diff --git a/.claude/skills/exec-local-compile/SKILL.md b/.claude/skills/exec-local-compile/SKILL.md index 6a2f3184895d..9a45283174fe 100644 --- a/.claude/skills/exec-local-compile/SKILL.md +++ b/.claude/skills/exec-local-compile/SKILL.md @@ -45,7 +45,7 @@ git checkout main && git pull Run the build command (**incremental by default** — omit `-c`/`--clean` unless explicitly requested or the incremental build fails): ```bash -./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache -a "" -f --nvtx +./scripts/build_wheel.py --use_ccache -a "" -f --nvtx ``` Replace `` with the target GPU architecture (see Architecture Reference below). If not specified by the user, auto-detect from `nvidia-smi`. @@ -66,8 +66,6 @@ python3 -c "import tensorrt_llm; print(tensorrt_llm.__version__)" | Flag | Description | |------|-------------| -| `--trt_root /usr/local/tensorrt` | TensorRT installation path (standard in NVIDIA containers) | -| `--benchmarks` | Build the C++ benchmarks | | `-a ""` | Target GPU architecture(s) | | `--nvtx` | Enable NVTX markers for profiling | | `--use_ccache` | Use ccache for faster recompilation | diff --git a/.claude/skills/exec-slurm-compile/SKILL.md b/.claude/skills/exec-slurm-compile/SKILL.md index 44c60ed374bd..8a8005d06d7a 100644 --- a/.claude/skills/exec-slurm-compile/SKILL.md +++ b/.claude/skills/exec-slurm-compile/SKILL.md @@ -204,8 +204,6 @@ A successful build ends with a message like `Successfully built tensorrt_llm` or | Flag | Description | |------|-------------| -| `--trt_root /usr/local/tensorrt` | TensorRT installation path (standard in NVIDIA containers) | -| `--benchmarks` | Build the C++ benchmarks | | `-a "100-real"` | Target architecture — `100` for Blackwell, `90` for Hopper, etc. | | `--nvtx` | Enable NVTX markers for profiling | | `--no-venv` | Skip virtual environment creation | @@ -228,7 +226,6 @@ Common architecture values: | `sbatch: error: invalid partition` | Verify partition name with `sinfo -s` | | `sbatch: error: invalid account` | Check available accounts with `sacctmgr show assoc user=$USER` | | Container image not found | Verify the `.sqsh` path exists and is readable | -| Build fails with missing TensorRT | Ensure `--trt_root` points to the correct path inside the container | | Build OOM (out of memory) | Reduce parallelism with `-j ` flag to `build_wheel.py` | | `srun: error: Unable to create step` | The node may lack enroot/pyxis — check with cluster admin | | Job stuck in `PD` state | Check `squeue -j -o %R` for the reason (e.g., resource limits, priority) | diff --git a/.claude/skills/exec-slurm-compile/scripts/compile.sh b/.claude/skills/exec-slurm-compile/scripts/compile.sh index 22b7882d55ff..4822dbceb677 100755 --- a/.claude/skills/exec-slurm-compile/scripts/compile.sh +++ b/.claude/skills/exec-slurm-compile/scripts/compile.sh @@ -19,7 +19,7 @@ # Usage: compile.sh [build_wheel_args...] # # Default build_wheel.py flags: -# --trt_root /usr/local/tensorrt --benchmarks -a "100-real" --nvtx --no-venv +# -a "100-real" --nvtx --no-venv # Any extra arguments after repo_dir are forwarded to build_wheel.py, # overriding the defaults above. @@ -36,8 +36,6 @@ if [[ $# -gt 0 ]]; then else echo "[compile.sh] Running default build command" python3 ./scripts/build_wheel.py \ - --trt_root /usr/local/tensorrt \ - --benchmarks \ -a "100-real" \ --nvtx fi diff --git a/.coderabbit.yaml b/.coderabbit.yaml index dc89ff60b2b8..2caf9db60128 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -22,7 +22,26 @@ reviews: auto_title_placeholder: '@coderabbitai title' auto_title_instructions: 'Format: "[] ". Category must be one of: fix, feat, doc, infra, style, refactor, perf, test, chore, revert. Enclose the category in square brackets. Title should be concise (<= 60 chars). Example: "[feat] Add logit_bias support".' commit_status: false - collapse_walkthrough: true + high_level_summary_instructions: | + Always produce two review sections in the summary: + + **Dev Engineer Review** + Review all changes for correctness and consistency, including: + - Code changes: correctness, performance, API consistency (CODING_GUIDELINES.md), error handling, regressions. + - Config files: valid values, no typos, consistency with related configs, no unintended scope changes. + - Test list files (test-db/, qa/, waives.txt): correct format, valid test paths, appropriate bug references, no duplicates. + + **QA Engineer Review** + Always include this section when any files under tests/ are touched. + For test-list-only changes (only tests/integration/test_lists/ files): + - List which test-db/ or qa/ files were modified and what entries were added or removed. + - Verdict: "needs follow-up" if CBTS coverage data is unavailable, otherwise "sufficient" or "insufficient". + For test-code changes (files outside tests/integration/test_lists/): + - List test functions added, modified, or removed. + - State whether each is covered in tests/integration/test_lists/ (test-db/ for CI, qa/ for manual QA). + - Verdict: sufficient, insufficient, or needs follow-up. + If no test files are touched, write "No test changes." + collapse_walkthrough: false assess_linked_issues: true related_issues: true related_prs: true @@ -31,15 +50,27 @@ reviews: poem: false review_status: false auto_review: - auto_incremental_review: false + auto_incremental_review: true drafts: false base_branches: ["main", "release/.+"] path_instructions: - path: "tests/**" instructions: | Act as a QA engineer reviewing test changes and coverage for TensorRT-LLM. - Keep feedback actionable: suggest concrete list file names and whether - coverage is sufficient, insufficient, or needs follow-up outside the PR. + Always produce a test coverage summary, even if no issues are found. + + If the change touches ONLY files under tests/integration/test_lists/ (no test-code changes): + - Report which test-db/ or qa/ list files were modified and what entries were added or removed. + - Do NOT require changed test functions for this path. + - Use verdict "needs follow-up" when cbts_touchmap.sqlite or a CBTS coverage report is unavailable + to confirm the impacted test scope; otherwise use "sufficient" or "insufficient". + + If the change includes test-code files (outside tests/integration/test_lists/), the summary must include: + 1. Which test functions were added, modified, or removed. + 2. Whether each changed test is listed in the appropriate test list files under + tests/integration/test_lists/ (test-db/ for CI, qa/ for manual QA). + 3. A coverage verdict: sufficient, insufficient, or needs follow-up. + Keep feedback actionable: reference concrete list file names when suggesting additions. - path: "tests/integration/test_lists/qa/**" instructions: | Files here are manually-triggered QA perf/regression lists, maintained diff --git a/.gitattributes b/.gitattributes index 5f797908576a..a8ca34bc5151 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,4 +16,5 @@ docs/source/blogs/media/tech_blog10_full_strategy_performance.png filter=lfs dif docs/source/blogs/media/tech_blog10_context_wait_performance.png filter=lfs diff=lfs merge=lfs -text cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/cubin/kernelMetaInfo_cubin.cpp filter=lfs diff=lfs merge=lfs -text cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/cubin/xqa_kernel_cubin.cpp filter=lfs diff=lfs merge=lfs -text -tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/*/*/*.so filter=lfs diff=lfs merge=lfs -text +docs/source/blogs/media/tech_blog26_deepseek_v4_hybrid_attention.png filter=lfs diff=lfs merge=lfs -text +docs/source/blogs/media/tech_blog26_deepseek_v4_mhc_moe.png filter=lfs diff=lfs merge=lfs -text diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 83e0275ad77c..1a01b72572c0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,7 +7,7 @@ # Infra @NVIDIA/trt-llm-infra-devs # Agent config @NVIDIA/trt-llm-agent-devs # Docs / Examples @NVIDIA/trt-llm-doc-owners -# QA @NVIDIA/trt-llm-qa +# QA @NVIDIA/trt-llm-qa / qa-perf / qa-function # Runtime @NVIDIA/trt-llm-runtime-devs # Kernels - Misc @NVIDIA/trt-llm-kernels-devs # Models @NVIDIA/trt-llm-models-devs @@ -31,14 +31,12 @@ /tensorrt_llm/commands/eval.py @NVIDIA/trt-llm-devs /tensorrt_llm/evaluate @NVIDIA/trt-llm-devs /tensorrt_llm/tools @NVIDIA/trt-llm-devs -/tests/integration/test_lists/test-db @NVIDIA/trt-llm-devs @NVIDIA/trt-llm-qa @NVIDIA/trt-llm-infra-devs -/tests/integration/test_lists/waives.txt @NVIDIA/trt-llm-devs @NVIDIA/trt-llm-qa @NVIDIA/trt-llm-infra-devs +/tests/integration/test_lists/test-db @NVIDIA/trt-llm-devs @NVIDIA/trt-llm-qa-function @NVIDIA/trt-llm-infra-devs +/tests/integration/test_lists/waives.txt @NVIDIA/trt-llm-devs @NVIDIA/trt-llm-qa-function @NVIDIA/trt-llm-infra-devs /tests/test_common @NVIDIA/trt-llm-devs /tests/unittest @NVIDIA/trt-llm-devs # ===== TensorRT backend (will be deprecated soon) — also on the trt-llm-devs fallback ===== -/cpp/include/tensorrt_llm/plugins @NVIDIA/trt-llm-devs -/cpp/tensorrt_llm/plugins @NVIDIA/trt-llm-devs /tensorrt_llm/builder.py @NVIDIA/trt-llm-devs /tensorrt_llm/commands/build.py @NVIDIA/trt-llm-devs /tensorrt_llm/commands/prune.py @NVIDIA/trt-llm-devs @@ -95,6 +93,8 @@ # ===== QA ===== /tests/integration/defs @NVIDIA/trt-llm-devs @NVIDIA/trt-llm-qa @NVIDIA/trt-llm-infra-devs /tests/integration/test_lists/qa @NVIDIA/trt-llm-qa +/tests/integration/test_lists/qa/llm_perf_* @NVIDIA/trt-llm-qa-perf +/tests/integration/test_lists/qa/llm_function_* @NVIDIA/trt-llm-qa-function # ===== RUNTIME ===== /cpp/include/tensorrt_llm/batch_manager @NVIDIA/trt-llm-runtime-devs @@ -105,7 +105,6 @@ /cpp/tensorrt_llm/batch_manager @NVIDIA/trt-llm-runtime-devs /cpp/tensorrt_llm/common @NVIDIA/trt-llm-runtime-devs /cpp/tensorrt_llm/executor @NVIDIA/trt-llm-runtime-devs -/cpp/tensorrt_llm/executor_worker @NVIDIA/trt-llm-runtime-devs /cpp/tensorrt_llm/layers @NVIDIA/trt-llm-runtime-devs /cpp/tensorrt_llm/nanobind @NVIDIA/trt-llm-runtime-devs /cpp/tensorrt_llm/runtime @NVIDIA/trt-llm-runtime-devs @@ -169,13 +168,12 @@ /examples/llm-api/quickstart_multimodal.py @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-doc-owners /examples/models @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-doc-owners /examples/serve/*multimodal* @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-doc-owners -/scripts/build_cpp_examples.py @NVIDIA/trt-llm-models-devs /scripts/generate_config_database_tests.py @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-doc-owners /scripts/generate_config_table.py @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-doc-owners /tensorrt_llm/_torch/models @NVIDIA/trt-llm-models-devs /tensorrt_llm/_torch/modules/mamba @NVIDIA/trt-llm-models-devs /tensorrt_llm/quantization @NVIDIA/trt-llm-models-devs -/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-qa +/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-qa-function /tests/unittest/_torch/modeling @NVIDIA/trt-llm-models-devs /tests/unittest/_torch/models @NVIDIA/trt-llm-models-devs /tests/unittest/_torch/modules/mamba @NVIDIA/trt-llm-models-devs @@ -212,6 +210,7 @@ /cpp/tensorrt_llm/batch_manager/blockKey* @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tensorrt_llm/batch_manager/evictionPolicy* @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tensorrt_llm/batch_manager/kvCache* @NVIDIA/trt-llm-kv-cache-manager-devs +/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2 @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager* @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tests/unit_tests/batch_manager/blockKey* @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tests/unit_tests/batch_manager/evictionPolicy* @NVIDIA/trt-llm-kv-cache-manager-devs @@ -246,9 +245,9 @@ /tensorrt_llm/disaggregated_params.py @NVIDIA/trt-llm-disagg-devs /tensorrt_llm/serve/openai_disagg_server.py @NVIDIA/trt-llm-disagg-devs # Disagg tests: co-own with the owning team so disagg-devs review disagg-test changes. -/tests/integration/defs/accuracy/*disagg* @NVIDIA/trt-llm-disagg-devs @NVIDIA/trt-llm-qa -/tests/integration/defs/disaggregated @NVIDIA/trt-llm-disagg-devs @NVIDIA/trt-llm-qa -/tests/integration/defs/stress_test/disagg_cancel @NVIDIA/trt-llm-disagg-devs @NVIDIA/trt-llm-qa +/tests/integration/defs/accuracy/*disagg* @NVIDIA/trt-llm-disagg-devs @NVIDIA/trt-llm-qa-serving +/tests/integration/defs/disaggregated @NVIDIA/trt-llm-disagg-devs @NVIDIA/trt-llm-qa-serving +/tests/integration/defs/stress_test/disagg_cancel @NVIDIA/trt-llm-disagg-devs @NVIDIA/trt-llm-qa-serving /tests/scripts/perf-sanity/disaggregated @NVIDIA/trt-llm-perf-devs @NVIDIA/trt-llm-disagg-devs /tests/scripts/perf/disaggregated @NVIDIA/trt-llm-perf-devs @NVIDIA/trt-llm-disagg-devs /tests/unittest/_torch/executor/*disagg* @NVIDIA/trt-llm-runtime-devs @NVIDIA/trt-llm-disagg-devs @@ -404,7 +403,7 @@ /scripts/check_auto_deploy_imports.py @NVIDIA/trt-llm-torch-autodeploy-devs /scripts/check_model_registry.py @NVIDIA/trt-llm-torch-autodeploy-devs /tensorrt_llm/_torch/auto_deploy @NVIDIA/trt-llm-torch-autodeploy-devs -/tests/integration/defs/accuracy/test_llm_api_autodeploy.py @NVIDIA/trt-llm-torch-autodeploy-devs @NVIDIA/trt-llm-qa +/tests/integration/defs/accuracy/test_llm_api_autodeploy.py @NVIDIA/trt-llm-torch-autodeploy-devs @NVIDIA/trt-llm-qa-function /tests/unittest/_torch/auto_deploy @NVIDIA/trt-llm-torch-autodeploy-devs /tests/unittest/auto_deploy @NVIDIA/trt-llm-torch-autodeploy-devs /tests/integration/defs/accuracy/test_llm_api_autodeploy.py @NVIDIA/trt-llm-torch-autodeploy-devs @NVIDIA/trt-llm-qa-function @@ -441,6 +440,7 @@ /cpp/tensorrt_llm/batch_manager/allocateKvCache.cpp @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tests/unit_tests/batch_manager/kvCacheUtilsTest.cpp @NVIDIA/trt-llm-kv-cache-manager-devs +/tensorrt_llm/_torch/attention_backend/sparse/*/cache_manager.py @NVIDIA/trt-llm-kv-cache-manager-devs /tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py @NVIDIA/trt-llm-kv-cache-manager-devs /tensorrt_llm/_torch/pyexecutor/resource_manager.py @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.h @NVIDIA/trt-llm-kv-cache-manager-devs diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml index d67685f216f2..91ca8ba0c6b0 100644 --- a/.github/workflows/blossom-ci.yml +++ b/.github/workflows/blossom-ci.yml @@ -48,7 +48,9 @@ jobs: "achartier", "ajrasane", "alec-flowers", + "AlessioNetti", "alexmsettle", + "allisonlim-nv", "ameynaik-hub", "amirkl94", "amitz-nv", @@ -94,6 +96,7 @@ jobs: "chzblych", "cjluo-nv", "crazydemo", + "daichu-nv", "DanBlanaru", "danielafrimi", "davidclark-nv", @@ -187,6 +190,7 @@ jobs: "JunyiXu-nv", "JyChang012", "kaiyux", + "Kambili", "kanghui0204", "karljang", "karthikvetrivel", @@ -228,6 +232,7 @@ jobs: "MatthiasKohl", "mayani-nv", "meenchen", + "MengmSun", "mgluhovskoi", "mikeiovine", "milesial", @@ -276,6 +281,7 @@ jobs: "PerkzZheng", "poweiw", "pranav-nvidia", + "pst2154", "qiangxu1996", "qiaoxj07", "QiJune", @@ -309,6 +315,7 @@ jobs: "shuyixiong", "shyeh25", "SimengLiu-nv", + "siyidNV", "sklevtsov-nvidia", "StanleySun639", "stnie", @@ -410,6 +417,7 @@ jobs: "zhangcl", "ZhanruiSunCh", "zhaoyangwang-nvidia", + "zhaoyuanh-nvidia", "zhengd-nv", "zhenhuaw-me", "zheyuf", diff --git a/.github/workflows/bot-command.yml b/.github/workflows/bot-command.yml index 0a112dbf5435..30b5de652f55 100644 --- a/.github/workflows/bot-command.yml +++ b/.github/workflows/bot-command.yml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -52,14 +52,14 @@ jobs: "`--disable-reuse-test ` *(OPTIONAL)* : Explicitly prevent the pipeline from reusing build artifacts and skipping successful test stages from a previous pipeline. Ensure that all builds and tests are run regardless of previous successes.\n\n" + "`--disable-fail-fast ` *(OPTIONAL)* : Disable fail fast on build/tests/infra failures.\n\n" + "`--skip-test ` *(OPTIONAL)* : Skip all test stages, but still run build stages, package stages and sanity check stages. Note: Does **NOT** update GitHub check status.\n\n" + - "`--stage-list \"A10-PyTorch-1, xxx\"` *(OPTIONAL)* : Only run the specified test stages. Supports wildcard `*` for pattern matching (e.g., `\"*PerfSanity*\"` matches all stages containing PerfSanity). Examples: \"A10-PyTorch-1, xxx\", \"*PerfSanity*\". Note: Does **NOT** update GitHub check status.\n\n" + + "`--stage-list \"A10-PyTorch-1, xxx\"` *(OPTIONAL)* : Only run the specified test stages. Supports wildcard `*` for pattern matching (e.g., `\"*PerfSanity*\"` matches all stages containing PerfSanity). Examples: \"A10-PyTorch-1, xxx\", \"*PerfSanity*\". The patterns `\"*\"`, `\"*Post-Merge*\"`, and `\"*PerfSanity*\"`, including equivalent escaped or repeated-star forms and their use in comma-separated lists, require the `ci: post-merge approved` PR label. Note: Does **NOT** update GitHub check status.\n\n" + "`--gpu-type \"A30, H100_PCIe\"` *(OPTIONAL)* : Only run the test stages on the specified GPU types. Examples: \"A30, H100_PCIe\". Note: Does **NOT** update GitHub check status.\n\n" + "`--test-backend \"pytorch, cpp\"` *(OPTIONAL)* : Skip test stages which don't match the specified backends. Only support [pytorch, cpp, tensorrt, triton]. Examples: \"pytorch, cpp\" (does not run test stages with tensorrt or triton backend). Note: Does **NOT** update GitHub pipeline status.\n\n" + "`--only-multi-gpu-test ` *(OPTIONAL)* : Only run the multi-GPU tests. Note: Does **NOT** update GitHub check status.\n\n" + "`--disable-multi-gpu-test ` *(OPTIONAL)* : Disable the multi-GPU tests. Note: Does **NOT** update GitHub check status.\n\n" + "`--add-multi-gpu-test ` *(OPTIONAL)* : Force run the multi-GPU tests in addition to running L0 pre-merge pipeline.\n\n" + - "`--post-merge ` *(OPTIONAL)* : Run the L0 post-merge pipeline instead of the ordinary L0 pre-merge pipeline.\n\n" + - "`--extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\"` *(OPTIONAL)* : Run the ordinary L0 pre-merge pipeline and specified test stages. Supports wildcard `*` for pattern matching. Examples: --extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\", --extra-stage \"*Post-Merge*\".\n\n" + + "`--post-merge ` *(OPTIONAL)* : Run the L0 post-merge pipeline instead of the ordinary L0 pre-merge pipeline. Requires the `ci: post-merge approved` PR label applied by an active member of `NVIDIA/trt-llm-ci-approvers`. The approval label remains in place when new commits are pushed.\n\n" + + "`--extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\"` *(OPTIONAL)* : Run the ordinary L0 pre-merge pipeline and specified test stages. Supports wildcard `*` for pattern matching. Examples: --extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\", --extra-stage \"*Post-Merge*\". The patterns `\"*\"`, `\"*Post-Merge*\"`, and `\"*PerfSanity*\"`, including equivalent escaped or repeated-star forms and their use in comma-separated lists, require the `ci: post-merge approved` PR label.\n\n" + "`--detailed-log ` *(OPTIONAL)* : Enable flushing out all logs to the Jenkins console. This will significantly increase the log volume and may slow down the job.\n\n" + "`--debug ` *(OPTIONAL)* : **Experimental feature**. Enable access to the CI container for debugging purpose. Note: Specify exactly one stage in the `stage-list` parameter to access the appropriate container environment. Note: Does **NOT** update GitHub check status.\n\n" + "`--high-priority ` *(OPTIONAL)* : Run the pipeline with high priority. This option is restricted to authorized users only and will route the job to a high-priority queue.\n\n" + diff --git a/.github/workflows/lfs-sync.yml b/.github/workflows/lfs-sync.yml index bccdedc858e1..3143afedbd90 100644 --- a/.github/workflows/lfs-sync.yml +++ b/.github/workflows/lfs-sync.yml @@ -44,9 +44,12 @@ jobs: steps: - name: Checkout merge commit without LFS smudge - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.merge_commit_sha }} + # This job only handles merged PRs and does not execute code from the + # checked-out tree, so explicitly allow the required merge checkout. + allow-unsafe-pr-checkout: true lfs: false fetch-depth: 2 token: ${{ secrets.GITHUB_TOKEN }} @@ -200,7 +203,7 @@ jobs: - name: Comment on PR — already in storage if: steps.detect-lfs.outputs.lfs_status == 'present' - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const files = require('fs').readFileSync('/tmp/lfs_files.txt', 'utf8').trim().split('\n'); @@ -221,7 +224,7 @@ jobs: - name: Comment on PR — sync succeeded if: steps.detect-lfs.outputs.lfs_status == 'pending' && steps.verify.outcome == 'success' - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const files = require('fs').readFileSync('/tmp/lfs_files.txt', 'utf8').trim().split('\n'); @@ -242,7 +245,7 @@ jobs: - name: Comment on PR — sync failed if: always() && steps.detect-lfs.outputs.lfs_status == 'pending' && (steps.fetch-lfs.outcome != 'success' || steps.push-lfs.outcome != 'success' || steps.verify.outcome != 'success') - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const forkRepo = context.payload.pull_request.head.repo.full_name; diff --git a/.github/workflows/post-merge-approval.yml b/.github/workflows/post-merge-approval.yml new file mode 100644 index 000000000000..66b3490843dd --- /dev/null +++ b/.github/workflows/post-merge-approval.yml @@ -0,0 +1,222 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Guard Post-Merge Approval Label + +on: + # Intentional: this runs static default-branch API logic only. It never checks + # out or executes PR code; it uses GitHub APIs to validate membership and + # manage this PR's approval label/comment. + pull_request_target: + types: [labeled] + +permissions: + contents: read + pull-requests: write + +jobs: + guard-post-merge-approval: + concurrency: + group: post-merge-approval-${{ github.event.pull_request.number }} + cancel-in-progress: false + # Keep a started job eligible to reach fail-closed cleanup after cancellation. + if: >- + always() && + github.repository == 'NVIDIA/TensorRT-LLM' && + github.event.action == 'labeled' && + github.event.label.name == 'ci: post-merge approved' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Validate post-merge approver + id: validate + if: github.event.action == 'labeled' + uses: actions/github-script@v8 + with: + github-token: ${{ secrets.TRTLLM_AGENT_SHARED_TOKEN }} + result-encoding: string + script: | + const approvalLabel = 'ci: post-merge approved'; + const owner = context.repo.owner; + const repo = context.repo.repo; + const issueNumber = context.payload.pull_request.number; + let actor = context.payload.sender?.login || context.actor; + let labelEventId = ''; + let timelineVerified = false; + + try { + const events = await github.paginate( + github.rest.issues.listEventsForTimeline, + { owner, repo, issue_number: issueNumber, per_page: 100 } + ); + const latestApprovalEvent = events + .filter( + (event) => + event.event === 'labeled' && + event.label?.name?.toLowerCase() === approvalLabel.toLowerCase() + ) + .at(-1); + if (latestApprovalEvent) { + actor = latestApprovalEvent.actor?.login || actor; + labelEventId = String(latestApprovalEvent.id); + timelineVerified = true; + } else { + core.warning('Could not identify the latest post-merge approval event.'); + } + } catch (error) { + core.warning( + 'Could not read the latest post-merge approval event: ' + error.message + ); + } + + core.setOutput('validated_actor', actor); + core.setOutput('label_event_id', labelEventId); + if (!timelineVerified) { + return 'false'; + } + try { + const response = await github.request( + 'GET /orgs/{org}/teams/{team_slug}/memberships/{username}', + { + org: 'NVIDIA', + team_slug: 'trt-llm-ci-approvers', + username: actor, + } + ); + const authorized = response.data.state === 'active'; + console.log( + actor + ' active membership in NVIDIA/trt-llm-ci-approvers: ' + authorized + ); + return authorized ? 'true' : 'false'; + } catch (error) { + if (error.status === 404) { + console.log( + actor + ' is not an active member of NVIDIA/trt-llm-ci-approvers.' + ); + } else { + core.warning( + 'Could not verify post-merge approver ' + actor + ': ' + error.message + ); + } + return 'false'; + } + + - name: Clear unauthorized post-merge approval + # Let a started job attempt fail-closed cleanup after normal cancellation. + if: always() && steps.validate.outputs.result != 'true' + uses: actions/github-script@v8 + env: + VALIDATED_ACTOR: ${{ steps.validate.outputs.validated_actor }} + VALIDATED_LABEL_EVENT_ID: ${{ steps.validate.outputs.label_event_id }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const approvalLabel = 'ci: post-merge approved'; + const actor = + process.env.VALIDATED_ACTOR || + context.payload.sender?.login || + context.actor; + const validatedEventId = process.env.VALIDATED_LABEL_EVENT_ID || ''; + const owner = context.repo.owner; + const repo = context.repo.repo; + const issueNumber = context.payload.pull_request.number; + + try { + const pullRequest = await github.rest.pulls.get({ + owner, + repo, + pull_number: issueNumber, + }); + const labelIsPresent = pullRequest.data.labels.some( + (label) => + label.name?.toLowerCase() === approvalLabel.toLowerCase() + ); + if (!labelIsPresent) { + console.log('Post-merge approval label is already absent; no cleanup needed.'); + return; + } + } catch (error) { + core.warning( + 'Could not read the current post-merge approval label state; continuing validation: ' + + error.message + ); + } + + try { + const events = await github.paginate( + github.rest.issues.listEventsForTimeline, + { owner, repo, issue_number: issueNumber, per_page: 100 } + ); + const latestApprovalEvent = events + .filter( + (event) => + event.event === 'labeled' && + event.label?.name?.toLowerCase() === approvalLabel.toLowerCase() + ) + .at(-1); + const latestEventId = latestApprovalEvent + ? String(latestApprovalEvent.id) + : ''; + const latestActor = latestApprovalEvent?.actor?.login || ''; + if (!latestApprovalEvent) { + core.warning( + 'Could not identify the latest post-merge approval event; removing the label to fail closed.' + ); + } else if (!latestActor) { + core.warning( + 'Could not identify the latest post-merge approval actor; removing the label to fail closed.' + ); + } else if (!validatedEventId) { + core.warning( + 'The validation run did not bind an approval event; removing the label to fail closed.' + ); + } else if ( + latestEventId !== validatedEventId || + latestActor !== actor + ) { + console.log( + 'A newer post-merge approval event was found; leaving it for its own validation run.' + ); + return; + } + } catch (error) { + core.warning( + 'Could not re-check the latest post-merge approval event; removing the label to fail closed: ' + + error.message + ); + } + + try { + await github.rest.issues.removeLabel({ + owner, + repo, + issue_number: issueNumber, + name: approvalLabel, + }); + } catch (error) { + if (error.status !== 404) { + throw error; + } + } + + await github.rest.issues.createComment({ + owner, + repo, + issue_number: issueNumber, + body: + 'Removed the "' + approvalLabel + '" label because @' + actor + + ' could not be verified as an active member of ' + + 'NVIDIA/trt-llm-ci-approvers. Ask a member of that team to apply it.', + }); diff --git a/.gitignore b/.gitignore index 47e39a1b2d71..9a9332335ca5 100644 --- a/.gitignore +++ b/.gitignore @@ -57,7 +57,6 @@ tensorrt_llm/flash_mla/ tensorrt_llm/flash_mla_cpp_tllm.*.so tensorrt_llm/flash_mla_cpp_tllm.pyi tensorrt_llm/runtime/kv_cache_manager_v2/**/*.so -!tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/**/*.so **/*__mypyc*.so tensorrt_llm/scripts *docs/cpp_docs* diff --git a/.gitmodules b/.gitmodules index e69de29bb2d1..627760b34da2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "3rdparty/MSA"] + path = 3rdparty/MSA + url = https://gitlab.com/nvidia/tensorrt-llm/oss-components/msa.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77f130db4363..5769139df446 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,14 +9,6 @@ common-files: &common_files | .devcontainer/make_env.py | .github/scripts/label_community_user.py | .github/scripts/pr_checklist_check.py | - benchmarks/cpp/__init__.py | - benchmarks/cpp/prepare_dataset.py | - benchmarks/cpp/utils/__init__.py | - benchmarks/cpp/utils/convert_nemo_dataset.py | - benchmarks/cpp/utils/generate_rand_loras.py | - benchmarks/cpp/utils/prepare_real_data.py | - benchmarks/cpp/utils/prepare_synthetic_data.py | - benchmarks/cpp/utils/utils.py | cpp/conanfile.py | cpp/kernels/fmha_v2/conftest.py | cpp/kernels/fmha_v2/fmha_test.py | @@ -41,58 +33,17 @@ common-files: &common_files | cpp/tensorrt_llm/deep_ep/strip_nvshmem_helper.py | cpp/tensorrt_llm/kernels/cutlass_kernels/python/generate_kernels.py | cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/copy_cu.py | - cpp/tests/resources/scripts/build_chatglm_engines.py | - cpp/tests/resources/scripts/build_eagle_engines.py | - cpp/tests/resources/scripts/build_enc_dec_engines.py | - cpp/tests/resources/scripts/build_engines_utils.py | - cpp/tests/resources/scripts/build_gpt_engines.py | - cpp/tests/resources/scripts/build_gptj_engines.py | - cpp/tests/resources/scripts/build_llama_engines.py | - cpp/tests/resources/scripts/build_mamba_engines.py | - cpp/tests/resources/scripts/build_medusa_engines.py | - cpp/tests/resources/scripts/build_recurrentgemma_engines.py | - cpp/tests/resources/scripts/build_redrafter_engines.py | - cpp/tests/resources/scripts/generate_expected_chatglm_output.py | - cpp/tests/resources/scripts/generate_expected_eagle_output.py | - cpp/tests/resources/scripts/generate_expected_enc_dec_output.py | - cpp/tests/resources/scripts/generate_expected_gpt_output.py | - cpp/tests/resources/scripts/generate_expected_gptj_output.py | - cpp/tests/resources/scripts/generate_expected_llama_output.py | - cpp/tests/resources/scripts/generate_expected_mamba_output.py | - cpp/tests/resources/scripts/generate_expected_medusa_output.py | - cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py | - cpp/tests/resources/scripts/generate_expected_redrafter_output.py | - cpp/tests/resources/scripts/generate_hf_gpt_output.py | cpp/tests/resources/scripts/generate_test_lora_weights.py | - cpp/tests/resources/scripts/io_converter.py | docs/source/conf.py | docs/source/helper.py | examples/apps/chat.py | examples/apps/fastapi_server.py | - examples/bindings/executor/example_advanced.py | - examples/bindings/executor/example_basic.py | - examples/bindings/executor/example_debug.py | - examples/bindings/executor/example_logits_processor.py | examples/disaggregated/clients/disagg_client.py | examples/disaggregated/slurm/benchmark/submit.py | - examples/dora/normalize_weights.py | - examples/eagle/convert_checkpoint.py | - examples/eval_long_context.py | - examples/generate_checkpoint_config.py | - examples/generate_xgrammar_tokenizer_info.py | - examples/hf_lora_convert.py | examples/infinitebench/args.py | examples/infinitebench/compute_scores.py | examples/infinitebench/construct_synthetic_dataset.py | examples/infinitebench/eval_utils.py | - examples/llm-api/_tensorrt_engine/llm_eagle2_decoding.py | - examples/llm-api/_tensorrt_engine/llm_eagle_decoding.py | - examples/llm-api/_tensorrt_engine/llm_inference_customize.py | - examples/llm-api/_tensorrt_engine/llm_inference_kv_events.py | - examples/llm-api/_tensorrt_engine/llm_lookahead_decoding.py | - examples/llm-api/_tensorrt_engine/llm_medusa_decoding.py | - examples/llm-api/_tensorrt_engine/llm_quantization.py | - examples/llm-api/_tensorrt_engine/quickstart_example.py | examples/llm-api/llm_guided_decoding.py | examples/llm-api/llm_inference.py | examples/llm-api/llm_inference_async.py | @@ -112,122 +63,17 @@ common-files: &common_files | examples/llm-api/quickstart_example.py | examples/llm-api/quickstart_multimodal.py | examples/llm-api/star_attention.py | - examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py | examples/longbench/eval_longbench_v1.py | - examples/medusa/convert_checkpoint.py | - examples/mmlu.py | - examples/models/contrib/baichuan/convert_checkpoint.py | - examples/models/contrib/bloom/convert_checkpoint.py | - examples/models/contrib/chatglm-6b/tokenization_chatglm.py | - examples/models/contrib/chatglm2-6b/tokenization_chatglm.py | - examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py | - examples/models/contrib/cogvlm/convert_checkpoint.py | - examples/models/contrib/dbrx/convert_checkpoint.py | - examples/models/contrib/deepseek_v1/__init__.py | - examples/models/contrib/deepseek_v1/convert_checkpoint.py | - examples/models/contrib/deepseek_v2/convert_checkpoint.py | - examples/models/contrib/dit/convert_checkpoint.py | - examples/models/contrib/dit/diffusion.py | - examples/models/contrib/dit/sample.py | - examples/models/contrib/dit/utils_modelopt.py | - examples/models/contrib/dit/vae_decoder_trt.py | - examples/models/contrib/falcon/convert_checkpoint.py | - examples/models/contrib/gptj/convert_checkpoint.py | - examples/models/contrib/gptneox/convert_checkpoint.py | - examples/models/contrib/grok/convert_checkpoint.py | - examples/models/contrib/mmdit/convert_checkpoint.py | - examples/models/contrib/mmdit/sample.py | - examples/models/contrib/mpt/convert_checkpoint.py | - examples/models/contrib/opt/convert_checkpoint.py | - examples/models/contrib/sdxl/build_sdxl_unet.py | - examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py | - examples/models/contrib/sdxl/run_sdxl.py | - examples/models/contrib/stdit/aspect.py | - examples/models/contrib/stdit/convert_checkpoint.py | - examples/models/contrib/stdit/pipeline_tllm.py | - examples/models/contrib/stdit/sample.py | - examples/models/contrib/stdit/scheduler.py | - examples/models/contrib/stdit/text_encoder.py | - examples/models/contrib/stdit/utils.py | - examples/models/contrib/stdit/vae.py | - examples/models/contrib/stdit/video_transforms.py | - examples/models/core/bert/__init__.py | - examples/models/core/bert/convert_checkpoint.py | - examples/models/core/bert/run.py | - examples/models/core/bert/utils.py | - examples/models/core/commandr/convert_checkpoint.py | - examples/models/core/enc_dec/__init__.py | - examples/models/core/enc_dec/convert_checkpoint.py | - examples/models/core/enc_dec/helper.py | - examples/models/core/enc_dec/run.py | - examples/models/core/gemma/convert_checkpoint.py | - examples/models/core/glm-4-9b/convert_checkpoint.py | - examples/models/core/glm-4-9b/tokenization_chatglm.py | - examples/models/core/gpt/convert_checkpoint.py | - examples/models/core/gpt/merge_ptuning_tables.py | - examples/models/core/gpt/nemo_lora_convert.py | - examples/models/core/gpt/nemo_prompt_convert.py | - examples/models/core/gpt/run_hf.py | examples/models/core/gpt_oss/openai_chat_client_function_calling.py | - examples/models/core/internlm2/convert_checkpoint.py | examples/models/core/kimi_k2/kimi_k2_tool_calling_example.py | - examples/models/core/llama/convert_checkpoint.py | - examples/models/core/llama/summarize_long.py | - examples/models/core/mamba/convert_checkpoint.py | - examples/models/core/mllama/convert_checkpoint.py | - examples/models/core/multimodal/__init__.py | - examples/models/core/multimodal/build_multimodal_engine.py | - examples/models/core/multimodal/eval.py | - examples/models/core/multimodal/run.py | - examples/models/core/multimodal/utils.py | - examples/models/core/nemotron_nas/calibration_utils.py | - examples/models/core/nemotron_nas/convert_checkpoint.py | - examples/models/core/phi/convert_checkpoint.py | - examples/models/core/qwen/convert_checkpoint.py | - examples/models/core/qwen2audio/run.py | - examples/models/core/qwen2audio/run_chat.py | - examples/models/core/qwen2audio/utils.py | - examples/models/core/qwenvl/run.py | - examples/models/core/qwenvl/run_chat.py | - examples/models/core/qwenvl/show_pic.py | - examples/models/core/qwenvl/vit_onnx_trt.py | - examples/models/core/recurrentgemma/convert_checkpoint.py | - examples/models/core/vit/convert_checkpoint.py | - examples/models/core/whisper/convert_checkpoint.py | - examples/models/core/whisper/distil_whisper/convert_from_distil_whisper.py | - examples/models/core/whisper/run.py | - examples/models/core/whisper/tokenizer.py | - examples/models/core/whisper/whisper_utils.py | - examples/ngram/run_dtm_ngram.py | - examples/openai_triton/manual_plugin/build.py | - examples/openai_triton/manual_plugin/fmha_triton.py | - examples/openai_triton/manual_plugin/plugin.py | - examples/openai_triton/manual_plugin/run.py | - examples/openai_triton/plugin_autogen/build_engine.py | - examples/openai_triton/plugin_autogen/kernel_config.py | - examples/openai_triton/plugin_autogen/run_engine.py | - examples/python_plugin/build_lookup.py | - examples/python_plugin/plugin_lib/__init__.py | - examples/python_plugin/plugin_lib/lookup_kernel.py | - examples/python_plugin/plugin_lib/lookup_plugin.py | - examples/python_plugin/run_lookup.py | - examples/quantization/quantize.py | examples/quantization/quantize_mixed_precision_moe.py | examples/ray_orchestrator/llm_inference_async_ray.py | examples/ray_orchestrator/llm_inference_distributed_ray.py | - examples/redrafter/convert_checkpoint.py | - examples/run.py | examples/scaffolding/contrib/AsyncGeneration/stream_generation_controller.py | examples/scaffolding/contrib/DeepConf/run_generation.py | examples/scaffolding/contrib/Dynasor/scaffolding_dynasor_run.py | examples/scaffolding/contrib/TreeInference/run_mcts_example.py | examples/scaffolding/contrib/TreeInference/run_tot_example.py | - examples/scaffolding/contrib/mcp/e2b/e2bserver.py | - examples/scaffolding/contrib/mcp/e2b/main.py | - examples/scaffolding/contrib/mcp/mcptest.py | - examples/scaffolding/contrib/mcp/weather/weather.py | - examples/scaffolding/contrib/mcp/websearch/main.py | - examples/scaffolding/contrib/mcp/websearch/websearch.py | examples/scaffolding/run_basic_generation.py | examples/scaffolding/run_best_of_n_with_reward.py | examples/scaffolding/run_majority_vote_aime24.py | @@ -237,8 +83,6 @@ common-files: &common_files | examples/serve/openai_completion_client.py | examples/serve/openai_completion_client_for_lora.py | examples/serve/openai_completion_client_json_schema.py | - examples/summarize.py | - examples/utils.py | examples/wide_ep/ep_load_balancer/generate_eplb_config.py | examples/wide_ep/ep_load_balancer/report_load_statistics.py | examples/wide_ep/ep_load_balancer/utils.py | @@ -246,12 +90,10 @@ common-files: &common_files | jenkins/scripts/mergeWaiveList.py | jenkins/scripts/open_search_db.py | jenkins/scripts/test_rerun.py | - scripts/build_cpp_examples.py | scripts/build_wheel.py | scripts/check_test_list.py | scripts/dco_check.py | scripts/format_test_list.py | - scripts/generate_duration.py | scripts/generate_lock_file.py | scripts/get_wheel_from_package.py | scripts/git_replace.py | @@ -262,7 +104,6 @@ common-files: &common_files | setup.py | tensorrt_llm/__init__.py | tensorrt_llm/_ray_utils.py | - tensorrt_llm/_tensorrt_engine/__init__.py | tensorrt_llm/_torch/__init__.py | tensorrt_llm/_torch/attention_backend/__init__.py | tensorrt_llm/_torch/attention_backend/flashinfer.py | @@ -466,7 +307,6 @@ common-files: &common_files | tensorrt_llm/_torch/pyexecutor/guided_decoder.py | tensorrt_llm/_torch/pyexecutor/handle_additional_outputs.py | tensorrt_llm/_torch/pyexecutor/handle_logits.py | - tensorrt_llm/_torch/pyexecutor/kv_cache_connector.py | tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py | tensorrt_llm/_torch/pyexecutor/layerwise_nvtx_marker.py | tensorrt_llm/_torch/pyexecutor/llm_request.py | @@ -504,11 +344,6 @@ common-files: &common_files | tensorrt_llm/bench/benchmark/utils/asynchronous.py | tensorrt_llm/bench/benchmark/utils/general.py | tensorrt_llm/bench/benchmark/utils/processes.py | - tensorrt_llm/bench/build/__init__.py | - tensorrt_llm/bench/build/build.py | - tensorrt_llm/bench/build/dataclasses.py | - tensorrt_llm/bench/build/tuning.py | - tensorrt_llm/bench/build/utils.py | tensorrt_llm/bench/dataclasses/__init__.py | tensorrt_llm/bench/dataclasses/configuration.py | tensorrt_llm/bench/dataclasses/engine.py | @@ -518,13 +353,9 @@ common-files: &common_files | tensorrt_llm/bench/dataclasses/statistics.py | tensorrt_llm/bench/utils/__init__.py | tensorrt_llm/bench/utils/data.py | - tensorrt_llm/builder.py | tensorrt_llm/commands/__init__.py | tensorrt_llm/commands/bench.py | - tensorrt_llm/commands/build.py | tensorrt_llm/commands/eval.py | - tensorrt_llm/commands/prune.py | - tensorrt_llm/commands/refit.py | tensorrt_llm/commands/serve.py | tensorrt_llm/evaluate/__init__.py | tensorrt_llm/evaluate/cnn_dailymail.py | @@ -560,23 +391,7 @@ common-files: &common_files | tensorrt_llm/inputs/multimodal.py | tensorrt_llm/inputs/registry.py | tensorrt_llm/inputs/utils.py | - tensorrt_llm/layers/__init__.py | - tensorrt_llm/layers/activation.py | - tensorrt_llm/layers/attention.py | - tensorrt_llm/layers/cast.py | - tensorrt_llm/layers/conv.py | - tensorrt_llm/layers/embedding.py | - tensorrt_llm/layers/language_adapter.py | - tensorrt_llm/layers/linear.py | - tensorrt_llm/layers/lora.py | - tensorrt_llm/layers/mlp.py | - tensorrt_llm/layers/moe.py | - tensorrt_llm/layers/normalization.py | - tensorrt_llm/layers/pooling.py | - tensorrt_llm/layers/recurrent.py | - tensorrt_llm/layers/ssm.py | tensorrt_llm/llmapi/__init__.py | - tensorrt_llm/llmapi/build_cache.py | tensorrt_llm/llmapi/disagg_utils.py | tensorrt_llm/llmapi/kv_cache_type.py | tensorrt_llm/llmapi/llm.py | @@ -599,179 +414,17 @@ common-files: &common_files | tensorrt_llm/metrics/enums.py | tensorrt_llm/models/__init__.py | tensorrt_llm/models/automodel.py | - tensorrt_llm/models/baichuan/__init__.py | - tensorrt_llm/models/baichuan/config.py | - tensorrt_llm/models/baichuan/convert.py | - tensorrt_llm/models/baichuan/model.py | - tensorrt_llm/models/bert/__init__.py | - tensorrt_llm/models/bert/config.py | - tensorrt_llm/models/bert/convert.py | - tensorrt_llm/models/bert/model.py | - tensorrt_llm/models/bloom/__init__.py | - tensorrt_llm/models/bloom/model.py | - tensorrt_llm/models/chatglm/__init__.py | - tensorrt_llm/models/chatglm/config.py | - tensorrt_llm/models/chatglm/convert.py | - tensorrt_llm/models/chatglm/model.py | - tensorrt_llm/models/clip/__init__.py | - tensorrt_llm/models/clip/model.py | - tensorrt_llm/models/cogvlm/__init__.py | - tensorrt_llm/models/cogvlm/config.py | - tensorrt_llm/models/cogvlm/convert.py | - tensorrt_llm/models/cogvlm/model.py | - tensorrt_llm/models/commandr/__init__.py | - tensorrt_llm/models/commandr/config.py | - tensorrt_llm/models/commandr/model.py | tensorrt_llm/models/convert_utils.py | - tensorrt_llm/models/dbrx/__init__.py | - tensorrt_llm/models/dbrx/config.py | - tensorrt_llm/models/dbrx/model.py | - tensorrt_llm/models/deepseek_v1/__init__.py | - tensorrt_llm/models/deepseek_v1/config.py | - tensorrt_llm/models/deepseek_v1/convert.py | - tensorrt_llm/models/deepseek_v1/model.py | - tensorrt_llm/models/deepseek_v2/__init__.py | - tensorrt_llm/models/deepseek_v2/config.py | - tensorrt_llm/models/deepseek_v2/convert.py | - tensorrt_llm/models/deepseek_v2/model.py | - tensorrt_llm/models/dit/__init__.py | - tensorrt_llm/models/dit/model.py | - tensorrt_llm/models/eagle/__init__.py | - tensorrt_llm/models/eagle/config.py | - tensorrt_llm/models/eagle/model.py | - tensorrt_llm/models/enc_dec/__init__.py | - tensorrt_llm/models/enc_dec/model.py | - tensorrt_llm/models/falcon/__init__.py | - tensorrt_llm/models/falcon/config.py | - tensorrt_llm/models/falcon/convert.py | - tensorrt_llm/models/falcon/model.py | - tensorrt_llm/models/gemma/__init__.py | - tensorrt_llm/models/gemma/config.py | - tensorrt_llm/models/gemma/convert.py | - tensorrt_llm/models/gemma/model.py | - tensorrt_llm/models/gemma/smoothquant.py | - tensorrt_llm/models/gemma/utils/__init__.py | - tensorrt_llm/models/gemma/utils/layers.py | - tensorrt_llm/models/gemma/utils/modules.py | - tensorrt_llm/models/gemma/utils/params.py | - tensorrt_llm/models/gemma/utils/positional_embeddings.py | - tensorrt_llm/models/gemma/utils/sampler.py | - tensorrt_llm/models/gemma/utils/transformer.py | - tensorrt_llm/models/gemma/weight.py | - tensorrt_llm/models/generation_mixin.py | - tensorrt_llm/models/gpt/__init__.py | - tensorrt_llm/models/gpt/config.py | - tensorrt_llm/models/gpt/convert.py | - tensorrt_llm/models/gpt/model.py | - tensorrt_llm/models/gptj/__init__.py | - tensorrt_llm/models/gptj/config.py | - tensorrt_llm/models/gptj/convert.py | - tensorrt_llm/models/gptj/model.py | - tensorrt_llm/models/gptneox/__init__.py | - tensorrt_llm/models/gptneox/model.py | - tensorrt_llm/models/grok/__init__.py | - tensorrt_llm/models/grok/convert.py | - tensorrt_llm/models/grok/model.py | - tensorrt_llm/models/grok/weight.py | - tensorrt_llm/models/llama/__init__.py | - tensorrt_llm/models/llama/config.py | - tensorrt_llm/models/llama/convert.py | - tensorrt_llm/models/llama/model.py | - tensorrt_llm/models/mamba/__init__.py | - tensorrt_llm/models/mamba/config.py | - tensorrt_llm/models/mamba/convert.py | - tensorrt_llm/models/mamba/model.py | - tensorrt_llm/models/medusa/__init__.py | - tensorrt_llm/models/medusa/config.py | - tensorrt_llm/models/medusa/model.py | - tensorrt_llm/models/medusa/weight.py | - tensorrt_llm/models/mllama/__init__.py | - tensorrt_llm/models/mllama/config.py | - tensorrt_llm/models/mllama/model.py | - tensorrt_llm/models/mmdit_sd3/__init__.py | - tensorrt_llm/models/mmdit_sd3/config.py | - tensorrt_llm/models/mmdit_sd3/model.py | - tensorrt_llm/models/model_weights_loader.py | tensorrt_llm/models/modeling_utils.py | - tensorrt_llm/models/mpt/__init__.py | - tensorrt_llm/models/mpt/model.py | - tensorrt_llm/models/multimodal_encoders/__init__.py | - tensorrt_llm/models/multimodal_encoders/config.py | - tensorrt_llm/models/multimodal_encoders/model.py | - tensorrt_llm/models/nemotron_nas/__init__.py | - tensorrt_llm/models/nemotron_nas/config.py | - tensorrt_llm/models/nemotron_nas/convert.py | - tensorrt_llm/models/nemotron_nas/layer_config.py | - tensorrt_llm/models/nemotron_nas/model.py | - tensorrt_llm/models/opt/__init__.py | - tensorrt_llm/models/opt/model.py | - tensorrt_llm/models/phi/__init__.py | - tensorrt_llm/models/phi/config.py | - tensorrt_llm/models/phi/convert.py | - tensorrt_llm/models/phi/model.py | - tensorrt_llm/models/phi3/__init__.py | - tensorrt_llm/models/phi3/config.py | - tensorrt_llm/models/phi3/convert.py | - tensorrt_llm/models/phi3/model.py | - tensorrt_llm/models/phi3/split_weights.py | - tensorrt_llm/models/qwen/__init__.py | - tensorrt_llm/models/qwen/config.py | - tensorrt_llm/models/qwen/convert.py | - tensorrt_llm/models/qwen/model.py | - tensorrt_llm/models/qwen/utils.py | - tensorrt_llm/models/recurrentgemma/__init__.py | - tensorrt_llm/models/recurrentgemma/model.py | - tensorrt_llm/models/redrafter/__init__.py | - tensorrt_llm/models/redrafter/drafter.py | - tensorrt_llm/models/redrafter/model.py | - tensorrt_llm/models/redrafter/redrafter_helper.py | - tensorrt_llm/models/stdit/__init__.py | - tensorrt_llm/models/stdit/config.py | - tensorrt_llm/models/stdit/model.py | - tensorrt_llm/models/unet/__init__.py | - tensorrt_llm/models/unet/attention.py | - tensorrt_llm/models/unet/embeddings.py | - tensorrt_llm/models/unet/pp/__init__.py | - tensorrt_llm/models/unet/pp/attention.py | - tensorrt_llm/models/unet/pp/conv2d.py | - tensorrt_llm/models/unet/pp/groupnorm.py | - tensorrt_llm/models/unet/pp/unet_pp.py | - tensorrt_llm/models/unet/resnet.py | - tensorrt_llm/models/unet/unet_2d_blocks.py | - tensorrt_llm/models/unet/unet_2d_condition.py | - tensorrt_llm/models/unet/weights.py | - tensorrt_llm/network.py | - tensorrt_llm/parameter.py | - tensorrt_llm/plugin/__init__.py | - tensorrt_llm/plugin/plugin.py | tensorrt_llm/quantization/__init__.py | tensorrt_llm/quantization/functional.py | - tensorrt_llm/quantization/image_processing.py | - tensorrt_llm/quantization/layers.py | tensorrt_llm/quantization/mode.py | - tensorrt_llm/quantization/quantize.py | - tensorrt_llm/quantization/quantize_by_modelopt.py | tensorrt_llm/quantization/utils/__init__.py | tensorrt_llm/quantization/utils/fp4_utils.py | tensorrt_llm/quantization/utils/fp8_utils.py | tensorrt_llm/ray_stub.py | tensorrt_llm/runtime/__init__.py | - tensorrt_llm/runtime/enc_dec_model_runner.py | - tensorrt_llm/runtime/generation.py | - tensorrt_llm/runtime/kv_cache_manager.py | - tensorrt_llm/runtime/medusa_utils.py | tensorrt_llm/runtime/memory_pools/__init__.py | - tensorrt_llm/runtime/memory_pools/memory_pools_allocator.py | - tensorrt_llm/runtime/memory_pools/pool.py | - tensorrt_llm/runtime/memory_pools/pools_kv_cache_manager.py | - tensorrt_llm/runtime/model_runner.py | - tensorrt_llm/runtime/model_runner_cpp.py | - tensorrt_llm/runtime/multimodal_model_runner.py | - tensorrt_llm/runtime/processor_wrapper/__init__.py | - tensorrt_llm/runtime/processor_wrapper/mllama_processor_wrapper.py | - tensorrt_llm/runtime/processor_wrapper/processor_wrapper.py | - tensorrt_llm/runtime/redrafter_utils.py | - tensorrt_llm/runtime/session.py | tensorrt_llm/scaffolding/__init__.py | tensorrt_llm/scaffolding/benchmark.py | tensorrt_llm/scaffolding/contrib/AsyncGeneration/stream_generation.py | @@ -826,12 +479,7 @@ common-files: &common_files | tensorrt_llm/tokenizer/tokenizer.py | tensorrt_llm/tools/__init__.py | tensorrt_llm/tools/importlib_utils.py | - tensorrt_llm/tools/multimodal_builder.py | - tensorrt_llm/tools/onnx_utils.py | tensorrt_llm/tools/plugin_gen/__init__.py | - tensorrt_llm/tools/plugin_gen/core.py | - tensorrt_llm/tools/plugin_gen/plugin_gen.py | - tensorrt_llm/tools/plugin_gen/shape_infer.py | tensorrt_llm/tools/ppl.py | tensorrt_llm/tools/profiler/nsys_profile_tools/gputrc2graph.py | tensorrt_llm/version.py | @@ -842,7 +490,6 @@ common-files: &common_files | tests/integration/defs/accuracy/scripts/compute_theta_and_thresholds.py | tests/integration/defs/accuracy/test_cli_flow.py | tests/integration/defs/accuracy/test_disaggregated_serving.py | - tests/integration/defs/accuracy/test_llm_api.py | tests/integration/defs/accuracy/test_llm_api_autodeploy.py | tests/integration/defs/accuracy/test_llm_api_pytorch.py | tests/integration/defs/accuracy/test_llm_api_pytorch_ray.py | @@ -851,63 +498,29 @@ common-files: &common_files | tests/integration/defs/conftest.py | tests/integration/defs/cpp/conftest.py | tests/integration/defs/cpp/cpp_common.py | - tests/integration/defs/cpp/test_e2e.py | tests/integration/defs/cpp/test_multi_gpu.py | tests/integration/defs/cpp/test_unit_tests.py | - tests/integration/defs/deterministic/mixtral_deterministic.py | - tests/integration/defs/deterministic/test_mixtral_deterministic.py | tests/integration/defs/disaggregated/test_auto_scaling.py | tests/integration/defs/disaggregated/test_disaggregated.py | tests/integration/defs/disaggregated/test_disaggregated_etcd.py | tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py | tests/integration/defs/disaggregated/test_workers.py | - tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py | tests/integration/defs/examples/run_llm_quickstart_atexit.py | tests/integration/defs/examples/serve/test_serve.py | tests/integration/defs/examples/serve/test_serve_negative.py | tests/integration/defs/examples/test_ad_guided_decoding.py | - tests/integration/defs/examples/test_bert.py | - tests/integration/defs/examples/test_bindings.py | - tests/integration/defs/examples/test_chatglm.py | - tests/integration/defs/examples/test_commandr.py | - tests/integration/defs/examples/test_draft_target_model.py | - tests/integration/defs/examples/test_eagle.py | - tests/integration/defs/examples/test_enc_dec.py | - tests/integration/defs/examples/test_exaone.py | - tests/integration/defs/examples/test_gemma.py | tests/integration/defs/examples/test_gpt.py | - tests/integration/defs/examples/test_gptj.py | - tests/integration/defs/examples/test_granite.py | - tests/integration/defs/examples/test_internlm.py | - tests/integration/defs/examples/test_llama.py | tests/integration/defs/examples/test_llm_api_with_mpi.py | - tests/integration/defs/examples/test_mamba.py | - tests/integration/defs/examples/test_medusa.py | - tests/integration/defs/examples/test_mistral.py | - tests/integration/defs/examples/test_mixtral.py | - tests/integration/defs/examples/test_multimodal.py | - tests/integration/defs/examples/test_nemotron.py | - tests/integration/defs/examples/test_nemotron_nas.py | - tests/integration/defs/examples/test_ngram.py | - tests/integration/defs/examples/test_openai.py | tests/integration/defs/examples/test_phi.py | - tests/integration/defs/examples/test_qwen.py | - tests/integration/defs/examples/test_qwen2audio.py | - tests/integration/defs/examples/test_qwenvl.py | tests/integration/defs/examples/test_ray.py | - tests/integration/defs/examples/test_recurrentgemma.py | - tests/integration/defs/examples/test_redrafter.py | - tests/integration/defs/examples/test_whisper.py | tests/integration/defs/llmapi/__init__.py | tests/integration/defs/llmapi/_run_llmapi_llm.py | tests/integration/defs/llmapi/test_llm_api_connector.py | tests/integration/defs/llmapi/test_llm_api_qa.py | - tests/integration/defs/llmapi/test_llm_e2e.py | tests/integration/defs/llmapi/test_llm_examples.py | tests/integration/defs/local_venv.py | tests/integration/defs/perf/__init__.py | tests/integration/defs/perf/allowed_configs.py | - tests/integration/defs/perf/build.py | tests/integration/defs/perf/create_perf_comparison_report.py | tests/integration/defs/perf/data.py | tests/integration/defs/perf/data_export.py | @@ -928,38 +541,21 @@ common-files: &common_files | tests/integration/defs/test_fmha.py | tests/integration/defs/test_list_parser.py | tests/integration/defs/test_list_validation.py | - tests/integration/defs/test_mlpf_results.py | tests/integration/defs/test_sanity.py | tests/integration/defs/test_unittests.py | tests/integration/defs/triton_server/__init__.py | - tests/integration/defs/triton_server/build_engines.py | tests/integration/defs/triton_server/common.py | tests/integration/defs/triton_server/conftest.py | - tests/integration/defs/triton_server/local_venv.py | - tests/integration/defs/triton_server/rcca/bug_4323566/inflight_batcher_llm_client_with_end_id.py | - tests/integration/defs/triton_server/runner_interface.py | tests/integration/defs/triton_server/test_list_parser.py | - tests/integration/defs/triton_server/test_triton.py | - tests/integration/defs/triton_server/test_triton_llm.py | - tests/integration/defs/triton_server/test_triton_memleak.py | - tests/integration/defs/triton_server/test_triton_multi_node.py | - tests/integration/defs/triton_server/test_triton_rcca.py | tests/integration/defs/triton_server/trt_test_alternative.py | tests/integration/defs/trt_test_alternative.py | tests/integration/defs/utils/__init__.py | tests/integration/defs/utils/periodic_junit.py | tests/integration/defs/utils/timeout_manager.py | tests/microbenchmarks/all_reduce.py | - tests/microbenchmarks/build_time_benchmark.py | - tests/microbenchmarks/build_time_dashboard.py | tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py | tests/scripts/allreduce_perf/allreduce_perf_viz.py | tests/scripts/iteration_log_parser.py | - tests/scripts/perf-sanity/parse_benchmark_results.py | - tests/scripts/perf-sanity/run_benchmark_serve.py | - tests/unittest/_torch/attention/sparse/test_dsa_indexer.py | - tests/unittest/_torch/attention/sparse/test_flash_mla.py | - tests/unittest/_torch/attention/sparse/test_rocketkv.py | tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py | tests/unittest/_torch/attention/test_attention.py | tests/unittest/_torch/attention/test_attention_mla.py | @@ -980,7 +576,6 @@ common-files: &common_files | tests/unittest/_torch/misc/test_virtual_memory.py | tests/unittest/_torch/modeling/test_modeling_bert.py | tests/unittest/_torch/modeling/test_modeling_clip.py | - tests/unittest/_torch/modeling/test_modeling_exaone4.py | tests/unittest/_torch/modeling/test_modeling_gemma3.py | tests/unittest/_torch/modeling/test_modeling_gpt_oss.py | tests/unittest/_torch/modeling/test_modeling_llama.py | @@ -1004,8 +599,6 @@ common-files: &common_files | tests/unittest/_torch/modules/test_moe_routing.py | tests/unittest/_torch/modules/test_rotary_embedding.py | tests/unittest/_torch/modules/test_triton_linear.py | - tests/unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py | - tests/unittest/_torch/modules/tests_lora_modules/test_lora_plugin_vs_lora_op.py | tests/unittest/_torch/multi_gpu/test_allreduce.py | tests/unittest/_torch/multi_gpu/test_alltoall.py | tests/unittest/_torch/multi_gpu/test_ar_residual_norm.py | @@ -1032,24 +625,10 @@ common-files: &common_files | tests/unittest/_torch/sampler/test_beam_search.py | tests/unittest/_torch/sampler/test_best_of_n.py | tests/unittest/_torch/sampler/test_trtllm_sampler.py | - tests/unittest/_torch/speculative/test_draft_target.py | - tests/unittest/_torch/speculative/test_draft_token_tree_sampling.py | - tests/unittest/_torch/speculative/test_draft_token_tree_verification.py | - tests/unittest/_torch/speculative/test_dynamic_spec_decode.py | tests/unittest/_torch/speculative/test_eagle3.py | - tests/unittest/_torch/speculative/test_kv_cache_reuse.py | - tests/unittest/_torch/speculative/test_mtp.py | - tests/unittest/_torch/speculative/test_ngram.py | - tests/unittest/_torch/speculative/test_save_state.py | - tests/unittest/_torch/speculative/test_spec_gate.py | - tests/unittest/_torch/speculative/test_torch_rejection_sampling.py | - tests/unittest/_torch/speculative/test_user_provided.py | tests/unittest/_torch/test_connector.py | tests/unittest/_torch/test_torch_multi_arange.py | tests/unittest/_torch/thop/parallel/deep_gemm_tests.py | - tests/unittest/_torch/thop/parallel/test_causal_conv1d_op.py | - tests/unittest/_torch/thop/parallel/test_cublas_mm.py | - tests/unittest/_torch/thop/parallel/test_custom_ops.py | tests/unittest/_torch/thop/parallel/test_dsv3_fused_a_gemm.py | tests/unittest/_torch/thop/parallel/test_dsv3_router_gemm.py | tests/unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py | @@ -1063,11 +642,6 @@ common-files: &common_files | tests/unittest/_torch/thop/parallel/test_fp8_per_tensor_scale_tllmg_gemm.py | tests/unittest/_torch/thop/parallel/test_fp8_quantize.py | tests/unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py | - tests/unittest/_torch/thop/parallel/test_fused_qk_norm_rope.py | - tests/unittest/_torch/thop/parallel/test_logits_bitmask_op.py | - tests/unittest/_torch/thop/parallel/test_mamba2_chunk_ss_update.py | - tests/unittest/_torch/thop/parallel/test_mamba_conv1d_op.py | - tests/unittest/_torch/thop/parallel/test_noaux_tc.py | tests/unittest/_torch/thop/parallel/test_scaled_mm.py | tests/unittest/_torch/thop/parallel/test_selective_scan_op.py | tests/unittest/_torch/thop/parallel/test_tinygemm2.py | @@ -1081,7 +655,6 @@ common-files: &common_files | tests/unittest/_torch/thop/serial/test_moe_alltoall.py | tests/unittest/api_stability/api_stability_core.py | tests/unittest/api_stability/test_llm_api.py | - tests/unittest/bindings/binding_test_utils.py | tests/unittest/bindings/test_bindings_moe.py | tests/unittest/bindings/test_bindings_ut.py | tests/unittest/bindings/test_executor_bindings.py | @@ -1112,12 +685,10 @@ common-files: &common_files | tests/unittest/llmapi/apps/_test_openai_chat_harmony.py | tests/unittest/llmapi/apps/_test_openai_chat_multimodal.py | tests/unittest/llmapi/apps/_test_openai_completions.py | - tests/unittest/llmapi/apps/_test_openai_consistent_chat.py | tests/unittest/llmapi/apps/_test_openai_lora.py | tests/unittest/llmapi/apps/_test_openai_metrics.py | tests/unittest/llmapi/apps/_test_openai_misc.py | tests/unittest/llmapi/apps/_test_openai_mmencoder.py | - tests/unittest/llmapi/apps/_test_openai_multi_chat.py | tests/unittest/llmapi/apps/_test_openai_multi_gpu.py | tests/unittest/llmapi/apps/_test_openai_multi_nodes.py | tests/unittest/llmapi/apps/_test_openai_perf_metrics.py | @@ -1140,15 +711,12 @@ common-files: &common_files | tests/unittest/llmapi/run_llm_exit.py | tests/unittest/llmapi/run_llm_with_postproc.py | tests/unittest/llmapi/test_additional_model_outputs.py | - tests/unittest/llmapi/test_build_cache.py | tests/unittest/llmapi/test_executor.py | tests/unittest/llmapi/test_gc_utils.py | tests/unittest/llmapi/test_llm.py | tests/unittest/llmapi/test_llm_args.py | tests/unittest/llmapi/test_llm_download.py | tests/unittest/llmapi/test_llm_kv_cache_events.py | - tests/unittest/llmapi/test_llm_models.py | - tests/unittest/llmapi/test_llm_multi_gpu.py | tests/unittest/llmapi/test_llm_multi_gpu_pytorch.py | tests/unittest/llmapi/test_llm_pytorch.py | tests/unittest/llmapi/test_llm_quant.py | @@ -1159,25 +727,14 @@ common-files: &common_files | tests/unittest/llmapi/test_serialization.py | tests/unittest/llmapi/test_utils.py | tests/unittest/others/__init__.py | - tests/unittest/others/test_builder.py | tests/unittest/others/test_convert_spec_decoding_mask_to_packed_mask.py | - tests/unittest/others/test_debugging_api.py | tests/unittest/others/test_exception.py | tests/unittest/others/test_export.py | - tests/unittest/others/test_graph_rewriter.py | - tests/unittest/others/test_kv_cache_manager.py | tests/unittest/others/test_kv_cache_transceiver.py | tests/unittest/others/test_kv_cache_update.py | - tests/unittest/others/test_layer.py | - tests/unittest/others/test_leak.py | tests/unittest/others/test_mapping.py | - tests/unittest/others/test_model_dtype.py | - tests/unittest/others/test_module.py | tests/unittest/others/test_multimodal_registry.py | - tests/unittest/others/test_plugins.py | - tests/unittest/others/test_precision_control.py | tests/unittest/others/test_pretrained_config.py | - tests/unittest/others/test_session.py | tests/unittest/others/test_time_breakdown.py | tests/unittest/profile_utils.py | tests/unittest/scaffolding/__init__.py | @@ -1186,141 +743,14 @@ common-files: &common_files | tests/unittest/scaffolding/test_scaffolding.py | tests/unittest/scaffolding/test_task_collection.py | tests/unittest/scaffolding/test_worker.py | - tests/unittest/test_model_runner_cpp.py | tests/unittest/test_pip_install.py | tests/unittest/tools/__init__.py | - tests/unittest/tools/plugin_gen/__init__.py | - tests/unittest/tools/plugin_gen/kernel_config.py | - tests/unittest/tools/plugin_gen/test_core.py | - tests/unittest/tools/plugin_gen/test_plugin_gen.py | - tests/unittest/tools/plugin_gen/test_shape_infer.py | tests/unittest/tools/test_prepare_dataset.py | tests/unittest/tools/test_test_to_stage_mapping.py | - tests/unittest/trt/__init__.py | - tests/unittest/trt/attention/test_bert_attention.py | - tests/unittest/trt/attention/test_gpt_attention.py | - tests/unittest/trt/attention/test_gpt_attention_IFB.py | - tests/unittest/trt/attention/test_gpt_attention_no_cache.py | - tests/unittest/trt/attention/test_sage_attention.py | - tests/unittest/trt/functional/__init__.py | - tests/unittest/trt/functional/test_alibi.py | - tests/unittest/trt/functional/test_allreduce_norm.py | - tests/unittest/trt/functional/test_allreduce_prepost_residual_norm.py | - tests/unittest/trt/functional/test_arange.py | - tests/unittest/trt/functional/test_argmax.py | - tests/unittest/trt/functional/test_assertion.py | - tests/unittest/trt/functional/test_avg_pool2d.py | - tests/unittest/trt/functional/test_cast.py | - tests/unittest/trt/functional/test_conv2d.py | - tests/unittest/trt/functional/test_conv3d.py | - tests/unittest/trt/functional/test_cos.py | - tests/unittest/trt/functional/test_cumsum.py | - tests/unittest/trt/functional/test_dora.py | - tests/unittest/trt/functional/test_einsum.py | - tests/unittest/trt/functional/test_embedding_single_gpu.py | - tests/unittest/trt/functional/test_exp.py | - tests/unittest/trt/functional/test_expand.py | - tests/unittest/trt/functional/test_flatten.py | - tests/unittest/trt/functional/test_flip.py | - tests/unittest/trt/functional/test_fp4_gemm.py | - tests/unittest/trt/functional/test_fp4_gemm_ootb.py | - tests/unittest/trt/functional/test_gather.py | - tests/unittest/trt/functional/test_gather_nd.py | - tests/unittest/trt/functional/test_geglu.py | - tests/unittest/trt/functional/test_gelu.py | - tests/unittest/trt/functional/test_gemm_swiglu.py | - tests/unittest/trt/functional/test_group_norm.py | - tests/unittest/trt/functional/test_identity.py | - tests/unittest/trt/functional/test_index_select.py | - tests/unittest/trt/functional/test_interpolate.py | - tests/unittest/trt/functional/test_logsoftmax.py | - tests/unittest/trt/functional/test_lora.py | - tests/unittest/trt/functional/test_low_latency_gemm.py | - tests/unittest/trt/functional/test_mamba_conv1d.py | - tests/unittest/trt/functional/test_masked_scatter.py | - tests/unittest/trt/functional/test_masked_select.py | - tests/unittest/trt/functional/test_matmul.py | - tests/unittest/trt/functional/test_meshgrid2d.py | - tests/unittest/trt/functional/test_moe.py | - tests/unittest/trt/functional/test_nccl.py | - tests/unittest/trt/functional/test_nonzero.py | - tests/unittest/trt/functional/test_outer.py | - tests/unittest/trt/functional/test_pad.py | - tests/unittest/trt/functional/test_permute.py | - tests/unittest/trt/functional/test_pp_reduce_scatter.py | - tests/unittest/trt/functional/test_quant.py | - tests/unittest/trt/functional/test_rearrange.py | - tests/unittest/trt/functional/test_repeat.py | - tests/unittest/trt/functional/test_repeat_interleave.py | - tests/unittest/trt/functional/test_rg_lru.py | - tests/unittest/trt/functional/test_sample.py | - tests/unittest/trt/functional/test_scatter.py | - tests/unittest/trt/functional/test_scatter_nd.py | - tests/unittest/trt/functional/test_select.py | - tests/unittest/trt/functional/test_selective_scan.py | - tests/unittest/trt/functional/test_sigmoid.py | - tests/unittest/trt/functional/test_silu.py | - tests/unittest/trt/functional/test_sin.py | - tests/unittest/trt/functional/test_slice.py | - tests/unittest/trt/functional/test_softplus.py | - tests/unittest/trt/functional/test_split.py | - tests/unittest/trt/functional/test_squeeze.py | - tests/unittest/trt/functional/test_swiglu.py | - tests/unittest/trt/functional/test_topk.py | - tests/unittest/trt/functional/test_transpose.py | - tests/unittest/trt/functional/test_unbind.py | - tests/unittest/trt/functional/test_unsqueeze.py | - tests/unittest/trt/functional/test_view.py | - tests/unittest/trt/functional/test_where.py | - tests/unittest/trt/model/__init__.py | - tests/unittest/trt/model/eagle/test_decode_draft_tokens_plugin.py | - tests/unittest/trt/model/eagle/test_prepare_drafter_inputs_plugin.py | - tests/unittest/trt/model/eagle/test_sample_accept_draft_tokens_plugin.py | - tests/unittest/trt/model/redrafter/test_beams2tree.py | - tests/unittest/trt/model/redrafter/test_draft_token.py | - tests/unittest/trt/model/redrafter/test_draft_token_indices.py | - tests/unittest/trt/model/redrafter/test_gather_beams.py | - tests/unittest/trt/model/redrafter/test_mask.py | - tests/unittest/trt/model/redrafter/test_packed_position_ids.py | - tests/unittest/trt/model/redrafter/test_prefix_match_indices.py | - tests/unittest/trt/model/redrafter/test_prepare_input.py | - tests/unittest/trt/model/redrafter/test_process_logits.py | - tests/unittest/trt/model/redrafter/test_top1.py | - tests/unittest/trt/model/redrafter/test_unpack_gen_data.py | - tests/unittest/trt/model/redrafter/test_validate.py | - tests/unittest/trt/model/test_gpt.py | - tests/unittest/trt/model/test_gpt_e2e.py | - tests/unittest/trt/model/test_llama.py | - tests/unittest/trt/model/test_mamba.py | - tests/unittest/trt/model/test_mistral.py | - tests/unittest/trt/model/test_nemotron_nas.py | - tests/unittest/trt/model/test_phi.py | - tests/unittest/trt/model/test_unet.py | - tests/unittest/trt/model_api/test_model_api_multi_gpu.py | - tests/unittest/trt/model_api/test_model_level_api.py | - tests/unittest/trt/model_api/test_model_quantization.py | - tests/unittest/trt/python_plugin/plugin_wrapper_utils.py | - tests/unittest/trt/python_plugin/test_plugin_wrapper.py | - tests/unittest/trt/quantization/__init__.py | - tests/unittest/trt/quantization/_utils.py | - tests/unittest/trt/quantization/test_fp8_quantization.py | - tests/unittest/trt/quantization/test_fp8_rowwise_gemm.py | - tests/unittest/trt/quantization/test_functional.py | - tests/unittest/trt/quantization/test_mode.py | - tests/unittest/trt/quantization/test_moe_weight_only_quant_matmul.py | - tests/unittest/trt/quantization/test_qserve_gemm.py | - tests/unittest/trt/quantization/test_quant.py | - tests/unittest/trt/quantization/test_quant_layer.py | - tests/unittest/trt/quantization/test_smooth_quant_gemm.py | - tests/unittest/trt/quantization/test_smooth_quant_layer_norm.py | - tests/unittest/trt/quantization/test_smooth_quant_rms_norm.py | - tests/unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py | - tests/unittest/trt/quantization/test_weight_only_quant_matmul.py | tests/unittest/utils/__init__.py | tests/unittest/utils/cpp_paths.py | tests/unittest/utils/llm_data.py | tests/unittest/utils/runtime_defaults.py | - tests/unittest/utils/test_medusa_utils.py | tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py | tests/unittest/utils/test_util.py | tests/unittest/utils/torch_ref.py | @@ -1350,14 +780,6 @@ legacy-files: &legacy_files | .devcontainer/make_env.py | .github/scripts/label_community_user.py | .github/scripts/pr_checklist_check.py | - benchmarks/cpp/__init__.py | - benchmarks/cpp/prepare_dataset.py | - benchmarks/cpp/utils/__init__.py | - benchmarks/cpp/utils/convert_nemo_dataset.py | - benchmarks/cpp/utils/generate_rand_loras.py | - benchmarks/cpp/utils/prepare_real_data.py | - benchmarks/cpp/utils/prepare_synthetic_data.py | - benchmarks/cpp/utils/utils.py | cpp/conanfile.py | cpp/kernels/fmha_v2/conftest.py | cpp/kernels/fmha_v2/fmha_test.py | @@ -1382,58 +804,17 @@ legacy-files: &legacy_files | cpp/tensorrt_llm/deep_ep/strip_nvshmem_helper.py | cpp/tensorrt_llm/kernels/cutlass_kernels/python/generate_kernels.py | cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/copy_cu.py | - cpp/tests/resources/scripts/build_chatglm_engines.py | - cpp/tests/resources/scripts/build_eagle_engines.py | - cpp/tests/resources/scripts/build_enc_dec_engines.py | - cpp/tests/resources/scripts/build_engines_utils.py | - cpp/tests/resources/scripts/build_gpt_engines.py | - cpp/tests/resources/scripts/build_gptj_engines.py | - cpp/tests/resources/scripts/build_llama_engines.py | - cpp/tests/resources/scripts/build_mamba_engines.py | - cpp/tests/resources/scripts/build_medusa_engines.py | - cpp/tests/resources/scripts/build_recurrentgemma_engines.py | - cpp/tests/resources/scripts/build_redrafter_engines.py | - cpp/tests/resources/scripts/generate_expected_chatglm_output.py | - cpp/tests/resources/scripts/generate_expected_eagle_output.py | - cpp/tests/resources/scripts/generate_expected_enc_dec_output.py | - cpp/tests/resources/scripts/generate_expected_gpt_output.py | - cpp/tests/resources/scripts/generate_expected_gptj_output.py | - cpp/tests/resources/scripts/generate_expected_llama_output.py | - cpp/tests/resources/scripts/generate_expected_mamba_output.py | - cpp/tests/resources/scripts/generate_expected_medusa_output.py | - cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py | - cpp/tests/resources/scripts/generate_expected_redrafter_output.py | - cpp/tests/resources/scripts/generate_hf_gpt_output.py | cpp/tests/resources/scripts/generate_test_lora_weights.py | - cpp/tests/resources/scripts/io_converter.py | docs/source/conf.py | docs/source/helper.py | examples/apps/chat.py | examples/apps/fastapi_server.py | - examples/bindings/executor/example_advanced.py | - examples/bindings/executor/example_basic.py | - examples/bindings/executor/example_debug.py | - examples/bindings/executor/example_logits_processor.py | examples/disaggregated/clients/disagg_client.py | examples/disaggregated/slurm/benchmark/submit.py | - examples/dora/normalize_weights.py | - examples/eagle/convert_checkpoint.py | - examples/eval_long_context.py | - examples/generate_checkpoint_config.py | - examples/generate_xgrammar_tokenizer_info.py | - examples/hf_lora_convert.py | examples/infinitebench/args.py | examples/infinitebench/compute_scores.py | examples/infinitebench/construct_synthetic_dataset.py | examples/infinitebench/eval_utils.py | - examples/llm-api/_tensorrt_engine/llm_eagle2_decoding.py | - examples/llm-api/_tensorrt_engine/llm_eagle_decoding.py | - examples/llm-api/_tensorrt_engine/llm_inference_customize.py | - examples/llm-api/_tensorrt_engine/llm_inference_kv_events.py | - examples/llm-api/_tensorrt_engine/llm_lookahead_decoding.py | - examples/llm-api/_tensorrt_engine/llm_medusa_decoding.py | - examples/llm-api/_tensorrt_engine/llm_quantization.py | - examples/llm-api/_tensorrt_engine/quickstart_example.py | examples/llm-api/llm_guided_decoding.py | examples/llm-api/llm_inference.py | examples/llm-api/llm_inference_async.py | @@ -1453,122 +834,17 @@ legacy-files: &legacy_files | examples/llm-api/quickstart_example.py | examples/llm-api/quickstart_multimodal.py | examples/llm-api/star_attention.py | - examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py | examples/longbench/eval_longbench_v1.py | - examples/medusa/convert_checkpoint.py | - examples/mmlu.py | - examples/models/contrib/baichuan/convert_checkpoint.py | - examples/models/contrib/bloom/convert_checkpoint.py | - examples/models/contrib/chatglm-6b/tokenization_chatglm.py | - examples/models/contrib/chatglm2-6b/tokenization_chatglm.py | - examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py | - examples/models/contrib/cogvlm/convert_checkpoint.py | - examples/models/contrib/dbrx/convert_checkpoint.py | - examples/models/contrib/deepseek_v1/__init__.py | - examples/models/contrib/deepseek_v1/convert_checkpoint.py | - examples/models/contrib/deepseek_v2/convert_checkpoint.py | - examples/models/contrib/dit/convert_checkpoint.py | - examples/models/contrib/dit/diffusion.py | - examples/models/contrib/dit/sample.py | - examples/models/contrib/dit/utils_modelopt.py | - examples/models/contrib/dit/vae_decoder_trt.py | - examples/models/contrib/falcon/convert_checkpoint.py | - examples/models/contrib/gptj/convert_checkpoint.py | - examples/models/contrib/gptneox/convert_checkpoint.py | - examples/models/contrib/grok/convert_checkpoint.py | - examples/models/contrib/mmdit/convert_checkpoint.py | - examples/models/contrib/mmdit/sample.py | - examples/models/contrib/mpt/convert_checkpoint.py | - examples/models/contrib/opt/convert_checkpoint.py | - examples/models/contrib/sdxl/build_sdxl_unet.py | - examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py | - examples/models/contrib/sdxl/run_sdxl.py | - examples/models/contrib/stdit/aspect.py | - examples/models/contrib/stdit/convert_checkpoint.py | - examples/models/contrib/stdit/pipeline_tllm.py | - examples/models/contrib/stdit/sample.py | - examples/models/contrib/stdit/scheduler.py | - examples/models/contrib/stdit/text_encoder.py | - examples/models/contrib/stdit/utils.py | - examples/models/contrib/stdit/vae.py | - examples/models/contrib/stdit/video_transforms.py | - examples/models/core/bert/__init__.py | - examples/models/core/bert/convert_checkpoint.py | - examples/models/core/bert/run.py | - examples/models/core/bert/utils.py | - examples/models/core/commandr/convert_checkpoint.py | - examples/models/core/enc_dec/__init__.py | - examples/models/core/enc_dec/convert_checkpoint.py | - examples/models/core/enc_dec/helper.py | - examples/models/core/enc_dec/run.py | - examples/models/core/gemma/convert_checkpoint.py | - examples/models/core/glm-4-9b/convert_checkpoint.py | - examples/models/core/glm-4-9b/tokenization_chatglm.py | - examples/models/core/gpt/convert_checkpoint.py | - examples/models/core/gpt/merge_ptuning_tables.py | - examples/models/core/gpt/nemo_lora_convert.py | - examples/models/core/gpt/nemo_prompt_convert.py | - examples/models/core/gpt/run_hf.py | examples/models/core/gpt_oss/openai_chat_client_function_calling.py | - examples/models/core/internlm2/convert_checkpoint.py | examples/models/core/kimi_k2/kimi_k2_tool_calling_example.py | - examples/models/core/llama/convert_checkpoint.py | - examples/models/core/llama/summarize_long.py | - examples/models/core/mamba/convert_checkpoint.py | - examples/models/core/mllama/convert_checkpoint.py | - examples/models/core/multimodal/__init__.py | - examples/models/core/multimodal/build_multimodal_engine.py | - examples/models/core/multimodal/eval.py | - examples/models/core/multimodal/run.py | - examples/models/core/multimodal/utils.py | - examples/models/core/nemotron_nas/calibration_utils.py | - examples/models/core/nemotron_nas/convert_checkpoint.py | - examples/models/core/phi/convert_checkpoint.py | - examples/models/core/qwen/convert_checkpoint.py | - examples/models/core/qwen2audio/run.py | - examples/models/core/qwen2audio/run_chat.py | - examples/models/core/qwen2audio/utils.py | - examples/models/core/qwenvl/run.py | - examples/models/core/qwenvl/run_chat.py | - examples/models/core/qwenvl/show_pic.py | - examples/models/core/qwenvl/vit_onnx_trt.py | - examples/models/core/recurrentgemma/convert_checkpoint.py | - examples/models/core/vit/convert_checkpoint.py | - examples/models/core/whisper/convert_checkpoint.py | - examples/models/core/whisper/distil_whisper/convert_from_distil_whisper.py | - examples/models/core/whisper/run.py | - examples/models/core/whisper/tokenizer.py | - examples/models/core/whisper/whisper_utils.py | - examples/ngram/run_dtm_ngram.py | - examples/openai_triton/manual_plugin/build.py | - examples/openai_triton/manual_plugin/fmha_triton.py | - examples/openai_triton/manual_plugin/plugin.py | - examples/openai_triton/manual_plugin/run.py | - examples/openai_triton/plugin_autogen/build_engine.py | - examples/openai_triton/plugin_autogen/kernel_config.py | - examples/openai_triton/plugin_autogen/run_engine.py | - examples/python_plugin/build_lookup.py | - examples/python_plugin/plugin_lib/__init__.py | - examples/python_plugin/plugin_lib/lookup_kernel.py | - examples/python_plugin/plugin_lib/lookup_plugin.py | - examples/python_plugin/run_lookup.py | - examples/quantization/quantize.py | examples/quantization/quantize_mixed_precision_moe.py | examples/ray_orchestrator/llm_inference_async_ray.py | examples/ray_orchestrator/llm_inference_distributed_ray.py | - examples/redrafter/convert_checkpoint.py | - examples/run.py | examples/scaffolding/contrib/AsyncGeneration/stream_generation_controller.py | examples/scaffolding/contrib/DeepConf/run_generation.py | examples/scaffolding/contrib/Dynasor/scaffolding_dynasor_run.py | examples/scaffolding/contrib/TreeInference/run_mcts_example.py | examples/scaffolding/contrib/TreeInference/run_tot_example.py | - examples/scaffolding/contrib/mcp/e2b/e2bserver.py | - examples/scaffolding/contrib/mcp/e2b/main.py | - examples/scaffolding/contrib/mcp/mcptest.py | - examples/scaffolding/contrib/mcp/weather/weather.py | - examples/scaffolding/contrib/mcp/websearch/main.py | - examples/scaffolding/contrib/mcp/websearch/websearch.py | examples/scaffolding/run_basic_generation.py | examples/scaffolding/run_best_of_n_with_reward.py | examples/scaffolding/run_majority_vote_aime24.py | @@ -1578,8 +854,6 @@ legacy-files: &legacy_files | examples/serve/openai_completion_client.py | examples/serve/openai_completion_client_for_lora.py | examples/serve/openai_completion_client_json_schema.py | - examples/summarize.py | - examples/utils.py | examples/wide_ep/ep_load_balancer/generate_eplb_config.py | examples/wide_ep/ep_load_balancer/report_load_statistics.py | examples/wide_ep/ep_load_balancer/utils.py | @@ -1587,12 +861,10 @@ legacy-files: &legacy_files | jenkins/scripts/mergeWaiveList.py | jenkins/scripts/open_search_db.py | jenkins/scripts/test_rerun.py | - scripts/build_cpp_examples.py | scripts/build_wheel.py | scripts/check_test_list.py | scripts/dco_check.py | scripts/format_test_list.py | - scripts/generate_duration.py | scripts/generate_lock_file.py | scripts/get_wheel_from_package.py | scripts/git_replace.py | @@ -1603,7 +875,6 @@ legacy-files: &legacy_files | setup.py | tensorrt_llm/__init__.py | tensorrt_llm/_ray_utils.py | - tensorrt_llm/_tensorrt_engine/__init__.py | tensorrt_llm/_torch/__init__.py | tensorrt_llm/_torch/attention_backend/__init__.py | tensorrt_llm/_torch/attention_backend/flashinfer.py | @@ -1807,7 +1078,6 @@ legacy-files: &legacy_files | tensorrt_llm/_torch/pyexecutor/guided_decoder.py | tensorrt_llm/_torch/pyexecutor/handle_additional_outputs.py | tensorrt_llm/_torch/pyexecutor/handle_logits.py | - tensorrt_llm/_torch/pyexecutor/kv_cache_connector.py | tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py | tensorrt_llm/_torch/pyexecutor/layerwise_nvtx_marker.py | tensorrt_llm/_torch/pyexecutor/llm_request.py | @@ -1845,11 +1115,6 @@ legacy-files: &legacy_files | tensorrt_llm/bench/benchmark/utils/asynchronous.py | tensorrt_llm/bench/benchmark/utils/general.py | tensorrt_llm/bench/benchmark/utils/processes.py | - tensorrt_llm/bench/build/__init__.py | - tensorrt_llm/bench/build/build.py | - tensorrt_llm/bench/build/dataclasses.py | - tensorrt_llm/bench/build/tuning.py | - tensorrt_llm/bench/build/utils.py | tensorrt_llm/bench/dataclasses/__init__.py | tensorrt_llm/bench/dataclasses/configuration.py | tensorrt_llm/bench/dataclasses/engine.py | @@ -1859,13 +1124,9 @@ legacy-files: &legacy_files | tensorrt_llm/bench/dataclasses/statistics.py | tensorrt_llm/bench/utils/__init__.py | tensorrt_llm/bench/utils/data.py | - tensorrt_llm/builder.py | tensorrt_llm/commands/__init__.py | tensorrt_llm/commands/bench.py | - tensorrt_llm/commands/build.py | tensorrt_llm/commands/eval.py | - tensorrt_llm/commands/prune.py | - tensorrt_llm/commands/refit.py | tensorrt_llm/commands/serve.py | tensorrt_llm/evaluate/__init__.py | tensorrt_llm/evaluate/cnn_dailymail.py | @@ -1901,23 +1162,7 @@ legacy-files: &legacy_files | tensorrt_llm/inputs/multimodal.py | tensorrt_llm/inputs/registry.py | tensorrt_llm/inputs/utils.py | - tensorrt_llm/layers/__init__.py | - tensorrt_llm/layers/activation.py | - tensorrt_llm/layers/attention.py | - tensorrt_llm/layers/cast.py | - tensorrt_llm/layers/conv.py | - tensorrt_llm/layers/embedding.py | - tensorrt_llm/layers/language_adapter.py | - tensorrt_llm/layers/linear.py | - tensorrt_llm/layers/lora.py | - tensorrt_llm/layers/mlp.py | - tensorrt_llm/layers/moe.py | - tensorrt_llm/layers/normalization.py | - tensorrt_llm/layers/pooling.py | - tensorrt_llm/layers/recurrent.py | - tensorrt_llm/layers/ssm.py | tensorrt_llm/llmapi/__init__.py | - tensorrt_llm/llmapi/build_cache.py | tensorrt_llm/llmapi/disagg_utils.py | tensorrt_llm/llmapi/kv_cache_type.py | tensorrt_llm/llmapi/llm.py | @@ -1940,179 +1185,17 @@ legacy-files: &legacy_files | tensorrt_llm/metrics/enums.py | tensorrt_llm/models/__init__.py | tensorrt_llm/models/automodel.py | - tensorrt_llm/models/baichuan/__init__.py | - tensorrt_llm/models/baichuan/config.py | - tensorrt_llm/models/baichuan/convert.py | - tensorrt_llm/models/baichuan/model.py | - tensorrt_llm/models/bert/__init__.py | - tensorrt_llm/models/bert/config.py | - tensorrt_llm/models/bert/convert.py | - tensorrt_llm/models/bert/model.py | - tensorrt_llm/models/bloom/__init__.py | - tensorrt_llm/models/bloom/model.py | - tensorrt_llm/models/chatglm/__init__.py | - tensorrt_llm/models/chatglm/config.py | - tensorrt_llm/models/chatglm/convert.py | - tensorrt_llm/models/chatglm/model.py | - tensorrt_llm/models/clip/__init__.py | - tensorrt_llm/models/clip/model.py | - tensorrt_llm/models/cogvlm/__init__.py | - tensorrt_llm/models/cogvlm/config.py | - tensorrt_llm/models/cogvlm/convert.py | - tensorrt_llm/models/cogvlm/model.py | - tensorrt_llm/models/commandr/__init__.py | - tensorrt_llm/models/commandr/config.py | - tensorrt_llm/models/commandr/model.py | tensorrt_llm/models/convert_utils.py | - tensorrt_llm/models/dbrx/__init__.py | - tensorrt_llm/models/dbrx/config.py | - tensorrt_llm/models/dbrx/model.py | - tensorrt_llm/models/deepseek_v1/__init__.py | - tensorrt_llm/models/deepseek_v1/config.py | - tensorrt_llm/models/deepseek_v1/convert.py | - tensorrt_llm/models/deepseek_v1/model.py | - tensorrt_llm/models/deepseek_v2/__init__.py | - tensorrt_llm/models/deepseek_v2/config.py | - tensorrt_llm/models/deepseek_v2/convert.py | - tensorrt_llm/models/deepseek_v2/model.py | - tensorrt_llm/models/dit/__init__.py | - tensorrt_llm/models/dit/model.py | - tensorrt_llm/models/eagle/__init__.py | - tensorrt_llm/models/eagle/config.py | - tensorrt_llm/models/eagle/model.py | - tensorrt_llm/models/enc_dec/__init__.py | - tensorrt_llm/models/enc_dec/model.py | - tensorrt_llm/models/falcon/__init__.py | - tensorrt_llm/models/falcon/config.py | - tensorrt_llm/models/falcon/convert.py | - tensorrt_llm/models/falcon/model.py | - tensorrt_llm/models/gemma/__init__.py | - tensorrt_llm/models/gemma/config.py | - tensorrt_llm/models/gemma/convert.py | - tensorrt_llm/models/gemma/model.py | - tensorrt_llm/models/gemma/smoothquant.py | - tensorrt_llm/models/gemma/utils/__init__.py | - tensorrt_llm/models/gemma/utils/layers.py | - tensorrt_llm/models/gemma/utils/modules.py | - tensorrt_llm/models/gemma/utils/params.py | - tensorrt_llm/models/gemma/utils/positional_embeddings.py | - tensorrt_llm/models/gemma/utils/sampler.py | - tensorrt_llm/models/gemma/utils/transformer.py | - tensorrt_llm/models/gemma/weight.py | - tensorrt_llm/models/generation_mixin.py | - tensorrt_llm/models/gpt/__init__.py | - tensorrt_llm/models/gpt/config.py | - tensorrt_llm/models/gpt/convert.py | - tensorrt_llm/models/gpt/model.py | - tensorrt_llm/models/gptj/__init__.py | - tensorrt_llm/models/gptj/config.py | - tensorrt_llm/models/gptj/convert.py | - tensorrt_llm/models/gptj/model.py | - tensorrt_llm/models/gptneox/__init__.py | - tensorrt_llm/models/gptneox/model.py | - tensorrt_llm/models/grok/__init__.py | - tensorrt_llm/models/grok/convert.py | - tensorrt_llm/models/grok/model.py | - tensorrt_llm/models/grok/weight.py | - tensorrt_llm/models/llama/__init__.py | - tensorrt_llm/models/llama/config.py | - tensorrt_llm/models/llama/convert.py | - tensorrt_llm/models/llama/model.py | - tensorrt_llm/models/mamba/__init__.py | - tensorrt_llm/models/mamba/config.py | - tensorrt_llm/models/mamba/convert.py | - tensorrt_llm/models/mamba/model.py | - tensorrt_llm/models/medusa/__init__.py | - tensorrt_llm/models/medusa/config.py | - tensorrt_llm/models/medusa/model.py | - tensorrt_llm/models/medusa/weight.py | - tensorrt_llm/models/mllama/__init__.py | - tensorrt_llm/models/mllama/config.py | - tensorrt_llm/models/mllama/model.py | - tensorrt_llm/models/mmdit_sd3/__init__.py | - tensorrt_llm/models/mmdit_sd3/config.py | - tensorrt_llm/models/mmdit_sd3/model.py | - tensorrt_llm/models/model_weights_loader.py | tensorrt_llm/models/modeling_utils.py | - tensorrt_llm/models/mpt/__init__.py | - tensorrt_llm/models/mpt/model.py | - tensorrt_llm/models/multimodal_encoders/__init__.py | - tensorrt_llm/models/multimodal_encoders/config.py | - tensorrt_llm/models/multimodal_encoders/model.py | - tensorrt_llm/models/nemotron_nas/__init__.py | - tensorrt_llm/models/nemotron_nas/config.py | - tensorrt_llm/models/nemotron_nas/convert.py | - tensorrt_llm/models/nemotron_nas/layer_config.py | - tensorrt_llm/models/nemotron_nas/model.py | - tensorrt_llm/models/opt/__init__.py | - tensorrt_llm/models/opt/model.py | - tensorrt_llm/models/phi/__init__.py | - tensorrt_llm/models/phi/config.py | - tensorrt_llm/models/phi/convert.py | - tensorrt_llm/models/phi/model.py | - tensorrt_llm/models/phi3/__init__.py | - tensorrt_llm/models/phi3/config.py | - tensorrt_llm/models/phi3/convert.py | - tensorrt_llm/models/phi3/model.py | - tensorrt_llm/models/phi3/split_weights.py | - tensorrt_llm/models/qwen/__init__.py | - tensorrt_llm/models/qwen/config.py | - tensorrt_llm/models/qwen/convert.py | - tensorrt_llm/models/qwen/model.py | - tensorrt_llm/models/qwen/utils.py | - tensorrt_llm/models/recurrentgemma/__init__.py | - tensorrt_llm/models/recurrentgemma/model.py | - tensorrt_llm/models/redrafter/__init__.py | - tensorrt_llm/models/redrafter/drafter.py | - tensorrt_llm/models/redrafter/model.py | - tensorrt_llm/models/redrafter/redrafter_helper.py | - tensorrt_llm/models/stdit/__init__.py | - tensorrt_llm/models/stdit/config.py | - tensorrt_llm/models/stdit/model.py | - tensorrt_llm/models/unet/__init__.py | - tensorrt_llm/models/unet/attention.py | - tensorrt_llm/models/unet/embeddings.py | - tensorrt_llm/models/unet/pp/__init__.py | - tensorrt_llm/models/unet/pp/attention.py | - tensorrt_llm/models/unet/pp/conv2d.py | - tensorrt_llm/models/unet/pp/groupnorm.py | - tensorrt_llm/models/unet/pp/unet_pp.py | - tensorrt_llm/models/unet/resnet.py | - tensorrt_llm/models/unet/unet_2d_blocks.py | - tensorrt_llm/models/unet/unet_2d_condition.py | - tensorrt_llm/models/unet/weights.py | - tensorrt_llm/network.py | - tensorrt_llm/parameter.py | - tensorrt_llm/plugin/__init__.py | - tensorrt_llm/plugin/plugin.py | tensorrt_llm/quantization/__init__.py | tensorrt_llm/quantization/functional.py | - tensorrt_llm/quantization/image_processing.py | - tensorrt_llm/quantization/layers.py | tensorrt_llm/quantization/mode.py | - tensorrt_llm/quantization/quantize.py | - tensorrt_llm/quantization/quantize_by_modelopt.py | tensorrt_llm/quantization/utils/__init__.py | tensorrt_llm/quantization/utils/fp4_utils.py | tensorrt_llm/quantization/utils/fp8_utils.py | tensorrt_llm/ray_stub.py | tensorrt_llm/runtime/__init__.py | - tensorrt_llm/runtime/enc_dec_model_runner.py | - tensorrt_llm/runtime/generation.py | - tensorrt_llm/runtime/kv_cache_manager.py | - tensorrt_llm/runtime/medusa_utils.py | tensorrt_llm/runtime/memory_pools/__init__.py | - tensorrt_llm/runtime/memory_pools/memory_pools_allocator.py | - tensorrt_llm/runtime/memory_pools/pool.py | - tensorrt_llm/runtime/memory_pools/pools_kv_cache_manager.py | - tensorrt_llm/runtime/model_runner.py | - tensorrt_llm/runtime/model_runner_cpp.py | - tensorrt_llm/runtime/multimodal_model_runner.py | - tensorrt_llm/runtime/processor_wrapper/__init__.py | - tensorrt_llm/runtime/processor_wrapper/mllama_processor_wrapper.py | - tensorrt_llm/runtime/processor_wrapper/processor_wrapper.py | - tensorrt_llm/runtime/redrafter_utils.py | - tensorrt_llm/runtime/session.py | tensorrt_llm/scaffolding/__init__.py | tensorrt_llm/scaffolding/benchmark.py | tensorrt_llm/scaffolding/contrib/AsyncGeneration/stream_generation.py | @@ -2167,12 +1250,7 @@ legacy-files: &legacy_files | tensorrt_llm/tokenizer/tokenizer.py | tensorrt_llm/tools/__init__.py | tensorrt_llm/tools/importlib_utils.py | - tensorrt_llm/tools/multimodal_builder.py | - tensorrt_llm/tools/onnx_utils.py | tensorrt_llm/tools/plugin_gen/__init__.py | - tensorrt_llm/tools/plugin_gen/core.py | - tensorrt_llm/tools/plugin_gen/plugin_gen.py | - tensorrt_llm/tools/plugin_gen/shape_infer.py | tensorrt_llm/tools/ppl.py | tensorrt_llm/tools/profiler/nsys_profile_tools/gputrc2graph.py | tensorrt_llm/version.py | @@ -2183,7 +1261,6 @@ legacy-files: &legacy_files | tests/integration/defs/accuracy/scripts/compute_theta_and_thresholds.py | tests/integration/defs/accuracy/test_cli_flow.py | tests/integration/defs/accuracy/test_disaggregated_serving.py | - tests/integration/defs/accuracy/test_llm_api.py | tests/integration/defs/accuracy/test_llm_api_autodeploy.py | tests/integration/defs/accuracy/test_llm_api_pytorch.py | tests/integration/defs/accuracy/test_llm_api_pytorch_ray.py | @@ -2192,63 +1269,29 @@ legacy-files: &legacy_files | tests/integration/defs/conftest.py | tests/integration/defs/cpp/conftest.py | tests/integration/defs/cpp/cpp_common.py | - tests/integration/defs/cpp/test_e2e.py | tests/integration/defs/cpp/test_multi_gpu.py | tests/integration/defs/cpp/test_unit_tests.py | - tests/integration/defs/deterministic/mixtral_deterministic.py | - tests/integration/defs/deterministic/test_mixtral_deterministic.py | tests/integration/defs/disaggregated/test_auto_scaling.py | tests/integration/defs/disaggregated/test_disaggregated.py | tests/integration/defs/disaggregated/test_disaggregated_etcd.py | tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py | tests/integration/defs/disaggregated/test_workers.py | - tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py | tests/integration/defs/examples/run_llm_quickstart_atexit.py | tests/integration/defs/examples/serve/test_serve.py | tests/integration/defs/examples/serve/test_serve_negative.py | tests/integration/defs/examples/test_ad_guided_decoding.py | - tests/integration/defs/examples/test_bert.py | - tests/integration/defs/examples/test_bindings.py | - tests/integration/defs/examples/test_chatglm.py | - tests/integration/defs/examples/test_commandr.py | - tests/integration/defs/examples/test_draft_target_model.py | - tests/integration/defs/examples/test_eagle.py | - tests/integration/defs/examples/test_enc_dec.py | - tests/integration/defs/examples/test_exaone.py | - tests/integration/defs/examples/test_gemma.py | tests/integration/defs/examples/test_gpt.py | - tests/integration/defs/examples/test_gptj.py | - tests/integration/defs/examples/test_granite.py | - tests/integration/defs/examples/test_internlm.py | - tests/integration/defs/examples/test_llama.py | tests/integration/defs/examples/test_llm_api_with_mpi.py | - tests/integration/defs/examples/test_mamba.py | - tests/integration/defs/examples/test_medusa.py | - tests/integration/defs/examples/test_mistral.py | - tests/integration/defs/examples/test_mixtral.py | - tests/integration/defs/examples/test_multimodal.py | - tests/integration/defs/examples/test_nemotron.py | - tests/integration/defs/examples/test_nemotron_nas.py | - tests/integration/defs/examples/test_ngram.py | - tests/integration/defs/examples/test_openai.py | tests/integration/defs/examples/test_phi.py | - tests/integration/defs/examples/test_qwen.py | - tests/integration/defs/examples/test_qwen2audio.py | - tests/integration/defs/examples/test_qwenvl.py | tests/integration/defs/examples/test_ray.py | - tests/integration/defs/examples/test_recurrentgemma.py | - tests/integration/defs/examples/test_redrafter.py | - tests/integration/defs/examples/test_whisper.py | tests/integration/defs/llmapi/__init__.py | tests/integration/defs/llmapi/_run_llmapi_llm.py | tests/integration/defs/llmapi/test_llm_api_connector.py | tests/integration/defs/llmapi/test_llm_api_qa.py | - tests/integration/defs/llmapi/test_llm_e2e.py | tests/integration/defs/llmapi/test_llm_examples.py | tests/integration/defs/local_venv.py | tests/integration/defs/perf/__init__.py | tests/integration/defs/perf/allowed_configs.py | - tests/integration/defs/perf/build.py | tests/integration/defs/perf/create_perf_comparison_report.py | tests/integration/defs/perf/data.py | tests/integration/defs/perf/data_export.py | @@ -2269,38 +1312,21 @@ legacy-files: &legacy_files | tests/integration/defs/test_fmha.py | tests/integration/defs/test_list_parser.py | tests/integration/defs/test_list_validation.py | - tests/integration/defs/test_mlpf_results.py | tests/integration/defs/test_sanity.py | tests/integration/defs/test_unittests.py | tests/integration/defs/triton_server/__init__.py | - tests/integration/defs/triton_server/build_engines.py | tests/integration/defs/triton_server/common.py | tests/integration/defs/triton_server/conftest.py | - tests/integration/defs/triton_server/local_venv.py | - tests/integration/defs/triton_server/rcca/bug_4323566/inflight_batcher_llm_client_with_end_id.py | - tests/integration/defs/triton_server/runner_interface.py | tests/integration/defs/triton_server/test_list_parser.py | - tests/integration/defs/triton_server/test_triton.py | - tests/integration/defs/triton_server/test_triton_llm.py | - tests/integration/defs/triton_server/test_triton_memleak.py | - tests/integration/defs/triton_server/test_triton_multi_node.py | - tests/integration/defs/triton_server/test_triton_rcca.py | tests/integration/defs/triton_server/trt_test_alternative.py | tests/integration/defs/trt_test_alternative.py | tests/integration/defs/utils/__init__.py | tests/integration/defs/utils/periodic_junit.py | tests/integration/defs/utils/timeout_manager.py | tests/microbenchmarks/all_reduce.py | - tests/microbenchmarks/build_time_benchmark.py | - tests/microbenchmarks/build_time_dashboard.py | tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py | tests/scripts/allreduce_perf/allreduce_perf_viz.py | tests/scripts/iteration_log_parser.py | - tests/scripts/perf-sanity/parse_benchmark_results.py | - tests/scripts/perf-sanity/run_benchmark_serve.py | - tests/unittest/_torch/attention/sparse/test_dsa_indexer.py | - tests/unittest/_torch/attention/sparse/test_flash_mla.py | - tests/unittest/_torch/attention/sparse/test_rocketkv.py | tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py | tests/unittest/_torch/attention/test_attention.py | tests/unittest/_torch/attention/test_attention_mla.py | @@ -2321,7 +1347,6 @@ legacy-files: &legacy_files | tests/unittest/_torch/misc/test_virtual_memory.py | tests/unittest/_torch/modeling/test_modeling_bert.py | tests/unittest/_torch/modeling/test_modeling_clip.py | - tests/unittest/_torch/modeling/test_modeling_exaone4.py | tests/unittest/_torch/modeling/test_modeling_gemma3.py | tests/unittest/_torch/modeling/test_modeling_gpt_oss.py | tests/unittest/_torch/modeling/test_modeling_llama.py | @@ -2345,8 +1370,6 @@ legacy-files: &legacy_files | tests/unittest/_torch/modules/test_moe_routing.py | tests/unittest/_torch/modules/test_rotary_embedding.py | tests/unittest/_torch/modules/test_triton_linear.py | - tests/unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py | - tests/unittest/_torch/modules/tests_lora_modules/test_lora_plugin_vs_lora_op.py | tests/unittest/_torch/multi_gpu/test_allreduce.py | tests/unittest/_torch/multi_gpu/test_alltoall.py | tests/unittest/_torch/multi_gpu/test_ar_residual_norm.py | @@ -2373,24 +1396,10 @@ legacy-files: &legacy_files | tests/unittest/_torch/sampler/test_beam_search.py | tests/unittest/_torch/sampler/test_best_of_n.py | tests/unittest/_torch/sampler/test_trtllm_sampler.py | - tests/unittest/_torch/speculative/test_draft_target.py | - tests/unittest/_torch/speculative/test_draft_token_tree_sampling.py | - tests/unittest/_torch/speculative/test_draft_token_tree_verification.py | - tests/unittest/_torch/speculative/test_dynamic_spec_decode.py | tests/unittest/_torch/speculative/test_eagle3.py | - tests/unittest/_torch/speculative/test_kv_cache_reuse.py | - tests/unittest/_torch/speculative/test_mtp.py | - tests/unittest/_torch/speculative/test_ngram.py | - tests/unittest/_torch/speculative/test_save_state.py | - tests/unittest/_torch/speculative/test_spec_gate.py | - tests/unittest/_torch/speculative/test_torch_rejection_sampling.py | - tests/unittest/_torch/speculative/test_user_provided.py | tests/unittest/_torch/test_connector.py | tests/unittest/_torch/test_torch_multi_arange.py | tests/unittest/_torch/thop/parallel/deep_gemm_tests.py | - tests/unittest/_torch/thop/parallel/test_causal_conv1d_op.py | - tests/unittest/_torch/thop/parallel/test_cublas_mm.py | - tests/unittest/_torch/thop/parallel/test_custom_ops.py | tests/unittest/_torch/thop/parallel/test_dsv3_fused_a_gemm.py | tests/unittest/_torch/thop/parallel/test_dsv3_router_gemm.py | tests/unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py | @@ -2404,11 +1413,6 @@ legacy-files: &legacy_files | tests/unittest/_torch/thop/parallel/test_fp8_per_tensor_scale_tllmg_gemm.py | tests/unittest/_torch/thop/parallel/test_fp8_quantize.py | tests/unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py | - tests/unittest/_torch/thop/parallel/test_fused_qk_norm_rope.py | - tests/unittest/_torch/thop/parallel/test_logits_bitmask_op.py | - tests/unittest/_torch/thop/parallel/test_mamba2_chunk_ss_update.py | - tests/unittest/_torch/thop/parallel/test_mamba_conv1d_op.py | - tests/unittest/_torch/thop/parallel/test_noaux_tc.py | tests/unittest/_torch/thop/parallel/test_scaled_mm.py | tests/unittest/_torch/thop/parallel/test_selective_scan_op.py | tests/unittest/_torch/thop/parallel/test_tinygemm2.py | @@ -2422,7 +1426,6 @@ legacy-files: &legacy_files | tests/unittest/_torch/thop/serial/test_moe_alltoall.py | tests/unittest/api_stability/api_stability_core.py | tests/unittest/api_stability/test_llm_api.py | - tests/unittest/bindings/binding_test_utils.py | tests/unittest/bindings/test_bindings_moe.py | tests/unittest/bindings/test_bindings_ut.py | tests/unittest/bindings/test_executor_bindings.py | @@ -2453,12 +1456,10 @@ legacy-files: &legacy_files | tests/unittest/llmapi/apps/_test_openai_chat_harmony.py | tests/unittest/llmapi/apps/_test_openai_chat_multimodal.py | tests/unittest/llmapi/apps/_test_openai_completions.py | - tests/unittest/llmapi/apps/_test_openai_consistent_chat.py | tests/unittest/llmapi/apps/_test_openai_lora.py | tests/unittest/llmapi/apps/_test_openai_metrics.py | tests/unittest/llmapi/apps/_test_openai_misc.py | tests/unittest/llmapi/apps/_test_openai_mmencoder.py | - tests/unittest/llmapi/apps/_test_openai_multi_chat.py | tests/unittest/llmapi/apps/_test_openai_multi_gpu.py | tests/unittest/llmapi/apps/_test_openai_multi_nodes.py | tests/unittest/llmapi/apps/_test_openai_perf_metrics.py | @@ -2481,15 +1482,12 @@ legacy-files: &legacy_files | tests/unittest/llmapi/run_llm_exit.py | tests/unittest/llmapi/run_llm_with_postproc.py | tests/unittest/llmapi/test_additional_model_outputs.py | - tests/unittest/llmapi/test_build_cache.py | tests/unittest/llmapi/test_executor.py | tests/unittest/llmapi/test_gc_utils.py | tests/unittest/llmapi/test_llm.py | tests/unittest/llmapi/test_llm_args.py | tests/unittest/llmapi/test_llm_download.py | tests/unittest/llmapi/test_llm_kv_cache_events.py | - tests/unittest/llmapi/test_llm_models.py | - tests/unittest/llmapi/test_llm_multi_gpu.py | tests/unittest/llmapi/test_llm_multi_gpu_pytorch.py | tests/unittest/llmapi/test_llm_pytorch.py | tests/unittest/llmapi/test_llm_quant.py | @@ -2500,25 +1498,14 @@ legacy-files: &legacy_files | tests/unittest/llmapi/test_serialization.py | tests/unittest/llmapi/test_utils.py | tests/unittest/others/__init__.py | - tests/unittest/others/test_builder.py | tests/unittest/others/test_convert_spec_decoding_mask_to_packed_mask.py | - tests/unittest/others/test_debugging_api.py | tests/unittest/others/test_exception.py | tests/unittest/others/test_export.py | - tests/unittest/others/test_graph_rewriter.py | - tests/unittest/others/test_kv_cache_manager.py | tests/unittest/others/test_kv_cache_transceiver.py | tests/unittest/others/test_kv_cache_update.py | - tests/unittest/others/test_layer.py | - tests/unittest/others/test_leak.py | tests/unittest/others/test_mapping.py | - tests/unittest/others/test_model_dtype.py | - tests/unittest/others/test_module.py | tests/unittest/others/test_multimodal_registry.py | - tests/unittest/others/test_plugins.py | - tests/unittest/others/test_precision_control.py | tests/unittest/others/test_pretrained_config.py | - tests/unittest/others/test_session.py | tests/unittest/others/test_time_breakdown.py | tests/unittest/profile_utils.py | tests/unittest/scaffolding/__init__.py | @@ -2527,141 +1514,14 @@ legacy-files: &legacy_files | tests/unittest/scaffolding/test_scaffolding.py | tests/unittest/scaffolding/test_task_collection.py | tests/unittest/scaffolding/test_worker.py | - tests/unittest/test_model_runner_cpp.py | tests/unittest/test_pip_install.py | tests/unittest/tools/__init__.py | - tests/unittest/tools/plugin_gen/__init__.py | - tests/unittest/tools/plugin_gen/kernel_config.py | - tests/unittest/tools/plugin_gen/test_core.py | - tests/unittest/tools/plugin_gen/test_plugin_gen.py | - tests/unittest/tools/plugin_gen/test_shape_infer.py | tests/unittest/tools/test_prepare_dataset.py | tests/unittest/tools/test_test_to_stage_mapping.py | - tests/unittest/trt/__init__.py | - tests/unittest/trt/attention/test_bert_attention.py | - tests/unittest/trt/attention/test_gpt_attention.py | - tests/unittest/trt/attention/test_gpt_attention_IFB.py | - tests/unittest/trt/attention/test_gpt_attention_no_cache.py | - tests/unittest/trt/attention/test_sage_attention.py | - tests/unittest/trt/functional/__init__.py | - tests/unittest/trt/functional/test_alibi.py | - tests/unittest/trt/functional/test_allreduce_norm.py | - tests/unittest/trt/functional/test_allreduce_prepost_residual_norm.py | - tests/unittest/trt/functional/test_arange.py | - tests/unittest/trt/functional/test_argmax.py | - tests/unittest/trt/functional/test_assertion.py | - tests/unittest/trt/functional/test_avg_pool2d.py | - tests/unittest/trt/functional/test_cast.py | - tests/unittest/trt/functional/test_conv2d.py | - tests/unittest/trt/functional/test_conv3d.py | - tests/unittest/trt/functional/test_cos.py | - tests/unittest/trt/functional/test_cumsum.py | - tests/unittest/trt/functional/test_dora.py | - tests/unittest/trt/functional/test_einsum.py | - tests/unittest/trt/functional/test_embedding_single_gpu.py | - tests/unittest/trt/functional/test_exp.py | - tests/unittest/trt/functional/test_expand.py | - tests/unittest/trt/functional/test_flatten.py | - tests/unittest/trt/functional/test_flip.py | - tests/unittest/trt/functional/test_fp4_gemm.py | - tests/unittest/trt/functional/test_fp4_gemm_ootb.py | - tests/unittest/trt/functional/test_gather.py | - tests/unittest/trt/functional/test_gather_nd.py | - tests/unittest/trt/functional/test_geglu.py | - tests/unittest/trt/functional/test_gelu.py | - tests/unittest/trt/functional/test_gemm_swiglu.py | - tests/unittest/trt/functional/test_group_norm.py | - tests/unittest/trt/functional/test_identity.py | - tests/unittest/trt/functional/test_index_select.py | - tests/unittest/trt/functional/test_interpolate.py | - tests/unittest/trt/functional/test_logsoftmax.py | - tests/unittest/trt/functional/test_lora.py | - tests/unittest/trt/functional/test_low_latency_gemm.py | - tests/unittest/trt/functional/test_mamba_conv1d.py | - tests/unittest/trt/functional/test_masked_scatter.py | - tests/unittest/trt/functional/test_masked_select.py | - tests/unittest/trt/functional/test_matmul.py | - tests/unittest/trt/functional/test_meshgrid2d.py | - tests/unittest/trt/functional/test_moe.py | - tests/unittest/trt/functional/test_nccl.py | - tests/unittest/trt/functional/test_nonzero.py | - tests/unittest/trt/functional/test_outer.py | - tests/unittest/trt/functional/test_pad.py | - tests/unittest/trt/functional/test_permute.py | - tests/unittest/trt/functional/test_pp_reduce_scatter.py | - tests/unittest/trt/functional/test_quant.py | - tests/unittest/trt/functional/test_rearrange.py | - tests/unittest/trt/functional/test_repeat.py | - tests/unittest/trt/functional/test_repeat_interleave.py | - tests/unittest/trt/functional/test_rg_lru.py | - tests/unittest/trt/functional/test_sample.py | - tests/unittest/trt/functional/test_scatter.py | - tests/unittest/trt/functional/test_scatter_nd.py | - tests/unittest/trt/functional/test_select.py | - tests/unittest/trt/functional/test_selective_scan.py | - tests/unittest/trt/functional/test_sigmoid.py | - tests/unittest/trt/functional/test_silu.py | - tests/unittest/trt/functional/test_sin.py | - tests/unittest/trt/functional/test_slice.py | - tests/unittest/trt/functional/test_softplus.py | - tests/unittest/trt/functional/test_split.py | - tests/unittest/trt/functional/test_squeeze.py | - tests/unittest/trt/functional/test_swiglu.py | - tests/unittest/trt/functional/test_topk.py | - tests/unittest/trt/functional/test_transpose.py | - tests/unittest/trt/functional/test_unbind.py | - tests/unittest/trt/functional/test_unsqueeze.py | - tests/unittest/trt/functional/test_view.py | - tests/unittest/trt/functional/test_where.py | - tests/unittest/trt/model/__init__.py | - tests/unittest/trt/model/eagle/test_decode_draft_tokens_plugin.py | - tests/unittest/trt/model/eagle/test_prepare_drafter_inputs_plugin.py | - tests/unittest/trt/model/eagle/test_sample_accept_draft_tokens_plugin.py | - tests/unittest/trt/model/redrafter/test_beams2tree.py | - tests/unittest/trt/model/redrafter/test_draft_token.py | - tests/unittest/trt/model/redrafter/test_draft_token_indices.py | - tests/unittest/trt/model/redrafter/test_gather_beams.py | - tests/unittest/trt/model/redrafter/test_mask.py | - tests/unittest/trt/model/redrafter/test_packed_position_ids.py | - tests/unittest/trt/model/redrafter/test_prefix_match_indices.py | - tests/unittest/trt/model/redrafter/test_prepare_input.py | - tests/unittest/trt/model/redrafter/test_process_logits.py | - tests/unittest/trt/model/redrafter/test_top1.py | - tests/unittest/trt/model/redrafter/test_unpack_gen_data.py | - tests/unittest/trt/model/redrafter/test_validate.py | - tests/unittest/trt/model/test_gpt.py | - tests/unittest/trt/model/test_gpt_e2e.py | - tests/unittest/trt/model/test_llama.py | - tests/unittest/trt/model/test_mamba.py | - tests/unittest/trt/model/test_mistral.py | - tests/unittest/trt/model/test_nemotron_nas.py | - tests/unittest/trt/model/test_phi.py | - tests/unittest/trt/model/test_unet.py | - tests/unittest/trt/model_api/test_model_api_multi_gpu.py | - tests/unittest/trt/model_api/test_model_level_api.py | - tests/unittest/trt/model_api/test_model_quantization.py | - tests/unittest/trt/python_plugin/plugin_wrapper_utils.py | - tests/unittest/trt/python_plugin/test_plugin_wrapper.py | - tests/unittest/trt/quantization/__init__.py | - tests/unittest/trt/quantization/_utils.py | - tests/unittest/trt/quantization/test_fp8_quantization.py | - tests/unittest/trt/quantization/test_fp8_rowwise_gemm.py | - tests/unittest/trt/quantization/test_functional.py | - tests/unittest/trt/quantization/test_mode.py | - tests/unittest/trt/quantization/test_moe_weight_only_quant_matmul.py | - tests/unittest/trt/quantization/test_qserve_gemm.py | - tests/unittest/trt/quantization/test_quant.py | - tests/unittest/trt/quantization/test_quant_layer.py | - tests/unittest/trt/quantization/test_smooth_quant_gemm.py | - tests/unittest/trt/quantization/test_smooth_quant_layer_norm.py | - tests/unittest/trt/quantization/test_smooth_quant_rms_norm.py | - tests/unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py | - tests/unittest/trt/quantization/test_weight_only_quant_matmul.py | tests/unittest/utils/__init__.py | tests/unittest/utils/cpp_paths.py | tests/unittest/utils/llm_data.py | tests/unittest/utils/runtime_defaults.py | - tests/unittest/utils/test_medusa_utils.py | tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py | tests/unittest/utils/test_util.py | tests/unittest/utils/torch_ref.py | @@ -2689,17 +1549,19 @@ legacy-files: &legacy_files | static-analysis-files: &static_analysis_files | (?x)^( tensorrt_llm/_torch/pyexecutor/sampler/sampler.py | + tensorrt_llm/_torch/pyexecutor/sampler/token_ban.py | tensorrt_llm/_torch/pyexecutor/sampler/sampling_utils.py | tensorrt_llm/_torch/pyexecutor/sampler/ops/interface.py | tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py | tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py | tests/unittest/_torch/sampler/test_torch_sampler.py | + tests/unittest/_torch/sampler/test_token_ban.py | tests/unittest/_torch/sampler/test_beam_search.py | tests/unittest/_torch/sampler/test_beam_search_util.py | )$ # Global exclude: vendored code + trtllm-gen FMHA artifacts (cubin pointers, export headers, cuda_ptx) -exclude: '(^triton_kernels/|trtllmGenKernels/fmha/cubin/kernelMetaInfo\.h$|cubin\.cpp$|cubin\.h$|trtllmGenKernels/fmha/trtllmGen_fmha_export/|trtllmGenKernels/fmha/cuda_ptx/|trtllmGenKernels/batchedGemm/trtllmGen_bmm_export/KernelMetaInfo\.h$)' +exclude: '(^cpp/tensorrt_llm/common/sha256/|^triton_kernels/|trtllmGenKernels/fmha/cubin/kernelMetaInfo\.h$|cubin\.cpp$|cubin\.h$|trtllmGenKernels/fmha/trtllmGen_fmha_export/|trtllmGenKernels/fmha/cuda_ptx/|trtllmGenKernels/batchedGemm/trtllmGen_bmm_export/KernelMetaInfo\.h$)' default_install_hook_types: [pre-commit, commit-msg] repos: diff --git a/3rdparty/MSA b/3rdparty/MSA new file mode 160000 index 000000000000..e2ebe7656649 --- /dev/null +++ b/3rdparty/MSA @@ -0,0 +1 @@ +Subproject commit e2ebe7656649f619af0ad1d457b534283034655e diff --git a/3rdparty/fetch_content.json b/3rdparty/fetch_content.json index d86ebb57fc46..31820f223a18 100644 --- a/3rdparty/fetch_content.json +++ b/3rdparty/fetch_content.json @@ -32,7 +32,7 @@ { "name": "deepgemm", "git_repository": "https://github.com/deepseek-ai/DeepGEMM", - "git_tag": "245dc5d6a5fe344c61505fe71011d203141d4479", + "git_tag": "f8e8fb5830fa5cda6e4ea73d360bb3f21f87a3ca", "git_submodules_recurse": true, "source_subdir": "dont-add-this-project-with-add-subdirectory" }, diff --git a/AGENTS.md b/AGENTS.md index 20fa71f70f7f..cf6c717d0928 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # AGENTS.md TensorRT-LLM: open-source library for optimized LLM inference on NVIDIA GPUs. -Python and C++ codebase supporting TensorRT engine-based and PyTorch-based execution paths. +Python and C++ codebase with PyTorch and AutoDeploy execution paths. > If a `CLAUDE.local.md` file exists alongside this file, read and respect it — it contains developer-specific overrides that supplement this shared guidance. @@ -55,17 +55,16 @@ See [architecture diagram](.github/tava_architecture_diagram.md) for the full Me |---------|--------|-------------|----------| | **PyTorch** | Default | `TorchLlmArgs` | `_torch/pyexecutor/` → `PyExecutor` → PyTorch Engine | | **AutoDeploy** | Beta | `_torch/auto_deploy/` shim | `_torch/auto_deploy/shim/ad_executor.py` → adapts `PyExecutor` → graph transforms + torch.export | -| **TensorRT** | Legacy | `TrtLlmArgs` | `builder.py` → `trtllm.Executor` → TensorRT Engine | ### Shared C++ Core (via Nanobind) -Both PyTorch and TensorRT backends share these C++ components: +Both backends share these C++ components: - **Scheduling pipeline**: Scheduler → BatchManager (in-flight batching) → KV Cache Manager - **Decoding pipeline**: Decoder (token generation orchestration) → Sampling ### Request Flow ```text -HuggingFace Model → LLM API → Executor (PyTorch/AutoDeploy/TensorRT) +HuggingFace Model → LLM API → Executor (PyTorch/AutoDeploy) → Scheduler → Model Forward → Decoder → Sampling → Generated Tokens ``` @@ -84,7 +83,7 @@ HuggingFace Model → LLM API → Executor (PyTorch/AutoDeploy/TensorRT) | `tensorrt_llm/models/modeling_utils.py` | Base classes for all models (`PretrainedConfig`, `PretrainedModel`) | | `tensorrt_llm/executor/executor.py` | Execution abstraction (`GenerationExecutor`) | | `tensorrt_llm/models/automodel.py` | Auto-discovery and model registry | -| `tensorrt_llm/_torch/models/` | PyTorch backend model implementations (distinct from `models/` used by TensorRT backend) | +| `tensorrt_llm/_torch/models/` | PyTorch backend model implementations (distinct from the top-level `models/` package) | | `tensorrt_llm/_torch/modules/ATTENTION_DEVELOPER_GUIDE.md` | Attention, MLA, backend families, sparse backends, metadata contracts, and KV-cache behavior - **read before modifying `tensorrt_llm/_torch/modules/attention.py`, `tensorrt_llm/_torch/modules/mla.py`, or `tensorrt_llm/_torch/attention_backend/`** | | `tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md` | MoE architecture, backends, communication, development patterns — **read before modifying MoE code** | | `CODING_GUIDELINES.md` | C++ and Python coding standards (referenced throughout, must read before contributing) | @@ -93,7 +92,7 @@ HuggingFace Model → LLM API → Executor (PyTorch/AutoDeploy/TensorRT) | Pattern | Key Points | |---------|------------| -| **Config hierarchy** | `BaseLlmArgs` → `TrtLlmArgs` / `TorchLlmArgs`, model-specific defaults override generics, Pydantic validation | +| **Config hierarchy** | `BaseLlmArgs` → `TorchLlmArgs`, model-specific defaults override generics, Pydantic validation | | **Model architecture** | Each model: `Config` (inherits `PretrainedConfig`) + `ForCausalLM` (inherits `PretrainedModel`) | | **Model defaults** | Architecture-specific overrides in `llm_utils.py` (attention kernels, quant, spec decoding, cache) | | **Attention backends** | `TorchLlmArgs.attn_backend` selects kernel: `TRTLLM` (default), `FlashInfer`, `FlashAttention` | @@ -125,7 +124,6 @@ Key files: - **Avoid broad exception handling** — catch specific exceptions, not bare `except:` (see `CODING_GUIDELINES.md`). - **One concern per PR** — avoid scope creep. If a PR touches unrelated areas, split it. - **User-facing configuration classes** - when editing or defining any user-facing configuration classes (particularly `BaseLlmArgs` or any class used in its fields), you **MUST** follow the Pydantic guidelines in `CODING_GUIDELINES.md`. -- **TensorRT backend is legacy** — `TrtLlmArgs` / `backend="tensorrt"` and all exclusive tooling (`trtllm-build`, `trtllm-refit`, `convert_checkpoint.py`, `ModelRunner*`) are legacy. Bug fixes OK; new features target PyTorch or AutoDeploy. ## Development Workflow diff --git a/ATTRIBUTIONS-Python.md b/ATTRIBUTIONS-Python.md index e43a78cafda0..7e74b846a85f 100644 --- a/ATTRIBUTIONS-Python.md +++ b/ATTRIBUTIONS-Python.md @@ -5261,7 +5261,7 @@ For more information, please refer to <http://unlicense.org> - `Tracker`: https://github.com/tox-dev/py-filelock/issues -## flashinfer-python (0.6.14) +## flashinfer-python (0.6.15) ### Licenses License: `Apache-2.0` diff --git a/LICENSE b/LICENSE index 8ba867f30567..bf9a933a6697 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2011-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +Copyright (c) 2011-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. This project is licensed under the Apache 2.0 license, whose full license text is available below. @@ -12,6 +12,14 @@ file headers for specific copyright and license information. Below is a list of other projects that have portions contained by this project: +-------------------------------------------------------------------------------- +Bitcoin Core +-------------------------------------------------------------------------------- +Original Source: https://github.com/bitcoin/bitcoin +Copyright (c) 2009-2026 The Bitcoin Core developers +Copyright (c) 2009-2026 Bitcoin Developers +Licensed under the MIT License + -------------------------------------------------------------------------------- causal-conv1d -------------------------------------------------------------------------------- @@ -19,6 +27,13 @@ Original Source: https://github.com/Dao-AILab/causal-conv1d Copyright (c) 2024, Tri Dao. Licensed under the BSD 3-Clause License +-------------------------------------------------------------------------------- +CUTLASS +-------------------------------------------------------------------------------- +Original Source: https://github.com/NVIDIA/cutlass +Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +Licensed under the BSD 3-Clause License + -------------------------------------------------------------------------------- flash-attention -------------------------------------------------------------------------------- @@ -34,6 +49,14 @@ Original Source: https://github.com/fla-org/flash-linear-attention Copyright (c) 2023-2025 Songlin Yang Licensed under the MIT License +-------------------------------------------------------------------------------- +FlashInfer +-------------------------------------------------------------------------------- +Original Source: https://github.com/flashinfer-ai/flashinfer +Copyright 2025-2026 NVIDIA +Copyright 2023-2026 FlashInfer community (https://flashinfer.ai/) +Licensed under the Apache License 2.0 + -------------------------------------------------------------------------------- InstructEval -------------------------------------------------------------------------------- @@ -59,6 +82,14 @@ Original Source: https://github.com/state-spaces/mamba Copyright 2023 Tri Dao, Albert Gu Licensed under the Apache License 2.0 +-------------------------------------------------------------------------------- +MSA (MiniMax Sparse Attention) +-------------------------------------------------------------------------------- +Original Source: https://github.com/MiniMax-AI/MSA +Copyright (c) 2026 MiniMax +Licensed under the MIT License + + -------------------------------------------------------------------------------- Quack -------------------------------------------------------------------------------- diff --git a/README.md b/README.md index 943083d6c4c6..6c5f86b73102 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ TensorRT LLM [![python](https://img.shields.io/badge/python-3.10-green)](https://www.python.org/downloads/release/python-31012/) [![cuda](https://img.shields.io/badge/cuda-13.2.1-green)](https://developer.nvidia.com/cuda-downloads) [![torch](https://img.shields.io/badge/torch-2.11.0-green)](https://pytorch.org) -[![version](https://img.shields.io/badge/release-1.3.0rc21-green)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/version.py) +[![version](https://img.shields.io/badge/release-1.3.0rc24-green)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/version.py) [![license](https://img.shields.io/badge/license-Apache%202-blue)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/LICENSE) [Architecture](https://nvidia.github.io/TensorRT-LLM/developer-guide/overview.html)   |   [Performance](https://nvidia.github.io/TensorRT-LLM/developer-guide/perf-overview.html)   |   [Examples](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html)   |   [Documentation](https://nvidia.github.io/TensorRT-LLM/)   |   [Roadmap](https://github.com/NVIDIA/TensorRT-LLM/issues?q=is%3Aissue%20state%3Aopen%20label%3Aroadmap) @@ -22,6 +22,9 @@ TensorRT LLM <!-- Use github markdown link to link for the latest blog since the doc build has not happened yet. When the doc build is updated, it should be updated to the webpage link. --> +* [07/17] DeepSeek-V4 on NVIDIA Blackwell: Model-Specific and Agentic-Workload Optimizations in TensorRT LLM +✨ [➡️ link](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md) + * [07/01] Scaling Video Generation Across NVL72 Rack with TensorRT-LLM ✨ [➡️ link](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog25_Scaling_Video_Generation_Across_NVL72_Rack_with_TensorRT-LLM.md) diff --git a/benchmarks/README.md b/benchmarks/README.md deleted file mode 100644 index 5d89f412ac5c..000000000000 --- a/benchmarks/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# TensorRT-LLM Benchmarks - -## Overview - -There are currently two workflows to benchmark TensorRT-LLM: -* [`trtllm-bench`](../docs/source/developer-guide/perf-benchmarking.md) - - `trtllm-bench` is native to TensorRT-LLM and is a Python benchmarker for reproducing and testing the performance of TensorRT-LLM. - - _NOTE_: This benchmarking suite is a current work in progress and is prone to large changes. -* [C++ benchmarks](./cpp) - - The recommended workflow that uses TensorRT-LLM C++ API and can take advantage of the latest features of TensorRT-LLM. diff --git a/benchmarks/cpp/CMakeLists.txt b/benchmarks/cpp/CMakeLists.txt deleted file mode 100644 index cb5ef1ee928b..000000000000 --- a/benchmarks/cpp/CMakeLists.txt +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -include_directories(${PROJECT_SOURCE_DIR}/include) - -set(TOP_LEVEL_DIR "${PROJECT_SOURCE_DIR}/..") - -add_custom_target(benchmarks) - -if(NOT TARGET cxxopts::cxxopts) - add_subdirectory(${CMAKE_BINARY_DIR}/_deps/cxxopts-src - ${CMAKE_CURRENT_BINARY_DIR}/cxxopts) -endif() - -function(add_benchmark test_name test_src) - add_executable(${test_name} ${test_src} utils/utils.cpp) - - target_link_libraries( - ${test_name} PUBLIC ${SHARED_TARGET} nvinfer_plugin_tensorrt_llm - cxxopts::cxxopts) - - target_compile_features(${test_name} PRIVATE cxx_std_17) - target_compile_definitions(${test_name} - PUBLIC TOP_LEVEL_DIR="${TOP_LEVEL_DIR}") - add_dependencies(benchmarks ${test_name}) -endfunction() - -add_benchmark(bertBenchmark bertBenchmark.cpp) -add_benchmark(gptManagerBenchmark gptManagerBenchmark.cpp) -add_benchmark(disaggServerBenchmark disaggServerBenchmark.cpp) diff --git a/benchmarks/cpp/README.md b/benchmarks/cpp/README.md deleted file mode 100644 index ae3287faf06c..000000000000 --- a/benchmarks/cpp/README.md +++ /dev/null @@ -1,367 +0,0 @@ -# Benchmark C++ Runtime - -This document explains how to benchmark the models supported by TensorRT-LLM on a single GPU, a single node with -multiple GPUs or multiple nodes with multiple GPUs using the C++ runtime. - -## Usage - -### 1. Build TensorRT-LLM and benchmarking source code - -Please follow the [`installation document`](../../README.md#installation) to build TensorRT-LLM. - -Note that the benchmarking source code for C++ runtime is not built by default, you can use the argument `--benchmarks` in [`build_wheel.py`](source:scripts/build_wheel.py) to build the corresponding executable. - -### 2. Launch C++ benchmarking (Inflight/V1 batching) - -#### Prepare dataset - -Run a preprocessing script to prepare/generate dataset into a json that `gptManagerBenchmark` can consume later. The processed output json has *input tokens length, input token ids and output tokens length*. - -For `tokenizer`, specifying the path to the local tokenizer that have already been downloaded, or simply the name of the tokenizer from HuggingFace like `meta-llama/Llama-2-7b` will both work. The tokenizer will be downloaded automatically for the latter case. - -This tool can be used in 3 different modes of traffic generation: `dataset`, `token-norm-dist` and `token-unif-dist`. - -##### 1 – Dataset - -The tool will tokenize the words and instruct the model to generate a specified number of output tokens for a request. - -``` -python3 prepare_dataset.py \ - --tokenizer <path/to/tokenizer> \ - --output preprocessed_dataset.json - dataset - --dataset-name <name of the dataset> \ - --dataset-split <split of the dataset to use> \ - --dataset-input-key <dataset dictionary key for input> \ - --dataset-prompt-key <dataset dictionary key for prompt> \ - --dataset-output-key <dataset dictionary key for output> \ - [--num-requests 100] \ - [--max-input-len 1000] \ - [--output-len-dist 100,10] -``` - -For datasets that don't have prompt key, set --dataset-prompt instead. -Take [cnn_dailymail dataset](https://huggingface.co/datasets/abisee/cnn_dailymail) for example: -``` -python3 prepare_dataset.py \ - --tokenizer <path/to/tokenizer> \ - --output cnn_dailymail.json - dataset - --dataset-name cnn_dailymail \ - --dataset-split validation \ - --dataset-config-name 3.0.0 \ - --dataset-input-key article \ - --dataset-prompt "Summarize the following article:" \ - --dataset-output-key "highlights" \ - [--num-requests 100] \ - [--max-input-len 1000] \ - [--output-len-dist 100,10] -``` - -##### 2 – Normal token length distribution - -This mode allows the user to generate normally distributed token lengths with a mean and std deviation specified. -For example, setting `mean=100` and `stdev=10` would generate requests where 95.4% of values are in <80,120> range following the normal probability distribution. Setting `stdev=0` will generate all requests with the same mean number of tokens. - -``` -python prepare_dataset.py \ - --output token-norm-dist.json \ - --tokenizer <path/to/tokenizer> \ - token-norm-dist \ - --num-requests 100 \ - --input-mean 100 --input-stdev 10 \ - --output-mean 15 --output-stdev 0 -``` - -##### 2 – Uniform token length distribution - -This mode allows the user to generate uniformly distributed token lengths with min and max lengths specified. -For example, setting `min=50` and `max=100` would generate requests where lengths are in the range `[50, 100]` following the uniform probability distribution. Setting `min=x` and `max=x` will generate all requests with the same mean number of tokens `x`. - -``` -python prepare_dataset.py \ - --output token-norm-dist.json \ - --tokenizer <path/to/tokenizer> \ - token-unif-dist \ - --num-requests 100 \ - --input-min 50 --input-max 100 \ - --output-min 10 --output-max 15 -``` - - -#### Prepare TensorRT-LLM engines - -Before you launch C++ benchmarking, please make sure that you have already built engine(s) using `trtllm-build` command. For more details on building engine(s), please refer to the [Quick Start Guide](../../docs/source/quick-start-guide.md). - -#### Launch benchmarking - -For detailed usage, you can do the following -``` -cd cpp/build - -# You can directly execute the binary for help information -./benchmarks/gptManagerBenchmark --help -``` - -`gptManagerBenchmark` now supports decoder-only models and encoder-decoder models. - -1. Decoder-only Models - - To benchmark decoder-only models, pass in the engine path with `--engine_dir` as executable input argument. - - Take GPT-350M as an example for 2-GPU inflight batching - ``` - mpirun -n 2 ./benchmarks/gptManagerBenchmark \ - --engine_dir ../../examples/models/core/gpt/trt_engine/gpt2-ib/fp16/2-gpu/ \ - --request_rate 10 \ - --dataset ../../benchmarks/cpp/preprocessed_dataset.json \ - --max_num_samples 500 - ``` - - `gptManagerBenchmark` by default uses the high-level C++ API defined by the `executor::Executor` class (see `cpp/include/tensorrt_llm/executor/executor.h`). - -2. Encoder-Decoder Models - To benchmark encoder-decoder models, pass in the encoder engine path with `--encoder_engine_dir` and the decoder engine path with `--decoder_engine_dir` as executable input arguments. `--decoder_engine_dir` is an alias of `--engine_dir`. - - Currently encoder-decoder engines only support `--api executor`, `--type IFB`, `--enable_kv_cache_reuse false`, which are all default values so no specific settings required. - - Prepare t5-small engine from [examples/models/core/enc_dec](/examples/models/core/enc_dec/README.md#convert-and-split-weights) for the encoder-decoder 4-GPU inflight batching example. - - Prepare the dataset suitable for engine input lengths. - ``` - python prepare_dataset.py \ - --tokenizer <path/to/tokenizer> \ - --output cnn_dailymail.json \ - dataset \ - --dataset-name cnn_dailymail \ - --dataset-split validation \ - --dataset-config-name 3.0.0 \ - --dataset-input-key article \ - --dataset-prompt "Summarize the following article:" \ - --dataset-output-key "highlights" \ - --num-requests 100 \ - --max-input-len 512 \ - --output-len-dist 128,20 - ``` - - Run the benchmark - ``` - mpirun --allow-run-as-root -np 4 ./benchmarks/gptManagerBenchmark \ - --encoder_engine_dir ../../examples/models/core/enc_dec/tmp/trt_engines/t5-small-4gpu/bfloat16/encoder \ - --decoder_engine_dir ../../examples/models/core/enc_dec/tmp/trt_engines/t5-small-4gpu/bfloat16/decoder \ - --dataset cnn_dailymail.json - ``` - - -#### Emulated static batching - -To emulate the deprecated `gptSessionBenchmark` static batching, you can use `gptManagerBenchmark` with the `--static_emulated_batch_size` and `--static_emulated-timeout` arguments. - -Given a `static_emulated_batch_size` of `n` the server will wait for `n` requests to arrive before submitting them to the batch manager at once. If the `static_emulated_timeout` (in ms) is reached before `n` requests are collected, the batch will be submitted prematurely with the current request count. New batches will only be submitted once the previous batch has been processed comepletely. - -Datasets with fixed input/output lengths for benchmarking can be generated with the preprocessing script, e.g. -``` - python prepare_dataset.py \ - --output tokens-fixed-lengths.json \ - --tokenizer <path/to/tokenizer> \ - token-norm-dist \ - --num-requests 128 \ - --input-mean 60 --input-stdev 0 \ - --output-mean 20 --output-stdev 0 -``` - -Take GPT-350M as an example for single GPU with static batching -``` -./benchmarks/gptManagerBenchmark \ - --engine_dir ../../examples/models/core/gpt/trt_engine/gpt2/fp16/1-gpu/ \ - --request_rate -1 \ - --static_emulated_batch_size 32 \ - --static_emulated_timeout 100 \ - --dataset ../../benchmarks/cpp/tokens-fixed-lengths.json -``` - -#### Benchmarking LoRA - -Using either of the `prepare_dataset.py` methods above, add `--rand-task-id <start-id> <end-id>` to the command. This will add a random `task_id` from `<start-id>` to `<end-id>` inclusive. -You can then use `utils/generate_rand_loras.py` to generate random LoRA weights for benchmarking purposes. `utils/generate_rand_loras.py` takes an example LoRA for the model you are benchmarking. -Then you can run `gptManagerBenchmark` with `--type IFB` and `--lora_dir /path/to/utils/generate_rand_loras/output` - -End-to-end LoRA benchmarking script - -``` -git-lfs clone https://huggingface.co/meta-llama/Llama-2-13b-hf -git-lfs clone https://huggingface.co/hfl/chinese-llama-2-lora-13b - -MODEL_CHECKPOINT=Llama-2-13b-hf -CONVERTED_CHECKPOINT=Llama-2-13b-hf-ckpt -TOKENIZER=Llama-2-13b-hf -LORA_ENGINE=Llama-2-13b-hf-engine - -DTYPE=float16 -TP=2 -PP=1 -MAX_LEN=1024 -MAX_BATCH=32 -NUM_LAYERS=40 -MAX_LORA_RANK=64 -NUM_LORA_MODS=7 -EOS_ID=2 - -SOURCE_LORA=chinese-llama-2-lora-13b -CPP_LORA=chinese-llama-2-lora-13b-cpp - -EG_DIR=/tmp/lora-eg - -# Build lora enabled engine -python examples/models/core/llama/convert_checkpoint.py --model_dir ${MODEL_CHECKPOINT} \ - --output_dir ${CONVERTED_CHECKPOINT} \ - --dtype ${DTYPE} \ - --tp_size ${TP} \ - --pp_size 1 - -${HOME}/.local/bin/trtllm-build \ - --checkpoint_dir ${CONVERTED_CHECKPOINT} \ - --output_dir ${LORA_ENGINE} \ - --max_batch_size ${MAX_BATCH} \ - --max_input_len $MAX_LEN \ - --max_seq_len $((2*${MAX_LEN})) \ - --gemm_plugin float16 \ - --lora_plugin float16 \ - --use_paged_context_fmha enable \ - --lora_target_modules attn_q attn_k attn_v attn_dense mlp_h_to_4h mlp_4h_to_h mlp_gate \ - --max_lora_rank ${MAX_LORA_RANK} - -NUM_LORAS=(8 16) -NUM_REQUESTS=1024 - -# Convert LoRA to cpp format -python examples/hf_lora_convert.py \ - -i $SOURCE_LORA \ - --storage-type $DTYPE \ - -o $CPP_LORA - -# Prepare datasets -mkdir -p $EG_DIR/data - -# Prepare dataset without lora_task_id -python benchmarks/cpp/prepare_dataset.py \ - --output "${EG_DIR}/data/token-norm-dist.json" \ - --tokenizer $TOKENIZER \ - token-norm-dist \ - --num-requests $NUM_REQUESTS \ - --input-mean 256 --input-stdev 16 --output-mean 128 --output-stdev 24 - -# Prepare dataset with lora_task_ids from 0 - $nloras -for nloras in ${NUM_LORAS[@]}; do - python benchmarks/cpp/prepare_dataset.py \ - --output "${EG_DIR}/data/token-norm-dist-lora-${nloras}.json" \ - --rand-task-id 0 $(( $nloras - 1 )) \ - --tokenizer $TOKENIZER \ - token-norm-dist \ - --num-requests $NUM_REQUESTS \ - --input-mean 256 --input-stdev 16 --output-mean 128 --output-stdev 24 -done - -# Generate random lora weights for 16 adapters -python benchmarks/cpp/utils/generate_rand_loras.py ${CPP_LORA} ${EG_DIR}/loras 16 - -# Perform benchmarking - -# First run inference without LoRAs -mkdir -p ${EG_DIR}/log-base-lora -mpirun -n ${TP} --output-filename ${EG_DIR}/log-base-lora \ - cpp/build/benchmarks/gptManagerBenchmark \ - --engine_dir $LORA_ENGINE \ - --type IFB \ - --dataset "${EG_DIR}/data/token-norm-dist.json" \ - --lora_host_cache_bytes 8589934592 \ - --lora_num_device_mod_layers $(( 32 * $NUM_LAYERS * $NUM_LORA_MODS * $MAX_LORA_RANK )) \ - --kv_cache_free_gpu_mem_fraction 0.70 \ - --log_level info \ - --eos_id ${EOS_ID} - -# Now run inference with various numbers or loras -# The host cache is set large enough to hold all the LoRAs in lora_dir -# GPU cache is set to hold 16 LoRAs -# This benchmark will preload all the LoRAs into the host cache -# We run inference on a range of active LoRAs exercising different cache miss rates. -for nloras in ${NUM_LORAS[@]}; do - mkdir -p ${EG_DIR}/log-lora-${nloras} - mpirun -n ${TP} --output-filename "${EG_DIR}/log-lora-${nloras}" \ - cpp/build/benchmarks/gptManagerBenchmark \ - --engine_dir $LORA_ENGINE \ - --type IFB \ - --dataset "${EG_DIR}/data/token-norm-dist-lora-${nloras}.json" \ - --lora_host_cache_bytes 8589934592 \ - --lora_num_device_mod_layers $(( 16 * $NUM_LAYERS * $NUM_LORA_MODS * $MAX_LORA_RANK )) \ - --kv_cache_free_gpu_mem_fraction 0.70 \ - --log_level info \ - --eos_id ${EOS_ID} \ - --lora_dir ${EG_DIR}/loras -done -``` - -### 3. [DEPRECATED] Launch C++ static batching benchmarking (Fixed BatchSize/InputLen/OutputLen) - -#### Prepare TensorRT-LLM engine(s) - -Before you launch C++ benchmarking, please make sure that you have already built engine(s) using TensorRT-LLM API, C++ benchmarking code cannot generate engine(s) for you. - -Use `trtllm-build` to build the TRT-LLM engine. Alternatively, if you have already benchmarked Python Runtime, you can reuse the engine(s) built previously, please see that [`document`](../python/README.md). - -#### Launch benchmarking - -For detailed usage, you can do the following -``` -cd cpp/build - -# You can directly execute the binary for help information -./benchmarks/bertBenchmark --help -``` - -*Please note that the expected outputs in that document are only for reference, specific performance numbers depend on the GPU you're using.* - - -### 4.launch C++ disaggServerBenchmark -Currently ,TensorRT-LLM has limited support for disaggregated inference, where context and generation phases of a request can run on different executors. `disaggServerBenchmark` is a tool to benchmark disaggregated inference. - -#### Usage -For detailed usage, you can do the following -``` -cd cpp/build - -# You can directly execute the binary for help information -./benchmarks/disaggServerBenchmark --help -``` -`disaggServerBenchmark` only supports `decoder-only` models. -Here is the basic usage: -``` -export TRTLLM_USE_UCX_KVCACHE=1 -mpirun -n ${proc} benchmarks/disaggServerBenchmark --context_engine_dirs ${context_engine_0},${context_engine_1}...,${context_engine_{m-1}} \ ---generation_engine_dirs ${generation_engine_0},${generation_engine_1}...,${generation_engine_{n-1}} --dataset ${dataset_path} -``` -This command will launch m context engines and n generation engines. You need to ensure `proc` is equal to the sum of the number of processes required for each engine plus 1. Since we use orchestrator mode for `disaggServerBenchmark` we need an additional process as the orchestrator. For example, if there are two context engines (one is TP2_PP1,another is TP1_PP1) and two generation engines(one is TP2_PP1,another is TP1_PP1), then the `proc` value should be set to 7. - -for example: -``` -export TRTLLM_USE_UCX_KVCACHE=1 -mpirun -n 7 benchmarks/disaggServerBenchmark --context_engine_dirs ${llama_7b_tp2_pp1_dir},${llama_7b_tp1_pp1_dir} --generation_engine_dirs ${llama_7b_tp1_pp1_dir},${llama_7b_tp2_pp1_dir} --dataset ${dataset_path} - -# need 6 gpus and 7 processes to launch the benchmark. -``` - -#### Known Issues - -##### 1. error `All available sequence slots are used` - -If generation_engine's pp_size >1, the error "All available sequence slots are used" may occur, setting and adjusting the parameter `--request_rate` may help alleviate the problem. - -##### 2.KVCache transfers are by default via PCIE on single node. -Currently, because of the dependency libraries,KVCache transfers are by default via PCIE on single node. - -If you want to use NVLink, please check the UCX version in the container by running: -``` -ucx_info -v -``` -If the UCX version is less than or equal to 1.17, set `UCX_RNDV_FRAG_MEM_TYPE=cuda` to enable KvCache transfers using NVLink. -If the UCX version is 1.18, please set `UCX_CUDA_COPY_ASYNC_MEM_TYPE=cuda` to enable KvCache transfers using NVLink. diff --git a/benchmarks/cpp/bertBenchmark.cpp b/benchmarks/cpp/bertBenchmark.cpp deleted file mode 100644 index cc10a5b49eee..000000000000 --- a/benchmarks/cpp/bertBenchmark.cpp +++ /dev/null @@ -1,260 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -#include <NvInfer.h> -#include <cxxopts.hpp> -#include <nlohmann/json.hpp> - -#include <chrono> -#include <filesystem> -#include <fstream> -#include <iostream> -#include <sstream> -#include <string> - -using namespace tensorrt_llm::runtime; - -namespace trt = nvinfer1; - -namespace -{ - -std::string engineFilename( - std::filesystem::path const& dataPath, WorldConfig const& worldConfig, std::string const& model) -{ - auto constexpr allowExceptions = true; - auto constexpr ignoreComments = true; - auto const jsonFilePath = dataPath / "config.json"; - TLLM_CHECK_WITH_INFO( - std::filesystem::exists(jsonFilePath), std::string("File does not exist: ") + jsonFilePath.string()); - std::ifstream jsonStream(jsonFilePath); - auto const json = nlohmann::json::parse(jsonStream, nullptr, allowExceptions, ignoreComments); - auto const& builderConfig = json.at("builder_config"); - auto const precision = builderConfig.at("precision").template get<std::string>(); - auto const worldSize = builderConfig.at("tensor_parallel").template get<SizeType32>(); - - TLLM_CHECK_WITH_INFO(worldSize == worldConfig.getSize(), "world size mismatch"); - return model + "_" + precision + "_tp" + std::to_string(worldConfig.getSize()) + "_rank" - + std::to_string(worldConfig.getRank()) + ".engine"; -} - -void benchmarkBert(std::string const& modelName, std::filesystem::path const& dataPath, - std::vector<int> const& batchSizes, std::vector<int> const& inLens, bool useGpuDirectStorage, - std::vector<float> const& gpuWeightsPercents, std::shared_ptr<nvinfer1::ILogger> const& logger, int warmUp, - int numRuns, int duration) -{ - auto const worldConfig = WorldConfig::mpi(); - auto const enginePath = dataPath / engineFilename(dataPath, worldConfig, modelName); - - for (float gpuWeightsPercent : gpuWeightsPercents) - { - auto rt = std::make_shared<TllmRuntime>( - RawEngine(enginePath), logger.get(), useGpuDirectStorage, gpuWeightsPercent); - rt->addContext(0); - for (auto inLen : inLens) - { - for (auto const batchSize : batchSizes) - { - auto& allocator = rt->getBufferManager(); - TllmRuntime::TensorMap tensorMap{}; - - // input_ids - std::vector<SizeType32> inputIdsHost(batchSize * inLen, inLen); - auto inputIdsBuffer = std::shared_ptr<ITensor>{ - allocator.copyFrom(inputIdsHost, ITensor::makeShape({batchSize, inLen}), MemoryType::kGPU)}; - allocator.setZero(*inputIdsBuffer); - tensorMap.insert(std::make_pair("input_ids", inputIdsBuffer)); - // input_lengths - std::vector<SizeType32> inputLengthsHost(batchSize); - auto inLensBuffer = std::shared_ptr<ITensor>{ - allocator.copyFrom(inputLengthsHost, ITensor::makeShape({batchSize}), MemoryType::kGPU)}; - allocator.setZero(*inLensBuffer); - tensorMap.insert(std::make_pair("input_lengths", inLensBuffer)); - - rt->setInputTensors(0, tensorMap); - rt->setOutputTensors(0, tensorMap); - cudaDeviceSynchronize(); - - for (auto r = 0; r < warmUp; ++r) - { - rt->executeContext(0); - rt->getStream().synchronize(); - } - cudaDeviceSynchronize(); - - int iterIdx = 0; - float curDuration = 0; - while (iterIdx < numRuns || curDuration / 1000 < duration) - { - auto const start = std::chrono::steady_clock::now(); - rt->executeContext(0); - rt->getStream().synchronize(); - auto const end = std::chrono::steady_clock::now(); - - iterIdx += 1; - curDuration += (static_cast<float>( - std::chrono::duration_cast<std::chrono::microseconds>(end - start).count()) - / 1000); - } - printf("Benchmarking done. Iteration: %d, duration: %.2f sec.\n", iterIdx, curDuration / 1000); - - auto averageLatency = curDuration / iterIdx; - - if (worldConfig.getRank() == 0) - { - printf("[BENCHMARK] batch_size %d input_length %d latency(ms) %.2f\n", batchSize, inLen, - averageLatency); - } - } - } - } -} - -} // namespace - -int main(int argc, char* argv[]) -{ - cxxopts::Options options("TensorRT LLM C++ Runtime Benchmark", "TensorRT LLM C++ Runtime Benchmark for BERT."); - options.add_options()("h,help", "Print usage"); - options.add_options()( - "m,model", "Model name specified for engines.", cxxopts::value<std::string>()->default_value("bert_base")); - options.add_options()("engine_dir", "Directory that store the engines.", cxxopts::value<std::string>()); - options.add_options()("batch_size", - "Specify batch size(s) you want to benchmark. Multiple batch sizes can be separated by \";\", example: " - "\"1;8;64\".", - cxxopts::value<std::string>()->default_value("8")); - options.add_options()("input_len", - "Specify input length(s) you want to benchmark. Multiple input lengths can be " - "separated by \";\", example: \"60;128\".", - cxxopts::value<std::string>()->default_value("128")); - - options.add_options()("log_level", "Choose log level between verbose/info/warning/error/internal_error.", - cxxopts::value<std::string>()->default_value("error")); - options.add_options()( - "warm_up", "Specify warm up iterations before benchmark starts.", cxxopts::value<int>()->default_value("2")); - options.add_options()("num_runs", "Minimal number of iterations to run during benchmarking.", - cxxopts::value<int>()->default_value("10")); - options.add_options()("duration", "Minimal duration of iterations to measure in seconds.", - cxxopts::value<int>()->default_value("60")); - options.add_options()("gpu_weights_percent", - "Specify the percentage of weights that reside on GPU (from 0.0 to 1.0). Multiple percentages can be separated " - "by \";\", " - "example: \"0.0;0.5;1.0\".", - cxxopts::value<std::string>()->default_value("1.0")); - options.add_options()("use_gpu_direct_storage", "Enable GPUDirect Storage (GDS) for loading engine.", - cxxopts::value<bool>()->default_value("false")); - - auto result = options.parse(argc, argv); - - if (result.count("help")) - { - std::cout << options.help() << std::endl; - exit(0); - } - - // Argument: Engine directory - if (!result.count("engine_dir")) - { - std::cout << options.help() << std::endl; - TLLM_LOG_ERROR("Please specify engine directory."); - return 1; - } - - // Argument: Batch sizes - std::istringstream ssBatchSizesArg; - ssBatchSizesArg.str(result["batch_size"].as<std::string>()); - std::vector<int> batchSizes; - for (std::string token; std::getline(ssBatchSizesArg, token, ';');) - { - batchSizes.push_back(std::stoi(token)); - } - - // Argument : Input lengths - std::istringstream ssInLenArg; - ssInLenArg.str(result["input_len"].as<std::string>()); - std::vector<int> inLens; - for (std::string token; std::getline(ssInLenArg, token, ';');) - { - inLens.push_back(std::stoi(token)); - } - - // Argument: GPU weights percentage - std::istringstream ssGpuPercentArg; - ssGpuPercentArg.str(result["gpu_weights_percent"].as<std::string>()); - std::vector<float> gpuWeightsPercents; - for (std::string token; std::getline(ssGpuPercentArg, token, ';');) - { - auto gpuWeightsPercent = std::stof(token); - if (gpuWeightsPercent < 0 || gpuWeightsPercent > 1) - { - TLLM_LOG_ERROR( - "--gpu_weights_percent must have percents between 0.0 and 1.0 but got: %f", gpuWeightsPercent); - return 1; - } - gpuWeightsPercents.push_back(gpuWeightsPercent); - } - - // Argument: Log level - auto logger = std::make_shared<TllmLogger>(); - auto const logLevel = result["log_level"].as<std::string>(); - if (logLevel == "verbose") - { - logger->setLevel(trt::ILogger::Severity::kVERBOSE); - } - else if (logLevel == "info") - { - logger->setLevel(trt::ILogger::Severity::kINFO); - } - else if (logLevel == "warning") - { - logger->setLevel(trt::ILogger::Severity::kWARNING); - } - else if (logLevel == "error") - { - logger->setLevel(trt::ILogger::Severity::kERROR); - } - else if (logLevel == "internal_error") - { - logger->setLevel(trt::ILogger::Severity::kINTERNAL_ERROR); - } - else - { - TLLM_LOG_ERROR("Unexpected log level: " + logLevel); - return 1; - } - initTrtLlmPlugins(logger.get()); - - try - { - benchmarkBert(result["model"].as<std::string>(), result["engine_dir"].as<std::string>(), batchSizes, inLens, - result["use_gpu_direct_storage"].as<bool>(), gpuWeightsPercents, logger, result["warm_up"].as<int>(), - result["num_runs"].as<int>(), result["duration"].as<int>()); - } - catch (std::exception const& e) - { - TLLM_LOG_ERROR(e.what()); - return 1; - } - return 0; -} diff --git a/benchmarks/cpp/disaggServerBenchmark.cpp b/benchmarks/cpp/disaggServerBenchmark.cpp deleted file mode 100644 index bc3a7a2659fd..000000000000 --- a/benchmarks/cpp/disaggServerBenchmark.cpp +++ /dev/null @@ -1,1582 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/disaggServerUtil.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/gptJsonConfig.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "utils/utils.h" - -#include "cxxopts.hpp" -#include <nlohmann/json.hpp> - -#include <chrono> -#include <cstdint> -#include <cstdio> -#include <filesystem> -#include <memory> -#include <mutex> -#include <numeric> -#include <optional> -#include <string> -#include <thread> -#include <unordered_map> -#include <vector> - -using namespace tensorrt_llm::batch_manager; -using namespace tensorrt_llm::runtime; -using namespace tensorrt_llm::benchmark; -using namespace tensorrt_llm::executor::disagg_executor; -namespace texec = tensorrt_llm::executor; -namespace trt = nvinfer1; - -namespace -{ - -class Recorder -{ - -public: - explicit Recorder(std::string opCsvFile, bool streaming = false, int beamWidth = 1, - bool calculateKvCacheTransferTime = true, bool calculateQueueTime = true, std::string responsesJsonFile = "", - bool excludeInputInOutput = false) - : mOpCsvFile(std::move(opCsvFile)) - , mStreaming(streaming) - , mBeamWidth(beamWidth) - , mRespJsonFile(std::move(responsesJsonFile)) - , mOutputHasInput(!excludeInputInOutput) - , mCalculateKVCacheTransferTime(calculateKvCacheTransferTime) - , mCalculateQueueTime(calculateQueueTime) - { - } - - void initialize() - { - mStart = std::chrono::steady_clock::now(); - mSeqLatency.mDataTimes.clear(); - mFtLatency.mDataTimes.clear(); - mGenLatency.mDataTimes.clear(); - mGenFirstTokenLatency.mDataTimes.clear(); - mGenT2TLatency.mDataTimes.clear(); - mGenExcludeFirstIterT2TLatency.mDataTimes.clear(); - mContextReqQueuingLatency.mDataTimes.clear(); - mGenReqQueuingLatency.mDataTimes.clear(); - mGenReqKvCacheTransferLatency.mDataTimes.clear(); - mKvCacheThroughput.mDataTps.clear(); - } - - void finalize() - { - mEnd = std::chrono::steady_clock::now(); - } - - void recordContextQueueLatency(std::vector<float> const& latencies) - { - mContextReqQueuingLatency.mDataTimes.insert( - mContextReqQueuingLatency.mDataTimes.end(), latencies.begin(), latencies.end()); - } - - void recordGenQueueLatency(std::vector<float> const& latencies) - { - mGenReqQueuingLatency.mDataTimes.insert( - mGenReqQueuingLatency.mDataTimes.end(), latencies.begin(), latencies.end()); - } - - void recordKvCacheTransferLatency(std::vector<float> const& latencies) - { - mGenReqKvCacheTransferLatency.mDataTimes.insert( - mGenReqKvCacheTransferLatency.mDataTimes.end(), latencies.begin(), latencies.end()); - } - - void recordKvCacheThroughput(std::vector<float> const& throughputs) - { - mKvCacheThroughput.mDataTps.insert(mKvCacheThroughput.mDataTps.end(), throughputs.begin(), throughputs.end()); - } - - void recordContextStart(SizeType32 inputLength, SizeType32 maxNewTokens, uint64_t requestId, - std::chrono::time_point<std::chrono::steady_clock> const& start) - { - mRequestBenchInfos[requestId] = BenchInfo(inputLength, start); - } - - void recordContextEnd(tensorrt_llm::executor::IdType requestId, bool hasError) - { - TLLM_CHECK(mRequestBenchInfos.find(requestId) != mRequestBenchInfos.end()); - mRequestBenchInfos.at(requestId).contextEnd = std::chrono::steady_clock::now(); - mRequestBenchInfos.at(requestId).contextHasError = hasError; - mRequestBenchInfos.at(requestId).decodingIter += 1; - } - - void recordToken(tensorrt_llm::executor::IdType requestId) - { - TLLM_CHECK(mStreaming); - TLLM_CHECK_WITH_INFO(mBeamWidth == 1, "gptManagerBenchmark streaming mode does not support beam > 1"); - TLLM_CHECK(mRequestBenchInfos.find(requestId) != mRequestBenchInfos.end()); - - if (!mRequestBenchInfos.at(requestId).genFirstTokenSeen) - { - mRequestBenchInfos.at(requestId).genFirstTokenTs = std::chrono::steady_clock::now(); - mRequestBenchInfos.at(requestId).genFirstTokenSeen = true; - } - mRequestBenchInfos.at(requestId).decodingIter += 1; - } - - void recordToken(tensorrt_llm::executor::IdType requestId, texec::Response const& response) - { - - TLLM_CHECK(mRequestBenchInfos.find(requestId) != mRequestBenchInfos.end()); - - auto outputTokenIds = response.getResult().outputTokenIds; - - int32_t outputLength = 1; - for (auto const& beam : outputTokenIds) - { - outputLength = std::max(static_cast<int32_t>(beam.size()), outputLength); - } - - mRequestBenchInfos[requestId].outputLength += outputLength; - this->recordToken(requestId); - } - - void recordGenStart( - tensorrt_llm::executor::IdType requestId, std::chrono::time_point<std::chrono::steady_clock> const& start) - { - - TLLM_CHECK(mRequestBenchInfos.find(requestId) != mRequestBenchInfos.end()); - mRequestBenchInfos.at(requestId).genStart = start; - } - - void recordGenEnd(tensorrt_llm::executor::IdType requestId, bool hasError) - { - TLLM_CHECK(mRequestBenchInfos.find(requestId) != mRequestBenchInfos.end()); - mRequestBenchInfos.at(requestId).genEnd = std::chrono::steady_clock::now(); - mRequestBenchInfos.at(requestId).genHasError = hasError; - } - - void recordGenEnd(tensorrt_llm::executor::IdType requestId, texec::Response const& response) - { - recordGenEnd(requestId, response.hasError()); - if (!response.hasError()) - { - if (!mStreaming) - { - TLLM_LOG_DEBUG("response.getResult().outputTokenIds"); - auto outputTokenIds = response.getResult().outputTokenIds; - - int32_t outSeqLen = 0; - for (auto const& beam : outputTokenIds) - { - outSeqLen = std::max(static_cast<int32_t>(beam.size()), outSeqLen); - } - if (mOutputHasInput) - { - int inputSeqLen = mRequestBenchInfos[requestId].inputLength; - outSeqLen -= inputSeqLen; - } - mRequestBenchInfos[requestId].outputLength = outSeqLen; - mRequestBenchInfos[requestId].decodingIter = response.getResult().decodingIter; - } - else - { - recordToken(requestId, response); - } - } - } - - void reserve(size_t size) - { - mRequestBenchInfos.reserve(size); - } - - void calculateLatencies() - { - for (auto& reqInfo : mRequestBenchInfos) - { - - reqInfo.second.latency - = std::chrono::duration<float, std::milli>(reqInfo.second.genEnd - reqInfo.second.contextStart).count(); - reqInfo.second.firstTokenLatency - = std::chrono::duration<float, std::milli>(reqInfo.second.contextEnd - reqInfo.second.contextStart) - .count(); - reqInfo.second.genLatency - = std::chrono::duration<float, std::milli>(reqInfo.second.genEnd - reqInfo.second.genStart).count(); - if (mStreaming) - { - reqInfo.second.genFirstTokenLatency - = std::chrono::duration<float, std::milli>(reqInfo.second.genFirstTokenTs - reqInfo.second.genStart) - .count(); - // include the latency of the second token+ kv Cache transfer latency - - if (reqInfo.second.outputLength > 1) - { - reqInfo.second.avgGenT2TLatency - = std::chrono::duration<float, std::milli>(reqInfo.second.genEnd - reqInfo.second.genStart) - .count() - / static_cast<float>(reqInfo.second.outputLength - 1); - } - if (reqInfo.second.outputLength > 2) - { - reqInfo.second.avgGenExcludeFirstIterT2TLatency - = std::chrono::duration<float, std::milli>( - reqInfo.second.genEnd - reqInfo.second.genFirstTokenTs) - .count() - / static_cast<float>(reqInfo.second.outputLength - 2); - } - } - } - } - - void calculateMetrics() - { - - calculateLatencies(); - - int totalOutputTokens{0}; - int totalDecodingIter{0}; - mNumContextErrorSamples = 0; - mNumGenErrorSamples = 0; - mNumSamples = 0; - for (auto const& reqInfo : mRequestBenchInfos) - { - - if (!reqInfo.second.contextHasError && !reqInfo.second.genHasError) - { - mSeqLatency.mDataTimes.push_back(reqInfo.second.latency); - mNumSamples++; - } - if (!reqInfo.second.contextHasError) - { - mFtLatency.mDataTimes.push_back(reqInfo.second.firstTokenLatency); - } - else - { - mNumContextErrorSamples++; - } - if (!reqInfo.second.genHasError) - { - mGenLatency.mDataTimes.push_back(reqInfo.second.genLatency); - totalOutputTokens += reqInfo.second.outputLength; - totalDecodingIter += reqInfo.second.decodingIter; - if (mStreaming) - { - mGenFirstTokenLatency.mDataTimes.push_back(reqInfo.second.genFirstTokenLatency); - - if (reqInfo.second.avgGenT2TLatency.has_value()) - { - mGenT2TLatency.mDataTimes.push_back(reqInfo.second.avgGenT2TLatency.value()); - } - if (reqInfo.second.avgGenExcludeFirstIterT2TLatency.has_value()) - { - mGenExcludeFirstIterT2TLatency.mDataTimes.push_back( - reqInfo.second.avgGenExcludeFirstIterT2TLatency.value()); - } - } - } - else - { - mNumGenErrorSamples++; - } - } - mTotalLatency = std::chrono::duration<float, std::milli>(mEnd - mStart).count(); - mSeqThroughput = mNumSamples / (mTotalLatency / 1000); - mTokenThroughput = totalOutputTokens / (mTotalLatency / 1000); - mAcceptanceRate = totalDecodingIter - ? (static_cast<float>(totalOutputTokens) / static_cast<float>(totalDecodingIter)) - : 0.0F; - - mSeqLatency.calculate(); - mFtLatency.calculate(); - mGenLatency.calculate(); - if (mStreaming) - { - - mGenFirstTokenLatency.calculate(); - - if (!mGenT2TLatency.mDataTimes.empty()) - { - mGenT2TLatency.calculate(); - std::vector<float> userTokensPerSecond; - userTokensPerSecond.reserve(mGenT2TLatency.mDataTimes.size()); - for (auto const& latency : mGenT2TLatency.mDataTimes) - { - userTokensPerSecond.push_back(1000.F / latency); - } - mAvgUserTokensPerSecond = std::accumulate(userTokensPerSecond.begin(), userTokensPerSecond.end(), 0.F) - / userTokensPerSecond.size(); - } - if (!mGenExcludeFirstIterT2TLatency.mDataTimes.empty()) - { - - mGenExcludeFirstIterT2TLatency.calculate(); - } - } - if (mCalculateQueueTime) - { - - mContextReqQueuingLatency.calculate(); - mGenReqQueuingLatency.calculate(); - } - if (mCalculateKVCacheTransferTime) - { - mGenReqKvCacheTransferLatency.calculate(); - mKvCacheThroughput.calculate(); - } - } - - void report() - { - printf("[BENCHMARK] num_samples %d\n", mNumSamples); - printf("[BENCHMARK] num_context_error_samples %d\n", mNumContextErrorSamples); - printf("[BENCHMARK] num_gen_error_samples %d\n", mNumGenErrorSamples); - printf("\n[BENCHMARK] num_samples %d\n", mNumSamples); - printf("[BENCHMARK] total_latency(ms) %.2f\n", mTotalLatency); - printf("[BENCHMARK] seq_throughput(seq/sec) %.2f\n", mSeqThroughput); - printf("[BENCHMARK] token_throughput(token/sec) %.2f\n", mTokenThroughput); - if (mStreaming) - { - printf("[BENCHMARK] user_tokens_per_second(tokens/sec/user) %.2f\n", mAvgUserTokensPerSecond); - } - printf("[BENCHMARK] avg_acceptance_rate(tokens/decoding steps) %.2f\n\n", mAcceptanceRate); - - mSeqLatency.report(); - mFtLatency.report(); - mGenLatency.report(); - if (mStreaming) - { - mGenFirstTokenLatency.report(); - mGenT2TLatency.report(); - mGenExcludeFirstIterT2TLatency.report(); - } - if (mCalculateQueueTime) - { - mContextReqQueuingLatency.report(); - mGenReqQueuingLatency.report(); - } - if (mCalculateKVCacheTransferTime) - { - mGenReqKvCacheTransferLatency.report(); - mKvCacheThroughput.report(); - } - } - - void writeOpMetricsToCsv() - { - if (!mOpCsvFile.empty()) - { - std::vector<std::string> headers{"num_samples", "num_context_error_samples", "num_gen_error_samples", - "total_latency(ms)", "seq_throughput(seq/sec)", "token_throughput(token/sec)"}; - auto seqLatencyHeader = mSeqLatency.genHeaders(); - headers.insert(headers.end(), std::make_move_iterator(seqLatencyHeader.begin()), - std::make_move_iterator(seqLatencyHeader.end())); - auto contextLatencyHeader = mFtLatency.genHeaders(); - headers.insert(headers.end(), std::make_move_iterator(contextLatencyHeader.begin()), - std::make_move_iterator(contextLatencyHeader.end())); - auto genLatencyHeader = mGenLatency.genHeaders(); - headers.insert(headers.end(), std::make_move_iterator(genLatencyHeader.begin()), - std::make_move_iterator(genLatencyHeader.end())); - if (mStreaming) - { - auto genFirstTokenHeader = mGenFirstTokenLatency.genHeaders(); - headers.insert(headers.end(), std::make_move_iterator(genFirstTokenHeader.begin()), - std::make_move_iterator(genFirstTokenHeader.end())); - auto genIngterHeader = mGenT2TLatency.genHeaders(); - headers.insert(headers.end(), std::make_move_iterator(genIngterHeader.begin()), - std::make_move_iterator(genIngterHeader.end())); - auto excludeFirstIterIngterHeader = mGenExcludeFirstIterT2TLatency.genHeaders(); - headers.insert(headers.end(), std::make_move_iterator(excludeFirstIterIngterHeader.begin()), - std::make_move_iterator(excludeFirstIterIngterHeader.end())); - headers.push_back("avg_user_tokens_per_second(tokens/sec/user)"); - } - if (mCalculateKVCacheTransferTime) - { - auto genReqKVCacheTransferHeader = mGenReqKvCacheTransferLatency.genHeaders(); - headers.insert(headers.end(), std::make_move_iterator(genReqKVCacheTransferHeader.begin()), - std::make_move_iterator(genReqKVCacheTransferHeader.end())); - auto kvCacheTpHeader = mKvCacheThroughput.genHeaders(); - headers.insert(headers.end(), std::make_move_iterator(kvCacheTpHeader.begin()), - std::make_move_iterator(kvCacheTpHeader.end())); - } - - std::ofstream outputFile(mOpCsvFile); - - if (outputFile.is_open()) - { - for (auto const& header : headers) - { - outputFile << header << ","; - } - outputFile << "\n"; - - outputFile << mNumSamples << "," << mNumContextErrorSamples << "," << mNumGenErrorSamples << "," - << mTotalLatency << "," << mSeqThroughput << "," << mTokenThroughput << "," << mSeqLatency - << "," << mFtLatency << "," << mGenLatency; - if (mStreaming) - { - - outputFile << "," << mGenFirstTokenLatency << "," << mGenT2TLatency << "," - << mGenExcludeFirstIterT2TLatency << "," << mAvgUserTokensPerSecond; - } - if (mCalculateKVCacheTransferTime) - { - outputFile << "," << mGenReqKvCacheTransferLatency << "," << mKvCacheThroughput; - } - - outputFile << "\n"; - } - else - { - std::cerr << "Error opening file '" << mOpCsvFile << "' for writing.\n"; - } - } - } - -private: - struct BenchInfo - { - BenchInfo() = default; - - BenchInfo(int inputLength, std::chrono::time_point<std::chrono::steady_clock> start) - : inputLength(inputLength) - , contextStart(start) - { - } - - int inputLength{}; - int outputLength{}; - std::chrono::time_point<std::chrono::steady_clock> contextStart; - std::chrono::time_point<std::chrono::steady_clock> contextEnd; - std::chrono::time_point<std::chrono::steady_clock> genFirstTokenTs; - std::chrono::time_point<std::chrono::steady_clock> genStart; - std::chrono::time_point<std::chrono::steady_clock> genEnd; - float latency{}; // millisecond - float genLatency{}; - bool contextHasError{false}; - bool genHasError{false}; - float firstTokenLatency{}; - float genFirstTokenLatency{}; - std::optional<float> avgGenT2TLatency; - std::optional<float> avgGenExcludeFirstIterT2TLatency; - bool genFirstTokenSeen{false}; - SizeType32 decodingIter{0}; - }; - - std::unordered_map<uint64_t, BenchInfo> mRequestBenchInfos; - - std::chrono::time_point<std::chrono::steady_clock> mStart; - std::chrono::time_point<std::chrono::steady_clock> mEnd; - int mNumSamples{}; - int mNumContextErrorSamples{}; - int mNumGenErrorSamples{}; - float mTotalLatency{}; - float mSeqThroughput{}; - RecordTimeMetric mSeqLatency{"sequence_latency"}; - RecordTimeMetric mFtLatency{"context_latency"}; - RecordTimeMetric mGenLatency{"gen_latency"}; - - RecordTimeMetric mGenFirstTokenLatency{"time_to_gen_first_token"}; - RecordTimeMetric mGenT2TLatency{"inter_token_latency"}; - RecordTimeMetric mGenExcludeFirstIterT2TLatency{"exclude_first_iter_inter_token_latency"}; - RecordTimeMetric mContextReqQueuingLatency{"context_req_queueing_latency"}; - - RecordTimeMetric mGenReqQueuingLatency{"gen_req_queueing_latency"}; - RecordTimeMetric mGenReqKvCacheTransferLatency{"gen_req_kv_cache_transfer_latency"}; - - RecordBwMetric mKvCacheThroughput{"gen_req_kv_cache_transfer_throughput"}; - - float mTokenThroughput{}; - float mAcceptanceRate{}; - - std::string mOpCsvFile; - bool mStreaming; - int mBeamWidth; - std::string mRespJsonFile; - std::unordered_map<uint64_t, tensorrt_llm::executor::TensorPtr> mResponseTensors; - bool mOutputHasInput; - bool mCalculateKVCacheTransferTime; - bool mCalculateQueueTime; - float mAvgUserTokensPerSecond{}; -}; - -texec::Request makeExecutorContextRequest(Sample const& sample, SizeType32 const& beamWidth, - std::optional<SizeType32> const& eosId, std::optional<SizeType32> const& padId, bool streaming = false, - bool const& returnContextLogits = false, bool const& returnGenerationLogits = false, - std::optional<texec::LoraConfig> const& loraConfig = std::nullopt, - std::optional<texec::LookaheadDecodingConfig> const& lookaheadConfig = std::nullopt, - std::optional<texec::VecTokens> const& encoderInputTokenIds = std::nullopt) -{ - auto samplingConfig = texec::SamplingConfig{beamWidth}; - auto outputConfig = texec::OutputConfig{false, returnContextLogits, returnGenerationLogits, false}; - auto request - = texec::Request(sample.inputIds, sample.outputLen, streaming, samplingConfig, outputConfig, eosId, padId, - std::nullopt, // positionIds - std::nullopt, // badWords - std::nullopt, // stopWords - std::nullopt, // embeddingBias - std::nullopt, // speculativeDecoding - std::nullopt, // pTuning - std::nullopt, // multimodalInput - std::nullopt, // multimodalEmbedding - std::nullopt, // mRopeConfig - loraConfig, // loraConfig - lookaheadConfig, // lookaheadConfig - std::nullopt, // kvCacheRetentionConfig - std::nullopt, // logitsPostProcessorName - std::nullopt, // logitsPostProcessor - encoderInputTokenIds.has_value() ? encoderInputTokenIds : std::nullopt, - std::nullopt); // cacheSalt - request.setRequestType(tensorrt_llm::executor::RequestType::REQUEST_TYPE_CONTEXT_ONLY); - return request; -} - -class DisaggExecutorServer -{ - -public: - DisaggExecutorServer(std::vector<std::filesystem::path> const& contextEnginePaths, - std::vector<std::filesystem::path> const& genEnginePaths, - std::optional<std::vector<std::vector<SizeType32>>> const& deviceIdsForInstance, int32_t maxBeamWidth, - texec::CapacitySchedulerPolicy capacitySchedulerPolicy, BenchmarkParams const& benchmarkParams, - std::shared_ptr<Recorder> recorder, std::chrono::milliseconds waitSleep, bool logIterationData, - bool hasContextAwaitThreads, bool hasGenAwaitThreads) - : mRecorder(std::move(recorder)) - , mWaitSleep(waitSleep) - , mConcurrency(benchmarkParams.concurrency) - , mShutdown(false) - , mLogIterationData(logIterationData) - , mEnableCollectKvCacheTransferTime(benchmarkParams.enableCollectkvCacheTransferTime) - , mEnableCollectIterStats(benchmarkParams.enableCollectIterStats) - { - - int worldRank = tensorrt_llm::mpi::MpiComm::world().getRank(); - int worldSize = tensorrt_llm::mpi::MpiComm::world().getSize(); - mIsOrchestrator = (worldRank == 0); - auto contextNum = contextEnginePaths.size(); - auto genNum = genEnginePaths.size(); - int deviceCount = -1; - TLLM_CUDA_CHECK(cudaGetDeviceCount(&deviceCount)); - - std::vector<std::unique_ptr<tensorrt_llm::executor::Executor>> instances; - auto instanceNum = genNum + contextNum; - if (worldRank == 0) - { - TLLM_LOG_INFO("context enigne num :%d gen enigne num:%d", contextNum, genNum); - } - - int startRank = 0; - std::vector<texec::ExecutorConfig> ctxExecutorConfigs; - std::vector<texec::ExecutorConfig> genExecutorConfigs; - for (auto in = 0; in < instanceNum; in++) - { - auto&& enginePath = in < contextNum ? contextEnginePaths.at(in) : genEnginePaths.at(in - contextNum); - auto decoderJsonConfig = tensorrt_llm::runtime::GptJsonConfig::parse(enginePath / "config.json"); - size_t instanceRanks = decoderJsonConfig.getWorldSize(); - std::vector<SizeType32> participateRank(instanceRanks); - std::vector<SizeType32> deviceIds; - if (deviceIdsForInstance.has_value()) - { - deviceIds = deviceIdsForInstance.value().at(in); - } - for (int i = 0; i < instanceRanks; i++) - { - startRank++; - participateRank.at(i) = startRank; - if (!deviceIdsForInstance.has_value()) - { - deviceIds.push_back((startRank - 1) % deviceCount); - } - } - texec::DynamicBatchConfig dynamicBatchConfig(benchmarkParams.enableBatchSizeTuning); - texec::SchedulerConfig schedulerConfig(capacitySchedulerPolicy, std::nullopt, dynamicBatchConfig); - texec::KvCacheConfig kvCacheConfig(benchmarkParams.enableBlockReuse, - benchmarkParams.maxTokensInPagedKvCache, benchmarkParams.maxAttentionWindowVec, - benchmarkParams.sinkTokenLength, benchmarkParams.freeGpuMemoryFractions.at(in), - benchmarkParams.kvHostCacheSize); - texec::ExtendedRuntimePerfKnobConfig extendedRuntimePerfKnobConfig(benchmarkParams.multiBlockMode, - benchmarkParams.enableContextFMHAFP32Acc, benchmarkParams.cudaGraphMode, - benchmarkParams.cudaGraphCacheSize); - texec::ExecutorConfig executorConfig(maxBeamWidth, schedulerConfig, kvCacheConfig, - benchmarkParams.enableChunekedContextVec.at(in).value_or(false)); - executorConfig.setGpuWeightsPercent(benchmarkParams.gpuWeightsPercent); - texec::OrchestratorConfig orchestratorConfig{mIsOrchestrator, "", nullptr, false}; - texec::ParallelConfig parallelConfig{tensorrt_llm::executor::CommunicationType::kMPI, - tensorrt_llm::executor::CommunicationMode::kORCHESTRATOR, deviceIds, participateRank, - orchestratorConfig}; - executorConfig.setParallelConfig(parallelConfig); - if (benchmarkParams.maxBatchSizes.at(in)) - { - executorConfig.setMaxBatchSize(benchmarkParams.maxBatchSizes.at(in).value()); - } - if (benchmarkParams.maxNumTokensVec.at(in)) - { - executorConfig.setMaxNumTokens(benchmarkParams.maxNumTokensVec.at(in).value()); - } - - executorConfig.setDecodingConfig( - texec::DecodingConfig(benchmarkParams.medusaChoices.has_value() ? texec::DecodingMode::Medusa() - : benchmarkParams.executorLookaheadConfig.has_value() ? texec::DecodingMode::Lookahead() - : texec::DecodingMode::Auto(), - benchmarkParams.executorLookaheadConfig, benchmarkParams.medusaChoices)); - executorConfig.setExtendedRuntimePerfKnobConfig(extendedRuntimePerfKnobConfig); - executorConfig.setCacheTransceiverConfig( - texec::CacheTransceiverConfig(texec::CacheTransceiverConfig::BackendType::DEFAULT)); - constexpr int maxIterationsForRequestStats = 1000; - if (mEnableCollectKvCacheTransferTime) - { - executorConfig.setRequestStatsMaxIterations(maxIterationsForRequestStats); - } - if (!benchmarkParams.enableCollectIterStats) - { - executorConfig.setIterStatsMaxIterations(0); - } - - if (in < contextNum) - { - ctxExecutorConfigs.push_back(executorConfig); - } - else - { - genExecutorConfigs.push_back(executorConfig); - } - } - - mDisaggExecutor = std::make_unique<DisaggExecutorOrchestrator>(contextEnginePaths, genEnginePaths, - ctxExecutorConfigs, genExecutorConfigs, hasContextAwaitThreads, hasGenAwaitThreads); - - if (mIsOrchestrator) - { - - if (mEnableCollectIterStats || mEnableCollectKvCacheTransferTime) - { - mCollectStatsThread = std::thread(&DisaggExecutorServer::collectStats, this); - } - } - tensorrt_llm::mpi::MpiComm::world().barrier(); - } - - std::vector<tensorrt_llm::executor::IdType> enqueueContext(std::vector<texec::Request> const& requests, - std::optional<int> selectContextId = std::nullopt, bool warmup = false, bool batch = false) - { - std::vector<SizeType32> inputLengths; - std::vector<SizeType32> maxNewTokens; - if (!warmup) - { - for (auto const& request : requests) - { - inputLengths.push_back(static_cast<SizeType32>(request.getInputTokenIds().size())); - maxNewTokens.push_back(request.getMaxTokens()); - } - } - auto const start = std::chrono::steady_clock::now(); - std::vector<tensorrt_llm::executor::IdType> globalReqIds - = mDisaggExecutor->enqueueContext(requests, selectContextId, batch); - if (!warmup) - { - for (size_t i = 0; i < requests.size(); ++i) - { - mRecorder->recordContextStart(inputLengths.at(i), maxNewTokens.at(i), globalReqIds.at(i), start); - } - } - mNumContextActive += requests.size(); - return globalReqIds; - } - - void enqueueGeneration(std::vector<texec::Request> const& requests, - std::vector<tensorrt_llm::executor::IdType> const& globalRequestIds, - std::optional<int> selectGenIdx = std::nullopt, bool warmup = false, bool batch = false) - { - TLLM_CHECK(globalRequestIds.size() == requests.size()); - auto const start = std::chrono::steady_clock::now(); - mDisaggExecutor->enqueueGeneration(requests, globalRequestIds, selectGenIdx, batch); - if (!warmup) - { - for (int i = 0; i < requests.size(); i++) - { - - mRecorder->recordGenStart(globalRequestIds.at(i), start); - } - } - mNumGenActive += requests.size(); - } - - std::vector<ResponseWithId> waitForContextResponse(SizeType32 numRequests, bool warmup = false) - { - std::vector<ResponseWithId> ret; - ret.reserve(numRequests); - while ((mNumContextActive != 0) || (mNumContextFinished < numRequests)) - { - auto responses = mDisaggExecutor->awaitContextResponses(mWaitSleep); - for (auto&& response : responses) - { - TLLM_CHECK(response.response.getResult().isFinal); - if (response.response.getResult().isFinal) - { - mNumContextActive--; - mNumContextFinished++; - } - if (!warmup) - { - mRecorder->recordContextEnd(response.gid, response.response.hasError()); - } - ret.emplace_back(std::move(response)); - } - } - return ret; - } - - void waitForGenResponse(SizeType32 numRequests, bool warmup = false) - { - while (mNumGenActive > 0 || (mNumGenFinished < numRequests)) - { - auto responses = mDisaggExecutor->awaitGenerationResponses(mWaitSleep); - for (auto&& response : responses) - { - if (response.response.getResult().isFinal) - { - mNumGenActive--; - mNumGenFinished++; - - if (!warmup) - { - mRecorder->recordGenEnd(response.gid, response.response); - } - } - else - { - // streaming - if (!warmup && !response.response.hasError()) - { - mRecorder->recordToken(response.gid, response.response); - } - } - } - } - } - - bool canEnqueue(int numSentRequests) const - { - return mIsOrchestrator && (!mConcurrency || (numSentRequests - mNumGenFinished < mConcurrency)); - } - - ~DisaggExecutorServer() - { - mShutdown = true; - if (mCollectStatsThread.joinable()) - { - mCollectStatsThread.join(); - } - } - - void resetNumFinished() - { - mNumContextFinished = 0; - mNumGenFinished = 0; - } - - void resetNumActive() - { - mNumContextActive = 0; - mNumGenActive = 0; - } - - void collectStats() const - { - while (!mShutdown) - { - std::vector<std::deque<tensorrt_llm::executor::IterationStats>> contextStats; - std::vector<std::deque<tensorrt_llm::executor::IterationStats>> generationStats; - std::vector<std::deque<tensorrt_llm::executor::RequestStatsPerIteration>> - generationRequestStatsPerIteration; - contextStats.reserve(mDisaggExecutor->getContextExecutors().size()); - for (auto&& executor : mDisaggExecutor->getContextExecutors()) - { - if (executor->canEnqueueRequests()) - { - contextStats.emplace_back(executor->getLatestIterationStats()); - } - } - generationStats.reserve(mDisaggExecutor->getGenExecutors().size()); - for (auto&& executor : mDisaggExecutor->getGenExecutors()) - { - if (executor->canEnqueueRequests()) - { - if (mEnableCollectIterStats) - { - generationStats.emplace_back(executor->getLatestIterationStats()); - } - if (mEnableCollectKvCacheTransferTime) - { - - generationRequestStatsPerIteration.emplace_back(executor->getLatestRequestStats()); - } - } - } - if (mEnableCollectIterStats) - { - for (std::size_t i = 0; i < contextStats.size(); i++) - { - auto const& iterStats = contextStats.at(i); - for (auto const& stat : iterStats) - { - SizeType32 numNewActiveRequests = stat.numNewActiveRequests; - if (numNewActiveRequests > 0) - { - auto avgQueueingTime - = static_cast<float>(stat.newActiveRequestsQueueLatencyMS / numNewActiveRequests); - std::vector<float> requestsQueueLatencyMS(numNewActiveRequests, avgQueueingTime); - mRecorder->recordContextQueueLatency(requestsQueueLatencyMS); - } - if (mLogIterationData) - { - TLLM_LOG_INFO( - "ctx_id %d, ctx_stat: %s", i, texec::JsonSerialization::toJsonStr(stat).c_str()); - } - } - } - - for (std::size_t i = 0; i < generationStats.size(); i++) - { - auto const& iterStats = generationStats.at(i); - for (auto const& stat : iterStats) - { - SizeType32 numNewActiveRequests = stat.numNewActiveRequests; - if (numNewActiveRequests > 0) - { - float avgQueueingTime - = static_cast<float>(stat.newActiveRequestsQueueLatencyMS / numNewActiveRequests); - std::vector<float> requestsQueueLatencyMS(numNewActiveRequests, avgQueueingTime); - mRecorder->recordGenQueueLatency(requestsQueueLatencyMS); - } - if (mLogIterationData) - { - TLLM_LOG_INFO( - "gen_id %d, gen_stat: %s", i, texec::JsonSerialization::toJsonStr(stat).c_str()); - } - } - } - } - - if (mEnableCollectKvCacheTransferTime) - { - for (std::size_t i = 0; i < generationRequestStatsPerIteration.size(); i++) - { - auto const& stats = generationRequestStatsPerIteration.at(i); - for (auto const& stat : stats) - { - std::vector<float> kvCacheTransferMs; - std::vector<float> kvCacheThroughput; - for (auto const& requestStat : stat.requestStats) - { - if (requestStat.stage == tensorrt_llm::executor::RequestStage::kGENERATION_COMPLETE) - { - kvCacheTransferMs.push_back( - static_cast<float>(requestStat.disServingStats->kvCacheTransferMS)); - kvCacheThroughput.push_back(static_cast<float>(requestStat.disServingStats->kvCacheSize) - * 8 / (static_cast<float>(requestStat.disServingStats->kvCacheTransferMS) / 1000) - / 1e9f); - } - } - if (kvCacheTransferMs.size() > 0) - { - mRecorder->recordKvCacheTransferLatency(kvCacheTransferMs); - } - if (kvCacheThroughput.size() > 0) - { - mRecorder->recordKvCacheThroughput(kvCacheThroughput); - } - if (mLogIterationData) - { - TLLM_LOG_INFO( - "gen_id %d, gen_req_stat: %s", i, texec::JsonSerialization::toJsonStr(stat).c_str()); - } - } - } - } - auto const waitSleep = std::chrono::milliseconds(50); - std::this_thread::sleep_for(waitSleep); - } - } - - std::unique_ptr<DisaggExecutorOrchestrator> const& getDisaggExecutor() const noexcept - { - return mDisaggExecutor; - } - -private: - std::unique_ptr<DisaggExecutorOrchestrator> mDisaggExecutor; - - std::atomic<bool> mShutdown{false}; - bool mIsOrchestrator{false}; - - std::shared_ptr<Recorder> mRecorder; - std::chrono::milliseconds mWaitSleep; - std::optional<int> mConcurrency; - bool mLogIterationData{false}; - bool const mEnableCollectKvCacheTransferTime; - bool const mEnableCollectIterStats; - std::thread mCollectStatsThread; - std::atomic<uint64_t> mNumGenFinished{0}; - std::atomic<uint64_t> mNumContextFinished{0}; - std::atomic<uint64_t> mNumGenActive{0}; - std::atomic<uint64_t> mNumContextActive{0}; -}; - -} // namespace - -void benchmark(std::vector<std::filesystem::path> const& contextEngineDirs, - std::vector<std::filesystem::path> const& generationEngineDirs, - std::optional<std::vector<std::vector<int>>> const& deviceIdsForInstances, std::string const& datasetPath, - std::string const& opCsvFile, int maxNumSamples, int beamWidth, int warmUp, std::optional<int32_t> const& eosId, - std::optional<int32_t> const& padId, BenchmarkParams const& benchmarkParams, - texec::CapacitySchedulerPolicy capacitySchedulerPolicy, std::chrono::milliseconds waitSleep, - bool returnContextLogits, bool returnGenerationLogits, std::optional<int> const staticEmulatedBatchSize, - bool logIterationData, std::optional<SizeType32> const maxPromptLen, bool hasContextAwait, bool hasGenAwait) -{ - - auto const& world = tensorrt_llm::mpi::MpiComm::world(); - auto worldRank = world.getRank(); - - // Load dataset - auto const samples = parseWorkloadJson(datasetPath, maxNumSamples, maxPromptLen); - auto const numSamples = samples.size(); - auto recorder = std::make_shared<Recorder>(opCsvFile, benchmarkParams.streaming, beamWidth, - benchmarkParams.enableCollectkvCacheTransferTime, benchmarkParams.enableCollectIterStats); - auto disaggExecutor = std::make_shared<DisaggExecutorServer>(contextEngineDirs, generationEngineDirs, - deviceIdsForInstances, beamWidth, capacitySchedulerPolicy, benchmarkParams, recorder, waitSleep, - logIterationData, hasContextAwait, hasGenAwait); - constexpr size_t numMap = 8; - std::vector<std::unordered_map<tensorrt_llm::executor::IdType, tensorrt_llm::executor::Request>> gidToRequestMaps( - numMap); - std::vector<std::mutex> mtxForMaps(numMap); - - auto fillRequestMap = [&](std::vector<tensorrt_llm::executor::IdType> const& reqIds, - std::vector<tensorrt_llm::executor::Request>&& requests) - { - TLLM_CHECK(reqIds.size() == requests.size()); - for (size_t i = 0; i < reqIds.size(); i++) - { - - size_t mapIdx = reqIds[i] % numMap; - std::scoped_lock<std::mutex> lock(mtxForMaps[mapIdx]); - gidToRequestMaps.at(mapIdx).emplace(reqIds[i], std::move(requests[i])); - } - }; - - auto makeGenRequest = [&](std::vector<ResponseWithId>&& contextResponse) - { - std::vector<tensorrt_llm::executor::IdType> gids; - gids.reserve(contextResponse.size()); - std::vector<tensorrt_llm::executor::Request> genRequest; - genRequest.reserve(contextResponse.size()); - for (auto&& ctxResponse : contextResponse) - { - gids.emplace_back(ctxResponse.gid); - size_t mapIdx = ctxResponse.gid % numMap; - - std::unique_lock<std::mutex> lock(mtxForMaps[mapIdx]); - TLLM_CHECK(gidToRequestMaps.at(mapIdx).find(ctxResponse.gid) != gidToRequestMaps.at(mapIdx).end()); - auto ctxRequest = std::move(gidToRequestMaps.at(mapIdx).at(ctxResponse.gid)); - gidToRequestMaps.at(mapIdx).erase(ctxResponse.gid); - lock.unlock(); - ctxRequest.setRequestType(tensorrt_llm::executor::RequestType::REQUEST_TYPE_GENERATION_ONLY); - ctxRequest.setContextPhaseParams(ctxResponse.response.getResult().contextPhaseParams.value()); - genRequest.emplace_back(std::move(ctxRequest)); - } - return std::make_pair(genRequest, gids); - }; - if (worldRank == 0) - { - { // warmup - TLLM_LOG_INFO("Warmup start"); - - size_t contextNum = contextEngineDirs.size(); - size_t generationNum = generationEngineDirs.size(); - for (auto con = 0; con < contextNum; con++) - { - for (auto gen = 0; gen < generationNum; gen++) - { - std::vector<tensorrt_llm::executor::Request> contextRequests; - contextRequests.reserve(warmUp); - for (int i = 0; i < warmUp; ++i) - { - contextRequests.emplace_back(makeExecutorContextRequest(samples[0], beamWidth, eosId, padId, - benchmarkParams.streaming, returnContextLogits, returnGenerationLogits, std::nullopt, - benchmarkParams.requestLookaheadConfig)); - } - auto reqIds = disaggExecutor->enqueueContext(contextRequests, con, true); - fillRequestMap(reqIds, std::move(contextRequests)); - auto contextResponse = disaggExecutor->waitForContextResponse(warmUp, true); - auto&& [genRequests, gids] = makeGenRequest(std::move(contextResponse)); - disaggExecutor->enqueueGeneration(genRequests, gids, gen, true); - disaggExecutor->waitForGenResponse(warmUp, true); - disaggExecutor->resetNumFinished(); - disaggExecutor->resetNumActive(); - } - } - - auto const warmUpWaitSleep = std::chrono::milliseconds(50); - std::this_thread::sleep_for(warmUpWaitSleep); - TLLM_LOG_INFO("Warmup done"); - } - - { - - auto timeDelays = computeTimeDelays(benchmarkParams, numSamples - 1); - - std::vector<texec::Request> contextRequests; - - for (std::size_t i = 0; i < numSamples; ++i) - { - std::optional<texec::LoraConfig> loraConfig = std::nullopt; - contextRequests.emplace_back(makeExecutorContextRequest(samples[i], beamWidth, eosId, padId, - benchmarkParams.streaming, returnContextLogits, returnGenerationLogits, loraConfig, - benchmarkParams.requestLookaheadConfig)); - } - - bool const hasDelay - = std::any_of(timeDelays.begin(), timeDelays.end(), [](auto const& delay) { return delay > 0.0; }); - disaggExecutor->resetNumFinished(); - disaggExecutor->resetNumActive(); - - recorder->reserve(numSamples); - recorder->initialize(); - if (!staticEmulatedBatchSize) - { - - std::thread waitContextResponseAndEnqueGenThread{[&]() - { - auto numRequest = numSamples; - while (numRequest > 0) - { - auto contextResponseWithIds - = disaggExecutor->getDisaggExecutor()->awaitContextResponses(waitSleep); - if (contextResponseWithIds.empty()) - { - continue; - } - for (auto&& contextResponseWithId : contextResponseWithIds) - { - recorder->recordContextEnd( - contextResponseWithId.gid, contextResponseWithId.response.hasError()); - } - numRequest -= contextResponseWithIds.size(); - auto&& [genReqeust, genGids] = makeGenRequest(std::move(contextResponseWithIds)); - disaggExecutor->enqueueGeneration(genReqeust, genGids); - } - }}; - - std::thread waitGenResponseThread{[&]() { disaggExecutor->waitForGenResponse(numSamples); }}; - int numSentRequests = 0; - while (numSentRequests < numSamples) - { - - if (disaggExecutor->canEnqueue(numSentRequests)) - { - auto gids = disaggExecutor->enqueueContext({contextRequests.at(numSentRequests)}); - fillRequestMap(gids, {contextRequests.at(numSentRequests)}); - - if (hasDelay && numSentRequests < numSamples - 1) - { - std::this_thread::sleep_for( - std::chrono::milliseconds(static_cast<int>(timeDelays.at(numSentRequests) * 1000))); - } - numSentRequests += 1; - } - } - waitContextResponseAndEnqueGenThread.join(); - waitGenResponseThread.join(); - } - else - { - TLLM_CHECK_WITH_INFO( - !hasDelay, "Executor benchmark doesn't support delays with emulated static batch sizes"); - auto numRequests = contextRequests.size(); - int maxBatchSize = staticEmulatedBatchSize.value(); - for (int req = 0; req < numRequests; req += maxBatchSize) - { - auto batchSize = std::min(static_cast<size_t>(maxBatchSize), numRequests - req); - - std::vector<texec::Request> requestsBatch(std::make_move_iterator(contextRequests.begin() + req), - std::make_move_iterator(contextRequests.begin() + req + static_cast<int64_t>(batchSize))); - // Enqueue in batches - - auto reqIds = disaggExecutor->enqueueContext(requestsBatch); - fillRequestMap(reqIds, std::move(requestsBatch)); - auto contextResponse = disaggExecutor->waitForContextResponse(static_cast<SizeType32>(batchSize)); - auto&& [genRequests, genReqIds] = makeGenRequest(std::move(contextResponse)); - disaggExecutor->enqueueGeneration(genRequests, genReqIds); - disaggExecutor->waitForGenResponse(static_cast<SizeType32>(batchSize)); - - // Wait for current batch to be done - } - } - } - recorder->finalize(); - // sleep for collect stats - if (benchmarkParams.enableCollectIterStats || benchmarkParams.enableCollectkvCacheTransferTime) - { - auto const collectWaitSleep = std::chrono::milliseconds(50); - std::this_thread::sleep_for(collectWaitSleep); - } - recorder->calculateMetrics(); - recorder->report(); - recorder->writeOpMetricsToCsv(); - } -} - -int main(int argc, char* argv[]) - -{ - cxxopts::Options options("TensorRT LLM DisaggServer Benchmark"); - options.add_options()("h,help", "Print usage"); - options.add_options()("context_engine_dirs", "Directories that store context engines,separator is a ,", - cxxopts::value<std::vector<std::string>>()); - options.add_options()("generation_engine_dirs", "Directories that store generation engines,separator is a , ", - cxxopts::value<std::vector<std::string>>()); - options.add_options()("device_ids_for_instances", - "device ids for each instances , example: \"[[0,1],[2,3],[4,5,6,7]]\" ", cxxopts::value<std::string>()); - options.add_options()("dataset", "Dataset that is used for benchmarking BatchManager.", - cxxopts::value<std::string>()->default_value("")); - options.add_options()( - "output_csv", "Write output metrics to CSV", cxxopts::value<std::string>()->default_value("")); - options.add_options()("max_num_samples", "maximum number of samples to use from dataset/generate", - cxxopts::value<int>()->default_value("100000")); - options.add_options()( - "beam_width", "Specify beam width you want to benchmark.", cxxopts::value<int>()->default_value("1")); - options.add_options()( - "warm_up", "Specify warm up iterations before benchmark starts.", cxxopts::value<int>()->default_value("2")); - options.add_options()( - "eos_id", "Specify the end-of-sequence token id.", cxxopts::value<TokenIdType>()->default_value("-1")); - options.add_options()("pad_id", "Specify the padding token id.", cxxopts::value<TokenIdType>()); - options.add_options()("max_tokens_in_paged_kvcache", "Max tokens in paged K-V Cache.", cxxopts::value<int>()); - options.add_options()( - "max_attention_window", "Max KV cache length per sequence", cxxopts::value<std::vector<int>>()); - options.add_options()("sink_token_len", "Sink token length in kv cache per sequence.", cxxopts::value<int>()); - options.add_options()( - "random_seed", "integer random seed for exponential time delays.", cxxopts::value<int>()->default_value("420")); - options.add_options()("kv_cache_free_gpu_mem_fractions", "K-V Cache Free Gpu Mem Fraction,each for per instance", - cxxopts::value<std::vector<float>>()); - options.add_options()("request_rate", - "request rate in reqs/sec. Skipping this arg or negative value will trigger offline/0-delay.", - cxxopts::value<float>()); - options.add_options()("concurrency", "Concurrent number of connections with the server.", cxxopts::value<int>()); - options.add_options()("max_batch_sizes", "The max runtime batch size when benchmarking, each for per instance", - cxxopts::value<std::vector<int>>()); - options.add_options()("max_num_tokens_per_instance", - "The max runtime number of tokens per batch when benchmarking, each for per instance", - cxxopts::value<std::vector<int>>()); - options.add_options()( - "enable_batch_size_tuning", "Dynamic tuning of batch size", cxxopts::value<bool>()->default_value("false")); - options.add_options()("enable_exp_delays", "Enables exponential delay distr to mimic real world request arrival", - cxxopts::value<bool>()->default_value("false")); - options.add_options()("streaming", "Operate in streaming mode", cxxopts::value<bool>()->default_value("false")); - options.add_options()( - "enable_kv_cache_reuse", "Enables the KV cache reuse.", cxxopts::value<bool>()->default_value("false")); - options.add_options()("enable_chunked_context_per_instance", "Whether to enable context chunking for per instance", - cxxopts::value<std::vector<bool>>()->default_value("false")); - options.add_options()( - "return_context_logits", "Whether to return context logits.", cxxopts::value<bool>()->default_value("false")); - options.add_options()("return_generation_logits", "Whether to return generation logits.", - cxxopts::value<bool>()->default_value("false")); - - options.add_options()("scheduler_policy", - "Choose scheduler policy between max_utilization/guaranteed_no_evict/static_batch.", - cxxopts::value<std::string>()->default_value("guaranteed_no_evict")); - - options.add_options()("static_emulated_batch_size", - "Emulate static batching performance with the provided batch size.", cxxopts::value<SizeType32>()); - options.add_options()("log_level", "Choose log level between verbose/info/warning/error/internal_error.", - cxxopts::value<std::string>()->default_value("error")); - options.add_options()("log_iteration_data", "On each decoder iteration, print batch state metadata.", - cxxopts::value<bool>()->default_value("false")); - options.add_options()("wait_sleep", "Specify how many milliseconds to sleep each iteration of waitForEmpty loop.", - cxxopts::value<int>()->default_value("25")); - options.add_options()("kv_host_cache_bytes", - "Size of secondary memory pool used for offloading kv cache blocks (in bytes).", - cxxopts::value<size_t>()->default_value("0")); - options.add_options()( - "max_prompt_len", "Truncate all prompts from dataset to the length specified.", cxxopts::value<SizeType32>()); - options.add_options()("gpu_weights_percent", - "Specify the percentage of weights that reside on GPU (from 0.0 to 1.0).", - cxxopts::value<float>()->default_value("1.0")); - options.add_options()( - "medusa_choices", "Medusa choices in the format of [[0], [0, 1], [0, 0, 1]]", cxxopts::value<std::string>()); - options.add_options()("multi_block_mode", - "Distribute the work across multiple CUDA thread-blocks on the GPU for masked MHA kernel", - cxxopts::value<bool>()->default_value("true")); - options.add_options()("cuda_graph_mode", "When enabled, inference is executed with cuda graph.", - cxxopts::value<bool>()->default_value("false")); - options.add_options()("cuda_graph_cache_size", - "Specify how many cuda graphs are cached in the runtime. Larger cache gives better perf, but consumes more GPU " - "memory.", - cxxopts::value<SizeType32>()->default_value("0")); - options.add_options()("enable_context_fmha_fp32_acc", "Enable FMHA runner FP32 accumulation", - cxxopts::value<bool>()->default_value("false")); - options.add_options()("executor_lookahead_config", - "lookahead config in the format of [max_window_size, max_ngram_size, max_verification_set_size]", - cxxopts::value<std::string>()); - options.add_options()("request_lookahead_config", - "lookahead config in the format of [max_window_size, max_ngram_size, max_verification_set_size], and each <= " - "executor lookahead config", - cxxopts::value<std::string>()); - options.add_options()("context_await", "When enabled, will has a thread to await context response.", - cxxopts::value<bool>()->default_value("true")); - options.add_options()("gen_await", "When enabled,will has a thread to await gen response.", - cxxopts::value<bool>()->default_value("true")); - options.add_options()("enable_collect_kvcache_transfer_time", "When enabled, will collect kvcache transfer time.", - cxxopts::value<bool>()->default_value("false")); - options.add_options()("enable_collect_iter_stats", "When enabled, will collect iteration stats.", - cxxopts::value<bool>()->default_value("false")); - - auto result = options.parse(argc, argv); - - if ((result.count("context_engine_dirs") == 0) || (result.count("generation_engine_dirs") == 0)) - { - std::cout << options.help() << std::endl; - TLLM_LOG_ERROR("Please specify context engine and generation engine directory."); - return 1; - } - // Argument: Log level - auto logger = std::make_shared<TllmLogger>(); - auto const logLevel = result["log_level"].as<std::string>(); - if (logLevel == "verbose") - { - logger->setLevel(trt::ILogger::Severity::kVERBOSE); - } - else if (logLevel == "info") - { - logger->setLevel(trt::ILogger::Severity::kINFO); - } - else if (logLevel == "warning") - { - logger->setLevel(trt::ILogger::Severity::kWARNING); - } - else if (logLevel == "error") - { - logger->setLevel(trt::ILogger::Severity::kERROR); - } - else if (logLevel == "internal_error") - { - logger->setLevel(trt::ILogger::Severity::kINTERNAL_ERROR); - } - else - { - TLLM_LOG_ERROR("Unexpected log level: " + logLevel); - return 1; - } - - initTrtLlmPlugins(logger.get()); - - // Argument: Dataset - auto const datasetPath = result["dataset"].as<std::string>(); - auto const maxNumSamples = result["max_num_samples"].as<int>(); - - // Argument: Output metrics CSV - auto const opCsvFile = result["output_csv"].as<std::string>(); - - // Argument: beam width - auto const beamWidth = result["beam_width"].as<int>(); - TLLM_CHECK_WITH_INFO(beamWidth == 1, "Currently only support beamWidth=1"); - // Argument: wait_sleep - auto const waitSleep = std::chrono::milliseconds(result["wait_sleep"].as<int>()); - auto const hasContextAwait = result["context_await"].as<bool>(); - auto const hasGenAwait = result["gen_await"].as<bool>(); - BenchmarkParams benchmarkParams; - benchmarkParams.enableCollectkvCacheTransferTime = result["enable_collect_kvcache_transfer_time"].as<bool>(); - benchmarkParams.enableCollectIterStats = result["enable_collect_iter_stats"].as<bool>(); - - std::vector<std::string> contextEngineDirs = result["context_engine_dirs"].as<std::vector<std::string>>(); - std::vector<std::string> generationEngineDirs = result["generation_engine_dirs"].as<std::vector<std::string>>(); - if (tensorrt_llm::mpi::MpiComm::world().getRank() == 0) - { - std::string contextEngineStrings; - for (auto&& contextEngineDir : contextEngineDirs) - { - contextEngineStrings += contextEngineDir + ","; - } - std::string generationEnginesStrings; - for (auto&& genEngineDir : generationEngineDirs) - { - generationEnginesStrings += genEngineDir + ","; - } - TLLM_LOG_INFO( - "Will Launch benchmark with %d context engines and %d generation engines. Context Engines:%s ; Generation " - "Engines:%s ;", - contextEngineDirs.size(), generationEngineDirs.size(), contextEngineStrings.c_str(), - generationEnginesStrings.c_str()); - } - std::vector<std::filesystem::path> contextEnigePaths; - std::vector<std::filesystem::path> generationEnginePaths; - - contextEnigePaths.reserve(contextEngineDirs.size()); - - for (auto& contextEngineDir : contextEngineDirs) - { - - contextEnigePaths.emplace_back(contextEngineDir); - } - generationEnginePaths.reserve(generationEngineDirs.size()); - for (auto& genEngineDir : generationEngineDirs) - { - - generationEnginePaths.emplace_back(genEngineDir); - } - - int const instanceNum = contextEngineDirs.size() + generationEngineDirs.size(); - // Argument: Max tokens in paged K-V Cache - if (result.count("max_tokens_in_paged_kvcache")) - { - benchmarkParams.maxTokensInPagedKvCache = result["max_tokens_in_paged_kvcache"].as<int>(); - } - - // Argument: Max KV cache length - if (result.count("max_attention_window")) - { - benchmarkParams.maxAttentionWindowVec = result["max_attention_window"].as<std::vector<int>>(); - } - - // Argument: Sink token length - if (result.count("sink_token_len")) - { - benchmarkParams.sinkTokenLength = result["sink_token_len"].as<int>(); - } - - if (result.count("random_seed")) - { - benchmarkParams.randomSeed = result["random_seed"].as<int>(); - } - - // Argument: K-V Cache Free Gpu Mem Fraction - benchmarkParams.freeGpuMemoryFractions.resize(instanceNum); - if (result.count("kv_cache_free_gpu_mem_fractions")) - { - auto fractions = result["kv_cache_free_gpu_mem_fractions"].as<std::vector<float>>(); - TLLM_CHECK_WITH_INFO(fractions.size() == instanceNum || fractions.size() == 1, - "the number of fraction should be equal to the number of instances or equal to 1"); - for (int i = 0; i < instanceNum; i++) - { - benchmarkParams.freeGpuMemoryFractions.at(i) = fractions.size() == 1 ? fractions[0] : fractions[i]; - } - } - - // Argument: Enable dynamic tuning of batch size - benchmarkParams.enableBatchSizeTuning = result["enable_batch_size_tuning"].as<bool>(); - - // Argument: Enable KV cache reuse - benchmarkParams.enableBlockReuse = result["enable_kv_cache_reuse"].as<bool>(); - - // Argument: streaming - benchmarkParams.streaming = result["streaming"].as<bool>(); - - TLLM_CHECK_WITH_INFO(!(result.count("request_rate") && result.count("concurrency")), - "request_rate and concurrency cannot be specified at the same time."); - - // Argument: request rate - if (result.count("request_rate")) - { - benchmarkParams.requestRate = result["request_rate"].as<float>(); - } - - // Argument: concurrency - if (result.count("concurrency")) - { - benchmarkParams.concurrency = result["concurrency"].as<int>(); - } - - // Argument: max_batch_sizes - benchmarkParams.maxBatchSizes.resize(instanceNum); - if (result.count("max_batch_sizes")) - { - auto batchSizes = result["max_batch_sizes"].as<std::vector<int>>(); - TLLM_CHECK_WITH_INFO(batchSizes.size() == instanceNum || batchSizes.size() == 1, - "the number of batch size should be equal to the number of instances or equal to 1"); - for (int i = 0; i < instanceNum; i++) - { - benchmarkParams.maxBatchSizes.at(i) = batchSizes.size() == 1 ? batchSizes[0] : batchSizes[i]; - } - } - - // Argument: max_num_tokens_per_instance - benchmarkParams.maxNumTokensVec.resize(instanceNum); - if (result.count("max_num_tokens_per_instance")) - { - auto maxNumTokensVec = result["max_num_tokens_per_instance"].as<std::vector<int>>(); - TLLM_CHECK_WITH_INFO(maxNumTokensVec.size() == instanceNum || maxNumTokensVec.size() == 1, - "the number of max_num_tokens should be equal to the number of instances or equal to 1"); - for (int i = 0; i < instanceNum; i++) - { - benchmarkParams.maxNumTokensVec.at(i) - = maxNumTokensVec.size() == 1 ? maxNumTokensVec[0] : maxNumTokensVec[i]; - } - } - - benchmarkParams.enableExpDelays = result["enable_exp_delays"].as<bool>(); - - // Argument: Enable batch stats output - bool logIterationData = result["log_iteration_data"].as<bool>(); - - // Argument: Enable chunked context - benchmarkParams.enableChunekedContextVec.resize(instanceNum); - if (result.count("enable_chunked_context_per_instance")) - { - auto enableChunkedContextVec = result["enable_chunked_context_per_instance"].as<std::vector<bool>>(); - - TLLM_CHECK_WITH_INFO(enableChunkedContextVec.size() == instanceNum || enableChunkedContextVec.size() == 1, - "the number of enable_chunked_context_per_instance should be equal to the number of instances or equal to " - "1"); - for (int i = 0; i < instanceNum; i++) - { - benchmarkParams.enableChunekedContextVec.at(i) - = enableChunkedContextVec.size() == 1 ? enableChunkedContextVec[0] : enableChunkedContextVec[i]; - } - } - // Argument: Enable return context logits - bool returnContextLogits = result["return_context_logits"].as<bool>(); - TLLM_CHECK_WITH_INFO(returnContextLogits == false, "Currently disaggServer don't support returnContextLogits!"); - // Argument: Enable return context logits - bool returnGenerationLogits = result["return_generation_logits"].as<bool>(); - TLLM_CHECK_WITH_INFO( - returnGenerationLogits == false, "Currently disaggServer don't support returnGenerationLogits!"); - - if (result.count("lora_dir")) - { - TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support lora!"); - benchmarkParams.loraDir = result["lora_dir"].as<std::string>(); - } - if (result.count("lora_host_cache_bytes")) - { - TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support lora!"); - - benchmarkParams.loraHostCacheSize = result["lora_host_cache_bytes"].as<size_t>(); - } - if (result.count("lora_num_device_mod_layers")) - { - TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support lora!"); - - benchmarkParams.loraDeviceNumModLayers = result["lora_num_device_mod_layers"].as<SizeType32>(); - } - - // Argument: How many KV cache blocks (as fraction of number of GPU kv cache blocks). - benchmarkParams.kvHostCacheSize = result["kv_host_cache_bytes"].as<size_t>(); - TLLM_CHECK_WITH_INFO( - benchmarkParams.kvHostCacheSize == false, "Currently disaggServer don't support kv_host_cache!"); - - // Argument: Medusa choices for the Medusa speculative decoding. - if (result.count("medusa_choices")) - { - TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support medusa!"); - - benchmarkParams.medusaChoices = parseVectorOfVectors(result["medusa_choices"].as<std::string>()); - } - if (result.count("executor_lookahead_config")) - { - TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support lookhead!"); - - benchmarkParams.executorLookaheadConfig - = parseLookaheadConfig(result["executor_lookahead_config"].as<std::string>()); - } - if (result.count("request_lookahead_config")) - { - TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support lookhead!"); - - benchmarkParams.requestLookaheadConfig - = parseLookaheadConfig(result["request_lookahead_config"].as<std::string>()); - } - - // Argument: multi_block_mode - benchmarkParams.multiBlockMode = result["multi_block_mode"].as<bool>(); - - // Argument: enable_context_fmha_fp32_acc - benchmarkParams.enableContextFMHAFP32Acc = result["enable_context_fmha_fp32_acc"].as<bool>(); - - // Argument: cuda_graph_mode - benchmarkParams.cudaGraphMode = result["cuda_graph_mode"].as<bool>(); - - // Argument: cuda_graph_cache_size - benchmarkParams.cudaGraphCacheSize = result["cuda_graph_cache_size"].as<SizeType32>(); - - std::optional<TokenIdType> padId; - // Argument: Padding token id - if (result.count("pad_id")) - { - padId = result["pad_id"].as<TokenIdType>(); - } - - // Argument: End-of-sentence token id - std::optional<TokenIdType> eosId = result["eos_id"].as<TokenIdType>(); - - std::optional<std::chrono::milliseconds> batchTimeout; - - std::optional<SizeType32> staticEmulatedBatchSize; - // Argument: Static emulated batch size - if (result.count("static_emulated_batch_size")) - { - staticEmulatedBatchSize = result["static_emulated_batch_size"].as<SizeType32>(); - } - - // Argument: Scheduler policy - texec::CapacitySchedulerPolicy capacitySchedulerPolicy; - auto const capacitySchedulerPolicyArg = result["scheduler_policy"].as<std::string>(); - if (capacitySchedulerPolicyArg == "max_utilization") - { - capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kMAX_UTILIZATION; - } - else if (capacitySchedulerPolicyArg == "guaranteed_no_evict") - { - capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT; - } - else if (capacitySchedulerPolicyArg == "static_batch") - { - capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kSTATIC_BATCH; - } - else - { - TLLM_LOG_ERROR("Unexpected scheduler policy: " + capacitySchedulerPolicyArg); - return 1; - } - - // Argument: max_prompt_len - std::optional<SizeType32> maxPromptLen; - if (result.count("max_prompt_len")) - { - maxPromptLen = result["max_prompt_len"].as<SizeType32>(); - } - - // Argument: GPU weights percentage - auto gpuWeightsPercent = result["gpu_weights_percent"].as<float>(); - if (gpuWeightsPercent < 0 || gpuWeightsPercent > 1) - { - TLLM_LOG_ERROR("--gpu_weights_percent must be between 0.0 and 1.0 but got: %f", gpuWeightsPercent); - return 1; - } - benchmarkParams.gpuWeightsPercent = gpuWeightsPercent; - - std::optional<std::vector<std::vector<int>>> deviceIdsForInstance = std::nullopt; - if (result.count("device_ids_for_instances")) - { - deviceIdsForInstance = parseVectorOfVectors(result["device_ids_for_instances"].as<std::string>()); - } - benchmark(contextEnigePaths, generationEnginePaths, deviceIdsForInstance, datasetPath, opCsvFile, maxNumSamples, - beamWidth, result["warm_up"].as<int>(), eosId, padId, benchmarkParams, capacitySchedulerPolicy, waitSleep, - returnContextLogits, returnContextLogits, staticEmulatedBatchSize, logIterationData, maxPromptLen, - hasContextAwait, hasGenAwait); -} diff --git a/benchmarks/cpp/gptManagerBenchmark.cpp b/benchmarks/cpp/gptManagerBenchmark.cpp deleted file mode 100644 index 287cbba343ce..000000000000 --- a/benchmarks/cpp/gptManagerBenchmark.cpp +++ /dev/null @@ -1,1557 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/tensor.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/gptJsonConfig.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "tensorrt_llm/runtime/utils/numpyUtils.h" -#include "tensorrt_llm/runtime/worldConfig.h" -#include "utils/utils.h" - -#include <chrono> -#include <cstdint> -#include <cxxopts.hpp> -#include <iostream> -#include <memory> -#include <nlohmann/json.hpp> -#include <numeric> -#include <optional> -#include <string> -#include <thread> -#include <utility> - -using namespace tensorrt_llm::batch_manager; -using namespace tensorrt_llm::runtime; -using namespace tensorrt_llm::benchmark; -namespace texec = tensorrt_llm::executor; -namespace trt = nvinfer1; -namespace fs = std::filesystem; - -namespace -{ - -using TensorPtr = ITensor::SharedPtr; - -class LoraLib -{ -public: - LoraLib(std::string const& loraDir) - : mLoraDir(loraDir) - , mBufferManager(std::make_shared<CudaStream>()) - , mTaskPaths(parseDirPaths(mLoraDir)) - , mLoras(readLoras(mTaskPaths)) - { - } - - TensorPtr getLoraWeights(uint64_t taskId) const - { - return mLoras.at(taskId).first; - } - - TensorPtr getLoraConfig(uint64_t taskId) const - { - return mLoras.at(taskId).second; - } - - void clear() - { - mLoras.clear(); - } - - std::map<uint64_t, std::pair<TensorPtr, TensorPtr>> const& getLoras() - { - return mLoras; - } - -private: - std::string const mLoraDir; - BufferManager mBufferManager; - std::map<uint64_t, fs::path> mTaskPaths; - std::map<uint64_t, std::pair<TensorPtr, TensorPtr>> mLoras; - - std::map<uint64_t, std::pair<TensorPtr, TensorPtr>> readLoras(std::map<uint64_t, fs::path> taskPaths) - { - std::map<uint64_t, std::pair<TensorPtr, TensorPtr>> loras; - for (auto const& [id, p] : taskPaths) - { - TensorPtr loraWeights - = utils::loadNpy(mBufferManager, (p / "model.lora_weights.npy").string(), MemoryType::kCPU); - TensorPtr loraConfig - = utils::loadNpy(mBufferManager, (p / "model.lora_config.npy").string(), MemoryType::kCPU); - loras.insert_or_assign(id, std::make_pair(loraWeights, loraConfig)); - } - return loras; - } - - std::map<uint64_t, fs::path> parseDirPaths(std::string const& loraDir) - { - std::map<uint64_t, fs::path> taskPaths; - if (loraDir == "") - { - return taskPaths; - } - for (auto const& entry : fs::recursive_directory_iterator(loraDir)) - { - if (entry.is_directory()) - { - auto taskId = parseId(entry.path()); - taskPaths.insert_or_assign(taskId, entry.path()); - } - } - return taskPaths; - } - - uint64_t parseId(fs::path p) - { - auto fn = p.filename().string(); - auto dashPos = fn.find_first_of("-"); - std::string idStr = fn; - if (dashPos != std::string::npos) - { - auto idStr = fn.substr(0, dashPos); - } - uint64_t id = static_cast<uint64_t>(std::stoi(idStr)); - return id; - } -}; - -} // namespace - -struct BenchInfo -{ - BenchInfo() = default; - - BenchInfo(int inputLength, std::chrono::time_point<std::chrono::steady_clock> start) - : inputLength(inputLength) - , start(start) - { - } - - int inputLength; - int outputLength{0}; - std::chrono::time_point<std::chrono::steady_clock> start; - std::chrono::time_point<std::chrono::steady_clock> end; - std::chrono::time_point<std::chrono::steady_clock> firstTokenTs; - float latency{}; // millisecond - bool hasError{false}; - float firstTokenLatency{}; - std::optional<float> avgGenT2TLatency{}; - bool firstTokenSeen{false}; - SizeType32 decodingIter{0}; -}; - -class Recorder -{ - using TensorPtr = ITensor::SharedPtr; - -public: - explicit Recorder(std::string opCsvFile, bool streaming = false, int beamWidth = 1, - std::string responsesJsonFile = "", bool excludeInputInOutput = false) - : mOpCsvFile(std::move(opCsvFile)) - , mStreaming(streaming) - , mBeamWidth(beamWidth) - , mRespJsonFile(std::move(responsesJsonFile)) - , mOutputHasInput(!excludeInputInOutput) - { - } - - void initialize() - { - mStart = std::chrono::steady_clock::now(); - mRequestsQueueingLatencies.clear(); - } - - void finalize() - { - mEnd = std::chrono::steady_clock::now(); - } - - void recordQueueLatency(std::vector<float> const& latencies) - { - mRequestsQueueingLatencies.insert(mRequestsQueueingLatencies.end(), latencies.begin(), latencies.end()); - } - - // number of output tokens not calculated from output sequence here, instead set to max_output_len - // - if eos_id == -1 (default behavior), this is correct since output seq will have max permissible length. - // - However, if eos_id != -1, the token size of output sequence may be less than max_output_len, and token - // throughput may be inaccurate - void recordStart( - SizeType32 inputLength, uint64_t requestId, std::chrono::time_point<std::chrono::steady_clock> const& start) - { - TLLM_CHECK_WITH_INFO(mRequestBenchInfos.find(requestId) == mRequestBenchInfos.end(), - "Request %lu already exists in record before start, please report a bug to developers.", requestId); - std::lock_guard<std::mutex> const lock(mRequestBenchInfosMutex); - mRequestBenchInfos[requestId] = BenchInfo(inputLength, start); - } - - void recordToken( - texec::Response const& response, std::chrono::time_point<std::chrono::steady_clock> const& tokenTime) - { - auto const requestId = response.getRequestId(); - auto outputTokenIds = response.getResult().outputTokenIds; - - int32_t outputLength = 1; - for (auto const& beam : outputTokenIds) - { - outputLength = std::max(static_cast<int32_t>(beam.size()), outputLength); - } - - std::lock_guard<std::mutex> const lock(mRequestBenchInfosMutex); - mRequestBenchInfos[requestId].outputLength += outputLength; - - if (!mRequestBenchInfos[requestId].firstTokenSeen) - { - mRequestBenchInfos[requestId].firstTokenTs = tokenTime; - mRequestBenchInfos[requestId].firstTokenSeen = true; - } - - mRequestBenchInfos[requestId].decodingIter += 1; - } - - void recordEnd(texec::Response const& response, std::chrono::time_point<std::chrono::steady_clock> const& end) - { - auto const requestId = response.getRequestId(); - // Get the actual output length - if (!response.hasError()) - { - if (!mStreaming) - { - TLLM_LOG_DEBUG("response.getResult().outputTokenIds"); - auto outputTokenIds = response.getResult().outputTokenIds; - - int32_t outSeqLen = 0; - for (auto const& beam : outputTokenIds) - { - outSeqLen = std::max(static_cast<int32_t>(beam.size()), outSeqLen); - } - if (mOutputHasInput) - { - int inputSeqLen = mRequestBenchInfos[requestId].inputLength; - outSeqLen -= inputSeqLen; - } - std::lock_guard<std::mutex> const lock(mRequestBenchInfosMutex); - mRequestBenchInfos[requestId].outputLength = outSeqLen; - mRequestBenchInfos[requestId].decodingIter = response.getResult().decodingIter; - - // We record the first beam for the response file - mResponseTensors[requestId] = outputTokenIds[0]; - } - else - { - TLLM_CHECK_WITH_INFO(mBeamWidth == 1, "gptManagerBenchmark streaming mode does not support beam > 1"); - this->recordToken(response, end); - } - } - - std::lock_guard<std::mutex> const lock(mRequestBenchInfosMutex); - mRequestBenchInfos[requestId].end = end; - mRequestBenchInfos[requestId].hasError = response.hasError(); - } - - float calcPercentile(std::vector<float> const& latencies, int percentile) - { - int const index = static_cast<int>(std::ceil((percentile / 100.0) * latencies.size())) - 1; - return latencies[index]; - } - - void calculateLatencies() - { - for (auto& reqInfo : mRequestBenchInfos) - { - reqInfo.second.latency - = std::chrono::duration<float, std::milli>(reqInfo.second.end - reqInfo.second.start).count(); - if (mStreaming) - { - reqInfo.second.firstTokenLatency - = std::chrono::duration<float, std::milli>(reqInfo.second.firstTokenTs - reqInfo.second.start) - .count(); - if (reqInfo.second.outputLength > 1) - { - reqInfo.second.avgGenT2TLatency - = std::chrono::duration<float, std::milli>(reqInfo.second.end - reqInfo.second.firstTokenTs) - .count() - / static_cast<float>(reqInfo.second.outputLength - 1); - } - } - } - } - - void calculateMetrics() - { - calculateLatencies(); - - std::vector<float> reqLatencies; - std::vector<float> ftLatencies; - std::vector<float> genT2TLatencies; - std::vector<float> userTokensPerSecond; - - int totalOutputTokens{0}; - int totalDecodingIter{0}; - mNumErrorSamples = 0; - mNumSamples = 0; - for (auto reqInfo : mRequestBenchInfos) - { - if (!reqInfo.second.hasError) - { - reqLatencies.push_back(reqInfo.second.latency); - totalOutputTokens += reqInfo.second.outputLength; - totalDecodingIter += reqInfo.second.decodingIter; - - if (mStreaming) - { - ftLatencies.push_back(reqInfo.second.firstTokenLatency); - - if (reqInfo.second.avgGenT2TLatency) - { - genT2TLatencies.push_back(reqInfo.second.avgGenT2TLatency.value()); - } - if (reqInfo.second.avgGenT2TLatency.value() > 0) - { - userTokensPerSecond.push_back(1000.F / reqInfo.second.avgGenT2TLatency.value()); - } - } - ++mNumSamples; - } - else - { - ++mNumErrorSamples; - } - } - - mTotalLatency = std::chrono::duration<float, std::milli>(mEnd - mStart).count(); - mSeqThroughput = mNumSamples / (mTotalLatency / 1000); - mTokenThroughput = totalOutputTokens / (mTotalLatency / 1000); - mAcceptanceRate = totalDecodingIter - ? (static_cast<float>(totalOutputTokens) / static_cast<float>(totalDecodingIter)) - : 0.0f; - - mAvgSeqLatency = std::accumulate(reqLatencies.begin(), reqLatencies.end(), 0.F) / reqLatencies.size(); - - std::sort(reqLatencies.begin(), reqLatencies.end()); - - mP99SeqLatency = calcPercentile(reqLatencies, 99); - mP90SeqLatency = calcPercentile(reqLatencies, 90); - mP50SeqLatency = calcPercentile(reqLatencies, 50); - mMaxSeqLatency = reqLatencies.back(); - mMinSeqLatency = reqLatencies.front(); - - if (mStreaming) - { - mAvgFtLatency = std::accumulate(ftLatencies.begin(), ftLatencies.end(), 0.F) / ftLatencies.size(); - - std::sort(ftLatencies.begin(), ftLatencies.end()); - - mP99FtLatency = calcPercentile(ftLatencies, 99); - mP90FtLatency = calcPercentile(ftLatencies, 90); - mP50FtLatency = calcPercentile(ftLatencies, 50); - mMaxFtLatency = ftLatencies.back(); - mMinFtLatency = ftLatencies.front(); - - if (!genT2TLatencies.empty()) - { - mAvgGenT2TLatency - = std::accumulate(genT2TLatencies.begin(), genT2TLatencies.end(), 0.F) / genT2TLatencies.size(); - - std::sort(genT2TLatencies.begin(), genT2TLatencies.end()); - - mP99GenT2TLatency = calcPercentile(genT2TLatencies, 99); - mP90GenT2TLatency = calcPercentile(genT2TLatencies, 90); - mP50GenT2TLatency = calcPercentile(genT2TLatencies, 50); - mMaxGenT2TLatency = genT2TLatencies.back(); - mMinGenT2TLatency = genT2TLatencies.front(); - } - - if (!userTokensPerSecond.empty()) - { - mAvgUserTokensPerSecond = std::accumulate(userTokensPerSecond.begin(), userTokensPerSecond.end(), 0.F) - / userTokensPerSecond.size(); - std::sort(userTokensPerSecond.begin(), userTokensPerSecond.end()); - mP99UserTokensPerSecond = calcPercentile(userTokensPerSecond, 99); - mP90UserTokensPerSecond = calcPercentile(userTokensPerSecond, 90); - mP50UserTokensPerSecond = calcPercentile(userTokensPerSecond, 50); - mMaxUserTokensPerSecond = userTokensPerSecond.back(); - mMinUserTokensPerSecond = userTokensPerSecond.front(); - } - - mAvgReqQueueingLatency - = std::accumulate(mRequestsQueueingLatencies.begin(), mRequestsQueueingLatencies.end(), 0.F) - / mRequestsQueueingLatencies.size(); - std::sort(mRequestsQueueingLatencies.begin(), mRequestsQueueingLatencies.end()); - mP99ReqQueueingLatency = calcPercentile(mRequestsQueueingLatencies, 99); - mP90ReqQueueingLatency = calcPercentile(mRequestsQueueingLatencies, 90); - mP50ReqQueueingLatency = calcPercentile(mRequestsQueueingLatencies, 50); - mMaxReqQueueingLatency = mRequestsQueueingLatencies.back(); - mMinReqQueueingLatency = mRequestsQueueingLatencies.front(); - } - } - - void report() - { - - printf("[BENCHMARK] num_samples %d\n", mNumSamples); - printf("[BENCHMARK] num_error_samples %d\n", mNumErrorSamples); - printf("\n[BENCHMARK] num_samples %d\n", mNumSamples); - printf("[BENCHMARK] total_latency(ms) %.2f\n", mTotalLatency); - printf("[BENCHMARK] seq_throughput(seq/sec) %.2f\n", mSeqThroughput); - printf("[BENCHMARK] token_throughput(token/sec) %.2f\n", mTokenThroughput); - printf("[BENCHMARK] avg_acceptance_rate(tokens/decoding steps) %.2f\n\n", mAcceptanceRate); - - printf("[BENCHMARK] avg_sequence_latency(ms) %.2f\n", mAvgSeqLatency); - printf("[BENCHMARK] max_sequence_latency(ms) %.2f\n", mMaxSeqLatency); - printf("[BENCHMARK] min_sequence_latency(ms) %.2f\n", mMinSeqLatency); - printf("[BENCHMARK] p99_sequence_latency(ms) %.2f\n", mP99SeqLatency); - printf("[BENCHMARK] p90_sequence_latency(ms) %.2f\n", mP90SeqLatency); - printf("[BENCHMARK] p50_sequence_latency(ms) %.2f\n\n", mP50SeqLatency); - - if (mStreaming) - { - printf("[BENCHMARK] avg_time_to_first_token(ms) %.2f\n", mAvgFtLatency); - printf("[BENCHMARK] max_time_to_first_token(ms) %.2f\n", mMaxFtLatency); - printf("[BENCHMARK] min_time_to_first_token(ms) %.2f\n", mMinFtLatency); - printf("[BENCHMARK] p99_time_to_first_token(ms) %.2f\n", mP99FtLatency); - printf("[BENCHMARK] p90_time_to_first_token(ms) %.2f\n", mP90FtLatency); - printf("[BENCHMARK] p50_time_to_first_token(ms) %.2f\n\n", mP50FtLatency); - - printf("[BENCHMARK] avg_inter_token_latency(ms) %.2f\n", mAvgGenT2TLatency); - printf("[BENCHMARK] max_inter_token_latency(ms) %.2f\n", mMaxGenT2TLatency); - printf("[BENCHMARK] min_inter_token_latency(ms) %.2f\n", mMinGenT2TLatency); - printf("[BENCHMARK] p99_inter_token_latency(ms) %.2f\n", mP99GenT2TLatency); - printf("[BENCHMARK] p90_inter_token_latency(ms) %.2f\n", mP90GenT2TLatency); - printf("[BENCHMARK] p50_inter_token_latency(ms) %.2f\n\n", mP50GenT2TLatency); - - printf("[BENCHMARK] avg_user_tokens_per_second(tokens/sec/user) %.2f\n", mAvgUserTokensPerSecond); - printf("[BENCHMARK] max_user_tokens_per_second(tokens/sec/user) %.2f\n", mMaxUserTokensPerSecond); - printf("[BENCHMARK] min_user_tokens_per_second(tokens/sec/user) %.2f\n", mMinUserTokensPerSecond); - printf("[BENCHMARK] p99_user_tokens_per_second(tokens/sec/user) %.2f\n", mP99UserTokensPerSecond); - printf("[BENCHMARK] p90_user_tokens_per_second(tokens/sec/user) %.2f\n", mP90UserTokensPerSecond); - printf("[BENCHMARK] p50_user_tokens_per_second(tokens/sec/user) %.2f\n\n", mP50UserTokensPerSecond); - - printf("[BENCHMARK] avg_request_queueing_latency(ms) %.2f\n", mAvgReqQueueingLatency); - printf("[BENCHMARK] max_request_queueing_latency(ms) %.2f\n", mMaxReqQueueingLatency); - printf("[BENCHMARK] min_request_queueing_latency(ms) %.2f\n", mMinReqQueueingLatency); - printf("[BENCHMARK] p99_request_queueing_latency(ms) %.2f\n", mP99ReqQueueingLatency); - printf("[BENCHMARK] p90_request_queueing_latency(ms) %.2f\n", mP90ReqQueueingLatency); - printf("[BENCHMARK] p50_request_queueing_latency(ms) %.2f\n\n", mP50ReqQueueingLatency); - } - } - - void writeOpMetricsToCsv() - { - if (!mOpCsvFile.empty()) - { - std::vector<std::string> headers = {"num_samples", "num_error_samples", "total_latency(ms)", - "seq_throughput(seq/sec)", "token_throughput(token/sec)", "avg_sequence_latency(ms)", - "max_sequence_latency(ms)", "min_sequence_latency(ms)", "p99_sequence_latency(ms)", - "p90_sequence_latency(ms)", "p50_sequence_latency(ms)", "avg_acceptance_rate(tokens/decoding steps)"}; - - if (mStreaming) - { - std::vector<std::string> streamingHeaders = { - "avg_time_to_first_token(ms)", - "max_time_to_first_token(ms)", - "min_time_to_first_token(ms)", - "p99_time_to_first_token(ms)", - "p90_time_to_first_token(ms)", - "p50_time_to_first_token(ms)", - "avg_inter_token_latency(ms)", - "max_inter_token_latency(ms)", - "min_inter_token_latency(ms)", - "p99_inter_token_latency(ms)", - "p90_inter_token_latency(ms)", - "p50_inter_token_latency(ms)", - "avg_user_tokens_per_second(tokens/sec/user)", - "max_user_tokens_per_second(tokens/sec/user)", - "min_user_tokens_per_second(tokens/sec/user)", - "p99_user_tokens_per_second(tokens/sec/user)", - "p90_user_tokens_per_second(tokens/sec/user)", - "p50_user_tokens_per_second(tokens/sec/user)", - }; - - headers.insert(headers.end(), streamingHeaders.begin(), streamingHeaders.end()); - } - - std::ofstream outputFile(mOpCsvFile); - - if (outputFile.is_open()) - { - for (auto const& header : headers) - { - outputFile << header << ","; - } - outputFile << "\n"; - outputFile << mNumSamples << "," << mNumErrorSamples << "," << mTotalLatency << "," << mSeqThroughput - << "," << mTokenThroughput << "," << mAvgSeqLatency << "," << mMaxSeqLatency << "," - << mMinSeqLatency << "," << mP99SeqLatency << "," << mP90SeqLatency << "," << mP50SeqLatency - << "," << mAcceptanceRate; - if (mStreaming) - { - outputFile << "," << mAvgFtLatency << "," << mMaxFtLatency << "," << mMinFtLatency << "," - << mP99FtLatency << "," << mP90FtLatency << "," << mP50FtLatency << "," - << mAvgGenT2TLatency << "," << mMaxGenT2TLatency << "," << mMinGenT2TLatency << "," - << mP99GenT2TLatency << "," << mP90GenT2TLatency << "," << mP50GenT2TLatency << "," - << mAvgUserTokensPerSecond << "," << mMaxUserTokensPerSecond << "," - << mMinUserTokensPerSecond << "," << mP99UserTokensPerSecond << "," - << mP90UserTokensPerSecond << "," << mP50UserTokensPerSecond << ","; - } - - outputFile << "\n"; - } - else - { - std::cerr << "Error opening file '" << mOpCsvFile << "' for writing.\n"; - } - } - } - - void dumpResponseSeqs() - { - if (mRespJsonFile.empty()) - return; - nlohmann::json jsonResponses = nlohmann::json::array(); - for (auto const& [respId, respTokensTensor] : mResponseTensors) - { - auto respTokens = mResponseTensors[respId]; - int respLength = respTokens.size(); - int* respBufferPtr = respTokens.data(); - - if (mOutputHasInput) - { - int inputSeqLen = mRequestBenchInfos[respId].inputLength; - respBufferPtr += inputSeqLen; - respLength -= inputSeqLen; - } - - std::vector<int32_t> outputTokens(respLength); - std::copy(respBufferPtr, respBufferPtr + respLength, outputTokens.begin()); - - nlohmann::json currResp; - currResp["response_id"] = respId; - currResp["response_tokens"] = outputTokens; - jsonResponses.push_back(currResp); - } - std::ofstream outFile(mRespJsonFile); - outFile << jsonResponses; - outFile.close(); - } - -private: - std::unordered_map<uint64_t, BenchInfo> mRequestBenchInfos; - - std::chrono::time_point<std::chrono::steady_clock> mStart; - std::chrono::time_point<std::chrono::steady_clock> mEnd; - int mNumSamples{}; - int mNumErrorSamples{}; - float mTotalLatency{}; - float mSeqThroughput{}; - float mAvgSeqLatency{}; - float mAvgGenT2TLatency{}; - float mAvgUserTokensPerSecond{}; - float mAvgFtLatency{}; - float mTokenThroughput{}; - float mAcceptanceRate{}; - float mP99SeqLatency{}; - float mP90SeqLatency{}; - float mP50SeqLatency{}; - float mMaxSeqLatency{}; - float mMinSeqLatency{}; - float mP99FtLatency{}; - float mP90FtLatency{}; - float mP50FtLatency{}; - float mMaxFtLatency{}; - float mMinFtLatency{}; - float mP99GenT2TLatency{}; - float mP90GenT2TLatency{}; - float mP50GenT2TLatency{}; - float mMaxGenT2TLatency{}; - float mMinGenT2TLatency{}; - float mP99UserTokensPerSecond{}; - float mP90UserTokensPerSecond{}; - float mP50UserTokensPerSecond{}; - float mMaxUserTokensPerSecond{}; - float mMinUserTokensPerSecond{}; - float mAvgReqQueueingLatency{}; - float mP99ReqQueueingLatency{}; - float mP90ReqQueueingLatency{}; - float mP50ReqQueueingLatency{}; - float mMaxReqQueueingLatency{}; - float mMinReqQueueingLatency{}; - std::vector<float> mRequestsQueueingLatencies{}; - - std::string mOpCsvFile; - bool mStreaming; - int mBeamWidth; - std::string mRespJsonFile; - std::unordered_map<uint64_t, texec::VecTokens> mResponseTensors; - bool mOutputHasInput; - std::mutex mRequestBenchInfosMutex; - -}; // class Recorder - -class ExecutorServer -{ -public: - ExecutorServer(std::optional<std::filesystem::path> const& decoderTrtEnginePath, - std::optional<std::filesystem::path> const& encoderTrtEnginePath, texec::BatchingType batchingType, - int32_t maxBeamWidth, texec::CapacitySchedulerPolicy capacitySchedulerPolicy, - BenchmarkParams const& benchmarkParams, std::shared_ptr<Recorder> recorder, std::chrono::milliseconds waitSleep, - bool logIterationData, texec::ModelType executorModelType) - : mRecorder(std::move(recorder)) - , mWaitSleep(waitSleep) - , mConcurrency(benchmarkParams.concurrency) - , mActiveCount(0) - , mNumFinished(0) - , mShutdown(false) - , mLogIterationData(logIterationData) - { - texec::DynamicBatchConfig dynamicBatchConfig( - benchmarkParams.enableBatchSizeTuning, benchmarkParams.enableMaxNumTokensTuning); - texec::SchedulerConfig schedulerConfig(capacitySchedulerPolicy, std::nullopt, dynamicBatchConfig); - - texec::KvCacheConfig kvCacheConfig(benchmarkParams.enableBlockReuse, benchmarkParams.maxTokensInPagedKvCache, - benchmarkParams.maxAttentionWindowVec, benchmarkParams.sinkTokenLength, - benchmarkParams.freeGpuMemoryFraction, benchmarkParams.kvHostCacheSize, - benchmarkParams.crossKvCacheFraction); - texec::PeftCacheConfig peftCacheConfig(0, benchmarkParams.loraDeviceNumModLayers, 8, 64, 4, 4, 4, 24, 8, - std::nullopt, benchmarkParams.loraHostCacheSize); - texec::ExtendedRuntimePerfKnobConfig extendedRuntimePerfKnobConfig(benchmarkParams.multiBlockMode, - benchmarkParams.enableContextFMHAFP32Acc, benchmarkParams.cudaGraphMode, - benchmarkParams.cudaGraphCacheSize); - texec::ExecutorConfig executorConfig( - maxBeamWidth, schedulerConfig, kvCacheConfig, benchmarkParams.enableChunkedContext, true); - executorConfig.setEnableTrtOverlap(benchmarkParams.enableTrtOverlap); - executorConfig.setGpuWeightsPercent(benchmarkParams.gpuWeightsPercent); - executorConfig.setPeftCacheConfig(peftCacheConfig); - executorConfig.setBatchingType(batchingType); - if (benchmarkParams.maxBatchSize) - { - executorConfig.setMaxBatchSize(benchmarkParams.maxBatchSize.value()); - } - if (benchmarkParams.maxNumTokens) - { - executorConfig.setMaxNumTokens(benchmarkParams.maxNumTokens.value()); - } - - auto decodingMode = texec::DecodingMode::Auto(); - if (benchmarkParams.medusaChoices.has_value()) - { - decodingMode = texec::DecodingMode::Medusa(); - } - else if (benchmarkParams.executorLookaheadConfig.has_value()) - { - decodingMode = texec::DecodingMode::Lookahead(); - } - else if (benchmarkParams.eagleConfig.has_value()) - { - decodingMode = texec::DecodingMode::Eagle(); - } - - executorConfig.setDecodingConfig(texec::DecodingConfig(decodingMode, benchmarkParams.executorLookaheadConfig, - benchmarkParams.medusaChoices, benchmarkParams.eagleConfig)); - executorConfig.setExtendedRuntimePerfKnobConfig(extendedRuntimePerfKnobConfig); - - if (executorModelType == texec::ModelType::kDECODER_ONLY) - { - mExecutor - = std::make_unique<texec::Executor>(decoderTrtEnginePath.value(), executorModelType, executorConfig); - } - else if (executorModelType == texec::ModelType::kENCODER_DECODER) - { - mExecutor = std::make_unique<texec::Executor>( - encoderTrtEnginePath.value(), decoderTrtEnginePath.value(), executorModelType, executorConfig); - } - else if (executorModelType == texec::ModelType::kENCODER_ONLY) - { - mExecutor - = std::make_unique<texec::Executor>(encoderTrtEnginePath.value(), executorModelType, executorConfig); - } - else - { - TLLM_LOG_ERROR("not a supported executor model type in executor server."); - } - - auto const& world = tensorrt_llm::mpi::MpiComm::world(); - auto worldRank = world.getRank(); - if (worldRank == 0) - { - mCollectStatsThread = std::thread(&ExecutorServer::collectStats, this); - } - } - - ~ExecutorServer() - { - mShutdown = true; - if (mCollectStatsThread.joinable()) - { - mCollectStatsThread.join(); - } - } - - void enqueue(std::vector<texec::Request> requests, bool warmup = false) - { - try - { - std::vector<SizeType32> inputLengths; - for (auto const& request : requests) - { - inputLengths.push_back(request.getInputTokenIds().size()); - } - auto const start = std::chrono::steady_clock::now(); - auto reqIds = mExecutor->enqueueRequests(std::move(requests)); - for (int req = 0; req < reqIds.size(); ++req) - { - if (!warmup) - { - mRecorder->recordStart(inputLengths.at(req), reqIds.at(req), start); - } - mActiveCount++; - } - } - catch (std::exception const& e) - { - TLLM_THROW("%s", e.what()); - } - } - - void resetNumFinished() - { - mNumFinished = 0; - } - - bool canEnqueue(int numSentRequests) const - { - return !mConcurrency || (numSentRequests - mNumFinished < mConcurrency); - } - - void waitForResponses(SizeType32 numRequests, bool warmup = false) - { - while (mActiveCount || (mNumFinished < numRequests)) - { - auto responses = mExecutor->awaitResponses(mWaitSleep); - auto const tokenTime = std::chrono::steady_clock::now(); - for (auto const& response : responses) - { - if (response.getResult().isFinal) - { - mActiveCount--; - mNumFinished++; - if (!warmup) - { - mRecorder->recordEnd(response, tokenTime); - } - } - else - { - if (!warmup && !response.hasError()) - { - mRecorder->recordToken(response, tokenTime); - } - } - } - } - } - - void collectStats() const - { - while (!mShutdown) - { - auto iterStats = mExecutor->getLatestIterationStats(); - for (auto const& iterStat : iterStats) - { - SizeType32 numNewActiveRequests = iterStat.numNewActiveRequests; - if (numNewActiveRequests > 0) - { - float avgQueueingTime - = static_cast<float>(iterStat.newActiveRequestsQueueLatencyMS / numNewActiveRequests); - std::vector<float> requestsQueueLatencyMS(numNewActiveRequests, avgQueueingTime); - mRecorder->recordQueueLatency(requestsQueueLatencyMS); - } - if (mLogIterationData) - { - TLLM_LOG_INFO(texec::JsonSerialization::toJsonStr(iterStat)); - } - } - auto const waitSleep = std::chrono::milliseconds(50); - std::this_thread::sleep_for(waitSleep); - } - } - -private: - std::unique_ptr<texec::Executor> mExecutor; - std::thread mCollectStatsThread; - std::shared_ptr<Recorder> mRecorder; - std::chrono::milliseconds mWaitSleep; - std::optional<int> mConcurrency; - std::atomic<uint64_t> mActiveCount; - std::atomic<uint64_t> mNumFinished; - std::atomic<bool> mShutdown; - bool mLogIterationData; -}; // class ExecutorServer - -namespace -{ - -texec::Request makeExecutorRequest(Sample const& sample, SizeType32 const& beamWidth, - std::optional<SizeType32> const& eosId, std::optional<SizeType32> const& padId, bool streaming = false, - bool const& returnContextLogits = false, bool const& returnGenerationLogits = false, - std::optional<texec::LoraConfig> const& loraConfig = std::nullopt, - std::optional<texec::LookaheadDecodingConfig> const& lookaheadConfig = std::nullopt, - std::optional<texec::VecTokens> encoderInputTokenIds = std::nullopt, - std::optional<float> temperature = std::nullopt) -{ - auto samplingConfig = texec::SamplingConfig{beamWidth}; - samplingConfig.setTemperature(temperature); - auto outputConfig = texec::OutputConfig{false, returnContextLogits, returnGenerationLogits, false}; - return texec::Request(sample.inputIds, sample.outputLen, streaming, samplingConfig, outputConfig, eosId, padId, - std::nullopt, // positionIds - std::nullopt, // badWords - std::nullopt, // stopWords - std::nullopt, // embeddingBias - std::nullopt, // speculativeDecoding - std::nullopt, // pTuning - std::nullopt, // multimodalInput - std::nullopt, // multimodalEmbedding - std::nullopt, // mRopeConfig - loraConfig, // loraConfig - lookaheadConfig, // lookaheadConfig - std::nullopt, // kvCacheRetentionConfig - std::nullopt, // logitsPostProcessorName - std::nullopt, // logitsPostProcessor - encoderInputTokenIds.has_value() ? encoderInputTokenIds : std::nullopt, - std::nullopt); // cacheSalt -} - -void benchmarkExecutor(std::optional<std::filesystem::path> const& decoderEngineDir, - std::optional<std::filesystem::path> const& encoderEngineDir, texec::BatchingType batchingType, - std::string const& datasetPath, std::string const& opCsvFile, int maxNumSamples, int beamWidth, int warmUp, - std::optional<int32_t> const& eosId, std::optional<int32_t> const& padId, BenchmarkParams const& benchmarkParams, - texec::CapacitySchedulerPolicy capacitySchedulerPolicy, std::chrono::milliseconds waitSleep, - bool returnContextLogits, bool returnGenerationLogits, std::optional<int> const staticEmulatedBatchSize, - bool logIterationData, std::optional<SizeType32> const maxPromptLen, texec::ModelType executorModelType, - std::string const& responsesJsonFile) -{ - auto const& world = tensorrt_llm::mpi::MpiComm::world(); - auto worldRank = world.getRank(); - - // Load dataset - auto const samples = parseWorkloadJson(datasetPath, maxNumSamples, maxPromptLen); - auto const numSamples = samples.size(); - - auto recorder = std::make_shared<Recorder>(opCsvFile, benchmarkParams.streaming, beamWidth, responsesJsonFile); - int32_t decoderStartTokenId = 0; - std::shared_ptr<ExecutorServer> executorServer; - - if (executorModelType == texec::ModelType::kDECODER_ONLY) - { - TLLM_CHECK_WITH_INFO( - decoderEngineDir.has_value(), "decoder models require a path to decoder engine in executor benchmark."); - executorServer - = std::make_shared<ExecutorServer>(decoderEngineDir.value(), std::nullopt, batchingType, beamWidth, - capacitySchedulerPolicy, benchmarkParams, recorder, waitSleep, logIterationData, executorModelType); - } - else if (executorModelType == texec::ModelType::kENCODER_DECODER) - { - TLLM_CHECK_WITH_INFO(encoderEngineDir.has_value(), - "encoder-decoder models require a path to encoder engine in executor benchmark."); - executorServer = std::make_shared<ExecutorServer>(decoderEngineDir.value(), encoderEngineDir.value(), - batchingType, beamWidth, capacitySchedulerPolicy, benchmarkParams, recorder, waitSleep, logIterationData, - executorModelType); - try - { - std::ifstream decoderJsonConfigPath(decoderEngineDir.value() / "config.json"); - auto const decoderPretrainedConfig - = nlohmann::json::parse(decoderJsonConfigPath, nullptr, true, true).at("pretrained_config"); - decoderStartTokenId = decoderPretrainedConfig.at("decoder_start_token_id").template get<int32_t>(); - } - catch (nlohmann::json::out_of_range& e) - { - TLLM_LOG_ERROR( - "Parameter %s cannot be read from decoder config.json in pretrained_config. Using default id %d.", - std::string("decoder_start_token_id").c_str(), decoderStartTokenId); - } - catch (nlohmann::json::type_error const& e) - { - TLLM_LOG_ERROR( - "Parameter %s has error type in decoder config.json in pretrained_config. Using default id %d.", - std::string("decoder_start_token_id").c_str(), decoderStartTokenId); - } - } - else if (executorModelType == texec::ModelType::kENCODER_ONLY) - { - TLLM_CHECK_WITH_INFO( - encoderEngineDir.has_value(), "encoder models require a path to encoder engine in executor benchmark."); - executorServer - = std::make_shared<ExecutorServer>(std::nullopt, encoderEngineDir.value(), batchingType, beamWidth, - capacitySchedulerPolicy, benchmarkParams, recorder, waitSleep, logIterationData, executorModelType); - } - else - { - TLLM_LOG_ERROR("not a supported executor model type in executor benchmark."); - return; - } - - if (worldRank == 0) - { - if (benchmarkParams.loraDir) - { - auto startLoraLoad = std::chrono::steady_clock::now(); - LoraLib loras(benchmarkParams.loraDir.value()); - std::vector<texec::Request> requests; - for (auto& [taskId, p] : loras.getLoras()) - { - // squeeze lora configs and weights since LoraConfig requires them to be 2D tensors - p.first->squeeze(0); - p.second->squeeze(0); - texec::LoraConfig loraConfig( - taskId, texec::detail::ofITensor(p.first), texec::detail::ofITensor(p.second)); - if (executorModelType == texec::ModelType::kENCODER_DECODER) - { - Sample s{std::vector<int32_t>{decoderStartTokenId}, 1, static_cast<int32_t>(taskId)}; - requests.emplace_back(makeExecutorRequest(s, beamWidth, eosId, padId, false, false, false, - loraConfig, std::nullopt, std::vector<int32_t>{1, 2, 3, 4, 5})); - } - else - { - Sample s{std::vector<int32_t>{1, 2, 3, 4, 5}, 1, static_cast<int32_t>(taskId)}; - requests.emplace_back( - makeExecutorRequest(s, beamWidth, eosId, padId, false, false, false, loraConfig, std::nullopt)); - } - } - executorServer->enqueue(std::move(requests), true); - executorServer->waitForResponses(loras.getLoras().size(), true); - auto endLoraLoad = std::chrono::steady_clock::now(); - printf("[BENCHMARK] time to preload LoRAs(ms) %.2f\n", - std::chrono::duration<float, std::milli>(endLoraLoad - startLoraLoad).count()); - } - // Warm up - { - std::vector<texec::Request> requests; - for (auto i = 0; i < warmUp; ++i) - { - if (executorModelType == texec::ModelType::kENCODER_DECODER) - { - Sample s{std::vector<int32_t>{decoderStartTokenId}, samples[0].outputLen, samples[0].taskId}; - requests.emplace_back(makeExecutorRequest(s, beamWidth, eosId, padId, benchmarkParams.streaming, - returnContextLogits, returnGenerationLogits, std::nullopt, - benchmarkParams.requestLookaheadConfig, samples[0].inputIds)); - } - else - { - requests.emplace_back(makeExecutorRequest(samples[0], beamWidth, eosId, padId, - benchmarkParams.streaming, returnContextLogits, returnGenerationLogits, std::nullopt, - benchmarkParams.requestLookaheadConfig, std::nullopt, benchmarkParams.temperature)); - } - } - executorServer->enqueue(std::move(requests), true); - executorServer->waitForResponses(warmUp, true); - } - - // Benchmark - { - auto timeDelays = computeTimeDelays(benchmarkParams, numSamples - 1); - - // Create requests - recorder->initialize(); - std::vector<texec::Request> requests; - - for (std::size_t i = 0; i < numSamples; ++i) - { - std::optional<texec::LoraConfig> loraConfig; - if (samples[i].taskId >= 0) - { - loraConfig = texec::LoraConfig(samples[i].taskId); - } - if (executorModelType == texec::ModelType::kENCODER_DECODER) - { - Sample s{std::vector<int32_t>{decoderStartTokenId}, samples[i].outputLen, samples[i].taskId}; - requests.emplace_back(makeExecutorRequest(s, beamWidth, eosId, padId, benchmarkParams.streaming, - returnContextLogits, returnGenerationLogits, loraConfig, benchmarkParams.requestLookaheadConfig, - samples[i].inputIds)); - } - else - { - requests.emplace_back(makeExecutorRequest(samples[i], beamWidth, eosId, padId, - benchmarkParams.streaming, returnContextLogits, returnGenerationLogits, loraConfig, - benchmarkParams.requestLookaheadConfig, std::nullopt, benchmarkParams.temperature)); - } - } - - bool const hasDelay - = std::any_of(timeDelays.begin(), timeDelays.end(), [](auto const& delay) { return delay > 0.0; }); - executorServer->resetNumFinished(); - if (!staticEmulatedBatchSize) - { - // Launch a thread that will wait for responses - std::thread waitThread( - [numSamples, executorServer]() { executorServer->waitForResponses(numSamples); }); - - // Enqueue requests one by one - int numSentRequests = 0; - while (numSentRequests < numSamples) - { - if (executorServer->canEnqueue(numSentRequests)) - { - executorServer->enqueue({requests.at(numSentRequests)}); - if (hasDelay && numSentRequests < numSamples - 1) - { - std::this_thread::sleep_for( - std::chrono::milliseconds(static_cast<int>(timeDelays.at(numSentRequests) * 1000))); - } - numSentRequests += 1; - } - } - waitThread.join(); - } - else - { - TLLM_CHECK_WITH_INFO( - !hasDelay, "Executor benchmark doesn't support delays with emulated static batch sizes"); - SizeType32 numRequests = requests.size(); - SizeType32 maxBatchSize = staticEmulatedBatchSize.value(); - for (SizeType32 req = 0; req < numRequests; req += maxBatchSize) - { - auto batchSize = std::min(maxBatchSize, numRequests - req); - - std::vector<texec::Request> requestsBatch(std::make_move_iterator(requests.begin() + req), - std::make_move_iterator(requests.begin() + req + batchSize)); - // Enqueue in batches - executorServer->enqueue(std::move(requestsBatch)); - // Wait for current batch to be done - executorServer->waitForResponses(batchSize); - } - } - } - recorder->finalize(); - recorder->calculateMetrics(); - recorder->report(); - recorder->writeOpMetricsToCsv(); - recorder->dumpResponseSeqs(); - // Send terminateReqId to terminate servers on all ranks - // Sever on rank 0 will broadcast the terminate signal to other servers on multi-GPU cases - } -} - -} // namespace - -int main(int argc, char* argv[]) -{ - cxxopts::Options options( - "TensorRT LLM BatchManager Benchmark", "TensorRT LLM BatchManager Benchmark for GPT and GPT-like models."); - options.add_options()("h,help", "Print usage"); - options.add_options()("engine_dir, decoder_engine_dir", "Directory that store the engines of decoder models.", - cxxopts::value<std::string>()); - options.add_options()( - "encoder_engine_dir", "Directory that store the engines of the encoder models.", cxxopts::value<std::string>()); - options.add_options()( - "api", "API type: gptManager or executor.", cxxopts::value<std::string>()->default_value("executor")); - options.add_options()("type", - "Batching type: choose between inflight/static. (IFB/V1 options are going to be deprecated)", - cxxopts::value<std::string>()->default_value("inflight")); - options.add_options()("dataset", "Dataset that is used for benchmarking BatchManager.", - cxxopts::value<std::string>()->default_value("")); - options.add_options()( - "output_csv", "Write output metrics to CSV", cxxopts::value<std::string>()->default_value("")); - options.add_options()("max_num_samples", "maximum number of samples to use from dataset/generate", - cxxopts::value<int>()->default_value("100000")); - options.add_options()( - "beam_width", "Specify beam width you want to benchmark.", cxxopts::value<int>()->default_value("1")); - options.add_options()( - "warm_up", "Specify warm up iterations before benchmark starts.", cxxopts::value<int>()->default_value("2")); - options.add_options()( - "eos_id", "Specify the end-of-sequence token id.", cxxopts::value<TokenIdType>()->default_value("-1")); - options.add_options()("pad_id", "Specify the padding token id.", cxxopts::value<TokenIdType>()); - options.add_options()("max_tokens_in_paged_kvcache", "Max tokens in paged K-V Cache.", cxxopts::value<int>()); - options.add_options()( - "max_attention_window", "Max KV cache length per sequence", cxxopts::value<std::vector<int>>()); - options.add_options()("sink_token_len", "Sink token length in kv cache per sequence.", cxxopts::value<int>()); - options.add_options()( - "random_seed", "integer random seed for exponential time delays.", cxxopts::value<int>()->default_value("420")); - options.add_options()( - "kv_cache_free_gpu_mem_fraction", "K-V Cache Free Gpu Mem Fraction.", cxxopts::value<float>()); - options.add_options()( - "cross_kv_cache_fraction", "Cross K-V Cache Fraction (from 0.0 to 1.0).", cxxopts::value<float>()); - options.add_options()("request_rate", - "request rate in reqs/sec. Skipping this arg or negative value will trigger offline/0-delay.", - cxxopts::value<float>()); - options.add_options()("concurrency", "Concurrent number of connections with the server.", cxxopts::value<int>()); - options.add_options()("max_batch_size", "The max runtime batch size when benchmarking", cxxopts::value<int>()); - options.add_options()( - "max_num_tokens", "The max runtime number of tokens per batch when benchmarking", cxxopts::value<int>()); - options.add_options()( - "enable_batch_size_tuning", "Dynamic tuning of batch size", cxxopts::value<bool>()->default_value("false")); - options.add_options()("enable_max_num_tokens_tuning", "Dynamic tuning of max num tokens", - cxxopts::value<bool>()->default_value("false")); - options.add_options()("enable_exp_delays", "Enables exponential delay distr to mimic real world request arrival", - cxxopts::value<bool>()->default_value("false")); - options.add_options()("streaming", - "Operate in streaming mode. Note: it reflects time-to-first-token and inter-token-latency", - cxxopts::value<bool>()->default_value("false")); - options.add_options()( - "enable_kv_cache_reuse", "Enables the KV cache reuse.", cxxopts::value<bool>()->default_value("true")); - options.add_options()( - "enable_chunked_context", "Whether to enable context chunking.", cxxopts::value<bool>()->default_value("true")); - options.add_options()( - "return_context_logits", "Whether to return context logits.", cxxopts::value<bool>()->default_value("false")); - options.add_options()("return_generation_logits", "Whether to return generation logits.", - cxxopts::value<bool>()->default_value("false")); - - options.add_options()("scheduler_policy", - "Choose scheduler policy between max_utilization/guaranteed_no_evict/static_batch.", - cxxopts::value<std::string>()->default_value("guaranteed_no_evict")); - - options.add_options()("static_emulated_batch_size", - "Emulate static batching performance with the provided batch size.", cxxopts::value<SizeType32>()); - options.add_options()("log_level", "Choose log level between verbose/info/warning/error/internal_error.", - cxxopts::value<std::string>()->default_value("warning")); - options.add_options()("log_iteration_data", "On each decoder iteration, print batch state metadata.", - cxxopts::value<bool>()->default_value("false")); - options.add_options()("wait_sleep", "Specify how many milliseconds to sleep each iteration of waitForEmpty loop.", - cxxopts::value<int>()->default_value("25")); - options.add_options()("lora_dir", "Directory containing LoRAs", cxxopts::value<std::string>()->default_value("")); - options.add_options()("lora_host_cache_bytes", "LoRA host cache memory in bytes", cxxopts::value<size_t>()); - options.add_options()("lora_num_device_mod_layers", "LoRA number 1d cache rows", cxxopts::value<int>()); - options.add_options()("kv_host_cache_bytes", - "Size of secondary memory pool used for offloading kv cache blocks (in bytes).", - cxxopts::value<size_t>()->default_value("0")); - options.add_options()( - "max_prompt_len", "Truncate all prompts from dataset to the length specified.", cxxopts::value<SizeType32>()); - - options.add_options()("gpu_weights_percent", - "Specify the percentage of weights that reside on GPU (from 0.0 to 1.0).", - cxxopts::value<float>()->default_value("1.0")); - options.add_options()( - "medusa_choices", "Medusa choices in the format of [[0], [0, 1], [0, 0, 1]]", cxxopts::value<std::string>()); - options.add_options()( - "eagle_choices", "Eagle choices in the format of [[0], [0, 1], [0, 0, 1]]", cxxopts::value<std::string>()); - options.add_options()("eagle_posterior_threshold", - "Minimum token probability threshold for typical acceptance. Enables typical acceptance in Eagle", - cxxopts::value<float>()); - options.add_options()("temperature", "Sampling temperature for each request", cxxopts::value<float>()); - options.add_options()( - "eagle_use_dynamic_tree", "Whether to use Eagle-2", cxxopts::value<bool>()->default_value("false")); - options.add_options()("eagle_dynamic_tree_max_top_k", - "The max topK for dynamic tree, also the number of draft tokens that will expand for each node", - cxxopts::value<SizeType32>()); - - options.add_options()("multi_block_mode", - "Distribute the work across multiple CUDA thread-blocks on the GPU for masked MHA kernel", - cxxopts::value<bool>()->default_value("true")); - options.add_options()("cuda_graph_mode", "When enabled, inference is executed with cuda graph.", - cxxopts::value<bool>()->default_value("false")); - options.add_options()("cuda_graph_cache_size", - "Specify how many cuda graphs are cached in the runtime. Larger cache gives better perf, but consumes more GPU " - "memory.", - cxxopts::value<SizeType32>()->default_value("0")); - options.add_options()("enable_trt_overlap", "Enable TRT Overlap", cxxopts::value<bool>()->default_value("false")); - - options.add_options()("enable_context_fmha_fp32_acc", "Enable FMHA runner FP32 accumulation", - cxxopts::value<bool>()->default_value("false")); - options.add_options()("executor_lookahead_config", - "lookahead config in the format of [max_window_size, max_ngram_size, max_verification_set_size]", - cxxopts::value<std::string>()); - options.add_options()("request_lookahead_config", - "lookahead config in the format of [max_window_size, max_ngram_size, max_verification_set_size], and each <= " - "executor lookahead config", - cxxopts::value<std::string>()); - options.add_options()("responses_json", "Write output response sequences to a json file", - cxxopts::value<std::string>()->default_value("")); - - auto result = options.parse(argc, argv); - - if (result.count("help")) - { - std::cout << options.help() << std::endl; - return 0; - } - - // Argument: Engine directory - if (!result.count("engine_dir") && !result.count("encoder_engine_dir")) - { - std::cout << options.help() << std::endl; - TLLM_LOG_ERROR("Please specify engine directory."); - return 1; - } - - // Argument: Batching Type - auto const type = result["type"].as<std::string>(); - texec::BatchingType batchingType{texec::BatchingType::kINFLIGHT}; - if (type == "V1" || type == "static") - { - if (type == "V1") - { - TLLM_LOG_WARNING("type option \"V1\" is going to be renamed to \"static\"."); - } - bool streaming = result["streaming"].as<bool>(); - if (streaming) - { - TLLM_LOG_ERROR("Streaming is not supported in static batching.\n"); - return 1; - } - batchingType = texec::BatchingType::kSTATIC; - } - else if (type == "IFB" || type == "inflight") - { - if (type == "IFB") - { - TLLM_LOG_WARNING("type option \"IFB\" is going to be renamed to \"inflight\"."); - } - batchingType = texec::BatchingType::kINFLIGHT; - } - else - { - TLLM_LOG_ERROR("Unexpected batching type: %s", type.c_str()); - return 1; - } - - // Argument: Dataset - auto const datasetPath = result["dataset"].as<std::string>(); - auto const maxNumSamples = result["max_num_samples"].as<int>(); - - // Argument: Output metrics CSV - auto const opCsvFile = result["output_csv"].as<std::string>(); - - // Argument: beam width - auto const beamWidth = result["beam_width"].as<int>(); - - // Argument: wait_sleep - auto const waitSleep = std::chrono::milliseconds(result["wait_sleep"].as<int>()); - BenchmarkParams benchmarkParams; - - // Argument: Max tokens in paged K-V Cache - if (result.count("max_tokens_in_paged_kvcache")) - { - benchmarkParams.maxTokensInPagedKvCache = result["max_tokens_in_paged_kvcache"].as<int>(); - } - - // Argument: Max KV cache length - if (result.count("max_attention_window")) - { - benchmarkParams.maxAttentionWindowVec = result["max_attention_window"].as<std::vector<int>>(); - } - - // Argument: Sink token length - if (result.count("sink_token_len")) - { - benchmarkParams.sinkTokenLength = result["sink_token_len"].as<int>(); - } - - if (result.count("random_seed")) - { - benchmarkParams.randomSeed = result["random_seed"].as<int>(); - } - - // Argument: K-V Cache Free Gpu Mem Fraction - if (result.count("kv_cache_free_gpu_mem_fraction")) - { - benchmarkParams.freeGpuMemoryFraction = result["kv_cache_free_gpu_mem_fraction"].as<float>(); - } - // Argument: K-V Cache Cross Attention Fraction. Only applicable to enc-dec models. - if (result.count("encoder_engine_dir") && result.count("decoder_engine_dir")) - { - if (result.count("cross_kv_cache_fraction")) - { - benchmarkParams.crossKvCacheFraction = result["cross_kv_cache_fraction"].as<float>(); - } - else - { - benchmarkParams.crossKvCacheFraction - = 0.5f; // default value if not set. but non enc-dec should not even have this param set - } - } - - // Argument: Enable dynamic tuning of batch size - benchmarkParams.enableBatchSizeTuning = result["enable_batch_size_tuning"].as<bool>(); - - // Argument: Enable dynamic tuning of max num tokens - benchmarkParams.enableMaxNumTokensTuning = result["enable_max_num_tokens_tuning"].as<bool>(); - - // Argument: Enable KV cache reuse - benchmarkParams.enableBlockReuse = result["enable_kv_cache_reuse"].as<bool>(); - - // Argument: streaming - benchmarkParams.streaming = result["streaming"].as<bool>(); - - TLLM_CHECK_WITH_INFO(!(result.count("request_rate") && result.count("concurrency")), - "request_rate and concurrency cannot be specified at the same time."); - - // Argument: request rate - if (result.count("request_rate")) - { - benchmarkParams.requestRate = result["request_rate"].as<float>(); - } - - // Argument: concurrency - if (result.count("concurrency")) - { - benchmarkParams.concurrency = result["concurrency"].as<int>(); - } - - // Argument: request rate - if (result.count("max_batch_size")) - { - benchmarkParams.maxBatchSize = result["max_batch_size"].as<int>(); - } - - // Argument: request rate - if (result.count("max_num_tokens")) - { - benchmarkParams.maxNumTokens = result["max_num_tokens"].as<int>(); - } - - benchmarkParams.enableExpDelays = result["enable_exp_delays"].as<bool>(); - - // Argument: Enable batch stats output - bool logIterationData = result["log_iteration_data"].as<bool>(); - - if (logIterationData) - { - TLLM_LOG_WARNING("Setting log_iteration_data to true adds overheads and may result in lower perf"); - } - - // Argument: Enable chunked context - benchmarkParams.enableChunkedContext = result["enable_chunked_context"].as<bool>(); - - // Argument: Enable return context logits - bool returnContextLogits = result["return_context_logits"].as<bool>(); - - // Argument: Enable return context logits - bool returnGenerationLogits = result["return_generation_logits"].as<bool>(); - - if (result.count("lora_dir")) - { - benchmarkParams.loraDir = result["lora_dir"].as<std::string>(); - } - if (result.count("lora_host_cache_bytes")) - { - benchmarkParams.loraHostCacheSize = result["lora_host_cache_bytes"].as<size_t>(); - } - if (result.count("lora_num_device_mod_layers")) - { - benchmarkParams.loraDeviceNumModLayers = result["lora_num_device_mod_layers"].as<SizeType32>(); - } - - // Argument: How many KV cache blocks (as fraction of number of GPU kv cache blocks). - benchmarkParams.kvHostCacheSize = result["kv_host_cache_bytes"].as<size_t>(); - - // Argument: Medusa choices for the Medusa speculative decoding. - if (result.count("medusa_choices")) - { - benchmarkParams.medusaChoices = parseVectorOfVectors(result["medusa_choices"].as<std::string>()); - } - // Argument: Eagle choices for the Eagle speculative decoding. - if (result.count("eagle_choices") || result.count("eagle_posterior_threshold") - || result.count("eagle_use_dynamic_tree") || result.count("eagle_dynamic_tree_max_top_k")) - { - std::optional<float> posteriorThreshold; - if (result.count("eagle_posterior_threshold")) - { - posteriorThreshold = result["eagle_posterior_threshold"].as<float>(); - } - std::optional<texec::EagleChoices> choices; - if (result.count("eagle_choices")) - { - choices = parseVectorOfVectors(result["eagle_choices"].as<std::string>()); - } - bool eagleUseDynamicTree = false; - if (result.count("eagle_use_dynamic_tree")) - { - eagleUseDynamicTree = result["eagle_use_dynamic_tree"].as<bool>(); - } - std::optional<SizeType32> eagleDynamicTreeMaxTopK; - if (result.count("eagle_dynamic_tree_max_top_k")) - { - eagleDynamicTreeMaxTopK = result["eagle_dynamic_tree_max_top_k"].as<SizeType32>(); - } - benchmarkParams.eagleConfig = texec::EagleConfig( - choices, !posteriorThreshold.has_value(), posteriorThreshold, eagleUseDynamicTree, eagleDynamicTreeMaxTopK); - } - if (result.count("temperature")) - { - benchmarkParams.temperature = result["temperature"].as<float>(); - } - - if (result.count("executor_lookahead_config")) - { - benchmarkParams.executorLookaheadConfig - = parseLookaheadConfig(result["executor_lookahead_config"].as<std::string>()); - } - if (result.count("request_lookahead_config")) - { - benchmarkParams.requestLookaheadConfig - = parseLookaheadConfig(result["request_lookahead_config"].as<std::string>()); - } - - // Argument: multi_block_mode - benchmarkParams.multiBlockMode = result["multi_block_mode"].as<bool>(); - - // Argument: enable_context_fmha_fp32_acc - benchmarkParams.enableContextFMHAFP32Acc = result["enable_context_fmha_fp32_acc"].as<bool>(); - - // Argument: cuda_graph_mode - benchmarkParams.cudaGraphMode = result["cuda_graph_mode"].as<bool>(); - - // Argument: cuda_graph_cache_size - benchmarkParams.cudaGraphCacheSize = result["cuda_graph_cache_size"].as<SizeType32>(); - - // Argument: enable_trt_overlap - benchmarkParams.enableTrtOverlap = result["enable_trt_overlap"].as<bool>(); - - std::optional<TokenIdType> padId; - // Argument: Padding token id - if (result.count("pad_id")) - { - padId = result["pad_id"].as<TokenIdType>(); - } - - // Argument: End-of-sentence token id - std::optional<TokenIdType> eosId = result["eos_id"].as<TokenIdType>(); - - std::optional<SizeType32> staticEmulatedBatchSize; - // Argument: Static emulated batch size - if (result.count("static_emulated_batch_size")) - { - staticEmulatedBatchSize = result["static_emulated_batch_size"].as<SizeType32>(); - } - - // Argument: Scheduler policy - texec::CapacitySchedulerPolicy capacitySchedulerPolicy; - auto const capacitySchedulerPolicyArg = result["scheduler_policy"].as<std::string>(); - if (capacitySchedulerPolicyArg == "max_utilization") - { - capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kMAX_UTILIZATION; - } - else if (capacitySchedulerPolicyArg == "guaranteed_no_evict") - { - capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT; - } - else if (capacitySchedulerPolicyArg == "static_batch") - { - capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kSTATIC_BATCH; - } - else - { - TLLM_LOG_ERROR("Unexpected scheduler policy: " + capacitySchedulerPolicyArg); - return 1; - } - - // Argument: max_prompt_len - std::optional<SizeType32> maxPromptLen; - if (result.count("max_prompt_len")) - { - maxPromptLen = result["max_prompt_len"].as<SizeType32>(); - } - - // Argument: GPU weights percentage - auto gpuWeightsPercent = result["gpu_weights_percent"].as<float>(); - if (gpuWeightsPercent < 0 || gpuWeightsPercent > 1) - { - TLLM_LOG_ERROR("--gpu_weights_percent must be between 0.0 and 1.0 but got: %f", gpuWeightsPercent); - return 1; - } - benchmarkParams.gpuWeightsPercent = gpuWeightsPercent; - - // Argument: Log level - auto logger = std::make_shared<TllmLogger>(); - auto const logLevel = result["log_level"].as<std::string>(); - if (logLevel == "verbose") - { - logger->setLevel(trt::ILogger::Severity::kVERBOSE); - } - else if (logLevel == "info") - { - logger->setLevel(trt::ILogger::Severity::kINFO); - } - else if (logLevel == "warning") - { - logger->setLevel(trt::ILogger::Severity::kWARNING); - } - else if (logLevel == "error") - { - logger->setLevel(trt::ILogger::Severity::kERROR); - } - else if (logLevel == "internal_error") - { - logger->setLevel(trt::ILogger::Severity::kINTERNAL_ERROR); - } - else - { - TLLM_LOG_ERROR("Unexpected log level: " + logLevel); - return 1; - } - - initTrtLlmPlugins(logger.get()); - - // Argument: output sequences JSON - auto const responsesJsonFile = result["responses_json"].as<std::string>(); - - // Argument: API - auto const api = result["api"].as<std::string>(); - if (api == "executor") - { - texec::ModelType executorModelType; - std::optional<std::string> decoderEngineDir = std::nullopt, encoderEngineDir = std::nullopt; - if (result.count("encoder_engine_dir") && result.count("decoder_engine_dir")) - { - TLLM_CHECK_WITH_INFO(api == "executor", "encoder-decoder only support executor api."); - TLLM_CHECK_WITH_INFO( - batchingType == texec::BatchingType::kINFLIGHT, "encoder-decoder only support inflight batching."); - executorModelType = texec::ModelType::kENCODER_DECODER; - encoderEngineDir = result["encoder_engine_dir"].as<std::string>(); - decoderEngineDir = result["decoder_engine_dir"].as<std::string>(); - } - else if (result.count("engine_dir")) - { - executorModelType = texec::ModelType::kDECODER_ONLY; - decoderEngineDir = result["engine_dir"].as<std::string>(); - } - else - { - executorModelType = texec::ModelType::kENCODER_ONLY; - encoderEngineDir = result["encoder_engine_dir"].as<std::string>(); - } - try - { - benchmarkExecutor(decoderEngineDir, encoderEngineDir, batchingType, datasetPath, opCsvFile, maxNumSamples, - beamWidth, result["warm_up"].as<int>(), eosId, padId, benchmarkParams, capacitySchedulerPolicy, - waitSleep, returnContextLogits, returnGenerationLogits, staticEmulatedBatchSize, logIterationData, - maxPromptLen, executorModelType, responsesJsonFile); - } - catch (std::exception const& e) - { - TLLM_LOG_ERROR(e.what()); - return 1; - } - } - else if (api == "gptManager") - { - TLLM_LOG_ERROR("gptManager is deprecated, please use the executor API."); - return 1; - } - else - { - TLLM_LOG_ERROR("api parameter must be gptManager or executor"); - return 1; - } - - return 0; -} diff --git a/benchmarks/cpp/prepare_dataset.py b/benchmarks/cpp/prepare_dataset.py deleted file mode 100644 index 3b9665fd2902..000000000000 --- a/benchmarks/cpp/prepare_dataset.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import Optional, Tuple - -import click -from pydantic import BaseModel, model_validator -from transformers import AutoTokenizer -from utils.prepare_real_data import dataset -from utils.prepare_synthetic_data import token_norm_dist, token_unif_dist - - -class RootArgs(BaseModel): - tokenizer: str - output: str - random_seed: int - task_id: int - std_out: bool - trust_remote_code: bool = False - rand_task_id: Optional[Tuple[int, int]] - lora_dir: Optional[str] = None - - @model_validator(mode='after') - def validate_tokenizer(self): - try: - tokenizer = AutoTokenizer.from_pretrained( - self.tokenizer, - padding_side='left', - trust_remote_code=self.trust_remote_code) - except EnvironmentError as e: - raise ValueError( - f"Cannot find a tokenizer from the given string because of {e}\nPlease set tokenizer to the directory that contains the tokenizer, or set to a model name in HuggingFace." - ) - tokenizer.pad_token = tokenizer.eos_token - self.tokenizer = tokenizer - - return self - - -@click.group(deprecated=True) -@click.option( - "--tokenizer", - required=True, - type=str, - help= - "Tokenizer dir for the model run by gptManagerBenchmark, or the model name from HuggingFace." -) -@click.option("--output", - type=str, - help="Output json filename.", - default="preprocessed_dataset.json") -@click.option( - "--stdout", - is_flag=True, - help="Print output to stdout with a JSON dataset entry on each line.", - default=False) -@click.option("--random-seed", - required=False, - type=int, - help="random seed for token_ids", - default=420) -@click.option("--task-id", type=int, default=-1, help="LoRA task id") -@click.option("--rand-task-id", - type=int, - default=None, - nargs=2, - help="Random LoRA Tasks") -@click.option("--lora-dir", - type=str, - default=None, - help="Directory containing LoRA adapters") -@click.option("--log-level", - default="info", - type=click.Choice(['info', 'debug']), - help="Logging level.") -@click.option("--trust-remote-code", - is_flag=True, - default=False, - envvar="TRUST_REMOTE_CODE", - help="Trust remote code.") -@click.pass_context -def cli(ctx, **kwargs): - """This script generates dataset input for gptManagerBenchmark.""" - if kwargs['log_level'] == 'info': - logging.basicConfig(level=logging.INFO) - elif kwargs['log_level'] == 'debug': - logging.basicConfig(level=logging.DEBUG) - else: - raise ValueError(f"Unsupported logging level {kwargs['log_level']}") - - ctx.obj = RootArgs(tokenizer=kwargs['tokenizer'], - output=kwargs['output'], - std_out=kwargs['stdout'], - random_seed=kwargs['random_seed'], - task_id=kwargs['task_id'], - rand_task_id=kwargs['rand_task_id'], - lora_dir=kwargs['lora_dir'], - trust_remote_code=kwargs['trust_remote_code']) - - -cli.add_command(dataset) -cli.add_command(token_norm_dist) -cli.add_command(token_unif_dist) - -if __name__ == "__main__": - cli() diff --git a/benchmarks/cpp/utils/convert_nemo_dataset.py b/benchmarks/cpp/utils/convert_nemo_dataset.py deleted file mode 100644 index 6f4884347677..000000000000 --- a/benchmarks/cpp/utils/convert_nemo_dataset.py +++ /dev/null @@ -1,47 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#!/usr/bin/env python3 - -import argparse -import json - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("input") - parser.add_argument("output") - - args = parser.parse_args() - - output_o = [] - - with open(args.input, 'r') as infile: - for _l in infile: - l = _l.strip() - if len(l) == 0: - continue - o = json.loads(l) - output_o.append({ - "input": o["prompt"], - "instruction": "", - "output": o["completion"] - }) - - with open(args.output, 'w') as outfile: - json.dump(output_o, outfile) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/cpp/utils/generate_rand_loras.py b/benchmarks/cpp/utils/generate_rand_loras.py deleted file mode 100644 index 12eb1fdc3648..000000000000 --- a/benchmarks/cpp/utils/generate_rand_loras.py +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#!/usr/bin/env python3 - -import argparse -import os -from pathlib import Path - -import numpy as np - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("input_lora") - parser.add_argument("output") - parser.add_argument("num_loras", type=int) - - args = parser.parse_args() - - lora_path = Path(args.input_lora) - weights_path = lora_path / "model.lora_weights.npy" - config_path = lora_path / "model.lora_config.npy" - - weights = np.load(weights_path) - config = np.load(config_path) - - for i in range(args.num_loras): - out_path = Path(args.output) / str(i) - os.makedirs(out_path, exist_ok=True) - w = np.random.normal(0, 2, weights.shape).astype(weights.dtype) - np.save(out_path / "model.lora_weights.npy", w) - np.save(out_path / "model.lora_config.npy", config) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/cpp/utils/prepare_real_data.py b/benchmarks/cpp/utils/prepare_real_data.py deleted file mode 100644 index 4441c57ee4bc..000000000000 --- a/benchmarks/cpp/utils/prepare_real_data.py +++ /dev/null @@ -1,434 +0,0 @@ -import logging -import random -import re -import tempfile -from pathlib import Path -from typing import Optional - -import click -import datasets -from PIL import Image -from pydantic import BaseModel, model_validator -from utils.utils import (get_norm_dist_lengths, multimodal_dataset_dump, - print_multimodal_dataset, print_text_dataset, - text_dataset_dump) - - -def validate_output_len_dist(ctx, param, value): - """Validate the --output-len-dist option.""" - if value is None: - return value - m = re.match(r"(\d+),(\d+)", value) - if m: - return int(m.group(1)), int(m.group(2)) - else: - raise AssertionError( - "Incorrect specification for --output-len-dist. Correct format: --output-len-dist <output_len_mean>,<output_len_stdev>" - ) - - -class DatasetConfig(BaseModel): - """Dataset configurations.""" - """Name of the dataset on HuggingFace.""" - name: Optional[str] = None - """Config name of the dataset if existing.""" - config_name: Optional[str] = None - """Split of the dataset. Typical values: train, validation, test. Setting to None will include all splits.""" - split: Optional[str] - """The dataset dictionary used for the input sentence.""" - input_key: Optional[str] = None - """The dataset dictionary key used for the prompt of the input sentence. Must not be set when prompt is set.""" - image_key: Optional[str] = None - """The dataset dictionary key used for the images.""" - prompt_key: Optional[str] = None - """The prompt sentence to be added to the input sentence. Must not be set when prompt_key is set.""" - prompt: Optional[str] = None - """The dataset dictionary key used to derive the output sequence length. Set to None if the dataset does not have a key for output.""" - output_key: Optional[str] - """The local path to the dataset to be loaded when using a local cache.""" - local_path: Optional[str] = None - - @model_validator(mode='after') - def check_prompt(self) -> 'DatasetConfig': - if self.prompt_key and self.prompt: - raise AssertionError( - "--prompt-key and --prompt cannot be set at the same time.") - if (not self.prompt_key) and (not self.prompt): - raise AssertionError("Either --prompt-key or --prompt must be set.") - return self - - @model_validator(mode='after') - def check_name_and_local_path(self) -> 'DatasetConfig': - if self.name and self.local_path: - raise AssertionError( - "--dataset-name and --dataset-local-path cannot be set at the same time." - ) - if (not self.name) and (not self.local_path): - raise AssertionError( - "Either --dataset-name or --dataset-local-path must be set.") - return self - - @property - def query(self): - """Generate the query for HuggingFace `datasets.load_dataset()`""" - first_arg = self.local_path if self.local_path else self.name - - if self.config_name: - return [first_arg, self.config_name] - else: - return [first_arg] - - @property - def display_name(self) -> str: - """Returns a human-readable identifier for error messages.""" - # model_validator ensures exactly one of name or local_path is set - if self.name is not None: - return self.name - return self.local_path - - def get_prompt(self, req): - """Get the prompt sentence from the given request.""" - if self.prompt_key: - assert self.prompt_key in req, ( - f"Dataset {self.display_name} does not have key '{self.prompt_key}'. " - "Please set --prompt-key to one of the available keys: " - f"{req.keys()}") - return req[self.prompt_key] - else: - return self.prompt - - def get_input(self, req): - """Get the input sentence from the given request.""" - assert self.input_key in req, ( - f"Dataset {self.display_name} does not have key '{self.input_key}'. " - "Please set --input-key to one of the available keys: " - f"{req.keys()}") - return req[self.input_key] - - def get_images(self, req): - """Get the images from the given request.""" - image_keys = [self.image_key - ] + [f"{self.image_key}_{i}" for i in range(1, 8)] - assert any(key in req for key in image_keys), ( - f"Dataset {self.display_name} does not have key '{self.image_key}'. " - "Please set --dataset-image-key to one of the available keys: " - f"{req.keys()}") - images = [] - for key in image_keys: - if key in req and req[key] is not None: - images.append(req[key]) - return images - - def get_output(self, req): - """Get the output sentence from the given request.""" - if self.output_key is None: - raise RuntimeError( - "--output-key is not set. Please either:\n" - "1. Define output length through --output-len-dist.\n" - f"2. If the dataset {self.display_name} has key for golden output and " - "you wish to set output length to the length of the golden " - "output, set --output-key.") - assert self.output_key in req, ( - f"Dataset {self.display_name} does not have key '{self.output_key}'. " - "Please set --output-key to one of the available keys: " - f"{req.keys()}") - return req[self.output_key] - - -def _create_dataset_load_error(e: ValueError) -> ValueError: - """Create a more informative ValueError from a dataset loading error. - - Args: - e: The original ValueError from datasets.load_dataset(). - Returns: - A new ValueError with additional context. - """ - error_msg = str(e) - if "Config" in error_msg: - error_msg += "\n Please add the config name to the dataset config yaml." - elif "split" in error_msg: - error_msg += "\n Please specify supported split in the dataset config yaml." - return ValueError(error_msg) - - -def load_dataset(dataset_config: DatasetConfig): - """Load dataset from local path or HuggingFace. - Args: - dataset_config: A `DatasetConfig` object that defines the dataset to load. - Returns: - Dataset iterator. - Raises: - ValueError: When dataset loading fails due to incorrect dataset config setting. - """ - if dataset_config.local_path: - return load_dataset_from_local(dataset_config) - else: - return load_dataset_from_hf(dataset_config) - - -def load_dataset_from_hf(dataset_config: DatasetConfig): - """Load dataset from HuggingFace. - - Args: - dataset_config: A `DatasetConfig` object that defines the dataset to load. - Returns: - Dataset iterator. - Raises: - ValueError: When dataset loading fails due to incorrect dataset config setting. - """ - logging.debug( - f"Loading dataset from HF: query={dataset_config.query}, split={dataset_config.split}" - ) - - try: - dataset = iter( - datasets.load_dataset(*dataset_config.query, - split=dataset_config.split, - streaming=True, - trust_remote_code=True)) - except ValueError as e: - raise _create_dataset_load_error(e) - - logging.debug("Finished loading HF dataset") - - return dataset - - -def load_dataset_from_local(dataset_config: DatasetConfig): - """Load dataset from local path. - - Args: - dataset_config: A `DatasetConfig` object that defines the dataset to load. - Returns: - Dataset iterator. - Raises: - FileNotFoundError: When local dataset path does not exist. - ValueError: When dataset loading fails due to incorrect dataset config setting. - """ - - local_path = Path(dataset_config.local_path) - - if not local_path.exists(): - raise FileNotFoundError( - f"Local dataset path {local_path} does not exist.") - - logging.debug( - f"Loading dataset from local path: path={local_path}, query={dataset_config.query}, split={dataset_config.split}" - ) - - # If it's a directory we can use the normal loader, otherwise custom loader - # depends on the file extension - if local_path.is_dir(): - try: - dataset = datasets.load_dataset(*dataset_config.query, - split=dataset_config.split, - trust_remote_code=True) - except ValueError as e: - raise _create_dataset_load_error(e) - else: - format_map = { - ".json": "json", - ".jsonl": "json", - ".csv": "csv", - ".parquet": "parquet", - } - - file_extension = local_path.suffix - dataset_type = format_map.get(file_extension) - - if dataset_type is None: - raise ValueError(f"Unsupported file extension: {file_extension}") - - try: - dataset = datasets.load_dataset(dataset_type, - data_files=str(local_path), - split=dataset_config.split) - except ValueError as e: - raise _create_dataset_load_error(e) - - logging.debug("Finished loading local dataset") - - return iter(dataset) - - -@click.command() -@click.option("--dataset-name", type=str, help="Dataset name in HuggingFace.") -@click.option("--dataset-config-name", - type=str, - default=None, - help="Dataset config name in HuggingFace (if exists).") -@click.option("--dataset-split", - type=str, - required=True, - help="Split of the dataset to use.") -@click.option("--dataset-input-key", - type=str, - help="The dataset dictionary key for input.") -@click.option("--dataset-image-key", - type=str, - default="image", - help="The dataset dictionary key for images.") -@click.option("--dataset-prompt-key", - type=str, - default=None, - help="The dataset dictionary key for prompt (if exists).") -@click.option( - "--dataset-local-path", - type=str, - default=None, - help= - "The local path to the dataset to be loaded when using an offline cache.") -@click.option( - "--dataset-prompt", - type=str, - default=None, - help="The prompt string when there is no prompt key for the dataset.") -@click.option("--dataset-output-key", - type=str, - default=None, - help="The dataset dictionary key for output (if exists).") -@click.option( - "--num-requests", - type=int, - default=None, - help= - "Number of requests to be generated. Will be capped to min(dataset.num_rows, num_requests)." -) -@click.option( - "--max-input-len", - type=int, - default=None, - help= - "Maximum input sequence length for a given request. This will be used to filter out the requests with long input sequence length. Default will include all the requests." -) -@click.option( - "--output-len-dist", - type=str, - default=None, - callback=validate_output_len_dist, - help= - "Output length distribution. Default will be the length of the golden output from the dataset. Format: <output_len_mean>,<output_len_stdev>. E.g. 100,10 will randomize the output length with mean=100 and variance=10." -) -@click.pass_obj -def dataset(root_args, **kwargs): - """Prepare dataset from real dataset.""" - dataset_config = DatasetConfig(**{ - k[8:]: v - for k, v in kwargs.items() if k.startswith('dataset_') - }) - - input_ids = [] - input_lens = [] - output_lens = [] - task_ids = [] - req_cnt = 0 - modality = None - multimodal_texts = [] - multimodal_image_paths = [] - for req in load_dataset(dataset_config): - if any(key in req for key in ['image', 'image_1', 'video']): - # multimodal input - if 'video' in req and req['video'] is not None: - assert "Not supported yet" - assert kwargs['output_len_dist'] is not None, ( - "Output length distribution must be set for multimodal requests." - ) - modality = 'image' - text = dataset_config.get_prompt(req) - images = dataset_config.get_images(req) - image_paths = [] - for image in images: - if image is not None: - if isinstance(image, str): - image_paths.append(image) - elif isinstance(image, Image.Image): - with tempfile.NamedTemporaryFile( - suffix=".jpg", delete=False) as tmp_file: - logging.debug(f"Saving image to {tmp_file.name}") - image = image.convert("RGB") - image.save(tmp_file, "JPEG") - filepath = tmp_file.name - image_paths.append(filepath) - else: - raise ValueError(f"Invalid image path: {image}") - multimodal_texts.append(text) - multimodal_image_paths.append(image_paths) - else: - # text input - prompt = dataset_config.get_prompt( - req) + ' ' + dataset_config.get_input(req) - logging.debug(f"Input sequence: {prompt}") - line = root_args.tokenizer.encode(prompt) - if kwargs['max_input_len'] and len(line) > kwargs['max_input_len']: - continue - input_ids.append(line) - input_lens.append(len(line)) - - # output if fetch from golden - if kwargs['output_len_dist'] is None: - output_lens.append( - len( - root_args.tokenizer.encode( - dataset_config.get_output(req)))) - - # lora task id - task_id = root_args.task_id - if root_args.rand_task_id is not None: - min_id, max_id = root_args.rand_task_id - task_id = random.randint(min_id, max_id) - task_ids.append(task_id) - - req_cnt += 1 - if kwargs['num_requests'] and req_cnt >= kwargs['num_requests']: - break - - if kwargs['num_requests'] and (len(input_ids) if modality is None else len( - multimodal_texts)) < kwargs['num_requests']: - logging.warning( - f"Number of requests={len(input_ids) if modality is None else len(multimodal_texts)} is" - f" smaller than the num-requests user set={kwargs['num_requests']}." - ) - - # output if randomized - if kwargs['output_len_dist'] is not None: - osl_mean, osl_stdev = kwargs['output_len_dist'] - output_lens = get_norm_dist_lengths( - osl_mean, osl_stdev, - len(input_ids) if modality is None else len(multimodal_texts), - root_args.random_seed) - logging.debug(f"Input lengths: {[len(i) for i in input_ids]}") - logging.debug(f"Output lengths: {output_lens}") - if modality is not None: - logging.debug(f"Modality: {modality}") - - if modality is not None: - if not root_args.std_out: - multimodal_dataset_dump( - multimodal_texts, multimodal_image_paths, output_lens, task_ids, - { - "workload_type": "dataset", - "tokenizer": root_args.tokenizer.__class__.__name__, - "num_requests": len(task_ids), - "max_output_len": max(output_lens) - }, root_args.output) - else: - print_multimodal_dataset( - multimodal_texts, - multimodal_image_paths, - output_lens, - ) - else: - if not root_args.std_out: - text_dataset_dump( - input_lens, input_ids, output_lens, task_ids, { - "workload_type": "dataset", - "tokenizer": root_args.tokenizer.__class__.__name__, - "num_requests": len(input_ids), - "max_input_len": max(input_lens), - "max_output_len": max(output_lens) - }, root_args.output) - else: - print_text_dataset( - input_ids, - output_lens, - ) diff --git a/benchmarks/cpp/utils/prepare_synthetic_data.py b/benchmarks/cpp/utils/prepare_synthetic_data.py deleted file mode 100644 index b072b712d085..000000000000 --- a/benchmarks/cpp/utils/prepare_synthetic_data.py +++ /dev/null @@ -1,164 +0,0 @@ -import random -import warnings - -import click -from utils.utils import (gen_random_tokens, get_norm_dist_lengths, - get_unif_dist_lengths, print_text_dataset, - text_dataset_dump) - - -def _generate_task_ids_and_lora_config(root_args, num_reqs): - """Generate task IDs and determine LoRA configuration based on root_args.""" - if root_args.rand_task_id is None: - task_ids = [root_args.task_id for _ in range(num_reqs)] - else: - min_id, max_id = root_args.rand_task_id - task_ids = [random.randint(min_id, max_id) for _ in range(num_reqs)] - - use_task_ids = root_args.task_id != -1 or root_args.rand_task_id is not None - - # Determine if LoRA should be used (requires both task IDs and lora_dir) - use_lora = use_task_ids and root_args.lora_dir is not None - - # Warn if task IDs are specified but no LoRA directory is provided - if use_task_ids and not use_lora: - warnings.warn( - "Task IDs require LoRA directory. Use --lora-dir or omit task IDs.", - UserWarning) - - return (task_ids, task_ids if use_task_ids else None, { - "lora_dir": root_args.lora_dir - } if use_lora else None) - - -@click.command() -@click.option("--num-requests", - required=True, - type=int, - help='Number of requests to be generated') -@click.option('--input-mean', - required=True, - type=int, - help='normal dist mean for input tokens') -@click.option('--input-stdev', - required=True, - type=int, - help='normal dist stdev for input tokens') -@click.option('--output-mean', - required=True, - type=int, - help='normal dist mean for output tokens') -@click.option('--output-stdev', - required=True, - type=int, - help='normal dist stdev for output tokens') -@click.pass_obj -def token_norm_dist(root_args, **kwargs): - """Prepare synthetic dataset by generating random tokens with normal dist lengths.""" - input_ids = [] - input_lens = [] - output_lens = [] - - input_lens = get_norm_dist_lengths(kwargs['input_mean'], - kwargs['input_stdev'], - kwargs['num_requests'], - root_args.random_seed) - - num_reqs = len(input_lens) - output_lens = get_norm_dist_lengths(kwargs['output_mean'], - kwargs['output_stdev'], num_reqs, - root_args.random_seed) - - max_input_len = max(input_lens) - max_output_len = max(output_lens) - - input_ids = gen_random_tokens(input_lens, root_args.tokenizer, - root_args.random_seed) - - task_ids, print_task_ids, lora_config = _generate_task_ids_and_lora_config( - root_args, num_reqs) - - if not root_args.std_out: - text_dataset_dump( - input_lens, input_ids, output_lens, task_ids, { - "workload_type": "token-norm-dist", - "input_mean": kwargs['input_mean'], - "input_stdev": kwargs['input_stdev'], - "output_mean": kwargs['output_mean'], - "output_stdev": kwargs['output_stdev'], - "num_requests": kwargs['num_requests'], - "tokenize_vocabsize": root_args.tokenizer.vocab_size, - "max_input_len": max_input_len, - "max_output_len": max_output_len - }, root_args.output) - else: - print_text_dataset(input_ids, - output_lens, - task_ids=print_task_ids, - lora_config=lora_config) - - -@click.command() -@click.option("--num-requests", - required=True, - type=int, - help='Number of requests to be generated') -@click.option('--input-min', - required=True, - type=int, - help='uniform dist (inclusive) min for input tokens') -@click.option('--input-max', - required=True, - type=int, - help='normal dist (inclusive) max for input tokens') -@click.option('--output-min', - required=True, - type=int, - help='normal dist (inclusive) min for output tokens') -@click.option('--output-max', - required=True, - type=int, - help='normal dist (inclusive) max for output tokens') -@click.pass_obj -def token_unif_dist(root_args, **kwargs): - """Prepare synthetic dataset by generating random tokens with normal uniformly lengths.""" - input_ids = [] - input_lens = [] - output_lens = [] - - input_lens = get_unif_dist_lengths(kwargs['input_min'], kwargs['input_max'], - kwargs['num_requests'], - root_args.random_seed) - - num_reqs = len(input_lens) - output_lens = get_unif_dist_lengths(kwargs['output_min'], - kwargs['output_max'], num_reqs, - root_args.random_seed) - - max_input_len = max(input_lens) - max_output_len = max(output_lens) - - input_ids = gen_random_tokens(input_lens, root_args.tokenizer, - root_args.random_seed) - - task_ids, print_task_ids, lora_config = _generate_task_ids_and_lora_config( - root_args, num_reqs) - - if not root_args.std_out: - text_dataset_dump( - input_lens, input_ids, output_lens, task_ids, { - "workload_type": "token-unif-dist", - "input_min": kwargs['input_min'], - "input_max": kwargs['input_max'], - "output_min": kwargs['output_min'], - "output_max": kwargs['output_max'], - "num_requests": kwargs['num_requests'], - "tokenize_vocabsize": root_args.tokenizer.vocab_size, - "max_input_len": max_input_len, - "max_output_len": max_output_len - }, root_args.output) - else: - print_text_dataset(input_ids, - output_lens, - task_ids=print_task_ids, - lora_config=lora_config) diff --git a/benchmarks/cpp/utils/utils.cpp b/benchmarks/cpp/utils/utils.cpp deleted file mode 100644 index 0cbcf1c0468d..000000000000 --- a/benchmarks/cpp/utils/utils.cpp +++ /dev/null @@ -1,170 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & - *AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "utils.h" -#include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/logger.h" -#include <random> - -#include <filesystem> -#include <fstream> - -TRTLLM_NAMESPACE_BEGIN - -namespace benchmark -{ - -std::vector<std::vector<SizeType32>> parseVectorOfVectors(std::string const& input) -{ - std::vector<std::vector<SizeType32>> result; - std::regex outer_regex(R"(\[(.*?)\])"); - std::regex inner_regex(R"(\d+)"); - auto outer_begin = std::sregex_iterator(input.begin(), input.end(), outer_regex); - auto outer_end = std::sregex_iterator(); - - for (std::sregex_iterator i = outer_begin; i != outer_end; ++i) - { - std::smatch match = *i; - std::string inner_str = match.str(1); - std::vector<int> inner_vec; - auto inner_begin = std::sregex_iterator(inner_str.begin(), inner_str.end(), inner_regex); - auto inner_end = std::sregex_iterator(); - - for (std::sregex_iterator j = inner_begin; j != inner_end; ++j) - { - std::smatch inner_match = *j; - inner_vec.push_back(std::stoi(inner_match.str())); - } - result.push_back(inner_vec); - } - return result; -} - -texec::LookaheadDecodingConfig parseLookaheadConfig(std::string const& input) -{ - std::regex regex("\\[ *(\\d+) *, *(\\d+) *, *(\\d+) *\\]"); - std::smatch match; - if (std::regex_match(input, match, regex)) - { - TLLM_CHECK(match.size() == 4); - auto w = std::stoi(match[1]); - auto n = std::stoi(match[2]); - auto g = std::stoi(match[3]); - return texec::LookaheadDecodingConfig(w, n, g); - } - else - { - TLLM_LOG_WARNING("cannot parse lookahead config from '%s'", input.c_str()); - return texec::LookaheadDecodingConfig(); - } -} - -Samples parseWorkloadJson( - std::filesystem::path const& datasetPath, int maxNumSamples, std::optional<SizeType32> const maxPromptLen) -{ - auto constexpr allowExceptions = true; - auto constexpr ignoreComments = true; - TLLM_CHECK_WITH_INFO(std::filesystem::exists(datasetPath), "File does not exist: %s", datasetPath.c_str()); - std::ifstream jsonStream(datasetPath); - auto json = nlohmann::json::parse(jsonStream, nullptr, allowExceptions, ignoreComments); - - Samples samples; - - for (auto const& sample : json["samples"]) - { - if (samples.size() >= maxNumSamples) - break; - int32_t taskId = sample.count("task_id") ? sample["task_id"].template get<int32_t>() : -1; - auto input_ids(sample["input_ids"].template get<std::vector<int32_t>>()); - if (maxPromptLen && (input_ids.size() > maxPromptLen.value())) - { - input_ids.resize(maxPromptLen.value()); - } - samples.emplace_back(Sample{std::move(input_ids), sample["output_len"], taskId}); - } - - if (samples.size() < maxNumSamples) - { - TLLM_LOG_WARNING( - "Dataset size %zu is smaller than given max_num_samples " - "%d, max_num_samples will be ignored.\n", - samples.size(), maxNumSamples); - } - return samples; -} - -std::vector<double> generateRandomExponentialValues(int count, float lambda, int seed) -{ - // Set a constant seed for reproducibility - std::mt19937 gen(seed); - - // Create an exponential distribution object - std::exponential_distribution<double> distribution(lambda); - - // Generate random numbers from the exponential distribution - std::vector<double> randomValues; - for (int i = 0; i < count; ++i) - { - double randomValue = distribution(gen); - randomValues.push_back(randomValue); - } - - return randomValues; -} - -std::vector<double> computeTimeDelays(BenchmarkParams const& benchmarkParams, int numDelays) -{ - std::vector<double> timeDelays; - if (benchmarkParams.requestRate.has_value() && benchmarkParams.requestRate.value() > 0.0) - { - if (benchmarkParams.enableExpDelays) - { - timeDelays = generateRandomExponentialValues( - numDelays, benchmarkParams.requestRate.value(), benchmarkParams.randomSeed); - } - else - { - timeDelays.assign(numDelays, 1.0 / benchmarkParams.requestRate.value()); - } - } - else - { - timeDelays.assign(numDelays, 0.0); - } - - return timeDelays; -} - -std::ostream& operator<<(std::ostream& os, RecordTimeMetric const& metric) -{ - os << metric.mAvg << "," << metric.mMax << "," << metric.mMin << "," << metric.mP99 << "," << metric.mP90 << "," - << metric.mP50; - return os; -} - -std::ostream& operator<<(std::ostream& os, RecordBwMetric const& metric) -{ - os << metric.mAvg << "," << metric.mMax << "," << metric.mMin << "," << metric.mP99 << "," << metric.mP90 << "," - << metric.mP50; - return os; -} - -} // namespace benchmark - -TRTLLM_NAMESPACE_END diff --git a/benchmarks/cpp/utils/utils.h b/benchmarks/cpp/utils/utils.h deleted file mode 100644 index fba30fee69ae..000000000000 --- a/benchmarks/cpp/utils/utils.h +++ /dev/null @@ -1,244 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/executor/executor.h" - -#include <cstdint> -#include <cxxopts.hpp> -#include <iostream> -#include <nlohmann/json.hpp> -#include <numeric> -#include <optional> -#include <string> -#include <utility> - -#pragma once - -TRTLLM_NAMESPACE_BEGIN - -namespace benchmark -{ - -// using namespace tensorrt_llm::batch_manager; -using namespace tensorrt_llm::runtime; - -namespace texec = tensorrt_llm::executor; - -std::vector<std::vector<SizeType32>> parseVectorOfVectors(std::string const& input); - -texec::LookaheadDecodingConfig parseLookaheadConfig(std::string const& input); - -struct BenchmarkParams -{ - std::optional<SizeType32> maxTokensInPagedKvCache{std::nullopt}; - std::optional<float> freeGpuMemoryFraction{std::nullopt}; - std::vector<std::optional<float>> freeGpuMemoryFractions{std::nullopt}; - - std::optional<float> crossKvCacheFraction{std::nullopt}; - bool enableTrtOverlap{false}; - bool enableBatchSizeTuning{false}; - bool enableMaxNumTokensTuning{false}; - bool enableBlockReuse{false}; - bool enableChunkedContext{true}; - bool streaming{false}; - bool enableExpDelays{false}; - std::vector<std::optional<bool>> enableChunekedContextVec{std::nullopt}; - std::optional<float> requestRate{std::nullopt}; - std::optional<int> concurrency{std::nullopt}; - std::optional<SizeType32> maxBatchSize{std::nullopt}; - std::vector<std::optional<SizeType32>> maxBatchSizes{std::nullopt}; - std::optional<SizeType32> maxNumTokens{std::nullopt}; - std::vector<std::optional<SizeType32>> maxNumTokensVec{std::nullopt}; - int randomSeed = 430; - std::optional<std::vector<int>> maxAttentionWindowVec{std::nullopt}; - std::optional<int> sinkTokenLength{std::nullopt}; - bool multiBlockMode{true}; - bool enableContextFMHAFP32Acc{false}; - bool cudaGraphMode{false}; - SizeType32 cudaGraphCacheSize{0}; - - // lora / peft params - std::optional<std::string> loraDir{std::nullopt}; - SizeType32 loraDeviceNumModLayers{0}; - size_t loraHostCacheSize{1024 * 2024 * 1024}; - - // KV cache block offloading - size_t kvHostCacheSize{0}; - - // Weights offloading - float gpuWeightsPercent{1.0}; - - // Decoding params - std::optional<std::vector<std::vector<SizeType32>>> medusaChoices; - - std::optional<texec::EagleConfig> eagleConfig; - std::optional<float> temperature; - - std::optional<texec::LookaheadDecodingConfig> executorLookaheadConfig; - std::optional<texec::LookaheadDecodingConfig> requestLookaheadConfig; - - bool enableCollectkvCacheTransferTime = false; - bool enableCollectIterStats = false; -}; - -struct RecordTimeMetric -{ - - RecordTimeMetric(std::string tag) - : mTag(std::move(tag)) - { - } - - std::string mTag; - - std::vector<float> mDataTimes; - - float mAvg; - float mP99; - float mP95; - float mP90; - float mP50; - float mMax; - float mMin; - - static float calcPercentile(std::vector<float> const& latencies, int percentile) - { - int const index = static_cast<int>(std::ceil((percentile / 100.0) * latencies.size())) - 1; - return latencies[index]; - } - - void calculate() - { - TLLM_CHECK_WITH_INFO(mDataTimes.size() > 0, "No data to calculate for tag:%s", mTag.c_str()); - mAvg = std::accumulate(mDataTimes.begin(), mDataTimes.end(), 0.F) / mDataTimes.size(); - - std::sort(mDataTimes.begin(), mDataTimes.end()); - - mP99 = calcPercentile(mDataTimes, 99); - mP90 = calcPercentile(mDataTimes, 90); - mP50 = calcPercentile(mDataTimes, 50); - mMax = mDataTimes.back(); - mMin = mDataTimes.front(); - } - - void report() const - { - - printf("[BENCHMARK] avg_%s(ms) %.2f\n", mTag.c_str(), mAvg); - printf("[BENCHMARK] max_%s(ms) %.2f\n", mTag.c_str(), mMax); - printf("[BENCHMARK] min_%s(ms) %.2f\n", mTag.c_str(), mMin); - - printf("[BENCHMARK] p99_%s(ms) %.2f\n", mTag.c_str(), mP99); - - printf("[BENCHMARK] p90_%s(ms) %.2f\n", mTag.c_str(), mP90); - - printf("[BENCHMARK] p50_%s(ms) %.2f\n\n", mTag.c_str(), mP50); - } - - std::vector<std::string> genHeaders() const - { - std::string timeTag = mTag + "(ms)"; - return { - "avg_" + timeTag, "max_" + timeTag, "min_" + timeTag, "p99" + timeTag, "p90" + timeTag, "p50" + timeTag}; - } -}; - -struct RecordBwMetric -{ - - RecordBwMetric(std::string tag) - : mTag(std::move(tag)) - { - } - - std::string mTag; - - std::vector<float> mDataTps; - - float mAvg; - float mP99; - float mP95; - float mP90; - float mP50; - float mMax; - float mMin; - - static float calcPercentile(std::vector<float> const& throughputs, int percentile) - { - int const index = static_cast<int>(std::ceil((percentile / 100.0) * throughputs.size())) - 1; - return throughputs[index]; - } - - void calculate() - { - TLLM_CHECK_WITH_INFO(mDataTps.size() > 0, "No data to calculate for tag:%s", mTag.c_str()); - mAvg = std::accumulate(mDataTps.begin(), mDataTps.end(), 0.F) / mDataTps.size(); - - std::sort(mDataTps.begin(), mDataTps.end(), std::greater<float>()); - - mP99 = calcPercentile(mDataTps, 99); - mP90 = calcPercentile(mDataTps, 90); - mP50 = calcPercentile(mDataTps, 50); - mMax = mDataTps.front(); - mMin = mDataTps.back(); - } - - void report() const - { - - printf("[BENCHMARK] avg_%s(Gb/sec) %.8f\n", mTag.c_str(), mAvg); - printf("[BENCHMARK] max_%s(Gb/sec) %.8f\n", mTag.c_str(), mMax); - printf("[BENCHMARK] min_%s(Gb/sec) %.8f\n", mTag.c_str(), mMin); - - printf("[BENCHMARK] p99_%s(Gb/sec) %.8f\n", mTag.c_str(), mP99); - - printf("[BENCHMARK] p90_%s(Gb/sec) %.8f\n", mTag.c_str(), mP90); - - printf("[BENCHMARK] p50_%s(Gb/sec) %.8f\n\n", mTag.c_str(), mP50); - } - - std::vector<std::string> genHeaders() const - { - std::string tpTag = mTag + "(Gb/sec)"; - return {"avg_" + tpTag, "max_" + tpTag, "min_" + tpTag, "p99" + tpTag, "p90" + tpTag, "p50" + tpTag}; - } -}; - -std::ostream& operator<<(std::ostream& os, RecordTimeMetric const& metric); -std::ostream& operator<<(std::ostream& os, RecordBwMetric const& metric); - -struct Sample -{ - std::vector<int32_t> inputIds; - int32_t outputLen; - int32_t taskId; -}; - -using Samples = std::vector<Sample>; - -Samples parseWorkloadJson( - std::filesystem::path const& datasetPath, int maxNumSamples, std::optional<SizeType32> const maxPromptLen); - -std::vector<double> generateRandomExponentialValues(int count, float lambda, int seed); - -std::vector<double> computeTimeDelays(BenchmarkParams const& benchmarkParams, int numDelays); - -} // namespace benchmark - -TRTLLM_NAMESPACE_END diff --git a/benchmarks/cpp/utils/utils.py b/benchmarks/cpp/utils/utils.py deleted file mode 100644 index c395cf6c9449..000000000000 --- a/benchmarks/cpp/utils/utils.py +++ /dev/null @@ -1,168 +0,0 @@ -import json -import math -import os -import random -from typing import List, Union - -import numpy as np -from pydantic import BaseModel - - -class TextSample(BaseModel): - input_len: int - input_ids: List[int] - output_len: int - task_id: int - - -class MultimodalSample(BaseModel): - task_id: int - prompt: str - media_paths: List[str] - output_len: int - - -class Workload(BaseModel): - metadata: dict - samples: List[Union[TextSample, MultimodalSample]] = [] - - def __init__(self, **kwargs) -> None: - super().__init__(**kwargs) - self.setup_workload_name() - - def setup_workload_name(self): - # Keys to ignore - ignore_keys = ['tokenizer'] - # Create a string by concatenating keys and values with "__" - workload_name = '__'.join(f'{key}:{value}' - for key, value in self.metadata.items() - if key not in ignore_keys) - self.metadata.setdefault('workload_name', workload_name) - - -def text_dataset_dump(input_lens, input_ids, output_lens, task_ids, metadata, - output_file): - samples = [] - for i in range(len(input_ids)): - samples.append( - TextSample(input_len=input_lens[i], - input_ids=input_ids[i], - output_len=output_lens[i], - task_id=task_ids[i])) - workload = Workload(metadata=metadata, samples=samples) - os.makedirs(os.path.dirname(output_file), exist_ok=True) - with open(output_file, 'w') as f: - json.dump(workload.model_dump(), f) - - -def multimodal_dataset_dump(multimodal_texts, multimodal_image_paths, - output_lens, task_ids, metadata, output_file): - samples = [] - for i in range(len(multimodal_texts)): - samples.append( - MultimodalSample(task_id=task_ids[i], - prompt=multimodal_texts[i], - media_paths=multimodal_image_paths[i], - output_len=output_lens[i])) - workload = Workload(metadata=metadata, samples=samples) - os.makedirs(os.path.dirname(output_file), exist_ok=True) - with open(output_file, 'w') as f: - json.dump(workload.model_dump(), f) - - -def print_text_dataset(input_ids, output_lens, task_ids=None, lora_config=None): - for i, input_tokens in enumerate(input_ids): - d = { - "task_id": i, - "input_ids": input_tokens, - "output_tokens": output_lens[i] - } - - # Add LoRA request if task_ids indicate LoRA usage - if task_ids is not None and lora_config is not None: - task_id = task_ids[i] - if task_id != -1: # -1 means no LoRA - d["lora_request"] = { - "lora_name": - f"lora_{task_id}", - "lora_int_id": - task_id, - "lora_path": - os.path.join(lora_config.get("lora_dir", "loras"), - str(task_id)) - } - - print(json.dumps(d, separators=(',', ':'), ensure_ascii=False)) - - -def print_multimodal_dataset(multimodal_texts, multimodal_image_paths, - output_lens): - for i, (text, image_paths) in enumerate( - zip(multimodal_texts, multimodal_image_paths)): - d = { - "task_id": i, - "prompt": text, - "media_paths": image_paths, - "output_tokens": output_lens[i] - } - print(json.dumps(d, separators=(',', ':'), ensure_ascii=False)) - - -def get_list_of_delays(delay_dist, mean_time_bet_reqs, num_reqs, random_seed): - if delay_dist == "constant": - delays = [mean_time_bet_reqs] * num_reqs - elif delay_dist == "exponential_dist": - delays = get_exponential_dist_delays(mean_time_bet_reqs, num_reqs, - random_seed) - - return delays - - -def get_exponential_dist_delays(mean_time_bet_reqs, num_reqs, random_seed): - # set seed for determinism - np.random.seed(random_seed) - return np.random.exponential(mean_time_bet_reqs, num_reqs).tolist() - - -def get_norm_dist_lengths(mean, stdev, num_reqs, random_seed): - # set seed for determinism - np.random.seed(random_seed) - numbers_list = np.random.normal(loc=mean, scale=stdev, - size=num_reqs).tolist() - return [max(1, math.ceil(x)) for x in numbers_list] - - -def get_unif_dist_lengths(min_len, max_len, num_reqs, random_seed): - # set seed for determinism - rng = np.random.default_rng(random_seed) - numbers = rng.integers(low=min_len, high=max_len + 1, size=num_reqs) - return numbers.tolist() - - -def gen_random_tokens(ip_lens, tokenizer, random_seed): - - def get_sample_from_population(population_range, sample_size): - # random.sample can not sample a value more than once. hence the check - if sample_size < len(population_range): - sample = random.sample(population_range, sample_size) - else: - sample = random.choices(population_range, k=sample_size) - - return sample - - input_ids = [] - random.seed(random_seed) - for ip_len in ip_lens: - start_ids = get_sample_from_population(range(0, tokenizer.vocab_size), - ip_len) - # Make sure it does not contain EOS token - eos_id = tokenizer.encode(tokenizer.eos_token, add_special_tokens=False) - while set(eos_id).issubset(start_ids): - tmp_id = (eos_id[0] + 1) % tokenizer.vocab_size - start_ids = [ - tmp_id if element == eos_id[0] else element - for element in start_ids - ] - input_ids.append(start_ids) - - return input_ids diff --git a/constraints.txt b/constraints.txt index 09a933566229..7de1c603d547 100644 --- a/constraints.txt +++ b/constraints.txt @@ -5,9 +5,20 @@ nvidia-cutlass-dsl>=4.4.2 # The `nvidia-cutlass-dsl` package does not pin numpy at all, which can be problematic in certain CI # stages. numpy>=2.0.0,<2.4 # numba 0.63.1 requires numpy<2.4 -# WAR against https://github.com/advisories/GHSA-8mp2-v27r-99xp -mistune>=3.2.1 +# WAR against https://github.com/advisories/GHSA-qcq2-496w-v96p +mistune>=3.3.0 # WAR against https://github.com/advisories/GHSA-rch3-82jr-f9w9 notebook>=7.5.6 -# WAR against https://github.com/advisories/GHSA-rch3-82jr-f9w9 -jupyter_server>=2.18.0 +# WAR against https://github.com/advisories/GHSA-fcw5-x6j4-ccmp +jupyter_server>=2.20.0 +# WAR against https://github.com/advisories/GHSA-2wc2-fm75-p42x +soupsieve>=2.8.4 +# WAR against https://github.com/advisories/GHSA-82w8-qh3p-5jfq +# ! Need to bump fastapi first +# starlette>=1.3.1 +# WAR against https://github.com/advisories/GHSA-mgf9-4vpg-hj56 +tornado>=6.5.6 +# WAR against https://github.com/advisories/GHSA- +gitpython>=3.1.55 +# WAR against https://github.com/advisories/GHSA-gx64-gj6p-pc4c +jupyterlab>=4.5.10 diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a323b32b82b5..5dec92599ae6 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -45,7 +45,6 @@ add_compile_definitions(TRTLLM_ABI_NAMESPACE=${TRTLLM_ABI_NAMESPACE}) # Build options option(BUILD_PYT "Build in PyTorch TorchScript class mode" ON) option(BUILD_TESTS "Build Google tests" ON) -option(BUILD_BENCHMARKS "Build benchmarks" ON) option(BUILD_DEEP_EP "Build the Deep EP module" ON) option(BUILD_DEEP_GEMM "Build the DeepGEMM module" ON) option(BUILD_FLASH_MLA "Build the FlashMLA module" ON) @@ -126,12 +125,6 @@ else() message(STATUS "Not building Google tests") endif() -if(BUILD_BENCHMARKS) - message(STATUS "Building benchmarks") -else() - message(STATUS "Not building benchmarks") -endif() - if(BUILD_MICRO_BENCHMARKS) message(STATUS "Building C++ micro benchmarks") else() @@ -249,8 +242,6 @@ if(ENABLE_MULTI_DEVICE) endif() # TRT dependencies -find_package(TensorRT 10 REQUIRED COMPONENTS OnnxParser) -set(TRT_LIB TensorRT::NvInfer) get_filename_component(TRT_LLM_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR} PATH) @@ -291,7 +282,6 @@ include_directories( ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_INCLUDE_DIRS}/cccl ${CUDNN_ROOT_DIR}/include - $<TARGET_PROPERTY:TensorRT::NvInfer,INTERFACE_INCLUDE_DIRECTORIES> ${maybe_nvtx_includedir} ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include ${CMAKE_BINARY_DIR}/_deps/cutlass-src/tools/util/include @@ -669,11 +659,6 @@ if(BUILD_TESTS) add_subdirectory(tests) endif() -if(BUILD_BENCHMARKS) - add_subdirectory(${TRT_LLM_ROOT_DIR}/benchmarks/cpp - ${CMAKE_BINARY_DIR}/benchmarks) -endif() - if(BUILD_MICRO_BENCHMARKS) add_subdirectory(${TRT_LLM_ROOT_DIR}/cpp/micro_benchmarks ${CMAKE_BINARY_DIR}/micro_benchmarks) @@ -688,6 +673,6 @@ if(MEASURE_BUILD_TIME) endif() set(BUILD_WHEEL_TARGETS - tensorrt_llm;nvinfer_plugin_tensorrt_llm + tensorrt_llm CACHE STRING "Targets used to build wheel") add_custom_target(build_wheel_targets DEPENDS ${BUILD_WHEEL_TARGETS}) diff --git a/cpp/cmake/modules/FindTensorRT.cmake b/cpp/cmake/modules/FindTensorRT.cmake deleted file mode 100644 index 9e7e35b51bae..000000000000 --- a/cpp/cmake/modules/FindTensorRT.cmake +++ /dev/null @@ -1,190 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# - -# TensorRT install path in docker image -set(TensorRT_WELL_KNOWN_ROOT /usr/local/tensorrt) - -find_path( - TensorRT_INCLUDE_DIR - NAMES NvInfer.h - PATHS ${TensorRT_WELL_KNOWN_ROOT}/include) - -function(_tensorrt_get_version) - unset(TensorRT_VERSION_STRING PARENT_SCOPE) - set(_hdr_file "${TensorRT_INCLUDE_DIR}/NvInferVersion.h") - - if(NOT EXISTS "${_hdr_file}") - return() - endif() - - file(STRINGS "${_hdr_file}" IS_10_11_NEW_MACRO REGEX "TRT_MAJOR_ENTERPRISE") - if(IS_10_11_NEW_MACRO) - file(STRINGS "${_hdr_file}" VERSION_STRINGS - REGEX "#define TRT_.+_ENTERPRISE.*") - foreach(TYPE MAJOR MINOR PATCH BUILD) - string(REGEX MATCH "TRT_${TYPE}_ENTERPRISE [0-9]+" TRT_TYPE_STRING - ${VERSION_STRINGS}) - string(REGEX MATCH "[0-9]+" TensorRT_VERSION_${TYPE} ${TRT_TYPE_STRING}) - endforeach(TYPE) - else() - file(STRINGS "${_hdr_file}" VERSION_STRINGS REGEX "#define NV_TENSORRT_.*") - foreach(TYPE MAJOR MINOR PATCH BUILD) - string(REGEX MATCH "NV_TENSORRT_${TYPE} [0-9]+" TRT_TYPE_STRING - ${VERSION_STRINGS}) - string(REGEX MATCH "[0-9]+" TensorRT_VERSION_${TYPE} ${TRT_TYPE_STRING}) - endforeach(TYPE) - endif() - - set(TensorRT_VERSION_MAJOR - ${TensorRT_VERSION_MAJOR} - PARENT_SCOPE) - set(TensorRT_VERSION_STRING - "${TensorRT_VERSION_MAJOR}.${TensorRT_VERSION_MINOR}.${TensorRT_VERSION_PATCH}.${TensorRT_VERSION_BUILD}" - PARENT_SCOPE) -endfunction(_tensorrt_get_version) - -_tensorrt_get_version() - -macro(_tensorrt_find_dll VAR) - find_file( - ${VAR} - NAMES ${ARGN} - HINTS ${TensorRT_ROOT} - PATH_SUFFIXES bin) -endmacro(_tensorrt_find_dll) - -find_library( - TensorRT_LIBRARY - NAMES "nvinfer_${TensorRT_VERSION_MAJOR}" nvinfer - PATHS ${TensorRT_WELL_KNOWN_ROOT}/lib) - -if(WIN32) - _tensorrt_find_dll(TensorRT_DLL "nvinfer_${TensorRT_VERSION_MAJOR}.dll" - nvinfer.dll) -endif() - -if(TensorRT_LIBRARY) - set(TensorRT_LIBRARIES ${TensorRT_LIBRARIES} ${TensorRT_LIBRARY}) -endif(TensorRT_LIBRARY) - -if(TensorRT_FIND_COMPONENTS) - list(REMOVE_ITEM TensorRT_FIND_COMPONENTS "nvinfer") - - if("OnnxParser" IN_LIST TensorRT_FIND_COMPONENTS) - find_path( - TensorRT_OnnxParser_INCLUDE_DIR - NAMES NvOnnxParser.h - PATHS ${TensorRT_WELL_KNOWN_ROOT}/include) - - find_library( - TensorRT_OnnxParser_LIBRARY - NAMES "nvonnxparser_${TensorRT_VERSION_MAJOR}" nvonnxparser - PATHS ${TensorRT_WELL_KNOWN_ROOT}/lib) - if(TensorRT_OnnxParser_LIBRARY AND TensorRT_LIBRARIES) - set(TensorRT_LIBRARIES ${TensorRT_LIBRARIES} - ${TensorRT_OnnxParser_LIBRARY}) - set(TensorRT_OnnxParser_FOUND TRUE) - endif() - - if(WIN32) - _tensorrt_find_dll( - TensorRT_OnnxParser_DLL "nvonnxparser_${TensorRT_VERSION_MAJOR}.dll" - nvonnxparser.dll) - endif() - endif() - - if("Plugin" IN_LIST TensorRT_FIND_COMPONENTS) - find_path( - TensorRT_Plugin_INCLUDE_DIR - NAMES NvInferPlugin.h - PATHS ${TensorRT_WELL_KNOWN_ROOT}/include) - - find_library( - TensorRT_Plugin_LIBRARY - NAMES "nvinfer_plugin_${TensorRT_VERSION_MAJOR}" nvinfer_plugin - PATHS ${TensorRT_WELL_KNOWN_ROOT}/lib) - - if(TensorRT_Plugin_LIBRARY AND TensorRT_LIBRARIES) - set(TensorRT_LIBRARIES ${TensorRT_LIBRARIES} ${TensorRT_Plugin_LIBRARY}) - set(TensorRT_Plugin_FOUND TRUE) - endif() - - if(WIN32) - _tensorrt_find_dll( - TensorRT_Plugin_DLL "nvinfer_plugin_${TensorRT_VERSION_MAJOR}.dll" - nvinfer_plugin.dll) - endif() - endif() -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - TensorRT - FOUND_VAR TensorRT_FOUND - REQUIRED_VARS TensorRT_LIBRARY TensorRT_LIBRARIES TensorRT_INCLUDE_DIR - VERSION_VAR TensorRT_VERSION_STRING - HANDLE_COMPONENTS) - -if(NOT TARGET TensorRT::NvInfer) - add_library(TensorRT::NvInfer SHARED IMPORTED) - target_include_directories(TensorRT::NvInfer SYSTEM - INTERFACE "${TensorRT_INCLUDE_DIR}") - if(WIN32) - set_property(TARGET TensorRT::NvInfer PROPERTY IMPORTED_LOCATION - "${TensorRT_DLL}") - set_property(TARGET TensorRT::NvInfer PROPERTY IMPORTED_IMPLIB - "${TensorRT_LIBRARY}") - else() - set_property(TARGET TensorRT::NvInfer PROPERTY IMPORTED_LOCATION - "${TensorRT_LIBRARY}") - endif() -endif() - -if(NOT TARGET TensorRT::OnnxParser AND "OnnxParser" IN_LIST - TensorRT_FIND_COMPONENTS) - add_library(TensorRT::OnnxParser SHARED IMPORTED) - target_include_directories(TensorRT::OnnxParser SYSTEM - INTERFACE "${TensorRT_OnnxParser_INCLUDE_DIR}") - target_link_libraries(TensorRT::OnnxParser INTERFACE TensorRT::NvInfer) - if(WIN32) - set_property(TARGET TensorRT::OnnxParser - PROPERTY IMPORTED_LOCATION "${TensorRT_OnnxParser_DLL}") - set_property(TARGET TensorRT::OnnxParser - PROPERTY IMPORTED_IMPLIB "${TensorRT_OnnxParser_LIBRARY}") - else() - set_property(TARGET TensorRT::OnnxParser - PROPERTY IMPORTED_LOCATION "${TensorRT_OnnxParser_LIBRARY}") - endif() -endif() - -if(NOT TARGET TensorRT::Plugin AND "Plugin" IN_LIST TensorRT_FIND_COMPONENTS) - add_library(TensorRT::Plugin SHARED IMPORTED) - target_include_directories(TensorRT::Plugin SYSTEM - INTERFACE "${TensorRT_Plugin_INCLUDE_DIR}") - target_link_libraries(TensorRT::Plugin INTERFACE TensorRT::NvInfer) - if(WIN32) - set_property(TARGET TensorRT::Plugin PROPERTY IMPORTED_LOCATION - "${TensorRT_Plugin_DLL}") - set_property(TARGET TensorRT::Plugin PROPERTY IMPORTED_IMPLIB - "${TensorRT_Plugin_LIBRARY}") - else() - set_property(TARGET TensorRT::Plugin PROPERTY IMPORTED_LOCATION - "${TensorRT_Plugin_LIBRARY}") - endif() -endif() - -mark_as_advanced(TensorRT_INCLUDE_DIR TensorRT_LIBRARY TensorRT_LIBRARIES) diff --git a/cpp/include/tensorrt_llm/batch_manager/cacheTransceiver.h b/cpp/include/tensorrt_llm/batch_manager/cacheTransceiver.h index 0fcee1435005..1b0fc4d23200 100644 --- a/cpp/include/tensorrt_llm/batch_manager/cacheTransceiver.h +++ b/cpp/include/tensorrt_llm/batch_manager/cacheTransceiver.h @@ -22,15 +22,20 @@ #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/batch_manager/rnnCacheTransBuffer.h" #include "tensorrt_llm/batch_manager/rnnStateManager.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cacheCommunicator.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/pgUtils.h" +#include <atomic> +#include <cstddef> +#include <fstream> #include <future> #include <memory> #include <mutex> #include <optional> #include <pybind11/pybind11.h> +#include <string> #include <torch/csrc/jit/python/pybind_utils.h> #include <torch/custom_class.h> #include <torch/python.h> @@ -54,6 +59,7 @@ class BaseKVCacheManager; class CacheSender; class CacheReceiver; +class ContextTransferCoordinator; class CacheTransceiverComm { @@ -148,6 +154,25 @@ class CacheTransceiverComm TLLM_THROW("Input arguments only supported in mpi"); } + [[nodiscard]] std::unique_ptr<mpi::MpiRequest> sendAsync( + void const* buffer, std::size_t size, mpi::MpiType dtype, int dest, mpi::MpiTag tag) const + { + TLLM_CHECK_WITH_INFO(isMpi(), "Point-to-point cache-transceiver status messages require MPI."); + return mMpiComm->sendAsync(buffer, size, dtype, dest, tag); + } + + [[nodiscard]] bool iprobe(int source, mpi::MpiTag tag, MPI_Status* status) const + { + TLLM_CHECK_WITH_INFO(isMpi(), "Point-to-point cache-transceiver status messages require MPI."); + return mMpiComm->iprobe(source, tag, status); + } + + void recv(void* buffer, std::size_t size, mpi::MpiType dtype, int source, mpi::MpiTag tag) const + { + TLLM_CHECK_WITH_INFO(isMpi(), "Point-to-point cache-transceiver status messages require MPI."); + static_cast<void>(mMpiComm->recv(buffer, size, dtype, source, tag)); + } + CacheTransceiverComm split(int color, int key) { if (isMpi()) @@ -227,6 +252,12 @@ class BaseCacheTransceiver virtual bool cancelRequest(std::shared_ptr<LlmRequest> llmRequest) = 0; + /// Get the serialized DataTransceiverState (CacheState + CommState) for this transceiver. + [[nodiscard]] virtual std::vector<char> getSerializedDataTransceiverState() const + { + return {}; + } + [[nodiscard]] virtual bool hasPoisonedTransferBuffer() const { return false; @@ -238,7 +269,7 @@ class CacheTransceiver : public BaseCacheTransceiver public: CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, executor::kv_cache::CacheState::ModelConfig const& cacheStateModelCfg, runtime::WorldConfig const& worldConfig, - std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, executor::kv_cache::CacheState::AttentionType attentionType = executor::kv_cache::CacheState::AttentionType::kDEFAULT, std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig = std::nullopt, @@ -246,7 +277,7 @@ class CacheTransceiver : public BaseCacheTransceiver CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, std::vector<SizeType32> numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, runtime::WorldConfig const& worldConfig, - std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, executor::kv_cache::CacheState::AttentionType attentionType = executor::kv_cache::CacheState::AttentionType::kDEFAULT, std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig = std::nullopt, @@ -276,13 +307,51 @@ class CacheTransceiver : public BaseCacheTransceiver virtual bool cancelRequest(std::shared_ptr<LlmRequest> llmRequest) override; + [[nodiscard]] std::vector<char> getSerializedDataTransceiverState() const override; + [[nodiscard]] bool hasPoisonedTransferBuffer() const override; + /// Return a human-readable dump of transceiver state for debugging hangs. + std::string getStatusDump() const; private: + struct StatusSnapshot + { + size_t senderAsyncActive{0}; + size_t requesterAsyncActive{0}; + size_t timedOutSenders{0}; + size_t timedOutRequesters{0}; + size_t cancelingSenders{0}; + size_t cancelingRequesters{0}; + size_t completedSenders{0}; + size_t completedRequesters{0}; + size_t failedSenders{0}; + size_t failedRequesters{0}; + size_t sendersAwaitingConsensus{0}; + size_t requestersAwaitingConsensus{0}; + }; + + class SyncRequesterStatusGuard + { + public: + explicit SyncRequesterStatusGuard(CacheTransceiver& transceiver); + ~SyncRequesterStatusGuard() noexcept; + + SyncRequesterStatusGuard(SyncRequesterStatusGuard const&) = delete; + SyncRequesterStatusGuard& operator=(SyncRequesterStatusGuard const&) = delete; + + private: + CacheTransceiver& mTransceiver; + }; + void initializeCommState(); void setContextState(LlmRequest* llmRequest); + // Append one row per completed request to the gen-side transfer summary CSV. Opens the file + // lazily on first use; expects timing to already be synced across ranks by the caller. + void writeGenTransferSummary(std::vector<LlmRequest*> const& completedRequests); + void publishStatusSnapshot() noexcept; + std::unique_ptr<CacheSender> mCacheSender; std::unique_ptr<CacheReceiver> mCacheReceiver; // shared_ptr (not raw LlmRequest*) so the futures hold a strong reference for @@ -302,10 +371,17 @@ class CacheTransceiver : public BaseCacheTransceiver std::unordered_set<LlmRequest::RequestIdType> mCompletedRequesterRequestIds; std::unordered_set<LlmRequest::RequestIdType> mFailedRequesterRequestIds; std::unordered_map<LlmRequest::RequestIdType, std::shared_ptr<LlmRequest>> mRequesterRequestsAwaitingConsensus; + std::atomic_size_t mSyncRequesterActive{0}; + // Live transfer containers are owned by the executor worker thread. Synchronous receive threads update only the + // atomic count above. The executor publishes snapshots after state transitions, while the hang-detector thread only + // copies the snapshot under this short lock. + mutable std::mutex mStatusSnapshotMutex; + StatusSnapshot mStatusSnapshot; mpi::MpiComm const* mMpiWorldComm{nullptr}; std::shared_ptr<CacheTransceiverComm> mGroupComm; std::shared_ptr<CacheTransceiverComm> mGroupTensorParaComm, mGroupPipeParaComm, mGroupDataComm, mGroupTPInDPComm; + std::unique_ptr<ContextTransferCoordinator> mContextTransferCoordinator; executor::kv_cache::CommState const* mCommState; std::unique_ptr<executor::kv_cache::CacheState> mCacheState; @@ -317,6 +393,13 @@ class CacheTransceiver : public BaseCacheTransceiver // TODO(shreyasm): update this to use same container as kv by using base trans buffers instead std::unique_ptr<rnn_state_manager::RnnCacheTransBufferManager> mRnnCacheTransBufferManager{nullptr}; + // Unique instance identifier for CSV file naming (avoids collisions across gen instances) + std::string mInstanceId; + + // Gen-side transfer summary CSV (written after timing sync) + std::ofstream mGenTransferSummaryFile; + std::mutex mGenTransferSummaryMutex; + // library handle to the communicator related features, // this is used to defer dependency resolution until needed. static std::mutex mDllMutex; diff --git a/cpp/include/tensorrt_llm/batch_manager/contextTransferCoordinator.h b/cpp/include/tensorrt_llm/batch_manager/contextTransferCoordinator.h new file mode 100644 index 000000000000..9b156c8887a0 --- /dev/null +++ b/cpp/include/tensorrt_llm/batch_manager/contextTransferCoordinator.h @@ -0,0 +1,109 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <cstddef> +#include <cstdint> +#include <memory> +#include <unordered_map> +#include <unordered_set> +#include <vector> + +namespace tensorrt_llm::batch_manager +{ + +class CacheTransceiverComm; + +enum class ContextTransferVote : std::uint64_t +{ + kCompleted = 1, + kFailed = 2, +}; + +struct ContextTransferConsensusResult +{ + std::unordered_set<std::uint64_t> completedRequestIds; + std::unordered_set<std::uint64_t> failedRequestIds; + std::unordered_set<std::uint64_t> timedOutRequestIds; +}; + +//! Accumulates one immutable terminal vote per participant and request. +class ContextTransferVoteReducer +{ +public: + explicit ContextTransferVoteReducer(int participantCount); + + void recordVote(int participantRank, std::uint64_t requestId, ContextTransferVote vote); + + //! Record a sticky nonterminal timeout proposal. The final outcome still waits for every terminal vote. + void recordTimeout(std::uint64_t requestId); + + [[nodiscard]] ContextTransferConsensusResult takeReady(); + + void clear() noexcept; + +private: + struct RequestVotes + { + explicit RequestVotes(int participantCount) + : votes(static_cast<std::size_t>(participantCount), 0) + { + } + + std::vector<std::uint64_t> votes; + int terminalCount{0}; + bool failed{false}; + bool timedOut{false}; + bool timeoutPending{false}; + }; + + int mParticipantCount; + std::unordered_map<std::uint64_t, RequestVotes> mRequestVotes; +}; + +//! Coordinates asynchronous context-transfer outcomes across a rank group. +//! +//! Timeout and terminal events share one ordered stream toward the coordinator. Timeout updates and final commits +//! share another ordered stream toward followers, so a request that times out can never commit success first. +class ContextTransferCoordinator +{ +public: + explicit ContextTransferCoordinator(std::shared_ptr<CacheTransceiverComm> comm); + ~ContextTransferCoordinator(); + + ContextTransferCoordinator(ContextTransferCoordinator const&) = delete; + ContextTransferCoordinator& operator=(ContextTransferCoordinator const&) = delete; + + //! Publish this rank's immutable local terminal outcome without waiting for peers. + void publishLocalOutcome(std::uint64_t requestId, bool failed); + + //! Publish an idempotent, sticky, nonterminal timeout proposal without waiting for peers. + void publishTimeout(std::uint64_t requestId); + + //! Make nonblocking protocol progress and return newly committed global outcomes. + [[nodiscard]] ContextTransferConsensusResult poll(); + + //! Exchange ordered close markers so no active MPI request outlives its backing buffer. Failure aborts closed. + void shutdown() noexcept; + +private: + class Impl; + std::unique_ptr<Impl> mImpl; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/createNewDecoderRequests.h b/cpp/include/tensorrt_llm/batch_manager/createNewDecoderRequests.h index bc619a34bc03..600927af9645 100644 --- a/cpp/include/tensorrt_llm/batch_manager/createNewDecoderRequests.h +++ b/cpp/include/tensorrt_llm/batch_manager/createNewDecoderRequests.h @@ -20,6 +20,7 @@ #include "tensorrt_llm/batch_manager/common.h" #include "tensorrt_llm/common/algorithm.h" #include "tensorrt_llm/common/optionalRef.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -66,16 +67,17 @@ class CreateNewDecoderRequests : Algorithm std::vector<executor::LookaheadDecodingConfig>> operator()(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, executor::DecodingConfig const& decodingConfig, RequestVector const& contextRequests, - nvinfer1::DataType logitsType, DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState, - CudaStream const& runtimeStream, CudaStream const& decoderStream, SizeType32 maxSequenceLength, - SizeType32 beamWidth, OptionalRef<MedusaBuffers const> medusaBuffers) const; + tensorrt_llm::DataType logitsType, DecoderInputBuffers& inputBuffers, + runtime::decoder::DecoderState& decoderState, CudaStream const& runtimeStream, CudaStream const& decoderStream, + SizeType32 maxSequenceLength, SizeType32 beamWidth, OptionalRef<MedusaBuffers const> medusaBuffers) const; [[nodiscard]] std::tuple<std::vector<SharedConstPtr>, std::vector<executor::LookaheadDecodingConfig>> createDecoderRequests(RequestVector const& finishedContextRequests, TensorPtr const& inputIds, executor::DecodingConfig const& decodingConfig, runtime::decoder::DecoderState& decoderState, - nvinfer1::DataType logitsType, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - runtime::CudaStream const& runtimeStream, runtime::CudaStream const& decoderStream, - SizeType32 maxSequenceLength, OptionalRef<MedusaBuffers const> medusaBuffers) const; + tensorrt_llm::DataType logitsType, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig, runtime::CudaStream const& runtimeStream, + runtime::CudaStream const& decoderStream, SizeType32 maxSequenceLength, + OptionalRef<MedusaBuffers const> medusaBuffers) const; private: bool mSpeculativeDecodingFastLogits; diff --git a/cpp/include/tensorrt_llm/batch_manager/guidedDecoder.h b/cpp/include/tensorrt_llm/batch_manager/guidedDecoder.h deleted file mode 100644 index 9a577b61ad51..000000000000 --- a/cpp/include/tensorrt_llm/batch_manager/guidedDecoder.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/iTensor.h" - -namespace xgrammar -{ -class GrammarMatcher; -class GrammarCompiler; -} // namespace xgrammar - -namespace tensorrt_llm::batch_manager -{ -class DecoderInputBuffers; - -class GuidedDecoder -{ -public: - using TensorPtr = runtime::ITensor::SharedPtr; - using SizeType32 = tensorrt_llm::runtime::SizeType32; - using BitmaskT = uint32_t; - - GuidedDecoder(executor::GuidedDecodingConfig const& guidedDecodingConfig, SizeType32 maxNumSequences, - SizeType32 vocabSizePadded, nvinfer1::DataType logitsDtype, runtime::BufferManager const& runtimeBufferManager); - void build(ScheduledRequests const& scheduledRequests); - void execute(DecoderInputBuffers const& decoderInputBuffers, runtime::BufferManager const& runtimeBufferManager); - -private: - executor::GuidedDecodingConfig::GuidedDecodingBackend mGuidedDecodingBackend; - std::vector<std::shared_ptr<xgrammar::GrammarMatcher>> mXGrammarMatchers; - std::shared_ptr<xgrammar::GrammarCompiler> mXGrammarCompiler; - - SizeType32 mMaxNumSequences; - SizeType32 mVocabSizePadded; - SizeType32 mBitmaskSize; // CeilDiv(vocabSizePadded, 32) - nvinfer1::DataType mLogitsDtype; - - TensorPtr mLogitsBitmask; // [mMaxNumRequests, mBitmaskSize] - TensorPtr mLogitsBitmaskHost; // [mMaxNumRequests, mBitmaskSize] - TensorPtr mLogitsBitmaskPtrVec; // [mMaxNumRequests], pointers to the logitsBitmask in a batch - TensorPtr mLogitsBitmaskPtrVecHost; // [mMaxNumRequests] - TensorPtr mLogitsPtrVec; // [mMaxNumRequests], pointers to the logits in a batch - TensorPtr mLogitsPtrVecHost; // [mMaxNumRequests] - - // BufferManager with a dedicated stream for async copy of buffers for guided decoding. - runtime::BufferManager mCopyBufferManager; -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/handleContextLogits.h b/cpp/include/tensorrt_llm/batch_manager/handleContextLogits.h deleted file mode 100644 index cb77545578c8..000000000000 --- a/cpp/include/tensorrt_llm/batch_manager/handleContextLogits.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/common/algorithm.h" -#include "tensorrt_llm/common/optionalRef.h" -#include "tensorrt_llm/runtime/modelConfig.h" - -namespace tensorrt_llm::runtime -{ -class BufferManager; -class CudaStream; -} // namespace tensorrt_llm::runtime - -namespace tensorrt_llm::batch_manager -{ - -class DecoderInputBuffers; -class MedusaBuffers; - -class HandleContextLogits : Algorithm -{ -public: - template <typename T> - using OptionalRef = tensorrt_llm::common::OptionalRef<T>; - - constexpr static auto name{"HandleContextLogits"}; - - HandleContextLogits() = default; - - runtime::SizeType32 operator()(DecoderInputBuffers& inputBuffers, RequestVector const& contextRequests, - runtime::ITensor::SharedPtr const& logits, std::vector<runtime::SizeType32> const& numContextLogitsVec, - runtime::ModelConfig const& modelConfig, runtime::BufferManager const& manager, - OptionalRef<MedusaBuffers> medusaBuffers) const; -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/handleGenerationLogits.h b/cpp/include/tensorrt_llm/batch_manager/handleGenerationLogits.h deleted file mode 100644 index f9fd58800a6f..000000000000 --- a/cpp/include/tensorrt_llm/batch_manager/handleGenerationLogits.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "common.h" -#include "tensorrt_llm/common/algorithm.h" -#include "tensorrt_llm/common/optionalRef.h" -#include "tensorrt_llm/runtime/modelConfig.h" - -namespace tensorrt_llm::runtime -{ -class BufferManager; -} // namespace tensorrt_llm::runtime - -namespace tensorrt_llm::batch_manager -{ - -class DecoderInputBuffers; -class RuntimeBuffers; -class MedusaBuffers; - -class HandleGenerationLogits : Algorithm -{ -public: - template <typename T> - using OptionalRef = tensorrt_llm::common::OptionalRef<T>; - - constexpr static auto name{"HandleGenerationLogits"}; - - HandleGenerationLogits() = default; - - void operator()(DecoderInputBuffers& inputBuffers, RequestVector const& generationRequests, - runtime::ITensor::SharedPtr const& logits, runtime::SizeType32 logitsIndex, - runtime::ModelConfig const& modelConfig, runtime::BufferManager const& manager, - OptionalRef<RuntimeBuffers> genRuntimeBuffers, OptionalRef<MedusaBuffers> medusaBuffers) const; -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h b/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h index e821cafd5c3f..04b6c230e75a 100644 --- a/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h +++ b/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h @@ -23,6 +23,7 @@ #include "tensorrt_llm/batch_manager/llmRequest.h" // TODO forward declare #include "tensorrt_llm/batch_manager/radixBlockTree.h" #include "tensorrt_llm/common/optionalRef.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/executor/transferAgent.h" #include "tensorrt_llm/kernels/kvCacheIndex.h" @@ -32,7 +33,6 @@ #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include <NvInferRuntime.h> #include <algorithm> #include <array> @@ -140,7 +140,7 @@ struct PoolConfiguration { SizeType32 windowSize; SizeType32 sizePerHead; - nvinfer1::DataType dtype; + tensorrt_llm::DataType dtype; }; struct LinearAttentionMetadata @@ -872,7 +872,7 @@ class WindowBlockManager using BlockMap = std::unordered_multimap<size_t, BlockPtr>; using BlockMapIterRange = std::pair<BlockMap::const_iterator, BlockMap::const_iterator>; - explicit WindowBlockManager(nvinfer1::DataType dtype, SizeType32 windowSize, + explicit WindowBlockManager(tensorrt_llm::DataType dtype, SizeType32 windowSize, std::vector<SizeType32> const& managedLayers, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, bool isSWA, SizeType32 blocksInPrimaryPool, SizeType32 blocksInSecondaryPool, SizeType32 maxNumSequences, std::shared_ptr<runtime::CudaStream> stream, @@ -1037,7 +1037,7 @@ class WindowBlockManager //! host pools with mixed precisions when constructed with a per-window //! dtype map. Empty pools or NVFP4-scale pools are routed through the //! per-pool tensor metadata instead. - [[nodiscard]] nvinfer1::DataType getDataType() const noexcept + [[nodiscard]] tensorrt_llm::DataType getDataType() const noexcept { return mDataType; } @@ -1127,7 +1127,7 @@ class WindowBlockManager [[nodiscard]] SizeType32 getNumEltsPerContainer() const { #ifdef ENABLE_FP4 - return mDataType == nvinfer1::DataType::kFP4 ? 2 : 1; + return mDataType == tensorrt_llm::DataType::kFP4 ? 2 : 1; #else return 1; #endif @@ -1192,7 +1192,7 @@ class WindowBlockManager return mLayerToIndexWithinPool.at(layerIdx); } - void setOffsets(kernels::KVCacheIndex* offsetsPtr, nvinfer1::Dims const& offsetsShape, SizeType32 beamIdx, + void setOffsets(kernels::KVCacheIndex* offsetsPtr, tensorrt_llm::Dims const& offsetsShape, SizeType32 beamIdx, SizeType32 blockIdx, KVCacheBlock::IdType blockId) const; //! \brief Bring offloaded block from secondary to primary memory. @@ -1251,14 +1251,29 @@ class WindowBlockManager return mEnablePartialReuse; } + //! \brief Look up the block chain matching blockKey in the reuse tree. [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKey(BlockKey const& blockKey); + //! \brief Same lookup, additionally pinning matched blocks; on a miss all pins are + //! rolled back and pinnedBlockIds is cleared. + [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKey( + BlockKey const& blockKey, std::vector<KVCacheBlock::IdType>& pinnedBlockIds); + [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKeys( std::vector<BlockKey> const& blockKeys); //! \brief Unpin blocks by block ids directly void unpinBlocksById(std::vector<KVCacheBlock::IdType> const& blockIds); + //! \brief Pin a block: claim it from the eviction policy if free, then take a reference. + //! Safe to call from cache-transceiver threads: block bookkeeping is serialized by the + //! lookup-tree mutex, which every mutating entry point acquires. + void pinBlock(BlockPtr const& block); + + //! \brief Inverse of pinBlock: drop one reference and release the block back to the + //! eviction policy once no references remain. + void unpinBlock(BlockPtr const& block); + void truncateBlocks(LlmRequest::VecTokens const& targetTokens, SizeType32 numTokensToKeep); void resetReuseState() @@ -1274,6 +1289,10 @@ class WindowBlockManager } private: + //! \brief Shared implementation of the findBlocksInReuseTreeByBlockKey overloads. + [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKeyImpl( + BlockKey const& blockKey, bool pinBlocks, std::vector<KVCacheBlock::IdType>& pinnedBlockIds); + //! \brief Walk the reuse tree with precomputed per-block keys (no lock; callers must hold mLookupTree->getMutex()). [[nodiscard]] std::shared_ptr<KVCacheBlock> searchReuseTree(std::vector<BlockKey> const& blockKeys); @@ -1353,7 +1372,7 @@ class WindowBlockManager } private: - nvinfer1::DataType mDataType; + tensorrt_llm::DataType mDataType; SizeType32 mWindowSize; // Number of blocks in pools @@ -1481,7 +1500,7 @@ class BlockManager explicit BlockManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkBubbleLength, + std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkBubbleLength, SizeType32 chunkSize, CacheType cacheType = CacheType::kSELF, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt, std::shared_ptr<KVCacheEventManager> eventManager = nullptr, bool enablePartialReuse = true, @@ -1563,7 +1582,7 @@ class BlockManager void releaseLastBlock(GenerationRequest& sequence, SizeType32 windowSize); - void setOffsets(kernels::KVCacheIndex* offsetsPtr, nvinfer1::Dims const& offsetsShape, SizeType32 beamIdx, + void setOffsets(kernels::KVCacheIndex* offsetsPtr, tensorrt_llm::Dims const& offsetsShape, SizeType32 beamIdx, SizeType32 blockIdx, KVCacheBlock::IdType blockId, SizeType32 windowSize) const; //! \brief Combined prefix reuse analysis — single radix tree walk. @@ -1621,9 +1640,9 @@ class BlockManager //! \brief Convenience: window_size -> dataType, derived from getPoolConfigurations(). //! For one-pool-per-window managers only; multi-pool-per-window will collide. - [[nodiscard]] std::map<SizeType32, nvinfer1::DataType> getDataTypePerWindow() const + [[nodiscard]] std::map<SizeType32, tensorrt_llm::DataType> getDataTypePerWindow() const { - std::map<SizeType32, nvinfer1::DataType> result; + std::map<SizeType32, tensorrt_llm::DataType> result; for (auto const& [windowSize, manager] : mWindowBlockManagers) { result[windowSize] = manager.getDataType(); @@ -1636,7 +1655,7 @@ class BlockManager return mWindowBlockManagers.at(windowSize).getSizePerHead(); } - [[nodiscard]] nvinfer1::DataType getDataTypeForWindow(SizeType32 windowSize) const + [[nodiscard]] tensorrt_llm::DataType getDataTypeForWindow(SizeType32 windowSize) const { return mWindowBlockManagers.at(windowSize).getDataType(); } @@ -1836,6 +1855,12 @@ class BlockManager return mWindowBlockManagers.at(windowSize).findBlocksInReuseTreeByBlockKey(blockKey); } + [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKey( + BlockKey const& blockKey, SizeType32 windowSize, std::vector<KVCacheBlock::IdType>& pinnedBlockIds) + { + return mWindowBlockManagers.at(windowSize).findBlocksInReuseTreeByBlockKey(blockKey, pinnedBlockIds); + } + [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKeys( std::vector<BlockKey> const& blockKeys, SizeType32 windowSize) { @@ -2189,7 +2214,7 @@ class BaseKVCacheManager /// head_dim=512). Empty vector = uniform @p sizePerHead / @p dtype across all windows. /// @return Map from window size to tuple of (primary blocks, secondary blocks) [[nodiscard]] static BlocksPerWindow calculateMaxNumBlocks(executor::KvCacheConfig const& config, - nvinfer1::DataType dtype, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, + tensorrt_llm::DataType dtype, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, tensorrt_llm::runtime::WorldConfig const& worldConfig, std::map<SizeType32, std::vector<SizeType32>> const& windowSizeToLayers, uint64_t allottedPrimaryMemBytes, uint64_t allottedSecondaryMemBytes, size_t extraCostMemory, SizeType32 kvFactor, SizeType32 maxBatchSize, @@ -2210,6 +2235,11 @@ class BaseKVCacheManager BlockKey const& blockKey, SizeType32 windowSize) = 0; + //! \brief Pinning lookup: pins matched blocks and records their ids for unpinBlocksById. + [[nodiscard]] virtual std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKey( + BlockKey const& blockKey, SizeType32 windowSize, std::vector<KVCacheBlock::IdType>& pinnedBlockIds) + = 0; + [[nodiscard]] virtual std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKeys( std::vector<BlockKey> const& blockKeys, SizeType32 windowSize) = 0; @@ -2276,7 +2306,7 @@ class KVCacheManager : public BaseKVCacheManager //! and disagg transfer machinery applies natively. Empty vector = uniform. KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, + std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = false, CacheType cacheType = CacheType::kSELF, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt, @@ -2290,7 +2320,7 @@ class KVCacheManager : public BaseKVCacheManager KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, + std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, int64_t stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = false, CacheType cacheType = CacheType::kSELF, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt, @@ -2304,7 +2334,7 @@ class KVCacheManager : public BaseKVCacheManager KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, + std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = true, CacheType cacheType = CacheType::kSELF, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt, @@ -2318,7 +2348,7 @@ class KVCacheManager : public BaseKVCacheManager KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, + std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, int64_t stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = false, CacheType cacheType = CacheType::kSELF, bool enablePartialReuse = true, bool copyOnpartialReuse = true, bool enableIndexerKCache = false, SizeType32 indexerKCacheQuantBlockSize = 128, @@ -2650,6 +2680,12 @@ class KVCacheManager : public BaseKVCacheManager return mBlockManager.findBlocksInReuseTreeByBlockKey(blockKey, windowSize); } + std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKey( + BlockKey const& blockKey, SizeType32 windowSize, std::vector<KVCacheBlock::IdType>& pinnedBlockIds) override + { + return mBlockManager.findBlocksInReuseTreeByBlockKey(blockKey, windowSize, pinnedBlockIds); + } + std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKeys( std::vector<BlockKey> const& blockKeys, SizeType32 windowSize) override { @@ -2682,7 +2718,7 @@ class KVCacheManager : public BaseKVCacheManager SizeType32 mMaxNumSequences; // Maximum beam width SizeType32 mMaxBeamWidth; - nvinfer1::DataType mDataType; + tensorrt_llm::DataType mDataType; // Maximum kv cache length per sequence SizeType32 mMaxAttentionWindow; // Number of tokens per block diff --git a/cpp/include/tensorrt_llm/batch_manager/llmRequest.h b/cpp/include/tensorrt_llm/batch_manager/llmRequest.h index bc1ca3e6d012..309e41c9e762 100644 --- a/cpp/include/tensorrt_llm/batch_manager/llmRequest.h +++ b/cpp/include/tensorrt_llm/batch_manager/llmRequest.h @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iBuffer.h" @@ -83,6 +84,15 @@ enum LlmRequestType class ContextProgress; +// Process-global offset between the local steady clock and the global steady +// clock (rank 0's steady clock). The storage lives in a single translation unit +// (llmRequest.cpp) and is reached through this accessor so that +// libtensorrt_llm.so and the nanobind extension module share one copy across +// .so boundaries. An inline-static member would instead give each shared object +// its own copy, so an offset calibrated on one side would be invisible to the +// other. +std::optional<std::chrono::steady_clock::duration>& globalSteadyClockOffset(); + template <typename TTensor, typename TStream = runtime::BufferManager::CudaStreamPtr> class GenericLlmRequest { @@ -221,6 +231,7 @@ class GenericLlmRequest mState = LlmRequestState::kENCODER_INIT; } + adoptContextPhaseDraftTokens(); initialize(*inputTokens, returnLogProbs, arrivalTime); } @@ -289,6 +300,7 @@ class GenericLlmRequest { mState = LlmRequestState::kENCODER_INIT; } + adoptContextPhaseDraftTokens(); initialize(inputTokens, returnLogProbs); } @@ -521,6 +533,7 @@ class GenericLlmRequest default: throw std::runtime_error("Unsupported request type found."); } + adoptContextPhaseDraftTokens(); initialize(req.getInputTokenIds(), req.getOutputConfig().returnLogProbs); } @@ -539,9 +552,29 @@ class GenericLlmRequest return mContextPhaseParams; } + /// @brief Get the number of generation tokens carried by context phase handoff. + /// @return Number of first generation tokens plus draft tokens. + [[nodiscard]] SizeType32 getNumContextPhaseGenerationTokens() const noexcept + { + if (!mContextPhaseParams.has_value()) + { + return 0; + } + + auto const& contextPhaseParams = mContextPhaseParams.value(); + auto numTokens = static_cast<SizeType32>(contextPhaseParams.getFirstGenTokens().size()); + auto const& draftTokens = contextPhaseParams.getDraftTokens(); + if (draftTokens.has_value()) + { + numTokens += static_cast<SizeType32>(draftTokens->size()); + } + return numTokens; + } + void setContextPhaseParams(executor::ContextPhaseParams contextPhaseParams) { mContextPhaseParams = std::move(contextPhaseParams); + adoptContextPhaseDraftTokens(); } /// @brief Get the state params of the context @@ -1233,6 +1266,18 @@ class GenericLlmRequest mEstimatedReusableTokens = estimatedReusableTokens; } + //! Get the absolute context positions at which recurrent-state snapshots are expected. + [[nodiscard]] std::vector<SizeType32> const& getExpectedSnapshotPoints() const noexcept + { + return mExpectedSnapshotPoints; + } + + //! Set the absolute context positions at which recurrent-state snapshots are expected. + void setExpectedSnapshotPoints(std::vector<SizeType32> expectedSnapshotPoints) + { + mExpectedSnapshotPoints = std::move(expectedSnapshotPoints); + } + void setDraftTokens(std::shared_ptr<VecTokens> const& draftTokens) { mDraftTokens = draftTokens; @@ -1312,7 +1357,7 @@ class GenericLlmRequest mEncoderOutput = std::move(encoderOutput); } - void allocEncoderOutputHost(SizeType32 encoderHiddenSize, nvinfer1::DataType dataType) + void allocEncoderOutputHost(SizeType32 encoderHiddenSize, tensorrt_llm::DataType dataType) { mEncoderOutputHost = runtime::BufferManager::pinned( runtime::ITensor::makeShape({getEncoderOutputLen(), encoderHiddenSize}), dataType); @@ -1328,13 +1373,13 @@ class GenericLlmRequest return mEncoderHiddenStates; } - void allocEncoderOutput(runtime::BufferManager const& manager, nvinfer1::DataType dataType) + void allocEncoderOutput(runtime::BufferManager const& manager, tensorrt_llm::DataType dataType) { // unique_ptr --> shared_ptr ownership move mEncoderOutput = std::move(manager.emptyTensor(runtime::MemoryType::kGPU, dataType)); } - void allocEncoderHiddenStates(runtime::BufferManager const& manager, nvinfer1::DataType dataType) + void allocEncoderHiddenStates(runtime::BufferManager const& manager, tensorrt_llm::DataType dataType) { // unique_ptr --> shared_ptr ownership move mEncoderHiddenStates = std::move(manager.emptyTensor(runtime::MemoryType::kGPU, dataType)); @@ -1452,7 +1497,7 @@ class GenericLlmRequest mContextLogitsHost = std::move(contextLogitsHost); } - void allocContextLogitsHost(SizeType32 vocabSizePadded, nvinfer1::DataType logitsDataType) + void allocContextLogitsHost(SizeType32 vocabSizePadded, tensorrt_llm::DataType logitsDataType) { mContextLogitsHost = runtime::BufferManager::pinnedPool( runtime::ITensor::makeShape({mPromptLen, vocabSizePadded}), logitsDataType); @@ -1471,7 +1516,7 @@ class GenericLlmRequest mGenerationLogitsHost = std::move(generationLogitsHost); } - void allocGenerationLogitsHost(SizeType32 vocabSizePadded, nvinfer1::DataType logitsDataType) + void allocGenerationLogitsHost(SizeType32 vocabSizePadded, tensorrt_llm::DataType logitsDataType) { if (mIsStreaming) { @@ -1490,7 +1535,7 @@ class GenericLlmRequest } } - void allocTargetModelAcceptedTokenLogitsHost(SizeType32 vocabSizePadded, nvinfer1::DataType logitsDataType) + void allocTargetModelAcceptedTokenLogitsHost(SizeType32 vocabSizePadded, tensorrt_llm::DataType logitsDataType) { mGenerationLogitsHost = runtime::BufferManager::pinnedPool( runtime::ITensor::makeShape({1, getNumDraftTokens() + 1, vocabSizePadded}), logitsDataType); @@ -1844,14 +1889,18 @@ class GenericLlmRequest mDecodingIter = iter; } + // Callers must pass a global-steady-clock time point (getSteadyClockNow(), + // or a value merged from such time points). Normalizing again here would + // apply the global steady clock offset twice, which corrupts cross-node + // min/max merging whenever the offset is non-zero. void setKvCacheTransferStart(TimePoint time) const { - mPerfMetrics.timingMetrics.kvCacheTransferStart = maybeToGlobalSteadyClock(time); + mPerfMetrics.timingMetrics.kvCacheTransferStart = time; } void setKvCacheTransferEnd(TimePoint time) const { - mPerfMetrics.timingMetrics.kvCacheTransferEnd = maybeToGlobalSteadyClock(time); + mPerfMetrics.timingMetrics.kvCacheTransferEnd = time; } TimePoint getKvCacheTransferStart() const @@ -2027,8 +2076,8 @@ class GenericLlmRequest return mUseDraftModel; } - // If sGlobalSteadyClockOffset is set, return a global steady clock time point, otherwise return local steady clock - // time point + // If the global steady clock offset is set, return a global steady clock time point, otherwise return local steady + // clock time point [[nodiscard]] static TimePoint getSteadyClockNow() { return maybeToGlobalSteadyClock(std::chrono::steady_clock::now()); @@ -2058,9 +2107,6 @@ class GenericLlmRequest // current position of the prompt tuning table (only used in chunked prefill mode) SizeType32 mPtableCurrentPosition{0}; - // The offset between local steady clock and global steady clock (at rank 0) - inline static std::optional<Duration> sGlobalSteadyClockOffset{std::nullopt}; - protected: bool mIsStreaming; @@ -2096,6 +2142,9 @@ class GenericLlmRequest // the authoritative mPrepopulatedPromptLen and advances context position. mutable SizeType32 mEstimatedReusableTokens{0}; + // Absolute context positions at which recurrent-state snapshots are expected. + std::vector<SizeType32> mExpectedSnapshotPoints; + SizeType32 mMaxSentTokenLen; std::optional<TensorPtr> mEmbeddingBias{std::nullopt}; @@ -2237,6 +2286,20 @@ class GenericLlmRequest std::optional<std::vector<std::tuple<std::string, int>>> mAgentHierarchy{std::nullopt}; private: + void adoptContextPhaseDraftTokens() + { + if (hasDraftTokens() || !mContextPhaseParams.has_value()) + { + return; + } + + auto const& draftTokens = mContextPhaseParams.value().getDraftTokens(); + if (draftTokens.has_value() && !draftTokens->empty()) + { + mDraftTokens = std::make_shared<VecTokens>(*draftTokens); + } + } + void initialize( VecTokens const& inputTokens, bool outputLogProbs, std::optional<TimePoint> arrivalTime = std::nullopt) { @@ -2356,7 +2419,7 @@ class GenericLlmRequest auto const numWords = static_cast<SizeType32>(words.size()); auto const shape = runtime::ITensor::makeShape({2, numWords}); - auto tensor = runtime::BufferManager::pinnedPool(shape, nvinfer1::DataType::kINT32); + auto tensor = runtime::BufferManager::pinnedPool(shape, tensorrt_llm::DataType::kINT32); auto* data = runtime::bufferCast<int32_t>(*tensor); std::memcpy(data, words.data(), numWords * sizeof(int32_t)); std::memcpy(data + numWords, offsets.data(), numWords * sizeof(int32_t)); @@ -2369,9 +2432,10 @@ class GenericLlmRequest static TimePoint maybeToGlobalSteadyClock(TimePoint const& time_point) { - if (sGlobalSteadyClockOffset.has_value()) + auto const& offset = globalSteadyClockOffset(); + if (offset.has_value()) { - return time_point + *sGlobalSteadyClockOffset; + return time_point + *offset; } return time_point; } diff --git a/cpp/include/tensorrt_llm/batch_manager/logitsPostProcessor.h b/cpp/include/tensorrt_llm/batch_manager/logitsPostProcessor.h deleted file mode 100644 index 1916a915e337..000000000000 --- a/cpp/include/tensorrt_llm/batch_manager/logitsPostProcessor.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "common.h" -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/common/algorithm.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -namespace tensorrt_llm::runtime -{ -class CudaStream; -} - -namespace tensorrt_llm::batch_manager -{ -class DecoderInputBuffers; - -class LogitsPostProcessor : Algorithm -{ -public: - using CudaStreamPtr = std::shared_ptr<runtime::CudaStream>; - - using LogitsPostProcessorBatched = std::function<void(std::vector<batch_manager::LlmRequest::RequestIdType> const&, - std::vector<batch_manager::LlmRequest::TensorPtr>&, - std::vector<std::reference_wrapper<batch_manager::LlmRequest::BeamTokens const>> const&, CudaStreamPtr const&, - std::vector<std::optional<batch_manager::LlmRequest::RequestIdType>> const&)>; - - constexpr static auto name{"LogitsPostProcessor"}; - - LogitsPostProcessor() = default; - - bool operator()(DecoderInputBuffers& inputBuffers, bool replicateLogitsPostProcessor, - runtime::WorldConfig const& worldConfig, CudaStreamPtr const& stream, - std::optional<LogitsPostProcessorBatched> const& logitsPostProcessorBatched = std::nullopt) const; -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h b/cpp/include/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h deleted file mode 100644 index 245f4b4b5286..000000000000 --- a/cpp/include/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "common.h" -#include "tensorrt_llm/common/algorithm.h" -#include "tensorrt_llm/common/optionalRef.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iGptDecoderBatched.h" -#include "tensorrt_llm/runtime/modelConfig.h" - -namespace tensorrt_llm::runtime::decoder -{ -class DecoderState; -} // namespace tensorrt_llm::runtime::decoder - -namespace tensorrt_llm::batch_manager -{ -class DecoderInputBuffers; -class RuntimeBuffers; - -class MakeDecodingBatchInputOutput : Algorithm -{ -public: - constexpr static auto name{"MakeDecodingBatchInputOutput"}; - - using SizeType32 = tensorrt_llm::runtime::SizeType32; - using TensorPtr = runtime::ITensor::SharedPtr; - template <typename T> - using OptionalRef = tensorrt_llm::common::OptionalRef<T>; - - MakeDecodingBatchInputOutput() = default; - - void operator()(DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState, - runtime::ModelConfig const& modelConfig, OptionalRef<RuntimeBuffers> fusedRuntimeBuffers) const; - - static void createDecoderBatchInputs(DecoderInputBuffers& inputBuffers, std::vector<SizeType32> const& activeSlots, - runtime::decoder::DecoderState const& decoderState); -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/medusaBuffers.h b/cpp/include/tensorrt_llm/batch_manager/medusaBuffers.h index ba29be6ede81..5342591840a8 100644 --- a/cpp/include/tensorrt_llm/batch_manager/medusaBuffers.h +++ b/cpp/include/tensorrt_llm/batch_manager/medusaBuffers.h @@ -22,7 +22,6 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/promptTuningParams.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" #include "tensorrt_llm/runtime/worldConfig.h" namespace tensorrt_llm::batch_manager @@ -36,10 +35,6 @@ class MedusaBuffers using TensorPtr = runtime::ITensor::SharedPtr; using TensorMap = runtime::StringPtrMap<runtime::ITensor>; - MedusaBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, runtime::BufferManager const& manager, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - executor::DecodingConfig const& decodingConfig, runtime::TllmRuntime const& runtime); - void reshape(SizeType32 numCtxSequences, SizeType32 numGenSequences, SizeType32 tokensPerStep); void insertInputTensors( diff --git a/cpp/include/tensorrt_llm/batch_manager/peftCacheManager.h b/cpp/include/tensorrt_llm/batch_manager/peftCacheManager.h index cf65753783e8..ed928e96d811 100644 --- a/cpp/include/tensorrt_llm/batch_manager/peftCacheManager.h +++ b/cpp/include/tensorrt_llm/batch_manager/peftCacheManager.h @@ -25,7 +25,7 @@ #include "tensorrt_llm/runtime/workerPool.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <future> #include <memory> @@ -147,7 +147,7 @@ class PeftCacheManager : public BasePeftCacheManager void updateTaskState(uint64_t taskId, uint64_t reqId, bool terminate = false, bool pause = false); static std::pair<uint64_t, uint64_t> getMaxNumSlots(PeftCacheManagerConfig const& config, - nvinfer1::DataType dataType, uint64_t pageWidth, uint64_t max1dModSize, + tensorrt_llm::DataType dataType, uint64_t pageWidth, uint64_t max1dModSize, runtime::BufferManager const& bufferManager); static std::pair<runtime::LoraCachePageManagerConfig, runtime::LoraCachePageManagerConfig> getPageManagerConfig( diff --git a/cpp/include/tensorrt_llm/batch_manager/promptTuningBuffers.h b/cpp/include/tensorrt_llm/batch_manager/promptTuningBuffers.h deleted file mode 100644 index a1d8849a8811..000000000000 --- a/cpp/include/tensorrt_llm/batch_manager/promptTuningBuffers.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/promptTuningParams.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -namespace tensorrt_llm::batch_manager -{ - -class PromptTuningBuffers -{ - -public: - using SizeType32 = tensorrt_llm::runtime::SizeType32; - using ITensor = tensorrt_llm::runtime::ITensor; - using TensorPtr = runtime::ITensor::SharedPtr; - - runtime::PromptTuningParams mPromptTuningParams; - SizeType32 mMaxPromptVocabSize; - - PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); - - PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, bool promptTableOffloading); - - void validate(std::optional<TensorPtr> const& optReqPromptEmbeddingTable, - std::optional<SizeType32> const& optReqPromptVocabSize); - - void fill(RequestVector const& contextRequests, RequestVector const& genRequests, - runtime::BufferManager const& manager, bool packed); - - /* - * The below functions are specific for Chunked Prefill mode - * Chunk Ptable with Ping-Pong Buffer Implementation - * ----------------------------------------------- - * - * Overview: - * The chunk ptable (prompt tuning table) system uses a ping-pong buffer mechanism to efficiently - * manage large embedding tables when operating in context Prefill mode. This allows - * for processing of large embedding tables by loading them in chunks from CPU to GPU memory, - * enabling support for tables that exceed available GPU memory. - * - * Key Components: - * 1. Ping-Pong Buffers (mChunkPtableBuffers): - * - Two alternating GPU buffers that store chunks of the embedding table - * - While the current buffer is being processed by the model, - * the next chunk can be asynchronously loaded into the other buffer - * - Managed through mChunkPtableCurrentIndex (toggles between 0 and 1) - * 2. Start Positions Tracking (mChunkPtableBufferStartPositions): - * - Mainly used for multi-batch processing - * - Maintains the starting position of each batch's data within each buffer - * - Maintained separately for each ping-pong buffer - * - * Memory Optimization: - * - Only two GPU buffers are maintained regardless of total embedding table size - * - Each buffer size is limited to contextChunkSize * hiddenSize - * - Efficient memory usage through chunk-based processing - */ - - bool mPromptTableOffloading; - - bool mChunkPtableInitialized{false}; - std::optional<std::array<TensorPtr, 2>> mChunkPtableBuffers; - std::optional<std::vector<std::vector<SizeType32>>> mChunkPtableBufferStartPositions; - size_t mChunkPtableCurrentIndex{0}; - - void initializeChunkPtableBuffers(runtime::BufferManager const& manager, runtime::ModelConfig const& modelConfig, - SizeType32 contextChunkSize, std::shared_ptr<LlmRequest> const& llmReq); - - void switchChunkPtableBuffer(); - - size_t getChunkPtableCurrentIndex(); - - [[nodiscard]] TensorPtr& getChunkPtableBuffer(size_t index); - - [[nodiscard]] SizeType32 getChunkPtableBufferSliceSize(size_t index, size_t batchIdx); - - [[nodiscard]] SizeType32 getChunkPtableBufferStartPosition(size_t index, size_t batchIdx); - - void updateBufferStartPosition(size_t index, SizeType32 numRows); - - void clearBufferStartPositions(size_t index); -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/rnnStateManager.h b/cpp/include/tensorrt_llm/batch_manager/rnnStateManager.h index 5c0bfe136de2..c4f97950a6b9 100644 --- a/cpp/include/tensorrt_llm/batch_manager/rnnStateManager.h +++ b/cpp/include/tensorrt_llm/batch_manager/rnnStateManager.h @@ -17,6 +17,7 @@ #pragma once #include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -42,8 +43,8 @@ class RnnStateManager runtime::WorldConfig const& worldConfig, tensorrt_llm::runtime::BufferManager const& bufferManager); RnnStateManager(SizeType32 dState, SizeType32 dConv, SizeType32 numHeads, SizeType32 nGroups, SizeType32 headDim, - SizeType32 maxBatchSize, runtime::WorldConfig const& worldConfig, int64_t stream, nvinfer1::DataType dtype, - nvinfer1::DataType ssmCacheDtype, std::vector<SizeType32> const& ppLayers, SizeType32 numLayers); + SizeType32 maxBatchSize, runtime::WorldConfig const& worldConfig, int64_t stream, tensorrt_llm::DataType dtype, + tensorrt_llm::DataType ssmCacheDtype, std::vector<SizeType32> const& ppLayers, SizeType32 numLayers); void getPtrBuffers(TensorMap& inputBuffers, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) const; @@ -68,9 +69,9 @@ class RnnStateManager [[nodiscard]] TensorPtr getSsmStates() const; - [[nodiscard]] nvinfer1::DataType getConvStateDataType() const noexcept; + [[nodiscard]] tensorrt_llm::DataType getConvStateDataType() const noexcept; - [[nodiscard]] nvinfer1::DataType getSsmStateDataType() const noexcept; + [[nodiscard]] tensorrt_llm::DataType getSsmStateDataType() const noexcept; [[nodiscard]] executor::kv_cache::CacheState::RnnModelConfig getRnnCacheStateModelConfig() const noexcept; @@ -111,8 +112,8 @@ class RnnStateManager std::vector<SizeType32> mFreeBlocks; std::unordered_map<RequestIdType, SizeType32> mCacheIndex; std::optional<runtime::BufferManager> mBufferManager; - nvinfer1::DataType mDtype{nvinfer1::DataType::kFLOAT}; - nvinfer1::DataType mSsmCacheDtype{nvinfer1::DataType::kFLOAT}; + tensorrt_llm::DataType mDtype{tensorrt_llm::DataType::kFLOAT}; + tensorrt_llm::DataType mSsmCacheDtype{tensorrt_llm::DataType::kFLOAT}; // RNN model config (global values before TP/PP split) SizeType32 mDState{0}; diff --git a/cpp/include/tensorrt_llm/batch_manager/runtimeBuffers.h b/cpp/include/tensorrt_llm/batch_manager/runtimeBuffers.h deleted file mode 100644 index 97a4ae67acdd..000000000000 --- a/cpp/include/tensorrt_llm/batch_manager/runtimeBuffers.h +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Copyright (c) 2023-2026, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/batch_manager/rnnStateManager.h" -#include "tensorrt_llm/common/optionalRef.h" -#include "tensorrt_llm/runtime/eagleBuffers.h" -#include "tensorrt_llm/runtime/explicitDraftTokensBuffers.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/lookaheadBuffers.h" -#include "tensorrt_llm/runtime/loraManager.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -#include <cstddef> -#include <memory> -#include <optional> -#include <vector> - -namespace tensorrt_llm::runtime -{ -class TllmRuntime; - -namespace decoder -{ -class DecoderState; -} // namespace decoder -} // namespace tensorrt_llm::runtime - -namespace tensorrt_llm::batch_manager -{ - -namespace kv_cache_manager -{ -class BaseKVCacheManager; -} // namespace kv_cache_manager - -class LlmRequest; - -class EncoderBuffers; -class LoraBuffers; -class MedusaBuffers; -class PromptTuningBuffers; -class RnnStateBuffers; -class TransformerBuffers; - -class RuntimeBuffers -{ -public: - static constexpr auto kLogitsTensorName = "logits"; - static constexpr auto kHiddenStatesOutputTensorName = "hidden_states_output"; - static constexpr auto kHiddenStatesInputTensorName = "hidden_states_input"; - static constexpr auto kInputIdsTensorName = "input_ids"; - static constexpr auto kLastTokenIdsTensorName = "last_token_ids"; - static constexpr auto kHostRequestTypesTensorName = "host_request_types"; - static constexpr auto kContextLengthsTensorName = "context_lengths"; - static constexpr auto kHostContextLengthsTensorName = "host_context_lengths"; - static constexpr auto kSequenceLengthsTensorName = "sequence_length"; - static constexpr auto kPromptEmbeddingTableTensorName = "prompt_embedding_table"; - static constexpr auto kTasksTensorName = "tasks"; - static constexpr auto kPromptVocabSizeTensorName = "prompt_vocab_size"; - static constexpr auto kMRopeRotaryCosSinTensorName = "mrope_rotary_cos_sin"; - static constexpr auto kMRopePositionDeltasTensorName = "mrope_position_deltas"; - - using SizeType32 = runtime::SizeType32; - using TensorPtr = runtime::ITensor::SharedPtr; - using TensorMap = runtime::ITensor::TensorMap; - using PeftTable = runtime::LoraManager::PeftTable; - template <typename T> - using OptionalRef = tensorrt_llm::common::OptionalRef<T>; - - [[nodiscard]] SizeType32 constexpr getContextIndex() const noexcept - { - return contextIndex; - }; - - void constexpr setContextIndex(SizeType32 index) noexcept - { - contextIndex = index; - }; - - [[nodiscard]] SizeType32 constexpr getNumContextTokens() const noexcept - { - return numContextTokens; - }; - - [[nodiscard]] BatchState getBatchState() const noexcept - { - return {numContextRequests, numGenRequests, getNumTokens(), maxKvCacheLengthRounded}; - }; - -private: - [[nodiscard]] SizeType32 constexpr getNumRequests() const noexcept - { - return numContextRequests + numGenRequests; - }; - - [[nodiscard]] SizeType32 constexpr getNumSequences() const noexcept - { - return numContextRequests + numGenSequences; - }; - - [[nodiscard]] SizeType32 constexpr getNumTokens() const noexcept - { - return numContextTokens + numGenTokens; - }; - - //! Sizes - SizeType32 numContextRequests{}; - SizeType32 numGenRequests{}; - SizeType32 numGenSequences{}; - SizeType32 numContextTokens{}; - SizeType32 numGenTokens{}; - SizeType32 numLogits{}; - SizeType32 maxKvCacheLengthRounded{}; - - //! General - TensorPtr inputsIds; - - TensorPtr contextLengthsHost; - TensorPtr contextLengthsDevice; - TensorPtr sequenceLengthsHost; - - //! Index of selected runtime context. - SizeType32 contextIndex{}; - SizeType32 maxContextLength{}; - -public: - TensorPtr sequenceLengthsDevice; - bool promptTableOffloading; - - //! Prompt-Tuning - std::unique_ptr<PromptTuningBuffers> promptTuningBuffers; - -private: - //! Runtime - //! Type of host tensor: 0 for context, 1 for generation - TensorPtr requestTypes; - - TensorPtr lastTokenIdsHost; - TensorPtr lastTokenIdsDevice; - TensorPtr logitsIdsHost; - - //! Pipeline-Parallelism - TensorPtr hiddenStates; - - //! Mrope - TensorPtr mropeRotaryCosSin; - TensorPtr mropePositionDeltas; - - //! LoRA - std::unique_ptr<LoraBuffers> loraBuffers; - -public: - //! Additional buffers depending on model type - std::unique_ptr<TransformerBuffers> transformerBuffers; - std::unique_ptr<RnnStateBuffers> rnnStateBuffers; - - //! Encoder-Decoder - std::unique_ptr<EncoderBuffers> encoderBuffers; - - //! Medusa - std::unique_ptr<MedusaBuffers> mMedusaBuffers; - //! Lookahead decoding - std::unique_ptr<runtime::LookaheadRuntimeBuffers> mLookaheadBuffers; - //! Explicit draft tokens decoding - std::unique_ptr<runtime::ExplicitDraftTokensBuffers> mExplicitDraftTokensBuffers; - //! Eagle decoding - std::unique_ptr<runtime::EagleBuffers> mEagleBuffers; - - //! Language adapter routing information if language adapter is presented, [numTokens, numLanguages] - TensorPtr languageAdapterRoutings; - - TensorPtr cacheIndirDecoderIOBatchedCopySrcOffsets; - TensorPtr cacheIndirDecoderIOBatchedCopyDstOffsets; - TensorPtr cacheIndirDecoderIOBatchedCopySizes; - - //! Logits - std::vector<SizeType32> numContextLogits; - TensorPtr logits; - - //! Helper cache for store generation logits - struct GenerationLogitsCache - { - static constexpr auto kCACHE_LENGTH = 8; - - //! Buffer for logits between steps to prevent from being overwritten - //! [kCACHE_LENGTH, maxBatchSize * maxBeamWidth, vocabSizePadded] - TensorPtr logits; - //! Record the usage offset of the cacheGenerationLogits buffer - SizeType32 offset{0}; - - //! Temporarily store the transposed results of multiple fragment logits, [maxBeamWidth, kCACHE_LENGTH] - TensorPtr transposedLogits; - - //! Temporarily store logits buffer address during the transposing, [maxBatchSize, kCACHE_LENGTH] - //! One row per batch slot (same layout as fragmentPointerHost) so concurrent flushes for - //! different requests in the same batch never clobber each other's pointer arrays. - TensorPtr fragmentPointerDevice; - - //! Temporarily store logits buffer address during the transposing, [maxBatchSize, kCACHE_LENGTH] - TensorPtr fragmentPointerHost; - - //! Cycling index for workspace - size_t workIdx{0}; - - void cycleWorkIdx() - { - workIdx = (workIdx + 1) % (fragmentPointerHost->getShape().d[0]); - } - - //! Returns matching host and device pointer rows for the current workIdx, then advances - //! workIdx. Always call this instead of the individual getters to avoid ordering bugs. - [[nodiscard]] std::pair<TensorPtr, TensorPtr> getFragmentPointerSlot() - { - TensorPtr host = runtime::ITensor::slice(fragmentPointerHost, workIdx, 1); - TensorPtr device = runtime::ITensor::slice(fragmentPointerDevice, workIdx, 1); - cycleWorkIdx(); - return {std::move(host), std::move(device)}; - }; - }; - - GenerationLogitsCache generationLogitsCache; - - //! Mapping from batch idx to slot id - TensorPtr seqSlots; - TensorPtr seqSlotsDevice; - - //! Explicitly device-copy src offsets to reduce warp stalls in copy batch kernel invocation - //! [mMaxNumRequests], on gpu - TensorPtr mCacheIndirDecoderIOBatchedCopySrcOffsetsSliceDevice; - //! Explicitly device-copy dst offsets to reduce warp stalls in copy batch kernel invocation - //! [mMaxNumRequests], on gpu - TensorPtr mCacheIndirDecoderIOBatchedCopyDstOffsetsSliceDevice; - //! Explicitly device-copy size to reduce warp stalls in copy batch kernel invocation - //! [mMaxNumRequests], on gpu - TensorPtr mCacheIndirDecoderIOBatchedCopyCopySizesDevice; - -private: - //! Re-capture cuda graph when max kv cache len of the batch has changed on kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE. - static SizeType32 constexpr kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE{256}; - - TensorMap mAdditionalOutputTensors; // Tensors storing additional output tensors. - - //! Engine I/O - TensorMap inputMap; - TensorMap outputMap; - -public: - RuntimeBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, - runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig, executor::DecodingConfig const& decodingConfig, - bool gatherGenerationLogits, std::optional<SizeType32> maxNumTokens = std::nullopt, - std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs = std::nullopt, - bool promptTableOffloading = false); - - RuntimeBuffers(RuntimeBuffers const& other) = delete; - RuntimeBuffers& operator=(RuntimeBuffers const& other) = delete; - RuntimeBuffers(RuntimeBuffers&& other) = delete; - RuntimeBuffers& operator=(RuntimeBuffers&& other) = delete; - - ~RuntimeBuffers(); - - std::tuple<SizeType32, TensorMap const&, TensorMap&> prepareStep(RequestVector const& contextRequests, - RequestVector const& genRequests, SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow, - runtime::decoder::DecoderState const& decoderState, kv_cache_manager::BaseKVCacheManager* kvCacheManager, - kv_cache_manager::BaseKVCacheManager* crossKvCacheManager, rnn_state_manager::RnnStateManager* rnnStateManager, - PeftTable const& peftTable, runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig, bool gatherGenerationLogits, bool trtOverlap, - OptionalRef<runtime::ITensor const> newOutputTokens = std::nullopt); - - void prepareBuffersForCudaGraph(SizeType32 maxSequenceLength); - - void prepareExplicitDraftTokenBuffers(runtime::ExplicitDraftTokensBuffers::Inputs const& explicitDraftTokensBuffers, - runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig); - - void prepareEagleBuffers(RequestVector const& contextRequests, RequestVector const& genRequests, - runtime::EagleBuffers::Inputs const& eagleBuffers, runtime::TllmRuntime const& runtime, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); - -private: - void create(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, - SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, runtime::TllmRuntime const& runtime, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - executor::DecodingConfig const& decodingConfig, bool gatherGenerationLogits, - std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs = std::nullopt); - - //! @brief set max sizes for pre-allocation - void setMaxBufferSizes(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, runtime::ModelConfig const& modelConfig, - std::optional<SizeType32> maxNumRuntimeTokens); - - //! @brief set sizes depending on scheduled requests - void setBufferSizes(RequestVector const& contextRequests, RequestVector const& genRequests); - - void reshape(runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig, bool gatherGenerationLogits); - - void setFromInputs(RequestVector const& contextRequests, RequestVector const& genRequests, SizeType32 maxBeamWidth, - SizeType32 maxAttentionWindow, runtime::decoder::DecoderState const& decoderState, - kv_cache_manager::BaseKVCacheManager* kvCacheManagerPtr, - kv_cache_manager::BaseKVCacheManager* crossKvCacheManagerPtr, - rnn_state_manager::RnnStateManager* rnnStateManagerPtr, PeftTable const& peftTable, - runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig, bool trtOverlap, OptionalRef<runtime::ITensor const> newOutputTokens); - - void fillIOMaps(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/transformerBuffers.h b/cpp/include/tensorrt_llm/batch_manager/transformerBuffers.h deleted file mode 100644 index b5254c6357b4..000000000000 --- a/cpp/include/tensorrt_llm/batch_manager/transformerBuffers.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/batch_manager/kvCacheType.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -namespace tensorrt_llm::runtime -{ -class TllmRuntime; -class MulticastTensor; -} // namespace tensorrt_llm::runtime - -namespace tensorrt_llm::batch_manager -{ - -namespace kv_cache_manager -{ -class BaseKVCacheManager; -} - -class TransformerBuffers -{ -public: - using SizeType32 = runtime::SizeType32; - using TensorPtr = runtime::ITensor::SharedPtr; - using TensorMap = runtime::StringPtrMap<runtime::ITensor>; - - static constexpr auto kCrossAttentionMaskTensorName = "cross_attention_mask"; - static constexpr auto kCrossAttentionPackedMaskTensorName = "cross_attention_packed_mask"; - static constexpr auto kPositionIdsTensorName = "position_ids"; - static constexpr auto kCacheIndirectionsTensorName = "cache_indirection"; - static constexpr auto kHostPastKeyValueLengthsTensorName = "host_past_key_value_lengths"; - static constexpr auto kHostSinkTokenLengthTensorName = "host_sink_token_length"; - static constexpr auto kHostMaxAttentionWindowSizesTensorName = "host_max_attention_window_sizes"; - static constexpr auto kHostContextProgressTensorName = "host_context_progress"; - static constexpr auto kKvCacheBlockOffsetsTensorName = "kv_cache_block_offsets"; - static constexpr auto kHostKvCacheBlockOffsetsTensorName = "host_kv_cache_block_offsets"; - static constexpr auto kCrossKvCacheBlockOffsetsTensorName = "cross_kv_cache_block_offsets"; - static constexpr auto kHostCrossKvCacheBlockOffsetsTensorName = "host_cross_kv_cache_block_offsets"; - static constexpr auto kHostCrossKvCachePoolPointersTensorName = "host_cross_kv_cache_pool_pointers"; - static constexpr auto kHostCrossKvCachePoolMappingTensorName = "host_cross_kv_cache_pool_mapping"; - static constexpr auto kSkipCrossAttentionBlocksTensorName = "skip_cross_attn_blocks"; - - TensorPtr pastKeyValueLengths; // Host tensor - TensorPtr positionIds; - - // max kv cache lengths. - TensorPtr maxAttentionWindows; - // sink token lengths. - TensorPtr sinkTokenLengths; - TensorPtr cacheIndirection; - TensorPtr kvCacheBlockOffsetsHost; // [numPools, maxBatch * maxBeamWidth, 2, maxBlocksPerSeq] - TensorPtr kvCacheBlockOffsetsDevice; // [numPools, maxBatch * maxBeamWidth, 2, maxBlocksPerSeq] - TensorPtr contextProgressHost; - - // Cross attention buffers - TensorPtr crossKvCacheBlockPoolPointers = nullptr; - TensorPtr crossKvCacheBlockPoolMapping = nullptr; - TensorPtr crossKvCacheBlockOffsetsHost = nullptr; - TensorPtr crossKvCacheBlockOffsetsDevice = nullptr; - TensorPtr crossAttentionMaskCopySrcOffsets = nullptr; // [maxNumRequest] pinned memory. - TensorPtr crossAttentionMaskCopyDstOffsets = nullptr; // [maxNumRequest] pinned memory. - TensorPtr crossAttentionMaskCopySizes = nullptr; // [maxNumRequest] pinned memory. - TensorPtr crossAttentionMaskDevice = nullptr; // [maxNumTokens, maxEncoderOutputLen] - // This is created to allow mixed memory types of crossAttentionMask (i.e. CPU and GPU). - TensorPtr crossAttentionMaskPinnedHost = nullptr; // [maxNumTokens, maxEncoderOutputLen] - // See more details in tensorrt_llm/kernels/contextFusedMultiHeadAttention/fmhaPackedMask.cu. - // The attention packed mask for FMHA where each bit represents one mask. - TensorPtr crossAttentionPackedMaskDevice - = nullptr; // [maxBatchSize, maxInputLengthInBatch, roundUp(maxEncoderOutputLen, 32)] - // The number of cumulative Q sequence lengths in the mask input, which is used to get mask offsets for different - // requests. - TensorPtr crossAttentionCuQSeqLensDevice = nullptr; // [maxBatchSize + 1] - // The number of cumulative Q sequence lengths in the packed mask, which is used to get mask offsets for different - // requests. - TensorPtr crossAttentionPackedMaskCuMaskRowsDevice = nullptr; // [maxBatchSize + 1] - - TensorPtr cacheIndirBatchedCopySrcOffsets; - TensorPtr cacheIndirBatchedCopyDstOffsets; - TensorPtr cacheIndirBatchedCopySizes; - - TensorPtr fillValuesAlt; - TensorPtr fillValuesAltDevice; - TensorPtr seqSlotsAlt; - TensorPtr seqSlotsAltDevice; - TensorPtr skipCrossAttnBlocks; - - std::shared_ptr<tensorrt_llm::runtime::MulticastTensor> gemmAllReduceOutput; - - TransformerBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, - runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig); - - void reshape(SizeType32 numSequences, SizeType32 numInputTokens); - - void reshapeKvTensors(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, SizeType32 maxBlocksPerSeq, - kv_cache_manager::CacheType kvCacheType, SizeType32 numPools, runtime::BufferManager const& manager); - - void getBuffers(TensorMap& inputBuffers, TensorMap& outputBuffers, runtime::ModelConfig const& modelConfig) const; - - void copyPositionIds(runtime::TllmRuntime const& runtime, std::vector<SizeType32> const& positionIdsHost, - bool isChatGlm, TensorPtr const& decoderPositionIds); - - void copyKvBlockOffsets(RequestVector const& contextRequests, RequestVector const& genRequests, - kv_cache_manager::BaseKVCacheManager const* kvCacheManager, - kv_cache_manager::BaseKVCacheManager const* crossKvCacheManager, runtime::BufferManager const& manager); - - // Copy CacheIndirection from `decoderCacheIndirectionOutput` to `this->cacheIndirection` - void copyCacheIndirection(RequestVector const& genRequests, TensorPtr const& decoderCacheIndirectionOutput, - runtime::CudaStream const& stream); - - void copyCrossAttentionMasks(RequestVector const& contextRequests, RequestVector const& genRequests, - TensorPtr const& decoderContextLengthsDevice, TensorPtr const& encoderInputLengths, - SizeType32 maxDecoderContextLength, SizeType32 maxEncoderInputLengthInBatch, - runtime::TllmRuntime const& runtime); - - void copySkipCrossAttnBlocks(bool const& _skipCrossAttnBlocks, runtime::TllmRuntime const& runtime); - -private: - SizeType32 maxInputLen; - SizeType32 maxEncoderOutputLen; - SizeType32 maxNumTokens; -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/updateDecoderBuffers.h b/cpp/include/tensorrt_llm/batch_manager/updateDecoderBuffers.h deleted file mode 100644 index 526a756e5546..000000000000 --- a/cpp/include/tensorrt_llm/batch_manager/updateDecoderBuffers.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/common/algorithm.h" -#include "tensorrt_llm/runtime/modelConfig.h" - -namespace tensorrt_llm::runtime -{ -class BufferManager; -class CudaEvent; - -namespace decoder -{ -class DecoderState; -} // namespace decoder -} // namespace tensorrt_llm::runtime - -namespace tensorrt_llm::batch_manager -{ - -class DecoderOutputBuffers; - -class UpdateDecoderBuffers : Algorithm -{ -public: - constexpr static auto name{"UpdateDecoderBuffers"}; - - UpdateDecoderBuffers() = default; - - runtime::CudaEvent operator()(runtime::ModelConfig const& modelConfig, DecoderOutputBuffers& decoderOutputBuffers, - runtime::BufferManager const& copyBufferManager, runtime::decoder::DecoderState const& decoderState, - bool returnLogProbs, runtime::CudaEvent const& decoderFinishEvent) const; -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/common/dataType.h b/cpp/include/tensorrt_llm/common/dataType.h index 2f19404f9c94..9b3bb5fdf0f0 100644 --- a/cpp/include/tensorrt_llm/common/dataType.h +++ b/cpp/include/tensorrt_llm/common/dataType.h @@ -19,7 +19,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/tllmException.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <map> TRTLLM_NAMESPACE_BEGIN @@ -27,61 +27,61 @@ TRTLLM_NAMESPACE_BEGIN namespace common { -constexpr static size_t getDTypeSize(nvinfer1::DataType type) +constexpr static size_t getDTypeSize(tensorrt_llm::DataType type) { switch (type) { - case nvinfer1::DataType::kINT64: return 8; - case nvinfer1::DataType::kINT32: [[fallthrough]]; - case nvinfer1::DataType::kFLOAT: return 4; - case nvinfer1::DataType::kBF16: [[fallthrough]]; - case nvinfer1::DataType::kHALF: return 2; - case nvinfer1::DataType::kBOOL: [[fallthrough]]; - case nvinfer1::DataType::kUINT8: [[fallthrough]]; - case nvinfer1::DataType::kINT8: [[fallthrough]]; - case nvinfer1::DataType::kFP8: return 1; - case nvinfer1::DataType::kINT4: TLLM_THROW("Cannot determine size of INT4 data type"); - case nvinfer1::DataType::kFP4: TLLM_THROW("Cannot determine size of FP4 data type"); + case tensorrt_llm::DataType::kINT64: return 8; + case tensorrt_llm::DataType::kINT32: [[fallthrough]]; + case tensorrt_llm::DataType::kFLOAT: return 4; + case tensorrt_llm::DataType::kBF16: [[fallthrough]]; + case tensorrt_llm::DataType::kHALF: return 2; + case tensorrt_llm::DataType::kBOOL: [[fallthrough]]; + case tensorrt_llm::DataType::kUINT8: [[fallthrough]]; + case tensorrt_llm::DataType::kINT8: [[fallthrough]]; + case tensorrt_llm::DataType::kFP8: return 1; + case tensorrt_llm::DataType::kINT4: TLLM_THROW("Cannot determine size of INT4 data type"); + case tensorrt_llm::DataType::kFP4: TLLM_THROW("Cannot determine size of FP4 data type"); default: TLLM_THROW("Unknown dtype %d", static_cast<int>(type)); } return 0; } -constexpr static size_t getDTypeSizeInBits(nvinfer1::DataType type) +constexpr static size_t getDTypeSizeInBits(tensorrt_llm::DataType type) { switch (type) { - case nvinfer1::DataType::kINT64: return 64; - case nvinfer1::DataType::kINT32: [[fallthrough]]; - case nvinfer1::DataType::kFLOAT: return 32; - case nvinfer1::DataType::kBF16: [[fallthrough]]; - case nvinfer1::DataType::kHALF: return 16; - case nvinfer1::DataType::kBOOL: [[fallthrough]]; - case nvinfer1::DataType::kUINT8: [[fallthrough]]; - case nvinfer1::DataType::kINT8: [[fallthrough]]; - case nvinfer1::DataType::kFP8: return 8; - case nvinfer1::DataType::kINT4: [[fallthrough]]; - case nvinfer1::DataType::kFP4: return 4; + case tensorrt_llm::DataType::kINT64: return 64; + case tensorrt_llm::DataType::kINT32: [[fallthrough]]; + case tensorrt_llm::DataType::kFLOAT: return 32; + case tensorrt_llm::DataType::kBF16: [[fallthrough]]; + case tensorrt_llm::DataType::kHALF: return 16; + case tensorrt_llm::DataType::kBOOL: [[fallthrough]]; + case tensorrt_llm::DataType::kUINT8: [[fallthrough]]; + case tensorrt_llm::DataType::kINT8: [[fallthrough]]; + case tensorrt_llm::DataType::kFP8: return 8; + case tensorrt_llm::DataType::kINT4: [[fallthrough]]; + case tensorrt_llm::DataType::kFP4: return 4; default: TLLM_THROW("Unknown dtype %d", static_cast<int>(type)); } return 0; } -[[maybe_unused]] static std::string getDtypeString(nvinfer1::DataType type) +[[maybe_unused]] static std::string getDtypeString(tensorrt_llm::DataType type) { switch (type) { - case nvinfer1::DataType::kFLOAT: return "fp32"; break; - case nvinfer1::DataType::kHALF: return "fp16"; break; - case nvinfer1::DataType::kINT8: return "int8"; break; - case nvinfer1::DataType::kINT32: return "int32"; break; - case nvinfer1::DataType::kBOOL: return "bool"; break; - case nvinfer1::DataType::kUINT8: return "uint8"; break; - case nvinfer1::DataType::kFP8: return "fp8"; break; - case nvinfer1::DataType::kBF16: return "bf16"; break; - case nvinfer1::DataType::kINT64: return "int64"; break; - case nvinfer1::DataType::kINT4: return "int4"; break; - case nvinfer1::DataType::kFP4: return "fp4"; break; + case tensorrt_llm::DataType::kFLOAT: return "fp32"; break; + case tensorrt_llm::DataType::kHALF: return "fp16"; break; + case tensorrt_llm::DataType::kINT8: return "int8"; break; + case tensorrt_llm::DataType::kINT32: return "int32"; break; + case tensorrt_llm::DataType::kBOOL: return "bool"; break; + case tensorrt_llm::DataType::kUINT8: return "uint8"; break; + case tensorrt_llm::DataType::kFP8: return "fp8"; break; + case tensorrt_llm::DataType::kBF16: return "bf16"; break; + case tensorrt_llm::DataType::kINT64: return "int64"; break; + case tensorrt_llm::DataType::kINT4: return "int4"; break; + case tensorrt_llm::DataType::kFP4: return "fp4"; break; default: throw std::runtime_error("Unsupported data type"); break; } diff --git a/cpp/include/tensorrt_llm/common/logger.h b/cpp/include/tensorrt_llm/common/logger.h index d14b4c02e992..9073d21f0088 100644 --- a/cpp/include/tensorrt_llm/common/logger.h +++ b/cpp/include/tensorrt_llm/common/logger.h @@ -50,8 +50,6 @@ constexpr std::string_view formatModule(std::string_view module) return "deepgemm"; else if (module == "executor") return "executor"; - else if (module == "executor_worker") - return "exec_wkr"; else if (module == "flash_mla") return "flashmla"; else if (module == "kernels") @@ -60,8 +58,6 @@ constexpr std::string_view formatModule(std::string_view module) return "layers"; else if (module == "nanobind") return "nanobind"; - else if (module == "plugins") - return "plugins"; else if (module == "runtime") return "runtime"; else if (module == "testing") diff --git a/cpp/include/tensorrt_llm/common/tllmDataType.h b/cpp/include/tensorrt_llm/common/tllmDataType.h new file mode 100644 index 000000000000..9e5567ce280e --- /dev/null +++ b/cpp/include/tensorrt_llm/common/tllmDataType.h @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/common/config.h" + +#include <cstdint> + +//! \file tllmDataType.h +//! +//! Standalone, TensorRT-free runtime types that replace the \c nvinfer1 +//! types the shared C++ core historically used as common currency: +//! \c tensorrt_llm::common::DataType and \c tensorrt_llm::common::Dims. +//! These are defined here so the retained tree (runtime, batch manager, executor, +//! kernels and the nanobind bridge) compiles and links without the TensorRT +//! library. They are hoisted into the \c tensorrt_llm namespace with +//! using-declarations because they are common currency across the whole tree +//! (\c tensorrt_llm::DataType / \c tensorrt_llm::Dims). +//! +//! The \c DataType enumerator values intentionally mirror the legacy +//! \c nvinfer1::DataType integer values so that previously-serialized executor +//! configs and KV-cache metadata remain byte-compatible. The \c Dims layout +//! mirrors the legacy \c nvinfer1::Dims (\c int32_t \c nbDims followed by +//! \c int64_t \c d[8]) for the same reason. + +TRTLLM_NAMESPACE_BEGIN + +namespace common +{ + +//! \brief Standalone data-type enum. Values mirror the legacy +//! \c nvinfer1::DataType for serialization/format compatibility. +enum class DataType : int32_t +{ + kFLOAT = 0, + kHALF = 1, + kINT8 = 2, + kINT32 = 3, + kBOOL = 4, + kUINT8 = 5, + kFP8 = 6, + kBF16 = 7, + kINT64 = 8, + kINT4 = 9, + kFP4 = 10, + kE8M0 = 11, +}; + +//! \brief Standalone dimensions type. Layout mirrors the legacy +//! \c nvinfer1::Dims (rank plus up to \c MAX_DIMS 64-bit extents) so serialized +//! shapes remain compatible. +class Dims +{ +public: + //! The maximum rank (number of dimensions) supported for a tensor. + static constexpr int32_t MAX_DIMS{8}; + + //! The rank (number of dimensions). + int32_t nbDims; + + //! The extent of each dimension. + int64_t d[MAX_DIMS]; +}; + +} // namespace common + +using common::DataType; +using common::Dims; + +TRTLLM_NAMESPACE_END diff --git a/cpp/include/tensorrt_llm/executor/dataTransceiverState.h b/cpp/include/tensorrt_llm/executor/dataTransceiverState.h index 5067ae61dc83..578e53b81dbf 100644 --- a/cpp/include/tensorrt_llm/executor/dataTransceiverState.h +++ b/cpp/include/tensorrt_llm/executor/dataTransceiverState.h @@ -18,6 +18,7 @@ #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/worldConfig.h" @@ -50,7 +51,7 @@ class CacheState final }; CacheState(ModelConfig modelConfig, runtime::WorldConfig const& worldConfig, - std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, AttentionType attentionType = AttentionType::kDEFAULT, int kvFactor = 2, bool enableBlockReuse = false, bool enablePartialReuse = false, bool hasIndexerKCache = false, SizeType32 indexerDimPerHead = 0, SizeType32 indexerKCacheQuantBlockSize = 128, bool indexerKCacheUseFp4 = false) @@ -71,7 +72,7 @@ class CacheState final CacheState(std::vector<SizeType32> nbKvHeadPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, SizeType32 tensorParallelism, SizeType32 pipelineParallelism, SizeType32 contextParallelism, - std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, AttentionType attentionType = AttentionType::kDEFAULT, int kvFactor = 2, bool enableAttentionDP = false, int DPrank = 0, int DPsize = 0, bool enableBlockReuse = false, bool enablePartialReuse = false, bool hasIndexerKCache = false, SizeType32 indexerDimPerHead = 0, SizeType32 indexerKCacheQuantBlockSize = 128, @@ -92,7 +93,7 @@ class CacheState final CacheState(SizeType32 nbAttentionLayers, SizeType32 nbKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, SizeType32 tensorParallelism, SizeType32 pipelineParallelism, SizeType32 contextParallelism, - std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, AttentionType attentionType = AttentionType::kDEFAULT, int kvFactor = 2, bool enableAttentionDP = false, int DPrank = 0, int DPsize = 0, bool enableBlockReuse = false, bool enablePartialReuse = false, bool hasIndexerKCache = false, SizeType32 indexerDimPerHead = 0, SizeType32 indexerKCacheQuantBlockSize = 128, @@ -238,8 +239,8 @@ class CacheState final RnnModelConfig mModelConfig; /// Number of RNN layers per pipeline parallelism rank. std::vector<SizeType32> mLayerNumPerPP; - nvinfer1::DataType mConvStateDataType; - nvinfer1::DataType mSsmStateDataType; + tensorrt_llm::DataType mConvStateDataType; + tensorrt_llm::DataType mSsmStateDataType; [[nodiscard]] bool operator==(RnnCacheState const& other) const noexcept { @@ -263,7 +264,7 @@ class CacheState final return mAttentionConfig; } - [[nodiscard]] nvinfer1::DataType const& getDataType() const + [[nodiscard]] tensorrt_llm::DataType const& getDataType() const { return mDataType; } @@ -308,7 +309,7 @@ class CacheState final } void setRnnConfig(RnnModelConfig rnnModelConfig, std::vector<SizeType32> rnnLayerNumPerPP, - nvinfer1::DataType convStateDataType, nvinfer1::DataType ssmStateDataType) + tensorrt_llm::DataType convStateDataType, tensorrt_llm::DataType ssmStateDataType) { mRnnCacheState = RnnCacheState{ std::move(rnnModelConfig), std::move(rnnLayerNumPerPP), convStateDataType, ssmStateDataType}; @@ -325,12 +326,12 @@ class CacheState final return getRnnCacheState().mModelConfig; } - [[nodiscard]] nvinfer1::DataType getConvStateDataType() const + [[nodiscard]] tensorrt_llm::DataType getConvStateDataType() const { return getRnnCacheState().mConvStateDataType; } - [[nodiscard]] nvinfer1::DataType getSsmStateDataType() const + [[nodiscard]] tensorrt_llm::DataType getSsmStateDataType() const { return getRnnCacheState().mSsmStateDataType; } @@ -395,7 +396,7 @@ class CacheState final friend class tensorrt_llm::executor::Serialization; ModelConfig mModelConfig; ParallelConfig mParallelConfig; - nvinfer1::DataType mDataType; + tensorrt_llm::DataType mDataType; AttentionConfig mAttentionConfig; bool mEnableBlockReuse{false}; bool mEnablePartialReuse{false}; @@ -618,9 +619,22 @@ class DataTransceiverState final return mCacheState.has_value() && mCacheState->hasRnnConfig(); } + /// @brief Set only when exported via CacheTransceiver::getSerializedDataTransceiverState: + /// transfers driven by such a state have no LlmRequest on the sender. + [[nodiscard]] bool isArbitraryTransferState() const noexcept + { + return mIsArbitraryTransferState; + } + + void setIsArbitraryTransferState(bool isArbitraryTransferState) noexcept + { + mIsArbitraryTransferState = isArbitraryTransferState; + } + [[nodiscard]] bool operator==(DataTransceiverState const& other) const noexcept { - return mCacheState == other.mCacheState && mCommState == other.mCommState; + return mCacheState == other.mCacheState && mCommState == other.mCommState + && mIsArbitraryTransferState == other.mIsArbitraryTransferState; } [[nodiscard]] std::string toString() const @@ -641,6 +655,7 @@ class DataTransceiverState final friend class Serialization; std::optional<kv_cache::CacheState> mCacheState; std::optional<kv_cache::CommState> mCommState; + bool mIsArbitraryTransferState{false}; }; } // namespace tensorrt_llm::executor diff --git a/cpp/include/tensorrt_llm/executor/disaggServerUtil.h b/cpp/include/tensorrt_llm/executor/disaggServerUtil.h deleted file mode 100644 index b68dce78738a..000000000000 --- a/cpp/include/tensorrt_llm/executor/disaggServerUtil.h +++ /dev/null @@ -1,158 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/executor/executor.h" - -#include <cstdio> -#include <filesystem> -#include <memory> -#include <optional> -#include <vector> - -namespace tensorrt_llm::executor::disagg_executor -{ - -namespace texec = tensorrt_llm::executor; - -struct ResponseWithId -{ - - tensorrt_llm::executor::Response response; - IdType gid; - - ResponseWithId(tensorrt_llm::executor::Response&& response, IdType gid) - : response(std::move(response)) - , gid(gid) - { - } - - ResponseWithId(tensorrt_llm::executor::Response const& response, IdType gid) - : response(response) - , gid(gid) - { - } - - ResponseWithId(ResponseWithId&& other) noexcept - : response(std::move(other.response)) - , gid(other.gid) - { - other.gid = {}; - } - - ResponseWithId(ResponseWithId const& other) = default; - - ResponseWithId& operator=(ResponseWithId&& other) noexcept - { - if (this != &other) - { - response = std::move(other.response); - gid = other.gid; - other.gid = {}; - } - return *this; - } - - ResponseWithId& operator=(ResponseWithId const& other) - { - - if (this != &other) - { - response = other.response; - gid = other.gid; - } - return *this; - } - - ~ResponseWithId() = default; -}; - -class DisaggExecutorOrchestrator -{ -public: - /// @brief Constructs a DisaggExecutorOrchestrator object. - /// - /// @param ctxEnginePaths A vector of file paths to context engine files. - /// @param genEnginePaths A vector of file paths to generation engine files. - /// @param ctxExecutorConfigs A vector of ExecutorConfig for context executors. - /// @param genExecutorConfigs A vector of ExecutorConfig for generation executors. - /// @param hasContextAwaitThreads Whether or not there are threads that receive response for each generation - /// executor. - /// @param hasGenAwaitThreads Whether or not there are threads that receive response for each generation executor. - - DisaggExecutorOrchestrator(std::vector<std::filesystem::path> const& ctxEnginePaths, - std::vector<std::filesystem::path> const& genEnginePaths, - std::vector<executor::ExecutorConfig> const& ctxExecutorConfigs, - std::vector<executor::ExecutorConfig> const& genExecutorConfigs, bool hasContextAwaitThreads, - bool hasGenAwaitThreads); - - /// @brief Enqueue context-only requests to context executors. - /// @param requests A vector of context-only requests. - /// @param selectContextId The index of the context executor to use. If `std::nullopt`, the executor that has the - /// smallest number of inflight requests will be used. - /// @param batch If true,enqueue requests in same context executor.If false, will try to use a different executor - /// for each request. - /// @return A vector of global request ids, corresponding to the order of the requests in `requests`, the id - /// returned may be different from the request id in each executor. - [[nodiscard]] std::vector<IdType> enqueueContext(std::vector<texec::Request> const& requests, - std::optional<int> selectContextId = std::nullopt, bool batch = false); - - /// @brief Enqueue generation-only requests to generation executors. - /// @param requests A vector of generation-only requests. - /// @param globalRequestIds A vector of global request ids, corresponding to the order of the requests,and must be - /// the ids returned by the enqueueContext function. - /// @param selectGenIdx The index of the generation executor to use. If `std::nullopt`, the executor that has the - /// smallest number of inflight requests will be used. - /// @param batch If true,enqueue requests in same generation executor.If false, will try to use a different executor - /// for each request. - - void enqueueGeneration(std::vector<texec::Request> const& requests, std::vector<IdType> const& globalRequestIds, - std::optional<int> selectGenIdx = std::nullopt, bool batch = false); - - /// @brief Await for context responses - /// @param timeout The maximum time to wait for new responses - /// @param contextIdx The index of the context executor to use. If `std::nullopt`, return ready responses in all - /// context executors,if `hasContextAwaitThreads` is true, then this parameter must be std::nullopt. - /// @return A vector of responses with corresponding global request ids - - [[nodiscard]] std::vector<ResponseWithId> awaitContextResponses( - std::optional<std::chrono::milliseconds> const& timeout, std::optional<int> contextIdx = std::nullopt); - - /// @brief Await for generation responses - /// @param timeout The maximum time to wait for new responses. - /// @param genIdx The index of the generation executor to use. If `std::nullopt`, return ready responses in all - /// generation executors,if `hasGenAwaitThreads` is true, then this parameter must be std::nullopt. - /// @return A vector of responses with corresponding global request ids. - [[nodiscard]] std::vector<ResponseWithId> awaitGenerationResponses( - std::optional<std::chrono::milliseconds> const& timeout, std::optional<int> genIdx = std::nullopt); - - /// @brief Indicates if the current process is allowed to enqueueRequests - [[nodiscard]] bool canEnqueue() const; - - /// @brief Get context executors - [[nodiscard]] std::vector<std::unique_ptr<texec::Executor>> const& getContextExecutors() const; - - /// @brief Get generation executors - [[nodiscard]] std::vector<std::unique_ptr<texec::Executor>> const& getGenExecutors() const; - - ~DisaggExecutorOrchestrator(); - -private: - class Impl; - std::unique_ptr<Impl> mImpl; -}; -} // namespace tensorrt_llm::executor::disagg_executor diff --git a/cpp/include/tensorrt_llm/executor/executor.h b/cpp/include/tensorrt_llm/executor/executor.h index 825b8ad75959..acc0efe18966 100644 --- a/cpp/include/tensorrt_llm/executor/executor.h +++ b/cpp/include/tensorrt_llm/executor/executor.h @@ -18,6 +18,7 @@ #include "tensorrt_llm/executor/tensor.h" #include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/executor/version.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/runtimeDefaults.h" @@ -50,9 +51,11 @@ namespace tensorrt_llm::executor using SizeType32 = tensorrt_llm::runtime::SizeType32; /// @brief Version of TRT-LLM -char const* version() noexcept; +inline char const* version() noexcept +{ + return kTensorRtLlmVersion; +} -class Model; class Serialization; class DataTransceiverState; @@ -1233,6 +1236,11 @@ class DebugConfig SizeType32 mDebugTensorsMaxIterations; }; +/// @brief Configuration for the orchestrator communication mode. +/// @deprecated Orchestrator mode is non-functional: the worker binary it spawned +/// (executorWorker) was removed together with the TensorRT backend. This class is +/// retained only for serialization and Python-binding compatibility and is a +/// candidate for removal in a follow-up (needs API-stability review). class OrchestratorConfig { public: @@ -1835,7 +1843,13 @@ using KVCacheEventData = std::variant<KVCacheCreatedData, KVCacheStoredData, KVC struct KVCacheEvent { KVCacheEvent(IdType eventId, KVCacheEventData data, SizeType32 windowSize, - std::optional<SizeType32> attentionDpRank = std::nullopt); + std::optional<SizeType32> attentionDpRank = std::nullopt) + : eventId{eventId} + , data{std::move(data)} + , windowSize{windowSize} + , attentionDpRank{attentionDpRank} + { + } /// @brief The unique id of this event IdType eventId; @@ -1863,119 +1877,6 @@ class KVCacheEventManager std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager> kvCacheManager; }; -/// @brief The executor is responsible for receiving new requests and sending responses, and running the inference -class Executor -{ - -public: - /// @brief - /// @param modelPath Path to the folder that defines the model to run - /// @param modelType The type of model - /// @param executorConfig The configuration for the executor - Executor(std::filesystem::path const& modelPath, ModelType modelType, ExecutorConfig const& executorConfig); - - Executor(std::filesystem::path const& encoderModelPath, std::filesystem::path const& decoderModelPath, - ModelType modelType, ExecutorConfig const& executorConfig); - - Executor(BufferView const& engineBuffer, std::string const& jsonConfigStr, ModelType modelType, - ExecutorConfig const& executorConfig, - std::optional<std::map<std::string, Tensor>> const& managedWeights = std::nullopt); - - Executor(BufferView const& encoderEngineBuffer, std::string const& encoderJsonConfigStr, - BufferView const& decoderEngineBuffer, std::string const& decoderJsonConfigStr, ModelType modelType, - ExecutorConfig const& executorConfig); - - Executor(std::shared_ptr<Model> model, ExecutorConfig const& executorConfig); - - Executor( - std::shared_ptr<Model> encoderModel, std::shared_ptr<Model> decoderModel, ExecutorConfig const& executorConfig); - - ~Executor(); - Executor(Executor const& executor) = delete; - Executor& operator=(Executor const& executor) = delete; - Executor(Executor&&) = default; - Executor& operator=(Executor&&) = default; - - /// @brief Enqueue a new request - /// @param request The LLM request which contains input tokens and request parameters - /// @return A unique id that identifies the request - [[nodiscard]] IdType enqueueRequest(Request const& request); - - /// @brief Enqueue a batch of request - [[nodiscard]] std::vector<IdType> enqueueRequests(std::vector<Request> const& requests); - - /// @brief Await for ready responses - /// - /// This overload awaits for any ready responses. In particular, if several requests - /// have been enqueued, this method will provide any ready responses without order guarantees. - /// @param timeout The maximum time to wait for new responses - /// @return A vector of responses - [[nodiscard]] std::vector<Response> awaitResponses( - std::optional<std::chrono::milliseconds> const& timeout = std::nullopt); - - /// @brief Await for ready responses - /// @param id A request id - /// @param timeout The maximum time to wait for new responses - /// @return A vector of responses - [[nodiscard]] std::vector<Response> awaitResponses( - IdType const& requestId, std::optional<std::chrono::milliseconds> const& timeout = std::nullopt); - - /// @brief Await for multiple ready responses - /// - /// A multiple ID request behaves as if awaitResponses(IdType, timeout) - /// were invoked on all IDs. The returned vector contains - /// a vector of responses per ID in the same order specified by the requestIds. - /// The same behaviour as awaitResponses(IdType, timeout) applies: - /// * Responses may be empty. - /// * If all responses have already been given for one of the requestIds, - /// then this method will hang unless a timeout is specified. - /// @param requestIds Ids requested - /// @param timeout The maximum time to wait for new responses - /// @return A vector of vector of responses - [[nodiscard]] std::vector<std::vector<Response>> awaitResponses( - std::vector<IdType> const& requestIds, std::optional<std::chrono::milliseconds> const& timeout = std::nullopt); - - /// @brief Get the number of ready responses - /// @param requestId An optional request id - /// @return The number of ready responses - [[nodiscard]] SizeType32 getNumResponsesReady(std::optional<IdType> const& requestId = std::nullopt) const; - - /// @brief Cancel the request with provided request id - /// @param id The request id for which to cancel the response - void cancelRequest(IdType requestId); - - /// @brief Signals the server to shutdown. - /// @details This call is blocking. Only returns when all requests have terminated or timeout has been reached - void shutdown(); - - /// @brief Returns the per-iterations statistics computed since last call to getLatestIterationStats. - /// Contains at most iterStatsMaxIterations iterations, or all iterations when set to -1. - /// @return Iteration stats - std::deque<IterationStats> getLatestIterationStats(); - - /// @brief Returns the request stats of each iteration computed since last call to getLatestRequestStats. - /// Contains at most requestStatsMaxIterations iterations, or all iterations when set to -1. - /// @return Request stats grouped by iterations - std::deque<RequestStatsPerIteration> getLatestRequestStats(); - - /// @brief Returns the debug tensors of each iteration computed since last call to getLatestDebugTensors. - /// Contains at most debugTensorsMaxIterations iterations. - /// @return Request debug tensors grouped by iterations - std::deque<DebugTensorsPerIteration> getLatestDebugTensors(); - - /// @brief Indicates if the current process is allowed to enqueueRequests - [[nodiscard]] bool canEnqueueRequests() const; - - /// @brief Indicates if the current process participates in this executor instance - [[nodiscard]] bool isParticipant() const; - - std::optional<std::shared_ptr<KVCacheEventManager>> getKVCacheEventManager() const; - -private: - class Impl; - std::unique_ptr<Impl> mImpl; -}; - /// @brief Class with utility functions to serialize statistics to json string class JsonSerialization { diff --git a/cpp/include/tensorrt_llm/executor/transferAgent.h b/cpp/include/tensorrt_llm/executor/transferAgent.h index e1685c7c4ba5..e94f636a8312 100644 --- a/cpp/include/tensorrt_llm/executor/transferAgent.h +++ b/cpp/include/tensorrt_llm/executor/transferAgent.h @@ -216,12 +216,20 @@ struct VmmDescSplitter /// For non-VRAM or addresses not in the map, descs pass through unchanged. [[nodiscard]] static MemoryDescs splitDescsWithRegionMap(MemoryDescs const& descs, VramRegionMap const& regionMap); - /// @brief Split paired src/dst descs using local and remote region maps. - /// src is split by localRegionMap, dst is split by remoteRegionMap. - /// The final piece size is min(srcPiece, dstPiece, remaining). - [[nodiscard]] static std::pair<MemoryDescs, MemoryDescs> splitTransferDescsWithRegionMaps( - MemoryDescs const& srcDescs, MemoryDescs const& dstDescs, VramRegionMap const& localRegionMap, - VramRegionMap const& remoteRegionMap); + /// @brief Split paired src/dst descs at chunk boundaries, then coalesce contiguous pieces. + /// src is split by localRegionMap, dst is split by remoteRegionMap; each piece size is + /// min(srcPiece, dstPiece, remaining). Pairs are sorted by src address, and adjacent pieces + /// whose src AND dst are both contiguous (same deviceId) are merged — but a merged desc never + /// crosses a chunk boundary on either side, and never spans two distinct regions, so every + /// output desc stays within a single registered memory region. Merging requires region + /// metadata: a piece whose address misses the region map on either side is never merged, + /// because two unknown regions are indistinguishable and a merge could cross a chunk or + /// registration boundary. With no region metadata the result is split-only. Non-kVRAM descs + /// pass through unchanged (no region info is available to bound the merge). + /// @param enableCoalesce When false, only split at chunk boundaries without merging pieces. + [[nodiscard]] static std::pair<MemoryDescs, MemoryDescs> splitAndCoalesceTransferDescs(MemoryDescs const& srcDescs, + MemoryDescs const& dstDescs, VramRegionMap const& localRegionMap, VramRegionMap const& remoteRegionMap, + bool enableCoalesce = true); /// @brief Split VRAM descs at VMM chunk boundaries detected via cuMemGetAddressRange. /// For cudaMalloc memory (single allocation), descs pass through unchanged. @@ -436,6 +444,14 @@ class BaseLoopbackAgent virtual void executeLoopbackRequest(MemoryDescs const& memoryDescs, FileDescs const& fileDescs, bool isOffload) = 0; }; +/// @brief Promote the shared library containing this code (libtensorrt_llm.so) to the +/// process's global symbol scope. The KV cache transfer-agent wrapper libraries +/// (libtensorrt_llm_{nixl,ucx,mooncake}_wrapper.so) intentionally carry no DT_NEEDED on +/// libtensorrt_llm.so (the dependency would be circular) and resolve its symbols from the +/// global symbol table, while Python extension modules and their dependencies load with +/// RTLD_LOCAL. Idempotent; a no-op when the code is statically linked (e.g. unit tests). +void promoteHostLibraryToGlobalScope(); + class DynLibLoader final { public: diff --git a/cpp/include/tensorrt_llm/plugins/api/tllmPlugin.h b/cpp/include/tensorrt_llm/plugins/api/tllmPlugin.h deleted file mode 100644 index e3d4613e3d00..000000000000 --- a/cpp/include/tensorrt_llm/plugins/api/tllmPlugin.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <cstdint> -#include <mutex> - -// Forward declarations -namespace nvinfer1 -{ -class ILoggerFinder; -class ILogger; - -namespace v_1_0 -{ -class IPluginCreator; -class IPluginCreatorV3One; -class IPluginCreatorInterface; -} // namespace v_1_0 - -} // namespace nvinfer1 - -namespace tensorrt_llm::plugins::api -{ - -auto constexpr kDefaultNamespace = "tensorrt_llm"; - -class LoggerManager -{ -public: - //! Set the logger finder. - void setLoggerFinder(nvinfer1::ILoggerFinder* finder); - - //! Get the logger. - [[maybe_unused]] nvinfer1::ILogger* logger(); - - static LoggerManager& getInstance() noexcept; - - static nvinfer1::ILogger* defaultLogger() noexcept; - -private: - LoggerManager() = default; - - nvinfer1::ILoggerFinder* mLoggerFinder{nullptr}; - std::mutex mMutex; -}; -} // namespace tensorrt_llm::plugins::api - -extern "C" -{ - // This function is used for explicitly registering the TRT-LLM plugins and the default logger. - bool initTrtLlmPlugins(void* logger = tensorrt_llm::plugins::api::LoggerManager::defaultLogger(), - char const* libNamespace = tensorrt_llm::plugins::api::kDefaultNamespace); - - // The functions below are used by TensorRT to when loading a shared plugin library with automatic registering. - // see https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#generating-plugin-library - [[maybe_unused]] void setLoggerFinder([[maybe_unused]] nvinfer1::ILoggerFinder* finder); - [[maybe_unused]] nvinfer1::v_1_0::IPluginCreator* const* getPluginCreators(std::int32_t& nbCreators); - [[maybe_unused]] nvinfer1::v_1_0::IPluginCreatorInterface* const* getCreators(std::int32_t& nbCreators); -} diff --git a/cpp/include/tensorrt_llm/runtime/bufferManager.h b/cpp/include/tensorrt_llm/runtime/bufferManager.h index 8357443dc5ea..321a96ba321e 100644 --- a/cpp/include/tensorrt_llm/runtime/bufferManager.h +++ b/cpp/include/tensorrt_llm/runtime/bufferManager.h @@ -17,10 +17,10 @@ #pragma once #include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" -#include <NvInferRuntime.h> #include <cstring> #include <memory> @@ -62,63 +62,63 @@ class BufferManager } } - static auto constexpr kBYTE_TYPE = nvinfer1::DataType::kUINT8; + static auto constexpr kBYTE_TYPE = tensorrt_llm::DataType::kUINT8; //! \brief Allocates an `IBuffer` of the given size on the GPU, using cudaMallocAsync. - [[nodiscard]] IBufferPtr gpu(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE) const; + [[nodiscard]] IBufferPtr gpu(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE) const; //! \brief Allocates an `ITensor` of the given dimensions on the GPU, using cudaMallocAsync. - [[nodiscard]] ITensorPtr gpu(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE) const; + [[nodiscard]] ITensorPtr gpu(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE) const; //! \brief Allocates an `IBuffer` of the given size on the GPU, using cudaMalloc. - [[nodiscard]] static IBufferPtr gpuSync(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE); + [[nodiscard]] static IBufferPtr gpuSync(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE); //! \brief Allocates an `ITensor` of the given dimensions on the GPU, using cudaMalloc. - [[nodiscard]] static ITensorPtr gpuSync(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE); + [[nodiscard]] static ITensorPtr gpuSync(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE); //! \brief Allocates an `IBuffer` of the given size on the CPU. - [[nodiscard]] static IBufferPtr cpu(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE); + [[nodiscard]] static IBufferPtr cpu(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE); //! \brief Allocates an `ITensor` of the given dimensions on the CPU. - [[nodiscard]] static ITensorPtr cpu(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE); + [[nodiscard]] static ITensorPtr cpu(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE); //! \brief Allocates a pinned `IBuffer` of the given size on the CPU. - [[nodiscard]] static IBufferPtr pinned(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE); + [[nodiscard]] static IBufferPtr pinned(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE); //! \brief Allocates a pinned `ITensor` of the given dimensions on the CPU. - [[nodiscard]] static ITensorPtr pinned(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE); + [[nodiscard]] static ITensorPtr pinned(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE); //! \brief Allocates a pinned `IBuffer` of the given size on the CPU in the default memory pool. - [[nodiscard]] static IBufferPtr pinnedPool(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE); + [[nodiscard]] static IBufferPtr pinnedPool(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE); //! \brief Allocates a pinned `ITensor` of the given dimensions on the CPU in the default memory pool. - [[nodiscard]] static ITensorPtr pinnedPool(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE); + [[nodiscard]] static ITensorPtr pinnedPool(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE); //! \brief Allocates an `IBuffer` of the given size in UVM. - [[nodiscard]] static IBufferPtr managed(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE); + [[nodiscard]] static IBufferPtr managed(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE); //! \brief Allocates an `ITensor` of the given dimensions in UVM. - [[nodiscard]] static ITensorPtr managed(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE); + [[nodiscard]] static ITensorPtr managed(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE); //! \brief Allocates an `ITensor` of the given dimensions for NVLS - [[nodiscard]] static ITensorPtr ipcNvls(std::set<int> ranks, nvinfer1::Dims dims, nvinfer1::DataType type); + [[nodiscard]] static ITensorPtr ipcNvls(std::set<int> ranks, tensorrt_llm::Dims dims, tensorrt_llm::DataType type); //! \brief Allocates an `IBuffer` of the given size and memory type. [[nodiscard]] IBufferPtr allocate( - MemoryType memoryType, std::size_t size, nvinfer1::DataType type = kBYTE_TYPE) const; + MemoryType memoryType, std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE) const; //! \brief Allocates an `ITensor` of the given dimensions and memory type. [[nodiscard]] ITensorPtr allocate( - MemoryType memoryType, nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE) const; + MemoryType memoryType, tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE) const; //! \brief Create an empty `IBuffer` of the given memory type. It may be resized later. - [[nodiscard]] IBufferPtr emptyBuffer(MemoryType memoryType, nvinfer1::DataType type = kBYTE_TYPE) const + [[nodiscard]] IBufferPtr emptyBuffer(MemoryType memoryType, tensorrt_llm::DataType type = kBYTE_TYPE) const { return allocate(memoryType, 0, type); } //! \brief Create an empty `ITensor` of the given memory type. It may be reshaped later. - [[nodiscard]] ITensorPtr emptyTensor(MemoryType memoryType, nvinfer1::DataType type = kBYTE_TYPE) const + [[nodiscard]] ITensorPtr emptyTensor(MemoryType memoryType, tensorrt_llm::DataType type = kBYTE_TYPE) const { return allocate(memoryType, ITensor::makeShape({}), type); } @@ -167,7 +167,7 @@ class BufferManager //! \brief Copy `src` into a new `ITensor` with a potentially different memory type. template <typename T> - [[nodiscard]] ITensorPtr copyFrom(T* src, nvinfer1::Dims dims, MemoryType memoryType) const + [[nodiscard]] ITensorPtr copyFrom(T* src, tensorrt_llm::Dims dims, MemoryType memoryType) const { auto buffer = allocate(memoryType, dims, TRTDataType<std::remove_cv_t<T>>::value); copy(src, *buffer); @@ -176,7 +176,7 @@ class BufferManager //! \brief Copy `src` into a new `ITensor` with a potentially different memory type. template <typename T> - [[nodiscard]] ITensorPtr copyFrom(std::vector<T> const& src, nvinfer1::Dims dims, MemoryType memoryType) const + [[nodiscard]] ITensorPtr copyFrom(std::vector<T> const& src, tensorrt_llm::Dims dims, MemoryType memoryType) const { TLLM_CHECK_WITH_INFO(src.size() == ITensor::volumeNonNegative(dims), common::fmtstr("[TensorRT-LLM][ERROR] Incompatible size %lu and dims %s", src.size(), diff --git a/cpp/include/tensorrt_llm/runtime/decoderState.h b/cpp/include/tensorrt_llm/runtime/decoderState.h index 95d7ff0ffac9..ea2c767c0478 100644 --- a/cpp/include/tensorrt_llm/runtime/decoderState.h +++ b/cpp/include/tensorrt_llm/runtime/decoderState.h @@ -18,6 +18,7 @@ #include "decodingInput.h" #include "decodingOutput.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/speculativeDecodingMode.h" @@ -52,7 +53,7 @@ class DecoderState //! @brief Setup buffers for the decoder excluding speculative decoding. void setup(SizeType32 maxNumSequences, SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow, - SizeType32 sinkTokenLength, SizeType32 maxSequenceLength, nvinfer1::DataType dtype, + SizeType32 sinkTokenLength, SizeType32 maxSequenceLength, tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager); //! @brief Setup buffers for the cache indirection. @@ -62,7 +63,7 @@ class DecoderState //! @brief Setup buffers for speculative decoding. void setupSpeculativeDecoding(SpeculativeDecodingMode const& speculativeDecodingMode, - SizeType32 maxTokensPerEngineStep, nvinfer1::DataType dtype, ModelConfig const& modelConfig, + SizeType32 maxTokensPerEngineStep, tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager); //! @brief Disable lookahead decoding. @@ -199,7 +200,7 @@ class DecoderState [[nodiscard]] DecodingOutput& getJointDecodingOutput() const; private: - void setupBuffers(nvinfer1::DataType dtype, BufferManager const& bufferManager); + void setupBuffers(tensorrt_llm::DataType dtype, BufferManager const& bufferManager); void reshapeBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLength, SizeType32 maxSequenceLength, ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager); @@ -208,8 +209,8 @@ class DecoderState void reshapeCacheIndirectionBuffers( SizeType32 maxBatchSize, SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow); - void setupSpeculativeDecodingBuffers( - SpeculativeDecodingMode speculativeDecodingMode, nvinfer1::DataType dtype, BufferManager const& bufferManager); + void setupSpeculativeDecodingBuffers(SpeculativeDecodingMode speculativeDecodingMode, tensorrt_llm::DataType dtype, + BufferManager const& bufferManager); void reshapeSpeculativeDecodingBuffers(SpeculativeDecodingMode const& speculativeDecodingMode, SizeType32 maxTokensPerEngineStep, ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager); diff --git a/cpp/include/tensorrt_llm/runtime/gptDecoder.h b/cpp/include/tensorrt_llm/runtime/gptDecoder.h index 7e0cc1bb56d2..5a785e84fe75 100644 --- a/cpp/include/tensorrt_llm/runtime/gptDecoder.h +++ b/cpp/include/tensorrt_llm/runtime/gptDecoder.h @@ -22,7 +22,7 @@ #include "tensorrt_llm/runtime/decodingOutput.h" #include "tensorrt_llm/runtime/samplingConfig.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <curand_kernel.h> #include <memory> @@ -55,7 +55,7 @@ class IGptDecoder /// @param explicitDraftTokensDType is only used by ExplicitDraftTokens model to WAR the lack of bf16 decoder. virtual void setup(SamplingConfig const& samplingConfig, size_t batchSize, TensorConstPtr const& batchSlots, std::optional<DecodingOutput> const& output = std::nullopt, - std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt, + std::optional<tensorrt_llm::DataType> explicitDraftTokensDType = std::nullopt, std::optional<std::vector<TensorConstPtr>> const& lookaheadPrompt = std::nullopt, std::optional<std::vector<executor::LookaheadDecodingConfig>> const& lookaheadAlgoConfigs = std::nullopt) = 0; @@ -70,7 +70,7 @@ class IGptDecoder std::optional<SamplingConfig> const& samplingConfig, SizeType32 batchSize, TensorConstPtr batchSlots) = 0; - static std::unique_ptr<IGptDecoder> create(executor::DecodingMode const& mode, nvinfer1::DataType dtype, + static std::unique_ptr<IGptDecoder> create(executor::DecodingMode const& mode, tensorrt_llm::DataType dtype, size_t maxNumSequences, size_t maxBeamWidth, size_t vocabSize, size_t vocabSizePadded, BufferManager::CudaStreamPtr const& stream, std::shared_ptr<SpeculativeDecodingModule const> const& speculativeDecodingModule = nullptr); @@ -90,7 +90,7 @@ class GptDecoder : public virtual IGptDecoder void setup(SamplingConfig const& samplingConfig, size_t batchSize, TensorConstPtr const& batchSlots, std::optional<DecodingOutput> const& output = std::nullopt, - std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt, + std::optional<tensorrt_llm::DataType> explicitDraftTokensDType = std::nullopt, std::optional<std::vector<TensorConstPtr>> const& lookaheadPrompt = std::nullopt, std::optional<std::vector<executor::LookaheadDecodingConfig>> const& lookaheadAlgoConfigs = std::nullopt) override; @@ -121,17 +121,17 @@ class GptDecoder : public virtual IGptDecoder executor::DecodingMode mDecodingMode; }; -inline std::unique_ptr<IGptDecoder> IGptDecoder::create(executor::DecodingMode const& mode, nvinfer1::DataType dtype, - size_t maxNumSequences, size_t maxBeamWidth, size_t vocabSize, size_t vocabSizePadded, +inline std::unique_ptr<IGptDecoder> IGptDecoder::create(executor::DecodingMode const& mode, + tensorrt_llm::DataType dtype, size_t maxNumSequences, size_t maxBeamWidth, size_t vocabSize, size_t vocabSizePadded, BufferManager::CudaStreamPtr const& stream, std::shared_ptr<SpeculativeDecodingModule const> const& speculativeDecodingModule) { switch (dtype) { - case nvinfer1::DataType::kFLOAT: + case tensorrt_llm::DataType::kFLOAT: return std::make_unique<GptDecoder<float>>( mode, maxNumSequences, maxBeamWidth, vocabSize, vocabSizePadded, stream, speculativeDecodingModule); - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: return std::make_unique<GptDecoder<half>>( mode, maxNumSequences, maxBeamWidth, vocabSize, vocabSizePadded, stream, speculativeDecodingModule); default: diff --git a/cpp/include/tensorrt_llm/runtime/gptDecoderBatched.h b/cpp/include/tensorrt_llm/runtime/gptDecoderBatched.h index 9fcd3262c8ca..d5447f441163 100644 --- a/cpp/include/tensorrt_llm/runtime/gptDecoderBatched.h +++ b/cpp/include/tensorrt_llm/runtime/gptDecoderBatched.h @@ -16,6 +16,7 @@ #pragma once +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaEvent.h" #include "tensorrt_llm/runtime/cudaStream.h" @@ -48,7 +49,7 @@ class GptDecoderBatched : public IGptDecoderBatched explicit GptDecoderBatched(CudaStreamPtr stream); void setup(executor::DecodingMode const& mode, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - nvinfer1::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) override; + tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) override; void disableLookahead(RequestVector const& genRequests, TensorPtr const& batchSlots) override; diff --git a/cpp/include/tensorrt_llm/runtime/iBuffer.h b/cpp/include/tensorrt_llm/runtime/iBuffer.h index 91d5cd739f32..bf63d3a0da7b 100644 --- a/cpp/include/tensorrt_llm/runtime/iBuffer.h +++ b/cpp/include/tensorrt_llm/runtime/iBuffer.h @@ -22,7 +22,7 @@ #include "tensorrt_llm/kernels/kvCacheIndex.h" #include "tensorrt_llm/runtime/common.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cstdint> #ifdef ENABLE_FP8 @@ -88,13 +88,13 @@ struct MemoryTypeString<MemoryType::kPINNEDPOOL> }; //! \brief For converting a TensorRT data type to a C++ data type. -template <nvinfer1::DataType kDataType, bool kIsUnsigned = false, bool kIsPointer = false> +template <tensorrt_llm::DataType kDataType, bool kIsUnsigned = false, bool kIsPointer = false> struct DataTypeTraits { }; template <> -struct DataTypeTraits<nvinfer1::DataType::kFLOAT> +struct DataTypeTraits<tensorrt_llm::DataType::kFLOAT> { using type = float; static char constexpr name[] = "float"; @@ -102,7 +102,7 @@ struct DataTypeTraits<nvinfer1::DataType::kFLOAT> }; template <> -struct DataTypeTraits<nvinfer1::DataType::kHALF> +struct DataTypeTraits<tensorrt_llm::DataType::kHALF> { using type = half; static char constexpr name[] = "half"; @@ -110,7 +110,7 @@ struct DataTypeTraits<nvinfer1::DataType::kHALF> }; template <> -struct DataTypeTraits<nvinfer1::DataType::kINT8> +struct DataTypeTraits<tensorrt_llm::DataType::kINT8> { using type = std::int8_t; static char constexpr name[] = "int8"; @@ -118,7 +118,7 @@ struct DataTypeTraits<nvinfer1::DataType::kINT8> }; template <> -struct DataTypeTraits<nvinfer1::DataType::kINT32> +struct DataTypeTraits<tensorrt_llm::DataType::kINT32> { using type = std::int32_t; static char constexpr name[] = "int32"; @@ -126,7 +126,7 @@ struct DataTypeTraits<nvinfer1::DataType::kINT32> }; template <> -struct DataTypeTraits<nvinfer1::DataType::kINT64> +struct DataTypeTraits<tensorrt_llm::DataType::kINT64> { using type = std::int64_t; static char constexpr name[] = "int64"; @@ -134,7 +134,7 @@ struct DataTypeTraits<nvinfer1::DataType::kINT64> }; template <> -struct DataTypeTraits<nvinfer1::DataType::kINT32, true> +struct DataTypeTraits<tensorrt_llm::DataType::kINT32, true> { using type = std::uint32_t; static char constexpr name[] = "uint32"; @@ -142,7 +142,7 @@ struct DataTypeTraits<nvinfer1::DataType::kINT32, true> }; template <> -struct DataTypeTraits<nvinfer1::DataType::kINT64, true> +struct DataTypeTraits<tensorrt_llm::DataType::kINT64, true> { using type = std::uint64_t; static char constexpr name[] = "uint64"; @@ -150,7 +150,7 @@ struct DataTypeTraits<nvinfer1::DataType::kINT64, true> }; template <bool kUnsigned> -struct DataTypeTraits<nvinfer1::DataType::kBOOL, kUnsigned> +struct DataTypeTraits<tensorrt_llm::DataType::kBOOL, kUnsigned> { using type = bool; static char constexpr name[] = "bool"; @@ -158,7 +158,7 @@ struct DataTypeTraits<nvinfer1::DataType::kBOOL, kUnsigned> }; template <bool kUnsigned> -struct DataTypeTraits<nvinfer1::DataType::kUINT8, kUnsigned> +struct DataTypeTraits<tensorrt_llm::DataType::kUINT8, kUnsigned> { using type = std::uint8_t; static char constexpr name[] = "uint8"; @@ -167,7 +167,7 @@ struct DataTypeTraits<nvinfer1::DataType::kUINT8, kUnsigned> #ifdef ENABLE_BF16 template <> -struct DataTypeTraits<nvinfer1::DataType::kBF16> +struct DataTypeTraits<tensorrt_llm::DataType::kBF16> { using type = __nv_bfloat16; static char constexpr name[] = "bfloat16"; @@ -177,7 +177,7 @@ struct DataTypeTraits<nvinfer1::DataType::kBF16> #ifdef ENABLE_FP8 template <> -struct DataTypeTraits<nvinfer1::DataType::kFP8> +struct DataTypeTraits<tensorrt_llm::DataType::kFP8> { using type = __nv_fp8_e4m3; static char constexpr name[] = "fp8"; @@ -185,7 +185,7 @@ struct DataTypeTraits<nvinfer1::DataType::kFP8> }; #endif -template <nvinfer1::DataType kDataType, bool kUnsigned> +template <tensorrt_llm::DataType kDataType, bool kUnsigned> struct DataTypeTraits<kDataType, kUnsigned, true> { using type = typename DataTypeTraits<kDataType, kUnsigned, false>::type*; @@ -193,26 +193,26 @@ struct DataTypeTraits<kDataType, kUnsigned, true> static auto constexpr size = sizeof(type); }; -//! \brief A wrapper around `nvinfer1::DataType` that provides a support for pointer types. +//! \brief A wrapper around `tensorrt_llm::DataType` that provides a support for pointer types. class BufferDataType { public: constexpr BufferDataType( // NOLINT(*-explicit-constructor) - nvinfer1::DataType dataType, bool _unsigned = false, bool pointer = false) + tensorrt_llm::DataType dataType, bool _unsigned = false, bool pointer = false) : mDataType{dataType} , mUnsigned{_unsigned} , mPointer{pointer} { } - static auto constexpr kTrtPointerType = nvinfer1::DataType::kINT64; + static auto constexpr kTrtPointerType = tensorrt_llm::DataType::kINT64; - constexpr operator nvinfer1::DataType() const noexcept // NOLINT(*-explicit-constructor) + constexpr operator tensorrt_llm::DataType() const noexcept // NOLINT(*-explicit-constructor) { return mPointer ? kTrtPointerType : mDataType; } - [[nodiscard]] constexpr nvinfer1::DataType getDataType() const noexcept + [[nodiscard]] constexpr tensorrt_llm::DataType getDataType() const noexcept { return mDataType; } @@ -226,24 +226,24 @@ class BufferDataType { switch (mDataType) { - case nvinfer1::DataType::kBOOL: [[fallthrough]]; - case nvinfer1::DataType::kUINT8: return true; + case tensorrt_llm::DataType::kBOOL: [[fallthrough]]; + case tensorrt_llm::DataType::kUINT8: return true; default: return mUnsigned; } } [[nodiscard]] constexpr std::size_t getSize() const noexcept { - return tensorrt_llm::common::getDTypeSize(static_cast<nvinfer1::DataType>(*this)); + return tensorrt_llm::common::getDTypeSize(static_cast<tensorrt_llm::DataType>(*this)); } [[nodiscard]] constexpr std::size_t getSizeInBits() const noexcept { - return tensorrt_llm::common::getDTypeSizeInBits(static_cast<nvinfer1::DataType>(*this)); + return tensorrt_llm::common::getDTypeSizeInBits(static_cast<tensorrt_llm::DataType>(*this)); } private: - nvinfer1::DataType mDataType; + tensorrt_llm::DataType mDataType; bool mUnsigned; bool mPointer; }; @@ -257,62 +257,62 @@ struct TRTDataType template <> struct TRTDataType<float> { - static constexpr auto value = nvinfer1::DataType::kFLOAT; + static constexpr auto value = tensorrt_llm::DataType::kFLOAT; }; template <> struct TRTDataType<half> { - static constexpr auto value = nvinfer1::DataType::kHALF; + static constexpr auto value = tensorrt_llm::DataType::kHALF; }; template <> struct TRTDataType<std::int8_t> { - static constexpr auto value = nvinfer1::DataType::kINT8; + static constexpr auto value = tensorrt_llm::DataType::kINT8; }; template <> struct TRTDataType<std::int32_t> { - static constexpr auto value = nvinfer1::DataType::kINT32; + static constexpr auto value = tensorrt_llm::DataType::kINT32; }; template <> struct TRTDataType<std::uint32_t> { - static constexpr auto value = BufferDataType{nvinfer1::DataType::kINT32, true}; + static constexpr auto value = BufferDataType{tensorrt_llm::DataType::kINT32, true}; }; template <> struct TRTDataType<std::int64_t> { - static constexpr auto value = nvinfer1::DataType::kINT64; + static constexpr auto value = tensorrt_llm::DataType::kINT64; }; template <> struct TRTDataType<std::uint64_t> { - static constexpr auto value = BufferDataType{nvinfer1::DataType::kINT64, true}; + static constexpr auto value = BufferDataType{tensorrt_llm::DataType::kINT64, true}; }; template <> struct TRTDataType<bool> { - static constexpr auto value = nvinfer1::DataType::kBOOL; + static constexpr auto value = tensorrt_llm::DataType::kBOOL; }; template <> struct TRTDataType<std::uint8_t> { - static constexpr auto value = nvinfer1::DataType::kUINT8; + static constexpr auto value = tensorrt_llm::DataType::kUINT8; }; #ifdef ENABLE_BF16 template <> struct TRTDataType<__nv_bfloat16> { - static constexpr auto value = nvinfer1::DataType::kBF16; + static constexpr auto value = tensorrt_llm::DataType::kBF16; }; #endif @@ -320,7 +320,7 @@ struct TRTDataType<__nv_bfloat16> template <> struct TRTDataType<__nv_fp8_e4m3> { - static constexpr auto value = nvinfer1::DataType::kFP8; + static constexpr auto value = tensorrt_llm::DataType::kFP8; }; #endif @@ -380,7 +380,7 @@ class IBuffer using SharedPtr = std::shared_ptr<IBuffer>; using UniqueConstPtr = std::unique_ptr<IBuffer const>; using SharedConstPtr = std::shared_ptr<IBuffer const>; - using DataType = nvinfer1::DataType; + using DataType = tensorrt_llm::DataType; //! //! \brief Returns a pointer to underlying array. diff --git a/cpp/include/tensorrt_llm/runtime/iGptDecoderBatched.h b/cpp/include/tensorrt_llm/runtime/iGptDecoderBatched.h index ab55b754f9be..b664bc007f0e 100644 --- a/cpp/include/tensorrt_llm/runtime/iGptDecoderBatched.h +++ b/cpp/include/tensorrt_llm/runtime/iGptDecoderBatched.h @@ -16,6 +16,7 @@ #pragma once +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/cudaEvent.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -51,7 +52,7 @@ class IGptDecoderBatched //! @brief Setup the decoder before calling `forward()` virtual void setup(executor::DecodingMode const& mode, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - nvinfer1::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) + tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) = 0; //! @brief Disable Lookahead decoding. diff --git a/cpp/include/tensorrt_llm/runtime/iTensor.h b/cpp/include/tensorrt_llm/runtime/iTensor.h index eb5c10eeb691..a85291dd8263 100644 --- a/cpp/include/tensorrt_llm/runtime/iTensor.h +++ b/cpp/include/tensorrt_llm/runtime/iTensor.h @@ -20,7 +20,7 @@ #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iBuffer.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <algorithm> #include <cstdint> @@ -33,11 +33,6 @@ #include <string> #include <type_traits> -namespace nvinfer1 -{ -class IExecutionContext; -} - namespace tensorrt_llm::runtime { @@ -50,7 +45,7 @@ class ITensor : virtual public IBuffer using SharedPtr = std::shared_ptr<ITensor>; using UniqueConstPtr = std::unique_ptr<ITensor const>; using SharedConstPtr = std::shared_ptr<ITensor const>; - using Shape = nvinfer1::Dims; + using Shape = tensorrt_llm::Dims; using DimType64 = std::remove_reference_t<decltype(Shape::d[0])>; using TensorMap = runtime::StringPtrMap<runtime::ITensor>; @@ -352,9 +347,9 @@ class ITensor : virtual public IBuffer //! \param shape The shape of the tensor. //! \param capacity The capacity of the buffer. //! \return An `ITensor`. - static UniquePtr wrap(void* data, nvinfer1::DataType type, Shape const& shape, std::size_t capacity); + static UniquePtr wrap(void* data, tensorrt_llm::DataType type, Shape const& shape, std::size_t capacity); - static UniquePtr wrap(void* data, nvinfer1::DataType type, Shape const& shape) + static UniquePtr wrap(void* data, tensorrt_llm::DataType type, Shape const& shape) { return wrap(data, type, shape, volumeNonNegative(shape)); } diff --git a/cpp/include/tensorrt_llm/runtime/lookaheadBuffers.h b/cpp/include/tensorrt_llm/runtime/lookaheadBuffers.h index ecaa439f2d52..26c6e3886be4 100644 --- a/cpp/include/tensorrt_llm/runtime/lookaheadBuffers.h +++ b/cpp/include/tensorrt_llm/runtime/lookaheadBuffers.h @@ -17,9 +17,9 @@ #pragma once #include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" #include "tensorrt_llm/runtime/worldConfig.h" namespace tensorrt_llm::runtime @@ -37,47 +37,4 @@ class LookaheadDecodingBuffers TensorPtr positionIds; }; -class LookaheadRuntimeBuffers -{ -public: - using TensorPtr = ITensor::SharedPtr; - using TensorMap = StringPtrMap<ITensor>; - - LookaheadRuntimeBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, BufferManager const& manager, - ModelConfig const& modelConfig, WorldConfig const& worldConfig, executor::DecodingConfig const& decodingConfig, - TllmRuntime const& runtime); - - void setFromInputs(SizeType32 numCtxSequences, SizeType32 numGenSequences, ITensor const& requestTypes, - ITensor const& seqSlots, LookaheadDecodingBuffers const& decoderLookaheadBuffers, TllmRuntime const& runtime, - ModelConfig const& modelConfig, WorldConfig const& worldConfig) const; - - void reshape(SizeType32 numCtxSequences, SizeType32 numGenSequences, SizeType32 tokensPerStep); - - void insertInputTensors(TensorMap& inputBuffers, TensorMap& outputBuffers, WorldConfig const& worldConfig) const; - - void enableLookaheadDecoding(SizeType32 maxBatchSize, SizeType32 tokensPerStep); - - void disableLookaheadDecoding(); - -public: - TensorPtr cumSumLength; // [1] the cumulative sum of generation length, on pinned - TensorPtr packedMasksDevice; // [forwardBatchSize, tokensPerStep, numPackedMasks], on gpu - TensorPtr generationLengthsDevice; // [forwardBatchSize], on gpu - TensorPtr positionOffsetsDevice; // [forwardBatchSize, tokensPerStep], on gpu - TensorPtr positionIdsDevice; // [forwardBatchSize, tokensPerStep], on gpu - - TensorPtr packedMaskHost; - TensorPtr generationLengthsHost; - TensorPtr positionOffsetsHost; - TensorPtr positionIdsHost; - - TensorPtr packedMaskHostCopy; - TensorPtr generationLengthsHostCopy; - TensorPtr positionOffsetsHostCopy; - TensorPtr positionIdsHostCopy; - TensorPtr useSpecDecoding; - - TensorPtr batchSlotsHostCopy; -}; - } // namespace tensorrt_llm::runtime diff --git a/cpp/include/tensorrt_llm/runtime/loraCache.h b/cpp/include/tensorrt_llm/runtime/loraCache.h index 1d242cdc80c5..eb4ef57494ea 100644 --- a/cpp/include/tensorrt_llm/runtime/loraCache.h +++ b/cpp/include/tensorrt_llm/runtime/loraCache.h @@ -25,8 +25,6 @@ #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include <NvInferRuntime.h> - #include <deque> #include <list> #include <map> diff --git a/cpp/include/tensorrt_llm/runtime/loraCachePageManagerConfig.h b/cpp/include/tensorrt_llm/runtime/loraCachePageManagerConfig.h index a995304e94ec..cf1b1a6aac18 100644 --- a/cpp/include/tensorrt_llm/runtime/loraCachePageManagerConfig.h +++ b/cpp/include/tensorrt_llm/runtime/loraCachePageManagerConfig.h @@ -20,7 +20,7 @@ #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iBuffer.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <ostream> #include <sstream> @@ -36,7 +36,7 @@ namespace tensorrt_llm::runtime class LoraCachePageManagerConfig { public: - explicit constexpr LoraCachePageManagerConfig(runtime::MemoryType memType, nvinfer1::DataType dType, + explicit constexpr LoraCachePageManagerConfig(runtime::MemoryType memType, tensorrt_llm::DataType dType, SizeType32 totalNumPages, SizeType32 maxPagesPerBlock, SizeType32 slotsPerPage, SizeType32 pageWidth, SizeType32 numCopyStreams) : mMemoryType(memType) @@ -59,12 +59,12 @@ class LoraCachePageManagerConfig mMemoryType = memoryType; } - [[nodiscard]] nvinfer1::DataType constexpr getDataType() const noexcept + [[nodiscard]] tensorrt_llm::DataType constexpr getDataType() const noexcept { return mDataType; } - void constexpr setDataType(nvinfer1::DataType const& dtype) noexcept + void constexpr setDataType(tensorrt_llm::DataType const& dtype) noexcept { mDataType = dtype; } @@ -131,7 +131,7 @@ class LoraCachePageManagerConfig private: runtime::MemoryType mMemoryType; - nvinfer1::DataType mDataType; + tensorrt_llm::DataType mDataType; /* * Number cache pages in the cache. @@ -154,7 +154,7 @@ inline std::ostream& operator<<(std::ostream& os, LoraCachePageManagerConfig con { os << "{" << "memoryType=" << static_cast<typename std::underlying_type<runtime::MemoryType>::type>(c.getMemoryType()) - << " dataType=" << static_cast<typename std::underlying_type<nvinfer1::DataType>::type>(c.getDataType()) + << " dataType=" << static_cast<typename std::underlying_type<tensorrt_llm::DataType>::type>(c.getDataType()) << " totalNumPages=" << c.getTotalNumPages() << " maxPagesPerBlock=" << c.getMaxPagesPerBlock() << " slotsPerPage=" << c.getSlotsPerPage() << " pageWidth=" << c.getPageWidth() << " initToZero=" << c.getInitToZero() << "}"; diff --git a/cpp/include/tensorrt_llm/runtime/modelConfig.h b/cpp/include/tensorrt_llm/runtime/modelConfig.h index 5bfe7bce9d58..b5f18da07f3b 100644 --- a/cpp/include/tensorrt_llm/runtime/modelConfig.h +++ b/cpp/include/tensorrt_llm/runtime/modelConfig.h @@ -23,7 +23,7 @@ #include "tensorrt_llm/runtime/speculativeDecodingMode.h" #include "tensorrt_llm/runtime/speculativeDecodingModule.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <array> namespace tensorrt_llm::runtime @@ -101,7 +101,7 @@ class ModelConfig }; explicit ModelConfig(SizeType32 vocabSize, SizeType32 nbLayers, SizeType32 nbAttentionLayers, - SizeType32 nbRnnLayers, SizeType32 nbHeads, SizeType32 hiddenSize, nvinfer1::DataType dtype) + SizeType32 nbRnnLayers, SizeType32 nbHeads, SizeType32 hiddenSize, tensorrt_llm::DataType dtype) : mVocabSize(vocabSize) , mNbLayers(nbLayers) , mNbAttentionLayers(nbAttentionLayers) @@ -137,7 +137,7 @@ class ModelConfig , mUsePositionEmbedding(false) , mUseTokenTypeEmbedding(false) , mSpeculativeDecodingMode(SpeculativeDecodingMode::None()) - , mLogitsDtype(nvinfer1::DataType::kFLOAT) + , mLogitsDtype(tensorrt_llm::DataType::kFLOAT) , mUseShapeInference(true) , mManageWeightsType(ManageWeightsType::kDisabled) , mSkipCrossAttnBlocks(false) @@ -331,7 +331,7 @@ class ModelConfig mSizePerHead = sizePerHead; } - [[nodiscard]] nvinfer1::DataType constexpr getDataType() const noexcept + [[nodiscard]] tensorrt_llm::DataType constexpr getDataType() const noexcept { return mDataType; } @@ -735,20 +735,20 @@ class ModelConfig resetSpeculativeDecodingModule(); } - [[nodiscard]] nvinfer1::DataType getKvDataType() const + [[nodiscard]] tensorrt_llm::DataType getKvDataType() const { if (getQuantMode().hasFp8KvCache()) { - return nvinfer1::DataType::kFP8; + return tensorrt_llm::DataType::kFP8; } if (getQuantMode().hasInt8KvCache()) { - return nvinfer1::DataType::kINT8; + return tensorrt_llm::DataType::kINT8; } else if (getQuantMode().hasFp4KvCache()) { #ifdef ENABLE_FP4 - return nvinfer1::DataType::kFP4; + return tensorrt_llm::DataType::kFP4; #else throw std::runtime_error("Model has FP4 KV cache, but TRT-LLM was not compiled with FP4 enabled."); #endif @@ -800,22 +800,22 @@ class ModelConfig return mSpeculativeDecodingMode; } - void setLogitsDtype(nvinfer1::DataType inputDtype) noexcept + void setLogitsDtype(tensorrt_llm::DataType inputDtype) noexcept { mLogitsDtype = inputDtype; } - [[nodiscard]] nvinfer1::DataType constexpr getLogitsDtype() const noexcept + [[nodiscard]] tensorrt_llm::DataType constexpr getLogitsDtype() const noexcept { return mLogitsDtype; } - void setGemmAllReduceDtype(nvinfer1::DataType inputDtype) noexcept + void setGemmAllReduceDtype(tensorrt_llm::DataType inputDtype) noexcept { mGemmAllReduceDtype = inputDtype; } - [[nodiscard]] nvinfer1::DataType constexpr getGemmAllReduceDtype() const noexcept + [[nodiscard]] tensorrt_llm::DataType constexpr getGemmAllReduceDtype() const noexcept { return mGemmAllReduceDtype; } @@ -945,10 +945,10 @@ class ModelConfig SizeType32 mNbHeads; SizeType32 mHiddenSize; SizeType32 mSizePerHead; - nvinfer1::DataType mDataType; + tensorrt_llm::DataType mDataType; bool mUseGptAttentionPlugin; bool mUseGemmAllReducePlugin; - nvinfer1::DataType mGemmAllReduceDtype; + tensorrt_llm::DataType mGemmAllReduceDtype; bool mUseMambaConv1dPlugin; bool mInputPacked; bool mPagedState; @@ -998,7 +998,7 @@ class ModelConfig SpeculativeDecodingMode mSpeculativeDecodingMode; // Logits datatype - nvinfer1::DataType mLogitsDtype; + tensorrt_llm::DataType mLogitsDtype; bool mUseShapeInference; ManageWeightsType mManageWeightsType; std::string mModelName; diff --git a/cpp/include/tensorrt_llm/runtime/rawEngine.h b/cpp/include/tensorrt_llm/runtime/rawEngine.h deleted file mode 100644 index b219cbe03382..000000000000 --- a/cpp/include/tensorrt_llm/runtime/rawEngine.h +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/executor/tensor.h" - -#include <NvInferRuntime.h> -#include <filesystem> -#include <map> -#include <optional> - -namespace tensorrt_llm::runtime -{ - -class RawEngine -{ -public: - enum Type - { - FilePath, - AddressWithSize, - HostMemory - }; - - explicit RawEngine(std::filesystem::path enginePath) noexcept - : mType(FilePath) - , mEnginePath(std::move(enginePath)) - { - } - - explicit RawEngine(void const* engineAddr, std::size_t engineSize) noexcept - : mType(AddressWithSize) - , mEngineAddr(engineAddr) - , mEngineSize(engineSize) - { - } - - explicit RawEngine(nvinfer1::IHostMemory const* engineBuffer) noexcept - : mType(HostMemory) - , mEngineBuffer(engineBuffer) - { - } - - [[nodiscard]] Type getType() const - { - return mType; - } - - [[nodiscard]] std::filesystem::path getPath() const - { - TLLM_CHECK(mEnginePath.has_value()); - return mEnginePath.value(); - } - - [[nodiscard]] std::optional<std::filesystem::path> getPathOpt() const - { - return mEnginePath; - } - - void setPath(std::filesystem::path enginePath) - { - mEnginePath = std::move(enginePath); - } - - [[nodiscard]] std::optional<std::map<std::string, tensorrt_llm::executor::Tensor>> const& - getManagedWeightsMapOpt() const - { - return mManagedWeightsMap; - } - - void setManagedWeightsMap(std::map<std::string, tensorrt_llm::executor::Tensor> managedWeightsMap) - { - mManagedWeightsMap = std::move(managedWeightsMap); - } - - [[nodiscard]] void const* getAddress() const - { - TLLM_CHECK(mType == AddressWithSize); - return mEngineAddr; - } - - [[nodiscard]] std::size_t getSize() const - { - TLLM_CHECK(mType == AddressWithSize); - return mEngineSize; - } - - [[nodiscard]] nvinfer1::IHostMemory const* getHostMemory() const - { - TLLM_CHECK(mType == HostMemory); - return mEngineBuffer; - } - -private: - Type mType; - std::optional<std::filesystem::path> mEnginePath; - - struct - { - void const* mEngineAddr{}; - std::size_t mEngineSize{}; - }; - - nvinfer1::IHostMemory const* mEngineBuffer{}; - std::optional<std::map<std::string, tensorrt_llm::executor::Tensor>> mManagedWeightsMap; -}; - -} // namespace tensorrt_llm::runtime diff --git a/cpp/include/tensorrt_llm/runtime/tllmLogger.h b/cpp/include/tensorrt_llm/runtime/tllmLogger.h deleted file mode 100644 index dd3806ec5242..000000000000 --- a/cpp/include/tensorrt_llm/runtime/tllmLogger.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <NvInferRuntime.h> - -namespace tensorrt_llm::runtime -{ - -class TllmLogger : public nvinfer1::ILogger -{ -public: - void log(Severity severity, nvinfer1::AsciiChar const* msg) noexcept override; - - Severity getLevel(); - - void setLevel(Severity level); -}; - -} // namespace tensorrt_llm::runtime diff --git a/cpp/include/tensorrt_llm/runtime/utils/debugUtils.h b/cpp/include/tensorrt_llm/runtime/utils/debugUtils.h index 68064d74c7e2..3f6c307a3cde 100644 --- a/cpp/include/tensorrt_llm/runtime/utils/debugUtils.h +++ b/cpp/include/tensorrt_llm/runtime/utils/debugUtils.h @@ -15,6 +15,7 @@ */ #pragma once +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" namespace tensorrt_llm::runtime::utils @@ -24,7 +25,7 @@ template <typename T> bool tensorHasInvalid(ITensor const& tensor, BufferManager const& manager, std::string const& infoStr); bool tensorHasInvalid( - size_t M, size_t K, nvinfer1::DataType type, void const* data, cudaStream_t stream, std::string const& infoStr); + size_t M, size_t K, tensorrt_llm::DataType type, void const* data, cudaStream_t stream, std::string const& infoStr); int stallStream( char const* name, std::optional<cudaStream_t> stream = std::nullopt, std::optional<int> delay = std::nullopt); diff --git a/cpp/include/tensorrt_llm/runtime/utils/mpiTags.h b/cpp/include/tensorrt_llm/runtime/utils/mpiTags.h index 32c086c84ee9..49c50e49b2e2 100644 --- a/cpp/include/tensorrt_llm/runtime/utils/mpiTags.h +++ b/cpp/include/tensorrt_llm/runtime/utils/mpiTags.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,7 +71,11 @@ enum class MpiTag : int // KvCacheEventManager kKvCacheEventSize = 1026, - kKvCacheEvent = 1027 + kKvCacheEvent = 1027, + + // Asynchronous context-transfer coordination. + kContextTransferEvent = 1028, + kContextTransferUpdate = 1029 }; } // namespace tensorrt_llm::mpi diff --git a/cpp/include/tensorrt_llm/runtime/utils/mpiUtils.h b/cpp/include/tensorrt_llm/runtime/utils/mpiUtils.h index 75ec7a534815..b474b4b12cbf 100644 --- a/cpp/include/tensorrt_llm/runtime/utils/mpiUtils.h +++ b/cpp/include/tensorrt_llm/runtime/utils/mpiUtils.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -235,6 +235,17 @@ class MpiRequest #endif } + [[nodiscard]] bool isCompleted() + { +#if ENABLE_MULTI_DEVICE + int completed = 0; + TLLM_MPI_CHECK(MPI_Test(&mRequest, &completed, MPI_STATUS_IGNORE)); + return completed != 0; +#else + TLLM_THROW("Multi device support is disabled."); +#endif + } + void cancel() { #if ENABLE_MULTI_DEVICE diff --git a/cpp/include/tensorrt_llm/runtime/worldConfig.h b/cpp/include/tensorrt_llm/runtime/worldConfig.h index 9ff2d0970df7..272b0fec5ada 100644 --- a/cpp/include/tensorrt_llm/runtime/worldConfig.h +++ b/cpp/include/tensorrt_llm/runtime/worldConfig.h @@ -18,7 +18,6 @@ #include "tensorrt_llm/runtime/common.h" -#include <NvInferRuntime.h> #include <optional> #include <vector> diff --git a/cpp/micro_benchmarks/mixtureOfExpertsBackendBenchmarkFixture.h b/cpp/micro_benchmarks/mixtureOfExpertsBackendBenchmarkFixture.h index f466a65e871f..354e11184f8d 100644 --- a/cpp/micro_benchmarks/mixtureOfExpertsBackendBenchmarkFixture.h +++ b/cpp/micro_benchmarks/mixtureOfExpertsBackendBenchmarkFixture.h @@ -31,6 +31,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/cutlass_kernels/cutlass_preprocessors.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" @@ -380,44 +381,44 @@ class MixtureOfExpertsBenchmark : public ::benchmark::Fixture int64_t mNumExpertsPerNode{}; int64_t mK{}; - constexpr static nvinfer1::DataType toDTypeID() + constexpr static tensorrt_llm::DataType toDTypeID() { if (FP8 || WFP4AFP8) - return nvinfer1::DataType::kFP8; + return tensorrt_llm::DataType::kFP8; if (NVFP4) - return nvinfer1::DataType::kFP4; + return tensorrt_llm::DataType::kFP4; if (INT_QUANT && INT4) - return nvinfer1::DataType::kINT4; + return tensorrt_llm::DataType::kINT4; if (INT_QUANT) - return nvinfer1::DataType::kINT8; + return tensorrt_llm::DataType::kINT8; if (std::is_same_v<DataType, float>) - return nvinfer1::DataType::kFLOAT; + return tensorrt_llm::DataType::kFLOAT; if (std::is_same_v<DataType, half>) - return nvinfer1::DataType::kHALF; + return tensorrt_llm::DataType::kHALF; #ifdef ENABLE_BF16 if (std::is_same_v<DataType, nv_bfloat16>) - return nvinfer1::DataType::kBF16; + return tensorrt_llm::DataType::kBF16; #endif TLLM_THROW("Unrecognised format"); }; - constexpr static nvinfer1::DataType toWTypeID() + constexpr static tensorrt_llm::DataType toWTypeID() { if (FP8) - return nvinfer1::DataType::kFP8; + return tensorrt_llm::DataType::kFP8; if (NVFP4 || WFP4AFP8) - return nvinfer1::DataType::kFP4; + return tensorrt_llm::DataType::kFP4; if (INT_QUANT && INT4) - return nvinfer1::DataType::kINT4; + return tensorrt_llm::DataType::kINT4; if (INT_QUANT) - return nvinfer1::DataType::kINT8; + return tensorrt_llm::DataType::kINT8; if (std::is_same_v<DataType, float>) - return nvinfer1::DataType::kFLOAT; + return tensorrt_llm::DataType::kFLOAT; if (std::is_same_v<DataType, half>) - return nvinfer1::DataType::kHALF; + return tensorrt_llm::DataType::kHALF; #ifdef ENABLE_BF16 if (std::is_same_v<DataType, nv_bfloat16>) - return nvinfer1::DataType::kBF16; + return tensorrt_llm::DataType::kBF16; #endif TLLM_THROW("Unrecognised format"); }; @@ -427,31 +428,31 @@ class MixtureOfExpertsBenchmark : public ::benchmark::Fixture { if constexpr (std::is_same_v<T, SafeFP8>) { - return nvinfer1::DataType::kFP8; + return tensorrt_llm::DataType::kFP8; } else if constexpr (std::is_same_v<T, SafeFP4>) { - return nvinfer1::DataType::kFP4; + return tensorrt_llm::DataType::kFP4; } else if constexpr (std::is_same_v<T, uint8_t>) { - return nvinfer1::DataType::kINT8; + return tensorrt_llm::DataType::kINT8; } else if constexpr (std::is_same_v<T, cutlass::uint4b_t>) { - return nvinfer1::DataType::kINT4; + return tensorrt_llm::DataType::kINT4; } else if constexpr (std::is_same_v<T, nv_bfloat16>) { - return nvinfer1::DataType::kBF16; + return tensorrt_llm::DataType::kBF16; } else if constexpr (std::is_same_v<T, half>) { - return nvinfer1::DataType::kHALF; + return tensorrt_llm::DataType::kHALF; } else if constexpr (std::is_same_v<T, float>) { - return nvinfer1::DataType::kFLOAT; + return tensorrt_llm::DataType::kFLOAT; } else { diff --git a/cpp/tensorrt_llm/CMakeLists.txt b/cpp/tensorrt_llm/CMakeLists.txt index 2ae0b74335f6..d9a7853bc9d5 100644 --- a/cpp/tensorrt_llm/CMakeLists.txt +++ b/cpp/tensorrt_llm/CMakeLists.txt @@ -144,8 +144,6 @@ add_subdirectory(common) add_subdirectory(kernels) add_subdirectory(layers) add_subdirectory(runtime) -add_subdirectory(testing) -add_subdirectory(executor_worker) set(BATCH_MANAGER_TARGET tensorrt_llm_batch_manager_static) set(BATCH_MANAGER_TARGET_ARCH ${TARGET_ARCH}) @@ -176,7 +174,6 @@ set(TRTLLM_LINK_LIBS ${CUBLASLT_LIB} ${CURAND_LIB} ${CMAKE_DL_LIBS} - ${TRT_LIB} common_src kernels_src flash_mla_src @@ -201,7 +198,6 @@ set(TRTLLM_LINK_LIBS cute_dsl_src layers_src runtime_src - testing_src compressorKernels_src mhcKernels_src userbuffers_src @@ -312,5 +308,3 @@ endif() if(BUILD_FLASH_MLA) add_subdirectory(flash_mla) endif() - -add_subdirectory(plugins) diff --git a/cpp/tensorrt_llm/batch_manager/CMakeLists.txt b/cpp/tensorrt_llm/batch_manager/CMakeLists.txt index 88e2484cc6f1..f61e58e16b28 100644 --- a/cpp/tensorrt_llm/batch_manager/CMakeLists.txt +++ b/cpp/tensorrt_llm/batch_manager/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & +# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & # AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -30,36 +30,23 @@ set(SRCS capacityScheduler.cpp createNewDecoderRequests.cpp contextProgress.cpp + contextTransferCoordinator.cpp dataTransceiver.cpp decoderBuffers.cpp - encoderBuffers.cpp - guidedDecoder.cpp - handleContextLogits.cpp - handleGenerationLogits.cpp kvCacheManager.cpp kvCacheEventManager.cpp kvCacheTransferManager.cpp kvCacheManagerV2Utils.cpp kvCacheManagerV2Utils.cu llmRequest.cpp - logitsPostProcessor.cpp - loraBuffers.cpp - makeDecodingBatchInputOutput.cpp medusaBuffers.cpp microBatchScheduler.cpp pauseRequests.cpp peftCacheManager.cpp - promptTuningBuffers.cpp - rnnStateBuffers.cpp rnnStateManager.cpp rnnCacheFormatter.cpp rnnCacheTransBuffer.cpp - runtimeBuffers.cpp sequenceSlotManager.cpp - transformerBuffers.cpp - trtEncoderModel.cpp - trtGptModelInflightBatching.cpp - updateDecoderBuffers.cpp utils/debugUtils.cpp utils/inflightBatchingUtils.cpp utils/logitsThread.cpp @@ -89,12 +76,18 @@ else() # Windows set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") endif() +# Include kv_cache_manager_v2 C++ sources. +include(${CMAKE_CURRENT_SOURCE_DIR}/kv_cache_manager_v2/CMakeLists.txt) +list(APPEND SRCS ${KV_CACHE_MANAGER_V2_SRCS}) + add_library(${BATCH_MANAGER_STATIC_TARGET} STATIC ${SRCS}) target_include_directories( ${BATCH_MANAGER_STATIC_TARGET} PUBLIC ${xgrammar_source_dir}/3rdparty/picojson ${xgrammar_source_dir}/3rdparty/dlpack/include - ${xgrammar_source_dir}/include) + ${xgrammar_source_dir}/include + ${KV_CACHE_MANAGER_V2_INCLUDE_DIR} + ${PROJECT_SOURCE_DIR}/tensorrt_llm/common/sha256) set_target_properties( ${BATCH_MANAGER_STATIC_TARGET} diff --git a/cpp/tensorrt_llm/batch_manager/baseTransBuffer.cpp b/cpp/tensorrt_llm/batch_manager/baseTransBuffer.cpp index cbb8159bce44..1fb522afd7be 100644 --- a/cpp/tensorrt_llm/batch_manager/baseTransBuffer.cpp +++ b/cpp/tensorrt_llm/batch_manager/baseTransBuffer.cpp @@ -20,6 +20,7 @@ #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/opUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include <chrono> #include <exception> @@ -98,7 +99,7 @@ void BufferIndexHolder::poison() noexcept } BaseTransBufferManager::BaseTransBufferManager( - size_t transferBufferSize, nvinfer1::DataType dataType, std::optional<size_t> maxNumTokens) + size_t transferBufferSize, tensorrt_llm::DataType dataType, std::optional<size_t> maxNumTokens) : mDataType{dataType} , mBufferManager{std::make_shared<runtime::CudaStream>()} , mMaxNumTokens{maxNumTokens} diff --git a/cpp/tensorrt_llm/batch_manager/baseTransBuffer.h b/cpp/tensorrt_llm/batch_manager/baseTransBuffer.h index 8c1fd313f5a6..88585818d4e5 100644 --- a/cpp/tensorrt_llm/batch_manager/baseTransBuffer.h +++ b/cpp/tensorrt_llm/batch_manager/baseTransBuffer.h @@ -17,6 +17,7 @@ #pragma once +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -225,7 +226,7 @@ class BaseTransBufferManager /// @param dataType Data type for the buffers. /// @param maxNumTokens Optional max tokens for sizing. BaseTransBufferManager( - size_t transferBufferSize, nvinfer1::DataType dataType, std::optional<size_t> maxNumTokens = std::nullopt); + size_t transferBufferSize, tensorrt_llm::DataType dataType, std::optional<size_t> maxNumTokens = std::nullopt); struct ConcurrenceResource { @@ -256,7 +257,7 @@ class BaseTransBufferManager bool mOnlyUseDynamicBuffer; bool mUseFabricMemory; size_t mNumberOfElements; - nvinfer1::DataType mDataType; + tensorrt_llm::DataType mDataType; ConcurrenceResource mConcurrenceSendResource; ConcurrenceResource mConcurrenceRecvResource; runtime::BufferManager mBufferManager; diff --git a/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp b/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp index 9dc44531d409..39cd7cc0f8de 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp @@ -183,7 +183,7 @@ void sendAllBuffers(TransferSession& session, int deviceId, namespace tensorrt_llm::batch_manager::kv_cache_manager { -BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, LlmRequest const& llmRequest, +BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, std::optional<LlmRequest const*> llmRequest, BlockKey const& lastBlockKey, int32_t indexFromEnd, bool recvSideHasCP, SizeType32 ppSize) { auto poolNum = cacheManager->getBlockManager().getNumPools( @@ -197,9 +197,10 @@ BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, LlmRequest || lastBlockKey.uniqueTokens.size() == 0 || recvSideHasCP || ppSize > 1) { // disable reuse path, and vwsa don't support reuse. + TLLM_CHECK_WITH_INFO(llmRequest.has_value(), "LlmRequest required for non-reuse-tree transfer path"); bool needSendAllForWindow = common::getEnvKVCacheTransferAllBlocksForWindow(); - auto blockRange = BlockRange::fromAllBlockIds(*cacheManager, llmRequest.mRequestId); + auto blockRange = BlockRange::fromAllBlockIds(*cacheManager, (*llmRequest)->mRequestId); auto const& windowsMetadata = cacheManager->getBlockManager().getWindowSizesMetadata(); @@ -235,16 +236,20 @@ BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, LlmRequest TLLM_CHECK_WITH_INFO(lastBlockKey.uniqueTokens.size() > 0, "lastBlockKey must be non-empty when reuse is enabled"); - auto multimodalHashes = llmRequest.getMultimodalHashes(); - bool isMultimodal = multimodalHashes.has_value() && *multimodalHashes && !(*multimodalHashes)->empty(); - if (isMultimodal) + // No request on the reuse-tree path: fall through to the plain lastBlockKey lookup. + if (llmRequest.has_value()) { - auto tokensPerBlock = cacheManager->getBlockManager().getTokensPerBlock(); - auto const usableSize = static_cast<SizeType32>(lastBlockKey.uniqueTokens.size()); - auto blockedUniqueTokens = chopVectorIntoBlocks<UniqueToken>( - lastBlockKey.uniqueTokens, usableSize, tokensPerBlock, /*allowPartial=*/true); - auto blockKeys = buildBlockKeys(blockedUniqueTokens, llmRequest); - return BlockRange::fromReuseTree(*cacheManager, blockKeys, indexFromEnd); + auto multimodalHashes = (*llmRequest)->getMultimodalHashes(); + bool isMultimodal = multimodalHashes.has_value() && *multimodalHashes && !(*multimodalHashes)->empty(); + if (isMultimodal) + { + auto tokensPerBlock = cacheManager->getBlockManager().getTokensPerBlock(); + auto const usableSize = static_cast<SizeType32>(lastBlockKey.uniqueTokens.size()); + auto blockedUniqueTokens = chopVectorIntoBlocks<UniqueToken>( + lastBlockKey.uniqueTokens, usableSize, tokensPerBlock, /*allowPartial=*/true); + auto blockKeys = buildBlockKeys(blockedUniqueTokens, **llmRequest); + return BlockRange::fromReuseTree(*cacheManager, blockKeys, indexFromEnd); + } } return BlockRange::fromReuseTree(*cacheManager, lastBlockKey, indexFromEnd); @@ -363,11 +368,15 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio { NVTX3_SCOPED_RANGE(CacheFormatter_format); session.setTime(TransferSession::kTimeFormatter); - auto const& llmRequest = session.getLlmRequest(); - TLLM_LOG_DEBUG( - mpi::MpiComm::world().getRank(), "Start sending KV cache for request ID: %ld.", llmRequest.mRequestId); + auto llmRequest = session.getLlmRequest(); + if (llmRequest.has_value()) + { + TLLM_LOG_DEBUG( + mpi::MpiComm::world().getRank(), "Start sending KV cache for request ID: %ld.", (*llmRequest)->mRequestId); + TLLM_CHECK_WITH_INFO( + (*llmRequest)->mSamplingConfig.beamWidth == 1, "Currently, only beam width 1 is supported."); + } - TLLM_CHECK_WITH_INFO(llmRequest.mSamplingConfig.beamWidth == 1, "Currently, only beam width 1 is supported."); auto const& connections = session.getConnections(); auto const& selfConfig = session.getSelfState().getCacheState().value(); auto const& destConfig = session.getOtherState().getCacheState().value(); @@ -385,7 +394,6 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio size_t targetNum = pickUpConnections.size(); if (targetNum == 0) { - TLLM_LOG_DEBUG("No targets to send KV cache to for request ID: %ld", llmRequest.mRequestId); return; } @@ -413,13 +421,15 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio SizeType32 const numKvPools = static_cast<SizeType32>(kvWindowSizes.size()); - TLLM_LOG_DEBUG("CacheFormatter::format: allWindowSizes=%zu, kvWindowSizes=%d, numPools=%d, requestId=%lu", - allWindowSizes.size(), numKvPools, numPools, llmRequest.mRequestId); + TLLM_LOG_DEBUG("CacheFormatter::format: allWindowSizes=%zu, kvWindowSizes=%d, numPools=%d, requestId=%s", + allWindowSizes.size(), numKvPools, numPools, + llmRequest.has_value() ? std::to_string((*llmRequest)->mRequestId).c_str() : "<request-free>"); bool layerWise = common::getEnvDisaggLayerwise() && numKvPools == 1; if (layerWise) { - auto& progress = llmRequest.getContextProgress(); + TLLM_CHECK_WITH_INFO(llmRequest.has_value(), "LlmRequest required for layer-wise transfer"); + auto& progress = (*llmRequest)->getContextProgress(); SizeType32 const numLayers = blockManager.getNumLayers(); runtime::ITensor::Shape offset = runtime::ITensor::makeShape({0, 0}); for (SizeType32 layerIdx = 0; layerIdx < numLayers; layerIdx++) @@ -515,8 +525,11 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio } } } - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", - llmRequest.mRequestId); + if (llmRequest.has_value()) + { + TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", + (*llmRequest)->mRequestId); + } return; } @@ -635,15 +648,20 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio sendHolder.release(); session.setTime(TransferSession::kTimePostprocess); } - TLLM_LOG_DEBUG( - mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID:%ld ", llmRequest.mRequestId); + if (llmRequest.has_value()) + { + TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID:%ld ", + (*llmRequest)->mRequestId); + } } void CacheFormatter::unformat(tensorrt_llm::batch_manager::TransferSession& session) { NVTX3_SCOPED_RANGE(CacheFormatter_unformat); session.setTime(TransferSession::kTimeFormatter); - auto const& llmRequest = session.getLlmRequest(); + auto llmRequestOpt = session.getLlmRequest(); + TLLM_CHECK_WITH_INFO(llmRequestOpt.has_value(), "LlmRequest required for receiving KV cache"); + auto const& llmRequest = **llmRequestOpt; auto const ctxReqId = llmRequest.getContextPhaseParams().value().getReqId(); TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "Start receiving KV cache for request ID: %ld, context request ID: %ld.", llmRequest.mRequestId, ctxReqId); @@ -729,12 +747,7 @@ void CacheFormatter::unformat(tensorrt_llm::batch_manager::TransferSession& sess { NVTX3_SCOPED_RANGE(formatInputRecvBuffer); - // TODO(disagg-multi-dtype): pool 0's dtype is treated as canonical for the wire - // transport here. Pools with differing dtypes are rejected up-front in - // CacheTransBufferManager's constructor (see cacheTransBuffer.cpp). When - // per-pool dtype dispatch lands, this single dataType variable must be replaced - // with a per-pool lookup keyed by the source pool of each block. - auto dataType = mCacheManager->getPrimaryPool(0)->getDataType(); + auto const dataType = mCacheTransBufferManager->getDataType(); bool layerWise = common::getEnvDisaggLayerwise() && numKvPools == 1; if (layerWise) { diff --git a/cpp/tensorrt_llm/batch_manager/cacheFormatter.h b/cpp/tensorrt_llm/batch_manager/cacheFormatter.h index 458cac8d4382..21356f3dd6c0 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheFormatter.h +++ b/cpp/tensorrt_llm/batch_manager/cacheFormatter.h @@ -28,7 +28,6 @@ #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include <NvInferRuntimeBase.h> #include <cstddef> #include <cstdint> #include <fstream> @@ -177,7 +176,7 @@ inline std::pair<std::vector<size_t>, std::vector<size_t>> pickRecvConnections(s namespace tensorrt_llm::batch_manager::kv_cache_manager { -BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, LlmRequest const& llmRequest, +BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, std::optional<LlmRequest const*> llmRequest, BlockKey const& lastBlockKey, SizeType32 indexFromEnd, bool recvSideHasCP = false, SizeType32 ppSize = 1); using DataContext = tensorrt_llm::executor::kv_cache::DataContext; diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp b/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp index 772c9555f0f3..1e8bbcb766bd 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,12 +21,78 @@ #include "tensorrt_llm/common/opUtils.h" #include "tensorrt_llm/executor/executor.h" -#include <NvInferRuntimeBase.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <mutex> namespace tensorrt_llm::batch_manager::kv_cache_manager { +namespace +{ + +bool isCachePool(BlockManager const& blockManager, SizeType32 poolIdx) +{ + auto const& pool = blockManager.getPool(poolIdx); + return !pool.containsBlockScales && !pool.containsIndexerKCache; +} + +bool isAttentionCachePool(BlockManager const& blockManager, SizeType32 poolIdx) +{ + return isCachePool(blockManager, poolIdx) + && !LinearAttentionMetadata::hasLinearCache(blockManager.getPoolWindowSize(poolIdx)); +} + +tensorrt_llm::DataType getTransferDataType(KVCacheManager::BaseKVCacheManager* cacheManager, bool transferIndexerKCache) +{ + TLLM_CHECK(cacheManager); + if (transferIndexerKCache) + { + auto const indexerKCachePool = cacheManager->getIndexerKCachePool(); + TLLM_CHECK(indexerKCachePool); + return indexerKCachePool->getDataType(); + } + + auto const& blockManager = cacheManager->getBlockManager(); + std::optional<tensorrt_llm::DataType> cacheDataType; + std::optional<tensorrt_llm::DataType> attentionDataType; + SizeType32 firstPoolIdx = -1; + // Recurrent-state pools have a separate transfer manager and formatter. Only + // attention pools determine the KV transfer-buffer dtype. + for (SizeType32 poolIdx = 0; poolIdx < blockManager.getNumPools(); ++poolIdx) + { + if (!isCachePool(blockManager, poolIdx)) + { + continue; + } + + auto const poolDataType = blockManager.getPrimaryPool(poolIdx)->getDataType(); + if (!cacheDataType.has_value()) + { + cacheDataType = poolDataType; + } + if (!isAttentionCachePool(blockManager, poolIdx)) + { + continue; + } + if (!attentionDataType.has_value()) + { + attentionDataType = poolDataType; + firstPoolIdx = poolIdx; + continue; + } + + TLLM_CHECK_WITH_INFO(poolDataType == attentionDataType.value(), + "Disaggregated KV cache transfer does not yet support attention pools with differing dtypes " + "(pool %d dtype=%d, pool %d dtype=%d). TODO(disagg-multi-dtype): per-pool dtype dispatch in formatter.", + firstPoolIdx, static_cast<int>(attentionDataType.value()), poolIdx, static_cast<int>(poolDataType)); + } + + TLLM_CHECK_WITH_INFO(cacheDataType.has_value(), "Disaggregated KV cache transfer requires a cache pool"); + return attentionDataType.value_or(cacheDataType.value()); +} + +} // namespace + // ============================================================================ // FabricMemory Implementation // ============================================================================ @@ -194,39 +260,38 @@ bool FabricMemory::supportFabricMemory() size_t CacheTransBufferManager::computeTransferBufferSize( KVCacheManager::BaseKVCacheManager* cacheManager, std::optional<size_t> maxNumTokens, bool transferIndexerKCache) { - nvinfer1::DataType dataType; - if (transferIndexerKCache) - { - dataType = cacheManager->getIndexerKCachePool()->getDataType(); - } - else + auto const dataType = getTransferDataType(cacheManager, transferIndexerKCache); + + auto const& blockManager = cacheManager->getBlockManager(); + auto const tokensPerBlock = blockManager.getTokensPerBlock(); + bool hasAttentionCachePool = false; + for (SizeType32 poolIdx = 0; poolIdx < blockManager.getNumPools(); ++poolIdx) { - dataType = cacheManager->getPrimaryPool(0)->getDataType(); + hasAttentionCachePool |= isAttentionCachePool(blockManager, poolIdx); } - - auto tokensPerBlock = cacheManager->getBlockManager().getTokensPerBlock(); size_t bufferSizeFromMaxNumToken = 0; if (maxNumTokens.has_value()) { TLLM_CHECK(maxNumTokens.value() % tokensPerBlock == 0); - auto dataSize = common::getDTypeSize(dataType); - SizeType32 kvCacheByteSizePerTokenPerLayer = 0; + auto const dataSize = common::getDTypeSize(dataType); + SizeType32 indexerCacheByteSizePerTokenPerLayer = 0; if (transferIndexerKCache) { - kvCacheByteSizePerTokenPerLayer + indexerCacheByteSizePerTokenPerLayer = cacheManager->getIndexerKCachePool()->getDimension<-1>() * dataSize / tokensPerBlock; } - else - { - auto primaryPool = cacheManager->getPrimaryPool(0); - kvCacheByteSizePerTokenPerLayer - = primaryPool->getDimension<-1>() * primaryPool->getDimension<2>() * dataSize / tokensPerBlock; - } - for (auto layerId = 0; layerId < cacheManager->getBlockManager().getNumLayers(); layerId++) + for (auto layerId = 0; layerId < blockManager.getNumLayers(); layerId++) { - auto poolIdx = cacheManager->getBlockManager().getLayerPoolIdx(layerId); - auto windowSize = static_cast<size_t>(cacheManager->getBlockManager().getPoolWindowSize(poolIdx)); + auto const poolIdx = blockManager.getLayerPoolIdx(layerId); + auto const encodedWindowSize = blockManager.getPoolWindowSize(poolIdx); + if (!transferIndexerKCache && hasAttentionCachePool + && LinearAttentionMetadata::hasLinearCache(encodedWindowSize)) + { + continue; + } + + auto const windowSize = static_cast<size_t>(encodedWindowSize); auto alignedWindowSize = (windowSize + tokensPerBlock - 1) / tokensPerBlock * tokensPerBlock; auto validTokenNum = (alignedWindowSize < maxNumTokens.value() ? alignedWindowSize : maxNumTokens.value()); if (common::getEnvKVCacheTransferAllBlocksForWindow()) @@ -235,7 +300,17 @@ size_t CacheTransBufferManager::computeTransferBufferSize( } validTokenNum += tokensPerBlock; // add one more block - bufferSizeFromMaxNumToken += validTokenNum * kvCacheByteSizePerTokenPerLayer; + if (transferIndexerKCache) + { + bufferSizeFromMaxNumToken += validTokenNum * indexerCacheByteSizePerTokenPerLayer; + } + else + { + auto const primaryPool = blockManager.getPrimaryPool(poolIdx); + auto const kvCacheByteSizePerTokenPerLayer + = primaryPool->getDimension<-1>() * primaryPool->getDimension<2>() * dataSize / tokensPerBlock; + bufferSizeFromMaxNumToken += validTokenNum * kvCacheByteSizePerTokenPerLayer; + } } } @@ -245,36 +320,12 @@ size_t CacheTransBufferManager::computeTransferBufferSize( CacheTransBufferManager::CacheTransBufferManager( KVCacheManager::BaseKVCacheManager* cacheManager, std::optional<size_t> maxNumTokens, bool transferIndexerKCache) : BaseTransBufferManager(computeTransferBufferSize(cacheManager, maxNumTokens, transferIndexerKCache), - transferIndexerKCache ? cacheManager->getIndexerKCachePool()->getDataType() - : cacheManager->getPrimaryPool(0)->getDataType(), - maxNumTokens) + getTransferDataType(cacheManager, transferIndexerKCache), maxNumTokens) , mCacheManager{cacheManager} , mTransferIndexerKCache{transferIndexerKCache} { // TODO: FP4 dataSize TLLM_CHECK(mCacheManager); - // TODO(disagg-multi-dtype): Per-pool dtype dispatch in formatter / transfer buffer - // not yet implemented. Disagg currently picks pool 0's dtype as the canonical - // transport type (above), so any KV pool with a different dtype would be silently - // miscoerced on the wire. Fail loudly until per-pool dispatch lands. We restrict - // the comparison to KV pools (getNumPools(false, false)) since block-scale and - // indexer-K pools legitimately have their own dtypes and travel through their own - // code paths. - if (!transferIndexerKCache) - { - auto const numKvPools = mCacheManager->getBlockManager().getNumPools( - /*includeBlockScalePools=*/false, /*includeIndexerKCachePools=*/false); - auto const dtype0 = mCacheManager->getPrimaryPool(0)->getDataType(); - for (SizeType32 i = 1; i < numKvPools; ++i) - { - auto const dtypeI = mCacheManager->getPrimaryPool(i)->getDataType(); - TLLM_CHECK_WITH_INFO(dtypeI == dtype0, - "Disaggregated KV cache transfer does not yet support pools with differing dtypes " - "(pool 0 dtype=%d, pool %d dtype=%d). TODO(disagg-multi-dtype): per-pool dtype " - "dispatch in formatter.", - static_cast<int>(dtype0), i, static_cast<int>(dtypeI)); - } - } TLLM_LOG_INFO("CacheTransBufferManager created for KV cache"); } diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.h b/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.h index 1635c11bc673..72f67e8ca3c9 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.h +++ b/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -74,6 +74,12 @@ class CacheTransBufferManager : public BaseTransBufferManager return mCacheManager; } + /// @brief Get the data type used by KV cache transfer buffers. + [[nodiscard]] tensorrt_llm::DataType getDataType() const noexcept + { + return mDataType; + } + [[nodiscard]] BufferKind getBufferKind() const override { return mTransferIndexerKCache ? BufferKind::kKV_INDEXER : BufferKind::kKV; diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp b/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp index aa05cd033892..1c65405aae4a 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp @@ -36,6 +36,7 @@ #include "tensorrt_llm/batch_manager/cacheFormatter.h" #include "tensorrt_llm/batch_manager/cacheTransceiver.h" #include "tensorrt_llm/batch_manager/contextProgress.h" +#include "tensorrt_llm/batch_manager/contextTransferCoordinator.h" #include "tensorrt_llm/batch_manager/dataTransceiver.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/kvCacheType.h" @@ -47,15 +48,24 @@ #include "tensorrt_llm/batch_manager/rnnStateManager.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/mpi_utils/connection.h" #include "tensorrt_llm/executor/dataTransceiverState.h" +#include "tensorrt_llm/executor/serialization.h" #include "tensorrt_llm/executor/serializeUtils.h" +#include "tensorrt_llm/executor/transferAgent.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/pgUtils.h" #include <algorithm> #include <chrono> #include <cstddef> +#include <filesystem> +#include <fstream> +#include <iomanip> #include <numeric> +#include <random> +#include <sstream> +#include <system_error> #include <thread> #include <unordered_map> #include <unordered_set> @@ -63,6 +73,28 @@ namespace tensorrt_llm::batch_manager { +namespace +{ + +/// Generate a UUID-like hex string (e.g. "a1b2c3d4-e5f6-7890-abcd-ef1234567890") +/// to uniquely identify a CacheTransceiver instance across gen instances. +std::string generateInstanceId() +{ + // The RNG state is comparatively expensive to construct/seed, so keep one + // per thread instead of building it on every call. + static thread_local std::mt19937_64 gen{std::random_device{}()}; + std::uniform_int_distribution<uint64_t> dis; + uint64_t a = dis(gen); + uint64_t b = dis(gen); + std::ostringstream oss; + oss << std::hex << std::setfill('0') << std::setw(8) << (a >> 32) << "-" << std::setw(4) << ((a >> 16) & 0xFFFF) + << "-" << std::setw(4) << (a & 0xFFFF) << "-" << std::setw(4) << (b >> 48) << "-" << std::setw(12) + << (b & 0xFFFFFFFFFFFF); + return oss.str(); +} + +} // anonymous namespace + std::mutex CacheTransceiver::mDllMutex; namespace @@ -72,6 +104,20 @@ using RequestIdType = LlmRequest::RequestIdType; constexpr int kTransferFuturePollIntervalMs = 10; +char const* cacheTransceiverBackendName(executor::CacheTransceiverConfig::BackendType backendType) +{ + using BackendType = executor::CacheTransceiverConfig::BackendType; + switch (backendType) + { + case BackendType::DEFAULT: return "DEFAULT"; + case BackendType::MPI: return "MPI"; + case BackendType::UCX: return "UCX"; + case BackendType::NIXL: return "NIXL"; + case BackendType::MOONCAKE: return "MOONCAKE"; + } + return "UNKNOWN"; +} + // Finite status checks are scheduler polls, not terminal deadlines. Pure polls // use short slices; calls that ask for at least one completion keep bounded // backpressure by waiting up to the configured future timeout. @@ -344,7 +390,7 @@ std::unique_ptr<BaseCacheTransceiver> CacheTransceiverFactory::createCacheTransc CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, executor::kv_cache::CacheState::ModelConfig const& cacheStateModelCfg, runtime::WorldConfig const& worldConfig, - std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, executor::kv_cache::CacheState::AttentionType attentionType, std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig, std::vector<SizeType32> const& rnnLayerNumPerPP) @@ -384,6 +430,89 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa mGroupComm = std::make_shared<CacheTransceiverComm>(tensorrt_llm::pg_utils::get_world_pg()); } + // Generate instance ID on rank 0 and broadcast to all ranks in the session + // so every rank in the same gen/ctx instance shares the same ID. + { + if (mGroupComm->getRank() == 0) + { + mInstanceId = generateInstanceId(); + } + if (useMPI()) + { + int len = static_cast<int>(mInstanceId.size()); + tensorrt_llm::mpi::MpiComm::session().bcast(&len, 1, mpi::MpiType::kINT32, 0); + mInstanceId.resize(len); + tensorrt_llm::mpi::MpiComm::session().bcast(mInstanceId.data(), len, mpi::MpiType::kCHAR, 0); + } + else + { + // PG path: rank 0 sends via allgather, others receive. + constexpr int kUuidLen = 36; + std::vector<char> sendBuf(kUuidLen, '\0'); + if (mGroupComm->getRank() == 0) + { + std::copy_n(mInstanceId.begin(), std::min<size_t>(mInstanceId.size(), kUuidLen), sendBuf.begin()); + } + std::vector<char> recvBuf(kUuidLen * mGroupComm->getSize(), '\0'); + mGroupComm->allgather(std::ref(sendBuf), std::ref(recvBuf), {}); + // Take rank 0's segment. + mInstanceId = std::string(recvBuf.begin(), recvBuf.begin() + kUuidLen); + } + } + + // Calibrate steady_clock across ranks so that cross-node allgather + // in batchUpdateKVCacheTransferBW can compare time points. + // globalSteadyClockOffset() reads a single process-global copy shared with + // the nanobind module, so if the Python runtime already calibrated the offset + // (PyExecutor::_set_global_steady_clock_offset) it is visible here and we skip; + // the pure-C++ path performs the calibration below. + // The check-and-set is guarded by a mutex so that CacheTransceiver instances + // constructed concurrently in the same process (e.g. multi-engine serving) do + // not race on the shared offset or issue mismatched collectives. + { + static std::mutex sSteadyClockCalibrationMutex; + std::lock_guard<std::mutex> lock(sSteadyClockCalibrationMutex); + if (!globalSteadyClockOffset().has_value()) + { + using Duration = LlmRequest::Duration; + // Synchronize all ranks immediately before sampling the local clock so + // every rank measures from a consistent point. + if (useMPI()) + { + tensorrt_llm::mpi::MpiComm::session().barrier(); + } + else + { + // CacheTransceiverComm exposes no barrier primitive, so use a cheap + // allgather as a pseudo-barrier for the process-group path. + int64_t const dummy = 0; + std::vector<int64_t> dummyRecv(mGroupComm->getSize(), 0); + mGroupComm->allgather(dummy, std::ref(dummyRecv), {}); + } + auto localNow = std::chrono::steady_clock::now(); + auto localNs = std::chrono::duration_cast<std::chrono::nanoseconds>(localNow.time_since_epoch()).count(); + + // Allgather timestamps from all ranks + std::vector<int64_t> allNs(mGroupComm->getSize(), 0); + if (useMPI()) + { + tensorrt_llm::mpi::MpiComm::session().allgather(&localNs, allNs.data(), 1, mpi::MpiType::kINT64); + } + else + { + mGroupComm->allgather(localNs, std::ref(allNs), {}); + } + + // Offset = rank0's timestamp - my timestamp (same formula as Python) + auto offsetNs = allNs[0] - localNs; + globalSteadyClockOffset() = Duration(offsetNs); + + TLLM_LOG_INFO(mGroupComm->getRank(), + "CacheTransceiver: set global steady clock offset = %.6f sec for rank %d", + static_cast<double>(offsetNs) / 1e9, mGroupComm->getRank()); + } + } + if (worldConfig.isTensorParallel() || worldConfig.isContextParallel()) { mGroupTensorParaComm = std::make_shared<CacheTransceiverComm>( @@ -457,20 +586,20 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa // Pool dtype is UINT8 (raw byte storage), so we cannot use pool->getDataType(). // Only the byte size matters for split/concat kernel stride calculations — the actual // dtype enum is not interpreted numerically, just used for getDTypeSize() dispatch. - auto dtypeFromSize = [](SizeType32 size) -> nvinfer1::DataType + auto dtypeFromSize = [](SizeType32 size) -> tensorrt_llm::DataType { switch (size) { - case 4: return nvinfer1::DataType::kFLOAT; - case 2: return nvinfer1::DataType::kBF16; - case 1: return nvinfer1::DataType::kFP8; + case 4: return tensorrt_llm::DataType::kFLOAT; + case 2: return tensorrt_llm::DataType::kBF16; + case 1: return tensorrt_llm::DataType::kFP8; default: TLLM_THROW("Unsupported RNN state dtype size: %d", size); } }; TLLM_CHECK_WITH_INFO(linearMeta->rnnSsmDtypeSize > 0, "rnnSsmDtypeSize not set in LinearAttentionMetadata"); TLLM_CHECK_WITH_INFO(linearMeta->rnnConvDtypeSize > 0, "rnnConvDtypeSize not set in LinearAttentionMetadata"); - nvinfer1::DataType ssmDtype = dtypeFromSize(linearMeta->rnnSsmDtypeSize); - nvinfer1::DataType convDtype = dtypeFromSize(linearMeta->rnnConvDtypeSize); + tensorrt_llm::DataType ssmDtype = dtypeFromSize(linearMeta->rnnSsmDtypeSize); + tensorrt_llm::DataType convDtype = dtypeFromSize(linearMeta->rnnConvDtypeSize); mCacheState->setRnnConfig(rnnModelCfg, rnnLayerNumPerPP, convDtype, ssmDtype); // Create RnnCacheTransBufferManager for unified pool path. @@ -498,6 +627,7 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa if (backendType.value() == executor::CacheTransceiverConfig::BackendType::UCX) { std::lock_guard<std::mutex> lock(mDllMutex); + executor::kv_cache::promoteHostLibraryToGlobalScope(); mWrapperLibHandle = dllOpen(UCX_WRAPPER_LIB_NAME); TLLM_CHECK_WITH_INFO( mWrapperLibHandle != nullptr, "UCX wrapper library is not open correctly. error : %s", dlerror()); @@ -565,8 +695,38 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa auto makeCacheTransferLayer = [&]() { return CacheTransferLayer(*mCacheState, makeFormatter(), makeRnnFormatter()); }; - mCacheSender = std::make_unique<CacheSender>(mManager.get(), worldConfig.getRank(), makeCacheTransferLayer()); - mCacheReceiver = std::make_unique<CacheReceiver>(mManager.get(), worldConfig.getRank(), makeCacheTransferLayer()); + mCacheSender + = std::make_unique<CacheSender>(mManager.get(), worldConfig.getRank(), makeCacheTransferLayer(), mInstanceId); + mCacheReceiver + = std::make_unique<CacheReceiver>(mManager.get(), worldConfig.getRank(), makeCacheTransferLayer(), mInstanceId); + + // Keep automatic enablement within the currently qualified C++ NIXL/UCX TP1/CP1 pipeline topology. + bool const coordinatorTopologyEligible = worldConfig.getPipelineParallelism() > 1 && useMPI() + && backendType.value() == executor::CacheTransceiverConfig::BackendType::NIXL + && common::getEnvNixlBackend() == "UCX" && worldConfig.getTensorParallelism() == 1 + && worldConfig.getContextParallelism() == 1 && !mCacheState->getParallelConfig().mEnableAttentionDP; + if (worldConfig.getPipelineParallelism() > 1 && useMPI()) + { + TLLM_CHECK(mGroupPipeParaComm != nullptr); + constexpr std::uint64_t kCoordinatorProtocolVersion = 1; + std::uint64_t const localVersion = coordinatorTopologyEligible ? kCoordinatorProtocolVersion : 0; + bool const cancellationEnabled = common::getEnvDisaggEnableInflightCancel(); + std::uint64_t const localProtocolMode = (localVersion << 1) | static_cast<std::uint64_t>(cancellationEnabled); + std::vector<std::uint64_t> protocolModes(static_cast<std::size_t>(mGroupPipeParaComm->getSize())); + mGroupPipeParaComm->allgather(&localProtocolMode, protocolModes.data(), 1, mpi::MpiType::kUINT64); + TLLM_CHECK_WITH_INFO(std::all_of(protocolModes.begin(), protocolModes.end(), + [&](std::uint64_t const mode) { return mode == localProtocolMode; }), + "Context-transfer consensus protocol version or cancellation mode differs across PP ranks."); + if (localVersion != 0) + { + mContextTransferCoordinator = std::make_unique<ContextTransferCoordinator>(mGroupPipeParaComm); + TLLM_LOG_INFO( + "Enable asynchronous context-transfer consensus version %llu for PP group of size %d; in-flight " + "cancellation=%s.", + static_cast<unsigned long long>(kCoordinatorProtocolVersion), mGroupPipeParaComm->getSize(), + cancellationEnabled ? "enabled" : "disabled"); + } + } initializeCommState(); } @@ -577,6 +737,7 @@ CacheTransceiver::~CacheTransceiver() // plugin are still alive. The workers can access both during termination. mCacheSender.reset(); mCacheReceiver.reset(); + mContextTransferCoordinator.reset(); if (mWrapperLibHandle) { @@ -585,11 +746,89 @@ CacheTransceiver::~CacheTransceiver() } } +std::string CacheTransceiver::getStatusDump() const +{ + auto const backendType = mCacheTransceiverConfig->getBackendType().value(); + auto const requesterSyncActive = mSyncRequesterActive.load(std::memory_order_relaxed); + StatusSnapshot snapshot; + { + std::unique_lock<std::mutex> lock(mStatusSnapshotMutex, std::try_to_lock); + if (!lock.owns_lock()) + { + std::ostringstream oss; + oss << "KV cache transceiver | backend=" << cacheTransceiverBackendName(backendType) + << " | snapshot=unavailable | RX(sync_active=" << requesterSyncActive + << ") | poisoned=" << (hasPoisonedTransferBuffer() ? "yes" : "no"); + return oss.str(); + } + snapshot = mStatusSnapshot; + } + std::ostringstream oss; + oss << "KV cache transceiver | backend=" << cacheTransceiverBackendName(backendType) + << " | TX(async_active=" << snapshot.senderAsyncActive << ", timed_out=" << snapshot.timedOutSenders + << ", cancel_requested=" << snapshot.cancelingSenders << ", local_completed=" << snapshot.completedSenders + << ", local_failed=" << snapshot.failedSenders << ", awaiting_consensus=" << snapshot.sendersAwaitingConsensus + << ") | RX(async_active=" << snapshot.requesterAsyncActive << ", sync_active=" << requesterSyncActive + << ", timed_out=" << snapshot.timedOutRequesters << ", cancel_requested=" << snapshot.cancelingRequesters + << ", local_completed=" << snapshot.completedRequesters << ", local_failed=" << snapshot.failedRequesters + << ", awaiting_consensus=" << snapshot.requestersAwaitingConsensus + << ") | poisoned=" << (hasPoisonedTransferBuffer() ? "yes" : "no"); + return oss.str(); +} + +void CacheTransceiver::publishStatusSnapshot() noexcept +{ + StatusSnapshot snapshot; + snapshot.senderAsyncActive = mSenderFutures.size(); + snapshot.requesterAsyncActive = mRequesterFutures.size(); + snapshot.timedOutSenders = mTimedOutSenderIds.size(); + snapshot.timedOutRequesters = mTimedOutRequesterIds.size(); + snapshot.cancelingSenders = mCancelRequestedSenderIds.size(); + snapshot.cancelingRequesters = mCancelRequestedRequesterIds.size(); + snapshot.completedSenders = mCompletedSenderRequestIds.size(); + snapshot.completedRequesters = mCompletedRequesterRequestIds.size(); + snapshot.failedSenders = mFailedSenderRequestIds.size(); + snapshot.failedRequesters = mFailedRequesterRequestIds.size(); + snapshot.sendersAwaitingConsensus = mSenderRequestsAwaitingConsensus.size(); + snapshot.requestersAwaitingConsensus = mRequesterRequestsAwaitingConsensus.size(); + try + { + std::lock_guard<std::mutex> lock(mStatusSnapshotMutex); + mStatusSnapshot = snapshot; + } + catch (std::system_error const&) + { + // Status publication is best-effort and must never fail a transfer path. + } +} + +CacheTransceiver::SyncRequesterStatusGuard::SyncRequesterStatusGuard(CacheTransceiver& transceiver) + : mTransceiver{transceiver} +{ + mTransceiver.mSyncRequesterActive.fetch_add(1, std::memory_order_relaxed); +} + +CacheTransceiver::SyncRequesterStatusGuard::~SyncRequesterStatusGuard() noexcept +{ + mTransceiver.mSyncRequesterActive.fetch_sub(1, std::memory_order_relaxed); +} + void CacheTransceiver::initializeCommState() { mCommState = std::addressof(mCacheSender->getCommState()); } +std::vector<char> CacheTransceiver::getSerializedDataTransceiverState() const +{ + TLLM_CHECK(mCommState != nullptr && mCacheState != nullptr); + executor::DataTransceiverState state; + state.setCommState(*mCommState); + state.setCacheState(*mCacheState); + // Only this API marks the state; context responses leave it unset. + state.setIsArbitraryTransferState(true); + return executor::Serialization::serialize(state); +} + void CacheTransceiver::setContextState(LlmRequest* llmRequest) { TLLM_CHECK(llmRequest && llmRequest->isContextOnlyRequest()); @@ -625,6 +864,7 @@ void CacheTransceiver::respondAndSendAsync(std::shared_ptr<LlmRequest> llmReques setContextState(llmRequest.get()); auto future = mCacheSender->sendAsync(llmRequest); mSenderFutures.emplace_back(std::move(llmRequest), std::move(future)); + publishStatusSnapshot(); } void CacheTransceiver::respondAndSendLayerWise( @@ -642,6 +882,7 @@ void CacheTransceiver::respondAndSendLayerWise( auto future = mCacheSender->sendAsync(llmRequest); mSenderFutures.emplace_back(llmRequest, std::move(future)); } + publishStatusSnapshot(); } void CacheTransceiver::requestAndReceiveSync(std::shared_ptr<LlmRequest> llmRequest) @@ -651,6 +892,7 @@ void CacheTransceiver::requestAndReceiveSync(std::shared_ptr<LlmRequest> llmRequ auto const contextRequestId = llmRequest->getContextPhaseParams().value().getReqId(); TLLM_LOG_DEBUG("Synchronous KV cache receive request %zu, context request %zu waiting for native completion.", requestId, contextRequestId); + SyncRequesterStatusGuard statusGuard{*this}; try { auto future = mCacheReceiver->receiveAsync(llmRequest); @@ -699,6 +941,7 @@ void CacheTransceiver::requestAndReceiveAsync(std::shared_ptr<LlmRequest> llmReq auto* requestPtr = llmRequest.get(); mRequesterFutures.emplace_back(std::move(llmRequest), std::move(future)); requestPtr->setState(LlmRequestState::kDISAGG_GENERATION_TRANS_IN_PROGRESS); + publishStatusSnapshot(); } std::vector<LlmRequest::RequestIdType> gatherRequestIds( @@ -731,67 +974,109 @@ std::vector<LlmRequest::RequestIdType> gatherRequestIds( return retData; } -void updateKVCacheTransferBW(std::shared_ptr<CacheTransceiverComm> const& mComm, LlmRequest* request) +void batchUpdateKVCacheTransferBW( + std::shared_ptr<CacheTransceiverComm> const& comm, std::vector<LlmRequest*> const& requests) { + // Key-based merge: each rank serializes (requestId, start, end, size) + // tuples and we use allgatherv so ranks may have different request counts. + // The merge matches by requestId, not by position — this tolerates + // ordering differences and count mismatches across ranks. + namespace su = executor::serialize_utils; - int worldSize = mComm->getSize(); + int const worldSize = comm->getSize(); + + // --- Serialize local entries keyed by requestId --- + std::size_t const numReqs = requests.size(); std::ostringstream oStream; - su::serialize(request->getKvCacheTransferStart(), oStream); - su::serialize(request->getKvCacheTransferEnd(), oStream); + su::serialize(numReqs, oStream); + for (auto* req : requests) + { + su::serialize(req->getContextPhaseParams().value().getReqId(), oStream); + su::serialize(req->getKvCacheTransferStart(), oStream); + su::serialize(req->getKvCacheTransferEnd(), oStream); + su::serialize(req->getKvCacheSize(), oStream); + } auto str = oStream.str(); std::vector<char> sendBuffer(str.begin(), str.end()); - auto sendBufferSize = sendBuffer.size(); - auto recvBufferSize = sendBufferSize * worldSize; - std::vector<char> recvBuffer(recvBufferSize); + int const sendSize = static_cast<int>(sendBuffer.size()); + // --- Step 1: allgather per-rank buffer sizes --- + std::vector<int> recvCounts(worldSize, 0); if (useMPI()) { - mComm->allgather(sendBuffer.data(), recvBuffer.data(), sendBufferSize, mpi::MpiType::kCHAR); + comm->allgather(&sendSize, recvCounts.data(), 1, mpi::MpiType::kINT32); } else { - mComm->allgather(std::ref(sendBuffer), std::ref(recvBuffer), {}); + comm->allgather(sendSize, std::ref(recvCounts), {}); } - su::VectorWrapBuf<char> strbuf(recvBuffer); - std::istream is(&strbuf); - - auto minStartTime = executor::RequestPerfMetrics::TimePoint::max(); - auto maxEndTime = executor::RequestPerfMetrics::TimePoint::min(); - - for (int rank = 0; rank < worldSize; rank++) + // --- Step 2: allgatherv the serialized data --- + std::vector<int> displs(worldSize, 0); + int totalRecvSize = 0; + for (int r = 0; r < worldSize; ++r) { - minStartTime = std::min(su::deserialize<executor::RequestPerfMetrics::TimePoint>(is), minStartTime); - maxEndTime = std::max(su::deserialize<executor::RequestPerfMetrics::TimePoint>(is), maxEndTime); + displs[r] = totalRecvSize; + totalRecvSize += recvCounts[r]; } - - // Handle KV cache size separately - gather all sizes to the leader rank - std::size_t localKVCacheSize = request->getKvCacheSize(); - std::vector<std::size_t> allKVCacheSizes(worldSize, 0); + std::vector<char> recvBuffer(totalRecvSize, 0); if (useMPI()) { - mComm->allgather(&localKVCacheSize, allKVCacheSizes.data(), 1, mpi::MpiType::kUINT64); + comm->allgatherv(sendBuffer.data(), sendSize, mpi::MpiType::kCHAR, recvBuffer.data(), recvCounts, displs, + mpi::MpiType::kCHAR); } else { - mComm->allgather(&localKVCacheSize, std::ref(allKVCacheSizes), {}); + comm->allgatherv(std::ref(sendBuffer), std::ref(recvBuffer), recvCounts, {}); } - std::size_t totalKVCacheSize = 0; - for (int rank = 0; rank < worldSize; rank++) + // --- Step 3: Deserialize and merge by requestId --- + using TimePoint = executor::RequestPerfMetrics::TimePoint; + using ReqIdType = LlmRequest::RequestIdType; + + struct MergedEntry { - totalKVCacheSize += allKVCacheSizes[rank]; + TimePoint minStart = TimePoint::max(); + TimePoint maxEnd = TimePoint::min(); + std::size_t totalSize = 0; + }; + + std::unordered_map<ReqIdType, MergedEntry> merged; + + su::VectorWrapBuf<char> strbuf(recvBuffer); + std::istream is(&strbuf); + + for (int rank = 0; rank < worldSize; ++rank) + { + auto rankNumReqs = su::deserialize<std::size_t>(is); + for (std::size_t i = 0; i < rankNumReqs; ++i) + { + auto rid = su::deserialize<ReqIdType>(is); + auto start = su::deserialize<TimePoint>(is); + auto end = su::deserialize<TimePoint>(is); + auto size = su::deserialize<std::size_t>(is); + + auto& entry = merged[rid]; + entry.minStart = std::min(entry.minStart, start); + entry.maxEnd = std::max(entry.maxEnd, end); + entry.totalSize += size; + } } - // Update the latest KV cache transfer time for leader rank - if (mComm->getRank() == 0) + // --- Step 4: Update local requests --- + for (auto* req : requests) { - request->setKvCacheTransferStart(minStartTime); - request->setKvCacheTransferEnd(maxEndTime); - request->setKvCacheSize(totalKVCacheSize); + auto reqId = req->getContextPhaseParams().value().getReqId(); + auto it = merged.find(reqId); + if (it != merged.end()) + { + req->setKvCacheTransferStart(it->second.minStart); + req->setKvCacheTransferEnd(it->second.maxEnd); + req->setKvCacheSize(it->second.totalSize); + } } } @@ -827,7 +1112,7 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( contextCompleteRequestIds.push_back(request->mRequestId); } } - + publishStatusSnapshot(); std::unordered_map<LlmRequest::RequestIdType, int> frequencyMap; if ((syncComm) && syncComm->getSize() > 1) { @@ -867,6 +1152,26 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( toCompleteIdSet.insert(request->mRequestId); } + auto recordTimeout = [&](RequestIdType const requestId) + { + bool const inserted = mTimedOutSenderIds.insert(requestId).second; + if (inserted && inflightCancelEnabled && mContextTransferCoordinator) + { + mContextTransferCoordinator->publishTimeout(requestId); + } + return inserted; + }; + auto recordOutcome + = [&](RequestIdType const requestId, std::shared_ptr<LlmRequest> const& request, bool const failed) + { + recordLocalTransferOutcome(requestId, request, failed, mCompletedSenderRequestIds, mFailedSenderRequestIds, + mSenderRequestsAwaitingConsensus); + if (mContextTransferCoordinator) + { + mContextTransferCoordinator->publishLocalOutcome(requestId, failed); + } + }; + // Record local terminal outcomes for requests selected this round. The // request is reported only after all ranks in the sync group agree that the // request reached a terminal state. @@ -880,7 +1185,7 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( auto const elapsedMs = getTransferElapsedMs(request, LlmRequest::getSteadyClockNow()); if (elapsedMs > kvTransferTimeoutMs.value()) { - if (mTimedOutSenderIds.insert(requestId).second) + if (recordTimeout(requestId)) { TLLM_LOG_WARNING( "Context KV cache transfer for request %ld exceeded configured timeout: " @@ -892,6 +1197,8 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( } if (blockAll || (toCompleteIdSet.find(requestId) != toCompleteIdSet.end())) { + bool terminal = false; + bool failed = false; try { auto const status = blockAll ? std::future_status::ready : future.wait_for(futureWaitInterval); @@ -901,7 +1208,7 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( if (kvTransferTimeoutMs.has_value()) { auto const elapsedMs = getTransferElapsedMs(request, request->getKvCacheTransferEnd()); - if (elapsedMs > kvTransferTimeoutMs.value() && mTimedOutSenderIds.insert(requestId).second) + if (elapsedMs > kvTransferTimeoutMs.value() && recordTimeout(requestId)) { TLLM_LOG_WARNING( "Context KV cache transfer for request %ld completed after its deadline: " @@ -910,9 +1217,8 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( inflightCancelEnabled ? "failing request" : "observe-only"); } } - recordLocalTransferOutcome(requestId, request, /*failed=*/false, mCompletedSenderRequestIds, - mFailedSenderRequestIds, mSenderRequestsAwaitingConsensus); - it = mSenderFutures.erase(it); + failed = request->getState() == LlmRequestState::kDISAGG_TRANS_ERROR; + terminal = true; } else if (status == std::future_status::timeout) { @@ -926,25 +1232,28 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( { TLLM_LOG_ERROR( "Future returned unexpected status for request %ld. Recording as failed.", requestId); - - recordLocalTransferOutcome(requestId, request, /*failed=*/true, mCompletedSenderRequestIds, - mFailedSenderRequestIds, mSenderRequestsAwaitingConsensus); - it = mSenderFutures.erase(it); + failed = true; + terminal = true; } } catch (std::exception const& e) { TLLM_LOG_ERROR("Error occurred during context transfer for request %ld: %s", requestId, e.what()); - recordLocalTransferOutcome(requestId, request, /*failed=*/true, mCompletedSenderRequestIds, - mFailedSenderRequestIds, mSenderRequestsAwaitingConsensus); - it = mSenderFutures.erase(it); + failed = true; + terminal = true; } catch (...) { TLLM_LOG_ERROR("Unknown error occurred during context transfer for request %ld", requestId); - recordLocalTransferOutcome(requestId, request, /*failed=*/true, mCompletedSenderRequestIds, - mFailedSenderRequestIds, mSenderRequestsAwaitingConsensus); + failed = true; + terminal = true; + } + if (terminal) + { + auto terminalRequest = request; it = mSenderFutures.erase(it); + // Publish outside the transfer-future try/catch. A protocol error must not rewrite an immutable vote. + recordOutcome(requestId, terminalRequest, failed); } } else @@ -953,13 +1262,54 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( } } + // Publish after local polling and before consensus, which may be the point at which a rank hangs. + publishStatusSnapshot(); RequestStatuses requestsStatus{}; - auto const consensusOutcome = reduceTransferStates(syncComm, mGroupPipeParaComm, mCompletedSenderRequestIds, - mFailedSenderRequestIds, inflightCancelEnabled ? mTimedOutSenderIds : std::unordered_set<RequestIdType>{}); + TransferConsensusOutcome consensusOutcome; + if (mContextTransferCoordinator) + { + auto mergeCoordinatorOutcome = [&]() + { + auto coordinatorOutcome = mContextTransferCoordinator->poll(); + consensusOutcome.completedRequestIds.insert( + coordinatorOutcome.completedRequestIds.begin(), coordinatorOutcome.completedRequestIds.end()); + consensusOutcome.failedRequestIds.insert( + coordinatorOutcome.failedRequestIds.begin(), coordinatorOutcome.failedRequestIds.end()); + consensusOutcome.timedOutRequestIds.insert( + coordinatorOutcome.timedOutRequestIds.begin(), coordinatorOutcome.timedOutRequestIds.end()); + }; + do + { + mergeCoordinatorOutcome(); + if (blockAll + && consensusOutcome.completedRequestIds.size() + consensusOutcome.failedRequestIds.size() + < mSenderRequestsAwaitingConsensus.size()) + { + std::this_thread::yield(); + } + } while (blockAll + && consensusOutcome.completedRequestIds.size() + consensusOutcome.failedRequestIds.size() + < mSenderRequestsAwaitingConsensus.size()); + + if (inflightCancelEnabled) + { + // A timeout update is transmitted once, but cancellation may be declined transiently. Keep the globally + // observed timeout active so rank-local cancellation is retried on every poll until terminal commit. + consensusOutcome.timedOutRequestIds.insert(mTimedOutSenderIds.begin(), mTimedOutSenderIds.end()); + } + } + else + { + consensusOutcome = reduceTransferStates(syncComm, mGroupPipeParaComm, mCompletedSenderRequestIds, + mFailedSenderRequestIds, inflightCancelEnabled ? mTimedOutSenderIds : std::unordered_set<RequestIdType>{}); + } if (inflightCancelEnabled) { for (auto const requestId : consensusOutcome.timedOutRequestIds) { + // Persist the global timeout even if this rank has not registered its local future yet. The one-shot + // coordinator update must remain actionable when that future appears on a later scheduler poll. + mTimedOutSenderIds.insert(requestId); auto const futureIt = std::find_if(mSenderFutures.begin(), mSenderFutures.end(), [requestId](auto const& entry) { return entry.first->mRequestId == requestId; }); if (futureIt == mSenderFutures.end() @@ -968,7 +1318,6 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( { continue; } - mTimedOutSenderIds.insert(requestId); if (requestCancellationNoThrow( requestId, "Context", [&]() { return mCacheSender->cancelRequest(*futureIt->first); })) { @@ -1012,6 +1361,7 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( requestId, mCompletedSenderRequestIds, mFailedSenderRequestIds, mSenderRequestsAwaitingConsensus); } + publishStatusSnapshot(); return requestsStatus; } @@ -1054,6 +1404,7 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR collectReadyRequestIds(); } } + publishStatusSnapshot(); std::unordered_map<LlmRequest::RequestIdType, int> frequencyMap; std::vector<LlmRequest::RequestIdType> toBlockRequestIds; @@ -1207,6 +1558,8 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR } } + // Publish after local polling and before collectives, which may be the point at which a rank hangs. + publishStatusSnapshot(); auto const consensusOutcome = reduceTransferStates(syncComm, mCompletedRequesterRequestIds, mFailedRequesterRequestIds, inflightCancelEnabled ? mTimedOutRequesterIds : std::unordered_set<RequestIdType>{}); @@ -1247,6 +1600,10 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR eraseLocalTransferOutcome( requestId, mCompletedRequesterRequestIds, mFailedRequesterRequestIds, mRequesterRequestsAwaitingConsensus); } + + // Collect consensus-completed requests so timing can be synced across ranks in a single + // batched allgather (instead of one collective per request). + std::vector<LlmRequest*> completedRequests; for (auto const requestId : sortedRequestIds(consensusOutcome.completedRequestIds)) { auto const requestIt = mRequesterRequestsAwaitingConsensus.find(requestId); @@ -1255,17 +1612,45 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR continue; } requestIt->second->setState(LlmRequestState::kDISAGG_GENERATION_TRANS_COMPLETE); - - // Gather the kv cache transfer time from all workers and update to leader rank. - if (!common::getEnvKVCacheTimeOutputPath().empty()) - { - updateKVCacheTransferBW(syncComm, requestIt->second.get()); - } + completedRequests.push_back(requestIt->second.get()); mTimedOutRequesterIds.erase(requestId); mCancelRequestedRequesterIds.erase(requestId); eraseLocalTransferOutcome( requestId, mCompletedRequesterRequestIds, mFailedRequesterRequestIds, mRequesterRequestsAwaitingConsensus); } + publishStatusSnapshot(); + + // Batch-sync timing across ranks in one allgather (instead of per-request), then write + // the gen-side transfer summary CSV. + if (!completedRequests.empty() && !common::getEnvKVCacheTimeOutputPath().empty()) + { + batchUpdateKVCacheTransferBW(syncComm, completedRequests); + writeGenTransferSummary(completedRequests); + } +} + +void CacheTransceiver::writeGenTransferSummary(std::vector<LlmRequest*> const& completedRequests) +{ + std::lock_guard<std::mutex> lock(mGenTransferSummaryMutex); + if (!mGenTransferSummaryFile.is_open()) + { + namespace fs = std::filesystem; + auto outputPath = fs::path(common::getEnvKVCacheTimeOutputPath()); + fs::create_directories(outputPath); + int rank = useMPI() ? mpi::MpiComm::world().getRank() : tensorrt_llm::pg_utils::get_world_pg()->getRank(); + auto filePath = outputPath / (mInstanceId + "_" + std::to_string(rank) + "_gen_transfer_summary.csv"); + mGenTransferSummaryFile.open(filePath); + TLLM_CHECK_WITH_INFO(mGenTransferSummaryFile.is_open(), "Failed to open gen transfer summary file: %s", + filePath.string().c_str()); + mGenTransferSummaryFile << "RequestID,gen_side_transfer_time(ms),kv_cache_size" << '\n'; + } + for (auto* req : completedRequests) + { + auto reqId = req->getContextPhaseParams().value().getReqId(); + mGenTransferSummaryFile << reqId << "," << req->getKvCacheTransferTimeMS() << "," << req->getKvCacheSize() + << '\n'; + } + mGenTransferSummaryFile << std::flush; } bool CacheTransceiver::checkGenTransferComplete() const diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.cpp b/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.cpp index d0a54dbb7d3c..b2e0a21537d1 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.cpp @@ -21,6 +21,7 @@ #include "tensorrt_llm/batch_manager/rnnCacheFormatter.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" #include "tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h" @@ -121,7 +122,8 @@ void CacheTransferLayer::unformat(TransferSession& session) const } void CacheTransferLayer::setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, nvinfer1::DataType ssmStateDataType) + std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, + tensorrt_llm::DataType ssmStateDataType) { mCacheState.setRnnConfig( std::move(rnnModelConfig), std::move(rnnLayerNumPerPP), convStateDataType, ssmStateDataType); diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.h b/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.h index 0506e98197e6..48a171a65c9c 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.h +++ b/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.h @@ -18,6 +18,7 @@ #pragma once #include "tensorrt_llm/batch_manager/rnnCacheFormatter.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/runtime/common.h" @@ -79,8 +80,8 @@ class CacheTransferLayer /// @brief Update the RNN config on the internal CacheState. /// Used by CppMambaHybridCacheManager path where RNN config is set after construction. void setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, - nvinfer1::DataType ssmStateDataType); + std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, + tensorrt_llm::DataType ssmStateDataType); [[nodiscard]] kv_cache_manager::BaseKVCacheManager* getCacheManager() const noexcept; diff --git a/cpp/tensorrt_llm/batch_manager/contextTransferCoordinator.cpp b/cpp/tensorrt_llm/batch_manager/contextTransferCoordinator.cpp new file mode 100644 index 000000000000..909b7858c574 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/contextTransferCoordinator.cpp @@ -0,0 +1,475 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/contextTransferCoordinator.h" + +#include "tensorrt_llm/batch_manager/cacheTransceiver.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" + +#include <array> +#include <chrono> +#include <cstdlib> +#include <exception> +#include <list> +#include <thread> +#include <utility> + +namespace tensorrt_llm::batch_manager +{ + +ContextTransferVoteReducer::ContextTransferVoteReducer(int const participantCount) + : mParticipantCount(participantCount) +{ + TLLM_CHECK_WITH_INFO(participantCount > 0, "Context-transfer consensus requires at least one participant."); +} + +void ContextTransferVoteReducer::recordVote( + int const participantRank, std::uint64_t const requestId, ContextTransferVote const vote) +{ + TLLM_CHECK_WITH_INFO(participantRank >= 0 && participantRank < mParticipantCount, + "Context-transfer consensus participant rank is out of range."); + TLLM_CHECK_WITH_INFO(vote == ContextTransferVote::kCompleted || vote == ContextTransferVote::kFailed, + "Context-transfer consensus received an invalid vote."); + + auto [requestIt, inserted] = mRequestVotes.try_emplace(requestId, mParticipantCount); + static_cast<void>(inserted); + auto& requestVotes = requestIt->second; + auto& recordedVote = requestVotes.votes.at(static_cast<std::size_t>(participantRank)); + auto const packedVote = static_cast<std::uint64_t>(vote); + if (recordedVote != 0) + { + TLLM_CHECK_WITH_INFO( + recordedVote == packedVote, "Context-transfer participant changed its terminal vote for a request."); + return; + } + + recordedVote = packedVote; + ++requestVotes.terminalCount; + requestVotes.failed = requestVotes.failed || vote == ContextTransferVote::kFailed; +} + +void ContextTransferVoteReducer::recordTimeout(std::uint64_t const requestId) +{ + auto [requestIt, inserted] = mRequestVotes.try_emplace(requestId, mParticipantCount); + static_cast<void>(inserted); + auto& requestVotes = requestIt->second; + if (!requestVotes.timedOut) + { + requestVotes.timedOut = true; + requestVotes.timeoutPending = true; + } +} + +ContextTransferConsensusResult ContextTransferVoteReducer::takeReady() +{ + ContextTransferConsensusResult result; + for (auto requestIt = mRequestVotes.begin(); requestIt != mRequestVotes.end();) + { + auto& requestVotes = requestIt->second; + if (requestVotes.timeoutPending) + { + result.timedOutRequestIds.insert(requestIt->first); + requestVotes.timeoutPending = false; + } + if (requestVotes.terminalCount != mParticipantCount) + { + ++requestIt; + continue; + } + + auto& terminalRequestIds + = (requestVotes.failed || requestVotes.timedOut) ? result.failedRequestIds : result.completedRequestIds; + terminalRequestIds.insert(requestIt->first); + requestIt = mRequestVotes.erase(requestIt); + } + return result; +} + +void ContextTransferVoteReducer::clear() noexcept +{ + mRequestVotes.clear(); +} + +class ContextTransferCoordinator::Impl +{ +public: + explicit Impl(std::shared_ptr<CacheTransceiverComm> comm) + : mComm(std::move(comm)) + , mCoordinatorRank(0) + , mReducer(mComm ? mComm->getSize() : 1) + { + TLLM_CHECK_WITH_INFO(mComm != nullptr, "Context-transfer coordination requires a communicator."); + TLLM_CHECK_WITH_INFO(mComm->isMpi(), "Asynchronous context-transfer coordination requires MPI."); + TLLM_CHECK_WITH_INFO( + mComm->getSize() > 1, "Asynchronous context-transfer coordination requires multiple participants."); + mCoordinatorRank = mComm->getSize() - 1; + } + + void publishLocalOutcome(std::uint64_t const requestId, bool const failed) + { + TLLM_CHECK_WITH_INFO(!mShutdown, "Cannot publish a context-transfer vote after coordinator shutdown."); + auto const vote = failed ? ContextTransferVote::kFailed : ContextTransferVote::kCompleted; + auto const [voteIt, inserted] = mPublishedLocalVotes.emplace(requestId, vote); + TLLM_CHECK_WITH_INFO( + inserted || voteIt->second == vote, "This rank changed its terminal vote for a context transfer."); + if (!inserted) + { + return; + } + + try + { + if (isCoordinator()) + { + mReducer.recordVote(mComm->getRank(), requestId, vote); + } + else + { + queuePacket( + requestId, static_cast<std::uint64_t>(vote), mCoordinatorRank, mpi::MpiTag::kContextTransferEvent); + } + } + catch (...) + { + mPublishedLocalVotes.erase(voteIt); + throw; + } + } + + void publishTimeout(std::uint64_t const requestId) + { + TLLM_CHECK_WITH_INFO(!mShutdown, "Cannot publish a context-transfer timeout after coordinator shutdown."); + TLLM_CHECK_WITH_INFO(mPublishedLocalVotes.find(requestId) == mPublishedLocalVotes.end(), + "A context-transfer timeout must be published before its immutable terminal vote."); + auto const [timeoutIt, inserted] = mPublishedTimeouts.insert(requestId); + if (!inserted) + { + return; + } + + try + { + if (isCoordinator()) + { + mReducer.recordTimeout(requestId); + } + else + { + queuePacket(requestId, kTimedOutSignal, mCoordinatorRank, mpi::MpiTag::kContextTransferEvent); + } + } + catch (...) + { + mPublishedTimeouts.erase(timeoutIt); + throw; + } + } + + [[nodiscard]] ContextTransferConsensusResult poll() + { + TLLM_CHECK_WITH_INFO(!mShutdown, "Cannot poll context-transfer coordination after shutdown."); + return progress(); + } + + void shutdown() noexcept + { + if (mShutdown) + { + return; + } + mShutdown = true; + + try + { + if (!isCoordinator()) + { + queuePacket(/*requestId=*/0, kCloseMarker, mCoordinatorRank, mpi::MpiTag::kContextTransferEvent); + } + + auto const deadline = std::chrono::steady_clock::now() + kShutdownTimeout; + while (!shutdownComplete()) + { + static_cast<void>(progress()); + if (!shutdownComplete()) + { + if (std::chrono::steady_clock::now() >= deadline) + { + TLLM_LOG_ERROR( + "Timed out shutting down asynchronous context-transfer coordinator; rank=%d " + "peer_closes=%zu/%d ack=%d pending_sends=%zu. Aborting to avoid freeing active MPI " + "requests.", + mComm->getRank(), mClosedPeers.size(), mComm->getSize() - 1, mCloseAcknowledged, + mPendingSends.size()); + std::abort(); + } + std::this_thread::yield(); + } + } + } + catch (std::exception const& error) + { + TLLM_LOG_ERROR("Failed to shut down asynchronous context-transfer coordinator: %s", error.what()); + std::abort(); + } + catch (...) + { + TLLM_LOG_ERROR("Failed to shut down asynchronous context-transfer coordinator with an unknown error."); + std::abort(); + } + } + +private: + static constexpr std::size_t kPacketFieldCount = 2; + static constexpr std::uint64_t kTimedOutSignal = 3; + static constexpr std::uint64_t kCloseMarker = 4; + static constexpr auto kShutdownTimeout = std::chrono::seconds(30); + + struct PendingSend + { + std::array<std::uint64_t, kPacketFieldCount> packet{}; + std::unique_ptr<mpi::MpiRequest> request; + }; + + [[nodiscard]] bool isCoordinator() const + { + return mComm->getRank() == mCoordinatorRank; + } + + void queuePacket(std::uint64_t const requestId, std::uint64_t const value, int const peer, mpi::MpiTag const tag) + { + mPendingSends.emplace_back(); + auto& pendingSend = mPendingSends.back(); + pendingSend.packet = {requestId, value}; + try + { + pendingSend.request = mComm->sendAsync( + pendingSend.packet.data(), pendingSend.packet.size(), mpi::MpiType::kUINT64, peer, tag); + } + catch (...) + { + mPendingSends.pop_back(); + throw; + } + } + + void queueUpdateForPeers(std::uint64_t const requestId, std::uint64_t const update) + { + for (int peer = 0; peer < mComm->getSize(); ++peer) + { + if (peer != mCoordinatorRank) + { + queuePacket(requestId, update, peer, mpi::MpiTag::kContextTransferUpdate); + } + } + } + + void reapCompletedSends() + { + for (auto sendIt = mPendingSends.begin(); sendIt != mPendingSends.end();) + { + TLLM_CHECK(sendIt->request); + if (sendIt->request->isCompleted()) + { + sendIt = mPendingSends.erase(sendIt); + } + else + { + ++sendIt; + } + } + } + + void drainVotes() + { + for (int peer = 0; peer < mComm->getSize(); ++peer) + { + if (peer == mCoordinatorRank) + { + continue; + } + + MPI_Status status{}; + while (mComm->iprobe(peer, mpi::MpiTag::kContextTransferEvent, &status)) + { + std::array<std::uint64_t, kPacketFieldCount> packet{}; + mComm->recv( + packet.data(), packet.size(), mpi::MpiType::kUINT64, peer, mpi::MpiTag::kContextTransferEvent); + if (packet.back() == kCloseMarker) + { + TLLM_CHECK_WITH_INFO( + mClosedPeers.insert(peer).second, "Received a duplicate context-transfer close marker."); + continue; + } + TLLM_CHECK_WITH_INFO(mClosedPeers.find(peer) == mClosedPeers.end(), + "Received a context-transfer vote after its peer close marker."); + if (packet.back() == kTimedOutSignal) + { + mReducer.recordTimeout(packet.front()); + continue; + } + mReducer.recordVote(peer, packet.front(), static_cast<ContextTransferVote>(packet.back())); + } + } + } + + ContextTransferConsensusResult completeCoordinatorUpdates() + { + auto result = mReducer.takeReady(); + for (auto const requestId : result.timedOutRequestIds) + { + queueUpdateForPeers(requestId, kTimedOutSignal); + } + for (auto const requestId : result.failedRequestIds) + { + queueUpdateForPeers(requestId, static_cast<std::uint64_t>(ContextTransferVote::kFailed)); + mPublishedLocalVotes.erase(requestId); + mPublishedTimeouts.erase(requestId); + } + for (auto const requestId : result.completedRequestIds) + { + queueUpdateForPeers(requestId, static_cast<std::uint64_t>(ContextTransferVote::kCompleted)); + mPublishedLocalVotes.erase(requestId); + mPublishedTimeouts.erase(requestId); + } + return result; + } + + ContextTransferConsensusResult drainUpdates() + { + ContextTransferConsensusResult result; + MPI_Status status{}; + while (mComm->iprobe(mCoordinatorRank, mpi::MpiTag::kContextTransferUpdate, &status)) + { + std::array<std::uint64_t, kPacketFieldCount> packet{}; + mComm->recv(packet.data(), packet.size(), mpi::MpiType::kUINT64, mCoordinatorRank, + mpi::MpiTag::kContextTransferUpdate); + if (packet.back() == kCloseMarker) + { + TLLM_CHECK_WITH_INFO(!mCloseAcknowledged, "Received a duplicate coordinator close marker."); + mCloseAcknowledged = true; + mPublishedLocalVotes.clear(); + mPublishedTimeouts.clear(); + continue; + } + + if (packet.back() == kTimedOutSignal) + { + result.timedOutRequestIds.insert(packet.front()); + continue; + } + + auto const outcome = static_cast<ContextTransferVote>(packet.back()); + TLLM_CHECK_WITH_INFO(outcome == ContextTransferVote::kCompleted || outcome == ContextTransferVote::kFailed, + "Received an invalid context-transfer commit outcome."); + auto const localVoteIt = mPublishedLocalVotes.find(packet.front()); + TLLM_CHECK_WITH_INFO(localVoteIt != mPublishedLocalVotes.end(), + "Received a context-transfer commit before publishing the local terminal vote."); + if (outcome == ContextTransferVote::kFailed) + { + result.failedRequestIds.insert(packet.front()); + } + else + { + result.completedRequestIds.insert(packet.front()); + } + mPublishedLocalVotes.erase(localVoteIt); + mPublishedTimeouts.erase(packet.front()); + } + return result; + } + + ContextTransferConsensusResult progress() + { + reapCompletedSends(); + if (isCoordinator()) + { + drainVotes(); + auto result = completeCoordinatorUpdates(); + if (mShutdown && !mCloseSent && mClosedPeers.size() == static_cast<std::size_t>(mComm->getSize() - 1)) + { + // Shutdown is an explicit abort epoch. A peer close proves that no more votes will arrive from that + // peer, so incomplete requests cannot reach a global decision and are intentionally abandoned. + mReducer.clear(); + mPublishedLocalVotes.clear(); + mPublishedTimeouts.clear(); + for (int peer = 0; peer < mCoordinatorRank; ++peer) + { + queuePacket(/*requestId=*/0, kCloseMarker, peer, mpi::MpiTag::kContextTransferUpdate); + } + mCloseSent = true; + } + return result; + } + return drainUpdates(); + } + + [[nodiscard]] bool shutdownComplete() const + { + if (isCoordinator()) + { + return mCloseSent && mPendingSends.empty(); + } + return mCloseAcknowledged && mPendingSends.empty(); + } + + std::shared_ptr<CacheTransceiverComm> mComm; + int mCoordinatorRank; + ContextTransferVoteReducer mReducer; + std::unordered_map<std::uint64_t, ContextTransferVote> mPublishedLocalVotes; + std::unordered_set<std::uint64_t> mPublishedTimeouts; + std::unordered_set<int> mClosedPeers; + std::list<PendingSend> mPendingSends; + bool mShutdown{false}; + bool mCloseSent{false}; + bool mCloseAcknowledged{false}; +}; + +ContextTransferCoordinator::ContextTransferCoordinator(std::shared_ptr<CacheTransceiverComm> comm) + : mImpl(std::make_unique<Impl>(std::move(comm))) +{ +} + +ContextTransferCoordinator::~ContextTransferCoordinator() +{ + shutdown(); +} + +void ContextTransferCoordinator::publishLocalOutcome(std::uint64_t const requestId, bool const failed) +{ + mImpl->publishLocalOutcome(requestId, failed); +} + +void ContextTransferCoordinator::publishTimeout(std::uint64_t const requestId) +{ + mImpl->publishTimeout(requestId); +} + +ContextTransferConsensusResult ContextTransferCoordinator::poll() +{ + return mImpl->poll(); +} + +void ContextTransferCoordinator::shutdown() noexcept +{ + if (mImpl) + { + mImpl->shutdown(); + } +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/createNewDecoderRequests.cpp b/cpp/tensorrt_llm/batch_manager/createNewDecoderRequests.cpp index 5c5d3e11a01c..2d090a5612a5 100644 --- a/cpp/tensorrt_llm/batch_manager/createNewDecoderRequests.cpp +++ b/cpp/tensorrt_llm/batch_manager/createNewDecoderRequests.cpp @@ -33,7 +33,7 @@ #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/speculativeChoicesUtils.h" -#include <NvInferRuntimeBase.h> +#include "tensorrt_llm/common/tllmDataType.h" using namespace tensorrt_llm::runtime; @@ -69,10 +69,8 @@ void copySequenceLengths(RequestVector const& contextRequests, DecoderInputBuffe SizeType32 batchIdx{0}; for (auto const& llmReq : contextRequests) { - auto const disaggFirstGenTokenSize - = llmReq->getContextPhaseParams() ? llmReq->getContextPhaseParams().value().getFirstGenTokens().size() : 0; auto const currentSequenceLen - = llmReq->mPromptLen + llmReq->getMaxNumGeneratedTokens() + disaggFirstGenTokenSize; + = llmReq->mPromptLen + llmReq->getMaxNumGeneratedTokens() + llmReq->getNumContextPhaseGenerationTokens(); // Get position of the current sequence in the decoder auto const seqSlot = llmReq->mSeqSlot.value(); batchSlotsRange[batchIdx] = seqSlot; @@ -93,7 +91,7 @@ void copySequenceLengths(RequestVector const& contextRequests, DecoderInputBuffe /// @brief Retrieve the embedding bias from the request. This potentially makes a copy of the tensor /// to the appropriate type if the input tensor does not match it. -[[nodiscard]] TensorPtr getEmbeddingBias(nvinfer1::DataType logitsType, TensorPtr const& tensor) +[[nodiscard]] TensorPtr getEmbeddingBias(tensorrt_llm::DataType logitsType, TensorPtr const& tensor) { // Check that embedding bias type is same as logits type. If so, we can return the tensor right away if (tensor->getDataType() == logitsType) @@ -102,7 +100,7 @@ void copySequenceLengths(RequestVector const& contextRequests, DecoderInputBuffe } // Support FP32 input for FP16 embedding bias (in the case of FP8 models) - if (tensor->getDataType() == nvinfer1::DataType::kFLOAT && logitsType == nvinfer1::DataType::kHALF) + if (tensor->getDataType() == tensorrt_llm::DataType::kFLOAT && logitsType == tensorrt_llm::DataType::kHALF) { // Do a deep copy of the tensor to the expected type TLLM_LOG_WARNING( @@ -133,10 +131,10 @@ void copySequenceLengths(RequestVector const& contextRequests, DecoderInputBuffe std::tuple<TensorPtr, std::vector<runtime::SamplingConfig>, std::vector<runtime::ITensor::SharedConstPtr>, std::vector<executor::LookaheadDecodingConfig>> CreateNewDecoderRequests::operator()(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - executor::DecodingConfig const& decodingConfig, RequestVector const& contextRequests, nvinfer1::DataType logitsType, - DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState, CudaStream const& runtimeStream, - CudaStream const& decoderStream, SizeType32 maxSequenceLength, SizeType32 beamWidth, - OptionalRef<MedusaBuffers const> medusaBuffers) const + executor::DecodingConfig const& decodingConfig, RequestVector const& contextRequests, + tensorrt_llm::DataType logitsType, DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState, + CudaStream const& runtimeStream, CudaStream const& decoderStream, SizeType32 maxSequenceLength, + SizeType32 beamWidth, OptionalRef<MedusaBuffers const> medusaBuffers) const { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); NVTX3_SCOPED_RANGE(CreateNewDecoderRequests); @@ -235,7 +233,7 @@ void initializeBeamSearch(DecodingInput& dJointInput, DecodingOutput& dJointOutp } void initializeEmbeddingBias(DecodingInput& dJointInput, SizeType32 batchSlot, - std::optional<TensorPtr> const& embeddingBias, nvinfer1::DataType logitsType, + std::optional<TensorPtr> const& embeddingBias, tensorrt_llm::DataType logitsType, runtime::ModelConfig const& modelConfig, BufferManager const& manager) { TensorPtr const embeddingBiasSlice = ITensor::slice(constPointerCast(dJointInput.embeddingBias), batchSlot, 1); @@ -631,7 +629,7 @@ void newRequestSpeculativeDecoding(DecodingInput& jointDecodingInput, DecodingOu std::tuple<std::vector<runtime::ITensor::SharedConstPtr>, std::vector<executor::LookaheadDecodingConfig>> CreateNewDecoderRequests::createDecoderRequests(RequestVector const& finishedContextRequests, TensorPtr const& inputIds, executor::DecodingConfig const& decodingConfig, runtime::decoder::DecoderState& decoderState, - nvinfer1::DataType logitsType, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + tensorrt_llm::DataType logitsType, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, runtime::CudaStream const& runtimeStream, runtime::CudaStream const& decoderStream, SizeType32 maxSequenceLength, OptionalRef<MedusaBuffers const> medusaBuffers) const { diff --git a/cpp/tensorrt_llm/batch_manager/dataTransceiver.cpp b/cpp/tensorrt_llm/batch_manager/dataTransceiver.cpp index 109417965a75..0f8ded65613f 100644 --- a/cpp/tensorrt_llm/batch_manager/dataTransceiver.cpp +++ b/cpp/tensorrt_llm/batch_manager/dataTransceiver.cpp @@ -20,9 +20,9 @@ #include "tensorrt_llm/batch_manager/cacheFormatter.h" #include "tensorrt_llm/batch_manager/common.h" #include "tensorrt_llm/batch_manager/kvCacheUtils.h" -#include "tensorrt_llm/batch_manager/runtimeBuffers.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/common/tllmException.h" #include "tensorrt_llm/common/utils.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" @@ -37,6 +37,7 @@ #include <optional> #include <stdexcept> #include <unordered_map> +#include <variant> namespace tensorrt_llm::batch_manager { @@ -81,6 +82,11 @@ void TransferSession::send(size_t idx, void const* data, size_t size) } catch (std::exception const& e) { + // Request-free (llmRequest-agnostic) transfer: there is no valid ID to attach. + if (mRequest == nullptr) + { + TLLM_THROW("%s", e.what()); + } throw common::RequestSpecificException( __FILE__, __LINE__, e.what(), mRequest->mRequestId, common::RequestErrorCode::kNETWORK_ERROR); } @@ -94,15 +100,23 @@ void TransferSession::recv(size_t idx, void* data, size_t size) } catch (std::exception const& e) { + // Request-free (llmRequest-agnostic) transfer: there is no valid ID to attach. + if (mRequest == nullptr) + { + TLLM_THROW("%s", e.what()); + } throw common::RequestSpecificException( __FILE__, __LINE__, e.what(), mRequest->mRequestId, common::RequestErrorCode::kNETWORK_ERROR); } } -LlmRequest const& TransferSession::getLlmRequest() const +std::optional<LlmRequest const*> TransferSession::getLlmRequest() const { - TLLM_CHECK(mRequest != nullptr); - return *mRequest; + if (mRequest == nullptr) + { + return std::nullopt; + } + return mRequest; } void TransferSession::setLlmRequest(LlmRequest const& llmRequest) @@ -171,7 +185,8 @@ void TransferSession::poisonReservedRecvBuffers() noexcept void TransferSession::exportMeasure(std::ofstream& outFile, bool isContext) const { - if (!mTimes || mTimes->measures.empty()) + // Request-free transfers are excluded: the exported row is keyed by the LlmRequest. + if (!mTimes || mTimes->measures.empty() || mRequest == nullptr) { return; } @@ -229,7 +244,7 @@ int32_t tagFromRequestId(LlmRequest::RequestIdType requestId) return ((requestId & 0xFFF) << 8) | (kDATA_TAG & 0xFF); } -std::filesystem::path getTransferOutputPath(char const* tag) +std::filesystem::path getTransferOutputPath(char const* tag, std::string const& instanceId = "") { namespace fs = std::filesystem; auto outputPath = common::getEnvKVCacheTimeOutputPath(); @@ -238,7 +253,9 @@ std::filesystem::path getTransferOutputPath(char const* tag) auto rank = mpi::MpiComm::world().getRank(); auto path = fs::path(outputPath); fs::create_directories(path); - return path / ("rank_" + std::to_string(rank) + "_" + tag + ".csv"); + std::string prefix + = instanceId.empty() ? "rank_" + std::to_string(rank) : instanceId + "_" + std::to_string(rank); + return path / (prefix + "_" + tag + ".csv"); } return {}; } @@ -275,7 +292,7 @@ RequestInfo::RequestInfo(LlmRequest::RequestIdType requestId, executor::DataTran bool RequestInfo::operator==(RequestInfo const& rhs) const { return mRequestId == rhs.mRequestId && mIndexFromEnd == rhs.mIndexFromEnd && mLastBlockKey == rhs.mLastBlockKey - && mTransState == rhs.mTransState; + && mIsArbitraryTransfer == rhs.mIsArbitraryTransfer && mTransState == rhs.mTransState; } LlmRequest::RequestIdType RequestInfo::getRequestId() const noexcept @@ -294,6 +311,7 @@ void RequestInfo::serialize(RequestInfo const& requestInfo, std::ostream& os) su::serialize(requestInfo.mRequestId, os); su::serialize(requestInfo.mIndexFromEnd, os); su::serialize(requestInfo.mLastBlockKey, os); + su::serialize(requestInfo.mIsArbitraryTransfer, os); su::serialize(requestInfo.mTransState, os); } @@ -303,8 +321,11 @@ RequestInfo RequestInfo::deserialize(std::istream& is) auto requestId = su::deserialize<decltype(mRequestId)>(is); auto indexFromEnd = su::deserialize<decltype(mIndexFromEnd)>(is); auto lastBlockKey = su::deserialize<decltype(mLastBlockKey)>(is); + auto isArbitraryTransfer = su::deserialize<decltype(mIsArbitraryTransfer)>(is); auto transState = su::deserialize<decltype(mTransState)>(is); - return RequestInfo{requestId, std::move(transState), indexFromEnd, lastBlockKey}; + auto requestInfo = RequestInfo{requestId, std::move(transState), indexFromEnd, lastBlockKey}; + requestInfo.setIsArbitraryTransfer(isArbitraryTransfer); + return requestInfo; } std::size_t RequestInfo::serializedSize(RequestInfo const& requestInfo) @@ -314,6 +335,7 @@ std::size_t RequestInfo::serializedSize(RequestInfo const& requestInfo) totalSize += su::serializedSize(requestInfo.mRequestId); totalSize += su::serializedSize(requestInfo.mIndexFromEnd); totalSize += su::serializedSize(requestInfo.mLastBlockKey); + totalSize += su::serializedSize(requestInfo.mIsArbitraryTransfer); totalSize += su::serializedSize(requestInfo.mTransState); return totalSize; } @@ -323,11 +345,13 @@ class CacheSender::Impl public: using RequestIdType = LlmRequest::RequestIdType; - Impl(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer) + Impl(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer, + std::string instanceId = "") : mManager{manager} , mSelfState{cacheLayer.getCacheState(), executor::kv_cache::CommState{manager->getCommState()}} , mCacheTransferLayer{std::move(cacheLayer)} , mBufferManager{std::make_shared<runtime::CudaStream>()} + , mInstanceId{std::move(instanceId)} { TLLM_CHECK(mManager); TLLM_CHECK(mManager->getCommState().getSelfIdx() == selfIndex); @@ -405,7 +429,7 @@ class CacheSender::Impl { if (!mMeasuresFile.is_open()) { - auto outputPath = getTransferOutputPath("send"); + auto outputPath = getTransferOutputPath("send", mInstanceId); mMeasuresFile.open(outputPath); TLLM_CHECK_WITH_INFO(mMeasuresFile.is_open(), "Failed to open transfer output file: %s", outputPath.string().c_str()); @@ -467,7 +491,6 @@ class CacheSender::Impl : mManager->recvConnect(DataContext{TransceiverTag::kID_TAG, mTerminate}, &id, sizeof(id)); if (connection == nullptr) { - TLLM_LOG_WARNING("recvRequestInfo connection is nullptr, maybe the server is terminating"); return std::nullopt; } @@ -625,10 +648,23 @@ class CacheSender::Impl private: struct Response { - // shared_ptr so this struct co-owns the request until the promise resolves; - // protects worker-side dereferences and the promise itself from premature destruction. - std::shared_ptr<LlmRequest> mRequest; + // An LlmRequest (co-owned until the promise resolves) for normal transfers, or + // just the request id for llmRequest-agnostic reuse-tree transfers. + std::variant<std::shared_ptr<LlmRequest>, RequestIdType> mRequestOrId; std::promise<void> mPromise; + std::vector<kv_cache_manager::KVCacheBlock::IdType> mPinnedBlockIds; + + [[nodiscard]] LlmRequest* getRequest() const + { + auto const* request = std::get_if<std::shared_ptr<LlmRequest>>(&mRequestOrId); + return request != nullptr ? request->get() : nullptr; + } + + [[nodiscard]] RequestIdType getRequestId() const + { + auto const* request = getRequest(); + return request != nullptr ? request->mRequestId : std::get<RequestIdType>(mRequestOrId); + } }; struct AsyncSendResource @@ -661,21 +697,44 @@ class CacheSender::Impl resp = std::move(resource.mSendQueue.front()); resource.mSendQueue.pop_front(); } - // Sequence the read before the move: argument initializations - // are indeterminately sequenced, so inlining resp.mRequest->... - // alongside std::move(resp) is UB once mRequest is a shared_ptr. - TLLM_CHECK(resp.mRequest != nullptr); - auto const reqId = resp.mRequest->mRequestId; - sendAndRemoveResponse(reqId, std::move(resp)); + // Read before std::move(resp): argument evaluations are indeterminately sequenced. + auto const requestId = resp.getRequestId(); + sendAndRemoveResponse(requestId, std::move(resp)); } } + //! Must not throw: called from noexcept send/failure paths. + void releasePinnedBlocks(Response& response) noexcept + { + if (response.mPinnedBlockIds.empty()) + { + return; + } + try + { + mCacheTransferLayer.getCacheManager()->unpinBlocksById(response.mPinnedBlockIds); + } + catch (std::exception const& err) + { + TLLM_LOG_ERROR("Failed to unpin reuse-tree blocks: %s", err.what()); + } + response.mPinnedBlockIds.clear(); + } + void sendAndRemoveResponse(RequestIdType id, Response resp) noexcept { try { TLLM_CUDA_CHECK(cudaSetDevice(mDeviceId)); - sendSync(*resp.mRequest); + if (auto* llmRequest = resp.getRequest(); llmRequest != nullptr) + { + sendSync(*llmRequest); + } + else + { + // Reuse tree path — no LlmRequest + sendSyncFromReuseTree(id); + } release(id); resp.mPromise.set_value(); } @@ -700,6 +759,7 @@ class CacheSender::Impl discardTransferState(id); failResponse(resp, exception); } + releasePinnedBlocks(resp); } void asyncSendAndRemoveResponse(RequestIdType id, Response resp) noexcept @@ -806,6 +866,37 @@ class CacheSender::Impl } } + void sendSyncFromReuseTree(RequestIdType requestId) + { + TransferSession* session = nullptr; + { + std::unique_lock<std::mutex> lk(mMtxForMap); + auto it = mRequestToSession.find(requestId); + TLLM_CHECK(it != mRequestToSession.end()); + session = std::addressof(it->second); + } + // READY was already sent by response(); the receiver consumes exactly one per transfer. + mCacheTransferLayer.format(*session); + } + + // Pin the requested chain in the reuse tree; an empty result means no full match. + // The caller must unpin once the transfer settles. + std::vector<kv_cache_manager::KVCacheBlock::IdType> pinReuseTreeBlocks(RequestIdType requestId) + { + std::unique_lock<std::mutex> lk(mMtxForMap); + auto it = mRequestToSession.find(requestId); + auto const& lastBlockKey = it->second.getLastBlockKey(); + auto* cacheManager = mCacheTransferLayer.getCacheManager(); + auto windowSize = cacheManager->getBlockManager().getWindowSizesMetadata().begin()->first; + std::vector<kv_cache_manager::KVCacheBlock::IdType> pinnedIds; + auto lastBlock = cacheManager->findBlocksInReuseTreeByBlockKey(lastBlockKey, windowSize, pinnedIds); + if (lastBlock == nullptr) + { + return {}; + } + return pinnedIds; + } + void response() noexcept { std::exception_ptr responseException; @@ -815,16 +906,13 @@ class CacheSender::Impl TLLM_CUDA_CHECK(cudaSetDevice(mDeviceId)); while (true) { + if (mTerminate) { - std::unique_lock lock(mSenderMutex); - mSenderCv.wait(lock, - [this]() { return mTerminate || !mReadyResponses.empty() || !mCancelledRequests.empty(); }); - if (mTerminate) - { - break; - } + break; } + // Arbitrary transfers arrive without a pre-registered response; do not gate on + // mReadyResponses. auto requestInfo = recvRequestInfo(); if (!requestInfo.has_value() || mTerminate || !mManager->isRunning()) { @@ -837,22 +925,70 @@ class CacheSender::Impl mRemainSendCount[reqId] = getCounterpartsCount(reqId); } + if (requestInfo->isArbitraryTransfer()) { - std::unique_lock lock(mSenderMutex); - mCurrentRequest = reqId; - mSenderCv.wait(lock, - [this, reqId]() + // No LlmRequest will ever be registered; serve from the reuse tree off-thread. + { + std::scoped_lock lock(mSenderMutex); + mCurrentRequest = reqId; + } + auto countIt = mRemainSendCount.find(reqId); + auto const count = --countIt->second; + TLLM_CHECK(count >= 0); + if (count == 0) + { + mRemainSendCount.erase(countIt); + auto pinnedIds = pinReuseTreeBlocks(reqId); + if (pinnedIds.empty()) + { + TLLM_LOG_ERROR( + "Requested blocks do not exist in the source's reuse tree (request id: %lu). Notifying " + "receiver.", + reqId); + sendReadySignal(reqId, false); + discardTransferState(reqId); + } + else { - return mTerminate || mReadyResponses.find(reqId) != mReadyResponses.end() - || mCancelledRequests.find(reqId) != mCancelledRequests.end(); - }); - if (mTerminate) + sendReadySignal(reqId, true); + std::promise<void> promise; + // Id-only response: the reuse-tree path has no LlmRequest. + Response resp{reqId, std::move(promise), std::move(pinnedIds)}; + if (dynamic_cast<executor::kv_cache::AgentConnectionManager*>(mManager) != nullptr) + { + sendAndRemoveResponse(reqId, std::move(resp)); + } + else + { + asyncSendAndRemoveResponse(reqId, std::move(resp)); + } + } + } { + std::scoped_lock lock(mSenderMutex); mCurrentRequest = std::nullopt; - break; } } - sendResponse(reqId); + else + { + // The RequestInfo may race ahead of sendAsync; wait for the specific response. + { + std::unique_lock lock(mSenderMutex); + mCurrentRequest = reqId; + mSenderCv.wait(lock, + [this, reqId]() + { + return mTerminate || mReadyResponses.find(reqId) != mReadyResponses.end() + || mCancelledRequests.find(reqId) != mCancelledRequests.end(); + }); + if (mTerminate) + { + mCurrentRequest = std::nullopt; + break; + } + } + sendResponse(reqId); + } } } catch (std::exception const& err) @@ -930,6 +1066,7 @@ class CacheSender::Impl { TLLM_LOG_ERROR("Failed to set CacheSender response exception: %s", err.what()); } + releasePinnedBlocks(response); } void failPendingResponses(std::exception_ptr const& exception) noexcept @@ -950,8 +1087,8 @@ class CacheSender::Impl public: void setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, - nvinfer1::DataType ssmStateDataType) + std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, + tensorrt_llm::DataType ssmStateDataType) { mCacheTransferLayer.setRnnConfig(rnnModelConfig, rnnLayerNumPerPP, convStateDataType, ssmStateDataType); mSelfState.setCacheState(mCacheTransferLayer.getCacheState()); @@ -979,16 +1116,19 @@ class CacheSender::Impl std::ofstream mMeasuresFile; std::mutex mInFlightCancelMutex; std::unordered_map<LlmRequest::RequestIdType, std::shared_ptr<std::atomic<bool>>> mInFlightCancelFlags; + std::string mInstanceId; }; class CacheReceiver::Impl { public: - Impl(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer) + Impl(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer, + std::string instanceId = "") : mManager{manager} , mSelfState{cacheLayer.getCacheState(), executor::kv_cache::CommState{manager->getCommState()}} , mCacheTransferLayer{std::move(cacheLayer)} , mBufferManager{std::make_shared<runtime::CudaStream>()} + , mInstanceId{std::move(instanceId)} { TLLM_CHECK(mManager); TLLM_CHECK(mManager->getCommState().getSelfIdx() == selfIndex); @@ -1056,7 +1196,7 @@ class CacheReceiver::Impl std::unique_lock<std::mutex> lock(mMeasuresFileMutex); if (!mMeasuresFile.is_open()) { - auto outputPath = getTransferOutputPath("recv"); + auto outputPath = getTransferOutputPath("recv", mInstanceId); mMeasuresFile.open(outputPath); TLLM_CHECK_WITH_INFO(mMeasuresFile.is_open(), "Failed to open transfer output file: %s", outputPath.string().c_str()); @@ -1115,6 +1255,9 @@ class CacheReceiver::Impl requestInfo = RequestInfo(requestId, mSelfState, indexFromEnd, lastBlockKey); } } + // The state's provenance marks llmRequest-agnostic transfers: only + // getSerializedDataTransceiverState sets it; context responses leave it unset. + requestInfo.setIsArbitraryTransfer(contextState.isArbitraryTransferState()); auto* agentConnectionManager = dynamic_cast<executor::kv_cache::AgentConnectionManager*>(mManager); std::vector<BufferIndexHolder> recvHolders; @@ -1697,8 +1840,8 @@ class CacheReceiver::Impl public: void setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, - nvinfer1::DataType ssmStateDataType) + std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, + tensorrt_llm::DataType ssmStateDataType) { mCacheTransferLayer.setRnnConfig(rnnModelConfig, rnnLayerNumPerPP, convStateDataType, ssmStateDataType); mSelfState.setCacheState(mCacheTransferLayer.getCacheState()); @@ -1720,6 +1863,7 @@ class CacheReceiver::Impl std::atomic<bool> mTerminate{false}; std::mutex mInFlightCancelMutex; std::unordered_map<LlmRequest::RequestIdType, std::shared_ptr<std::atomic<bool>>> mInFlightCancelFlags; + std::string mInstanceId; }; void CacheSender::ImplDeleter::operator()(Impl* ptr) @@ -1732,9 +1876,10 @@ void CacheReceiver::ImplDeleter::operator()(Impl* ptr) delete ptr; } -CacheSender::CacheSender( - executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer) - : mImpl{std::unique_ptr<Impl, ImplDeleter>(new Impl(manager, selfIndex, std::move(cacheLayer)))} +CacheSender::CacheSender(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, + CacheTransferLayer cacheLayer, std::string instanceId) + : mImpl{ + std::unique_ptr<Impl, ImplDeleter>(new Impl(manager, selfIndex, std::move(cacheLayer), std::move(instanceId)))} { } @@ -1778,14 +1923,16 @@ void CacheSender::sendReadySignal(LlmRequest::RequestIdType requestId, bool isRe } void CacheSender::setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, nvinfer1::DataType ssmStateDataType) + std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, + tensorrt_llm::DataType ssmStateDataType) { mImpl->setRnnConfig(std::move(rnnModelConfig), std::move(rnnLayerNumPerPP), convStateDataType, ssmStateDataType); } -CacheReceiver::CacheReceiver( - executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer) - : mImpl{std::unique_ptr<Impl, ImplDeleter>(new Impl(manager, selfIndex, std::move(cacheLayer)))} +CacheReceiver::CacheReceiver(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, + CacheTransferLayer cacheLayer, std::string instanceId) + : mImpl{ + std::unique_ptr<Impl, ImplDeleter>(new Impl(manager, selfIndex, std::move(cacheLayer), std::move(instanceId)))} { } @@ -1817,7 +1964,8 @@ bool CacheReceiver::receiveReadySignal(TransferSession& session) } void CacheReceiver::setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, nvinfer1::DataType ssmStateDataType) + std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, + tensorrt_llm::DataType ssmStateDataType) { mImpl->setRnnConfig(std::move(rnnModelConfig), std::move(rnnLayerNumPerPP), convStateDataType, ssmStateDataType); } diff --git a/cpp/tensorrt_llm/batch_manager/dataTransceiver.h b/cpp/tensorrt_llm/batch_manager/dataTransceiver.h index 8e84a71556af..778e5e80c7f4 100644 --- a/cpp/tensorrt_llm/batch_manager/dataTransceiver.h +++ b/cpp/tensorrt_llm/batch_manager/dataTransceiver.h @@ -19,6 +19,7 @@ #include <fstream> #include <future> #include <map> +#include <optional> #include <string> #include <vector> @@ -29,6 +30,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cacheCommunicator.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/serializeUtils.h" @@ -120,7 +122,7 @@ class TransferSession void recv(size_t idx, void* data, size_t size); - [[nodiscard]] LlmRequest const& getLlmRequest() const; + [[nodiscard]] std::optional<LlmRequest const*> getLlmRequest() const; // in CacheSender, the LlmRequest is not available until the sendSync is called void setLlmRequest(LlmRequest const& llmRequest); @@ -233,6 +235,17 @@ class RequestInfo return mLastBlockKey; } + /// @brief Arbitrary (llmRequest-agnostic) transfer served from the sender's reuse tree. + [[nodiscard]] bool isArbitraryTransfer() const noexcept + { + return mIsArbitraryTransfer; + } + + void setIsArbitraryTransfer(bool isArbitraryTransfer) noexcept + { + mIsArbitraryTransfer = isArbitraryTransfer; + } + /// @brief Serialization. /// @param requestInfo Request information to be serialized. /// @param os The output stream to which the serialization result points. @@ -256,6 +269,9 @@ class RequestInfo // Last block key, used to derive other block keys on receiver BlockKey mLastBlockKey{}; + // True for arbitrary (llmRequest-agnostic) transfers served from the sender's reuse tree. + bool mIsArbitraryTransfer{false}; + // The state of the data transceiver. executor::DataTransceiverState mTransState; }; @@ -267,7 +283,8 @@ class CacheSender /// @param manager The connection manager. /// @param selfIndex The sequential index of the current executor process. /// @param cacheLayer The cache layer bundling all cache states and formatters. - CacheSender(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer); + CacheSender(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer, + std::string instanceId = ""); CacheSender() = default; @@ -306,8 +323,8 @@ class CacheSender /// @brief Update the RNN config on the internal CacheState copies. /// Used by CppMambaHybridCacheManager path where RNN config is set after construction. void setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, - nvinfer1::DataType ssmStateDataType); + std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, + tensorrt_llm::DataType ssmStateDataType); /// @brief Destructor. virtual ~CacheSender(); @@ -330,7 +347,8 @@ class CacheReceiver /// @param manager The connection manager. /// @param selfIndex The sequential index of the current executor process. /// @param cacheLayer The cache layer bundling all cache states and formatters. - CacheReceiver(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer); + CacheReceiver(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer, + std::string instanceId = ""); CacheReceiver() = default; @@ -357,8 +375,8 @@ class CacheReceiver /// @brief Update the RNN config on the internal CacheState copies. /// Used by CppMambaHybridCacheManager path where RNN config is set after construction. void setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, - nvinfer1::DataType ssmStateDataType); + std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, + tensorrt_llm::DataType ssmStateDataType); /// @brief Destructor. virtual ~CacheReceiver(); diff --git a/cpp/tensorrt_llm/batch_manager/decoderBuffers.cpp b/cpp/tensorrt_llm/batch_manager/decoderBuffers.cpp index fd67bb55e89d..fecc0851d361 100644 --- a/cpp/tensorrt_llm/batch_manager/decoderBuffers.cpp +++ b/cpp/tensorrt_llm/batch_manager/decoderBuffers.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/batch_manager/decoderBuffers.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/decoderState.h" @@ -70,21 +71,21 @@ DecoderOutputBuffers::DecoderOutputBuffers(SizeType32 maxNumSequences, SizeType3 auto constexpr TRTTokenIdType = runtime::TRTDataType<runtime::TokenIdType>::value; sequenceLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), tensorrt_llm::DataType::kINT32); - finishedSumHost = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + finishedSumHost = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); newOutputTokensHost = BufferManager::pinned(ITensor::makeShape({maxTokensPerStep, maxNumSequences, maxBeamWidth}), TRTTokenIdType); cumLogProbsHost - = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), tensorrt_llm::DataType::kFLOAT); logProbsHost = BufferManager::pinned( - ITensor::makeShape({maxNumSequences, maxBeamWidth, maxSeqLen}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({maxNumSequences, maxBeamWidth, maxSeqLen}), tensorrt_llm::DataType::kFLOAT); finishReasonsHost - = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), nvinfer1::DataType::kUINT8); + = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), tensorrt_llm::DataType::kUINT8); } void DecoderOutputBuffers::enableLookaheadDecoding(SizeType32 maxNumSequences, SizeType32 maxTokensPerStep) @@ -115,9 +116,9 @@ void DecoderOutputBuffers::setupSpeculativeDecoding( if (speculativeDecodingMode.variableDraftLength()) { nextDraftTokensLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); prevDraftTokensLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); } } } @@ -307,17 +308,18 @@ DecoderSlotAsyncSend::~DecoderSlotAsyncSend() SlotDecoderBuffers::SlotDecoderBuffers(SizeType32 maxBeamWidth, SizeType32 maxSeqLen, BufferManager const& manager) { - outputIds = manager.gpu(ITensor::makeShape({maxBeamWidth, maxSeqLen}), nvinfer1::DataType::kINT32); - outputIdsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth, maxSeqLen}), nvinfer1::DataType::kINT32); + outputIds = manager.gpu(ITensor::makeShape({maxBeamWidth, maxSeqLen}), tensorrt_llm::DataType::kINT32); + outputIdsHost + = BufferManager::pinned(ITensor::makeShape({maxBeamWidth, maxSeqLen}), tensorrt_llm::DataType::kINT32); - sequenceLengths = manager.gpu(ITensor::makeShape({maxBeamWidth}), nvinfer1::DataType::kINT32); - sequenceLengthsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth}), nvinfer1::DataType::kINT32); + sequenceLengths = manager.gpu(ITensor::makeShape({maxBeamWidth}), tensorrt_llm::DataType::kINT32); + sequenceLengthsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth}), tensorrt_llm::DataType::kINT32); - cumLogProbs = manager.gpu(ITensor::makeShape({maxBeamWidth}), nvinfer1::DataType::kFLOAT); - cumLogProbsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth}), nvinfer1::DataType::kFLOAT); + cumLogProbs = manager.gpu(ITensor::makeShape({maxBeamWidth}), tensorrt_llm::DataType::kFLOAT); + cumLogProbsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth}), tensorrt_llm::DataType::kFLOAT); - logProbs = manager.gpu(ITensor::makeShape({maxBeamWidth, maxSeqLen}), nvinfer1::DataType::kFLOAT); - logProbsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth, maxSeqLen}), nvinfer1::DataType::kFLOAT); + logProbs = manager.gpu(ITensor::makeShape({maxBeamWidth, maxSeqLen}), tensorrt_llm::DataType::kFLOAT); + logProbsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth, maxSeqLen}), tensorrt_llm::DataType::kFLOAT); } } // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/encoderBuffers.cpp b/cpp/tensorrt_llm/batch_manager/encoderBuffers.cpp deleted file mode 100644 index 56fd393c68d7..000000000000 --- a/cpp/tensorrt_llm/batch_manager/encoderBuffers.cpp +++ /dev/null @@ -1,560 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "encoderBuffers.h" - -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/iTensor.h" - -#include <valarray> - -using namespace tensorrt_llm::runtime; - -namespace tensorrt_llm::batch_manager -{ - -EncoderBuffers::EncoderBuffers( - SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, TllmRuntime const& runtime) -{ - // init empty buffers on cpu/gpu/pinned - init(maxBatchSize, modelConfig, worldConfig, runtime); - - // pre-allocate based on max buffer sizes - // Note: pre-allocation can be done directly instead of empty-->reshape, but it is ok extract the common reshape() - // utility because the buffer shapes can be dynamically set during runtime as well - initBufferSizes(maxBatchSize, modelConfig, worldConfig, runtime); -} - -void EncoderBuffers::init( - SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, TllmRuntime const& runtime) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const& manager = runtime.getBufferManager(); - - auto hiddenStatesType = modelConfig.getDataType(); - - inputFeatures = manager.emptyTensor(MemoryType::kGPU, hiddenStatesType); - inputIds = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - - // in PP, only rank 0 needs the following input fields - if (modelConfig.usePositionEmbedding() && worldConfig.isFirstPipelineParallelRank()) - { - positionIds = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - positionIdsReserved.resize(maxBatchSize * modelConfig.getMaxInputLen()); - std::iota(positionIdsReserved.begin(), positionIdsReserved.end(), 0); - } - if (modelConfig.useTokenTypeEmbedding() && worldConfig.isFirstPipelineParallelRank()) - { - tokenTypeIds = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - tokenTypeIdsReserved.resize(maxBatchSize * modelConfig.getMaxInputLen()); - std::fill(tokenTypeIdsReserved.begin(), tokenTypeIdsReserved.end(), 0); - } - - inputLengths = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - maxInputLength = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - - if (worldConfig.isPipelineParallel()) - { - hiddenStates = manager.emptyTensor(MemoryType::kGPU, hiddenStatesType); - } - if (worldConfig.isLastPipelineParallelRank()) - { - encoderOutput = manager.emptyTensor(MemoryType::kGPU, hiddenStatesType); - } - - if (modelConfig.useLanguageAdapter()) - { - languageAdapterRoutings = manager.emptyTensor(MemoryType::kGPU, TRTDataType<SizeType32>::value); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::initBufferSizes( - SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, TllmRuntime const& runtime) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // get buffer shape based on max values - numRequests = maxBatchSize; - encoderInputLen = maxBatchSize * modelConfig.getMaxInputLen(); - encoderOutputLen = maxBatchSize * modelConfig.getMaxInputLen(); // assume output length <= input length - maxInputLengthInBatch = modelConfig.getMaxInputLen(); - - // update buffer shapes - reshape(runtime, modelConfig, worldConfig); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::updateBufferSizes(RequestVector const& requests, ModelConfig const& modelConfig, - WorldConfig const& worldConfig, TllmRuntime const& runtime) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - numRequests = requests.size(); - encoderInputLen = 0; - encoderOutputLen = 0; - maxInputLengthInBatch = 0; - - // get buffer shape based on actual batched requests - for (auto const& req : requests) - { - encoderInputLen += req->getEncoderInputLen(); - encoderOutputLen += req->getEncoderOutputLen(); - maxInputLengthInBatch - = std::max(maxInputLengthInBatch, req->getEncoderInputLen()); // Decoder input is encoder output - } - - // update buffer shapes - reshape(runtime, modelConfig, worldConfig); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::reshape(TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - if (modelConfig.isMultiModal()) - { - return; // multimodal models do not need to set position id, etc. or any output tensors - } - - inputIds->reshape(ITensor::makeShape({encoderInputLen})); - if (positionIds) - { - if (modelConfig.isWhisper()) - { - positionIds->reshape(ITensor::makeShape({encoderOutputLen})); - } - else - { - positionIds->reshape(ITensor::makeShape({encoderInputLen})); - } - } - if (tokenTypeIds) - { - tokenTypeIds->reshape(ITensor::makeShape({encoderInputLen})); - } - - inputLengths->reshape(ITensor::makeShape({numRequests})); - maxInputLength->reshape(ITensor::makeShape({maxInputLengthInBatch})); - - if (worldConfig.isPipelineParallel()) - { - hiddenStates->reshape( - ITensor::makeShape({encoderOutputLen, modelConfig.getHiddenSize() * worldConfig.getTensorParallelism()})); - } - if (worldConfig.isLastPipelineParallelRank()) - { - encoderOutput->reshape( - ITensor::makeShape({encoderOutputLen, modelConfig.getHiddenSize() * worldConfig.getTensorParallelism()})); - } - if (modelConfig.useLanguageAdapter()) - { - languageAdapterRoutings->reshape(ITensor::makeShape({encoderInputLen, 1})); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::setFromInputs(RequestVector const& requests, ModelConfig const& modelConfig, - WorldConfig const& worldConfig, TllmRuntime const& runtime) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(encoderBuffersSetFromInputs); - - if (!worldConfig.isFirstPipelineParallelRank()) - { - return; - } - - auto const& manager = runtime.getBufferManager(); - - std::vector<TokenIdType> inputIdsAll; - std::vector<SizeType32> positionIdsAll; - std::vector<SizeType32> tokenTypeIdsAll; - std::vector<SizeType32> inputLengthsAll; - std::vector<SizeType32> languageAdapterRoutingAll; - // use shape to indicates max input length, content is not important - // TODO: change to a scalar value for this from engine side - std::vector<SizeType32> maxInputLengthAll(maxInputLengthInBatch, 0); - - if (requests.front()->getEncoderInputFeatures()) - { - if (modelConfig.isMultiModal()) - { - auto batchedInputShape = requests.front()->getEncoderInputFeatures()->getShape(); // [1, 3, H, W] - batchedInputShape.d[0] = encoderInputLen; // [batch_size, 3, H, W] - inputFeatures->reshape(batchedInputShape); - } - else - { - SizeType32 const featureDim = requests.front()->getEncoderInputFeatures()->getShape().d[1]; - TLLM_LOG_DEBUG("EncoderBuffers::setFromInputs - featureDim = %d", featureDim); - inputFeatures->reshape(ITensor::makeShape({encoderInputLen, featureDim})); - } - } - - SizeType32 offset = 0; - - for (auto const& llmReq : requests) - { - SizeType32 const inputLength = llmReq->getEncoderInputLen(); - SizeType32 const outputLength = llmReq->getEncoderOutputLen(); - if (llmReq->getEncoderInputFeatures()) - { - auto const& reqFeatures - = llmReq - ->getEncoderInputFeatures(); // whisper: [length, featureDim]; Vision: [batch_size, channel, W, H] - TLLM_LOG_DEBUG("EncoderBuffers::setFromInputs - request id = %d, input features length = %d", - llmReq->mRequestId, inputLength); - manager.copy(*reqFeatures, *ITensor::slice(inputFeatures, offset, inputLength)); - offset += inputLength; - } - else - { - auto const& reqTokens = *llmReq->getEncoderTokens().value(); - inputIdsAll.insert(inputIdsAll.end(), reqTokens.begin(), reqTokens.end()); - if (tokenTypeIds) - { - tokenTypeIdsAll.insert( - tokenTypeIdsAll.end(), tokenTypeIdsReserved.begin(), tokenTypeIdsReserved.begin() + inputLength); - } - } - if (positionIds) - { - SizeType32 const length = modelConfig.isWhisper() ? outputLength : inputLength; - positionIdsAll.insert( - positionIdsAll.end(), positionIdsReserved.begin(), positionIdsReserved.begin() + length); - } - if (modelConfig.useLanguageAdapter()) - { - auto const languageAdapterRouting - = llmReq->getLanguageAdapterRouting(modelConfig.getNumLanguages().value(), inputLength); - languageAdapterRoutingAll.insert( - languageAdapterRoutingAll.end(), std::begin(languageAdapterRouting), std::end(languageAdapterRouting)); - } - inputLengthsAll.push_back(inputLength); - } - - // copy inputs from host to device - { - NVTX3_SCOPED_RANGE(bufferCopies); - if (requests.front()->getEncoderTokens()) - { - manager.copy(inputIdsAll.data(), *inputIds); - if (tokenTypeIds) - { - manager.copy(tokenTypeIdsAll.data(), *tokenTypeIds); - } - manager.copy(maxInputLengthAll.data(), *maxInputLength); - } - if (positionIds) - { - manager.copy(positionIdsAll.data(), *positionIds); - } - manager.copy(inputLengthsAll.data(), *inputLengths); - if (modelConfig.useLanguageAdapter()) - { - manager.copy(languageAdapterRoutingAll.data(), *languageAdapterRoutings); - } - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::fillIOMaps(ModelConfig const& modelConfig, WorldConfig const& worldConfig) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(runtimeBuffersFillIOMaps); - - inputMap.clear(); - outputMap.clear(); - - // inputs - if (modelConfig.isMultiModal()) - { - inputMap.insert_or_assign("input", inputFeatures); - } - else if (modelConfig.isWhisper()) - { - inputMap.insert_or_assign("input_features", inputFeatures); - inputMap.insert_or_assign("input_lengths", inputLengths); - inputMap.insert_or_assign("position_ids", positionIds); - } - else - { - if (worldConfig.isFirstPipelineParallelRank()) - { - inputMap.insert_or_assign("input_ids", inputIds); - if (positionIds) - { - inputMap.insert_or_assign("position_ids", positionIds); - } - if (tokenTypeIds) - { - inputMap.insert_or_assign("token_type_ids", tokenTypeIds); - } - } - else - { - inputMap.insert_or_assign("hidden_states_input", hiddenStates); - } - inputMap.insert_or_assign("input_lengths", inputLengths); - inputMap.insert_or_assign("max_input_length", maxInputLength); - if (modelConfig.useLanguageAdapter()) - { - inputMap.insert_or_assign("language_adapter_routings", languageAdapterRoutings); - } - } - - // outputs - if (worldConfig.isLastPipelineParallelRank()) - { - outputMap.insert_or_assign("encoder_output", encoderOutput); - } - else - { - outputMap.insert_or_assign("hidden_states_output", hiddenStates); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -std::pair<EncoderBuffers::TensorMap const&, EncoderBuffers::TensorMap&> EncoderBuffers::prepareIO( - RequestVector const& requests, ModelConfig const& modelConfig, WorldConfig const& worldConfig, - TllmRuntime const& runtime) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - updateBufferSizes(requests, modelConfig, worldConfig, runtime); - - setFromInputs(requests, modelConfig, worldConfig, runtime); - - fillIOMaps(modelConfig, worldConfig); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - - return {inputMap, outputMap}; -} - -void EncoderBuffers::rearrangeOutputs(RequestVector const& requests, ModelConfig const& modelConfig, - WorldConfig const& worldConfig, TllmRuntime const& runtime) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(encoderBuffersRearrangeOutput); - - auto const& manager = runtime.getBufferManager(); - - SizeType32 offset = 0, size = 0; - - updateReqOutputShape(requests, runtime, worldConfig, modelConfig); - - for (auto const& req : requests) - { - // copy from internal buffer to request-owned external buffers - size = req->getEncoderOutputLen(); - TLLM_LOG_DEBUG("EncoderBuffers::rearrangeOutputs - req: %d, encoderOutput shape = (%d, %d)", req->mClientId, - req->getEncoderOutput()->getShape().d[0], req->getEncoderOutput()->getShape().d[1]); - TLLM_LOG_DEBUG("EncoderBuffers::rearrangeOutputs - req: %d, enc output size = %d", req->mClientId, size); - - if (worldConfig.isPipelineParallel()) - { - manager.copy(*ITensor::slice(hiddenStates, offset, size), *req->getEncoderHiddenStates()); - } - if (worldConfig.isLastPipelineParallelRank()) - { - if (modelConfig.isMultiModal()) - { - manager.copy( - *ITensor::slice(encoderOutput, offset, size), *(req->getPromptEmbeddingTableMutable().value())); - } - else - { - manager.copy(*ITensor::slice(encoderOutput, offset, size), *req->getEncoderOutput()); - } - } - offset += size; - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::updateReqOutputShape(RequestVector const& requests, TllmRuntime const& runtime, - WorldConfig const& worldConfig, ModelConfig const& modelConfig) -{ - auto const& manager = runtime.getBufferManager(); - - for (auto const& req : requests) - { - if (modelConfig.isMultiModal()) - { - auto shape = encoderOutput->getShape(); // [batch_size, prompt_vocab_size, feature_dim] - shape.d[0] = req->getEncoderOutputLen(); - req->getPromptEmbeddingTableMutable() = manager.emptyTensor(MemoryType::kGPU, encoderOutput->getDataType()); - req->getPromptEmbeddingTableMutable().value()->reshape(shape); - req->setPromptVocabSize(shape.d[1]); - // TODO: extra ids for kv cache reuse - } - else - { - auto encOutLen = req->getEncoderOutputLen(); - // update request-owned external buffer for each request - if (worldConfig.isPipelineParallel()) - { - req->getEncoderHiddenStates()->reshape( - ITensor::makeShape({encOutLen, modelConfig.getHiddenSize() * worldConfig.getTensorParallelism()})); - } - if (worldConfig.isLastPipelineParallelRank()) - { - req->getEncoderOutput()->reshape( - ITensor::makeShape({encOutLen, modelConfig.getHiddenSize() * worldConfig.getTensorParallelism()})); - } - } - } -} - -void EncoderBuffers::create(SizeType32 maxBatchSize, ModelConfig const& modelConfig, TllmRuntime const& runtime) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const& manager = runtime.getBufferManager(); - - inputLengths = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - maxInputLength = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - - hiddenSize = modelConfig.getEncoderHiddenSize(); // full hidden size - // assume encoder & decoder use the same data type - encoderOutput = manager.emptyTensor(MemoryType::kGPU, modelConfig.getDataType()); - encoderOutputReserved = manager.gpu(ITensor::makeShape({1, hiddenSize}), modelConfig.getDataType()); - - crossKvCacheGen = manager.gpu(ITensor::makeShape({1}), nvinfer1::DataType::kBOOL); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::setMaxBufferSizes(SizeType32 maxBatchSize, runtime::ModelConfig const& modelConfig) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - numRequests = maxBatchSize; - encoderInputLen = maxBatchSize * modelConfig.getMaxEncoderLen(); - encoderOutputLen = maxBatchSize * modelConfig.getMaxEncoderLen(); - maxInputLengthInBatch = modelConfig.getMaxEncoderLen(); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::setBufferSizes(RequestVector const& contextRequests, RequestVector const& genRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - numRequests = 0; /// total number of requests that need encoder information (context requests + - /// generation requests * beam width) - encoderInputLen = 0; - encoderOutputLen = 0; - maxInputLengthInBatch = 1; /// maximum encoder length in a batch - - for (auto const& llmReq : contextRequests) - { - numRequests += 1; - encoderInputLen += llmReq->getEncoderInputLen(); - encoderOutputLen += llmReq->getEncoderOutputLen(); - maxInputLengthInBatch = std::max(maxInputLengthInBatch, llmReq->getEncoderInputLen()); - } - - for (auto const& llmReq : genRequests) - { - auto const reqBeamWidth = llmReq->getBeamWidthByIter(); - numRequests += reqBeamWidth; // tile by beam width - maxInputLengthInBatch = std::max(maxInputLengthInBatch, llmReq->getEncoderInputLen()); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::reshape() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - inputLengths->reshape(ITensor::makeShape({numRequests})); - maxInputLength->reshape(ITensor::makeShape({maxInputLengthInBatch})); - encoderOutput->reshape(ITensor::makeShape({encoderOutputLen, hiddenSize})); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::fill( - RequestVector const& ctxRequests, RequestVector const& genRequests, runtime::BufferManager const& manager) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(encoderBufferCopies); - - std::vector<SizeType32> inputLengthsAll; - std::vector<SizeType32> maxInputLengthAll(maxInputLength->getShape().d[0], 0); - - SizeType32 offset = 0, size = 0; - for (auto const& requests : {ctxRequests, genRequests}) - { - for (auto const& llmReq : requests) - { - // 1. only ctx requests should gather the encoder output - // 2. only gen requests should tile encoder input lengths info by beam width - bool isCtx = llmReq->isContextInitState(); - if (isCtx) - { - size = llmReq->getEncoderOutputLen(); - auto const encoderOutputSlice = runtime::ITensor::slice(encoderOutput, offset, size); - manager.copy(*llmReq->getEncoderOutput(), *encoderOutputSlice); - offset += size; - - inputLengthsAll.emplace_back(size); - } - else - { - auto const reqBeamWidth = llmReq->getBeamWidthByIter(); - std::fill_n(std::back_inserter(inputLengthsAll), reqBeamWidth, - llmReq->getEncoderOutputLen()); // although encoder output is not needed, gen phase still needs the - // encoder length info for cross kv cache. Also tile by beam width - } - } - } - manager.copy(inputLengthsAll.data(), *inputLengths); - manager.copy(maxInputLengthAll.data(), *maxInputLength); - // crossKvCacheGen unused in engine for now, use default tensor - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EncoderBuffers::insertInputTensors(TensorMap& inputMap) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - inputMap.insert_or_assign("encoder_output", encoderOutput); - inputMap.insert_or_assign("encoder_input_lengths", inputLengths); - inputMap.insert_or_assign("encoder_max_input_length", maxInputLength); - inputMap.insert_or_assign("cross_kv_cache_gen", crossKvCacheGen); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/encoderBuffers.h b/cpp/tensorrt_llm/batch_manager/encoderBuffers.h deleted file mode 100644 index 64d416280f21..000000000000 --- a/cpp/tensorrt_llm/batch_manager/encoderBuffers.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -namespace tensorrt_llm::batch_manager -{ - -class EncoderBuffers -{ -public: - using SizeType32 = tensorrt_llm::runtime::SizeType32; - using ITensor = tensorrt_llm::runtime::ITensor; - using TensorPtr = runtime::ITensor::SharedPtr; - using TensorMap = runtime::StringPtrMap<runtime::ITensor>; - using ModelConfig = runtime::ModelConfig; - using WorldConfig = runtime::WorldConfig; - using TllmRuntime = runtime::TllmRuntime; - - TensorPtr inputIds; - TensorPtr positionIds = nullptr; - TensorPtr tokenTypeIds = nullptr; - - TensorPtr inputLengths; // [numEncoderRequests] - TensorPtr maxInputLength; // [maxInputLengthInBatch] - - // intermediate states in pipeline parallelism - TensorPtr hiddenStates; // [numTokens, hiddenSize] - - // features for multimodal encoders (audio, image, etc.) - TensorPtr - inputFeatures; // [totalNumOfFeatures, featureDim] if remove_padding else [batchSize, featureDim, featureLength] - - // language adapter routing information for encoders if language adapter is presented. - TensorPtr languageAdapterRoutings; // [numTokens, numLanguages] - - // encoder output - TensorPtr encoderOutput; // [numEncoderTokens, hiddenSize] - - // output buffer owned by llmRequest, such that it's per-request output buffer - // encoderBuffers class can init and reshape each buffer, without maintaining a list/set of inflight buffers - // TODO in progress: to support BS>1 encoder, need (1) internal scratch space tensors to save the contiguous - // batched output (2) copy from CONTIGUOUS scratch tensor to individual request's DISCRETE output tensor after - // execution To standardize the implementation, for both BS=1 and BS>1, we use internal buffer to store BS=1/BS>1 - // results, and copy to request's external buffers. For BS=1, this introduces a redundancy copy, but ok for now. - - EncoderBuffers() = default; - EncoderBuffers(SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, - TllmRuntime const& runtime); - - std::pair<EncoderBuffers::TensorMap const&, EncoderBuffers::TensorMap&> prepareIO(RequestVector const& requests, - ModelConfig const& modelConfig, WorldConfig const& worldConfig, TllmRuntime const& runtime); - - void rearrangeOutputs(RequestVector const& requests, ModelConfig const& modelConfig, WorldConfig const& worldConfig, - TllmRuntime const& runtime); - - //! @brief set shape of individual request's encoder output (Ptuning embedding table if multimodal) - void updateReqOutputShape(RequestVector const& requests, TllmRuntime const& runtime, WorldConfig const& worldConfig, - ModelConfig const& modelConfig); - -private: - SizeType32 numRequests{}; - SizeType32 encoderInputLen{}; - SizeType32 encoderOutputLen{}; - SizeType32 maxInputLengthInBatch{}; // max input length in a batch - - // prefilled with deterministic values to avoid runtime creation - std::vector<SizeType32> positionIdsReserved; - std::vector<SizeType32> tokenTypeIdsReserved; - - // engine I/O - TensorMap inputMap; - TensorMap outputMap; - - void init(SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, - TllmRuntime const& runtime); - - //! @brief pre-allocate max buffer sizes during init - void initBufferSizes(SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, - TllmRuntime const& runtime); - - //! @brief update actual buffer usage of requests during runtime - void updateBufferSizes(RequestVector const& requests, ModelConfig const& modelConfig, - WorldConfig const& worldConfig, TllmRuntime const& runtime); - - void reshape(TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig); - - void setFromInputs(RequestVector const& requests, ModelConfig const& modelConfig, WorldConfig const& worldConfig, - TllmRuntime const& runtime); - - void fillIOMaps(ModelConfig const& modelConfig, WorldConfig const& worldConfig); - - // additional members that are Encoder-Decoder specific -private: - TensorPtr encoderOutputReserved; // [1, hiddenSize], dummy tensor for gen phase - TensorPtr crossKvCacheGen; // [1] - SizeType32 hiddenSize; // full hidden size (after multiplying tensor parallelism) - -public: - void create(SizeType32 maxBatchSize, ModelConfig const& modelConfig, TllmRuntime const& runtime); - - SizeType32 getMaxInputLengthInBatch() const - { - return maxInputLengthInBatch; - }; - - void setMaxBufferSizes(SizeType32 maxBatchSize, runtime::ModelConfig const& modelConfig); - - void setBufferSizes(RequestVector const& contextRequests, RequestVector const& genRequests); - - void reshape(); - - void fill( - RequestVector const& ctxRequests, RequestVector const& genRequests, runtime::BufferManager const& manager); - - void insertInputTensors(TensorMap& inputMap); -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/guidedDecoder.cpp b/cpp/tensorrt_llm/batch_manager/guidedDecoder.cpp deleted file mode 100644 index cb2264ec8003..000000000000 --- a/cpp/tensorrt_llm/batch_manager/guidedDecoder.cpp +++ /dev/null @@ -1,224 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/guidedDecoder.h" -#include "tensorrt_llm/batch_manager/decoderBuffers.h" -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/kernels/logitsBitmask.h" - -#include <nlohmann/json.hpp> -#include <xgrammar/xgrammar.h> - -using namespace tensorrt_llm::runtime; - -namespace tensorrt_llm::batch_manager -{ - -GuidedDecoder::GuidedDecoder(executor::GuidedDecodingConfig const& guidedDecodingConfig, SizeType32 maxNumSequences, - SizeType32 vocabSizePadded, nvinfer1::DataType logitsDtype, BufferManager const& runtimeBufferManager) - : mGuidedDecodingBackend{guidedDecodingConfig.getBackend()} - , mMaxNumSequences{maxNumSequences} - , mVocabSizePadded{vocabSizePadded} - , mBitmaskSize{common::ceilDiv(mVocabSizePadded, 32)} - , mLogitsDtype{logitsDtype} - , mCopyBufferManager{std::make_shared<CudaStream>()} -{ - TLLM_CHECK_WITH_INFO(mGuidedDecodingBackend != executor::GuidedDecodingConfig::GuidedDecodingBackend::kLLGUIDANCE, - "LLGuidance is not supported for guided decoding in C++ runtime."); - if (mGuidedDecodingBackend == executor::GuidedDecodingConfig::GuidedDecodingBackend::kXGRAMMAR) - { - mXGrammarMatchers.resize(mMaxNumSequences); - xgrammar::VocabType vocabType = xgrammar::VocabType::RAW; - bool addPrefixSpace = false; - auto const& tokenizerStr = guidedDecodingConfig.getTokenizerStr(); - if (tokenizerStr) - { - auto const& metadata = xgrammar::TokenizerInfo::DetectMetadataFromHF(tokenizerStr.value()); - auto const& metadataJson = nlohmann::json::parse(metadata); - vocabType = metadataJson.at("vocab_type").template get<xgrammar::VocabType>(); - addPrefixSpace = metadataJson.at("add_prefix_space").template get<bool>(); - } - auto const& tokenizerInfo = xgrammar::TokenizerInfo(guidedDecodingConfig.getEncodedVocab().value(), vocabType, - mVocabSizePadded, guidedDecodingConfig.getStopTokenIds(), addPrefixSpace); - - auto const cacheLimitGb = common::getFloatEnv("XGRAMMAR_CACHE_LIMIT_GB"); - mXGrammarCompiler = std::make_shared<xgrammar::GrammarCompiler>(tokenizerInfo, /*max_threads=*/8, - /*cache_enabled=*/true, - /*cache_limit_bytes=*/static_cast<long long>(cacheLimitGb.value_or(1.0f) * 1024 * 1024 * 1024)); - - auto const logitsPtrDtype = BufferDataType{mLogitsDtype, false, true}; - auto constexpr bitmaskDtype = TRTDataType<BitmaskT>::value; - auto constexpr bitmaskPtrDtype = TRTDataType<BitmaskT*>::value; - - mLogitsBitmask = runtimeBufferManager.gpu(ITensor::makeShape({mMaxNumSequences, mBitmaskSize}), bitmaskDtype); - mLogitsBitmaskHost = BufferManager::pinned(ITensor::makeShape({mMaxNumSequences, mBitmaskSize}), bitmaskDtype); - mLogitsBitmaskPtrVec = runtimeBufferManager.gpu(ITensor::makeShape({mMaxNumSequences}), bitmaskPtrDtype); - mLogitsBitmaskPtrVecHost = BufferManager::pinned(ITensor::makeShape({mMaxNumSequences}), bitmaskPtrDtype); - mLogitsPtrVec = runtimeBufferManager.gpu(ITensor::makeShape({mMaxNumSequences}), logitsPtrDtype); - mLogitsPtrVecHost = BufferManager::pinned(ITensor::makeShape({mMaxNumSequences}), logitsPtrDtype); - } -} - -void GuidedDecoder::build(ScheduledRequests const& scheduledRequests) -{ - if (mGuidedDecodingBackend == executor::GuidedDecodingConfig::GuidedDecodingBackend::kXGRAMMAR) - { - for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) - { - for (auto const& llmReq : requests) - { - auto const& guidedDecodingParams = llmReq->getGuidedDecodingParams(); - if (!guidedDecodingParams.has_value()) - { - continue; - } - auto const seqSlot = llmReq->mSeqSlot.value(); - if (llmReq->isContextInitState() && llmReq->isFirstContextChunk()) - { - // The request is in the first context forward step (considering kv cache reuse). - auto const& guideType = guidedDecodingParams->getGuideType(); - auto const& guide = guidedDecodingParams->getGuide(); - switch (guideType) - { - case executor::GuidedDecodingParams::GuideType::kJSON: - { - mXGrammarMatchers.at(seqSlot) = std::make_shared<xgrammar::GrammarMatcher>( - mXGrammarCompiler->CompileBuiltinJSONGrammar()); - break; - } - case executor::GuidedDecodingParams::GuideType::kJSON_SCHEMA: - { - mXGrammarMatchers.at(seqSlot) = std::make_shared<xgrammar::GrammarMatcher>( - mXGrammarCompiler->CompileJSONSchema(guide.value())); - break; - } - case executor::GuidedDecodingParams::GuideType::kREGEX: - { - mXGrammarMatchers.at(seqSlot) = std::make_shared<xgrammar::GrammarMatcher>( - mXGrammarCompiler->CompileRegex(guide.value())); - break; - } - case executor::GuidedDecodingParams::GuideType::kEBNF_GRAMMAR: - { - mXGrammarMatchers.at(seqSlot) = std::make_shared<xgrammar::GrammarMatcher>( - mXGrammarCompiler->CompileGrammar(guide.value())); - break; - } - case executor::GuidedDecodingParams::GuideType::kSTRUCTURAL_TAG: - { - mXGrammarMatchers.at(seqSlot) = std::make_shared<xgrammar::GrammarMatcher>( - mXGrammarCompiler->CompileStructuralTag(guide.value())); - break; - } - default: - { - TLLM_THROW("Unsupported guide type."); - } - } - } - else if (llmReq->isGenerationInProgressState()) - { - // The request is in a generation forward step. - // Currently, guided decoding does not support with beam search. - mXGrammarMatchers.at(seqSlot)->AcceptToken(llmReq->getLastTokens(0)); - } - else - { - continue; - } - - // Fill the bitmask on host and asynchorously copy to device using mCopyBufferManager. - auto const logitsBitmask = ITensor::at(mLogitsBitmask, {seqSlot}); - auto const logitsBitmaskHost = ITensor::at(mLogitsBitmaskHost, {seqSlot}); - - std::array<int64_t, 1> bitmaskShape{mBitmaskSize}; - DLTensor logitsBitmaskDlt{logitsBitmaskHost->data(), DLDevice{kDLCPU, 0}, 1, DLDataType{kDLInt, 32, 1}, - bitmaskShape.data(), nullptr, 0}; - mXGrammarMatchers.at(seqSlot)->FillNextTokenBitmask(&logitsBitmaskDlt); - mCopyBufferManager.copy(*logitsBitmaskHost, *logitsBitmask); - } - } - } -} - -void GuidedDecoder::execute(DecoderInputBuffers const& decoderInputBuffers, BufferManager const& runtimeBufferManager) -{ - auto const& stream = runtimeBufferManager.getStream(); - - // Wait for mCopyBufferManager finishing the H2D copy of logitsBitmask - // TODO(enweiz): Move the H2D copy of logitsBitmaskPtrVec to buildGuidedDecoding. - // This may not bring too much perf gain because of the small size of logitsBitmaskPtrVec. - // TODO(enweiz): For chunked context, we currently build mask cache at the first context chunk, and apply - // the mask at the last context chunk. So, ideally we should sync the stream at the last context chunk. - CudaEvent event{}; - mCopyBufferManager.getStream().record(event); - stream.wait(event); - - if (mGuidedDecodingBackend == executor::GuidedDecodingConfig::GuidedDecodingBackend::kXGRAMMAR - && !decoderInputBuffers.decoderRequests.empty()) - { - SizeType32 batchIdx{0}; - for (size_t requestIdx = 0; requestIdx < decoderInputBuffers.decoderRequests.size(); ++requestIdx) - { - auto const& llmReq = decoderInputBuffers.decoderRequests.at(requestIdx); - - auto const& guidedDecodingParams = llmReq->getGuidedDecodingParams(); - if (guidedDecodingParams.has_value()) - { - auto const seqSlot = llmReq->mSeqSlot.value(); - - auto const& logits = decoderInputBuffers.decoderLogits.at(requestIdx); - auto const logitsBitmask = ITensor::at(mLogitsBitmask, {seqSlot}); - - // Use void* to unify the code for different mLogitsDtype - *reinterpret_cast<void**>(ITensor::at(mLogitsPtrVecHost, {batchIdx})->data()) = logits->data(); - *reinterpret_cast<void**>(ITensor::at(mLogitsBitmaskPtrVecHost, {batchIdx})->data()) - = logitsBitmask->data(); - - ++batchIdx; - } - } - if (batchIdx > 0) - { - runtimeBufferManager.copy( - *ITensor::slice(mLogitsPtrVecHost, 0, batchIdx), *ITensor::slice(mLogitsPtrVec, 0, batchIdx)); - runtimeBufferManager.copy(*ITensor::slice(mLogitsBitmaskPtrVecHost, 0, batchIdx), - *ITensor::slice(mLogitsBitmaskPtrVec, 0, batchIdx)); - - auto logitsBitmaskPtrVec = bufferCast<BitmaskT const*>(*mLogitsBitmaskPtrVec); - if (mLogitsDtype == nvinfer1::DataType::kFLOAT) - { - auto logitsPtrVec = bufferCast<float*>(*mLogitsPtrVec); - tensorrt_llm::kernels::invokeLogitsBitmask<float>( - logitsPtrVec, logitsBitmaskPtrVec, batchIdx, mVocabSizePadded, stream.get()); - } - else if (mLogitsDtype == nvinfer1::DataType::kHALF) - { - auto logitsPtrVec = bufferCast<half*>(*mLogitsPtrVec); - tensorrt_llm::kernels::invokeLogitsBitmask<half>( - logitsPtrVec, logitsBitmaskPtrVec, batchIdx, mVocabSizePadded, stream.get()); - } - else - { - TLLM_THROW("Unsupported logits data type."); - } - } - } -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/handleContextLogits.cpp b/cpp/tensorrt_llm/batch_manager/handleContextLogits.cpp deleted file mode 100644 index 6f4a541ffcbb..000000000000 --- a/cpp/tensorrt_llm/batch_manager/handleContextLogits.cpp +++ /dev/null @@ -1,176 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/handleContextLogits.h" - -#include "tensorrt_llm/batch_manager/decoderBuffers.h" -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/batch_manager/medusaBuffers.h" -#include "tensorrt_llm/batch_manager/runtimeBuffers.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/runtimeKernels.h" -#include "tensorrt_llm/runtime/utils/debugUtils.h" - -namespace tr = tensorrt_llm::runtime; -namespace tru = tensorrt_llm::runtime::utils; - -namespace tensorrt_llm::batch_manager -{ - -using BufferManager = tensorrt_llm::runtime::BufferManager; -using TensorPtr = runtime::ITensor::SharedPtr; -using ITensor = runtime::ITensor; -using SizeType32 = tensorrt_llm::runtime::SizeType32; - -namespace -{ - -//! @brief Copy logits from context phase to beginning of generation logits. -//! @details Usually, this concerns logits of 1 token. In speculative decoding this concerns draftLen + 1 tokens. -void copyLastContextLogits(TensorPtr const& contextLogits, LlmRequest& llmReq, BufferManager const& bufferManager) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - auto const numLogits = contextLogits->getShape().d[0]; - for (int beam = 0; beam < llmReq.getBeamWidthByIter(); beam++) - { - // [beamWidth, mMaxNewTokens, vocabSizePadded] -> [numLogits, vocabSizePadded] - auto beamHostTensorPtr = ITensor::slice(llmReq.getGenerationLogitsHost(), {beam, 0}, numLogits); - bufferManager.copy(*contextLogits, *beamHostTensorPtr); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void setupMedusaLogits(std::vector<TensorPtr>& medusaLogitsHeads, TensorPtr const& medusaLogitsDevice, - SizeType32 medusaHeads, SizeType32 logitsIndex, SizeType32 numLogits) -{ - for (SizeType32 hi = 0; hi < medusaHeads; ++hi) - { - TensorPtr logitsHead = ITensor::slice(medusaLogitsDevice, hi, 1); - logitsHead->squeeze(0); - medusaLogitsHeads[hi] = ITensor::slice(logitsHead, logitsIndex, numLogits); - } -} - -} // namespace - -SizeType32 HandleContextLogits::operator()(DecoderInputBuffers& inputBuffers, RequestVector const& contextRequests, - tr::ITensor::SharedPtr const& logits, std::vector<tr::SizeType32> const& numContextLogitsVec, - tr::ModelConfig const& modelConfig, tr::BufferManager const& manager, - OptionalRef<MedusaBuffers> medusaBuffers) const -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(HandleContextLogits); - - auto& decoderRequests = inputBuffers.decoderRequests; - decoderRequests.clear(); - decoderRequests.reserve(contextRequests.size()); - auto& allDecoderLogits = inputBuffers.decoderLogits; - allDecoderLogits.clear(); - allDecoderLogits.reserve(contextRequests.size()); - - SizeType32 batchIndex{0}; - SizeType32 logitsIndex{0}; - // Copy logits into decoderBuffers.logits - for (auto const& llmReq : contextRequests) - { - auto const numContextLogits = numContextLogitsVec.at(batchIndex); - auto const draftLength = llmReq->isLastContextChunk() ? llmReq->getNumDraftTokens() : 0; - - TLLM_LOG_DEBUG("logitsIndex: %d", logitsIndex); - TLLM_LOG_DEBUG("numContextLogits %d", numContextLogits); - TLLM_LOG_DEBUG("draftLength: %d", draftLength); - - if (modelConfig.computeContextLogits()) - { - // Since the computational graph has been modified, only the last token is needed. - TLLM_CHECK_WITH_INFO(!modelConfig.getSpeculativeDecodingMode().isMedusa() - && !modelConfig.getSpeculativeDecodingMode().isLookaheadDecoding(), - "Return context logits is not supported with Medusa and Lookahead decoding"); - - if (llmReq->getReturnContextLogits()) - { - if (llmReq->getPrepopulatedPromptLen() > 0) - { - TLLM_LOG_WARNING( - "Because of KV cache reuse, not all context logits could be produced for request %lu.", - llmReq->mRequestId); - } - TensorPtr contextLogitsDeviceView = ITensor::slice(logits, logitsIndex, numContextLogits); - TensorPtr contextLogitsHostView = ITensor::slice( - llmReq->getContextLogitsHost(), llmReq->getContextCurrentPosition(), numContextLogits); - // Copy to host directly - manager.copy(*contextLogitsDeviceView, *contextLogitsHostView); - } - } - logitsIndex += numContextLogits + draftLength; - - // Get the logits from the last context token and draft tokens - auto const numDecoderLogits = 1 + draftLength; - auto const seqSlot = llmReq->mSeqSlot.value(); - TensorPtr logitsView = ITensor::slice(logits, logitsIndex - numDecoderLogits, numDecoderLogits); - - if (modelConfig.getSpeculativeDecodingMode().hasDraftLogits()) - { - auto& medusaLogitsHeads = inputBuffers.predictedDraftLogits.at(seqSlot); - TLLM_CHECK(medusaBuffers); - setupMedusaLogits(medusaLogitsHeads, medusaBuffers->medusaLogitsDevice, - modelConfig.getSpeculativeDecodingModule().getMaxDraftPathLen(), logitsIndex - numDecoderLogits, - numDecoderLogits); - } - - // Save the last token logits of context into generation logits or - // save the accepted token logits from target model - if (llmReq->getReturnGenerationLogits()) - { - copyLastContextLogits(logitsView, *llmReq, manager); - } - - TLLM_CHECK_DEBUG_WITH_INFO(tru::tensorHasInvalid<float>(*logitsView, manager, "logits") == false, - "Found invalid number (NaN or Inf) in logits"); - - if (llmReq->isLastContextChunk()) - { - TensorPtr decoderLogits; - auto const reqBeamWidth = llmReq->getBeamWidthByIter(); - if (reqBeamWidth > 1) - { - // Tile logits of context requests - auto const& logitsShape = logitsView->getShape(); - auto const logitsType = logitsView->getDataType(); - decoderLogits = manager.gpu(ITensor::makeShape({reqBeamWidth, logitsShape.d[1]}), logitsType); - tensorrt_llm::runtime::kernels::tileTensor( - *decoderLogits, *logitsView, reqBeamWidth, manager.getStream()); - decoderLogits->unsqueeze(0); - } - else - { - decoderLogits = logitsView; - decoderLogits->unsqueeze(1); - } - decoderRequests.push_back(llmReq); - allDecoderLogits.emplace_back(std::move(decoderLogits)); - } - - ++batchIndex; - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return logitsIndex; -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/handleGenerationLogits.cpp b/cpp/tensorrt_llm/batch_manager/handleGenerationLogits.cpp deleted file mode 100644 index e2a7486b050a..000000000000 --- a/cpp/tensorrt_llm/batch_manager/handleGenerationLogits.cpp +++ /dev/null @@ -1,161 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/handleGenerationLogits.h" - -#include "tensorrt_llm/batch_manager/decoderBuffers.h" -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/batch_manager/medusaBuffers.h" -#include "tensorrt_llm/batch_manager/runtimeBuffers.h" -#include "tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/utils/debugUtils.h" - -namespace tr = tensorrt_llm::runtime; -namespace tru = tensorrt_llm::runtime::utils; - -namespace tensorrt_llm::batch_manager -{ - -using BufferManager = tensorrt_llm::runtime::BufferManager; -using TensorPtr = runtime::ITensor::SharedPtr; -using ITensor = runtime::ITensor; -using SizeType32 = tensorrt_llm::runtime::SizeType32; - -namespace -{ - -//! @brief Copy logits from generation phase under streaming mode. -void copyStreamingGenerationLogits(BufferManager const& bufferManager, LlmRequest& llmReq) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - // If llmRequest is streaming, directly copy to host. - // Only one token's logits needs to be copied each time. - TLLM_CHECK(llmReq.getGenerationLogitsFragmentsSize() == 1); - - SizeType32 numGenerationToken = llmReq.getMaxBeamNumTokens() - llmReq.mPromptLen; - TensorPtr const& generationLogitsHost - = llmReq.getGenerationLogitsHost(); // [mMaxNewTokens (or 1), beamWidth, vocabSizePadded] - - TensorPtr hostTensorPtr - = ITensor::slice(generationLogitsHost, numGenerationToken, 1); // [1, beamWidth, vocabSizePadded] - TensorPtr deviceTensorPtr = *(llmReq.getGenerationLogitsFragments().begin()); - - bufferManager.copy(*deviceTensorPtr, *hostTensorPtr); - llmReq.clearGenerationLogitsFragments(); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void setupMedusaLogits(std::vector<TensorPtr>& medusaLogitsHeads, TensorPtr const& medusaLogitsDevice, - SizeType32 medusaHeads, SizeType32 logitsIndex, SizeType32 numLogits) -{ - for (SizeType32 hi = 0; hi < medusaHeads; ++hi) - { - TensorPtr logitsHead = ITensor::slice(medusaLogitsDevice, hi, 1); - logitsHead->squeeze(0); - medusaLogitsHeads[hi] = ITensor::slice(logitsHead, logitsIndex, numLogits); - } -} - -} // namespace - -void HandleGenerationLogits::operator()(DecoderInputBuffers& inputBuffers, RequestVector const& generationRequests, - tr::ITensor::SharedPtr const& logits, tr::SizeType32 logitsIndex, tr::ModelConfig const& modelConfig, - tr::BufferManager const& manager, OptionalRef<RuntimeBuffers> genRuntimeBuffers, - OptionalRef<MedusaBuffers> medusaBuffers) const -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(HandleGenerationLogits); - - auto& decoderRequests = inputBuffers.decoderRequests; - decoderRequests.reserve(decoderRequests.size() + generationRequests.size()); - auto& allDecoderLogits = inputBuffers.decoderLogits; - allDecoderLogits.reserve(allDecoderLogits.size() + generationRequests.size()); - - for (auto const& llmReq : generationRequests) - { - auto const reqBeamWidth = llmReq->getBeamWidthByIter(); - auto const seqSlot = llmReq->mSeqSlot.value(); - - auto const draftLength = llmReq->getNumDraftTokens(); - auto const numLogits = draftLength + reqBeamWidth; - - TLLM_CHECK(draftLength == 0 || reqBeamWidth == 1); - - TLLM_LOG_DEBUG("logitsIndex: %d", logitsIndex); - TLLM_LOG_DEBUG("draftLength: %d", draftLength); - TLLM_LOG_DEBUG("reqBeamWidth: %d", reqBeamWidth); - - // genRuntimeBuffers.logits shape: [numGen*reqBeamWidth, vocabSize] - // logitsView shape: [numLogits, vocabSize] - TensorPtr logitsView = ITensor::slice(logits, logitsIndex, numLogits); - TLLM_CHECK_DEBUG_WITH_INFO(tru::tensorHasInvalid<float>(*logitsView, manager, "logits") == false, - "Found invalid number (NaN or Inf) in logits"); - - TLLM_CHECK(llmReq->isGenerationInProgressState()); - TensorPtr decoderLogits; - if (reqBeamWidth > 1) - { - decoderLogits = logitsView; - decoderLogits->unsqueeze(0); - } - else - { - decoderLogits = logitsView; - decoderLogits->unsqueeze(1); - } - decoderRequests.push_back(llmReq); - allDecoderLogits.emplace_back(std::move(decoderLogits)); - - if (llmReq->getReturnGenerationLogits()) - { - TLLM_CHECK_WITH_INFO(modelConfig.getSpeculativeDecodingMode().isNone() - || modelConfig.getSpeculativeDecodingMode().isDraftTokensExternal(), - "Only speculative decoding with external draft tokens supports returning generation logits"); - - // Push into fragments vector - llmReq->addGenerationLogitsFragment(logitsView); - TLLM_CHECK( - llmReq->getGenerationLogitsFragmentsSize() <= RuntimeBuffers::GenerationLogitsCache::kCACHE_LENGTH); - if (llmReq->isStreaming()) - { - copyStreamingGenerationLogits(manager, *llmReq); - } - // Copy back to host for every kCACHE_LENGTH steps to mitigate GPU memory pressure - else if (llmReq->getGenerationLogitsFragmentsSize() == RuntimeBuffers::GenerationLogitsCache::kCACHE_LENGTH) - { - TLLM_CHECK(genRuntimeBuffers); - auto constexpr beforeDecoder = true; - utils::copyGenerationLogits(genRuntimeBuffers->generationLogitsCache, manager, *llmReq, beforeDecoder); - } - } - if (modelConfig.getSpeculativeDecodingMode().hasDraftLogits()) - { - auto& medusaLogitsHeads = inputBuffers.predictedDraftLogits.at(seqSlot); - TLLM_CHECK(medusaBuffers); - setupMedusaLogits(medusaLogitsHeads, medusaBuffers->medusaLogitsDevice, - modelConfig.getSpeculativeDecodingModule().getMaxDraftPathLen(), logitsIndex, draftLength); - } - logitsIndex += numLogits; - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp b/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp index fd5f8d4ddd75..8c1ffb70e372 100644 --- a/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp +++ b/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp @@ -27,6 +27,7 @@ #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/kernels/kvCacheIndex.h" #include "tensorrt_llm/runtime/common.h" @@ -586,7 +587,7 @@ std::map<SizeType32, float> BlockManager::calculateWindowSizeToShare( BlockManager::BlockManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, - nvinfer1::DataType dtype, SizeType32 sinkBubbleLength, SizeType32 chunkSize, CacheType cacheType, + tensorrt_llm::DataType dtype, SizeType32 sinkBubbleLength, SizeType32 chunkSize, CacheType cacheType, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority, std::shared_ptr<KVCacheEventManager> eventManager, bool enablePartialReuse, bool copyOnPartialReuse, std::shared_ptr<kv_connector::KvCacheConnectorManager> kvCacheConnectorManager, @@ -740,7 +741,7 @@ BlockManager::BlockManager(std::vector<SizeType32> const& numKvHeadsPerLayer, Si "Maybe you tried changing either of them to an std::unordered_map?"); } -WindowBlockManager::WindowBlockManager(nvinfer1::DataType dtype, SizeType32 windowSize, +WindowBlockManager::WindowBlockManager(tensorrt_llm::DataType dtype, SizeType32 windowSize, std::vector<SizeType32> const& managedLayers, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, bool isSWA, SizeType32 blocksInPrimaryPool, SizeType32 blocksInSecondaryPool, SizeType32 maxNumSequences, std::shared_ptr<runtime::CudaStream> stream, @@ -840,7 +841,7 @@ WindowBlockManager::WindowBlockManager(nvinfer1::DataType dtype, SizeType32 wind // to specify FP4 related parameters (scale dtypes, etc)? This can also be passed // in the constructor. constexpr SizeType32 kQuantBlockSizeNVFP4 = 16; - if (dtype == nvinfer1::DataType::kFP4) + if (dtype == tensorrt_llm::DataType::kFP4) { createBlockScalePools(kQuantBlockSizeNVFP4); } @@ -936,6 +937,7 @@ bool BlockManager::verifyQueueIntegrity(SizeType32 windowSize) const bool WindowBlockManager::verifyQueueIntegrity() const { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); return mEvictionPolicy->verifyQueueIntegrity(); } @@ -1092,7 +1094,7 @@ void BlockManager::allocatePools(bool useUvm) void WindowBlockManager::allocatePools(bool useUvm) { - constexpr nvinfer1::DataType kScaleDtypeNVFP4 = nvinfer1::DataType::kFP8; + constexpr tensorrt_llm::DataType kScaleDtypeNVFP4 = tensorrt_llm::DataType::kFP8; bool const requestFabricMemory = tc::getEnvKVCachePoolUseFabricMemory(); bool const fabricMemorySupported = FabricMemory::supportFabricMemory(); @@ -1118,21 +1120,21 @@ void WindowBlockManager::allocatePools(bool useUvm) auto blockSize = pool.blockSize; auto poolDtype = pool.containsBlockScales ? kScaleDtypeNVFP4 : mDataType; #ifdef ENABLE_FP4 - auto const poolIsFP4 = poolDtype == nvinfer1::DataType::kFP4; + auto const poolIsFP4 = poolDtype == tensorrt_llm::DataType::kFP4; #else auto const poolIsFP4 = false; #endif if (poolIsFP4) { - poolDtype = nvinfer1::DataType::kINT8; + poolDtype = tensorrt_llm::DataType::kINT8; } if (pool.containsIndexerKCache) { - poolDtype = nvinfer1::DataType::kUINT8; + poolDtype = tensorrt_llm::DataType::kUINT8; } - nvinfer1::Dims cacheShape = isRecurrentState() + tensorrt_llm::Dims cacheShape = isRecurrentState() ? ITensor::makeShape({pool.numLayers, mNumPrimaryBlocks, mKVFactor, blockSize}) : ITensor::makeShape({mNumPrimaryBlocks, pool.numLayers, mKVFactor, blockSize}); pool.layerFirstLayout = isRecurrentState(); @@ -1166,7 +1168,7 @@ void WindowBlockManager::allocatePools(bool useUvm) if (mNumSecondaryBlocks > 0) { - nvinfer1::Dims cacheShapeOffload = isRecurrentState() + tensorrt_llm::Dims cacheShapeOffload = isRecurrentState() ? ITensor::makeShape({pool.numLayers, mNumSecondaryBlocks, mKVFactor, blockSize}) : ITensor::makeShape({mNumSecondaryBlocks, pool.numLayers, mKVFactor, blockSize}); TLLM_LOG_DEBUG("[%s] Allocating secondary pool with %d blocks for %d layers with %d kv heads", @@ -1218,6 +1220,7 @@ void BlockManager::startScheduling() void WindowBlockManager::startScheduling() { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); mSchedulingNumFreeBlocks = mEvictionPolicy->getNumFreeBlocks(kPrimaryLevel); for (auto& [requestId, slotAllocatedBlocks] : mAllocatedBlocksPerSeq) { @@ -1237,6 +1240,7 @@ void WindowBlockManager::freeLeafBlock(BlockPtr const& block) void WindowBlockManager::releaseSubtree(BlockPtr const& block) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); // Iterative pre-order DFS over `block` and all its descendants. Collect // first, then detach in reverse order: cascade-prune in freeLeafBlock() // removes empty parent nodes from the trie, so leaves must be detached @@ -1274,6 +1278,7 @@ BlockPtr WindowBlockManager::getFreeBlock(GenerationRequest& sequence, executor: std::optional<std::chrono::milliseconds> durationMs, executor::KvCacheTransferMode mode, std::string const& directory, bool wantPlaceholder) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); // eviction policy get free primary block auto [block, canOffload] = mEvictionPolicy->getFreeBlock(kPrimaryLevel, wantPlaceholder); if (block->getUniqueTokens().empty()) @@ -1344,7 +1349,7 @@ BlockPtr WindowBlockManager::getFreeBlock(GenerationRequest& sequence, executor: return block; } -void WindowBlockManager::setOffsets(tk::KVCacheIndex* offsetsPtr, nvinfer1::Dims const& offsetsShape, +void WindowBlockManager::setOffsets(tk::KVCacheIndex* offsetsPtr, tensorrt_llm::Dims const& offsetsShape, SizeType32 beamIdx, SizeType32 blockIdx, KVCacheBlock::IdType blockId) const { auto constexpr kIdx = 0; @@ -1382,7 +1387,7 @@ void WindowBlockManager::setOffsets(tk::KVCacheIndex* offsetsPtr, nvinfer1::Dims } } -void BlockManager::setOffsets(tk::KVCacheIndex* offsetsPtr, nvinfer1::Dims const& offsetsShape, SizeType32 beamIdx, +void BlockManager::setOffsets(tk::KVCacheIndex* offsetsPtr, tensorrt_llm::Dims const& offsetsShape, SizeType32 beamIdx, SizeType32 blockIdx, KVCacheBlock::IdType blockId, SizeType32 windowSize) const { mWindowBlockManagers.at(windowSize).setOffsets(offsetsPtr, offsetsShape, beamIdx, blockIdx, blockId); @@ -1397,6 +1402,7 @@ void BlockManager::onboardBlock(GenerationRequest& sequence, BlockPtr const& off void WindowBlockManager::onboardBlock(GenerationRequest& sequence, BlockPtr const& offloadBlock, executor::KvCacheTransferMode mode, std::string const& directory) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); if (!offloadBlock->isPlaceholder() && !offloadBlock->isPrimary()) { auto block = getFreeBlock( @@ -1425,6 +1431,7 @@ void BlockManager::offloadBlock( void WindowBlockManager::offloadBlock( BlockPtr const& block, executor::KvCacheTransferMode mode, std::string const& directory) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); // The current default behavior is to offload the out-of-window block // to secondary block pool to allow more free primary blocks for reuse. // However, such behavior does not take account whether the offloaded @@ -2130,6 +2137,19 @@ bool WindowBlockManager::blockInRadixTree(BlockPtr const& block) } std::shared_ptr<KVCacheBlock> WindowBlockManager::findBlocksInReuseTreeByBlockKey(BlockKey const& blockKey) +{ + std::vector<KVCacheBlock::IdType> unusedPinnedBlockIds; + return findBlocksInReuseTreeByBlockKeyImpl(blockKey, /*pinBlocks=*/false, unusedPinnedBlockIds); +} + +std::shared_ptr<KVCacheBlock> WindowBlockManager::findBlocksInReuseTreeByBlockKey( + BlockKey const& blockKey, std::vector<KVCacheBlock::IdType>& pinnedBlockIds) +{ + return findBlocksInReuseTreeByBlockKeyImpl(blockKey, /*pinBlocks=*/true, pinnedBlockIds); +} + +std::shared_ptr<KVCacheBlock> WindowBlockManager::findBlocksInReuseTreeByBlockKeyImpl( + BlockKey const& blockKey, bool pinBlocks, std::vector<KVCacheBlock::IdType>& pinnedBlockIds) { std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto blockedUniqueTokens @@ -2142,7 +2162,42 @@ std::shared_ptr<KVCacheBlock> WindowBlockManager::findBlocksInReuseTreeByBlockKe blockKeys.emplace_back(blockKey.usesExtraIds, blockKey.loraTaskId, blockedUniqueTokensList, blockKey.extraKeys, blockKey.cacheSalt); } - return searchReuseTree(blockKeys); + auto searchRoot = mCachedBlocksRoot; + std::vector<BlockPtr> pinnedInScope; + for (auto const& blockKey : blockKeys) + { + auto [partialMatch, numMatched, matchingBlock] = searchRoot != nullptr + ? searchRoot->findMatchingBlock(blockKey, true, true) + : std::make_tuple(false, 0, nullptr); + + // A prefix-only match lacks KV for the requested tail, and pinning (transfer) + // lookups read primary-pool buffers directly, so offloaded and placeholder blocks + // are misses too (isPlaceholder first: isPrimary asserts on placeholders). + bool const fullyMatched + = matchingBlock != nullptr && numMatched == static_cast<SizeType32>(blockKey.uniqueTokens.size()); + bool const transferable + = fullyMatched && (!pinBlocks || (!matchingBlock->isPlaceholder() && matchingBlock->isPrimary())); + if (!transferable) + { + // Roll back pins taken during the partial walk. + for (auto const& block : pinnedInScope) + { + unpinBlock(block); + } + pinnedBlockIds.clear(); + return nullptr; + } + + if (pinBlocks) + { + pinBlock(matchingBlock); + pinnedInScope.push_back(matchingBlock); + pinnedBlockIds.push_back(matchingBlock->getBlockId()); + } + + searchRoot = std::move(matchingBlock); + } + return searchRoot; } std::shared_ptr<KVCacheBlock> WindowBlockManager::findBlocksInReuseTreeByBlockKeys( @@ -2191,6 +2246,7 @@ void BlockManager::refreshBlocks() void WindowBlockManager::refreshBlocks() { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); mEvictionPolicy->refresh(); mTransferManager->syncTransfers(); } @@ -2321,6 +2377,7 @@ bool BlockManager::copyLinearAttentionBlock(GenerationRequest& sequence, LlmRequ bool WindowBlockManager::tryAllocatePlaceholderForLinearAttention(GenerationRequest& sequence, bool shareAmongBeams) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const beamWidth = sequence.getBeamWidth(); auto const newBlockIdx = sequence.getCacheBlockIds(mWindowSize).at(0).size(); // The first block is not a placeholder. @@ -2410,6 +2467,7 @@ bool WindowBlockManager::tryAllocatePlaceholderForLinearAttention(GenerationRequ void WindowBlockManager::allocateBlock(GenerationRequest& sequence, bool shareAmongBeams) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const beamWidth = sequence.getBeamWidth(); auto const requiredBlocks = shareAmongBeams ? 1 : beamWidth; @@ -2691,16 +2749,7 @@ std::pair<SizeType32, std::vector<KVCacheBlock::IdType>> WindowBlockManager::sto if (pinBlocks) { - // If the block has no refs it sits in the eviction policy's free - // queue. Claim it first so that the later unpinBlocksById / - // releaseBlock cycle does not create a duplicate queue entry. - // Pass the block's existing priority and duration so that - // claimBlock does not clear its retention/expiry metadata. - if (!prevBlock->hasRefs()) - { - mEvictionPolicy->claimBlock(prevBlock, prevBlock->getPriority(), prevBlock->getDurationMs()); - } - prevBlock->incRefCount(); + pinBlock(prevBlock); pinnedBlockIds.push_back(prevBlock->getBlockId()); } } @@ -2735,6 +2784,7 @@ void BlockManager::replaceSharedBlock(GenerationRequest& sequence, SizeType32 wi void WindowBlockManager::replaceSharedBlock(GenerationRequest& sequence, SizeType32 blockIdx) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const requestId = sequence.getRequestId(); auto const beamWidth = sequence.getBeamWidth(); auto& allocatedBlocks = mAllocatedBlocksPerSeq.at(requestId); @@ -2786,6 +2836,7 @@ void BlockManager::releaseLastBlock(GenerationRequest& sequence, SizeType32 wind void WindowBlockManager::releaseLastBlock(GenerationRequest& sequence) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); if (isRecurrentState()) { // In recurrent state, the last block always contains the current state and should not be released. @@ -2834,6 +2885,7 @@ void WindowBlockManager::releaseLastBlock(GenerationRequest& sequence) [[nodiscard]] SizeType32 WindowBlockManager::getNumFreeBlocks() const { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const numFree = mEvictionPolicy->getNumFreeBlocks(kPrimaryLevel); TLLM_CHECK_WITH_INFO(numFree <= getMaxNumBlocks(), "%s::getNumFreeBlocks - primary free block count (%d) exceeds total block count (%d). " @@ -2845,6 +2897,7 @@ void WindowBlockManager::releaseLastBlock(GenerationRequest& sequence) [[nodiscard]] SizeType32 WindowBlockManager::getNumFreeSecondaryBlocks() const noexcept { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); return mEvictionPolicy->getNumFreeBlocks(kSecondaryLevel); } @@ -2975,18 +3028,41 @@ void BlockManager::unpinBlocksById(std::vector<KVCacheBlock::IdType> const& bloc firstManager.unpinBlocksById(blockIds); } +void WindowBlockManager::pinBlock(BlockPtr const& block) +{ + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); + // Claim free blocks out of the eviction queue first, keeping their retention metadata. + if (!block->hasRefs()) + { + mEvictionPolicy->claimBlock(block, block->getPriority(), block->getDurationMs()); + } + block->incRefCount(); +} + +void WindowBlockManager::unpinBlock(BlockPtr const& block) +{ + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); + block->decRefCount(); + if (!block->hasRefs()) + { + mEvictionPolicy->releaseBlock(block); + } +} + void WindowBlockManager::pinBlocks(GenerationRequest& sequence) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const requestId = sequence.getRequestId(); auto& allocatedBlocks = mAllocatedBlocksPerSeq.at(requestId); for (auto& block : allocatedBlocks) { - block->incRefCount(); + pinBlock(block); } } void WindowBlockManager::unpinBlocksById(std::vector<KVCacheBlock::IdType> const& blockIds) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); if (blockIds.empty()) { return; @@ -2999,11 +3075,7 @@ void WindowBlockManager::unpinBlocksById(std::vector<KVCacheBlock::IdType> const auto block = mAllBlocksById[blockId]; if (block && block->getBlockId() != KVCacheBlock::kCachedBlocksRootId) { - block->decRefCount(); - if (!block->hasRefs()) - { - mEvictionPolicy->releaseBlock(block); - } + unpinBlock(block); } } } @@ -3019,6 +3091,7 @@ void BlockManager::storeNewBlock(GenerationRequest& sequence, OptionalRef<LlmReq void WindowBlockManager::storeNewBlock(GenerationRequest& sequence, OptionalRef<LlmRequest const> llmRequest) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto constexpr beamIdx = 0; auto const& uniqueTokens = llmRequest->getUniqueTokens(beamIdx); @@ -3125,6 +3198,7 @@ std::vector<KVCacheBlock::IdType> WindowBlockManager::storeBlocksForReuse( std::optional<KVCacheBlock::IdType> WindowBlockManager::releaseBlocks( GenerationRequest& sequence, OptionalRef<LlmRequest const> llmRequest) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const requestId = sequence.getRequestId(); TLLM_LOG_DEBUG("%s::releaseBlocks - requestId=%lu, llmRequest.id=%s", mLogPrefix.c_str(), requestId, llmRequest.has_value() ? std::to_string(llmRequest->mRequestId).c_str() : "null"); @@ -3229,7 +3303,7 @@ void WindowBlockManager::schedulingReleaseBlocks(RequestIdType requestId) KVCacheManager::KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, - SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, + SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, int64_t stream, runtime::SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse, CacheType cacheType, bool enablePartialReuse, bool copyOnPartialReuse, bool enableIndexerKCache, SizeType32 indexerKCacheQuantBlockSize, SizeType32 indexerKCacheIndexHeadDim, @@ -3246,7 +3320,7 @@ KVCacheManager::KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, Size KVCacheManager::KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, - SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, + SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, int64_t stream, runtime::SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse, CacheType cacheType, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority, std::shared_ptr<KVCacheEventManager> eventManager, bool enablePartialReuse, bool copyOnPartialReuse, @@ -3265,7 +3339,7 @@ KVCacheManager::KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer KVCacheManager::KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, - SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, + SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, CudaStreamPtr stream, runtime::SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse, CacheType cacheType, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority, std::shared_ptr<KVCacheEventManager> eventManager, bool enablePartialReuse, bool copyOnPartialReuse, @@ -3307,7 +3381,7 @@ KVCacheManager::KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer KVCacheManager::KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, - SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, + SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, CudaStreamPtr stream, runtime::SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse, CacheType cacheType, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority, std::shared_ptr<KVCacheEventManager> eventManager, bool enablePartialReuse, bool copyOnPartialReuse, @@ -3340,7 +3414,7 @@ void KVCacheManager::allocatePools(bool useUvm) // a future per-window override map can mix precisions inside a single manager. auto const poolDataType = primaryPool->getDataType(); #ifdef ENABLE_FP4 - auto const isFp4 = poolDataType == nvinfer1::DataType::kFP4; + auto const isFp4 = poolDataType == tensorrt_llm::DataType::kFP4; #else auto const isFp4 = false; #endif @@ -3711,6 +3785,7 @@ bool KVCacheManager::copyLinearAttentionBlockBatch(std::vector<std::shared_ptr<L void WindowBlockManager::detachFrontBlock(GenerationRequest& sequence) { + std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); // streamLLM is not supported at the moment. The out of window block will // always be the 0th block. TLLM_CHECK_WITH_INFO( @@ -4283,7 +4358,7 @@ std::map<SizeType32, float> computeWindowSizeShares( } // namespace BlocksPerWindow BaseKVCacheManager::calculateMaxNumBlocks(executor::KvCacheConfig const& config, - nvinfer1::DataType dtype, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, + tensorrt_llm::DataType dtype, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, WorldConfig const& worldConfig, std::map<SizeType32, std::vector<SizeType32>> const& windowSizeToLayers, uint64_t allottedPrimaryMemBytes, uint64_t allottedSecondaryMemBytes, size_t extraCostMemory, SizeType32 kvFactor, SizeType32 maxBatchSize, diff --git a/cpp/tensorrt_llm/batch_manager/kvCacheManagerV2Utils.cu b/cpp/tensorrt_llm/batch_manager/kvCacheManagerV2Utils.cu index a2778e169154..b1855ab43b64 100644 --- a/cpp/tensorrt_llm/batch_manager/kvCacheManagerV2Utils.cu +++ b/cpp/tensorrt_llm/batch_manager/kvCacheManagerV2Utils.cu @@ -119,7 +119,7 @@ CUresult launchBatchedCopyImpl( } uint32_t const nbSplits = lowBandwidth ? 1 : divUp(nbBytes, grainBytes * ctaSize * 2); void* args[] = {(void*) pTasks, (void*) &nbBytes}; - static CUkernel const kernel = [] -> CUkernel + static CUkernel const kernel = []() -> CUkernel { cudaKernel_t kernel = nullptr; TLLM_CUDA_CHECK(cudaGetKernel(&kernel, reinterpret_cast<void const*>(&batchedCopy<N>))); diff --git a/cpp/tensorrt_llm/batch_manager/kvCacheManagerV2Utils.h b/cpp/tensorrt_llm/batch_manager/kvCacheManagerV2Utils.h index e32c727d0d81..5fa88a78468e 100644 --- a/cpp/tensorrt_llm/batch_manager/kvCacheManagerV2Utils.h +++ b/cpp/tensorrt_llm/batch_manager/kvCacheManagerV2Utils.h @@ -17,6 +17,7 @@ #pragma once +#include "kv_cache_manager_v2/common.h" #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/kernels/kvCacheIndex.h" #include "tensorrt_llm/runtime/iBuffer.h" @@ -34,13 +35,7 @@ using ITensor = tensorrt_llm::runtime::ITensor; namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 { -struct DiskAddress -{ - int fd; - ssize_t pos; -}; - -using MemAddress = std::uintptr_t; +// DiskAddress and MemAddress are defined in kv_cache_manager_v2/common.h (included above). // Please make sure to align with the definition in tensorrt_llm/runtime/kv_cache_manager_v2/_common.py constexpr tk::KVCacheIndex::UnderlyingType BAD_PAGE_INDEX = -1; diff --git a/cpp/tensorrt_llm/batch_manager/kvCacheTransferManager.cpp b/cpp/tensorrt_llm/batch_manager/kvCacheTransferManager.cpp index c28d1e476137..33ad5b4a1675 100644 --- a/cpp/tensorrt_llm/batch_manager/kvCacheTransferManager.cpp +++ b/cpp/tensorrt_llm/batch_manager/kvCacheTransferManager.cpp @@ -22,6 +22,7 @@ #include "tensorrt_llm/batch_manager/kvCacheEventManager.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/kernels/kvCachePartialCopy.h" #include "tensorrt_llm/runtime/bufferManager.h" @@ -163,8 +164,8 @@ void KVCacheTransferManager::copyBlock(BlockPtr const& src, BlockPtr const& dst, // If no partial tokens or if the dataType is not supported for partial copy, copy entire block. // Note that nvfp4 kv cache SFs use an interleaved layout, so we need to copy the entire block. - if (numTokensToCopy <= 0 || srcPtr->getDataType() == nvinfer1::DataType::kINT4 - || srcPtr->getDataType() == nvinfer1::DataType::kFP4 || containsBlockScales) + if (numTokensToCopy <= 0 || srcPtr->getDataType() == tensorrt_llm::DataType::kINT4 + || srcPtr->getDataType() == tensorrt_llm::DataType::kFP4 || containsBlockScales) { // For partial copy not implemented with these data types, // just do a full copy. @@ -461,8 +462,8 @@ std::size_t KVCacheTransferManager::computeBlockTransferBytes( // Mirror the logic in copyBlock: a partial copy only happens when numTokensToCopy > 0, // the data type supports it (not kINT4/kFP4), not block scales, and numTokensToCopy < tokensPerBlock. - bool const isPartialCopy = numTokensToCopy > 0 && dataType != nvinfer1::DataType::kINT4 - && dataType != nvinfer1::DataType::kFP4 && !pool.containsBlockScales + bool const isPartialCopy = numTokensToCopy > 0 && dataType != tensorrt_llm::DataType::kINT4 + && dataType != tensorrt_llm::DataType::kFP4 && !pool.containsBlockScales && numTokensToCopy < pool.tokensPerBlock; if (isPartialCopy) diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/CMakeLists.txt b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/CMakeLists.txt new file mode 100644 index 000000000000..163ad82ffd3f --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/CMakeLists.txt @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# Sources for the C++ KVCacheManagerV2 implementation. These are added to the +# tensorrt_llm_batch_manager_static target by the parent CMakeLists.txt. + +# Bitcoin Core SHA-256 (vendored, MIT; see common/sha256/README.md) — needed by +# blockRadixTree.cpp. Slimmed to the single-block CSHA256 path: portable scalar +# core + runtime dispatch (sha256.cpp) plus a per-architecture hardware +# transform. Plain C++17; the include root common/sha256 is added on the +# batch_manager target so "sha256.h" resolves. +set(SHA256_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../common/sha256) +set(SHA256_SRCS ${SHA256_DIR}/sha256.cpp) +# x86/x86-64: Intel SHA-NI transform. +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64|i[3-6]86") + set(SHA256_X86_SHANI ${SHA256_DIR}/sha256_x86_shani.cpp) + set_source_files_properties( + ${SHA256_X86_SHANI} PROPERTIES COMPILE_OPTIONS "-msse4.1;-msse4;-msha") + list(APPEND SHA256_SRCS ${SHA256_X86_SHANI}) +endif() +# AArch64: ARMv8 crypto-extension (SHA-256) transform. +if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") + set(SHA256_ARM_SHANI ${SHA256_DIR}/sha256_arm_shani.cpp) + set_source_files_properties( + ${SHA256_ARM_SHANI} PROPERTIES COMPILE_OPTIONS "-march=armv8-a+crypto") + list(APPEND SHA256_SRCS ${SHA256_ARM_SHANI}) +endif() + +set(KV_CACHE_MANAGER_V2_SRCS + kv_cache_manager_v2/common.cpp + kv_cache_manager_v2/config.cpp + kv_cache_manager_v2/lifeCycleRegistry.cpp + kv_cache_manager_v2/utils/cudaEvent.cpp + kv_cache_manager_v2/utils/hostMem.cpp + kv_cache_manager_v2/cudaVirtMem.cpp + kv_cache_manager_v2/storage/config.cpp + kv_cache_manager_v2/storage/core.cpp + kv_cache_manager_v2/evictionController.cpp + kv_cache_manager_v2/copyEngine.cpp + kv_cache_manager_v2/blockRadixTree.cpp + kv_cache_manager_v2/eventManager.cpp + kv_cache_manager_v2/page.cpp + kv_cache_manager_v2/storageManager.cpp + kv_cache_manager_v2/introspection.cpp + kv_cache_manager_v2/kvCache.cpp + kv_cache_manager_v2/kvCacheManager.cpp + ${SHA256_SRCS}) + +# Headers and sources are co-located; the include root is batch_manager/ so that +# #include "kv_cache_manager_v2/common.h" resolves correctly. NOTE: this file is +# include()'d (not add_subdirectory()'d), so CMAKE_CURRENT_SOURCE_DIR is already +# batch_manager/ — no PARENT_SCOPE needed. +set(KV_CACHE_MANAGER_V2_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/blockRadixTree.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/blockRadixTree.cpp new file mode 100644 index 000000000000..212161acfda0 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/blockRadixTree.cpp @@ -0,0 +1,880 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/blockRadixTree.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/page.h" +#include "kv_cache_manager_v2/storageManager.h" +#include "kv_cache_manager_v2/utils/math.h" + +#include "sha256.h" + +#include "tensorrt_llm/common/assert.h" +#include <algorithm> +#include <cstring> +#include <stdexcept> +#include <utility> +#include <variant> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// ReuseScope +// --------------------------------------------------------------------------- + +// Serialized layout of a ReuseScope, consumed by Hasher::update(ReuseScope). +// Must stay byte-identical to the Python ReuseScope.to_bytes(): a mask byte +// followed by one little-endian uint64 per present field (signed=False). +template <typename Emit> +static void emitReuseScopeBytes(ReuseScope const& scope, Emit&& emit) +{ + uint8_t mask = 0; + if (scope.loraId.has_value()) + { + mask |= 1U << 0; + } + if (scope.salt.has_value()) + { + mask |= 1U << 1; + } + emit(&mask, sizeof(mask)); + if (scope.loraId.has_value()) + { + std::uint64_t const value = *scope.loraId; + emit(reinterpret_cast<uint8_t const*>(&value), sizeof(value)); + } + if (scope.salt.has_value()) + { + std::uint64_t const value = *scope.salt; + emit(reinterpret_cast<uint8_t const*>(&value), sizeof(value)); + } +} + +// --------------------------------------------------------------------------- +// Hasher +// --------------------------------------------------------------------------- + +namespace +{ +// Select the best available SHA-256 back-end (x86 SHA-NI, ARMv8 crypto, SSE4, +// AVX2) once, falling back to a portable scalar transform. CSHA256 dispatches +// through function pointers that SHA256AutoDetect() installs; the magic-static +// guarantees this runs exactly once and is thread-safe. +void ensureSha256Detected() +{ + static std::string const impl = SHA256AutoDetect(); + (void) impl; +} +} // namespace + +static void hashInt64(CSHA256& h, int64_t v) +{ + unsigned char buf[8]; + auto const unsignedValue = static_cast<uint64_t>(v); + for (int i = 0; i < 8; ++i) + { + buf[i] = static_cast<unsigned char>((unsignedValue >> (8 * i)) & 0xFFU); + } + h.Write(buf, sizeof(buf)); +} + +Hasher::Hasher() +{ + ensureSha256Detected(); +} + +Hasher::Hasher(ReuseScope const& seed) +{ + ensureSha256Detected(); + update(seed); +} + +Hasher& Hasher::update(ReuseScope const& scope) +{ + // Feed the serialized ReuseScope straight into the hash state without any + // intermediate heap buffer. + emitReuseScopeBytes(scope, [this](uint8_t const* data, size_t count) { mState.Write(data, count); }); + return *this; +} + +Hasher& Hasher::update(TokenId token) +{ + hashInt64(mState, static_cast<int64_t>(token)); + return *this; +} + +Hasher& Hasher::update(BlockKey const& key) +{ + mState.Write(reinterpret_cast<unsigned char const*>(key.data()), key.size()); + return *this; +} + +Hasher& Hasher::update(std::vector<uint8_t> const& bytes) +{ + mState.Write(bytes.data(), bytes.size()); + return *this; +} + +Hasher& Hasher::update(TokenIdExt const& tokenExt) +{ + std::visit( + [this](auto const& v) + { + using T = std::decay_t<decltype(v)>; + if constexpr (std::is_same_v<T, TokenId>) + hashInt64(mState, static_cast<int64_t>(v)); + else + mState.Write(reinterpret_cast<unsigned char const*>(v.data()), v.size()); + }, + tokenExt); + return *this; +} + +Hasher& Hasher::update(TokenIdExt const* tokens, size_t count) +{ + // Python uses array("Q", data).tobytes() to reduce per-token interpreter + // overhead. In C++ the compiler inlines each update() call, so the loop + // is already optimal; batching would only add a heap allocation. + for (size_t i = 0; i < count; ++i) + update(tokens[i]); + return *this; +} + +BlockKey Hasher::digest() const +{ + // Finalize into a 32-byte key. CSHA256::Finalize consumes the state, so we + // finalize a copy to keep this method const and allow further updates. + BlockKey out; + CSHA256 copy = mState; + copy.Finalize(reinterpret_cast<unsigned char*>(out.data())); + return out; +} + +// --------------------------------------------------------------------------- +// genMultiModalTokens +// --------------------------------------------------------------------------- + +std::vector<TokenIdExt> genMultiModalTokens( + int idOffset, std::vector<uint8_t> const& multiModalDataDigest, int numTokens, int tokenOffset) +{ + TLLM_CHECK_DEBUG(numTokens > 0); + TLLM_CHECK_DEBUG(tokenOffset >= 0); + TLLM_CHECK_DEBUG(multiModalDataDigest.size() == kDIGEST_LEN); + std::vector<TokenIdExt> result; + result.reserve(static_cast<size_t>(numTokens)); + for (int i = 0; i < numTokens; ++i) + { + if (tokenOffset + i == 0) + { + Digest d; + std::memcpy(d.data(), multiModalDataDigest.data(), kDIGEST_LEN); + result.emplace_back(DigestToken(d)); + } + else + { + result.emplace_back(TokenId(idOffset + tokenOffset + i)); + } + } + return result; +} + +// --------------------------------------------------------------------------- +// makeBlockchainKeyGenerator — lazy key generator. +// Returns a callable that yields one BlockKey per call (nullopt when done). +// First call yields root entry (empty token block). Mirrors Python's generator. +// --------------------------------------------------------------------------- + +static auto makeBlockchainKeyGenerator( + int tokensPerBlock, ReuseScope reuseScope, TokenIdExt const* tokens, size_t numTokens) +{ + // digest carries the running hash from the previous block. + BlockKey digest = Hasher(reuseScope).digest(); + // ordinal = -1: next call yields root (reuseScope digest). + // ordinal >= 0: next call yields key for tokens[ordinal*tpb .. (ordinal+1)*tpb). + int ordinal = -1; + + return [=]() mutable -> std::optional<BlockKey> + { + if (ordinal == -1) + { + ordinal++; + return digest; // root key + } + + size_t beg = static_cast<size_t>(ordinal) * static_cast<size_t>(tokensPerBlock); + if (beg >= numTokens) + return std::nullopt; + + size_t end = std::min(beg + static_cast<size_t>(tokensPerBlock), numTokens); + + Hasher h; + h.update(digest); + h.update(tokens + beg, end - beg); + digest = h.digest(); + + ordinal++; + return digest; + }; +} + +// Eager wrapper for callers that need all keys at once. +std::vector<BlockKey> sequenceToBlockchainKeys( + int tokensPerBlock, ReuseScope const& reuseScope, std::vector<TokenIdExt> const& tokens) +{ + std::vector<BlockKey> result; + auto gen = makeBlockchainKeyGenerator(tokensPerBlock, reuseScope, tokens.data(), tokens.size()); + while (auto key = gen()) + result.push_back(*key); + return result; +} + +// --------------------------------------------------------------------------- +// RootBlock +// --------------------------------------------------------------------------- + +BlockKey RootBlock::makeKey(ReuseScope const& reuseScope) +{ + return Hasher(reuseScope).digest(); +} + +RootBlock::RootBlock(ReuseScope reuseScope_, BlockRadixTree* treePtr) + : NodeBase(makeKey(reuseScope_), treePtr->eventSink().get()) + , reuseScope(std::move(reuseScope_)) + , tree(treePtr) +{ +} + +int RootBlock::tokensPerBlock() const noexcept +{ + return tree->tokensPerBlock(); +} + +// --------------------------------------------------------------------------- +// NodeBase +// --------------------------------------------------------------------------- + +NodeBase::~NodeBase() +{ + // Detach children before next is destroyed (implicit member destruction). + // This ensures that when a child's ~Block() runs, it sees prev == nullptr + // and skips parent cleanup — avoiding virtual calls on a mid-destruction parent. + for (auto& [k, child] : next) + { + child->prev = nullptr; + } +} + +SharedPtr<Block> NodeBase::detachNext(BlockKey const& blockKey) +{ + auto it = next.find(blockKey); + if (it == next.end()) + { + return nullptr; + } + + auto block = it->second; + block->prev = nullptr; + next.erase(it); + if (eventSink) + { + eventSink->addRemovedBlock(block->key); + } + if (type() == Type::kROOT_BLOCK && next.empty()) + { + auto* root = static_cast<RootBlock*>(this); + root->tree->proposeToEraseEmptyRoot(root->key); + } + return block; +} + +// --------------------------------------------------------------------------- +// Block +// --------------------------------------------------------------------------- + +namespace +{ + +static bool isPrefix(std::vector<TokenIdExt> const& prefix, std::vector<TokenIdExt> const& full) +{ + if (prefix.size() > full.size()) + return false; + for (size_t i = 0; i < prefix.size(); ++i) + { + if (prefix[i] != full[i]) + return false; + } + return true; +} + +} // anonymous namespace + +BlockKey Block::makeKey(BlockKey const& prevKey, TokenIdExt const* tokens, size_t count) +{ + Hasher h; + h.update(prevKey); + h.update(tokens, count); + return h.digest(); +} + +Block::Block(BlockKey k, std::vector<TokenIdExt> toks, NodeBase* prevNode, LifeCycleId numLifeCycles) + : NodeBase(k, prevNode->eventSink) + , tokens(std::move(toks)) + , prev(prevNode) + , storage(numLifeCycles, nullptr) + , mOrdinal(prevNode->ordinal() + 1) +{ +} + +int Block::tokensPerBlock() const noexcept +{ + TLLM_CHECK_DEBUG_WITH_INFO(prev, "Block must have a prev"); + // Mirrors Python: prev.tokens_per_block if isinstance(prev, RootBlock) else len(prev.tokens) + if (prev->type() == Type::kROOT_BLOCK) + return prev->tokensPerBlock(); + return static_cast<int>(static_cast<Block const*>(prev)->tokens.size()); +} + +void Block::releasePages() +{ + // Mirrors Python Block._release_pages(): for each stored page, if alive and + // DROPPABLE and scheduled for eviction, exclude from eviction. Also null out + // the page's back-pointer so that CommittedPage::~CommittedPage() doesn't + // attempt cleanup through this Block. Idempotent — storage is empty afterwards. + for (LifeCycleId lcIdx{0}; lcIdx < storage.size(); ++lcIdx) + { + auto const page = storage[lcIdx]; + if (page != nullptr) + { + TLLM_CHECK_DEBUG(page->block == this); + unlinkPage(lcIdx); + if (page->status() == PageStatus::DROPPABLE && page->scheduledForEviction()) + { + page->manager->excludeFromEviction(*page); + } + } + } +} + +Block::~Block() +{ + releasePages(); +} + +bool Block::isOrphan() const noexcept +{ + TLLM_CHECK_DEBUG(prev == nullptr || (prev->next.count(key) == 1 && prev->next.at(key).get() == this)); + return prev == nullptr; +} + +int Block::partialMatchThisNode(TokenIdExt const* otherTokens, size_t otherCount) const +{ + int count = 0; + for (size_t i = 0; i < std::min(tokens.size(), otherCount); ++i) + { + if (tokens[i] != otherTokens[i]) + break; + ++count; + } + return count; +} + +CommittedPage* Block::unlinkPage(LifeCycleId lcIdx, CommittedPage* expectedPage) +{ + auto& slot = storage.at(lcIdx); + CommittedPage* page = slot; + if (page == nullptr) + return nullptr; + if (expectedPage != nullptr && page != expectedPage) + return nullptr; + page->block = nullptr; + slot = nullptr; + return page; +} + +std::vector<SharedPtr<Block>> Block::clearStaleBlocksAfterPageUnlink( + Block& block, LifeCycleId lcIdx, LifeCycle const& lc) +{ + std::vector<SharedPtr<Block>> detachedBlocks; + TLLM_CHECK_DEBUG(block.storage.at(lcIdx) == nullptr); + if (block.isOrphan()) + { + return detachedBlocks; + } + + // Reuse cleanup only applies to attention lifecycles. + // SSM lifecycles are allowed in the tree but don't trigger subtree eviction. + auto const* const alc = std::get_if<AttnLifeCycle>(&lc); + NodeBase* pruneStart = █ + + // If this is a full-attention block or a sink block: evict subtree. + // Mirrors Python: pages = remove_subtree(self) + if (alc && (!alc->windowSize.has_value() || block.ordinal() < BlockOrdinal{alc->numSinkBlocks})) + { + pruneStart = block.prev; + detachedBlocks.push_back(removeSubtree(block)); + } + else if (block.eventSink) + { + block.eventSink->addRemovedLifeCycle(block.key, lcIdx); + } + + // Prune empty tail nodes up the chain. + // Save prev, key, and type before erasing, because the erase may destroy + // curr when its last shared_ptr is dropped. + Block* curr + = pruneStart && pruneStart->type() == NodeBase::Type::kBLOCK ? static_cast<Block*>(pruneStart) : nullptr; + while (curr && curr->next.empty() && curr->storage.at(lcIdx) == nullptr) + { + NodeBase* prevNode = curr->prev; + BlockKey const currKey = curr->key; + bool const prevIsBlock = prevNode && prevNode->type() == NodeBase::Type::kBLOCK; + if (prevNode) + { + auto detached = prevNode->detachNext(currKey); // may destroy curr + TLLM_CHECK_DEBUG(detached && detached.get() == curr); + detachedBlocks.push_back(std::move(detached)); + } + // Walk up only through Block nodes; stop at RootBlock. + curr = prevIsBlock ? static_cast<Block*>(prevNode) : nullptr; + } + return detachedBlocks; +} + +// --------------------------------------------------------------------------- +// addOrGetExistingBlock +// --------------------------------------------------------------------------- + +SharedPtr<Block> addOrGetExistingBlock( + NodeBase* prev, LifeCycleId numLifeCycles, std::vector<TokenIdExt> tokens, bool* isNew) +{ + TLLM_CHECK_DEBUG_WITH_INFO(prev, "prev must not be null"); + + // Prev must be a full block if it is a Block (mirrors Python: "prev must be a full block"). + if (prev->type() == NodeBase::Type::kBLOCK) + { + TLLM_CHECK_DEBUG_WITH_INFO(static_cast<Block*>(prev)->isFull(), "prev must be a full block"); + } + + auto& prevNext = prev->next; + int const tpb = prev->tokensPerBlock(); + BlockKey newKey = Block::makeKey(prev->key, tokens.data(), tokens.size()); + + // Exact match: return existing block (not new — mirrors Python's UselessBlockError path). + auto it = prevNext.find(newKey); + if (it != prevNext.end()) + { + if (isNew) + *isNew = false; + return it->second; + } + + // Useless check: is this block's token prefix covered by a sibling? + // Mirrors Python's UselessBlockError — throw with the sibling block. + if (static_cast<int>(tokens.size()) < tpb) + { + for (auto const& [k, sibling] : prevNext) + { + if (sibling->tokens.size() >= tokens.size() && isPrefix(tokens, sibling->tokens)) + throw UselessBlockError(sibling); + } + } + + // Remove siblings whose tokens are a strict prefix of ours. + std::vector<BlockKey> toRemove; + for (auto const& [k, sibling] : prevNext) + { + if (sibling->tokens.size() < tokens.size() && isPrefix(sibling->tokens, tokens)) + { + TLLM_CHECK_DEBUG(!sibling->isFull() && sibling->key == k && sibling->next.empty()); + toRemove.push_back(k); + } + } + for (auto const& k : toRemove) + { + auto erasedBlock = prev->detachNext(k); + TLLM_CHECK_DEBUG(erasedBlock); + TLLM_CHECK_DEBUG_WITH_INFO(erasedBlock->isOrphan(), "erased sibling must be orphan after removal"); + (void) erasedBlock; + } + + // Create the new block. ordinal and tokensPerBlock are derived from prev inside the Block ctor. + auto block = makeShared<Block>(newKey, std::move(tokens), prev, numLifeCycles); + + prevNext[newKey] = block; + if (isNew) + *isNew = true; + return block; +} + +// --------------------------------------------------------------------------- +// removeSubtree +// --------------------------------------------------------------------------- + +SharedPtr<Block> removeSubtree(Block& root) +{ + Block* current = &root; + SharedPtr<Block> detachedRoot; + + // Post-order traversal using prev/next links — O(1) extra space. + // Descend to leaves first, remove on the way back up. + // Each block's pages are reclaimed eagerly via releasePages() while the + // StorageManager is still alive, rather than deferring to ~Block(): an external + // reference can keep a Block alive past StorageManager teardown, after which + // page->manager would be dangling. Mirrors Python's remove_subtree(). + while (true) + { + // Descend: if the current block has children, go to the first child. + if (!current->next.empty()) + { + current = current->next.begin()->second.get(); + } + else + { + current->releasePages(); + // Remove this block from its parent's next map. + // Null prev to detach — the block may outlive the tree if held + // externally (e.g., by nanobind/Python shared_ptr). + NodeBase* parent = current->prev; + BlockKey const currentKey = current->key; + auto detached = parent->detachNext(currentKey); + TLLM_CHECK_DEBUG(detached && detached.get() == current); + (void) detached; + + if (current == &root) + { + detachedRoot = std::move(detached); + break; + } + + TLLM_CHECK_DEBUG(parent->type() == NodeBase::Type::kBLOCK); + current = static_cast<Block*>(parent); + } + } + TLLM_CHECK_DEBUG(detachedRoot); + return detachedRoot; +} + +// --------------------------------------------------------------------------- +// BlockRadixTree +// --------------------------------------------------------------------------- + +BlockRadixTree::BlockRadixTree( + LifeCycleRegistry const& lifeCycles, int tokensPerBlock, std::shared_ptr<EventSink> eventSink) + : mLifeCycles(lifeCycles) + , mTokensPerBlock(tokensPerBlock) + , mEventSink(std::move(eventSink)) +{ +} + +BlockRadixTree::~BlockRadixTree() +{ + // Clear all roots (which will drop all blocks). + mRoots.clear(); +} + +LifeCycleId BlockRadixTree::numLifeCycles() const noexcept +{ + return mLifeCycles.size(); +} + +void BlockRadixTree::drainPendingRootErases() const +{ + if (mPendingRootErases.empty()) + { + return; + } + // Move to local to allow re-entrancy (proposeToEraseEmptyRoot during erase). + std::vector<BlockKey> pending; + pending.swap(mPendingRootErases); + auto& roots = const_cast<std::unordered_map<BlockKey, SharedPtr<RootBlock>>&>(mRoots); + for (auto const& key : pending) + { + auto it = roots.find(key); + // Only erase if the root exists and is still childless. + if (it != roots.end() && it->second->next.empty()) + { + roots.erase(it); + } + } +} + +RootBlock& BlockRadixTree::addOrGetExisting(ReuseScope const& reuseScope) +{ + drainPendingRootErases(); + + BlockKey key = RootBlock::makeKey(reuseScope); + auto it = mRoots.find(key); + if (it != mRoots.end()) + { + return *it->second; + } + + auto rb = makeShared<RootBlock>(reuseScope, this); + auto [newIt, inserted] = mRoots.emplace(key, std::move(rb)); + return *newIt->second; +} + +// Among all child nodes, find the one whose tokens have the longest leading match. +// Returns (block, numMatchedTokens) or (nullptr, 0) if no match. +// Mirrors Python's find_best_partial_match_in_next_nodes(). +std::pair<Block*, int> findBestPartialMatchInNextNodes( + std::unordered_map<BlockKey, SharedPtr<Block>> const& nextMap, TokenIdExt const* tokens, size_t tokenCount) +{ + // Skip heuristic: too many children would be slow to iterate. + if (nextMap.size() >= 32) + return {nullptr, 0}; + Block* best = nullptr; + int bestMatch = 0; + for (auto const& [k, child] : nextMap) + { + int m = child->partialMatchThisNode(tokens, tokenCount); + if (m > bestMatch) + { + bestMatch = m; + best = child.get(); + } + } + return {best, bestMatch}; +} + +namespace +{ + +int numMatchedTokens(std::vector<BlockRadixTree::MatchResult> const& matched, int tokensPerBlock) +{ + if (matched.empty()) + { + return 0; + } + return tokensPerBlock * (static_cast<int>(matched.size()) - 1) + matched.back().numMatchedTokens; +} + +bool hasPage(Block const& block, LifeCycleId lcId) +{ + return block.storage.at(lcId) != nullptr; +} + +} // anonymous namespace + +std::vector<BlockRadixTree::MatchResult> BlockRadixTree::matchTokenPath( + ReuseScope const& reuseScope, std::vector<TokenIdExt> const& tokens, bool enablePartialMatch) const +{ + drainPendingRootErases(); + + std::vector<MatchResult> results; + + // Lazily compute one key per iteration — no wasted hashing on early miss. + auto gen = makeBlockchainKeyGenerator(mTokensPerBlock, reuseScope, tokens.data(), tokens.size()); + + // First key is the root key. + auto rootKey = gen(); + if (!rootKey) + return results; + auto rootIt = mRoots.find(*rootKey); + if (rootIt == mRoots.end()) + return results; + + RootBlock const& root = *rootIt->second; + std::unordered_map<BlockKey, SharedPtr<Block>> const* currentNext = &root.next; + // ordinal tracks which block we're on (0-based, after root). + BlockOrdinal ordinal{0}; + bool missed = false; + + while (auto key = gen()) + { + auto blockIt = currentNext->find(*key); + if (blockIt == currentNext->end()) + { + missed = true; + break; + } + size_t beg = toSizeT(ordinal) * static_cast<size_t>(mTokensPerBlock); + int numTokens = static_cast<int>(std::min(static_cast<size_t>(mTokensPerBlock), tokens.size() - beg)); + Block* block = blockIt->second.get(); + results.push_back({block, numTokens}); + currentNext = &block->next; + ordinal++; + } + + // Partial match in children of current node. + if (missed && enablePartialMatch) + { + size_t beg = toSizeT(ordinal) * static_cast<size_t>(mTokensPerBlock); + size_t missedCount = std::min(static_cast<size_t>(mTokensPerBlock), tokens.size() - beg); + auto [best, bestMatch] = findBestPartialMatchInNextNodes(*currentNext, tokens.data() + beg, missedCount); + if (best) + results.push_back({best, bestMatch}); + } + + return results; +} + +std::vector<BlockRadixTree::MatchResult> BlockRadixTree::pruneMatch(std::vector<MatchResult> matched) const +{ + // All blocks except the last must be fully matched (mirrors Python: matched[:-1]). + TLLM_CHECK_DEBUG(matched.size() <= 1 + || std::all_of(matched.begin(), matched.end() - 1, + [this](auto const& m) { return m.numMatchedTokens == mTokensPerBlock; })); + + auto attnLcs = mLifeCycles.attentionLifeCycles(); + + // Full-attention layers require pages on every matched block. + std::vector<LifeCycleId> fullAttnLcList; + for (auto [lcId, attn] : attnLcs) + { + if (!attn->windowSize.has_value()) + { + fullAttnLcList.push_back(lcId); + } + } + if (!fullAttnLcList.empty()) + { + int n = findIndex(matched.begin(), matched.end(), + [&](auto const& match) + { + return std::any_of(fullAttnLcList.begin(), fullAttnLcList.end(), + [&](LifeCycleId lcId) { return !hasPage(*match.block, lcId); }); + }); + matched.resize(static_cast<size_t>(n)); + } + + std::vector<std::pair<LifeCycleId, AttnLifeCycle const*>> swaLcs; + for (auto [lcId, attn] : attnLcs) + { + if (attn->windowSize.has_value()) + { + swaLcs.push_back({lcId, attn}); + } + } + + // SWA sink blocks must all be available. + for (auto [lcId, attn] : swaLcs) + { + int const sinkBlocks = attn->numSinkBlocks; + int const limit = std::min(sinkBlocks, static_cast<int>(matched.size())); + int n = findIndex(matched.begin(), matched.begin() + limit, + [&, lcId = lcId](auto const& match) { return !hasPage(*match.block, lcId); }); + if (n < sinkBlocks) + { + matched.resize(static_cast<size_t>(n)); + } + } + + auto ssmLcId = mLifeCycles.ssmLifeCycleId(); + while (!matched.empty()) + { + if (ssmLcId.has_value()) + { + // Truncate to the last block whose SSM snapshot is reusable at that + // block's matched-token count, then clamp the tail entry's matched + // token count to the snapshot length (mirrors _block_radix_tree.py). + int ssmTrunc = 0; + int ssmMatchLen = 0; + for (int i = static_cast<int>(matched.size()) - 1; i >= 0; --i) + { + CommittedPage* page = matched[static_cast<size_t>(i)].block->storage.at(*ssmLcId); + if (page == nullptr) + { + continue; + } + auto* ssmPage = dynamic_cast<SsmCommittedPage*>(page); + TLLM_CHECK_DEBUG(ssmPage != nullptr); + int const snapshotLen = ssmPage->numTokensInBlock; + if (matched[static_cast<size_t>(i)].numMatchedTokens >= snapshotLen) + { + ssmTrunc = i + 1; + ssmMatchLen = snapshotLen; + break; + } + } + matched.resize(static_cast<size_t>(ssmTrunc)); + if (matched.empty()) + { + break; + } + matched.back().numMatchedTokens = ssmMatchLen; + } + + int const numTok = numMatchedTokens(matched, mTokensPerBlock); + bool trimmed = false; + for (auto [lcId, attn] : swaLcs) + { + int n = findIndex(matched.rbegin(), matched.rend(), + [&, lcId = lcId](auto const& match) { return hasPage(*match.block, lcId); }); + if (n != 0) + { + matched.resize(matched.size() - static_cast<size_t>(n)); + trimmed = true; + break; + } + + auto staleRange = attn->getStaleRange(numTok, mTokensPerBlock); + BlockOrdinal const staleEnd = staleRange.end; + if (staleEnd < BlockOrdinal{static_cast<int>(matched.size())}) + { + auto tailBegin = matched.begin() + static_cast<ptrdiff_t>(toSizeT(staleEnd)); + int nMissing = findIndex(matched.rbegin(), std::make_reverse_iterator(tailBegin), + [&, lcId = lcId](auto const& match) { return !hasPage(*match.block, lcId); }); + if (BlockOrdinal{static_cast<int>(matched.size()) - nMissing} > staleEnd) + { + matched.resize(matched.size() - static_cast<size_t>(nMissing) - 1); + trimmed = true; + break; + } + } + } + if (!trimmed) + { + break; + } + } + + return matched; +} + +BlockRadixTree::ReuseMatch BlockRadixTree::match( + ReuseScope const& reuseScope, std::vector<TokenIdExt> const& tokens, bool enablePartialMatch) const +{ + auto const matched = pruneMatch(matchTokenPath(reuseScope, tokens, enablePartialMatch)); + ReuseMatch result{}; + result.numTokens = numMatchedTokens(matched, mTokensPerBlock); + result.numLookupTokens = static_cast<int>(tokens.size()); + result.blocks.reserve(BlockOrdinal{static_cast<int>(matched.size())}); + for (auto const& match : matched) + { + result.blocks.push_back(match.block); + } + return result; +} + +void BlockRadixTree::clear() +{ + // detachNext() may call proposeToEraseEmptyRoot, but won't modify mRoots directly. + for (auto& [rootKey, root] : mRoots) + { + while (!root->next.empty()) + { + removeSubtree(*root->next.begin()->second); + } + } + TLLM_CHECK_DEBUG(mRoots.size() == mPendingRootErases.size()); + mRoots.clear(); + mPendingRootErases.clear(); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/blockRadixTree.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/blockRadixTree.h new file mode 100644 index 000000000000..c3ae3035415c --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/blockRadixTree.h @@ -0,0 +1,356 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/eventSink.h" +#include "kv_cache_manager_v2/lifeCycleRegistry.h" +#include "kv_cache_manager_v2/utils/sharedPtr.h" + +#include "sha256.h" + +#include <array> +#include <cstdint> +#include <iterator> +#include <memory> +#include <optional> +#include <unordered_map> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// Forward declarations +class CommittedPage; +class BlockRadixTree; +struct NodeBase; +struct RootBlock; +struct Block; + +// --------------------------------------------------------------------------- +// ReuseScope — per-request namespace for prefix reuse. +// Mirrors Python's ReuseScope(lora_id, salt). +// --------------------------------------------------------------------------- +struct ReuseScope +{ + std::optional<LoraTaskIdType> loraId; + std::optional<std::uint64_t> salt; + + bool operator==(ReuseScope const& other) const noexcept + { + return loraId == other.loraId && salt == other.salt; + } +}; + +// --------------------------------------------------------------------------- +// BlockKey — SHA-256 digest (32 bytes), used as radix-tree node identifier. +// Matches Python's hashlib.sha256 32-byte digest. +// +// SECURITY INVARIANT: the block hash MUST remain cryptographically +// collision-resistant and >= 256-bit. The radix tree is a globally shared, +// cross-request/cross-tenant cache index, prefix matching is decided purely by +// digest equality with NO re-verification of the underlying tokens, and the +// hashed input (tokens, the user-supplied cache_salt in ReuseScope, multimodal +// content bytes) is attacker-influenceable. A hash collision therefore silently +// reuses another request's KV blocks (cross-request corruption / data leak), +// and tenant isolation via cache_salt relies entirely on this hash's collision +// resistance. Do NOT substitute a non-cryptographic hash (xxHash, HighwayHash, +// City, ...) or truncate below 256 bits without first adding a token-content +// equality check on match. +// --------------------------------------------------------------------------- +using BlockKey = Digest; +static_assert(kDIGEST_LEN == CSHA256::OUTPUT_SIZE); // 32 bytes + +// --------------------------------------------------------------------------- +// Hasher — thin wrapper around SHA-256 (CSHA256) for incremental digests. +// Mirrors Python's Hasher class (hashlib.sha256). See the SECURITY INVARIANT on +// BlockKey above before changing the hash algorithm or digest width. +// --------------------------------------------------------------------------- +class Hasher +{ +public: + Hasher(); + explicit Hasher(ReuseScope const& seed); + + Hasher& update(TokenId token); + Hasher& update(BlockKey const& key); + Hasher& update(ReuseScope const& scope); + Hasher& update(std::vector<uint8_t> const& bytes); + Hasher& update(TokenIdExt const& tokenExt); + Hasher& update(TokenIdExt const* tokens, size_t count); + + BlockKey digest() const; + +private: + CSHA256 mState; +}; + +// --------------------------------------------------------------------------- +// Utility: convert a token sequence → list of BlockKeys. +// First key is the root (reuseScope digest), then one per token block. +// Mirrors Python's sequence_to_blockchain_keys(). +// --------------------------------------------------------------------------- +std::vector<BlockKey> sequenceToBlockchainKeys( + int tokensPerBlock, ReuseScope const& reuseScope, std::vector<TokenIdExt> const& tokens); + +// Generate multi-modal token IDs (mirrors gen_multi_modal_tokens in Python). +std::vector<TokenIdExt> genMultiModalTokens( + int idOffset, std::vector<uint8_t> const& multiModalDataDigest, int numTokens, int tokenOffset = 0); + +// --------------------------------------------------------------------------- +// NodeBase — common base for RootBlock and Block (nodes in the radix tree). +// Holds shared fields: key, next map, ordinal, and tokens-per-block. +// Mirrors Python's common interface between RootBlock and Block. +// --------------------------------------------------------------------------- +struct NodeBase +{ + enum class Type : uint8_t + { + kROOT_BLOCK, + kBLOCK + }; + + BlockKey key; + std::unordered_map<BlockKey, SharedPtr<Block>> next; + EventSink* eventSink; + + virtual ~NodeBase(); + + virtual Type type() const noexcept = 0; + virtual BlockOrdinal ordinal() const noexcept = 0; + + SharedPtr<Block> detachNext(BlockKey const& key); + + /// RootBlock: delegates to tree. Block: len(prev->tokens) or prev->tokensPerBlock(). + virtual int tokensPerBlock() const noexcept = 0; + +protected: + NodeBase(BlockKey k, EventSink* sink) + : key(k) + , eventSink(sink) + { + } +}; + +// --------------------------------------------------------------------------- +// RootBlock — one root per ReuseScope in a BlockRadixTree. +// Holds a map of child Blocks keyed by BlockKey. +// Mirrors Python's RootBlock. +// --------------------------------------------------------------------------- +struct RootBlock : NodeBase +{ + ReuseScope reuseScope; + BlockRadixTree* tree; // back-reference (non-owning) + + RootBlock(ReuseScope reuseScope, BlockRadixTree* tree); + + static BlockKey makeKey(ReuseScope const& reuseScope); + + Type type() const noexcept override + { + return Type::kROOT_BLOCK; + } + + BlockOrdinal ordinal() const noexcept override + { + return kBadBlockOrdinal; + } + + int tokensPerBlock() const noexcept override; +}; + +// --------------------------------------------------------------------------- +// Block — one full (or partial) token block in the radix tree. +// storage[lifeCycleId] = raw observer pointer to CommittedPage (null if not cached). +// Mirrors Python's Block. +// --------------------------------------------------------------------------- +struct Block : NodeBase, EnableSharedFromThis<Block> +{ + std::vector<TokenIdExt> tokens; + + // Previous node in the chain (RootBlock or Block). Null after detaching from the tree. + // Raw non-owning pointer: while attached, the prev node's `next` map owns us via shared_ptr. + NodeBase* prev{nullptr}; + + TypedVec<LifeCycleId, CommittedPage*> storage; + + Block(BlockKey key, std::vector<TokenIdExt> tokens, NodeBase* prev, LifeCycleId numLifeCycles); + ~Block() override; + + static BlockKey makeKey(BlockKey const& prevKey, TokenIdExt const* tokens, size_t count); + + Type type() const noexcept override + { + return Type::kBLOCK; + } + + BlockOrdinal ordinal() const noexcept override + { + return mOrdinal; + } + + int tokensPerBlock() const noexcept override; + + LifeCycleId numLifeCycles() const noexcept + { + return storage.size(); + } + + bool isFull() const noexcept + { + return static_cast<int>(tokens.size()) == tokensPerBlock(); + } + + bool isOrphan() const noexcept; + + // Returns how many leading tokens match `otherTokens`. + int partialMatchThisNode(TokenIdExt const* otherTokens, size_t count) const; + + // Break the bidirectional link to the cached page for a lifecycle. + // Returns the previously-stored CommittedPage* (nullptr if already unlinked). + // If `expectedPage` is non-null and the stored page differs from it, the link + // is left untouched and nullptr is returned (mirrors Python's unset_page + // `expected_page` guard: a newer page may already occupy the slot). + CommittedPage* unlinkPage(LifeCycleId lcIdx, CommittedPage* expectedPage = nullptr); + + // Return the cached page for a lifecycle (nullptr if none). Mirrors Python's Block.get_page(). + CommittedPage* getPage(LifeCycleId lcIdx) const + { + return storage[lcIdx]; + } + + // Clear stale tree nodes after a lifecycle page has been unlinked. + // Returns detached blocks that must stay alive until cleanup completes. + static std::vector<SharedPtr<Block>> clearStaleBlocksAfterPageUnlink( + Block& block, LifeCycleId lcIdx, LifeCycle const& lc); + + // Reclaim every page held by this block: null each page's back-pointer and, for + // DROPPABLE pages still scheduled for eviction, remove them from the eviction + // controller (releasing their storage slots). Idempotent. Must run during tree + // teardown (removeSubtree) rather than being deferred to ~Block(), so page + // reclamation does not depend on this Block's destruction timing — an external + // reference can keep a Block alive past StorageManager teardown, after which + // page->manager would be dangling. Mirrors Python's Block._release_pages(). + void releasePages(); + +private: + BlockOrdinal mOrdinal; +}; + +// --------------------------------------------------------------------------- +// BlockRadixTree — the global cache index. +// next: reuseScope digest → RootBlock. +// Mirrors Python's BlockRadixTree. +// --------------------------------------------------------------------------- +class BlockRadixTree +{ +public: + BlockRadixTree( + LifeCycleRegistry const& lifeCycles, int tokensPerBlock, std::shared_ptr<EventSink> eventSink = nullptr); + ~BlockRadixTree(); + + // Get (or create) the RootBlock for the given reuse scope. + RootBlock& addOrGetExisting(ReuseScope const& reuseScope); + + // Match tokens against the tree, yielding (block, numMatchedTokens) pairs. + // Partial matching: if enablePartialMatch, also yields blocks with a partial + // leading-token match. + struct MatchResult + { + Block* block; + int numMatchedTokens; + }; + + struct ReuseMatch + { + TypedVec<BlockOrdinal, Block*> blocks; + int numTokens; + // Total query length passed to match() (== len(tokens)). + int numLookupTokens; + }; + + ReuseMatch match( + ReuseScope const& reuseScope, std::vector<TokenIdExt> const& tokens, bool enablePartialMatch = false) const; + + // Clear all cached pages. ~Block() handles excludeFromEviction for DROPPABLE pages. + void clear(); + + int tokensPerBlock() const noexcept + { + return mTokensPerBlock; + } + + LifeCycleId numLifeCycles() const noexcept; + + LifeCycleRegistry const& lifeCycles() const noexcept + { + return mLifeCycles; + } + + std::shared_ptr<EventSink> const& eventSink() const noexcept + { + return mEventSink; + } + + // Read-only access to the root map (used by nanobind introspection). + std::unordered_map<BlockKey, SharedPtr<RootBlock>> const& roots() const noexcept + { + return mRoots; + } + + // Propose removal of an empty root block. Deferred to avoid destroying + // objects during destructor chains. Drained at safe points (addOrGetExisting, match). + void proposeToEraseEmptyRoot(BlockKey const& key) + { + mPendingRootErases.push_back(key); + } + +private: + std::vector<MatchResult> matchTokenPath( + ReuseScope const& reuseScope, std::vector<TokenIdExt> const& tokens, bool enablePartialMatch) const; + std::vector<MatchResult> pruneMatch(std::vector<MatchResult> matched) const; + + // Erase any pending empty root blocks from mRoots. + // Const-qualified: deferred cleanup is not a logical mutation. + void drainPendingRootErases() const; + + LifeCycleRegistry const& mLifeCycles; + int mTokensPerBlock; + std::shared_ptr<EventSink> mEventSink; + + std::unordered_map<BlockKey, SharedPtr<RootBlock>> mRoots; + mutable std::vector<BlockKey> mPendingRootErases; +}; + +// --------------------------------------------------------------------------- +// Helpers used by Block and the tree traversal. +// --------------------------------------------------------------------------- + +// Add a block to prev's `next` map, or return the existing one on collision. +// Throws UselessBlockError (with the sibling block) if the block's tokens are a +// prefix of an existing sibling — mirrors Python's UselessBlockError. +// If isNew is non-null, *isNew is set to true if a new block was created, false +// if an existing block was returned. +SharedPtr<Block> addOrGetExistingBlock( + NodeBase* prev, LifeCycleId numLifeCycles, std::vector<TokenIdExt> tokens, bool* isNew = nullptr); + +// Post-order traversal: remove a subtree rooted at `root` from its parent's +// next map. ~Block() handles page cleanup. Mirrors Python's remove_subtree(). +SharedPtr<Block> removeSubtree(Block& root); + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/common.cpp similarity index 64% rename from cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.h rename to cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/common.cpp index d8d8af1ef220..bd2928adb2f3 100644 --- a/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.h +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/common.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,16 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#pragma once +#include "kv_cache_manager_v2/common.h" #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/cudaUtils.h" -namespace tensorrt_llm::plugins +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 { -void logError(char const* msg, char const* file, char const* fn, int line); +bool const gDebug = tensorrt_llm::DebugConfig::isCheckDebugEnabled(); -void caughtError(std::exception const& e); - -} // namespace tensorrt_llm::plugins +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/common.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/common.h new file mode 100644 index 000000000000..0f53938b95e2 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/common.h @@ -0,0 +1,243 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/utils/typedIndex.h" +#include "tensorrt_llm/batch_manager/common.h" + +#include <array> +#include <cstdint> +#include <cstring> +#include <memory> +#include <optional> +#include <string> +#include <sys/types.h> +#include <variant> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// Debug flag — true when TLLM_DEBUG_MODE=1. +// Delegates to DebugConfig::isCheckDebugEnabled() for consistency with TLLM_CHECK_DEBUG. +// --------------------------------------------------------------------------- +extern bool const gDebug; // true == debug mode (expensive assertions enabled) + +// --------------------------------------------------------------------------- +// Enumerations +// --------------------------------------------------------------------------- + +enum class PageStatus : int +{ + LOCKED = 0, // Required in GPU. Eviction/dropping not allowed. + HELD = 1, // Allow eviction but not dropping. + DROPPABLE = 2, // Allow eviction and dropping. +}; + +enum class CacheTier : int +{ + GPU_MEM = 0, + HOST_MEM = 1, + DISK = 2, +}; + +// PageIndexMode — how converted page indices relate to layers within a layer group. +// Mirrors _common.py::PageIndexMode. +enum class PageIndexMode : int +{ + // Converted index list is shared across layers in the same LayerGroup. + // Base pointer is per-layer (includes attr.offset). + SHARED = 0, + // Converted index list is per-layer. + // Base pointer is shared (pool group base, no attr.offset). + PER_LAYER = 1, +}; + +// --------------------------------------------------------------------------- +// Strongly-typed integer aliases (mirroring Python NewType wrappers). +// --------------------------------------------------------------------------- + +// Index of a cache level (0 = GPU, 1 = host, 2 = disk, ...). +using CacheLevel = StrongIndex<int, struct CacheLevelTag, 0>; +inline constexpr CacheLevel kGpuLevel{0}; + +// Vocabulary token identifier (normal tokens only). +using TokenId = int64_t; + +// Opaque request identifier shared with the rest of the batch manager. +using RequestIdType = tensorrt_llm::batch_manager::RequestIdType; + +// Opaque LoRA task identifier shared with the rest of the batch manager. +using LoraTaskIdType = tensorrt_llm::runtime::LoraTaskIdType; + +// 32-byte aligned to enable SIMD. +inline constexpr int kDIGEST_LEN = 32; + +struct alignas(kDIGEST_LEN) Digest : std::array<std::byte, kDIGEST_LEN> +{ + // Custom operator== needed to emit SIMD code + bool operator==(Digest const& o) const noexcept + { + return std::memcmp(this, &o, kDIGEST_LEN) == 0; + } + + bool operator!=(Digest const& o) const noexcept + { + return !(*this == o); + } +}; + +// Heap-allocated digest token for multi-modal tokens. +// Copyable (deep-copies the digest) with value-based equality. +// Digest tokens are rare, so unique_ptr keeps sizeof(TokenIdExt) small. +class DigestToken +{ +public: + explicit DigestToken(Digest const& d) + : mData(std::make_unique<Digest>(d)) + { + } + + explicit DigestToken(std::unique_ptr<Digest> d) + : mData(std::move(d)) + { + } + + DigestToken(DigestToken const& o) + : mData(std::make_unique<Digest>(*o.mData)) + { + } + + DigestToken(DigestToken&&) noexcept = default; + + DigestToken& operator=(DigestToken const& o) + { + if (this != &o) + mData = std::make_unique<Digest>(*o.mData); + return *this; + } + + DigestToken& operator=(DigestToken&&) noexcept = default; + + bool operator==(DigestToken const& o) const + { + return *mData == *o.mData; + } + + bool operator!=(DigestToken const& o) const + { + return !(*this == o); + } + + std::byte const* data() const noexcept + { + return mData->data(); + } + + size_t size() const noexcept + { + return mData->size(); + } + + Digest const& digest() const noexcept + { + return *mData; + } + +private: + std::unique_ptr<Digest> mData; +}; + +// Extended token id: normal TokenId or a heap-allocated digest for multi-modal tokens. +using TokenIdExt = std::variant<TokenId, DigestToken>; + +// Ordinal index of a KV cache block (sequence of tokens). +using BlockOrdinal = StrongIndex<int, struct BlockOrdinalTag, -1>; +inline constexpr BlockOrdinal kBadBlockOrdinal{-1}; + +// Identifier of an attention layer. +using LayerId = int; + +// Raw CUDA stream handle (CUstream cast to uintptr_t). +using CudaStream = uintptr_t; + +// Index of a beam in beam-search. +using BeamIndex = StrongIndex<int, struct BeamIndexTag, 0>; +inline constexpr BeamIndex kDefaultBeamIndex{0}; + +// User-defined request/session identifier. +using UserId = int64_t; + +// Host or device memory address (uintptr_t). +using MemAddress = std::uintptr_t; + +// OS file descriptor. +using FileDescriptor = int; +inline constexpr FileDescriptor kBadFileDescriptor = -1; + +// Index into a page table. +using PageIndex = StrongIndex<int, struct PageIndexTag, -1>; +inline constexpr PageIndex kBadPageIndex{-1}; + +// Eviction priority (0 = highest priority to evict, 100 = lowest). +using Priority = int; +inline constexpr Priority kPriorityMin = 0; +inline constexpr Priority kPriorityMax = 100; +inline constexpr Priority kPriorityDefault = 35; + +// Optional sliding window size (nullopt = no sliding window). +using SlidingWindowSize = std::optional<int>; + +// --------------------------------------------------------------------------- +// Address types +// --------------------------------------------------------------------------- + +// Disk address: (fd, byte-offset). +struct DiskAddress +{ + int fd = kBadFileDescriptor; + ssize_t pos = 0; + + bool operator==(DiskAddress const& o) const noexcept + { + return fd == o.fd && pos == o.pos; + } +}; + +// Unified address: either a host/device memory pointer or a disk address. +using Address = std::variant<MemAddress, DiskAddress>; + +// --------------------------------------------------------------------------- +// DataRole — string-typed tag for a buffer inside one attention layer. +// --------------------------------------------------------------------------- +using DataRole = std::string; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 + +// std::hash specialization for Digest/BlockKey so unordered_map works without a custom hasher. +template <> +struct std::hash<tensorrt_llm::batch_manager::kv_cache_manager_v2::Digest> +{ + size_t operator()(tensorrt_llm::batch_manager::kv_cache_manager_v2::Digest const& k) const noexcept + { + // First 8 bytes of a SHA-256 digest are already well-distributed. + uint64_t v; + std::memcpy(&v, k.data(), sizeof(v)); + return static_cast<size_t>(v); + } +}; diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/config.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/config.cpp new file mode 100644 index 000000000000..c31ea7be3586 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/config.cpp @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/config.h" +#include "kv_cache_manager_v2/exceptions.h" + +#include <filesystem> +#include <set> +#include <stdexcept> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +void DiskCacheTierConfig::assertValid() const +{ + if (quota == 0) + { + throw std::invalid_argument("DiskCacheTierConfig: quota must be > 0"); + } + if (!std::filesystem::is_directory(path)) + { + throw std::invalid_argument("DiskCacheTierConfig: path '" + path + "' is not a directory"); + } +} + +void KVCacheManagerConfig::validate() const +{ + if (swaScratchReuse.has_value()) + { + swaScratchReuse->validate(); + } + + // These mirror Python's KVCacheManagerConfig.__post_init__ asserts, so they + // throw AssertionError (translated in the binding layer) rather than ValueError. + if (cacheTiers.empty() || cacheTierOf(cacheTiers[0]) != CacheTier::GPU_MEM) + { + throw AssertionError("KVCacheManagerConfig: first cache tier must be GPU memory"); + } + + // Check for duplicate layer ids. + std::set<LayerId> seenLayerIds; + for (auto const& layer : layers) + { + std::visit( + [&](auto const& cfg) + { + if (!seenLayerIds.insert(cfg.layerId).second) + { + throw AssertionError("KVCacheManagerConfig: duplicate layer id"); + } + for (auto const& buf : cfg.buffers) + { + if (buf.tokensPerBlockOverride.has_value() + && (*buf.tokensPerBlockOverride <= 0 || tokensPerBlock % *buf.tokensPerBlockOverride != 0)) + { + throw AssertionError( + "KVCacheManagerConfig: tokensPerBlockOverride must be a divisor of " + "tokensPerBlock"); + } + } + }, + layer); + } + + // SSM-specific validation. + bool hasSSM = false; + for (auto const& layer : layers) + { + if (std::holds_alternative<SsmLayerConfig>(layer)) + { + hasSSM = true; + break; + } + } + if (hasSSM) + { + if (!commitMinSnapshot) + throw AssertionError("KVCacheManagerConfig: commit_min_snapshot must be True when SSM layers are present"); + } +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/config.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/config.h new file mode 100644 index 000000000000..6852ed51f5a1 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/config.h @@ -0,0 +1,304 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" + +#include "tensorrt_llm/common/assert.h" +#include <optional> +#include <stdexcept> +#include <string> +#include <unordered_set> +#include <variant> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// Cache tier configuration structs (mirrors _config.py) +// --------------------------------------------------------------------------- + +struct GpuCacheTierConfig +{ + size_t quota = 0; // bytes + + CacheTier tier() const noexcept + { + return CacheTier::GPU_MEM; + } + + void assertValid() const + { + if (quota == 0) + throw std::invalid_argument("GpuCacheTierConfig: quota must be > 0"); + } +}; + +struct HostCacheTierConfig +{ + size_t quota = 0; // bytes + + CacheTier tier() const noexcept + { + return CacheTier::HOST_MEM; + } + + void assertValid() const + { + if (quota == 0) + throw std::invalid_argument("HostCacheTierConfig: quota must be > 0"); + } +}; + +struct DiskCacheTierConfig +{ + size_t quota = 0; // bytes + std::string path; // directory for temp files + + CacheTier tier() const noexcept + { + return CacheTier::DISK; + } + + void assertValid() const; +}; + +// Variant holding any tier config. +using CacheTierConfig = std::variant<GpuCacheTierConfig, HostCacheTierConfig, DiskCacheTierConfig>; + +// Helper to extract tier from a variant. +inline CacheTier cacheTierOf(CacheTierConfig const& cfg) +{ + return std::visit([](auto const& c) { return c.tier(); }, cfg); +} + +inline size_t cacheTierQuota(CacheTierConfig const& cfg) +{ + return std::visit([](auto const& c) { return c.quota; }, cfg); +} + +// --------------------------------------------------------------------------- +// Buffer configuration (one KV buffer inside an attention layer). +// --------------------------------------------------------------------------- + +struct BufferConfig +{ + DataRole role; + size_t size = 0; // bytes per page (without expansion) + + // If set, overrides tokens_per_block for this buffer. + // Must be a divisor of KVCacheManagerConfig::tokensPerBlock. + std::optional<int> tokensPerBlockOverride; +}; + +// --------------------------------------------------------------------------- +// Layer type discriminator. +// --------------------------------------------------------------------------- + +enum class LayerType : int +{ + ATTENTION = 0, + SSM = 1, +}; + +// --------------------------------------------------------------------------- +// Attention layer configuration. +// --------------------------------------------------------------------------- + +namespace detail +{ + +inline void validateNoDuplicateBufferRoles(std::vector<BufferConfig> const& buffers) +{ + std::unordered_set<DataRole> roles; + for (auto const& buf : buffers) + { + if (!roles.insert(buf.role).second) + throw std::invalid_argument("duplicate buffer role"); + } +} + +} // namespace detail + +struct AttentionLayerConfig +{ + static constexpr LayerType type = LayerType::ATTENTION; + + LayerId layerId = 0; + std::vector<BufferConfig> buffers; + + // nullopt = no sliding window. + std::optional<int> slidingWindowSize; + + // nullopt or 0 = no sink tokens. + std::optional<int> numSinkTokens; + + std::optional<int> windowSize() const noexcept + { + return slidingWindowSize; + } + + void validate() const + { + detail::validateNoDuplicateBufferRoles(buffers); + } +}; + +// --------------------------------------------------------------------------- +// SSM (State Space Model) layer configuration. +// --------------------------------------------------------------------------- + +struct SsmLayerConfig +{ + static constexpr LayerType type = LayerType::SSM; + + LayerId layerId = 0; + std::vector<BufferConfig> buffers; + + void validate() const + { + detail::validateNoDuplicateBufferRoles(buffers); + for (auto const& buf : buffers) + { + if (buf.tokensPerBlockOverride.has_value()) + throw std::invalid_argument("tokensPerBlockOverride not supported for SSM layers"); + } + } +}; + +using LayerConfig = std::variant<AttentionLayerConfig, SsmLayerConfig>; + +// --------------------------------------------------------------------------- +// KVCacheDesc — describes one KV cache request's capacity and history length. +// Mirrors _config.py::KVCacheDesc. +// --------------------------------------------------------------------------- +struct KVCacheDesc +{ + int capacity = 0; + int historyLength = 0; + + void validate() const + { + TLLM_CHECK_DEBUG(0 <= historyLength && historyLength <= capacity); + } + + // Value equality, mirroring the Python @dataclass(frozen=True) semantics the + // bindings replace. Required so tests can compare descs by value. + bool operator==(KVCacheDesc const& other) const noexcept + { + return capacity == other.capacity && historyLength == other.historyLength; + } + + bool operator!=(KVCacheDesc const& other) const noexcept + { + return !(*this == other); + } +}; + +// --------------------------------------------------------------------------- +// BatchDesc — a batch of requests that the KVCacheManager must support. +// Mirrors _config.py::BatchDesc. +// --------------------------------------------------------------------------- +struct BatchDesc +{ + std::vector<KVCacheDesc> kvCaches; + int systemPromptLength = 0; // tokens shared by all requests (0 if no reuse) + + void validate() const + { + TLLM_CHECK_DEBUG(systemPromptLength >= 0); + } + + // Value equality, mirroring the Python @dataclass(frozen=True) semantics the + // bindings replace. Uses KVCacheDesc::operator== elementwise. + bool operator==(BatchDesc const& other) const noexcept + { + return systemPromptLength == other.systemPromptLength && kvCaches == other.kvCaches; + } + + bool operator!=(BatchDesc const& other) const noexcept + { + return !(*this == other); + } +}; + +// --------------------------------------------------------------------------- +// SWA scratch reuse configuration. +// --------------------------------------------------------------------------- +struct SwaScratchReuseConfig +{ + int maxRewindLen = 0; + + void validate() const + { + if (maxRewindLen < 0) + { + throw std::invalid_argument("SwaScratchReuseConfig: max_rewind_len must be non-negative"); + } + } +}; + +// --------------------------------------------------------------------------- +// Top-level KV cache manager configuration (mirrors _config.py::KVCacheManagerConfig). +// --------------------------------------------------------------------------- + +struct KVCacheManagerConfig +{ + int tokensPerBlock = 0; + + // Ordered from warm (GPU) to cold (disk). First must be GPU memory. + std::vector<CacheTierConfig> cacheTiers; + + // Layer configs (attention or SSM). Layer IDs must be unique. + std::vector<LayerConfig> layers; + + // Suspend/resume threshold: if utilization > this, resuming will fail. + float maxUtilForResume = 0.97f; + + // Try to reuse tokens from partially matched blocks. + bool enablePartialReuse = true; + + // Constraint-based memory partitioning. + std::vector<BatchDesc> constraints; // batches that must always be supportable + std::optional<BatchDesc> typicalStep; // typical step for initial ratio computation + std::optional<std::vector<float>> initialPoolRatio; // explicit initial ratio, overrides inferred sizing inputs + + // When set, SWA layers reuse physical pages for out-of-window blocks during prefill. + // Scratch blocks share coalesced slot sub-pages across blocks for the currently executing + // layer, reducing peak memory. Trade-off: KV cache reuse is degraded because scratch blocks + // have no preserved data after the step. + std::optional<SwaScratchReuseConfig> swaScratchReuse; + + // If true, commit() records only the minimum cache snapshot reusable at the post-call + // numCommittedTokens. Only the minimum amount of pages required for such reuse will be + // preserved. Required when SSM layers are present. + bool commitMinSnapshot = false; + + // Collect V2 KV cache allocation, reuse, and transfer statistics. + bool enableStats = true; + + bool enableSwaScratchReuse() const noexcept + { + return swaScratchReuse.has_value(); + } + + void validate() const; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/copyEngine.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/copyEngine.cpp new file mode 100644 index 000000000000..d8579005b2f1 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/copyEngine.cpp @@ -0,0 +1,256 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/copyEngine.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/exceptions.h" +#include "kv_cache_manager_v2/utils/math.h" + +// Reuse existing copy implementations (no Python round-trip). +#include "tensorrt_llm/batch_manager/kvCacheManagerV2Utils.h" + +#include "tensorrt_llm/common/assert.h" +#include <stdexcept> +#include <variant> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// dispatchCopy<DstTier, SrcTier> — template that maps CacheTier pair to +// address types and the underlying copy function via if-constexpr. +// Replaces 7 near-identical static dispatcher functions. +// --------------------------------------------------------------------------- + +template <CacheTier Tier> +using TierAddr = std::conditional_t<Tier == CacheTier::DISK, DiskAddress, MemAddress>; + +template <CacheTier DstTier, CacheTier SrcTier> +void dispatchCopy(std::vector<CopyTask> const& tasks, size_t numBytes, CUstream stream) +{ + using DstAddr = TierAddr<DstTier>; + using SrcAddr = TierAddr<SrcTier>; + + std::vector<Task<DstAddr, SrcAddr>> t; + t.reserve(tasks.size()); + for (auto const& task : tasks) + t.push_back({std::get<DstAddr>(task.dst), std::get<SrcAddr>(task.src)}); + + constexpr auto G = CacheTier::GPU_MEM; + constexpr auto H = CacheTier::HOST_MEM; + constexpr auto D = CacheTier::DISK; + + // clang-format off + if constexpr (DstTier == G && SrcTier == G) cuCheck(copyDeviceToDevice(std::move(t), static_cast<ssize_t>(numBytes), stream)); + else if constexpr (DstTier == G && SrcTier == H) cuCheck(copyHostToDevice(std::move(t), static_cast<ssize_t>(numBytes), stream)); + else if constexpr (DstTier == H && SrcTier == G) cuCheck(copyDeviceToHost(std::move(t), static_cast<ssize_t>(numBytes), stream)); + else if constexpr (DstTier == H && SrcTier == H) cuCheck(copyHostToHost(std::move(t), static_cast<ssize_t>(numBytes), stream)); + else if constexpr (DstTier == D && SrcTier == D) cuCheck(copyDiskToDisk(std::move(t), static_cast<ssize_t>(numBytes), stream)); + else if constexpr (DstTier == H && SrcTier == D) cuCheck(copyDiskToHost(std::move(t), static_cast<ssize_t>(numBytes), stream)); + else if constexpr (DstTier == D && SrcTier == H) cuCheck(copyHostToDisk(std::move(t), static_cast<ssize_t>(numBytes), stream)); + // clang-format on +} + +// --------------------------------------------------------------------------- +// StagingBuffer +// --------------------------------------------------------------------------- + +StagingBuffer::StagingBuffer(StagingBufferManager& manager, size_t minSize, size_t maxSize, CUstream stream) + : mManager(manager) + , mStream(stream) +{ + if (minSize > manager.totalSize()) + { + throw std::invalid_argument("StagingBuffer: minSize exceeds total staging buffer size"); + } + + std::unique_lock<std::mutex> lock(mManager.mMutex); + + // Compute how many contiguous grains to use. If the suffix cannot satisfy + // the required minimum, skip it and wrap before allocating. + size_t const minGrains = divUp(minSize, kGranularity); + size_t availableGrains = mManager.suggestNextMaxGrains(); + if (minGrains > availableGrains) + { + mManager.mNext = 0; + availableGrains = mManager.suggestNextMaxGrains(); + } + TLLM_CHECK_DEBUG(minGrains <= availableGrains); + + size_t const available = availableGrains * kGranularity; + size_t actualSize = std::min(maxSize, available); + actualSize = std::max(actualSize, minSize); + mNumGrains = divUp(actualSize, kGranularity); + TLLM_CHECK_DEBUG(mNumGrains <= availableGrains); + mSize = actualSize; + mStartGrain = mManager.mNext; + mManager.mNext += mNumGrains; + TLLM_CHECK_DEBUG(mManager.mNext <= mManager.numGrains()); + if (mManager.mNext == mManager.numGrains()) + { + mManager.mNext = 0; + } + + mAddress = mManager.baseAddress() + mStartGrain * kGranularity; + lock.unlock(); + + // Lock grains and collect their ready events for deduplicated waiting. + // Mirrors Python's stream_wait_events(stream, lock_and_consume_events()) which + // deduplicates via set() — adjacent grains often share the same event. + std::vector<CachedCudaEvent const*> readyEvents; + readyEvents.reserve(mNumGrains); + for (size_t i = 0; i < mNumGrains; ++i) + { + GrainMetadata& g = mManager.mGrains[mStartGrain + i]; + g.mutex.lock(); + readyEvents.push_back(&g.readyEvent); + } + streamWaitEvents(reinterpret_cast<CudaStream>(mStream), readyEvents); + for (size_t i = 0; i < mNumGrains; ++i) + mManager.mGrains[mStartGrain + i].readyEvent.close(); +} + +StagingBuffer::~StagingBuffer() +{ + // One shared completion event for all grains (all on the same stream → same completion point). + CachedCudaEvent finishEvent(reinterpret_cast<CudaStream>(mStream)); + for (int i = static_cast<int>(mNumGrains) - 1; i >= 0; --i) + { + GrainMetadata& g = mManager.mGrains[mStartGrain + static_cast<size_t>(i)]; + g.readyEvent = finishEvent; + g.mutex.unlock(); + } +} + +// --------------------------------------------------------------------------- +// StagingBufferManager +// --------------------------------------------------------------------------- + +StagingBufferManager::StagingBufferManager(size_t size) + : mBuffer(size) + , mGrains(size / kGranularity) +{ + TLLM_CHECK_DEBUG(size % kGranularity == 0); +} + +StagingBuffer StagingBufferManager::acquire(size_t minSize, size_t maxSize, CUstream stream) +{ + return StagingBuffer(*this, minSize, maxSize, stream); +} + +// --------------------------------------------------------------------------- +// CopyEngine +// --------------------------------------------------------------------------- + +StagingBufferManager& CopyEngine::getStagingManager() +{ + if (!mStagingManager) + mStagingManager = std::make_unique<StagingBufferManager>(64u << 20u); // 64 MB + return *mStagingManager; +} + +// Two-hop transfer via host staging buffer (e.g., GPU→Disk or Disk→GPU). +// SrcTier → MidTier (staging) → DstTier. MidTier is the staging tier (HOST_MEM). +template <CacheTier DstTier, CacheTier MidTier, CacheTier SrcTier> +static void twoHopTransfer( + StagingBufferManager& manager, size_t numBytes, std::vector<CopyTask> const& tasks, CUstream stream) +{ + size_t remaining = tasks.size(); + size_t offset = 0; + + while (remaining > 0) + { + StagingBuffer buf = manager.acquire(numBytes, numBytes * remaining, stream); + MemAddress addr = buf.address(); + size_t n = buf.size() / numBytes; + TLLM_CHECK_DEBUG(n > 0 && n <= remaining); + + // First hop: src → staging + { + std::vector<CopyTask> hop1; + hop1.reserve(n); + for (size_t i = 0; i < n; ++i) + hop1.push_back({Address{addr + numBytes * i}, tasks[offset + i].src}); + dispatchCopy<MidTier, SrcTier>(hop1, numBytes, buf.stream()); + } + + // Second hop: staging → dst + { + std::vector<CopyTask> hop2; + hop2.reserve(n); + for (size_t i = 0; i < n; ++i) + hop2.push_back({tasks[offset + i].dst, Address{addr + numBytes * i}}); + dispatchCopy<DstTier, MidTier>(hop2, numBytes, buf.stream()); + } + + offset += n; + remaining -= n; + } +} + +void CopyEngine::transfer( + CacheTier dstTier, CacheTier srcTier, size_t numBytes, std::vector<CopyTask> const& tasks, CUstream stream) +{ + // Nothing to copy. Also guards the two-hop path against a div-by-zero on + // `buf.size() / numBytes` when numBytes == 0. + if (tasks.empty() || numBytes == 0) + { + return; + } + + constexpr auto G = CacheTier::GPU_MEM; + constexpr auto H = CacheTier::HOST_MEM; + constexpr auto D = CacheTier::DISK; + + if (dstTier == G && srcTier == G) + dispatchCopy<G, G>(tasks, numBytes, stream); + else if (dstTier == G && srcTier == H) + dispatchCopy<G, H>(tasks, numBytes, stream); + else if (dstTier == G && srcTier == D) + twoHopTransfer<G, H, D>(getStagingManager(), numBytes, tasks, stream); + else if (dstTier == H && srcTier == G) + dispatchCopy<H, G>(tasks, numBytes, stream); + else if (dstTier == H && srcTier == H) + dispatchCopy<H, H>(tasks, numBytes, stream); + else if (dstTier == H && srcTier == D) + dispatchCopy<H, D>(tasks, numBytes, stream); + else if (dstTier == D && srcTier == G) + twoHopTransfer<D, H, G>(getStagingManager(), numBytes, tasks, stream); + else if (dstTier == D && srcTier == H) + dispatchCopy<D, H>(tasks, numBytes, stream); + else if (dstTier == D && srcTier == D) + dispatchCopy<D, D>(tasks, numBytes, stream); + else + throw std::invalid_argument("CopyEngine::transfer: unsupported tier combination"); +} + +// --------------------------------------------------------------------------- +// Module-level singleton +// --------------------------------------------------------------------------- + +CopyEngine& globalCopyEngine() +{ + static CopyEngine engine; + return engine; +} + +void batchedCopy(CacheTier dstTier, CacheTier srcTier, size_t numBytes, std::vector<CopyTask> tasks, CUstream stream) +{ + globalCopyEngine().transfer(dstTier, srcTier, numBytes, std::move(tasks), stream); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/copyEngine.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/copyEngine.h new file mode 100644 index 000000000000..5fca2de91abc --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/copyEngine.h @@ -0,0 +1,199 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/utils/cudaEvent.h" +#include "kv_cache_manager_v2/utils/hostMem.h" + +#include "tensorrt_llm/common/assert.h" +#include <cuda.h> +#include <memory> +#include <mutex> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// CopyTask — source and destination address pair for a bulk copy. +// --------------------------------------------------------------------------- +struct CopyTask +{ + Address dst; + Address src; +}; + +// --------------------------------------------------------------------------- +// GrainMetadata — per-grain synchronization state inside StagingBufferManager. +// --------------------------------------------------------------------------- +struct GrainMetadata +{ + std::mutex mutex; + CachedCudaEvent readyEvent; // event protecting this grain + + GrainMetadata() + : readyEvent(CachedCudaEvent::makeNull()) + { + } + + GrainMetadata(GrainMetadata const&) = delete; + GrainMetadata& operator=(GrainMetadata const&) = delete; + GrainMetadata(GrainMetadata&&) = delete; + GrainMetadata& operator=(GrainMetadata&&) = delete; +}; + +class StagingBufferManager; + +// --------------------------------------------------------------------------- +// StagingBuffer — RAII handle to a slice of the StagingBufferManager's buffer. +// +// On construction, acquires grain locks and waits for readyEvents. +// On destruction, records a new event and releases grain locks. +// --------------------------------------------------------------------------- +class StagingBuffer +{ +public: + static constexpr size_t kGranularity = 1u << 20; // 1 MB grains + + StagingBuffer(StagingBufferManager& manager, size_t minSize, size_t maxSize, CUstream stream); + ~StagingBuffer(); + + StagingBuffer(StagingBuffer const&) = delete; + StagingBuffer& operator=(StagingBuffer const&) = delete; + StagingBuffer(StagingBuffer&&) = delete; + StagingBuffer& operator=(StagingBuffer&&) = delete; + + MemAddress address() const noexcept + { + return mAddress; + } + + size_t size() const noexcept + { + return mSize; + } + + CUstream stream() const noexcept + { + return mStream; + } + +private: + StagingBufferManager& mManager; + size_t mStartGrain{0}; + size_t mNumGrains{0}; + size_t mSize{0}; + MemAddress mAddress{0}; + CUstream mStream; +}; + +// --------------------------------------------------------------------------- +// StagingBufferManager — ring-buffer allocator over a CUDA-registered HostMem. +// Used for two-hop GPU↔Disk transfers. +// --------------------------------------------------------------------------- +class StagingBufferManager +{ +public: + static constexpr size_t kGranularity = StagingBuffer::kGranularity; // 1 MB + + explicit StagingBufferManager(size_t size); + + // Not movable: GrainMetadata has mutexes. + StagingBufferManager(StagingBufferManager const&) = delete; + StagingBufferManager& operator=(StagingBufferManager const&) = delete; + StagingBufferManager(StagingBufferManager&&) = delete; + StagingBufferManager& operator=(StagingBufferManager&&) = delete; + + // Acquire a staging slice. Thread-safe. + // minSize: minimum required bytes. maxSize: best-effort upper bound. + // Returns an RAII StagingBuffer that holds grain locks until destroyed. + StagingBuffer acquire(size_t minSize, size_t maxSize, CUstream stream); + + size_t totalSize() const noexcept + { + TLLM_CHECK_DEBUG_WITH_INFO( + mGrains.size() * kGranularity == mBuffer.size(), "grain count * granularity must equal buffer size"); + return mBuffer.size(); + } + + size_t numGrains() const noexcept + { + return mGrains.size(); + } + + MemAddress baseAddress() const noexcept + { + return mBuffer.address(); + } + +private: + friend class StagingBuffer; + + // Caller must hold mMutex. + size_t suggestNextMaxGrains() const noexcept + { + return numGrains() - mNext; + } + + std::mutex mMutex; + HostMem mBuffer; + std::vector<GrainMetadata> mGrains; + size_t mNext{0}; +}; + +// --------------------------------------------------------------------------- +// CopyEngine — dispatches bulk transfers between cache tiers. +// +// Single-hop pairs call the appropriate copy function from kvCacheManagerV2Utils. +// Two-hop pairs (GPU↔Disk) route through a lazily-allocated StagingBuffer. +// --------------------------------------------------------------------------- +class CopyEngine +{ +public: + CopyEngine() = default; + ~CopyEngine() = default; + + CopyEngine(CopyEngine const&) = delete; + CopyEngine& operator=(CopyEngine const&) = delete; + + // Transfer num_bytes per task. tasks must all share the same (dstTier, srcTier). + // stream: the CUDA stream on which GPU ops are enqueued. + void transfer( + CacheTier dstTier, CacheTier srcTier, size_t numBytes, std::vector<CopyTask> const& tasks, CUstream stream); + + void close() noexcept + { + mStagingManager.reset(); + } + +private: + StagingBufferManager& getStagingManager(); + + std::unique_ptr<StagingBufferManager> mStagingManager; +}; + +// --------------------------------------------------------------------------- +// Module-level singleton (mirrors Python's _copy_engine global). +// --------------------------------------------------------------------------- +CopyEngine& globalCopyEngine(); + +// Convenience wrapper used by higher layers. +void batchedCopy(CacheTier dstTier, CacheTier srcTier, size_t numBytes, std::vector<CopyTask> tasks, CUstream stream); + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/cudaVirtMem.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/cudaVirtMem.cpp new file mode 100644 index 000000000000..240f7d08f1d5 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/cudaVirtMem.cpp @@ -0,0 +1,217 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/cudaVirtMem.h" +#include "kv_cache_manager_v2/exceptions.h" +#include "kv_cache_manager_v2/utils/math.h" +#include "tensorrt_llm/common/assert.h" + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// PooledPhysMemAllocator +// --------------------------------------------------------------------------- + +static bool isPropSupported(CUmemAllocationProp const& prop) +{ + CUmemGenericAllocationHandle handle; + CUresult err = cuMemCreate(&handle, 2ULL << 20, &prop, 0); + if (err == CUDA_ERROR_NOT_PERMITTED || err == CUDA_ERROR_NOT_SUPPORTED || err == CUDA_ERROR_INVALID_DEVICE + || err == CUDA_ERROR_INVALID_VALUE) + { + return false; + } + if (err == CUDA_SUCCESS) + { + cuMemRelease(handle); + return true; + } + throw CuError(err); +} + +// --------------------------------------------------------------------------- +// PhysMemChunk +// --------------------------------------------------------------------------- + +PhysMemWrapper::PhysMemWrapper(size_t size, CUmemAllocationProp const& prop) +{ + cuCheck(cuMemCreate(&mHandle, size, &prop, 0)); +} + +PhysMemWrapper::~PhysMemWrapper() +{ + cuMemRelease(mHandle); +} + +// --------------------------------------------------------------------------- +// PooledPhysMemAllocator +// --------------------------------------------------------------------------- + +PooledPhysMemAllocator::PooledPhysMemAllocator(size_t physMemSize) + : mPhysMemSize(physMemSize) + , mPool([this]() -> PhysMemWrapper* { return new PhysMemWrapper(mPhysMemSize, mProp); }, + [](PhysMemWrapper* chunk) { delete chunk; }) +{ + // Get current device. + cuCheck(cuCtxGetDevice(&mDeviceId)); + + // Build the best allocation property. + mProp.type = CU_MEM_ALLOCATION_TYPE_PINNED; + mProp.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + mProp.location.id = mDeviceId; + mProp.allocFlags.gpuDirectRDMACapable = 1; + mProp.requestedHandleTypes = CU_MEM_HANDLE_TYPE_FABRIC; + + if (!isPropSupported(mProp)) + { + mProp.requestedHandleTypes = CU_MEM_HANDLE_TYPE_NONE; + if (!isPropSupported(mProp)) + { + mProp.allocFlags.gpuDirectRDMACapable = 0; + if (!isPropSupported(mProp)) + { + throw std::runtime_error("PooledPhysMemAllocator: no supported physical memory allocation property"); + } + } + } +} + +PooledPhysMemAllocator::~PooledPhysMemAllocator() +{ + mPool.clear(); +} + +PooledPhysMemAllocator::PooledPhysMem PooledPhysMemAllocator::acquire() +{ + return mPool.get(); +} + +// --------------------------------------------------------------------------- +// VirtMem +// --------------------------------------------------------------------------- + +VirtMem::VirtMem(size_t vmSize, PooledPhysMemAllocator& physMemAllocator, size_t initNumPhysMem) + : mVmSize(vmSize) + , mPhysMemAllocator(physMemAllocator) +{ + TLLM_CHECK_DEBUG(vmSize % physMemAllocator.physMemSize() == 0); + + cuCheck(cuMemAddressReserve(&mAddr, vmSize, 0, 0, 0)); + + mAccessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + mAccessDesc.location.id = physMemAllocator.deviceId(); + mAccessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + + extend(initNumPhysMem); +} + +VirtMem::~VirtMem() noexcept +{ + try + { + destroy(); + } + catch (...) + { + // Destructors cannot surface CUDA cleanup failures. Explicit destroy() + // still reports them to match Python VirtMem.destroy(). + } +} + +void VirtMem::push(PooledPhysMemAllocator::PooledPhysMem handle) +{ + size_t physSize = mPhysMemAllocator.physMemSize(); + CUdeviceptr offset = mAddr + physSize * static_cast<size_t>(mPhysHandles.size()); + TLLM_CHECK_DEBUG(physSize * (mPhysHandles.size() + 1) <= mVmSize); + + cuCheck(cuMemMap(offset, physSize, 0, handle->handle(), 0)); + cuCheck(cuMemSetAccess(offset, physSize, &mAccessDesc, 1)); + mPhysHandles.push_back(std::move(handle)); +} + +void VirtMem::pop() +{ + TLLM_CHECK_DEBUG(!mPhysHandles.empty()); + size_t physSize = mPhysMemAllocator.physMemSize(); + CUdeviceptr offset = mAddr + physSize * (mPhysHandles.size() - 1); + cuCheck(cuMemUnmap(offset, physSize)); + mPhysHandles.pop_back(); // PhysMemHandle destructor returns to pool +} + +void VirtMem::extend(size_t numToAdd) +{ + size_t old = numPhysMem(); + try + { + for (size_t i = 0; i < numToAdd; ++i) + { + push(mPhysMemAllocator.acquire()); + } + } + catch (...) + { + // Rollback: remove any newly-mapped chunks. + while (numPhysMem() > old) + { + pop(); + } + throw; + } +} + +void VirtMem::shrink(size_t numToRemove) +{ + cuCheck(cuCtxSynchronize()); + for (size_t i = 0; i < numToRemove; ++i) + { + pop(); + } +} + +void VirtMem::realloc(size_t numBytes) +{ + size_t physSize = mPhysMemAllocator.physMemSize(); + size_t required = divUp(numBytes, physSize); + size_t current = numPhysMem(); + if (required > current) + { + extend(required - current); + } + else if (required < current) + { + shrink(current - required); + } +} + +void VirtMem::destroy() +{ + if (mVmSize == 0) + { + return; + } + cuCheck(cuCtxSynchronize()); + while (!mPhysHandles.empty()) + { + pop(); + } + cuCheck(cuMemAddressFree(mAddr, mVmSize)); + mAddr = 0; + mVmSize = 0; +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/cudaVirtMem.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/cudaVirtMem.h new file mode 100644 index 000000000000..5c560f036594 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/cudaVirtMem.h @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/utils/cudaEvent.h" + +#include <cuda.h> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// PhysMemChunk — RAII wrapper for a CUmemGenericAllocationHandle. +// Constructor calls cuMemCreate, destructor calls cuMemRelease. +// --------------------------------------------------------------------------- +class PhysMemWrapper +{ +public: + PhysMemWrapper(size_t size, CUmemAllocationProp const& prop); + ~PhysMemWrapper(); + + PhysMemWrapper(PhysMemWrapper const&) = delete; + PhysMemWrapper& operator=(PhysMemWrapper const&) = delete; + + [[nodiscard]] CUmemGenericAllocationHandle handle() const noexcept + { + return mHandle; + } + +private: + CUmemGenericAllocationHandle mHandle; +}; + +// --------------------------------------------------------------------------- +// PooledPhysMemAllocator — creates and pools physical GPU memory chunks. +// Mirrors _cuda_virt_mem.py::PooledPhysMemAllocator. +// --------------------------------------------------------------------------- +class PooledPhysMemAllocator +{ +public: + using PooledPhysMem = SimplePool<PhysMemWrapper>::PoolItem; + + // physMemSize: size of each physical chunk in bytes. + explicit PooledPhysMemAllocator(size_t physMemSize); + ~PooledPhysMemAllocator(); + + PooledPhysMemAllocator(PooledPhysMemAllocator const&) = delete; + PooledPhysMemAllocator& operator=(PooledPhysMemAllocator const&) = delete; + + // Borrow a physical memory handle from the pool (or allocate a new one). + // Dropping the returned PhysMemHandle returns it to the pool. + [[nodiscard]] PooledPhysMem acquire(); + + // Release all cached (unused) physical memory back to the driver. + // Mirrors Python PooledPhysMemAllocator.clear(). + void clear() + { + mPool.clear(); + } + + [[nodiscard]] size_t physMemSize() const noexcept + { + return mPhysMemSize; + } + + int deviceId() const noexcept + { + return mDeviceId; + } + +private: + size_t mPhysMemSize{}; + int mDeviceId{}; + CUmemAllocationProp mProp{}; + SimplePool<PhysMemWrapper> mPool; +}; + +// --------------------------------------------------------------------------- +// VirtMem — a virtual address range backed by physical GPU memory chunks. +// Physical chunks are mapped/unmapped at the end of the range (stack discipline). +// Mirrors _cuda_virt_mem.py::VirtMem. +// +// Invariant: mappedBytes() == numPhysMem() * physMemSize() +// mappedBytes() <= virtualBytes() +// --------------------------------------------------------------------------- +class VirtMem +{ +public: + // vmSize: total virtual address space reserved (bytes). + // Must be a multiple of physMemSize. + // physMemAllocator: shared allocator for physical chunks. + // initNumPhysMem: number of physical chunks to map immediately. + VirtMem(size_t vmSize, PooledPhysMemAllocator& physMemAllocator, size_t initNumPhysMem = 0); + ~VirtMem() noexcept; + + VirtMem(VirtMem const&) = delete; + VirtMem& operator=(VirtMem const&) = delete; + + // Map numPhysMem additional chunks at the top of the address range. + void extend(size_t numPhysMem); + + // Unmap numPhysMem chunks from the top (synchronizes CUDA first). + void shrink(size_t numPhysMem); + + // Adjust mapped bytes to exactly numBytes (extend or shrink). + void realloc(size_t numBytes); + + void destroy(); + + [[nodiscard]] MemAddress address() const noexcept + { + return static_cast<MemAddress>(mAddr); + } + + [[nodiscard]] size_t physMemSize() const noexcept + { + return mPhysMemAllocator.physMemSize(); + } + + [[nodiscard]] size_t mappedBytes() const noexcept + { + return mPhysMemAllocator.physMemSize() * numPhysMem(); + } + + [[nodiscard]] size_t virtualBytes() const noexcept + { + return mVmSize; + } + + [[nodiscard]] size_t numPhysMem() const noexcept + { + return mPhysHandles.size(); + } + +private: + void push(PooledPhysMemAllocator::PooledPhysMem handle); + void pop(); + + CUdeviceptr mAddr = 0; + size_t mVmSize = 0; + PooledPhysMemAllocator& mPhysMemAllocator; + std::vector<PooledPhysMemAllocator::PooledPhysMem> mPhysHandles; + CUmemAccessDesc mAccessDesc{}; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/eventManager.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/eventManager.cpp new file mode 100644 index 000000000000..73fec6f57cd1 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/eventManager.cpp @@ -0,0 +1,683 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/eventManager.h" + +#include "kv_cache_manager_v2/blockRadixTree.h" +#include "kv_cache_manager_v2/page.h" +#include "tensorrt_llm/common/logger.h" + +#include <algorithm> +#include <chrono> +#include <iterator> +#include <stdexcept> +#include <utility> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ +namespace +{ + +constexpr uint32_t kUint32HashConst = 0x045D9F3BU; +constexpr uint64_t kUint64HashConst1 = 0xBF58476D1CE4E5B9ULL; +constexpr uint64_t kUint64HashConst2 = 0x94D049BB133111EBULL; +constexpr uint32_t kHashCombineConst = 0x9E3779B9U; +constexpr uint64_t kParentHashConst = 0xBF58476D1CE4E5B9ULL; + +uint64_t hash32Mix(int64_t input, uint64_t seed) +{ + uint32_t value = static_cast<uint32_t>(input); + value = ((value >> 16U) ^ value) * kUint32HashConst; + value = ((value >> 16U) ^ value) * kUint32HashConst; + value = (value >> 16U) ^ value; + value += kHashCombineConst; + return seed ^ (static_cast<uint64_t>(value) + (seed << 6U) + (seed >> 2U)); +} + +uint64_t hash64Mix(int64_t input, uint64_t seed) +{ + uint64_t value = static_cast<uint64_t>(input); + value = (value ^ (value >> 30U)) * kUint64HashConst1; + value = (value ^ (value >> 27U)) * kUint64HashConst2; + value ^= value >> 31U; + return seed ^ (value + static_cast<uint64_t>(kHashCombineConst) + (seed << 6U) + (seed >> 2U)); +} + +} // namespace + +EventManager::EventManager(int maxKvEventEntries, int windowSize, std::optional<int> attentionDpRank, + AttentionDpGatherFn attentionDpGather, std::string hashAlgo, std::map<int, int> windowSizeByLayerGroup) + : mMaxKvEventEntries(maxKvEventEntries) + , mWindowSize(windowSize) + , mWindowSizeByLayerGroup(std::move(windowSizeByLayerGroup)) + , mAttentionDpRank(attentionDpRank) + , mAttentionDpGather(std::move(attentionDpGather)) +{ + std::tie(mHashAlgo, mHashAlgoName) = parseHashAlgorithm(hashAlgo); +} + +std::pair<EventManager::HashAlgorithm, std::string> EventManager::parseHashAlgorithm(std::string const& hashAlgo) +{ + if (hashAlgo == "auto" || hashAlgo == "v1_block_key") + { + return {HashAlgorithm::kV1, "v1_block_key"}; + } + if (hashAlgo == "v2_sha256") + { + return {HashAlgorithm::kV2Sha256, "v2_sha256"}; + } + if (hashAlgo == "v2_sha256_64") + { + return {HashAlgorithm::kV2Sha256_64, "v2_sha256_64"}; + } + throw std::invalid_argument("Unsupported V2 KV cache event hash algorithm: " + hashAlgo); +} + +void EventManager::addCreatedEvent( + std::vector<int> numBlocksPerCacheLevel, std::optional<std::vector<int>> layerGroupIds) +{ + if (mMaxKvEventEntries <= 0) + { + return; + } + std::lock_guard<std::mutex> lock(mMutex); + KVCacheCreatedData data{std::move(numBlocksPerCacheLevel)}; + if (!layerGroupIds.has_value()) + { + addEventUnlocked(std::move(data), std::nullopt); + return; + } + for (int layerGroupId : *layerGroupIds) + { + addEventUnlocked(data, layerGroupId); + } +} + +void EventManager::setLayerGroupWindowSizes(std::map<int, int> windowSizes) +{ + std::lock_guard<std::mutex> lock(mMutex); + mWindowSizeByLayerGroup = std::move(windowSizes); +} + +void EventManager::addStoredEvent(KVCacheStoredData data, EventLayerGroupId layerGroupId) +{ + if (data.blocks.empty() || mMaxKvEventEntries <= 0) + { + return; + } + std::lock_guard<std::mutex> lock(mMutex); + flushRemovedEventUnlocked(layerGroupId); + addStoredEventUnlocked(std::move(data), layerGroupId); +} + +void EventManager::addRemovedEvent(std::vector<EventBlockHash> blockHashes, EventLayerGroupId layerGroupId) +{ + if (blockHashes.empty() || mMaxKvEventEntries <= 0) + { + return; + } + std::lock_guard<std::mutex> lock(mMutex); + enqueueRemovedEventUnlocked(std::move(blockHashes), layerGroupId); +} + +void EventManager::addUpdatedEvent(EventBlockHash blockHash, std::optional<KVCacheEventDiff> cacheLevel, + std::optional<KVCacheEventDiff> priority, EventLayerGroupId layerGroupId) +{ + if ((!cacheLevel.has_value() && !priority.has_value()) || mMaxKvEventEntries <= 0) + { + return; + } + std::lock_guard<std::mutex> lock(mMutex); + addEventUnlocked(KVCacheUpdatedData{std::move(blockHash), cacheLevel, priority}, layerGroupId); +} + +void EventManager::addUpdatedEvent(Digest const& blockKey, std::optional<KVCacheEventDiff> cacheLevel, + std::optional<KVCacheEventDiff> priority, EventLayerGroupId layerGroupId) +{ + if ((!cacheLevel.has_value() && !priority.has_value()) || mMaxKvEventEntries <= 0) + { + return; + } + std::lock_guard<std::mutex> lock(mMutex); + auto const state = mStoredBlocks.find(blockKey); + if (state == mStoredBlocks.end()) + { + return; + } + addEventUnlocked(KVCacheUpdatedData{state->second.blockHash, cacheLevel, priority}, layerGroupId); +} + +void EventManager::addStoredBlock(Block const& block) +{ + if (mMaxKvEventEntries <= 0) + { + return; + } + std::lock_guard<std::mutex> lock(mMutex); + addStoredBlockUnlocked(block); +} + +void EventManager::addStoredBlockUnlocked(Block const& block) +{ + std::set<int> lifeCycleIds; + for (LifeCycleId lifeCycle{0}; lifeCycle < block.storage.size(); ++lifeCycle) + { + if (block.storage[lifeCycle] != nullptr) + { + lifeCycleIds.insert(lifeCycle.value()); + } + } + if (lifeCycleIds.empty()) + { + return; + } + + EventBlockHash blockHash = hashFromBlock(block); + mStoredBlocks.insert_or_assign(block.key, StoredBlockState{blockHash, lifeCycleIds}); + auto parentHash = parentHashFromBlock(block); + for (int lifeCycleId : lifeCycleIds) + { + auto blockData = storedBlockFromBlock(block, std::set<int>{lifeCycleId}); + if (blockData.has_value()) + { + flushRemovedEventUnlocked(lifeCycleId); + addStoredEventUnlocked(KVCacheStoredData{parentHash, {std::move(*blockData)}}, lifeCycleId); + } + } +} + +void EventManager::addStoredLifeCycle(Block const& block, LifeCycleId lifeCycle) +{ + if (mMaxKvEventEntries <= 0) + { + return; + } + std::lock_guard<std::mutex> lock(mMutex); + auto state = mStoredBlocks.find(block.key); + if (state == mStoredBlocks.end()) + { + addStoredBlockUnlocked(block); + return; + } + int const lifeCycleId = lifeCycle.value(); + if (state->second.lifeCycleIds.count(lifeCycleId) != 0) + { + return; + } + auto blockData = storedBlockFromBlock(block, std::set<int>{lifeCycleId}); + if (!blockData.has_value()) + { + return; + } + state->second.lifeCycleIds.insert(lifeCycleId); + flushRemovedEventUnlocked(lifeCycleId); + addStoredEventUnlocked(KVCacheStoredData{parentHashFromBlock(block), {std::move(*blockData)}}, lifeCycleId); +} + +void EventManager::addRemovedBlock(Digest const& blockKey) +{ + if (mMaxKvEventEntries <= 0) + { + return; + } + std::lock_guard<std::mutex> lock(mMutex); + auto state = mStoredBlocks.find(blockKey); + if (state == mStoredBlocks.end()) + { + return; + } + EventBlockHash blockHash = state->second.blockHash; + auto lifeCycleIds = state->second.lifeCycleIds; + mStoredBlocks.erase(state); + dropHashCache(blockKey); + + if (lifeCycleIds.empty()) + { + enqueueRemovedEventUnlocked({std::move(blockHash)}, std::nullopt); + return; + } + for (int lifeCycleId : lifeCycleIds) + { + enqueueRemovedEventUnlocked({blockHash}, lifeCycleId); + } +} + +void EventManager::addRemovedLifeCycle(Digest const& blockKey, LifeCycleId lifeCycle) +{ + if (mMaxKvEventEntries <= 0) + { + return; + } + std::lock_guard<std::mutex> lock(mMutex); + auto state = mStoredBlocks.find(blockKey); + int const lifeCycleId = lifeCycle.value(); + if (state == mStoredBlocks.end() || state->second.lifeCycleIds.erase(lifeCycleId) == 0) + { + return; + } + EventBlockHash blockHash = state->second.blockHash; + if (state->second.lifeCycleIds.empty()) + { + mStoredBlocks.erase(state); + dropHashCache(blockKey); + } + enqueueRemovedEventUnlocked({std::move(blockHash)}, lifeCycleId); +} + +void EventManager::addCacheLevelUpdated( + Digest const& blockKey, CacheLevel oldLevel, CacheLevel newLevel, LifeCycleId lifeCycle) +{ + if (mMaxKvEventEntries <= 0) + { + return; + } + std::lock_guard<std::mutex> lock(mMutex); + auto state = mStoredBlocks.find(blockKey); + if (state == mStoredBlocks.end()) + { + return; + } + addEventUnlocked( + KVCacheUpdatedData{state->second.blockHash, KVCacheEventDiff{oldLevel.value(), newLevel.value()}, std::nullopt}, + lifeCycle.value()); +} + +void EventManager::addStoredEventUnlocked(KVCacheStoredData data, EventLayerGroupId layerGroupId) +{ + bool const hasPendingRemovedEvents = !mLatestRemovedBlockHashes.empty(); + auto latest = mLatestStoredEventIds.find(layerGroupId); + if (!hasPendingRemovedEvents && latest != mLatestStoredEventIds.end()) + { + auto pending = std::find_if(mPendingEvents.rbegin(), mPendingEvents.rend(), + [&](KVCacheEvent const& event) { return event.eventId == latest->second; }); + if (pending == mPendingEvents.rend()) + { + throw std::logic_error("Stored event coalescing lost the pending event"); + } + if (auto* stored = std::get_if<KVCacheStoredData>(&pending->data); stored != nullptr && !stored->blocks.empty() + && data.parentHash.has_value() && stored->blocks.back().blockHash == *data.parentHash) + { + std::move(data.blocks.begin(), data.blocks.end(), std::back_inserter(stored->blocks)); + return; + } + } + + auto& event = addEventUnlocked(std::move(data), layerGroupId); + mLatestStoredEventIds.insert_or_assign(layerGroupId, event.eventId); +} + +void EventManager::enqueueRemovedEventUnlocked(std::vector<EventBlockHash> blockHashes, EventLayerGroupId layerGroupId) +{ + if (blockHashes.empty()) + { + return; + } + auto& pending = mLatestRemovedBlockHashes[layerGroupId]; + std::move(blockHashes.begin(), blockHashes.end(), std::back_inserter(pending)); + mLatestStoredEventIds.erase(layerGroupId); +} + +void EventManager::flushRemovedEventUnlocked(EventLayerGroupId layerGroupId) +{ + auto removed = mLatestRemovedBlockHashes.find(layerGroupId); + if (removed == mLatestRemovedBlockHashes.end() || removed->second.empty()) + { + return; + } + auto blockHashes = std::move(removed->second); + mLatestRemovedBlockHashes.erase(removed); + addEventUnlocked(KVCacheRemovedData{std::move(blockHashes)}, layerGroupId); +} + +void EventManager::flushAllRemovedEventsUnlocked() +{ + while (!mLatestRemovedBlockHashes.empty()) + { + flushRemovedEventUnlocked(mLatestRemovedBlockHashes.begin()->first); + } +} + +KVCacheEvent& EventManager::addEventUnlocked(KVCacheEventData data, EventLayerGroupId layerGroupId) +{ + if (mMaxKvEventEntries <= 0) + { + throw std::logic_error("Cannot add an event when the event queue is disabled"); + } + if (!std::holds_alternative<KVCacheRemovedData>(data)) + { + flushAllRemovedEventsUnlocked(); + } + mPendingEvents.push_back(KVCacheEvent{ + mNextEventId++, std::move(data), getWindowSize(layerGroupId), mHashAlgoName, mAttentionDpRank, layerGroupId}); + if (!std::holds_alternative<KVCacheStoredData>(mPendingEvents.back().data)) + { + mLatestStoredEventIds.erase(layerGroupId); + } + return mPendingEvents.back(); +} + +std::vector<KVCacheEvent> EventManager::drainPendingEventsUnlocked() +{ + flushAllRemovedEventsUnlocked(); + auto events = std::move(mPendingEvents); + mPendingEvents.clear(); + mLatestStoredEventIds.clear(); + return events; +} + +void EventManager::publishEventsUnlocked(std::vector<KVCacheEvent> events, std::optional<int> maxKvEventEntries) +{ + if (events.empty()) + { + return; + } + int const capacity = maxKvEventEntries.value_or(mMaxKvEventEntries); + std::move(events.begin(), events.end(), std::back_inserter(mEvents)); + while (static_cast<int>(mEvents.size()) > capacity) + { + mEvents.pop_front(); + } +} + +std::vector<KVCacheEvent> EventManager::trimEvents(std::vector<KVCacheEvent> events, int maxKvEventEntries) +{ + if (maxKvEventEntries <= 0) + { + return {}; + } + if (static_cast<int>(events.size()) > maxKvEventEntries) + { + events.erase(events.begin(), events.end() - maxKvEventEntries); + } + return events; +} + +void EventManager::flushIterationEvents() +{ + if (mAttentionDpGather) + { + std::vector<KVCacheEvent> localEvents; + { + std::lock_guard<std::mutex> lock(mMutex); + localEvents = trimEvents(drainPendingEventsUnlocked(), mMaxKvEventEntries); + } + auto gatheredEvents = mAttentionDpGather(localEvents); + if (mAttentionDpRank != std::optional<int>{0}) + { + return; + } + + std::vector<KVCacheEvent> events; + for (auto& rankEvents : gatheredEvents) + { + auto trimmed = trimEvents(std::move(rankEvents), mMaxKvEventEntries); + std::move(trimmed.begin(), trimmed.end(), std::back_inserter(events)); + } + { + std::lock_guard<std::mutex> lock(mMutex); + publishEventsUnlocked(std::move(events), mMaxKvEventEntries * std::max<int>(1, gatheredEvents.size())); + } + mCondition.notify_all(); + return; + } + + { + std::lock_guard<std::mutex> lock(mMutex); + publishEventsUnlocked(drainPendingEventsUnlocked()); + } + mCondition.notify_all(); +} + +std::vector<KVCacheEvent> EventManager::getLatestEvents(std::optional<double> timeoutMs) +{ + std::unique_lock<std::mutex> lock(mMutex); + if (mEvents.empty() && !timeoutMs.has_value()) + { + mCondition.wait(lock, [&] { return !mEvents.empty(); }); + } + else if (mEvents.empty() && *timeoutMs > 0) + { + mCondition.wait_for( + lock, std::chrono::duration<double, std::milli>(*timeoutMs), [&] { return !mEvents.empty(); }); + } + std::vector<KVCacheEvent> events; + events.reserve(mEvents.size()); + std::move(mEvents.begin(), mEvents.end(), std::back_inserter(events)); + mEvents.clear(); + return events; +} + +int EventManager::getWindowSize(EventLayerGroupId layerGroupId) const +{ + if (!layerGroupId.has_value()) + { + return mWindowSize; + } + auto const windowSize = mWindowSizeByLayerGroup.find(*layerGroupId); + return windowSize == mWindowSizeByLayerGroup.end() ? mWindowSize : windowSize->second; +} + +std::string EventManager::digestToHex(Digest const& digest) +{ + constexpr char kHex[] = "0123456789abcdef"; + std::string result; + result.resize(digest.size() * 2); + for (size_t i = 0; i < digest.size(); ++i) + { + auto const value = std::to_integer<uint8_t>(digest[i]); + result[2 * i] = kHex[value >> 4U]; + result[2 * i + 1] = kHex[value & 0x0FU]; + } + return result; +} + +uint64_t EventManager::truncateDigestToInt64(Digest const& digest) +{ + uint64_t result = 0; + for (int i = 0; i < 8; ++i) + { + result = (result << 8U) | std::to_integer<uint8_t>(digest[static_cast<size_t>(i)]); + } + return result; +} + +EventBlockHash EventManager::normalizeDigest(Digest const& digest) const +{ + if (mHashAlgo == HashAlgorithm::kV2Sha256_64) + { + return truncateDigestToInt64(digest); + } + return digestToHex(digest); +} + +EventBlockHash EventManager::hashFromBlock(Block const& block) +{ + if (mHashAlgo == HashAlgorithm::kV1) + { + return v1HashFromBlock(block); + } + return normalizeDigest(block.key); +} + +std::optional<EventBlockHash> EventManager::parentHashFromBlock(Block const& block) +{ + if (block.prev == nullptr) + { + throw std::logic_error("Cannot hash an orphan KV cache block"); + } + if (block.prev->type() == NodeBase::Type::kROOT_BLOCK) + { + return std::nullopt; + } + return hashFromBlock(*static_cast<Block const*>(block.prev)); +} + +std::optional<KVCacheStoredBlockData> EventManager::storedBlockFromBlock( + Block const& block, std::optional<std::set<int>> const& lifeCycleIds) +{ + CacheLevel cacheLevel = kGpuLevel; + Priority priority = kPriorityDefault; + bool foundPage = false; + for (LifeCycleId lifeCycle{0}; lifeCycle < block.storage.size(); ++lifeCycle) + { + if (lifeCycleIds.has_value() && lifeCycleIds->count(lifeCycle.value()) == 0) + { + continue; + } + auto const* page = block.storage[lifeCycle]; + if (page != nullptr) + { + cacheLevel = page->cacheLevel; + priority = page->priority; + foundPage = true; + break; + } + } + if (lifeCycleIds.has_value() && !foundPage) + { + return std::nullopt; + } + + std::vector<UniqueToken> tokens; + tokens.reserve(block.tokens.size()); + for (auto const& token : block.tokens) + { + if (auto const* tokenId = std::get_if<TokenId>(&token)) + { + UniqueToken uniqueToken; + uniqueToken.tokenId = EventTokenId{std::in_place_index<0>, *tokenId}; + tokens.push_back(std::move(uniqueToken)); + } + else + { + UniqueToken uniqueToken; + uniqueToken.tokenId + = EventTokenId{std::in_place_index<1>, digestToHex(std::get<DigestToken>(token).digest())}; + tokens.push_back(std::move(uniqueToken)); + } + } + return KVCacheStoredBlockData{ + hashFromBlock(block), std::move(tokens), cacheLevel.value(), priority, {}, std::nullopt}; +} + +uint64_t EventManager::hashV1BlockKey(std::vector<TokenId> const& tokens, uint64_t parentHash, + std::optional<LoraTaskIdType> loraTaskId, std::optional<std::uint64_t> cacheSaltId) +{ + uint64_t seed = static_cast<uint64_t>(tokens.size()) ^ (parentHash * kParentHashConst); + if (parentHash == 0 && cacheSaltId.has_value()) + { + seed = hash64Mix(*cacheSaltId, seed); + } + for (TokenId token : tokens) + { + seed = hash32Mix(token, seed); + } + if (loraTaskId.has_value()) + { + seed = hash64Mix(*loraTaskId, seed); + } + return seed; +} + +uint64_t EventManager::v1HashFromBlock(Block const& block) +{ + if (auto const cached = mV1HashByBlockKey.find(block.key); cached != mV1HashByBlockKey.end()) + { + return cached->second; + } + + std::vector<Block const*> chain; + NodeBase const* current = █ + uint64_t parentHash = 0; + bool parentIsV1Compatible = true; + V1RootAttrs rootAttrs; + while (current->type() == NodeBase::Type::kBLOCK) + { + auto const* currentBlock = static_cast<Block const*>(current); + if (auto const cached = mV1HashByBlockKey.find(currentBlock->key); cached != mV1HashByBlockKey.end()) + { + parentHash = cached->second; + parentIsV1Compatible = mV1HashCompatibleKeys.count(currentBlock->key) != 0; + rootAttrs = mV1RootAttrsByBlockKey.at(currentBlock->key); + break; + } + chain.push_back(currentBlock); + current = currentBlock->prev; + if (current == nullptr) + { + throw std::logic_error("Cannot hash an orphan KV cache block"); + } + } + if (current->type() == NodeBase::Type::kROOT_BLOCK) + { + auto const& reuseScope = static_cast<RootBlock const*>(current)->reuseScope; + rootAttrs = {reuseScope.loraId, reuseScope.salt}; + } + + for (auto chainIter = chain.rbegin(); chainIter != chain.rend(); ++chainIter) + { + Block const& currentBlock = **chainIter; + std::vector<TokenId> textTokens; + textTokens.reserve(currentBlock.tokens.size()); + if (parentIsV1Compatible) + { + for (auto const& token : currentBlock.tokens) + { + auto const* tokenId = std::get_if<TokenId>(&token); + if (tokenId == nullptr) + { + parentIsV1Compatible = false; + break; + } + textTokens.push_back(*tokenId); + } + } + if (parentIsV1Compatible) + { + parentHash = hashV1BlockKey(textTokens, parentHash, rootAttrs.first, rootAttrs.second); + mV1HashCompatibleKeys.insert(currentBlock.key); + } + else + { + parentHash = fallbackV1Hash(currentBlock.key); + } + mV1HashByBlockKey.insert_or_assign(currentBlock.key, parentHash); + mV1RootAttrsByBlockKey.insert_or_assign(currentBlock.key, rootAttrs); + } + return parentHash; +} + +uint64_t EventManager::fallbackV1Hash(Digest const& blockKey) +{ + if (!mWarnedV1HashFallback) + { + TLLM_LOG_WARNING( + "V2 KV cache event hash algorithm v1_block_key only matches v1 for text-token radix blocks. " + "Falling back to truncated SHA-256 block hash for unsupported blocks."); + mWarnedV1HashFallback = true; + } + return truncateDigestToInt64(blockKey); +} + +void EventManager::dropHashCache(Digest const& blockKey) +{ + mV1HashByBlockKey.erase(blockKey); + mV1HashCompatibleKeys.erase(blockKey); + mV1RootAttrsByBlockKey.erase(blockKey); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/eventManager.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/eventManager.h new file mode 100644 index 000000000000..871a8bf4199a --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/eventManager.h @@ -0,0 +1,262 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/eventSink.h" + +#include <condition_variable> +#include <cstdint> +#include <deque> +#include <functional> +#include <map> +#include <mutex> +#include <optional> +#include <set> +#include <string> +#include <tuple> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <variant> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +using EventBlockHash = std::variant<uint64_t, std::string>; +using EventTokenId = std::variant<int64_t, std::string>; +using EventLayerGroupId = std::optional<int>; + +struct UniqueToken +{ + EventTokenId tokenId; + int64_t tokenExtraId = 0; + + bool operator==(UniqueToken const& other) const + { + return tokenId == other.tokenId && tokenExtraId == other.tokenExtraId; + } +}; + +struct KVCacheCreatedData +{ + std::vector<int> numBlocksPerCacheLevel; + + bool operator==(KVCacheCreatedData const& other) const + { + return numBlocksPerCacheLevel == other.numBlocksPerCacheLevel; + } +}; + +struct MmKey +{ + std::string hash; + int startOffset = 0; + std::optional<std::string> uuid; + bool hasUuidField = false; + + bool operator==(MmKey const& other) const + { + return hash == other.hash && startOffset == other.startOffset && uuid == other.uuid + && hasUuidField == other.hasUuidField; + } +}; + +struct KVCacheStoredBlockData +{ + EventBlockHash blockHash; + std::vector<UniqueToken> tokens; + int cacheLevel = kGpuLevel.value(); + int priority = kPriorityDefault; + std::vector<MmKey> mmKeys; + std::optional<std::string> cacheSalt; + + bool operator==(KVCacheStoredBlockData const& other) const + { + return blockHash == other.blockHash && tokens == other.tokens && cacheLevel == other.cacheLevel + && priority == other.priority && mmKeys == other.mmKeys && cacheSalt == other.cacheSalt; + } +}; + +struct KVCacheStoredData +{ + std::optional<EventBlockHash> parentHash; + std::vector<KVCacheStoredBlockData> blocks; + + bool operator==(KVCacheStoredData const& other) const + { + return parentHash == other.parentHash && blocks == other.blocks; + } +}; + +struct KVCacheRemovedData +{ + std::vector<EventBlockHash> blockHashes; + + bool operator==(KVCacheRemovedData const& other) const + { + return blockHashes == other.blockHashes; + } +}; + +struct KVCacheEventDiff +{ + int oldValue = 0; + int newValue = 0; + + bool operator==(KVCacheEventDiff const& other) const + { + return oldValue == other.oldValue && newValue == other.newValue; + } +}; + +struct KVCacheUpdatedData +{ + EventBlockHash blockHash; + std::optional<KVCacheEventDiff> cacheLevel; + std::optional<KVCacheEventDiff> priority; + + bool operator==(KVCacheUpdatedData const& other) const + { + return blockHash == other.blockHash && cacheLevel == other.cacheLevel && priority == other.priority; + } +}; + +using KVCacheEventData = std::variant<KVCacheCreatedData, KVCacheStoredData, KVCacheRemovedData, KVCacheUpdatedData>; + +struct KVCacheEvent +{ + int64_t eventId = 0; + KVCacheEventData data; + int windowSize = 0; + std::optional<std::string> hashAlgo; + std::optional<int> attentionDpRank; + EventLayerGroupId layerGroupId; + + bool operator==(KVCacheEvent const& other) const + { + return eventId == other.eventId && data == other.data && windowSize == other.windowSize + && hashAlgo == other.hashAlgo && attentionDpRank == other.attentionDpRank + && layerGroupId == other.layerGroupId; + } +}; + +class EventManager final : public EventSink +{ +public: + using AttentionDpGatherFn = std::function<std::vector<std::vector<KVCacheEvent>>(std::vector<KVCacheEvent> const&)>; + + EventManager(int maxKvEventEntries, int windowSize = 0, std::optional<int> attentionDpRank = std::nullopt, + AttentionDpGatherFn attentionDpGather = {}, std::string hashAlgo = "v2_sha256", + std::map<int, int> windowSizeByLayerGroup = {}); + + void addCreatedEvent( + std::vector<int> numBlocksPerCacheLevel, std::optional<std::vector<int>> layerGroupIds = std::nullopt); + void setLayerGroupWindowSizes(std::map<int, int> windowSizes); + void addStoredEvent(KVCacheStoredData data, EventLayerGroupId layerGroupId = std::nullopt); + void addRemovedEvent(std::vector<EventBlockHash> blockHashes, EventLayerGroupId layerGroupId = std::nullopt); + void addUpdatedEvent(EventBlockHash blockHash, std::optional<KVCacheEventDiff> cacheLevel = std::nullopt, + std::optional<KVCacheEventDiff> priority = std::nullopt, EventLayerGroupId layerGroupId = std::nullopt); + void addUpdatedEvent(Digest const& blockKey, std::optional<KVCacheEventDiff> cacheLevel = std::nullopt, + std::optional<KVCacheEventDiff> priority = std::nullopt, EventLayerGroupId layerGroupId = std::nullopt); + + void flushIterationEvents(); + std::vector<KVCacheEvent> getLatestEvents(std::optional<double> timeoutMs = std::nullopt); + + std::string const& hashAlgorithm() const noexcept + { + return mHashAlgoName; + } + + static uint64_t hashV1BlockKey(std::vector<TokenId> const& tokens, uint64_t parentHash = 0, + std::optional<LoraTaskIdType> loraTaskId = std::nullopt, + std::optional<std::uint64_t> cacheSaltId = std::nullopt); + + void addStoredBlock(Block const& block) override; + void addStoredLifeCycle(Block const& block, LifeCycleId lifeCycle) override; + void addRemovedBlock(Digest const& blockKey) override; + void addRemovedLifeCycle(Digest const& blockKey, LifeCycleId lifeCycle) override; + void addCacheLevelUpdated( + Digest const& blockKey, CacheLevel oldLevel, CacheLevel newLevel, LifeCycleId lifeCycle) override; + +private: + enum class HashAlgorithm + { + kV1, + kV2Sha256, + kV2Sha256_64, + }; + + struct StoredBlockState + { + EventBlockHash blockHash; + std::set<int> lifeCycleIds; + }; + + using V1RootAttrs = std::pair<std::optional<LoraTaskIdType>, std::optional<std::uint64_t>>; + + static std::pair<HashAlgorithm, std::string> parseHashAlgorithm(std::string const& hashAlgo); + static std::string digestToHex(Digest const& digest); + static uint64_t truncateDigestToInt64(Digest const& digest); + static std::vector<KVCacheEvent> trimEvents(std::vector<KVCacheEvent> events, int maxKvEventEntries); + + EventBlockHash normalizeDigest(Digest const& digest) const; + EventBlockHash hashFromBlock(Block const& block); + uint64_t v1HashFromBlock(Block const& block); + uint64_t fallbackV1Hash(Digest const& blockKey); + std::optional<EventBlockHash> parentHashFromBlock(Block const& block); + std::optional<KVCacheStoredBlockData> storedBlockFromBlock( + Block const& block, std::optional<std::set<int>> const& lifeCycleIds = std::nullopt); + + void addStoredBlockUnlocked(Block const& block); + void addStoredEventUnlocked(KVCacheStoredData data, EventLayerGroupId layerGroupId); + void enqueueRemovedEventUnlocked(std::vector<EventBlockHash> blockHashes, EventLayerGroupId layerGroupId); + void flushRemovedEventUnlocked(EventLayerGroupId layerGroupId); + void flushAllRemovedEventsUnlocked(); + KVCacheEvent& addEventUnlocked(KVCacheEventData data, EventLayerGroupId layerGroupId); + std::vector<KVCacheEvent> drainPendingEventsUnlocked(); + void publishEventsUnlocked(std::vector<KVCacheEvent> events, std::optional<int> maxKvEventEntries = std::nullopt); + int getWindowSize(EventLayerGroupId layerGroupId) const; + void dropHashCache(Digest const& blockKey); + + int mMaxKvEventEntries; + int mWindowSize; + std::map<int, int> mWindowSizeByLayerGroup; + std::optional<int> mAttentionDpRank; + AttentionDpGatherFn mAttentionDpGather; + HashAlgorithm mHashAlgo; + std::string mHashAlgoName; + int64_t mNextEventId = 0; + + std::unordered_map<Digest, StoredBlockState> mStoredBlocks; + std::map<EventLayerGroupId, int64_t> mLatestStoredEventIds; + std::map<EventLayerGroupId, std::vector<EventBlockHash>> mLatestRemovedBlockHashes; + std::vector<KVCacheEvent> mPendingEvents; + std::deque<KVCacheEvent> mEvents; + + std::unordered_map<Digest, uint64_t> mV1HashByBlockKey; + std::unordered_set<Digest> mV1HashCompatibleKeys; + std::unordered_map<Digest, V1RootAttrs> mV1RootAttrsByBlockKey; + bool mWarnedV1HashFallback = false; + + mutable std::mutex mMutex; + std::condition_variable mCondition; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/eventSink.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/eventSink.h new file mode 100644 index 000000000000..0be2a6d10fa1 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/eventSink.h @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/lifeCycleRegistry.h" + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +struct Block; + +// Boundary between the cache implementation and the native event manager. +class EventSink +{ +public: + virtual ~EventSink() = default; + + virtual void addStoredBlock(Block const& block) = 0; + virtual void addStoredLifeCycle(Block const& block, LifeCycleId lifeCycle) = 0; + virtual void addRemovedBlock(Digest const& blockKey) = 0; + virtual void addRemovedLifeCycle(Digest const& blockKey, LifeCycleId lifeCycle) = 0; + virtual void addCacheLevelUpdated( + Digest const& blockKey, CacheLevel oldLevel, CacheLevel newLevel, LifeCycleId lifeCycle) + = 0; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/evictionController.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/evictionController.cpp new file mode 100644 index 000000000000..f29a9bc046ff --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/evictionController.cpp @@ -0,0 +1,242 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/evictionController.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/exceptions.h" +#include "kv_cache_manager_v2/page.h" + +#include "tensorrt_llm/common/assert.h" +#include <cstddef> +#include <numeric> +#include <set> +#include <stdexcept> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// LRUEvictionPolicy +// --------------------------------------------------------------------------- + +NodeRef LRUEvictionPolicy::push(SharedPtr<Page> page, bool evictFirst) +{ + TLLM_CHECK_DEBUG_WITH_INFO(!page->nodeRef.has_value(), "page must not already be scheduled for eviction"); + if (evictFirst) + { + mQueue.push_front(std::move(page)); + return mQueue.begin(); + } + else + { + mQueue.push_back(std::move(page)); + auto it = mQueue.end(); + --it; + return it; + } +} + +SharedPtr<Page> LRUEvictionPolicy::pop() +{ + TLLM_CHECK_DEBUG(!mQueue.empty()); + auto page = mQueue.front(); + mQueue.pop_front(); + return page; +} + +SharedPtr<Page> LRUEvictionPolicy::remove(NodeRef node) +{ + auto page = *node; + TLLM_CHECK_DEBUG_WITH_INFO( + page->nodeRef.has_value() && page->nodeRef.value() == node, "node's page must reference this node"); + mQueue.erase(node); + return page; +} + +// --------------------------------------------------------------------------- +// PrioritizedEvictionPolicy +// --------------------------------------------------------------------------- + +LRUEvictionPolicy& PrioritizedEvictionPolicy::getOrCreate(Priority p) +{ + return mPolicies[p]; // std::map default-constructs if not present +} + +NodeRef PrioritizedEvictionPolicy::push(SharedPtr<Page> page, bool evictFirst) +{ + Priority p = page->priority; + LRUEvictionPolicy& policy = getOrCreate(p); + return policy.push(std::move(page), evictFirst); +} + +SharedPtr<Page> PrioritizedEvictionPolicy::pop() +{ + TLLM_CHECK_DEBUG(!mPolicies.empty()); + // Lowest priority key evicted first (std::map iterates in ascending key order) + auto it = mPolicies.begin(); + auto page = it->second.pop(); + if (it->second.empty()) + { + mPolicies.erase(it); + } + return page; +} + +SharedPtr<Page> PrioritizedEvictionPolicy::remove(NodeRef node) +{ + auto page = *node; + Priority p = page->priority; + auto it = mPolicies.find(p); + TLLM_CHECK_DEBUG(it != mPolicies.end()); + it->second.remove(node); + if (it->second.empty()) + { + mPolicies.erase(it); + } + return page; +} + +SlotCount PrioritizedEvictionPolicy::size() const noexcept +{ + SlotCount total = 0; + for (auto const& [p, policy] : mPolicies) + { + total += slotCountValueFromSize(policy.size()); + } + return total; +} + +// allPages() removed — use begin()/end() iterator instead. + +// --------------------------------------------------------------------------- +// PerLevelEvictionController +// --------------------------------------------------------------------------- + +PerLevelEvictionController::PerLevelEvictionController( + TypedVec<LifeCycleId, PoolGroupIndex> const& lifeCycleGrouping, CacheLevel cacheLevel) + : mCacheLevel(cacheLevel) + , mLifeCycleGrouping(lifeCycleGrouping) +{ + // Compute number of pool groups = max(grouping) + 1 + PoolGroupIndex numPoolGroups{0}; + for (auto g : mLifeCycleGrouping) + { + if (g + 1 > numPoolGroups) + numPoolGroups = g + 1; + } + // Pool group indices must be contiguous 0..N-1. + TLLM_CHECK_DEBUG(numPoolGroups + == PoolGroupIndex{ + static_cast<int>(std::set<PoolGroupIndex>(mLifeCycleGrouping.begin(), mLifeCycleGrouping.end()).size())}); + mPolicies.resize(numPoolGroups); +} + +PerLevelEvictionController::~PerLevelEvictionController() +{ + TLLM_CHECK_DEBUG_WITH_INFO(std::all_of(mPolicies.begin(), mPolicies.end(), [](auto const& p) { return p.empty(); }), + "Eviction controller is not empty on destruction"); +} + +PrioritizedEvictionPolicy& PerLevelEvictionController::getPolicy(LifeCycleId lcId) +{ + PoolGroupIndex pgIdx = mLifeCycleGrouping.at(lcId); + return mPolicies.at(pgIdx); +} + +void PerLevelEvictionController::scheduleForEviction(Page& page, bool evictFirst) +{ + TLLM_CHECK_DEBUG(page.nodeRef == std::nullopt); + TLLM_CHECK_DEBUG(page.cacheLevel == mCacheLevel); + auto sharedPage = page.sharedFromThis(); + NodeRef ref = getPolicy(page.lifeCycle).push(sharedPage, evictFirst); + page.nodeRef = ref; + TLLM_CHECK_DEBUG_WITH_INFO(*ref == sharedPage, "stored iterator must dereference to this page"); +} + +TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> PerLevelEvictionController::evict( + TypedVec<PoolGroupIndex, SlotCount> const& minNumPages) +{ + TLLM_CHECK_DEBUG(minNumPages.size() == numPoolGroups()); + + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> ret(numPoolGroups()); + + try + { + for (PoolGroupIndex pgIdx{0}; pgIdx < minNumPages.size(); ++pgIdx) + { + auto& policy = mPolicies.at(pgIdx); + SlotCount const count = minNumPages.at(pgIdx); + if (count < 0) + { + throw LogicError("PerLevelEvictionController::evict: page count must be non-negative"); + } + SlotCount const available = policy.size() + slotCountValueFromSize(ret[pgIdx].size()); + if (available < count) + { + throw OutOfPagesError("Not enough pages to evict in group " + std::to_string(pgIdx.value())); + } + while (slotCountValueFromSize(ret[pgIdx].size()) < count) + { + auto page = policy.pop(); + page->nodeRef = std::nullopt; + ret[pgIdx].push_back(page); + // @TODO: evict dependencies (like Python _evict_dependencies) + } + } + } + catch (...) + { + // Re-queue evicted pages in reverse order (push to front so they are evicted first next time) + for (PoolGroupIndex pgIdx = ret.size(); pgIdx > PoolGroupIndex{0};) + { + --pgIdx; + auto& group = ret[pgIdx]; + while (!group.empty()) + { + auto page = std::move(group.back()); + group.pop_back(); + scheduleForEviction(*page, /*evictFirst=*/true); + } + } + throw; + } + + TLLM_CHECK_DEBUG_WITH_INFO(std::all_of(ret.begin(), ret.end(), + [this](auto const& group) { + return std::all_of(group.begin(), group.end(), + [this](auto const& p) { return p->cacheLevel == mCacheLevel; }); + }), + "Corrupted eviction controller"); + + return ret; +} + +void PerLevelEvictionController::remove(NodeRef node) +{ + auto page = *node; + TLLM_CHECK_DEBUG_WITH_INFO( + page->nodeRef.has_value() && page->nodeRef.value() == node, "page's nodeRef must match the node being removed"); + getPolicy(page->lifeCycle).remove(node); + page->nodeRef = std::nullopt; +} + +SlotCount PerLevelEvictionController::numEvictablePages(PoolGroupIndex pgIdx) const +{ + return mPolicies.at(pgIdx).size(); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/evictionController.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/evictionController.h new file mode 100644 index 000000000000..b7ff84816d7f --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/evictionController.h @@ -0,0 +1,198 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/exceptions.h" +#include "kv_cache_manager_v2/storage/config.h" +#include "kv_cache_manager_v2/utils/sharedPtr.h" + +#include <cstddef> +#include <list> +#include <map> +#include <optional> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// Forward declaration: Page is defined in page.h, but evictionController.h +// must not include page.h to avoid circular dependencies. +// We use a forward-declared abstract interface instead. +class Page; + +// --------------------------------------------------------------------------- +// NodeRef — stable iterator into an LRU list. +// Using list<shared_ptr<Page>>: erasing any element does not invalidate +// other iterators (std::list guarantee). This is the C++ equivalent of the +// Python dllistnode. +// +// IMPORTANT: A NodeRef is always tied to the specific list that issued it. +// Never mix NodeRefs from different LRUEvictionPolicy instances. +// --------------------------------------------------------------------------- +using EvictionList = std::list<SharedPtr<Page>>; +using NodeRef = EvictionList::iterator; + +// --------------------------------------------------------------------------- +// LRUEvictionPolicy — LRU queue backed by std::list. +// Mirrors _eviction_controller.py::LRUEvictionPolicy. +// push() → O(1), pop() → O(1), remove(NodeRef) → O(1). +// --------------------------------------------------------------------------- +class LRUEvictionPolicy +{ +public: + // Push a page into the eviction queue. + // evictFirst=true puts it at the front (will be evicted first). + // Returns a stable iterator (NodeRef) for later O(1) removal. + NodeRef push(SharedPtr<Page> page, bool evictFirst = false); + + // Remove and return the front (least-recently-used) page. + SharedPtr<Page> pop(); + + // Remove an arbitrary page via its iterator. O(1). + SharedPtr<Page> remove(NodeRef node); + + size_t size() const noexcept + { + return mQueue.size(); + } + + bool empty() const noexcept + { + return mQueue.empty(); + } + + EvictionList::const_iterator cbegin() const + { + return mQueue.cbegin(); + } + + EvictionList::const_iterator cend() const + { + return mQueue.cend(); + } + +private: + EvictionList mQueue; +}; + +// --------------------------------------------------------------------------- +// PrioritizedEvictionPolicy — wraps per-priority LRU sub-queues. +// Mirrors _eviction_controller.py::PrioritizedEvictionPolicy. +// Lower priority key = evicted first. +// --------------------------------------------------------------------------- +class PrioritizedEvictionPolicy +{ +public: + NodeRef push(SharedPtr<Page> page, bool evictFirst = false); + SharedPtr<Page> pop(); + SharedPtr<Page> remove(NodeRef node); + + SlotCount size() const noexcept; + + bool empty() const noexcept + { + return size() == 0; + } + + // Generator: returns a mutable lambda that yields shared_ptr<Page> const* + // in eviction order (lowest priority first, LRU within). Returns nullptr when exhausted. + // No extra strong refs — the pointer references the shared_ptr inside the list node. + [[nodiscard]] auto pageGenerator() const + { + using MapIt = std::map<Priority, LRUEvictionPolicy>::const_iterator; + MapIt mapIt = mPolicies.cbegin(); + MapIt mapEnd = mPolicies.cend(); + EvictionList::const_iterator listIt; + // Advance to first non-empty sub-queue. + while (mapIt != mapEnd && mapIt->second.empty()) + ++mapIt; + if (mapIt != mapEnd) + listIt = mapIt->second.cbegin(); + return [=]() mutable -> SharedPtr<Page> const* + { + if (mapIt == mapEnd) + return nullptr; + auto* result = &(*listIt); + ++listIt; + if (listIt == mapIt->second.cend()) + { + ++mapIt; + while (mapIt != mapEnd && mapIt->second.empty()) + ++mapIt; + if (mapIt != mapEnd) + listIt = mapIt->second.cbegin(); + } + return result; + }; + } + +private: + LRUEvictionPolicy& getOrCreate(Priority p); + + // std::map keeps keys sorted. We evict from the lowest-priority key first. + std::map<Priority, LRUEvictionPolicy> mPolicies; +}; + +// --------------------------------------------------------------------------- +// PerLevelEvictionController — one eviction controller per cache level. +// Holds one PrioritizedEvictionPolicy per pool group. +// Mirrors _eviction_controller.py::PerLevelEvictionController. +// --------------------------------------------------------------------------- +class PerLevelEvictionController +{ +public: + // lifeCycleGrouping: maps LifeCycleId → PoolGroupIndex. + // cacheLevel: the level this controller manages. + PerLevelEvictionController(TypedVec<LifeCycleId, PoolGroupIndex> const& lifeCycleGrouping, CacheLevel cacheLevel); + + ~PerLevelEvictionController(); + + void scheduleForEviction(Page& page, bool evictFirst = false); + + // Evict at least minNumPages[pgIdx] pages per pool group. + // Returns evicted pages per pool group. + // On failure, re-queues any already-evicted pages and throws OutOfPagesError. + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> evict( + TypedVec<PoolGroupIndex, SlotCount> const& minNumPages); + + // Remove a page from the queue by its NodeRef. + void remove(NodeRef node); + + [[nodiscard]] SlotCount numEvictablePages(PoolGroupIndex pgIdx) const; + + [[nodiscard]] PoolGroupIndex numPoolGroups() const noexcept + { + return mPolicies.size(); + } + + // All pages in eviction order for a pool group. + [[nodiscard]] auto pageGenerator(PoolGroupIndex pgIdx) const + { + return mPolicies.at(pgIdx).pageGenerator(); + } + +private: + PrioritizedEvictionPolicy& getPolicy(LifeCycleId lcId); + + CacheLevel mCacheLevel; + TypedVec<LifeCycleId, PoolGroupIndex> mLifeCycleGrouping; + TypedVec<PoolGroupIndex, PrioritizedEvictionPolicy> mPolicies; // one per pool group +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/exceptions.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/exceptions.h new file mode 100644 index 000000000000..0b83db95a010 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/exceptions.h @@ -0,0 +1,184 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/utils/sharedPtr.h" + +#include <cuda.h> +#include <stdexcept> +#include <string> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// Exception hierarchy (mirrors _exceptions.py) +// --------------------------------------------------------------------------- + +class OutOfMemoryError : public std::runtime_error +{ +public: + explicit OutOfMemoryError(std::string const& msg = "Out of memory") + : std::runtime_error(msg) + { + } +}; + +class HostOOMError : public OutOfMemoryError +{ +public: + explicit HostOOMError(std::string const& msg = "Host out of memory") + : OutOfMemoryError(msg) + { + } +}; + +class DiskOOMError : public OutOfMemoryError +{ +public: + explicit DiskOOMError(std::string const& msg = "Disk out of memory") + : OutOfMemoryError(msg) + { + } +}; + +class CuOOMError : public OutOfMemoryError +{ +public: + explicit CuOOMError(std::string const& msg = "CUDA out of memory") + : OutOfMemoryError(msg) + { + } +}; + +// Indicates a bug in the KV cache manager code. +class LogicError : public std::logic_error +{ +public: + explicit LogicError(std::string const& msg) + : std::logic_error(msg) + { + } +}; + +// Mirrors a Python `assert` failure: the binding layer translates this to a +// Python AssertionError so shared tests observe the same exception type as the +// pure-Python backend. +class AssertionError : public std::logic_error +{ +public: + explicit AssertionError(std::string const& msg) + : std::logic_error(msg) + { + } +}; + +// Wraps a CUDA driver API error (CUresult). +class CuError : public std::runtime_error +{ +public: + CUresult errorCode; + + explicit CuError(CUresult result) + : std::runtime_error(makeMessage(result)) + , errorCode(result) + { + } + +private: + static std::string makeMessage(CUresult result) + { + char const* errStr = nullptr; + cuGetErrorString(result, &errStr); + std::string msg = "CUDA driver error: "; + msg += errStr ? errStr : "<unknown>"; + return msg; + } +}; + +// A resource (e.g., a page lock) is still in use. +class ResourceBusyError : public std::runtime_error +{ +public: + explicit ResourceBusyError(std::string const& msg = "Resource is busy") + : std::runtime_error(msg) + { + } +}; + +// Not enough free pages to satisfy an allocation request. +class OutOfPagesError : public std::runtime_error +{ +public: + explicit OutOfPagesError(std::string const& msg = "Out of pages") + : std::runtime_error(msg) + { + } +}; + +// Block creation rejected because its tokens are fully covered by an existing sibling. +// Mirrors Python's UselessBlockError — carries the sibling block. +// TODO: Once Python is removed and C++ becomes the primary development target, +// replace this exception-based flow with a simple if-condition return in +// addOrGetExistingBlock (returning the sibling block directly instead of throwing). +// The exception pattern exists only to maintain parity with the Python code path. +// Forward-declared; Block definition is in blockRadixTree.h. +struct Block; + +class UselessBlockError : public std::runtime_error +{ +public: + SharedPtr<Block> block; + + explicit UselessBlockError(SharedPtr<Block> blk) + : std::runtime_error("Block is useless — covered by existing sibling") + , block(std::move(blk)) + { + } +}; + +// --------------------------------------------------------------------------- +// Helper: unwrap a weak_ptr, throw LogicError on dangling reference. +// Mirrors Python's unwrap_rawref(_utils.py:163). +// --------------------------------------------------------------------------- +template <typename T> +SharedPtr<T> unwrap(WeakPtr<T> const& ref) +{ + auto ptr = ref.lock(); + if (!ptr) + throw LogicError("Dereferencing a dangling weak_ptr"); + return ptr; +} + +// --------------------------------------------------------------------------- +// Helper: unwrap CUresult, throw CuError/CuOOMError on failure. +// --------------------------------------------------------------------------- +inline void cuCheck(CUresult result) +{ + if (result == CUDA_SUCCESS) + { + return; + } + if (result == CUDA_ERROR_OUT_OF_MEMORY) + { + throw CuOOMError(); + } + throw CuError(result); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/introspection.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/introspection.cpp new file mode 100644 index 000000000000..835c48c9a13b --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/introspection.cpp @@ -0,0 +1,131 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/introspection.h" + +#include "kv_cache_manager_v2/blockRadixTree.h" + +#include "kv_cache_manager_v2/kvCache.h" +#include "kv_cache_manager_v2/kvCacheManager.h" +#include "kv_cache_manager_v2/storageManager.h" + +#include <utility> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +namespace +{ + +bool allBlockPagesDroppable(Block const& block) +{ + for (auto const* page : block.storage) + { + if (page != nullptr && page->status() != PageStatus::DROPPABLE) + { + return false; + } + } + + for (auto const& [_, child] : block.next) + { + if (!allBlockPagesDroppable(*child)) + { + return false; + } + } + return true; +} + +} // namespace + +KvCacheIntrospection::ActivePageStats KvCacheIntrospection::activePageStats(KvCache const& kvCache) +{ + auto& storageMgr = kvCache.manager().storage(); + CacheLevel const numTiers = storageMgr.numCacheLevels(); + TypedVec<CacheLevel, int> counts(numTiers, 0); + TypedVec<CacheLevel, int> unscheduledEvictable(numTiers, 0); + + for (auto const& activePage : kvCache._activePages()) + { + auto page = kvCache._page(activePage.ordinal, activePage.beamIdx, activePage.lcId); + if (!page) + { + continue; + } + + CacheLevel const level = page->cacheLevel; + counts.at(level) += 1; + if (storageMgr.isEvictable(*page) && !page->scheduledForEviction()) + { + unscheduledEvictable.at(level) += 1; + } + } + + return {std::move(counts), std::move(unscheduledEvictable)}; +} + +TypedVec<PoolGroupIndex, StorageStatistics> KvCacheIntrospection::storageStatistics( + KvCacheManager& manager, CacheLevel level) +{ + TypedVec<PoolGroupIndex, StorageStatistics> result; + PoolGroupIndex const numPoolGroups = manager.storage().numPoolGroups(); + result.reserve(numPoolGroups); + for (PoolGroupIndex pgIdx{0}; pgIdx < numPoolGroups; ++pgIdx) + { + result.push_back(manager.storage().getStatistics(level, pgIdx)); + } + return result; +} + +TypedVec<PoolGroupIndex, SlotCount> KvCacheIntrospection::computeSlotsForBatch(KvCacheManager& manager, + BatchDesc const& batch, int tokensPerBlock, std::optional<SwaScratchReuseConfig> const& swaScratchReuse) +{ + return manager.storage().computeSlotsForBatch(batch, tokensPerBlock, swaScratchReuse); +} + +bool KvCacheIntrospection::allTreePagesDroppable(KvCacheManager& manager) +{ + for (auto const& [_, root] : manager.radixTree().roots()) + { + for (auto const& [__, block] : root->next) + { + if (!allBlockPagesDroppable(*block)) + { + return false; + } + } + } + return true; +} + +void KvCacheIntrospection::setNumSampledKvCaches(KvCacheManager& manager, int value) +{ + manager.mNumSampledKvCaches = value; +} + +void KvCacheIntrospection::setLastAdjustmentTime(KvCacheManager& manager, double value) +{ + manager.mLastAdjustmentTime = value; +} + +void KvCacheIntrospection::setTargetRatioListGpu(KvCacheManager& manager, TypedVec<PoolGroupIndex, float> value) +{ + manager.mTargetRatioListGpu = std::move(value); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/introspection.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/introspection.h new file mode 100644 index 000000000000..07f0dd9acef0 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/introspection.h @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/kvCache.h" +#include "kv_cache_manager_v2/storageManager.h" + +#include <cstddef> +#include <cstdint> +#include <optional> +#include <tuple> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +class KvCacheIntrospection +{ +public: + using ActivePageStats = std::tuple<TypedVec<CacheLevel, int>, TypedVec<CacheLevel, int>>; + + static ActivePageStats activePageStats(KvCache const& kvCache); + static bool allTreePagesDroppable(KvCacheManager& manager); + static TypedVec<PoolGroupIndex, StorageStatistics> storageStatistics(KvCacheManager& manager, CacheLevel level); + + // White-box hook: minimum per-pool-group slot counts to support a BatchDesc. + // Reaches StorageManager::computeSlotsForBatch() (private) via friendship. + static TypedVec<PoolGroupIndex, SlotCount> computeSlotsForBatch(KvCacheManager& manager, BatchDesc const& batch, + int tokensPerBlock, std::optional<SwaScratchReuseConfig> const& swaScratchReuse); + + // White-box test hooks: mutate auto-tuner state so accuracy tests can force a + // pool rebalance. Reach KvCacheManager's private members via friendship. + static void setNumSampledKvCaches(KvCacheManager& manager, int value); + static void setLastAdjustmentTime(KvCacheManager& manager, double value); + static void setTargetRatioListGpu(KvCacheManager& manager, TypedVec<PoolGroupIndex, float> value); +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCache.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCache.cpp new file mode 100644 index 000000000000..1715f44c7463 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCache.cpp @@ -0,0 +1,2567 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/kvCache.h" +#include "kv_cache_manager_v2/blockRadixTree.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/copyEngine.h" +#include "kv_cache_manager_v2/exceptions.h" +#include "kv_cache_manager_v2/kvCacheManager.h" +#include "kv_cache_manager_v2/storageManager.h" +#include "kv_cache_manager_v2/utils/math.h" + +#include "tensorrt_llm/common/assert.h" +#include <algorithm> +#include <stdexcept> +#include <unordered_set> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +namespace +{ + +// Copy one slot's data across all pools in a pool group. +// Used by resume() (GPU→GPU partial block copy) and _snapshotSsmToTreeBlock(). +void copySlotData(StorageManager& storageMgr, CacheLevel dstLevel, CacheLevel srcLevel, PoolGroupIndex pgIdx, + SlotId dstSlotId, SlotId srcSlotId, CUstream stream) +{ + auto slotSizes = storageMgr.slotSize(pgIdx); + CacheTier dstTier = storageMgr.cacheTier(dstLevel); + CacheTier srcTier = storageMgr.cacheTier(srcLevel); + for (PoolIndex poolIdx{0}; poolIdx < slotSizes.size(); ++poolIdx) + { + Address dst = storageMgr.slotAddress(dstLevel, pgIdx, dstSlotId, poolIdx); + Address src = storageMgr.slotAddress(srcLevel, pgIdx, srcSlotId, poolIdx); + batchedCopy(dstTier, srcTier, static_cast<size_t>(slotSizes[poolIdx]), {{dst, src}}, stream); + } +} + +} // anonymous namespace + +// --------------------------------------------------------------------------- +// KvCache constructor +// --------------------------------------------------------------------------- + +KvCache::KvCache(KvCacheManager& manager, ReuseScope reuseScope, std::optional<BlockRadixTree::ReuseMatch> reuseMatch, + std::optional<RequestIdType> mId, PriorityCb priorityCb, std::optional<int> expectedPromptLength) + : id(mId) + , mManager(manager.shared_from_this()) + , mReuseScope(std::move(reuseScope)) + , mPriorityCb(priorityCb ? std::move(priorityCb) : [](BlockOrdinal, LifeCycleId) { return kPriorityDefault; }) + , mStatus(Status::SUSPENDED) + , mCommitState(CommitState::ALLOWED) + , mBeamWidth(BeamIndex{1}) + , mCapacity(0) + , mHistoryLength(0) + , mExpectedPromptLength( + expectedPromptLength.has_value() ? std::optional<int>{std::max(*expectedPromptLength, 0)} : std::nullopt) + , mNumCommittedBlocks(0) + , mTokensPerBlock(manager.tokensPerBlock()) +{ + LifeCycleId numLc = manager.storage().numLifeCycles(); + + // Initialise page index buffers: [beamIdx][lcId] = empty vector + mBasePageIndices.resize(mBeamWidth); + for (auto& beamIndices : mBasePageIndices) + { + beamIndices.resize(numLc, PageIndexBuf{std::vector<int>{}}); + } + + // Always initialise mSsmBlocks (matching Python: no longer optional). + mSsmBlocks.resize(mBeamWidth); + for (auto& beam : mSsmBlocks) + { + beam.resize(numLc); // default-constructs to monostate + } + + mEnableSwaScratchReuse = manager.isSwaScratchReuseEnabled(); + mScratchSlots.resize(manager.storage().numLifeCycles()); + + if (reuseMatch.has_value()) + { + _setupForReuse(*reuseMatch); + } + + _refreshGenerationAllocReady(); + + mAvgHistoryLength.update(static_cast<double>(mHistoryLength)); + + mManager->registerKvCache(this); + mManager->updateAvgReusedLength(static_cast<double>(mHistoryLength)); + TLLM_CHECK_DEBUG(_checkSanity()); +} + +KvCache::~KvCache() +{ + try + { + close(); + } + catch (...) + { + // Destructors must not propagate exceptions (implicitly noexcept in C++11). + // close() should not throw in normal usage; if it does, suppress and accept leak. + } +} + +// --------------------------------------------------------------------------- +// State machine +// --------------------------------------------------------------------------- + +CUstream KvCache::cudaStream() const +{ + TLLM_CHECK_DEBUG_WITH_INFO(mCudaStream.has_value(), "No CUDA stream attached"); + return *mCudaStream; +} + +CachedCudaEvent KvCache::finishEvent() const +{ + return mFinishEvent.value(); +} + +Priority KvCache::getPriority(BlockOrdinal ordinal, LifeCycleId lc) const +{ + return mPriorityCb(ordinal, lc); +} + +StorageManager* KvCache::storageManager() const +{ + return &mManager->storage(); +} + +std::vector<KvCache::ActivePage> KvCache::_activePages() const +{ + std::vector<ActivePage> result; + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + auto const& lcs = mManager->lifeCycles(); + LifeCycleId numLc = mManager->storage().numLifeCycles(); + + for (LifeCycleId lcId{0}; lcId < numLc; ++lcId) + { + // SSM lifecycle → yield from mSsmBlocks (check individual entries). + if (ssmLcId.has_value() && lcId == *ssmLcId) + { + for (BeamIndex bi{0}; bi < mBeamWidth; ++bi) + { + if (!blockPageIsNull(mSsmBlocks[bi][*ssmLcId])) + result.push_back({kBadBlockOrdinal, bi, lcId}); + } + continue; + } + + // Attention lifecycle: yield non-stale blocks (sink + window). + LifeCycle const& lc = lcs.getLifeCycle(lcId); + auto staleRange = _getStaleRange(mHistoryLength, lc); + BlockOrdinal staleBeg = staleRange.beg; + BlockOrdinal staleEnd = staleRange.end; + auto scratchRange = _getScratchRange(lc); + + // Sink blocks: [0, staleBeg) + for (BlockOrdinal ord{0}; ord < staleBeg; ++ord) + for (BeamIndex bi{0}; bi < mBeamWidth; ++bi) + result.push_back({ord, bi, lcId}); + + // Window blocks: [staleEnd, numBlocks) — skip scratch blocks. + for (BlockOrdinal ord{staleEnd}; ord < mBlocks.size(); ++ord) + { + auto& block = mBlocks[ord]; + for (BeamIndex bi{0}; bi < mBeamWidth; ++bi) + { + bool isScratch = scratchRange.contains(ord); + TLLM_CHECK_DEBUG(isScratch == blockPageIsNull(block.pages[bi][lcId])); + if (!isScratch) + result.push_back({ord, bi, lcId}); + } + } + } + return result; +} + +SharedPtr<Page> KvCache::_page(BlockOrdinal ordinal, BeamIndex beamIdx, LifeCycleId lcId) const +{ + bool const isSsm = ordinal == kBadBlockOrdinal; + auto const ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + TLLM_CHECK_DEBUG((ssmLcId.has_value() && lcId == *ssmLcId) == isSsm); + auto const& blockPage = isSsm ? mSsmBlocks.at(beamIdx).at(lcId) : mBlocks.at(ordinal).pages.at(beamIdx).at(lcId); + return blockPageGetPage(blockPage); +} + +void KvCache::activate() +{ + TLLM_CHECK_DEBUG(mStatus == Status::SUSPENDED); + TLLM_CHECK_DEBUG_WITH_INFO(mCudaStream.has_value(), "cuda_stream must be set before activate()"); + + mFinishEvent.reset(); + + // Lock only active (non-stale) pages to GPU — mirrors Python's _active_pages(). + auto activePages = _activePages(); + std::vector<BatchedLockTarget> targets; + targets.reserve(activePages.size()); + + for (auto const& ap : activePages) + { + BlockPage* bp = nullptr; + if (ap.ordinal == kBadBlockOrdinal) + { + bp = &mSsmBlocks[ap.beamIdx][ap.lcId]; + } + else + { + bp = &mBlocks[ap.ordinal].pages[ap.beamIdx][ap.lcId]; + } + auto& holder = std::get<SharedPtr<PageHolder>>(*bp); + TLLM_CHECK_DEBUG(holder); + targets.push_back({holder->page, ap.beamIdx, ap.ordinal, ap.lcId}); + } + + { + auto locks = batchedLockToGpu(*this, targets); + size_t idx = 0; + for (auto& t : targets) + { + TLLM_CHECK_DEBUG(t.page == locks[idx].page()); + BeamIndex bi = t.beamIndex; + LifeCycleId lc = t.lifeCycle; + if (t.ordinal == kBadBlockOrdinal) + mSsmBlocks[bi][lc] = std::move(locks[idx++]); + else + mBlocks[t.ordinal].pages[bi][lc] = std::move(locks[idx++]); + } + } +} + +bool KvCache::resume(std::optional<CUstream> stream) +{ + TLLM_CHECK_DEBUG(mStatus == Status::SUSPENDED); + + // Set stream first (mirrors Python: self.cuda_stream = cuda_stream). + if (stream.has_value()) + { + setCudaStream(*stream); + } + TLLM_CHECK_DEBUG_WITH_INFO(mCudaStream.has_value(), "cuda_stream is never set"); + TLLM_CHECK_DEBUG(!mFinishEvent.has_value()); + + // Check utilization against threshold. + auto const utilizations = mManager->storage().getUtilization(kGpuLevel); + float const utilization = utilizations.empty() ? 0.f : *std::max_element(utilizations.begin(), utilizations.end()); + if (utilization > mManager->config().maxUtilForResume) + { + return false; + } + + auto& storageMgr = mManager->storage(); + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + LifeCycleId numLc = storageMgr.numLifeCycles(); + + // Pre-allocate GPU slots for deferred copies (partial blocks + SSM) and scratch slots + // before locking, so we never end up in a state where pages are locked but we can't allocate. + TypedVec<LifeCycleId, std::optional<Slot>> deferredSlots(numLc); + + // Compute scratch slot deltas UNCONDITIONALLY (mirrors Python: _take_excess_scratch_slots + // is called outside _never_resumed). + auto [excessScratch, scratchDeltaCounts, scratchRanges] = _takeExcessScratchSlots(mCapacity, mHistoryLength); + TLLM_CHECK_DEBUG(excessScratch.size() == numLc + && std::all_of(excessScratch.begin(), excessScratch.end(), [](auto const& s) { return s.empty(); })); + + TypedVec<LifeCycleId, SlotCount> numSlotsNeeded(numLc, 0); + bool hasPartial = false; + if (mNeverResumed) + { + TLLM_CHECK_DEBUG(mBeamWidth == BeamIndex{1}); + hasPartial = numCommittedTokens() % mTokensPerBlock != 0; + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + bool isSsm = ssmLcId.has_value() && lc == *ssmLcId; + if (isSsm || hasPartial) + numSlotsNeeded[lc] += 1; + } + } + + // Add scratch slot needs UNCONDITIONALLY (mirrors Python: delta loop is outside _never_resumed). + for (LifeCycleId lc{0}; lc < numLc; ++lc) + numSlotsNeeded[lc] += std::max(0, scratchDeltaCounts[lc]); + + // Only allocate if any slots are needed. + bool anyNeeded = std::any_of(numSlotsNeeded.begin(), numSlotsNeeded.end(), [](SlotCount n) { return n > 0; }); + if (anyNeeded) + { + TypedVec<LifeCycleId, std::vector<Slot>> tmpSlots; + try + { + MigrationRecorder const migrationRecorder + = [this](std::vector<SharedPtr<Page>> const& pages, std::vector<Slot> const& slots, CacheLevel srcLevel, + CacheLevel dstLevel) { _recordMigratedSlots(pages, slots, srcLevel, dstLevel); }; + DropRecorder const dropRecorder = [this](std::vector<SharedPtr<Page>> const& pages, CacheLevel cacheLevel) + { _recordDroppedPages(pages, cacheLevel); }; + tmpSlots = storageMgr.newGpuSlots(numSlotsNeeded, migrationRecorder, dropRecorder); + } + catch (OutOfPagesError const&) + { + return false; + } + + // Separate deferred vs scratch slots, and collect scratch ready events. + std::vector<CachedCudaEvent const*> scratchReadyEvents; + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + if (!tmpSlots[lc].empty()) + { + // Mirrors Python: `if self._never_resumed and (... SsmLifeCycle or has_partial):` + bool needsDeferred = mNeverResumed && ((ssmLcId.has_value() && lc == *ssmLcId) || hasPartial); + if (needsDeferred) + { + // Python uses pop() here: reserve one slot for deferred copy, then treat the rest as scratch. + deferredSlots[lc] = std::move(tmpSlots[lc].back()); + tmpSlots[lc].pop_back(); + } + // Remaining slots are scratch slots. + for (auto& slot : tmpSlots[lc]) + { + mScratchSlots[lc].emplace_back(std::move(slot), *this, lc, + /*skipWait=*/true); + scratchReadyEvents.push_back(&mScratchSlots[lc].back().slot().readyEvent); + } + } + } + + // Wait only for newly-added scratch slots (mirrors Python's + // stream_wait_events for scratch_slots_to_add). + if (!scratchReadyEvents.empty()) + streamWaitEvents(reinterpret_cast<CudaStream>(cudaStream()), scratchReadyEvents); + } + + try + { + activate(); + } + catch (OutOfPagesError const&) + { + // Release pre-allocated deferred slots on failure. + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + if (deferredSlots[lc].has_value()) + storageMgr.releaseSlot(lc, kGpuLevel, std::move(*deferredSlots[lc])); + } + // Scratch slots stay in mScratchSlots — they'll be freed by close() inside + // a recordEventScope, matching Python behavior. + return false; + } + + // Deferred copy: for partial blocks and SSM, copy from now-locked source pages + // to pre-allocated GPU slots, then unlock sources and replace with new pages. + if (mNeverResumed) + { + BeamIndex beamIdx = kDefaultBeamIndex; + auto const lastOrdinal = BlockOrdinal{mBlocks.empty() ? 0 : (numCommittedTokens() - 1) / mTokensPerBlock}; + CUstream cudaStr = cudaStream(); + + // Wait for all new slots to be ready (deduplicated). + { + std::vector<CachedCudaEvent const*> slotEvents; + for (auto& optSlot : deferredSlots) + { + if (optSlot.has_value()) + slotEvents.push_back(&optSlot->readyEvent); + } + streamWaitEvents(reinterpret_cast<CudaStream>(cudaStr), slotEvents); + } + + // Phase 1: Copy GPU→GPU from locked source pages to pre-allocated slots. + std::vector<SharedPageLock*> srcLocks; + for (LifeCycleId lcIdx{0}; lcIdx < numLc; ++lcIdx) + { + if (!deferredSlots[lcIdx].has_value()) + continue; + auto& newSlot = *deferredSlots[lcIdx]; + + BlockPage* sourcePage = nullptr; + if (ssmLcId.has_value() && lcIdx == *ssmLcId) + { + if (numCommittedTokens() == 0) + continue; // fresh SSM — no source to copy from + sourcePage = &mSsmBlocks[beamIdx][lcIdx]; + } + else + { + sourcePage = &mBlocks[lastOrdinal].pages[beamIdx][lcIdx]; + } + auto* lock = std::get_if<SharedPageLock>(sourcePage); + TLLM_CHECK_DEBUG(lock && lock->isValid()); + bool const hasPartialReuseSource = _hasReuseSource(*sourcePage); + srcLocks.push_back(lock); + + PoolGroupIndex pgIdx = storageMgr.getPoolGroupIndex(lcIdx); + copySlotData(storageMgr, kGpuLevel, kGpuLevel, pgIdx, newSlot.slotId(), lock->page()->slotId(), cudaStr); + if ((!ssmLcId.has_value() || lcIdx != *ssmLcId) && _shouldRecordStats()) + { + bool const changed = mPendingStats.recordAllocationRange(lcIdx, lastOrdinal, lastOrdinal + 1, + /*beamWidth=*/1, /*countAsMissed=*/!hasPartialReuseSource); + if (changed) + { + mManager->markStatsDirty(id); + } + } + KVCacheIterationStatsDelta iterationStats; + iterationStats.iterIntraDeviceCopyBlocks = 1; + for (size_t const size : storageMgr.slotSize(pgIdx)) + { + iterationStats.iterIntraDeviceCopyBytes += static_cast<int64_t>(size); + } + _recordDirectIterationStats(lcIdx, iterationStats); + } + + // Unlock source pages — recordEventScope captures all prior CUDA work + // so the original pages know when we're done reading from them. + if (!srcLocks.empty()) + { + auto scope = recordEventScope(); + for (auto* lock : srcLocks) + lock->unlock(); + } + + // Phase 2: Replace with new UncommittedPages (both copied and fresh SSM). + for (LifeCycleId lcIdx{0}; lcIdx < numLc; ++lcIdx) + { + if (!deferredSlots[lcIdx].has_value()) + continue; + auto& newSlot = *deferredSlots[lcIdx]; + + BlockPage* targetBp; + BlockOrdinal blockOrdinal; + if (ssmLcId.has_value() && lcIdx == *ssmLcId) + { + targetBp = &mSsmBlocks[beamIdx][lcIdx]; + blockOrdinal = kBadBlockOrdinal; + } + else + { + targetBp = &mBlocks[lastOrdinal].pages[beamIdx][lcIdx]; + blockOrdinal = lastOrdinal; + } + + auto newPage = makeShared<UncommittedPage>(*this, blockOrdinal, lcIdx, kGpuLevel, beamIdx); + newPage->setSlot(newSlot); + auto newLock = newPage->lock(*this, beamIdx, blockOrdinal, lcIdx, /*skipWait=*/true); + *targetBp = std::move(newLock); + } + + // Clear treeBlock for partial last block (mirrors Python: partial block is uncommitted). + if (numCommittedTokens() % mTokensPerBlock != 0) + mBlocks[lastOrdinal].treeBlock = nullptr; + } + + mNeverResumed = false; + mStatus = Status::ACTIVE; + return true; +} + +bool KvCache::prefetch(CacheLevel target) +{ + TLLM_CHECK_DEBUG(mStatus == Status::SUSPENDED); + auto& storageMgr = mManager->storage(); + CacheLevel const numTiers = storageMgr.numCacheLevels(); + TLLM_CHECK_DEBUG(kGpuLevel <= target && target < numTiers); + + PoolGroupIndex const numPoolGroups = storageMgr.numPoolGroups(); + TypedVec<PoolGroupIndex, TypedVec<CacheLevel, std::vector<SharedPtr<Page>>>> allPages( + numPoolGroups, TypedVec<CacheLevel, std::vector<SharedPtr<Page>>>(numTiers)); + + for (auto const& activePage : _activePages()) + { + auto page = _page(activePage.ordinal, activePage.beamIdx, activePage.lcId); + if (!page) + { + continue; + } + CacheLevel const level = page->cacheLevel; + if (level < target) + { + continue; + } + auto const pgIdx = storageMgr.getPoolGroupIndex(activePage.lcId); + allPages.at(pgIdx).at(level).push_back(std::move(page)); + } + + try + { + storageMgr.prefetch(target, allPages); + } + catch (OutOfPagesError const&) + { + return false; + } + return true; +} + +void KvCache::suspend() +{ + TLLM_CHECK_DEBUG(mStatus == Status::ACTIVE); + TLLM_CHECK_DEBUG(_checkSanity()); + TLLM_CHECK_DEBUG(!mFinishEvent.has_value()); + + // Copy data from external buffers back to internal vectors (mirrors Python's suspend). + for (BeamIndex bi{0}; bi < mBeamWidth; ++bi) + for (LifeCycleId lcId{0}; lcId < mBasePageIndices[bi].size(); ++lcId) + if (std::holds_alternative<Span<int>>(mBasePageIndices[bi][lcId])) + setBasePageIndexBuf(bi, lcId, nullptr, 0); + + // Record event scope — mirrors Python's `with self._record_event()`. + // SharedPageLock destructors inside the scope use finishEvent() to synchronize. + { + auto scope = recordEventScope(); + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + + // Convert SharedPageLocks → PageHolders for active (non-stale) pages only. + // Mirrors Python: for ordinal, beam_idx, lc_idx in self._active_pages() + for (auto const& ap : _activePages()) + { + auto& bp = (ap.lcId != ssmLcId) ? mBlocks[ap.ordinal].pages[ap.beamIdx][ap.lcId] + : mSsmBlocks[ap.beamIdx][ap.lcId]; + // expect_type(_SharedPageLock, beam_block[lc_idx]) → std::get raises on wrong type + auto& lock = std::get<SharedPageLock>(bp); + auto holder = lock.page()->hold(); + bp = std::move(holder); // ~SharedPageLock calls unlock() → notifyFinish(finishEvent()) + } + // Free scratch slots inside scope — unlock() needs finishEvent(). + // Mirrors Python: _free_scratch_slots() inside `with self._record_event()`. + _freeScratchSlots(); + } + mStatus = Status::SUSPENDED; +} + +void KvCache::close() +{ + TLLM_CHECK_DEBUG(_checkSanity()); + if (mStatus == Status::CLOSED) + return; + + discardPendingStats(); + stopCommitting(); + TLLM_CHECK_DEBUG(_checkSanity()); + + if (mCapacity > 0) + { + mAvgCapacity.update(static_cast<double>(mCapacity)); + mManager->updateAvgSqrCapacity(mAvgCapacity.value() * mAvgCapacity.value()); + mManager->updateAvgSqrHistoryLength(mAvgHistoryLength.value() * mAvgHistoryLength.value()); + mManager->incrementNumSampledKvCaches(); + mManager->tryUpdateTargetRatios(); + } + + // Record event scope — mirrors Python's `with self._record_event()`. + // Python always enters _record_event() here; _cuda_stream is valid for both ACTIVE and SUSPENDED. + { + auto scope = recordEventScope(); + _clearBlocks(); + } + mStatus = Status::CLOSED; + mManager->unregisterKvCache(this); +} + +KVCacheStatsDelta KvCache::commitPendingStats() +{ + if (!_shouldRecordStats()) + { + discardPendingStats(); + return {}; + } + + mManager->commitStats(mPendingStats.globalStats(), mPendingStats.iterationStatsByLifeCycle()); + mManager->commitSsmSnapshotIterationStats(mPendingStats.ssmSnapshotIterationStatsByLifeCycle()); + KVCacheStatsDelta const requestStats = mPendingStats.requestStats().copy(); + mPendingStats.clear(); + mManager->clearStatsDirty(id); + return requestStats; +} + +void KvCache::discardPendingStats() +{ + mPendingStats.clear(); + mManager->clearStatsDirty(id); +} + +bool KvCache::_shouldRecordStats() const +{ + return mManager->config().enableStats && !mManager->isStatsExcluded(id); +} + +void KvCache::_refreshStatsDirtyState() +{ + if (!mPendingStats.empty()) + { + mManager->markStatsDirty(id); + } + else + { + mManager->clearStatsDirty(id); + } +} + +void KvCache::_recordDirectIterationStats(LifeCycleId lifeCycle, KVCacheIterationStatsDelta const& iterationStats) +{ + if (!_shouldRecordStats() || iterationStats.empty() + || !std::holds_alternative<AttnLifeCycle>(mManager->lifeCycles().getLifeCycle(lifeCycle))) + { + return; + } + IterationStatsByLifeCycle iterationStatsByLifeCycle; + iterationStatsByLifeCycle.emplace(lifeCycle, iterationStats.copy()); + mManager->commitStats({}, iterationStatsByLifeCycle); +} + +void KvCache::_recordMigratedSlots( + std::vector<SharedPtr<Page>> const& pages, std::vector<Slot> const& slots, CacheLevel srcLevel, CacheLevel dstLevel) +{ + if (!_shouldRecordStats()) + { + return; + } + TLLM_CHECK_DEBUG(pages.size() == slots.size()); + for (auto const& page : pages) + { + LifeCycleId const lifeCycle = page->lifeCycle; + if (!std::holds_alternative<AttnLifeCycle>(mManager->lifeCycles().getLifeCycle(lifeCycle))) + { + continue; + } + + PoolGroupIndex const poolGroup = mManager->storage().getPoolGroupIndex(lifeCycle); + int64_t pageSize = 0; + for (size_t const size : mManager->storage().slotSize(poolGroup)) + { + pageSize += static_cast<int64_t>(size); + } + + KVCacheStatsDelta stats; + KVCacheIterationStatsDelta iterationStats; + if (srcLevel == kGpuLevel && dstLevel > kGpuLevel) + { + iterationStats.iterOffloadBlocks = 1; + iterationStats.iterOffloadBytes = pageSize; + } + else if (dstLevel == kGpuLevel) + { + stats.allocTotalBlocks = 1; + stats.allocNewBlocks = 1; + iterationStats.iterAllocTotalBlocks = 1; + iterationStats.iterAllocNewBlocks = 1; + if (srcLevel > kGpuLevel) + { + iterationStats.iterOnboardBlocks = 1; + iterationStats.iterOnboardBytes = pageSize; + } + else if (srcLevel == kGpuLevel) + { + iterationStats.iterIntraDeviceCopyBlocks = 1; + iterationStats.iterIntraDeviceCopyBytes = pageSize; + } + } + + if (!stats.empty() || !iterationStats.empty()) + { + IterationStatsByLifeCycle iterationStatsByLifeCycle; + iterationStatsByLifeCycle.emplace(lifeCycle, iterationStats); + mManager->commitStats(stats, iterationStatsByLifeCycle); + } + } +} + +void KvCache::_recordDroppedPages(std::vector<SharedPtr<Page>> const& pages, CacheLevel cacheLevel) +{ + (void) cacheLevel; + if (!_shouldRecordStats()) + { + return; + } + for (auto const& page : pages) + { + LifeCycleId const lifeCycle = page->lifeCycle; + if (!std::holds_alternative<AttnLifeCycle>(mManager->lifeCycles().getLifeCycle(lifeCycle))) + { + continue; + } + PoolGroupIndex const poolGroup = mManager->storage().getPoolGroupIndex(lifeCycle); + int64_t pageSize = 0; + for (size_t const size : mManager->storage().slotSize(poolGroup)) + { + pageSize += static_cast<int64_t>(size); + } + KVCacheIterationStatsDelta iterationStats; + iterationStats.iterHostDroppedBlocks = 1; + iterationStats.iterHostDroppedBytes = pageSize; + _recordDirectIterationStats(lifeCycle, iterationStats); + } +} + +void KvCache::_recordResizePendingAllocations(BlockOrdinal blockBegin, BlockOrdinal blockEnd, + TypedVec<LifeCycleId, HalfOpenRange<BlockOrdinal>> const& excludedRanges, bool countAsGeneration) +{ + if (!_shouldRecordStats() || blockBegin >= blockEnd) + { + return; + } + + bool changed = false; + for (auto const& [lifeCycle, unused] : mManager->lifeCycles().attentionLifeCycles()) + { + (void) unused; + auto const& excluded = excludedRanges[lifeCycle]; + BlockOrdinal const firstEnd = std::min(blockEnd, excluded.beg); + if (blockBegin < firstEnd) + { + changed |= mPendingStats.recordAllocationRange( + lifeCycle, blockBegin, firstEnd, mBeamWidth.value(), !countAsGeneration, countAsGeneration); + } + BlockOrdinal const secondBegin = std::max(blockBegin, excluded.end); + if (secondBegin < blockEnd) + { + changed |= mPendingStats.recordAllocationRange( + lifeCycle, secondBegin, blockEnd, mBeamWidth.value(), !countAsGeneration, countAsGeneration); + } + } + if (changed) + { + mManager->markStatsDirty(id); + } +} + +void KvCache::_subtractPendingAllocationRange(BlockOrdinal blockBegin, BlockOrdinal blockEnd) +{ + if (mPendingStats.subtractAllocationRange(blockBegin, blockEnd)) + { + _refreshStatsDirtyState(); + } +} + +bool KvCache::_hasReuseSource(BlockPage const& page) +{ + auto const committedPage = dynamicPointerCast<CommittedPage>(blockPageGetPage(page)); + return committedPage && committedPage->block != nullptr; +} + +// --------------------------------------------------------------------------- +// _clearBlocks +// --------------------------------------------------------------------------- + +void KvCache::_clearBlocks() +{ + // Drop last block first (mirrors Python: while self._blocks: self._blocks.pop()). + while (!mBlocks.empty()) + mBlocks.pop_back(); + _freeScratchSlots(); + // Clear SSM blocks. + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + if (ssmLcId.has_value()) + { + for (auto& beamBlock : mSsmBlocks) + beamBlock[*ssmLcId] = std::monostate{}; + } +} + +// --------------------------------------------------------------------------- +// _copyPageToTreeBlock: copy a page into a new committed (or SSM committed) page +// attached to a radix tree block. Mirrors Python's _copy_page_to_tree_block. +// --------------------------------------------------------------------------- + +void KvCache::_copyPageToTreeBlock(SharedPtr<Block> const& treeBlock, LifeCycleId lcIdx, SharedPtr<Page> const& srcPage, + std::optional<int> ssmNumTokensInBlock) +{ + if (treeBlock->storage.at(lcIdx) != nullptr) + { + return; // block already holds a page for this lifecycle + } + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + bool const isSsm = ssmLcId.has_value() && lcIdx == *ssmLcId; + TLLM_CHECK_DEBUG(isSsm == ssmNumTokensInBlock.has_value()); + + auto& storageMgr = mManager->storage(); + PoolGroupIndex pgIdx = storageMgr.getPoolGroupIndex(lcIdx); + + for (CacheLevel lvl = srcPage->cacheLevel; lvl < storageMgr.numCacheLevels(); ++lvl) + { + Slot newSlot; + try + { + auto slots = storageMgr.newSlotsForPoolGroup(lvl, pgIdx, 1); + newSlot = std::move(slots[0]); + } + catch (OutOfPagesError const&) + { + continue; + } + + CUstream stream = cudaStream(); + newSlot.readyEvent.waitInStream(reinterpret_cast<CudaStream>(stream)); + copySlotData(storageMgr, lvl, srcPage->cacheLevel, pgIdx, newSlot.slotId(), srcPage->slotId(), stream); + + CachedCudaEvent readyEv(reinterpret_cast<CudaStream>(stream)); + auto tempPage = makeShared<UncommittedPage>(*this, treeBlock->ordinal(), lcIdx, lvl, kDefaultBeamIndex); + tempPage->setSlot(newSlot); + SharedPtr<CommittedPage> committed; + if (ssmNumTokensInBlock.has_value()) + { + committed = tempPage->convertToSsmCommitted(treeBlock, std::move(readyEv), *ssmNumTokensInBlock); + } + else + { + committed = tempPage->convertToCommitted(treeBlock, std::move(readyEv)); + } + + // Schedule for eviction so eviction controller keeps a strong reference, + // preventing the page from being destroyed. + storageMgr.scheduleForEviction(*committed); + return; // success + } + // No pages available in any level, silently skip snapshot (matches Python). +} + +// --------------------------------------------------------------------------- +// _snapshotSsmToTreeBlock: snapshot live SSM state to a radix tree block for the +// given committed token count. Mirrors Python's _snapshot_ssm_to_tree_block. +// --------------------------------------------------------------------------- + +void KvCache::_snapshotSsmToTreeBlock(SharedPtr<Block> const& treeBlock, LifeCycleId ssmLcId, int numTokens, bool move) +{ + int const numTokensInBlock = numTokens - treeBlock->ordinal().value() * mTokensPerBlock; + TLLM_CHECK_DEBUG(0 < numTokensInBlock && numTokensInBlock <= mTokensPerBlock); + + CommittedPage* existingRaw = treeBlock->storage.at(ssmLcId); + int existingNumTokens = 0; + if (existingRaw != nullptr) + { + auto* existingSsm = dynamic_cast<SsmCommittedPage*>(existingRaw); + TLLM_CHECK_DEBUG(existingSsm != nullptr); + existingNumTokens = existingSsm->numTokensInBlock; + } + if (existingNumTokens >= numTokensInBlock) + { + return; + } + if (existingRaw != nullptr) + { + // Detach the smaller snapshot; the CommittedPage dtor's expected-page guard + // makes the later unlink a no-op (the slot is already null). + treeBlock->storage.at(ssmLcId) = nullptr; + if (existingRaw->scheduledForEviction()) + { + existingRaw->manager->excludeFromEviction(*existingRaw); + } + } + + auto& ssmBlock = mSsmBlocks[kDefaultBeamIndex]; + auto* ssmLock = std::get_if<SharedPageLock>(&ssmBlock[ssmLcId]); + TLLM_CHECK_DEBUG(ssmLock && ssmLock->isValid()); + auto srcPage = ssmLock->page(); + if (move) + { + auto unlocked = ssmLock->unlock(); + auto up = dynamicPointerCast<UncommittedPage>(unlocked); + TLLM_CHECK_DEBUG(up != nullptr); + ssmBlock[ssmLcId] = std::monostate{}; + auto committed = up->convertToSsmCommitted(treeBlock, finishEvent(), numTokensInBlock); + mManager->storage().scheduleForEviction(*committed); + return; + } + + _copyPageToTreeBlock(treeBlock, ssmLcId, srcPage, numTokensInBlock); +} + +// --------------------------------------------------------------------------- +// _snapshotPartialBlockToTree: snapshot a partial final block into the radix +// tree. Mirrors Python's _snapshot_partial_block_to_tree. +// --------------------------------------------------------------------------- + +void KvCache::_snapshotPartialBlockToTree(BlockOrdinal ordinal, bool commitSsm) +{ + int const start = ordinal.value() * mTokensPerBlock; + int const end = std::min(start + mTokensPerBlock, static_cast<int>(mCommittedTokens.size())); + std::vector<TokenIdExt> tokens(mCommittedTokens.begin() + start, mCommittedTokens.begin() + end); + int const numTokens = static_cast<int>(tokens.size()); + TLLM_CHECK_DEBUG(0 < numTokens && numTokens < mTokensPerBlock); + + LifeCycleId numLc = mManager->storage().numLifeCycles(); + NodeBase* prevNode = nullptr; + RootBlock& root = mManager->radixTree().addOrGetExisting(mReuseScope); + if (ordinal == BlockOrdinal{0}) + { + prevNode = &root; + } + else + { + prevNode = _getTreeBlock(BlockOrdinal{ordinal.value() - 1}).get(); + } + + bool isNew = false; + SharedPtr<Block> treeBlock; + try + { + treeBlock = addOrGetExistingBlock(prevNode, numLc, tokens, &isNew); + } + catch (UselessBlockError const& e) + { + treeBlock = e.block; + isNew = false; + } + TLLM_CHECK_DEBUG(treeBlock); + TLLM_CHECK_DEBUG(isNew || std::equal(tokens.begin(), tokens.end(), treeBlock->tokens.begin())); + + auto& beamBlock = mBlocks.at(ordinal).pages[kDefaultBeamIndex]; + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + // Only attach partial attention pages to a tree block whose token span is + // exactly the partial snapshot. A longer existing sibling may already have + // full attention pages; otherwise a partial page would make it look more + // reusable than it is. + if (static_cast<int>(treeBlock->tokens.size()) == numTokens) + { + for (auto const& [lcIdx, attn] : mManager->lifeCycles().attentionLifeCycles()) + { + (void) attn; + auto& bp = beamBlock[lcIdx]; + if (blockPageIsNull(bp) || treeBlock->storage.at(lcIdx) != nullptr) + { + continue; + } + _copyPageToTreeBlock(treeBlock, lcIdx, blockPageGetPage(bp)); + } + } + if (commitSsm) + { + TLLM_CHECK_DEBUG(ssmLcId.has_value()); + _snapshotSsmToTreeBlock(treeBlock, *ssmLcId, start + numTokens); + } + if (isNew && treeBlock->eventSink) + { + treeBlock->eventSink->addStoredBlock(*treeBlock); + } +} + +// --------------------------------------------------------------------------- +// resize +// --------------------------------------------------------------------------- + +bool KvCache::resize(std::optional<int> capacity, std::optional<int> historyLength) +{ + TLLM_CHECK_DEBUG(mStatus == Status::ACTIVE); + TLLM_CHECK_DEBUG(mBlocks.size() == BlockOrdinal{divUp(mCapacity, mTokensPerBlock)}); + + int newCap = capacity.value_or(mCapacity); + int newHist = historyLength.value_or(mHistoryLength); + + if (capacity.has_value()) + mAvgCapacity.update(static_cast<double>(newCap)); + if (historyLength.has_value()) + mAvgHistoryLength.update(static_cast<double>(newHist)); + + if (newHist < mHistoryLength) + throw std::invalid_argument("History length cannot be decreased"); + if (newCap < newHist) + throw std::invalid_argument("History length cannot exceed capacity"); + + // Scratch reuse: enforce constraint. + bool enableScratch = mEnableSwaScratchReuse; + if (TLLM_UNLIKELY(gDebug) && enableScratch && newCap != mCapacity) + { + int const maxRewindLen = _swaScratchMaxRewindLen(); + int const minHistoryLength = std::max(0, mCapacity - maxRewindLen); + bool const validSwaScratchHistory = minHistoryLength <= newHist && newHist <= mCapacity; + TLLM_CHECK_WITH_INFO(validSwaScratchHistory, + "SWA scratch requires old_capacity - max_rewind_len <= history_length <= old_capacity"); + (void) validSwaScratchHistory; + } + + bool const recordGenerationAllocStats = mGenerationAllocReady && newCap > mCapacity; + if (!enableScratch && _shortcutSetCapacity(newCap) && _shortcutSetHistoryLength(newHist)) + { + _refreshGenerationAllocReady(); + return true; + } + + BlockOrdinal oldNumBlocks{divUp(mCapacity, mTokensPerBlock)}; + BlockOrdinal newNumBlocks{divUp(newCap, mTokensPerBlock)}; + LifeCycleId numLc = mManager->storage().numLifeCycles(); + auto const& lcs = mManager->lifeCycles(); + + _checkPageIndexBufferCapacity(newNumBlocks); + + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + auto backupHolders = _unlockStaleBlocks(newHist); + + if (newNumBlocks < oldNumBlocks) + { + TLLM_CHECK_DEBUG_WITH_INFO(!hasScratchSlots(), "Cannot shrink while scratch slots exist"); + _subtractPendingAllocationRange(newNumBlocks, oldNumBlocks); + auto scope = recordEventScope(); + _decreaseCapacity(newNumBlocks); + } + + // Compute scratch deltas. + auto [excessScratchSlots, deltaScratchSlots, scratchRanges] = _takeExcessScratchSlots(newCap, newHist); + + if (newNumBlocks >= oldNumBlocks) + { + // Compute new normal slots needed per lifecycle. + TypedVec<LifeCycleId, SlotCount> numNewSlots(numLc, 0); + TypedVec<LifeCycleId, HalfOpenRange<BlockOrdinal>> staleRanges(numLc); + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + if (ssmLcId.has_value() && lc == *ssmLcId) + continue; + staleRanges[lc] = _getStaleRange(newHist, lcs.getLifeCycle(lc)); + + if (enableScratch) + { + HalfOpenRange<BlockOrdinal> const newBlockRange{oldNumBlocks, newNumBlocks}; + int const numNewBlocksUsingScratch = intersect(scratchRanges[lc], newBlockRange).length(); + int const numNewNormalBlocks = newBlockRange.length() - numNewBlocksUsingScratch; + numNewSlots[lc] = static_cast<SlotCount>(numNewNormalBlocks) * mBeamWidth.value(); + } + else + { + auto [staleBeg, staleEnd] = staleRanges[lc]; + int numNewBlocksToAdd; + if (oldNumBlocks < staleBeg) + { + TLLM_CHECK_DEBUG(newNumBlocks >= staleEnd); + numNewBlocksToAdd = (staleBeg - oldNumBlocks) + (newNumBlocks - staleEnd); + } + else + { + numNewBlocksToAdd = newNumBlocks - std::max(staleEnd, oldNumBlocks); + } + numNewSlots[lc] = static_cast<SlotCount>(numNewBlocksToAdd) * mBeamWidth.value(); + } + } + + // Compute net allocation counts (normal + scratch delta). + TypedVec<LifeCycleId, SlotCount> netAllocCounts(numLc, 0); + for (LifeCycleId lc{0}; lc < numLc; ++lc) + netAllocCounts[lc] = numNewSlots[lc] + static_cast<SlotCount>(deltaScratchSlots[lc]); + + // Allocate new slots. + TypedVec<LifeCycleId, std::vector<Slot>> newSlots; + bool anyPositive = std::any_of(netAllocCounts.begin(), netAllocCounts.end(), [](SlotCount c) { return c > 0; }); + if (anyPositive) + { + TypedVec<LifeCycleId, SlotCount> allocCounts(numLc); + for (LifeCycleId lc{0}; lc < numLc; ++lc) + allocCounts[lc] = std::max(SlotCount{0}, netAllocCounts[lc]); + try + { + MigrationRecorder const migrationRecorder + = [this](std::vector<SharedPtr<Page>> const& pages, std::vector<Slot> const& slots, + CacheLevel srcLevel, CacheLevel dstLevel) + { _recordMigratedSlots(pages, slots, srcLevel, dstLevel); }; + DropRecorder const dropRecorder + = [this](std::vector<SharedPtr<Page>> const& pages, CacheLevel cacheLevel) + { _recordDroppedPages(pages, cacheLevel); }; + newSlots = mManager->storage().newGpuSlots(allocCounts, migrationRecorder, dropRecorder); + } + catch (OutOfPagesError const&) + { + _recoverExcessScratchSlots(excessScratchSlots); + _lockHeldBlocks(backupHolders); + return false; + } + } + else + { + newSlots.resize(numLc); + } + + // Wait on newly allocated slots. + { + std::vector<CachedCudaEvent const*> readyEvents; + for (auto const& lcSlots : newSlots) + for (auto const& slot : lcSlots) + readyEvents.push_back(&slot.readyEvent); + if (!readyEvents.empty()) + streamWaitEvents(reinterpret_cast<CudaStream>(cudaStream()), readyEvents); + } + + // Combine: new slots + excess scratch detached slots. + TypedVec<LifeCycleId, std::vector<Slot>> slots(numLc); + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + auto& combined = slots[lc]; + combined = std::move(newSlots[lc]); + for (auto& lock : excessScratchSlots[lc]) + combined.push_back(lock.detachSlot()); + excessScratchSlots[lc].clear(); + } + + // Release excess if net is negative. + if (std::any_of(netAllocCounts.begin(), netAllocCounts.end(), [](SlotCount c) { return c < 0; })) + { + auto scope = recordEventScope(); + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + for (SlotCount i = 0; i < -netAllocCounts[lc]; ++i) + { + auto slot = std::move(slots[lc].back()); + slots[lc].pop_back(); + slot.readyEvent = finishEvent(); + mManager->storage().releaseSlot(lc, kGpuLevel, std::move(slot)); + } + } + } + + // Assert correct combined slot count. + if (TLLM_UNLIKELY(gDebug)) + { + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + SlotCount const expected + = numNewSlots[lc] + std::max(SlotCount{0}, static_cast<SlotCount>(deltaScratchSlots[lc])); + TLLM_CHECK(static_cast<SlotCount>(slots[lc].size()) == expected); + } + } + + // Fulfill additional scratch slots (pop from end of combined slots). + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + for (int i = 0; i < deltaScratchSlots[lc]; ++i) + { + auto slot = std::move(slots[lc].back()); + slots[lc].pop_back(); + mScratchSlots[lc].emplace_back(std::move(slot), *this, lc, /*skipWait=*/true); + } + } + + // Wait for all slots that will back new pages. This includes detached excess scratch slots, + // which are not covered by the earlier wait on newly allocated slots. + { + std::vector<CachedCudaEvent const*> readyEvents; + for (auto const& lcSlots : slots) + for (auto const& slot : lcSlots) + readyEvents.push_back(&slot.readyEvent); + if (!readyEvents.empty()) + streamWaitEvents(reinterpret_cast<CudaStream>(cudaStream()), readyEvents); + } + + // Scratch and stale blocks do not consume per-request KV pages, so exclude them from allocation stats. + auto const& excludedRanges = enableScratch ? scratchRanges : staleRanges; + _recordResizePendingAllocations(oldNumBlocks, newNumBlocks, excludedRanges, recordGenerationAllocStats); + + // Resize page index buffers. + TLLM_CHECK_DEBUG(std::all_of(mBasePageIndices.begin(), mBasePageIndices.end(), + [oldNumBlocks](auto const& beamIndices) + { + return std::all_of(beamIndices.begin(), beamIndices.end(), + [oldNumBlocks](auto const& buf) + { + auto const* vec = std::get_if<std::vector<int>>(&buf); + return !vec || vec->size() == toSizeT(oldNumBlocks); + }); + })); + + // Create SeqBlocks for new ordinals (pop slots from end, matching Python). + _resizePageIndexBuffers(newNumBlocks); + for (BlockOrdinal ord = oldNumBlocks; ord < newNumBlocks; ++ord) + { + SeqBlock sb; + sb.pages.resize(mBeamWidth); + for (auto& row : sb.pages) + row.resize(numLc); + for (BeamIndex bi{0}; bi < mBeamWidth; ++bi) + { + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + if (ssmLcId.has_value() && lc == *ssmLcId) + continue; // SSM pages live in mSsmBlocks, not in mBlocks + if (enableScratch) + { + if (scratchRanges[lc].contains(ord)) + continue; // Scratch block — no per-block page allocation. + } + else + { + auto [staleBeg, staleEnd] = staleRanges[lc]; + if (staleBeg <= ord && ord < staleEnd) + continue; + } + auto slot = std::move(slots[lc].back()); + slots[lc].pop_back(); + auto page = makeShared<UncommittedPage>(*this, ord, lc, kGpuLevel, bi); + page->setSlot(slot); + sb.pages[bi][lc] = page->lock(*this, bi, ord, lc, /*skipWait=*/true); + } + } + mBlocks.push_back(std::move(sb)); + } + TLLM_CHECK_DEBUG(std::all_of(slots.begin(), slots.end(), [](auto const& vec) { return vec.empty(); })); + } + + mCapacity = newCap; + mHistoryLength = newHist; + _refreshGenerationAllocReady(); + _evictOutOfWindowBlocks(newHist); + TLLM_CHECK_DEBUG(_checkSanity()); + return true; +} + +void KvCache::setCapacity(int cap) +{ + if (mEnableSwaScratchReuse) + { + throw std::invalid_argument( + "Cannot use capacity setter when SWA scratch reuse is enabled. " + "Use resize(capacity, history_length) instead."); + } + if (!resize(cap, std::nullopt)) + throw OutOfPagesError("Not enough pages in GPU memory"); +} + +void KvCache::setHistoryLength(int hist) +{ + bool success = resize(std::nullopt, hist); + TLLM_CHECK_DEBUG(success); + (void) success; +} + +bool KvCache::_shortcutSetCapacity(int newCap) +{ + if (newCap == mCapacity) + return true; + // No shortcut if block count changes. + if (divUp(newCap, mTokensPerBlock) != divUp(mCapacity, mTokensPerBlock)) + return false; + mCapacity = newCap; + return true; +} + +bool KvCache::_shortcutSetHistoryLength(int newHist) +{ + if (newHist == mHistoryLength) + return true; + // Check if stale range changes for any lifecycle. + for (auto [lcId, lc] : mManager->lifeCycles()) + { + bool changed = std::visit( + [&](auto const& v) -> bool + { + using T = std::decay_t<decltype(v)>; + if constexpr (std::is_same_v<T, SsmLifeCycle>) + { + // history_length change does not impact blocks at all. + return false; + } + else + { + static_assert(std::is_same_v<T, AttnLifeCycle>); + if (!v.windowSize.has_value()) + return false; + return v.getStaleRange(newHist, mTokensPerBlock) + != v.getStaleRange(mHistoryLength, mTokensPerBlock); + } + }, + lc); + if (changed) + return false; + } + mHistoryLength = newHist; + return true; +} + +void KvCache::_refreshGenerationAllocReady() +{ + if (mExpectedPromptLength.has_value() && mHistoryLength >= *mExpectedPromptLength) + { + mGenerationAllocReady = true; + } +} + +// --------------------------------------------------------------------------- +// Capacity management +// --------------------------------------------------------------------------- + +void KvCache::_increaseCapacity(BlockOrdinal newNumBlocks, int newHistoryLength) +{ + BlockOrdinal curNumBlocks = mBlocks.size(); + LifeCycleId numLc = mManager->storage().numLifeCycles(); + auto const& lcs = mManager->lifeCycles(); + auto ssmLcId = lcs.ssmLifeCycleId(); + + // Compute stale ranges using new history length so stale SWA blocks get no pages. + TypedVec<LifeCycleId, HalfOpenRange<BlockOrdinal>> staleRanges(numLc); + TypedVec<LifeCycleId, SlotCount> numSlotsPerLc(numLc, 0); + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + // SSM slots are handled separately below — don't allocate block-level slots for SSM. + if (ssmLcId.has_value() && lc == *ssmLcId) + continue; + staleRanges[lc] = _getStaleRange(newHistoryLength, lcs.getLifeCycle(lc)); + auto [staleBeg, staleEnd] = staleRanges[lc]; + int numNewBlocks; + if (curNumBlocks < staleBeg) + { + TLLM_CHECK_DEBUG(newNumBlocks >= staleEnd); + numNewBlocks = (staleBeg - curNumBlocks) + (newNumBlocks - staleEnd); + } + else + { + numNewBlocks = newNumBlocks - std::max(staleEnd, curNumBlocks); + } + numSlotsPerLc[lc] = static_cast<SlotCount>(numNewBlocks) * mBeamWidth.value(); + } + + // SSM slots are now allocated lazily in resume() via deferred copy, not here. + + MigrationRecorder const migrationRecorder + = [this](std::vector<SharedPtr<Page>> const& pages, std::vector<Slot> const& slots, CacheLevel srcLevel, + CacheLevel dstLevel) { _recordMigratedSlots(pages, slots, srcLevel, dstLevel); }; + DropRecorder const dropRecorder = [this](std::vector<SharedPtr<Page>> const& pages, CacheLevel cacheLevel) + { _recordDroppedPages(pages, cacheLevel); }; + auto allSlots = mManager->storage().newGpuSlots(numSlotsPerLc, migrationRecorder, dropRecorder); + + // Assert that internal index buffer sizes match expected old_num_blocks (mirrors Python line ~463). + TLLM_CHECK_DEBUG(std::all_of(mBasePageIndices.begin(), mBasePageIndices.end(), + [curNumBlocks](auto const& beamIndices) + { + return std::all_of(beamIndices.begin(), beamIndices.end(), + [curNumBlocks](auto const& buf) + { + auto const* vec = std::get_if<std::vector<int>>(&buf); + return !vec || vec->size() == toSizeT(curNumBlocks); + }); + })); + + // Create SeqBlocks for the new ordinals. + TypedVec<LifeCycleId, size_t> slotCounters(numLc, 0); + _resizePageIndexBuffers(newNumBlocks); + for (BlockOrdinal ord = curNumBlocks; ord < newNumBlocks; ++ord) + { + SeqBlock sb; + sb.pages.resize(mBeamWidth); + for (auto& row : sb.pages) + row.resize(numLc); // default-constructs to monostate + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + // SSM pages live in mSsmBlocks, not in _blocks. + if (ssmLcId.has_value() && lc == *ssmLcId) + continue; + + auto [staleBeg, staleEnd] = staleRanges[lc]; + if (staleBeg <= ord && ord < staleEnd) + continue; // stale block for this lc — no page allocated + + size_t si = slotCounters[lc]++; + auto& slot = allSlots[lc][si]; + auto page = makeShared<UncommittedPage>(*this, ord, lc, kGpuLevel, kDefaultBeamIndex); + page->setSlot(slot); + sb.pages[kDefaultBeamIndex][lc] = page->lock(*this, kDefaultBeamIndex, ord, lc); + } + mBlocks.push_back(std::move(sb)); + } + // Assert all allocated slots were consumed (mirrors Python line ~488). + if (TLLM_UNLIKELY(gDebug)) + { + for (LifeCycleId lc{0}; lc < numLc; ++lc) + TLLM_CHECK(slotCounters[lc] == allSlots[lc].size()); + } +} + +void KvCache::_decreaseCapacity(BlockOrdinal newNumBlocks) +{ + while (mBlocks.size() > newNumBlocks) + { + auto& sb = mBlocks.back(); + for (auto& beamPages : sb.pages) + for (auto& bp : beamPages) + bp = std::monostate{}; + sb.treeBlock.reset(); + mBlocks.pop_back(); + } + _resizePageIndexBuffers(newNumBlocks); +} + +HalfOpenRange<BlockOrdinal> KvCache::_getStaleRange(int historyLength, LifeCycle const& lc) const +{ + return kv_cache_manager_v2::getStaleRange(lc, historyLength, mTokensPerBlock); +} + +std::vector<KvCache::StaleBackup> KvCache::_unlockStaleBlocks(int newHistoryLength) +{ + std::vector<StaleBackup> ret; + if (newHistoryLength == mHistoryLength) + return ret; + + auto scope = recordEventScope(); + auto const& lcs = mManager->lifeCycles(); + LifeCycleId numLc = mManager->storage().numLifeCycles(); + + for (LifeCycleId lcIdx{0}; lcIdx < numLc; ++lcIdx) + { + LifeCycle const& lc = lcs.getLifeCycle(lcIdx); + // SSM pages live in mSsmBlocks, not _blocks — skip. + if (std::holds_alternative<SsmLifeCycle>(lc)) + continue; + // Full-attention (no SWA) has empty stale range — skip. + auto const& alc = std::get<AttnLifeCycle>(lc); + if (!alc.windowSize.has_value()) + continue; + + auto oldRange = _getStaleRange(mHistoryLength, lc); + auto newRange = _getStaleRange(newHistoryLength, lc); + + BlockOrdinal unlockStart = std::max(oldRange.end, newRange.beg); + BlockOrdinal unlockEnd = std::min(mBlocks.size(), newRange.end); + + for (BlockOrdinal ord = unlockStart; ord < unlockEnd; ++ord) + { + auto& sb = mBlocks[ord]; + bool isCommitted = sb.isCommitted(); + bool holdForCommit + = !mManager->commitMinSnapshot() && !isCommitted && (mCommitState == CommitState::ALLOWED); + + for (BeamIndex bi{0}; bi < sb.pages.size(); ++bi) + { + auto& bp = sb.pages[bi][lcIdx]; + if (blockPageIsNull(bp)) + { + // No page to unlock: scratch block, commit_min_snapshot early + // release, or a stale block created unallocated by resize(). + continue; + } + TLLM_CHECK_DEBUG(std::holds_alternative<SharedPageLock>(bp)); + auto holder = blockPageGetPage(bp)->hold(); + ret.push_back({ord, bi, lcIdx, holder}); + bp = holdForCommit ? BlockPage{std::move(holder)} : BlockPage{std::monostate{}}; + } + } + } + return ret; +} + +void KvCache::_lockHeldBlocks(std::vector<StaleBackup> const& backup) +{ + std::vector<BatchedLockTarget> targets; + targets.reserve(backup.size()); + for (auto const& b : backup) + targets.push_back({b.holder->page, b.beamIdx, b.ordinal, b.lcId}); + + auto locks = batchedLockToGpu(*this, targets); + for (size_t i = 0; i < locks.size(); ++i) + { + auto const& t = backup[i]; + mBlocks[t.ordinal].pages[t.beamIdx][t.lcId] = std::move(locks[i]); + } +} + +// --------------------------------------------------------------------------- +// _takeUncommittedPage — extract uncommitted pages from a SeqBlock. +// Mirrors Python's _take_uncommitted_page(). +// --------------------------------------------------------------------------- + +TypedVec<LifeCycleId, KvCache::TakenPage> KvCache::_takeUncommittedPage( + SeqBlock& sb, BeamIndex beamIdx, std::optional<LifeCycleId> skipLc) +{ + LifeCycleId numLc = mManager->storage().numLifeCycles(); + TypedVec<LifeCycleId, TakenPage> result(numLc, TakenPage{nullptr, false}); + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + if (skipLc.has_value() && lc == *skipLc) + continue; + auto& bp = sb.pages[beamIdx][lc]; + if (auto* lock = std::get_if<SharedPageLock>(&bp)) + { + auto up = dynamicPointerCast<UncommittedPage>(lock->page()); + TLLM_CHECK_DEBUG_WITH_INFO(up, "page must be UncommittedPage"); + result[lc] = {up, true}; + } + else if (auto* holder = std::get_if<SharedPtr<PageHolder>>(&bp)) + { + TLLM_CHECK_DEBUG(*holder); + auto up = dynamicPointerCast<UncommittedPage>((*holder)->page); + TLLM_CHECK_DEBUG_WITH_INFO(up, "page must be UncommittedPage"); + result[lc] = {up, false}; + } + bp = std::monostate{}; + } + return result; +} + +// --------------------------------------------------------------------------- +// _commitBlock — shared logic for committing a single block. +// Mirrors Python's _commit_block(ordinal, is_last). +// Caller must have recordEventScope() open so finishEvent() works. +// On VIRTUAL_STOP or when isLast is true, transitions to USER_STOP and +// calls _onStopCommitting() — callers do not need post-call cleanup. +// --------------------------------------------------------------------------- + +void KvCache::_commitBlock(int ord, bool isLast, bool commitSsm, bool moveSsm) +{ + TLLM_CHECK_DEBUG(mCommitState == CommitState::ALLOWED); + TLLM_CHECK_DEBUG(ord == mNumCommittedBlocks); + + auto& sb = mBlocks.at(BlockOrdinal{ord}); + TLLM_CHECK_DEBUG_WITH_INFO(sb.pages.size() == BeamIndex{1}, "Must have 1 beam only"); + + // Build token block — always slice up to tokens_per_block; is_full tells us + // whether we got a full block's worth. Mirrors Python's: + // tokens = self._committed_tokens[start : start + tokens_per_block] + // is_full = len(tokens) == tokens_per_block + int start = ord * mTokensPerBlock; + int end = std::min(start + mTokensPerBlock, static_cast<int>(mCommittedTokens.size())); + std::vector<TokenIdExt> tokenBlock(mCommittedTokens.begin() + start, mCommittedTokens.begin() + end); + bool isFull = static_cast<int>(tokenBlock.size()) == mTokensPerBlock; + + if (!isLast && !isFull) + throw LogicError("Cannot commit block that is not full except last block"); + + // Prev node lookup (root or previous committed block). + RootBlock& root = mManager->radixTree().addOrGetExisting(mReuseScope); + LifeCycleId numLc = mManager->storage().numLifeCycles(); + + NodeBase* prevNode = &root; + if (ord > 0) + { + TLLM_CHECK_DEBUG_WITH_INFO(mBlocks[BlockOrdinal{ord - 1}].treeBlock, "prev block must be committed"); + prevNode = mBlocks[BlockOrdinal{ord - 1}].treeBlock.get(); + } + + // Try to find or create a block in the radix tree. + // Mirrors Python's try/except UselessBlockError pattern. + // TODO: Replace with if-condition once Python is removed and C++ is the primary codebase. + bool blockIsNew = false; + SharedPtr<Block> newBlock; + try + { + newBlock = addOrGetExistingBlock(prevNode, numLc, tokenBlock, &blockIsNew); + } + catch (UselessBlockError const& e) + { + newBlock = e.block; + blockIsNew = false; + } + TLLM_CHECK_DEBUG(newBlock); + TLLM_CHECK_DEBUG(newBlock->tokensPerBlock() == mTokensPerBlock); + // In reuse case, verify token match (mirrors Python: tree_block.tokens[:num_tokens] == tokens). + TLLM_CHECK_DEBUG(blockIsNew || std::equal(tokenBlock.begin(), tokenBlock.end(), newBlock->tokens.begin())); + + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + bool didCommit = false; + + if (blockIsNew) + { + // New block: take uncommitted pages, convert to committed. + // Mirrors Python's _take_uncommitted_page + convert path. + auto taken = _takeUncommittedPage(sb, kDefaultBeamIndex, ssmLcId); + sb.treeBlock = newBlock; + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + auto& [up, locked] = taken[lc]; + if (!up) + continue; + auto committed = up->convertToCommitted(newBlock, finishEvent()); + if (locked) + sb.pages[kDefaultBeamIndex][lc] + = committed->lock(*this, kDefaultBeamIndex, static_cast<BlockOrdinal>(ord), lc); + else + sb.pages[kDefaultBeamIndex][lc] = committed->hold(); + } + TLLM_CHECK_DEBUG(_getTreeBlock(static_cast<BlockOrdinal>(ord)) == newBlock); + ++mNumCommittedBlocks; + if (newBlock->eventSink) + { + newBlock->eventSink->addStoredBlock(*newBlock); + } + didCommit = true; + } + else if (newBlock->isFull() && mManager->allowSeqRebasing() && isFull) + { + // Existing block: rebase — reuse existing block's committed pages. + // Mirrors Python's `elif tree_block.is_full and allow_seq_rebasing and is_full` path. + std::vector<BatchedLockTarget> reuseTasks; + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + if (ssmLcId.has_value() && lc == *ssmLcId) + continue; + auto& bp = sb.pages[kDefaultBeamIndex][lc]; + if (blockPageIsNull(bp)) + continue; + auto* existingPage = newBlock->storage.at(lc); + bool isLocked = std::holds_alternative<SharedPageLock>(bp); + if (existingPage == nullptr) + { + // Existing page gone — put our uncommitted page into the tree block. + if (auto* lock = std::get_if<SharedPageLock>(&bp)) + { + auto up = dynamicPointerCast<UncommittedPage>(lock->page()); + if (up) + { + bp = std::monostate{}; + auto committed = up->convertToCommitted(newBlock, finishEvent()); + if (newBlock->eventSink) + { + newBlock->eventSink->addStoredLifeCycle(*newBlock, lc); + } + bp = isLocked + ? BlockPage{committed->lock(*this, kDefaultBeamIndex, static_cast<BlockOrdinal>(ord), lc)} + : BlockPage{committed->hold()}; + } + } + else if (auto* holder = std::get_if<SharedPtr<PageHolder>>(&bp)) + { + if (*holder) + { + auto up = dynamicPointerCast<UncommittedPage>((*holder)->page); + if (up) + { + bp = std::monostate{}; + auto committed = up->convertToCommitted(newBlock, finishEvent()); + if (newBlock->eventSink) + { + newBlock->eventSink->addStoredLifeCycle(*newBlock, lc); + } + bp = committed->hold(); + } + } + } + } + else + { + // Downgrade lock to holder for our page; reuse the existing page. + if (isLocked) + { + auto holder = blockPageGetPage(bp)->hold(); + bp = std::move(holder); + } + reuseTasks.push_back( + {existingPage->sharedFromThis(), kDefaultBeamIndex, static_cast<BlockOrdinal>(ord), lc}); + } + } + if (!reuseTasks.empty()) + { + auto locks = batchedLockToGpu(*this, reuseTasks); + for (size_t ri = 0; ri < reuseTasks.size(); ++ri) + { + LifeCycleId lc = reuseTasks[ri].lifeCycle; + sb.pages[kDefaultBeamIndex][lc] = std::move(locks[ri]); + } + } + // Don't clear SSM storage on rebase — the existing block may have a valid snapshot. + sb.treeBlock = newBlock; + TLLM_CHECK_DEBUG(_getTreeBlock(static_cast<BlockOrdinal>(ord)) == newBlock); + ++mNumCommittedBlocks; + didCommit = true; + } + else + { + // Can't commit and can't reuse existing block. Just stop committing. + mCommitState = CommitState::VIRTUAL_STOP; + } + + if (didCommit && commitSsm) + { + TLLM_CHECK_DEBUG(ssmLcId.has_value()); + _snapshotSsmToTreeBlock(newBlock, *ssmLcId, start + static_cast<int>(tokenBlock.size()), moveSsm); + } + + if (sb.isCommitted()) + { + auto const& lifeCycles = mManager->lifeCycles(); + for (LifeCycleId lcIdx{0}; lcIdx < numLc; ++lcIdx) + { + if (ssmLcId.has_value() && lcIdx == *ssmLcId) + { + continue; + } + LifeCycle const& lc = lifeCycles.getLifeCycle(lcIdx); + if (!std::holds_alternative<AttnLifeCycle>(lc)) + { + continue; + } + auto const staleRange = _getStaleRange(mHistoryLength, lc); + if (staleRange.contains(BlockOrdinal{ord})) + { + for (auto& beamBlock : sb.pages) + { + beamBlock[lcIdx] = std::monostate{}; + } + } + } + } + + // Mirrors Python's tail of _commit_block: + // if is_last or self._commit_state == self.CommitState.VIRTUAL_STOP: + // self._commit_state = self.CommitState.USER_STOP + // self._on_stop_committing() + if (isLast || mCommitState == CommitState::VIRTUAL_STOP) + { + mCommitState = CommitState::USER_STOP; + _onStopCommitting(); + } +} + +// --------------------------------------------------------------------------- +// commit +// --------------------------------------------------------------------------- + +void KvCache::commit(std::vector<TokenIdExt> const& tokens, bool isEnd) +{ + TLLM_CHECK_DEBUG(mStatus == Status::ACTIVE); + if (mBeamWidth != BeamIndex{1}) + throw LogicError("Not implemented yet for beam search"); + if (tokens.empty()) + { + if (isEnd) + stopCommitting(); + return; + } + if (mCommitState == CommitState::USER_STOP) + throw LogicError("Cannot commit tokens after stop_committing()"); + + bool const commitMinSnapshot = mManager->commitMinSnapshot(); + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + if (commitMinSnapshot) + { + int const newNumCommitted = static_cast<int>(mCommittedTokens.size()) + static_cast<int>(tokens.size()); + if (mHistoryLength != static_cast<int>(mCommittedTokens.size()) && mHistoryLength != newNumCommitted) + { + throw AssertionError("commit_min_snapshot requires commit() to start or end at history_length"); + } + } + + // Append tokens to committed list. + mCommittedTokens.insert(mCommittedTokens.end(), tokens.begin(), tokens.end()); + if (mCommitState == CommitState::VIRTUAL_STOP) + { + if (isEnd) + mCommitState = CommitState::USER_STOP; + return; + } + + // Bump history_length to cover newly committed tokens (mirrors Python — done + // BEFORE the commit loop so stale-range computation sees the new history). + int const numCommitted = static_cast<int>(mCommittedTokens.size()); + if (mHistoryLength < numCommitted) + setHistoryLength(numCommitted); + + int const numCommittedBlocksBefore = mNumCommittedBlocks; + int const newNumFullBlocks = numCommitted / mTokensPerBlock; + bool const hasPartialSnapshot = commitMinSnapshot && (numCommitted % mTokensPerBlock != 0) && numCommitted > 0; + bool const hasNewFullBlocks = newNumFullBlocks > numCommittedBlocksBefore; + if (hasNewFullBlocks || hasPartialSnapshot) + { + // Block whose end is the last committed token — where the SSM snapshot lives. + int const ssmSnapshotOrdinal = (numCommitted - 1) / mTokensPerBlock; + // Wrapped in recordEventScope() so SharedPageLock::unlock() shares one finish + // event (mirrors Python's `with self._record_event()`). + auto scope = recordEventScope(); + for (int ordinal = numCommittedBlocksBefore; ordinal < newNumFullBlocks; ++ordinal) + { + bool const commitSsm = commitMinSnapshot && ssmLcId.has_value() && ordinal == ssmSnapshotOrdinal; + _commitBlock(ordinal, /*isLast=*/false, commitSsm, /*moveSsm=*/isEnd); + // _commitBlock transitions to USER_STOP on VIRTUAL_STOP internally. + if (mCommitState != CommitState::ALLOWED) + break; + } + if (hasPartialSnapshot && mCommitState == CommitState::ALLOWED) + { + BlockOrdinal const partialOrdinal{newNumFullBlocks}; + if (isEnd) + { + _commitBlock(newNumFullBlocks, /*isLast=*/true, /*commitSsm=*/ssmLcId.has_value(), + /*moveSsm=*/ssmLcId.has_value()); + } + else + { + _snapshotPartialBlockToTree(partialOrdinal, /*commitSsm=*/ssmLcId.has_value()); + } + } + } + + if (isEnd && mCommitState != CommitState::USER_STOP) + stopCommitting(); +} + +void KvCache::stopCommitting() +{ + TLLM_CHECK_DEBUG(mStatus != Status::CLOSED); + if (mCommitState == CommitState::USER_STOP) + return; + TLLM_CHECK_DEBUG(_checkSanity()); + + // Mirrors Python's stop_committing() which calls _commit_block(ordinal, True). + if (mCommitState == CommitState::VIRTUAL_STOP) + { + mCommitState = CommitState::USER_STOP; + return; + } + + TLLM_CHECK_DEBUG(mCommitState == CommitState::ALLOWED); + + int tokensLeft = static_cast<int>(mCommittedTokens.size()) - mNumCommittedBlocks * mTokensPerBlock; + if (tokensLeft > 0) + { + TLLM_CHECK_DEBUG(BlockOrdinal{mNumCommittedBlocks} < mBlocks.size()); + auto scope = recordEventScope(); + // isLast=true: _commitBlock handles USER_STOP + _onStopCommitting() internally. + _commitBlock(mNumCommittedBlocks, /*isLast=*/true); + } + else + { + mCommitState = CommitState::USER_STOP; + _onStopCommitting(); + } + TLLM_CHECK_DEBUG(mCommitState == CommitState::USER_STOP); +} + +// --------------------------------------------------------------------------- +// PlannedDropHandle — mirrors Python's PlannedDropHandle. +// --------------------------------------------------------------------------- + +PlannedDropHandle::PlannedDropHandle(std::vector<CommittedPage*> const& pages) +{ + // Deduplicate by identity (mirrors Python's {id(page): page} dict). + std::vector<CommittedPage*> unique; + std::unordered_set<CommittedPage*> seen; + unique.reserve(pages.size()); + for (auto* page : pages) + { + if (seen.insert(page).second) + unique.push_back(page); + } + + std::vector<WeakPtr<CommittedPage>> refs; + refs.reserve(unique.size()); + for (auto* page : unique) + { + refs.emplace_back(dynamicPointerCast<CommittedPage>(page->sharedFromThis())); + page->plannedDropCount += 1; + } + mPageRefs = std::move(refs); +} + +void PlannedDropHandle::drop() +{ + if (!mPageRefs.has_value()) + throw std::invalid_argument("Planned drop handle has already been dropped"); + + std::vector<SharedPtr<CommittedPage>> pages; + for (auto const& ref : *mPageRefs) + { + auto page = ref.lock(); + if (page) + { + if (page->plannedDropCount <= 0) + throw std::invalid_argument("Committed page has no planned drop"); + pages.push_back(std::move(page)); + } + } + + mPageRefs.reset(); + for (auto const& page : pages) + { + page->plannedDropCount -= 1; + if (page->plannedDropCount == 0 && page->status() == PageStatus::DROPPABLE && page->scheduledForEviction()) + { + page->manager->excludeFromEviction(*page); + } + } +} + +PlannedDropHandle::~PlannedDropHandle() +{ + if (mPageRefs.has_value()) + { + // Mirror Python's __del__: apply the plan if not already dropped. + // Destructors must not throw; swallow any error. + try + { + drop(); + } + catch (...) + { + } + } +} + +std::unique_ptr<PlannedDropHandle> KvCache::planCommittedBlockDrop() +{ + if (mCommitState != CommitState::USER_STOP) + throw LogicError("plan_committed_block_drop() requires stop_committing()"); + + // A cache with no committed tokens has no preceding turn to drop. + if (numCommittedTokens() == 0) + return nullptr; + + auto const match = mManager->matchReuse(mReuseScope, mCommittedTokens); + if (match.numTokens != numCommittedTokens() || match.blocks.empty()) + return nullptr; + + BlockOrdinal const end = match.blocks.size(); + std::vector<CommittedPage*> pagesToDrop; + for (auto const item : mManager->lifeCycles()) + { + LifeCycleId const lcIdx = item.id; + LifeCycle const& lc = item.lc; + BlockOrdinal windowStart; + if (auto const* attn = std::get_if<AttnLifeCycle>(&lc)) + { + // Full-attention blocks may still be needed by later turns. + if (!attn->windowSize.has_value()) + continue; + auto const staleRange = _getStaleRange(numCommittedTokens(), lc); + windowStart = std::min(staleRange.end, end); + } + else + { + // SSM: only the last committed block carries the reusable snapshot. + windowStart = BlockOrdinal{end.value() - 1}; + } + for (BlockOrdinal ordinal = windowStart; ordinal < end; ++ordinal) + { + CommittedPage* page = match.blocks[ordinal]->getPage(lcIdx); + if (page == nullptr) + return nullptr; + pagesToDrop.push_back(page); + } + } + return std::make_unique<PlannedDropHandle>(pagesToDrop); +} + +// --------------------------------------------------------------------------- +// _onStopCommitting: release stale held uncommitted pages for SWA layers. +// Mirrors Python's _on_stop_committing(). +// --------------------------------------------------------------------------- + +void KvCache::_onStopCommitting() +{ + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + auto const& lcs = mManager->lifeCycles(); + + for (auto [lcIdx, lc] : lcs) + { + if (ssmLcId.has_value() && lcIdx == *ssmLcId) + continue; // SSM pages live in _ssm_blocks, not in _blocks + + auto staleRange = _getStaleRange(mHistoryLength, lc); + BlockOrdinal start = std::max(staleRange.beg, BlockOrdinal{mNumCommittedBlocks}); + BlockOrdinal end = staleRange.end; + + TLLM_CHECK_DEBUG(end <= mBlocks.size()); + for (BlockOrdinal ord = start; ord < end; ++ord) + { + auto& sb = mBlocks[ord]; + TLLM_CHECK_DEBUG(!sb.isCommitted()); + for (auto& beamPages : sb.pages) + { + auto& bp = beamPages[lcIdx]; + if (blockPageIsNull(bp)) + { + // Nothing to release: scratch block, commit_min_snapshot early + // release, or a stale block created unallocated by resize(). + continue; + } + TLLM_CHECK_DEBUG(std::holds_alternative<SharedPtr<PageHolder>>(bp)); + bp = std::monostate{}; + } + } + } + TLLM_CHECK_DEBUG(_checkSanity()); +} + +// --------------------------------------------------------------------------- +// _setupForReuse: find existing blocks in radix tree matching input tokens. +// --------------------------------------------------------------------------- + +void KvCache::_setupForReuse(BlockRadixTree::ReuseMatch const& match) +{ + auto const& matched = match.blocks; + int const numTokens = match.numTokens; + + auto& lifeCycles = mManager->lifeCycles(); + auto const& allLc = lifeCycles.getAll(); + LifeCycleId numLc = lifeCycles.size(); + auto ssmLcId = lifeCycles.ssmLifeCycleId(); + BlockOrdinal const fullReusedEnd{numTokens / mTokensPerBlock}; + bool const hasPartialMatch = numTokens % mTokensPerBlock != 0; + bool const shouldRecordStats = _shouldRecordStats(); + + // --- Build blocks with stale range handling --- + + BlockOrdinal const numMatchedBlocks = matched.size(); + _resizePageIndexBuffers(numMatchedBlocks); + + for (BlockOrdinal i{0}; i < numMatchedBlocks; ++i) + { + SeqBlock sb; + sb.treeBlock = matched[i]->sharedFromThis(); + sb.pages.resize(BeamIndex{1}); + sb.pages[kDefaultBeamIndex].resize(numLc); + mBlocks.push_back(std::move(sb)); + } + + BeamIndex beamIdx = kDefaultBeamIndex; + + for (LifeCycleId lcId{0}; lcId < numLc; ++lcId) + { + // SSM is handled separately below. + if (ssmLcId.has_value() && lcId == *ssmLcId) + continue; + + auto staleRange = getStaleRange(allLc[lcId], numTokens, mTokensPerBlock); + BlockOrdinal staleStart = staleRange.beg; + BlockOrdinal staleEnd = staleRange.end; + bool const isAttention = std::holds_alternative<AttnLifeCycle>(allLc[lcId]); + int fullReusedBlocks = 0; + int partialReusedBlocks = 0; + + // Process a non-stale ordinal: hold the page. + // For partial blocks (last block, not full), defer the copy to first resume(). + auto processOrdinal = [&](BlockOrdinal ordinal) + { + auto& blk = *matched.at(ordinal); + auto* page = blk.storage.at(lcId); + TLLM_CHECK_DEBUG_WITH_INFO(page, "Expected page in non-stale block"); + auto& bpSlot = mBlocks[ordinal].pages[beamIdx][lcId]; + bpSlot = page->hold(); + if (shouldRecordStats && isAttention) + { + if (ordinal < fullReusedEnd) + { + ++fullReusedBlocks; + } + else if (hasPartialMatch && ordinal == fullReusedEnd && _hasReuseSource(bpSlot)) + { + partialReusedBlocks = 1; + } + } + }; + + for (BlockOrdinal ord{0}; ord < staleStart; ++ord) + processOrdinal(ord); + for (BlockOrdinal ord = staleEnd; ord < numMatchedBlocks; ++ord) + processOrdinal(ord); + + if (shouldRecordStats && isAttention && mPendingStats.recordReuse(lcId, fullReusedBlocks, partialReusedBlocks)) + { + mManager->markStatsDirty(id); + } + } + + // SSM reuse: hold the snapshot from the last matched block. Copy is deferred to first resume(). + if (ssmLcId.has_value() && !matched.empty()) + { + auto& snapshotBlock = *matched.back(); + auto* snapshotPage = snapshotBlock.storage[*ssmLcId]; + TLLM_CHECK_DEBUG_WITH_INFO(snapshotPage, "Last matched block must have SSM snapshot after truncation"); + mSsmBlocks[kDefaultBeamIndex][*ssmLcId] = snapshotPage->hold(); + } + // Record one SSM snapshot lookup for this reuse-match onboarding (a miss when + // nothing matched). Mirrors Python's record_ssm_snapshot_lookup call. + if (shouldRecordStats && ssmLcId.has_value()) + { + if (mPendingStats.recordSsmSnapshotLookup(*ssmLcId, match.numLookupTokens, numTokens, mTokensPerBlock)) + { + mManager->markStatsDirty(id); + } + } + + // Append matched tokens (reconstructed from the matched blocks). + mCommittedTokens = _getMatchedTokens(match); + mNumCommittedBlocks = numTokens / mTokensPerBlock; + mHistoryLength = numTokens; + mCapacity = numTokens; +} + +// --------------------------------------------------------------------------- +// _getMatchedTokens — reconstruct the committed token sequence from a match. +// Mirrors Python's _get_matched_tokens(). +// --------------------------------------------------------------------------- + +std::vector<TokenIdExt> KvCache::_getMatchedTokens(BlockRadixTree::ReuseMatch const& match) const +{ + std::vector<TokenIdExt> ret; + ret.reserve(static_cast<size_t>(match.numTokens)); + int remaining = match.numTokens; + for (auto const* block : match.blocks) + { + TLLM_CHECK_DEBUG(remaining > 0); + int const numBlockTokens = std::min(remaining, static_cast<int>(block->tokens.size())); + ret.insert(ret.end(), block->tokens.begin(), block->tokens.begin() + numBlockTokens); + remaining -= numBlockTokens; + } + TLLM_CHECK_DEBUG(remaining == 0); + return ret; +} + +// --------------------------------------------------------------------------- +// _getTreeBlock — get and validate the tree block at a committed ordinal. +// Mirrors Python's _get_tree_block(). +// --------------------------------------------------------------------------- + +SharedPtr<Block> const& KvCache::_getTreeBlock(BlockOrdinal ordinal) const +{ + TLLM_CHECK_DEBUG(mBlocks[ordinal].isCommitted()); + auto const& ret = mBlocks[ordinal].treeBlock; + TLLM_CHECK_DEBUG(ret); + if (TLLM_UNLIKELY(gDebug)) + { + auto ssmLcId = mManager->lifeCycles().ssmLifeCycleId(); + auto const& beamBlock = mBlocks[ordinal].pages[kDefaultBeamIndex]; + for (LifeCycleId lcId{0}; lcId < beamBlock.size(); ++lcId) + { + if (ssmLcId.has_value() && lcId == *ssmLcId) + { + TLLM_CHECK_WITH_INFO(blockPageIsNull(beamBlock[lcId]), "SSM pages live in mSsmBlocks"); + } + else if (!blockPageIsNull(beamBlock[lcId])) + { + auto page = blockPageGetPage(beamBlock[lcId]); + auto committed = dynamicPointerCast<CommittedPage>(page); + TLLM_CHECK(committed && committed->block == ret.get()); + } + } + } + return ret; +} + +// --------------------------------------------------------------------------- +// _checkSanity — comprehensive invariant check. +// Mirrors Python's _check_sanity(). +// --------------------------------------------------------------------------- + +bool KvCache::_checkSanity() const +{ + if (mStatus == Status::CLOSED) + return numBlocks() == BlockOrdinal{0}; + + TLLM_CHECK_DEBUG(numCommittedTokens() <= mHistoryLength && mHistoryLength <= mCapacity); + TLLM_CHECK_DEBUG(numBlocks() == BlockOrdinal{divUp(mCapacity, mTokensPerBlock)}); + + auto const& lcs = mManager->lifeCycles(); + LifeCycleId numLc = mManager->storage().numLifeCycles(); + auto ssmLcId = lcs.ssmLifeCycleId(); + + // Precompute stale and scratch ranges for each lifecycle. + TypedVec<LifeCycleId, HalfOpenRange<BlockOrdinal>> staleRanges(numLc); + TypedVec<LifeCycleId, HalfOpenRange<BlockOrdinal>> scratchRangesVec(numLc); + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + auto const& lifecycle = lcs.getLifeCycle(lc); + staleRanges[lc] = _getStaleRange(mHistoryLength, lifecycle); + scratchRangesVec[lc] = _getScratchRange(lifecycle); + } + + for (BlockOrdinal ordinal{0}; ordinal < numBlocks(); ++ordinal) + { + auto const& block = mBlocks[ordinal]; + bool isCommitted = mNeverResumed || ordinal < BlockOrdinal{mNumCommittedBlocks}; + TLLM_CHECK_DEBUG(isCommitted == block.isCommitted()); + + for (auto const& beamBlock : block.pages) + { + TLLM_CHECK_DEBUG(beamBlock.size() == numLc); + for (LifeCycleId lc{0}; lc < numLc; ++lc) + { + auto const& bp = beamBlock[lc]; + if (ssmLcId.has_value() && lc == *ssmLcId) + { + // SSM pages live in mSsmBlocks, not in mBlocks. + // When mNeverResumed and SSM snapshot is held, the block is committed + // but SSM page entry remains null (SSM is in mSsmBlocks). + TLLM_CHECK_DEBUG(blockPageIsNull(bp)); + continue; + } + + auto const& staleRange = staleRanges[lc]; + auto const& scratchRange = scratchRangesVec[lc]; + + if (scratchRange.contains(ordinal)) + { + // Scratch blocks have no per-block pages. + TLLM_CHECK_DEBUG(blockPageIsNull(bp)); + } + else if (staleRange.beg <= ordinal && ordinal < staleRange.end) + { + if (isCommitted || mCommitState != CommitState::ALLOWED) + { + TLLM_CHECK_DEBUG(blockPageIsNull(bp)); + } + else + { + // For the decoder-side disagg case, for the first step, we will skip the + // out-of-window blocks. + TLLM_CHECK_DEBUG(std::holds_alternative<SharedPtr<PageHolder>>(bp) + || (blockPageIsNull(bp) && (mCommittedTokens.empty() || mManager->commitMinSnapshot()))); + } + } + else + { + if (mStatus == Status::ACTIVE) + TLLM_CHECK_DEBUG(std::holds_alternative<SharedPageLock>(bp)); + else + TLLM_CHECK_DEBUG(std::holds_alternative<SharedPtr<PageHolder>>(bp)); + } + + if (!blockPageIsNull(bp)) + { + auto page = blockPageGetPage(bp); + TLLM_CHECK_DEBUG(isCommitted == (dynamicPointerCast<CommittedPage>(page) != nullptr)); + } + } + } + } + + // Check SSM blocks (mirrors Python lines 1342-1353). + if (ssmLcId.has_value()) + { + for (BeamIndex bi{0}; bi < mBeamWidth; ++bi) + { + auto const& bp = mSsmBlocks[bi][*ssmLcId]; + if (!blockPageIsNull(bp)) + { + if (mNeverResumed) + { + // Deferred copy: SSM holds CommittedPage from matched snapshot. + TLLM_CHECK_DEBUG(std::holds_alternative<SharedPtr<PageHolder>>(bp)); + auto page = blockPageGetPage(bp); + TLLM_CHECK_DEBUG(dynamicPointerCast<CommittedPage>(page) != nullptr); + } + else + { + TLLM_CHECK_DEBUG(std::holds_alternative<SharedPageLock>(bp)); + auto page = blockPageGetPage(bp); + TLLM_CHECK_DEBUG(dynamicPointerCast<UncommittedPage>(page) != nullptr); + } + } + } + } + + return true; +} + +// --------------------------------------------------------------------------- +// Page index tables +// --------------------------------------------------------------------------- + +void KvCache::_checkPageIndexBufferCapacity(BlockOrdinal newNumBlocks) const +{ + for (auto const& beamIndices : mBasePageIndices) + { + for (auto const& buf : beamIndices) + { + if (auto const* ext = std::get_if<Span<int>>(&buf)) + { + if (ext->len < newNumBlocks.value()) + { + throw std::invalid_argument("User-provided base page indices is too short"); + } + } + } + } +} + +void KvCache::_resizePageIndexBuffers(BlockOrdinal newNumBlocks) +{ + for (BeamIndex bi{0}; bi < mBeamWidth; ++bi) + { + for (LifeCycleId lcId{0}; lcId < mBasePageIndices[bi].size(); ++lcId) + { + auto& buf = mBasePageIndices[bi][lcId]; + if (auto* vec = std::get_if<std::vector<int>>(&buf)) + { + // When shrinking, assert tail entries are already BAD (mirrors Python line ~432). + auto const newSize = toSizeT(newNumBlocks); + TLLM_CHECK_DEBUG(newSize >= vec->size() + || std::all_of(vec->begin() + static_cast<ptrdiff_t>(newSize), vec->end(), + [](int idx) { return idx == kBadPageIndex.value(); })); + // Growing fills new entries with kBadPageIndex; shrinking truncates. + vec->resize(newSize, kBadPageIndex.value()); + } + else + { + // Span<int>: caller-provided buffer must be large enough, + // and tail beyond active blocks must already be BAD + // (lock destructors set indices via updateBasePageIndex). + auto& ext = std::get<Span<int>>(buf); + int const newLen = newNumBlocks.value(); + if (ext.len < newLen) + { + throw std::invalid_argument("User-provided base page indices is too short"); + } + for (int i = newLen; i < ext.len; ++i) + TLLM_CHECK_DEBUG(ext[i] == kBadPageIndex.value()); + } + } + } +} + +int KvCache::updateBasePageIndex(BeamIndex bi, BlockOrdinal ord, LifeCycleId lc, int value) +{ + if (ord == kBadBlockOrdinal) + return kBadPageIndex.value(); // SSM pages use BAD_BLOCK_ORDINAL + auto& buf = mBasePageIndices[bi][lc]; + return std::visit( + [&](auto& b) -> int + { + using T = std::decay_t<decltype(b)>; + if constexpr (std::is_same_v<T, std::vector<int>>) + { + TLLM_CHECK_DEBUG(b.size() > toSizeT(ord)); + int old = b[toSizeT(ord)]; + b[toSizeT(ord)] = value; + return old; + } + else + { + TLLM_CHECK_DEBUG(ord < b.len); + int old = b[ord.value()]; + b[ord.value()] = value; + return old; + } + }, + buf); +} + +Span<int const> KvCache::getBasePageIndices(LayerGroupId lgId, BeamIndex beamIdx) const +{ + auto const& buf = mBasePageIndices.at(beamIdx).at(lgId); + auto result = std::visit( + [](auto const& b) -> Span<int const> { + return {b.data(), static_cast<int32_t>(b.size())}; + }, + buf); + // Cross-validate cached indices against freshly computed reference (mirrors Python lines ~350-354). + if (TLLM_UNLIKELY(gDebug) && isActive()) + { + auto ref = getAggregatedPageIndices(lgId, beamIdx); + auto len = static_cast<size_t>(std::min(result.len, static_cast<int32_t>(ref.size()))); + TLLM_CHECK(std::equal(result.data(), result.data() + len, ref.begin())); + } + return result; +} + +std::vector<int> KvCache::getAggregatedPageIndices(LayerGroupId lgId, BeamIndex beamIdx, bool validOnly) const +{ + std::vector<int> result; + result.reserve(mBlocks.stdSize()); + for (auto const& sb : mBlocks) + { + auto const& pg = blockPageGetPage(sb.pages[beamIdx][lgId]); + if (!pg) + { + if (!validOnly) + result.push_back(kBadPageIndex.value()); + } + else + { + result.push_back(slotIdToPageIndexValue(pg->slotId())); + } + } + return result; +} + +void KvCache::setBasePageIndexBuf(BeamIndex beamIdx, LayerGroupId lgId, int32_t* buf, int len) +{ + auto& slot = mBasePageIndices[beamIdx][lgId]; + BlockOrdinal const numBlocks = mBlocks.size(); + + if (!buf || len == 0) + { + // Revert to internal vector: copy current data out of the old buffer. + auto const& old = slot; + if (auto const* ext = std::get_if<Span<int>>(&old)) + { + auto const n = std::min(toSizeT(numBlocks), static_cast<size_t>(ext->len)); + std::vector<int> vec(ext->ptr, ext->ptr + n); + slot = std::move(vec); + } + // If already a vector, nothing to do. + return; + } + + if (len < numBlocks.value()) + { + throw std::invalid_argument("setBasePageIndexBuf: buffer length must be >= num_blocks"); + } + + // Copy current indices into the external buffer (mirrors Python: buf[:length] = old_indices[:length]). + int const* oldData = nullptr; + int oldLen = 0; + if (auto* vec = std::get_if<std::vector<int>>(&slot)) + { + oldData = vec->data(); + oldLen = static_cast<int>(vec->size()); + } + else + { + auto& span = std::get<Span<int>>(slot); + oldData = span.data(); + oldLen = span.len; + } + auto const copyLen = std::min(static_cast<size_t>(oldLen), toSizeT(numBlocks)); + std::copy(oldData, oldData + copyLen, buf); + std::fill(buf + copyLen, buf + len, kBadPageIndex.value()); + slot = Span<int>{buf, len}; +} + +int KvCache::getSsmBlockBaseIndex(LayerGroupId lgId, BeamIndex beamIdx) const +{ + auto const& bp = mSsmBlocks.at(beamIdx).at(lgId); + if (blockPageIsNull(bp)) + return kBadPageIndex.value(); + TLLM_CHECK_DEBUG(std::holds_alternative<SharedPageLock>(bp)); + auto const& pg = blockPageGetPage(bp); + TLLM_CHECK_DEBUG_WITH_INFO(pg, "SSM block must have a valid page"); + return slotIdToPageIndexValue(pg->slotId()); // asserts valid slot +} + +// --------------------------------------------------------------------------- +// SWA scratch slot methods +// --------------------------------------------------------------------------- + +bool KvCache::hasScratchSlots() const +{ + return std::any_of(mScratchSlots.begin(), mScratchSlots.end(), [](auto const& v) { return !v.empty(); }); +} + +bool KvCache::isSwaScratchReuseEnabled() const noexcept +{ + return mEnableSwaScratchReuse; +} + +bool KvCache::supportsIndexMode(PageIndexMode mode) const +{ + switch (mode) + { + case PageIndexMode::PER_LAYER: return true; + case PageIndexMode::SHARED: return !hasScratchSlots(); + } + return false; +} + +HalfOpenRange<BlockOrdinal> KvCache::_getScratchRange( + LifeCycle const& lc, std::optional<int> hlOverride, std::optional<int> capOverride) const +{ + if (!mEnableSwaScratchReuse) + return {0, 0}; + int hist = hlOverride.value_or(mHistoryLength); + int cap = capOverride.value_or(mCapacity); + return computeScratchRange(lc, hist, cap, mTokensPerBlock, _swaScratchMaxRewindLen()); +} + +bool KvCache::_wouldUseSwaScratchBlocks() const +{ + int const maxRewindLen = _swaScratchMaxRewindLen(); + for (auto const& [lcId, lc] : mManager->lifeCycles()) + { + if (computeScratchRange(lc, mHistoryLength, mCapacity, mTokensPerBlock, maxRewindLen)) + return true; + } + return false; +} + +int KvCache::_swaScratchMaxRewindLen() const +{ + auto const& cfg = mManager->config().swaScratchReuse; + TLLM_CHECK_DEBUG(cfg.has_value()); + return cfg->maxRewindLen; +} + +std::optional<ScratchDesc> KvCache::getScratchDesc(LayerGroupId lgId) const +{ + auto const& lc = mManager->lifeCycles().getLifeCycle(lgId); + auto sr = _getScratchRange(lc); + if (!sr) + return std::nullopt; + std::vector<int> slotIds; + slotIds.reserve(mScratchSlots[lgId].size()); + for (auto const& lock : mScratchSlots[lgId]) + slotIds.push_back(slotIdToPageIndexValue(lock.slot().slotId())); + return ScratchDesc{sr, std::move(slotIds)}; +} + +void KvCache::setEnableSwaScratchReuse(bool enable) +{ + if (enable == mEnableSwaScratchReuse) + return; + if (enable) + { + if (!mManager->isSwaScratchReuseEnabled()) + throw std::invalid_argument( + "Cannot enable SWA scratch reuse for a request when it is disabled in KV cache manager config"); + if (_wouldUseSwaScratchBlocks()) + throw std::invalid_argument( + "Cannot enable SWA scratch reuse while the current request state would need scratch blocks"); + mEnableSwaScratchReuse = true; + return; + } + if (_wouldUseSwaScratchBlocks()) + throw std::invalid_argument("Cannot disable SWA scratch reuse while scratch blocks are needed"); + TLLM_CHECK_DEBUG(!hasScratchSlots()); + mEnableSwaScratchReuse = false; +} + +KvCache::DeltaScratchSlots KvCache::_takeExcessScratchSlots(int capacity, int historyLength) +{ + LifeCycleId numLc = mManager->storage().numLifeCycles(); + DeltaScratchSlots result; + result.excess.resize(numLc); + result.deltaCnt.resize(numLc, 0); + result.scratchRanges.resize(numLc); + + for (auto const& [lcIdx, lc] : mManager->lifeCycles()) + { + auto scratchRange = _getScratchRange(lc, historyLength, capacity); + result.scratchRanges[lcIdx] = scratchRange; + int numScratchBlocks = scratchRange.length(); + auto const& fracMax = mManager->storage().slotUtilFracMax(lcIdx); + int neededSlots = fracMax.ceilMul(numScratchBlocks); + int existingSlots = static_cast<int>(mScratchSlots[lcIdx].size()); + int delta = neededSlots - existingSlots; + result.deltaCnt[lcIdx] = delta; + + if (delta < 0) + { + for (int i = 0; i < -delta; ++i) + { + result.excess[lcIdx].push_back(std::move(mScratchSlots[lcIdx].back())); + mScratchSlots[lcIdx].pop_back(); + } + } + } + return result; +} + +void KvCache::_recoverExcessScratchSlots(TypedVec<LifeCycleId, std::vector<ScratchSlotLock>>& excess) +{ + for (LifeCycleId lcId{0}; lcId < excess.size(); ++lcId) + { + for (auto& lock : excess[lcId]) + { + mScratchSlots[lcId].push_back(std::move(lock)); + } + excess[lcId].clear(); + } +} + +void KvCache::_freeScratchSlots() +{ + for (auto& lcSlots : mScratchSlots) + { + for (auto& lock : lcSlots) + lock.unlock(); + lcSlots.clear(); + } +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCache.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCache.h new file mode 100644 index 000000000000..bd090197433b --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCache.h @@ -0,0 +1,629 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/blockRadixTree.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/lifeCycleRegistry.h" +#include "kv_cache_manager_v2/movingAverage.h" +#include "kv_cache_manager_v2/page.h" +#include "kv_cache_manager_v2/pendingStats.h" +#include "kv_cache_manager_v2/utils/cudaEvent.h" + +#include "tensorrt_llm/common/assert.h" +#include <functional> +#include <memory> +#include <optional> +#include <variant> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// Forward declarations. +class KvCacheIntrospection; +class KvCacheManager; +class StorageManager; +struct ScratchDesc; + +// --------------------------------------------------------------------------- +// BlockPage — what a SeqBlock holds per (beamIndex, lifeCycleId): +// - nullptr → no page (block not allocated for this lifecycle/beam) +// - SharedPageLock → locked (ACTIVE inference) +// - shared_ptr<PageHolder> → held (suspended, waiting for activation) +// --------------------------------------------------------------------------- +using BlockPage = std::variant<std::monostate, // nullptr + SharedPageLock, // locked + SharedPtr<PageHolder> // held + >; + +inline bool blockPageIsNull(BlockPage const& bp) noexcept +{ + return std::holds_alternative<std::monostate>(bp); +} + +inline SharedPtr<Page> const& blockPageGetPage(BlockPage const& bp) noexcept +{ + static SharedPtr<Page> const sNull{}; + if (auto* lock = std::get_if<SharedPageLock>(&bp)) + return lock->isValid() ? lock->page() : sNull; + if (auto* holder = std::get_if<SharedPtr<PageHolder>>(&bp)) + return holder->get() ? (*holder)->page : sNull; + return sNull; +} + +// --------------------------------------------------------------------------- +// SeqBlock — one block-slot in a KvCache sequence. +// pages[beamIdx][lcId] tracks who holds/locks each page. +// treeBlock: non-null only for committed blocks (strong ref in rare cases). +// --------------------------------------------------------------------------- +using LifeCycleBlockPages = TypedVec<LifeCycleId, BlockPage>; +using BeamBlockPages = TypedVec<BeamIndex, LifeCycleBlockPages>; + +struct SeqBlock +{ + BeamBlockPages pages; + SharedPtr<Block> treeBlock; // non-null iff committed + + bool isCommitted() const noexcept + { + bool ret = treeBlock != nullptr; + if (TLLM_UNLIKELY(gDebug)) + { + // When committed: must have 1 beam, all non-null pages must be CommittedPage. + if (ret) + { + TLLM_CHECK(pages.size() == BeamIndex{1}); + for (auto const& beamBlock : pages) + for (auto const& bp : beamBlock) + if (!blockPageIsNull(bp)) + { + auto pg = blockPageGetPage(bp); + TLLM_CHECK(!pg || dynamicPointerCast<CommittedPage>(pg)); + } + } + else + { + // When not committed: all non-null pages must be UncommittedPage. + for (auto const& beamBlock : pages) + for (auto const& bp : beamBlock) + if (!blockPageIsNull(bp)) + { + auto pg = blockPageGetPage(bp); + TLLM_CHECK(!pg || dynamicPointerCast<UncommittedPage>(pg)); + } + } + } + return ret; + } +}; + +// --------------------------------------------------------------------------- +// Span<T> — non-owning view into a contiguous buffer. +// Supports operator[] for uniform access with std::vector<int>. +// --------------------------------------------------------------------------- +template <typename T> +struct Span +{ + T* ptr; + int32_t len; + + T& operator[](int idx) + { + return ptr[idx]; + } + + T operator[](int idx) const + { + return ptr[idx]; + } + + int size() const noexcept + { + return len; + } + + T* data() const noexcept + { + return ptr; + } + + T* begin() const noexcept + { + return ptr; + } + + T* end() const noexcept + { + return ptr + len; + } +}; + +// --------------------------------------------------------------------------- +// PlannedDropHandle — tracks committed pages planned for dropping without +// owning them. Mirrors Python's PlannedDropHandle in _core/_kv_cache.py. +// +// The handle stores weak references and does not keep pages alive. Dropping it +// decrements each live page's planned-drop count and removes an already- +// droppable page from eviction tracking when no plans remain. +// --------------------------------------------------------------------------- +class PlannedDropHandle +{ +public: + // Deduplicates `pages` by identity, stores weak references, and increments + // each page's plannedDropCount. + explicit PlannedDropHandle(std::vector<CommittedPage*> const& pages); + + // Mirrors Python's __del__: applies the plan if not already dropped. + ~PlannedDropHandle(); + + PlannedDropHandle(PlannedDropHandle const&) = delete; + PlannedDropHandle& operator=(PlannedDropHandle const&) = delete; + + // Apply this drop plan and invalidate the handle. + // + // A live page is removed from eviction tracking only when this is its final + // plan and it is already droppable and queued for eviction. Calling this + // method twice throws (translated to Python ValueError). + void drop(); + +private: + // nullopt once dropped (mirrors Python's `_page_refs is None`). + std::optional<std::vector<WeakPtr<CommittedPage>>> mPageRefs; +}; + +// --------------------------------------------------------------------------- +// KvCache — manages the per-sequence KV cache state. +// Mirrors Python's _KVCache. +// --------------------------------------------------------------------------- +class KvCache : public std::enable_shared_from_this<KvCache> +{ +public: + enum class Status + { + ACTIVE, + SUSPENDED, + CLOSED + }; + enum class CommitState + { + ALLOWED, + VIRTUAL_STOP, + USER_STOP + }; + + // Priority callback: (blockOrdinal, lifeCycleId) → Priority. + using PriorityCb = std::function<Priority(BlockOrdinal, LifeCycleId)>; + + KvCache(KvCacheManager& manager, ReuseScope reuseScope, std::optional<BlockRadixTree::ReuseMatch> reuseMatch, + std::optional<RequestIdType> id, PriorityCb priorityCb, std::optional<int> expectedPromptLength = std::nullopt); + + ~KvCache(); + + KvCache(KvCache const&) = delete; + KvCache& operator=(KvCache const&) = delete; + + // ---- State machine ----------------------------------------------------- + + // Resume: check utilization and lock all pages to GPU. + // Optionally sets a new CUDA stream; if nullopt, uses the existing one. + // Returns false if utilization too high or out of memory. + bool resume(std::optional<CUstream> stream = std::nullopt); + + // Suspend: detach from CUDA stream, unlock pages → PageHolder. + void suspend(); + + // Close: release all blocks back to KvCacheManager. + void close(); + + // Commit or discard request-local statistics accumulated since the previous scheduler commit. + KVCacheStatsDelta commitPendingStats(); + void discardPendingStats(); + + // Best-effort prefetch active pages to the target cache level. + bool prefetch(CacheLevel target); + + // ---- Capacity / history ------------------------------------------------ + + // Resize capacity and/or history_length. + // Returns true if the resize was a no-op shortcut. + bool resize(std::optional<int> capacity, std::optional<int> historyLength = std::nullopt); + + // Convenience: set only capacity or history length. + void setCapacity(int capacity); + void setHistoryLength(int historyLength); + + // ---- Committing tokens ------------------------------------------------- + + // Commit tokens: finalises the oldest uncommitted block and makes it + // available for reuse by other KvCaches. + // tokens must contain exactly tokensPerBlock tokens per call (until the last). + // is_end: if true, records a final reusable snapshot and stops committing. + // This is a terminal-memory contract: callers must not perform later writes + // to this KvCache's memory. The final live pages may be moved into the radix + // tree instead of copied (SSM state and the last partial block). + void commit(std::vector<TokenIdExt> const& tokens, bool isEnd = false); + + // Stop committing (called by close() automatically). + void stopCommitting(); + + // ---- Page index queries ------------------------------------------------ + + // Get base page indices (slot_id) for beamIdx × layerGroupId. + // Returns a non-owning Span into the internal page-index buffer. + Span<int const> getBasePageIndices(LayerGroupId lgId, BeamIndex beamIdx = kDefaultBeamIndex) const; + + // Get aggregated (slot-level) page indices for one layer group + beam. + // Returns one entry per block; bad blocks yield kBadPageIndex. + // If valid_only=true, bad-index blocks are skipped entirely. + std::vector<int> getAggregatedPageIndices( + LayerGroupId lgId, BeamIndex beamIdx = kDefaultBeamIndex, bool validOnly = false) const; + + // Zero-copy page index buffer: copy current base indices into [buf, buf+len) + // and arrange that future updateBasePageIndex calls write there too. + // Pass buf=nullptr / len=0 to revert to the internal vector. + void setBasePageIndexBuf(BeamIndex beamIdx, LayerGroupId lgId, int32_t* buf, int len); + + // ---- Introspection ----------------------------------------------------- + + Status status() const noexcept + { + return mStatus; + } + + CommitState commitState() const noexcept + { + return mCommitState; + } + + bool isActive() const noexcept + { + return mStatus == Status::ACTIVE; + } + + bool isClosed() const noexcept + { + return mStatus == Status::CLOSED; + } + + BlockOrdinal numBlocks() const noexcept + { + return mBlocks.size(); + } + + TypedVec<BlockOrdinal, SeqBlock> const& blocks() const noexcept + { + return mBlocks; + } + + int numCommittedBlocks() const noexcept + { + return mNumCommittedBlocks; + } + + int numCommittedTokens() const noexcept + { + return static_cast<int>(mCommittedTokens.size()); + } + + std::vector<TokenIdExt> const& committedTokens() const noexcept + { + return mCommittedTokens; + } + + ReuseScope const& reuseScope() const noexcept + { + return mReuseScope; + } + + // Plan dropping SWA blocks needed only by the next conversation turn. + // + // The plan covers committed pages in each SWA life cycle's current attention + // window. Full-attention and attention-sink blocks are excluded because + // later turns may still need them. SSM state is not yet supported. Must be + // called after stopCommitting(). Returns nullptr without creating a plan if + // any required SWA page is unavailable. Mirrors Python's + // _KVCache.plan_committed_block_drop(). + std::unique_ptr<PlannedDropHandle> planCommittedBlockDrop(); + + int historyLength() const noexcept + { + return mHistoryLength; + } + + int capacity() const noexcept + { + return mCapacity; + } + + int tokensPerBlock() const noexcept + { + return mTokensPerBlock; + } + + BeamIndex beamWidth() const noexcept + { + return mBeamWidth; + } + + CUstream cudaStream() const; + + // Mirrors Python's cuda_stream setter: if already on a stream AND active, + // make the new stream wait for the old one before switching (cross-stream sync). + void setCudaStream(CUstream stream) + { + if (mCudaStream.has_value()) + { + if (mStatus == Status::ACTIVE) + { + CachedCudaEvent ev(reinterpret_cast<CudaStream>(*mCudaStream)); + ev.waitInStream(reinterpret_cast<CudaStream>(stream)); + } + } + else + { + TLLM_CHECK_DEBUG(mStatus == Status::SUSPENDED && !mFinishEvent.has_value()); + } + mCudaStream = stream; + } + + CachedCudaEvent finishEvent() const; + + // RAII scope guard for _record_event() context manager. + // Sets mFinishEvent on construction, clears it on destruction (= Python's finally). + [[nodiscard]] auto recordEventScope() + { + TLLM_CHECK_DEBUG(!mFinishEvent.has_value()); + // When mCudaStream is nullopt the cache was never resumed — no GPU work + // was performed, so no CUDA event synchronization is needed. Blocks only + // contain PageHolders (not SharedPageLocks) whose destructors do not read + // finishEvent. Mirrors Python's _record_event() early-return path. + if (mCudaStream.has_value()) + { + mFinishEvent = CachedCudaEvent(reinterpret_cast<CudaStream>(*mCudaStream)); + } + return FuncGuard([this]() { mFinishEvent.reset(); }); + } + + // Priority for (blockOrdinal, lifeCycleId) based on the callback. + Priority getPriority(BlockOrdinal ordinal, LifeCycleId lc) const; + + // Reference to StorageManager (for page acquisition/release). + StorageManager* storageManager() const; + + KvCacheManager& manager() const noexcept + { + return *mManager; + } + + // ---- SSM support -------------------------------------------------------- + + // Return the slot ID for the SSM block at the given layer group / beam. + // Returns kBadPageIndex if no SSM blocks are allocated. + int getSsmBlockBaseIndex(LayerGroupId lgId, BeamIndex beamIdx = kDefaultBeamIndex) const; + + // ---- SWA scratch slot management ------------------------------------------ + + // Return scratch metadata for a layer group, or nullopt if no scratch blocks. + std::optional<ScratchDesc> getScratchDesc(LayerGroupId lgId) const; + + // True if any lifecycle has scratch slots allocated. + bool hasScratchSlots() const; + + // Whether SWA scratch reuse is enabled for this KvCache. + bool isSwaScratchReuseEnabled() const noexcept; + + // Enable or disable SWA scratch reuse. Throws if the transition is invalid. + void setEnableSwaScratchReuse(bool enable); + + // Whether the given page index mode is supported (SHARED requires no scratch slots). + bool supportsIndexMode(PageIndexMode mode) const; + + // ---- Internal callbacks (called by SharedPageLock) ---------------------- + + int updateBasePageIndex(BeamIndex bi, BlockOrdinal ord, LifeCycleId lc, int value); + + std::optional<RequestIdType> id; // opaque identifier (mirrors Python's id field) + +private: + friend class KvCacheIntrospection; + friend std::vector<SharedPageLock> batchedLockToGpu( + KvCache& kvCache, std::vector<BatchedLockTarget> const& targets); + + // Activate: lock all pages to GPU. mCudaStream must already be set. + // Internal — called by resume(). Not public (mirrors Python where activate() doesn't exist). + void activate(); + + // Internal helpers. + void _setupForReuse(BlockRadixTree::ReuseMatch const& match); + // Reconstruct the committed token sequence from a match's blocks (mirrors + // Python's _get_matched_tokens); used when reuse-matching no longer has the + // raw input tokens in scope. + std::vector<TokenIdExt> _getMatchedTokens(BlockRadixTree::ReuseMatch const& match) const; + void _clearBlocks(); + // Copy `srcPage` into a new committed page attached to `treeBlock` for lifecycle + // `lcIdx`. When `ssmNumTokensInBlock` is set, the copy is an SsmCommittedPage + // covering that many tokens; otherwise a plain attention CommittedPage. No-op if + // the block already holds a page for this lifecycle, or on OOM in all levels. + void _copyPageToTreeBlock(SharedPtr<Block> const& treeBlock, LifeCycleId lcIdx, SharedPtr<Page> const& srcPage, + std::optional<int> ssmNumTokensInBlock = std::nullopt); + + // Snapshot live SSM state to `treeBlock` reusable at `numTokens` committed tokens. + // If `move`, the live SSM page is moved (not copied) into the tree — the caller + // must guarantee no later writes to this KvCache's memory. + void _snapshotSsmToTreeBlock( + SharedPtr<Block> const& treeBlock, LifeCycleId ssmLcId, int numTokens, bool move = false); + + // Snapshot a partial (non-full) final block at `ordinal` into the radix tree, + // copying partial attention pages and optionally the SSM snapshot. + void _snapshotPartialBlockToTree(BlockOrdinal ordinal, bool commitSsm); + // Returns [stale_begin, stale_end) block ordinal range for a SWA lifecycle. + HalfOpenRange<BlockOrdinal> _getStaleRange(int historyLength, LifeCycle const& lc) const; + + // Backup entry for rollback after _unlockStaleBlocks. + struct StaleBackup + { + BlockOrdinal ordinal; + BeamIndex beamIdx; + LifeCycleId lcId; + SharedPtr<PageHolder> holder; + }; + + // Unlock stale SWA blocks. Returns backup holders for rollback. + std::vector<StaleBackup> _unlockStaleBlocks(int historyLength); + + // Re-lock previously unlocked stale blocks (rollback on OOM). + void _lockHeldBlocks(std::vector<StaleBackup> const& backup); + + // Iterator over (ordinal, beamIdx, lcIdx) tuples for active (non-stale) pages. + // Mirrors Python's _active_pages(). Used by activate() for efficient lock. + struct ActivePage + { + BlockOrdinal ordinal; + BeamIndex beamIdx; + LifeCycleId lcId; + }; + + std::vector<ActivePage> _activePages() const; + SharedPtr<Page> _page(BlockOrdinal ordinal, BeamIndex beamIdx, LifeCycleId lcId) const; + + bool _shortcutSetCapacity(int capacity); + bool _shortcutSetHistoryLength(int historyLength); + bool _shouldRecordStats() const; + void _refreshStatsDirtyState(); + void _recordDirectIterationStats(LifeCycleId lifeCycle, KVCacheIterationStatsDelta const& iterationStats); + void _recordMigratedSlots(std::vector<SharedPtr<Page>> const& pages, std::vector<Slot> const& slots, + CacheLevel srcLevel, CacheLevel dstLevel); + void _recordDroppedPages(std::vector<SharedPtr<Page>> const& pages, CacheLevel cacheLevel); + void _refreshGenerationAllocReady(); + void _recordResizePendingAllocations(BlockOrdinal blockBegin, BlockOrdinal blockEnd, + TypedVec<LifeCycleId, HalfOpenRange<BlockOrdinal>> const& excludedRanges, bool countAsGeneration); + void _subtractPendingAllocationRange(BlockOrdinal blockBegin, BlockOrdinal blockEnd); + static bool _hasReuseSource(BlockPage const& page); + void _increaseCapacity(BlockOrdinal newNumBlocks, int newHistoryLength); + void _decreaseCapacity(BlockOrdinal newNumBlocks); + + void _evictOutOfWindowBlocks(int historyLength) + { + (void) historyLength; + } // handled by _unlockStaleBlocks + + // Release stale held uncommitted pages for SWA layers after committing stops. + // Mirrors Python's _on_stop_committing(). + void _onStopCommitting(); + + // Commit a single block at ordinal `ord`. + // `isLast` mirrors Python's is_last parameter: when True (or on VIRTUAL_STOP), + // transitions to USER_STOP and calls _onStopCommitting() internally. + // Caller must have recordEventScope() open so finishEvent() works. + // `commitSsm` snapshots the current SSM state for this block; `moveSsm` + // moves (vs copies) the live SSM page into the tree (caller must guarantee + // no later writes to this KvCache's memory). Mirrors Python's _commit_block. + void _commitBlock(int ord, bool isLast, bool commitSsm = false, bool moveSsm = false); + + struct TakenPage + { + SharedPtr<UncommittedPage> page; + bool locked; + }; + + // Extract uncommitted pages from a SeqBlock, resetting block page entries. + // Returns one TakenPage per lifecycle. Mirrors Python's _take_uncommitted_page(). + TypedVec<LifeCycleId, TakenPage> _takeUncommittedPage( + SeqBlock& sb, BeamIndex beamIdx, std::optional<LifeCycleId> skipLc = std::nullopt); + + // Get and validate the tree block at a committed ordinal. + // Mirrors Python's _get_tree_block(). Asserts committed pages reference the correct block. + SharedPtr<Block> const& _getTreeBlock(BlockOrdinal ordinal) const; + + // Comprehensive sanity check of KvCache invariants. + // Mirrors Python's _check_sanity(). Returns true on success (asserts internally). + bool _checkSanity() const; + + // ---- SWA scratch private helpers ------------------------------------------ + + // Compute the scratch block range for a lifecycle. + HalfOpenRange<BlockOrdinal> _getScratchRange(LifeCycle const& lc, std::optional<int> hlOverride = std::nullopt, + std::optional<int> capOverride = std::nullopt) const; + + // Result of _takeExcessScratchSlots: excess locks, per-lc delta counts, and scratch ranges. + struct DeltaScratchSlots + { + TypedVec<LifeCycleId, std::vector<ScratchSlotLock>> excess; + TypedVec<LifeCycleId, int> deltaCnt; + TypedVec<LifeCycleId, HalfOpenRange<BlockOrdinal>> scratchRanges; + }; + + // Compute and remove excess scratch slots for a new capacity/historyLength. + DeltaScratchSlots _takeExcessScratchSlots(int capacity, int historyLength); + + // Recover previously taken excess scratch slots back into mScratchSlots. + void _recoverExcessScratchSlots(TypedVec<LifeCycleId, std::vector<ScratchSlotLock>>& excess); + + // Release all scratch slots back to storage. + void _freeScratchSlots(); + + // Whether any lifecycle would require scratch blocks at the current state. + bool _wouldUseSwaScratchBlocks() const; + int _swaScratchMaxRewindLen() const; + + // Page index table management. + // _basePageIndices[beamIdx][lcId][blockOrdinal] = slotId or BAD + void _checkPageIndexBufferCapacity(BlockOrdinal newNumBlocks) const; + void _resizePageIndexBuffers(BlockOrdinal newNumBlocks); + + std::shared_ptr<KvCacheManager> mManager; + ReuseScope mReuseScope; + PriorityCb mPriorityCb; + std::optional<CUstream> mCudaStream; + Status mStatus; + CommitState mCommitState; + BeamIndex mBeamWidth; + int mCapacity; + int mHistoryLength; + std::optional<int> mExpectedPromptLength; + bool mGenerationAllocReady = false; + + // Page index tables: [beamIdx][lcId] → either an internal vector or an external span. + // Mirrors Python's IndexSeq = array.array | memoryview. + using PageIndexBuf = std::variant<std::vector<int>, Span<int>>; + using LifeCyclePageIndexBuffers = TypedVec<LifeCycleId, PageIndexBuf>; + using BeamPageIndexBuffers = TypedVec<BeamIndex, LifeCyclePageIndexBuffers>; + BeamPageIndexBuffers mBasePageIndices; + + TypedVec<BlockOrdinal, SeqBlock> mBlocks; + + std::vector<TokenIdExt> mCommittedTokens; + int mNumCommittedBlocks; + std::optional<CachedCudaEvent> mFinishEvent; + int mTokensPerBlock; + Average mAvgHistoryLength; + Average mAvgCapacity; + + // SSM pages: [beamIdx][lcId] — always initialized (empty entries = monostate). + BeamBlockPages mSsmBlocks; + bool mNeverResumed = true; + + PendingStats mPendingStats; + + // SWA scratch slot support. + bool mEnableSwaScratchReuse = false; + TypedVec<LifeCycleId, std::vector<ScratchSlotLock>> mScratchSlots; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCacheManager.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCacheManager.cpp new file mode 100644 index 000000000000..ae136b0306f7 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCacheManager.cpp @@ -0,0 +1,860 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/kvCacheManager.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/exceptions.h" +#include "kv_cache_manager_v2/storage/config.h" +#include "kv_cache_manager_v2/storage/core.h" +#include "kv_cache_manager_v2/utils/math.h" + +#include "tensorrt_llm/common/assert.h" +#include <algorithm> +#include <chrono> +#include <cmath> +#include <map> +#include <numeric> +#include <stdexcept> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +static double nowSeconds() +{ + return static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>( + std::chrono::steady_clock::now().time_since_epoch()) + .count()) + / 1e6; +} + +// --------------------------------------------------------------------------- +// PageIndexConverter +// --------------------------------------------------------------------------- + +std::vector<int> PageIndexConverter::operator()( + std::vector<int> const& baseIndices, std::optional<PageIndexMode> indexMode, ScratchDesc const* scratch) const +{ + if (!indexMode.has_value()) + { + if (scratch) + { + throw std::invalid_argument("index_mode must be provided when scratch is active"); + } + indexMode = PageIndexMode::SHARED; + } + + int appliedLayerOffset = (*indexMode == PageIndexMode::PER_LAYER) ? layerOffset : 0; + int scratchPages = scratchPagesPerBlock; + + std::vector<int> result; + result.reserve(baseIndices.size() * static_cast<size_t>(expansion)); + + for (BlockOrdinal ordinal{0}; ordinal < BlockOrdinal{static_cast<int>(baseIndices.size())}; ++ordinal) + { + int index; + if (scratch && scratch->range.contains(ordinal)) + { + // Scratch block: slot IDs come from ScratchDesc, not base_indices. + int blockPos = ordinal - scratch->range.beg; + int totalOffset = blockPos * scratchPages; + int slotIdx = totalOffset / scale; + int slotId = scratch->slotIds[static_cast<size_t>(slotIdx)]; + int offset = totalOffset % scale; + index = slotId * scale + (offset + appliedLayerOffset) % scale; + } + else if (baseIndices[toSizeT(ordinal)] == kBadPageIndex.value()) + { + index = kBadPageIndex.value(); + } + else + { + index = baseIndices[toSizeT(ordinal)] * scale + appliedLayerOffset; + } + for (int i = 0; i < expansion; ++i) + { + result.push_back(index != kBadPageIndex.value() ? index * expansion + i : kBadPageIndex.value()); + } + } + return result; +} + +std::vector<int> PageIndexConverter::operator()(int baseIndex) const +{ + return operator()(std::vector<int>{baseIndex}, std::nullopt, nullptr); +} + +// --------------------------------------------------------------------------- +// KvCacheManager +// --------------------------------------------------------------------------- + +KvCacheManager::KvCacheManager(KVCacheManagerConfig const& config, std::shared_ptr<EventSink> eventSink) + : mConfig(config) + , mLifeCycles(config) + , mEventSink(std::move(eventSink)) + , mAvgReusedLength(0.9999) + , mAvgSqrCapacity(0.9999) + , mAvgSqrHistoryLength(0.9999) +{ + mConfig.validate(); + + mRadixTree = std::make_shared<BlockRadixTree>(mLifeCycles, mConfig.tokensPerBlock, mEventSink); + + StorageConfig storageConfig = createStorageConfig(mConfig); + mStorage + = std::make_shared<StorageManager>(mLifeCycles, storageConfig, mConfig.tokensPerBlock, mConfig.swaScratchReuse, + mConfig.typicalStep, mConfig.constraints, mConfig.initialPoolRatio, mEventSink, mConfig.maxUtilForResume); + + mTargetRatioListGpu = _currentGpuRatio(); + mTargetRatioListOther = _currentOtherRatios(); + _resetIterationPeakNumBlocks(); + + mLastAdjustmentTime = nowSeconds(); +} + +KvCacheManager::~KvCacheManager() +{ + shutdown(); +} + +void KvCacheManager::shutdown() +{ + clearReusableBlocks(); + TLLM_CHECK_DEBUG(mStorage); + + // Post-condition: after clearing all reusable blocks and with no active KvCaches, + // no evictable pages should remain. + for (auto const& lvl : mStorage->mLevels) + { + for (PoolGroupIndex pgIdx{0}; pgIdx < lvl.numPoolGroups(); ++pgIdx) + { + TLLM_CHECK_DEBUG(lvl.controller.numEvictablePages(pgIdx) == 0); + } + } + + mStorage->destroy(); +} + +void KvCacheManager::clearReusableBlocks() +{ + TLLM_CHECK_DEBUG(mRadixTree); + mRadixTree->clear(); +} + +std::shared_ptr<KvCache> KvCacheManager::createKvCache(ReuseScope reuseScope, + std::vector<TokenIdExt> const& inputTokens, std::optional<RequestIdType> id, KvCache::PriorityCb priorityCb, + std::optional<int> expectedPromptLength) +{ + if (!priorityCb) + { + priorityCb = [](BlockOrdinal, LifeCycleId) { return kPriorityDefault; }; + } + + if (!expectedPromptLength.has_value() && !inputTokens.empty()) + { + expectedPromptLength = static_cast<int>(inputTokens.size()); + } + + // Compute the reuse match once here (shared conceptually with probeReuse) and + // hand it to the KvCache, rather than having the cache re-walk the radix tree. + // Mirrors Python KVCacheManager.allocate() passing a ReuseMatch into _KVCache. + std::optional<BlockRadixTree::ReuseMatch> reuseMatch; + if (!inputTokens.empty()) + { + reuseMatch = matchReuse(reuseScope, inputTokens); + } + + return std::make_shared<KvCache>(*this, std::move(reuseScope), std::move(reuseMatch), std::move(id), + std::move(priorityCb), expectedPromptLength); +} + +BlockRadixTree::ReuseMatch KvCacheManager::matchReuse( + ReuseScope const& reuseScope, std::vector<TokenIdExt> const& inputTokens) const +{ + return mRadixTree->match(reuseScope, inputTokens, enablePartialMatch()); +} + +int KvCacheManager::probeReuse(ReuseScope reuseScope, std::vector<TokenIdExt> const& inputTokens) const +{ + return matchReuse(reuseScope, inputTokens).numTokens; +} + +// ---- Memory pool queries -------------------------------------------------- + +MemAddress KvCacheManager::getMemPoolBaseAddress( + LayerId layerId, DataRole role, std::optional<PageIndexMode> indexMode) const +{ + auto const& attr = mStorage->getBufferAttr(layerId, role); + + if (!indexMode.has_value()) + { + if (mConfig.enableSwaScratchReuse()) + { + throw std::invalid_argument("index_mode must be provided when SWA scratch reuse is enabled"); + } + indexMode = PageIndexMode::SHARED; + } + + PoolGroupIndex pgIdx = mStorage->getPoolGroupIndex(attr.lifeCycleId); + MemAddress addr = mStorage->getMemPoolBaseAddress(pgIdx, attr.poolIndex); + if (*indexMode == PageIndexMode::SHARED) + { + addr = MemAddress(addr + attr.offset); + } + return addr; +} + +int KvCacheManager::getPageStride(LayerId layerId, DataRole role) const +{ + auto const& attr = mStorage->getBufferAttr(layerId, role); + return exactDiv(static_cast<int>(attr.size), attr.expansion); +} + +size_t KvCacheManager::getPageIndexUpperBound(LayerId layerId, DataRole role) const +{ + auto const& attr = mStorage->getBufferAttr(layerId, role); + LifeCycleId lc = attr.lifeCycleId; + PoolGroupIndex pg = mStorage->getPoolGroupIndex(lc); + SlotCount const numSlots = mStorage->numSlots(pg, kGpuLevel); + auto slotSizes = mStorage->slotSize(pg); + size_t slotSize = slotSizes.at(attr.poolIndex); + return (exactDiv(slotSize, attr.size) * slotCountToSizeT(numSlots) - exactDiv(attr.offset, attr.size)) + * static_cast<size_t>(attr.expansion); +} + +int KvCacheManager::getPageIndexScale(LayerId layerId, DataRole role) const +{ + auto const& attr = mStorage->getBufferAttr(layerId, role); + return mStorage->mSlotToPageIndices.at(attr.lifeCycleId).at(attr.poolIndex); +} + +PageIndexConverter KvCacheManager::getPageIndexConverter(LayerId layerId, DataRole role) const +{ + auto const& attr = mStorage->getBufferAttr(layerId, role); + auto const& layerAttr = mStorage->getLayerAttr(layerId); + int scale = mStorage->mSlotToPageIndices.at(attr.lifeCycleId).at(attr.poolIndex); + int offset = exactDiv(static_cast<int>(attr.offset), static_cast<int>(attr.size)); + int scratchPages = layerAttr.slotUtil.at(attr.poolIndex); + return PageIndexConverter{scale, attr.expansion, offset, scratchPages}; +} + +std::optional<bool> KvCacheManager::supportsIndexMode(PageIndexMode mode) const +{ + switch (mode) + { + case PageIndexMode::PER_LAYER: return true; + case PageIndexMode::SHARED: return mConfig.enableSwaScratchReuse() ? std::optional<bool>(std::nullopt) : true; + } + return std::nullopt; +} + +// ---- getAggregatedPages --------------------------------------------------- + +std::vector<AggregatedPageDesc> KvCacheManager::getAggregatedPages(std::vector<BufferId> const& buffers) const +{ + using Key = std::pair<LifeCycleId, PoolIndex>; + + struct Entry + { + size_t start; + size_t end; + ExpandedBuffer eb; + }; + + std::map<Key, std::vector<Entry>> groups; + + for (auto const& bufferId : buffers) + { + auto it = mStorage->mBufferAttr.find(bufferId); + if (it == mStorage->mBufferAttr.end()) + throw std::out_of_range("getAggregatedPages: unknown buffer id"); + + auto const& attr = it->second; + size_t start = attr.offset; + size_t end = attr.offset + attr.size; + Key key{attr.lifeCycleId, attr.poolIndex}; + groups[key].push_back({start, end, ExpandedBuffer{bufferId, attr.expansion}}); + } + + std::vector<AggregatedPageDesc> result; + for (auto& [key, entries] : groups) + { + auto [lifeCycleId, poolIdx] = key; + auto pgIdx = mStorage->getPoolGroupIndex(lifeCycleId); + + std::sort(entries.begin(), entries.end(), [](Entry const& a, Entry const& b) { return a.start < b.start; }); + + auto const poolBase = mStorage->getMemPoolBaseAddress(pgIdx, poolIdx); + size_t stride = mStorage->slotSize(pgIdx).at(poolIdx); + + auto flush + = [&, lifeCycleId = lifeCycleId](size_t start, size_t end, std::vector<ExpandedBuffer>& buffersInRange) + { + result.push_back(AggregatedPageDesc{ + MemAddress(poolBase + start), end - start, stride, lifeCycleId, std::move(buffersInRange)}); + }; + + size_t currentStart = entries.front().start; + size_t currentEnd = entries.front().end; + std::vector<ExpandedBuffer> currentBuffers{entries.front().eb}; + for (size_t i = 1; i < entries.size(); ++i) + { + if (entries[i].start == currentEnd) + { + currentEnd = entries[i].end; + currentBuffers.push_back(entries[i].eb); + continue; + } + + flush(currentStart, currentEnd, currentBuffers); + currentStart = entries[i].start; + currentEnd = entries[i].end; + currentBuffers = {entries[i].eb}; + } + flush(currentStart, currentEnd, currentBuffers); + } + + return result; +} + +// ---- Pool group layout ----------------------------------------------------- + +TypedVec<PoolGroupIndex, PoolGroupDesc> KvCacheManager::poolGroupDescs() const +{ + auto const& slotDescList = mStorage->slotDescList(); + TypedVec<PoolGroupIndex, PoolGroupDesc> result; + result.reserve(slotDescList.size()); + + for (PoolGroupIndex pgIdx{0}; pgIdx < slotDescList.size(); ++pgIdx) + { + auto const& slotDesc = slotDescList.at(pgIdx); + auto slotSizeList = mStorage->slotSize(pgIdx); + + TypedVec<PoolIndex, PoolDesc> pools; + pools.reserve(slotSizeList.size()); + for (PoolIndex poolIdx{0}; poolIdx < slotSizeList.size(); ++poolIdx) + { + pools.push_back( + PoolDesc{poolIdx, mStorage->getMemPoolBaseAddress(pgIdx, poolIdx), slotSizeList.at(poolIdx)}); + } + + result.push_back(PoolGroupDesc{pgIdx, mStorage->numSlots(pgIdx, kGpuLevel), slotDesc, std::move(pools)}); + } + + return result; +} + +// ---- Query / info --------------------------------------------------------- + +int KvCacheManager::tokensPerBlock() const noexcept +{ + return mRadixTree->tokensPerBlock(); +} + +bool KvCacheManager::enablePartialMatch() const noexcept +{ + return mConfig.enablePartialReuse; +} + +int KvCacheManager::numLayers() const noexcept +{ + return static_cast<int>(mStorage->layerToLifeCycleIds().size()); +} + +std::vector<LayerId> KvCacheManager::layerIds() const +{ + std::vector<LayerId> ids; + for (auto const& [lid, lc] : mStorage->layerToLifeCycleIds()) + ids.push_back(lid); + return ids; +} + +LayerGroupId KvCacheManager::getLayerGroupId(LayerId layerId) const +{ + return mStorage->layerToLifeCycleIds().at(layerId); +} + +TypedVec<LayerGroupId, std::vector<LayerId>> KvCacheManager::layerGrouping() const +{ + LifeCycleId numLc = mLifeCycles.size(); + TypedVec<LayerGroupId, std::vector<LayerId>> result(numLc); + for (auto const& [lid, lc] : mStorage->layerToLifeCycleIds()) + { + result.at(lc).push_back(lid); + } + return result; +} + +// ---- Resize --------------------------------------------------------------- + +bool KvCacheManager::resize(CacheLevel level, size_t quota, bool bestEfforts) +{ + if (bestEfforts) + throw std::runtime_error("best_efforts resize not implemented"); + try + { + _adjustLevel(level, quota); + return true; + } + catch (std::exception const& e) + { + return false; + } +} + +size_t KvCacheManager::getQuota(CacheLevel level) const +{ + return mStorage->mLevels.at(level).storage->totalQuota(); +} + +// ---- Statistics ---------------------------------------------------------- + +void KvCacheManager::commitStats( + KVCacheStatsDelta const& stats, IterationStatsByLifeCycle const& iterationStatsByLifeCycle) +{ + if (!mConfig.enableStats) + { + return; + } + + _updateIterationPeakNumBlocks(); + mCommittedStats.add(stats); + for (auto const& [lifeCycle, iterationStats] : iterationStatsByLifeCycle) + { + if (!iterationStats.empty()) + { + mIterationStatsByLifeCycle[lifeCycle].add(iterationStats); + } + } +} + +KVCacheStatsDelta KvCacheManager::getCommittedStats() const +{ + return mCommittedStats.copy(); +} + +IterationStatsByLifeCycle KvCacheManager::getAndResetIterationStats() +{ + IterationStatsByLifeCycle stats; + for (auto const& [lifeCycle, delta] : mIterationStatsByLifeCycle) + { + if (!delta.empty()) + { + stats.emplace(lifeCycle, delta.copy()); + } + } + mIterationStatsByLifeCycle.clear(); + return stats; +} + +void KvCacheManager::commitSsmSnapshotIterationStats(SsmSnapshotIterationStatsByLifeCycle const& statsByLifeCycle) +{ + if (!mConfig.enableStats) + { + return; + } + for (auto const& [lifeCycle, iterationStats] : statsByLifeCycle) + { + if (!iterationStats.empty()) + { + mSsmSnapshotIterationStatsByLifeCycle[lifeCycle].add(iterationStats); + } + } +} + +SsmSnapshotIterationStatsByLifeCycle KvCacheManager::getAndResetSsmSnapshotIterationStats() +{ + SsmSnapshotIterationStatsByLifeCycle stats; + for (auto const& [lifeCycle, delta] : mSsmSnapshotIterationStatsByLifeCycle) + { + if (!delta.empty()) + { + stats.emplace(lifeCycle, delta.copy()); + } + } + mSsmSnapshotIterationStatsByLifeCycle.clear(); + return stats; +} + +PeakBlockStatsByCacheLevel KvCacheManager::_currentBlockStatsByCacheLevel() const +{ + PeakBlockStatsByCacheLevel result(mStorage->numCacheLevels()); + for (CacheLevel cacheLevel{0}; cacheLevel < mStorage->numCacheLevels(); ++cacheLevel) + { + auto& levelStats = result[cacheLevel]; + levelStats.resize(mStorage->numPoolGroups()); + for (PoolGroupIndex poolGroup{0}; poolGroup < mStorage->numPoolGroups(); ++poolGroup) + { + auto const stats = mStorage->getStatistics(cacheLevel, poolGroup); + levelStats[poolGroup] = {stats.available(), stats.unavailable(), stats.evictable}; + } + } + return result; +} + +void KvCacheManager::_resetIterationPeakNumBlocks(std::optional<CacheLevel> cacheLevel) +{ + if (!cacheLevel.has_value()) + { + mIterationPeakNumBlocksByCacheLevel = _currentBlockStatsByCacheLevel(); + return; + } + + PeakBlockStatsByPoolGroup levelStats(mStorage->numPoolGroups()); + for (PoolGroupIndex poolGroup{0}; poolGroup < mStorage->numPoolGroups(); ++poolGroup) + { + auto const stats = mStorage->getStatistics(*cacheLevel, poolGroup); + levelStats[poolGroup] = {stats.available(), stats.unavailable(), stats.evictable}; + } + mIterationPeakNumBlocksByCacheLevel[*cacheLevel] = std::move(levelStats); +} + +void KvCacheManager::_updateIterationPeakNumBlocks() +{ + auto const current = _currentBlockStatsByCacheLevel(); + for (CacheLevel cacheLevel{0}; cacheLevel < current.size(); ++cacheLevel) + { + auto& peakLevel = mIterationPeakNumBlocksByCacheLevel[cacheLevel]; + for (PoolGroupIndex poolGroup{0}; poolGroup < current[cacheLevel].size(); ++poolGroup) + { + auto& peak = peakLevel[poolGroup]; + auto const& value = current[cacheLevel][poolGroup]; + peak.available = std::max(peak.available, value.available); + peak.unavailable = std::max(peak.unavailable, value.unavailable); + peak.evictable = std::max(peak.evictable, value.evictable); + } + } +} + +PeakBlockStatsByPoolGroup KvCacheManager::getAndResetIterationPeakBlockStats(CacheLevel cacheLevel) +{ + _updateIterationPeakNumBlocks(); + PeakBlockStatsByPoolGroup peak = mIterationPeakNumBlocksByCacheLevel.at(cacheLevel); + _resetIterationPeakNumBlocks(cacheLevel); + return peak; +} + +void KvCacheManager::markStatsDirty(std::optional<RequestIdType> kvCacheId) +{ + if (kvCacheId.has_value()) + { + mDirtyStatsKvCacheIds.insert(*kvCacheId); + } +} + +void KvCacheManager::clearStatsDirty(std::optional<RequestIdType> kvCacheId) +{ + if (kvCacheId.has_value()) + { + mDirtyStatsKvCacheIds.erase(*kvCacheId); + } +} + +std::unordered_set<RequestIdType> KvCacheManager::getDirtyStatsKvCacheIds() const +{ + return mDirtyStatsKvCacheIds; +} + +void KvCacheManager::markStatsExcluded(std::optional<RequestIdType> kvCacheId) +{ + if (kvCacheId.has_value()) + { + mStatsExcludedKvCacheIds.insert(*kvCacheId); + clearStatsDirty(kvCacheId); + } +} + +void KvCacheManager::clearStatsExcluded(std::optional<RequestIdType> kvCacheId) +{ + if (kvCacheId.has_value()) + { + mStatsExcludedKvCacheIds.erase(*kvCacheId); + } +} + +bool KvCacheManager::isStatsExcluded(std::optional<RequestIdType> kvCacheId) const +{ + return kvCacheId.has_value() && mStatsExcludedKvCacheIds.find(*kvCacheId) != mStatsExcludedKvCacheIds.end(); +} + +TypedVec<CacheLevel, CacheTier> KvCacheManager::cacheTierList() const +{ + TypedVec<CacheLevel, CacheTier> result; + result.reserve(mStorage->mLevels.size()); + for (auto const& lvl : mStorage->mLevels) + { + result.push_back(lvl.cacheTier); + } + return result; +} + +std::vector<BufferId> KvCacheManager::allBufferIds() const +{ + std::vector<BufferId> result; + result.reserve(mStorage->mBufferAttr.size()); + for (auto const& item : mStorage->mBufferAttr) + result.push_back(item.first); + return result; +} + +int KvCacheManager::clampMaxSeqLenForMem(int batchSize, int tokenNumUpperBound) const +{ + TLLM_CHECK_DEBUG(batchSize > 0); + int tokPerBlock = tokensPerBlock(); + PoolGroupIndex numPg = mStorage->numPoolGroups(); + auto const& lcs = mLifeCycles; + auto const& lcGrouping = mStorage->mLifeCycleGrouping; + + // Remaining slot counts per pool group. + TypedVec<PoolGroupIndex, SlotCount> remainingSlots(numPg); + for (PoolGroupIndex pgIdx{0}; pgIdx < numPg; ++pgIdx) + { + remainingSlots[pgIdx] = mStorage->numSlots(pgIdx); + } + + // Compute required slot counts per pool group for a given seq_len. + auto getNumSlots = [&](int seqLen) -> TypedVec<PoolGroupIndex, SlotCount> + { + TypedVec<PoolGroupIndex, SlotCount> ret(numPg, 0); + for (LifeCycleId lifeCycleId{0}; lifeCycleId < lcs.size(); ++lifeCycleId) + { + auto staleRange = getStaleRange(lcs[lifeCycleId], seqLen, tokPerBlock); + int numStaleBlocks = staleRange.end - staleRange.beg; + int numSlots = divUp(seqLen, tokPerBlock) - numStaleBlocks; + auto pgIdx = lcGrouping[lifeCycleId]; + ret[pgIdx] += numSlots; + } + return ret; + }; + + // Reserve slots for (batch_size - 1) minimal sequences. + auto minSlots = getNumSlots(1); + for (PoolGroupIndex pgIdx{0}; pgIdx < numPg; ++pgIdx) + { + TLLM_CHECK_DEBUG(minSlots[pgIdx] >= 0); + SlotCount const reservedSlots = minSlots[pgIdx] * (batchSize - 1); + remainingSlots[pgIdx] -= reservedSlots; + if (remainingSlots[pgIdx] < 0) + { + return 0; + } + } + + auto isEnough = [&](int numBlocks) -> bool + { + auto needed = getNumSlots(numBlocks * tokPerBlock); + for (PoolGroupIndex pgIdx{0}; pgIdx < numPg; ++pgIdx) + { + TLLM_CHECK_DEBUG(needed[pgIdx] >= 0); + if (needed[pgIdx] > remainingSlots[pgIdx]) + { + return false; + } + } + return true; + }; + + if (!isEnough(1)) + { + return 0; + } + int lb = 1; + int ub = divUp(tokenNumUpperBound, tokPerBlock); + if (isEnough(ub)) + { + return tokenNumUpperBound; + } + while (lb < ub - 1) + { + int mid = (lb + ub) / 2; + if (isEnough(mid)) + { + lb = mid; + } + else + { + ub = mid; + } + } + return std::min(lb * tokPerBlock, tokenNumUpperBound); +} + +void KvCacheManager::_adjustLevel(CacheLevel level, size_t quota) +{ + auto const& ratioList = _getTargetRatioList(level); + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> const* persistent = nullptr; + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> persistentPages; + if (mStorage->isLastLevel(level)) + { + persistentPages = _gatherPersistentPages(); + persistent = &persistentPages; + } + mStorage->adjustCacheLevel(level, quota, ratioList, persistent); +} + +TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> KvCacheManager::_gatherPersistentPages() const +{ + CacheLevel lastLevel = mStorage->numCacheLevels() - 1; + PoolGroupIndex numPg = mStorage->numPoolGroups(); + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> result(numPg); + + for (KvCache* kvc : mLivingKvCaches) + { + TLLM_CHECK_DEBUG(kvc->status() == KvCache::Status::SUSPENDED); + for (auto const& sb : kvc->blocks()) + { + for (auto const& beamPages : sb.pages) + { + for (LifeCycleId lc{0}; lc < beamPages.size(); ++lc) + { + // Mirrors Python: holder must be _PageHolder type (suspended state). + if (blockPageIsNull(beamPages[lc])) + { + continue; + } + TLLM_CHECK_DEBUG_WITH_INFO(std::holds_alternative<SharedPtr<PageHolder>>(beamPages[lc]), + "Non-null holder must be PageHolder in suspended state"); + auto const& pg = blockPageGetPage(beamPages[lc]); + if (!pg) + { + continue; + } + // Mirrors Python assertions for invariant checking. + TLLM_CHECK_DEBUG_WITH_INFO( + pg->status() == PageStatus::HELD, "Page in suspended KvCache must be HELD"); + TLLM_CHECK_DEBUG_WITH_INFO((pg->scheduledForEviction() == (pg->cacheLevel != lastLevel)), + "Eviction scheduling invariant violated"); + if (pg->scheduledForEviction()) + { + continue; + } + PoolGroupIndex pgIdx = mStorage->getPoolGroupIndex(lc); + result[pgIdx].push_back(pg); + } + } + } + } + return result; +} + +// ---- KvCache registry ----------------------------------------------------- + +void KvCacheManager::registerKvCache(KvCache* kvc) +{ + mLivingKvCaches.insert(kvc); + ++mNumCreatedKvCaches; +} + +void KvCacheManager::unregisterKvCache(KvCache* kvc) +{ + mLivingKvCaches.erase(kvc); +} + +void KvCacheManager::tryUpdateTargetRatios() +{ + if (mNumSampledKvCaches - mLastUpdateNumSampledKvCaches < 100) + return; + mLastUpdateNumSampledKvCaches = mNumSampledKvCaches; + + int tokensPerBlock = mConfig.tokensPerBlock; + int avgReusedLength = static_cast<int>(std::round(mAvgReusedLength.value())); + int avgCapacity = static_cast<int>(std::round(std::sqrt(mAvgSqrCapacity.value()))); + int avgHistoryLength = static_cast<int>(std::round(std::sqrt(mAvgSqrHistoryLength.value()))); + if (avgCapacity > 0) + mTargetRatioListGpu + = mStorage->constrainRatio(mStorage->ratioFromLength(tokensPerBlock, avgHistoryLength, avgCapacity)); + if (avgReusedLength > 0) + mTargetRatioListOther = mStorage->ratioFromLength(tokensPerBlock, avgReusedLength, avgReusedLength); +} + +TypedVec<PoolGroupIndex, float> KvCacheManager::_currentGpuRatio() const +{ + return mStorage->getRatioList(kGpuLevel); +} + +TypedVec<PoolGroupIndex, float> KvCacheManager::_currentOtherRatios() const +{ + CacheLevel numLevels = mStorage->numCacheLevels(); + if (numLevels == CacheLevel{1}) + { + return _currentGpuRatio(); + } + PoolGroupIndex numPg = mStorage->numPoolGroups(); + TypedVec<PoolGroupIndex, float> result(numPg, 0.f); + for (CacheLevel lvl{1}; lvl < numLevels; ++lvl) + { + auto ratios = mStorage->getRatioList(lvl); + for (PoolGroupIndex pgIdx{0}; pgIdx < numPg; ++pgIdx) + { + result[pgIdx] += ratios[pgIdx]; + } + } + float denom = static_cast<float>((numLevels - 1).value()); + for (auto& r : result) + { + r /= denom; + } + return result; +} + +// ---- needAdjustment / adjust ----------------------------------------------- + +TypedVec<PoolGroupIndex, float> const& KvCacheManager::_getTargetRatioList(CacheLevel level) const +{ + return (level == kGpuLevel) ? mTargetRatioListGpu : mTargetRatioListOther; +} + +bool KvCacheManager::_needAdjustment(CacheLevel level) const +{ + auto const& target = _getTargetRatioList(level); + auto current = (level == kGpuLevel) ? _currentGpuRatio() : _currentOtherRatios(); + constexpr float kThreshold = 1.25f; + for (PoolGroupIndex pgIdx{0}; pgIdx < target.size() && pgIdx < current.size(); ++pgIdx) + { + TLLM_CHECK_DEBUG_WITH_INFO(current[pgIdx] > 0.f && target[pgIdx] > 0.f, "ratios must not be zero"); + float ratio = target[pgIdx] / current[pgIdx]; + if (ratio < 1.f / kThreshold || ratio > kThreshold) + return true; + } + return false; +} + +bool KvCacheManager::needAdjustment() const +{ + if (mNumSampledKvCaches < 2000) + return false; + double now = nowSeconds(); + if (now - mLastAdjustmentTime < 120.0) + return false; + CacheLevel lastLevel = mStorage->numCacheLevels() - 1; + return _needAdjustment(kGpuLevel) || _needAdjustment(lastLevel); +} + +void KvCacheManager::adjust() +{ + for (KvCache* kvc : mLivingKvCaches) + TLLM_CHECK_DEBUG(kvc->status() == KvCache::Status::SUSPENDED); + + CacheLevel numLevels = mStorage->numCacheLevels(); + for (CacheLevel level{0}; level < numLevels; ++level) + { + if (_needAdjustment(level)) + _adjustLevel(level, getQuota(level)); + } + mLastAdjustmentTime = nowSeconds(); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCacheManager.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCacheManager.h new file mode 100644 index 000000000000..a9737b5b282c --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCacheManager.h @@ -0,0 +1,344 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/blockRadixTree.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/config.h" +#include "kv_cache_manager_v2/eventSink.h" +#include "kv_cache_manager_v2/kvCache.h" +#include "kv_cache_manager_v2/lifeCycleRegistry.h" +#include "kv_cache_manager_v2/movingAverage.h" +#include "kv_cache_manager_v2/stats.h" +#include "kv_cache_manager_v2/storageManager.h" + +#include <functional> +#include <memory> +#include <optional> +#include <set> +#include <unordered_set> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// PoolDesc / PoolGroupDesc — describe GPU memory pool layout. +// --------------------------------------------------------------------------- +struct PoolDesc +{ + PoolIndex poolIndex{0}; + MemAddress baseAddress = 0; + size_t slotBytes = 0; +}; + +struct PoolGroupDesc +{ + PoolGroupIndex poolGroupIndex{0}; + SlotCount numSlots = 0; + SlotDesc slotDesc; + TypedVec<PoolIndex, PoolDesc> pools; +}; + +// --------------------------------------------------------------------------- +// ExpandedBuffer / AggregatedPageDesc — returned by getAggregatedPages(). +// --------------------------------------------------------------------------- +struct ExpandedBuffer +{ + BufferId id; + int expansion; // expansion factor (tokens_per_block / tokens_per_block_override) +}; + +struct AggregatedPageDesc +{ + MemAddress base; // pool base address + buffer offset + size_t size; // byte span of this aggregated buffer group + size_t stride; // slot size (bytes per slot in the pool group) + LifeCycleId layerGroupId; // pool group / life-cycle id + std::vector<ExpandedBuffer> buffers; // constituent buffers in offset order +}; + +// --------------------------------------------------------------------------- +// ScratchDesc — scratch metadata for one layer group of one sequence. +// Scratch blocks store ephemeral KV data using shared coalesced slots. +// Mirrors _kv_cache_manager.py::ScratchDesc. +// --------------------------------------------------------------------------- +struct ScratchDesc +{ + HalfOpenRange<BlockOrdinal> range; // block ordinal range [beg, end) + std::vector<int> slotIds; // scratch slot IDs, length = ceil(numScratchBlocks / scale) + + explicit operator bool() const noexcept + { + return static_cast<bool>(range); + } +}; + +// --------------------------------------------------------------------------- +// PageIndexConverter — convert base page index → kernel page indices. +// --------------------------------------------------------------------------- +struct PageIndexConverter +{ + int scale; + int expansion; + int layerOffset = 0; // sub-page offset within coalesced slot + int scratchPagesPerBlock = 1; // sub-pages per block for scratch allocation + + // Convert a sequence of base page indices to per-layer page indices. + // indexMode: SHARED (default) or PER_LAYER. When scratch is active, must be PER_LAYER. + // scratch: optional scratch descriptor from KvCache::getScratchDesc(). + // Mirrors _kv_cache_manager.py::PageIndexConverter.__call__. + std::vector<int> operator()(std::vector<int> const& baseIndices, + std::optional<PageIndexMode> indexMode = std::nullopt, ScratchDesc const* scratch = nullptr) const; + + // Backward-compatible single-index overload. + std::vector<int> operator()(int baseIndex) const; +}; + +// --------------------------------------------------------------------------- +// KvCacheManager — top-level KV cache manager. +// Mirrors Python's KVCacheManager. +// --------------------------------------------------------------------------- +class KvCacheManager : public std::enable_shared_from_this<KvCacheManager> +{ +public: + explicit KvCacheManager(KVCacheManagerConfig const& config, std::shared_ptr<EventSink> eventSink = nullptr); + ~KvCacheManager(); + + KvCacheManager(KvCacheManager const&) = delete; + KvCacheManager& operator=(KvCacheManager const&) = delete; + + // ---- Lifecycle -------------------------------------------------------- + + void shutdown(); + + // Clear all reusable (committed) blocks from the radix tree. + void clearReusableBlocks(); + + // ---- KvCache creation ------------------------------------------------- + + // Create a new KvCache. Returned cache is SUSPENDED; call activate() with a stream. + // input_tokens: optional sequence to match against existing cached blocks. + // priorityCb: optional priority override per block. + std::shared_ptr<KvCache> createKvCache(ReuseScope reuseScope = {}, std::vector<TokenIdExt> const& inputTokens = {}, + std::optional<RequestIdType> id = std::nullopt, KvCache::PriorityCb priorityCb = {}, + std::optional<int> expectedPromptLength = std::nullopt); + + BlockRadixTree::ReuseMatch matchReuse( + ReuseScope const& reuseScope, std::vector<TokenIdExt> const& inputTokens) const; + int probeReuse(ReuseScope reuseScope = {}, std::vector<TokenIdExt> const& inputTokens = {}) const; + + // ---- Memory pool queries ----------------------------------------------- + + // Base address of the memory pool. When indexMode is PER_LAYER, returns pool group base + // (without per-layer offset). When SHARED, returns per-layer base (with offset baked in). + MemAddress getMemPoolBaseAddress( + LayerId layerId, DataRole role, std::optional<PageIndexMode> indexMode = std::nullopt) const; + + int getPageStride(LayerId layerId, DataRole role) const; + size_t getPageIndexUpperBound(LayerId layerId, DataRole role) const; + + // Scale factor: base_page_index * scale → kernel page index. + int getPageIndexScale(LayerId layerId, DataRole role) const; + + // Composite converter (scale + expansion). + PageIndexConverter getPageIndexConverter(LayerId layerId, DataRole role) const; + + // Group a set of BufferIds into contiguous AggregatedPageDesc descriptors. + // Mirrors Python's KVCacheManager.get_aggregated_pages(). + std::vector<AggregatedPageDesc> getAggregatedPages(std::vector<BufferId> const& buffers) const; + + TypedVec<PoolGroupIndex, PoolGroupDesc> poolGroupDescs() const; + + // ---- Query / info ------------------------------------------------------ + + int tokensPerBlock() const noexcept; + bool enablePartialMatch() const noexcept; + + bool commitMinSnapshot() const noexcept + { + return mConfig.commitMinSnapshot; + } + + bool isSwaScratchReuseEnabled() const noexcept + { + return mConfig.enableSwaScratchReuse(); + } + + // Whether managed KV caches support the given page index mode. + // Returns true/false for a definitive answer, nullopt for per-instance check. + std::optional<bool> supportsIndexMode(PageIndexMode mode) const; + + bool allowSeqRebasing() const noexcept + { + return true; + } + + int numLayers() const noexcept; + + std::vector<LayerId> layerIds() const; + LayerGroupId getLayerGroupId(LayerId layerId) const; + + // Layer grouping: layers with the same lifecycle share pool allocation. + // NOTE: the iteration order of the layer lists (and of the groups) is NOT + // part of the API contract and may differ across backends/runs. Do not rely + // on it to infer buffer/pool memory order — query poolGroupDescs() + // (PoolGroupDesc::pools[i].baseAddress + coalescedBuffers) for that. + TypedVec<LayerGroupId, std::vector<LayerId>> layerGrouping() const; + + // Iterator over all buffer identifiers. Mirrors Python's all_buffer_ids property. + std::vector<BufferId> allBufferIds() const; + + // Sorted by CacheLevel from warm to cold. Mirrors Python's cache_tier_list property. + TypedVec<CacheLevel, CacheTier> cacheTierList() const; + + // Get the max possible sequence length limited by GPU memory pools. + // Mirrors Python's clamp_max_seq_len_for_mem(). + int clampMaxSeqLenForMem(int batchSize, int tokenNumUpperBound) const; + + // ---- Resize ----------------------------------------------------------- + + bool resize(CacheLevel level, size_t quota, bool bestEfforts = false); + size_t getQuota(CacheLevel level) const; + + // ---- Statistics ------------------------------------------------------- + + void commitStats(KVCacheStatsDelta const& stats, IterationStatsByLifeCycle const& iterationStatsByLifeCycle = {}); + KVCacheStatsDelta getCommittedStats() const; + IterationStatsByLifeCycle getAndResetIterationStats(); + PeakBlockStatsByPoolGroup getAndResetIterationPeakBlockStats(CacheLevel cacheLevel); + + void commitSsmSnapshotIterationStats(SsmSnapshotIterationStatsByLifeCycle const& statsByLifeCycle); + SsmSnapshotIterationStatsByLifeCycle getAndResetSsmSnapshotIterationStats(); + + void markStatsDirty(std::optional<RequestIdType> kvCacheId); + void clearStatsDirty(std::optional<RequestIdType> kvCacheId); + std::unordered_set<RequestIdType> getDirtyStatsKvCacheIds() const; + void markStatsExcluded(std::optional<RequestIdType> kvCacheId); + void clearStatsExcluded(std::optional<RequestIdType> kvCacheId); + bool isStatsExcluded(std::optional<RequestIdType> kvCacheId) const; + + // Mirrors Python's need_adjustment property and adjust() method. + // All KvCaches must be suspended before calling adjust(). + bool needAdjustment() const; + void adjust(); + + // ---- Internals used by KvCache ---------------------------------------- + + StorageManager& storage() noexcept + { + return *mStorage; + } + + KVCacheManagerConfig const& config() const noexcept + { + return mConfig; + } + + LifeCycleRegistry const& lifeCycles() const noexcept + { + return mLifeCycles; + } + + BlockRadixTree& radixTree() noexcept + { + return *mRadixTree; + } + + std::shared_ptr<EventSink> const& eventSink() const noexcept + { + return mEventSink; + } + + // Called by KvCache constructor/destructor. + void registerKvCache(KvCache* kvc); + void unregisterKvCache(KvCache* kvc); + + // Moving-average updates from closed KvCaches. + void updateAvgReusedLength(double v) + { + mAvgReusedLength.update(v); + } + + void updateAvgSqrCapacity(double v) + { + mAvgSqrCapacity.update(v); + } + + void updateAvgSqrHistoryLength(double v) + { + mAvgSqrHistoryLength.update(v); + } + + void incrementNumSampledKvCaches() + { + ++mNumSampledKvCaches; + } + + // Try to rebalance memory pool ratios based on usage statistics. + void tryUpdateTargetRatios(); + + // White-box introspection (incl. test-only auto-tuner state mutation) reaches + // private members directly rather than widening the public API. + friend class KvCacheIntrospection; + +private: + void _adjustLevel(CacheLevel level, size_t quota); + bool _needAdjustment(CacheLevel level) const; + TypedVec<PoolGroupIndex, float> const& _getTargetRatioList(CacheLevel level) const; + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> _gatherPersistentPages() const; + + PeakBlockStatsByCacheLevel _currentBlockStatsByCacheLevel() const; + void _resetIterationPeakNumBlocks(std::optional<CacheLevel> cacheLevel = std::nullopt); + void _updateIterationPeakNumBlocks(); + + // Current per-pool-group GPU utilization ratios. + TypedVec<PoolGroupIndex, float> _currentGpuRatio() const; + TypedVec<PoolGroupIndex, float> _currentOtherRatios() const; + + KVCacheManagerConfig mConfig; + LifeCycleRegistry mLifeCycles; + std::shared_ptr<EventSink> mEventSink; + std::shared_ptr<BlockRadixTree> mRadixTree; + std::shared_ptr<StorageManager> mStorage; + + // Weak references to all living KvCaches. + std::set<KvCache*> mLivingKvCaches; + + // Moving averages used for ratio rebalancing. + MovingAverage mAvgReusedLength; + MovingAverage mAvgSqrCapacity; + MovingAverage mAvgSqrHistoryLength; + + TypedVec<PoolGroupIndex, float> mTargetRatioListGpu; + TypedVec<PoolGroupIndex, float> mTargetRatioListOther; + + int mNumCreatedKvCaches{0}; + int mNumSampledKvCaches{0}; + double mLastAdjustmentTime{0.0}; + int mLastUpdateNumSampledKvCaches{0}; + + KVCacheStatsDelta mCommittedStats; + IterationStatsByLifeCycle mIterationStatsByLifeCycle; + SsmSnapshotIterationStatsByLifeCycle mSsmSnapshotIterationStatsByLifeCycle; + PeakBlockStatsByCacheLevel mIterationPeakNumBlocksByCacheLevel; + std::unordered_set<RequestIdType> mDirtyStatsKvCacheIds; + std::unordered_set<RequestIdType> mStatsExcludedKvCacheIds; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/lifeCycleRegistry.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/lifeCycleRegistry.cpp new file mode 100644 index 000000000000..c26e77f61b2f --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/lifeCycleRegistry.cpp @@ -0,0 +1,99 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/lifeCycleRegistry.h" + +#include "tensorrt_llm/common/assert.h" +#include <stdexcept> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// makeLifeCycle — factory dispatching on LayerConfig variant. +// --------------------------------------------------------------------------- + +LifeCycle makeLifeCycle(LayerConfig const& layer, int tokensPerBlock) +{ + return std::visit( + [&](auto const& cfg) -> LifeCycle + { + cfg.validate(); + using T = std::decay_t<decltype(cfg)>; + if constexpr (std::is_same_v<T, SsmLayerConfig>) + return SsmLifeCycle{}; + else + return AttnLifeCycle::make(cfg.slidingWindowSize, cfg.numSinkTokens, tokensPerBlock); + }, + layer); +} + +// --------------------------------------------------------------------------- +// LifeCycleRegistry +// --------------------------------------------------------------------------- + +LifeCycleRegistry::LifeCycleRegistry(KVCacheManagerConfig const& config) +{ + for (auto const& layer : config.layers) + { + LifeCycle lc = makeLifeCycle(layer, config.tokensPerBlock); + if (mLifeCycleIdMap.find(lc) == mLifeCycleIdMap.end()) + { + check(); + LifeCycleId id = mLifeCycleList.size(); + mLifeCycleList.push_back(lc); + mLifeCycleIdMap[lc] = id; + if (std::holds_alternative<SsmLifeCycle>(lc)) + mSsmLifeCycleId = id; + } + } + check(); +} + +LifeCycle const& LifeCycleRegistry::operator[](LifeCycleId id) const +{ + return mLifeCycleList.at(id); +} + +LifeCycle const& LifeCycleRegistry::getLifeCycle(LifeCycleId id) const +{ + return (*this)[id]; +} + +LifeCycleId LifeCycleRegistry::getId(LifeCycle const& lc) const +{ + auto it = mLifeCycleIdMap.find(lc); + if (it == mLifeCycleIdMap.end()) + { + throw std::out_of_range("LifeCycleRegistry::getId: life cycle not found"); + } + return it->second; +} + +LifeCycleId LifeCycleRegistry::size() const noexcept +{ + check(); + return mLifeCycleList.size(); +} + +inline void LifeCycleRegistry::check() const +{ + TLLM_CHECK_DEBUG_WITH_INFO( + toSizeT(mLifeCycleList.size()) == mLifeCycleIdMap.size(), "corrupted life cycle registry"); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/lifeCycleRegistry.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/lifeCycleRegistry.h new file mode 100644 index 000000000000..f655fa7ce092 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/lifeCycleRegistry.h @@ -0,0 +1,251 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/config.h" +#include "kv_cache_manager_v2/utils/math.h" +#include "tensorrt_llm/common/assert.h" + +#include <algorithm> +#include <map> +#include <optional> +#include <utility> +#include <variant> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// AttnLifeCycle — lifecycle for attention layers (SWA + sink blocks). +// Mirrors _life_cycle_registry.py::AttnLifeCycle. +// --------------------------------------------------------------------------- +struct AttnLifeCycle +{ + std::optional<int> windowSize; // nullopt = no sliding window + int numSinkBlocks = 0; // divUp(numSinkTokens, tokensPerBlock) + + HalfOpenRange<BlockOrdinal> getStaleRange(int historyLength, int tokensPerBlock) const + { + int numBlocks = divUp(historyLength, tokensPerBlock); + BlockOrdinal start{std::min(numBlocks, numSinkBlocks)}; + if (!windowSize.has_value()) + return {start, start}; + // `+ 1` is intentional: attention always runs for >= 1 in-flight input + // token at position `historyLength`, so the live window is + // [historyLength + 1 - windowSize, historyLength]. Do not drop it. + BlockOrdinal windowStart{(historyLength + 1 - *windowSize) / tokensPerBlock}; + return {start, std::max(start, windowStart)}; + } + + bool operator==(AttnLifeCycle const& o) const noexcept + { + return windowSize == o.windowSize && numSinkBlocks == o.numSinkBlocks; + } + + bool operator<(AttnLifeCycle const& o) const noexcept + { + if (windowSize != o.windowSize) + return windowSize < o.windowSize; + return numSinkBlocks < o.numSinkBlocks; + } + + static AttnLifeCycle make(std::optional<int> ws, std::optional<int> numSinkTokens, int tokensPerBlock) + { + TLLM_CHECK_DEBUG(tokensPerBlock > 0); + TLLM_CHECK_DEBUG(!ws.has_value() || *ws > 0); + TLLM_CHECK_DEBUG(!numSinkTokens.has_value() || *numSinkTokens >= 0); + TLLM_CHECK_DEBUG((!numSinkTokens.has_value() || *numSinkTokens == 0) || ws.has_value()); + int sinkBlocks = divUp(numSinkTokens.value_or(0), tokensPerBlock); + return AttnLifeCycle{ws, sinkBlocks}; + } +}; + +// --------------------------------------------------------------------------- +// SsmLifeCycle — lifecycle for SSM (State Space Model) layers. +// All blocks before the last full block are stale (recurrent state). +// Singleton: all SSM layers share the same lifecycle. +// --------------------------------------------------------------------------- +struct SsmLifeCycle +{ + HalfOpenRange<BlockOrdinal> getStaleRange(int historyLength, int tokensPerBlock) const + { + return {BlockOrdinal{0}, BlockOrdinal{historyLength / tokensPerBlock}}; + } + + bool operator==(SsmLifeCycle const&) const noexcept + { + return true; + } + + bool operator<(SsmLifeCycle const&) const noexcept + { + return false; + } +}; + +// --------------------------------------------------------------------------- +// LifeCycle — variant of attention or SSM lifecycle. +// --------------------------------------------------------------------------- +using LifeCycle = std::variant<AttnLifeCycle, SsmLifeCycle>; + +// Free function: compute stale range via std::visit. +inline HalfOpenRange<BlockOrdinal> getStaleRange(LifeCycle const& lc, int historyLength, int tokensPerBlock) +{ + return std::visit([&](auto const& v) { return v.getStaleRange(historyLength, tokensPerBlock); }, lc); +} + +// Compute the range of blocks that should use scratch (shared) slots during SWA prefill. +// Scratch = stale_at_capacity ∩ input_blocks, where: +// stale_at_capacity: blocks out-of-window when all capacity tokens become history. +// input_blocks: [divUp(historyLength, tpb), divUp(capacity, tpb)) — new blocks +// for the current chunk. +// Mirrors _life_cycle_registry.py::compute_scratch_range(). +inline HalfOpenRange<BlockOrdinal> computeScratchRange( + LifeCycle const& lc, int historyLength, int capacity, int tokensPerBlock, int maxRewindLen) +{ + auto const* attn = std::get_if<AttnLifeCycle>(&lc); + if (!attn || !attn->windowSize.has_value()) + { + return {BlockOrdinal{0}, BlockOrdinal{0}}; + } + int const nonRewindableCapacity = std::max(0, capacity - maxRewindLen); + auto capStale = attn->getStaleRange(nonRewindableCapacity, tokensPerBlock); + HalfOpenRange<BlockOrdinal> inputRange{divUp(historyLength, tokensPerBlock), divUp(capacity, tokensPerBlock)}; + return intersect(capStale, inputRange); +} + +// Factory: create a LifeCycle from a LayerConfig variant. +LifeCycle makeLifeCycle(LayerConfig const& layer, int tokensPerBlock); + +// Integer id assigned to each unique LifeCycle. +using LifeCycleId = StrongIndex<int, struct LifeCycleIdTag>; + +// Alias for public exposure (same meaning as LifeCycleId). +using LayerGroupId = LifeCycleId; + +// --------------------------------------------------------------------------- +// LifeCycleRegistry — deduplicates LifeCycle objects and assigns integer ids. +// Mirrors _life_cycle_registry.py::LifeCycleRegistry. +// --------------------------------------------------------------------------- +class LifeCycleRegistry +{ +public: + explicit LifeCycleRegistry(KVCacheManagerConfig const& config); + + // Look up a LifeCycle by id. + LifeCycle const& operator[](LifeCycleId id) const; + LifeCycle const& getLifeCycle(LifeCycleId id) const; + + // Look up the id for a LifeCycle (throws if not found). + LifeCycleId getId(LifeCycle const& lc) const; + + // Number of unique life cycles. + LifeCycleId size() const noexcept; + + // Iteration over LifeCycles in registration order. + TypedVec<LifeCycleId, LifeCycle> const& getAll() const noexcept + { + return mLifeCycleList; + } + + bool contains(LifeCycle const& lc) const noexcept + { + return mLifeCycleIdMap.count(lc) > 0; + } + + // SSM helpers. + std::optional<LifeCycleId> ssmLifeCycleId() const noexcept + { + return mSsmLifeCycleId; + } + + bool hasSSM() const noexcept + { + return mSsmLifeCycleId.has_value(); + } + + // Return (id, AttnLifeCycle*) pairs for attention lifecycles only. Used by _setupForReuse. + std::vector<std::pair<LifeCycleId, AttnLifeCycle const*>> attentionLifeCycles() const + { + std::vector<std::pair<LifeCycleId, AttnLifeCycle const*>> result; + for (LifeCycleId lcId{0}; lcId < mLifeCycleList.size(); ++lcId) + { + if (auto const* attn = std::get_if<AttnLifeCycle>(&mLifeCycleList[lcId])) + result.emplace_back(lcId, attn); + } + return result; + } + + // Iterate: (id, lifecycle) pairs — all entries. + struct Item + { + LifeCycleId id; + LifeCycle const& lc; + }; + + class ItemIterator + { + public: + ItemIterator(TypedVec<LifeCycleId, LifeCycle> const& list, LifeCycleId pos) + : mList(&list) + , mPos(pos) + { + } + + Item operator*() const + { + return {mPos, (*mList)[mPos]}; + } + + ItemIterator& operator++() + { + ++mPos; + return *this; + } + + bool operator!=(ItemIterator const& o) const + { + return mPos != o.mPos; + } + + private: + TypedVec<LifeCycleId, LifeCycle> const* mList; + LifeCycleId mPos; + }; + + ItemIterator begin() const + { + return {mLifeCycleList, LifeCycleId{0}}; + } + + ItemIterator end() const + { + return {mLifeCycleList, mLifeCycleList.size()}; + } + +private: + void check() const; + + TypedVec<LifeCycleId, LifeCycle> mLifeCycleList; + std::map<LifeCycle, LifeCycleId> mLifeCycleIdMap; + std::optional<LifeCycleId> mSsmLifeCycleId; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/movingAverage.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/movingAverage.h new file mode 100644 index 000000000000..c2f53af570ff --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/movingAverage.h @@ -0,0 +1,99 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/common/assert.h" + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// Exponential moving average (mirrors _core/_moving_average.py::MovingAverage). +// weight-corrected bias: avg += (value - avg) / weight, weight = 1 + decay*weight +// --------------------------------------------------------------------------- +class MovingAverage +{ +public: + explicit MovingAverage(double decay = 0.9999) noexcept + : mDecay(decay) + , mAvg(0.0) + , mWeight(0.0) + , mNumUpdates(0) + { + } + + double update(double value) noexcept + { + mWeight = 1.0 + mDecay * mWeight; + mAvg += (value - mAvg) / mWeight; + ++mNumUpdates; + return mAvg; + } + + double value() const noexcept + { + return mAvg; + } + + int numUpdates() const noexcept + { + return mNumUpdates; + } + +private: + double mDecay; + double mAvg; + double mWeight; + int mNumUpdates; +}; + +// --------------------------------------------------------------------------- +// Simple arithmetic mean (mirrors _core/_moving_average.py::Average). +// --------------------------------------------------------------------------- +class Average +{ +public: + Average() noexcept + : mSum(0.0) + , mCount(0) + { + } + + void update(double value) noexcept + { + mSum += value; + ++mCount; + } + + double value() const noexcept + { + TLLM_CHECK_DEBUG(mCount > 0); + return mSum / mCount; + } + + int count() const noexcept + { + return mCount; + } + +private: + double mSum; + int mCount; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/page.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/page.cpp new file mode 100644 index 000000000000..91f9921df948 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/page.cpp @@ -0,0 +1,570 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/page.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/exceptions.h" +#include "kv_cache_manager_v2/kvCache.h" // for KvCache +#include "kv_cache_manager_v2/storageManager.h" // for StorageManager + +#include "tensorrt_llm/common/assert.h" +#include <stdexcept> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// Page +// --------------------------------------------------------------------------- + +Page::Page(StorageManager* mgr, LifeCycleId lc, CacheLevel level, Priority prio) + : manager(mgr) + , lifeCycle(lc) + , cacheLevel(level) + , priority(prio) + , nodeRef(std::nullopt) +{ +} + +Page::~Page() +{ + TLLM_CHECK_DEBUG_WITH_INFO(status() == PageStatus::DROPPABLE && !scheduledForEviction(), + "Page destroyed while still held or scheduled for eviction"); + if (hasValidSlot()) + { + Slot s; + s.setSlotId(slotId()); + s.readyEvent = std::move(readyEvent); + resetSlot(); + manager->releaseSlot(lifeCycle, cacheLevel, std::move(s)); + } +} + +PageStatus Page::status() const noexcept +{ + auto h = holder.lock(); + if (!h) + return PageStatus::DROPPABLE; + if (h->uniqLock.expired()) + return PageStatus::HELD; + return PageStatus::LOCKED; +} + +SharedPtr<PageHolder> Page::hold() +{ + // Return existing holder if any. + auto h = holder.lock(); + if (h) + return h; + + auto self = sharedFromThis(); + h = makeShared<PageHolder>(self); + holder = h; + + // If we were scheduled for eviction but are no longer evictable (just got held), remove. + if (scheduledForEviction()) + { + if (!manager->isEvictable(*this)) + { + manager->excludeFromEviction(*this); + TLLM_CHECK_DEBUG(!scheduledForEviction()); + } + } + return h; +} + +SharedPageLock Page::lock(KvCache& kvCache, BeamIndex beamIndex, BlockOrdinal ordinal, LifeCycleId lc, bool skipWait) +{ + return hold()->lock(kvCache, beamIndex, ordinal, lc, skipWait); +} + +// --------------------------------------------------------------------------- +// CommittedPage +// --------------------------------------------------------------------------- + +CommittedPage::CommittedPage(StorageManager* mgr, SharedPtr<Block> blk, LifeCycleId lc, CacheLevel level, Priority prio) + : Page(mgr, lc, level, prio) + , block(blk.get()) +{ +} + +SsmCommittedPage::SsmCommittedPage( + StorageManager* mgr, SharedPtr<Block> blk, LifeCycleId lc, CacheLevel level, Priority prio, int numTokensInBlock_) + : CommittedPage(mgr, std::move(blk), lc, level, prio) + , numTokensInBlock(numTokensInBlock_) +{ + TLLM_CHECK_DEBUG(numTokensInBlock_ > 0); +} + +CommittedPage::~CommittedPage() +{ + if (block != nullptr) + { + // unlinkPage nulls storage[lc]->block (i.e. our member), so capture + // the block pointer first. Pass `this` as the expected page: if a newer + // page already replaced us in the slot (e.g. a larger SSM snapshot), the + // slot is left alone and prev is nullptr — skip stale-block cleanup then. + Block* blk = block; + auto* prev = blk->unlinkPage(lifeCycle, this); + if (prev != nullptr) + { + TLLM_CHECK_DEBUG_WITH_INFO(prev == this, "unlinkPage returned unexpected page"); + LifeCycle const& lc = manager->lifeCycles().getLifeCycle(lifeCycle); + Block::clearStaleBlocksAfterPageUnlink(*blk, lifeCycle, lc); + } + } + // Delegate slot release to Page::~Page(). +} + +// --------------------------------------------------------------------------- +// UncommittedPage +// --------------------------------------------------------------------------- + +UncommittedPage::UncommittedPage(KvCache& kvc, BlockOrdinal ord, LifeCycleId lc, CacheLevel level, BeamIndex bi) + : Page(kvc.storageManager(), lc, level, kvc.getPriority(ord, lc)) + , kvCache(&kvc) + , ordinal(ord) + , beamIndex(bi) +{ +} + +UncommittedPage::~UncommittedPage() +{ + // Mirrors Python UncommittedPage.__del__: for attention LCs, the page must be either: + // - part of an SSM lifecycle (different rules), + // - at an ordinal beyond the current block list (block already removed), + // - the slot at this position is null, CommittedPage, or this page itself (self-destruction). + // The "p == this" condition is C++-specific: std::variant destroys the old value before + // switching to monostate, so during destruction the slot still references this page. + if (TLLM_UNLIKELY(gDebug)) + { + auto ssmLcId = manager->lifeCycles().ssmLifeCycleId(); + bool isSsm = ssmLcId.has_value() && lifeCycle == *ssmLcId; + if (!isSsm) + { + [[maybe_unused]] bool blockRemoved = kvCache->blocks().size() <= ordinal; + [[maybe_unused]] bool pageOk = true; + if (!blockRemoved) + { + auto const& bp = kvCache->blocks()[ordinal].pages[beamIndex][lifeCycle]; + auto page = blockPageGetPage(bp); + pageOk + = blockPageIsNull(bp) || page.get() == this || dynamicPointerCast<CommittedPage>(page) != nullptr; + } + TLLM_CHECK_WITH_INFO( + blockRemoved || pageOk, "UncommittedPage destroyed but slot still holds a different uncommitted page"); + } + } + // Delegate slot release to Page::~Page(). +} + +SharedPtr<CommittedPage> UncommittedPage::convertToCommitted(SharedPtr<Block> blk, CachedCudaEvent readyEv) +{ + TLLM_CHECK_DEBUG(!scheduledForEviction()); + TLLM_CHECK_DEBUG_WITH_INFO( + blk->storage.at(lifeCycle) == nullptr, "Block slot for this lifecycle already has a committed page"); + TLLM_CHECK_DEBUG_WITH_INFO(status() == PageStatus::DROPPABLE, "Release holder/lock before converting"); + + // Set the ready event before transfer (matches Python: self.ready_event = ready_event). + this->readyEvent = std::move(readyEv); + + auto committed = makeShared<CommittedPage>(manager, blk, lifeCycle, cacheLevel, priority); + // Move slot id to the committed page; invalidate our slot. + committed->setSlotId(slotId()); // asserts valid + committed->readyEvent = std::move(readyEvent); + resetSlot(); + readyEvent = CachedCudaEvent::makeNull(); + + TLLM_CHECK_DEBUG(!hasValidSlot() && readyEvent.isClosed()); + TLLM_CHECK_DEBUG_WITH_INFO(committed->hasValidSlot(), "committed page must have a valid slot after transfer"); + + // Register in block storage. + blk->storage.at(lifeCycle) = committed.get(); + + return committed; +} + +SharedPtr<SsmCommittedPage> UncommittedPage::convertToSsmCommitted( + SharedPtr<Block> blk, CachedCudaEvent readyEv, int numTokensInBlock) +{ + TLLM_CHECK_DEBUG(!scheduledForEviction()); + TLLM_CHECK_DEBUG_WITH_INFO( + blk->storage.at(lifeCycle) == nullptr, "Block slot for this lifecycle already has a committed page"); + TLLM_CHECK_DEBUG_WITH_INFO(status() == PageStatus::DROPPABLE, "Release holder/lock before converting"); + + this->readyEvent = std::move(readyEv); + + auto committed = makeShared<SsmCommittedPage>(manager, blk, lifeCycle, cacheLevel, priority, numTokensInBlock); + committed->setSlotId(slotId()); // asserts valid + committed->readyEvent = std::move(readyEvent); + resetSlot(); + readyEvent = CachedCudaEvent::makeNull(); + + TLLM_CHECK_DEBUG(!hasValidSlot() && readyEvent.isClosed()); + TLLM_CHECK_DEBUG_WITH_INFO(committed->hasValidSlot(), "committed page must have a valid slot after transfer"); + + blk->storage.at(lifeCycle) = committed.get(); + + return committed; +} + +// --------------------------------------------------------------------------- +// PageHolder +// --------------------------------------------------------------------------- + +PageHolder::PageHolder(SharedPtr<Page> p) + : page(std::move(p)) +{ +} + +PageHolder::~PageHolder() +{ + TLLM_CHECK_DEBUG_WITH_INFO(uniqLock.expired(), "PageHolder destroyed while lock still active"); + + page->holder.reset(); // clear back-reference + auto const manager = page->manager; + + // If it's a committed page, schedule for eviction (if evictable). + if (page->isCommitted()) + { + if (!page->scheduledForEviction()) + manager->scheduleForEviction(*page); + + // If the block is orphan, exclude from eviction immediately. + auto* cp = dynamic_cast<CommittedPage*>(page.get()); + if (cp) + { + if (cp->block == nullptr || cp->block->isOrphan()) + manager->excludeFromEviction(*page); + } + } + else + { + // Uncommitted page: if scheduled for eviction, remove it. + if (page->scheduledForEviction()) + manager->excludeFromEviction(*page); + } +} + +SharedPageLock PageHolder::lock( + KvCache& kvCache, BeamIndex beamIndex, BlockOrdinal ordinal, LifeCycleId lc, bool skipWait) +{ + // Create or reuse UniqPageLock. + auto ul = uniqLock.lock(); + if (!ul) + { + ul = makeShared<UniqPageLock>(sharedFromThis()); + uniqLock = ul; + } + + // Remove from eviction queue if scheduled. + if (page->scheduledForEviction()) + { + page->manager->excludeFromEviction(*page); + TLLM_CHECK_DEBUG(!page->scheduledForEviction()); + } + + return ul->share(kvCache, beamIndex, ordinal, lc, skipWait); +} + +// --------------------------------------------------------------------------- +// UniqPageLock +// --------------------------------------------------------------------------- + +UniqPageLock::UniqPageLock(SharedPtr<PageHolder> h) + : holder(std::move(h)) +{ + if (holder->page->cacheLevel != kGpuLevel) + throw LogicError("Lock can only be applied to GPU-memory pages"); +} + +UniqPageLock::~UniqPageLock() +{ + Page& p = *page(); + TLLM_CHECK_DEBUG(p.cacheLevel == kGpuLevel && !p.scheduledForEviction()); + // Set readyEvent to the merged finish events of all readers. For committed (read-only) + // pages, this means the next reader will wait for prior reads to complete, which is + // unnecessary but correct. See the CommittedPage comment in page.h for rationale. + p.readyEvent = mergeEvents(finishEvents); + + // Clear the holder's lock reference. + TLLM_CHECK_DEBUG(holder); + holder->uniqLock.reset(); + + // Optimized path (mirrors Python): set holder=nullptr, then check if still evictable. + auto holderCopy = std::move(holder); + holder = nullptr; + + // If the page is not droppable (still held by someone else) and evictable, + // schedule for eviction. + if (p.status() != PageStatus::DROPPABLE) + { + auto const manager = p.manager; + if (manager->isEvictable(p)) + manager->scheduleForEviction(p); + } +} + +void UniqPageLock::notifyFinish(CachedCudaEvent event) +{ + finishEvents.push_back(std::move(event)); + // Avoid unbounded growth for system prompt pages shared by all requests. + if (finishEvents.size() > 32) + { + CachedCudaEvent merged = mergeEvents(finishEvents); + finishEvents.clear(); + finishEvents.push_back(std::move(merged)); + } +} + +SharedPtr<Page> const& UniqPageLock::page() const +{ + TLLM_CHECK_DEBUG(holder && holder->page); + return holder->page; +} + +SharedPageLock UniqPageLock::share( + KvCache& kvCache, BeamIndex beamIndex, BlockOrdinal ordinal, LifeCycleId lc, bool skipWait) +{ + return SharedPageLock(sharedFromThis(), kvCache, beamIndex, ordinal, lc, skipWait); +} + +// --------------------------------------------------------------------------- +// SharedPageLock +// --------------------------------------------------------------------------- + +SharedPageLock::SharedPageLock(SharedPtr<UniqPageLock> ul, KvCache& kvCache, BeamIndex beamIndex, BlockOrdinal ordinal, + LifeCycleId lc, bool skipWait) + : mUniqLock(std::move(ul)) + , mUser{&kvCache, beamIndex, ordinal, lc} +{ + if (!skipWait) + page()->readyEvent.waitInStream(reinterpret_cast<CudaStream>(kvCache.cudaStream())); + + acquirePageIndex(); +} + +SharedPageLock::~SharedPageLock() +{ + if (mUniqLock) + unlock(); +} + +SharedPageLock::SharedPageLock(SharedPageLock&& other) noexcept + : mUniqLock(std::move(other.mUniqLock)) + , mUser(std::move(other.mUser)) +{ +} + +SharedPageLock& SharedPageLock::operator=(SharedPageLock&& other) noexcept +{ + if (this != &other) + { + if (mUniqLock) + unlock(); + mUniqLock = std::move(other.mUniqLock); + mUser = std::move(other.mUser); + } + return *this; +} + +SharedPtr<Page> const& SharedPageLock::page() const +{ + TLLM_CHECK_DEBUG(mUniqLock); + return mUniqLock->page(); +} + +SharedPtr<Page> SharedPageLock::unlock() +{ + TLLM_CHECK_DEBUG(mUniqLock); + + // Record finish event from the KvCache stream. + mUniqLock->notifyFinish(mUser.kvCache->finishEvent()); + + releasePageIndex(); + auto p = page(); // copy shared_ptr before reset + mUniqLock.reset(); + return p; +} + +void SharedPageLock::acquirePageIndex() +{ + auto* kvc = mUser.kvCache; + auto& pg = *page(); + int old = kvc->updateBasePageIndex( + mUser.beamIndex, mUser.ordinal, mUser.lifeCycle, slotIdToPageIndexValue(pg.slotId())); + // Mirrors Python assertion: old base index must be BAD (prevents double-locking same slot). + TLLM_CHECK_DEBUG_WITH_INFO( + old == kBadPageIndex.value(), "Double-lock: page index already acquired for this (beam, ordinal, lc)"); + (void) old; +} + +void SharedPageLock::releasePageIndex() +{ + int oldBaseIndex + = mUser.kvCache->updateBasePageIndex(mUser.beamIndex, mUser.ordinal, mUser.lifeCycle, kBadPageIndex.value()); + // Mirrors Python assertion: old base index must match this page's slot ID. + TLLM_CHECK_DEBUG(oldBaseIndex == slotIdToPageIndexValue(page()->slotId())); + (void) oldBaseIndex; +} + +// --------------------------------------------------------------------------- +// batchedLockToGpu +// --------------------------------------------------------------------------- + +std::vector<SharedPageLock> batchedLockToGpu(KvCache& kvCache, std::vector<BatchedLockTarget> const& targets) +{ + auto* storeMgr = kvCache.storageManager(); + TLLM_CHECK_DEBUG(storeMgr); + // All pages must belong to the same storage manager. + TLLM_CHECK_DEBUG(targets.empty() + || std::all_of(targets.begin(), targets.end(), [&](auto const& t) { return t.page->manager == storeMgr; })); + + // Determine how many GPU slots are needed per pool group. + TypedVec<PoolGroupIndex, SlotCount> requirements(storeMgr->numPoolGroups(), 0); + std::vector<bool> wasScheduled(targets.size(), false); + + for (size_t i = 0; i < targets.size(); ++i) + { + auto const& t = targets[i]; + wasScheduled[i] = t.page->scheduledForEviction(); + if (wasScheduled[i]) + storeMgr->excludeFromEviction(*t.page); + if (t.page->cacheLevel != kGpuLevel) + { + PoolGroupIndex pgIdx = storeMgr->getPoolGroupIndex(t.lifeCycle); + requirements[pgIdx] += 1; + } + } + + try + { + MigrationRecorder const migrationRecorder + = [&kvCache](std::vector<SharedPtr<Page>> const& pages, std::vector<Slot> const& slots, CacheLevel srcLevel, + CacheLevel dstLevel) { kvCache._recordMigratedSlots(pages, slots, srcLevel, dstLevel); }; + DropRecorder const dropRecorder = [&kvCache](std::vector<SharedPtr<Page>> const& pages, CacheLevel cacheLevel) + { kvCache._recordDroppedPages(pages, cacheLevel); }; + storeMgr->prepareFreeSlots(kGpuLevel, requirements, migrationRecorder, dropRecorder); + // Migrate non-GPU pages. + storeMgr->batchedMigrateToGpu(targets, kvCache, migrationRecorder); + } + catch (...) + { + // Restore eviction scheduling. + for (size_t i = 0; i < targets.size(); ++i) + if (wasScheduled[i]) + storeMgr->scheduleForEviction(*targets[i].page); + throw; + } + + // Wait for all ready events on KvCache's stream (deduplicated). + { + std::vector<CachedCudaEvent const*> readyEvents; + readyEvents.reserve(targets.size()); + for (auto const& t : targets) + readyEvents.push_back(&t.page->readyEvent); + streamWaitEvents(reinterpret_cast<CudaStream>(kvCache.cudaStream()), readyEvents); + } + + // Lock all pages. + std::vector<SharedPageLock> locks; + locks.reserve(targets.size()); + for (auto const& t : targets) + locks.emplace_back(t.page->lock(kvCache, t.beamIndex, t.ordinal, t.lifeCycle, + /*skipWait=*/true)); + return locks; +} + +// --------------------------------------------------------------------------- +// ScratchSlotLock +// --------------------------------------------------------------------------- + +ScratchSlotLock::ScratchSlotLock(Slot slot, KvCache& owner, LifeCycleId lifeCycle, bool skipWait) + : mOwner(&owner) + , mLifeCycle(lifeCycle) +{ + if (!skipWait) + { + slot.readyEvent.waitInStream(reinterpret_cast<CudaStream>(owner.cudaStream())); + } + mSlot.setSlot(slot); +} + +ScratchSlotLock::~ScratchSlotLock() +{ + if (mSlot.hasValidSlot()) + { + try + { + unlock(); + } + catch (...) + { + } + } +} + +ScratchSlotLock::ScratchSlotLock(ScratchSlotLock&& other) noexcept + : mSlot(std::move(other.mSlot)) + , mOwner(other.mOwner) + , mLifeCycle(other.mLifeCycle) +{ + // Invalidate moved-from: mSlot move only transfers readyEvent, not slotId (trivially-copyable). + other.mSlot.resetSlot(); + other.mOwner = nullptr; +} + +ScratchSlotLock& ScratchSlotLock::operator=(ScratchSlotLock&& other) noexcept +{ + if (this != &other) + { + if (mSlot.hasValidSlot()) + { + try + { + unlock(); + } + catch (...) + { + } + } + mSlot = std::move(other.mSlot); + other.mSlot.resetSlot(); // Invalidate moved-from slotId. + mOwner = other.mOwner; + mLifeCycle = other.mLifeCycle; + other.mOwner = nullptr; + } + return *this; +} + +Slot ScratchSlotLock::detachSlot() +{ + TLLM_CHECK_DEBUG(mSlot.hasValidSlot()); + Slot result; + result.setSlot(mSlot); + return result; +} + +void ScratchSlotLock::unlock() +{ + TLLM_CHECK_DEBUG(mSlot.hasValidSlot()); + mSlot.readyEvent = mOwner->finishEvent(); + mOwner->storageManager()->releaseSlot(mLifeCycle, kGpuLevel, std::move(mSlot)); + TLLM_CHECK_DEBUG(!mSlot.hasValidSlot()); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/page.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/page.h new file mode 100644 index 000000000000..9f52dca1f0ee --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/page.h @@ -0,0 +1,307 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/blockRadixTree.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/evictionController.h" +#include "kv_cache_manager_v2/lifeCycleRegistry.h" +#include "kv_cache_manager_v2/storage/core.h" +#include "kv_cache_manager_v2/utils/cudaEvent.h" +#include "kv_cache_manager_v2/utils/sharedPtr.h" + +#include <functional> +#include <optional> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// Forward declarations to break circular includes. +class StorageManager; +class KvCache; +class PageHolder; +class UniqPageLock; +class SharedPageLock; + +// --------------------------------------------------------------------------- +// Page — base class for all KV-cache pages. +// Inherits from Slot (holds slotId + readyEvent). +// Mirrors Python's Page(Slot) dataclass. +// --------------------------------------------------------------------------- +class Page : public Slot, public EnableSharedFromThis<Page> +{ +public: + StorageManager* manager; + LifeCycleId lifeCycle; + CacheLevel cacheLevel; + Priority priority; + WeakPtr<PageHolder> holder; // empty → DROPPABLE + std::optional<NodeRef> nodeRef; // present → scheduled for eviction + + Page(StorageManager* mgr, LifeCycleId lc, CacheLevel level, Priority prio); + + virtual ~Page(); + + virtual bool isCommitted() const = 0; + + PageStatus status() const noexcept; + + bool scheduledForEviction() const noexcept + { + return nodeRef.has_value(); + } + + // Prevent the page from being dropped (returns/creates a PageHolder). + SharedPtr<PageHolder> hold(); + + // Acquire a shared lock (migrates to GPU if needed). + // skip_wait: caller guarantees the page is ready on kvCache's stream. + SharedPageLock lock( + KvCache& kvCache, BeamIndex beamIndex, BlockOrdinal ordinal, LifeCycleId lifeCycle, bool skipWait = false); +}; + +// --------------------------------------------------------------------------- +// CommittedPage — page associated with a Block in the radix tree. +// +// A committed page is immutable — all access after commit is read-only. +// +// We intentionally do not add a separate read event to track read completion. +// The inherited Slot::readyEvent serves double duty: after commit or migration +// it represents write completion; after UniqPageLock is destroyed it is set to +// the merged finish events of all prior readers. This means a new reader may +// unnecessarily wait for a prior reader (read-after-read on immutable data), +// but this is functionally correct, only occurs when the lock is fully released +// between reuses, and saves one event field per committed page — a worthwhile +// tradeoff given the potentially huge number of committed pages in the system. +// --------------------------------------------------------------------------- +class CommittedPage : public Page +{ +public: + Block* block; + + // Number of outstanding PlannedDropHandles that intend to drop this page. + // Mirrors Python's CommittedPage.planned_drop_count. + int plannedDropCount{0}; + + CommittedPage(StorageManager* mgr, SharedPtr<Block> blk, LifeCycleId lc, CacheLevel level, Priority prio); + + ~CommittedPage() override; + + bool isCommitted() const override + { + return true; + } +}; + +// --------------------------------------------------------------------------- +// SsmCommittedPage — a committed SSM snapshot page. +// +// Unlike attention CommittedPages (which always cover a full block), an SSM +// snapshot may cover only a prefix of its block. `numTokensInBlock` records how +// many tokens of the block this snapshot is reusable for. +// --------------------------------------------------------------------------- +class SsmCommittedPage : public CommittedPage +{ +public: + int numTokensInBlock; + + SsmCommittedPage(StorageManager* mgr, SharedPtr<Block> blk, LifeCycleId lc, CacheLevel level, Priority prio, + int numTokensInBlock); +}; + +// --------------------------------------------------------------------------- +// UncommittedPage — page associated with a live KvCache sequence. +// --------------------------------------------------------------------------- +class UncommittedPage : public Page +{ +public: + KvCache* kvCache; + BlockOrdinal ordinal; + BeamIndex beamIndex; + std::vector<TokenIdExt> tokens; + + UncommittedPage(KvCache& kvc, BlockOrdinal ord, LifeCycleId lc, CacheLevel level, BeamIndex bi = kDefaultBeamIndex); + + ~UncommittedPage() override; + + bool isCommitted() const override + { + return false; + } + + // Convert this UncommittedPage into a CommittedPage and attach to `block`. + // The UncommittedPage becomes invalid (slot transferred to CommittedPage). + SharedPtr<CommittedPage> convertToCommitted(SharedPtr<Block> block, CachedCudaEvent readyEvent); + + // Convert this UncommittedPage into an SsmCommittedPage covering + // `numTokensInBlock` tokens and attach to `block`. Invalidates this page. + SharedPtr<SsmCommittedPage> convertToSsmCommitted( + SharedPtr<Block> block, CachedCudaEvent readyEvent, int numTokensInBlock); +}; + +// --------------------------------------------------------------------------- +// PageHolder — prevents a page from being dropped (HELD status). +// Mirrors Python's _PageHolder. +// --------------------------------------------------------------------------- +class PageHolder : public EnableSharedFromThis<PageHolder> +{ +public: + explicit PageHolder(SharedPtr<Page> page); + ~PageHolder(); + + PageHolder(PageHolder const&) = delete; + PageHolder& operator=(PageHolder const&) = delete; + + // Acquire a shared lock (creates or reuses the UniqPageLock). + SharedPageLock lock( + KvCache& kvCache, BeamIndex beamIndex, BlockOrdinal ordinal, LifeCycleId lifeCycle, bool skipWait = false); + + SharedPtr<Page> page; + WeakPtr<UniqPageLock> uniqLock; // non-null → LOCKED +}; + +// --------------------------------------------------------------------------- +// UniqPageLock — locks a page to prevent eviction (LOCKED status). +// Owns finish events from all SharedPageLocks it issued. +// Mirrors Python's _UniqPageLock. +// --------------------------------------------------------------------------- +class UniqPageLock : public EnableSharedFromThis<UniqPageLock> +{ +public: + explicit UniqPageLock(SharedPtr<PageHolder> holder); + ~UniqPageLock(); + + UniqPageLock(UniqPageLock const&) = delete; + UniqPageLock& operator=(UniqPageLock const&) = delete; + + // Issue a SharedPageLock to a specific (kvCache, beam, ordinal, lifecycle). + SharedPageLock share( + KvCache& kvCache, BeamIndex beamIndex, BlockOrdinal ordinal, LifeCycleId lifeCycle, bool skipWait); + + SharedPtr<Page> const& page() const; + + // Append a finish event, merging when count exceeds 32 to prevent unbounded growth. + void notifyFinish(CachedCudaEvent event); + + SharedPtr<PageHolder> holder; + std::vector<CachedCudaEvent> finishEvents; +}; + +// --------------------------------------------------------------------------- +// LockOwner — identifies who holds a SharedPageLock. +// --------------------------------------------------------------------------- +struct LockOwner +{ + KvCache* kvCache; + BeamIndex beamIndex; + BlockOrdinal ordinal; + LifeCycleId lifeCycle; +}; + +// --------------------------------------------------------------------------- +// SharedPageLock — one user's hold on an active page lock. +// Mirrors Python's _SharedPageLock. +// --------------------------------------------------------------------------- +class SharedPageLock +{ +public: + SharedPageLock(SharedPtr<UniqPageLock> uniqLock, KvCache& kvCache, BeamIndex beamIndex, BlockOrdinal ordinal, + LifeCycleId lifeCycle, bool skipWait); + + ~SharedPageLock(); + + SharedPageLock(SharedPageLock&&) noexcept; + SharedPageLock& operator=(SharedPageLock&&) noexcept; + + SharedPageLock(SharedPageLock const&) = delete; + SharedPageLock& operator=(SharedPageLock const&) = delete; + + // Explicitly release the lock (called by destructor if not already released). + SharedPtr<Page> unlock(); + + SharedPtr<Page> const& page() const; + + bool isValid() const noexcept + { + return mUniqLock != nullptr; + } + +private: + // Internal helpers that update KvCache page index tables. + void acquirePageIndex(); + void releasePageIndex(); + + SharedPtr<UniqPageLock> mUniqLock; + LockOwner mUser; +}; + +// --------------------------------------------------------------------------- +// BatchedLockTarget — input for batched_lock_to_gpu. +// --------------------------------------------------------------------------- +struct BatchedLockTarget +{ + SharedPtr<Page> page; + BeamIndex beamIndex; + BlockOrdinal ordinal; + LifeCycleId lifeCycle; +}; + +// --------------------------------------------------------------------------- +// batchedLockToGpu — migrate pages to GPU then lock them. +// Returns one SharedPageLock per target. +// Mirrors Python's batched_lock_to_gpu(). +// --------------------------------------------------------------------------- +std::vector<SharedPageLock> batchedLockToGpu(KvCache& kvCache, std::vector<BatchedLockTarget> const& targets); + +// --------------------------------------------------------------------------- +// ScratchSlotLock — manages a scratch slot for SWA prefill memory reuse. +// Wraps a Slot with owner (KvCache) and lifecycle references. +// On destruction, releases the slot back to the StorageManager. +// Mirrors _page.py::ScratchSlotLock. +// --------------------------------------------------------------------------- +class ScratchSlotLock +{ +public: + ScratchSlotLock(Slot slot, KvCache& owner, LifeCycleId lifeCycle, bool skipWait = false); + ~ScratchSlotLock(); + + ScratchSlotLock(ScratchSlotLock&& other) noexcept; + ScratchSlotLock& operator=(ScratchSlotLock&& other) noexcept; + + ScratchSlotLock(ScratchSlotLock const&) = delete; + ScratchSlotLock& operator=(ScratchSlotLock const&) = delete; + + // Detach and return the slot (transfers ownership to caller). + Slot detachSlot(); + + // Release the slot back to storage manager. + void unlock(); + + Slot const& slot() const noexcept + { + return mSlot; + } + +private: + Slot mSlot; + KvCache* mOwner; + LifeCycleId mLifeCycle; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/pendingStats.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/pendingStats.h new file mode 100644 index 000000000000..3ebb785dfb5f --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/pendingStats.h @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/stats.h" + +#include "tensorrt_llm/common/assert.h" +#include <algorithm> +#include <optional> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +struct PendingAllocationSegment +{ + LifeCycleId lifeCycle; + BlockOrdinal blockBegin; + BlockOrdinal blockEnd; + int beamWidth; + bool countAsMissed; + bool countAsGeneration; +}; + +struct PendingStatsDelta +{ + KVCacheStatsDelta globalStats; + KVCacheStatsDelta requestStats; + KVCacheIterationStatsDelta iterationStats; + std::optional<LifeCycleId> lifeCycle; + + [[nodiscard]] bool empty() const noexcept + { + return globalStats.empty() && requestStats.empty() && iterationStats.empty(); + } +}; + +class PendingStats +{ +public: + [[nodiscard]] bool empty() const noexcept + { + return mRequestStats.empty() && mGlobalStats.empty() && mIterationStatsByLifeCycle.empty() + && mSsmSnapshotIterationStatsByLifeCycle.empty(); + } + + void clear() noexcept + { + mRequestStats.clear(); + mGlobalStats.clear(); + mIterationStatsByLifeCycle.clear(); + mSsmSnapshotIterationStatsByLifeCycle.clear(); + mAllocationSegments.clear(); + } + + bool recordAllocationRange(LifeCycleId lifeCycle, BlockOrdinal blockBegin, BlockOrdinal blockEnd, int beamWidth, + bool countAsMissed, bool countAsGeneration = false) + { + if (blockBegin >= blockEnd) + { + return false; + } + PendingAllocationSegment segment{lifeCycle, blockBegin, blockEnd, beamWidth, countAsMissed, countAsGeneration}; + if (!add(allocationDelta(segment, blockBegin, blockEnd))) + { + return false; + } + mAllocationSegments.push_back(segment); + return true; + } + + bool recordReuse(LifeCycleId lifeCycle, int fullReusedBlocks, int partialReusedBlocks) + { + int const reusedBlocks = fullReusedBlocks + partialReusedBlocks; + if (reusedBlocks == 0) + { + return false; + } + + PendingStatsDelta delta; + delta.globalStats.reusedBlocks = reusedBlocks; + delta.requestStats.reusedBlocks = reusedBlocks; + delta.iterationStats.iterReusedBlocks = reusedBlocks; + delta.iterationStats.iterFullReusedBlocks = fullReusedBlocks; + delta.iterationStats.iterPartialReusedBlocks = partialReusedBlocks; + delta.lifeCycle = lifeCycle; + return add(delta); + } + + // Record one SSM snapshot lookup for a lifecycle. Mirrors Python's + // _PendingStats.record_ssm_snapshot_lookup(). Returns false if lookupTokens == 0. + bool recordSsmSnapshotLookup(LifeCycleId lifeCycle, int lookupTokens, int reusedTokens, int tokensPerBlock) + { + if (lookupTokens == 0) + { + return false; + } + TLLM_CHECK_DEBUG(lookupTokens > 0); + TLLM_CHECK_DEBUG(0 <= reusedTokens && reusedTokens <= lookupTokens); + TLLM_CHECK_DEBUG(tokensPerBlock > 0); + + bool const isHit = reusedTokens > 0; + SsmSnapshotIterationStatsDelta delta; + delta.iterSnapshotLookups = 1; + delta.iterSnapshotHits = isHit ? 1 : 0; + delta.iterSnapshotMisses = isHit ? 0 : 1; + delta.iterReusedTokens = reusedTokens; + delta.iterUnreusedTokens = lookupTokens - reusedTokens; + delta.iterAlignedSnapshotHits = (isHit && reusedTokens % tokensPerBlock == 0) ? 1 : 0; + delta.iterUnalignedSnapshotHits = (isHit && reusedTokens % tokensPerBlock != 0) ? 1 : 0; + mSsmSnapshotIterationStatsByLifeCycle[lifeCycle].add(delta); + return true; + } + + bool subtractAllocationRange(BlockOrdinal blockBegin, BlockOrdinal blockEnd) + { + if (blockBegin >= blockEnd || mAllocationSegments.empty()) + { + return false; + } + + bool changed = false; + int index = static_cast<int>(mAllocationSegments.size()) - 1; + while (index >= 0) + { + auto& segment = mAllocationSegments[static_cast<size_t>(index)]; + if (segment.blockEnd <= blockBegin) + { + break; + } + BlockOrdinal const removedBegin = std::max(blockBegin, segment.blockBegin); + BlockOrdinal const removedEnd = std::min(blockEnd, segment.blockEnd); + if (removedBegin >= removedEnd) + { + --index; + continue; + } + + changed = true; + subtract(allocationDelta(segment, removedBegin, removedEnd)); + if (removedBegin <= segment.blockBegin) + { + mAllocationSegments.erase(mAllocationSegments.begin() + index); + } + else + { + TLLM_CHECK_DEBUG(removedEnd == segment.blockEnd); + segment.blockEnd = removedBegin; + } + --index; + } + return changed; + } + + KVCacheStatsDelta const& globalStats() const noexcept + { + return mGlobalStats; + } + + KVCacheStatsDelta const& requestStats() const noexcept + { + return mRequestStats; + } + + IterationStatsByLifeCycle const& iterationStatsByLifeCycle() const noexcept + { + return mIterationStatsByLifeCycle; + } + + SsmSnapshotIterationStatsByLifeCycle const& ssmSnapshotIterationStatsByLifeCycle() const noexcept + { + return mSsmSnapshotIterationStatsByLifeCycle; + } + +private: + static PendingStatsDelta allocationDelta( + PendingAllocationSegment const& segment, BlockOrdinal blockBegin, BlockOrdinal blockEnd) + { + int64_t const numBlocks = static_cast<int64_t>(std::max(0, blockEnd - blockBegin)) * segment.beamWidth; + PendingStatsDelta delta; + delta.globalStats.allocTotalBlocks = numBlocks; + delta.globalStats.allocNewBlocks = numBlocks; + delta.globalStats.missedBlocks = segment.countAsMissed ? numBlocks : 0; + delta.requestStats = delta.globalStats.copy(); + delta.iterationStats.iterAllocTotalBlocks = numBlocks; + delta.iterationStats.iterAllocNewBlocks = numBlocks; + delta.iterationStats.iterMissedBlocks = segment.countAsMissed ? numBlocks : 0; + delta.iterationStats.iterGenAllocBlocks = segment.countAsGeneration ? numBlocks : 0; + delta.lifeCycle = segment.lifeCycle; + return delta; + } + + bool add(PendingStatsDelta const& delta) + { + if (delta.empty()) + { + return false; + } + mGlobalStats.add(delta.globalStats); + mRequestStats.add(delta.requestStats); + if (!delta.iterationStats.empty()) + { + TLLM_CHECK_DEBUG(delta.lifeCycle.has_value()); + mIterationStatsByLifeCycle[*delta.lifeCycle].add(delta.iterationStats); + } + return true; + } + + bool subtract(PendingStatsDelta const& delta) + { + if (delta.empty()) + { + return false; + } + mGlobalStats.subtract(delta.globalStats); + mRequestStats.subtract(delta.requestStats); + if (!delta.iterationStats.empty()) + { + TLLM_CHECK_DEBUG(delta.lifeCycle.has_value()); + auto const it = mIterationStatsByLifeCycle.find(*delta.lifeCycle); + if (it != mIterationStatsByLifeCycle.end()) + { + it->second.subtract(delta.iterationStats); + if (it->second.empty()) + { + mIterationStatsByLifeCycle.erase(it); + } + } + } + return true; + } + + KVCacheStatsDelta mRequestStats; + KVCacheStatsDelta mGlobalStats; + IterationStatsByLifeCycle mIterationStatsByLifeCycle; + SsmSnapshotIterationStatsByLifeCycle mSsmSnapshotIterationStatsByLifeCycle; + std::vector<PendingAllocationSegment> mAllocationSegments; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/stats.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/stats.h new file mode 100644 index 000000000000..e81e936f6e30 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/stats.h @@ -0,0 +1,265 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/lifeCycleRegistry.h" +#include "kv_cache_manager_v2/storage/config.h" + +#include <cstdint> +#include <unordered_map> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +struct KVCacheStatsDelta +{ + int64_t allocTotalBlocks = 0; + int64_t allocNewBlocks = 0; + int64_t reusedBlocks = 0; + int64_t missedBlocks = 0; + + void add(KVCacheStatsDelta const& other) noexcept + { + allocTotalBlocks += other.allocTotalBlocks; + allocNewBlocks += other.allocNewBlocks; + reusedBlocks += other.reusedBlocks; + missedBlocks += other.missedBlocks; + } + + void subtract(KVCacheStatsDelta const& other) noexcept + { + allocTotalBlocks -= other.allocTotalBlocks; + allocNewBlocks -= other.allocNewBlocks; + reusedBlocks -= other.reusedBlocks; + missedBlocks -= other.missedBlocks; + } + + void clear() noexcept + { + *this = {}; + } + + [[nodiscard]] KVCacheStatsDelta copy() const noexcept + { + return *this; + } + + [[nodiscard]] bool empty() const noexcept + { + return allocTotalBlocks == 0 && allocNewBlocks == 0 && reusedBlocks == 0 && missedBlocks == 0; + } + + [[nodiscard]] bool operator==(KVCacheStatsDelta const& other) const noexcept + { + return allocTotalBlocks == other.allocTotalBlocks && allocNewBlocks == other.allocNewBlocks + && reusedBlocks == other.reusedBlocks && missedBlocks == other.missedBlocks; + } +}; + +struct KVCacheIterationStatsDelta +{ + int64_t iterAllocTotalBlocks = 0; + int64_t iterAllocNewBlocks = 0; + int64_t iterReusedBlocks = 0; + int64_t iterFullReusedBlocks = 0; + int64_t iterPartialReusedBlocks = 0; + int64_t iterMissedBlocks = 0; + int64_t iterGenAllocBlocks = 0; + int64_t iterOnboardBlocks = 0; + int64_t iterOnboardBytes = 0; + int64_t iterOffloadBlocks = 0; + int64_t iterOffloadBytes = 0; + int64_t iterIntraDeviceCopyBlocks = 0; + int64_t iterIntraDeviceCopyBytes = 0; + int64_t iterHostDroppedBlocks = 0; + int64_t iterHostDroppedBytes = 0; + + void add(KVCacheIterationStatsDelta const& other) noexcept + { + iterAllocTotalBlocks += other.iterAllocTotalBlocks; + iterAllocNewBlocks += other.iterAllocNewBlocks; + iterReusedBlocks += other.iterReusedBlocks; + iterFullReusedBlocks += other.iterFullReusedBlocks; + iterPartialReusedBlocks += other.iterPartialReusedBlocks; + iterMissedBlocks += other.iterMissedBlocks; + iterGenAllocBlocks += other.iterGenAllocBlocks; + iterOnboardBlocks += other.iterOnboardBlocks; + iterOnboardBytes += other.iterOnboardBytes; + iterOffloadBlocks += other.iterOffloadBlocks; + iterOffloadBytes += other.iterOffloadBytes; + iterIntraDeviceCopyBlocks += other.iterIntraDeviceCopyBlocks; + iterIntraDeviceCopyBytes += other.iterIntraDeviceCopyBytes; + iterHostDroppedBlocks += other.iterHostDroppedBlocks; + iterHostDroppedBytes += other.iterHostDroppedBytes; + } + + void subtract(KVCacheIterationStatsDelta const& other) noexcept + { + iterAllocTotalBlocks -= other.iterAllocTotalBlocks; + iterAllocNewBlocks -= other.iterAllocNewBlocks; + iterReusedBlocks -= other.iterReusedBlocks; + iterFullReusedBlocks -= other.iterFullReusedBlocks; + iterPartialReusedBlocks -= other.iterPartialReusedBlocks; + iterMissedBlocks -= other.iterMissedBlocks; + iterGenAllocBlocks -= other.iterGenAllocBlocks; + iterOnboardBlocks -= other.iterOnboardBlocks; + iterOnboardBytes -= other.iterOnboardBytes; + iterOffloadBlocks -= other.iterOffloadBlocks; + iterOffloadBytes -= other.iterOffloadBytes; + iterIntraDeviceCopyBlocks -= other.iterIntraDeviceCopyBlocks; + iterIntraDeviceCopyBytes -= other.iterIntraDeviceCopyBytes; + iterHostDroppedBlocks -= other.iterHostDroppedBlocks; + iterHostDroppedBytes -= other.iterHostDroppedBytes; + } + + void clear() noexcept + { + *this = {}; + } + + [[nodiscard]] KVCacheIterationStatsDelta copy() const noexcept + { + return *this; + } + + [[nodiscard]] bool empty() const noexcept + { + return iterAllocTotalBlocks == 0 && iterAllocNewBlocks == 0 && iterReusedBlocks == 0 + && iterFullReusedBlocks == 0 && iterPartialReusedBlocks == 0 && iterMissedBlocks == 0 + && iterGenAllocBlocks == 0 && iterOnboardBlocks == 0 && iterOnboardBytes == 0 && iterOffloadBlocks == 0 + && iterOffloadBytes == 0 && iterIntraDeviceCopyBlocks == 0 && iterIntraDeviceCopyBytes == 0 + && iterHostDroppedBlocks == 0 && iterHostDroppedBytes == 0; + } + + [[nodiscard]] double iterCacheHitRate() const noexcept + { + int64_t const total = iterReusedBlocks + iterMissedBlocks; + if (iterReusedBlocks == 0 || total == 0) + { + return 0.0; + } + return static_cast<double>(iterReusedBlocks) / static_cast<double>(total); + } + + [[nodiscard]] bool operator==(KVCacheIterationStatsDelta const& other) const noexcept + { + return iterAllocTotalBlocks == other.iterAllocTotalBlocks && iterAllocNewBlocks == other.iterAllocNewBlocks + && iterReusedBlocks == other.iterReusedBlocks && iterFullReusedBlocks == other.iterFullReusedBlocks + && iterPartialReusedBlocks == other.iterPartialReusedBlocks && iterMissedBlocks == other.iterMissedBlocks + && iterGenAllocBlocks == other.iterGenAllocBlocks && iterOnboardBlocks == other.iterOnboardBlocks + && iterOnboardBytes == other.iterOnboardBytes && iterOffloadBlocks == other.iterOffloadBlocks + && iterOffloadBytes == other.iterOffloadBytes + && iterIntraDeviceCopyBlocks == other.iterIntraDeviceCopyBlocks + && iterIntraDeviceCopyBytes == other.iterIntraDeviceCopyBytes + && iterHostDroppedBlocks == other.iterHostDroppedBlocks + && iterHostDroppedBytes == other.iterHostDroppedBytes; + } +}; + +using IterationStatsByLifeCycle = std::unordered_map<LifeCycleId, KVCacheIterationStatsDelta>; + +// --------------------------------------------------------------------------- +// SsmSnapshotIterationStatsDelta — per-lifecycle counters for SSM snapshot +// reuse in one iteration. Mirrors Python's SsmSnapshotIterationStatsDelta. +// --------------------------------------------------------------------------- +struct SsmSnapshotIterationStatsDelta +{ + int64_t iterSnapshotLookups = 0; + int64_t iterSnapshotHits = 0; + int64_t iterSnapshotMisses = 0; + int64_t iterReusedTokens = 0; + int64_t iterUnreusedTokens = 0; + int64_t iterAlignedSnapshotHits = 0; + int64_t iterUnalignedSnapshotHits = 0; + + void add(SsmSnapshotIterationStatsDelta const& other) noexcept + { + iterSnapshotLookups += other.iterSnapshotLookups; + iterSnapshotHits += other.iterSnapshotHits; + iterSnapshotMisses += other.iterSnapshotMisses; + iterReusedTokens += other.iterReusedTokens; + iterUnreusedTokens += other.iterUnreusedTokens; + iterAlignedSnapshotHits += other.iterAlignedSnapshotHits; + iterUnalignedSnapshotHits += other.iterUnalignedSnapshotHits; + } + + void subtract(SsmSnapshotIterationStatsDelta const& other) noexcept + { + iterSnapshotLookups -= other.iterSnapshotLookups; + iterSnapshotHits -= other.iterSnapshotHits; + iterSnapshotMisses -= other.iterSnapshotMisses; + iterReusedTokens -= other.iterReusedTokens; + iterUnreusedTokens -= other.iterUnreusedTokens; + iterAlignedSnapshotHits -= other.iterAlignedSnapshotHits; + iterUnalignedSnapshotHits -= other.iterUnalignedSnapshotHits; + } + + void clear() noexcept + { + *this = {}; + } + + [[nodiscard]] SsmSnapshotIterationStatsDelta copy() const noexcept + { + return *this; + } + + [[nodiscard]] bool empty() const noexcept + { + return iterSnapshotLookups == 0 && iterSnapshotHits == 0 && iterSnapshotMisses == 0 && iterReusedTokens == 0 + && iterUnreusedTokens == 0 && iterAlignedSnapshotHits == 0 && iterUnalignedSnapshotHits == 0; + } + + [[nodiscard]] double iterSnapshotHitRate() const noexcept + { + if (iterSnapshotHits == 0 || iterSnapshotLookups == 0) + { + return 0.0; + } + return static_cast<double>(iterSnapshotHits) / static_cast<double>(iterSnapshotLookups); + } + + [[nodiscard]] bool operator==(SsmSnapshotIterationStatsDelta const& other) const noexcept + { + return iterSnapshotLookups == other.iterSnapshotLookups && iterSnapshotHits == other.iterSnapshotHits + && iterSnapshotMisses == other.iterSnapshotMisses && iterReusedTokens == other.iterReusedTokens + && iterUnreusedTokens == other.iterUnreusedTokens + && iterAlignedSnapshotHits == other.iterAlignedSnapshotHits + && iterUnalignedSnapshotHits == other.iterUnalignedSnapshotHits; + } +}; + +using SsmSnapshotIterationStatsByLifeCycle = std::unordered_map<LifeCycleId, SsmSnapshotIterationStatsDelta>; + +struct PoolGroupPeakBlockStats +{ + SlotCount available = 0; + SlotCount unavailable = 0; + SlotCount evictable = 0; + + [[nodiscard]] bool operator==(PoolGroupPeakBlockStats const& other) const noexcept + { + return available == other.available && unavailable == other.unavailable && evictable == other.evictable; + } +}; + +using PeakBlockStatsByPoolGroup = TypedVec<PoolGroupIndex, PoolGroupPeakBlockStats>; +using PeakBlockStatsByCacheLevel = TypedVec<CacheLevel, PeakBlockStatsByPoolGroup>; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/config.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/config.cpp new file mode 100644 index 000000000000..7411577d03b1 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/config.cpp @@ -0,0 +1,247 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/storage/config.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/utils/math.h" + +#include "tensorrt_llm/common/assert.h" +#include <algorithm> +#include <map> +#include <stdexcept> +#include <unordered_set> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// StorageConfig methods +// --------------------------------------------------------------------------- + +LifeCycleId StorageConfig::numLifeCycles() const +{ + LifeCycleId count{0}; + for (auto const& sd : slotDescList) + { + count += static_cast<int>(sd.variants.size()); + } + return count; +} + +TypedVec<LifeCycleId, PoolGroupIndex> StorageConfig::lifeCycleGrouping() const +{ + LifeCycleId n = numLifeCycles(); + TypedVec<LifeCycleId, PoolGroupIndex> ret(n, PoolGroupIndex{-1}); + for (PoolGroupIndex poolGroupIndex{0}; poolGroupIndex < slotDescList.size(); ++poolGroupIndex) + { + for (auto const& variant : slotDescList[poolGroupIndex].variants) + { + ret[variant.lifeCycleId] = poolGroupIndex; + } + } + return ret; +} + +std::map<BufferId, BufferAttr> StorageConfig::bufferAttributes() const +{ + std::map<BufferId, BufferAttr> ret; + for (auto const& sd : slotDescList) + { + for (auto const& variant : sd.variants) + { + LifeCycleId lcId = variant.lifeCycleId; + for (PoolIndex poolIndex{0}; poolIndex < variant.coalescedBuffers.size(); ++poolIndex) + { + auto const& cb = variant.coalescedBuffers[poolIndex]; + size_t offset = 0; + for (auto const& bufId : cb.bufferIds) + { + int exp = 1; + auto it = expansion.find(bufId); + if (it != expansion.end()) + exp = it->second; + + ret[bufId] = BufferAttr{lcId, poolIndex, offset, cb.singleBufferSize, exp}; + offset += cb.singleBufferSize; + } + } + } + } + return ret; +} + +std::unordered_map<LayerId, LifeCycleId> StorageConfig::layerToLifeCycleIds() const +{ + std::unordered_map<LayerId, LifeCycleId> map; + for (auto const& [bufId, attr] : bufferAttributes()) + { + auto [it, inserted] = map.emplace(bufId.layerId, attr.lifeCycleId); + if (!inserted) + { + TLLM_CHECK_DEBUG(it->second == attr.lifeCycleId); + } + } + return map; +} + +std::map<LayerId, LayerAttr> StorageConfig::layerAttributes() const +{ + std::map<LayerId, LayerAttr> ret; + for (auto const& pg : slotDescList) + { + for (auto const& variant : pg.variants) + { + LifeCycleId lcId = variant.lifeCycleId; + for (PoolIndex poolIndex{0}; poolIndex < variant.coalescedBuffers.size(); ++poolIndex) + { + auto const& cb = variant.coalescedBuffers[poolIndex]; + // Count how many buffers per layer in this coalesced buffer. + std::unordered_map<LayerId, int> slotUtilPerLayer; + for (auto const& bufId : cb.bufferIds) + { + slotUtilPerLayer[bufId.layerId] += 1; + } + int buffersPerSlot = cb.numBuffers(); + + for (auto const& [layerId, count] : slotUtilPerLayer) + { + auto it = ret.find(layerId); + if (it == ret.end()) + { + LayerAttr attr; + attr.lifeCycleId = lcId; + attr.slotUtil.resize(variant.coalescedBuffers.size(), 0); + attr.slotUtilFracMax = Rational{0, 1}; + it = ret.emplace(layerId, std::move(attr)).first; + } + auto& attr = it->second; + TLLM_CHECK_DEBUG(attr.lifeCycleId == lcId); + attr.slotUtil[poolIndex] = count; + Rational frac{count, buffersPerSlot}; + if (frac > attr.slotUtilFracMax) + { + attr.slotUtilFracMax = frac; + } + } + } + } + } + return ret; +} + +// --------------------------------------------------------------------------- +// createStorageConfig — factory function. +// Mirrors _storage/_config.py::create_storage_config. +// --------------------------------------------------------------------------- +StorageConfig createStorageConfig(KVCacheManagerConfig const& config) +{ + LifeCycleRegistry registry{config}; + int tokensPerBlock = config.tokensPerBlock; + + // Map: lifeCycleId → (bufferSize → list of BufferId). + // Outer map key is LifeCycleId; inner map key is expanded buffer size. + // NOTE: Python uses insertion-ordered dict/defaultdict here. The std::map + // grouping below intentionally remains sorted, which can make observable + // pool-group indices/layout order differ from Python. Later lookups use + // StorageConfig mappings, so this is not expected to affect correctness. + std::map<LifeCycleId, std::map<size_t, std::vector<BufferId>>> bufferGroups; + std::unordered_map<BufferId, int, BufferIdHash> expansionMap; + + for (auto const& layer : config.layers) + { + LifeCycle lc = makeLifeCycle(layer, tokensPerBlock); + LifeCycleId lcId = registry.getId(lc); + + std::visit( + [&](auto const& cfg) + { + for (auto const& buf : cfg.buffers) + { + int tpbo = buf.tokensPerBlockOverride.value_or(tokensPerBlock); + int exp = exactDiv(tokensPerBlock, tpbo); + size_t expandedSize = buf.size * static_cast<size_t>(exp); + + BufferId bid{cfg.layerId, buf.role}; + expansionMap[bid] = exp; + bufferGroups[lcId][expandedSize].push_back(bid); + } + }, + layer); + } + + // Build one SlotDescVariant per life cycle. + // Each variant: coalesced buffers sorted by size descending. + std::vector<SlotDescVariant> slotGroups; + slotGroups.reserve(bufferGroups.size()); + for (auto const& [lcId, sizeMap] : bufferGroups) + { + SlotDescVariant var; + var.lifeCycleId = lcId; + for (auto const& [sz, bufIds] : sizeMap) + { + CoalescedBuffer cb; + cb.singleBufferSize = sz; + cb.bufferIds = bufIds; + var.coalescedBuffers.push_back(std::move(cb)); + } + // Sort descending by size. + std::sort(var.coalescedBuffers.begin(), var.coalescedBuffers.end(), + [](CoalescedBuffer const& a, CoalescedBuffer const& b) { return a.size() > b.size(); }); + slotGroups.push_back(std::move(var)); + } + + // Merge SlotDescVariants that share the same slotSizeList. + // Key: tuple of sizes (sorted desc). + std::map<std::vector<size_t>, std::vector<SlotDescVariant>> poolGroupsBySizes; + for (auto& sg : slotGroups) + { + auto sizes = sg.slotSizeList(); + poolGroupsBySizes[sizes.raw()].push_back(std::move(sg)); + } + + StorageConfig out; + out.cacheTiers = TypedVec<CacheLevel, CacheTierConfig>{config.cacheTiers}; + out.expansion = expansionMap; + + for (auto& [sizes, variants] : poolGroupsBySizes) + { + SlotDesc sd; + sd.variants = std::move(variants); + out.slotDescList.push_back(std::move(sd)); + } + + // A21: Assert all life_cycle_ids across all SlotDescVariants are unique. + // Mirrors Python StorageConfig.__post_init__: + // all_life_cycle_ids = [lc_id for variant in self.slot_desc_list for lc_id in variant.life_cycle_ids] + // assert len(all_life_cycle_ids) == len(set(all_life_cycle_ids)) + if (TLLM_UNLIKELY(gDebug)) + { + std::unordered_set<LifeCycleId> allLcIds; + for (auto const& sd : out.slotDescList) + { + for (auto const& variant : sd.variants) + { + [[maybe_unused]] bool inserted = allLcIds.insert(variant.lifeCycleId).second; + TLLM_CHECK_WITH_INFO(inserted, "Duplicate life_cycle_id across SlotDescVariants"); + } + } + } + + return out; +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/config.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/config.h new file mode 100644 index 000000000000..b09d0dbbc9a0 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/config.h @@ -0,0 +1,249 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/config.h" +#include "kv_cache_manager_v2/lifeCycleRegistry.h" + +#include "tensorrt_llm/common/assert.h" +#include <cstddef> +#include <cstdint> +#include <limits> +#include <map> +#include <stdexcept> +#include <unordered_map> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// Index types (mirrors _storage/_core.py) +// --------------------------------------------------------------------------- +// Index of a pool group (= set of pools with mirrored allocation). +using PoolGroupIndex = StrongIndex<int, struct PoolGroupIndexTag>; +// Index of a pool within a pool group. +using PoolIndex = StrongIndex<int, struct PoolIndexTag>; +// Plain count of slots when the value is not itself a slot id. +using SlotCount = std::int64_t; +// Slot index within a pool. +using SlotId = StrongIndex<SlotCount, struct SlotIdTag>; + +[[nodiscard]] inline SlotCount slotCountValueFromSize(std::size_t count) noexcept +{ + TLLM_CHECK_DEBUG(count <= static_cast<std::size_t>(std::numeric_limits<SlotCount>::max())); + return static_cast<SlotCount>(count); +} + +[[nodiscard]] inline std::size_t slotCountToSizeT(SlotCount count) noexcept +{ + TLLM_CHECK_DEBUG_WITH_INFO(count >= 0, "Slot count must be non-negative for size_t conversion"); + return static_cast<std::size_t>(count); +} + +[[nodiscard]] inline int slotIdToPageIndexValue(SlotId slotId) +{ + if (slotId.value() < 0 || slotId.value() > std::numeric_limits<int>::max()) + { + throw std::overflow_error("SlotId does not fit in int page-index storage"); + } + return static_cast<int>(slotId.value()); +} + +// --------------------------------------------------------------------------- +// BufferId — (layer_id, role) pair identifying one buffer in a layer. +// Mirrors _storage/_config.py::BufferId. +// --------------------------------------------------------------------------- +struct BufferId +{ + LayerId layerId = 0; + DataRole role; + + bool operator==(BufferId const& o) const noexcept + { + return layerId == o.layerId && role == o.role; + } + + bool operator<(BufferId const& o) const noexcept + { + if (layerId != o.layerId) + return layerId < o.layerId; + return role < o.role; + } +}; + +struct BufferIdHash +{ + size_t operator()(BufferId const& id) const noexcept + { + size_t h = std::hash<int>{}(id.layerId); + h ^= std::hash<std::string>{}(id.role) + 0x9e3779b9 + (h << 6) + (h >> 2); + return h; + } +}; + +// --------------------------------------------------------------------------- +// CoalescedBuffer — several buffers of the same size and life cycle, +// laid out contiguously within one slot. +// Mirrors _storage/_config.py::CoalescedBuffer. +// --------------------------------------------------------------------------- +struct CoalescedBuffer +{ + size_t singleBufferSize = 0; + std::vector<BufferId> bufferIds; + + size_t size() const noexcept + { + return singleBufferSize * bufferIds.size(); + } + + int numBuffers() const noexcept + { + return static_cast<int>(bufferIds.size()); + } +}; + +// --------------------------------------------------------------------------- +// SlotDescVariant — one life cycle's view of a pool group. +// Mirrors _storage/_config.py::SlotDescVariant. +// --------------------------------------------------------------------------- +struct SlotDescVariant +{ + LifeCycleId lifeCycleId{0}; + TypedVec<PoolIndex, CoalescedBuffer> coalescedBuffers; // sorted size desc + + // Slot size for each pool in this group. + TypedVec<PoolIndex, size_t> slotSizeList() const + { + TypedVec<PoolIndex, size_t> out; + out.reserve(coalescedBuffers.size()); + for (auto const& cb : coalescedBuffers) + out.push_back(cb.size()); + // Coalesced buffers must be sorted in descending size order. + TLLM_CHECK_DEBUG(std::is_sorted(out.begin(), out.end(), std::greater<>())); + return out; + } +}; + +// --------------------------------------------------------------------------- +// SlotDesc — a pool group descriptor. Variants share the same slotSizeList +// but may differ in which buffers they contain. +// Mirrors _storage/_config.py::SlotDesc. +// --------------------------------------------------------------------------- +struct SlotDesc +{ + std::vector<SlotDescVariant> variants; // different life cycles sharing this pool group + + TypedVec<PoolIndex, size_t> slotSizeList() const + { + return getUniformAttribute(variants, [](auto const& v) { return v.slotSizeList(); }); + } +}; + +// --------------------------------------------------------------------------- +// BufferAttr — metadata for one buffer within storage. +// Mirrors _storage/_config.py::BufferAttr. +// --------------------------------------------------------------------------- +struct BufferAttr +{ + LifeCycleId lifeCycleId{0}; + PoolIndex poolIndex{0}; + size_t offset = 0; // byte offset within the slot + size_t size = 0; // expanded size of the buffer (after expansion) + int expansion = 1; // expansion factor (tokens_per_block / tokens_per_block_override) +}; + +// --------------------------------------------------------------------------- +// Rational — exact fraction for slot utilization computations. +// Mirrors Python's fractions.Fraction used in _storage/_config.py::LayerAttr. +// --------------------------------------------------------------------------- +struct Rational +{ + int num = 0; + int den = 1; + + // ceil(n * num / den). Uses int64_t intermediates to avoid overflow. + [[nodiscard]] int ceilMul(int value) const noexcept + { + auto n = static_cast<int64_t>(value) * num; + return static_cast<int>((n + den - 1) / den); + } + + bool operator>(Rational const& other) const noexcept + { + return static_cast<int64_t>(num) * other.den > static_cast<int64_t>(other.num) * den; + } + + bool operator==(Rational const& other) const noexcept + { + return static_cast<int64_t>(num) * other.den == static_cast<int64_t>(other.num) * den; + } +}; + +// --------------------------------------------------------------------------- +// LayerAttr — per-layer storage attributes for scratch slot management. +// Mirrors _storage/_config.py::LayerAttr. +// --------------------------------------------------------------------------- +struct LayerAttr +{ + LifeCycleId lifeCycleId{0}; + + // Number of sub-pages within a single coalesced slot that belong to this layer, + // per pool within the pool group. + TypedVec<PoolIndex, int> slotUtil; + + // Fraction of slot_util to total number of buffers in the slot, max over all pools. + Rational slotUtilFracMax; +}; + +// --------------------------------------------------------------------------- +// StorageConfig — complete storage layout for the KV cache. +// Mirrors _storage/_config.py::StorageConfig. +// --------------------------------------------------------------------------- +struct StorageConfig +{ + TypedVec<CacheLevel, CacheTierConfig> cacheTiers; + TypedVec<PoolGroupIndex, SlotDesc> slotDescList; + + // Expansion factor per buffer (for heterogeneous tokens_per_block). + std::unordered_map<BufferId, int, BufferIdHash> expansion; + + // Map each LifeCycleId → its PoolGroupIndex. + TypedVec<LifeCycleId, PoolGroupIndex> lifeCycleGrouping() const; + + // Attribute map for each buffer. + std::map<BufferId, BufferAttr> bufferAttributes() const; + + // Map LayerId → LifeCycleId. + std::unordered_map<LayerId, LifeCycleId> layerToLifeCycleIds() const; + + // Per-layer storage attributes (slot utilization within coalesced slots). + // Mirrors _storage/_config.py::StorageConfig.layer_attributes(). + std::map<LayerId, LayerAttr> layerAttributes() const; + + LifeCycleId numLifeCycles() const; +}; + +// --------------------------------------------------------------------------- +// Factory: create StorageConfig from KVCacheManagerConfig. +// Mirrors _storage/_config.py::create_storage_config. +// --------------------------------------------------------------------------- +StorageConfig createStorageConfig(KVCacheManagerConfig const& config); + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/core.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/core.cpp new file mode 100644 index 000000000000..892687304452 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/core.cpp @@ -0,0 +1,946 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/storage/core.h" + +#include "tensorrt_llm/common/assert.h" +#include <algorithm> +#include <cerrno> +#include <climits> +#include <cmath> +#include <cstring> +#include <fcntl.h> +#include <limits> +#include <numeric> +#include <set> +#include <stdexcept> +#include <unistd.h> +#include <unordered_map> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// SlotAllocator +// --------------------------------------------------------------------------- + +SlotAllocator::SlotAllocator(SlotCount capacity) + : mCapacity(capacity) + , mTargetCapacity(capacity) + , mNumActiveSlots(0) + , mNumReadyRecycledSlots(0) + , mOccupiedMask(slotCountToSizeT(capacity)) +{ +} + +SlotAllocator::~SlotAllocator() +{ + // Mirrors Python SlotAllocator.__del__ (assert_critical checks). + if (TLLM_UNLIKELY(gDebug)) + { + TLLM_CHECK_WITH_INFO(slotCountToSizeT(mNumReadyRecycledSlots) == mRecycledSlots.size(), + "SlotAllocator destroyed with unfinished events — did you call synchronize()?"); + TLLM_CHECK_WITH_INFO(mTargetCapacity == mCapacity && mOverflowSlots.empty(), + "SlotAllocator destroyed while resize is in progress"); + TLLM_CHECK_WITH_INFO( + mOccupiedMask.numSetBits() == 0, "SlotAllocator destroyed with occupied slots still in use"); + TLLM_CHECK_WITH_INFO(mRecycledSlots.size() == slotCountToSizeT(mNumActiveSlots), + "SlotAllocator destroyed with some slots not recycled"); + } +} + +SlotCount SlotAllocator::numFreeSlots() const noexcept +{ + SlotCount const inactiveSlots = mTargetCapacity > mNumActiveSlots ? mTargetCapacity - mNumActiveSlots : 0; + return slotCountValueFromSize(mRecycledSlots.size()) + inactiveSlots; +} + +SlotCount SlotAllocator::numOccupiedSlots() const noexcept +{ + return slotCountValueFromSize(mOccupiedMask.numSetBits()); +} + +Slot SlotAllocator::allocate() +{ + if (numFreeSlots() == 0) + { + throw OutOfPagesError("SlotAllocator: no free slots"); + } + scrubEvents(); + + Slot slot; + if (mNumReadyRecycledSlots > 0) + { + TLLM_CHECK_DEBUG_WITH_INFO(!mRecycledSlots.empty(), "ready recycled slots > 0 but deque is empty"); + slot = std::move(mRecycledSlots.front()); + mRecycledSlots.pop_front(); + TLLM_CHECK_DEBUG_WITH_INFO(slot.hasValidSlot(), "ready recycled slot has no valid id"); + --mNumReadyRecycledSlots; + TLLM_CHECK_DEBUG_WITH_INFO(slot.readyEvent.isClosed(), "ready recycled slot has non-null event"); + } + else if (mNumActiveSlots < std::min(mCapacity, mTargetCapacity)) + { + slot.setSlotId(SlotId{mNumActiveSlots++}); + } + else + { + slot = std::move(mRecycledSlots.front()); + mRecycledSlots.pop_front(); + TLLM_CHECK_DEBUG_WITH_INFO(slot.hasValidSlot(), "non-ready recycled slot has no valid id"); + } + mOccupiedMask.set(toSizeT(slot.slotId())); + return slot; +} + +std::vector<Slot> SlotAllocator::allocateMultiple(SlotCount numSlots) +{ + if (numSlots < 0) + { + throw LogicError("SlotAllocator::allocateMultiple: slot count must be non-negative"); + } + if (numFreeSlots() < numSlots) + { + throw OutOfPagesError("SlotAllocator: not enough free slots"); + } + std::vector<Slot> result; + result.reserve(slotCountToSizeT(numSlots)); + for (SlotCount slotCount{0}; slotCount < numSlots; ++slotCount) + { + result.push_back(allocate()); + } + return result; +} + +void SlotAllocator::release(Slot slot) +{ + if (!slot.hasValidSlot()) + { + throw LogicError("SlotAllocator::release: slot has no valid id"); + } + SlotId const slotId = slot.slotId(); + if (slotId >= numSlots() || !mOccupiedMask.get(toSizeT(slotId))) + { + throw LogicError("SlotAllocator::release: slot is not occupied"); + } + mOccupiedMask.clear(toSizeT(slotId)); + if (slotId < mTargetCapacity) + { + mRecycledSlots.push_back(std::move(slot)); + } + else + { + mOverflowSlots.push_back(std::move(slot)); + } + scrubEvents(); + TLLM_CHECK_DEBUG(check()); +} + +void SlotAllocator::expand(SlotCount newNumSlots) +{ + TLLM_CHECK_DEBUG(check()); + TLLM_CHECK_DEBUG(mTargetCapacity == mCapacity); + TLLM_CHECK_DEBUG(newNumSlots > mCapacity); + mOccupiedMask.resize(slotCountToSizeT(newNumSlots)); + mCapacity = newNumSlots; + mTargetCapacity = newNumSlots; + TLLM_CHECK_DEBUG(check()); +} + +void SlotAllocator::prepareForShrink(SlotCount newNumSlots) +{ + TLLM_CHECK_DEBUG(check()); + TLLM_CHECK_DEBUG(mTargetCapacity == mCapacity); + TLLM_CHECK_DEBUG(newNumSlots < mCapacity); + std::deque<Slot> newRecycled; + SlotCount newNumReady = 0; + SlotCount const oldNumReady = mNumReadyRecycledSlots; + SlotCount idx = 0; + for (auto& s : mRecycledSlots) + { + if (s.slotId() < newNumSlots) + { + if (idx < oldNumReady) + ++newNumReady; + newRecycled.push_back(std::move(s)); + } + else + { + mOverflowSlots.push_back(std::move(s)); + } + ++idx; + } + mRecycledSlots = std::move(newRecycled); + mNumReadyRecycledSlots = newNumReady; + mTargetCapacity = newNumSlots; + TLLM_CHECK_DEBUG(check()); +} + +bool SlotAllocator::finishShrink() +{ + TLLM_CHECK_DEBUG(check()); + // Overflow-range IDs that were ever issued are exactly + // max(0, _num_active_slots - _target_capacity); the underused case + // (_num_active_slots <= _target_capacity) collapses to zero. + SlotCount const expectedOverflow = std::max(SlotCount{0}, mNumActiveSlots - mTargetCapacity); + if (shrinkInProgress() && slotCountValueFromSize(mOverflowSlots.size()) == expectedOverflow) + { + // Validate uniqueness of slot IDs in overflow (debug only). + if (TLLM_UNLIKELY(gDebug)) + { + std::set<SlotId> ids; + for (auto const& s : mOverflowSlots) + { + TLLM_CHECK(s.hasValidSlot()); + ids.insert(s.slotId()); + } + TLLM_CHECK_WITH_INFO(ids.size() == mOverflowSlots.size(), "Duplicate slot IDs in overflow slots"); + } + // Synchronize overflow events (deduplicated — slots often share events). + { + std::vector<CachedCudaEvent*> overflowEvents; + overflowEvents.reserve(mOverflowSlots.size()); + for (auto& s : mOverflowSlots) + overflowEvents.push_back(&s.readyEvent); + synchronizeAll(overflowEvents); + } + for (auto& s : mOverflowSlots) + s.resetSlot(); + mOverflowSlots.clear(); + mCapacity = mTargetCapacity; + mNumActiveSlots = std::min(mNumActiveSlots, mCapacity); + scrubEvents(); + TLLM_CHECK_DEBUG(check()); + return true; + } + throw std::runtime_error("SlotAllocator::finishShrink: cannot finish shrink yet"); +} + +std::vector<SlotId> SlotAllocator::getSlotsBlockingShrink() const +{ + std::vector<SlotId> result; + for (SlotCount id = mTargetCapacity; id < mCapacity; ++id) + { + if (mOccupiedMask.get(slotCountToSizeT(id))) + result.push_back(SlotId{id}); + } + return result; +} + +void SlotAllocator::synchronize() +{ + while (slotCountToSizeT(mNumReadyRecycledSlots) != mRecycledSlots.size()) + { + scrubEvents(); + } +} + +bool SlotAllocator::check() const noexcept +{ + // Mirrors Python SlotAllocator._check(). + if (mNumActiveSlots > mCapacity) + return false; + if (mTargetCapacity > mCapacity) + return false; + if (!shrinkInProgress() && !mOverflowSlots.empty()) + return false; + for (auto const& slot : mOverflowSlots) + { + if (!slot.hasValidSlot()) + return false; + SlotCount id = slot.slotId().value(); + if (id < mTargetCapacity || id >= mCapacity) + return false; + } + SlotCount const accountedSlots = slotCountValueFromSize(mRecycledSlots.size()) + + slotCountValueFromSize(mOverflowSlots.size()) + numOccupiedSlots(); + if (accountedSlots != mNumActiveSlots) + return false; + return true; +} + +void SlotAllocator::scrubEvents() +{ + for (size_t i = slotCountToSizeT(mNumReadyRecycledSlots); i < mRecycledSlots.size(); ++i) + { + if (mRecycledSlots[i].queryReady()) + { + ++mNumReadyRecycledSlots; + } + else + { + break; + } + } +} + +// --------------------------------------------------------------------------- +// GpuSlotPool +// --------------------------------------------------------------------------- + +GpuSlotPool::GpuSlotPool(size_t slotSize, size_t vmSize, PooledPhysMemAllocator& physMemAllocator, SlotCount numSlots) + : SlotPoolBase(slotSize) + , mVirtMem(vmSize, physMemAllocator) +{ + TLLM_CHECK_DEBUG_WITH_INFO( + vmSize % physMemAllocator.physMemSize() == 0, "vm_size must be aligned to phys_mem_size"); + resize(numSlots); +} + +size_t GpuSlotPool::computeNumPhysMem(size_t slotSize, SlotCount numSlots, size_t physMemSize) noexcept +{ + return divUp(slotCountToSizeT(numSlots) * slotSize, physMemSize); +} + +SlotCount GpuSlotPool::computeNumSlots(size_t slotSize, size_t numPhysMem, size_t physMemSize) noexcept +{ + return slotCountValueFromSize(numPhysMem * physMemSize / slotSize); +} + +SlotCount GpuSlotPool::numSlots() const noexcept +{ + return computeNumSlots(mSlotSize, mVirtMem.numPhysMem(), mVirtMem.physMemSize()); +} + +void GpuSlotPool::destroy() +{ + mVirtMem.destroy(); +} + +void GpuSlotPool::resize(SlotCount newNumSlots) +{ + size_t physSize = mVirtMem.physMemSize(); + size_t newPhysMem = computeNumPhysMem(mSlotSize, newNumSlots, physSize); + mVirtMem.realloc(physSize * newPhysMem); +} + +SlotCount GpuSlotPool::extendByOnePhysMem() +{ + mVirtMem.extend(1); + return numSlots(); +} + +Address GpuSlotPool::slotAddress(SlotId slot) const +{ + TLLM_CHECK_DEBUG_WITH_INFO(slot < numSlots(), "GpuSlotPool::slotAddress: slot index out of bounds"); + return MemAddress(mVirtMem.address() + mSlotSize * toSizeT(slot)); +} + +// --------------------------------------------------------------------------- +// HostSlotPool +// --------------------------------------------------------------------------- + +HostSlotPool::HostSlotPool(size_t slotSize, SlotCount numSlots) + : SlotPoolBase(slotSize) + , mHostMem(alignedSize(numSlots)) +{ +} + +size_t HostSlotPool::alignedSize(SlotCount numSlots) const noexcept +{ + return roundUp(slotCountToSizeT(numSlots) * mSlotSize, HostMem::kAlignment); +} + +SlotCount HostSlotPool::numSlots() const noexcept +{ + return slotCountValueFromSize(mHostMem.size() / mSlotSize); +} + +void HostSlotPool::destroy() +{ + mHostMem.destroy(); +} + +void HostSlotPool::resize(SlotCount newNumSlots) +{ + mHostMem.resize(alignedSize(newNumSlots)); +} + +Address HostSlotPool::slotAddress(SlotId slot) const +{ + TLLM_CHECK_DEBUG_WITH_INFO(slot < numSlots(), "HostSlotPool::slotAddress: slot index out of bounds"); + return MemAddress(mHostMem.address() + mSlotSize * toSizeT(slot)); +} + +// --------------------------------------------------------------------------- +// DiskSlotPool +// --------------------------------------------------------------------------- + +DiskSlotPool::DiskSlotPool(std::string const& directory, size_t slotSize, SlotCount numSlots) + : SlotPoolBase(slotSize) +{ + // Try O_TMPFILE first, fall back to mkstemp. + mFd = ::open(directory.c_str(), O_TMPFILE | O_RDWR | O_EXCL, 0664); + if (mFd < 0) + { + if (errno == EOPNOTSUPP) + { + char tmpl[PATH_MAX]; + snprintf(tmpl, sizeof(tmpl), "%s/kvXXXXXX", directory.c_str()); + mFd = ::mkstemp(tmpl); + if (mFd < 0) + { + throw DiskOOMError("DiskSlotPool: mkstemp failed: " + std::string(std::strerror(errno))); + } + ::unlink(tmpl); + } + else + { + throw DiskOOMError("DiskSlotPool: open O_TMPFILE failed: " + std::string(std::strerror(errno))); + } + } + resize(numSlots); +} + +DiskSlotPool::~DiskSlotPool() +{ + destroy(); +} + +SlotCount DiskSlotPool::numSlots() const noexcept +{ + TLLM_CHECK_DEBUG(mFd != kBadFileDescriptor); + off_t sz = ::lseek(mFd, 0, SEEK_END); + return (sz < 0 || mSlotSize == 0) ? 0 : slotCountValueFromSize(static_cast<size_t>(sz) / mSlotSize); +} + +void DiskSlotPool::destroy() +{ + if (mFd != kBadFileDescriptor) + { + ::close(mFd); + mFd = kBadFileDescriptor; + } +} + +void DiskSlotPool::resize(SlotCount newNumSlots) +{ + resizeFile(mFd, slotCountToSizeT(newNumSlots) * mSlotSize); +} + +Address DiskSlotPool::slotAddress(SlotId slot) const +{ + TLLM_CHECK_DEBUG_WITH_INFO(slot < numSlots(), "DiskSlotPool::slotAddress: slot index out of bounds"); + size_t const byteOffset = toSizeT(slot) * mSlotSize; + TLLM_CHECK_DEBUG_WITH_INFO(byteOffset <= static_cast<size_t>(std::numeric_limits<ssize_t>::max()), + "DiskSlotPool::slotAddress: byte offset out of range"); + return DiskAddress{mFd, static_cast<ssize_t>(byteOffset)}; +} + +// --------------------------------------------------------------------------- +// PoolGroupBase +// --------------------------------------------------------------------------- + +PoolGroupBase::PoolGroupBase(SlotCount numSlots) + : mSlotAllocator(numSlots) +{ +} + +SlotCount PoolGroupBase::getNumSlotsFromPools() const noexcept +{ + if (mPools.empty()) + return 0; + SlotCount minSlots = mPools.front()->numSlots(); + for (PoolIndex poolIdx{1}; poolIdx < mPools.size(); ++poolIdx) + { + minSlots = std::min(minSlots, mPools[poolIdx]->numSlots()); + } + return minSlots; +} + +PoolGroupBase::~PoolGroupBase() +{ + destroy(); +} + +SlotCount PoolGroupBase::numSlots() const noexcept +{ + SlotCount n = mSlotAllocator.numSlots(); + if (TLLM_UNLIKELY(gDebug)) + { + // Mirrors Python PoolGroupBase.num_slots: assert num_slots <= self._get_num_slots_from_pools() + [[maybe_unused]] SlotCount poolSlots = getNumSlotsFromPools(); + TLLM_CHECK_WITH_INFO(n <= poolSlots, "SlotAllocator capacity exceeds pool capacity"); + } + return n; +} + +Slot PoolGroupBase::allocate() +{ + return mSlotAllocator.allocate(); +} + +std::vector<Slot> PoolGroupBase::allocateMultiple(SlotCount numSlots) +{ + return mSlotAllocator.allocateMultiple(numSlots); +} + +void PoolGroupBase::release(Slot slot) +{ + mSlotAllocator.release(std::move(slot)); +} + +void PoolGroupBase::destroy() +{ + if (mDestroyed) + return; + if (mSlotAllocator.numSlots() != 0) + { + mSlotAllocator.synchronize(); + mSlotAllocator.prepareForShrink(0); + mSlotAllocator.finishShrink(); + } + for (auto& p : mPools) + p->destroy(); + mDestroyed = true; +} + +void PoolGroupBase::resizePools(std::optional<SlotCount> newNumSlots) +{ + SlotCount n = newNumSlots.value_or(mSlotAllocator.numSlots()); + for (auto& p : mPools) + p->resize(n); + // Mirrors Python PoolGroupBase.resize_pools: assert NDEBUG or self._check(True) + // After resize, allocator capacity must not exceed pool capacity (allow mismatch). + TLLM_CHECK_DEBUG_WITH_INFO(mSlotAllocator.numSlots() <= getNumSlotsFromPools(), + "After resizePools: allocator capacity exceeds pool capacity"); +} + +TypedVec<PoolIndex, Address> PoolGroupBase::slotAddress(SlotId slotId) const +{ + TypedVec<PoolIndex, Address> addrs; + addrs.reserve(mPools.size()); + for (auto const& p : mPools) + addrs.push_back(p->slotAddress(slotId)); + return addrs; +} + +TypedVec<PoolIndex, size_t> PoolGroupBase::slotSize() const +{ + TypedVec<PoolIndex, size_t> sizes; + sizes.reserve(mPools.size()); + for (auto const& p : mPools) + sizes.push_back(p->slotSize()); + return sizes; +} + +// --------------------------------------------------------------------------- +// GpuPoolGroup +// --------------------------------------------------------------------------- + +GpuPoolGroup::GpuPoolGroup( + SlotCount numSlots, TypedVec<PoolIndex, size_t> const& slotSizeList, PooledPhysMemAllocator& physMemAllocator) + : PoolGroupBase(numSlots) +{ + size_t physMemSize = physMemAllocator.physMemSize(); + // Query total GPU memory to size virtual address space (mirrors Python GpuPoolGroup). + size_t totalGpuMem = 0; + { + CUdevice dev{}; + cuCheck(cuCtxGetDevice(&dev)); + cuCheck(cuDeviceTotalMem(&totalGpuMem, dev)); + } + // @TODO: We should replace maxSlotSize with sum. This should also be updated in Python. Will do it later. + TLLM_CHECK_WITH_INFO(!slotSizeList.empty(), "GpuPoolGroup: slotSizeList must not be empty"); + size_t maxSlotSize = *std::max_element(slotSizeList.begin(), slotSizeList.end()); + for (size_t sz : slotSizeList) + { + // VA proportional to GPU memory, scaled by slot size ratio (mirrors Python). + // Compute ratio as double first to avoid size_t overflow. + double sizeRatio = static_cast<double>(sz) / static_cast<double>(maxSlotSize); + size_t vmSize = roundDown(static_cast<size_t>(static_cast<double>(totalGpuMem) * sizeRatio), physMemSize); + vmSize = std::max(vmSize, roundUp(slotCountToSizeT(numSlots) * sz, physMemSize)); + mPools.push_back(std::make_unique<GpuSlotPool>(sz, vmSize, physMemAllocator, numSlots)); + } +} + +// --------------------------------------------------------------------------- +// HostPoolGroup +// --------------------------------------------------------------------------- + +HostPoolGroup::HostPoolGroup(SlotCount numSlots, TypedVec<PoolIndex, size_t> const& slotSizeList) + : PoolGroupBase(numSlots) +{ + for (size_t sz : slotSizeList) + { + mPools.push_back(std::make_unique<HostSlotPool>(sz, numSlots)); + } +} + +// --------------------------------------------------------------------------- +// DiskPoolGroup +// --------------------------------------------------------------------------- + +DiskPoolGroup::DiskPoolGroup( + SlotCount numSlots, TypedVec<PoolIndex, size_t> const& slotSizeList, std::string const& directory) + : PoolGroupBase(numSlots) +{ + for (size_t sz : slotSizeList) + { + mPools.push_back(std::make_unique<DiskSlotPool>(directory, sz, numSlots)); + } +} + +// --------------------------------------------------------------------------- +// CacheLevelStorage +// --------------------------------------------------------------------------- + +std::vector<Slot> CacheLevelStorage::allocateMultiple(PoolGroupIndex pgIdx, SlotCount numSlots) +{ + return mPoolGroups.at(pgIdx)->allocateMultiple(numSlots); +} + +void CacheLevelStorage::release(PoolGroupIndex pgIdx, Slot slot) +{ + mPoolGroups.at(pgIdx)->release(std::move(slot)); +} + +SlotCount CacheLevelStorage::numFreeSlots(PoolGroupIndex pgIdx) const +{ + return mPoolGroups.at(pgIdx)->numFreeSlots(); +} + +TypedVec<PoolIndex, Address> CacheLevelStorage::slotAddress(PoolGroupIndex pgIdx, SlotId slotId) const +{ + return mPoolGroups.at(pgIdx)->slotAddress(slotId); +} + +// --------------------------------------------------------------------------- +// GpuCacheLevelStorage +// --------------------------------------------------------------------------- + +GpuCacheLevelStorage::GpuCacheLevelStorage( + StorageConfig const& storageCfg, TypedVec<PoolGroupIndex, SlotCount> const& slotCountList, size_t physMemSize) +{ + TLLM_CHECK_DEBUG_WITH_INFO(slotCountList.size() == storageCfg.slotDescList.size(), + "GpuCacheLevelStorage: slotCountList and slotDescList must have the same length"); + mPhysMemAllocator = std::make_unique<PooledPhysMemAllocator>(physMemSize); + + for (PoolGroupIndex pgIdx{0}; pgIdx < storageCfg.slotDescList.size(); ++pgIdx) + { + mPoolGroups.push_back(std::make_unique<GpuPoolGroup>( + slotCountList[pgIdx], storageCfg.slotDescList[pgIdx].slotSizeList(), *mPhysMemAllocator)); + } +} + +// --------------------------------------------------------------------------- +// HostCacheLevelStorage +// --------------------------------------------------------------------------- + +HostCacheLevelStorage::HostCacheLevelStorage( + StorageConfig const& storageCfg, TypedVec<PoolGroupIndex, SlotCount> const& slotCountList) +{ + TLLM_CHECK_DEBUG_WITH_INFO(slotCountList.size() == storageCfg.slotDescList.size(), + "HostCacheLevelStorage: slotCountList and slotDescList must have the same length"); + for (PoolGroupIndex pgIdx{0}; pgIdx < storageCfg.slotDescList.size(); ++pgIdx) + { + mPoolGroups.push_back( + std::make_unique<HostPoolGroup>(slotCountList[pgIdx], storageCfg.slotDescList[pgIdx].slotSizeList())); + } +} + +// --------------------------------------------------------------------------- +// DiskCacheLevelStorage +// --------------------------------------------------------------------------- + +DiskCacheLevelStorage::DiskCacheLevelStorage( + StorageConfig const& storageCfg, TypedVec<PoolGroupIndex, SlotCount> const& slotCountList, std::string directory) + : mDirectory(std::move(directory)) +{ + TLLM_CHECK_DEBUG_WITH_INFO(slotCountList.size() == storageCfg.slotDescList.size(), + "DiskCacheLevelStorage: slotCountList and slotDescList must have the same length"); + for (PoolGroupIndex pgIdx{0}; pgIdx < storageCfg.slotDescList.size(); ++pgIdx) + { + mPoolGroups.push_back(std::make_unique<DiskPoolGroup>( + slotCountList[pgIdx], storageCfg.slotDescList[pgIdx].slotSizeList(), mDirectory)); + } +} + +// --------------------------------------------------------------------------- +// Factory +// --------------------------------------------------------------------------- + +std::unique_ptr<CacheLevelStorage> createCacheLevelStorage(CacheTierConfig const& tierCfg, + StorageConfig const& storageCfg, TypedVec<PoolGroupIndex, SlotCount> const& slotCountList) +{ + return std::visit( + [&](auto const& cfg) -> std::unique_ptr<CacheLevelStorage> + { + using T = std::decay_t<decltype(cfg)>; + if constexpr (std::is_same_v<T, GpuCacheTierConfig>) + { + // Compute phys mem size (granularity) from quota. + constexpr size_t kPageSize = 2ULL << 20; + // Guard std::log2(0) (UB when cast to int) for quotas below 1 GiB, + // where the integer ratio is 0 and the exponent floor is used. + size_t const ratio = cfg.quota / (kPageSize * 512); + int const exponent = ratio == 0 ? 0 : std::min(4, std::max(0, static_cast<int>(std::log2(ratio)))); + size_t physMemSize = kPageSize << exponent; + return std::make_unique<GpuCacheLevelStorage>(storageCfg, slotCountList, physMemSize); + } + else if constexpr (std::is_same_v<T, HostCacheTierConfig>) + { + return std::make_unique<HostCacheLevelStorage>(storageCfg, slotCountList); + } + else + { + return std::make_unique<DiskCacheLevelStorage>(storageCfg, slotCountList, cfg.path); + } + }, + tierCfg); +} + +// --------------------------------------------------------------------------- +// CacheLevelStorage helper: grainsToSlots +// Distribute grains among pools within a pool group. +// Returns {num_slots, grains_consumed}. +// Mirrors Python CacheLevelStorage._grains_to_slots. +// --------------------------------------------------------------------------- + +std::pair<SlotCount, size_t> CacheLevelStorage::grainsToSlots( + size_t pgGrains, TypedVec<PoolIndex, size_t> const& slotSizeList, size_t granularity) +{ + TypedVec<PoolIndex, size_t> minPoolGrains(slotSizeList.size()); + for (PoolIndex poolIdx{0}; poolIdx < slotSizeList.size(); ++poolIdx) + { + minPoolGrains[poolIdx] = divUp(slotSizeList[poolIdx], granularity); + } + + size_t minTotal = 0; + for (auto g : minPoolGrains) + { + minTotal += g; + } + if (pgGrains < minTotal) + return {0, 0}; + + SlotCount numSlots{std::numeric_limits<SlotCount>::max()}; + size_t remainingPgGrains = pgGrains; + + // Sort pools by slot size ascending. + std::vector<PoolIndex> poolOrder; + poolOrder.reserve(slotSizeList.stdSize()); + for (PoolIndex poolIdx{0}; poolIdx < slotSizeList.size(); ++poolIdx) + { + poolOrder.push_back(poolIdx); + } + std::sort(poolOrder.begin(), poolOrder.end(), + [&](PoolIndex a, PoolIndex b) { return slotSizeList[a] < slotSizeList[b]; }); + + for (size_t j = 0; j < poolOrder.size(); ++j) + { + PoolIndex const poolIdx = poolOrder[j]; + size_t slotSz = slotSizeList[poolIdx]; + size_t poolSzSum = 0; + for (size_t k = j; k < poolOrder.size(); ++k) + poolSzSum += slotSizeList[poolOrder[k]]; + double poolFrac = (poolSzSum > 0) ? static_cast<double>(slotSz) / static_cast<double>(poolSzSum) : 1.0; + size_t roundedGrains = static_cast<size_t>(std::nearbyint(static_cast<double>(remainingPgGrains) * poolFrac)); + size_t poolGrains = std::max(minPoolGrains[poolIdx], roundedGrains); + SlotCount const poolSlots = slotCountValueFromSize(poolGrains * granularity / slotSz); + numSlots = std::min(numSlots, poolSlots); + TLLM_CHECK_DEBUG(poolGrains <= remainingPgGrains); + remainingPgGrains -= poolGrains; + } + TLLM_CHECK_DEBUG(remainingPgGrains == 0); + TLLM_CHECK_DEBUG(numSlots > 0); + + auto slotsToGrains = [&](SlotCount slots) { return grainsForSlots(slots, slotSizeList, granularity); }; + SlotCount lo = numSlots; + SlotCount step = 1; + SlotCount hi = lo + step; + while (slotsToGrains(hi) <= pgGrains) + { + lo = hi; + step *= 2; + hi = lo + step; + } + while (lo + 1 < hi) + { + SlotCount const mid = lo + ((hi - lo) / 2); + if (slotsToGrains(mid) <= pgGrains) + { + lo = mid; + } + else + { + hi = mid; + } + } + size_t const used = slotsToGrains(lo); + TLLM_CHECK_DEBUG(used <= pgGrains); + TLLM_CHECK_DEBUG(slotsToGrains(lo + 1) > pgGrains); + return {lo, used}; +} + +// --------------------------------------------------------------------------- +// CacheLevelStorage helper: grainsForSlots +// Compute minimum grains needed for numSlots in a pool group. +// Mirrors Python CacheLevelStorage._grains_for_slots. +// --------------------------------------------------------------------------- + +size_t CacheLevelStorage::grainsForSlots( + SlotCount numSlots, TypedVec<PoolIndex, size_t> const& slotSizeList, size_t granularity) +{ + size_t total = 0; + for (auto s : slotSizeList) + total += divUp(slotCountToSizeT(numSlots) * s, granularity); + return total; +} + +// --------------------------------------------------------------------------- +// CacheLevelStorage::ratioToSlotCountList (static) +// Mirrors Python CacheLevelStorage.ratio_to_slot_count_list. +// --------------------------------------------------------------------------- + +TypedVec<PoolGroupIndex, SlotCount> CacheLevelStorage::ratioToSlotCountList(size_t quota, + TypedVec<PoolGroupIndex, TypedVec<PoolIndex, size_t>> const& sizeLists, + TypedVec<PoolGroupIndex, float> const& ratioList, size_t granularity, + TypedVec<PoolGroupIndex, SlotCount> const& minSlots) +{ + PoolGroupIndex numPg = sizeLists.size(); + TLLM_CHECK_DEBUG(ratioList.size() == numPg); + TLLM_CHECK_DEBUG_WITH_INFO(std::all_of(ratioList.begin(), ratioList.end(), [](auto x) { return x > 0; }), + "ratioToSlotCountList: all ratios must be positive"); + TLLM_CHECK_DEBUG(quota % granularity == 0); + size_t totalGrains = quota / granularity; + if (TLLM_UNLIKELY(gDebug)) + { + [[maybe_unused]] size_t minGrains = 0; + for (auto const& sizes : sizeLists) + minGrains += toSizeT(sizes.size()); + TLLM_CHECK_WITH_INFO(totalGrains >= minGrains, + "ratioToSlotCountList: insufficient total grains for at least 1 slot per pool group"); + } + + TypedVec<PoolGroupIndex, SlotCount> slotCntList(numPg, 0); + size_t remainingGrains = totalGrains; + std::vector<PoolGroupIndex> activePgs(toSizeT(numPg)); + std::iota(activePgs.begin(), activePgs.end(), PoolGroupIndex{0}); + + // Iteratively peel off constrained PGs until all active PGs are + // unconstrained: + // 1. Distribute remaining quota among active PGs by ratio. + // 2. Any PG with slots <= min_slots is constrained — pin it to + // min_slots and subtract its grains from the budget. + // 3. Repeat with the remaining PGs and re-normalized ratios. + // Each iteration removes at least one PG, so this terminates. + while (!activePgs.empty()) + { + // Distribute remainingGrains among active PGs by ratio. + size_t nActive = activePgs.size(); + std::vector<float> activeRatio(nActive); + for (size_t i = 0; i < nActive; ++i) + activeRatio[i] = ratioList[activePgs[i]]; + + std::vector<SlotCount> slotsForActive(nActive, 0); + std::vector<size_t> grainsForActive(nActive, 0); + size_t budget = remainingGrains; + + // Sort indices by ratio ascending. + std::vector<size_t> idxLst(nActive); + std::iota(idxLst.begin(), idxLst.end(), size_t{0}); + std::sort(idxLst.begin(), idxLst.end(), [&](size_t a, size_t b) { return activeRatio[a] < activeRatio[b]; }); + + for (size_t i = 0; i < idxLst.size(); ++i) + { + size_t idx = idxLst[i]; + double ratioSum = 0.0; + for (size_t j = i; j < idxLst.size(); ++j) + ratioSum += static_cast<double>(activeRatio[idxLst[j]]); + double pct = (ratioSum > 0.0) ? static_cast<double>(activeRatio[idx]) / ratioSum : 1.0; + auto [slots, used] = grainsToSlots(static_cast<size_t>(std::nearbyint(static_cast<double>(budget) * pct)), + sizeLists[activePgs[idx]], granularity); + slotsForActive[idx] = slots; + grainsForActive[idx] = used; + TLLM_CHECK_DEBUG(used <= budget); + budget -= used; + } + + // Identify constrained PGs (slots <= min_slots). + std::vector<size_t> constrained; + std::vector<size_t> unconstrained; + for (size_t idx = 0; idx < nActive; ++idx) + { + PoolGroupIndex pgIdx = activePgs[idx]; + SlotCount const minSlotCount = minSlots[pgIdx]; + if (slotsForActive[idx] <= minSlotCount) + constrained.push_back(idx); + else + unconstrained.push_back(idx); + } + + if (constrained.empty()) + { + // All active PGs are unconstrained — accept their allocations. + for (size_t idx = 0; idx < nActive; ++idx) + slotCntList[activePgs[idx]] = slotsForActive[idx]; + break; + } + + // Pin constrained PGs to min_slots and subtract from budget. + for (size_t idx : constrained) + { + PoolGroupIndex pgIdx = activePgs[idx]; + SlotCount const minSlotCount = minSlots[pgIdx]; + size_t minGrains = grainsForSlots(minSlotCount, sizeLists[pgIdx], granularity); + auto [slots, used] = grainsToSlots(minGrains, sizeLists[pgIdx], granularity); + slotCntList[pgIdx] = slots; + TLLM_CHECK_DEBUG(used <= remainingGrains); + remainingGrains -= used; + } + + if (unconstrained.empty()) + { + // All PGs are constrained — nothing left to redistribute. + break; + } + + if (remainingGrains == 0) + throw std::runtime_error("Insufficient quota to satisfy min_slots constraints"); + + // Continue with unconstrained PGs only. + std::vector<PoolGroupIndex> newActivePgs; + newActivePgs.reserve(unconstrained.size()); + for (size_t idx : unconstrained) + newActivePgs.push_back(activePgs[idx]); + activePgs = std::move(newActivePgs); + } + + // _g2s may under-count slots due to imperfect grain distribution + // across pools. Try bumping each PG's slot count while it still fits + // within the same grain budget. + for (PoolGroupIndex pgIdx{0}; pgIdx < sizeLists.size(); ++pgIdx) + { + size_t grainsNow = grainsForSlots(slotCntList[pgIdx], sizeLists[pgIdx], granularity); + while (grainsForSlots(slotCntList[pgIdx] + 1, sizeLists[pgIdx], granularity) <= grainsNow) + slotCntList[pgIdx] += 1; + } + + return slotCntList; +} + +// Instance convenience wrapper. +TypedVec<PoolGroupIndex, SlotCount> CacheLevelStorage::computeSlotCountList( + TypedVec<PoolGroupIndex, float> const& ratioList, TypedVec<PoolGroupIndex, SlotCount> const& minSlots, + std::optional<size_t> quota) const +{ + size_t q = quota.value_or(totalQuota()); + TLLM_CHECK_DEBUG(ratioList.size() == mPoolGroups.size()); + return ratioToSlotCountList(q, slotSizeLists(), ratioList, poolSizeGranularity(), minSlots); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/core.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/core.h new file mode 100644 index 000000000000..eaea0c4a26b2 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/core.h @@ -0,0 +1,563 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/cudaVirtMem.h" +#include "kv_cache_manager_v2/exceptions.h" +#include "kv_cache_manager_v2/storage/config.h" +#include "kv_cache_manager_v2/utils/cudaEvent.h" +#include "kv_cache_manager_v2/utils/hostMem.h" +#include "kv_cache_manager_v2/utils/math.h" +#include "tensorrt_llm/common/assert.h" + +#include <deque> +#include <memory> +#include <optional> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// Slot — represents ownership of one allocated slot in a pool group. +// Mirrors _storage/_core.py::Slot. +// +// ready_event: completes when the slot's data is safe to use. +// - Newly allocated: completes when previous users are done. +// - After migration: completes when the copy is done. +// - Passed to release(): completes when current users are done. +// --------------------------------------------------------------------------- +struct Slot +{ + CachedCudaEvent readyEvent = CachedCudaEvent::makeNull(); + + // Mirrors Python @property slot_id: asserts valid, returns unwrapped value. + [[nodiscard]] SlotId slotId() const + { + return mSlotId.value(); + } + + [[nodiscard]] bool hasValidSlot() const noexcept + { + return mSlotId.has_value(); + } + + void setSlotId(SlotId id) noexcept + { + mSlotId = id; + } + + void resetSlot() noexcept + { + mSlotId.reset(); + } + + bool queryReady() + { + return readyEvent.queryComplete(); + } + + // Transfer slot ownership: moves slotId and readyEvent from src to this. + void setSlot(Slot& src) + { + if (hasValidSlot()) + { + throw LogicError("Slot::setSlot: already has a valid slot"); + } + mSlotId = src.mSlotId; + readyEvent = std::move(src.readyEvent); + src.mSlotId.reset(); + } + +private: + std::optional<SlotId> mSlotId; +}; + +// --------------------------------------------------------------------------- +// SlotAllocator — manages a fixed-capacity array of slot ids. +// Mirrors _storage/_core.py::SlotAllocator. +// --------------------------------------------------------------------------- +class SlotAllocator +{ +public: + explicit SlotAllocator(SlotCount capacity); + ~SlotAllocator(); + + [[nodiscard]] SlotCount numFreeSlots() const noexcept; + [[nodiscard]] SlotCount numOccupiedSlots() const noexcept; + + [[nodiscard]] SlotCount numSlots() const noexcept + { + return mCapacity; + } + + Slot allocate(); + std::vector<Slot> allocateMultiple(SlotCount numSlots); + void release(Slot slot); + + void expand(SlotCount newNumSlots); + void prepareForShrink(SlotCount newNumSlots); + bool finishShrink(); + + [[nodiscard]] bool shrinkInProgress() const noexcept + { + return mTargetCapacity < mCapacity; + } + + [[nodiscard]] std::vector<SlotId> getSlotsBlockingShrink() const; + + // Read-only accessors for debug assertions (mirrors Python's direct attribute access). + [[nodiscard]] SlotCount numOverflowSlots() const noexcept + { + return slotCountValueFromSize(mOverflowSlots.size()); + } + + [[nodiscard]] SlotCount numActiveSlots() const noexcept + { + return mNumActiveSlots; + } + + [[nodiscard]] SlotCount targetCapacity() const noexcept + { + return mTargetCapacity; + } + + void synchronize(); + +private: + void scrubEvents(); + [[nodiscard]] bool check() const noexcept; + + SlotCount mCapacity; + SlotCount mTargetCapacity; + SlotCount mNumActiveSlots; + SlotCount mNumReadyRecycledSlots; + std::deque<Slot> mRecycledSlots; + std::vector<Slot> mOverflowSlots; + DynamicBitset mOccupiedMask; +}; + +// --------------------------------------------------------------------------- +// SlotPoolBase — abstract base for a single memory pool. +// Mirrors _storage/_core.py::SlotPoolBase. +// --------------------------------------------------------------------------- +class SlotPoolBase +{ +public: + explicit SlotPoolBase(size_t slotSize) + : mSlotSize(slotSize) + { + } + + virtual ~SlotPoolBase() = default; + + size_t slotSize() const noexcept + { + return mSlotSize; + } + + virtual SlotCount numSlots() const noexcept = 0; + + size_t numBytes() const noexcept + { + return mSlotSize * slotCountToSizeT(numSlots()); + } + + virtual void destroy() = 0; + virtual void resize(SlotCount newNumSlots) = 0; + virtual Address slotAddress(SlotId slot) const = 0; + +protected: + size_t mSlotSize; +}; + +// --------------------------------------------------------------------------- +// GpuSlotPool — GPU virtual memory pool. +// --------------------------------------------------------------------------- +class GpuSlotPool : public SlotPoolBase +{ +public: + GpuSlotPool(size_t slotSize, size_t vmSize, PooledPhysMemAllocator& physMemAllocator, SlotCount numSlots); + + SlotCount numSlots() const noexcept override; + void destroy() override; + void resize(SlotCount newNumSlots) override; + Address slotAddress(SlotId slot) const override; + + // Extend by exactly one physical memory chunk; returns new numSlots. + SlotCount extendByOnePhysMem(); + + static size_t computeNumPhysMem(size_t slotSize, SlotCount numSlots, size_t physMemSize) noexcept; + static SlotCount computeNumSlots(size_t slotSize, size_t numPhysMem, size_t physMemSize) noexcept; + +private: + VirtMem mVirtMem; +}; + +// --------------------------------------------------------------------------- +// HostSlotPool — pinned host memory pool. +// --------------------------------------------------------------------------- +class HostSlotPool : public SlotPoolBase +{ +public: + HostSlotPool(size_t slotSize, SlotCount numSlots); + + SlotCount numSlots() const noexcept override; + void destroy() override; + void resize(SlotCount newNumSlots) override; + Address slotAddress(SlotId slot) const override; + + size_t alignedSize(SlotCount numSlots) const noexcept; + +private: + HostMem mHostMem; +}; + +// --------------------------------------------------------------------------- +// DiskSlotPool — temp-file backed disk pool. +// --------------------------------------------------------------------------- +class DiskSlotPool : public SlotPoolBase +{ +public: + // directory: path under which to create the temp file. + DiskSlotPool(std::string const& directory, size_t slotSize, SlotCount numSlots); + ~DiskSlotPool() override; + + SlotCount numSlots() const noexcept override; + void destroy() override; + void resize(SlotCount newNumSlots) override; + Address slotAddress(SlotId slot) const override; + + int fd() const noexcept + { + return mFd; + } + +private: + int mFd = kBadFileDescriptor; +}; + +// --------------------------------------------------------------------------- +// PoolGroupBase — manages multiple pools with mirrored slot allocation. +// Mirrors _storage/_core.py::PoolGroupBase. +// --------------------------------------------------------------------------- +class PoolGroupBase +{ +public: + explicit PoolGroupBase(SlotCount numSlots); + virtual ~PoolGroupBase(); + + PoolIndex numPools() const noexcept + { + return mPools.size(); + } + + SlotCount numSlots() const noexcept; + + SlotCount numFreeSlots() const noexcept + { + return mSlotAllocator.numFreeSlots(); + } + + SlotAllocator& slotAllocator() noexcept + { + return mSlotAllocator; + } + + SlotAllocator const& slotAllocator() const noexcept + { + return mSlotAllocator; + } + + Slot allocate(); + std::vector<Slot> allocateMultiple(SlotCount numSlots); + void release(Slot slot); + void destroy(); + void resizePools(std::optional<SlotCount> newNumSlots = std::nullopt); + + // Addresses for all pools at a given slot id. + TypedVec<PoolIndex, Address> slotAddress(SlotId slotId) const; + // Sizes per pool. + TypedVec<PoolIndex, size_t> slotSize() const; + + // Total bytes across all pools for one slot. + size_t numBytes() const noexcept + { + size_t total = 0; + for (auto const& pool : mPools) + total += pool->numBytes(); + return total; + } + + // Total bytes across all pools, rounding each pool's bytes up to granularity. + // Mirrors Python total_quota: sum(round_up(p.num_bytes, granularity) for p in pg._pools). + size_t roundedNumBytes(size_t granularity) const noexcept + { + size_t total = 0; + for (auto const& pool : mPools) + total += roundUp(pool->numBytes(), granularity); + return total; + } + +protected: + // Mirrors Python _get_num_slots_from_pools: min(p.num_slots for p in pools). + SlotCount getNumSlotsFromPools() const noexcept; + + SlotAllocator mSlotAllocator; + TypedVec<PoolIndex, std::unique_ptr<SlotPoolBase>> mPools; + bool mDestroyed = false; +}; + +// --------------------------------------------------------------------------- +// GpuPoolGroup / HostPoolGroup / DiskPoolGroup +// --------------------------------------------------------------------------- +class GpuPoolGroup : public PoolGroupBase +{ +public: + GpuPoolGroup( + SlotCount numSlots, TypedVec<PoolIndex, size_t> const& slotSizeList, PooledPhysMemAllocator& physMemAllocator); +}; + +class HostPoolGroup : public PoolGroupBase +{ +public: + HostPoolGroup(SlotCount numSlots, TypedVec<PoolIndex, size_t> const& slotSizeList); +}; + +class DiskPoolGroup : public PoolGroupBase +{ +public: + DiskPoolGroup(SlotCount numSlots, TypedVec<PoolIndex, size_t> const& slotSizeList, std::string const& directory); +}; + +// --------------------------------------------------------------------------- +// CacheLevelStorage — manages all pool groups for one cache tier. +// Mirrors _storage/_core.py::CacheLevelStorage. +// --------------------------------------------------------------------------- +class CacheLevelStorage +{ +public: + virtual ~CacheLevelStorage() = default; + + virtual CacheTier cacheTier() const noexcept = 0; + + PoolGroupIndex numPoolGroups() const noexcept + { + return mPoolGroups.size(); + } + + std::vector<Slot> allocateMultiple(PoolGroupIndex pgIdx, SlotCount numSlots); + void release(PoolGroupIndex pgIdx, Slot slot); + SlotCount numFreeSlots(PoolGroupIndex pgIdx) const; + TypedVec<PoolIndex, Address> slotAddress(PoolGroupIndex pgIdx, SlotId slotId) const; + + // Additional accessors used by StorageManager and KvCacheManager. + virtual void destroy() + { + for (auto& pg : mPoolGroups) + pg->destroy(); + } + + PoolIndex numPools(PoolGroupIndex pgIdx) const + { + return mPoolGroups.at(pgIdx)->numPools(); + } + + SlotCount numSlots(PoolGroupIndex pgIdx) const + { + return mPoolGroups.at(pgIdx)->numSlots(); + } + + TypedVec<PoolIndex, size_t> slotSize(PoolGroupIndex pgIdx) const + { + auto szl = mPoolGroups.at(pgIdx)->slotSize(); + TypedVec<PoolIndex, size_t> ret; + ret.reserve(szl.size()); + for (auto sz : szl) + ret.push_back(sz); + return ret; + } + + PoolGroupBase& poolGroup(PoolGroupIndex pgIdx) + { + return *mPoolGroups.at(pgIdx); + } + + MemAddress getBaseAddress(PoolGroupIndex pgIdx, PoolIndex poolIdx, SlotId slotId) const + { + return std::get<MemAddress>(mPoolGroups.at(pgIdx)->slotAddress(slotId).at(poolIdx)); + } + + size_t totalQuota() const noexcept + { + size_t granularity = poolSizeGranularity(); + size_t total = 0; + for (auto const& pg : mPoolGroups) + total += pg->roundedNumBytes(granularity); + return total; + } + + // Returns numSlots() per pool group. + TypedVec<PoolGroupIndex, SlotCount> slotCountList() const + { + TypedVec<PoolGroupIndex, SlotCount> ret; + ret.reserve(mPoolGroups.size()); + for (auto const& pg : mPoolGroups) + ret.push_back(pg->numSlots()); + return ret; + } + + TypedVec<PoolGroupIndex, TypedVec<PoolIndex, size_t>> slotSizeLists() const + { + TypedVec<PoolGroupIndex, TypedVec<PoolIndex, size_t>> ret; + ret.reserve(mPoolGroups.size()); + for (auto const& pg : mPoolGroups) + { + auto szl = pg->slotSize(); + TypedVec<PoolIndex, size_t> sizes; + sizes.reserve(szl.size()); + for (auto sz : szl) + sizes.push_back(sz); + ret.push_back(std::move(sizes)); + } + return ret; + } + + // Current ratio list: proportion of bytes per pool group. + TypedVec<PoolGroupIndex, float> ratioList() const + { + TypedVec<PoolGroupIndex, float> ret(mPoolGroups.size(), 0.f); + float total = 0.f; + for (PoolGroupIndex pgIdx{0}; pgIdx < mPoolGroups.size(); ++pgIdx) + { + auto sz = static_cast<float>(mPoolGroups[pgIdx]->numBytes()); + ret[pgIdx] = sz; + total += sz; + } + TLLM_CHECK_DEBUG(total > 0.f); + for (auto& r : ret) + r /= total; + return ret; + } + + virtual size_t poolSizeGranularity() const noexcept + { + return size_t{2} << 20; + } + + // Compute slot counts per pool group for a given ratio, min_slots, and optional quota. + // Instance convenience method — delegates to the static version below. + TypedVec<PoolGroupIndex, SlotCount> computeSlotCountList(TypedVec<PoolGroupIndex, float> const& ratioList, + TypedVec<PoolGroupIndex, SlotCount> const& minSlots, std::optional<size_t> quota = std::nullopt) const; + + // Static version: compute slot counts from quota, slot size lists, ratio, granularity, and min_slots. + // Mirrors Python CacheLevelStorage.ratio_to_slot_count_list. + static TypedVec<PoolGroupIndex, SlotCount> ratioToSlotCountList(size_t quota, + TypedVec<PoolGroupIndex, TypedVec<PoolIndex, size_t>> const& slotSizeLists, + TypedVec<PoolGroupIndex, float> const& ratioList, size_t granularity, + TypedVec<PoolGroupIndex, SlotCount> const& minSlots); + + // Distribute grains among pools within a pool group. + // Returns {num_slots, grains_consumed}. + static std::pair<SlotCount, size_t> grainsToSlots( + size_t pgGrains, TypedVec<PoolIndex, size_t> const& slotSizeList, size_t granularity); + + // Compute minimum grains needed for numSlots in a pool group. + static size_t grainsForSlots( + SlotCount numSlots, TypedVec<PoolIndex, size_t> const& slotSizeList, size_t granularity); + + virtual void postResize() {} + +protected: + TypedVec<PoolGroupIndex, std::unique_ptr<PoolGroupBase>> mPoolGroups; +}; + +class GpuCacheLevelStorage : public CacheLevelStorage +{ +public: + GpuCacheLevelStorage( + StorageConfig const& storageCfg, TypedVec<PoolGroupIndex, SlotCount> const& slotCountList, size_t physMemSize); + + CacheTier cacheTier() const noexcept override + { + return CacheTier::GPU_MEM; + } + + size_t poolSizeGranularity() const noexcept override + { + return mPhysMemAllocator->physMemSize(); + } + + void postResize() override + { + CacheLevelStorage::postResize(); + mPhysMemAllocator->clear(); + } + + void destroy() override + { + CacheLevelStorage::destroy(); + mPhysMemAllocator->clear(); + } + +private: + std::unique_ptr<PooledPhysMemAllocator> mPhysMemAllocator; +}; + +class HostCacheLevelStorage : public CacheLevelStorage +{ +public: + HostCacheLevelStorage(StorageConfig const& storageCfg, TypedVec<PoolGroupIndex, SlotCount> const& slotCountList); + + CacheTier cacheTier() const noexcept override + { + return CacheTier::HOST_MEM; + } + + size_t poolSizeGranularity() const noexcept override + { + return HostMem::kAlignment; + } +}; + +class DiskCacheLevelStorage : public CacheLevelStorage +{ +public: + DiskCacheLevelStorage(StorageConfig const& storageCfg, TypedVec<PoolGroupIndex, SlotCount> const& slotCountList, + std::string directory); + + CacheTier cacheTier() const noexcept override + { + return CacheTier::DISK; + } + + std::string const& directory() const noexcept + { + return mDirectory; + } + +private: + std::string mDirectory; +}; + +// Factory: create appropriate CacheLevelStorage for a given tier config. +std::unique_ptr<CacheLevelStorage> createCacheLevelStorage(CacheTierConfig const& tierCfg, + StorageConfig const& storageCfg, TypedVec<PoolGroupIndex, SlotCount> const& slotCountList); + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storageManager.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storageManager.cpp new file mode 100644 index 000000000000..724a33e308b0 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storageManager.cpp @@ -0,0 +1,1292 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/storageManager.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/copyEngine.h" +#include "kv_cache_manager_v2/exceptions.h" +#include "kv_cache_manager_v2/page.h" +#include "kv_cache_manager_v2/utils/hostMem.h" +#include "kv_cache_manager_v2/utils/math.h" +#include "tensorrt_llm/common/logger.h" + +#include "tensorrt_llm/common/assert.h" +#include <algorithm> +#include <cmath> +#include <cstddef> +#include <numeric> +#include <set> +#include <string> +#include <utility> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// CacheLevelManager +// --------------------------------------------------------------------------- + +CacheLevelManager::CacheLevelManager(TypedVec<LifeCycleId, PoolGroupIndex> const& lifeCycleGrouping, CacheLevel cl, + CacheTierConfig const& tierConfig, StorageConfig const& storageConfig, + TypedVec<PoolGroupIndex, SlotCount> const& slotCountList) + : cacheLevel(cl) + , cacheTier(CacheTier(tierConfig.index())) + , controller(lifeCycleGrouping, cl) +{ + storage = createCacheLevelStorage(tierConfig, storageConfig, slotCountList); +} + +size_t CacheLevelManager::cacheTierGranularity(CacheTier tier, size_t quota) +{ + switch (tier) + { + case CacheTier::GPU_MEM: + { + constexpr size_t kPageSize = 2ULL << 20; + return kPageSize << std::min(4, std::max(0, static_cast<int>(std::log2(quota / (kPageSize * 512))))); + } + case CacheTier::HOST_MEM: return HostMem::kAlignment; // 4 KiB + case CacheTier::DISK: return size_t{2} << 20; // DiskCacheLevelStorage::POOL_SIZE_GRANULARITY + default: throw std::invalid_argument("Invalid cache tier"); + } +} + +// --------------------------------------------------------------------------- +// StorageManager constructor helpers +// --------------------------------------------------------------------------- + +namespace +{ + +// Compute the slot-to-page-indices scale factors. +// For each (lcId, poolIdx), scale = numBuffersInCoalescedSlot. +// Python: _slot_to_page_indices[lc_id][pool_idx] = numBuffers +TypedVec<LifeCycleId, TypedVec<PoolIndex, int>> computeSlotToPageIndices(StorageConfig const& config) +{ + LifeCycleId numLc = config.numLifeCycles(); + TypedVec<LifeCycleId, TypedVec<PoolIndex, int>> result(numLc); + + auto const& slotDescList = config.slotDescList; + auto const& grouping = config.lifeCycleGrouping(); + + for (LifeCycleId lcId{0}; lcId < result.size(); ++lcId) + { + PoolGroupIndex pgIdx = grouping[lcId]; + SlotDesc const& sd = slotDescList.at(pgIdx); + // Find the variant that corresponds to this lifecycle. + for (auto const& variant : sd.variants) + { + if (variant.lifeCycleId == lcId) + { + // Each coalesced buffer contributes its numBuffers as the scale. + result[lcId].reserve(variant.coalescedBuffers.size()); + for (auto const& cb : variant.coalescedBuffers) + result[lcId].push_back(cb.numBuffers()); + break; + } + } + if (result[lcId].empty()) + result[lcId].push_back(1); // fallback + } + return result; +} + +} // namespace + +// --------------------------------------------------------------------------- +// StorageManager +// --------------------------------------------------------------------------- + +StorageManager::StorageManager(LifeCycleRegistry const& lifeCycles, StorageConfig const& config, int tokensPerBlock, + std::optional<SwaScratchReuseConfig> swaScratchReuse, std::optional<BatchDesc> const& typicalBatch, + std::vector<BatchDesc> const& constraints, std::optional<std::vector<float>> const& initialPoolRatio, + std::shared_ptr<EventSink> eventSink, float maxUtilForResume) + : mLifeCycles(lifeCycles) + , mEventSink(std::move(eventSink)) + , mStorageConfig(config) + , mSwaScratchReuse(std::move(swaScratchReuse)) +{ + mLifeCycleGrouping = config.lifeCycleGrouping(); + mLayerToLifeCycleIds = config.layerToLifeCycleIds(); + mSlotToPageIndices = computeSlotToPageIndices(config); + mBufferAttr = config.bufferAttributes(); + mSlotDescList = config.slotDescList; + + // Compute layer attributes and slot utilization fractions for scratch support. + mLayerAttributes = config.layerAttributes(); + mSlotUtilFracMax.resize(lifeCycles.size(), Rational{0, 1}); + for (auto const& [layerId, layerAttr] : mLayerAttributes) + { + LifeCycleId const lcIdx = layerAttr.lifeCycleId; + if (layerAttr.slotUtilFracMax > mSlotUtilFracMax[lcIdx]) + { + mSlotUtilFracMax[lcIdx] = layerAttr.slotUtilFracMax; + } + } + + TLLM_CHECK_DEBUG(std::all_of(mLifeCycleGrouping.begin(), mLifeCycleGrouping.end(), + [this](PoolGroupIndex pg) { return pg < numPoolGroups(); })); + TLLM_CHECK_DEBUG(numPoolGroups() + == PoolGroupIndex{ + static_cast<int>(std::set<PoolGroupIndex>(mLifeCycleGrouping.begin(), mLifeCycleGrouping.end()).size())}); + + // Build one CacheLevelManager per tier. + TLLM_CHECK_DEBUG(!config.cacheTiers.empty()); + TLLM_CHECK_DEBUG_WITH_INFO( + std::holds_alternative<GpuCacheTierConfig>(config.cacheTiers[kGpuLevel]), "First cache tier must be GPU"); + + // Compute slot size lists for all pool groups. + TypedVec<PoolGroupIndex, TypedVec<PoolIndex, size_t>> slotSizeLists; + slotSizeLists.reserve(mSlotDescList.size()); + for (auto const& sd : mSlotDescList) + { + slotSizeLists.push_back(sd.slotSizeList()); + } + + size_t gpuQuota = cacheTierQuota(config.cacheTiers[kGpuLevel]); + size_t gpuGranularity = CacheLevelManager::cacheTierGranularity(CacheTier::GPU_MEM, gpuQuota); + + // Constraints stay feasibility floors even under an explicit initial pool + // ratio (a share below what a declared batch needs is clamped up), and the + // floors are scaled by 1/maxUtilForResume because KvCache::resume rejects any + // pool group above that utilization. Mirrors PR#16269 on the Python side. + mMinSlots = computeMinSlotsFromConstraints(constraints, tokensPerBlock, mSwaScratchReuse, maxUtilForResume); + + // Compute init_ratio from explicit config, typical_batch, constraints, or fallback. + TypedVec<PoolGroupIndex, float> initRatio; + if (initialPoolRatio.has_value()) + { + if (initialPoolRatio->size() != toSizeT(numPoolGroups())) + { + throw std::invalid_argument("initial_pool_ratio length must match number of pool groups (" + + std::to_string(toSizeT(numPoolGroups())) + "), got " + std::to_string(initialPoolRatio->size())); + } + if (std::any_of(initialPoolRatio->begin(), initialPoolRatio->end(), [](float ratio) { return ratio <= 0.0F; })) + { + throw std::invalid_argument("initial_pool_ratio values must be positive"); + } + + constexpr double kExpectedRatioSum = 1.0; + constexpr double kRatioSumTolerance = 1e-6; + double const ratioSum = std::accumulate(initialPoolRatio->begin(), initialPoolRatio->end(), 0.0); + if (!std::isfinite(ratioSum) || std::abs(ratioSum - kExpectedRatioSum) > kRatioSumTolerance) + { + throw std::invalid_argument("initial_pool_ratio values must sum to 1.0"); + } + initRatio = TypedVec<PoolGroupIndex, float>(*initialPoolRatio); + } + else if (typicalBatch.has_value()) + { + initRatio = ratioFromBatch(*typicalBatch, tokensPerBlock, mSwaScratchReuse, gpuGranularity); + } + else if (!constraints.empty()) + { + // Use the constraint slot counts as the ratio basis. + auto minBytes = slotsToBytes(mMinSlots, gpuGranularity); + initRatio = normalizeToRatio(minBytes); + } + else + { + // Fallback: average history length 2048. + BatchDesc fallback; + fallback.kvCaches.push_back(KVCacheDesc{2049, 2048}); + initRatio = ratioFromBatch(fallback, tokensPerBlock, mSwaScratchReuse, gpuGranularity); + } + + mLevels.reserve(config.cacheTiers.size()); + for (CacheLevel level{0}; level < config.cacheTiers.size(); ++level) + { + auto slotCountList = computeSlotCountForLevel(config.cacheTiers[level], slotSizeLists, initRatio); + mLevels.emplace_back(mLifeCycleGrouping, level, config.cacheTiers[level], config, slotCountList); + } + + TLLM_CHECK_DEBUG(mLevels.empty() + || numPoolGroups() + == getUniformAttribute(mLevels, [](auto const& lvl) { return lvl.storage->numPoolGroups(); })); +} + +StorageManager::~StorageManager() +{ + destroy(); +} + +void StorageManager::destroy() +{ + for (auto& lvl : mLevels) + { + TLLM_CHECK_DEBUG(lvl.storage); + lvl.storage->destroy(); + } + mLevels.clear(); +} + +// --------------------------------------------------------------------------- +// newSlots +// --------------------------------------------------------------------------- + +TypedVec<LifeCycleId, std::vector<Slot>> StorageManager::newSlots(CacheLevel level, + TypedVec<LifeCycleId, SlotCount> const& numSlotsPerLc, MigrationRecorder const& migrationRecorder, + DropRecorder const& dropRecorder) +{ + TLLM_CHECK_DEBUG(numSlotsPerLc.size() == numLifeCycles()); + auto& storage = *mLevels.at(level).storage; + + // Aggregate by pool group. + TypedVec<PoolGroupIndex, SlotCount> pgNumSlots(numPoolGroups(), 0); + for (LifeCycleId lcId{0}; lcId < numSlotsPerLc.size(); ++lcId) + { + SlotCount const numSlots = numSlotsPerLc[lcId]; + if (numSlots < 0) + { + throw LogicError("StorageManager::newSlots: slot count must be non-negative"); + } + pgNumSlots[mLifeCycleGrouping[lcId]] += numSlots; + } + + // Prepare free slots if needed. + bool needMore = false; + for (PoolGroupIndex pgIdx{0}; pgIdx < pgNumSlots.size(); ++pgIdx) + { + if (pgNumSlots[pgIdx] > storage.numFreeSlots(pgIdx)) + { + needMore = true; + break; + } + } + + if (needMore) + { + prepareFreeSlots(level, pgNumSlots, migrationRecorder, dropRecorder); + } + + // A14: post-condition — free-slot counts satisfy requirements. + for (PoolGroupIndex pgIdx{0}; pgIdx < pgNumSlots.size(); ++pgIdx) + { + TLLM_CHECK_DEBUG_WITH_INFO(pgNumSlots[pgIdx] <= storage.numFreeSlots(pgIdx), + "Free slot count does not satisfy requirement after prepareFreeSlots"); + } + + // Allocate. + TypedVec<LifeCycleId, std::vector<Slot>> ret(numLifeCycles()); + try + { + for (LifeCycleId lcId{0}; lcId < ret.size(); ++lcId) + { + PoolGroupIndex pg = mLifeCycleGrouping[lcId]; + ret[lcId] = storage.allocateMultiple(pg, numSlotsPerLc[lcId]); + } + } + catch (...) + { + for (LifeCycleId lcId{0}; lcId < ret.size(); ++lcId) + { + PoolGroupIndex pg = mLifeCycleGrouping[lcId]; + for (auto& s : ret[lcId]) + storage.release(pg, std::move(s)); + } + throw; + } + return ret; +} + +TypedVec<LifeCycleId, std::vector<Slot>> StorageManager::newGpuSlots( + TypedVec<LifeCycleId, SlotCount> const& numSlotsPerLc, MigrationRecorder const& migrationRecorder, + DropRecorder const& dropRecorder) +{ + return newSlots(kGpuLevel, numSlotsPerLc, migrationRecorder, dropRecorder); +} + +std::vector<Slot> StorageManager::newSlotsForPoolGroup(CacheLevel level, PoolGroupIndex pgIdx, SlotCount numSlots, + MigrationRecorder const& migrationRecorder, DropRecorder const& dropRecorder) +{ + if (numSlots < 0) + { + throw LogicError("StorageManager::newSlotsForPoolGroup: numSlots must be non-negative"); + } + auto& storage = *mLevels.at(level).storage; + if (numSlots > storage.numFreeSlots(pgIdx)) + { + TypedVec<PoolGroupIndex, SlotCount> requirements(numPoolGroups(), 0); + requirements.at(pgIdx) = numSlots; + prepareFreeSlots(level, requirements, migrationRecorder, dropRecorder); + } + TLLM_CHECK_DEBUG(numSlots <= storage.numFreeSlots(pgIdx)); + return storage.allocateMultiple(pgIdx, numSlots); +} + +Address StorageManager::slotAddress(CacheLevel level, PoolGroupIndex pgIdx, SlotId slotId, PoolIndex poolIdx) const +{ + return mLevels.at(level).storage->slotAddress(pgIdx, slotId).at(poolIdx); +} + +CacheTier StorageManager::cacheTier(CacheLevel level) const +{ + return mLevels.at(level).cacheTier; +} + +void StorageManager::releaseSlot(LifeCycleId lc, CacheLevel level, Slot slot) +{ + PoolGroupIndex pg = mLifeCycleGrouping.at(lc); + mLevels.at(level).storage->release(pg, std::move(slot)); +} + +// --------------------------------------------------------------------------- +// isEvictable +// --------------------------------------------------------------------------- + +bool StorageManager::isEvictable(Page const& page, std::optional<CacheLevel> level) const noexcept +{ + PageStatus s = page.status(); + CacheLevel lvl = level.value_or(page.cacheLevel); + return (s == PageStatus::DROPPABLE && page.isCommitted()) || (s == PageStatus::HELD && lvl < numCacheLevels() - 1); +} + +// --------------------------------------------------------------------------- +// scheduleForEviction / excludeFromEviction +// --------------------------------------------------------------------------- + +void StorageManager::scheduleForEviction(Page& page) +{ + if (isEvictable(page)) + mLevels.at(page.cacheLevel).controller.scheduleForEviction(page); +} + +void StorageManager::excludeFromEviction(Page& page) +{ + TLLM_CHECK_DEBUG(page.nodeRef.has_value()); + mLevels.at(page.cacheLevel).controller.remove(*page.nodeRef); +} + +// --------------------------------------------------------------------------- +// prepareFreeSlots +// --------------------------------------------------------------------------- + +void StorageManager::prepareFreeSlots(CacheLevel level, TypedVec<PoolGroupIndex, SlotCount> const& requirements, + MigrationRecorder const& migrationRecorder, DropRecorder const& dropRecorder) +{ + TypedVec<CacheLevel, TypedVec<PoolGroupIndex, SlotCount>> goals(numCacheLevels()); + for (CacheLevel lvl{0}; lvl < goals.size(); ++lvl) + { + goals[lvl].resize(numPoolGroups(), 0); + } + for (PoolGroupIndex pgIdx{0}; pgIdx < requirements.size(); ++pgIdx) + { + goals.at(level).at(pgIdx) = requirements.at(pgIdx); + } + + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> fallenPages(numPoolGroups()); + _prepareFreeSlots(goals, level, fallenPages, migrationRecorder, dropRecorder); +} + +void StorageManager::forceEvict( + CacheLevel level, TypedVec<PoolGroupIndex, SlotCount> const& minNumPages, DropRecorder const& dropRecorder) +{ + auto evicted = mLevels.at(level).controller.evict(minNumPages); + + if (isLastLevel(level)) + { + // Last level: all evicted pages must be DROPPABLE (they get dropped, not migrated). + for (auto const& pages : evicted) + { + for (auto const& page : pages) + { + TLLM_CHECK_DEBUG_WITH_INFO(page->status() == PageStatus::DROPPABLE, "Corrupted eviction controller"); + } + } + if (dropRecorder) + { + for (auto const& pages : evicted) + { + if (!pages.empty()) + { + dropRecorder(pages, level); + } + } + } + return; + } + + TypedVec<CacheLevel, TypedVec<PoolGroupIndex, SlotCount>> goals(numCacheLevels()); + for (CacheLevel lvl{0}; lvl < goals.size(); ++lvl) + { + goals[lvl].resize(numPoolGroups(), 0); + } + CacheLevel nextLvl = level + 1; + + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> fallen(numPoolGroups()); + for (PoolGroupIndex pgIdx{0}; pgIdx < fallen.size(); ++pgIdx) + { + for (auto& sp : evicted.at(pgIdx)) + fallen.at(pgIdx).push_back(sp); + } + _prepareFreeSlots(goals, nextLvl, fallen, MigrationRecorder{}, dropRecorder); +} + +// --------------------------------------------------------------------------- +// _prepareFreeSlots (recursive) +// --------------------------------------------------------------------------- + +void StorageManager::_prepareFreeSlots(TypedVec<CacheLevel, TypedVec<PoolGroupIndex, SlotCount>>& goals, + CacheLevel lvlId, TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>>& fallenPages, + MigrationRecorder const& migrationRecorder, DropRecorder const& dropRecorder) +{ + // A7: goals dimensions must match [numCacheLevels][numPoolGroups]. + if (TLLM_UNLIKELY(gDebug)) + { + TLLM_CHECK_WITH_INFO(goals.size() == numCacheLevels(), "goals.rows must equal numCacheLevels"); + TLLM_CHECK_DEBUG_WITH_INFO( + std::all_of(goals.begin(), goals.end(), [this](auto const& row) { return row.size() == numPoolGroups(); }), + "goals.cols must equal numPoolGroups"); + } + + // A8: all fallen pages must come from upper cache levels (cache_level < lvlId). + TLLM_CHECK_DEBUG_WITH_INFO(std::all_of(fallenPages.begin(), fallenPages.end(), + [lvlId](auto const& pages) { + return std::all_of(pages.begin(), pages.end(), + [lvlId](auto const& p) { return p->cacheLevel < lvlId; }); + }), + "Fallen pages must come from upper cache levels"); + + auto& lvl = mLevels.at(lvlId); + auto& storage = *lvl.storage; + auto& ctrl = lvl.controller; + bool isLast = isLastLevel(lvlId); + + TypedVec<PoolGroupIndex, SlotCount> numToEvict(numPoolGroups(), 0); + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> heldPages(numPoolGroups()); + + for (PoolGroupIndex pgIdx{0}; pgIdx < numToEvict.size(); ++pgIdx) + { + SlotCount const goal = goals.at(lvlId).at(pgIdx); + SlotCount const fallen = slotCountValueFromSize(fallenPages.at(pgIdx).size()); + SlotCount const oldFree = storage.numFreeSlots(pgIdx); + SlotCount const evictableCount = ctrl.numEvictablePages(pgIdx); + SlotCount const required = goal + fallen; + SlotCount const shortage = required > oldFree ? required - oldFree : 0; + numToEvict.at(pgIdx) = std::min(shortage, evictableCount); + + SlotCount fallenHeld = 0; + if (isLast) + { + // Separate held pages from fallen_pages (mirrors Python's remove_if). + auto& fp = fallenPages.at(pgIdx); + heldPages.at(pgIdx) = stealIf(fp, [](SharedPtr<Page> const& p) { return p->status() == PageStatus::HELD; }); + fallenHeld = slotCountValueFromSize(heldPages.at(pgIdx).size()); + + if (fallenHeld > oldFree + evictableCount) + throw OutOfPagesError( + "Too many held pages falling to last-level cache for group " + std::to_string(pgIdx.value())); + } + + if (oldFree + evictableCount < fallenHeld + goal) + throw OutOfPagesError("Impossible to meet free-slot goal " + std::to_string(goal) + " for group " + + std::to_string(pgIdx.value())); + } + + auto evicted = ctrl.evict(numToEvict); + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> acceptedPages(numPoolGroups()); + + if (isLast) + { + for (PoolGroupIndex pgIdx{0}; pgIdx < evicted.size(); ++pgIdx) + { + auto& ev = evicted.at(pgIdx); + SlotCount const oldFree = storage.numFreeSlots(pgIdx); + SlotCount const numEvicted = slotCountValueFromSize(ev.size()); + // A9: all evicted pages at last level must be DROPPABLE. + TLLM_CHECK_DEBUG_WITH_INFO( + std::all_of(ev.begin(), ev.end(), [](auto const& p) { return p->status() == PageStatus::DROPPABLE; }), + "Evicted page at last level must be DROPPABLE"); + // Drop droppable evicted pages (GC). + if (dropRecorder && !ev.empty()) + { + dropRecorder(ev, lvlId); + } + ev.clear(); + SlotCount const newFree = storage.numFreeSlots(pgIdx); + TLLM_CHECK_DEBUG(newFree >= numEvicted + oldFree); + + // A10: held_pages count must not exceed new_free. + TLLM_CHECK_DEBUG_WITH_INFO(slotCountValueFromSize(heldPages.at(pgIdx).size()) <= newFree, + "held_pages count exceeds new free slot count"); + + // Add held pages from upper levels. + auto& hp = heldPages.at(pgIdx); + auto& fp = fallenPages.at(pgIdx); + fp.insert(fp.end(), hp.begin(), hp.end()); + hp.clear(); + + SlotCount const goal = goals.at(lvlId).at(pgIdx); + SlotCount const freeAfterGoal = newFree > goal ? newFree - goal : 0; + SlotCount const numAccepted = std::min(freeAfterGoal, slotCountValueFromSize(fp.size())); + if (numAccepted > 0) + { + acceptedPages.at(pgIdx).assign(fp.end() - static_cast<std::ptrdiff_t>(numAccepted), fp.end()); + } + fp.clear(); + } + } + else + { + // A12: no held pages at non-last level. + TLLM_CHECK_DEBUG_WITH_INFO( + std::all_of(heldPages.begin(), heldPages.end(), [](auto const& hp) { return hp.empty(); }), + "held_pages must be empty at non-last level"); + + CacheLevel nextLvl = lvlId + 1; + for (PoolGroupIndex pgIdx{0}; pgIdx < evicted.size(); ++pgIdx) + { + auto& ev = evicted.at(pgIdx); + SlotCount const oldFree = storage.numFreeSlots(pgIdx); + SlotCount const numEvicted = slotCountValueFromSize(ev.size()); + auto& fp = fallenPages.at(pgIdx); + fp.insert(fp.begin(), ev.begin(), ev.end()); // prepend evicted to fallen (preserving order) + ev.clear(); + + SlotCount const goal = goals.at(lvlId).at(pgIdx); + SlotCount const availableAfterGoal = oldFree + numEvicted > goal ? oldFree + numEvicted - goal : 0; + SlotCount const numAccepted = std::min(availableAfterGoal, slotCountValueFromSize(fp.size())); + if (numAccepted > 0) + { + acceptedPages.at(pgIdx).assign(fp.end() - static_cast<std::ptrdiff_t>(numAccepted), fp.end()); + fp.erase(fp.end() - static_cast<std::ptrdiff_t>(numAccepted), fp.end()); + } + } + _prepareFreeSlots(goals, nextLvl, fallenPages, migrationRecorder, dropRecorder); + } + + // A13: all fallen pages must have been consumed. + TLLM_CHECK_DEBUG_WITH_INFO( + std::all_of(fallenPages.begin(), fallenPages.end(), [](auto const& fp) { return fp.empty(); }), + "All fallen pages must be consumed after level loop"); + + // Migrate accepted pages into lvlId. + for (PoolGroupIndex pgIdx{0}; pgIdx < acceptedPages.size(); ++pgIdx) + { + // Group by source level (mirrors Python's partition()). + auto bySrcLevel = partition(acceptedPages.at(pgIdx), [](SharedPtr<Page> const& p) { return p->cacheLevel; }); + + for (auto& [srcLvl, pages] : bySrcLevel) + { + _batchedMigrate(pgIdx, lvlId, srcLvl, pages, /*updateSrc=*/true, migrationRecorder); + for (auto const& p : pages) + { + if (isLast && p->status() == PageStatus::HELD) + continue; + lvl.controller.scheduleForEviction(*p); + } + } + } +} + +// --------------------------------------------------------------------------- +// _batchedMigrate +// --------------------------------------------------------------------------- + +void StorageManager::_batchedMigrate(PoolGroupIndex pgIdx, CacheLevel dstLevel, CacheLevel srcLevel, + std::vector<SharedPtr<Page>> const& srcPages, bool updateSrc, MigrationRecorder const& migrationRecorder, + bool defrag) +{ + TLLM_CHECK_DEBUG(defrag || dstLevel != srcLevel); + SlotCount const numSlots = slotCountValueFromSize(srcPages.size()); + + auto& srcPoolGroup = poolGroup(srcLevel, pgIdx); + auto& dstPoolGroup = poolGroup(dstLevel, pgIdx); + + if (dstPoolGroup.numFreeSlots() < numSlots) + throw OutOfPagesError("Not enough free slots for migration"); + + auto dstSlots = dstPoolGroup.allocateMultiple(numSlots); + // A15: allocated slot count must match the request. + TLLM_CHECK_DEBUG_WITH_INFO(slotCountValueFromSize(dstSlots.size()) == numSlots, "dst_slots size mismatch"); + try + { + CacheTier dstTier = mLevels.at(dstLevel).cacheTier; + CacheTier srcTier = mLevels.at(srcLevel).cacheTier; + + PoolIndex numPools = mNumPools(pgIdx); + + // Build copy tasks per pool. + TypedVec<PoolIndex, std::vector<CopyTask>> tasksPerPool(numPools); + for (std::size_t i = 0; i < srcPages.size(); ++i) + { + auto const& src = srcPages.at(i); + auto const& dst = dstSlots.at(i); + // Fix #8: assert non-defrag migrations only accept pages not scheduled for eviction. + TLLM_CHECK_DEBUG(defrag || !src->scheduledForEviction()); + for (PoolIndex poolIdx{0}; poolIdx < tasksPerPool.size(); ++poolIdx) + { + Address dstAddr = dstPoolGroup.slotAddress(dst.slotId()).at(poolIdx); + Address srcAddr = srcPoolGroup.slotAddress(src->slotId()).at(poolIdx); + tasksPerPool.at(poolIdx).push_back({dstAddr, srcAddr}); + } + } + + // Collect prior events (src + dst ready events) — mirrors Python's prior_events set. + std::vector<CachedCudaEvent const*> priorEvents; + priorEvents.reserve(2 * srcPages.size()); + for (std::size_t i = 0; i < srcPages.size(); ++i) + { + priorEvents.push_back(&srcPages.at(i)->readyEvent); + priorEvents.push_back(&dstSlots.at(i).readyEvent); + } + + // Create a temporary CUDA stream that waits for all prior events before copying. + TemporaryCudaStream tempStream(priorEvents); + { + auto scope = tempStream.enter(); + CUstream stream = tempStream.get(); + auto slotSizes = slotSize(pgIdx); + for (PoolIndex poolIdx{0}; poolIdx < numPools; ++poolIdx) + { + batchedCopy(dstTier, srcTier, slotSizes.at(poolIdx), tasksPerPool.at(poolIdx), stream); + } + } // ~Scope records finish event + + CachedCudaEvent finishEvent = tempStream.takeFinishEvent(); + if (migrationRecorder && !defrag) + { + migrationRecorder(srcPages, dstSlots, srcLevel, dstLevel); + } + std::set<std::pair<std::string, int>> emittedCacheLevelUpdates; + bool const emitCacheLevelUpdates + = updateSrc && !defrag && srcLevel != dstLevel && static_cast<bool>(mEventSink); + for (std::size_t i = 0; i < srcPages.size(); ++i) + { + dstSlots.at(i).readyEvent = finishEvent; + // Fix #6: set src.ready_event unconditionally — compulsory for the next owner + // getting this slot from the pool. Mirrors Python: `src.ready_event = finish_event`. + srcPages.at(i)->readyEvent = finishEvent; + if (updateSrc) + { + bool wasScheduled = srcPages.at(i)->scheduledForEviction(); + if (wasScheduled) + excludeFromEviction(*srcPages.at(i)); + // Extract source slot from the page and release it back to the pool. + Slot srcSlot; + srcSlot.setSlotId(srcPages.at(i)->slotId()); // asserts valid + srcSlot.readyEvent = finishEvent; + srcPages.at(i)->resetSlot(); + srcPoolGroup.release(std::move(srcSlot)); + // Transfer dst slot ownership to the page. + srcPages.at(i)->setSlot(dstSlots.at(i)); + srcPages.at(i)->cacheLevel = dstLevel; + if (emitCacheLevelUpdates && srcPages.at(i)->isCommitted()) + { + auto const& page = static_cast<CommittedPage const&>(*srcPages.at(i)); + Block const* block = page.block; + std::string const blockKey = block + ? std::string(reinterpret_cast<char const*>(block->key.data()), block->key.size()) + : std::string{}; + if (block && !block->isOrphan() + && emittedCacheLevelUpdates.insert({blockKey, page.lifeCycle.value()}).second) + { + mEventSink->addCacheLevelUpdated(block->key, srcLevel, dstLevel, page.lifeCycle); + } + } + if (wasScheduled) + scheduleForEviction(*srcPages.at(i)); + } + } + } + catch (...) + { + for (auto& s : dstSlots) + dstPoolGroup.release(std::move(s)); + throw; + } +} + +// --------------------------------------------------------------------------- +// batchedMigrateToGpu +// --------------------------------------------------------------------------- + +void StorageManager::batchedMigrateToGpu( + std::vector<BatchedLockTarget> const& targets, KvCache& /*kvCache*/, MigrationRecorder const& migrationRecorder) +{ + // Group by (srcLevel, pgIdx). + std::map<std::pair<CacheLevel, PoolGroupIndex>, std::vector<SharedPtr<Page>>> groups; + for (auto const& t : targets) + { + if (t.page->cacheLevel == kGpuLevel) + continue; + PoolGroupIndex pg = mLifeCycleGrouping.at(t.lifeCycle); + groups[{t.page->cacheLevel, pg}].push_back(t.page); + } + for (auto& [key, pages] : groups) + _batchedMigrate(key.second, kGpuLevel, key.first, pages, /*updateSrc=*/true, migrationRecorder); +} + +void StorageManager::prefetch( + CacheLevel dstLevel, TypedVec<PoolGroupIndex, TypedVec<CacheLevel, std::vector<SharedPtr<Page>>>> const& pages) +{ + TypedVec<PoolGroupIndex, SlotCount> numSlotsToMigrate(numPoolGroups(), 0); + std::vector<SharedPtr<Page>> scheduled; + + struct ReschedulePagesGuard + { + StorageManager& storageManager; + std::vector<SharedPtr<Page>>& scheduled; + + ~ReschedulePagesGuard() + { + for (auto const& page : scheduled) + { + storageManager.scheduleForEviction(*page); + } + scheduled.clear(); + } + } reschedulePagesGuard{*this, scheduled}; + + for (PoolGroupIndex pgIndex{0}; pgIndex < pages.size(); ++pgIndex) + { + auto const& poolGroupPages = pages.at(pgIndex); + for (CacheLevel level{0}; level < poolGroupPages.size(); ++level) + { + auto const& levelPages = poolGroupPages.at(level); + TLLM_CHECK_DEBUG(level >= dstLevel || levelPages.empty()); + for (auto const& page : levelPages) + { + if (page->scheduledForEviction()) + { + excludeFromEviction(*page); + scheduled.push_back(page); + } + else if (isEvictable(*page, dstLevel)) + { + scheduled.push_back(page); + } + TLLM_CHECK_DEBUG(level >= dstLevel); + if (level == dstLevel) + { + continue; + } + numSlotsToMigrate.at(pgIndex) += 1; + } + } + } + + prepareFreeSlots(dstLevel, numSlotsToMigrate); + for (PoolGroupIndex pgIndex{0}; pgIndex < pages.size(); ++pgIndex) + { + auto const& poolGroupPages = pages.at(pgIndex); + for (CacheLevel lvl = dstLevel + 1; lvl < numCacheLevels(); ++lvl) + { + _batchedMigrate(pgIndex, dstLevel, lvl, poolGroupPages.at(lvl), /*updateSrc=*/true); + } + } +} + +// --------------------------------------------------------------------------- +// Query helpers +// --------------------------------------------------------------------------- + +LifeCycle const& StorageManager::getLifeCycle(LifeCycleId lc) const +{ + return mLifeCycles[lc]; +} + +PoolGroupIndex StorageManager::getPoolGroupIndex(LifeCycleId lc) const +{ + return mLifeCycleGrouping.at(lc); +} + +PoolIndex StorageManager::mNumPools(PoolGroupIndex pgIdx) const +{ + TLLM_CHECK_DEBUG(!mLevels.empty()); + return getUniformAttribute(mLevels, [pgIdx](auto const& lvl) { return lvl.storage->numPools(pgIdx); }); +} + +PoolIndex StorageManager::numPools(PoolGroupIndex pgIdx) const +{ + return mNumPools(pgIdx); +} + +TypedVec<PoolIndex, size_t> StorageManager::slotSize(PoolGroupIndex pgIdx) const +{ + return mSlotDescList.at(pgIdx).slotSizeList(); +} + +PoolGroupBase& StorageManager::poolGroup(CacheLevel lvl, PoolGroupIndex pgIdx) +{ + return mLevels.at(lvl).storage->poolGroup(pgIdx); +} + +MemAddress StorageManager::getMemPoolBaseAddress(LayerId layerId, DataRole role) const +{ + auto it = mBufferAttr.find(BufferId{layerId, role}); + if (it == mBufferAttr.end()) + throw std::out_of_range("Unknown BufferId"); + auto const& attr = it->second; + PoolGroupIndex pgIdx = mLifeCycleGrouping.at(attr.lifeCycleId); + return mLevels[kGpuLevel].storage->getBaseAddress(pgIdx, attr.poolIndex, SlotId{0}) + attr.offset; +} + +MemAddress StorageManager::getMemPoolBaseAddress(PoolGroupIndex pgIdx, PoolIndex poolIdx) const +{ + return mLevels[kGpuLevel].storage->getBaseAddress(pgIdx, poolIdx, SlotId{0}); +} + +LayerAttr const& StorageManager::getLayerAttr(LayerId layerId) const +{ + auto it = mLayerAttributes.find(layerId); + if (it == mLayerAttributes.end()) + throw std::out_of_range("Unknown LayerId for LayerAttr"); + return it->second; +} + +SlotCount StorageManager::numSlots(PoolGroupIndex pgIdx, CacheLevel level) const +{ + return mLevels.at(level).storage->numSlots(pgIdx); +} + +StorageStatistics StorageManager::getStatistics(CacheLevel level, PoolGroupIndex pgIdx) const +{ + auto const& lvl = mLevels.at(level); + SlotCount freeSlots = lvl.storage->numFreeSlots(pgIdx); + SlotCount totalSlots = lvl.storage->numSlots(pgIdx); + SlotCount evictable = lvl.controller.numEvictablePages(pgIdx); + auto sizes = lvl.storage->slotSize(pgIdx); + return StorageStatistics{sizes, totalSlots, freeSlots, evictable}; +} + +TypedVec<PoolGroupIndex, float> StorageManager::getUtilization(CacheLevel level) const +{ + TypedVec<PoolGroupIndex, float> result; + result.reserve(numPoolGroups()); + for (PoolGroupIndex pgIdx{0}; pgIdx < numPoolGroups(); ++pgIdx) + { + auto const s = getStatistics(level, pgIdx); + TLLM_CHECK_DEBUG(s.total > 0); + result.push_back(static_cast<float>(s.unavailable()) / static_cast<float>(s.total)); + } + return result; +} + +float StorageManager::getOverallUtilization(CacheLevel level) const +{ + float num = 0.f, den = 0.f; + for (PoolGroupIndex pgIdx{0}; pgIdx < numPoolGroups(); ++pgIdx) + { + auto s = getStatistics(level, pgIdx); + float sz = 0.f; + for (auto v : s.slotSizes) + sz += static_cast<float>(v); + num += sz * static_cast<float>(s.unavailable()); + den += sz * static_cast<float>(s.total); + } + TLLM_CHECK_DEBUG(den > 0.f); + return num / den; +} + +// --------------------------------------------------------------------------- +// expandPoolGroup +// --------------------------------------------------------------------------- + +void StorageManager::expandPoolGroup(CacheLevel level, PoolGroupIndex pgIdx, SlotCount newNumSlots) +{ + auto& pg = poolGroup(level, pgIdx); + TLLM_CHECK_DEBUG(newNumSlots > pg.numSlots()); + pg.resizePools(newNumSlots); + pg.slotAllocator().expand(newNumSlots); +} + +// --------------------------------------------------------------------------- +// shrinkPoolGroup — mirrors Python _storage_manager.py::shrink_pool_group +// --------------------------------------------------------------------------- + +void StorageManager::shrinkPoolGroup( + CacheLevel level, PoolGroupIndex pgIdx, SlotCount newNumSlots, std::vector<SharedPtr<Page>> const& persistentPages) +{ + auto& pg = poolGroup(level, pgIdx); + auto& allocator = pg.slotAllocator(); + auto& ctrl = mLevels.at(level).controller; + TLLM_CHECK_DEBUG(newNumSlots < pg.numSlots()); + + // A16: persistent_pages preconditions. + TLLM_CHECK_DEBUG_WITH_INFO( + persistentPages.size() <= slotCountToSizeT(newNumSlots), "Not enough slots to hold all persistent pages"); + TLLM_CHECK_DEBUG_WITH_INFO(std::all_of(persistentPages.begin(), persistentPages.end(), + [this, level, pgIdx](auto const& p) + { return p->cacheLevel == level && mLifeCycleGrouping.at(p->lifeCycle) == pgIdx; }), + "Persistent page cache level or pool group mismatch"); + + // Fast path: when no slot id has ever been issued in the to-be-removed + // range [newNumSlots, capacity), there is nothing to migrate. + // numActiveSlots() is a monotone high-water mark of issued ids. + if (allocator.numActiveSlots() <= newNumSlots) + { + allocator.prepareForShrink(newNumSlots); + allocator.finishShrink(); + pg.resizePools(newNumSlots); + return; + } + + // Find overflow pages: scheduled pages with slot_id >= newNumSlots. + auto gen = ctrl.pageGenerator(pgIdx); + std::deque<std::pair<SlotCount, SharedPtr<Page>>> overflowSlots; + { + SlotCount idx = 0; + while (auto const* page = gen()) + { + if ((*page)->slotId() >= newNumSlots) + overflowSlots.emplace_back(idx, *page); + ++idx; + } + } + + // Persistent pages in overflow range. + std::vector<SharedPtr<Page>> overflowPersistent; + for (auto const& p : persistentPages) + { + if (p->slotId() >= newNumSlots) + overflowPersistent.push_back(p); + } + SlotCount numOverflowPersistent = slotCountValueFromSize(overflowPersistent.size()); + + // A2: RUNTIME check — persistent overflow pages must fit in the new capacity. + if (numOverflowPersistent > newNumSlots) + { + throw OutOfPagesError("Not enough slots to hold all persistent pages"); + } + + // Mark the allocator for shrink. + allocator.prepareForShrink(newNumSlots); + + // Calculate minimum number of lowest-priority pages to evict. + // Need numEvictedOverflowSlots because evicted overflow pages won't become free, + // because only free non-overflow slots can be used for defragmentation. + SlotCount minNumEvicted = 0; + SlotCount numEvictedOverflowSlots = 0; + while (!overflowSlots.empty() + && slotCountValueFromSize(overflowSlots.size()) + numOverflowPersistent + > std::min(newNumSlots, overflowSlots.front().first + allocator.numFreeSlots() - numEvictedOverflowSlots)) + { + minNumEvicted = overflowSlots.front().first + 1; + overflowSlots.pop_front(); + ++numEvictedOverflowSlots; + } + + // Force-evict the required pages. + TypedVec<PoolGroupIndex, SlotCount> evictReqs(numPoolGroups(), 0); + evictReqs[pgIdx] = minNumEvicted; + forceEvict(level, evictReqs); + + // Remaining overflow pages to defragment. + std::vector<SharedPtr<Page>> overflowPages; + overflowPages.reserve(overflowSlots.size() + overflowPersistent.size()); + for (auto& [idx, p] : overflowSlots) + overflowPages.push_back(p); + for (auto& p : overflowPersistent) + overflowPages.push_back(p); + + // Ensure free slots for the overflow pages. + TypedVec<PoolGroupIndex, SlotCount> reqs(numPoolGroups(), 0); + reqs[pgIdx] = slotCountValueFromSize(overflowPages.size()); + prepareFreeSlots(level, reqs); + + // A17: all overflow pages must be at the expected cache level. + TLLM_CHECK_DEBUG_WITH_INFO(std::all_of(overflowPages.begin(), overflowPages.end(), + [level](auto const& p) { return p->cacheLevel == level; }), + "Overflow page cache level mismatch"); + + // Defragment: migrate overflow pages to free slots within the same level. + _batchedMigrate(pgIdx, level, level, overflowPages, /*updateSrc=*/true, MigrationRecorder{}, /*defrag=*/true); + + // A18: post-defrag overflow assertion — overflow slot count matches expectations. + TLLM_CHECK_DEBUG_WITH_INFO(allocator.numOverflowSlots() == allocator.numActiveSlots() - allocator.targetCapacity(), + "Post-defrag overflow slot count mismatch"); + + // Finalize shrink and resize pools. + allocator.finishShrink(); + pg.resizePools(newNumSlots); +} + +// --------------------------------------------------------------------------- +// adjustCacheLevel — mirrors Python _storage_manager.py::adjust_cache_level +// --------------------------------------------------------------------------- + +void StorageManager::adjustCacheLevel(CacheLevel level, std::optional<size_t> newQuota, + TypedVec<PoolGroupIndex, float> const& ratioList, + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> const* persistentPages) +{ + auto& lvlStorage = *mLevels.at(level).storage; + auto oldNumSlots = lvlStorage.slotCountList(); + size_t quota = newQuota.has_value() + ? roundUp(newQuota.value(), static_cast<size_t>(lvlStorage.poolSizeGranularity())) + : lvlStorage.totalQuota(); + size_t minQuota = minQuotaForLevel(lvlStorage.slotSizeLists(), lvlStorage.poolSizeGranularity()); + if (quota < minQuota) + { + throw std::invalid_argument("Quota " + std::to_string(quota) + + " is insufficient for min_slots constraints (requires at least " + std::to_string(minQuota) + ")"); + } + auto newNumSlots = lvlStorage.computeSlotCountList(ratioList, mMinSlots, quota); + + if (!isLastLevel(level)) + TLLM_CHECK_DEBUG(persistentPages == nullptr); + + // Shrink first. + for (PoolGroupIndex pgIdx{0}; pgIdx < newNumSlots.size(); ++pgIdx) + { + if (newNumSlots[pgIdx] >= oldNumSlots[pgIdx]) + continue; + std::vector<SharedPtr<Page>> pages; + if (persistentPages) + pages = (*persistentPages)[pgIdx]; + shrinkPoolGroup(level, pgIdx, newNumSlots[pgIdx], pages); + } + // Then expand. + for (PoolGroupIndex pgIdx{0}; pgIdx < newNumSlots.size(); ++pgIdx) + { + if (newNumSlots[pgIdx] <= oldNumSlots[pgIdx]) + continue; + expandPoolGroup(level, pgIdx, newNumSlots[pgIdx]); + } + lvlStorage.postResize(); +} + +TypedVec<PoolGroupIndex, float> StorageManager::getRatioList(CacheLevel level) const +{ + return mLevels.at(level).storage->ratioList(); +} + +TypedVec<PoolGroupIndex, float> StorageManager::ratioFromLength( + int tokensPerBlock, int historyLength, int capacity) const +{ + if (capacity < historyLength) + { + TLLM_LOG_WARNING("Bad sampling for capacity and history_length"); + capacity = historyLength; + } + int numBlocks = divUp(capacity, tokensPerBlock); + TypedVec<PoolGroupIndex, size_t> numBytes(numPoolGroups(), 0); + auto ssmLcId = mLifeCycles.ssmLifeCycleId(); + auto const& lifecycles = mLifeCycles.getAll(); + for (LifeCycleId lcId{0}; lcId < lifecycles.size(); ++lcId) + { + PoolGroupIndex pgIdx = mLifeCycleGrouping[lcId]; + auto ss = slotSize(pgIdx); + size_t slotSizeSum = 0; + for (auto s : ss) + slotSizeSum += s; + int numRequiredBlocks; + if (ssmLcId.has_value() && lcId == *ssmLcId) + { + numRequiredBlocks = 1; + } + else + { + auto stale = getStaleRange(lifecycles[lcId], historyLength, tokensPerBlock); + numRequiredBlocks = std::max(numBlocks - stale.length(), 1); + } + numBytes[pgIdx] += static_cast<size_t>(numRequiredBlocks) * slotSizeSum; + } + return normalizeToRatio(numBytes); +} + +// --------------------------------------------------------------------------- +// ratioFromBatch +// --------------------------------------------------------------------------- + +TypedVec<PoolGroupIndex, float> StorageManager::ratioFromBatch(BatchDesc const& batch, int tokensPerBlock, + std::optional<SwaScratchReuseConfig> const& swaScratchReuse, size_t granularity) const +{ + auto numSlots = computeSlotsForBatch(batch, tokensPerBlock, swaScratchReuse); + auto numBytes = slotsToBytes(numSlots, granularity); + return normalizeToRatio(numBytes); +} + +// --------------------------------------------------------------------------- +// computeMinSlotsFromConstraints +// --------------------------------------------------------------------------- + +TypedVec<PoolGroupIndex, SlotCount> StorageManager::computeMinSlotsFromConstraints( + std::vector<BatchDesc> const& constraints, int tokensPerBlock, + std::optional<SwaScratchReuseConfig> const& swaScratchReuse, float maxUtilForResume) const +{ + TLLM_CHECK_DEBUG(maxUtilForResume > 0.0f && maxUtilForResume <= 1.0f); + // All returned elements are positive. Constraint-derived floors include headroom + // for the utilization gate checked by KvCache::resume. + TypedVec<PoolGroupIndex, SlotCount> maxSlots(numPoolGroups(), 0); + + auto swaFloorBlocks = [tokensPerBlock](AttnLifeCycle const& lc) -> int + { + int window = *lc.windowSize; + // Handle oscillation of slot count required by SWA while the window slides. + return lc.numSinkBlocks + (window + tokensPerBlock - 2) / tokensPerBlock + 1; + }; + + // Full-attention lifecycles share the largest SWA floor: all attention + // lifecycles see the same seq_len, so this is a valid lower bound. + int floorNumBlocks = 1; + for (auto const& [lcId, attn] : mLifeCycles.attentionLifeCycles()) + { + if (attn->windowSize.has_value()) + floorNumBlocks = std::max(floorNumBlocks, swaFloorBlocks(*attn)); + } + for (auto const& [lcIdx, lc] : mLifeCycles) + { + PoolGroupIndex pgIdx = getPoolGroupIndex(lcIdx); + auto const* attn = std::get_if<AttnLifeCycle>(&lc); + if (attn == nullptr) + { + // SSM / non-attention: 1 slot floor per life cycle. + maxSlots[pgIdx] += 1; + } + else if (attn->windowSize.has_value()) + { + maxSlots[pgIdx] += swaFloorBlocks(*attn); + } + else + { + maxSlots[pgIdx] += floorNumBlocks; + } + } + for (auto const& batch : constraints) + { + auto slots = computeSlotsForBatch(batch, tokensPerBlock, swaScratchReuse); + for (PoolGroupIndex pgIdx{0}; pgIdx < slots.size(); ++pgIdx) + { + auto const scaledSlots = static_cast<SlotCount>( + std::ceil(static_cast<double>(slots[pgIdx]) / static_cast<double>(maxUtilForResume))); + maxSlots[pgIdx] = std::max(maxSlots[pgIdx], scaledSlots); + } + } + return maxSlots; +} + +// --------------------------------------------------------------------------- +// computeSlotsForBatch +// --------------------------------------------------------------------------- + +TypedVec<PoolGroupIndex, SlotCount> StorageManager::computeSlotsForBatch( + BatchDesc const& batch, int tokensPerBlock, std::optional<SwaScratchReuseConfig> const& swaScratchReuse) const +{ + TypedVec<PoolGroupIndex, SlotCount> numSlots(numPoolGroups(), 0); + auto ssmLcId = mLifeCycles.ssmLifeCycleId(); + int sysBlocks = batch.systemPromptLength / tokensPerBlock; + + for (auto const& [lcIdx, lc] : mLifeCycles) + { + PoolGroupIndex pgIdx = mLifeCycleGrouping[lcIdx]; + if (ssmLcId.has_value() && lcIdx == *ssmLcId) + { + // SSM: always 1 dedicated block per request, never shared. + numSlots[pgIdx] += slotCountValueFromSize(batch.kvCaches.size()); + continue; + } + // Shared sys blocks (counted once): union of non-stale sys blocks across all requests. + HalfOpenRange<BlockOrdinal> sysRange{0, sysBlocks}; + HalfOpenRange<BlockOrdinal> staleIntersection = sysRange; + for (auto const& kv : batch.kvCaches) + { + auto stale = getStaleRange(lc, kv.historyLength, tokensPerBlock); + staleIntersection = intersect(staleIntersection, stale); + } + numSlots[pgIdx] += sysBlocks - staleIntersection.length(); + + // Per-request unique blocks (excluding shared sys blocks already counted above). + for (auto const& kv : batch.kvCaches) + { + int totalBlocks = divUp(kv.capacity, tokensPerBlock); + auto stale = getStaleRange(lc, kv.historyLength, tokensPerBlock); + int nonStale = totalBlocks - stale.length(); + int nonStaleSys = sysBlocks - intersect(stale, sysRange).length(); + int uniqueNonStale = std::max(0, nonStale - nonStaleSys); + if (swaScratchReuse.has_value()) + { + auto scratch = computeScratchRange( + lc, kv.historyLength, kv.capacity, tokensPerBlock, swaScratchReuse->maxRewindLen); + int numScratch = scratch.length(); + // Scratch blocks share coalesced slots: actual slots = ceil(numScratch * fracMax). + numSlots[pgIdx] += (uniqueNonStale - numScratch) + mSlotUtilFracMax[lcIdx].ceilMul(numScratch); + } + else + { + numSlots[pgIdx] += uniqueNonStale; + } + } + } + return numSlots; +} + +// --------------------------------------------------------------------------- +// slotsToBytes +// --------------------------------------------------------------------------- + +TypedVec<PoolGroupIndex, size_t> StorageManager::slotsToBytes( + TypedVec<PoolGroupIndex, SlotCount> const& numSlots, size_t granularity) const +{ + TypedVec<PoolGroupIndex, size_t> numBytes(numPoolGroups(), 0); + for (PoolGroupIndex pgIdx{0}; pgIdx < numSlots.size(); ++pgIdx) + { + for (auto poolSize : slotSize(pgIdx)) + { + numBytes[pgIdx] += roundUp(slotCountToSizeT(numSlots[pgIdx]) * poolSize, granularity); + } + } + return numBytes; +} + +// --------------------------------------------------------------------------- +// computeSlotCountForLevel +// --------------------------------------------------------------------------- + +TypedVec<PoolGroupIndex, SlotCount> StorageManager::computeSlotCountForLevel(CacheTierConfig const& tierConfig, + TypedVec<PoolGroupIndex, TypedVec<PoolIndex, size_t>> const& slotSizeLists, + TypedVec<PoolGroupIndex, float> const& ratio) const +{ + CacheTier tier = cacheTierOf(tierConfig); + size_t quota = cacheTierQuota(tierConfig); + size_t granularity = CacheLevelManager::cacheTierGranularity(tier, quota); + quota = std::max(minQuotaForLevel(slotSizeLists, granularity), roundUp(quota, granularity)); + return CacheLevelStorage::ratioToSlotCountList(quota, slotSizeLists, ratio, granularity, mMinSlots); +} + +// --------------------------------------------------------------------------- +// minQuotaForLevel +// --------------------------------------------------------------------------- + +size_t StorageManager::minQuotaForLevel( + TypedVec<PoolGroupIndex, TypedVec<PoolIndex, size_t>> const& slotSizeLists, size_t granularity) const +{ + size_t total = 0; + for (PoolGroupIndex pgIdx{0}; pgIdx < slotSizeLists.size(); ++pgIdx) + { + for (auto slotSize : slotSizeLists[pgIdx]) + { + total += roundUp(slotCountToSizeT(mMinSlots[pgIdx]) * slotSize, granularity); + } + } + return total; +} + +// --------------------------------------------------------------------------- +// constrainRatio +// --------------------------------------------------------------------------- + +TypedVec<PoolGroupIndex, float> StorageManager::constrainRatio(TypedVec<PoolGroupIndex, float> const& ratio) const +{ + auto& gpuStorage = *mLevels[kGpuLevel].storage; + size_t granularity = gpuStorage.poolSizeGranularity(); + auto slotCountList = gpuStorage.computeSlotCountList(ratio, mMinSlots); + auto numBytes = slotsToBytes(slotCountList, granularity); + return normalizeToRatio(numBytes); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storageManager.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storageManager.h new file mode 100644 index 000000000000..27979516059e --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/storageManager.h @@ -0,0 +1,331 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/config.h" +#include "kv_cache_manager_v2/eventSink.h" +#include "kv_cache_manager_v2/evictionController.h" +#include "kv_cache_manager_v2/lifeCycleRegistry.h" +#include "kv_cache_manager_v2/storage/config.h" +#include "kv_cache_manager_v2/storage/core.h" +#include "tensorrt_llm/common/assert.h" + +#include <functional> +#include <map> +#include <memory> +#include <unordered_map> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// Forward declarations. +class Page; +class KvCache; +struct BatchedLockTarget; + +using MigrationRecorder + = std::function<void(std::vector<SharedPtr<Page>> const&, std::vector<Slot> const&, CacheLevel, CacheLevel)>; +using DropRecorder = std::function<void(std::vector<SharedPtr<Page>> const&, CacheLevel)>; + +// --------------------------------------------------------------------------- +// StorageStatistics — per-pool-group slot counts. +// --------------------------------------------------------------------------- +struct StorageStatistics +{ + TypedVec<PoolIndex, size_t> slotSizes; + SlotCount total; // total slots + SlotCount free; // free (unallocated) slots + SlotCount evictable; // scheduled for eviction + + SlotCount available() const noexcept + { + return free + evictable; + } + + SlotCount unavailable() const noexcept + { + return total - available(); + } +}; + +// --------------------------------------------------------------------------- +// CacheLevelManager — one storage level (GPU/Host/Disk) + its eviction controller. +// --------------------------------------------------------------------------- +class CacheLevelManager +{ +public: + CacheLevelManager(TypedVec<LifeCycleId, PoolGroupIndex> const& lifeCycleGrouping, CacheLevel cacheLevel, + CacheTierConfig const& tierConfig, StorageConfig const& storageConfig, + TypedVec<PoolGroupIndex, SlotCount> const& slotCountList); + + // Compute pool size granularity for a given cache tier and quota. + static size_t cacheTierGranularity(CacheTier tier, size_t quota); + + CacheLevel cacheLevel; + CacheTier cacheTier; + std::unique_ptr<CacheLevelStorage> storage; + PerLevelEvictionController controller; + + PoolGroupIndex numPoolGroups() const noexcept + { + TLLM_CHECK_DEBUG_WITH_INFO( + storage->numPoolGroups() == controller.numPoolGroups(), "Storage and controller disagree on numPoolGroups"); + return controller.numPoolGroups(); + } +}; + +// --------------------------------------------------------------------------- +// StorageManager — manages all cache levels and the eviction pipeline. +// Mirrors Python's StorageManager. +// --------------------------------------------------------------------------- +class StorageManager : public std::enable_shared_from_this<StorageManager> +{ +public: + StorageManager(LifeCycleRegistry const& lifeCycles, StorageConfig const& config, int tokensPerBlock, + std::optional<SwaScratchReuseConfig> swaScratchReuse = std::nullopt, + std::optional<BatchDesc> const& typicalBatch = std::nullopt, std::vector<BatchDesc> const& constraints = {}, + std::optional<std::vector<float>> const& initialPoolRatio = std::nullopt, + std::shared_ptr<EventSink> eventSink = nullptr, float maxUtilForResume = 1.0f); + ~StorageManager(); + + StorageManager(StorageManager const&) = delete; + StorageManager& operator=(StorageManager const&) = delete; + + void destroy(); + + // ---- Allocation ------------------------------------------------------- + + // Allocate slots for all life cycles at the given cache level. + // numSlotsPerLc[lcId] = how many slots to allocate for that life cycle. + // Returns a vector indexed by lcId. + TypedVec<LifeCycleId, std::vector<Slot>> newSlots(CacheLevel level, + TypedVec<LifeCycleId, SlotCount> const& numSlotsPerLc, MigrationRecorder const& migrationRecorder = {}, + DropRecorder const& dropRecorder = {}); + + TypedVec<LifeCycleId, std::vector<Slot>> newGpuSlots(TypedVec<LifeCycleId, SlotCount> const& numSlotsPerLc, + MigrationRecorder const& migrationRecorder = {}, DropRecorder const& dropRecorder = {}); + + // Allocate slots for a single pool group at the given cache level. + // Returns numSlots Slot objects. Throws OutOfPagesError if allocation fails. + std::vector<Slot> newSlotsForPoolGroup(CacheLevel level, PoolGroupIndex pgIdx, SlotCount numSlots, + MigrationRecorder const& migrationRecorder = {}, DropRecorder const& dropRecorder = {}); + + // Release a slot back to its pool. + void releaseSlot(LifeCycleId lc, CacheLevel level, Slot slot); + + // ---- Eviction ---------------------------------------------------------- + + // Schedule a page for eviction (if evictable). + void scheduleForEviction(Page& page); + + // Remove a page from the eviction queue. + void excludeFromEviction(Page& page); + + // Check if a page is evictable (optionally at a target level). + bool isEvictable(Page const& page, std::optional<CacheLevel> level = std::nullopt) const noexcept; + + // Ensure numFreeSlots[pgIdx] free GPU slots exist (evicting pages as needed). + void prepareFreeSlots(CacheLevel level, TypedVec<PoolGroupIndex, SlotCount> const& requirements, + MigrationRecorder const& migrationRecorder = {}, DropRecorder const& dropRecorder = {}); + + // Force-evict pages from a level to free space. + void forceEvict(CacheLevel level, TypedVec<PoolGroupIndex, SlotCount> const& minNumPages, + DropRecorder const& dropRecorder = {}); + + // Dynamic cache level resizing. + void adjustCacheLevel(CacheLevel level, std::optional<size_t> newQuota, + TypedVec<PoolGroupIndex, float> const& ratioList, + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>> const* persistentPages); + void shrinkPoolGroup(CacheLevel level, PoolGroupIndex pgIdx, SlotCount newNumSlots, + std::vector<SharedPtr<Page>> const& persistentPages); + void expandPoolGroup(CacheLevel level, PoolGroupIndex pgIdx, SlotCount newNumSlots); + + // ---- Migration --------------------------------------------------------- + + // Migrate a batch of pages to GPU (used by batchedLockToGpu). + void batchedMigrateToGpu( + std::vector<BatchedLockTarget> const& targets, KvCache& kvCache, MigrationRecorder const& migrationRecorder); + + // Best-effort migration of grouped pages to a destination cache level. + void prefetch( + CacheLevel dstLevel, TypedVec<PoolGroupIndex, TypedVec<CacheLevel, std::vector<SharedPtr<Page>>>> const& pages); + + // ---- Query helpers ----------------------------------------------------- + + LifeCycleRegistry const& lifeCycles() const noexcept + { + return mLifeCycles; + } + + LifeCycle const& getLifeCycle(LifeCycleId lc) const; + + LifeCycleId numLifeCycles() const noexcept + { + return mLifeCycleGrouping.size(); + } + + PoolGroupIndex numPoolGroups() const noexcept + { + return mSlotDescList.size(); + } + + TypedVec<PoolGroupIndex, SlotDesc> const& slotDescList() const noexcept + { + return mSlotDescList; + } + + CacheLevel numCacheLevels() const noexcept + { + return mLevels.size(); + } + + bool isLastLevel(CacheLevel lvl) const noexcept + { + return lvl == numCacheLevels() - 1; + } + + PoolGroupIndex getPoolGroupIndex(LifeCycleId lc) const; + PoolIndex numPools(PoolGroupIndex pgIdx) const; + + // Return the byte size of each pool in a pool group. + TypedVec<PoolIndex, size_t> slotSize(PoolGroupIndex pgIdx) const; + + // Current ratio list for a cache level (proportional to byte usage per pool group). + TypedVec<PoolGroupIndex, float> getRatioList(CacheLevel level) const; + + // Compute init ratio from an assumed average history length and capacity. + TypedVec<PoolGroupIndex, float> ratioFromLength(int tokensPerBlock, int historyLength, int capacity) const; + + // Compute ratio from a BatchDesc. + TypedVec<PoolGroupIndex, float> ratioFromBatch(BatchDesc const& batch, int tokensPerBlock, + std::optional<SwaScratchReuseConfig> const& swaScratchReuse, size_t granularity) const; + + // Apply stored min_slots constraint to a ratio list for GPU level. + TypedVec<PoolGroupIndex, float> constrainRatio(TypedVec<PoolGroupIndex, float> const& ratio) const; + + // Byte address of a slot's buffer in GPU memory (per-layer, with offset). + MemAddress getMemPoolBaseAddress(LayerId layerId, DataRole role) const; + + // Pool group base address without per-layer offset. + MemAddress getMemPoolBaseAddress(PoolGroupIndex pgIdx, PoolIndex poolIdx) const; + + // Per-layer storage attributes. + LayerAttr const& getLayerAttr(LayerId layerId) const; + + // Address of a slot's buffer in a specific pool at a cache level. + Address slotAddress(CacheLevel level, PoolGroupIndex pgIdx, SlotId slotId, PoolIndex poolIdx) const; + + // Cache tier for a given level. + CacheTier cacheTier(CacheLevel level) const; + + // NOTE: Python's get_statistics(level) returns a list over all pool groups. + // C++ takes a single pgIdx for flexibility; the nanobind wrapper loops over + // all pool groups to match Python's signature. + StorageStatistics getStatistics(CacheLevel level = kGpuLevel, PoolGroupIndex pgIdx = PoolGroupIndex{0}) const; + TypedVec<PoolGroupIndex, float> getUtilization(CacheLevel level = kGpuLevel) const; + float getOverallUtilization(CacheLevel level = kGpuLevel) const; + + // Pool-group slot count (number of pages). + SlotCount numSlots(PoolGroupIndex pgIdx, CacheLevel level = kGpuLevel) const; + + // Layer-to-lifecycle mapping (for KvCacheManager queries). + std::unordered_map<LayerId, LifeCycleId> const& layerToLifeCycleIds() const noexcept + { + return mLayerToLifeCycleIds; + } + + // Per-lifecycle max slot utilization fraction (for scratch slot computation). + Rational const& slotUtilFracMax(LifeCycleId lcId) const + { + return mSlotUtilFracMax.at(lcId); + } + + friend class KvCacheManager; + // White-box test introspection reaches computeSlotsForBatch() via friendship + // (mirrors Python's _compute_slots_for_batch). + friend class KvCacheIntrospection; + +private: + // Minimum per-pool-group slot counts to support a BatchDesc. + TypedVec<PoolGroupIndex, SlotCount> computeSlotsForBatch( + BatchDesc const& batch, int tokensPerBlock, std::optional<SwaScratchReuseConfig> const& swaScratchReuse) const; + + // Constraint-based partitioning helpers. + TypedVec<PoolGroupIndex, SlotCount> computeMinSlotsFromConstraints(std::vector<BatchDesc> const& constraints, + int tokensPerBlock, std::optional<SwaScratchReuseConfig> const& swaScratchReuse, + float maxUtilForResume = 1.0f) const; + TypedVec<PoolGroupIndex, size_t> slotsToBytes( + TypedVec<PoolGroupIndex, SlotCount> const& numSlots, size_t granularity) const; + TypedVec<PoolGroupIndex, SlotCount> computeSlotCountForLevel(CacheTierConfig const& tierConfig, + TypedVec<PoolGroupIndex, TypedVec<PoolIndex, size_t>> const& slotSizeLists, + TypedVec<PoolGroupIndex, float> const& ratio) const; + size_t minQuotaForLevel( + TypedVec<PoolGroupIndex, TypedVec<PoolIndex, size_t>> const& slotSizeLists, size_t granularity) const; + + PoolIndex mNumPools(PoolGroupIndex pgIdx) const; + + // Internal helpers. + void _prepareFreeSlots(TypedVec<CacheLevel, TypedVec<PoolGroupIndex, SlotCount>>& goals, CacheLevel lvlId, + TypedVec<PoolGroupIndex, std::vector<SharedPtr<Page>>>& fallenPages, + MigrationRecorder const& migrationRecorder = {}, DropRecorder const& dropRecorder = {}); + + void _batchedMigrate(PoolGroupIndex pgIdx, CacheLevel dstLevel, CacheLevel srcLevel, + std::vector<SharedPtr<Page>> const& srcPages, bool updateSrc, MigrationRecorder const& migrationRecorder = {}, + bool defrag = false); + + PoolGroupBase& poolGroup(CacheLevel lvl, PoolGroupIndex pgIdx); + + LifeCycleRegistry const& mLifeCycles; + std::shared_ptr<EventSink> mEventSink; + TypedVec<LifeCycleId, PoolGroupIndex> mLifeCycleGrouping; // lcId → pgIdx + std::unordered_map<LayerId, LifeCycleId> mLayerToLifeCycleIds; + StorageConfig mStorageConfig; + + // slot-to-page-index scale factors: [lcId][poolIdx] + TypedVec<LifeCycleId, TypedVec<PoolIndex, int>> mSlotToPageIndices; + + // Per-layer storage attributes for scratch slot management. + std::map<LayerId, LayerAttr> mLayerAttributes; + + // Max slot utilization fraction per lifecycle (across all layers in that lifecycle). + TypedVec<LifeCycleId, Rational> mSlotUtilFracMax; + + // Whether SWA scratch reuse is enabled. + std::optional<SwaScratchReuseConfig> mSwaScratchReuse; + + // Get buffer attributes for a (LayerId, DataRole) pair. Throws std::out_of_range if not found. + // Mirrors Python's get_buffer_attr(). + BufferAttr const& getBufferAttr(LayerId layerId, DataRole role) const + { + auto it = mBufferAttr.find(BufferId{layerId, role}); + if (it == mBufferAttr.end()) + throw std::out_of_range("Unknown buffer id"); + return it->second; + } + + // Buffer attributes keyed by BufferId. + std::map<BufferId, BufferAttr> mBufferAttr; + + TypedVec<PoolGroupIndex, SlotDesc> mSlotDescList; + TypedVec<PoolGroupIndex, SlotCount> mMinSlots; + TypedVec<CacheLevel, CacheLevelManager> mLevels; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/cudaEvent.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/cudaEvent.cpp new file mode 100644 index 000000000000..865374f765ea --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/cudaEvent.cpp @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/utils/cudaEvent.h" +#include "kv_cache_manager_v2/exceptions.h" + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// CudaEventPool / CudaStreamPool singleton implementations +// --------------------------------------------------------------------------- + +CudaEventPool::CudaEventPool() + : SimplePool( + []() -> CUevent + { + CUevent ev; + cuCheck(cuEventCreate(&ev, CU_EVENT_DISABLE_TIMING)); + return ev; + }, + [](CUevent ev) { cuEventDestroy(ev); }, + /*initSize=*/1024) +{ +} + +CudaEventPool& CudaEventPool::instance() +{ + static CudaEventPool pool; + return pool; +} + +CudaStreamPool::CudaStreamPool() + : SimplePool( + []() -> CUstream + { + CUstream s; + cuCheck(cuStreamCreate(&s, CU_STREAM_NON_BLOCKING)); + return s; + }, + [](CUstream s) { cuStreamDestroy(s); }, + /*initSize=*/128) +{ +} + +CudaStreamPool& CudaStreamPool::instance() +{ + static CudaStreamPool pool; + return pool; +} + +// --------------------------------------------------------------------------- +// CachedCudaEvent implementation +// --------------------------------------------------------------------------- + +CachedCudaEvent CachedCudaEvent::makeNull() noexcept +{ + return CachedCudaEvent{}; +} + +CachedCudaEvent::CachedCudaEvent(CudaStream stream) + : mEvent(std::make_shared<CudaEventPool::PoolItem>(CudaEventPool::instance().get())) +{ + cuCheck(cuEventRecord(mEvent->get(), reinterpret_cast<CUstream>(stream))); +} + +bool CachedCudaEvent::queryComplete() +{ + if (isClosed()) + { + return true; + } + CUresult result = cuEventQuery(mEvent->get()); + if (result == CUDA_SUCCESS) + { + close(); + return true; + } + if (result == CUDA_ERROR_NOT_READY) + { + return false; + } + throw CuError(result); +} + +void CachedCudaEvent::synchronize() +{ + if (isClosed()) + { + return; + } + cuCheck(cuEventSynchronize(mEvent->get())); + close(); +} + +void CachedCudaEvent::waitInStream(CudaStream stream) const +{ + if (isClosed()) + { + return; + } + cuCheck(cuStreamWaitEvent(reinterpret_cast<CUstream>(stream), mEvent->get(), 0)); +} + +void CachedCudaEvent::close() +{ + if (mEvent) + { + mEvent->reset(); + } +} + +// --------------------------------------------------------------------------- +// CachedCudaStream implementation +// --------------------------------------------------------------------------- + +CachedCudaStream::CachedCudaStream() + : mPoolItem(CudaStreamPool::instance().get()) +{ +} + +CachedCudaEvent CachedCudaStream::recordEvent() +{ + return CachedCudaEvent{reinterpret_cast<CudaStream>(handle())}; +} + +void CachedCudaStream::synchronize() +{ + cuCheck(cuStreamSynchronize(handle())); +} + +// --------------------------------------------------------------------------- +// TemporaryCudaStream implementation +// --------------------------------------------------------------------------- + +TemporaryCudaStream::TemporaryCudaStream(std::vector<CachedCudaEvent const*> const& priorEvents) + : mStream() +{ + CudaStream cs = reinterpret_cast<CudaStream>(mStream.handle()); + streamWaitEvents(cs, priorEvents); +} + +// --------------------------------------------------------------------------- +// mergeEvents — merge multiple CUDA events into one. +// Mirrors Python's merge_events() in _utils.py. +// --------------------------------------------------------------------------- + +CachedCudaEvent mergeEvents(std::vector<CachedCudaEvent>& events) +{ + // Filter out closed events (optimization: skip cuStreamWaitEvent calls). + std::vector<CachedCudaEvent*> live; + for (auto& ev : events) + { + if (!ev.isClosed()) + live.push_back(&ev); + } + if (live.empty()) + return CachedCudaEvent::makeNull(); + if (live.size() == 1) + return std::move(*live[0]); + // Multiple live events: merge via TemporaryCudaStream. + std::vector<CachedCudaEvent const*> priors; + priors.reserve(live.size()); + for (auto* ev : live) + priors.push_back(ev); + TemporaryCudaStream tempStream(priors); + { + auto scope = tempStream.enter(); + } + return tempStream.takeFinishEvent(); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/cudaEvent.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/cudaEvent.h new file mode 100644 index 000000000000..5621ca4b04b5 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/cudaEvent.h @@ -0,0 +1,443 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/exceptions.h" + +#include <algorithm> +#include <cuda.h> +#include <deque> +#include <exception> +#include <functional> +#include <memory> +#include <optional> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// FuncGuard<F> — generic RAII scope guard that calls a void() callable on destruction. +// Movable (moved-from instance is disarmed). Not copyable. +// --------------------------------------------------------------------------- +template <typename F> +class FuncGuard +{ +public: + explicit FuncGuard(F&& func) + : mFunc(std::forward<F>(func)) + , mActive(true) + { + } + + ~FuncGuard() + { + if (mActive) + { + mFunc(); + } + } + + FuncGuard(FuncGuard&& other) noexcept + : mFunc(std::move(other.mFunc)) + , mActive(other.mActive) + { + other.mActive = false; + } + + FuncGuard(FuncGuard const&) = delete; + FuncGuard& operator=(FuncGuard const&) = delete; + FuncGuard& operator=(FuncGuard&&) = delete; + +private: + F mFunc; + bool mActive; +}; + +// --------------------------------------------------------------------------- +// SimplePool<T, Derived> — generic resource pool for opaque handle types. +// Mirrors _utils.py::SimplePool. +// +// T is the pointed-to type (e.g. CUevent_st, CUstream_st). +// CreateFn returns T*, DestroyFn takes T*. +// get() returns a PoolItem (unique_ptr<T, Deleter>) with zero extra allocation — +// the unique_ptr directly wraps the handle pointer. +// +// Derived (CRTP, default void): +// - void: instance pool — Deleter stores a SimplePool* pointer (8 bytes). +// - non-void: singleton pool — Deleter is stateless (0 bytes), calls +// Derived::instance() to find the pool. PoolItem is pointer-sized. +// --------------------------------------------------------------------------- + +// Forward declare so Deleters can reference it. +template <typename T, typename Derived = void> +class SimplePool; + +// Deleter for instance pools (Derived == void): stores a pool pointer. +template <typename T> +struct InstancePoolDeleter +{ + SimplePool<T, void>* pool = nullptr; + + void operator()(T* ptr) const noexcept; +}; + +// Deleter for singleton pools (Derived != void): stateless, zero-size. +template <typename T, typename Derived> +struct SingletonPoolDeleter +{ + void operator()(T* ptr) const noexcept; +}; + +template <typename T, typename Derived> +class SimplePool +{ +public: + using CreateFn = std::function<T*()>; + using DestroyFn = std::function<void(T*)>; + + using Deleter + = std::conditional_t<std::is_void_v<Derived>, InstancePoolDeleter<T>, SingletonPoolDeleter<T, Derived>>; + using PoolItem = std::unique_ptr<T, Deleter>; + + SimplePool(CreateFn createFn, DestroyFn destroyFn, int initSize = 0, std::optional<int> maxSize = std::nullopt) + : mCreateFn(std::move(createFn)) + , mDestroyFn(std::move(destroyFn)) + , mMaxSize(maxSize) + , mOutstandingCount(0) + { + for (int i = 0; i < initSize; ++i) + { + mItems.push_back(mCreateFn()); + } + } + + ~SimplePool() + { + clear(); + } + + SimplePool(SimplePool const&) = delete; + SimplePool& operator=(SimplePool const&) = delete; + + // Get a resource wrapped in a PoolItem that auto-returns to pool on destruction. + [[nodiscard]] PoolItem get() + { + // Increment only after the item is successfully obtained, so a throwing + // mCreateFn() leaves mOutstandingCount unchanged (no leak in stats). + T* item = mItems.empty() ? mCreateFn() : popFront(); + ++mOutstandingCount; + if constexpr (std::is_void_v<Derived>) + { + return PoolItem(item, Deleter{this}); + } + else + { + return PoolItem(item, Deleter{}); + } + } + + void clear() + { + while (!mItems.empty()) + { + mDestroyFn(popFront()); + } + } + + [[nodiscard]] int outstandingCount() const noexcept + { + return mOutstandingCount; + } + + [[nodiscard]] int cachedCount() const noexcept + { + return static_cast<int>(mItems.size()); + } + +private: + friend struct InstancePoolDeleter<T>; + friend struct SingletonPoolDeleter<T, Derived>; + + T* popFront() + { + T* item = mItems.front(); + mItems.pop_front(); + return item; + } + + void put(T* item) + { + --mOutstandingCount; + if (mMaxSize.has_value() && static_cast<int>(mItems.size()) >= *mMaxSize) + { + mDestroyFn(item); + } + else + { + mItems.push_back(item); + } + } + + CreateFn mCreateFn; + DestroyFn mDestroyFn; + std::optional<int> mMaxSize; + std::deque<T*> mItems; + int mOutstandingCount; +}; + +// Deleter implementations (after SimplePool is fully defined). +template <typename T> +void InstancePoolDeleter<T>::operator()(T* ptr) const noexcept +{ + if (pool) + { + pool->put(ptr); + } +} + +template <typename T, typename Derived> +void SingletonPoolDeleter<T, Derived>::operator()(T* ptr) const noexcept +{ + Derived::instance().put(ptr); +} + +// --------------------------------------------------------------------------- +// CudaEventPool — singleton CRTP pool for CUevent handles. +// --------------------------------------------------------------------------- +class CudaEventPool : public SimplePool<CUevent_st, CudaEventPool> +{ +public: + static CudaEventPool& instance(); + +private: + CudaEventPool(); +}; + +// --------------------------------------------------------------------------- +// CudaStreamPool — singleton CRTP pool for CUstream handles. +// --------------------------------------------------------------------------- +class CudaStreamPool : public SimplePool<CUstream_st, CudaStreamPool> +{ +public: + static CudaStreamPool& instance(); + +private: + CudaStreamPool(); +}; + +// --------------------------------------------------------------------------- +// CachedCudaEvent — pooled CUevent (no timing). +// Mirrors _utils.py::CachedCudaEvent. +// +// On construction: gets an event from the global pool and records it to stream. +// Copyable: copies share the same underlying CUevent via shared_ptr. +// Last copy returns the event to the pool. +// NULL sentinel: always considered complete, no event in flight. +// --------------------------------------------------------------------------- +class CachedCudaEvent +{ +public: + // NULL sentinel: always considered complete, no event in flight. + static CachedCudaEvent makeNull() noexcept; + + // Normal constructor: gets an event and records it on stream. + explicit CachedCudaEvent(CudaStream stream); + + // Copyable and movable (shared ownership of the underlying CUevent). + CachedCudaEvent(CachedCudaEvent const&) = default; + CachedCudaEvent& operator=(CachedCudaEvent const&) = default; + CachedCudaEvent(CachedCudaEvent&&) noexcept = default; + CachedCudaEvent& operator=(CachedCudaEvent&&) noexcept = default; + ~CachedCudaEvent() = default; + + // Query if the recorded work is done. + bool queryComplete(); + + // Block until complete. + void synchronize(); + + // Insert a stream dependency on this event. + void waitInStream(CudaStream stream) const; + + // True if no CUevent is held (NULL or already closed by any copy). + [[nodiscard]] bool isClosed() const noexcept + { + return !mEvent || !*mEvent; + } + + // Release the event back to pool. Visible to ALL copies sharing this event. + void close(); + + // Raw CUevent handle. Returns nullptr for NULL/closed events. + // Also serves as identity key for deduplication. + [[nodiscard]] CUevent handle() const noexcept + { + return isClosed() ? nullptr : mEvent->get(); + } + +private: + explicit CachedCudaEvent() noexcept = default; // used by makeNull() + + // Shared ownership of the PoolItem. close() resets the inner unique_ptr, + // visible to all copies. Last shared_ptr drop is a no-op (inner already empty). + std::shared_ptr<CudaEventPool::PoolItem> mEvent; +}; + +// --------------------------------------------------------------------------- +// Stream-level helpers. +// --------------------------------------------------------------------------- + +// Wait for all events on the given stream. Deduplicates internally. +// Mirrors Python's stream_wait_events() which converts to set() before iterating. +inline void streamWaitEvents(CudaStream stream, std::vector<CachedCudaEvent const*> const& events) +{ + thread_local std::vector<CUevent> handles; + handles.clear(); + handles.reserve(events.size()); + for (auto const* ev : events) + { + if (ev && !ev->isClosed()) + handles.push_back(ev->handle()); + } + std::sort(handles.begin(), handles.end()); + handles.erase(std::unique(handles.begin(), handles.end()), handles.end()); + for (CUevent h : handles) + cuCheck(cuStreamWaitEvent(reinterpret_cast<CUstream>(stream), h, 0)); +} + +// Synchronize and close all events. Deduplicates internally. +// Mirrors Python's set()-based synchronization pattern. +inline void synchronizeAll(std::vector<CachedCudaEvent*> const& events) +{ + thread_local std::vector<CUevent> handles; + handles.clear(); + handles.reserve(events.size()); + for (auto* ev : events) + { + if (!ev->isClosed()) + handles.push_back(ev->handle()); + } + std::sort(handles.begin(), handles.end()); + handles.erase(std::unique(handles.begin(), handles.end()), handles.end()); + for (CUevent h : handles) + cuCheck(cuEventSynchronize(h)); + for (auto* ev : events) + ev->close(); +} + +// --------------------------------------------------------------------------- +// CachedCudaStream — pooled non-blocking CUstream. +// Mirrors _utils.py::CachedCudaStream. +// --------------------------------------------------------------------------- +class CachedCudaStream +{ +public: + CachedCudaStream(); + + CachedCudaStream(CachedCudaStream&&) noexcept = default; + CachedCudaStream& operator=(CachedCudaStream&&) noexcept = default; + CachedCudaStream(CachedCudaStream const&) = delete; + CachedCudaStream& operator=(CachedCudaStream const&) = delete; + + [[nodiscard]] CUstream handle() const noexcept + { + return mPoolItem.get(); // CUstream = CUstream_st* + } + + // Wait for a single event on this stream. + void waitEvent(CachedCudaEvent const& event) const + { + event.waitInStream(reinterpret_cast<CudaStream>(handle())); + } + + // Wait for all events on this stream. Deduplicates internally. + void waitEvents(std::vector<CachedCudaEvent const*> const& events) + { + streamWaitEvents(reinterpret_cast<CudaStream>(handle()), events); + } + + CachedCudaEvent recordEvent(); + void synchronize(); + +private: + CudaStreamPool::PoolItem mPoolItem; // returns to pool on destruction +}; + +// --------------------------------------------------------------------------- +// TemporaryCudaStream — pooled stream with finish-event tracking. +// Mirrors Python's TemporaryCudaStream context manager. +// +// Usage (matches Python's `with TemporaryCudaStream(events) as stream:`): +// +// TemporaryCudaStream tempStream(priorEvents); +// { +// auto scope = tempStream.enter(); // __enter__ +// launchKernel(tempStream.get()); +// } // ~Scope → __exit__ records finish event +// auto ev = tempStream.takeFinishEvent(); // after with block +// +// --------------------------------------------------------------------------- +class TemporaryCudaStream +{ +public: + // Acquire a stream from pool and issue cuStreamWaitEvent for each prior event. + explicit TemporaryCudaStream(std::vector<CachedCudaEvent const*> const& priorEvents); + + // Begin a scoped block. Destructor records the finish event (= Python __exit__). + // Skips recording during stack unwinding to match Python's `if not exc_type:` guard. + [[nodiscard]] auto enter() + { + int const exCount = std::uncaught_exceptions(); + return FuncGuard( + [this, exCount]() + { + if (std::uncaught_exceptions() == exCount) + mFinishEvent = mStream.recordEvent(); + }); + } + + [[nodiscard]] CUstream get() const noexcept + { + return mStream.handle(); + } + + // Consume the finish event recorded by Scope destructor. + [[nodiscard]] CachedCudaEvent takeFinishEvent() + { + auto result = std::move(mFinishEvent); + mFinishEvent = CachedCudaEvent::makeNull(); + return result; + } + + TemporaryCudaStream(TemporaryCudaStream const&) = delete; + TemporaryCudaStream& operator=(TemporaryCudaStream const&) = delete; + +private: + CachedCudaStream mStream; + CachedCudaEvent mFinishEvent = CachedCudaEvent::makeNull(); +}; + +// Merge multiple CUDA events into one. +// Returns makeNull() for 0 live events, the single live event for 1, +// or a TemporaryCudaStream-merged event for many. +// Mirrors Python's merge_events() utility. +CachedCudaEvent mergeEvents(std::vector<CachedCudaEvent>& events); + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/hostMem.cpp b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/hostMem.cpp new file mode 100644 index 000000000000..e51cc2ceaa4b --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/hostMem.cpp @@ -0,0 +1,334 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_cache_manager_v2/utils/hostMem.h" +#include "kv_cache_manager_v2/exceptions.h" + +#include "tensorrt_llm/common/assert.h" +#include <algorithm> +#include <atomic> +#include <cerrno> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <cuda.h> +#include <exception> +#include <fcntl.h> +#include <mutex> +#include <stdexcept> +#include <string> +#include <string_view> +#include <sys/mman.h> +#include <sys/utsname.h> +#include <system_error> +#include <thread> +#include <unistd.h> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// Low-level helpers +// --------------------------------------------------------------------------- + +MemAddress hostMmap(size_t size) +{ + void* ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr == MAP_FAILED || ptr == nullptr) + { + throw HostOOMError(std::string("mmap failed: ") + std::strerror(errno)); + } + return reinterpret_cast<MemAddress>(ptr); +} + +void hostMunmap(MemAddress ptr, size_t size) noexcept +{ + int ret = ::munmap(reinterpret_cast<void*>(ptr), size); + if (ret != 0) + { + std::fprintf(stderr, "munmap failed with errno %d\n", errno); + } +} + +MemAddress hostMremap(MemAddress ptr, size_t oldSize, size_t newSize) +{ + void* newPtr = ::mremap(reinterpret_cast<void*>(ptr), oldSize, newSize, MREMAP_MAYMOVE); + if (newPtr == MAP_FAILED || newPtr == nullptr) + { + throw HostOOMError(std::string("mremap failed: ") + std::strerror(errno)); + } + return reinterpret_cast<MemAddress>(newPtr); +} + +void resizeFile(int fd, size_t newSize) +{ + off_t oldSize = ::lseek(fd, 0, SEEK_END); + if (static_cast<size_t>(oldSize) < newSize) + { + int ret = ::posix_fallocate(fd, oldSize, static_cast<off_t>(newSize - oldSize)); + if (ret != 0) + { + throw DiskOOMError("posix_fallocate failed: " + std::string(std::strerror(ret))); + } + } + else if (static_cast<size_t>(oldSize) > newSize) + { + if (::ftruncate(fd, static_cast<off_t>(newSize)) != 0) + { + throw DiskOOMError("ftruncate failed: " + std::string(std::strerror(errno))); + } + } +} + +bool hostUseThp() +{ + char const* value = std::getenv("TLLM_KV_CACHE_MANAGER_V2_THP"); + return value == nullptr || std::string_view(value) == "1"; +} + +int hostPrefaultThreads() +{ + char const* value = std::getenv("TLLM_KV_CACHE_MANAGER_V2_PREFAULT_THREADS"); + if (value != nullptr) + { + return std::stoi(value); + } + unsigned int const detectedCpuCount = std::thread::hardware_concurrency(); + unsigned int const cpuCount = detectedCpuCount == 0 ? 32 : detectedCpuCount; + return static_cast<int>(std::min(64U, cpuCount / 2)); +} + +void hostMadvisePageMode(MemAddress ptr, size_t size, bool useThp, HostMadviseFn madviseFn) noexcept +{ + HostMadviseFn const fn = madviseFn != nullptr ? madviseFn : ::madvise; + int const advice = useThp ? MADV_HUGEPAGE : MADV_NOHUGEPAGE; + if (fn(reinterpret_cast<void*>(ptr), size, advice) != 0) + { + std::fprintf(stderr, "madvise failed with errno %d\n", errno); + } +} + +void hostPrefaultChunk(MemAddress ptr, size_t size, HostMadviseFn madviseFn, HostMemsetFn memsetFn) +{ + HostMadviseFn const advise = madviseFn != nullptr ? madviseFn : ::madvise; + HostMemsetFn const touch = memsetFn != nullptr ? memsetFn : ::memset; +#ifdef MADV_POPULATE_WRITE + if (advise(reinterpret_cast<void*>(ptr), size, MADV_POPULATE_WRITE) == 0) + { + return; + } + + int const errorCode = errno; + if (errorCode == EINVAL || errorCode == ENOSYS) + { + touch(reinterpret_cast<void*>(ptr), 0, size); + return; + } + if (errorCode == ENOMEM) + { + throw HostOOMError("madvise(MADV_POPULATE_WRITE) failed: " + std::string(std::strerror(errorCode))); + } + throw std::system_error(errorCode, std::generic_category(), "madvise(MADV_POPULATE_WRITE) failed"); +#else + // MADV_POPULATE_WRITE requires glibc >= 2.34 / Linux >= 5.14 headers and is not defined in + // older build environments (e.g. Rocky8 package-sanity images). Fall back to explicitly + // touching the pages to force population, matching the EINVAL/ENOSYS runtime path above. + (void) advise; + touch(reinterpret_cast<void*>(ptr), 0, size); +#endif +} + +// --------------------------------------------------------------------------- +// HostMem implementation +// --------------------------------------------------------------------------- + +bool HostMem::shouldUseChunkedRegistration() +{ + struct utsname u + { + }; + + if (::uname(&u) != 0) + { + return false; + } + // Check for Linux kernel 6.11, 6.12, 6.13 prefix. + std::string_view rel{u.release}; + for (auto prefix : {"6.11", "6.12", "6.13"}) + { + if (rel.substr(0, 4) == prefix) + { + return true; + } + } + return false; +} + +HostMem::HostMem(size_t size) + : mUseThp(hostUseThp()) +{ + if (size == 0) + { + return; + } + mAddr = hostMmap(size); + TLLM_CHECK_DEBUG(mAddr % kAlignment == 0); + mSize = size; + try + { + madvisePageMode(); + int const prefaultThreads = hostPrefaultThreads(); + if (prefaultThreads > 0) + { + parallelPrefault(prefaultThreads); + } + registerToCuda(); + } + catch (...) + { + unregisterFromCuda(); + hostMunmap(mAddr, mSize); + mAddr = 0; + mSize = 0; + throw; + } +} + +HostMem::~HostMem() +{ + destroy(); +} + +void HostMem::resize(size_t newSize) +{ + unregisterFromCuda(); + try + { + mAddr = hostMremap(mAddr, mSize, newSize); + TLLM_CHECK_DEBUG(mAddr % kAlignment == 0); + mSize = newSize; + madvisePageMode(); + } + catch (...) + { + registerToCuda(); + throw; + } + registerToCuda(); +} + +void HostMem::destroy() +{ + if (mAddr == 0) + { + return; + } + unregisterFromCuda(); + hostMunmap(mAddr, mSize); + mAddr = 0; + mSize = 0; +} + +void HostMem::madvisePageMode() +{ + TLLM_CHECK_DEBUG(mAddr && mSize); + hostMadvisePageMode(mAddr, mSize, mUseThp); +} + +void HostMem::parallelPrefault(int numThreads) +{ + size_t const numChunks = (mSize + kPrefaultChunkSize - 1) / kPrefaultChunkSize; + int const workerCount = std::min<int>(numThreads, static_cast<int>(numChunks)); + std::atomic_size_t nextChunk{0}; + std::atomic_bool failed{false}; + std::exception_ptr error; + std::mutex errorMutex; + + auto worker = [&]() + { + while (!failed.load()) + { + size_t const chunkIndex = nextChunk.fetch_add(1); + if (chunkIndex >= numChunks) + { + return; + } + size_t const offset = chunkIndex * kPrefaultChunkSize; + size_t const chunkSize = std::min(kPrefaultChunkSize, mSize - offset); + try + { + hostPrefaultChunk(mAddr + offset, chunkSize); + } + catch (...) + { + failed.store(true); + std::lock_guard<std::mutex> lock(errorMutex); + if (error == nullptr) + { + error = std::current_exception(); + } + return; + } + } + }; + + std::vector<std::thread> workers; + workers.reserve(static_cast<size_t>(workerCount)); + for (int i = 0; i < workerCount; ++i) + { + workers.emplace_back(worker); + } + for (auto& thread : workers) + { + thread.join(); + } + if (error != nullptr) + { + std::rethrow_exception(error); + } +} + +void HostMem::registerToCuda() +{ + TLLM_CHECK_DEBUG(mNumRegisteredChunks == 0); + static bool chunked = shouldUseChunkedRegistration(); + + size_t chunkSize = (chunked && mSize > kChunkSize) ? kChunkSize : mSize; + for (size_t offset = 0; offset < mSize; offset += chunkSize) + { + size_t sz = std::min(chunkSize, mSize - offset); + CUresult res = cuMemHostRegister( + reinterpret_cast<void*>(mAddr + offset), sz, CU_MEMHOSTREGISTER_PORTABLE | CU_MEMHOSTREGISTER_DEVICEMAP); + cuCheck(res); + ++mNumRegisteredChunks; + } +} + +void HostMem::unregisterFromCuda() +{ + static bool chunked = shouldUseChunkedRegistration(); + size_t chunkSize = (chunked && mSize > kChunkSize) ? kChunkSize : mSize; + for (size_t offset = 0; offset < mSize && mNumRegisteredChunks > 0; offset += chunkSize) + { + cuMemHostUnregister(reinterpret_cast<void*>(mAddr + offset)); + --mNumRegisteredChunks; + } + TLLM_CHECK_DEBUG(mNumRegisteredChunks == 0); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/hostMem.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/hostMem.h new file mode 100644 index 000000000000..cf498653d2ee --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/hostMem.h @@ -0,0 +1,100 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/common.h" + +#include <cstddef> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// HostMem — mmap-backed pinned host memory, resizable via mremap. +// Mirrors _utils.py::HostMem. +// +// Memory is: +// - Anonymous private mmap +// - Advised according to TLLM_KV_CACHE_MANAGER_V2_THP +// - Optionally prefaulted in parallel before CUDA registration +// - Registered to CUDA as page-locked (CU_MEMHOSTREGISTER_DEVICEMAP) +// +// On kernels 6.11/6.12/6.13, pinning is chunked in 2GB pieces to work around +// a kernel bug that prevents pinning more than 2GB in one call. +// --------------------------------------------------------------------------- +class HostMem +{ +public: + static constexpr size_t kAlignment = 4096; // 4 KB + static constexpr size_t kChunkSize = 2ULL << 30; // 2 GB + static constexpr size_t kPrefaultChunkSize = 512ULL << 20; // 512 MB + + explicit HostMem(size_t size); + ~HostMem(); + + HostMem(HostMem const&) = delete; + HostMem& operator=(HostMem const&) = delete; + + // Resize in-place (mremap, preserves data). Unregisters and re-registers with CUDA. + void resize(size_t newSize); + + // Unregister from CUDA and unmap. Safe to call multiple times. + void destroy(); + + MemAddress address() const noexcept + { + return mAddr; + } + + size_t size() const noexcept + { + return mSize; + } + +private: + void registerToCuda(); + void unregisterFromCuda(); + void madvisePageMode(); + void parallelPrefault(int numThreads); + + MemAddress mAddr = 0; + size_t mSize = 0; + int mNumRegisteredChunks = 0; + bool mUseThp = true; + + // Detect kernel version once at startup. + static bool shouldUseChunkedRegistration(); +}; + +// --------------------------------------------------------------------------- +// Low-level wrappers used internally (also exposed for storage pool use). +// --------------------------------------------------------------------------- +MemAddress hostMmap(size_t size); // throws HostOOMError +void hostMunmap(MemAddress ptr, size_t size) noexcept; +MemAddress hostMremap(MemAddress ptr, size_t oldSize, size_t newSize); // throws HostOOMError +void resizeFile(int fd, size_t newSize); // throws DiskOOMError + +using HostMadviseFn = int (*)(void*, size_t, int); +using HostMemsetFn = void* (*) (void*, int, size_t); + +bool hostUseThp(); +int hostPrefaultThreads(); +void hostMadvisePageMode(MemAddress ptr, size_t size, bool useThp, HostMadviseFn madviseFn = nullptr) noexcept; +void hostPrefaultChunk(MemAddress ptr, size_t size, HostMadviseFn madviseFn = nullptr, HostMemsetFn memsetFn = nullptr); + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/math.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/math.h new file mode 100644 index 000000000000..bdd1fd503708 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/math.h @@ -0,0 +1,384 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "kv_cache_manager_v2/utils/typedIndex.h" + +#include "tensorrt_llm/common/assert.h" +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <map> +#include <numeric> +#include <stdexcept> +#include <utility> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// --------------------------------------------------------------------------- +// Integer math helpers (mirrors _utils.py) +// --------------------------------------------------------------------------- + +template <typename T> +[[nodiscard]] inline T divUp(T x, T y) noexcept +{ + return (x + y - 1) / y; +} + +template <typename T> +[[nodiscard]] inline T roundUp(T x, T y) noexcept +{ + return divUp(x, y) * y; +} + +template <typename T> +[[nodiscard]] inline T roundDown(T x, T y) noexcept +{ + return (x / y) * y; +} + +template <typename T> +[[nodiscard]] inline T exactDiv(T x, T y) +{ + TLLM_CHECK_DEBUG(x % y == 0); + return x / y; +} + +template <typename T> +[[nodiscard]] inline bool inRange(T x, T lower, T upper) noexcept +{ + return lower <= x && x < upper; +} + +// Returns the intersection of [a.first, a.second) and [b.first, b.second). +// If the ranges are disjoint, the result has first > second; the caller must +// check first < second before using it as a non-empty range. +template <typename T> +[[nodiscard]] inline std::pair<T, T> overlap(std::pair<T, T> a, std::pair<T, T> b) noexcept +{ + T lo = a.first > b.first ? a.first : b.first; + T hi = a.second < b.second ? a.second : b.second; + return {lo, hi}; +} + +// Extract an attribute from the first element and assert (debug) all elements agree. +// Mirrors Python's get_uniform_attribute(_utils.py:202). +template <typename Range, typename Func> +[[nodiscard]] auto getUniformAttribute(Range const& range, Func&& func) -> decltype(func(*range.begin())) +{ + auto it = range.begin(); + TLLM_CHECK_DEBUG(it != range.end()); + auto result = func(*it); + TLLM_CHECK_DEBUG(std::all_of(range.begin(), range.end(), [&](auto const& item) { return func(item) == result; })); + return result; +} + +// Find the first index in [begin, end) where predicate is true. +// Returns distance(begin, end) if not found. +// Mirrors Python's find_index(_utils.py:366). +template <typename Iter, typename Pred> +[[nodiscard]] int findIndex(Iter begin, Iter end, Pred pred) +{ + return static_cast<int>(std::distance(begin, std::find_if(begin, end, pred))); +} + +// Steal items matching predicate from the container and return them (stable). +// Single-pass O(n), mirrors Python's remove_if(_utils.py:174). +template <typename T, typename Pred> +std::vector<T> stealIf(std::vector<T>& original, Pred pred) +{ + std::vector<T> removed; + size_t writeIdx = 0; + for (size_t i = 0; i < original.size(); ++i) + { + if (pred(original[i])) + removed.push_back(std::move(original[i])); + else + original[writeIdx++] = std::move(original[i]); + } + original.erase(original.begin() + static_cast<ptrdiff_t>(writeIdx), original.end()); + return removed; +} + +// Group items by a classifier function, returning a map of key → vector of items. +// Mirrors Python's partition(_utils.py:195). +template <typename T, typename Classifier> +auto partition(std::vector<T> const& items, Classifier classifier) + -> std::map<decltype(classifier(std::declval<T const&>())), std::vector<T>> +{ + using Key = decltype(classifier(std::declval<T const&>())); + std::map<Key, std::vector<T>> result; + for (auto const& item : items) + result[classifier(item)].push_back(item); + return result; +} + +// Normalize a vector of values to a ratio vector summing to 1.0. +// Mirrors Python's typed_map(values, lambda x: x / total). +template <typename T> +[[nodiscard]] std::vector<float> normalizeToRatio(std::vector<T> const& values) +{ + auto total = std::accumulate(values.begin(), values.end(), static_cast<T>(0)); + TLLM_CHECK_DEBUG(total > 0); + std::vector<float> ratio(values.size()); + for (size_t i = 0; i < values.size(); ++i) + ratio[i] = static_cast<float>(values[i]) / static_cast<float>(total); + return ratio; +} + +template <typename Index, typename T> +[[nodiscard]] TypedVec<Index, float> normalizeToRatio(TypedVec<Index, T> const& values) +{ + auto total = std::accumulate(values.begin(), values.end(), static_cast<T>(0)); + TLLM_CHECK_DEBUG(total > 0); + TypedVec<Index, float> ratio(values.size()); + for (Index index{0}; index < values.size(); ++index) + { + ratio[index] = static_cast<float>(values[index]) / static_cast<float>(total); + } + return ratio; +} + +// --------------------------------------------------------------------------- +// HalfOpenRange — a half-open range [beg, end). Empty when beg >= end. +// Mirrors _utils.py::HalfOpenRange. +// --------------------------------------------------------------------------- +template <typename Index = int> +struct HalfOpenRange +{ + using IndexType = Index; + using DifferenceType = decltype(std::declval<Index>() - std::declval<Index>()); + + Index beg{0}; + Index end{0}; + + constexpr HalfOpenRange() noexcept = default; + + template <typename Beg, typename End> + constexpr HalfOpenRange(Beg b, End e) noexcept + : beg(Index{b}) + , end(Index{e}) + { + } + + [[nodiscard]] DifferenceType length() const noexcept + { + return beg < end ? end - beg : DifferenceType{0}; + } + + [[nodiscard]] bool empty() const noexcept + { + return beg >= end; + } + + explicit operator bool() const noexcept + { + return beg < end; + } + + bool operator==(HalfOpenRange const& o) const noexcept + { + if (beg >= end && o.beg >= o.end) + return true; + return beg == o.beg && end == o.end; + } + + bool operator!=(HalfOpenRange const& o) const noexcept + { + return !(*this == o); + } + + // Membership test: is value in [beg, end)? + // Mirrors Python's HalfOpenRange.__contains__. + [[nodiscard]] bool contains(Index value) const noexcept + { + return beg <= value && value < end; + } +}; + +// Returns the intersection of two half-open ranges. +// The result may be empty (beg >= end), which is safe to chain. +template <typename Index> +[[nodiscard]] inline HalfOpenRange<Index> intersect(HalfOpenRange<Index> a, HalfOpenRange<Index> b) noexcept +{ + return {std::max(a.beg, b.beg), std::min(a.end, b.end)}; +} + +// --------------------------------------------------------------------------- +// DynamicBitset — resizable bitset using 64-bit words. +// Mirrors _utils.py::DynamicBitset. +// --------------------------------------------------------------------------- +class DynamicBitset +{ +public: + explicit DynamicBitset(size_t capacity) + : mWords(divUp(capacity, size_t{64}), uint64_t{0}) + , mNumSetBits(0) + { + } + + void set(size_t index) + { + if (!get(index)) + { + mWords[index / 64] |= (uint64_t{1} << (index % 64)); + ++mNumSetBits; + } + } + + [[nodiscard]] bool get(size_t index) const noexcept + { + return (mWords[index / 64] & (uint64_t{1} << (index % 64))) != 0; + } + + void clear(size_t index) + { + if (get(index)) + { + mWords[index / 64] &= ~(uint64_t{1} << (index % 64)); + --mNumSetBits; + } + } + + [[nodiscard]] size_t numSetBits() const noexcept + { + return mNumSetBits; + } + + void resize(size_t newCapacity) + { + size_t const oldWords = mWords.size(); + size_t const newWords = divUp(newCapacity, size_t{64}); + + // When the capacity shrinks, every set bit at or above newCapacity is + // dropped. Account for those bits so numSetBits() stays accurate, and + // mask the retained partial word so anySet() cannot observe stale bits. + // This covers both fewer-words and same-word-count (newWords == oldWords + // with a smaller newCapacity) shrinks. + if (newWords <= oldWords) + { + for (size_t w = newWords; w < oldWords; ++w) + { + mNumSetBits -= static_cast<size_t>(__builtin_popcountll(mWords[w])); + } + if (newWords >= 1 && newCapacity % 64 != 0) + { + uint64_t const keepMask = (uint64_t{1} << (newCapacity % 64)) - 1; + uint64_t& word = mWords[newWords - 1]; + mNumSetBits -= static_cast<size_t>(__builtin_popcountll(word & ~keepMask)); + word &= keepMask; + } + } + + // Grow (zero-filled) or shrink storage to the new word count. + mWords.resize(newWords, uint64_t{0}); + } + + // Returns true if any bit in [start, end) is set. + [[nodiscard]] bool anySet(size_t start, size_t end) const noexcept + { + if (start >= end) + { + return false; + } + size_t startWord = start / 64; + size_t endWord = (end - 1) / 64; + uint64_t startMask = ~uint64_t{0} << (start % 64); + if (startWord == endWord) + { + size_t bitsInWord = end % 64; + uint64_t endMask = bitsInWord ? ((uint64_t{1} << bitsInWord) - 1) : ~uint64_t{0}; + return (mWords[startWord] & startMask & endMask) != 0; + } + if (mWords[startWord] & startMask) + { + return true; + } + for (size_t w = startWord + 1; w < endWord; ++w) + { + if (mWords[w]) + { + return true; + } + } + size_t bitsInLastWord = end % 64; + if (bitsInLastWord == 0) + { + return mWords[endWord] != 0; + } + return (mWords[endWord] & ((uint64_t{1} << bitsInLastWord) - 1)) != 0; + } + +private: + std::vector<uint64_t> mWords; + size_t mNumSetBits; +}; + +// --------------------------------------------------------------------------- +// Array2D — row-major 2D array with typed row/column indices. +// Mirrors _utils.py::Array2D. +// --------------------------------------------------------------------------- +template <typename T> +class Array2D +{ +public: + Array2D(int rows, int cols, T initVal = T{}) + : mData(static_cast<size_t>(rows * cols), initVal) + , mCols(cols) + { + } + + T& operator()(int row, int col) noexcept + { + return mData[static_cast<size_t>(row * mCols + col)]; + } + + T const& operator()(int row, int col) const noexcept + { + return mData[static_cast<size_t>(row * mCols + col)]; + } + + [[nodiscard]] int rows() const noexcept + { + return static_cast<int>(mData.size()) / mCols; + } + + [[nodiscard]] int cols() const noexcept + { + return mCols; + } + + // Pointer to start of row (for slicing / iteration). + T* rowData(int row) noexcept + { + return mData.data() + row * mCols; + } + + T const* rowData(int row) const noexcept + { + return mData.data() + row * mCols; + } + +private: + std::vector<T> mData; + int mCols; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/sharedPtr.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/sharedPtr.h new file mode 100644 index 000000000000..604a8a0e9c12 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/sharedPtr.h @@ -0,0 +1,593 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Non-atomic SharedPtr / WeakPtr — drop-in replacements for std::shared_ptr +// and std::weak_ptr that use plain int refcounts instead of atomics. +// +// Motivation: std::shared_ptr uses atomic increments/decrements for thread +// safety. On ARM (Grace), atomics are significantly more expensive than on +// x86. The KV cache manager's shared_ptr usage is entirely single-threaded, +// so the atomics are pure overhead. +// +// Usage: +// SharedPtr<T> replaces std::shared_ptr<T> +// WeakPtr<T> replaces std::weak_ptr<T> +// EnableSharedFromThis<T> replaces std::enable_shared_from_this<T> +// makeShared<T>(args...) replaces std::make_shared<T>(args...) +// dynamicPointerCast<T>(p) replaces std::dynamic_pointer_cast<T>(p) +// toStd(SharedPtr<T>) bridges SharedPtr<T> → std::shared_ptr<T> + +#pragma once + +#include "tensorrt_llm/common/assert.h" +#include <cstddef> +#include <memory> +#include <new> +#include <type_traits> +#include <utility> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +// Forward declarations. +template <typename T> +class SharedPtr; +template <typename T> +class WeakPtr; +template <typename T> +class EnableSharedFromThis; + +// ========================================================================= +// Control block +// ========================================================================= + +namespace detail +{ + +struct ControlBlockBase +{ + int strongCount = 1; + int weakCount = 1; // +1 bias while strongCount > 0 + + virtual void destroyObject() noexcept = 0; + virtual void deallocate() noexcept = 0; + + void addStrongRef() noexcept + { + ++strongCount; + } + + void releaseStrongRef() noexcept + { + if (--strongCount == 0) + { + destroyObject(); + releaseWeakRef(); // release the bias + } + } + + void addWeakRef() noexcept + { + ++weakCount; + } + + void releaseWeakRef() noexcept + { + if (--weakCount == 0) + { + deallocate(); + } + } + +protected: + ~ControlBlockBase() = default; // prevent polymorphic delete via base +}; + +template <typename T> +struct InplaceControlBlock final : ControlBlockBase +{ + // Aligned uninitialized storage for T. + alignas(T) unsigned char storage[sizeof(T)]; + + T* ptr() noexcept + { + return std::launder(reinterpret_cast<T*>(&storage)); + } + + void destroyObject() noexcept override + { + ptr()->~T(); + } + + void deallocate() noexcept override + { + delete this; + } +}; + +// --------------------------------------------------------------------------- +// EnableSharedFromThis detection — handles polymorphic inheritance. +// E.g. makeShared<CommittedPage>() detects EnableSharedFromThis<Page> +// inherited through Page. +// --------------------------------------------------------------------------- +template <typename T> +auto detectEnableImpl(EnableSharedFromThis<T>*) -> EnableSharedFromThis<T>; + +auto detectEnableImpl(...) -> void; + +template <typename T> +using EnableBase = decltype(detectEnableImpl(std::declval<T*>())); + +template <typename T> +inline constexpr bool hasEnable = !std::is_void_v<EnableBase<T>>; + +} // namespace detail + +// ========================================================================= +// SharedPtr<T> +// ========================================================================= + +template <typename T> +class SharedPtr +{ +public: + // -- Constructors ------------------------------------------------------- + + SharedPtr() noexcept + : mPtr(nullptr) + , mCb(nullptr) + { + } + + SharedPtr(std::nullptr_t) noexcept // NOLINT(google-explicit-constructor) + : mPtr(nullptr) + , mCb(nullptr) + { + } + + SharedPtr(SharedPtr const& other) noexcept + : mPtr(other.mPtr) + , mCb(other.mCb) + { + if (mCb) + mCb->addStrongRef(); + } + + SharedPtr(SharedPtr&& other) noexcept + : mPtr(other.mPtr) + , mCb(other.mCb) + { + other.mPtr = nullptr; + other.mCb = nullptr; + } + + // Converting copy (U* implicitly convertible to T*). + template <typename U, std::enable_if_t<std::is_convertible_v<U*, T*>, int> = 0> + SharedPtr(SharedPtr<U> const& other) noexcept // NOLINT(google-explicit-constructor) + : mPtr(other.mPtr) + , mCb(other.mCb) + { + if (mCb) + mCb->addStrongRef(); + } + + // Converting move. + template <typename U, std::enable_if_t<std::is_convertible_v<U*, T*>, int> = 0> + SharedPtr(SharedPtr<U>&& other) noexcept // NOLINT(google-explicit-constructor) + : mPtr(other.mPtr) + , mCb(other.mCb) + { + other.mPtr = nullptr; + other.mCb = nullptr; + } + + ~SharedPtr() + { + if (mCb) + mCb->releaseStrongRef(); + } + + // -- Assignment --------------------------------------------------------- + + SharedPtr& operator=(SharedPtr const& other) noexcept + { + if (this != &other) + { + SharedPtr tmp(other); + swap(tmp); + } + return *this; + } + + SharedPtr& operator=(SharedPtr&& other) noexcept + { + if (this != &other) + { + SharedPtr tmp(std::move(other)); + swap(tmp); + } + return *this; + } + + template <typename U, std::enable_if_t<std::is_convertible_v<U*, T*>, int> = 0> + SharedPtr& operator=(SharedPtr<U> const& other) noexcept + { + SharedPtr tmp(other); + swap(tmp); + return *this; + } + + template <typename U, std::enable_if_t<std::is_convertible_v<U*, T*>, int> = 0> + SharedPtr& operator=(SharedPtr<U>&& other) noexcept + { + SharedPtr tmp(std::move(other)); + swap(tmp); + return *this; + } + + SharedPtr& operator=(std::nullptr_t) noexcept + { + reset(); + return *this; + } + + // -- Observers ---------------------------------------------------------- + + T* get() const noexcept + { + return mPtr; + } + + T& operator*() const noexcept + { + return *mPtr; + } + + T* operator->() const noexcept + { + return mPtr; + } + + explicit operator bool() const noexcept + { + return mPtr != nullptr; + } + + int useCount() const noexcept + { + return mCb ? mCb->strongCount : 0; + } + + // -- Modifiers ---------------------------------------------------------- + + void reset() noexcept + { + SharedPtr().swap(*this); + } + + void swap(SharedPtr& other) noexcept + { + std::swap(mPtr, other.mPtr); + std::swap(mCb, other.mCb); + } + + // -- Comparisons -------------------------------------------------------- + + bool operator==(SharedPtr const& other) const noexcept + { + return mPtr == other.mPtr; + } + + bool operator!=(SharedPtr const& other) const noexcept + { + return mPtr != other.mPtr; + } + + bool operator==(std::nullptr_t) const noexcept + { + return mPtr == nullptr; + } + + bool operator!=(std::nullptr_t) const noexcept + { + return mPtr != nullptr; + } + +private: + // Aliasing constructor — shares control block from `owner`, stores `ptr`. + // Used by dynamicPointerCast and EnableSharedFromThis. + SharedPtr(detail::ControlBlockBase* cb, T* ptr) noexcept + : mPtr(ptr) + , mCb(cb) + { + if (mCb) + mCb->addStrongRef(); + } + + template <typename U> + friend class SharedPtr; + template <typename U> + friend class WeakPtr; + template <typename U> + friend class EnableSharedFromThis; + template <typename U, typename... Args> + friend SharedPtr<U> makeShared(Args&&... args); + template <typename To, typename From> + friend SharedPtr<To> dynamicPointerCast(SharedPtr<From> const&); + template <typename U> + friend std::shared_ptr<U> toStd(SharedPtr<U> const&); + + T* mPtr; + detail::ControlBlockBase* mCb; +}; + +// Free-standing comparison with nullptr (reversed operand order). +template <typename T> +bool operator==(std::nullptr_t, SharedPtr<T> const& sp) noexcept +{ + return sp == nullptr; +} + +template <typename T> +bool operator!=(std::nullptr_t, SharedPtr<T> const& sp) noexcept +{ + return sp != nullptr; +} + +// ========================================================================= +// WeakPtr<T> +// ========================================================================= + +template <typename T> +class WeakPtr +{ +public: + WeakPtr() noexcept + : mPtr(nullptr) + , mCb(nullptr) + { + } + + WeakPtr(SharedPtr<T> const& sp) noexcept // NOLINT(google-explicit-constructor) + : mPtr(sp.mPtr) + , mCb(sp.mCb) + { + if (mCb) + mCb->addWeakRef(); + } + + // Converting constructor from SharedPtr<U>. + template <typename U, std::enable_if_t<std::is_convertible_v<U*, T*>, int> = 0> + WeakPtr(SharedPtr<U> const& sp) noexcept // NOLINT(google-explicit-constructor) + : mPtr(sp.mPtr) + , mCb(sp.mCb) + { + if (mCb) + mCb->addWeakRef(); + } + + WeakPtr(WeakPtr const& other) noexcept + : mPtr(other.mPtr) + , mCb(other.mCb) + { + if (mCb) + mCb->addWeakRef(); + } + + WeakPtr(WeakPtr&& other) noexcept + : mPtr(other.mPtr) + , mCb(other.mCb) + { + other.mPtr = nullptr; + other.mCb = nullptr; + } + + ~WeakPtr() + { + if (mCb) + mCb->releaseWeakRef(); + } + + // -- Assignment --------------------------------------------------------- + + WeakPtr& operator=(SharedPtr<T> const& sp) noexcept + { + WeakPtr tmp(sp); + swap(tmp); + return *this; + } + + template <typename U, std::enable_if_t<std::is_convertible_v<U*, T*>, int> = 0> + WeakPtr& operator=(SharedPtr<U> const& sp) noexcept + { + WeakPtr tmp(sp); + swap(tmp); + return *this; + } + + WeakPtr& operator=(WeakPtr const& other) noexcept + { + if (this != &other) + { + WeakPtr tmp(other); + swap(tmp); + } + return *this; + } + + WeakPtr& operator=(WeakPtr&& other) noexcept + { + if (this != &other) + { + WeakPtr tmp(std::move(other)); + swap(tmp); + } + return *this; + } + + // -- Observers ---------------------------------------------------------- + + bool expired() const noexcept + { + return !mCb || mCb->strongCount == 0; + } + + SharedPtr<T> lock() const noexcept + { + if (expired()) + return SharedPtr<T>(); + // Object still alive — construct a SharedPtr sharing the control block. + SharedPtr<T> result; + result.mPtr = mPtr; + result.mCb = mCb; + mCb->addStrongRef(); + return result; + } + + // -- Modifiers ---------------------------------------------------------- + + void reset() noexcept + { + WeakPtr().swap(*this); + } + + void swap(WeakPtr& other) noexcept + { + std::swap(mPtr, other.mPtr); + std::swap(mCb, other.mCb); + } + +private: + template <typename U> + friend class SharedPtr; + template <typename U> + friend class WeakPtr; + template <typename U> + friend class EnableSharedFromThis; + + T* mPtr; + detail::ControlBlockBase* mCb; +}; + +// ========================================================================= +// EnableSharedFromThis<T> +// ========================================================================= + +template <typename T> +class EnableSharedFromThis +{ +public: + SharedPtr<T> sharedFromThis() + { + auto sp = mWeakThis.lock(); + TLLM_CHECK_DEBUG_WITH_INFO(sp, "sharedFromThis() called on object not owned by SharedPtr"); + return sp; + } + + SharedPtr<T const> sharedFromThis() const + { + auto sp = mWeakThis.lock(); + TLLM_CHECK_DEBUG_WITH_INFO(sp, "sharedFromThis() called on object not owned by SharedPtr"); + // Convert SharedPtr<T> to SharedPtr<T const> via the converting constructor. + return sp; + } + +protected: + EnableSharedFromThis() noexcept = default; + + // Copy/move must NOT copy mWeakThis — the new object is a distinct entity. + EnableSharedFromThis(EnableSharedFromThis const&) noexcept {} + + EnableSharedFromThis& operator=(EnableSharedFromThis const&) noexcept + { + return *this; + } + + ~EnableSharedFromThis() = default; + +private: + template <typename U, typename... Args> + friend SharedPtr<U> makeShared(Args&&... args); + + mutable WeakPtr<T> mWeakThis; +}; + +// ========================================================================= +// makeShared<T>(args...) +// ========================================================================= + +template <typename T, typename... Args> +SharedPtr<T> makeShared(Args&&... args) +{ + auto* cb = new detail::InplaceControlBlock<T>(); + try + { + new (cb->storage) T(std::forward<Args>(args)...); + } + catch (...) + { + // T's constructor threw — control block was never fully initialized. + // Release directly; destroyObject() must not run. + delete cb; + throw; + } + + SharedPtr<T> result; + result.mPtr = cb->ptr(); + result.mCb = cb; + + // Wire up EnableSharedFromThis if T (or a base) derives from it. + if constexpr (detail::hasEnable<T>) + { + result.mPtr->detail::template EnableBase<T>::mWeakThis = result; + } + + return result; +} + +// ========================================================================= +// dynamicPointerCast<To>(SharedPtr<From>) +// ========================================================================= + +template <typename To, typename From> +SharedPtr<To> dynamicPointerCast(SharedPtr<From> const& src) +{ + To* raw = dynamic_cast<To*>(src.get()); + if (!raw) + return SharedPtr<To>(); + // Aliasing: share control block, different pointer. + return SharedPtr<To>(src.mCb, raw); +} + +// ========================================================================= +// toStd — one-way bridge to std::shared_ptr for nanobind boundary +// ========================================================================= + +template <typename T> +std::shared_ptr<T> toStd(SharedPtr<T> const& sp) +{ + if (!sp) + return nullptr; + // The custom deleter captures a copy of the SharedPtr, keeping + // the non-atomic refcount alive. When the last std::shared_ptr + // copy dies, the captured SharedPtr is destroyed and releases its + // non-atomic reference. + SharedPtr<T> copy(sp); + return std::shared_ptr<T>(sp.get(), [captured = std::move(copy)](T*) mutable { captured.reset(); }); +} + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 diff --git a/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/typedIndex.h b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/typedIndex.h new file mode 100644 index 000000000000..bc50645d4941 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/typedIndex.h @@ -0,0 +1,377 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/common/assert.h" +#include <cstddef> +#include <functional> +#include <initializer_list> +#include <type_traits> +#include <utility> +#include <vector> + +namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 +{ + +template <typename T, typename Tag, T DefaultValue = T{}> +class StrongIndex +{ + static_assert(std::is_integral<T>::value, "StrongIndex requires an integral underlying type"); + +public: + using ValueType = T; + + constexpr StrongIndex() noexcept = default; + + explicit constexpr StrongIndex(T value) noexcept + : mValue(value) + { + } + + [[nodiscard]] constexpr T value() const noexcept + { + return mValue; + } + + template <typename U, typename = std::enable_if_t<std::is_integral<U>::value>> + constexpr StrongIndex& operator+=(U rhs) noexcept + { + mValue = static_cast<T>(mValue + static_cast<T>(rhs)); + return *this; + } + + template <typename U, typename = std::enable_if_t<std::is_integral<U>::value>> + constexpr StrongIndex& operator-=(U rhs) noexcept + { + mValue = static_cast<T>(mValue - static_cast<T>(rhs)); + return *this; + } + + constexpr StrongIndex& operator++() noexcept + { + ++mValue; + return *this; + } + + constexpr StrongIndex operator++(int) noexcept + { + StrongIndex old{*this}; + ++(*this); + return old; + } + + constexpr StrongIndex& operator--() noexcept + { + --mValue; + return *this; + } + + constexpr StrongIndex operator--(int) noexcept + { + StrongIndex old{*this}; + --(*this); + return old; + } + +private: + T mValue{DefaultValue}; +}; + +template <typename T, typename Tag, T DefaultValue> +constexpr bool operator==(StrongIndex<T, Tag, DefaultValue> lhs, StrongIndex<T, Tag, DefaultValue> rhs) noexcept +{ + return lhs.value() == rhs.value(); +} + +template <typename T, typename Tag, T DefaultValue> +constexpr bool operator!=(StrongIndex<T, Tag, DefaultValue> lhs, StrongIndex<T, Tag, DefaultValue> rhs) noexcept +{ + return !(lhs == rhs); +} + +template <typename T, typename Tag, T DefaultValue> +constexpr bool operator<(StrongIndex<T, Tag, DefaultValue> lhs, StrongIndex<T, Tag, DefaultValue> rhs) noexcept +{ + return lhs.value() < rhs.value(); +} + +template <typename T, typename Tag, T DefaultValue> +constexpr bool operator<(StrongIndex<T, Tag, DefaultValue> lhs, T rhs) noexcept +{ + return lhs.value() < rhs; +} + +template <typename T, typename Tag, T DefaultValue> +constexpr bool operator>(StrongIndex<T, Tag, DefaultValue> lhs, StrongIndex<T, Tag, DefaultValue> rhs) noexcept +{ + return rhs < lhs; +} + +template <typename T, typename Tag, T DefaultValue> +constexpr bool operator<=(StrongIndex<T, Tag, DefaultValue> lhs, StrongIndex<T, Tag, DefaultValue> rhs) noexcept +{ + return !(rhs < lhs); +} + +template <typename T, typename Tag, T DefaultValue> +constexpr bool operator>=(StrongIndex<T, Tag, DefaultValue> lhs, StrongIndex<T, Tag, DefaultValue> rhs) noexcept +{ + return !(lhs < rhs); +} + +template <typename T, typename Tag, T DefaultValue> +constexpr bool operator>=(StrongIndex<T, Tag, DefaultValue> lhs, T rhs) noexcept +{ + return lhs.value() >= rhs; +} + +template <typename T, typename Tag, T DefaultValue, typename U, typename = std::enable_if_t<std::is_integral<U>::value>> +constexpr StrongIndex<T, Tag, DefaultValue> operator+(StrongIndex<T, Tag, DefaultValue> lhs, U rhs) noexcept +{ + lhs += rhs; + return lhs; +} + +template <typename U, typename T, typename Tag, T DefaultValue, typename = std::enable_if_t<std::is_integral<U>::value>> +constexpr StrongIndex<T, Tag, DefaultValue> operator+(U lhs, StrongIndex<T, Tag, DefaultValue> rhs) noexcept +{ + rhs += lhs; + return rhs; +} + +template <typename T, typename Tag, T DefaultValue, typename U, typename = std::enable_if_t<std::is_integral<U>::value>> +constexpr StrongIndex<T, Tag, DefaultValue> operator-(StrongIndex<T, Tag, DefaultValue> lhs, U rhs) noexcept +{ + lhs -= rhs; + return lhs; +} + +template <typename T, typename Tag, T LhsDefaultValue, T RhsDefaultValue> +constexpr T operator-(StrongIndex<T, Tag, LhsDefaultValue> lhs, StrongIndex<T, Tag, RhsDefaultValue> rhs) noexcept +{ + return static_cast<T>(lhs.value() - rhs.value()); +} + +template <typename T, typename Tag, T DefaultValue> +[[nodiscard]] std::size_t toSizeT(StrongIndex<T, Tag, DefaultValue> index) noexcept +{ + if constexpr (std::is_signed<T>::value) + { + TLLM_CHECK_DEBUG_WITH_INFO(index.value() >= 0, "StrongIndex value must be non-negative for size_t conversion"); + } + return static_cast<std::size_t>(index.value()); +} + +template <typename Index, typename T> +class TypedVec +{ +public: + using IndexType = Index; + using ValueType = T; + using ContainerType = std::vector<T>; + using iterator = typename ContainerType::iterator; + using const_iterator = typename ContainerType::const_iterator; + + TypedVec() = default; + + explicit TypedVec(Index count) + : mData(toSizeT(count)) + { + } + + TypedVec(Index count, T const& value) + : mData(toSizeT(count), value) + { + } + + TypedVec(std::initializer_list<T> values) + : mData(values) + { + } + + explicit TypedVec(ContainerType data) + : mData(std::move(data)) + { + } + + [[nodiscard]] T& operator[](Index index) noexcept + { + return mData[toSizeT(index)]; + } + + [[nodiscard]] T const& operator[](Index index) const noexcept + { + return mData[toSizeT(index)]; + } + + [[nodiscard]] T& at(Index index) + { + return mData.at(toSizeT(index)); + } + + [[nodiscard]] T const& at(Index index) const + { + return mData.at(toSizeT(index)); + } + + [[nodiscard]] Index size() const noexcept + { + return Index{static_cast<typename Index::ValueType>(mData.size())}; + } + + [[nodiscard]] std::size_t stdSize() const noexcept + { + return mData.size(); + } + + [[nodiscard]] bool empty() const noexcept + { + return mData.empty(); + } + + void clear() noexcept + { + mData.clear(); + } + + void reserve(Index count) + { + mData.reserve(toSizeT(count)); + } + + void resize(Index count) + { + mData.resize(toSizeT(count)); + } + + void resize(Index count, T const& value) + { + mData.resize(toSizeT(count), value); + } + + void push_back(T const& value) + { + mData.push_back(value); + } + + void push_back(T&& value) + { + mData.push_back(std::move(value)); + } + + template <typename... Args> + T& emplace_back(Args&&... args) + { + return mData.emplace_back(std::forward<Args>(args)...); + } + + void pop_back() + { + mData.pop_back(); + } + + [[nodiscard]] T& front() + { + return mData.front(); + } + + [[nodiscard]] T const& front() const + { + return mData.front(); + } + + [[nodiscard]] T& back() + { + return mData.back(); + } + + [[nodiscard]] T const& back() const + { + return mData.back(); + } + + [[nodiscard]] iterator begin() noexcept + { + return mData.begin(); + } + + [[nodiscard]] const_iterator begin() const noexcept + { + return mData.begin(); + } + + [[nodiscard]] const_iterator cbegin() const noexcept + { + return mData.cbegin(); + } + + [[nodiscard]] iterator end() noexcept + { + return mData.end(); + } + + [[nodiscard]] const_iterator end() const noexcept + { + return mData.end(); + } + + [[nodiscard]] const_iterator cend() const noexcept + { + return mData.cend(); + } + + [[nodiscard]] ContainerType& raw() noexcept + { + return mData; + } + + [[nodiscard]] ContainerType const& raw() const noexcept + { + return mData; + } + + friend bool operator==(TypedVec const& lhs, TypedVec const& rhs) + { + return lhs.mData == rhs.mData; + } + + friend bool operator!=(TypedVec const& lhs, TypedVec const& rhs) + { + return !(lhs == rhs); + } + +private: + ContainerType mData; +}; + +} // namespace tensorrt_llm::batch_manager::kv_cache_manager_v2 + +namespace std +{ + +template <typename T, typename Tag, T DefaultValue> +struct hash<tensorrt_llm::batch_manager::kv_cache_manager_v2::StrongIndex<T, Tag, DefaultValue>> +{ + size_t operator()( + tensorrt_llm::batch_manager::kv_cache_manager_v2::StrongIndex<T, Tag, DefaultValue> index) const noexcept + { + return std::hash<T>{}(index.value()); + } +}; + +} // namespace std diff --git a/cpp/tensorrt_llm/batch_manager/llmRequest.cpp b/cpp/tensorrt_llm/batch_manager/llmRequest.cpp index e51fad8ba149..d5466b4a7539 100644 --- a/cpp/tensorrt_llm/batch_manager/llmRequest.cpp +++ b/cpp/tensorrt_llm/batch_manager/llmRequest.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,6 +22,16 @@ namespace tensorrt_llm::batch_manager { +// Single, process-global storage for the steady-clock offset. Keeping it in this +// translation unit (rather than as an inline-static member reachable from every +// shared object) guarantees that libtensorrt_llm.so and the nanobind extension +// module observe the same value once either side calibrates it. +std::optional<std::chrono::steady_clock::duration>& globalSteadyClockOffset() +{ + static std::optional<std::chrono::steady_clock::duration> offset{std::nullopt}; + return offset; +} + template <typename TTensor, typename TStream> runtime::SizeType32 GenericLlmRequest<TTensor, TStream>::getBeamWidthByIter(bool const forNextIteration) { diff --git a/cpp/tensorrt_llm/batch_manager/logitsPostProcessor.cpp b/cpp/tensorrt_llm/batch_manager/logitsPostProcessor.cpp deleted file mode 100644 index 95b324f0f2ec..000000000000 --- a/cpp/tensorrt_llm/batch_manager/logitsPostProcessor.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/logitsPostProcessor.h" - -#include "tensorrt_llm/batch_manager/decoderBuffers.h" -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/batch_manager/runtimeBuffers.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/runtime/iTensor.h" - -namespace tr = tensorrt_llm::runtime; - -namespace tensorrt_llm::batch_manager -{ - -using TensorPtr = runtime::ITensor::SharedPtr; -using ITensor = runtime::ITensor; -using SizeType32 = tensorrt_llm::runtime::SizeType32; - -bool LogitsPostProcessor::operator()(DecoderInputBuffers& inputBuffers, bool replicateLogitsPostProcessor, - tr::WorldConfig const& worldConfig, CudaStreamPtr const& stream, - std::optional<LogitsPostProcessorBatched> const& logitsPostProcessorBatched) const -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(LogitsPostProcessor); - - // Arguments for batched processor - std::vector<LlmRequest::RequestIdType> reqIdsVec; - std::vector<LlmRequest::TensorPtr> logitsVec; - std::vector<std::reference_wrapper<LlmRequest::BeamTokens const>> beamTokensVec; - std::vector<std::optional<LlmRequest::RequestIdType>> clientIdsVec; - - bool logitsPostProcessorIsApplied = false; - for (size_t batchIdx = 0; batchIdx < inputBuffers.decoderRequests.size(); ++batchIdx) - { - auto const& llmReq = inputBuffers.decoderRequests.at(batchIdx); - auto& logits = inputBuffers.decoderLogits.at(batchIdx); - - // Invoke non-batched processor or collect arguments for batched processor - if (llmReq->mLogitsPostProcessor) - { - logitsPostProcessorIsApplied = true; - if (replicateLogitsPostProcessor || worldConfig.isFirstTensorParallelRank()) - { - (*llmReq->mLogitsPostProcessor)( - llmReq->mRequestId, logits, llmReq->getTokens(), stream, llmReq->mClientId); - } - } - else if (llmReq->mApplyLogitsPostProcessorBatched) - { - reqIdsVec.push_back(llmReq->mRequestId); - logitsVec.push_back(logits); - beamTokensVec.emplace_back(llmReq->getTokens()); - clientIdsVec.push_back(llmReq->mClientId); - } - } - - // Invoke batched processor - if (!reqIdsVec.empty()) - { - logitsPostProcessorIsApplied = true; - if (replicateLogitsPostProcessor || worldConfig.isFirstTensorParallelRank()) - { - (*logitsPostProcessorBatched)(reqIdsVec, logitsVec, beamTokensVec, stream, clientIdsVec); - } - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - - return logitsPostProcessorIsApplied; -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/loraBuffers.cpp b/cpp/tensorrt_llm/batch_manager/loraBuffers.cpp deleted file mode 100644 index b67b72f6c49a..000000000000 --- a/cpp/tensorrt_llm/batch_manager/loraBuffers.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "loraBuffers.h" - -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/runtime/loraUtils.h" - -namespace tensorrt_llm::batch_manager -{ - -LoraBuffers::LoraBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, runtime::TllmRuntime const& tllmRuntime, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) -{ - auto const localNbLayers - = modelConfig.getNbAttentionLayers(worldConfig.getPipelineParallelism(), worldConfig.getPipelineParallelRank()); - auto const firstLayerId = worldConfig.getPipelineParallelRank() * localNbLayers; - - auto nbModelConfigs = static_cast<SizeType32>(modelConfig.getLoraModules().size()); - - // there are 3 pointers: LoRA A, LoRA B, and a DoRA magnitude (null if not DoRA) - auto loraWeightsPtrsShape - = runtime::ITensor::makeShape({nbModelConfigs, localNbLayers, maxBatchSize * maxBeamWidth, 3}); - auto loraAdapterSizesShape - = runtime::ITensor::makeShape({nbModelConfigs, localNbLayers, maxBatchSize * maxBeamWidth}); - - auto firstModuleName = std::string(modelConfig.getLoraModules().front().name()); - auto ptrsFieldName = firstModuleName + "_lora_weights_pointers_" + std::to_string(firstLayerId); - auto rankFieldName = firstModuleName + "_lora_ranks_" + std::to_string(firstLayerId); - auto weightsPtrDtype = tllmRuntime.getEngine().getTensorDataType(ptrsFieldName.c_str()); - auto ranksDtype = tllmRuntime.getEngine().getTensorDataType(rankFieldName.c_str()); - - mLoraManager.create(modelConfig); - - mLoraWeightsPointersHost = runtime::BufferManager::pinned(loraWeightsPtrsShape, weightsPtrDtype); - mLoraAdapterSizesHost = runtime::BufferManager::pinned(loraAdapterSizesShape, ranksDtype); -} - -void LoraBuffers::fill(RequestVector const& contextRequests, RequestVector const& genRequests, - PeftTable const& peftTable, runtime::BufferManager const& manager, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig) -{ - manager.setZero(*mLoraWeightsPointersHost); - manager.setZero(*mLoraAdapterSizesHost); - - SizeType32 batchIdx{0}; - for (auto const& requests : {contextRequests, genRequests}) - { - for (auto const& llmReq : requests) - { - auto const optReqLoraWeights = llmReq->getLoraWeights(); - auto const optReqLoraConfig = llmReq->getLoraConfig(); - - auto const isContextRequest = llmReq->isContextInitState(); - auto const beamWidth = isContextRequest ? 1 : llmReq->mSamplingConfig.beamWidth; - auto const peftIt = peftTable.find(llmReq->mRequestId); - if (peftIt != peftTable.end()) - { - auto const& peftValues = peftIt->second; - if (!peftValues.empty()) - { - mLoraManager.fillInputTensors(mLoraWeightsPointersHost, mLoraAdapterSizesHost, peftIt->second, - batchIdx, beamWidth, modelConfig, worldConfig); - } - } - ++batchIdx; - } - } -} - -void LoraBuffers::validate(std::optional<std::uint64_t> const& optTaskId, - std::optional<TensorPtr> const& optReqLoraWeights, std::optional<TensorPtr> const& optReqLoraConfig, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) -{ - runtime::lora::loraValidateRequestTensors(optTaskId, optReqLoraWeights, optReqLoraConfig, modelConfig, worldConfig); -} - -void LoraBuffers::insertInputTensors(TensorMap& inputTensors, TensorPtr weightsPtrs, TensorPtr adapterSizes, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) const -{ - mLoraManager.insertInputTensors(inputTensors, weightsPtrs, adapterSizes, modelConfig, worldConfig); -} - -void LoraBuffers::reshape(SizeType32 numSequences) -{ - auto weightsPtrsShape = mLoraWeightsPointersHost->getShape(); - weightsPtrsShape.d[2] = numSequences; - mLoraWeightsPointersHost->reshape(weightsPtrsShape); - - auto adapterSizesShape = mLoraAdapterSizesHost->getShape(); - adapterSizesShape.d[2] = numSequences; - mLoraAdapterSizesHost->reshape(adapterSizesShape); -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/loraBuffers.h b/cpp/tensorrt_llm/batch_manager/loraBuffers.h deleted file mode 100644 index 3ba68995518f..000000000000 --- a/cpp/tensorrt_llm/batch_manager/loraBuffers.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/loraManager.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -namespace tensorrt_llm::batch_manager -{ - -class LoraBuffers -{ -public: - using SizeType32 = tensorrt_llm::runtime::SizeType32; - using PeftTable = runtime::LoraManager::PeftTable; - using TensorPtr = runtime::ITensor::SharedPtr; - using TensorMap = runtime::StringPtrMap<runtime::ITensor>; - - TensorPtr mLoraWeightsPointersHost; - TensorPtr mLoraAdapterSizesHost; - - runtime::LoraManager mLoraManager; - - LoraBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, runtime::TllmRuntime const& tllmRuntime, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); - - static void validate(std::optional<std::uint64_t> const& optTaskId, - std::optional<TensorPtr> const& optReqLoraWeights, std::optional<TensorPtr> const& optReqLoraConfig, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); - - void fill(RequestVector const& contextRequests, RequestVector const& genRequests, PeftTable const& peftTable, - runtime::BufferManager const& manager, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig); - - void insertInputTensors(TensorMap& inputTensors, TensorPtr weightsPtrs, TensorPtr adapterSizes, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) const; - - void reshape(SizeType32 numSequences); -}; -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.cpp b/cpp/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.cpp deleted file mode 100644 index 3e494a6383ec..000000000000 --- a/cpp/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.cpp +++ /dev/null @@ -1,198 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h" -#include "tensorrt_llm/batch_manager/decoderBuffers.h" -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/batch_manager/runtimeBuffers.h" -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/runtime/decoderState.h" -#include "tensorrt_llm/runtime/iGptDecoderBatched.h" - -namespace tr = tensorrt_llm::runtime; - -namespace tensorrt_llm::batch_manager -{ -using SizeType32 = MakeDecodingBatchInputOutput::SizeType32; -using TensorPtr = MakeDecodingBatchInputOutput::TensorPtr; - -void MakeDecodingBatchInputOutput::createDecoderBatchInputs(DecoderInputBuffers& inputBuffers, - std::vector<SizeType32> const& activeSlots, runtime::decoder::DecoderState const& decoderState) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const& numDecodingEngineTokens = decoderState.getNumDecodingEngineTokens(); - auto const& maxDecodingEngineTokens = decoderState.getMaxDecodingEngineTokens(); - auto const& maxDecodingDecoderTokens = decoderState.getMaxDecodingDecoderTokens(); - auto const maxDecoderSteps = common::ceilDiv(maxDecodingEngineTokens, maxDecodingDecoderTokens); - - auto& batchSlots = inputBuffers.forwardBatchSlots; - auto& decoderLogits = inputBuffers.decoderLogits; - - for (SizeType32 step = 0; step < maxDecoderSteps; ++step) - { - batchSlots.at(step)->resize(activeSlots.size()); - } - - auto constexpr singleRequest = 1; - - std::vector<SizeType32> batchSizes(maxDecoderSteps); - std::vector<std::vector<tr::ITensor::SharedConstPtr>> batchLogits(maxDecoderSteps); - auto maxActiveDecoderSteps = 1; - for (size_t batchIdx = 0; batchIdx < activeSlots.size(); ++batchIdx) - { - auto const slot = activeSlots.at(batchIdx); - auto const& logits = decoderLogits.at(batchIdx); - - auto const numDecoderSteps = common::ceilDiv(numDecodingEngineTokens.at(slot), maxDecodingDecoderTokens); - maxActiveDecoderSteps = std::max(maxActiveDecoderSteps, numDecoderSteps); - for (SizeType32 step = 0; step < numDecoderSteps; ++step) - { - auto batchSlotsRange = tr::BufferRange<SizeType32>(*batchSlots.at(step)); - batchSlotsRange[batchSizes[step]] = slot; - batchSizes[step]++; - auto logitsSlice = tr::ITensor::slice(logits, step, singleRequest); - batchLogits[step].emplace_back(std::move(logitsSlice)); - } - } - - for (SizeType32 step = 0; step < maxDecoderSteps; ++step) - { - batchSlots.at(step)->resize(batchSizes[step]); - } - batchLogits.resize(maxActiveDecoderSteps); - - inputBuffers.maxDecoderSteps = maxActiveDecoderSteps; - inputBuffers.batchLogits = batchLogits; - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -namespace -{ - -std::pair<std::vector<SizeType32>, std::vector<SizeType32>> getActiveSlots(RequestVector const& decoderRequests) -{ - std::vector<SizeType32> activeSlots; - std::vector<SizeType32> generationSteps; - for (auto const& llmReq : decoderRequests) - { - activeSlots.push_back(llmReq->mSeqSlot.value()); - generationSteps.push_back(llmReq->getDecodingIter()); - } - - return {activeSlots, generationSteps}; -} - -//! @brief Sets inputs for explicit draft tokens. -void setExplicitDraftTokensInputs(tr::DecodingInput& dInput, RuntimeBuffers const& fusedRuntimeBuffers) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - TLLM_CHECK(fusedRuntimeBuffers.mExplicitDraftTokensBuffers); - auto const& explicitDraftTokensInputs = fusedRuntimeBuffers.mExplicitDraftTokensBuffers->engineOutputs; - auto const& explicitDraftTokensLastInputs = fusedRuntimeBuffers.mExplicitDraftTokensBuffers->engineInputs; - - dInput.explicitDraftTokensInputs = tr::DecodingInput::ExplicitDraftTokensInputs(); - dInput.explicitDraftTokensInputs->nextDraftTokens = explicitDraftTokensInputs.nextDraftTokens; - dInput.explicitDraftTokensInputs->nextFlatTokens = explicitDraftTokensInputs.nextFlatTokens; - dInput.explicitDraftTokensInputs->nextDraftIndices = explicitDraftTokensInputs.nextDraftIndices; - dInput.explicitDraftTokensInputs->nextDraftProbs = explicitDraftTokensInputs.nextDraftProbs; - dInput.explicitDraftTokensInputs->lastDraftTokens = explicitDraftTokensLastInputs.draftTokens; - dInput.explicitDraftTokensInputs->lastDraftIndices = explicitDraftTokensLastInputs.draftIndices; - dInput.explicitDraftTokensInputs->lastPositionIdsBase = explicitDraftTokensLastInputs.positionIdsBase; - dInput.explicitDraftTokensInputs->masks = explicitDraftTokensInputs.masks; - dInput.explicitDraftTokensInputs->packedPositionIds = explicitDraftTokensInputs.packedPositionIds; - dInput.explicitDraftTokensInputs->bestPathLengths = explicitDraftTokensInputs.bestPathLengths; - dInput.explicitDraftTokensInputs->bestPathIndices = explicitDraftTokensInputs.bestPathIndices; - dInput.explicitDraftTokensInputs->nextGenerationLengths = explicitDraftTokensInputs.nextGenerationLengths; - dInput.explicitDraftTokensInputs->lastGenerationLengths = explicitDraftTokensLastInputs.generationLengths; - dInput.explicitDraftTokensInputs->maxGenLengthDevice = explicitDraftTokensInputs.maxGenToken; - // Slots in request order - dInput.explicitDraftTokensInputs->seqSlots = fusedRuntimeBuffers.seqSlots; - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -//! @brief Sets inputs for eagle decoding. -void setEagleInputs(tr::DecodingInput& dInput, RuntimeBuffers const& fusedRuntimeBuffers) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - TLLM_CHECK(fusedRuntimeBuffers.mEagleBuffers); - auto const& eagleInputs = fusedRuntimeBuffers.mEagleBuffers->engineOutputs; - auto const& eagleLastInputs = fusedRuntimeBuffers.mEagleBuffers->engineInputs; - - dInput.eagleInputs = tr::DecodingInput::EagleInputs(); - dInput.eagleInputs->nextDraftTokens = eagleInputs.nextDraftTokens; - dInput.eagleInputs->nextDraftLens = eagleInputs.nextDraftLens; - dInput.eagleInputs->nextDraftPaths = eagleInputs.nextDraftPaths; - dInput.eagleInputs->lastDraftTokens = eagleLastInputs.draftTokens; - dInput.eagleInputs->lastDraftLens = eagleLastInputs.draftLens; - dInput.eagleInputs->lastDraftPaths = eagleLastInputs.draftPaths; - dInput.eagleInputs->acceptedTokens = eagleInputs.acceptedTokens; - dInput.eagleInputs->acceptedLens = eagleInputs.acceptedLens; - dInput.eagleInputs->acceptedPathIds = eagleInputs.acceptedPaths; - dInput.eagleInputs->chunkedContextNextTokens = eagleInputs.chunkedContextNextTokens; - // Slots in request order - dInput.eagleInputs->seqSlots = fusedRuntimeBuffers.seqSlots; - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -} // namespace - -void MakeDecodingBatchInputOutput::operator()(DecoderInputBuffers& inputBuffers, - runtime::decoder::DecoderState& decoderState, runtime::ModelConfig const& modelConfig, - OptionalRef<RuntimeBuffers> fusedRuntimeBuffers) const -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto [activeSlots, generationSteps] = getActiveSlots(inputBuffers.decoderRequests); - - createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); - - auto const maxBeamWidth = decoderState.getMaxBeamWidth(); - if (maxBeamWidth > 1) - { - // For Variable-Beam-Width-Search - decoderState.getJointDecodingInput().generationSteps = generationSteps; - } - - if (modelConfig.getSpeculativeDecodingMode().hasDraftLogits()) - { - decoderState.getJointDecodingInput().medusaInputs->medusaLogits = inputBuffers.predictedDraftLogits; - } - - if (modelConfig.getSpeculativeDecodingMode().isExplicitDraftTokens()) - { - TLLM_CHECK(fusedRuntimeBuffers); - // requires mCtxGenFusion == true - setExplicitDraftTokensInputs(decoderState.getJointDecodingInput(), *fusedRuntimeBuffers); - } - else if (modelConfig.getSpeculativeDecodingMode().isEagle()) - { - TLLM_CHECK(fusedRuntimeBuffers); - // requires mCtxGenFusion == true - setEagleInputs(decoderState.getJointDecodingInput(), *fusedRuntimeBuffers); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/medusaBuffers.cpp b/cpp/tensorrt_llm/batch_manager/medusaBuffers.cpp index eb40208739cf..32935e683b83 100644 --- a/cpp/tensorrt_llm/batch_manager/medusaBuffers.cpp +++ b/cpp/tensorrt_llm/batch_manager/medusaBuffers.cpp @@ -17,99 +17,10 @@ #include "tensorrt_llm/batch_manager/medusaBuffers.h" #include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/medusaModule.h" -#include "tensorrt_llm/runtime/utils/speculativeChoicesUtils.h" namespace tensorrt_llm::batch_manager { -MedusaBuffers::MedusaBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, runtime::BufferManager const& manager, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - executor::DecodingConfig const& decodingConfig, runtime::TllmRuntime const& runtime) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - TLLM_CHECK_WITH_INFO(maxBeamWidth == 1, "Medusa does not support beam search"); - - auto const& engine = runtime.getEngine(); - - auto const maxNumSequences = maxBatchSize; - - auto const medusaModule = std::dynamic_pointer_cast<tensorrt_llm::runtime::MedusaModule const>( - modelConfig.getSpeculativeDecodingModulePtr()); - - auto const medusaHeads = medusaModule->getMaxDraftPathLen(); - auto const maxPathLen = medusaModule->getMaxPathLen(); // medusaHeads + 1 - auto const maxMedusaTokens = medusaModule->getMaxDecodingDraftTokens(); - auto const maxDecodingTokens = medusaModule->getMaxDecodingTokens(); // maxMedusaTokens + 1 - auto const numPackedMasks = medusaModule->getNumPackedMasks(); - - auto const vocabSizePadded = modelConfig.getVocabSizePadded(worldConfig.getSize()); - - if (worldConfig.isLastPipelineParallelRank()) - { - auto logitsType = engine.getTensorDataType("medusa_logits"); - medusaLogitsDevice = manager.gpu( - ITensor::makeShape({medusaHeads, maxBatchSize, maxDecodingTokens, vocabSizePadded}), logitsType); - } - - // Note: reserved for variable sequence length support. - medusaGenerationLengthsHost - = runtime::BufferManager::pinned(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); - // TODO: pack batch and tokensPerStep into one dim to support variable sequence length without padddings. - attentionPackedMaskHost = runtime::BufferManager::pinned( - ITensor::makeShape({maxNumSequences, maxDecodingTokens, numPackedMasks}), nvinfer1::DataType::kINT32); - medusaPositionOffsetsHost = runtime::BufferManager::pinned( - ITensor::makeShape({maxNumSequences, maxDecodingTokens}), nvinfer1::DataType::kINT32); - medusaTreeIdsHost = runtime::BufferManager::pinned( - ITensor::makeShape({maxNumSequences, maxMedusaTokens}), nvinfer1::DataType::kINT32); - medusaPathsHost = runtime::BufferManager::pinned( - ITensor::makeShape({maxNumSequences, maxDecodingTokens, maxPathLen}), nvinfer1::DataType::kINT32); - - TensorPtr medusaPositionOffsetsHostSlice = ITensor::slice(medusaPositionOffsetsHost, 0, 1); - medusaPositionOffsetsHostSlice->squeeze(0); - TensorPtr medusaTreeIdsHostSlice = ITensor::slice(medusaTreeIdsHost, 0, 1); - medusaTreeIdsHostSlice->squeeze(0); - TensorPtr medusaPathsHostSlice = ITensor::slice(medusaPathsHost, 0, 1); - medusaPathsHostSlice->squeeze(0); - TensorPtr attentionPackedMaskHostSlice = ITensor::slice(attentionPackedMaskHost, 0, 1); - attentionPackedMaskHostSlice->squeeze(0); - - // Init buffers for 1 request - auto const& choices = decodingConfig.getMedusaChoices().value_or(medusaModule->getMedusaChoices()); - runtime::utils::initTensorsFromChoices(*medusaModule, choices, mTopKs, medusaGenerationLengthsHost, - medusaPositionOffsetsHostSlice, medusaTreeIdsHostSlice, medusaPathsHostSlice, attentionPackedMaskHostSlice); - - auto scatterToBatch = [maxBatchSize, &manager](TensorPtr& data) - { - auto srcSlice = ITensor::slice(data, 0, 1); - // Populate data from the 1st request to the other requests in the batch - for (SizeType32 bi = 1; bi < maxBatchSize; ++bi) - { - auto dstSlice = ITensor::slice(data, bi, 1); - manager.copy(*srcSlice, *dstSlice); - } - }; - - scatterToBatch(medusaPositionOffsetsHost); - scatterToBatch(medusaTreeIdsHost); - scatterToBatch(medusaPathsHost); - scatterToBatch(attentionPackedMaskHost); - - // Copy buffers to device - // 1st dimension of packed mask is num_total_generation_tokens now (packed without paddings). - attentionPackedMaskHost->reshape(ITensor::makeShape({maxNumSequences * maxDecodingTokens, numPackedMasks})); - attentionPackedMaskDevice = manager.copyFrom(*attentionPackedMaskHost, runtime::MemoryType::kGPU); - medusaGenerationLengthsDevice = manager.copyFrom(*medusaGenerationLengthsHost, runtime::MemoryType::kGPU); - medusaPositionOffsetsDevice = manager.copyFrom(*medusaPositionOffsetsHost, runtime::MemoryType::kGPU); - medusaTreeIdsDevice = manager.copyFrom(*medusaTreeIdsHost, runtime::MemoryType::kGPU); - medusaPathsDevice = manager.copyFrom(*medusaPathsHost, runtime::MemoryType::kGPU); - - // use speculative decoding buffer - medusaUseSpecDecoding = manager.cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - runtime::bufferCast<SizeType32>(*medusaUseSpecDecoding)[0] = 1; - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - void MedusaBuffers::reshape(SizeType32 /* numCtxSequences */, SizeType32 numGenSequences, SizeType32 tokensPerStep) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); diff --git a/cpp/tensorrt_llm/batch_manager/microBatchScheduler.cpp b/cpp/tensorrt_llm/batch_manager/microBatchScheduler.cpp index 3e8fca0be052..2807f33463a1 100644 --- a/cpp/tensorrt_llm/batch_manager/microBatchScheduler.cpp +++ b/cpp/tensorrt_llm/batch_manager/microBatchScheduler.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -230,8 +230,9 @@ void MicroBatchScheduler::setCtxRequestsChunkSize<MicroBatchScheduler::ContextCh // Assigns chunk sizes to context requests under the kFORCE_CHUNK policy. // -// Every request is assigned exactly min(contextRemainingLength, chunkUnitSize) tokens. -// Requests whose chunk would push the running total past ctxTokensCapacity are zeroed. +// Requests with expected snapshot points advance to their next absolute snapshot position. +// Otherwise, every request consumes its full remaining context. +// Capacity and context-length truncation are rounded down to a chunk-unit boundary. // // This policy is designed for linear attention state caching, so reusable KV-cache tokens are NOT // calculated because it's not supported yet. @@ -248,16 +249,36 @@ void MicroBatchScheduler::setCtxRequestsChunkSize<MicroBatchScheduler::ContextCh SizeType32 totalTokens{0}; for (auto& llmReq : contextsToBeChunked) { - SizeType32 const chunkSize = std::min(llmReq->getContextRemainingLength(), chunkUnitSize); - if (ctxTokensCapacity && totalTokens + chunkSize > ctxTokensCapacity.value()) + SizeType32 chunkSize = llmReq->getContextRemainingLength(); + auto const& expectedSnapshotPoints = llmReq->getExpectedSnapshotPoints(); + if (!expectedSnapshotPoints.empty()) + { + auto const currentPosition = llmReq->getContextCurrentPosition(); + std::optional<SizeType32> nextSnapshotPoint; + for (auto const point : expectedSnapshotPoints) + { + if (point > currentPosition && (!nextSnapshotPoint || point < nextSnapshotPoint.value())) + { + nextSnapshotPoint = point; + } + } + chunkSize = nextSnapshotPoint + ? std::max<SizeType32>(0, std::min(nextSnapshotPoint.value(), llmReq->getPromptLen()) - currentPosition) + : llmReq->getContextRemainingLength(); + } + + if (maxContextLength && chunkSize > maxContextLength.value()) { - llmReq->setContextChunkSize(0); + chunkSize = maxContextLength.value() / chunkUnitSize * chunkUnitSize; } - else + if (ctxTokensCapacity && totalTokens + chunkSize > ctxTokensCapacity.value()) { - llmReq->setContextChunkSize(chunkSize); - totalTokens += llmReq->getContextChunkSize(); + auto const remainingCapacity = std::max<SizeType32>(0, ctxTokensCapacity.value() - totalTokens); + chunkSize = std::min(chunkSize, remainingCapacity) / chunkUnitSize * chunkUnitSize; } + + llmReq->setContextChunkSize(chunkSize); + totalTokens += llmReq->getContextChunkSize(); } } @@ -267,8 +288,9 @@ void MicroBatchScheduler::setCtxRequestsChunkSize<MicroBatchScheduler::ContextCh // kEQUAL_PROGRESS — all requests advance together one chunkUnitSize at a time. // kFIRST_COME_FIRST_SERVED — requests are served greedily in order until the budget // is exhausted. -// kFORCE_CHUNK — every request gets exactly min(remaining, chunkUnitSize) -// tokens; budget is charged at face value (no reuse discount). +// kFORCE_CHUNK — requests advance to the next expected snapshot point, or consume +// the remaining context when none are configured; budget is charged +// at face value (no reuse discount). // // EQUAL_PROGRESS and FIRST_COME_FIRST_SERVED are compute-aware: tokens covered by the // reusable KV-cache prefix are not charged against ctxTokensCapacity. @@ -436,7 +458,7 @@ std::tuple<RequestVector, RequestVector> MicroBatchScheduler::operator()(Request allContextRequestsFit = false; } - // For FORCE_CHUNK policy, always re-chunk regardless of whether all contexts fit. + // FORCE_CHUNK must always run boundary selection even when all contexts fit. if (mCtxChunkConfig && mCtxChunkConfig.value().chunkingPolicy == ContextChunkingPolicy::kFORCE_CHUNK) { allContextRequestsFit = false; diff --git a/cpp/tensorrt_llm/batch_manager/mlaCacheFormatter.cpp b/cpp/tensorrt_llm/batch_manager/mlaCacheFormatter.cpp index 24b7182fe628..2631df9aa9a2 100644 --- a/cpp/tensorrt_llm/batch_manager/mlaCacheFormatter.cpp +++ b/cpp/tensorrt_llm/batch_manager/mlaCacheFormatter.cpp @@ -124,9 +124,13 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses { NVTX3_SCOPED_RANGE(MLACacheFormatter_format); session.setTime(TransferSession::kTimeFormatter); - auto const& llmRequest = session.getLlmRequest(); - TLLM_LOG_DEBUG( - mpi::MpiComm::world().getRank(), "Start sending KV cache for request ID: %ld.", llmRequest.mRequestId); + auto llmRequest = session.getLlmRequest(); + if (llmRequest.has_value()) + { + TLLM_LOG_DEBUG( + mpi::MpiComm::world().getRank(), "Start sending KV cache for request ID: %ld.", (*llmRequest)->mRequestId); + TLLM_CHECK_WITH_INFO((*llmRequest)->mSamplingConfig.beamWidth == 1, "Currently only supports beam width 1."); + } auto const& selfConfig = session.getSelfState().getCacheState().value(); auto const& destConfig = session.getOtherState().getCacheState().value(); auto const selfIdx = session.getSelfState().getCommState().value().getSelfIdx(); @@ -134,7 +138,6 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses auto const& lastBlockKey = session.getLastBlockKey(); auto const& connections = session.getConnections(); auto& bufferManager = session.getBufferManager(); - TLLM_CHECK_WITH_INFO(llmRequest.mSamplingConfig.beamWidth == 1, "Currently only supports beam width 1."); TLLM_CHECK(!connections.empty()); if (!needSendCache(selfConfig, destConfig, selfIdx)) { @@ -145,7 +148,6 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses auto targetNum = pickUpConnections.size(); if (targetNum == 0) { - TLLM_LOG_DEBUG("No targets to send KV cache to for request ID: %ld", llmRequest.mRequestId); return; } @@ -219,8 +221,11 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses } } - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", - llmRequest.mRequestId); + if (llmRequest.has_value()) + { + TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", + (*llmRequest)->mRequestId); + } return; } @@ -412,15 +417,20 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses session.setTime(TransferSession::kTimeTransmissions); session.setTime(TransferSession::kTimePostprocess); - TLLM_LOG_DEBUG( - mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", llmRequest.mRequestId); + if (llmRequest.has_value()) + { + TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", + (*llmRequest)->mRequestId); + } } void MLACacheFormatter::unformat(tensorrt_llm::batch_manager::TransferSession& session) { NVTX3_SCOPED_RANGE(MLACacheFormatter_unformat); session.setTime(TransferSession::kTimeFormatter); - auto const& llmRequest = session.getLlmRequest(); + auto llmRequestOpt = session.getLlmRequest(); + TLLM_CHECK_WITH_INFO(llmRequestOpt.has_value(), "LlmRequest required for receiving KV cache"); + auto const& llmRequest = **llmRequestOpt; TLLM_CHECK_WITH_INFO(llmRequest.mSamplingConfig.beamWidth == 1, "Currently only supports beam width 1."); auto const ctxReqId = llmRequest.getContextPhaseParams().value().getReqId(); TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), diff --git a/cpp/tensorrt_llm/batch_manager/peftCacheManager.cpp b/cpp/tensorrt_llm/batch_manager/peftCacheManager.cpp index 0bf9a989fd65..89cc475b82ee 100644 --- a/cpp/tensorrt_llm/batch_manager/peftCacheManager.cpp +++ b/cpp/tensorrt_llm/batch_manager/peftCacheManager.cpp @@ -30,7 +30,7 @@ #include "tensorrt_llm/runtime/workerPool.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cstdint> #include <limits> @@ -52,7 +52,8 @@ PeftTaskNotCachedException::PeftTaskNotCachedException(std::string const& msg) PeftTaskNotCachedException::~PeftTaskNotCachedException() noexcept = default; std::pair<uint64_t, uint64_t> PeftCacheManager::getMaxNumSlots(PeftCacheManagerConfig const& config, - nvinfer1::DataType dataType, uint64_t pageWidth, uint64_t max1dModSize, runtime::BufferManager const& bufferManager) + tensorrt_llm::DataType dataType, uint64_t pageWidth, uint64_t max1dModSize, + runtime::BufferManager const& bufferManager) { TLLM_LOG_DEBUG("max1dModeSize=%llu", max1dModSize); TLLM_LOG_DEBUG("pageWidth=%llu", pageWidth); diff --git a/cpp/tensorrt_llm/batch_manager/promptTuningBuffers.cpp b/cpp/tensorrt_llm/batch_manager/promptTuningBuffers.cpp deleted file mode 100644 index 1cf73a2c0d21..000000000000 --- a/cpp/tensorrt_llm/batch_manager/promptTuningBuffers.cpp +++ /dev/null @@ -1,323 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/promptTuningBuffers.h" - -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/common/nvtxUtils.h" - -namespace tensorrt_llm::batch_manager -{ -using SizeType32 = tensorrt_llm::runtime::SizeType32; -using TensorPtr = runtime::ITensor::SharedPtr; - -PromptTuningBuffers::PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) -{ - auto maxPromptEmbeddingTableSize = modelConfig.getMaxPromptEmbeddingTableSize(); - auto const hiddenSize = modelConfig.getHiddenSize() * worldConfig.getTensorParallelism(); - - // vocabSize and mMaxPromptVocabSize - mPromptTuningParams.vocabSize = manager.gpu(runtime::ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - mMaxPromptVocabSize = maxPromptEmbeddingTableSize / maxBatchSize; - - auto promptVocabSizeHost - = runtime::BufferManager::pinned(runtime::ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - auto promptVocabSizeHostData = runtime::bufferCast<SizeType32>(*promptVocabSizeHost); - promptVocabSizeHostData[0] = mMaxPromptVocabSize; - manager.copy(*promptVocabSizeHost, *mPromptTuningParams.vocabSize); - - // embeddingTable - mPromptTuningParams.embeddingTable = manager.gpu( - runtime::ITensor::makeShape({maxPromptEmbeddingTableSize, hiddenSize}), modelConfig.getDataType()); - - // tasks - mPromptTuningParams.tasks = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); -} - -PromptTuningBuffers::PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, bool promptTableOffloading) -{ - auto maxPromptEmbeddingTableSize = modelConfig.getMaxPromptEmbeddingTableSize(); - auto const hiddenSize = modelConfig.getHiddenSize() * worldConfig.getTensorParallelism(); - - // vocabSize and mMaxPromptVocabSize - mPromptTuningParams.vocabSize = manager.gpu(runtime::ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - mMaxPromptVocabSize = maxPromptEmbeddingTableSize / maxBatchSize; - mPromptTableOffloading = promptTableOffloading; - - auto promptVocabSizeHost - = runtime::BufferManager::pinned(runtime::ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - auto promptVocabSizeHostData = runtime::bufferCast<SizeType32>(*promptVocabSizeHost); - promptVocabSizeHostData[0] = mMaxPromptVocabSize; - manager.copy(*promptVocabSizeHost, *mPromptTuningParams.vocabSize); - - // embeddingTable - mPromptTuningParams.embeddingTable = manager.gpu( - runtime::ITensor::makeShape({maxPromptEmbeddingTableSize, hiddenSize}), modelConfig.getDataType()); - - // tasks - mPromptTuningParams.tasks = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); -} - -void PromptTuningBuffers::validate( - std::optional<TensorPtr> const& optReqPromptEmbeddingTable, std::optional<SizeType32> const& optReqPromptVocabSize) -{ - // Need to copy request embeddingTable to promptEmbeddingTable - if (optReqPromptEmbeddingTable.has_value()) - { - - auto reqPromptEmbeddingTable = optReqPromptEmbeddingTable.value(); - auto reqPromptVocabSize = optReqPromptVocabSize.value(); - - if (reqPromptVocabSize > mMaxPromptVocabSize) - { - std::string errStr = "Prompt vocab size" + std::to_string(reqPromptVocabSize) - + " is larger than max prompt vocab size of " + std::to_string(mMaxPromptVocabSize) - + ". Max prompt vocab size is computed from max_prompt_embedding_table_size / max_batch_size. "; - TLLM_LOG_ERROR(errStr); - throw std::runtime_error(errStr); - } - else - { - // Check that type matches model weights - if (reqPromptEmbeddingTable->getDataType() != mPromptTuningParams.embeddingTable->getDataType()) - { - std::string errStr = "Request embedding table data type doesn't match model weight data type."; - TLLM_LOG_ERROR(errStr); - throw std::runtime_error(errStr); - } - - if (reqPromptEmbeddingTable->getShape().d[1] != reqPromptVocabSize) - { - std::string errStr - = "First dimension of request embedding table is expected to be equal to prompt vocab size"; - TLLM_LOG_ERROR(errStr); - throw std::runtime_error(errStr); - } - } - } -} - -void PromptTuningBuffers::fill(RequestVector const& contextRequests, RequestVector const& genRequests, - runtime::BufferManager const& manager, bool packed) -{ - NVTX3_SCOPED_RANGE_WITH_NAME(range, "PromptTuningBuffers::fill"); - - auto const numContextRequests = static_cast<SizeType32>(contextRequests.size()); - - std::vector<SizeType32> reqBeamWidths; - std::vector<SizeType32> reqPromptLengths; - mPromptTuningParams.promptTuningEnabled.clear(); - - SizeType32 batchIdx{0}; - for (auto const& requests : {contextRequests, genRequests}) - { - for (auto const& llmReq : requests) - { - reqBeamWidths.push_back(llmReq->mSamplingConfig.beamWidth); - if (batchIdx < numContextRequests) - { - SizeType32 numContextTokens = 0; - auto const draftLength = llmReq->isLastContextChunk() ? llmReq->getNumDraftTokens() : 0; - auto const contextChunkSize = llmReq->getContextChunkSize(); - numContextTokens += contextChunkSize + draftLength; - reqPromptLengths.push_back(numContextTokens); - } - - std::optional<TensorPtr> optReqPromptEmbeddingTable = std::nullopt; - std::optional<SizeType32> optReqPromptVocabSize = std::nullopt; - - if (mPromptTableOffloading) - { - optReqPromptEmbeddingTable = getChunkPtableBuffer(getChunkPtableCurrentIndex()); - optReqPromptVocabSize = getChunkPtableBufferSliceSize(getChunkPtableCurrentIndex(), batchIdx); - } - else - { - optReqPromptEmbeddingTable = llmReq->getPromptEmbeddingTable(); - optReqPromptVocabSize = llmReq->getPromptVocabSize(); - } - - mPromptTuningParams.promptTuningEnabled.push_back(optReqPromptEmbeddingTable.has_value()); - - // If context request & has embedding table, validate it - if (optReqPromptEmbeddingTable.has_value()) - { - // If a context request, validate prompt tensors and move to GPU - if (batchIdx < numContextRequests) - { - if (mPromptTableOffloading) - { - // Need to slice the ptable since we don't need the entire buffer - // The size depends on optReqPromptVocabSize which stores how many fake prompts are in the chunk - auto slicedPtable = runtime::ITensor::slice( - optReqPromptEmbeddingTable.value(), 0, optReqPromptVocabSize.value()); - slicedPtable->unsqueeze(0); - optReqPromptEmbeddingTable = std::move(slicedPtable); - } - else - { - // Move to GPU - llmReq->movePromptEmbeddingTableToGpu(manager); - optReqPromptEmbeddingTable = llmReq->getPromptEmbeddingTable(); - } - - // Validate the table, prompt_vocab_size - validate(optReqPromptEmbeddingTable, optReqPromptVocabSize); - } - - auto const reqPromptEmbeddingTable = optReqPromptEmbeddingTable.value(); - auto const reqPromptVocabSize = optReqPromptVocabSize.value(); - - // TODO: Use invokeCopyBatch to avoid multiple bs1 copies - // Copy into large prompt embedding table - TensorPtr reqPromptEmbeddingTableView = runtime::ITensor::view(reqPromptEmbeddingTable); - reqPromptEmbeddingTableView->squeeze(0); - auto const promptEmbeddingTableSlice = runtime::ITensor::slice( - mPromptTuningParams.embeddingTable, batchIdx * mMaxPromptVocabSize, reqPromptVocabSize); - manager.copy(*reqPromptEmbeddingTable, *promptEmbeddingTableSlice); - // TODO: src: 2007040 (llmReq->getPromptEmbeddingTable()) != dst: 1003520 (reqPromptVocabSize) - // (original shape passed from - // python == 196 * 5120, fp16) - // VILA mode 1 , 2 images in one request - } - ++batchIdx; - } - } - - auto const batchSize = batchIdx; - std::vector<SizeType32> tasksHostVec(batchSize); - std::iota(tasksHostVec.begin(), tasksHostVec.end(), 0); - - // Create a tensor that wraps the vector and convert unique_ptr to shared_ptr - auto tasksHost = std::shared_ptr<runtime::ITensor>( - runtime::ITensor::wrap(tasksHostVec, runtime::ITensor::makeShape({batchSize})).release()); - - mPromptTuningParams.fillTasksTensor( - tasksHost, batchSize, numContextRequests, reqBeamWidths, reqPromptLengths, manager, packed); -} - -void PromptTuningBuffers::initializeChunkPtableBuffers(runtime::BufferManager const& manager, - runtime::ModelConfig const& modelConfig, SizeType32 contextChunkSize, std::shared_ptr<LlmRequest> const& llmReq) -{ - if (mChunkPtableInitialized) - { - return; - } - - std::array<TensorPtr, 2> buffers; - std::vector<std::vector<SizeType32>> startPositions(2); - for (int i = 0; i < 2; i++) - { - startPositions[i].emplace_back(0); - auto memType = llmReq->getPromptEmbeddingTable().value()->getDataType(); - buffers[i] = manager.gpu(runtime::ITensor::makeShape({contextChunkSize, modelConfig.getHiddenSize()}), memType); - } - - mChunkPtableBuffers = std::move(buffers); - mChunkPtableBufferStartPositions = std::move(startPositions); - - mChunkPtableCurrentIndex = 0; - mChunkPtableInitialized = true; -} - -void PromptTuningBuffers::switchChunkPtableBuffer() -{ - mChunkPtableCurrentIndex = 1 - mChunkPtableCurrentIndex; - clearBufferStartPositions(mChunkPtableCurrentIndex); -} - -size_t PromptTuningBuffers::getChunkPtableCurrentIndex() -{ - return mChunkPtableCurrentIndex; -} - -TensorPtr& PromptTuningBuffers::getChunkPtableBuffer(size_t index) -{ - if (!mChunkPtableBuffers.has_value()) - { - TLLM_THROW("Chunk ptable buffers not initialized"); - } - if (!mChunkPtableBuffers.value()[index]) - { - TLLM_THROW("Chunk ptable buffer at index %zu is null", index); - } - return mChunkPtableBuffers.value()[index]; -} - -SizeType32 PromptTuningBuffers::getChunkPtableBufferSliceSize(size_t index, size_t batchIdx) -{ - if (!mChunkPtableBufferStartPositions.has_value()) - { - return 0; - } - - if (batchIdx + 1 >= mChunkPtableBufferStartPositions.value()[index].size()) - { - TLLM_THROW("Batch index %zu + 1 out of bounds for buffer %zu (size: %zu)", batchIdx, index, - mChunkPtableBufferStartPositions.value()[index].size()); - } - - return mChunkPtableBufferStartPositions.value()[index][batchIdx + 1] - - mChunkPtableBufferStartPositions.value()[index][batchIdx]; -} - -SizeType32 PromptTuningBuffers::getChunkPtableBufferStartPosition(size_t index, size_t batchIdx) -{ - if (!mChunkPtableBufferStartPositions.has_value()) - { - return 0; - } - - if (batchIdx >= mChunkPtableBufferStartPositions.value()[index].size()) - { - TLLM_THROW("Batch index %zu out of bounds for buffer %zu (size: %zu)", batchIdx, index, - mChunkPtableBufferStartPositions.value()[index].size()); - } - - // For first batch, return the value directly - if (batchIdx == 0) - { - return mChunkPtableBufferStartPositions.value()[index][0]; - } - - // For other batches, return difference from previous position - return mChunkPtableBufferStartPositions.value()[index][batchIdx] - - mChunkPtableBufferStartPositions.value()[index][batchIdx - 1]; -} - -void PromptTuningBuffers::updateBufferStartPosition(size_t index, SizeType32 numRows) -{ - if (!mChunkPtableBufferStartPositions.has_value()) - { - return; - } - auto& positions = mChunkPtableBufferStartPositions.value()[index]; - positions.push_back(positions.back() + numRows); -} - -void PromptTuningBuffers::clearBufferStartPositions(size_t index) -{ - if (mChunkPtableBufferStartPositions.has_value()) - { - mChunkPtableBufferStartPositions.value()[index].clear(); - mChunkPtableBufferStartPositions.value()[index].emplace_back(0); - } -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/rnnCacheFormatter.cpp b/cpp/tensorrt_llm/batch_manager/rnnCacheFormatter.cpp index c58733f25885..b7ceb727efa3 100644 --- a/cpp/tensorrt_llm/batch_manager/rnnCacheFormatter.cpp +++ b/cpp/tensorrt_llm/batch_manager/rnnCacheFormatter.cpp @@ -47,7 +47,9 @@ void RnnCacheFormatter::format(TransferSession& session) NVTX3_SCOPED_RANGE(RnnCacheFormatter_formatUnifiedPool); session.setTime(TransferSession::kTimeFormatter); - auto const& llmRequest = session.getLlmRequest(); + auto llmRequestOpt = session.getLlmRequest(); + TLLM_CHECK_WITH_INFO(llmRequestOpt.has_value(), "LlmRequest required for RNN state transfer"); + auto const& llmRequest = **llmRequestOpt; TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "Start sending unified pool RNN state for request ID: %ld.", llmRequest.mRequestId); TLLM_CHECK_WITH_INFO(llmRequest.mSamplingConfig.beamWidth == 1, "Currently, only beam width 1 is supported."); @@ -79,7 +81,7 @@ void RnnCacheFormatter::format(TransferSession& session) bool const recvSideHasCP = destConfig.getParallelConfig().mContextParallelism > 1; auto const indexFromEnd = session.getIndexFromEnd(); auto blockRange = kv_cache_manager::getBlockRangeForSending( - mKvCacheManager, llmRequest, lastBlockKey, indexFromEnd, recvSideHasCP, ppSize); + mKvCacheManager, llmRequestOpt, lastBlockKey, indexFromEnd, recvSideHasCP, ppSize); auto const& blockIdsPerWindow = blockRange.getBlockIdsPerWindow(); auto const allWindowSizes = blockRange.getWindowSizes(); @@ -251,7 +253,9 @@ void RnnCacheFormatter::unformat(TransferSession& session) NVTX3_SCOPED_RANGE(RnnCacheFormatter_unformatUnifiedPool); session.setTime(TransferSession::kTimeFormatter); - auto const& llmRequest = session.getLlmRequest(); + auto llmRequestOpt = session.getLlmRequest(); + TLLM_CHECK_WITH_INFO(llmRequestOpt.has_value(), "LlmRequest required for RNN state transfer"); + auto const& llmRequest = **llmRequestOpt; TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "Start receiving unified pool RNN state for request ID: %ld.", llmRequest.mRequestId); TLLM_CHECK_WITH_INFO(llmRequest.mSamplingConfig.beamWidth == 1, "Currently, only beam width 1 is supported."); diff --git a/cpp/tensorrt_llm/batch_manager/rnnCacheTransBuffer.cpp b/cpp/tensorrt_llm/batch_manager/rnnCacheTransBuffer.cpp index 37af8e31baf9..f9c04200e8d2 100644 --- a/cpp/tensorrt_llm/batch_manager/rnnCacheTransBuffer.cpp +++ b/cpp/tensorrt_llm/batch_manager/rnnCacheTransBuffer.cpp @@ -21,6 +21,7 @@ #include "tensorrt_llm/common/dataType.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include <algorithm> @@ -95,7 +96,7 @@ size_t RnnCacheTransBufferManager::computeTransferBufferSizeFromPool( RnnCacheTransBufferManager::RnnCacheTransBufferManager(kv_cache_manager::BaseKVCacheManager* kvCacheManager, executor::kv_cache::CacheState const& cacheState, std::optional<size_t> maxNumTokens) : BaseTransBufferManager(computeTransferBufferSizeFromPool(kvCacheManager, cacheState, maxNumTokens), - nvinfer1::DataType::kUINT8, maxNumTokens) + tensorrt_llm::DataType::kUINT8, maxNumTokens) { TLLM_CHECK(kvCacheManager != nullptr); TLLM_LOG_INFO("RnnCacheTransBufferManager created for unified pool RNN cache"); diff --git a/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.cpp b/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.cpp deleted file mode 100644 index 6fc7977ef8f1..000000000000 --- a/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "rnnStateBuffers.h" - -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/batch_manager/rnnStateManager.h" -#include "tensorrt_llm/batch_manager/runtimeBuffers.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" - -using namespace tensorrt_llm::runtime; - -namespace tensorrt_llm::batch_manager -{ - -RnnStateBuffers::RnnStateBuffers(SizeType32 maxBatchSize, runtime::TllmRuntime const& runtime) -{ - auto const& manager = runtime.getBufferManager(); - - slotMappingHost = BufferManager::cpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - slotMappingDevice = manager.gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); -} - -void RnnStateBuffers::reshape(SizeType32 numSequences) -{ - slotMappingHost->reshape(ITensor::makeShape({numSequences})); - slotMappingDevice->reshape(ITensor::makeShape({numSequences})); -} - -void RnnStateBuffers::fillSlotMappings( - RequestVector const& contextRequests, rnn_state_manager::RnnStateManager* rnnStateManager) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(rnnStateBuffersFillSlotMappings); - - SizeType32 batchIdx{0}; - for (auto const& llmReq : contextRequests) - { - auto const seqSlot = llmReq->mSeqSlot.value(); - auto const reqBeamWidth = llmReq->mSamplingConfig.beamWidth; - rnnStateManager->fillSlotMapping(*slotMappingHost, batchIdx, seqSlot, reqBeamWidth); - ++batchIdx; - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void RnnStateBuffers::copySlotMappingH2D(runtime::TllmRuntime const& runtime) -{ - auto const& manager = runtime.getBufferManager(); - manager.copy(*slotMappingHost, *slotMappingDevice); -} - -void RnnStateBuffers::getBuffers(TensorMap& inputBuffers) const -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(rnnStateBuffersGetBuffers); - - inputBuffers.insert_or_assign("slot_mapping", slotMappingDevice); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.h b/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.h deleted file mode 100644 index e25df47382a1..000000000000 --- a/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/runtime/iTensor.h" - -namespace tensorrt_llm::runtime -{ -class TllmRuntime; -} // namespace tensorrt_llm::runtime - -namespace tensorrt_llm::batch_manager -{ - -namespace rnn_state_manager -{ -class RnnStateManager; -} - -class RnnStateBuffers -{ -public: - using SizeType32 = tensorrt_llm::runtime::SizeType32; - using TensorPtr = runtime::ITensor::SharedPtr; - using TensorMap = runtime::StringPtrMap<runtime::ITensor>; - - // others should be in rnnStateManager, we only need slotMapping here. - TensorPtr slotMappingHost; // [batch_size] - TensorPtr slotMappingDevice; // [batch_size] - - RnnStateBuffers(SizeType32 maxBatchSize, runtime::TllmRuntime const& runtime); - - void reshape(SizeType32 numSequences); - - void fillSlotMappings(RequestVector const& contextRequests, rnn_state_manager::RnnStateManager* rnnStateManager); - - void copySlotMappingH2D(runtime::TllmRuntime const& runtime); - - void getBuffers(TensorMap& inputBuffers) const; -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/rnnStateManager.cpp b/cpp/tensorrt_llm/batch_manager/rnnStateManager.cpp index 7608079fb396..7d032a268fdd 100644 --- a/cpp/tensorrt_llm/batch_manager/rnnStateManager.cpp +++ b/cpp/tensorrt_llm/batch_manager/rnnStateManager.cpp @@ -17,6 +17,7 @@ #include "tensorrt_llm/batch_manager/rnnStateManager.h" #include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/utils/runtimeUtils.h" @@ -80,7 +81,7 @@ RnnStateManager::RnnStateManager(SizeType32 maxNumSequences, tensorrt_llm::runti {localNbLayers, mMaxNumSequences * mBeamSlotsPerSequence, convKernel - 1, rnnConvDimSize}); mDtype = dataType; - mSsmCacheDtype = nvinfer1::DataType::kFLOAT; + mSsmCacheDtype = tensorrt_llm::DataType::kFLOAT; // Store RNN model config for CacheTransceiver mDState = stateSize; @@ -117,7 +118,7 @@ RnnStateManager::RnnStateManager(SizeType32 maxNumSequences, tensorrt_llm::runti RnnStateManager::RnnStateManager(SizeType32 dState, SizeType32 dConv, SizeType32 numHeads, SizeType32 nGroups, SizeType32 headDim, SizeType32 maxBatchSize, WorldConfig const& worldConfig, int64_t stream, - nvinfer1::DataType dtype, nvinfer1::DataType ssmCacheDtype, std::vector<SizeType32> const& ppLayers, + tensorrt_llm::DataType dtype, tensorrt_llm::DataType ssmCacheDtype, std::vector<SizeType32> const& ppLayers, SizeType32 numLayers) : mMaxNumSequences(maxBatchSize) , mMaxBeamWidth{1} @@ -297,12 +298,12 @@ RnnStateManager::TensorPtr RnnStateManager::getSsmStates() const return pagedRnnStates; } -nvinfer1::DataType RnnStateManager::getConvStateDataType() const noexcept +tensorrt_llm::DataType RnnStateManager::getConvStateDataType() const noexcept { return mDtype; } -nvinfer1::DataType RnnStateManager::getSsmStateDataType() const noexcept +tensorrt_llm::DataType RnnStateManager::getSsmStateDataType() const noexcept { return mSsmCacheDtype; } diff --git a/cpp/tensorrt_llm/batch_manager/runtimeBuffers.cpp b/cpp/tensorrt_llm/batch_manager/runtimeBuffers.cpp deleted file mode 100644 index ea5b9b06a96e..000000000000 --- a/cpp/tensorrt_llm/batch_manager/runtimeBuffers.cpp +++ /dev/null @@ -1,1029 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/runtimeBuffers.h" - -#include "tensorrt_llm/batch_manager/encoderBuffers.h" -#include "tensorrt_llm/batch_manager/kvCacheManager.h" -#include "tensorrt_llm/batch_manager/loraBuffers.h" -#include "tensorrt_llm/batch_manager/medusaBuffers.h" -#include "tensorrt_llm/batch_manager/promptTuningBuffers.h" -#include "tensorrt_llm/batch_manager/rnnStateBuffers.h" -#include "tensorrt_llm/batch_manager/rnnStateManager.h" -#include "tensorrt_llm/batch_manager/transformerBuffers.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/common/stlUtils.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/decoderState.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/runtimeKernels.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" - -#include <algorithm> -#include <iterator> -#include <memory> -#include <numeric> -#include <vector> - -using namespace tensorrt_llm::runtime; - -namespace tensorrt_llm::batch_manager -{ - -RuntimeBuffers::RuntimeBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, - TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig, - executor::DecodingConfig const& decodingConfig, bool gatherGenerationLogits, std::optional<SizeType32> maxNumTokens, - std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs, - bool promptTableOffloadingParam) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - promptTableOffloading = promptTableOffloadingParam; - - create(maxBatchSize, maxBeamWidth, maxAttentionWindowVec, maxAttentionWindow, sinkTokenLen, runtime, modelConfig, - worldConfig, decodingConfig, gatherGenerationLogits, additionalModelOutputs); - - // pre-allocate - setMaxBufferSizes(maxBatchSize, maxBeamWidth, modelConfig, maxNumTokens); - reshape(runtime, modelConfig, worldConfig, gatherGenerationLogits); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -RuntimeBuffers::~RuntimeBuffers() = default; - -void RuntimeBuffers::create(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, - TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig, - executor::DecodingConfig const& decodingConfig, bool gatherGenerationLogits, - std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const& manager = runtime.getBufferManager(); - auto const& engine = runtime.getEngine(); - - if (modelConfig.isTransformerBased()) - { - transformerBuffers = std::make_unique<TransformerBuffers>(maxBatchSize, maxBeamWidth, maxAttentionWindowVec, - maxAttentionWindow, sinkTokenLen, runtime, modelConfig, worldConfig); - } - if (modelConfig.isRnnBased()) - { - rnnStateBuffers = std::make_unique<RnnStateBuffers>(maxBatchSize, runtime); - } - - auto constexpr nvTokenIdType = TRTDataType<TokenIdType>::value; - inputsIds = manager.emptyTensor(MemoryType::kGPU, nvTokenIdType); - - mropeRotaryCosSin = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kFLOAT); - mropePositionDeltas = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - - if (worldConfig.isLastPipelineParallelRank()) - { - auto const logitsType = engine.getTensorDataType(batch_manager::RuntimeBuffers::kLogitsTensorName); - logits = manager.emptyTensor(MemoryType::kGPU, logitsType); - } - - // TODO: check which tensors can be allocated as pinned for max size - requestTypes = manager.emptyTensor(MemoryType::kCPU, TRTDataType<runtime::RequestType>::value); - - contextLengthsHost = manager.emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kINT32); - contextLengthsDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - sequenceLengthsHost = manager.emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kINT32); - sequenceLengthsDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - - lastTokenIdsHost = manager.emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kINT32); - lastTokenIdsDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - logitsIdsHost = manager.emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kINT32); - - inputsIds = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - - if (worldConfig.isPipelineParallel()) - { - hiddenStates = manager.emptyTensor(MemoryType::kGPU, modelConfig.getDataType()); - } - - auto const maxBatchSizeShape = ITensor::makeShape({maxBatchSize}); - seqSlots = tensorrt_llm::runtime::BufferManager::pinnedPool(maxBatchSizeShape, nvinfer1::DataType::kINT32); - seqSlotsDevice = manager.gpu(maxBatchSizeShape, nvinfer1::DataType::kINT32); - - cacheIndirDecoderIOBatchedCopySrcOffsets - = tensorrt_llm::runtime::BufferManager::pinnedPool(maxBatchSizeShape, nvinfer1::DataType::kINT64); - cacheIndirDecoderIOBatchedCopyDstOffsets - = tensorrt_llm::runtime::BufferManager::pinnedPool(maxBatchSizeShape, nvinfer1::DataType::kINT64); - cacheIndirDecoderIOBatchedCopySizes - = tensorrt_llm::runtime::BufferManager::pinnedPool(maxBatchSizeShape, nvinfer1::DataType::kINT64); - mCacheIndirDecoderIOBatchedCopySrcOffsetsSliceDevice = manager.gpu(maxBatchSizeShape, nvinfer1::DataType::kINT64); - mCacheIndirDecoderIOBatchedCopyDstOffsetsSliceDevice = manager.gpu(maxBatchSizeShape, nvinfer1::DataType::kINT64); - mCacheIndirDecoderIOBatchedCopyCopySizesDevice = manager.gpu(maxBatchSizeShape, nvinfer1::DataType::kINT64); - - // Pre-allocate buffer for saving generation logits for model w/o draft tokens - if (gatherGenerationLogits - && (modelConfig.getSpeculativeDecodingMode().isDraftTokensExternal() - || modelConfig.getSpeculativeDecodingMode().isNone()) - && worldConfig.isLastPipelineParallelRank()) - { - auto const vocabSizePadded = modelConfig.getVocabSizePadded(worldConfig.getSize()); - auto const logitsType = engine.getTensorDataType(batch_manager::RuntimeBuffers::kLogitsTensorName); - - generationLogitsCache.transposedLogits = manager.gpu( - ITensor::makeShape({maxBeamWidth, GenerationLogitsCache::kCACHE_LENGTH, vocabSizePadded}), logitsType); - generationLogitsCache.logits = manager.gpu( - ITensor::makeShape({GenerationLogitsCache::kCACHE_LENGTH, maxBatchSize * maxBeamWidth, vocabSizePadded}), - logitsType); - - generationLogitsCache.fragmentPointerDevice = manager.gpu( - ITensor::makeShape({maxBatchSize, GenerationLogitsCache::kCACHE_LENGTH}), nvinfer1::DataType::kINT64); - generationLogitsCache.fragmentPointerHost = tensorrt_llm::runtime::BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, GenerationLogitsCache::kCACHE_LENGTH}), nvinfer1::DataType::kINT64); - } - - if (modelConfig.useCrossAttention()) - { - encoderBuffers = std::make_unique<EncoderBuffers>(); - encoderBuffers->create(maxBatchSize, modelConfig, runtime); - } - - if (modelConfig.usePromptTuning()) - { - promptTuningBuffers = std::make_unique<PromptTuningBuffers>( - maxBatchSize, manager, modelConfig, worldConfig, promptTableOffloading); - } - - if (modelConfig.useLoraPlugin()) - { - loraBuffers = std::make_unique<LoraBuffers>(maxBatchSize, maxBeamWidth, runtime, modelConfig, worldConfig); - } - - if (modelConfig.getSpeculativeDecodingMode().isMedusa()) - { - mMedusaBuffers = std::make_unique<MedusaBuffers>( - maxBatchSize, maxBeamWidth, manager, modelConfig, worldConfig, decodingConfig, runtime); - } - else if (modelConfig.getSpeculativeDecodingMode().isLookaheadDecoding()) - { - mLookaheadBuffers = std::make_unique<runtime::LookaheadRuntimeBuffers>( - maxBatchSize, maxBeamWidth, manager, modelConfig, worldConfig, decodingConfig, runtime); - } - else if (modelConfig.getSpeculativeDecodingMode().isExplicitDraftTokens()) - { - mExplicitDraftTokensBuffers = std::make_unique<runtime::ExplicitDraftTokensBuffers>( - maxBatchSize, maxBeamWidth, manager, modelConfig, worldConfig); - } - else if (modelConfig.getSpeculativeDecodingMode().isEagle()) - { - mEagleBuffers = std::make_unique<runtime::EagleBuffers>( - maxBatchSize, maxBeamWidth, manager, modelConfig, worldConfig, decodingConfig); - } - - if (modelConfig.useLanguageAdapter()) - { - languageAdapterRoutings = manager.emptyTensor(MemoryType::kGPU, TRTDataType<SizeType32>::value); - } - - for (auto const& output : additionalModelOutputs.value_or(std::vector<executor::AdditionalModelOutput>{})) - { - auto const& engine = runtime.getEngine(); - auto const dataType = engine.getTensorDataType(output.name.c_str()); - mAdditionalOutputTensors.emplace(output.name, manager.emptyTensor(runtime::MemoryType::kGPU, dataType)); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void RuntimeBuffers::setMaxBufferSizes(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, - runtime::ModelConfig const& modelConfig, std::optional<SizeType32> maxNumRuntimeTokens) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // `maxNumSequences` is reached when all requests are in generation - numContextRequests = 0; - numGenRequests = maxBatchSize; - numGenSequences = maxBatchSize * maxBeamWidth; - - auto const maxDraftTokens = modelConfig.getMaxDecodingDraftTokens(); - // Draft-Tokens and Beam-Search are mutually exclusive - numLogits = maxBatchSize * std::max(1 + maxDraftTokens, maxBeamWidth); - auto const maxNumModelTokens = modelConfig.getMaxNumTokens(); - auto const maxNumContextTokens = maxBatchSize * modelConfig.getMaxInputLen(); - auto const maxNumGenTokens = numLogits; - // For pre-allocation - numContextTokens = 0; // Set in `setBufferSizes` rather than here for `computeContextLogits` - numGenTokens - = maxNumRuntimeTokens.value_or(maxNumModelTokens.value_or(std::max(maxNumContextTokens, maxNumGenTokens))); - - if (modelConfig.useCrossAttention()) - { - encoderBuffers->setMaxBufferSizes(maxBatchSize, modelConfig); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void RuntimeBuffers::setBufferSizes(RequestVector const& contextRequests, RequestVector const& genRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(runtimeBuffersSetBufferSizes); - - // set context sizes - numContextRequests = static_cast<SizeType32>(contextRequests.size()); - auto numContextLogits = numContextRequests; - numContextTokens = 0; - maxContextLength = 0; - for (auto const& llmReq : contextRequests) - { - auto const draftLength = llmReq->isLastContextChunk() ? llmReq->getNumDraftTokens() : 0; - numContextLogits += draftLength; - - auto const contextChunkSize = llmReq->getContextChunkSize(); - numContextTokens += contextChunkSize + draftLength; - if (maxContextLength < llmReq->mPromptLen) - { - maxContextLength = llmReq->mPromptLen; - } - } - - // set generation sizes - numGenRequests = static_cast<SizeType32>(genRequests.size()); - numGenSequences = 0; - numGenTokens = 0; - for (auto const& llmReq : genRequests) - { - auto const reqBeamWidth = llmReq->getBeamWidthByIter(); - numGenSequences += reqBeamWidth; - auto const draftLen = llmReq->getNumDraftTokens(); - numGenTokens += draftLen + reqBeamWidth; - } - - numLogits = numContextLogits + numGenTokens; - - if (encoderBuffers) - { - encoderBuffers->setBufferSizes(contextRequests, genRequests); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void RuntimeBuffers::reshape(TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig, - bool gatherGenerationLogits) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(runtimeBuffersReshape); - - if (worldConfig.isLastPipelineParallelRank()) - { - auto const vocabSizePadded = modelConfig.getVocabSizePadded(worldConfig.getSize()); - - if (modelConfig.computeContextLogits() && (numContextRequests > 0)) - { - // Only when need to return context logits, and there are new requests will execute context phase, - // logits buffer need to be re-allocated with size of [numContextTokens + numGenSequences, vocabSizePadded] - auto const& engine = runtime.getEngine(); - auto const& manager = runtime.getBufferManager(); - auto const logitsType = engine.getTensorDataType(kLogitsTensorName); - logits = manager.gpu(ITensor::makeShape({numContextTokens + numGenSequences, vocabSizePadded}), logitsType); - } - else if (gatherGenerationLogits && modelConfig.getSpeculativeDecodingMode().isNone()) - { - // If need to return generation logits, re-point the logit buffer to avoid overwrite, - // so we could write back GenerationLogitsCache::kCACHE_LENGTH steps' logits together - // logits shape: [1, maxBatchSize * maxBeamWidth, vocabSizePadded] - // which is large enough to cover both numContextRequests and numGenSequences - logits = ITensor::slice(generationLogitsCache.logits, generationLogitsCache.offset, 1); - generationLogitsCache.offset = (generationLogitsCache.offset + 1) % GenerationLogitsCache::kCACHE_LENGTH; - logits->squeeze(0); - } - else - { - logits->reshape(ITensor::makeShape({numLogits, vocabSizePadded})); - } - } - - auto const numSequences = getNumSequences(); - auto const numSequencesShape = ITensor::makeShape({numSequences}); - requestTypes->reshape(numSequencesShape); - contextLengthsHost->reshape(numSequencesShape); - contextLengthsDevice->reshape(numSequencesShape); - sequenceLengthsHost->reshape(numSequencesShape); - sequenceLengthsDevice->reshape(numSequencesShape); - - auto const numLogitsShape = ITensor::makeShape({numLogits}); - lastTokenIdsHost->reshape(numLogitsShape); - lastTokenIdsDevice->reshape(numLogitsShape); - logitsIdsHost->reshape(numLogitsShape); - - if (transformerBuffers) - { - transformerBuffers->reshape(numSequences, numContextTokens + numGenTokens); - } - - if (rnnStateBuffers) - { - rnnStateBuffers->reshape(numSequences); - } - - if (modelConfig.useCrossAttention()) - { - encoderBuffers->reshape(); - } - - if (modelConfig.useLoraPlugin()) - { - loraBuffers->reshape(numSequences); - } - - if (mMedusaBuffers) - { - mMedusaBuffers->reshape( - numContextRequests, numGenRequests, modelConfig.getSpeculativeDecodingModulePtr()->getMaxDecodingTokens()); - } - - if (mLookaheadBuffers && modelConfig.getSpeculativeDecodingMode().isLookaheadDecoding()) - { - mLookaheadBuffers->reshape( - numContextRequests, numGenRequests, modelConfig.getSpeculativeDecodingModulePtr()->getMaxDecodingTokens()); - } - - if (mExplicitDraftTokensBuffers) - { - mExplicitDraftTokensBuffers->reshape(numContextRequests, numGenRequests, modelConfig); - } - - if (mEagleBuffers) - { - mEagleBuffers->reshape(numContextRequests, numGenRequests, modelConfig); - } - - auto const numRequests = getNumRequests(); - auto const numRequestsShape = ITensor::makeShape({numRequests}); - seqSlots->reshape(numRequestsShape); - seqSlotsDevice->reshape(numRequestsShape); - - auto const numTokens = getNumTokens(); - inputsIds->reshape(ITensor::makeShape({numTokens})); - - if (modelConfig.useMrope()) - { - auto const mropeRotaryCosSinSize = modelConfig.getMaxPositionEmbeddings() * modelConfig.getRotaryEmbeddingDim(); - mropeRotaryCosSin->reshape(ITensor::makeShape({numSequences, mropeRotaryCosSinSize})); - mropePositionDeltas->reshape(ITensor::makeShape({numSequences, 1})); - } - - if (worldConfig.isPipelineParallel()) - { - auto const hiddenSize = (!modelConfig.getPpReduceScatter() || worldConfig.isFirstPipelineParallelRank()) - ? modelConfig.getHiddenSize() * worldConfig.getTensorParallelism() - : modelConfig.getHiddenSize(); - - auto const hiddenStatesShape = ITensor::makeShape({numTokens, hiddenSize}); - hiddenStates->reshape(hiddenStatesShape); - } - - if (modelConfig.useLanguageAdapter()) - { - languageAdapterRoutings->reshape(ITensor::makeShape({numTokens, 1})); - } - - for (auto const& outputTensor : mAdditionalOutputTensors) - { - auto const& [name, tensor] = outputTensor; - auto const& engine = runtime.getEngine(); - auto shape = engine.getTensorShape(name.c_str()); - TLLM_CHECK_WITH_INFO( - shape.d[0] == -1, "First dimension of additional output tensor '%s' must be dynamic", name.c_str()); - shape.d[0] = numTokens; - tensor->reshape(shape); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void RuntimeBuffers::prepareBuffersForCudaGraph(SizeType32 maxSequenceLength) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(prepareBuffersForCudaGraph); - - TLLM_CHECK(numContextRequests == 0); - - if (transformerBuffers) - { - // Set pastKeyValueLength for graph capturing. This way we will capture graph with - // maxKvCacheLengthRounded rounded to the next kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE. - // MMHA will launch excessive amount of blocks and some of them will exit early during the actual launch. - // We can reuse the same graph for the next kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE iterations. - - // make sure the size does not overflow the max allowed pastKvCacheLength - auto const pastKvCacheLength = std::min(maxSequenceLength - 1, maxKvCacheLengthRounded); - - auto* pastKeyValueLengthsPtr = bufferCast<SizeType32>(*transformerBuffers->pastKeyValueLengths); - std::fill_n(pastKeyValueLengthsPtr, getNumSequences(), pastKvCacheLength); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void RuntimeBuffers::setFromInputs(RequestVector const& contextRequests, RequestVector const& genRequests, - SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow, runtime::decoder::DecoderState const& decoderState, - kv_cache_manager::BaseKVCacheManager* kvCacheManagerPtr, - kv_cache_manager::BaseKVCacheManager* crossKvCacheManagerPtr, - rnn_state_manager::RnnStateManager* rnnStateManagerPtr, PeftTable const& peftTable, - runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig, bool trtOverlap, OptionalRef<runtime::ITensor const> newOutputTokens) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(runtimeBuffersSetFromInputs); - - auto const& manager = runtime.getBufferManager(); - auto const& stream = runtime.getStream(); - - // Fill requestTypes - { - auto* hostRequestTypes = bufferCast<runtime::RequestType>(*requestTypes); - std::fill_n(hostRequestTypes, numContextRequests, runtime::RequestType::kCONTEXT); - std::fill_n(hostRequestTypes + numContextRequests, numGenSequences, runtime::RequestType::kGENERATION); - } - - SizeType32 totalInputSize = 0; - std::vector<TokenIdType> inputHost; - std::vector<SizeType32> positionIdsHost; - std::vector<SizeType32> positionIdsHostRow2; - std::vector<SizeType32> mropePositionDeltasHost; - std::vector<SizeType32> languageAdapterRoutingsHost; - - auto* contextLengthsHostPtr = bufferCast<SizeType32>(*contextLengthsHost); - auto* sequenceLengthsHostPtr = bufferCast<SizeType32>(*sequenceLengthsHost); - auto* pastKeyValueLengthsPtr - = transformerBuffers ? bufferCast<SizeType32>(*transformerBuffers->pastKeyValueLengths) : nullptr; - SizeType32 totalNumLogits{0}; - auto* logitsIdsHostPtr = bufferCast<SizeType32>(*logitsIdsHost); - bool const isChatGlm = modelConfig.getModelVariant() == ModelConfig::ModelVariant::kChatGlm; - bool const isGlm = modelConfig.getModelVariant() == ModelConfig::ModelVariant::kGlm; - auto const mropeRotaryCosSinSize = modelConfig.getMaxPositionEmbeddings() * modelConfig.getRotaryEmbeddingDim(); - - { - NVTX3_SCOPED_RANGE(seqSlotsLoop); - auto* seqSlotIndices = bufferCast<SizeType32>(*seqSlots); - - SizeType32 batchIdx{0}; - for (auto const& requests : {contextRequests, genRequests}) - { - for (auto const& llmReq : requests) - { - // Get position of the current sequence in the decoder - auto const seqSlot = llmReq->mSeqSlot.value(); - seqSlotIndices[batchIdx] = seqSlot; - ++batchIdx; - } - } - - TLLM_CHECK(seqSlots->getSize() == static_cast<std::size_t>(batchIdx)); - manager.copy(*seqSlots, *seqSlotsDevice); - } - - // context preparation loop - if (!contextRequests.empty()) - { - NVTX3_SCOPED_RANGE(contextPrepareLoop); - numContextLogits.resize(contextRequests.size()); - - SizeType32 batchIdx{0}; - for (auto const& llmReq : contextRequests) - { - TLLM_CHECK_WITH_INFO(llmReq->isContextInitState() || llmReq->isDisaggGenerationTransmissionComplete(), - "The request should be in context phase or disaggregated generation tranmissionComplete phase."); - TLLM_CHECK_WITH_INFO( - llmReq->getMaxNumGeneratedTokens() == 0, "Context request should not have generated tokens."); - - auto const& reqTokens = llmReq->getTokens(0); - auto const& draftTokens = llmReq->getDraftTokens(); - auto const draftLength = llmReq->getNumDraftTokens(); - auto const& positionIds = llmReq->getPositionIds(); - - auto const contextChunkSize = llmReq->getContextChunkSize(); - auto const beginCompute = llmReq->getContextCurrentPosition(); - auto const endCompute = beginCompute + contextChunkSize; - inputHost.insert(inputHost.end(), reqTokens.begin() + beginCompute, reqTokens.begin() + endCompute); - - logitsIdsHostPtr[totalNumLogits++] = contextChunkSize; - numContextLogits.at(batchIdx) = modelConfig.computeContextLogits() ? contextChunkSize : 1; - - if (llmReq->isLastContextChunk()) - { - inputHost.insert(inputHost.end(), draftTokens->begin(), draftTokens->end()); - std::fill_n(logitsIdsHostPtr + totalNumLogits, draftLength, 1); - totalNumLogits += draftLength; - } - auto const inputLength = contextChunkSize + (llmReq->isLastContextChunk() ? draftLength : 0); - contextLengthsHostPtr[batchIdx] = inputLength; - auto const sequenceLen = inputLength + llmReq->getContextCurrentPosition(); - sequenceLengthsHostPtr[batchIdx] = sequenceLen; - - if (static_cast<bool>(pastKeyValueLengthsPtr)) - { - pastKeyValueLengthsPtr[batchIdx] = beginCompute + inputLength; - } - - if (positionIds.has_value()) - { - TLLM_CHECK_WITH_INFO(!(isChatGlm || isGlm), "ChatGLM-6B and Glm only use the default initialization"); - positionIdsHost.insert(positionIdsHost.end(), positionIds.value()->begin() + beginCompute, - positionIds.value()->begin() + endCompute); - } - else - { - if (isChatGlm) - { - // Specialize for ChatGLM-6B with 2D-Position-Embedding - positionIdsHost.resize(totalInputSize + inputLength); - std::iota(std::begin(positionIdsHost) + totalInputSize, std::end(positionIdsHost), 0); - positionIdsHost.back() = positionIdsHost.back() - 1; - - positionIdsHostRow2.resize(totalInputSize + inputLength); - positionIdsHostRow2.back() = 1; - } - else if (isGlm) - { - // Specialize for GLM-10B with 2D-Position-Embedding and special value of the mask id position - auto start = inputHost.begin() + totalInputSize; - auto end = start + inputLength; - auto it = std::find_if( - start, end, [](SizeType32 id) { return id == 50260 || id == 50263 || id == 50264; }); - llmReq->mMaskPosition = (it != end) ? std::distance(start, it) : maxContextLength; - - positionIdsHost.resize(totalInputSize + inputLength); - std::iota(std::begin(positionIdsHost) + totalInputSize, std::end(positionIdsHost), 0); - positionIdsHost.back() = llmReq->mMaskPosition; - - positionIdsHostRow2.resize(totalInputSize + inputLength); - positionIdsHostRow2.back() = 1; - } - else - { - // Other models - positionIdsHost.resize(totalInputSize + inputLength); - std::iota(std::begin(positionIdsHost) + totalInputSize, - std::begin(positionIdsHost) + totalInputSize + inputLength, beginCompute); - } - } - if (modelConfig.useMrope()) - { - auto optMropeRotaryCosSin = llmReq->getMropeRotaryCosSin().value(); - TLLM_CHECK_WITH_INFO(optMropeRotaryCosSin->getShape().d[0] == mropeRotaryCosSinSize, - "Provided MropeRotarySinCos is %ld and expected is %d.\n", optMropeRotaryCosSin->getShape().d[0], - int(mropeRotaryCosSinSize)); - - auto const mropeRotaryCosSinCtx = ITensor::slice(mropeRotaryCosSin, batchIdx, 1); - manager.copy(*optMropeRotaryCosSin, *mropeRotaryCosSinCtx); - } - - if (modelConfig.useLanguageAdapter()) - { - auto const languageAdapterRouting = llmReq->getLanguageAdapterRouting( - modelConfig.getNumLanguages().value(), endCompute - beginCompute); - languageAdapterRoutingsHost.insert(languageAdapterRoutingsHost.end(), - std::begin(languageAdapterRouting), std::end(languageAdapterRouting)); - } - totalInputSize += inputLength; - ++batchIdx; - } - - if (rnnStateBuffers) - { - rnnStateBuffers->fillSlotMappings(contextRequests, rnnStateManagerPtr); - } - } - - // generation preparation loop - if (!genRequests.empty()) - { - NVTX3_SCOPED_RANGE(genPrepareLoop); - - auto const numContextRequests = static_cast<SizeType32>(contextRequests.size()); - auto numSequences = numContextRequests; - for (auto const& llmReq : genRequests) - { - auto const reqBeamWidth = llmReq->getBeamWidthByIter(); - auto const draftLength = llmReq->getNumDraftTokens(); - auto const& draftTokens = llmReq->getDraftTokens(); - auto const numLogits = draftLength + reqBeamWidth; - TLLM_CHECK(draftLength == 0 || reqBeamWidth == 1); - - auto const promptLen = llmReq->mPromptLen; - auto const sequenceLen - = promptLen + llmReq->getMaxNumGeneratedTokens() + static_cast<SizeType32>(trtOverlap); - auto const& positionIds = llmReq->getPositionIds(); - for (int beam = 0; beam < reqBeamWidth; ++beam) - { - auto const numTokens = llmReq->getNumTokens(beam) + static_cast<SizeType32>(trtOverlap); - // TODO: can this be removed completely? - if (!trtOverlap) - { - auto const lastToken = llmReq->getLastTokens(beam); - inputHost.push_back(lastToken); - if (draftLength > 0) - { - inputHost.insert(inputHost.end(), draftTokens->begin(), draftTokens->end()); - } - } - - // If model updates generation position ids do not append them here. - if (!modelConfig.getSpeculativeDecodingMode().updatesPositionIds()) - { - if (positionIds.has_value()) - { - TLLM_CHECK_WITH_INFO( - !(isChatGlm || isGlm), "ChatGLM-6B and Glm only use the default initialization"); - auto last_context_position_id = positionIds.value()->back(); - positionIdsHost.push_back( - static_cast<SizeType32>(last_context_position_id + sequenceLen - promptLen)); - } - else - { - if (isChatGlm) // ChatGLM-6B - { - positionIdsHost.push_back(static_cast<SizeType32>(promptLen - 2)); - positionIdsHostRow2.push_back(static_cast<SizeType32>(sequenceLen - promptLen + 1)); - } - else if (isGlm) - { - positionIdsHost.push_back(llmReq->mMaskPosition); - positionIdsHostRow2.push_back(static_cast<SizeType32>(sequenceLen - promptLen + 1)); - } - else // GPT / ChatGLM2-6B / ChatGLM3-6B / BART - { - // positionIds is just the size of tokens -1 - positionIdsHost.push_back(numTokens - 1); - } - } - } - - if (modelConfig.useMrope()) - { - auto optMropePositionDeltas = llmReq->getMropePositionDeltas().value(); - mropePositionDeltasHost.push_back(optMropePositionDeltas); - } - - if (modelConfig.useLanguageAdapter()) - { - // Generation requests only have one token per sequence - auto const languageAdapterRouting - = llmReq->getLanguageAdapterRouting(modelConfig.getNumLanguages().value(), 1); - languageAdapterRoutingsHost.insert(languageAdapterRoutingsHost.end(), - std::begin(languageAdapterRouting), std::end(languageAdapterRouting)); - } - } - - if (static_cast<bool>(pastKeyValueLengthsPtr)) - { - SizeType32 pastKeyValueLength = sequenceLen - 1; - std::fill_n(pastKeyValueLengthsPtr + numSequences, reqBeamWidth, pastKeyValueLength); - } - totalInputSize += numLogits; - - std::fill_n(logitsIdsHostPtr + totalNumLogits, numLogits, 1); - - totalNumLogits += numLogits; - - if (rnnStateBuffers) - { - auto const seqSlot = llmReq->mSeqSlot.value(); - auto& rnnStateManager = *rnnStateManagerPtr; - rnnStateManager.fillSlotMapping(*rnnStateBuffers->slotMappingHost, numSequences, seqSlot, reqBeamWidth); - } - numSequences += reqBeamWidth; - } - - if (transformerBuffers && maxBeamWidth > 1) - { - transformerBuffers->copyCacheIndirection(genRequests, decoderState.getCacheIndirectionOutput(), stream); - } - - numSequences = numContextRequests; - for (auto const& llmReq : genRequests) - { - auto const reqBeamWidth = llmReq->getBeamWidthByIter(); - auto const draftLength = llmReq->getNumDraftTokens(); - - auto const contextQLength = llmReq->mPromptLen + draftLength; - auto const sequenceLen - = contextQLength + llmReq->getMaxNumGeneratedTokens() + static_cast<SizeType32>(trtOverlap); - - std::fill_n(contextLengthsHostPtr + numSequences, reqBeamWidth, contextQLength); - std::fill_n(sequenceLengthsHostPtr + numSequences, reqBeamWidth, sequenceLen); - numSequences += reqBeamWidth; - } - if (modelConfig.getSpeculativeDecodingMode().isLookaheadDecoding()) - { - // copy from lookahead decoding buffer - mLookaheadBuffers->setFromInputs(numContextRequests, numGenRequests, *requestTypes, *seqSlots, - decoderState.getLookaheadBuffers(), runtime, modelConfig, worldConfig); - } - } - - // check skipCrossAttnBlocks - if (transformerBuffers && modelConfig.skipCrossAttnBlocks()) - { - bool isSkipCrossAttn = true; - for (auto const& requests : {contextRequests, genRequests}) - { - for (auto const& llmReq : requests) - { - bool tmpValue = false; - if (llmReq->getSkipCrossAttnBlocks() != nullptr) - { - manager.copy(*llmReq->getSkipCrossAttnBlocks(), &tmpValue); - } - isSkipCrossAttn &= tmpValue; - } - } - transformerBuffers->copySkipCrossAttnBlocks(isSkipCrossAttn, runtime); - } - - if (isChatGlm || isGlm) - { - positionIdsHost.reserve(totalInputSize * 2); - positionIdsHost.insert(positionIdsHost.end(), positionIdsHostRow2.begin(), positionIdsHostRow2.end()); - } - - if (modelConfig.useCrossAttention()) - { - encoderBuffers->fill(contextRequests, genRequests, manager); - } - if (modelConfig.usePromptTuning()) - { - promptTuningBuffers->fill(contextRequests, genRequests, manager, modelConfig.usePackedInput()); - } - if (modelConfig.useLoraPlugin()) - { - loraBuffers->fill(contextRequests, genRequests, peftTable, manager, modelConfig, worldConfig); - } - if (modelConfig.useMrope()) - { - if (!mropePositionDeltasHost.empty()) - { - auto mropePositionDeltasGen = ITensor::slice(mropePositionDeltas, 0, numGenSequences); - manager.copy(mropePositionDeltasHost.data(), *mropePositionDeltasGen); - } - } - - { - NVTX3_SCOPED_RANGE(bufferCopies); - if (trtOverlap) - { - auto contextInputsIds = ITensor::slice(inputsIds, 0, numContextTokens); - manager.copy(inputHost.data(), *contextInputsIds); - - if (!genRequests.empty()) - { - auto generationInputsIds = ITensor::slice(inputsIds, numContextTokens); - auto seqSlotsDeviceSlice = ITensor::slice(seqSlotsDevice, numContextRequests); - runtime::kernels::invokeGatherBatch( - *generationInputsIds, *newOutputTokens, *seqSlotsDeviceSlice, maxBeamWidth, stream); - } - } - else - { - manager.copy(inputHost.data(), *inputsIds); - } - // In generation phase, device ptr of context lengths need to be tiled. - manager.copy(*contextLengthsHost, *contextLengthsDevice); - manager.copy(*sequenceLengthsHost, *sequenceLengthsDevice); - auto const logitsIdsHostRange = BufferRange<SizeType32>(*logitsIdsHost); - auto lastTokenIdsHostRange = BufferRange<SizeType32>(*lastTokenIdsHost); - common::stl_utils::inclusiveScan( - logitsIdsHostRange.begin(), logitsIdsHostRange.end(), lastTokenIdsHostRange.begin()); - manager.copy(*lastTokenIdsHost, *lastTokenIdsDevice); - if (transformerBuffers) - { - TensorPtr decoderPositionIds = modelConfig.getSpeculativeDecodingMode().isLookaheadDecoding() - ? mLookaheadBuffers->positionIdsDevice - : nullptr; - transformerBuffers->copyPositionIds(runtime, positionIdsHost, isChatGlm || isGlm, decoderPositionIds); - } - if (rnnStateBuffers) - { - rnnStateBuffers->copySlotMappingH2D(runtime); - } - if (modelConfig.useLanguageAdapter()) - { - manager.copy(languageAdapterRoutingsHost.data(), *languageAdapterRoutings); - } - } - - if (transformerBuffers && static_cast<bool>(kvCacheManagerPtr)) - { - transformerBuffers->copyKvBlockOffsets( - contextRequests, genRequests, kvCacheManagerPtr, crossKvCacheManagerPtr, manager); - } - - if (modelConfig.useCrossAttention()) - { - transformerBuffers->copyCrossAttentionMasks(contextRequests, genRequests, contextLengthsDevice, - encoderBuffers->inputLengths, maxContextLength, encoderBuffers->getMaxInputLengthInBatch(), runtime); - } - - maxKvCacheLengthRounded = 0; - if (static_cast<bool>(pastKeyValueLengthsPtr)) - { - auto const maxKvCacheLength - = *std::max_element(pastKeyValueLengthsPtr, pastKeyValueLengthsPtr + getNumSequences()); - // Round up kv cache length - maxKvCacheLengthRounded = common::ceilDiv(maxKvCacheLength, kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE) - * kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE; - } - - if (modelConfig.getSpeculativeDecodingMode().needsDecoderPrologue()) - { - if (modelConfig.getSpeculativeDecodingMode().isExplicitDraftTokens()) - { - prepareExplicitDraftTokenBuffers( - decoderState.getExplicitDraftTokensBuffers(), runtime, modelConfig, worldConfig); - } - if (modelConfig.getSpeculativeDecodingMode().isEagle()) - { - prepareEagleBuffers( - contextRequests, genRequests, decoderState.getEagleBuffers(), runtime, modelConfig, worldConfig); - } - } - - sync_check_cuda_error(stream.get()); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void RuntimeBuffers::prepareExplicitDraftTokenBuffers( - runtime::ExplicitDraftTokensBuffers::Inputs const& explicitDraftTokensBuffers, TllmRuntime const& runtime, - ModelConfig const& modelConfig, WorldConfig const& worldConfig) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - TLLM_CHECK(mExplicitDraftTokensBuffers); - - mExplicitDraftTokensBuffers->setFromInputs(numContextRequests, numGenRequests, *requestTypes, *seqSlots, - explicitDraftTokensBuffers, *transformerBuffers->positionIds, modelConfig, worldConfig, - runtime.getBufferManager(), runtime.getStream()); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void RuntimeBuffers::prepareEagleBuffers(RequestVector const& contextRequests, RequestVector const& genRequests, - runtime::EagleBuffers::Inputs const& eagleBuffers, TllmRuntime const& runtime, ModelConfig const& modelConfig, - WorldConfig const& worldConfig) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - TLLM_CHECK(mEagleBuffers); - - mEagleBuffers->setFromInputs(contextRequests, genRequests, *requestTypes, *seqSlots, eagleBuffers, - runtime.getBufferManager(), modelConfig, worldConfig); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -std::tuple<SizeType32, RuntimeBuffers::TensorMap const&, RuntimeBuffers::TensorMap&> RuntimeBuffers::prepareStep( - RequestVector const& contextRequests, RequestVector const& genRequests, SizeType32 maxBeamWidth, - SizeType32 maxAttentionWindow, runtime::decoder::DecoderState const& decoderState, - kv_cache_manager::BaseKVCacheManager* kvCacheManager, kv_cache_manager::BaseKVCacheManager* crossKvCacheManager, - rnn_state_manager::RnnStateManager* rnnStateManager, PeftTable const& peftTable, TllmRuntime const& runtime, - ModelConfig const& modelConfig, WorldConfig const& worldConfig, bool gatherGenerationLogits, bool trtOverlap, - OptionalRef<runtime::ITensor const> newOutputTokens) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(runtimeBuffersPrepareStep); - - setBufferSizes(contextRequests, genRequests); - reshape(runtime, modelConfig, worldConfig, gatherGenerationLogits); - - setFromInputs(contextRequests, genRequests, maxBeamWidth, maxAttentionWindow, decoderState, kvCacheManager, - crossKvCacheManager, rnnStateManager, peftTable, runtime, modelConfig, worldConfig, trtOverlap, - newOutputTokens); - - fillIOMaps(modelConfig, worldConfig); - - auto const numTokens = getNumTokens(); - auto const optProfileId = runtime.getOptProfileId(numTokens, ModelConfig::getOptProfilesSplitPoints()); - setContextIndex(optProfileId); - TLLM_LOG_DEBUG("numTokens: %d, optProfileId: %d", numTokens, optProfileId); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return {optProfileId, inputMap, outputMap}; -} - -void RuntimeBuffers::fillIOMaps(ModelConfig const& modelConfig, WorldConfig const& worldConfig) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(runtimeBuffersFillIOMaps); - - inputMap.clear(); - outputMap.clear(); - - if (transformerBuffers) - { - transformerBuffers->getBuffers(inputMap, outputMap, modelConfig); - } - if (rnnStateBuffers) - { - rnnStateBuffers->getBuffers(inputMap); - } - - if (worldConfig.isLastPipelineParallelRank()) - { - // feed a view to TensorRT runtime so reshaping does not change logits buffer - outputMap.insert_or_assign(kLogitsTensorName, ITensor::view(logits)); - } - else - { - outputMap.insert_or_assign(kHiddenStatesOutputTensorName, hiddenStates); - } - - if (worldConfig.isFirstPipelineParallelRank()) - { - inputMap.insert_or_assign(kInputIdsTensorName, inputsIds); - } - else - { - inputMap.insert_or_assign(kHiddenStatesInputTensorName, hiddenStates); - } - - inputMap.insert_or_assign(kLastTokenIdsTensorName, lastTokenIdsDevice); - - inputMap.insert_or_assign(kHostRequestTypesTensorName, requestTypes); - // In the generation phase, we still pass context lengths. - inputMap.insert_or_assign(kContextLengthsTensorName, contextLengthsDevice); - inputMap.insert_or_assign(kHostContextLengthsTensorName, contextLengthsHost); - inputMap.insert_or_assign(kSequenceLengthsTensorName, sequenceLengthsDevice); - - if (modelConfig.useCrossAttention()) - { - encoderBuffers->insertInputTensors(inputMap); - } - if (modelConfig.usePromptTuning()) - { - auto const& promptTuningParams = promptTuningBuffers->mPromptTuningParams; - inputMap.insert_or_assign(kPromptEmbeddingTableTensorName, promptTuningParams.embeddingTable); - inputMap.insert_or_assign(kTasksTensorName, promptTuningParams.tasks); - inputMap.insert_or_assign(kPromptVocabSizeTensorName, promptTuningParams.vocabSize); - } - if (modelConfig.useMrope()) - { - - inputMap.insert_or_assign(kMRopeRotaryCosSinTensorName, mropeRotaryCosSin); - inputMap.insert_or_assign(kMRopePositionDeltasTensorName, mropePositionDeltas); - } - if (modelConfig.useLoraPlugin()) - { - loraBuffers->insertInputTensors(inputMap, loraBuffers->mLoraWeightsPointersHost, - loraBuffers->mLoraAdapterSizesHost, modelConfig, worldConfig); - } - if (modelConfig.useLanguageAdapter()) - { - inputMap.insert_or_assign("language_adapter_routings", languageAdapterRoutings); - } - - if (mMedusaBuffers) - { - mMedusaBuffers->insertInputTensors(inputMap, outputMap, worldConfig); - } - if (mLookaheadBuffers) - { - mLookaheadBuffers->insertInputTensors(inputMap, outputMap, worldConfig); - } - if (mExplicitDraftTokensBuffers) - { - mExplicitDraftTokensBuffers->insertInputTensors(inputMap, outputMap, worldConfig); - } - if (mEagleBuffers) - { - mEagleBuffers->insertInputTensors(inputMap, outputMap, worldConfig); - } - - for (auto const& outputTensor : mAdditionalOutputTensors) - { - outputMap.insert_or_assign(outputTensor.first, outputTensor.second); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/transformerBuffers.cpp b/cpp/tensorrt_llm/batch_manager/transformerBuffers.cpp deleted file mode 100644 index 4f81c8926682..000000000000 --- a/cpp/tensorrt_llm/batch_manager/transformerBuffers.cpp +++ /dev/null @@ -1,679 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/transformerBuffers.h" - -#include "tensorrt_llm/batch_manager/kvCacheManager.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/kernels/attentionMask.h" -#include "tensorrt_llm/kernels/contextFusedMultiHeadAttention/fmhaPackedMask.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/runtimeKernels.h" -#include "tensorrt_llm/runtime/tllmBuffers.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" -#include <cstdint> - -using namespace tensorrt_llm::runtime; -namespace tk = tensorrt_llm::kernels; - -namespace tensorrt_llm::batch_manager -{ - -TransformerBuffers::TransformerBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, - runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig) - : maxInputLen(modelConfig.getMaxInputLen()) - , maxEncoderOutputLen(modelConfig.getMaxEncoderLen()) -{ - auto const& manager = runtime.getBufferManager(); - auto const& engine = runtime.getEngine(); - - positionIds = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - - auto const localNbAttnLayers - = modelConfig.getNbAttentionLayers(worldConfig.getPipelineParallelism(), worldConfig.getPipelineParallelRank()); - // find the index of the first attention layer in the current rank - auto const firstLayerId = modelConfig.countLowerRankLayers(runtime::ModelConfig::LayerType::kATTENTION, - worldConfig.getPipelineParallelism(), worldConfig.getPipelineParallelRank()); - - cacheIndirection - = manager.gpu(ITensor::makeShape({maxBatchSize, maxBeamWidth, maxAttentionWindow}), nvinfer1::DataType::kINT32); - - if (!modelConfig.getMaxNumTokens().has_value()) - { - TLLM_THROW("Model must configure a max number of tokens."); - } - maxNumTokens = modelConfig.getMaxNumTokens().value(); - - if (modelConfig.isKVCacheEnabled()) - { - auto const kvCacheBlockOffsetsType = engine.getTensorDataType("kv_cache_block_offsets"); - kvCacheBlockOffsetsHost = manager.emptyTensor(MemoryType::kPINNEDPOOL, kvCacheBlockOffsetsType); - kvCacheBlockOffsetsDevice = manager.emptyTensor(MemoryType::kGPU, kvCacheBlockOffsetsType); - - if (modelConfig.useCrossAttention()) - { - crossKvCacheBlockOffsetsHost = manager.emptyTensor(MemoryType::kPINNEDPOOL, kvCacheBlockOffsetsType); - crossKvCacheBlockOffsetsDevice = manager.emptyTensor(MemoryType::kGPU, kvCacheBlockOffsetsType); - crossAttentionMaskDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kBOOL); - crossAttentionMaskPinnedHost = tensorrt_llm::runtime::BufferManager::pinnedPool( - ITensor::makeShape({maxNumTokens, maxEncoderOutputLen}), nvinfer1::DataType::kBOOL); - crossAttentionPackedMaskDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - crossAttentionCuQSeqLensDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - crossAttentionPackedMaskCuMaskRowsDevice - = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); - - // Pinned memory for batch copy of attention masks. - // There will be paddings in the dim1, so copy it by tokens. - crossAttentionMaskCopySrcOffsets = tensorrt_llm::runtime::BufferManager::pinnedPool( - ITensor::makeShape({maxNumTokens}), nvinfer1::DataType::kINT64); - crossAttentionMaskCopyDstOffsets = tensorrt_llm::runtime::BufferManager::pinnedPool( - ITensor::makeShape({maxNumTokens}), nvinfer1::DataType::kINT64); - crossAttentionMaskCopySizes = tensorrt_llm::runtime::BufferManager::pinnedPool( - ITensor::makeShape({maxNumTokens}), nvinfer1::DataType::kINT64); - } - } - - fillValuesAlt = tensorrt_llm::runtime::BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - fillValuesAltDevice = manager.gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - seqSlotsAlt = tensorrt_llm::runtime::BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - seqSlotsAltDevice = manager.gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - - cacheIndirBatchedCopySrcOffsets = tensorrt_llm::runtime::BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT64); - cacheIndirBatchedCopyDstOffsets = tensorrt_llm::runtime::BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT64); - cacheIndirBatchedCopySizes = tensorrt_llm::runtime::BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT64); - skipCrossAttnBlocks - = tensorrt_llm::runtime::BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kBOOL); - - pastKeyValueLengths = manager.emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kINT32); - - maxAttentionWindows = BufferManager::cpu(ITensor::makeShape({localNbAttnLayers}), nvinfer1::DataType::kINT32); - auto* maxAttentionWindowsPtr = bufferCast<SizeType32>(*maxAttentionWindows); - auto const attentionWindowLength = maxAttentionWindowVec.size(); - for (SizeType32 i = 0; i < localNbAttnLayers; ++i) - { - maxAttentionWindowsPtr[i] = maxAttentionWindowVec[(firstLayerId + i) % attentionWindowLength]; - } - - sinkTokenLengths = BufferManager::cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - bufferCast<SizeType32>(*sinkTokenLengths)[0] = sinkTokenLen; - - contextProgressHost = BufferManager::cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT64); - bufferCast<int64_t>(*contextProgressHost)[0] = 0; - - if (modelConfig.useGemmAllReducePlugin() && worldConfig.isTensorParallel()) - { - nvinfer1::DataType ARType = modelConfig.getGemmAllReduceDtype(); - - auto hiddenSize = modelConfig.getHiddenSize() * worldConfig.getTensorParallelism(); - - auto tpGroup = worldConfig.getTensorParallelGroup(); - std::set<int> tpGroupSet(tpGroup.begin(), tpGroup.end()); - - auto outputDims = ITensor::makeShape({modelConfig.getMaxNumTokens().value() * hiddenSize}); - - gemmAllReduceOutput = std::make_shared<MulticastTensor>(outputDims, ARType, tpGroupSet); - } -} - -void TransformerBuffers::reshape(SizeType32 numSequences, SizeType32 numInputTokens) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - pastKeyValueLengths->reshape(ITensor::makeShape({numSequences})); - - if (kvCacheBlockOffsetsHost) - { - auto cacheBlockOffsetsShape = kvCacheBlockOffsetsHost->getShape(); - if (cacheBlockOffsetsShape.nbDims > 0) - { - cacheBlockOffsetsShape.d[1] = numSequences; - kvCacheBlockOffsetsHost->reshape(cacheBlockOffsetsShape); - kvCacheBlockOffsetsDevice->reshape(cacheBlockOffsetsShape); - } - else - { - TLLM_LOG_DEBUG("kvCacheBlockOffsets not allocated yet"); - } - } - - if (crossKvCacheBlockOffsetsHost) - { - TLLM_CHECK_WITH_INFO( - crossKvCacheBlockOffsetsDevice, "crossKvCacheBlockOffsetsDevice is empty for model with cross attention!"); - auto crossCacheBlockOffsetsShape = crossKvCacheBlockOffsetsHost->getShape(); - if (crossCacheBlockOffsetsShape.nbDims > 0) - { - crossCacheBlockOffsetsShape.d[1] = numSequences; - crossKvCacheBlockOffsetsHost->reshape(crossCacheBlockOffsetsShape); - crossKvCacheBlockOffsetsDevice->reshape(crossCacheBlockOffsetsShape); - } - else - { - TLLM_LOG_DEBUG("crossKvCacheBlockOffsets not allocated yet"); - } - } - - if (crossAttentionMaskDevice) - { - auto crossAttentionMaskShape = crossAttentionMaskDevice->getShape(); - if (crossAttentionMaskShape.nbDims > 0) - { - crossAttentionMaskShape.d[0] = numInputTokens; - crossAttentionMaskDevice->reshape(crossAttentionMaskShape); - crossAttentionMaskPinnedHost->reshape(crossAttentionMaskShape); - crossAttentionMaskCopySrcOffsets->reshape(ITensor::makeShape({numInputTokens})); - crossAttentionMaskCopyDstOffsets->reshape(ITensor::makeShape({numInputTokens})); - crossAttentionMaskCopySizes->reshape(ITensor::makeShape({numInputTokens})); - } - else - { - TLLM_LOG_DEBUG("crossAttentionMaskDevice not allocated yet"); - } - } - - if (crossAttentionPackedMaskDevice) - { - auto crossAttentionMaskPackedShape = crossAttentionPackedMaskDevice->getShape(); - if (crossAttentionMaskPackedShape.nbDims > 0) - { - crossAttentionMaskPackedShape.d[0] = numInputTokens; - crossAttentionPackedMaskDevice->reshape(crossAttentionMaskPackedShape); - } - else - { - TLLM_LOG_DEBUG("crossAttentionPackedMaskDevice not allocated yet"); - } - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TransformerBuffers::reshapeKvTensors(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, SizeType32 maxBlocksPerSeq, - kv_cache_manager::CacheType kvCacheType, SizeType32 numPools, BufferManager const& manager) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // allocate with max shape during init - if (kvCacheType == kv_cache_manager::CacheType::kSELF) - { - auto const cacheBlockOffsetsShape - = ITensor::makeShape({numPools, maxBatchSize * maxBeamWidth, 2, maxBlocksPerSeq}); - - kvCacheBlockOffsetsHost->reshape(cacheBlockOffsetsShape); - manager.setZero(*kvCacheBlockOffsetsHost); - - kvCacheBlockOffsetsDevice->reshape(cacheBlockOffsetsShape); - manager.setZero(*kvCacheBlockOffsetsDevice); - } - else if (kvCacheType == kv_cache_manager::CacheType::kCROSS) - { - auto const crossCacheBlockOffsetsShape - = ITensor::makeShape({numPools, maxBatchSize * maxBeamWidth, 2, maxBlocksPerSeq}); - - crossKvCacheBlockOffsetsHost->reshape(crossCacheBlockOffsetsShape); - manager.setZero(*crossKvCacheBlockOffsetsHost); - - crossKvCacheBlockOffsetsDevice->reshape(crossCacheBlockOffsetsShape); - manager.setZero(*crossKvCacheBlockOffsetsDevice); - - crossAttentionMaskDevice->reshape(ITensor::makeShape({maxNumTokens, maxEncoderOutputLen})); - manager.setZero(*crossAttentionMaskDevice); - manager.setZero(*crossAttentionMaskPinnedHost); - - // Only context attention needs this, so allocate it by shape [maxBatchSize, maxInputLen, maxEncoderOutputLen]. - auto [packedMaskM, packedMaskN] = tk::roundUpPackedMaskMNDims(maxInputLen, maxEncoderOutputLen); - crossAttentionPackedMaskDevice->reshape(ITensor::makeShape({maxBatchSize * packedMaskM, packedMaskN})); - manager.setZero(*crossAttentionPackedMaskDevice); - - crossAttentionCuQSeqLensDevice->reshape(ITensor::makeShape({maxBatchSize + 1})); - manager.setZero(*crossAttentionCuQSeqLensDevice); - - crossAttentionPackedMaskCuMaskRowsDevice->reshape(ITensor::makeShape({maxBatchSize + 1})); - manager.setZero(*crossAttentionPackedMaskCuMaskRowsDevice); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TransformerBuffers::getBuffers( - TensorMap& inputBuffers, TensorMap& outputBuffers, runtime::ModelConfig const& modelConfig) const -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(transformerBuffersGetBuffers); - - inputBuffers.insert_or_assign(kPositionIdsTensorName, positionIds); - inputBuffers.insert_or_assign(kHostPastKeyValueLengthsTensorName, pastKeyValueLengths); - inputBuffers.insert_or_assign(kCacheIndirectionsTensorName, cacheIndirection); - inputBuffers.insert_or_assign(kHostSinkTokenLengthTensorName, sinkTokenLengths); - - inputBuffers.insert_or_assign(kHostMaxAttentionWindowSizesTensorName, maxAttentionWindows); - inputBuffers.insert_or_assign(kKvCacheBlockOffsetsTensorName, kvCacheBlockOffsetsDevice); - inputBuffers.insert_or_assign(kHostKvCacheBlockOffsetsTensorName, kvCacheBlockOffsetsHost); - inputBuffers.insert_or_assign(kHostContextProgressTensorName, contextProgressHost); - - if (crossKvCacheBlockOffsetsHost) - { - inputBuffers.insert_or_assign(kCrossKvCacheBlockOffsetsTensorName, crossKvCacheBlockOffsetsDevice); - inputBuffers.insert_or_assign(kHostCrossKvCacheBlockOffsetsTensorName, crossKvCacheBlockOffsetsHost); - inputBuffers.insert_or_assign(kHostCrossKvCachePoolPointersTensorName, crossKvCacheBlockPoolPointers); - inputBuffers.insert_or_assign(kHostCrossKvCachePoolMappingTensorName, crossKvCacheBlockPoolMapping); - inputBuffers.insert_or_assign(kCrossAttentionMaskTensorName, crossAttentionMaskDevice); - inputBuffers.insert_or_assign(kCrossAttentionPackedMaskTensorName, crossAttentionPackedMaskDevice); - } - - if (skipCrossAttnBlocks) - { - inputBuffers.insert_or_assign(kSkipCrossAttentionBlocksTensorName, skipCrossAttnBlocks); - } - - if (modelConfig.useGemmAllReducePlugin()) - { - for (int idx = 0; idx < modelConfig.getNbAttentionLayers() * 2; ++idx) - { - // XXX (xsimmons): this is a bit hacky as it assumes - // 2x RowLinear layers per attention block. - // This will be fixed soon when I remove coupling between model - // and runtime. - auto gemmARViewUC = gemmAllReduceOutput->getTensorView(MulticastTensorView::ViewType::kUNICAST); - auto gemmARViewMC = gemmAllReduceOutput->getTensorView(MulticastTensorView::ViewType::kMULTICAST); - auto gemmARViewIpc = gemmAllReduceOutput->getTensorView(MulticastTensorView::ViewType::kIPC_LIST); - - outputBuffers.insert_or_assign("gemm_allreduce_uc_out_" + std::to_string(idx), gemmARViewUC); - outputBuffers.insert_or_assign("gemm_allreduce_mc_out_" + std::to_string(idx), gemmARViewMC); - outputBuffers.insert_or_assign("gemm_allreduce_ipc_out_" + std::to_string(idx), gemmARViewIpc); - } - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TransformerBuffers::copyPositionIds(runtime::TllmRuntime const& runtime, - std::vector<SizeType32> const& positionIdsHost, bool isChatGlm, TensorPtr const& decoderPositionIds) -{ - auto const& manager = runtime.getBufferManager(); - if (isChatGlm) - { - positionIds->reshape(ITensor::makeShape({2, static_cast<int>(positionIdsHost.size()) / 2})); - manager.copy(positionIdsHost.data(), *positionIds); - } - else if (decoderPositionIds == nullptr) - { - positionIds->reshape(ITensor::makeShape({static_cast<int>(positionIdsHost.size())})); - manager.copy(positionIdsHost.data(), *positionIds); - } - else - { - // concat context phase and generation phase positionIds. - auto const contextPositionIdsLen = static_cast<ITensor::DimType64>(positionIdsHost.size()); - auto const generationPositionIdsLen = ITensor::volume(decoderPositionIds->getShape()); - positionIds->reshape(ITensor::makeShape({contextPositionIdsLen + generationPositionIdsLen})); - manager.copy(positionIdsHost.data(), *ITensor::slice(positionIds, 0, contextPositionIdsLen)); - manager.copy(*decoderPositionIds, *ITensor::slice(positionIds, contextPositionIdsLen)); - } -} - -void TransformerBuffers::copyKvBlockOffsets(RequestVector const& contextRequests, RequestVector const& genRequests, - kv_cache_manager::BaseKVCacheManager const* kvCacheManager, - kv_cache_manager::BaseKVCacheManager const* crossKvCacheManager, BufferManager const& manager) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(copyKvBlockOffsets); - - auto const& cudaStream = manager.getStream(); - - SizeType32 constexpr contextBeamWidth{1}; - SizeType32 numSequences{0}; - SizeType32 maxBlockCount{0}; - SizeType32 maxCrossBlockCount{0}; - for (auto const& requests : {contextRequests, genRequests}) - { - for (auto const& llmReq : requests) - { - auto const requestId = llmReq->mRequestId; - auto const isContextRequest = llmReq->isContextInitState(); - auto const beamWidth = isContextRequest ? contextBeamWidth : llmReq->getBeamWidthByIter(); - auto const maxBeamBlockCount - = kvCacheManager->copyBlockOffsets(*kvCacheBlockOffsetsHost, numSequences, requestId); - maxBlockCount = std::max(maxBlockCount, maxBeamBlockCount); - if (crossKvCacheBlockOffsetsHost) - { - auto const maxCrossBeamBlockCount - = crossKvCacheManager->copyBlockOffsets(*crossKvCacheBlockOffsetsHost, numSequences, requestId); - maxCrossBlockCount = std::max(maxCrossBlockCount, maxCrossBeamBlockCount); - } - numSequences += beamWidth; - } - } - - // requests' block offsets collected as [totalNumSequences, 2, maxBlocksPerSeq], copy to device - auto copyOffsetsToDevice = [&cudaStream](TensorPtr& offsetsHost, TensorPtr& offsetsDevice, SizeType32 maxBlockCount) - { - // shape should be [totalNumSequences, 2, maxBlocksPerSeq] - auto const& offsetsShape = offsetsHost->getShape(); - auto const maxBlocksPerSeq = offsetsShape.d[3]; - auto const offsetsTypeSize = tensorrt_llm::common::getDTypeSize(offsetsHost->getDataType()); - auto const copyPitch = maxBlocksPerSeq * offsetsTypeSize; - auto const copyHeight = offsetsShape.d[0] * offsetsShape.d[1] * offsetsShape.d[2]; - auto const copyWidth = maxBlockCount * offsetsTypeSize; - auto* srcPtr = bufferCast<tk::KVCacheIndex>(*offsetsHost); - auto* dstPtr = bufferCast<tk::KVCacheIndex>(*offsetsDevice); - - TLLM_CUDA_CHECK(cudaMemcpy2DAsync( - dstPtr, copyPitch, srcPtr, copyPitch, copyWidth, copyHeight, cudaMemcpyHostToDevice, cudaStream.get())); - }; - - copyOffsetsToDevice(kvCacheBlockOffsetsHost, kvCacheBlockOffsetsDevice, maxBlockCount); - if (crossKvCacheBlockOffsetsHost) - { - copyOffsetsToDevice(crossKvCacheBlockOffsetsHost, crossKvCacheBlockOffsetsDevice, maxCrossBlockCount); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TransformerBuffers::copyCacheIndirection( - RequestVector const& genRequests, TensorPtr const& decoderCacheIndirectionOutput, CudaStream const& stream) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(copyCacheIndirection); - - auto const numGenerationRequests = genRequests.size(); - - auto batchedCopySrcOffsets = BufferRange<SizeType64>(*cacheIndirBatchedCopySrcOffsets); - auto batchedCopyDstOffsets = BufferRange<SizeType64>(*cacheIndirBatchedCopyDstOffsets); - auto batchedCopySizes = BufferRange<SizeType64>(*cacheIndirBatchedCopySizes); - - auto cacheIndirShape = decoderCacheIndirectionOutput->getShape(); - - // At present, all requests of a batch must have the same beam width in one generation step (or they will not - // be batched together). So, the beam width of the first request is taken here to reshape the buffer. - // Corresponding changes must be done if Diverse-Beam-Width-Search (DBWS, requests with diverse beam width in - // a batch in one generation step) is supported in the future. - auto reqBeamWidth = genRequests[0]->getBeamWidthByIter(); - - // Get size of copying from shape of `CacheIndirectionOutput` - cacheIndirShape.d[0] = 1; - cacheIndirShape.d[1] = reqBeamWidth; // Use beam width of current step rather than max beam width as dst offset - auto const copySize = static_cast<SizeType64>(ITensor::volume(cacheIndirShape)); - - std::transform(genRequests.begin(), genRequests.end(), batchedCopySrcOffsets.begin(), - [copySize](auto const& llmReq) { return llmReq->mSeqSlot.value() * copySize; }); - std::generate_n( - batchedCopyDstOffsets.begin(), numGenerationRequests, [copySize, i = 0]() mutable { return (i++) * copySize; }); - std::fill_n(batchedCopySizes.begin(), numGenerationRequests, copySize); - - auto const batchedCopySrcOffsetsSlice = ITensor::slice(cacheIndirBatchedCopySrcOffsets, 0, numGenerationRequests); - auto const batchedCopyDstOffsetsSlice = ITensor::slice(cacheIndirBatchedCopyDstOffsets, 0, numGenerationRequests); - auto const batchedCopySizesSlice = ITensor::slice(cacheIndirBatchedCopySizes, 0, numGenerationRequests); - runtime::kernels::invokeCopyBatch(*decoderCacheIndirectionOutput, *cacheIndirection, *batchedCopySrcOffsetsSlice, - *batchedCopyDstOffsetsSlice, *batchedCopySizesSlice, copySize, stream); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TransformerBuffers::copyCrossAttentionMasks(RequestVector const& contextRequests, RequestVector const& genRequests, - TensorPtr const& decoderContextLengthsDevice, TensorPtr const& encoderInputLengths, - SizeType32 maxDecoderContextLength, SizeType32 maxEncoderInputLengthInBatch, TllmRuntime const& runtime) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - auto const& manager = runtime.getBufferManager(); - - // Reshape the tensor to make sure the dim1 matches maxEncoderInputLengthInBatch. - auto crossAttentionMaskShape = crossAttentionMaskDevice->getShape(); - crossAttentionMaskShape.d[1] = maxEncoderInputLengthInBatch; - crossAttentionMaskDevice->reshape(crossAttentionMaskShape); - // Set crossAttentionMask to true by default if it is not provided. - manager.setMem(*crossAttentionMaskDevice, 1); - - // Check if all context requests have cross attention mask. - bool allContextCrossAttentionMaskProvided = true; - for (auto const& llmReq : contextRequests) - { - auto const& crossAttentionMaskRequest = llmReq->getCrossAttentionMask(); - if (bufferCastOrNull<bool>(crossAttentionMaskRequest) == nullptr) - { - allContextCrossAttentionMaskProvided = false; - break; - } - } - // If not all requests have cross attention mask, let us create the default ones. - auto const& stream = runtime.getStream(); - if (!allContextCrossAttentionMaskProvided) - { - TLLM_LOG_WARNING("Default padding attention mask will be used as not all requests have cross attention mask."); - tk::AttentionMaskParams<bool> attentionMaskParams; - memset((void*) &attentionMaskParams, 0, sizeof(attentionMaskParams)); - // Set parameters. - attentionMaskParams.mask = bufferCastOrNull<bool>(crossAttentionMaskDevice); - attentionMaskParams.cuQSeqLens = bufferCastOrNull<SizeType32>(crossAttentionCuQSeqLensDevice); - attentionMaskParams.actualQSeqLens = bufferCastOrNull<SizeType32>(decoderContextLengthsDevice); - attentionMaskParams.actualKvSeqLens = bufferCastOrNull<SizeType32>(encoderInputLengths); - attentionMaskParams.attentionMaskType = tk::AttentionMaskType::PADDING; - attentionMaskParams.batchSize = static_cast<SizeType32>(contextRequests.size()); - attentionMaskParams.maxQSeqLen = maxDecoderContextLength; - attentionMaskParams.maxKvSeqLen = maxEncoderInputLengthInBatch; - // Launch the kernel. - tk::invokeBuildAttentionMask(attentionMaskParams, stream.get()); - sync_check_cuda_error(stream.get()); - } - // Use the first request's cross attention mask tensor's pointer address as the primary source pointer. - auto const& attentionMaskSrc = !contextRequests.empty() ? contextRequests[0]->getCrossAttentionMask() - : genRequests[0]->getCrossAttentionMask(); - bool const* primarySrcPtr = bufferCastOrNull<bool>(attentionMaskSrc); - - // Pinned-memory buffer preparation for batch copy. - auto batchedCopySrcOffsets = BufferRange<SizeType64>(*crossAttentionMaskCopySrcOffsets); - auto batchedCopyDstOffsets = BufferRange<SizeType64>(*crossAttentionMaskCopyDstOffsets); - auto batchedCopySizes = BufferRange<SizeType64>(*crossAttentionMaskCopySizes); - // Requests with cross-attention-mask don't need to copy. - manager.setZero(*crossAttentionMaskCopySizes); - sync_check_cuda_error(stream.get()); - - SizeType32 numTokens = 0; - SizeType32 numCopiedTokens = 0; - bool* pinnedMemPtr = bufferCastOrNull<bool>(crossAttentionMaskPinnedHost); - for (auto const& llmReq : contextRequests) - { - auto const& crossAttentionMaskRequest = llmReq->getCrossAttentionMask(); - auto const position = llmReq->getContextCurrentPosition(); - auto const size = llmReq->getContextChunkSize(); - if (bufferCastOrNull<bool>(crossAttentionMaskRequest) != nullptr) - { - auto memType = crossAttentionMaskRequest->getMemoryType(); - auto const crossAttentionMaskRequestDim0 - = static_cast<SizeType64>(crossAttentionMaskRequest->getShape().d[0]); - auto const crossAttentionMaskRequestDim1 - = static_cast<SizeType64>(crossAttentionMaskRequest->getShape().d[1]); - TLLM_LOG_DEBUG("copyCrossAttentionMasks (shape [%d, %d]) from contextRequests position %d chunkSize %d", - crossAttentionMaskRequestDim0, crossAttentionMaskRequestDim1, position, size); - if ((position + size - 1) >= crossAttentionMaskRequestDim0) - { - TLLM_LOG_WARNING( - "The provided crossAttentionMask input is not complete for context phases, the last row " - "will be " - "used by default."); - } - // copy it to pinned memory if it is a cpu tensor. - if (memType == MemoryType::kCPU) - { - TLLM_LOG_DEBUG("CrossAttentionMask tensor is on CPU."); - auto const copiedPosition - = std::min(crossAttentionMaskRequestDim0 - 1, static_cast<SizeType64>(position)); - auto const copiedSize - = std::min(crossAttentionMaskRequestDim0 - copiedPosition, static_cast<SizeType64>(size)); - SizeType64 inputMaskOffset = (copiedPosition * crossAttentionMaskRequestDim1); - SizeType64 inputMaskSize = (copiedSize * crossAttentionMaskRequestDim1); - std::memcpy( - pinnedMemPtr, bufferCastOrNull<bool>(crossAttentionMaskRequest) + inputMaskOffset, inputMaskSize); - pinnedMemPtr += inputMaskSize; - for (SizeType32 tokenId = position; tokenId < position + size; tokenId++) - { - SizeType64 tokenIdInPinnedMem - = std::min(copiedSize - 1, static_cast<SizeType64>(tokenId - position)); - batchedCopySrcOffsets.begin()[numCopiedTokens] - = (pinnedMemPtr - primarySrcPtr) + tokenIdInPinnedMem * crossAttentionMaskRequestDim1; - batchedCopyDstOffsets.begin()[numCopiedTokens] - = numTokens * static_cast<SizeType64>(maxEncoderInputLengthInBatch); - batchedCopySizes.begin()[numCopiedTokens] = crossAttentionMaskRequestDim1; - numCopiedTokens++; - numTokens++; - } - } - else - { - TLLM_LOG_DEBUG("CrossAttentionMask tensor is on GPU."); - for (SizeType32 tokenId = position; tokenId < position + size; tokenId++) - { - batchedCopySrcOffsets.begin()[numCopiedTokens] - = static_cast<SizeType64>(bufferCastOrNull<bool>(crossAttentionMaskRequest) - primarySrcPtr) - + std::min(crossAttentionMaskRequestDim0 - 1, static_cast<SizeType64>(tokenId)) - * crossAttentionMaskRequestDim1; - batchedCopyDstOffsets.begin()[numCopiedTokens] - = numTokens * static_cast<SizeType64>(maxEncoderInputLengthInBatch); - batchedCopySizes.begin()[numCopiedTokens] = crossAttentionMaskRequestDim1; - numCopiedTokens++; - numTokens++; - } - } - } - else - { - numTokens += size; - TLLM_LOG_WARNING( - "CrossAttentionMask is not provided for the request. Default padding attention mask will be " - "created."); - } - } - sync_check_cuda_error(stream.get()); - - for (auto const& llmReq : genRequests) - { - auto const promptLen = llmReq->mPromptLen; - auto const decodingIter = llmReq->getDecodingIter(); - auto const& crossAttentionMaskRequest = llmReq->getCrossAttentionMask(); - if (bufferCastOrNull<bool>(crossAttentionMaskRequest) != nullptr) - { - auto const memType = crossAttentionMaskRequest->getMemoryType(); - auto const crossAttentionMaskRequestDim0 - = static_cast<SizeType64>(crossAttentionMaskRequest->getShape().d[0]); - auto const crossAttentionMaskRequestDim1 - = static_cast<SizeType64>(crossAttentionMaskRequest->getShape().d[1]); - TLLM_LOG_DEBUG("copyCrossAttentionMasks (shape [%d, %d]) from genRequests decodingIter %d", - crossAttentionMaskRequestDim0, crossAttentionMaskRequestDim1, decodingIter); - if (promptLen + decodingIter - 1 >= crossAttentionMaskRequestDim0) - { - TLLM_LOG_WARNING( - "The provided crossAttentionMask input is not complete for generation phases, the last row " - "will be " - "used by default."); - } - // copy it to pinned memory if it is a cpu tensor. - if (memType == MemoryType::kCPU) - { - TLLM_LOG_DEBUG("CrossAttentionMask tensor is on CPU."); - SizeType64 copiedPosition = std::min( - crossAttentionMaskRequestDim0 - 1, static_cast<SizeType64>(promptLen + decodingIter - 1)); - SizeType64 inputMaskOffset = (copiedPosition * crossAttentionMaskRequestDim1); - SizeType64 inputMaskSize = crossAttentionMaskRequestDim1; - std::memcpy( - pinnedMemPtr, bufferCastOrNull<bool>(crossAttentionMaskRequest) + inputMaskOffset, inputMaskSize); - pinnedMemPtr += inputMaskSize; - batchedCopySrcOffsets.begin()[numCopiedTokens] = static_cast<SizeType64>(pinnedMemPtr - primarySrcPtr); - batchedCopyDstOffsets.begin()[numCopiedTokens] - = numTokens * static_cast<SizeType64>(maxEncoderInputLengthInBatch); - batchedCopySizes.begin()[numCopiedTokens] = crossAttentionMaskRequestDim1; - } - else - { - TLLM_LOG_DEBUG("CrossAttentionMask tensor is on GPU."); - batchedCopySrcOffsets.begin()[numCopiedTokens] - = static_cast<SizeType64>(bufferCastOrNull<bool>(crossAttentionMaskRequest) - primarySrcPtr) - + std::min(crossAttentionMaskRequestDim0 - 1, static_cast<SizeType64>(promptLen + decodingIter - 1)) - * crossAttentionMaskRequestDim1; - batchedCopyDstOffsets.begin()[numCopiedTokens] - = numTokens * static_cast<SizeType64>(maxEncoderInputLengthInBatch); - batchedCopySizes.begin()[numCopiedTokens] = crossAttentionMaskRequestDim1; - } - numCopiedTokens++; - numTokens++; - } - else - { - numTokens++; - TLLM_LOG_WARNING( - "CrossAttentionMask is not provided for the generation request. Full valid attentionMask will " - "be used " - "by default."); - } - } - sync_check_cuda_error(stream.get()); - - // Copy all requests' attention mask in one kernel. - if (attentionMaskSrc != nullptr) - { - crossAttentionMaskCopySrcOffsets->reshape(ITensor::makeShape({numCopiedTokens})); - crossAttentionMaskCopyDstOffsets->reshape(ITensor::makeShape({numCopiedTokens})); - crossAttentionMaskCopySizes->reshape(ITensor::makeShape({numCopiedTokens})); - runtime::kernels::invokeCopyBatch(*attentionMaskSrc, *crossAttentionMaskDevice, - *crossAttentionMaskCopySrcOffsets, *crossAttentionMaskCopyDstOffsets, *crossAttentionMaskCopySizes, - maxEncoderInputLengthInBatch, stream); - } - sync_check_cuda_error(stream.get()); - - // The packed mask is only needed by context requests now. - if (!contextRequests.empty()) - { - // Set the parameters for creating packed mask for context FMHA. - tk::PackedMaskParams<bool> maskParams{}; - maskParams.maskInput = bufferCastOrNull<bool>(crossAttentionMaskDevice); - maskParams.cuQSeqLens = bufferCastOrNull<SizeType32>(crossAttentionCuQSeqLensDevice); - maskParams.packedMask = bufferCastOrNull<uint32_t>(crossAttentionPackedMaskDevice); - maskParams.cuMaskRows = bufferCastOrNull<SizeType32>(crossAttentionPackedMaskCuMaskRowsDevice); - maskParams.actualQSeqLens = bufferCastOrNull<SizeType32>(decoderContextLengthsDevice); - maskParams.actualKvSeqLens = bufferCastOrNull<SizeType32>(encoderInputLengths); - maskParams.batchSize = contextRequests.size(); - maskParams.maxQSeqLen = maxDecoderContextLength; - maskParams.maxKvSeqLen = maxEncoderInputLengthInBatch; - maskParams.attentionMaskType = tk::ContextAttentionMaskType::CUSTOM_MASK; - maskParams.validPosVal = true; - - // Launch the pack mask kernel. - tk::invokeBuildPackedMask(maskParams, stream.get()); - sync_check_cuda_error(stream.get()); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TransformerBuffers::copySkipCrossAttnBlocks(bool const& _skipCrossAttnBlocks, runtime::TllmRuntime const& runtime) -{ - auto const& manager = runtime.getBufferManager(); - manager.copy(&_skipCrossAttnBlocks, *skipCrossAttnBlocks); -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtEncoderModel.cpp b/cpp/tensorrt_llm/batch_manager/trtEncoderModel.cpp deleted file mode 100644 index 0d7dbfde42e6..000000000000 --- a/cpp/tensorrt_llm/batch_manager/trtEncoderModel.cpp +++ /dev/null @@ -1,618 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "trtEncoderModel.h" -#include "encoderBuffers.h" -#include "tensorrt_llm/batch_manager/capacityScheduler.h" -#include "tensorrt_llm/batch_manager/microBatchScheduler.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" -#include "tensorrt_llm/runtime/utils/runtimeUtils.h" - -#include <algorithm> -#include <cstddef> -#include <vector> - -using namespace tensorrt_llm::runtime; -using namespace tensorrt_llm::mpi; - -namespace tensorrt_llm::batch_manager -{ - -TrtEncoderModel::TrtEncoderModel(runtime::ModelConfig const& modelConfig, WorldConfig const& worldConfig, - runtime::RawEngine const& rawEngine, std::shared_ptr<nvinfer1::ILogger> logger, - executor::ExecutorConfig const& executorConfig) - : TrtGptModel(modelConfig, worldConfig, executorConfig) - , mModelConfig{modelConfig} - , mWorldConfig{worldConfig} - , mDevice{runtime::utils::initDevice(worldConfig)} - , mLogger{logger ? std::move(logger) : std::make_shared<TllmLogger>()} - , mRuntime{std::make_shared<TllmRuntime>( - rawEngine, mLogger.get(), executorConfig.getUseGpuDirectStorage(), executorConfig.getGpuWeightsPercent())} - , mNumMicroBatches{1} - , mNumBuffers{mNumMicroBatches} - , mCopyBufferManager{std::make_shared<CudaStream>()} -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - TLLM_CHECK_WITH_INFO( - !mWorldConfig.isPipelineParallel(), "Pipeline parallelism is currently not supported for encoder models."); - - createRuntimeContexts(); - - createBuffers(); - - if (mWorldConfig.isPipelineParallel()) - { - auto const& commSession = COMM_SESSION; - mMpiCommPipelinePara = std::make_shared<tensorrt_llm::mpi::MpiComm>( - commSession.split(mWorldConfig.getTensorParallelRank(), mWorldConfig.getPipelineParallelRank())); - } - - mMicroBatchScheduledRequests.resize(mNumMicroBatches); - // mEncoderWaitEvents.resize(mNumMicroBatches); - - // set noScheduleUntilState to LlmRequestState::kENCODER_INIT for encoder model - // when null kv cache manager is given, request scheduler will use MaxRequests as capacity scheduler, i.e. no - // handling of maximizing utilization or pause/evict - // TODO: finer control on encoder requests scheduling - mCapacityScheduler = std::make_unique<tensorrt_llm::batch_manager::CapacityScheduler>( - getMaxBatchSize() * mNumMicroBatches, executorConfig.getSchedulerConfig().getCapacitySchedulerPolicy(), - /*hasKvCacheManager=*/false, /*twoStepsLookAhead=*/false, - /*noScheduleUntilState=*/LlmRequestState::kENCODER_INIT, - /*noScheduleAfterState=*/LlmRequestState::kCONTEXT_INIT, - /*enablePrefixAwareScheduling=*/executorConfig.getSchedulerConfig().getEnablePrefixAwareScheduling()); - - mMicroBatchScheduler = std::make_unique<tensorrt_llm::batch_manager::MicroBatchScheduler>( - std::nullopt, mModelConfig.getMaxInputLen(), LlmRequestState::kENCODER_INIT, LlmRequestState::kCONTEXT_INIT); - - mHiddenSize = modelConfig.getHiddenSize(); - - mMaxInputLen = mModelConfig.getMaxInputLen(); - TLLM_LOG_INFO("TRTEncoderModel mMaxInputLen: reset to %d from build config.", mMaxInputLen); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -BufferManager const& TrtEncoderModel::getBufferManager() const -{ - return mRuntime->getBufferManager(); -} - -BufferManager::CudaStreamPtr TrtEncoderModel::getRuntimeStreamPtr() const -{ - return mRuntime->getStreamPtr(); -} - -nvinfer1::DataType TrtEncoderModel::getTensorDataType(std::string const& name) const -{ - auto const& engine = mRuntime->getEngine(); - return engine.getTensorDataType(name.c_str()); -} - -nvinfer1::Dims TrtEncoderModel::getTensorShape(std::string const& name) const -{ - auto const& engine = mRuntime->getEngine(); - return engine.getTensorShape(name.c_str()); -} - -void TrtEncoderModel::getCurrentIterationStats(executor::IterationStats& stats) const -{ - stats.iter = mIterCounter; -} - -void TrtEncoderModel::getCurrentRequestStats(executor::RequestStatsPerIteration& stats) const -{ - stats.iter = mIterCounter; -} - -executor::DebugTensorsPerIteration TrtEncoderModel::getCurrentDebugTensors() const -{ - executor::DebugTensorsPerIteration debugTensors; - debugTensors.iter = mIterCounter; - - TLLM_LOG_WARNING("TrtEncoderModel doesn't support getting debug tensors."); - - return debugTensors; -} - -void TrtEncoderModel::setLayerProfiler() -{ - TLLM_CHECK(mRuntime); - mRuntime->setLayerProfiler(); -} - -std::string TrtEncoderModel::getLayerProfileInfo() const -{ - TLLM_CHECK(mRuntime); - return mRuntime->getLayerProfileInfo(); -} - -void TrtEncoderModel::createRuntimeContexts() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - mRuntime->clearContexts(); - auto const numProfiles = mRuntime->getNbProfiles(); - TLLM_CHECK_WITH_INFO(numProfiles == 1, "Encoder only expects one optimization profile"); - for (auto i = 0; i < numProfiles; ++i) - { - mRuntime->addContext(i); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtEncoderModel::executeContext(SizeType32 runtimeContextId) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(executeContext); - auto enqueueSuccessful = mRuntime->executeContext(runtimeContextId); - if (!enqueueSuccessful) - { - throw std::runtime_error("Executing TRT engine failed!"); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtEncoderModel::createBuffers() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - for (SizeType32 i = 0; i < mNumBuffers; ++i) - { - mBuffers.emplace_back( - std::make_shared<EncoderBuffers>(getMaxBatchSize(), mModelConfig, mWorldConfig, *mRuntime)); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtEncoderModel::executeBatch(ScheduledRequests const& scheduledRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(executeBatch); - - // encoder model only have one optimization profile for now, so no optimization profile switch - SizeType32 optProfileIndex = 0; - auto const bufferId = getBufferId(); - if (!scheduledRequests.contextRequests.empty()) - { - // engine I/O - auto [inputMap, outputMap] - = mBuffers[bufferId]->prepareIO(scheduledRequests.contextRequests, mModelConfig, mWorldConfig, *mRuntime); - mRuntime->setInputTensors(optProfileIndex, inputMap); - mRuntime->setOutputTensors(optProfileIndex, outputMap); - - // engine run - executeContext(optProfileIndex); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtEncoderModel::rearrangeOutputs(ScheduledRequests const& scheduledRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(rearrangeOutputs); - - auto const bufferId = getBufferId(); - if (!scheduledRequests.contextRequests.empty()) - { - mBuffers[bufferId]->rearrangeOutputs(scheduledRequests.contextRequests, mModelConfig, mWorldConfig, *mRuntime); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtEncoderModel::forwardSync() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE_WITH_NAME(range, "TrtEncoderModel::forwardSync"); - - auto const device = mWorldConfig.getDevice(); - TLLM_CUDA_CHECK(cudaSetDevice(device)); - - auto& currRequests = mMicroBatchScheduledRequests.at(mMicroBatchId); - // auto& encoderWaitEvent = mEncoderWaitEvents.at(mMicroBatchId); - - if (!currRequests.empty()) - { - if (!mWorldConfig.isPipelineParallel() || !mWorldConfig.isLastPipelineParallelRank()) - { - // TLLM_CHECK_WITH_INFO(mEncStepAsyncSndHdl.get() == nullptr, "encoderSync handle must be nullptr."); - // // Wait for encoding for requests in flight for the current micro batch - // mEncStepAsyncSndHdl = encoderSync(currRequests, encoderWaitEvent); - } - else - { - } - - NVTX3_SCOPED_RANGE(pauseFlaggedCurrRequests); - for (auto const& requests : {currRequests.contextRequests}) - { - for (auto const& llmReq : requests) - { - auto const reqId = llmReq->mRequestId; - mInflightReqIds.erase(reqId); - TLLM_LOG_DEBUG("request ID %u removed from ENCODER inflight set", reqId); - - // If a request in encoder phase had been flagged to be paused, pause it right away - if (mReqIdsToPause.find(reqId) != mReqIdsToPause.end()) - { - terminateRequest(llmReq, true); - mReqIdsToPause.erase(reqId); - } - } - } - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtEncoderModel::forwardAsync(RequestList const& activeRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE_WITH_NAME(range, "TrtEncoderModel::ForwardAsync"); - auto const device = mWorldConfig.getDevice(); - TLLM_CUDA_CHECK(cudaSetDevice(device)); - - try - { - auto& currRequests = mMicroBatchScheduledRequests.at(mMicroBatchId); - // auto& encoderWaitEvent = mEncoderWaitEvents.at(mMicroBatchId); - - // Get a new set of requests for encoder - // The scheduler will not include any requests that are already in flight for encoder models - // TODO: add pause handling logic - TLLM_LOG_DEBUG("Running ENCODER request scheduler"); - - auto [fittingRequests, fittingDisaggeGenInitReuqests, requestsToPause] = (*mCapacityScheduler)(activeRequests); - - TLLM_CHECK_WITH_INFO( - fittingDisaggeGenInitReuqests.empty(), "Disaggregated servering is not support by encoder model."); - - std::tie(currRequests.contextRequests, std::ignore) = (*mMicroBatchScheduler)( - fittingRequests, mInflightReqIds, getMaxBatchSize(), mModelConfig.getMaxNumTokens()); - - { - NVTX3_SCOPED_RANGE(pauseRequestsFlaggedByScheduler); - // Loop over requests flagged to be paused, and if not in flight pause it right away - for (auto const& llmReq : requestsToPause) - { - auto const reqId = llmReq->mRequestId; - if (mInflightReqIds.find(reqId) == mInflightReqIds.end()) - { - // Not in flight, can terminate right away - terminateRequest(llmReq, true); - } - else - { - // In flight, add to set for pausing later - mReqIdsToPause.insert(reqId); - } - } - } - - TLLM_CHECK(currRequests.size() <= static_cast<size_t>(getMaxBatchSize())); - - if (!currRequests.empty()) - { - TLLM_LOG_DEBUG("Running ENCODER model with batch size: %u", currRequests.size()); - { - NVTX3_SCOPED_RANGE(updateInflightReqIds); - // Add to set of requests in flight - for (auto const& requests : {currRequests.contextRequests}) - { - for (auto const& llmReq : requests) - { - TLLM_LOG_DEBUG("request ID %u added to ENCODER inflight set", llmReq->mRequestId); - mInflightReqIds.insert(llmReq->mRequestId); - } - } - } - - executeBatch(currRequests); - - sync_check_cuda_error(mRuntime->getStream().get()); - - rearrangeOutputs(currRequests); - - sync_check_cuda_error(mRuntime->getStream().get()); - - // encoderWaitEvent = encoderStepAsync(currRequests); - - for (auto const& requests : {currRequests.contextRequests}) - { - for (auto const& llmReq : requests) - { - if (llmReq->isEncoderInitState()) - { - llmReq->setState(LlmRequestState::kCONTEXT_INIT); - TLLM_LOG_DEBUG("request ID: %u finishes encoder phase", llmReq->mRequestId); - } - } - } - } - - // TODO: PP handling - if (!currRequests.empty()) - { - if (mWorldConfig.isPipelineParallel() && mWorldConfig.isLastPipelineParallelRank()) - { - // TLLM_CHECK_WITH_INFO(mEncStepAsyncSndHdl.get() == nullptr, "decoderSync handle must be nullptr."); - // Wait for encoding for requests in flight for the current micro batch - // mEncStepAsyncSndHdl = encoderSync(currRequests, encoderWaitEvent); - } - } - - // Update the micro batch ID - mMicroBatchId = (mMicroBatchId + 1) % mNumMicroBatches; - } - // In case of error, we need to free the batch slot associated with those requests - catch (std::exception const& e) - { - for (auto const& llmReq : activeRequests) - { - terminateRequest(llmReq); - } - throw; - } - - ++mIterCounter; - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtEncoderModel::terminateRequest(std::shared_ptr<LlmRequest> const& llmReq, bool pause) -{ - // For encoder-only models, just change req state here. might need to do more when using an asynced forward - // For enc-dec models, only remove cross kv cache after decoder - // genenration has finished - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - if (llmReq->isEncoderInitState()) - { - llmReq->setState(LlmRequestState::kCONTEXT_INIT); - } - else - { - TLLM_LOG_DEBUG("Non-encoder request terminated in encoder model: id %lu", llmReq->mRequestId); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtEncoderModel::terminateRequestSync( - std::shared_ptr<LlmRequest> const& llmReq, executor::FinishReason finishReason) -{ - terminateRequest(llmReq, false); - llmReq->finishByReason(finishReason); - llmReq->clearGeneratedTokens(); -} - -void TrtEncoderModel::fillEncoderOutputSync(RequestVector const& requestList, TensorMap outputTensors) -{ - auto const totalTokensNb = outputTensors["encoder_output"]->getShape().d[0]; - auto const encoderOutputDtype = mRuntime->getEngine().getTensorDataType("encoder_output"); - SizeType32 const bytesPerValue = (encoderOutputDtype == nvinfer1::DataType::kFLOAT) ? 4 : 2; - std::vector<std::byte> encoderOutputHost( - totalTokensNb * mHiddenSize * bytesPerValue * mWorldConfig.getTensorParallelism()); - TLLM_CHECK_WITH_INFO(encoderOutputHost.size() > 0, "Encoder output size is 0!"); - getBufferManager().copy(*(outputTensors["encoder_output"]), reinterpret_cast<void*>(encoderOutputHost.data())); - getBufferManager().getStream().synchronize(); // TODO: change engine call to async to improve perf. Also - // need to store output buffers, cuda events, etc. - - auto encoderOutputHostPtr = encoderOutputHost.data(); - for (auto const& llmReq : requestList) - { - SizeType32 const seqLen = llmReq->getEncoderOutputLen(); - TensorPtr currentEncoderOutput - = mCopyBufferManager.copyFrom(reinterpret_cast<half const*>(encoderOutputHostPtr), - ITensor::makeShape({seqLen, mHiddenSize * mWorldConfig.getTensorParallelism()}), MemoryType::kCPU); - llmReq->setEncoderOutputHost(currentEncoderOutput); - encoderOutputHostPtr += seqLen * mHiddenSize * bytesPerValue * mWorldConfig.getTensorParallelism(); - - if (llmReq->isEncoderInitState()) - { - llmReq->setState(LlmRequestState::kCONTEXT_INIT); - } - else - { - TLLM_LOG_DEBUG("Non-encoder request terminated in encoder model: id %lu", llmReq->mRequestId); - } - } -} - -void TrtEncoderModel::executeBatch(RequestVector const& requestList) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(executeBatch); - - auto const modelName = mModelConfig.getModelName(); - TLLM_CHECK_WITH_INFO(modelName == "EncoderModel" || modelName == "WhisperEncoder", "Model not supported."); - TensorMap inputTensors; - TensorMap outputTensors; - TensorPtr rankOutput; - - std::vector<TokenIdType> inputIdsHost; - std::vector<SizeType32> positionIdsHost; - SizeType32 totalOutputLength = 0; - SizeType32 totalInputLength = 0; - std::vector<SizeType32> inputLengthsHost; - std::vector<std::byte> inputFeaturesHost; - - inputLengthsHost.reserve(requestList.size()); - SizeType32 maxInputLengthHost = 0; - - for (auto const& llmReq : requestList) - { - SizeType32 length = 0; - if (mModelConfig.getModelName() == "EncoderModel") - { - auto const& reqTokens = *(llmReq->getEncoderTokens().value()); - length = reqTokens.size(); - - inputIdsHost.insert(inputIdsHost.end(), reqTokens.begin(), reqTokens.end()); - maxInputLengthHost = std::max(maxInputLengthHost, static_cast<SizeType32>(length)); - } - else if (mModelConfig.getModelName() == "WhisperEncoder") - { - auto const& reqFeatures = llmReq->getEncoderInputFeatures(); // [length, featureDim] - length = reqFeatures->getShape().d[0]; - - auto const curFeatureBytes = reqFeatures->getSizeInBytes(); - auto const srcPtr = reinterpret_cast<std::byte*>(reqFeatures->data()); - inputFeaturesHost.insert(inputFeaturesHost.end(), srcPtr, srcPtr + curFeatureBytes); - } - positionIdsHost.reserve(positionIdsHost.size() + length); - auto const newReqPosBegin = positionIdsHost.end(); - positionIdsHost.resize(positionIdsHost.size() + length); - std::iota(newReqPosBegin, positionIdsHost.end(), 0); - - totalOutputLength += llmReq->getEncoderOutputLen(); - totalInputLength += length; - inputLengthsHost.push_back(length); - } - - TensorPtr hiddenStatesInput; - TensorPtr inputLengths = getBufferManager().copyFrom( - inputLengthsHost, ITensor::makeShape({static_cast<SizeType32>(inputLengthsHost.size())}), MemoryType::kGPU); - inputTensors.emplace("input_lengths", inputLengths); - - if (mModelConfig.getModelName() == "EncoderModel") - { - // use shape of maxInputLength to indicates max length, content is not important - TensorPtr maxInputLength - = getBufferManager().gpu(ITensor::makeShape({maxInputLengthHost}), nvinfer1::DataType::kINT32); - inputTensors.emplace("max_input_length", maxInputLength); - } - - // engine outputs - rankOutput = getBufferManager().gpu( - ITensor::makeShape({totalOutputLength, mHiddenSize * mWorldConfig.getTensorParallelism()}), - mModelConfig.getDataType()); - - if (mWorldConfig.isFirstPipelineParallelRank()) - { - if (mModelConfig.getModelName() == "EncoderModel") - { - // Engine inputs - TensorPtr inputIds - = getBufferManager().copyFrom(inputIdsHost, ITensor::makeShape({totalInputLength}), MemoryType::kGPU); - TensorPtr positionIds = getBufferManager().copyFrom( - positionIdsHost, ITensor::makeShape({totalInputLength}), MemoryType::kGPU); - inputTensors.emplace("input_ids", inputIds); - inputTensors.emplace("position_ids", positionIds); - } - else if (mModelConfig.getModelName() == "WhisperEncoder") - { - auto inputFeaturesHostPtr = inputFeaturesHost.data(); - auto const featureDim = requestList.front()->getEncoderInputFeatures()->getShape().d[1]; - auto const dtype = requestList.front()->getEncoderInputFeatures()->getDataType(); - TensorPtr inputFeatures = getBufferManager().gpu(ITensor::makeShape({totalInputLength, featureDim}), dtype); - getBufferManager().copy( - reinterpret_cast<void const*>(inputFeaturesHostPtr), *inputFeatures, runtime::MemoryType::kCPU); - TensorPtr positionIds = getBufferManager().copyFrom( - positionIdsHost, ITensor::makeShape({totalOutputLength}), MemoryType::kGPU); - inputTensors.emplace("input_features", inputFeatures); - inputTensors.emplace("position_ids", positionIds); - } - } - else - { - SizeType32 length = mModelConfig.getModelName() == "WhisperEncoder" ? totalOutputLength : totalInputLength; - hiddenStatesInput - = getBufferManager().gpu(ITensor::makeShape({length, mHiddenSize * mWorldConfig.getTensorParallelism()}), - mModelConfig.getDataType()); - - inputTensors.emplace("hidden_states_input", hiddenStatesInput); - } - - auto const outputName = mWorldConfig.isLastPipelineParallelRank() ? "encoder_output" : "hidden_states_output"; - outputTensors.emplace(outputName, rankOutput); - - // Set input / output tensors to context, encoder model only have one context - mRuntime->setInputTensors(0, inputTensors); - mRuntime->setOutputTensors(0, outputTensors); - - executeContext(0); - - // copy encoder output to llmRequest, if last PP rank - // dispatch result to each llmReq, only needed by the last PP rank - // TODO: more dtypes support - if (mWorldConfig.isLastPipelineParallelRank()) - { - fillEncoderOutputSync(requestList, outputTensors); - } - else - { - getBufferManager().getStream().synchronize(); - } - - // Update the micro batch ID for next microbatches - mMicroBatchId = (mMicroBatchId + 1) % mWorldConfig.getPipelineParallelism(); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtEncoderModel::forward(RequestVector& activeRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - auto const device = mWorldConfig.getDevice(); - TLLM_CUDA_CHECK(cudaSetDevice(device)); - - try - { - if (activeRequests.empty()) - { - return; - } - - executeBatch(activeRequests); - } - catch (std::exception const& e) - { - for (auto& req : activeRequests) - { - terminateRequest(req); - } - throw; - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtEncoderModel::setLogitsPostProcessorBatched( - std::optional<LogitsPostProcessorBatched> logitsPostProcessorBatched) -{ - TLLM_CHECK_WITH_INFO(!logitsPostProcessorBatched.has_value(), "TrtEncoderModel does not use logits processor."); -} - -void TrtEncoderModel::setReplicateLogitsPostProcessor(bool replicateLogitsPostProcessor) -{ - TLLM_THROW("TrtEncoderModel does not use logits processor."); -} - -bool TrtEncoderModel::getReplicateLogitsPostProcessor() const -{ - TLLM_THROW("TrtEncoderModel does not use logits processor."); -} - -TrtEncoderModel::~TrtEncoderModel() = default; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtEncoderModel.h b/cpp/tensorrt_llm/batch_manager/trtEncoderModel.h deleted file mode 100644 index 31f7d3d0c89b..000000000000 --- a/cpp/tensorrt_llm/batch_manager/trtEncoderModel.h +++ /dev/null @@ -1,205 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "trtGptModel.h" - -#include <NvInferRuntime.h> - -namespace tensorrt_llm::runtime -{ -class TllmRuntime; -class NcclCommunicator; -} // namespace tensorrt_llm::runtime - -namespace tensorrt_llm::batch_manager -{ -class CapacityScheduler; -class MicroBatchScheduler; -class EncoderBuffers; - -class TrtEncoderModel : public TrtGptModel -{ -public: - using SizeType32 = tensorrt_llm::runtime::SizeType32; - using TokenIdType = tensorrt_llm::runtime::TokenIdType; - using BufferManager = tensorrt_llm::runtime::BufferManager; - using TensorMap = runtime::StringPtrMap<runtime::ITensor>; - using TensorPtr = runtime::ITensor::SharedPtr; - - TrtEncoderModel(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - runtime::RawEngine const& rawEngine, std::shared_ptr<nvinfer1::ILogger> logger, - executor::ExecutorConfig const& executorConfig); - - ~TrtEncoderModel() override; - - void terminateRequest(std::shared_ptr<LlmRequest> const& llmRequest, bool pause = false) override; - void terminateRequestSync( - std::shared_ptr<LlmRequest> const& llmRequest, executor::FinishReason finishReason) override; - - void forward(RequestVector& activeRequests); - - void forwardSync() override; - - void forwardAsync(RequestList const& activeRequests) override; - - [[nodiscard]] runtime::BufferManager const& getBufferManager() const override; - [[nodiscard]] runtime::BufferManager::CudaStreamPtr getRuntimeStreamPtr() const override; - - runtime::ModelConfig const& getModelConfig() const override - { - return mModelConfig; - } - - [[nodiscard]] bool getGatherGenerationLogits() const override - { - return getModelConfig().computeGenerationLogits(); - } - - runtime::WorldConfig const& getWorldConfig() const override - { - return mWorldConfig; - } - - [[nodiscard]] SizeType32 getHiddenSize() const override - { - return mHiddenSize; - } - - [[nodiscard]] SizeType32 getMaxInputLen() const override - { - return mMaxInputLen; - } - - [[nodiscard]] SizeType32 getNumMicroBatches() const override - { - return mNumMicroBatches; - } - - [[nodiscard]] nvinfer1::DataType getLogitDataType() const override - { - return getModelConfig().getDataType(); - } - - nvinfer1::DataType getTensorDataType(std::string const& name) const override; - nvinfer1::Dims getTensorShape(std::string const& name) const override; - - [[nodiscard]] TrtGptModelType getModelType() const override - { - throw std::runtime_error("TrtEncoderModel does not have model type."); // FIXME: - } - - [[nodiscard]] executor::IterationType getIterCounter() const noexcept override - { - return mIterCounter; - } - - void updatePeftCache(std::shared_ptr<LlmRequest> const& /*llmRequest*/) override - { - throw std::runtime_error("TrtEncoderModel does not have Peft Cache."); - } - - void getCurrentIterationStats(executor::IterationStats& stats) const override; - void getCurrentRequestStats(executor::RequestStatsPerIteration& stats) const override; - [[nodiscard]] executor::DebugTensorsPerIteration getCurrentDebugTensors() const override; - - void setLayerProfiler() override; - std::string getLayerProfileInfo() const override; - - void setLogitsPostProcessorBatched(std::optional<LogitsPostProcessorBatched> logitsPostProcessorBatched) override; - void setReplicateLogitsPostProcessor(bool replicateLogitsPostProcessor) override; - [[nodiscard]] bool getReplicateLogitsPostProcessor() const override; - - void resetIterationStats() override {} - - [[nodiscard]] SizeType32 getMaxCapacityBatchSize(SizeType32 inputLength, SizeType32 outputLength) const override - { - return 0; - }; - -protected: - std::shared_ptr<kv_cache_manager::BaseKVCacheManager> getKVCacheManager() override - { - throw std::runtime_error("TrtEncoderModel does not have KVCache."); - } - - [[nodiscard]] std::shared_ptr<kv_cache_manager::BaseKVCacheManager const> getKVCacheManager() const override - { - throw std::runtime_error("TrtEncoderModel does not have KVCache."); - } - - [[nodiscard]] std::shared_ptr<BasePeftCacheManager> getPeftCacheManager() override - { - throw std::runtime_error("TrtEncoderModel does not use PEFT."); - } - - [[nodiscard]] std::shared_ptr<BasePeftCacheManager const> getPeftCacheManager() const override - { - throw std::runtime_error("TrtEncoderModel does not use PEFT."); - } - -private: - [[nodiscard]] SizeType32 getBufferId() const - { - return mMicroBatchId; - } - - void createRuntimeContexts(); - void executeContext(SizeType32 runtimeContextId); - void createBuffers(); - void executeBatch(RequestVector const& requestList); - void executeBatch(ScheduledRequests const& scheduledRequests); - void rearrangeOutputs(ScheduledRequests const& scheduledRequests); - void createCustomAllReduceWorkspace(); - void fillEncoderOutputSync(RequestVector const& requestList, TensorMap outputTensors); - - runtime::ModelConfig const mModelConfig; - runtime::WorldConfig const mWorldConfig; - int mDevice{-1}; - std::shared_ptr<tensorrt_llm::mpi::MpiComm> mMpiCommPipelinePara; - - std::shared_ptr<nvinfer1::ILogger> mLogger; - std::shared_ptr<runtime::TllmRuntime> mRuntime; - - SizeType32 mMicroBatchId{0}; - - // TODO: Add runtime buffers for async PP - std::vector<std::shared_ptr<EncoderBuffers>> mBuffers; - - SizeType32 mNumMicroBatches; - SizeType32 mNumBuffers; - - std::vector<ScheduledRequests> mMicroBatchScheduledRequests; - ReqIdsSet mInflightReqIds; - ReqIdsSet mReqIdsToPause; - - std::unique_ptr<tensorrt_llm::batch_manager::CapacityScheduler const> mCapacityScheduler; - std::unique_ptr<tensorrt_llm::batch_manager::MicroBatchScheduler const> mMicroBatchScheduler; - - SizeType32 mHiddenSize; // already divided by Tensor Parallelism - SizeType32 mMaxInputLen; // WAR for max_input_len == max_seq_len at all circumstances - - runtime::BufferManager mCopyBufferManager; - - // Iteration counter used to distinguish debug output - executor::IterationType mIterCounter{0}; -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtGptModel.h b/cpp/tensorrt_llm/batch_manager/trtGptModel.h deleted file mode 100644 index 54ad36b13895..000000000000 --- a/cpp/tensorrt_llm/batch_manager/trtGptModel.h +++ /dev/null @@ -1,339 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/peftCacheManager.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/stlUtils.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/model.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -#include <memory> - -namespace tc = tensorrt_llm::common; - -namespace tensorrt_llm::batch_manager -{ -enum class TrtGptModelType -{ - InflightBatching, - InflightFusedBatching -}; - -class LlmRequest; - -namespace kv_cache_manager -{ -class BaseKVCacheManager; -} // namespace kv_cache_manager - -class TrtGptModel : public executor::Model -{ -public: - using SizeType32 = tensorrt_llm::runtime::SizeType32; - - TrtGptModel(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - executor::ExecutorConfig const& executorConfig) - : mMaxBatchSize{executorConfig.getMaxBatchSize().value_or(modelConfig.getMaxBatchSize())} - , mMaxBeamWidth{executorConfig.getMaxBeamWidth()} - , mMaxSequenceLen{modelConfig.getMaxSequenceLen()} - , mMaxDraftLen{modelConfig.getMaxDecodingDraftTokens()} - , mVocabSizePadded{modelConfig.getVocabSizePadded(worldConfig.getSize())} - , mNormalizeLogProbs{executorConfig.getNormalizeLogProbs()} - , mEnableTrtOverlap{executorConfig.getEnableTrtOverlap()} - , mCudaGraphMode{executorConfig.getExtendedRuntimePerfKnobConfig().getCudaGraphMode()} - { - TLLM_CHECK_WITH_INFO(mMaxBeamWidth <= modelConfig.getMaxBeamWidth(), - "Runtime configured max beam width (%d) must not exceed engine max beam width (%d)", mMaxBeamWidth, - modelConfig.getMaxBeamWidth()); - TLLM_CHECK_WITH_INFO(mMaxBatchSize <= modelConfig.getMaxBatchSize(), - "Runtime configured max batch size (%d) must not exceed engine max batch size (%d)", mMaxBatchSize, - modelConfig.getMaxBatchSize()); - if (executorConfig.getEnableTrtOverlap()) - { - if (mMaxBeamWidth > 1) - { - mEnableTrtOverlap = false; - TLLM_LOG_WARNING( - "TRT overlap is not supported with beam search (maxBeamWidth is set to %d) and will be disabled.", - mMaxBeamWidth); - } - if (!modelConfig.getSpeculativeDecodingMode().isNone()) - { - mEnableTrtOverlap = false; - TLLM_LOG_WARNING("TRT overlap is not supported with speculative decoding and will be disabled."); - } - } - - mMaxAttentionWindow = 0; - if (executorConfig.getKvCacheConfig().getMaxAttentionWindowVec().has_value()) - { - bool warning = false; - auto const& maxAttentionWindowVec = executorConfig.getKvCacheConfig().getMaxAttentionWindowVec(); - for (int maxAttenWin : maxAttentionWindowVec.value()) - { - mMaxAttentionWindowVec.push_back(std::min(maxAttenWin, mMaxSequenceLen)); - mMaxAttentionWindow = std::max(mMaxAttentionWindow, mMaxAttentionWindowVec.back()); - if (maxAttenWin > mMaxSequenceLen) - { - warning = true; - } - TLLM_CHECK_WITH_INFO(mMaxAttentionWindowVec.back() > 0, - "Attention window sizes (elements in maxAttentionWindowVec) must be > 0"); - } - if (warning) - { - TLLM_LOG_WARNING( - "The value of maxAttentionWindow cannot exceed mMaxSequenceLen. " - "Therefore, it has been adjusted to match the value of mMaxSequenceLen."); - } - } - else - { - mMaxAttentionWindowVec.push_back(mMaxSequenceLen); - mMaxAttentionWindow = mMaxSequenceLen; - } - - mSinkTokenLen = executorConfig.getKvCacheConfig().getSinkTokenLength().has_value() - ? executorConfig.getKvCacheConfig().getSinkTokenLength().value() - : 0; - - mMaxNumSequences = mMaxBatchSize * worldConfig.getPipelineParallelism(); - - auto const numTotalAttenLayers = modelConfig.getNbAttentionLayers(); - auto const numRepeatsAttenWindow = numTotalAttenLayers / mMaxAttentionWindowVec.size(); - auto const numRemainsAttenWindow = numTotalAttenLayers % mMaxAttentionWindowVec.size(); - std::string attenWindowRemainInfo = numRemainsAttenWindow > 0 - ? " + " + tc::arr2str(mMaxAttentionWindowVec.data(), numRemainsAttenWindow) - : ""; - - TLLM_LOG_INFO("TRTGptModel maxNumSequences: %d", mMaxNumSequences); - TLLM_LOG_INFO("TRTGptModel maxBatchSize: %d", mMaxBatchSize); - TLLM_LOG_INFO("TRTGptModel maxBeamWidth: %d", mMaxBeamWidth); - TLLM_LOG_INFO("TRTGptModel maxSequenceLen: %d", mMaxSequenceLen); - TLLM_LOG_INFO("TRTGptModel maxDraftLen: %d", mMaxDraftLen); - TLLM_LOG_INFO("TRTGptModel mMaxAttentionWindowSize: %s * %d%s", tc::vec2str(mMaxAttentionWindowVec).c_str(), - numRepeatsAttenWindow, attenWindowRemainInfo.c_str()); - TLLM_LOG_INFO("TRTGptModel enableTrtOverlap: %d", mEnableTrtOverlap); - TLLM_LOG_INFO("TRTGptModel normalizeLogProbs: %d", mNormalizeLogProbs); - - mMaxNumTokens = modelConfig.getMaxNumTokens(); - if (executorConfig.getMaxNumTokens().has_value() && mMaxNumTokens) - { - if (executorConfig.getMaxNumTokens().value() > mMaxNumTokens.value()) - { - TLLM_LOG_WARNING( - "Runtime configured max num tokens (%d) is larger than model max num tokens (%d) and will be " - "ignored.", - executorConfig.getMaxNumTokens().value(), mMaxNumTokens.value()); - } - else - { - mMaxNumTokens = executorConfig.getMaxNumTokens(); - } - } - if (mMaxNumTokens) - { - TLLM_LOG_INFO("TRTGptModel maxNumTokens: %d", mMaxNumTokens.value()); - } - - if (executorConfig.getEnableChunkedContext()) - { - mMaxInputLen = mMaxSequenceLen - 1; - TLLM_LOG_INFO( - "TRTGptModel maxInputLen: %d = maxSequenceLen - 1 since chunked context is enabled", mMaxInputLen); - TLLM_LOG_INFO( - "TRTGptModel If model type is encoder, maxInputLen would be reset in trtEncoderModel to maxInputLen: " - "%d = maxSequenceLen.", - mMaxSequenceLen); - } - else if (modelConfig.getContextFMHA() && modelConfig.usePackedInput()) - { - TLLM_CHECK_WITH_INFO( - mMaxNumTokens, "Max number of tokens has to be set for context FMHA and usePackedInput case."); - mMaxInputLen = std::min(mMaxSequenceLen - 1, mMaxNumTokens.value()); - TLLM_LOG_INFO( - "TRTGptModel maxInputLen: %d = min(maxSequenceLen - 1, maxNumTokens) since context FMHA " - "and usePackedInput are enabled", - mMaxInputLen); - TLLM_LOG_INFO( - "TRTGptModel If model type is encoder, maxInputLen would be reset in trtEncoderModel to maxInputLen: " - "min(maxSequenceLen, maxNumTokens)."); - } - else - { - mMaxInputLen = modelConfig.getMaxInputLen(); - TLLM_LOG_INFO("TRTGptModel maxInputLen: %d = max_input_len (in trtllm-build args)", mMaxInputLen); - } - - using tensorrt_llm::common::stl_utils::toString; - - TLLM_LOG_INFO("Capacity Scheduler Policy: %s", - toString(executorConfig.getSchedulerConfig().getCapacitySchedulerPolicy()).c_str()); - TLLM_LOG_INFO("Context Chunking Scheduler Policy: %s", - toString(executorConfig.getSchedulerConfig().getContextChunkingPolicy()).c_str()); - } - - [[nodiscard]] std::optional<SizeType32> getMaxNumTokens() const - { - return mMaxNumTokens; - } - - [[nodiscard]] SizeType32 getMaxNumSequences() const override - { - return mMaxNumSequences; - } - - [[nodiscard]] SizeType32 getMaxBatchSize() const - { - return mMaxBatchSize; - } - - [[nodiscard]] SizeType32 getMaxInputLen() const override - { - return mMaxInputLen; - } - - [[nodiscard]] SizeType32 getHiddenSize() const override - { - return getModelConfig().getHiddenSize(); - }; - - [[nodiscard]] SizeType32 getMaxSequenceLen() const override - { - return mMaxSequenceLen; - } - - [[nodiscard]] virtual TrtGptModelType getModelType() const = 0; - - [[nodiscard]] SizeType32 getVocabSizePadded() const override - { - return mVocabSizePadded; - } - - [[nodiscard]] SizeType32 getMaxDraftLen() const override - { - return mMaxDraftLen; - } - - [[nodiscard]] SizeType32 getOperatingBeamWidth() const override - { - return mMaxBeamWidth; - } - - [[nodiscard]] bool hasSpeculativeDecodingFastLogits() const noexcept override - { - return false; - } - - [[nodiscard]] bool hasGuidedDecoder() const noexcept override - { - return false; - } - - virtual void setLayerProfiler() = 0; - [[nodiscard]] virtual std::string getLayerProfileInfo() const = 0; - - [[nodiscard]] bool hasKVCacheManager() const - { - return getKVCacheManager() != nullptr; - } - -protected: - [[nodiscard]] SizeType32 getMaxBeamWidth() const - { - return mMaxBeamWidth; - } - - [[nodiscard]] std::vector<SizeType32> getMaxAttentionWindowVec() const - { - return mMaxAttentionWindowVec; - } - - [[nodiscard]] SizeType32 getMaxAttentionWindow() const - { - return mMaxAttentionWindow; - } - - [[nodiscard]] SizeType32 getSinkTokenLen() const - { - return mSinkTokenLen; - } - - [[nodiscard]] bool isNormalizeLogProbs() const - { - return mNormalizeLogProbs; - } - - [[nodiscard]] bool isTrtOverlap() const - { - return mEnableTrtOverlap; - } - - [[nodiscard]] bool isCudaGraphMode() const - { - return mCudaGraphMode; - } - - void setMaxAttentionWindowVec(std::vector<SizeType32> const& maxAttentionWindowVec) - { - TLLM_CHECK_WITH_INFO(maxAttentionWindowVec.size() == mMaxAttentionWindowVec.size(), - "The size of maxAttentionWindowVec must match the size of mMaxAttentionWindowVec"); - mMaxAttentionWindowVec = maxAttentionWindowVec; - mMaxAttentionWindow = *std::max_element(std::begin(mMaxAttentionWindowVec), std::end(mMaxAttentionWindowVec)); - } - - void setMaxSequenceLen(SizeType32 maxSequenceLen) - { - mMaxSequenceLen = maxSequenceLen; - } - - void setMaxInputLen(SizeType32 maxInputLen) - { - mMaxInputLen = maxInputLen; - } - - [[nodiscard]] std::shared_ptr<kv_cache_manager::BaseKVCacheManager> getKVCacheManager() override = 0; - [[nodiscard]] std::shared_ptr<kv_cache_manager::BaseKVCacheManager const> getKVCacheManager() const override = 0; - - [[nodiscard]] virtual std::shared_ptr<BasePeftCacheManager> getPeftCacheManager() = 0; - [[nodiscard]] virtual std::shared_ptr<BasePeftCacheManager const> getPeftCacheManager() const = 0; - -private: - std::optional<SizeType32> mMaxNumTokens; - SizeType32 mMaxNumSequences; - SizeType32 mMaxBatchSize; - SizeType32 mMaxBeamWidth; - SizeType32 mMaxInputLen; - SizeType32 mMaxSequenceLen; - SizeType32 mMaxDraftLen; - - SizeType32 mVocabSizePadded; - std::vector<SizeType32> mMaxAttentionWindowVec; - SizeType32 mMaxAttentionWindow; - SizeType32 mSinkTokenLen; - - bool mNormalizeLogProbs; - bool mEnableTrtOverlap; - bool mCudaGraphMode; -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtGptModelFactory.h b/cpp/tensorrt_llm/batch_manager/trtGptModelFactory.h deleted file mode 100644 index bd4d7c767378..000000000000 --- a/cpp/tensorrt_llm/batch_manager/trtGptModelFactory.h +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/trtGptModel.h" -#include "tensorrt_llm/batch_manager/trtGptModelInflightBatching.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/runtime/gptJsonConfig.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -#include <NvInferPlugin.h> - -#include <memory> -#include <optional> - -namespace tensorrt_llm::batch_manager -{ - -class TrtGptModelFactory -{ -public: - using SizeType32 = tensorrt_llm::runtime::SizeType32; - - static std::shared_ptr<TrtGptModel> create(std::filesystem::path const& trtEnginePath, TrtGptModelType modelType, - executor::ExecutorConfig const& executorConfig, bool isLeaderInOrchMode) - { - auto const jsonConfig = runtime::GptJsonConfig::parse(trtEnginePath / "config.json"); - auto const& deviceIds = executorConfig.getParallelConfig().value_or(executor::ParallelConfig()).getDeviceIds(); - auto const worldConfig = getWorldConfig(jsonConfig, deviceIds); - auto const enginePath = trtEnginePath / jsonConfig.engineFilename(worldConfig); - - auto const& modelConfig = jsonConfig.getModelConfig(); - return create( - runtime::RawEngine(enginePath), modelConfig, worldConfig, modelType, executorConfig, isLeaderInOrchMode); - } - - static std::shared_ptr<TrtGptModel> create(std::filesystem::path const& trtEnginePath, TrtGptModelType modelType, - runtime::GptJsonConfig const& jsonConfig, runtime::WorldConfig const& worldConfig, - executor::ExecutorConfig const& executorConfig, bool isLeaderInOrchMode) - { - auto const enginePath = trtEnginePath / jsonConfig.engineFilename(worldConfig); - auto const& modelConfig = jsonConfig.getModelConfig(); - return create( - runtime::RawEngine(enginePath), modelConfig, worldConfig, modelType, executorConfig, isLeaderInOrchMode); - } - - static std::shared_ptr<TrtGptModel> create(runtime::RawEngine const& rawEngine, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, TrtGptModelType modelType, - executor::ExecutorConfig const& executorConfig, bool isLeaderInOrchMode) - { - auto logger = std::make_shared<runtime::TllmLogger>(); - auto const device = worldConfig.getDevice(); - auto const rank = worldConfig.getRank(); - TLLM_LOG_INFO("Rank %d is using GPU %d", rank, device); - TLLM_CUDA_CHECK(cudaSetDevice(device)); - - if ((modelType == TrtGptModelType::InflightBatching) || (modelType == TrtGptModelType::InflightFusedBatching)) - { - executor::ExecutorConfig const& fixedExecutorConfig - = TrtGptModelInflightBatching::executorConfigIsValid(modelConfig, executorConfig) - ? executorConfig - : TrtGptModelInflightBatching::fixExecutorConfig(modelConfig, executorConfig); - bool const ctxGenFusion = modelType == TrtGptModelType::InflightFusedBatching; - return std::make_shared<TrtGptModelInflightBatching>( - logger, modelConfig, worldConfig, rawEngine, ctxGenFusion, fixedExecutorConfig, isLeaderInOrchMode); - } - - throw std::runtime_error("Invalid modelType in trtGptModelFactory"); - } - -private: - static runtime::WorldConfig getWorldConfig( - runtime::GptJsonConfig const& json, std::optional<std::vector<SizeType32>> const& deviceIds) - { - return runtime::WorldConfig::mpi(json.getGpusPerNode(), json.getTensorParallelism(), - json.getPipelineParallelism(), json.getContextParallelism(), deviceIds); - } -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.cpp b/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.cpp deleted file mode 100644 index 7a0d78beb8a0..000000000000 --- a/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.cpp +++ /dev/null @@ -1,3136 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "trtGptModelInflightBatching.h" - -#include "tensorrt_llm/batch_manager/allocateKvCache.h" -#include "tensorrt_llm/batch_manager/assignReqSeqSlots.h" -#include "tensorrt_llm/batch_manager/cacheTransceiver.h" -#include "tensorrt_llm/batch_manager/capacityScheduler.h" -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/batch_manager/contextProgress.h" -#include "tensorrt_llm/batch_manager/createNewDecoderRequests.h" -#include "tensorrt_llm/batch_manager/decoderBuffers.h" -#include "tensorrt_llm/batch_manager/disaggTransferAdmissionController.h" -#include "tensorrt_llm/batch_manager/guidedDecoder.h" -#include "tensorrt_llm/batch_manager/handleContextLogits.h" -#include "tensorrt_llm/batch_manager/handleGenerationLogits.h" -#include "tensorrt_llm/batch_manager/kvCacheEventManager.h" -#include "tensorrt_llm/batch_manager/kvCacheManager.h" -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/batch_manager/logitsPostProcessor.h" -#include "tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h" -#include "tensorrt_llm/batch_manager/microBatchScheduler.h" -#include "tensorrt_llm/batch_manager/pauseRequests.h" -#include "tensorrt_llm/batch_manager/peftCacheManager.h" -#include "tensorrt_llm/batch_manager/promptTuningBuffers.h" -#include "tensorrt_llm/batch_manager/rnnStateManager.h" -#include "tensorrt_llm/batch_manager/runtimeBuffers.h" -#include "tensorrt_llm/batch_manager/sequenceSlotManager.h" -#include "tensorrt_llm/batch_manager/transformerBuffers.h" -#include "tensorrt_llm/batch_manager/updateDecoderBuffers.h" -#include "tensorrt_llm/batch_manager/utils/debugUtils.h" -#include "tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h" -#include "tensorrt_llm/batch_manager/utils/logitsThread.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/common/timestampUtils.h" -#include "tensorrt_llm/kernels/decodingCommon.h" -#include "tensorrt_llm/layers/defaultDecodingParams.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/gptDecoderBatched.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/ipcUtils.h" -#include "tensorrt_llm/runtime/lookaheadModule.h" -#include "tensorrt_llm/runtime/memoryCounters.h" -#include "tensorrt_llm/runtime/runtimeKernels.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "tensorrt_llm/runtime/utils/runtimeUtils.h" - -#include <algorithm> -#include <cstddef> -#include <cstring> -#include <memory> -#include <numeric> -#include <optional> -#include <stdexcept> -#include <thread> -#include <utility> -#include <vector> - -using namespace tensorrt_llm::runtime; -namespace tc = tensorrt_llm::common; -namespace tk = tensorrt_llm::kernels; - -using tensorrt_llm::batch_manager::CacheTransceiverFactory; - -namespace tensorrt_llm::batch_manager -{ - -std::map<SizeType32, SizeType32> TrtGptModelInflightBatching::calculateCacheSizePerTokenForDisagg( - ModelConfig const& modelConfig, WorldConfig const& worldConfig, - std::vector<SizeType32> const& maxAttentionWindowVec, bool isCrossAttention, SizeType32 kvFactor) -{ - // These are the number of attention layers on this PP rank. - auto const numLocalAttnLayers - = modelConfig.getNbAttentionLayers(worldConfig.getPipelineParallelism(), worldConfig.getPipelineParallelRank()); - // These are the number of attention layers on all previous PP ranks. - auto const numLowerRankAttnLayers = modelConfig.countLowerRankLayers(ModelConfig::LayerType::kATTENTION, - worldConfig.getPipelineParallelism(), worldConfig.getPipelineParallelRank()); - // Use global ranks of attention layers to lookup from maxAttentionWindowVec. - auto const startAttnLayerId = numLowerRankAttnLayers; - auto const endAttnLayerId = numLowerRankAttnLayers + numLocalAttnLayers; - auto const numNonUniqueWindowSizes = static_cast<SizeType32>(maxAttentionWindowVec.size()); - std::map<SizeType32, std::vector<SizeType32>> uniqueWindowSizeToLayers; - for (SizeType32 layerIdx = startAttnLayerId; layerIdx < endAttnLayerId; layerIdx++) - { - // maxAttentionWindowVec may or may not be stretched to the length of numLayers yet. - // If not stretched yet, we cycle through the window sizes. - auto const windowSize = maxAttentionWindowVec.at(layerIdx % numNonUniqueWindowSizes); - uniqueWindowSizeToLayers[windowSize].push_back(layerIdx); - } - std::map<SizeType32, SizeType32> cacheSizeBytesPerTokenPerWindow; - for (auto const& [windowSize, globalLayerIds] : uniqueWindowSizeToLayers) - { - auto const nkvh = modelConfig.getNumKvHeadsForGivenLayers(globalLayerIds, isCrossAttention); - auto const sumLocalHeads = std::reduce(nkvh.cbegin(), nkvh.cend()); - auto const cacheSizePerToken = sumLocalHeads * kvFactor * modelConfig.getSizePerHead(); - auto const cacheSizeBytesPerToken = cacheSizePerToken * BufferDataType(modelConfig.getKvDataType()).getSize(); - cacheSizeBytesPerTokenPerWindow[windowSize] = cacheSizeBytesPerToken; - } - - return cacheSizeBytesPerTokenPerWindow; -}; - -bool TrtGptModelInflightBatching::executorConfigIsValid( - ModelConfig const& modelConfig, executor::ExecutorConfig const& executorConfig) -{ - // Make sure logic in this function matches fixExecutorConfig - if (executorConfig.getKvCacheConfig().getEnableBlockReuse()) - { - if (!modelConfig.getPagedContextFMHA()) - { - return false; - } - // Context logits cannot be returned for reused tokens, so disable reuse - if (modelConfig.computeContextLogits()) - { - return false; - } - } - return true; -} - -executor::ExecutorConfig TrtGptModelInflightBatching::fixExecutorConfig( - ModelConfig const& modelConfig, executor::ExecutorConfig const& executorConfig) -{ - // Make sure logic in this function matches executorConfigIsValid - if (executorConfig.getKvCacheConfig().getEnableBlockReuse()) - { - auto kvCacheConfig = executorConfig.getKvCacheConfig(); - - if (!modelConfig.getPagedContextFMHA()) - { - TLLM_LOG_WARNING( - "Fixing executorConfig: KV cache reuse disabled because model was not built with paged context FMHA " - "support"); - kvCacheConfig.setEnableBlockReuse(false); - } - if (modelConfig.computeContextLogits()) - { - TLLM_LOG_WARNING( - "Fixing executorConfig: KV cache reuse disabled because model was built to return context logits"); - kvCacheConfig.setEnableBlockReuse(false); - } - - auto fixedExecutorConfig = executor::ExecutorConfig(executorConfig); - fixedExecutorConfig.setKvCacheConfig(kvCacheConfig); - return fixedExecutorConfig; - } - return executorConfig; -} - -TrtGptModelInflightBatching::TrtGptModelInflightBatching(std::shared_ptr<nvinfer1::ILogger> logger, - ModelConfig const& modelConfig, WorldConfig const& worldConfig, RawEngine const& rawEngine, bool ctxGenFusion, - executor::ExecutorConfig const& executorConfig, bool isLeaderInOrchMode) - : TrtGptModel(modelConfig, worldConfig, executorConfig) - , mModelConfig(modelConfig) - , mWorldConfig(worldConfig) - , mDevice{runtime::utils::initDevice(worldConfig)} - , mDecodingConfig{executorConfig.getDecodingConfig().value_or(executor::DecodingConfig{})} - , mExtendedRuntimePerfKnobConfig{executorConfig.getExtendedRuntimePerfKnobConfig()} - , mDebugConfig{executorConfig.getDebugConfig()} - , mAdditionalModelOutputs{worldConfig.isLastPipelineParallelRank() ? executorConfig.getAdditionalModelOutputs() - : std::nullopt} - , mLogger{logger ? std::move(logger) : std::make_shared<TllmLogger>()} - , mRuntime{std::make_unique<TllmRuntime>(rawEngine, mLogger.get(), executorConfig.getUseGpuDirectStorage(), - executorConfig.getGpuWeightsPercent(), modelConfig.useShapeInference())} - , mCopyBufferManager{std::make_shared<CudaStream>()} - , mCtxGenFusion(ctxGenFusion) - , mOperatingBeamWidth{getMaxBeamWidth()} - , mGatherGenerationLogits{executorConfig.getGatherGenerationLogits()} - , mPromptTableOffloading{executorConfig.getPromptTableOffloading()} - , mIsLeaderInOrchMode{isLeaderInOrchMode} -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - TLLM_LOG_INFO("gatherContextLogits: %d", mModelConfig.computeContextLogits()); - TLLM_LOG_INFO("gatherGenerationLogits: %d", getGatherGenerationLogits()); - - if (!(mModelConfig.supportsInflightBatching())) - { - throw std::runtime_error( - "TrtGptModelInflightBatching requires GPT attention/Mamba Conv 1d plugin with " - "packed input and paged KV cache."); - } - if (mWorldConfig.isTensorParallel()) - { - mRuntime->initializeUserBuffer(mWorldConfig, mModelConfig.getMaxBatchSize(), mModelConfig.getMaxBeamWidth(), - mModelConfig.getMaxSequenceLen(), mModelConfig.getHiddenSize(), getMaxNumTokens()); - } - if (mWorldConfig.isPipelineParallel()) - { - mNumMicroBatches = mWorldConfig.getPipelineParallelism(); - } - else - { - mNumMicroBatches = isTrtOverlap() ? 2 : 1; - } - - mNumBuffers = (mCtxGenFusion ? 1 : 2) * mNumMicroBatches; - - auto const& kvCacheConfig = executorConfig.getKvCacheConfig(); - - if (mModelConfig.getSpeculativeDecodingMode().isDraftTokensExternal()) - { - TLLM_CHECK_WITH_INFO(kvCacheConfig.getEnableBlockReuse(), - "KV cache block reuse must be enabled for speculative decoding target model"); - } - - if (mCtxGenFusion) - { - TLLM_CHECK_WITH_INFO(!mModelConfig.isRnnBased(), "RNN based model doesn't support context generation fusion."); - TLLM_CHECK_WITH_INFO( - mModelConfig.isTransformerBased(), "Only transformer based model support context generation fusion now."); - } - - if (mModelConfig.getSpeculativeDecodingMode().isLookaheadDecoding()) - { - mSeamlessLADMaxDraftLen = modelConfig.getMaxDecodingDraftTokens(); - // TODO: enable it when speculativeDecodingMode is None and run with '--lookahead_config' - mUseSeamlessLookahead = false; - } - - setupSpeculativeDecodingModule(mDecodingConfig); - - if (mWorldConfig.isLastPipelineParallelRank() && executorConfig.getGuidedDecodingConfig()) - { - mGuidedDecoder = std::make_unique<GuidedDecoder>(executorConfig.getGuidedDecodingConfig().value(), - getMaxNumSequences(), mModelConfig.getVocabSizePadded(mWorldConfig.getSize()), - mModelConfig.getLogitsDtype(), mRuntime->getBufferManager()); - } - - createRuntimeContexts(); - - if (mWorldConfig.isTensorParallel()) - { - createCustomAllReduceWorkspace(); - } - - if (mModelConfig.isTransformerBased()) - { - createRuntimePerfKnobsTensor(mExtendedRuntimePerfKnobConfig); - } - - auto& memCounter = MemoryCounters::getInstance(); - auto const gpuUsage1 = memCounter.getGpu(); - createBuffers(mDecodingConfig, mAdditionalModelOutputs); - auto const gpuUsage2 = memCounter.getGpu(); - TLLM_LOG_INFO("[MemUsageChange] Allocated %s GPU memory for runtime buffers.", - memCounter.bytesToString(gpuUsage2 - gpuUsage1).c_str()); - - createDecoder(mDecodingConfig.getDecodingMode()); - auto const gpuUsage3 = memCounter.getGpu(); - TLLM_LOG_INFO("[MemUsageChange] Allocated %s GPU memory for decoder.", - memCounter.bytesToString(gpuUsage3 - gpuUsage2).c_str()); - - if (modelConfig.getManageWeightsType() != ModelConfig::ManageWeightsType::kDisabled) - { - mRuntime->loadManagedWeights(rawEngine, worldConfig.getLocalRank()); - } - - if (mModelConfig.useLoraPlugin()) - { - auto const peftCacheManagerConfig - = PeftCacheManagerConfig(executorConfig.getPeftCacheConfig().value_or(executor::PeftCacheConfig())); - mPeftCacheManager = std::make_shared<PeftCacheManager>( - peftCacheManagerConfig, mModelConfig, mWorldConfig, mRuntime->getBufferManager()); - } - else - { - mPeftCacheManager = std::make_shared<NoOpPeftCacheManager>(); - } - - if (mModelConfig.isRnnBased()) - { - createRnnStateManager(); - } - if (mModelConfig.isTransformerBased() && modelConfig.isKVCacheEnabled()) - { - auto cacheTransceiverConfig - = executorConfig.getCacheTransceiverConfig().value_or(executor::CacheTransceiverConfig()); - - auto const cacheSizeBytesPerTokenPerWindow = calculateCacheSizePerTokenForDisagg( - mModelConfig, mWorldConfig, getMaxAttentionWindowVec(), mModelConfig.useCrossAttention(), 2); - auto cacheTransPreAllocaSize = kv_cache_manager::CacheTransBufferManager::preAllocBufferSize( - cacheSizeBytesPerTokenPerWindow, mModelConfig.getTokensPerBlock(), cacheTransceiverConfig); - - auto const [freePrimaryMemBytes, freeSecondaryMemBytes] - = BaseKVCacheManager::calculateFreeMemBytes(mRuntime->getBufferManager(), kvCacheConfig); - if (mModelConfig.useCrossAttention()) - { - TLLM_CHECK_WITH_INFO(kvCacheConfig.getCrossKvCacheFraction().has_value(), - "Must set crossKvCacheFraction for encoder-decoder model"); - auto const crossKvCacheFraction = kvCacheConfig.getCrossKvCacheFraction().value(); - mKvCacheManager = createKvCacheManager(kvCacheConfig, KvCacheType::kSELF, - freePrimaryMemBytes * (1.0f - crossKvCacheFraction), - freeSecondaryMemBytes * (1.0f - crossKvCacheFraction), cacheTransPreAllocaSize, - executorConfig.getFailFastOnAttentionWindowTooLarge()); - mCrossKvCacheManager = createKvCacheManager(kvCacheConfig, KvCacheType::kCROSS, - freePrimaryMemBytes * crossKvCacheFraction, freeSecondaryMemBytes * crossKvCacheFraction, - cacheTransPreAllocaSize, executorConfig.getFailFastOnAttentionWindowTooLarge()); - TLLM_LOG_INFO("This is an Encoder-Decoder model, set %0.1f cross KV cache fraction based on the config.", - crossKvCacheFraction); - } - else - { - TLLM_CHECK_WITH_INFO(!kvCacheConfig.getCrossKvCacheFraction().has_value(), - "Do not set crossKvCacheFraction for decoder-only model"); - mKvCacheManager = createKvCacheManager(kvCacheConfig, KvCacheType::kSELF, freePrimaryMemBytes, - freeSecondaryMemBytes, cacheTransPreAllocaSize, executorConfig.getFailFastOnAttentionWindowTooLarge()); - } - - mCacheTransceiver - = CacheTransceiverFactory::createCacheTransceiver(mKvCacheManager.get(), mModelConfig, mWorldConfig, - executor::kv_cache::CacheState::AttentionType::kDEFAULT, executorConfig.getCacheTransceiverConfig()); - mDisaggTransferAdmissionController = std::make_unique<DisaggTransferAdmissionController>( - cacheTransceiverConfig.getMaxTokensInBuffer(), mModelConfig.getTokensPerBlock()); - } - - if (mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind()) - { - TLLM_CHECK_WITH_INFO( - mModelConfig.isKVCacheEnabled(), "When needsKVCacheRewind() returns true, KV cache needs to be enabled."); - auto const& blockManager = mKvCacheManager->getBlockManager(); - - TLLM_CHECK_WITH_INFO(blockManager.getNumPools() == 1, - "Rewinding KV cache blocks for models with multiple pools is not supported"); - - // Two "redundant" checks given the pool size check above, but those below don't rely on an implementation - // detail I guess. - TLLM_CHECK_WITH_INFO( - !blockManager.isVariableWindow(), "Rewinding KV cache blocks for variable SWA models isn't supported"); - auto const maxBlocksPerSeq = blockManager.getMaxBlockPerSeqWhenSingleWindowSize(); - - // TODO(oargov): VGQA is not supported, assume all layers have the same num_kv_heads - TLLM_CHECK_WITH_INFO( - !blockManager.isVariableGQA(), "Rewinding KV cache blocks for variable GQA models isn't supported"); - auto const numKvHeads = mModelConfig.getNbKvHeads(0); - - mRewindInputs = RewindInputs{maxBlocksPerSeq, /*isUseOneMoreBlock*/ false, numKvHeads}; - } - - if (mWorldConfig.isPipelineParallel()) - { - mAsyncSendWaitThread = std::make_unique<tensorrt_llm::mpi::MpiWaitThread>( - "asyncSendWaitThread", - [this]() - { - mDecStepAsyncSndHdls.clear(); - mDecSlotAsyncSndHdls.clear(); - }, - [this]() { TLLM_CUDA_CHECK(cudaSetDevice(mWorldConfig.getDevice())); }); - - auto const& commSession = COMM_SESSION; - mMpiCommPipelinePara = std::make_unique<tensorrt_llm::mpi::MpiComm>( - commSession.split(mWorldConfig.getTensorParallelRank(), mWorldConfig.getPipelineParallelRank())); - mDecSlotAsyncSndHdls.reserve(getMaxBatchSize()); - } - if (mWorldConfig.isTensorParallel()) - { - auto const& commSession = COMM_SESSION; - mMpiCommTensorPara = std::make_unique<tensorrt_llm::mpi::MpiComm>( - commSession.split(mWorldConfig.getPipelineParallelRank(), mWorldConfig.getTensorParallelRank())); - } - - mSeqSlotManager - = std::make_shared<SequenceSlotManager>(getMaxNumSequences(), executorConfig.getMaxSeqIdleMicroseconds()); - - mMicroBatchScheduledRequests.resize(mNumMicroBatches); - mDecoderFinishedEvents.resize(mNumMicroBatches); - mPeftTables.resize(mNumMicroBatches); - - if (modelConfig.isRnnBased()) - { - TLLM_CHECK_WITH_INFO(modelConfig.getMaxBeamWidth() == 1, "RNN based model doesn't support beam search now."); - TLLM_CHECK_WITH_INFO( - !executorConfig.getEnableChunkedContext(), "RNN based model doesn't support Chunked Context now."); - TLLM_CHECK_WITH_INFO( - modelConfig.getSpeculativeDecodingMode().isNone(), "RNN based model doesn't support speculative decoding."); - } - - std::optional<batch_scheduler::ContextChunkingConfig> ctxChunkConfig; - if (executorConfig.getEnableChunkedContext()) - { - TLLM_CHECK_WITH_INFO(modelConfig.isKVCacheEnabled() && mModelConfig.getPagedContextFMHA(), - "Chunked context requires context FMHA, paged kv_cache and paged context FMHA all enabled at the same " - "time."); - SizeType32 chunkUnitSize = mKvCacheManager->getTokensPerBlock(); - // If sliding window attention is used, then make sure the unit size aligns with the paged context fmha's kv - // step size. - if (getMaxInputLen() > getMaxAttentionWindow()) // TODO(nhaber): minAttentionWindow - { - chunkUnitSize = std::max(/* maxKvStepSizeInFmha */ 256, chunkUnitSize); - TLLM_LOG_INFO("ChunkUnitSize is set to %d as sliding window attention is used.", chunkUnitSize); - } - ctxChunkConfig = batch_scheduler::ContextChunkingConfig{ - executorConfig.getSchedulerConfig().getContextChunkingPolicy().value_or( - executor::ContextChunkingPolicy::kFIRST_COME_FIRST_SERVED), - chunkUnitSize}; - } - - auto maxNumTokens = getMaxNumTokens(); - TLLM_CHECK_WITH_INFO(maxNumTokens, "Max number of tokens is not set in model config."); - - // Max context size is limited by `max_num_tokens` for chunked-context or context-FMHA, - // or by `max_input_len` of the model. - auto const maxContextLength = (executorConfig.getEnableChunkedContext() || mModelConfig.getContextFMHA()) - ? maxNumTokens - : std::make_optional<SizeType32>(mModelConfig.getMaxInputLen()); - - mMaxBatchSizeTunerRecommended = 0; - mMaxBatchSizeRuntime = getMaxBatchSize(); - mMaxNumTokensStatic = maxNumTokens; - mMaxNumTokensTunerRecommended = 0; - mMaxNumTokensRuntime = maxNumTokens; - - if (mKvCacheManager && ctxChunkConfig) - { - TLLM_CHECK_WITH_INFO(ctxChunkConfig.value().chunkUnitSize % mKvCacheManager->getTokensPerBlock() == 0, - "To prevent cache fragmentation, the context chunk unit size (%d) should be divisible by the number of " - "tokens per kv-cache block (%d).", - ctxChunkConfig.value().chunkUnitSize, mKvCacheManager->getTokensPerBlock()); - } - - mCapacityScheduler = std::make_unique<CapacityScheduler>(getMaxNumSequences(), - executorConfig.getSchedulerConfig().getCapacitySchedulerPolicy(), mKvCacheManager != nullptr, - /*twoStepsLookAhead=*/mWorldConfig.isPipelineParallel(), - /*noScheduleUntilState=*/LlmRequestState::kCONTEXT_INIT, - /*noScheduleAfterState=*/LlmRequestState::kGENERATION_COMPLETE, - /*enablePrefixAwareScheduling=*/executorConfig.getSchedulerConfig().getEnablePrefixAwareScheduling()); - - mMicroBatchScheduler = std::make_unique<MicroBatchScheduler>(ctxChunkConfig, maxContextLength); - - if (ctxChunkConfig) - { - if (maxContextLength) - { - ctxChunkConfig.value().chunkUnitSize - = std::min(ctxChunkConfig.value().chunkUnitSize, maxContextLength.value()); - } - TLLM_CHECK_WITH_INFO(ctxChunkConfig.value().chunkUnitSize > 0, - "Context chunk size (%d) must be a positive integer.", maxContextLength.value()); - } - else - { - if (maxContextLength && maxNumTokens) - { - TLLM_CHECK_WITH_INFO(maxContextLength.value() <= maxNumTokens.value(), - "Without enabling chunked context, the max context length (%d) needs to be less than or equal to the " - "max number of tokens (%d).", - maxContextLength.value(), maxNumTokens.value()); - } - } - - mPauseRequests = std::make_unique<PauseRequests>(getMaxInputLen()); - mAssignReqSeqSlots = std::make_unique<AssignReqSeqSlots>(); - mAllocateKvCache = std::make_unique<AllocateKvCache>(); - - if (isCudaGraphMode()) - { - // Limit cuda graph cache size. Depending on the model one graph is 4-10MB of GPU memory. - SizeType32 cudaGraphCacheSize - = std::min(getMaxBatchSize(), std::max(mExtendedRuntimePerfKnobConfig.getCudaGraphCacheSize(), 1)); - // We can't have common cache for all microbatches as cuda graph is tied to the memory pointers of the runtime - // buffers. - mCudaGraphExecutorCaches.resize(mNumBuffers, utils::CudaGraphExecutorCache(cudaGraphCacheSize)); - } - - mSpeculativeDecodingFastLogits - = executorConfig.getSpecDecConfig().has_value() && executorConfig.getSpecDecConfig()->fastLogits; - if (mSpeculativeDecodingFastLogits && modelConfig.getSpeculativeDecodingMode().isNone() && mIsLeaderInOrchMode) - { - mDraftModelSendLogitsThread - = std::make_unique<std::thread>(&utils::draftModelSendLogitsThread, mDevice, &mDraftModelThreadShouldExit, - &mDraftRequestsWaitingToSendLogits, &mDraftRequestsDoneSendingLogits, &mDraftRequestsMtx); - } - - mCreateNewDecoderRequests = std::make_unique<CreateNewDecoderRequests>( - mSpeculativeDecodingFastLogits, mIsLeaderInOrchMode, isNormalizeLogProbs()); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -TrtGptModelInflightBatching::~TrtGptModelInflightBatching() -{ - if (mCacheTransceiver) - { - mCacheTransceiver->checkContextTransferStatus(1, true); - TLLM_CHECK_WITH_INFO(mCacheTransceiver->checkGenTransferComplete(), "Generation transfer not complete"); - } - if (mAsyncSendWaitThread) - { - mAsyncSendWaitThread.reset(nullptr); - } - if (mDraftModelSendLogitsThread) - { - mDraftModelThreadShouldExit = true; - mDraftModelSendLogitsThread->join(); - mDraftModelSendLogitsThread.reset(nullptr); - } -} - -void TrtGptModelInflightBatching::setupSpeculativeDecodingModule(executor::DecodingConfig const& decodingConfig) -{ - if (mModelConfig.getSpeculativeDecodingMode().isExplicitDraftTokens() - || mModelConfig.getSpeculativeDecodingMode().isEagle()) - { - TLLM_CHECK_WITH_INFO(mCtxGenFusion, "Current speculative decoding mode requires context-gen fusion IFB"); - } - - if (mModelConfig.getSpeculativeDecodingMode().isLookaheadDecoding() && decodingConfig.getLookaheadDecodingConfig()) - { - // FIXME choose defaults - auto maxLookaheadConfig = decodingConfig.getLookaheadDecodingConfig().value(); - - SizeType32 maxDraftTokens{0}; - SizeType32 maxDraftPathLen{0}; - std::tie(std::ignore, std::ignore, maxDraftTokens, maxDraftPathLen) - = maxLookaheadConfig.calculateSpeculativeResource(); - TLLM_CHECK(maxDraftTokens <= mModelConfig.getMaxDecodingDraftTokens()); - mModelConfig.getSpeculativeDecodingModulePtr()->setMaxDraftTokens(maxDraftTokens); - mModelConfig.getSpeculativeDecodingModulePtr()->setMaxDraftPathLen(maxDraftPathLen); - - auto lookaheadModulePtr - = std::dynamic_pointer_cast<runtime::LookaheadModule>(mModelConfig.getSpeculativeDecodingModulePtr()); - lookaheadModulePtr->setExecutionConfig(maxLookaheadConfig); - } -} - -void TrtGptModelInflightBatching::reshapeKvTensors(OffsetTableDimensions const& dims) -{ - TLLM_CHECK(mBuffers.size() == static_cast<size_t>(mNumBuffers)); - auto const& manager = mRuntime->getBufferManager(); - for (auto& buffers : mBuffers) - { - TLLM_CHECK(buffers->transformerBuffers); - // any method that operates on transformerBuffers must distinguish between self and cross cache, because - // transformerBuffers is not managed by KVCacheManager same rule applies to kv pool pointers below - buffers->transformerBuffers->reshapeKvTensors( - getMaxBatchSize(), mOperatingBeamWidth, dims.maxBlocksPerSeq, dims.cacheType, dims.numPools, manager); - } -} - -using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; - -std::pair<BlocksPerWindow, std::vector<SizeType32>> -TrtGptModelInflightBatching::clampWindowSizesToFitAtLeastOneSequence( - BlocksPerWindow const& blocksPerWindow, bool const failFastOnAttentionWindowTooLarge) -{ - // At this point, we can only validate that the cheapest sequence in terms of kv-cache resources still fits. More - // validation is needed on a per-request basis, once the prompt / output lengths and the actual beam width are - // known. - auto const promptLength = getMaxInputLen(); - auto const outputLength - = getMaxSequenceLen() - promptLength; // This makes it the best case scenario, as context tokens are 'cheaper' - // in terms of kv-cache resources on average. - auto const sinkTokenLength = getSinkTokenLen(); - auto const maxBeamWidth = getMaxBeamWidth(); - auto const tokensPerBlock = mModelConfig.getTokensPerBlock(); - auto const& oldMaxAttentionWindowVec = getMaxAttentionWindowVec(); - std::vector<SizeType32> newMaxAttentionWindowVec; - BlocksPerWindow newBlocksPerWindow; - - newMaxAttentionWindowVec.reserve(oldMaxAttentionWindowVec.size()); - for (auto const windowSize : oldMaxAttentionWindowVec) - { - auto const bestCaseBlockRequirements = kv_cache_manager::KVCacheManager::calculateMaxBlockRequirements( - promptLength, outputLength, sinkTokenLength, windowSize, maxBeamWidth, tokensPerBlock); - auto const [numPrimaryBlocks, numSecondaryBlocks] = blocksPerWindow.at(windowSize); - if (bestCaseBlockRequirements > numPrimaryBlocks) - { - auto const newMaxAttentionWindow = KVCacheManager::calculateMaxAttentionWindow( - promptLength, outputLength, sinkTokenLength, numPrimaryBlocks, maxBeamWidth, tokensPerBlock); - newMaxAttentionWindowVec.push_back(newMaxAttentionWindow); - newBlocksPerWindow[newMaxAttentionWindow] = std::make_tuple(numPrimaryBlocks, numSecondaryBlocks); - } - else - { - newMaxAttentionWindowVec.push_back(windowSize); - newBlocksPerWindow[windowSize] = std::make_tuple(numPrimaryBlocks, numSecondaryBlocks); - } - } - if (newMaxAttentionWindowVec == getMaxAttentionWindowVec()) - { - return {blocksPerWindow, newMaxAttentionWindowVec}; - } - TLLM_LOG_WARNING("maxAttentionWindowVec too large to fit at least one sequence in kvCache. Old: %s, New: %s", - common::vec2str(getMaxAttentionWindowVec()).c_str(), common::vec2str(newMaxAttentionWindowVec).c_str()); - - if (failFastOnAttentionWindowTooLarge) - { - throw std::runtime_error( - "Attention window too large to fit even a single sequence in the KV cache. Failing fast rather than " - "attempting an adjustment of the window sizes. " - "Old: " - + common::vec2str(getMaxAttentionWindowVec()) + ", New: " + common::vec2str(newMaxAttentionWindowVec)); - } - - setMaxAttentionWindowVec(newMaxAttentionWindowVec); - if (getMaxSequenceLen() > getMaxAttentionWindow()) - { - TLLM_LOG_WARNING("maxSequenceLen is reduced to maxAttentionWindow: %d", getMaxAttentionWindow()); - setMaxSequenceLen(getMaxAttentionWindow()); - if (getMaxInputLen() > getMaxSequenceLen() - 1) - { - setMaxInputLen(getMaxSequenceLen() - 1); - TLLM_LOG_WARNING("maxInputLen is reduced to %d", getMaxInputLen()); - } - } - // createBuffers depends on: - // maxAttentionWindow; maxAttentionWindowVec; maxSequenceLen; - // TODO: This is problematic, as createBuffers edits the state of trtGptModelInflightBatching, but - // what if there are different window values for cross+self etc. in encoder+decoder scenario... - createBuffers(mDecodingConfig, mAdditionalModelOutputs); - createDecoder(mDecodingConfig.getDecodingMode()); - return {newBlocksPerWindow, newMaxAttentionWindowVec}; -} - -std::unique_ptr<kv_cache_manager::KVCacheManager> TrtGptModelInflightBatching::createKvCacheManager( - KvCacheConfig const& kvCacheConfig, KvCacheType kvCacheType, uint64_t freePrimaryMemBytes, - uint64_t freeSecondaryMemBytes, size_t extraCostMemory, bool const failFastOnAttentionWindowTooLarge) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - bool isCrossAttention = kvCacheType == KvCacheType::kCROSS; - TLLM_CHECK_WITH_INFO( - mModelConfig.isTransformerBased(), "KvCacheManager is only needed by transformer based model."); - - auto const tokensPerBlock = mModelConfig.getTokensPerBlock(); - auto const kvDtype = mModelConfig.getKvDataType(); - - // init KV cache block manager - auto [numKvHeadsPerLayerBegin, numKvHeadsPerLayerEnd] = mModelConfig.getNumKvHeadsPerLayerLocalRange( - mWorldConfig.getPipelineParallelism(), mWorldConfig.getPipelineParallelRank(), isCrossAttention); - auto numKvHeadsPerLayer = std::vector<SizeType32>(numKvHeadsPerLayerBegin, numKvHeadsPerLayerEnd); - - auto maxAttentionWindowVec = getMaxAttentionWindowVec(); - if (kvCacheType != KvCacheType::kSELF) // TODO(nhaber): more foolproof way of initing cross-kvcache-manager - { - maxAttentionWindowVec = std::vector<SizeType32>{mModelConfig.getMaxEncoderLen()}; - } - - auto const numLayers = static_cast<SizeType32>(numKvHeadsPerLayer.size()); - auto const windowSizeToLayers = KVCacheManager::groupLayersByWindowSize(maxAttentionWindowVec, numLayers); - auto const sizePerHead = mModelConfig.getSizePerHead(); - auto blocksPerWindow = KVCacheManager::calculateMaxNumBlocks(kvCacheConfig, kvDtype, numKvHeadsPerLayer, - sizePerHead, tokensPerBlock, mWorldConfig, windowSizeToLayers, freePrimaryMemBytes, freeSecondaryMemBytes, - extraCostMemory, 2, getMaxBatchSize()); - - // now we check if any of the window sizes is too large for at least one sequence to fit in kvCache - // this can happen if e.g. maxSeqLen is deduced from the model and is too large - // and user also didn't provide maxAttentionWindow, which leads it to be equal to maxSeqLen - if (kvCacheType == KvCacheType::kSELF) - { - std::tie(blocksPerWindow, maxAttentionWindowVec) - = clampWindowSizesToFitAtLeastOneSequence(blocksPerWindow, failFastOnAttentionWindowTooLarge); - } - - if (kvCacheType == KvCacheType::kCROSS && kvCacheConfig.getEnableBlockReuse()) - { - TLLM_LOG_INFO( - "Cross KV cache does not support reuse because cross attention depends on encoder and decoder input ids. " - "Thus, KV cache reuse is disabled for cross KV cache."); - } - auto const enableBlockReuse = kvCacheType == KvCacheType::kSELF ? kvCacheConfig.getEnableBlockReuse() : false; - - auto kvCacheManager = std::make_unique<KVCacheManager>(numKvHeadsPerLayer, sizePerHead, tokensPerBlock, - blocksPerWindow, getMaxNumSequences(), getMaxBeamWidth(), maxAttentionWindowVec, kvDtype, getSinkTokenLen(), - mRuntime->getStreamPtr(), - kvCacheType == KvCacheType::kCROSS ? mModelConfig.getMaxEncoderLen() : getMaxSequenceLen(), - getMaxNumTokens().value(), enableBlockReuse, kvCacheType, kvCacheConfig.getSecondaryOffloadMinPriority(), - kvCacheConfig.getEventBufferMaxSize() > 0 - ? std::make_unique<kv_cache_manager::KVCacheEventManager>(kvCacheConfig.getEventBufferMaxSize()) - : nullptr, - kvCacheConfig.getEnablePartialReuse(), kvCacheConfig.getCopyOnPartialReuse()); - - reshapeKvTensors(kvCacheManager->getOffsetTableDimensions()); - - kvCacheManager->allocatePools(kvCacheConfig.getUseUvm()); - - TensorMap inputBuffers; - TensorPtr poolPointers = kvCacheManager->getBlockPoolPointers(); - TensorPtr poolMapping = kvCacheManager->getLayerToPoolMapping(); - - if (kvCacheType == KvCacheType::kSELF) - { - inputBuffers.insert_or_assign("host_kv_cache_pool_pointers", std::move(poolPointers)); - inputBuffers.insert_or_assign("host_kv_cache_pool_mapping", std::move(poolMapping)); - } - else - { - inputBuffers.insert_or_assign("host_cross_kv_cache_pool_pointers", std::move(poolPointers)); - inputBuffers.insert_or_assign("host_cross_kv_cache_pool_mapping", std::move(poolMapping)); - } - mRuntime->setStaticInputTensors(inputBuffers); - - // Emit the `created` event - kvCacheManager->flushIterationEvents(); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return kvCacheManager; -} - -void TrtGptModelInflightBatching::createRnnStateManager() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - TLLM_CHECK_WITH_INFO(mModelConfig.isRnnBased(), "RnnStateManager is only needed by RNN based model."); - - mRnnStateManager = std::make_unique<RnnStateManager>( - getMaxNumSequences(), mModelConfig, mWorldConfig, mRuntime->getBufferManager()); - - TensorMap inputBuffers; - mRnnStateManager->getPtrBuffers(inputBuffers, mModelConfig, mWorldConfig); - mRuntime->setStaticInputTensors(inputBuffers); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::createCustomAllReduceWorkspace() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - TLLM_CHECK(mWorldConfig.isTensorParallel()); - - auto const& manager = mRuntime->getBufferManager(); - auto const hiddenSize = mModelConfig.getHiddenSize(); - - mAllReduceBuffers = std::make_unique<AllReduceBuffers>(getMaxBatchSize(), getMaxBeamWidth(), getMaxSequenceLen(), - hiddenSize, manager, mWorldConfig, mRuntime->isUserBufferEnabled()); - - TensorMap inputBuffers; - inputBuffers.insert_or_assign("all_reduce_workspace", mAllReduceBuffers->mAllReduceCommPtrs); - mRuntime->setStaticInputTensors(inputBuffers); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::createRuntimePerfKnobsTensor( - executor::ExtendedRuntimePerfKnobConfig const& extendedRuntimePerfKnobConfig) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - SizeType32 constexpr perfKnobSize{16}; - mExtendedRuntimePerfKnobsHost = BufferManager::cpu(ITensor::makeShape({perfKnobSize}), nvinfer1::DataType::kINT64); - auto* runtimePerfKnobsHostPtr = bufferCast<int64_t>(*mExtendedRuntimePerfKnobsHost); - std::fill_n(runtimePerfKnobsHostPtr, perfKnobSize, -1); - SizeType32 multiBlockModeVal = extendedRuntimePerfKnobConfig.getMultiBlockMode() ? 1 : 0; - SizeType32 enableContextFMHAFP32AccVal = extendedRuntimePerfKnobConfig.getEnableContextFMHAFP32Acc() ? 1 : 0; - runtimePerfKnobsHostPtr[0] = multiBlockModeVal; - runtimePerfKnobsHostPtr[1] = enableContextFMHAFP32AccVal; - - TensorMap inputBuffers; - inputBuffers.insert_or_assign("host_runtime_perf_knobs", mExtendedRuntimePerfKnobsHost); - mRuntime->setStaticInputTensors(inputBuffers); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::terminateRequest(LlmRequestPtr const& llmReq, bool pause) -{ - utils::terminateRequest( - *mSeqSlotManager, *llmReq, getMaxInputLen(), mKvCacheManager, mCrossKvCacheManager, mPeftCacheManager, pause); -} - -void TrtGptModelInflightBatching::terminateRequestSync( - LlmRequestPtr const& llmRequest, executor::FinishReason finishReason) -{ - TLLM_LOG_DEBUG("Registering termination for request %lu with finish reason %d", llmRequest->mRequestId, - static_cast<int>(finishReason)); - mReqIdsToTerminate.try_emplace(llmRequest->mRequestId, finishReason); -} - -TrtGptModelInflightBatching::IterationStatsIFB TrtGptModelInflightBatching::fillIterationStats( - ScheduledRequests const& scheduledRequests, RequestVector const& requestsToPause) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(fillIterationStats); - - IterationStatsIFB iterationStatsIfb{mMicroBatchId}; - iterationStatsIfb.numCtxRequests = scheduledRequests.contextRequests.size(); - iterationStatsIfb.numGenRequests = scheduledRequests.generationRequests.size(); - iterationStatsIfb.avgNumDecodedTokensPerIter = 0; - - auto const contextBufferId = mCtxGenFusion ? getFusedBufferId() : getContextBufferId(); - auto const& buffers = mBuffers.at(contextBufferId); - iterationStatsIfb.numCtxTokens = buffers->getNumContextTokens(); - - for (auto const& llmReq : scheduledRequests.contextRequests) - { - iterationStatsIfb.scheduledRequests.insert(llmReq->mRequestId); - } - for (auto const& llmReq : scheduledRequests.generationRequests) - { - iterationStatsIfb.scheduledRequests.insert(llmReq->mRequestId); - iterationStatsIfb.avgNumDecodedTokensPerIter += llmReq->getAvgDecodedTokensPerIter(); - } - if (iterationStatsIfb.numGenRequests > 0) - { - iterationStatsIfb.avgNumDecodedTokensPerIter /= iterationStatsIfb.numGenRequests; - TLLM_LOG_DEBUG( - "iterationStatsIfb.avgNumDecodedTokensPerIter = %.2f", iterationStatsIfb.avgNumDecodedTokensPerIter); - } - for (auto const& llmReq : requestsToPause) - { - iterationStatsIfb.pausedRequests.insert(llmReq->mRequestId); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return iterationStatsIfb; -} - -void TrtGptModelInflightBatching::forwardSync() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE_WITH_NAME(range, "TrtGptModelInflightBatching::forwardSync"); - - TLLM_CUDA_CHECK(cudaSetDevice(mWorldConfig.getDevice())); - - if (!mWorldConfig.isLastPipelineParallelRank()) - { - mAsyncSendWaitThread->waitStop(); - } - - auto& currRequests = mMicroBatchScheduledRequests.at(mMicroBatchId); - - if (!currRequests.empty()) - { - if (!mWorldConfig.isPipelineParallel() || !mWorldConfig.isLastPipelineParallelRank()) - { - for (auto& hdl : mDecStepAsyncSndHdls) - { - TLLM_CHECK_WITH_INFO(hdl.get() == nullptr, "decoderSync handle must be nullptr."); - } - // Wait for decoding for requests in flight for the current micro batch - auto& decoderWaitEvent = mDecoderFinishedEvents.at(mMicroBatchId); - mDecStepAsyncSndHdls = decoderSync(currRequests, decoderWaitEvent); - decoderWaitEvent.reset(); - - if (!mWorldConfig.isLastPipelineParallelRank()) - { - mAsyncSendWaitThread->notifyStart(); - } - } - else - { - for (auto const& requests : {currRequests.contextRequests, currRequests.generationRequests}) - { - for (auto const& llmReq : requests) - { - for (SizeType32 beam = 0; beam < llmReq->mSamplingConfig.beamWidth; ++beam) - { - llmReq->setNumPreDecodedTokens(0, beam); - } - if (llmReq->isGenerationToCompleteState()) - { - llmReq->setState(LlmRequestState::kGENERATION_COMPLETE); - terminateRequest(llmReq); - } - } - } - } - - (*mPauseRequests)(currRequests.generationRequests, mInflightReqIds, mReqIdsToPause, true, *mSeqSlotManager, - mKvCacheManager, mCrossKvCacheManager, mPeftCacheManager); - - if (!mReqIdsToTerminate.empty()) - { - for (auto const& requests : {currRequests.contextRequests, currRequests.generationRequests}) - { - for (auto const& llmReq : requests) - { - if (mReqIdsToTerminate.count(llmReq->mRequestId) != 0U) - { - if (!llmReq->isGenerationCompleteState()) - { - TLLM_LOG_DEBUG("Terminating request %lu with finish reason %d", llmReq->mRequestId, - static_cast<int>(mReqIdsToTerminate[llmReq->mRequestId])); - terminateRequest(llmReq); - llmReq->finishByReason(mReqIdsToTerminate[llmReq->mRequestId]); - llmReq->clearGeneratedTokens(); - } - mReqIdsToTerminate.erase(llmReq->mRequestId); - } - } - } - } - - // Terminate draft requests whose logits have been sent by the background thread. - { - RequestVector doneSending; - { - std::lock_guard<std::mutex> lk(mDraftRequestsMtx); - doneSending.swap(mDraftRequestsDoneSendingLogits); - } - for (auto const& llmReq : doneSending) - { - terminateRequest(llmReq); - } - } - - // Finished context requests have been moved to generationRequests by moveFinishedContextRequestsToGeneration - for (auto const& llmReq : currRequests.generationRequests) - { - // If a context-only request is finished, send its KV cache and mark it. - if (llmReq->isContextOnlyRequest() && llmReq->isContextFinished()) - { - // TODO: skip if sending layer-wise - { - TLLM_CHECK_WITH_INFO(mCacheTransceiver, - "Disaggregated serving is not enabled, please check the configuration of " - "cacheTransceiverConfig."); - mCacheTransceiver->respondAndSendAsync(llmReq); - } - mSeqSlotManager->freeSequenceSlot(llmReq->mRequestId); - } - } - } - // report profile data - auto const bufferId = getFusedBufferId(); - auto const contextId = mBuffers[bufferId]->getContextIndex(); - if (mRuntime->hasLayerProfiler(contextId)) - { - mRuntime->reportToProfiler(contextId); - } - if (mCacheTransceiver) - { - mCacheTransceiver->checkContextTransferStatus(0, true); - } - ++mIterCounter; - - if (mKvCacheManager) - { - mKvCacheManager->flushIterationEvents(); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::storeContextBlocks(std::shared_ptr<LlmRequest> const& llmReq) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // TMJ - Note - // Make context blocks reusable immediately after context phase finishes. - // For chunked contexts, this occurs in step that processes last context chunk. - // isLastContextChunk() is always true for non-chunked contexts. - // This check is made in code that calls storeContextBlocks, so omitted here. - if (mKvCacheManager) - { - mKvCacheManager->storeContextBlocks(*llmReq); - } - if (mCrossKvCacheManager) - { - mCrossKvCacheManager->storeContextBlocks(*llmReq); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::storeNewBlock(std::shared_ptr<LlmRequest> const& llmReq) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // TMJ - Note - // Make context blocks reusable immediately after each generation step. - - if (mKvCacheManager) - { - mKvCacheManager->storeNewBlock(*llmReq); - } - if (mCrossKvCacheManager) - { - mCrossKvCacheManager->storeNewBlock(*llmReq); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::resetIterationStats() -{ - mLastIterationStatsIFB = IterationStatsIFB{mMicroBatchId}; -} - -void TrtGptModelInflightBatching::forwardAsync(RequestList const& activeRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE_WITH_NAME(range, "TrtGptModelInflightBatching::forwardAsync"); - - TLLM_CUDA_CHECK(cudaSetDevice(mWorldConfig.getDevice())); - - try - { - verifyRequests(activeRequests); - if (mModelConfig.isTransformerBased() && getKVCacheManager() && mCacheTransceiver) - { - checkDisaggGenTransferStatus(activeRequests); - } - auto& currRequests = mMicroBatchScheduledRequests.at(mMicroBatchId); - - // Get a new set of requests for that context - // The scheduler will not include any requests that are (i) still in encoder state if encoder-decoder models OR - // (ii) already in flight for decoder models - TLLM_LOG_DEBUG("Running DECODER request scheduler"); - auto [fittingRequests, fittingDisaggGenInitRequests, requestsToPause] - = (*mCapacityScheduler)(activeRequests, mKvCacheManager, mPeftCacheManager, mCrossKvCacheManager); - // Remove from fitting requests the requests that cannot be scheduled due to disagg KV cache transfer - bool waitForDisaggGenTransferProgress = false; - if (mModelConfig.isTransformerBased() && getKVCacheManager() && mCacheTransceiver) - { - if (mDisaggTransferAdmissionController && mDisaggTransferAdmissionController->enabled() - && !fittingDisaggGenInitRequests.empty()) - { - auto admissionResult - = mDisaggTransferAdmissionController->select(activeRequests, fittingDisaggGenInitRequests); - waitForDisaggGenTransferProgress = admissionResult.isBlockedByActiveTransfers(); - if (admissionResult.deferredRequestCount > 0) - { - TLLM_LOG_DEBUG( - "Disagg transfer admission deferred %zu requests; active transfer blocks=%zu, admitted " - "transfer blocks=%zu, budget=%zu", - admissionResult.deferredRequestCount, admissionResult.activeTransferBlocks, - admissionResult.admittedTransferBlocks, - mDisaggTransferAdmissionController->getMaxTransferBlocks().value_or(0)); - } - fittingDisaggGenInitRequests = std::move(admissionResult.admittedRequests); - } - prepareDisaggGenInitRequests(activeRequests, fittingDisaggGenInitRequests); - } - if (fittingRequests.empty() && fittingDisaggGenInitRequests.empty()) - { - TLLM_LOG_WARNING( - "CapacityScheduler didn't schedule any requests in iteration %lu, " - "probably because of insufficient resources such as KV cache, " - "will try wait for KV cache transfer to complete", - mIterCounter); - if (mCacheTransceiver) - { - if (waitForDisaggGenTransferProgress) - { - TLLM_LOG_DEBUG("Waiting for generation KV cache transfer progress to free disagg admission budget"); - mCacheTransceiver->checkGenTransferStatus(1); - } - else - { - mCacheTransceiver->checkContextTransferStatus(1, true); - // will free kvCache in next iteration. - } - } - } - std::tie(currRequests.contextRequests, currRequests.generationRequests) - = (*mMicroBatchScheduler)(fittingRequests, mInflightReqIds, mMaxBatchSizeRuntime, mMaxNumTokensRuntime); - TLLM_CHECK(currRequests.size() <= static_cast<size_t>(getMaxBatchSize())); - - (*mPauseRequests)(requestsToPause, mInflightReqIds, mReqIdsToPause, false, *mSeqSlotManager, mKvCacheManager, - mCrossKvCacheManager, mPeftCacheManager); - - if (mUseSeamlessLookahead) - { - changeSpecDecMode(currRequests); - } - - if (!currRequests.empty()) - { - TLLM_LOG_DEBUG("Running DECODER model with batch size: %lu", currRequests.size()); - // For overlap don't store inflight requests, so they are not skipped in scheduler - if (!isTrtOverlap()) - { - NVTX3_SCOPED_RANGE(updateInflightReqIds); - // Add requests to in-flight set, so they can be skipped in other micro batches - for (auto const& llmReq : currRequests.contextRequests) - { - // Context requests that are chunking are not added to inflight set, so they are scheduled in the - // next micro batch. - if (llmReq->isLastContextChunk()) - { - TLLM_LOG_DEBUG( - "Context request with ID %lu added to DECODER model inflight set", llmReq->mRequestId); - mInflightReqIds.insert(llmReq->mRequestId); - } - } - for (auto const& llmReq : currRequests.generationRequests) - { - TLLM_LOG_DEBUG( - "Generation request with ID %lu added to DECODER model inflight set", llmReq->mRequestId); - mInflightReqIds.insert(llmReq->mRequestId); - } - } - - (*mAssignReqSeqSlots)(*mSeqSlotManager, currRequests.contextRequests, currRequests.generationRequests); - - if (mKvCacheManager) - { - (*mAllocateKvCache)(*mKvCacheManager, currRequests.contextRequests, currRequests.generationRequests, - mModelConfig, mCrossKvCacheManager); - } - - mPeftTables.at(mMicroBatchId) - = mPeftCacheManager->ensureBatch(currRequests.contextRequests, currRequests.generationRequests, true); - - // Do decoder setup before context phase if model needs to setup buffers for the context phase. - if (mModelConfig.getSpeculativeDecodingMode().needsDecoderPrologue()) - { - auto const contextBufferId = mCtxGenFusion ? getFusedBufferId() : getContextBufferId(); - setupDecoderStep(currRequests.contextRequests, *mBuffers.at(contextBufferId), - mDecoderInputBuffers.at(getFusedBufferId())); - // WAR: Sync to ensure that the decoder setup is complete before the context phase starts. - // Without this, there may be a race condition between the decoder setup and the context phase - // which also leads to spurious test failure in trtGptModelRealDecoderTest. - mRuntime->getStream().synchronize(); - } - else - { - prepareDistGenBufferAndDecoder(currRequests.generationRequests); - } - sync_check_cuda_error(mRuntime->getStream().get()); - - executeBatch(currRequests); - if (mWorldConfig.isLastPipelineParallelRank() && mGuidedDecoder) - { - // XGrammar: build maskcache for context requests and perform maskgen for all requests - // These need to be overlapped with the kernel execution of forward step - mGuidedDecoder->build(currRequests); - } - - sync_check_cuda_error(mRuntime->getStream().get()); - - // Postpone decoder setup if model does not need to setup buffers for the context phase. - if (!mModelConfig.getSpeculativeDecodingMode().needsDecoderPrologue()) - { - auto const contextBufferId = mCtxGenFusion ? getFusedBufferId() : getContextBufferId(); - setupDecoderStep(currRequests.contextRequests, *mBuffers.at(contextBufferId), - mDecoderInputBuffers.at(getFusedBufferId())); - } - - sync_check_cuda_error(mRuntime->getStream().get()); - - if (isTrtOverlap()) - { - // WAR: Because the decoder is not stateless (yet) a sync is needed between - // decoder execution and next decoder step preparation. - auto const prevMicroBatchId = getPrevMicroBatchId(mMicroBatchId); - auto& prevDecoderFinishedEvent = mDecoderFinishedEvents.at(prevMicroBatchId); - if (prevDecoderFinishedEvent) - { - prevDecoderFinishedEvent->synchronize(); - } - } - - auto& decoderFinishedEvent = mDecoderFinishedEvents.at(mMicroBatchId); - TLLM_CHECK_WITH_INFO(!decoderFinishedEvent.has_value(), "decoderFinishedEvent must be nullopt."); - decoderFinishedEvent = mWorldConfig.isLastPipelineParallelRank() - ? std::make_optional(decoderStepAsync(currRequests)) - : std::nullopt; - - sync_check_cuda_error(mRuntime->getStream().get()); - - mLastIterationStatsIFB = fillIterationStats(currRequests, requestsToPause); - for (auto const& requests : {currRequests.contextRequests, currRequests.generationRequests}) - { - for (auto const& llmReq : requests) - { - if (llmReq->isContextInitState()) - { - llmReq->moveToNextContextChunk(); - if (llmReq->getContextRemainingLength() == 0) - { - TLLM_LOG_DEBUG("[RANK %d] request with ID %lu finishes decoder ctx phase", - COMM_SESSION.getRank(), llmReq->mRequestId); - - llmReq->setState(LlmRequestState::kGENERATION_IN_PROGRESS); - - // for encoder-decoder models, free encoder output buffers after decoder context phase is - // completed - if (llmReq->getEncoderTokens().has_value()) - { - llmReq->freeEncoderOutputBuffers(); - } - storeContextBlocks(llmReq); - - if (isTrtOverlap() && llmReq->willCompleteNextIteration()) - { - // This prohibits the request from being scheduled for another iteration if only one - // iteration is expected. - llmReq->setState(LlmRequestState::kGENERATION_TO_COMPLETE); - } - } - } - else if (llmReq->isGenerationInProgressState()) - { - storeNewBlock(llmReq); - TLLM_LOG_DEBUG("request with ID %lu forwards a step in decoder gen phase", llmReq->mRequestId); - } - } - } - - utils::moveFinishedContextRequestsToGeneration(currRequests); - } - else - { - mLastIterationStatsIFB = IterationStatsIFB{mMicroBatchId}; - } - - if (mWorldConfig.isPipelineParallel() && mWorldConfig.isLastPipelineParallelRank()) - { - mAsyncSendWaitThread->waitStop(); - if (!currRequests.empty()) - { - for (auto& hdl : mDecStepAsyncSndHdls) - { - TLLM_CHECK_WITH_INFO(hdl.get() == nullptr, "decoderSync handle must be nullptr."); - } - // Wait for decoding for requests in flight for the current micro batch - auto& decoderFinishedEvent = mDecoderFinishedEvents.at(mMicroBatchId); - mDecStepAsyncSndHdls = decoderSync(currRequests, decoderFinishedEvent); - decoderFinishedEvent.reset(); - - mAsyncSendWaitThread->notifyStart(); - } - } - - // Update the micro batch ID - mMicroBatchId = getNextMicroBatchId(mMicroBatchId); - } - // In case of error, we need to free the batch slot associated with those requests - catch (std::exception const&) - { - try - { - for (auto const& llmReq : activeRequests) - { - // Remove from mInflightReqIds so changeBeamWidth can proceed on the next iteration. - // terminateRequest frees seqSlot/KV cache but does not clean up mInflightReqIds. - mInflightReqIds.erase(llmReq->mRequestId); - terminateRequest(llmReq); - } - // Force buffer/decoder reset to clean up any partial state from the aborted batch - // (e.g. partially-filled cross-KV block offsets from mid-context-chunk processing). - // Guard on mInflightReqIds.empty(): in pipeline-parallel multi-micro-batch mode, - // other micro-batches may still have requests tracked here; changeBeamWidth asserts - // emptiness so we skip the reset and let the next successful forwardAsync iteration - // perform it when the set is clear. - if (mWorldConfig.isLastPipelineParallelRank() && mInflightReqIds.empty()) - { - changeBeamWidth(mOperatingBeamWidth); - } - } - catch (std::exception const& e) - { - TLLM_LOG_ERROR("forwardAsync catch-all catch block that runs `terminateRequest` has failed with:"); - TLLM_LOG_EXCEPTION(e); - TLLM_LOG_ERROR("Rethrowing *outer* exception:"); - } - throw; - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::setRuntimeBatchSize(SizeType32 runtimeMaxBatchSize) -{ - mMaxBatchSizeTunerRecommended = runtimeMaxBatchSize; - mMaxBatchSizeRuntime = std::min(getMaxBatchSize(), runtimeMaxBatchSize); -} - -SizeType32 TrtGptModelInflightBatching::getRuntimeBatchSize() const -{ - return mMaxBatchSizeRuntime; -} - -void TrtGptModelInflightBatching::setRuntimeMaxNumTokens(SizeType32 runtimeMaxNumTokens) -{ - mMaxNumTokensTunerRecommended = runtimeMaxNumTokens; - mMaxNumTokensRuntime - = (mMaxNumTokensStatic) ? std::min(mMaxNumTokensStatic.value(), runtimeMaxNumTokens) : runtimeMaxNumTokens; -} - -void TrtGptModelInflightBatching::updatePeftCache(std::shared_ptr<LlmRequest> const& llmRequest) -{ - mPeftCacheManager->addRequestPeft(llmRequest, true); -} - -runtime::BufferManager const& TrtGptModelInflightBatching::getBufferManager() const -{ - return mRuntime->getBufferManager(); -} - -BufferManager::CudaStreamPtr TrtGptModelInflightBatching::getRuntimeStreamPtr() const -{ - return mRuntime->getStreamPtr(); -} - -void TrtGptModelInflightBatching::executeContext(SizeType32 runtimeContextId, SizeType32 bufferId) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(executeContext); - - auto const& currBatchState = mBuffers[bufferId]->getBatchState(); - - bool hasCudaGraph = false; - // If batch state is context only, do not capture/launch graph and execute the engine as is. - if (isCudaGraphMode() && !currBatchState.isAnyContext()) - { - auto cudaGraphOpt = mCudaGraphExecutorCaches[bufferId].get(currBatchState); - // If graph exists for current batch state, launch it. - if (cudaGraphOpt.has_value()) - { - hasCudaGraph = true; - } - } - - // If there is no graph for current state, execute the engine. - if (!hasCudaGraph) - { - auto enqueueSuccessful = mRuntime->executeContext(runtimeContextId); - if (!enqueueSuccessful) - { - throw std::runtime_error("Executing TRT engine failed!"); - } - } - else - { - // Launch graph. - auto cudaGraphOpt = mCudaGraphExecutorCaches[bufferId].get(currBatchState); - cudaGraphOpt.value()->launch(mRuntime->getStream()); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::setLayerProfiler() -{ - mRuntime->setLayerProfiler(); -} - -std::string TrtGptModelInflightBatching::getLayerProfileInfo() const -{ - return mRuntime->getLayerProfileInfo(); -} - -void TrtGptModelInflightBatching::verifyRequests(RequestList const& activeRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(verifyRequests); - - if (activeRequests.empty()) - { - return; - } - - auto const& firstRequest = activeRequests.front(); - auto const firstRequestId = firstRequest->mRequestId; - auto const firstBeamWidth = firstRequest->mSamplingConfig.beamWidth; - - for (auto const& llmReq : activeRequests) - { - auto const beamWidth = llmReq->mSamplingConfig.beamWidth; - auto const draftLength = llmReq->getNumDraftTokens(); - auto const maxDraftLength = mModelConfig.getMaxDecodingDraftTokens(); - - TLLM_CHECK_WITH_INFO(beamWidth == 1 || draftLength == 0, "Can't use speculative decoding with beam search."); - TLLM_CHECK_WITH_INFO(draftLength <= maxDraftLength, - "Number of draft tokens (%d) is larger than maximum number of draft tokens (%d)", draftLength, - maxDraftLength); - - // FIXME: Remove this check when varying beam width is supported - { - TLLM_CHECK_WITH_INFO(beamWidth == firstBeamWidth, - "All active requests must have same beam width, " - "but request %lu with beam width %d differs from first request %lu with beam width %d", - llmReq->mRequestId, beamWidth, firstRequestId, firstBeamWidth); - } - } - - if (firstBeamWidth != mOperatingBeamWidth) - { - changeBeamWidth(firstBeamWidth); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::executeBatch(ScheduledRequests const& scheduledRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(executeBatch); - - if (!mCtxGenFusion) - { - if (!scheduledRequests.contextRequests.empty()) - { - auto const bufferId = getContextBufferId(); - executeStep(scheduledRequests.contextRequests, {}, bufferId); - } - if (!scheduledRequests.generationRequests.empty()) - { - auto const bufferId = getGenerationBufferId(); - executeStep({}, scheduledRequests.generationRequests, bufferId); - } - } - else - { - auto const bufferId = getFusedBufferId(); - executeStep(scheduledRequests.contextRequests, scheduledRequests.generationRequests, bufferId); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::createRuntimeContexts() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - mRuntime->clearContexts(); - auto const numProfiles = mRuntime->getNbProfiles(); - for (auto i = 0; i < numProfiles; ++i) - { - mRuntime->addContext(i); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -namespace -{ -// TODO: move this somewhere else? -/** - * This function logic is also implemented in tensorrt_llm/_torch/pyexecutor/_util.py get_decoding_mode(). - */ -executor::DecodingMode getDecodingMode(SpeculativeDecodingMode specDecodingMode, - std::optional<executor::DecodingMode> const& decodingModeOpt, runtime::SizeType32 const beamWidth) -{ - auto getDefaultDecodingMode = [beamWidth](std::optional<executor::DecodingMode> const& decodingModeOpt) - { - if (decodingModeOpt.has_value() && !decodingModeOpt->isAuto()) - { - return decodingModeOpt.value(); - } - return (beamWidth == 1) ? executor::DecodingMode::TopKTopP() : executor::DecodingMode::BeamSearch(); - }; - - auto decodingMode = getDefaultDecodingMode(decodingModeOpt); - // Variable-Beam-Width-Search (special mode of Beam-Search) is enabled. - if (decodingMode.isBeamSearch() && decodingMode.isUseVariableBeamWidthSearch()) - { - TLLM_LOG_INFO("Variable-Beam-Width-Search is enabled"); - } - // Overwrite decoding mode when beam width is one. - if (beamWidth == 1 && decodingMode.isBeamSearch()) - { - TLLM_LOG_WARNING( - "Beam width is set to 1, but decoding mode is BeamSearch. Overwriting decoding mode to TopKTopP."); - decodingMode = executor::DecodingMode::TopKTopP(); - } - // Overwrite decoding mode when Medusa is used. - if (specDecodingMode.isMedusa() && !decodingMode.isMedusa()) - { - TLLM_LOG_WARNING("Model is Medusa, but decoding mode is not Medusa. Overwriting decoding mode to Medusa."); - decodingMode = executor::DecodingMode::Medusa(); - } - // Overwrite decoding mode when Medusa is not used. - if (!specDecodingMode.isMedusa() && decodingMode.isMedusa()) - { - TLLM_LOG_WARNING("Model is not Medusa, but decoding mode is Medusa. Overwriting decoding mode."); - decodingMode = getDefaultDecodingMode(decodingModeOpt); - } - // Overwrite decoding mode when lookahead decoding is used. - if (specDecodingMode.isLookaheadDecoding() && !decodingMode.isLookahead()) - { - TLLM_LOG_WARNING( - "Model is Lookahead, but decoding mode is not Lookahead. Overwriting decoding mode to Lookahead."); - decodingMode = executor::DecodingMode::Lookahead(); - } - // Overwrite decoding mode when lookahead decoding is not used. - if (!specDecodingMode.isLookaheadDecoding() && decodingMode.isLookahead()) - { - TLLM_LOG_WARNING( - "Model is not built with Lookahead decoding, but decoding mode is Lookahead. Overwriting decoding " - "mode."); - decodingMode = getDefaultDecodingMode(decodingModeOpt); - } - // Overwrite decoding mode when 'explicit draft tokens' is used. - if (specDecodingMode.isExplicitDraftTokens() && !decodingMode.isExplicitDraftTokens()) - { - TLLM_LOG_WARNING( - "Model is built with 'explicit draft tokens' decoding, but decoding mode is something else. Overwriting " - "decoding mode."); - decodingMode = executor::DecodingMode::ExplicitDraftTokens(); - } - // Overwrite decoding mode when 'explicit draft tokens' is not used. - if (!specDecodingMode.isExplicitDraftTokens() && decodingMode.isExplicitDraftTokens()) - { - TLLM_LOG_WARNING( - "Model is not built with 'explicit draft tokens' decoding, but decoding mode is set to it. Overwriting " - "decoding " - "mode to default."); - decodingMode = getDefaultDecodingMode(decodingModeOpt); - } - // Overwrite decoding mode when EAGLE is used. - if (specDecodingMode.isEagle() && !decodingMode.isEagle()) - { - TLLM_LOG_WARNING("Model is Eagle, but decoding mode is not Eagle. Overwriting decoding mode to Eagle."); - decodingMode = executor::DecodingMode::Eagle(); - } - // Overwrite decoding mode when Eagle is not used. - if (!specDecodingMode.isEagle() && decodingMode.isEagle()) - { - TLLM_LOG_WARNING("Model is not Eagle, but decoding mode is Eagle. Overwriting decoding mode."); - decodingMode = getDefaultDecodingMode(decodingModeOpt); - } - if (specDecodingMode.isDraftTokensExternal()) - { - TLLM_LOG_WARNING("Overwriting decoding mode to external draft token"); - decodingMode = executor::DecodingMode::ExternalDraftTokens(); - } - TLLM_LOG_DEBUG("DecodingMode: %s", decodingMode.getName()); - return decodingMode; -} -} // namespace - -void TrtGptModelInflightBatching::createDecoder(std::optional<executor::DecodingMode> const& decodingModeOpt) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - mDecoderState = std::make_unique<runtime::decoder::DecoderState>(); - - if (mWorldConfig.isLastPipelineParallelRank()) - { - auto decoderType = mRuntime->getEngine().getTensorDataType("logits"); - - auto const decodingMode - = getDecodingMode(mModelConfig.getSpeculativeDecodingMode(), decodingModeOpt, mOperatingBeamWidth); - - if (decodingMode.isExplicitDraftTokens()) - { - // There are no logits in Explicit draft tokens model. - decoderType = mModelConfig.getDataType(); - // Decoder is not instantiated for bf16. We use half to get the same data size - // and explicitly pass dtype to redrafter that has bf16 kernels. - if (decoderType == nvinfer1::DataType::kBF16) - { - decoderType = nvinfer1::DataType::kHALF; - } - } - - mDecoder = std::make_unique<runtime::GptDecoderBatched>(mRuntime->getStreamPtr()); - mDecoder->setup( - decodingMode, getMaxNumSequences(), mOperatingBeamWidth, decoderType, mModelConfig, mWorldConfig); - - mDecoderState->setup(getMaxNumSequences(), mOperatingBeamWidth, getMaxAttentionWindow(), getSinkTokenLen(), - getMaxSequenceLen(), decoderType, mModelConfig, mWorldConfig, mRuntime->getBufferManager()); - - if (!mModelConfig.getSpeculativeDecodingMode().isNone()) - { - mDecoderState->setupSpeculativeDecoding(mModelConfig.getSpeculativeDecodingMode(), - mModelConfig.getMaxDecodingTokens(), decoderType, mModelConfig, mWorldConfig, - mRuntime->getBufferManager()); - } - } - else - { - mDecoderState->setupCacheIndirection( - getMaxNumSequences(), mOperatingBeamWidth, getMaxAttentionWindow(), mRuntime->getBufferManager()); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::createBuffers(executor::DecodingConfig const& decodingConfig, - std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - mBuffers.clear(); - for (SizeType32 i = 0; i < mNumBuffers; ++i) - { - mBuffers.emplace_back( - std::make_unique<RuntimeBuffers>(getMaxBatchSize(), mOperatingBeamWidth, getMaxAttentionWindowVec(), - getMaxAttentionWindow(), getSinkTokenLen(), *mRuntime, mModelConfig, mWorldConfig, decodingConfig, - getGatherGenerationLogits(), getMaxNumTokens(), additionalModelOutputs, mPromptTableOffloading)); - } - - mDecoderInputBuffers.clear(); - mDecoderOutputBuffers.clear(); - for (SizeType32 i = 0; i < mNumMicroBatches; ++i) - { - mDecoderInputBuffers.emplace_back( - getMaxBatchSize(), mModelConfig.getMaxDecodingTokens(), mRuntime->getBufferManager()); - mDecoderInputBuffers.back().setupMedusaLogits(getMaxNumSequences(), mModelConfig); - mDecoderOutputBuffers.emplace_back(getMaxNumSequences(), mOperatingBeamWidth, getMaxSequenceLen(), - mModelConfig.getMaxDecodingTokens(), mRuntime->getBufferManager()); - mDecoderOutputBuffers.back().setupSpeculativeDecoding( - getMaxNumSequences(), mModelConfig.getMaxDecodingTokens(), mModelConfig); - } - - mSlotDecoderBuffers.clear(); - for (SizeType32 i = 0; i < getMaxNumSequences(); ++i) - { - mSlotDecoderBuffers.emplace_back(std::make_unique<SlotDecoderBuffers>( - mOperatingBeamWidth, getMaxSequenceLen(), mRuntime->getBufferManager())); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::prepareDisaggGenInitRequests( - RequestList const& activeRequests, RequestVector& newGenReqs) -{ - NVTX3_SCOPED_RANGE(prepareDisaggGenInitRequests); - - // Allocate KV cache by treating them as context requests - (*mAllocateKvCache)(*mKvCacheManager, newGenReqs, {}, mModelConfig, mCrossKvCacheManager); - - // Initiate KV cache transfer - auto timeStart = std::chrono::steady_clock::now(); - - if (tc::getEnvDisaggBenchmarkGenOnly()) - { - TLLM_LOG_DEBUG("Disaggregated generation only benchmark mode is enabled"); - for (auto& req : newGenReqs) - { - req->setState(LlmRequestState::kDISAGG_GENERATION_TRANS_COMPLETE); - } - return; - } - - auto const genInitReqNum = std::count_if(activeRequests.begin(), activeRequests.end(), - [](auto const& req) { return req->isDisaggGenerationInitState(); }); - - // Loop over the new disagg gen requests and trigger receive of KV cache - for (auto& newGenReq : newGenReqs) - { - TLLM_CHECK_WITH_INFO( - mCacheTransceiver, "Disaggregated serving is not enabled, please check the configuration."); - if (common::getEnvDisableKVCacheTransferOverlap()) - { - mCacheTransceiver->requestAndReceiveSync(newGenReq); - } - else - { - mCacheTransceiver->requestAndReceiveAsync(newGenReq); - } - } - if (!common::getEnvDisableKVCacheTransferOverlap()) - { - auto const blockTransfer = std::all_of(activeRequests.begin(), activeRequests.end(), - [](auto const& req) { return req->isDisaggGenerationTransmissionInProgress(); }); - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), - "newGenReqs.size():%ld requests, activeRequests.size():%ld allTransferInProgress:%d original " - "gen_only_requests_num:%ld", - newGenReqs.size(), activeRequests.size(), blockTransfer, genInitReqNum); - mCacheTransceiver->checkGenTransferStatus(0); - auto timeEnd = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration<float, std::milli>(timeEnd - timeStart).count(); - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), - "receiveDisaggGenCache time:%f ms, " - "blockTransfer:%d,genInitReqNum:%ld,newGenReqs.size():%ld,activeRequests.size():%ld", - duration, blockTransfer, genInitReqNum, newGenReqs.size(), activeRequests.size()); - } - - return; -} - -void TrtGptModelInflightBatching::checkDisaggGenTransferStatus(RequestList const& activeRequests) -{ - NVTX3_SCOPED_RANGE(checkDisaggGenTransferStatus); - - if (common::getEnvDisableKVCacheTransferOverlap()) - { - return; - } - - auto timeStart = std::chrono::steady_clock::now(); - - // TODO: - auto const needCheck = std::any_of(activeRequests.begin(), activeRequests.end(), - [](auto const& req) { return req->isDisaggGenerationTransmissionInProgress(); }); - - if (needCheck) - { - mCacheTransceiver->checkGenTransferStatus(0); - - auto timeEnd = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration<float, std::milli>(timeEnd - timeStart).count(); - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), - "no Prepare checkDisaggGenTransferStatus time:%f ms, " - "needCheck:%d,activeRequests.size():%ld", - duration, needCheck, activeRequests.size()); - } -} - -void TrtGptModelInflightBatching::prepareDistGenBufferAndDecoder(RequestVector const& generationRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // set decoderStep for disagg_generation - RequestVector cacheTransCompleteRequests; - for (auto const& request : generationRequests) - { - if (request->isDisaggGenerationTransmissionComplete()) - { - cacheTransCompleteRequests.push_back((request)); - } - } - if (!cacheTransCompleteRequests.empty()) - { - auto timeStart = std::chrono::steady_clock::now(); - auto const bufferId = getFusedBufferId(); - auto& runtimeBuffers = *mBuffers[bufferId]; - runtimeBuffers.prepareStep(cacheTransCompleteRequests, {}, getMaxBeamWidth(), getMaxAttentionWindow(), - *mDecoderState, mKvCacheManager.get(), mCrossKvCacheManager.get(), mRnnStateManager.get(), - mPeftTables[mMicroBatchId], *mRuntime, mModelConfig, mWorldConfig, getGatherGenerationLogits(), - isTrtOverlap()); - auto const contextBufferId = mCtxGenFusion ? getFusedBufferId() : getContextBufferId(); - setupDecoderStep( - cacheTransCompleteRequests, *mBuffers.at(contextBufferId), mDecoderInputBuffers.at(getFusedBufferId())); - sync_check_cuda_error(mRuntime->getStream().get()); - auto timeEnd = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration<float, std::milli>(timeEnd - timeStart).count(); - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), - "prepareDistGenBufferAndDecoder time:%f ms , cacheTransCompleteRequests.size():%ld", duration, - cacheTransCompleteRequests.size()); - } - for (auto& request : cacheTransCompleteRequests) - { - request->setState(LlmRequestState::kGENERATION_IN_PROGRESS); - request->setContextCurrentPosition(request->mPromptLen); - request->setDecodingIter(1); - auto const reqBeamWidth = request->mSamplingConfig.beamWidth; - auto firstGenTokens = request->getContextPhaseParams().value().getFirstGenTokens(); - for (SizeType32 beam = 0; beam < reqBeamWidth; ++beam) - { - request->addNewToken(firstGenTokens.at(beam), beam); - } - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::debugIOTensors(RequestVector const& contextRequests, - RequestVector const& generationRequests, TensorMap const& inputMap, TensorMap const& outputMap) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - TLLM_CHECK(mDebugConfig); - - auto const& manager = mRuntime->getBufferManager(); - auto requestIds = utils::collectRequestIds(contextRequests, generationRequests); - - if (mDebugConfig->getDebugTensorsMaxIterations() > 0) - { - mLastIterationDebugTensors.clear(); - mLastIterationDebugTensors = utils::storeIOTensors(*mDebugConfig, requestIds, inputMap, outputMap, manager); - } - else - { - utils::dumpIOTensors(*mDebugConfig, mIterCounter, requestIds, inputMap, outputMap, mWorldConfig, manager); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -std::tuple<SizeType32, runtime::StringPtrMap<runtime::ITensor> const&, runtime::StringPtrMap<runtime::ITensor>&> -TrtGptModelInflightBatching::prepareBuffers( - RequestVector const& contextRequests, RequestVector const& generationRequests, SizeType32 bufferId) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(prepareBuffers); - - auto& runtimeBuffers = *mBuffers.at(bufferId); - - auto allNewTokens = mWorldConfig.isLastPipelineParallelRank() - ? RuntimeBuffers::OptionalRef<runtime::ITensor const>(mDecoderState->getAllNewTokens()) - : std::nullopt; - - auto [optProfileId, inputMap, outputMap] = runtimeBuffers.prepareStep(contextRequests, generationRequests, - mOperatingBeamWidth, getMaxAttentionWindow(), *mDecoderState, mKvCacheManager.get(), mCrossKvCacheManager.get(), - mRnnStateManager.get(), mPeftTables[bufferId], *mRuntime, mModelConfig, mWorldConfig, - getGatherGenerationLogits(), isTrtOverlap(), allNewTokens); - - // For Variable-Beam-Width-Search - mRuntime->setCurrentBeamWidths( - tensorrt_llm::batch_manager::utils::getRequestBeamWidths(contextRequests, generationRequests)); - - mRuntime->setInputTensors(optProfileId, inputMap); - mRuntime->setOutputTensors(optProfileId, outputMap); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return {optProfileId, inputMap, outputMap}; -} - -void TrtGptModelInflightBatching::prepareGraph(SizeType32 bufferId, SizeType32 optProfileId) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(prepareGraph); - - auto const nextBatchState = mBuffers[bufferId]->getBatchState(); - auto cudaGraphOpt = mCudaGraphExecutorCaches[bufferId].get(nextBatchState); - // If graph is not found in the cache, capture it. - if (!cudaGraphOpt.has_value()) - { - // We need to prepare some tensors once again to properly set values for graph capture. - // Graph capture requires setting some tensors (e.g. past_kv_len) - // to the round_up(max_kv_cache_len, kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE) - // in order to capture the kernels with the large enough grid. - mBuffers[bufferId]->prepareBuffersForCudaGraph(getMaxSequenceLen()); - - auto cudaGraph = std::make_shared<utils::CudaGraphExecutor>(); - cudaGraph->prepareNextGraph(mRuntime, optProfileId); - mCudaGraphExecutorCaches[bufferId].put(nextBatchState, cudaGraph); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::executeStep( - RequestVector const& contextRequests, RequestVector const& generationRequests, SizeType32 bufferId) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE_WITH_NAME(range, - "executeStep: " + std::to_string(contextRequests.size()) + " ctx reqs, " - + std::to_string(generationRequests.size()) + " gen reqs"); - - if (mPromptTableOffloading) - { - prefetchNextPromptTableChunk(contextRequests, /* isFirstChunk */ true, bufferId); - } - - auto [optProfileId, inputMap, outputMap] = prepareBuffers(contextRequests, generationRequests, bufferId); - - if (mBuffers[bufferId]->transformerBuffers) - { - // Creation of context progress, or remains nullptr if not needed - std::shared_ptr<ContextProgress> progress = nullptr; - RequestVector layerWiseRequests; - if (common::getEnvDisaggLayerwise()) - { - for (auto const& request : contextRequests) - { - bool const enableLayerWise = request->isContextOnlyRequest() && request->isLastContextChunk(); - if (enableLayerWise) - { - layerWiseRequests.push_back(request); - } - } - } - // TODO: support layer-wise cross kv cache in encoder-decoder models - if (!layerWiseRequests.empty() && !mModelConfig.useCrossAttention()) - { - int const numLayers = mModelConfig.getNbAttentionLayers( - mWorldConfig.getPipelineParallelism(), mWorldConfig.getPipelineParallelRank()); - progress = std::make_shared<ContextProgress>(numLayers); - } - bufferCast<void*>(*mBuffers[bufferId]->transformerBuffers->contextProgressHost)[0] = progress.get(); - if (progress) - { - TLLM_CHECK_WITH_INFO(mCacheTransceiver, - "Disaggregated serving is not enabled, please check the configuration of cacheTransceiverConfig."); - mCacheTransceiver->respondAndSendLayerWise(layerWiseRequests, progress); - } - } - - if (mPromptTableOffloading) - { - prefetchNextPromptTableChunk(contextRequests, /* isFirstChunk */ false, bufferId); - } - - executeContext(optProfileId, bufferId); - - // If batch state has any context request, do not capture this graph. - if (isCudaGraphMode() && contextRequests.empty()) - { - // Capture graph of current batch state during engine execution. - // This is based on the assumptions that - // a) We can hide CPU graph capture behind the GPU engine execution. - // b) Batch size in the next iterations won't change and we can reuse the graph multiple times. - prepareGraph(bufferId, optProfileId); - } - - if (mDebugConfig) - { - debugIOTensors(contextRequests, generationRequests, inputMap, outputMap); - } - - if (mAdditionalModelOutputs.has_value() && !mAdditionalModelOutputs.value().empty()) - { - utils::copyAdditionalOutputs( - mAdditionalModelOutputs.value(), contextRequests, generationRequests, outputMap, getBufferManager()); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::setupDecoderStep( - RequestVector const& contextRequests, RuntimeBuffers const& buffers, DecoderInputBuffers& inputBuffers) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(setupDecoderStep); - - if (mWorldConfig.isLastPipelineParallelRank() && !contextRequests.empty()) - { - auto const logitsType = mRuntime->getEngine().getTensorDataType("logits"); - - auto [batchSlots, samplingConfigs, lookaheadPrompt, lookaheadAlgoConfigs] - = (*mCreateNewDecoderRequests)(mModelConfig, mWorldConfig, mDecodingConfig, contextRequests, logitsType, - inputBuffers, *mDecoderState, mRuntime->getStream(), *mDecoder->getDecoderStream(), getMaxSequenceLen(), - mOperatingBeamWidth, buffers.mMedusaBuffers); - - auto const localBatchSize = batchSlots->getSize(); - if (localBatchSize > 0) - { - auto samplingConfig = SamplingConfig(samplingConfigs); - mDecoder->getUnderlyingDecoder().setup(samplingConfig, localBatchSize, batchSlots, - {mDecoderState->getJointDecodingOutput()}, mModelConfig.getDataType(), lookaheadPrompt, - lookaheadAlgoConfigs); - - auto const& stream = mDecoder->getDecoderStream(); - CudaEvent event{}; - stream->record(event); - mRuntime->getStreamPtr()->wait(event); - } - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::postProcessRequest( - LlmRequest& llmReq, std::vector<SizeType32> const& numDroppedTokens) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - auto const seqSlot = llmReq.mSeqSlot.value(); - auto const reqBeamWidth = llmReq.getBeamWidthByIter(true); - auto const& bufferManager = getBufferManager(); - - if (llmReq.getReturnGenerationLogits() && !llmReq.getGenerationLogitsFragments().empty()) - { - TLLM_CHECK(!llmReq.isStreaming()); - auto const genBufferId = mCtxGenFusion ? getFusedBufferId() : getGenerationBufferId(); - auto& genRuntimeBuffers = *mBuffers.at(genBufferId); - - auto constexpr beforeDecoder = false; - utils::copyGenerationLogits( - genRuntimeBuffers.generationLogitsCache, bufferManager, llmReq, beforeDecoder, numDroppedTokens); - - bufferManager.getStream().synchronize(); - } - - if (reqBeamWidth == 1) - { - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return; - } - - // Update mDecoderBuffers->slotOutputIdsHost and synchronize - getDecoderSlotHostOutputs(seqSlot, llmReq.returnLogProbs(), llmReq.mSamplingConfig, llmReq.isStreaming()); - - auto const* outputIdsHostData = bufferCast<TokenIdType>(*mSlotDecoderBuffers[seqSlot]->outputIdsHost); - auto const* sequenceLengthsHostData = bufferCast<SizeType32>(*mSlotDecoderBuffers[seqSlot]->sequenceLengthsHost); - auto const* cumLogProbsHostData = bufferCast<float>(*mSlotDecoderBuffers[seqSlot]->cumLogProbsHost); - auto logProbsHost = mSlotDecoderBuffers[seqSlot]->logProbsHost; - auto const* logProbsHostData = bufferCast<float>(*logProbsHost); - - auto const& outputIdsShape = mSlotDecoderBuffers[seqSlot]->outputIdsHost->getShape(); - auto const maxSeqLength = outputIdsShape.d[1]; - - std::vector<std::vector<TokenIdType>> generatedTokens(reqBeamWidth); - for (SizeType32 beam = 0; beam < reqBeamWidth; ++beam) - { - auto const* const begin = outputIdsHostData + tc::flat_index2(beam, llmReq.mPromptLen, maxSeqLength); - auto const generatedLength = sequenceLengthsHostData[beam] - llmReq.mPromptLen; - auto const* const end = begin + generatedLength; - generatedTokens[beam].assign(begin, end); - - if (llmReq.returnLogProbs()) - { - llmReq.setCumLogProb(cumLogProbsHostData[beam], beam); - - auto const beginLogProbsOffset = reqBeamWidth == 1 ? llmReq.mPromptLen : 0; - auto const* const begin = logProbsHostData + beam * logProbsHost->getShape().d[1] + beginLogProbsOffset; - auto const* const end = begin + generatedLength; - LlmRequest::VecLogProbs logProbs(begin, end); - llmReq.setLogProbs(logProbs, beam); - } - } - - // store the generated tokens into the mTokensGathered buffer - llmReq.setGeneratedTokens(generatedTokens); - - if (llmReq.getReturnGenerationLogits() && llmReq.getGenerationLogitsHost() - && mWorldConfig.isLastPipelineParallelRank()) - { - reorderGenerationLogitsForBeamSearch( - llmReq, seqSlot, reqBeamWidth, maxSeqLength, outputIdsHostData, sequenceLengthsHostData); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::reorderGenerationLogitsForBeamSearch(LlmRequest& llmReq, SizeType32 seqSlot, - SizeType32 reqBeamWidth, SizeType32 maxSeqLength, TokenIdType const* outputIdsHostData, - SizeType32 const* sequenceLengthsHostData) -{ - // Reorder generation logits to match the gathered (finalized) beam ordering. - // During generation, logits are stored indexed by beam SLOT position. After beam search - // finalization (gatherTree), output_ids are reordered by tracing parentIds to reconstruct - // the correct beam paths. However, generation_logits are NOT reordered by gatherTree. - // We fix this here by tracing parentIds on the host to build the beam-slot mapping, - // then reindexing the logits accordingly. - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const promptLen = llmReq.mPromptLen; - - // Copy parentIds and ids (ungathered step IDs) from GPU to temporary host buffers. - // parentIds[slot][t] = the parent slot of beam slot `slot` at position t. - // ids[slot][t] = the token in beam slot `slot` at position t (before gather). - auto parentIdsDevice = ITensor::at(mDecoderState->getParentIds(), {seqSlot}); - auto idsDevice = mDecoderState->getIds(seqSlot); - - auto parentIdsHost = runtime::BufferManager::pinnedPool(parentIdsDevice->getShape(), nvinfer1::DataType::kINT32); - auto idsHost = runtime::BufferManager::pinnedPool(idsDevice->getShape(), nvinfer1::DataType::kINT32); - - mCopyBufferManager.copy(*parentIdsDevice, *parentIdsHost); - mCopyBufferManager.copy(*idsDevice, *idsHost); - mCopyBufferManager.getStream().synchronize(); - - auto const* parentIdsData = bufferCast<TokenIdType>(*parentIdsHost); - auto const* idsData = bufferCast<TokenIdType>(*idsHost); - - // For each final beam b, find the beam slot at the last generated step, then - // trace back through parentIds to build the slot trace for every generation step. - // slotTrace[beam][genStep] = the beam slot that produced the logits at that step. - auto const generationLogitsHost = llmReq.getGenerationLogitsHost(); - auto const& logitsShape = generationLogitsHost->getShape(); - // Non-streaming shape: [beamWidth, maxNewTokens, vocabSizePadded] - TLLM_CHECK_WITH_INFO(logitsShape.d[0] == reqBeamWidth, - "Generation logits beam dimension (%ld) does not match beam width (%d).", logitsShape.d[0], reqBeamWidth); - auto const maxNewTokens = logitsShape.d[1]; - auto const vocabSizePadded = logitsShape.d[2]; - - std::vector<std::vector<SizeType32>> slotTrace(reqBeamWidth, std::vector<SizeType32>(maxNewTokens, 0)); - bool anyReorderNeeded = false; - - for (SizeType32 beam = 0; beam < reqBeamWidth; ++beam) - { - auto const seqLen = sequenceLengthsHostData[beam]; - auto const genLen = seqLen - promptLen; - if (genLen <= 0) - { - continue; - } - - // Find the starting beam slot at the last generated step by matching the - // backtracked token sequence against the gathered (finalized) output. - SizeType32 startSlot = -1; - for (SizeType32 s = 0; s < reqBeamWidth; ++s) - { - SizeType32 slot = s; - bool matches = true; - for (SizeType32 t = seqLen - 1; t >= promptLen; --t) - { - if (idsData[slot * maxSeqLength + t] != outputIdsHostData[beam * maxSeqLength + t]) - { - matches = false; - break; - } - if (t > promptLen) - { - slot = parentIdsData[slot * maxSeqLength + t]; - } - } - if (matches) - { - startSlot = s; - break; - } - } - - TLLM_CHECK_WITH_INFO(startSlot >= 0, - "Could not determine beam slot mapping for beam %d during generation logits reordering.", beam); - - // Build the slot trace: slotTrace[beam][g] = the pre-reassignment slot whose - // logits correspond to generation step g of this beam. - // - // The model runs BEFORE beam search reassigns beams to slots, so - // generationLogits[slot][g] was produced by the pre-reassignment slot — - // i.e. the slot the beam occupied in the *previous* step. - // parentIds[postSlot][promptLen+g] gives exactly that pre-reassignment slot, - // so taking the parentIds lookup before storing (rather than after) yields - // the correct source slot in a single pass. - SizeType32 slot = startSlot; - for (SizeType32 t = seqLen - 1; t >= promptLen; --t) - { - slot = parentIdsData[slot * maxSeqLength + t]; - slotTrace[beam][t - promptLen] = slot; - } - - // Check if any reordering is actually needed for this beam - auto& slotTraceIds = slotTrace[beam]; - anyReorderNeeded |= std::any_of( - slotTraceIds.begin(), slotTraceIds.begin() + genLen, [beam](SizeType32 s) { return s != beam; }); - } - - // Reorder the generation logits in-place using a per-step temporary buffer. - if (anyReorderNeeded) - { - auto const logitsDataType = generationLogitsHost->getDataType(); - auto const elemSize = runtime::BufferDataType(logitsDataType).getSize(); - auto const stepSize = static_cast<size_t>(vocabSizePadded) * elemSize; - - // Temp buffer for one generation step across all beams: [beamWidth, vocabSizePadded] - auto tempLogits - = runtime::BufferManager::pinnedPool(ITensor::makeShape({reqBeamWidth, vocabSizePadded}), logitsDataType); - - auto* logitsPtr = static_cast<uint8_t*>(generationLogitsHost->data()); - auto* tempPtr = static_cast<uint8_t*>(tempLogits->data()); - - std::vector<SizeType32> genLens(reqBeamWidth); - SizeType32 maxGenLen = 0; - for (SizeType32 b = 0; b < reqBeamWidth; ++b) - { - genLens[b] = std::max(SizeType32{0}, sequenceLengthsHostData[b] - promptLen); - maxGenLen = std::max(maxGenLen, genLens[b]); - } - - for (SizeType32 g = 0; g < maxGenLen; ++g) - { - // Check if any beam that generated this step needs reordering - bool stepNeedsReorder = false; - for (SizeType32 b = 0; b < reqBeamWidth; ++b) - { - if (g < genLens[b] && slotTrace[b][g] != b) - { - stepNeedsReorder = true; - break; - } - } - if (!stepNeedsReorder) - { - continue; - } - - // Copy all beams' logits at this step to the temp buffer - for (SizeType32 b = 0; b < reqBeamWidth; ++b) - { - // logits layout: [beamWidth, maxNewTokens, vocabSizePadded] - auto const offset = (static_cast<size_t>(b) * maxNewTokens + g) * stepSize; - std::memcpy(tempPtr + static_cast<size_t>(b) * stepSize, logitsPtr + offset, stepSize); - } - - // Reorder: logits[b][g] = temp[slotTrace[b][g]] - for (SizeType32 b = 0; b < reqBeamWidth; ++b) - { - if (g >= genLens[b]) - { - continue; - } - auto const dstOffset = (static_cast<size_t>(b) * maxNewTokens + g) * stepSize; - auto const srcSlot = slotTrace[b][g]; - std::memcpy(logitsPtr + dstOffset, tempPtr + static_cast<size_t>(srcSlot) * stepSize, stepSize); - } - } - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::getDecoderSlotHostOutputs( - SizeType32 seqSlot, bool returnLogProbs, SamplingConfig const& samplingConfig, bool streaming) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - if (mWorldConfig.isLastPipelineParallelRank()) - { - auto event = mDecoder->finalize(*mDecoderState, seqSlot, samplingConfig, streaming); - // Make sure that postprocessing is done before copying outputIds - mCopyBufferManager.getStream().wait(event.get()); - - auto sequenceLengths = mDecoderState->getSequenceLengths(seqSlot); - auto outputIds = mDecoderState->getGatheredIds(seqSlot); - auto cumLogProbs = mDecoderState->getCumLogProbs(seqSlot); - auto logProbs = mDecoderState->getLogProbs(seqSlot); - - mCopyBufferManager.copy(*sequenceLengths, *mSlotDecoderBuffers[seqSlot]->sequenceLengths); - mCopyBufferManager.copy(*outputIds, *mSlotDecoderBuffers[seqSlot]->outputIds); - if (returnLogProbs) - { - mCopyBufferManager.copy(*cumLogProbs, *mSlotDecoderBuffers[seqSlot]->cumLogProbs); - mCopyBufferManager.copy(*logProbs, *mSlotDecoderBuffers[seqSlot]->logProbs); - } - - if (mWorldConfig.isPipelineParallel()) - { - // Make sure that postprocessing is done before sending outputIds - event.synchronize(); - - auto const peerSend = 0; - mDecSlotAsyncSndHdls.emplace_back(std::make_unique<DecoderSlotAsyncSend>( - outputIds, sequenceLengths, cumLogProbs, logProbs, returnLogProbs, *mMpiCommPipelinePara, peerSend)); - } - } - else - { - auto const peerRecv = mWorldConfig.getPipelineParallelRank() == 0 ? mWorldConfig.getPipelineParallelism() - 1 - : mWorldConfig.getPipelineParallelRank() - 1; - DecoderSlotAsyncSend::recv(*mSlotDecoderBuffers[seqSlot], returnLogProbs, *mMpiCommPipelinePara, peerRecv); - - auto const peerSend = mWorldConfig.getPipelineParallelRank() + 1; - if (peerSend != mWorldConfig.getPipelineParallelism() - 1) - { - mDecSlotAsyncSndHdls.emplace_back(std::make_unique<DecoderSlotAsyncSend>( - *mSlotDecoderBuffers[seqSlot], returnLogProbs, *mMpiCommPipelinePara, peerSend)); - } - } - sync_check_cuda_error(mRuntime->getStream().get()); - - // Here copy stream is synchronized after receiving decoderSlotOutputIdsView either by copy or by receive - // before copying to host on copy stream - runtime::CudaEvent beforeEvent{}; - mRuntime->getStreamPtr()->record(beforeEvent); - mCopyBufferManager.getStream().wait(beforeEvent); - mCopyBufferManager.copy(*mSlotDecoderBuffers[seqSlot]->outputIds, *mSlotDecoderBuffers[seqSlot]->outputIdsHost); - mCopyBufferManager.copy( - *mSlotDecoderBuffers[seqSlot]->sequenceLengths, *mSlotDecoderBuffers[seqSlot]->sequenceLengthsHost); - - if (returnLogProbs) - { - mCopyBufferManager.copy( - *mSlotDecoderBuffers[seqSlot]->cumLogProbs, *mSlotDecoderBuffers[seqSlot]->cumLogProbsHost); - mCopyBufferManager.copy(*mSlotDecoderBuffers[seqSlot]->logProbs, *mSlotDecoderBuffers[seqSlot]->logProbsHost); - } - - // Make sure copy is done before continuing on host - mCopyBufferManager.getStream().synchronize(); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -namespace -{ -// Check if one of the request needs log probs, need to get from decoder and communicate -bool batchReturnLogProbs(ScheduledRequests const& scheduledRequests) -{ - auto pred = [](auto const& llmReq) { return llmReq->returnLogProbs(); }; - return std::any_of(scheduledRequests.contextRequests.begin(), scheduledRequests.contextRequests.end(), pred) - || std::any_of(scheduledRequests.generationRequests.begin(), scheduledRequests.generationRequests.end(), pred); -} -} // namespace - -runtime::CudaEvent TrtGptModelInflightBatching::decoderStepAsync(ScheduledRequests const& scheduledRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(decoderStepAsync); - - auto& decoderInputBuffers = mDecoderInputBuffers.at(getFusedBufferId()); - - auto const contextBufferId = mCtxGenFusion ? getFusedBufferId() : getContextBufferId(); - auto& contextRuntimeBuffers = mBuffers.at(contextBufferId); - auto const logitsIndex = (*mHandleContextLogits)(decoderInputBuffers, scheduledRequests.contextRequests, - contextRuntimeBuffers->logits, contextRuntimeBuffers->numContextLogits, mModelConfig, - mRuntime->getBufferManager(), contextRuntimeBuffers->mMedusaBuffers); - - auto const genLogitsIndex = mCtxGenFusion ? logitsIndex : 0; - auto const genBufferId = mCtxGenFusion ? getFusedBufferId() : getGenerationBufferId(); - auto& genRuntimeBuffers = mBuffers.at(genBufferId); - (*mHandleGenerationLogits)(decoderInputBuffers, scheduledRequests.generationRequests, genRuntimeBuffers->logits, - genLogitsIndex, mModelConfig, mRuntime->getBufferManager(), *genRuntimeBuffers, - genRuntimeBuffers->mMedusaBuffers); - - if (mOperatingBeamWidth > 1) - { - copyCacheIndirectionFromOutputsToInputs(scheduledRequests, genBufferId); - } - - mLogitsPostProcessorIsApplied = (*mLogitsPostProcessor)(decoderInputBuffers, mReplicateLogitsPostProcessor, - mWorldConfig, mRuntime->getStreamPtr(), mLogitsPostProcessorBatched); - - if (mGuidedDecoder) - { - mGuidedDecoder->execute(decoderInputBuffers, mRuntime->getBufferManager()); - } - - auto const fusedBufferId = getFusedBufferId(); - auto& fusedRuntimeBuffers = mBuffers.at(fusedBufferId); - - (*mMakeDecodingBatchInputOutput)(decoderInputBuffers, *mDecoderState, mModelConfig, *fusedRuntimeBuffers); - - auto decoderFinishEvent = mDecoder->forwardAsync(*mDecoderState, decoderInputBuffers); - - auto const returnLogProbs = batchReturnLogProbs(scheduledRequests); - auto updateDecoderBuffersEvent = (*mUpdateDecoderBuffers)(mModelConfig, mDecoderOutputBuffers.at(fusedBufferId), - mRuntime->getBufferManager(), *mDecoderState, returnLogProbs, decoderFinishEvent); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return updateDecoderBuffersEvent; -} - -void TrtGptModelInflightBatching::copyCacheIndirectionFromOutputsToInputs( - ScheduledRequests const& scheduledRequests, SizeType32 genBufferId) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(copyCacheIndirectionFromOutputsToInputs); - - auto& genRuntimeBuffers = *mBuffers.at(genBufferId); - auto* srcOffsetsPtr = bufferCast<SizeType64>(*genRuntimeBuffers.cacheIndirDecoderIOBatchedCopySrcOffsets); - auto* dstOffsetsPtr = bufferCast<SizeType64>(*genRuntimeBuffers.cacheIndirDecoderIOBatchedCopyDstOffsets); - auto* copySizesPtr = bufferCast<SizeType64>(*genRuntimeBuffers.cacheIndirDecoderIOBatchedCopySizes); - - // Only `cacheIndirShape.d[2]` is used - auto const& cacheIndirShape = mDecoderState->getCacheIndirectionOutput()->getShape(); - auto const maxBeamWidth = cacheIndirShape.d[1]; - auto const maxAttentionWindow = cacheIndirShape.d[2]; - auto const slotOffset = maxBeamWidth * maxAttentionWindow; - - SizeType32 batchIdx{0}; - SizeType64 maxCopySize{0}; - auto& manager = mRuntime->getBufferManager(); - for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) - { - for (auto const& llmReq : requests) - { - auto const reqBeamWidth = llmReq->getBeamWidthByIter(); - auto const seqSlot = llmReq->mSeqSlot.value(); - auto const copySize = reqBeamWidth * maxAttentionWindow; - srcOffsetsPtr[batchIdx] = seqSlot * slotOffset; - dstOffsetsPtr[batchIdx] = seqSlot * slotOffset; - copySizesPtr[batchIdx] = copySize; - maxCopySize = std::max(maxCopySize, copySize); - batchIdx++; - } - } - if (batchIdx != 0) - { - auto const srcOffsetsSlice - = ITensor::slice(genRuntimeBuffers.cacheIndirDecoderIOBatchedCopySrcOffsets, 0, batchIdx); - auto const srcOffsetsSliceDeviceSlice - = ITensor::slice(genRuntimeBuffers.mCacheIndirDecoderIOBatchedCopySrcOffsetsSliceDevice, 0, batchIdx); - manager.copy(srcOffsetsSlice->data(), *srcOffsetsSliceDeviceSlice, - runtime::MemoryType::kGPU); // Explicitly move to device for faster access. - auto const dstOffsetsSlice - = ITensor::slice(genRuntimeBuffers.cacheIndirDecoderIOBatchedCopyDstOffsets, 0, batchIdx); - auto const dstOffsetsSliceDeviceSlice - = ITensor::slice(genRuntimeBuffers.mCacheIndirDecoderIOBatchedCopyDstOffsetsSliceDevice, 0, batchIdx); - manager.copy(dstOffsetsSlice->data(), *dstOffsetsSliceDeviceSlice, - runtime::MemoryType::kGPU); // Explicitly move to device for faster access. - auto const sizesSlice = ITensor::slice(genRuntimeBuffers.cacheIndirDecoderIOBatchedCopySizes, 0, batchIdx); - auto const copySizesDeviceSlice - = ITensor::slice(genRuntimeBuffers.mCacheIndirDecoderIOBatchedCopyCopySizesDevice, 0, batchIdx); - manager.copy(sizesSlice->data(), *copySizesDeviceSlice); // Explicitly move to device for faster access. - runtime::kernels::invokeCopyBatch(*mDecoderState->getCacheIndirectionOutput(), - *mDecoderState->getCacheIndirectionInput(), *srcOffsetsSliceDeviceSlice, *dstOffsetsSliceDeviceSlice, - *copySizesDeviceSlice, maxCopySize, manager.getStream()); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -std::vector<std::unique_ptr<DecoderStepAsyncSend>> TrtGptModelInflightBatching::communicateDecoderBuffers( - bool returnLogProbs) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(communicateDecoderBuffers); - - auto& decoderOutputBuffers = mDecoderOutputBuffers.at(getFusedBufferId()); - - std::vector<std::unique_ptr<DecoderStepAsyncSend>> asyncHandles; - if (mWorldConfig.isLastPipelineParallelRank()) - { - if (broadcastPostDecoder()) - { - DecoderStepAsyncSend::bcast(decoderOutputBuffers, *mDecoderState, returnLogProbs, mOperatingBeamWidth, - mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind(), *mMpiCommTensorPara, 0); - } - - if (mWorldConfig.isPipelineParallel()) - { - auto const peerSend = 0; - asyncHandles.emplace_back(std::make_unique<DecoderStepAsyncSend>(decoderOutputBuffers, *mDecoderState, - returnLogProbs, mOperatingBeamWidth, mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind(), - *mMpiCommPipelinePara, peerSend)); - } - } - else - { - auto const peerRecv = mWorldConfig.isFirstPipelineParallelRank() ? mWorldConfig.getPipelineParallelism() - 1 - : mWorldConfig.getPipelineParallelRank() - 1; - DecoderStepAsyncSend::recv(decoderOutputBuffers, *mDecoderState, returnLogProbs, mOperatingBeamWidth, - mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind(), *mMpiCommPipelinePara, peerRecv); - auto const peerSend = mWorldConfig.getPipelineParallelRank() + 1; - if (peerSend != mWorldConfig.getPipelineParallelism() - 1) - { - asyncHandles.emplace_back(std::make_unique<DecoderStepAsyncSend>(decoderOutputBuffers, *mDecoderState, - returnLogProbs, mOperatingBeamWidth, mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind(), - *mMpiCommPipelinePara, peerSend)); - } - } - TLLM_CHECK_WITH_INFO(asyncHandles.size() <= 2, "Up to two decoder step async handles expected"); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return asyncHandles; -} - -void TrtGptModelInflightBatching::updateRequests(ScheduledRequests const& scheduledRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(updateRequests); - - auto const& decoderOutputBuffers = mDecoderOutputBuffers.at(getFusedBufferId()); - - auto const hostNewOutputTokensShape = decoderOutputBuffers.newOutputTokensHost->getShape(); - auto const* const hostNewOutputTokensData - = bufferCast<TokenIdType const>(*decoderOutputBuffers.newOutputTokensHost); - auto const* const sequenceLengthsHostData = bufferCast<SizeType32 const>(*decoderOutputBuffers.sequenceLengthsHost); - auto const* const decoderFinishedSumPtr = bufferCast<SizeType32 const>(*decoderOutputBuffers.finishedSumHost); - auto const* const cumLogProbsPtr = bufferCast<float const>(*decoderOutputBuffers.cumLogProbsHost); - auto const* const logProbsPtr = bufferCast<float const>(*decoderOutputBuffers.logProbsHost); - auto const* const finishReasonsHostData - = bufferCast<kernels::FinishedState>(*decoderOutputBuffers.finishReasonsHost); - - // Update only requests that ran through the decoder - for (auto const& llmReq : scheduledRequests.generationRequests) - { - if (llmReq->isGenerationCompleteState()) - { - continue; - } - auto const reqBeamWidth = llmReq->getBeamWidthByIter(true); - auto const seqSlot = llmReq->mSeqSlot.value(); - auto const currentNumOfTokens = llmReq->getMaxBeamNumTokens(); - - // Save the accepted token logits from target model - if (mModelConfig.getSpeculativeDecodingMode().isDraftTokensExternal() && llmReq->getReturnGenerationLogits() - && llmReq->hasDraftTokens()) - { - TLLM_CHECK_WITH_INFO(reqBeamWidth == 1, "Speculative decoding only works for beam width == 1"); - - SizeType32 numAcceptedTokens - = sequenceLengthsHostData[seqSlot * mOperatingBeamWidth + 0] - llmReq->getMaxBeamNumTokens(); - - auto const& generationLogitsHost = llmReq->getGenerationLogitsHost(); - auto shape = generationLogitsHost->getShape(); - shape.d[1] = numAcceptedTokens; - generationLogitsHost->reshape(shape); - } - - std::vector<SizeType32> numNewTokens(reqBeamWidth); - std::vector<SizeType32> numDroppedTokens(reqBeamWidth); - - // numGeneratedTokens is the number of tokens generated by the decoder. - // Some tokens might be dropped due to end token or rejected draft tokens. - auto const numGeneratedTokens = llmReq->getNumDraftTokens() + 1; - - for (SizeType32 beam = 0; beam < reqBeamWidth; ++beam) - { - // Sequence length is only advanced for accepted tokens. - auto const seqLen = sequenceLengthsHostData[seqSlot * mOperatingBeamWidth + beam]; - // Actual number of tokens that should be added to the request. - auto const numNewOutputTokens = seqLen - llmReq->getNumTokens(beam); - if (reqBeamWidth == 1) - { - TLLM_CHECK_WITH_INFO(numGeneratedTokens >= numNewOutputTokens, - "numNewOutputTokens must not be greater than numGeneratedTokens: " - "numGeneratedTokens %d < numNewOutputTokens %d", - numGeneratedTokens, numNewOutputTokens); - } - numNewTokens[beam] = std::min(numGeneratedTokens, numNewOutputTokens); - numDroppedTokens[beam] = numGeneratedTokens - numNewTokens[beam]; - for (SizeType32 step = 0; step < numNewTokens[beam]; ++step) - { - auto const newTokenIdx = tc::flat_index(hostNewOutputTokensShape.d, step, seqSlot, beam); - auto const newToken = hostNewOutputTokensData[newTokenIdx]; - llmReq->addNewToken(newToken, beam); - TLLM_LOG_DEBUG("request ID %ld beam %d newToken %d", llmReq->mRequestId, beam, newToken); - - if (llmReq->returnLogProbs()) - { - auto const cumLogProb = cumLogProbsPtr[seqSlot * mOperatingBeamWidth + beam]; - llmReq->setCumLogProb(cumLogProb, beam); - - auto const beginLogProbsOffset = reqBeamWidth == 1 ? llmReq->mPromptLen : 0; - SizeType32 offset - = (seqSlot * mOperatingBeamWidth + beam) * getMaxSequenceLen() + beginLogProbsOffset; - auto const generatedLength = seqLen - llmReq->mPromptLen; - std::vector<float> logProbs(logProbsPtr + offset, logProbsPtr + offset + generatedLength); - llmReq->setLogProbs(logProbs, beam); - } - } - - auto const finishReason = finishReasonsHostData[seqSlot * mOperatingBeamWidth + beam]; - llmReq->setFinishedReason(finishReason.toFinishReason(), beam); - - TLLM_LOG_DEBUG("[RANK %d] decoderSync: request ID %lu beam %d tokens %s finished %d", - COMM_SESSION.getRank(), llmReq->mRequestId, beam, common::vec2str(llmReq->getTokens(beam)).c_str(), - static_cast<int>(finishReason.toFinishReason())); - } - - // Set number of tokens predicted per runtime iteration. Will be > 1 for speculative decoding. - llmReq->updateNumTokensPerIteration(llmReq->getMaxBeamNumTokens() - currentNumOfTokens, mModelConfig); - - // Fill new draft tokens for the next step - if (decoderFinishedSumPtr[seqSlot] != reqBeamWidth - && (mModelConfig.getSpeculativeDecodingMode().predictsDraftTokens() - || mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind())) - { - auto const maxDraftTokensLen = mModelConfig.getMaxDecodingDraftTokens(); - auto prevDraftTokensLen = llmReq->getNumDraftTokens(); - - // We overallocate KV cache for EAGLE to the maxDecodingTokens + maxPathLen in order to fit both - // Base model verification (needs up to maxDecodingTokens) and - // Drafter (needs up to maxPathLen of accepted tokens and maxDecodingDraftTokens for new draft tokens). - if (mModelConfig.getSpeculativeDecodingMode().isEagle()) - { - prevDraftTokensLen = mModelConfig.getSpeculativeDecodingModule().getMaxDecodingTokens() - + mModelConfig.getSpeculativeDecodingModule().getMaxPathLen() - 1; - } - - auto nextDraftTokensLen = mModelConfig.getSpeculativeDecodingModule().getMaxDecodingDraftTokens(); - if (mModelConfig.getSpeculativeDecodingMode().variableDraftLength()) - { - auto const* const nextDraftTokensLengthsHostData - = bufferCast<SizeType32 const>(*decoderOutputBuffers.nextDraftTokensLengthsHost); - nextDraftTokensLen = nextDraftTokensLengthsHostData[seqSlot]; - } - TLLM_CHECK(nextDraftTokensLen <= maxDraftTokensLen); - - auto const* const nextDraftTokensHostData - = bufferCast<TokenIdType const>(*decoderOutputBuffers.nextDraftTokensHost); - auto draftTokensShared - = std::make_shared<std::vector<TokenIdType>>(nextDraftTokensHostData + seqSlot * maxDraftTokensLen, - nextDraftTokensHostData + seqSlot * maxDraftTokensLen + nextDraftTokensLen); - - llmReq->setDraftTokens(draftTokensShared); - - // For all phases except context that does not have draft tokens - if (!llmReq->isGenerationCompleteState() && prevDraftTokensLen != 0 - && mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind()) - { - // -1 here is for current 'main' token - auto const acceptedTokensLen = llmReq->getMaxBeamNumTokens() - currentNumOfTokens - 1; - auto const rewindLength = prevDraftTokensLen - acceptedTokensLen; - - TLLM_LOG_DEBUG("request ID %lu (seqSlot %d): accepted %d of %d draft tokens, rewind %d tokens", - llmReq->mRequestId, seqSlot, acceptedTokensLen, prevDraftTokensLen, rewindLength); - TLLM_CHECK(0 <= acceptedTokensLen && acceptedTokensLen <= prevDraftTokensLen); - - // At this point, KV cache rows are already gathered and moved to the right location. - // We can safely rewind (draft - accepted) tokens - mKvCacheManager->rewindKVCache(llmReq->mRequestId, rewindLength); - } - } - - // Terminate if request has finished or if it is speculative decoding target model - if (decoderFinishedSumPtr[seqSlot] == reqBeamWidth - || (mModelConfig.getSpeculativeDecodingMode().isDraftTokensExternal() && llmReq->hasDraftTokens())) - { - postProcessRequest(*llmReq, numDroppedTokens); - - if (!mWorldConfig.isPipelineParallel() || !mWorldConfig.isLastPipelineParallelRank()) - { - if (llmReq->getReturnGenerationLogits() && mSpeculativeDecodingFastLogits && mIsLeaderInOrchMode) - { - std::lock_guard<std::mutex> lk(mDraftRequestsMtx); - mDraftRequestsWaitingToSendLogits.push_back(llmReq); - } - else - { - terminateRequest(llmReq); - } - llmReq->setState(LlmRequestState::kGENERATION_COMPLETE); - } - else - { - llmReq->setState(LlmRequestState::kGENERATION_TO_COMPLETE); - } - } - else - { - // gather tokens in the case of streaming and beam search - if (llmReq->isStreaming() && llmReq->mSamplingConfig.beamWidth > 1) - { - postProcessRequest(*llmReq, numDroppedTokens); - } - if (llmReq->isContextInitState()) - { - llmReq->setState(LlmRequestState::kGENERATION_IN_PROGRESS); - } - - if (isTrtOverlap() && llmReq->willCompleteNextIteration()) - { - // This state prohibits the request from being scheduled for another iteration. It assumes that the next - // iteration has already been scheduled and the request can finish in the next call to updateRequests(). - llmReq->setState(LlmRequestState::kGENERATION_TO_COMPLETE); - } - } - - if (llmReq->getReturnPerfMetrics()) - { - llmReq->updatePerfMetrics(mIterCounter); - } - - llmReq->advanceDecodingIter(); - - if (mWorldConfig.isPipelineParallel() && mWorldConfig.isLastPipelineParallelRank()) - { - for (SizeType32 beam = 0; beam < reqBeamWidth; ++beam) - { - llmReq->setNumPreDecodedTokens(numNewTokens[beam], beam); - } - } - } - - if (mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind()) - { - SizeType32 numSequences{0}; - for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) - { - for (auto const& llmReq : requests) - { - auto const reqBeamWidth = llmReq->mSamplingConfig.beamWidth; - numSequences += reqBeamWidth; - } - } - - TLLM_CHECK_WITH_INFO(mCtxGenFusion, "Current speculative decoding mode requires context-gen fusion IFB"); - rewindKVCacheBlocks(numSequences); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -std::vector<std::unique_ptr<DecoderStepAsyncSend>> TrtGptModelInflightBatching::decoderSync( - ScheduledRequests const& scheduledRequests, std::optional<runtime::CudaEvent> const& decoderFinishEvent) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(decoderSync); - - if (mWorldConfig.isLastPipelineParallelRank()) - { - decoderFinishEvent->synchronize(); - } - - auto const returnLogProbs = batchReturnLogProbs(scheduledRequests); - auto asyncHandles = communicateDecoderBuffers(returnLogProbs); - - updateRequests(scheduledRequests); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return asyncHandles; -} - -void TrtGptModelInflightBatching::rewindKVCacheBlocks(SizeType32 numSequences) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - auto const bufferId = getFusedBufferId(); - auto& runtimeBuffers = *mBuffers.at(bufferId); - auto& decoderOutputBuffers = mDecoderOutputBuffers.at(bufferId); - - auto localNbLayers = mModelConfig.getNbAttentionLayers( - mWorldConfig.getPipelineParallelism(), mWorldConfig.getPipelineParallelRank()); - if (mWorldConfig.isLastPipelineParallelRank() && mModelConfig.getSpeculativeDecodingMode().isEagle()) - { - // Do not correct the last kv caches, which are for EagleNet drafter. Those KV caches are managed separately. - auto eagleModulePtr - = std::dynamic_pointer_cast<runtime::EagleModule>(mModelConfig.getSpeculativeDecodingModulePtr()); - localNbLayers -= eagleModulePtr->getNumTransformerLayers(); - } - - auto const tokensPerBlock = mModelConfig.getTokensPerBlock(); - auto const elemSize = BufferDataType(mModelConfig.getKvDataType()).getSize(); - auto const sizeInBytesPerKVHead = mModelConfig.getSizePerHead() * elemSize; - - auto const poolPointers = mKvCacheManager->getBlockPoolPointers(); - auto* const* pointerArrayPtr = bufferCast<void*>(*poolPointers); - auto const* offsetArrayPtr - = bufferCast<tk::KVCacheIndex>(*runtimeBuffers.transformerBuffers->kvCacheBlockOffsetsDevice); - - auto commonRewindLen = mModelConfig.getSpeculativeDecodingModule().getMaxDecodingDraftTokens(); - SizeType32 const* rewindLens = nullptr; - if (mModelConfig.getSpeculativeDecodingMode().variableDraftLength()) - { - commonRewindLen = 0; - rewindLens = bufferCast<SizeType32 const>(*decoderOutputBuffers.prevDraftTokensLengthsHost); - } - - tensorrt_llm::runtime::kernels::invokeUpdateKVBlockArrayDraftTokenLocation( - *mDecoderState->getAcceptedLengthsCumSum(), *mDecoderState->getAcceptedPackedPaths(), - *runtimeBuffers.sequenceLengthsDevice, pointerArrayPtr, offsetArrayPtr, localNbLayers, numSequences, - mRewindInputs.numKvHeads, sizeInBytesPerKVHead, commonRewindLen, rewindLens, *runtimeBuffers.seqSlots, - getMaxAttentionWindow(), mRewindInputs.maxBlocksPerSeq, tokensPerBlock, mRewindInputs.isUseOneMoreBlock, - mRuntime->getStreamPtr()->get()); - - sync_check_cuda_error(mRuntime->getStream().get()); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -nvinfer1::DataType TrtGptModelInflightBatching::getLogitDataType() const -{ - return mModelConfig.getLogitsDtype(); -} - -TrtGptModelInflightBatching::SizeType32 TrtGptModelInflightBatching::numCachedCudaGraphs() const -{ - return std::accumulate(mCudaGraphExecutorCaches.begin(), mCudaGraphExecutorCaches.end(), SizeType32{0}, - [](SizeType32 sum, auto const& cache) { return sum + cache.size(); }); -} - -void TrtGptModelInflightBatching::changeBeamWidth(SizeType32 beamWidth) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - TLLM_CHECK(mInflightReqIds.empty()); - - TLLM_CHECK_WITH_INFO(beamWidth <= getMaxBeamWidth(), - "Requested beam width %d is larger than configured max beam width %d", beamWidth, getMaxBeamWidth()); - TLLM_LOG_DEBUG("Changing operating beam width from %d to %d", mOperatingBeamWidth, beamWidth); - mOperatingBeamWidth = beamWidth; - - if (isCudaGraphMode()) - { - for (auto& cache : mCudaGraphExecutorCaches) - { - cache.clear(); - } - } - createBuffers(mDecodingConfig, mAdditionalModelOutputs); - createDecoder(mDecodingConfig.getDecodingMode()); - - if (static_cast<bool>(mKvCacheManager)) - { - auto const dims = mKvCacheManager->getOffsetTableDimensions(); - reshapeKvTensors(dims); - } - if (static_cast<bool>(mCrossKvCacheManager)) - { - auto const dims = mCrossKvCacheManager->getOffsetTableDimensions(); - reshapeKvTensors(dims); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::changeSpecDecMode(ScheduledRequests const& scheduledRequests) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - if ((!mModelConfig.getSpeculativeDecodingMode().isLookaheadDecoding() - && !mModelConfig.getSpeculativeDecodingMode().isNone()) - || scheduledRequests.empty() || mSeamlessLADMaxDraftLen == 0 || getGatherGenerationLogits() - || mModelConfig.isRnnBased()) - { - return; - } - - bool canUseLookahead = false; - auto maxNumRequestForLad = mDecodingConfig.getLookaheadDecodingMaxNumRequest(); - SizeType32 numRequests = scheduledRequests.contextRequests.size() + scheduledRequests.generationRequests.size(); - if (numRequests > maxNumRequestForLad) - { - if (mModelConfig.getSpeculativeDecodingMode().isLookaheadDecoding()) - { - canUseLookahead = false; - } - else - { - return; - } - } - { - bool useTopKTopP = false; - bool useBanWords = false; - bool useTempAccVocabPenalties = false; // use temperature and penalties that need to accumulate #vocab. - SizeType32 beamWidth = 1; - for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) - { - for (auto const& llmReq : requests) - { - useTopKTopP |= !(llmReq->mSamplingConfig.useDefaultValues( - llmReq->mSamplingConfig.topK, layers::DefaultDecodingParams::getTopK()) - || llmReq->mSamplingConfig.useDefaultValues(llmReq->mSamplingConfig.topK, 1)); - useTopKTopP |= !llmReq->mSamplingConfig.useDefaultValues( - llmReq->mSamplingConfig.topP, layers::DefaultDecodingParams::getTopP()); - useBanWords |= llmReq->getBadWordsList().has_value(); - useBanWords |= !llmReq->mSamplingConfig.useDefaultValues( - llmReq->mSamplingConfig.noRepeatNgramSize, layers::DefaultDecodingParams::getNoRepeatNgramSize()); - useTempAccVocabPenalties |= !llmReq->mSamplingConfig.useDefaultValues( - llmReq->mSamplingConfig.temperature, layers::DefaultDecodingParams::getTemperature()); - useTempAccVocabPenalties |= !llmReq->mSamplingConfig.useDefaultValues( - llmReq->mSamplingConfig.repetitionPenalty, layers::DefaultDecodingParams::getRepetitionPenalty()); - useTempAccVocabPenalties |= !llmReq->mSamplingConfig.useDefaultValues( - llmReq->mSamplingConfig.presencePenalty, layers::DefaultDecodingParams::getPresencePenalty()); - useTempAccVocabPenalties |= !llmReq->mSamplingConfig.useDefaultValues( - llmReq->mSamplingConfig.frequencyPenalty, layers::DefaultDecodingParams::getFrequencyPenalty()); - beamWidth = llmReq->mSamplingConfig.beamWidth; - if (useTopKTopP || useBanWords || useTempAccVocabPenalties || beamWidth > 1) - { - break; - } - } - canUseLookahead = !(useTopKTopP || useBanWords || useTempAccVocabPenalties || beamWidth > 1); - } - } - - // Change speculative decoding mode - auto const bufferId = mCtxGenFusion - ? getFusedBufferId() - : (!scheduledRequests.contextRequests.empty() ? getContextBufferId() : getGenerationBufferId()); - // TODO: enable lookahead for generation requests. - bool canChangeToLookahead = scheduledRequests.generationRequests.empty(); - if (mModelConfig.getSpeculativeDecodingMode().isNone() && canUseLookahead && canChangeToLookahead) - { - // None -> Lookahead - mModelConfig.enableSeamlessLookaheadDecoding(mSeamlessLADMaxDraftLen); - mDecodingConfig.enableSeamlessLookaheadDecoding(); - setupSpeculativeDecodingModule(mDecodingConfig); - mBuffers.at(bufferId)->mLookaheadBuffers->enableLookaheadDecoding( - getMaxBatchSize(), mModelConfig.getMaxDecodingTokens()); - mDecoderOutputBuffers.at(getFusedBufferId()) - .enableLookaheadDecoding(getMaxNumSequences(), mModelConfig.getMaxDecodingTokens()); - createDecoder(mDecodingConfig.getDecodingMode()); - } - else if (mModelConfig.getSpeculativeDecodingMode().isLookaheadDecoding() - && (!canUseLookahead || numRequests > maxNumRequestForLad)) - { - // Lookahead -> None - mModelConfig.disableSeamlessLookaheadDecoding(); - mDecodingConfig.setDecodingMode(executor::DecodingMode::Auto()); - mBuffers.at(bufferId)->mLookaheadBuffers->disableLookaheadDecoding(); - mDecoderOutputBuffers.at(getFusedBufferId()).disableLookaheadDecoding(getMaxNumSequences()); - mDecoder->disableLookahead( - scheduledRequests.generationRequests, mDecoderInputBuffers.at(getFusedBufferId()).setupBatchSlots); - mDecoderState->disableLookahead(scheduledRequests.generationRequests); - for (auto const& llmReq : scheduledRequests.generationRequests) - { - if (llmReq->getNumDraftTokens() > 0) - { - llmReq->discardDraftTokens(llmReq->getNumDraftTokens()); - } - } - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TrtGptModelInflightBatching::getCurrentIterationStats(executor::IterationStats& stats) const -{ - stats.iter = mIterCounter; - - // Max batch size and max num tokens can be tuned at runtime - stats.maxBatchSizeStatic = getMaxBatchSize(); - stats.maxBatchSizeTunerRecommended = mMaxBatchSizeTunerRecommended; - stats.maxBatchSizeRuntime = mMaxBatchSizeRuntime; - stats.maxNumTokensStatic = mMaxNumTokensStatic.value_or(0); - stats.maxNumTokensTunerRecommended = mMaxNumTokensTunerRecommended; - stats.maxNumTokensRuntime = mMaxNumTokensRuntime.value_or(0); - - // KVCacheManager statistics - auto const& kvCacheManager = getKVCacheManager(); - if (kvCacheManager) - { - executor::KvCacheStats kvStats{}; - auto kvCacheStats = kvCacheManager->getKvCacheStats(); - kvStats.maxNumBlocks = kvCacheStats.maxNumBlocks; - kvStats.freeNumBlocks = kvCacheStats.freeNumBlocks; - kvStats.usedNumBlocks = kvCacheStats.usedNumBlocks; - kvStats.tokensPerBlock = kvCacheStats.toksPerBlock; - kvStats.allocTotalBlocks = kvCacheStats.allocTotalBlocks; - kvStats.allocNewBlocks = kvCacheStats.allocNewBlocks; - kvStats.reusedBlocks = kvCacheStats.reusedBlocks; - kvStats.missedBlocks = kvCacheStats.missedBlocks; - kvStats.cacheHitRate = kvCacheStats.cacheHitRate; - stats.kvCacheStats = kvStats; - } - auto const& crossKvCacheManager = getCrossKVCacheManager(); - if (crossKvCacheManager) - { - executor::KvCacheStats kvStats{}; - auto kvCacheStats = crossKvCacheManager->getKvCacheStats(); - kvStats.maxNumBlocks = kvCacheStats.maxNumBlocks; - kvStats.freeNumBlocks = kvCacheStats.freeNumBlocks; - kvStats.usedNumBlocks = kvCacheStats.usedNumBlocks; - kvStats.tokensPerBlock = kvCacheStats.toksPerBlock; - kvStats.allocTotalBlocks = kvCacheStats.allocTotalBlocks; - kvStats.allocNewBlocks = kvCacheStats.allocNewBlocks; - kvStats.reusedBlocks = kvCacheStats.reusedBlocks; - kvStats.missedBlocks = kvCacheStats.missedBlocks; - kvStats.cacheHitRate = kvCacheStats.cacheHitRate; - stats.crossKvCacheStats = kvStats; - } - executor::InflightBatchingStats modelStats{}; - modelStats.numScheduledRequests = mLastIterationStatsIFB.scheduledRequests.size(); - modelStats.numContextRequests = mLastIterationStatsIFB.numCtxRequests; - modelStats.numGenRequests = mLastIterationStatsIFB.numGenRequests; - modelStats.numPausedRequests = mLastIterationStatsIFB.pausedRequests.size(); - modelStats.avgNumDecodedTokensPerIter = mLastIterationStatsIFB.avgNumDecodedTokensPerIter; - modelStats.numCtxTokens = mLastIterationStatsIFB.numCtxTokens; - modelStats.microBatchId = mLastIterationStatsIFB.microBatchId; - stats.inflightBatchingStats = modelStats; -} - -void TrtGptModelInflightBatching::getCurrentRequestStats(executor::RequestStatsPerIteration& stats) const -{ - stats.iter = mIterCounter; - for (auto& requestStat : stats.requestStats) - { - requestStat.scheduled - = mLastIterationStatsIFB.scheduledRequests.count(static_cast<RequestIdType>(requestStat.id)); - requestStat.paused = mLastIterationStatsIFB.pausedRequests.count(static_cast<RequestIdType>(requestStat.id)); - } -} - -executor::DebugTensorsPerIteration TrtGptModelInflightBatching::getCurrentDebugTensors() const -{ - executor::DebugTensorsPerIteration debugTensors; - debugTensors.iter = mIterCounter; - - for (auto const& [name, tensor] : mLastIterationDebugTensors) - { - debugTensors.debugTensors.emplace(name, executor::detail::ofITensor(tensor)); - } - - return debugTensors; -} - -nvinfer1::DataType TrtGptModelInflightBatching::getTensorDataType(std::string const& name) const -{ - auto const& engine = mRuntime->getEngine(); - return engine.getTensorDataType(name.c_str()); -} - -nvinfer1::Dims TrtGptModelInflightBatching::getTensorShape(std::string const& name) const -{ - auto const& engine = mRuntime->getEngine(); - return engine.getTensorShape(name.c_str()); -} - -SizeType32 TrtGptModelInflightBatching::getMaxCapacityBatchSize(SizeType32 inputLength, SizeType32 outputLength) const -{ - return mKvCacheManager->getMaxCapacityBatchSize(inputLength, outputLength); -} - -/* - * Manages prefetching of prompt table chunks using a double-buffer strategy - * - * Function Flow: - * 1. First Chunk Processing (isFirstChunk == true): - * - Uses blocking prefetch on main runtime stream - * - Ensures initial data is ready before computation starts - * - * 2. Subsequent Chunks (isFirstChunk == false): - * - Uses non-blocking prefetch on separate copy stream - * - Overlaps data transfer with computation - * - * Synchronization: - * - First prefetch: No wait needed (fresh start) - * - Later prefetches: Wait for previous copy to complete - * - Uses mPtableCopyDoneEvent to track completion - * - * Key Functions: - * 1. prefetchNextPromptTableChunk: - * - Calls the correct function based on position in code (before or after prepareBuffers()) - * - Waits for previous copy to complete if not the first chunk - * - * 2. remapInputTokensForPromptTable: - * - Identifies tokens that need prompt table embeddings (tokens that are greater than vocabSize) - * - Remaps IDs to match chunked prompt table layout - * - * 3. copyPromptTableToGpuInChunk: - * - Handles actual transfer from CPU pinned memory to GPU - * - Uses appropriate buffer manager based on isFirstChunk - */ -void TrtGptModelInflightBatching::prefetchNextPromptTableChunk( - RequestVector const& contextRequests, bool isFirstChunk, SizeType32 bufferId) -{ - auto& promptTuningBuffers = mBuffers[bufferId]->promptTuningBuffers; - - if (!isFirstChunk) - { - // Only switch buffer after prepareBuffer() - promptTuningBuffers->switchChunkPtableBuffer(); - } - - SizeType32 contextId = 0; - for (auto const& llmReq : contextRequests) - { - if (llmReq->isFirstContextChunk() && isFirstChunk) - { - // For first chunk: Blocking prefetch on runtime stream to ensure data is ready - remapInputTokensForPromptTable(llmReq, true, bufferId, contextId); - } - else if (!isFirstChunk) // prefetching for subsequent chunks - { - // For the first prefetch chunk, don't need to wait for previous prefetch to complete - // For subsequent chunks: Need to wait for previous prefetch to complete - if (!llmReq->isFirstContextChunk()) - { - mRuntime->getBufferManager().getStream().wait(mPtableCopyDoneEvent); - } - - // Non-blocking prefetch on copy stream to prepare next chunk in pong buffer - if (llmReq->getContextRemainingLength() > 0) - { - remapInputTokensForPromptTable(llmReq, false, bufferId, contextId); - } - } - - ++contextId; - } -} - -void TrtGptModelInflightBatching::remapInputTokensForPromptTable( - std::shared_ptr<LlmRequest> const& llmReq, bool isFirstChunk, SizeType32 bufferId, SizeType32 contextId) -{ - NVTX3_SCOPED_RANGE_WITH_NAME(range, "remapInputTokensForPromptTable"); - auto& promptTuningBuffers = mBuffers[bufferId]->promptTuningBuffers; - auto const chunkSize = llmReq->getContextChunkSize(); - auto& inputTokensMutable = llmReq->getTokensMutable(0); - auto vocabSize = mModelConfig.getVocabSize(); - - if (isFirstChunk) - { - promptTuningBuffers->initializeChunkPtableBuffers( - mRuntime->getBufferManager(), mModelConfig, chunkSize, llmReq); - } - - size_t processChunkSize; - size_t beginPos; - - if (!isFirstChunk) - { - processChunkSize = std::min(chunkSize, llmReq->getContextRemainingLength() - chunkSize); - } - else - { - processChunkSize = std::min(chunkSize, llmReq->getContextRemainingLength()); - } - - if (!isFirstChunk) - { - // For prefetching next chunk - if (llmReq->getContextRemainingLength() - chunkSize <= 0) - { - promptTuningBuffers->updateBufferStartPosition(promptTuningBuffers->getChunkPtableCurrentIndex(), 0); - return; // No more chunks to prefetch - } - beginPos = llmReq->getContextCurrentPosition() + chunkSize; - } - else - { - // For current chunk - beginPos = llmReq->getContextCurrentPosition(); - } - - TLLM_CHECK_WITH_INFO(beginPos + processChunkSize <= inputTokensMutable.size(), - "Invalid chunk access: beginPos(%zu) + processChunkSize(%zu) > totalSize(%zu)", beginPos, processChunkSize, - inputTokensMutable.size()); - - auto inputTokensChunk = inputTokensMutable.begin() + beginPos; - std::vector<SizeType32> outOfVocabTokens; - SizeType32 ptableTokenId = vocabSize; - for (size_t i = 0; i < processChunkSize; i++) - { - if (inputTokensChunk[i] >= vocabSize) - { - outOfVocabTokens.push_back(inputTokensChunk[i]); - inputTokensChunk[i] = ptableTokenId++; - } - } - - copyPromptTableToGpuInChunk(llmReq, outOfVocabTokens, isFirstChunk, bufferId, contextId); -} - -void TrtGptModelInflightBatching::copyPromptTableToGpuInChunk(std::shared_ptr<LlmRequest> const& llmReq, - std::vector<int32_t> const& outOfVocabTokens, bool isFirstChunk, SizeType32 bufferId, SizeType32 contextId) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE_WITH_NAME(range, "copyPromptTableToGpuInChunk"); - auto& promptTuningBuffers = mBuffers[bufferId]->promptTuningBuffers; - - if (outOfVocabTokens.empty()) - { - return; - } - - auto const& promptTable = llmReq->getPromptEmbeddingTable(); - TLLM_CHECK_WITH_INFO(promptTable.has_value(), "promptTable is empty but there's fake_prompt"); - TLLM_CHECK_WITH_INFO(promptTable.value() != nullptr, "promptTable value is null but there's fake_prompt"); - - auto currentBufferManager = isFirstChunk ? mRuntime->getBufferManager() : mCopyBufferManager; - auto const hiddenSize = mModelConfig.getHiddenSize(); - auto numRows = outOfVocabTokens.size(); - std::size_t sliceSize = static_cast<size_t>(numRows * hiddenSize); - auto currentIndex = promptTuningBuffers->getChunkPtableCurrentIndex(); - - // Calculate the offset based on current position - size_t srcOffset = llmReq->mPtableCurrentPosition * hiddenSize; - size_t dstOffset = promptTuningBuffers->getChunkPtableBufferStartPosition(currentIndex, contextId); - - auto gpuBuffer = promptTuningBuffers->getChunkPtableBuffer(currentIndex); - - // First view as 1D tensor of elements - auto totalElements = promptTable.value()->getSize(); - auto table1D = runtime::ITensor::view( - promptTable.value(), runtime::ITensor::makeShape({static_cast<int64_t>(totalElements)})); - - TLLM_CHECK_WITH_INFO(srcOffset + sliceSize <= totalElements, - "Buffer bounds violation: Trying to access up to %zu elements but buffer only has %zu elements (offset: %zu, " - "slice size: %zu)", - srcOffset + sliceSize, totalElements, srcOffset, sliceSize); - - auto table1DShared = runtime::ITensor::SharedPtr(table1D.release()); - auto pTableView = runtime::ITensor::slice(table1DShared, srcOffset, sliceSize); - - auto gpuBufferSlice = runtime::ITensor::slice(gpuBuffer, dstOffset, numRows); - - currentBufferManager.copy(*pTableView, *gpuBufferSlice); - - promptTuningBuffers->updateBufferStartPosition(currentIndex, outOfVocabTokens.size()); - - llmReq->mPtableCurrentPosition += outOfVocabTokens.size(); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.h b/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.h deleted file mode 100644 index d6550281a758..000000000000 --- a/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.h +++ /dev/null @@ -1,639 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/batch_manager/kvCacheType.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "tensorrt_llm/runtime/worldConfig.h" -#include "trtGptModel.h" - -#include <NvInferRuntime.h> - -namespace tensorrt_llm::runtime -{ -class TllmRuntime; -class GptDecoderBatched; -class AllReduceBuffers; -class NcclCommunicator; -class SpeculativeDecodingMode; - -namespace decoder -{ -class DecoderState; -} // namespace decoder - -namespace decoder_batch -{ -class Input; -class Output; -} // namespace decoder_batch - -} // namespace tensorrt_llm::runtime - -namespace tensorrt_llm::mpi -{ -class MpiWaitThread; -} // namespace tensorrt_llm::mpi - -namespace tensorrt_llm::batch_manager -{ -class BaseCacheTransceiver; -} - -namespace tensorrt_llm::batch_manager -{ - -namespace kv_cache_manager -{ -class KVCacheManager; -struct OffsetTableDimensions; -} // namespace kv_cache_manager - -namespace rnn_state_manager -{ -class RnnStateManager; -} // namespace rnn_state_manager - -class SequenceSlotManager; -class DecoderStepAsyncSend; -class DecoderSlotAsyncSend; -class DecoderInputBuffers; -class DecoderOutputBuffers; -class SlotDecoderBuffers; -class LlmRequest; -class RuntimeBuffers; -class BasePeftCacheManager; -class GuidedDecoder; -class TrtGptModelTest; - -// Algorithms -class CapacityScheduler; -class DisaggTransferAdmissionController; -class MicroBatchScheduler; -class PauseRequests; -class AssignReqSeqSlots; -class AllocateKvCache; -class HandleContextLogits; -class HandleGenerationLogits; -class GenerateRequestOptions; -class LogitsPostProcessor; -class MakeDecodingBatchInputOutput; -class CreateNewDecoderRequests; -class UpdateDecoderBuffers; - -namespace utils -{ -class CudaGraphExecutorCache; -} // namespace utils - -struct RewindInputs -{ - SizeType32 maxBlocksPerSeq; - bool isUseOneMoreBlock; - SizeType32 numKvHeads; -}; - -class TrtGptModelInflightBatching : public TrtGptModel -{ - using BaseKVCacheManager = kv_cache_manager::BaseKVCacheManager; - using OffsetTableDimensions = kv_cache_manager::OffsetTableDimensions; - using KVCacheManager = kv_cache_manager::KVCacheManager; - using KvCacheType = kv_cache_manager::CacheType; - using KvCacheConfig = executor::KvCacheConfig; - using RnnStateManager = rnn_state_manager::RnnStateManager; - using LlmRequestPtr = std::shared_ptr<batch_manager::LlmRequest>; - -public: - class IterationStatsIFB - { - public: - explicit IterationStatsIFB(SizeType32 microBatchId) - : microBatchId{microBatchId} - { - } - - SizeType32 microBatchId; - SizeType32 numCtxRequests{}; - SizeType32 numGenRequests{}; - SizeType32 numCtxTokens{}; - float avgNumDecodedTokensPerIter{}; - ReqIdsSet scheduledRequests; - ReqIdsSet pausedRequests; - }; - - using SizeType32 = tensorrt_llm::runtime::SizeType32; - using TokenIdType = tensorrt_llm::runtime::TokenIdType; - using BufferManager = tensorrt_llm::runtime::BufferManager; - using PeftTable = PeftCacheManager::PeftTable; - using TensorMap = runtime::StringPtrMap<runtime::ITensor>; - using TensorPtr = runtime::ITensor::SharedPtr; - - TrtGptModelInflightBatching(std::shared_ptr<nvinfer1::ILogger> logger, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig, runtime::RawEngine const& rawEngine, bool ctxGenFusion, - executor::ExecutorConfig const& executorConfig, bool isLeaderInOrchMode); - - ~TrtGptModelInflightBatching() override; - - /// @brief Calculate the cache size per token for the disaggregated serving. - /// @param modelConfig Model configuration. - /// @param worldConfig World configuration. - /// @param maxAttentionWindowVec Maximum attention window vector. (may have fewer elements than numLayers, in which - /// case it cycles) - /// @param isCrossAttention Whether the attention is cross attention. - /// @param kvFactor KV factor. - /// @return Cache size per token for the disaggregated layers. Note that window size is not included in the result - /// here. - [[nodiscard]] static std::map<SizeType32, SizeType32> calculateCacheSizePerTokenForDisagg( - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - std::vector<SizeType32> const& maxAttentionWindowVec, bool isCrossAttention, SizeType32 kvFactor); - - void terminateRequest(LlmRequestPtr const& llmRequest, bool pause = false) override; - - /// @brief Terminate request in the next forwardSync call that includes the request. - /// @details This function does not terminate requests immediately. It will add the requests to the - /// mReqIdsToTerminate set. The requests will be terminated in the next forwardSync call that - /// includes the request in the batch. - void terminateRequestSync(LlmRequestPtr const& llmRequest, executor::FinishReason finishReason) override; - - /// @brief Function that waits for the decoding of requests in flight. - /// When the requests have finished or using speculative decoding, the state of requests - /// will become LlmRequestState::kGENERATION_COMPLETE. Else, it will be set to - /// LlmRequestState::kGENERATION_IN_PROGRESS. - void forwardSync() override; - - /// @brief Function that tries to advance the active requests. - /// Depending on resources available, it's possible that not all requests will get advanced. - /// Requests that may be in state LlmRequestState::kCONTEXT_INIT become - /// LlmRequestState::kGENERATION_IN_PROGRESS or LlmRequestState::kGENERATION_TO_COMPLETE. - /// @param activeRequests The list of request to try to advance. - void forwardAsync(RequestList const& activeRequests) override; - - /// @brief Override the runtime batch size for the model - void setRuntimeBatchSize(SizeType32 runtimeMaxBatchSize) override; - - /// @brief Get the runtime batch size for the model - [[nodiscard]] SizeType32 getRuntimeBatchSize() const override; - - /// @brief Override the runtime max num tokens for the model - void setRuntimeMaxNumTokens(SizeType32 runtimeMaxNumTokens) override; - - void updatePeftCache(std::shared_ptr<LlmRequest> const& llmRequest) override; - - [[nodiscard]] IterationStatsIFB getLastIterationStats() const - { - return mLastIterationStatsIFB; - } - - [[nodiscard]] TrtGptModelType getModelType() const override - { - return mCtxGenFusion ? TrtGptModelType::InflightFusedBatching : TrtGptModelType::InflightBatching; - }; - - [[nodiscard]] runtime::BufferManager const& getBufferManager() const override; - [[nodiscard]] runtime::BufferManager::CudaStreamPtr getRuntimeStreamPtr() const override; - - void getCurrentIterationStats(executor::IterationStats& stats) const override; - void getCurrentRequestStats(executor::RequestStatsPerIteration& stats) const override; - [[nodiscard]] executor::DebugTensorsPerIteration getCurrentDebugTensors() const override; - - [[nodiscard]] executor::IterationType getIterCounter() const noexcept override - { - return mIterCounter; - } - - [[nodiscard]] static bool executorConfigIsValid( - runtime::ModelConfig const& modelConfig, executor::ExecutorConfig const& executorConfig); - [[nodiscard]] static executor::ExecutorConfig fixExecutorConfig( - runtime::ModelConfig const& modelConfig, executor::ExecutorConfig const& executorConfig); - - void prepareDisaggGenInitRequests(RequestList const& activeRequests, RequestVector& newGenReques); - void checkDisaggGenTransferStatus(RequestList const& activeRequests); - void prepareDistGenBufferAndDecoder(RequestVector const& generationRequests); - - void resetIterationStats() override; - - runtime::SpeculativeDecodingMode getSpeculativeDecodingMode() const noexcept - { - return mModelConfig.getSpeculativeDecodingMode(); - } - - [[nodiscard]] SizeType32 numCachedCudaGraphs() const; - -private: - friend class TrtGptModelTest; - - [[nodiscard]] SizeType32 getContextBufferId() const - { - return mMicroBatchId; - } - - [[nodiscard]] SizeType32 getGenerationBufferId() const - { - return mNumMicroBatches + mMicroBatchId; - } - - [[nodiscard]] SizeType32 getFusedBufferId() const - { - return mMicroBatchId; - } - - [[nodiscard]] SizeType32 getNextMicroBatchId(SizeType32 bufferId) const - { - return (bufferId + 1) % mNumMicroBatches; - } - - [[nodiscard]] SizeType32 getPrevMicroBatchId(SizeType32 bufferId) const - { - return (bufferId + mNumMicroBatches - 1) % mNumMicroBatches; - } - - //! @brief Store full kv cache blocks contributed by req. - //! These blocks become reusable from next step. - void storeContextBlocks(std::shared_ptr<LlmRequest> const& req); - - //! @brief Store newest kv cache block for reuse. - //! The block become reusable from next step. - void storeNewBlock(std::shared_ptr<LlmRequest> const& req); - - //! @brief Set LayerProfiler to collect performance per layer. - void setLayerProfiler() override; - - //! @brief Print profile information per layer. - std::string getLayerProfileInfo() const override; - - std::tuple<SizeType32, TensorMap const&, TensorMap&> prepareBuffers( - RequestVector const& contextRequests, RequestVector const& generationRequests, SizeType32 bufferId); - - //! @brief Capture graph of current batch state during engine execution. - //! This is based on the assumptions that - //! a) We can hide CPU graph capture behind the GPU engine execution. - //! b) Batch size in the next iterations won't change and we can reuse the graph multiple times. - void prepareGraph(SizeType32 bufferId, SizeType32 optProfileId); - - void executeContext(SizeType32 runtimeContextId, SizeType32 bufferId); - void executeBatch(ScheduledRequests const& scheduledRequests); - void executeStep( - RequestVector const& contextRequests, RequestVector const& generationRequests, SizeType32 bufferId); - - void debugIOTensors(RequestVector const& contextRequests, RequestVector const& generationRequests, - TensorMap const& inputMap, TensorMap const& outputMap); - - void createRuntimeContexts(); - void createDecoder(std::optional<executor::DecodingMode> const& decodingModeOpt); - void createBuffers(executor::DecodingConfig const& decodingConfig, - std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs); - std::unique_ptr<KVCacheManager> createKvCacheManager(KvCacheConfig const& kvCacheConfig, KvCacheType kvCacheType, - uint64_t freePrimaryMemBytes, uint64_t freeSecondaryMemBytes, size_t extraCostMemory, - bool const failFastOnAttentionWindowTooLarge = false); - void createRnnStateManager(); - void createCustomAllReduceWorkspace(); - void createRuntimePerfKnobsTensor(executor::ExtendedRuntimePerfKnobConfig const& extendedRuntimePerfKnobConfig); - - /// @brief Verify draft token length and beam width of all active requests. - /// May change operating beam width if all requests agree on same beam width. - void verifyRequests(RequestList const& activeRequests); - - /// @brief Change the operating beam width. - /// Only possible if no requests are currently in-flight. - /// @param beamWidth New operating beam width. Must be smaller than initial maxBeamWidth. - void changeBeamWidth(SizeType32 beamWidth); - - SizeType32 getOperatingBeamWidth() const override - { - return mOperatingBeamWidth; - } - - /// @details Should be called after setting up the current batch in executeBatch to get the correct number of - /// context tokens. - IterationStatsIFB fillIterationStats( - ScheduledRequests const& scheduledRequests, RequestVector const& requestsToPause); - - /// @brief Function that sets up the TensorRT execution context that is going to be used for execution. If multiple - /// TensorRT optimization profiles are built in the engine, it selects the corresponding context that is going to be - /// used, and prepares the input and output tensors so that both buffers and the context is ready for the execution. - /// @return The TensorRT execution context index that has been setup. - void setupContext( - RequestVector const& contextRequests, RequestVector const& generationRequests, SizeType32 bufferId); - - void setupDecoderStep( - RequestVector const& contextRequests, RuntimeBuffers const& buffers, DecoderInputBuffers& inputBuffers); - runtime::CudaEvent decoderStepAsync(ScheduledRequests const& scheduledRequests); - std::vector<std::unique_ptr<DecoderStepAsyncSend>> decoderSync( - ScheduledRequests const& scheduledRequests, std::optional<runtime::CudaEvent> const& decoderFinishEvent); - - std::vector<std::unique_ptr<DecoderStepAsyncSend>> communicateDecoderBuffers(bool returnLogProbs); - void updateRequests(ScheduledRequests const& scheduledRequests); - - /// @brief It gathers the logits if they need to be returned, calls getDecoderSlotHostOutputs, - /// and overwrites the llmRequest tokens buffer. - /// Called either on request finishing, or at every step when doing beam search and streaming. - void postProcessRequest(LlmRequest& llmReq, std::vector<SizeType32> const& numDroppedTokens); - /// @brief Reorders generation logits to match finalized beam paths after gatherTree. - /// During beam search, logits are stored by beam slot. After finalization, output_ids are - /// reordered by parentIds, but logits are not. This method traces parentIds on the host - /// to build the slot mapping and reindexes the logits accordingly. - void reorderGenerationLogitsForBeamSearch(LlmRequest& llmReq, SizeType32 seqSlot, SizeType32 reqBeamWidth, - SizeType32 maxSeqLength, TokenIdType const* outputIdsHostData, SizeType32 const* sequenceLengthsHostData); - /// @brief Calls gatherTree (via finalize) and transmits the received data across ranks if PP>1 - void getDecoderSlotHostOutputs( - SizeType32 seqSlot, bool returnLogProbs, runtime::SamplingConfig const& samplingConfig, bool streaming); - void rewindKVCacheBlocks(SizeType32 numSequences); - void setupSpeculativeDecodingModule(executor::DecodingConfig const& decodingConfig); - - /// @brief Copies the content of the cache indirection outputs to the cache indirection inputs. - /// @param[in] scheduledRequests The requests to copy the cache indirections for. - /// @param[in] genBufferId The id of the generation buffers for those requests. - void copyCacheIndirectionFromOutputsToInputs(ScheduledRequests const& scheduledRequests, SizeType32 genBufferId); - - [[nodiscard]] bool getGatherGenerationLogits() const override - { - return getModelConfig().computeGenerationLogits() || mGatherGenerationLogits; - } - - [[nodiscard]] runtime::ModelConfig const& getModelConfig() const override - { - return mModelConfig; - } - - [[nodiscard]] runtime::WorldConfig const& getWorldConfig() const override - { - return mWorldConfig; - } - - [[nodiscard]] SizeType32 getNumMicroBatches() const override - { - return mNumMicroBatches; - } - - [[nodiscard]] nvinfer1::DataType getLogitDataType() const override; - - [[nodiscard]] nvinfer1::DataType getTensorDataType(std::string const& name) const override; - - [[nodiscard]] nvinfer1::Dims getTensorShape(std::string const& name) const override; - - void reshapeKvTensors(OffsetTableDimensions const& dims); - - [[nodiscard]] bool hasSpeculativeDecodingFastLogits() const noexcept override - { - return mSpeculativeDecodingFastLogits; - } - - [[nodiscard]] bool hasGuidedDecoder() const noexcept override - { - return static_cast<bool>(mGuidedDecoder); - } - - using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; - /// @brief Based on the KV-cache manager's capacity and configuration, we adjust the maximum supported attention - /// window. - /// - /// @param blocksPerWindow map of window size to number of blocks. - /// @param failFastOnAttentionWindowTooLarge if true, the function will report a runtime error if the attention - /// window is too large to fit even a single sequence in the KV cache. - /// @return pair of new blocks per window and new maxAttentionWindowVec - [[nodiscard]] std::pair<BlocksPerWindow, std::vector<SizeType32>> clampWindowSizesToFitAtLeastOneSequence( - BlocksPerWindow const& blocksPerWindow, bool const failFastOnAttentionWindowTooLarge = false); - - /// @brief Change the speculative decoding mode. - void changeSpecDecMode(ScheduledRequests const& scheduledRequests); - - void prefetchNextPromptTableChunk(RequestVector const& contextRequests, bool isFirstChunk, SizeType32 bufferId); - - void remapInputTokensForPromptTable( - std::shared_ptr<LlmRequest> const& llmReq, bool isCurrentChunk, SizeType32 bufferId, SizeType32 contextId); - - void copyPromptTableToGpuInChunk(std::shared_ptr<LlmRequest> const& llmReq, - std::vector<int32_t> const& outOfVocabTokens, bool useCurrentBuffer, SizeType32 bufferId, SizeType32 contextId); - -protected: - std::shared_ptr<BaseKVCacheManager> getKVCacheManager() override - { - return mKvCacheManager; - } - - [[nodiscard]] std::shared_ptr<BaseKVCacheManager const> getKVCacheManager() const override - { - return mKvCacheManager; - } - - std::shared_ptr<BaseKVCacheManager> getCrossKVCacheManager() - { - return mCrossKvCacheManager; - } - - [[nodiscard]] std::shared_ptr<BaseKVCacheManager const> getCrossKVCacheManager() const - { - return mCrossKvCacheManager; - } - - [[nodiscard]] std::shared_ptr<BasePeftCacheManager> getPeftCacheManager() override - { - return mPeftCacheManager; - } - - [[nodiscard]] std::shared_ptr<BasePeftCacheManager const> getPeftCacheManager() const override - { - return mPeftCacheManager; - } - - void setLogitsPostProcessorBatched(std::optional<LogitsPostProcessorBatched> logitsPostProcessorBatched) override - { - mLogitsPostProcessorBatched = logitsPostProcessorBatched; - } - - void setReplicateLogitsPostProcessor(bool replicateLogitsPostProcessor) override - { - mReplicateLogitsPostProcessor = replicateLogitsPostProcessor; - } - - [[nodiscard]] bool getReplicateLogitsPostProcessor() const override - { - return mReplicateLogitsPostProcessor; - } - - SizeType32 getMaxCapacityBatchSize(SizeType32 inputLength, SizeType32 outputLength) const override; - -private: - /******************** Configs ********************/ - // Parameters of the model (TRT engine) - runtime::ModelConfig mModelConfig; - // Parameters of the execution environment - runtime::WorldConfig mWorldConfig; - // Device ID of this instance - int mDevice{-1}; - // Config for (speculative) decoding - executor::DecodingConfig mDecodingConfig; - // Performance knobs for the engine. - executor::ExtendedRuntimePerfKnobConfig mExtendedRuntimePerfKnobConfig; - TensorPtr mExtendedRuntimePerfKnobsHost; - // Config for debugging output - std::optional<executor::DebugConfig> mDebugConfig; - // List of additional outputs for each request - std::optional<std::vector<executor::AdditionalModelOutput>> mAdditionalModelOutputs; - - /******************** Components ********************/ - std::shared_ptr<nvinfer1::ILogger> mLogger; - // Runner for the TRT engine. The engine produces logits. - std::unique_ptr<runtime::TllmRuntime> mRuntime; - // Decoder that generates new tokens from the logits. - std::unique_ptr<runtime::GptDecoderBatched> mDecoder; - // Decoder state for all requests - std::unique_ptr<runtime::decoder::DecoderState> mDecoderState; - // Synchronization handles for decoder - std::vector<std::optional<runtime::CudaEvent>> mDecoderFinishedEvents; - - // Manager that maps requests to slots - std::shared_ptr<SequenceSlotManager> mSeqSlotManager; - // KV cache manager for attention layers (optional) - std::shared_ptr<BaseKVCacheManager> mKvCacheManager; - // KV cache manager for cross attention in enc-dec models (optional) - std::shared_ptr<BaseKVCacheManager> mCrossKvCacheManager = nullptr; - // RNN state manager for recurrent layers (optional) - std::unique_ptr<RnnStateManager> mRnnStateManager; - // PEFT cache manager for LoRA tasks (optional) - std::shared_ptr<BasePeftCacheManager> mPeftCacheManager; - // BufferManager using a separate stream for async copy operations. - runtime::BufferManager mCopyBufferManager; - // Event for async data transfers - runtime::CudaEvent mPtableCopyDoneEvent; - - /******************** Logits Post-Processor ********************/ - std::optional<LogitsPostProcessorBatched> mLogitsPostProcessorBatched; - bool mReplicateLogitsPostProcessor{true}; - // Set if any request invoked a logits processor in current step - bool mLogitsPostProcessorIsApplied{false}; - - constexpr bool broadcastPostDecoder() - { - return mWorldConfig.isTensorParallel() && !mReplicateLogitsPostProcessor && mLogitsPostProcessorIsApplied; - } - - std::unique_ptr<tensorrt_llm::batch_manager::GuidedDecoder> mGuidedDecoder; - - /******************** Pipeline parallelism ********************/ - std::unique_ptr<tensorrt_llm::mpi::MpiComm> mMpiCommPipelinePara; - std::vector<std::unique_ptr<DecoderStepAsyncSend>> mDecStepAsyncSndHdls; - std::vector<std::unique_ptr<DecoderSlotAsyncSend>> mDecSlotAsyncSndHdls; - std::unique_ptr<tensorrt_llm::mpi::MpiWaitThread> mAsyncSendWaitThread; - - /******************** Tensor parallelism ********************/ - std::unique_ptr<tensorrt_llm::mpi::MpiComm> mMpiCommTensorPara; - std::unique_ptr<runtime::AllReduceBuffers> mAllReduceBuffers; - - /******************** Runtime parameters ********************/ - // Flag to select fused or unfused context+generation execution - bool mCtxGenFusion; - // ID of current micro batch, changes after each iteration - SizeType32 mMicroBatchId{0}; - // Number of micro batches. Multiple batches are used for overlapping setup and execution, - // and in pipeline parallelism. - SizeType32 mNumMicroBatches; - // Number of buffers to be added to mBuffers. - SizeType32 mNumBuffers; - // Current operating beam width. Can be changed with changeBeamWidth function. - SizeType32 mOperatingBeamWidth; - // Runtime batch size optimized during execution for microBatchScheduler: - /// The max batch size recommended by the dynamic tuner - SizeType32 mMaxBatchSizeTunerRecommended; - /// The min of mMaxBatchSize and mMaxBatchSizeTunerRecommended - SizeType32 mMaxBatchSizeRuntime; - // Runtime max num tokens optimized during execution for microBatchScheduler: - /// Build time max num tokens - std::optional<SizeType32> mMaxNumTokensStatic; - /// The max num tokens recommended by the dynamic tuner - SizeType32 mMaxNumTokensTunerRecommended; - /// The min of mMaxNumTokens and mMaxNumTokensTunerRecommended - std::optional<SizeType32> mMaxNumTokensRuntime; - // Controls if generation logits should be gathered, so that returnGenerationLogits can be requested. - bool mGatherGenerationLogits{false}; - // offloading and prefetching the prompt tuning table (only effective in chunked prefill mode) - bool mPromptTableOffloading; - - /******************** Buffers ********************/ - // Buffers for each micro batch. Unfused path (mCtxGenFusion==false) uses two times the buffers. - std::vector<std::unique_ptr<RuntimeBuffers>> mBuffers; - // Decoder input buffers for each micro batch. - std::vector<DecoderInputBuffers> mDecoderInputBuffers; - // Decoder output buffers for each micro batch. - std::vector<DecoderOutputBuffers> mDecoderOutputBuffers; - // Buffers for each slot in the decoder - std::vector<std::unique_ptr<SlotDecoderBuffers>> mSlotDecoderBuffers; - // PEFT table for each micro batch - std::vector<PeftTable> mPeftTables; - - /******************** Book keeping ********************/ - // List of requests in each micro batch - std::vector<ScheduledRequests> mMicroBatchScheduledRequests; - // Set of in-flight requests of *all* micro batches - ReqIdsSet mInflightReqIds; - // Requests that should be terminated (requested from outside the model) - std::unordered_map<RequestIdType, executor::FinishReason> mReqIdsToTerminate; - // Requests that the scheduler selected to be paused - ReqIdsSet mReqIdsToPause; - // Stats collected in last iteration - IterationStatsIFB mLastIterationStatsIFB{-1}; - // Iteration counter used to distinguish debug output - executor::IterationType mIterCounter{0}; - // Debug tensors of last itreation - TensorMap mLastIterationDebugTensors; - // Cuda graph instances for each microbatch. - std::vector<utils::CudaGraphExecutorCache> mCudaGraphExecutorCaches; - - /******************** Cache transceiver ********************/ - std::unique_ptr<BaseCacheTransceiver> mCacheTransceiver; - std::unique_ptr<DisaggTransferAdmissionController> mDisaggTransferAdmissionController; - - /******************** Spec dec ***********************/ - std::unique_ptr<std::thread> mDraftModelSendLogitsThread; - bool mSpeculativeDecodingFastLogits; - std::atomic<bool> mDraftModelThreadShouldExit{false}; - bool mIsLeaderInOrchMode{false}; - // List of completed draft requests which logits will need to be sent to the target model. - // Guarded by mDraftRequestsMtx (shared with the background logits sender thread). - RequestVector mDraftRequestsWaitingToSendLogits; - // Draft requests whose logits have been sent — pending termination by main thread. - // Guarded by mDraftRequestsMtx. - RequestVector mDraftRequestsDoneSendingLogits; - std::mutex mDraftRequestsMtx; - SizeType32 mSeamlessLADMaxDraftLen{0}; - bool mUseSeamlessLookahead{false}; - RewindInputs mRewindInputs; - - /******************** Algorithms ********************/ - // Algorithms are reentrant, they are assigned a state at - // construction time and it is not modified through execution, hence they are const. - // Schedulers that select which requests to run in each iteration - std::unique_ptr<tensorrt_llm::batch_manager::CapacityScheduler const> mCapacityScheduler; - std::unique_ptr<tensorrt_llm::batch_manager::MicroBatchScheduler const> mMicroBatchScheduler; - std::unique_ptr<tensorrt_llm::batch_manager::PauseRequests const> mPauseRequests; - std::unique_ptr<tensorrt_llm::batch_manager::AssignReqSeqSlots const> mAssignReqSeqSlots; - std::unique_ptr<tensorrt_llm::batch_manager::AllocateKvCache const> mAllocateKvCache; - std::unique_ptr<tensorrt_llm::batch_manager::HandleContextLogits const> mHandleContextLogits; - std::unique_ptr<tensorrt_llm::batch_manager::HandleGenerationLogits const> mHandleGenerationLogits; - std::unique_ptr<tensorrt_llm::batch_manager::LogitsPostProcessor const> mLogitsPostProcessor; - std::unique_ptr<tensorrt_llm::batch_manager::MakeDecodingBatchInputOutput const> mMakeDecodingBatchInputOutput; - std::unique_ptr<tensorrt_llm::batch_manager::CreateNewDecoderRequests const> mCreateNewDecoderRequests; - std::unique_ptr<tensorrt_llm::batch_manager::UpdateDecoderBuffers const> mUpdateDecoderBuffers; -}; - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/updateDecoderBuffers.cpp b/cpp/tensorrt_llm/batch_manager/updateDecoderBuffers.cpp deleted file mode 100644 index ead120135f3a..000000000000 --- a/cpp/tensorrt_llm/batch_manager/updateDecoderBuffers.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/updateDecoderBuffers.h" -#include "tensorrt_llm/batch_manager/decoderBuffers.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/runtime/decoderState.h" -#include "tensorrt_llm/runtime/iTensor.h" - -namespace tensorrt_llm::batch_manager -{ - -using BufferManager = tensorrt_llm::runtime::BufferManager; -using TensorPtr = runtime::ITensor::SharedPtr; -using ITensor = runtime::ITensor; -using SizeType32 = tensorrt_llm::runtime::SizeType32; - -runtime::CudaEvent UpdateDecoderBuffers::operator()(runtime::ModelConfig const& modelConfig, - DecoderOutputBuffers& decoderOutputBuffers, runtime::BufferManager const& copyBufferManager, - runtime::decoder::DecoderState const& decoderState, bool returnLogProbs, - runtime::CudaEvent const& decoderFinishEvent) const -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(updateDecoderBuffers); - - // Chain copy after decoder event, using a different stream - copyBufferManager.getStream().wait(decoderFinishEvent); - - copyBufferManager.copy(*decoderState.getAllNewTokens(), *decoderOutputBuffers.newOutputTokensHost); - copyBufferManager.copy(*decoderState.getSequenceLengths(), *decoderOutputBuffers.sequenceLengthsHost); - - auto const finishedSumDevice = decoderState.getFinishedSum(); - copyBufferManager.copy(*finishedSumDevice, *decoderOutputBuffers.finishedSumHost); - auto const finishReasonsDevice = decoderState.getFinishReasons(); - copyBufferManager.copy(*finishReasonsDevice, *decoderOutputBuffers.finishReasonsHost); - - if (returnLogProbs) - { - copyBufferManager.copy(*decoderState.getCumLogProbs(), *decoderOutputBuffers.cumLogProbsHost); - copyBufferManager.copy(*decoderState.getLogProbs(), *decoderOutputBuffers.logProbsHost); - } - - if (modelConfig.getSpeculativeDecodingMode().predictsDraftTokens()) - { - // TODO: keep data on device for next iteration - copyBufferManager.copy(*decoderState.getNextDraftTokens(), *decoderOutputBuffers.nextDraftTokensHost); - - if (modelConfig.getSpeculativeDecodingMode().variableDraftLength()) - { - copyBufferManager.copy( - *decoderState.getNextDraftTokensLengths(), *decoderOutputBuffers.nextDraftTokensLengthsHost); - copyBufferManager.copy( - *decoderState.getPrevDraftTokensLengths(), *decoderOutputBuffers.prevDraftTokensLengthsHost); - } - } - - runtime::CudaEvent copyEvent{}; - copyBufferManager.getStream().record(copyEvent); - // Store the event for later sync. Sync stream before calling next decoder. Sync host before updating requests. - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return copyEvent; -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/utils/debugUtils.h b/cpp/tensorrt_llm/batch_manager/utils/debugUtils.h index e4732a75f649..c041c7a71de8 100644 --- a/cpp/tensorrt_llm/batch_manager/utils/debugUtils.h +++ b/cpp/tensorrt_llm/batch_manager/utils/debugUtils.h @@ -23,11 +23,6 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/worldConfig.h" -namespace tensorrt_llm::runtime -{ -class TllmRuntime; -} // namespace tensorrt_llm::runtime - namespace tensorrt_llm::batch_manager::utils { diff --git a/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.cpp b/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.cpp index 416235f347b8..a3e54a6b0f9b 100644 --- a/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.cpp +++ b/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.cpp @@ -16,7 +16,6 @@ */ #include "inflightBatchingUtils.h" -#include "tensorrt_llm/runtime/runtimeKernels.h" namespace tensorrt_llm::batch_manager::utils { @@ -88,170 +87,6 @@ void moveFinishedContextRequestsToGeneration(ScheduledRequests& scheduledRequest TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); } -void copyGenerationLogits(RuntimeBuffers::GenerationLogitsCache& generationLogitsCache, - runtime::BufferManager const& bufferManager, LlmRequest& llmReq, bool beforeDecoder, - std::vector<SizeType32> const& numDroppedTokens) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - TLLM_CHECK_WITH_INFO( - !beforeDecoder || numDroppedTokens.empty(), "numDroppedTokens are only possible after decoder."); - - auto const reqBeamWidth = llmReq.getBeamWidthByIter(); - TLLM_CHECK_WITH_INFO(numDroppedTokens.empty() || numDroppedTokens.size() == static_cast<size_t>(reqBeamWidth), - "Dropped tokens have to be defined for all beams."); - - auto const fragmentSize = llmReq.getGenerationLogitsFragmentsSize(); - - // Merge logits fragments on device. getFragmentPointerSlot() returns the matching host and - // device rows for the current workIdx and advances the index atomically, so concurrent flushes - // for different requests in the same batch never clobber each other's pointer arrays. - auto const& transposeBufferPtr = generationLogitsCache.transposedLogits; - auto [cachePointerHost, cachePointerDevice] = generationLogitsCache.getFragmentPointerSlot(); - tensorrt_llm::runtime::kernels::mergeLogitsFragments(bufferManager, *transposeBufferPtr, - llmReq.getGenerationLogitsFragments(), *cachePointerDevice, *cachePointerHost, 0, 1, reqBeamWidth, - bufferManager.getStream(), 0); - llmReq.clearGenerationLogitsFragments(); - - // Copy logits to host - for (SizeType32 beam = 0; beam < reqBeamWidth; beam++) - { - auto const droppedSize = !numDroppedTokens.empty() ? numDroppedTokens.at(beam) : 0; - // Ignore logits of dropped tokens - auto const beamFragmentSize = fragmentSize - droppedSize; - // If this function is called before the decoder, the request does not contain the generated token of the - // current iteration, so we add 1 to the number of tokens. - auto const numGenerationToken - = static_cast<SizeType32>(beforeDecoder) + llmReq.getNumTokens(beam) - llmReq.mPromptLen; - auto const hostOffset = numGenerationToken - beamFragmentSize; - - // [beamWidth, GENERATION_LOGITS_BUFFER_LENGTH, vocabSizePadded] -> [beamFragmentSize, vocabSizePadded] - auto beamDeviceTensorPtr = ITensor::slice(transposeBufferPtr, {beam, 0}, beamFragmentSize); - // [beamWidth, mMaxNewTokens, vocabSizePadded] -> [beamFragmentSize, vocabSizePadded] - auto beamHostTensorPtr = ITensor::slice(llmReq.getGenerationLogitsHost(), {beam, hostOffset}, beamFragmentSize); - bufferManager.copy(*beamDeviceTensorPtr, *beamHostTensorPtr); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -namespace -{ - -std::pair<TensorPtr const&, bool> findOutputTensor(std::string const& outputTensorName, - std::vector<executor::AdditionalModelOutput> const& additionalModelOutputs, - RuntimeBuffers::TensorMap const& outputMap, bool isContext) -{ - auto const aoIter = std::find_if(additionalModelOutputs.cbegin(), additionalModelOutputs.cend(), - [&outputTensorName](auto const& ao) { return ao.name == outputTensorName; }); - TLLM_CHECK_WITH_INFO(aoIter != additionalModelOutputs.cend(), "Additional %s output tensor not found: %s", - isContext ? "context" : "generation", outputTensorName.c_str()); - - auto const gatherContext = aoIter->gatherContext; - if (isContext) - { - TLLM_CHECK_WITH_INFO( - gatherContext, "Additional context output tensor not gathered: %s", outputTensorName.c_str()); - } - - auto const tensorIt = outputMap.find(outputTensorName); - TLLM_CHECK_WITH_INFO(tensorIt != outputMap.end(), "Additional %s output tensor not found: %s", - isContext ? "context" : "generation", outputTensorName.c_str()); - - return {tensorIt->second, gatherContext}; -} - -} // namespace - -void copyAdditionalOutputs(std::vector<executor::AdditionalModelOutput> const& additionalModelOutputs, - RequestVector const& contextRequests, RequestVector const& generationRequests, - RuntimeBuffers::TensorMap const& outputMap, runtime::BufferManager const& manager) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // One index shared across all output tensors that have gatherContext - SizeType32 srcTensorIndexWithContext{0}; - // One index shared across all output tensors that do not have gatherContext - SizeType32 srcTensorIndexWithoutContext{0}; - - for (auto const& llmReq : contextRequests) - { - auto numContextTokens = llmReq->getContextChunkSize(); - for (auto const& outputTensor : llmReq->getAdditionalContextOutputs()) - { - auto const& [tensor, gatherContext] - = findOutputTensor(outputTensor.first, additionalModelOutputs, outputMap, true); - - auto const srcTensorIndex = srcTensorIndexWithContext; - auto srcView = ITensor::slice(tensor, srcTensorIndex, numContextTokens); - auto dstView = ITensor::slice(outputTensor.second, llmReq->getContextCurrentPosition(), numContextTokens); - manager.copy(*srcView, *dstView); - } - srcTensorIndexWithContext += numContextTokens; - srcTensorIndexWithoutContext += 1; - - // Copy output of last token to generation outputs - if (llmReq->isLastContextChunk()) - { - for (auto const& outputTensor : llmReq->getAdditionalGenerationOutputs()) - { - auto const& [tensor, gatherContext] - = findOutputTensor(outputTensor.first, additionalModelOutputs, outputMap, false); - - auto const srcTensorIndex = gatherContext ? srcTensorIndexWithContext : srcTensorIndexWithoutContext; - auto srcView = ITensor::slice(tensor, srcTensorIndex - 1, 1); - for (SizeType32 beam = 0; beam < llmReq->getBeamWidthByIter(); beam++) - { - auto dstView = ITensor::slice(outputTensor.second, {beam, 0}, 1); - manager.copy(*srcView, *dstView); - } - } - } - } - - for (auto const& llmReq : generationRequests) - { - auto const reqBeamWidth = llmReq->getBeamWidthByIter(); - for (auto const& outputTensor : llmReq->getAdditionalGenerationOutputs()) - { - auto const& [tensor, gatherContext] - = findOutputTensor(outputTensor.first, additionalModelOutputs, outputMap, false); - - auto const srcTensorIndex = gatherContext ? srcTensorIndexWithContext : srcTensorIndexWithoutContext; - for (SizeType32 beam = 0; beam < reqBeamWidth; beam++) - { - auto const generatedLength = llmReq->getNumTokens(beam) - llmReq->getPromptLen(); - TLLM_CHECK(generatedLength >= 1); - auto srcView = ITensor::slice(tensor, srcTensorIndex + beam, 1); - auto dstView = ITensor::slice(outputTensor.second, {beam, generatedLength}, 1); - manager.copy(*srcView, *dstView); - } - } - srcTensorIndexWithContext += reqBeamWidth; - srcTensorIndexWithoutContext += reqBeamWidth; - } - - // Check final indices - for (auto const& outputTensor : additionalModelOutputs) - { - auto const& outputTensorName = outputTensor.name; - auto const gatherContext = outputTensor.gatherContext; - - auto const tensorIt = outputMap.find(outputTensorName); - TLLM_CHECK_WITH_INFO( - tensorIt != outputMap.end(), "Additional output tensor not found: %s", outputTensorName.c_str()); - - auto const& outputShape = tensorIt->second->getShape(); - auto const outputSize = outputShape.d[0]; - auto const finalIndex = gatherContext ? srcTensorIndexWithContext : srcTensorIndexWithoutContext; - - TLLM_CHECK_WITH_INFO(finalIndex == outputSize, "Additional %s output tensor final index mismatch %d != %ld: %s", - gatherContext ? "context" : "generation", finalIndex, outputSize, outputTensorName.c_str()); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - void terminateRequest(SequenceSlotManager& seqSlotManager, LlmRequest& llmReq, SizeType32 maxInputLen, OptionalRef<kv_cache_manager::BaseKVCacheManager> kvCacheManager, OptionalRef<kv_cache_manager::BaseKVCacheManager> crossKvCacheManager, @@ -300,109 +135,4 @@ std::vector<SizeType32> getRequestBeamWidths( return beamWidths; } -void CudaGraphExecutor::create(cudaGraph_t const& graph) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - assert(mInstance == nullptr); - TLLM_CUDA_CHECK(cudaGraphInstantiate(&mInstance, graph, nullptr, nullptr, 0)); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void CudaGraphExecutor::uploadToStream(runtime::CudaStream const& stream) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - assert(hasInstance()); - TLLM_CUDA_CHECK(cudaGraphUpload(mInstance, stream.get())); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void CudaGraphExecutor::launch(runtime::CudaStream const& stream) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - TLLM_CUDA_CHECK(cudaGraphLaunch(mInstance, stream.get())); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -bool CudaGraphExecutor::update(cudaGraph_t const& graph) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - return cudaGraphExecUpdate(mInstance, graph, nullptr) != cudaSuccess; -} - -void CudaGraphExecutor::clear() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - if (mInstance != nullptr) - { - TLLM_CUDA_CHECK(cudaGraphExecDestroy(mInstance)); - mInstance = nullptr; - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void CudaGraphExecutor::prepareNextGraph(std::unique_ptr<runtime::TllmRuntime>& runtime, SizeType32 nextContextId) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - auto& stream = runtime->getStream(); - - cudaGraph_t nextGraph; - TLLM_CUDA_CHECK(cudaStreamBeginCapture(stream.get(), cudaStreamCaptureModeThreadLocal)); - runtime->executeContext(nextContextId); - TLLM_CUDA_CHECK(cudaStreamEndCapture(stream.get(), &nextGraph)); - - if (hasInstance()) - { - if (update(nextGraph)) - { - clear(); - create(nextGraph); - } - } - else - { - create(nextGraph); - } - - TLLM_CUDA_CHECK(cudaGraphDestroy(nextGraph)); - uploadToStream(stream); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -std::optional<std::shared_ptr<CudaGraphExecutor>> CudaGraphExecutorCache::get(BatchState const& state) -{ - auto it = mMap.find(state); - if (it == mMap.end()) - { - return std::nullopt; - } - mCache.splice(mCache.begin(), mCache, it->second); - return it->second->second; -} - -void CudaGraphExecutorCache::put(BatchState const& state, std::shared_ptr<CudaGraphExecutor> const& value) -{ - auto it = mMap.find(state); - if (it != mMap.end()) - { - mCache.erase(it->second); - } - mCache.emplace_front(state, value); - mMap[state] = mCache.begin(); - - if (static_cast<runtime::SizeType32>(mMap.size()) > mCapacity) - { - auto lastState = mCache.back().first; - mCache.pop_back(); - mMap.erase(lastState); - } -} - -void CudaGraphExecutorCache::clear() -{ - // Releasing the shared_ptrs runs ~CudaGraphExecutor, which calls - // cudaGraphExecDestroy on each cached instance. - mMap.clear(); - mCache.clear(); -} - } // namespace tensorrt_llm::batch_manager::utils diff --git a/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h b/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h index fe0c4e505218..374ae398f781 100644 --- a/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h +++ b/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h @@ -20,7 +20,6 @@ #include "tensorrt_llm/batch_manager/common.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/peftCacheManager.h" -#include "tensorrt_llm/batch_manager/runtimeBuffers.h" #include "tensorrt_llm/batch_manager/sequenceSlotManager.h" #include "tensorrt_llm/common/optionalRef.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -50,17 +49,6 @@ void sortRequests(RequestVector& contextRequests, RequestVector& generationReque //! @param scheduledRequests The scheduled context and generation requests. void moveFinishedContextRequestsToGeneration(ScheduledRequests& scheduledRequests); -//! @param beforeDecoder Whether the function is called before the decoder. If it is true, correct the output offset. -//! @param numDroppedTokens The number of dropped tokens for each beam (e.g. when the requests finished early). -//! Generation logits for dropped tokens are ignored. -void copyGenerationLogits(RuntimeBuffers::GenerationLogitsCache& generationLogitsCache, - runtime::BufferManager const& bufferManager, LlmRequest& llmReq, bool beforeDecoder, - std::vector<SizeType32> const& numDroppedTokens = {}); - -void copyAdditionalOutputs(std::vector<executor::AdditionalModelOutput> const& additionalModelOutputs, - RequestVector const& contextRequests, RequestVector const& generationRequests, - RuntimeBuffers::TensorMap const& outputMap, runtime::BufferManager const& manager); - void terminateRequest(SequenceSlotManager& seqSlotManager, LlmRequest& llmRequest, SizeType32 maxInputLen, OptionalRef<kv_cache_manager::BaseKVCacheManager> kvCacheManager = std::nullopt, OptionalRef<kv_cache_manager::BaseKVCacheManager> crossKvCacheManager = std::nullopt, @@ -68,66 +56,4 @@ void terminateRequest(SequenceSlotManager& seqSlotManager, LlmRequest& llmReques std::vector<SizeType32> getRequestBeamWidths( RequestVector const& contextRequests, RequestVector const& generationRequests); - -class CudaGraphExecutor -{ -public: - CudaGraphExecutor() = default; - - ~CudaGraphExecutor() - { - try - { - clear(); - } - catch (std::exception& e) - { - TLLM_LOG_EXCEPTION(e); - } - } - - bool hasInstance() const - { - return mInstance != nullptr; - } - - void clear(); - void prepareNextGraph(std::unique_ptr<runtime::TllmRuntime>& runtime, SizeType32 nextContextId); - void launch(runtime::CudaStream const& stream); - -private: - void create(cudaGraph_t const& graph); - bool update(cudaGraph_t const& graph); - void uploadToStream(runtime::CudaStream const& stream); - - cudaGraphExec_t mInstance = nullptr; -}; - -class CudaGraphExecutorCache -{ - /// @brief LRU cache to store cuda graph instances. -public: - explicit CudaGraphExecutorCache(runtime::SizeType32 capacity) - : mCapacity(capacity) - { - } - - std::optional<std::shared_ptr<CudaGraphExecutor>> get(BatchState const& state); - - void put(BatchState const& state, std::shared_ptr<CudaGraphExecutor> const& value); - - void clear(); - - [[nodiscard]] runtime::SizeType32 size() const noexcept - { - return static_cast<runtime::SizeType32>(mCache.size()); - } - -private: - using BatchStateGraphExecutorPair = std::pair<BatchState, std::shared_ptr<CudaGraphExecutor>>; - using GraphExecutorLruCache = std::list<BatchStateGraphExecutorPair>; - SizeType32 mCapacity; - GraphExecutorLruCache mCache; - std::unordered_map<BatchState, GraphExecutorLruCache::iterator, BatchStateHash> mMap; -}; } // namespace tensorrt_llm::batch_manager::utils diff --git a/cpp/tensorrt_llm/batch_manager/utils/logitsThread.cpp b/cpp/tensorrt_llm/batch_manager/utils/logitsThread.cpp index 941c1b655073..4f978a302187 100644 --- a/cpp/tensorrt_llm/batch_manager/utils/logitsThread.cpp +++ b/cpp/tensorrt_llm/batch_manager/utils/logitsThread.cpp @@ -19,6 +19,7 @@ #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/utils/mpiTags.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" @@ -124,7 +125,7 @@ void draftModelSendLogitsThread(int device, std::atomic<bool>* draftModelThreadS } void targetModelReceiveLogits(runtime::ITensor::SharedPtr& draftLogitsHost, - executor::SpeculativeDecodingFastLogitsInfo const& fastLogitsInfo, nvinfer1::DataType logitsDtype) + executor::SpeculativeDecodingFastLogitsInfo const& fastLogitsInfo, tensorrt_llm::DataType logitsDtype) { #if ENABLE_MULTI_DEVICE auto const& worldComm = tensorrt_llm::mpi::MpiComm::world(); diff --git a/cpp/tensorrt_llm/batch_manager/utils/logitsThread.h b/cpp/tensorrt_llm/batch_manager/utils/logitsThread.h index 637f8a850610..7af3b1762d20 100644 --- a/cpp/tensorrt_llm/batch_manager/utils/logitsThread.h +++ b/cpp/tensorrt_llm/batch_manager/utils/logitsThread.h @@ -18,6 +18,7 @@ #pragma once #include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -46,6 +47,6 @@ void draftModelSendLogitsThread(int device, std::atomic<bool>* draftModelThreadS std::mutex* draftRequestsMtx); void targetModelReceiveLogits(runtime::ITensor::SharedPtr& draftLogitsHost, - executor::SpeculativeDecodingFastLogitsInfo const& fastLogitsInfo, nvinfer1::DataType logitsDtype); + executor::SpeculativeDecodingFastLogitsInfo const& fastLogitsInfo, tensorrt_llm::DataType logitsDtype); } // namespace tensorrt_llm::batch_manager::utils diff --git a/cpp/tensorrt_llm/common/attentionOp.cpp b/cpp/tensorrt_llm/common/attentionOp.cpp index b91c0ef98df6..746decd7de3c 100644 --- a/cpp/tensorrt_llm/common/attentionOp.cpp +++ b/cpp/tensorrt_llm/common/attentionOp.cpp @@ -22,6 +22,7 @@ #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/sageQuant.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decoderMaskedMultiheadAttention.h" #include "tensorrt_llm/kernels/decoderMaskedMultiheadAttention/cascadeAttentionKernel.h" #include "tensorrt_llm/kernels/flashMLA/flash_mla.h" @@ -216,6 +217,7 @@ bool AttentionOp::convertMMHAParamsToXQAParams(tensorrt_llm::kernels::XQAParams& // Medusa mode will have multiple query tokens. xqaParams.multi_query_tokens = mIsSpecDecodingEnabled && mUseSpecDecoding; xqaParams.is_spec_dec_tree = mIsSpecDecTree; + xqaParams.force_prepare_spec_dec_tree_mask = mForcePrepareSpecDecTreeMask; xqaParams.layer_idx = generationsParams.layer_idx; if (mKVCacheQuantMode.hasInt8KvCache()) @@ -691,6 +693,14 @@ void fusedQKV_masked_attention_dispatch(Multihead_attention_params<T_MMHA, CROSS } params.multi_block_mode = input_params.multi_block_mode; + // Cascade-attention partials must be wired regardless of multi_block_mode. + // Cascade decode runs with multi_block disabled (short-decode workloads have + // max_num_seq_len_tiles == 1, so enable_multi_block is structurally false). + // Gating these behind multi_block_mode leaves cascade_partial_* null and makes + // launch_cascade_attention fall back with "cascade workspace not provisioned". + params.cascade_partial_out = input_params.cascade_partial_out; + params.cascade_partial_max = input_params.cascade_partial_max; + params.cascade_partial_sum = input_params.cascade_partial_sum; if (input_params.multi_block_mode) { params.min_seq_len_tile = input_params.min_seq_len_tile; @@ -700,10 +710,6 @@ void fusedQKV_masked_attention_dispatch(Multihead_attention_params<T_MMHA, CROSS params.partial_sum = input_params.partial_sum; params.partial_max = input_params.partial_max; - params.cascade_partial_out = input_params.cascade_partial_out; - params.cascade_partial_max = input_params.cascade_partial_max; - params.cascade_partial_sum = input_params.cascade_partial_sum; - params.block_counter = input_params.block_counter; } @@ -758,8 +764,8 @@ size_t AttentionOp::getFmhaMultiCtasKvScratchSize() const noexcept return partialStatsSize + partialOSize; } -size_t AttentionOp::getWorkspaceSizeForContext(nvinfer1::DataType type, int32_t max_num_seq, int32_t input_seq_length, - int32_t cross_kv_length, int32_t max_num_tokens, int32_t total_kv_len) const noexcept +size_t AttentionOp::getWorkspaceSizeForContext(tensorrt_llm::DataType type, int32_t max_num_seq, + int32_t input_seq_length, int32_t cross_kv_length, int32_t max_num_tokens, int32_t total_kv_len) const noexcept { if (max_num_tokens == 0) { @@ -911,7 +917,7 @@ size_t AttentionOp::getWorkspaceSizeForContext(nvinfer1::DataType type, int32_t return context_workspace_size; } -size_t AttentionOp::getWorkspaceSizeForGeneration(nvinfer1::DataType type, int32_t max_num_seq, +size_t AttentionOp::getWorkspaceSizeForGeneration(tensorrt_llm::DataType type, int32_t max_num_seq, int32_t max_attention_window_size, int32_t max_num_tokens, int32_t max_blocks_per_sequence) const noexcept { if (max_num_tokens == 0) @@ -2818,7 +2824,7 @@ int AttentionOp::initialize() noexcept if (mEnableContextFMHA) { mEnableContextFMHA = false; - if (!(mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kBF16)) + if (!(mType == tensorrt_llm::DataType::kHALF || mType == tensorrt_llm::DataType::kBF16)) { TLLM_LOG_WARNING("Fall back to unfused MHA because of unsupported data type."); } @@ -2863,7 +2869,7 @@ int AttentionOp::initialize() noexcept "mFP8ContextFMHA must enable if FP4 KV cache is enabled"); TLLM_CHECK(isRoPE() == (mRotaryEmbeddingDim != 0)); - TLLM_CHECK_WITH_INFO((mSM >= 80) || (mType != nvinfer1::DataType::kBF16), + TLLM_CHECK_WITH_INFO((mSM >= 80) || (mType != tensorrt_llm::DataType::kBF16), "Unsupported data type, pre SM 80 GPUs do not support bfloat16"); // Pre-check whether the head size is supported by MMHA. @@ -2915,11 +2921,11 @@ int AttentionOp::initialize() noexcept // Pre-checked during constructing. Data_type data_type, data_type_kv; - if (mType == nvinfer1::DataType::kHALF) + if (mType == tensorrt_llm::DataType::kHALF) { data_type = DATA_TYPE_FP16; } - else if (mType == nvinfer1::DataType::kBF16) + else if (mType == tensorrt_llm::DataType::kBF16) { data_type = DATA_TYPE_BF16; } @@ -3079,13 +3085,13 @@ int AttentionOp::initialize() noexcept Data_type kvDataType = DATA_TYPE_FP32; Data_type outputDataType = DATA_TYPE_FP32; - if (mType == nvinfer1::DataType::kHALF) + if (mType == tensorrt_llm::DataType::kHALF) { qDataType = DATA_TYPE_FP16; kvDataType = DATA_TYPE_FP16; outputDataType = DATA_TYPE_FP16; } - else if (mType == nvinfer1::DataType::kBF16) + else if (mType == tensorrt_llm::DataType::kBF16) { qDataType = DATA_TYPE_BF16; kvDataType = DATA_TYPE_BF16; @@ -3175,7 +3181,7 @@ int AttentionOp::initialize() noexcept } mEnableXQA = (mEnableXQA || mIsSpecDecodingEnabled) - && (mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kBF16) && mUseKVCache; + && (mType == tensorrt_llm::DataType::kHALF || mType == tensorrt_llm::DataType::kBF16) && mUseKVCache; if (mEnableXQA) { @@ -3185,12 +3191,12 @@ int AttentionOp::initialize() noexcept fixedParams.isMLA = mIsGenerationMLA; // TODO: support more combinations. // Update Q and O dtype. - if (mType == nvinfer1::DataType::kHALF) + if (mType == tensorrt_llm::DataType::kHALF) { fixedParams.inputDataType = DATA_TYPE_FP16; fixedParams.outputDataType = DATA_TYPE_FP16; } - else if (mType == nvinfer1::DataType::kBF16) + else if (mType == tensorrt_llm::DataType::kBF16) { fixedParams.inputDataType = DATA_TYPE_BF16; fixedParams.outputDataType = DATA_TYPE_BF16; @@ -3258,10 +3264,6 @@ int AttentionOp::initialize() noexcept reserveSemaphoreArray(mNbMultiBlockSemaphores); } - if (isBuilding()) - { - return 0; - } #if ENABLE_MULTI_DEVICE if (mCpSize > 1 && COMM_SESSION.getSize() > 1) { diff --git a/cpp/tensorrt_llm/common/attentionOp.h b/cpp/tensorrt_llm/common/attentionOp.h index f7337c9c9cb2..438489348577 100644 --- a/cpp/tensorrt_llm/common/attentionOp.h +++ b/cpp/tensorrt_llm/common/attentionOp.h @@ -20,6 +20,7 @@ #include "tensorrt_llm/common/cublasMMWrapper.h" #include "tensorrt_llm/common/opUtils.h" #include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/contextFusedMultiHeadAttention/fused_multihead_attention_common.h" #include "tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_gemm.h" #include "tensorrt_llm/kernels/decoderMaskedMultiheadAttention/decoderXQARunner.h" @@ -56,10 +57,11 @@ class AttentionOp [[nodiscard]] size_t getFmhaMultiCtasKvScratchSize() const noexcept; [[nodiscard]] int getHeadSize(bool checkInit = true) const; [[nodiscard]] int getMaxNumSeqLenTile(int batch_beam_size = 1) const; - [[nodiscard]] size_t getWorkspaceSizeForContext(nvinfer1::DataType type, int32_t nbReq, int32_t max_input_length, - int32_t cross_kv_length = 0, int32_t max_num_tokens = 0, int32_t total_kv_len = 0) const noexcept; + [[nodiscard]] size_t getWorkspaceSizeForContext(tensorrt_llm::DataType type, int32_t nbReq, + int32_t max_input_length, int32_t cross_kv_length = 0, int32_t max_num_tokens = 0, + int32_t total_kv_len = 0) const noexcept; // total_num_seq is the sum of beam_width for multiple requests - [[nodiscard]] size_t getWorkspaceSizeForGeneration(nvinfer1::DataType type, int32_t total_num_seq, + [[nodiscard]] size_t getWorkspaceSizeForGeneration(tensorrt_llm::DataType type, int32_t total_num_seq, int32_t max_attention_window_size, int32_t max_num_tokens, int32_t max_blocks_per_sequence) const noexcept; template <typename T> @@ -181,14 +183,14 @@ class AttentionOp if (this->context_lengths && batch_size > 0) { ss << "context_lengths: " - << *(runtime::ITensor::wrap((void*) this->context_lengths, nvinfer1::DataType::kINT32, + << *(runtime::ITensor::wrap((void*) this->context_lengths, tensorrt_llm::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))) << std::endl; } if (this->sequence_lengths && batch_size > 0) { ss << "sequence_lengths: " - << *(runtime::ITensor::wrap((void*) this->sequence_lengths, nvinfer1::DataType::kINT32, + << *(runtime::ITensor::wrap((void*) this->sequence_lengths, tensorrt_llm::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))) << std::endl; } @@ -478,7 +480,7 @@ class AttentionOp int mTpSize = 1; int mTpRank = 0; bool mUnfuseQkvGemm = false; - nvinfer1::DataType mType; + tensorrt_llm::DataType mType; int32_t mMaxContextLength = 0; int32_t mMaxSeqLen = 0; int32_t mMaxNumRequests = 0; @@ -501,6 +503,7 @@ class AttentionOp int32_t mSpecDecodingMaxGenerationLength = 1; // Static spec-dec tree length used by FMHA autotuning. int32_t mSpecDecodingTargetMaxGenLen = 0; + bool mForcePrepareSpecDecTreeMask = false; bool mIsMLAEnabled = false; bool mIsGenerationMLA = false; bool mUseGenFlashMLA = false; @@ -571,11 +574,11 @@ class AttentionOp mCrossAttention, mMaxDistance, mPosShiftEnabled, mPagedContextFMHA, mFP8ContextFMHA, mFP8AttenOutput, mFP8ContextMLA, mFP8GenerationMLA, mChunkPrefillBufferBatchSize, mDenseContextFMHA, mHasFullAttentionMask, mIsSpecDecodingEnabled, mUseSpecDecoding, mIsSpecDecTree, mSpecDecodingIsGenerationLengthVariable, - mSpecDecodingMaxGenerationLength, mSpecDecodingTargetMaxGenLen, mIsMLAEnabled, mIsGenerationMLA, - mUseGenFlashMLA, mUseSparseAttention, mUseTllmGenSparseAttentionPaged, mUseTllmGenSparseAttention, - mMLAParams.data(), mCpSize, mCpRank, mCpGroup, mNumAttnHeads, mNumAttnKVHeads, mNumKVHeadsOrigin, - mAttnTpSize, mAttnTpRank, mAttnCpSize, mAttnCpRank, mUlyssesMQABroadcast, mEnableContextFMHA, - mFMHAForceFP32Acc, mMultiBlockMode, mEnableXQA, mUseKVCache, mSkipAttn, mFuseFp4Quant, + mSpecDecodingMaxGenerationLength, mSpecDecodingTargetMaxGenLen, mForcePrepareSpecDecTreeMask, mIsMLAEnabled, + mIsGenerationMLA, mUseGenFlashMLA, mUseSparseAttention, mUseTllmGenSparseAttentionPaged, + mUseTllmGenSparseAttention, mMLAParams.data(), mCpSize, mCpRank, mCpGroup, mNumAttnHeads, mNumAttnKVHeads, + mNumKVHeadsOrigin, mAttnTpSize, mAttnTpRank, mAttnCpSize, mAttnCpRank, mUlyssesMQABroadcast, + mEnableContextFMHA, mFMHAForceFP32Acc, mMultiBlockMode, mEnableXQA, mUseKVCache, mSkipAttn, mFuseFp4Quant, mFusesDsv4InvRopeFp8Quant, mNbMultiBlockSemaphores, mAttentionChunkSize.value_or(-1), mSkipSoftmaxThresholdScaleFactorPrefill, mSkipSoftmaxThresholdScaleFactorDecode, mSageAttnNumEltsPerBlkQ, mSageAttnNumEltsPerBlkK, mSageAttnNumEltsPerBlkV, mSageAttnQkInt8); diff --git a/cpp/tensorrt_llm/common/cudaFp8Utils.cu b/cpp/tensorrt_llm/common/cudaFp8Utils.cu index 9cd0227739d8..03aae114c94f 100644 --- a/cpp/tensorrt_llm/common/cudaFp8Utils.cu +++ b/cpp/tensorrt_llm/common/cudaFp8Utils.cu @@ -421,6 +421,45 @@ __global__ void computeFP8QuantizeScale(T_S* quant_ptr, const T_W* weights, cons } } +// Vectorized PER_TENSOR amax→scale for bf16 input: 128-bit loads (8 bf16/thread), +// block reduce, one atomicMax per block. Paired with a numel-sized grid +// (scaleMatrixVecGridSize) it launches only as many blocks as the input needs, so +// a skinny decode activation ([64, 2048]) contends ~16 blocks on the scale address +// instead of the scalar path's fixed grid(1024) — whose ~1024-way single-address +// atomicMax plus ~900 idle blocks give a data-independent ~6us floor. Mirrors the +// existing vectorized PER_TENSOR *apply* kernel (scaleMatrixPerTensorVec). +template <typename T_S> +__global__ void computeFP8QuantizeScalePerTensorVec(T_S* quant_ptr, __nv_bfloat16 const* weights, int64_t const numel) +{ + constexpr float min_scaling_factor = 1.0f / (FP8_E4M3_MAX * 512.f); + int64_t const vecElements = numel / kVecSize; + int64_t const stride = static_cast<int64_t>(blockDim.x) * gridDim.x; + float max = 0.f; + for (int64_t vi = threadIdx.x + static_cast<int64_t>(blockIdx.x) * blockDim.x; vi < vecElements; vi += stride) + { + float4 raw = *reinterpret_cast<float4 const*>(weights + vi * kVecSize); + __nv_bfloat162 const* pairs = reinterpret_cast<__nv_bfloat162 const*>(&raw); +#pragma unroll + for (int p = 0; p < kPairsPerVec; ++p) + { + float2 const f2 = __bfloat1622float2(pairs[p]); + max = fmaxf(max, fmaxf(fabsf(f2.x), fabsf(f2.y))); + } + } + // scalar tail (numel not divisible by 8) + int64_t const tailStart = vecElements * kVecSize; + for (int64_t i = tailStart + threadIdx.x + static_cast<int64_t>(blockIdx.x) * blockDim.x; i < numel; i += stride) + { + max = fmaxf(max, fabsf(static_cast<float>(weights[i]))); + } + max = blockReduceMax<float>(max); + if (threadIdx.x == 0) + { + auto const scale = (T_S) std::max(max / FP8_E4M3_MAX, min_scaling_factor); + atomicMaxExtd(quant_ptr, scale); + } +} + template <typename T_S, typename T_W> void invokeComputeFP8QuantizeScale(T_S* quant_ptr, const T_W* weights, const int64_t numel, const int64_t lda, QuantizeMode quantize_mode, cudaStream_t stream) @@ -441,11 +480,34 @@ void invokeComputeFP8QuantizeScale(T_S* quant_ptr, const T_W* weights, const int } else if (quantize_mode == QuantizeMode::PER_TENSOR) { - dim3 block(1024); - dim3 grid(1024); cudaMemsetAsync(quant_ptr, 0, sizeof(T_S), stream); sync_check_cuda_error(stream); - computeFP8QuantizeScale<QuantizeMode::PER_TENSOR><<<grid, block, 0, stream>>>(quant_ptr, weights, numel, lda); + // Size the grid to the input (scaleMatrixVecGridSize), NOT a fixed 1024, so + // small activations launch only a handful of blocks — this is the fix for + // the ~6us fixed floor at skinny decode shapes. For 16B-aligned bf16 input + // use the vectorized amax kernel; otherwise the scalar kernel, still with a + // numel-sized grid. + dim3 const block(CTA_SIZE); + dim3 const grid(static_cast<unsigned int>(scaleMatrixVecGridSize(numel))); + bool const aligned = (reinterpret_cast<uintptr_t>(weights) % 16 == 0); + if constexpr (std::is_same_v<T_W, __nv_bfloat16>) + { + if (aligned) + { + computeFP8QuantizeScalePerTensorVec<<<grid, block, 0, stream>>>( + quant_ptr, reinterpret_cast<__nv_bfloat16 const*>(weights), numel); + } + else + { + computeFP8QuantizeScale<QuantizeMode::PER_TENSOR> + <<<grid, block, 0, stream>>>(quant_ptr, weights, numel, lda); + } + } + else + { + computeFP8QuantizeScale<QuantizeMode::PER_TENSOR> + <<<grid, block, 0, stream>>>(quant_ptr, weights, numel, lda); + } } sync_check_cuda_error(stream); } @@ -535,11 +597,28 @@ void invokeComputeScalesAndQuantizeMatrix(T_OUT* output, T_S* quant_ptr, const T } else if (quantize_mode == QuantizeMode::PER_TENSOR) { - dim3 block(1024); - dim3 grid(1024); cudaMemsetAsync(quant_ptr, 0, sizeof(T_S), stream); sync_check_cuda_error(stream); - computeFP8QuantizeScale<QuantizeMode::PER_TENSOR><<<grid, block, 0, stream>>>(quant_ptr, input, numel, lda); + // Size the amax grid to the input (scaleMatrixVecGridSize), NOT a fixed 1024, + // so skinny decode activations launch only a handful of blocks — this is the + // fix for the ~6us fixed floor. bf16 (16B-aligned) uses the vectorized amax + // kernel; otherwise the scalar kernel, still numel-sized. Then apply as before. + dim3 const block(CTA_SIZE); + dim3 const grid(static_cast<unsigned int>(scaleMatrixVecGridSize(numel))); + bool const aligned = (reinterpret_cast<uintptr_t>(input) % 16 == 0); + if constexpr (std::is_same_v<T_IN, __nv_bfloat16>) + { + if (aligned) + computeFP8QuantizeScalePerTensorVec<<<grid, block, 0, stream>>>( + quant_ptr, reinterpret_cast<__nv_bfloat16 const*>(input), numel); + else + computeFP8QuantizeScale<QuantizeMode::PER_TENSOR> + <<<grid, block, 0, stream>>>(quant_ptr, input, numel, lda); + } + else + { + computeFP8QuantizeScale<QuantizeMode::PER_TENSOR><<<grid, block, 0, stream>>>(quant_ptr, input, numel, lda); + } sync_check_cuda_error(stream); invokeQuantizeMatrix(output, quant_ptr, input, numel, lda, quantize_mode, stream); } diff --git a/cpp/tensorrt_llm/common/envUtils.cpp b/cpp/tensorrt_llm/common/envUtils.cpp index 14c637699584..9dcf6c81634e 100644 --- a/cpp/tensorrt_llm/common/envUtils.cpp +++ b/cpp/tensorrt_llm/common/envUtils.cpp @@ -517,10 +517,10 @@ uint16_t getEnvNixlPort() return nixlPort; } -bool getEnvNixlEnableCoalesce() +bool getEnvNixlDisableCoalesce() { - static bool const enableCoalesce = getBoolEnv("TRTLLM_NIXL_ENABLE_COALESCE"); - return enableCoalesce; + static bool const disableCoalesce = getBoolEnv("TRTLLM_NIXL_DISABLE_COALESCE"); + return disableCoalesce; } bool getEnvDisaggBenchmarkGenOnly() diff --git a/cpp/tensorrt_llm/common/envUtils.h b/cpp/tensorrt_llm/common/envUtils.h index 37b5934b4515..13ad0399d574 100644 --- a/cpp/tensorrt_llm/common/envUtils.h +++ b/cpp/tensorrt_llm/common/envUtils.h @@ -151,7 +151,8 @@ bool getEnvKVCachePoolUseFabricMemory(); uint16_t getEnvNixlPort(); -bool getEnvNixlEnableCoalesce(); +// Whether to disable coalescing of contiguous NIXL transfer descriptors (coalescing is on by default). +bool getEnvNixlDisableCoalesce(); bool getEnvDisaggBenchmarkGenOnly(); diff --git a/cpp/tensorrt_llm/common/opUtils.cpp b/cpp/tensorrt_llm/common/opUtils.cpp index ff9b57cdd099..8622e6bd9545 100644 --- a/cpp/tensorrt_llm/common/opUtils.cpp +++ b/cpp/tensorrt_llm/common/opUtils.cpp @@ -17,6 +17,7 @@ #include "tensorrt_llm/common/opUtils.h" #include "tensorrt_llm/common/ncclUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/ipcNvlsMemory.h" #include "tensorrt_llm/runtime/utils/mpiTags.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" @@ -33,18 +34,18 @@ TRTLLM_NAMESPACE_BEGIN #if ENABLE_MULTI_DEVICE -std::unordered_map<nvinfer1::DataType, ncclDataType_t>* getDtypeMap() +std::unordered_map<tensorrt_llm::DataType, ncclDataType_t>* getDtypeMap() { - static std::unordered_map<nvinfer1::DataType, ncclDataType_t> dtypeMap = { - {nvinfer1::DataType::kFLOAT, ncclFloat32}, - {nvinfer1::DataType::kHALF, ncclFloat16}, - {nvinfer1::DataType::kBF16, ncclBfloat16}, - {nvinfer1::DataType::kFP8, ncclInt8}, - {nvinfer1::DataType::kBOOL, ncclInt8}, - {nvinfer1::DataType::kINT32, ncclInt32}, - {nvinfer1::DataType::kINT64, ncclInt64}, - {nvinfer1::DataType::kUINT8, ncclUint8}, - {nvinfer1::DataType::kINT8, ncclInt8}, + static std::unordered_map<tensorrt_llm::DataType, ncclDataType_t> dtypeMap = { + {tensorrt_llm::DataType::kFLOAT, ncclFloat32}, + {tensorrt_llm::DataType::kHALF, ncclFloat16}, + {tensorrt_llm::DataType::kBF16, ncclBfloat16}, + {tensorrt_llm::DataType::kFP8, ncclInt8}, + {tensorrt_llm::DataType::kBOOL, ncclInt8}, + {tensorrt_llm::DataType::kINT32, ncclInt32}, + {tensorrt_llm::DataType::kINT64, ncclInt64}, + {tensorrt_llm::DataType::kUINT8, ncclUint8}, + {tensorrt_llm::DataType::kINT8, ncclInt8}, }; return &dtypeMap; } @@ -193,7 +194,6 @@ void const* tensorrt_llm::common::op::getCommSessionHandle() namespace { -using tensorrt_llm::common::op::hash; // Get current cuda context, a default context will be created if there is no context. inline CUcontext getCurrentCudaCtx() @@ -228,15 +228,14 @@ class PerCudaCtxPerThreadSingletonCreator PerCudaCtxPerThreadSingletonCreator(CreatorFunc creator, DeleterFunc deleter) : mCreator{std::move(creator)} , mDeleter{std::move(deleter)} - , mObservers{new std::unordered_map<CacheKey, std::weak_ptr<T>, hash<CacheKey>>()} + , mObservers{std::make_unique<CacheType>()} { } ~PerCudaCtxPerThreadSingletonCreator() { std::lock_guard<std::mutex> lk{mMutex}; - delete mObservers; - mObservers = nullptr; + mObservers.reset(); } std::shared_ptr<T> operator()() @@ -298,7 +297,8 @@ class PerCudaCtxPerThreadSingletonCreator mutable std::mutex mMutex; // CUDA resources are per-context and per-thread. using CacheKey = std::tuple<CUcontext, std::thread::id>; - std::unordered_map<CacheKey, std::weak_ptr<T>, hash<CacheKey>>* mObservers; + using CacheType = std::unordered_map<CacheKey, std::weak_ptr<T>, common::op::OpCustomHash<CacheKey>>; + std::unique_ptr<CacheType> mObservers; }; // Structure to hold memory information diff --git a/cpp/tensorrt_llm/common/opUtils.h b/cpp/tensorrt_llm/common/opUtils.h index 72e5a5ea3e09..fb0f2aca3d62 100644 --- a/cpp/tensorrt_llm/common/opUtils.h +++ b/cpp/tensorrt_llm/common/opUtils.h @@ -21,7 +21,7 @@ #include "tensorrt_llm/common/cublasMMWrapper.h" #include "tensorrt_llm/common/workspace.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cublasLt.h> #include <cublas_v2.h> #include <cuda_runtime.h> @@ -62,14 +62,14 @@ void read(char const*& buffer, T& val) buffer += sizeof(T); } -inline cudaDataType_t trtToCublasDtype(nvinfer1::DataType type) +inline cudaDataType_t trtToCublasDtype(tensorrt_llm::DataType type) { switch (type) { - case nvinfer1::DataType::kFLOAT: return CUDA_R_32F; - case nvinfer1::DataType::kHALF: return CUDA_R_16F; + case tensorrt_llm::DataType::kFLOAT: return CUDA_R_32F; + case tensorrt_llm::DataType::kHALF: return CUDA_R_16F; #if defined(NV_TENSORRT_MAJOR) && NV_TENSORRT_MAJOR >= 9 - case nvinfer1::DataType::kBF16: return CUDA_R_16BF; + case tensorrt_llm::DataType::kBF16: return CUDA_R_16BF; #endif default: TLLM_THROW("Not supported data type for cuBLAS"); } @@ -108,21 +108,20 @@ namespace { template <typename T> -struct hash_helper; - -// Base case: use std::hash for basic types -template <typename T> -struct hash_helper +struct OpCustomHash : public std::hash<T> { - size_t operator()(T const& v) const - { - return std::hash<T>{}(v); - } }; -// Specialization for std::set +template <class T> +inline size_t hash_combine(size_t hash, T const& value) +{ + static constexpr size_t seed = 0x9e3779b9ULL; + using RemoveCVRefT = std::remove_cv_t<std::remove_reference_t<T>>; + return OpCustomHash<RemoveCVRefT>{}(value) + seed + (hash << 6) + (hash >> 2); +} + template <typename T> -struct hash_helper<std::set<T>> +struct OpCustomHash<std::set<T>> { size_t operator()(std::set<T> const& s) const { @@ -130,68 +129,34 @@ struct hash_helper<std::set<T>> for (auto const& item : s) { // Recursively hash each element - hash_value ^= hash_helper<T>{}(item) + 0x9e3779b9 + (hash_value << 6) + (hash_value >> 2); + hash_value ^= hash_combine(hash_value, item); } return hash_value; } }; -// Helper for tuple hashing -template <typename Tuple, size_t Index = std::tuple_size<Tuple>::value - 1> -struct tuple_hash_helper +template <class... Args> +class OpCustomHash<std::tuple<Args...>> { - static size_t hash(Tuple const& tuple) + template <std::size_t... Idx> + static size_t hash_impl(std::tuple<Args...> const& t, std::integer_sequence<std::size_t, Idx...>) { - size_t hash_value = tuple_hash_helper<Tuple, Index - 1>::hash(tuple); - return hash_value - ^ (hash_helper<typename std::tuple_element<Index, Tuple>::type>{}(std::get<Index>(tuple)) + 0x9e3779b9 - + (hash_value << 6) + (hash_value >> 2)); + size_t value = 0; + return ((value ^= hash_combine(value, std::get<Idx>(t))), ...); } -}; - -// Base case for tuple hashing -template <typename Tuple> -struct tuple_hash_helper<Tuple, 0> -{ - static size_t hash(Tuple const& tuple) - { - return hash_helper<typename std::tuple_element<0, Tuple>::type>{}(std::get<0>(tuple)); - } -}; -// Specialization for std::tuple -template <typename... Args> -struct hash_helper<std::tuple<Args...>> -{ +public: size_t operator()(std::tuple<Args...> const& t) const { - return tuple_hash_helper<std::tuple<Args...>>::hash(t); + return hash_impl(t, std::make_index_sequence<sizeof...(Args)>{}); } }; - } // namespace -// Main hash struct to be used -template <typename T> -struct hash -{ - size_t operator()(T const& v) const - { - return hash_helper<T>{}(v); - } -}; - // for testing only void const* getCommSessionHandle(); } // namespace common::op -inline bool isBuilding() -{ - auto constexpr key = "IS_BUILDING"; - auto const val = getenv(key); - return val != nullptr && std::string(val) == "1"; -} - #if ENABLE_MULTI_DEVICE #define NCCLCHECK(cmd) \ do \ @@ -214,7 +179,7 @@ inline bool isBuilding() } \ } while (0) -std::unordered_map<nvinfer1::DataType, ncclDataType_t>* getDtypeMap(); +std::unordered_map<tensorrt_llm::DataType, ncclDataType_t>* getDtypeMap(); std::shared_ptr<ncclComm_t> getComm(std::set<int> const& group); diff --git a/cpp/tensorrt_llm/common/safetensors.cpp b/cpp/tensorrt_llm/common/safetensors.cpp index 9171f79e44e5..8bd91ccfbd51 100644 --- a/cpp/tensorrt_llm/common/safetensors.cpp +++ b/cpp/tensorrt_llm/common/safetensors.cpp @@ -18,7 +18,7 @@ #include "nlohmann/json.hpp" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/config.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cstdint> #include <fstream> #include <map> @@ -30,7 +30,7 @@ TRTLLM_NAMESPACE_BEGIN namespace common::safetensors { -using nvinfer1::DataType; +using tensorrt_llm::DataType; static DataType convertDataTypeStrToEnum(std::string const& str) { diff --git a/cpp/tensorrt_llm/common/safetensors.h b/cpp/tensorrt_llm/common/safetensors.h index e31225f1be24..bdecf95e5909 100644 --- a/cpp/tensorrt_llm/common/safetensors.h +++ b/cpp/tensorrt_llm/common/safetensors.h @@ -18,7 +18,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/logger.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cstdint> #include <map> #include <memory> @@ -34,13 +34,13 @@ class INdArray [[nodiscard]] virtual void const* data() const = 0; [[nodiscard]] virtual int ndim() const = 0; [[nodiscard]] virtual std::vector<int64_t> const& dims() const = 0; - [[nodiscard]] virtual nvinfer1::DataType dtype() const = 0; + [[nodiscard]] virtual tensorrt_llm::DataType dtype() const = 0; - [[nodiscard]] nvinfer1::Dims trtDims() const + [[nodiscard]] tensorrt_llm::Dims trtDims() const { - nvinfer1::Dims dims; + tensorrt_llm::Dims dims; dims.nbDims = ndim(); - TLLM_CHECK(dims.nbDims <= nvinfer1::Dims::MAX_DIMS); + TLLM_CHECK(dims.nbDims <= tensorrt_llm::Dims::MAX_DIMS); memset(dims.d, 0, sizeof(dims.d)); for (int i = 0; i < dims.nbDims; ++i) { diff --git a/cpp/tensorrt_llm/common/sha256/README.md b/cpp/tensorrt_llm/common/sha256/README.md new file mode 100644 index 000000000000..99c61e7a76ef --- /dev/null +++ b/cpp/tensorrt_llm/common/sha256/README.md @@ -0,0 +1,53 @@ +# SHA-256 (vendored from Bitcoin Core, modified) + +This directory contains the SHA-256 implementation from the +[Bitcoin Core](https://github.com/bitcoin/bitcoin) project, **reduced and +adapted by NVIDIA** to the minimal single-block hasher needed by TensorRT-LLM. +It provides a portable scalar SHA-256 with runtime dispatch to a +hardware-accelerated transform (x86 SHA-NI or ARMv8 crypto extensions), exposed +through the `CSHA256` class. + +TensorRT-LLM uses it in the C++ KVCacheManagerV2 (`blockRadixTree`) to hash +token sequences into block keys, byte-identically to the Python backend's +`hashlib.sha256` block keys. + +## Provenance + +- **Upstream project:** Bitcoin Core — https://github.com/bitcoin/bitcoin +- **Source path upstream:** `src/`, `src/crypto/`, and `src/compat` +- **Upstream commit:** `70d9ec7f3d452789d04dce81dc02db0b3b778bb5` (branch `master`) + +## Contents + +| File | Notes | +|------|-------| +| `sha256.h` | `CSHA256` API; `SHA256D64` declaration removed | +| `sha256.cpp` | Scalar core + `CSHA256` + slim runtime dispatch | +| `sha256_x86_shani.cpp` | x86 SHA-NI transform (guard changed) | +| `sha256_arm_shani.cpp` | ARMv8 crypto transform (guard changed) | +| `attributes.h` | `ALWAYS_INLINE` macro used by the x86 transform | +| `sha256_endian.h` | Big-endian helpers replacing upstream `common.h` | + +## NVIDIA modifications + +The files are reduced to the single-block path TensorRT-LLM needs. + +Changes vs. upstream: + +- **Removed** the public `SHA256D64` double-hash API (and its scalar + `TransformD64`) and the standalone SSE4 / SSE4.1 / AVX2 multi-block transform + files (unused — TensorRT-LLM only calls single-block `CSHA256`). The two + SHA-NI files retain their upstream 2-way `Transform_2way` helpers; these are + now unreferenced but were left in place to keep the transforms byte-close to + upstream. +- **Removed** the upstream support-header chain (`crypto/common.h`, + `compat/endian.h`, `compat/byteswap.h`, `compat/cpuid.h`), which pulled in + C++20 (`<bit>`, `<concepts>`). Endian helpers are now the small, + NVIDIA-authored `sha256_endian.h`; CPU detection uses a `CPUID` leaf-7 check + via `__get_cpuid_count` (x86) / `getauxval(AT_HWCAP)` (aarch64). The vendored + sources now build as plain **C++17**. +- **Flattened** the directory (no `crypto/` / `compat/` subtree) and switched + the HW-transform build guards from the upstream `ENABLE_*` macros to target + architecture macros. Build wiring (per-file `-msha` / `-march=armv8-a+crypto`) + lives in `cpp/tensorrt_llm/batch_manager/kv_cache_manager_v2/CMakeLists.txt`. + diff --git a/cpp/tensorrt_llm/common/sha256/attributes.h b/cpp/tensorrt_llm/common/sha256/attributes.h new file mode 100644 index 000000000000..275dad9f8ede --- /dev/null +++ b/cpp/tensorrt_llm/common/sha256/attributes.h @@ -0,0 +1,27 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ATTRIBUTES_H +#define BITCOIN_ATTRIBUTES_H + +#if defined(__clang__) +# if __has_attribute(lifetimebound) +# define LIFETIMEBOUND [[clang::lifetimebound]] +# else +# define LIFETIMEBOUND +# endif +#else +# define LIFETIMEBOUND +#endif + +#if defined(__GNUC__) +# define ALWAYS_INLINE inline __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define ALWAYS_INLINE __forceinline +#else +# error No known always_inline attribute for this platform. +#endif + +#endif // BITCOIN_ATTRIBUTES_H diff --git a/cpp/tensorrt_llm/common/sha256/sha256.cpp b/cpp/tensorrt_llm/common/sha256/sha256.cpp new file mode 100644 index 000000000000..c2200bc71586 --- /dev/null +++ b/cpp/tensorrt_llm/common/sha256/sha256.cpp @@ -0,0 +1,302 @@ +// SPDX-FileCopyrightText: Copyright (c) 2014-present The Bitcoin Core developers +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// +// Modifications for TensorRT-LLM: +// - Reduced to the single-block CSHA256 hasher. +// - Removed the SHA256D64 double-hash API and the SSE4/SSE4.1/AVX2 multi-block +// transforms; runtime dispatch selects scalar / x86 SHA-NI / ARMv8 crypto. +// - Replaced <crypto/common.h> / <compat/*> endian helpers with sha256_endian.h. +// - CPU detection via CPUID leaf 7 (x86) / getauxval (aarch64); plain C++17 (no <bit>). + +#include "sha256.h" +#include "sha256_endian.h" + +#include <algorithm> +#include <cassert> +#include <cstring> +#include <string> + +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#include <cpuid.h> +namespace sha256_x86_shani { +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); +} +#endif + +#if defined(__aarch64__) +#include <sys/auxv.h> +#include <asm/hwcap.h> +namespace sha256_arm_shani { +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); +} +#endif + +// Internal implementation code. +namespace +{ +/// Internal SHA-256 implementation. +namespace sha256 +{ +uint32_t inline Ch(uint32_t x, uint32_t y, uint32_t z) { return z ^ (x & (y ^ z)); } +uint32_t inline Maj(uint32_t x, uint32_t y, uint32_t z) { return (x & y) | (z & (x | y)); } +uint32_t inline Sigma0(uint32_t x) { return (x >> 2 | x << 30) ^ (x >> 13 | x << 19) ^ (x >> 22 | x << 10); } +uint32_t inline Sigma1(uint32_t x) { return (x >> 6 | x << 26) ^ (x >> 11 | x << 21) ^ (x >> 25 | x << 7); } +uint32_t inline sigma0(uint32_t x) { return (x >> 7 | x << 25) ^ (x >> 18 | x << 14) ^ (x >> 3); } +uint32_t inline sigma1(uint32_t x) { return (x >> 17 | x << 15) ^ (x >> 19 | x << 13) ^ (x >> 10); } + +/** One round of SHA-256. */ +void inline Round(uint32_t a, uint32_t b, uint32_t c, uint32_t& d, uint32_t e, uint32_t f, uint32_t g, uint32_t& h, uint32_t k) +{ + uint32_t t1 = h + Sigma1(e) + Ch(e, f, g) + k; + uint32_t t2 = Sigma0(a) + Maj(a, b, c); + d += t1; + h = t1 + t2; +} + +/** Initialize SHA-256 state. */ +void inline Initialize(uint32_t* s) +{ + s[0] = 0x6a09e667ul; + s[1] = 0xbb67ae85ul; + s[2] = 0x3c6ef372ul; + s[3] = 0xa54ff53aul; + s[4] = 0x510e527ful; + s[5] = 0x9b05688cul; + s[6] = 0x1f83d9abul; + s[7] = 0x5be0cd19ul; +} + +/** Perform a number of SHA-256 transformations, processing 64-byte chunks. */ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) +{ + while (blocks--) { + uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7]; + uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; + + Round(a, b, c, d, e, f, g, h, 0x428a2f98 + (w0 = ReadBE32(chunk + 0))); + Round(h, a, b, c, d, e, f, g, 0x71374491 + (w1 = ReadBE32(chunk + 4))); + Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf + (w2 = ReadBE32(chunk + 8))); + Round(f, g, h, a, b, c, d, e, 0xe9b5dba5 + (w3 = ReadBE32(chunk + 12))); + Round(e, f, g, h, a, b, c, d, 0x3956c25b + (w4 = ReadBE32(chunk + 16))); + Round(d, e, f, g, h, a, b, c, 0x59f111f1 + (w5 = ReadBE32(chunk + 20))); + Round(c, d, e, f, g, h, a, b, 0x923f82a4 + (w6 = ReadBE32(chunk + 24))); + Round(b, c, d, e, f, g, h, a, 0xab1c5ed5 + (w7 = ReadBE32(chunk + 28))); + Round(a, b, c, d, e, f, g, h, 0xd807aa98 + (w8 = ReadBE32(chunk + 32))); + Round(h, a, b, c, d, e, f, g, 0x12835b01 + (w9 = ReadBE32(chunk + 36))); + Round(g, h, a, b, c, d, e, f, 0x243185be + (w10 = ReadBE32(chunk + 40))); + Round(f, g, h, a, b, c, d, e, 0x550c7dc3 + (w11 = ReadBE32(chunk + 44))); + Round(e, f, g, h, a, b, c, d, 0x72be5d74 + (w12 = ReadBE32(chunk + 48))); + Round(d, e, f, g, h, a, b, c, 0x80deb1fe + (w13 = ReadBE32(chunk + 52))); + Round(c, d, e, f, g, h, a, b, 0x9bdc06a7 + (w14 = ReadBE32(chunk + 56))); + Round(b, c, d, e, f, g, h, a, 0xc19bf174 + (w15 = ReadBE32(chunk + 60))); + + Round(a, b, c, d, e, f, g, h, 0xe49b69c1 + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0xefbe4786 + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x0fc19dc6 + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x240ca1cc + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x2de92c6f + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x4a7484aa + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x5cb0a9dc + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x76f988da + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0x983e5152 + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0xa831c66d + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0xb00327c8 + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0xbf597fc7 + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0xc6e00bf3 + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xd5a79147 + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0x06ca6351 + (w14 += sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0x14292967 + (w15 += sigma1(w13) + w8 + sigma0(w0))); + + Round(a, b, c, d, e, f, g, h, 0x27b70a85 + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0x2e1b2138 + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x4d2c6dfc + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x53380d13 + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x650a7354 + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x766a0abb + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x81c2c92e + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x92722c85 + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1 + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0xa81a664b + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0xc24b8b70 + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0xc76c51a3 + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0xd192e819 + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xd6990624 + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0xf40e3585 + (w14 += sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0x106aa070 + (w15 += sigma1(w13) + w8 + sigma0(w0))); + + Round(a, b, c, d, e, f, g, h, 0x19a4c116 + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0x1e376c08 + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x2748774c + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x34b0bcb5 + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x391c0cb3 + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x4ed8aa4a + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x5b9cca4f + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x682e6ff3 + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0x748f82ee + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0x78a5636f + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0x84c87814 + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0x8cc70208 + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0x90befffa + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xa4506ceb + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0xbef9a3f7 + (w14 + sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0xc67178f2 + (w15 + sigma1(w13) + w8 + sigma0(w0))); + + s[0] += a; + s[1] += b; + s[2] += c; + s[3] += d; + s[4] += e; + s[5] += f; + s[6] += g; + s[7] += h; + chunk += 64; + } +} + +} // namespace sha256 + +typedef void (*TransformType)(uint32_t*, const unsigned char*, size_t); +TransformType Transform = sha256::Transform; + +[[maybe_unused]] bool SelfTest() { + // Input state (equal to the initial SHA256 state) + static const uint32_t init[8] = { + 0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul + }; + // Some random input data to test with + static const unsigned char data[641] = "-" // Intentionally not aligned + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua. Et m" + "olestie ac feugiat sed lectus vestibulum mattis ullamcorper. Mor" + "bi blandit cursus risus at ultrices mi tempus imperdiet nulla. N" + "unc congue nisi vita suscipit tellus mauris. Imperdiet proin fer" + "mentum leo vel orci. Massa tempor nec feugiat nisl pretium fusce" + " id velit. Telus in metus vulputate eu scelerisque felis. Mi tem" + "pus imperdiet nulla malesuada pellentesque. Tristique magna sit."; + // Expected output state for hashing the i*64 first input bytes above (excluding SHA256 padding). + static const uint32_t result[9][8] = { + {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul}, + {0x91f8ec6bul, 0x4da10fe3ul, 0x1c9c292cul, 0x45e18185ul, 0x435cc111ul, 0x3ca26f09ul, 0xeb954caeul, 0x402a7069ul}, + {0xcabea5acul, 0x374fb97cul, 0x182ad996ul, 0x7bd69cbful, 0x450ff900ul, 0xc1d2be8aul, 0x6a41d505ul, 0xe6212dc3ul}, + {0xbcff09d6ul, 0x3e76f36eul, 0x3ecb2501ul, 0x78866e97ul, 0xe1c1e2fdul, 0x32f4eafful, 0x8aa6c4e5ul, 0xdfc024bcul}, + {0xa08c5d94ul, 0x0a862f93ul, 0x6b7f2f40ul, 0x8f9fae76ul, 0x6d40439ful, 0x79dcee0cul, 0x3e39ff3aul, 0xdc3bdbb1ul}, + {0x216a0895ul, 0x9f1a3662ul, 0xe99946f9ul, 0x87ba4364ul, 0x0fb5db2cul, 0x12bed3d3ul, 0x6689c0c7ul, 0x292f1b04ul}, + {0xca3067f8ul, 0xbc8c2656ul, 0x37cb7e0dul, 0x9b6b8b0ful, 0x46dc380bul, 0xf1287f57ul, 0xc42e4b23ul, 0x3fefe94dul}, + {0x3e4c4039ul, 0xbb6fca8cul, 0x6f27d2f7ul, 0x301e44a4ul, 0x8352ba14ul, 0x5769ce37ul, 0x48a1155ful, 0xc0e1c4c6ul}, + {0xfe2fa9ddul, 0x69d0862bul, 0x1ae0db23ul, 0x471f9244ul, 0xf55c0145ul, 0xc30f9c3bul, 0x40a84ea0ul, 0x5b8a266cul}, + }; + // Test Transform() for 0 through 8 transformations. + for (size_t i = 0; i <= 8; ++i) { + uint32_t state[8]; + std::copy(init, init + 8, state); + Transform(state, data + 1, i); + if (!std::equal(state, state + 8, result[i])) return false; + } + + return true; +} +} // namespace + +std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implementation) +{ + std::string ret = "standard"; + Transform = sha256::Transform; + +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) + if (use_implementation & sha256_implementation::USE_SHANI) { + // Intel SHA extensions: CPUID.(EAX=7,ECX=0):EBX[bit 29]. Portable across + // GCC and Clang (unlike __builtin_cpu_supports("sha")). + unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; + if (__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx) && (ebx & (1u << 29))) { + Transform = sha256_x86_shani::Transform; + ret = "x86_shani"; + } + } +#endif + +#if defined(__aarch64__) + if ((use_implementation & sha256_implementation::USE_SHANI) && (getauxval(AT_HWCAP) & HWCAP_SHA2)) { + Transform = sha256_arm_shani::Transform; + ret = "arm_shani"; + } +#endif + + assert(SelfTest()); + return ret; +} + +////// SHA-256 + +CSHA256::CSHA256() +{ + sha256::Initialize(s); +} + +CSHA256& CSHA256::Write(const unsigned char* data, size_t len) +{ + const unsigned char* end = data + len; + size_t bufsize = bytes % 64; + if (bufsize && bufsize + len >= 64) { + // Fill the buffer, and process it. + memcpy(buf + bufsize, data, 64 - bufsize); + bytes += 64 - bufsize; + data += 64 - bufsize; + Transform(s, buf, 1); + bufsize = 0; + } + if (end - data >= 64) { + size_t blocks = (end - data) / 64; + Transform(s, data, blocks); + data += 64 * blocks; + bytes += 64 * blocks; + } + if (end > data) { + // Fill the buffer with what remains. + memcpy(buf + bufsize, data, end - data); + bytes += end - data; + } + return *this; +} + +void CSHA256::Finalize(unsigned char hash[OUTPUT_SIZE]) +{ + static const unsigned char pad[64] = {0x80}; + unsigned char sizedesc[8]; + WriteBE64(sizedesc, bytes << 3); + Write(pad, 1 + ((119 - (bytes % 64)) % 64)); + Write(sizedesc, 8); + WriteBE32(hash, s[0]); + WriteBE32(hash + 4, s[1]); + WriteBE32(hash + 8, s[2]); + WriteBE32(hash + 12, s[3]); + WriteBE32(hash + 16, s[4]); + WriteBE32(hash + 20, s[5]); + WriteBE32(hash + 24, s[6]); + WriteBE32(hash + 28, s[7]); +} + +CSHA256& CSHA256::Reset() +{ + bytes = 0; + sha256::Initialize(s); + return *this; +} diff --git a/cpp/tensorrt_llm/common/sha256/sha256.h b/cpp/tensorrt_llm/common/sha256/sha256.h new file mode 100644 index 000000000000..ff89b71459fc --- /dev/null +++ b/cpp/tensorrt_llm/common/sha256/sha256.h @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2014-present The Bitcoin Core developers +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// +// Modifications for TensorRT-LLM: +// removed the SHA256D64 double-hash declaration (unused). + +#ifndef BITCOIN_CRYPTO_SHA256_H +#define BITCOIN_CRYPTO_SHA256_H + +#include <cstdint> +#include <cstdlib> +#include <string> + +/** A hasher class for SHA-256. */ +class CSHA256 +{ +private: + uint32_t s[8]; + unsigned char buf[64]; + uint64_t bytes{0}; + +public: + static const size_t OUTPUT_SIZE = 32; + + CSHA256(); + CSHA256& Write(const unsigned char* data, size_t len); + void Finalize(unsigned char hash[OUTPUT_SIZE]); + CSHA256& Reset(); +}; + +namespace sha256_implementation { +enum UseImplementation : uint8_t { + STANDARD = 0, + USE_SSE4 = 1 << 0, + USE_AVX2 = 1 << 1, + USE_SHANI = 1 << 2, + USE_SSE4_AND_AVX2 = USE_SSE4 | USE_AVX2, + USE_SSE4_AND_SHANI = USE_SSE4 | USE_SHANI, + USE_ALL = USE_SSE4 | USE_AVX2 | USE_SHANI, +}; +} + +/** Autodetect the best available SHA256 implementation. + * Returns the name of the implementation. + */ +std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implementation = sha256_implementation::USE_ALL); + +#endif // BITCOIN_CRYPTO_SHA256_H diff --git a/cpp/tensorrt_llm/common/sha256/sha256_arm_shani.cpp b/cpp/tensorrt_llm/common/sha256/sha256_arm_shani.cpp new file mode 100644 index 000000000000..228ea3908e73 --- /dev/null +++ b/cpp/tensorrt_llm/common/sha256/sha256_arm_shani.cpp @@ -0,0 +1,916 @@ +// SPDX-FileCopyrightText: Copyright (c) 2022-present The Bitcoin Core developers +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// +// Based on https://github.com/noloader/SHA-Intrinsics/blob/master/sha256-arm.c, +// Written and placed in public domain by Jeffrey Walton. +// Based on code from ARM, and by Johannes Schneiders, Skip Hovsmith and +// Barry O'Rourke for the mbedTLS project. +// Variant specialized for 64-byte inputs added by Pieter Wuille. + +#if defined(__aarch64__) + +#include <array> +#include <cstdint> +#include <cstddef> +#include <arm_neon.h> + +namespace { +alignas(uint32x4_t) static constexpr std::array<uint32_t, 64> K = +{ + 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5, + 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5, + 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3, + 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174, + 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC, + 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA, + 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7, + 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967, + 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13, + 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85, + 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3, + 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070, + 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5, + 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3, + 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208, + 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2, +}; +} + +namespace sha256_arm_shani { +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) +{ + uint32x4_t STATE0, STATE1, ABCD_SAVE, EFGH_SAVE; + uint32x4_t MSG0, MSG1, MSG2, MSG3; + uint32x4_t TMP0, TMP2; + + // Load state + STATE0 = vld1q_u32(&s[0]); + STATE1 = vld1q_u32(&s[4]); + + while (blocks--) + { + // Save state + ABCD_SAVE = STATE0; + EFGH_SAVE = STATE1; + + // Load and convert input chunk to Big Endian + MSG0 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 0))); + MSG1 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 16))); + MSG2 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 32))); + MSG3 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 48))); + chunk += 64; + + // Original implementation preloaded message and constant addition which was 1-3% slower. + // Now included as first step in quad round code saving one Q Neon register + // "TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[0]));" + + // Rounds 1-4 + TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[0])); + TMP2 = STATE0; + MSG0 = vsha256su0q_u32(MSG0, MSG1); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG0 = vsha256su1q_u32(MSG0, MSG2, MSG3); + + // Rounds 5-8 + TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[4])); + TMP2 = STATE0; + MSG1 = vsha256su0q_u32(MSG1, MSG2); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG1 = vsha256su1q_u32(MSG1, MSG3, MSG0); + + // Rounds 9-12 + TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[8])); + TMP2 = STATE0; + MSG2 = vsha256su0q_u32(MSG2, MSG3); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG2 = vsha256su1q_u32(MSG2, MSG0, MSG1); + + // Rounds 13-16 + TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[12])); + TMP2 = STATE0; + MSG3 = vsha256su0q_u32(MSG3, MSG0); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG3 = vsha256su1q_u32(MSG3, MSG1, MSG2); + + // Rounds 17-20 + TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[16])); + TMP2 = STATE0; + MSG0 = vsha256su0q_u32(MSG0, MSG1); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG0 = vsha256su1q_u32(MSG0, MSG2, MSG3); + + // Rounds 21-24 + TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[20])); + TMP2 = STATE0; + MSG1 = vsha256su0q_u32(MSG1, MSG2); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG1 = vsha256su1q_u32(MSG1, MSG3, MSG0); + + // Rounds 25-28 + TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[24])); + TMP2 = STATE0; + MSG2 = vsha256su0q_u32(MSG2, MSG3); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG2 = vsha256su1q_u32(MSG2, MSG0, MSG1); + + // Rounds 29-32 + TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[28])); + TMP2 = STATE0; + MSG3 = vsha256su0q_u32(MSG3, MSG0); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG3 = vsha256su1q_u32(MSG3, MSG1, MSG2); + + // Rounds 33-36 + TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[32])); + TMP2 = STATE0; + MSG0 = vsha256su0q_u32(MSG0, MSG1); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG0 = vsha256su1q_u32(MSG0, MSG2, MSG3); + + // Rounds 37-40 + TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[36])); + TMP2 = STATE0; + MSG1 = vsha256su0q_u32(MSG1, MSG2); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG1 = vsha256su1q_u32(MSG1, MSG3, MSG0); + + // Rounds 41-44 + TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[40])); + TMP2 = STATE0; + MSG2 = vsha256su0q_u32(MSG2, MSG3); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG2 = vsha256su1q_u32(MSG2, MSG0, MSG1); + + // Rounds 45-48 + TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[44])); + TMP2 = STATE0; + MSG3 = vsha256su0q_u32(MSG3, MSG0); + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + MSG3 = vsha256su1q_u32(MSG3, MSG1, MSG2); + + // Rounds 49-52 + TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[48])); + TMP2 = STATE0; + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + + // Rounds 53-56 + TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[52])); + TMP2 = STATE0; + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + + // Rounds 57-60 + TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[56])); + TMP2 = STATE0; + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + + // Rounds 61-64 + TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[60])); + TMP2 = STATE0; + STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0); + STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0); + + // Update state + STATE0 = vaddq_u32(STATE0, ABCD_SAVE); + STATE1 = vaddq_u32(STATE1, EFGH_SAVE); + } + + // Save final state + vst1q_u32(&s[0], STATE0); + vst1q_u32(&s[4], STATE1); +} +} + +namespace sha256d64_arm_shani { +void Transform_2way(unsigned char* output, const unsigned char* input) +{ + /* Initial state. */ + alignas(uint32x4_t) static constexpr std::array<uint32_t, 8> INIT = { + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 + }; + + /* Precomputed message schedule for the 2nd transform. */ + alignas(uint32x4_t) static constexpr std::array<uint32_t, 64> MIDS = { + 0xc28a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf374, + 0x649b69c1, 0xf0fe4786, 0x0fe1edc6, 0x240cf254, + 0x4fe9346f, 0x6cc984be, 0x61b9411e, 0x16f988fa, + 0xf2c65152, 0xa88e5a6d, 0xb019fc65, 0xb9d99ec7, + 0x9a1231c3, 0xe70eeaa0, 0xfdb1232b, 0xc7353eb0, + 0x3069bad5, 0xcb976d5f, 0x5a0f118f, 0xdc1eeefd, + 0x0a35b689, 0xde0b7a04, 0x58f4ca9d, 0xe15d5b16, + 0x007f3e86, 0x37088980, 0xa507ea32, 0x6fab9537, + 0x17406110, 0x0d8cd6f1, 0xcdaa3b6d, 0xc0bbbe37, + 0x83613bda, 0xdb48a363, 0x0b02e931, 0x6fd15ca7, + 0x521afaca, 0x31338431, 0x6ed41a95, 0x6d437890, + 0xc39c91f2, 0x9eccabbd, 0xb5c9a0e6, 0x532fb63c, + 0xd2c741c6, 0x07237ea3, 0xa4954b68, 0x4c191d76 + }; + + /* A few precomputed message schedule values for the 3rd transform. */ + alignas(uint32x4_t) static constexpr std::array<uint32_t, 12> FINS = { + 0x5807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x80000000, 0x00000000, 0x00000000, 0x00000000, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf274 + }; + + /* Padding processed in the 3rd transform (byteswapped). */ + alignas(uint32x4_t) static constexpr std::array<uint32_t, 8> FINAL = {0x80000000, 0, 0, 0, 0, 0, 0, 0x100}; + + uint32x4_t STATE0A, STATE0B, STATE1A, STATE1B, ABCD_SAVEA, ABCD_SAVEB, EFGH_SAVEA, EFGH_SAVEB; + uint32x4_t MSG0A, MSG0B, MSG1A, MSG1B, MSG2A, MSG2B, MSG3A, MSG3B; + uint32x4_t TMP0A, TMP0B, TMP2A, TMP2B, TMP; + + // Transform 1: Load state + STATE0A = vld1q_u32(&INIT[0]); + STATE0B = STATE0A; + STATE1A = vld1q_u32(&INIT[4]); + STATE1B = STATE1A; + + // Transform 1: Load and convert input chunk to Big Endian + MSG0A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 0))); + MSG1A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 16))); + MSG2A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 32))); + MSG3A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 48))); + MSG0B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 64))); + MSG1B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 80))); + MSG2B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 96))); + MSG3B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 112))); + + // Transform 1: Rounds 1-4 + TMP = vld1q_u32(&K[0]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 1: Rounds 5-8 + TMP = vld1q_u32(&K[4]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 1: Rounds 9-12 + TMP = vld1q_u32(&K[8]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vsha256su0q_u32(MSG2A, MSG3A); + MSG2B = vsha256su0q_u32(MSG2B, MSG3B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 1: Rounds 13-16 + TMP = vld1q_u32(&K[12]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 1: Rounds 17-20 + TMP = vld1q_u32(&K[16]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 1: Rounds 21-24 + TMP = vld1q_u32(&K[20]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 1: Rounds 25-28 + TMP = vld1q_u32(&K[24]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vsha256su0q_u32(MSG2A, MSG3A); + MSG2B = vsha256su0q_u32(MSG2B, MSG3B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 1: Rounds 29-32 + TMP = vld1q_u32(&K[28]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 1: Rounds 33-36 + TMP = vld1q_u32(&K[32]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 1: Rounds 37-40 + TMP = vld1q_u32(&K[36]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 1: Rounds 41-44 + TMP = vld1q_u32(&K[40]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vsha256su0q_u32(MSG2A, MSG3A); + MSG2B = vsha256su0q_u32(MSG2B, MSG3B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 1: Rounds 45-48 + TMP = vld1q_u32(&K[44]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 1: Rounds 49-52 + TMP = vld1q_u32(&K[48]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 1: Rounds 53-56 + TMP = vld1q_u32(&K[52]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 1: Rounds 57-60 + TMP = vld1q_u32(&K[56]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 1: Rounds 61-64 + TMP = vld1q_u32(&K[60]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 1: Update state + TMP = vld1q_u32(&INIT[0]); + STATE0A = vaddq_u32(STATE0A, TMP); + STATE0B = vaddq_u32(STATE0B, TMP); + TMP = vld1q_u32(&INIT[4]); + STATE1A = vaddq_u32(STATE1A, TMP); + STATE1B = vaddq_u32(STATE1B, TMP); + + // Transform 2: Save state + ABCD_SAVEA = STATE0A; + ABCD_SAVEB = STATE0B; + EFGH_SAVEA = STATE1A; + EFGH_SAVEB = STATE1B; + + // Transform 2: Rounds 1-4 + TMP = vld1q_u32(&MIDS[0]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 5-8 + TMP = vld1q_u32(&MIDS[4]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 9-12 + TMP = vld1q_u32(&MIDS[8]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 13-16 + TMP = vld1q_u32(&MIDS[12]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 17-20 + TMP = vld1q_u32(&MIDS[16]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 21-24 + TMP = vld1q_u32(&MIDS[20]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 25-28 + TMP = vld1q_u32(&MIDS[24]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 29-32 + TMP = vld1q_u32(&MIDS[28]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 33-36 + TMP = vld1q_u32(&MIDS[32]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 37-40 + TMP = vld1q_u32(&MIDS[36]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 41-44 + TMP = vld1q_u32(&MIDS[40]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 45-48 + TMP = vld1q_u32(&MIDS[44]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 49-52 + TMP = vld1q_u32(&MIDS[48]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 53-56 + TMP = vld1q_u32(&MIDS[52]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 57-60 + TMP = vld1q_u32(&MIDS[56]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Rounds 61-64 + TMP = vld1q_u32(&MIDS[60]); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + + // Transform 2: Update state + STATE0A = vaddq_u32(STATE0A, ABCD_SAVEA); + STATE0B = vaddq_u32(STATE0B, ABCD_SAVEB); + STATE1A = vaddq_u32(STATE1A, EFGH_SAVEA); + STATE1B = vaddq_u32(STATE1B, EFGH_SAVEB); + + // Transform 3: Pad previous output + MSG0A = STATE0A; + MSG0B = STATE0B; + MSG1A = STATE1A; + MSG1B = STATE1B; + MSG2A = vld1q_u32(&FINAL[0]); + MSG2B = MSG2A; + MSG3A = vld1q_u32(&FINAL[4]); + MSG3B = MSG3A; + + // Transform 3: Load state + STATE0A = vld1q_u32(&INIT[0]); + STATE0B = STATE0A; + STATE1A = vld1q_u32(&INIT[4]); + STATE1B = STATE1A; + + // Transform 3: Rounds 1-4 + TMP = vld1q_u32(&K[0]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 3: Rounds 5-8 + TMP = vld1q_u32(&K[4]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 3: Rounds 9-12 + TMP = vld1q_u32(&FINS[0]); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vld1q_u32(&FINS[4]); + MSG2B = MSG2A; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 3: Rounds 13-16 + TMP = vld1q_u32(&FINS[8]); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 3: Rounds 17-20 + TMP = vld1q_u32(&K[16]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 3: Rounds 21-24 + TMP = vld1q_u32(&K[20]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 3: Rounds 25-28 + TMP = vld1q_u32(&K[24]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vsha256su0q_u32(MSG2A, MSG3A); + MSG2B = vsha256su0q_u32(MSG2B, MSG3B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 3: Rounds 29-32 + TMP = vld1q_u32(&K[28]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 3: Rounds 33-36 + TMP = vld1q_u32(&K[32]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG0A = vsha256su0q_u32(MSG0A, MSG1A); + MSG0B = vsha256su0q_u32(MSG0B, MSG1B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A); + MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B); + + // Transform 3: Rounds 37-40 + TMP = vld1q_u32(&K[36]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG1A = vsha256su0q_u32(MSG1A, MSG2A); + MSG1B = vsha256su0q_u32(MSG1B, MSG2B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A); + MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B); + + // Transform 3: Rounds 41-44 + TMP = vld1q_u32(&K[40]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG2A = vsha256su0q_u32(MSG2A, MSG3A); + MSG2B = vsha256su0q_u32(MSG2B, MSG3B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A); + MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B); + + // Transform 3: Rounds 45-48 + TMP = vld1q_u32(&K[44]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + MSG3A = vsha256su0q_u32(MSG3A, MSG0A); + MSG3B = vsha256su0q_u32(MSG3B, MSG0B); + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A); + MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B); + + // Transform 3: Rounds 49-52 + TMP = vld1q_u32(&K[48]); + TMP0A = vaddq_u32(MSG0A, TMP); + TMP0B = vaddq_u32(MSG0B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 3: Rounds 53-56 + TMP = vld1q_u32(&K[52]); + TMP0A = vaddq_u32(MSG1A, TMP); + TMP0B = vaddq_u32(MSG1B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 3: Rounds 57-60 + TMP = vld1q_u32(&K[56]); + TMP0A = vaddq_u32(MSG2A, TMP); + TMP0B = vaddq_u32(MSG2B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 3: Rounds 61-64 + TMP = vld1q_u32(&K[60]); + TMP0A = vaddq_u32(MSG3A, TMP); + TMP0B = vaddq_u32(MSG3B, TMP); + TMP2A = STATE0A; + TMP2B = STATE0B; + STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A); + STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B); + STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A); + STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B); + + // Transform 3: Update state + TMP = vld1q_u32(&INIT[0]); + STATE0A = vaddq_u32(STATE0A, TMP); + STATE0B = vaddq_u32(STATE0B, TMP); + TMP = vld1q_u32(&INIT[4]); + STATE1A = vaddq_u32(STATE1A, TMP); + STATE1B = vaddq_u32(STATE1B, TMP); + + // Store result + vst1q_u8(output, vrev32q_u8(vreinterpretq_u8_u32(STATE0A))); + vst1q_u8(output + 16, vrev32q_u8(vreinterpretq_u8_u32(STATE1A))); + vst1q_u8(output + 32, vrev32q_u8(vreinterpretq_u8_u32(STATE0B))); + vst1q_u8(output + 48, vrev32q_u8(vreinterpretq_u8_u32(STATE1B))); +} +} + +#endif diff --git a/cpp/tensorrt_llm/common/sha256/sha256_endian.h b/cpp/tensorrt_llm/common/sha256/sha256_endian.h new file mode 100644 index 000000000000..82af31f00060 --- /dev/null +++ b/cpp/tensorrt_llm/common/sha256/sha256_endian.h @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2014-present The Bitcoin Core developers +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// +// Minimal big-endian read/write helpers for the vendored Bitcoin Core SHA-256 +// (sha256.cpp). This NVIDIA-authored header replaces the upstream +// <crypto/common.h> / <compat/endian.h> / <compat/byteswap.h> dependency chain +// (which required C++20) so the vendored sources build as plain C++17. The +// helper names and byte order match the upstream common.h functions they +// replace, so the SHA-256 output is unchanged. + +#ifndef TENSORRT_LLM_3RDPARTY_SHA256_ENDIAN_H +#define TENSORRT_LLM_3RDPARTY_SHA256_ENDIAN_H + +#include <cstdint> + +inline uint32_t ReadBE32(unsigned char const* ptr) +{ + return (static_cast<uint32_t>(ptr[0]) << 24) | (static_cast<uint32_t>(ptr[1]) << 16) + | (static_cast<uint32_t>(ptr[2]) << 8) | static_cast<uint32_t>(ptr[3]); +} + +inline void WriteBE32(unsigned char* ptr, uint32_t x) +{ + ptr[0] = static_cast<unsigned char>(x >> 24); + ptr[1] = static_cast<unsigned char>(x >> 16); + ptr[2] = static_cast<unsigned char>(x >> 8); + ptr[3] = static_cast<unsigned char>(x); +} + +inline void WriteBE64(unsigned char* ptr, uint64_t x) +{ + ptr[0] = static_cast<unsigned char>(x >> 56); + ptr[1] = static_cast<unsigned char>(x >> 48); + ptr[2] = static_cast<unsigned char>(x >> 40); + ptr[3] = static_cast<unsigned char>(x >> 32); + ptr[4] = static_cast<unsigned char>(x >> 24); + ptr[5] = static_cast<unsigned char>(x >> 16); + ptr[6] = static_cast<unsigned char>(x >> 8); + ptr[7] = static_cast<unsigned char>(x); +} + +#endif // TENSORRT_LLM_3RDPARTY_SHA256_ENDIAN_H diff --git a/cpp/tensorrt_llm/common/sha256/sha256_x86_shani.cpp b/cpp/tensorrt_llm/common/sha256/sha256_x86_shani.cpp new file mode 100644 index 000000000000..5f3477c692f4 --- /dev/null +++ b/cpp/tensorrt_llm/common/sha256/sha256_x86_shani.cpp @@ -0,0 +1,377 @@ +// SPDX-FileCopyrightText: Copyright (c) 2018-present The Bitcoin Core developers +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// +// Based on https://github.com/noloader/SHA-Intrinsics/blob/master/sha256-x86.c, +// Written and placed in public domain by Jeffrey Walton. +// Based on code from Intel, and by Sean Gulley for the miTLS project. + +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) + +#include <cstddef> +#include <cstdint> +#include <immintrin.h> + +#include <attributes.h> + +namespace { + +alignas(__m128i) const uint8_t MASK[16] = {0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04, 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c}; +alignas(__m128i) const uint8_t INIT0[16] = {0x8c, 0x68, 0x05, 0x9b, 0x7f, 0x52, 0x0e, 0x51, 0x85, 0xae, 0x67, 0xbb, 0x67, 0xe6, 0x09, 0x6a}; +alignas(__m128i) const uint8_t INIT1[16] = {0x19, 0xcd, 0xe0, 0x5b, 0xab, 0xd9, 0x83, 0x1f, 0x3a, 0xf5, 0x4f, 0xa5, 0x72, 0xf3, 0x6e, 0x3c}; + +void ALWAYS_INLINE QuadRound(__m128i& state0, __m128i& state1, uint64_t k1, uint64_t k0) +{ + const __m128i msg = _mm_set_epi64x(k1, k0); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e)); +} + +void ALWAYS_INLINE QuadRound(__m128i& state0, __m128i& state1, __m128i m, uint64_t k1, uint64_t k0) +{ + const __m128i msg = _mm_add_epi32(m, _mm_set_epi64x(k1, k0)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e)); +} + +void ALWAYS_INLINE ShiftMessageA(__m128i& m0, __m128i m1) +{ + m0 = _mm_sha256msg1_epu32(m0, m1); +} + +void ALWAYS_INLINE ShiftMessageC(__m128i& m0, __m128i m1, __m128i& m2) +{ + m2 = _mm_sha256msg2_epu32(_mm_add_epi32(m2, _mm_alignr_epi8(m1, m0, 4)), m1); +} + +void ALWAYS_INLINE ShiftMessageB(__m128i& m0, __m128i m1, __m128i& m2) +{ + ShiftMessageC(m0, m1, m2); + ShiftMessageA(m0, m1); +} + +void ALWAYS_INLINE Shuffle(__m128i& s0, __m128i& s1) +{ + const __m128i t1 = _mm_shuffle_epi32(s0, 0xB1); + const __m128i t2 = _mm_shuffle_epi32(s1, 0x1B); + s0 = _mm_alignr_epi8(t1, t2, 0x08); + s1 = _mm_blend_epi16(t2, t1, 0xF0); +} + +void ALWAYS_INLINE Unshuffle(__m128i& s0, __m128i& s1) +{ + const __m128i t1 = _mm_shuffle_epi32(s0, 0x1B); + const __m128i t2 = _mm_shuffle_epi32(s1, 0xB1); + s0 = _mm_blend_epi16(t1, t2, 0xF0); + s1 = _mm_alignr_epi8(t2, t1, 0x08); +} + +__m128i ALWAYS_INLINE Load(const unsigned char* in) +{ + return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)in), _mm_load_si128((const __m128i*)MASK)); +} + +void ALWAYS_INLINE Save(unsigned char* out, __m128i s) +{ + _mm_storeu_si128((__m128i*)out, _mm_shuffle_epi8(s, _mm_load_si128((const __m128i*)MASK))); +} +} + +namespace sha256_x86_shani { +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) +{ + __m128i m0, m1, m2, m3, s0, s1, so0, so1; + + /* Load state */ + s0 = _mm_loadu_si128((const __m128i*)s); + s1 = _mm_loadu_si128((const __m128i*)(s + 4)); + Shuffle(s0, s1); + + while (blocks--) { + /* Remember old state */ + so0 = s0; + so1 = s1; + + /* Load data and transform */ + m0 = Load(chunk); + QuadRound(s0, s1, m0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull); + m1 = Load(chunk + 16); + QuadRound(s0, s1, m1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); + ShiftMessageA(m0, m1); + m2 = Load(chunk + 32); + QuadRound(s0, s1, m2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull); + ShiftMessageA(m1, m2); + m3 = Load(chunk + 48); + QuadRound(s0, s1, m3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull); + ShiftMessageB(m2, m3, m0); + QuadRound(s0, s1, m0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull); + ShiftMessageB(m3, m0, m1); + QuadRound(s0, s1, m1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full); + ShiftMessageB(m0, m1, m2); + QuadRound(s0, s1, m2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull); + ShiftMessageB(m1, m2, m3); + QuadRound(s0, s1, m3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull); + ShiftMessageB(m2, m3, m0); + QuadRound(s0, s1, m0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull); + ShiftMessageB(m3, m0, m1); + QuadRound(s0, s1, m1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull); + ShiftMessageB(m0, m1, m2); + QuadRound(s0, s1, m2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull); + ShiftMessageB(m1, m2, m3); + QuadRound(s0, s1, m3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull); + ShiftMessageB(m2, m3, m0); + QuadRound(s0, s1, m0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull); + ShiftMessageB(m3, m0, m1); + QuadRound(s0, s1, m1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull); + ShiftMessageC(m0, m1, m2); + QuadRound(s0, s1, m2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); + ShiftMessageC(m1, m2, m3); + QuadRound(s0, s1, m3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); + + /* Combine with old state */ + s0 = _mm_add_epi32(s0, so0); + s1 = _mm_add_epi32(s1, so1); + + /* Advance */ + chunk += 64; + } + + Unshuffle(s0, s1); + _mm_storeu_si128((__m128i*)s, s0); + _mm_storeu_si128((__m128i*)(s + 4), s1); +} +} + +namespace sha256d64_x86_shani { + +void Transform_2way(unsigned char* out, const unsigned char* in) +{ + __m128i am0, am1, am2, am3, as0, as1, aso0, aso1; + __m128i bm0, bm1, bm2, bm3, bs0, bs1, bso0, bso1; + + /* Transform 1 */ + bs0 = as0 = _mm_load_si128((const __m128i*)INIT0); + bs1 = as1 = _mm_load_si128((const __m128i*)INIT1); + am0 = Load(in); + bm0 = Load(in + 64); + QuadRound(as0, as1, am0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull); + QuadRound(bs0, bs1, bm0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull); + am1 = Load(in + 16); + bm1 = Load(in + 80); + QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); + QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); + ShiftMessageA(am0, am1); + ShiftMessageA(bm0, bm1); + am2 = Load(in + 32); + bm2 = Load(in + 96); + QuadRound(as0, as1, am2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull); + QuadRound(bs0, bs1, bm2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull); + ShiftMessageA(am1, am2); + ShiftMessageA(bm1, bm2); + am3 = Load(in + 48); + bm3 = Load(in + 112); + QuadRound(as0, as1, am3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull); + QuadRound(bs0, bs1, bm3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull); + ShiftMessageB(am2, am3, am0); + ShiftMessageB(bm2, bm3, bm0); + QuadRound(as0, as1, am0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull); + QuadRound(bs0, bs1, bm0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull); + ShiftMessageB(am3, am0, am1); + ShiftMessageB(bm3, bm0, bm1); + QuadRound(as0, as1, am1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full); + QuadRound(bs0, bs1, bm1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full); + ShiftMessageB(am0, am1, am2); + ShiftMessageB(bm0, bm1, bm2); + QuadRound(as0, as1, am2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull); + QuadRound(bs0, bs1, bm2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull); + ShiftMessageB(am1, am2, am3); + ShiftMessageB(bm1, bm2, bm3); + QuadRound(as0, as1, am3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull); + QuadRound(bs0, bs1, bm3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull); + ShiftMessageB(am2, am3, am0); + ShiftMessageB(bm2, bm3, bm0); + QuadRound(as0, as1, am0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull); + QuadRound(bs0, bs1, bm0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull); + ShiftMessageB(am3, am0, am1); + ShiftMessageB(bm3, bm0, bm1); + QuadRound(as0, as1, am1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull); + QuadRound(bs0, bs1, bm1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull); + ShiftMessageB(am0, am1, am2); + ShiftMessageB(bm0, bm1, bm2); + QuadRound(as0, as1, am2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull); + QuadRound(bs0, bs1, bm2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull); + ShiftMessageB(am1, am2, am3); + ShiftMessageB(bm1, bm2, bm3); + QuadRound(as0, as1, am3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull); + QuadRound(bs0, bs1, bm3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull); + ShiftMessageB(am2, am3, am0); + ShiftMessageB(bm2, bm3, bm0); + QuadRound(as0, as1, am0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull); + QuadRound(bs0, bs1, bm0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull); + ShiftMessageB(am3, am0, am1); + ShiftMessageB(bm3, bm0, bm1); + QuadRound(as0, as1, am1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull); + QuadRound(bs0, bs1, bm1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull); + ShiftMessageC(am0, am1, am2); + ShiftMessageC(bm0, bm1, bm2); + QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); + QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); + ShiftMessageC(am1, am2, am3); + ShiftMessageC(bm1, bm2, bm3); + QuadRound(as0, as1, am3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); + QuadRound(bs0, bs1, bm3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); + as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0)); + bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0)); + as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1)); + bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1)); + + /* Transform 2 */ + aso0 = as0; + bso0 = bs0; + aso1 = as1; + bso1 = bs1; + QuadRound(as0, as1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull); + QuadRound(bs0, bs1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull); + QuadRound(as0, as1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); + QuadRound(bs0, bs1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); + QuadRound(as0, as1, 0x550c7dc3243185beull, 0x12835b01d807aa98ull); + QuadRound(bs0, bs1, 0x550c7dc3243185beull, 0x12835b01d807aa98ull); + QuadRound(as0, as1, 0xc19bf3749bdc06a7ull, 0x80deb1fe72be5d74ull); + QuadRound(bs0, bs1, 0xc19bf3749bdc06a7ull, 0x80deb1fe72be5d74ull); + QuadRound(as0, as1, 0x240cf2540fe1edc6ull, 0xf0fe4786649b69c1ull); + QuadRound(bs0, bs1, 0x240cf2540fe1edc6ull, 0xf0fe4786649b69c1ull); + QuadRound(as0, as1, 0x16f988fa61b9411eull, 0x6cc984be4fe9346full); + QuadRound(bs0, bs1, 0x16f988fa61b9411eull, 0x6cc984be4fe9346full); + QuadRound(as0, as1, 0xb9d99ec7b019fc65ull, 0xa88e5a6df2c65152ull); + QuadRound(bs0, bs1, 0xb9d99ec7b019fc65ull, 0xa88e5a6df2c65152ull); + QuadRound(as0, as1, 0xc7353eb0fdb1232bull, 0xe70eeaa09a1231c3ull); + QuadRound(bs0, bs1, 0xc7353eb0fdb1232bull, 0xe70eeaa09a1231c3ull); + QuadRound(as0, as1, 0xdc1eeefd5a0f118full, 0xcb976d5f3069bad5ull); + QuadRound(bs0, bs1, 0xdc1eeefd5a0f118full, 0xcb976d5f3069bad5ull); + QuadRound(as0, as1, 0xe15d5b1658f4ca9dull, 0xde0b7a040a35b689ull); + QuadRound(bs0, bs1, 0xe15d5b1658f4ca9dull, 0xde0b7a040a35b689ull); + QuadRound(as0, as1, 0x6fab9537a507ea32ull, 0x37088980007f3e86ull); + QuadRound(bs0, bs1, 0x6fab9537a507ea32ull, 0x37088980007f3e86ull); + QuadRound(as0, as1, 0xc0bbbe37cdaa3b6dull, 0x0d8cd6f117406110ull); + QuadRound(bs0, bs1, 0xc0bbbe37cdaa3b6dull, 0x0d8cd6f117406110ull); + QuadRound(as0, as1, 0x6fd15ca70b02e931ull, 0xdb48a36383613bdaull); + QuadRound(bs0, bs1, 0x6fd15ca70b02e931ull, 0xdb48a36383613bdaull); + QuadRound(as0, as1, 0x6d4378906ed41a95ull, 0x31338431521afacaull); + QuadRound(bs0, bs1, 0x6d4378906ed41a95ull, 0x31338431521afacaull); + QuadRound(as0, as1, 0x532fb63cb5c9a0e6ull, 0x9eccabbdc39c91f2ull); + QuadRound(bs0, bs1, 0x532fb63cb5c9a0e6ull, 0x9eccabbdc39c91f2ull); + QuadRound(as0, as1, 0x4c191d76a4954b68ull, 0x07237ea3d2c741c6ull); + QuadRound(bs0, bs1, 0x4c191d76a4954b68ull, 0x07237ea3d2c741c6ull); + as0 = _mm_add_epi32(as0, aso0); + bs0 = _mm_add_epi32(bs0, bso0); + as1 = _mm_add_epi32(as1, aso1); + bs1 = _mm_add_epi32(bs1, bso1); + + /* Extract hash */ + Unshuffle(as0, as1); + Unshuffle(bs0, bs1); + am0 = as0; + bm0 = bs0; + am1 = as1; + bm1 = bs1; + + /* Transform 3 */ + bs0 = as0 = _mm_load_si128((const __m128i*)INIT0); + bs1 = as1 = _mm_load_si128((const __m128i*)INIT1); + QuadRound(as0, as1, am0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull); + QuadRound(bs0, bs1, bm0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull); + QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); + QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); + ShiftMessageA(am0, am1); + ShiftMessageA(bm0, bm1); + bm2 = am2 = _mm_set_epi64x(0x0ull, 0x80000000ull); + QuadRound(as0, as1, 0x550c7dc3243185beull, 0x12835b015807aa98ull); + QuadRound(bs0, bs1, 0x550c7dc3243185beull, 0x12835b015807aa98ull); + ShiftMessageA(am1, am2); + ShiftMessageA(bm1, bm2); + bm3 = am3 = _mm_set_epi64x(0x10000000000ull, 0x0ull); + QuadRound(as0, as1, 0xc19bf2749bdc06a7ull, 0x80deb1fe72be5d74ull); + QuadRound(bs0, bs1, 0xc19bf2749bdc06a7ull, 0x80deb1fe72be5d74ull); + ShiftMessageB(am2, am3, am0); + ShiftMessageB(bm2, bm3, bm0); + QuadRound(as0, as1, am0, 0x240ca1cc0fc19dc6ull, 0xefbe4786e49b69c1ull); + QuadRound(bs0, bs1, bm0, 0x240ca1cc0fc19dc6ull, 0xefbe4786e49b69c1ull); + ShiftMessageB(am3, am0, am1); + ShiftMessageB(bm3, bm0, bm1); + QuadRound(as0, as1, am1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full); + QuadRound(bs0, bs1, bm1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full); + ShiftMessageB(am0, am1, am2); + ShiftMessageB(bm0, bm1, bm2); + QuadRound(as0, as1, am2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull); + QuadRound(bs0, bs1, bm2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull); + ShiftMessageB(am1, am2, am3); + ShiftMessageB(bm1, bm2, bm3); + QuadRound(as0, as1, am3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull); + QuadRound(bs0, bs1, bm3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull); + ShiftMessageB(am2, am3, am0); + ShiftMessageB(bm2, bm3, bm0); + QuadRound(as0, as1, am0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull); + QuadRound(bs0, bs1, bm0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull); + ShiftMessageB(am3, am0, am1); + ShiftMessageB(bm3, bm0, bm1); + QuadRound(as0, as1, am1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull); + QuadRound(bs0, bs1, bm1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull); + ShiftMessageB(am0, am1, am2); + ShiftMessageB(bm0, bm1, bm2); + QuadRound(as0, as1, am2, 0xc76c51a3c24b8b70ull, 0xa81a664ba2bfe8A1ull); + QuadRound(bs0, bs1, bm2, 0xc76c51a3c24b8b70ull, 0xa81a664ba2bfe8A1ull); + ShiftMessageB(am1, am2, am3); + ShiftMessageB(bm1, bm2, bm3); + QuadRound(as0, as1, am3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull); + QuadRound(bs0, bs1, bm3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull); + ShiftMessageB(am2, am3, am0); + ShiftMessageB(bm2, bm3, bm0); + QuadRound(as0, as1, am0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull); + QuadRound(bs0, bs1, bm0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull); + ShiftMessageB(am3, am0, am1); + ShiftMessageB(bm3, bm0, bm1); + QuadRound(as0, as1, am1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull); + QuadRound(bs0, bs1, bm1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull); + ShiftMessageC(am0, am1, am2); + ShiftMessageC(bm0, bm1, bm2); + QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); + QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); + ShiftMessageC(am1, am2, am3); + ShiftMessageC(bm1, bm2, bm3); + QuadRound(as0, as1, am3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull); + QuadRound(bs0, bs1, bm3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull); + as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0)); + bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0)); + as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1)); + bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1)); + + /* Extract hash into out */ + Unshuffle(as0, as1); + Unshuffle(bs0, bs1); + Save(out, as0); + Save(out + 16, as1); + Save(out + 32, bs0); + Save(out + 48, bs1); +} + +} + +#endif diff --git a/cpp/tensorrt_llm/executor/CMakeLists.txt b/cpp/tensorrt_llm/executor/CMakeLists.txt index ca1ab298d7f4..358494e9989c 100644 --- a/cpp/tensorrt_llm/executor/CMakeLists.txt +++ b/cpp/tensorrt_llm/executor/CMakeLists.txt @@ -27,9 +27,7 @@ set(SRCS contextPhaseParams.cpp debugConfig.cpp decodingConfig.cpp - executor.cpp executorConfig.cpp - executorImpl.cpp executorKVCacheEventManager.cpp extendedRuntimePerfKnobConfig.cpp guidedDecodingConfig.cpp @@ -52,7 +50,6 @@ set(SRCS response.cpp samplingConfig.cpp dynamicBatchConfig.cpp - dynamicBatchTuner.cpp schedulerConfig.cpp serialization.cpp speculativeDecodingConfig.cpp @@ -60,7 +57,6 @@ set(SRCS types.cpp requestUtils.cpp contextPhaseParams.cpp - disaggServerUtil.cpp cacheTransceiverConfig.cpp) if(NOT WIN32) diff --git a/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.cu b/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.cu index 6e8c68d7efa7..d57ed7530e28 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.cu +++ b/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.cu @@ -20,6 +20,7 @@ #include "tensorrt_llm/common/cudaFp8Utils.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/tensor.h" #include "tensorrt_llm/executor/types.h" @@ -27,7 +28,6 @@ #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include <NvInferRuntimeBase.h> #include <cstddef> #include <cstdint> #include <sstream> @@ -467,7 +467,7 @@ void concatKVCache(runtime::ITensor::SharedPtr* inputBlocks, int inputBlockNum, blockInfos[outputBlockNum * inputAllRankNum + oi] = fillBlockInfo(oCacheState, outputBlocks[oi], oRank); } runtime::BufferManager::IBufferPtr blockInfosDeviceBuffer - = bufferManager.gpu(sizeof(BlockInfo<T>) * (blockInfos.size()), nvinfer1::DataType::kUINT8); + = bufferManager.gpu(sizeof(BlockInfo<T>) * (blockInfos.size()), tensorrt_llm::DataType::kUINT8); bufferManager.copy((blockInfos.data()), *blockInfosDeviceBuffer, runtime::MemoryType::kCPU); BlockInfo<T>* iBlockInfoDevice = static_cast<BlockInfo<T>*>(blockInfosDeviceBuffer->data()); @@ -594,7 +594,7 @@ void concatKVCacheDispatch(runtime::ITensor::SharedPtr* inputBlocks, int inputBl } } -nvinfer1::Dims makeShapeFromCacheState(kv_cache::CacheState const& cacheState) +tensorrt_llm::Dims makeShapeFromCacheState(kv_cache::CacheState const& cacheState) { int64_t blockSize = static_cast<int64_t>(cacheState.getModelConfig().mNbKvHeadsPerLayer[0] @@ -1130,8 +1130,8 @@ void splitKVCache(std::map<SizeType32, std::vector<runtime::ITensor::SharedPtr>> std::vector<SizeType32> layersInWindow; size_t cacheBlockSizeSum = 0; size_t inputBlockLayerNumSum = 0; - auto cacheDataType - = isIndexerKCache ? nvinfer1::DataType::kUINT8 : kVCacheBlocksPerWindow.begin()->second.front()->getDataType(); + auto cacheDataType = isIndexerKCache ? tensorrt_llm::DataType::kUINT8 + : kVCacheBlocksPerWindow.begin()->second.front()->getDataType(); for (auto const& [window, blocks] : kVCacheBlocksPerWindow) { @@ -1170,7 +1170,7 @@ void splitKVCache(std::map<SizeType32, std::vector<runtime::ITensor::SharedPtr>> bool const isWindow = windowSizes.size() > 1; runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(T*)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); @@ -1182,7 +1182,7 @@ void splitKVCache(std::map<SizeType32, std::vector<runtime::ITensor::SharedPtr>> windowInfoHostBuffer.insert(windowInfoHostBuffer.end(), blockNumInwindow.begin(), blockNumInwindow.end()); windowInfoHostBuffer.insert(windowInfoHostBuffer.end(), layersInWindow.begin(), layersInWindow.end()); - windowInfoDeviceBuffer = bufferManager.gpu(windowInfoHostBuffer.size(), nvinfer1::DataType::kINT32); + windowInfoDeviceBuffer = bufferManager.gpu(windowInfoHostBuffer.size(), tensorrt_llm::DataType::kINT32); bufferManager.copy(windowInfoHostBuffer.data(), *windowInfoDeviceBuffer, runtime::MemoryType::kCPU); for (auto layerNum : layersInWindow) @@ -1404,8 +1404,8 @@ void splitKVCacheDispatch(std::map<SizeType32, std::vector<runtime::ITensor::Sha bool isIndexerKCache) { TLLM_CHECK(!kVCacheBlocksPerWindow.empty()); - auto dataType - = isIndexerKCache ? nvinfer1::DataType::kUINT8 : kVCacheBlocksPerWindow.begin()->second.front()->getDataType(); + auto dataType = isIndexerKCache ? tensorrt_llm::DataType::kUINT8 + : kVCacheBlocksPerWindow.begin()->second.front()->getDataType(); auto dataSize = tensorrt_llm::common::getDTypeSize(dataType); switch (dataSize) @@ -1513,7 +1513,7 @@ void concatKVCache(std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlo } cachePtrs.insert(cachePtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(uint64_t)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); bool const isWindow = windowSizes.size() > 1; @@ -1525,7 +1525,7 @@ void concatKVCache(std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlo windowInfoHostBuffer.insert(windowInfoHostBuffer.end(), blockNumInwindow.begin(), blockNumInwindow.end()); windowInfoHostBuffer.insert(windowInfoHostBuffer.end(), layersInWindow.begin(), layersInWindow.end()); - windowInfoDeviceBuffer = bufferManager.gpu(windowInfoHostBuffer.size(), nvinfer1::DataType::kINT32); + windowInfoDeviceBuffer = bufferManager.gpu(windowInfoHostBuffer.size(), tensorrt_llm::DataType::kINT32); bufferManager.copy(windowInfoHostBuffer.data(), *windowInfoDeviceBuffer, runtime::MemoryType::kCPU); } constexpr int subWarpSize = 8; diff --git a/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h b/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h index c7036b219612..80816bc5c3a0 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h +++ b/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h @@ -27,7 +27,7 @@ #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" -#include <NvInferRuntimeBase.h> +#include "tensorrt_llm/common/tllmDataType.h" namespace tensorrt_llm::executor::kv_cache { @@ -78,7 +78,7 @@ void concatKVCacheDispatch(runtime::ITensor::SharedPtr* inputBlocks, int inputBl runtime::ITensor::SharedPtr* outputBlocks, int outputBlockNum, int selfRank, kv_cache::CacheState const& selfCacheState, runtime::BufferManager const& bufferManager); -nvinfer1::Dims makeShapeFromCacheState(kv_cache::CacheState const& cacheState); +tensorrt_llm::Dims makeShapeFromCacheState(kv_cache::CacheState const& cacheState); void splitKVCacheDispatch(std::map<SizeType32, std::vector<runtime::ITensor::SharedPtr>> const& kVCacheBlocksPerWindow, std::vector<runtime::ITensor::SharedPtr>& ouputSplitBlocks, kv_cache::CacheState const& peerCacheState, @@ -147,7 +147,7 @@ void concatRnnSsmStateDispatch(std::vector<runtime::ITensor::SharedPtr> const& i void splitUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr>& outputSplitBlocks, kv_cache::CacheState const& destCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, - size_t ssmBytes, size_t blockSizeBytes, nvinfer1::DataType ssmDataType, + size_t ssmBytes, size_t blockSizeBytes, tensorrt_llm::DataType ssmDataType, runtime::BufferManager const& bufferManager); /** @@ -156,7 +156,7 @@ void splitUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, void splitUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr>& outputSplitBlocks, kv_cache::CacheState const& destCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, - size_t ssmBytes, size_t blockSizeBytes, nvinfer1::DataType convDataType, + size_t ssmBytes, size_t blockSizeBytes, tensorrt_llm::DataType convDataType, runtime::BufferManager const& bufferManager); /** @@ -165,7 +165,7 @@ void splitUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, void concatUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlocks, kv_cache::CacheState const& srcCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, size_t ssmBytes, - size_t blockSizeBytes, nvinfer1::DataType ssmDataType, runtime::BufferManager const& bufferManager); + size_t blockSizeBytes, tensorrt_llm::DataType ssmDataType, runtime::BufferManager const& bufferManager); /** * @brief Concat conv state from per-source buffers into unified pool blocks (section-aware). @@ -173,6 +173,6 @@ void concatUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, void concatUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlocks, kv_cache::CacheState const& srcCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, size_t ssmBytes, - size_t blockSizeBytes, nvinfer1::DataType convDataType, runtime::BufferManager const& bufferManager); + size_t blockSizeBytes, tensorrt_llm::DataType convDataType, runtime::BufferManager const& bufferManager); } // namespace tensorrt_llm::executor::rnn_cache diff --git a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.cpp b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.cpp index 711750f84f54..5ab589d7ca75 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.cpp @@ -347,164 +347,6 @@ NixlTransferStatus::~NixlTransferStatus() noexcept } } -[[nodiscard]] MemoryDescs NixlHelper::coalesceMemoryDescs(MemoryDescs const& descs) -{ - auto const& descVec = descs.getDescs(); - - // If empty or single element, return as-is - if (descVec.size() <= 1) - { - return descs; - } - - size_t const numDescs = descVec.size(); - - // Create index array and sort by address - std::vector<size_t> sortedIndices(numDescs); - std::iota(sortedIndices.begin(), sortedIndices.end(), 0); - - std::sort(sortedIndices.begin(), sortedIndices.end(), - [&descVec](size_t lhs, size_t rhs) - { - // Sort by deviceId first, then by address - if (descVec[lhs].getDeviceId() != descVec[rhs].getDeviceId()) - { - return descVec[lhs].getDeviceId() < descVec[rhs].getDeviceId(); - } - return descVec[lhs].getAddr() < descVec[rhs].getAddr(); - }); - - std::vector<MemoryDesc> coalesced; - coalesced.reserve(numDescs); - - // Start with the first entry - size_t firstIdx = sortedIndices[0]; - uintptr_t currentAddr = descVec[firstIdx].getAddr(); - size_t currentLen = descVec[firstIdx].getLen(); - uint32_t currentDeviceId = descVec[firstIdx].getDeviceId(); - - for (size_t idx = 1; idx < numDescs; ++idx) - { - size_t sortedIdx = sortedIndices[idx]; - auto const& desc = descVec[sortedIdx]; - - // Check if current can be coalesced with previous - bool isContiguous = (currentAddr + currentLen == desc.getAddr()) && (currentDeviceId == desc.getDeviceId()); - - if (isContiguous) - { - // Coalesce: extend the current region - currentLen += desc.getLen(); - } - else - { - // Cannot coalesce: save the current region and start a new one - coalesced.emplace_back(currentAddr, currentLen, currentDeviceId); - - currentAddr = desc.getAddr(); - currentLen = desc.getLen(); - currentDeviceId = desc.getDeviceId(); - } - } - - // Add the last region - coalesced.emplace_back(currentAddr, currentLen, currentDeviceId); - - TLLM_LOG_DEBUG("NixlHelper::coalesceMemoryDescs: coalesced %zu -> %zu entries", descVec.size(), coalesced.size()); - - return MemoryDescs{descs.getType(), std::move(coalesced)}; -} - -[[nodiscard]] std::pair<MemoryDescs, MemoryDescs> NixlHelper::coalesceTransferDescs( - TransferDescs const& srcDescs, TransferDescs const& dstDescs) -{ - auto const& srcVec = srcDescs.getDescs(); - auto const& dstVec = dstDescs.getDescs(); - - // If sizes don't match or empty, return as-is - if (srcVec.size() != dstVec.size() || srcVec.empty()) - { - return {srcDescs, dstDescs}; - } - - size_t const numDescs = srcVec.size(); - - // Create index array and sort by src address - // This allows us to find contiguous regions even if the original order is scattered - std::vector<size_t> sortedIndices(numDescs); - std::iota(sortedIndices.begin(), sortedIndices.end(), 0); - - std::sort(sortedIndices.begin(), sortedIndices.end(), - [&srcVec](size_t lhs, size_t rhs) - { - // Sort by deviceId first, then by address - if (srcVec[lhs].getDeviceId() != srcVec[rhs].getDeviceId()) - { - return srcVec[lhs].getDeviceId() < srcVec[rhs].getDeviceId(); - } - return srcVec[lhs].getAddr() < srcVec[rhs].getAddr(); - }); - - std::vector<MemoryDesc> coalescedSrc; - std::vector<MemoryDesc> coalescedDst; - coalescedSrc.reserve(numDescs); - coalescedDst.reserve(numDescs); - - // Start with the first entry (using sorted order) - size_t firstIdx = sortedIndices[0]; - uintptr_t currentSrcAddr = srcVec[firstIdx].getAddr(); - size_t currentSrcLen = srcVec[firstIdx].getLen(); - uint32_t currentSrcDeviceId = srcVec[firstIdx].getDeviceId(); - - uintptr_t currentDstAddr = dstVec[firstIdx].getAddr(); - size_t currentDstLen = dstVec[firstIdx].getLen(); - uint32_t currentDstDeviceId = dstVec[firstIdx].getDeviceId(); - - for (size_t idx = 1; idx < numDescs; ++idx) - { - size_t sortedIdx = sortedIndices[idx]; - auto const& src = srcVec[sortedIdx]; - auto const& dst = dstVec[sortedIdx]; - - // Check if current src and dst can be coalesced with previous - bool srcContiguous - = (currentSrcAddr + currentSrcLen == src.getAddr()) && (currentSrcDeviceId == src.getDeviceId()); - bool dstContiguous - = (currentDstAddr + currentDstLen == dst.getAddr()) && (currentDstDeviceId == dst.getDeviceId()); - - if (srcContiguous && dstContiguous) - { - // Coalesce: extend the current region - currentSrcLen += src.getLen(); - currentDstLen += dst.getLen(); - } - else - { - // Cannot coalesce: save the current region and start a new one - coalescedSrc.emplace_back(currentSrcAddr, currentSrcLen, currentSrcDeviceId); - coalescedDst.emplace_back(currentDstAddr, currentDstLen, currentDstDeviceId); - - currentSrcAddr = src.getAddr(); - currentSrcLen = src.getLen(); - currentSrcDeviceId = src.getDeviceId(); - - currentDstAddr = dst.getAddr(); - currentDstLen = dst.getLen(); - currentDstDeviceId = dst.getDeviceId(); - } - } - - // Don't forget to add the last region - coalescedSrc.emplace_back(currentSrcAddr, currentSrcLen, currentSrcDeviceId); - coalescedDst.emplace_back(currentDstAddr, currentDstLen, currentDstDeviceId); - - TLLM_LOG_DEBUG( - "NixlHelper::coalesceTransferDescs: coalesced %zu -> %zu transfer entries", srcVec.size(), coalescedSrc.size()); - - return {MemoryDescs{srcDescs.getType(), std::move(coalescedSrc)}, - MemoryDescs{dstDescs.getType(), std::move(coalescedDst)}}; -} - TransferState NixlTransferStatus::wait(int64_t timeout_ms) const { auto startTime = std::chrono::steady_clock::now(); @@ -693,12 +535,8 @@ void NixlTransferAgent::registerMemory(RegisterDescs const& descs) auto detectedRegionMap = VmmDescSplitter::detectVramRegionMap(descs); mLocalVramRegionInfo.merge(detectedRegionMap); - // Coalesce contiguous memory regions to reduce registration overhead (disabled by default) - // Set TRTLLM_NIXL_ENABLE_COALESCE=1 to enable this optimization - auto coalescedDescs = common::getEnvNixlEnableCoalesce() ? NixlHelper::coalesceMemoryDescs(splitDescs) : splitDescs; - nixl_status_t status; - status = mRawAgent->registerMem(NixlHelper::convertRegDlist(coalescedDescs), &mExtraParams); + status = mRawAgent->registerMem(NixlHelper::convertRegDlist(splitDescs), &mExtraParams); TLLM_CHECK(status == NIXL_SUCCESS); std::string localMD; @@ -713,12 +551,8 @@ void NixlTransferAgent::deregisterMemory(RegisterDescs const& descs) // Split using per-region registry info to match what was registered auto splitDescs = VmmDescSplitter::splitDescsWithRegionMap(descs, mLocalVramRegionInfo); - // Coalesce contiguous memory regions to match what was registered (disabled by default) - // Set TRTLLM_NIXL_ENABLE_COALESCE=1 to enable this optimization - auto coalescedDescs = common::getEnvNixlEnableCoalesce() ? NixlHelper::coalesceMemoryDescs(splitDescs) : splitDescs; - nixl_status_t status; - status = mRawAgent->deregisterMem(NixlHelper::convertRegDlist(coalescedDescs), &mExtraParams); + status = mRawAgent->deregisterMem(NixlHelper::convertRegDlist(splitDescs), &mExtraParams); TLLM_CHECK(status == NIXL_SUCCESS); // Remove entries from registry @@ -743,7 +577,7 @@ void NixlTransferAgent::loadRemoteAgent(std::string const& name, AgentDesc const name == remoteName, "loadRemoteAgent gets error agent name: %s != %s", name.c_str(), remoteName.c_str()); // Store remote VMM region info for chunk boundary calculations in - // VmmDescSplitter::splitTransferDescsWithRegionMaps. Per-agent map because different remote agents may have + // VmmDescSplitter::splitAndCoalesceTransferDescs. Per-agent map because different remote agents may have // overlapping virtual addresses. auto const& regions = agentDesc.getVramRegions(); if (!regions.empty()) @@ -764,14 +598,13 @@ AgentDesc NixlTransferAgent::getLocalAgentDesc() nixl_status_t status = mRawAgent->getLocalMD(nixlBlob); TLLM_CHECK(status == NIXL_SUCCESS); - // Pack local VMM region info so remote agents can compute chunk boundaries. + // Pack ALL local region info (VMM multi-chunk and single-allocation alike) so remote agents can + // compute chunk boundaries and never coalesce transfer descs across separately registered regions. std::vector<VramRegionMeta> regions; + regions.reserve(mLocalVramRegionInfo.size()); for (auto const& [base, info] : mLocalVramRegionInfo) { - if (info.chunkSize > 0) - { - regions.push_back({base, info.totalLen, info.chunkSize}); - } + regions.push_back({base, info.totalLen, info.chunkSize}); } return AgentDesc{nixlBlob, std::move(regions)}; @@ -809,32 +642,25 @@ void NixlTransferAgent::invalidateRemoteAgent(std::string const& name) { reqParams.hasNotif = false; } - // Split transfer descriptors at VMM chunk boundaries to match registered memory. - // Both src and dst are split at chunk boundaries to ensure each descriptor - // falls within a single registered memory region on both local and remote sides. - // Find remote agent's VMM region map (empty map if not found). + // Split transfer descriptors at VMM chunk boundaries to match registered memory, then coalesce + // contiguous pieces. A coalesced descriptor never crosses a chunk boundary or a registered + // region boundary on either side, so every descriptor still falls within a single registered + // memory region on both local and remote sides. Set TRTLLM_NIXL_DISABLE_COALESCE=1 to fall back + // to split-only descriptors. Find remote agent's region map (empty map if not found — e.g. the + // peer's AgentDesc carried no region info; addresses missing from a map are never coalesced, + // so an empty remote map degrades to split-only rather than risking merges across unknown + // remote chunk/registration boundaries). static VramRegionMap const kEmptyMap; auto remoteIt = mRemoteVramRegionInfo.find(request.getRemoteName()); auto const& remoteRegionMap = (remoteIt != mRemoteVramRegionInfo.end()) ? remoteIt->second : kEmptyMap; - auto [splitSrc, splitDst] = VmmDescSplitter::splitTransferDescsWithRegionMaps( - request.getSrcDescs(), request.getDstDescs(), mLocalVramRegionInfo, remoteRegionMap); + auto [xferSrc, xferDst] = VmmDescSplitter::splitAndCoalesceTransferDescs(request.getSrcDescs(), + request.getDstDescs(), mLocalVramRegionInfo, remoteRegionMap, !common::getEnvNixlDisableCoalesce()); - // Coalesce contiguous memory regions to reduce transfer count (disabled by default) - // This matches the coalescing done during registerMemory() - // Set TRTLLM_NIXL_ENABLE_COALESCE=1 to enable this optimization - if (common::getEnvNixlEnableCoalesce()) - { - NVTX3_SCOPED_RANGE(coalesceTransferDescs_CreateXferReq); - auto [coalescedSrc, coalescedDst] = NixlHelper::coalesceTransferDescs(splitSrc, splitDst); - status - = mRawAgent->createXferReq(NixlHelper::convert(request.getOp()), NixlHelper::convertXferDist(coalescedSrc), - NixlHelper::convertXferDist(coalescedDst), request.getRemoteName(), handle, &reqParams); - } - else { - status = mRawAgent->createXferReq(NixlHelper::convert(request.getOp()), NixlHelper::convertXferDist(splitSrc), - NixlHelper::convertXferDist(splitDst), request.getRemoteName(), handle, &reqParams); + NVTX3_SCOPED_RANGE(createXferReq); + status = mRawAgent->createXferReq(NixlHelper::convert(request.getOp()), NixlHelper::convertXferDist(xferSrc), + NixlHelper::convertXferDist(xferDst), request.getRemoteName(), handle, &reqParams); } TLLM_CHECK_WITH_INFO(status == NIXL_SUCCESS, diff --git a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.h b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.h index cb371f02439e..31e62fd6d822 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.h +++ b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.h @@ -39,21 +39,6 @@ struct NixlHelper [[nodiscard]] static nixl_xfer_dlist_t convertXferDist(FileDescs const& descs); static void posixGpuToFileFallback(MemoryDescs const& memoryDesc, FileDescs const& fileDescs); static void posixFileToGpuFallback(MemoryDescs const& memoryDesc, FileDescs const& fileDescs); - - /// @brief Coalesce contiguous memory regions to reduce memory registration overhead. - /// Adjacent memory regions with the same deviceId will be merged into a single region. - /// @param descs Memory descriptors to coalesce - /// @return Coalesced MemoryDescs - [[nodiscard]] static MemoryDescs coalesceMemoryDescs(MemoryDescs const& descs); - - /// @brief Coalesce contiguous memory regions in src and dst to reduce transfer count. - /// If src[i] and src[i+1] are contiguous, and dst[i] and dst[i+1] are also contiguous - /// (with same deviceId), they will be merged into a single transfer. - /// @param srcDescs Source memory descriptors - /// @param dstDescs Destination memory descriptors - /// @return Pair of coalesced (src, dst) MemoryDescs - [[nodiscard]] static std::pair<MemoryDescs, MemoryDescs> coalesceTransferDescs( - TransferDescs const& srcDescs, TransferDescs const& dstDescs); }; class NixlTransferStatus final : public TransferStatus diff --git a/cpp/tensorrt_llm/executor/cache_transmission/rnnCacheSplitConcat.cu b/cpp/tensorrt_llm/executor/cache_transmission/rnnCacheSplitConcat.cu index fb6837e8d188..f464c41aadc9 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/rnnCacheSplitConcat.cu +++ b/cpp/tensorrt_llm/executor/cache_transmission/rnnCacheSplitConcat.cu @@ -24,6 +24,7 @@ #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/tensor.h" #include "tensorrt_llm/executor/types.h" @@ -31,7 +32,6 @@ #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include <NvInferRuntimeBase.h> #include <cstddef> #include <cstdint> #include <sstream> @@ -366,7 +366,7 @@ void splitRnnConvState(std::vector<runtime::ITensor::SharedPtr> const& inputConv // Allocate and copy pointer array to device runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(uint64_t)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); @@ -497,7 +497,7 @@ void splitRnnSsmState(std::vector<runtime::ITensor::SharedPtr> const& inputSsmBl cachePtrs.insert(cachePtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(uint64_t)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); @@ -625,7 +625,7 @@ void concatRnnConvState(std::vector<runtime::ITensor::SharedPtr> const& inputSpl cachePtrs.insert(cachePtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(uint64_t)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); @@ -747,7 +747,7 @@ void concatRnnSsmState(std::vector<runtime::ITensor::SharedPtr> const& inputSpli cachePtrs.insert(cachePtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(uint64_t)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); @@ -1348,7 +1348,7 @@ void splitUnifiedPoolSsm(runtime::ITensor::SharedPtr const& pool, std::vector<Si } allPtrs.insert(allPtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); - auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), nvinfer1::DataType::kINT64); + auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), tensorrt_llm::DataType::kINT64); bufferManager.copy(allPtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); T const** inputPtrsDev = static_cast<T const**>(PtrsDeviceBuffer->data()); @@ -1481,10 +1481,10 @@ void splitUnifiedPoolConv(runtime::ITensor::SharedPtr const& pool, std::vector<S sectionInfo.insert(sectionInfo.end(), sectionDimsDomainTP.begin(), sectionDimsDomainTP.end()); sectionInfo.insert(sectionInfo.end(), sectionOffsetsLocal.begin(), sectionOffsetsLocal.end()); - auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), nvinfer1::DataType::kINT64); + auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), tensorrt_llm::DataType::kINT64); bufferManager.copy(allPtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); - auto sectionInfoBuffer = bufferManager.gpu(sectionInfo.size(), nvinfer1::DataType::kINT32); + auto sectionInfoBuffer = bufferManager.gpu(sectionInfo.size(), tensorrt_llm::DataType::kINT32); bufferManager.copy(sectionInfo.data(), *sectionInfoBuffer, runtime::MemoryType::kCPU); T const** inputPtrsDev = static_cast<T const**>(PtrsDeviceBuffer->data()); @@ -1606,7 +1606,7 @@ void concatUnifiedPoolSsm(runtime::ITensor::SharedPtr const& pool, std::vector<S } allPtrs.insert(allPtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); - auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), nvinfer1::DataType::kINT64); + auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), tensorrt_llm::DataType::kINT64); bufferManager.copy(allPtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); T** outputPtrsDev = static_cast<T**>(PtrsDeviceBuffer->data()); @@ -1731,10 +1731,10 @@ void concatUnifiedPoolConv(runtime::ITensor::SharedPtr const& pool, std::vector< sectionInfo.insert(sectionInfo.end(), sectionDimsDomainTP.begin(), sectionDimsDomainTP.end()); sectionInfo.insert(sectionInfo.end(), sectionOffsetsLocal.begin(), sectionOffsetsLocal.end()); - auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), nvinfer1::DataType::kINT64); + auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), tensorrt_llm::DataType::kINT64); bufferManager.copy(allPtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); - auto sectionInfoBuffer = bufferManager.gpu(sectionInfo.size(), nvinfer1::DataType::kINT32); + auto sectionInfoBuffer = bufferManager.gpu(sectionInfo.size(), tensorrt_llm::DataType::kINT32); bufferManager.copy(sectionInfo.data(), *sectionInfoBuffer, runtime::MemoryType::kCPU); T** outputPtrsDev = static_cast<T**>(PtrsDeviceBuffer->data()); @@ -1810,7 +1810,8 @@ void concatUnifiedPoolConv(runtime::ITensor::SharedPtr const& pool, std::vector< void splitUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr>& outputSplitBlocks, kv_cache::CacheState const& destCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, - size_t ssmBytes, size_t blockSizeBytes, nvinfer1::DataType ssmDataType, runtime::BufferManager const& bufferManager) + size_t ssmBytes, size_t blockSizeBytes, tensorrt_llm::DataType ssmDataType, + runtime::BufferManager const& bufferManager) { auto dataSize = tensorrt_llm::common::getDTypeSize(ssmDataType); switch (dataSize) @@ -1834,7 +1835,7 @@ void splitUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, void splitUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr>& outputSplitBlocks, kv_cache::CacheState const& destCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, - size_t ssmBytes, size_t blockSizeBytes, nvinfer1::DataType convDataType, + size_t ssmBytes, size_t blockSizeBytes, tensorrt_llm::DataType convDataType, runtime::BufferManager const& bufferManager) { auto dataSize = tensorrt_llm::common::getDTypeSize(convDataType); @@ -1859,7 +1860,7 @@ void splitUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, void concatUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlocks, kv_cache::CacheState const& srcCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, size_t ssmBytes, - size_t blockSizeBytes, nvinfer1::DataType ssmDataType, runtime::BufferManager const& bufferManager) + size_t blockSizeBytes, tensorrt_llm::DataType ssmDataType, runtime::BufferManager const& bufferManager) { auto dataSize = tensorrt_llm::common::getDTypeSize(ssmDataType); switch (dataSize) @@ -1883,7 +1884,7 @@ void concatUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, void concatUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlocks, kv_cache::CacheState const& srcCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, size_t ssmBytes, - size_t blockSizeBytes, nvinfer1::DataType convDataType, runtime::BufferManager const& bufferManager) + size_t blockSizeBytes, tensorrt_llm::DataType convDataType, runtime::BufferManager const& bufferManager) { auto dataSize = tensorrt_llm::common::getDTypeSize(convDataType); switch (dataSize) diff --git a/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp b/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp index fd60651dc6a0..9c5aaaac78ae 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,13 +19,44 @@ #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/executor/serializeUtils.h" +#include <algorithm> #include <cuda.h> #include <dlfcn.h> +#include <mutex> +#include <numeric> #include <sstream> +#include <tuple> namespace tensorrt_llm::executor::kv_cache { +void promoteHostLibraryToGlobalScope() +{ + static std::once_flag once; + std::call_once(once, + []() + { + Dl_info info{}; + if (dladdr(reinterpret_cast<void*>(&promoteHostLibraryToGlobalScope), &info) == 0 + || info.dli_fname == nullptr) + { + TLLM_LOG_DEBUG("dladdr failed; skipping global-scope promotion"); + return; + } + // RTLD_NOLOAD promotes the visibility of the already-loaded library without + // loading a second copy. It fails when this code is statically linked into an + // executable (e.g. unit tests) — nothing to promote there. + void* handle = dlopen(info.dli_fname, RTLD_NOW | RTLD_GLOBAL | RTLD_NOLOAD); + if (handle == nullptr) + { + TLLM_LOG_DEBUG("global-scope promotion skipped for %s: %s", info.dli_fname, dlerror()); + return; + } + TLLM_LOG_DEBUG("promoted %s to the global symbol scope", info.dli_fname); + // Keep the extra reference: the library must stay resident for the wrappers. + }); +} + [[nodiscard]] DynLibLoader& DynLibLoader::getInstance() { static DynLibLoader instance; @@ -34,6 +65,7 @@ namespace tensorrt_llm::executor::kv_cache [[nodiscard]] void* DynLibLoader::getHandle(std::string const& name) { + promoteHostLibraryToGlobalScope(); std::lock_guard<std::mutex> lock(mDllMutex); auto it = mHandlers.find(name); if (it != mHandlers.end()) @@ -152,8 +184,9 @@ MemoryDescs VmmDescSplitter::splitDescsWithRegionMap(MemoryDescs const& descs, V return MemoryDescs{descs.getType(), std::move(result)}; } -std::pair<MemoryDescs, MemoryDescs> VmmDescSplitter::splitTransferDescsWithRegionMaps(MemoryDescs const& srcDescs, - MemoryDescs const& dstDescs, VramRegionMap const& localRegionMap, VramRegionMap const& remoteRegionMap) +std::pair<MemoryDescs, MemoryDescs> VmmDescSplitter::splitAndCoalesceTransferDescs(MemoryDescs const& srcDescs, + MemoryDescs const& dstDescs, VramRegionMap const& localRegionMap, VramRegionMap const& remoteRegionMap, + bool enableCoalesce) { if (srcDescs.getType() != MemoryType::kVRAM) return {srcDescs, dstDescs}; @@ -161,54 +194,143 @@ std::pair<MemoryDescs, MemoryDescs> VmmDescSplitter::splitTransferDescsWithRegio auto const& srcVec = srcDescs.getDescs(); auto const& dstVec = dstDescs.getDescs(); TLLM_CHECK(srcVec.size() == dstVec.size()); + if (srcVec.empty()) + return {srcDescs, dstDescs}; - std::vector<MemoryDesc> splitSrc, splitDst; - splitSrc.reserve(srcVec.size()); - splitDst.reserve(dstVec.size()); - - for (size_t i = 0; i < srcVec.size(); ++i) + // Sort pair indices by (src deviceId, src addr) so pairs that are contiguous in memory become + // adjacent, maximizing coalescing regardless of input order. Pairs are independent transfers, + // so reordering is safe. The sort only serves coalescing; with it disabled, keep input order. + std::vector<size_t> order(srcVec.size()); + std::iota(order.begin(), order.end(), 0); + if (enableCoalesce) { - auto [srcChunkSize, srcBase] = lookupChunkInfo(srcVec[i].getAddr(), localRegionMap); - auto [dstChunkSize, dstBase] = lookupChunkInfo(dstVec[i].getAddr(), remoteRegionMap); + std::sort(order.begin(), order.end(), + [&srcVec](size_t lhs, size_t rhs) + { + if (srcVec[lhs].getDeviceId() != srcVec[rhs].getDeviceId()) + { + return srcVec[lhs].getDeviceId() < srcVec[rhs].getDeviceId(); + } + return srcVec[lhs].getAddr() < srcVec[rhs].getAddr(); + }); + } - // If neither side is multi-chunk VMM, no splitting is needed. - if (srcChunkSize == 0 && dstChunkSize == 0) + std::vector<MemoryDesc> outSrc, outDst; + outSrc.reserve(srcVec.size()); + outDst.reserve(dstVec.size()); + + // Region info of the last emitted piece. Invariant: every emitted desc lies within a single + // chunk on both sides, so a merge is legal iff the new piece is contiguous, in the same region, + // and does not start on a chunk boundary (starting on a boundary means the merge would cross it). + size_t prevSrcChunkSize = 0, prevDstChunkSize = 0; + uintptr_t prevSrcBase = 0, prevDstBase = 0; + + auto emitPiece + = [&](uintptr_t srcAddr, uintptr_t dstAddr, size_t len, uint32_t srcDev, uint32_t dstDev, size_t srcChunkSize, + uintptr_t srcBase, size_t dstChunkSize, uintptr_t dstBase, bool regionsKnown) + { + if (enableCoalesce && regionsKnown && !outSrc.empty()) { - splitSrc.push_back(srcVec[i]); - splitDst.push_back(dstVec[i]); - continue; + auto const& lastSrc = outSrc.back(); + auto const& lastDst = outDst.back(); + bool contiguous = lastSrc.getAddr() + lastSrc.getLen() == srcAddr && lastSrc.getDeviceId() == srcDev + && lastDst.getAddr() + lastDst.getLen() == dstAddr && lastDst.getDeviceId() == dstDev; + bool sameSrcRegion = srcChunkSize == prevSrcChunkSize && srcBase == prevSrcBase; + bool sameDstRegion = dstChunkSize == prevDstChunkSize && dstBase == prevDstBase; + bool srcWithinChunk = srcChunkSize == 0 || (srcAddr - srcBase) % srcChunkSize != 0; + bool dstWithinChunk = dstChunkSize == 0 || (dstAddr - dstBase) % dstChunkSize != 0; + if (contiguous && sameSrcRegion && sameDstRegion && srcWithinChunk && dstWithinChunk) + { + outSrc.back() = MemoryDesc{lastSrc.getAddr(), lastSrc.getLen() + len, srcDev}; + outDst.back() = MemoryDesc{lastDst.getAddr(), lastDst.getLen() + len, dstDev}; + return; + } + } + outSrc.emplace_back(srcAddr, len, srcDev); + outDst.emplace_back(dstAddr, len, dstDev); + prevSrcChunkSize = srcChunkSize; + prevSrcBase = srcBase; + prevDstChunkSize = dstChunkSize; + prevDstBase = dstBase; + }; + + // One-entry region cache per side: after sorting, consecutive pairs almost always fall in the + // same region (typically one KV pool), so the O(log R) map lookup is skipped on cache hits. + struct RegionCache + { + uintptr_t base = 0; + size_t totalLen = 0; + size_t chunkSize = 0; + bool valid = false; + }; + + // Returns {chunkSize, regionBase, found}. A miss means the address is not covered by any + // region metadata (e.g. the peer did not send its region info). Two misses both look like + // {0, 0} yet may belong to two distinct regions, so pieces with a missed lookup on either + // side are never merged — a merge could silently cross a chunk or registration boundary. + auto cachedLookup = [](uintptr_t addr, VramRegionMap const& regionMap, RegionCache& cache) + { + if (cache.valid && addr >= cache.base && addr - cache.base < cache.totalLen) + { + return std::tuple<size_t, uintptr_t, bool>{cache.chunkSize, cache.base, true}; + } + auto it = regionMap.upper_bound(addr); + if (it != regionMap.begin()) + { + --it; + if (addr >= it->first && addr - it->first < it->second.totalLen) + { + cache = {it->first, it->second.totalLen, it->second.chunkSize, true}; + return std::tuple<size_t, uintptr_t, bool>{cache.chunkSize, cache.base, true}; + } } + return std::tuple<size_t, uintptr_t, bool>{0, 0, false}; + }; + + RegionCache srcCache, dstCache; + size_t numPieces = 0; + for (size_t idx : order) + { + auto const& src = srcVec[idx]; + auto const& dst = dstVec[idx]; + auto [srcChunkSize, srcBase, srcFound] = cachedLookup(src.getAddr(), localRegionMap, srcCache); + auto [dstChunkSize, dstBase, dstFound] = cachedLookup(dst.getAddr(), remoteRegionMap, dstCache); - uintptr_t srcAddr = srcVec[i].getAddr(); - uintptr_t dstAddr = dstVec[i].getAddr(); - size_t remaining = srcVec[i].getLen(); + uintptr_t srcAddr = src.getAddr(); + uintptr_t dstAddr = dst.getAddr(); + size_t remaining = src.getLen(); while (remaining > 0) { size_t srcPieceSize = remaining; if (srcChunkSize > 0) { - size_t srcOffsetInChunk = static_cast<size_t>((srcAddr - srcBase) % srcChunkSize); - srcPieceSize = srcChunkSize - srcOffsetInChunk; + srcPieceSize = srcChunkSize - static_cast<size_t>((srcAddr - srcBase) % srcChunkSize); } size_t dstPieceSize = remaining; if (dstChunkSize > 0) { - size_t dstOffsetInChunk = static_cast<size_t>((dstAddr - dstBase) % dstChunkSize); - dstPieceSize = dstChunkSize - dstOffsetInChunk; + dstPieceSize = dstChunkSize - static_cast<size_t>((dstAddr - dstBase) % dstChunkSize); } size_t pieceSize = std::min({remaining, srcPieceSize, dstPieceSize}); - splitSrc.emplace_back(srcAddr, pieceSize, srcVec[i].getDeviceId()); - splitDst.emplace_back(dstAddr, pieceSize, dstVec[i].getDeviceId()); + emitPiece(srcAddr, dstAddr, pieceSize, src.getDeviceId(), dst.getDeviceId(), srcChunkSize, srcBase, + dstChunkSize, dstBase, srcFound && dstFound); srcAddr += pieceSize; dstAddr += pieceSize; remaining -= pieceSize; + ++numPieces; } } - return {MemoryDescs{srcDescs.getType(), std::move(splitSrc)}, MemoryDescs{dstDescs.getType(), std::move(splitDst)}}; + if (outSrc.size() != srcVec.size()) + { + TLLM_LOG_DEBUG("VmmDescSplitter::splitAndCoalesceTransferDescs: %zu pairs -> %zu pieces -> %zu transfers", + srcVec.size(), numPieces, outSrc.size()); + } + + return {MemoryDescs{srcDescs.getType(), std::move(outSrc)}, MemoryDescs{dstDescs.getType(), std::move(outDst)}}; } MemoryDescs VmmDescSplitter::splitVmmDescs(MemoryDescs const& descs, size_t& detectedChunkSize) diff --git a/cpp/tensorrt_llm/executor/cache_transmission/ucx_utils/ucxCacheCommunicator.cpp b/cpp/tensorrt_llm/executor/cache_transmission/ucx_utils/ucxCacheCommunicator.cpp index 4ad1e7bffc86..7b817545b39c 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/ucx_utils/ucxCacheCommunicator.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/ucx_utils/ucxCacheCommunicator.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -611,23 +611,41 @@ UcxConnection::ConnectionIdType UcxConnectionManager::getNewConnectionId(std::sh Connection const* UcxConnectionManager::recvConnect(DataContext const& ctx, void* data, size_t size) { - std::vector<char> buffer(size + sizeof(UcxConnection::ConnectionIdType)); - std::promise<void> promise; - std::future<void> future = promise.get_future(); - auto completionCallback = [&](ucs_status_t, ucxx::RequestCallbackUserData) -> void { promise.set_value(); }; + // Co-owned by the completion callback: the terminate path returns while the + // cancelled request may still write the buffer and fire the callback. + struct RecvState + { + std::vector<char> mBuffer; + std::promise<void> mPromise; + }; - std::shared_ptr<ucxx::Request> req = mWorkersPool.front()->tagRecv( - buffer.data(), buffer.size(), ucxx::Tag(ctx.getTag()), ucxx::TagMask(0xFFFFFFFF), false, completionCallback); + auto state = std::make_shared<RecvState>(); + state->mBuffer.resize(size + sizeof(UcxConnection::ConnectionIdType)); + std::future<void> future = state->mPromise.get_future(); + auto completionCallback + = [state](ucs_status_t, ucxx::RequestCallbackUserData) -> void { state->mPromise.set_value(); }; + + std::shared_ptr<ucxx::Request> req = mWorkersPool.front()->tagRecv(state->mBuffer.data(), state->mBuffer.size(), + ucxx::Tag(ctx.getTag()), ucxx::TagMask(0xFFFFFFFF), false, completionCallback); if (!req->isCompleted()) { - future.get(); + // Poll with timeout to allow checking the terminate flag + auto const& terminate = ctx.getTransferTerminate(); + while (future.wait_for(std::chrono::milliseconds(100)) != std::future_status::ready) + { + if (terminate.load()) + { + req->cancel(); + return nullptr; + } + } } TLLM_CHECK_WITH_INFO(req->isCompleted(), "recv SendConnectionId should be completed"); req->checkError(); - memcpy(data, buffer.data(), size); + memcpy(data, state->mBuffer.data(), size); UcxConnection::ConnectionIdType connectionId - = *reinterpret_cast<UcxConnection::ConnectionIdType*>(buffer.data() + size); + = *reinterpret_cast<UcxConnection::ConnectionIdType*>(state->mBuffer.data() + size); std::scoped_lock lock(mConnectionsMutex, mConnectionFuturesMutex); TLLM_CHECK_WITH_INFO(mConnectionFutures.find(connectionId) != mConnectionFutures.end(), "connectionFuture not found In recvConnect connectionId : %lu , worldRank: %d", connectionId, mRank); @@ -642,7 +660,7 @@ Connection const* UcxConnectionManager::recvConnect(DataContext const& ctx, void TLLM_CHECK(!mConnections[connectionId]->isFromRequester()); TLLM_LOG_DEBUG(mRank, "recvConnect connectionId: %lu , sendIDData:%lu", connectionId, - *reinterpret_cast<uint64_t*>(buffer.data())); + *reinterpret_cast<uint64_t*>(state->mBuffer.data())); return mConnections[connectionId].get(); } diff --git a/cpp/tensorrt_llm/executor/disaggServerUtil.cpp b/cpp/tensorrt_llm/executor/disaggServerUtil.cpp deleted file mode 100644 index 6be2e4fb8ae8..000000000000 --- a/cpp/tensorrt_llm/executor/disaggServerUtil.cpp +++ /dev/null @@ -1,555 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/executor/disaggServerUtil.h" -#include "tensorrt_llm/common/utils.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" - -#include <mutex> - -namespace tensorrt_llm::executor::disagg_executor -{ - -class DisaggExecutorOrchestrator::Impl -{ -public: - Impl(std::vector<std::filesystem::path> const& ctxEnginePaths, - std::vector<std::filesystem::path> const& genEnginePaths, - std::vector<texec::ExecutorConfig> const& ctxExecutorConfigs, - std::vector<texec::ExecutorConfig> const& genExecutorConfigs, bool hasContextAwaitThreads, - bool hasGenAwaitThreads) - : mhasContextAwaitThreads(hasContextAwaitThreads) - , mhasGenAwaitThreads(hasGenAwaitThreads) - { - TLLM_CHECK(ctxEnginePaths.size() == ctxExecutorConfigs.size()); - TLLM_CHECK(genEnginePaths.size() == genExecutorConfigs.size()); - TLLM_CHECK(!(ctxEnginePaths.empty() || genEnginePaths.empty())); - int worldRank = tensorrt_llm::mpi::MpiComm::world().getRank(); - mIsOrchestrator = (worldRank == 0); - auto contextNum = ctxEnginePaths.size(); - mContextReqIdToGlobalId = std::vector<std::unordered_map<IdType, IdType>>(contextNum); - mContextMapMutexs = std::vector<std::mutex>(contextNum); - auto genNum = genEnginePaths.size(); - mGenerationReqIdToGlobalId = std::vector<std::unordered_map<IdType, IdType>>(genNum); - mGenerationMapMutexs = std::vector<std::mutex>(genNum); - - for (size_t cN = 0; cN < contextNum; cN++) - { - mContextExecutors.push_back(std::make_unique<texec::Executor>( - ctxEnginePaths[cN], texec::ModelType::kDECODER_ONLY, ctxExecutorConfigs[cN])); - } - - for (size_t gN = 0; gN < genNum; gN++) - { - mGenerationExecutors.push_back(std::make_unique<texec::Executor>( - genEnginePaths[gN], texec::ModelType::kDECODER_ONLY, genExecutorConfigs[gN])); - } - - if (mIsOrchestrator) - { - if (mhasContextAwaitThreads) - { - for (size_t contextIdx = 0; contextIdx < contextNum; contextIdx++) - { - mContextThreads.emplace_back( - [this, contextIdx]() { this->waitResponseAndAppendThreadFun(true, contextIdx); }); - } - } - if (mhasGenAwaitThreads) - { - - for (size_t genIdx = 0; genIdx < genNum; genIdx++) - { - mGenerationThreads.emplace_back( - [this, genIdx]() { this->waitResponseAndAppendThreadFun(false, genIdx); }); - } - } - } - tensorrt_llm::mpi::MpiComm::world().barrier(); - } - - std::vector<IdType> enqueueContext(std::vector<texec::Request> const& requests, - std::optional<int> selectContextId = std::nullopt, bool batch = false) - { - - std::vector<IdType> globalReqIds; - for (auto const& request : requests) - { - globalReqIds.push_back(generatedGlobalId()); - TLLM_CHECK(request.getRequestType() == tensorrt_llm::executor::RequestType::REQUEST_TYPE_CONTEXT_ONLY); - } - - if (batch) - { - size_t contextId = selectContextId.has_value() ? selectContextId.value() : selectContextExecutor(); - auto contextReqIds = mContextExecutors[contextId]->enqueueRequests(requests); - { - std::scoped_lock<std::mutex> lock{mContextMapMutexs[contextId]}; - for (size_t i = 0; i < requests.size(); ++i) - { - mContextReqIdToGlobalId[contextId][contextReqIds[i]] = globalReqIds[i]; - } - } - } - else - { - for (size_t i = 0; i < requests.size(); ++i) - { - size_t contextId = selectContextId.has_value() ? selectContextId.value() : selectContextExecutor(); - - auto contextReqId = mContextExecutors[contextId]->enqueueRequest(requests[i]); - { - std::scoped_lock<std::mutex> lock{mContextMapMutexs[contextId]}; - mContextReqIdToGlobalId[contextId][contextReqId] = globalReqIds[i]; - } - } - } - return globalReqIds; - } - - void enqueueGeneration(std::vector<texec::Request> const& requests, std::vector<IdType> const& globalRequestIds, - std::optional<int> selectGenIdx = std::nullopt, bool batch = false) - { - - TLLM_CHECK(globalRequestIds.size() == requests.size()); - - for (auto const& request : requests) - { - - TLLM_CHECK(request.getRequestType() == tensorrt_llm::executor::RequestType::REQUEST_TYPE_GENERATION_ONLY); - } - if (batch) - { - size_t genIdx = selectGenIdx.has_value() ? selectGenIdx.value() : selectGenerationExecutor(); - auto genReqIds = mGenerationExecutors[genIdx]->enqueueRequests(requests); - { - std::scoped_lock<std::mutex> lock{mGenerationMapMutexs[genIdx]}; - for (size_t i = 0; i < requests.size(); ++i) - { - mGenerationReqIdToGlobalId[genIdx][genReqIds[i]] = globalRequestIds[i]; - } - } - } - else - { - for (size_t i = 0; i < requests.size(); ++i) - { - size_t genIdx = selectGenIdx.has_value() ? selectGenIdx.value() : selectGenerationExecutor(); - - auto genReqId = mGenerationExecutors[genIdx]->enqueueRequest(requests[i]); - { - std::scoped_lock<std::mutex> lock{mGenerationMapMutexs[genIdx]}; - mGenerationReqIdToGlobalId[genIdx][genReqId] = globalRequestIds[i]; - } - } - } - } - - std::vector<ResponseWithId> awaitContextResponses( - std::optional<int> contextIdx, std::optional<std::chrono::milliseconds> const& timeout) - { - - std::vector<ResponseWithId> responses; - - if (mhasContextAwaitThreads) - { - - std::unique_lock<std::mutex> lock(mResponsesContextMtx); - auto pred = [&mShutdown = mShutdown, &resp = this->mContextResponses]() -> bool - { return !resp.empty() || mShutdown; }; - auto storeResponses = [&resp = this->mContextResponses, &responses]() - { - responses = std::move(resp); - resp.clear(); - }; - if (timeout) - { - if (mContextResponsesCV.wait_for(lock, timeout.value(), pred)) - { - storeResponses(); - } - } - else - { - mContextResponsesCV.wait(lock, pred); - storeResponses(); - } - TLLM_CHECK_WITH_INFO( - !contextIdx.has_value(), "contextIdx should not be provided when mhasContextAwaitThreads is true"); - - return responses; - } - - if (contextIdx.has_value()) - { - TLLM_CHECK(!mhasContextAwaitThreads); - auto responseFromExecutor = mContextExecutors[contextIdx.value()]->awaitResponses(timeout); - for (auto&& resp : responseFromExecutor) - { - - auto reqId = resp.getRequestId(); - IdType globalId{0}; - - { - std::scoped_lock<std::mutex> lock{mContextMapMutexs.at(contextIdx.value())}; - globalId = mContextReqIdToGlobalId.at(contextIdx.value()).at(reqId); - } - TLLM_CHECK(globalId != 0); - responses.emplace_back(std::move(resp), globalId); - } - return responses; - } - TLLM_CHECK(timeout.has_value()); - auto timeouP = timeout.value() / mContextExecutors.size(); - for (size_t ci = 0; ci < mContextExecutors.size(); ci++) - { - auto responseFromExecutor = mContextExecutors.at(ci)->awaitResponses(timeouP); - for (auto&& resp : responseFromExecutor) - { - auto reqId = resp.getRequestId(); - IdType globalId{0}; - - { - std::scoped_lock<std::mutex> lock{mContextMapMutexs.at(ci)}; - globalId = mContextReqIdToGlobalId.at(ci).at(reqId); - } - TLLM_CHECK(globalId != 0); - responses.emplace_back(std::move(resp), globalId); - } - } - - return responses; - }; - - std::vector<ResponseWithId> awaitGenerationResponses( - std::optional<int> genIdx, std::optional<std::chrono::milliseconds> const& timeout) - { - - std::vector<ResponseWithId> responses; - - if (mhasGenAwaitThreads) - { - - std::unique_lock<std::mutex> lock(mResponseGenerationMtx); - auto pred = [&mShutdown = mShutdown, &resp = this->mGenerationResponses]() -> bool - { return !resp.empty() || mShutdown; }; - auto storeResponses = [&resp = this->mGenerationResponses, &responses]() - { - responses = std::move(resp); - resp.clear(); - }; - if (timeout) - { - if (mGenerationResponsesCv.wait_for(lock, timeout.value(), pred)) - { - storeResponses(); - } - } - else - { - mGenerationResponsesCv.wait(lock, pred); - storeResponses(); - } - TLLM_CHECK_WITH_INFO(!genIdx.has_value(), "genIdx should not be provided when mhasGenAwaitThreads is true"); - return responses; - } - - if (genIdx.has_value()) - { - TLLM_CHECK(!mhasGenAwaitThreads); - auto responseFromExecutor = mGenerationExecutors[genIdx.value()]->awaitResponses(timeout); - for (auto&& resp : responseFromExecutor) - { - - auto reqId = resp.getRequestId(); - IdType globalId{0}; - - { - std::scoped_lock<std::mutex> lock{mGenerationMapMutexs.at(genIdx.value())}; - globalId = mGenerationReqIdToGlobalId.at(genIdx.value()).at(reqId); - } - TLLM_CHECK(globalId != 0); - responses.emplace_back(std::move(resp), globalId); - } - return responses; - } - TLLM_CHECK(timeout.has_value()); - auto timeouP = timeout.value() / mGenerationExecutors.size(); - - for (size_t gi = 0; gi < mGenerationExecutors.size(); gi++) - { - auto responseFromExecutor = mGenerationExecutors.at(gi)->awaitResponses(timeouP); - for (auto&& resp : responseFromExecutor) - { - - auto reqId = resp.getRequestId(); - IdType globalId{0}; - - { - std::scoped_lock<std::mutex> lock{mGenerationMapMutexs.at(gi)}; - globalId = mGenerationReqIdToGlobalId.at(gi).at(reqId); - } - TLLM_CHECK(globalId != 0); - responses.emplace_back(std::move(resp), globalId); - } - } - - return responses; - }; - - [[nodiscard]] bool canEnqueue() const - { - return mIsOrchestrator; - } - - [[nodiscard]] std::vector<std::unique_ptr<texec::Executor>> const& getContextExecutors() const - { - return mContextExecutors; - } - - [[nodiscard]] std::vector<std::unique_ptr<texec::Executor>> const& getGenExecutors() const - { - return mGenerationExecutors; - } - - ~Impl() - { - - mShutdown = true; - - mContextResponsesCV.notify_all(); - mGenerationResponsesCv.notify_all(); - for (auto&& executor : mContextExecutors) - { - executor->shutdown(); - } - for (auto&& executor : mGenerationExecutors) - { - executor->shutdown(); - } - - if (mIsOrchestrator) - { - if (mhasContextAwaitThreads) - { - for (auto&& contextThread : mContextThreads) - { - if (contextThread.joinable()) - { - contextThread.join(); - } - } - } - if (mhasGenAwaitThreads) - { - for (auto&& genThread : mGenerationThreads) - { - if (genThread.joinable()) - { - genThread.join(); - } - } - } - } - } - -private: - IdType generatedGlobalId() - { - return (++mLastId % UINT64_MAX); - }; - - size_t selectContextExecutor() - { - static size_t selectContextId = 0; - auto contextId = (selectContextId++) % mContextExecutors.size(); - if (selectContextId >= mContextExecutors.size()) - { - selectContextId = 0; - } - return contextId; - } - - size_t selectGenerationExecutor() - { - static size_t selectGenerationId = 0; - auto generationIdx = (selectGenerationId++) % mGenerationExecutors.size(); - if (selectGenerationId >= mGenerationExecutors.size()) - { - selectGenerationId = 0; - } - return generationIdx; - } - - void appendNewContextResponse(std::vector<ResponseWithId>&& newResponses) - { - { - std::scoped_lock<std::mutex> lock(mResponsesContextMtx); - for (auto&& response : newResponses) - { - mContextResponses.emplace_back(std::move(response)); - } - } - mContextResponsesCV.notify_all(); - } - - void appendNewGenerationResponse(std::vector<ResponseWithId>&& newResponses) - { - { - std::scoped_lock<std::mutex> lock(mResponseGenerationMtx); - for (auto&& response : newResponses) - { - mGenerationResponses.emplace_back(std::move(response)); - } - } - mGenerationResponsesCv.notify_all(); - } - - void waitResponseAndAppendThreadFun(bool isContext, int executorIdx) - { - - tensorrt_llm::common::setThreadName("waitResponseAndAppendThreadFun"); - - auto& executor = isContext ? mContextExecutors[executorIdx] : mGenerationExecutors[executorIdx]; - - while (!mShutdown) - { - auto responses = executor->awaitResponses(); - - if (responses.empty()) - { - continue; - } - std::vector<ResponseWithId> responseWithIds; - if (isContext) - { - for (auto&& response : responses) - { - auto reqId = response.getRequestId(); - IdType globalId{0}; - - { - std::scoped_lock<std::mutex> lock{mContextMapMutexs.at(executorIdx)}; - globalId = mContextReqIdToGlobalId.at(executorIdx).at(reqId); - } - TLLM_CHECK(globalId != 0); - responseWithIds.emplace_back(std::move(response), globalId); - } - if (responseWithIds.size() > 0) - { - appendNewContextResponse(std::move(responseWithIds)); - } - } - else - { - - for (auto&& response : responses) - { - auto reqId = response.getRequestId(); - IdType globalId{0}; - - { - std::scoped_lock<std::mutex> lock{mGenerationMapMutexs.at(executorIdx)}; - globalId = mGenerationReqIdToGlobalId.at(executorIdx).at(reqId); - } - TLLM_CHECK(globalId != 0); - responseWithIds.emplace_back(std::move(response), globalId); - } - if (responseWithIds.size() > 0) - { - appendNewGenerationResponse(std::move(responseWithIds)); - } - } - } - }; - - std::vector<std::unique_ptr<texec::Executor>> mContextExecutors; - std::vector<std::unique_ptr<texec::Executor>> mGenerationExecutors; - std::vector<std::thread> mContextThreads; - std::vector<std::thread> mGenerationThreads; - - std::atomic<IdType> mLastId{0}; - std::vector<std::unordered_map<IdType, IdType>> mContextReqIdToGlobalId; - std::vector<std::unordered_map<IdType, IdType>> mGenerationReqIdToGlobalId; - std::vector<std::mutex> mContextMapMutexs; - std::vector<std::mutex> mGenerationMapMutexs; - std::vector<ResponseWithId> mContextResponses; - std::condition_variable mContextResponsesCV; - std::mutex mResponsesContextMtx; - - std::vector<ResponseWithId> mGenerationResponses; - std::condition_variable mGenerationResponsesCv; - std::mutex mResponseGenerationMtx; - std::atomic<bool> mShutdown{false}; - std::atomic<bool> mhasContextAwaitThreads{false}; - std::atomic<bool> mhasGenAwaitThreads{false}; - bool mIsOrchestrator{false}; -}; - -DisaggExecutorOrchestrator::DisaggExecutorOrchestrator(std::vector<std::filesystem::path> const& ctxEnginePaths, - std::vector<std::filesystem::path> const& genEnginePaths, - std::vector<executor::ExecutorConfig> const& ctxExecutorConfigs, - std::vector<executor::ExecutorConfig> const& genExecutorConfigs, bool hasContextAwaitThreads, - bool hasGenAwaitThreads) - : mImpl(std::make_unique<DisaggExecutorOrchestrator::Impl>(ctxEnginePaths, genEnginePaths, ctxExecutorConfigs, - genExecutorConfigs, hasContextAwaitThreads, hasGenAwaitThreads)) -{ -} - -std::vector<IdType> DisaggExecutorOrchestrator::enqueueContext( - std::vector<texec::Request> const& requests, std::optional<int> selectContextId, bool batch) -{ - return mImpl->enqueueContext(requests, selectContextId, batch); -} - -void DisaggExecutorOrchestrator::enqueueGeneration(std::vector<texec::Request> const& requests, - std::vector<IdType> const& globalRequestIds, std::optional<int> selectGenIdx, bool batch) -{ - mImpl->enqueueGeneration(requests, globalRequestIds, selectGenIdx, batch); -} - -std::vector<ResponseWithId> DisaggExecutorOrchestrator::awaitContextResponses( - std::optional<std::chrono::milliseconds> const& timeout, std::optional<int> contextIdx) -{ - return mImpl->awaitContextResponses(contextIdx, timeout); -} - -std::vector<ResponseWithId> DisaggExecutorOrchestrator::awaitGenerationResponses( - std::optional<std::chrono::milliseconds> const& timeout, std::optional<int> genIdx) -{ - return mImpl->awaitGenerationResponses(genIdx, timeout); -} - -bool DisaggExecutorOrchestrator::canEnqueue() const -{ - return mImpl->canEnqueue(); -}; - -std::vector<std::unique_ptr<texec::Executor>> const& DisaggExecutorOrchestrator::getContextExecutors() const -{ - return mImpl->getContextExecutors(); -} - -std::vector<std::unique_ptr<texec::Executor>> const& DisaggExecutorOrchestrator::getGenExecutors() const -{ - return mImpl->getGenExecutors(); -} - -DisaggExecutorOrchestrator::~DisaggExecutorOrchestrator() = default; - -} // namespace tensorrt_llm::executor::disagg_executor diff --git a/cpp/tensorrt_llm/executor/dynamicBatchTuner.cpp b/cpp/tensorrt_llm/executor/dynamicBatchTuner.cpp deleted file mode 100644 index b7cd49f5430b..000000000000 --- a/cpp/tensorrt_llm/executor/dynamicBatchTuner.cpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/executor/dynamicBatchTuner.h" -#include "tensorrt_llm/common/logger.h" -#include <cmath> - -namespace -{ -using namespace tensorrt_llm::executor; - -void updateStats(SizeType32 value, std::deque<SizeType32>& stats, int64_t& sum, SizeType32 windowSize) -{ - while (static_cast<SizeType32>(stats.size()) >= windowSize) - { - sum -= stats.front(); - stats.pop_front(); - } - stats.push_back(value); - sum += value; -} -} // namespace - -namespace tensorrt_llm::executor -{ - -DynamicBatchTuner::DynamicBatchTuner(DynamicBatchConfig const& config) - : mEnableBatchSizeTuning(config.getEnableBatchSizeTuning()) - , mEnableMaxNumTokensTuning(config.getEnableMaxNumTokensTuning()) - , mDynamicBatchMovingAverageWindow(config.getDynamicBatchMovingAverageWindow()) - , mBatchSizeTable(config.getBatchSizeTable()) -{ - TLLM_CHECK_WITH_INFO(!mBatchSizeTable.empty(), "Batch size table is empty."); - for (size_t i = 1; i < mBatchSizeTable.size(); ++i) - { - TLLM_CHECK_WITH_INFO(mBatchSizeTable[i - 1].first < mBatchSizeTable[i].first, - "Batch size table is not sorted in ascending order."); - } -} - -void DynamicBatchTuner::updateStats(SizeType32 inputLength, SizeType32 outputLength) -{ - ::updateStats(inputLength, mInputLengthStats, mInputLengthSum, mDynamicBatchMovingAverageWindow); - ::updateStats(outputLength, mOutputLengthStats, mOutputLengthSum, mDynamicBatchMovingAverageWindow); -} - -double DynamicBatchTuner::getAverageInputLength() const -{ - return mInputLengthStats.empty() ? 0 : static_cast<double>(mInputLengthSum) / mInputLengthStats.size(); -} - -double DynamicBatchTuner::getAverageOutputLength() const -{ - return mOutputLengthStats.empty() ? 0 : static_cast<double>(mOutputLengthSum) / mOutputLengthStats.size(); -} - -SizeType32 DynamicBatchTuner::getRuntimeBatchSize(SizeType32 maxCapacityBatchSize) const -{ - for (auto const& [batchSizeLimit, batchSize] : mBatchSizeTable) - { - if (maxCapacityBatchSize < batchSizeLimit) - { - return batchSize; - } - } - SizeType32 threshold = maxCapacityBatchSize / kBatchSizeFallbackGranularity * kBatchSizeFallbackGranularity; - if (maxCapacityBatchSize < (threshold + kBatchSizeFallbackThreshold)) - { - return threshold; - } - return maxCapacityBatchSize; -} - -SizeType32 DynamicBatchTuner::getRuntimeMaxNumTokens(SizeType32 maxRuntimeBatchSize) const -{ - // calculate max num token in fully overlapped case - SizeType32 adjustedNumTokens - = 1.0 * (maxRuntimeBatchSize * getAverageInputLength() / getAverageOutputLength() + maxRuntimeBatchSize); - SizeType32 tokenThreshold; - // context heavy (avg ISL/OSL > kMaxNumTokensRatioContextHeavy) - if (getAverageInputLength() / getAverageOutputLength() > kMaxNumTokensRatioContextHeavy) - { - tokenThreshold = kMaxNumTokensThresholdContextHeavy; - } - // balanced case (kMaxNumTokensRatioBalanced < avg ISL/OSL < kMaxNumTokensRatioContextHeavy) - else if (getAverageInputLength() / getAverageOutputLength() > kMaxNumTokensRatioBalanced) - { - tokenThreshold = kMaxNumTokensThresholdBalanced; - } - // gen heavy (avg ISL/OSL < kMaxNumTokensRatioBalanced) - else - { - tokenThreshold = kMaxNumTokensThresholdGenHeavy; - } - // pad it to pow of 2 and max of this value and threshold. - return (std::max(1 << int(ceil(log2(adjustedNumTokens))), tokenThreshold)); -} - -} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/dynamicBatchTuner.h b/cpp/tensorrt_llm/executor/dynamicBatchTuner.h deleted file mode 100644 index df38cd157fde..000000000000 --- a/cpp/tensorrt_llm/executor/dynamicBatchTuner.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" - -#include <deque> - -namespace tensorrt_llm::executor -{ - -/// @brief A class that maintains runtime input and output length statistics and computes runtime dynamic batch size. -class DynamicBatchTuner -{ -public: - explicit DynamicBatchTuner(DynamicBatchConfig const& config); - - /// @brief Check if dynamic batch size tuning is enabled. - [[nodiscard]] bool isBatchSizeTuningEnabled() const - { - return mEnableBatchSizeTuning; - } - - /// @brief Check if max num tokens tuning is enabled. - [[nodiscard]] bool isMaxNumTokensTuningEnabled() const - { - return mEnableMaxNumTokensTuning; - } - - /// @brief Update current stats given the input and output length from a single request. - void updateStats(SizeType32 inputLen, SizeType32 outputLen); - - /// @brief Get average input length. - [[nodiscard]] double getAverageInputLength() const; - - /// @brief Get average output length. - [[nodiscard]] double getAverageOutputLength() const; - - /// @brief Get the dynamic batch size based on the current statistics. - [[nodiscard]] SizeType32 getRuntimeBatchSize(SizeType32 maxCapacityBatchSize) const; - - /// @brief Get the dynamic max num tokens based on the current statistics. - [[nodiscard]] SizeType32 getRuntimeMaxNumTokens(SizeType32 runtimeBatchSize) const; - -private: - bool mEnableBatchSizeTuning = false; - - bool mEnableMaxNumTokensTuning = false; - - SizeType32 mDynamicBatchMovingAverageWindow = 0; - - std::vector<std::pair<SizeType32, SizeType32>> mBatchSizeTable; - - int64_t mInputLengthSum = 0; - std::deque<SizeType32> mInputLengthStats; - - int64_t mOutputLengthSum = 0; - std::deque<SizeType32> mOutputLengthStats; - - static SizeType32 const kBatchSizeFallbackGranularity = 512; - static SizeType32 const kBatchSizeFallbackThreshold = 128; - - static double constexpr kMaxNumTokensRatioContextHeavy = 2.0; - static double constexpr kMaxNumTokensRatioBalanced = 0.5; - - static SizeType32 const kMaxNumTokensThresholdContextHeavy = 8192; - static SizeType32 const kMaxNumTokensThresholdBalanced = 4096; - static SizeType32 const kMaxNumTokensThresholdGenHeavy = 2048; -}; - -} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/executor.cpp b/cpp/tensorrt_llm/executor/executor.cpp deleted file mode 100644 index 091bb5128230..000000000000 --- a/cpp/tensorrt_llm/executor/executor.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <tensorrt_llm/executor/executor.h> -#include <tensorrt_llm/executor/executorImpl.h> - -namespace tensorrt_llm::executor -{ - -Executor::Executor(std::filesystem::path const& modelPath, ModelType modelType, ExecutorConfig const& executorConfig) - : mImpl(std::make_unique<Executor::Impl>(modelPath, std::nullopt, modelType, executorConfig)) -{ -} - -Executor::Executor(std::filesystem::path const& encoderModelPath, std::filesystem::path const& decoderModelPath, - ModelType modelType, ExecutorConfig const& executorConfig) - : mImpl(std::make_unique<Executor::Impl>(decoderModelPath, encoderModelPath, modelType, executorConfig)) -{ -} - -Executor::Executor(BufferView const& engineBuffer, std::string const& jsonConfigStr, ModelType modelType, - ExecutorConfig const& executorConfig, std::optional<std::map<std::string, Tensor>> const& managedWeights) - : mImpl(std::make_unique<Executor::Impl>( - engineBuffer, jsonConfigStr, std::nullopt, std::nullopt, modelType, executorConfig, managedWeights)) -{ -} - -Executor::Executor(BufferView const& encoderEngineBuffer, std::string const& encoderJsonConfigStr, - BufferView const& decoderEngineBuffer, std::string const& decoderJsonConfigStr, ModelType modelType, - ExecutorConfig const& executorConfig) - : mImpl(std::make_unique<Executor::Impl>(decoderEngineBuffer, decoderJsonConfigStr, encoderEngineBuffer, - encoderJsonConfigStr, modelType, executorConfig, std::nullopt)) -{ -} - -Executor::Executor(std::shared_ptr<Model> model, ExecutorConfig const& executorConfig) - : mImpl(std::make_unique<Executor::Impl>(std::move(model), std::nullopt, executorConfig)) -{ -} - -Executor::Executor( - std::shared_ptr<Model> encoderModel, std::shared_ptr<Model> decoderModel, ExecutorConfig const& executorConfig) - : mImpl(std::make_unique<Executor::Impl>(std::move(decoderModel), std::move(encoderModel), executorConfig)) -{ -} - -Executor::~Executor() = default; - -IdType Executor::enqueueRequest(Request const& llmRequest) -{ - return mImpl->enqueueRequest(llmRequest); -} - -std::vector<IdType> Executor::enqueueRequests(std::vector<Request> const& llmRequests) -{ - return mImpl->enqueueRequests(llmRequests); -} - -std::vector<Response> Executor::awaitResponses(std::optional<std::chrono::milliseconds> const& timeout) -{ - return mImpl->awaitResponses(timeout); -} - -std::vector<Response> Executor::awaitResponses( - IdType const& requestId, std::optional<std::chrono::milliseconds> const& timeout) -{ - return mImpl->awaitResponses(requestId, timeout); -} - -std::vector<std::vector<Response>> Executor::awaitResponses( - std::vector<IdType> const& requestIds, std::optional<std::chrono::milliseconds> const& timeout) -{ - return mImpl->awaitResponses(requestIds, timeout); -} - -SizeType32 Executor::getNumResponsesReady(std::optional<IdType> const& requestId) const -{ - return mImpl->getNumResponsesReady(requestId); -} - -void Executor::cancelRequest(IdType requestId) -{ - return mImpl->cancelRequest(requestId); -} - -void Executor::shutdown() -{ - return mImpl->shutdown(); -} - -std::deque<IterationStats> Executor::getLatestIterationStats() -{ - return mImpl->getLatestIterationStats(); -} - -std::deque<RequestStatsPerIteration> Executor::getLatestRequestStats() -{ - return mImpl->getLatestRequestStats(); -} - -std::deque<DebugTensorsPerIteration> Executor::getLatestDebugTensors() -{ - return mImpl->getLatestDebugTensors(); -} - -bool Executor::canEnqueueRequests() const -{ - return mImpl->canEnqueueRequests(); -} - -bool Executor::isParticipant() const -{ - return mImpl->isParticipant(); -} - -std::optional<std::shared_ptr<KVCacheEventManager>> Executor::getKVCacheEventManager() const -{ - return mImpl->getKVCacheEventManager(); -} - -KVCacheEvent::KVCacheEvent( - size_t eventId, KVCacheEventData data, SizeType32 windowSize, std::optional<SizeType32> attentionDpRank) - : eventId{eventId} - , data{std::move(data)} - , windowSize{windowSize} - , attentionDpRank{attentionDpRank} -{ -} - -} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/executorImpl.cpp b/cpp/tensorrt_llm/executor/executorImpl.cpp deleted file mode 100644 index 9f7fb654a2d5..000000000000 --- a/cpp/tensorrt_llm/executor/executorImpl.cpp +++ /dev/null @@ -1,2791 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/executor/executorImpl.h" -#include "tensorrt_llm/batch_manager/trtEncoderModel.h" -#include "tensorrt_llm/batch_manager/trtGptModelFactory.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/cudaProfilerUtils.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/common/timestampUtils.h" -#include "tensorrt_llm/common/utils.h" -#include "tensorrt_llm/executor/dataTransceiverState.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/orchestratorUtils.h" -#include "tensorrt_llm/executor/requestUtils.h" -#include "tensorrt_llm/executor/serialization.h" -#include "tensorrt_llm/executor/serializeUtils.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/executor/version.h" -#include "tensorrt_llm/runtime/loraCache.h" -#include "tensorrt_llm/runtime/memoryCounters.h" -#include "tensorrt_llm/runtime/utils/mpiTags.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" - -#include <algorithm> -#include <cstddef> -#include <cstdint> -#include <cuda_profiler_api.h> -#include <iterator> -#include <memory> -#include <optional> -#include <utility> - -namespace tensorrt_llm::executor -{ - -namespace -{ - -[[nodiscard]] bool executorConfigIsValid( - ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, runtime::ModelConfig const& modelConfig) -{ - // Make sure logic in this function matches fixExecutorConfig - if (executorConfig.getEnableChunkedContext()) - { - if (modelConfig.isRnnBased() || !modelConfig.isKVCacheEnabled() || !modelConfig.getPagedContextFMHA()) - { - return false; - } - } - return true; -} - -[[nodiscard]] ::tensorrt_llm::executor::ExecutorConfig fixExecutorConfig( - ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, runtime::ModelConfig const& modelConfig) -{ - // Make sure logic in this function matches executorConfigIsValid - auto fixedExecutorConfig = executorConfig; - // Disable chunked context when not supported - if (executorConfig.getEnableChunkedContext()) - { - if (modelConfig.isRnnBased() || !modelConfig.isKVCacheEnabled() || !modelConfig.getPagedContextFMHA()) - { - fixedExecutorConfig.setEnableChunkedContext(false); - TLLM_LOG_WARNING( - "Chunked context is not supported for this configuration and will be disabled. " - "Related configs: RNNBased: %d, KVCacheEnabled: %d, PagedContextFMHA: %d", - modelConfig.isRnnBased(), modelConfig.isKVCacheEnabled(), modelConfig.getPagedContextFMHA()); - } - } - return fixedExecutorConfig; -} - -[[nodiscard]] bool statsBufferIsEnabled(SizeType32 maxIterations) -{ - return maxIterations != 0; -} - -[[nodiscard]] bool statsBufferIsBounded(SizeType32 maxIterations) -{ - return maxIterations > 0; -} - -SizeType32 getNumChildRequests(Request const& request) -{ - auto samplingConfig = request.getSamplingConfig(); - return samplingConfig.getBeamWidth() > 1 ? 0 : samplingConfig.getNumReturnSequences().value_or(1) - 1; -} - -} // namespace - -/// @brief Version of TRT-LLM as defined in tensorrt_llm/version.py -char const* version() noexcept -{ - return kTensorRtLlmVersion; -} - -class CancelledRequestsAsyncSend -{ -public: - CancelledRequestsAsyncSend(std::shared_ptr<tensorrt_llm::mpi::MpiComm> const& commSession, - std::unordered_set<IdType> const& cancelledReqIds, int peer) - { - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - mNumReq = static_cast<int64_t>(cancelledReqIds.size()); - TLLM_LOG_DEBUG("start send %ld cancelled requests to rank %d", mNumReq, peer); - mRequest1 - = commSession->sendAsync(&mNumReq, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kCancelledRequestsNumReq); - if (mNumReq > 0) - { - mIds.assign(cancelledReqIds.begin(), cancelledReqIds.end()); - mRequest2 = commSession->sendAsync( - mIds.data(), mIds.size(), mpi::MpiType::kUINT64, peer, mpi::MpiTag::kCancelledRequestsIds); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - } - - ~CancelledRequestsAsyncSend() - { - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - mRequest1->wait(); - if (mRequest2) - { - mRequest2->wait(); - } - TLLM_LOG_DEBUG("end send cancelled requests"); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - } - - CancelledRequestsAsyncSend(CancelledRequestsAsyncSend const& executor) = delete; - CancelledRequestsAsyncSend& operator=(CancelledRequestsAsyncSend const& executor) = delete; - CancelledRequestsAsyncSend(CancelledRequestsAsyncSend&&) = delete; - CancelledRequestsAsyncSend& operator=(CancelledRequestsAsyncSend&&) = delete; - - static std::unordered_set<IdType> cancelledRequestsRecv( - std::shared_ptr<tensorrt_llm::mpi::MpiComm> const& commSession, int peer) - { - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - TLLM_LOG_DEBUG("start recv cancelled requests from rank %d", peer); - std::unordered_set<IdType> cancelledReqIds; - int64_t numReq{0}; - commSession->recv(&numReq, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kCancelledRequestsNumReq); - TLLM_LOG_DEBUG("recv %ld cancelled requests", numReq); - if (numReq > 0) - { - std::vector<IdType> buffer(numReq); - commSession->recv( - buffer.data(), buffer.size(), mpi::MpiType::kUINT64, peer, mpi::MpiTag::kCancelledRequestsIds); - cancelledReqIds = std::unordered_set<IdType>(buffer.begin(), buffer.end()); - } - TLLM_LOG_DEBUG("end recv cancelled requests from rank %d", peer); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return cancelledReqIds; - } - -private: - int64_t mNumReq; - std::vector<IdType> mIds; - std::shared_ptr<tensorrt_llm::mpi::MpiRequest> mRequest1; - std::shared_ptr<tensorrt_llm::mpi::MpiRequest> mRequest2; -}; - -class RequestWithIdAsyncSend -{ -public: - RequestWithIdAsyncSend(std::shared_ptr<tensorrt_llm::mpi::MpiComm> const& commSession, - std::vector<RequestWithId> const& reqWithIds, int peer) - { - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - TLLM_LOG_DEBUG("start send requests to rank %d", peer); - mNumReq = static_cast<int64_t>(reqWithIds.size()); - mRequest1 = commSession->sendAsync(&mNumReq, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kRequestWithIdNumReq); - if (mNumReq > 0) - { - mPacked = RequestWithId::serializeReqWithIds(reqWithIds); - mVecSize = static_cast<int64_t>(mPacked.size()); - mRequest2 - = commSession->sendAsync(&mVecSize, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kRequestWithIdVecSize); - mRequest3 = commSession->sendAsync( - mPacked.data(), mPacked.size(), mpi::MpiType::kCHAR, peer, mpi::MpiTag::kRequestWithIdPacked); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - } - - ~RequestWithIdAsyncSend() - { - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - mRequest1->wait(); - if (mRequest2) - { - mRequest2->wait(); - } - if (mRequest3) - { - mRequest3->wait(); - } - TLLM_LOG_DEBUG("end send requests"); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - } - - RequestWithIdAsyncSend(RequestWithIdAsyncSend const& executor) = delete; - RequestWithIdAsyncSend& operator=(RequestWithIdAsyncSend const& executor) = delete; - RequestWithIdAsyncSend(RequestWithIdAsyncSend&&) = delete; - RequestWithIdAsyncSend& operator=(RequestWithIdAsyncSend&&) = delete; - - static std::vector<RequestWithId> requestWithIdRecv( - std::shared_ptr<tensorrt_llm::mpi::MpiComm> const& commSession, int peer) - { - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - TLLM_LOG_DEBUG("start recv requests from rank %d", peer); - std::vector<RequestWithId> reqWithIds; - int64_t numReq{0}; - commSession->recv(&numReq, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kRequestWithIdNumReq); - if (numReq > 0) - { - std::vector<char> buffer; - int64_t vecSize = 0; - commSession->recv(&vecSize, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kRequestWithIdVecSize); - buffer.resize(vecSize); - commSession->recv( - buffer.data(), buffer.size(), mpi::MpiType::kCHAR, peer, mpi::MpiTag::kRequestWithIdPacked); - reqWithIds = RequestWithId::deserializeReqWithIds(buffer); - } - TLLM_LOG_DEBUG("end recv requests from rank %d", peer); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return reqWithIds; - } - -private: - int64_t mNumReq; - int64_t mVecSize; - std::vector<char> mPacked; - std::shared_ptr<tensorrt_llm::mpi::MpiRequest> mRequest1; - std::shared_ptr<tensorrt_llm::mpi::MpiRequest> mRequest2; - std::shared_ptr<tensorrt_llm::mpi::MpiRequest> mRequest3; -}; - -void Executor::Impl::loadModel(std::optional<std::filesystem::path> const& modelPathOpt, - std::optional<BufferView> const& engineBufferOpt, runtime::GptJsonConfig const& jsonConfig, - ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, bool isEncoder, - std::optional<std::map<std::string, Tensor>> const& managedWeightsOpt) -{ - auto const gpusPerNode = jsonConfig.getGpusPerNode(); - auto const tp = jsonConfig.getTensorParallelism(); - auto const pp = jsonConfig.getPipelineParallelism(); - auto const cp = jsonConfig.getContextParallelism(); - auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); - auto worldConfig = runtime::WorldConfig::mpi(gpusPerNode, tp, pp, cp, parallelConfig.getDeviceIds()); - - TLLM_CHECK_WITH_INFO(modelPathOpt.has_value() || engineBufferOpt.has_value(), - "Either engine path or deserialized engine buffer should be given to load the model properly."); - auto rawEngine = engineBufferOpt.has_value() - ? runtime::RawEngine(engineBufferOpt.value().data(), engineBufferOpt.value().size()) - : runtime::RawEngine(modelPathOpt.value() / jsonConfig.engineFilename(worldConfig)); - - if (rawEngine.getType() != tensorrt_llm::runtime::RawEngine::FilePath) - { - if (modelPathOpt.has_value()) - { - rawEngine.setPath(modelPathOpt.value() / jsonConfig.engineFilename(worldConfig)); - if (managedWeightsOpt.has_value()) - { - TLLM_LOG_WARNING( - "Executor::Impl::loadModel: managedWeightsOpt argument is ignored when loading engine from file."); - } - } - else if (managedWeightsOpt.has_value()) - { - rawEngine.setManagedWeightsMap(managedWeightsOpt.value()); - } - } - - auto const& modelConfig = jsonConfig.getModelConfig(); - - if (isEncoder) - { - mEncoderModel = createEncoderModel(rawEngine, modelConfig, worldConfig, executorConfig); - } - else - { - mModel = createModel(rawEngine, modelConfig, worldConfig, executorConfig); - } -}; - -Executor::Impl::Impl(std::filesystem::path const& modelPath, - std::optional<std::filesystem::path> const& encoderModelPath, ModelType const modelType, - ::tensorrt_llm::executor::ExecutorConfig const& executorConfig) -{ - auto decoderJsonConfig = runtime::GptJsonConfig::parse(modelPath / "config.json"); - - // for now, assume encoder & decoder models share the same MPI config - auto const tp = decoderJsonConfig.getTensorParallelism(); - auto const pp = decoderJsonConfig.getPipelineParallelism(); - auto const cp = decoderJsonConfig.getContextParallelism(); - initializeCommAndWorkers(tp, pp, cp, executorConfig, modelType, modelPath, std::nullopt, decoderJsonConfig); - - if (mIsWorker) - { - if (modelType == ModelType::kENCODER_DECODER) - { - if (encoderModelPath.has_value()) - { - auto const encoderJsonConfig = runtime::GptJsonConfig::parse(encoderModelPath.value() / "config.json"); - - auto const encoderMaxInputLen = encoderJsonConfig.getModelConfig().getMaxInputLen(); - auto const encoderHiddenSize = encoderJsonConfig.getModelConfig().getHiddenSize() - * encoderJsonConfig.getTensorParallelism(); // recover full hidden size - // add encoder info to decoder for encoder-decoder models - // note: GptJsonConfig can no longer have modelConfig as const member since it must be mutable here - decoderJsonConfig.getModelConfigMutable().setMaxEncoderLen(encoderMaxInputLen); - decoderJsonConfig.getModelConfigMutable().setEncoderHiddenSize(encoderHiddenSize); - - loadModel( - encoderModelPath.value(), std::nullopt, encoderJsonConfig, executorConfig, true, std::nullopt); - } - else - { - TLLM_LOG_WARNING("Encoder model path not provided. Skipping Encoder Run."); - } - } - loadModel(modelPath, std::nullopt, decoderJsonConfig, executorConfig, false, std::nullopt); - } - initialize(executorConfig); -} - -Executor::Impl::Impl(BufferView const& engineBufferView, std::string const& jsonConfigStr, - std::optional<BufferView> const& encoderEngineBufferView, std::optional<std::string> const& encoderJsonConfigStr, - ModelType const modelType, ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, - std::optional<std::map<std::string, Tensor>> const& managedWeightsOpt) -{ - auto decoderJsonConfig = runtime::GptJsonConfig::parse(jsonConfigStr); - - // for now, assume encoder & decoder models share the same MPI config - auto const tp = decoderJsonConfig.getTensorParallelism(); - auto const pp = decoderJsonConfig.getPipelineParallelism(); - auto const cp = decoderJsonConfig.getContextParallelism(); - initializeCommAndWorkers(tp, pp, cp, executorConfig, modelType, std::nullopt, std::nullopt, decoderJsonConfig); - - if (mIsWorker) - { - if (modelType == ModelType::kENCODER_DECODER) - { - TLLM_CHECK(encoderEngineBufferView.has_value() && encoderJsonConfigStr.has_value()); - TLLM_CHECK_WITH_INFO( - !managedWeightsOpt.has_value(), "Managed weights are not supported for enc-dec models"); - - auto const encoderJsonConfig = runtime::GptJsonConfig::parse(encoderJsonConfigStr.value()); - - auto const encoderMaxInputLen = encoderJsonConfig.getModelConfig().getMaxInputLen(); - auto const encoderHiddenSize = encoderJsonConfig.getModelConfig().getHiddenSize() - * encoderJsonConfig.getTensorParallelism(); // recover full hidden size - // add encoder info to decoder for encoder-decoder models - // note: GptJsonConfig can no longer have modelConfig as const member since it must be mutable here - decoderJsonConfig.getModelConfigMutable().setMaxEncoderLen(encoderMaxInputLen); - decoderJsonConfig.getModelConfigMutable().setEncoderHiddenSize(encoderHiddenSize); - - loadModel( - std::nullopt, encoderEngineBufferView.value(), encoderJsonConfig, executorConfig, true, std::nullopt); - } - loadModel(std::nullopt, engineBufferView, decoderJsonConfig, executorConfig, false, managedWeightsOpt); - } - initialize(executorConfig); -} - -Executor::Impl::Impl(std::shared_ptr<Model> model, std::optional<std::shared_ptr<Model>> encoderModel, - ::tensorrt_llm::executor::ExecutorConfig const& executorConfig) -{ - auto const& worldConfig = model->getWorldConfig(); - auto const tp = worldConfig.getTensorParallelism(); - auto const pp = worldConfig.getPipelineParallelism(); - auto const cp = worldConfig.getContextParallelism(); - auto const modelType = encoderModel.has_value() ? ModelType::kENCODER_DECODER : ModelType::kDECODER_ONLY; - initializeCommAndWorkers(tp, pp, cp, executorConfig, modelType, std::nullopt, worldConfig); - if (modelType == ModelType::kENCODER_DECODER) - { - mEncoderModel = encoderModel.value(); - } - mModel = std::move(model); - initialize(executorConfig); -} - -Executor::Impl::~Impl() -{ - shutdown(); -} - -void Executor::Impl::initialize(::tensorrt_llm::executor::ExecutorConfig const& executorConfig) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - mShutdown = false; - mShutdownCalled = false; - mIterStatsMaxIterations = executorConfig.getIterStatsMaxIterations(); - mRequestStatsMaxIterations = executorConfig.getRequestStatsMaxIterations(); - mDebugTensorsMaxIterations - = executorConfig.getDebugConfig() ? executorConfig.getDebugConfig()->getDebugTensorsMaxIterations() : 0; - TLLM_CHECK_WITH_INFO(mDebugTensorsMaxIterations == 0 || mCommMode == CommunicationMode::kLEADER, - "debugTensorsMaxIterations > 0 is only allowed in leader mode."); - mBatchingType = executorConfig.getBatchingType(); - mIsSchedulerMaxUtilization = (executorConfig.getSchedulerConfig().getCapacitySchedulerPolicy() - == CapacitySchedulerPolicy::kMAX_UTILIZATION); - mIsSchedulerGuaranteedNoEvict = (executorConfig.getSchedulerConfig().getCapacitySchedulerPolicy() - == CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT); - mIsChunkedContext = executorConfig.getEnableChunkedContext(); - mPromptTableOffloading = executorConfig.getPromptTableOffloading(); - mMaxQueueSize = executorConfig.getMaxQueueSize(); - - mLastReqId = 1; - - auto const& logitsProcConfig = executorConfig.getLogitsPostProcessorConfig(); - if (logitsProcConfig.has_value()) - { - mLogitsPostProcessorMap = logitsProcConfig.value().getProcessorMap().value_or(LogitsPostProcessorMap{}); - initializeLogitsPostProcessorBatched(logitsProcConfig.value()); - if (!logitsProcConfig.value().getReplicate()) - { - mModel->setReplicateLogitsPostProcessor(false); - } - } - - auto const& commComm = COMM_SESSION; - int32_t const commSize = commComm.getSize(); - if (mIsWorker) - { - if (commSize > 1) - { - auto const& worldConfig = mModel->getWorldConfig(); - auto const& commSession = COMM_SESSION; - auto const& rank = commSession.getRank(); - auto const& tp = worldConfig.getTensorParallelism(); - auto const& cp = worldConfig.getContextParallelism(); - - mCommTensorParallel = std::make_shared<tensorrt_llm::mpi::MpiComm>( - commSession.split(rank / tp, worldConfig.getTensorParallelRank())); - mCommContextParallel = std::make_shared<tensorrt_llm::mpi::MpiComm>( - commSession.split(rank / (tp * cp) * tp + rank % tp, worldConfig.getContextParallelRank())); - mCommPipelineParallel = std::make_shared<tensorrt_llm::mpi::MpiComm>( - commSession.split(rank % (tp * cp), worldConfig.getPipelineParallelRank())); - - if (worldConfig.isPipelineParallel()) - { - mRequestWithIdWaitThread = std::make_unique<tensorrt_llm::mpi::MpiWaitThread>( - "requestWithIdWaitThread", [this]() { mRequestWithIdAsyncSndHdl.reset(nullptr); }); - mCancelledRequestsWaitThread = std::make_unique<tensorrt_llm::mpi::MpiWaitThread>( - "cancelledRequestsWaitThread", [this]() { mCancelledRequestsAsyncSndHdl.reset(nullptr); }); - if (mIsLeader) - { - mRequestWithIdLeaderThread - = std::make_unique<std::thread>(&Executor::Impl::requestWithIdLeaderThread, this); - mCancelledRequestsLeaderThread - = std::make_unique<std::thread>(&Executor::Impl::cancelledRequestsLeaderThread, this); - } - } - } - // Launch the execution thread - mMaxNumActiveRequests = mModel->getMaxNumSequences(); - mExecutionThread = std::thread(&Impl::executionLoop, this); - } - - mEnableBlockReuse = executorConfig.getKvCacheConfig().getEnableBlockReuse(); - - auto const& dynamicBatchConfig = executorConfig.getSchedulerConfig().getDynamicBatchConfig(); - if (dynamicBatchConfig) - { - if (mIsWorker) - { - if (mModel->getModelConfig().isTransformerBased() && mModel->getModelConfig().isKVCacheEnabled()) - { - mDynamicBatchTuner = std::make_shared<DynamicBatchTuner>(dynamicBatchConfig.value()); - } - else - { - TLLM_LOG_WARNING("Dynamic batch tuner can only support transformer models that use KV cache."); - } - } - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -std::shared_ptr<Model> Executor::Impl::createModel(runtime::RawEngine const& rawEngine, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - ::tensorrt_llm::executor::ExecutorConfig const& executorConfig) -{ - auto const gptModelType = [&executorConfig, &modelConfig]() - { - switch (executorConfig.getBatchingType()) - { - case BatchingType::kSTATIC: - TLLM_THROW( - "Static batching type is deprecated. Please use in-flight batching with " - "CapacitySchedulerPolicy::kSTATIC_BATCH instead."); - case BatchingType::kINFLIGHT: - return modelConfig.isRnnBased() ? batch_manager::TrtGptModelType::InflightBatching - : batch_manager::TrtGptModelType::InflightFusedBatching; - default: TLLM_THROW("Invalid batching strategy"); - } - }(); - - bool const isLeaderInOrchMode = (mCommMode == CommunicationMode::kORCHESTRATOR) && mIsLeader; - auto const& fixedExecutorConfig = executorConfigIsValid(executorConfig, modelConfig) - ? executorConfig - : fixExecutorConfig(executorConfig, modelConfig); - - return batch_manager::TrtGptModelFactory::create( - rawEngine, modelConfig, worldConfig, gptModelType, fixedExecutorConfig, isLeaderInOrchMode); -} - -std::shared_ptr<Model> Executor::Impl::createEncoderModel(runtime::RawEngine const& rawEngine, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - ::tensorrt_llm::executor::ExecutorConfig const& executorConfig) -{ - auto fixedExecutorConfig = ExecutorConfig{}; - fixedExecutorConfig.setSchedulerConfig(executorConfig.getSchedulerConfig()); - return std::make_shared<batch_manager::TrtEncoderModel>( - modelConfig, worldConfig, rawEngine, std::make_shared<runtime::TllmLogger>(), fixedExecutorConfig); -} - -void Executor::Impl::setOrchLeaderComm( - SizeType32 tp, SizeType32 pp, SizeType32 cp, ParallelConfig const& parallelConfig) -{ -#if ENABLE_MULTI_DEVICE - auto optOrchestratorConfig = parallelConfig.getOrchestratorConfig(); - if (optOrchestratorConfig.value().getIsOrchestrator()) - { - TLLM_CHECK_WITH_INFO(mWorldRank == 0, "Rank 0 must be orchestrator"); - } - - TLLM_CHECK_WITH_INFO(parallelConfig.getParticipantIds(), - "When not spawning processes in orchestrator mode, participant IDs must be provided"); - auto participantIds = parallelConfig.getParticipantIds().value(); - - TLLM_CHECK_WITH_INFO(static_cast<SizeType32>(participantIds.size()) == tp * pp * cp, - "When specifying participantIds, participantIds size must be equal to tp*pp*cp"); - - bool isLeader = (mWorldRank == participantIds.front()); - bool isOrchestrator = (mWorldRank == 0); - - // OrchLeaderComm rank 0 is orchestrator, rank 1 is leader - mOrchRank = 0; - mLeaderRank = 1; - - // Create a leaderOrch comm - std::vector<int32_t> leaderOrchRanks{0, participantIds.front()}; - - MPI_Group worldGroup = nullptr; - MPICHECK(MPI_Comm_group(MPI_COMM_WORLD, &worldGroup)); // NOLINT - int worldGroupRank = 0; - MPI_Group_rank(worldGroup, &worldGroupRank); - - int worldSize = 0; - MPICHECK(MPI_Group_size(worldGroup, &worldSize)); // NOLINT - TLLM_CHECK_WITH_INFO(participantIds.front() < worldSize, "Not enough ranks in world"); - - MPI_Group leaderOrchCommGroup = nullptr; - MPICHECK( - MPI_Group_incl(worldGroup, leaderOrchRanks.size(), leaderOrchRanks.data(), &leaderOrchCommGroup)); // NOLINT - int leaderOrchGroupRank = 0; - int leaderOrchGroupSize = 0; - MPI_Group_rank(leaderOrchCommGroup, &leaderOrchGroupRank); - MPI_Group_size(leaderOrchCommGroup, &leaderOrchGroupSize); - - if (isOrchestrator || isLeader) - { - MPI_Comm leaderOrchComm = nullptr; - MPICHECK(MPI_Comm_create_group( - MPI_COMM_WORLD, leaderOrchCommGroup, participantIds.front(), &leaderOrchComm)); // NOLINT - mOrchLeaderComm = std::make_shared<tensorrt_llm::mpi::MpiComm>(leaderOrchComm, false); - } - else - { - mOrchLeaderComm = nullptr; - } -#endif // ENABLE_MULTI_DEVICE -} - -void Executor::Impl::initializeCommAndWorkers(SizeType32 tp, SizeType32 pp, SizeType32 cp, - ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, std::optional<ModelType> modelType, - std::optional<std::filesystem::path> const& modelPath, std::optional<runtime::WorldConfig> const& worldConfig, - std::optional<runtime::GptJsonConfig> const& decoderGptJsonConfig) -{ - if (modelType.has_value() && modelType.value() == ModelType::kENCODER_DECODER) - { - TLLM_CHECK_WITH_INFO(pp == 1, - "Encoder-Decoder C++ runtime doesn't support Pipeline Parallelism currently. Please switch to Python " - "runtime for PP mode, if necessary."); - } - - tensorrt_llm::mpi::initialize(tensorrt_llm::mpi::MpiThreadSupport::THREAD_MULTIPLE); - mWorldRank = tensorrt_llm::mpi::MpiComm::world().getRank(); - mUsePipelineParallel = pp > 1; - - auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); - validateParallelConfig(parallelConfig, modelType, modelPath); - - mCommMode = parallelConfig.getCommunicationMode(); - auto optOrchestratorConfig = parallelConfig.getOrchestratorConfig(); - - mRecvPollPeriodMs = executorConfig.getRecvPollPeriodMs(); - - // Need to create communicator between orchestrator and leader if not spawning processes in orchestrator mode - if (mCommMode == CommunicationMode::kORCHESTRATOR && !optOrchestratorConfig.value().getSpawnProcesses()) - { - setOrchLeaderComm(tp, pp, cp, parallelConfig); - } - - if (mCommMode == CommunicationMode::kORCHESTRATOR && optOrchestratorConfig.value().getIsOrchestrator()) - { - initializeOrchestrator(tp, pp, cp, executorConfig, parallelConfig, modelType.value(), modelPath.value()); - } - else - { - initializeWorkers(tp, pp, cp, parallelConfig, worldConfig, decoderGptJsonConfig); - } -} - -void Executor::Impl::validateParallelConfig(ParallelConfig const& parallelConfig, std::optional<ModelType> modelType, - std::optional<std::filesystem::path> const& modelPath) -{ - TLLM_CHECK_WITH_INFO(parallelConfig.getCommunicationType() == CommunicationType::kMPI, - "Only CommunicationType kMPI is supported for now."); - - auto optOrchestratorConfig = parallelConfig.getOrchestratorConfig(); - - if (parallelConfig.getCommunicationMode() == CommunicationMode::kORCHESTRATOR) - { - TLLM_CHECK_WITH_INFO( - optOrchestratorConfig, "OrchestratorConfig must be set when using ORCHESTRATOR communication mode."); - - TLLM_CHECK_WITH_INFO(modelPath, "OrchestratorMode only supports reading model weight from disk currently."); - - TLLM_CHECK_WITH_INFO(modelType, "OrchestratorMode requires modelType to be specified."); - } -} - -void Executor::Impl::initializeOrchestrator(SizeType32 tp, SizeType32 pp, SizeType32 cp, - ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, ParallelConfig parallelConfig, ModelType modelType, - std::filesystem::path const& modelPath) -{ -#if ENABLE_MULTI_DEVICE - namespace su = tensorrt_llm::executor::serialize_utils; - - auto const& worldComm = tensorrt_llm::mpi::MpiComm::world(); - int32_t const worldSize = worldComm.getSize(); - - auto orchestratorConfig = parallelConfig.getOrchestratorConfig().value(); - - mIsWorker = false; - mIsLeader = false; - mIsPipelineLeader = false; - mIsOrchestrator = true; - - // Verify that worldSize is 1 - if (orchestratorConfig.getSpawnProcesses()) - { - TLLM_CHECK_WITH_INFO(worldSize == 1, - "When using the orchestrator mode and isOrchestrator is true, expect MPI worldSize to be 1."); - - // Spawn the worker threads - auto workerExecPath = orchestratorConfig.getWorkerExecutablePath(); - MPI_Comm intercomm = nullptr; - MPI_Info mpiInfo = nullptr; - MPICHECK(MPI_Info_create(&mpiInfo)); - MPICHECK(MPI_Info_set(mpiInfo, "env", "FORCE_NCCL_ALL_REDUCE_STRATEGY")); - - // Binding policy is not inherited for dynamically spawned jobs, resulting in the worker being bound - // to a single core. Override the setting to avoid perf issue - see https://nvbugs/4574329 - MPICHECK(MPI_Info_set(mpiInfo, "bind_to", "none")); - - MPICHECK(MPI_Comm_spawn(workerExecPath.c_str(), MPI_ARGV_NULL, tp * pp * cp, mpiInfo, 0, MPI_COMM_SELF, - &intercomm, MPI_ERRCODES_IGNORE)); - - mOrchLeaderComm = std::make_shared<tensorrt_llm::mpi::MpiComm>(intercomm, true); - // With intercomm, leader is rank 0 in the local group - mLeaderRank = 0; - mOrchRank = 0; - - // Copy the executor config, but set the orchestrator flag to false - auto newOrchConfig = OrchestratorConfig(false, orchestratorConfig.getWorkerExecutablePath()); - parallelConfig.setOrchestratorConfig(newOrchConfig); - auto execConfig = executorConfig; - execConfig.setParallelConfig(parallelConfig); - - // Serialize and send the executorConfig, the modelType and the modelPath - std::ostringstream oStream; - su::serialize(modelPath.string(), oStream); - su::serialize(modelType, oStream); - su::serialize(execConfig, oStream); - - auto str = oStream.str(); - std::vector<char> buffer(str.begin(), str.end()); - auto bufferSize = static_cast<int64_t>(buffer.size()); - mOrchLeaderComm->bcast(&bufferSize, 1, mpi::MpiType::kINT64, MPI_ROOT); - mOrchLeaderComm->bcast(buffer.data(), buffer.size(), mpi::MpiType::kCHAR, MPI_ROOT); - - // Wait for workers to have created their executor instance - MPICHECK(MPI_Barrier(intercomm)); - } - - // Spawn the thread responsible for sending new requests to the leader of the model - mOrchSendReqThread = std::thread(&Impl::orchSendReqThread, this); - - // Spawn the thread responsible for receiving new responses from the leader of the model - mOrchRecvThread - = std::thread([&]() { this->orchRecvThread(mpi::MpiTag::kOrchestratorId, mpi::MpiTag::kOrchestratorData); }); - -#endif // ENABLE_MULTI_DEVICE -} - -void Executor::Impl::initializeWorkers(SizeType32 tp, SizeType32 pp, SizeType32 cp, ParallelConfig& parallelConfig, - std::optional<runtime::WorldConfig> const& worldConfig, - std::optional<runtime::GptJsonConfig> const& decoderGptJsonConfig) -{ - auto const& worldComm = tensorrt_llm::mpi::MpiComm::world(); - int32_t const worldSize = worldComm.getSize(); - - auto const& orchestratorConfig = parallelConfig.getOrchestratorConfig(); - mIsOrchestrator = mCommMode == CommunicationMode::kORCHESTRATOR && orchestratorConfig.value().getIsOrchestrator(); - - TLLM_CHECK_WITH_INFO(mCommMode != CommunicationMode::kORCHESTRATOR || orchestratorConfig.has_value(), - "When using ORCHESTRATOR mode, orchestrator config must be set"); - - if (mCommMode == CommunicationMode::kORCHESTRATOR && !orchestratorConfig.value().getSpawnProcesses()) - { - TLLM_CHECK_WITH_INFO(parallelConfig.getParticipantIds(), - "When not spawning processes in orchestrator mode, participant IDs must be provided"); - - // Check that rank 0 is reserved for the orchestrator - auto const participantIds = parallelConfig.getParticipantIds().value(); - for (auto const& participantId : participantIds) - { - TLLM_CHECK_WITH_INFO(participantId != 0, "Rank 0 is reserved for the orchestrator"); - } - } - - // Participant ids - std::vector<SizeType32> participantIds; - if (!parallelConfig.getParticipantIds()) - { - TLLM_CHECK_WITH_INFO(worldSize == tp * pp * cp, - "With communicationMode kLEADER, MPI worldSize is expected to be equal to tp*pp*cp when " - "participantIds are not specified"); - - participantIds.resize(tp * pp * cp); - std::iota(participantIds.begin(), participantIds.end(), 0); - } - else - { - if (mCommMode == CommunicationMode::kORCHESTRATOR && orchestratorConfig.value().getSpawnProcesses()) - { - TLLM_THROW( - "Participant ids should not be set when using CommunicationMode::kORCHESTRATOR with " - "spawnProcesses=true"); - } - participantIds = parallelConfig.getParticipantIds().value(); - TLLM_CHECK_WITH_INFO(static_cast<SizeType32>(participantIds.size()) == tp * pp * cp, - tensorrt_llm::common::fmtstr("When specifying participantIds, participantIds size (%lu) must be equal to " - "tp*pp*cp (tp is %u, pp is %u, cp is %u)", - participantIds.size(), tp, pp, cp)); - } - - // If deviceIds are specified, check that they match tp*pp*cp - if (parallelConfig.getDeviceIds()) - { - auto deviceIds = parallelConfig.getDeviceIds().value(); - auto const hasNumNodes = parallelConfig.getNumNodes().has_value(); - if (hasNumNodes || static_cast<SizeType32>(deviceIds.size()) != tp * pp * cp) - { - auto const numNodes = hasNumNodes ? parallelConfig.getNumNodes().value() : tensorrt_llm::mpi::getNumNodes(); - TLLM_CHECK_WITH_INFO(static_cast<SizeType32>(deviceIds.size() * numNodes) == tp * pp * cp, - tensorrt_llm::common::fmtstr("When specifying deviceIds, deviceIds (%lu) * numNodes (%u) must be equal " - "to tp*pp*cp (tp is %u, pp is %u, cp is %u)", - deviceIds.size(), numNodes, tp, pp, cp)); - } - } - - // Bool that indicates if current process is worker for this model or not - auto participantIt = std::find(participantIds.begin(), participantIds.end(), mWorldRank); - mIsWorker = participantIt != participantIds.end(); - // Bool that indicates if current ranks is leader for this model - mIsLeader = (mWorldRank == participantIds.front()); - mIsPipelineLeader = (mWorldRank == participantIds[tp * (pp - 1)]); - -#if ENABLE_MULTI_DEVICE - if (mIsWorker) - { - // Create a session, but only assign to COMM_SESSION for ranks participating in this model - MPI_Group worldGroup = MPI_GROUP_NULL; - MPICHECK(MPI_Comm_group(MPI_COMM_WORLD, &worldGroup)); // NOLINT - MPI_Group sessionGroup = MPI_GROUP_NULL; - if (pp > 1) - { - // reverse participantIds to move leader to last pp rank. retain order in each tp group - std::reverse(participantIds.begin(), participantIds.end()); - if (tp > 1) - { - for (SizeType32 ppRank = 0; ppRank < pp; ppRank++) - { - std::reverse(participantIds.begin() + ppRank * tp, participantIds.begin() + (ppRank + 1) * tp); - } - } - } - MPICHECK(MPI_Group_incl(worldGroup, participantIds.size(), participantIds.data(), &sessionGroup)); // NOLINT - MPI_Comm sessionComm = MPI_COMM_NULL; - MPICHECK( - MPI_Comm_create_group(MPI_COMM_WORLD, sessionGroup, 1000 + participantIds.front(), &sessionComm)); // NOLINT - - tensorrt_llm::mpi::MpiComm::setSession(tensorrt_llm::mpi::MpiComm(sessionComm, false)); - } - - if (mIsLeader && mCommMode == CommunicationMode::kORCHESTRATOR) - { - auto optOrchestratorConfig = parallelConfig.getOrchestratorConfig(); - if (orchestratorConfig.has_value() && orchestratorConfig.value().getSpawnProcesses()) - { - mOrchLeaderComm = optOrchestratorConfig.value().getOrchLeaderComm(); - } - else - { - // mOrchLeaderComm has already been created - } - TLLM_CHECK(mOrchLeaderComm.get() != nullptr); - - TLLM_CHECK(worldConfig.has_value() || decoderGptJsonConfig.has_value()); - if (worldConfig.has_value()) - { - mDeviceId = worldConfig->getDevice(); - } - else - { - auto gpusPerNode = decoderGptJsonConfig->getGpusPerNode(); - auto worldConfig = runtime::WorldConfig::mpi(gpusPerNode, tp, pp, cp, parallelConfig.getDeviceIds()); - mDeviceId = worldConfig.getDevice(); - } - // Spawn the thread responsible for receiving new requests from the orchestrator - mLeaderRecvReqThread = std::thread(&Impl::leaderRecvReqThread, this); - - // Spawn the thread responsible for sending new responses to the orchestrator - mLeaderSendThread = std::thread([&]() - { this->leaderSendThread(mSendQueue, mpi::MpiTag::kOrchestratorId, mpi::MpiTag::kOrchestratorData); }); - } -#endif // ENABLE_MULTI_DEVICE -} - -void Executor::Impl::initializeLogitsPostProcessorBatched(LogitsPostProcessorConfig const& logitsProcConfig) -{ - if (logitsProcConfig.getProcessorBatched().has_value()) - { - mLogitsPostProcessorBatched - = [cb = logitsProcConfig.getProcessorBatched().value()]( - std::vector<batch_manager::LlmRequest::RequestIdType> const& reqIdsVec, - std::vector<batch_manager::LlmRequest::TensorPtr>& logitsVec, - std::vector<std::reference_wrapper<batch_manager::LlmRequest::BeamTokens const>> const& beamTokensVec, - CudaStreamPtr const& cudaStreamPtr, - std::vector<std::optional<batch_manager::LlmRequest::RequestIdType>> const& clientIdsVec) - { - std::vector<Tensor> cbLogitsVec; - cbLogitsVec.reserve(logitsVec.size()); - for (auto& logits : logitsVec) - { - cbLogitsVec.emplace_back(executor::detail::ofITensor(logits)); - } - - cb(reqIdsVec, cbLogitsVec, beamTokensVec, cudaStreamPtr, clientIdsVec); - }; - - mModel->setLogitsPostProcessorBatched(mLogitsPostProcessorBatched); - } -} - -IdType Executor::Impl::enqueueRequest(Request const& request) -{ - return enqueueRequests({&request, 1}).at(0); -} - -std::vector<IdType> Executor::Impl::enqueueRequests(std::vector<Request> const& requests) -{ - return enqueueRequests({requests.data(), requests.size()}); -} - -std::vector<IdType> Executor::Impl::enqueueRequests(common::ArrayView<Request const> const& requests) -{ - TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called, cannot enqueue requests"); - checkParallelApiUsage(__func__); - - TLLM_LOG_DEBUG("Enqueuing %lu requests", requests.size()); - std::vector<RequestWithId> requestWithIds; - requestWithIds.reserve(requests.size()); - - // First check valid of request in enqueue thread, so Exceptions can be thrown to user. - for (auto const& req : requests) - { - auto logitsPostProcessorName = req.getLogitsPostProcessorName(); - if (logitsPostProcessorName && logitsPostProcessorName.value() != Request::kBatchedPostProcessorName) - { - getLogitsPostProcessor(*logitsPostProcessorName); - } - } - - std::vector<IdType> ids; - { - auto now = std::chrono::steady_clock::now(); - for (auto const& req : requests) - { - ids.emplace_back(generateReqId(req)); - TLLM_LOG_DEBUG("Enqueue new request with id %d", ids.back()); - - std::vector<IdType> childReqIds; - auto numChildRequests = getNumChildRequests(req); - if (numChildRequests > 0) - { - childReqIds.reserve(numChildRequests); - for (int childId = 0; childId < numChildRequests; childId++) - { - childReqIds.emplace_back(generateLocalReqId()); - TLLM_LOG_DEBUG("Add new child request with id %d", childReqIds.back()); - } - } - requestWithIds.emplace_back(RequestWithId{req, ids.back(), std::move(childReqIds), now}); - } - } - - if (mCommMode == CommunicationMode::kLEADER) - { - { - std::scoped_lock<std::mutex> const lck(mQueuedReqMtx); - if (mMaxQueueSize) - { - auto const maxQueueSize = mMaxQueueSize.value(); - - auto totalRequestSize = 0; - for (auto&& reqWithId : requestWithIds) - { - totalRequestSize += (getNumChildRequests(reqWithId.req) + 1); - } - - if (maxQueueSize > 0 && mQueuedRequests.size() + totalRequestSize > static_cast<size_t>(maxQueueSize)) - { - TLLM_THROW("Maximum queue size of %d has been reached, please try again later", maxQueueSize); - } - } - - for (auto&& req : requestWithIds) - { - insertRequestInOrder(mQueuedRequests, std::move(req)); - } - } - mQueuedReqCv.notify_one(); - } - else if (mCommMode == CommunicationMode::kORCHESTRATOR) - { - MpiMessage message(MpiId::PENDING_REQUEST); - message.data = PendingRequestData{std::move(requestWithIds)}; - mSendQueue.push(std::move(message)); - } - return ids; -} - -std::vector<Response> Executor::Impl::awaitResponses(std::optional<std::chrono::milliseconds> const& timeout) -{ - TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); - checkParallelApiUsage(__func__); - std::unique_lock<std::mutex> lck(mResponsesMtx); - auto pred = [this]() -> bool { return !mResponses.empty() || mShutdown; }; - auto storeResponses = [this]() - { - std::vector<Response> responses; - for (auto it = mResponses.begin(); it != mResponses.end();) - { - responses.insert(responses.end(), it->second.begin(), it->second.end()); - addTerminatedReqId(it->second, it->first); - it = mResponses.erase(it); - } - return responses; - }; - - std::vector<Response> responses; - if (timeout) - { - if (mResponsesCv.wait_for(lck, timeout.value(), pred)) - { - responses = storeResponses(); - } - } - else - { - mResponsesCv.wait(lck, pred); - responses = storeResponses(); - } - return responses; -} - -std::vector<Response> Executor::Impl::awaitResponses( - IdType const& reqId, std::optional<std::chrono::milliseconds> const& timeout) -{ - TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); - checkParallelApiUsage(__func__); - std::unique_lock<std::mutex> lck(mResponsesMtx); - auto pred = [this, reqId]() -> bool - { return (mResponses.find(reqId) != mResponses.end() && !mResponses.at(reqId).empty()) || mShutdown; }; - auto storeIdResponse = [this, reqId]() - { - std::vector<Response> responses; - responses.swap(mResponses.at(reqId)); - mResponses.erase(reqId); - addTerminatedReqId(responses, reqId); - return responses; - }; - - // We don't process a terminated request again. Terminated request is defined as a response - // with isFinal = true for a given requestId. - if (mTerminatedReqIds.contains(reqId)) - { - if (mResponses.find(reqId) != mResponses.end()) - { - TLLM_THROW("ReqId should already be removed from responses!"); - } - std::string const err = "ReqId " + std::to_string(reqId) + " has already been processed and was terminated."; - TLLM_LOG_ERROR("%s", err.c_str()); - - return {Response(reqId, err)}; - } - - std::vector<Response> responses; - if (timeout) - { - if (mResponsesCv.wait_for(lck, timeout.value(), pred)) - { - responses = storeIdResponse(); - } - } - else - { - mResponsesCv.wait(lck, pred); - responses = storeIdResponse(); - } - return responses; -} - -std::vector<std::vector<Response>> Executor::Impl::awaitResponses( - std::vector<IdType> const& requestIds, std::optional<std::chrono::milliseconds> const& timeout) -{ - TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); - checkParallelApiUsage(__func__); - std::vector<std::vector<Response>> responses; - responses.reserve(requestIds.size()); - if (timeout) - { - auto const start_time = std::chrono::high_resolution_clock::now(); - for (auto const requestId : requestIds) - { - auto const elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::high_resolution_clock::now() - start_time); - responses.emplace_back(awaitResponses( - requestId, timeout.value() > elapsed_ms ? timeout.value() - elapsed_ms : std::chrono::milliseconds{0})); - } - } - else - { - for (auto const requestId : requestIds) - { - responses.emplace_back(awaitResponses(requestId)); - } - } - return responses; -} - -SizeType32 Executor::Impl::getNumResponsesReady(std::optional<IdType> const& optId) const -{ - TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); - checkParallelApiUsage(__func__); - std::scoped_lock<std::mutex> lck(mResponsesMtx); - SizeType32 numResponsesReady = 0; - if (optId) - { - auto const reqId = optId.value(); - auto const respIt = mResponses.find(reqId); - if (respIt != mResponses.end()) - { - numResponsesReady = static_cast<SizeType32>(respIt->second.size()); - } - } - else - { - for (auto const& [id, responses] : mResponses) - { - numResponsesReady += static_cast<SizeType32>(responses.size()); - } - } - return numResponsesReady; -} - -void Executor::Impl::shutdown() -{ - // Cannot call shutdown multiple times - if (mShutdownCalled) - { - return; - } - mShutdownCalled = true; - - if (!mShutdown) - { - if (mCommMode == CommunicationMode::kLEADER && mIsLeader) - { - // Enqueue a request to indicate to other ranks to terminate - enqueueTerminateRequest(); - } - else if (mCommMode == CommunicationMode::kORCHESTRATOR) - { - if (mIsOrchestrator) - { - // Send to the leader the termination signal - mShutdown = true; - mResponsesCv.notify_all(); - - mSendQueue.push(MpiMessage(MpiId::TERMINATION)); - - // Wait for sender thread to exit - if (mOrchSendReqThread.joinable()) - { - mOrchSendReqThread.join(); - } - // Wait for recv response thread to exit - if (mOrchRecvThread.joinable()) - { - mOrchRecvThread.join(); - } - } - else if (mIsLeader) - { - // Wait for sender thread to exit - if (mLeaderRecvReqThread.joinable()) - { - mLeaderRecvReqThread.join(); - } - // Wait for send response thread to exit - if (mLeaderSendThread.joinable()) - { - mLeaderSendThread.join(); - } - } - } - } - - // Wait for execution thread to terminate - if (mExecutionThread.joinable()) - { - mExecutionThread.join(); - } - - // If we overwrote COMM_SESSION with split, free it now. Otherwise, since - // COMM_SESSION is a global static object, it will be destroyed in an - // undefined order and can cause crashes on program exit. - if (mIsWorker) - { - tensorrt_llm::mpi::MpiComm::setSession(tensorrt_llm::mpi::MpiComm(MPI_COMM_WORLD, false)); - } -} - -void Executor::Impl::cancelRequest(IdType requestId) -{ - TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); - checkParallelApiUsage(__func__); - - // Check if the request is terminated already. If so, return - { - std::scoped_lock<std::mutex> lckResp(mResponsesMtx); - if (mTerminatedReqIds.contains(requestId)) - { - TLLM_LOG_INFO("Ignoring already terminated request %lu", requestId); - return; - } - } - - if (mCommMode == CommunicationMode::kLEADER) - { - std::scoped_lock<std::mutex> lck(mCancelReqMtx); - auto& selCancelledReqIds = mUsePipelineParallel ? mPipelineCancelledReqIds : mCancelledReqIds; - selCancelledReqIds.insert(requestId); - } - else if (mCommMode == CommunicationMode::kORCHESTRATOR) - { - MpiMessage message(MpiId::CANCEL_REQUEST); - std::vector<IdType> cancelledReqIds{requestId}; - message.data = RequestIdsData{std::move(cancelledReqIds)}; - mSendQueue.push(std::move(message)); - } -} - -std::deque<IterationStats> Executor::Impl::getLatestIterationStats() -{ - TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); - checkParallelApiUsage(__func__); - std::scoped_lock<std::mutex> lck(mIterStatsMtx); - return std::exchange(mIterationStats, {}); -} - -std::deque<RequestStatsPerIteration> Executor::Impl::getLatestRequestStats() -{ - TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); - checkParallelApiUsage(__func__); - - std::scoped_lock<std::mutex> lck(mRequestStatsMtx); - return std::exchange(mRequestStats, {}); -} - -std::deque<DebugTensorsPerIteration> Executor::Impl::getLatestDebugTensors() -{ - TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); - if (mCommMode == CommunicationMode::kORCHESTRATOR) - { - TLLM_LOG_WARNING("getLatestDebugTensors is not supported in ORCHESTRATOR mode yet"); - return {}; - } - if (mEncoderModel) - { - TLLM_LOG_WARNING("getLatestDebugTensors is not supported for encoder model yet"); - } - std::scoped_lock<std::mutex> lck(mDebugTensorsMtx); - return std::exchange(mDebugTensors, {}); -} - -bool Executor::Impl::canEnqueueRequests() const -{ - return !mShutdownCalled - && ((mCommMode == CommunicationMode::kLEADER && mIsLeader) - || (mCommMode == CommunicationMode::kORCHESTRATOR && mIsOrchestrator)); -} - -bool Executor::Impl::isParticipant() const -{ - return mIsWorker; -} - -std::optional<std::shared_ptr<KVCacheEventManager>> Executor::Impl::getKVCacheEventManager() const -{ - if (!mModel) - { - return std::nullopt; - } - auto cacheEventManager = mModel->getKVCacheManager(); - return cacheEventManager ? std::optional(std::make_shared<KVCacheEventManager>(cacheEventManager)) : std::nullopt; -} - -void Executor::Impl::requestWithIdLeaderThread() -{ - TLLM_CUDA_CHECK(cudaSetDevice(mModel->getWorldConfig().getDevice())); - auto constexpr peer = 0; - while (true) - { - int64_t numActiveRequests; - mCommPipelineParallel->recv( - &numActiveRequests, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kExecutorNumActiveRequests); - if (numActiveRequests < 0) - { - break; - } - - bool lowestPriorityActiveHasValue; - std::optional<PriorityType> lowestPriorityActive; - mCommPipelineParallel->recv(&lowestPriorityActiveHasValue, 1, mpi::MpiType::kBOOL, peer, - mpi::MpiTag::kExecutorLowestPriorityActiveHasValue); - if (lowestPriorityActiveHasValue) - { - PriorityType lowestPriorityActiveValue; - mCommPipelineParallel->recv( - &lowestPriorityActiveValue, 1, mpi::MpiType::kFLOAT, peer, mpi::MpiTag::kExecutorLowestPriorityActive); - lowestPriorityActive = lowestPriorityActiveValue; - } - - auto reqWithIds = getLeaderNewReqWithIds(numActiveRequests, lowestPriorityActive); - setupDynamicLogitsPostProcessors(reqWithIds); - auto requestWithIdAsyncSndHdl - = std::make_unique<RequestWithIdAsyncSend>(mCommPipelineParallel, reqWithIds, peer); - requestWithIdAsyncSndHdl.reset(nullptr); - } -} - -void Executor::Impl::cancelledRequestsLeaderThread() -{ - TLLM_CUDA_CHECK(cudaSetDevice(mModel->getWorldConfig().getDevice())); - auto constexpr peer = 0; - while (true) - { - bool shouldExit; - mCommPipelineParallel->recv(&shouldExit, 1, mpi::MpiType::kBOOL, peer, mpi::MpiTag::kExecutorShouldExit); - if (shouldExit) - { - break; - } - - std::unique_ptr<CancelledRequestsAsyncSend> cancelledRequestsAsyncSndHdl; - { - std::scoped_lock<std::mutex> lck(mCancelReqMtx); - cancelledRequestsAsyncSndHdl - = std::make_unique<CancelledRequestsAsyncSend>(mCommPipelineParallel, mPipelineCancelledReqIds, peer); - mPipelineCancelledReqIds.clear(); - } - cancelledRequestsAsyncSndHdl.reset(nullptr); - } -} - -std::vector<RequestWithId> Executor::Impl::getLeaderNewReqWithIds( - SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive) -{ - std::unique_lock<std::mutex> lck(mQueuedReqMtx); - mQueuedReqCv.wait(lck, [&]() { return (!mQueuedRequests.empty() || numActiveRequests > 0 || mShutdown); }); - - std::vector<RequestWithId> reqWithIds; - - if (mQueuedRequests.empty() || mShutdown) - { - return reqWithIds; - } - - if (mQueuedRequests.front().id == kTerminateReqId) - { - reqWithIds.emplace_back(std::move(mQueuedRequests.front())); - mQueuedRequests.pop_front(); - return reqWithIds; - } - - auto const& firstRequest = mQueuedRequests.front(); - auto const firstBeamWidth = firstRequest.req.getSamplingConfig().getBeamWidth(); - auto const operatingBeamWidth = numActiveRequests > 0 ? mModel->getOperatingBeamWidth() : firstBeamWidth; - - auto const tryInsertQueuedRequestIntoReqWithIds = [this, &reqWithIds, operatingBeamWidth]() -> bool - { - auto& nextRequest = mQueuedRequests.front(); - auto const beamWidth = nextRequest.req.getSamplingConfig().getBeamWidth(); - if (beamWidth != operatingBeamWidth) - { - TLLM_LOG_INFO( - "Can't dequeue request with ID %ld because beam width %d differs from operating beam width %d.", - nextRequest.id, beamWidth, operatingBeamWidth); - return false; - } - - TLLM_LOG_DEBUG("Dequeue request with ID %ld", nextRequest.id); - reqWithIds.emplace_back(std::move(nextRequest)); - mQueuedRequests.pop_front(); - return true; - }; - - auto const maxNewRequests = static_cast<size_t>(std::max(mMaxNumActiveRequests - numActiveRequests, 0)); - for (size_t req = 0; !mQueuedRequests.empty() && req < maxNewRequests;) - { - req += (getNumChildRequests(mQueuedRequests.front().req) + 1); - if (req > maxNewRequests) - { - break; - } - if (!tryInsertQueuedRequestIntoReqWithIds()) - { - break; - } - } - - if (lowestPriorityActive) - { - while (!mQueuedRequests.empty() && mQueuedRequests.front().req.getPriority() > (*lowestPriorityActive)) - { - if (!tryInsertQueuedRequestIntoReqWithIds()) - { - break; - } - } - } - return reqWithIds; -} - -std::vector<RequestWithId> Executor::Impl::getNewReqWithIds( - SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - auto const& worldConfig = mModel->getWorldConfig(); - - if (worldConfig.isPipelineParallel()) - { - mRequestWithIdWaitThread->waitStop(); - } - - TLLM_CUDA_CHECK(cudaSetDevice(mModel->getWorldConfig().getDevice())); - std::vector<RequestWithId> reqWithIds; - if (mIsPipelineLeader) - { - if (!worldConfig.isPipelineParallel()) - { - reqWithIds = getLeaderNewReqWithIds(numActiveRequests, lowestPriorityActive); - setupDynamicLogitsPostProcessors(reqWithIds); - } - else - { - auto const peer = worldConfig.getPipelineParallelism() - 1; - auto numActiveRequestsValue = static_cast<int64_t>(numActiveRequests); - auto request1 = mCommPipelineParallel->sendAsync( - &numActiveRequestsValue, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kExecutorNumActiveRequests); - bool lowestPriorityActiveHasValue = lowestPriorityActive.has_value(); - auto request2 = mCommPipelineParallel->sendAsync(&lowestPriorityActiveHasValue, 1, mpi::MpiType::kBOOL, - peer, mpi::MpiTag::kExecutorLowestPriorityActiveHasValue); - auto request3 = lowestPriorityActiveHasValue - ? mCommPipelineParallel->sendAsync(&lowestPriorityActive.value(), 1, mpi::MpiType::kFLOAT, peer, - mpi::MpiTag::kExecutorLowestPriorityActive) - : nullptr; - request1->wait(); - request2->wait(); - if (request3) - { - request3->wait(); - } - reqWithIds = RequestWithIdAsyncSend::requestWithIdRecv(mCommPipelineParallel, peer); - } - if (worldConfig.isTensorParallel() || worldConfig.isContextParallel()) - { - auto packed = RequestWithId::serializeReqWithIds(reqWithIds); - if (worldConfig.isTensorParallel()) - { - mCommTensorParallel->bcast(packed, 0); - } - if (worldConfig.isContextParallel()) - { - mCommContextParallel->bcast(packed, 0); - } - } - } - else - { - if (worldConfig.isFirstPipelineParallelRank()) - { - std::vector<char> buffer; - mCommTensorParallel->bcast(buffer, 0); - mCommContextParallel->bcast(buffer, 0); - reqWithIds = RequestWithId::deserializeReqWithIds(buffer); - } - else - { - auto const peer = worldConfig.getPipelineParallelRank() - 1; - reqWithIds = RequestWithIdAsyncSend::requestWithIdRecv(mCommPipelineParallel, peer); - } - } - if (!worldConfig.isLastPipelineParallelRank()) - { - auto const peer = worldConfig.getPipelineParallelRank() + 1; - mRequestWithIdAsyncSndHdl = std::make_unique<RequestWithIdAsyncSend>(mCommPipelineParallel, reqWithIds, peer); - mRequestWithIdWaitThread->notifyStart(); - } - TLLM_CUDA_CHECK(cudaSetDevice(mModel->getWorldConfig().getDevice())); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return reqWithIds; -} - -std::tuple<Executor::Impl::RequestList, double> Executor::Impl::fetchNewRequests( - SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_SCOPED_RANGE(fetchNewRequests); - - // If grab requests from queue, do exchange between ranks - auto reqWithIds = getNewReqWithIds(numActiveRequests, lowestPriorityActive); - RequestList newRequests; - double newActiveRequestsQueueLatencyMS{0.}; - for (auto& reqWithId : reqWithIds) - { - if (reqWithId.id == kTerminateReqId) - { - mShutdown = true; - mResponsesCv.notify_all(); - return {}; - } - - try - { - std::optional<LlmRequestLogitsPostProcessor> llmRequestLogitsPostProcessor; - bool applyLogitsPostProcessorBatched{false}; - if (mModel->getWorldConfig().isLastPipelineParallelRank()) - { - auto logitsPostProcessorName = reqWithId.req.getLogitsPostProcessorName(); - if (logitsPostProcessorName) - { - if (logitsPostProcessorName.value() == Request::kBatchedPostProcessorName) - { - TLLM_CHECK_WITH_INFO( - mLogitsPostProcessorBatched, "Batched logits post processor is not defined."); - applyLogitsPostProcessorBatched = true; - } - else - { - if (logitsPostProcessorName->compare(0, - std::char_traits<char>::length(Request::kDynamicPostProcessorNamePrefix), - Request::kDynamicPostProcessorNamePrefix) - == 0) - { - TLLM_CHECK_WITH_INFO(!mModel->getReplicateLogitsPostProcessor() - || mModel->getWorldConfig().getTensorParallelism() == 1, - "Dynamic logits postprocessor must be used with replicate=false or no tensor " - "parallelism."); - } - if (mModel->getWorldConfig().isFirstTensorParallelRank() - || mModel->getReplicateLogitsPostProcessor()) - { - llmRequestLogitsPostProcessor = getLogitsPostProcessor(logitsPostProcessorName.value()); - } - else - { - llmRequestLogitsPostProcessor - = [](IdType reqId, RtTensorPtr& logits, BeamTokens const& beamTokens, - CudaStreamPtr const& cudaStreamPtr, std::optional<IdType> clientId) {}; - } - } - } - } - auto newLlmReq = std::make_shared<batch_manager::LlmRequest>( - reqWithId.id, reqWithId.req, llmRequestLogitsPostProcessor, applyLogitsPostProcessorBatched); - - auto numReturnSequences = newLlmReq->getNumSubRequests(); - if (numReturnSequences > 1) - { - TLLM_CHECK(reqWithId.childReqIds.size() == static_cast<size_t>(numReturnSequences - 1)); - mChildReqIdsMap[reqWithId.id] = reqWithId.childReqIds; - } - - for (auto seqIdx = 0; seqIdx < numReturnSequences; seqIdx++) - { - auto newReq - = seqIdx == 0 ? newLlmReq : newLlmReq->createChildRequest(reqWithId.childReqIds.at(seqIdx - 1)); - - // If static batching and streaming, disable streaming and exclude input - if (mBatchingType == BatchingType::kSTATIC && newReq->isStreaming()) - { - newReq->setStreaming(false); - newReq->setExcludeInputFromOutput(true); - } - - // Validate the request parameters - newReq->validate(mModel->getMaxInputLen(), mModel->getMaxSequenceLen(), mModel->getMaxDraftLen(), - mModel->getVocabSizePadded(), - mEncoderModel ? std::optional<SizeType32>(mEncoderModel->getMaxInputLen()) : std::nullopt, - mEnableBlockReuse); - - TLLM_CHECK_WITH_INFO(!mEncoderModel || !mIsSchedulerMaxUtilization, - "Encoder or Encoder-Decoder model don't support max utilization scheduler yet. Only max requests " - "or guaranteed no evict."); - - // When streaming is enabled and scheduling policy permits evict/restart, need to guard against the case - // where the sequence is truncated on eviction (to respect maxInputLen limits), resulting in loss of - // some tokens that have been streamed out. In this case, resuming generation may result in different - // completion for locations whose tokens have already been returned. There is no way to protect against - // this, so disallowing. - if (newReq->isStreaming() && !mIsSchedulerGuaranteedNoEvict && !mIsChunkedContext) - { - auto const maxReqSeqLen = newReq->mPromptLen + newReq->mMaxNewTokens; - auto const maxRestartLen = maxReqSeqLen - 1; - TLLM_CHECK_WITH_INFO(maxRestartLen <= mModel->getMaxInputLen(), - "Request sequence length is potentially greater than max input length. This cannot be run " - "unless streaming is disabled, context chunking is enabled or the GUARANTEED_NO_EVICT " - "scheduling policy is used"); - } - - // Create the encoder output tensor - if (mEncoderModel) - { - TLLM_CHECK_WITH_INFO(mModel || (!mModel && newReq->getReturnEncoderOutput()), - "Encoder-Decoder models allow optionally returning encoder output. But if it is Encoder-only " - "models, please make sure returnEncoderOutput is always true."); - - // gpu buffers for passing to the next phase - newReq->allocEncoderOutput(mEncoderModel->getBufferManager(), mEncoderModel->getLogitDataType()); - newReq->allocEncoderHiddenStates( - mEncoderModel->getBufferManager(), mEncoderModel->getLogitDataType()); - // pinned buffers for returning results to host - if (newReq->getReturnEncoderOutput()) - { - newReq->allocEncoderOutputHost( - mEncoderModel->getHiddenSize() * mEncoderModel->getWorldConfig().getTensorParallelism(), - mEncoderModel->getLogitDataType()); - } - } - - if (!mEncoderModel && newReq->getEncoderInputFeatures()) - { - TLLM_LOG_INFO("Allocating buffers for encoder output"); - // gpu buffers for passing to the next phase - newReq->allocEncoderOutput(mModel->getBufferManager(), mModel->getLogitDataType()); - newReq->allocEncoderHiddenStates(mModel->getBufferManager(), mModel->getLogitDataType()); - } - - // Create the context logits tensor - if (newReq->getReturnContextLogits()) - { - TLLM_CHECK_WITH_INFO(mModel->getModelConfig().computeContextLogits(), - "Return context logit need to build engine with gather_context_logits"); - newReq->allocContextLogitsHost(mModel->getVocabSizePadded(), mModel->getLogitDataType()); - } - - // Create the generation logits tensor - if (newReq->getReturnGenerationLogits()) - { - TLLM_CHECK_WITH_INFO(mModel->getGatherGenerationLogits(), - "To return generation logits, gather_generation_logits must be enabled in ExecutorConfig"); - - if (mModel->getModelConfig().getSpeculativeDecodingMode().isDraftTokensExternal() - && newReq->hasDraftTokens()) - { - newReq->allocTargetModelAcceptedTokenLogitsHost( - mModel->getVocabSizePadded(), mModel->getLogitDataType()); - } - else - { - newReq->allocGenerationLogitsHost(mModel->getVocabSizePadded(), mModel->getLogitDataType()); - } - } - - if (mModel->getWorldConfig().isLastPipelineParallelRank() && newReq->getGuidedDecodingParams()) - { - TLLM_CHECK_WITH_INFO(mModel->hasGuidedDecoder(), - "Request is specified with GuidedDecodingParams, but GuidedDecoder is not setup. Please " - "provide a valid GuidedDecodingConfig to setup GuidedDecoder."); - } - - if (mModel->getWorldConfig().isLastPipelineParallelRank() && newReq->hasAdditionalOutputs()) - { - newReq->allocAdditionalOutputs([this](std::string const& name) - { return mModel->getTensorDataType(name); }, - [this](std::string const& name) { return mModel->getTensorShape(name); }); - } - - mModel->updatePeftCache(newReq); - - newRequests.emplace_back(std::move(newReq)); - } - - auto queuedEnd = std::chrono::steady_clock::now(); - auto reqQueueLatencyMS - = std::chrono::duration<double, std::milli>(queuedEnd - reqWithId.queuedStart).count(); - newActiveRequestsQueueLatencyMS += reqQueueLatencyMS; - } - catch (runtime::LoraExpectedException const& e) - { - if (mIsLeader) - { - // In case of an expected LoRA exception (e.g. cache full, cache miss), log a warning and enqueue - // response - TLLM_LOG_WARNING("%s", e.what()); - enqueueNewResponses({{reqWithId.id, e.what(), reqWithId.req.getClientId()}}); - } - } - catch (std::exception const& e) - { - if (mIsLeader) - { - // In case of error, create a response with error for this request - auto err = std::string("Encountered an error when fetching new request: ") + e.what(); - TLLM_LOG_ERROR("%s", err.c_str()); - enqueueNewResponses({{reqWithId.id, err, reqWithId.req.getClientId()}}); - } - } - } - TLLM_LOG_DEBUG("[RANK %d] num new requests fetched from queue: %d", COMM_SESSION.getRank(), newRequests.size()); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return {newRequests, newActiveRequestsQueueLatencyMS}; -} - -void Executor::Impl::terminateActiveRequests(RequestList& activeRequests, std::string const& err) -{ - TLLM_LOG_ERROR("%s", err.c_str()); - - // Create a response for all requests and add to queue - for (auto it = activeRequests.cbegin(); it != activeRequests.cend();) - { - auto llmReq = (*it); - - llmReq->setState(batch_manager::LlmRequestState::kGENERATION_COMPLETE); - mModel->terminateRequest(llmReq); - - if (mIsLeader) - { - enqueueNewResponses({{llmReq->mRequestId, err, llmReq->mClientId}}); - } - - // Remove from the requestList - it = activeRequests.erase(it); - } -} - -void Executor::Impl::forwardSync(RequestList& activeRequests) -{ - TLLM_LOG_TRACE("[RANK %d] %s start", COMM_SESSION.getRank(), __PRETTY_FUNCTION__); - try - { - if (mEncoderModel) - { - mEncoderModel->forwardSync(); - } - mModel->forwardSync(); - } - catch (std::exception const& e) - { - std::string const err = std::string("Encountered an error in forwardSync function: ") + e.what(); - terminateActiveRequests(activeRequests, err); - } - TLLM_LOG_TRACE("[RANK %d] %s stop", COMM_SESSION.getRank(), __PRETTY_FUNCTION__); -} - -// The function is used to change the state of a request to context_init from encoder_init for enc-dec model whose -// encoder is skipped. The encoder output is populated accordingly with input features given through model executor of -// decoder. -void Executor::Impl::prepRequestsForEncoderSkip(RequestList& activeRequests) -{ - - for (auto& req : activeRequests) - { - - if (req->isEncoderInitState() && req->getEncoderInputFeatures()) - { - TLLM_LOG_INFO("Changing state of request and setting encoder output to skip encoder run"); - req->setState(batch_manager::LlmRequestState::kCONTEXT_INIT); - req->setEncoderOutput(req->getEncoderInputFeatures()); - } - } -} - -void Executor::Impl::finishTimedOutRequests(RequestList const& activeRequests) -{ - if (mIsLeader) - { - for (auto const& request : activeRequests) - { - if (request->isTimedOut() && !request->isFinished()) - { - // workaround to cancelRequest since it throws an error if - // mCommMode == CommunicationMode::kORCHESTRATOR && !mIsOrchestrator - { - std::scoped_lock<std::mutex> lck(mCancelReqMtx); - auto& selCancelledReqIds = mUsePipelineParallel ? mPipelineCancelledReqIds : mCancelledReqIds; - selCancelledReqIds.insert(request->mRequestId); - } - } - } - } -} - -void Executor::Impl::forwardAsync(RequestList& activeRequests) -{ - try - { - TLLM_LOG_DEBUG("num active requests in scope: %d", activeRequests.size()); - - if (mDynamicBatchTuner) - { - auto const averageInputLength = static_cast<SizeType32>(mDynamicBatchTuner->getAverageInputLength()); - auto const averageOutputLength = static_cast<SizeType32>(mDynamicBatchTuner->getAverageOutputLength()); - auto const maxCapacityBatchSize = mModel->getMaxCapacityBatchSize(averageInputLength, averageOutputLength); - - if (mDynamicBatchTuner->isBatchSizeTuningEnabled()) - { - auto runtimeBatchSize = mDynamicBatchTuner->getRuntimeBatchSize(maxCapacityBatchSize); - mModel->setRuntimeBatchSize(runtimeBatchSize); - } - - if (mDynamicBatchTuner->isMaxNumTokensTuningEnabled()) - { - auto runtimeBatchSize = mModel->getRuntimeBatchSize(); - auto runtimeMaxNumTokens = mDynamicBatchTuner->getRuntimeMaxNumTokens(runtimeBatchSize); - mModel->setRuntimeMaxNumTokens(runtimeMaxNumTokens); - } - } - - if (mEncoderModel) - { - mEncoderModel->forwardAsync(activeRequests); - auto const& encoderStream = *(mEncoderModel->getRuntimeStreamPtr()); - auto const& decoderStream = *(mModel->getRuntimeStreamPtr()); - runtime::CudaEvent encoderFinished; - encoderStream.record(encoderFinished); - decoderStream.wait(encoderFinished); - } - - if (!mEncoderModel) - { - prepRequestsForEncoderSkip(activeRequests); - } - - mModel->forwardAsync(activeRequests); - } - catch (std::exception const& e) - { - std::string err = std::string("Encountered an error in forwardAsync function: ") + e.what(); - terminateActiveRequests(activeRequests, err); - } -} - -IterationStats Executor::Impl::getCurrentIterationStats(RequestList const& activeRequests, double iterLatencyMS, - SizeType32 numNewActiveRequests, double newActiveRequestsQueueLatencyMS, SizeType32 numCompletedRequests) -{ - IterationStats stats; - // Timestamp - stats.timestamp = tensorrt_llm::common::getCurrentTimestamp(); - stats.numNewActiveRequests = numNewActiveRequests; - stats.iterLatencyMS = iterLatencyMS; - stats.newActiveRequestsQueueLatencyMS = newActiveRequestsQueueLatencyMS; - // Active request count - stats.numActiveRequests = static_cast<SizeType32>(activeRequests.size()); - // Queued request count - { - std::scoped_lock<std::mutex> lck(mQueuedReqMtx); - stats.numQueuedRequests = static_cast<SizeType32>(mQueuedRequests.size()); - } - stats.numCompletedRequests = numCompletedRequests; - // Max number of requests - stats.maxNumActiveRequests = mMaxNumActiveRequests; - // Runtime memory allocation statistics - auto const& memoryCounters = runtime::MemoryCounters::getInstance(); - stats.gpuMemUsage = memoryCounters.getGpu(); - stats.cpuMemUsage = memoryCounters.getCpu(); - stats.pinnedMemUsage = memoryCounters.getPinned(); - - // Model specific stats - mModel->getCurrentIterationStats(stats); - return stats; -} - -RequestStatsPerIteration Executor::Impl::getCurrentRequestStats( - RequestList const& activeRequests, RequestList const& finishedRequests) -{ - std::vector<RequestStats> requestStatsVec; - - auto includeDisServingStats = [](LlmRequestPtr const& request, tensorrt_llm::executor::RequestStats& requestStats) - { - auto requestType = request->getLlmRequestType(); - if (requestType == batch_manager::LlmRequestType::LLMREQUEST_TYPE_CONTEXT_ONLY - || requestType == batch_manager::LlmRequestType::LLMREQUEST_TYPE_GENERATION_ONLY) - { - requestStats.disServingStats - = executor::DisServingRequestStats{request->getKvCacheTransferTimeMS(), request->getKvCacheSize()}; - } - }; - - for (auto const& request : activeRequests) - { - RequestStats requestStats; - requestStats.id = request->mRequestId; - requestStats.stage = request->getRequestStage(); - requestStats.contextPrefillPosition = request->getContextCurrentPosition(); - requestStats.numGeneratedTokens = request->getMaxBeamNumTokens() - request->getOrigPromptLen(); - requestStats.avgNumDecodedTokensPerIter = request->getAvgDecodedTokensPerIter(); - includeDisServingStats(request, requestStats); - requestStats.allocTotalBlocksPerRequest = request->getAllocTotalBlocksPerRequest(); - requestStats.allocNewBlocksPerRequest = request->getAllocNewBlocksPerRequest(); - requestStats.reusedBlocksPerRequest = request->getReusedBlocksPerRequest(); - requestStats.missedBlocksPerRequest = request->getMissedBlocksPerRequest(); - requestStats.kvCacheHitRatePerRequest = request->getKVCacheHitRatePerRequest(); - requestStatsVec.emplace_back(requestStats); - } - - { - std::unique_lock<std::mutex> lck(mQueuedReqMtx); - for (auto const& request : mQueuedRequests) - { - // Still waiting for the first scheduling - RequestStats requestStats; - requestStats.id = static_cast<executor::IdType>(request.id); - requestStats.stage = executor::RequestStage::kQUEUED; - requestStats.contextPrefillPosition = 0; - requestStats.numGeneratedTokens = 0; - requestStats.avgNumDecodedTokensPerIter = 0; - requestStats.allocTotalBlocksPerRequest = 0; - requestStats.allocNewBlocksPerRequest = 0; - requestStats.reusedBlocksPerRequest = 0; - requestStats.missedBlocksPerRequest = 0; - requestStats.kvCacheHitRatePerRequest = 0; - requestStatsVec.emplace_back(requestStats); - } - } - - for (auto const& request : finishedRequests) - { - // Still waiting for the first scheduling - RequestStats requestStats; - requestStats.id = static_cast<executor::IdType>(request->mRequestId); - requestStats.stage = executor::RequestStage::kGENERATION_COMPLETE; - requestStats.contextPrefillPosition = request->getContextCurrentPosition(); - requestStats.numGeneratedTokens = request->getMaxBeamNumTokens() - request->getOrigPromptLen(); - requestStats.avgNumDecodedTokensPerIter = request->getAvgDecodedTokensPerIter(); - includeDisServingStats(request, requestStats); - requestStats.allocTotalBlocksPerRequest = request->getAllocTotalBlocksPerRequest(); - requestStats.allocNewBlocksPerRequest = request->getAllocNewBlocksPerRequest(); - requestStats.reusedBlocksPerRequest = request->getReusedBlocksPerRequest(); - requestStats.missedBlocksPerRequest = request->getMissedBlocksPerRequest(); - requestStats.kvCacheHitRatePerRequest = request->getKVCacheHitRatePerRequest(); - requestStatsVec.emplace_back(requestStats); - } - - RequestStatsPerIteration stats{0, std::move(requestStatsVec)}; - - // Model specific stats - mModel->getCurrentRequestStats(stats); - return stats; -} - -void Executor::Impl::appendCurrentIterStats(IterationStats&& currentIterStats) -{ - std::scoped_lock<std::mutex> lck(mIterStatsMtx); - if (statsBufferIsBounded(mIterStatsMaxIterations)) - { - auto const maxIterStats = static_cast<std::size_t>(mIterStatsMaxIterations); - if (mIterationStats.size() >= maxIterStats) - { - mIterationStats.pop_front(); - } - } - mIterationStats.emplace_back(std::move(currentIterStats)); -} - -void Executor::Impl::appendMultipleIterStats(std::vector<IterationStats>&& currentIterStatsVec) -{ - std::scoped_lock<std::mutex> lck(mIterStatsMtx); - mIterationStats.insert(mIterationStats.end(), std::make_move_iterator(currentIterStatsVec.begin()), - std::make_move_iterator(currentIterStatsVec.end())); - if (statsBufferIsBounded(mIterStatsMaxIterations)) - { - auto const maxIterStats = static_cast<std::size_t>(mIterStatsMaxIterations); - while (mIterationStats.size() > maxIterStats) - { - mIterationStats.pop_front(); - } - } -} - -void Executor::Impl::updateIterationStats(RequestList const& activeRequests, double iterLatencyMS, - SizeType32 numNewActiveRequests, double newActiveRequestsQueueLatencyMS, SizeType32 numCompletedRequests, - bool flushToOrchestrator) -{ - NVTX3_SCOPED_RANGE(updateIterationStats); - if (statsBufferIsEnabled(mIterStatsMaxIterations) && mIsLeader) - { - auto currentIterStats = getCurrentIterationStats( - activeRequests, iterLatencyMS, numNewActiveRequests, newActiveRequestsQueueLatencyMS, numCompletedRequests); - // Send the stats to the orchestrator - if (mCommMode == CommunicationMode::kORCHESTRATOR) - { - bool hasSchedThisIter = (currentIterStats.inflightBatchingStats - && currentIterStats.inflightBatchingStats->numScheduledRequests > 0) - || (currentIterStats.staticBatchingStats - && currentIterStats.staticBatchingStats->numScheduledRequests > 0); - appendCurrentIterStats(std::move(currentIterStats)); - if (hasSchedThisIter || flushToOrchestrator) - { - std::deque<IterationStats> iterStatsQueue; - { - std::scoped_lock<std::mutex> lck(mIterStatsMtx); - iterStatsQueue = std::exchange(mIterationStats, {}); - } - MpiMessage message(MpiId::ITER_STATS); - std::vector<IterationStats> iterStates( - std::make_move_iterator(iterStatsQueue.begin()), std::make_move_iterator(iterStatsQueue.end())); - message.data = IterStatsData{std::move(iterStates)}; - mSendQueue.push(std::move(message)); - } - } - else - { - // Add current iteration stats - appendCurrentIterStats(std::move(currentIterStats)); - } - } -} - -void Executor::Impl::appendCurrentRequestStats(RequestStatsPerIteration&& currentRequestStats) -{ - std::scoped_lock<std::mutex> lck(mRequestStatsMtx); - if (statsBufferIsBounded(mRequestStatsMaxIterations)) - { - auto const maxRequestStats = static_cast<std::size_t>(mRequestStatsMaxIterations); - if (mRequestStats.size() >= maxRequestStats) - { - mRequestStats.pop_front(); - } - } - mRequestStats.emplace_back(std::move(currentRequestStats)); -} - -void Executor::Impl::appendMultipleRequestStats(std::vector<RequestStatsPerIteration>&& currentRequestStatsVec) -{ - std::scoped_lock<std::mutex> lck(mRequestStatsMtx); - mRequestStats.insert(mRequestStats.end(), std::make_move_iterator(currentRequestStatsVec.begin()), - std::make_move_iterator(currentRequestStatsVec.end())); - if (statsBufferIsBounded(mRequestStatsMaxIterations)) - { - auto const maxRequestStats = static_cast<std::size_t>(mRequestStatsMaxIterations); - while (mRequestStats.size() > maxRequestStats) - { - mRequestStats.pop_front(); - } - } -} - -void Executor::Impl::updateRequestStats( - RequestList const& activeRequests, RequestList const& finishedRequests, bool flushToOrchestrator) -{ - NVTX3_SCOPED_RANGE(updateRequestStats); - if (statsBufferIsEnabled(mRequestStatsMaxIterations) && mIsLeader) - { - // Add current iteration request stats - auto currentRequestStats = getCurrentRequestStats(activeRequests, finishedRequests); - // Send the stats to the orchestrator - if (mCommMode == CommunicationMode::kORCHESTRATOR) - { - bool hasScheduledReqs = false; - if (!flushToOrchestrator) - { - size_t activeSize = activeRequests.size(); - TLLM_CHECK_WITH_INFO(currentRequestStats.requestStats.size() >= activeSize, - "currentRequestStats num is %ld should >= activeRequest num:%zu", - currentRequestStats.requestStats.size(), activeSize); - hasScheduledReqs = std::any_of(currentRequestStats.requestStats.begin(), - currentRequestStats.requestStats.begin() + static_cast<int64_t>(activeSize), - [](RequestStats const& requestStat) { return requestStat.scheduled; }); - } - appendCurrentRequestStats(std::move(currentRequestStats)); - if (hasScheduledReqs || flushToOrchestrator) - { - std::deque<RequestStatsPerIteration> requestStatsQueue; - { - std::scoped_lock<std::mutex> lck(mRequestStatsMtx); - requestStatsQueue = std::exchange(mRequestStats, {}); - } - std::vector<RequestStatsPerIteration> requestIterStates( - std::make_move_iterator(requestStatsQueue.begin()), - std::make_move_iterator(requestStatsQueue.end())); - MpiMessage message(MpiId::REQUEST_ITER_STATS); - message.data = RequestStatsPerIterationData{std::move(requestIterStates)}; - mSendQueue.push(std::move(message)); - } - } - else - { - // Add current iteration stats - appendCurrentRequestStats(std::move(currentRequestStats)); - } - } -} - -void Executor::Impl::appendCurrentDebugTensors() -{ - if (mDebugTensorsMaxIterations > 0) - { - std::scoped_lock<std::mutex> lck(mDebugTensorsMtx); - if (mDebugTensors.size() >= mDebugTensorsMaxIterations) - { - mDebugTensors.pop_front(); - } - mDebugTensors.emplace_back(mModel->getCurrentDebugTensors()); - } -} - -void Executor::Impl::terminateCancelledRequests(RequestList& activeRequests) -{ - NVTX3_SCOPED_RANGE(terminateCancelledRequests); - auto const& worldConfig = mModel->getWorldConfig(); - auto const broadcastCancelledRequests = [this, &activeRequests, &worldConfig] - { - auto const& commSession = COMM_SESSION; - - if (worldConfig.isPipelineParallel()) - { - mCancelledRequestsWaitThread->waitStop(); - } - - if (commSession.getSize() > 1 && !activeRequests.empty()) - { - if (mIsPipelineLeader) - { - if (worldConfig.isPipelineParallel()) - { - auto const peer = worldConfig.getPipelineParallelism() - 1; - bool shouldExit = false; - mCommPipelineParallel->send( - &shouldExit, 1, mpi::MpiType::kBOOL, peer, mpi::MpiTag::kExecutorShouldExit); - auto pipelineCancelledReqIds - = CancelledRequestsAsyncSend::cancelledRequestsRecv(mCommPipelineParallel, peer); - mCancelledReqIds.insert(pipelineCancelledReqIds.begin(), pipelineCancelledReqIds.end()); - } - - auto numCancelledRequests = static_cast<int64_t>(mCancelledReqIds.size()); - if (worldConfig.isTensorParallel()) - { - mCommTensorParallel->bcastValue(numCancelledRequests, 0); - if (numCancelledRequests > 0) - { - std::vector<IdType> cancelledReqIdsVec(mCancelledReqIds.begin(), mCancelledReqIds.end()); - mCommTensorParallel->bcast( - cancelledReqIdsVec.data(), cancelledReqIdsVec.size(), mpi::MpiType::kUINT64, 0); - } - } - if (worldConfig.isContextParallel()) - { - mCommContextParallel->bcastValue(numCancelledRequests, 0); - if (numCancelledRequests > 0) - { - std::vector<IdType> cancelledReqIdsVec(mCancelledReqIds.begin(), mCancelledReqIds.end()); - mCommContextParallel->bcast( - cancelledReqIdsVec.data(), cancelledReqIdsVec.size(), mpi::MpiType::kUINT64, 0); - } - } - } - // If not leader - else - { - if (worldConfig.isFirstPipelineParallelRank()) - { - int64_t numCancelledRequests = 0; - mCommTensorParallel->bcastValue(numCancelledRequests, 0); - mCommContextParallel->bcastValue(numCancelledRequests, 0); - if (numCancelledRequests > 0) - { - std::vector<IdType> cancelledReqIdsVec(numCancelledRequests); - mCommTensorParallel->bcast( - cancelledReqIdsVec.data(), cancelledReqIdsVec.size(), mpi::MpiType::kUINT64, 0); - mCommContextParallel->bcast( - cancelledReqIdsVec.data(), cancelledReqIdsVec.size(), mpi::MpiType::kUINT64, 0); - mCancelledReqIds - = std::unordered_set<IdType>(cancelledReqIdsVec.begin(), cancelledReqIdsVec.end()); - } - } - else - { - auto const peer = worldConfig.getPipelineParallelRank() - 1; - mCancelledReqIds = CancelledRequestsAsyncSend::cancelledRequestsRecv(mCommPipelineParallel, peer); - } - } - if (!worldConfig.isLastPipelineParallelRank()) - { - auto const peer = worldConfig.getPipelineParallelRank() + 1; - mCancelledRequestsAsyncSndHdl - = std::make_unique<CancelledRequestsAsyncSend>(mCommPipelineParallel, mCancelledReqIds, peer); - mCancelledRequestsWaitThread->notifyStart(); - } - } - }; - - std::unique_lock<std::mutex> lck{mCancelReqMtx, std::defer_lock}; - if (!worldConfig.isPipelineParallel()) - { - lck.lock(); - } - - broadcastCancelledRequests(); - - if (!mCancelledReqIds.empty()) - { - // Loop over active requests and terminate those that have been cancelled - std::unordered_set<IdType> terminatedReqIds; - for (auto& req : activeRequests) - { - auto reqId = req->isChild() ? req->getParentRequestId() : req->mRequestId; - if (mCancelledReqIds.find(reqId) != mCancelledReqIds.end()) - { - auto finishReason = req->isTimedOut() ? FinishReason::kTIMED_OUT : FinishReason::kCANCELLED; - mModel->terminateRequestSync(req, finishReason); - // Parent and child requests share the same request id. - // Mark it terminated first and remove from the set later. - terminatedReqIds.insert(reqId); - } - } - - for (auto const& reqId : terminatedReqIds) - { - mCancelledReqIds.erase(reqId); - } - } -} - -void Executor::Impl::terminateContextFinishedRequests(InTransList& inTransmissionRequests) -{ - NVTX3_SCOPED_RANGE(terminateContextFinishedRequests); - for (auto it = inTransmissionRequests.begin(); it != inTransmissionRequests.end();) - { - auto& item = *it; - auto req = item.request; - if (req->isDisaggContextCompleteState()) - { - // If pinnedBlockIds were tracked, unpin them. Otherwise, just terminate. - auto kvMgr = mModel->getKVCacheManager(); - if (kvMgr && !item.pinnedBlockIds.empty()) - { - kvMgr->unpinBlocksById(item.pinnedBlockIds); - } - else - { - mModel->terminateRequest(req); - } - it = inTransmissionRequests.erase(it); - } - else - { - ++it; - } - } -} - -void Executor::Impl::appendNewResponses(std::vector<Response>&& newResponses) -{ - { - std::scoped_lock<std::mutex> lck(mResponsesMtx); - for (auto& response : newResponses) - { - mResponses[response.getRequestId()].emplace_back(std::move(response)); - } - } - mResponsesCv.notify_all(); -} - -Executor::Impl::RequestList Executor::Impl::populateNewResponses( - RequestList& activeRequests, InTransList& inTransmissionRequests, std::vector<Response>& newResponses) -{ - NVTX3_SCOPED_RANGE(populateNewResponses); - RequestList finishedRequests; - for (auto it = activeRequests.begin(); it != activeRequests.end();) - { - auto const& llmReq = (*it); - bool const requestDone = llmReq->isFinished(); - // Only leader should store responses - if (mIsLeader) - { - auto response = llmReq->createResponse(mModel->hasSpeculativeDecodingFastLogits(), mWorldRank); - if (response) - { - newResponses.emplace_back(std::move(response.value())); - } - } - // Remove from active requests if last response has been generated - if (requestDone) - { - // move the in transmission requests to another tracker - if (llmReq->isDisaggContextTransmissionState()) - { - std::vector<SizeType32> pinnedBlockIds{}; - auto kvMgr = mModel->getKVCacheManager(); - if (kvMgr && kvMgr->isEnableBlockReuse() && !kvMgr->getBlockManager().isVariableWindow()) - { - pinnedBlockIds = kvMgr->storeBlocksForReuse(llmReq->mRequestId, llmReq, /*pinBlocks=*/true); - mModel->terminateRequest(llmReq); - } - inTransmissionRequests.push_back(InTransmissionItem{*it, pinnedBlockIds}); - } - finishedRequests.push_back(*it); - it = activeRequests.erase(it); - } - else - { - ++it; - } - } - return finishedRequests; -} - -void Executor::Impl::executionLoop() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - tensorrt_llm::common::setThreadName("executionLoop"); - - auto const& worldConfig = mModel->getWorldConfig(); - TLLM_CUDA_CHECK(cudaSetDevice(worldConfig.getDevice())); - - auto const [profileIterIdxs, stopIterIdxs] = tensorrt_llm::common::populateIterationIndexes( - kPROFILE_START_STOP_ENV_VAR_NAME, kLEGACY_PROFILE_START_STOP_ENV_VAR_NAME); - - SizeType32 numNewActiveRequests{0}; - std::chrono::time_point<std::chrono::steady_clock> iterStart; - std::chrono::time_point<std::chrono::steady_clock> iterEnd; - bool firstIteration{true}; - RequestList activeRequests; - InTransList inTransmissionRequests; - std::vector<Response> newResponses; - while (!mShutdown || !activeRequests.empty()) - { - double iterLatencyMS{0.0}; - double newActiveRequestsQueueLatencyMS{0.0}; - bool reportFinishedRequests = true; - RequestList finishedRequests; - if (!activeRequests.empty()) - { - finishTimedOutRequests(activeRequests); - terminateCancelledRequests(activeRequests); - forwardSync(activeRequests); - finishedRequests = populateNewResponses(activeRequests, inTransmissionRequests, newResponses); - cleanupDynamicLogitsPostProcessors(finishedRequests); - auto const iterCounter = mModel->getIterCounter(); - auto const stopIter = !stopIterIdxs.empty() && (stopIterIdxs.count(iterCounter - 1) > 0); - if (stopIter) - { - cudaProfilerStop(); - } - - // When there are no active or inflight requests, we need to update the stats before calling - // fetchNewRequests to make sure that the stats are reported accurately. - if (activeRequests.empty() && (!firstIteration)) - { - mModel->resetIterationStats(); - updateIterationStats(activeRequests, iterLatencyMS, numNewActiveRequests, - newActiveRequestsQueueLatencyMS, static_cast<SizeType32>(finishedRequests.size()), true); - updateRequestStats(activeRequests, finishedRequests, true); - reportFinishedRequests = false; - } - if (!newResponses.empty()) - { - enqueueNewResponses(std::move(newResponses)); - newResponses.clear(); - } - iterEnd = std::chrono::steady_clock::now(); - iterLatencyMS = std::chrono::duration<double, std::milli>(iterEnd - iterStart).count(); - } - - if (!inTransmissionRequests.empty()) - { - terminateContextFinishedRequests(inTransmissionRequests); - } - - if (!mShutdown) - { - auto const iterCounter = mModel->getIterCounter(); - auto const profileIter = !profileIterIdxs.empty() && (profileIterIdxs.count(iterCounter) > 0); - if (profileIter) - { - cudaProfilerStart(); - } - iterStart = std::chrono::steady_clock::now(); - std::optional<PriorityType> lowestPriority = std::nullopt; - if (!activeRequests.empty()) - { - lowestPriority = activeRequests.back()->priority(); - } - - auto [newRequests, newActiveRequestsQueueLatency] - = fetchNewRequests(static_cast<SizeType32>(activeRequests.size()), lowestPriority); - newActiveRequestsQueueLatencyMS = newActiveRequestsQueueLatency; - numNewActiveRequests = newRequests.size(); - - if (firstIteration) - { - firstIteration = false; - } - - for (auto const& newRequest : newRequests) - { - insertRequestInOrder(activeRequests, newRequest); - } - - // Update dynamic tuning stats - if (mDynamicBatchTuner) - { - for (auto const& req : activeRequests) - { - auto const inputLength = req->mPromptLen; - auto const outputLength = req->mMaxNewTokens; - mDynamicBatchTuner->updateStats(inputLength, outputLength); - } - } - } - if (!activeRequests.empty()) - { - forwardAsync(activeRequests); - updateIterationStats(activeRequests, iterLatencyMS, numNewActiveRequests, newActiveRequestsQueueLatencyMS, - static_cast<SizeType32>(finishedRequests.size()), false); - // Finished requests were reported once. Avoid reporting it twice. - if (reportFinishedRequests) - { - updateRequestStats(activeRequests, finishedRequests, false); - } - else - { - updateRequestStats(activeRequests, {}, false); - } - appendCurrentDebugTensors(); - } - } - - if (mCancelledRequestsWaitThread) - { - mCancelledRequestsWaitThread.reset(nullptr); - } - if (mRequestWithIdWaitThread) - { - mRequestWithIdWaitThread.reset(nullptr); - } - if (worldConfig.isPipelineParallel() && mIsPipelineLeader) - { - auto const peer = worldConfig.getPipelineParallelism() - 1; - int64_t numActiveRequests = -1; - mCommPipelineParallel->send( - &numActiveRequests, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kExecutorNumActiveRequests); - bool shouldExit = true; - mCommPipelineParallel->send(&shouldExit, 1, mpi::MpiType::kBOOL, peer, mpi::MpiTag::kExecutorShouldExit); - } - if (mRequestWithIdLeaderThread) - { - mRequestWithIdLeaderThread->join(); - mRequestWithIdLeaderThread.reset(nullptr); - } - if (mCancelledRequestsLeaderThread) - { - mCancelledRequestsLeaderThread->join(); - mCancelledRequestsLeaderThread.reset(nullptr); - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void Executor::Impl::enqueueTerminateRequest() -{ - { - std::scoped_lock<std::mutex> lck(mQueuedReqMtx); - Request dummyReq({1}, 1); - RequestWithId reqWithId{std::move(dummyReq), kTerminateReqId}; - mQueuedRequests.emplace_back(reqWithId); - } - mQueuedReqCv.notify_one(); -} - -void Executor::Impl::enqueueNewResponses(std::vector<Response>&& newResponses) -{ - TLLM_CHECK_WITH_INFO(mIsLeader, "Only leader should store responses"); - - if (mCommMode == CommunicationMode::kLEADER) - { - appendNewResponses(std::move(newResponses)); - } - else if (mCommMode == CommunicationMode::kORCHESTRATOR) - { - MpiMessage message(MpiId::RESPONSE); - message.data = ResponseData{std::move(newResponses)}; - mSendQueue.push(std::move(message)); - } -} - -// Orchestrator thread sending new requests to leader of the model -void Executor::Impl::orchSendReqThread() -{ - tensorrt_llm::common::setThreadName("orchSendReq"); - - while (true) - { - auto message = mSendQueue.pop(); - - if (message.id == MpiId::TERMINATION) - { - mOrchLeaderComm->send(&message.id, 1, mpi::MpiType::kUINT64, mLeaderRank, mpi::MpiTag::kOrchestratorId); - TLLM_LOG_INFO("Orchestrator sendReq thread exiting"); - break; - } - if (message.id == MpiId::PENDING_REQUEST) - { - auto& reqWithIds = std::get<PendingRequestData>(message.data); - auto packed = RequestWithId::serializeReqWithIds(reqWithIds.requests); - - TLLM_LOG_DEBUG("Orchestrator sendReq thread sending %d pending requests", reqWithIds.requests.size()); - // Temporary WAR to indicate to client that we cannot send the serialized request - // because it exceeds int32_t size limit. - // TODO: Should fix as part of https://jirasw.nvidia.com/browse/TRTLLM-708 - if (packed.size() > std::numeric_limits<int32_t>::max()) - { - for (auto const& reqWithId : reqWithIds.requests) - { - { - std::scoped_lock<std::mutex> lck(mResponsesMtx); - mResponses[reqWithId.id].emplace_back(reqWithId.id, - "Request is too large, or you are enqueuing too many requests at once " - "to be sent via MPI_Send, please try to enqueue the request(s) again. " - "This issue will be resolved in a future version of TRT-LLM."); - } - mResponsesCv.notify_all(); - } - } - else - { - mOrchLeaderComm->send(&message.id, 1, mpi::MpiType::kUINT64, mLeaderRank, mpi::MpiTag::kOrchestratorId); - mOrchLeaderComm->send( - packed.data(), packed.size(), mpi::MpiType::kCHAR, mLeaderRank, mpi::MpiTag::kOrchestratorData); - } - } - else if (message.id == MpiId::CANCEL_REQUEST) - { - auto& data = std::get<RequestIdsData>(message.data); - - mOrchLeaderComm->send(&message.id, 1, mpi::MpiType::kUINT64, mLeaderRank, mpi::MpiTag::kOrchestratorId); - mOrchLeaderComm->send( - data.ids.data(), data.ids.size(), mpi::MpiType::kUINT64, mLeaderRank, mpi::MpiTag::kOrchestratorData); - } - else - { - TLLM_THROW("Invalid message id"); - } - } -} - -// Leader thread receiving new requests from orchestrator -void Executor::Impl::leaderRecvReqThread() -{ - tensorrt_llm::common::setThreadName("leaderRecvReq"); - TLLM_CUDA_CHECK(cudaSetDevice(mDeviceId)); -#if ENABLE_MULTI_DEVICE - auto& selCancelledReqIds = mUsePipelineParallel ? mPipelineCancelledReqIds : mCancelledReqIds; - while (true) - { - if (mRecvPollPeriodMs > 0) - { - mOrchLeaderComm->recvPoll(mOrchRank, mpi::MpiTag::kOrchestratorId, mRecvPollPeriodMs); - } - - // Blocking is okay: terminate message is expected to arrive here - MPI_Message msg = nullptr; - MPI_Status status; - mOrchLeaderComm->mprobe(mOrchRank, mpi::MpiTag::kOrchestratorId, &msg, &status); - - int32_t count = 0; - MPICHECK(MPI_Get_count(&status, MPI_UINT64_T, &count)); // NOLINT - TLLM_CHECK(count == 1); - - MpiId mpiId{}; - MPICHECK(MPI_Mrecv(&mpiId, count, MPI_UINT64_T, &msg, &status)); // NOLINT - - // EXIT condition from receiving TERMINATE msg - if (mpiId == MpiId::TERMINATION) - { - // Enqueue a request to indicate to other ranks to terminate - enqueueTerminateRequest(); - - // Send message to orchestrator to indicate to terminate orch recv thread - mSendQueue.push(MpiMessage(mpiId)); - TLLM_LOG_INFO("Leader recvReq thread exiting"); - break; - } - if (mpiId == MpiId::PENDING_REQUEST) - { - mOrchLeaderComm->mprobe(mOrchRank, mpi::MpiTag::kOrchestratorData, &msg, &status); - MPICHECK(MPI_Get_count(&status, MPI_CHAR, &count)); // NOLINT - std::vector<char> buffer(count); - MPICHECK(MPI_Mrecv(buffer.data(), count, MPI_CHAR, &msg, &status)); // NOLINT - - auto requestWithIds = RequestWithId::deserializeReqWithIds(buffer); - TLLM_LOG_DEBUG("Leader recvReq thread receiving %d pending requests", requestWithIds.size()); - { - std::scoped_lock<std::mutex> lck(mQueuedReqMtx); - if (mMaxQueueSize) - { - auto const maxQueueSize = mMaxQueueSize.value(); - if (maxQueueSize > 0 && mQueuedRequests.size() >= static_cast<size_t>(maxQueueSize)) - { - auto err = tensorrt_llm::common::fmtstr( - "Maximum queue size of %d has been reached, please try again later", maxQueueSize); - TLLM_LOG_ERROR("%s", err.c_str()); - std::vector<Response> responses; - responses.reserve(requestWithIds.size()); - for (auto const& reqWithId : requestWithIds) - { - responses.emplace_back(reqWithId.id, err); - } - enqueueNewResponses(std::move(responses)); - continue; - } - } - for (auto&& req : requestWithIds) - { - req.queuedStart = std::chrono::steady_clock::now(); - insertRequestInOrder(mQueuedRequests, std::move(req)); - } - } - mQueuedReqCv.notify_one(); - } - else if (mpiId == MpiId::CANCEL_REQUEST) - { - // Prepare receiving data - mOrchLeaderComm->mprobe(mOrchRank, mpi::MpiTag::kOrchestratorData, &msg, &status); - MPICHECK(MPI_Get_count(&status, MPI_UINT64_T, &count)); // NOLINT - std::vector<uint64_t> cancelledReqIds(count); - MPICHECK(MPI_Mrecv(cancelledReqIds.data(), count, MPI_UINT64_T, &msg, &status)); // NOLINT - - std::scoped_lock<std::mutex> lck(mCancelReqMtx); - selCancelledReqIds.insert(cancelledReqIds.begin(), cancelledReqIds.end()); - } - else - { - TLLM_THROW("Invalid message id"); - } - } -#endif // ENABLE_MULTI_DEVICE -} - -// Leader thread sending responses to orchestrator -void Executor::Impl::leaderSendThread(MpiMessageQueue& sendQueue, mpi::MpiTag idTag, mpi::MpiTag dataTag) -{ - tensorrt_llm::common::setThreadName("leaderSend"); - TLLM_CUDA_CHECK(cudaSetDevice(mDeviceId)); - -#if ENABLE_MULTI_DEVICE - while (true) - { - auto message = sendQueue.pop(); - - if (message.id == MpiId::TERMINATION) - { - mOrchLeaderComm->send(&message.id, 1, mpi::MpiType::kUINT64, mOrchRank, idTag); - TLLM_LOG_INFO("Leader sendThread exiting"); - break; - } - if (message.id == MpiId::RESPONSE || message.id == MpiId::ITER_STATS - || message.id == MpiId ::REQUEST_ITER_STATS) - { - std::vector<char> buffer; - if (message.id == MpiId::RESPONSE) - { - auto& responseData = std::get<ResponseData>(message.data); - TLLM_LOG_DEBUG("Leader sendResp thread sending %d responses", responseData.responses.size()); - buffer = Serialization::serialize(responseData.responses); - } - else if (message.id == MpiId::ITER_STATS) - { - auto& iterStatsData = std::get<IterStatsData>(message.data); - TLLM_LOG_DEBUG("Leader sendResp thread sending iter stats"); - buffer = Serialization::serialize(iterStatsData.iterStatsVec); - } - else if (message.id == MpiId::REQUEST_ITER_STATS) - { - auto& requestIterStatsData = std::get<RequestStatsPerIterationData>(message.data); - TLLM_LOG_DEBUG("Leader sendResp thread sending iter request stats"); - buffer = Serialization::serialize(requestIterStatsData.requestStatsPerIterationVec); - } - mOrchLeaderComm->send(&message.id, 1, mpi::MpiType::kUINT64, mOrchRank, idTag); - mOrchLeaderComm->send(buffer.data(), buffer.size(), mpi::MpiType::kCHAR, mOrchRank, dataTag); - } - else - { - TLLM_THROW("Invalid message id"); - } - } -#endif // ENABLE_MULTI_DEVICE -} - -void Executor::Impl::orchRecvThread(mpi::MpiTag idTag, mpi::MpiTag dataTag) -{ - tensorrt_llm::common::setThreadName("orchRecv"); - -#if ENABLE_MULTI_DEVICE - while (true) - { - if (mRecvPollPeriodMs > 0) - { - mOrchLeaderComm->recvPoll(mOrchRank, mpi::MpiTag::kOrchestratorId, mRecvPollPeriodMs); - } - - MPI_Message msg = nullptr; - MPI_Status status; - mOrchLeaderComm->mprobe(mLeaderRank, idTag, &msg, &status); - - int32_t count = 0; - MPICHECK(MPI_Get_count(&status, MPI_UINT64_T, &count)); // NOLINT - TLLM_CHECK(count == 1); - - MpiId mpiId{}; - MPICHECK(MPI_Mrecv(&mpiId, count, MPI_UINT64_T, &msg, &status)); // NOLINT - - if (mpiId == MpiId::TERMINATION) - { - TLLM_LOG_INFO("Orchestrator recv thread exiting"); - break; - } - if (mpiId == MpiId::RESPONSE || mpiId == MpiId::ITER_STATS || mpiId == MpiId::REQUEST_ITER_STATS) - { - mOrchLeaderComm->mprobe(mLeaderRank, dataTag, &msg, &status); - MPICHECK(MPI_Get_count(&status, MPI_CHAR, &count)); // NOLINT - - std::vector<char> buffer(count); - MPICHECK(MPI_Mrecv(buffer.data(), count, MPI_CHAR, &msg, &status)); // NOLINT - - if (mpiId == MpiId::RESPONSE) - { - auto newResponses = Serialization::deserializeResponses(buffer); - TLLM_LOG_DEBUG("Orchestrator recv thread receiving %d responses", newResponses.size()); - appendNewResponses(std::move(newResponses)); - } - else if (mpiId == MpiId::ITER_STATS) - { - appendMultipleIterStats(Serialization::deserializeIterationStatsVec(buffer)); - } - else if (mpiId == MpiId::REQUEST_ITER_STATS) - { - appendMultipleRequestStats(Serialization::deserializeRequestStatsPerIterationVec(buffer)); - } - } - else - { - TLLM_THROW("Invalid message id"); - } - } -#endif // ENABLE_MULTI_DEVICE -} - -Executor::Impl::LlmRequestLogitsPostProcessor Executor::Impl::getLogitsPostProcessor(std::string const& name) -{ - auto const postProcIt = mLogitsPostProcessorMap.find(name); - TLLM_CHECK_WITH_INFO( - postProcIt != mLogitsPostProcessorMap.end(), "LogitsPostProcessor %s not found.", name.c_str()); - auto executorLogitsPostProcessor = postProcIt->second; - return [executorLogitsPostProcessor](IdType reqId, RtTensorPtr& logits, BeamTokens const& beamTokens, - CudaStreamPtr const& cudaStreamPtr, std::optional<IdType> clientId) - { - auto logitsTensor = executor::detail::ofITensor(logits); - executorLogitsPostProcessor(reqId, logitsTensor, beamTokens, cudaStreamPtr, clientId); - }; -} - -void Executor::Impl::setupDynamicLogitsPostProcessors(std::vector<RequestWithId>& newReqWithIds) -{ - for (auto& reqWithId : newReqWithIds) - { - auto logitsPostProcessor = reqWithId.req.getLogitsPostProcessor(); - if (logitsPostProcessor) - { - std::string const name = Request::kDynamicPostProcessorNamePrefix + std::to_string(reqWithId.id); - mLogitsPostProcessorMap[name] = logitsPostProcessor.value(); - reqWithId.req.setLogitsPostProcessor(std::nullopt); - reqWithId.req.setLogitsPostProcessorName(name); - } - } -} - -void Executor::Impl::cleanupDynamicLogitsPostProcessors(RequestList const& finishedRequests) -{ - for (auto& req : finishedRequests) - { - std::string const name = Request::kDynamicPostProcessorNamePrefix + std::to_string(req->mRequestId); - auto const postProcIt = mLogitsPostProcessorMap.find(name); - if (postProcIt != mLogitsPostProcessorMap.end()) - { - mLogitsPostProcessorMap.erase(name); - } - } -} - -void Executor::Impl::addTerminatedReqId(std::vector<Response> const& responses, IdType const& reqId) -{ - for (auto const& response : responses) - { - if (response.hasError() || (!response.hasError() && response.getResult().isFinal)) - { - mTerminatedReqIds.insert(reqId); - if (mChildReqIdsMap.find(reqId) != mChildReqIdsMap.end()) - { - for (auto childReqId : mChildReqIdsMap.at(reqId)) - { - mTerminatedReqIds.insert(childReqId); - } - mChildReqIdsMap.erase(reqId); - } - } - } -} - -void Executor::Impl::checkParallelApiUsage(std::string const& methodName) const -{ - // If leader mode, and not leader, throw error - if (mCommMode == CommunicationMode::kLEADER && !mIsLeader) - { - // Non-leader are not expected to call cancelRequest - TLLM_THROW("With LEADER communication mode, only leader rank is expected to call %s", methodName.c_str()); - } - if (mCommMode == CommunicationMode::kORCHESTRATOR && !mIsOrchestrator) - { - TLLM_THROW( - "With ORCHESTRATOR communication mode, only orchestrator rank is expected to call %s", methodName.c_str()); - } -} - -} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/executorImpl.h b/cpp/tensorrt_llm/executor/executorImpl.h deleted file mode 100644 index f812b55a3fa0..000000000000 --- a/cpp/tensorrt_llm/executor/executorImpl.h +++ /dev/null @@ -1,385 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/common/arrayView.h" -#include "tensorrt_llm/executor/dynamicBatchTuner.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/intervalSet.h" -#include "tensorrt_llm/executor/model.h" -#include "tensorrt_llm/executor/orchestratorUtils.h" -#include "tensorrt_llm/executor/requestWithId.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/runtime/gptJsonConfig.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -#include <atomic> -#include <condition_variable> -#include <list> -#include <mutex> -#include <optional> -#include <queue> -#include <thread> -#include <unordered_map> -#include <unordered_set> - -namespace tensorrt_llm::executor -{ - -class RequestWithIdAsyncSend; -class CancelledRequestsAsyncSend; - -class MpiMessageQueue -{ -public: - void push(MpiMessage&& message) - { - std::lock_guard<std::mutex> const lock(mMutex); - mQueue.push(std::move(message)); - mCv.notify_one(); - } - - MpiMessage pop() - { - std::unique_lock<std::mutex> lock(mMutex); - mCv.wait(lock, [this] { return !mQueue.empty(); }); - MpiMessage message = std::move(mQueue.front()); - mQueue.pop(); - return message; - } - -private: - std::queue<MpiMessage> mQueue; - std::mutex mMutex; - std::condition_variable mCv; -}; - -class Executor::Impl - -{ - using LlmRequestPtr = std::shared_ptr<batch_manager::LlmRequest>; - using RequestList = std::list<LlmRequestPtr>; - - // When block reuse is enabled for context worker for disaggregated serving, - // we need to store the pinned block ids so that we can unpin them when - // the request is finished. - struct InTransmissionItem - { - LlmRequestPtr request; - std::vector<SizeType32> pinnedBlockIds; - }; - - using InTransList = std::list<InTransmissionItem>; - -public: - Impl(std::filesystem::path const& modelPath, std::optional<std::filesystem::path> const& encoderModelPath, - [[maybe_unused]] ModelType modelType, ExecutorConfig const& executorConfig); - - Impl(BufferView const& engineBufferView, std::string const& jsonConfigStr, - std::optional<BufferView> const& encoderEngineBufferView, - std::optional<std::string> const& encoderJsonConfigStr, [[maybe_unused]] ModelType modelType, - ExecutorConfig const& executorConfig, std::optional<std::map<std::string, Tensor>> const& managedWeightsOpt); - - Impl(std::shared_ptr<Model> model, std::optional<std::shared_ptr<Model>> encoderModel, - ExecutorConfig const& executorConfig); - - ~Impl(); - - Impl(Impl const& executor) = delete; - Impl& operator=(Impl const& executor) = delete; - Impl(Impl&&) = delete; - Impl& operator=(Impl&&) = delete; - - IdType enqueueRequest(Request const& request); - - std::vector<IdType> enqueueRequests(std::vector<Request> const& requests); - - std::vector<IdType> enqueueRequests(common::ArrayView<Request const> const& requests); - - std::vector<Response> awaitResponses(std::optional<std::chrono::milliseconds> const& timeout = std::nullopt); - - std::vector<Response> awaitResponses( - IdType const& reqId, std::optional<std::chrono::milliseconds> const& optTimeout = std::nullopt); - - std::vector<std::vector<Response>> awaitResponses( - std::vector<IdType> const& requestIds, std::optional<std::chrono::milliseconds> const& timeout); - - SizeType32 getNumResponsesReady(std::optional<IdType> const& optId = std::nullopt) const; - - void cancelRequest(IdType requestId); - - void shutdown(); - - std::deque<IterationStats> getLatestIterationStats(); - std::deque<RequestStatsPerIteration> getLatestRequestStats(); - std::deque<DebugTensorsPerIteration> getLatestDebugTensors(); - - bool canEnqueueRequests() const; - - bool isParticipant() const; - - std::optional<std::shared_ptr<KVCacheEventManager>> getKVCacheEventManager() const; - -private: - using RtTensorPtr = runtime::ITensor::SharedPtr; - using CudaStreamPtr = runtime::BufferManager::CudaStreamPtr; - using LlmRequestLogitsPostProcessor - = std::function<void(IdType, RtTensorPtr&, BeamTokens const&, CudaStreamPtr, std::optional<IdType>)>; - - void initialize(ExecutorConfig const& executorConfig); - - void loadModel(std::optional<std::filesystem::path> const& modelPath, std::optional<BufferView> const& engineBuffer, - runtime::GptJsonConfig const& jsonConfig, ExecutorConfig const& executorConfig, bool isEncoder, - std::optional<std::map<std::string, Tensor>> const& managedWeightsOpt); - - std::shared_ptr<Model> createModel(runtime::RawEngine const& rawEngine, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig, ExecutorConfig const& executorConfig); - - std::shared_ptr<Model> createEncoderModel(runtime::RawEngine const& rawEngine, - runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - ExecutorConfig const& executorConfig); - - void setOrchLeaderComm(SizeType32 tp, SizeType32 pp, SizeType32 cp, ParallelConfig const& parallelConfig); - - void initializeCommAndWorkers(SizeType32 tp, SizeType32 pp, SizeType32 cp, ExecutorConfig const& executorConfig, - std::optional<ModelType> modelType = std::nullopt, - std::optional<std::filesystem::path> const& modelPath = std::nullopt, - std::optional<runtime::WorldConfig> const& worldConfig = std::nullopt, - std::optional<runtime::GptJsonConfig> const& decoderGptJsonConfig = std::nullopt); - - static void validateParallelConfig(ParallelConfig const& parallelConfig, std::optional<ModelType> modelType, - std::optional<std::filesystem::path> const& modelPath); - - void initializeOrchestrator(SizeType32 tp, SizeType32 pp, SizeType32 cp, ExecutorConfig const& executorConfig, - ParallelConfig parallelConfig, ModelType modelType, std::filesystem::path const& modelPath); - - void initializeWorkers(SizeType32 tp, SizeType32 pp, SizeType32 cp, ParallelConfig& parallelConfig, - std::optional<runtime::WorldConfig> const& worldConfig = std::nullopt, - std::optional<runtime::GptJsonConfig> const& decoderGptJsonConfig = std::nullopt); - - void initializeLogitsPostProcessorBatched(LogitsPostProcessorConfig const& logitsProcConfig); - - IdType generateReqId(Request const& request) - { - // If the request has a disaggregated request id, prefer it. - if (request.getDisaggRequestId().has_value() && request.getDisaggRequestId().value() > kMaxLocalReqId) - { - return request.getDisaggRequestId().value(); - } - // Otherwise, generate a local request id in range [1, kMaxLocalReqId). - return generateLocalReqId(); - } - - IdType generateLocalReqId() - { - return (mLastReqId++ % kMaxLocalReqId); - } - - std::vector<RequestWithId> getLeaderNewReqWithIds( - SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive); - std::vector<RequestWithId> getNewReqWithIds( - SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive); - - std::tuple<Executor::Impl::RequestList, double> fetchNewRequests( - SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive); - - void forwardSync(RequestList& activeRequests); - - void forwardAsync(RequestList& activeRequests); - - void prepRequestsForEncoderSkip(RequestList& activeRequests); - - void terminateActiveRequests(RequestList& activeRequests, std::string const& err); - - IterationStats getCurrentIterationStats(RequestList const& activeRequests, double iterLatencyMS, - SizeType32 numNewActiveRequests, double newActiveRequestsQueueLatencyMS, SizeType32 numCompletedRequests); - - void appendCurrentIterStats(IterationStats&& currentIterStats); - void appendMultipleIterStats(std::vector<IterationStats>&& currentIterStatsVec); - void updateIterationStats(RequestList const& activeRequests, double iterLatencyMS, SizeType32 numNewActiveRequests, - double newActiveRequestsQueueLatencyMS, SizeType32 numCompletedRequests, bool flushToOrchestrator); - void appendCurrentRequestStats(RequestStatsPerIteration&& currentRequestStats); - void appendMultipleRequestStats(std::vector<RequestStatsPerIteration>&& currentRequestStatsVec); - RequestStatsPerIteration getCurrentRequestStats( - RequestList const& activeRequests, RequestList const& finishedRequests); - void updateRequestStats( - RequestList const& activeRequests, RequestList const& finishedRequests, bool flushToOrchestrator); - - void appendCurrentDebugTensors(); - - void terminateCancelledRequests(RequestList& activeRequests); - - void terminateContextFinishedRequests(InTransList& inTransmissionRequests); - - void appendNewResponses(std::vector<Response>&& newResponses); - - /// @brief Populates new responses from active requests. - /// Active requests that have completed are erased from activeRequests - /// and returned for bookkeeping. - /// @return A list of requests that have completed. - RequestList populateNewResponses( - RequestList& activeRequests, InTransList& inTransmissionRequests, std::vector<Response>& newResponses); - - void executionLoop(); - - void enqueueTerminateRequest(); - void enqueueNewResponses(std::vector<Response>&& newResponses); - - LlmRequestLogitsPostProcessor getLogitsPostProcessor(std::string const& name); - void setupDynamicLogitsPostProcessors(std::vector<RequestWithId>& newReqWithIds); - void cleanupDynamicLogitsPostProcessors(RequestList const& finishedRequests); - - void orchSendReqThread(); - void orchRecvThread(mpi::MpiTag idTag, mpi::MpiTag dataTag); - void leaderRecvReqThread(); - void leaderSendThread(MpiMessageQueue& sendQueue, mpi::MpiTag idTag, mpi::MpiTag dataTag); - - void addTerminatedReqId(std::vector<Response> const& responses, IdType const& reqId); - - // Check that the current process is the leader or orchestrator - void checkParallelApiUsage(std::string const& methodName) const; - - // These functions wait for MPI async sends on separate threads - void requestWithIdWaitThread(); - void cancelledRequestsWaitThread(); - // These functions send data from leader to pipeline leader on separate threads - void requestWithIdLeaderThread(); - void cancelledRequestsLeaderThread(); - - /// @brief mark requests that have timed out before ever being executed as finished. - /// uses cancellation based on communication mode. - /// - /// @param activeRequests [in] List of active requests to check for timeouts - void finishTimedOutRequests(RequestList const& activeRequests); - - // The model to execute - std::shared_ptr<Model> mModel = nullptr; - std::shared_ptr<Model> mEncoderModel = nullptr; - - // The maximum number of activeRequests - SizeType32 mMaxNumActiveRequests; - - // Thread the executes the main loop - std::thread mExecutionThread; - - // Atomic that indicates threads should shutdown - std::atomic<bool> mShutdown; - - // Atomic that indicates if shutdown method has been called - std::atomic<bool> mShutdownCalled = false; - - // Queued requests - std::mutex mQueuedReqMtx; - std::condition_variable mQueuedReqCv; - std::deque<RequestWithId> mQueuedRequests; - std::optional<SizeType32> mMaxQueueSize; - - // Cancelled requests - std::mutex mCancelReqMtx; - std::unordered_set<IdType> mCancelledReqIds; - std::unordered_set<IdType> mPipelineCancelledReqIds; - - // Ready responses - std::unordered_map<IdType, std::vector<Response>> mResponses; - mutable std::mutex mResponsesMtx; - std::condition_variable mResponsesCv; - - // Since the request IDs are generated sequentially, IntervalSet is preferred over unordered_set for its efficient - // memory usage to stores request ID intervals rather than individual request ID numbers. - IntervalSet<IdType> mTerminatedReqIds; - - std::unordered_map<IdType, std::vector<IdType>> mChildReqIdsMap; - - // Iteration stats - SizeType32 mIterStatsMaxIterations; - std::mutex mIterStatsMtx; - std::deque<IterationStats> mIterationStats; - - // Request stats - SizeType32 mRequestStatsMaxIterations; - std::mutex mRequestStatsMtx; - std::deque<RequestStatsPerIteration> mRequestStats; - - // Debug - IterationType mDebugTensorsMaxIterations; - std::mutex mDebugTensorsMtx; - std::deque<DebugTensorsPerIteration> mDebugTensors; - - IdType mLastReqId = 1; - - static constexpr IdType kTerminateReqId = 0; - // Request id > kMaxLocalReqId is reserved for disaggregated requests. - // This max ID is also in Python side. - static constexpr IdType kMaxLocalReqId = 1ULL << 42U; - - BatchingType mBatchingType; - bool mIsSchedulerMaxUtilization; - bool mIsSchedulerGuaranteedNoEvict; - bool mIsChunkedContext; - bool mPromptTableOffloading; - - CommunicationMode mCommMode; - bool mIsWorker = false; - bool mIsLeader = false; - bool mIsPipelineLeader = false; - bool mUsePipelineParallel = false; - - std::unordered_map<std::string, LogitsPostProcessor> mLogitsPostProcessorMap; - std::optional<Model::LogitsPostProcessorBatched> mLogitsPostProcessorBatched; - - bool mIsOrchestrator = false; - std::shared_ptr<tensorrt_llm::mpi::MpiComm> mOrchLeaderComm; - - std::thread mOrchSendReqThread; - std::thread mOrchRecvThread; - std::thread mLeaderRecvReqThread; - std::thread mLeaderSendThread; - - int32_t mRecvPollPeriodMs = 0; - - int32_t mLeaderRank = -1; - int32_t mOrchRank = 0; - int32_t mWorldRank = -1; - int32_t mDeviceId = 0; - - MpiMessageQueue mSendQueue; - - std::shared_ptr<tensorrt_llm::mpi::MpiComm> mCommTensorParallel; - std::shared_ptr<tensorrt_llm::mpi::MpiComm> mCommPipelineParallel; - std::shared_ptr<tensorrt_llm::mpi::MpiComm> mCommContextParallel; - std::unique_ptr<RequestWithIdAsyncSend> mRequestWithIdAsyncSndHdl; - std::unique_ptr<CancelledRequestsAsyncSend> mCancelledRequestsAsyncSndHdl; - std::unique_ptr<std::thread> mRequestWithIdLeaderThread; - std::unique_ptr<std::thread> mCancelledRequestsLeaderThread; - std::unique_ptr<tensorrt_llm::mpi::MpiWaitThread> mRequestWithIdWaitThread; - std::unique_ptr<tensorrt_llm::mpi::MpiWaitThread> mCancelledRequestsWaitThread; - - // for validating requests - bool mEnableBlockReuse; - - inline static std::string const kPROFILE_START_STOP_ENV_VAR_NAME = "TLLM_PROFILE_START_STOP"; - inline static std::string const kLEGACY_PROFILE_START_STOP_ENV_VAR_NAME = "TLLM_GPTM_PROFILE_START_STOP"; - - std::shared_ptr<DynamicBatchTuner> mDynamicBatchTuner; -}; - -} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/intervalSet.h b/cpp/tensorrt_llm/executor/intervalSet.h deleted file mode 100644 index 01f40a685f46..000000000000 --- a/cpp/tensorrt_llm/executor/intervalSet.h +++ /dev/null @@ -1,141 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/executor/types.h" - -namespace tensorrt_llm::executor -{ - -/// @brief An interval inclusive on both ends. -/// A single number interval is represented as [num, num]. -template <typename T> -struct Interval -{ - T lowerEnd; - T upperEnd; -}; - -template <typename T> -bool operator<(Interval<T> const& a, Interval<T> const& b) -{ - return a.lowerEnd < b.lowerEnd; -} - -/// @brief A container to store unique numbers, represented as a vector of ordered and disjoint intervals. -template <typename NumType> -class IntervalSet -{ -public: - /// @brief Check if the given number is in set. - bool contains(NumType num) const - { - // Binary search - SizeType32 left = 0; - SizeType32 right = static_cast<SizeType32>(mIntervals.size()) - 1; - while (left <= right) - { - SizeType32 mid = left + (right - left) / 2; - if (mIntervals[mid].lowerEnd <= num && num <= mIntervals[mid].upperEnd) - { - return true; - } - else if (num < mIntervals[mid].lowerEnd) - { - right = mid - 1; - } - else - { - left = mid + 1; - } - } - return false; - } - - /// @brief Insert a number into set. Do nothing if the number is already in the set. - void insert(NumType num) - { - auto intervalToAdd = Interval<NumType>{num, num}; - - if (mIntervals.size() == 0) - { - mIntervals.insert(mIntervals.begin(), intervalToAdd); - mNumElements++; - return; - } - - // Iter is the first place in mIntervals such that num <= it.lowerEnd - auto iter = std::lower_bound(mIntervals.begin(), mIntervals.end(), intervalToAdd); - - bool iterAtBegin = iter == mIntervals.begin(); - bool iterAtEnd = iter == mIntervals.end(); - - if ((!iterAtEnd && iter->lowerEnd == num) || (!iterAtBegin && num <= (iter - 1)->upperEnd)) - { - // Number falls within the current interval or previous interval. No need to add again. - return; - } - - if (!iterAtBegin && !iterAtEnd && (iter - 1)->upperEnd + 1 == num && iter->lowerEnd - 1 == num) - { - // Merge two adjacent intervals - (iter - 1)->upperEnd = iter->upperEnd; - mIntervals.erase(iter); - } - else if (!iterAtBegin && (iter - 1)->upperEnd + 1 == num) - { - // Number is adjacent to the upper end of the previous interval. Merge left. - (iter - 1)->upperEnd = num; - } - else if (!iterAtEnd && iter->lowerEnd - 1 == num) - { - // Number is adjacent to the lower end of the current interval. Merge right. - iter->lowerEnd = num; - } - else - { - mIntervals.insert(iter, intervalToAdd); - } - mNumElements++; - } - - /// @brief Clear interval set and reset numElements to 0. - void clear() - { - mIntervals.clear(); - mNumElements = 0; - } - - /// @brief Return the size of the set. - SizeType32 getNumElements() const - { - return mNumElements; - } - - /// @brief Return the underlying mIntervals. - std::vector<Interval<NumType>> const& getIntervals() const - { - return mIntervals; - } - -private: - std::vector<Interval<NumType>> mIntervals; - SizeType32 mNumElements{0}; -}; - -} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/model.h b/cpp/tensorrt_llm/executor/model.h deleted file mode 100644 index 52fedf1d1113..000000000000 --- a/cpp/tensorrt_llm/executor/model.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/batch_manager/logitsPostProcessor.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/worldConfig.h" - -#include <nlohmann/json.hpp> - -namespace tensorrt_llm::executor -{ - -class Model -{ - using LlmRequestPtr = std::shared_ptr<batch_manager::LlmRequest>; - -public: - Model() = default; - - virtual ~Model() = default; - - /// @brief Function that marks a request Id as complete and cleans up associated state - virtual void terminateRequest(LlmRequestPtr const& llmRequest, bool pause) = 0; - - void terminateRequest(LlmRequestPtr const& llmRequest) - { - terminateRequest(llmRequest, false); - } - - /// @brief Terminate request in the next forwardSync call that includes the request. - virtual void terminateRequestSync(LlmRequestPtr const& llmRequest, FinishReason finishReason) = 0; - - /// @brief Function that synchronizes the decoder - virtual void forwardSync() = 0; - - /// @brief Function that tries to advance the active requests - /// Depending on resources available, it's possible that not all requests will get advanced - /// @param activeRequests The list of request to try to advance - virtual void forwardAsync(batch_manager::RequestList const& activeRequests) = 0; - - /// @brief Override the runtime batch size for the model - virtual void setRuntimeBatchSize(SizeType32 runtimeBatchSize) - { - // By default, we ignore the runtimeBatchSize unless the model actively supports it - } - - /// @brief Get the runtime batch size for the model - [[nodiscard]] virtual SizeType32 getRuntimeBatchSize() const - { - TLLM_CHECK_WITH_INFO(false, "getRuntimeBatchSize is not implemented"); - } - - /// @brieft Override the runtime max num tokens for the model - virtual void setRuntimeMaxNumTokens(SizeType32 runtimeMaxNumTokens) - { - // By default, we ignore the runtimeMaxNumTokens unless the model actively supports it - } - - virtual void updatePeftCache(LlmRequestPtr const& llmRequest) = 0; - - /// @brief Reset the iteration stats when there are no inflight requests - virtual void resetIterationStats() = 0; - - [[nodiscard]] virtual SizeType32 getMaxNumSequences() const = 0; - [[nodiscard]] virtual SizeType32 getMaxInputLen() const = 0; - [[nodiscard]] virtual SizeType32 getHiddenSize() const = 0; - [[nodiscard]] virtual SizeType32 getMaxSequenceLen() const = 0; - [[nodiscard]] virtual SizeType32 getVocabSizePadded() const = 0; - [[nodiscard]] virtual SizeType32 getMaxDraftLen() const = 0; - [[nodiscard]] virtual SizeType32 getNumMicroBatches() const = 0; - [[nodiscard]] virtual SizeType32 getOperatingBeamWidth() const = 0; - [[nodiscard]] virtual nvinfer1::DataType getLogitDataType() const = 0; - [[nodiscard]] virtual runtime::WorldConfig const& getWorldConfig() const = 0; - [[nodiscard]] virtual runtime::ModelConfig const& getModelConfig() const = 0; - [[nodiscard]] virtual runtime::BufferManager const& getBufferManager() const = 0; - [[nodiscard]] virtual runtime::BufferManager::CudaStreamPtr getRuntimeStreamPtr() const = 0; - [[nodiscard]] virtual IterationType getIterCounter() const noexcept = 0; - [[nodiscard]] virtual bool hasSpeculativeDecodingFastLogits() const noexcept = 0; - [[nodiscard]] virtual bool getGatherGenerationLogits() const = 0; - [[nodiscard]] virtual nvinfer1::DataType getTensorDataType(std::string const& name) const = 0; - [[nodiscard]] virtual nvinfer1::Dims getTensorShape(std::string const& name) const = 0; - - /// @brief Function that provides per iteration stats specific to a certain model - /// @param stats The json object to write stats to - virtual void getCurrentIterationStats(IterationStats& stats) const = 0; - - /// @brief Function that provides per request stats specific to a certain model - /// @param stats The request stats to be updated - virtual void getCurrentRequestStats(RequestStatsPerIteration& stats) const = 0; - - [[nodiscard]] virtual DebugTensorsPerIteration getCurrentDebugTensors() const = 0; - - using LogitsPostProcessorBatched = tensorrt_llm::batch_manager::LogitsPostProcessor::LogitsPostProcessorBatched; - - virtual void setLogitsPostProcessorBatched(std::optional<LogitsPostProcessorBatched> logitsPostProcessorBatched) - = 0; - virtual void setReplicateLogitsPostProcessor(bool replicateLogitsPostProcessor) = 0; - [[nodiscard]] virtual bool getReplicateLogitsPostProcessor() const = 0; - - [[nodiscard]] virtual bool hasGuidedDecoder() const noexcept = 0; - - [[nodiscard]] virtual std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager> - getKVCacheManager() = 0; - [[nodiscard]] virtual std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager const> - getKVCacheManager() const = 0; - - //! \brief Get the batch size that can fill the kv cache to the maximum capacity give the sequence length - //! \param seqLen The sequence length - //! \return The batch size that can fill the kv cache to the maximum capacity. If unsuporrted, return 0. - [[nodiscard]] virtual SizeType32 getMaxCapacityBatchSize(SizeType32 inputLength, SizeType32 outputLength) const = 0; -}; - -} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/serialization.cpp b/cpp/tensorrt_llm/executor/serialization.cpp index 020306e03e56..02077b6857ad 100644 --- a/cpp/tensorrt_llm/executor/serialization.cpp +++ b/cpp/tensorrt_llm/executor/serialization.cpp @@ -17,6 +17,7 @@ #include "tensorrt_llm/executor/serialization.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/executor/requestImpl.h" @@ -626,8 +627,8 @@ kv_cache::CacheState Serialization::deserializeCacheState(std::istream& is) auto hasRnnConfig = su::deserialize<bool>(is); std::optional<CacheState::RnnModelConfig> rnnModelConfig; std::vector<SizeType32> rnnLayerNumPerPP; - nvinfer1::DataType convStateDataType{nvinfer1::DataType::kFLOAT}; - nvinfer1::DataType ssmStateDataType{nvinfer1::DataType::kFLOAT}; + tensorrt_llm::DataType convStateDataType{tensorrt_llm::DataType::kFLOAT}; + tensorrt_llm::DataType ssmStateDataType{tensorrt_llm::DataType::kFLOAT}; if (hasRnnConfig) { CacheState::RnnModelConfig rnnCfg; @@ -641,8 +642,8 @@ kv_cache::CacheState Serialization::deserializeCacheState(std::istream& is) rnnCfg.mNumHeads = su::deserialize<decltype(CacheState::RnnModelConfig::mNumHeads)>(is); rnnCfg.mConvSectionLayout = static_cast<CacheState::RnnModelConfig::ConvSectionLayout>(su::deserialize<SizeType32>(is)); - convStateDataType = su::deserialize<nvinfer1::DataType>(is); - ssmStateDataType = su::deserialize<nvinfer1::DataType>(is); + convStateDataType = su::deserialize<tensorrt_llm::DataType>(is); + ssmStateDataType = su::deserialize<tensorrt_llm::DataType>(is); rnnLayerNumPerPP = su::deserialize<std::vector<SizeType32>>(is); rnnModelConfig = std::move(rnnCfg); } @@ -764,6 +765,8 @@ DataTransceiverState Serialization::deserializeDataTransceiverState(std::istream { state.setCacheState(std::move(cacheState).value()); } + auto isArbitraryTransferState = su::deserialize<decltype(DataTransceiverState::mIsArbitraryTransferState)>(is); + state.setIsArbitraryTransferState(isArbitraryTransferState); return state; } @@ -771,6 +774,7 @@ void Serialization::serialize(DataTransceiverState const& state, std::ostream& o { su::serialize(state.mCommState, os); su::serialize(state.mCacheState, os); + su::serialize(state.mIsArbitraryTransferState, os); } std::vector<char> Serialization::serialize(DataTransceiverState const& state) @@ -789,6 +793,7 @@ size_t Serialization::serializedSize(DataTransceiverState const& state) size_t totalSize = 0; totalSize += su::serializedSize(state.mCommState); totalSize += su::serializedSize(state.mCacheState); + totalSize += su::serializedSize(state.mIsArbitraryTransferState); return totalSize; } diff --git a/cpp/tensorrt_llm/executor/tensor.cpp b/cpp/tensorrt_llm/executor/tensor.cpp index c38feb0e34b8..9c508c0ec5c3 100644 --- a/cpp/tensorrt_llm/executor/tensor.cpp +++ b/cpp/tensorrt_llm/executor/tensor.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/executor/tensor.h" #include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -53,17 +54,17 @@ DataType Tensor::getDataType() const } switch (mTensor->getDataType()) { - case nvinfer1::DataType::kBOOL: return DataType::kBOOL; - case nvinfer1::DataType::kINT8: return DataType::kINT8; - case nvinfer1::DataType::kINT32: return DataType::kINT32; - case nvinfer1::DataType::kUINT8: return DataType::kUINT8; - case nvinfer1::DataType::kFP8: return DataType::kFP8; - case nvinfer1::DataType::kHALF: return DataType::kFP16; - case nvinfer1::DataType::kFLOAT: return DataType::kFP32; - case nvinfer1::DataType::kBF16: return DataType::kBF16; - case nvinfer1::DataType::kINT64: return DataType::kINT64; - case nvinfer1::DataType::kINT4: [[fallthrough]] /* do nothing */; - case nvinfer1::DataType::kFP4: [[fallthrough]] /* do nothing */; + case tensorrt_llm::DataType::kBOOL: return DataType::kBOOL; + case tensorrt_llm::DataType::kINT8: return DataType::kINT8; + case tensorrt_llm::DataType::kINT32: return DataType::kINT32; + case tensorrt_llm::DataType::kUINT8: return DataType::kUINT8; + case tensorrt_llm::DataType::kFP8: return DataType::kFP8; + case tensorrt_llm::DataType::kHALF: return DataType::kFP16; + case tensorrt_llm::DataType::kFLOAT: return DataType::kFP32; + case tensorrt_llm::DataType::kBF16: return DataType::kBF16; + case tensorrt_llm::DataType::kINT64: return DataType::kINT64; + case tensorrt_llm::DataType::kINT4: [[fallthrough]] /* do nothing */; + case tensorrt_llm::DataType::kFP4: [[fallthrough]] /* do nothing */; default: TLLM_THROW("Unsupported data type"); } } @@ -135,19 +136,19 @@ tr::ITensor::Shape toDims(Shape const& shape) return dims; } -nvinfer1::DataType toDataType(DataType dataType) +tensorrt_llm::DataType toDataType(DataType dataType) { switch (dataType) { - case DataType::kBOOL: return nvinfer1::DataType::kBOOL; - case DataType::kUINT8: return nvinfer1::DataType::kUINT8; - case DataType::kINT8: return nvinfer1::DataType::kINT8; - case DataType::kINT32: return nvinfer1::DataType::kINT32; - case DataType::kINT64: return nvinfer1::DataType::kINT64; - case DataType::kBF16: return nvinfer1::DataType::kBF16; - case DataType::kFP8: return nvinfer1::DataType::kFP8; - case DataType::kFP16: return nvinfer1::DataType::kHALF; - case DataType::kFP32: return nvinfer1::DataType::kFLOAT; + case DataType::kBOOL: return tensorrt_llm::DataType::kBOOL; + case DataType::kUINT8: return tensorrt_llm::DataType::kUINT8; + case DataType::kINT8: return tensorrt_llm::DataType::kINT8; + case DataType::kINT32: return tensorrt_llm::DataType::kINT32; + case DataType::kINT64: return tensorrt_llm::DataType::kINT64; + case DataType::kBF16: return tensorrt_llm::DataType::kBF16; + case DataType::kFP8: return tensorrt_llm::DataType::kFP8; + case DataType::kFP16: return tensorrt_llm::DataType::kHALF; + case DataType::kFP32: return tensorrt_llm::DataType::kFLOAT; case DataType::kUNKNOWN: TLLM_THROW("Unsupported data type"); } diff --git a/cpp/tensorrt_llm/executor_worker/CMakeLists.txt b/cpp/tensorrt_llm/executor_worker/CMakeLists.txt deleted file mode 100644 index 2feb6dfe5790..000000000000 --- a/cpp/tensorrt_llm/executor_worker/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -set(SRCS executorWorker.cpp) - -include_directories(${PROJECT_SOURCE_DIR}/include) - -set(EXECUTOR_WORKER_TARGET executorWorker) - -add_executable(${EXECUTOR_WORKER_TARGET} ${SRCS}) - -target_link_libraries(${EXECUTOR_WORKER_TARGET} - PUBLIC ${SHARED_TARGET} nvinfer_plugin_tensorrt_llm) - -target_compile_features(${EXECUTOR_WORKER_TARGET} PRIVATE cxx_std_17) diff --git a/cpp/tensorrt_llm/executor_worker/executorWorker.cpp b/cpp/tensorrt_llm/executor_worker/executorWorker.cpp deleted file mode 100644 index aa1b06c2cb74..000000000000 --- a/cpp/tensorrt_llm/executor_worker/executorWorker.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/serialization.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include <csignal> - -namespace tle = tensorrt_llm::executor; - -int main(int argc, char* argv[]) -{ -#if ENABLE_MULTI_DEVICE - - if (std::getenv("FORCE_NCCL_ALL_REDUCE_STRATEGY") != nullptr) - { - TLLM_LOG_INFO("FORCE_NCCL_ALL_REDUCE_STRATEGY env variable detected in worker"); - } - - // Register the TRT-LLM plugins - initTrtLlmPlugins(); - - tensorrt_llm::mpi::initialize(tensorrt_llm::mpi::MpiThreadSupport::THREAD_MULTIPLE, true); - - MPI_Comm parentComm; - MPI_Comm_get_parent(&parentComm); - if (parentComm == MPI_COMM_NULL) - { - TLLM_LOG_ERROR("TRT-LLM worker has no parent!"); - return -1; - } - - int size; - MPI_Comm_remote_size(parentComm, &size); - if (size != 1) - { - TLLM_LOG_ERROR("Parent size is %d, must be 1", size); - return -1; - } - - // Since parentComm is an intercommunicator, input root - // is the rank of the parent process in his group - // (always 0 as the parent size is checked before) - - // Receive from the parent the executor configuration - int64_t bufferSize; - MPICHECK(MPI_Bcast(&bufferSize, 1, MPI_INT64_T, 0, parentComm)); - std::vector<char> buffer(bufferSize); - MPICHECK(MPI_Bcast(buffer.data(), bufferSize, MPI_CHAR, 0, parentComm)); - std::istringstream is(std::string(buffer.begin(), buffer.end())); - auto modelPath = tle::Serialization::deserializeString(is); - auto modelType = tle::Serialization::deserializeModelType(is); - auto executorConfig = tle::Serialization::deserializeExecutorConfig(is); - - // Create the orchestrator config for workers - auto orchLeaderComm = std::make_shared<tensorrt_llm::mpi::MpiComm>(parentComm, true); - auto parallelConfig = executorConfig.getParallelConfig(); - TLLM_CHECK_WITH_INFO(parallelConfig.has_value(), "Parallel config should have a value."); - TLLM_CHECK_WITH_INFO( - parallelConfig.value().getOrchestratorConfig().has_value(), "Orchestrator config should have a value."); - auto orchConfig = parallelConfig.value().getOrchestratorConfig().value(); - TLLM_CHECK_WITH_INFO(parallelConfig.has_value(), "Parallel config should have a value."); - auto newOrchConfig = tle::OrchestratorConfig(false, orchConfig.getWorkerExecutablePath(), orchLeaderComm); - parallelConfig.value().setOrchestratorConfig(newOrchConfig); - executorConfig.setParallelConfig(parallelConfig.value()); - // In orchestrator mode, the spawned threads will wait for termination signal from orchestrator - auto executor = tle::Executor(modelPath, modelType, executorConfig); - - // Wait for all workers to have created their instances - MPI_Barrier(parentComm); - TLLM_LOG_INFO("Executor instance created by worker"); - -#endif // ENABLE_MULTI_DEVICE - - return 0; -} diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.cu b/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.cu index 5be8b1c2ff78..cf0ad7040f59 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.cu @@ -16,6 +16,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h" #include "tensorrt_llm/kernels/quantization.cuh" #include <cooperative_groups.h> @@ -816,7 +817,7 @@ void dispatch_dtype(MiniMaxReduceRMSParams const& params) bool use_float4 = (params.allreduce_in_k != nullptr) && (params.hidden_dim * params.nranks == 6144) && (params.hidden_dim_k * params.nranks == 1024); - if (params.dtype == nvinfer1::DataType::kHALF) + if (params.dtype == tensorrt_llm::DataType::kHALF) { if (use_float4) { @@ -827,7 +828,7 @@ void dispatch_dtype(MiniMaxReduceRMSParams const& params) minimax_reduce_rms_kernel_launcher<half, NRanks>(params); } } - else if (params.dtype == nvinfer1::DataType::kBF16) + else if (params.dtype == tensorrt_llm::DataType::kBF16) { if (use_float4) { @@ -838,7 +839,7 @@ void dispatch_dtype(MiniMaxReduceRMSParams const& params) minimax_reduce_rms_kernel_launcher<__nv_bfloat16, NRanks>(params); } } - else if (params.dtype == nvinfer1::DataType::kFLOAT) + else if (params.dtype == tensorrt_llm::DataType::kFLOAT) { if (use_float4) { diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h b/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h index b0cfd0ca074c..bf5775f96de9 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h @@ -15,7 +15,7 @@ */ #pragma once #include "tensorrt_llm/common/assert.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_bf16.h> #include <cuda_fp16.h> @@ -59,7 +59,7 @@ struct MiniMaxReduceRMSParams { int nranks{}; int rank{}; - nvinfer1::DataType dtype; + tensorrt_llm::DataType dtype; int size_q{}; // numel of Q (num_token * head_dim_q) int hidden_dim{}; // head_dim_q int size_k{}; // numel of K (num_token * head_dim_k) diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu index 5a3edda04a70..d9fbc9da0a0c 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu @@ -16,6 +16,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h" #include "tensorrt_llm/kernels/quantization.cuh" #include <cooperative_groups.h> @@ -795,15 +796,15 @@ void allreduce_fusion_op(AllReduceFusionParams const& params) } #define DISPATCH_DTYPE(NRanks) \ - if (params.dtype == nvinfer1::DataType::kHALF) \ + if (params.dtype == tensorrt_llm::DataType::kHALF) \ { \ DISPATCH_PATTERN(half, NRanks); \ } \ - else if (params.dtype == nvinfer1::DataType::kBF16) \ + else if (params.dtype == tensorrt_llm::DataType::kBF16) \ { \ DISPATCH_PATTERN(__nv_bfloat16, NRanks); \ } \ - else if (params.dtype == nvinfer1::DataType::kFLOAT) \ + else if (params.dtype == tensorrt_llm::DataType::kFLOAT) \ { \ DISPATCH_PATTERN(float, NRanks); \ } \ diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h index 6d2074a6589e..769776273ef6 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h @@ -16,7 +16,7 @@ #pragma once #include "tensorrt_llm/common/assert.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_bf16.h> #include <cuda_fp16.h> @@ -124,7 +124,7 @@ struct AllReduceFusionParams { int nranks; int rank; - nvinfer1::DataType dtype; + tensorrt_llm::DataType dtype; int size; int hidden_dim; void** workspace; diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.cu index f1d5c08bda6b..09a742494a9a 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.cu @@ -21,6 +21,7 @@ #include "tensorrt_llm/common/customAllReduceUtils.h" #include "tensorrt_llm/common/dataType.h" #include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h" #include <cooperative_groups.h> #include <tuple> @@ -1357,7 +1358,7 @@ std::vector<size_t> splitNumber(size_t number) } LowPrecisionAllReduceParams LowPrecisionAllReduceParams::deserialize( - size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, int token_num, int hidden_size) + size_t tpSize, size_t tpRank, tensorrt_llm::DataType dataType, int token_num, int hidden_size) { // Get appropriate static buffer @@ -1401,7 +1402,7 @@ LowPrecisionAllReduceParams LowPrecisionAllReduceParams::deserialize( } LowPrecisionAllReduceParams LowPrecisionAllReduceParams::deserialize_hier( - size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, int token_num, int hidden_size) + size_t tpSize, size_t tpRank, tensorrt_llm::DataType dataType, int token_num, int hidden_size) { // Get appropriate static buffer @@ -1616,7 +1617,7 @@ int32_t max_workspace_size_lowprecision(int32_t tp_size) } void customLowPrecisionAllReduce( - kernels::LowPrecisionAllReduceParams& params, nvinfer1::DataType dataType, cudaStream_t stream) + kernels::LowPrecisionAllReduceParams& params, tensorrt_llm::DataType dataType, cudaStream_t stream) { TLLM_CHECK_WITH_INFO(lowPrecisionConfigurationSupported(params.ranks_per_node, params.elts_total), "Low Precision Custom all-reduce configuration unsupported"); @@ -1625,10 +1626,10 @@ void customLowPrecisionAllReduce( switch (dataType) { - case nvinfer1::DataType::kFLOAT: lowPrecisionAllReduceDispatchType<float>(params, stream); break; - case nvinfer1::DataType::kHALF: lowPrecisionAllReduceDispatchType<half>(params, stream); break; + case tensorrt_llm::DataType::kFLOAT: lowPrecisionAllReduceDispatchType<float>(params, stream); break; + case tensorrt_llm::DataType::kHALF: lowPrecisionAllReduceDispatchType<half>(params, stream); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: lowPrecisionAllReduceDispatchType<__nv_bfloat16>(params, stream); break; + case tensorrt_llm::DataType::kBF16: lowPrecisionAllReduceDispatchType<__nv_bfloat16>(params, stream); break; #endif default: TLLM_THROW("Unsupported dataType for customAllReduce"); } diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h index 5fc87ef1a523..62d19039cc4e 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h @@ -19,8 +19,8 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/customAllReduceKernels.h" -#include <NvInferRuntime.h> #include <cuda_bf16.h> #include <cuda_fp16.h> #include <vector> @@ -111,15 +111,15 @@ struct LowPrecisionAllReduceParams uint64_t* ag_notify_peer_inside_numa_flags[LP_ALLREDUCE_MAX_BLOCKS * 4]; // 3*flags , 3 is other rank inside numa static LowPrecisionAllReduceParams deserialize( - size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, int token_num, int hidden_size); + size_t tpSize, size_t tpRank, tensorrt_llm::DataType dataType, int token_num, int hidden_size); static LowPrecisionAllReduceParams deserialize_hier( - size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, int token_num, int hidden_size); + size_t tpSize, size_t tpRank, tensorrt_llm::DataType dataType, int token_num, int hidden_size); }; bool lowPrecisionConfigurationSupported(size_t msg_size, size_t n_ranks); void customLowPrecisionAllReduce( - kernels::LowPrecisionAllReduceParams& params, nvinfer1::DataType dataType, cudaStream_t stream); + kernels::LowPrecisionAllReduceParams& params, tensorrt_llm::DataType dataType, cudaStream_t stream); int32_t max_workspace_size_lowprecision(int32_t tp_size); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.cu index d376e9791474..1cf871b7459d 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.cu @@ -33,6 +33,7 @@ #include "tensorrt_llm/common/lamportUtils.cuh" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/quantization.cuh" TRTLLM_NAMESPACE_BEGIN @@ -903,9 +904,9 @@ void oneshotAllreduceFusionOp(AllReduceFusionParams const& params) }; #undef LAUNCH_ALLREDUCE_KERNEL #undef DISPATCH_ALLREDUCE_PATTERN - bool launched = (params.dType == nvinfer1::DataType::kBF16 && dispatchImpl((__nv_bfloat16*) nullptr)) - || (params.dType == nvinfer1::DataType::kFLOAT && dispatchImpl((float*) nullptr)) - || (params.dType == nvinfer1::DataType::kHALF && dispatchImpl((__nv_half*) nullptr)); + bool launched = (params.dType == tensorrt_llm::DataType::kBF16 && dispatchImpl((__nv_bfloat16*) nullptr)) + || (params.dType == tensorrt_llm::DataType::kFLOAT && dispatchImpl((float*) nullptr)) + || (params.dType == tensorrt_llm::DataType::kHALF && dispatchImpl((__nv_half*) nullptr)); if (!launched) { TLLM_CHECK_WITH_INFO(false, "Failed to dispatch MNNVL AllReduceOneShot kernel."); @@ -987,24 +988,24 @@ void oneshotMoeFinalizeAllreduceRMSNormOp(MoeFinalizeAllReduceRMSNormParams cons #undef LAUNCH_MNNVL_MOE_FINALIZE_KERNEL bool launched = false; - if (params.dType == nvinfer1::DataType::kBF16) + if (params.dType == tensorrt_llm::DataType::kBF16) { - if (params.scaleDType == nvinfer1::DataType::kFLOAT) + if (params.scaleDType == tensorrt_llm::DataType::kFLOAT) { launched = dispatchImpl((__nv_bfloat16*) nullptr, (float*) nullptr); } - else if (params.scaleDType == nvinfer1::DataType::kBF16) + else if (params.scaleDType == tensorrt_llm::DataType::kBF16) { launched = dispatchImpl((__nv_bfloat16*) nullptr, (__nv_bfloat16*) nullptr); } } - else if (params.dType == nvinfer1::DataType::kHALF) + else if (params.dType == tensorrt_llm::DataType::kHALF) { - if (params.scaleDType == nvinfer1::DataType::kFLOAT) + if (params.scaleDType == tensorrt_llm::DataType::kFLOAT) { launched = dispatchImpl((__nv_half*) nullptr, (float*) nullptr); } - else if (params.scaleDType == nvinfer1::DataType::kHALF) + else if (params.scaleDType == tensorrt_llm::DataType::kHALF) { launched = dispatchImpl((__nv_half*) nullptr, (__nv_half*) nullptr); } @@ -1395,9 +1396,9 @@ void twoshotAllreduceFusionOp(AllReduceFusionParams const& params) #undef LAUNCH_ALLREDUCE_KERNEL - bool launched = (params.dType == nvinfer1::DataType::kFLOAT && dispatchAR((float*) nullptr)) - || (params.dType == nvinfer1::DataType::kBF16 && dispatchAR((__nv_bfloat16*) nullptr)) - || (params.dType == nvinfer1::DataType::kHALF && dispatchAR((__nv_half*) nullptr)); + bool launched = (params.dType == tensorrt_llm::DataType::kFLOAT && dispatchAR((float*) nullptr)) + || (params.dType == tensorrt_llm::DataType::kBF16 && dispatchAR((__nv_bfloat16*) nullptr)) + || (params.dType == tensorrt_llm::DataType::kHALF && dispatchAR((__nv_half*) nullptr)); if (!launched) { TLLM_CHECK_WITH_INFO(false, "[MNNVL AllReduceTwoShot] Failed to dispatch twoshotAllreduce kernel."); @@ -1537,9 +1538,9 @@ void twoshotAllreduceFusionOp(AllReduceFusionParams const& params) return true; }; - launched = (params.dType == nvinfer1::DataType::kFLOAT && dispatchRN((float*) nullptr)) - || (params.dType == nvinfer1::DataType::kBF16 && dispatchRN((__nv_bfloat16*) nullptr)) - || (params.dType == nvinfer1::DataType::kHALF && dispatchRN((__nv_half*) nullptr)); + launched = (params.dType == tensorrt_llm::DataType::kFLOAT && dispatchRN((float*) nullptr)) + || (params.dType == tensorrt_llm::DataType::kBF16 && dispatchRN((__nv_bfloat16*) nullptr)) + || (params.dType == tensorrt_llm::DataType::kHALF && dispatchRN((__nv_half*) nullptr)); if (!launched) { TLLM_CHECK_WITH_INFO(false, "[MNNVL AllReduceTwoShot] Failed to dispatch rmsnorm lamport kernel."); diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.h index 95f6dcda80f3..aa9702a80263 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.h @@ -17,8 +17,8 @@ #define TRTLLM_MNNVL_ALLREDUCE_KERNELS_H #include "tensorrt_llm/common/config.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h" -#include <NvInferRuntime.h> #include <cstdint> TRTLLM_NAMESPACE_BEGIN @@ -39,16 +39,16 @@ struct AllReduceFusionParams //! \name Environmental and Auxiliary Data //! @{ - int nRanks; //!< Total number of participating ranks in the AllReduce operation - int rank; //!< Current rank ID - nvinfer1::DataType dType; //!< Data type of the tensors (e.g., FP16, BF16, FP32) - int numTokens; //!< Number of tokens in the input tensor - int tokenDim; //!< Hidden Dimension - void** bufferPtrsDev; //!< Unicast Device pointers to communication buffers for each rank - void* bufferPtrLocal; //!< Local buffer pointer for temporary storage (i.e., bufferPtrsDev[rank]) - void* multicastPtr; //!< Multicast buffer pointer. - uint32_t* bufferFlags; //!< Synchronization flags for coordinating communication phases - bool rmsNormFusion; //!< Whether to fuse RMS normalization with the AllReduce operation + int nRanks; //!< Total number of participating ranks in the AllReduce operation + int rank; //!< Current rank ID + tensorrt_llm::DataType dType; //!< Data type of the tensors (e.g., FP16, BF16, FP32) + int numTokens; //!< Number of tokens in the input tensor + int tokenDim; //!< Hidden Dimension + void** bufferPtrsDev; //!< Unicast Device pointers to communication buffers for each rank + void* bufferPtrLocal; //!< Local buffer pointer for temporary storage (i.e., bufferPtrsDev[rank]) + void* multicastPtr; //!< Multicast buffer pointer. + uint32_t* bufferFlags; //!< Synchronization flags for coordinating communication phases + bool rmsNormFusion; //!< Whether to fuse RMS normalization with the AllReduce operation ar_fusion::AllReduceFusionPattern pattern = ar_fusion::AllReduceFusionPattern::kAllReduce; //!< Fused epilogue pattern @@ -83,7 +83,7 @@ struct AllReduceFusionParams struct MoeFinalizeAllReduceRMSNormParams : public AllReduceFusionParams { int topK; //!< Number of selected routes per token - nvinfer1::DataType scaleDType; //!< Data type of expertScaleFactor + tensorrt_llm::DataType scaleDType; //!< Data type of expertScaleFactor void const* expertScaleFactor; //!< Per-token route weights [numTokens, topK] int32_t const* expandedIdxToPermutedIdx; //!< Route-to-permuted-row map [numTokens, topK] }; diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.cu index 306d42677e2f..5b0da0d0e80c 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.cu @@ -16,6 +16,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h" #include "tensorrt_llm/kernels/quantization.cuh" #include <cooperative_groups.h> @@ -442,11 +443,11 @@ void moereduction_allreduce_fusion_op(MoeReductionAllReduceFusionParams const& p #define MOE_DISPATCH1(DTYPE, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT) \ return moereduction_allreduce_fusion_kernel_launcher<DTYPE, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT>(params); #define MOE_DISPATCH0(NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT) \ - if (params.nranks == NRANKS && params.dtype == nvinfer1::DataType::kHALF) \ + if (params.nranks == NRANKS && params.dtype == tensorrt_llm::DataType::kHALF) \ { \ MOE_DISPATCH1(half, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT); \ } \ - else if (params.nranks == NRANKS && params.dtype == nvinfer1::DataType::kBF16) \ + else if (params.nranks == NRANKS && params.dtype == tensorrt_llm::DataType::kBF16) \ { \ MOE_DISPATCH1(__nv_bfloat16, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT); \ } @@ -556,11 +557,20 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport(MoeFinalizeA } // * MoE finalize - ACC_TYPE accumulator; + // Accumulate the top-k weighted expert sum and the shared-expert add in + // fp32 (local `facc`), rounding to DType (bf16/fp16) only once when packing + // into `accumulator` for the 128-bit Lamport all-reduce store below. + // Accumulating directly in DType here rounds after every one of the top_k + // terms; across the many routed MoE layers this rounding bias is large + // enough to visibly degrade the routed output, and with attention-DP + // disabled + MTP speculative decoding it drifts the target hidden states + // enough to lower the acceptance length. The non-deferred in-kernel + // finalize (do_finalize=true) already accumulates in fp32; match it here. + float facc[kElemsPerAccess]; #pragma unroll for (int i = 0; i < kElemsPerAccess; ++i) { - accumulator.unpacked[i] = static_cast<DType>(0); + facc[i] = 0.f; } for (int k = 0; k < top_k; k++) @@ -582,17 +592,15 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport(MoeFinalizeA permuted_data.packed = reinterpret_cast<float4 const*>(params.allreduce_in)[thread_offset_across_token / kElemsPerAccess]; - // * acc += scale(data) + // * acc += scale(data) (fp32 accumulation) #pragma unroll for (int i = 0; i < kElemsPerAccess; ++i) { - // assume computation is done in ScaleType - accumulator.unpacked[i] - += static_cast<DType>((static_cast<float>(permuted_data.unpacked[i]) * block_scale)); + facc[i] += static_cast<float>(permuted_data.unpacked[i]) * block_scale; } } - // * Add shared expert output + // * Add shared expert output (fp32 accumulation) if (params.shared_expert_output) { // * Load shared expert output @@ -603,10 +611,18 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport(MoeFinalizeA #pragma unroll for (int i = 0; i < kElemsPerAccess; ++i) { - accumulator.unpacked[i] += shared_expert_output.unpacked[i]; + facc[i] += static_cast<float>(shared_expert_output.unpacked[i]); } } + // Round the fp32 accumulator to DType once, packed for the Lamport AR store. + ACC_TYPE accumulator; +#pragma unroll + for (int i = 0; i < kElemsPerAccess; ++i) + { + accumulator.unpacked[i] = static_cast<DType>(facc[i]); + } + // * AR Store int access_id = token_id * params.hidden_dim / kElemsPerAccess + access_id_in_token; int idx = access_id; @@ -727,13 +743,13 @@ void moefinalize_allreduce_fusion_op(MoeFinalizeAllReduceFusionParams const& par #define MOE_FINALIZE_DISPATCH1(DTYPE, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT) \ return moefinalize_allreduce_fusion_kernel_launcher<DTYPE, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT>(params); #define MOE_FINALIZE_DISPATCH0(NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT) \ - if (params.nranks == NRANKS && params.dtype == nvinfer1::DataType::kHALF \ - && params.scale_dtype == nvinfer1::DataType::kHALF) \ + if (params.nranks == NRANKS && params.dtype == tensorrt_llm::DataType::kHALF \ + && params.scale_dtype == tensorrt_llm::DataType::kHALF) \ { \ MOE_FINALIZE_DISPATCH1(half, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT); \ } \ - else if (params.nranks == NRANKS && params.dtype == nvinfer1::DataType::kBF16 \ - && params.scale_dtype == nvinfer1::DataType::kBF16) \ + else if (params.nranks == NRANKS && params.dtype == tensorrt_llm::DataType::kBF16 \ + && params.scale_dtype == tensorrt_llm::DataType::kBF16) \ { \ MOE_FINALIZE_DISPATCH1(__nv_bfloat16, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT); \ } diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h index 556dd4e5cd24..e526a70268b3 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h @@ -16,7 +16,7 @@ #pragma once #include "tensorrt_llm/common/assert.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_bf16.h> #include <cuda_fp16.h> @@ -44,7 +44,7 @@ struct AllReduceFusionParams { int nranks; int rank; - nvinfer1::DataType dtype; + tensorrt_llm::DataType dtype; // size = token_num * hidden_dim int size; int hidden_dim; @@ -94,7 +94,7 @@ struct MoeFinalizeAllReduceFusionParams : public AllReduceFusionParams // Refer to kernel implementation on layout of those params // number of active experts on current device int top_k; - nvinfer1::DataType scale_dtype; + tensorrt_llm::DataType scale_dtype; // [num_tokens, top_k] void* expert_scale_factor = nullptr; void* shared_expert_output = nullptr; diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu index 472a5877a80d..74e40dbb2b81 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu @@ -17,6 +17,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/common/vec_dtypes.cuh" #include "tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h" #include "tensorrt_llm/kernels/quantization.cuh" @@ -129,25 +130,25 @@ using tensorrt_llm::common::launchWithPdlWhenEnabled; #define SWITCH_DTYPE(dtype, TYPE, ...) \ switch (dtype) \ { \ - case nvinfer1::DataType::kHALF: \ + case tensorrt_llm::DataType::kHALF: \ { \ using TYPE = half; \ __VA_ARGS__; \ break; \ } \ - case nvinfer1::DataType::kBF16: \ + case tensorrt_llm::DataType::kBF16: \ { \ using TYPE = __nv_bfloat16; \ __VA_ARGS__; \ break; \ } \ - case nvinfer1::DataType::kFLOAT: \ + case tensorrt_llm::DataType::kFLOAT: \ { \ using TYPE = float; \ __VA_ARGS__; \ break; \ } \ - case nvinfer1::DataType::kFP8: \ + case tensorrt_llm::DataType::kFP8: \ { \ using TYPE = __nv_fp8_e4m3; \ __VA_ARGS__; \ @@ -1403,7 +1404,7 @@ void moe_a2a_combine_launch(MoeA2ACombineParams const& params) // When use_low_precision is set the recv buffers contain FP8 data regardless of params.dtype, // so dispatch the FP8 accumulation kernel in that case. - auto const effective_dtype = params.use_low_precision ? nvinfer1::DataType::kFP8 : params.dtype; + auto const effective_dtype = params.use_low_precision ? tensorrt_llm::DataType::kFP8 : params.dtype; // Launch appropriate kernel with compact macros SWITCH_BOOL(params.enable_rank_mask, ENABLE_RANK_MASK, { diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h index 177293684874..5184878ffc51 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h @@ -16,7 +16,7 @@ #pragma once #include "tensorrt_llm/common/config.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_bf16.h> #include <cuda_fp16.h> @@ -174,8 +174,8 @@ struct MoeA2ACombineParams // Output tensor void* output_data; // Output buffer [local_num_tokens, elements_per_token] // Payload information - int elements_per_token; // Number of elements per token - nvinfer1::DataType dtype; // Data type of the payload (used for combine kernel dispatch) + int elements_per_token; // Number of elements per token + tensorrt_llm::DataType dtype; // Data type of the payload (used for combine kernel dispatch) bool use_low_precision; // If true, prepare kernel quantizes payload→FP8; combine kernel accumulates FP8→output dtype diff --git a/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.cu b/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.cu index 81e947977797..8af43b2b4914 100644 --- a/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.cu +++ b/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.cu @@ -23,6 +23,7 @@ #include "cutlass/cutlass.h" #include "cutlass/gemm/device/gemm_grouped.h" #include "cutlass/gemm/kernel/default_gemm_grouped.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/device/splitk_gemm_grouped.h" #include "tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/default_splitk_gemm_grouped.h" @@ -117,16 +118,16 @@ void cudaGraphGroupedGemmTemplate(cutlass::gemm::GemmCoord* problemSizesPtr, int template <int M1, int N1, int K1, int M2, int N2, int K2, int kAlignmentAB, int kAlignmentC, int kStages> void cudaGraphGroupedGemmType(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, - nvinfer1::DataType dataType, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream) + tensorrt_llm::DataType dataType, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream) { - if (dataType == nvinfer1::DataType::kHALF) + if (dataType == tensorrt_llm::DataType::kHALF) { cudaGraphGroupedGemmTemplate<M1, N1, K1, M2, N2, K2, cutlass::half_t, kAlignmentAB, kAlignmentC, kStages>( problemSizesPtr, problemCount, ptrAGpu, ptrBGpu, ptrCGpu, ptrDGpu, ldaGpu, ldbGpu, ldcGpu, lddGpu, hostMaxProblemSizesPtr, stream); } #ifdef ENABLE_BF16 - else if (dataType == nvinfer1::DataType::kBF16) + else if (dataType == tensorrt_llm::DataType::kBF16) { cudaGraphGroupedGemmTemplate<M1, N1, K1, M2, N2, K2, cutlass::bfloat16_t, kAlignmentAB, kAlignmentC, kStages>( problemSizesPtr, problemCount, ptrAGpu, ptrBGpu, ptrCGpu, ptrDGpu, ldaGpu, ldbGpu, ldcGpu, lddGpu, @@ -141,7 +142,7 @@ void cudaGraphGroupedGemmType(cutlass::gemm::GemmCoord* problemSizesPtr, int pro void cudaGraphGroupedGemm(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, bool isLoraIn, - nvinfer1::DataType dataType, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream) + tensorrt_llm::DataType dataType, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream) { if (isLoraIn) { @@ -283,17 +284,17 @@ void cudaGraphSplitKGroupedGemmTemplate(cutlass::gemm::GemmCoord* problemSizesPt template <int M1, int N1, int K1, int M2, int N2, int K2, int kAlignmentAB, int kAlignmentC, int kStages> void cudaGraphSplitKGroupedGemmType(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, - nvinfer1::DataType dataType, int splitKSlices, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, + tensorrt_llm::DataType dataType, int splitKSlices, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, int64_t* splitKOffsetsGpu, cudaStream_t stream) { - if (dataType == nvinfer1::DataType::kHALF) + if (dataType == tensorrt_llm::DataType::kHALF) { cudaGraphSplitKGroupedGemmTemplate<M1, N1, K1, M2, N2, K2, cutlass::half_t, kAlignmentAB, kAlignmentC, kStages>( problemSizesPtr, problemCount, ptrAGpu, ptrBGpu, ptrCGpu, ptrDGpu, ldaGpu, ldbGpu, ldcGpu, lddGpu, splitKSlices, hostMaxProblemSizesPtr, splitKOffsetsGpu, stream); } #ifdef ENABLE_BF16 - else if (dataType == nvinfer1::DataType::kBF16) + else if (dataType == tensorrt_llm::DataType::kBF16) { cudaGraphSplitKGroupedGemmTemplate<M1, N1, K1, M2, N2, K2, cutlass::bfloat16_t, kAlignmentAB, kAlignmentC, kStages>(problemSizesPtr, problemCount, ptrAGpu, ptrBGpu, ptrCGpu, ptrDGpu, ldaGpu, ldbGpu, ldcGpu, lddGpu, @@ -308,7 +309,7 @@ void cudaGraphSplitKGroupedGemmType(cutlass::gemm::GemmCoord* problemSizesPtr, i void cudaGraphSplitKGroupedGemm(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, - bool isLoraIn, nvinfer1::DataType dataType, int splitKSlices, int minKN, + bool isLoraIn, tensorrt_llm::DataType dataType, int splitKSlices, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, int64_t* splitKOffsetsGpu, cudaStream_t stream) { if (isLoraIn) diff --git a/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.h b/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.h index 0eecccb78852..b447bba3a785 100644 --- a/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.h +++ b/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.h @@ -18,7 +18,7 @@ #include "cutlass/gemm_coord.h" #include "tensorrt_llm/common/config.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_runtime.h> TRTLLM_NAMESPACE_BEGIN @@ -45,7 +45,7 @@ namespace kernels */ void cudaGraphGroupedGemm(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, bool isLoraIn, - nvinfer1::DataType dataType, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream); + tensorrt_llm::DataType dataType, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream); /** * @brief CUDA Graph compatible wrapper for split-K grouped GEMM operations. @@ -55,7 +55,7 @@ void cudaGraphGroupedGemm(cutlass::gemm::GemmCoord* problemSizesPtr, int problem */ void cudaGraphSplitKGroupedGemm(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, - bool isLoraIn, nvinfer1::DataType dataType, int splitKSlices, int minKN, + bool isLoraIn, tensorrt_llm::DataType dataType, int splitKSlices, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, int64_t* splitKOffsetsGpu, cudaStream_t stream); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/customAllReduceKernels.cu b/cpp/tensorrt_llm/kernels/customAllReduceKernels.cu index 9cf2b51eb583..ea217d465be1 100644 --- a/cpp/tensorrt_llm/kernels/customAllReduceKernels.cu +++ b/cpp/tensorrt_llm/kernels/customAllReduceKernels.cu @@ -22,6 +22,7 @@ #include "tensorrt_llm/common/customAllReduceUtils.h" #include "tensorrt_llm/common/dataType.h" #include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include <cooperative_groups.h> #include <cstdint> #include <tuple> @@ -1187,14 +1188,14 @@ bool is_lamport_supported(int token_num, int hidden_size) return true; } -bool is_lamport_supported(nvinfer1::DataType dataType, int token_num, int hidden_size) +bool is_lamport_supported(tensorrt_llm::DataType dataType, int token_num, int hidden_size) { switch (dataType) { - case nvinfer1::DataType::kFLOAT: return is_lamport_supported<float>(token_num, hidden_size); - case nvinfer1::DataType::kHALF: return is_lamport_supported<half>(token_num, hidden_size); + case tensorrt_llm::DataType::kFLOAT: return is_lamport_supported<float>(token_num, hidden_size); + case tensorrt_llm::DataType::kHALF: return is_lamport_supported<half>(token_num, hidden_size); #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: return is_lamport_supported<__nv_bfloat16>(token_num, hidden_size); + case tensorrt_llm::DataType::kBF16: return is_lamport_supported<__nv_bfloat16>(token_num, hidden_size); #endif default: return false; } @@ -1658,7 +1659,7 @@ static __global__ void __launch_bounds__(512, 1) twoShotAllReduceKernel(AllReduc update_barrier_flag(params.barrier_flag_ptr, params.barrier_flag_counter_ptr); } -bool configurationSupported(AllReduceStrategyType algo, size_t msg_size, size_t n_ranks, nvinfer1::DataType type) +bool configurationSupported(AllReduceStrategyType algo, size_t msg_size, size_t n_ranks, tensorrt_llm::DataType type) { size_t elts_per_thread = 16 / common::getDTypeSize(type); int const msg_align = (algo == AllReduceStrategyType::TWOSHOT) ? n_ranks * elts_per_thread : elts_per_thread; @@ -1894,8 +1895,8 @@ void AllReduceDispatchType(AllReduceParams& params, AllReduceStrategyType strat, } } -AllReduceParams AllReduceParams::deserialize(int64_t* buffer, size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, - int token_num, int hidden_size, AllReduceFusionOp op) +AllReduceParams AllReduceParams::deserialize(int64_t* buffer, size_t tpSize, size_t tpRank, + tensorrt_llm::DataType dataType, int token_num, int hidden_size, AllReduceFusionOp op) { void* const* buffer_ptrs = reinterpret_cast<void* const*>(buffer); int flag_offset; @@ -1933,7 +1934,7 @@ AllReduceParams AllReduceParams::deserialize(int64_t* buffer, size_t tpSize, siz return params; } -void customAllReduce(kernels::AllReduceParams& params, nvinfer1::DataType dataType, AllReduceStrategyType strat, +void customAllReduce(kernels::AllReduceParams& params, tensorrt_llm::DataType dataType, AllReduceStrategyType strat, AllReduceStrategyConfig config, AllReduceFusionOp fusionOp, cudaStream_t stream) { TLLM_CHECK_WITH_INFO(configurationSupported(strat, params.elts_total, params.ranks_per_node, dataType), @@ -1943,10 +1944,10 @@ void customAllReduce(kernels::AllReduceParams& params, nvinfer1::DataType dataTy switch (dataType) { - case nvinfer1::DataType::kFLOAT: AllReduceDispatchType<float>(params, strat, config, fusionOp, stream); break; - case nvinfer1::DataType::kHALF: AllReduceDispatchType<half>(params, strat, config, fusionOp, stream); break; + case tensorrt_llm::DataType::kFLOAT: AllReduceDispatchType<float>(params, strat, config, fusionOp, stream); break; + case tensorrt_llm::DataType::kHALF: AllReduceDispatchType<half>(params, strat, config, fusionOp, stream); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: AllReduceDispatchType<__nv_bfloat16>(params, strat, config, fusionOp, stream); break; #endif @@ -1991,22 +1992,22 @@ void launchResidualRmsNormKernel(kernels::AllReduceParams& params, cudaStream_t } void residualRmsNorm( - kernels::AllReduceParams& params, nvinfer1::DataType dataType, cudaStream_t stream, AllReduceFusionOp fusionOp) + kernels::AllReduceParams& params, tensorrt_llm::DataType dataType, cudaStream_t stream, AllReduceFusionOp fusionOp) { sync_check_cuda_error(stream); switch (dataType) { - case nvinfer1::DataType::kFLOAT: launchResidualRmsNormKernel<float>(params, stream, fusionOp); break; - case nvinfer1::DataType::kHALF: launchResidualRmsNormKernel<half>(params, stream, fusionOp); break; + case tensorrt_llm::DataType::kFLOAT: launchResidualRmsNormKernel<float>(params, stream, fusionOp); break; + case tensorrt_llm::DataType::kHALF: launchResidualRmsNormKernel<half>(params, stream, fusionOp); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: launchResidualRmsNormKernel<__nv_bfloat16>(params, stream, fusionOp); break; + case tensorrt_llm::DataType::kBF16: launchResidualRmsNormKernel<__nv_bfloat16>(params, stream, fusionOp); break; #endif default: TLLM_THROW("Unsupported dataType for customAllReduce"); } sync_check_cuda_error(stream); } -void lamportInitialize(void* buffer, size_t size, nvinfer1::DataType dataType, cudaStream_t stream) +void lamportInitialize(void* buffer, size_t size, tensorrt_llm::DataType dataType, cudaStream_t stream) { sync_check_cuda_error(stream); if (size == 0) @@ -2015,14 +2016,14 @@ void lamportInitialize(void* buffer, size_t size, nvinfer1::DataType dataType, c } switch (dataType) { - case nvinfer1::DataType::kFLOAT: + case tensorrt_llm::DataType::kFLOAT: reduce_fusion::lamport_initialize_kernel_launcher<float>(buffer, size, stream); break; - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: reduce_fusion::lamport_initialize_kernel_launcher<half>(buffer, size, stream); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: reduce_fusion::lamport_initialize_kernel_launcher<__nv_bfloat16>(buffer, size, stream); break; #endif diff --git a/cpp/tensorrt_llm/kernels/customAllReduceKernels.h b/cpp/tensorrt_llm/kernels/customAllReduceKernels.h index f7151f1cd0ab..93f67ffdd911 100644 --- a/cpp/tensorrt_llm/kernels/customAllReduceKernels.h +++ b/cpp/tensorrt_llm/kernels/customAllReduceKernels.h @@ -17,7 +17,7 @@ #pragma once #include "tensorrt_llm/common/assert.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_bf16.h> #include <cuda_fp16.h> #include <limits> @@ -178,23 +178,23 @@ struct AllReduceParams AllReduceFusionParams fusion_params; - static AllReduceParams deserialize(int64_t* buffer, size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, + static AllReduceParams deserialize(int64_t* buffer, size_t tpSize, size_t tpRank, tensorrt_llm::DataType dataType, int token_num, int hidden_size, AllReduceFusionOp op); }; -bool configurationSupported(AllReduceStrategyType algo, size_t msg_size, size_t n_ranks, nvinfer1::DataType type); +bool configurationSupported(AllReduceStrategyType algo, size_t msg_size, size_t n_ranks, tensorrt_llm::DataType type); -void customAllReduce(kernels::AllReduceParams& params, nvinfer1::DataType dataType, AllReduceStrategyType strat, +void customAllReduce(kernels::AllReduceParams& params, tensorrt_llm::DataType dataType, AllReduceStrategyType strat, AllReduceStrategyConfig config, AllReduceFusionOp fusionOp, cudaStream_t stream); void residualRmsNorm( - kernels::AllReduceParams& params, nvinfer1::DataType dataType, cudaStream_t stream, AllReduceFusionOp fusionOp); + kernels::AllReduceParams& params, tensorrt_llm::DataType dataType, cudaStream_t stream, AllReduceFusionOp fusionOp); -void lamportInitialize(void* buffer, size_t size, nvinfer1::DataType dataType, cudaStream_t stream); +void lamportInitialize(void* buffer, size_t size, tensorrt_llm::DataType dataType, cudaStream_t stream); namespace reduce_fusion { -bool is_lamport_supported(nvinfer1::DataType dataType, int token_num, int hidden_size); +bool is_lamport_supported(tensorrt_llm::DataType dataType, int token_num, int hidden_size); } } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.cpp b/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.cpp index 7bba57a03d5e..bcc0eb1165ac 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.cpp +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #include "tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.h" #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaBf16Wrapper.h" +#include "tensorrt_llm/common/cudaUtils.h" #ifdef __GNUC__ // Check if the compiler is GCC or Clang #pragma GCC diagnostic push @@ -31,6 +32,7 @@ #pragma GCC diagnostic pop #endif // __GNUC +#include <algorithm> #include <cuda_runtime_api.h> #include <set> #include <vector> @@ -573,7 +575,7 @@ std::vector<CutlassGemmConfig> get_candidate_configs_sm120(CutlassGemmConfig::Ca std::vector<CutlassGemmConfig> candidate_configs; if (config & CutlassGemmConfig::FP8FP4_MIXED) { - // Mixed FP8 x FP4: restrict to 128x128x128B only + // Mixed FP8 x FP4 only supports the 128x128x128B tile. candidate_configs.push_back(CutlassGemmConfig{CutlassTileConfigSM120::CtaShape128x128x128B, MainloopScheduleType::AUTO, EpilogueScheduleType::AUTO, ClusterShape::ClusterShape_1x1x1}); return candidate_configs; @@ -589,9 +591,34 @@ std::vector<CutlassGemmConfig> get_candidate_configs_sm120(CutlassGemmConfig::Ca MainloopScheduleType::AUTO, EpilogueScheduleType::AUTO, ClusterShape::ClusterShape_1x1x1}); candidate_configs.push_back(CutlassGemmConfig{CutlassTileConfigSM120::CtaShape256x128x64B, MainloopScheduleType::AUTO, EpilogueScheduleType::AUTO, ClusterShape::ClusterShape_1x1x1}); - return candidate_configs; } - TLLM_THROW("Not Implemented: SM120 group GEMM only supports mxfp8-mxfp4 mixed or nvfp4."); + else + { + TLLM_THROW("Not Implemented: SM120 group GEMM only supports mxfp8-mxfp4 mixed or nvfp4."); + } + // Filter configs by device shared memory. SM100 (B200) has 228 KiB, but + // consumer Blackwell (SM120 RTX PRO 6000, SM121 GB10 / DGX Spark) has only + // 99 KiB. On these constrained devices, keep only CtaShape128x128x64B which + // fits within 99 KiB including FINALIZE epilogue (~80 KiB total). + // CtaShape128x256x64B/256x128x64B overflow with FINALIZE (~100 KiB). + // CtaShape128x128x128B also exceeds 99 KiB at typical stage counts. + { + constexpr int kMinSmemForFullTileSet = 120 * 1024; + int device = 0; + tensorrt_llm::common::check_cuda_error(cudaGetDevice(&device)); + int maxSmem = 0; + tensorrt_llm::common::check_cuda_error( + cudaDeviceGetAttribute(&maxSmem, cudaDevAttrMaxSharedMemoryPerBlockOptin, device)); + + if (maxSmem < kMinSmemForFullTileSet) + { + auto const it = std::remove_if(candidate_configs.begin(), candidate_configs.end(), + [](CutlassGemmConfig const& config) + { return config.tile_config_sm120 != CutlassTileConfigSM120::CtaShape128x128x64B; }); + candidate_configs.erase(it, candidate_configs.end()); + } + } + return candidate_configs; } else { diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h b/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h index dbbed4e08c97..6632f273cc35 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h @@ -17,7 +17,7 @@ #pragma once #include "tensorrt_llm/common/config.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include "cutlass/half.h" #include <cuda_fp16.h> @@ -38,34 +38,34 @@ namespace kernels namespace cutlass_kernels { /////////////////////////////////////////////////////////////////////////////////////////////////// -// nvinfer1::DataType to Cutlass +// tensorrt_llm::DataType to Cutlass /////////////////////////////////////////////////////////////////////////////////////////////////// -template <nvinfer1::DataType> +template <tensorrt_llm::DataType> struct CutlassType { using type = void; }; template <> -struct CutlassType<nvinfer1::DataType::kHALF> +struct CutlassType<tensorrt_llm::DataType::kHALF> { using type = cutlass::half_t; }; template <> -struct CutlassType<nvinfer1::DataType::kBF16> +struct CutlassType<tensorrt_llm::DataType::kBF16> { using type = cutlass::bfloat16_t; }; template <> -struct CutlassType<nvinfer1::DataType::kFP8> +struct CutlassType<tensorrt_llm::DataType::kFP8> { using type = cutlass::float_e4m3_t; }; template <> -struct CutlassType<nvinfer1::DataType::kFP4> +struct CutlassType<tensorrt_llm::DataType::kFP4> { using type = cutlass::float_e2m1_t; }; diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.cu b/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.cu index a4923fdbd072..5d6d37237776 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.cu +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.cu @@ -40,6 +40,17 @@ namespace kernels::fp8_blockscale_gemm namespace { +// CUDA limits grid.y and grid.z to 65535. Distribute row blocks across both +// dimensions while keeping the total number of launched blocks near mBlocks. +dim3 makeQuantizeGrid(int numPackedSfK, int mBlocks) +{ + constexpr int kMaxGridDimY = 65535; + int const gridZ = (mBlocks + kMaxGridDimY - 1) / kMaxGridDimY; + int const gridY = (mBlocks + gridZ - 1) / gridZ; + return { + static_cast<unsigned int>(numPackedSfK), static_cast<unsigned int>(gridY), static_cast<unsigned int>(gridZ)}; +} + __device__ __forceinline__ float reciprocal_approximate_ftz_local(float a) { float b; @@ -52,7 +63,7 @@ __device__ __forceinline__ float reciprocal_approximate_ftz_local(float a) // (8 lanes × 16 BF16 elems = 128 elems). After per-block amax, lanes // 0/8/16/24 each hold one UE8M0 scale byte; lane 0 packs them into a uint32 // and stores in the deep_gemm-expected MN-major layout. -template <int WarpsPerBlock> +template <int WarpsPerBlock, bool OutputCuteDslSf> __global__ void fp8_quantize_1x128_packed_kernel_impl(__nv_fp8_e4m3* __restrict__ fp8_output, int32_t* __restrict__ packed_scale_output, __nv_bfloat16 const* __restrict__ input, int const m, int const k, int const scale_leading_dim_uint32) @@ -60,7 +71,8 @@ __global__ void fp8_quantize_1x128_packed_kernel_impl(__nv_fp8_e4m3* __restrict_ int const packed_sf_k_idx = static_cast<int>(blockIdx.x); int const warp_id = static_cast<int>(threadIdx.x) >> 5; int const lane_id = static_cast<int>(threadIdx.x) & 31; - int const m_idx = static_cast<int>(blockIdx.y) * WarpsPerBlock + warp_id; + int64_t const mBlockIdx = static_cast<int64_t>(blockIdx.z) * gridDim.y + blockIdx.y; + int64_t const m_idx = mBlockIdx * WarpsPerBlock + warp_id; #if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) cudaGridDependencySynchronize(); @@ -72,6 +84,7 @@ __global__ void fp8_quantize_1x128_packed_kernel_impl(__nv_fp8_e4m3* __restrict_ bool const row_in_range = (m_idx < m); uint32_t packed = 0u; + uint32_t scale_byte = 0u; if (row_in_range) { int const k_base = packed_sf_k_idx * 512 + lane_id * 16; @@ -120,6 +133,7 @@ __global__ void fp8_quantize_1x128_packed_kernel_impl(__nv_fp8_e4m3* __restrict_ float const dequant_scale_raw = amax * reciprocal_approximate_ftz_local(448.0f); __nv_fp8_e8m0 ue8m0_scale; ue8m0_scale.__x = __nv_cvt_float_to_e8m0(dequant_scale_raw, __NV_SATFINITE, cudaRoundPosInf); + scale_byte = static_cast<uint32_t>(ue8m0_scale.__x); // Recover quant_scale = 1 / 2^(exp - 127) for fp8 conversion. constexpr uint32_t FP32_EXPONENT_BIAS = 127u; @@ -162,32 +176,58 @@ __global__ void fp8_quantize_1x128_packed_kernel_impl(__nv_fp8_e4m3* __restrict_ } // ---- 5. Pack 4 UE8M0 scales (lanes 0/8/16/24). ---- - uint32_t const s0 = __shfl_sync(0xFFFFFFFFu, static_cast<uint32_t>(ue8m0_scale.__x), 0); - uint32_t const s1 = __shfl_sync(0xFFFFFFFFu, static_cast<uint32_t>(ue8m0_scale.__x), 8); - uint32_t const s2 = __shfl_sync(0xFFFFFFFFu, static_cast<uint32_t>(ue8m0_scale.__x), 16); - uint32_t const s3 = __shfl_sync(0xFFFFFFFFu, static_cast<uint32_t>(ue8m0_scale.__x), 24); - if (lane_id == 0) + if constexpr (!OutputCuteDslSf) { - // Mask off scale bytes whose sf_k is past the actual K. - int const num_sf_k = (k + 127) / 128; - int const sf_k_base = packed_sf_k_idx * 4; - if (sf_k_base + 0 < num_sf_k) - packed |= s0; - if (sf_k_base + 1 < num_sf_k) - packed |= (s1 << 8); - if (sf_k_base + 2 < num_sf_k) - packed |= (s2 << 16); - if (sf_k_base + 3 < num_sf_k) - packed |= (s3 << 24); + uint32_t const s0 = __shfl_sync(0xFFFFFFFFu, scale_byte, 0); + uint32_t const s1 = __shfl_sync(0xFFFFFFFFu, scale_byte, 8); + uint32_t const s2 = __shfl_sync(0xFFFFFFFFu, scale_byte, 16); + uint32_t const s3 = __shfl_sync(0xFFFFFFFFu, scale_byte, 24); + if (lane_id == 0) + { + // Mask off scale bytes whose sf_k is past the actual K. + int const num_sf_k = (k + 127) / 128; + int const sf_k_base = packed_sf_k_idx * 4; + if (sf_k_base + 0 < num_sf_k) + packed |= s0; + if (sf_k_base + 1 < num_sf_k) + packed |= (s1 << 8); + if (sf_k_base + 2 < num_sf_k) + packed |= (s2 << 16); + if (sf_k_base + 3 < num_sf_k) + packed |= (s3 << 24); + } } } - // Always write the packed scale — `packed` is 0 for padded rows. The grid - // covers the full [0, scale_leading_dim_uint32) leading dim (rounded up to - // WarpsPerBlock), and the m_idx guard drops the few rows past the buffer end. - if (lane_id == 0 && m_idx < scale_leading_dim_uint32) + if constexpr (OutputCuteDslSf) { - packed_scale_output[static_cast<int64_t>(packed_sf_k_idx) * scale_leading_dim_uint32 + m_idx] = packed; + // Native MXF8 MMA consumes one UE8M0 scale per 32 K values. Preserve + // the production 1x128 quantization contract by replicating each scale + // four times directly into CUTLASS/CuTe's 128x4 swizzled layout. + if (lane_id % 8 == 0 && m_idx < scale_leading_dim_uint32) + { + int const sf128_idx = packed_sf_k_idx * 4 + lane_id / 8; + int const num_sf128 = (k + 127) / 128; + if (sf128_idx < num_sf128) + { + int const num_sf32 = (k + 31) / 32; + int const num_k_tiles = (num_sf32 + 3) / 4; + int64_t const dst_offset = static_cast<int64_t>(m_idx / 128) * num_k_tiles * 512 + + static_cast<int64_t>(sf128_idx) * 512 + (m_idx % 32) * 16 + ((m_idx % 128) / 32) * 4; + uint32_t const replicated = row_in_range ? scale_byte * 0x01010101u : 0u; + *reinterpret_cast<uint32_t*>(reinterpret_cast<uint8_t*>(packed_scale_output) + dst_offset) = replicated; + } + } + } + else + { + // Always write the packed scale — `packed` is 0 for padded rows. The grid + // covers the full [0, scale_leading_dim_uint32) leading dim (rounded up to + // WarpsPerBlock), and the m_idx guard drops the few rows past the buffer end. + if (lane_id == 0 && m_idx < scale_leading_dim_uint32) + { + packed_scale_output[static_cast<int64_t>(packed_sf_k_idx) * scale_leading_dim_uint32 + m_idx] = packed; + } } #if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -211,12 +251,31 @@ void launch_fp8_quantize_1x128_packed_bf16_e4m3(__nv_fp8_e4m3* fp8_output, int32 // extent of packed_scale_output is written (in-kernel zero for rows past `m`), // regardless of how the caller padded the input. int const m_blocks = (scale_leading_dim_uint32 + kWarpsPerBlock - 1) / kWarpsPerBlock; - dim3 const grid(num_packed_sf_k, m_blocks, 1); + dim3 const grid = makeQuantizeGrid(num_packed_sf_k, m_blocks); dim3 const block(kWarpsPerBlock * 32, 1, 1); tensorrt_llm::common::launchWithPdlWhenEnabled("fp8_quantize_1x128_packed_kernel_impl", - fp8_quantize_1x128_packed_kernel_impl<kWarpsPerBlock>, grid, block, 0, stream, fp8_output, packed_scale_output, - input, m, k, scale_leading_dim_uint32); + fp8_quantize_1x128_packed_kernel_impl<kWarpsPerBlock, false>, grid, block, 0, stream, fp8_output, + packed_scale_output, input, m, k, scale_leading_dim_uint32); +} + +void launch_fp8_quantize_1x128_cutedsl_bf16_e4m3(__nv_fp8_e4m3* fp8_output, uint8_t* swizzled_scale_output, + __nv_bfloat16 const* input, int m, int k, int padded_m, cudaStream_t stream) +{ + if (m <= 0 || k <= 0) + { + return; + } + + constexpr int kWarpsPerBlock = 4; + int const num_packed_sf_k = (((k + 127) / 128) + 3) / 4; + int const m_blocks = (padded_m + kWarpsPerBlock - 1) / kWarpsPerBlock; + dim3 const grid = makeQuantizeGrid(num_packed_sf_k, m_blocks); + dim3 const block(kWarpsPerBlock * 32, 1, 1); + + tensorrt_llm::common::launchWithPdlWhenEnabled("fp8_quantize_1x128_cutedsl_kernel_impl", + fp8_quantize_1x128_packed_kernel_impl<kWarpsPerBlock, true>, grid, block, 0, stream, fp8_output, + reinterpret_cast<int32_t*>(swizzled_scale_output), input, m, k, padded_m); } } // namespace kernels::fp8_blockscale_gemm diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.h b/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.h index a4079cd9b054..7f8c4f07d33d 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.h +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.h @@ -47,6 +47,12 @@ namespace kernels::fp8_blockscale_gemm void launch_fp8_quantize_1x128_packed_bf16_e4m3(__nv_fp8_e4m3* fp8_output, int32_t* packed_scale_output, __nv_bfloat16 const* input, int m, int k, int scale_leading_dim_uint32, cudaStream_t stream); +// Quantizes with the same 1x128 scale as above, but replicates each UE8M0 +// scale over four 32-wide groups and writes the native SM100 CuTe/CUTLASS +// 128x4 swizzled scale layout. +void launch_fp8_quantize_1x128_cutedsl_bf16_e4m3(__nv_fp8_e4m3* fp8_output, uint8_t* swizzled_scale_output, + __nv_bfloat16 const* input, int m, int k, int padded_m, cudaStream_t stream); + } // namespace kernels::fp8_blockscale_gemm TRTLLM_NAMESPACE_END diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_kernels.h b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_kernels.h index 24781bec76e7..ab7ed876257d 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_kernels.h +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_kernels.h @@ -27,7 +27,7 @@ #include <cuda_fp4.h> #endif #include "tensorrt_llm/common/config.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <array> #include <cuda_runtime_api.h> #include <map> @@ -1032,8 +1032,8 @@ struct GemmProfilerBackend using Config = cutlass_extensions::CutlassGemmConfig; using GemmToProfile = MoeGemmId; - void init(CutlassMoeFCRunnerInterface& runner, GemmToProfile gemm_to_profile, nvinfer1::DataType dtype, - nvinfer1::DataType wtype, nvinfer1::DataType otype, int num_experts, int k, int64_t hidden_size, + void init(CutlassMoeFCRunnerInterface& runner, GemmToProfile gemm_to_profile, tensorrt_llm::DataType dtype, + tensorrt_llm::DataType wtype, tensorrt_llm::DataType otype, int num_experts, int k, int64_t hidden_size, int64_t unpadded_hidden_size, int64_t inter_size, int64_t group_size, ActivationType activation_type, bool bias, bool use_lora, bool min_latency_mode, bool need_weights, MOEParallelismConfig parallelism_config, bool const enable_alltoall, bool use_mxfp8_weight_scaling = false) @@ -1061,20 +1061,21 @@ struct GemmProfilerBackend mSM = common::getSMVersion(); mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::NONE; - if (dtype == nvinfer1::DataType::kFP8 - && (wtype == nvinfer1::DataType::kFP4 || wtype == nvinfer1::DataType::kINT64)) + if (dtype == tensorrt_llm::DataType::kFP8 + && (wtype == tensorrt_llm::DataType::kFP4 || wtype == tensorrt_llm::DataType::kINT64)) { mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::MXFPX; } - else if (dtype == nvinfer1::DataType::kFP8 && wtype == nvinfer1::DataType::kFP8 && use_mxfp8_weight_scaling) + else if (dtype == tensorrt_llm::DataType::kFP8 && wtype == tensorrt_llm::DataType::kFP8 + && use_mxfp8_weight_scaling) { // MXFP8 W8A8: e4m3 acts × e4m3 weights with UE8M0 1x32 block scales on both sides. // Profiler must produce MXFPX block-scaled inputs (otherwise the per-expert SF // pointer arrays stay uninitialized and the kernel reads garbage SF addresses). mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::MXFPX; } - else if ((dtype == nvinfer1::DataType::kFP4 || dtype == nvinfer1::DataType::kINT64) - && (wtype == nvinfer1::DataType::kFP4 || wtype == nvinfer1::DataType::kINT64)) + else if ((dtype == tensorrt_llm::DataType::kFP4 || dtype == tensorrt_llm::DataType::kINT64) + && (wtype == tensorrt_llm::DataType::kFP4 || wtype == tensorrt_llm::DataType::kINT64)) { mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::NVFP4; } @@ -1106,9 +1107,9 @@ struct GemmProfilerBackend int mSampleIndex = 0; - nvinfer1::DataType mDType{}; - nvinfer1::DataType mWType{}; - nvinfer1::DataType mOType{}; + tensorrt_llm::DataType mDType{}; + tensorrt_llm::DataType mWType{}; + tensorrt_llm::DataType mOType{}; // This will be a unique value for every iteration of warmup and actual bench constexpr static int64_t NUM_ROUTING_SAMPLES = 16; diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_lora_grouped_gemm.h b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_lora_grouped_gemm.h index 55ab4e40a3ae..15af49fc1839 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_lora_grouped_gemm.h +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_lora_grouped_gemm.h @@ -18,7 +18,7 @@ #include "tensorrt_llm/common/config.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cstdint> #include <cuda_runtime.h> @@ -56,7 +56,7 @@ struct MoeLoraGroupedGemmModule; // stream: CUDA stream to launch onto. using MoeLoraGroupedGemmRunFn = void (*)(MoeLoraGroupedGemmModule const& mod, int64_t num_permuted_tokens, int64_t in_hidden_size, int64_t max_lora_rank, int64_t dtype_bytes, int64_t splitk_slices, void const* input_base, - void* output_base, nvinfer1::DataType data_type, cudaStream_t stream); + void* output_base, tensorrt_llm::DataType data_type, cudaStream_t stream); // Per-module device-resident scratch for the MoE LoRA capture-safe path. // Pointers refer to device memory unless noted. diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_util_kernels.h b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_util_kernels.h index e902e2c9d6d3..f3e8940b0c28 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_util_kernels.h +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_util_kernels.h @@ -25,7 +25,6 @@ #ifdef ENABLE_FP4 #include <cuda_fp4.h> #endif -#include <NvInferRuntime.h> #include <array> #include <cuda_runtime_api.h> #include <map> diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_launcher.inl b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_launcher.inl index 0044528b4dff..9c5ebbdaa19d 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_launcher.inl +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_launcher.inl @@ -15,6 +15,8 @@ */ #pragma once +#include "tensorrt_llm/common/cudaUtils.h" + #include "cutlass/array.h" #include "cutlass/numeric_conversion.h" @@ -677,6 +679,19 @@ using namespace cutlass::epilogue; "Workspace is size %zu but only %zu were allocated", calculated_ws_size, \ tma_ws_input.gemm_workspace_size); \ \ + /* Check if kernel SMEM fits on the active device before launch. */ \ + { \ + using GemmKernel_ = typename GemmGrouped::GemmKernel; \ + int smem_size = static_cast<int>(sizeof(typename GemmKernel_::SharedStorage)); \ + int device_ = 0; \ + tensorrt_llm::common::check_cuda_error(cudaGetDevice(&device_)); \ + int maxSmem_ = 0; \ + tensorrt_llm::common::check_cuda_error( \ + cudaDeviceGetAttribute(&maxSmem_, cudaDevAttrMaxSharedMemoryPerBlockOptin, device_)); \ + TLLM_CHECK_WITH_INFO(smem_size <= maxSmem_, \ + "MoE grouped GEMM requires %d bytes shared memory but device supports %d", smem_size, maxSmem_); \ + } \ + \ auto can_implement = gemm.can_implement(args); \ TLLM_CHECK_WITH_INFO(can_implement == cutlass::Status::kSuccess, \ "Grouped GEMM kernel will fail for params. Error: " \ diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_mixed_input_launcher.inl b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_mixed_input_launcher.inl index f37920dcf73c..cdf5ea8dc3c9 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_mixed_input_launcher.inl +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_mixed_input_launcher.inl @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,8 @@ #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif // __GNUC__ +#include "tensorrt_llm/common/cudaUtils.h" + #include "cutlass/epilogue/collective/default_epilogue.hpp" #include "cutlass/epilogue/thread/linear_combination.h" #include "cutlass/gemm/collective/collective_builder.hpp" @@ -273,6 +275,17 @@ void sm90_generic_mixed_moe_gemm_kernelLauncher(GroupedGemmInput<T, WeightType, // This is not initialized during workspace size calculation so check after TLLM_CHECK_WITH_INFO(hopper_inputs.swap_ab, "swap_ab must be true for mixed dtype WS grouped GEMM"); + { + int smem_size = static_cast<int>(sizeof(typename GemmKernel::SharedStorage)); + int device = 0; + tensorrt_llm::common::check_cuda_error(cudaGetDevice(&device)); + int maxSmem = 0; + tensorrt_llm::common::check_cuda_error( + cudaDeviceGetAttribute(&maxSmem, cudaDevAttrMaxSharedMemoryPerBlockOptin, device)); + TLLM_CHECK_WITH_INFO(smem_size <= maxSmem, + "Mixed dtype WS grouped GEMM requires %d bytes shared memory but device supports %d", smem_size, maxSmem); + } + auto can_implement = gemm.can_implement(arguments); if (can_implement != cutlass::Status::kSuccess) { diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_kernels.cu b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_kernels.cu index b7a32be2e285..8bed9c16b58e 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_kernels.cu +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_kernels.cu @@ -58,6 +58,7 @@ #include "tensorrt_llm/kernels/preQuantScaleKernel.h" #include "tensorrt_llm/kernels/quantization.cuh" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/cutlass_kernels/include/moe_lora_pointer_expand.h" #include "tensorrt_llm/kernels/cutlass_kernels/include/moe_util_kernels.h" // NOTE: the grouped-GEMM dispatch (cudaGraph(SplitK)GroupedGemm, @@ -3686,7 +3687,7 @@ void CutlassMoeFCRunner<T, WeightType, OutputType, InputType, BackBoneType, Enab inline void runMoeLoraGroupedGemmModule(::tensorrt_llm::kernels::cutlass_kernels::MoeLoraGroupedGemmModule const& mod, int64_t num_permuted_tokens, int64_t in_hidden_size, int64_t max_lora_rank, int64_t dtype_bytes, int64_t splitk_slices, void const* input_base, void* output_base, - ::tensorrt_llm::kernels::cutlass_kernels::MoeLoraGroupedGemmRunFn run, nvinfer1::DataType data_type, + ::tensorrt_llm::kernels::cutlass_kernels::MoeLoraGroupedGemmRunFn run, tensorrt_llm::DataType data_type, cudaStream_t stream) { TLLM_CHECK_WITH_INFO(mod.permuted_ranks_dev != nullptr, @@ -3698,25 +3699,25 @@ inline void runMoeLoraGroupedGemmModule(::tensorrt_llm::kernels::cutlass_kernels data_type, stream); } -// Map the activation/back-bone type to the nvinfer1 enum the +// Map the activation/back-bone type to the DataType enum the // cuda_graph_grouped_gemm wrappers expect. Only fp16/bf16/fp32 are handled; // anything else is a compile-time error rather than a silent fall-through. template <class ScaleBiasType> -constexpr nvinfer1::DataType moeLoraNvInferType() +constexpr tensorrt_llm::DataType moeLoraDataType() { if constexpr (std::is_same_v<ScaleBiasType, half>) { - return nvinfer1::DataType::kHALF; + return tensorrt_llm::DataType::kHALF; } #if defined(ENABLE_BF16) else if constexpr (std::is_same_v<ScaleBiasType, __nv_bfloat16>) { - return nvinfer1::DataType::kBF16; + return tensorrt_llm::DataType::kBF16; } #endif else if constexpr (std::is_same_v<ScaleBiasType, float>) { - return nvinfer1::DataType::kFLOAT; + return tensorrt_llm::DataType::kFLOAT; } else { @@ -3925,7 +3926,7 @@ auto CutlassMoeFCRunner<T, WeightType, OutputType, InputType, BackBoneType, Enab if (lora_params.grouped_gemm.enabled) { auto const& grouped_gemm = lora_params.grouped_gemm; - nvinfer1::DataType const data_type = moeLoraNvInferType<ScaleBiasType>(); + tensorrt_llm::DataType const data_type = moeLoraDataType<ScaleBiasType>(); // The grouped-GEMM GEMM skips rank-0 rows, but the bias/reorder paths // read lora_fc1_result_ for every valid row. Zero the buffer first so @@ -4020,7 +4021,7 @@ void CutlassMoeFCRunner<T, WeightType, OutputType, InputType, BackBoneType, Enab if (lora_params.grouped_gemm.enabled) { auto const& grouped_gemm = lora_params.grouped_gemm; - nvinfer1::DataType const data_type = moeLoraNvInferType<ScaleBiasType>(); + tensorrt_llm::DataType const data_type = moeLoraDataType<ScaleBiasType>(); // As in loraFC1, zero the output so rank-0 rows the GEMM skips do not // feed stale data into the downstream add. @@ -4750,18 +4751,18 @@ std::map<std::string, std::pair<size_t, size_t>> GemmProfilerBackend::getProfile size_t k = mK; size_t num_expanded_tokens = mMinLatencyMode ? maxM * mNumExpertsPerNode : maxM * k; - TLLM_CHECK(mDType != nvinfer1::DataType::kINT4); + TLLM_CHECK(mDType != tensorrt_llm::DataType::kINT4); // nvllm still uses int64 because torch doesn't have fp4 yet. - bool is_4bit_act = mDType == nvinfer1::DataType::kFP4 || mDType == nvinfer1::DataType::kINT64; - bool is_4bit_weight = mWType == nvinfer1::DataType::kINT4 || mWType == nvinfer1::DataType::kFP4 - || mWType == nvinfer1::DataType::kINT64; + bool is_4bit_act = mDType == tensorrt_llm::DataType::kFP4 || mDType == tensorrt_llm::DataType::kINT64; + bool is_4bit_weight = mWType == tensorrt_llm::DataType::kINT4 || mWType == tensorrt_llm::DataType::kFP4 + || mWType == tensorrt_llm::DataType::kINT64; TLLM_CHECK_WITH_INFO(!is_4bit_act || is_4bit_weight, "Cannot have 4-bit activation with non-4-bit weight"); float dtype_bytes = is_4bit_act ? 0.5f - : static_cast<float>(mWType == nvinfer1::DataType::kINT4 ? getDTypeSize(mOType) : getDTypeSize(mDType)); + : static_cast<float>(mWType == tensorrt_llm::DataType::kINT4 ? getDTypeSize(mOType) : getDTypeSize(mDType)); float weight_bytes = is_4bit_weight ? 0.5f : static_cast<float>(getDTypeSize(mWType)); size_t output_bytes = getDTypeSize(mOType); - size_t gemm_output_bytes = (mOType == nvinfer1::DataType::kFP8) + size_t gemm_output_bytes = (mOType == tensorrt_llm::DataType::kFP8) ? sizeof(TmaWarpSpecializedGroupedGemmInput::OutputTypeAdaptor_t<__nv_fp8_e4m3>) : output_bytes; @@ -4803,18 +4804,18 @@ std::map<std::string, std::pair<size_t, size_t>> GemmProfilerBackend::getProfile // TODO Make quant 2 & 4 bigger for FP8 if we ever change to scaling per expert bool is_int_w_quant - = (mWType == nvinfer1::DataType::kINT8 || mWType == nvinfer1::DataType::kINT4) && mGroupSize <= 0; + = (mWType == tensorrt_llm::DataType::kINT8 || mWType == tensorrt_llm::DataType::kINT4) && mGroupSize <= 0; bool is_int_groupwise_w_quant - = (mWType == nvinfer1::DataType::kINT8 || mWType == nvinfer1::DataType::kINT4) && mGroupSize > 0; - bool is_fp8_act_quant = mDType == nvinfer1::DataType::kFP8; - bool is_fp8_w_quant = mWType == nvinfer1::DataType::kFP8; + = (mWType == tensorrt_llm::DataType::kINT8 || mWType == tensorrt_llm::DataType::kINT4) && mGroupSize > 0; + bool is_fp8_act_quant = mDType == tensorrt_llm::DataType::kFP8; + bool is_fp8_w_quant = mWType == tensorrt_llm::DataType::kFP8; // nvllm still uses int64 because torch doesn't have fp4 yet. - // bool is_fp4_act_quant = mDType == nvinfer1::DataType::kFP4 || mDType == nvinfer1::DataType::kINT64; - bool is_fp4_w_quant = mWType == nvinfer1::DataType::kFP4 || mWType == nvinfer1::DataType::kINT64; + // bool is_fp4_act_quant = mDType == tensorrt_llm::DataType::kFP4 || mDType == tensorrt_llm::DataType::kINT64; + bool is_fp4_w_quant = mWType == tensorrt_llm::DataType::kFP4 || mWType == tensorrt_llm::DataType::kINT64; bool is_w4afp8_quant = is_int_groupwise_w_quant && is_fp8_act_quant; // bool is_wfp4afp8_quant = is_fp4_w_quant && is_fp8_act_quant; - bool is_wfp4a16_quant = (mDType == nvinfer1::DataType::kHALF || mDType == nvinfer1::DataType::kBF16) - && mWType == nvinfer1::DataType::kUINT8; + bool is_wfp4a16_quant = (mDType == tensorrt_llm::DataType::kHALF || mDType == tensorrt_llm::DataType::kBF16) + && mWType == tensorrt_llm::DataType::kUINT8; // Int sizes size_t quant_1_size = is_int_w_quant ? fc1_out_size * num_experts_per_node * dtype_bytes : 0; @@ -5047,19 +5048,19 @@ void GemmProfilerBackend::prepareQuantParams(int num_tokens, char* workspace_ptr GET_WS_PTR(float const*, w4a8_alpha); #undef GET_WS_PTR - if ((mWType == nvinfer1::DataType::kINT8 || mWType == nvinfer1::DataType::kINT4 - || mWType == nvinfer1::DataType::kUINT8) + if ((mWType == tensorrt_llm::DataType::kINT8 || mWType == tensorrt_llm::DataType::kINT4 + || mWType == tensorrt_llm::DataType::kUINT8) && mGroupSize < 0) { TLLM_CHECK(quant_1 && quant_2); mQuantParams = QuantParams::Int(quant_1, quant_2); } - else if (mWType == nvinfer1::DataType::kINT4 || mWType == nvinfer1::DataType::kUINT8) + else if (mWType == tensorrt_llm::DataType::kINT4 || mWType == tensorrt_llm::DataType::kUINT8) { TLLM_CHECK(quant_1 && quant_2); - if (mDType == nvinfer1::DataType::kFP8 - || (mWType == nvinfer1::DataType::kUINT8 - && (mDType == nvinfer1::DataType::kHALF || mDType == nvinfer1::DataType::kBF16))) + if (mDType == tensorrt_llm::DataType::kFP8 + || (mWType == tensorrt_llm::DataType::kUINT8 + && (mDType == tensorrt_llm::DataType::kHALF || mDType == tensorrt_llm::DataType::kBF16))) { TLLM_CHECK(w4a8_alpha); mQuantParams = QuantParams::GroupWise( @@ -5070,7 +5071,7 @@ void GemmProfilerBackend::prepareQuantParams(int num_tokens, char* workspace_ptr mQuantParams = QuantParams::GroupWise(mGroupSize, quant_1, quant_2, nullptr, nullptr, quant_3, quant_4); } } - else if (mWType == nvinfer1::DataType::kFP8) + else if (mWType == tensorrt_llm::DataType::kFP8) { if (mUseMxfp8WeightScaling) { @@ -5089,8 +5090,8 @@ void GemmProfilerBackend::prepareQuantParams(int num_tokens, char* workspace_ptr static_cast<float const*>(quant_3), static_cast<float const*>(quant_4)); } } - else if (mDType == nvinfer1::DataType::kFP8 - && (mWType == nvinfer1::DataType::kFP4 || mWType == nvinfer1::DataType::kINT64)) + else if (mDType == tensorrt_llm::DataType::kFP8 + && (mWType == tensorrt_llm::DataType::kFP4 || mWType == tensorrt_llm::DataType::kINT64)) { TLLM_CHECK(quant_1 && quant_2 && quant_3 && quant_4 && quant_5 && quant_6); mQuantParams = QuantParams::FP8MXFP4(static_cast<float const*>(quant_1), @@ -5099,8 +5100,8 @@ void GemmProfilerBackend::prepareQuantParams(int num_tokens, char* workspace_ptr static_cast<TmaWarpSpecializedGroupedGemmInput::MXFPXElementSF const*>(quant_5), static_cast<float const*>(quant_6)); } - else if ((mDType == nvinfer1::DataType::kFP4 || mDType == nvinfer1::DataType::kINT64) - && (mWType == nvinfer1::DataType::kFP4 || mWType == nvinfer1::DataType::kINT64)) + else if ((mDType == tensorrt_llm::DataType::kFP4 || mDType == tensorrt_llm::DataType::kINT64) + && (mWType == tensorrt_llm::DataType::kFP4 || mWType == tensorrt_llm::DataType::kINT64)) { // nvllm still uses int64 because torch doesn't have fp4 yet. TLLM_CHECK(quant_1 && quant_2 && quant_3 && quant_4 && quant_5 && quant_6); @@ -5120,9 +5121,9 @@ void GemmProfilerBackend::prepareTmaWsInputs(int num_tokens, char* workspace_ptr return; } - bool use_w4afp8 = (mDType == nvinfer1::DataType::kFP8 && mWType == nvinfer1::DataType::kINT4); - bool use_wfp4a16 = ((mDType == nvinfer1::DataType::kHALF || mDType == nvinfer1::DataType::kBF16) - && mWType == nvinfer1::DataType::kUINT8); + bool use_w4afp8 = (mDType == tensorrt_llm::DataType::kFP8 && mWType == tensorrt_llm::DataType::kINT4); + bool use_wfp4a16 = ((mDType == tensorrt_llm::DataType::kHALF || mDType == tensorrt_llm::DataType::kBF16) + && mWType == tensorrt_llm::DataType::kUINT8); bool const use_finalize_fusion = fusion == TmaWarpSpecializedGroupedGemmInput::EpilogueFusion::FINALIZE; bool const finalize_fusion_not_supported = !mInterface->use_fused_finalize_ || mMinLatencyMode || use_wfp4a16 || mGemmToProfile != GemmToProfile::GEMM_2; diff --git a/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/xqaParams.h b/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/xqaParams.h index e421be0a6bd7..dc7794752e47 100644 --- a/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/xqaParams.h +++ b/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/xqaParams.h @@ -63,6 +63,7 @@ struct XQAParams int64_t* spec_decoding_bl_tree_mask_offset; // for blackwell spec-dec tree mask offset uint32_t* spec_decoding_bl_tree_mask; // for blackwell spec-dec tree mask int32_t* spec_bl_tree_first_sparse_mask_offset_kv; // for blackwell spec-dec tree first sparse mask offset kv + bool force_prepare_spec_dec_tree_mask = false; int32_t const* mrope_position_deltas = nullptr; // Helix parallelism params. int32_t const* helix_position_offsets = nullptr; diff --git a/cpp/tensorrt_llm/kernels/fusedAdaptiveLayerNormKernel.cu b/cpp/tensorrt_llm/kernels/fusedAdaptiveLayerNormKernel.cu new file mode 100644 index 000000000000..3564f94c9f74 --- /dev/null +++ b/cpp/tensorrt_llm/kernels/fusedAdaptiveLayerNormKernel.cu @@ -0,0 +1,466 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Fused DiT LayerNorm + optional AdaLN/affine + optional NVFP4 quantization kernel. +// Three mode combos × two output dtypes = 6 compile-time instantiations; see +// launchFusedAdaptiveLayerNormKernel and fusedAdaptiveLayerNormKernel.h. + +#include "fusedAdaptiveLayerNormKernel.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/kernels/quantization.cuh" +#include <cuda_bf16.h> +#include <cuda_fp8.h> +#include <cuda_pipeline.h> +#include <optional> + +TRTLLM_NAMESPACE_BEGIN + +namespace kernels +{ + +namespace +{ + +// TMA / mbarrier helpers (shared with fusedDiTGateResidNormShiftScaleKernel). + +__device__ __forceinline__ uint32_t cvta_to_smem(void const* ptr) +{ + return static_cast<uint32_t>(__cvta_generic_to_shared(const_cast<void*>(ptr))); +} + +__device__ __forceinline__ void mbar_init(uint64_t* bar, uint32_t count) +{ +#if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900) + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" : : "r"(cvta_to_smem(bar)), "r"(count)); +#endif +} + +__device__ __forceinline__ void mbar_arrive(uint64_t* bar) +{ +#if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900) + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];" : : "r"(cvta_to_smem(bar))); +#endif +} + +__device__ __forceinline__ void mbar_arrive_expect_tx(uint64_t* bar, uint32_t tx_bytes) +{ +#if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900) + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" : : "r"(cvta_to_smem(bar)), "r"(tx_bytes)); +#endif +} + +__device__ __forceinline__ void mbar_wait(uint64_t* bar, uint32_t phase) +{ +#if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900) + asm volatile( + "{ .reg .pred P; \n" + " WAIT: mbarrier.try_wait.parity.shared::cta.b64 P, [%0], %1; \n" + " @P bra DONE; \n" + " bra WAIT; \n" + " DONE: }" + : + : "r"(cvta_to_smem(bar)), "r"(phase)); +#endif +} + +__device__ __forceinline__ void cp_async_bulk(void* smem_dst, void const* global_src, uint32_t bytes, uint64_t* bar) +{ +#if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900) + asm volatile( + "cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" + : + : "r"(cvta_to_smem(smem_dst)), "l"(reinterpret_cast<uint64_t>(global_src)), "r"(bytes), "r"(cvta_to_smem(bar)) + : "memory"); +#endif +} + +} // anonymous namespace + +template <int D, int BLOCK_SIZE, bool HAS_LN_AFFINE, bool HAS_MODULATION, bool HAS_QUANT> +__global__ void +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +__launch_bounds__(BLOCK_SIZE, 4) +#elif defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) +__launch_bounds__(BLOCK_SIZE, 8) +#endif + fusedAdaptiveLayerNormKernel(AdaptiveLayerNormParams p) +{ + static_assert(D % 8 == 0, "D must be a multiple of 8"); + static_assert(D % 16 == 0, "D must be a multiple of 16 (NVFP4 SF group size)"); + static_assert(BLOCK_SIZE % 32 == 0, "BLOCK_SIZE must be a multiple of warp size"); + static_assert(!HAS_LN_AFFINE || !HAS_MODULATION, "HAS_LN_AFFINE and HAS_MODULATION are mutually exclusive"); + + constexpr int ELEMS_PER_THREAD = D / BLOCK_SIZE; // 40 for D=5120, BLOCK_SIZE=128 + constexpr int CHUNKS_PER_THREAD = ELEMS_PER_THREAD / 8; // 5 + constexpr int NUM_WARPS = BLOCK_SIZE / 32; // 4 + constexpr int SF_VEC_SIZE = 16; + constexpr int SF_PER_ROW = D / SF_VEC_SIZE; // 320 + constexpr int NUM_THREADS_PER_SF = SF_VEC_SIZE / 8; // 2 + +#if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900) + asm volatile("griddepcontrol.wait;"); +#endif + + // TMA is beneficial for large rows on sm>=900 + constexpr bool USE_TMA = (D >= 4096); + + int const tid = threadIdx.x; + int const warpId = tid / 32; + int const laneId = tid % 32; + int const row = blockIdx.x; + + int64_t const rowBase = static_cast<int64_t>(row) * D; + + // Shared memory for cross-warp reductions and mean/rstd broadcast. + __shared__ float warpSums[NUM_WARPS]; + __shared__ float warpSqSums[NUM_WARPS]; + __shared__ float meanRstd[2]; + + // Static mbarrier for TMA (compiler elides on non-TMA paths). + __shared__ alignas(8) uint64_t mbar; + + // Dynamic SMEM: holds one row of bf16 x when USE_TMA; zero-sized otherwise. + // The launcher passes dyn_smem = USE_TMA ? D * sizeof(bf16) : 0. + extern __shared__ __align__(16) unsigned char smem_raw[]; + __nv_bfloat16* smem_x = reinterpret_cast<__nv_bfloat16*>(smem_raw); + + // Phase 0a: bulk-load x[row] into SMEM while Phase 0b loads modulators. +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if constexpr (USE_TMA) + { + constexpr uint32_t kXBytes = static_cast<uint32_t>(D) * sizeof(__nv_bfloat16); + static_assert(kXBytes % 16 == 0, "cp.async.bulk requires nbBytes multiple of 16"); + if (tid == 0) + { + mbar_init(&mbar, 1); + mbar_arrive_expect_tx(&mbar, kXBytes); + cp_async_bulk(smem_x, p.x + rowBase, kXBytes, &mbar); + } + __syncthreads(); // ensure mbar is initialized before all threads see it + } + else +#endif + { + // Non-TMA path: cp.async into SMEM (fallback, not production path for D=5120). +#pragma unroll + for (int chunk = 0; chunk < CHUNKS_PER_THREAD; ++chunk) + { + int const vecIdx = chunk * BLOCK_SIZE + tid; + int const elemOff = vecIdx * 8; + __pipeline_memcpy_async(smem_x + elemOff, p.x + rowBase + elemOff, 16); + } + __pipeline_commit(); + } + + // Phase 0b: load LN affine or AdaLN modulator rows into register caches. + float wVals[ELEMS_PER_THREAD]; // weight (or 1 + scale_msa) + float bVals[ELEMS_PER_THREAD]; // bias (or shift_msa) + + if constexpr (HAS_LN_AFFINE) + { + // ln_weight[D], ln_bias[D] -- direct GMEM reads while waiting for SMEM fill. +#pragma unroll + for (int chunk = 0; chunk < CHUNKS_PER_THREAD; ++chunk) + { + int const vecIdx = chunk * BLOCK_SIZE + tid; + int const elemOff = vecIdx * 8; + uint4 const wVec = *reinterpret_cast<uint4 const*>(p.ln_weight + elemOff); + uint4 const bVec = *reinterpret_cast<uint4 const*>(p.ln_bias + elemOff); + __nv_bfloat162 const* wVec2 = reinterpret_cast<__nv_bfloat162 const*>(&wVec); + __nv_bfloat162 const* bVec2 = reinterpret_cast<__nv_bfloat162 const*>(&bVec); +#pragma unroll + for (int i = 0; i < 4; ++i) + { + float2 wf = __bfloat1622float2(wVec2[i]); + float2 bf = __bfloat1622float2(bVec2[i]); + wVals[chunk * 8 + i * 2 + 0] = wf.x; + wVals[chunk * 8 + i * 2 + 1] = wf.y; + bVals[chunk * 8 + i * 2 + 0] = bf.x; + bVals[chunk * 8 + i * 2 + 1] = bf.y; + } + } + } + else if constexpr (HAS_MODULATION) + { + // scale_msa[B, D], shift_msa[B, D] -- batch_idx = row / seq_len_per_batch. + int const batchIdx = row / p.seq_len_per_batch; + int64_t const modBase = static_cast<int64_t>(batchIdx) * D; +#pragma unroll + for (int chunk = 0; chunk < CHUNKS_PER_THREAD; ++chunk) + { + int const vecIdx = chunk * BLOCK_SIZE + tid; + int const elemOff = vecIdx * 8; + uint4 const sVec = *reinterpret_cast<uint4 const*>(p.scale_msa + modBase + elemOff); + uint4 const shVec = *reinterpret_cast<uint4 const*>(p.shift_msa + modBase + elemOff); + __nv_bfloat162 const* sVec2 = reinterpret_cast<__nv_bfloat162 const*>(&sVec); + __nv_bfloat162 const* shVec2 = reinterpret_cast<__nv_bfloat162 const*>(&shVec); +#pragma unroll + for (int i = 0; i < 4; ++i) + { + float2 sf = __bfloat1622float2(sVec2[i]); + float2 shf = __bfloat1622float2(shVec2[i]); + // AdaLN: y = normalized * (1 + scale_msa) + shift_msa. + // Fold +1 into wVals so Phase 2 is one fma. + wVals[chunk * 8 + i * 2 + 0] = 1.0f + sf.x; + wVals[chunk * 8 + i * 2 + 1] = 1.0f + sf.y; + bVals[chunk * 8 + i * 2 + 0] = shf.x; + bVals[chunk * 8 + i * 2 + 1] = shf.y; + } + } + } + + // Phase 0c: wait for x load, then read x chunks into xVals registers. + float xVals[ELEMS_PER_THREAD]; + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if constexpr (USE_TMA) + { + mbar_wait(&mbar, /*phase=*/0); + __syncthreads(); + // Read x chunks from SMEM. +#pragma unroll + for (int chunk = 0; chunk < CHUNKS_PER_THREAD; ++chunk) + { + int const vecIdx = chunk * BLOCK_SIZE + tid; + int const elemOff = vecIdx * 8; + uint4 const xVec = *reinterpret_cast<uint4 const*>(smem_x + elemOff); + __nv_bfloat162 const* xVec2 = reinterpret_cast<__nv_bfloat162 const*>(&xVec); +#pragma unroll + for (int i = 0; i < 4; ++i) + { + float2 xf = __bfloat1622float2(xVec2[i]); + xVals[chunk * 8 + i * 2 + 0] = xf.x; + xVals[chunk * 8 + i * 2 + 1] = xf.y; + } + } + } + else +#endif + { + __pipeline_wait_prior(0); + __syncthreads(); + // Read x chunks from SMEM (non-TMA path). +#pragma unroll + for (int chunk = 0; chunk < CHUNKS_PER_THREAD; ++chunk) + { + int const vecIdx = chunk * BLOCK_SIZE + tid; + int const elemOff = vecIdx * 8; + uint4 const xVec = *reinterpret_cast<uint4 const*>(smem_x + elemOff); + __nv_bfloat162 const* xVec2 = reinterpret_cast<__nv_bfloat162 const*>(&xVec); +#pragma unroll + for (int i = 0; i < 4; ++i) + { + float2 xf = __bfloat1622float2(xVec2[i]); + xVals[chunk * 8 + i * 2 + 0] = xf.x; + xVals[chunk * 8 + i * 2 + 1] = xf.y; + } + } + } + + // Phase 1: warp-reduce sum/sum-of-squares → mean and rstd (LayerNorm, not RMSNorm). + float localSum = 0.0f; + float localSqSum = 0.0f; +#pragma unroll + for (int i = 0; i < ELEMS_PER_THREAD; ++i) + { + localSum += xVals[i]; + localSqSum += xVals[i] * xVals[i]; + } + + // Warp-level reduction. +#pragma unroll + for (int offset = 16; offset > 0; offset /= 2) + { + localSum += __shfl_xor_sync(0xffffffff, localSum, offset); + localSqSum += __shfl_xor_sync(0xffffffff, localSqSum, offset); + } + + if (laneId == 0) + { + warpSums[warpId] = localSum; + warpSqSums[warpId] = localSqSum; + } + __syncthreads(); + + // Cross-warp reduction in warp 0. + if (warpId == 0) + { + float s = (laneId < NUM_WARPS) ? warpSums[laneId] : 0.0f; + float s2 = (laneId < NUM_WARPS) ? warpSqSums[laneId] : 0.0f; +#pragma unroll + for (int offset = 16; offset > 0; offset /= 2) + { + s += __shfl_xor_sync(0xffffffff, s, offset); + s2 += __shfl_xor_sync(0xffffffff, s2, offset); + } + if (laneId == 0) + { + float const invD = 1.0f / static_cast<float>(D); + float const mean = s * invD; + float const var = s2 * invD - mean * mean; + meanRstd[0] = mean; + meanRstd[1] = rsqrtf(var + p.eps); + } + } + __syncthreads(); + + float const mean = meanRstd[0]; + float const rstd = meanRstd[1]; + + // Pre-read sf_scale scalar (HAS_QUANT only). + float sfScaleVal = 1.0f; + if constexpr (HAS_QUANT) + { + sfScaleVal = (p.sf_scale != nullptr) ? p.sf_scale[0] : 1.0f; + } + + // Phase 2: normalize, apply affine/modulation, write output. +#pragma unroll + for (int chunk = 0; chunk < CHUNKS_PER_THREAD; ++chunk) + { + int const vecIdx = chunk * BLOCK_SIZE + tid; + int const elemOff = vecIdx * 8; + + // Normalize and apply affine/modulation, producing 8 float yVals. + float yVals[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + float const xn = (xVals[chunk * 8 + i] - mean) * rstd; + if constexpr (HAS_LN_AFFINE || HAS_MODULATION) + { + yVals[i] = xn * wVals[chunk * 8 + i] + bVals[chunk * 8 + i]; + } + else + { + yVals[i] = xn; + } + } + + if constexpr (HAS_QUANT) + { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + // FP4 quantization path (Blackwell+). + // 1. Compute local max-abs across 8 elements held by this thread. + float localMax = fabsf(yVals[0]); +#pragma unroll + for (int i = 1; i < 8; ++i) + { + localMax = fmaxf(localMax, fabsf(yVals[i])); + } + + // 2. Pair-lane max across the 16-element SF block (threads pair via XOR-1). + float const blockMax = fmaxf(__shfl_xor_sync(0xffffffff, localMax, 1), localMax); + + // 3. Compute FP8-e4m3 scale factor (same formula as LTX-2 kernel). + constexpr float kE2M1MaxRcp = 1.0f / 6.0f; + float const sfValue = sfScaleVal * (blockMax * kE2M1MaxRcp); + __nv_fp8_e4m3 const sfFp8 = __nv_fp8_e4m3(sfValue); + uint8_t const sfByte = sfFp8.__x; + float const sfValueQuant = static_cast<float>(sfFp8); + float const outScale = (blockMax != 0.0f) ? (sfScaleVal / sfValueQuant) : 0.0f; + + // 4. Scale yVals and convert to packed FP4 (e2m1). +#pragma unroll + for (int i = 0; i < 8; ++i) + yVals[i] *= outScale; + uint32_t const fp4Packed = fp32_vec_to_e2m1(yVals); + + // 5. Write SF via swizzled layout helper. + uint8_t* sfOutPtr = cvt_quant_get_sf_out_offset<uint32_t, NUM_THREADS_PER_SF>(std::nullopt, row, vecIdx, + std::optional<int>(p.M), SF_PER_ROW, reinterpret_cast<uint32_t*>(p.out_sf), + QuantizationSFLayout::SWIZZLED); + if (sfOutPtr != nullptr) + { + *sfOutPtr = sfByte; + } + + // 6. Write packed FP4. + int64_t const fp4Off = static_cast<int64_t>(row) * (D / 8) + vecIdx; + p.out_fp4[fp4Off] = fp4Packed; +#endif // __CUDA_ARCH__ >= 1000 + } + else + { + // bf16 output path. + uint4 outVec; + __nv_bfloat162* outVec2 = reinterpret_cast<__nv_bfloat162*>(&outVec); +#pragma unroll + for (int i = 0; i < 4; ++i) + { + outVec2[i] = __float22bfloat162_rn(make_float2(yVals[i * 2 + 0], yVals[i * 2 + 1])); + } + *reinterpret_cast<uint4*>(p.out_bf16 + rowBase + elemOff) = outVec; + } + } + +#if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900) + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +void launchFusedAdaptiveLayerNormKernel( + AdaptiveLayerNormParams const& params, bool has_ln_affine, bool has_modulation, bool has_quant, cudaStream_t stream) +{ + TLLM_CHECK_WITH_INFO(params.D == 5120, "fusedAdaptiveLayerNormKernel only supports D=5120 (got %d)", params.D); + TLLM_CHECK_WITH_INFO(!(has_ln_affine && has_modulation), "has_ln_affine and has_modulation are mutually exclusive"); + TLLM_CHECK_WITH_INFO(!has_quant || tensorrt_llm::common::getSMVersion() >= 100, + "fusedAdaptiveLayerNormKernel quant mode requires SM >= 100 (Blackwell+), got SM %d", + tensorrt_llm::common::getSMVersion()); + + constexpr int D = 5120; + constexpr int BLOCK_SIZE = 128; + constexpr size_t dynSmem = D * sizeof(__nv_bfloat16); + + cudaLaunchConfig_t cfg = {}; + cfg.gridDim = dim3(params.M); + cfg.blockDim = dim3(BLOCK_SIZE); + cfg.dynamicSmemBytes = dynSmem; + cfg.stream = stream; + cudaLaunchAttribute attrs[1] = {}; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = tensorrt_llm::common::getEnvEnablePDL(); + cfg.attrs = attrs; + cfg.numAttrs = 1; + +#define LAUNCH(AFF, MOD, QUANT) \ + cudaLaunchKernelEx(&cfg, fusedAdaptiveLayerNormKernel<D, BLOCK_SIZE, AFF, MOD, QUANT>, params) + + if (!has_ln_affine && !has_modulation && !has_quant) + LAUNCH(false, false, false); + else if (!has_ln_affine && !has_modulation && has_quant) + LAUNCH(false, false, true); + else if (has_ln_affine && !has_modulation && !has_quant) + LAUNCH(true, false, false); + else if (has_ln_affine && !has_modulation && has_quant) + LAUNCH(true, false, true); + else if (!has_ln_affine && has_modulation && !has_quant) + LAUNCH(false, true, false); + else if (!has_ln_affine && has_modulation && has_quant) + LAUNCH(false, true, true); + else + TLLM_CHECK_WITH_INFO(false, "Unsupported flag combination in launchFusedAdaptiveLayerNormKernel"); + +#undef LAUNCH + + sync_check_cuda_error(stream); +} + +} // namespace kernels + +TRTLLM_NAMESPACE_END diff --git a/cpp/tensorrt_llm/kernels/fusedAdaptiveLayerNormKernel.h b/cpp/tensorrt_llm/kernels/fusedAdaptiveLayerNormKernel.h new file mode 100644 index 000000000000..35db16b54f30 --- /dev/null +++ b/cpp/tensorrt_llm/kernels/fusedAdaptiveLayerNormKernel.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRTLLM_FUSEDADAPTIVELAYERNORMKERNEL_H +#define TRTLLM_FUSEDADAPTIVELAYERNORMKERNEL_H + +#include "tensorrt_llm/common/config.h" +#include <cstdint> +#include <cuda_bf16.h> +#include <cuda_runtime.h> + +TRTLLM_NAMESPACE_BEGIN + +namespace kernels +{ + +// Parameters for the fused DiT LayerNorm + optional AdaLN/affine + optional NVFP4 quant kernel. +// +// Supported modes (compile-time template flags): +// HAS_LN_AFFINE : LayerNorm with learned weight/bias (norm2 in WanBlock) +// HAS_MODULATION : LayerNorm + AdaLN: y = (1 + scale_msa) * x_hat + shift_msa (norm1/norm3) +// HAS_QUANT : write NVFP4 packed output + swizzled scale factors +// +// HAS_LN_AFFINE and HAS_MODULATION are mutually exclusive. +// +// Input shape: x [M, D] +// Modulator shapes: scale_msa/shift_msa [B, D] where B = M / seq_len_per_batch +// ln_weight/ln_bias [D] +// Output shapes: +// HAS_QUANT=false: out_bf16 [M, D] +// HAS_QUANT=true: out_fp4 [M, D/8] (8 FP4 nibbles packed per uint32) +// out_sf swizzled NVFP4 scale factors (uint8 array) +struct AdaptiveLayerNormParams +{ + // Input + __nv_bfloat16 const* x = nullptr; // [M, D] bf16 + + // Affine LN params (HAS_LN_AFFINE only) + __nv_bfloat16 const* ln_weight = nullptr; // [D] bf16 + __nv_bfloat16 const* ln_bias = nullptr; // [D] bf16 + + // AdaLN modulation params (HAS_MODULATION only) + __nv_bfloat16 const* scale_msa = nullptr; // [B, D] bf16 + __nv_bfloat16 const* shift_msa = nullptr; // [B, D] bf16 + + // bf16 output path (HAS_QUANT=false) + __nv_bfloat16* out_bf16 = nullptr; // [M, D] bf16 + + // FP4 output path (HAS_QUANT=true) + uint32_t* out_fp4 = nullptr; // [M, D/8] packed FP4 (8 nibbles per uint32) + uint32_t* out_sf = nullptr; // swizzled NVFP4 scale factors (uint8 recast as uint32*) + float const* sf_scale = nullptr; // scalar global activation scale (calibrated input_scale) + + // Shape + int M = 0; // total rows + int D = 0; // hidden dim (must equal 5120 for now) + int seq_len_per_batch = 0; // HAS_MODULATION: batch_idx = row / seq_len_per_batch + float eps = 1e-6f; +}; + +// Launch the fused kernel. The bool flags select the matching compile-time specialization. +// has_ln_affine and has_modulation must not both be true. Supported hidden_dim: 5120 (Wan 14B). +void launchFusedAdaptiveLayerNormKernel(AdaptiveLayerNormParams const& params, bool has_ln_affine, bool has_modulation, + bool has_quant, cudaStream_t stream); + +} // namespace kernels + +TRTLLM_NAMESPACE_END + +#endif // TRTLLM_FUSEDADAPTIVELAYERNORMKERNEL_H diff --git a/cpp/tensorrt_llm/kernels/fusedQKNormRopeKernel.cu b/cpp/tensorrt_llm/kernels/fusedQKNormRopeKernel.cu index e06b0f200e4b..6e26dfce65bf 100644 --- a/cpp/tensorrt_llm/kernels/fusedQKNormRopeKernel.cu +++ b/cpp/tensorrt_llm/kernels/fusedQKNormRopeKernel.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -174,6 +174,17 @@ __global__ void fusedQKNormRopeKernel( // pos_id is selected per rotary half-dim (interleaved mRoPE); for plain RoPE // selectMRopePosId always returns position_ids[tokenIdx]. + // Hoist log2(base) and the loop-invariant constants out of the per-thread + // per-elem loop. powf(base, -2*hd/rd) == exp2f(-2*hd/rd * log2(base)); base + // and rotary_dim are kernel-uniform, so one MUFU.LG2 per warp instead of + // one per (thread, iter). Uses the fast __log2f intrinsic (a few ULPs of + // error, absorbed by bf16 downcast at store time). + float const neg2_log2base_over_rd = -2.0f * __log2f(base) / static_cast<float>(rotary_dim); + // rotary_dim is even by contract; when it's also a power of 2 (always in + // practice — 64/128/256) '% rotary_dim' becomes '& (rotary_dim - 1)'. + // The bool is warp-uniform → predicated select, no branch divergence. + int const rd_mask = rotary_dim - 1; + bool const rd_is_pow2 = ((rotary_dim & rd_mask) == 0); // TODO: cos sin calculation could be halved. if constexpr (interleave) { @@ -191,7 +202,7 @@ __global__ void fusedQKNormRopeKernel( int dim_idx = laneId * numElemsPerThread + i; int half_dim = dim_idx / 2; - float freq = powf(base, -2.0f * half_dim / static_cast<float>(rotary_dim)); + float freq = exp2f(static_cast<float>(half_dim) * neg2_log2base_over_rd); if (factor != 1.0f) { @@ -232,9 +243,9 @@ __global__ void fusedQKNormRopeKernel( } int dim_idx = laneId * numElemsPerThread + i; - dim_idx = (dim_idx * 2) % rotary_dim; + dim_idx = rd_is_pow2 ? ((dim_idx * 2) & rd_mask) : ((dim_idx * 2) % rotary_dim); int half_dim = dim_idx / 2; - float freq = powf(base, -2.0f * half_dim / static_cast<float>(rotary_dim)); + float freq = exp2f(static_cast<float>(half_dim) * neg2_log2base_over_rd); if (factor != 1.0f) { diff --git a/cpp/tensorrt_llm/kernels/gptKernels.h b/cpp/tensorrt_llm/kernels/gptKernels.h index e13e9bca4d6a..d855aade79c2 100644 --- a/cpp/tensorrt_llm/kernels/gptKernels.h +++ b/cpp/tensorrt_llm/kernels/gptKernels.h @@ -16,6 +16,7 @@ #pragma once #include "tensorrt_llm/common/config.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/contextFusedMultiHeadAttention/fused_multihead_attention_common.h" #include "tensorrt_llm/runtime/iTensor.h" #include <cstdint> @@ -227,11 +228,11 @@ struct BuildDecoderInfoParams std::string toString() const { std::stringstream ss; - auto printTensor = [&ss](char const* name, void* ptr, nvinfer1::Dims shape) + auto printTensor = [&ss](char const* name, void* ptr, tensorrt_llm::Dims shape) { ss << name << ": "; if (ptr) - ss << *(runtime::ITensor::wrap((void*) ptr, nvinfer1::DataType::kINT32, shape)); + ss << *(runtime::ITensor::wrap((void*) ptr, tensorrt_llm::DataType::kINT32, shape)); else ss << "nullptr"; ss << std::endl; diff --git a/cpp/tensorrt_llm/kernels/groupGemm.cu b/cpp/tensorrt_llm/kernels/groupGemm.cu index 5b8c0d929150..b41021ffe6f7 100644 --- a/cpp/tensorrt_llm/kernels/groupGemm.cu +++ b/cpp/tensorrt_llm/kernels/groupGemm.cu @@ -28,6 +28,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" TRTLLM_NAMESPACE_BEGIN @@ -63,7 +64,7 @@ template <int M1, int N1, int K1, int M2, int N2, int K2, typename cutlassType, void groupedGemm_(std::vector<cutlass::gemm::GemmCoord> problem_sizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkSpace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkspaceSize, - nvinfer1::DataType dataType, cudaStream_t stream) + tensorrt_llm::DataType dataType, cudaStream_t stream) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); using ElementA = cutlassType; @@ -178,20 +179,20 @@ template <int M1, int N1, int K1, int M2, int N2, int K2, int kAlignmentAB, int void groupedGemmType_(std::vector<cutlass::gemm::GemmCoord> problem_sizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkSpace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkspaceSize, - nvinfer1::DataType dataType, cudaStream_t stream) + tensorrt_llm::DataType dataType, cudaStream_t stream) { - if (dataType == nvinfer1::DataType::kHALF) + if (dataType == tensorrt_llm::DataType::kHALF) { groupedGemm_<M1, N1, K1, M2, N2, K2, cutlass::half_t, kAlignmentAB, kAlignmentC, kStages>(problem_sizes, ptrA, ptrB, ptrC, ptrD, gemmParamsWorkSpace, gemmParamsWorkSpaceSize, gemmWorkSpace, gemmWorkspaceSize, dataType, stream); } - else if (dataType == nvinfer1::DataType::kFLOAT) + else if (dataType == tensorrt_llm::DataType::kFLOAT) { TLLM_CHECK_WITH_INFO(false, "not support float input/output"); } #ifdef ENABLE_BF16 - else if (dataType == nvinfer1::DataType::kBF16) + else if (dataType == tensorrt_llm::DataType::kBF16) { groupedGemm_<M1, N1, K1, M2, N2, K2, cutlass::bfloat16_t, kAlignmentAB, kAlignmentC, kStages>(problem_sizes, ptrA, ptrB, ptrC, ptrD, gemmParamsWorkSpace, gemmParamsWorkSpaceSize, gemmWorkSpace, gemmWorkspaceSize, @@ -203,7 +204,7 @@ void groupedGemmType_(std::vector<cutlass::gemm::GemmCoord> problem_sizes, std:: void groupedGemm(std::vector<cutlass::gemm::GemmCoord> problem_sizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkSpace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkspaceSize, - bool isLoraIn, nvinfer1::DataType dataType, int minKN, cudaStream_t stream) + bool isLoraIn, tensorrt_llm::DataType dataType, int minKN, cudaStream_t stream) { TLLM_LOG_TRACE("%s start, isLoraIn: %d, minKN = %d", __PRETTY_FUNCTION__, static_cast<int>(isLoraIn), minKN); if (isLoraIn) diff --git a/cpp/tensorrt_llm/kernels/groupGemm.h b/cpp/tensorrt_llm/kernels/groupGemm.h index dbc1e498b7b2..c526e08e986c 100644 --- a/cpp/tensorrt_llm/kernels/groupGemm.h +++ b/cpp/tensorrt_llm/kernels/groupGemm.h @@ -17,7 +17,7 @@ #include "cutlass/gemm_coord.h" #include "tensorrt_llm/common/config.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" TRTLLM_NAMESPACE_BEGIN @@ -29,7 +29,7 @@ int64_t getGroupedGemmParamsWorkSpaceSize(int64_t problem_count); void groupedGemm(std::vector<cutlass::gemm::GemmCoord> problem_sizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkspace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkspaceSize, - bool isLoraIn, nvinfer1::DataType dataType, int minKN, cudaStream_t stream); + bool isLoraIn, tensorrt_llm::DataType dataType, int minKN, cudaStream_t stream); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.cu b/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.cu index 409968bb510d..3332c2918a3f 100644 --- a/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.cu +++ b/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.cu @@ -22,6 +22,7 @@ #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h" TRTLLM_NAMESPACE_BEGIN @@ -655,9 +656,9 @@ void GroupRMSNormBaseKernelLauncher(GroupRMSParams<n>& params) switch (params.dtype) { - case nvinfer1::DataType::kHALF: GROUP_RMS_NORM_DISPATCH(half); break; - case nvinfer1::DataType::kBF16: GROUP_RMS_NORM_DISPATCH(__nv_bfloat16); break; - case nvinfer1::DataType::kFLOAT: GROUP_RMS_NORM_DISPATCH(float); break; + case tensorrt_llm::DataType::kHALF: GROUP_RMS_NORM_DISPATCH(half); break; + case tensorrt_llm::DataType::kBF16: GROUP_RMS_NORM_DISPATCH(__nv_bfloat16); break; + case tensorrt_llm::DataType::kFLOAT: GROUP_RMS_NORM_DISPATCH(float); break; default: TLLM_CHECK_WITH_INFO(false, "Unsupported data type for GroupRMSNorm"); } @@ -750,9 +751,9 @@ void GroupRMSNormKernelLargeBatchLauncher(GroupRMSParams<n>& params) switch (params.dtype) { - case nvinfer1::DataType::kHALF: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(half); break; - case nvinfer1::DataType::kBF16: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(__nv_bfloat16); break; - case nvinfer1::DataType::kFLOAT: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(float); break; + case tensorrt_llm::DataType::kHALF: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(half); break; + case tensorrt_llm::DataType::kBF16: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(__nv_bfloat16); break; + case tensorrt_llm::DataType::kFLOAT: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(float); break; default: TLLM_CHECK_WITH_INFO(false, "Unsupported data type for GroupRMSNormV2"); } @@ -813,15 +814,15 @@ void GroupRMSNormKernelLauncherWithHeuristic(GroupRMSParams<n>& params) // Choose the appropriate DType switch (params.dtype) { - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: base_warps = calculateNumWarpsBase<half, n>(params); large_batch_warps = calculateNumWarpsLargeBatch<half, n>(params).num_warps_to_launch; break; - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: base_warps = calculateNumWarpsBase<__nv_bfloat16, n>(params); large_batch_warps = calculateNumWarpsLargeBatch<__nv_bfloat16, n>(params).num_warps_to_launch; break; - case nvinfer1::DataType::kFLOAT: + case tensorrt_llm::DataType::kFLOAT: base_warps = calculateNumWarpsBase<float, n>(params); large_batch_warps = calculateNumWarpsLargeBatch<float, n>(params).num_warps_to_launch; break; diff --git a/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h b/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h index 335adf44ed67..70425f924217 100644 --- a/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h +++ b/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h @@ -15,7 +15,7 @@ */ #pragma once #include "tensorrt_llm/common/assert.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_bf16.h> #include <cuda_fp16.h> #include <map> @@ -44,7 +44,7 @@ struct GroupRMSParams float eps; float weight_bias; bool enable_weights; - nvinfer1::DataType dtype; + tensorrt_llm::DataType dtype; cudaStream_t stream; }; diff --git a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/include/moe_kernels.h b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/include/moe_kernels.h index 132990603db9..0b02686f5e53 100644 --- a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/include/moe_kernels.h +++ b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/include/moe_kernels.h @@ -26,7 +26,7 @@ #ifdef ENABLE_FP4 #include <cuda_fp4.h> #endif -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <array> #include <cuda_runtime_api.h> #include <map> @@ -869,8 +869,8 @@ struct GemmProfilerBackend GEMM_2 }; - void init(CutlassMoeFCRunnerInterface& runner, GemmToProfile gemm_to_profile, nvinfer1::DataType dtype, - nvinfer1::DataType wtype, nvinfer1::DataType otype, int num_experts, int k, int64_t hidden_size, + void init(CutlassMoeFCRunnerInterface& runner, GemmToProfile gemm_to_profile, tensorrt_llm::DataType dtype, + tensorrt_llm::DataType wtype, tensorrt_llm::DataType otype, int num_experts, int k, int64_t hidden_size, int64_t inter_size, int64_t group_size, ActivationType activation_type, bool bias, bool use_lora, bool min_latency_mode, bool need_weights, MOEParallelismConfig parallelism_config) { @@ -895,13 +895,13 @@ struct GemmProfilerBackend mSorter.updateNumExperts(mNumExpertsPerNode); mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::NONE; - if (dtype == nvinfer1::DataType::kFP8 - && (wtype == nvinfer1::DataType::kFP4 || wtype == nvinfer1::DataType::kINT64)) + if (dtype == tensorrt_llm::DataType::kFP8 + && (wtype == tensorrt_llm::DataType::kFP4 || wtype == tensorrt_llm::DataType::kINT64)) { mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::MXFPX; } - else if ((dtype == nvinfer1::DataType::kFP4 || dtype == nvinfer1::DataType::kINT64) - && (wtype == nvinfer1::DataType::kFP4 || wtype == nvinfer1::DataType::kINT64)) + else if ((dtype == tensorrt_llm::DataType::kFP4 || dtype == tensorrt_llm::DataType::kINT64) + && (wtype == tensorrt_llm::DataType::kFP4 || wtype == tensorrt_llm::DataType::kINT64)) { mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::NVFP4; } @@ -932,9 +932,9 @@ struct GemmProfilerBackend int mSampleIndex = 0; - nvinfer1::DataType mDType{}; - nvinfer1::DataType mWType{}; - nvinfer1::DataType mOType{}; + tensorrt_llm::DataType mDType{}; + tensorrt_llm::DataType mWType{}; + tensorrt_llm::DataType mOType{}; // This will be a unique value for every iteration of warmup and actual bench constexpr static int64_t NUM_ROUTING_SAMPLES = 16; diff --git a/cpp/tensorrt_llm/kernels/kvCachePartialCopy.cu b/cpp/tensorrt_llm/kernels/kvCachePartialCopy.cu index 3b91cf3f1776..04105721dfca 100644 --- a/cpp/tensorrt_llm/kernels/kvCachePartialCopy.cu +++ b/cpp/tensorrt_llm/kernels/kvCachePartialCopy.cu @@ -15,6 +15,7 @@ */ #include "tensorrt_llm/common/config.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/kvCachePartialCopy.h" #include <cstdint> #include <cuda_runtime_api.h> @@ -89,42 +90,42 @@ void kvCacheBlockPartialCopy(IBuffer& dst, IBuffer const& src, unsigned int numL TLLM_CHECK_WITH_INFO(dataType == dst.getDataType(), "src and dst dataType does not match"); switch (dataType) { - case nvinfer1::DataType::kINT64: + case tensorrt_llm::DataType::kINT64: hostKVCacheBlockPartialCopy<SizeType64>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; - case nvinfer1::DataType::kINT32: + case tensorrt_llm::DataType::kINT32: hostKVCacheBlockPartialCopy<std::int32_t>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; - case nvinfer1::DataType::kFLOAT: + case tensorrt_llm::DataType::kFLOAT: hostKVCacheBlockPartialCopy<float>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: hostKVCacheBlockPartialCopy<__nv_bfloat16>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; #endif - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: hostKVCacheBlockPartialCopy<half>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; - case nvinfer1::DataType::kBOOL: + case tensorrt_llm::DataType::kBOOL: hostKVCacheBlockPartialCopy<bool>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; - case nvinfer1::DataType::kUINT8: + case tensorrt_llm::DataType::kUINT8: hostKVCacheBlockPartialCopy<std::uint8_t>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; - case nvinfer1::DataType::kINT8: + case tensorrt_llm::DataType::kINT8: hostKVCacheBlockPartialCopy<std::int8_t>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; #ifdef ENABLE_FP8 - case nvinfer1::DataType::kFP8: + case tensorrt_llm::DataType::kFP8: hostKVCacheBlockPartialCopy<__nv_fp8_e4m3>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; diff --git a/cpp/tensorrt_llm/kernels/lora/dora.cpp b/cpp/tensorrt_llm/kernels/lora/dora.cpp index 43dbf4fdccb0..883d02df9291 100644 --- a/cpp/tensorrt_llm/kernels/lora/dora.cpp +++ b/cpp/tensorrt_llm/kernels/lora/dora.cpp @@ -20,13 +20,13 @@ #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/kernels/doraScaling.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <numeric> #include <vector> using tensorrt_llm::kernels::DoraImpl; -DoraImpl::DoraImpl(std::vector<int> const& outHiddenSizes, nvinfer1::DataType type) +DoraImpl::DoraImpl(std::vector<int> const& outHiddenSizes, tensorrt_llm::DataType type) : mType(type) { mCumModuleSizes.resize(outHiddenSizes.size()); @@ -73,14 +73,14 @@ int DoraImpl::run(int64_t numTokens, void const* input, void const* const* loraW auto const* deviceCumModuleSizes = reinterpret_cast<int64_t const*>(workspace); auto const* deviceScalePtrs = reinterpret_cast<void const* const*>((&deviceCumModuleSizes[numModules])); - if (mType == nvinfer1::DataType::kHALF) + if (mType == tensorrt_llm::DataType::kHALF) { tokenPerChannelScale<half>(numel, numModules, numTokens, deviceCumModuleSizes, reinterpret_cast<half const*>(input), reinterpret_cast<half const* const*>(deviceScalePtrs), reinterpret_cast<half*>(outputs[0]), stream); } #ifdef ENABLE_BF16 - else if (mType == nvinfer1::DataType::kBF16) + else if (mType == tensorrt_llm::DataType::kBF16) { tokenPerChannelScale<nv_bfloat16>(numel, numModules, numTokens, deviceCumModuleSizes, reinterpret_cast<nv_bfloat16 const*>(input), reinterpret_cast<nv_bfloat16 const* const*>(deviceScalePtrs), diff --git a/cpp/tensorrt_llm/kernels/lora/dora.h b/cpp/tensorrt_llm/kernels/lora/dora.h index fc21fe669366..02cd68e7c1f0 100644 --- a/cpp/tensorrt_llm/kernels/lora/dora.h +++ b/cpp/tensorrt_llm/kernels/lora/dora.h @@ -17,7 +17,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaUtils.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" TRTLLM_NAMESPACE_BEGIN @@ -28,7 +28,7 @@ class DoraImpl public: DoraImpl() = delete; - DoraImpl(std::vector<int> const& outHiddenSizes, nvinfer1::DataType type); + DoraImpl(std::vector<int> const& outHiddenSizes, tensorrt_llm::DataType type); ~DoraImpl() = default; @@ -41,7 +41,7 @@ class DoraImpl private: std::vector<int64_t> mCumModuleSizes; std::vector<int64_t> mHostBuf; - nvinfer1::DataType mType; + tensorrt_llm::DataType mType; }; } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/lora/lora.cpp b/cpp/tensorrt_llm/kernels/lora/lora.cpp index 61f6af00fedc..7a2b7d330afc 100644 --- a/cpp/tensorrt_llm/kernels/lora/lora.cpp +++ b/cpp/tensorrt_llm/kernels/lora/lora.cpp @@ -19,6 +19,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/groupGemm.h" #include "tensorrt_llm/kernels/lora/lora.h" #include "tensorrt_llm/kernels/splitkGroupGemm.h" @@ -48,8 +49,9 @@ void _getProblemParams(cublasOperation_t& transa, cublasOperation_t& transb, int // TODO should reuse the function in gemmPlugin void _runGemm(int const M, int const N, int const K, bool const transA, bool const transB, - nvinfer1::DataType const type, CublasGemmWrapperPtr const& cublasWrapperPtr, void const* act, void const* weight, - void* output, std::optional<cublasLtMatmulHeuristicResult_t> const& heuristic, void* workspace, cudaStream_t stream) + tensorrt_llm::DataType const type, CublasGemmWrapperPtr const& cublasWrapperPtr, void const* act, + void const* weight, void* output, std::optional<cublasLtMatmulHeuristicResult_t> const& heuristic, void* workspace, + cudaStream_t stream) { cublasWrapperPtr->setStream(stream); cublasWrapperPtr->setWorkspace(workspace); @@ -65,7 +67,8 @@ void _runGemm(int const M, int const N, int const K, bool const transA, bool con } LoraImpl::LoraImpl(int in_hidden_size, std::vector<int> out_hidden_sizes, bool transA, bool transB, - int num_lora_modules, nvinfer1::DataType type, int max_low_rank, std::shared_ptr<CublasGemmWrapper> cublasWrapper) + int num_lora_modules, tensorrt_llm::DataType type, int max_low_rank, + std::shared_ptr<CublasGemmWrapper> cublasWrapper) : mInHiddenSize(in_hidden_size) , mTransA(transA) , mTransB(transB) @@ -82,16 +85,16 @@ LoraImpl::LoraImpl(int in_hidden_size, std::vector<int> out_hidden_sizes, bool t void LoraImpl::setGemmConfig() { TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - if (mType == nvinfer1::DataType::kHALF) + if (mType == tensorrt_llm::DataType::kHALF) { mCublasWrapper->setFP16GemmConfig(); } - else if (mType == nvinfer1::DataType::kFLOAT) + else if (mType == tensorrt_llm::DataType::kFLOAT) { mCublasWrapper->setFP32GemmConfig(); } #ifdef ENABLE_BF16 - else if (mType == nvinfer1::DataType::kBF16) + else if (mType == tensorrt_llm::DataType::kBF16) { mCublasWrapper->setBF16GemmConfig(); } @@ -121,7 +124,7 @@ int64_t getGemmWorkSpaceSize(int64_t numTokens, int64_t maxLoraModuleNum, int64_ } size_t LoraImpl::getWorkspaceSize( - int64_t const numTokens, int64_t const numReqs, nvinfer1::DataType const type) const noexcept + int64_t const numTokens, int64_t const numReqs, tensorrt_llm::DataType const type) const noexcept { TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); auto const typeSize = tensorrt_llm::common::getDTypeSize(type); diff --git a/cpp/tensorrt_llm/kernels/lora/lora.h b/cpp/tensorrt_llm/kernels/lora/lora.h index 7215a7af74d4..73a2cbe330be 100644 --- a/cpp/tensorrt_llm/kernels/lora/lora.h +++ b/cpp/tensorrt_llm/kernels/lora/lora.h @@ -19,7 +19,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cublasMMWrapper.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cassert> #include <vector> @@ -37,9 +37,10 @@ class LoraImpl { public: LoraImpl(int in_hidden_size, std::vector<int> out_hidden_sizes, bool transA, bool transB, int num_lora_modules, - nvinfer1::DataType type, int max_low_rank, std::shared_ptr<CublasGemmWrapper> cublasWrapper); + tensorrt_llm::DataType type, int max_low_rank, std::shared_ptr<CublasGemmWrapper> cublasWrapper); - [[nodiscard]] size_t getWorkspaceSize(int64_t numTokens, int64_t numReqs, nvinfer1::DataType type) const noexcept; + [[nodiscard]] size_t getWorkspaceSize( + int64_t numTokens, int64_t numReqs, tensorrt_llm::DataType type) const noexcept; void setBestTactic(std::optional<Config> config); int run(int64_t numTokens, int64_t numReqs, void const* input, int32_t const* loraRanks, void const* const* loraWeightsPtr, int weightIndex, void* const* outputs, void* workspace, cudaStream_t stream); @@ -54,7 +55,7 @@ class LoraImpl private: bool mTransA; bool mTransB; - nvinfer1::DataType mType; + tensorrt_llm::DataType mType; int mNumLoraModules; // @fixme: seems this is shared across multiple clones. diff --git a/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.cu b/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.cu index c3276ea487bc..1165f39b3e5d 100644 --- a/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.cu +++ b/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.cu @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include <cub/block/block_load.cuh> #include <cub/block/block_scan.cuh> @@ -358,7 +359,7 @@ void launchLoraGroupGEMMParamFillRowReorderFusion(int32_t* in_sizes, int32_t* ou int64_t a_base, int64_t d_base, int64_t d_prime_base, int32_t const* slot_counts, int32_t const* slot_ranks, int64_t const* slot_offsets, int32_t const* module_out_sizes, int64_t const* module_out_prefix, int64_t const* b_ptrs, int64_t const* b_prime_ptrs, void const* input, int64_t const* sorted_ids, - int32_t module_count, nvinfer1::DataType dtype, cudaStream_t stream) + int32_t module_count, tensorrt_llm::DataType dtype, cudaStream_t stream) { // Determine block dimensions (1D) // Requirements: 1) >= max_lora_count * module_count 2) >= 256 3) divisible by 32 diff --git a/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.h b/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.h index 3043054ca4b5..835b9f8bed96 100644 --- a/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.h +++ b/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.h @@ -17,7 +17,7 @@ #pragma once #include "tensorrt_llm/common/config.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cstdint> #include <cuda_runtime.h> @@ -70,7 +70,7 @@ void launchLoraGroupGEMMParamFillRowReorderFusion(int32_t* in_sizes, int32_t* ou int64_t a_base, int64_t d_base, int64_t d_prime_base, int32_t const* slot_counts, int32_t const* slot_ranks, int64_t const* slot_offsets, int32_t const* module_out_sizes, int64_t const* module_out_prefix, int64_t const* b_ptrs, int64_t const* b_prime_ptrs, void const* input, int64_t const* sorted_ids, - int32_t module_count, nvinfer1::DataType dtype, cudaStream_t stream); + int32_t module_count, tensorrt_llm::DataType dtype, cudaStream_t stream); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/mhcKernels/fused_tf32_pmap_gemm.cuh b/cpp/tensorrt_llm/kernels/mhcKernels/fused_tf32_pmap_gemm.cuh index ed2841f63ce2..a2bdcec84cf5 100644 --- a/cpp/tensorrt_llm/kernels/mhcKernels/fused_tf32_pmap_gemm.cuh +++ b/cpp/tensorrt_llm/kernels/mhcKernels/fused_tf32_pmap_gemm.cuh @@ -52,40 +52,48 @@ #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wunknown-attributes" -#include <cuda/std/utility> #include <cuda_bf16.h> +#include <cute/atom/mma_traits_sm100.hpp> #include <cutlass/arch/barrier.h> -namespace deep_gemm::sm90 -{ -using cuda::std::swap; -} - -namespace deep_gemm::sm100 -{ -using cuda::std::swap; -} - -#include <deep_gemm/common/reduction.cuh> -#include <deep_gemm/common/sm100_utils.cuh> -#include <deep_gemm/common/sm90_utils.cuh> -#include <deep_gemm/common/tma_utils.cuh> +#include <deep_gemm/common/tma_copy.cuh> #include <deep_gemm/common/utils.cuh> #include <deep_gemm/ptx/ld_st.cuh> +#include <deep_gemm/ptx/tcgen05.cuh> #include <deep_gemm/ptx/utils.cuh> namespace fused_mhc { -// Reuse DeepGEMM's swizzle helper -using deep_gemm::sm100::make_umma_desc; -using deep_gemm::sm100::get_num_aligned_tmem_cols; -using deep_gemm::sm100::tcgen05_before_thread_sync; -using deep_gemm::sm100::tcgen05_after_thread_sync; -using deep_gemm::sm100::advance_umma_desc_lo; -using deep_gemm::tma_copy; -using deep_gemm::utils::PatternVisitor; using deep_gemm::ptx::get_lane_idx; +using deep_gemm::ptx::tcgen05_after_thread_sync; +using deep_gemm::ptx::tcgen05_before_thread_sync; +using deep_gemm::utils::get_num_aligned_tmem_cols; +using deep_gemm::utils::PatternVisitor; + +template <cute::UMMA::Major kMajor, uint32_t kBlockN, uint32_t kBlockSwizzledK> +__device__ __forceinline__ cute::UMMA::SmemDescriptor makeBUmmaDescriptor(float* smemPtr) +{ + constexpr uint32_t kSwizzleBytes = 128U; + static_assert(kMajor == cute::UMMA::Major::K, "B UMMA descriptor must be K-major"); + static_assert(kBlockSwizzledK * sizeof(float) == kSwizzleBytes, "B UMMA descriptor must use 128-byte swizzling"); + + auto const smemLayoutB = cute::tile_to_shape( + cute::UMMA::Layout_K_SW128_Atom<float>{}, cute::Shape<cute::Int<kBlockN>, cute::Int<kBlockSwizzledK>>{}); + auto const tensorB = cute::make_tensor(cute::make_smem_ptr(smemPtr), smemLayoutB); + auto desc = cute::UMMA::make_umma_desc<kMajor>(tensorB); + // A single K swizzle atom does not use the leading byte offset. Keep it zero to preserve the existing descriptor. + desc.leading_byte_offset_ = 0U; + return desc; +} + +__device__ __forceinline__ uint32_t advanceBUmmaDescriptorLo( + uint32_t const base, uint32_t const elementOffset, uint32_t const kIndex) +{ + constexpr uint32_t kElementBytes = static_cast<uint32_t>(sizeof(float)); + constexpr uint32_t kDescriptorAddressShift = 4U; + return base + (((elementOffset + kIndex) * kElementBytes) >> kDescriptorAddressShift); +} template <uint32_t kSwizzleMode, uint32_t kSwizzleBase = 16> __device__ __forceinline__ uint32_t get_swizzled_smem_offset(uint32_t const& offset, uint32_t const& lane_idx) @@ -99,6 +107,64 @@ __device__ __forceinline__ uint32_t get_swizzled_smem_offset(uint32_t const& off return row * 128 + col * kSwizzleBase; } +// Packed fp32x2 helpers (sm_100 f32x2 ALU). Per-lane IEEE fp32, identical +// numerics to scalar fmaf/mul chains, at half the instruction count. +__device__ __forceinline__ unsigned long long bitcast_f32x2_to_u64(float2 v) +{ + unsigned long long r; + memcpy(&r, &v, sizeof(r)); + return r; +} + +__device__ __forceinline__ float2 bitcast_u64_to_f32x2(unsigned long long v) +{ + float2 r; + memcpy(&r, &v, sizeof(r)); + return r; +} + +__device__ __forceinline__ float2 mul_f32x2(float a, float2 b) +{ +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) && (__CUDA_ARCH__ < 1100) + unsigned long long d; + asm("mul.rn.ftz.f32x2 %0, %1, %2;" + : "=l"(d) + : "l"(bitcast_f32x2_to_u64(float2{a, a})), "l"(bitcast_f32x2_to_u64(b))); + return bitcast_u64_to_f32x2(d); +#else + return float2{a * b.x, a * b.y}; +#endif +} + +// Scalar `a` is broadcast to both lanes: {a * b.x + c.x, a * b.y + c.y}. +__device__ __forceinline__ float2 fma_f32x2(float a, float2 b, float2 c) +{ +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) && (__CUDA_ARCH__ < 1100) + unsigned long long d; + asm("fma.rn.ftz.f32x2 %0, %1, %2, %3;" + : "=l"(d) + : "l"(bitcast_f32x2_to_u64(float2{a, a})), "l"(bitcast_f32x2_to_u64(b)), "l"(bitcast_f32x2_to_u64(c))); + return bitcast_u64_to_f32x2(d); +#else + return float2{fmaf(a, b.x, c.x), fmaf(a, b.y, c.y)}; +#endif +} + +// `_vv`: vector-by-vector, i.e. all three operands are per-lane vectors +// (no broadcast): {a.x * b.x + c.x, a.y * b.y + c.y}. +__device__ __forceinline__ float2 fma_f32x2_vv(float2 a, float2 b, float2 c) +{ +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) && (__CUDA_ARCH__ < 1100) + unsigned long long d; + asm("fma.rn.ftz.f32x2 %0, %1, %2, %3;" + : "=l"(d) + : "l"(bitcast_f32x2_to_u64(a)), "l"(bitcast_f32x2_to_u64(b)), "l"(bitcast_f32x2_to_u64(c))); + return bitcast_u64_to_f32x2(d); +#else + return float2{fmaf(a.x, b.x, c.x), fmaf(a.y, b.y, c.y)}; +#endif +} + __device__ __forceinline__ void stsm_x4_b16_rout(void* smem_dst, uint32_t a, uint32_t b, uint32_t c, uint32_t d) { asm volatile( @@ -237,39 +303,9 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) fused_tf3 const uint32_t h_tile_start = k_split_idx * H_TILES_PER_SPLIT; constexpr uint32_t num_total_stages = H_TILES_PER_SPLIT * HC_MULT; - // Prologue: pmap warp group loads post_mix, comb_mix into SMEM - if (warp_idx >= kNumMMAThreads / 32) - { - const uint32_t pmap_tid = threadIdx.x - kNumMMAThreads; -#pragma unroll - for (uint32_t t = 0; t < 2; ++t) - { - uint32_t idx = pmap_tid + t * kNumPmapThreads; - if (idx < BLOCK_M * HC_MULT) - { - uint32_t m = idx / HC_MULT; - uint32_t hc = idx % HC_MULT; - uint32_t gmem_m = m_offset + m; - float v = (gmem_m < shape_m) ? post_mix[gmem_m * HC_MULT + hc] : 0.f; - smem_post[idx] = v; - } - } -#pragma unroll - for (uint32_t t = 0; t < 8; ++t) - { - uint32_t idx = pmap_tid + t * kNumPmapThreads; - if (idx < BLOCK_M * HC_MULT * HC_MULT) - { - uint32_t m = idx / (HC_MULT * HC_MULT); - uint32_t jk = idx % (HC_MULT * HC_MULT); - uint32_t gmem_m = m_offset + m; - float v = (gmem_m < shape_m) ? comb_mix[gmem_m * HC_MULT * HC_MULT + jk] : 0.f; - smem_comb[idx] = v; - } - } - } - __syncthreads(); - + // Prologue removed: pmap threads load their own post_mix/comb_mix rows + // directly into registers below, so the MMA/TMA warps never wait on those + // global loads and the TMA pipeline starts immediately. if (warp_idx < kNumMMAThreads / 32) { // ----- TMA warp (warp 0) ----- @@ -287,10 +323,10 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) fused_tf3 #pragma unroll for (uint32_t j = 0; j < HC_MULT; ++j) { - tma_copy<BLOCK_K, BLOCK_M, kSwizzleResMode>(&tensor_map_residual, full_input[i_stage], + deep_gemm::tma::copy<BLOCK_K, BLOCK_M, kSwizzleResMode>(&tensor_map_residual, full_input[i_stage], smem_res[i_stage] + j * BLOCK_M * BLOCK_K, j * HIDDEN + h_idx, m_idx); } - tma_copy<BLOCK_K, BLOCK_M, kSwizzleXMode>( + deep_gemm::tma::copy<BLOCK_K, BLOCK_M, kSwizzleXMode>( &tensor_map_x, full_input[i_stage], smem_x_stg[i_stage], h_idx, m_idx); constexpr uint32_t kInputBytes = SMEM_RES_PER_ISTG + SMEM_X_PER_ISTG; full_input[i_stage]->arrive_and_expect_tx(kInputBytes); @@ -300,7 +336,7 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) fused_tf3 { empty_B[b_stage]->wait(((s / N_B_STAGES) & 1) ^ 1); uint32_t k_idx = hc * HIDDEN + h_idx; - tma_copy<BLOCK_K, BLOCK_N, kSwizzleBMode>( + deep_gemm::tma::copy<BLOCK_K, BLOCK_N, kSwizzleBMode>( &tensor_map_b, full_B[b_stage], smem_b[b_stage], k_idx, 0); full_B[b_stage]->arrive_and_expect_tx(SMEM_B_PER_STAGE); b_stage = (b_stage + 1) % N_B_STAGES; @@ -323,7 +359,7 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) fused_tf3 UMMA_N, kMajorA, kMajorB>(); auto const& runtime_instr_desc = cute::UMMA::make_runtime_instr_desc(instr_desc); static_assert(N_B_STAGES <= 32, "Too many B stages"); - auto b_desc = make_umma_desc<kMajorB, BLOCK_N, BLOCK_SWIZZLED_BK, kSwizzleBMode>(smem_b[0], 0, 0); + auto b_desc = makeBUmmaDescriptor<kMajorB, BLOCK_N, BLOCK_SWIZZLED_BK>(smem_b[0]); uint32_t const& b_desc_lo = lane_idx < N_B_STAGES ? b_desc.lo + lane_idx * SMEM_B_PER_STAGE / 16 : 0u; for (uint32_t s = 0; s < num_total_stages; ++s) @@ -340,8 +376,7 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) fused_tf3 uint32_t const& atom_idx = (k * UMMA_K) / BLOCK_SWIZZLED_BK; uint32_t const& in_atom_idx = (k * UMMA_K) % BLOCK_SWIZZLED_BK; uint32_t const& offset = atom_idx * BLOCK_N * BLOCK_SWIZZLED_BK; - b_desc.lo = advance_umma_desc_lo<kMajorB, BLOCK_N, kSwizzleBMode, float>( - b_desc_base_lo, offset, in_atom_idx); + b_desc.lo = advanceBUmmaDescriptorLo(b_desc_base_lo, offset, in_atom_idx); umma_t::fma(BLOCK_K * cast_stage_idx + k * UMMA_K, b_desc, BLOCK_K * kNumCastStages, s > 0 or k > 0, runtime_instr_desc); } @@ -417,24 +452,30 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) fused_tf3 float pm_u[HC_MULT], pm_l[HC_MULT]; float cm_u[HC_MULT][HC_MULT], cm_l[HC_MULT][HC_MULT]; -#pragma unroll - for (uint32_t hc = 0; hc < HC_MULT; ++hc) - { - pm_u[hc] = smem_post[upper_row * HC_MULT + hc]; - pm_l[hc] = smem_post[lower_row * HC_MULT + hc]; - } -#pragma unroll - for (uint32_t j = 0; j < HC_MULT; ++j) { + static_assert(HC_MULT == 4, "float4 row loads assume HC_MULT == 4"); + const uint32_t gm_u = m_offset + upper_row; + const uint32_t gm_l = m_offset + lower_row; + auto load_row4 + = [](float const* base, uint32_t row, uint32_t row_floats, uint32_t vec_idx, bool valid) -> float4 + { + if (!valid) + return float4{0.f, 0.f, 0.f, 0.f}; + return __ldg(reinterpret_cast<float4 const*>(base + row * row_floats) + vec_idx); + }; + bool const valid_u = gm_u < shape_m; + bool const valid_l = gm_l < shape_m; + *reinterpret_cast<float4*>(pm_u) = load_row4(post_mix, gm_u, HC_MULT, 0, valid_u); + *reinterpret_cast<float4*>(pm_l) = load_row4(post_mix, gm_l, HC_MULT, 0, valid_l); #pragma unroll - for (uint32_t hc = 0; hc < HC_MULT; ++hc) + for (uint32_t j = 0; j < HC_MULT; ++j) { - cm_u[j][hc] = smem_comb[upper_row * HC_MULT * HC_MULT + j * HC_MULT + hc]; - cm_l[j][hc] = smem_comb[lower_row * HC_MULT * HC_MULT + j * HC_MULT + hc]; + *reinterpret_cast<float4*>(cm_u[j]) = load_row4(comb_mix, gm_u, HC_MULT * HC_MULT, j, valid_u); + *reinterpret_cast<float4*>(cm_l[j]) = load_row4(comb_mix, gm_l, HC_MULT * HC_MULT, j, valid_l); } } - float sqr_u = 0.f, sqr_l = 0.f; + float2 sqr2_u{0.f, 0.f}, sqr2_l{0.f, 0.f}; constexpr uint32_t kNumBankGroupBytes = 16; constexpr uint32_t kNumElemsPerBankGroup = kNumBankGroupBytes / sizeof(nv_bfloat16); constexpr uint32_t kNumLoads = BLOCK_K / kNumElemsPerBankGroup; @@ -456,7 +497,7 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) fused_tf3 for (uint32_t i = 0; i < kNumLoads; i += 2) { auto smem_ptr = x_base + get_swizzled_smem_offset<kSwizzleXMode>(i + lane_idx / 16, lane_idx % 16); - deep_gemm::sm90::SM90_U32x4_LDSM_N::copy(x_vals[0][i + 0], x_vals[1][i + 0], x_vals[0][i + 1], + deep_gemm::ptx::SM90_U32x4_LDSM_N::copy(x_vals[0][i + 0], x_vals[1][i + 0], x_vals[0][i + 1], x_vals[1][i + 1], const_cast<uint8_t*>(smem_ptr)); } } @@ -472,7 +513,7 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) fused_tf3 { auto smem_ptr = r_base + get_swizzled_smem_offset<kSwizzleResMode>(i + lane_idx / 16, lane_idx % 16); - deep_gemm::sm90::SM90_U32x4_LDSM_N::copy(r_vals[j][0][i + 0], r_vals[j][1][i + 0], + deep_gemm::ptx::SM90_U32x4_LDSM_N::copy(r_vals[j][0][i + 0], r_vals[j][1][i + 0], r_vals[j][0][i + 1], r_vals[j][1][i + 1], const_cast<uint8_t*>(smem_ptr)); } } @@ -493,71 +534,89 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) fused_tf3 empty_input[i_stage]->arrive(); } - // Wait for previous ht's residual_out TMA_STOREs to drain before we - // overwrite single-buffered smem_rc with new hc values. + // ---- tile-batched f32x2 post-mapping ---- + // s is a multiple of kNumCastStages at tile start and all four cast + // stages share one phase parity, so claim them all up front, then + // process column pairs with hc innermost: residual bf16->f32 + // conversion happens once per (j, column) instead of once per + // (j, column, hc). + static_assert(kNumCastStages == HC_MULT, "tile-batched cast protocol expects one stage per hc"); +#pragma unroll + for (uint32_t c = 0; c < kNumCastStages; ++c) + { + empty_cast[c]->wait(((s / kNumCastStages) & 1) ^ 1); + } + + // smem_rc is single-buffered: previous tile's residual_cur TMA + // stores must drain before this tile overwrites it. if (ht > 0) { cute::tma_store_wait<0>(); } #pragma unroll - for (uint32_t hc = 0; hc < HC_MULT; ++hc) + for (uint32_t i = 0; i < kNumLoads; i += 2) { - const uint32_t cast_stage_idx = s % kNumCastStages; - empty_cast[cast_stage_idx]->wait(((s / kNumCastStages) & 1) ^ 1); - - uint32_t rc_u_buf[kNumLoads], rc_l_buf[kNumLoads]; + float2 r_u[HC_MULT][2], r_l[HC_MULT][2]; #pragma unroll - for (uint32_t i = 0; i < kNumLoads; ++i) + for (uint32_t j = 0; j < HC_MULT; ++j) { - float2 nu{pm_u[hc] * xf[0][i].x, pm_u[hc] * xf[0][i].y}; - float2 nl{pm_l[hc] * xf[1][i].x, pm_l[hc] * xf[1][i].y}; + r_u[j][0] = __bfloat1622float2(*reinterpret_cast<nv_bfloat162*>(&r_vals[j][0][i + 0])); + r_u[j][1] = __bfloat1622float2(*reinterpret_cast<nv_bfloat162*>(&r_vals[j][0][i + 1])); + r_l[j][0] = __bfloat1622float2(*reinterpret_cast<nv_bfloat162*>(&r_vals[j][1][i + 0])); + r_l[j][1] = __bfloat1622float2(*reinterpret_cast<nv_bfloat162*>(&r_vals[j][1][i + 1])); + } +#pragma unroll + for (uint32_t hc = 0; hc < HC_MULT; ++hc) + { + float2 nu0 = mul_f32x2(pm_u[hc], xf[0][i + 0]); + float2 nu1 = mul_f32x2(pm_u[hc], xf[0][i + 1]); + float2 nl0 = mul_f32x2(pm_l[hc], xf[1][i + 0]); + float2 nl1 = mul_f32x2(pm_l[hc], xf[1][i + 1]); #pragma unroll for (uint32_t j = 0; j < HC_MULT; ++j) { - float2 ruj = __bfloat1622float2(*reinterpret_cast<nv_bfloat162*>(&r_vals[j][0][i])); - float2 rlj = __bfloat1622float2(*reinterpret_cast<nv_bfloat162*>(&r_vals[j][1][i])); - nu.x = fmaf(cm_u[j][hc], ruj.x, nu.x); - nu.y = fmaf(cm_u[j][hc], ruj.y, nu.y); - nl.x = fmaf(cm_l[j][hc], rlj.x, nl.x); - nl.y = fmaf(cm_l[j][hc], rlj.y, nl.y); + nu0 = fma_f32x2(cm_u[j][hc], r_u[j][0], nu0); + nu1 = fma_f32x2(cm_u[j][hc], r_u[j][1], nu1); + nl0 = fma_f32x2(cm_l[j][hc], r_l[j][0], nl0); + nl1 = fma_f32x2(cm_l[j][hc], r_l[j][1], nl1); } - nv_bfloat162 b_up = __float22bfloat162_rn(nu); - nv_bfloat162 b_lo = __float22bfloat162_rn(nl); - uint32_t b_up_bits = *reinterpret_cast<uint32_t*>(&b_up); - uint32_t b_lo_bits = *reinterpret_cast<uint32_t*>(&b_lo); - rc_u_buf[i] = b_up_bits; - rc_l_buf[i] = b_lo_bits; - float2 ru = __bfloat1622float2(b_up); - float2 rl = __bfloat1622float2(b_lo); - sqr_u = fmaf(ru.x, ru.x, sqr_u); - sqr_u = fmaf(ru.y, ru.y, sqr_u); - sqr_l = fmaf(rl.x, rl.x, sqr_l); - sqr_l = fmaf(rl.y, rl.y, sqr_l); - cute::SM100_TMEM_STORE_16dp256b1x::copy(*reinterpret_cast<uint32_t*>(&ru.x), - *reinterpret_cast<uint32_t*>(&ru.y), *reinterpret_cast<uint32_t*>(&rl.x), - *reinterpret_cast<uint32_t*>(&rl.y), cast_stage_idx * BLOCK_K + i * 8); - } - cutlass::arch::fence_view_async_tmem_store(); - tcgen05_before_thread_sync(); - full_cast[cast_stage_idx]->arrive(); - ++s; + nv_bfloat162 b_u0 = __float22bfloat162_rn(nu0); + nv_bfloat162 b_u1 = __float22bfloat162_rn(nu1); + nv_bfloat162 b_l0 = __float22bfloat162_rn(nl0); + nv_bfloat162 b_l1 = __float22bfloat162_rn(nl1); + float2 ru0 = __bfloat1622float2(b_u0); + float2 ru1 = __bfloat1622float2(b_u1); + float2 rl0 = __bfloat1622float2(b_l0); + float2 rl1 = __bfloat1622float2(b_l1); + sqr2_u = fma_f32x2_vv(ru0, ru0, sqr2_u); + sqr2_u = fma_f32x2_vv(ru1, ru1, sqr2_u); + sqr2_l = fma_f32x2_vv(rl0, rl0, sqr2_l); + sqr2_l = fma_f32x2_vv(rl1, rl1, sqr2_l); + cute::SM100_TMEM_STORE_16dp256b1x::copy(*reinterpret_cast<uint32_t*>(&ru0.x), + *reinterpret_cast<uint32_t*>(&ru0.y), *reinterpret_cast<uint32_t*>(&rl0.x), + *reinterpret_cast<uint32_t*>(&rl0.y), hc * BLOCK_K + (i + 0) * 8); + cute::SM100_TMEM_STORE_16dp256b1x::copy(*reinterpret_cast<uint32_t*>(&ru1.x), + *reinterpret_cast<uint32_t*>(&ru1.y), *reinterpret_cast<uint32_t*>(&rl1.x), + *reinterpret_cast<uint32_t*>(&rl1.y), hc * BLOCK_K + (i + 1) * 8); - // STSM bf16 new_r values into smem_rc[hc] sub-region for this warp. - uint8_t* rc_base = reinterpret_cast<uint8_t*>(smem_rc) + hc * SMEM_RC_PER_HC - + sub_warp_idx * BLOCK_M_PER_WARP * kSwizzleRoutMode; -#pragma unroll - for (uint32_t i = 0; i < kNumLoads; i += 2) - { + uint8_t* rc_base = reinterpret_cast<uint8_t*>(smem_rc) + hc * SMEM_RC_PER_HC + + sub_warp_idx * BLOCK_M_PER_WARP * kSwizzleRoutMode; auto smem_ptr = rc_base + get_swizzled_smem_offset<kSwizzleRoutMode>(i + lane_idx / 16, lane_idx % 16); - stsm_x4_b16_rout(smem_ptr, rc_u_buf[i + 0], rc_l_buf[i + 0], rc_u_buf[i + 1], rc_l_buf[i + 1]); + stsm_x4_b16_rout(smem_ptr, *reinterpret_cast<uint32_t*>(&b_u0), *reinterpret_cast<uint32_t*>(&b_l0), + *reinterpret_cast<uint32_t*>(&b_u1), *reinterpret_cast<uint32_t*>(&b_l1)); } } - if constexpr (!kEarlyRelease) + + cutlass::arch::fence_view_async_tmem_store(); + tcgen05_before_thread_sync(); +#pragma unroll + for (uint32_t c = 0; c < kNumCastStages; ++c) { - empty_input[i_stage]->arrive(); + full_cast[c]->arrive(); } + s += kNumCastStages; // Emit HC_MULT TMA_STOREs of residual_cur: one per hc slice, per-warp rows. cute::tma_store_fence(); @@ -574,12 +633,18 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) fused_tf3 cute::tma_store_arrive(); } } + if constexpr (!kEarlyRelease) + { + empty_input[i_stage]->arrive(); + } } // Drain any in-flight residual_out TMA stores before exit. cute::tma_store_wait<0>(); // Warp-reduce sqr across 4 col_lanes then atomicAdd to global. + float sqr_u = sqr2_u.x + sqr2_u.y; + float sqr_l = sqr2_l.x + sqr2_l.y; sqr_u += __shfl_xor_sync(0xffffffff, sqr_u, 1); sqr_u += __shfl_xor_sync(0xffffffff, sqr_u, 2); sqr_l += __shfl_xor_sync(0xffffffff, sqr_l, 1); @@ -839,10 +904,10 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) #pragma unroll for (uint32_t j = 0; j < HC_MULT; ++j) { - tma_copy<BLOCK_K, BLOCK_M, kSwizzleResMode>(&tensor_map_residual, full_input[i_stage], + deep_gemm::tma::copy<BLOCK_K, BLOCK_M, kSwizzleResMode>(&tensor_map_residual, full_input[i_stage], smem_res[i_stage] + j * BLOCK_M * BLOCK_K, j * HIDDEN + h_idx, m_idx); } - tma_copy<BLOCK_K, BLOCK_M, kSwizzleXMode>( + deep_gemm::tma::copy<BLOCK_K, BLOCK_M, kSwizzleXMode>( &tensor_map_x, full_input[i_stage], smem_x_stg[i_stage], h_idx, m_idx); constexpr uint32_t kInputBytes = SMEM_RES_PER_ISTG + SMEM_X_PER_ISTG; full_input[i_stage]->arrive_and_expect_tx(kInputBytes); @@ -852,7 +917,7 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) { empty_B[b_stage]->wait(((s / N_B_STAGES) & 1) ^ 1); uint32_t k_idx = hc * HIDDEN + h_idx; - tma_copy<BLOCK_K, BLOCK_N, kSwizzleBMode>( + deep_gemm::tma::copy<BLOCK_K, BLOCK_N, kSwizzleBMode>( &tensor_map_b, full_B[b_stage], smem_b[b_stage], k_idx, 0); full_B[b_stage]->arrive_and_expect_tx(SMEM_B_PER_STAGE); b_stage = (b_stage + 1) % N_B_STAGES; @@ -875,7 +940,7 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) UMMA_N, kMajorA, kMajorB>(); auto const& runtime_instr_desc = cute::UMMA::make_runtime_instr_desc(instr_desc); static_assert(N_B_STAGES <= 32, "Too many B stages"); - auto b_desc = make_umma_desc<kMajorB, BLOCK_N, BLOCK_SWIZZLED_BK, kSwizzleBMode>(smem_b[0], 0, 0); + auto b_desc = makeBUmmaDescriptor<kMajorB, BLOCK_N, BLOCK_SWIZZLED_BK>(smem_b[0]); uint32_t const& b_desc_lo = lane_idx < N_B_STAGES ? b_desc.lo + lane_idx * SMEM_B_PER_STAGE / 16 : 0u; for (uint32_t s = 0; s < num_total_stages; ++s) @@ -892,8 +957,7 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) uint32_t const& atom_idx = (k * UMMA_K) / BLOCK_SWIZZLED_BK; uint32_t const& in_atom_idx = (k * UMMA_K) % BLOCK_SWIZZLED_BK; uint32_t const& offset = atom_idx * BLOCK_N * BLOCK_SWIZZLED_BK; - b_desc.lo = advance_umma_desc_lo<kMajorB, BLOCK_N, kSwizzleBMode, float>( - b_desc_base_lo, offset, in_atom_idx); + b_desc.lo = advanceBUmmaDescriptorLo(b_desc_base_lo, offset, in_atom_idx); umma_t::fma(BLOCK_K * cast_stage_idx + k * UMMA_K, b_desc, BLOCK_K * kNumCastStages, s > 0 or k > 0, runtime_instr_desc); } @@ -1008,7 +1072,7 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) for (uint32_t i = 0; i < kNumLoads; i += 2) { auto smem_ptr = x_base + get_swizzled_smem_offset<kSwizzleXMode>(i + lane_idx / 16, lane_idx % 16); - deep_gemm::sm90::SM90_U32x4_LDSM_N::copy(x_vals[0][i + 0], x_vals[1][i + 0], x_vals[0][i + 1], + deep_gemm::ptx::SM90_U32x4_LDSM_N::copy(x_vals[0][i + 0], x_vals[1][i + 0], x_vals[0][i + 1], x_vals[1][i + 1], const_cast<uint8_t*>(smem_ptr)); } } @@ -1024,7 +1088,7 @@ __global__ void __launch_bounds__(kNumMMAThreads + kNumPmapThreads, 1) { auto smem_ptr = r_base + get_swizzled_smem_offset<kSwizzleResMode>(i + lane_idx / 16, lane_idx % 16); - deep_gemm::sm90::SM90_U32x4_LDSM_N::copy(r_vals[j][0][i + 0], r_vals[j][1][i + 0], + deep_gemm::ptx::SM90_U32x4_LDSM_N::copy(r_vals[j][0][i + 0], r_vals[j][1][i + 0], r_vals[j][0][i + 1], r_vals[j][1][i + 1], const_cast<uint8_t*>(smem_ptr)); } } diff --git a/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.cu b/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.cu index b70a9549df86..45b1a38266b1 100644 --- a/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.cu +++ b/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.cu @@ -389,7 +389,7 @@ void launchRmsNormFp4QuantKernel(RmsNormFp4QuantParams const& params, cudaStream } // namespace rms_norm_fp4_quant -void residualRmsNormFp4Quant(RmsNormFp4QuantParams const& params, nvinfer1::DataType dataType, cudaStream_t stream) +void residualRmsNormFp4Quant(RmsNormFp4QuantParams const& params, tensorrt_llm::DataType dataType, cudaStream_t stream) { // The NVFP4 epilogue (cvt_warp_fp16_to_fp4) is compiled only for // __CUDA_ARCH__ >= 1000 and emits zeros otherwise, so this kernel is correct @@ -410,11 +410,11 @@ void residualRmsNormFp4Quant(RmsNormFp4QuantParams const& params, nvinfer1::Data switch (dataType) { #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: rms_norm_fp4_quant::launchRmsNormFp4QuantKernel<__nv_bfloat16, /*OutNorm=*/true>(params, stream); break; #endif - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: rms_norm_fp4_quant::launchRmsNormFp4QuantKernel<half, /*OutNorm=*/true>(params, stream); break; default: TLLM_THROW("Unsupported dataType for residualRmsNormFp4Quant"); @@ -425,11 +425,11 @@ void residualRmsNormFp4Quant(RmsNormFp4QuantParams const& params, nvinfer1::Data switch (dataType) { #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: rms_norm_fp4_quant::launchRmsNormFp4QuantKernel<__nv_bfloat16, /*OutNorm=*/false>(params, stream); break; #endif - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: rms_norm_fp4_quant::launchRmsNormFp4QuantKernel<half, /*OutNorm=*/false>(params, stream); break; default: TLLM_THROW("Unsupported dataType for residualRmsNormFp4Quant"); diff --git a/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.h b/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.h index 86e998fea209..0ecba538d567 100644 --- a/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.h +++ b/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.h @@ -17,8 +17,8 @@ #pragma once #include "tensorrt_llm/common/config.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/quantization.h" -#include <NvInferRuntime.h> #include <cstdint> TRTLLM_NAMESPACE_BEGIN @@ -80,7 +80,7 @@ struct RmsNormFp4QuantParams // fp4_quantize) pair becomes one launch on the attention-DP path. All inputs, // outputs, and layout configuration are carried in params (see the struct // field docs above); dataType selects the fp16/bf16 instantiation. -void residualRmsNormFp4Quant(RmsNormFp4QuantParams const& params, nvinfer1::DataType dataType, cudaStream_t stream); +void residualRmsNormFp4Quant(RmsNormFp4QuantParams const& params, tensorrt_llm::DataType dataType, cudaStream_t stream); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/splitkGroupGemm.cu b/cpp/tensorrt_llm/kernels/splitkGroupGemm.cu index 6397396ea6f6..1f63189ec657 100644 --- a/cpp/tensorrt_llm/kernels/splitkGroupGemm.cu +++ b/cpp/tensorrt_llm/kernels/splitkGroupGemm.cu @@ -26,6 +26,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/device/splitk_gemm_grouped.h" #include "tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/default_splitk_gemm_grouped.h" #include "tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/splitk_gemm_grouped.h" @@ -203,20 +204,20 @@ template <int M1, int N1, int K1, int M2, int N2, int K2, int kAlignmentAB, int void splitkGroupedGemmType_(std::vector<cutlass::gemm::GemmCoord> const& problemSizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkSpace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkSpaceSize, - nvinfer1::DataType dataType, int splitKSlices, cudaStream_t stream) + tensorrt_llm::DataType dataType, int splitKSlices, cudaStream_t stream) { - if (dataType == nvinfer1::DataType::kHALF) + if (dataType == tensorrt_llm::DataType::kHALF) { splitkGroupedGemm_<M1, N1, K1, M2, N2, K2, cutlass::half_t, kAlignmentAB, kAlignmentC, kStages>(problemSizes, ptrA, ptrB, ptrC, ptrD, gemmParamsWorkSpace, gemmParamsWorkSpaceSize, gemmWorkSpace, gemmWorkSpaceSize, splitKSlices, stream); } - else if (dataType == nvinfer1::DataType::kFLOAT) + else if (dataType == tensorrt_llm::DataType::kFLOAT) { TLLM_CHECK_WITH_INFO(false, "not support float input/output"); } #ifdef ENABLE_BF16 - else if (dataType == nvinfer1::DataType::kBF16) + else if (dataType == tensorrt_llm::DataType::kBF16) { splitkGroupedGemm_<M1, N1, K1, M2, N2, K2, cutlass::bfloat16_t, kAlignmentAB, kAlignmentC, kStages>( problemSizes, ptrA, ptrB, ptrC, ptrD, gemmParamsWorkSpace, gemmParamsWorkSpaceSize, gemmWorkSpace, @@ -228,7 +229,7 @@ void splitkGroupedGemmType_(std::vector<cutlass::gemm::GemmCoord> const& problem void splitkGroupedGemm(std::vector<cutlass::gemm::GemmCoord> const& problemSizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkSpace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkSpaceSize, - bool isLoraIn, nvinfer1::DataType dataType, int splitKSlices, int minKN, cudaStream_t stream) + bool isLoraIn, tensorrt_llm::DataType dataType, int splitKSlices, int minKN, cudaStream_t stream) { TLLM_LOG_TRACE("%s start, isLoraIn: %d, minKN = %d", __PRETTY_FUNCTION__, static_cast<int>(isLoraIn), minKN); if (isLoraIn) diff --git a/cpp/tensorrt_llm/kernels/splitkGroupGemm.h b/cpp/tensorrt_llm/kernels/splitkGroupGemm.h index 6ada8255292e..bcde457db7d2 100644 --- a/cpp/tensorrt_llm/kernels/splitkGroupGemm.h +++ b/cpp/tensorrt_llm/kernels/splitkGroupGemm.h @@ -17,7 +17,7 @@ #include "cutlass/gemm_coord.h" #include "tensorrt_llm/common/config.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <vector> TRTLLM_NAMESPACE_BEGIN @@ -30,7 +30,7 @@ int64_t getSplitkGroupedGemmParamsWorkSpaceSize(int64_t problem_count); void splitkGroupedGemm(std::vector<cutlass::gemm::GemmCoord> const& problem_sizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkspace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkspaceSize, - bool isLoraIn, nvinfer1::DataType dataType, int splitKSlices, int minKN, cudaStream_t stream); + bool isLoraIn, tensorrt_llm::DataType dataType, int splitKSlices, int minKN, cudaStream_t stream); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/trtllmGenKernels/blockScaleMoe/runner.cu b/cpp/tensorrt_llm/kernels/trtllmGenKernels/blockScaleMoe/runner.cu index 480d4b67e1cf..88e2f3c35143 100644 --- a/cpp/tensorrt_llm/kernels/trtllmGenKernels/blockScaleMoe/runner.cu +++ b/cpp/tensorrt_llm/kernels/trtllmGenKernels/blockScaleMoe/runner.cu @@ -174,9 +174,9 @@ void Runner::run(void* routingLogits, void* routingBias, int32_t numTokens, int3 } else if (routingMethodType == RoutingMethodType::MiniMax2) { - // MiniMaxM2: sigmoid(logit) + bias → topK → renormalize un-biased sigmoid scores. - // Similar to DeepSeek no-groups but with routeScale = 1.0 and epsilon = 1e-20 - // to match the Python reference: weight / (sum + 1e-20). + // MiniMaxM2/M3: sigmoid(logit) + bias → topK → renormalize un-biased sigmoid scores. + // MiniMaxM2 uses the default routeScale = 1.0, while MiniMaxM3 supplies its + // model-specific scale. Both use epsilon = 1e-20 to match the Python reference. moe::dev::routing::routingCustom::Data routingData; // @@ -189,7 +189,7 @@ void Runner::run(void* routingLogits, void* routingBias, int32_t numTokens, int3 routingData.mPostprocessType = moe::dev::routing::RoutingPostprocessType::ScaledSumNormalize; routingData.mPtrRoutingBias = routingBias; routingData.mDtypeBias = dtypeRoutingBias; - routingData.mRouteScale = 1.0f; + routingData.mRouteScale = routedScalingFactor; routingData.mSumEpsilon = 1e-20f; // Pass-through raw pointer; kernels will cast to the proper InputT based on routing method diff --git a/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaKernels.h b/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaKernels.h index d1681c39a56e..8bcc29e0dbdb 100644 --- a/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaKernels.h +++ b/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaKernels.h @@ -496,7 +496,11 @@ class TllmGenFmhaKernel tg::CudaRunner::Grid grid{numCtasX, numCtasY, numCtasZ}; // Prepare custom mask for spec-decoding generation kernels if needed. - if (params.mLayerIdx == 0 && params.mIsSpecDecTree) + bool const prepareSpecDecTreeMask = params.mIsSpecDecTree + && (params.mForcePrepareSpecDecTreeMask || params.mLayerIdx == 0 + || (params.mSpecDecodingTargetMaxGenLen > 0 + && params.mMaxSeqLenQ != params.mSpecDecodingTargetMaxGenLen)); + if (prepareSpecDecTreeMask) { int32_t stepQ = options.mTileSizeQ * options.mNumInstsQ; int32_t stepKv = options.mTileSizeKv * options.mNumInstsKv; diff --git a/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaRunnerParams.h b/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaRunnerParams.h index 83b816c346f5..9252650ac67b 100644 --- a/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaRunnerParams.h +++ b/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaRunnerParams.h @@ -369,6 +369,7 @@ struct TllmGenFmhaRunnerParams // row stride ceilDiv(mPackedMaskMaxSeqLenQ, 32) rather than ceilDiv(seqLenQ, 32). int32_t mPackedMaskMaxSeqLenQ = 0; int32_t mSpecDecodingTargetMaxGenLen = 0; + bool mForcePrepareSpecDecTreeMask = false; // set the attention mask type TllmGenFmhaRunnerParams& setAttentionMaskType(std::int8_t maskType) diff --git a/cpp/tensorrt_llm/kernels/unfusedAttentionKernels.h b/cpp/tensorrt_llm/kernels/unfusedAttentionKernels.h index 302c278f7a2a..3a757c3263d9 100644 --- a/cpp/tensorrt_llm/kernels/unfusedAttentionKernels.h +++ b/cpp/tensorrt_llm/kernels/unfusedAttentionKernels.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/gptKernels.h" #include "tensorrt_llm/kernels/kvCacheUtils.h" #include "tensorrt_llm/kernels/mlaKernels.h" @@ -242,42 +243,42 @@ struct QKVPreprocessingParams { ss << "seq_lens: " << *(runtime::ITensor::wrap( - (void*) seq_lens, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); + (void*) seq_lens, tensorrt_llm::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); } if (cache_seq_lens && batch_size > 0) { ss << "cache_seq_lens: " - << *(runtime::ITensor::wrap( - (void*) cache_seq_lens, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); + << *(runtime::ITensor::wrap((void*) cache_seq_lens, tensorrt_llm::DataType::kINT32, + runtime::ITensor::makeShape({batch_size}))); } if (encoder_seq_lens && batch_size > 0) { ss << "encoder_seq_lens: " - << *(runtime::ITensor::wrap( - (void*) encoder_seq_lens, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); + << *(runtime::ITensor::wrap((void*) encoder_seq_lens, tensorrt_llm::DataType::kINT32, + runtime::ITensor::makeShape({batch_size}))); } if (cu_seq_lens && batch_size > 0) { ss << "cu_seq_lens: " << *(runtime::ITensor::wrap( - (void*) cu_seq_lens, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); + (void*) cu_seq_lens, tensorrt_llm::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); } if (cu_kv_seq_lens && batch_size > 0) { ss << "cu_kv_seq_lens: " - << *(runtime::ITensor::wrap( - (void*) cu_kv_seq_lens, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); + << *(runtime::ITensor::wrap((void*) cu_kv_seq_lens, tensorrt_llm::DataType::kINT32, + runtime::ITensor::makeShape({batch_size}))); } if (sparse_kv_offsets) { ss << "sparse_kv_offsets: " - << *(runtime::ITensor::wrap((void*) sparse_kv_offsets, nvinfer1::DataType::kINT32, + << *(runtime::ITensor::wrap((void*) sparse_kv_offsets, tensorrt_llm::DataType::kINT32, runtime::ITensor::makeShape({batch_size + 1}))); } if (rotary_embedding_inv_freq && batch_size > 0 && rotary_embedding_dim > 0) { ss << "rotary_embedding_inv_freq: " - << *(runtime::ITensor::wrap((void*) rotary_embedding_inv_freq, nvinfer1::DataType::kFLOAT, + << *(runtime::ITensor::wrap((void*) rotary_embedding_inv_freq, tensorrt_llm::DataType::kFLOAT, runtime::ITensor::makeShape({batch_size, rotary_embedding_dim / 2}))); } ss << "rotary_coef_cache_buffer: " << rotary_coef_cache_buffer << std::endl; @@ -417,6 +418,16 @@ template <typename T, typename KVCacheBuffer> void invokeDebugSparseKvCacheParams( QKVPreprocessingParams<T, KVCacheBuffer> params, int* debug_output, cudaStream_t stream); +//! Compact a uniform group of KVCacheManagerV2 layer pools in one batched launch +//! (per request and head, moves are ascending and never overtake their sources: +//! the copy runs in place). +template <typename T> +void invokeSparseKvCacheCompactLayers(int64_t const* poolPointers, int32_t const* pageTable, int32_t numLayers, + int64_t pageTableRequestStride, int32_t const* sparseKvIndices, int32_t const* sourceLayerIndices, + int64_t sourceLayerStride, int64_t sourceHeadStride, int32_t const* sparseKvOffsets, + int32_t const* destinationBases, int32_t batchSize, int32_t numKvHeads, int32_t tokensPerBlock, int32_t headDim, + cudaStream_t stream); + template <typename T, typename KVCacheBuffer> void invokeKvCachePostprocessing(QKVPreprocessingParams<T, KVCacheBuffer> params, cudaStream_t stream) { diff --git a/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_bf16_bf16.cu b/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_bf16_bf16.cu index 5d006ef4a979..8c2d11804d38 100644 --- a/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_bf16_bf16.cu +++ b/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_bf16_bf16.cu @@ -26,6 +26,7 @@ namespace kernels #ifdef ENABLE_BF16 INSTANTIATE_ATTENTION_INPUT_OUTPUT_PROCESSING(__nv_bfloat16, __nv_bfloat16, KVBlockArray); INSTANTIATE_ATTENTION_INPUT_OUTPUT_PROCESSING(__nv_bfloat16, __nv_bfloat16, KVLinearBuffer); +INSTANTIATE_SPARSE_KV_CACHE_COMPACT_LAYERS(__nv_bfloat16); #endif } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_template.h b/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_template.h index 560a153ffa70..22e17b438d58 100644 --- a/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_template.h +++ b/cpp/tensorrt_llm/kernels/unfusedAttentionKernels/unfusedAttentionKernels_2_template.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2024, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. * Copyright (c) 2021, NAVER Corp. Authored by CLOVA. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,6 +29,8 @@ #include "tensorrt_llm/kernels/quantization.cuh" #include "tensorrt_llm/kernels/unfusedAttentionKernels.h" +#include <type_traits> + using namespace tensorrt_llm::common; TRTLLM_NAMESPACE_BEGIN @@ -1754,6 +1756,213 @@ void invokeUpdateCyclicKvCacheAfterFmha(QKVPreprocessingParams<T, KVCacheBuffer> //////////////////////////////////////////////////////////////////////////////////////////////////// +#ifdef ENABLE_BF16 + +// Pipelined bf16 compaction kernels: double-buffered cp.async copies that +// compact paged KV pools in place through the V2 block-offset table. + +namespace compact_detail +{ +// Vendored cp.async wrappers (xqa's ldgsts.cuh cannot be included here). +template <uint32_t size> +__device__ __forceinline__ void copyAsync(void* dst, void const* src, uint32_t srcSize = size) +{ + static_assert(size == 16, "only the 16B cp.async variant is vendored"); + // srcSize == 0 zero-fills shared memory instead of reading global. + if (srcSize == 0) + { + src = nullptr; + } + asm volatile( + "cp.async.cg.shared.global [%0], [%1], 16, %2;\n" ::"l"(__cvta_generic_to_shared(dst)), "l"(src), "r"(srcSize)); +} + +__device__ __forceinline__ void commitGroup() +{ + asm volatile("cp.async.commit_group;\n"); +} + +// Wait until at most InFlightGroups cp.async groups remain in flight. +template <uint32_t InFlightGroups> +__device__ __forceinline__ void waitGroup() +{ + asm volatile("cp.async.wait_group %0;\n" ::"n"(InFlightGroups)); +} +} // namespace compact_detail + +// One pipeline stage moves a 32-token tile regardless of the page geometry. +constexpr int32_t kSparseKvCompactTokensPerTile = 32; + +//! Launch parameters for the pipelined bf16 fast-path compaction kernels. +struct SparseKvCacheCompactBf16Params +{ + int64_t const* poolPointers; + int32_t const* pageTable; + int32_t const* sourceIndices; + int32_t const* sourceOffsets; + int32_t const* sourceLayerIndices; + int32_t const* destinationBases; + int64_t sourceLayerStride; + int64_t sourceHeadStride; + int64_t pageTableRequestStride; + int32_t numLayers; + int32_t batchSize; + int32_t numKvHeads; + size_t bytesPerKvHalf; + size_t bytesPerPage; +}; + +//! Double-buffered cp.async pipeline: the next tile streams in while the +//! current tile drains. One CTA per (layer, KV head, request). +template <typename T, int32_t HeadDim, int32_t TokensPerBlock> +__global__ __launch_bounds__(HeadDim * sizeof(T) / sizeof(uint4) + * kSparseKvCompactTokensPerTile) void sparseKvCacheCompactV2Bf16PipelineKernel(SparseKvCacheCompactBf16Params + params) +{ + static_assert(std::is_same_v<T, __nv_bfloat16>); + static_assert(HeadDim == 64 || HeadDim == 128); + // 128-token pages are the geometry the kernel was written for; 32-token + // pages cover the supported production configuration (one tile == one page). + static_assert(TokensPerBlock == 32 || TokensPerBlock == 128); + constexpr int32_t kVectorsPerHead = HeadDim * sizeof(T) / sizeof(uint4); + constexpr int32_t kTokensPerTile = kSparseKvCompactTokensPerTile; + constexpr int32_t kVectorsPerTile = kTokensPerTile * kVectorsPerHead; + // A buffer holds one K tile plus one V tile; two buffers ping-pong. + constexpr int32_t kVectorsPerBuffer = 2 * kVectorsPerTile; + + int32_t const layerIdx = static_cast<int32_t>(blockIdx.x); + int32_t const kvHeadIdx = static_cast<int32_t>(blockIdx.y); + int32_t const batchIdx = static_cast<int32_t>(blockIdx.z); + int32_t const moveBegin = params.sourceOffsets[batchIdx]; + int32_t const moveEnd = params.sourceOffsets[batchIdx + 1]; + int32_t const moveCount = moveEnd - moveBegin; + if (moveCount <= 0) + { + return; + } + + // Without an explicit layer map, launch layer i reads source plane i. + int32_t const sourceLayer = params.sourceLayerIndices == nullptr ? layerIdx : params.sourceLayerIndices[layerIdx]; + int64_t const sourceMoveBase = static_cast<int64_t>(sourceLayer) * params.sourceLayerStride + + static_cast<int64_t>(kvHeadIdx) * params.sourceHeadStride + moveBegin; + int32_t const destinationBase = params.destinationBases[batchIdx]; + auto* const pool = reinterpret_cast<uint8_t*>(static_cast<uintptr_t>(params.poolPointers[layerIdx])); + // Block-offset entries decode to a page with >> 1. + int32_t const* const pageTable = params.pageTable + static_cast<int64_t>(batchIdx) * params.pageTableRequestStride; + + extern __shared__ uint4 sharedVectors[]; + int32_t const sharedVector + = static_cast<int32_t>(threadIdx.y) * kVectorsPerHead + static_cast<int32_t>(threadIdx.x); + int32_t currentRequestMove = static_cast<int32_t>(threadIdx.y); + bool currentValid = currentRequestMove < moveCount; + int32_t currentSourceToken = currentValid ? params.sourceIndices[sourceMoveBase + currentRequestMove] : -1; + uint4* currentSharedK = sharedVectors; + uint4* currentSharedV = currentSharedK + kVectorsPerTile; + + // Prologue: explicitly wait for tile 0 and synchronize the CTA before any thread stores it. + uint4 const* currentSourceKVector = nullptr; + uint4 const* currentSourceVVector = nullptr; + if (currentValid) + { + int32_t const sourcePage = pageTable[currentSourceToken / TokensPerBlock] >> 1; + auto* const sourcePageBase = pool + static_cast<size_t>(sourcePage) * params.bytesPerPage; + auto const* const sourceK = reinterpret_cast<uint4 const*>(sourcePageBase); + auto const* const sourceV = reinterpret_cast<uint4 const*>(sourcePageBase + params.bytesPerKvHalf); + int32_t const localVector = (kvHeadIdx * TokensPerBlock + currentSourceToken % TokensPerBlock) * kVectorsPerHead + + static_cast<int32_t>(threadIdx.x); + currentSourceKVector = &sourceK[localVector]; + currentSourceVVector = &sourceV[localVector]; + } + uint32_t const currentSourceBytes = currentValid ? sizeof(uint4) : 0U; + compact_detail::copyAsync<sizeof(uint4)>(¤tSharedK[sharedVector], currentSourceKVector, currentSourceBytes); + compact_detail::copyAsync<sizeof(uint4)>(¤tSharedV[sharedVector], currentSourceVVector, currentSourceBytes); + compact_detail::commitGroup(); + compact_detail::waitGroup<0>(); + __syncthreads(); + + for (int32_t nextTileBegin = kTokensPerTile; nextTileBegin < moveCount; nextTileBegin += kTokensPerTile) + { + int32_t const nextRequestMove = nextTileBegin + static_cast<int32_t>(threadIdx.y); + bool const nextValid = nextRequestMove < moveCount; + int32_t const nextSourceToken = nextValid ? params.sourceIndices[sourceMoveBase + nextRequestMove] : -1; + int32_t const nextBuffer = (nextTileBegin / kTokensPerTile) & 1; + uint4* const nextSharedK = sharedVectors + nextBuffer * kVectorsPerBuffer; + uint4* const nextSharedV = nextSharedK + kVectorsPerTile; + + uint4 const* nextSourceKVector = nullptr; + uint4 const* nextSourceVVector = nullptr; + if (nextValid) + { + int32_t const sourcePage = pageTable[nextSourceToken / TokensPerBlock] >> 1; + auto* const sourcePageBase = pool + static_cast<size_t>(sourcePage) * params.bytesPerPage; + auto const* const sourceK = reinterpret_cast<uint4 const*>(sourcePageBase); + auto const* const sourceV = reinterpret_cast<uint4 const*>(sourcePageBase + params.bytesPerKvHalf); + int32_t const localVector + = (kvHeadIdx * TokensPerBlock + nextSourceToken % TokensPerBlock) * kVectorsPerHead + + static_cast<int32_t>(threadIdx.x); + nextSourceKVector = &sourceK[localVector]; + nextSourceVVector = &sourceV[localVector]; + } + uint32_t const nextSourceBytes = nextValid ? sizeof(uint4) : 0U; + compact_detail::copyAsync<sizeof(uint4)>(&nextSharedK[sharedVector], nextSourceKVector, nextSourceBytes); + compact_detail::copyAsync<sizeof(uint4)>(&nextSharedV[sharedVector], nextSourceVVector, nextSourceBytes); + compact_detail::commitGroup(); + + // In-place safety: sources are strictly increasing with dst(i) <= src(i), + // so current stores never alias future prefetch sources. + int32_t const destinationToken = destinationBase + currentRequestMove; + if (currentValid && currentSourceToken != destinationToken) + { + int32_t const destinationPage = pageTable[destinationToken / TokensPerBlock] >> 1; + auto* const destinationPageBase = pool + static_cast<size_t>(destinationPage) * params.bytesPerPage; + auto* const destinationK = reinterpret_cast<uint4*>(destinationPageBase); + auto* const destinationV = reinterpret_cast<uint4*>(destinationPageBase + params.bytesPerKvHalf); + int32_t const localVector + = (kvHeadIdx * TokensPerBlock + destinationToken % TokensPerBlock) * kVectorsPerHead + + static_cast<int32_t>(threadIdx.x); + destinationK[localVector] = currentSharedK[sharedVector]; + destinationV[localVector] = currentSharedV[sharedVector]; + } + + // waitGroup<0> completes the next tile's copies before the buffer swap. + compact_detail::waitGroup<0>(); + __syncthreads(); + currentRequestMove = nextRequestMove; + currentValid = nextValid; + currentSourceToken = nextSourceToken; + currentSharedK = nextSharedK; + currentSharedV = nextSharedV; + } + + // Epilogue: the final tile already completed its async wait and CTA barrier. + int32_t const destinationToken = destinationBase + currentRequestMove; + if (currentValid && currentSourceToken != destinationToken) + { + int32_t const destinationPage = pageTable[destinationToken / TokensPerBlock] >> 1; + auto* const destinationPageBase = pool + static_cast<size_t>(destinationPage) * params.bytesPerPage; + auto* const destinationK = reinterpret_cast<uint4*>(destinationPageBase); + auto* const destinationV = reinterpret_cast<uint4*>(destinationPageBase + params.bytesPerKvHalf); + int32_t const localVector = (kvHeadIdx * TokensPerBlock + destinationToken % TokensPerBlock) * kVectorsPerHead + + static_cast<int32_t>(threadIdx.x); + destinationK[localVector] = currentSharedK[sharedVector]; + destinationV[localVector] = currentSharedV[sharedVector]; + } +} + +template <typename T, int32_t HeadDim, int32_t TokensPerBlock> +void launchSparseKvCacheCompactV2Bf16Pipeline(SparseKvCacheCompactBf16Params const& params, cudaStream_t stream) +{ + constexpr int32_t kVectorsPerHead = HeadDim * sizeof(T) / sizeof(uint4); + dim3 const block(kVectorsPerHead, kSparseKvCompactTokensPerTile); + dim3 const grid(params.numLayers, params.numKvHeads, params.batchSize); + // Two ping-pong buffers of (K + V) tiles; both geometries fit the 48 KiB + // per-CTA dynamic shared memory default. + size_t const sharedBytes = 4 * kSparseKvCompactTokensPerTile * kVectorsPerHead * sizeof(uint4); + sparseKvCacheCompactV2Bf16PipelineKernel<T, HeadDim, TokensPerBlock><<<grid, block, sharedBytes, stream>>>(params); +} + +#endif // ENABLE_BF16 + template <typename T, typename TCache, int BLOCK_SIZE, int Dh, typename KVCacheBuffer> __global__ __launch_bounds__(BLOCK_SIZE) void updateSparseKvCacheAfterFmha( QKVPreprocessingParams<T, KVCacheBuffer> params) @@ -1876,6 +2085,60 @@ void invokeUpdateSparseKvCacheAfterFmha(QKVPreprocessingParams<T, KVCacheBuffer> } } +template <typename T> +void invokeSparseKvCacheCompactLayers(int64_t const* poolPointers, int32_t const* pageTable, int32_t numLayers, + int64_t pageTableRequestStride, int32_t const* sparseKvIndices, int32_t const* sourceLayerIndices, + int64_t sourceLayerStride, int64_t sourceHeadStride, int32_t const* sparseKvOffsets, + int32_t const* destinationBases, int32_t batchSize, int32_t numKvHeads, int32_t tokensPerBlock, int32_t headDim, + cudaStream_t stream) +{ +#ifdef ENABLE_BF16 + if constexpr (std::is_same_v<T, __nv_bfloat16>) + { + if ((headDim == 64 || headDim == 128) && (tokensPerBlock == 32 || tokensPerBlock == 128)) + { + SparseKvCacheCompactBf16Params fastParams{}; + fastParams.poolPointers = poolPointers; + fastParams.pageTable = pageTable; + fastParams.sourceIndices = sparseKvIndices; + fastParams.sourceOffsets = sparseKvOffsets; + fastParams.sourceLayerIndices = sourceLayerIndices; + fastParams.destinationBases = destinationBases; + fastParams.sourceLayerStride = sourceLayerStride; + fastParams.sourceHeadStride = sourceHeadStride; + fastParams.pageTableRequestStride = pageTableRequestStride; + fastParams.numLayers = numLayers; + fastParams.batchSize = batchSize; + fastParams.numKvHeads = numKvHeads; + fastParams.bytesPerKvHalf = static_cast<size_t>(numKvHeads) * tokensPerBlock * headDim * sizeof(T); + fastParams.bytesPerPage = 2 * fastParams.bytesPerKvHalf; + if (headDim == 64 && tokensPerBlock == 32) + { + launchSparseKvCacheCompactV2Bf16Pipeline<T, 64, 32>(fastParams, stream); + } + else if (headDim == 64 && tokensPerBlock == 128) + { + launchSparseKvCacheCompactV2Bf16Pipeline<T, 64, 128>(fastParams, stream); + } + else if (headDim == 128 && tokensPerBlock == 32) + { + launchSparseKvCacheCompactV2Bf16Pipeline<T, 128, 32>(fastParams, stream); + } + else + { + launchSparseKvCacheCompactV2Bf16Pipeline<T, 128, 128>(fastParams, stream); + } + return; + } + } +#endif // ENABLE_BF16 + + TLLM_CHECK_WITH_INFO(false, + "Sparse KV compaction ships only the pipelined bf16 kernels (head size 64/128, page size 32/128 " + "tokens); got element size %zu, head size %d, %d tokens per page", + sizeof(T), headDim, tokensPerBlock); +} + //////////////////////////////////////////////////////////////////////////////////////////////////// #define INSTANTIATE_ATTENTION_INPUT_PROCESSING(T, TCache, KVCacheBuffer) \ @@ -1891,6 +2154,11 @@ void invokeUpdateSparseKvCacheAfterFmha(QKVPreprocessingParams<T, KVCacheBuffer> QKVPreprocessingParams<T, KVCacheBuffer> params, cudaStream_t stream); \ //////////////////////////////////////////////////////////////////////////////////////////////////// +#define INSTANTIATE_SPARSE_KV_CACHE_COMPACT_LAYERS(T) \ + template void invokeSparseKvCacheCompactLayers<T>(int64_t const*, int32_t const*, int32_t, int64_t, \ + int32_t const*, int32_t const*, int64_t, int64_t, int32_t const*, int32_t const*, int32_t, int32_t, int32_t, \ + int32_t, cudaStream_t); + } // namespace kernels TRTLLM_NAMESPACE_END diff --git a/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.cpp b/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.cpp index 3e19f9ebe72a..d219572d2707 100644 --- a/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.cpp +++ b/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.cpp @@ -16,6 +16,7 @@ #include "ub_interface.h" #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaDriverWrapper.h" +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_runtime.h> #include <cuda_runtime_api.h> @@ -80,13 +81,13 @@ namespace kernels::ub { void allreduce2_userbuff_inplace_launcher(int const handler, size_t const offset, size_t const elements, - nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) + tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { allreduce2_userbuff_inplace_impl(handler, offset, elements, dataType, comm, stream); } int allgather2_userbuff_residual_launcher(int const handler, size_t const offset, size_t const elements, - int const hidden_size, void* residual, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream, + int const hidden_size, void* residual, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream, bool force_enable) { return allgather2_userbuff_residual_impl( @@ -95,7 +96,7 @@ int allgather2_userbuff_residual_launcher(int const handler, size_t const offset int allreduce2_userbuff_rmsnorm_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) + void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { return allreduce2_userbuff_rmsnorm_impl(handler, offset, out_handler, out_offset, elements, hidden_size, beta, gamma, eps, residual_in, residual_out, dataType, comm, stream); @@ -103,7 +104,7 @@ int allreduce2_userbuff_rmsnorm_launcher(int const handler, size_t const offset, int allreduce2_userbuff_inplace_rmsnorm_quant_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - float* scalefactor, void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, + float* scalefactor, void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { return allreduce2_userbuff_inplace_rmsnorm_quant_impl(handler, offset, out_handler, out_offset, elements, @@ -113,7 +114,7 @@ int allreduce2_userbuff_inplace_rmsnorm_quant_launcher(int const handler, size_t int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, int const scale_handler, size_t const scale_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, float* scalefactor, - void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) + void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { return allreduce2_userbuff_inplace_rmsnorm_quant_fp4_impl(handler, offset, out_handler, out_offset, scale_handler, scale_offset, elements, hidden_size, beta, gamma, eps, scalefactor, residual_in, residual_out, dataType, comm, @@ -165,12 +166,12 @@ TRTLLM_NAMESPACE_BEGIN namespace kernels::ub { void allreduce2_userbuff_inplace_launcher(int const handler, size_t const offset, size_t const elements, - nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) + tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { } int allgather2_userbuff_residual_launcher(int const handler, size_t const offset, size_t const elements, - int const hidden_size, void* residual, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream, + int const hidden_size, void* residual, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream, bool force_enable) { return 0; @@ -178,7 +179,7 @@ int allgather2_userbuff_residual_launcher(int const handler, size_t const offset int allreduce2_userbuff_inplace_rmsnorm_quant_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - float* scalefactor, void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, + float* scalefactor, void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { return 0; @@ -187,7 +188,7 @@ int allreduce2_userbuff_inplace_rmsnorm_quant_launcher(int const handler, size_t int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, int const scale_handler, size_t const scale_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, float* scalefactor, - void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) + void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { return 0; } diff --git a/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.h b/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.h index e8a48e2c680b..dc68154fb462 100644 --- a/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.h +++ b/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.h @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/dataType.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "ub_allocator.h" namespace tensorrt_llm::runtime::ub @@ -40,24 +41,24 @@ namespace kernels::ub using ::tensorrt_llm::runtime::ub::communicator; void allreduce2_userbuff_inplace_launcher(int const handler, size_t const offset, size_t const elements, - nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream = 0); + tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream = 0); int allgather2_userbuff_residual_launcher(int const handler, size_t const offset, size_t const elements, - int const hidden_size, void* residual, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream, + int const hidden_size, void* residual, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream, bool force_enable = false); int allreduce2_userbuff_rmsnorm_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream); + void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream); int allreduce2_userbuff_inplace_rmsnorm_quant_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - float* scalefactor, void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, + float* scalefactor, void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream); int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, int const scale_handler, size_t const scale_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, float* scalefactor, - void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream); + void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream); } // namespace kernels::ub TRTLLM_NAMESPACE_END diff --git a/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.cu b/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.cu index 8cb5814e0398..a19059e9010c 100644 --- a/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.cu +++ b/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.cu @@ -15,6 +15,7 @@ */ #include "tensorrt_llm/common/config.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/quantization.cuh" #include "userbuffers.h" #include "utils.h" @@ -1774,11 +1775,11 @@ int allgather2_userbuff_residual(int const handler, size_t const offset, size_t } void allreduce2_userbuff_inplace_impl(int const handler, size_t const offset, size_t const elements, - nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) + tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { switch (dataType) { - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1791,7 +1792,7 @@ void allreduce2_userbuff_inplace_impl(int const handler, size_t const offset, si break; } #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1809,17 +1810,17 @@ void allreduce2_userbuff_inplace_impl(int const handler, size_t const offset, si } int allgather2_userbuff_residual_impl(int const handler, size_t const offset, size_t const elements, - int const hidden_size, void* residual, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream, + int const hidden_size, void* residual, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream, bool force_enable) { switch (dataType) { - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: return allgather2_userbuff_residual<half>( handler, offset, elements, hidden_size, residual, comm, stream, force_enable); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: return allgather2_userbuff_residual<__nv_bfloat16>( handler, offset, elements, hidden_size, residual, comm, stream, force_enable); break; @@ -1830,11 +1831,11 @@ int allgather2_userbuff_residual_impl(int const handler, size_t const offset, si int allreduce2_userbuff_rmsnorm_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) + void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { switch (dataType) { - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1849,7 +1850,7 @@ int allreduce2_userbuff_rmsnorm_impl(int const handler, size_t const offset, int break; } #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1870,12 +1871,12 @@ int allreduce2_userbuff_rmsnorm_impl(int const handler, size_t const offset, int int allreduce2_userbuff_inplace_rmsnorm_quant_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - float* scalefactor, void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, + float* scalefactor, void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { switch (dataType) { - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1890,7 +1891,7 @@ int allreduce2_userbuff_inplace_rmsnorm_quant_impl(int const handler, size_t con break; } #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1914,11 +1915,11 @@ int allreduce2_userbuff_inplace_rmsnorm_quant_impl(int const handler, size_t con int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, int const scale_handler, size_t const scale_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, float* scalefactor, void* residual_in, - void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) + void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) { switch (dataType) { - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1935,7 +1936,7 @@ int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_impl(int const handler, size_t break; } #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: { if (kDISABLE_FP32_ACCUMULATION) { diff --git a/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.h b/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.h index 96f21b748282..5d3ffe0cc950 100644 --- a/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.h +++ b/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.h @@ -14,6 +14,7 @@ * limitations under the License. */ #pragma once +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/worldConfig.h" @@ -120,25 +121,25 @@ namespace kernels::ub { using namespace ::tensorrt_llm::runtime::ub; void allreduce2_userbuff_inplace_impl(int const handler, size_t const offset, size_t const elements, - nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream = 0); + tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream = 0); // for TP-parallelism, only single node is implemented int allgather2_userbuff_residual_impl(int const handler, size_t const offset, size_t const elements, - int const hidden_size, void* residual, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream, + int const hidden_size, void* residual, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream, bool force_enable); int allreduce2_userbuff_rmsnorm_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream); + void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream); int allreduce2_userbuff_inplace_rmsnorm_quant_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - float* scalefactor, void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, + float* scalefactor, void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream); int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, int const scale_handler, size_t const scale_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, float* scalefactor, void* residual_in, - void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream); + void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream); } // namespace kernels::ub TRTLLM_NAMESPACE_END diff --git a/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemm.h b/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemm.h index eb939b57c2db..c2bf35175391 100644 --- a/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemm.h +++ b/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemm.h @@ -24,8 +24,6 @@ #include "tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h" #include "tensorrt_llm/runtime/common.h" -#include <NvInferRuntime.h> - #include <cassert> #include <cmath> #include <cstdint> diff --git a/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemmNVFP4.h b/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemmNVFP4.h index 616f9d25c2bf..6e901846ed25 100644 --- a/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemmNVFP4.h +++ b/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemmNVFP4.h @@ -24,8 +24,6 @@ #include "tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h" #include "tensorrt_llm/runtime/common.h" -#include <NvInferRuntime.h> - #include <cassert> #include <cmath> #include <cstdint> diff --git a/cpp/tensorrt_llm/kernels/xqaDispatcher.cpp b/cpp/tensorrt_llm/kernels/xqaDispatcher.cpp index 35fd02e7f127..8a5eeee91c6b 100644 --- a/cpp/tensorrt_llm/kernels/xqaDispatcher.cpp +++ b/cpp/tensorrt_llm/kernels/xqaDispatcher.cpp @@ -534,6 +534,7 @@ void XqaDispatcher::runImpl( tllmRunnerParams.generalPackedCustoMaskPtr = params.spec_decoding_packed_mask; tllmRunnerParams.mPackedMaskMaxSeqLenQ = params.spec_decoding_max_generation_length; tllmRunnerParams.mSpecDecodingTargetMaxGenLen = mFixedParams.specDecodingTargetMaxGenLen; + tllmRunnerParams.mForcePrepareSpecDecTreeMask = params.force_prepare_spec_dec_tree_mask; tllmRunnerParams.customMaskPtr = params.spec_decoding_bl_tree_mask; tllmRunnerParams.customMaskOffsetsPtr = params.spec_decoding_bl_tree_mask_offset; tllmRunnerParams.firstSparseMaskOffsetsKvPtr = params.spec_bl_tree_first_sparse_mask_offset_kv; diff --git a/cpp/tensorrt_llm/layers/decodingParams.h b/cpp/tensorrt_llm/layers/decodingParams.h index 1e77b8919ca1..76c5cedd637b 100644 --- a/cpp/tensorrt_llm/layers/decodingParams.h +++ b/cpp/tensorrt_llm/layers/decodingParams.h @@ -17,6 +17,7 @@ #pragma once #include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/kernels/beamSearchKernels.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -192,9 +193,9 @@ class ExplicitDraftTokensSetupParams : public DecodingSetupParams public: OptVec<float> temperature; // [setupBatchSize] // Hack to init some data for the context phase in the setup. - TensorPtr randomDataSample; // [maxBatchSize], on gpu - TensorPtr temperatures; // [maxBatchSize], on gpu - nvinfer1::DataType dtype; // [1] + TensorPtr randomDataSample; // [maxBatchSize], on gpu + TensorPtr temperatures; // [maxBatchSize], on gpu + tensorrt_llm::DataType dtype; // [1] }; class EagleSetupParams : public DecodingSetupParams @@ -202,9 +203,9 @@ class EagleSetupParams : public DecodingSetupParams public: OptVec<float> temperature; // [setupBatchSize] // Hack to init some data for the context phase in the setup. - TensorPtr randomDataSample; // [maxBatchSize], on gpu - TensorPtr temperatures; // [maxBatchSize], on gpu - nvinfer1::DataType dtype; // [1] + TensorPtr randomDataSample; // [maxBatchSize], on gpu + TensorPtr temperatures; // [maxBatchSize], on gpu + tensorrt_llm::DataType dtype; // [1] }; class DynamicDecodeSetupParams : public BaseSetupParams diff --git a/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.cpp b/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.cpp index e014ee4535e5..aedeb731574f 100644 --- a/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.cpp +++ b/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.cpp @@ -16,6 +16,7 @@ #include "explicitDraftTokensLayer.h" #include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/penaltyTypes.h" #include "tensorrt_llm/kernels/speculativeDecoding/common.h" #include "tensorrt_llm/kernels/speculativeDecoding/explicitDraftTokensKernels.h" @@ -93,15 +94,15 @@ void ExplicitDraftTokensLayer<T>::setup(SizeType32 batchSize, SizeType32 beamWid batchSlots, getLimitsPenalty(DecodingPenaltyType::Temperature), "temperature penalty"); // Dispatch context buffer fill - if (mDecoderDtype == nvinfer1::DataType::kFLOAT) + if (mDecoderDtype == tensorrt_llm::DataType::kFLOAT) { fillContextBuffers<float>(batchSize, batchSlots, *setupParams, workspace); } - else if (mDecoderDtype == nvinfer1::DataType::kHALF) + else if (mDecoderDtype == tensorrt_llm::DataType::kHALF) { fillContextBuffers<half>(batchSize, batchSlots, *setupParams, workspace); } - else if (mDecoderDtype == nvinfer1::DataType::kBF16) + else if (mDecoderDtype == tensorrt_llm::DataType::kBF16) { fillContextBuffers<__nv_bfloat16>(batchSize, batchSlots, *setupParams, workspace); } @@ -126,15 +127,15 @@ void ExplicitDraftTokensLayer<T>::forwardAsync(std::shared_ptr<BaseDecodingOutpu convertPackedMask(*outputs, *inputs, workspace); // Slice output ids, pos ids, next draft tokens. - if (mDecoderDtype == nvinfer1::DataType::kFLOAT) + if (mDecoderDtype == tensorrt_llm::DataType::kFLOAT) { splitInputDataToBatchSlots<float>(*outputs, *inputs, workspace); } - else if (mDecoderDtype == nvinfer1::DataType::kHALF) + else if (mDecoderDtype == tensorrt_llm::DataType::kHALF) { splitInputDataToBatchSlots<half>(*outputs, *inputs, workspace); } - else if (mDecoderDtype == nvinfer1::DataType::kBF16) + else if (mDecoderDtype == tensorrt_llm::DataType::kBF16) { splitInputDataToBatchSlots<__nv_bfloat16>(*outputs, *inputs, workspace); } diff --git a/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.h b/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.h index 75883ded6e5a..17fca4513cf1 100644 --- a/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.h +++ b/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.h @@ -16,6 +16,7 @@ #pragma once +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/layers/baseLayer.h" #include "tensorrt_llm/layers/decodingParams.h" #include "tensorrt_llm/runtime/common.h" @@ -83,7 +84,7 @@ class ExplicitDraftTokensLayer : public BaseLayer TensorPtr mTemperature; - std::optional<nvinfer1::DataType> mDecoderDtype{std::nullopt}; + std::optional<tensorrt_llm::DataType> mDecoderDtype{std::nullopt}; }; } // namespace tensorrt_llm::layers diff --git a/cpp/tensorrt_llm/layers/lookaheadAlgorithm.cpp b/cpp/tensorrt_llm/layers/lookaheadAlgorithm.cpp index 09843fd7ce44..76da89dfec0d 100644 --- a/cpp/tensorrt_llm/layers/lookaheadAlgorithm.cpp +++ b/cpp/tensorrt_llm/layers/lookaheadAlgorithm.cpp @@ -17,6 +17,7 @@ #include "tensorrt_llm/layers/lookaheadAlgorithm.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" #include "tensorrt_llm/runtime/common.h" @@ -35,14 +36,14 @@ LookaheadAlgorithm::LookaheadAlgorithm( runtime::SizeType32 maxW, runtime::SizeType32 maxN, runtime::SizeType32 maxG, runtime::SizeType32 id) : mPoolManager(maxG) , mPrefillsMax(runtime::BufferManager::cpu( - runtime::ITensor::makeShape({(maxN <= 1 ? 0 : maxN - 2)}), nvinfer1::DataType::kINT32)) + runtime::ITensor::makeShape({(maxN <= 1 ? 0 : maxN - 2)}), tensorrt_llm::DataType::kINT32)) , mPastTokensMax( - runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxW * (maxN - 1)}), nvinfer1::DataType::kINT32)) - , mKeyTokensMax(runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxW}), nvinfer1::DataType::kINT32)) + runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxW * (maxN - 1)}), tensorrt_llm::DataType::kINT32)) + , mKeyTokensMax(runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxW}), tensorrt_llm::DataType::kINT32)) , mGoldenTokensMax( - runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxN * 2 - 1}), nvinfer1::DataType::kINT32)) + runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxN * 2 - 1}), tensorrt_llm::DataType::kINT32)) , mGuessTokensMax( - runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxG * (maxN - 1)}), nvinfer1::DataType::kINT32)) + runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxG * (maxN - 1)}), tensorrt_llm::DataType::kINT32)) , mMaxW(maxW) , mMaxN(maxN) , mMaxG(maxG) @@ -52,12 +53,13 @@ LookaheadAlgorithm::LookaheadAlgorithm( std::tie(maxGeneratedLen, std::ignore, maxDraftLen, std::ignore) = executor::LookaheadDecodingConfig(maxW, maxN, maxG).calculateSpeculativeResource(); mAttentionMask = runtime::BufferManager::cpu( - runtime::ITensor::makeShape({maxDraftLen, maxDraftLen}), nvinfer1::DataType::kBOOL); + runtime::ITensor::makeShape({maxDraftLen, maxDraftLen}), tensorrt_llm::DataType::kBOOL); mDraftTokensMax - = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxDraftLen}), nvinfer1::DataType::kINT32); + = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxDraftLen}), tensorrt_llm::DataType::kINT32); mSampledTokensMax - = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxGeneratedLen}), nvinfer1::DataType::kINT32); - mEncodeMapMax = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxDraftLen}), nvinfer1::DataType::kINT32); + = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxGeneratedLen}), tensorrt_llm::DataType::kINT32); + mEncodeMapMax + = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxDraftLen}), tensorrt_llm::DataType::kINT32); } void LookaheadAlgorithm::setup(TensorConstPtr const& prompt, SizeType32 w, SizeType32 n, SizeType32 g, uint64_t seed) diff --git a/cpp/tensorrt_llm/layers/lookaheadDecodingLayer.cpp b/cpp/tensorrt_llm/layers/lookaheadDecodingLayer.cpp index bf6e15080f3c..986f0e0b978e 100644 --- a/cpp/tensorrt_llm/layers/lookaheadDecodingLayer.cpp +++ b/cpp/tensorrt_llm/layers/lookaheadDecodingLayer.cpp @@ -19,6 +19,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/kernels/samplingTopKKernels.h" #include "tensorrt_llm/layers/decodingParams.h" @@ -64,38 +65,42 @@ LookaheadDecodingLayer<T>::CpuAlgorithmResources::CpuAlgorithmResources(DecoderD mPrompts.reserve(maxBatchSize); for (auto bi = 0; bi < maxBatchSize; bi++) { - mPrompts.emplace_back(BufferManager::cpu(ITensor::makeShape({0}), nvinfer1::DataType::kINT32)); + mPrompts.emplace_back(BufferManager::cpu(ITensor::makeShape({0}), tensorrt_llm::DataType::kINT32)); } auto const maxBatchShape1D = ITensor::makeShape({maxBatchSize}); - mBatchSlots = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); mTargetTokens - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), nvinfer1::DataType::kINT32); - mTokensPerStep = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); - mEndIds = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), tensorrt_llm::DataType::kINT32); + mTokensPerStep = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + mEndIds = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - mOutputIds = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxNumNewTokens}), nvinfer1::DataType::kINT32); + mOutputIds + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxNumNewTokens}), tensorrt_llm::DataType::kINT32); mNewTokens = BufferManager::cpu( - ITensor::makeShape({maxTokensPerStep, maxBatchSize, beamWidth}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxTokensPerStep, maxBatchSize, beamWidth}), tensorrt_llm::DataType::kINT32); mPathsOffsets - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), nvinfer1::DataType::kINT32); + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), tensorrt_llm::DataType::kINT32); mPathsOffsetsBatch - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), nvinfer1::DataType::kINT32); - mNumNewTokens = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); - mNumNewTokensCumSum = BufferManager::cpu(ITensor::makeShape({maxBatchSize + 1}), nvinfer1::DataType::kINT32); - mNextDraftTokens = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxDraftLen}), nvinfer1::DataType::kINT32); - mNextDraftPosIds = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxDraftLen}), nvinfer1::DataType::kINT32); - mGenerationLengths = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), tensorrt_llm::DataType::kINT32); + mNumNewTokens = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + mNumNewTokensCumSum = BufferManager::cpu(ITensor::makeShape({maxBatchSize + 1}), tensorrt_llm::DataType::kINT32); + mNextDraftTokens + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxDraftLen}), tensorrt_llm::DataType::kINT32); + mNextDraftPosIds + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxDraftLen}), tensorrt_llm::DataType::kINT32); + mGenerationLengths = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); mPositionOffsets - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), nvinfer1::DataType::kINT32); - mPositionIds = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), nvinfer1::DataType::kINT32); + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), tensorrt_llm::DataType::kINT32); + mPositionIds + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), tensorrt_llm::DataType::kINT32); mAttentionMask - = BufferManager::cpu(ITensor::makeShape({maxTokensPerStep, maxTokensPerStep}), nvinfer1::DataType::kBOOL); + = BufferManager::cpu(ITensor::makeShape({maxTokensPerStep, maxTokensPerStep}), tensorrt_llm::DataType::kBOOL); mPackedMask = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep, static_cast<ITensor::DimType64>(divUp(maxTokensPerStep, 32))}), - nvinfer1::DataType::kINT32); - mNextDraftLengths = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); - mSequenceLengths = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); + mNextDraftLengths = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + mSequenceLengths = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); } template <typename T> @@ -117,12 +122,12 @@ LookaheadDecodingLayer<T>::LookaheadDecodingLayer( auto const maxBatchShape2D = ITensor::makeShape({maxBatchSize, maxTokensPerStep}); mWorkspaceSize = getTopKWorkspaceSize<T>(maxBatchSize, maxTokensPerStep, maxTopK, vocabSizePadded); - mTargetTokensDevice = mBufferManager->gpu(maxBatchShape2D, nvinfer1::DataType::kINT32); + mTargetTokensDevice = mBufferManager->gpu(maxBatchShape2D, tensorrt_llm::DataType::kINT32); mCurandStatesDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), nvinfer1::DataType::kINT8); + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); mSetupWorkspaceSize = DecodingLayerWorkspace::calculateRequiredWorkspaceSize( - std::make_pair(maxBatchShape1D, nvinfer1::DataType::kINT64)); + std::make_pair(maxBatchShape1D, tensorrt_llm::DataType::kINT64)); TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); } diff --git a/cpp/tensorrt_llm/layers/lookaheadDecodingUtils.h b/cpp/tensorrt_llm/layers/lookaheadDecodingUtils.h index 739cf65001ab..8e3e8f6c590d 100644 --- a/cpp/tensorrt_llm/layers/lookaheadDecodingUtils.h +++ b/cpp/tensorrt_llm/layers/lookaheadDecodingUtils.h @@ -16,6 +16,7 @@ #pragma once +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -318,12 +319,12 @@ class DebugTensor { switch (mTensor.getDataType()) { - case nvinfer1::DataType::kBOOL: return values<bool>(); - case nvinfer1::DataType::kFLOAT: return values<float>(); - case nvinfer1::DataType::kINT8: return values<std::int8_t>(); - case nvinfer1::DataType::kINT32: return values<std::int32_t>(); - case nvinfer1::DataType::kINT64: return values<std::int64_t>(); - case nvinfer1::DataType::kUINT8: return values<std::uint8_t>(); + case tensorrt_llm::DataType::kBOOL: return values<bool>(); + case tensorrt_llm::DataType::kFLOAT: return values<float>(); + case tensorrt_llm::DataType::kINT8: return values<std::int8_t>(); + case tensorrt_llm::DataType::kINT32: return values<std::int32_t>(); + case tensorrt_llm::DataType::kINT64: return values<std::int64_t>(); + case tensorrt_llm::DataType::kUINT8: return values<std::uint8_t>(); default: return std::string(mName + ": Unsupported data type"); } } @@ -376,12 +377,12 @@ class DebugTensor { switch (mTensor.getDataType()) { - case nvinfer1::DataType::kBOOL: return randomize<bool>(3); - case nvinfer1::DataType::kFLOAT: return randomize<float>(3); - case nvinfer1::DataType::kINT8: return randomize<std::int8_t>(3); - case nvinfer1::DataType::kINT32: return randomize<std::int32_t>(3); - case nvinfer1::DataType::kINT64: return randomize<std::int64_t>(3); - case nvinfer1::DataType::kUINT8: return randomize<std::uint8_t>(3); + case tensorrt_llm::DataType::kBOOL: return randomize<bool>(3); + case tensorrt_llm::DataType::kFLOAT: return randomize<float>(3); + case tensorrt_llm::DataType::kINT8: return randomize<std::int8_t>(3); + case tensorrt_llm::DataType::kINT32: return randomize<std::int32_t>(3); + case tensorrt_llm::DataType::kINT64: return randomize<std::int64_t>(3); + case tensorrt_llm::DataType::kUINT8: return randomize<std::uint8_t>(3); default: return; } } @@ -391,12 +392,12 @@ class DebugTensor { switch (mTensor.getDataType()) { - case nvinfer1::DataType::kBOOL: return randomize<bool>(0); - case nvinfer1::DataType::kFLOAT: return randomize<float>(0); - case nvinfer1::DataType::kINT8: return randomize<std::int8_t>(0); - case nvinfer1::DataType::kINT32: return randomize<std::int32_t>(0); - case nvinfer1::DataType::kINT64: return randomize<std::int64_t>(0); - case nvinfer1::DataType::kUINT8: return randomize<std::uint8_t>(0); + case tensorrt_llm::DataType::kBOOL: return randomize<bool>(0); + case tensorrt_llm::DataType::kFLOAT: return randomize<float>(0); + case tensorrt_llm::DataType::kINT8: return randomize<std::int8_t>(0); + case tensorrt_llm::DataType::kINT32: return randomize<std::int32_t>(0); + case tensorrt_llm::DataType::kINT64: return randomize<std::int64_t>(0); + case tensorrt_llm::DataType::kUINT8: return randomize<std::uint8_t>(0); default: return; } } @@ -405,12 +406,12 @@ class DebugTensor { switch (mTensor.getDataType()) { - case nvinfer1::DataType::kBOOL: return randomize<bool>(1); - case nvinfer1::DataType::kFLOAT: return randomize<float>(1); - case nvinfer1::DataType::kINT8: return randomize<std::int8_t>(1); - case nvinfer1::DataType::kINT32: return randomize<std::int32_t>(1); - case nvinfer1::DataType::kINT64: return randomize<std::int64_t>(1); - case nvinfer1::DataType::kUINT8: return randomize<std::uint8_t>(1); + case tensorrt_llm::DataType::kBOOL: return randomize<bool>(1); + case tensorrt_llm::DataType::kFLOAT: return randomize<float>(1); + case tensorrt_llm::DataType::kINT8: return randomize<std::int8_t>(1); + case tensorrt_llm::DataType::kINT32: return randomize<std::int32_t>(1); + case tensorrt_llm::DataType::kINT64: return randomize<std::int64_t>(1); + case tensorrt_llm::DataType::kUINT8: return randomize<std::uint8_t>(1); default: return; } } diff --git a/cpp/tensorrt_llm/layers/lookaheadPoolManager.cpp b/cpp/tensorrt_llm/layers/lookaheadPoolManager.cpp index 5954bc520ad0..397b4262226a 100644 --- a/cpp/tensorrt_llm/layers/lookaheadPoolManager.cpp +++ b/cpp/tensorrt_llm/layers/lookaheadPoolManager.cpp @@ -16,6 +16,7 @@ #include "tensorrt_llm/layers/lookaheadPoolManager.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" #include <cstddef> @@ -67,7 +68,7 @@ void LookaheadPoolManager::accept(TensorConstPtr const& prompt, SizeType32 level for (SizeType32 ti = 0; ti + level - 1 < length; ti++) { auto key = promptRange[ti]; - TensorPtr ngram = BufferManager::cpu(ITensor::makeShape({level - 1}), nvinfer1::DataType::kINT32); + TensorPtr ngram = BufferManager::cpu(ITensor::makeShape({level - 1}), tensorrt_llm::DataType::kINT32); BufferRange<TokenIdType const> sourceRange(*ITensor::slice(prompt, ti + 1, level - 1)); BufferRange<TokenIdType> ngramRange(*ngram); std::copy(sourceRange.begin(), sourceRange.end(), ngramRange.begin()); @@ -107,7 +108,7 @@ void LookaheadPoolManager::update(TensorConstPtr const& keyTokens, TensorConstPt for (SizeType32 wi = 0; wi < window; wi++) { TensorConstPtr source = ITensor::at(ngramTokens, {wi}); - TensorPtr ngram = BufferManager::cpu(source->getShape(), nvinfer1::DataType::kINT32); + TensorPtr ngram = BufferManager::cpu(source->getShape(), tensorrt_llm::DataType::kINT32); BufferRange<TokenIdType const> sourceRange(*source); BufferRange<TokenIdType> ngramRange(*ngram); std::copy(sourceRange.begin(), sourceRange.end(), ngramRange.begin()); diff --git a/cpp/tensorrt_llm/layers/medusaDecodingLayer.cpp b/cpp/tensorrt_llm/layers/medusaDecodingLayer.cpp index 40eff62c17d6..9e4098b34ebf 100644 --- a/cpp/tensorrt_llm/layers/medusaDecodingLayer.cpp +++ b/cpp/tensorrt_llm/layers/medusaDecodingLayer.cpp @@ -16,6 +16,7 @@ #include "medusaDecodingLayer.h" #include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/kernels/samplingTopKKernels.h" #include "tensorrt_llm/kernels/speculativeDecoding/medusaDecodingKernels.h" @@ -88,10 +89,10 @@ void MedusaDecodingLayer<T>::allocateBuffer() mTiledBatchSlotsSetup = BufferManager::pinnedPool( ITensor::makeShape({static_cast<SizeType32>(mDecoderDomain.getBatchSize() * maxDraftPathLen)}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mTiledBatchSlotsForward = BufferManager::pinnedPool( ITensor::makeShape({static_cast<SizeType32>(mDecoderDomain.getBatchSize() * maxDraftPathLen)}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mMedusaInputLogitsPtrs = BufferManager::pinnedPool( ITensor::makeShape({static_cast<SizeType32>(mDecoderDomain.getBatchSize() * maxDraftPathLen)}), TRTDataType<T*>::value); diff --git a/cpp/tensorrt_llm/layers/penaltyLayer.cpp b/cpp/tensorrt_llm/layers/penaltyLayer.cpp index c6c57ca5034d..c72b8e463bc6 100644 --- a/cpp/tensorrt_llm/layers/penaltyLayer.cpp +++ b/cpp/tensorrt_llm/layers/penaltyLayer.cpp @@ -18,6 +18,7 @@ #include "penaltyLayer.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/penaltyKernels.h" #include "tensorrt_llm/kernels/penaltyTypes.h" #include "tensorrt_llm/layers/defaultDecodingParams.h" @@ -84,11 +85,11 @@ void PenaltyLayer<T>::allocateWorkspace() auto const workspaceSize = mDecoderDomain.getBatchSize() * mDecoderDomain.getMaxDecodingTokens() * mConfiguredBeamWidth * mDecoderDomain.getVocabSize() * 2; - mPenaltyWorkspaceDevice = mBufferManager->gpu(workspaceSize, nvinfer1::DataType::kINT32); + mPenaltyWorkspaceDevice = mBufferManager->gpu(workspaceSize, tensorrt_llm::DataType::kINT32); if (mDecodingMode.isBeamSearch()) { - mPenaltyWorkspacePrevDevice = mBufferManager->gpu(workspaceSize, nvinfer1::DataType::kINT32); + mPenaltyWorkspacePrevDevice = mBufferManager->gpu(workspaceSize, tensorrt_llm::DataType::kINT32); } } @@ -111,27 +112,27 @@ void PenaltyLayer<T>::allocateBuffer() if (mDecodingMode.isUseTemperature()) { - mTemperatureDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kFLOAT); + mTemperatureDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kFLOAT); } if (mDecodingMode.isUseRepetitionPenalty()) { - mRepetitionPenaltyDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kFLOAT); + mRepetitionPenaltyDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kFLOAT); } if (mDecodingMode.isUsePresencePenalty()) { - mPresencePenaltyDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kFLOAT); + mPresencePenaltyDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kFLOAT); } if (mDecodingMode.isUseFrequencyPenalty()) { - mFrequencyPenaltyDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kFLOAT); + mFrequencyPenaltyDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kFLOAT); } if (mDecodingMode.isUseMinLength()) { - mMinLengthDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kINT32); + mMinLengthDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kINT32); } if (mDecodingMode.isUseOccurrencePenalty()) { - mPromptIgnoreLengthDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kINT32); + mPromptIgnoreLengthDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kINT32); } auto const logitsPtrDeviceDesc = std::make_pair(batchSizeShape, TRTDataType<T*>::value); diff --git a/cpp/tensorrt_llm/nanobind/CMakeLists.txt b/cpp/tensorrt_llm/nanobind/CMakeLists.txt index b523ae193871..5dc1de84a308 100755 --- a/cpp/tensorrt_llm/nanobind/CMakeLists.txt +++ b/cpp/tensorrt_llm/nanobind/CMakeLists.txt @@ -11,10 +11,10 @@ set(SRCS batch_manager/kvCacheConnector.cpp batch_manager/kvCacheManager.cpp batch_manager/kvCacheManagerV2Utils.cpp + batch_manager/kvCacheManagerV2.cpp batch_manager/llmRequest.cpp common/tllmExceptions.cpp executor/bindings.cpp - executor/executor.cpp executor/executorConfig.cpp executor/request.cpp process_group/bindings.cpp @@ -23,7 +23,6 @@ set(SRCS runtime/moeBindings.cpp suffixAutomaton/bindings.cpp testing/kvCacheManagerTestUtilBinding.cpp - testing/modelSpecBinding.cpp userbuffers/bindings.cpp thop/bindings.cpp ../runtime/ipcNvlsMemory.cu @@ -31,6 +30,15 @@ set(SRCS include_directories(${PROJECT_SOURCE_DIR}/include) +# KV Cache Manager V2 include directory (global, needed during compilation of +# SRCS) NOTE: PROJECT_SOURCE_DIR = cpp/ (build_wheel.py passes cpp/ as -S), so +# batch_manager is one level down and the vendored SHA-256 is at +# tensorrt_llm/common/sha256. +set(KV_CACHE_MANAGER_V2_INCLUDE + ${PROJECT_SOURCE_DIR}/tensorrt_llm/batch_manager) +include_directories(${KV_CACHE_MANAGER_V2_INCLUDE} + ${PROJECT_SOURCE_DIR}/tensorrt_llm/common/sha256) + # NOSTRIP: Disable stripping for BOLT compatibility (requires --emit-relocs) if(ENABLE_BOLT_COMPATIBLE) nanobind_add_module(${TRTLLM_NB_MODULE} NOSTRIP ${SRCS}) @@ -38,6 +46,12 @@ else() nanobind_add_module(${TRTLLM_NB_MODULE} ${SRCS}) endif() +# SHA-256 symbols are provided by tensorrt_llm_batch_manager_static (via the +# shared lib). Only add the include directory so headers resolve during +# compilation of kvCacheManagerV2.cpp. +target_include_directories( + ${TRTLLM_NB_MODULE} PRIVATE ${PROJECT_SOURCE_DIR}/tensorrt_llm/common/sha256) + set_property(TARGET ${TRTLLM_NB_MODULE} PROPERTY POSITION_INDEPENDENT_CODE ON) target_link_directories(${TRTLLM_NB_MODULE} PUBLIC diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/algorithms.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/algorithms.cpp index 4070811b2d72..c13466565342 100644 --- a/cpp/tensorrt_llm/nanobind/batch_manager/algorithms.cpp +++ b/cpp/tensorrt_llm/nanobind/batch_manager/algorithms.cpp @@ -23,11 +23,11 @@ #include "tensorrt_llm/batch_manager/createNewDecoderRequests.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/batch_manager/logitsPostProcessor.h" #include "tensorrt_llm/batch_manager/medusaBuffers.h" #include "tensorrt_llm/batch_manager/microBatchScheduler.h" #include "tensorrt_llm/batch_manager/pauseRequests.h" #include "tensorrt_llm/batch_manager/peftCacheManager.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/nanobind/common/customCasters.h" #include "tensorrt_llm/runtime/decoderState.h" #include "tensorrt_llm/runtime/torch.h" @@ -129,13 +129,6 @@ void tensorrt_llm::nanobind::batch_manager::algorithms::initBindings(nb::module_ nb::call_guard<nb::gil_scoped_release>()) .def("name", [](AllocateKvCache const&) { return AllocateKvCache::name; }); - nb::class_<LogitsPostProcessor>(m, LogitsPostProcessor::name) - .def(nb::init<>()) - .def("__call__", &LogitsPostProcessor::operator(), nb::arg("decoder_input_buffers"), - nb::arg("replicate_logits_post_processor"), nb::arg("world_config"), nb::arg("stream"), - nb::arg("logits_post_processor_batched") = std::nullopt) - .def("name", [](LogitsPostProcessor const&) { return LogitsPostProcessor::name; }); - nb::class_<CreateNewDecoderRequests>(m, CreateNewDecoderRequests::name) .def(nb::init<bool, bool, bool>(), nb::arg("speculative_decoding_fast_logits"), nb::arg("is_leader_in_orch_mode"), nb::arg("is_normalize_log_probs")) @@ -143,7 +136,7 @@ void tensorrt_llm::nanobind::batch_manager::algorithms::initBindings(nb::module_ "__call__", [](CreateNewDecoderRequests& self, tr::ModelConfig const& modelConfig, tr::WorldConfig const& worldConfig, executor::DecodingConfig const& decodingConfig, RequestVector const& contextRequests, - nvinfer1::DataType logitsType, DecoderInputBuffers& inputBuffers, + tensorrt_llm::DataType logitsType, DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState, tensorrt_llm::runtime::CudaStream const& runtimeStream, tensorrt_llm::runtime::CudaStream const& decoderStream, SizeType32 maxSequenceLength, SizeType32 beamWidth) diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp index 0846663dafad..f1c57ec064ac 100644 --- a/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp @@ -24,6 +24,7 @@ #include "tensorrt_llm/batch_manager/peftCacheManager.h" #include "tensorrt_llm/batch_manager/rnnStateManager.h" #include "tensorrt_llm/batch_manager/sequenceSlotManager.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/nanobind/common/bindTypes.h" #include "tensorrt_llm/runtime/gptDecoderBatched.h" #include "tensorrt_llm/runtime/runtimeKernels.h" @@ -100,6 +101,34 @@ void initBindings(nb::module_& m) .def("get_token", &GenLlmReq::getToken, nb::arg("beam"), nb::arg("pos")) .def("get_tokens", nb::overload_cast<GenLlmReq::SizeType32>(&GenLlmReq::getTokens, nb::const_), nb::arg("beam")) .def("get_tokens", nb::overload_cast<>(&GenLlmReq::getTokens, nb::const_)) + // Copies only [begin, end) -> O(end-begin), vs get_tokens(beam) which + // marshals the whole O(seq_len) VecTokens into a Python list. + .def( + "get_tokens_range", + [](GenLlmReq const& self, GenLlmReq::SizeType32 beam, GenLlmReq::SizeType32 begin, + GenLlmReq::SizeType32 end) + { + auto const& tokens = self.getTokens(beam); + auto const n = static_cast<GenLlmReq::SizeType32>(tokens.size()); + if (begin < 0) + { + begin = 0; + } + if (begin > n) + { + begin = n; + } + if (end < begin) + { + end = begin; + } + if (end > n) + { + end = n; + } + return GenLlmReq::VecTokens(tokens.begin() + begin, tokens.begin() + end); + }, + nb::arg("beam"), nb::arg("begin"), nb::arg("end")) .def("get_last_tokens", nb::overload_cast<GenLlmReq::SizeType32>(&GenLlmReq::getLastTokens), nb::arg("beam")) .def("get_last_tokens", nb::overload_cast<>(&GenLlmReq::getLastTokens)) .def("get_beam_width_by_iter", &GenLlmReq::getBeamWidthByIter, nb::arg("for_next_iteration") = false) @@ -173,6 +202,8 @@ void initBindings(nb::module_& m) nb::arg("kv_tokens_per_block")) .def_prop_rw( "estimated_reusable_tokens", &GenLlmReq::getEstimatedReusableTokens, &GenLlmReq::setEstimatedReusableTokens) + .def_prop_rw( + "expect_snapshot_points", &GenLlmReq::getExpectedSnapshotPoints, &GenLlmReq::setExpectedSnapshotPoints) .def_prop_rw("guided_decoding_params", &GenLlmReq::getGuidedDecodingParams, &GenLlmReq::setGuidedDecodingParams) .def_prop_rw("context_phase_params", &GenLlmReq::getContextPhaseParams, &GenLlmReq::setContextPhaseParams) .def_prop_ro("is_context_only_request", &GenLlmReq::isContextOnlyRequest) @@ -193,6 +224,8 @@ void initBindings(nb::module_& m) .def_prop_ro("kv_cache_transfer_time_ms", &GenLlmReq::getKvCacheTransferTimeMS) .def_prop_ro("kv_cache_transfer_start", &GenLlmReq::getKvCacheTransferStart) .def_prop_ro("kv_cache_transfer_end", &GenLlmReq::getKvCacheTransferEnd) + .def("get_kv_cache_transfer_start", &GenLlmReq::getKvCacheTransferStart) + .def("get_kv_cache_transfer_end", &GenLlmReq::getKvCacheTransferEnd) .def_prop_ro("kv_cache_size", &GenLlmReq::getKvCacheSize) .def("set_kv_cache_transfer_start", &GenLlmReq::setKvCacheTransferStart, nb::arg("time")) .def("set_kv_cache_transfer_end", &GenLlmReq::setKvCacheTransferEnd, nb::arg("time")) @@ -476,7 +509,11 @@ void initBindings(nb::module_& m) .def("set_first_scheduled_time", &tb::LlmRequest::setFirstScheduledTime) .def("update_perf_metrics", &tb::LlmRequest::updatePerfMetrics, nb::arg("iter_counter")) .def("remove_lora_tensors", &tb::LlmRequest::removeLoraTensors) - .def_rw_static("global_steady_clock_offset", &tb::LlmRequest::sGlobalSteadyClockOffset); + // Bind to the single storage owned by libtensorrt_llm.so (reached through + // globalSteadyClockOffset()) instead of an inline-static member, so the + // offset is shared with the native library rather than living in this + // module's separate copy. + .def_rw_static("global_steady_clock_offset", &tb::globalSteadyClockOffset()); nb::class_<tb::SequenceSlotManager>(m, "SequenceSlotManager") .def(nb::init<tb::SequenceSlotManager::SlotIdType, uint64_t>(), nb::arg("max_num_slots"), @@ -491,7 +528,7 @@ void initBindings(nb::module_& m) nb::arg("max_num_sequences"), nb::arg("model_config"), nb::arg("world_config"), nb::arg("buffer_manager"), nb::call_guard<nb::gil_scoped_release>()) .def(nb::init<tr::SizeType32, tr::SizeType32, tr::SizeType32, tr::SizeType32, tr::SizeType32, tr::SizeType32, - tr::WorldConfig const&, int64_t, nvinfer1::DataType, nvinfer1::DataType, + tr::WorldConfig const&, int64_t, tensorrt_llm::DataType, tensorrt_llm::DataType, std::vector<tr::SizeType32> const&, tr::SizeType32>(), nb::arg("d_state"), nb::arg("d_conv"), nb::arg("num_heads"), nb::arg("n_groups"), nb::arg("head_dim"), nb::arg("max_batch_size"), nb::arg("world_config"), nb::arg("stream"), nb::arg("dtype"), diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/cacheTransceiver.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/cacheTransceiver.cpp index 70be9c2f353e..cc00ea0259c5 100644 --- a/cpp/tensorrt_llm/nanobind/batch_manager/cacheTransceiver.cpp +++ b/cpp/tensorrt_llm/nanobind/batch_manager/cacheTransceiver.cpp @@ -20,12 +20,14 @@ #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/rnnStateManager.h" #include "tensorrt_llm/common/bindingUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/nanobind/common/customCasters.h" #include <ATen/ATen.h> #include <nanobind/nanobind.h> #include <nanobind/stl/optional.h> #include <nanobind/stl/shared_ptr.h> +#include <nanobind/stl/string.h> #include <nanobind/stl/unique_ptr.h> #include <nanobind/stl/vector.h> #include <nanobind/trampoline.h> @@ -91,6 +93,12 @@ void tb::CacheTransceiverBindings::initBindings(nb::module_& m) .def("request_and_receive_sync", &BaseCacheTransceiver::requestAndReceiveSync, nb::call_guard<nb::gil_scoped_release>()) .def("request_and_receive_async", &BaseCacheTransceiver::requestAndReceiveAsync) + .def("get_serialized_data_transceiver_state", + [](tb::BaseCacheTransceiver& self) + { + auto serialized = self.getSerializedDataTransceiverState(); + return nb::bytes(serialized.data(), serialized.size()); + }) .def( "check_context_transfer_status", [](tb::BaseCacheTransceiver& self, std::optional<int> const& atLeastRequestNum, bool markComplete = false) @@ -120,13 +128,14 @@ void tb::CacheTransceiverBindings::initBindings(nb::module_& m) nb::class_<tb::CacheTransceiver, tb::BaseCacheTransceiver>(m, "CacheTransceiver") .def(nb::init<tb::kv_cache_manager::BaseKVCacheManager*, std::vector<SizeType32>, SizeType32, SizeType32, - runtime::WorldConfig, std::vector<SizeType32>, nvinfer1::DataType, + runtime::WorldConfig, std::vector<SizeType32>, tensorrt_llm::DataType, executor::kv_cache::CacheState::AttentionType, std::optional<executor::CacheTransceiverConfig>, std::vector<SizeType32>>(), nb::arg("cache_manager"), nb::arg("num_kv_heads_per_layer"), nb::arg("size_per_head"), nb::arg("tokens_per_block"), nb::arg("world_config"), nb::arg("attention_layer_num_per_pp"), nb::arg("dtype"), nb::arg("attention_type"), nb::arg("cache_transceiver_config") = std::nullopt, - nb::arg("rnn_layer_num_per_pp") = std::vector<SizeType32>{}); + nb::arg("rnn_layer_num_per_pp") = std::vector<SizeType32>{}) + .def("get_status_dump", &tb::CacheTransceiver::getStatusDump); nb::class_<tb::CacheTransceiverComm>(m, "CacheTransceiverComm") .def( diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp index 12c352e4a427..b1c4391c0e26 100644 --- a/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp +++ b/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp @@ -18,6 +18,7 @@ #include "kvCacheManager.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/peftCacheManager.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/nanobind/common/bindTypes.h" #include "tensorrt_llm/nanobind/common/customCasters.h" #include "tensorrt_llm/runtime/torch.h" @@ -348,8 +349,8 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) nb::class_<tbk::PoolConfiguration>(m, "PoolConfiguration") .def(nb::init<>()) - .def(nb::init<SizeType32, SizeType32, nvinfer1::DataType>(), nb::arg("window_size"), nb::arg("size_per_head"), - nb::arg("dtype")) + .def(nb::init<SizeType32, SizeType32, tensorrt_llm::DataType>(), nb::arg("window_size"), + nb::arg("size_per_head"), nb::arg("dtype")) .def_rw("window_size", &tbk::PoolConfiguration::windowSize) .def_rw("size_per_head", &tbk::PoolConfiguration::sizePerHead) .def_rw("dtype", &tbk::PoolConfiguration::dtype); @@ -661,8 +662,8 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) nb::class_<tbk::KVCacheManager, tbk::BaseKVCacheManager>(m, "KVCacheManager") .def(nb::init<std::vector<SizeType32> const&, SizeType32, SizeType32, std::map<SizeType32, std::tuple<SizeType32, SizeType32>> const&, SizeType32, SizeType32, - std::vector<SizeType32> const&, nvinfer1::DataType, SizeType32, int64_t, SizeType32, SizeType32, bool, - tbk::CacheType, std::optional<tensorrt_llm::executor::RetentionPriority>, + std::vector<SizeType32> const&, tensorrt_llm::DataType, SizeType32, int64_t, SizeType32, SizeType32, + bool, tbk::CacheType, std::optional<tensorrt_llm::executor::RetentionPriority>, std::shared_ptr<tbk::KVCacheEventManager>, bool, bool, std::shared_ptr<tbc::KvCacheConnectorManager>, bool, SizeType32, SizeType32, bool, std::optional<tbk::LinearAttentionMetadata>, std::vector<tbk::PoolConfiguration> const&>(), diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManagerV2.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManagerV2.cpp new file mode 100644 index 000000000000..7cda4942c104 --- /dev/null +++ b/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManagerV2.cpp @@ -0,0 +1,1903 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/nanobind/batch_manager/kvCacheManagerV2.h" + +#include "kv_cache_manager_v2/blockRadixTree.h" +#include "kv_cache_manager_v2/common.h" +#include "kv_cache_manager_v2/config.h" +#include "kv_cache_manager_v2/eventManager.h" +#include "kv_cache_manager_v2/exceptions.h" +#include "kv_cache_manager_v2/introspection.h" +#include "kv_cache_manager_v2/kvCache.h" +#include "kv_cache_manager_v2/kvCacheManager.h" +#include "kv_cache_manager_v2/lifeCycleRegistry.h" +#include "kv_cache_manager_v2/page.h" +#include "kv_cache_manager_v2/stats.h" +#include "kv_cache_manager_v2/storage/config.h" +#include "kv_cache_manager_v2/storage/core.h" + +#include <algorithm> +#include <cassert> +#include <cstring> +#include <memory> +#include <nanobind/nanobind.h> +#include <nanobind/ndarray.h> +#include <nanobind/stl/function.h> +#include <nanobind/stl/map.h> +#include <nanobind/stl/optional.h> +#include <nanobind/stl/pair.h> +#include <nanobind/stl/shared_ptr.h> +#include <nanobind/stl/string.h> +#include <nanobind/stl/tuple.h> +#include <nanobind/stl/unique_ptr.h> +#include <nanobind/stl/variant.h> +#include <nanobind/stl/vector.h> +#include <optional> +#include <sstream> +#include <stdexcept> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> + +namespace nb = nanobind; +namespace kv = tensorrt_llm::batch_manager::kv_cache_manager_v2; + +namespace tensorrt_llm::nanobind::batch_manager +{ + +// Helper: convert a Python iterable of int|bytes to vector<TokenIdExt>. +// nanobind's variant caster can't auto-convert bytes → DigestToken. +static std::vector<kv::TokenIdExt> castTokenIterable(nb::handle tokens) +{ + std::vector<kv::TokenIdExt> vec; + for (auto item : nb::cast<nb::iterable>(tokens)) + { + if (nb::isinstance<nb::bytes>(item)) + { + auto b = nb::cast<nb::bytes>(item); + if (nb::len(b) != kv::kDIGEST_LEN) + { + throw std::invalid_argument("Token bytes must have length kDIGEST_LEN"); + } + kv::Digest d; + std::memcpy(d.data(), b.c_str(), kv::kDIGEST_LEN); + vec.emplace_back(kv::DigestToken(d)); + } + else + { + vec.emplace_back(nb::cast<kv::TokenId>(item)); + } + } + return vec; +} + +static kv::TypedVec<kv::PoolIndex, size_t> typedPoolSizeList(std::vector<size_t> const& slotSizeList) +{ + return kv::TypedVec<kv::PoolIndex, size_t>{slotSizeList}; +} + +static kv::TypedVec<kv::PoolGroupIndex, kv::TypedVec<kv::PoolIndex, size_t>> typedSlotSizeLists( + std::vector<std::vector<size_t>> const& slotSizeLists) +{ + kv::TypedVec<kv::PoolGroupIndex, kv::TypedVec<kv::PoolIndex, size_t>> result; + for (auto const& slotSizeList : slotSizeLists) + { + result.push_back(typedPoolSizeList(slotSizeList)); + } + return result; +} + +static kv::TypedVec<kv::PoolGroupIndex, kv::SlotCount> typedSlotCounts(std::vector<kv::SlotCount> const& slotCounts) +{ + kv::TypedVec<kv::PoolGroupIndex, kv::SlotCount> result; + for (kv::SlotCount slotCount : slotCounts) + { + if (slotCount < 0) + { + throw std::invalid_argument("slot counts must be non-negative"); + } + result.push_back(slotCount); + } + return result; +} + +static nb::object castLifeCycle(kv::LifeCycle const& lifeCycle) +{ + return std::visit([](auto const& concreteLifeCycle) { return nb::cast(concreteLifeCycle); }, lifeCycle); +} + +static kv::KvCache::PriorityCb castPriorityCallback(kv::KvCacheManager const& manager, nb::object callback) +{ + if (callback.is_none()) + { + return {}; + } + if (!PyCallable_Check(callback.ptr())) + { + throw std::invalid_argument("custom_priority_callback must be callable"); + } + + kv::LifeCycleRegistry const* lifeCycles = &manager.lifeCycles(); + return [callback = std::move(callback), lifeCycles](kv::BlockOrdinal ordinal, kv::LifeCycleId lifeCycleId) + { + nb::gil_scoped_acquire acquire; + return nb::cast<kv::Priority>(callback(ordinal.value(), castLifeCycle(lifeCycles->getLifeCycle(lifeCycleId)))); + }; +} + +static nb::tuple bufferIdTuple(kv::BufferId const& self) +{ + return nb::make_tuple(self.layerId, self.role); +} + +static nb::object optionalIntToObject(std::optional<std::uint64_t> value) +{ + if (!value.has_value()) + { + return nb::none(); + } + return nb::cast(*value); +} + +static nb::tuple reuseScopeTuple(kv::ReuseScope const& self) +{ + return nb::make_tuple(optionalIntToObject(self.loraId), optionalIntToObject(self.salt)); +} + +static nb::list tokenList(std::vector<kv::TokenIdExt> const& tokens) +{ + nb::list result; + for (auto const& tok : tokens) + { + if (auto* id = std::get_if<kv::TokenId>(&tok)) + { + result.append(*id); + } + else + { + auto const& d = std::get<kv::DigestToken>(tok); + result.append(nb::bytes(reinterpret_cast<char const*>(d.data()), d.size())); + } + } + return result; +} + +static nb::list committedTokensList(kv::KvCache const& self) +{ + return tokenList(self.committedTokens()); +} + +static nb::bytes digestBytes(kv::Digest const& digest) +{ + return nb::bytes(reinterpret_cast<char const*>(digest.data()), digest.size()); +} + +static kv::Digest castDigest(nb::handle value) +{ + if (!nb::isinstance<nb::bytes>(value)) + { + throw std::invalid_argument("block hash must be bytes"); + } + auto bytes = nb::cast<nb::bytes>(value); + if (nb::len(bytes) != kv::kDIGEST_LEN) + { + throw std::invalid_argument("block hash bytes must have length kDIGEST_LEN"); + } + kv::Digest digest; + std::memcpy(digest.data(), bytes.c_str(), digest.size()); + return digest; +} + +static bool hasDigestSize(nb::handle value) +{ + return nb::len(value) == kv::kDIGEST_LEN; +} + +static kv::EventBlockHash castEventBlockHash(nb::handle value) +{ + if (PyLong_Check(value.ptr())) + { + return nb::cast<uint64_t>(value); + } + return nb::cast<std::string>(value); +} + +static std::optional<kv::EventBlockHash> castOptionalEventBlockHash(nb::handle value) +{ + return value.is_none() ? std::nullopt : std::optional<kv::EventBlockHash>{castEventBlockHash(value)}; +} + +static std::vector<kv::KVCacheStoredBlockData> castStoredBlocks(nb::handle values) +{ + std::vector<kv::KVCacheStoredBlockData> result; + for (nb::handle value : nb::cast<nb::iterable>(values)) + { + result.push_back(nb::cast<kv::KVCacheStoredBlockData>(value)); + } + return result; +} + +static std::vector<kv::MmKey> castMmKeys(nb::handle values) +{ + std::vector<kv::MmKey> result; + for (nb::handle value : nb::cast<nb::iterable>(values)) + { + nb::tuple tuple = nb::cast<nb::tuple>(value); + if (tuple.size() != 2 && tuple.size() != 3) + { + throw std::invalid_argument("mm_key must have two or three entries"); + } + std::optional<std::string> uuid; + if (tuple.size() == 3 && !tuple[2].is_none()) + { + uuid = nb::cast<std::string>(tuple[2]); + } + if (!nb::isinstance<nb::bytes>(tuple[0])) + { + throw std::invalid_argument("mm_key hash must be bytes"); + } + auto hash = nb::cast<nb::bytes>(tuple[0]); + result.push_back(kv::MmKey{std::string(hash.c_str(), static_cast<size_t>(nb::len(hash))), + nb::cast<int>(tuple[1]), std::move(uuid), tuple.size() == 3}); + } + return result; +} + +static nb::list castMmKeys(kv::KVCacheStoredBlockData const& data) +{ + nb::list result; + for (auto const& mmKey : data.mmKeys) + { + auto hash = nb::bytes(mmKey.hash.data(), mmKey.hash.size()); + if (mmKey.hasUuidField) + { + result.append(nb::make_tuple(std::move(hash), mmKey.startOffset, mmKey.uuid)); + } + else + { + result.append(nb::make_tuple(std::move(hash), mmKey.startOffset)); + } + } + return result; +} + +static nb::object castEventData(kv::KVCacheEventData const& data) +{ + return std::visit([](auto const& concreteData) { return nb::cast(concreteData); }, data); +} + +static kv::KVCacheEventData castEventData(nb::handle data) +{ + if (nb::isinstance<kv::KVCacheCreatedData>(data)) + { + return nb::cast<kv::KVCacheCreatedData>(data); + } + if (nb::isinstance<kv::KVCacheStoredData>(data)) + { + return nb::cast<kv::KVCacheStoredData>(data); + } + if (nb::isinstance<kv::KVCacheRemovedData>(data)) + { + return nb::cast<kv::KVCacheRemovedData>(data); + } + if (nb::isinstance<kv::KVCacheUpdatedData>(data)) + { + return nb::cast<kv::KVCacheUpdatedData>(data); + } + throw std::invalid_argument("Unsupported KV cache event data type"); +} + +static std::string pythonHandleRepr(nb::handle value) +{ + nb::object result = nb::steal<nb::object>(PyObject_Repr(value.ptr())); + if (!result) + { + throw nb::python_error(); + } + return nb::cast<std::string>(result); +} + +template <typename T> +static std::string pythonRepr(T const& value) +{ + return pythonHandleRepr(nb::cast(value)); +} + +class PythonAttentionDpGather +{ +public: + explicit PythonAttentionDpGather(nb::handle callback) + : mCallback(callback.ptr()) + { + Py_INCREF(mCallback); + } + + ~PythonAttentionDpGather() + { + if (Py_IsInitialized()) + { + nb::gil_scoped_acquire acquire; + Py_DECREF(mCallback); + } + } + + PythonAttentionDpGather(PythonAttentionDpGather const&) = delete; + PythonAttentionDpGather& operator=(PythonAttentionDpGather const&) = delete; + + std::vector<std::vector<kv::KVCacheEvent>> operator()(std::vector<kv::KVCacheEvent> const& events) const + { + nb::gil_scoped_acquire acquire; + return nb::cast<std::vector<std::vector<kv::KVCacheEvent>>>(nb::borrow<nb::object>(mCallback)(events)); + } + +private: + PyObject* mCallback; +}; + +static kv::EventManager::AttentionDpGatherFn castAttentionDpGather(nb::handle callback) +{ + if (callback.is_none()) + { + return {}; + } + if (!PyCallable_Check(callback.ptr())) + { + throw std::invalid_argument("attention_dp_gather must be callable"); + } + auto gather = std::make_shared<PythonAttentionDpGather>(callback); + return [gather = std::move(gather)](std::vector<kv::KVCacheEvent> const& events) { return (*gather)(events); }; +} + +static nb::object castStatsDelta(kv::KVCacheStatsDelta const& stats) +{ + return nb::cast(stats); +} + +static nb::object castIterationStatsDelta(kv::KVCacheIterationStatsDelta const& stats) +{ + return nb::cast(stats); +} + +static nb::dict castIterationStatsByLifeCycle(kv::IterationStatsByLifeCycle const& statsByLifeCycle) +{ + nb::dict result; + for (auto const& [lifeCycle, stats] : statsByLifeCycle) + { + result[nb::int_(lifeCycle.value())] = castIterationStatsDelta(stats); + } + return result; +} + +static nb::dict castSsmSnapshotIterationStatsByLifeCycle( + kv::SsmSnapshotIterationStatsByLifeCycle const& statsByLifeCycle) +{ + nb::dict result; + for (auto const& [lifeCycle, stats] : statsByLifeCycle) + { + result[nb::int_(lifeCycle.value())] = nb::cast(stats); + } + return result; +} + +static nb::list castPeakBlockStats(kv::PeakBlockStatsByPoolGroup const& statsByPoolGroup) +{ + nb::list result; + for (auto const& stats : statsByPoolGroup) + { + result.append(nb::cast(stats)); + } + return result; +} + +static std::string statsDeltaRepr(kv::KVCacheStatsDelta const& stats) +{ + std::ostringstream stream; + stream << "KVCacheStatsDelta(alloc_total_blocks=" << stats.allocTotalBlocks + << ", alloc_new_blocks=" << stats.allocNewBlocks << ", reused_blocks=" << stats.reusedBlocks + << ", missed_blocks=" << stats.missedBlocks << ')'; + return stream.str(); +} + +static std::string iterationStatsDeltaRepr(kv::KVCacheIterationStatsDelta const& stats) +{ + std::ostringstream stream; + stream << "KVCacheIterationStatsDelta(iter_alloc_total_blocks=" << stats.iterAllocTotalBlocks + << ", iter_alloc_new_blocks=" << stats.iterAllocNewBlocks + << ", iter_reused_blocks=" << stats.iterReusedBlocks + << ", iter_full_reused_blocks=" << stats.iterFullReusedBlocks + << ", iter_partial_reused_blocks=" << stats.iterPartialReusedBlocks + << ", iter_missed_blocks=" << stats.iterMissedBlocks + << ", iter_gen_alloc_blocks=" << stats.iterGenAllocBlocks + << ", iter_onboard_blocks=" << stats.iterOnboardBlocks << ", iter_onboard_bytes=" << stats.iterOnboardBytes + << ", iter_offload_blocks=" << stats.iterOffloadBlocks << ", iter_offload_bytes=" << stats.iterOffloadBytes + << ", iter_intra_device_copy_blocks=" << stats.iterIntraDeviceCopyBlocks + << ", iter_intra_device_copy_bytes=" << stats.iterIntraDeviceCopyBytes + << ", iter_host_dropped_blocks=" << stats.iterHostDroppedBlocks + << ", iter_host_dropped_bytes=" << stats.iterHostDroppedBytes << ')'; + return stream.str(); +} + +static std::string ssmSnapshotStatsDeltaRepr(kv::SsmSnapshotIterationStatsDelta const& stats) +{ + std::ostringstream stream; + stream << "SsmSnapshotIterationStatsDelta(iter_snapshot_lookups=" << stats.iterSnapshotLookups + << ", iter_snapshot_hits=" << stats.iterSnapshotHits << ", iter_snapshot_misses=" << stats.iterSnapshotMisses + << ", iter_reused_tokens=" << stats.iterReusedTokens << ", iter_unreused_tokens=" << stats.iterUnreusedTokens + << ", iter_aligned_snapshot_hits=" << stats.iterAlignedSnapshotHits + << ", iter_unaligned_snapshot_hits=" << stats.iterUnalignedSnapshotHits << ')'; + return stream.str(); +} + +static std::string peakBlockStatsRepr(kv::PoolGroupPeakBlockStats const& stats) +{ + std::ostringstream stream; + stream << "PoolGroupPeakBlockStats(available=" << stats.available << ", unavailable=" << stats.unavailable + << ", evictable=" << stats.evictable << ')'; + return stream.str(); +} + +static nb::object castRequestIds(std::unordered_set<kv::RequestIdType> const& requestIds) +{ + nb::object result = nb::module_::import_("builtins").attr("set")(); + for (kv::RequestIdType const requestId : requestIds) + { + result.attr("add")(requestId); + } + return result; +} + +static std::optional<std::uint64_t> castOptionalIntAttr(nb::handle obj, char const* attrName) +{ + nb::object attr = nb::steal(PyObject_GetAttrString(obj.ptr(), attrName)); + if (!attr) + { + throw nb::python_error(); + } + if (attr.is_none()) + { + return std::nullopt; + } + return nb::cast<std::uint64_t>(attr); +} + +static kv::ReuseScope castReuseScope(nb::object reuseScope) +{ + if (reuseScope.is_none()) + { + return {}; + } + if (nb::isinstance<kv::ReuseScope>(reuseScope)) + { + return nb::cast<kv::ReuseScope>(reuseScope); + } + // Backward-compatible bridge for old callers that still pass lora_task_id as the first argument. + if (PyLong_Check(reuseScope.ptr())) + { + return {nb::cast<kv::LoraTaskIdType>(reuseScope), std::nullopt}; + } + if (PyObject_HasAttrString(reuseScope.ptr(), "lora_id") && PyObject_HasAttrString(reuseScope.ptr(), "salt")) + { + return {castOptionalIntAttr(reuseScope, "lora_id"), castOptionalIntAttr(reuseScope, "salt")}; + } + throw std::invalid_argument( + "reuse_scope must be None, ReuseScope, an int lora_task_id, or an object with lora_id and salt"); +} + +void KvCacheManagerV2Bindings::initBindings(nb::module_& m) +{ + // Export the C++ debug mode as an immutable Python bool snapshot. + m.attr("NDEBUG") = nb::bool_(!kv::gDebug); + + // ---- Exceptions -------------------------------------------------------- + static nb::object sOutOfMemoryError = nb::exception<kv::OutOfMemoryError>(m, "OutOfMemoryError"); + static nb::object sHostOOMError = nb::exception<kv::HostOOMError>(m, "HostOOMError"); + static nb::object sDiskOOMError = nb::exception<kv::DiskOOMError>(m, "DiskOOMError"); + static nb::object sCuOOMError = nb::exception<kv::CuOOMError>(m, "CuOOMError"); + static nb::object sLogicError = nb::exception<kv::LogicError>(m, "LogicError"); + static nb::object sResourceBusyError = nb::exception<kv::ResourceBusyError>(m, "ResourceBusyError"); + static nb::object sOutOfPagesError = nb::exception<kv::OutOfPagesError>(m, "OutOfPagesError"); + + // Map kv::AssertionError to Python's builtin AssertionError so shared tests see + // the same exception type as the pure-Python backend (which uses `assert`). + // Registered last so it is tried before the generic std::exception fallback. + nb::register_exception_translator( + [](std::exception_ptr const& p, void*) + { + try + { + if (p) + std::rethrow_exception(p); + } + catch (kv::AssertionError const& e) + { + PyErr_SetString(PyExc_AssertionError, e.what()); + } + }); + + // ---- Enums ------------------------------------------------------------- + nb::enum_<kv::PageStatus>(m, "PageStatus") + .value("LOCKED", kv::PageStatus::LOCKED) + .value("HELD", kv::PageStatus::HELD) + .value("DROPPABLE", kv::PageStatus::DROPPABLE) + .export_values(); + + nb::enum_<kv::CacheTier>(m, "CacheTier") + .value("GPU_MEM", kv::CacheTier::GPU_MEM) + .value("HOST_MEM", kv::CacheTier::HOST_MEM) + .value("DISK", kv::CacheTier::DISK) + .export_values(); + + // ---- KvCache::Status enum (also accessible as _KVCache.Status) --------- + auto kvCacheStatus = nb::enum_<kv::KvCache::Status>(m, "KvCacheStatus") + .value("ACTIVE", kv::KvCache::Status::ACTIVE) + .value("SUSPENDED", kv::KvCache::Status::SUSPENDED) + .value("CLOSED", kv::KvCache::Status::CLOSED); + + // ---- KV cache events ---------------------------------------------------- + nb::class_<kv::UniqueToken>(m, "UniqueToken") + .def(nb::init<kv::EventTokenId, int64_t>(), nb::arg("token_id"), nb::arg("token_extra_id") = 0) + .def_ro("token_id", &kv::UniqueToken::tokenId) + .def_ro("token_extra_id", &kv::UniqueToken::tokenExtraId) + .def("__eq__", &kv::UniqueToken::operator==, nb::arg("other")) + .def("__repr__", + [](kv::UniqueToken const& self) + { + return "UniqueToken(token_id=" + pythonRepr(self.tokenId) + + ", token_extra_id=" + pythonRepr(self.tokenExtraId) + ")"; + }) + .def("__reduce__", + [](kv::UniqueToken const& self) + { return nb::make_tuple(nb::type<kv::UniqueToken>(), nb::make_tuple(self.tokenId, self.tokenExtraId)); }); + + nb::class_<kv::KVCacheCreatedData>(m, "KVCacheCreatedData") + .def(nb::init<std::vector<int>>(), nb::arg("num_blocks_per_cache_level")) + .def_ro("num_blocks_per_cache_level", &kv::KVCacheCreatedData::numBlocksPerCacheLevel) + .def("__eq__", &kv::KVCacheCreatedData::operator==, nb::arg("other")) + .def("__repr__", + [](kv::KVCacheCreatedData const& self) { + return "KVCacheCreatedData(num_blocks_per_cache_level=" + pythonRepr(self.numBlocksPerCacheLevel) + ")"; + }) + .def("__reduce__", + [](kv::KVCacheCreatedData const& self) { + return nb::make_tuple(nb::type<kv::KVCacheCreatedData>(), nb::make_tuple(self.numBlocksPerCacheLevel)); + }); + + nb::class_<kv::KVCacheStoredBlockData>(m, "KVCacheStoredBlockData") + .def( + "__init__", + [](kv::KVCacheStoredBlockData* self, kv::EventBlockHash blockHash, std::vector<kv::UniqueToken> tokens, + int cacheLevel, int priority, nb::handle mmKeys, std::optional<std::string> cacheSalt) + { + new (self) kv::KVCacheStoredBlockData{std::move(blockHash), std::move(tokens), cacheLevel, priority, + castMmKeys(mmKeys), std::move(cacheSalt)}; + }, + nb::arg("block_hash"), nb::arg("tokens"), nb::arg("cache_level"), nb::arg("priority"), + nb::arg("mm_keys") = nb::make_tuple(), nb::arg("cache_salt") = std::nullopt) + .def_ro("block_hash", &kv::KVCacheStoredBlockData::blockHash) + .def_ro("tokens", &kv::KVCacheStoredBlockData::tokens) + .def_ro("cache_level", &kv::KVCacheStoredBlockData::cacheLevel) + .def_ro("priority", &kv::KVCacheStoredBlockData::priority) + .def_prop_ro("mm_keys", [](kv::KVCacheStoredBlockData const& self) { return castMmKeys(self); }) + .def_ro("cache_salt", &kv::KVCacheStoredBlockData::cacheSalt) + .def("__eq__", &kv::KVCacheStoredBlockData::operator==, nb::arg("other")) + .def("__repr__", + [](kv::KVCacheStoredBlockData const& self) + { + return "KVCacheStoredBlockData(block_hash=" + pythonRepr(self.blockHash) + + ", tokens=" + pythonRepr(self.tokens) + ", cache_level=" + pythonRepr(self.cacheLevel) + + ", priority=" + pythonRepr(self.priority) + ", mm_keys=" + pythonHandleRepr(castMmKeys(self)) + + ", cache_salt=" + pythonRepr(self.cacheSalt) + ")"; + }) + .def("__reduce__", + [](kv::KVCacheStoredBlockData const& self) + { + return nb::make_tuple(nb::type<kv::KVCacheStoredBlockData>(), + nb::make_tuple( + self.blockHash, self.tokens, self.cacheLevel, self.priority, castMmKeys(self), self.cacheSalt)); + }); + + nb::class_<kv::KVCacheStoredData>(m, "KVCacheStoredData") + .def( + "__init__", + [](kv::KVCacheStoredData* self, nb::object parentHash, nb::object blocks) { + new (self) kv::KVCacheStoredData{castOptionalEventBlockHash(parentHash), castStoredBlocks(blocks)}; + }, + nb::arg("parent_hash").none(), nb::arg("blocks")) + .def_ro("parent_hash", &kv::KVCacheStoredData::parentHash) + .def_ro("blocks", &kv::KVCacheStoredData::blocks) + .def("__eq__", &kv::KVCacheStoredData::operator==, nb::arg("other")) + .def("__repr__", + [](kv::KVCacheStoredData const& self) + { + return "KVCacheStoredData(parent_hash=" + pythonRepr(self.parentHash) + + ", blocks=" + pythonRepr(self.blocks) + ")"; + }) + .def("__reduce__", + [](kv::KVCacheStoredData const& self) { + return nb::make_tuple(nb::type<kv::KVCacheStoredData>(), nb::make_tuple(self.parentHash, self.blocks)); + }); + + nb::class_<kv::KVCacheRemovedData>(m, "KVCacheRemovedData") + .def(nb::init<std::vector<kv::EventBlockHash>>(), nb::arg("block_hashes")) + .def_ro("block_hashes", &kv::KVCacheRemovedData::blockHashes) + .def("__eq__", &kv::KVCacheRemovedData::operator==, nb::arg("other")) + .def("__repr__", + [](kv::KVCacheRemovedData const& self) + { return "KVCacheRemovedData(block_hashes=" + pythonRepr(self.blockHashes) + ")"; }) + .def("__reduce__", + [](kv::KVCacheRemovedData const& self) + { return nb::make_tuple(nb::type<kv::KVCacheRemovedData>(), nb::make_tuple(self.blockHashes)); }); + + nb::class_<kv::KVCacheEventDiff>(m, "KVCacheEventDiff") + .def(nb::init<int, int>(), nb::arg("old_value"), nb::arg("new_value")) + .def_ro("old_value", &kv::KVCacheEventDiff::oldValue) + .def_ro("new_value", &kv::KVCacheEventDiff::newValue) + .def("__eq__", &kv::KVCacheEventDiff::operator==, nb::arg("other")) + .def("__repr__", + [](kv::KVCacheEventDiff const& self) + { + return "KVCacheEventDiff(old_value=" + pythonRepr(self.oldValue) + + ", new_value=" + pythonRepr(self.newValue) + ")"; + }) + .def("__reduce__", + [](kv::KVCacheEventDiff const& self) + { return nb::make_tuple(nb::type<kv::KVCacheEventDiff>(), nb::make_tuple(self.oldValue, self.newValue)); }); + + nb::class_<kv::KVCacheUpdatedData>(m, "KVCacheUpdatedData") + .def(nb::init<kv::EventBlockHash, std::optional<kv::KVCacheEventDiff>, std::optional<kv::KVCacheEventDiff>>(), + nb::arg("block_hash"), nb::arg("cache_level").none(), nb::arg("priority").none()) + .def_ro("block_hash", &kv::KVCacheUpdatedData::blockHash) + .def_ro("cache_level", &kv::KVCacheUpdatedData::cacheLevel) + .def_ro("priority", &kv::KVCacheUpdatedData::priority) + .def("__eq__", &kv::KVCacheUpdatedData::operator==, nb::arg("other")) + .def("__repr__", + [](kv::KVCacheUpdatedData const& self) + { + return "KVCacheUpdatedData(block_hash=" + pythonRepr(self.blockHash) + + ", cache_level=" + pythonRepr(self.cacheLevel) + ", priority=" + pythonRepr(self.priority) + ")"; + }) + .def("__reduce__", + [](kv::KVCacheUpdatedData const& self) + { + return nb::make_tuple( + nb::type<kv::KVCacheUpdatedData>(), nb::make_tuple(self.blockHash, self.cacheLevel, self.priority)); + }); + + nb::class_<kv::KVCacheEvent>(m, "KVCacheEvent") + .def( + "__init__", + [](kv::KVCacheEvent* self, int64_t eventId, nb::handle data, int windowSize, + std::optional<std::string> hashAlgo, std::optional<int> attentionDpRank, + kv::EventLayerGroupId layerGroupId) + { + new (self) kv::KVCacheEvent{ + eventId, castEventData(data), windowSize, std::move(hashAlgo), attentionDpRank, layerGroupId}; + }, + nb::arg("event_id"), nb::arg("data"), nb::arg("window_size"), nb::arg("hash_algo") = std::nullopt, + nb::arg("attention_dp_rank") = std::nullopt, nb::arg("layer_group_id") = std::nullopt) + .def_ro("event_id", &kv::KVCacheEvent::eventId) + .def_prop_ro("data", [](kv::KVCacheEvent const& self) { return castEventData(self.data); }) + .def_ro("window_size", &kv::KVCacheEvent::windowSize) + .def_ro("hash_algo", &kv::KVCacheEvent::hashAlgo) + .def_ro("attention_dp_rank", &kv::KVCacheEvent::attentionDpRank) + .def_ro("layer_group_id", &kv::KVCacheEvent::layerGroupId) + .def("__eq__", &kv::KVCacheEvent::operator==, nb::arg("other")) + .def("__repr__", + [](kv::KVCacheEvent const& self) + { + return "KVCacheEvent(event_id=" + pythonRepr(self.eventId) + ", data=" + + pythonHandleRepr(castEventData(self.data)) + ", window_size=" + pythonRepr(self.windowSize) + + ", hash_algo=" + pythonRepr(self.hashAlgo) + ", attention_dp_rank=" + + pythonRepr(self.attentionDpRank) + ", layer_group_id=" + pythonRepr(self.layerGroupId) + ")"; + }) + .def("__reduce__", + [](kv::KVCacheEvent const& self) + { + return nb::make_tuple(nb::type<kv::KVCacheEvent>(), + nb::make_tuple(self.eventId, castEventData(self.data), self.windowSize, self.hashAlgo, + self.attentionDpRank, self.layerGroupId)); + }); + + nb::class_<kv::EventManager>(m, "KVCacheEventManager") + .def( + "__init__", + [](kv::EventManager* self, int maxKvEventEntries, int windowSize, std::optional<int> attentionDpRank, + nb::handle attentionDpGather, std::string hashAlgo, nb::handle windowSizeByLayerGroup) + { + std::map<int, int> windowSizes; + if (!windowSizeByLayerGroup.is_none()) + { + windowSizes = nb::cast<std::map<int, int>>(windowSizeByLayerGroup); + } + new (self) kv::EventManager(maxKvEventEntries, windowSize, attentionDpRank, + castAttentionDpGather(attentionDpGather), std::move(hashAlgo), std::move(windowSizes)); + }, + nb::arg("max_kv_event_entries"), nb::kw_only(), nb::arg("window_size") = 0, + nb::arg("attention_dp_rank") = std::nullopt, nb::arg("attention_dp_gather") = nb::none(), + nb::arg("hash_algo") = "v2_sha256", nb::arg("window_size_by_layer_group").none() = nb::none()) + .def("add_created_event", &kv::EventManager::addCreatedEvent, nb::arg("num_blocks_per_cache_level"), + nb::arg("layer_group_ids") = std::nullopt, nb::call_guard<nb::gil_scoped_release>()) + .def("set_layer_group_window_sizes", &kv::EventManager::setLayerGroupWindowSizes, nb::arg("window_sizes"), + nb::call_guard<nb::gil_scoped_release>()) + .def( + "add_stored_event", + [](kv::EventManager& self, nb::object parentHash, nb::object blocks, kv::EventLayerGroupId layerGroupId) + { + self.addStoredEvent( + kv::KVCacheStoredData{castOptionalEventBlockHash(parentHash), castStoredBlocks(blocks)}, + layerGroupId); + }, + nb::arg("parent_hash").none(), nb::arg("blocks"), nb::arg("layer_group_id") = std::nullopt) + .def( + "add_removed_event", + [](kv::EventManager& self, nb::handle blockHashes) + { + if (nb::isinstance<nb::bytes>(blockHashes)) + { + if (hasDigestSize(blockHashes)) + { + self.addRemovedBlock(castDigest(blockHashes)); + } + return; + } + if (PyLong_Check(blockHashes.ptr()) || nb::isinstance<nb::str>(blockHashes)) + { + self.addRemovedEvent({castEventBlockHash(blockHashes)}); + return; + } + for (nb::handle blockHash : nb::cast<nb::iterable>(blockHashes)) + { + if (nb::isinstance<nb::bytes>(blockHash)) + { + if (hasDigestSize(blockHash)) + { + self.addRemovedBlock(castDigest(blockHash)); + } + } + else + { + self.addRemovedEvent({castEventBlockHash(blockHash)}); + } + } + }, + nb::arg("block_hashes")) + .def( + "add_removed_life_cycle_event", + [](kv::EventManager& self, nb::handle blockHash, int lifeCycleId) + { + if (!nb::isinstance<nb::bytes>(blockHash)) + { + throw std::invalid_argument("block hash must be bytes"); + } + if (hasDigestSize(blockHash)) + { + self.addRemovedLifeCycle(castDigest(blockHash), kv::LifeCycleId{lifeCycleId}); + } + }, + nb::arg("block_hash"), nb::arg("life_cycle_id")) + .def( + "add_updated_event", + [](kv::EventManager& self, nb::handle blockHash, std::optional<kv::KVCacheEventDiff> cacheLevel, + std::optional<kv::KVCacheEventDiff> priority, kv::EventLayerGroupId layerGroupId) + { + if (!cacheLevel.has_value() && !priority.has_value()) + { + return; + } + if (nb::isinstance<nb::bytes>(blockHash)) + { + if (!hasDigestSize(blockHash)) + { + return; + } + auto digest = castDigest(blockHash); + nb::gil_scoped_release release; + self.addUpdatedEvent(digest, cacheLevel, priority, layerGroupId); + return; + } + auto eventBlockHash = castEventBlockHash(blockHash); + nb::gil_scoped_release release; + self.addUpdatedEvent(std::move(eventBlockHash), cacheLevel, priority, layerGroupId); + }, + nb::arg("block_hash"), nb::kw_only(), nb::arg("cache_level") = std::nullopt, + nb::arg("priority") = std::nullopt, nb::arg("layer_group_id") = std::nullopt) + .def( + "flush_iteration_events", &kv::EventManager::flushIterationEvents, nb::call_guard<nb::gil_scoped_release>()) + .def("get_latest_events", &kv::EventManager::getLatestEvents, nb::arg("timeout_ms") = std::nullopt, + nb::call_guard<nb::gil_scoped_release>()) + .def_prop_ro("hash_algo", &kv::EventManager::hashAlgorithm) + .def_prop_ro("_hash_algo", &kv::EventManager::hashAlgorithm) + .def_static("_hash_block_key", &kv::EventManager::hashV1BlockKey, nb::arg("tokens"), nb::arg("parent_hash") = 0, + nb::arg("lora_task_id") = std::nullopt, nb::arg("cache_salt_id") = std::nullopt); + + // ---- Statistics -------------------------------------------------------- + nb::class_<kv::KVCacheStatsDelta>(m, "KVCacheStatsDelta") + .def( + "__init__", + [](kv::KVCacheStatsDelta* self, int64_t allocTotalBlocks, int64_t allocNewBlocks, int64_t reusedBlocks, + int64_t missedBlocks) + { + new (self) kv::KVCacheStatsDelta{}; + self->allocTotalBlocks = allocTotalBlocks; + self->allocNewBlocks = allocNewBlocks; + self->reusedBlocks = reusedBlocks; + self->missedBlocks = missedBlocks; + }, + nb::arg("alloc_total_blocks") = 0, nb::arg("alloc_new_blocks") = 0, nb::arg("reused_blocks") = 0, + nb::arg("missed_blocks") = 0) + .def_rw("alloc_total_blocks", &kv::KVCacheStatsDelta::allocTotalBlocks) + .def_rw("alloc_new_blocks", &kv::KVCacheStatsDelta::allocNewBlocks) + .def_rw("reused_blocks", &kv::KVCacheStatsDelta::reusedBlocks) + .def_rw("missed_blocks", &kv::KVCacheStatsDelta::missedBlocks) + .def("add", &kv::KVCacheStatsDelta::add, nb::arg("other")) + .def("subtract", &kv::KVCacheStatsDelta::subtract, nb::arg("other")) + .def("clear", &kv::KVCacheStatsDelta::clear) + .def("copy", &kv::KVCacheStatsDelta::copy) + .def_prop_ro("empty", &kv::KVCacheStatsDelta::empty) + .def("__eq__", &kv::KVCacheStatsDelta::operator==, nb::arg("other")) + .def("__repr__", &statsDeltaRepr); + + nb::class_<kv::KVCacheIterationStatsDelta>(m, "KVCacheIterationStatsDelta") + .def( + "__init__", + [](kv::KVCacheIterationStatsDelta* self, int64_t iterAllocTotalBlocks, int64_t iterAllocNewBlocks, + int64_t iterReusedBlocks, int64_t iterFullReusedBlocks, int64_t iterPartialReusedBlocks, + int64_t iterMissedBlocks, int64_t iterGenAllocBlocks, int64_t iterOnboardBlocks, + int64_t iterOnboardBytes, int64_t iterOffloadBlocks, int64_t iterOffloadBytes, + int64_t iterIntraDeviceCopyBlocks, int64_t iterIntraDeviceCopyBytes, int64_t iterHostDroppedBlocks, + int64_t iterHostDroppedBytes) + { + new (self) kv::KVCacheIterationStatsDelta{}; + self->iterAllocTotalBlocks = iterAllocTotalBlocks; + self->iterAllocNewBlocks = iterAllocNewBlocks; + self->iterReusedBlocks = iterReusedBlocks; + self->iterFullReusedBlocks = iterFullReusedBlocks; + self->iterPartialReusedBlocks = iterPartialReusedBlocks; + self->iterMissedBlocks = iterMissedBlocks; + self->iterGenAllocBlocks = iterGenAllocBlocks; + self->iterOnboardBlocks = iterOnboardBlocks; + self->iterOnboardBytes = iterOnboardBytes; + self->iterOffloadBlocks = iterOffloadBlocks; + self->iterOffloadBytes = iterOffloadBytes; + self->iterIntraDeviceCopyBlocks = iterIntraDeviceCopyBlocks; + self->iterIntraDeviceCopyBytes = iterIntraDeviceCopyBytes; + self->iterHostDroppedBlocks = iterHostDroppedBlocks; + self->iterHostDroppedBytes = iterHostDroppedBytes; + }, + nb::arg("iter_alloc_total_blocks") = 0, nb::arg("iter_alloc_new_blocks") = 0, + nb::arg("iter_reused_blocks") = 0, nb::arg("iter_full_reused_blocks") = 0, + nb::arg("iter_partial_reused_blocks") = 0, nb::arg("iter_missed_blocks") = 0, + nb::arg("iter_gen_alloc_blocks") = 0, nb::arg("iter_onboard_blocks") = 0, nb::arg("iter_onboard_bytes") = 0, + nb::arg("iter_offload_blocks") = 0, nb::arg("iter_offload_bytes") = 0, + nb::arg("iter_intra_device_copy_blocks") = 0, nb::arg("iter_intra_device_copy_bytes") = 0, + nb::arg("iter_host_dropped_blocks") = 0, nb::arg("iter_host_dropped_bytes") = 0) + .def_rw("iter_alloc_total_blocks", &kv::KVCacheIterationStatsDelta::iterAllocTotalBlocks) + .def_rw("iter_alloc_new_blocks", &kv::KVCacheIterationStatsDelta::iterAllocNewBlocks) + .def_rw("iter_reused_blocks", &kv::KVCacheIterationStatsDelta::iterReusedBlocks) + .def_rw("iter_full_reused_blocks", &kv::KVCacheIterationStatsDelta::iterFullReusedBlocks) + .def_rw("iter_partial_reused_blocks", &kv::KVCacheIterationStatsDelta::iterPartialReusedBlocks) + .def_rw("iter_missed_blocks", &kv::KVCacheIterationStatsDelta::iterMissedBlocks) + .def_rw("iter_gen_alloc_blocks", &kv::KVCacheIterationStatsDelta::iterGenAllocBlocks) + .def_rw("iter_onboard_blocks", &kv::KVCacheIterationStatsDelta::iterOnboardBlocks) + .def_rw("iter_onboard_bytes", &kv::KVCacheIterationStatsDelta::iterOnboardBytes) + .def_rw("iter_offload_blocks", &kv::KVCacheIterationStatsDelta::iterOffloadBlocks) + .def_rw("iter_offload_bytes", &kv::KVCacheIterationStatsDelta::iterOffloadBytes) + .def_rw("iter_intra_device_copy_blocks", &kv::KVCacheIterationStatsDelta::iterIntraDeviceCopyBlocks) + .def_rw("iter_intra_device_copy_bytes", &kv::KVCacheIterationStatsDelta::iterIntraDeviceCopyBytes) + .def_rw("iter_host_dropped_blocks", &kv::KVCacheIterationStatsDelta::iterHostDroppedBlocks) + .def_rw("iter_host_dropped_bytes", &kv::KVCacheIterationStatsDelta::iterHostDroppedBytes) + .def("add", &kv::KVCacheIterationStatsDelta::add, nb::arg("other")) + .def("subtract", &kv::KVCacheIterationStatsDelta::subtract, nb::arg("other")) + .def("clear", &kv::KVCacheIterationStatsDelta::clear) + .def("copy", &kv::KVCacheIterationStatsDelta::copy) + .def_prop_ro("empty", &kv::KVCacheIterationStatsDelta::empty) + .def_prop_ro("iter_cache_hit_rate", &kv::KVCacheIterationStatsDelta::iterCacheHitRate) + .def("__eq__", &kv::KVCacheIterationStatsDelta::operator==, nb::arg("other")) + .def("__repr__", &iterationStatsDeltaRepr); + + m.attr("KVCacheIterationStatsDelta").attr("_field_names") = nb::make_tuple("iter_alloc_total_blocks", + "iter_alloc_new_blocks", "iter_reused_blocks", "iter_full_reused_blocks", "iter_partial_reused_blocks", + "iter_missed_blocks", "iter_gen_alloc_blocks", "iter_onboard_blocks", "iter_onboard_bytes", + "iter_offload_blocks", "iter_offload_bytes", "iter_intra_device_copy_blocks", "iter_intra_device_copy_bytes", + "iter_host_dropped_blocks", "iter_host_dropped_bytes"); + + nb::class_<kv::SsmSnapshotIterationStatsDelta>(m, "SsmSnapshotIterationStatsDelta") + .def( + "__init__", + [](kv::SsmSnapshotIterationStatsDelta* self, int64_t iterSnapshotLookups, int64_t iterSnapshotHits, + int64_t iterSnapshotMisses, int64_t iterReusedTokens, int64_t iterUnreusedTokens, + int64_t iterAlignedSnapshotHits, int64_t iterUnalignedSnapshotHits) + { + new (self) kv::SsmSnapshotIterationStatsDelta{}; + self->iterSnapshotLookups = iterSnapshotLookups; + self->iterSnapshotHits = iterSnapshotHits; + self->iterSnapshotMisses = iterSnapshotMisses; + self->iterReusedTokens = iterReusedTokens; + self->iterUnreusedTokens = iterUnreusedTokens; + self->iterAlignedSnapshotHits = iterAlignedSnapshotHits; + self->iterUnalignedSnapshotHits = iterUnalignedSnapshotHits; + }, + nb::arg("iter_snapshot_lookups") = 0, nb::arg("iter_snapshot_hits") = 0, + nb::arg("iter_snapshot_misses") = 0, nb::arg("iter_reused_tokens") = 0, nb::arg("iter_unreused_tokens") = 0, + nb::arg("iter_aligned_snapshot_hits") = 0, nb::arg("iter_unaligned_snapshot_hits") = 0) + .def_rw("iter_snapshot_lookups", &kv::SsmSnapshotIterationStatsDelta::iterSnapshotLookups) + .def_rw("iter_snapshot_hits", &kv::SsmSnapshotIterationStatsDelta::iterSnapshotHits) + .def_rw("iter_snapshot_misses", &kv::SsmSnapshotIterationStatsDelta::iterSnapshotMisses) + .def_rw("iter_reused_tokens", &kv::SsmSnapshotIterationStatsDelta::iterReusedTokens) + .def_rw("iter_unreused_tokens", &kv::SsmSnapshotIterationStatsDelta::iterUnreusedTokens) + .def_rw("iter_aligned_snapshot_hits", &kv::SsmSnapshotIterationStatsDelta::iterAlignedSnapshotHits) + .def_rw("iter_unaligned_snapshot_hits", &kv::SsmSnapshotIterationStatsDelta::iterUnalignedSnapshotHits) + .def("add", &kv::SsmSnapshotIterationStatsDelta::add, nb::arg("other")) + .def("subtract", &kv::SsmSnapshotIterationStatsDelta::subtract, nb::arg("other")) + .def("clear", &kv::SsmSnapshotIterationStatsDelta::clear) + .def("copy", &kv::SsmSnapshotIterationStatsDelta::copy) + .def_prop_ro("empty", &kv::SsmSnapshotIterationStatsDelta::empty) + .def_prop_ro("iter_snapshot_hit_rate", &kv::SsmSnapshotIterationStatsDelta::iterSnapshotHitRate) + .def("__eq__", &kv::SsmSnapshotIterationStatsDelta::operator==, nb::arg("other")) + .def("__repr__", &ssmSnapshotStatsDeltaRepr); + + nb::class_<kv::PoolGroupPeakBlockStats>(m, "PoolGroupPeakBlockStats") + .def( + "__init__", + [](kv::PoolGroupPeakBlockStats* self, kv::SlotCount available, kv::SlotCount unavailable, + kv::SlotCount evictable) { + new (self) kv::PoolGroupPeakBlockStats{available, unavailable, evictable}; + }, + nb::arg("available"), nb::arg("unavailable"), nb::arg("evictable")) + .def_ro("available", &kv::PoolGroupPeakBlockStats::available) + .def_ro("unavailable", &kv::PoolGroupPeakBlockStats::unavailable) + .def_ro("evictable", &kv::PoolGroupPeakBlockStats::evictable) + .def("__eq__", &kv::PoolGroupPeakBlockStats::operator==, nb::arg("other")) + .def("__repr__", &peakBlockStatsRepr); + + // ---- Life cycle helpers ------------------------------------------------ + using BlockRange = kv::HalfOpenRange<kv::BlockOrdinal>; + nb::class_<BlockRange>(m, "HalfOpenRange") + .def(nb::init<int, int>(), nb::arg("beg"), nb::arg("end")) + .def_prop_ro("beg", [](BlockRange const& self) { return self.beg.value(); }) + .def_prop_ro("end", [](BlockRange const& self) { return self.end.value(); }) + .def("__bool__", [](BlockRange const& self) { return static_cast<bool>(self); }) + .def("__len__", &BlockRange::length) + .def("__eq__", [](BlockRange const& self, BlockRange const& other) { return self == other; }) + .def( + "__contains__", [](BlockRange const& self, int item) { return self.contains(kv::BlockOrdinal{item}); }, + nb::arg("item")); + + nb::enum_<kv::PageIndexMode>(m, "PageIndexMode") + .value("SHARED", kv::PageIndexMode::SHARED) + .value("PER_LAYER", kv::PageIndexMode::PER_LAYER); + + nb::class_<kv::ScratchDesc>(m, "ScratchDesc") + .def_ro("range", &kv::ScratchDesc::range) + .def_ro("slot_ids", &kv::ScratchDesc::slotIds) + .def("__bool__", [](kv::ScratchDesc const& self) { return static_cast<bool>(self); }); + + nb::class_<kv::AttnLifeCycle>(m, "AttnLifeCycle") + .def(nb::init<std::optional<int>, int>(), nb::arg("window_size"), nb::arg("num_sink_blocks")) + .def_prop_ro("window_size", [](kv::AttnLifeCycle const& self) { return self.windowSize; }) + .def_ro("num_sink_blocks", &kv::AttnLifeCycle::numSinkBlocks) + .def("get_stale_range", &kv::AttnLifeCycle::getStaleRange, nb::arg("history_length"), + nb::arg("tokens_per_block")) + .def("__eq__", &kv::AttnLifeCycle::operator==); + + nb::class_<kv::SsmLifeCycle>(m, "SsmLifeCycle") + .def(nb::init<>()) + .def( + "get_stale_range", &kv::SsmLifeCycle::getStaleRange, nb::arg("history_length"), nb::arg("tokens_per_block")) + .def("__eq__", &kv::SsmLifeCycle::operator==); + + // ---- CUDA event -------------------------------------------------------- + auto cachedCudaEvent + = nb::class_<kv::CachedCudaEvent>(m, "CachedCudaEvent") + .def(nb::init<kv::CudaStream>(), nb::arg("stream")) + .def("query_complete", &kv::CachedCudaEvent::queryComplete, nb::call_guard<nb::gil_scoped_release>()) + .def("synchronize", &kv::CachedCudaEvent::synchronize, nb::call_guard<nb::gil_scoped_release>()) + .def( + "wait_in_stream", + [](kv::CachedCudaEvent const& self, kv::CudaStream stream) { self.waitInStream(stream); }, + nb::arg("stream"), nb::call_guard<nb::gil_scoped_release>()) + .def("close", &kv::CachedCudaEvent::close) + .def("is_closed", &kv::CachedCudaEvent::isClosed); + cachedCudaEvent.attr("NULL") = kv::CachedCudaEvent::makeNull(); + + // ---- ReuseScope -------------------------------------------------------- + nb::class_<kv::ReuseScope>(m, "ReuseScope") + .def(nb::init<std::optional<kv::LoraTaskIdType>, std::optional<std::uint64_t>>(), + nb::arg("lora_id").none() = std::nullopt, nb::arg("salt").none() = std::nullopt) + .def_ro("lora_id", &kv::ReuseScope::loraId) + .def_ro("salt", &kv::ReuseScope::salt) + .def("__len__", [](kv::ReuseScope const&) { return 2; }) + .def( + "__getitem__", + [](kv::ReuseScope const& self, int index) -> nb::object + { + if (index < 0) + { + index += 2; + } + if (index == 0) + { + return optionalIntToObject(self.loraId); + } + if (index == 1) + { + return optionalIntToObject(self.salt); + } + throw nb::index_error("ReuseScope index out of range"); + }, + nb::arg("index")) + .def("__iter__", + [](kv::ReuseScope const& self) + { return nb::steal<nb::iterator>(PyObject_GetIter(reuseScopeTuple(self).ptr())); }) + .def("__hash__", [](kv::ReuseScope const& self) { return PyObject_Hash(reuseScopeTuple(self).ptr()); }) + .def( + "__eq__", + [](kv::ReuseScope const& self, nb::object other) -> nb::object + { + if (nb::isinstance<kv::ReuseScope>(other)) + { + return nb::bool_(self == nb::cast<kv::ReuseScope>(other)); + } + if (PyTuple_Check(other.ptr()) && PyTuple_GET_SIZE(other.ptr()) == 2) + { + return nb::bool_(PyObject_RichCompareBool(reuseScopeTuple(self).ptr(), other.ptr(), Py_EQ) == 1); + } + return nb::not_implemented(); + }, + nb::arg("other")) + .def("__repr__", + [](kv::ReuseScope const& self) + { + std::string repr = "ReuseScope(lora_id="; + repr += self.loraId.has_value() ? std::to_string(*self.loraId) : "None"; + repr += ", salt="; + repr += self.salt.has_value() ? std::to_string(*self.salt) : "None"; + repr += ")"; + return repr; + }); + + // ---- BufferId ---------------------------------------------------------- + nb::class_<kv::BufferId>(m, "BufferId") + .def(nb::init<kv::LayerId, kv::DataRole>(), nb::arg("layer_id"), nb::arg("role")) + .def_ro("layer_id", &kv::BufferId::layerId) + .def_ro("role", &kv::BufferId::role) + .def("__len__", [](kv::BufferId const&) { return 2; }) + .def( + "__getitem__", + [](kv::BufferId const& self, int index) -> nb::object + { + if (index < 0) + { + index += 2; + } + if (index == 0) + { + return nb::cast(self.layerId); + } + if (index == 1) + { + return nb::cast(self.role); + } + throw nb::index_error("BufferId index out of range"); + }, + nb::arg("index")) + .def("__iter__", + [](kv::BufferId const& self) + { return nb::steal<nb::iterator>(PyObject_GetIter(bufferIdTuple(self).ptr())); }) + .def("__hash__", [](kv::BufferId const& self) { return PyObject_Hash(bufferIdTuple(self).ptr()); }) + .def( + "__eq__", + [](kv::BufferId const& self, nb::object other) -> nb::object + { + if (nb::isinstance<kv::BufferId>(other)) + { + return nb::bool_(self == nb::cast<kv::BufferId>(other)); + } + if (PyTuple_Check(other.ptr()) && PyTuple_GET_SIZE(other.ptr()) == 2) + { + return nb::bool_(PyObject_RichCompareBool(bufferIdTuple(self).ptr(), other.ptr(), Py_EQ) == 1); + } + return nb::not_implemented(); + }, + nb::arg("other")); + + // ---- Storage layout structs ------------------------------------------- + nb::class_<kv::CoalescedBuffer>(m, "CoalescedBuffer") + .def_ro("single_buffer_size", &kv::CoalescedBuffer::singleBufferSize) + .def_ro("buffer_ids", &kv::CoalescedBuffer::bufferIds) + .def_prop_ro("size", &kv::CoalescedBuffer::size) + .def_prop_ro("num_buffers", &kv::CoalescedBuffer::numBuffers); + + nb::class_<kv::SlotDescVariant>(m, "SlotDescVariant") + .def_prop_ro("layer_group_id", [](kv::SlotDescVariant const& self) { return self.lifeCycleId.value(); }) + .def_prop_ro("coalesced_buffers", [](kv::SlotDescVariant const& self) { return self.coalescedBuffers.raw(); }) + .def_prop_ro("slot_size_list", + [](kv::SlotDescVariant const& self) + { + auto sizes = self.slotSizeList(); + return std::vector<int>(sizes.begin(), sizes.end()); + }); + + nb::class_<kv::SlotDesc>(m, "SlotDesc") + .def_ro("variants", &kv::SlotDesc::variants) + .def_prop_ro("slot_size_list", + [](kv::SlotDesc const& self) + { + auto sizes = self.slotSizeList(); + return std::vector<int>(sizes.begin(), sizes.end()); + }); + + nb::class_<kv::PoolDesc>(m, "PoolDesc") + .def_prop_ro("pool_index", [](kv::PoolDesc const& self) { return self.poolIndex.value(); }) + .def_ro("base_address", &kv::PoolDesc::baseAddress) + .def_ro("slot_bytes", &kv::PoolDesc::slotBytes); + + nb::class_<kv::PoolGroupDesc>(m, "PoolGroupDesc") + .def_prop_ro("pool_group_index", [](kv::PoolGroupDesc const& self) { return self.poolGroupIndex.value(); }) + .def_ro("num_slots", &kv::PoolGroupDesc::numSlots) + .def_ro("slot_desc", &kv::PoolGroupDesc::slotDesc) + .def_prop_ro("pools", [](kv::PoolGroupDesc const& self) { return self.pools.raw(); }); + + nb::class_<kv::ExpandedBuffer>(m, "ExpandedBuffer") + .def_ro("id", &kv::ExpandedBuffer::id) + .def_ro("expansion", &kv::ExpandedBuffer::expansion) + .def("__eq__", + [](kv::ExpandedBuffer const& a, kv::ExpandedBuffer const& b) + { return a.id == b.id && a.expansion == b.expansion; }); + + nb::class_<kv::AggregatedPageDesc>(m, "AggregatedPageDesc") + .def_ro("base", &kv::AggregatedPageDesc::base) + .def_ro("size", &kv::AggregatedPageDesc::size) + .def_ro("stride", &kv::AggregatedPageDesc::stride) + .def_prop_ro("layer_group_id", [](kv::AggregatedPageDesc const& self) { return self.layerGroupId.value(); }) + .def_ro("buffers", &kv::AggregatedPageDesc::buffers); + + // ---- Config structs ---------------------------------------------------- + // Helper: add __copy__ and __deepcopy__ for aggregate config types. + // All config structs are simple aggregates — default copy construction works. +#define DEF_COPY(cls) \ + .def("__copy__", [](cls const& self) { return cls(self); }) \ + .def( \ + "__deepcopy__", [](cls const& self, nb::dict) { return cls(self); }, nb::arg("memo")) + + nb::class_<kv::GpuCacheTierConfig>(m, "GpuCacheTierConfig") + .def(nb::init<size_t>(), nb::arg("quota")) + .def_rw("quota", &kv::GpuCacheTierConfig::quota) + .def_prop_ro("tier", &kv::GpuCacheTierConfig::tier) + .def("assert_valid", &kv::GpuCacheTierConfig::assertValid) DEF_COPY(kv::GpuCacheTierConfig); + + nb::class_<kv::HostCacheTierConfig>(m, "HostCacheTierConfig") + .def(nb::init<size_t>(), nb::arg("quota")) + .def_rw("quota", &kv::HostCacheTierConfig::quota) + .def_prop_ro("tier", &kv::HostCacheTierConfig::tier) + .def("assert_valid", &kv::HostCacheTierConfig::assertValid) DEF_COPY(kv::HostCacheTierConfig); + + nb::class_<kv::DiskCacheTierConfig>(m, "DiskCacheTierConfig") + .def(nb::init<size_t, std::string>(), nb::arg("quota"), nb::arg("path")) + .def_rw("quota", &kv::DiskCacheTierConfig::quota) + .def_rw("path", &kv::DiskCacheTierConfig::path) + .def_prop_ro("tier", &kv::DiskCacheTierConfig::tier) + .def("assert_valid", &kv::DiskCacheTierConfig::assertValid) DEF_COPY(kv::DiskCacheTierConfig); + + nb::class_<kv::BufferConfig>(m, "BufferConfig") + .def(nb::init<kv::DataRole, size_t, std::optional<int>>(), nb::arg("role"), nb::arg("size"), + nb::arg("tokens_per_block_override") = std::nullopt) + .def_rw("role", &kv::BufferConfig::role) + .def_rw("size", &kv::BufferConfig::size) + .def_rw("tokens_per_block_override", &kv::BufferConfig::tokensPerBlockOverride) DEF_COPY(kv::BufferConfig); + + nb::class_<kv::AttentionLayerConfig>(m, "AttentionLayerConfig") + .def(nb::init<kv::LayerId, std::vector<kv::BufferConfig>, std::optional<int>, std::optional<int>>(), + nb::arg("layer_id"), nb::arg("buffers"), nb::arg("sliding_window_size") = std::nullopt, + nb::arg("num_sink_tokens") = std::nullopt) + .def_rw("layer_id", &kv::AttentionLayerConfig::layerId) + .def_rw("buffers", &kv::AttentionLayerConfig::buffers) + .def_rw("sliding_window_size", &kv::AttentionLayerConfig::slidingWindowSize) + .def_rw("num_sink_tokens", &kv::AttentionLayerConfig::numSinkTokens) + .def_prop_ro("window_size", &kv::AttentionLayerConfig::windowSize) DEF_COPY(kv::AttentionLayerConfig); + + nb::enum_<kv::LayerType>(m, "LayerType") + .value("ATTENTION", kv::LayerType::ATTENTION) + .value("SSM", kv::LayerType::SSM); + + nb::class_<kv::SsmLayerConfig>(m, "SsmLayerConfig") + .def(nb::init<kv::LayerId, std::vector<kv::BufferConfig>>(), nb::arg("layer_id"), nb::arg("buffers")) + .def_rw("layer_id", &kv::SsmLayerConfig::layerId) + .def_rw("buffers", &kv::SsmLayerConfig::buffers) DEF_COPY(kv::SsmLayerConfig); + + nb::class_<kv::KVCacheDesc>(m, "KVCacheDesc") + .def(nb::init<int, int>(), nb::arg("capacity"), nb::arg("history_length")) + .def_rw("capacity", &kv::KVCacheDesc::capacity) + .def_rw("history_length", &kv::KVCacheDesc::historyLength) + .def("__eq__", + [](kv::KVCacheDesc const& self, nb::handle other) + { + if (!nb::isinstance<kv::KVCacheDesc>(other)) + { + return false; + } + return self == nb::cast<kv::KVCacheDesc>(other); + }) + .def("__repr__", + [](kv::KVCacheDesc const& self) + { + return "KVCacheDesc(capacity=" + std::to_string(self.capacity) + + ", history_length=" + std::to_string(self.historyLength) + ")"; + }) DEF_COPY(kv::KVCacheDesc); + + nb::class_<kv::BatchDesc>(m, "BatchDesc") + .def( + "__init__", + [](kv::BatchDesc* bd, std::vector<kv::KVCacheDesc> kvCaches, int systemPromptLength) + { + new (bd) kv::BatchDesc(); + bd->kvCaches = std::move(kvCaches); + bd->systemPromptLength = systemPromptLength; + }, + nb::arg("kv_caches"), nb::arg("system_prompt_length") = 0) + .def_rw("kv_caches", &kv::BatchDesc::kvCaches) + .def_rw("system_prompt_length", &kv::BatchDesc::systemPromptLength) + .def("__eq__", + [](kv::BatchDesc const& self, nb::handle other) + { + if (!nb::isinstance<kv::BatchDesc>(other)) + { + return false; + } + return self == nb::cast<kv::BatchDesc>(other); + }) + .def("__repr__", + [](kv::BatchDesc const& self) + { + std::string repr = "BatchDesc(kv_caches=["; + for (size_t i = 0; i < self.kvCaches.size(); ++i) + { + if (i != 0) + { + repr += ", "; + } + repr += "KVCacheDesc(capacity=" + std::to_string(self.kvCaches[i].capacity) + + ", history_length=" + std::to_string(self.kvCaches[i].historyLength) + ")"; + } + repr += "], system_prompt_length=" + std::to_string(self.systemPromptLength) + ")"; + return repr; + }) DEF_COPY(kv::BatchDesc); + + nb::class_<kv::SwaScratchReuseConfig>(m, "SwaScratchReuseConfig") + .def( + "__init__", + [](kv::SwaScratchReuseConfig* cfg, int maxRewindLen) + { + new (cfg) kv::SwaScratchReuseConfig(); + cfg->maxRewindLen = maxRewindLen; + cfg->validate(); + }, + nb::arg("max_rewind_len") = 0) + .def_rw("max_rewind_len", &kv::SwaScratchReuseConfig::maxRewindLen) DEF_COPY(kv::SwaScratchReuseConfig); + + nb::class_<kv::KVCacheManagerConfig>(m, "KVCacheManagerConfig") + .def( + "__init__", + [](kv::KVCacheManagerConfig* cfg, int tokensPerBlock, std::vector<kv::CacheTierConfig> cacheTiers, + nb::list layers, float maxUtilForResume, bool enablePartialReuse, + std::optional<kv::BatchDesc> typicalStep, std::vector<kv::BatchDesc> constraints, + std::optional<std::vector<float>> initialPoolRatio, + std::optional<kv::SwaScratchReuseConfig> swaScratchReuse, bool commitMinSnapshot, bool enableStats) + { + new (cfg) kv::KVCacheManagerConfig(); + cfg->tokensPerBlock = tokensPerBlock; + cfg->cacheTiers = std::move(cacheTiers); + // Convert Python list of AttentionLayerConfig|SsmLayerConfig to vector<LayerConfig>. + for (auto item : layers) + { + if (nb::isinstance<kv::SsmLayerConfig>(item)) + cfg->layers.push_back(nb::cast<kv::SsmLayerConfig>(item)); + else + cfg->layers.push_back(nb::cast<kv::AttentionLayerConfig>(item)); + } + cfg->maxUtilForResume = maxUtilForResume; + cfg->enablePartialReuse = enablePartialReuse; + cfg->typicalStep = std::move(typicalStep); + cfg->constraints = std::move(constraints); + cfg->initialPoolRatio = std::move(initialPoolRatio); + cfg->swaScratchReuse = std::move(swaScratchReuse); + cfg->commitMinSnapshot = commitMinSnapshot; + cfg->enableStats = enableStats; + // Mirror Python's __post_init__: validate at construction. Config-integrity + // failures raise AssertionError (translated below). + cfg->validate(); + }, + nb::arg("tokens_per_block"), nb::arg("cache_tiers"), nb::arg("layers"), + nb::arg("max_util_for_resume") = 0.97f, nb::arg("enable_partial_reuse") = true, + nb::arg("typical_step") = std::nullopt, nb::arg("constraints") = std::vector<kv::BatchDesc>{}, + nb::arg("initial_pool_ratio").none() = std::nullopt, nb::arg("swa_scratch_reuse").none() = std::nullopt, + nb::arg("commit_min_snapshot") = false, nb::arg("enable_stats") = true) + .def_rw("tokens_per_block", &kv::KVCacheManagerConfig::tokensPerBlock) + .def_rw("cache_tiers", &kv::KVCacheManagerConfig::cacheTiers) + .def_rw("layers", &kv::KVCacheManagerConfig::layers) + .def_rw("max_util_for_resume", &kv::KVCacheManagerConfig::maxUtilForResume) + .def_rw("enable_partial_reuse", &kv::KVCacheManagerConfig::enablePartialReuse) + .def_rw("typical_step", &kv::KVCacheManagerConfig::typicalStep) + .def_rw("constraints", &kv::KVCacheManagerConfig::constraints) + .def_rw("initial_pool_ratio", &kv::KVCacheManagerConfig::initialPoolRatio) + .def_rw("swa_scratch_reuse", &kv::KVCacheManagerConfig::swaScratchReuse) + .def_rw("commit_min_snapshot", &kv::KVCacheManagerConfig::commitMinSnapshot) + .def_rw("enable_stats", &kv::KVCacheManagerConfig::enableStats) + .def_prop_ro("enable_swa_scratch_reuse", &kv::KVCacheManagerConfig::enableSwaScratchReuse) + .def("validate", &kv::KVCacheManagerConfig::validate) DEF_COPY(kv::KVCacheManagerConfig); + +#undef DEF_COPY + + // ---- PlannedDropHandle ------------------------------------------------- + nb::class_<kv::PlannedDropHandle>(m, "PlannedDropHandle") + .def("drop", &kv::PlannedDropHandle::drop, nb::call_guard<nb::gil_scoped_release>()); + + // ---- KvCache ----------------------------------------------------------- + nb::class_<kv::KvCache>(m, "_KVCache") + .def( + "resume", + [](kv::KvCache& self, nb::object stream) + { + std::optional<CUstream> optStream; + if (!stream.is_none()) + optStream = reinterpret_cast<CUstream>(nb::cast<intptr_t>(stream)); + nb::gil_scoped_release rel; + return self.resume(optStream); + }, + nb::arg("cuda_stream") = nb::none()) + .def("suspend", &kv::KvCache::suspend, nb::call_guard<nb::gil_scoped_release>()) + .def( + "prefetch", [](kv::KvCache& self, int target) { return self.prefetch(kv::CacheLevel{target}); }, + nb::arg("target"), nb::call_guard<nb::gil_scoped_release>()) + .def("close", &kv::KvCache::close, nb::call_guard<nb::gil_scoped_release>()) + .def("commit_pending_stats", [](kv::KvCache& self) { return castStatsDelta(self.commitPendingStats()); }) + .def("discard_pending_stats", &kv::KvCache::discardPendingStats) + .def( + "resize", + [](kv::KvCache& self, std::optional<int> capacity, std::optional<int> historyLength) -> bool + { return self.resize(capacity, historyLength); }, + nb::arg("capacity") = std::nullopt, nb::arg("history_length") = std::nullopt, + nb::call_guard<nb::gil_scoped_release>()) + .def( + "commit", + [](kv::KvCache& self, nb::object acceptedInputTokens, nb::object beamSearchIndices, bool isEnd) + { + auto vec = castTokenIterable(acceptedInputTokens); + if (!beamSearchIndices.is_none()) + { + PyErr_SetString(PyExc_AssertionError, "beam_search_indices must be None"); + throw nb::python_error(); + } + // Note: an empty token list with is_end=True must still stop committing, + // so we do not early-return on empty; commit() handles it. + nb::gil_scoped_release release; + self.commit(vec, isEnd); + }, + nb::arg("accepted_input_tokens"), nb::arg("beam_search_indices").none() = nb::none(), + nb::arg("is_end") = false) + .def("stop_committing", &kv::KvCache::stopCommitting, nb::call_guard<nb::gil_scoped_release>()) + .def( + "get_base_page_indices", + [](kv::KvCache const& self, int layerGroupId, int beamIdx) + { + kv::Span<int const> span; + { + nb::gil_scoped_release release; + span = self.getBasePageIndices(kv::LayerGroupId{layerGroupId}, kv::BeamIndex{beamIdx}); + } + // Zero-copy: return a read-only numpy ndarray referencing the internal buffer. + // nb::handle() = no owner; the returned array does not own the data. + // Contract: callers must keep the KvCache alive and must not mutate/resize it + // while using this view. The view is intended for read-only use, matching the + // practical use of Python's array.array/memoryview index buffers. + // TODO(yaoy): switch to nb::ndarray<nb::memview> when we have nanobind >= 2.9.0, + // or nb::memoryview for nanobind >= 2.12.0. + return nb::ndarray<nb::numpy, int const, nb::ndim<1>>( + span.ptr, {static_cast<size_t>(span.len)}, nb::handle()); + }, + nb::arg("layer_group_id"), nb::arg("beam_idx") = 0) + .def( + "get_ssm_block_base_index", + [](kv::KvCache const& self, int layerGroupId, int beamId) + { return self.getSsmBlockBaseIndex(kv::LayerGroupId{layerGroupId}, kv::BeamIndex{beamId}); }, + nb::arg("layer_group_id"), nb::arg("beam_id") = 0, nb::call_guard<nb::gil_scoped_release>()) + .def( + "get_scratch_desc", + [](kv::KvCache const& self, int layerGroupId) + { return self.getScratchDesc(kv::LayerGroupId{layerGroupId}); }, + nb::arg("layer_group_id"), nb::call_guard<nb::gil_scoped_release>()) + .def_prop_ro("has_scratch_slots", &kv::KvCache::hasScratchSlots) + .def_prop_rw("enable_swa_scratch_reuse", &kv::KvCache::isSwaScratchReuseEnabled, + [](kv::KvCache& self, bool enable) { self.setEnableSwaScratchReuse(enable); }) + .def("supports_index_mode", &kv::KvCache::supportsIndexMode, nb::arg("mode")) + .def_prop_ro("status", [](kv::KvCache const& kvc) { return kvc.status(); }) + .def_prop_ro("is_active", &kv::KvCache::isActive) + .def_prop_ro("finish_event", + [](kv::KvCache const& self) + { + try + { + return self.finishEvent(); + } + catch (std::bad_optional_access const&) + { + // Python unwrap_optional(None) raises ValueError with this message. + PyErr_SetString(PyExc_ValueError, "Expected non-None value"); + throw nb::python_error(); + } + }) + .def_prop_ro("num_blocks", [](kv::KvCache const& self) { return self.numBlocks().value(); }) + .def_prop_ro("num_committed_blocks", &kv::KvCache::numCommittedBlocks) + .def_prop_ro("num_committed_tokens", &kv::KvCache::numCommittedTokens) + .def_prop_rw("history_length", &kv::KvCache::historyLength, + [](kv::KvCache& self, int hist) { self.setHistoryLength(hist); }) + .def_prop_rw("capacity", &kv::KvCache::capacity, [](kv::KvCache& self, int cap) { self.setCapacity(cap); }) + .def_prop_ro("tokens_per_block", &kv::KvCache::tokensPerBlock) + .def_prop_ro("beam_width", [](kv::KvCache const& self) { return self.beamWidth().value(); }) + .def_prop_rw( + "cuda_stream", + [](kv::KvCache const& self) -> intptr_t { return reinterpret_cast<intptr_t>(self.cudaStream()); }, + [](kv::KvCache& self, intptr_t stream) { self.setCudaStream(reinterpret_cast<CUstream>(stream)); }) + .def_rw("id", &kv::KvCache::id) + .def_prop_ro( + "manager", [](kv::KvCache& self) -> kv::KvCacheManager& { return self.manager(); }, + nb::rv_policy::reference_internal) + .def_prop_ro("committed_tokens", &committedTokensList) + .def_prop_ro("reuse_scope", &kv::KvCache::reuseScope) + .def( + "plan_committed_block_drop", [](kv::KvCache& self) { return self.planCommittedBlockDrop(); }, + nb::call_guard<nb::gil_scoped_release>()) + .def( + "get_aggregated_page_indices", + [](kv::KvCache const& self, int layerGroupId, int beamIdx, bool validOnly) { + return self.getAggregatedPageIndices(kv::LayerGroupId{layerGroupId}, kv::BeamIndex{beamIdx}, validOnly); + }, + nb::arg("layer_group_id"), nb::arg("beam_idx") = 0, nb::arg("valid_only") = false) + .def( + "set_base_page_index_buf", + [](kv::KvCache& self, int beamIdx, int layerGroupId, nb::object bufObj) + { + kv::BeamIndex const typedBeamIdx{beamIdx}; + kv::LayerGroupId const typedLayerGroupId{layerGroupId}; + if (bufObj.is_none()) + { + self.setBasePageIndexBuf(typedBeamIdx, typedLayerGroupId, nullptr, 0); + return; + } + // Accept any object exporting a 1-D writable int32 ('i') buffer. + // PyBUF_ND is required so that memoryview exports shape + format together; + // without it, Python refuses to present a non-byte format as a flat buffer. + Py_buffer view; + if (PyObject_GetBuffer(bufObj.ptr(), &view, PyBUF_WRITABLE | PyBUF_FORMAT | PyBUF_ND) != 0) + throw nb::python_error(); + struct Cleanup + { + Py_buffer* v; + ~Cleanup() + { + PyBuffer_Release(v); + } + } cleanup{&view}; + if (std::string(view.format) != "i" || view.ndim != 1) + throw std::invalid_argument("set_base_page_index_buf: buffer must be 1-D int32 ('i')"); + self.setBasePageIndexBuf(typedBeamIdx, typedLayerGroupId, static_cast<int32_t*>(view.buf), + static_cast<int>(view.len / sizeof(int32_t))); + }, + nb::arg("beam_idx"), nb::arg("layer_group_id"), nb::arg("buf").none()); + + // Make Status accessible as _KVCache.Status. + m.attr("_KVCache").attr("Status") = kvCacheStatus; + + // ---- Introspection ------------------------------------------------------- + auto mIntrospection = m.def_submodule("_introspection", "KV cache manager v2 introspection helpers"); + nb::class_<kv::StorageStatistics>(mIntrospection, "StorageStatistics") + .def_prop_ro("slot_sizes", [](kv::StorageStatistics const& self) { return self.slotSizes.raw(); }) + .def_ro("total", &kv::StorageStatistics::total) + .def_ro("free", &kv::StorageStatistics::free) + .def_ro("evictable", &kv::StorageStatistics::evictable) + .def_prop_ro("available", &kv::StorageStatistics::available) + .def_prop_ro("unavailable", &kv::StorageStatistics::unavailable); + mIntrospection.def( + "active_page_stats", + [](kv::KvCache const& kvCache) + { + auto [counts, unscheduledEvictable] = kv::KvCacheIntrospection::activePageStats(kvCache); + return std::make_tuple(std::move(counts.raw()), std::move(unscheduledEvictable.raw())); + }, + nb::arg("kv_cache"), nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def("all_tree_pages_droppable", &kv::KvCacheIntrospection::allTreePagesDroppable, nb::arg("manager"), + nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "is_commit_allowed", + [](kv::KvCache const& kvCache) { return kvCache.commitState() == kv::KvCache::CommitState::ALLOWED; }, + nb::arg("kv_cache"), nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "current_gpu_ratio", + [](kv::KvCacheManager& manager) + { + auto ratio = manager.storage().getRatioList(kv::kGpuLevel); + return std::move(ratio.raw()); + }, + nb::arg("manager"), nb::call_guard<nb::gil_scoped_release>()); + // White-box test hooks: mutate auto-tuner state so accuracy tests can force + // a pool rebalance. Mirror the Python manager's internal attributes. + mIntrospection.def( + "set_num_sampled_kv_caches", + [](kv::KvCacheManager& manager, int value) { kv::KvCacheIntrospection::setNumSampledKvCaches(manager, value); }, + nb::arg("manager"), nb::arg("value"), nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "set_last_adjustment_time", + [](kv::KvCacheManager& manager, double value) + { kv::KvCacheIntrospection::setLastAdjustmentTime(manager, value); }, + nb::arg("manager"), nb::arg("value"), nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "set_target_ratio_list_gpu", + [](kv::KvCacheManager& manager, std::vector<float> ratios) + { + kv::KvCacheIntrospection::setTargetRatioListGpu( + manager, kv::TypedVec<kv::PoolGroupIndex, float>{std::move(ratios)}); + }, + nb::arg("manager"), nb::arg("ratios"), nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "storage_statistics", + [](kv::KvCacheManager& manager, int cacheLevel) + { + auto stats = kv::KvCacheIntrospection::storageStatistics(manager, kv::CacheLevel{cacheLevel}); + return std::move(stats.raw()); + }, + nb::arg("manager"), nb::arg("cache_level") = kv::kGpuLevel.value(), nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "life_cycle_pool_group_indices", + [](kv::KvCacheManager& manager) + { + std::vector<int> result; + result.reserve(manager.lifeCycles().size().value()); + for (kv::LifeCycleId lifeCycle{0}; lifeCycle < manager.lifeCycles().size(); ++lifeCycle) + { + result.push_back(manager.storage().getPoolGroupIndex(lifeCycle).value()); + } + return result; + }, + nb::arg("manager"), nb::call_guard<nb::gil_scoped_release>()); + // White-box reuse-tree introspection: mirror the Python manager's _life_cycles + // and _radix_tree attributes so shared tests can inspect reuse state. + mIntrospection.def( + "attention_life_cycle_ids", + [](kv::KvCacheManager& manager) + { + std::vector<int> result; + for (auto const& [lcId, attn] : manager.lifeCycles().attentionLifeCycles()) + result.push_back(lcId.value()); + return result; + }, + nb::arg("manager"), nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "swa_life_cycle_ids", + [](kv::KvCacheManager& manager) + { + std::vector<int> result; + for (auto const& [lcId, attn] : manager.lifeCycles().attentionLifeCycles()) + if (attn->windowSize.has_value()) + result.push_back(lcId.value()); + return result; + }, + nb::arg("manager"), nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "ssm_life_cycle_id", + [](kv::KvCacheManager& manager) -> std::optional<int> + { + auto id = manager.lifeCycles().ssmLifeCycleId(); + if (id.has_value()) + return id->value(); + return std::nullopt; + }, + nb::arg("manager"), nb::call_guard<nb::gil_scoped_release>()); + // Returns (num_tokens, pages) where pages[i] is None for a block with no page in + // this lifecycle, else (slot_id, num_tokens_in_block) with num_tokens_in_block = -1 + // for a non-SSM (attention) page. + mIntrospection.def( + "reuse_match_pages", + [](kv::KvCacheManager& manager, nb::object reuseScope, nb::object tokens, int lcId, bool enablePartial) + { + auto rs = castReuseScope(reuseScope); + auto vec = castTokenIterable(tokens); + int numTokens = 0; + std::vector<std::optional<std::pair<int, int>>> pages; + { + nb::gil_scoped_release release; + auto matchResult = manager.radixTree().match(rs, vec, enablePartial); + numTokens = matchResult.numTokens; + kv::LifeCycleId lc{lcId}; + pages.reserve(matchResult.blocks.stdSize()); + for (auto* block : matchResult.blocks) + { + auto* page = block->storage.at(lc); + if (page == nullptr) + { + pages.emplace_back(std::nullopt); + continue; + } + int const slotId = page->slotId().value(); + int numTokensInBlock = -1; + if (auto* ssm = dynamic_cast<kv::SsmCommittedPage*>(page)) + numTokensInBlock = ssm->numTokensInBlock; + pages.emplace_back(std::make_pair(slotId, numTokensInBlock)); + } + } + return std::make_tuple(numTokens, std::move(pages)); + }, + nb::arg("manager"), nb::arg("reuse_scope"), nb::arg("tokens"), nb::arg("lc_id"), + nb::arg("enable_partial") = false); + // Returns (num_tokens, counts) where counts[i] is None for a block with no page in + // this lifecycle, else the matched page's planned_drop_count. + mIntrospection.def( + "reuse_match_planned_drop_counts", + [](kv::KvCacheManager& manager, nb::object reuseScope, nb::object tokens, int lcId, bool enablePartial) + { + auto rs = castReuseScope(reuseScope); + auto vec = castTokenIterable(tokens); + int numTokens = 0; + std::vector<std::optional<int>> counts; + { + nb::gil_scoped_release release; + auto matchResult = manager.radixTree().match(rs, vec, enablePartial); + numTokens = matchResult.numTokens; + kv::LifeCycleId lc{lcId}; + counts.reserve(matchResult.blocks.stdSize()); + for (auto* block : matchResult.blocks) + { + auto* page = block->storage.at(lc); + if (page == nullptr) + counts.emplace_back(std::nullopt); + else + counts.emplace_back(page->plannedDropCount); + } + } + return std::make_tuple(numTokens, std::move(counts)); + }, + nb::arg("manager"), nb::arg("reuse_scope"), nb::arg("tokens"), nb::arg("lc_id"), + nb::arg("enable_partial") = false); + mIntrospection.def( + "pool_group_index", + [](kv::KvCacheManager& manager, int lcId) + { return manager.storage().getPoolGroupIndex(kv::LifeCycleId{lcId}).value(); }, + nb::arg("manager"), nb::arg("lc_id")); + mIntrospection.def( + "compute_slots_for_batch", + [](kv::KvCacheManager& manager, kv::BatchDesc const& batch, int tokensPerBlock, + std::optional<kv::SwaScratchReuseConfig> const& swaScratchReuse) { + return kv::KvCacheIntrospection::computeSlotsForBatch(manager, batch, tokensPerBlock, swaScratchReuse) + .raw(); + }, + nb::arg("manager"), nb::arg("batch"), nb::arg("tokens_per_block"), nb::arg("swa_scratch_reuse") = std::nullopt); + mIntrospection.def( + "storage_utilization", + [](kv::KvCacheManager& manager, int cacheLevel) + { + auto utilization = manager.storage().getUtilization(kv::CacheLevel{cacheLevel}); + return std::move(utilization.raw()); + }, + nb::arg("manager"), nb::arg("cache_level") = kv::kGpuLevel.value(), nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "grains_for_slots", + [](kv::SlotCount numSlots, std::vector<size_t> const& slotSizeList, size_t granularity) + { + if (numSlots < 0) + { + throw std::invalid_argument("num_slots must be non-negative"); + } + return kv::CacheLevelStorage::grainsForSlots(numSlots, typedPoolSizeList(slotSizeList), granularity); + }, + nb::arg("num_slots"), nb::arg("slot_size_list"), nb::arg("granularity"), + nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "grains_to_slots", + [](size_t pgGrains, std::vector<size_t> const& slotSizeList, size_t granularity) + { + auto [slots, used] + = kv::CacheLevelStorage::grainsToSlots(pgGrains, typedPoolSizeList(slotSizeList), granularity); + return std::make_tuple(slots, used); + }, + nb::arg("pg_grains"), nb::arg("slot_size_list"), nb::arg("granularity"), + nb::call_guard<nb::gil_scoped_release>()); + mIntrospection.def( + "ratio_to_slot_count_list", + [](size_t quota, std::vector<std::vector<size_t>> const& slotSizeLists, std::vector<float> const& ratioList, + size_t granularity, std::vector<kv::SlotCount> const& minSlots) + { + auto slotCountList = kv::CacheLevelStorage::ratioToSlotCountList(quota, typedSlotSizeLists(slotSizeLists), + kv::TypedVec<kv::PoolGroupIndex, float>{ratioList}, granularity, typedSlotCounts(minSlots)); + std::vector<kv::SlotCount> result; + result.reserve(slotCountList.stdSize()); + for (kv::SlotCount slotCount : slotCountList) + { + result.push_back(slotCount); + } + return result; + }, + nb::arg("quota"), nb::arg("slot_size_lists"), nb::arg("ratio_list"), nb::arg("granularity"), + nb::arg("min_slots"), nb::call_guard<nb::gil_scoped_release>()); + + // ---- KvCacheManager ---------------------------------------------------- + nb::class_<kv::KvCacheManager>(m, "KVCacheManager") + .def( + "__init__", + [](kv::KvCacheManager* self, kv::KVCacheManagerConfig const& config, nb::object eventManager) + { + std::shared_ptr<kv::EventSink> eventSink; + if (!eventManager.is_none()) + { + eventSink = nb::cast<std::shared_ptr<kv::EventManager>>(eventManager); + } + nb::gil_scoped_release release; + new (self) kv::KvCacheManager(config, std::move(eventSink)); + }, + nb::arg("config"), nb::arg("event_manager").none() = nb::none()) + .def("shutdown", &kv::KvCacheManager::shutdown, nb::call_guard<nb::gil_scoped_release>()) + .def( + "clear_reusable_blocks", &kv::KvCacheManager::clearReusableBlocks, nb::call_guard<nb::gil_scoped_release>()) + .def( + "create_kv_cache", + [](std::shared_ptr<kv::KvCacheManager> self, nb::object reuseScopeObj, nb::object inputTokens, + std::optional<kv::RequestIdType> id, nb::object customPriorityCallback, + std::optional<int> expectedPromptLength) + { + kv::ReuseScope reuseScope = castReuseScope(std::move(reuseScopeObj)); + std::vector<kv::TokenIdExt> tokens; + bool const hasInputTokens = !inputTokens.is_none(); + if (!inputTokens.is_none()) + { + tokens = castTokenIterable(inputTokens); + } + if (!expectedPromptLength.has_value() && hasInputTokens) + { + expectedPromptLength = static_cast<int>(tokens.size()); + } + kv::KvCache::PriorityCb priorityCb = castPriorityCallback(*self, std::move(customPriorityCallback)); + nb::gil_scoped_release release; + return self->createKvCache( + std::move(reuseScope), tokens, id, std::move(priorityCb), expectedPromptLength); + }, + nb::arg("reuse_scope") = nb::none(), nb::arg("input_tokens") = nb::none(), nb::arg("id") = std::nullopt, + nb::arg("custom_priority_callback") = nb::none(), nb::arg("expected_prompt_length") = std::nullopt) + .def( + "probe_reuse", + [](std::shared_ptr<kv::KvCacheManager> self, nb::object reuseScopeObj, nb::object inputTokens) + { + kv::ReuseScope reuseScope = castReuseScope(std::move(reuseScopeObj)); + std::vector<kv::TokenIdExt> tokens; + if (!inputTokens.is_none()) + { + tokens = castTokenIterable(inputTokens); + } + nb::gil_scoped_release release; + return self->probeReuse(std::move(reuseScope), tokens); + }, + nb::arg("reuse_scope") = nb::none(), nb::arg("input_tokens") = nb::none()) + .def("get_mem_pool_base_address", &kv::KvCacheManager::getMemPoolBaseAddress, nb::arg("layer_id"), + nb::arg("data_role"), nb::arg("index_mode") = std::nullopt, nb::call_guard<nb::gil_scoped_release>()) + .def("get_page_stride", &kv::KvCacheManager::getPageStride, nb::arg("layer_id"), nb::arg("data_role")) + .def("get_page_index_scale", &kv::KvCacheManager::getPageIndexScale, nb::arg("layer_id"), nb::arg("data_role")) + .def("get_page_index_upper_bound", &kv::KvCacheManager::getPageIndexUpperBound, nb::arg("layer_id"), + nb::arg("data_role")) + .def( + "resize", + [](kv::KvCacheManager& self, int cacheLevel, size_t quota, bool bestEfforts) + { return self.resize(kv::CacheLevel{cacheLevel}, quota, bestEfforts); }, + nb::arg("cache_level"), nb::arg("quota"), nb::arg("best_efforts") = false, + nb::call_guard<nb::gil_scoped_release>()) + .def( + "get_quota", + [](kv::KvCacheManager const& self, int cacheLevel) { return self.getQuota(kv::CacheLevel{cacheLevel}); }, + nb::arg("cache_level")) + .def("get_committed_stats", + [](kv::KvCacheManager const& self) { return castStatsDelta(self.getCommittedStats()); }) + .def("get_and_reset_iteration_stats", + [](kv::KvCacheManager& self) { return castIterationStatsByLifeCycle(self.getAndResetIterationStats()); }) + .def("get_and_reset_ssm_snapshot_iteration_stats", + [](kv::KvCacheManager& self) + { return castSsmSnapshotIterationStatsByLifeCycle(self.getAndResetSsmSnapshotIterationStats()); }) + .def( + "get_and_reset_iteration_peak_block_stats", + [](kv::KvCacheManager& self, int cacheLevel) + { return castPeakBlockStats(self.getAndResetIterationPeakBlockStats(kv::CacheLevel{cacheLevel})); }, + nb::arg("cache_level")) + .def("mark_stats_dirty", &kv::KvCacheManager::markStatsDirty, nb::arg("kv_cache_id").none()) + .def("clear_stats_dirty", &kv::KvCacheManager::clearStatsDirty, nb::arg("kv_cache_id").none()) + .def("get_dirty_stats_kv_cache_ids", + [](kv::KvCacheManager const& self) { return castRequestIds(self.getDirtyStatsKvCacheIds()); }) + .def("mark_stats_excluded", &kv::KvCacheManager::markStatsExcluded, nb::arg("kv_cache_id").none()) + .def("clear_stats_excluded", &kv::KvCacheManager::clearStatsExcluded, nb::arg("kv_cache_id").none()) + .def("is_stats_excluded", &kv::KvCacheManager::isStatsExcluded, nb::arg("kv_cache_id").none()) + .def_prop_ro("tokens_per_block", &kv::KvCacheManager::tokensPerBlock) + .def_prop_ro("event_manager", + [](kv::KvCacheManager const& self) + { return std::dynamic_pointer_cast<kv::EventManager>(self.eventSink()); }) + .def_prop_ro("init_config", [](kv::KvCacheManager const& self) { return self.config(); }) + .def_prop_ro("cache_tier_list", [](kv::KvCacheManager const& self) { return self.cacheTierList().raw(); }) + .def_prop_ro("all_buffer_ids", &kv::KvCacheManager::allBufferIds) + .def_prop_ro("pool_group_descs", [](kv::KvCacheManager const& self) { return self.poolGroupDescs().raw(); }) + .def("clamp_max_seq_len_for_mem", &kv::KvCacheManager::clampMaxSeqLenForMem, nb::arg("batch_size"), + nb::arg("token_num_upper_bound"), nb::call_guard<nb::gil_scoped_release>()) + .def_prop_ro("allow_seq_rebasing", &kv::KvCacheManager::allowSeqRebasing) + .def_prop_ro("enable_partial_match", &kv::KvCacheManager::enablePartialMatch) + .def_prop_ro("enable_swa_scratch_reuse", &kv::KvCacheManager::isSwaScratchReuseEnabled) + .def( + "supports_index_mode", + [](kv::KvCacheManager const& self, kv::PageIndexMode mode) -> nb::object + { + auto result = self.supportsIndexMode(mode); + if (!result.has_value()) + return nb::none(); + return nb::cast(*result); + }, + nb::arg("mode")) + .def_prop_ro("commit_min_snapshot", &kv::KvCacheManager::commitMinSnapshot) + .def_prop_ro("num_layers", &kv::KvCacheManager::numLayers) + .def_prop_ro("layer_ids", &kv::KvCacheManager::layerIds) + .def_prop_ro( + "layer_grouping", [](kv::KvCacheManager const& self) { return self.layerGrouping().raw(); }, + "Layers grouped by shared lifecycle/pool allocation. The iteration order of the " + "layer lists (and of the groups) is NOT an API contract and may differ across " + "backends/runs; do not rely on it for buffer/pool memory order -- use " + "pool_group_descs (PoolGroupDesc.pools[i].base_address + coalesced_buffers) instead.") + .def( + "get_layer_group_id", + [](kv::KvCacheManager const& self, kv::LayerId layerId) { return self.getLayerGroupId(layerId).value(); }, + nb::arg("layer_id")) + .def("get_page_index_converter", &kv::KvCacheManager::getPageIndexConverter, nb::arg("layer_id"), + nb::arg("data_role")) + .def( + "get_aggregated_pages", + [](kv::KvCacheManager const& self, nb::object buffers) + { + std::vector<kv::BufferId> ids; + for (auto item : nb::cast<nb::iterable>(buffers)) + ids.push_back(nb::cast<kv::BufferId>(item)); + nb::gil_scoped_release release; + return self.getAggregatedPages(ids); + }, + nb::arg("buffers")) + .def("adjust", &kv::KvCacheManager::adjust, nb::call_guard<nb::gil_scoped_release>()) + .def_prop_ro("need_adjustment", &kv::KvCacheManager::needAdjustment); + + // ---- PageIndexConverter ------------------------------------------------ + nb::class_<kv::PageIndexConverter>(m, "PageIndexConverter") + .def_ro("scale", &kv::PageIndexConverter::scale) + .def_ro("expansion", &kv::PageIndexConverter::expansion) + .def_ro("layer_offset", &kv::PageIndexConverter::layerOffset) + .def_ro("scratch_pages_per_block", &kv::PageIndexConverter::scratchPagesPerBlock) + .def( + "__call__", + [](kv::PageIndexConverter const& self, std::vector<int> const& baseIndices, + std::optional<kv::PageIndexMode> indexMode, nb::object scratchObj) -> std::vector<int> + { + kv::ScratchDesc const* scratch = nullptr; + std::optional<kv::ScratchDesc> scratchHolder; + if (!scratchObj.is_none()) + { + scratchHolder = nb::cast<kv::ScratchDesc>(scratchObj); + scratch = &*scratchHolder; + } + return self(baseIndices, indexMode, scratch); + }, + nb::arg("base_indices"), nb::arg("index_mode") = nb::none(), nb::arg("scratch") = nb::none()); +} + +} // namespace tensorrt_llm::nanobind::batch_manager diff --git a/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.h b/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManagerV2.h similarity index 69% rename from cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.h rename to cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManagerV2.h index 1aababc6ff89..c2bb4a51c1b6 100644 --- a/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.h +++ b/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManagerV2.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,11 +19,12 @@ #include <nanobind/nanobind.h> -namespace nb = nanobind; - -namespace tensorrt_llm::nanobind::testing +namespace tensorrt_llm::nanobind::batch_manager { -void initBindings(nb::module_& m); +struct KvCacheManagerV2Bindings +{ + static void initBindings(::nanobind::module_& m); +}; -} // namespace tensorrt_llm::nanobind::testing +} // namespace tensorrt_llm::nanobind::batch_manager diff --git a/cpp/tensorrt_llm/nanobind/bindings.cpp b/cpp/tensorrt_llm/nanobind/bindings.cpp index db263fa639f9..a2054dbd7217 100644 --- a/cpp/tensorrt_llm/nanobind/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/bindings.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -32,12 +32,14 @@ #include "tensorrt_llm/batch_manager/peftCacheManagerConfig.h" #include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/nanobind/batch_manager/algorithms.h" #include "tensorrt_llm/nanobind/batch_manager/bindings.h" #include "tensorrt_llm/nanobind/batch_manager/buffers.h" #include "tensorrt_llm/nanobind/batch_manager/cacheTransceiver.h" #include "tensorrt_llm/nanobind/batch_manager/kvCacheConnector.h" #include "tensorrt_llm/nanobind/batch_manager/kvCacheManager.h" +#include "tensorrt_llm/nanobind/batch_manager/kvCacheManagerV2.h" #include "tensorrt_llm/nanobind/batch_manager/kvCacheManagerV2Utils.h" #include "tensorrt_llm/nanobind/batch_manager/llmRequest.h" #include "tensorrt_llm/nanobind/common/tllmExceptions.h" @@ -46,7 +48,6 @@ #include "tensorrt_llm/nanobind/runtime/bindings.h" #include "tensorrt_llm/nanobind/suffixAutomaton/bindings.h" #include "tensorrt_llm/nanobind/testing/kvCacheManagerTestUtilBinding.h" -#include "tensorrt_llm/nanobind/testing/modelSpecBinding.h" #include "tensorrt_llm/nanobind/thop/bindings.h" #include "tensorrt_llm/nanobind/userbuffers/bindings.h" #include "tensorrt_llm/runtime/common.h" @@ -135,6 +136,9 @@ NB_MODULE(TRTLLM_NB_MODULE, m) auto mInternalBatchManager = mInternal.def_submodule("batch_manager", "Batch manager internal bindings"); auto mInternalBatchManagerKvCacheV2Utils = mInternalBatchManager.def_submodule("kv_cache_manager_v2_utils", "KV Cache Manager V2 Utils bindings"); + auto mInternalBatchManagerKvCacheV2 + = mInternalBatchManager.def_submodule("kv_cache_manager_v2", "KV Cache Manager V2 bindings"); + tensorrt_llm::nanobind::batch_manager::KvCacheManagerV2Bindings::initBindings(mInternalBatchManagerKvCacheV2); auto mInternalThop = mInternal.def_submodule("thop", "Torch op internal bindings"); auto mExceptions = m.def_submodule("exceptions", "Exceptions internal bindings"); @@ -168,17 +172,17 @@ NB_MODULE(TRTLLM_NB_MODULE, m) .def_rw("host_cache_size", &tb::PeftCacheManagerConfig::hostCacheSize) .def_rw("lora_prefetch_dir", &tb::PeftCacheManagerConfig::loraPrefetchDir); - nb::enum_<nvinfer1::DataType>(m, "DataType") - .value("FLOAT", nvinfer1::DataType::kFLOAT) - .value("HALF", nvinfer1::DataType::kHALF) - .value("INT8", nvinfer1::DataType::kINT8) - .value("INT32", nvinfer1::DataType::kINT32) - .value("BOOL", nvinfer1::DataType::kBOOL) - .value("UINT8", nvinfer1::DataType::kUINT8) - .value("FP8", nvinfer1::DataType::kFP8) - .value("BF16", nvinfer1::DataType::kBF16) - .value("INT64", nvinfer1::DataType::kINT64) - .value("NVFP4", nvinfer1::DataType::kFP4) + nb::enum_<tensorrt_llm::DataType>(m, "DataType") + .value("FLOAT", tensorrt_llm::DataType::kFLOAT) + .value("HALF", tensorrt_llm::DataType::kHALF) + .value("INT8", tensorrt_llm::DataType::kINT8) + .value("INT32", tensorrt_llm::DataType::kINT32) + .value("BOOL", tensorrt_llm::DataType::kBOOL) + .value("UINT8", tensorrt_llm::DataType::kUINT8) + .value("FP8", tensorrt_llm::DataType::kFP8) + .value("BF16", tensorrt_llm::DataType::kBF16) + .value("INT64", tensorrt_llm::DataType::kINT64) + .value("NVFP4", tensorrt_llm::DataType::kFP4) .export_values(); nb::enum_<tr::ModelConfig::ModelVariant>(m, "GptModelVariant") @@ -295,7 +299,7 @@ NB_MODULE(TRTLLM_NB_MODULE, m) .def(nb::self != nb::self); nb::class_<tr::ModelConfig>(m, "ModelConfig") - .def(nb::init<SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, nvinfer1::DataType>(), + .def(nb::init<SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, tensorrt_llm::DataType>(), nb::arg("vocab_size"), nb::arg("num_layers"), nb::arg("num_attention_layers"), nb::arg("num_rnn_layers"), nb::arg("num_heads"), nb::arg("hidden_size"), nb::arg("data_type")) .def_prop_ro("vocab_size", &tr::ModelConfig::getVocabSize) @@ -512,7 +516,6 @@ NB_MODULE(TRTLLM_NB_MODULE, m) tensorrt_llm::nanobind::process_group::initBindings(mInternalProcessGroup); tpb::Buffers::initBindings(mInternalBatchManager); tensorrt_llm::nanobind::runtime::initBindings(mInternalRuntime); - tensorrt_llm::nanobind::testing::initBindings(mInternalTesting); tensorrt_llm::nanobind::testing::initKvCacheTestUtilBindings(mInternalTesting); tpb::initBindings(mInternalBatchManager); @@ -545,4 +548,8 @@ NB_MODULE(TRTLLM_NB_MODULE, m) m.def("ipc_nvls_supported", &tr::ipcNvlsSupported); m.def("steady_clock_now", []() { return std::chrono::steady_clock::now(); }); + // Global (offset-normalized) steady clock, matching what + // LlmRequest::setKvCacheTransferStart/End expect. Reads the process-global + // steady clock offset, set by PyExecutor at startup. + m.def("global_steady_clock_now", []() { return tb::LlmRequest::getSteadyClockNow(); }); } diff --git a/cpp/tensorrt_llm/nanobind/executor/bindings.cpp b/cpp/tensorrt_llm/nanobind/executor/bindings.cpp index b0ad31b7347e..a8d2301fa43d 100644 --- a/cpp/tensorrt_llm/nanobind/executor/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/executor/bindings.cpp @@ -16,7 +16,6 @@ */ #include "bindings.h" -#include "executor.h" #include "executorConfig.h" #include "request.h" #include "tensorrt_llm/executor/executor.h" @@ -287,7 +286,6 @@ void initBindings(nb::module_& m) tensorrt_llm::nanobind::executor::initRequestBindings(m); tensorrt_llm::nanobind::executor::initConfigBindings(m); - tensorrt_llm::nanobind::executor::Executor::initBindings(m); } } // namespace tensorrt_llm::nanobind::executor diff --git a/cpp/tensorrt_llm/nanobind/executor/executor.cpp b/cpp/tensorrt_llm/nanobind/executor/executor.cpp deleted file mode 100644 index 34cc8182d1bb..000000000000 --- a/cpp/tensorrt_llm/nanobind/executor/executor.cpp +++ /dev/null @@ -1,225 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "executor.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/tensor.h" -#include "tensorrt_llm/nanobind/common/customCasters.h" - -#include <nanobind/nanobind.h> -#include <nanobind/ndarray.h> -#include <nanobind/stl/chrono.h> -#include <nanobind/stl/filesystem.h> -#include <nanobind/stl/map.h> -#include <nanobind/stl/optional.h> -#include <nanobind/stl/shared_ptr.h> -#include <nanobind/stl/string.h> -#include <nanobind/stl/vector.h> -#include <torch/extension.h> - -namespace nb = nanobind; -namespace tle = tensorrt_llm::executor; - -namespace nanobind::detail -{ - -template <> -struct dtype_traits<half> -{ - static constexpr dlpack::dtype value{ - (uint8_t) dlpack::dtype_code::Float, // type code - 16, // size in bits - 1 // lanes (simd), usually set to 1 - }; - static constexpr auto name = const_name("float16"); -}; -} // namespace nanobind::detail - -namespace -{ -tle::Tensor numpyToTensor(nb::object const& object) -{ - std::string dtype_name = nb::cast<std::string>(object.attr("dtype").attr("name")); - nb::object metadata = object.attr("dtype").attr("metadata"); - - tle::DataType dtype; - if (dtype_name == "float16") - { - dtype = tle::DataType::kFP16; - } - else if (dtype_name == "float32") - { - dtype = tle::DataType::kFP32; - } - else if (dtype_name == "int8") - { - dtype = tle::DataType::kINT8; - } - else if (dtype_name == "int32") - { - dtype = tle::DataType::kINT32; - } - else if (dtype_name == "int64") - { - dtype = tle::DataType::kINT64; - } - else if (dtype_name == "void8" && !metadata.is_none() && nb::cast<std::string>(metadata["dtype"]) == "float8") - { - dtype = tle::DataType::kFP8; - } - else if (dtype_name == "void16" && !metadata.is_none() && nb::cast<std::string>(metadata["dtype"]) == "bfloat16") - { - dtype = tle::DataType::kBF16; - } - else - { - TLLM_THROW("Unsupported numpy dtype."); - } - - nb::object array_interface = object.attr("__array_interface__"); - nb::object shape_obj = array_interface["shape"]; - std::vector<int64_t> dims; - dims.reserve(nb::len(shape_obj)); - - for (size_t i = 0; i < nb::len(shape_obj); ++i) - { - dims.push_back(nb::cast<int64_t>(shape_obj[i])); - } - - nb::object data_obj = array_interface["data"]; - uintptr_t addr = nb::cast<uintptr_t>(data_obj[0]); - void* data_ptr = reinterpret_cast<void*>(addr); - tle::Shape shape(dims.data(), dims.size()); - return tle::Tensor::of(dtype, data_ptr, shape); -} - -} // namespace - -namespace tensorrt_llm::nanobind::executor -{ - -Executor::Executor( - std::filesystem::path const& modelPath, tle::ModelType modelType, tle::ExecutorConfig const& executorConfig) -{ - mExecutor = std::make_unique<tle::Executor>(modelPath, modelType, executorConfig); -} - -Executor::Executor(std::filesystem::path const& encoderModelPath, std::filesystem::path const& decoderModelPath, - tle::ModelType modelType, tle::ExecutorConfig const& executorConfig) -{ - mExecutor = std::make_unique<tle::Executor>(encoderModelPath, decoderModelPath, modelType, executorConfig); -} - -Executor::Executor(nb::bytes const& engineBuffer, std::string const& jsonConfigStr, tle::ModelType modelType, - tle::ExecutorConfig const& executorConfig, std::optional<nb::dict> managedWeights) -{ - uint8_t const* data = static_cast<uint8_t const*>(engineBuffer.data()); - size_t size = engineBuffer.size(); - std::optional<std::map<std::string, tle::Tensor>> managedWeightsMap = std::nullopt; - if (managedWeights.has_value() && !managedWeights.value().empty()) - { - managedWeightsMap = std::map<std::string, tle::Tensor>(); - for (auto const& [rawName, rawArray] : managedWeights.value()) - { - std::string name = nb::cast<std::string>(rawName); - nb::object array_obj = nb::cast<nb::object>(rawArray); - managedWeightsMap->emplace(name, numpyToTensor(array_obj)); - } - } - mExecutor = std::make_unique<tle::Executor>( - tle::BufferView(data, size), jsonConfigStr, modelType, executorConfig, managedWeightsMap); -} - -Executor::Executor(std::string const& encoderEngineBuffer, std::string const& encoderJsonConfigStr, - std::string const& decoderEngineBuffer, std::string const& decoderJsonConfigStr, tle::ModelType modelType, - tle::ExecutorConfig const& executorConfig) -{ - uint8_t const* encoderData = reinterpret_cast<uint8_t const*>(encoderEngineBuffer.data()); - size_t encoderSize = encoderEngineBuffer.size(); - uint8_t const* decoderData = reinterpret_cast<uint8_t const*>(decoderEngineBuffer.data()); - size_t decoderSize = decoderEngineBuffer.size(); - mExecutor = std::make_unique<tle::Executor>(tle::BufferView(encoderData, encoderSize), encoderJsonConfigStr, - tle::BufferView(decoderData, decoderSize), decoderJsonConfigStr, modelType, executorConfig); -} - -nb::object Executor::enter() -{ - TLLM_CHECK(static_cast<bool>(mExecutor)); - return nb::cast(this); -} - -void Executor::exit( - [[maybe_unused]] nb::handle type, [[maybe_unused]] nb::handle value, [[maybe_unused]] nb::handle traceback) -{ - shutdown(); - mExecutor = nullptr; -} - -void Executor::shutdown() -{ - // NOTE: we must release the GIL here. Executor has spawned a thread for the execution loop. That thread must be - // able to do forward progress for the shutdown process to succeed. It takes the GIL during its callbacks, so - // we release it now. Note that we shouldn't do anything related to python objects after that. - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - nb::gil_scoped_release release; - mExecutor->shutdown(); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void Executor::initBindings(nb::module_& m) -{ - nb::class_<Executor>(m, "Executor") - .def(nb::init<std::filesystem::path const&, tle::ModelType, tle::ExecutorConfig const&>(), - nb::arg("model_path"), nb::arg("model_type"), nb::arg("executor_config")) - .def(nb::init<std::filesystem::path const&, std::filesystem::path const&, tle::ModelType, - tle::ExecutorConfig const&>(), - nb::arg("encoder_model_path"), nb::arg("decoder_model_path"), nb::arg("model_type"), - nb::arg("executor_config")) - .def(nb::init<nb::bytes, std::string const&, tle::ModelType, tle::ExecutorConfig const&, nb::dict>(), - nb::arg("engine_buffer"), nb::arg("json_config_str"), nb::arg("model_type"), nb::arg("executor_config"), - nb::arg("managed_weights") = nb::dict()) - .def(nb::init<std::string const&, std::string const&, std::string const&, std::string const&, tle::ModelType, - tle::ExecutorConfig const&>(), - nb::arg("encoder_engine_buffer"), nb::arg("encoder_json_config_str"), nb::arg("decoder_engine_buffer"), - nb::arg("decoder_json_config_str"), nb::arg("model_type"), nb::arg("executor_config")) - .def("shutdown", &Executor::shutdown) - .def("__enter__", &Executor::enter) - .def("__exit__", &Executor::exit, nb::arg("type").none(), nb::arg("value").none(), nb::arg("traceback").none()) - .def("enqueue_request", &Executor::enqueueRequest, nb::arg("request")) - .def("enqueue_requests", &Executor::enqueueRequests, nb::arg("requests")) - .def("await_responses", - nb::overload_cast<std::optional<std::chrono::milliseconds> const&>(&Executor::awaitResponses), - nb::arg("timeout") = nb::none()) - .def("await_responses", - nb::overload_cast<tle::IdType const&, std::optional<std::chrono::milliseconds> const&>( - &Executor::awaitResponses), - nb::arg("id"), nb::arg("timeout") = nb::none()) - .def("await_responses", - nb::overload_cast<std::vector<tle::IdType> const&, std::optional<std::chrono::milliseconds> const&>( - &Executor::awaitResponses), - nb::arg("ids"), nb::arg("timeout") = nb::none()) - .def("get_num_responses_ready", &Executor::getNumResponsesReady, nb::arg("id") = nb::none()) - .def("cancel_request", &Executor::cancelRequest, nb::arg("id") = nb::none()) - .def("get_latest_iteration_stats", &Executor::getLatestIterationStats) - .def("get_latest_request_stats", &Executor::getLatestRequestStats) - .def("get_latest_debug_tensors", &Executor::getLatestDebugTensors) - .def("can_enqueue_requests", &Executor::canEnqueueRequests) - .def("get_kv_cache_event_manager", &Executor::getKVCacheEventManager); -} - -} // namespace tensorrt_llm::nanobind::executor diff --git a/cpp/tensorrt_llm/nanobind/executor/executor.h b/cpp/tensorrt_llm/nanobind/executor/executor.h deleted file mode 100644 index 22c24abb4bfd..000000000000 --- a/cpp/tensorrt_llm/nanobind/executor/executor.h +++ /dev/null @@ -1,129 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" -#include <nanobind/nanobind.h> - -namespace nb = nanobind; -namespace tle = tensorrt_llm::executor; - -namespace tensorrt_llm::nanobind::executor -{ - -class Executor -{ -public: - Executor( - std::filesystem::path const& modelPath, tle::ModelType modelType, tle::ExecutorConfig const& executorConfig); - - Executor(std::filesystem::path const& encoderModelPath, std::filesystem::path const& decoderModelPath, - tle::ModelType modelType, tle::ExecutorConfig const& executorConfig); - - Executor(nb::bytes const& engineBuffer, std::string const& jsonConfigStr, tle::ModelType modelType, - tle::ExecutorConfig const& executorConfig, std::optional<nb::dict> managedWeights); - - Executor(std::string const& encoderEngineBuffer, std::string const& encoderJsonConfigStr, - std::string const& decoderEngineBuffer, std::string const& decoderJsonConfigStr, tle::ModelType modelType, - tle::ExecutorConfig const& executorConfig); - - nb::object enter(); - void exit( - [[maybe_unused]] nb::handle type, [[maybe_unused]] nb::handle value, [[maybe_unused]] nb::handle traceback); - void shutdown(); - - [[nodiscard]] tle::IdType enqueueRequest(tle::Request const& request) - { - return mExecutor->enqueueRequest(request); - } - - [[nodiscard]] std::vector<tle::IdType> enqueueRequests(std::vector<tle::Request> const& requests) - { - return mExecutor->enqueueRequests(requests); - } - - [[nodiscard]] std::vector<tle::Response> awaitResponses( - std::optional<std::chrono::milliseconds> const& timeout = std::nullopt) - { - // Await responses blocks until a response is received. Release GIL so that it can be ran in a background - // thread. - nb::gil_scoped_release release; - return mExecutor->awaitResponses(timeout); - } - - [[nodiscard]] std::vector<tle::Response> awaitResponses( - tle::IdType const& requestId, std::optional<std::chrono::milliseconds> const& timeout = std::nullopt) - { - // Await responses blocks until a response is received. Release GIL so that it can be ran in a background - // thread. - nb::gil_scoped_release release; - return mExecutor->awaitResponses(requestId, timeout); - } - - [[nodiscard]] std::vector<std::vector<tle::Response>> awaitResponses(std::vector<tle::IdType> const& requestIds, - std::optional<std::chrono::milliseconds> const& timeout = std::nullopt) - { - // Await responses blocks until a response is received. Release GIL so that it can be ran in a background - // thread. - nb::gil_scoped_release release; - return mExecutor->awaitResponses(requestIds, timeout); - } - - [[nodiscard]] tle::SizeType32 getNumResponsesReady(std::optional<tle::IdType> const& requestId = std::nullopt) const - { - return mExecutor->getNumResponsesReady(requestId); - } - - void cancelRequest(tle::IdType requestId) - { - mExecutor->cancelRequest(requestId); - } - - std::deque<tle::IterationStats> getLatestIterationStats() - { - return mExecutor->getLatestIterationStats(); - } - - std::deque<tle::RequestStatsPerIteration> getLatestRequestStats() - { - return mExecutor->getLatestRequestStats(); - } - - std::deque<tle::DebugTensorsPerIteration> getLatestDebugTensors() - { - return mExecutor->getLatestDebugTensors(); - } - - [[nodiscard]] bool canEnqueueRequests() const - { - return mExecutor->canEnqueueRequests(); - } - - [[nodiscard]] std::optional<std::shared_ptr<tle::KVCacheEventManager>> getKVCacheEventManager() const - { - return mExecutor->getKVCacheEventManager(); - } - - static void initBindings(nb::module_& m); - -private: - std::unique_ptr<tle::Executor> mExecutor; -}; - -} // namespace tensorrt_llm::nanobind::executor diff --git a/cpp/tensorrt_llm/nanobind/executor/executorConfig.cpp b/cpp/tensorrt_llm/nanobind/executor/executorConfig.cpp index 830f30ab9c67..acd33c0df769 100644 --- a/cpp/tensorrt_llm/nanobind/executor/executorConfig.cpp +++ b/cpp/tensorrt_llm/nanobind/executor/executorConfig.cpp @@ -171,6 +171,9 @@ void initConfigBindings(nb::module_& m) .def("__getstate__", kvCacheConfigGetstate) .def("__setstate__", kvCacheConfigSetstate); + // Deprecated: orchestrator mode is non-functional (its executorWorker binary was + // removed with the TensorRT backend); binding kept for compatibility, removal is a + // follow-up pending API-stability review. nb::class_<tle::OrchestratorConfig>(m, "OrchestratorConfig") .def(nb::init<bool, std::string, std::shared_ptr<mpi::MpiComm>, bool>(), nb::arg("is_orchestrator") = true, nb::arg("worker_executable_path") = "", nb::arg("orch_leader_comm").none() = nullptr, diff --git a/cpp/tensorrt_llm/nanobind/runtime/bindings.cpp b/cpp/tensorrt_llm/nanobind/runtime/bindings.cpp index 6d5d70aafb6b..eec3cd79bac1 100644 --- a/cpp/tensorrt_llm/nanobind/runtime/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/runtime/bindings.cpp @@ -18,6 +18,7 @@ #include "bindings.h" #include "hostfunc.h" #include "moeBindings.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceWorkspace.h" #include "tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h" #include "tensorrt_llm/kernels/customAllReduceKernels.h" @@ -39,7 +40,6 @@ #include "tensorrt_llm/runtime/loraCache.h" #include "tensorrt_llm/runtime/mcastGPUBuffer.h" #include "tensorrt_llm/runtime/speculativeDecodingMode.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" #include "tensorrt_llm/runtime/torchView.h" #include "tensorrt_llm/runtime/virtualMemory.h" @@ -68,7 +68,7 @@ class PyIGptDecoder : public tr::IGptDecoder void setup(tr::SamplingConfig const& samplingConfig, size_t batchSize, tr::DecodingInput::TensorConstPtr const& batchSlots, std::optional<tr::DecodingOutput> const& output = std::nullopt, - std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt, + std::optional<tensorrt_llm::DataType> explicitDraftTokensDType = std::nullopt, std::optional<std::vector<tr::ITensor::SharedConstPtr>> const& lookaheadPrompt = std::nullopt, std::optional<std::vector<te::LookaheadDecodingConfig>> const& lookaheadAlgoConfigs = std::nullopt) override { @@ -125,47 +125,6 @@ void initBindings(nb::module_& m) .def("materialize_with_tag", &tr::CudaVirtualMemoryManager::materializeWithTag, nb::arg("tag"), nb::call_guard<nb::gil_scoped_release>()); - nb::class_<tr::TllmRuntime>(m, "TllmRuntime") - .def( - "__init__", - [](tr::TllmRuntime* self, std::filesystem::path engine_path, float gpu_weights_percent = 1.0f, - bool use_shape_inference = true) - { - // Using default logger by passing nullptr - new (self) - tr::TllmRuntime(tr::RawEngine(engine_path), nullptr, gpu_weights_percent, use_shape_inference); - }, - nb::arg("engine_path"), nb::arg("gpu_weights_percent") = 1.0f, nb::arg("use_shape_inference") = true) - .def( - "__init__", - [](tr::TllmRuntime* self, nb::ndarray<nb::numpy, uint8_t> engine_buffer, float gpu_weights_percent = 1.0f, - bool use_shape_inference = true) - { - if (engine_buffer.ndim() != 1) - throw std::runtime_error("Expected 1-D array for engine buffer"); - new (self) tr::TllmRuntime(tr::RawEngine(engine_buffer.data(), engine_buffer.size()), nullptr, - gpu_weights_percent, use_shape_inference); - }, - nb::arg("engine_buffer"), nb::arg("gpu_weights_percent") = 1.0f, nb::arg("use_shape_inference") = true) - .def_prop_ro("num_contexts", &tr::TllmRuntime::getNbContexts) - .def_prop_ro("num_profiles", &tr::TllmRuntime::getNbProfiles) - .def("get_opt_profile_id", &tr::TllmRuntime::getOptProfileId, nb::arg("num_tokens"), nb::arg("split_points"), - nb::call_guard<nb::gil_scoped_release>()) - .def("clear_contexts", &tr::TllmRuntime::clearContexts, nb::call_guard<nb::gil_scoped_release>()) - .def("execute_context", &tr::TllmRuntime::executeContext, nb::arg("context_id"), - nb::call_guard<nb::gil_scoped_release>()) - .def_prop_ro("stream_ptr", &tr::TllmRuntime::getStreamPtr) - .def_prop_ro("buffer_manager", - static_cast<tr::BufferManager& (tr::TllmRuntime::*) ()>(&tr::TllmRuntime::getBufferManager)) - .def("set_layer_profiler", &tr::TllmRuntime::setLayerProfiler, nb::call_guard<nb::gil_scoped_release>()) - .def("has_layer_profiler", &tr::TllmRuntime::hasLayerProfiler, nb::arg("context_id"), - nb::call_guard<nb::gil_scoped_release>()) - .def_prop_ro("layer_profiler_info", &tr::TllmRuntime::getLayerProfileInfo) - .def("report_to_profiler", &tr::TllmRuntime::reportToProfiler, nb::arg("context_id"), - nb::call_guard<nb::gil_scoped_release>()) - .def_prop_ro("logits_dtype_from_engine", - [](tr::TllmRuntime& self) { return self.getEngine().getTensorDataType("logits"); }); - nb::class_<tr::LookaheadDecodingBuffers>(m, "LookaheadDecodingBuffers") .def(nb::init<tr::SizeType32, tr::SizeType32, tr::BufferManager const&>(), nb::arg("max_num_sequences"), nb::arg("max_tokens_per_step"), nb::arg("buffer_manager"), nb::call_guard<nb::gil_scoped_release>()) @@ -204,7 +163,7 @@ void initBindings(nb::module_& m) "setup", [](tr::IGptDecoder& self, tr::SamplingConfig const& samplingConfig, size_t batchSize, at::Tensor const& batchSlots, std::optional<tr::DecodingOutput> const& output = std::nullopt, - std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt, + std::optional<tensorrt_llm::DataType> explicitDraftTokensDType = std::nullopt, std::optional<std::vector<tr::ITensor::SharedConstPtr>> const& lookaheadPrompt = std::nullopt, std::optional<std::vector<te::LookaheadDecodingConfig>> const& lookaheadAlgoConfigs = std::nullopt) { diff --git a/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.cpp b/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.cpp deleted file mode 100644 index caef94c5defd..000000000000 --- a/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "modelSpecBinding.h" -#include "tensorrt_llm/nanobind/common/customCasters.h" -#include "tensorrt_llm/testing/modelSpec.h" - -#include <nanobind/nanobind.h> - -namespace nb = nanobind; -using tensorrt_llm::testing::ModelSpec; -using tensorrt_llm::testing::KVCacheType; -using tensorrt_llm::testing::QuantMethod; -using tensorrt_llm::testing::OutputContentType; - -namespace tensorrt_llm::nanobind::testing -{ - -void initBindings(nb::module_& m) -{ - nb::enum_<QuantMethod>(m, "QuantMethod", nb::is_arithmetic(), "Quantization Method") - .value("NONE", QuantMethod::kNONE, "No Quantization") - .value("SMOOTH_QUANT", QuantMethod::kSMOOTH_QUANT, "Smooth Quantization"); - - nb::enum_<OutputContentType>(m, "OutputContentType", nb::is_arithmetic(), "Output Content Type") - .value("NONE", OutputContentType::kNONE, "No Output Content") - .value("CONTEXT_LOGITS", OutputContentType::kCONTEXT_LOGITS, "Context Logits") - .value("GENERATION_LOGITS", OutputContentType::kGENERATION_LOGITS, "Generation Logits") - .value("LOG_PROBS", OutputContentType::kLOG_PROBS, "Log Probs") - .value("CUM_LOG_PROBS", OutputContentType::kCUM_LOG_PROBS, "Cumulative Log"); - - nb::class_<ModelSpec>(m, "ModelSpec") - .def(nb::init<std::string const&, nvinfer1::DataType>()) - .def("use_gpt_plugin", &ModelSpec::useGptAttentionPlugin, nb::rv_policy::reference_internal) - .def("use_packed_input", &ModelSpec::usePackedInput, nb::rv_policy::reference_internal) - .def("set_kv_cache_type", &ModelSpec::setKVCacheType, nb::rv_policy::reference_internal) - .def("use_decoder_per_request", &ModelSpec::useDecoderPerRequest, nb::rv_policy::reference_internal) - .def("use_tensor_parallelism", &ModelSpec::useTensorParallelism, nb::rv_policy::reference_internal) - .def("use_pipeline_parallelism", &ModelSpec::usePipelineParallelism, nb::rv_policy::reference_internal) - .def("use_context_parallelism", &ModelSpec::useContextParallelism, nb::rv_policy::reference_internal) - .def("set_draft_tokens", &ModelSpec::setDraftTokens, nb::rv_policy::reference_internal) - .def("use_accept_by_logits", &ModelSpec::useAcceptByLogits, nb::rv_policy::reference_internal) - .def("use_mamba_plugin", &ModelSpec::useMambaPlugin, nb::rv_policy::reference_internal) - .def("gather_logits", &ModelSpec::gatherLogits, nb::rv_policy::reference_internal) - .def("replace_logits", &ModelSpec::replaceLogits, nb::rv_policy::reference_internal) - .def("return_log_probs", &ModelSpec::returnLogProbs, nb::rv_policy::reference_internal) - .def("smoke_test", &ModelSpec::smokeTest, nb::rv_policy::reference_internal) - .def("use_medusa", &ModelSpec::useMedusa, nb::rv_policy::reference_internal) - .def("use_eagle", &ModelSpec::useEagle, nb::rv_policy::reference_internal) - .def("use_lookahead_decoding", &ModelSpec::useLookaheadDecoding, nb::rv_policy::reference_internal) - .def("use_explicit_draft_tokens_decoding", &ModelSpec::useExplicitDraftTokensDecoding, - nb::rv_policy::reference_internal) - .def("use_draft_tokens_external_decoding", &ModelSpec::useDraftTokensExternalDecoding, - nb::rv_policy::reference_internal) - .def("use_logits", &ModelSpec::useLogits) - .def("use_multiple_profiles", &ModelSpec::useMultipleProfiles, nb::rv_policy::reference_internal) - .def("set_max_input_length", &ModelSpec::setMaxInputLength, nb::rv_policy::reference_internal) - .def("set_max_output_length", &ModelSpec::setMaxOutputLength, nb::rv_policy::reference_internal) - .def("set_quant_method", &ModelSpec::setQuantMethod, nb::rv_policy::reference_internal) - .def("use_lora_plugin", &ModelSpec::useLoraPlugin, nb::rv_policy::reference_internal) - .def("get_input_file", &ModelSpec::getInputFile) - .def("get_model_path", &ModelSpec::getModelPath) - .def("get_results_file", &ModelSpec::getResultsFile) - .def("get_generation_logits_file", &ModelSpec::getGenerationLogitsFile) - .def("get_context_logits_file", &ModelSpec::getContextLogitsFile) - .def("get_cum_log_probs_file", &ModelSpec::getCumLogProbsFile) - .def("get_log_probs_file", &ModelSpec::getLogProbsFile) - .def("enable_context_fmha_fp32_acc", &ModelSpec::enableContextFMHAFp32Acc, nb::rv_policy::reference_internal) - .def("get_enable_context_fmha_fp32_acc", &ModelSpec::getEnableContextFMHAFp32Acc) - .def("__copy__", [](ModelSpec const& self) { return ModelSpec(self); }); -} - -} // namespace tensorrt_llm::nanobind::testing diff --git a/cpp/tensorrt_llm/nanobind/thop/bindings.cpp b/cpp/tensorrt_llm/nanobind/thop/bindings.cpp index 23bd36d67b82..dd1ff0db5410 100644 --- a/cpp/tensorrt_llm/nanobind/thop/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/thop/bindings.cpp @@ -179,7 +179,8 @@ void initBindings(nb::module_& m) nb::arg("relative_attention_bias") = std::nullopt, nb::arg("relative_attention_max_distance") = 0, nb::arg("spec_decoding_target_max_draft_tokens") = std::nullopt, nb::arg("quant_scale_qkv") = std::nullopt, nb::arg("dsv4_inv_rope_cos_sin_cache") = std::nullopt, nb::arg("enable_dsv4_epilogue_fusion") = false, - "Multi-head attention operation", nb::call_guard<nb::gil_scoped_release>()); + nb::arg("force_prepare_spec_dec_tree_mask") = false, "Multi-head attention operation", + nb::call_guard<nb::gil_scoped_release>()); m.def( "get_helix_workspace_size_per_rank", diff --git a/cpp/tensorrt_llm/plugins/CMakeLists.txt b/cpp/tensorrt_llm/plugins/CMakeLists.txt deleted file mode 100755 index 8b89cccdc813..000000000000 --- a/cpp/tensorrt_llm/plugins/CMakeLists.txt +++ /dev/null @@ -1,183 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# - -set(PLUGIN_TARGET_NAME nvinfer_plugin_tensorrt_llm) -set(PLUGIN_SHARED_TARGET ${PLUGIN_TARGET_NAME}) - -set(TARGET_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(PLUGIN_EXPORT_MAP ${TARGET_DIR}/exports.map) # Linux -set(PLUGIN_EXPORT_DEF ${TARGET_DIR}/exports.def) # Windows - -if(${CMAKE_BUILD_TYPE} MATCHES "Debug") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") -endif() - -set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --Wno-deprecated-declarations") -set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --diag-suppress 997") - -if(NOT WIN32) - # additional warnings - # - # Ignore overloaded-virtual warning. We intentionally change parameters of - # some methods in derived class. - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-overloaded-virtual") - if(WARNING_IS_ERROR) - message(STATUS "Treating warnings as errors in GCC compilation") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") - endif() -else() # Windows - # warning level 4 - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") -endif() - -set(PLUGIN_SOURCES) -set(PLUGIN_CU_SOURCES) - -set(PLUGIN_LISTS - bertAttentionPlugin - cpSplitPlugin - fusedLayernormPlugin - gptAttentionCommon - gptAttentionPlugin - identityPlugin - gemmPlugin - gemmSwigluPlugin - fp8RowwiseGemmPlugin - smoothQuantGemmPlugin - fp4GemmPlugin - quantizePerTokenPlugin - quantizeTensorPlugin - quantizeToFP4Plugin - layernormQuantizationPlugin - rmsnormQuantizationPlugin - weightOnlyGroupwiseQuantMatmulPlugin - weightOnlyQuantMatmulPlugin - lookupPlugin - loraPlugin - doraPlugin - mixtureOfExperts - selectiveScanPlugin - mambaConv1dPlugin - lruPlugin - cumsumLastDimPlugin - topkLastDimPlugin - lowLatencyGemmPlugin - eaglePlugin - lowLatencyGemmSwigluPlugin - qserveGemmPlugin - cudaStreamPlugin - gemmAllReducePlugin) - -foreach(PLUGIN_ITER ${PLUGIN_LISTS}) - include_directories(${PLUGIN_ITER}) - add_subdirectory(${PLUGIN_ITER}) -endforeach(PLUGIN_ITER) - -if(ENABLE_MULTI_DEVICE) - include_directories(ncclPlugin) - add_subdirectory(ncclPlugin) -endif() -include_directories(common) -add_subdirectory(common) - -# Set gencodes -list(APPEND PLUGIN_SOURCES "${PLUGIN_CU_SOURCES}") - -list(APPEND PLUGIN_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/api/tllmPlugin.cpp") - -# ################################# SHARED LIBRARY -# ############################################################################## - -if(WIN32) - set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS 1) -endif() - -add_library(${PLUGIN_SHARED_TARGET} SHARED ${PLUGIN_SOURCES}) -add_cuda_architectures(${PLUGIN_SHARED_TARGET} 89) - -target_include_directories( - ${PLUGIN_SHARED_TARGET} - PUBLIC ${CUDA_INSTALL_DIR}/include - PUBLIC - $<TARGET_PROPERTY:${INTERNAL_CUTLASS_KERNELS_TARGET},INTERFACE_INCLUDE_DIRECTORIES> - PRIVATE ${TARGET_DIR}) - -if(USING_OSS_CUTLASS_FP4_GEMM) - target_compile_definitions(${PLUGIN_SHARED_TARGET} - PUBLIC USING_OSS_CUTLASS_FP4_GEMM) -endif() - -if(USING_OSS_CUTLASS_ALLREDUCE_GEMM) - target_compile_definitions(${PLUGIN_SHARED_TARGET} - PUBLIC USING_OSS_CUTLASS_ALLREDUCE_GEMM) -endif() - -if(USING_OSS_CUTLASS_MOE_GEMM) - target_compile_definitions(${PLUGIN_SHARED_TARGET} - PUBLIC USING_OSS_CUTLASS_MOE_GEMM) -endif() - -if(ENABLE_MULTI_DEVICE) - target_include_directories(${PLUGIN_SHARED_TARGET} - PUBLIC ${MPI_C_INCLUDE_DIRS}) -endif() - -if(CUDA_VERSION VERSION_LESS 11.0) - target_include_directories(${PLUGIN_SHARED_TARGET} PUBLIC ${CUB_ROOT_DIR}) -endif() - -set_target_properties( - ${PLUGIN_SHARED_TARGET} - PROPERTIES CXX_STANDARD "17" - CXX_STANDARD_REQUIRED "YES" - CXX_EXTENSIONS "NO" - ARCHIVE_OUTPUT_DIRECTORY "${TRT_OUT_DIR}" - LIBRARY_OUTPUT_DIRECTORY "${TRT_OUT_DIR}" - RUNTIME_OUTPUT_DIRECTORY "${TRT_OUT_DIR}") - -if(WIN32) - set_target_properties( - ${PLUGIN_SHARED_TARGET} - PROPERTIES LINK_FLAGS "/DEF:${PLUGIN_EXPORT_DEF} ${UNDEFINED_FLAG}") -else() - set_target_properties( - ${PLUGIN_SHARED_TARGET} - PROPERTIES - LINK_FLAGS - "-Wl,--exclude-libs,ALL -Wl,--version-script=${PLUGIN_EXPORT_MAP} -Wl,-rpath,'$ORIGIN' ${AS_NEEDED_FLAG} ${UNDEFINED_FLAG}" - ) -endif() - -set_property(TARGET ${PLUGIN_SHARED_TARGET} PROPERTY CUDA_STANDARD 17) - -target_link_libraries( - ${PLUGIN_SHARED_TARGET} - ${CUBLAS_LIB} - ${CUBLASLT_LIB} - ${TRT_LIB} - ${CUDA_DRV_LIB} - ${CUDA_RT_LIB} - ${CMAKE_DL_LIBS} - ${SHARED_TARGET}) - -if(WIN32) - target_link_libraries(${PLUGIN_SHARED_TARGET} context_attention_src) -endif() - -if(ENABLE_MULTI_DEVICE) - target_link_libraries(${PLUGIN_SHARED_TARGET} ${MPI_C_LIBRARIES} ${NCCL_LIB}) -endif() diff --git a/cpp/tensorrt_llm/plugins/api/tllmPlugin.cpp b/cpp/tensorrt_llm/plugins/api/tllmPlugin.cpp deleted file mode 100644 index f0dceb2f4a99..000000000000 --- a/cpp/tensorrt_llm/plugins/api/tllmPlugin.cpp +++ /dev/null @@ -1,313 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "tensorrt_llm/plugins/api/tllmPlugin.h" - -#include "tensorrt_llm/common/stringUtils.h" -#include "tensorrt_llm/runtime/tllmLogger.h" - -#include "tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.h" -#include "tensorrt_llm/plugins/doraPlugin/doraPlugin.h" -#include "tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.h" -#include "tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.h" -#include "tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h" -#include "tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.h" -#include "tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.h" -#include "tensorrt_llm/plugins/identityPlugin/identityPlugin.h" -#include "tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.h" -#include "tensorrt_llm/plugins/lookupPlugin/lookupPlugin.h" -#include "tensorrt_llm/plugins/loraPlugin/loraPlugin.h" -#include "tensorrt_llm/plugins/lruPlugin/lruPlugin.h" -#include "tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.h" -#include "tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h" -#include "tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.h" -#if ENABLE_MULTI_DEVICE -#include "tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.h" -#include "tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.h" -#include "tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.h" -#include "tensorrt_llm/plugins/ncclPlugin/allreducePlugin.h" -#include "tensorrt_llm/plugins/ncclPlugin/recvPlugin.h" -#include "tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.h" -#include "tensorrt_llm/plugins/ncclPlugin/sendPlugin.h" -#endif // ENABLE_MULTI_DEVICE -#include "tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.h" -#include "tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.h" -#include "tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.h" -#include "tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.h" -#include "tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.h" -#include "tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.h" -#include "tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.h" -#include "tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.h" -#include "tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.h" -#include "tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.h" -#include "tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.h" -#include "tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.h" -#include "tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.h" -#include "tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.h" -#include "tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.h" -#include "tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.h" -#include "tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.h" - -#include <array> -#include <cstdlib> - -#include <NvInferRuntime.h> - -namespace tc = tensorrt_llm::common; - -namespace -{ - -nvinfer1::IPluginCreator* creatorPtr(nvinfer1::IPluginCreator& creator) -{ - return &creator; -} - -nvinfer1::IPluginCreatorInterface* creatorInterfacePtr(nvinfer1::IPluginCreatorInterface& creator) -{ - return &creator; -} - -auto tllmLogger = tensorrt_llm::runtime::TllmLogger(); - -nvinfer1::ILogger* gLogger{&tllmLogger}; - -class GlobalLoggerFinder : public nvinfer1::ILoggerFinder -{ -public: - nvinfer1::ILogger* findLogger() override - { - return gLogger; - } -}; - -GlobalLoggerFinder gGlobalLoggerFinder{}; - -#if !defined(_MSC_VER) -[[maybe_unused]] __attribute__((constructor)) -#endif -void initOnLoad() -{ - auto constexpr kLoadPlugins = "TRT_LLM_LOAD_PLUGINS"; - auto const loadPlugins = std::getenv(kLoadPlugins); - if (loadPlugins && loadPlugins[0] == '1') - { - initTrtLlmPlugins(gLogger); - } -} - -bool pluginsInitialized = false; - -} // namespace - -namespace tensorrt_llm::plugins::api -{ - -LoggerManager& tensorrt_llm::plugins::api::LoggerManager::getInstance() noexcept -{ - static LoggerManager instance; - return instance; -} - -void LoggerManager::setLoggerFinder(nvinfer1::ILoggerFinder* finder) -{ - std::lock_guard<std::mutex> lk(mMutex); - if (mLoggerFinder == nullptr && finder != nullptr) - { - mLoggerFinder = finder; - } -} - -[[maybe_unused]] nvinfer1::ILogger* LoggerManager::logger() -{ - std::lock_guard<std::mutex> lk(mMutex); - if (mLoggerFinder != nullptr) - { - return mLoggerFinder->findLogger(); - } - return nullptr; -} - -nvinfer1::ILogger* LoggerManager::defaultLogger() noexcept -{ - return gLogger; -} -} // namespace tensorrt_llm::plugins::api - -// New Plugin APIs - -extern "C" -{ - bool initTrtLlmPlugins(void* logger, char const* libNamespace) - { - if (pluginsInitialized) - { - return true; - } - - if (logger) - { - gLogger = static_cast<nvinfer1::ILogger*>(logger); - } - setLoggerFinder(&gGlobalLoggerFinder); - - auto registry = getPluginRegistry(); - - { - std::int32_t nbCreators; - auto creators = getPluginCreators(nbCreators); - - for (std::int32_t i = 0; i < nbCreators; ++i) - { - auto const creator = creators[i]; - creator->setPluginNamespace(libNamespace); - registry->registerCreator(*creator, libNamespace); - if (gLogger) - { - auto const msg = tc::fmtstr("Registered plugin creator %s version %s in namespace %s", - creator->getPluginName(), creator->getPluginVersion(), libNamespace); - gLogger->log(nvinfer1::ILogger::Severity::kVERBOSE, msg.c_str()); - } - } - } - - { - std::int32_t nbCreators; - auto creators = getCreators(nbCreators); - - for (std::int32_t i = 0; i < nbCreators; ++i) - { - auto const creator = creators[i]; - registry->registerCreator(*creator, libNamespace); - } - } - - pluginsInitialized = true; - return true; - } - - [[maybe_unused]] void setLoggerFinder([[maybe_unused]] nvinfer1::ILoggerFinder* finder) - { - tensorrt_llm::plugins::api::LoggerManager::getInstance().setLoggerFinder(finder); - } - - [[maybe_unused]] nvinfer1::IPluginCreator* const* getPluginCreators(std::int32_t& nbCreators) - { - static tensorrt_llm::plugins::IdentityPluginCreator identityPluginCreator; - static tensorrt_llm::plugins::BertAttentionPluginCreator bertAttentionPluginCreator; - static tensorrt_llm::plugins::FusedLayernormPluginCreator fusedLayernormPluginCreator; - static tensorrt_llm::plugins::GPTAttentionPluginCreator gptAttentionPluginCreator; - static tensorrt_llm::plugins::GemmPluginCreator gemmPluginCreator; - static tensorrt_llm::plugins::GemmSwigluPluginCreator gemmSwigluPluginCreator; - static tensorrt_llm::plugins::Fp8RowwiseGemmPluginCreator fp8RowwiseGemmPluginCreator; - static tensorrt_llm::plugins::MixtureOfExpertsPluginCreator moePluginCreator; -#if ENABLE_MULTI_DEVICE - static tensorrt_llm::plugins::SendPluginCreator sendPluginCreator; - static tensorrt_llm::plugins::RecvPluginCreator recvPluginCreator; - static tensorrt_llm::plugins::AllreducePluginCreator allreducePluginCreator; - static tensorrt_llm::plugins::AllgatherPluginCreator allgatherPluginCreator; - static tensorrt_llm::plugins::ReduceScatterPluginCreator reduceScatterPluginCreator; - static tensorrt_llm::plugins::GemmAllReducePluginCreator gemmAllReducePluginCreator; -#endif // ENABLE_MULTI_DEVICE - static tensorrt_llm::plugins::SmoothQuantGemmPluginCreator smoothQuantGemmPluginCreator; - static tensorrt_llm::plugins::QServeGemmPluginCreator qserveGemmPluginCreator; - static tensorrt_llm::plugins::LayernormQuantizationPluginCreator layernormQuantizationPluginCreator; - static tensorrt_llm::plugins::QuantizeToFP4PluginCreator quantizeToFP4PluginCreator; - static tensorrt_llm::plugins::QuantizePerTokenPluginCreator quantizePerTokenPluginCreator; - static tensorrt_llm::plugins::QuantizeTensorPluginCreator quantizeTensorPluginCreator; - static tensorrt_llm::plugins::RmsnormQuantizationPluginCreator rmsnormQuantizationPluginCreator; - static tensorrt_llm::plugins::WeightOnlyGroupwiseQuantMatmulPluginCreator - weightOnlyGroupwiseQuantMatmulPluginCreator; - static tensorrt_llm::plugins::WeightOnlyQuantMatmulPluginCreator weightOnlyQuantMatmulPluginCreator; - static tensorrt_llm::plugins::LookupPluginCreator lookupPluginCreator; - static tensorrt_llm::plugins::LoraPluginCreator loraPluginCreator; - static tensorrt_llm::plugins::SelectiveScanPluginCreator selectiveScanPluginCreator; - static tensorrt_llm::plugins::Fp4GemmPluginCreator fp4GemmPluginCreator; - static tensorrt_llm::plugins::MambaConv1dPluginCreator mambaConv1DPluginCreator; - static tensorrt_llm::plugins::lruPluginCreator lruPluginCreator; - static tensorrt_llm::plugins::CumsumLastDimPluginCreator cumsumLastDimPluginCreator; - static tensorrt_llm::plugins::TopkLastDimPluginCreator topkLastDimPluginCreator; - static tensorrt_llm::plugins::LowLatencyGemmPluginCreator lowLatencyGemmPluginCreator; - static tensorrt_llm::plugins::LowLatencyGemmSwigluPluginCreator lowLatencyGemmSwigluPluginCreator; - static tensorrt_llm::plugins::EagleDecodeDraftTokensPluginCreator eagleDecodeDraftTokensPluginCreator; - static tensorrt_llm::plugins::EagleSampleAndAcceptDraftTokensPluginCreator - eagleSampleAndAcceptDraftTokensPluginCreator; - static tensorrt_llm::plugins::CudaStreamPluginCreator cudaStreamPluginCreator; - - static std::array pluginCreators - = { creatorPtr(identityPluginCreator), - creatorPtr(bertAttentionPluginCreator), - creatorPtr(gptAttentionPluginCreator), - creatorPtr(gemmPluginCreator), - creatorPtr(gemmSwigluPluginCreator), - creatorPtr(fp8RowwiseGemmPluginCreator), - creatorPtr(moePluginCreator), -#if ENABLE_MULTI_DEVICE - creatorPtr(sendPluginCreator), - creatorPtr(recvPluginCreator), - creatorPtr(allreducePluginCreator), - creatorPtr(allgatherPluginCreator), - creatorPtr(reduceScatterPluginCreator), - creatorPtr(gemmAllReducePluginCreator), -#endif // ENABLE_MULTI_DEVICE - creatorPtr(fusedLayernormPluginCreator), - creatorPtr(smoothQuantGemmPluginCreator), - creatorPtr(qserveGemmPluginCreator), - creatorPtr(layernormQuantizationPluginCreator), - creatorPtr(quantizeToFP4PluginCreator), - creatorPtr(quantizePerTokenPluginCreator), - creatorPtr(quantizeTensorPluginCreator), - creatorPtr(rmsnormQuantizationPluginCreator), - creatorPtr(weightOnlyGroupwiseQuantMatmulPluginCreator), - creatorPtr(weightOnlyQuantMatmulPluginCreator), - creatorPtr(lookupPluginCreator), - creatorPtr(loraPluginCreator), - creatorPtr(selectiveScanPluginCreator), - creatorPtr(fp4GemmPluginCreator), - creatorPtr(mambaConv1DPluginCreator), - creatorPtr(lruPluginCreator), - creatorPtr(cumsumLastDimPluginCreator), - creatorPtr(topkLastDimPluginCreator), - creatorPtr(lowLatencyGemmPluginCreator), - creatorPtr(eagleDecodeDraftTokensPluginCreator), - creatorPtr(eagleSampleAndAcceptDraftTokensPluginCreator), - creatorPtr(lowLatencyGemmSwigluPluginCreator), - creatorPtr(cudaStreamPluginCreator), - }; - nbCreators = pluginCreators.size(); - return pluginCreators.data(); - } - - [[maybe_unused]] nvinfer1::IPluginCreatorInterface* const* getCreators(std::int32_t& nbCreators) - { - static tensorrt_llm::plugins::EaglePrepareDrafterInputsPluginCreator eaglePrepareDrafterInputsPluginCreator; -#if ENABLE_MULTI_DEVICE - static tensorrt_llm::plugins::CpSplitPluginCreator cpSplitPluginCreator; -#endif // ENABLE_MULTI_DEVICE - - static tensorrt_llm::plugins::DoraPluginCreator doraPluginCreator; - - static std::array creators - = { creatorInterfacePtr(eaglePrepareDrafterInputsPluginCreator), -#if ENABLE_MULTI_DEVICE - creatorInterfacePtr(cpSplitPluginCreator), -#endif // ENABLE_MULTI_DEVICE - creatorInterfacePtr(doraPluginCreator) }; - - nbCreators = creators.size(); - return creators.data(); - } -} // extern "C" diff --git a/cpp/tensorrt_llm/plugins/bertAttentionPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/bertAttentionPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/bertAttentionPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.cpp b/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.cpp deleted file mode 100644 index 6acf0b3a9d25..000000000000 --- a/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.cpp +++ /dev/null @@ -1,1206 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "bertAttentionPlugin.h" -#include "tensorrt_llm/kernels/gptKernels.h" -#include "tensorrt_llm/kernels/recoverFromRingAtten.h" -#include "tensorrt_llm/kernels/sageAttentionKernels.h" -#include "tensorrt_llm/kernels/unfusedAttentionKernels.h" -#include "tensorrt_llm/runtime/iBuffer.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -namespace tc = tensorrt_llm::common; - -using tensorrt_llm::plugins::BertAttentionPluginCreator; -using tensorrt_llm::plugins::BertAttentionPlugin; - -static char const* BERT_ATTENTION_PLUGIN_VERSION{"1"}; -static char const* BERT_ATTENTION_PLUGIN_NAME{"BertAttention"}; -PluginFieldCollection BertAttentionPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> BertAttentionPluginCreator::mPluginAttributes; - -BertAttentionPlugin::BertAttentionPlugin(int num_heads, int head_size, float q_scaling, - ContextFMHAType context_fmha_type, nvinfer1::DataType type, bool do_relative_attention, int max_distance, - bool remove_padding, bool sage_attn, int sage_attn_q_block_size, int sage_attn_k_block_size, - int sage_attn_v_block_size, int cp_size, int cp_rank, std::set<int> cp_group) - : mNumHeads(num_heads) - , mHeadSize(head_size) - , mQScaling(q_scaling) - , mType(type) - , mRelativeAttention(do_relative_attention) - , mMaxDistance(max_distance) - , mRemovePadding(remove_padding) - , mEnableContextFMHA(context_fmha_type != ContextFMHAType::DISABLED) - , mFMHAForceFP32Acc(context_fmha_type == ContextFMHAType::ENABLED_WITH_FP32_ACC) - , mSageAttn(sage_attn) - , mCpSize(cp_size) - , mCpRank(cp_rank) - , mCpGroup(std::move(cp_group)) -{ - // pre-check whether FMHA is supported in order to save memory allocation - if (mEnableContextFMHA) - { - mEnableContextFMHA = false; - if (!(mType == DataType::kHALF || mType == DataType::kBF16)) - { - TLLM_LOG_WARNING("Fall back to unfused MHA because of unsupported data type."); - } - else if (mRelativeAttention) - { - TLLM_LOG_WARNING("Fall back to unfused MHA because of relative position embedding."); - } - else - { - mEnableContextFMHA = true; - } - } - - if (mSageAttn) - { - mSageAttnQBlockSize = sage_attn_q_block_size; - mSageAttnKBlockSize = sage_attn_k_block_size; - mSageAttnVBlockSize = sage_attn_v_block_size; - std::vector<int> blockSizeCombination - = {sage_attn_q_block_size, sage_attn_k_block_size, sage_attn_v_block_size}; - if (mSageAttnSupportedBlockSizes.find(blockSizeCombination) == mSageAttnSupportedBlockSizes.end() - || (head_size != 128 && head_size != 72 && head_size != 80)) - { - TLLM_LOG_WARNING(" Q, k ,v quant block size not support. disable sage attention"); - mSageAttn = false; - } - else - { - TLLM_LOG_INFO("SageAttnQBlockSize: %d, SageAttnKBlockSize: %d, SageAttnVBlockSize: %d", mSageAttnQBlockSize, - mSageAttnKBlockSize, mSageAttnVBlockSize); - } - } - - if (cp_group.size() > 1 && !mEnableContextFMHA) - { - TLLM_LOG_ERROR("Unfused MHA do not support context parallel now."); - } -} - -// Parameterized constructor -BertAttentionPlugin::BertAttentionPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mNumHeads); - read(d, mHeadSize); - read(d, mQScaling); - read(d, mQKHalfAccum); - read(d, mEnableContextFMHA); - read(d, mFMHAForceFP32Acc); - read(d, mType); - read(d, mRelativeAttention); - read(d, mMaxDistance); - read(d, mRemovePadding); - read(d, mSageAttn); - read(d, mSageAttnQBlockSize); - read(d, mSageAttnKBlockSize); - read(d, mSageAttnVBlockSize); - read(d, mCpSize); - read(d, mCpRank); - mCpGroup.clear(); - int groupItem = 0; - while (d != a + length) - { - read(d, groupItem); - mCpGroup.insert(groupItem); - } - - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* BertAttentionPlugin::clone() const noexcept -{ - auto* plugin = new BertAttentionPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - plugin->initialize(); - return plugin; -} - -nvinfer1::DimsExprs BertAttentionPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - TLLM_CHECK(outputIndex == 0); - auto ret = inputs[0]; - ret.d[mRemovePadding ? 1 : 2] = exprBuilder.constant(ret.d[mRemovePadding ? 1 : 2]->getConstantValue() / 3); - return ret; -} - -bool BertAttentionPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - // inputs: [0] qkv, [1] input_lengths, [2] max_input_length (optional), [3] relative_attention_bias (optional) - // outputs: [X] hidden_states - if (nbInputs == 2) - { // BERT - if (pos == 1) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } - if (nbInputs > 2) - { // Encoder in encoder-decoder - if (pos == 1 || pos == 2) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } - - return false; -} - -void BertAttentionPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t BertAttentionPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - // if remove padding, inputs[0] "qkv_hidden_states" dim is [num_tokens, 3*hidden_dim] which doesn't have shape - // info should get max_batch_size and max_input_length from inputs[1] "input_lengths" and input[2] - // "max_input_length" - int const batch_size = mRemovePadding ? inputs[1].dims.d[0] : inputs[0].dims.d[0]; - int const input_seq_len = mRemovePadding ? inputs[2].dims.d[0] : inputs[0].dims.d[1]; - int const local_hidden_units_ = inputs[0].dims.d[mRemovePadding ? 1 : 2] / 3; - - auto const size = tensorrt_llm::runtime::BufferDataType(inputs[0].type).getSize(); - - size_t const attention_mask_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_len * input_seq_len; - size_t const cu_seqlens_size = sizeof(int) * (batch_size + 1); - size_t const q_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_len * local_hidden_units_; - size_t const k_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_len * local_hidden_units_; - size_t const v_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_len * local_hidden_units_; - size_t const qk_buf_size = mEnableContextFMHA ? 0 : size * batch_size * mNumHeads * input_seq_len * input_seq_len; - size_t const qkv_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_len * local_hidden_units_; - size_t const qk_buf_float_size - = mEnableContextFMHA ? 0 : sizeof(float) * batch_size * mNumHeads * input_seq_len * input_seq_len; - size_t const padding_offset_size = mEnableContextFMHA ? 0 : sizeof(int) * batch_size * input_seq_len; - size_t const fmha_scheduler_counter = mEnableContextFMHA ? sizeof(uint32_t) : 0; - int const paddedHeadSize = mSageAttn ? ((mHeadSize + 15) / 16) * 16 : mHeadSize; - const size_t quanted_qkv_size - = mSageAttn ? sizeof(__nv_fp8_e4m3) * batch_size * input_seq_len * mNumHeads * paddedHeadSize * 3 : 0; - const size_t q_scale_size = mSageAttn - ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnQBlockSize - 1) / mSageAttnQBlockSize) * mNumHeads - : 0; - const size_t k_scale_size = mSageAttn - ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnKBlockSize - 1) / mSageAttnKBlockSize) * mNumHeads - : 0; - const size_t v_scale_size = mSageAttn - ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnVBlockSize - 1) / mSageAttnVBlockSize) * mNumHeads - : 0; - const size_t scale_bmm1_device_size = mSageAttn ? sizeof(float) * 2 : 0; - const size_t scale_bmm2_device_size = mSageAttn ? sizeof(float) : 0; - size_t sage_quant_space_size = mSageAttn ? sizeof(float) * batch_size * mNumHeads * mHeadSize : 0; - - if (paddedHeadSize != mHeadSize) - sage_quant_space_size - = sage_quant_space_size < (batch_size * input_seq_len * mNumHeads * paddedHeadSize * sizeof(__nv_bfloat16)) - ? (batch_size * input_seq_len * mNumHeads * paddedHeadSize * sizeof(__nv_bfloat16)) - : sage_quant_space_size; - - // workspace for RingAttention ping-pong buffer - bool const enableRingAttn = (mCpGroup.size() > 1); - const size_t ring_q_buf_size = enableRingAttn ? size * batch_size * input_seq_len * local_hidden_units_ : 0; - const size_t ring_kv_buf_size = enableRingAttn - ? 2 * size * batch_size * input_seq_len * local_hidden_units_ + sizeof(int) * (batch_size + 1) - : 0; - const size_t ring_softmax_stats_buf_size - = enableRingAttn ? 2 * sizeof(float) * batch_size * input_seq_len * mNumHeads : 0; - const size_t ring_softmax_stats_accu_buf_size - = enableRingAttn ? 2 * sizeof(float) * batch_size * input_seq_len * mNumHeads : 0; - const size_t ring_block_output_size = enableRingAttn ? size * batch_size * input_seq_len * local_hidden_units_ : 0; - - int const NUM_BUFFERS = 24; - - size_t workspaces[NUM_BUFFERS]; - workspaces[0] = CUBLAS_WORKSPACE_SIZE; - workspaces[1] = attention_mask_size; - workspaces[2] = cu_seqlens_size; - workspaces[3] = q_buf_2_size; - workspaces[4] = k_buf_2_size; - workspaces[5] = v_buf_2_size; - workspaces[6] = qk_buf_size; - workspaces[7] = qkv_buf_2_size; - workspaces[8] = qk_buf_float_size; - workspaces[9] = padding_offset_size; - workspaces[10] = fmha_scheduler_counter; - workspaces[11] = quanted_qkv_size; - workspaces[12] = q_scale_size; - workspaces[13] = v_scale_size; - workspaces[14] = k_scale_size; - workspaces[15] = scale_bmm1_device_size; - workspaces[16] = scale_bmm2_device_size; - workspaces[17] = sage_quant_space_size; - workspaces[18] = ring_q_buf_size; - workspaces[19] = ring_kv_buf_size; // kv1 - workspaces[20] = ring_kv_buf_size; // kv2 - workspaces[21] = ring_softmax_stats_buf_size; - workspaces[22] = ring_softmax_stats_accu_buf_size; - workspaces[23] = ring_block_output_size; - - return tc::calculateTotalWorkspaceSize(workspaces, NUM_BUFFERS); -} - -template <typename T> -int BertAttentionPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) -{ - - // inputs - // input_tensor [batch_size, seq_len, local_hidden_size*3] or [num_tokens, local_hidden_size*3] - // input_lengths [batch_size] - // max_input_length [max_input_length] -- use shape dim to represent max value. If remove padding, this records - // the max input length among sequences; otherwise same as input_tensor's padded dim[1] relative_attention_bias - // [num_heads, num_buckets] (optional) - // outputs - // output_tensor [batch_size, seq_len, local_hidden_size] or [num_tokens, local_hidden_size] - - // if remove padding, inputs[0] dim is [num_tokens] which doesn't have workspace info - // should get max_batch_size from inputs[1] and max_input_length from plugin attribute - int const batch_size = mRemovePadding ? inputDesc[1].dims.d[0] : inputDesc[0].dims.d[0]; - int const input_seq_len = mRemovePadding ? inputDesc[2].dims.d[0] : inputDesc[0].dims.d[1]; - int const num_tokens = mRemovePadding ? inputDesc[0].dims.d[0] : batch_size * input_seq_len; - int const request_batch_size = batch_size; - int const request_seq_len = input_seq_len; - int const local_hidden_units_ = inputDesc[0].dims.d[mRemovePadding ? 1 : 2] / 3; - float const q_scaling = mQScaling; - - T const* attention_input = reinterpret_cast<T const*>(inputs[0]); - int const* input_lengths = reinterpret_cast<int const*>(inputs[1]); - T const* relative_attn_table = mRelativeAttention ? reinterpret_cast<T const*>(inputs[3]) : nullptr; - T* context_buf_ = (T*) (outputs[0]); - - auto cublasHandle = mCublasWrapper->getCublasHandle(); - TLLM_CUDA_CHECK(cublasSetStream(cublasHandle, stream)); - mCublasWrapper->setStream(stream); - mCublasWrapper->setWorkspace(workspace); - if (inputDesc[0].type == DataType::kHALF) - { - mCublasWrapper->setFP16GemmConfig(); - } - else if (inputDesc[0].type == DataType::kFLOAT) - { - mCublasWrapper->setFP32GemmConfig(); - } -#ifdef ENABLE_BF16 - else if constexpr (std::is_same_v<T, __nv_bfloat16>) - { - mCublasWrapper->setBF16GemmConfig(); - } -#endif - - size_t const attention_mask_size = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * input_seq_len * input_seq_len; - size_t const cu_seqlens_size = sizeof(int) * (batch_size + 1); - size_t const q_buf_2_size = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * input_seq_len * local_hidden_units_; - size_t const k_buf_2_size = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * input_seq_len * local_hidden_units_; - size_t const v_buf_2_size = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * input_seq_len * local_hidden_units_; - size_t const qk_buf_size - = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * mNumHeads * input_seq_len * input_seq_len; - size_t const qkv_buf_2_size = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * input_seq_len * local_hidden_units_; - size_t const qk_buf_float_size - = mEnableContextFMHA ? 0 : sizeof(float) * batch_size * mNumHeads * input_seq_len * input_seq_len; - size_t const padding_offset_size = mEnableContextFMHA ? 0 : sizeof(int) * batch_size * input_seq_len; - size_t const fmha_scheduler_counter = mEnableContextFMHA ? sizeof(uint32_t) : 0; - - int const paddedHeadSize = mSageAttn ? ((mHeadSize + 15) / 16) * 16 : mHeadSize; - const size_t quanted_qkv_size - = mSageAttn ? sizeof(__nv_fp8_e4m3) * batch_size * input_seq_len * mNumHeads * paddedHeadSize * 3 : 0; - const size_t q_scale_size = mSageAttn - ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnQBlockSize - 1) / mSageAttnQBlockSize) * mNumHeads - : 0; - const size_t k_scale_size = mSageAttn - ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnKBlockSize - 1) / mSageAttnKBlockSize) * mNumHeads - : 0; - const size_t v_scale_size = mSageAttn - ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnVBlockSize - 1) / mSageAttnVBlockSize) * mNumHeads - : 0; - const size_t scale_bmm1_device_size = mSageAttn ? sizeof(float) * 2 : 0; - const size_t scale_bmm2_device_size = mSageAttn ? sizeof(float) : 0; - size_t sage_quant_space_size = mSageAttn ? sizeof(float) * batch_size * mNumHeads * mHeadSize : 0; - - if (paddedHeadSize != mHeadSize) - sage_quant_space_size - = sage_quant_space_size < (batch_size * input_seq_len * mNumHeads * paddedHeadSize * sizeof(__nv_bfloat16)) - ? (batch_size * input_seq_len * mNumHeads * paddedHeadSize * sizeof(__nv_bfloat16)) - : sage_quant_space_size; - - bool const enableRingAttn = (mCpGroup.size() > 1); - const size_t ring_q_buf_size = enableRingAttn ? sizeof(T) * batch_size * input_seq_len * local_hidden_units_ : 0; - const size_t ring_kv_buf_size - = enableRingAttn ? 2 * sizeof(T) * batch_size * input_seq_len * local_hidden_units_ : 0; - const size_t ring_softmax_stats_buf_size - = enableRingAttn ? 2 * sizeof(float) * batch_size * input_seq_len * mNumHeads : 0; - const size_t ring_block_output_size - = enableRingAttn ? sizeof(T) * batch_size * input_seq_len * local_hidden_units_ : 0; - - // Workspace pointer shift - int8_t* workspace_byte_ptr = reinterpret_cast<int8_t*>(workspace); - size_t offset = CUBLAS_WORKSPACE_SIZE; - - T* attention_mask = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, attention_mask_size)); - int* cu_seqlens = reinterpret_cast<int*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, cu_seqlens_size)); - T* q_buf_2_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, q_buf_2_size)); - T* k_buf_2_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, k_buf_2_size)); - T* v_buf_2_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, v_buf_2_size)); - T* qk_buf_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, qk_buf_size)); - T* qkv_buf_2_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, qkv_buf_2_size)); - float* qk_buf_float_ - = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, qk_buf_float_size)); - int* padding_offset = reinterpret_cast<int*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, padding_offset_size)); - uint32_t* fmha_tile_counter_ptr - = reinterpret_cast<uint32_t*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, fmha_scheduler_counter)); - - __nv_fp8_e4m3* quanted_qkv_ptr - = reinterpret_cast<__nv_fp8_e4m3*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, quanted_qkv_size)); - float* q_scale_ptr = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, q_scale_size)); - float* k_scale_ptr = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, k_scale_size)); - float* v_scale_ptr = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, v_scale_size)); - float* scale_bmm1_ptr - = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, scale_bmm1_device_size)); - float* scale_bmm2_ptr - = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, scale_bmm2_device_size)); - void* sage_quant_space_ptr - = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, sage_quant_space_size)); - - T* ring_q_buf_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_q_buf_size)); - T* ring_kv_buf_1_ = reinterpret_cast<T*>( - tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_kv_buf_size + sizeof(int) * (batch_size + 1))); - T* ring_kv_buf_2_ = reinterpret_cast<T*>( - tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_kv_buf_size + sizeof(int) * (batch_size + 1))); - float* ring_softmax_stats_buf_ - = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_softmax_stats_buf_size)); - float* ring_softmax_accu_stats_buf_ - = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_softmax_stats_buf_size)); - T* ring_block_output_ - = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_block_output_size)); - - // build attention_mask, cu_seqlens, and padding_offset tensors - BuildDecoderInfoParams<T> params{}; - params.seqQOffsets = cu_seqlens; - params.paddingOffsets = padding_offset; - params.attentionMask = attention_mask; - params.seqQLengths = input_lengths; - params.batchSize = batch_size; - params.maxQSeqLength = input_seq_len; - params.numTokens = num_tokens; - params.attentionMaskType = AttentionMaskType::PADDING; - params.fmhaTileCounter = fmha_tile_counter_ptr; - if (mSageAttn) - { - params.fmhaHostBmm1Scale = 1.0f / (sqrtf(mHeadSize * 1.0f) * q_scaling); - params.fmhaBmm1Scale = scale_bmm1_ptr; - params.fmhaBmm2Scale = scale_bmm2_ptr; - } - invokeBuildDecoderInfo(params, stream); - sync_check_cuda_error(stream); - - auto const gemm_data_type = tc::CudaDataType<T>::value; - int const attention_seq_len_1 = request_seq_len; // q length - int const attention_seq_len_2 = request_seq_len; // kv length - - // If the model has relative attentiona bias, q scaling should be applied in QK gemm stage and use 1 in - // softamax stage (because to get softmax[scale(Q*K) + rel pos bias] here, q_scaling can't be applied during - // softmax phase by qk_scale); otherwise, use 1 in gemm stage and apply scaling in softmax stage - float const qk_scale - = 1.0f / (sqrtf(mHeadSize * 1.0f) * q_scaling); // q_scaling in denominator. by default q_scaling =1.0f - float const qk_scale_gemm = mRelativeAttention ? qk_scale : 1.0f; - T const qk_scale_softmax = static_cast<T>(mRelativeAttention ? 1.0f : qk_scale); - - T* linear_bias_slopes = nullptr; - - // FMHA doesn't apply to MHA with relative attention bias, i.e. softmax(QK + bias) * V - // We update mEnableContextFMHA in constructor to check this condition - if (mEnableContextFMHA) - { - if (enableRingAttn) - { - // make sure the padding part of key/value buffer is 0 - cudaMemsetAsync(ring_kv_buf_1_, 0, - reinterpret_cast<int8_t*>(ring_kv_buf_2_) - reinterpret_cast<int8_t*>(ring_kv_buf_1_), stream); - - cudaMemcpyAsync(ring_q_buf_, attention_input, ring_q_buf_size, cudaMemcpyDeviceToDevice, stream); - cudaMemcpyAsync(ring_kv_buf_1_, - const_cast<char*>(reinterpret_cast<char const*>(attention_input)) + ring_q_buf_size, ring_kv_buf_size, - cudaMemcpyDeviceToDevice, stream); - cudaMemcpyAsync(reinterpret_cast<char*>(ring_kv_buf_1_) + ring_kv_buf_size, cu_seqlens, - sizeof(int) * (batch_size + 1), cudaMemcpyDeviceToDevice, stream); - // init softmax_stats - cudaMemsetAsync(ring_softmax_accu_stats_buf_, 0, ring_softmax_stats_buf_size, stream); - -#if ENABLE_MULTI_DEVICE - // relative position of prev/next rank in cp group - int prev_rank = mCpRank > 0 ? mCpRank - 1 : mCpGroup.size() - 1; - int next_rank = (mCpRank == static_cast<int>(mCpGroup.size() - 1)) ? 0 : mCpRank + 1; -#endif // ENABLE_MULTI_DEVICE - - common::check_cuda_error(cudaStreamCreate(&mNcclStream)); - common::check_cuda_error(cudaStreamSynchronize(stream)); - - uint32_t* fmha_scheduler_counter_h = (uint32_t*) malloc(sizeof(uint32_t)); - cudaMemcpyAsync( - fmha_scheduler_counter_h, fmha_tile_counter_ptr, sizeof(uint32_t), cudaMemcpyDeviceToHost, stream); - for (size_t iter = 0; iter < mCpGroup.size(); ++iter) - { - // KV buffer used by fmha - T* ring_fmha_kv_buf_ = (iter % 2 == 0) ? ring_kv_buf_1_ : ring_kv_buf_2_; -#if ENABLE_MULTI_DEVICE - T* ring_send_kv_buf_ = (iter % 2 == 0) ? ring_kv_buf_1_ : ring_kv_buf_2_; - T* ring_recv_kv_buf_ = (iter % 2 == 0) ? ring_kv_buf_2_ : ring_kv_buf_1_; - if (iter < mCpGroup.size() - 1) - { - NCCLCHECK(ncclGroupStart()); - TLLM_CHECK_WITH_INFO(mNcclComm.get() != nullptr, "mNcclComm should be initialized before used"); - NCCLCHECK(ncclSend(ring_send_kv_buf_, - ring_kv_buf_size / sizeof(T) + sizeof(int) / sizeof(T) * (batch_size + 1), - (*getDtypeMap())[inputDesc[0].type], next_rank, *mNcclComm, mNcclStream)); - NCCLCHECK(ncclRecv(ring_recv_kv_buf_, - ring_kv_buf_size / sizeof(T) + sizeof(int) / sizeof(T) * (batch_size + 1), - (*getDtypeMap())[inputDesc[0].type], prev_rank, *mNcclComm, mNcclStream)); - NCCLCHECK(ncclGroupEnd()); - } -#else - TLLM_LOG_ERROR("Please set ENABLE_MULTI_DEVICE to enable RingAttention"); - return 1; -#endif // ENABLE_MULTI_DEVICE - // Construct the fmha params for running kernels. - MHARunnerParams fmhaParams{}; - fmhaParams.b = request_batch_size; - fmhaParams.qSeqLen = request_seq_len; - fmhaParams.kvSeqLen = request_seq_len; - fmhaParams.totalQSeqLen = request_batch_size * request_seq_len; - // Device buffer pointers. - fmhaParams.qPtr = ring_q_buf_; - fmhaParams.kvPtr = ring_fmha_kv_buf_; - if (iter == 0) - { - fmhaParams.outputPtr = context_buf_; - fmhaParams.softmaxStatsPtr = ring_softmax_accu_stats_buf_; - } - else - { - cudaMemsetAsync(ring_softmax_stats_buf_, 0, ring_softmax_stats_buf_size, stream); - fmhaParams.outputPtr = ring_block_output_; - fmhaParams.softmaxStatsPtr = ring_softmax_stats_buf_; - } - fmhaParams.cuQSeqLenPtr = cu_seqlens; - fmhaParams.cuKvSeqLenPtr - = reinterpret_cast<int*>(reinterpret_cast<char*>(ring_fmha_kv_buf_) + ring_kv_buf_size); - - fmhaParams.tileCounterPtr = fmha_tile_counter_ptr; - fmhaParams.stream = stream; - // Run the fmha kernel. - cudaMemsetAsync(fmhaParams.outputPtr, 0, ring_block_output_size, stream); - cudaMemcpyAsync(fmhaParams.tileCounterPtr, fmha_scheduler_counter_h, sizeof(uint32_t), - cudaMemcpyHostToDevice, stream); - mFmhaDispatcher->run(fmhaParams); - if (iter != 0) - { - invokeRecoverFromRA<T>((T*) context_buf_, (float*) ring_softmax_accu_stats_buf_, - (T*) ring_block_output_, (float*) ring_softmax_stats_buf_, fmhaParams.b, fmhaParams.qSeqLen, - mNumHeads, mHeadSize, cu_seqlens, stream); - } - cudaStreamSynchronize(stream); - cudaStreamSynchronize(mNcclStream); - } - common::check_cuda_error(cudaStreamDestroy(mNcclStream)); - free(fmha_scheduler_counter_h); - } - - else - { - if (mSageAttn && mHeadSize == 72 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 64 - && mSageAttnVBlockSize == 256) - { - sage_quant<72, 80, 64, 64, 256, __nv_bfloat16, __nv_fp8_e4m3, float>( - // host var - batch_size, mNumHeads, input_seq_len, true, true, - // device var - // q k v - attention_input, attention_input + mNumHeads * mHeadSize, - attention_input + 2 * mNumHeads * mHeadSize, - // stride - 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, - cu_seqlens, sage_quant_space_ptr, - // quant q k v - quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, - quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, - // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, - 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, - // scales - q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); - - sync_check_cuda_error(stream); - } - if (mSageAttn && mHeadSize == 80 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 64 - && mSageAttnVBlockSize == 256) - { - sage_quant<80, 80, 64, 64, 256, __nv_bfloat16, __nv_fp8_e4m3, float>( - // host var - batch_size, mNumHeads, input_seq_len, true, true, - // device var - // q k v - attention_input, attention_input + mNumHeads * mHeadSize, - attention_input + 2 * mNumHeads * mHeadSize, - // stride - 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, - cu_seqlens, sage_quant_space_ptr, - // quant q k v - quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, - quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, - // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, - 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, - // scales - q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); - - sync_check_cuda_error(stream); - } - if (mSageAttn && mHeadSize == 128 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 64 - && mSageAttnVBlockSize == 256) - { - sage_quant<128, 128, 64, 64, 256, __nv_bfloat16, __nv_fp8_e4m3, float>( - // host var - batch_size, mNumHeads, input_seq_len, true, true, - // device var - // q k v - attention_input, attention_input + mNumHeads * mHeadSize, - attention_input + 2 * mNumHeads * mHeadSize, - // stride - 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, - cu_seqlens, sage_quant_space_ptr, - // quant q k v - quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, - quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, - // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, - 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, - // scales - q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); - - sync_check_cuda_error(stream); - } - if (mSageAttn && mHeadSize == 128 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 32 - && mSageAttnVBlockSize == 32) - { - sage_quant<128, 128, 64, 32, 32, __nv_bfloat16, __nv_fp8_e4m3, float>( - // host var - batch_size, mNumHeads, input_seq_len, true, true, - // device var - // q k v - attention_input, attention_input + mNumHeads * mHeadSize, - attention_input + 2 * mNumHeads * mHeadSize, - // stride - 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, - cu_seqlens, sage_quant_space_ptr, - // quant q k v - quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, - quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, - // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, - 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, - // scales - q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); - - sync_check_cuda_error(stream); - } - if (mSageAttn && mHeadSize == 80 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 32 - && mSageAttnVBlockSize == 32) - { - sage_quant<80, 80, 64, 32, 32, __nv_bfloat16, __nv_fp8_e4m3, float>( - // host var - batch_size, mNumHeads, input_seq_len, true, true, - // device var - // q k v - attention_input, attention_input + mNumHeads * mHeadSize, - attention_input + 2 * mNumHeads * mHeadSize, - // stride - 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, - cu_seqlens, sage_quant_space_ptr, - // quant q k v - quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, - quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, - // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, - 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, - // scales - q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); - - sync_check_cuda_error(stream); - } - if (mSageAttn && mHeadSize == 72 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 32 - && mSageAttnVBlockSize == 32) - { - sage_quant<72, 80, 64, 32, 32, __nv_bfloat16, __nv_fp8_e4m3, float>( - // host var - batch_size, mNumHeads, input_seq_len, true, true, - // device var - // q k v - attention_input, attention_input + mNumHeads * mHeadSize, - attention_input + 2 * mNumHeads * mHeadSize, - // stride - 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, - cu_seqlens, sage_quant_space_ptr, - // quant q k v - quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, - quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, - // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, - 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, - // scales - q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); - - sync_check_cuda_error(stream); - } - - // Construct the fmha params for running kernels. - MHARunnerParams fmhaParams{}; - fmhaParams.b = request_batch_size; - fmhaParams.qSeqLen = request_seq_len; - fmhaParams.kvSeqLen = request_seq_len; - fmhaParams.totalQSeqLen = request_batch_size * request_seq_len; - // Device buffer pointers. - fmhaParams.qkvPtr = attention_input; - fmhaParams.outputPtr = context_buf_; - fmhaParams.cuQSeqLenPtr = cu_seqlens; - fmhaParams.cuKvSeqLenPtr = cu_seqlens; - fmhaParams.tileCounterPtr = fmha_tile_counter_ptr; - fmhaParams.stream = stream; - if (mSageAttn) - { - if (paddedHeadSize != mHeadSize) - fmhaParams.outputPtr = sage_quant_space_ptr; - fmhaParams.qkvPtr = quanted_qkv_ptr; - fmhaParams.scaleBmm1Ptr = scale_bmm1_ptr; - fmhaParams.scaleBmm2Ptr = scale_bmm2_ptr; - fmhaParams.qScalePtr = q_scale_ptr; - fmhaParams.kScalePtr = k_scale_ptr; - fmhaParams.vScalePtr = v_scale_ptr; - fmhaParams.qMaxNBlock = (input_seq_len + mSageAttnQBlockSize - 1) / mSageAttnQBlockSize; - fmhaParams.kMaxNBlock = (input_seq_len + mSageAttnKBlockSize - 1) / mSageAttnKBlockSize; - fmhaParams.vMaxNBlock = (input_seq_len + mSageAttnVBlockSize - 1) / mSageAttnVBlockSize; - } - - // Run the fmha kernel. - - // TODO: set it correctly for contiguous kv buffer (cross-attention). - fmhaParams.totalKvSeqLen = num_tokens; - - fmhaParams.cuKvSeqLenPtr = cu_seqlens; - fmhaParams.cuMaskRowsPtr = cu_seqlens; - fmhaParams.tileCounterPtr = fmha_tile_counter_ptr; - - fmhaParams.scaleBmm1Ptr = scale_bmm1_ptr; - fmhaParams.scaleBmm2Ptr = scale_bmm2_ptr; - fmhaParams.forceFp32Acc = mFMHAForceFP32Acc; - mFmhaDispatcher->run(fmhaParams); - sync_check_cuda_error(stream); - if (mSageAttn) - { - if (paddedHeadSize != mHeadSize && mHeadSize == 72) - { - unpadding<80, 72, __nv_bfloat16>(batch_size, mNumHeads, input_seq_len, sage_quant_space_ptr, - mNumHeads * 72, mNumHeads * 80, cu_seqlens, context_buf_, stream); - } - } - } - } - else - { - // FIXME: a temporary solution to make sure the padding part of key/value buffer is 0 - // NOTE: pointer subtraction is used below since there could be some extra gap due to alignment. - // Otherwise, we could do cudaMemsetAsync(k_buf_2_, 0, k_buf_2_size + v_buf_2_size, stream); - // cudaMemsetAsync(k_buf_2_, 0, reinterpret_cast<int8_t*>(qk_buf_) - reinterpret_cast<int8_t*>(k_buf_2_), - // stream); - // FIXME: the final solution is to change the add_fusedQKV_bias_transpose_kernel to map CTAs corresponding to - // the output shape, and set the padding part to 0. Without zero-initialize guarantee, these workspace buffers - // may contain random NaN values when IFB workload is high. - cudaMemsetAsync(k_buf_2_, 0, - reinterpret_cast<int8_t*>(v_buf_2_) - reinterpret_cast<int8_t*>(k_buf_2_) + v_buf_2_size, stream); - - // only non-FMHA path needs to split Q,K,V from QKV - invokeAddFusedQKVBiasTranspose(q_buf_2_, k_buf_2_, v_buf_2_, const_cast<T*>(attention_input), input_lengths, - mRemovePadding ? padding_offset : nullptr, batch_size, input_seq_len, num_tokens, mNumHeads, mNumHeads, - mHeadSize, 0, 0.0f, RotaryScalingType::kNONE, 0.0f, 0, PositionEmbeddingType::kLEARNED_ABSOLUTE, - (float*) nullptr, 0, stream); - - if (!mQKHalfAccum && gemm_data_type != CUDA_R_32F) - { - mCublasWrapper->stridedBatchedGemm(CUBLAS_OP_T, CUBLAS_OP_N, - attention_seq_len_2, // n - attention_seq_len_1, // m - mHeadSize, // k - qk_scale_gemm, k_buf_2_, gemm_data_type, - mHeadSize, // k - attention_seq_len_2 * mHeadSize, // n * k - q_buf_2_, gemm_data_type, - mHeadSize, // k - attention_seq_len_1 * mHeadSize, // m * k - 0.0f, qk_buf_float_, CUDA_R_32F, - attention_seq_len_2, // n - attention_seq_len_2 * attention_seq_len_1, - request_batch_size * mNumHeads, // global batch size - CUDA_R_32F); - - // add relative position bias - if (mRelativeAttention) - { - // add rel pos bias - // QK is (batch_size, local_head_num, q_length, k_length), rel pos bias is (1, local_head_num, - // max_output_len + 1, max_output_len + 1). broadcast along 1st dim. max_seq_len is already - // max_output_len + 1. In implicit mode, relative_attention_bias is rel attn table - // [num_heads, num_buckets], with necessary params (max_distance, num_buckets) passed at the end - invokeAddRelativeAttentionBiasUnaligned(qk_buf_float_, relative_attn_table, request_batch_size, - mNumHeads, attention_seq_len_1, attention_seq_len_2, stream, mMaxDistance > 0, - inputDesc[3].dims.d[1], mMaxDistance, true /* bidirectional */); - } - - MaskedSoftmaxParam<T, float> param; - param.attention_score = qk_buf_; // (batch_size, head_num, q_length, k_length) - param.qk = qk_buf_float_; // (batch_size, head_num, q_length, k_length) - param.attention_mask = attention_mask; // (batch_size, q_length, k_length) - param.batch_size = request_batch_size; - param.q_length = attention_seq_len_1; - param.k_length = attention_seq_len_2; - param.num_heads = mNumHeads; - param.qk_scale = qk_scale_softmax; - param.linear_bias_slopes = const_cast<T*>(linear_bias_slopes); // (head_num,), optional - invokeMaskedSoftmax(param, stream); - } - else - { - mCublasWrapper->stridedBatchedGemm(CUBLAS_OP_T, CUBLAS_OP_N, attention_seq_len_2, attention_seq_len_1, - mHeadSize, k_buf_2_, mHeadSize, attention_seq_len_2 * mHeadSize, q_buf_2_, mHeadSize, - attention_seq_len_1 * mHeadSize, qk_buf_, attention_seq_len_2, - attention_seq_len_2 * attention_seq_len_1, request_batch_size * mNumHeads, qk_scale_gemm, - 0.0f); // alpha, beta - - // add relative position bias - if (mRelativeAttention) - { - // add rel pos bias - // QK is (batch_size, local_head_num, q_length, k_length), rel pos bias is (1, local_head_num, - // max_output_len + 1, max_output_len + 1). broadcast along 1st dim. max_seq_len is already - // max_output_len + 1. In implicit mode, relative_attention_bias is rel attn table - // [num_heads, num_buckets], with necessary params (max_distance, num_buckets) passed at the end - invokeAddRelativeAttentionBiasUnaligned(qk_buf_, relative_attn_table, request_batch_size, mNumHeads, - attention_seq_len_1, attention_seq_len_2, stream, mMaxDistance > 0, inputDesc[3].dims.d[1], - mMaxDistance, true /* bidirectional */); - } - - MaskedSoftmaxParam<T, T> param; - param.attention_score = qk_buf_; // (batch_size, head_num, q_length, k_length) - param.qk = qk_buf_; // (batch_size, head_num, q_length, k_length) - param.attention_mask = attention_mask; // (batch_size, q_length, k_length) - param.batch_size = request_batch_size; - param.q_length = attention_seq_len_1; - param.k_length = attention_seq_len_2; - param.num_heads = mNumHeads; - param.qk_scale = qk_scale_softmax; - param.linear_bias_slopes = const_cast<T*>(linear_bias_slopes); // (head_num,), optional - invokeMaskedSoftmax(param, stream); - } - - mCublasWrapper->stridedBatchedGemm(CUBLAS_OP_N, CUBLAS_OP_N, mHeadSize, attention_seq_len_1, - attention_seq_len_2, v_buf_2_, mHeadSize, attention_seq_len_2 * mHeadSize, qk_buf_, attention_seq_len_2, - attention_seq_len_1 * attention_seq_len_2, qkv_buf_2_, mHeadSize, attention_seq_len_1 * mHeadSize, - request_batch_size * mNumHeads); - - if (!mRemovePadding) - { - invokeTransposeQKV(context_buf_, qkv_buf_2_, request_batch_size, attention_seq_len_1, mNumHeads, mHeadSize, - (float*) nullptr, 0, stream); - } - else - { - invokeTransposeAttentionOutRemovePadding(qkv_buf_2_, context_buf_, num_tokens, request_batch_size, - request_seq_len, mNumHeads, mHeadSize, padding_offset, (float*) nullptr, 0, stream); - } - } - sync_check_cuda_error(stream); - return 0; -} - -template int BertAttentionPlugin::enqueueImpl<half>(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream); - -template int BertAttentionPlugin::enqueueImpl<float>(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream); - -#ifdef ENABLE_BF16 -template int BertAttentionPlugin::enqueueImpl<__nv_bfloat16>(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream); -#endif - -int BertAttentionPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - if (mType == DataType::kHALF) - { - return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (mType == DataType::kFLOAT) - { - return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#ifdef ENABLE_BF16 - else if (mType == DataType::kBF16) - { - return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#endif - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType BertAttentionPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return inputTypes[0]; -} - -// IPluginV2 Methods - -char const* BertAttentionPlugin::getPluginType() const noexcept -{ - return BERT_ATTENTION_PLUGIN_NAME; -} - -char const* BertAttentionPlugin::getPluginVersion() const noexcept -{ - return BERT_ATTENTION_PLUGIN_VERSION; -} - -int BertAttentionPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int BertAttentionPlugin::initialize() noexcept -{ - auto cublasHandle = getCublasHandle(); - auto cublasLtHandle = getCublasLtHandle(); - mCublasWrapper.reset(new tc::CublasMMWrapper(cublasHandle, cublasLtHandle, nullptr, nullptr)); - if (mEnableContextFMHA) - { - // Pre-checked during constructing. - Data_type data_type; - if (mType == DataType::kHALF) - { - data_type = DATA_TYPE_FP16; - } - else if (mType == DataType::kBF16) - { - data_type = DATA_TYPE_BF16; - } - else - { - TLLM_CHECK_WITH_INFO(false, "GPTAttentionPlugin received wrong data type."); - } - - // Construct the fmha runner. - MHARunnerFixedParams fmhaParams{}; - if (mSageAttn) - { - fmhaParams.dataType = DATA_TYPE_E4M3; - } - else - { - fmhaParams.dataType = data_type; - } - fmhaParams.dataTypeOut = data_type; - fmhaParams.forceFp32Acc = mFMHAForceFP32Acc; - fmhaParams.attentionMaskType = ContextAttentionMaskType::PADDING; - fmhaParams.isSPadded = !mRemovePadding; - fmhaParams.numQHeads = mNumHeads; - fmhaParams.numKvHeads = mNumHeads; - fmhaParams.headSize = mHeadSize; - fmhaParams.qScaling = mQScaling; - fmhaParams.sageBlockSizeQ = mSageAttnQBlockSize; - fmhaParams.sageBlockSizeK = mSageAttnKBlockSize; - fmhaParams.sageBlockSizeV = mSageAttnVBlockSize; - if (mSageAttn) - { - int const paddedHeadSize = ((mHeadSize + 15) / 16) * 16; - fmhaParams.headSize = paddedHeadSize; - } - - if (mCpGroup.size() > 1) - { - fmhaParams.attentionInputLayout = AttentionInputLayout::Q_CONTIGUOUS_KV; - fmhaParams.saveSoftmax = true; - } - - // Load kernels from the pre-compiled cubins. - // The KV input data type. The default is same as dataType. - fmhaParams.dataTypeKv = data_type; - fmhaParams.headSizeV = mHeadSize; - - // Load kernels from the pre-compiled cubins. - mFmhaDispatcher.reset(new FmhaDispatcher(fmhaParams)); - // Fall back to unfused MHA kernels if not supported. - mEnableContextFMHA = mFmhaDispatcher->isSupported(); - } - -#if ENABLE_MULTI_DEVICE - if (mCpGroup.size() > 1 && COMM_SESSION.getSize() > 1) - { - TLLM_LOG_TRACE("%s start for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); - mNcclComm = getComm(mCpGroup); - TLLM_LOG_TRACE("%s stop for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); - } -#endif // ENABLE_MULTI_DEVICE - - return 0; -} - -void BertAttentionPlugin::destroy() noexcept -{ - delete this; -} - -size_t BertAttentionPlugin::getSerializationSize() const noexcept -{ - return sizeof(mNumHeads) + sizeof(mHeadSize) + sizeof(mQScaling) + sizeof(mQKHalfAccum) + sizeof(mEnableContextFMHA) - + sizeof(mFMHAForceFP32Acc) + sizeof(mType) + sizeof(mRelativeAttention) + sizeof(mMaxDistance) - + sizeof(mRemovePadding) + sizeof(mSageAttn) + sizeof(mSageAttnQBlockSize) + sizeof(mSageAttnKBlockSize) - + sizeof(mSageAttnVBlockSize) + sizeof(mCpSize) + sizeof(mCpRank) + sizeof(int32_t) * mCpGroup.size(); -} - -void BertAttentionPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mNumHeads); - write(d, mHeadSize); - write(d, mQScaling); - write(d, mQKHalfAccum); - write(d, mEnableContextFMHA); - write(d, mFMHAForceFP32Acc); - write(d, mType); - write(d, mRelativeAttention); - write(d, mMaxDistance); - write(d, mRemovePadding); - write(d, mSageAttn); - write(d, mSageAttnQBlockSize); - write(d, mSageAttnKBlockSize); - write(d, mSageAttnVBlockSize); - write(d, mCpSize); - write(d, mCpRank); - for (auto it = mCpGroup.begin(); it != mCpGroup.end(); ++it) - { - write(d, *it); - } - TLLM_CHECK(d == a + getSerializationSize()); -} - -void BertAttentionPlugin::terminate() noexcept {} - -/////////////// - -BertAttentionPluginCreator::BertAttentionPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - - mPluginAttributes.emplace_back(PluginField("num_heads", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("head_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("q_scaling", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("context_fmha_type", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("do_relative_attention", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("max_distance", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("remove_padding", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("sage_attn", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("sage_attn_q_block_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("sage_attn_k_block_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("sage_attn_v_block_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("cp_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("cp_rank", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("cp_group", nullptr, PluginFieldType::kINT32)); - - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* BertAttentionPluginCreator::getPluginName() const noexcept -{ - return BERT_ATTENTION_PLUGIN_NAME; -} - -char const* BertAttentionPluginCreator::getPluginVersion() const noexcept -{ - return BERT_ATTENTION_PLUGIN_VERSION; -} - -PluginFieldCollection const* BertAttentionPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* BertAttentionPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int num_heads{}; - int head_size{}; - ContextFMHAType context_fmha_type{}; - float q_scaling{}; - nvinfer1::DataType type{}; - bool do_relative_attention{}; - int max_distance{}; - bool remove_padding{}; - bool sage_attn{}; - int sage_attn_q_block_size{}; - int sage_attn_k_block_size{}; - int sage_attn_v_block_size{}; - int cp_size{}; - int cp_rank{}; - std::set<int> cp_group{}; - - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "num_heads")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - num_heads = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "head_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - head_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "q_scaling")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - q_scaling = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - else if (!strcmp(attrName, "context_fmha_type")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - context_fmha_type = static_cast<ContextFMHAType>(*(static_cast<int8_t const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "do_relative_attention")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - do_relative_attention = static_cast<bool>(*(static_cast<int8_t const*>(fields[i].data))); - } - else if (!strcmp(attrName, "max_distance")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - max_distance = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "remove_padding")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - remove_padding = static_cast<bool>(*(static_cast<int8_t const*>(fields[i].data))); - } - else if (!strcmp(attrName, "sage_attn")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - sage_attn = static_cast<bool>(*(static_cast<int8_t const*>(fields[i].data))); - if (sage_attn) - { - std::cout << "sage attn true!" << std::endl; - } - } - else if (!strcmp(attrName, "sage_attn_q_block_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - sage_attn_q_block_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "sage_attn_k_block_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - sage_attn_k_block_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "sage_attn_v_block_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - sage_attn_v_block_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "cp_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - cp_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "cp_rank")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - cp_rank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "cp_group")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - auto const* r = static_cast<int const*>(fields[i].data); - for (int j = 0; j < fields[i].length; ++j) - { - cp_group.insert(*r); - ++r; - } - } - } - try - { - auto* obj = new BertAttentionPlugin(num_heads, head_size, q_scaling, context_fmha_type, type, - do_relative_attention, max_distance, remove_padding, sage_attn, sage_attn_q_block_size, - sage_attn_k_block_size, sage_attn_v_block_size, cp_size, cp_rank, cp_group); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* BertAttentionPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call BertAttentionPlugin::destroy() - try - { - auto* obj = new BertAttentionPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.h b/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.h deleted file mode 100644 index 2eb39086a005..000000000000 --- a/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.h +++ /dev/null @@ -1,142 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/cublasMMWrapper.h" -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/kernels/fmhaDispatcher.h" -#include "tensorrt_llm/kernels/gptKernels.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include <cassert> -#include <cuda_runtime.h> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class BertAttentionPlugin : public BasePlugin -{ -public: - BertAttentionPlugin() = delete; - - BertAttentionPlugin(int num_heads, int head_size, float q_scaling, - tensorrt_llm::kernels::ContextFMHAType context_fmha_type, nvinfer1::DataType type, - bool do_relative_attention = false, int max_distance = 0, bool remove_padding = false, bool sage_attn = false, - int sage_attn_q_block_size = 0, int sage_attn_k_block_size = 0, int sage_attn_v_block_size = 0, int cp_size = 1, - int cp_rank = 0, std::set<int> cp_group = {}); - - BertAttentionPlugin(void const* data, size_t length); - - ~BertAttentionPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - template <typename T> - int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - const std::string mLayerName; - - int mNumHeads; - int mHeadSize; - float mQScaling; - nvinfer1::DataType mType; - bool mRelativeAttention = false; - int mMaxDistance = 0; - bool mRemovePadding = false; - - // unfused mha - bool mQKHalfAccum = false; - - // fmha runner (disable by default) - bool mEnableContextFMHA = false; - bool mFMHAForceFP32Acc = false; - - // sage attention - bool mSageAttn = false; - int mSageAttnQBlockSize = 0; - int mSageAttnKBlockSize = 0; - int mSageAttnVBlockSize = 0; - std::set<std::vector<int>> mSageAttnSupportedBlockSizes{{64, 64, 256}, {64, 32, 32}}; - - int mSM = tensorrt_llm::common::getSMVersion(); - - // comm group for RingAttention - int mCpSize = 1; - int mCpRank = 0; - std::set<int> mCpGroup = {}; -#if ENABLE_MULTI_DEVICE - std::shared_ptr<ncclComm_t> mNcclComm; -#endif // ENABLE_MULTI_DEVICE - cudaStream_t mNcclStream; - - // The default copy constructor will leave them as nullptr. clone() shall initialize it. - UniqPtrWNullCopy<tensorrt_llm::kernels::FmhaDispatcher> mFmhaDispatcher; - UniqPtrWNullCopy<tensorrt_llm::common::CublasMMWrapper> mCublasWrapper; -}; - -class BertAttentionPluginCreator : public BaseCreator -{ -public: - BertAttentionPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/common/CMakeLists.txt b/cpp/tensorrt_llm/plugins/common/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/common/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.cpp b/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.cpp deleted file mode 100644 index 2aab6b3675d8..000000000000 --- a/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "checkMacrosPlugin.h" - -#include "tensorrt_llm/common/logger.h" - -namespace tensorrt_llm::plugins -{ - -void caughtError(std::exception const& e) -{ - TLLM_LOG_EXCEPTION(e); -} - -void logError(char const* msg, char const* file, char const* fn, int line) -{ - TLLM_LOG_ERROR("Parameter check failed at: %s::%s::%d, condition: %s", file, fn, line, msg); -} - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.cpp b/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.cpp deleted file mode 100644 index e5d6650648ab..000000000000 --- a/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.cpp +++ /dev/null @@ -1,404 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/common/cublasMMWrapper.h" -#include "tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm.h" -#include "tensorrt_llm/kernels/cutlass_kernels/fpA_intB_gemm/fpA_intB_gemm.h" -#include "tensorrt_llm/kernels/cutlass_kernels/fused_gated_gemm/fused_gated_gemm.h" -#include "tensorrt_llm/kernels/cutlass_kernels/int8_gemm/int8_gemm.h" -#include "tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.h" -#include "tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.h" -#include "tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.h" -#include "tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h" -#if defined(USING_OSS_CUTLASS_FP4_GEMM) -#include "tensorrt_llm/kernels/cutlass_kernels/include/fp4_gemm.h" -#else -#include "fp4_gemm.h" -#endif -#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) -#include "tensorrt_llm/kernels/cutlass_kernels/include/allreduce_gemm_runner.h" -using GemmAllReduceImplInterface = tensorrt_llm::kernels::opened_cutlass_kernels::GemmAllReduceImplInterface; -#else -#include "allreduce_gemm_runner.h" -using GemmAllReduceImplInterface = tensorrt_llm::kernels::cutlass_kernels::GemmAllReduceImplInterface; -#endif - -#include <cstddef> - -namespace tensorrt_llm::plugins -{ - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::GemmPluginProfiler() -{ - mMNKProfileMap = std::make_shared<MNKProfileMap>(); - - // set SKIP_GEMM_PLUGIN_PROFILINGS=1 to avoid tactics profilings - auto const skipEnv = std::getenv("SKIP_GEMM_PLUGIN_PROFILINGS"); - mSkip = (skipEnv != NULL && std::stoi(skipEnv)); - if (mSkip) - { - TLLM_LOG_DEBUG( - "SKIP_GEMM_PLUGIN_PROFILINGS is set. Skipping GEMM plugin profilings. It could result in runtime error " - "if default tactic is not defined."); - } -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::serialize( - char*& buffer, GemmIdType const& gemmId) const -{ - auto mProfileMap = mMNKProfileMap->getMProfileMap(gemmId); - - // Save number of profiles for given GEMM ID - write(buffer, static_cast<int>(mProfileMap->size())); - for (auto const& pair : *mProfileMap) - { - // Save pair of M to the best GEMM config - write(buffer, pair); - } -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::deserialize( - char const*& data, GemmDims& dims, GemmIdType const& gemmId) -{ - // NOTE: this mutex is not needed since each thread owns its private map, but will put here for - // consistency - writer_lock lock(mMNKProfileMap->mutex); - - mDims = dims; - - // GemmId gemmId(dims.n, dims.k); - if (!mMNKProfileMap->existsMProfileMap(gemmId)) - { - // Create GEMM with GEMM ID if it does not exist - mMNKProfileMap->createMProfileMap(gemmId); - } - // Populate map with profiles of GEMM ID - auto profileMap = mMNKProfileMap->getMProfileMap(gemmId); - int selectedMapSize; - read(data, selectedMapSize); - for (int ii = 0; ii < selectedMapSize; ++ii) - { - std::pair<int, std::optional<Config>> config; - read(data, config); - profileMap->insert(config); - } -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -size_t GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::getSerializationSize( - GemmIdType const& gemmId) const -{ - reader_lock lock(mMNKProfileMap->mutex); - return sizeof(int) + // size of the tactics map - mMNKProfileMap->getMProfileMap(gemmId)->size() - * sizeof(std::pair<int, std::optional<Config>>); // size of the tactics map -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -int GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::getMaxProfileM() const -{ - return 8192; -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::initTmpData( - int m, int n, int k, char* workspace, size_t size, cudaStream_t stream) -{ - /* Do nothing */ -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::profileTactics(RunnerPtr const& runner, - nvinfer1::DataType const& type, GemmDims const& dims, GemmIdType const& gemmId, bool hasWeightOnlyCudaKernel) -{ - writer_lock lock(mMNKProfileMap->mutex); - - if (!dims.isInitialized()) - { - return; - } - - mRunner = runner; - mType = type; - - int const maxM = std::min(nextPowerOfTwo(dims.maxM), getMaxProfileM()); - computeTmpSize(maxM, dims.n, dims.k); - - if (!mMNKProfileMap->existsMProfileMap(gemmId)) - { - // Create map for GEMM ID - mMNKProfileMap->createMProfileMap(gemmId); - } - - if (mSkip) - { - return; - } - - auto mProfileMap = mMNKProfileMap->getMProfileMap(gemmId); - bool isAllocated{false}; - - auto profileTactics = [&mProfileMap, &isAllocated, this](int m, int n, int k) - { - if (mProfileMap->count(m) == 0) - { - if (!isAllocated) - { - // Allocate tmp data to run GEMMs - allocateTmpData(); - isAllocated = true; - } - initTmpData(m, n, k, mWorkspaceTmp, mTmpWorkspaceSizeInBytes, mStream); - auto tactics = this->getTactics(m, n, k); - - // Profile different tactics for particular m and insert best config to the map - mProfileMap->insert({m, this->profileTacticsForProblem(m, n, k, tactics)}); - } - }; - - common::check_cuda_error(cudaStreamCreate(&mStream)); - - int const startMinMRounded = nextPowerOfTwo(dims.minM); - - if (hasWeightOnlyCudaKernel) - { - // Profile tactics for finer granularity of M, - // if CUDA kernel is enabled for weight-only plugins - int minM = dims.minM; - for (int m = std::max(1, minM); m < std::min(16, maxM); m += 1) - { - profileTactics(m, dims.n, dims.k); - } - - for (int m = 16; m < maxM; m *= 2) - { - profileTactics(m, dims.n, dims.k); - } - } - else - { - // Profile tactics for CUTLASS kernel only - for (int m = std::max(1, startMinMRounded); m < maxM; m *= 2) - { - profileTactics(m, dims.n, dims.k); - } - } - - profileTactics(maxM, dims.n, dims.k); - - if (isAllocated) - { - // Free tmp data - freeTmpData(); - } - common::check_cuda_error(cudaStreamDestroy(mStream)); -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -std::optional<Config> GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::getBestConfig( - int m, GemmIdType const& gemmId) const -{ - reader_lock lock(mMNKProfileMap->mutex); - - if (mSkip) - { - TLLM_LOG_TRACE("Skip is set, no best config is set for this instance"); - return std::nullopt; - } - - int const mRounded = std::min(std::max(1, nextPowerOfTwo(m)), getMaxProfileM()); - fflush(stdout); - - if (mMNKProfileMap->getMProfileMap(gemmId)->count(m) > 0) - { - return mMNKProfileMap->getMProfileMap(gemmId)->at(m); - } - else if (mMNKProfileMap->getMProfileMap(gemmId)->count(mRounded) > 0) - { - return mMNKProfileMap->getMProfileMap(gemmId)->at(mRounded); - } - else - { - std::ostringstream msg; - msg << "Cannot find best tactic for m=" << m << " and GEMM ID " << gemmId; - TLLM_LOG_WARNING(msg.str()); - return std::nullopt; - } -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::allocateTmpData() -{ - TLLM_CHECK_WITH_INFO(mTmpWorkspaceSizeInBytes > 0, "tmpWorkspaceSizeInBytes must be larger than 0"); - auto const status = cudaMalloc(&mWorkspaceTmp, mTmpWorkspaceSizeInBytes); - TLLM_CHECK_WITH_INFO(status == cudaSuccess, "Can't allocate tmp workspace for GEMM tactics profiling."); -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::freeTmpData() -{ - auto const status = cudaFree(mWorkspaceTmp); - TLLM_CHECK_WITH_INFO(status == cudaSuccess, "Can't free tmp workspace for GEMM tactics profiling."); -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -std::optional<Config> GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::profileTacticsForProblem( - int m, int n, int k, std::vector<Config> const& tactics) -{ - TLLM_LOG_DEBUG(__PRETTY_FUNCTION__); - - float bestTime = std::numeric_limits<float>::max(); - Config bestConfig; - bool foundOne = false; - - // Iterate over all tactics for given M, N and K - for (size_t ii = 0; ii < tactics.size(); ++ii) - { - Config const& candidateConfig = tactics[ii]; - float time = std::numeric_limits<float>::max(); - try - { - if (!checkTactic(m, n, k, candidateConfig)) - { - continue; - } - // Profile particular tactic for given M, N and K - time = profileTacticForProblem(m, n, k, candidateConfig); - foundOne = true; - } - catch (std::exception const& e) - { - std::ostringstream msg; - msg << "Cannot profile configuration " << ii; - if constexpr (std::is_same_v<Config, tensorrt_llm::cutlass_extensions::CutlassGemmConfig>) - { - msg << ": " << candidateConfig.toString(); - } - msg << "\n (for" - << " m=" << m << ", n=" << n << ", k=" << k << ")" - << ", reason: \"" << e.what() << "\". Skipped"; - TLLM_LOG_TRACE(msg.str()); - cudaGetLastError(); // Reset the last cudaError to cudaSuccess. - continue; - } - - // Choose the fastest tactic - if (time < bestTime) - { - bestConfig = candidateConfig; - bestTime = time; - } - } - - if (!foundOne) - { - std::ostringstream msg; - msg << "Have not found any valid GEMM config for shape (" - << "m=" << m << ", n=" << n << ", k=" << k << "). Will try to use default or fail at runtime"; - TLLM_LOG_WARNING(msg.str()); - return std::nullopt; - } - - return {bestConfig}; -} - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -float GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::profileTacticForProblem( - int m, int n, int k, Config const& tactic) -{ - constexpr int warmup = 5; - constexpr int runs = 10; - - cudaStream_t stream = mStream; - - // Warmup the execution - for (int i = 0; i < warmup; ++i) - { - runTactic(m, n, k, tactic, mWorkspaceTmp, stream); - } - - cudaEvent_t start; - cudaEvent_t stop; - common::check_cuda_error(cudaEventCreate(&start)); - common::check_cuda_error(cudaEventCreate(&stop)); - common::check_cuda_error(cudaStreamSynchronize(stream)); - common::check_cuda_error(cudaEventRecord(start, stream)); - - // Profile GEMM - for (int i = 0; i < runs; ++i) - { - runTactic(m, n, k, tactic, mWorkspaceTmp, stream); - } - - common::check_cuda_error(cudaEventRecord(stop, stream)); - - common::check_cuda_error(cudaEventSynchronize(stop)); - - float elapsed; - common::check_cuda_error(cudaEventElapsedTime(&elapsed, start, stop)); - - common::check_cuda_error(cudaEventDestroy(start)); - common::check_cuda_error(cudaEventDestroy(stop)); - - return elapsed / runs; -} - -template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassInt8GemmRunnerInterface>, GemmIdCore, - GemmIdCoreHash>; - -template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFpAIntBGemmRunnerInterface>, GemmIdCore, - GemmIdCoreHash>; - -template class GemmPluginProfiler<cublasLtMatmulHeuristicResult_t, - std::shared_ptr<tensorrt_llm::common::CublasMMWrapper>, GemmIdCublas, GemmIdCublasHash>; - -// TODO I dont like the dependency on the MOE plugin here, but MOE needs the full context to run profiles -template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, MixtureOfExpertsPlugin*, - GemmIDMoe, GemmIDMoeHash>; - -template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFusedGatedGemmRunnerInterface>, GemmIdCore, - GemmIdCoreHash>; - -template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFp8RowwiseGemmRunnerInterface>, GemmIdCore, - GemmIdCoreHash>; - -#if defined(USING_OSS_CUTLASS_FP4_GEMM) -template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFp4GemmRunnerInterface>, GemmIdCore, GemmIdCoreHash>; -#else -template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - std::shared_ptr<tensorrt_llm::kernels::internal_cutlass_kernels::CutlassFp4GemmRunnerInterface>, GemmIdCore, - GemmIdCoreHash>; -#endif - -template class GemmPluginProfiler<LowLatencyGemmPluginProfiler::Config, LowLatencyGemmRunnerPtr, GemmIdCore, - GemmIdCoreHash>; - -template class GemmPluginProfiler<LowLatencyGemmSwigluPluginProfiler::Config, LowLatencyGemmSwigluRunnerPtr, GemmIdCore, - GemmIdCoreHash>; - -template class GemmPluginProfiler<GemmAllReduceImplInterface::LaunchConfig, std::shared_ptr<GemmAllReduceImplInterface>, - GemmIdCore, GemmIdCoreHash>; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.h b/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.h deleted file mode 100644 index fe85b3b7e456..000000000000 --- a/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.h +++ /dev/null @@ -1,332 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "pluginUtils.h" - -#include <cuda_runtime.h> - -#include <cstdlib> -#include <iostream> -#include <memory> -#include <mutex> -#include <optional> -#include <shared_mutex> -#include <sstream> -#include <unordered_map> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -struct GemmDims -{ - using DimType64 = utils::DimType64; - - DimType64 minM; - DimType64 maxM; - DimType64 n; - DimType64 k; - - GemmDims() - : minM(-1) - , maxM(-1) - , n(-1) - , k(-1) - { - } - - GemmDims(DimType64 minM_, DimType64 maxM_, DimType64 n_, DimType64 k_) - : minM(minM_) - , maxM(maxM_) - , n(n_) - , k(k_) - { - } - - [[nodiscard]] bool isInitialized() const - { - return minM >= 0 && maxM >= 0 && n >= 0 && k >= 0; - } -}; - -// Unique ID of GEMM -// In our case GEMM is uniqly identified by N and K -class GemmIdCore -{ -public: - int n; - int k; - nvinfer1::DataType dtype; - - GemmIdCore(int n_, int k_, nvinfer1::DataType const& dtype_) - : n(n_) - , k(k_) - , dtype(dtype_) - { - } - - GemmIdCore() - : n(-1) - , k(-1) - , dtype(nvinfer1::DataType::kFLOAT) // dtype does not matter here - { - } - - bool operator==(GemmIdCore const& id) const - { - return isEqual(id); - } - - friend std::ostream& operator<<(std::ostream& out, GemmIdCore const& id) - { - out << "(N;K)=(" << id.n << ";" << id.k << "),"; - out << " type=" << static_cast<int>(id.dtype); - return out; - } - -protected: - bool isEqual(GemmIdCore const& id) const - { - return n == id.n && k == id.k && dtype == id.dtype; - } -}; - -// Hash of GemmId -struct GemmIdCoreHash -{ - std::size_t operator()(GemmIdCore const& id) const - { - auto h1 = std::hash<int>{}(id.n); - auto h2 = std::hash<int>{}(id.k); - auto h3 = std::hash<int>{}(static_cast<int>(id.dtype)); - return h1 ^ h2 ^ h3; - } -}; - -class GemmIdCublas : public GemmIdCore -{ -public: - bool transA{}; - bool transB{}; - nvinfer1::DataType outputDtype; - - GemmIdCublas(int n_, int k_, nvinfer1::DataType const& dtype_, bool transA_, bool transB_, - nvinfer1::DataType const& output_dtype_) - : GemmIdCore(n_, k_, dtype_) - , transA(transA_) - , transB(transB_) - , outputDtype(output_dtype_) - { - } - - GemmIdCublas() {} - - bool operator==(GemmIdCublas const& id) const - { - return isEqual(id) && transA == id.transA && transB == id.transB && outputDtype == id.outputDtype; - } - - friend std::ostream& operator<<(std::ostream& out, GemmIdCublas const& id) - { - out << "(N;K)=(" << id.n << ";" << id.k << "),"; - out << " type=" << static_cast<int>(id.dtype); - out << " transA=" << id.transA; - out << " transB=" << id.transB; - out << " outputDtype=" << static_cast<int>(id.outputDtype); - return out; - } -}; - -// Hash of GemmIdCublas -struct GemmIdCublasHash -{ - std::size_t operator()(GemmIdCublas const& id) const - { - auto h1 = std::hash<int>{}(id.n); - auto h2 = std::hash<int>{}(id.k); - auto h3 = std::hash<int>{}(static_cast<int>(id.dtype)); - auto h4 = std::hash<bool>{}(id.transA); - auto h5 = std::hash<bool>{}(id.transB); - auto h6 = std::hash<bool>{}(static_cast<int>(id.outputDtype)); - return h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ h6; - } -}; - -template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> -class GemmPluginProfiler -{ -public: - // Map for single GEMM for different Ms (GEMM dimension) to the best config for particular M - using MProfileMap = std::unordered_map<int, std::optional<Config>>; - using MProfileMapPtr = std::shared_ptr<MProfileMap>; - - // requires exclusive ownership to write to *this - using reader_lock = std::unique_lock<std::shared_timed_mutex>; - // requires shared ownership to read from other - using writer_lock = std::shared_lock<std::shared_timed_mutex>; - - // Struct of continuing map if GEMMs to the best profiles for different Ms - struct MNKProfileMap - { - // Mutex guarding map - std::shared_timed_mutex mutex; - // Map from GEMM Id to profile for particular GEMM - std::unordered_map<GemmIdType, MProfileMapPtr, GemmIdHashType> profileMap; - - bool existsMProfileMap(GemmIdType const& id) - { - auto const iter = profileMap.find(id); - return iter != profileMap.end(); - } - - void createMProfileMap(GemmIdType const& id) - { - profileMap[id] = std::make_shared<MProfileMap>(); - } - - MProfileMapPtr getMProfileMap(GemmIdType const& id) - { - auto const iter = profileMap.find(id); - if (iter == profileMap.end()) - { - std::ostringstream msg; - msg << "Cannot find ID (" << id << ") in the profile map. Abort."; - TLLM_THROW(msg.str()); - } - return iter->second; - } - }; - - using MNKProfileMapPtr = std::shared_ptr<MNKProfileMap>; - - GemmPluginProfiler(); - - virtual ~GemmPluginProfiler() = default; - - void serialize(char*& buffer, GemmIdType const& gemmId) const; - - void deserialize(char const*& data, GemmDims& dims, GemmIdType const& gemmId); - size_t getSerializationSize(GemmIdType const& gemmId) const; - - void profileTactics(RunnerPtr const& runner, nvinfer1::DataType const& type, GemmDims const& dims, - GemmIdType const& gemmId, bool hasWeightOnlyCudaKernel = false); - - void setSelectionTactics(MNKProfileMapPtr const& map) - { - mMNKProfileMap = map; - } - - void setTmpWorkspaceSizeInBytes(size_t bytes) - { - mTmpWorkspaceSizeInBytes = bytes; - } - - void setSkip(bool skip) - { - mSkip = mSkip || skip; - } - - std::optional<Config> getBestConfig(int m, GemmIdType const& gemmId) const; - - virtual int getMaxProfileM() const; - -protected: - virtual void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) = 0; - - virtual void computeTmpSize(size_t maxM, size_t n, size_t k) = 0; - - virtual bool checkTactic(int m, int n, int k, Config const& tactic) const - { - return true; - } - - virtual std::vector<Config> getTactics(int m, int n, int k) const = 0; - - virtual void initTmpData(int m, int n, int k, char* workspace, size_t size, cudaStream_t stream); - -private: - void allocateTmpData(); - - void freeTmpData(); - - std::optional<Config> profileTacticsForProblem(int m, int n, int k, std::vector<Config> const& tactics); - - float profileTacticForProblem(int m, int n, int k, Config const& tactic); - - int nextPowerOfTwo(int v) const - { - --v; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return ++v; - } - -protected: - RunnerPtr mRunner{nullptr}; - - nvinfer1::DataType mType{}; - -private: - MNKProfileMapPtr mMNKProfileMap{}; - - size_t mTmpWorkspaceSizeInBytes{0}; - - char* mWorkspaceTmp{nullptr}; - - cudaStream_t mStream; - - GemmDims mDims{}; - - bool mSkip{false}; -}; - -template <typename GemmPluginProfilerType> -class GemmPluginProfilerManager -{ -public: - using MNKProfileMap = typename GemmPluginProfilerType::MNKProfileMap; - using MNKProfileMapPtr = typename GemmPluginProfilerType::MNKProfileMapPtr; - using GemmPluginProfilerPtr = std::shared_ptr<GemmPluginProfilerType>; - - GemmPluginProfilerManager() - { - mMNKProfileMap = std::make_shared<MNKProfileMap>(); - } - - GemmPluginProfilerPtr createGemmPluginProfiler(bool inference, bool skip = false) - { - auto profiler = std::make_shared<GemmPluginProfilerType>(); - profiler->setSkip(skip); - // If the profiler is created during the engine build, - // mMNKProfileMap is shared between different profilers to minimize the time spent on the profiling - // and do not repeat profiling for the GEMMs of the same shape. - if (!inference) - { - profiler->setSelectionTactics(mMNKProfileMap); - } - return profiler; - } - -private: - MNKProfileMapPtr mMNKProfileMap{}; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/common/plugin.cpp b/cpp/tensorrt_llm/plugins/common/plugin.cpp deleted file mode 100644 index 82c8bf93b13c..000000000000 --- a/cpp/tensorrt_llm/plugins/common/plugin.cpp +++ /dev/null @@ -1,124 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "tensorrt_llm/plugins/common/plugin.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" - -#include "checkMacrosPlugin.h" -#include <cstdint> -#include <functional> - -#ifdef _MSC_VER -#define FN_NAME __FUNCTION__ -#else -#define FN_NAME __func__ -#endif - -PluginFieldParser::PluginFieldParser(int32_t nbFields, nvinfer1::PluginField const* fields) - : mFields{fields} -{ - for (int32_t i = 0; i < nbFields; i++) - { - mMap.emplace(fields[i].name, PluginFieldParser::Record{i}); - } -} - -PluginFieldParser::~PluginFieldParser() -{ - for (auto const& [name, record] : mMap) - { - if (!record.retrieved) - { - std::stringstream ss; - ss << "unused plugin field with name: " << name; - tensorrt_llm::plugins::logError(ss.str().c_str(), __FILE__, FN_NAME, __LINE__); - } - } -} - -template <typename T> -nvinfer1::PluginFieldType toFieldType(); -#define SPECIALIZE_TO_FIELD_TYPE(T, type) \ - template <> \ - nvinfer1::PluginFieldType toFieldType<T>() \ - { \ - return nvinfer1::PluginFieldType::type; \ - } -SPECIALIZE_TO_FIELD_TYPE(half, kFLOAT16) -SPECIALIZE_TO_FIELD_TYPE(float, kFLOAT32) -SPECIALIZE_TO_FIELD_TYPE(double, kFLOAT64) -SPECIALIZE_TO_FIELD_TYPE(int8_t, kINT8) -SPECIALIZE_TO_FIELD_TYPE(int16_t, kINT16) -SPECIALIZE_TO_FIELD_TYPE(int32_t, kINT32) -SPECIALIZE_TO_FIELD_TYPE(char, kCHAR) -SPECIALIZE_TO_FIELD_TYPE(nvinfer1::Dims, kDIMS) -SPECIALIZE_TO_FIELD_TYPE(void, kUNKNOWN) -#undef SPECIALIZE_TO_FIELD_TYPE - -template <typename T> -std::optional<T> PluginFieldParser::getScalar(std::string_view const& name) -{ - auto const iter = mMap.find(name); - if (iter == mMap.end()) - { - return std::nullopt; - } - auto& record = mMap.at(name); - auto const& f = mFields[record.index]; - TLLM_CHECK(toFieldType<T>() == f.type && f.length == 1); - record.retrieved = true; - return std::optional{*static_cast<T const*>(f.data)}; -} - -#define INSTANTIATE_PluginFieldParser_getScalar(T) \ - template std::optional<T> PluginFieldParser::getScalar(std::string_view const&) -INSTANTIATE_PluginFieldParser_getScalar(half); -INSTANTIATE_PluginFieldParser_getScalar(float); -INSTANTIATE_PluginFieldParser_getScalar(double); -INSTANTIATE_PluginFieldParser_getScalar(int8_t); -INSTANTIATE_PluginFieldParser_getScalar(int16_t); -INSTANTIATE_PluginFieldParser_getScalar(int32_t); -INSTANTIATE_PluginFieldParser_getScalar(char); -INSTANTIATE_PluginFieldParser_getScalar(nvinfer1::Dims); -#undef INSTANTIATE_PluginFieldParser_getScalar - -template <typename T> -std::optional<std::set<T>> PluginFieldParser::getSet(std::string_view const& name) -{ - auto const iter = mMap.find(name); - if (iter == mMap.end()) - { - return std::nullopt; - } - auto& record = mMap.at(name); - auto const& f = mFields[record.index]; - TLLM_CHECK(toFieldType<T>() == f.type); - std::set<T> group; - auto const* r = static_cast<T const*>(f.data); - for (int j = 0; j < f.length; ++j) - { - group.insert(*r); - ++r; - } - - record.retrieved = true; - return std::optional{group}; -} - -#define INSTANTIATE_PluginFieldParser_getVector(T) \ - template std::optional<std::set<T>> PluginFieldParser::getSet(std::string_view const&) -INSTANTIATE_PluginFieldParser_getVector(int32_t); -#undef INSTANTIATE_PluginFieldParser_getVector diff --git a/cpp/tensorrt_llm/plugins/common/plugin.h b/cpp/tensorrt_llm/plugins/common/plugin.h deleted file mode 100644 index a7febe4cc13d..000000000000 --- a/cpp/tensorrt_llm/plugins/common/plugin.h +++ /dev/null @@ -1,143 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/common/opUtils.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/plugins/common/checkMacrosPlugin.h" - -#include <NvInferRuntime.h> - -#include <cstring> -#include <map> -#include <memory> -#include <optional> -#include <set> -#include <string> -#include <unordered_map> - -namespace tensorrt_llm::plugins -{ - -using namespace tensorrt_llm::common::op; - -class BasePlugin : public nvinfer1::IPluginV2DynamicExt -{ -public: - void setPluginNamespace(char const* libNamespace) noexcept override - { - mNamespace = libNamespace; - } - - [[nodiscard]] char const* getPluginNamespace() const noexcept override - { - return mNamespace.c_str(); - } - -protected: - std::string mNamespace{api::kDefaultNamespace}; -}; - -class BasePluginV3 : public nvinfer1::IPluginV3, - public nvinfer1::IPluginV3OneCore, - public nvinfer1::IPluginV3OneBuild, - public nvinfer1::IPluginV3OneRuntime -{ -public: - void setPluginNamespace(char const* libNamespace) noexcept - { - mNamespace = libNamespace; - } - - [[nodiscard]] char const* getPluginNamespace() const noexcept override - { - return mNamespace.c_str(); - } - -protected: - std::string mNamespace{api::kDefaultNamespace}; -}; - -class BaseCreator : public nvinfer1::IPluginCreator -{ -public: - void setPluginNamespace(char const* libNamespace) noexcept override - { - mNamespace = libNamespace; - } - - [[nodiscard]] char const* getPluginNamespace() const noexcept override - { - return mNamespace.c_str(); - } - -protected: - std::string mNamespace{api::kDefaultNamespace}; -}; - -class BaseCreatorV3 : public nvinfer1::IPluginCreatorV3One -{ -public: - void setPluginNamespace(char const* libNamespace) noexcept - { - mNamespace = libNamespace; - } - - [[nodiscard]] char const* getPluginNamespace() const noexcept override - { - return mNamespace.c_str(); - } - -protected: - std::string mNamespace{api::kDefaultNamespace}; -}; - -} // namespace tensorrt_llm::plugins - -// Init with O(n) and retrieve with O(1) -class PluginFieldParser -{ -public: - // field array must remain valid when calling getScalar() later. - PluginFieldParser(int32_t nbFields, nvinfer1::PluginField const* fields); - // delete to remind accidental mis-use (copy) which may result in false-alarm warnings about unused fields. - PluginFieldParser(PluginFieldParser const&) = delete; - PluginFieldParser& operator=(PluginFieldParser const&) = delete; - // check if all fields are retrieved and emit warning if some of them are not. - ~PluginFieldParser(); - template <typename T> - std::optional<T> getScalar(std::string_view const& name); - template <typename T> - std::optional<std::set<T>> getSet(std::string_view const& name); - -private: - nvinfer1::PluginField const* mFields; - - struct Record - { - Record(int32_t idx) - : index{idx} - { - } - - int32_t const index; - bool retrieved{false}; - }; - - std::unordered_map<std::string_view, Record> mMap; -}; diff --git a/cpp/tensorrt_llm/plugins/common/pluginUtils.h b/cpp/tensorrt_llm/plugins/common/pluginUtils.h deleted file mode 100644 index ee3e59d57c6d..000000000000 --- a/cpp/tensorrt_llm/plugins/common/pluginUtils.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <NvInferRuntime.h> - -#include "tensorrt_llm/common/logger.h" - -namespace tensorrt_llm::plugins::utils -{ -using DimType64 = int64_t; - -inline DimType64 computeMDimension(bool transA, nvinfer1::Dims const& dims) -{ - DimType64 M{1}; - if (transA) - { - for (int i = dims.nbDims - 1; i > 0; --i) - { - M *= dims.d[i]; - } - } - else - { - for (int i = 0; i < dims.nbDims - 1; ++i) - { - M *= dims.d[i]; - } - } - return M; -} - -inline DimType64 computeNDimension(bool transB, nvinfer1::Dims const& dims) -{ - DimType64 N{1}; - if (transB) - { - for (int32_t i = 0; i < dims.nbDims - 1; ++i) - { - N *= dims.d[i]; - } - } - else - { - for (int32_t i = dims.nbDims - 1; i > 0; --i) - { - N *= dims.d[i]; - } - } - return N; -} - -inline std::int32_t logErrorReturn0(char const* variable) -{ - TLLM_LOG_ERROR("Value of %s is out of range for int32_t", variable); - return 0; -} - -#define TLLM_INT32_CAST(value) \ - ((value > 0x7FFFFFFFLL || value < -0x80000000LL) ? tensorrt_llm::plugins::utils::logErrorReturn0(#value) \ - : static_cast<int32_t>(value)) - -} // namespace tensorrt_llm::plugins::utils diff --git a/cpp/tensorrt_llm/plugins/cpSplitPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/cpSplitPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/cpSplitPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.cpp b/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.cpp deleted file mode 100644 index 221d5dac2da1..000000000000 --- a/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.cpp +++ /dev/null @@ -1,356 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <cstdio> - -#include "cpSplitPlugin.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::CpSplitPluginCreator; -using tensorrt_llm::plugins::CpSplitPlugin; - -static char const* CPSPLIT_PLUGIN_VERSION{"1"}; -static char const* CPSPLIT_PLUGIN_NAME{"CpSplit"}; -PluginFieldCollection CpSplitPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> CpSplitPluginCreator::mPluginAttributes; - -CpSplitPlugin::CpSplitPlugin() -{ - initFieldsToSerialize(); -} - -CpSplitPlugin::CpSplitPlugin(int cpSize, int cpRank) - : mCpSize(cpSize) - , mCpRank(cpRank) -{ - initFieldsToSerialize(); -} - -void CpSplitPlugin::initFieldsToSerialize() -{ - mDataToSerialize.clear(); - mDataToSerialize.emplace_back(PluginField("cp_size", &mCpSize, PluginFieldType::kINT32, 1)); - mDataToSerialize.emplace_back(PluginField("cp_rank", &mCpRank, PluginFieldType::kINT32, 1)); - mFCToSerialize.nbFields = mDataToSerialize.size(); - mFCToSerialize.fields = mDataToSerialize.data(); -} - -// IPluginV3 methods -nvinfer1::IPluginCapability* CpSplitPlugin::getCapabilityInterface(nvinfer1::PluginCapabilityType type) noexcept -{ - switch (type) - { - case PluginCapabilityType::kBUILD: return static_cast<IPluginV3OneBuild*>(this); - case PluginCapabilityType::kRUNTIME: return static_cast<IPluginV3OneRuntime*>(this); - case PluginCapabilityType::kCORE: return static_cast<IPluginV3OneCore*>(this); - } - return nullptr; -} - -nvinfer1::IPluginV3* CpSplitPlugin::clone() noexcept -{ - std::unique_ptr<CpSplitPlugin> plugin{std::make_unique<CpSplitPlugin>(*this)}; - plugin->setPluginNamespace(mNamespace.c_str()); - plugin->initFieldsToSerialize(); - return plugin.release(); -} - -// IPluginV3OneCore methods -char const* CpSplitPlugin::getPluginName() const noexcept -{ - return CPSPLIT_PLUGIN_NAME; -} - -char const* CpSplitPlugin::getPluginVersion() const noexcept -{ - return CPSPLIT_PLUGIN_VERSION; -} - -// IPluginV3OneBuild methods -int32_t CpSplitPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept -{ - return 0; -} - -int32_t CpSplitPlugin::getOutputDataTypes( - DataType* outputTypes, int32_t nbOutputs, DataType const* inputTypes, int32_t nbInputs) const noexcept -{ - outputTypes[0] = inputTypes[0]; - outputTypes[1] = DataType::kINT32; - outputTypes[2] = DataType::kINT32; - return 0; -} - -int32_t CpSplitPlugin::getOutputShapes(DimsExprs const* inputs, int32_t nbInputs, DimsExprs const* shapeInputs, - int32_t nbShapeInputs, DimsExprs* outputs, int32_t nbOutputs, IExprBuilder& exprBuilder) noexcept -{ - outputs[0].nbDims = 1; - - auto cpSize = exprBuilder.constant(mCpSize); - auto upper = inputs[0].d[0]; - auto opt = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *upper, *cpSize); - outputs[0].d[0] = exprBuilder.declareSizeTensor(1, *opt, *upper); - - // We must have such an output size tensor (with dim == 0) to notify the shape of output tensor above - outputs[1].nbDims = 0; - outputs[2].nbDims = 1; - outputs[2].d[0] = upper; - return 0; -} - -bool CpSplitPlugin::supportsFormatCombination( - int32_t pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept -{ - if (pos == IdxEntry::INPUT_IDS) - { - return ((inOut[pos].desc.type == DataType::kINT32) && (inOut[pos].desc.format == TensorFormat::kLINEAR)); - } - else if (pos == IdxEntry::REQUEST_TYPES || pos == IdxEntry::HOST_CONTEXT_LENGTH) - { - return inOut[pos].desc.type == DataType::kINT32; - } - else - { - return ((inOut[pos].desc.type == DataType::kINT32) && (inOut[pos].desc.format == TensorFormat::kLINEAR)); - } - return false; -} - -int32_t CpSplitPlugin::getNbOutputs() const noexcept -{ - return 3; -} - -size_t CpSplitPlugin::getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int32_t nbInputs, - nvinfer1::DynamicPluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept -{ - return 0; -} - -int32_t CpSplitPlugin::getValidTactics(int32_t* tactics, int32_t nbTactics) noexcept -{ - return 0; -} - -int32_t CpSplitPlugin::getNbTactics() noexcept -{ - return 0; -} - -char const* CpSplitPlugin::getTimingCacheID() noexcept -{ - return nullptr; -} - -int32_t CpSplitPlugin::getFormatCombinationLimit() noexcept -{ - return 1; -} - -char const* CpSplitPlugin::getMetadataString() noexcept -{ - return nullptr; -} - -// IPluginV3OneRuntime methods -int32_t CpSplitPlugin::setTactic(int32_t tactic) noexcept -{ - return 0; -} - -int32_t CpSplitPlugin::onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, - nvinfer1::PluginTensorDesc const* out, int32_t nbOutputs) noexcept -{ - return 0; -} - -int32_t CpSplitPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // @param inputIds [tokenNum] - // @param host_request_types [batchSize]: Tensor = None (On CPU) - // The tensor on the host that indicates if a request is in context or - // generation phase. Its shape is [batch_size]. See Inflight Batching - // in docs/gpt_attention.md, - // @param host_context_lengths [batchSize]: Tensor = None (On CPU) - // A host tensor that contains the lengths of the different inputs - // outputs - // @param outputIds [tokenNum spiltted by cp] - // @param outputLength scalar - // @param joinIdx [tokenNum] - - int64_t tokenNum = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) - { - tokenNum *= inputDesc[0].dims.d[i]; - } - - RequestType const* reqTypes = static_cast<RequestType const*>(inputs[IdxEntry::REQUEST_TYPES]); - int32_t const* hContextLengths = static_cast<int32_t const*>(inputs[IdxEntry::HOST_CONTEXT_LENGTH]); - int const* inputIds = reinterpret_cast<int const*>(inputs[IdxEntry::INPUT_IDS]); - int* outputIds = reinterpret_cast<int*>(outputs[0]); - int32_t* outputLength = reinterpret_cast<int32_t*>(outputs[1]); - int32_t* outputJoinIdx = reinterpret_cast<int32_t*>(outputs[2]); - - int32_t const nbSeq = inputDesc[IdxEntry::HOST_CONTEXT_LENGTH].dims.d[0]; - - int32_t* hInputs = new int[inputDesc[IdxEntry::INPUT_IDS].dims.d[0]]; - int32_t* hOutputs = new int[inputDesc[IdxEntry::INPUT_IDS].dims.d[0]]; - int32_t* hOutputJoinIdx = new int[inputDesc[IdxEntry::INPUT_IDS].dims.d[0]]; - cudaMemcpyAsync( - hInputs, inputIds, sizeof(int32_t) * inputDesc[IdxEntry::INPUT_IDS].dims.d[0], cudaMemcpyDeviceToHost, stream); - sync_check_cuda_error(stream); - - int32_t inputIdx = 0; - int32_t outputIdx = 0; - for (int32_t seqIdx = 0; seqIdx < nbSeq; ++seqIdx) - { - if (reqTypes[seqIdx] == RequestType::kCONTEXT) - { - auto const& ctxLength = hContextLengths[seqIdx]; - int32_t partialAverageLength = (ctxLength + mCpSize - 1) / mCpSize; - int32_t partialLength - = mCpRank == mCpSize - 1 ? ctxLength - partialAverageLength * (mCpSize - 1) : partialAverageLength; - for (int i = 0; i < partialLength; i++) - { - hOutputs[outputIdx + i] = hInputs[inputIdx + partialAverageLength * mCpRank + i]; - } - inputIdx += ctxLength; - outputIdx += partialAverageLength; - } - else if (reqTypes[seqIdx] == RequestType::kGENERATION) - { - auto const& genLength = nbSeq - seqIdx; - int32_t partialAverageLength = (genLength + mCpSize - 1) / mCpSize; - int32_t partialLength - = mCpRank == mCpSize - 1 ? genLength - partialAverageLength * (mCpSize - 1) : partialAverageLength; - for (int i = 0; i < partialLength; i++) - { - hOutputs[outputIdx + i] = hInputs[inputIdx + partialAverageLength * mCpRank + i]; - } - outputIdx += partialAverageLength; - break; - } - } - int32_t hOutputLength = outputIdx; - inputIdx = 0; - outputIdx = 0; - for (int32_t seqIdx = 0; seqIdx < nbSeq; ++seqIdx) - { - if (reqTypes[seqIdx] == RequestType::kCONTEXT) - { - auto const& ctxLength = hContextLengths[seqIdx]; - int32_t partialAverageLength = (ctxLength + mCpSize - 1) / mCpSize; - for (int32_t idx = 0; idx < ctxLength; ++idx) - { - hOutputJoinIdx[inputIdx + idx] - = idx % partialAverageLength + idx / partialAverageLength * hOutputLength + outputIdx; - } - inputIdx += ctxLength; - outputIdx += partialAverageLength; - } - else if (reqTypes[seqIdx] == RequestType::kGENERATION) - { - auto const& genLength = nbSeq - seqIdx; - int32_t partialAverageLength = (genLength + mCpSize - 1) / mCpSize; - for (int32_t idx = 0; idx < genLength; ++idx) - { - hOutputJoinIdx[inputIdx + idx] - = idx % partialAverageLength + idx / partialAverageLength * hOutputLength + outputIdx; - } - break; - } - } - cudaMemcpyAsync(outputIds, hOutputs, sizeof(int32_t) * hOutputLength, cudaMemcpyHostToDevice, stream); - cudaMemcpyAsync(outputLength, &hOutputLength, sizeof(int32_t), cudaMemcpyHostToDevice, stream); - cudaMemcpyAsync(outputJoinIdx, hOutputJoinIdx, sizeof(int32_t) * tokenNum, cudaMemcpyHostToDevice, stream); - sync_check_cuda_error(stream); - return 0; -} - -nvinfer1::IPluginV3* CpSplitPlugin::attachToContext(nvinfer1::IPluginResourceContext* context) noexcept -{ - return clone(); -} - -nvinfer1::PluginFieldCollection const* CpSplitPlugin::getFieldsToSerialize() noexcept -{ - return &mFCToSerialize; -} - -CpSplitPluginCreator::CpSplitPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("cp_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("cp_rank", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* CpSplitPluginCreator::getPluginName() const noexcept -{ - return CPSPLIT_PLUGIN_NAME; -} - -char const* CpSplitPluginCreator::getPluginVersion() const noexcept -{ - return CPSPLIT_PLUGIN_VERSION; -} - -nvinfer1::PluginFieldCollection const* CpSplitPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -nvinfer1::IPluginV3* CpSplitPluginCreator::createPlugin( - char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept -{ - PluginField const* fields = fc->fields; - int cp_size{}; - int cp_rank{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "cp_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - cp_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "cp_rank")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - cp_rank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - } - try - { - auto* obj = new CpSplitPlugin(cp_size, cp_rank); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.h b/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.h deleted file mode 100644 index 1dc8c15b355a..000000000000 --- a/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class CpSplitPlugin : public BasePluginV3 -{ -public: - CpSplitPlugin(); - CpSplitPlugin(int cpSize, int cpRank); - CpSplitPlugin(CpSplitPlugin const& p) = default; - void initFieldsToSerialize(); - - // IPluginV3 methods - nvinfer1::IPluginCapability* getCapabilityInterface(nvinfer1::PluginCapabilityType type) noexcept override; - nvinfer1::IPluginV3* clone() noexcept override; - - // IPluginV3OneCore methods - char const* getPluginName() const noexcept override; - char const* getPluginVersion() const noexcept override; - - // IPluginV3OneBuild methods - int32_t configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept override; // nochange - int32_t getOutputDataTypes(nvinfer1::DataType* outputTypes, int32_t nbOutputs, nvinfer1::DataType const* inputTypes, - int32_t nbInputs) const noexcept override; // fixed - int32_t getOutputShapes(nvinfer1::DimsExprs const* inputs, int32_t nbInputs, nvinfer1::DimsExprs const* shapeInputs, - int32_t nbShapeInputs, nvinfer1::DimsExprs* outputs, int32_t nbOutputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; // fixed - bool supportsFormatCombination(int32_t pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int32_t nbInputs, - int32_t nbOutputs) noexcept override; // fixed - int32_t getNbOutputs() const noexcept override; // fixed - size_t getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int32_t nbInputs, - nvinfer1::DynamicPluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept override; // fixed - int32_t getValidTactics(int32_t* tactics, int32_t nbTactics) noexcept override; - int32_t getNbTactics() noexcept override; - char const* getTimingCacheID() noexcept override; - int32_t getFormatCombinationLimit() noexcept override; - char const* getMetadataString() noexcept override; - - // IPluginV3OneRuntime methods - int32_t setTactic(int32_t tactic) noexcept override; - int32_t onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, nvinfer1::PluginTensorDesc const* out, - int32_t nbOutputs) noexcept override; - int32_t enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept override; // fixed - nvinfer1::IPluginV3* attachToContext(nvinfer1::IPluginResourceContext* context) noexcept override; - nvinfer1::PluginFieldCollection const* getFieldsToSerialize() noexcept override; - -private: - int mCpSize; - int mCpRank; - std::vector<nvinfer1::PluginField> mDataToSerialize; - nvinfer1::PluginFieldCollection mFCToSerialize; - - enum IdxEntry - { - INPUT_IDS, - REQUEST_TYPES, - HOST_CONTEXT_LENGTH, - }; - - enum class RequestType : int32_t - { - kCONTEXT = 0, - kGENERATION = 1 - }; -}; - -class CpSplitPluginCreator : public BaseCreatorV3 -{ -public: - CpSplitPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV3* createPlugin( - char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/cudaStreamPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/cudaStreamPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/cudaStreamPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.cpp b/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.cpp deleted file mode 100644 index 802e828c9250..000000000000 --- a/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.cpp +++ /dev/null @@ -1,293 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "cudaStreamPlugin.h" -#include "tensorrt_llm/runtime/iBuffer.h" - -#include <cassert> - -using namespace nvinfer1; -using tensorrt_llm::plugins::CudaStreamPluginCreator; -using tensorrt_llm::plugins::CudaStreamPlugin; - -static char const* CUDA_STREAM_PLUGIN_VERSION{"1"}; -static char const* CUDA_STREAM_PLUGIN_NAME{"CudaStream"}; -PluginFieldCollection CudaStreamPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> CudaStreamPluginCreator::mPluginAttributes; - -CudaStreamPlugin::CudaStreamPlugin(int sideStreamId, int nbInputs, nvinfer1::DataType type) - : mSideStreamId(sideStreamId) - , mNbInputs(nbInputs) - , mType(type) -{ - init(); -} - -CudaStreamPlugin::CudaStreamPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mSideStreamId); - read(d, mNbInputs); - read(d, mType); - - init(); - - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -CudaStreamPlugin::CudaStreamPlugin(CudaStreamPlugin const& other) - : mSideStreamId(other.mSideStreamId) - , mNbInputs(other.mNbInputs) - , mType(other.mType) -{ - init(); -} - -void CudaStreamPlugin::init() -{ - mSideStreamPtr = nullptr; -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* CudaStreamPlugin::clone() const noexcept -{ - auto* plugin = new CudaStreamPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs CudaStreamPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - assert(outputIndex == 0); - return inputs[outputIndex]; -} - -bool CudaStreamPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - TLLM_CHECK_WITH_INFO(nbInputs == mNbInputs, "CudaStreamPlugin only accepts mNbInputs inputs"); - TLLM_CHECK_WITH_INFO(nbOutputs == 1, "CudaStreamPlugin only accepts 1 output"); - - auto const& desc = inOut[pos]; - if (desc.format != TensorFormat::kLINEAR) - { - return false; - } - - if (pos > 0 && pos < nbInputs) - { - return true; - } - return desc.type == mType; -} - -void CudaStreamPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t CudaStreamPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -int CudaStreamPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - if (!mSideStreamPtr) - { - auto const resource_name = nvinfer1::pluginInternal::SideStream::getResourceKey(mSideStreamId); - nvinfer1::pluginInternal::SideStream side_stream{}; - mSideStreamPtr = reinterpret_cast<nvinfer1::pluginInternal::SideStream*>( - getPluginRegistry()->acquirePluginResource(resource_name.c_str(), &side_stream)); - } - mSideStreamPtr->waitSideStreamOnMainStream(stream); - size_t count = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) - { - count *= inputDesc[0].dims.d[i]; - } - count *= tensorrt_llm::runtime::BufferDataType(inputDesc[0].type).getSize(); - TLLM_CUDA_CHECK(cudaMemcpyAsync(outputs[0], inputs[0], count, cudaMemcpyDeviceToDevice, stream)); - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType CudaStreamPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* CudaStreamPlugin::getPluginType() const noexcept -{ - return CUDA_STREAM_PLUGIN_NAME; -} - -char const* CudaStreamPlugin::getPluginVersion() const noexcept -{ - return CUDA_STREAM_PLUGIN_VERSION; -} - -int CudaStreamPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int CudaStreamPlugin::initialize() noexcept -{ - return 0; -} - -void CudaStreamPlugin::terminate() noexcept -{ - if (mSideStreamPtr) - { - auto const resource_name = nvinfer1::pluginInternal::SideStream::getResourceKey(mSideStreamId); - getPluginRegistry()->releasePluginResource(resource_name.c_str()); - mSideStreamPtr = nullptr; - } -} - -size_t CudaStreamPlugin::getSerializationSize() const noexcept -{ - return sizeof(mSideStreamId) + sizeof(mNbInputs) + sizeof(mType); -} - -void CudaStreamPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mSideStreamId); - write(d, mNbInputs); - write(d, mType); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void CudaStreamPlugin::destroy() noexcept -{ - delete this; -} - -/////////////// - -CudaStreamPluginCreator::CudaStreamPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("side_stream_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("num_inputs", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* CudaStreamPluginCreator::getPluginName() const noexcept -{ - return CUDA_STREAM_PLUGIN_NAME; -} - -char const* CudaStreamPluginCreator::getPluginVersion() const noexcept -{ - return CUDA_STREAM_PLUGIN_VERSION; -} - -PluginFieldCollection const* CudaStreamPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* CudaStreamPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int sideStreamId; - int nbInputs; - int type; - - // Read configurations from each fields - struct MapPair - { - char const* key; - int& field; - bool optional = false; - bool set = false; - }; - - std::array input_map{ - MapPair{"side_stream_id", std::ref(sideStreamId)}, - MapPair{"num_inputs", std::ref(nbInputs)}, - MapPair{"type_id", std::ref(type)}, - }; - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - for (auto& item : input_map) - { - if (!strcmp(item.key, attrName)) - { - TLLM_CHECK(fields[i].type == nvinfer1::PluginFieldType::kINT32); - TLLM_CHECK_WITH_INFO(!item.set, "Parameter %s was set twice", item.key); - item.field = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - item.set = true; - } - } - } - - for (auto& item : input_map) - { - TLLM_CHECK_WITH_INFO(item.set || item.optional, "Parameter %s is required but not set", item.key); - } - - try - { - auto* obj = new CudaStreamPlugin(sideStreamId, nbInputs, static_cast<nvinfer1::DataType>(type)); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* CudaStreamPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call CudaStreamPlugin::destroy() - try - { - auto* obj = new CudaStreamPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.h b/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.h deleted file mode 100644 index 5b78c3b873bb..000000000000 --- a/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.h +++ /dev/null @@ -1,265 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "NvInferPlugin.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include "tensorrt_llm/runtime/cudaMemPool.h" -#include "tensorrt_llm/runtime/utils/debugUtils.h" -#include <memory> -#include <string> -#include <vector> - -namespace nvinfer1 -{ -namespace pluginInternal -{ -class SideWorkspace -{ -public: - SideWorkspace(cudaStream_t stream) - : mWorkspaceSize{0} - , mWorkspacePtr{nullptr} - , mStream{stream} - { - } - - ~SideWorkspace() - { - if (mWorkspacePtr) - { - TLLM_CUDA_CHECK(cudaFreeAsync(mWorkspacePtr, mStream)); - } - } - - void* get(size_t workspaceSize) - { - if (mWorkspacePtr && mWorkspaceSize < workspaceSize) - { - TLLM_CUDA_CHECK(cudaFreeAsync(mWorkspacePtr, mStream)); - mWorkspacePtr = nullptr; - } - if (!mWorkspacePtr) - { - mWorkspaceSize = workspaceSize; - auto pool_ptr - = tensorrt_llm::runtime::CudaMemPool::getPrimaryPoolForDevice(tensorrt_llm::common::getDevice()); - TLLM_CUDA_CHECK(cudaMallocFromPoolAsync(&mWorkspacePtr, mWorkspaceSize, pool_ptr->getPool(), mStream)); - } - return mWorkspacePtr; - } - -private: - size_t mWorkspaceSize; - void* mWorkspacePtr; - cudaStream_t mStream; -}; - -class SideStream : public IPluginResource -{ -public: - SideStream(bool init = false) - : mStream{} - , mMainEvent{} - , mSideEvent{} - , mWorkspace{} - , mInit{init} - { - // The object passed to acquirePluginResource should use the default value init=false - if (init) - { - TLLM_CUDA_CHECK(cudaStreamCreate(&mStream)); - TLLM_CUDA_CHECK(cudaEventCreateWithFlags(&mMainEvent, cudaEventDisableTiming)); - TLLM_CUDA_CHECK(cudaEventCreateWithFlags(&mSideEvent, cudaEventDisableTiming)); - mWorkspace = std::make_shared<SideWorkspace>(mStream); - } - } - - void free() - { - if (mInit) - { - mWorkspace = nullptr; - TLLM_CUDA_CHECK(cudaStreamSynchronize(mStream)); - TLLM_CUDA_CHECK(cudaStreamDestroy(mStream)); - TLLM_CUDA_CHECK(cudaEventDestroy(mMainEvent)); - TLLM_CUDA_CHECK(cudaEventDestroy(mSideEvent)); - mInit = false; - } - } - - int32_t release() noexcept override - { - try - { - free(); - } - catch (std::exception const& e) - { - return -1; - } - return 0; - } - - IPluginResource* clone() noexcept override - { - // An object is cloned only when calling acquirePluginResource for the first time for each key - std::unique_ptr<SideStream> cloned{}; - try - { - if (!mInit) - { - cloned = std::make_unique<SideStream>(/* init */ true); - } - else - { - return nullptr; - } - } - catch (std::exception const& e) - { - return nullptr; - } - return cloned.release(); - } - - ~SideStream() override - { - free(); - } - - void* getWorkspacePtr(size_t workspaceSize) - { - return mWorkspace->get(workspaceSize); - } - - cudaStream_t getStream() const - { - return mStream; - } - - void waitMainStreamOnSideStream(cudaStream_t const stream) const - { - TLLM_CUDA_CHECK(cudaEventRecord(mMainEvent, stream)); - TLLM_CUDA_CHECK(cudaStreamWaitEvent(mStream, mMainEvent)); - } - - void waitSideStreamOnMainStream(cudaStream_t const stream) const - { - TLLM_CUDA_CHECK(cudaEventRecord(mSideEvent, mStream)); - TLLM_CUDA_CHECK(cudaStreamWaitEvent(stream, mSideEvent)); - } - - void stallMainStream(char const* name, cudaStream_t const stream, std::optional<int> delay = std::nullopt) const - { - tensorrt_llm::runtime::utils::stallStream(name, stream, delay); - } - - void stallSideStream(char const* name, std::optional<int> delay = std::nullopt) const - { - tensorrt_llm::runtime::utils::stallStream(name, mStream, delay); - } - - static std::string getResourceKey(int const stream_id) - { - return "side_stream_" + std::to_string(stream_id); - } - -private: - cudaStream_t mStream; - cudaEvent_t mMainEvent; - cudaEvent_t mSideEvent; - std::shared_ptr<SideWorkspace> mWorkspace; - bool mInit; -}; - -} // namespace pluginInternal -} // namespace nvinfer1 - -namespace tensorrt_llm::plugins -{ - -class CudaStreamPlugin : public BasePlugin -{ -public: - CudaStreamPlugin(int sideStreamId, int nbInputs, nvinfer1::DataType type); - - CudaStreamPlugin(void const* data, size_t length); - - CudaStreamPlugin(CudaStreamPlugin const&); - - void init(); - - ~CudaStreamPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - const std::string mLayerName; - int mSideStreamId; - int mNbInputs; - nvinfer1::DataType mType; - nvinfer1::pluginInternal::SideStream* mSideStreamPtr; -}; - -class CudaStreamPluginCreator : public BaseCreator -{ -public: - CudaStreamPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/CMakeLists.txt deleted file mode 100644 index ea25de075f34..000000000000 --- a/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# - -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.cpp b/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.cpp deleted file mode 100644 index 927a42ebac2f..000000000000 --- a/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.cpp +++ /dev/null @@ -1,299 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "cumsumLastDimPlugin.h" -#include "tensorrt_llm/common/assert.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::CumsumLastDimPluginCreator; -using tensorrt_llm::plugins::CumsumLastDimPlugin; - -static char const* CUMSUM_LAST_DIM_PLUGIN_VERSION{"1"}; -static char const* CUMSUM_LAST_DIM_PLUGIN_NAME{"CumsumLastDim"}; -PluginFieldCollection CumsumLastDimPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> CumsumLastDimPluginCreator::mPluginAttributes; - -static constexpr SizeType32 LENGTH_LIMIT_FOR_BLOCKSCAN = 4096; - -CumsumLastDimPlugin::CumsumLastDimPlugin(SizeType32 inputLength, nvinfer1::DataType type, size_t temp_storage_bytes) - : mInputLength(inputLength) - , mTempStorageBytes(temp_storage_bytes) - , mType(type) -{ - TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF) - || (mType == DataType::kINT32), - "Only support int, float, half, and bfloat16."); - if (mTempStorageBytes == 0) - { - mTempStorageBytes = getWorkspaceSizeNeeded(inputLength, type); - } -} - -// Parameterized constructor -CumsumLastDimPlugin::CumsumLastDimPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mInputLength); - read(d, mTempStorageBytes); - read(d, mType); - TLLM_CHECK(d == a + length); - TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF) - || (mType == DataType::kINT32), - "Only support int, float, half, and bfloat16."); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* CumsumLastDimPlugin::clone() const noexcept -{ - auto* plugin = new CumsumLastDimPlugin(mInputLength, mType, mTempStorageBytes); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -// Outputs -// output_tensor: [batch_size, inputLength] -nvinfer1::DimsExprs CumsumLastDimPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - TLLM_CHECK_WITH_INFO(outputIndex == 0, "Only one output."); - return inputs[getInputTensorIdx()]; -} - -bool CumsumLastDimPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); -} - -void CumsumLastDimPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t CumsumLastDimPlugin::getWorkspaceSizeNeeded(SizeType32 inputLength, nvinfer1::DataType type) -{ - size_t tempStorageBytes{0}; - if (inputLength < LENGTH_LIMIT_FOR_BLOCKSCAN) // last dim unknown or small, use BlockScan - { - tempStorageBytes = 0; - } - else if (type == DataType::kINT32) - { - tempStorageBytes = invokeComputeCumsumLastDimWorkspaceSize<int>(inputLength); - } - else if (type == DataType::kHALF) - { - tempStorageBytes = invokeComputeCumsumLastDimWorkspaceSize<half>(inputLength); - } - else if (type == DataType::kFLOAT) - { - tempStorageBytes = invokeComputeCumsumLastDimWorkspaceSize<float>(inputLength); - } -#ifdef ENABLE_BF16 - else if (type == DataType::kBF16) - { - tempStorageBytes = invokeComputeCumsumLastDimWorkspaceSize<__nv_bfloat16>(inputLength); - } -#endif - return tempStorageBytes; -} - -size_t CumsumLastDimPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return mTempStorageBytes; -} - -template <typename T> -int CumsumLastDimPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) -{ - // inputs - // 0. input_tensor [batch_size, inputLength] - // outputs - // 0. output_tensor [batch_size, inputLength] - auto const batchSize = inputDesc[getInputTensorIdx()].dims.d[0]; - auto const inputLength = inputDesc[getInputTensorIdx()].dims.d[1]; - /* - Two cases where we should use BlockScan: - 1. inputLength is small - 2. batchSize is large (since DeviceScan causes kernel launch per row) - */ - void* wp = inputLength < LENGTH_LIMIT_FOR_BLOCKSCAN || batchSize > 2 ? nullptr : workspace; - invokeCumsumLastDim<T>( - batchSize, inputLength, inputs[getInputTensorIdx()], outputs[0], wp, mTempStorageBytes, stream); - - sync_check_cuda_error(stream); - return 0; -} - -int CumsumLastDimPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - if (mType == DataType::kINT32) - { - return enqueueImpl<int>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (mType == DataType::kHALF) - { - return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (mType == DataType::kFLOAT) - { - return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#ifdef ENABLE_BF16 - else if (mType == DataType::kBF16) - { - return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#endif - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType CumsumLastDimPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK_WITH_INFO(index == 0, "Only one output."); - return inputTypes[getInputTensorIdx()]; -} - -// IPluginV2 Methods - -char const* CumsumLastDimPlugin::getPluginType() const noexcept -{ - return CUMSUM_LAST_DIM_PLUGIN_NAME; -} - -char const* CumsumLastDimPlugin::getPluginVersion() const noexcept -{ - return CUMSUM_LAST_DIM_PLUGIN_VERSION; -} - -int CumsumLastDimPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int CumsumLastDimPlugin::initialize() noexcept -{ - return 0; -} - -void CumsumLastDimPlugin::terminate() noexcept {} - -size_t CumsumLastDimPlugin::getSerializationSize() const noexcept -{ - return sizeof(mInputLength) + sizeof(mTempStorageBytes) + sizeof(mType); -} - -void CumsumLastDimPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mInputLength); - write(d, mTempStorageBytes); - write(d, mType); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void CumsumLastDimPlugin::destroy() noexcept -{ - delete this; -} - -/////////////// - -CumsumLastDimPluginCreator::CumsumLastDimPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("input_length", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* CumsumLastDimPluginCreator::getPluginName() const noexcept -{ - return CUMSUM_LAST_DIM_PLUGIN_NAME; -} - -char const* CumsumLastDimPluginCreator::getPluginVersion() const noexcept -{ - return CUMSUM_LAST_DIM_PLUGIN_VERSION; -} - -PluginFieldCollection const* CumsumLastDimPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* CumsumLastDimPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int inputLength{}; - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "input_length")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - inputLength = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - try - { - auto* obj = new CumsumLastDimPlugin(inputLength, type); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* CumsumLastDimPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call CumsumLastDimPlugin::destroy() - try - { - auto* obj = new CumsumLastDimPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.h b/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.h deleted file mode 100644 index 3cbf4e2356dd..000000000000 --- a/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TRT_CUMSUM_LAST_DIM_PLUGIN_H -#define TRT_CUMSUM_LAST_DIM_PLUGIN_H - -#include "tensorrt_llm/kernels/cumsumLastDim.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> - -namespace tensorrt_llm::plugins -{ -class CumsumLastDimPlugin : public BasePlugin -{ -public: - using SizeType32 = tensorrt_llm::kernels::SizeType32; - - CumsumLastDimPlugin(SizeType32 inputLength, nvinfer1::DataType type, size_t tempStorageBytes = 0); - CumsumLastDimPlugin(void const* data, size_t length); - ~CumsumLastDimPlugin() override = default; - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - template <typename T> - int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); - size_t getWorkspaceSizeNeeded(SizeType32 inputLength, nvinfer1::DataType type); - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - using IndexType = std::int32_t; - - IndexType getInputTensorIdx() const - { - return 0; - }; - -private: - SizeType32 mInputLength; - size_t mTempStorageBytes; - nvinfer1::DataType mType; -}; - -class CumsumLastDimPluginCreator : public BaseCreator -{ -public: - CumsumLastDimPluginCreator(); - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins - -#endif diff --git a/cpp/tensorrt_llm/plugins/doraPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/doraPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/doraPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.cpp b/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.cpp deleted file mode 100644 index 7c980f079ceb..000000000000 --- a/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.cpp +++ /dev/null @@ -1,392 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "doraPlugin.h" - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/runtime/iBuffer.h" - -#include <numeric> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::DoraPlugin; -using tensorrt_llm::plugins::DoraPluginCreator; - -static char const* DORA_PLUGIN_VERSION{"1"}; -static char const* DORA_PLUGIN_NAME{"Dora"}; -PluginFieldCollection DoraPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> DoraPluginCreator::mPluginAttributes; - -DoraPlugin::DoraPlugin(std::vector<int32_t> const& outHiddenSizes, nvinfer1::DataType type, bool removeInputPadding) - : mType(type) - , mRemoveInputPadding(removeInputPadding) - , mDoraImpl(outHiddenSizes, type) -{ - mOutHiddenSizes.resize(outHiddenSizes.size()); - mOutHiddenSizes.assign(outHiddenSizes.cbegin(), outHiddenSizes.cend()); - init(); -} - -void DoraPlugin::init() -{ - // initialize data to serialize - mDataToSerialize.clear(); - mDataToSerialize.emplace_back( - "out_hidden_sizes", mOutHiddenSizes.data(), PluginFieldType::kINT32, mOutHiddenSizes.size()); - mDataToSerialize.emplace_back("type", &mType, PluginFieldType::kINT32, 1); - mDataToSerialize.emplace_back("remove_input_padding", &mRemoveInputPadding, PluginFieldType::kINT8, 1); - mFieldsToSerialize.nbFields = static_cast<int32_t>(mDataToSerialize.size()); - mFieldsToSerialize.fields = mDataToSerialize.data(); -} - -// IPluginV3 methods -nvinfer1::IPluginCapability* DoraPlugin::getCapabilityInterface(nvinfer1::PluginCapabilityType type) noexcept -{ - switch (type) - { - case PluginCapabilityType::kBUILD: return static_cast<IPluginV3OneBuild*>(this); - case PluginCapabilityType::kRUNTIME: return static_cast<IPluginV3OneRuntime*>(this); - case PluginCapabilityType::kCORE: return static_cast<IPluginV3OneCore*>(this); - } - return nullptr; -} - -nvinfer1::IPluginV3* DoraPlugin::clone() noexcept -{ - std::unique_ptr<DoraPlugin> plugin{std::make_unique<DoraPlugin>(mOutHiddenSizes, mType, mRemoveInputPadding)}; - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin.release(); -} - -// IPluginV3OneCore methods -char const* DoraPlugin::getPluginName() const noexcept -{ - return DORA_PLUGIN_NAME; -} - -char const* DoraPlugin::getPluginVersion() const noexcept -{ - return DORA_PLUGIN_VERSION; -} - -// IPluginV3OneBuild methods -int32_t DoraPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept -{ - return 0; -} - -int32_t DoraPlugin::getOutputDataTypes( - DataType* outputTypes, int32_t nbOutputs, DataType const* inputTypes, int32_t nbInputs) const noexcept -{ - try - { - TLLM_CHECK(nbOutputs == 1); - TLLM_CHECK(nbInputs == 2 + static_cast<int32_t>(mOutHiddenSizes.size()) + (mRemoveInputPadding ? 1 : 0)); - TLLM_CHECK(inputTypes[IdxEntry::kINPUT_TENSOR] == mType); - // output has the same dtype as the input, the plugin just applies scaling - outputTypes[0] = inputTypes[IdxEntry::kINPUT_TENSOR]; - } - catch (std::exception const& e) - { - caughtError(e); - } - return 0; -} - -int32_t DoraPlugin::getOutputShapes(DimsExprs const* inputs, int32_t nbInputs, DimsExprs const* shapeInputs, - int32_t nbShapeInputs, DimsExprs* outputs, int32_t nbOutputs, IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbOutputs == 1); - TLLM_CHECK(nbShapeInputs == 0); - TLLM_CHECK(nbInputs == 2 + static_cast<int32_t>(mOutHiddenSizes.size()) + (mRemoveInputPadding ? 1 : 0)); - - auto const inputTensorDims = inputs[IdxEntry::kINPUT_TENSOR]; - TLLM_CHECK(inputTensorDims.nbDims == (mRemoveInputPadding ? 2 : 3)); - - auto const lastDim = inputTensorDims.d[inputTensorDims.nbDims - 1]; - TLLM_CHECK(lastDim->isConstant()); - TLLM_CHECK(lastDim->getConstantValue() == std::accumulate(mOutHiddenSizes.cbegin(), mOutHiddenSizes.cend(), 0)); - - outputs[0].nbDims = inputTensorDims.nbDims; - for (auto dim = 0; dim < inputTensorDims.nbDims; ++dim) - { - outputs[0].d[dim] = inputTensorDims.d[dim]; - } - } - catch (std::exception const& e) - { - caughtError(e); - } - return 0; -} - -bool DoraPlugin::supportsFormatCombination( - int32_t pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - auto const numModules = static_cast<int32_t>(mOutHiddenSizes.size()); - if (nbInputs != 2 + numModules + (mRemoveInputPadding ? 1 : 0)) - { - return false; - } - - bool const isInput = pos < nbInputs; - if (pos == IdxEntry::kHOST_REQUEST_TYPES) - { - return (inOut[pos].desc.type == nvinfer1::DataType::kINT32); - } - // optional host_context_lens after lora pointers - else if (pos == IdxEntry::kLORA_WEIGHTS_PTRS_START + numModules and isInput) - { - return (inOut[pos].desc.type == nvinfer1::DataType::kINT32 and mRemoveInputPadding); - } - // lora weight pointers - else if (pos >= IdxEntry::kLORA_WEIGHTS_PTRS_START and pos < IdxEntry::kLORA_WEIGHTS_PTRS_START + numModules) - { - return (inOut[pos].desc.type == nvinfer1::DataType::kINT64); - } - else if (pos != 0 and isInput) - { - TLLM_LOG_WARNING("%s: got an unexpected input at position %d", __PRETTY_FUNCTION__, pos); - return false; - } - - return (inOut[pos].desc.type == mType) and (inOut[pos].desc.format == TensorFormat::kLINEAR); -} - -int32_t DoraPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -size_t DoraPlugin::getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int32_t nbInputs, - nvinfer1::DynamicPluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept -{ - auto const inputTensorMax = inputs[IdxEntry::kINPUT_TENSOR].max; - auto const maxNumTokens = mRemoveInputPadding ? inputTensorMax.d[0] : inputTensorMax.d[0] * inputTensorMax.d[1]; - auto const size = mDoraImpl.getWorkspaceSize(maxNumTokens); - return size; -} - -int32_t DoraPlugin::getValidTactics(int32_t* tactics, int32_t nbTactics) noexcept -{ - return 0; -} - -int32_t DoraPlugin::getNbTactics() noexcept -{ - return 0; -} - -char const* DoraPlugin::getTimingCacheID() noexcept -{ - return nullptr; -} - -int32_t DoraPlugin::getFormatCombinationLimit() noexcept -{ - return 1; -} - -char const* DoraPlugin::getMetadataString() noexcept -{ - return nullptr; -} - -// IPluginV3OneRuntime methods -int32_t DoraPlugin::setTactic(int32_t tactic) noexcept -{ - return 0; -} - -int32_t DoraPlugin::onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, - nvinfer1::PluginTensorDesc const* out, int32_t nbOutputs) noexcept -{ - return 0; -} - -int32_t DoraPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - - auto const numModules = static_cast<int32_t>(mOutHiddenSizes.size()); - auto const numReqs = inputDesc[IdxEntry::kHOST_REQUEST_TYPES].dims.d[0]; - - auto const inputTensorDesc = inputDesc[IdxEntry::kINPUT_TENSOR]; - auto const numTokens - = mRemoveInputPadding ? inputTensorDesc.dims.d[0] : inputTensorDesc.dims.d[0] * inputTensorDesc.dims.d[1]; - auto const seqLen = mRemoveInputPadding ? 0 : inputTensorDesc.dims.d[1]; - - void const* inputTensor = inputs[IdxEntry::kINPUT_TENSOR]; - auto const* hostRequestTypes = static_cast<int32_t const*>(inputs[IdxEntry::kHOST_REQUEST_TYPES]); - void const* const* loraWeightsPtrs = &inputs[IdxEntry::kLORA_WEIGHTS_PTRS_START]; - - int32_t const* hostContextLengths = mRemoveInputPadding - ? static_cast<int32_t const*>(inputs[IdxEntry::kLORA_WEIGHTS_PTRS_START + numModules]) - : nullptr; - - mExpandDoraWeightPtrs.clear(); - mExpandDoraWeightPtrs.reserve(numModules * numTokens); - - bool hasAnyDora = false; - - for (auto moduleIdx = 0; moduleIdx < numModules; moduleIdx++) - { - auto const loraWeightModulePtrs = static_cast<int64_t const*>(loraWeightsPtrs[moduleIdx]); - - int idx = 0; - for (int reqId = 0; reqId < numReqs; reqId++) - { - // loraWeightModulePtrs has 3 pointers for each module: A,B, and an optional DoRA magnitude - // the current DoRA plugin does not apply LoRA, so A and B are ignored. - RequestType const reqType = static_cast<RequestType const>(hostRequestTypes[reqId]); - auto const* modulePtr = reinterpret_cast<void const*>(loraWeightModulePtrs[reqId * 3 + 2]); - hasAnyDora = hasAnyDora or modulePtr != nullptr; - - if (reqType == RequestType::kGENERATION) - { - mExpandDoraWeightPtrs.push_back(modulePtr); - idx += 1; - } - else - { - int contextLen = (mRemoveInputPadding ? hostContextLengths[reqId] : seqLen); - - for (int contextId = 0; contextId < contextLen; contextId++) - { - mExpandDoraWeightPtrs.push_back(modulePtr); - idx += 1; - } - } - } - if (idx != numTokens) - { - TLLM_LOG_ERROR("LoraParams and input dims don't match, lora tokens %d input tokens %d", idx, numTokens); - return -1; - } - } - - if (hasAnyDora) - { - mDoraImpl.run(numTokens, inputTensor, mExpandDoraWeightPtrs.data(), outputs, workspace, stream); - } - else - { - // skip dora scaling if all requests are pure-lora - auto const inputRank = inputTensorDesc.dims.nbDims; - auto const numel - = std::accumulate(inputTensorDesc.dims.d, inputTensorDesc.dims.d + inputRank, 1, std::multiplies()); - auto const elemSize = tensorrt_llm::common::getDTypeSize(mType); - tensorrt_llm::common::cudaAutoCpy((int8_t*) outputs[0], (int8_t*) inputTensor, numel * elemSize, stream); - } - - sync_check_cuda_error(stream); - return 0; -} - -nvinfer1::IPluginV3* DoraPlugin::attachToContext(nvinfer1::IPluginResourceContext* context) noexcept -{ - return clone(); -} - -nvinfer1::PluginFieldCollection const* DoraPlugin::getFieldsToSerialize() noexcept -{ - return &mFieldsToSerialize; -} - -DoraPluginCreator::DoraPluginCreator() -{ - mPluginAttributes.clear(); - mPluginAttributes.emplace_back("num_modules", nullptr, PluginFieldType::kINT32, 1); - mPluginAttributes.emplace_back("type", nullptr, PluginFieldType::kINT32, 1); - mPluginAttributes.emplace_back("remove_input_padding", nullptr, PluginFieldType::kINT8, 1); - mFC.nbFields = static_cast<int32_t>(mPluginAttributes.size()); - mFC.fields = mPluginAttributes.data(); -} - -char const* DoraPluginCreator::getPluginName() const noexcept -{ - return DORA_PLUGIN_NAME; -} - -char const* DoraPluginCreator::getPluginVersion() const noexcept -{ - return DORA_PLUGIN_VERSION; -} - -nvinfer1::PluginFieldCollection const* DoraPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -nvinfer1::IPluginV3* DoraPluginCreator::createPlugin( - char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept -{ - PluginField const* fields = fc->fields; - nvinfer1::DataType type{}; - bool removeInputPadding{}; - std::vector<int32_t> outHiddenSizes; - - // Read configurations from each field - for (int i = 0; i < fc->nbFields; ++i) - { - auto const field = fields[i]; - char const* attrName = fields[i].name; - if (!strcmp(attrName, "type")) - { - TLLM_CHECK(field.type == PluginFieldType::kINT32 and field.length == 1); - type = *static_cast<nvinfer1::DataType const*>(field.data); - } - else if (!strcmp(attrName, "remove_input_padding")) - { - TLLM_CHECK(field.type == PluginFieldType::kINT8 and field.length == 1); - removeInputPadding = *static_cast<bool const*>(field.data); - } - else if (!strcmp(attrName, "out_hidden_sizes")) - { - TLLM_CHECK(field.type == PluginFieldType::kINT32); - auto const* outHiddenSizesPtr = static_cast<int32_t const*>(field.data); - outHiddenSizes.resize(field.length); - outHiddenSizes.assign(outHiddenSizesPtr, outHiddenSizesPtr + field.length); - } - else - { - TLLM_LOG_WARNING("%s: got an unexpected attribute: %s", __PRETTY_FUNCTION__, attrName); - } - } - - try - { - auto* obj = new DoraPlugin(outHiddenSizes, type, removeInputPadding); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.h b/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.h deleted file mode 100644 index dfee11fdc90e..000000000000 --- a/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.h +++ /dev/null @@ -1,114 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "tensorrt_llm/kernels/lora/dora.h" -#include "tensorrt_llm/plugins/common/plugin.h" - -namespace tensorrt_llm::plugins -{ - -class DoraPlugin : public BasePluginV3 -{ -public: - DoraPlugin() = delete; - DoraPlugin(std::vector<int32_t> const& outHiddenSizes, nvinfer1::DataType type, bool removeInputPadding); - DoraPlugin(DoraPlugin const& p) = default; - - // IPluginV3 methods - nvinfer1::IPluginCapability* getCapabilityInterface(nvinfer1::PluginCapabilityType type) noexcept override; - nvinfer1::IPluginV3* clone() noexcept override; - - // IPluginV3OneCore methods - char const* getPluginName() const noexcept override; - char const* getPluginVersion() const noexcept override; - - // IPluginV3OneBuild methods - int32_t configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept override; - int32_t getOutputDataTypes(nvinfer1::DataType* outputTypes, int32_t nbOutputs, nvinfer1::DataType const* inputTypes, - int32_t nbInputs) const noexcept override; - int32_t getOutputShapes(nvinfer1::DimsExprs const* inputs, int32_t nbInputs, nvinfer1::DimsExprs const* shapeInputs, - int32_t nbShapeInputs, nvinfer1::DimsExprs* outputs, int32_t nbOutputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination(int32_t pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int32_t nbInputs, - int32_t nbOutputs) noexcept override; - int32_t getNbOutputs() const noexcept override; - size_t getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int32_t nbInputs, - nvinfer1::DynamicPluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept override; - int32_t getValidTactics(int32_t* tactics, int32_t nbTactics) noexcept override; - int32_t getNbTactics() noexcept override; - char const* getTimingCacheID() noexcept override; - int32_t getFormatCombinationLimit() noexcept override; - char const* getMetadataString() noexcept override; - - // IPluginV3OneRuntime methods - int32_t setTactic(int32_t tactic) noexcept override; - int32_t onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, nvinfer1::PluginTensorDesc const* out, - int32_t nbOutputs) noexcept override; - int32_t enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept override; // fixed - nvinfer1::IPluginV3* attachToContext(nvinfer1::IPluginResourceContext* context) noexcept override; - nvinfer1::PluginFieldCollection const* getFieldsToSerialize() noexcept override; - -private: - void init(); - - std::vector<nvinfer1::PluginField> mDataToSerialize; - nvinfer1::PluginFieldCollection mFieldsToSerialize; - - enum IdxEntry - { - kINPUT_TENSOR = 0, - kHOST_REQUEST_TYPES = 1, - kLORA_WEIGHTS_PTRS_START = 2 - }; - - // TODO(oargov) this is shared with the LoRA plugin, put it somewhere else - enum class RequestType : int32_t - { - kCONTEXT = 0, - kGENERATION = 1 - }; - - std::vector<int32_t> mOutHiddenSizes; - nvinfer1::DataType mType; - bool mRemoveInputPadding; - tensorrt_llm::kernels::DoraImpl mDoraImpl; - - std::vector<void const*> mExpandDoraWeightPtrs{}; -}; - -class DoraPluginCreator : public BaseCreatorV3 -{ -public: - DoraPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV3* createPlugin( - char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -}; // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/eaglePlugin/CMakeLists.txt deleted file mode 100644 index b6bd0439cc0c..000000000000 --- a/cpp/tensorrt_llm/plugins/eaglePlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.cpp b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.cpp deleted file mode 100644 index 899c93855b9f..000000000000 --- a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.cpp +++ /dev/null @@ -1,945 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "eagleDecodeDraftTokensPlugin.h" - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/kernels/samplingTopKKernels.h" -#include "tensorrt_llm/kernels/speculativeDecoding/eagleDecodingKernels.h" -#include "tensorrt_llm/kernels/speculativeDecoding/medusaDecodingKernels.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iTensor.h" - -using namespace nvinfer1; -using tensorrt_llm::plugins::EagleDecodeDraftTokensPluginCreator; -using tensorrt_llm::plugins::EagleDecodeDraftTokensPlugin; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::kernels::speculative_decoding; -using namespace tensorrt_llm::runtime; -namespace tc = tensorrt_llm::common; - -static char const* EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_VERSION{"1"}; -static char const* EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_NAME{"EagleDecodeDraftTokens"}; -PluginFieldCollection EagleDecodeDraftTokensPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> EagleDecodeDraftTokensPluginCreator::mPluginAttributes; - -EagleDecodeDraftTokensPlugin::EagleDecodeDraftTokensPlugin( - nvinfer1::DataType type, int32_t layerIdx, int32_t numEagleLayers, bool topKSampling) - : mDtype(type) - , mLayerIdx(layerIdx) - , mNumEagleLayers(numEagleLayers) - , mTopKSampling(topKSampling) -{ - TLLM_CHECK_WITH_INFO(mTopKSampling, "Multinomial sampling is not supported yet."); -} - -// Parameterized constructor -EagleDecodeDraftTokensPlugin::EagleDecodeDraftTokensPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mDtype); - read(d, mLayerIdx); - read(d, mNumEagleLayers); - read(d, mTopKSampling); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - static_cast<int>(length), static_cast<int>(d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* EagleDecodeDraftTokensPlugin::clone() const noexcept -{ - auto* plugin = new EagleDecodeDraftTokensPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs EagleDecodeDraftTokensPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - TLLM_CHECK(outputIndex < getNbOutputs()); - TLLM_CHECK(nbInputs == 12); - auto const batchSizeExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[0]; - auto const maxDecodingTokensExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[1]; - auto const maxPathLengthExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[2]; - auto const maxDecodingDraftTokensExpr - = exprBuilder.operation(DimensionOperation::kSUB, *maxDecodingTokensExpr, *exprBuilder.constant(1)); - - auto const numEagleLayersExpr - = exprBuilder.operation(DimensionOperation::kSUB, *maxPathLengthExpr, *exprBuilder.constant(1)); - auto const maxDecodingDraftTokensSquareExpr - = exprBuilder.operation(DimensionOperation::kPROD, *maxDecodingDraftTokensExpr, - *maxDecodingDraftTokensExpr); // maxDecodingDraftTokensExpr * maxDecodingDraftTokensExpr - - nvinfer1::DimsExprs ret; - if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_DRAFT_TOKEN_IDS)) - { - // output_draft_token_ids: [batch_size, max_decoding_draft_tokens] - ret.nbDims = 2; - ret.d[0] = batchSizeExpr; - ret.d[1] = maxDecodingDraftTokensExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_DRAFT_LENS)) - { - // output_draft_lens: [batch_size] - ret.nbDims = 1; - ret.d[0] = batchSizeExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_PATHS)) - { - // output_path: [batch_size, max_decoding_tokens, max_path_len] - ret.nbDims = 3; - ret.d[0] = batchSizeExpr; - ret.d[1] = maxDecodingTokensExpr; - ret.d[2] = maxPathLengthExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_CURRENT_SCORES)) - { - // output_current_scores: [batch_size, max_decoding_draft_tokens] - ret.nbDims = 2; - ret.d[0] = batchSizeExpr; - ret.d[1] = maxDecodingDraftTokensExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_NEXT_EXPAND_INDICES)) - { - // output_next_expand_index - ret.nbDims = 2; - ret.d[0] = batchSizeExpr; - ret.d[1] = maxDecodingDraftTokensExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_SCORES)) - { - // output_all_layers_scores: - // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] - ret.nbDims = 3; - ret.d[0] = batchSizeExpr; - ret.d[1] = numEagleLayersExpr; - ret.d[2] = maxDecodingDraftTokensSquareExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS)) - { - // output_all_layers_draft_token_ids: - // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] - ret.nbDims = 3; - ret.d[0] = batchSizeExpr; - ret.d[1] = numEagleLayersExpr; - ret.d[2] = maxDecodingDraftTokensSquareExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS_PREDECESSOR)) - { - // output_all_layers_draft_token_ids_predecessor - // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] - ret.nbDims = 3; - ret.d[0] = batchSizeExpr; - ret.d[1] = numEagleLayersExpr; - ret.d[2] = maxDecodingDraftTokensSquareExpr; - } - else - { - TLLM_CHECK_WITH_INFO( - false, "Wrong outputIndex %d in EagleDecodeDraftTokensPlugin::getOutputDimensions", outputIndex); - } - return ret; -} - -bool EagleDecodeDraftTokensPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - TLLM_CHECK(nbInputs == 12 && nbOutputs == getNbOutputs()); - TLLM_CHECK(pos < nbInputs + nbOutputs); - - if (pos == getIdx(InputIdxEntry::LOGITS)) - { - // input: logits - // output: output_all_layers_scores - return (inOut[pos].type == mDtype) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == getIdx(InputIdxEntry::INPUT_ALL_LAYERS_SCORES) || pos == getIdx(InputIdxEntry::INPUT_PREV_SCORES) - || pos == nbInputs + getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_SCORES) - || pos == nbInputs + getIdx(OutputIdxEntry::OUTPUT_CURRENT_SCORES)) - { - // input: rand_sample, input_all_layers_scores, input_prev_scores - // output: output_all_layers_scores, output_current_scores - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else - { - // input: path, num_valid_logits, use_dynamic_tree, dynamic_tree_max_topK, input_draft_token_ids, - // input_draft_lens, input_current_expand_index, input_all_layers_draft_token_ids - // output: output_draft_token_ids, output_draft_lens, output_path, output_next_expand_index - // output_all_layers_draft_token_ids, output_all_alyers_draft_token_predecessor - return (inOut[pos].type == nvinfer1::DataType::kINT32) && (inOut[pos].format == TensorFormat::kLINEAR); - } -} - -void EagleDecodeDraftTokensPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -template <typename T> -size_t EagleDecodeDraftTokensPlugin::getWorkspaceSizeType(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - size_t workspaceSize{0}; - auto const numInputLogits = inputs[getIdx(InputIdxEntry::LOGITS)].dims.d[0]; - auto const batchSize = inputs[getIdx(InputIdxEntry::PATHS)].dims.d[0]; - auto const vocabSizePadded = inputs[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; - auto const maxDecodingTokens = inputs[getIdx(InputIdxEntry::PATHS)].dims.d[1]; - auto const maxDecodingDraftTokens = maxDecodingTokens - 1; - auto const maxTopK = maxDecodingDraftTokens; - auto const mNumEagleLayers = inputs[getIdx(InputIdxEntry::INPUT_ALL_LAYERS_SCORES)].dims.d[1]; - - // Greedy sampling - if (mTopKSampling) - { - // 0. The first topK sampling workspace - auto const draftTokenSamplingWorkspaceSize - = getTopKWorkspaceSize<T>(numInputLogits, /* maxTokensPerStep */ 1, /* maxTopK */ maxTopK, vocabSizePadded); - - // 1. The first TopKs [numInputLogits] - auto const topKsSize = numInputLogits * sizeof(SizeType32); - - // 2. Topks offset [batchSize] - // Each request will have different number of logits that need to be sampled - // This tensor will record the start offset of the topK for each request - auto const topKOffsetSize = batchSize * sizeof(SizeType32); - - // 3. Logits ptrs [numInputLogits] - auto const logitsPtrsSize = numInputLogits * sizeof(T*); - - // 4. The first topK sampling's output ids ptrs [numInputLogits][maxDecodingDraftTokens] - auto const firstTopKOutputIdsPtrsSize = numInputLogits * sizeof(TokenIdType*); - - // 5. The first topK sampling's output ids (temporary buffer) [numInputLogits * maxDecodingDraftTokens] - auto const firstTopKOutputIdsSize = numInputLogits * maxDecodingDraftTokens * sizeof(TokenIdType); - - // 6. Number of successors for each nodes, extract from the paths and layerId - // [batchSize * maxDecodingTokens] - auto const numSuccessorsForEachNodeSize = batchSize * maxDecodingTokens * sizeof(SizeType32); - - // 7. Flag whether to do decoding or not. SamplingTopK is done for numInputLogits tokens. - // But only sum(numValidLogitsPerRequest[:]) of them are valid. - // [batchSize * maxDecodingTokens] - auto const skipDecodeSize = numInputLogits * sizeof(bool); - - // 8. The first topK sampling's logprobs [batchSize * maxDecodingDraftTokens] - auto const firstTopKOutputLogProbsSize = numInputLogits * maxDecodingDraftTokens * sizeof(float); - - // 9. Eagle-2, the second topK sampling workspace - // Sampling from [batchSize, maxTopK * maxTopK] to [batchSize, maxTopK] - auto const secondTopKSamplingWorkspaceSize = getTopKWorkspaceSize<float>( - batchSize, /* maxTokensPerStep */ 1, /* maxTopK */ maxTopK, maxTopK * maxTopK); - - // 10. Eagle-2, the outputIds of the second topK sampling, shape [batchSize, maxDecodingTokens] - auto const secondTopKOutputIdsSize = batchSize * maxDecodingTokens * sizeof(TokenIdType); - // 11. Eagle-2, the outputIdsPtr of the second topK sampling, shape [batchSize] - auto const secondTopKOutputIdsPtrSize = batchSize * sizeof(TokenIdType*); - // 12. Eagle-2, the inputScoresPtrs of the second topK sampling, shape [batchSize] - auto const secondTopKInputScoresPtrsSize = batchSize * sizeof(float*); - // 13. Eagle-2, the outpuLogProbs of the second topK samplig, shape [batchSize, maxDecodingDraftTokens] - auto const secondTopKOutputLogProbsSize = batchSize * maxDecodingDraftTokens * sizeof(float); - - // 14. Eagle-2, the input scores pointers of the third topK sampling, shape [batchSize] - // Each points to a vocabSize = '(mNumEagleLayers - 1) * dynamicTreeMaxTopK * dynamicTreeMaxTopK + - // dynamicTreeMaxTopK' - auto const thirdTopKInputScoresPtrsSize = batchSize * sizeof(float*); - // 15. Eagle-2, the output of the third topK sampling, shape [batchSize, maxDecodingDraftTokens] - auto const thirdTopKOutputIdsSize = batchSize * maxDecodingDraftTokens * sizeof(TokenIdType); - // 16. Eagle-2, the output pointers of the third topK sampling, shape [batchSize] - auto const thirdTopKOutputIdsPtrsSize = batchSize * sizeof(TokenIdType*); - // 17. Eagle-2, the workspace of the third topK sampling - // Sampling from [batchSize, '(mNumEagleLayers - 1) * dynamicTreeMaxTopK * dynamicTreeMaxTopK + - // dynamicTreeMaxTopK'] to [batchSize, maxDecodingDraftTokens] We over-set the vocabsize here. - auto const thridTopKSamplingWorkspaceSize = getTopKWorkspaceSize<float>(batchSize, /* maxTokensPerStep */ 1, - /* maxTopK */ maxDecodingDraftTokens, mNumEagleLayers * maxDecodingDraftTokens * maxDecodingDraftTokens); - - // 18. Eagle-2, the topKs for each request in the third topK sampling - // The real topK value is min(maxDecodingDraftTokens, totalNumDraftTokensForAllLayers) - auto const thirdTopKsSize = batchSize * sizeof(SizeType32); - - SizeType32 constexpr NUM_BUFFERS{19}; - size_t workspaces[NUM_BUFFERS]; - workspaces[0] = draftTokenSamplingWorkspaceSize; - workspaces[1] = topKsSize; - workspaces[2] = topKOffsetSize; - workspaces[3] = logitsPtrsSize; - workspaces[4] = firstTopKOutputIdsPtrsSize; - workspaces[5] = firstTopKOutputIdsSize; - workspaces[6] = numSuccessorsForEachNodeSize; - workspaces[7] = skipDecodeSize; - workspaces[8] = firstTopKOutputLogProbsSize; - workspaces[9] = secondTopKSamplingWorkspaceSize; - workspaces[10] = secondTopKOutputIdsSize; - workspaces[11] = secondTopKOutputIdsPtrSize; - workspaces[12] = secondTopKInputScoresPtrsSize; - workspaces[13] = secondTopKOutputLogProbsSize; - workspaces[14] = thirdTopKInputScoresPtrsSize; - workspaces[15] = thirdTopKOutputIdsSize; - workspaces[16] = thirdTopKOutputIdsPtrsSize; - workspaces[17] = thridTopKSamplingWorkspaceSize; - workspaces[18] = thirdTopKsSize; - workspaceSize = tc::calculateTotalWorkspaceSize(workspaces, NUM_BUFFERS); - } - else - { - // TODO fill me - // Multinomial sampling - TLLM_CHECK_WITH_INFO(false, "Multinomial sampling is not supported yet."); - } - - return workspaceSize; -} - -size_t EagleDecodeDraftTokensPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - auto const logitsType = inputs[getIdx(InputIdxEntry::LOGITS)].type; - if (logitsType == nvinfer1::DataType::kFLOAT) - { - return getWorkspaceSizeType<float>(inputs, nbInputs, outputs, nbOutputs); - } - else if (logitsType == nvinfer1::DataType::kHALF) - { - return getWorkspaceSizeType<__half>(inputs, nbInputs, outputs, nbOutputs); - } - else - { - TLLM_CHECK_WITH_INFO(false, "Unsupported logits type"); - } - return 0; -} - -template <typename T> -void EagleDecodeDraftTokensPlugin::doTopKSampling(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // We allocate many buffers with 'numInputLogits' size, but the input logits will include some padding logits. - // So only 'batchSize' or 'numValidLogits' size will be actually used. - auto const numInputLogits = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[0]; - auto const vocabSizePadded = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; - auto const batchSize = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[0]; - auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[1]; - auto const maxPathLen = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[2]; - auto const maxDecodingDraftTokens = maxDecodingTokens - 1; - auto const maxTopK = maxDecodingDraftTokens; - - ////////////////////////////////////////// Get plugin inputs ////////////////////////////////////////// - // Plugin inputs - // Input logits for sampling, shape: [numInputLogits, vocabSizePadded] - auto pluginInputLogits = static_cast<T const*>(inputs[getIdx(InputIdxEntry::LOGITS)]); - // Input paths, shape: [batchSize, maxDecodingTokens, maxPathLen] - auto pluginInputPaths = static_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::PATHS)]); - auto numValidLogits = static_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::NUM_VALID_LOGITS)]); - // For Eagle-2 - // Whether to use dynamic tree (i.e., Eagle-2) - auto useDynamicTree = *(static_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::USE_DYNAMIC_TREE)])); - // The max topK for dynamic tree. All the requests have the same expand topK. - // In Eagle-2, dynamicTreeMaxTopK is equal to maxNonLeavesPerLayer in the internal EagleNets. - auto dynamicTreeMaxTopK = *(static_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::DYNAMIC_TREE_MAX_TOPK)])); - // All layer's draft tokenIds, shape: [batchSize, maxDecodingDraftTokens] - auto pluginInputDraftTokenIds - = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::INPUT_DRAFT_TOKEN_IDS)]); - // The number of all layer's draft tokenIds, shape: [batchSize] - auto pluginInputDraftLens = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::INPUT_DRAFT_LENS)]); - // The previous EagleNet's scores, shape: [batchSize, maxDecodingDraftTokens] - auto pluginInputPrevScores = static_cast<float const*>(inputs[getIdx(InputIdxEntry::INPUT_PREV_SCORES)]); - // The indices of the nodes that will be expand in this layer, shape: [batchSize, maxDecodingDraftTokens] - // The index is related to the final output tree, which has max_decoding_draft_tokens draft tokens. - auto pluginInputCurrentExpandIndices - = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::INPUT_CURRENT_EXPAND_INDICES)]); - // The scores from all previous EagleNets, - // shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x maxDecodingDraftTokens] - auto pluginInputAllLayersScores = static_cast<float const*>(inputs[getIdx(InputIdxEntry::INPUT_ALL_LAYERS_SCORES)]); - // The draft tokens from all previous EagleNets, - // shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x maxDecodingDraftTokens] - auto pluginInputAllLayersDraftTokenIds - = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::INPUT_ALL_LAYERS_DRAFT_TOKEN_IDS)]); - // The predecessor of all the draft tokens, - // shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x maxDecodingDraftTokens] - auto pluginInputAllLayersDraftTokenIdsPredecessor = reinterpret_cast<SizeType32 const*>( - inputs[getIdx(InputIdxEntry::INPUT_ALL_LAYERS_DRAFT_TOKEN_IDS_PREDECESSOR)]); - - ////////////////////////////////////////// Get plugin outputs ////////////////////////////////////////// - // Plugin outputs - // All layer's draft tokenIds, shape: [batchSize, maxDecodingDraftTokens] - auto pluginOutputDraftTokenIds - = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::OUTPUT_DRAFT_TOKEN_IDS)]); - // The number of all layer's draft tokenIds, shape: [batchSize] - auto pluginOutputDraftLens = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::OUTPUT_DRAFT_LENS)]); - // For Eagle-2 - // Updated paths base on this layer's sampling result, shape: [batchSize, maxDecodingTokens, maxPathLen] - auto pluginOutputPaths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::OUTPUT_PATHS)]); - // This layer's scores, which will be used in next layers [batchSize, maxDecodingDraftTokens] - auto pluginOutputCurrentScores = static_cast<float*>(outputs[getIdx(OutputIdxEntry::OUTPUT_CURRENT_SCORES)]); - // The indices of the nodes that will be expand in next layer, shape: [batchSize, maxDecodingDraftTokens] - // The index is related to the final output tree, which has max_decoding_draft_tokens draft tokens. - auto pluginOutputNextExpandIndices - = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::OUTPUT_NEXT_EXPAND_INDICES)]); - // Updated scores, shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x maxDecodingDraftTokens] - auto pluginOutputAllLayersScores = static_cast<float*>(outputs[getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_SCORES)]); - // Updated draft tokens, shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x maxDecodingDraftTokens] - auto pluginOutputAllLayersDraftTokenIds - = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS)]); - // Update the predecessor of the draft tokens, shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x - // maxDecodingDraftTokens] - auto pluginOutputAllLayersDraftTokenIdsPredecessor - = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS_PREDECESSOR)]); - - ////////////////////////////////////////// Get workspaces ////////////////////////////////////////// - int8_t* workspaceBytePtr = reinterpret_cast<int8_t*>(workspace); - size_t offset{0}; - // Workspace 0: Sampling workspace. - // Treat numInputLogits as batchSize - auto const samplingWorkspaceSize - = getTopKWorkspaceSize<T>(numInputLogits, /* maxTokensPerStep */ 1, /* maxTopK */ maxTopK, vocabSizePadded); - void* workspaceSampling - = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, samplingWorkspaceSize)); - - // Workspace 1: Topks tensor: shape [numInputLogits] - SizeType32* topKs = reinterpret_cast<SizeType32*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * sizeof(SizeType32))); - - // Workspace 2: topKOffset tensor: shape: [batchSize], number of nodes that have successors for each requests - SizeType32* topKOffset - = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(SizeType32))); - - // Workspace 3: logits pointers tensor: shape: [numInputLogits] - T const** logitsPtrs - = reinterpret_cast<T const**>(tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * sizeof(T*))); - - // Workspace 4: outputIds pointers tensor: shape [numInputLogits], each points to a [maxDecodingDraftTokens] buffer - TokenIdType** firstTopKOutputIdsPtrs = reinterpret_cast<TokenIdType**>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * sizeof(TokenIdType*))); - - // Workspace 5: outputIds tensor: flatten outputIds, shape [numInputLogits * maxDecodingDraftTokens] - TokenIdType* firstTopKOutputIdsFlatten = reinterpret_cast<TokenIdType*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * maxDecodingDraftTokens * sizeof(TokenIdType))); - - // Workspace 6: number of successors for each nodes tensor: shape [batchSize * maxDecodingTokens] - SizeType32* numSuccessorsForEachNode = reinterpret_cast<SizeType32*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(SizeType32))); - - // Workspace 7: skip decoding mask [numInputLogits] - bool* skipDecode - = reinterpret_cast<bool*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * sizeof(bool))); - - // In Eagle-1, we do not need to return logProbs - float* firstTopKOutputLogProbs = nullptr; - if (useDynamicTree) - { - // Workspace 8. The output logProbs of the first topK sampling. - // Which will be updated with the previous layer's scores (i.e., pluginInputPrevScores), and will be treat as - // the input of the second topK sampling. For mLayerIdx == 0, shape: [numInputLogits(batchSize), - // maxDecodingDraftTokens] For mLayerIdx > 0, shape: [numInputLogits(batchSize * dynamicTreeMaxTopK), - // maxDecodingDraftTokens] - firstTopKOutputLogProbs = reinterpret_cast<float*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * maxDecodingDraftTokens * sizeof(float))); - } - - SizeType32 const secondTopKVocabSize = dynamicTreeMaxTopK * maxDecodingDraftTokens; - // Workspace 9: Sampling from [batchSize, dynamicTreeMaxTopK * maxDecodingDraftTokens] to [batchSize, - // dynamicTreeMaxTopK] - auto const secondTopKSamplingWorkspaceSize - = getTopKWorkspaceSize<float>(batchSize, /* maxTokensPerStep */ 1, /* maxTopK */ maxTopK, secondTopKVocabSize); - void* workspaceScoresSampling - = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, secondTopKSamplingWorkspaceSize)); - - // Workspace 10: the second (scores) sampling's outputIds, shape: [batchSize, maxDecodingDraftTokens] - TokenIdType* secondTopKOutputIdsFlatten = reinterpret_cast<TokenIdType*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingDraftTokens * sizeof(TokenIdType))); - - // Workspace 11: the second (scores) sampling's outputIdsPtrs - TokenIdType** secondTopKOutputIdsPtrs = reinterpret_cast<TokenIdType**>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(TokenIdType*))); - - // Workspace 12: input scores pointers - float** secondTopKInputScoresPtrs - = reinterpret_cast<float**>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(float*))); - - // Workspace 13: the second sampling's outputLogProbs - float* secondTopKOutputLogProbs = reinterpret_cast<float*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingDraftTokens * sizeof(float))); - - // Workspace 14: The input scores pointers of the third topK sampling, shape [batchSize] - float** thirdTopKInputScoresPtrs - = reinterpret_cast<float**>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(float*))); - - // Workspace 15: The output of the third topK sampling, shape [batchSize, maxDecodingDraftTokens] - TokenIdType* thirdTopKOutputIds = reinterpret_cast<TokenIdType*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingDraftTokens * sizeof(TokenIdType))); - - // Workspace 16: The output pointers of the third topK sampling, shape [batchSize] - TokenIdType** thirdTopKOutputIdsPtrs = reinterpret_cast<TokenIdType**>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(TokenIdType*))); - - // The number of draft tokens among all layers - long const totalNumDraftTokensForAllLayers - = (mNumEagleLayers - 1) * dynamicTreeMaxTopK * dynamicTreeMaxTopK + dynamicTreeMaxTopK; - - auto const thridTopKSamplingWorkspaceSize = getTopKWorkspaceSize<float>( - batchSize, /* maxTokensPerStep */ 1, /* maxTopK */ maxDecodingDraftTokens, totalNumDraftTokensForAllLayers); - // Workspace 17: The workspace of the third topK sampling - void* workspaceThirdTopKSampling - = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, thridTopKSamplingWorkspaceSize)); - - // Workspace 18. Eagle-2, the topKs for each request in the third topK sampling, shape [batchSize] - // The real topK value is min(maxDecodingDraftTokens, totalNumDraftTokensForAllLayers) - SizeType32* thirdTopKs - = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(SizeType32))); - - ////////////////////////////////////////// Main logic ////////////////////////////////////////// - // Fill logitsPtrs from plugin input logits - // And fill firstTopKOutputIdsPtrs from firstTopKOutputIdsFlatten - invokeAssembleDraftLogitsOffsets(logitsPtrs, pluginInputLogits, firstTopKOutputIdsPtrs, firstTopKOutputIdsFlatten, - skipDecode, numValidLogits, numInputLogits, batchSize, maxDecodingDraftTokens, vocabSizePadded, stream); - sync_check_cuda_error(stream); - - if (useDynamicTree) - { - // For Eagle-2, the topK value between different requests are the same, all set to 'dynamicTreeMaxTopK'. - invokeSetTopKsFromDyanmicTreeMaxTopK( - mLayerIdx, batchSize, numInputLogits, topKs, topKOffset, dynamicTreeMaxTopK, numValidLogits, stream); - sync_check_cuda_error(stream); - - // Do softmax for the input logits - // We set the 'batchSize' and 'maxBatchSize' to 'numInputLogits', while 'numInputLogits' logits may contain - // some padding logits, which do not need to be calculated. - // We use 'skipDecode' list to skip these padding logits. This could avoid redundant calculations. - BiasSoftmaxParams<T> biasSoftmaxParams; - biasSoftmaxParams.logits = const_cast<T*>(pluginInputLogits); - biasSoftmaxParams.logitsPtrs = nullptr; - biasSoftmaxParams.probs = const_cast<T*>(pluginInputLogits); - biasSoftmaxParams.maxBeamWidth = 1; - biasSoftmaxParams.batchSlots = nullptr; - biasSoftmaxParams.batchSize = numInputLogits; - biasSoftmaxParams.maxBatchSize = numInputLogits; - biasSoftmaxParams.vocabSize = vocabSizePadded; - biasSoftmaxParams.vocabSizePadded = vocabSizePadded; - biasSoftmaxParams.skipSoftMax = false; - biasSoftmaxParams.batchSlotsLogits = false; - biasSoftmaxParams.skipDecode = skipDecode; - biasSoftmaxParams.checkParams(); - - invokeAddBiasSoftMax(biasSoftmaxParams, stream); - sync_check_cuda_error(stream); - } - else - { - // For Eagle-1, extract topK value from input path. - invokeExtractTopKsFromPath(pluginInputPaths, topKs, topKOffset, numSuccessorsForEachNode, mLayerIdx, batchSize, - maxDecodingTokens, maxPathLen, stream); - sync_check_cuda_error(stream); - } - - TopKSamplingKernelParams<T> params{}; - params.logProbsPtrs = logitsPtrs; // [numInputLogits][vocabSizePadded] - params.outputIdsPtrs = firstTopKOutputIdsPtrs; // [numInputLogits][maxDecodingDraftTokens] - params.workspace = workspaceSampling; - params.maxTopK = maxTopK; - params.topKs = topKs; // [numInputLogits] - params.batchSize = numInputLogits; - params.maxBatchSize = numInputLogits; - params.maxTokensPerStep = 1; - params.vocabSizePadded = vocabSizePadded; - params.returnAllSelectedTokens = true; - params.strictTopPBoundary = false; - params.skipDecode = skipDecode; - params.outputLogProbs = firstTopKOutputLogProbs; // [numInputLogits * maxDecodingDraftTokens] - params.logitsHasProbs = true; - - invokeBatchTopKSampling(params, stream); - sync_check_cuda_error(stream); - - if (useDynamicTree) - { - // When mLayerIdx == 0, we do not need to update scores. - // We take the outputLogProbs of the first topK sampling as the scores directly. - if (mLayerIdx != 0) - { - // Update firstTopKOutputLogProbs with pluginInputPrevScores, which is the scores from the previous layer - invokeUpdateScores(batchSize, dynamicTreeMaxTopK, maxDecodingDraftTokens, firstTopKOutputLogProbs, - pluginInputPrevScores, stream); - sync_check_cuda_error(stream); - - // Do the second top-dynamicTreeMaxTopK sampling among this dynamicTreeMaxTopK x dynamicTreeMaxTopK draft - // tokens. Through the second topK sampling, we obtain the dynamicTreeMaxTopK output draft tokens of this - // layer. - - // Although theoretically we only need to select 'dynamicTreeMaxTopK' draft tokens from 'dynamicTreeMaxTopK - // * dynamicTreeMaxTopK' draft tokens, we over-set vocabSize here. This is because when we write the scores - // into firstTopKOutputLogProbs, we store it in the form of [batchSize * dynamicTreeMaxTopK, - // maxDecodingDraftTokens]. For each request, these 'dynamicTreeMaxTopK * dynamicTreeMaxTopK' scores are not - // saved continuously, but in the format of [dynamicTreeMaxTopK, maxDecodingDraftTokens]. For unused - // positions, we set '-inf' to ensure that they will not be sampled. Examples: For a request, - // dynamicTreeMaxTopK == 3, the scores in its buffer ([dynamicTreeMaxTopK, maxDecodingDraftTokens]) are as - // follow: - // [[1.1, 2.2, 3.3, -inf, -inf, ...], - // [4.4, 5.5, 6.6, -inf, -inf, ...], - // [7.7, 8.8, 9.9, -inf, -inf, ...]] - - // Prepare the input of the second topK sampling. - invokeAssembleSecondTopKSamplingInputs(batchSize, dynamicTreeMaxTopK, maxDecodingDraftTokens, - firstTopKOutputLogProbs, secondTopKInputScoresPtrs, secondTopKOutputIdsFlatten, secondTopKOutputIdsPtrs, - stream); - sync_check_cuda_error(stream); - - TopKSamplingKernelParams<float> params{}; - params.logProbsPtrs = secondTopKInputScoresPtrs; - params.outputIdsPtrs = secondTopKOutputIdsPtrs; - params.workspace = workspaceScoresSampling; - params.maxTopK = maxTopK; // Same to maxDecodingTokens - params.topKs = topKs; // [batchSize], all set to dynamicTreeMaxTopK - params.batchSize = batchSize; - params.maxBatchSize = batchSize; - params.maxTokensPerStep = 1; - params.vocabSizePadded = secondTopKVocabSize; - params.returnAllSelectedTokens = true; - params.strictTopPBoundary = false; - - invokeBatchTopKSampling(params, stream); - sync_check_cuda_error(stream); - } - - // Copy this layer's scores and draft tokensId: - // 1) Copy this layer's scores to pluginOutputAllLayersScores - // 2) Copy dynamicTreeMaxTopK (or dynamicTreeMaxTopK * dynamicTreeMaxTopK) draft tokens to - // pluginOutputAllLayersDraftTokenIds 3) Set the predecessors of these draft tokens and save to - // pluginOutputAllLayersDraftTokenIdsPredecessor, - // which will be used to reconstruct the final output tree at the last layer - invokeCopyScoresAndDraftTokenIds(mLayerIdx, mNumEagleLayers, maxDecodingDraftTokens, batchSize, - dynamicTreeMaxTopK, - pluginInputCurrentExpandIndices, // The indices of the nodes that expand in this layer (i.e., the input - // logits). The index is related to the final tree. - pluginInputAllLayersScores, pluginInputAllLayersDraftTokenIds, pluginInputAllLayersDraftTokenIdsPredecessor, - pluginOutputAllLayersScores, pluginOutputAllLayersDraftTokenIds, - pluginOutputAllLayersDraftTokenIdsPredecessor, - firstTopKOutputLogProbs, // This layer's scores - firstTopKOutputIdsFlatten, // This layer's draft tokens - stream); - sync_check_cuda_error(stream); - - // Update Path - // For mLayerIdx == 0, the output of the first topK sampling are the output draft tokens of this layers. The - // update logic is simple. For mLayerIdx > 0, the output of the second topK sampling are the output draft tokens - // of this layers. 'secondTopKOutputIdsPtrs' contains the top-dynamicTreeMaxTopK selected from the second topK - // sampling. 'pluginOutputNextExpandIndices' record the selected the top-dynamicTreeMaxTopK draft token's Id of - // this layer, - // which will be used in the next layer to compute the predecessors. - // The last layer will completely reconstruct the paths, so there is no need to update the paths here. - if (mLayerIdx != mNumEagleLayers - 1) - { - invokeUpdatePath(mLayerIdx, batchSize, dynamicTreeMaxTopK, maxDecodingTokens, maxPathLen, pluginInputPaths, - pluginOutputPaths, - secondTopKOutputIdsPtrs, // if mLayerIdx == 0, secondTopKOutputIdsPtrs == nullptr, and it's useless - // during update paths - pluginOutputNextExpandIndices, stream); - sync_check_cuda_error(stream); - } - - if (mLayerIdx != 0) - { - // We will extract the real draft tokenIds and scores from 'firstTopKOutputIdsFlatten' and - // 'secondTopKInputScoresPtrs' according to the 'secondTopKOutputIdsPtrs'. And store them into - // 'secondTopKOutputIdsPtrs' and 'secondTopKOutputLogProbs' (reuse these buffers). - // secondTopKInputScoresPtrs: shape [batchSize * dynamicTreeMaxTopK, maxDecodingDraftTokens] - // The original scores, which were used to do the second TopK sampling - // secondTopKOutputIdsPtrs: shape [batchSize], each points to a [maxDecodingDraftTokens] buffer - // The output of the second TopK sampling, which are the indices of the top-dynamicTreeMaxTopK among - // 'dynamicTreeMaxTopK * dynamicTreeMaxTopK'. We need to figure out what these top-dynamicTreeMaxTopK - // draft tokens' real tokenIds. - // firstTopKOutputIdsFlatten: shape [batchSize * dynamicTreeMaxTopK, maxDecodingDraftTokens] - // The value are related to the vocabSize, which is the real tokenIds. - invokeExtractScoresAndRealDraftTokensIds(batchSize, dynamicTreeMaxTopK, maxDecodingDraftTokens, - secondTopKInputScoresPtrs, secondTopKOutputIdsPtrs, firstTopKOutputIdsFlatten, secondTopKOutputLogProbs, - stream); - sync_check_cuda_error(stream); - } - - // Copy this layer's output draft tokens and scores. - // This layer's output scores is next layer's previous scores. - // if mLayerIdx == 0, directly use the first topK's outputIds / logProbs as this layer's output draft tokens / - // scores if mLayerIdx > 0, we use the second topK's outputIds / logProbs, - // which is updated with the real draft tokenIds / logprobs in 'invokeExtractScoresAndRealDraftTokensIds' - invokeUpdateDraftTokensAndLensAndCurScores(mLayerIdx, batchSize, dynamicTreeMaxTopK, maxDecodingDraftTokens, - mLayerIdx == 0 ? firstTopKOutputIdsPtrs : secondTopKOutputIdsPtrs, pluginInputDraftTokenIds, - pluginInputDraftLens, pluginOutputDraftTokenIds, pluginOutputDraftLens, - mLayerIdx == 0 ? firstTopKOutputLogProbs : secondTopKOutputLogProbs, pluginOutputCurrentScores, stream); - sync_check_cuda_error(stream); - - if (mLayerIdx == mNumEagleLayers - 1) - { - // The maximum number of nodes on the final tree (exclude the root node) - auto const maxNodesOnFinalTree = std::min(maxDecodingDraftTokens, totalNumDraftTokensForAllLayers); - - // When reach the last EagleNet, we need to do the third sampling, which take all layers' draft tokens and - // scores as input, and then select top-maxDecodingDraftTokens draft tokens among them. We need to - // reconstruct the path/tree after the third topK sampling. - invokeAssembleThridTopKSamplingInputs(batchSize, maxDecodingDraftTokens, mNumEagleLayers, - maxNodesOnFinalTree, thirdTopKs, pluginOutputAllLayersScores, thirdTopKInputScoresPtrs, - thirdTopKOutputIds, thirdTopKOutputIdsPtrs, stream); - sync_check_cuda_error(stream); - - // 1) Do topK sampling among all previous draft tokens - TopKSamplingKernelParams<float> params{}; - params.logProbsPtrs = thirdTopKInputScoresPtrs; - params.outputIdsPtrs = thirdTopKOutputIdsPtrs; - params.workspace = workspaceThirdTopKSampling; - params.topKs = thirdTopKs; // All set to 'maxNodesOnFinalTree' - params.maxTopK = maxDecodingDraftTokens; // We set maxTopK to 'maxDecodingDraftTokens' to align the - // outputIdsPtrs offsets when written back. - params.batchSize = batchSize; - params.maxBatchSize = batchSize; - params.maxTokensPerStep = 1; - params.vocabSizePadded = totalNumDraftTokensForAllLayers; - params.returnAllSelectedTokens = true; - params.strictTopPBoundary = false; // Make sure to select topK tokens. - - invokeBatchTopKSampling(params, stream); - sync_check_cuda_error(stream); - - // 2) Reconstruct the Path - invokeReconstructFinalPath(batchSize, dynamicTreeMaxTopK, maxDecodingDraftTokens, maxDecodingTokens, - maxPathLen, mNumEagleLayers, maxNodesOnFinalTree, thirdTopKOutputIdsPtrs, - pluginOutputAllLayersDraftTokenIdsPredecessor, pluginOutputPaths, stream); - sync_check_cuda_error(stream); - - // 3) Copy this layer's outputIds to outputDraftTokenIds - invokeCopyFinalDraftTokens(batchSize, maxDecodingDraftTokens, mNumEagleLayers, maxNodesOnFinalTree, - thirdTopKOutputIdsPtrs, pluginOutputAllLayersDraftTokenIds, pluginOutputDraftTokenIds, - pluginOutputDraftLens, stream); - sync_check_cuda_error(stream); - } - } - else - { - // Eagle-1: Copy output token id from outputIdsPtrs to the plugin output buffer - invokeCopyOutputTokensIds(firstTopKOutputIdsPtrs, topKs, topKOffset, pluginInputDraftTokenIds, - pluginInputDraftLens, numValidLogits, pluginOutputDraftTokenIds, pluginOutputDraftLens, mLayerIdx, - batchSize, maxDecodingDraftTokens, pluginInputPaths, pluginOutputPaths, maxPathLen, stream); - sync_check_cuda_error(stream); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -template <typename T> -void EagleDecodeDraftTokensPlugin::enqueueType(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // TODO split batch into greedy and non-greedy and execute both paths - if (mTopKSampling) - { - doTopKSampling<T>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else - { - // TODO fill me - TLLM_CHECK_WITH_INFO(false, "Multinomial sampling is not supported yet"); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -int EagleDecodeDraftTokensPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - auto const logitsType = inputDesc[getIdx(InputIdxEntry::LOGITS)].type; - if (logitsType == nvinfer1::DataType::kFLOAT) - { - enqueueType<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (logitsType == nvinfer1::DataType::kHALF) - { - enqueueType<__half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else - { - TLLM_CHECK_WITH_INFO(false, "Unsupported logits type"); - } - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType EagleDecodeDraftTokensPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index < getNbOutputs()); - TLLM_CHECK(index < getNbOutputs()); - if (index == getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_SCORES) - || index == getIdx(OutputIdxEntry::OUTPUT_CURRENT_SCORES)) - { - // Only output_prev_socres are float - return inputTypes[getIdx(InputIdxEntry::INPUT_ALL_LAYERS_SCORES)]; - } - else - { - // output_draft_token_ids, output_draft_lens, output_paths, output_next_expand_index, - // output_all_layers_draft_token_ids, output_all_layers_draft_token_ids_predecessor - // are all int32 type, same as path - return inputTypes[getIdx(InputIdxEntry::PATHS)]; - } -} - -// IPluginV2 Methods - -char const* EagleDecodeDraftTokensPlugin::getPluginType() const noexcept -{ - return EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_NAME; -} - -char const* EagleDecodeDraftTokensPlugin::getPluginVersion() const noexcept -{ - return EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_VERSION; -} - -int EagleDecodeDraftTokensPlugin::getNbOutputs() const noexcept -{ - return 8; -} - -int EagleDecodeDraftTokensPlugin::initialize() noexcept -{ - return 0; -} - -void EagleDecodeDraftTokensPlugin::terminate() noexcept {} - -size_t EagleDecodeDraftTokensPlugin::getSerializationSize() const noexcept -{ - return sizeof(mDtype) + sizeof(mLayerIdx) + sizeof(mNumEagleLayers) + sizeof(mTopKSampling); -} - -void EagleDecodeDraftTokensPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mDtype); - write(d, mLayerIdx); - write(d, mNumEagleLayers); - write(d, mTopKSampling); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void EagleDecodeDraftTokensPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -EagleDecodeDraftTokensPluginCreator::EagleDecodeDraftTokensPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("layer_idx", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("num_eagle_layers", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("top_k_sampling", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* EagleDecodeDraftTokensPluginCreator::getPluginName() const noexcept -{ - return EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_NAME; -} - -char const* EagleDecodeDraftTokensPluginCreator::getPluginVersion() const noexcept -{ - return EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_VERSION; -} - -PluginFieldCollection const* EagleDecodeDraftTokensPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* EagleDecodeDraftTokensPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int32_t layerIdx{}; - int32_t numEagleLayers{}; - nvinfer1::DataType type{}; - bool topKSampling{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "layer_idx")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - layerIdx = *static_cast<int32_t const*>(fields[i].data); - } - else if (!strcmp(attrName, "num_eagle_layers")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - numEagleLayers = *static_cast<int32_t const*>(fields[i].data); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "top_k_sampling")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - topKSampling = static_cast<bool>(*static_cast<int32_t const*>(fields[i].data)); - } - } - - try - { - auto* obj = new EagleDecodeDraftTokensPlugin(type, layerIdx, numEagleLayers, topKSampling); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* EagleDecodeDraftTokensPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call EagleDecodeDraftTokensPlugin::destroy() - try - { - auto* obj = new EagleDecodeDraftTokensPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.h b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.h deleted file mode 100644 index 8c144a1bc073..000000000000 --- a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.h +++ /dev/null @@ -1,174 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class EagleDecodeDraftTokensPlugin : public BasePlugin -{ -public: - EagleDecodeDraftTokensPlugin(nvinfer1::DataType type, int32_t layerIdx, int32_t numEagleLayers, bool topKSampling); - - EagleDecodeDraftTokensPlugin(void const* data, size_t length); - - ~EagleDecodeDraftTokensPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - enum class InputIdxEntry : int32_t - { - // 12 inputs - // [num_input_logits, vocab_size_padded] - LOGITS = 0, - // [batch_size, max_decoding_tokens, max_path_len] - PATHS, - // [1] - NUM_VALID_LOGITS, - // [1] - USE_DYNAMIC_TREE, - // [1] - DYNAMIC_TREE_MAX_TOPK, - - // [batch_size, max_decoding_draft_tokens] - INPUT_DRAFT_TOKEN_IDS, - // [batch_size] - INPUT_DRAFT_LENS, - - // [batch_size, max_decoding_draft_tokens] - INPUT_PREV_SCORES, - - // [batch_size, max_decoding_draft_tokens] - INPUT_CURRENT_EXPAND_INDICES, - - // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] - INPUT_ALL_LAYERS_SCORES, - // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] - INPUT_ALL_LAYERS_DRAFT_TOKEN_IDS, - // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] - INPUT_ALL_LAYERS_DRAFT_TOKEN_IDS_PREDECESSOR - }; - - enum class OutputIdxEntry : int32_t - { - // 8 outputs - // [batch_size, max_decoding_draft_tokens] - OUTPUT_DRAFT_TOKEN_IDS = 0, - // [batch_size] - OUTPUT_DRAFT_LENS, - - // [batch_size, max_decoding_tokens, max_path_len] - OUTPUT_PATHS, - - // [batch_size, max_decoding_draft_tokens] - OUTPUT_CURRENT_SCORES, - - // [batch_size, max_decoding_draft_tokens] - OUTPUT_NEXT_EXPAND_INDICES, - - // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] - OUTPUT_ALL_LAYERS_SCORES, - // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] - OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS, - // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] - OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS_PREDECESSOR - }; - - int32_t getIdx(InputIdxEntry idx) const - { - return static_cast<int32_t>(idx); - } - - int32_t getIdx(OutputIdxEntry idx) const - { - return static_cast<int32_t>(idx); - } - -private: - template <typename T> - size_t getWorkspaceSizeType(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept; - - template <typename T> - void enqueueType(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept; - - template <typename T> - void doTopKSampling(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept; - -private: - nvinfer1::DataType mDtype; // Logit datatype - int32_t mLayerIdx{-1}; // Index of eagle layer - int32_t mNumEagleLayers{-1}; // Number of eagle layers - bool mTopKSampling; // Use TopK sampling or multinomial sampling -}; - -class EagleDecodeDraftTokensPluginCreator : public BaseCreator -{ -public: - EagleDecodeDraftTokensPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.cpp b/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.cpp deleted file mode 100644 index 2cd8c695e296..000000000000 --- a/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.cpp +++ /dev/null @@ -1,548 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "eaglePrepareDrafterInputsPlugin.h" - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/kernels/speculativeDecoding/eagleDecodingKernels.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iTensor.h" - -using namespace nvinfer1; -using tensorrt_llm::plugins::EaglePrepareDrafterInputsPluginCreator; -using tensorrt_llm::plugins::EaglePrepareDrafterInputsPlugin; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::kernels::speculative_decoding; -using namespace tensorrt_llm::runtime; -namespace tc = tensorrt_llm::common; - -static char const* EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_VERSION{"1"}; -static char const* EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_NAME{"EaglePrepareDrafterInputs"}; -PluginFieldCollection EaglePrepareDrafterInputsPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> EaglePrepareDrafterInputsPluginCreator::mPluginAttributes; - -EaglePrepareDrafterInputsPlugin::EaglePrepareDrafterInputsPlugin( - int32_t layerIdx, int32_t numLayers, int32_t maxNonLeavesPerLayer) - : mLayerIdx(layerIdx) - , mNumLayers(numLayers) - , mMaxNonLeavesPerLayer(maxNonLeavesPerLayer) -{ -} - -void EaglePrepareDrafterInputsPlugin::initFieldsToSerialize() -{ - mDataToSerialize.clear(); - mDataToSerialize.emplace_back(PluginField("layer_idx", &mLayerIdx, PluginFieldType::kINT32, 1)); - mDataToSerialize.emplace_back(PluginField("num_layers", &mNumLayers, PluginFieldType::kINT32, 1)); - mDataToSerialize.emplace_back( - PluginField("max_non_leaves_per_layer", &mMaxNonLeavesPerLayer, PluginFieldType::kINT32, 1)); - mFCToSerialize.nbFields = mDataToSerialize.size(); - mFCToSerialize.fields = mDataToSerialize.data(); -} - -nvinfer1::IPluginCapability* EaglePrepareDrafterInputsPlugin::getCapabilityInterface( - nvinfer1::PluginCapabilityType type) noexcept -{ - try - { - if (type == nvinfer1::PluginCapabilityType::kBUILD) - { - return static_cast<nvinfer1::IPluginV3OneBuild*>(this); - } - if (type == nvinfer1::PluginCapabilityType::kRUNTIME) - { - return static_cast<nvinfer1::IPluginV3OneRuntime*>(this); - } - TLLM_CHECK(type == nvinfer1::PluginCapabilityType::kCORE); - return static_cast<nvinfer1::IPluginV3OneCore*>(this); - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -// IPluginV3 methods -nvinfer1::IPluginV3* EaglePrepareDrafterInputsPlugin::clone() noexcept -{ - auto clone = std::make_unique<EaglePrepareDrafterInputsPlugin>(*this); - clone->initFieldsToSerialize(); - return clone.release(); -} - -// IPluginV3OneCore methods -char const* EaglePrepareDrafterInputsPlugin::getPluginName() const noexcept -{ - return EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_NAME; -} - -char const* EaglePrepareDrafterInputsPlugin::getPluginVersion() const noexcept -{ - return EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_VERSION; -} - -char const* EaglePrepareDrafterInputsPlugin::getPluginNamespace() const noexcept -{ - return tensorrt_llm::plugins::api::kDefaultNamespace; -} - -// IPluginV3OneBuild methods -int32_t EaglePrepareDrafterInputsPlugin::getNbOutputs() const noexcept -{ - return 11; -} - -int32_t EaglePrepareDrafterInputsPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept -{ - return 0; -} - -bool EaglePrepareDrafterInputsPlugin::supportsFormatCombination( - int32_t pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept -{ - return (inOut[pos].desc.type == nvinfer1::DataType::kINT32) && (inOut[pos].desc.format == TensorFormat::kLINEAR); -} - -int32_t EaglePrepareDrafterInputsPlugin::getOutputDataTypes(nvinfer1::DataType* outputTypes, int32_t nbOutputs, - nvinfer1::DataType const* inputTypes, int32_t nbInputs) const noexcept -{ - outputTypes[0] = nvinfer1::DataType::kINT32; - outputTypes[1] = nvinfer1::DataType::kINT32; - outputTypes[2] = nvinfer1::DataType::kINT32; - outputTypes[3] = nvinfer1::DataType::kINT32; - outputTypes[4] = nvinfer1::DataType::kINT32; - outputTypes[5] = nvinfer1::DataType::kINT32; - outputTypes[6] = nvinfer1::DataType::kINT32; - outputTypes[7] = nvinfer1::DataType::kINT32; - outputTypes[8] = nvinfer1::DataType::kINT32; - outputTypes[9] = nvinfer1::DataType::kINT32; - outputTypes[10] = nvinfer1::DataType::kINT32; - outputTypes[11] = nvinfer1::DataType::kINT32; - return 0; -} - -int32_t EaglePrepareDrafterInputsPlugin::getOutputShapes(nvinfer1::DimsExprs const* inputs, int32_t nbInputs, - nvinfer1::DimsExprs const* shapeInputs, int32_t nbShapeInputs, nvinfer1::DimsExprs* outputs, int32_t nbOutputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - TLLM_CHECK(nbOutputs == 11); - TLLM_CHECK(nbInputs == 15); - TLLM_CHECK(nbShapeInputs == 0); - auto const numTokens = inputs[getIdx(InputIdxEntry::INPUT_IDS)].d[0]; - auto const batchSizeExpr = inputs[getIdx(InputIdxEntry::PREV_DRAFT_PATHS)].d[0]; - auto const numGenRequestsExpr = inputs[getIdx(InputIdxEntry::SPEC_DECODING_GENERATION_LENGTHS)].d[0]; - auto const numInputGenTokensExpr = inputs[getIdx(InputIdxEntry::INPUT_GEN_TOKENS)].d[0]; - auto const maxDecodingLenExpr = inputs[getIdx(InputIdxEntry::PREV_DRAFT_PATHS)].d[1]; - auto const maxPathLenExpr = inputs[getIdx(InputIdxEntry::PREV_DRAFT_PATHS)].d[2]; - - for (SizeType32 outputIndex = 0; outputIndex < nbOutputs; ++outputIndex) - { - if (outputIndex == getIdx(OutputIdxEntry::SEQUENCE_LENGTHS) - || outputIndex == getIdx(OutputIdxEntry::CONTEXT_LENGTHS) - || outputIndex == getIdx(OutputIdxEntry::SPEC_DECODING_GENERATION_LENGTHS)) - { - outputs[outputIndex] = inputs[getIdx(InputIdxEntry::SEQUENCE_LENGTHS)]; - } - else if (outputIndex == getIdx(OutputIdxEntry::SPEC_DECODING_PACKED_MASK)) - { - outputs[outputIndex].nbDims = 3; - outputs[outputIndex].d[0] = batchSizeExpr; - outputs[outputIndex].d[1] = maxDecodingLenExpr; - outputs[outputIndex].d[2] - = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *maxDecodingLenExpr, *exprBuilder.constant(32)); - } - else if (outputIndex == getIdx(OutputIdxEntry::SPEC_DECODING_POSITION_OFFSETS)) - { - outputs[outputIndex].nbDims = 2; - outputs[outputIndex].d[0] = batchSizeExpr; - outputs[outputIndex].d[1] = maxDecodingLenExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_IDS) - || outputIndex == getIdx(OutputIdxEntry::HIDDEN_STATES_INDICES) - || (mLayerIdx == 0 && outputIndex == getIdx(OutputIdxEntry::POSITION_IDS))) - { - if (mLayerIdx == 0) - { - // We have at most numGenRequests * (mNumLayers + 1) accepted tokens per step for gen requests and - // input_ids - numGenTokens tokens for context requests. - auto numOutputGenTokensExpr = exprBuilder.operation( - DimensionOperation::kPROD, *numGenRequestsExpr, *exprBuilder.constant(mNumLayers + 1)); - auto numInputCtxTokensExpr - = exprBuilder.operation(DimensionOperation::kSUB, *numTokens, *numInputGenTokensExpr); - outputs[outputIndex].nbDims = 1; - outputs[outputIndex].d[0] = exprBuilder.operation(DimensionOperation::kMAX, *exprBuilder.constant(1), - *exprBuilder.operation(DimensionOperation::kSUM, *numOutputGenTokensExpr, *numInputCtxTokensExpr)); - } - else - { - // At most we have mMaxNonLeavesPerLayer non-leaves at this layer. - // And in total we pass all non-leaves + all their preceding nodes. - // batchSize * mMaxNonLeavesPerLayer * layerIdx - outputs[outputIndex].nbDims = 1; - outputs[outputIndex].d[0] = exprBuilder.operation(DimensionOperation::kPROD, - *exprBuilder.operation(DimensionOperation::kPROD, *exprBuilder.constant(mLayerIdx), - *exprBuilder.constant(mMaxNonLeavesPerLayer)), - *batchSizeExpr); - } - } - else if (mLayerIdx > 0 && outputIndex == getIdx(OutputIdxEntry::POSITION_IDS)) - { - outputs[outputIndex].nbDims = 1; - outputs[outputIndex].d[0] = batchSizeExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::LAST_TOKEN_INDICES)) - { - outputs[outputIndex].nbDims = 1; - outputs[outputIndex].d[0] = exprBuilder.operation( - DimensionOperation::kPROD, *exprBuilder.constant(mMaxNonLeavesPerLayer), *batchSizeExpr); - } - else if (outputIndex == getIdx(OutputIdxEntry::NUM_LAST_TOKEN_INDICES)) - { - outputs[outputIndex].nbDims = 1; - outputs[outputIndex].d[0] = exprBuilder.constant(1); - } - else if (outputIndex == getIdx(OutputIdxEntry::HIDDEN_SIZE_BATCH_LEVEL_STARTS)) - { - // batchSize * (maxPathLen - 1) + 1 - outputs[outputIndex].nbDims = 1; - outputs[outputIndex].d[0] = exprBuilder.operation(DimensionOperation::kSUM, *exprBuilder.constant(1), - *exprBuilder.operation(DimensionOperation::kPROD, *batchSizeExpr, - *exprBuilder.operation(DimensionOperation::kSUB, *maxPathLenExpr, *exprBuilder.constant(1)))); - } - } - return 0; -} - -int32_t EaglePrepareDrafterInputsPlugin::onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, - nvinfer1::PluginTensorDesc const* out, int32_t nbOutputs) noexcept -{ - return 0; -} - -nvinfer1::IPluginV3* EaglePrepareDrafterInputsPlugin::attachToContext( - nvinfer1::IPluginResourceContext* context) noexcept -{ - return clone(); -} - -PluginFieldCollection const* EaglePrepareDrafterInputsPlugin::getFieldsToSerialize() noexcept -{ - return &mFCToSerialize; -} - -size_t EaglePrepareDrafterInputsPlugin::getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - size_t workspaceSize{0}; - - auto const batchSize = inputs[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)].max.d[0]; - auto const maxDecodingTokens = inputs[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)].max.d[1]; - - if (mLayerIdx > 0) - { - SizeType32 constexpr NUM_BUFFERS{9}; - size_t workspaces[NUM_BUFFERS]; - workspaces[0] = batchSize * maxDecodingTokens * sizeof(int8_t); // isLeafMask - workspaces[1] = batchSize * maxDecodingTokens * sizeof(SizeType32); // selectedDraftIndices - workspaces[2] = batchSize * maxDecodingTokens * sizeof(SizeType32); // selectedDraftPosOffsets - workspaces[3] = batchSize * sizeof(SizeType32); // numSelectedDraftIndices - workspaces[4] = batchSize * maxDecodingTokens * maxDecodingTokens * sizeof(int8_t); // selectedMasks - workspaces[5] = (batchSize + 1) * sizeof(SizeType32); // cumSumGenerationLengths - workspaces[6] = batchSize * maxDecodingTokens * sizeof(SizeType32); // nonLeavesInLevelOffsets - workspaces[7] = batchSize * maxDecodingTokens * sizeof(SizeType32); // parentNonLeafInLevelOffset - workspaces[8] = 1 * sizeof(SizeType32); // maxGenerationLength - workspaceSize = tc::calculateTotalWorkspaceSize(workspaces, NUM_BUFFERS); - } - - return workspaceSize; -} - -void EaglePrepareDrafterInputsPlugin::prepareCtxEagleNetData(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const batchSize = inputDesc[getIdx(InputIdxEntry::SEQUENCE_LENGTHS)].dims.d[0]; - - auto const numTokens = inputDesc[getIdx(InputIdxEntry::INPUT_IDS)].dims.d[0]; - auto const numGenRequests = inputDesc[getIdx(InputIdxEntry::SPEC_DECODING_GENERATION_LENGTHS)].dims.d[0]; - auto const numInputGenTokens = inputDesc[getIdx(InputIdxEntry::INPUT_GEN_TOKENS)].dims.d[0]; - - auto const maxPathLen = inputDesc[getIdx(InputIdxEntry::ACCEPTED_TOKENS)].dims.d[1]; - auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)].dims.d[1]; - - auto eagleNetSequenceLengths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::SEQUENCE_LENGTHS)]); - auto eagleNetContextLengths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::CONTEXT_LENGTHS)]); - auto outputIds = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::OUTPUT_IDS)]); - auto positionIds = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::POSITION_IDS)]); - auto hiddenStatesIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::HIDDEN_STATES_INDICES)]); - auto lastTokenIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::LAST_TOKEN_INDICES)]); - auto numLastTokenIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::NUM_LAST_TOKEN_INDICES)]); - auto hiddenSizeBatchLevelStarts - = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::HIDDEN_SIZE_BATCH_LEVEL_STARTS)]); - - auto inputIds = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::INPUT_IDS)]); - auto chunkedContextNextTokens - = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::CHUNKED_CONTEXT_NEXT_TOKENS)]); - auto baseNetSequenceLengths = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::SEQUENCE_LENGTHS)]); - auto baseNetContextLengths = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::CONTEXT_LENGTHS)]); - auto acceptedTokens = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::ACCEPTED_TOKENS)]); - auto acceptedLens = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::ACCEPTED_LENS)]); - auto prevDraftLens = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::PREV_DRAFT_LENS)]); - auto prevPaths = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::PREV_DRAFT_PATHS)]); - auto bestPathIds = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::ACCEPTED_PATHS)]); - - auto const numOutputTokens = (numTokens - numInputGenTokens) + (numGenRequests * (mNumLayers + 1)); - cudaMemsetAsync(positionIds, 0, numOutputTokens * sizeof(SizeType32), stream); - cudaMemsetAsync(hiddenStatesIndices, 0, numOutputTokens * sizeof(SizeType32), stream); - - invokePrepareCtxEagleNetInputs(eagleNetSequenceLengths, eagleNetContextLengths, outputIds, positionIds, - hiddenStatesIndices, lastTokenIndices, numLastTokenIndices, hiddenSizeBatchLevelStarts, inputIds, - chunkedContextNextTokens, baseNetSequenceLengths, baseNetContextLengths, acceptedTokens, acceptedLens, - prevDraftLens, prevPaths, bestPathIds, batchSize, maxPathLen, maxDecodingTokens, mMaxNonLeavesPerLayer, stream); - - sync_check_cuda_error(stream); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void EaglePrepareDrafterInputsPlugin::prepareGenEagleNetData(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const batchSize = inputDesc[getIdx(InputIdxEntry::SEQUENCE_LENGTHS)].dims.d[0]; - auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)].dims.d[1]; - auto const maxPathLen = inputDesc[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)].dims.d[2]; - - auto eagleNetSequenceLengths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::SEQUENCE_LENGTHS)]); - auto eagleNetContextLengths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::CONTEXT_LENGTHS)]); - auto outputIds = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::OUTPUT_IDS)]); - auto positionIds = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::POSITION_IDS)]); - auto specDecodingGenLengths - = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::SPEC_DECODING_GENERATION_LENGTHS)]); - auto specDecodingPositionOffsets - = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::SPEC_DECODING_POSITION_OFFSETS)]); - auto specDecodingPackedMasks - = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::SPEC_DECODING_PACKED_MASK)]); - auto hiddenStatesIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::HIDDEN_STATES_INDICES)]); - auto lastTokenIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::LAST_TOKEN_INDICES)]); - auto numLastTokenIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::NUM_LAST_TOKEN_INDICES)]); - auto outputHiddenSizeBatchStartsPerLevel - = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::HIDDEN_SIZE_BATCH_LEVEL_STARTS)]); - - auto eagleNet0SequenceLengths - = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::SEQUENCE_LENGTHS)]); - auto eagleNet0ContextLength = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::CONTEXT_LENGTHS)]); - auto nextDraftPaths = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)]); - auto nextDraftIds = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::NEXT_DRAFT_TOKENS)]); - auto inputHiddenSizeBatchStartsPerLevel - = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::HIDDEN_SIZE_BATCH_LEVEL_STARTS)]); - - int8_t* workspaceBytePtr = reinterpret_cast<int8_t*>(workspace); - size_t offset{0}; - - int8_t* isLeafMask = reinterpret_cast<int8_t*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(int8_t))); - TokenIdType* selectedDraftIndices = reinterpret_cast<TokenIdType*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(TokenIdType))); - SizeType32* selectedDraftPosOffsets = reinterpret_cast<SizeType32*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(SizeType32))); - SizeType32* numSelectedDraftIndices - = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(SizeType32))); - bool* selectedMasks = reinterpret_cast<bool*>(tc::nextWorkspacePtr( - workspaceBytePtr, offset, batchSize * maxDecodingTokens * maxDecodingTokens * sizeof(int8_t))); - SizeType32* cumSumGenerationLengths = reinterpret_cast<SizeType32*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, (batchSize + 1) * sizeof(SizeType32))); - SizeType32* nonLeavesInLevelOffsets = reinterpret_cast<SizeType32*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(SizeType32))); - SizeType32* parentNonLeafInLevelOffset = reinterpret_cast<SizeType32*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(SizeType32))); - SizeType32* maxGenerationLength - = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, 1 * sizeof(SizeType32))); - - cudaMemsetAsync(hiddenStatesIndices, 0, batchSize * mMaxNonLeavesPerLayer * mLayerIdx * sizeof(SizeType32), stream); - cudaMemsetAsync(selectedMasks, 0, batchSize * maxDecodingTokens * maxDecodingTokens * sizeof(int8_t), stream); - // Prefill mask setting all to leaves. - cudaMemsetAsync(isLeafMask, 1, batchSize * maxDecodingTokens * sizeof(int8_t), stream); - - PrepareGenEagleNetInputsParams params; - params.nextSequenceLengths = eagleNetSequenceLengths; - params.nextContextLengths = eagleNetContextLengths; - params.outputIds = outputIds; - params.positionIds = positionIds; - params.specDecodingGenLengths = specDecodingGenLengths; - params.specDecodingPositionOffsets = specDecodingPositionOffsets; - params.specDecodingPackedMasks = specDecodingPackedMasks; - params.hiddenStatesIndices = hiddenStatesIndices; - params.lastTokenIndices = lastTokenIndices; - params.numLastTokenIndices = numLastTokenIndices; - params.outputHiddenSizeBatchStartsPerLevel = outputHiddenSizeBatchStartsPerLevel; - - // tmp data - params.isLeafMask = isLeafMask; - params.selectedDraftIndices = selectedDraftIndices; - params.selectedDraftPosOffsets = selectedDraftPosOffsets; - params.numSelectedDraftIndices = numSelectedDraftIndices; - params.selectedMasks = selectedMasks; - params.cumSumGenerationLengths = cumSumGenerationLengths; - params.maxGenerationLength = maxGenerationLength; - params.nonLeavesInLevelOffsets = nonLeavesInLevelOffsets; - params.parentNonLeafInLevelOffset = parentNonLeafInLevelOffset; - - params.nextDraftIds = nextDraftIds; - params.eagleNet0SequenceLengths = eagleNet0SequenceLengths; - params.prevContextLengths = eagleNet0ContextLength; - params.nextPaths = nextDraftPaths; - params.inputHiddenSizeBatchStartsPerLevel = inputHiddenSizeBatchStartsPerLevel; - params.levelIdx = mLayerIdx; - params.batchSize = batchSize; - params.maxPathLen = maxPathLen; - params.maxDecodingTokens = maxDecodingTokens; - params.maxNonLeavesPerLayer = mMaxNonLeavesPerLayer; - params.stream = stream; - - params.checkParams(); - - invokePrepareGenEagleNetInputs(params); - - sync_check_cuda_error(stream); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -int EaglePrepareDrafterInputsPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // First EagleNet instance (EagleNet0) is always chunked context attn, - // where we process either context tokens or newly accepted tokens and append them to EagleNet KV cache. - - // For all following EagleNetX (X > 0) instances there is need for masked spec decoding attn. - // Ideally with mask for context. - // Let's say we have prompt ABCD and two variants of tokens spec decoding tokens E and F - // predicted by EagleNet0. If we draw full attn mask, it becomes: - // |A|B|C|D|E|F - // E|1|1|1|1|1|0 - // F|1|1|1|1|0|1 - // - // In the next step we predict token G from ABCDE branch and token H from ABCDF branch -- like beam search. - // And we'd need spec decoding mask that includes kv cache: - // |A|B|C|D|E|F|G|H - // G|1|1|1|1|1|0|1|0 - // H|1|1|1|1|0|1|0|1 - // - // But TRT-LLM does not support such mask for now. We can only provide - // |G|H - // G|1|0 - // H|0|1 - // , which is wrong mask. - // - // For now we WAR this by passing EFGH for the EagleNet1 with right mask - // and using only G and H logits for sampling, but that's redundant compute: - // |E|F|G|H - // E|1|0|0|0 - // F|0|1|0|0 - // G|1|0|1|0 - // H|0|1|0|1 - - if (mLayerIdx == 0) - { - prepareCtxEagleNetData(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else - { - prepareGenEagleNetData(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - - return 0; -} - -/////////////// - -EaglePrepareDrafterInputsPluginCreator::EaglePrepareDrafterInputsPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("layer_idx", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("num_layers", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("max_non_leaves_per_layer", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* EaglePrepareDrafterInputsPluginCreator::getPluginName() const noexcept -{ - return EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_NAME; -} - -char const* EaglePrepareDrafterInputsPluginCreator::getPluginVersion() const noexcept -{ - return EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_VERSION; -} - -PluginFieldCollection const* EaglePrepareDrafterInputsPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -nvinfer1::IPluginV3* EaglePrepareDrafterInputsPluginCreator::createPlugin( - char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept -{ - try - { - int32_t layerIdx{0}; - int32_t numLayers{0}; - int32_t maxNonLeavesPerLayer{0}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fc->fields[i].name; - if (!strcmp(attrName, "layer_idx")) - { - TLLM_CHECK(fc->fields[i].type == PluginFieldType::kINT32); - layerIdx = *static_cast<int32_t const*>(fc->fields[i].data); - } - else if (!strcmp(attrName, "num_layers")) - { - TLLM_CHECK(fc->fields[i].type == PluginFieldType::kINT32); - numLayers = *static_cast<int32_t const*>(fc->fields[i].data); - } - else if (!strcmp(attrName, "max_non_leaves_per_layer")) - { - TLLM_CHECK(fc->fields[i].type == PluginFieldType::kINT32); - maxNonLeavesPerLayer = *static_cast<int32_t const*>(fc->fields[i].data); - } - } - return new EaglePrepareDrafterInputsPlugin(layerIdx, numLayers, maxNonLeavesPerLayer); - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -char const* EaglePrepareDrafterInputsPluginCreator::getPluginNamespace() const noexcept -{ - return tensorrt_llm::plugins::api::kDefaultNamespace; -} diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.h b/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.h deleted file mode 100644 index 0059c46f6c8d..000000000000 --- a/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.h +++ /dev/null @@ -1,186 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class EaglePrepareDrafterInputsPlugin : public nvinfer1::IPluginV3, - public nvinfer1::IPluginV3OneCore, - public nvinfer1::IPluginV3OneBuild, - public nvinfer1::IPluginV3OneRuntime -{ -public: - EaglePrepareDrafterInputsPlugin(EaglePrepareDrafterInputsPlugin const& p) = default; - - EaglePrepareDrafterInputsPlugin(int32_t layerIdx, int32_t numLayers, int32_t maxNonLeavesPerLayer); - - nvinfer1::IPluginV3* clone() noexcept override; - - nvinfer1::IPluginCapability* getCapabilityInterface(nvinfer1::PluginCapabilityType type) noexcept override; - - void initFieldsToSerialize(); - - char const* getPluginName() const noexcept override; - char const* getPluginVersion() const noexcept override; - char const* getPluginNamespace() const noexcept override; - - int32_t getNbOutputs() const noexcept override; - - bool supportsFormatCombination( - int pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept override; - int32_t configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept override; - - int32_t getOutputDataTypes(nvinfer1::DataType* outputTypes, int32_t nbOutputs, nvinfer1::DataType const* inputTypes, - int32_t nbInputs) const noexcept override; - - int32_t getOutputShapes(nvinfer1::DimsExprs const* inputs, int32_t nbInputs, nvinfer1::DimsExprs const* shapeInputs, - int32_t nbShapeInputs, nvinfer1::DimsExprs* outputs, int32_t nbOutputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - - int32_t onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, nvinfer1::PluginTensorDesc const* out, - int32_t nbOutputs) noexcept override; - - nvinfer1::IPluginV3* attachToContext(nvinfer1::IPluginResourceContext* context) noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldsToSerialize() noexcept override; - - size_t getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - -private: - enum class InputIdxEntry : int32_t - { - //! [batch_size] - SEQUENCE_LENGTHS = 0, - //! [batch_size] - CONTEXT_LENGTHS, - //! [num_tokens] - INPUT_IDS, - //! [batch_size] - CHUNKED_CONTEXT_NEXT_TOKENS, - //! [batch_size, max_path_len] - ACCEPTED_TOKENS, - //! [batch_size] - ACCEPTED_LENS, - //! [batch_size] - ACCEPTED_PATHS, - //! [batch_size, max_decoding_draft_tokens] - NEXT_DRAFT_TOKENS, - //! [batch_size] - NEXT_DRAFT_LENS, - //! [batch_size, max_decoding_tokens, max_path_len] - NEXT_DRAFT_PATHS, - //! [batch_size] - PREV_DRAFT_LENS, - //! [batch_size, max_decoding_tokens, max_path_len] - PREV_DRAFT_PATHS, - //! [(max_path_len - 1) * batch_size + 1] - HIDDEN_SIZE_BATCH_LEVEL_STARTS, - //! [num_gen_tokens] - INPUT_GEN_TOKENS, - //! [num_gen_requests] - SPEC_DECODING_GENERATION_LENGTHS, - }; - - enum class OutputIdxEntry : int32_t - { - //! [batch_size] - SEQUENCE_LENGTHS = 0, - //! [batch_size] - CONTEXT_LENGTHS, - //! [batch_size] - SPEC_DECODING_GENERATION_LENGTHS, - //! [batch_size, max_decoding_tokens] - SPEC_DECODING_POSITION_OFFSETS, - //! [batchSize, maxDecodingTokens, ceil(maxDecodingTokens / 32)] - SPEC_DECODING_PACKED_MASK, - //! [batchSize * mMaxNonLeavesPerLayer * layerIdx] for layerIdx > 0 - //! [num_tokens - numGenTokens + numGenRequests * (mNumLayers + 1)] for layerIdx == 0 - OUTPUT_IDS, - //! [batchSize] for layerIdx > 0 - //! [num_tokens - numGenTokens + numGenRequests * (mNumLayers + 1)] for layerIdx == 0 - POSITION_IDS, - //! [batchSize * mMaxNonLeavesPerLayer * layerIdx] for layerIdx > 0 - //! [num_tokens - numGenTokens + numGenRequests * (mNumLayers + 1)] for layerIdx == 0 - HIDDEN_STATES_INDICES, - //! [batchSize * mMaxNonLeavesPerLayer] - LAST_TOKEN_INDICES, - //! [1] - NUM_LAST_TOKEN_INDICES, - //! [(max_path_len - 1) * batch_size + 1] - HIDDEN_SIZE_BATCH_LEVEL_STARTS, - }; - - int32_t getIdx(InputIdxEntry idx) const - { - return static_cast<int32_t>(idx); - } - - int32_t getIdx(OutputIdxEntry idx) const - { - return static_cast<int32_t>(idx); - } - -private: - void prepareCtxEagleNetData(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept; - - void prepareGenEagleNetData(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept; - -private: - int32_t mLayerIdx{0}; - int32_t mNumLayers{0}; - int32_t mMaxNonLeavesPerLayer{0}; - std::vector<nvinfer1::PluginField> mDataToSerialize; - nvinfer1::PluginFieldCollection mFCToSerialize; -}; - -class EaglePrepareDrafterInputsPluginCreator : public nvinfer1::IPluginCreatorV3One -{ -public: - EaglePrepareDrafterInputsPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - char const* getPluginNamespace() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV3* createPlugin( - char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.cpp b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.cpp deleted file mode 100644 index 5fb30f583712..000000000000 --- a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.cpp +++ /dev/null @@ -1,565 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "eagleSampleAndAcceptDraftTokensPlugin.h" - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/kernels/samplingTopKKernels.h" -#include "tensorrt_llm/kernels/speculativeDecoding/common.h" -#include "tensorrt_llm/kernels/speculativeDecoding/eagleDecodingKernels.h" -#include "tensorrt_llm/kernels/speculativeDecoding/medusaDecodingKernels.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iTensor.h" - -using namespace nvinfer1; -using tensorrt_llm::plugins::EagleSampleAndAcceptDraftTokensPluginCreator; -using tensorrt_llm::plugins::EagleSampleAndAcceptDraftTokensPlugin; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::kernels::speculative_decoding; -using namespace tensorrt_llm::runtime; -namespace tc = tensorrt_llm::common; - -static char const* EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_VERSION{"1"}; -static char const* EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_NAME{"EagleSampleAndAcceptDraftTokens"}; -PluginFieldCollection EagleSampleAndAcceptDraftTokensPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> EagleSampleAndAcceptDraftTokensPluginCreator::mPluginAttributes; - -EagleSampleAndAcceptDraftTokensPlugin::EagleSampleAndAcceptDraftTokensPlugin(nvinfer1::DataType type) - : mDtype(type) -{ -} - -// Parameterized constructor -EagleSampleAndAcceptDraftTokensPlugin::EagleSampleAndAcceptDraftTokensPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mDtype); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* EagleSampleAndAcceptDraftTokensPlugin::clone() const noexcept -{ - auto* plugin = new EagleSampleAndAcceptDraftTokensPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs EagleSampleAndAcceptDraftTokensPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - TLLM_CHECK(nbInputs == 10); - TLLM_CHECK(outputIndex < 7); - auto const batchSizeExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[0]; - auto const maxDecodingDraftTokensExpr = inputs[getIdx(InputIdxEntry::DRAFT_TOKEN_IDS)].d[1]; - auto const maxDecodingTokensExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[1]; - auto const maxPathLenExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[2]; - - nvinfer1::DimsExprs ret; - if (outputIndex == getIdx(OutputIdxEntry::ACCEPTED_TOKENS)) - { - ret.nbDims = 2; - ret.d[0] = batchSizeExpr; - ret.d[1] = maxPathLenExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::ACCEPTED_LENS)) - { - ret.nbDims = 1; - ret.d[0] = batchSizeExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::BEST_ACCEPTED_PATHS)) - { - ret.nbDims = 1; - ret.d[0] = batchSizeExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::NEXT_DRAFT_TOKEN_IDS)) - { - ret.nbDims = 2; - ret.d[0] = batchSizeExpr; - ret.d[1] = maxDecodingDraftTokensExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::NEXT_DRAFT_LENS)) - { - ret.nbDims = 1; - ret.d[0] = batchSizeExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::NEXT_DRAFT_PATHS)) - { - ret.nbDims = 3; - ret.d[0] = batchSizeExpr; - ret.d[1] = maxDecodingTokensExpr; - ret.d[2] = maxPathLenExpr; - } - else if (outputIndex == getIdx(OutputIdxEntry::HIDDEN_SIZE_BATCH_LEVEL_STARTS)) - { - ret.nbDims = 1; - ret.d[0] = exprBuilder.operation(DimensionOperation::kSUM, *exprBuilder.constant(1), - *exprBuilder.operation(DimensionOperation::kPROD, - *exprBuilder.operation(DimensionOperation::kSUB, *maxPathLenExpr, *exprBuilder.constant(1)), - *batchSizeExpr)); - } - return ret; -} - -bool EagleSampleAndAcceptDraftTokensPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - if (pos == getIdx(InputIdxEntry::LOGITS)) // logits - { - return (inOut[pos].type == mDtype) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == getIdx(InputIdxEntry::TEMPERATURE) || pos == getIdx(InputIdxEntry::RAND_VALIDATION) - || pos == getIdx(InputIdxEntry::POSTERIOR_ALPHA) - || pos == getIdx(InputIdxEntry::POSTERIOR_THRESHOLD)) // temperature, rand_validation - { - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else // everything else - { - return (inOut[pos].type == nvinfer1::DataType::kINT32) && (inOut[pos].format == TensorFormat::kLINEAR); - } -} - -void EagleSampleAndAcceptDraftTokensPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -template <typename T> -size_t EagleSampleAndAcceptDraftTokensPlugin::getWorkspaceSizeType(nvinfer1::PluginTensorDesc const* inputs, - int nbInputs, nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - size_t workspaceSize{0}; - - auto const vocabSizePadded = inputs[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; - auto const batchSize = inputs[getIdx(InputIdxEntry::PATHS)].dims.d[0]; - auto const maxDecodingTokens = inputs[getIdx(InputIdxEntry::PATHS)].dims.d[1]; - - // Greedy sampling - // Top1 sampling workspace - auto const greedySamplingWorkspaceSize - = getTopKWorkspaceSize<T>(batchSize, maxDecodingTokens, /* maxTopK */ 1, vocabSizePadded); - - // Multinomial sampling - auto const typicalSamplingWorkspaceSize - = getTypicalAcceptanceWorkspaceSize<T>(batchSize, maxDecodingTokens, vocabSizePadded); - - auto const primarySamplingWorkspaceSize = std::max(greedySamplingWorkspaceSize, typicalSamplingWorkspaceSize); - - // Target output ids - auto const targetOutputIdsSize = batchSize * maxDecodingTokens * sizeof(TokenIdType); - // Logits ptrs - auto const logitsPtrsSize = batchSize * maxDecodingTokens * sizeof(T*); - SizeType32 constexpr NUM_BUFFERS{4}; - size_t workspaces[NUM_BUFFERS]; - workspaces[0] = targetOutputIdsSize; - workspaces[1] = primarySamplingWorkspaceSize; - workspaces[2] = logitsPtrsSize; - workspaces[3] = batchSize * sizeof(SizeType32); - workspaceSize = tc::calculateTotalWorkspaceSize(workspaces, NUM_BUFFERS); - - return workspaceSize; -} - -size_t EagleSampleAndAcceptDraftTokensPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - auto const logitsType = inputs[getIdx(InputIdxEntry::LOGITS)].type; - if (logitsType == nvinfer1::DataType::kFLOAT) - { - return getWorkspaceSizeType<float>(inputs, nbInputs, outputs, nbOutputs); - } - else if (logitsType == nvinfer1::DataType::kHALF) - { - return getWorkspaceSizeType<__half>(inputs, nbInputs, outputs, nbOutputs); - } - else - { - TLLM_CHECK_WITH_INFO(false, "Unsupported logits type"); - } - return 0; -} - -template <typename T> -void EagleSampleAndAcceptDraftTokensPlugin::samplePrimeHeadTokens(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // auto const maxNumTokens = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[0]; - auto const vocabSizePadded = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; - auto const batchSize = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[0]; - auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[1]; - - auto logits = static_cast<T const*>(inputs[getIdx(InputIdxEntry::LOGITS)]); - auto prevDraftLens = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::DRAFT_LENS)]); - - int8_t* workspaceBytePtr = reinterpret_cast<int8_t*>(workspace); - size_t offset{0}; - - auto const samplingWorkspaceSize - = getTopKWorkspaceSize<T>(batchSize, maxDecodingTokens, /* maxTopK */ 1, vocabSizePadded); - - TokenIdType* outputIds = reinterpret_cast<TokenIdType*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(TokenIdType))); - void* workspaceSampling - = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, samplingWorkspaceSize)); - T const** logitsPtrs = reinterpret_cast<T const**>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(T*))); - SizeType32* decodingTokens - = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(SizeType32))); - - // Assemble pointers to logits - invokeAssembleTargetLogitsOffsets( - logitsPtrs, decodingTokens, logits, prevDraftLens, batchSize, maxDecodingTokens, vocabSizePadded, stream); - - sync_check_cuda_error(stream); - - TopKSamplingKernelParams<T> params; - params.logProbsPtrs = logitsPtrs; - params.outputIds = outputIds; - params.workspace = workspaceSampling; - params.maxTopK = 1; - params.batchSize = batchSize; - params.maxBatchSize = batchSize; - params.tokensPerStep = decodingTokens; - params.maxTokensPerStep = maxDecodingTokens; - params.maxSeqLen = maxDecodingTokens; - params.vocabSizePadded = vocabSizePadded; - - invokeBatchTopKSampling(params, stream); - - sync_check_cuda_error(stream); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -template <typename T> -void EagleSampleAndAcceptDraftTokensPlugin::doTypicalAcceptance(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // auto const maxNumTokens = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[0]; - auto const vocabSizePadded = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; - - auto const batchSize = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[0]; - auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[1]; - // auto const maxPathLen = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[2]; - // auto const maxDraftPathLen = maxPathLen - 1; - - auto logits = static_cast<T const*>(inputs[getIdx(InputIdxEntry::LOGITS)]); - auto prevDraftLens = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::DRAFT_LENS)]); - - int8_t* workspaceBytePtr = reinterpret_cast<int8_t*>(workspace); - size_t offset{0}; - - // Multinomial sampling - auto const primarySamplingWorkspaceSize - = getTypicalAcceptanceWorkspaceSize<T>(batchSize, maxDecodingTokens, vocabSizePadded); - - TokenIdType* outputIds = reinterpret_cast<TokenIdType*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(TokenIdType))); - void* workspaceSampling - = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, primarySamplingWorkspaceSize)); - T** logitsPtrs = reinterpret_cast<T**>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(T*))); - SizeType32* decodingTokens - = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(SizeType32))); - - // Assemble pointers to logits - invokeAssembleTargetLogitsOffsets(const_cast<T const**>(logitsPtrs), decodingTokens, logits, prevDraftLens, - batchSize, maxDecodingTokens, vocabSizePadded, stream); - - sync_check_cuda_error(stream); - - TypicalAcceptanceSampling<T> params; - params.logitsPtrs = logitsPtrs; - params.generationLengths = decodingTokens; - params.temperatures = reinterpret_cast<float const*>(inputs[getIdx(InputIdxEntry::TEMPERATURE)]); - params.posteriorThresholds = reinterpret_cast<float const*>(inputs[getIdx(InputIdxEntry::POSTERIOR_THRESHOLD)]); - params.posteriorAlphas = reinterpret_cast<float const*>(inputs[getIdx(InputIdxEntry::POSTERIOR_ALPHA)]); - params.outputIds = outputIds; - params.workspace = reinterpret_cast<int8_t*>(workspaceSampling); - params.randomVals = reinterpret_cast<float const*>(inputs[getIdx(InputIdxEntry::RAND_VALIDATION)]); - - params.batchSize = batchSize; - params.maxBatchSize = batchSize; - params.maxDecodingTokens = maxDecodingTokens; - params.vocabSize = vocabSizePadded; - - if (mSmCnt <= 0) - { - auto const deviceId = tensorrt_llm::common::getDevice(); - cudaDeviceProp prop{}; - TLLM_CUDA_CHECK(cudaGetDeviceProperties(&prop, deviceId)); - mSmCnt = prop.multiProcessorCount; - } - params.smCnt = mSmCnt; - - params.checkParams(); - - typicalAcceptanceSampling(params, stream); - - sync_check_cuda_error(stream); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -template <typename T> -void EagleSampleAndAcceptDraftTokensPlugin::acceptDraftTokens(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - // auto const maxNumTokens = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[0]; - auto const vocabSizePadded = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; - - auto const batchSize = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[0]; - auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[1]; - auto const maxPathLen = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[2]; - auto const maxDraftPathLen = maxPathLen - 1; - - auto const useDynamicTree = *(reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::USE_DYNAMIC_TREE)])); - - int8_t* workspaceBytePtr = reinterpret_cast<int8_t*>(workspace); - size_t offset{0}; - - // auto const samplingWorkspaceSize - // = getTopKWorkspaceSize<T>(batchSize, maxDecodingTokens, /* maxTopK */ 1, vocabSizePadded); - - TokenIdType* outputIds = reinterpret_cast<TokenIdType*>( - tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(TokenIdType))); - - AcceptDraftTokensByIdsWithPathsParams<T> params; - params.outputIds = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::ACCEPTED_TOKENS)]); - params.draftIds = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::DRAFT_TOKEN_IDS)]); - params.targetIds = outputIds; - params.acceptedLengths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::ACCEPTED_LENS)]); - params.paths = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::PATHS)]); - params.bestPathIds = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::BEST_ACCEPTED_PATHS)]); - params.batchSize = batchSize; - params.maxBatchSize = batchSize; - params.vocabSize = vocabSizePadded; - params.maxSeqLen = maxPathLen; - params.maxDraftPathLen = maxDraftPathLen; - params.maxDecodingTokens = maxDecodingTokens; - params.stream = stream; - - params.checkParams(); - - acceptDraftTokensByIdsWithPaths(params); - - if (useDynamicTree) - { - // For Eagle-2, after verification and acceptance, the original path becomes useless. - // All set to '-1' - cudaMemsetAsync(outputs[getIdx(OutputIdxEntry::NEXT_DRAFT_PATHS)], -1, - batchSize * maxDecodingTokens * maxPathLen * sizeof(SizeType32), stream); - } - else - { - // For Eagle-1 - // Copy input paths to the output - cudaMemcpyAsync(outputs[getIdx(OutputIdxEntry::NEXT_DRAFT_PATHS)], inputs[getIdx(InputIdxEntry::PATHS)], - batchSize * maxDecodingTokens * maxPathLen * sizeof(SizeType32), cudaMemcpyDeviceToDevice, stream); - } - - sync_check_cuda_error(stream); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -template <typename T> -void EagleSampleAndAcceptDraftTokensPlugin::enqueueType(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const greedySampling = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::GREEDY_SAMPLING)])[0]; - // TODO split batch into greedy and non-greedy and execute both paths - if (greedySampling) - { - // Sample all main head tokens with Top-1. - samplePrimeHeadTokens<T>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else - { - // Typical sampling for typical acceptance. - doTypicalAcceptance<T>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - - // Accept tokens based on token ids, write the best path and best token id. - acceptDraftTokens<T>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -int EagleSampleAndAcceptDraftTokensPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - auto const logitsType = inputDesc[getIdx(InputIdxEntry::LOGITS)].type; - if (logitsType == nvinfer1::DataType::kFLOAT) - { - enqueueType<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (logitsType == nvinfer1::DataType::kHALF) - { - enqueueType<__half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else - { - TLLM_CHECK_WITH_INFO(false, "Unsupported logits type"); - } - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType EagleSampleAndAcceptDraftTokensPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index < 7); - // input 1 is draft tokens now of int32 type. All outputs are int32_t as well. - return inputTypes[getIdx(InputIdxEntry::DRAFT_TOKEN_IDS)]; -} - -// IPluginV2 Methods - -char const* EagleSampleAndAcceptDraftTokensPlugin::getPluginType() const noexcept -{ - return EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_NAME; -} - -char const* EagleSampleAndAcceptDraftTokensPlugin::getPluginVersion() const noexcept -{ - return EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_VERSION; -} - -int EagleSampleAndAcceptDraftTokensPlugin::getNbOutputs() const noexcept -{ - return 7; -} - -int EagleSampleAndAcceptDraftTokensPlugin::initialize() noexcept -{ - return 0; -} - -void EagleSampleAndAcceptDraftTokensPlugin::terminate() noexcept {} - -size_t EagleSampleAndAcceptDraftTokensPlugin::getSerializationSize() const noexcept -{ - return sizeof(mDtype); -} - -void EagleSampleAndAcceptDraftTokensPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mDtype); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void EagleSampleAndAcceptDraftTokensPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -EagleSampleAndAcceptDraftTokensPluginCreator::EagleSampleAndAcceptDraftTokensPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* EagleSampleAndAcceptDraftTokensPluginCreator::getPluginName() const noexcept -{ - return EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_NAME; -} - -char const* EagleSampleAndAcceptDraftTokensPluginCreator::getPluginVersion() const noexcept -{ - return EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_VERSION; -} - -PluginFieldCollection const* EagleSampleAndAcceptDraftTokensPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* EagleSampleAndAcceptDraftTokensPluginCreator::createPlugin( - char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - - try - { - auto* obj = new EagleSampleAndAcceptDraftTokensPlugin(type); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* EagleSampleAndAcceptDraftTokensPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call EagleSampleAndAcceptDraftTokensPlugin::destroy() - try - { - auto* obj = new EagleSampleAndAcceptDraftTokensPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.h b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.h deleted file mode 100644 index 3b14bab83170..000000000000 --- a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.h +++ /dev/null @@ -1,167 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" - -#include <cassert> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class EagleSampleAndAcceptDraftTokensPlugin : public BasePlugin -{ -public: - EagleSampleAndAcceptDraftTokensPlugin(nvinfer1::DataType type); - - EagleSampleAndAcceptDraftTokensPlugin(void const* data, size_t length); - - ~EagleSampleAndAcceptDraftTokensPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - enum class InputIdxEntry : int32_t - { - //! [num_tokens, vocab_size_padded] - LOGITS = 0, - //! [batch_size, max_decoding_draft_tokens] - DRAFT_TOKEN_IDS, - //! [batch_size] - DRAFT_LENS, - //! [batch_size] - TEMPERATURE, - //! [batch_size, max_decoding_tokens] - RAND_VALIDATION, - //! [batch_size] - POSTERIOR_ALPHA, - //! [batch_size] - POSTERIOR_THRESHOLD, - //! [batch_size, max_decoding_tokens, max_path_len] - PATHS, - //! [1] - GREEDY_SAMPLING, - //! [1] - USE_DYNAMIC_TREE - }; - - enum class OutputIdxEntry : int32_t - { - //! [batch_size, max_path_len] - ACCEPTED_TOKENS = 0, - //! [batch_size] - ACCEPTED_LENS, - //! [batch_size] - BEST_ACCEPTED_PATHS, - //! [batch_size, max_decoding_draft_tokens] - NEXT_DRAFT_TOKEN_IDS, - //! [batch_size] - NEXT_DRAFT_LENS, - //! [batch_size, max_decoding_tokens, max_path_len] - NEXT_DRAFT_PATHS, - //! [max_draft_path_len * batch_size] - HIDDEN_SIZE_BATCH_LEVEL_STARTS, - }; - - int32_t getIdx(InputIdxEntry idx) const - { - return static_cast<int32_t>(idx); - } - - int32_t getIdx(OutputIdxEntry idx) const - { - return static_cast<int32_t>(idx); - } - -private: - template <typename T> - size_t getWorkspaceSizeType(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept; - - template <typename T> - void samplePrimeHeadTokens(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept; - - template <typename T> - void doTypicalAcceptance(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept; - - template <typename T> - void acceptDraftTokens(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept; - - template <typename T> - void enqueueType(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept; - -private: - nvinfer1::DataType mDtype; - int32_t mSmCnt{0}; -}; - -class EagleSampleAndAcceptDraftTokensPluginCreator : public BaseCreator -{ -public: - EagleSampleAndAcceptDraftTokensPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/exports.def b/cpp/tensorrt_llm/plugins/exports.def deleted file mode 100644 index 5d4ac9e3e793..000000000000 --- a/cpp/tensorrt_llm/plugins/exports.def +++ /dev/null @@ -1,19 +0,0 @@ -; SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -; SPDX-License-Identifier: Apache-2.0 -; -; Licensed under the Apache License, Version 2.0 (the "License"); -; you may not use this file except in compliance with the License. -; You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, software -; distributed under the License is distributed on an "AS IS" BASIS, -; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and -; limitations under the License. - -LIBRARY nvinfer_plugin_tensorrt_llm -EXPORTS -getPluginRegistry -initLibNvInferPlugins diff --git a/cpp/tensorrt_llm/plugins/exports.map b/cpp/tensorrt_llm/plugins/exports.map deleted file mode 100644 index c6c949775079..000000000000 --- a/cpp/tensorrt_llm/plugins/exports.map +++ /dev/null @@ -1,34 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* Hides all symbols except those specified in the global section */ -{ - global: - initTrtLlmPlugins; - setLoggerFinder; - getPluginCreators; - getCreators; - extern "C++" { - nvinfer1::IPluginCreator::*; - nvinfer1::IPluginV2Ext::*; - nvinfer1::IPluginV2IOExt::*; - nvinfer1::PluginRegistrar*; - tensorrt_llm::plugins::api::*; - tensorrt_llm::plugins::*; - }; - local: *; -}; diff --git a/cpp/tensorrt_llm/plugins/fp4GemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/fp4GemmPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/fp4GemmPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.cpp b/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.cpp deleted file mode 100644 index 05f06ae38feb..000000000000 --- a/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.cpp +++ /dev/null @@ -1,434 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <numeric> - -#include "fp4GemmPlugin.h" -#include "tensorrt_llm/common/assert.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::Fp4GemmPluginCreator; -using tensorrt_llm::plugins::Fp4GemmPlugin; -using tensorrt_llm::plugins::Fp4GemmPluginProfiler; -#if defined(USING_OSS_CUTLASS_FP4_GEMM) -using namespace tensorrt_llm::kernels::cutlass_kernels; -#else -using namespace tensorrt_llm::kernels::internal_cutlass_kernels; -#endif - -constexpr nvinfer1::DataType FP4_DTYPE = nvinfer1::DataType::kFP4; -constexpr nvinfer1::DataType FP8_DTYPE = nvinfer1::DataType::kFP8; - -static char const* FP4_GEMM_PLUGIN_VERSION{"1"}; -static char const* FP4_GEMM_PLUGIN_NAME{"Fp4Gemm"}; -PluginFieldCollection Fp4GemmPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> Fp4GemmPluginCreator::mPluginAttributes; - -void Fp4GemmPluginProfiler::runTactic( - int m, int n, int k, Fp4GemmPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) -{ - // Workspace size required by gemm runner - // NB: this function will throw exception when selected tactic exceeds SMEM, which is then - // caught by gemmPluginProfiler and it will register this tactic as invalid - size_t wsSizeRunner = mRunner->getWorkspaceSize(m, n, k, /* batch_count */ 1); - - // Workspace size required by profiling - size_t wsByteOffset = 0; - int8_t* wsBytePointer = reinterpret_cast<int8_t*>(workspace); - void* aTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, (m * k) / 2)); - void* bTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, (n * k) / 2)); - void* dTmp = reinterpret_cast<void*>( - nextWorkspacePtr(wsBytePointer, wsByteOffset, m * n * (mType == nvinfer1::DataType::kFLOAT ? 4u : 2u))); - // SF M/N is padded along 128 and K is padded along 4. - int vector_size = 16; - int sf_round_m = ((m + 127) / 128) * 128; - int sf_round_n = ((n + 127) / 128) * 128; - int sf_round_k = ((k / vector_size + 3) / 4) * 4; - float* a_sf = reinterpret_cast<float*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, sf_round_m * sf_round_k)); - float* b_sf = reinterpret_cast<float*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, sf_round_n * sf_round_k)); - float* global_sf = reinterpret_cast<float*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, sizeof(float))); - char* workspaceTmp = reinterpret_cast<char*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, wsSizeRunner)); - - // Run profiling - mRunner->gemm(dTmp, aTmp, bTmp, a_sf, b_sf, global_sf, m, n, k, /* batch_count */ 1, tactic, workspaceTmp, - wsSizeRunner, stream); - sync_check_cuda_error(stream); -} - -void Fp4GemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - size_t vector_size = 16; - size_t sf_round_m = ((maxM + 127) / 128) * 128; - size_t sf_round_n = ((n + 127) / 128) * 128; - size_t sf_round_k = ((k / vector_size + 3) / 4) * 4; - std::vector<size_t> workspaces = { - (size_t) (maxM * k / 2), // A - (size_t) (n * k / 2), // B - maxM * n * (mType == nvinfer1::DataType::kFLOAT ? 4u : 2u), // D - (size_t) (sf_round_m * sf_round_k), // A_SF - (size_t) (sf_round_n * sf_round_k), // B_SF - sizeof(float), // Global_SF - mRunner->getWorkspaceSize(maxM, n, k, /* batch_count */ 1) // workspace - }; - size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); - setTmpWorkspaceSizeInBytes(bytes); -} - -std::vector<Fp4GemmPluginProfiler::Config> Fp4GemmPluginProfiler::getTactics(int m, int n, int k) const -{ - return mRunner->getConfigs(); -} - -Fp4GemmPlugin::Fp4GemmPlugin( - int sfVecSize, nvinfer1::DataType OutputType, Fp4GemmPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) - , mSfVecSize(sfVecSize) - , mOutputType(OutputType) -{ - init(OutputType); -} - -Fp4GemmPlugin::Fp4GemmPlugin(void const* data, size_t length, Fp4GemmPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mSfVecSize); - read(d, mOutputType); - read(d, mDims); - - init(mOutputType); - mPluginProfiler->deserialize(d, mDims, mGemmId); - - TLLM_CHECK(d == a + length); -} - -void Fp4GemmPlugin::init(nvinfer1::DataType type) -{ - TLLM_CHECK_WITH_INFO((getSMVersion() >= 100), "FP4 Gemm not supported before Blackwell"); - TLLM_CHECK_WITH_INFO( - (mOutputType == DataType::kBF16) || (mOutputType == DataType::kFLOAT) || (mOutputType == DataType::kHALF), - "Only support float, half, bfloat16, got %d.", (int) mOutputType); - mOutputType = type; - if (mOutputType == nvinfer1::DataType::kHALF) - { - mGemmRunner = std::make_shared<CutlassFp4GemmRunner<half>>(); - } - else if (mOutputType == nvinfer1::DataType::kFLOAT) - { - mGemmRunner = std::make_shared<CutlassFp4GemmRunner<float>>(); - } -#ifdef ENABLE_BF16 - else if (mOutputType == nvinfer1::DataType::kBF16) - { - mGemmRunner = std::make_shared<CutlassFp4GemmRunner<__nv_bfloat16>>(); - } -#endif - - mGemmId = GemmIdCore(mDims.n, mDims.k, mOutputType); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* Fp4GemmPlugin::clone() const noexcept -{ - auto* plugin = new Fp4GemmPlugin(*this); - return plugin; -} - -nvinfer1::DimsExprs Fp4GemmPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - TLLM_CHECK_WITH_INFO(outputIndex == 0, "Only support one output"); - auto const& dimsInput = inputs[getInputTensorIdx()]; - auto const& dimsWeights = inputs[getWeightsTensorIdx()]; - TLLM_CHECK_WITH_INFO(dimsInput.nbDims >= 2 && dimsWeights.nbDims == 2, "Fp4GemmPlugin input dim=%d, weights dim=%d", - dimsInput.nbDims, dimsWeights.nbDims); - nvinfer1::DimsExprs ret; - if (outputIndex == 0) - { - ret.nbDims = dimsInput.nbDims; - for (int i = 0; i < dimsInput.nbDims - 1; ++i) - { - ret.d[i] = dimsInput.d[i]; - } - ret.d[dimsInput.nbDims - 1] = dimsWeights.d[0]; - } - else - { - TLLM_CHECK_WITH_INFO(outputIndex == 0, "output fp4 not supported now."); - ret.nbDims = 1; - auto vecCount = dimsInput.d[0]; - int numDim = dimsInput.nbDims; - for (int idx = 1; idx < numDim - 1; ++idx) - { - vecCount = exprBuilder.operation(nvinfer1::DimensionOperation::kPROD, *vecCount, *dimsInput.d[idx]); - } - auto constant128 = exprBuilder.constant(128); - auto alignedRowCount = exprBuilder.operation(nvinfer1::DimensionOperation::kCEIL_DIV, *vecCount, *constant128); - alignedRowCount = exprBuilder.operation(nvinfer1::DimensionOperation::kPROD, *alignedRowCount, *constant128); - auto constant4 = exprBuilder.constant(4); - auto constantSFSize = exprBuilder.constant(mSfVecSize); - auto sfColumn - = exprBuilder.operation(nvinfer1::DimensionOperation::kCEIL_DIV, *dimsInput.d[numDim - 1], *constantSFSize); - auto alignedColumnCount = exprBuilder.operation(nvinfer1::DimensionOperation::kCEIL_DIV, *sfColumn, *constant4); - alignedColumnCount - = exprBuilder.operation(nvinfer1::DimensionOperation::kPROD, *alignedColumnCount, *constant4); - auto totalSize - = exprBuilder.operation(nvinfer1::DimensionOperation::kPROD, *alignedColumnCount, *alignedRowCount); - ret.d[0] = totalSize; - } - return ret; -} - -bool Fp4GemmPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - if (inOut[pos].format != TensorFormat::kLINEAR) - { - return false; - } - if (pos == getInputTensorIdx()) - { - return (inOut[pos].type == FP4_DTYPE); - } - else if (pos == getWeightsTensorIdx()) - { - return (inOut[pos].type == FP4_DTYPE); - } - else if (pos == getInputSFTensorIdx() || pos == getWeightsSFTensorIdx()) - { - return (inOut[pos].type == FP8_DTYPE); - } - else if (pos == getGlobalSFTensorIdx()) - { - return (inOut[pos].type == DataType::kFLOAT); - } - else if (pos == nbInputs) - { - // Output - return (inOut[pos].type == DataType::kFLOAT || inOut[pos].type == DataType::kBF16 - || inOut[pos].type == DataType::kHALF); - } - return false; -} - -void Fp4GemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); - auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); - - int const maxK = in[0].max.d[in[0].max.nbDims - 1]; - int const maxN = in[2].max.d[0]; - int const minK = in[0].min.d[in[0].min.nbDims - 1]; - int const minN = in[2].min.d[0]; - - TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); - TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, maxN, maxK}; - } - mGemmId = {maxN, maxK, mOutputType}; - m_workspaceMaxSize = mGemmRunner->getWorkspaceSize(maxM, maxN, maxK, /* batch_count */ 1); -} - -size_t Fp4GemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return m_workspaceMaxSize; -} - -int Fp4GemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - // inputs - // 0. input_tensor [num_tokens, dim] - // 1. input_block_scale [num_tokens, dim / SFVecSize] (padded) - // 2. weights_tensor [out_dim, dim] - // 3. weights_block_scale [out_dim, dim / SFVecSize] (padded) - // 4. alpha (global scaling factor) [1] - // outputs - // 0. output_tensor [num_tokens, out_dim] - int64_t m = 1; - for (int i = 0; i < inputDesc[getInputTensorIdx()].dims.nbDims - 1; ++i) - { - m *= inputDesc[getInputTensorIdx()].dims.d[i]; - } - int const n = inputDesc[getWeightsTensorIdx()].dims.d[0]; - int const k = inputDesc[getWeightsTensorIdx()].dims.d[1]; - TLLM_CHECK_WITH_INFO(k % 32 == 0, "K dim should be aligned to 16 Bytes"); - int N_align = mOutputType == nvinfer1::DataType::kFLOAT ? 4u : 8u; - TLLM_CHECK_WITH_INFO(n % N_align == 0, "N dim should be aligned to 16 Bytes"); - size_t const wsSize = mGemmRunner->getWorkspaceSize(m, n, k, /* batch_count */ 1); - auto const bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); - TLLM_CHECK_WITH_INFO(bestTactic, "No valid FP4 GEMM tactic"); - if (m >= 1) - { - mGemmRunner->gemm(outputs[0], inputs[0], inputs[2], inputs[1], inputs[3], - reinterpret_cast<float const*>(inputs[4]), m, n, k, /* batch_count */ 1, *bestTactic, - reinterpret_cast<char*>(workspace), wsSize, stream); - } - sync_check_cuda_error(stream); - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType Fp4GemmPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK_WITH_INFO(index == 0, "Only support one output"); - return mOutputType; -} - -// IPluginV2 Methods - -char const* Fp4GemmPlugin::getPluginType() const noexcept -{ - return FP4_GEMM_PLUGIN_NAME; -} - -char const* Fp4GemmPlugin::getPluginVersion() const noexcept -{ - return FP4_GEMM_PLUGIN_VERSION; -} - -int Fp4GemmPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int Fp4GemmPlugin::initialize() noexcept -{ - configGemm(); - return 0; -} - -void Fp4GemmPlugin::terminate() noexcept {} - -size_t Fp4GemmPlugin::getSerializationSize() const noexcept -{ - return sizeof(mSfVecSize) + // mSfVecSize - sizeof(nvinfer1::DataType) + // dtype - sizeof(mDims) + // Dimensions - mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size -} - -void Fp4GemmPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mSfVecSize); - write(d, mOutputType); - write(d, mDims); - mPluginProfiler->serialize(d, mGemmId); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void Fp4GemmPlugin::destroy() noexcept -{ - delete this; -} - -void Fp4GemmPlugin::configGemm() -{ - mPluginProfiler->profileTactics(mGemmRunner, mOutputType, mDims, mGemmId); -} - -/////////////// - -Fp4GemmPluginCreator::Fp4GemmPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("sv_vec_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("output_type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* Fp4GemmPluginCreator::getPluginName() const noexcept -{ - return FP4_GEMM_PLUGIN_NAME; -} - -char const* Fp4GemmPluginCreator::getPluginVersion() const noexcept -{ - return FP4_GEMM_PLUGIN_VERSION; -} - -PluginFieldCollection const* Fp4GemmPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* Fp4GemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - TLLM_CHECK(fc->nbFields == 2); - int sf_vec_size{}; - nvinfer1::DataType output_type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "sf_vec_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - sf_vec_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "output_type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - output_type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - try - { - // Fp4GemmPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); - auto* obj = new Fp4GemmPlugin(sf_vec_size, output_type, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* Fp4GemmPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call CumsumLastDimPlugin::destroy() - try - { - // Create plugin profiler with private tactics map which is read from the serialized engine - auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); - auto* obj = new Fp4GemmPlugin(serialData, serialLength, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.h b/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.h deleted file mode 100644 index 9947e849d84f..000000000000 --- a/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.h +++ /dev/null @@ -1,162 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#if defined(USING_OSS_CUTLASS_FP4_GEMM) -#include "tensorrt_llm/kernels/cutlass_kernels/include/fp4_gemm.h" -#else -#include "fp4_gemm.h" -#endif - -#include <cassert> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -#if defined(USING_OSS_CUTLASS_FP4_GEMM) -using Fp4GemmRunnerPtr = std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFp4GemmRunnerInterface>; -#else -using Fp4GemmRunnerPtr - = std::shared_ptr<tensorrt_llm::kernels::internal_cutlass_kernels::CutlassFp4GemmRunnerInterface>; -#endif - -class Fp4GemmPluginProfiler : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - Fp4GemmRunnerPtr, GemmIdCore, GemmIdCoreHash> -{ -public: - using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; - -protected: - void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; - - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - - std::vector<Config> getTactics(int m, int n, int k) const override; - -private: - tensorrt_llm::common::QuantMode mQuantMode; -}; - -class Fp4GemmPlugin : public BasePlugin -{ -public: - using PluginProfilerPtr = std::shared_ptr<Fp4GemmPluginProfiler>; - - Fp4GemmPlugin() = delete; - - Fp4GemmPlugin(int sfVecSize, nvinfer1::DataType OutputType, PluginProfilerPtr const& pluginProfiler); - - Fp4GemmPlugin(void const* data, size_t length, PluginProfilerPtr const& pluginProfiler); - - ~Fp4GemmPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - using IndexType = std::int32_t; - - IndexType getInputTensorIdx() const - { - return 0; - }; - - IndexType getInputSFTensorIdx() const - { - return 1; - }; - - IndexType getWeightsTensorIdx() const - { - return 2; - }; - - IndexType getWeightsSFTensorIdx() const - { - return 3; - }; - - IndexType getGlobalSFTensorIdx() const - { - return 4; - } - - void init(nvinfer1::DataType type); - void configGemm(); - - Fp4GemmRunnerPtr mGemmRunner; - PluginProfilerPtr mPluginProfiler; - - int mSfVecSize; - nvinfer1::DataType mOutputType; - size_t m_workspaceMaxSize; - GemmDims mDims{}; - GemmIdCore mGemmId{}; -}; - -class Fp4GemmPluginCreator : public BaseCreator -{ -public: - Fp4GemmPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - GemmPluginProfilerManager<Fp4GemmPluginProfiler> mGemmPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/CMakeLists.txt deleted file mode 100644 index 3b714a3928fb..000000000000 --- a/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp *.cu) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.cpp b/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.cpp deleted file mode 100644 index 84963df50a21..000000000000 --- a/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.cpp +++ /dev/null @@ -1,422 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fp8RowwiseGemmPlugin.h" -#include "cutlass_extensions/gemm_configs.h" - -#include <NvInferRuntimeBase.h> -#include <numeric> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using namespace tensorrt_llm::kernels::cutlass_kernels; -using tensorrt_llm::plugins::Fp8RowwiseGemmPluginCreator; -using tensorrt_llm::plugins::Fp8RowwiseGemmPlugin; -using tensorrt_llm::plugins::Fp8RowwiseGemmPluginProfiler; - -static char const* FP8_ROWWISE_GEMM_PLUGIN_VERSION{"1"}; -static char const* FP8_ROWWISE_GEMM_PLUGIN_NAME{"Fp8RowwiseGemm"}; -PluginFieldCollection Fp8RowwiseGemmPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> Fp8RowwiseGemmPluginCreator::mPluginAttributes; - -size_t Fp8RowwiseGemmPluginProfiler::getBytePerElement(nvinfer1::DataType type) -{ - size_t bpe; - if (type == nvinfer1::DataType::kHALF || type == nvinfer1::DataType::kBF16) - { - bpe = 2; - } - else if (type == nvinfer1::DataType::kINT8 || type == nvinfer1::DataType::kFP8) - { - bpe = 1; - } - else - { - TLLM_THROW("Not recognized/implemented"); - } - return bpe; -} - -void Fp8RowwiseGemmPluginProfiler::setQuantMode(tensorrt_llm::common::QuantMode const& quantMode) -{ - mQuantMode = quantMode; -} - -void Fp8RowwiseGemmPluginProfiler::runTactic(int m, int n, int k, Fp8RowwiseGemmPluginProfiler::Config const& tactic, - char* workspace, cudaStream_t const& stream) -{ - size_t bpeIn = getBytePerElement(nvinfer1::DataType::kFP8); - size_t bpeOut = getBytePerElement(mType); - - // Workspace size required by gemm runner - // NB: this function will throw exception when selected tactic exceeds SMEM, which is then - // caught by gemmPluginProfiler and it will register this tactic as invalid - size_t wsSizeRunner = mRunner->getWorkspaceSize(m, n, k); - - // Workspace size required by profiling - size_t wsByteOffset = 0; - int8_t* wsBytePointer = reinterpret_cast<int8_t*>(workspace); - void* aTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, m * k * bpeIn)); - void* bTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, n * k * bpeIn)); - // void* cTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, n * bpeOut)); - void* dTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, m * n * bpeOut)); - float* scaleD0Tmp = reinterpret_cast<float*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, m * sizeof(float))); - float* scaleD1Tmp = reinterpret_cast<float*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, n * sizeof(float))); - char* workspaceTmp = reinterpret_cast<char*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, wsSizeRunner)); - - // Run profiling - mRunner->gemm(dTmp, aTmp, bTmp, nullptr, mQuantMode, m, n, k, scaleD0Tmp, scaleD1Tmp, tactic, workspaceTmp, - wsSizeRunner, stream); - sync_check_cuda_error(stream); -} - -int Fp8RowwiseGemmPluginProfiler::getMaxProfileM() const -{ - // Max_num_tokens are not suggested to be set larger than 16k. - return 16384; -} - -void Fp8RowwiseGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - std::vector<size_t> workspaces = { - maxM * k * getBytePerElement(nvinfer1::DataType::kFP8), // A - n * k * getBytePerElement(nvinfer1::DataType::kFP8), // B - // n * getBytePerElement(mType), // C_bias - maxM * n * getBytePerElement(mType), // D - maxM * sizeof(float), // alphaRow - n * sizeof(float), // alphaCol - maxM * sizeof(float), // alphaOutput - mRunner->getWorkspaceSize(maxM, n, k) // workspace - }; - size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); - setTmpWorkspaceSizeInBytes(bytes); -} - -std::vector<Fp8RowwiseGemmPluginProfiler::Config> Fp8RowwiseGemmPluginProfiler::getTactics(int m, int n, int k) const -{ - return mRunner->getConfigs(); -} - -Fp8RowwiseGemmPlugin::Fp8RowwiseGemmPlugin( - QuantMode quantMode, nvinfer1::DataType type, Fp8RowwiseGemmPlugin::PluginProfilerPtr const& pluginProfiler) - : mQuantMode(quantMode) - , mPluginProfiler(pluginProfiler) -{ - init(type); -} - -// Parameterized constructor -Fp8RowwiseGemmPlugin::Fp8RowwiseGemmPlugin( - void const* data, size_t length, Fp8RowwiseGemmPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - nvinfer1::DataType type; - unsigned int quantMode; - read(d, quantMode); - read(d, type); - read(d, mDims); - - mQuantMode = QuantMode(quantMode); - - init(type); - - mPluginProfiler->deserialize(d, mDims, mGemmId); - - TLLM_CHECK(d == a + length); -} - -void Fp8RowwiseGemmPlugin::init(nvinfer1::DataType type) -{ - mType = type; - if (mType == nvinfer1::DataType::kHALF) - { - mGemmRunner = std::make_shared<CutlassFp8RowwiseGemmRunner<half>>(); - } -#ifdef ENABLE_BF16 - else if (mType == nvinfer1::DataType::kBF16) - { - mGemmRunner = std::make_shared<CutlassFp8RowwiseGemmRunner<__nv_bfloat16>>(); - } -#endif - else - { - TLLM_THROW("Fp8 Rowwise Gemm plugin doesn't support this type now"); - } - - mPluginProfiler->setQuantMode(mQuantMode); - - mGemmId = GemmIdCore(mDims.n, mDims.k, mType); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* Fp8RowwiseGemmPlugin::clone() const noexcept -{ - auto* plugin = new Fp8RowwiseGemmPlugin(*this); - return plugin; -} - -nvinfer1::DimsExprs Fp8RowwiseGemmPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs == 4); - TLLM_CHECK(outputIndex == 0); - int const nbDimsA = inputs[0].nbDims; - TLLM_CHECK(nbDimsA >= 2); - DimsExprs ret; - ret.nbDims = nbDimsA; - for (int ii = 0; ii < nbDimsA - 1; ++ii) - { - ret.d[ii] = inputs[0].d[ii]; - } - ret.d[nbDimsA - 1] = inputs[1].d[0]; - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool Fp8RowwiseGemmPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - switch (pos) - { - case 0: - // activation - return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; - case 1: - // weights - // Weights stored in checkpoint must have fp8 type - return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; - case 2: - // scales channels - case 3: - // scales tokens - return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; - case 4: - // out - return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - default: - // All other format combinations are unsupported. - return false; - } -} - -void Fp8RowwiseGemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); - auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); - - int const maxK = in[0].max.d[in[0].max.nbDims - 1]; - int const maxN = in[1].max.d[0]; - int const minK = in[0].min.d[in[0].min.nbDims - 1]; - int const minN = in[1].min.d[0]; - - TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); - TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, maxN, maxK}; - } - mGemmId = {maxN, maxK, mType}; - - mWorkspaceMaxSize = mGemmRunner->getWorkspaceSize(maxM, maxN, maxK); -} - -size_t Fp8RowwiseGemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return mWorkspaceMaxSize; -} - -int Fp8RowwiseGemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // mat1 [M(*), K] - // mat2 [N, K] - // scale_tokens [M, 1] if has_per_token_scaling else [1, 1] - // scale_channels [1, N] if has_per_channel_scaling else [1, 1] - // outputs - // mat [M(*), N] - int m = 1; - for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) - { - m *= inputDesc[0].dims.d[ii]; - } - int const n = inputDesc[1].dims.d[0]; - int const k = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; - size_t const wsSize = mGemmRunner->getWorkspaceSize(m, n, k); - - auto const bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); - TLLM_CHECK_WITH_INFO(bestTactic, "No valid GEMM tactic"); - mGemmRunner->gemm(outputs[0], inputs[0], inputs[1], nullptr, mQuantMode, m, n, k, - reinterpret_cast<float const*>(inputs[2]), reinterpret_cast<float const*>(inputs[3]), *bestTactic, - reinterpret_cast<char*>(workspace), wsSize, stream); - sync_check_cuda_error(stream); - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType Fp8RowwiseGemmPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* Fp8RowwiseGemmPlugin::getPluginType() const noexcept -{ - return FP8_ROWWISE_GEMM_PLUGIN_NAME; -} - -char const* Fp8RowwiseGemmPlugin::getPluginVersion() const noexcept -{ - return FP8_ROWWISE_GEMM_PLUGIN_VERSION; -} - -int Fp8RowwiseGemmPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int Fp8RowwiseGemmPlugin::initialize() noexcept -{ - configGemm(); // gemm profiler in action - return 0; -} - -void Fp8RowwiseGemmPlugin::terminate() noexcept {} - -size_t Fp8RowwiseGemmPlugin::getSerializationSize() const noexcept -{ - return sizeof(unsigned int) + // QuantMode - sizeof(nvinfer1::DataType) + // dtype - sizeof(mDims) + // Dimensions - mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size -} - -void Fp8RowwiseGemmPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mQuantMode.value()); - write(d, mType); - write(d, mDims); - - mPluginProfiler->serialize(d, mGemmId); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void Fp8RowwiseGemmPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -void Fp8RowwiseGemmPlugin::configGemm() -{ - mPluginProfiler->profileTactics(mGemmRunner, mType, mDims, mGemmId); -} - -Fp8RowwiseGemmPluginCreator::Fp8RowwiseGemmPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("has_per_channel_scaling", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("has_per_token_scaling", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* Fp8RowwiseGemmPluginCreator::getPluginName() const noexcept -{ - return FP8_ROWWISE_GEMM_PLUGIN_NAME; -} - -char const* Fp8RowwiseGemmPluginCreator::getPluginVersion() const noexcept -{ - return FP8_ROWWISE_GEMM_PLUGIN_VERSION; -} - -PluginFieldCollection const* Fp8RowwiseGemmPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* Fp8RowwiseGemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - TLLM_CHECK(fc->nbFields == 3); - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - try - { - // Fp8RowwiseGemmPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); - QuantMode quantMode = QuantMode{}; - auto* obj = new Fp8RowwiseGemmPlugin(quantMode, type, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* Fp8RowwiseGemmPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call Fp8RowwiseGemmPlugin::destroy() - try - { - // Create plugin profiler with private tactics map which is read from the serialized engine - auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); - auto* obj = new Fp8RowwiseGemmPlugin(serialData, serialLength, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.h b/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.h deleted file mode 100644 index 36f22ad5885d..000000000000 --- a/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm.h" -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -using Fp8RowwiseGemmRunnerPtr - = std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFp8RowwiseGemmRunnerInterface>; - -class Fp8RowwiseGemmPluginProfiler : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - Fp8RowwiseGemmRunnerPtr, GemmIdCore, GemmIdCoreHash> - -{ -public: - using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; - - void setQuantMode(tensorrt_llm::common::QuantMode const& quantMode); - - virtual int getMaxProfileM() const override; - -protected: - void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; - - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - - std::vector<Config> getTactics(int m, int n, int k) const override; - -private: - size_t getBytePerElement(nvinfer1::DataType type); - - tensorrt_llm::common::QuantMode mQuantMode; -}; - -class Fp8RowwiseGemmPlugin : public BasePlugin -{ -public: - using PluginProfilerPtr = std::shared_ptr<Fp8RowwiseGemmPluginProfiler>; - - Fp8RowwiseGemmPlugin() = delete; - - Fp8RowwiseGemmPlugin( - tensorrt_llm::common::QuantMode quantMode, nvinfer1::DataType type, PluginProfilerPtr const& pluginProfiler); - - Fp8RowwiseGemmPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); - - ~Fp8RowwiseGemmPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - void init(nvinfer1::DataType type); - - void configGemm(); - -private: - const std::string mLayerName; - - Fp8RowwiseGemmRunnerPtr mGemmRunner; - tensorrt_llm::common::QuantMode mQuantMode; // not configurable yet - size_t mWorkspaceMaxSize; - - GemmDims mDims{}; - GemmIdCore mGemmId{}; - - PluginProfilerPtr mPluginProfiler; - - nvinfer1::DataType mType; -}; - -class Fp8RowwiseGemmPluginCreator : public BaseCreator -{ -public: - Fp8RowwiseGemmPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - GemmPluginProfilerManager<Fp8RowwiseGemmPluginProfiler> mGemmPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/CMakeLists.txt deleted file mode 100755 index 7cc985b60b7a..000000000000 --- a/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.cpp b/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.cpp deleted file mode 100644 index 541afdadc4c8..000000000000 --- a/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.cpp +++ /dev/null @@ -1,388 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "fusedLayernormPlugin.h" -#include "pluginUtils.h" -#include "tensorrt_llm/common/assert.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::FusedLayernormPluginCreator; -using tensorrt_llm::plugins::FusedLayernormPlugin; - -static char const* FUSED_LAYERNORM_PLUGIN_VERSION{"1"}; -static char const* FUSED_LAYERNORM_PLUGIN_NAME{"FusedLayernorm"}; -PluginFieldCollection FusedLayernormPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> FusedLayernormPluginCreator::mPluginAttributes; - -FusedLayernormPlugin::FusedLayernormPlugin(float eps, bool needFP32Output, bool needQuantize, nvinfer1::DataType type) - : mEps(eps) - , mNeedFP32Output(needFP32Output) - , mNeedQuantize(needQuantize) - , mType(type) -{ -} - -// Parameterized constructor -FusedLayernormPlugin::FusedLayernormPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mEps); - read(d, mNeedFP32Output); - read(d, mNeedQuantize); - read(d, mType); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* FusedLayernormPlugin::clone() const noexcept -{ - auto* plugin = new FusedLayernormPlugin(mEps, mNeedFP32Output, mNeedQuantize, mType); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs FusedLayernormPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - // Dim should be the same as input hidden states - if (!mNeedQuantize) - { - return inputs[0]; - } - - if (outputIndex == 1) // un-normed output fp16 - { - return inputs[0]; - } - if (outputIndex == 0) // quantized normed output - { - // Quantized output with int64_t data type (16 FP4 values per element). - DimsExprs ret; - ret.nbDims = inputs[0].nbDims; - for (int di = 0; di < ret.nbDims; ++di) - { - ret.d[di] = inputs[0].d[di]; - } - return ret; - } - - // Scaling Factors. - try - { - TLLM_CHECK(outputIndex == 2); - - DimsExprs ret; - ret.nbDims = inputs[0].nbDims; - for (int di = 0; di < ret.nbDims; ++di) - { - ret.d[di] = inputs[0].d[di]; - } - // Sequence dimension or token dimension. - // Pad to multiple of 128. - auto dimM - = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 2], *exprBuilder.constant(128)); - ret.d[ret.nbDims - 2] = exprBuilder.operation(DimensionOperation::kPROD, *dimM, *exprBuilder.constant(128)); - // Hidden size dimension. - ret.d[ret.nbDims - 1] - = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 1], *exprBuilder.constant(16)); - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool FusedLayernormPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - int const totalPoses = 5 + 2 * static_cast<int>(mNeedQuantize); - TLLM_CHECK(0 <= pos && pos < totalPoses); - TLLM_CHECK(nbInputs == 3 + static_cast<int>(mNeedQuantize)); - if (pos < nbInputs) - { - switch (pos) - { - case 0: - case 1: - case 2: return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - case 3: return (inOut[pos].type == nvinfer1::DataType::kFLOAT); - } - } - if (pos == nbInputs) // Normed output - { - if (mNeedQuantize) - { - // fp4 quantized output -- fp4 padded tp int64 - return (inOut[pos].type == nvinfer1::DataType::kFP4) && (inOut[pos].format == TensorFormat::kLINEAR); - } - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == nbInputs + 1) // Un-normed output - { - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } - // fp4 act_per_block_scale -- fp8 padded to int32 - return (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); -} - -void FusedLayernormPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t FusedLayernormPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return sizeof(WarpSpecializedCounters); -} - -int FusedLayernormPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // input [M(*), N] - // residual [M(*), N] - // weight [N, ] - // scale [1, ] - if needQuantize - // outputs - // output [M(*), N] - fp4 padded to int64 / fp16 - // un-normed output [M(*), N] - fp16 - // act_per_block_scale - fp8 padded to int32 - if needQuantize - -#define SETUP_PARAM \ - Param param; \ - int64_t m64 = 1; \ - for (int i = 0; i < inputDesc[0].dims.nbDims - 1; ++i) \ - { \ - m64 *= inputDesc[0].dims.d[i]; \ - } \ - int const m = TLLM_INT32_CAST(m64); \ - int const n = TLLM_INT32_CAST(inputDesc[2].dims.d[0]); \ - param.m = m; \ - param.n = n; \ - param.layernorm_eps = mEps; \ - param.input = const_cast<Input*>(reinterpret_cast<Input const*>(inputs[0])); \ - param.residual = const_cast<Input*>(reinterpret_cast<Input const*>(inputs[1])); \ - param.gamma = const_cast<Input*>(reinterpret_cast<Input const*>(inputs[2])); \ - if (mNeedQuantize) \ - { \ - param.sf_scale = const_cast<float*>(reinterpret_cast<float const*>(inputs[3])); \ - } \ - param.counters = reinterpret_cast<WarpSpecializedCounters*>(workspace); \ - param.stream = stream; \ - param.normed_output = reinterpret_cast<uint32_t*>(outputs[0]); \ - param.output = reinterpret_cast<Input*>(outputs[1]); \ - param.sf_out = reinterpret_cast<uint32_t*>(outputs[2]); - -#define CLEANUP_AND_INVOKE \ - TLLM_CUDA_CHECK(cudaMemsetAsync(workspace, 0, sizeof(WarpSpecializedCounters), stream)); \ - invokeWSLayerNorm(param, true, num_sms); - - int num_sms = tensorrt_llm::common::getMultiProcessorCount(); - - if (mType == DataType::kHALF) - { - using Input = half; - using Param = WarpSpecializedParam<GeneralFP4AddBiasResidualPreLayerNormParam<Input>>; - SETUP_PARAM - CLEANUP_AND_INVOKE - } -#ifdef ENABLE_BF16 - else if (mType == DataType::kBF16) - { - using Input = __nv_bfloat16; - using Param = WarpSpecializedParam<GeneralFP4AddBiasResidualPreLayerNormParam<Input>>; - SETUP_PARAM - CLEANUP_AND_INVOKE - } -#endif - else - { - TLLM_LOG_ERROR("Unsupported data type"); - return 1; - } - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType FusedLayernormPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - // assert((mNeedFP32Output && index < 3) || (!mNeedFP32Output && index < 2)); - assert((mNeedQuantize && index < 3) || (!mNeedQuantize && index < 2)); - if (index == 0) - { - // Output 0 quantized output of layernorm - fp4 padded to int64 - if (mNeedQuantize) - { - return nvinfer1::DataType::kFP4; - } - return mType; - } - else if (index == 1) - { - // Output 1 un-normed output - return mType; - } - // Output 2 act_per_block_scale - fp8 padded to int32 - return nvinfer1::DataType::kFP8; -} - -// IPluginV2 Methods - -char const* FusedLayernormPlugin::getPluginType() const noexcept -{ - return FUSED_LAYERNORM_PLUGIN_NAME; -} - -char const* FusedLayernormPlugin::getPluginVersion() const noexcept -{ - return FUSED_LAYERNORM_PLUGIN_VERSION; -} - -int FusedLayernormPlugin::getNbOutputs() const noexcept -{ - return 2 + static_cast<int>(mNeedQuantize); -} - -int FusedLayernormPlugin::initialize() noexcept -{ - return 0; -} - -void FusedLayernormPlugin::terminate() noexcept {} - -size_t FusedLayernormPlugin::getSerializationSize() const noexcept -{ - return sizeof(mEps) + sizeof(mNeedFP32Output) + sizeof(mNeedQuantize) + sizeof(mType); -} - -void FusedLayernormPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mEps); - write(d, mNeedFP32Output); - write(d, mNeedQuantize); - write(d, mType); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void FusedLayernormPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -FusedLayernormPluginCreator::FusedLayernormPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("eps", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("need_fp32_output", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("need_quantize", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* FusedLayernormPluginCreator::getPluginName() const noexcept -{ - return FUSED_LAYERNORM_PLUGIN_NAME; -} - -char const* FusedLayernormPluginCreator::getPluginVersion() const noexcept -{ - return FUSED_LAYERNORM_PLUGIN_VERSION; -} - -PluginFieldCollection const* FusedLayernormPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* FusedLayernormPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - float eps{}; - nvinfer1::DataType type{}; - bool needFP32Output{}; - bool needQuantize{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "eps")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - eps = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "need_fp32_output")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - needFP32Output = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "need_quantize")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - needQuantize = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - } - try - { - auto* obj = new FusedLayernormPlugin(eps, needFP32Output, needQuantize, type); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* FusedLayernormPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call FusedLayernormPlugin::destroy() - try - { - auto* obj = new FusedLayernormPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.h b/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.h deleted file mode 100755 index c6c899950fdc..000000000000 --- a/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.h +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/kernels/fusedLayernormKernels/layernorm_param.h" -#include "tensorrt_llm/kernels/fusedLayernormKernels/ws_layernorm.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class FusedLayernormPlugin : public BasePlugin -{ -public: - FusedLayernormPlugin() = delete; - - FusedLayernormPlugin(float eps, bool needFP32Output, bool needQuantize, nvinfer1::DataType type); - - FusedLayernormPlugin(void const* data, size_t length); - - ~FusedLayernormPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - float mEps; - bool mNeedFP32Output; - bool mNeedQuantize; - nvinfer1::DataType mType; - - const std::string mLayerName; -}; - -class FusedLayernormPluginCreator : public BaseCreator -{ -public: - FusedLayernormPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/CMakeLists.txt deleted file mode 100644 index 1d1fa98f4132..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.cpp b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.cpp deleted file mode 100644 index 08ee2af55406..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.cpp +++ /dev/null @@ -1,721 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gemmAllReducePlugin.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h" -#include "tensorrt_llm/plugins/common/pluginUtils.h" - -#include <unistd.h> - -static char const* GEMM_ALLREDUCE_PLUGIN_VERSION = "1"; -static char const* GEMM_ALLREDUCE_PLUGIN_NAME = "GemmAllReduce"; -template <nvinfer1::DataType T> -using CutlassType = ::tensorrt_llm::kernels::cutlass_kernels::CutlassType<T>; - -namespace tensorrt_llm::plugins -{ -template <typename K, typename V, DataType ElementA, DataType ElementB, DataType ElementD> -static std::pair<K, V> makeEntry() -{ - return {std::make_tuple(ElementA, ElementB, ElementD), - [&]() - { - using GemmTraits - = cutlass_kernels::GemmTypes<typename CutlassType<ElementA>::type, typename CutlassType<ElementB>::type, - typename CutlassType<ElementD>::type, // C, unused - typename CutlassType<ElementD>::type, - std::conditional_t<ElementA == DataType::kFP4, cutlass::float_ue4m3_t, void>, // SFA - std::conditional_t<ElementB == DataType::kFP4, cutlass::float_ue4m3_t, void>, // SFB - cutlass::layout::RowMajor, cutlass::layout::ColumnMajor, - cutlass::layout::RowMajor, // C, unused - cutlass::layout::RowMajor>; - return new cutlass_kernels::GemmAllReduceImplRunner<GemmTraits>(); - }}; -} - -template <typename K, typename V> -static std::map<K, V> getTypedInstantiators() -{ - return std::map<K, V>({makeEntry<K, V, DataType::kHALF, DataType::kHALF, DataType::kHALF>(), - makeEntry<K, V, DataType::kBF16, DataType::kBF16, DataType::kBF16>(), - makeEntry<K, V, DataType::kFP8, DataType::kFP8, DataType::kHALF>(), - makeEntry<K, V, DataType::kFP8, DataType::kFP8, DataType::kBF16>(), - makeEntry<K, V, DataType::kFP4, DataType::kFP4, DataType::kHALF>(), - makeEntry<K, V, DataType::kFP4, DataType::kFP4, DataType::kBF16>()}); -} - -//////////////////////////////////////////////////////////// -// GemmAllReducePlugin Methods -//////////////////////////////////////////////////////////// -GemmAllReducePlugin::GemmAllReducePlugin(GemmAllReducePluginOptions const& options) - : mOptions(options) - , mGemmId(GemmIdCore(options.maxProblemShape.n, options.maxProblemShape.k, options.typeD)) - , mProfiler(mGemmPluginProfileManager.createGemmPluginProfiler(/*inference=*/options.deserialize)) -{ - // construct mapping of input/output pos to argument - int argIdx = 0; - // inputs - mArgMap[argIdx++] = TensorArg::IN_ACTIVATION; - mArgMap[argIdx++] = TensorArg::IN_WEIGHT; - if (mOptions.hasSFA) - { - mArgMap[argIdx++] = TensorArg::IN_ACTIVATION_SF; - } - if (mOptions.hasSFB) - { - mArgMap[argIdx++] = TensorArg::IN_WEIGHT_SF; - } - if (mOptions.alphaIsPtr) - { - mArgMap[argIdx++] = TensorArg::IN_ALPHA; - } - mNbInputs = argIdx; - // outputs - mArgMap[argIdx++] = TensorArg::OUT_D_UC; - mArgMap[argIdx++] = TensorArg::OUT_D_MC; - mArgMap[argIdx++] = TensorArg::OUT_D_IPC; - mNbOutputs = argIdx - mNbInputs; - - // Create mapping of argument to tensor pos - for (auto const& pair : mArgMap) - { - mArgInvMap[pair.second] = pair.first; - } - - // Use map instead of huge switch case - mTypedInstantiators = getTypedInstantiators<KeyType, ValueType>(); - - auto key = std::make_tuple(mOptions.typeA, mOptions.typeB, mOptions.typeD); - - TLLM_CHECK_WITH_INFO(mTypedInstantiators.count(key) > 0, "No cutlass gemm for impl."); - mGemm = std::shared_ptr<cutlass_kernels::GemmAllReduceImplInterface>(mTypedInstantiators[key]()); -} - -void GemmAllReducePlugin::allocatePersistentWorkspace() -{ - TLLM_CHECK(mOptions.maxProblemShape.isInitialized()); - - mWorkspaceKey = "gemm_allreduce_workspace_m" + std::to_string(mOptions.maxProblemShape.maxM); - - cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig smallest_tile_config - = mGemm->getSupportedLaunchConfigs()[0]; - cutlass_kernels::GemmAllReduceImplInterface::ProblemArgs args; - args.argProblemShape(mOptions.maxProblemShape.maxM, mOptions.maxProblemShape.n, mOptions.maxProblemShape.k, 1) - .argRanks(mRank, mOptions.group) - .argLaunchConfig(smallest_tile_config); - - TLLM_CHECK(mWorkspace == nullptr); - - // Wrap persistent workspace in IPluginResource type - // so that clone() can be called to allocate memory - GemmAllReducePersistentWorkspace unallocated_resource(mGemm->getPersistentWorkspace(args)); - - // Register and allocate workspace - mWorkspace = static_cast<GemmAllReducePersistentWorkspace*>( - getPluginRegistry()->acquirePluginResource(mWorkspaceKey.c_str(), &unallocated_resource)); - TLLM_CHECK(mWorkspace != nullptr); -} - -LaunchConfig GemmAllReducePlugin::getStaticHeuristicLaunchConfig(int M) const -{ - using namespace tensorrt_llm::cutlass_extensions; - // This is only applicable when we swap and transpose A & B. - // When M is small we want to select tile that best fits it to maximize MMA efficiency. - auto filterByM = [&](std::vector<LaunchConfig> candidateConfigs) - { - std::vector<LaunchConfig> result; - if (M <= 16) - { - std::copy_if(candidateConfigs.begin(), candidateConfigs.end(), std::back_inserter(result), - [](const LaunchConfig& config) - { return config.tile_shape == TileShape::TileShape_128x16x128 and config.transposed; }); - } - else if (M <= 32) - { - std::copy_if(candidateConfigs.begin(), candidateConfigs.end(), std::back_inserter(result), - [](const LaunchConfig& config) - { return config.tile_shape == TileShape::TileShape_128x32x128 and config.transposed; }); - } - else if (M <= 64) - { - std::copy_if(candidateConfigs.begin(), candidateConfigs.end(), std::back_inserter(result), - [](const LaunchConfig& config) - { return config.tile_shape == TileShape::TileShape_128x64x128 and config.transposed; }); - } - else - { - std::copy_if(candidateConfigs.begin(), candidateConfigs.end(), std::back_inserter(result), - [](const LaunchConfig& config) - { return config.tile_shape == TileShape::TileShape_128x128x128 and config.transposed; }); - } - // If result empty then use any. - if (result.empty()) - { - result = candidateConfigs; - } - return result; - }; - - auto bestLaunchConfigs = mGemm->getSupportedLaunchConfigs(); - bestLaunchConfigs = filterByM(bestLaunchConfigs); - TLLM_CHECK(!bestLaunchConfigs.empty()); - // Return first one, because who knows which is best. - return bestLaunchConfigs.front(); -} - -static GemmAllReducePluginOptions deserializeOptions(void const*& data, size_t length) -{ - char const* begin = reinterpret_cast<char const*>(data); - char const*& end = reinterpret_cast<char const*&>(data); - GemmAllReducePluginOptions options; - options.deserialize = true; - - read(end, options.typeA); - read(end, options.typeB); - read(end, options.typeD); - read(end, options.transA); - read(end, options.transB); - read(end, options.alpha); - read(end, options.maxProblemShape); - read(end, options.groupSize); - for (int i = 0; i < options.groupSize; ++i) - { - int rank = -1; - read(end, rank); - options.group.insert(rank); - } - read(end, options.hasSFA); - read(end, options.hasSFB); - read(end, options.alphaIsPtr); - - TLLM_CHECK_WITH_INFO(end == begin + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (end - begin)); - - return options; -} - -GemmAllReducePlugin::GemmAllReducePlugin(void const* data, size_t length) - : GemmAllReducePlugin(deserializeOptions(std::ref(data), length)) -{ - if (mProfiler->useProfiler()) - { - mProfiler->deserializeFromOwnFile(mGemmId, mOptions.maxProblemShape); - } -} - -////////////////////////////////// -// IPluginV2DynamicExt Methods -////////////////////////////////// -IPluginV2DynamicExt* GemmAllReducePlugin::clone() const noexcept -{ - return new GemmAllReducePlugin(*this); -} - -DimsExprs GemmAllReducePlugin::getOutputDimensions( - int outputIndex, DimsExprs const* inputs, int nbInputs, IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs == mNbInputs); // number of input tensors - TLLM_CHECK(inputs[0].nbDims == inputs[1].nbDims); - TLLM_CHECK(outputIndex < getNbOutputs()); - - // List of pointers to D on each rank - if ((nbInputs + outputIndex) == TensorArg::OUT_D_IPC) - { - DimsExprs out_dims; - out_dims.nbDims = 1; - out_dims.d[0] = exprBuilder.constant(mOptions.groupSize); - return out_dims; - } - - TLLM_CHECK(mOptions.transA == false); - TLLM_CHECK(mOptions.transB == true); - - int const nbDimsA = inputs[0].nbDims; // number of dims - int const nbDimsB = inputs[1].nbDims; - - DimsExprs out_dims; - // subtract 2 -> K from each input - out_dims.nbDims = nbDimsA + nbDimsB - 2; - - if (mOptions.transA) - { - for (int i = 1; i < nbDimsA; ++i) - { - out_dims.d[i - 1] = inputs[0].d[i]; - } - } - else - { - for (int i = 0; i < nbDimsA - 1; ++i) - { - out_dims.d[i] = inputs[0].d[i]; - } - } - if (mOptions.transB) - { - for (int i = 0; i < nbDimsB - 1; ++i) - { - out_dims.d[nbDimsA - 1 + i] = inputs[1].d[i]; - } - } - else - { - for (int i = 1; i < nbDimsB; ++i) - { - out_dims.d[nbDimsA - 2 + i] = inputs[1].d[i]; - } - } - return out_dims; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool GemmAllReducePlugin::supportsFormatCombination( - int32_t pos, PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept -{ - // inOut[0] -> activation - // inOut[1] -> weight - // inOut[1+hasInputSF] -> activation_sf - // inOut[1+hasInputSF*2] -> weight_sf - // inOut[2+hasInputSF*2] -> output[0] = D_uc - // inOut[3+hasInputSF*2] -> output[1] = D_mc - - TLLM_CHECK_WITH_INFO(pos < mNbInputs + mNbOutputs, "Unexpected pos: %d", pos); - auto const& desc = inOut[pos]; - - TLLM_CHECK_WITH_INFO(mArgMap.count(pos) > 0, "pos %d not found in mArgMap.", pos); - TensorArg arg = mArgMap[pos]; - - auto typeExists = [&](DataType dtype, auto idx) -> bool - { - for (const auto& [key, value] : mTypedInstantiators) - { - // key format: <ActivationType, WeightType, OutputType> - if (std::get<decltype(idx)::value>(key) == dtype) - { - return true; - } - } - return false; - }; - - switch (arg) - { - case TensorArg::IN_ACTIVATION: return typeExists(desc.type, std::integral_constant<size_t, 0>{}); - case TensorArg::IN_WEIGHT: return typeExists(desc.type, std::integral_constant<size_t, 1>{}); - case TensorArg::IN_ACTIVATION_SF: - case TensorArg::IN_WEIGHT_SF: - // Assumed SF for only FP4 at the moment - return desc.type == DataType::kFP8; - case TensorArg::IN_ALPHA: return desc.type == DataType::kFLOAT; - case TensorArg::OUT_D_UC: - case TensorArg::OUT_D_MC: - case TensorArg::OUT_D_IPC: return typeExists(desc.type, std::integral_constant<size_t, 2>{}); - default: return false; - } -} - -void GemmAllReducePlugin::configurePlugin( - DynamicPluginTensorDesc const* in, int32_t nbInputs, DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept -{ - // Get problem shape - int const nbDimsA = in[0].max.nbDims; - int const minM = utils::computeMDimension(mOptions.transA, in[0].min); - int const maxM = utils::computeMDimension(mOptions.transA, in[0].max); - int const N = utils::computeNDimension(mOptions.transB, in[1].max); - int const K = mOptions.transA ? in[0].max.d[0] : in[0].max.d[nbDimsA - 1]; - - TLLM_CHECK_WITH_INFO(out[0].desc.type == mOptions.typeD, "Output type mismatch."); - - // Ensure call from execution phase does - // not override call from build phase - if (!mOptions.maxProblemShape.isInitialized()) - { - mOptions.maxProblemShape = {minM, maxM, N, K}; - mGemmId = {N, K, mOptions.typeD}; - } - - // Build phase doesn't have COMM_SESSION (i.e built on single rank) - // so do not allocate persistent workspace - if (!isBuilding()) - { - auto getTPRank = [&]() - { - int rank = COMM_SESSION.getRank(); - auto it = std::find(mOptions.group.begin(), mOptions.group.end(), rank); - TLLM_CHECK_WITH_INFO(it != mOptions.group.end(), - "Incorrect group specified - rank " + std::to_string(rank) + " not found in group"); - return std::distance(mOptions.group.begin(), it); - }; - - mRank = getTPRank(); - - if (mWorkspace == nullptr) - { - allocatePersistentWorkspace(); - } - } -} - -size_t GemmAllReducePlugin::getWorkspaceSize( - PluginTensorDesc const* inputs, int32_t nbInputs, PluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept -{ - return 0; -} - -int GemmAllReducePlugin::enqueue(PluginTensorDesc const* inputDesc, PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - // inputs[0] -> [M(*), K] - // inputs[1] -> [K, N] - // outputs[0] -> [M(*), N] unicast ptr - // outputs[1] -> [M(*), N] multicast ptr - auto const nbDimsA = inputDesc[0].dims.nbDims; - auto const M = utils::computeMDimension(mOptions.transA, inputDesc[0].dims); - auto const N = utils::computeNDimension(mOptions.transB, inputDesc[1].dims); - auto const K = mOptions.transA ? inputDesc[0].dims.d[0] : inputDesc[0].dims.d[nbDimsA - 1]; - - TLLM_CHECK_WITH_INFO(M <= mOptions.maxProblemShape.maxM, "GemmAllReducePlugin M > maxM."); - TLLM_CHECK_WITH_INFO(M > 0, "GemmAllReducePlugin M is 0."); - TLLM_CHECK_WITH_INFO(N > 0, "GemmAllReducePlugin N is 0."); - TLLM_CHECK_WITH_INFO(K > 0, "GemmAllReducePlugin K is 0."); - TLLM_CHECK_WITH_INFO(mWorkspace != nullptr, "GemmAllReducePlugin workspace is null."); - - LaunchConfig bestLaunchConfig; - if (mProfiler->useProfiler()) - { - bestLaunchConfig = mProfiler->getBestConfig(M, mGemmId).value(); - } - else - { - bestLaunchConfig = getStaticHeuristicLaunchConfig(M); - } - - void const* activation = inputs[mArgInvMap[TensorArg::IN_ACTIVATION]]; - void const* weight = inputs[mArgInvMap[TensorArg::IN_WEIGHT]]; - void* D_out_uc = outputs[mArgInvMap[TensorArg::OUT_D_UC] - mNbInputs]; - void* D_out_mc = outputs[mArgInvMap[TensorArg::OUT_D_MC] - mNbInputs]; - void* D_out_ipc = outputs[mArgInvMap[TensorArg::OUT_D_IPC] - mNbInputs]; - - TLLM_CHECK_WITH_INFO(activation != nullptr, "GemmAllReducePlugin activation is NULL"); - TLLM_CHECK_WITH_INFO(weight != nullptr, "GemmAllReducePlugin weight is NULL"); - TLLM_CHECK_WITH_INFO(D_out_uc != nullptr, "GemmAllReducePlugin out_uc is NULL"); - TLLM_CHECK_WITH_INFO(D_out_mc != nullptr, "GemmAllReducePlugin out_mc is NULL"); - TLLM_CHECK_WITH_INFO(D_out_ipc != nullptr, "GemmAllReducePlugin out_ipc is NULL"); - - cutlass_kernels::GemmAllReduceImplInterface::ProblemArgs args; - args.argProblemShape(M, N, K, 1) - .argA(activation) - .argB(weight) - .argC(nullptr) - .argD(D_out_uc, D_out_mc, (void**) D_out_ipc) - .argRanks(mRank, mOptions.group) - .argBeta(0.f) // no bias - .argLaunchConfig(bestLaunchConfig) - .argWorkspace(mWorkspace->mWorkspace.get()); - // tensor for scaling input A - if (mOptions.hasSFA) - { - void const* activation_sf = inputs[mArgInvMap[TensorArg::IN_ACTIVATION_SF]]; - TLLM_CHECK_WITH_INFO(activation_sf != nullptr, "GemmAllReducePlugin activation_sf is NULL"); - args.argAScale(activation_sf); - } - // tensor for scaling input B - if (mOptions.hasSFB) - { - void const* weight_sf = inputs[mArgInvMap[TensorArg::IN_WEIGHT_SF]]; - TLLM_CHECK_WITH_INFO(weight_sf != nullptr, "GemmAllReducePlugin weight_sf is NULL"); - args.argBScale(weight_sf); - } - // tensor for scaling output D - if (mOptions.alphaIsPtr) - { - void const* alpha_vec = inputs[mArgInvMap[TensorArg::IN_ALPHA]]; - TLLM_CHECK_WITH_INFO(alpha_vec != nullptr, "GemmAllReducePlugin alpha_vec is NULL"); - args.argAlphaPtr(reinterpret_cast<float const*>(alpha_vec)); - } - else - { - args.argAlpha(mOptions.alpha); - } - - mGemm->run(args, stream); - - return 0; -} - -////////////////////////////////// -// IPluginV2Ext Methods -////////////////////////////////// -DataType GemmAllReducePlugin::getOutputDataType(int index, DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK_WITH_INFO(index < getNbOutputs(), "Output index out of bounds: %d", index); - return mOptions.typeD; -} - -////////////////////////////////// -// IPluginV2 Methods -////////////////////////////////// -char const* GemmAllReducePlugin::getPluginType() const noexcept -{ - return GEMM_ALLREDUCE_PLUGIN_NAME; -} - -char const* GemmAllReducePlugin::getPluginVersion() const noexcept -{ - return GEMM_ALLREDUCE_PLUGIN_VERSION; -} - -int GemmAllReducePlugin::getNbOutputs() const noexcept -{ - return mNbOutputs; -} - -int GemmAllReducePlugin::initialize() noexcept -{ - if (isBuilding() && mProfiler->useProfiler()) - { - // TODO (xsimmons): interfaces between GemmPluginProfiler and Plugin - // needs to be relooked at - current interface implicitly assigns runner to profiler - // object in profileTactics() - assert(mOptions.maxProblemShape.isInitialized()); - mProfiler->profileTactics(mGemm, mOptions.typeD, mOptions.maxProblemShape, mGemmId); - } - return 0; -} - -void GemmAllReducePlugin::terminate() noexcept -{ - if (isBuilding()) // need this otherwise getComm will crash during build phase - { - return; - } - - // free mWorkspace - if (mWorkspace) - { - getPluginRegistry()->releasePluginResource(mWorkspaceKey.c_str()); - mWorkspace = nullptr; - } -} - -size_t GemmAllReducePlugin::getSerializationSize() const noexcept -{ - // cannot use sizeof(GemmAllReducePluginOptions) - // becaused need packed attribute which doesn't work on enum - // without making the enum also packed - size_t size = 0; - size += sizeof(mOptions.typeA); - size += sizeof(mOptions.typeB); - size += sizeof(mOptions.typeD); - size += sizeof(mOptions.transA); - size += sizeof(mOptions.transB); - size += sizeof(mOptions.alpha); - size += sizeof(mOptions.maxProblemShape); - size += sizeof(mOptions.groupSize); - size += mOptions.group.size() * sizeof(int); - size += sizeof(mOptions.hasSFA); - size += sizeof(mOptions.hasSFB); - size += sizeof(mOptions.alphaIsPtr); - return size; -} - -void GemmAllReducePlugin::serialize(void* buffer) const noexcept -{ - char* begin = reinterpret_cast<char*>(buffer); - char* end = reinterpret_cast<char*>(buffer); - - write(end, mOptions.typeA); - write(end, mOptions.typeB); - write(end, mOptions.typeD); - write(end, mOptions.transA); - write(end, mOptions.transB); - write(end, mOptions.alpha); - write(end, mOptions.maxProblemShape); - write(end, mOptions.groupSize); - for (auto const& rank : mOptions.group) - { - write(end, rank); - } - write(end, mOptions.hasSFA); - write(end, mOptions.hasSFB); - write(end, mOptions.alphaIsPtr); - TLLM_CHECK(end == begin + getSerializationSize()); - - // Profiler MNK->kernel mappings need to be deterministic and consistent across ranks - // to ensure correct functionality (unlike standalone GEMMs). - // Since by default each rank will generate and serialize its own profiler mapping - // this can lead to different mappings between ranks which will result in fatal - // error. Therefore only generate and use profiler mapping for single rank. - if (mProfiler->useProfiler() && COMM_SESSION.getRank() == 0) - { - mProfiler->serializeToOwnFile(mGemmId); - } -} - -void GemmAllReducePlugin::destroy() noexcept -{ - delete this; -} - -//////////////////////////////////////////////////////////// -// GemmAllReducePluginCreator Methods -//////////////////////////////////////////////////////////// -PluginFieldCollection GemmAllReducePluginCreator::mFC; -std::vector<PluginField> GemmAllReducePluginCreator::mPluginAttributes; - -GemmAllReducePluginCreator::GemmAllReducePluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back("type_a", nullptr, PluginFieldType::kINT32, 1); - mPluginAttributes.emplace_back("type_b", nullptr, PluginFieldType::kINT32, 1); - mPluginAttributes.emplace_back("type_d", nullptr, PluginFieldType::kINT32, 1); - mPluginAttributes.emplace_back("transa", nullptr, PluginFieldType::kINT32, 1); - mPluginAttributes.emplace_back("transb", nullptr, PluginFieldType::kINT32, 1); - mPluginAttributes.emplace_back("alpha", nullptr, PluginFieldType::kFLOAT32, 1); - mPluginAttributes.emplace_back("group", nullptr, PluginFieldType::kINT32, 1); - mPluginAttributes.emplace_back("has_sfa", nullptr, PluginFieldType::kINT8, 1); - mPluginAttributes.emplace_back("has_sfb", nullptr, PluginFieldType::kINT8, 1); - mPluginAttributes.emplace_back("alpha_is_ptr", nullptr, PluginFieldType::kINT8, 1); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* GemmAllReducePluginCreator::getPluginName() const noexcept -{ - return GEMM_ALLREDUCE_PLUGIN_NAME; -} - -char const* GemmAllReducePluginCreator::getPluginVersion() const noexcept -{ - return GEMM_ALLREDUCE_PLUGIN_VERSION; -} - -PluginFieldCollection const* GemmAllReducePluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* GemmAllReducePluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - GemmAllReducePluginOptions options; - options.deserialize = false; - - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "type_a")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - options.typeA = *static_cast<DataType const*>(fields[i].data); - } - else if (!strcmp(attrName, "type_b")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - options.typeB = *static_cast<DataType const*>(fields[i].data); - } - else if (!strcmp(attrName, "type_d")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - options.typeD = *static_cast<DataType const*>(fields[i].data); - } - else if (!strcmp(attrName, "transa")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - options.transA = *static_cast<int const*>(fields[i].data); - } - else if (!strcmp(attrName, "transb")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - options.transB = *static_cast<int const*>(fields[i].data); - } - else if (!strcmp(attrName, "alpha")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - options.alpha = *static_cast<float const*>(fields[i].data); - } - else if (!strcmp(attrName, "group")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - auto const* ranks = static_cast<int const*>(fields[i].data); - for (int j = 0; j < fields[i].length; ++j) - { - options.group.insert(ranks[j]); - } - options.groupSize = options.group.size(); - } - else if (!strcmp(attrName, "has_sfa")) // passed in as input tensor - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - options.hasSFA = *static_cast<int8_t const*>(fields[i].data); - } - else if (!strcmp(attrName, "has_sfb")) // passed in as input tensor - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - options.hasSFB = *static_cast<int8_t const*>(fields[i].data); - } - else if (!strcmp(attrName, "alpha_is_ptr")) // passed in as input tensor - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - options.alphaIsPtr = *static_cast<int8_t const*>(fields[i].data); - } - } - - try - { - // GemmAllReducePluginCreator is unique and shared for an engine generation - auto* obj = new GemmAllReducePlugin(options); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - return nullptr; - } -} - -IPluginV2* GemmAllReducePluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call GemmAllReducePlugin::destroy() - try - { - auto* obj = new GemmAllReducePlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.h b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.h deleted file mode 100644 index 457926246002..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.h +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) -#include "tensorrt_llm/kernels/cutlass_kernels/include/allreduce_gemm_runner.h" -#else -#include "allreduce_gemm_runner.h" -#endif - -#include "gemmAllReducePluginProfiler.h" -#include "gemmAllReducePluginResource.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" - -using namespace nvinfer1; - -using nvinfer1::DataType; -#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) -namespace cutlass_kernels = ::tensorrt_llm::kernels::opened_cutlass_kernels; -#else -namespace cutlass_kernels = ::tensorrt_llm::kernels::cutlass_kernels; -#endif - -using LaunchConfig = typename cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig; - -namespace tensorrt_llm::plugins -{ -struct GemmAllReducePluginOptions -{ - // Don't need to specify problem shape, this - // is specified in configurePlugin - DataType typeA; - DataType typeB; - DataType typeD; - int transA; - int transB; - float alpha; - // ranks participating in collective - std::set<int> group; - int groupSize; - // Set in configurePlugin during build phase - GemmDims maxProblemShape; - bool deserialize; // used for profiler instantiation - int8_t hasSFA = 0; - int8_t hasSFB = 0; - int8_t alphaIsPtr = 0; -}; - -class GemmAllReducePlugin : public BasePlugin -{ - friend class GemmAllReducePluginCreator; - -public: - ~GemmAllReducePlugin() override = default; - - ////////////////////////////////// - // IPluginV2DynamicExt Methods - ////////////////////////////////// - IPluginV2DynamicExt* clone() const noexcept override; - - DimsExprs getOutputDimensions( - int outputIndex, DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept override; - - // inOut[0] -> activation - // inOut[1] -> weight - // inOut[2] -> result - bool supportsFormatCombination( - int32_t pos, PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept override; - - // in[0] -> activation - // in[1] -> weight - // no bias needed - void configurePlugin(DynamicPluginTensorDesc const* in, int32_t nbInputs, DynamicPluginTensorDesc const* out, - int32_t nbOutputs) noexcept override; - - size_t getWorkspaceSize(PluginTensorDesc const* inputs, int32_t nbInputs, PluginTensorDesc const* outputs, - int32_t nbOutputs) const noexcept override; - - // in[0] -> activation - // in[1] -> weight - // out[0] -> result_uc - // out[1] -> result_mc - int enqueue(PluginTensorDesc const* inputDesc, PluginTensorDesc const* outputDesc, void const* const* inputs, - void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - ////////////////////////////////// - // IPluginV2Ext Methods - ////////////////////////////////// - DataType getOutputDataType(int index, DataType const* inputTypes, int nbInputs) const noexcept override; - - ////////////////////////////////// - // IPluginV2 Methods - ////////////////////////////////// - char const* getPluginType() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - int getNbOutputs() const noexcept override; - - int initialize() noexcept override; - - void terminate() noexcept override; - - size_t getSerializationSize() const noexcept override; - - void serialize(void* buffer) const noexcept override; - - void destroy() noexcept override; - -private: - explicit GemmAllReducePlugin(GemmAllReducePluginOptions const& options); - // Parameterized constructor - explicit GemmAllReducePlugin(void const* data, size_t length); - - void allocatePersistentWorkspace(); - - LaunchConfig getStaticHeuristicLaunchConfig(int M) const; - - // Params that are initialized during constructor - using KeyType = std::tuple<DataType, DataType, DataType>; - using ValueType = std::function<cutlass_kernels::GemmAllReduceImplInterface*()>; - GemmAllReducePluginOptions mOptions; - int mRank = 0; - - enum TensorArg - { - IN_ACTIVATION, - IN_ACTIVATION_SF, - IN_WEIGHT, - IN_WEIGHT_SF, - IN_ALPHA, - OUT_D_UC, - OUT_D_MC, - OUT_D_IPC - }; - - std::unordered_map<int, TensorArg> mArgMap; - std::unordered_map<TensorArg, int> mArgInvMap; - int mNbInputs = 0; - int mNbOutputs = 0; - - std::map<KeyType, ValueType> mTypedInstantiators; - std::string mWorkspaceKey; - std::shared_ptr<cutlass_kernels::GemmAllReduceImplInterface> mGemm; - // Params that are initialized during configurePlugin() - GemmAllReducePersistentWorkspace* mWorkspace = nullptr; - - // Used for selecting best GEMM for given problem shapes - GemmIdCore mGemmId{}; - GemmPluginProfilerManager<GemmAllReducePluginProfiler> mGemmPluginProfileManager; - std::shared_ptr<GemmAllReducePluginProfiler> mProfiler; -}; - -class GemmAllReducePluginCreator : public BaseCreator -{ -public: - GemmAllReducePluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.cpp b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.cpp deleted file mode 100644 index a6f7ca2615df..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.cpp +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gemmAllReducePlugin.h" -#include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h" -#include "tensorrt_llm/plugins/common/pluginUtils.h" - -namespace tc = tensorrt_llm::common; - -namespace tensorrt_llm::plugins -{ -void GemmAllReducePluginProfiler::serializeToOwnFile(GemmIdCore gemmId) -{ - std::vector<char> file_buf(getSerializationSize(gemmId)); - char* begin = file_buf.data(); - char* end = file_buf.data(); - serialize(end, gemmId); - assert(end == begin + file_buf.size()); - - auto fileName = getCacheFileName(gemmId); - std::ofstream file(fileName, std::ios::binary); - TLLM_CHECK(file.is_open()); - file.write(begin, file_buf.size()); - file.flush(); - file.close(); -} - -void GemmAllReducePluginProfiler::deserializeFromOwnFile(GemmIdCore gemmId, GemmDims problemShape) -{ - auto fileName = getCacheFileName(gemmId); - std::ifstream file(fileName, std::ios::binary); - TLLM_CHECK(file.is_open()); - file.seekg(0, std::ios::end); - std::streamsize size = file.tellg(); - TLLM_CHECK(size > 0); - file.seekg(0, std::ios::beg); - - std::vector<char> file_buf(size); - file.read(file_buf.data(), size); - file.close(); - - char const* begin = const_cast<char const*>(file_buf.data()); - char const* end = begin; - deserialize(end, problemShape, gemmId); - assert(end == begin + size); -} - -bool GemmAllReducePluginProfiler::useProfiler() -{ - // char const* envDir = getenv("GEMM_AR_PLUGIN_PROFILE_DIR"); - // return envDir != nullptr; - // TODO(xsimmons): currently the profiler does not add any perf gain - // due to static heuristics being sufficient. We can re-enable this - // when we need more configurations. - return false; -} - -std::string GemmAllReducePluginProfiler::getCacheFileName(GemmIdCore gemmId) -{ - std::stringstream fileName; - char const* envDir = getenv("GEMM_AR_PLUGIN_PROFILE_DIR"); - std::string directory = envDir ? std::string(envDir) : "/tmp/"; - fileName << directory + "/gemm-AR"; - fileName << "-n" << std::to_string(gemmId.n); - fileName << "-k" << std::to_string(gemmId.k); - fileName << "-" << tc::getDtypeString(gemmId.dtype); - fileName << ".prof_cache"; - return fileName.str(); -} - -void GemmAllReducePluginProfiler::runTactic(int m, int n, int k, - cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig const& tactic, char* workspace, - cudaStream_t const& stream) -{ - const size_t dtype_size = tc::getDTypeSize(mType); - char* inputA = workspace; - char* inputB = inputA + m * k * dtype_size; - char* outputD = inputB + n * k * dtype_size; - char* inputSFA = outputD + m * n * dtype_size; - char* inputSFB = inputSFA + m * k * dtype_size; - std::set<int> tpGroup = {0}; - - // Run on single-GPU - cutlass_kernels::GemmAllReduceImplInterface::ProblemArgs args; - args.argProblemShape(m, n, k, 1) - .argA((void*) inputA) - .argB((void*) inputB) - .argD((void*) outputD, /*output_mc=*/nullptr) - .argAScale((void*) inputSFA) - .argBScale((void*) inputSFB) - .argRanks(0, tpGroup) - .argAlpha(1.f) - .argBeta(0.f) // no bias - .argLaunchConfig(tactic); - - TLLM_CHECK(mRunner != nullptr); - mRunner->run(args, stream); -} - -void GemmAllReducePluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - TLLM_CHECK(maxM != 0); - TLLM_CHECK(n != 0); - TLLM_CHECK(k != 0); - // mType refers to the output data type - // WARNING: This code assumes that the output precision is >= to input precision - const size_t dtype_size = tc::getDTypeSize(mType); - size_t bytes = 0; - bytes += maxM * k * dtype_size; // A - bytes += n * k * dtype_size; // B - // No C - // Note that D is typically IPC, however, when tuning GEMM we need it to run on single GPU - bytes += maxM * n * dtype_size; // D - // scale tensors for A & B - will at most be same size as A/B - bytes += maxM * k * dtype_size; // A - bytes += n * k * dtype_size; // B - - setTmpWorkspaceSizeInBytes(bytes); -} - -std::vector<cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig> GemmAllReducePluginProfiler::getTactics( - int m, int n, int k) const -{ - TLLM_CHECK(mRunner != nullptr); - return mRunner->getSupportedLaunchConfigs(); -} -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.h b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.h deleted file mode 100644 index faacbb3b8c0f..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) -#include "tensorrt_llm/kernels/cutlass_kernels/include/allreduce_gemm_runner.h" -#else -#include "allreduce_gemm_runner.h" -#endif -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" - -namespace tensorrt_llm::plugins -{ -/* - * Used for tuning to find best GEMM configs for different problem shapes. - * WARNING: Tuning GEMM+AR kernel may not be fully representable of real - * multi-GPU workloads as tuning only runs on single-GPU. - * IMPORTANT: TRT-LLM does not support deterministic tuning across ranks. - * Because of this, we have to serialize/deserialize our own configuration file. - */ - -#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) -namespace cutlass_kernels = ::tensorrt_llm::kernels::opened_cutlass_kernels; -#else -namespace cutlass_kernels = ::tensorrt_llm::kernels::cutlass_kernels; -#endif -class GemmAllReducePluginProfiler - : public GemmPluginProfiler<cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig, - std::shared_ptr<cutlass_kernels::GemmAllReduceImplInterface>, GemmIdCore, GemmIdCoreHash> -{ -public: - void serializeToOwnFile(GemmIdCore gemmId); - - void deserializeFromOwnFile(GemmIdCore gemmId, GemmDims problemShape); - - bool useProfiler(); - -protected: - //////////////////////////////////// - // GemmPluginProfiler methods - //////////////////////////////////// - void runTactic(int m, int n, int k, cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig const& tactic, - char* workspace, cudaStream_t const& stream) override; - - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - - std::vector<cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig> getTactics( - int m, int n, int k) const override; - -private: - static std::string getCacheFileName(GemmIdCore gemmId); -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginResource.h b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginResource.h deleted file mode 100644 index 8136bd363bd7..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginResource.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "NvInferPlugin.h" - -#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) -#include "tensorrt_llm/kernels/cutlass_kernels/include/allreduce_gemm_runner.h" -#else -#include "allreduce_gemm_runner.h" -#endif -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/plugins/common/plugin.h" - -using namespace nvinfer1; - -namespace tensorrt_llm::plugins -{ - -#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) -namespace cutlass_kernels = ::tensorrt_llm::kernels::opened_cutlass_kernels; -#else -namespace cutlass_kernels = ::tensorrt_llm::kernels::cutlass_kernels; -#endif -class GemmAllReducePersistentWorkspace : public IPluginResource -{ -public: - GemmAllReducePersistentWorkspace(std::shared_ptr<cutlass_kernels::PersistentWorkspaceInterface> workspace) - : mWorkspace(workspace) - { - } - - ////////////////////////////////// - // IPluginResource Methods - ////////////////////////////////// - IPluginResource* clone() noexcept override - { - auto copy = new GemmAllReducePersistentWorkspace(mWorkspace); - // Resource initialization (if any) may be skipped for non-cloned objects - // since only clones will be registered by TensorRT. - try - { - copy->mWorkspace->allocate(); - return copy; - } - catch (std::exception const& e) - { - TLLM_LOG_ERROR(e.what()); - return nullptr; - } - } - - int32_t release() noexcept override - { - try - { - return mWorkspace->free(); - } - catch (std::exception const& e) - { - TLLM_LOG_ERROR(e.what()); - return -1; - } - } - - std::shared_ptr<cutlass_kernels::PersistentWorkspaceInterface> mWorkspace; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/gemmPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.cpp b/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.cpp deleted file mode 100644 index 9e06ad01d10f..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.cpp +++ /dev/null @@ -1,614 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "gemmPlugin.h" - -#include "gemmPluginProfiler.h" -#include "plugin.h" -#include "pluginUtils.h" -#include "tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemm.h" -#include "tensorrt_llm/runtime/utils/debugUtils.h" - -#include <NvInferRuntime.h> - -#include <cassert> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::GemmDims; -using tensorrt_llm::plugins::GemmPluginCreator; -using tensorrt_llm::plugins::GemmPlugin; -using tensorrt_llm::plugins::CublasLtGemmPluginProfiler; -using tensorrt_llm::plugins::CublasGemmWrapperPtr; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; - -static char const* GEMM_PLUGIN_VERSION{"1"}; -static char const* GEMM_PLUGIN_NAME{"Gemm"}; -PluginFieldCollection GemmPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> GemmPluginCreator::mPluginAttributes; - -void getProblemParams(cublasOperation_t& transa, cublasOperation_t& transb, int& m, int& n, int& k, int& lda, int& ldb, - int& ldc, bool transA, bool transB, int M, int N, int K, int padLda, int padLdb, int padLdc) -{ - transa = transB ? CUBLAS_OP_T : CUBLAS_OP_N; - transb = transA ? CUBLAS_OP_T : CUBLAS_OP_N; - m = N; - n = M; - k = K; - lda = transB ? K + padLdb : N + padLdb; - ldb = transA ? M + padLda : K + padLda; - ldc = N + padLdc; -} - -void runGemm(int const M, int const N, int const K, bool const transA, bool const transB, int const padLda, - int const padLdb, int const padLdc, nvinfer1::DataType const type, CublasGemmWrapperPtr const& cublasWrapperPtr, - void const* act, void const* weight, float const alpha, void* output, - std::optional<cublasLtMatmulHeuristicResult_t> const& heuristic, void* workspace, cudaStream_t stream) -{ - if (M == 0 || N == 0 || K == 0) - return; - - cublasWrapperPtr->setStream(stream); - cublasWrapperPtr->setWorkspace(workspace); - - cublasOperation_t transa, transb; - int m, n, k; - int lda, ldb, ldc; - getProblemParams(transa, transb, m, n, k, lda, ldb, ldc, transA, transB, M, N, K, padLda, padLdb, padLdc); - - cublasWrapperPtr->createDescriptors(transa, transb, m, n, k, lda, ldb, ldc); - cublasWrapperPtr->Gemm(transa, transb, m, n, k, weight, lda, act, ldb, output, ldc, alpha, 0.0f, heuristic); - cublasWrapperPtr->destroyDescriptors(); -} - -void CublasLtGemmPluginProfiler::runTactic( - int m, int n, int k, CublasLtGemmPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) -{ - size_t dataSize = sizeof(half); - if (mType == nvinfer1::DataType::kFLOAT) - { - dataSize = sizeof(float); - } - - void* actPtr = reinterpret_cast<void*>(workspace); - void* weightPtr = reinterpret_cast<void*>( - nextWorkspacePtrWithAlignment(reinterpret_cast<int8_t*>(actPtr), m * k * dataSize, ALIGNMENT)); - void* outputPtr = reinterpret_cast<void*>( - nextWorkspacePtrWithAlignment(reinterpret_cast<int8_t*>(weightPtr), n * k * dataSize, ALIGNMENT)); - char* workspacePtr = reinterpret_cast<char*>( - nextWorkspacePtrWithAlignment(reinterpret_cast<int8_t*>(outputPtr), m * (n + mPadLdc) * dataSize, ALIGNMENT)); - runGemm(m, n, k, mTransA, mTransB, mPadLda, mPadLdb, mPadLdc, mType, mRunner, actPtr, weightPtr, 1.0f, outputPtr, - {tactic}, workspacePtr, stream); -} - -bool CublasLtGemmPluginProfiler::checkTactic(int m, int n, int k, Config const& tactic) const -{ - cublasOperation_t transa, transb; - int M = m, N = n, K = k; - int lda, ldb, ldc; - getProblemParams(transa, transb, m, n, k, lda, ldb, ldc, mTransA, mTransB, M, N, K, mPadLda, mPadLdb, mPadLdc); - - mRunner->createDescriptors(transa, transb, m, n, k, lda, ldb, ldc); - - auto const checkResult = mRunner->checkTactic(transa, transb, m, n, k, lda, ldb, ldc, tactic.algo); - - mRunner->destroyDescriptors(); - - return checkResult; -} - -void CublasLtGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - size_t dataSize = getDTypeSize(mType); - size_t outputDataSize = getDTypeSize(mOutputType); - - std::vector<size_t> workspaces = { - maxM * k * dataSize, // A - n * k * dataSize, // B - maxM * (n + mPadLdc) * outputDataSize, // C - CUBLAS_WORKSPACE_SIZE // workspace - }; - size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size(), ALIGNMENT); - setTmpWorkspaceSizeInBytes(bytes); -} - -std::vector<CublasLtGemmPluginProfiler::Config> CublasLtGemmPluginProfiler::getTactics(int M, int N, int K) const -{ - cublasOperation_t transa, transb; - int m, n, k; - int lda, ldb, ldc; - getProblemParams(transa, transb, m, n, k, lda, ldb, ldc, mTransA, mTransB, M, N, K, mPadLda, mPadLdb, mPadLdc); - - mRunner->createDescriptors(transa, transb, m, n, k, lda, ldb, ldc); - auto const heruistics = mRunner->getTactics(transa, transb, m, n, k, lda, ldb, ldc); - mRunner->destroyDescriptors(); - - return heruistics; -} - -GemmPlugin::GemmPlugin(int transA, int transB, int padLda, int padLdb, int padLdc, nvinfer1::DataType type, bool useFp8, - float alpha, GemmPlugin::PluginProfilerPtr const& pluginProfiler) - : mTransA(transA) - , mTransB(transB) - , mPadLda(padLda) - , mPadLdb(padLdb) - , mPadLdc(padLdc) - , mType(type) - , mOutputType(type) - , mUseFp8(useFp8) - , mAlpha(alpha) - , mPluginProfiler(pluginProfiler) -{ - init(); -} - -// Parameterized constructor -GemmPlugin::GemmPlugin(void const* data, size_t length, GemmPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mTransA); - read(d, mTransB); - read(d, mPadLda); - read(d, mPadLdb); - read(d, mPadLdc); - read(d, mType); - read(d, mUseFp8); - read(d, mAlpha); - read(d, mDims); - read(d, mOutputType); - - init(); - - mPluginProfiler->deserialize(d, mDims, mGemmId); - - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -thread_local CublasGemmWrapperPtr GemmPlugin::mCublasWrapper = nullptr; - -void GemmPlugin::init() -{ - auto cublasHandle = getCublasHandle(); - auto cublasLtHandle = getCublasLtHandle(); - mCublasWrapper = std::make_shared<CublasMMWrapper>(cublasHandle, cublasLtHandle, nullptr, nullptr); - - mPluginProfiler->setTranspose(mTransA, mTransB); - mPluginProfiler->setOutputType(mOutputType); - mPluginProfiler->setPadLd(mPadLda, mPadLdb, mPadLdc); - - mGemmId = GemmIdCublas(mDims.n, mDims.k, mType, mTransA, mTransB, mOutputType); - - mArch = tensorrt_llm::common::getSMVersion(); -} - -void GemmPlugin::setGemmConfig() -{ - if (mType == nvinfer1::DataType::kHALF) - { - mCublasWrapper->setFP16GemmConfig(trtToCublasDtype(mOutputType)); - } - else if (mType == nvinfer1::DataType::kFLOAT) - { - mCublasWrapper->setFP32GemmConfig(); - } -#ifdef ENABLE_BF16 - else if (mType == nvinfer1::DataType::kBF16) - { - mCublasWrapper->setBF16GemmConfig(trtToCublasDtype(mOutputType)); - } -#endif - -#ifdef ENABLE_FP8 - if (mUseFp8) - { - mCublasWrapper->setFP8GemmConfig(trtToCublasDtype(mOutputType)); - } -#endif -} - -void GemmPlugin::configGemm() -{ - if (!mDims.isInitialized()) - { - return; - } - - setGemmConfig(); - - mPluginProfiler->profileTactics(mCublasWrapper, mType, mDims, mGemmId); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* GemmPlugin::clone() const noexcept -{ - auto* plugin = new GemmPlugin(*this); - return plugin; -} - -nvinfer1::DimsExprs GemmPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs == 2); - TLLM_CHECK(outputIndex == 0); - int const nbDimsA = inputs[0].nbDims; - int const nbDimsB = inputs[1].nbDims; - DimsExprs ret; - ret.nbDims = nbDimsA + nbDimsB - 2; - - if (mTransA) - { - for (int i = 1; i < nbDimsA; ++i) - { - ret.d[i - 1] = inputs[0].d[i]; - } - } - else - { - for (int i = 0; i < nbDimsA - 1; ++i) - { - ret.d[i] = inputs[0].d[i]; - } - } - if (mTransB) - { - for (int i = 0; i < nbDimsB - 1; ++i) - { - ret.d[nbDimsA - 1 + i] = exprBuilder.constant(inputs[1].d[i]->getConstantValue() + mPadLdc); - } - } - else - { - for (int i = 1; i < nbDimsB; ++i) - { - ret.d[nbDimsA - 2 + i] = exprBuilder.constant(inputs[1].d[i]->getConstantValue() + mPadLdc); - } - } - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool GemmPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - auto const& desc = inOut[pos]; - if (desc.format != TensorFormat::kLINEAR) - { - return false; - } - - if (pos < nbInputs) - { - // If use FP8, act/weight dtype should be kFP8 - if (mUseFp8) - { - return desc.type == nvinfer1::DataType::kFP8; - } - else - { - return desc.type == mType; - } - } - - return desc.type == mType || desc.type == nvinfer1::DataType::kFLOAT; -} - -void GemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - auto const nbDimsA = in[0].max.nbDims; - - auto const minM = utils::computeMDimension(mTransA, in[0].min); - auto const maxM = utils::computeMDimension(mTransA, in[0].max); - auto const N = utils::computeNDimension(mTransB, in[1].max); - auto const K = static_cast<utils::DimType64>(mTransA ? in[0].max.d[0] : in[0].max.d[nbDimsA - 1]); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, N, K}; - } - mGemmId.n = N; - mGemmId.k = K; - - mOutputType = out[0].desc.type; -} - -size_t GemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return CUBLAS_WORKSPACE_SIZE; -} - -int GemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - // inputs - // mat1 [M, K] (mTransA = False) - // mat2 [K, N] (mTransB = False) - // outputs - // mat [M, N] - if (mCublasWrapper == nullptr) - { - auto cublasHandle = getCublasHandle(); - auto cublasLtHandle = getCublasLtHandle(); - mCublasWrapper = std::make_shared<CublasMMWrapper>(cublasHandle, cublasLtHandle, nullptr, nullptr); - } - setGemmConfig(); - - int const nbDimsA = inputDesc[0].dims.nbDims; - int const padM = mTransA ? mPadLda : 0; - int const padN = mTransB ? 0 : mPadLdb; - int const padK = mTransA ? 0 : mPadLda; - auto const M = utils::computeMDimension(mTransA, inputDesc[0].dims) - padM; - auto const N = utils::computeNDimension(mTransB, inputDesc[1].dims) - padN; - int const K = static_cast<utils::DimType64>( - mTransA ? inputDesc[0].dims.d[0] - padK : inputDesc[0].dims.d[nbDimsA - 1] - padK); - - bool noPadDim = padM == 0 && padN == 0 && padK == 0 && mPadLdc == 0; - bool cudaKernelSupportType = mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kFLOAT - || mType == nvinfer1::DataType::kBF16; - - // skip computation for a TRT empty tensor - if (M == 0) - { - return 0; - } - - std::string mnkStr = "MNK={" + std::to_string(M) + ", " + std::to_string(N) + ", " + std::to_string(K) + "}"; - { - std::string const activationStr = "GEMM layer's activation before GEMM with " + mnkStr; - TLLM_CHECK_DEBUG_WITH_INFO( - tensorrt_llm::runtime::utils::tensorHasInvalid(M, K, mType, inputs[0], stream, activationStr) == false, - "Found invalid number (NaN or Inf) in " + activationStr); - } - - bool cudaKernelFinished = false; - bool isArch90or100 = mArch >= 90 && mArch < 120; - // TODO: sub tensor matmul is not supported in fp8 gemm cuda kernel - if (!isArch90or100 && M <= 4 && N <= 128000 && mUseFp8 && noPadDim && cudaKernelSupportType) - { - tensorrt_llm::kernels::cuda_core_gemm::Params params(reinterpret_cast<void const*>(inputs[0]), - reinterpret_cast<void const*>(inputs[1]), mAlpha, reinterpret_cast<void*>(outputs[0]), M, N, K, - CUDA_R_8F_E4M3, trtToCublasDtype(mOutputType)); - cudaKernelFinished = tensorrt_llm::kernels::cuda_core_gemm::cudaCoreGemmDispatcher(params, stream); - } - else if (!isArch90or100 && ((mArch < 90 && M <= 6) || (isArch90or100 && M <= 2)) && N <= 128000 && !mUseFp8 - && noPadDim && cudaKernelSupportType) - { - tensorrt_llm::kernels::cuda_core_gemm::Params params(reinterpret_cast<void const*>(inputs[0]), - reinterpret_cast<void const*>(inputs[1]), mAlpha, reinterpret_cast<void*>(outputs[0]), M, N, K, - trtToCublasDtype(mType), trtToCublasDtype(mOutputType)); - cudaKernelFinished = tensorrt_llm::kernels::cuda_core_gemm::cudaCoreGemmDispatcher(params, stream); - } - - if (!cudaKernelFinished) - { - auto bestTactic = mPluginProfiler->getBestConfig(M, mGemmId); - runGemm(M, N, K, mTransA, mTransB, mPadLda, mPadLdb, mPadLdc, mType, mCublasWrapper, inputs[0], inputs[1], - mAlpha, outputs[0], bestTactic, workspace, stream); - } - - { - std::string const outputStr = "GEMM layer's output after GEMM with " + mnkStr; - TLLM_CHECK_DEBUG_WITH_INFO( - tensorrt_llm::runtime::utils::tensorHasInvalid(M, N + mPadLdc, mType, outputs[0], stream, outputStr) - == false, - "Found invalid number (NaN or Inf) in " + outputStr); - } - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType GemmPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* GemmPlugin::getPluginType() const noexcept -{ - return GEMM_PLUGIN_NAME; -} - -char const* GemmPlugin::getPluginVersion() const noexcept -{ - return GEMM_PLUGIN_VERSION; -} - -int GemmPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int GemmPlugin::initialize() noexcept -{ - configGemm(); - return 0; -} - -void GemmPlugin::destroy() noexcept -{ - delete this; -} - -size_t GemmPlugin::getSerializationSize() const noexcept -{ - return sizeof(mTransA) + sizeof(mTransB) + sizeof(mPadLda) + sizeof(mPadLdb) + sizeof(mPadLdc) + sizeof(mType) - + sizeof(mDims) + sizeof(mUseFp8) + sizeof(mAlpha) + mPluginProfiler->getSerializationSize(mGemmId) - + sizeof(mOutputType); // selected tactics container size -} - -void GemmPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mTransA); - write(d, mTransB); - write(d, mPadLda); - write(d, mPadLdb); - write(d, mPadLdc); - write(d, mType); - write(d, mUseFp8); - write(d, mAlpha); - write(d, mDims); - write(d, mOutputType); - mPluginProfiler->serialize(d, mGemmId); - - TLLM_CHECK(d == a + getSerializationSize()); -} - -void GemmPlugin::terminate() noexcept {} - -/////////////// - -GemmPluginCreator::GemmPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("transA", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("transB", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("padLda", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("padLdb", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("padLdc", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("use_fp8", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* GemmPluginCreator::getPluginName() const noexcept -{ - return GEMM_PLUGIN_NAME; -} - -char const* GemmPluginCreator::getPluginVersion() const noexcept -{ - return GEMM_PLUGIN_VERSION; -} - -PluginFieldCollection const* GemmPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* GemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int transA{}; - int transB{}; - int padLda{}; - int padLdb{}; - int padLdc{}; - nvinfer1::DataType type{}; - int useFp8{}; - float alpha = 1.F; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "transa")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - transA = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "transb")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - transB = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "pad_lda")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - padLda = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "pad_ldb")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - padLdb = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "pad_ldc")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - padLdc = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "use_fp8")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - useFp8 = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "alpha")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - alpha = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - } - try - { - // GemmPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - // FIXME enable tactic profiler - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false, /* skip */ true); - auto* obj = new GemmPlugin(transA, transB, padLda, padLdb, padLdc, type, useFp8, alpha, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* GemmPluginCreator::deserializePlugin(char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call GemmPlugin::destroy() - try - { - // GemmPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - // FIXME enable tactic profiler - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true, /* skip */ true); - auto* obj = new GemmPlugin(serialData, serialLength, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h b/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h deleted file mode 100644 index 1ba553c23d4b..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h +++ /dev/null @@ -1,169 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef TRT_GEMM_PLUGIN_H -#define TRT_GEMM_PLUGIN_H - -#include "tensorrt_llm/common/cublasMMWrapper.h" -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" - -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -using CublasGemmWrapper = tensorrt_llm::common::CublasMMWrapper; -using CublasGemmWrapperPtr = std::shared_ptr<CublasGemmWrapper>; - -class CublasLtGemmPluginProfiler - : public GemmPluginProfiler<cublasLtMatmulHeuristicResult_t, CublasGemmWrapperPtr, GemmIdCublas, GemmIdCublasHash> -{ -public: - using Config = cublasLtMatmulHeuristicResult_t; - - void setTranspose(bool transposeA, bool transposeB) - { - mTransA = transposeA; - mTransB = transposeB; - } - - void setPadLd(int padLda, int padLdb, int padLdc) - { - mPadLda = padLda; - mPadLdb = padLdb; - mPadLdc = padLdc; - } - - void setOutputType(nvinfer1::DataType type) - { - mOutputType = type; - } - -protected: - void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; - - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - - bool checkTactic(int m, int n, int k, Config const& tactic) const override; - - std::vector<Config> getTactics(int m, int n, int k) const override; - -private: - bool mTransA; - bool mTransB; - int mPadLda; - int mPadLdb; - int mPadLdc; - nvinfer1::DataType mOutputType; - - static constexpr size_t ALIGNMENT = 256; -}; - -class GemmPlugin : public BasePlugin -{ -public: - using PluginProfilerPtr = std::shared_ptr<CublasLtGemmPluginProfiler>; - - GemmPlugin() = delete; - - GemmPlugin(int transA, int transB, int padLda, int padLdb, int padLdc, nvinfer1::DataType type, bool useFp8, - float alpha, PluginProfilerPtr const& profiler); - - GemmPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); - - ~GemmPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - void init(); - void configGemm(); - void setGemmConfig(); - -private: - const std::string mLayerName; - - int mTransA; - int mTransB; - int mPadLda; - int mPadLdb; - int mPadLdc; - int mArch; - nvinfer1::DataType mType; - nvinfer1::DataType mOutputType; - - static thread_local CublasGemmWrapperPtr mCublasWrapper; - - GemmDims mDims{}; - GemmIdCublas mGemmId{}; - bool mUseFp8{false}; - float mAlpha{1.f}; - - PluginProfilerPtr mPluginProfiler; -}; - -class GemmPluginCreator : public BaseCreator -{ -public: - GemmPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - GemmPluginProfilerManager<CublasLtGemmPluginProfiler> gemmPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins - -#endif // TRT_GEMM_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/CMakeLists.txt deleted file mode 100644 index 3b714a3928fb..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp *.cu) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cpp b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cpp deleted file mode 100644 index ed964ace695f..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cpp +++ /dev/null @@ -1,446 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "gemmSwigluPlugin.h" -#include "cutlass_extensions/gemm_configs.h" - -#include <NvInferRuntimeBase.h> -#include <numeric> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using namespace tensorrt_llm::kernels::cutlass_kernels; -using tensorrt_llm::plugins::GemmSwigluPluginCreator; -using tensorrt_llm::plugins::GemmSwigluPlugin; -using tensorrt_llm::plugins::GemmSwigluPluginProfiler; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; - -static char const* GEMM_SWIGLU_PLUGIN_VERSION{"1"}; -static char const* GEMM_SWIGLU_PLUGIN_NAME{"GemmSwiglu"}; -PluginFieldCollection GemmSwigluPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> GemmSwigluPluginCreator::mPluginAttributes; - -size_t GemmSwigluPluginProfiler::getBytePerElement(nvinfer1::DataType type) -{ - size_t bpe; - if (type == nvinfer1::DataType::kHALF || type == nvinfer1::DataType::kBF16) - { - bpe = 2; - } - else if (type == nvinfer1::DataType::kINT8 || type == nvinfer1::DataType::kFP8) - { - bpe = 1; - } - else - { - TLLM_THROW("Not recognized/implemented"); - } - return bpe; -} - -void GemmSwigluPluginProfiler::setQuantMode(tensorrt_llm::common::QuantMode const& quantMode) -{ - mQuantMode = quantMode; -} - -void GemmSwigluPluginProfiler::runTactic( - int m, int n, int k, GemmSwigluPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) -{ - size_t bpe = getBytePerElement(mType); - - // Workspace size required by gemm runner - // NB: this function will throw exception when selected tactic exceeds SMEM, which is then - // caught by gemmPluginProfiler and it will register this tactic as invalid - size_t wsSizeRunner = mRunner->getWorkspaceSize(m, n, k); - - // Workspace size required by profiling - size_t wsByteOffset = 0; - int8_t* wsBytePointer = reinterpret_cast<int8_t*>(workspace); - void* aTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, m * k * bpe)); - void* bTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, n * k * bpe)); - void* cTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, 1 * n * bpe)); - void* dTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, m * (n / 2) * bpe)); - char* workspaceTmp = reinterpret_cast<char*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, wsSizeRunner)); - - // Run profiling - mRunner->gemm( - dTmp, aTmp, bTmp, cTmp, mQuantMode, m, n, k, 1.0, 1.0, 1.0, tactic, workspaceTmp, wsSizeRunner, stream); -} - -int GemmSwigluPluginProfiler::getMaxProfileM() const -{ - return 32768; -} - -void GemmSwigluPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - std::vector<size_t> workspaces = { - maxM * k * getBytePerElement(mType), // A - n * k * getBytePerElement(mType), // B - 1 * n * getBytePerElement(mType), // C_bias - maxM * (n / 2) * getBytePerElement(mType), // D - mRunner->getWorkspaceSize(maxM, n, k) // workspace - }; - size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); - setTmpWorkspaceSizeInBytes(bytes); -} - -std::vector<GemmSwigluPluginProfiler::Config> GemmSwigluPluginProfiler::getTactics(int m, int n, int k) const -{ - return mRunner->getConfigs(); -} - -GemmSwigluPlugin::GemmSwigluPlugin(QuantMode quantMode, nvinfer1::DataType type, bool hasBias, float scale_d0, - float scale_d1, float scale_output, GemmSwigluPlugin::PluginProfilerPtr const& pluginProfiler) - : mQuantMode(quantMode) - , mPluginProfiler(pluginProfiler) - , mHasBias(hasBias) - , mScaleD0(scale_d0) - , mScaleD1(scale_d1) - , mScaleOutput(scale_output) -{ - init(type); -} - -// Parameterized constructor -GemmSwigluPlugin::GemmSwigluPlugin( - void const* data, size_t length, GemmSwigluPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - nvinfer1::DataType type; - unsigned int quantMode; - read(d, quantMode); - read(d, type); - read(d, mHasBias); - read(d, mScaleD0); - read(d, mScaleD1); - read(d, mScaleOutput); - read(d, mDims); - - mQuantMode = QuantMode(quantMode); - - init(type); - - mPluginProfiler->deserialize(d, mDims, mGemmId); - - TLLM_CHECK(d == a + length); -} - -void GemmSwigluPlugin::init(nvinfer1::DataType type) -{ - mType = type; - if (mType == nvinfer1::DataType::kFP8) - { - mGemmRunner = std::make_shared<CutlassFusedGatedGemmRunner<__nv_fp8_e4m3>>(); - } - else - { - TLLM_THROW("Gemm Swiglu plugin only supports fp8 now"); - } - - mPluginProfiler->setQuantMode(mQuantMode); - - mGemmId = GemmIdCore(mDims.n, mDims.k, mType); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* GemmSwigluPlugin::clone() const noexcept -{ - auto* plugin = new GemmSwigluPlugin(*this); - return plugin; -} - -nvinfer1::DimsExprs GemmSwigluPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs == 3); - TLLM_CHECK(outputIndex == 0); - int const nbDimsA = inputs[0].nbDims; - TLLM_CHECK(nbDimsA >= 2); - DimsExprs ret; - ret.nbDims = nbDimsA; - for (int ii = 0; ii < nbDimsA - 1; ++ii) - { - ret.d[ii] = inputs[0].d[ii]; - } - ret.d[nbDimsA - 1] = exprBuilder.constant(inputs[1].d[1]->getConstantValue() / 2); - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool GemmSwigluPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - switch (pos) - { - case 0: - // activation - return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - case 1: - // weights - return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - case 2: - // bias - return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - case 3: - // out - return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - default: - // Never should be here - TLLM_CHECK(false); - return false; - } -} - -void GemmSwigluPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); - auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); - - int const maxK = in[0].max.d[in[0].max.nbDims - 1]; - int const maxN = in[1].max.d[1]; - int const minK = in[0].min.d[in[0].min.nbDims - 1]; - int const minN = in[1].min.d[1]; - - TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); - TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, maxN, maxK}; - } - mGemmId = {maxN, maxK, mType}; - - mWorkspaceMaxSize = mGemmRunner->getWorkspaceSize(maxM, maxN, maxK); -} - -size_t GemmSwigluPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return mWorkspaceMaxSize; -} - -int GemmSwigluPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - // inputs - // mat1 [M(*), K] - // mat2 [K, N] - // bias [1, N] - // outputs - // mat [M(*), N / 2] - int m = 1; - for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) - { - m *= inputDesc[0].dims.d[ii]; - } - int const n = inputDesc[1].dims.d[1]; - int const k = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; - size_t const wsSize = mGemmRunner->getWorkspaceSize(m, n, k); - - auto const bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); - TLLM_CHECK_WITH_INFO(bestTactic, "No valid GEMM tactic"); - mGemmRunner->gemm(outputs[0], inputs[0], inputs[1], inputs[2], mQuantMode, m, n, k, mScaleD0, mScaleD1, - mScaleOutput, *bestTactic, reinterpret_cast<char*>(workspace), wsSize, stream); - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType GemmSwigluPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* GemmSwigluPlugin::getPluginType() const noexcept -{ - return GEMM_SWIGLU_PLUGIN_NAME; -} - -char const* GemmSwigluPlugin::getPluginVersion() const noexcept -{ - return GEMM_SWIGLU_PLUGIN_VERSION; -} - -int GemmSwigluPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int GemmSwigluPlugin::initialize() noexcept -{ - configGemm(); // gemm profiler in action - return 0; -} - -void GemmSwigluPlugin::terminate() noexcept {} - -size_t GemmSwigluPlugin::getSerializationSize() const noexcept -{ - return sizeof(unsigned int) + // QuantMode - sizeof(nvinfer1::DataType) + // dtype - sizeof(bool) + // hasBias - sizeof(float) * 3 + // scales - sizeof(mDims) + // Dimensions - mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size -} - -void GemmSwigluPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mQuantMode.value()); - write(d, mType); - write(d, mHasBias); - write(d, mScaleD0); - write(d, mScaleD1); - write(d, mScaleOutput); - write(d, mDims); - - mPluginProfiler->serialize(d, mGemmId); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void GemmSwigluPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -void GemmSwigluPlugin::configGemm() -{ - mPluginProfiler->profileTactics(mGemmRunner, mType, mDims, mGemmId); -} - -/////////////// - -GemmSwigluPluginCreator::GemmSwigluPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("has_bias", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("scale_d0", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("scale_d1", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("scale_output", nullptr, PluginFieldType::kFLOAT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* GemmSwigluPluginCreator::getPluginName() const noexcept -{ - return GEMM_SWIGLU_PLUGIN_NAME; -} - -char const* GemmSwigluPluginCreator::getPluginVersion() const noexcept -{ - return GEMM_SWIGLU_PLUGIN_VERSION; -} - -PluginFieldCollection const* GemmSwigluPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* GemmSwigluPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - TLLM_CHECK(fc->nbFields == 5); - nvinfer1::DataType type{}; - bool hasBias{}; - float scale_d0{}; - float scale_d1{}; - float scale_output{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "has_bias")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - hasBias = static_cast<bool>(*(static_cast<int8_t const*>(fields[i].data))); - } - else if (!strcmp(attrName, "scale_d0")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - scale_d0 = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - else if (!strcmp(attrName, "scale_d1")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - scale_d1 = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - else if (!strcmp(attrName, "scale_output")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - scale_output = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - } - try - { - // GemmSwigluPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); - QuantMode quantMode = QuantMode{}; - auto* obj = new GemmSwigluPlugin(quantMode, type, hasBias, scale_d0, scale_d1, scale_output, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* GemmSwigluPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call GemmSwigluPlugin::destroy() - try - { - // Create plugin profiler with private tactics map which is read from the serialized engine - auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); - auto* obj = new GemmSwigluPlugin(serialData, serialLength, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cu b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cu deleted file mode 100644 index 339c432b1113..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cu +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "gemmSwigluPlugin.h" - -#include "cutlass/util/reference/device/tensor_fill.h" -#include "cutlass_extensions/gemm_configs.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using namespace tensorrt_llm::kernels::cutlass_kernels; -using tensorrt_llm::plugins::GemmSwigluPluginCreator; -using tensorrt_llm::plugins::GemmSwigluPlugin; -using tensorrt_llm::plugins::GemmSwigluPluginProfiler; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; - -void GemmSwigluPluginProfiler::initTmpData(int m, int n, int k, char* workspace, size_t size, cudaStream_t stream) -{ - size_t bpe = getBytePerElement(mType); - - if (mType == nvinfer1::DataType::kFP8) - { - cutlass::reference::device::BlockFillRandomUniform(reinterpret_cast<cutlass::float_e4m3_t*>(workspace), - m * k + n * k + 1 * n, 42, cutlass::float_e4m3_t{128}, -cutlass::float_e4m3_t{128}, -1, 0, stream); - } -} diff --git a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.h b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.h deleted file mode 100644 index 766e59aad258..000000000000 --- a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.h +++ /dev/null @@ -1,150 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/kernels/cutlass_kernels/fused_gated_gemm/fused_gated_gemm.h" -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -using GemmSwigluRunnerPtr - = std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFusedGatedGemmRunnerInterface>; - -class GemmSwigluPluginProfiler : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - GemmSwigluRunnerPtr, GemmIdCore, GemmIdCoreHash> - -{ -public: - using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; - - void setQuantMode(tensorrt_llm::common::QuantMode const& quantMode); - - virtual int getMaxProfileM() const override; - -protected: - void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; - - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - - // TODO(anchengc) implement checkTactic - // bool checkTactic(int m, int n, int k, const Config& tactic) const override; - - std::vector<Config> getTactics(int m, int n, int k) const override; - - void initTmpData(int m, int n, int k, char* workspace, size_t size, cudaStream_t stream) override; - -private: - size_t getBytePerElement(nvinfer1::DataType type); - - tensorrt_llm::common::QuantMode mQuantMode; -}; - -class GemmSwigluPlugin : public BasePlugin -{ -public: - using PluginProfilerPtr = std::shared_ptr<GemmSwigluPluginProfiler>; - - GemmSwigluPlugin() = delete; - - GemmSwigluPlugin(tensorrt_llm::common::QuantMode quantMode, nvinfer1::DataType type, bool hasBias, float scale_d0, - float scale_d1, float scale_output, PluginProfilerPtr const& pluginProfiler); - - GemmSwigluPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); - - ~GemmSwigluPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - void init(nvinfer1::DataType type); - - void configGemm(); - // void setGemmConfig(); - -private: - const std::string mLayerName; - - GemmSwigluRunnerPtr mGemmRunner; - tensorrt_llm::common::QuantMode mQuantMode; // not configurable yet - size_t mWorkspaceMaxSize; - - GemmDims mDims{}; - GemmIdCore mGemmId{}; - - PluginProfilerPtr mPluginProfiler; - - nvinfer1::DataType mType; - bool mHasBias; - float mScaleD0; - float mScaleD1; - float mScaleOutput; -}; - -class GemmSwigluPluginCreator : public BaseCreator -{ -public: - GemmSwigluPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - GemmPluginProfilerManager<GemmSwigluPluginProfiler> mGemmPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gptAttentionCommon/CMakeLists.txt b/cpp/tensorrt_llm/plugins/gptAttentionCommon/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/gptAttentionCommon/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.cpp b/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.cpp deleted file mode 100644 index 717ab3083e5f..000000000000 --- a/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.cpp +++ /dev/null @@ -1,380 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gptAttentionCommon.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/kernels/decoderMaskedMultiheadAttention/decoderXQARunner.h" -#include "tensorrt_llm/kernels/gptKernels.h" -#include <NvInferRuntimePlugin.h> -#include <cstdint> - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -namespace tc = tensorrt_llm::common; -using tensorrt_llm::plugins::GPTAttentionPluginCreatorCommon; -using tensorrt_llm::plugins::GPTAttentionPluginCommon; - -GPTAttentionPluginCommon::GPTAttentionPluginCommon(int layer_idx, int num_heads, int vision_start, int vision_length, - int num_kv_heads, int num_kv_heads_origin, int head_size, int unidirectional, float q_scaling, - float attn_logit_softcapping_scale, tensorrt_llm::kernels::PositionEmbeddingType position_embedding_type, - int rotary_embedding_dim, // for RoPE. Use 0 for non-RoPE - float rotary_embedding_base, tensorrt_llm::kernels::RotaryScalingType rotary_embedding_scale_type, - float rotary_embedding_scale, float rotary_embedding_short_m_scale, float rotary_embedding_long_m_scale, - int rotary_embedding_max_positions, int rotary_embedding_original_max_positions, int tp_size, - int tp_rank, // for ALiBi - bool unfuse_qkv_gemm, // for AutoPP - bool use_logn_scaling, // for LognScaling - tensorrt_llm::kernels::ContextFMHAType context_fmha_type, int kv_cache_quant_mode, bool remove_input_padding, - tensorrt_llm::kernels::AttentionMaskType mask_type, tensorrt_llm::kernels::BlockSparseParams block_sparse_params, - bool paged_kv_cache, int tokens_per_block, nvinfer1::DataType type, int32_t max_context_length, - bool qkv_bias_enabled, bool cross_attention, int max_distance, bool pos_shift_enabled, bool dense_context_fmha, - bool use_paged_context_fmha, bool use_fp8_context_fmha, bool has_full_attention_mask, bool use_cache, - bool is_spec_decoding_enabled, bool spec_decoding_is_generation_length_variable, - int32_t spec_decoding_max_generation_length, bool is_mla_enabled, int q_lora_rank, int kv_lora_rank, - int qk_nope_head_dim, int qk_rope_head_dim, int v_head_dim, bool fuse_fp4_quant, bool skip_attn, int cp_size, - int cp_rank, std::set<int32_t> cp_group) - : mResource{DecoderXQARunner::getResourceGlobal()} -{ - mLayerIdx = layer_idx; - mNumHeads = num_heads; - mVisionStart = vision_start; - mVisionLength = vision_length; - mNumKVHeads = num_kv_heads; - mNumKVHeadsOrigin = num_kv_heads_origin; - mHeadSize = head_size; - mUnidirectional = unidirectional; - mQScaling = q_scaling; - mAttnLogitSoftcappingScale = attn_logit_softcapping_scale; - mRotaryEmbeddingDim = rotary_embedding_dim; - mRotaryEmbeddingBase = rotary_embedding_base; - mRotaryEmbeddingScaleType = rotary_embedding_scale_type; - mRotaryEmbeddingScale = rotary_embedding_scale; - mRotaryEmbeddingShortMscale = rotary_embedding_short_m_scale; - mRotaryEmbeddingLongMscale = rotary_embedding_long_m_scale; - mRotaryEmbeddingMaxPositions = rotary_embedding_max_positions; - mRotaryEmbeddingOriginalMaxPositions = rotary_embedding_original_max_positions; - mPositionEmbeddingType = position_embedding_type; - mEnableContextFMHA = context_fmha_type != ContextFMHAType::DISABLED; - mFMHAForceFP32Acc = type == nvinfer1::DataType::kBF16; - mMaskType = mask_type; - mBlockSparseParams = block_sparse_params; - mType = type; - mMultiBlockMode = true; - mEnableXQA = true; - mKVCacheQuantMode = tc::QuantMode(kv_cache_quant_mode); - mRemovePadding = remove_input_padding; - mPagedKVCache = paged_kv_cache; - mTokensPerBlock = tokens_per_block; - mTpSize = tp_size; - mTpRank = tp_rank; - mUnfuseQkvGemm = unfuse_qkv_gemm; - mUseLognScaling = use_logn_scaling; - mMaxContextLength = max_context_length; - mQKVBiasEnabled = qkv_bias_enabled; - mCrossAttention = cross_attention; - mMaxDistance = max_distance; - mPosShiftEnabled = pos_shift_enabled; - mDenseContextFMHA = dense_context_fmha; - mPagedContextFMHA = use_paged_context_fmha; - mFP8ContextFMHA = use_fp8_context_fmha; - mFP8AttenOutput = use_fp8_context_fmha; - mHasFullAttentionMask = has_full_attention_mask; - mUseKVCache = use_cache; - mIsSpecDecodingEnabled = is_spec_decoding_enabled; - mSpecDecodingIsGenerationLengthVariable = spec_decoding_is_generation_length_variable; - mSpecDecodingMaxGenerationLength = spec_decoding_max_generation_length; - mIsMLAEnabled = is_mla_enabled; - mMLAParams = {q_lora_rank, kv_lora_rank, qk_nope_head_dim, qk_rope_head_dim, v_head_dim}; - mCpSize = cp_size; - mCpRank = cp_rank; - mCpGroup = std::move(cp_group); - mFuseFp4Quant = fuse_fp4_quant; - mSkipAttn = skip_attn; -} - -// Parameterized constructor -GPTAttentionPluginCommon::GPTAttentionPluginCommon(void const* data, size_t length) - : mResource{DecoderXQARunner::getResourceGlobal()} -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - unsigned int kvCacheQuantMode; - - read(d, mLayerIdx); - read(d, mNumHeads); - read(d, mVisionStart); - read(d, mVisionLength); - read(d, mNumKVHeads); - read(d, mNumKVHeadsOrigin); - read(d, mHeadSize); - read(d, mUnidirectional); - read(d, mQScaling); - read(d, mAttnLogitSoftcappingScale); - read(d, mPositionEmbeddingType); - read(d, mRotaryEmbeddingDim); - read(d, mRotaryEmbeddingBase); - read(d, mRotaryEmbeddingScaleType); - read(d, mRotaryEmbeddingScale); - read(d, mRotaryEmbeddingShortMscale); - read(d, mRotaryEmbeddingLongMscale); - read(d, mRotaryEmbeddingMaxPositions); - read(d, mRotaryEmbeddingOriginalMaxPositions); - read(d, mTpSize); - read(d, mTpRank); - read(d, mUnfuseQkvGemm); - read(d, mUseLognScaling); - read(d, mEnableContextFMHA); - read(d, mFMHAForceFP32Acc); - read(d, mMultiBlockMode); - read(d, mEnableXQA); - read(d, kvCacheQuantMode); - read(d, mRemovePadding); - read(d, mMaskType); - read(d, mBlockSparseParams); - read(d, mPagedKVCache); - read(d, mTokensPerBlock); - read(d, mType); - read(d, mMaxContextLength); - read(d, mQKVBiasEnabled); - read(d, mCrossAttention); - read(d, mMaxDistance); - read(d, mPosShiftEnabled); - read(d, mDenseContextFMHA); - read(d, mPagedContextFMHA); - read(d, mFP8ContextFMHA); - read(d, mFP8AttenOutput); - read(d, mHasFullAttentionMask); - read(d, mUseKVCache); - read(d, mIsSpecDecodingEnabled); - read(d, mUseSpecDecoding); - read(d, mSpecDecodingIsGenerationLengthVariable); - read(d, mSpecDecodingMaxGenerationLength); - read(d, mIsMLAEnabled); - read(d, mMLAParams); - read(d, mNbMultiBlockSemaphores); - read(d, mFuseFp4Quant); - read(d, mSkipAttn); - read(d, mCpSize); - read(d, mCpRank); - - mKVCacheQuantMode = tc::QuantMode(kvCacheQuantMode); - - uint32_t decoderXQARunnerResourceSerializedSize; - read(d, decoderXQARunnerResourceSerializedSize); - mResource->merge(DecoderXQARunnerResource(d, decoderXQARunnerResourceSerializedSize), /*initialize=*/true); - d += decoderXQARunnerResourceSerializedSize; - - mCpGroup.clear(); - int32_t groupItem = 0; - while (d != a + length) - { - read(d, groupItem); - mCpGroup.insert(groupItem); - } - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); - TLLM_CHECK_WITH_INFO((smVersion() >= 80) || (mType != nvinfer1::DataType::kBF16), - "Unsupported data type, pre SM 80 GPUs do not support bfloat16"); -} - -int GPTAttentionPluginCommon::initialize() noexcept -{ - return AttentionOp::initialize(); -} - -void GPTAttentionPluginCommon::destroy() noexcept -{ - delete this; -} - -size_t GPTAttentionPluginCommon::getCommonSerializationSize() const noexcept -{ - return sizeof(mLayerIdx) + sizeof(mNumHeads) + +sizeof(mVisionStart) + sizeof(mVisionLength) + sizeof(mNumKVHeads) - + sizeof(mNumKVHeadsOrigin) + sizeof(mHeadSize) + sizeof(mUnidirectional) + sizeof(mQScaling) - + sizeof(mAttnLogitSoftcappingScale) + sizeof(mPositionEmbeddingType) + sizeof(mRotaryEmbeddingDim) - + sizeof(mRotaryEmbeddingBase) + sizeof(mRotaryEmbeddingScaleType) + sizeof(mRotaryEmbeddingScale) - + sizeof(mRotaryEmbeddingShortMscale) + sizeof(mRotaryEmbeddingLongMscale) - + sizeof(mRotaryEmbeddingMaxPositions) + sizeof(mRotaryEmbeddingOriginalMaxPositions) + sizeof(mTpSize) - + sizeof(mTpRank) + sizeof(mEnableContextFMHA) + sizeof(mFMHAForceFP32Acc) + sizeof(mMultiBlockMode) - + sizeof(mEnableXQA) + sizeof(unsigned int) // mKVCacheQuantMode - + sizeof(mRemovePadding) + sizeof(mMaskType) + sizeof(mBlockSparseParams) + sizeof(mPagedKVCache) - + sizeof(mTokensPerBlock) + sizeof(mType) + sizeof(mMaxContextLength) + sizeof(mQKVBiasEnabled) - + sizeof(mCrossAttention) + sizeof(mMaxDistance) + sizeof(mPosShiftEnabled) + sizeof(mDenseContextFMHA) - + sizeof(mPagedContextFMHA) + sizeof(mFP8ContextFMHA) + sizeof(mFP8AttenOutput) + sizeof(mHasFullAttentionMask) - + sizeof(mUseKVCache) + sizeof(mUnfuseQkvGemm) + sizeof(mUseLognScaling) + sizeof(mIsSpecDecodingEnabled) - + sizeof(mUseSpecDecoding) + sizeof(mSpecDecodingIsGenerationLengthVariable) - + sizeof(mSpecDecodingMaxGenerationLength) + sizeof(mNbMultiBlockSemaphores) + sizeof(mIsMLAEnabled) - + sizeof(mMLAParams) + sizeof(mFuseFp4Quant) + sizeof(mSkipAttn) - + sizeof(uint32_t) // size of DecoderXQARunnerResource buffer. - + sizeof(mCpSize) + sizeof(mCpRank) + sizeof(int32_t) * mCpGroup.size() + mResource->getSerializationSize(); -} - -void GPTAttentionPluginCommon::serializeCommon(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mLayerIdx); - write(d, mNumHeads); - write(d, mVisionStart); - write(d, mVisionLength); - write(d, mNumKVHeads); - write(d, mNumKVHeadsOrigin); - write(d, mHeadSize); - write(d, mUnidirectional); - write(d, mQScaling); - write(d, mAttnLogitSoftcappingScale); - write(d, mPositionEmbeddingType); - write(d, mRotaryEmbeddingDim); - write(d, mRotaryEmbeddingBase); - write(d, mRotaryEmbeddingScaleType); - write(d, mRotaryEmbeddingScale); - write(d, mRotaryEmbeddingShortMscale); - write(d, mRotaryEmbeddingLongMscale); - write(d, mRotaryEmbeddingMaxPositions); - write(d, mRotaryEmbeddingOriginalMaxPositions); - write(d, mTpSize); - write(d, mTpRank); - write(d, mUnfuseQkvGemm); - write(d, mUseLognScaling); - write(d, mEnableContextFMHA); - write(d, mFMHAForceFP32Acc); - write(d, mMultiBlockMode); - write(d, mEnableXQA); - write(d, mKVCacheQuantMode.value()); - write(d, mRemovePadding); - write(d, mMaskType); - write(d, mBlockSparseParams); - write(d, mPagedKVCache); - write(d, mTokensPerBlock); - write(d, mType); - write(d, mMaxContextLength); - write(d, mQKVBiasEnabled); - write(d, mCrossAttention); - write(d, mMaxDistance); - write(d, mPosShiftEnabled); - write(d, mDenseContextFMHA); - write(d, mPagedContextFMHA); - write(d, mFP8ContextFMHA); - write(d, mFP8AttenOutput); - write(d, mHasFullAttentionMask); - write(d, mUseKVCache); - write(d, mIsSpecDecodingEnabled); - write(d, mUseSpecDecoding); - write(d, mSpecDecodingIsGenerationLengthVariable); - write(d, mSpecDecodingMaxGenerationLength); - write(d, mIsMLAEnabled); - write(d, mMLAParams); - write(d, mNbMultiBlockSemaphores); - write(d, mFuseFp4Quant); - write(d, mSkipAttn); - write(d, mCpSize); - write(d, mCpRank); - - // An uint32_t that specifies the size of the serialized buffer, followed by the actual content. - uint32_t decoderXQARunnerResourceSerializedSize = mResource->getSerializationSize(); - write(d, decoderXQARunnerResourceSerializedSize); - mResource->serialize(d, decoderXQARunnerResourceSerializedSize); - d += decoderXQARunnerResourceSerializedSize; - - for (auto it = mCpGroup.begin(); it != mCpGroup.end(); ++it) - { - write(d, *it); - } - TLLM_CHECK(d == a + getCommonSerializationSize()); -} - -void GPTAttentionPluginCommon::terminate() noexcept -{ - // Do nothing, destroy will always be called, so release the resources there. -} - -/////////////// - -GPTAttentionPluginCreatorCommon::GPTAttentionPluginCreatorCommon() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("layer_idx", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("num_heads", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("vision_start", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("vision_length", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("num_kv_heads", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("num_kv_heads_origin", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("layer_idx_in_cache_pool", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("head_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("unidirectional", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("q_scaling", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("attn_logit_softcapping_scale", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("position_embedding_type", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("rotary_embedding_dim", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("rotary_embedding_base", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("rotary_embedding_scale_type", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("rotary_embedding_scale", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("rotary_embedding_short_m_scale", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("rotary_embedding_long_m_scale", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("rotary_embedding_max_positions", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back( - PluginField("rotary_embedding_original_max_positions", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("tp_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("tp_rank", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("unfuse_qkv_gemm", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("use_logn_scaling", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("context_fmha_type", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("kv_cache_quant_mode", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("remove_input_padding", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("mask_type", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("block_sparse_block_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("block_sparse_homo_head_pattern", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("block_sparse_num_local_blocks", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("block_sparse_vertical_stride", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("paged_kv_cache", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("tokens_per_block", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("max_context_length", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("qkv_bias_enabled", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("do_cross_attention", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("max_distance", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("pos_shift_enabled", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("dense_context_fmha", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("use_paged_context_fmha", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("use_fp8_context_fmha", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("has_full_attention_mask", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("use_cache", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("is_spec_decoding_enabled", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back( - PluginField("spec_decoding_is_generation_length_variable", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back( - PluginField("spec_decoding_max_generation_length", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("is_mla_enabled", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("q_lora_rank", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("kv_lora_rank", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("qk_nope_head_dim", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("qk_rope_head_dim", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("v_head_dim", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("fuse_fp4_quant", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("skip_attn", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("cp_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("cp_rank", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("cp_group", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -PluginFieldCollection const* GPTAttentionPluginCreatorCommon::getFieldNames() noexcept -{ - return &mFC; -} diff --git a/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h b/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h deleted file mode 100644 index dd87d67aab9e..000000000000 --- a/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/attentionOp.h" -#include "tensorrt_llm/common/cublasMMWrapper.h" -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/kernels/gptKernels.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::kernels -{ -class DecoderXQARunnerResource; -} - -namespace tensorrt_llm::plugins -{ - -class GPTAttentionPluginCommon : public BasePlugin, public tensorrt_llm::common::op::AttentionOp -{ -public: - GPTAttentionPluginCommon() = delete; - - GPTAttentionPluginCommon(int layer_idx, int num_heads, int vision_start, int vision_length, int num_kv_heads, - int num_kv_heads_origin, int head_size, int unidirectional, float q_scaling, float attn_logit_softcapping_scale, - tensorrt_llm::kernels::PositionEmbeddingType position_embedding_type, - int rotary_embedding_dim, // for RoPE. Use 0 for non-RoPE - float rotary_embedding_base, tensorrt_llm::kernels::RotaryScalingType rotary_embedding_scale_type, - float rotary_embedding_scale, float rotary_embedding_short_m_scale, float rotary_embedding_long_m_scale, - int rotary_embedding_max_positions, int rotary_embedding_original_max_positions, int tp_size, - int tp_rank, // for ALiBi - bool unfuse_qkv_gemm, // for AutoPP - bool use_logn_scaling, // for LognScaling - tensorrt_llm::kernels::ContextFMHAType context_fmha_type, int kv_cache_quant_mode, bool remove_input_padding, - tensorrt_llm::kernels::AttentionMaskType mask_type, - tensorrt_llm::kernels::BlockSparseParams block_sparse_params, bool paged_kv_cache, int tokens_per_block, - nvinfer1::DataType type, int32_t max_context_length, bool qkv_bias_enabled, bool cross_attention = false, - int max_distance = 0, bool pos_shift_enabled = false, bool dense_context_fmha = false, - bool use_paged_context_fmha = true, bool use_fp8_context_fmha = true, bool has_full_attention_mask = false, - bool use_cache = true, bool is_spec_decoding_enabled = false, - bool spec_decoding_is_generation_length_variable = false, int32_t spec_decoding_max_generation_length = 1, - bool is_mla_enabled = false, int q_lora_rank = 0, int kv_lora_rank = 0, int qk_nope_head_dim = 0, - int qk_rope_head_dim = 0, int v_head_dim = 0, bool fuse_fp4_quant = false, bool skip_attn = false, - int cp_size = 1, int cp_rank = 0, std::set<int32_t> cp_group = {}); - - GPTAttentionPluginCommon(void const* data, size_t length); - - ~GPTAttentionPluginCommon() override = default; - - template <typename T> - int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); - - //! This is called on every trt Engine creation - int initialize() noexcept override; - //! This is called on every trt Engine destroy - void terminate() noexcept override; - - //! This is called on every trt ExecutionContext creation by TRT - //! Note TRT does not call the initialize on cloned plugin, so clone internally should do initialization. - template <typename T> - T* cloneImpl() const noexcept; - - //! This is called on evert trt Engine or ExecutionContext destroy. - //! None-cloned plugins will call terminate and then call destroy, while the cloned plugins will call destroy only - //! So plugin should put the resource release inside destroy. - void destroy() noexcept override; - - size_t getCommonSerializationSize() const noexcept; - void serializeCommon(void* buffer) const noexcept; - -protected: - std::string const mLayerName; - -private: - std::shared_ptr<tensorrt_llm::kernels::DecoderXQARunnerResource> mResource; -}; - -class GPTAttentionPluginCreatorCommon : public BaseCreator -{ -public: - GPTAttentionPluginCreatorCommon(); - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - template <typename T> - T* deserializePluginImpl(char const* name, void const* serialData, size_t serialLength) noexcept; - -protected: - std::vector<nvinfer1::PluginField> mPluginAttributes; - nvinfer1::PluginFieldCollection mFC{}; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommonImpl.h b/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommonImpl.h deleted file mode 100644 index 51462cee6f40..000000000000 --- a/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommonImpl.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "gptAttentionCommon.h" - -namespace tensorrt_llm::plugins -{ -template <typename T> -T* GPTAttentionPluginCommon::cloneImpl() const noexcept -{ - static_assert(std::is_base_of_v<GPTAttentionPluginCommon, T>); - auto* plugin = new T(static_cast<T const&>(*this)); - plugin->setPluginNamespace(mNamespace.c_str()); - - // Cloned plugins should be in initialized state with correct resources ready to be enqueued. - plugin->initialize(); - return plugin; -} - -template <typename T> -T* GPTAttentionPluginCreatorCommon::deserializePluginImpl( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call GPTAttentionPluginCommon::destroy() - try - { - auto* obj = new T(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gptAttentionPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/gptAttentionPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/gptAttentionPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.cpp b/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.cpp deleted file mode 100644 index 6f8c41c94131..000000000000 --- a/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.cpp +++ /dev/null @@ -1,1387 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "gptAttentionPlugin.h" - -#include "tensorrt_llm/batch_manager/contextProgress.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/kernels/decoderMaskedMultiheadAttention.h" -#include "tensorrt_llm/kernels/gptKernels.h" -#include "tensorrt_llm/kernels/unfusedAttentionKernels.h" -#include "tensorrt_llm/plugins/common/checkMacrosPlugin.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include "tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h" -#include "tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommonImpl.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/utils/debugUtils.h" - -#include <NvInferRuntimeBase.h> -#include <algorithm> -#include <cstdint> -#include <functional> -#include <numeric> - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::GPTAttentionPluginCreator; -using tensorrt_llm::plugins::GPTAttentionPlugin; - -static char const* GPT_ATTENTION_PLUGIN_VERSION{"1"}; -static char const* GPT_ATTENTION_PLUGIN_NAME{"GPTAttention"}; - -GPTAttentionPlugin::GPTAttentionPlugin(int layer_idx, int num_heads, int vision_start, int vision_length, - int num_kv_heads, int num_kv_heads_origin, int head_size, int unidirectional, float q_scaling, - float attn_logit_softcapping_scale, tensorrt_llm::kernels::PositionEmbeddingType position_embedding_type, - int rotary_embedding_dim, // for RoPE. 0 for non-RoPE - float rotary_embedding_base, tensorrt_llm::kernels::RotaryScalingType rotary_embedding_scale_type, - float rotary_embedding_scale, float rotary_embedding_short_m_scale, - float rotary_embedding_long_m_scale, // magnitude scaling factors for Phi-3 long RoPE - int rotary_embedding_max_positions, int rotary_embedding_original_max_positions, int tp_size, - int tp_rank, // for ALiBi - bool unfuse_qkv_gemm, // for AutoPP - bool use_logn_scaling, // for LognScaling - tensorrt_llm::kernels::ContextFMHAType context_fmha_type, int kv_cache_quant_mode, bool remove_input_padding, - tensorrt_llm::kernels::AttentionMaskType mask_type, tensorrt_llm::kernels::BlockSparseParams block_sparse_params, - bool paged_kv_cache, int tokens_per_block, nvinfer1::DataType type, int32_t max_context_length, - bool qkv_bias_enabled, bool cross_attention, int max_distance, bool pos_shift_enabled, bool dense_context_fmha, - bool use_paged_context_fmha, bool use_fp8_context_fmha, bool has_full_attention_mask, bool use_cache, - bool is_spec_decoding_enabled, bool spec_decoding_is_generation_length_variable, - int spec_decoding_max_generation_length, bool is_mla_enabled, int q_lora_rank, int kv_lora_rank, - int qk_nope_head_dim, int qk_rope_head_dim, int v_head_dim, bool fuse_fp4_quant, bool skip_attn, int cp_size, - int cp_rank, std::set<int32_t> cp_group) - : GPTAttentionPluginCommon(layer_idx, num_heads, vision_start, vision_length, num_kv_heads, num_kv_heads_origin, - head_size, unidirectional, q_scaling, attn_logit_softcapping_scale, position_embedding_type, - rotary_embedding_dim, rotary_embedding_base, rotary_embedding_scale_type, rotary_embedding_scale, - rotary_embedding_short_m_scale, rotary_embedding_long_m_scale, rotary_embedding_max_positions, - rotary_embedding_original_max_positions, tp_size, tp_rank, unfuse_qkv_gemm, use_logn_scaling, context_fmha_type, - kv_cache_quant_mode, remove_input_padding, mask_type, block_sparse_params, paged_kv_cache, tokens_per_block, - type, max_context_length, qkv_bias_enabled, cross_attention, max_distance, pos_shift_enabled, - dense_context_fmha, use_paged_context_fmha, use_fp8_context_fmha, has_full_attention_mask, use_cache, - is_spec_decoding_enabled, spec_decoding_is_generation_length_variable, spec_decoding_max_generation_length, - is_mla_enabled, q_lora_rank, kv_lora_rank, qk_nope_head_dim, qk_rope_head_dim, v_head_dim, fuse_fp4_quant, - skip_attn, cp_size, cp_rank, cp_group) -{ - TLLM_CHECK_WITH_INFO( - !is_mla_enabled, "GPTAttentionPlugin no longer supports MLA. Please use the PyTorch workflow instead."); - initEntryIdx(); -} - -GPTAttentionPlugin::GPTAttentionPlugin(void const* data, size_t length) - : GPTAttentionPluginCommon(data, length) -{ - initEntryIdx(); -} - -std::string GPTAttentionPlugin::toString(IdxEntry const& entry) const -{ -#define TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(name) \ - case IdxEntry::name: return #name - - switch (entry) - { - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(QKV_TENSOR); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(K_TENSOR); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(V_TENSOR); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ATTENTION_MASK); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ATTENTION_PACKED_MASK); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SEQUENCE_LENGTH); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_PAST_KEY_VALUE_LENGTHS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_MAX_ATTENTION_WINDOW); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_SINK_TOKEN_LENGTH); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(CONTEXT_LENGTHS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(CACHE_INDIR); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(REQUEST_TYPES); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(KV_CACHE_BLOCK_OFFSETS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_KV_CACHE_BLOCK_OFFSETS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_KV_CACHE_POOL_POINTERS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_KV_CACHE_POOL_MAPPING); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(PAST_KEY_VALUE); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(KV_CACHE_QUANTIZATION_SCALE); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(KV_CACHE_DEQUANTIZATION_SCALE); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ATTENTION_OUTPUT_QUANTIZATION_SCALE); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ATTENTION_OUTPUT_SF_SCALE); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ROTARY_INV_FREQ); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ROTARY_COS_SIN); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ALIBI_SLOPES); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(RELATIVE_ATTENTION_BIAS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(CROSS_KV); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(CROSS_KV_LENGTH); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ENCODER_INPUT_LENGTH); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_CONTEXT_LENGTH); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(QKV_BIAS_TENSOR); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SPEC_DECODING_GENERATION_LENGTHS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SPEC_DECODING_PACKED_MASK); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SPEC_DECODING_POSITION_OFFSETS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SPEC_DECODING_USE); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(LONG_ROPE_ROTARY_INV_FREQ); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(LONG_ROPE_ROTARY_COS_SIN); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(MROPE_ROTARY_COS_SIN); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(MROPE_POSITION_DELTAS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_RUNTIME_PERF_KNOBS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_CONTEXT_PROGRESS); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(MLA_Q_B_PROJ_TENSOR); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(MLA_KV_B_PROJ_TENSOR); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(MLA_K_B_PROJ_TRANS_TENSOR); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SKIP_ATTN); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(LOGN_SCALING); - TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ENUM_SIZE); - } -#undef TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING - TLLM_LOG_TRACE(common::fmtstr("Missing string description for IdxEntry enum %lu.\n", static_cast<size_t>(entry))); - return ""; -} - -bool GPTAttentionPlugin::isEntryUsed(IdxEntry const& entry) const -{ - switch (entry) - { - case IdxEntry::QKV_TENSOR: return true; - case IdxEntry::K_TENSOR: return mUnfuseQkvGemm; - case IdxEntry::V_TENSOR: return mUnfuseQkvGemm; - case IdxEntry::ATTENTION_MASK: return useFullCustomMask(); - case IdxEntry::ATTENTION_PACKED_MASK: return useCustomMask(); - case IdxEntry::SEQUENCE_LENGTH: return useKVCache(); - case IdxEntry::HOST_PAST_KEY_VALUE_LENGTHS: return useKVCache(); - case IdxEntry::HOST_MAX_ATTENTION_WINDOW: return true; - case IdxEntry::HOST_SINK_TOKEN_LENGTH: return true; - case IdxEntry::CONTEXT_LENGTHS: return true; - case IdxEntry::CACHE_INDIR: return useKVCache(); - case IdxEntry::REQUEST_TYPES: return true; - case IdxEntry::KV_CACHE_BLOCK_OFFSETS: return useKVCache() && mPagedKVCache; - case IdxEntry::HOST_KV_CACHE_BLOCK_OFFSETS: return useKVCache() && mPagedKVCache; - case IdxEntry::HOST_KV_CACHE_POOL_POINTERS: return useKVCache() && mPagedKVCache; - case IdxEntry::HOST_KV_CACHE_POOL_MAPPING: return useKVCache() && mPagedKVCache; - case IdxEntry::PAST_KEY_VALUE: return useKVCache() && !mPagedKVCache; - case IdxEntry::KV_CACHE_QUANTIZATION_SCALE: return useKVCache() && mKVCacheQuantMode.hasKvCacheQuant(); - case IdxEntry::KV_CACHE_DEQUANTIZATION_SCALE: return useKVCache() && mKVCacheQuantMode.hasKvCacheQuant(); - case IdxEntry::ATTENTION_OUTPUT_QUANTIZATION_SCALE: return mFP8ContextFMHA; - case IdxEntry::ATTENTION_OUTPUT_SF_SCALE: return mFuseFp4Quant; - case IdxEntry::ROTARY_INV_FREQ: return isRoPE(); - case IdxEntry::ROTARY_COS_SIN: return isRoPE(); - case IdxEntry::ALIBI_SLOPES: return isALiBi(); - case IdxEntry::RELATIVE_ATTENTION_BIAS: return isRelativePosition(); - case IdxEntry::CROSS_KV: return isCrossAttention(); - case IdxEntry::CROSS_KV_LENGTH: return isCrossAttention(); - case IdxEntry::LOGN_SCALING: return isLognScaling(); - case IdxEntry::ENCODER_INPUT_LENGTH: return isCrossAttention(); - case IdxEntry::HOST_CONTEXT_LENGTH: return mRemovePadding; - case IdxEntry::QKV_BIAS_TENSOR: return mQKVBiasEnabled; - case IdxEntry::SPEC_DECODING_GENERATION_LENGTHS: return mIsSpecDecodingEnabled; - case IdxEntry::SPEC_DECODING_PACKED_MASK: return mIsSpecDecodingEnabled; - case IdxEntry::SPEC_DECODING_POSITION_OFFSETS: return mIsSpecDecodingEnabled; - case IdxEntry::SPEC_DECODING_USE: return mIsSpecDecodingEnabled; - case IdxEntry::LONG_ROPE_ROTARY_INV_FREQ: return isLongRoPE(); - case IdxEntry::LONG_ROPE_ROTARY_COS_SIN: return isLongRoPE(); - case IdxEntry::MROPE_ROTARY_COS_SIN: return isMRoPE(); - case IdxEntry::MROPE_POSITION_DELTAS: return isMRoPE(); - case IdxEntry::HOST_RUNTIME_PERF_KNOBS: return true; - case IdxEntry::HOST_CONTEXT_PROGRESS: return true; - case IdxEntry::MLA_Q_B_PROJ_TENSOR: return mIsMLAEnabled; - case IdxEntry::MLA_KV_B_PROJ_TENSOR: return mIsMLAEnabled; - case IdxEntry::MLA_K_B_PROJ_TRANS_TENSOR: return mIsMLAEnabled; - case IdxEntry::SKIP_ATTN: return mSkipAttn; - default: return false; - } -} - -void GPTAttentionPlugin::initEntryIdx() -{ - mEntryIdx.resize(static_cast<size_t>(IdxEntry::ENUM_SIZE)); - size_t entryIdx = 0; - for (size_t i = 0; i < static_cast<size_t>(IdxEntry::ENUM_SIZE); i++) - { - mEntryIdx[i] = entryIdx; - entryIdx += isEntryUsed(static_cast<IdxEntry>(i)); - } -} - -GPTAttentionPlugin::IndexType GPTAttentionPlugin::getIdx(IdxEntry const& entry) const -{ - TLLM_CHECK_WITH_INFO( - isEntryUsed(entry), common::fmtstr("getIdx() should not be used with entry %s.\n", toString(entry).data())); - return mEntryIdx[static_cast<size_t>(entry)]; -} - -// IPluginV2DynamicExt Methods -GPTAttentionPlugin* GPTAttentionPlugin::clone() const noexcept -{ - return dynamic_cast<GPTAttentionPlugin*>(this->cloneImpl<GPTAttentionPlugin>()); -} - -static int getPackedTensorHiddenDimIndex(bool removePadding) -{ - return removePadding ? 1 : 2; -} - -// NOTE: generation input length might be larger than one in the spec decoding mode. -int GPTAttentionPlugin::getGenerationInputSequenceLength( - nvinfer1::PluginTensorDesc const* inputDesc, int32_t localNbSeq, int32_t localNbTokens) const -{ - if (mRemovePadding) - { - // Speculative decoding mode might need variable generation input sequence length. - if (mIsSpecDecodingEnabled && mUseSpecDecoding) - { - TLLM_CHECK_WITH_INFO(mCpSize <= 1, "Context Parallel does not support speculative decoding mode for now"); - // SPEC_DECODING_POSITION_OFFSETS: [batch_size, max_generation_input_length]. - return inputDesc[getIdx(IdxEntry::SPEC_DECODING_POSITION_OFFSETS)].dims.d[1]; - } - else - { - if (mCpSize > 1) - { - // Given that localNbTokens == (beamSize * localNbSeq + mCpSize - 1) / mCpSize, but when mCpSize - 1 > - // localNbSeq, there are multiple choices for beamSize. Assume beamSize == 1 here. - TLLM_CHECK_WITH_INFO(localNbTokens == (localNbSeq + mCpSize - 1) / mCpSize, - "Context Parallel does not support beamSize > 1 for non-speculative decoding mode, " - "localNbTokens=%d, localNbSeq=%d", - localNbTokens, localNbSeq); - return 1; - } - // [num_tokens, local_hidden_size] where num_tokens = batch_size * generation_input_length - TLLM_CHECK_WITH_INFO(localNbTokens % localNbSeq == 0, - "seq_len should be same for all generation requests, localNbTokens=%d, localNbSeq=%d", localNbTokens, - localNbSeq); - return localNbTokens / localNbSeq; - } - } - else - { - // We don't have IFB without mRemovePadding, so just take it out from inputDesc - // [batch_size, seq_len, local_hidden_size] - return inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[1]; - } -} - -// outputs -// output_tensor [batch_size, seq_len, local_hidden_size] or [num_tokens, local_hidden_size] -// present_key_value_pool (optional if mPagedKVCache is false) [batch_size, 2, local_num_kv_heads, max_seq_len, -// head_size] -nvinfer1::DimsExprs GPTAttentionPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - if (mFuseFp4Quant) - { - TLLM_CHECK(outputIndex == 0 || outputIndex == 1 || (!mPagedKVCache && useKVCache() && outputIndex == 2)); - // Compute the output dimension for FP4 quantized tensor. Consistent with QuantizeToFP4Plugin. - if (outputIndex == 0) - { - auto ret = inputs[getIdx(IdxEntry::QKV_TENSOR)]; - return ret; - } - // Compute the output dimension for output scaling factor tensor. Consistent with QuantizeToFP4Plugin. - if (outputIndex == 1) - { - auto ret = inputs[getIdx(IdxEntry::QKV_TENSOR)]; - // Sequence dimension or token dimension. - // Pad to multiple of 128. - auto dimM = exprBuilder.operation(DimensionOperation::kCEIL_DIV, - *ret.d[getPackedTensorHiddenDimIndex(mRemovePadding) - 1], *exprBuilder.constant(128)); - ret.d[getPackedTensorHiddenDimIndex(mRemovePadding) - 1] - = exprBuilder.operation(DimensionOperation::kPROD, *dimM, *exprBuilder.constant(128)); - // Hidden size dimension. - // Div (rounding up) by 16 since 16 elements share one SF and SF padded to k%4==0. - ret.d[getPackedTensorHiddenDimIndex(mRemovePadding)] = exprBuilder.operation(DimensionOperation::kCEIL_DIV, - *ret.d[getPackedTensorHiddenDimIndex(mRemovePadding)], *exprBuilder.constant(16)); - return ret; - } - } - else - { - TLLM_CHECK(outputIndex == 0 || (!mPagedKVCache && useKVCache() && outputIndex == 1)); - if (outputIndex == 0) - { - auto ret = inputs[getIdx(IdxEntry::QKV_TENSOR)]; - // In MLA, the output dim is v_head_dim - auto const head_size = mHeadSize; - ret.d[getPackedTensorHiddenDimIndex(mRemovePadding)] = exprBuilder.operation( - DimensionOperation::kPROD, *exprBuilder.constant(head_size), *exprBuilder.constant(mNumHeads)); - return ret; - } - } - return inputs[getIdx(IdxEntry::PAST_KEY_VALUE)]; -} - -bool GPTAttentionPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - bool result = false; - int posCaseLine = -1; - if (pos == getIdx(IdxEntry::CONTEXT_LENGTHS) || pos == getIdx(IdxEntry::REQUEST_TYPES) - || pos == getIdx(IdxEntry::HOST_MAX_ATTENTION_WINDOW) || pos == getIdx(IdxEntry::HOST_SINK_TOKEN_LENGTH) - || (isEntryUsed(IdxEntry::SPEC_DECODING_PACKED_MASK) && pos == getIdx(IdxEntry::SPEC_DECODING_PACKED_MASK)) - || (isEntryUsed(IdxEntry::SPEC_DECODING_POSITION_OFFSETS) - && pos == getIdx(IdxEntry::SPEC_DECODING_POSITION_OFFSETS)) - || (isEntryUsed(IdxEntry::SPEC_DECODING_GENERATION_LENGTHS) - && pos == getIdx(IdxEntry::SPEC_DECODING_GENERATION_LENGTHS)) - || (isEntryUsed(IdxEntry::SPEC_DECODING_USE) && pos == getIdx(IdxEntry::SPEC_DECODING_USE))) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (isMRoPE() && (pos == getIdx(IdxEntry::MROPE_ROTARY_COS_SIN))) - { - return inOut[pos].type == nvinfer1::DataType::kFLOAT; - } - else if (isMRoPE() && (pos == getIdx(IdxEntry::MROPE_POSITION_DELTAS))) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (pos == getIdx(IdxEntry::HOST_RUNTIME_PERF_KNOBS) || pos == getIdx(IdxEntry::HOST_CONTEXT_PROGRESS)) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kINT64; - } - else if (useKVCache() - && (pos == getIdx(IdxEntry::SEQUENCE_LENGTH) || pos == getIdx(IdxEntry::HOST_PAST_KEY_VALUE_LENGTHS) - || pos == getIdx(IdxEntry::CACHE_INDIR))) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (isRoPE() && (pos == getIdx(IdxEntry::ROTARY_INV_FREQ) || pos == getIdx(IdxEntry::ROTARY_COS_SIN))) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kFLOAT; - } - else if (isLongRoPE() - && (pos == getIdx(IdxEntry::LONG_ROPE_ROTARY_INV_FREQ) || pos == getIdx(IdxEntry::LONG_ROPE_ROTARY_COS_SIN))) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kFLOAT; - } - else if (useKVCache() && mKVCacheQuantMode.hasKvCacheQuant() - && (pos == getIdx(IdxEntry::KV_CACHE_DEQUANTIZATION_SCALE) - || pos == getIdx(IdxEntry::KV_CACHE_QUANTIZATION_SCALE))) - { - // kv_scale for mType->int8/fp8 and int8/fp8->mType conversion - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (mFP8ContextFMHA && pos == getIdx(IdxEntry::ATTENTION_OUTPUT_QUANTIZATION_SCALE)) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (mFuseFp4Quant && pos == getIdx(IdxEntry::ATTENTION_OUTPUT_SF_SCALE)) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (useFullCustomMask() && pos == getIdx(IdxEntry::ATTENTION_MASK)) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kBOOL && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (useCustomMask() && pos == getIdx(IdxEntry::ATTENTION_PACKED_MASK)) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kINT32 && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (useKVCache() && mPagedKVCache - && (pos == getIdx(IdxEntry::KV_CACHE_BLOCK_OFFSETS) || pos == getIdx(IdxEntry::HOST_KV_CACHE_BLOCK_OFFSETS))) - { - // kv cache block offsets - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kINT32 && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (useKVCache() && mPagedKVCache && (pos == getIdx(IdxEntry::HOST_KV_CACHE_POOL_POINTERS))) - { - // kv cache pool pointers - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kINT64 && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (useKVCache() && mPagedKVCache && (pos == getIdx(IdxEntry::HOST_KV_CACHE_POOL_MAPPING))) - { - // kv cache pool mapping - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kINT32 && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (useKVCache() && mKVCacheQuantMode.hasInt8KvCache() - && (!mPagedKVCache && (pos == getIdx(IdxEntry::PAST_KEY_VALUE) || pos == nbInputs + 1))) - { - // If use Int8 K/V cache we require I/O KV values to int8 - posCaseLine = __LINE__; - result = (inOut[pos].type == nvinfer1::DataType::kINT8) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (useKVCache() && mKVCacheQuantMode.hasFp8KvCache() - && (!mPagedKVCache && (pos == getIdx(IdxEntry::PAST_KEY_VALUE) || pos == nbInputs + 1))) - { - // If use FP8 K/V cache we require I/O KV values to FP8 - posCaseLine = __LINE__; - result = (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (mRemovePadding && (pos == getIdx(IdxEntry::HOST_CONTEXT_LENGTH))) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kINT32 && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (mCrossAttention - && (pos == getIdx(IdxEntry::CROSS_KV_LENGTH) || pos == getIdx(IdxEntry::ENCODER_INPUT_LENGTH))) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (isLognScaling() && pos == getIdx(IdxEntry::LOGN_SCALING)) - { - return inOut[pos].type == nvinfer1::DataType::kFLOAT; - } - else if (pos == nbInputs && mFuseFp4Quant) - { - // Set dtype for output FP4 quantized tensor. - posCaseLine = __LINE__; - result = (inOut[pos].type == nvinfer1::DataType::kFP4) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == nbInputs + 1 && mFuseFp4Quant) - { - // Set dtype for output scaling factor tensor. Use kINT32 as storage type (same as QuantizeToFP4Plugin). - posCaseLine = __LINE__; - result = (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == nbInputs && mFP8ContextFMHA) - { - // Output tensor now supports fp8 data type. - posCaseLine = __LINE__; - result = (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (mSkipAttn && pos == getIdx(IdxEntry::SKIP_ATTN)) - { - posCaseLine = __LINE__; - result = inOut[pos].type == nvinfer1::DataType::kBOOL && inOut[pos].format == TensorFormat::kLINEAR; - } - else - { - posCaseLine = __LINE__; - result = (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } - TLLM_LOG_DEBUG( - "%s: pos: %d, result: %d, posCaseLine: %d", __PRETTY_FUNCTION__, pos, static_cast<int>(result), posCaseLine); - return result; -} - -template <typename T, typename KVCacheBuffer> -void GPTAttentionPlugin::configurePluginImpl(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - TLLM_CHECK(mHeadSize > 0); - - int beamWidth = -1; - if (!isCrossAttention() && useKVCache()) - { - // desc_val == -1 means beam_width is not static, we should look at min/max/opt. - // - // In prepareEnqueueGeneration, we'll prepare for all cases where beam_width doesn't exceed max. - // TODO: pass min AND max to prepareEnqueueGeneration instead of max only. - int desc_val = in[getIdx(IdxEntry::CACHE_INDIR)].desc.dims.d[1]; - int max_val = in[getIdx(IdxEntry::CACHE_INDIR)].max.d[1]; - beamWidth = desc_val == -1 ? max_val : desc_val; - } - else - { - beamWidth = 1; - } - TLLM_CHECK(beamWidth != -1); - - // Commonly, cyclic_attention_window_size, and max_attention_window_size will be the same - // unless each layer has different attention window sizes. - // the kv_cache capacity. - int max_encoder_context_len = isCrossAttention() ? in[getIdx(IdxEntry::CROSS_KV_LENGTH)].desc.dims.d[0] : 0; - int const max_attention_window_size = isCrossAttention() - ? max_encoder_context_len - : (useKVCache() ? in[getIdx(IdxEntry::CACHE_INDIR)].desc.dims.d[2] : 0); - int const cyclic_attention_window_size = max_attention_window_size; - - int const num_requests = 256; - int const sink_token_length = 0; - - EnqueueGenerationParams<T> enqueueParams; - enqueueParams.max_attention_window_size = max_attention_window_size; - enqueueParams.cyclic_attention_window_size = cyclic_attention_window_size; - enqueueParams.max_cyclic_attention_window_size = cyclic_attention_window_size; - enqueueParams.sink_token_length = sink_token_length; - enqueueParams.beam_width = beamWidth; - enqueueParams.num_requests = num_requests; - - prepareEnqueueGeneration<T, KVCacheBuffer>(enqueueParams); - - // Always reserve SemaphoreArray (for multi-block mode) as MMHA may enable multi-block mode when shared memory is - // not enough. - auto const& ctxLenTensor = in[getIdx(IdxEntry::CONTEXT_LENGTHS)]; - TLLM_CHECK_DEBUG(ctxLenTensor.max.nbDims == 1); - int32_t const max_batch_beam = in[getIdx(IdxEntry::CONTEXT_LENGTHS)].max.d[0]; - reserveSemaphoreArray(mNumHeads * max_batch_beam); -} - -template <typename T> -void GPTAttentionPlugin::configurePluginDispatchKVCacheType(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - if (mPagedKVCache) - { - configurePluginImpl<T, KVBlockArray>(in, nbInputs, out, nbOutputs); - } - else - { - configurePluginImpl<T, KVLinearBuffer>(in, nbInputs, out, nbOutputs); - } -} - -void GPTAttentionPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - if (mType == nvinfer1::DataType::kHALF) - { - configurePluginDispatchKVCacheType<half>(in, nbInputs, out, nbOutputs); - } - else if (mType == nvinfer1::DataType::kFLOAT) - { - configurePluginDispatchKVCacheType<float>(in, nbInputs, out, nbOutputs); - } -#ifdef ENABLE_BF16 - else if (mType == nvinfer1::DataType::kBF16) - { - configurePluginDispatchKVCacheType<__nv_bfloat16>(in, nbInputs, out, nbOutputs); - } -#endif -} - -size_t GPTAttentionPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - int const max_context_length = mMaxContextLength; - int const cross_kv_length = isCrossAttention() ? inputs[getIdx(IdxEntry::CROSS_KV_LENGTH)].dims.d[0] : 0; - int const max_num_seq = inputs[getIdx(IdxEntry::CONTEXT_LENGTHS)].dims.d[0]; - auto const type = inputs[getIdx(IdxEntry::QKV_TENSOR)].type; - int const max_kv_cache_length - = isCrossAttention() ? cross_kv_length : (useKVCache() ? inputs[getIdx(IdxEntry::CACHE_INDIR)].dims.d[2] : 0); - int const max_num_tokens - = mRemovePadding ? inputs[getIdx(IdxEntry::QKV_TENSOR)].dims.d[0] : max_num_seq * max_context_length; - int const max_blocks_per_sequence - = (useKVCache() && mPagedKVCache) ? inputs[getIdx(IdxEntry::KV_CACHE_BLOCK_OFFSETS)].dims.d[3] : 0; - - size_t const context_workspace_size - = getWorkspaceSizeForContext(type, max_num_seq, max_context_length, cross_kv_length, max_num_tokens); - - size_t const generation_workspace_size = getWorkspaceSizeForGeneration( - type, max_num_seq, max_kv_cache_length, max_num_tokens, max_blocks_per_sequence); - - size_t attention_input_workspace_size = 0; - - if (mUnfuseQkvGemm) - { - int const local_hidden_units_q - = inputs[getIdx(IdxEntry::QKV_TENSOR)].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)]; - int const local_hidden_units_kv - = inputs[getIdx(IdxEntry::K_TENSOR)].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)]; - size_t const size = tensorrt_llm::runtime::BufferDataType(type).getSize(); - size_t const attention_input_size = size * max_num_tokens * (local_hidden_units_q + 2 * local_hidden_units_kv); - size_t workspaces[1]; - workspaces[0] = attention_input_size; - attention_input_workspace_size = tensorrt_llm::common::calculateTotalWorkspaceSize(workspaces, 1); - } - - return std::max(context_workspace_size, generation_workspace_size) + attention_input_workspace_size; -} - -static size_t getStride(nvinfer1::Dims const& dims, int n) -{ - TLLM_CHECK(n >= 0 && n < dims.nbDims); - return std::accumulate(dims.d + n + 1, dims.d + dims.nbDims, 1, std::multiplies<size_t>{}); -} - -template <typename T, typename AttentionOutT, typename KVCacheBuffer> -int GPTAttentionPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) -{ - TLLM_LOG_TRACE("Attention plugin start at layer %d", mLayerIdx); - - using runtime::RequestType; - - int32_t const nbSeq = inputDesc[getIdx(IdxEntry::CONTEXT_LENGTHS)].dims.d[0]; - RequestType const* reqTypes = static_cast<RequestType const*>(inputs[getIdx(IdxEntry::REQUEST_TYPES)]); - - int32_t nbContextRequests = 0; - int32_t contextTokenIdxEnd = 0; - int32_t contextTokenIdxEndForCp = 0; - // count context requests - for (int32_t seqIdx = 0; seqIdx < nbSeq; seqIdx++) - { - if (reqTypes[seqIdx] != RequestType::kCONTEXT) - { - break; - } - ++nbContextRequests; - contextTokenIdxEnd += mRemovePadding - ? static_cast<int32_t const*>(inputs[getIdx(IdxEntry::HOST_CONTEXT_LENGTH)])[seqIdx] - : inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[1]; - contextTokenIdxEndForCp += mRemovePadding - ? (static_cast<int32_t const*>(inputs[getIdx(IdxEntry::HOST_CONTEXT_LENGTH)])[seqIdx] + mCpSize - 1) - / mCpSize - : (inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[1] + mCpSize - 1) / mCpSize; - } - - for (int32_t seqIdx = nbContextRequests; seqIdx < nbSeq; seqIdx++) - { - TLLM_CHECK(reqTypes[seqIdx] == RequestType::kGENERATION); - } - - // mixed requests require mRemovePadding and mPagedKVCache - if (nbContextRequests != 0 && nbContextRequests != nbSeq) - { - TLLM_CHECK(mRemovePadding && mPagedKVCache); - } - - if (nbContextRequests > 0) - { - auto seqIdxBeg = 0; - auto tokenIdxBeg = 0; - auto localNbTokens = contextTokenIdxEnd; - enqueueSome<T, AttentionOutT, KVCacheBuffer>(seqIdxBeg, nbContextRequests, tokenIdxBeg, localNbTokens, - inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - - if (auto nbGenerationSeq = nbSeq - nbContextRequests; nbGenerationSeq > 0) - { - auto seqIdxBeg = nbContextRequests; - auto tokenIdxBeg = mCpSize > 1 ? contextTokenIdxEndForCp : contextTokenIdxEnd; - // if mRemovePadding is true, we may have IFB, and need to remove context tokens. - // if mRemovePadding is false, it is only generation requests, so just multiply batch_beam and seq_len (May not - // 1 for Parallel Decoding) - auto localNbTokens = mRemovePadding - ? inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[0] - tokenIdxBeg - : inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[0] * inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[1]; - enqueueSome<T, AttentionOutT, KVCacheBuffer>(seqIdxBeg, nbGenerationSeq, tokenIdxBeg, localNbTokens, inputDesc, - outputDesc, inputs, outputs, workspace, stream); - } - - sync_check_cuda_error(stream); - TLLM_LOG_TRACE("Attention plugin stop at layer %d", mLayerIdx); - - return 0; -} - -template <typename T, typename AttentionOutT, typename KVCacheBuffer> -int GPTAttentionPlugin::enqueueSome(int32_t seqIdxBeg, int32_t localNbSeq, int32_t tokenIdxBeg, int32_t localNbTokens, - nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) -{ - // relative_attention_bias [head_num, max_seq_len, max_seq_len] (optional in relative position) - // or [head_num, num_buckets] (optional in implicit relative attention) - // cross_kv [batch_size, seq_len, 2 * local_hidden_size] or [num_tokens, 2 * local_hidden_size] - // when enable remove_input_padding (optional in cross attention mode) - // cross_kv_length [int] max encoder input context length (optional in cross attention mode) - // encoder_input_lengths [batch_size] raw sequence lengths (optional in cross attention mode) - - using runtime::RequestType; - - auto const* const reqTypeInBatchPtr - = static_cast<RequestType const*>(inputs[getIdx(IdxEntry::REQUEST_TYPES)]) + seqIdxBeg; - bool const is_context = (reqTypeInBatchPtr[0] == RequestType::kCONTEXT); - - T const* attention_input = static_cast<T const*>(inputs[getIdx(IdxEntry::QKV_TENSOR)]) - + inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)] - * size_t(tokenIdxBeg); - - bool changeSpecDecodingMode = false; - if (mIsSpecDecodingEnabled) - { - bool useSpecDecoding - = static_cast<bool>(reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::SPEC_DECODING_USE)])[0]); - changeSpecDecodingMode = mUseSpecDecoding != useSpecDecoding; - mUseSpecDecoding = useSpecDecoding; - } - - [[maybe_unused]] MlaParams<T> mla_params; - - T const* qkv_bias = nullptr; - if (mQKVBiasEnabled) - { - qkv_bias = reinterpret_cast<T const*>(inputs[getIdx(IdxEntry::QKV_BIAS_TENSOR)]); - } - - // Note we still need context length during generation for MMHA optimization. - int32_t const max_context_q_len = [&]() - { - if (!mRemovePadding) - { - return static_cast<int>(inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[1]); - } - auto const host_context_lengths - = static_cast<int32_t const*>(inputs[getIdx(IdxEntry::HOST_CONTEXT_LENGTH)]) + seqIdxBeg; - return *std::max_element(host_context_lengths, host_context_lengths + localNbSeq); - }(); - - // Rotary inv_freq, cos_sin cache to avoid re-computing. - float const* rotary_inv_freq = nullptr; - float2 const* rotary_cos_sin = nullptr; - - bool const useLongRoPECache = isLongRoPE() && max_context_q_len > mRotaryEmbeddingOriginalMaxPositions; - if (isRoPE()) - { - auto inputName = useLongRoPECache ? IdxEntry::LONG_ROPE_ROTARY_INV_FREQ : IdxEntry::ROTARY_INV_FREQ; - rotary_inv_freq = reinterpret_cast<float const*>(inputs[getIdx(inputName)]); - } - if (isRoPE()) - { - auto inputName = useLongRoPECache ? IdxEntry::LONG_ROPE_ROTARY_COS_SIN : IdxEntry::ROTARY_COS_SIN; - rotary_cos_sin = reinterpret_cast<float2 const*>(inputs[getIdx(inputName)]); - } - - auto const mrope_rotary_cos_sin - = isMRoPE() ? reinterpret_cast<float2 const*>(inputs[getIdx(IdxEntry::MROPE_ROTARY_COS_SIN)]) : nullptr; - - auto const mrope_position_deltas - = isMRoPE() ? reinterpret_cast<int32_t const*>(inputs[getIdx(IdxEntry::MROPE_POSITION_DELTAS)]) : nullptr; - - if (mUnfuseQkvGemm) - { - int const max_seqlen = inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[mRemovePadding ? 0 : 1]; - int const batch_size = mRemovePadding ? 1 : inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[0]; - - T const* attention_input_q = static_cast<T const*>(inputs[getIdx(IdxEntry::QKV_TENSOR)]); - T const* attention_input_k = static_cast<T const*>(inputs[getIdx(IdxEntry::K_TENSOR)]); - T const* attention_input_v = static_cast<T const*>(inputs[getIdx(IdxEntry::V_TENSOR)]); - size_t const hidden_units_q - = inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)]; - size_t const hidden_units_kv - = inputDesc[getIdx(IdxEntry::K_TENSOR)].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)]; - size_t const hidden_units = hidden_units_q + 2 * hidden_units_kv; - size_t const size_qkv = sizeof(T) * hidden_units; - size_t const size_q = sizeof(T) * hidden_units_q; - size_t const size_kv = sizeof(T) * hidden_units_kv; - size_t const total_size = size_qkv * batch_size * max_seqlen; - int8_t* workspace_byte_ptr = reinterpret_cast<int8_t*>(workspace); - size_t offset = 0; - T* attention_input_qkv = reinterpret_cast<T*>(nextWorkspacePtr(workspace_byte_ptr, offset, total_size)); - workspace = reinterpret_cast<void*>(workspace_byte_ptr + offset); - - cudaMemcpy2DAsync(attention_input_qkv, size_qkv, attention_input_q, size_q, size_q, batch_size * max_seqlen, - cudaMemcpyDeviceToDevice, stream); - cudaMemcpy2DAsync(attention_input_qkv + hidden_units_q, size_qkv, attention_input_k, size_kv, size_kv, - batch_size * max_seqlen, cudaMemcpyDeviceToDevice, stream); - cudaMemcpy2DAsync(attention_input_qkv + hidden_units_q + hidden_units_kv, size_qkv, attention_input_v, size_kv, - size_kv, batch_size * max_seqlen, cudaMemcpyDeviceToDevice, stream); - - attention_input = attention_input_qkv + hidden_units * tokenIdxBeg; - } - - int const* context_q_lengths = reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::CONTEXT_LENGTHS)]) + seqIdxBeg; - int const* sequence_kv_length = useKVCache() - ? static_cast<int const*>(inputs[getIdx(IdxEntry::SEQUENCE_LENGTH)]) + seqIdxBeg - : context_q_lengths; - - int max_encoder_context_len = isCrossAttention() ? inputDesc[getIdx(IdxEntry::CROSS_KV_LENGTH)].dims.d[0] : 0; - // for enc-dec model, since decoder_input_ids could be longer than 1, - // such model has an encoder context (for cross attn) and an decoder context (for self attn) - // clarify 3 lens: - // -- max_context_q_len: len of decoder input. No "max" concept, it's what it is given. - // Also called (decoder_)input_seq_length, normally 1 for encoder-decoder start token - // -- max_seq_len: max allowed len of decoder output, i.e. final results - // -- max_encoder_context_len: len of encoder input (in cross attn). Also called encoder_input_seq_length - - int const beamWidth - = isCrossAttention() ? 1 : (useKVCache() ? inputDesc[getIdx(IdxEntry::CACHE_INDIR)].dims.d[1] : 1); - - // Commonly, cyclic_attention_window_size, and max_attention_window_size will be the same - // unless each layer has different attention window sizes. - // the kv_cache capacity. - int const max_attention_window_size = isCrossAttention() - ? max_encoder_context_len - : (useKVCache() ? inputDesc[getIdx(IdxEntry::CACHE_INDIR)].dims.d[2] : 0); - // The cyclic_attention_window_size will determine the cyclic kv cache position of new tokens. - // Note that this cyclic_attention_window_size might be smaller than the actual kv cache capactity. - int const* cyclic_attention_window_sizes - = reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::HOST_MAX_ATTENTION_WINDOW)]); - int const cyclic_attention_window_size - = isCrossAttention() ? max_encoder_context_len : cyclic_attention_window_sizes[mLayerIdx]; - int const sink_token_length = reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::HOST_SINK_TOKEN_LENGTH)])[0]; - int const num_attn_layer = inputDesc[getIdx(IdxEntry::HOST_MAX_ATTENTION_WINDOW)].dims.d[0]; - int const max_cyclic_attention_window_size = isCrossAttention() - ? max_encoder_context_len - : *std::max_element(cyclic_attention_window_sizes, cyclic_attention_window_sizes + num_attn_layer); - bool const can_use_one_more_block = beamWidth > 1; - - float const* kv_scale_orig_quant = nullptr; - float const* kv_scale_quant_orig = nullptr; - if (useKVCache() && mKVCacheQuantMode.hasKvCacheQuant()) - { - assert(inputDesc[getIdx(IdxEntry::KV_CACHE_QUANTIZATION_SCALE)].type == nvinfer1::DataType::kFLOAT); - assert(inputDesc[getIdx(IdxEntry::KV_CACHE_DEQUANTIZATION_SCALE)].type == nvinfer1::DataType::kFLOAT); - kv_scale_orig_quant = reinterpret_cast<float const*>(inputs[getIdx(IdxEntry::KV_CACHE_QUANTIZATION_SCALE)]); - kv_scale_quant_orig = reinterpret_cast<float const*>(inputs[getIdx(IdxEntry::KV_CACHE_DEQUANTIZATION_SCALE)]); - } - - float const* attention_output_orig_quant = nullptr; - if (mFP8ContextFMHA) - { - assert(inputDesc[getIdx(IdxEntry::ATTENTION_OUTPUT_QUANTIZATION_SCALE)].type == nvinfer1::DataType::kFLOAT); - attention_output_orig_quant - = reinterpret_cast<float const*>(inputs[getIdx(IdxEntry::ATTENTION_OUTPUT_QUANTIZATION_SCALE)]); - } - float const* attention_output_sf_scale = nullptr; - if (mFuseFp4Quant) - { - assert(inputDesc[getIdx(IdxEntry::ATTENTION_OUTPUT_SF_SCALE)].type == nvinfer1::DataType::kFLOAT); - attention_output_sf_scale = reinterpret_cast<float const*>(inputs[getIdx(IdxEntry::ATTENTION_OUTPUT_SF_SCALE)]); - } - uint32_t const* attention_packed_mask = nullptr; - if (useCustomMask()) - { - assert(inputDesc[getIdx(IdxEntry::ATTENTION_PACKED_MASK)].type == nvinfer1::DataType::kINT32); - attention_packed_mask = reinterpret_cast<uint32_t const*>(inputs[getIdx(IdxEntry::ATTENTION_PACKED_MASK)]); - } - bool const* attention_mask = nullptr; - int attention_mask_stride = 0; - if (useFullCustomMask()) - { - attention_mask_stride = static_cast<int>(inputDesc[getIdx(IdxEntry::ATTENTION_MASK)].dims.d[1]); - attention_mask = reinterpret_cast<bool const*>(inputs[getIdx(IdxEntry::ATTENTION_MASK)]) - + attention_mask_stride * static_cast<size_t>(tokenIdxBeg); - } - - int max_blocks_per_sequence = 0; - kernels::KVBlockArray::DataType* block_offsets = nullptr; - void* host_primary_pool_pointer = nullptr; - void* host_secondary_pool_pointer = nullptr; - if (useKVCache() && mPagedKVCache) - { - auto const& kvCacheBlockOffsetsShape = inputDesc[getIdx(IdxEntry::KV_CACHE_BLOCK_OFFSETS)].dims; - max_blocks_per_sequence = kvCacheBlockOffsetsShape.d[kvCacheBlockOffsetsShape.nbDims - 1]; - - std::int32_t const* host_pool_mapping - = static_cast<std::int32_t const*>(inputs[getIdx(IdxEntry::HOST_KV_CACHE_POOL_MAPPING)]); - - int32_t const layerToPool = host_pool_mapping[mLayerIdx * 2]; - int32_t const layerIdxInCachePool = host_pool_mapping[mLayerIdx * 2 + 1]; - TLLM_LOG_TRACE("Layer%d: LayerCachePoolLocator{.indexOfPool=%d, .layerIdxInCachePool=%d}", mLayerIdx, - layerToPool, layerIdxInCachePool); - auto const seqStride = getStride(kvCacheBlockOffsetsShape, 1); - auto const poolStride = getStride(kvCacheBlockOffsetsShape, 0); - auto const seqOffset = seqIdxBeg * seqStride; - auto const poolOffset = layerToPool * poolStride; - - block_offsets - = reinterpret_cast<kernels::KVBlockArray::DataType*>(inputs[getIdx(IdxEntry::KV_CACHE_BLOCK_OFFSETS)]) - + poolOffset + seqOffset; - - auto const* const typed_host_pool_pointers - = static_cast<char* const*>(inputs[getIdx(IdxEntry::HOST_KV_CACHE_POOL_POINTERS)]); - - auto const cacheElemSize = (mKVCacheQuantMode.hasKvCacheQuant() ? 1 : sizeof(T)); - - auto const kv_cache_head_num = (mNumKVHeads + mCpSize - 1) / mCpSize; - auto const blockSize = mTokensPerBlock * kv_cache_head_num * mHeadSize; - auto const bytesPerBlock = blockSize * cacheElemSize; - auto const layerOffset = layerIdxInCachePool * 2 * bytesPerBlock; - - host_primary_pool_pointer = reinterpret_cast<void*>(typed_host_pool_pointers[layerToPool * 2] + layerOffset); - host_secondary_pool_pointer - = reinterpret_cast<void*>(typed_host_pool_pointers[layerToPool * 2 + 1] + layerOffset); - } - - // The index of kv cache tensor in outputs. If fuse FP4 quant, an additional scaling factor output is added before - // the kv cache tensor. - int const kvCacheIdxInOutputs = mFuseFp4Quant ? 2 : 1; - // The number of elements per storage type. For FP4 output, storage type is uint8_t. - int const numEltsPerStorageType = mFuseFp4Quant ? 2 : 1; - - AttentionOutT* context_buf_ = static_cast<AttentionOutT*>(outputs[0]) - + outputDesc[0].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)] * tokenIdxBeg / numEltsPerStorageType; - - __nv_fp8_e4m3* context_buf_sf_ = nullptr; - if (mFuseFp4Quant) - { - // The output address for FP4 scaling factor. - context_buf_sf_ = static_cast<__nv_fp8_e4m3*>(outputs[1]); - } - - void* key_value_cache = nullptr; - if (useKVCache() && !mPagedKVCache) - { - auto const cacheElemSize = (mKVCacheQuantMode.hasKvCacheQuant() ? 1 : sizeof(T)); - key_value_cache = static_cast<std::byte*>(outputs[kvCacheIdxInOutputs]) - + cacheElemSize * getStride(outputDesc[kvCacheIdxInOutputs].dims, 0) * seqIdxBeg; - void const* past_key_value_cache = inputs[getIdx(IdxEntry::PAST_KEY_VALUE)]; - if (past_key_value_cache != outputs[kvCacheIdxInOutputs]) - { - auto shape = outputDesc[kvCacheIdxInOutputs].dims; - auto const size - = cacheElemSize * std::accumulate(shape.d, shape.d + shape.nbDims, 1, std::multiplies<size_t>{}); - cudaMemcpyAsync(outputs[kvCacheIdxInOutputs], past_key_value_cache, size, cudaMemcpyDeviceToDevice, stream); - } - } - - T const* alibi_slopes = isALiBi() ? static_cast<T const*>(inputs[getIdx(IdxEntry::ALIBI_SLOPES)]) : nullptr; - - int const* spec_decoding_packed_mask = nullptr; - int const* spec_decoding_position_offsets = nullptr; - int const* spec_decoding_generation_lengths = nullptr; - int num_decoding_draft_tokens = 0; - if (mIsSpecDecodingEnabled && mUseSpecDecoding) - { - // Second dimension of spec_decoding_position_offsets is num_decoding_draft_tokens + 1. - // [batch_size, num_decoding_draft_tokens + 1] - num_decoding_draft_tokens = inputDesc[getIdx(IdxEntry::SPEC_DECODING_POSITION_OFFSETS)].dims.d[1] - 1; - if (num_decoding_draft_tokens > 0) - { - // spec_decoding_* tensors are not filled for context requests. Hence, always strting from 0th index - int32_t constexpr genSeqIdx = 0; - spec_decoding_packed_mask = static_cast<int const*>(inputs[getIdx(IdxEntry::SPEC_DECODING_PACKED_MASK)]) - + genSeqIdx * getStride(inputDesc[getIdx(IdxEntry::SPEC_DECODING_PACKED_MASK)].dims, 0); - // Packed as [num_tokens, packed_mask_size] - // Use seqIdxBeg * (num_decoding_draft_tokens + 1) here as only generation tokens have the packed_mask - // buffer. - // TODO: support variable sequence length based on generationTokenIdxBeg. - spec_decoding_packed_mask = static_cast<int const*>(inputs[getIdx(IdxEntry::SPEC_DECODING_PACKED_MASK)]) - + genSeqIdx * (num_decoding_draft_tokens + 1) - * getStride(inputDesc[getIdx(IdxEntry::SPEC_DECODING_PACKED_MASK)].dims, 0); - spec_decoding_position_offsets - = static_cast<int const*>(inputs[getIdx(IdxEntry::SPEC_DECODING_POSITION_OFFSETS)]) - + genSeqIdx * getStride(inputDesc[getIdx(IdxEntry::SPEC_DECODING_POSITION_OFFSETS)].dims, 0); - spec_decoding_generation_lengths - = static_cast<int const*>(inputs[getIdx(IdxEntry::SPEC_DECODING_GENERATION_LENGTHS)]) + genSeqIdx; - } - } - - int32_t const* host_past_kv_len_list = useKVCache() - ? static_cast<int const*>(inputs[getIdx(IdxEntry::HOST_PAST_KEY_VALUE_LENGTHS)]) + seqIdxBeg - : nullptr; - int32_t const max_context_kv_len = useKVCache() - ? *std::max_element(host_past_kv_len_list, host_past_kv_len_list + localNbSeq) - : max_context_q_len; - - int const* host_context_lengths - = mRemovePadding ? reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::HOST_CONTEXT_LENGTH)]) : nullptr; - - int64_t const* runtime_perf_knobs = static_cast<int64_t const*>(inputs[getIdx(IdxEntry::HOST_RUNTIME_PERF_KNOBS)]); - - EnqueueParams<T> common_enqueue_params; - common_enqueue_params.attention_input = attention_input; - common_enqueue_params.qkv_bias = qkv_bias; - common_enqueue_params.attention_mask = attention_mask; - common_enqueue_params.rotary_inv_freq = rotary_inv_freq; - common_enqueue_params.rotary_cos_sin = rotary_cos_sin; - common_enqueue_params.max_attention_window_size = max_attention_window_size; - common_enqueue_params.cyclic_attention_window_size = cyclic_attention_window_size; - common_enqueue_params.max_cyclic_attention_window_size = max_cyclic_attention_window_size; - common_enqueue_params.can_use_one_more_block = can_use_one_more_block; - common_enqueue_params.sink_token_length = sink_token_length; - common_enqueue_params.kv_scale_orig_quant = kv_scale_orig_quant; - common_enqueue_params.kv_scale_quant_orig = kv_scale_quant_orig; - common_enqueue_params.attention_output_orig_quant = attention_output_orig_quant; - common_enqueue_params.attention_output_sf_scale = attention_output_sf_scale; - common_enqueue_params.alibi_slopes = alibi_slopes; - common_enqueue_params.context_buf = context_buf_; - common_enqueue_params.context_buf_sf = context_buf_sf_; - common_enqueue_params.key_value_cache = key_value_cache; - common_enqueue_params.block_offsets = block_offsets; - common_enqueue_params.host_primary_pool_pointer = host_primary_pool_pointer; - common_enqueue_params.host_secondary_pool_pointer = host_secondary_pool_pointer; - common_enqueue_params.num_tokens = localNbTokens; - common_enqueue_params.max_blocks_per_sequence = max_blocks_per_sequence; - common_enqueue_params.sequence_lengths = sequence_kv_length; - common_enqueue_params.context_lengths = context_q_lengths; - common_enqueue_params.host_context_lengths = host_context_lengths; - common_enqueue_params.workspace = workspace; - common_enqueue_params.runtime_perf_knobs = runtime_perf_knobs; - - if (isRelativePosition()) - { - common_enqueue_params.relative_attention_bias - = static_cast<T const*>(inputs[getIdx(IdxEntry::RELATIVE_ATTENTION_BIAS)]); - common_enqueue_params.relative_attention_bias_stride - = inputDesc[getIdx(IdxEntry::RELATIVE_ATTENTION_BIAS)].dims.d[1]; // max_seq_len or num_buckets - } - if (isLognScaling()) - { - common_enqueue_params.logn_scaling_ptr = static_cast<float const*>(inputs[getIdx(IdxEntry::LOGN_SCALING)]); - } - if (isCrossAttention()) - { - common_enqueue_params.encoder_input_lengths - = reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::ENCODER_INPUT_LENGTH)]) + seqIdxBeg; - } - - if (is_context) // context stage - { - int const batch_size = localNbSeq; - int const request_batch_size = batch_size; - // num of total tokens (without paddings when remove paddings). - int num_encoder_tokens = 0; - if (isCrossAttention()) - { - if (!mRemovePadding) - { - num_encoder_tokens = request_batch_size * max_encoder_context_len; - } - else - { - num_encoder_tokens = inputDesc[getIdx(IdxEntry::CROSS_KV)].dims.d[0]; - } - } - - common_enqueue_params.input_seq_length = max_context_q_len; - common_enqueue_params.max_past_kv_length = max_context_kv_len; - EnqueueContextParams<T> enqueue_params{common_enqueue_params}; - enqueue_params.attention_packed_mask = attention_packed_mask; - enqueue_params.batch_size = batch_size; - enqueue_params.mrope_rotary_cos_sin = mrope_rotary_cos_sin; - enqueue_params.total_kv_len = enqueue_params.num_tokens; - - if (isCrossAttention()) - { - enqueue_params.cross_kv = static_cast<T const*>(inputs[getIdx(IdxEntry::CROSS_KV)]); - enqueue_params.cross_kv_length = max_encoder_context_len; - enqueue_params.num_encoder_tokens = num_encoder_tokens; - } - - enqueueContext<T, KVCacheBuffer>(enqueue_params, stream); - - { - std::string const afterContexStr = "ctx attention at layer " + std::to_string(mLayerIdx); - TLLM_LOG_TRACE("GPTAttentionPlugin - %s", afterContexStr.c_str()); - - auto progress = static_cast<batch_manager::ContextProgress* const*>( - inputs[getIdx(IdxEntry::HOST_CONTEXT_PROGRESS)])[0]; - if (progress != nullptr) - { - progress->recordEvent(mLayerIdx, stream); - } - - if (!mFuseFp4Quant) - { - TLLM_CHECK_DEBUG_WITH_INFO( - tensorrt_llm::runtime::utils::tensorHasInvalid(localNbTokens, - outputDesc[0].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)], - mFP8ContextFMHA ? nvinfer1::DataType::kFP8 : mType, context_buf_, stream, afterContexStr) - == false, - "Found invalid number (NaN or Inf) in " + afterContexStr); - } - } - } - else // generation stage; max_context_q_len == input_seq_len == 1 - { - TLLM_CHECK_WITH_INFO(useKVCache(), "KV-cache-less is only supported for context"); - int batch_beam = localNbSeq; - TLLM_CHECK(batch_beam % beamWidth == 0); - int32_t const num_requests = batch_beam / beamWidth; - - int const* cache_indir - = beamWidth == 1 ? nullptr : reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::CACHE_INDIR)]); - - // Medusa: the max input sequence length if variable sequence length is needed. - int const input_seq_length = getGenerationInputSequenceLength(inputDesc, localNbSeq, localNbTokens); - int const max_past_kv_length = isCrossAttention() ? max_encoder_context_len : max_context_kv_len; - auto qkvDims = inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims; - TLLM_CHECK_WITH_INFO(input_seq_length == 1 || (mIsSpecDecodingEnabled && mUseSpecDecoding), - "Only speculative decoding mode supports input length > 1 in the generation phase, input_seq_length=%d, " - "mIsSpecDecodingEnabled=%s, nDims=%d, (" FMT_DIM ", " FMT_DIM ", " FMT_DIM ")", - input_seq_length, mIsSpecDecodingEnabled ? "true" : "false", qkvDims.nbDims, qkvDims.d[0], qkvDims.d[1], - qkvDims.d[2]); - TLLM_CHECK_WITH_INFO( - input_seq_length == num_decoding_draft_tokens + 1, "The generation input length is not expected."); - common_enqueue_params.input_seq_length = input_seq_length; - common_enqueue_params.max_past_kv_length = max_past_kv_length; - EnqueueGenerationParams<T> enqueue_params{common_enqueue_params}; - enqueue_params.beam_width = beamWidth; - enqueue_params.attention_mask_stride = attention_mask_stride; - enqueue_params.num_requests = num_requests; - enqueue_params.cache_indir = cache_indir; - enqueue_params.semaphores = multiBlockSemaphores(); - enqueue_params.host_past_key_value_lengths = host_past_kv_len_list; - enqueue_params.mrope_position_deltas = mrope_position_deltas; - if (mIsSpecDecodingEnabled && mUseSpecDecoding) - { - enqueue_params.spec_decoding_packed_mask = spec_decoding_packed_mask; - enqueue_params.spec_decoding_position_offsets = spec_decoding_position_offsets; - enqueue_params.spec_decoding_generation_lengths = spec_decoding_generation_lengths; - enqueue_params.spec_decoding_is_generation_length_variable = mSpecDecodingIsGenerationLengthVariable; - enqueue_params.spec_decoding_max_generation_length = mSpecDecodingMaxGenerationLength; - } - if (mFuseFp4Quant) - { - enqueue_params.start_token_idx_sf = tokenIdxBeg; - } - - if (changeSpecDecodingMode) - { - // mUseSpecDecoding is changed, need to re-prepare the DecoderXQARunner - prepareEnqueueGeneration<T, KVCacheBuffer>(enqueue_params); - } - - enqueueGeneration<T, KVCacheBuffer>(enqueue_params, stream); - - { - std::string const afterGenStr = "gen attention at layer " + std::to_string(mLayerIdx); - { - TLLM_CHECK_DEBUG_WITH_INFO( - tensorrt_llm::runtime::utils::tensorHasInvalid(localNbTokens, - outputDesc[0].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)], - mFP8ContextFMHA ? nvinfer1::DataType::kFP8 : mType, context_buf_, stream, afterGenStr) - == false, - "Found invalid number (NaN or Inf) in " + afterGenStr); - } - } - } - - return 0; -} - -template <typename T, typename AttentionOutT> -int GPTAttentionPlugin::enqueueDispatchKVCacheType(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) -{ - if (mPagedKVCache) - { - return enqueueImpl<T, AttentionOutT, KVBlockArray>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else - { - return enqueueImpl<T, AttentionOutT, KVLinearBuffer>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - return 0; -} - -int GPTAttentionPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - if (mSkipAttn) - { - bool const* SKIP_ATTN = reinterpret_cast<bool const*>(inputs[getIdx(IdxEntry::SKIP_ATTN)]); - if (SKIP_ATTN[0]) - { - return 0; - } - } - - if (mType == nvinfer1::DataType::kHALF) - { - if (mFuseFp4Quant) - { - return enqueueDispatchKVCacheType<half, uint8_t>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#ifdef ENABLE_FP8 - if (mFP8ContextFMHA) - { - return enqueueDispatchKVCacheType<half, __nv_fp8_e4m3>( - inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#endif - return enqueueDispatchKVCacheType<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (mType == nvinfer1::DataType::kFLOAT) - { - return enqueueDispatchKVCacheType<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#ifdef ENABLE_BF16 - else if (mType == nvinfer1::DataType::kBF16) - { - if (mFuseFp4Quant) - { - return enqueueDispatchKVCacheType<__nv_bfloat16, uint8_t>( - inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#ifdef ENABLE_FP8 - if (mFP8ContextFMHA) - { - return enqueueDispatchKVCacheType<__nv_bfloat16, __nv_fp8_e4m3>( - inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#endif - return enqueueDispatchKVCacheType<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#endif - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType GPTAttentionPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - if (mFuseFp4Quant) - { - TLLM_CHECK(index == 0 || index == 1 || (!mPagedKVCache && useKVCache() && index == 2)); - } - else - { - TLLM_CHECK(index == 0 || (!mPagedKVCache && useKVCache() && index == 1)); - } - if (index == 0) - { - if (mFuseFp4Quant) - { - return nvinfer1::DataType::kFP4; - } - return mFP8ContextFMHA && mEnableContextFMHA ? nvinfer1::DataType::kFP8 - : inputTypes[getIdx(IdxEntry::QKV_TENSOR)]; - } - if (mFuseFp4Quant && index == 1) - { - return nvinfer1::DataType::kFP8; - } - return inputTypes[getIdx(IdxEntry::PAST_KEY_VALUE)]; -} - -// IPluginV2 Methods - -char const* GPTAttentionPlugin::getPluginType() const noexcept -{ - return GPT_ATTENTION_PLUGIN_NAME; -} - -char const* GPTAttentionPlugin::getPluginVersion() const noexcept -{ - return GPT_ATTENTION_PLUGIN_VERSION; -} - -int GPTAttentionPlugin::getNbOutputs() const noexcept -{ - int nbOutputs = mFuseFp4Quant ? 2 : 1; - if (!mPagedKVCache && useKVCache()) - { - nbOutputs += 1; - } - return nbOutputs; -} - -size_t GPTAttentionPlugin::getSerializationSize() const noexcept -{ - return GPTAttentionPluginCommon::getCommonSerializationSize(); -} - -void GPTAttentionPlugin::serialize(void* buffer) const noexcept -{ - GPTAttentionPluginCommon::serializeCommon(buffer); -} - -/////////////// - -GPTAttentionPluginCreator::GPTAttentionPluginCreator() - : GPTAttentionPluginCreatorCommon() -{ -} - -char const* GPTAttentionPluginCreator::getPluginName() const noexcept -{ - return GPT_ATTENTION_PLUGIN_NAME; -} - -char const* GPTAttentionPluginCreator::getPluginVersion() const noexcept -{ - return GPT_ATTENTION_PLUGIN_VERSION; -} - -PluginFieldCollection const* GPTAttentionPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* GPTAttentionPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginFieldParser p{fc->nbFields, fc->fields}; - - try - { - auto* obj = new GPTAttentionPlugin(p.getScalar<int32_t>("layer_idx").value(), - p.getScalar<int32_t>("num_heads").value(), p.getScalar<int32_t>("vision_start").value(), - p.getScalar<int32_t>("vision_length").value(), p.getScalar<int32_t>("num_kv_heads").value(), - p.getScalar<int32_t>("num_kv_heads_origin").value(), p.getScalar<int32_t>("head_size").value(), - p.getScalar<int32_t>("unidirectional").value(), p.getScalar<float>("q_scaling").value(), - p.getScalar<float>("attn_logit_softcapping_scale").value(), - static_cast<PositionEmbeddingType>(p.getScalar<int8_t>("position_embedding_type").value()), - p.getScalar<int32_t>("rotary_embedding_dim").value(), p.getScalar<float>("rotary_embedding_base").value(), - static_cast<RotaryScalingType>(p.getScalar<int8_t>("rotary_embedding_scale_type").value()), - p.getScalar<float>("rotary_embedding_scale").value(), - p.getScalar<float>("rotary_embedding_short_m_scale").value(), - p.getScalar<float>("rotary_embedding_long_m_scale").value(), - p.getScalar<int32_t>("rotary_embedding_max_positions").value(), - p.getScalar<int32_t>("rotary_embedding_original_max_positions").value(), - static_cast<int32_t>(p.getScalar<int32_t>("tp_size").value()), - static_cast<int32_t>(p.getScalar<int32_t>("tp_rank").value()), - static_cast<bool>(p.getScalar<int8_t>("unfuse_qkv_gemm").value()), - static_cast<bool>(p.getScalar<int8_t>("use_logn_scaling").value()), - static_cast<ContextFMHAType>(p.getScalar<int8_t>("context_fmha_type").value()), - p.getScalar<int32_t>("kv_cache_quant_mode").value(), - static_cast<bool>(p.getScalar<int8_t>("remove_input_padding").value()), - static_cast<AttentionMaskType>(p.getScalar<int32_t>("mask_type").value()), - BlockSparseParams{p.getScalar<int32_t>("block_sparse_block_size").value(), - static_cast<bool>(p.getScalar<int8_t>("block_sparse_homo_head_pattern").value()), - p.getScalar<int32_t>("block_sparse_num_local_blocks").value(), - p.getScalar<int32_t>("block_sparse_vertical_stride").value()}, - static_cast<bool>(p.getScalar<int32_t>("paged_kv_cache").value()), - p.getScalar<int32_t>("tokens_per_block").value(), - static_cast<nvinfer1::DataType>(p.getScalar<int32_t>("type_id").value()), - p.getScalar<int32_t>("max_context_length").value(), - static_cast<bool>(p.getScalar<int8_t>("qkv_bias_enabled").value()), - static_cast<bool>(p.getScalar<int8_t>("do_cross_attention").value()), - static_cast<int32_t>(p.getScalar<int32_t>("max_distance").value()), - static_cast<bool>(p.getScalar<int8_t>("pos_shift_enabled").value()), - static_cast<bool>(p.getScalar<int8_t>("dense_context_fmha").value()), - static_cast<bool>(p.getScalar<int8_t>("use_paged_context_fmha").value()), - static_cast<bool>(p.getScalar<int8_t>("use_fp8_context_fmha").value()), - static_cast<bool>(p.getScalar<int8_t>("has_full_attention_mask").value()), - static_cast<bool>(p.getScalar<int32_t>("use_cache").value()), - static_cast<bool>(p.getScalar<int8_t>("is_spec_decoding_enabled").value()), - static_cast<bool>(p.getScalar<int8_t>("spec_decoding_is_generation_length_variable").value()), - p.getScalar<int32_t>("spec_decoding_max_generation_length").value(), - static_cast<int8_t>(p.getScalar<int8_t>("is_mla_enabled").value()), - static_cast<int32_t>(p.getScalar<int32_t>("q_lora_rank").value()), - static_cast<int32_t>(p.getScalar<int32_t>("kv_lora_rank").value()), - static_cast<int32_t>(p.getScalar<int32_t>("qk_nope_head_dim").value()), - static_cast<int32_t>(p.getScalar<int32_t>("qk_rope_head_dim").value()), - static_cast<int32_t>(p.getScalar<int32_t>("v_head_dim").value()), - static_cast<bool>(p.getScalar<int8_t>("fuse_fp4_quant").value()), - static_cast<bool>(p.getScalar<int8_t>("skip_attn").value()), - static_cast<int32_t>(p.getScalar<int32_t>("cp_size").value()), - static_cast<int32_t>(p.getScalar<int32_t>("cp_rank").value()), - static_cast<std::set<int32_t>>(p.getSet<int32_t>("cp_group").value())); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* GPTAttentionPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call GPTAttentionPlugin::destroy() - try - { - auto* obj = new GPTAttentionPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.h b/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.h deleted file mode 100644 index 3e34703c6221..000000000000 --- a/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.h +++ /dev/null @@ -1,259 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "checkMacrosPlugin.h" -#include "tensorrt_llm/common/cublasMMWrapper.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/common/stringUtils.h" -#include "tensorrt_llm/kernels/contextFusedMultiHeadAttention/fmhaRunner.h" -#include "tensorrt_llm/kernels/contextFusedMultiHeadAttention/fused_multihead_attention_common.h" -#include "tensorrt_llm/kernels/gptKernels.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include "tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h" -#include <cassert> -#include <cstddef> -#include <cstdint> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ -// batch_size = num_ctx_requests + num_gen_requests * beam_width -// num_ctx_requests = number of context requests (single sequence per request). -// num_gen_requests = number of generation requests (beam_width sequences per request). -// Context sequences have to appear first, generation sequences after - -// inputs (see GPTAttentionPlugin::isEntryUsed for when each tensor is actually used) -// 0. input_tensor [batch_size, seq_len, local_hidden_size + 2 * local_num_kv_heads * head_size] or -// [num_tokens, local_hidden_size + 2 * local_num_kv_heads * head_size] when -// enable_remove_input_padding -// 1. sequence_length [batch_size] (optional) -// 2. host_past_key_value_lengths [batch_size] (int32) (optional) -// 3. host_max_attention_window_sizes [num_layers] (int32) -// 4. host_sink_token_length [1] (int32) -// 5. context_lengths [batch_size] -// 6. cache_indir [num_gen_requests, beam_width, memory_max_len] (required in beamsearch) (optional) -// 7. host_request_types [batch_size] int32. 0: context; 1: generation: 2: none. When not in inflight-batching -// mode, -// all elements must be identical. -// 8. past_key_value_pool [batch_size, 2, local_num_kv_heads, max_seq_len, head_size] or -// block_offsets [batch_size, 2, max_blocks_per_seq] if paged kv cache (optional) -// 8.1 host_pool_pointers [2] if paged kv cache (optional) -// 9. kv_cache_quantization_scale [1] (optional) -// 10. kv_cache_dequantization_scale [1] (optional) -// 11. attention_output_quantization_scale [1] (on device, optional) -// 12. attention_mask [num_tokens, kv_seqlen] (on device, bool, optional) -// 13. attention_packed_mask [num_tokens, kv_seqlen / 32] (on device, uint32_t, optional) -// - pack masks by encoding multiple mask positions into a single 32-bit unsigned integer. -// - see kernels/contextMultiHeadAttention/fmhaPackedMask.cpp for more details. -// 14. rotary_inv_freq [head_size / 2] or [head_size] (longrope type) (float) (on device, optional) -// 15. rotary_cos_sin [max_num_embedding_positions, 2] (float) (on device, optional) -// 16. alibi_slopes [num_heads] (optional for ALiBi position embedding) -// 17. relative_attention_bias [num_heads] (optional for ALiBi position embedding) -// 18. host_context_lengths [batch_size] int32. (optional, required when remove_input_padding is true) -// 19. qkv_bias (optional) [local_hidden_size * 3] -// 20. spec_decoding_generation_lengths (optional, required when medusa is enabled) (int32_t) [batch_size] -// 21. spec_decoding_packed_mask (optional, required when medusa is enabled) (int32_t) [num_tokens, packed_mask_dim] -// packed_mask_dim = divUp(max_num_spec_decoding_tokens + 1, 32) -// 22. spec_decoding_position_offsets (optional, required when medusa is enabled) (int32_t) [batch_size, -// max_num_spec_decoding_tokens + 1] -// 23. spec_decoding_use (optional, bool) [1]: If it is set as true, enable speculative decoding -// 24. long_rope_rotary_inv_freq [head / 2] (float) (on device, optional) -// 25. long_rope_rotary_cos_sin [max_num_embedding_positions, 2] (float) (on device, optional) -// 26. host_runtime_perf_knobs (int64) -// 27. host_context_progress (void*) -// 28. position_id_tensor(MLA) [total_tokens], used for rope embedding in MLA -// 29. q_a_proj_tensor(MLA) [hidden_dim, c_q_dim + c_k_dim + ropd_dim], used to proj compacted QKV -// 30. q_a_layernorm_tensor(MLA) [c_q_dim], rmsnorm weight for compacted q -// 31. q_b_proj_tensor(MLA) [c_q_dim, head_num * head_size], weight for companted q to q in context -// 32. kv_a_proj_with_mqa_tensor(MLA) [c_q_dim, head_num * (c_k_dim + rope_dim)], weight for companted q to kdim in -// generation -// 33. kv_a_layernorm_tensor(MLA) [c_k_dim], rmsnorm weight for compacted kv -// 34. kv_b_proj_tensor(MLA) [c_k_dim, head_num * 2 * (head_size - rope_dim)], weight for compacted kv to kv in -// context -// 35. skip_attn (optional, bool) [1]: If it is set as true, skip the atteniton plugin and return -// directly. -// -// outputs -// output_tensor [batch_size, seq_len, local_hidden_size] -// present_key_value_pool (optional if not paged kv cache) [batch_size, 2, local_num_kv_heads, max_seq_len, -// head_size] - -class GPTAttentionPlugin : public GPTAttentionPluginCommon -{ -public: - GPTAttentionPlugin(int layer_idx, int num_heads, int vision_start, int vision_length, int num_kv_heads, - int num_kv_heads_origin, int head_size, int unidirectional, float q_scaling, float attn_logit_softcapping_scale, - tensorrt_llm::kernels::PositionEmbeddingType position_embedding_type, - int rotary_embedding_dim, // for RoPE. 0 for non-RoPE - float rotary_embedding_base, tensorrt_llm::kernels::RotaryScalingType rotary_embedding_scale_type, - float rotary_embedding_scale, float rotary_embedding_short_m_scale, float rotary_embedding_long_m_scale, - int rotary_embedding_max_positions, int rotary_embedding_original_max_positions, int tp_size, - int tp_rank, // for ALiBi - bool unfuse_qkv_gemm, // for AutoPP - bool use_logn_scaling, // for LognScaling - tensorrt_llm::kernels::ContextFMHAType context_fmha_type, int kv_cache_quant_mode, bool remove_input_padding, - tensorrt_llm::kernels::AttentionMaskType mask_type, - tensorrt_llm::kernels::BlockSparseParams block_sparse_params, bool paged_kv_cache, int tokens_per_block, - nvinfer1::DataType type, int32_t max_context_length, bool qkv_bias_enabled, bool cross_attention = false, - int max_distance = 0, bool pos_shift_enabled = false, bool dense_context_fmha = false, - bool use_paged_context_fmha = true, bool use_fp8_context_fmha = true, bool has_full_attention_mask = false, - bool use_cache = true, bool is_spec_decoding_enabled = false, - bool spec_decoding_is_generation_length_variable = false, int spec_decoding_max_generation_length = 1, - bool is_mla_enabled = false, int q_lora_rank = 0, int kv_lora_rank = 0, int qk_nope_head_dim = 0, - int qk_rope_head_dim = 0, int v_head_dim = 0, bool fuse_fp4_quant = false, bool skip_attn = false, - int cp_size = 1, int cp_rank = 0, std::set<int32_t> cp_group = {}); - - GPTAttentionPlugin(void const* data, size_t length); - - ~GPTAttentionPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - template <typename T, typename AttentionOutT, typename KVCacheBuffer> - int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); - - template <typename T, typename AttentionOutT = T> - int enqueueDispatchKVCacheType(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream); - - template <typename T, typename KVCacheBuffer> - void configurePluginImpl(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept; - template <typename T> - void configurePluginDispatchKVCacheType(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - - //! This is called on every trt ExecutionContext creation by TRT - //! Note TRT does not call the initialize on cloned plugin, so clone internally should do initialization. - GPTAttentionPlugin* clone() const noexcept override; - - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - -private: - template <typename T, typename AttentionOutT, typename KVCacheBuffer> - int enqueueSome(int32_t seqIdxBeg, int32_t localNbSeq, int32_t tokenIdxBeg, int32_t localNbTokens, - nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); - - using IndexType = std::int32_t; - - std::vector<size_t> mEntryIdx; - enum class IdxEntry : size_t - { - QKV_TENSOR, - K_TENSOR, - V_TENSOR, - ATTENTION_MASK, - ATTENTION_PACKED_MASK, - SEQUENCE_LENGTH, - HOST_PAST_KEY_VALUE_LENGTHS, - HOST_MAX_ATTENTION_WINDOW, - HOST_SINK_TOKEN_LENGTH, - CONTEXT_LENGTHS, - CACHE_INDIR, - REQUEST_TYPES, - KV_CACHE_BLOCK_OFFSETS, - HOST_KV_CACHE_BLOCK_OFFSETS, - HOST_KV_CACHE_POOL_POINTERS, - HOST_KV_CACHE_POOL_MAPPING, - PAST_KEY_VALUE, - KV_CACHE_QUANTIZATION_SCALE, - KV_CACHE_DEQUANTIZATION_SCALE, - ATTENTION_OUTPUT_QUANTIZATION_SCALE, - ATTENTION_OUTPUT_SF_SCALE, - ROTARY_INV_FREQ, - ROTARY_COS_SIN, - ALIBI_SLOPES, - RELATIVE_ATTENTION_BIAS, - CROSS_KV, - CROSS_KV_LENGTH, - ENCODER_INPUT_LENGTH, - HOST_CONTEXT_LENGTH, - QKV_BIAS_TENSOR, - SPEC_DECODING_GENERATION_LENGTHS, - SPEC_DECODING_PACKED_MASK, - SPEC_DECODING_POSITION_OFFSETS, - SPEC_DECODING_USE, - LONG_ROPE_ROTARY_INV_FREQ, - LONG_ROPE_ROTARY_COS_SIN, - MROPE_ROTARY_COS_SIN, - MROPE_POSITION_DELTAS, - HOST_RUNTIME_PERF_KNOBS, - HOST_CONTEXT_PROGRESS, - MLA_Q_B_PROJ_TENSOR, - MLA_KV_B_PROJ_TENSOR, - MLA_K_B_PROJ_TRANS_TENSOR, - SKIP_ATTN, - LOGN_SCALING, - ENUM_SIZE, // Used to count the number of IdxEntry, must put in last - }; - - std::string toString(IdxEntry const& entry) const; - bool isEntryUsed(IdxEntry const& entry) const; - void initEntryIdx(); - IndexType getIdx(IdxEntry const& entry) const; - - // Get generation input sequence length (might be larger than 1 in the speculative decoding mode). - int getGenerationInputSequenceLength( - nvinfer1::PluginTensorDesc const* inputDesc, int32_t localNbSeq, int32_t localNbTokens) const; -}; - -class GPTAttentionPluginCreator : public GPTAttentionPluginCreatorCommon -{ -public: - GPTAttentionPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/identityPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/identityPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/identityPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.cpp b/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.cpp deleted file mode 100644 index 109010e7a933..000000000000 --- a/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.cpp +++ /dev/null @@ -1,199 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "identityPlugin.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/iTensor.h" - -using namespace nvinfer1; -using tensorrt_llm::plugins::IdentityPluginCreator; -using tensorrt_llm::plugins::IdentityPlugin; - -static char const* IDENTITY_PLUGIN_VERSION{"1"}; -static char const* IDENTITY_PLUGIN_NAME{"Identity"}; -PluginFieldCollection IdentityPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> IdentityPluginCreator::mPluginAttributes; - -IdentityPlugin::IdentityPlugin() {} - -// Parameterized constructor -IdentityPlugin::IdentityPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* IdentityPlugin::clone() const noexcept -{ - auto* plugin = new IdentityPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs IdentityPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - return inputs[outputIndex]; -} - -bool IdentityPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - assert(0 <= pos && pos < 2); - PluginTensorDesc const& input = inOut[0]; - PluginTensorDesc const& output = inOut[1]; - switch (pos) - { - case 0: return input.format == nvinfer1::TensorFormat::kLINEAR; - case 1: return output.type == input.type && output.format == nvinfer1::TensorFormat::kLINEAR; - } - return false; -} - -void IdentityPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t IdentityPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -int IdentityPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - size_t count = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) - { - count *= inputDesc[0].dims.d[i]; - } - count *= tensorrt_llm::runtime::BufferDataType(inputDesc[0].type).getSize(); - - cudaMemcpyAsync(outputs[0], inputs[0], count, cudaMemcpyDeviceToDevice, stream); - - sync_check_cuda_error(stream); - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType IdentityPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - assert(index == 0); - return inputTypes[0]; -} - -// IPluginV2 Methods - -char const* IdentityPlugin::getPluginType() const noexcept -{ - return IDENTITY_PLUGIN_NAME; -} - -char const* IdentityPlugin::getPluginVersion() const noexcept -{ - return IDENTITY_PLUGIN_VERSION; -} - -int IdentityPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int IdentityPlugin::initialize() noexcept -{ - return 0; -} - -void IdentityPlugin::terminate() noexcept {} - -size_t IdentityPlugin::getSerializationSize() const noexcept -{ - return 0; -} - -void IdentityPlugin::serialize(void* buffer) const noexcept {} - -void IdentityPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -IdentityPluginCreator::IdentityPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* IdentityPluginCreator::getPluginName() const noexcept -{ - return IDENTITY_PLUGIN_NAME; -} - -char const* IdentityPluginCreator::getPluginVersion() const noexcept -{ - return IDENTITY_PLUGIN_VERSION; -} - -PluginFieldCollection const* IdentityPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* IdentityPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - try - { - auto* obj = new IdentityPlugin(); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* IdentityPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call IdentityPlugin::destroy() - try - { - auto* obj = new IdentityPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.h b/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.h deleted file mode 100644 index 9ab10601ae59..000000000000 --- a/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.h +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class IdentityPlugin : public BasePlugin -{ -public: - IdentityPlugin(); - - IdentityPlugin(void const* data, size_t length); - - ~IdentityPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - const std::string mLayerName; -}; - -class IdentityPluginCreator : public BaseCreator -{ -public: - IdentityPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/CMakeLists.txt deleted file mode 100755 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.cpp b/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.cpp deleted file mode 100644 index 02a40a00c919..000000000000 --- a/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.cpp +++ /dev/null @@ -1,472 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "layernormQuantizationPlugin.h" -#include "pluginUtils.h" -#include "tensorrt_llm/kernels/layernormKernels.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::LayernormQuantizationPluginCreator; -using tensorrt_llm::plugins::LayernormQuantizationPlugin; - -static char const* LAYERNORM_QUANTIZATION_PLUGIN_VERSION{"1"}; -static char const* LAYERNORM_QUANTIZATION_PLUGIN_NAME{"LayernormQuantization"}; -PluginFieldCollection LayernormQuantizationPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> LayernormQuantizationPluginCreator::mPluginAttributes; - -LayernormQuantizationPlugin::LayernormQuantizationPlugin(float eps, bool useDiffOfSquares, - bool dynamicActivationScaling, bool sumPerToken, bool clampValEnabled, tensorrt_llm::common::QuantMode quantMode, - nvinfer1::DataType type, nvinfer1::DataType outputType) - : mEps(eps) - , mUseDiffOfSquares(useDiffOfSquares) - , mDynActScaling(dynamicActivationScaling) - , mType(type) - , mOutputType(outputType) - , mClampValEnabled(clampValEnabled) - , mQuantMode(quantMode) - , mSumPerToken(sumPerToken) -{ - TLLM_CHECK_WITH_INFO(mOutputType == nvinfer1::DataType::kINT8 || mOutputType == nvinfer1::DataType::kFP8, - "Only int8 or fp8 output type is allowed."); - // Check if the quant mode is valid. - TLLM_CHECK_WITH_INFO(mQuantMode.hasPerTokenScaling(), "The quant mode is not valid."); -} - -// Parameterized constructor -LayernormQuantizationPlugin::LayernormQuantizationPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mEps); - read(d, mUseDiffOfSquares); - read(d, mDynActScaling); - read(d, mSumPerToken); - read(d, mClampValEnabled); - read(d, mQuantMode); - read(d, mType); - read(d, mOutputType); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* LayernormQuantizationPlugin::clone() const noexcept -{ - auto* plugin = new LayernormQuantizationPlugin( - mEps, mUseDiffOfSquares, mDynActScaling, mSumPerToken, mClampValEnabled, mQuantMode, mType, mOutputType); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs LayernormQuantizationPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - if (outputIndex == 0) - { - // Quantized output - return inputs[outputIndex]; - } - - // Dynamic scaling or per-token sum if enabled - try - { - if (outputIndex == 1) - { - TLLM_CHECK(mDynActScaling); - } - else if (outputIndex == 2) - { - TLLM_CHECK(mSumPerToken); - } - else - { - TLLM_CHECK(false); - } - DimsExprs ret; - ret.nbDims = inputs[0].nbDims; - for (int di = 0; di < ret.nbDims - 1; ++di) - { - ret.d[di] = inputs[0].d[di]; - } - ret.d[ret.nbDims - 1] = exprBuilder.constant(1); - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool LayernormQuantizationPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - int const totalPoses - = 6 + static_cast<int>(mClampValEnabled) + static_cast<int>(mDynActScaling) + static_cast<int>(mSumPerToken); - TLLM_CHECK(0 <= pos && pos < totalPoses); - TLLM_CHECK(nbInputs == 4 + static_cast<int>(mClampValEnabled)); - if (pos < nbInputs) - { - if (pos < 3) - { - // activatation, weight, bias - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == 3) - { - // scale - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == 4 && mClampValEnabled) - { - // clamp_max_v - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - } - else - { - auto const output_pos = pos - nbInputs; - if (output_pos == 0) - { - // Quantized output - return (inOut[pos].type == mOutputType) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (output_pos == 1 && mDynActScaling) - { - // Dynamic scaling if enabled - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (output_pos == 2 && static_cast<int>(mClampValEnabled)) - { - // Clamp value - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - } - - // We should never reach this point - TLLM_CHECK_WITH_INFO(false, "The input/output is not supported."); - return false; -} - -void LayernormQuantizationPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t LayernormQuantizationPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -int LayernormQuantizationPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // input [M(*), N] - // weight [N, ] - // bias [N, ] - // scale_to_int [1] - // clamp_max_v [2], contains min val, and max val (optional) - // outputs - // output [M(*), N] Normalized activations, potentially with quantization applied. - // dynamic_scaling [M(*), 1] (Optional) Per-token scales if quantization is enabled. - // token_sums [M(*), 1] (Optional) Per-token sums of all the channels (before quantization). - - int64_t m64 = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims - 1; ++i) - { - m64 *= inputDesc[0].dims.d[i]; - } - int const m = TLLM_INT32_CAST(m64); - int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[0]); - - void const* input = inputs[0]; - void const* weight = inputs[1]; - void const* bias = inputs[2]; - void const* scale = inputs[3]; - void const* clampValPtr = mClampValEnabled ? inputs[4] : nullptr; - void* output = outputs[0]; - void* dynamic_scale = mDynActScaling ? outputs[1] : nullptr; - void* sum_per_token = mSumPerToken ? outputs[2] : nullptr; - - if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kINT8) - { - dispatchDataType<float, int8_t>(nullptr, input, weight, bias, mEps, m, n, stream, mUseDiffOfSquares, - clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#ifdef ENABLE_FP8 - else if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kFP8) - { - dispatchDataType<float, __nv_fp8_e4m3>(nullptr, input, weight, bias, mEps, m, n, stream, mUseDiffOfSquares, - clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#endif // ENABLE_FP8 - else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kINT8) - { - dispatchDataType<half, int8_t>(nullptr, input, weight, bias, mEps, m, n, stream, mUseDiffOfSquares, clampValPtr, - scale, dynamic_scale, sum_per_token, output); - } -#ifdef ENABLE_FP8 - else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kFP8) - { - dispatchDataType<half, __nv_fp8_e4m3>(nullptr, input, weight, bias, mEps, m, n, stream, mUseDiffOfSquares, - clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#endif // ENABLE_FP8 -#ifdef ENABLE_BF16 - else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kINT8) - { - dispatchDataType<__nv_bfloat16, int8_t>(nullptr, input, weight, bias, mEps, m, n, stream, mUseDiffOfSquares, - clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#ifdef ENABLE_FP8 - else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kFP8) - { - dispatchDataType<__nv_bfloat16, __nv_fp8_e4m3>(nullptr, input, weight, bias, mEps, m, n, stream, - mUseDiffOfSquares, clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#endif // ENABLE_FP8 -#endif // ENABLE_BF16 - sync_check_cuda_error(stream); - return 0; -} - -template <typename T, typename QuantT> -void LayernormQuantizationPlugin::dispatchDataType(void* out, void const* input, void const* gamma, void const* beta, - float const eps, int const tokens, int const hidden_dim, cudaStream_t stream, bool use_diff_of_squares, - void const* clampValPtr, void const* scale, void* dynamic_scale, void* sum_per_token, - void* normed_output_quant) noexcept -{ - // inputs - // activation [dim0(*), dim1] - // clamp_value [2], contains min val, and max val (optional) - // outputs - // quant [dim0(*), dim1] - // scale_tokens [dim0(*), 1] - - invokeGeneralLayerNorm(reinterpret_cast<T*>(out), reinterpret_cast<T const*>(input), - reinterpret_cast<T const*>(gamma), reinterpret_cast<T const*>(beta), eps, tokens, hidden_dim, mQuantMode, - stream, use_diff_of_squares, reinterpret_cast<float const*>(clampValPtr), reinterpret_cast<float const*>(scale), - reinterpret_cast<float*>(dynamic_scale), reinterpret_cast<float*>(sum_per_token), - reinterpret_cast<QuantT*>(normed_output_quant)); -} - -// IPluginV2Ext Methods -nvinfer1::DataType LayernormQuantizationPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - assert(index <= 2); - - if (index == 0) - { - // Output 0 quantized output of layer norm - return mOutputType; - } - else if (index == 1) - { - assert(mDynActScaling); - // Output 1 dynamic act scaling - return nvinfer1::DataType::kFLOAT; - } - else if (index == 2) - { - assert(mDynActScaling && mSumPerToken); - // Output 2 per-token sums - return nvinfer1::DataType::kFLOAT; - } - - // We should never reach this point - TLLM_CHECK_WITH_INFO(false, "The output index is not supported."); - return nvinfer1::DataType::kFLOAT; -} - -// IPluginV2 Methods - -char const* LayernormQuantizationPlugin::getPluginType() const noexcept -{ - return LAYERNORM_QUANTIZATION_PLUGIN_NAME; -} - -char const* LayernormQuantizationPlugin::getPluginVersion() const noexcept -{ - return LAYERNORM_QUANTIZATION_PLUGIN_VERSION; -} - -int LayernormQuantizationPlugin::getNbOutputs() const noexcept -{ - return 1 + static_cast<int>(mDynActScaling) + static_cast<int>(mSumPerToken); -} - -int LayernormQuantizationPlugin::initialize() noexcept -{ - return 0; -} - -void LayernormQuantizationPlugin::terminate() noexcept {} - -size_t LayernormQuantizationPlugin::getSerializationSize() const noexcept -{ - return sizeof(mEps) + sizeof(mUseDiffOfSquares) + sizeof(mDynActScaling) + sizeof(mSumPerToken) - + sizeof(mClampValEnabled) + sizeof(mQuantMode) + sizeof(mType) + sizeof(mOutputType); -} - -void LayernormQuantizationPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mEps); - write(d, mUseDiffOfSquares); - write(d, mDynActScaling); - write(d, mSumPerToken); - write(d, mClampValEnabled); - write(d, mQuantMode); - write(d, mType); - write(d, mOutputType); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void LayernormQuantizationPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -LayernormQuantizationPluginCreator::LayernormQuantizationPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("eps", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("use_diff_of_squares", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("dyn_act_scaling", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("sum_per_token", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("clamp_val_enabled", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("quant_mode", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("out_type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* LayernormQuantizationPluginCreator::getPluginName() const noexcept -{ - return LAYERNORM_QUANTIZATION_PLUGIN_NAME; -} - -char const* LayernormQuantizationPluginCreator::getPluginVersion() const noexcept -{ - return LAYERNORM_QUANTIZATION_PLUGIN_VERSION; -} - -PluginFieldCollection const* LayernormQuantizationPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* LayernormQuantizationPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - tensorrt_llm::common::QuantMode quantMode{}; - float eps{}; - nvinfer1::DataType type{}; - nvinfer1::DataType outputType{}; - bool useDiffOfSquares{}; - bool dynamicActivationScaling{}; - bool sumPerToken{}; - bool clampValEnabled{}; - - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "eps")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - eps = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "dyn_act_scaling")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - dynamicActivationScaling = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "use_diff_of_squares")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - useDiffOfSquares = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "sum_per_token")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - sumPerToken = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "clamp_val_enabled")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - clampValEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "quant_mode")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - quantMode = QuantMode(*(static_cast<int32_t const*>(fields[i].data))); - } - else if (!strcmp(attrName, "out_type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - outputType = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - try - { - auto* obj = new LayernormQuantizationPlugin( - eps, useDiffOfSquares, dynamicActivationScaling, sumPerToken, clampValEnabled, quantMode, type, outputType); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* LayernormQuantizationPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call LayernormQuantizationPlugin::destroy() - try - { - auto* obj = new LayernormQuantizationPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.h b/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.h deleted file mode 100644 index 5cf3fa7e022f..000000000000 --- a/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.h +++ /dev/null @@ -1,110 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class LayernormQuantizationPlugin : public BasePlugin -{ -public: - LayernormQuantizationPlugin(float eps, bool useDiffOfSquares, bool dynamicActivationScaling, bool sumPerToken, - bool clampValEnabled, tensorrt_llm::common::QuantMode quantMode, nvinfer1::DataType type, - nvinfer1::DataType outputType); - - LayernormQuantizationPlugin(void const* data, size_t length); - - ~LayernormQuantizationPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - template <typename T, typename QuantT> - void dispatchDataType(void* out, void const* input, void const* gamma, void const* beta, float const eps, - int const tokens, int const hidden_dim, cudaStream_t stream, bool use_diff_of_squares, void const* clampValPtr, - void const* scale, void* dynamic_scale, void* sum_per_token, void* normed_output_quant) noexcept; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - float mEps; - bool mUseDiffOfSquares; - bool mDynActScaling; - nvinfer1::DataType mType; - - const std::string mLayerName; - // The quantized output data type - nvinfer1::DataType mOutputType; - // Do we clamp the input tensor? - bool mClampValEnabled; - // The quantization mode - tensorrt_llm::common::QuantMode mQuantMode; - // Should we output the sum of channels per-token? (Used by QServe GEMM) - bool mSumPerToken; -}; - -class LayernormQuantizationPluginCreator : public BaseCreator -{ -public: - LayernormQuantizationPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/lookupPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/lookupPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/lookupPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.cpp b/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.cpp deleted file mode 100644 index e4d26f9e5ec6..000000000000 --- a/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.cpp +++ /dev/null @@ -1,341 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <cstdio> - -#include "lookupPlugin.h" -#include "tensorrt_llm/kernels/lookupKernels.h" -#include "tensorrt_llm/plugins/common/plugin.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::LookupPluginCreator; -using tensorrt_llm::plugins::LookupPlugin; - -static char const* LOOKUP_PLUGIN_VERSION{"1"}; -static char const* LOOKUP_PLUGIN_NAME{"Lookup"}; -PluginFieldCollection LookupPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> LookupPluginCreator::mPluginAttributes; - -LookupPlugin::LookupPlugin(nvinfer1::DataType type, int rank) - : mType(type) - , mRank(rank) -{ - mArch = tensorrt_llm::common::getSMVersion(); -} - -// Parameterized constructor -LookupPlugin::LookupPlugin(void const* data, size_t length) -{ - mArch = tensorrt_llm::common::getSMVersion(); - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mType); - read(d, mRank); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* LookupPlugin::clone() const noexcept -{ - auto* plugin = new LookupPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - plugin->initialize(); - return plugin; -} - -nvinfer1::DimsExprs LookupPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs == 2 || nbInputs == 3); - TLLM_CHECK(outputIndex == 0); - DimsExprs ret; - int const nbDimsInput = inputs[0].nbDims; - int const nbDimsWeight = inputs[1].nbDims; - ret.nbDims = nbDimsInput + 1; - - for (int i = 0; i < nbDimsInput; ++i) - { - ret.d[i] = inputs[0].d[i]; - } - ret.d[nbDimsInput] = inputs[1].d[nbDimsWeight - 1]; - - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool LookupPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - bool res = false; - if (nbInputs == 2) - { - switch (pos) - { - case 0: res = ((inOut[0].type == DataType::kINT32) && (inOut[0].format == TensorFormat::kLINEAR)); break; - case 1: res = ((inOut[1].type == mType) && (inOut[1].format == TensorFormat::kLINEAR)); break; - case 2: res = ((inOut[2].type == mType) && (inOut[2].format == TensorFormat::kLINEAR)); break; - default: // should NOT be here! - res = false; - } - } - else - { - TLLM_CHECK_WITH_INFO(mArch == 90, "int8 weight only lookupPlugin is only supported in SM 90 now."); - switch (pos) - { - case 0: res = ((inOut[0].type == DataType::kINT32) && (inOut[0].format == TensorFormat::kLINEAR)); break; - case 1: - res = ((inOut[1].type == DataType::kINT8 || inOut[1].type == mType) - && (inOut[1].format == TensorFormat::kLINEAR)); - break; - case 2: res = ((inOut[2].type == mType) && (inOut[2].format == TensorFormat::kLINEAR)); break; - case 3: res = ((inOut[3].type == mType) && (inOut[3].format == TensorFormat::kLINEAR)); break; - default: // should NOT be here! - res = false; - } - } - return res; -} - -void LookupPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - mNbInputs = nbInputs; -} - -size_t LookupPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -int LookupPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - // inputs - // input [tokenNum] - // weight [localVocabSize, hidden] - // per_token_scales [localVocabSize], optional - // outputs - // embedding [tokenNum, hidden] - - int64_t tokenNum = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) - { - tokenNum *= inputDesc[0].dims.d[i]; - } - - int const localVocabSize = inputDesc[1].dims.d[0]; - int const hidden = inputDesc[1].dims.d[inputDesc[1].dims.nbDims - 1]; - int const* input = reinterpret_cast<int const*>(inputs[0]); - - int offset = mRank * localVocabSize; - - if (mNbInputs == 3) - { - int8_t const* weight = reinterpret_cast<int8_t const*>(inputs[1]); - if (mType == DataType::kHALF) - { - half const* per_token_scales = reinterpret_cast<half const*>(inputs[2]); - half* output = reinterpret_cast<half*>(outputs[0]); - invokeLookUp<half, int8_t, int>( - output, input, weight, tokenNum, offset, localVocabSize, hidden, per_token_scales, stream); - } - else if (mType == DataType::kFLOAT) - { - float const* per_token_scales = reinterpret_cast<float const*>(inputs[2]); - float* output = reinterpret_cast<float*>(outputs[0]); - invokeLookUp<float, int8_t, int>( - output, input, weight, tokenNum, offset, localVocabSize, hidden, per_token_scales, stream); - } - else if (mType == DataType::kBF16) - { - __nv_bfloat16 const* per_token_scales = reinterpret_cast<__nv_bfloat16 const*>(inputs[2]); - __nv_bfloat16* output = reinterpret_cast<__nv_bfloat16*>(outputs[0]); - invokeLookUp<__nv_bfloat16, int8_t, int>( - output, input, weight, tokenNum, offset, localVocabSize, hidden, per_token_scales, stream); - } - } - else - { - if (mType == DataType::kHALF) - { - half const* weight = reinterpret_cast<half const*>(inputs[1]); - half* output = reinterpret_cast<half*>(outputs[0]); - invokeLookUp<half, half, int>( - output, input, weight, tokenNum, offset, localVocabSize, hidden, nullptr, stream); - } - else if (mType == DataType::kFLOAT) - { - float const* weight = reinterpret_cast<float const*>(inputs[1]); - float* output = reinterpret_cast<float*>(outputs[0]); - invokeLookUp<float, float, int>( - output, input, weight, tokenNum, offset, localVocabSize, hidden, nullptr, stream); - } - else if (mType == DataType::kBF16) - { - __nv_bfloat16 const* weight = reinterpret_cast<__nv_bfloat16 const*>(inputs[1]); - __nv_bfloat16* output = reinterpret_cast<__nv_bfloat16*>(outputs[0]); - invokeLookUp<__nv_bfloat16, __nv_bfloat16, int>( - output, input, weight, tokenNum, offset, localVocabSize, hidden, nullptr, stream); - } - } - sync_check_cuda_error(stream); - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType LookupPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* LookupPlugin::getPluginType() const noexcept -{ - return LOOKUP_PLUGIN_NAME; -} - -char const* LookupPlugin::getPluginVersion() const noexcept -{ - return LOOKUP_PLUGIN_VERSION; -} - -int LookupPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int LookupPlugin::initialize() noexcept -{ - return 0; -} - -void LookupPlugin::destroy() noexcept -{ - delete this; -} - -size_t LookupPlugin::getSerializationSize() const noexcept -{ - return sizeof(mType) + sizeof(mRank); -} - -void LookupPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - write(d, mRank); - - TLLM_CHECK(d == a + getSerializationSize()); -} - -void LookupPlugin::terminate() noexcept {} - -/////////////// - -LookupPluginCreator::LookupPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("rank", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* LookupPluginCreator::getPluginName() const noexcept -{ - return LOOKUP_PLUGIN_NAME; -} - -char const* LookupPluginCreator::getPluginVersion() const noexcept -{ - return LOOKUP_PLUGIN_VERSION; -} - -PluginFieldCollection const* LookupPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* LookupPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - nvinfer1::DataType type{}; - int rank{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "rank")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - rank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - } - try - { - auto* obj = new LookupPlugin(type, rank); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* LookupPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call LookupPlugin::destroy() - try - { - auto* obj = new LookupPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.h b/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.h deleted file mode 100644 index 4dddaa1d8bdc..000000000000 --- a/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class LookupPlugin : public BasePlugin -{ -public: - LookupPlugin() = delete; - - LookupPlugin(nvinfer1::DataType type, int rank); - - LookupPlugin(void const* data, size_t length); - - ~LookupPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - const std::string mLayerName; - - nvinfer1::DataType mType; - int mRank; - int mNbInputs = 0; - int mArch; -}; - -class LookupPluginCreator : public BaseCreator -{ -public: - LookupPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/loraPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/loraPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/loraPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.cpp b/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.cpp deleted file mode 100644 index 7a7d925a74f6..000000000000 --- a/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.cpp +++ /dev/null @@ -1,525 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "loraPlugin.h" - -#include "pluginUtils.h" -#include "tensorrt_llm/common/assert.h" - -#include <vector> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::LoraPluginCreator; -using tensorrt_llm::plugins::LoraPlugin; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; - -static char const* LORA_PLUGIN_VERSION{"1"}; -static char const* LORA_PLUGIN_NAME{"Lora"}; -PluginFieldCollection LoraPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> LoraPluginCreator::mPluginAttributes; - -LoraPlugin::LoraPlugin(int in_hidden_size, std::vector<int> out_hidden_sizes, int transA, int transB, - int num_lora_modules, nvinfer1::DataType type, LoraPlugin::PluginProfilerPtr const& pluginProfiler, - bool remove_input_padding, int max_low_rank, int weight_index) - : mTransA(transA) - , mTransB(transB) - , mType(type) - , mRemoveInputPadding(remove_input_padding) - , mNumLoraModules(num_lora_modules) - , mInHiddenSize(in_hidden_size) - , mMaxLowRank(max_low_rank) - , mWeightIndex(weight_index) - , mPluginProfiler(pluginProfiler) -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - mOutHiddenSizes.resize(mNumLoraModules); - mOutHiddenSizes.assign(out_hidden_sizes.begin(), out_hidden_sizes.end()); - init(); -} - -// Parameterized constructor -LoraPlugin::LoraPlugin(void const* data, size_t length, LoraPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mInHiddenSize); - read(d, mTransA); - read(d, mTransB); - read(d, mNumLoraModules); - read(d, mType); - read(d, mRemoveInputPadding); - read(d, mMaxLowRank); - read(d, mWeightIndex); - mOutHiddenSizes.resize(mNumLoraModules); - for (int i = 0; i < mNumLoraModules; i++) - { - read(d, mOutHiddenSizes[i]); - } - init(); - - mPluginProfiler->deserialize(d, mDims, mGemmId); - - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -void LoraPlugin::init() -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - - auto cublasHandle = getCublasHandle(); - auto cublasLtHandle = getCublasLtHandle(); - auto cublasWraper = std::make_shared<CublasMMWrapper>(cublasHandle, cublasLtHandle, nullptr, nullptr); - - mLoraImpl = std::make_shared<kernels::LoraImpl>( - mInHiddenSize, mOutHiddenSizes, mTransA, mTransB, mNumLoraModules, mType, mMaxLowRank, cublasWraper); - - mPluginProfiler->setTranspose(mTransA, mTransB); - mGemmId = GemmIdCublas(mDims.n, mDims.k, mType, mTransA, mTransB, mType); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* LoraPlugin::clone() const noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - auto* plugin = new LoraPlugin(*this); - return plugin; -} - -nvinfer1::DimsExprs LoraPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - try - { - TLLM_CHECK(outputIndex < mNumLoraModules); - int const nbDimsA = inputs[getInputTensorIdx()].nbDims; - DimsExprs ret; - ret.nbDims = nbDimsA; - - for (int i = 0; i < ret.nbDims; ++i) - { - ret.d[0] = 0; - } - - if (mTransA) - { - for (int i = 1; i < nbDimsA; ++i) - { - ret.d[i - 1] = inputs[getInputTensorIdx()].d[i]; - } - } - else - { - for (int i = 0; i < nbDimsA - 1; ++i) - { - ret.d[i] = inputs[getInputTensorIdx()].d[i]; - } - } - - auto const* outHiddenSize = exprBuilder.constant(mOutHiddenSizes.at(outputIndex)); - TLLM_CHECK(outHiddenSize != nullptr); - ret.d[ret.nbDims - 1] = outHiddenSize; - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool LoraPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - if (pos == getHostRequestTypesIdx()) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (pos >= getLoraRanksIdx() && pos < getLoraRanksIdx() + mNumLoraModules) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (pos >= getLoraWeightsPtrsIdx() && pos < getLoraWeightsPtrsIdx() + mNumLoraModules) - { - return inOut[pos].type == nvinfer1::DataType::kINT64; - } - else if (mRemoveInputPadding && pos == getHostContextLengthsIdx()) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else - { - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } -} - -void LoraPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - - auto const input = in[getInputTensorIdx()]; - - int const nbDimsA = input.max.nbDims; - - auto const minM = utils::computeMDimension(mTransA, input.min); - auto const maxM = utils::computeMDimension(mTransA, input.max); - auto const N = utils::computeNDimension(mTransB, in[getHostRequestTypesIdx()].max); - auto const K = static_cast<utils::DimType64>(mTransA ? input.max.d[0] : input.max.d[nbDimsA - 1]); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, N, K}; - } - mGemmId.n = N; - mGemmId.k = K; - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -size_t LoraPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - - int const nbReq = inputs[getLoraRanksIdx()].dims.d[0]; - auto const type = inputs[getInputTensorIdx()].type; - auto const numTokens = getNumTokens(inputs); - return mLoraImpl->getWorkspaceSize(numTokens, nbReq, type); -} - -int64_t LoraPlugin::getNumTokens(nvinfer1::PluginTensorDesc const* input_tensors) const -{ - int ndim = input_tensors[getInputTensorIdx()].dims.nbDims; - TLLM_CHECK_WITH_INFO( - 3 == ndim || 2 == ndim, "hidden_state dimension should be either 2 [numTokens, hidden], or 3 [b, s, hidden]"); - int64_t num_tokens = input_tensors[getInputTensorIdx()].dims.d[0]; - if (ndim == 3) - { - num_tokens *= input_tensors[getInputTensorIdx()].dims.d[1]; - } - return num_tokens; -} - -int LoraPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - if (isBuilding()) - { - return 0; - } - - auto const numReqs = inputDesc[getLoraRanksIdx()].dims.d[0]; - void const* input = inputs[getInputTensorIdx()]; - int const seqLen = mRemoveInputPadding ? 0 : inputDesc[getInputTensorIdx()].dims.d[1]; - int32_t const* reqTypes = static_cast<int32_t const*>(inputs[getHostRequestTypesIdx()]); - void const* const* loraRanks = &inputs[getLoraRanksIdx()]; - void const* const* loraWeightPtrs = &inputs[getLoraWeightsPtrsIdx()]; - int32_t const* hostContextLengths - = mRemoveInputPadding ? static_cast<int32_t const*>(inputs[getHostContextLengthsIdx()]) : nullptr; - - int numTokens = getNumTokens(inputDesc); - mExpandLoraWeightPtrs.clear(); - mExpandLoraRanks.clear(); - mExpandLoraWeightPtrs.reserve(mNumLoraModules * numTokens * 2); - mExpandLoraRanks.reserve(mNumLoraModules * numTokens); - - for (int loraModuleIdx = 0; loraModuleIdx < mNumLoraModules; loraModuleIdx++) - { - auto const loraWeightModulePtrs = static_cast<int64_t const*>(loraWeightPtrs[loraModuleIdx]); - auto const loraRankModule = static_cast<int32_t const*>(loraRanks[loraModuleIdx]); - - int idx = 0; - for (int reqId = 0; reqId < numReqs; reqId++) - { - // loraWeightModulePtrs has 3 pointers for each module: A,B, and an optional DoRA magnitude - // the current LoRA plugin does not apply DoRA scaling, so the magnitude is ignored - RequestType const reqType = static_cast<RequestType>(reqTypes[reqId]); - if (reqType == RequestType::kGENERATION) - { - mExpandLoraWeightPtrs.push_back(reinterpret_cast<void const*>(loraWeightModulePtrs[reqId * 3])); - mExpandLoraWeightPtrs.push_back(reinterpret_cast<void const*>(loraWeightModulePtrs[reqId * 3 + 1])); - mExpandLoraRanks.push_back(loraRankModule[reqId]); - idx += 1; - } - else - { - int contextLen = (mRemoveInputPadding ? hostContextLengths[reqId] : seqLen); - - for (int contextId = 0; contextId < contextLen; contextId++) - { - mExpandLoraWeightPtrs.push_back(reinterpret_cast<void const*>(loraWeightModulePtrs[reqId * 3])); - mExpandLoraWeightPtrs.push_back(reinterpret_cast<void const*>(loraWeightModulePtrs[reqId * 3 + 1])); - mExpandLoraRanks.push_back(loraRankModule[reqId]); - idx += 1; - } - } - } - - // In 1st generation phase cross attention qkv lora, cross qkv is skipped by passing an empty encoder_output - // (passing 0 to dim) getNumTokens() will get in cross qkv_lora. Skipping the check for this case. - if (numTokens > 0) - { - TLLM_CHECK_WITH_INFO(idx == numTokens, - fmtstr("LoraParams and input dims don't match, lora tokens %d input tokens %d", idx, numTokens)); - } - } - - // only used for unified gemm - auto bestTactic = mPluginProfiler->getBestConfig(numTokens, mGemmId); - mLoraImpl->setBestTactic(bestTactic); - mLoraImpl->run(numTokens, numReqs, input, mExpandLoraRanks.data(), mExpandLoraWeightPtrs.data(), mWeightIndex, - outputs, workspace, stream); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType LoraPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - TLLM_CHECK(index < mNumLoraModules); - return mType; -} - -// IPluginV2 Methods - -char const* LoraPlugin::getPluginType() const noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - return LORA_PLUGIN_NAME; -} - -char const* LoraPlugin::getPluginVersion() const noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - return LORA_PLUGIN_VERSION; -} - -int LoraPlugin::getNbOutputs() const noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - return mNumLoraModules; -} - -int LoraPlugin::initialize() noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - if (!mDims.isInitialized()) - { - return 0; - } - - mLoraImpl->setGemmConfig(); - - mPluginProfiler->profileTactics(mLoraImpl->getCublasWrapper(), mType, mDims, mGemmId); - return 0; -} - -void LoraPlugin::destroy() noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - delete this; -} - -size_t LoraPlugin::getSerializationSize() const noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - return sizeof(mInHiddenSize) + sizeof(mTransA) + sizeof(mTransB) + sizeof(mNumLoraModules) + sizeof(mType) - + mPluginProfiler->getSerializationSize(mGemmId) + sizeof(mRemoveInputPadding) + sizeof(mMaxLowRank) - + sizeof(mWeightIndex) + sizeof(int) * mNumLoraModules; // selected tactics container size -} - -void LoraPlugin::serialize(void* buffer) const noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - char *d = static_cast<char*>(buffer), *a = d; - write(d, mInHiddenSize); - write(d, mTransA); - write(d, mTransB); - write(d, mNumLoraModules); - write(d, mType); - write(d, mRemoveInputPadding); - write(d, mMaxLowRank); - write(d, mWeightIndex); - for (int i = 0; i < mNumLoraModules; i++) - { - write(d, mOutHiddenSizes.at(i)); - } - mPluginProfiler->serialize(d, mGemmId); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void LoraPlugin::terminate() noexcept {} - -/////////////// - -LoraPluginCreator::LoraPluginCreator() -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("transA", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("transB", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("num_lora_modules", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("weight_index", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* LoraPluginCreator::getPluginName() const noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - return LORA_PLUGIN_NAME; -} - -char const* LoraPluginCreator::getPluginVersion() const noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - return LORA_PLUGIN_VERSION; -} - -PluginFieldCollection const* LoraPluginCreator::getFieldNames() noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - return &mFC; -} - -IPluginV2* LoraPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - - PluginField const* fields = fc->fields; - nvinfer1::DataType type{}; - int num_lora_modules{}; - int in_hidden_size{}; - int transA{}; - int transB{}; - bool remove_input_padding{}; - int max_low_rank{}; - int weight_index{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "in_hidden_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - in_hidden_size = *(static_cast<int32_t const*>(fields[i].data)); - } - else if (!strcmp(attrName, "transa")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - transA = *(static_cast<int const*>(fields[i].data)); - } - else if (!strcmp(attrName, "transb")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - transB = *(static_cast<int const*>(fields[i].data)); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "remove_input_padding")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - remove_input_padding = static_cast<bool>(*(static_cast<int8_t const*>(fields[i].data))); - } - else if (!strcmp(attrName, "max_low_rank")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - max_low_rank = *(static_cast<int const*>(fields[i].data)); - } - else if (!strcmp(attrName, "num_lora_modules")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - num_lora_modules = *(static_cast<int const*>(fields[i].data)); - } - else if (!strcmp(attrName, "weight_index")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - weight_index = *(static_cast<int const*>(fields[i].data)); - } - } - std::vector<int> out_hidden_sizes; - out_hidden_sizes.resize(num_lora_modules); - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - for (int j = 0; j < num_lora_modules; j++) - { - if (!strcmp(attrName, fmtstr("out_hidden_size_%d", j).c_str())) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - out_hidden_sizes.at(j) = *(static_cast<int const*>(fields[i].data)); - } - } - } - try - { - // LoraPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - // FIXME enable tactic profiler - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false, /* skip */ true); - auto* obj = new LoraPlugin(in_hidden_size, out_hidden_sizes, transA, transB, num_lora_modules, type, - pluginProfiler, remove_input_padding, max_low_rank, weight_index); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* LoraPluginCreator::deserializePlugin(char const* name, void const* serialData, size_t serialLength) noexcept -{ - TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - // This object will be deleted when the network is destroyed, which will - // call LoraPlugin::destroy() - try - { - // LoraPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - // FIXME enable tactic profiler - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true, /* skip */ true); - auto* obj = new LoraPlugin(serialData, serialLength, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.h b/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.h deleted file mode 100644 index 7795f7b7c76d..000000000000 --- a/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.h +++ /dev/null @@ -1,159 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef TRT_LORA_PLUGIN_H -#define TRT_LORA_PLUGIN_H -#include "tensorrt_llm/kernels/lora/lora.h" -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include "tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h" -#include <cassert> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class LoraPlugin : public BasePlugin -{ -public: - using PluginProfilerPtr = std::shared_ptr<CublasLtGemmPluginProfiler>; - using ImplPtr = std::shared_ptr<kernels::LoraImpl>; - using Config = cublasLtMatmulHeuristicResult_t; - - LoraPlugin() = delete; - - LoraPlugin(int in_hidden_size, std::vector<int> out_hidden_sizes, int transA, int transB, int num_lora_modules, - nvinfer1::DataType type, PluginProfilerPtr const& profiler, bool remove_input_padding, int max_low_rank, - int weight_index); - - LoraPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); - - ~LoraPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - int64_t getNumTokens(nvinfer1::PluginTensorDesc const* input_tensors) const; - void init(); - - using IndexType = std::int32_t; - - IndexType getInputTensorIdx() const - { - return 0; - } - - IndexType getHostRequestTypesIdx() const - { - return 1; - } - - IndexType getLoraRanksIdx() const - { - return 2; - } - - IndexType getLoraWeightsPtrsIdx() const - { - return 2 + mNumLoraModules; - } - - IndexType getHostContextLengthsIdx() const - { - TLLM_CHECK(mRemoveInputPadding); - return 2 + mNumLoraModules + mNumLoraModules; - } - - enum class RequestType : int32_t - { - kCONTEXT = 0, - kGENERATION = 1 - }; - -private: - const std::string mLayerName; - - std::vector<int> mOutHiddenSizes; - int mTransA; - int mTransB; - nvinfer1::DataType mType; - bool mRemoveInputPadding; - int mNumLoraModules; - int mInHiddenSize; - int mMaxLowRank; - int mWeightIndex; - - std::vector<void const*> mExpandLoraWeightPtrs{}; - std::vector<int32_t> mExpandLoraRanks{}; - - GemmDims mDims{}; - GemmIdCublas mGemmId{}; - - PluginProfilerPtr mPluginProfiler; - ImplPtr mLoraImpl; -}; - -class LoraPluginCreator : public BaseCreator -{ -public: - LoraPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - GemmPluginProfilerManager<CublasLtGemmPluginProfiler> gemmPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins - -#endif // TRT_LORA_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/CMakeLists.txt deleted file mode 100644 index b6bd0439cc0c..000000000000 --- a/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.cpp b/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.cpp deleted file mode 100644 index 6165d6210f29..000000000000 --- a/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.cpp +++ /dev/null @@ -1,425 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "lowLatencyGemmPlugin.h" -#include "low_latency_gemm.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/cudaFp8Utils.h" -#include "tensorrt_llm/common/logger.h" -#include <NvInferRuntime.h> -#include <NvInferRuntimeBase.h> -#include <NvInferRuntimePlugin.h> -#include <cstddef> -#include <cstdint> -#include <cstdio> -#include <numeric> -#include <optional> -#include <vector> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using namespace tensorrt_llm::kernels::internal_cutlass_kernels; -using tensorrt_llm::plugins::LowLatencyGemmPluginCreator; -using tensorrt_llm::plugins::LowLatencyGemmPlugin; -using tensorrt_llm::plugins::LowLatencyGemmPluginProfiler; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; - -static char const* LOW_LATENCY_GEMM_PLUGIN_VERSION{"1"}; -static char const* LOW_LATENCY_GEMM_PLUGIN_NAME{"LowLatencyGemm"}; - -PluginFieldCollection LowLatencyGemmPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> LowLatencyGemmPluginCreator::mPluginAttributes; - -using FP8Type = __nv_fp8_e4m3; - -static std::optional<float> getFloatEnv(char const* name) -{ - char const* const env = std::getenv(name); - if (env == nullptr) - { - return std::nullopt; - } - try - { - float value = std::stof(env); - return {value}; - } - catch (std::invalid_argument const& e) - { - return std::nullopt; - } - catch (std::out_of_range const& e) - { - return std::nullopt; - } -}; - -void LowLatencyGemmPluginProfiler::runTactic(int m, int n, int k, LowLatencyGemmPluginProfiler::Config const& tactic, - char* workspace, cudaStream_t const& stream) -{ - - float default_pdl_overlap_ratio = 0.5; - float default_prefetch_ratio = -1.0; - FP8Type* aTmp = reinterpret_cast<FP8Type*>(workspace); - FP8Type* bTmp - = reinterpret_cast<FP8Type*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(aTmp), m * k * sizeof(FP8Type))); - void* cTmp = reinterpret_cast<void*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(bTmp), n * k * sizeof(FP8Type))); - size_t workspaceSize = mRunner->getWorkspaceSize(m, n, k); - char* workspaceTmp = reinterpret_cast<char*>(nextWorkspacePtr( - reinterpret_cast<int8_t*>(cTmp), m * n * (mType == nvinfer1::DataType::kFLOAT ? sizeof(float) : sizeof(half)))); - mRunner->gemm(aTmp, bTmp, 1.0f, 0.0f, nullptr, cTmp, m, n, k, default_pdl_overlap_ratio, default_prefetch_ratio, - tactic, workspaceTmp, workspaceSize, stream); -} - -void LowLatencyGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - - std::vector<size_t> workspaces = {maxM * k * sizeof(FP8Type), n * k * sizeof(FP8Type), - maxM * n * (mType == nvinfer1::DataType::kFLOAT ? sizeof(float) : sizeof(half)), - mRunner->getWorkspaceSize(maxM, n, k)}; - - size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); - setTmpWorkspaceSizeInBytes(bytes); -} - -std::vector<LowLatencyGemmPluginProfiler::Config> LowLatencyGemmPluginProfiler::getTactics(int m, int n, int k) const -{ - return mRunner->getConfigs(); -} - -LowLatencyGemmPlugin::LowLatencyGemmPlugin( - nvinfer1::DataType type, float alpha, PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) - , mAplha(alpha) -{ - init(type); -} - -LowLatencyGemmPlugin::LowLatencyGemmPlugin(void const* data, size_t length, PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - - char const *d = reinterpret_cast<char const*>(data), *a = d; - nvinfer1::DataType type; - read(d, type); - read(d, mAplha); - read(d, mDims); - init(type); - mPluginProfiler->deserialize(d, mDims, mGemmId); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -void LowLatencyGemmPlugin::init(nvinfer1::DataType type) -{ - - mType = type; - - if (mType == nvinfer1::DataType::kFLOAT) - { - m_lowLatencyGemmRunner = std::make_shared<CutlassLowLatencyFp8GemmRunner<float>>(); - } - else if (mType == nvinfer1::DataType::kHALF) - { - m_lowLatencyGemmRunner = std::make_shared<CutlassLowLatencyFp8GemmRunner<half>>(); - } -#ifdef ENABLE_BF16 - - else if (mType == nvinfer1::DataType::kBF16) - { - m_lowLatencyGemmRunner = std::make_shared<CutlassLowLatencyFp8GemmRunner<__nv_bfloat16>>(); - } -#endif - else - { - TLLM_THROW("Unsupported data type"); - } - mGemmId = GemmIdCore(mDims.n, mDims.k, mType); -} - -nvinfer1::DimsExprs LowLatencyGemmPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs == 2); - TLLM_CHECK(outputIndex == 0); - int const nbDimsA = inputs[0].nbDims; - TLLM_CHECK(nbDimsA >= 2); - DimsExprs ret; - ret.nbDims = nbDimsA; - for (int ii = 0; ii < nbDimsA - 1; ++ii) - { - ret.d[ii] = inputs[0].d[ii]; - } - // input[1] , weights [n,k] - ret.d[nbDimsA - 1] = inputs[1].d[0]; - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool LowLatencyGemmPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - switch (pos) - { - case 0: - // activation - return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; - case 1: - // weights - // Weights stored in checkpoint must have fp8 type - return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; - case 2: - // out - return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - default: - // Never should be here - assert(false); - return false; - } -} - -void LowLatencyGemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); - auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); - - int const maxK = in[0].max.d[in[0].max.nbDims - 1]; - int const maxN = in[1].max.d[0]; - int const minK = in[0].min.d[in[0].min.nbDims - 1]; - int const minN = in[1].min.d[0]; - - TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); - TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, maxN, maxK}; - } - mGemmId = {maxN, maxK, mType}; - - m_workspaceMaxSize = m_lowLatencyGemmRunner->getWorkspaceSize(maxM, maxN, maxK); -} - -size_t LowLatencyGemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return m_workspaceMaxSize; -} - -int LowLatencyGemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - - // input0 activation [M,K] - // input1 weights [N,K] - // output0 [M,N] - - int64_t m64 = 1; - for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) - { - m64 *= inputDesc[0].dims.d[ii]; - } - int const m = TLLM_INT32_CAST(m64); - int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[0]); - int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); - int const wsSize = m_lowLatencyGemmRunner->getWorkspaceSize(m, n, k); - auto const& bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); - TLLM_CHECK_WITH_INFO(bestTactic, "No valid Low Latency GEMM tactic"); - - auto env_pdl_overlap_ratio = getFloatEnv("TRTLLM_PDL_OVERLAP_RATIO"); - auto env_prefetch_ratio = getFloatEnv("TRTLLM_PREFETCH_RATIO"); - auto valid_ratio = [](std::optional<float>& env_val, float default_val) - { - if (env_val.has_value()) - { - TLLM_CHECK_WITH_INFO(env_val.value() <= 1.0f, "Valid ratio should be less than or equal to 1.0"); - return env_val.value(); - } - return default_val; - }; - float pdl_overlap_ratio = valid_ratio(env_pdl_overlap_ratio, /*default_val=*/0.5); - float prefetch_ratio = valid_ratio(env_prefetch_ratio, /*default_val=*/-1.0); - m_lowLatencyGemmRunner->gemm(const_cast<FP8Type*>(reinterpret_cast<FP8Type const*>(inputs[0])), - const_cast<FP8Type*>(reinterpret_cast<FP8Type const*>(inputs[1])), mAplha, 0.0F, nullptr, outputs[0], m, n, k, - pdl_overlap_ratio, prefetch_ratio, *bestTactic, reinterpret_cast<char*>(workspace), wsSize, stream); - - return 0; -} - -nvinfer1::DataType LowLatencyGemmPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* LowLatencyGemmPlugin::getPluginType() const noexcept -{ - return LOW_LATENCY_GEMM_PLUGIN_NAME; -} - -char const* LowLatencyGemmPlugin::getPluginVersion() const noexcept -{ - return LOW_LATENCY_GEMM_PLUGIN_VERSION; -} - -int LowLatencyGemmPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int LowLatencyGemmPlugin::initialize() noexcept -{ - configGemm(); - return 0; -} - -void LowLatencyGemmPlugin::terminate() noexcept {} - -nvinfer1::IPluginV2DynamicExt* LowLatencyGemmPlugin::clone() const noexcept -{ - auto* plugin = new LowLatencyGemmPlugin(*this); - return plugin; -} - -size_t LowLatencyGemmPlugin::getSerializationSize() const noexcept -{ - return sizeof(nvinfer1::DataType) + // dtype - sizeof(float) * 1 + // alpha - sizeof(mDims) + mPluginProfiler->getSerializationSize(mGemmId); -} - -void LowLatencyGemmPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - write(d, mAplha); - write(d, mDims); - mPluginProfiler->serialize(d, mGemmId); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void LowLatencyGemmPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -void LowLatencyGemmPlugin::configGemm() -{ - mPluginProfiler->profileTactics(m_lowLatencyGemmRunner, mType, mDims, mGemmId); -} - -LowLatencyGemmPluginCreator::LowLatencyGemmPluginCreator() -{ - - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("alpha", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* LowLatencyGemmPluginCreator::getPluginName() const noexcept -{ - return LOW_LATENCY_GEMM_PLUGIN_NAME; -} - -char const* LowLatencyGemmPluginCreator::getPluginVersion() const noexcept -{ - return LOW_LATENCY_GEMM_PLUGIN_VERSION; -} - -PluginFieldCollection const* LowLatencyGemmPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* LowLatencyGemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - float alpha{}; - nvinfer1::DataType type{}; - for (int i = 0; i < fc->nbFields; i++) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "alpha")) - { - - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - alpha = *(static_cast<float const*>(fields[i].data)); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - - try - { - - // - // GemmPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/*inference=*/false); - auto* obj = new LowLatencyGemmPlugin(type, alpha, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* LowLatencyGemmPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - try - { - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/*inference=*/true); - auto* obj = new LowLatencyGemmPlugin(serialData, serialLength, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.h b/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.h deleted file mode 100644 index 98b8f4807174..000000000000 --- a/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.h +++ /dev/null @@ -1,135 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "low_latency_gemm.h" - -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <cstddef> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -using LowLatencyGemmRunnerPtr - = std::shared_ptr<tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmRunnerInterface>; - -class LowLatencyGemmPluginProfiler - : public GemmPluginProfiler< - tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmRunnerInterface::ConfigType, - LowLatencyGemmRunnerPtr, GemmIdCore, GemmIdCoreHash> -{ - -public: - using Config = tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmRunnerInterface::ConfigType; - -protected: - void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; - - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - - std::vector<Config> getTactics(int m, int n, int k) const override; -}; - -class LowLatencyGemmPlugin : public BasePlugin -{ - -public: - using PluginProfilerPtr = std::shared_ptr<LowLatencyGemmPluginProfiler>; - - LowLatencyGemmPlugin() = delete; - - LowLatencyGemmPlugin(nvinfer1::DataType type, float alpha, PluginProfilerPtr const& pluginProfiler); - - LowLatencyGemmPlugin(void const* data, size_t length, PluginProfilerPtr const& pluginProfiler); - ~LowLatencyGemmPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - void init(nvinfer1::DataType type); - void configGemm(); - -private: - std::string const mLayerName; - - LowLatencyGemmRunnerPtr m_lowLatencyGemmRunner; - size_t m_workspaceMaxSize; - - GemmDims mDims{}; - GemmIdCore mGemmId{}; - - PluginProfilerPtr mPluginProfiler; - - nvinfer1::DataType mType; - float mAplha{1.0F}; -}; - -class LowLatencyGemmPluginCreator : public BaseCreator -{ -public: - LowLatencyGemmPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - GemmPluginProfilerManager<LowLatencyGemmPluginProfiler> gemmPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/CMakeLists.txt deleted file mode 100644 index b6bd0439cc0c..000000000000 --- a/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.cpp b/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.cpp deleted file mode 100644 index a1aa11c2f165..000000000000 --- a/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.cpp +++ /dev/null @@ -1,468 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "lowLatencyGemmSwigluPlugin.h" -#include "low_latency_gemm_swiglu.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/cudaFp8Utils.h" -#include "tensorrt_llm/common/logger.h" -#include <NvInferRuntime.h> -#include <NvInferRuntimeBase.h> -#include <NvInferRuntimePlugin.h> -#include <cstddef> -#include <cstdint> -#include <cstdio> -#include <numeric> -#include <optional> -#include <vector> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using namespace tensorrt_llm::kernels::internal_cutlass_kernels; -using tensorrt_llm::plugins::LowLatencyGemmSwigluPluginCreator; -using tensorrt_llm::plugins::LowLatencyGemmSwigluPlugin; -using tensorrt_llm::plugins::LowLatencyGemmSwigluPluginProfiler; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; - -static char const* LOW_LATENCY_GEMM_SWIGLU_PLUGIN_VERSION{"1"}; -static char const* LOW_LATENCY_GEMM_SWIGLU_PLUGIN_NAME{"LowLatencyGemmSwiglu"}; - -PluginFieldCollection LowLatencyGemmSwigluPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> LowLatencyGemmSwigluPluginCreator::mPluginAttributes; - -using FP8Type = __nv_fp8_e4m3; - -static std::optional<float> getFloatEnv(char const* name) -{ - char const* const env = std::getenv(name); - if (env == nullptr) - { - return std::nullopt; - } - try - { - float value = std::stof(env); - return {value}; - } - catch (std::invalid_argument const& e) - { - return std::nullopt; - } - catch (std::out_of_range const& e) - { - return std::nullopt; - } -}; - -static size_t getBytePerElement(nvinfer1::DataType type) -{ - size_t bpe; - if (type == nvinfer1::DataType::kFLOAT) - { - bpe = 4; - } - else if (type == nvinfer1::DataType::kHALF || type == nvinfer1::DataType::kBF16) - { - bpe = 2; - } - else if (type == nvinfer1::DataType::kINT8 || type == nvinfer1::DataType::kFP8) - { - bpe = 1; - } - else - { - TLLM_THROW("Not recognized/implemented"); - } - return bpe; -} - -void LowLatencyGemmSwigluPluginProfiler::runTactic(int m, int n, int k, - LowLatencyGemmSwigluPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) -{ - - float default_pdl_overlap_ratio = 0.5; - float default_prefetch_ratio = -1.0; - FP8Type* aTmp = reinterpret_cast<FP8Type*>(workspace); - FP8Type* bTmp - = reinterpret_cast<FP8Type*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(aTmp), m * k * sizeof(FP8Type))); - void* dTmp = reinterpret_cast<void*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(bTmp), n * k * sizeof(FP8Type))); - size_t workspaceSize = mRunner->getWorkspaceSize(m, n, k); - char* workspaceTmp = reinterpret_cast<char*>( - nextWorkspacePtr(reinterpret_cast<int8_t*>(dTmp), (n / 2 * m * getBytePerElement(mType)))); - mRunner->gemm(aTmp, bTmp, 1.0f, 0.0f, 1.0f, 1.0f, nullptr, dTmp, m, n, k, default_pdl_overlap_ratio, - default_prefetch_ratio, tactic, workspaceTmp, workspaceSize, stream); -} - -int LowLatencyGemmSwigluPluginProfiler::getMaxProfileM() const -{ - return 32768; -} - -void LowLatencyGemmSwigluPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - - std::vector<size_t> workspaces = {maxM * k * sizeof(FP8Type), // A - n * k * sizeof(FP8Type), // B - maxM * (n / 2) * getBytePerElement(mType), // D - mRunner->getWorkspaceSize(maxM, n, k)}; // workspace - - size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); - setTmpWorkspaceSizeInBytes(bytes); -} - -std::vector<LowLatencyGemmSwigluPluginProfiler::Config> LowLatencyGemmSwigluPluginProfiler::getTactics( - int m, int n, int k) const -{ - return mRunner->getConfigs(); -} - -LowLatencyGemmSwigluPlugin::LowLatencyGemmSwigluPlugin(nvinfer1::DataType type, float scale_output, float scale_d0, - float scale_d1, PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) - , mScaleOutput(scale_output) - , mScaleD0(scale_d0) - , mScaleD1(scale_d1) -{ - init(type); -} - -LowLatencyGemmSwigluPlugin::LowLatencyGemmSwigluPlugin( - void const* data, size_t length, PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - - char const *d = reinterpret_cast<char const*>(data), *a = d; - nvinfer1::DataType type; - read(d, type); - read(d, mScaleOutput); - read(d, mScaleD0); - read(d, mScaleD1); - read(d, mDims); - - init(type); - mPluginProfiler->deserialize(d, mDims, mGemmId); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -void LowLatencyGemmSwigluPlugin::init(nvinfer1::DataType type) -{ - - mType = type; - - if (mType == nvinfer1::DataType::kFP8) - { - mLowLatencyGemmSwigluRunner = std::make_shared<CutlassLowLatencyFp8GemmSwigluRunner<__nv_fp8_e4m3>>(); - } - else - { - TLLM_THROW("Unsupported data type"); - } - mGemmId = GemmIdCore(mDims.n, mDims.k, mType); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* LowLatencyGemmSwigluPlugin::clone() const noexcept -{ - auto* plugin = new LowLatencyGemmSwigluPlugin(*this); - return plugin; -} - -nvinfer1::DimsExprs LowLatencyGemmSwigluPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs == 2); - TLLM_CHECK(outputIndex == 0); - int const nbDimsA = inputs[0].nbDims; - TLLM_CHECK(nbDimsA >= 2); - DimsExprs ret; - ret.nbDims = nbDimsA; - for (int ii = 0; ii < nbDimsA - 1; ++ii) - { - ret.d[ii] = inputs[0].d[ii]; - } - ret.d[nbDimsA - 1] = exprBuilder.constant(inputs[1].d[1]->getConstantValue() / 2); - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool LowLatencyGemmSwigluPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - switch (pos) - { - case 0: - // activation - return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; - case 1: - // weights - // Weights stored in checkpoint must have fp8 type - return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; - case 2: - // out - return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - default: - // Never should be here - TLLM_CHECK(false); - return false; - } -} - -void LowLatencyGemmSwigluPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); - auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); - - int const maxK = in[0].max.d[in[0].max.nbDims - 1]; - int const maxN = in[1].max.d[1]; - int const minK = in[0].min.d[in[0].min.nbDims - 1]; - int const minN = in[1].min.d[1]; - - TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); - TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, maxN, maxK}; - } - mGemmId = {maxN, maxK, mType}; - - mWorkspaceMaxSize = mLowLatencyGemmSwigluRunner->getWorkspaceSize(maxM, maxN, maxK); -} - -size_t LowLatencyGemmSwigluPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return mWorkspaceMaxSize; -} - -int LowLatencyGemmSwigluPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - - // input0 activation [M,K] row-major - // input1 weights [K, N] col-major - // output0 [M,N / 2] row-major - - int64_t m64 = 1; - for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) - { - m64 *= inputDesc[0].dims.d[ii]; - } - int const m = TLLM_INT32_CAST(m64); - int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[1]); - int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); - int const wsSize = mLowLatencyGemmSwigluRunner->getWorkspaceSize(m, n, k); - auto const& bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); - TLLM_CHECK_WITH_INFO(bestTactic, "No valid Low Latency GEMM SWIGLU tactic"); - - auto env_pdl_overlap_ratio = getFloatEnv("TRTLLM_PDL_OVERLAP_RATIO"); - auto env_prefetch_ratio = getFloatEnv("TRTLLM_PREFETCH_RATIO"); - auto valid_ratio = [](std::optional<float>& env_val, float default_val) - { - if (env_val.has_value()) - { - TLLM_CHECK_WITH_INFO(env_val.value() <= 1.0f, "Valid ratio should be less than or equal to 1.0"); - return env_val.value(); - } - return default_val; - }; - float pdl_overlap_ratio = valid_ratio(env_pdl_overlap_ratio, /*default_val=*/0.5); - float prefetch_ratio = valid_ratio(env_prefetch_ratio, /*default_val=*/-1.0); - mLowLatencyGemmSwigluRunner->gemm(const_cast<FP8Type*>(reinterpret_cast<FP8Type const*>(inputs[0])), - const_cast<FP8Type*>(reinterpret_cast<FP8Type const*>(inputs[1])), mScaleOutput, 0.0F, mScaleD0, mScaleD1, - nullptr, outputs[0], m, n, k, pdl_overlap_ratio, prefetch_ratio, *bestTactic, - reinterpret_cast<char*>(workspace), wsSize, stream); - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType LowLatencyGemmSwigluPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* LowLatencyGemmSwigluPlugin::getPluginType() const noexcept -{ - return LOW_LATENCY_GEMM_SWIGLU_PLUGIN_NAME; -} - -char const* LowLatencyGemmSwigluPlugin::getPluginVersion() const noexcept -{ - return LOW_LATENCY_GEMM_SWIGLU_PLUGIN_VERSION; -} - -int LowLatencyGemmSwigluPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int LowLatencyGemmSwigluPlugin::initialize() noexcept -{ - configGemm(); - return 0; -} - -void LowLatencyGemmSwigluPlugin::terminate() noexcept {} - -size_t LowLatencyGemmSwigluPlugin::getSerializationSize() const noexcept -{ - return sizeof(nvinfer1::DataType) + // dtype - sizeof(float) * 3 + // scales - sizeof(mDims) + mPluginProfiler->getSerializationSize(mGemmId); -} - -void LowLatencyGemmSwigluPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - write(d, mScaleOutput); - write(d, mScaleD0); - write(d, mScaleD1); - write(d, mDims); - mPluginProfiler->serialize(d, mGemmId); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void LowLatencyGemmSwigluPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -void LowLatencyGemmSwigluPlugin::configGemm() -{ - mPluginProfiler->profileTactics(mLowLatencyGemmSwigluRunner, mType, mDims, mGemmId); -} - -////////////////////////////////////////////////////////////////////////// - -LowLatencyGemmSwigluPluginCreator::LowLatencyGemmSwigluPluginCreator() -{ - - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("scale_output", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("scale_d0", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("scale_d1", nullptr, PluginFieldType::kFLOAT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* LowLatencyGemmSwigluPluginCreator::getPluginName() const noexcept -{ - return LOW_LATENCY_GEMM_SWIGLU_PLUGIN_NAME; -} - -char const* LowLatencyGemmSwigluPluginCreator::getPluginVersion() const noexcept -{ - return LOW_LATENCY_GEMM_SWIGLU_PLUGIN_VERSION; -} - -PluginFieldCollection const* LowLatencyGemmSwigluPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* LowLatencyGemmSwigluPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - TLLM_CHECK(fc->nbFields == 4); - nvinfer1::DataType type{}; - float scale_output{}; - float scale_d0{}; - float scale_d1{}; - for (int i = 0; i < fc->nbFields; i++) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "scale_output")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - scale_output = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - else if (!strcmp(attrName, "scale_d0")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - scale_d0 = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - else if (!strcmp(attrName, "scale_d1")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - scale_d1 = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - } - - try - { - - // - // LowLatencyGemmSwigluPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/*inference=*/false); - auto* obj = new LowLatencyGemmSwigluPlugin(type, scale_output, scale_d0, scale_d1, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* LowLatencyGemmSwigluPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - try - { - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/*inference=*/true); - auto* obj = new LowLatencyGemmSwigluPlugin(serialData, serialLength, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.h b/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.h deleted file mode 100644 index 3f73324e7740..000000000000 --- a/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "low_latency_gemm_swiglu.h" - -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <cstddef> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ -using LowLatencyGemmSwigluRunnerPtr - = std::shared_ptr<tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmSwigluRunnerInterface>; - -class LowLatencyGemmSwigluPluginProfiler - : public GemmPluginProfiler< - tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmSwigluRunnerInterface::ConfigType, - LowLatencyGemmSwigluRunnerPtr, GemmIdCore, GemmIdCoreHash> -{ - -public: - using Config - = tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmSwigluRunnerInterface::ConfigType; - - virtual int getMaxProfileM() const override; - -protected: - void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; - - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - - std::vector<Config> getTactics(int m, int n, int k) const override; -}; - -class LowLatencyGemmSwigluPlugin : public BasePlugin -{ - -public: - using PluginProfilerPtr = std::shared_ptr<LowLatencyGemmSwigluPluginProfiler>; - - LowLatencyGemmSwigluPlugin() = delete; - - LowLatencyGemmSwigluPlugin(nvinfer1::DataType type, float scale_output, float scale_d0, float scale_d1, - PluginProfilerPtr const& pluginProfiler); - - LowLatencyGemmSwigluPlugin(void const* data, size_t length, PluginProfilerPtr const& pluginProfiler); - ~LowLatencyGemmSwigluPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - void init(nvinfer1::DataType type); - void configGemm(); - -private: - std::string const mLayerName; - - LowLatencyGemmSwigluRunnerPtr mLowLatencyGemmSwigluRunner; - size_t mWorkspaceMaxSize; - - GemmDims mDims{}; - GemmIdCore mGemmId{}; - - PluginProfilerPtr mPluginProfiler; - - nvinfer1::DataType mType; - float mScaleOutput; - float mScaleD0; - float mScaleD1; -}; - -class LowLatencyGemmSwigluPluginCreator : public BaseCreator -{ -public: - LowLatencyGemmSwigluPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - GemmPluginProfilerManager<LowLatencyGemmSwigluPluginProfiler> gemmPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/lruPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/lruPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/lruPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.cpp b/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.cpp deleted file mode 100644 index 9d86b8cb8acd..000000000000 --- a/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.cpp +++ /dev/null @@ -1,431 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "lruPlugin.h" -#include "tensorrt_llm/common/assert.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::lruPluginCreator; -using tensorrt_llm::plugins::lruPlugin; - -static char const* LRU_PLUGIN_VERSION{"1"}; -static char const* LRU_PLUGIN_NAME{"LRU"}; -PluginFieldCollection lruPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> lruPluginCreator::mPluginAttributes; - -lruPlugin::lruPlugin(int dim, int block_size, nvinfer1::DataType type, bool removePadding, bool pagedState, - bool yEnabled, bool yBiasEnabled, bool fuseGateEnabled, bool gateBiasEnabled) - : mDim(dim) - , mBlockSize(block_size) - , mType(type) - , mRemovePadding(removePadding) - , mPagedState(pagedState) - , mYEnabled(yEnabled) - , mYBiasEnabled(yBiasEnabled) - , mFuseGateEnabled(fuseGateEnabled) - , mGateBiasEnabled(gateBiasEnabled) -{ - TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), - "Only support float, half, and bfloat16."); -} - -// Parameterized constructor -lruPlugin::lruPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mDim); - read(d, mBlockSize); - read(d, mType); - read(d, mRemovePadding); - read(d, mPagedState); - read(d, mYEnabled); - read(d, mYBiasEnabled); - read(d, mFuseGateEnabled); - read(d, mGateBiasEnabled); - TLLM_CHECK(d == a + length); - TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), - "Only support float, half, and bfloat16."); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* lruPlugin::clone() const noexcept -{ - auto* plugin = new lruPlugin(mDim, mBlockSize, mType, mRemovePadding, mPagedState, mYEnabled, mYBiasEnabled, - mFuseGateEnabled, mGateBiasEnabled); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -// Outputs -// output_tensor: [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// state: [batch_size, dim] -nvinfer1::DimsExprs lruPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - if (outputIndex == 0) - { - return inputs[getXIdx()]; - } - return inputs[getStateIdx()]; -} - -bool lruPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - if (pos == getHostRequestTypesIdx() || pos == getLastTokenIdsIdx() || (mPagedState && pos == getSlotMappingIdx())) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (mPagedState && pos == getStateIdx()) - { - return inOut[pos].type == nvinfer1::DataType::kINT64; - } - else if (pos == getStateIdx() || pos == (nbInputs + 1)) - { - // Use float for both input and output state - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else - { - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } -} - -void lruPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t lruPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -void lruPlugin::setLruParams(lruParams& params, const size_t batch, const size_t dim, const size_t block_size, - const size_t maxSeqLen, void* statePtr, void const* x, void const* gate, void const* gate_bias, void const* gate_x, - void const* gate_x_bias, void const* gate_a, void const* gate_a_bias, void const* y, void const* y_bias, - void const* A, int const* lastTokenIds, int const* slotMapping, void* out, bool removePadding) -{ - // Reset the parameters - memset(¶ms, 0, sizeof(params)); - - params.batch = batch; - params.width = dim; - params.block_size = block_size; - params.max_seqlen = maxSeqLen; - params.remove_padding = removePadding; - - // Set the pointers and strides. - params.A_ptr = const_cast<void*>(A); - params.x_ptr = const_cast<void*>(x); - params.y_ptr = const_cast<void*>(y); - params.y_bias_ptr = const_cast<void*>(y_bias); - params.gate_ptr = const_cast<void*>(gate); - params.gate_bias_ptr = const_cast<void*>(gate_bias); - params.gate_x_ptr = const_cast<void*>(gate_x); - params.gate_x_bias_ptr = const_cast<void*>(gate_x_bias); - params.gate_a_ptr = const_cast<void*>(gate_a); - params.gate_a_bias_ptr = const_cast<void*>(gate_a_bias); - params.state_ptr = statePtr; - params.out_ptr = out; - params.last_token_ids_ptr = lastTokenIds; - params.slot_mapping_ptr = slotMapping; -} - -template <typename T> -int lruPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) -{ - // inputs - // 0. x [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding - // 1. A [dim] - // 2. state [batch_size, dim] or host [1] containing only pointer for paged_state - // 3. host_request_types [batch_size] int32. 0: context; 1: generation; 2: none. - // 4. last_token_ids [batch_size] int32 - // 5. state_slot_mapping [batch_size] int32, optional for paged state - // 6. y [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding - // 7. y_bias [dim] - // 8. gate [batch_size, seq_len, 2 * dim] or [num_tokens, 2 * dim] for remove_input_padding - // 9. gate_bias [2 * dim] - // 10. gate_x [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding - // 11. gate_a [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding - // 12. gate_x_bias [2 * dim] - // 13. gate_a_bias [2 * dim] - // outputs - // 0. output_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding - // 1. state [batch_size, dim] - auto const batch_size = inputDesc[getHostRequestTypesIdx()].dims.d[0]; - int max_seq_len; - if (mRemovePadding) - { - max_seq_len = -1; - } - else - { - max_seq_len = inputDesc[getXIdx()].dims.d[1]; - } - - // only support context or generation, not for both of them - RequestType const* reqTypes = static_cast<RequestType const*>(inputs[getHostRequestTypesIdx()]); - - lruParams lru_params; - - int const* slotMapping = mPagedState ? static_cast<int const*>(inputs[getSlotMappingIdx()]) : nullptr; - void const* y = mYEnabled ? inputs[getYIdx()] : nullptr; - void const* y_bias = mYBiasEnabled ? inputs[getYBiasIdx()] : nullptr; - void const* gate = mFuseGateEnabled ? inputs[getGateIdx()] : nullptr; - void const* gate_bias = (mFuseGateEnabled && mGateBiasEnabled) ? inputs[getGateBiasIdx()] : nullptr; - void const* gate_x = mFuseGateEnabled ? nullptr : inputs[getGateXIdx()]; - void const* gate_a = mFuseGateEnabled ? nullptr : inputs[getGateAIdx()]; - void const* gate_x_bias = (!mFuseGateEnabled && mGateBiasEnabled) ? inputs[getGateXBiasIdx()] : nullptr; - void const* gate_a_bias = (!mFuseGateEnabled && mGateBiasEnabled) ? inputs[getGateABiasIdx()] : nullptr; - - void* statePtr = mPagedState ? *reinterpret_cast<void**>(const_cast<void*>(inputs[getStateIdx()])) : outputs[1]; - - setLruParams(lru_params, batch_size, mDim, mBlockSize, max_seq_len, statePtr, inputs[getXIdx()], gate, gate_bias, - gate_x, gate_x_bias, gate_a, gate_a_bias, y, y_bias, inputs[getAIdx()], - static_cast<int const*>(inputs[getLastTokenIdsIdx()]), slotMapping, outputs[0], mRemovePadding); - - if (reqTypes[0] == RequestType::kCONTEXT) - { - invokeRGLRU<T>(lru_params, stream); - } - else if (reqTypes[0] == RequestType::kGENERATION) - { - invokeRGLRUUpdate<T>(lru_params, stream); - } - sync_check_cuda_error(stream); - return 0; -} - -int lruPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - if (mType == DataType::kHALF) - { - return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (mType == DataType::kFLOAT) - { - return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#ifdef ENABLE_BF16 - else if (mType == DataType::kBF16) - { - return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#endif - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType lruPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - if (index == 0) - { - return inputTypes[getXIdx()]; - } - else - { - return inputTypes[getStateIdx()]; - } -} - -// IPluginV2 Methods - -char const* lruPlugin::getPluginType() const noexcept -{ - return LRU_PLUGIN_NAME; -} - -char const* lruPlugin::getPluginVersion() const noexcept -{ - return LRU_PLUGIN_VERSION; -} - -int lruPlugin::getNbOutputs() const noexcept -{ - return mPagedState ? 1 : 2; -} - -int lruPlugin::initialize() noexcept -{ - return 0; -} - -void lruPlugin::terminate() noexcept {} - -size_t lruPlugin::getSerializationSize() const noexcept -{ - return sizeof(mDim) + sizeof(mBlockSize) + sizeof(mType) + sizeof(mRemovePadding) + sizeof(mPagedState) - + sizeof(mYEnabled) + sizeof(mYBiasEnabled) + sizeof(mFuseGateEnabled) + sizeof(mGateBiasEnabled); -} - -void lruPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mDim); - write(d, mBlockSize); - write(d, mType); - write(d, mRemovePadding); - write(d, mPagedState); - write(d, mYEnabled); - write(d, mYBiasEnabled); - write(d, mFuseGateEnabled); - write(d, mGateBiasEnabled); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void lruPlugin::destroy() noexcept -{ - delete this; -} - -/////////////// - -lruPluginCreator::lruPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("dim", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("block_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("remove_input_padding", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("paged_state", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("y_enabled", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("y_bias_enabled", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("fuse_gate_enabled", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("gate_bias_enabled", nullptr, PluginFieldType::kINT8)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* lruPluginCreator::getPluginName() const noexcept -{ - return LRU_PLUGIN_NAME; -} - -char const* lruPluginCreator::getPluginVersion() const noexcept -{ - return LRU_PLUGIN_VERSION; -} - -PluginFieldCollection const* lruPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* lruPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int dim{}; - int block_size{}; - bool removePadding{}; - bool pagedState{}; - bool yEnabled{}; - bool yBiasEnabled{}; - bool fuseGateEnabled{}; - bool gateBiasEnabled{}; - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "dim")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - dim = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - if (!strcmp(attrName, "block_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - block_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "remove_input_padding")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - removePadding = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "paged_state")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - pagedState = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "y_enabled")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - yEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "y_bias_enabled")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - yBiasEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "fuse_gate_enabled")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - fuseGateEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "gate_bias_enabled")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - gateBiasEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - } - try - { - auto* obj = new lruPlugin( - dim, block_size, type, removePadding, pagedState, yEnabled, yBiasEnabled, fuseGateEnabled, gateBiasEnabled); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* lruPluginCreator::deserializePlugin(char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call lruPlugin::destroy() - try - { - auto* obj = new lruPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.h b/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.h deleted file mode 100644 index ee4e0b989b34..000000000000 --- a/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.h +++ /dev/null @@ -1,239 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TRT_LRU_PLUGIN_H -#define TRT_LRU_PLUGIN_H -#include "tensorrt_llm/kernels/lruKernel.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> - -namespace tensorrt_llm::plugins -{ -// batch_size = num_ctx_requests or num_gen_requests -// num_ctx_requests = number of context requests (single sequence per request). -// num_gen_requests = number of generation requests (single sequences per request). -// can not support beam search - -// inputs -// 0. x [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// 1. A [dim] -// 2. state [batch_size, dim] or host [1] containing only pointer for paged_state -// 3. host_request_types [batch_size] int32. 0: context; 1: generation; 2: none. -// 4. last_token_ids [batch_size] int32 -// 5. state_slot_mapping [batch_size] int32, optional for paged state -// 6. y [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// 7. y_bias [dim] -// 8. gate [batch_size, seq_len, 2 * dim] or [num_tokens, 2 * dim] for remove_input_padding -// 9. gate_bias [2 * dim] -// 10. gate_x [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// 11. gate_a [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// 12. gate_x_bias [2 * dim] -// 13. gate_a_bias [2 * dim] -// outputs -// 0. output_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// 1. state [batch_size, dim] - -class lruPlugin : public BasePlugin -{ -public: - lruPlugin(int dim, int block_size, nvinfer1::DataType type, bool removePadding, bool pagedState, bool yEnabled, - bool yBiasEnabled, bool fuseGateEnabled, bool gateBiasEnabled); - - lruPlugin(void const* data, size_t length); - - ~lruPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - template <typename T> - int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - - enum class RequestType : int32_t - { - kCONTEXT = 0, - kGENERATION = 1 - }; - -private: - using IndexType = std::int32_t; - - IndexType getXIdx() const - { - return 0; - }; - - IndexType getAIdx() const - { - return 1; - }; - - IndexType getStateIdx() const - { - return 2; - }; - - IndexType getHostRequestTypesIdx() const - { - return 3; - }; - - IndexType getLastTokenIdsIdx() const - { - return 4; - }; - - IndexType getSlotMappingIdx() const - { - if (mPagedState) - return 5; - else - return 4; - }; - - IndexType getYIdx() const - { - if (mYEnabled) - return getSlotMappingIdx() + 1; - else - return getSlotMappingIdx(); - }; - - IndexType getYBiasIdx() const - { - if (mYBiasEnabled) - return getYIdx() + 1; - else - return getYIdx(); - }; - - IndexType getGateIdx() const - { - if (mFuseGateEnabled) - return getYBiasIdx() + 1; - else - return getYBiasIdx(); - }; - - IndexType getGateBiasIdx() const - { - if (mFuseGateEnabled && mGateBiasEnabled) - return getGateIdx() + 1; - else - return getGateIdx(); - }; - - IndexType getGateXIdx() const - { - if (mFuseGateEnabled) - return getGateBiasIdx(); - else - return getGateBiasIdx() + 1; - }; - - IndexType getGateAIdx() const - { - if (mFuseGateEnabled) - return getGateXIdx(); - else - return getGateXIdx() + 1; - }; - - IndexType getGateXBiasIdx() const - { - if (!mFuseGateEnabled && mGateBiasEnabled) - return getGateAIdx() + 1; - else - return getGateAIdx(); - }; - - IndexType getGateABiasIdx() const - { - if (!mFuseGateEnabled && mGateBiasEnabled) - return getGateXBiasIdx() + 1; - else - return getGateXBiasIdx(); - }; - - static void setLruParams(tensorrt_llm::kernels::lruParams& params, - // sizes - const size_t batch, const size_t dim, const size_t block_size, const size_t maxSeqLen, - // device pointers - void* statePtr, void const* x, void const* gate, void const* gate_bias, void const* gate_x, - void const* gate_x_bias, void const* gate_a, void const* gate_a_bias, void const* y, void const* y_bias, - void const* A, int const* lastTokenIds, int const* slotMapping, void* out, bool removePadding); - -private: - int mDim; - int mBlockSize; - nvinfer1::DataType mType; - bool mRemovePadding = false; - bool mPagedState = false; - bool mYEnabled = false; - bool mYBiasEnabled = false; - bool mFuseGateEnabled = false; - bool mGateBiasEnabled = false; -}; - -class lruPluginCreator : public BaseCreator -{ -public: - lruPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins - -#endif // TRT_LRU_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.cpp b/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.cpp deleted file mode 100644 index 16754248b84d..000000000000 --- a/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.cpp +++ /dev/null @@ -1,404 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mambaConv1dPlugin.h" -#include "tensorrt_llm/common/assert.h" -#include <algorithm> - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::MambaConv1dPluginCreator; -using tensorrt_llm::plugins::MambaConv1dPlugin; - -static char const* MAMBA_CONV1D_PLUGIN_VERSION{"1"}; -static char const* MAMBA_CONV1D_PLUGIN_NAME{"MambaConv1d"}; - -PluginFieldCollection MambaConv1dPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> MambaConv1dPluginCreator::mPluginAttributes; - -MambaConv1dPlugin::MambaConv1dPlugin(int dim, int dconv, int preStride, int postStride, nvinfer1::DataType type, - bool removePadding, bool pagedState, bool applySilu) - : mDim(dim) - , mDConv(dconv) - , mPreStride(preStride) - , mPostStride(postStride) - , mType(type) - , mRemovePadding(removePadding) - , mPagedState(pagedState) - , mApplySilu(applySilu) -{ - TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), - "Only support float, half, and bfloat16."); -} - -// Parameterized constructor -MambaConv1dPlugin::MambaConv1dPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mDim); - read(d, mDConv); - read(d, mPreStride); - read(d, mPostStride); - read(d, mType); - read(d, mRemovePadding); - read(d, mPagedState); - read(d, mApplySilu); - TLLM_CHECK(d == a + length); - TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), - "Only support float, half, and bfloat16."); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* MambaConv1dPlugin::clone() const noexcept -{ - auto* plugin - = new MambaConv1dPlugin(mDim, mDConv, mPreStride, mPostStride, mType, mRemovePadding, mPagedState, mApplySilu); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -// Outputs -// output_tensor: [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// state: [batch_size, dconv - 1, dim] -nvinfer1::DimsExprs MambaConv1dPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - if (outputIndex == 0) - { - auto ret = inputs[getInputTensorIdx()]; - ret.d[mRemovePadding ? 1 : 2] = exprBuilder.constant(mDim); - return ret; - } - return inputs[getConvStateIdx()]; -} - -bool MambaConv1dPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - if (pos == getHostRequestTypesIdx() || pos == getLastTokenIdsIdx() - || (mRemovePadding && pos == getHostContextLengthIdx()) || (mPagedState && pos == getSlotMappingIdx())) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (mPagedState && pos == getConvStateIdx()) - { - return inOut[pos].type == nvinfer1::DataType::kINT64; - } - else - { - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } -} - -void MambaConv1dPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t MambaConv1dPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -void MambaConv1dPlugin::setMambaConv1dParams(tensorrt_llm::kernels::MambaConv1dParamsBase& params, const size_t batch, - const size_t dim, const size_t maxSeqLen, const size_t dconv, const size_t preStride, const size_t postStride, - void const* inPtr, void const* stateInPtr, void* stateOutPtr, void const* convWeight, void const* convBias, - void* outPtr, int const* lastTokenIds, int const* stateSlotMapping, bool removePadding, bool applySilu) -{ - // Reset the parameters - memset(¶ms, 0, sizeof(params)); - - params.batch = batch; - params.dim = dim; - params.max_seqlen = maxSeqLen; - params.dconv = dconv; - params.pre_stride = preStride; - params.post_stride = postStride; - - params.remove_padding = removePadding; - params.apply_silu = applySilu; - - // Set the pointers and strides. - params.in_ptr = const_cast<void*>(inPtr); - params.state_in_ptr = const_cast<void*>(stateInPtr); - params.state_out_ptr = stateOutPtr; - params.weight_ptr = const_cast<void*>(convWeight); - params.bias_ptr = const_cast<void*>(convBias); - params.out_ptr = outPtr; - params.last_token_ids_ptr = lastTokenIds; - params.state_slot_mapping_ptr = stateSlotMapping; -} - -template <typename T> -int MambaConv1dPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) -{ - // inputs - // 0. input_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding - // 1. conv_state [batch_size, dconv - 1, dim] or host [1] containing only pointer for paged_state - // 2. weight [dim, 1, dconv] - // 3. bias [dim] - // 4. host_request_types [batch_size] int32. 0: context; 1: generation; 2: none. - // 5. last_token_ids [batch_size] int32 - // 6. host_context_lengths [batch_size] int32, optional for remove_input_padding - // 7. state_slot_mapping [batch_size] int32, optional - // outputs - // 0. output_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding - // 1. conv_state [batch_size, dconv - 1, dim] - auto const batchSize = inputDesc[getHostRequestTypesIdx()].dims.d[0]; - int maxSeqLen; - if (mRemovePadding) - { - int const* host_context_length = static_cast<int const*>(inputs[getHostContextLengthIdx()]); - maxSeqLen = *std::max_element(host_context_length, host_context_length + batchSize); - } - else - { - maxSeqLen = inputDesc[getInputTensorIdx()].dims.d[1]; - } - - // only support context or generation, not for both of them - RequestType const* reqTypes = static_cast<RequestType const*>(inputs[getHostRequestTypesIdx()]); - - MambaConv1dParamsBase mambaConv1dParams; - - int const* slotMapping = mPagedState ? static_cast<int const*>(inputs[getSlotMappingIdx()]) : nullptr; - void* stateInPtr = mPagedState ? *reinterpret_cast<void**>(const_cast<void*>(inputs[getConvStateIdx()])) - : const_cast<void*>(inputs[getConvStateIdx()]); - void* stateOutPtr - = mPagedState ? *reinterpret_cast<void**>(const_cast<void*>(inputs[getConvStateIdx()])) : outputs[1]; - - setMambaConv1dParams(mambaConv1dParams, batchSize, mDim, maxSeqLen, mDConv, mPreStride, mPostStride, - inputs[getInputTensorIdx()], stateInPtr, stateOutPtr, inputs[getWeightIdx()], inputs[getBiasIdx()], outputs[0], - static_cast<int const*>(inputs[getLastTokenIdsIdx()]), slotMapping, mRemovePadding, mApplySilu); - - if (reqTypes[0] == RequestType::kCONTEXT) - { - invokeMambaConv1dContext<T>(mambaConv1dParams, stream); - } - else if (reqTypes[0] == RequestType::kGENERATION) - { - invokeMambaConv1dGeneration<T>(mambaConv1dParams, stream); - } - sync_check_cuda_error(stream); - return 0; -} - -int MambaConv1dPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - if (mType == DataType::kHALF) - { - return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (mType == DataType::kFLOAT) - { - return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#ifdef ENABLE_BF16 - else if (mType == DataType::kBF16) - { - return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#endif - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType MambaConv1dPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - return inputTypes[getInputTensorIdx()]; -} - -// IPluginV2 Methods - -char const* MambaConv1dPlugin::getPluginType() const noexcept -{ - return MAMBA_CONV1D_PLUGIN_NAME; -} - -char const* MambaConv1dPlugin::getPluginVersion() const noexcept -{ - return MAMBA_CONV1D_PLUGIN_VERSION; -} - -int MambaConv1dPlugin::getNbOutputs() const noexcept -{ - return 2; -} - -int MambaConv1dPlugin::initialize() noexcept -{ - return 0; -} - -void MambaConv1dPlugin::terminate() noexcept {} - -size_t MambaConv1dPlugin::getSerializationSize() const noexcept -{ - return sizeof(mDim) + sizeof(mDConv) + sizeof(mPreStride) + sizeof(mPostStride) + sizeof(mType) - + sizeof(mRemovePadding) + sizeof(mPagedState) + sizeof(mApplySilu); -} - -void MambaConv1dPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mDim); - write(d, mDConv); - write(d, mPreStride); - write(d, mPostStride); - write(d, mType); - write(d, mRemovePadding); - write(d, mPagedState); - write(d, mApplySilu); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void MambaConv1dPlugin::destroy() noexcept -{ - delete this; -} - -/////////////// - -MambaConv1dPluginCreator::MambaConv1dPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("dim", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("dconv", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("pre_stride", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("post_stride", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("remove_input_padding", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("paged_state", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("apply_silu", nullptr, PluginFieldType::kINT8)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* MambaConv1dPluginCreator::getPluginName() const noexcept -{ - return MAMBA_CONV1D_PLUGIN_NAME; -} - -char const* MambaConv1dPluginCreator::getPluginVersion() const noexcept -{ - return MAMBA_CONV1D_PLUGIN_VERSION; -} - -PluginFieldCollection const* MambaConv1dPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* MambaConv1dPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int dim{}; - int dconv{}; - int pre_stride{}; - int post_stride{}; - bool removePadding{}; - bool pagedState{}; - bool applySilu{}; - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "dim")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - dim = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "dconv")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - dconv = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "pre_stride")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - pre_stride = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "post_stride")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - post_stride = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "remove_input_padding")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - removePadding = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "paged_state")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - pagedState = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "apply_silu")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - applySilu = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - } - try - { - auto* obj - = new MambaConv1dPlugin(dim, dconv, pre_stride, post_stride, type, removePadding, pagedState, applySilu); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* MambaConv1dPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call MambaConv1dPlugin::destroy() - try - { - auto* obj = new MambaConv1dPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.h b/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.h deleted file mode 100644 index d351b1cdc237..000000000000 --- a/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.h +++ /dev/null @@ -1,176 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TRT_MAMBA_CONV1D_PLUGIN_H -#define TRT_MAMBA_CONV1D_PLUGIN_H -#include "tensorrt_llm/kernels/mambaConv1dKernels.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> - -namespace tensorrt_llm::plugins -{ -// batch_size = num_ctx_requests or num_gen_requests -// num_ctx_requests = number of context requests (single sequence per request). -// num_gen_requests = number of generation requests (single sequences per request). -// can not support beam search - -// inputs -// 0. input_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// 1. conv_state [batch_size, dconv - 1, dim] or host [1] containing only pointer for paged_state -// 2. weight [1, dconv, dim] -// 3. bias [dim] -// 4. host_request_types [batch_size] int32. 0: context; 1: generation; 2: none. -// 5. last_token_ids [batch_size] int32 -// 6. host_context_lengths [batch_size] int32, optional for remove_input_padding -// 7. state_slot_mapping [batch_size] int32, optional -// outputs -// 0. output_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// 1. conv_state [batch_size, dconv - 1, dim] - -class MambaConv1dPlugin : public BasePlugin -{ -public: - MambaConv1dPlugin(int dim, int dconv, int preStride, int postStride, nvinfer1::DataType type, bool removePadding, - bool pagedState, bool applySilu); - - MambaConv1dPlugin(void const* data, size_t length); - - ~MambaConv1dPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - template <typename T> - int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - - enum class RequestType : int32_t - { - kCONTEXT = 0, - kGENERATION = 1 - }; - -private: - using IndexType = std::int32_t; - - IndexType getInputTensorIdx() const - { - return 0; - }; - - IndexType getConvStateIdx() const - { - return 1; - }; - - IndexType getWeightIdx() const - { - return 2; - }; - - IndexType getBiasIdx() const - { - return 3; - }; - - IndexType getHostRequestTypesIdx() const - { - return 4; - }; - - IndexType getLastTokenIdsIdx() const - { - return 5; - }; - - IndexType getHostContextLengthIdx() const - { - return 6; - }; - - IndexType getSlotMappingIdx() const - { - // if not remove input padding, host_context_length is not used, so the index is 6 - return mRemovePadding ? 7 : 6; - }; - - void setMambaConv1dParams(tensorrt_llm::kernels::MambaConv1dParamsBase& params, - // sizes - const size_t batch, const size_t dim, const size_t maxSeqLen, const size_t dconv, const size_t preStride, - const size_t postStride, - // device pointers - void const* inPtr, void const* stateInPtr, void* stateOutPtr, void const* convWeight, void const* convBias, - void* outPtr, int const* lastTokenIds, int const* stateSlotMapping, bool removePadding, bool applySilu); - -private: - int mDim; - int mDConv; - int mPreStride; - int mPostStride; - nvinfer1::DataType mType; - bool mRemovePadding = false; - bool mPagedState = false; - bool mApplySilu = true; -}; - -class MambaConv1dPluginCreator : public BaseCreator -{ -public: - MambaConv1dPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins - -#endif // TRT_MAMBA_CONV1D_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/mixtureOfExperts/CMakeLists.txt b/cpp/tensorrt_llm/plugins/mixtureOfExperts/CMakeLists.txt deleted file mode 100644 index 7cc985b60b7a..000000000000 --- a/cpp/tensorrt_llm/plugins/mixtureOfExperts/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.cpp b/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.cpp deleted file mode 100644 index ccce34850730..000000000000 --- a/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.cpp +++ /dev/null @@ -1,1314 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h" -#include "tensorrt_llm/common/cudaBf16Wrapper.h" -#include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/utils/debugUtils.h" -#include <numeric> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using namespace tensorrt_llm::plugins; -using tensorrt_llm::common::QuantMode; -using tensorrt_llm::common::nextWorkspacePtr; -using tensorrt_llm::common::calculateTotalWorkspaceSize; -using tensorrt_llm::plugins::MixtureOfExpertsPluginCreator; -using tensorrt_llm::plugins::MixtureOfExpertsPlugin; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; - -using LoraImpl = tensorrt_llm::kernels::LoraImpl; -using LoraParams = tensorrt_llm::kernels::LoraParams; - -static char const* MIXTURE_OF_EXPERTS_PLUGIN_VERSION{"1"}; -static char const* MIXTURE_OF_EXPERTS_PLUGIN_NAME{"MixtureOfExperts"}; -nvinfer1::PluginFieldCollection MixtureOfExpertsPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> MixtureOfExpertsPluginCreator::mPluginAttributes; - -MixtureOfExpertsPlugin::MixtureOfExpertsPlugin(bool remove_input_padding, int number_of_experts, int experts_per_token, - int expert_hidden_size, int expert_inter_size, int groupwise_quant_algo, int group_size, - ActivationType activation_type, nvinfer1::DataType type, nvinfer1::DataType weight_type, - nvinfer1::DataType output_type, QuantMode quant_mode, bool use_final_scales, bool use_bias, int tp_size, - int tp_rank, int ep_size, int ep_rank, bool force_determinism, int side_stream_id, - MixtureOfExpertsPluginProfilerPtr gemm_profiler_ptr, bool use_lora, nvinfer1::DataType lora_type, - LoraPluginProfilerPtr lora_profiler, int max_low_rank) - : mNumExperts(number_of_experts) - , mExpertsPerToken(experts_per_token) - , mExpertHiddenSize(expert_hidden_size) - , mExpertInterSize(expert_inter_size) - , mGroupwiseQuantAlgo(groupwise_quant_algo) - , mGroupSize(group_size) - , mActivationType(activation_type) - , mType(type) - , mWeightType(weight_type) - , mOutputType(output_type) - , mQuantMode(quant_mode) - , mUseFinalScales(use_final_scales) - , mUseBias(use_bias) - , mParallelismConfig(MOEParallelismConfig{tp_size, tp_rank, ep_size, ep_rank}) - , mUseDeterministicKernels(force_determinism) - , mSideStreamId(side_stream_id) - , mGemmProfiler(std::move(gemm_profiler_ptr)) - , mUseLora(use_lora) - , mLoraType(lora_type) - , mMaxLowRank(max_low_rank) - , mRemoveInputPadding(remove_input_padding) - , mLoraProfiler(std::move(lora_profiler)) -{ - init(); -} - -tensorrt_llm::plugins::MixtureOfExpertsPlugin::MixtureOfExpertsPlugin(MixtureOfExpertsPlugin const& other) - : mMOERunner() - , mNumExperts(other.mNumExperts) - , mExpertsPerToken(other.mExpertsPerToken) - , mExpertHiddenSize(other.mExpertHiddenSize) - , mExpertInterSize(other.mExpertInterSize) - , mGroupwiseQuantAlgo(other.mGroupwiseQuantAlgo) - , mGroupSize(other.mGroupSize) - , mActivationType(other.mActivationType) - , mType(other.mType) - , mWeightType(other.mWeightType) - , mOutputType(other.mOutputType) - , mQuantMode(other.mQuantMode) - , mUseFinalScales(other.mUseFinalScales) - , mUseBias(other.mUseBias) - , mParallelismConfig(other.mParallelismConfig) - , mDims(other.mDims) - , mUseDeterministicKernels(other.mUseDeterministicKernels) - , mSideStreamId(other.mSideStreamId) - , mGemmId1(other.mGemmId1) - , mGemmId2(other.mGemmId2) - , mGemmProfiler(other.mGemmProfiler) - , mUseLora(other.mUseLora) - , mLoraType(other.mLoraType) - , mMaxLowRank(other.mMaxLowRank) - , mRemoveInputPadding(other.mRemoveInputPadding) - , mLoraImpl1(other.mLoraImpl1) - , mLoraImpl2(other.mLoraImpl2) - , mLoraGemmId1(other.mLoraGemmId1) - , mLoraGemmId2(other.mLoraGemmId2) - , mLoraProfiler(other.mLoraProfiler) - , mLayerName(other.mLayerName) - , mNamespace(other.mNamespace) -{ - init(); -} - -size_t MixtureOfExpertsPlugin::getSerializationSize() const noexcept -{ - size_t size = sizeof(mRemoveInputPadding) + sizeof(mNumExperts) + sizeof(mExpertsPerToken) - + sizeof(mExpertHiddenSize) + sizeof(mExpertInterSize) + sizeof(mGroupwiseQuantAlgo) + sizeof(mGroupSize) - + sizeof(mActivationType) + sizeof(mType) + sizeof(mWeightType) + sizeof(mOutputType) - + sizeof(QuantMode::BaseType) + sizeof(mUseFinalScales) + sizeof(mUseBias) + sizeof(mParallelismConfig) - + sizeof(mDims) + sizeof(mUseDeterministicKernels) + sizeof(mSideStreamId) - + mGemmProfiler->getSerializationSize(mGemmId1) + mGemmProfiler->getSerializationSize(mGemmId2) - + sizeof(mUseLora) + sizeof(mLoraType) + sizeof(mMaxLowRank); - - if (hasLora()) - { - size += mLoraProfiler->getSerializationSize(mLoraGemmId1); - size += mLoraProfiler->getSerializationSize(mLoraGemmId2); - } - - return size; -} - -MixtureOfExpertsPlugin::MixtureOfExpertsPlugin(void const* data, size_t length, - MixtureOfExpertsPluginProfilerPtr gemm_profiler_ptr, LoraPluginProfilerPtr lora_profiler) - : mGemmProfiler(gemm_profiler_ptr) - , mLoraProfiler(lora_profiler) -{ - char const* d = reinterpret_cast<char const*>(data); - char const* a = d; - read(d, mRemoveInputPadding); - read(d, mNumExperts); - read(d, mExpertsPerToken); - read(d, mExpertHiddenSize); - read(d, mExpertInterSize); - read(d, mGroupwiseQuantAlgo); - read(d, mGroupSize); - read(d, mActivationType); - read(d, mType); - read(d, mWeightType); - read(d, mOutputType); - QuantMode::BaseType quant_mode; - read(d, quant_mode); - mQuantMode = QuantMode{quant_mode}; - read(d, mUseFinalScales); - read(d, mUseBias); - read(d, mParallelismConfig); - read(d, mDims); - read(d, mUseDeterministicKernels); - read(d, mSideStreamId); - read(d, mUseLora); - read(d, mLoraType); - read(d, mMaxLowRank); - - // Call init before deserialising the profiler to initialize mGemmId - init(); - mGemmProfiler->deserialize(d, mDims, mGemmId1); - mGemmProfiler->deserialize(d, mDims, mGemmId2); - - if (hasLora()) - { - mLoraProfiler->deserialize(d, mDims, mLoraGemmId1); - mLoraProfiler->deserialize(d, mDims, mLoraGemmId2); - } - - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -void MixtureOfExpertsPlugin::serialize(void* buffer) const noexcept -{ - char* d = static_cast<char*>(buffer); - char* a = d; - - write(d, mRemoveInputPadding); - write(d, mNumExperts); - write(d, mExpertsPerToken); - write(d, mExpertHiddenSize); - write(d, mExpertInterSize); - write(d, mGroupwiseQuantAlgo); - write(d, mGroupSize); - write(d, mActivationType); - write(d, mType); - write(d, mWeightType); - write(d, mOutputType); - write(d, mQuantMode.value()); - write(d, mUseFinalScales); - write(d, mUseBias); - write(d, mParallelismConfig); - write(d, mDims); - write(d, mUseDeterministicKernels); - write(d, mSideStreamId); - write(d, mUseLora); - write(d, mLoraType); - write(d, mMaxLowRank); - - mGemmProfiler->serialize(d, mGemmId1); - mGemmProfiler->serialize(d, mGemmId2); - - if (hasLora()) - { - mLoraProfiler->serialize(d, mLoraGemmId1); - mLoraProfiler->serialize(d, mLoraGemmId2); - } - - TLLM_CHECK(d == a + getSerializationSize()); -} - -template <typename Type, bool NeedQuant = false> -std::unique_ptr<kernels::CutlassMoeFCRunnerInterface> switch_output_type(nvinfer1::DataType output_type) -{ - switch (output_type) - { - case nvinfer1::DataType::kFP4: - case nvinfer1::DataType::kFP8: - // TODO We need an atomic FP8 reduction for the finalize fusions - TLLM_THROW("Outputting %d directly is not currently supported", static_cast<int>(output_type)); - // return std::make_unique<kernels::CutlassMoeFCRunner<Type, Type>>(); - case nvinfer1::DataType::kHALF: - if constexpr (NeedQuant) - { - return std::make_unique<kernels::CutlassMoeFCRunner<Type, Type, half, half>>(); - } - else - { - return std::make_unique<kernels::CutlassMoeFCRunner<Type, Type, half, Type>>(); - } -#ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: - if constexpr (NeedQuant) - { - return std::make_unique<kernels::CutlassMoeFCRunner<Type, Type, __nv_bfloat16, __nv_bfloat16>>(); - } - else - { - return std::make_unique<kernels::CutlassMoeFCRunner<Type, Type, __nv_bfloat16, Type>>(); - } -#endif - default: TLLM_THROW("Invalid output type %d", static_cast<int>(output_type)); - } -}; - -void MixtureOfExpertsPlugin::init() -{ - TLLM_CHECK_WITH_INFO(mType == DataType::kFP8 || mType == DataType::kFP4 || mOutputType == mType, - "MOE plugin only supports a different output type for FP4/FP8"); - TLLM_CHECK_WITH_INFO(mType != DataType::kFP8 || tensorrt_llm::common::getSMVersion() >= 89, - "MoE FP8 is not supported for architectures less than SM89"); - TLLM_CHECK_WITH_INFO(mType != DataType::kFP4 || (tensorrt_llm::common::getSMVersion() >= 100), - "MoE FP4 is only supported on architecture SM100 or later"); - - TLLM_CHECK_WITH_INFO(!hasLora() || mLoraType == mOutputType, "The LoraType need to keep same with moe OutputType."); - - if (mWeightType == nvinfer1::DataType::kINT8 && mQuantMode.hasInt4Weights()) - { - mWeightType = DataType::kINT4; - } - - if (mType == DataType::kHALF && mWeightType == DataType::kHALF) - { - mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<half, half>>(); - } - else if (mType == DataType::kFLOAT && mWeightType == DataType::kFLOAT) - { - mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<float, float>>(); - } - else if (mType == DataType::kHALF && mWeightType == DataType::kINT8) - { - mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<half, uint8_t>>(); - } - else if (mType == DataType::kHALF && mWeightType == DataType::kINT4) - { - mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<half, cutlass::uint4b_t>>(); - } -#ifdef ENABLE_FP8 - else if (mType == DataType::kFP8 && mWeightType == DataType::kINT4 && mOutputType == DataType::kHALF) - { - mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<__nv_fp8_e4m3, cutlass::uint4b_t, half, half>>(); - } -#endif -#ifdef ENABLE_BF16 - else if (mType == DataType::kBF16 && mWeightType == DataType::kBF16) - { - mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<__nv_bfloat16, __nv_bfloat16>>(); - } - else if (mType == DataType::kBF16 && mWeightType == DataType::kINT8) - { - mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<__nv_bfloat16, uint8_t>>(); - } - else if (mType == DataType::kBF16 && mWeightType == DataType::kINT4) - { - mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<__nv_bfloat16, cutlass::uint4b_t>>(); - } -#ifdef ENABLE_FP8 - else if (mType == DataType::kFP8 && mWeightType == DataType::kINT4 && mOutputType == DataType::kBF16) - { - mMOERunner = std::make_unique< - kernels::CutlassMoeFCRunner<__nv_fp8_e4m3, cutlass::uint4b_t, __nv_bfloat16, __nv_bfloat16>>(); - } -#endif -#endif - -#ifdef ENABLE_FP8 - if (mType == DataType::kFP8 && mWeightType == DataType::kFP8) - { - mMOERunner = switch_output_type<__nv_fp8_e4m3>(mOutputType); - } -#endif -#ifdef ENABLE_FP4 - if (mType == DataType::kFP4 && mWeightType == DataType::kFP4) - { - mMOERunner = switch_output_type<__nv_fp4_e2m1, true>(mOutputType); - } -#endif - - if (!mMOERunner) - { - TLLM_THROW( - "Could not construct the mixture of experts plugin with the requested input combination Activation: %d " - "Weight: %d Output: %d", - static_cast<int>(mType), static_cast<int>(mWeightType), static_cast<int>(mOutputType)); - } - - // Finalize fusion should be disabled if Lora is used. - mMOERunner->use_fused_finalize_ - = (mExpertsPerToken < 3 || !mUseDeterministicKernels) && !getEnvMOEDisableFinalizeFusion() && !hasLora(); - - mGemmId1 = GemmIDMoe{1, mNumExperts, mExpertsPerToken, mParallelismConfig, mExpertHiddenSize, mExpertInterSize, - mGroupSize, mActivationType, mType, mWeightType, mQuantMode, !mMOERunner->use_fused_finalize_}; - mGemmId2 = GemmIDMoe{2, mNumExperts, mExpertsPerToken, mParallelismConfig, mExpertHiddenSize, mExpertInterSize, - mGroupSize, mActivationType, mType, mWeightType, mQuantMode, !mMOERunner->use_fused_finalize_}; - mGemmProfiler->setMaxProfileM(16384 * mNumExperts / mExpertsPerToken); - - if (hasLora()) - { - auto cublasHandle = getCublasHandle(); - auto cublasLtHandle = getCublasLtHandle(); - auto cublasWrapper = std::make_shared<CublasMMWrapper>(cublasHandle, cublasLtHandle, nullptr, nullptr); - mLoraGemmId1 = GemmIdCublas(mExpertInterSize, mExpertHiddenSize, mLoraType, false, true, mLoraType); - mLoraGemmId2 = GemmIdCublas(mExpertHiddenSize, mExpertInterSize, mLoraType, false, true, mLoraType); - std::vector<int> loraOutSizes1 = {static_cast<int>(mExpertInterSize)}; - mLoraImpl1 = std::make_shared<LoraImpl>( - mExpertHiddenSize, loraOutSizes1, false, true, 1, mLoraType, mMaxLowRank, cublasWrapper); - std::vector<int> loraOutSizes2 = {static_cast<int>(mExpertHiddenSize)}; - mLoraImpl2 = std::make_shared<LoraImpl>( - mExpertInterSize, loraOutSizes2, false, true, 1, mLoraType, mMaxLowRank, cublasWrapper); - - TLLM_CUDA_CHECK(cudaEventCreate(&mMemcpyEvent)); - } - mSideStreamPtr = nullptr; - mDebugStallMain = tensorrt_llm::runtime::utils::stallStream("TLLM_DEBUG_MOE_STALL_MAIN"); - mDebugStallSide = tensorrt_llm::runtime::utils::stallStream("TLLM_DEBUG_MOE_STALL_SIDE"); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* MixtureOfExpertsPlugin::clone() const noexcept -{ - auto* plugin = new MixtureOfExpertsPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs MixtureOfExpertsPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - assert(outputIndex == getOutputTensorIndex() || outputIndex == getOutputDummyTensorIndex()); - return inputs[getInputTensorIndex()]; -} - -bool MixtureOfExpertsPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - TLLM_CHECK(0 <= pos && pos < getNbInputs() + getNbOutputs()); - TLLM_CHECK_WITH_INFO( - nbInputs == getNbInputs(), "Required input to plugin is missing. Expected %d Got %d", getNbInputs(), nbInputs); - TLLM_CHECK_WITH_INFO(nbOutputs == getNbOutputs(), "Required output to plugin is missing. Expected %d Got %d", - getNbOutputs(), nbOutputs); - - if (inOut[pos].format != TensorFormat::kLINEAR) - { - return false; - } - - if (pos == getExpertWeights1Index() || pos == getExpertWeights2Index()) - { - if (mGroupwiseQuantAlgo == 0) - { - auto normalized_weight_type - = mWeightType == nvinfer1::DataType::kINT4 ? nvinfer1::DataType::kINT8 : mWeightType; - return inOut[pos].type == normalized_weight_type; - } - else - { - return inOut[pos].type == mOutputType; - } - } - else if (pos == getTokenSelectedExpertsIndex()) - { - return inOut[pos].type == DataType::kINT32; - } - else if (pos == getTokenFinalScalesIndex()) - { - return inOut[pos].type == DataType::kFLOAT; - } - else if (pos == getExpertBias1Index() || pos == getExpertBias2Index()) - { - return inOut[pos].type == mOutputType; - } - else if (pos == nbInputs + getOutputTensorIndex()) - { - return inOut[pos].type == mOutputType; - } - else if (useSideStream() && pos == nbInputs + getOutputDummyTensorIndex()) - { - return inOut[pos].type == inOut[getInputDummyTensorIndex()].type; - } - else if (useSideStream() && pos == getInputDummyTensorIndex()) - { - return true; - } - else if (hasExpertFp8QuantScales() && getExpertFP8Dequant1Index() <= pos && pos <= getExpertFP8QuantFinalIndex()) - { - return inOut[pos].type == DataType::kFLOAT; - } - else if (hasExpertIntQuantScales() && getExpertIntQuantScale1Index() <= pos - && pos <= getExpertIntQuantScale2Index()) - { - return inOut[pos].type == mOutputType; - } - else if (hasFP4QuantScales() && getFP4GlobalActSF1Index() <= pos && pos <= getFP4GlobalSF2Index()) - { - if (pos == getFP4WeightSF1Index() || pos == getFP4WeightSF2Index()) - return inOut[pos].type == nvinfer1::DataType::kFP8; - else - return inOut[pos].type == nvinfer1::DataType::kFLOAT; - } - else if (hasLora() && hasExpertFp8QuantScales() && pos == getInputFP8DequantIndex()) - { - return inOut[pos].type == nvinfer1::DataType::kFLOAT; - } - else if (hasExpertWeightQuantZeros() && getExpertIntQuantZeros1Index() <= pos - && pos <= getExpertIntQuantZeros2Index()) - { - return inOut[pos].type == mOutputType; - } - else if (hasExpertPrequantScales() && getExpertPrequantScales1Index() <= pos - && pos <= getExpertPrequantScales2Index()) - { - return inOut[pos].type == mOutputType; - } - else if (hasGroupwiseFp8Alpha() && getExpertFp8Alpha1Index() <= pos && pos <= getExpertFp8Alpha2Index()) - { - return inOut[pos].type == DataType::kFLOAT; - } - else if (hasLora() && pos == getHostRequestTypeIndex()) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (hasLora() && (pos == getLoraFC1RanksIndex() || pos == getLoraFC2RanksIndex())) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (hasGatedLoraWeightsAndRanks() && pos == getLoraGatedRanksIndex()) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (hasLora() && (pos == getLoraFC1WeightPtrsIndex() || pos == getLoraFC2WeightPtrsIndex())) - { - return inOut[pos].type == nvinfer1::DataType::kINT64; - } - else if (hasGatedLoraWeightsAndRanks() && pos == getLoraGatedWeightPtrsIndex()) - { - return inOut[pos].type == nvinfer1::DataType::kINT64; - } - else if (hasLora() && mRemoveInputPadding && pos == getHostContextLengthIndex()) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if ((hasFP4QuantScales() || hasGroupwiseFp8Alpha()) && pos == getInputTensorIndex()) - { - return inOut[pos].type == mOutputType; - } - else - { - return inOut[pos].type == mType; - } - - return false; -} - -void MixtureOfExpertsPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - TLLM_CHECK_WITH_INFO( - nbInputs == getNbInputs(), "Required input to plugin is missing. Expected %d Got %d", getNbInputs(), nbInputs); - TLLM_CHECK_WITH_INFO(nbOutputs == getNbOutputs(), "Required output to plugin is missing. Expected %d Got %d", - getNbOutputs(), nbOutputs); - - auto in_tensor = in[getInputTensorIndex()]; - - auto const minM - = std::accumulate(in_tensor.min.d, in_tensor.min.d + in_tensor.min.nbDims - 1, 1, std::multiplies<int>()); - auto const maxM - = std::accumulate(in_tensor.max.d, in_tensor.max.d + in_tensor.max.nbDims - 1, 1, std::multiplies<int>()); - - auto weights_1 = in[getExpertWeights1Index()]; - auto weights_2 = in[getExpertWeights2Index()]; - int inner_dim_idx = getGemmShapeInnerDimIndex(); - int const maxK = weights_1.max.d[inner_dim_idx]; - int const maxN = weights_2.max.d[inner_dim_idx]; - int const minK = weights_1.min.d[inner_dim_idx]; - int const minN = weights_2.min.d[inner_dim_idx]; - - TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); - TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); - TLLM_CHECK_WITH_INFO(maxK == mExpertHiddenSize && maxN == mExpertInterSize, - "Configured tensor sizes %dx%d does not match constructor param size %ldx%ld", maxK, maxN, mExpertHiddenSize, - mExpertInterSize); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, maxN, maxK}; - } - - mGemmId1 = GemmIDMoe{1, mNumExperts, mExpertsPerToken, mParallelismConfig, mExpertHiddenSize, mExpertInterSize, - mGroupSize, mActivationType, mType, mWeightType, mQuantMode, !mMOERunner->use_fused_finalize_}; - mGemmId2 = GemmIDMoe{2, mNumExperts, mExpertsPerToken, mParallelismConfig, mExpertHiddenSize, mExpertInterSize, - mGroupSize, mActivationType, mType, mWeightType, mQuantMode, !mMOERunner->use_fused_finalize_}; - - if (hasLora()) - { - auto const N = utils::computeNDimension(true, in[getHostRequestTypeIndex()].max); - mLoraGemmId1 = GemmIdCublas(N, mExpertHiddenSize, mLoraType, false, true, mLoraType); - mLoraGemmId2 = GemmIdCublas(N, mExpertInterSize, mLoraType, false, true, mLoraType); - } -} - -auto MixtureOfExpertsPlugin::setupWorkspace(void* base_ptr, int64_t num_tokens, int num_reqs) const -> WorkspaceInfo -{ - size_t moe_workspace_size - = mMOERunner->getWorkspaceSize(num_tokens, mExpertHiddenSize, mExpertInterSize, mNumExperts, mExpertsPerToken, - mActivationType, mParallelismConfig, hasLora(), /*use_deepseek_fp8_block_scale=*/false, - /*min_latency_mode=*/false, hasExpertPrequantScales()); - - // Permutation map - size_t src_to_dest_map_size = mExpertsPerToken * num_tokens * sizeof(int); - - size_t lora_workspace_size = 0; - if (hasLora()) - { - int64_t num_reqs_lora = std::min(num_tokens * mExpertsPerToken, static_cast<int64_t>(num_reqs * mNumExperts)); - lora_workspace_size - = std::max(mLoraImpl1->getWorkspaceSize(num_tokens * mExpertsPerToken, num_reqs_lora, mLoraType), - mLoraImpl2->getWorkspaceSize(num_tokens * mExpertsPerToken, num_reqs_lora, mLoraType)); - } - - std::vector<size_t> workspaces{ - moe_workspace_size, - src_to_dest_map_size, - lora_workspace_size, - }; - - WorkspaceInfo info{}; - info.size = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); - - if (base_ptr) - { - info.workspace = base_ptr; - info.src_to_dest_map = nextWorkspacePtr((int8_t*) info.workspace, moe_workspace_size); - info.lora_workspace = nextWorkspacePtr((int8_t*) info.src_to_dest_map, src_to_dest_map_size); - } - - return info; -} - -int64_t MixtureOfExpertsPlugin::getNumTokens(nvinfer1::PluginTensorDesc const* input_tensors) const -{ - int ndim = input_tensors[getInputTensorIndex()].dims.nbDims; - TLLM_CHECK_WITH_INFO( - 3 == ndim || 2 == ndim, "hidden_state dimension should be either 2 [b*s, hidden], or 3 [b, s, hidden]"); - int64_t num_tokens = input_tensors[getInputTensorIndex()].dims.d[0]; - if (ndim == 3) - { - num_tokens *= input_tensors[getInputTensorIndex()].dims.d[1]; - } - return num_tokens; -} - -size_t MixtureOfExpertsPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - TLLM_CHECK_WITH_INFO( - nbInputs == getNbInputs(), "Required input to plugin is missing. Expected %d Got %d", getNbInputs(), nbInputs); - TLLM_CHECK_WITH_INFO(nbOutputs == getNbOutputs(), "Required output to plugin is missing. Expected %d Got %d", - getNbOutputs(), nbOutputs); - - if (useSideStream()) - { - return 0; - } - int const num_tokens = getNumTokens(inputs); - int const num_lora_reqs = getNumLoraRequests(inputs); - return setupWorkspace(nullptr, num_tokens, num_lora_reqs).size; -} - -MOEParallelismConfig MixtureOfExpertsPlugin::getParallelismConfig() const -{ - return mParallelismConfig; -} - -QuantParams tensorrt_llm::plugins::MixtureOfExpertsPlugin::getQuantParams(nvinfer1::PluginTensorDesc const* inputDesc, - void const* const* inputs, int scale_1_idx, int scale_2_idx, int scale_3_idx, int scale_4_idx, int scale_5_idx, - int scale_6_idx, int scale_7_idx, int scale_8_idx) const -{ - void const* scale_1 = scale_1_idx >= 0 ? inputs[scale_1_idx] : nullptr; - void const* scale_2 = scale_2_idx >= 0 ? inputs[scale_2_idx] : nullptr; - void const* scale_3 = scale_3_idx >= 0 ? inputs[scale_3_idx] : nullptr; - void const* scale_4 = scale_4_idx >= 0 ? inputs[scale_4_idx] : nullptr; - void const* scale_5 = scale_5_idx >= 0 ? inputs[scale_5_idx] : nullptr; - void const* scale_6 = scale_6_idx >= 0 ? inputs[scale_6_idx] : nullptr; - void const* scale_7 = scale_7_idx >= 0 ? inputs[scale_7_idx] : nullptr; - void const* scale_8 = scale_8_idx >= 0 ? inputs[scale_8_idx] : nullptr; - nvinfer1::PluginTensorDesc const* desc_1 = scale_1_idx >= 0 ? &inputDesc[scale_1_idx] : nullptr; - nvinfer1::PluginTensorDesc const* desc_2 = scale_2_idx >= 0 ? &inputDesc[scale_2_idx] : nullptr; - nvinfer1::PluginTensorDesc const* desc_3 = scale_3_idx >= 0 ? &inputDesc[scale_3_idx] : nullptr; - nvinfer1::PluginTensorDesc const* desc_4 = scale_4_idx >= 0 ? &inputDesc[scale_4_idx] : nullptr; - nvinfer1::PluginTensorDesc const* desc_5 = scale_5_idx >= 0 ? &inputDesc[scale_5_idx] : nullptr; - nvinfer1::PluginTensorDesc const* desc_6 = scale_6_idx >= 0 ? &inputDesc[scale_6_idx] : nullptr; - auto const gated_inter_size = isGatedActivation(mActivationType) ? mExpertInterSize * 2 : mExpertInterSize; - auto const experts_per_node = mNumExperts / mParallelismConfig.ep_size; - if (hasExpertIntQuantScales()) - { - TLLM_CHECK(scale_1 && scale_2); - if (!hasGroupwiseIntQuantScales()) - { - TLLM_CHECK(!scale_3 && !scale_4 && !scale_5 && !scale_6); - TLLM_CHECK(desc_1->dims.nbDims == 2); - TLLM_CHECK(desc_2->dims.nbDims == 2); - TLLM_CHECK_WITH_INFO( - desc_1->dims.d[0] == experts_per_node, "Incorrect number of experts in int quant scale"); - TLLM_CHECK(desc_1->dims.d[1] == gated_inter_size); - TLLM_CHECK_WITH_INFO( - desc_2->dims.d[0] == experts_per_node, "Incorrect number of experts in int quant scale"); - TLLM_CHECK(desc_2->dims.d[1] == mExpertHiddenSize); - return QuantParams::Int(scale_1, scale_2); - } - else - { - TLLM_CHECK(desc_1->dims.nbDims == 3); - TLLM_CHECK(desc_2->dims.nbDims == 3); - TLLM_CHECK((scale_3 && scale_4) || !hasExpertPrequantScales()); - TLLM_CHECK((scale_5 && scale_6) || !hasExpertWeightQuantZeros()); - TLLM_CHECK((scale_7 && scale_8) || !hasGroupwiseFp8Alpha()); - return QuantParams::GroupWise(mGroupSize, scale_1, scale_2, scale_3, scale_4, scale_5, scale_6, - static_cast<float const*>(scale_7), static_cast<float const*>(scale_8)); - } - } - else if (hasExpertFp8QuantScales()) - { - TLLM_CHECK(scale_1 && scale_2 && scale_3); - TLLM_CHECK(scale_4 || !hasExpertFp8FinalQuantScales()); - TLLM_CHECK((scale_5 != nullptr) == hasLora()); - TLLM_CHECK(!scale_6); - TLLM_CHECK(desc_1->dims.nbDims == 2); - TLLM_CHECK(desc_2->dims.nbDims == 1); - TLLM_CHECK(desc_3->dims.nbDims == 2); - TLLM_CHECK_WITH_INFO( - desc_1->dims.d[0] == experts_per_node && desc_1->dims.d[1] == 1, "Incorrect shape for weight FP8 scale"); - TLLM_CHECK(desc_2->dims.d[0] == 1); - TLLM_CHECK_WITH_INFO( - desc_3->dims.d[0] == experts_per_node && desc_3->dims.d[1] == 1, "Incorrect shape for weight FP8 scale"); - return QuantParams::FP8(static_cast<float const*>(scale_1), static_cast<float const*>(scale_2), - static_cast<float const*>(scale_3), static_cast<float const*>(scale_4), static_cast<float const*>(scale_5)); - } - else if (hasFP4QuantScales()) - { - TLLM_CHECK(scale_1 && scale_2 && scale_3 && scale_4 && scale_5 && scale_6); - TLLM_CHECK(desc_1->dims.nbDims == 1); - TLLM_CHECK(desc_2->dims.nbDims == 3); - TLLM_CHECK(desc_3->dims.nbDims == 1); - TLLM_CHECK(desc_4->dims.nbDims == 1); - TLLM_CHECK(desc_5->dims.nbDims == 3); - TLLM_CHECK(desc_6->dims.nbDims == 1); - TLLM_CHECK(desc_1->dims.d[0] == 1); - TLLM_CHECK_WITH_INFO(desc_2->dims.d[0] == experts_per_node && desc_2->dims.d[1] == gated_inter_size - && desc_2->dims.d[2] - == mExpertHiddenSize / TmaWarpSpecializedGroupedGemmInput::NVFP4BlockScaleVectorSize, - "Incorrect shape for FP4 scale"); - TLLM_CHECK_WITH_INFO(desc_3->dims.d[0] == experts_per_node, "Incorrect shape for FP4 scale"); - TLLM_CHECK(desc_4->dims.d[0] == 1); - TLLM_CHECK_WITH_INFO(desc_5->dims.d[0] == experts_per_node && desc_5->dims.d[1] == mExpertHiddenSize - && desc_5->dims.d[2] - == mExpertInterSize / TmaWarpSpecializedGroupedGemmInput::NVFP4BlockScaleVectorSize, - "Incorrect shape for FP4 scale"); - TLLM_CHECK_WITH_INFO(desc_6->dims.d[0] == experts_per_node, "Incorrect shape for FP4 scale"); - return QuantParams::FP4(static_cast<float const*>(scale_1), - static_cast<TmaWarpSpecializedGroupedGemmInput::ElementSF const*>(scale_2), - static_cast<float const*>(scale_3), static_cast<float const*>(scale_4), - static_cast<TmaWarpSpecializedGroupedGemmInput::ElementSF const*>(scale_5), - static_cast<float const*>(scale_6)); - } - return {}; -} - -int MixtureOfExpertsPlugin::getNumLoraRequests(nvinfer1::PluginTensorDesc const* input_tensors) const -{ - if (!hasLora()) - return 0; - int num_reqs = input_tensors[getLoraFC1RanksIndex()].dims.d[0]; - return num_reqs; -} - -LoraParams MixtureOfExpertsPlugin::getLoraParams( - nvinfer1::PluginTensorDesc const* inputDesc, void const* const* inputs, void* workspace) -{ - TLLM_CHECK(hasLora()); - - int const num_reqs = getNumLoraRequests(inputDesc); - int64_t const num_tokens = getNumTokens(inputDesc); - bool is_gated_actiation = isGatedActivation(mActivationType); - - mLoraExpandFC1WeightPtrs.clear(); - mLoraExpandFC2WeightPtrs.clear(); - mLoraExpandFC1Ranks.clear(); - mLoraExpandFC2Ranks.clear(); - - mLoraExpandFC1WeightPtrs.reserve(num_tokens * 2); - mLoraExpandFC2WeightPtrs.reserve(num_tokens * 2); - mLoraExpandFC1Ranks.reserve(num_tokens); - mLoraExpandFC2Ranks.reserve(num_tokens); - - if (is_gated_actiation) - { - mLoraExpandGatedWeightPtrs.clear(); - mLoraExpandGatedRanks.clear(); - mLoraExpandGatedWeightPtrs.reserve(num_tokens * 2); - mLoraExpandGatedRanks.reserve(num_tokens); - } - - int const seq_len = mRemoveInputPadding ? 0 : inputDesc[getInputTensorIndex()].dims.d[1]; - int32_t const* req_types = static_cast<int32_t const*>(inputs[getHostRequestTypeIndex()]); - int32_t const* host_context_lens - = mRemoveInputPadding ? static_cast<int32_t const*>(inputs[getHostContextLengthIndex()]) : nullptr; - - auto const fc1_lora_weight_ptrs = static_cast<void const* const*>(inputs[getLoraFC1WeightPtrsIndex()]); - auto const fc1_lora_ranks = static_cast<int32_t const*>(inputs[getLoraFC1RanksIndex()]); - - auto const fc2_lora_weight_ptrs = static_cast<void const* const*>(inputs[getLoraFC2WeightPtrsIndex()]); - auto const fc2_lora_ranks = static_cast<int32_t const*>(inputs[getLoraFC2RanksIndex()]); - - auto const gated_lora_weight_ptrs - = is_gated_actiation ? static_cast<void const* const*>(inputs[getLoraGatedWeightPtrsIndex()]) : nullptr; - auto const gated_lora_ranks - = is_gated_actiation ? static_cast<int32_t const*>(inputs[getLoraGatedRanksIndex()]) : nullptr; - - int idx = 0; - for (int req_id = 0; req_id < num_reqs; req_id++) - { - RequestType const reqType = static_cast<RequestType const>(req_types[req_id]); - if (reqType == RequestType::kGENERATION) - { - // lora_weight_ptrs has 3 pointers for each module: A,B, and an optional DoRA magnitude - // the current LoRA implementation does not apply DoRA scaling, so the magnitude is ignored - mLoraExpandFC1WeightPtrs.push_back(fc1_lora_weight_ptrs[req_id * 3]); - mLoraExpandFC1WeightPtrs.push_back(fc1_lora_weight_ptrs[req_id * 3 + 1]); - mLoraExpandFC1Ranks.push_back(fc1_lora_ranks[req_id]); - - mLoraExpandFC2WeightPtrs.push_back(fc2_lora_weight_ptrs[req_id * 3]); - mLoraExpandFC2WeightPtrs.push_back(fc2_lora_weight_ptrs[req_id * 3 + 1]); - mLoraExpandFC2Ranks.push_back(fc2_lora_ranks[req_id]); - - if (is_gated_actiation) - { - mLoraExpandGatedWeightPtrs.push_back(gated_lora_weight_ptrs[req_id * 3]); - mLoraExpandGatedWeightPtrs.push_back(gated_lora_weight_ptrs[req_id * 3 + 1]); - mLoraExpandGatedRanks.push_back(gated_lora_ranks[req_id]); - } - - idx += 1; - } - else - { - int context_len = (mRemoveInputPadding ? host_context_lens[req_id] : seq_len); - - for (int context_id = 0; context_id < context_len; context_id++) - { - mLoraExpandFC1WeightPtrs.push_back(fc1_lora_weight_ptrs[req_id * 3]); - mLoraExpandFC1WeightPtrs.push_back(fc1_lora_weight_ptrs[req_id * 3 + 1]); - mLoraExpandFC1Ranks.push_back(fc1_lora_ranks[req_id]); - - mLoraExpandFC2WeightPtrs.push_back(fc2_lora_weight_ptrs[req_id * 3]); - mLoraExpandFC2WeightPtrs.push_back(fc2_lora_weight_ptrs[req_id * 3 + 1]); - mLoraExpandFC2Ranks.push_back(fc2_lora_ranks[req_id]); - - if (is_gated_actiation) - { - mLoraExpandGatedWeightPtrs.push_back(gated_lora_weight_ptrs[req_id * 3]); - mLoraExpandGatedWeightPtrs.push_back(gated_lora_weight_ptrs[req_id * 3 + 1]); - mLoraExpandGatedRanks.push_back(gated_lora_ranks[req_id]); - } - } - idx += context_len; - } - } - - TLLM_CHECK_WITH_INFO(idx == num_tokens, fmtstr("idx %d num_tokens %ld", idx, num_tokens)); - - return LoraParams(num_reqs, mLoraExpandFC1Ranks.data(), mLoraExpandFC1WeightPtrs.data(), mLoraExpandFC2Ranks.data(), - mLoraExpandFC2WeightPtrs.data(), mLoraImpl1, mLoraImpl2, workspace, &mMemcpyEvent, mLoraExpandGatedRanks.data(), - mLoraExpandGatedWeightPtrs.data()); -} - -int MixtureOfExpertsPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace_ptr, - cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - - int64_t const num_tokens = getNumTokens(inputDesc); - int64_t const num_reqs = getNumLoraRequests(inputDesc); - - if (useSideStream()) - { - // Prepare the side stream - if (!mSideStreamPtr) - { - auto const resource_name = nvinfer1::pluginInternal::SideStream::getResourceKey(mSideStreamId); - nvinfer1::pluginInternal::SideStream side_stream{}; - mSideStreamPtr = reinterpret_cast<nvinfer1::pluginInternal::SideStream*>( - getPluginRegistry()->acquirePluginResource(resource_name.c_str(), &side_stream)); - } - // Debug the code with the main stream stalled (only executed when the environment variable - // TLLM_DEBUG_MOE_STALL_MAIN is set and has a positive value) - mSideStreamPtr->stallMainStream("TLLM_DEBUG_MOE_STALL_MAIN", stream, mDebugStallMain); - // The side stream waits for the inputs managed by the main stream to be ready - mSideStreamPtr->waitMainStreamOnSideStream(stream); - // Provide data dependency for the shared experts running after this plugin by copying inputs on the main stream - size_t count = 1; - for (int i = 0; i < inputDesc[getInputDummyTensorIndex()].dims.nbDims; ++i) - { - count *= inputDesc[getInputDummyTensorIndex()].dims.d[i]; - } - count *= tensorrt_llm::runtime::BufferDataType(inputDesc[getInputDummyTensorIndex()].type).getSize(); - TLLM_CUDA_CHECK(cudaMemcpyAsync(outputs[getOutputDummyTensorIndex()], inputs[getInputDummyTensorIndex()], count, - cudaMemcpyDeviceToDevice, stream)); - // Switch from the main stream to the side stream - stream = mSideStreamPtr->getStream(); - // The workspace is managed by the side stream (otherwise, the lifetime of workspace may be incorrect) - auto const workspace_size = setupWorkspace(nullptr, num_tokens, num_reqs).size; - workspace_ptr = mSideStreamPtr->getWorkspacePtr(workspace_size); - } - auto workspace = setupWorkspace(workspace_ptr, num_tokens, num_reqs); - - auto w1_desc = inputDesc[getExpertWeights1Index()]; - auto w2_desc = inputDesc[getExpertWeights2Index()]; - TLLM_CHECK(w1_desc.dims.nbDims == 3); - auto const experts_per_node = mNumExperts / mParallelismConfig.ep_size; - TLLM_CHECK(w1_desc.dims.d[0] == experts_per_node); - TLLM_CHECK(w2_desc.dims.nbDims == 3); - TLLM_CHECK(w2_desc.dims.d[0] == experts_per_node); - - auto [inner_packed_elements, outer_packed_elements] = getWeightPackedElements(); - int inner_dim_idx = getGemmShapeInnerDimIndex(); - int outer_dim_idx = getGemmShapeOuterDimIndex(); - TLLM_CHECK(w1_desc.dims.d[inner_dim_idx] * inner_packed_elements == mExpertHiddenSize); - if (isGatedActivation(mActivationType)) - { - TLLM_CHECK(w1_desc.dims.d[outer_dim_idx] * outer_packed_elements == mExpertInterSize * 2); - } - else - { - TLLM_CHECK(w1_desc.dims.d[outer_dim_idx] * outer_packed_elements == mExpertInterSize); - } - - TLLM_CHECK(w2_desc.dims.d[inner_dim_idx] * inner_packed_elements == mExpertInterSize); - TLLM_CHECK(w2_desc.dims.d[outer_dim_idx] * outer_packed_elements == mExpertHiddenSize); - - QuantParams quant_params{}; - if (hasExpertIntQuantScales()) - { - if (mGroupSize > 0) - { - quant_params = getQuantParams(inputDesc, inputs, getExpertIntQuantScale1Index(), - getExpertIntQuantScale2Index(), hasExpertPrequantScales() ? getExpertPrequantScales1Index() : -1, - hasExpertPrequantScales() ? getExpertPrequantScales2Index() : -1, - hasExpertWeightQuantZeros() ? getExpertIntQuantZeros1Index() : -1, - hasExpertWeightQuantZeros() ? getExpertIntQuantZeros2Index() : -1, - hasGroupwiseFp8Alpha() ? getExpertFp8Alpha1Index() : -1, - hasGroupwiseFp8Alpha() ? getExpertFp8Alpha2Index() : -1); - } - else - { - quant_params - = getQuantParams(inputDesc, inputs, getExpertIntQuantScale1Index(), getExpertIntQuantScale2Index()); - } - } - else if (hasExpertFp8QuantScales()) - { - quant_params = getQuantParams(inputDesc, inputs, // - getExpertFP8Dequant1Index(), // - getExpertFP8Quant2Index(), // - getExpertFP8Dequant2Index(), // - hasExpertFp8FinalQuantScales() ? getExpertFP8QuantFinalIndex() : -1, - hasLora() ? getInputFP8DequantIndex() : -1); - } - else if (hasFP4QuantScales()) - { - quant_params = getQuantParams(inputDesc, inputs, // - getFP4GlobalActSF1Index(), // - getFP4WeightSF1Index(), // - getFP4GlobalSF1Index(), // - getFP4GlobalActSF2Index(), // - getFP4WeightSF2Index(), // - getFP4GlobalSF2Index() // - ); - } - - LoraParams lora_params{}; - - if (hasLora()) - { - lora_params = getLoraParams(inputDesc, inputs, workspace.lora_workspace); - auto lora_gemm1 = mLoraProfiler->getBestConfig(num_tokens, mLoraGemmId1); - auto lora_gemm2 = mLoraProfiler->getBestConfig(num_tokens, mLoraGemmId2); - - mLoraImpl1->setBestTactic(lora_gemm1); - mLoraImpl2->setBestTactic(lora_gemm2); - } - - std::optional<tensorrt_llm::cutlass_extensions::CutlassGemmConfig> gemm1; - std::optional<tensorrt_llm::cutlass_extensions::CutlassGemmConfig> gemm2; - if (common::getEnvForceDeterministicMOE()) - { - gemm1 = mMOERunner->getTactics(MoeGemmId::GEMM_1)[0]; - gemm2 = mMOERunner->getTactics(MoeGemmId::GEMM_2)[0]; - } - else - { - gemm1 = mGemmProfiler->getBestConfig(num_tokens, mGemmId1); - gemm2 = mGemmProfiler->getBestConfig(num_tokens, mGemmId2); - } - - MoeMinLatencyParams min_latency_params{}; - mMOERunner->setTactic(gemm1, gemm2); -#ifdef USING_OSS_CUTLASS_MOE_GEMM - mMOERunner->runMoe(inputs[getInputTensorIndex()], nullptr, true, - static_cast<int const*>(inputs[getTokenSelectedExpertsIndex()]), - hasFinalScales() ? static_cast<float const*>(inputs[getTokenFinalScalesIndex()]) : nullptr, - inputs[getExpertWeights1Index()], hasBias() ? inputs[getExpertBias1Index()] : nullptr, - ActivationParams(mActivationType), inputs[getExpertWeights2Index()], - hasBias() ? inputs[getExpertBias2Index()] : nullptr, quant_params, num_tokens, num_tokens, mExpertHiddenSize, - mExpertHiddenSize /*TRT does not support padding, safe to assume padded/unpadded hidden sizes are the same*/, - mExpertInterSize, mNumExperts, mExpertsPerToken, static_cast<char*>(workspace.workspace), - // Outputs - outputs[getOutputTensorIndex()], static_cast<int*>(workspace.src_to_dest_map), mParallelismConfig, - /*enable_alltoall=*/false, hasLora(), lora_params, /*use_deepseek_fp8_block_scale=*/false, - /*min_latency_mode=*/false, min_latency_params, stream); -#else - mMOERunner->runMoe(inputs[getInputTensorIndex()], nullptr, true, - static_cast<int const*>(inputs[getTokenSelectedExpertsIndex()]), - hasFinalScales() ? static_cast<float const*>(inputs[getTokenFinalScalesIndex()]) : nullptr, - inputs[getExpertWeights1Index()], hasBias() ? inputs[getExpertBias1Index()] : nullptr, - ActivationParams(mActivationType), inputs[getExpertWeights2Index()], - hasBias() ? inputs[getExpertBias2Index()] : nullptr, quant_params, num_tokens, num_tokens, mExpertHiddenSize, - mExpertInterSize, mNumExperts, mExpertsPerToken, static_cast<char*>(workspace.workspace), - // Outputs - outputs[getOutputTensorIndex()], static_cast<int*>(workspace.src_to_dest_map), mParallelismConfig, hasLora(), - lora_params, /*use_deepseek_fp8_block_scale=*/false, - /*min_latency_mode=*/false, min_latency_params, stream); -#endif - - if (useSideStream()) - { - // Debug the code with the side stream stalled (only executed when the environment variable - // TLLM_DEBUG_MOE_STALL_SIDE is set and has a positive value) - mSideStreamPtr->stallSideStream("TLLM_DEBUG_MOE_STALL_SIDE", mDebugStallSide); - } - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType MixtureOfExpertsPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == getOutputTensorIndex() || index == getOutputDummyTensorIndex()); - if (useSideStream() && index == getOutputDummyTensorIndex()) - { - return inputTypes[getInputDummyTensorIndex()]; - } - return mOutputType; -} - -// IPluginV2 Methods -char const* MixtureOfExpertsPlugin::getPluginType() const noexcept -{ - return MIXTURE_OF_EXPERTS_PLUGIN_NAME; -} - -char const* MixtureOfExpertsPlugin::getPluginVersion() const noexcept -{ - return MIXTURE_OF_EXPERTS_PLUGIN_VERSION; -} - -int MixtureOfExpertsPlugin::initialize() noexcept -{ - mGemmProfiler->setGemmToProfile(kernels::GemmProfilerBackend::GemmToProfile::GEMM_1); - mGemmProfiler->profileTactics(this, mType, mDims, mGemmId1); - mGemmProfiler->setGemmToProfile(kernels::GemmProfilerBackend::GemmToProfile::GEMM_2); - mGemmProfiler->profileTactics(this, mType, mDims, mGemmId2); - - if (hasLora()) - { - mLoraImpl1->setGemmConfig(); - mLoraImpl2->setGemmConfig(); - - mLoraProfiler->profileTactics(mLoraImpl1->getCublasWrapper(), mType, mDims, mLoraGemmId1); - mLoraProfiler->profileTactics(mLoraImpl2->getCublasWrapper(), mType, mDims, mLoraGemmId2); - } - return 0; -} - -void MixtureOfExpertsPlugin::terminate() noexcept -{ - if (mSideStreamPtr) - { - auto const resource_name = nvinfer1::pluginInternal::SideStream::getResourceKey(mSideStreamId); - getPluginRegistry()->releasePluginResource(resource_name.c_str()); - mSideStreamPtr = nullptr; - } -} - -void MixtureOfExpertsPlugin::destroy() noexcept -{ - if (hasLora()) - { - TLLM_CUDA_CHECK(cudaEventDestroy(mMemcpyEvent)); - } - // This gets called when the network containing plugin is destroyed - delete this; -} - -void MixtureOfExpertsPlugin::setPluginNamespace(char const* libNamespace) noexcept -{ - mNamespace = libNamespace; -} - -char const* MixtureOfExpertsPlugin::getPluginNamespace() const noexcept -{ - return mNamespace.c_str(); -} - -/////////////// - -char const* MixtureOfExpertsPluginCreator::getPluginName() const noexcept -{ - return MIXTURE_OF_EXPERTS_PLUGIN_NAME; -} - -char const* MixtureOfExpertsPluginCreator::getPluginVersion() const noexcept -{ - return MIXTURE_OF_EXPERTS_PLUGIN_VERSION; -} - -nvinfer1::PluginFieldCollection const* MixtureOfExpertsPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -MixtureOfExpertsPluginCreator::MixtureOfExpertsPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(nvinfer1::PluginField("remove_input_padding", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("number_of_experts", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("experts_per_token", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("expert_hidden_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("expert_inter_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("groupwise_quant_algo", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("group_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("activation_type", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("weight_type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("quant_mode", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("use_final_scales", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("use_bias", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("tp_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("tp_rank", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("ep_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("ep_rank", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("side_stream_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("use_lora", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("lora_type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(nvinfer1::PluginField("max_low_rank", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -IPluginV2* MixtureOfExpertsPluginCreator::createPlugin( - char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept -{ - nvinfer1::PluginField const* fields = fc->fields; - int mRemoveInputPadding{}; - int mNumExperts{}; - int mExpertsPerToken{}; - int mExpertHiddenSize{}; - int mExpertInterSize{}; - int mGroupwiseQuantAlgo{}; - int mGroupSize{}; - int mActivationType{}; - int mType{}; - int mWeightType{}; - int mOutputType{INT_MAX}; - int mQuantMode{}; - int mUseFinalScales{1}; // Default to true - int mUseBias{0}; - int mTPSize{}; - int mTPRank{}; - int mEPSize{}; - int mEPRank{}; - int mRequiresDeterminism{0}; - int mSideStreamId{0}; - int mUseLora{}; - int mLoraType{INT_MAX}; - int mMaxLowRank{0}; - - // Read configurations from each fields - struct MapPair - { - char const* key; - int& field; - bool optional = false; - bool set = false; - }; - - std::array input_map{ - MapPair{"remove_input_padding", std::ref(mRemoveInputPadding)}, - MapPair{"number_of_experts", std::ref(mNumExperts)}, - MapPair{"experts_per_token", std::ref(mExpertsPerToken)}, - MapPair{"expert_hidden_size", std::ref(mExpertHiddenSize)}, - MapPair{"expert_inter_size", std::ref(mExpertInterSize)}, - MapPair{"groupwise_quant_algo", std::ref(mGroupwiseQuantAlgo)}, - MapPair{"group_size", std::ref(mGroupSize)}, - MapPair{"activation_type", std::ref(mActivationType)}, - MapPair{"type_id", std::ref(mType)}, - MapPair{"weight_type_id", std::ref(mWeightType)}, - MapPair{"quant_mode", std::ref(mQuantMode)}, - MapPair{"tp_size", std::ref(mTPSize)}, - MapPair{"tp_rank", std::ref(mTPRank)}, - MapPair{"ep_size", std::ref(mEPSize)}, - MapPair{"ep_rank", std::ref(mEPRank)}, - MapPair{"use_lora", std::ref(mUseLora)}, - MapPair{"use_final_scales", std::ref(mUseFinalScales)}, - - // Optional - MapPair{"use_bias", std::ref(mUseBias), true}, - MapPair{"output_type_id", std::ref(mOutputType), true}, - MapPair{"force_determinism", std::ref(mRequiresDeterminism), true}, - MapPair{"side_stream_id", std::ref(mSideStreamId), true}, - MapPair{"lora_type_id", std::ref(mLoraType), true}, - MapPair{"max_low_rank", std::ref(mMaxLowRank), true}, - }; - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - for (auto& item : input_map) - { - if (!strcmp(item.key, attrName)) - { - TLLM_CHECK(fields[i].type == nvinfer1::PluginFieldType::kINT32); - TLLM_CHECK_WITH_INFO(!item.set, "Parameter %s was set twice", item.key); - item.field = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - item.set = true; - } - } - } - - for (auto& item : input_map) - { - TLLM_CHECK_WITH_INFO(item.set || item.optional, "Parameter %s is required but not set", item.key); - } - - // Output type is optional, if not set it to the same as mType - if (mOutputType == INT_MAX) - { - mOutputType = mType; - } - - if (mUseLora) - { - TLLM_CHECK_WITH_INFO(mLoraType != INT_MAX && mMaxLowRank != 0, - "MoE fuse lora, lora_type_id and max_low_rank are required but not set"); - } - - try - { - auto gemmProfiler = moePluginProfiler.createGemmPluginProfiler(/* inference */ false); - auto loraProfiler = loraPluginProfileManager.createGemmPluginProfiler(/* inference */ false, /* skip */ true); - auto* obj = new MixtureOfExpertsPlugin( - // Constructor parameters - mRemoveInputPadding, mNumExperts, mExpertsPerToken, mExpertHiddenSize, mExpertInterSize, - mGroupwiseQuantAlgo, mGroupSize, static_cast<ActivationType>(mActivationType), - static_cast<nvinfer1::DataType>(mType), static_cast<nvinfer1::DataType>(mWeightType), - static_cast<nvinfer1::DataType>(mOutputType), QuantMode(mQuantMode), mUseFinalScales != 0, mUseBias != 0, - mTPSize, mTPRank, mEPSize, mEPRank, mRequiresDeterminism != 0, mSideStreamId, gemmProfiler, mUseLora != 0, - static_cast<nvinfer1::DataType>(mLoraType), loraProfiler, mMaxLowRank); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* MixtureOfExpertsPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call MixtureOfExpertsPlugin::destroy() - try - { - auto gemmProfiler = moePluginProfiler.createGemmPluginProfiler(/* inference */ true); - auto loraProfiler = loraPluginProfileManager.createGemmPluginProfiler(/* inference */ false, /* skip */ true); - - auto* obj = new MixtureOfExpertsPlugin( - // Constructor parameters - serialData, serialLength, gemmProfiler, loraProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -void MixtureOfExpertsPluginCreator::setPluginNamespace(char const* libNamespace) noexcept -{ - mNamespace = libNamespace; -} - -char const* MixtureOfExpertsPluginCreator::getPluginNamespace() const noexcept -{ - return mNamespace.c_str(); -} - -void MixtureOfExpertsGemmProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - checkInit(); - size_t bytes = backend.getWorkspaceSize(maxM); - this->setTmpWorkspaceSizeInBytes(bytes); -} - -void MixtureOfExpertsGemmProfiler::runTactic(int m, int n, int k, MixtureOfExpertsGemmProfiler::Config const& tactic, - char* workspace_ptr_char, cudaStream_t const& stream) -{ - checkInit(); - backend.runProfiler(m, tactic, workspace_ptr_char, /*expert_weights*/ nullptr, stream); -} - -auto MixtureOfExpertsGemmProfiler::getTactics(int m, int n, int k) const -> std::vector<Config> -{ - assert(mRunner); - return mRunner->mMOERunner->getTactics(backend.mGemmToProfile); -} - -void MixtureOfExpertsGemmProfiler::initTmpData( - int m, int n, int k, char* workspace, size_t ws_size, cudaStream_t stream) -{ - checkInit(); - backend.prepare(m, workspace, /*expert_weights*/ nullptr, stream); -} - -void MixtureOfExpertsGemmProfiler::checkInit() -{ - assert(mRunner); - if (init_backend) - { - return; - } - init_backend = true; - auto& plugin = *mRunner; -#ifdef USING_OSS_CUTLASS_MOE_GEMM - backend.init(*plugin.mMOERunner, backend.mGemmToProfile, plugin.mType, plugin.mWeightType, plugin.mOutputType, - plugin.mNumExperts, plugin.mExpertsPerToken, plugin.mExpertHiddenSize, - plugin.mExpertHiddenSize /*TRT backend does not support unpadded hidden size*/, plugin.mExpertInterSize, - plugin.mGroupSize, plugin.mActivationType, plugin.hasBias(), plugin.hasLora(), /*min_latency_mode=*/false, - /*need_weights=*/true, plugin.getParallelismConfig(), /*enable_alltoall=*/false); -#else - backend.init(*plugin.mMOERunner, backend.mGemmToProfile, plugin.mType, plugin.mWeightType, plugin.mOutputType, - plugin.mNumExperts, plugin.mExpertsPerToken, plugin.mExpertHiddenSize, plugin.mExpertInterSize, - plugin.mGroupSize, plugin.mActivationType, plugin.hasBias(), plugin.hasLora(), /*min_latency_mode=*/false, - /*need_weights=*/true, plugin.getParallelismConfig()); -#endif -} diff --git a/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h b/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h deleted file mode 100644 index feb1f10cdc70..000000000000 --- a/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h +++ /dev/null @@ -1,637 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef TRT_MIXTURE_OF_EXPERTS_PLUGIN_H -#define TRT_MIXTURE_OF_EXPERTS_PLUGIN_H - -#include "NvInferPlugin.h" -#include "tensorrt_llm/kernels/cutlass_kernels/include/cutlass_kernel_selector.h" -#if defined(USING_OSS_CUTLASS_MOE_GEMM) -#include "tensorrt_llm/kernels/cutlass_kernels/include/moe_kernels.h" -#else -#include "moe_kernels.h" -#endif -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/kernels/lora/lora.h" -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include "tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.h" -#include "tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h" -#include "tensorrt_llm/runtime/cudaStream.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ -namespace kernels = CUTLASS_MOE_GEMM_KERNELS_NAMESPACE; -using MoeMinLatencyParams = CUTLASS_MOE_GEMM_KERNELS_NAMESPACE::MoeMinLatencyParams; -using MOEParallelismConfig = CUTLASS_MOE_GEMM_KERNELS_NAMESPACE::MOEParallelismConfig; -using QuantParams = CUTLASS_MOE_GEMM_KERNELS_NAMESPACE::QuantParams; -using MoeGemmId = CUTLASS_MOE_GEMM_NAMESPACE::MoeGemmId; -using ActivationType = CUTLASS_MOE_GEMM_NAMESPACE::ActivationType; -using ActivationParams = CUTLASS_MOE_GEMM_KERNELS_NAMESPACE::ActivationParams; -using TmaWarpSpecializedGroupedGemmInput = CUTLASS_MOE_GEMM_NAMESPACE::TmaWarpSpecializedGroupedGemmInput; -using CUTLASS_MOE_GEMM_NAMESPACE::isGatedActivation; - -class MixtureOfExpertsGemmProfiler; -using MixtureOfExpertsPluginProfilerPtr = std::shared_ptr<MixtureOfExpertsGemmProfiler>; -using GroupwiseQuantAlgo = tensorrt_llm::common::GroupwiseQuantAlgo; - -struct GemmIDMoe -{ - int gemm_idx; - int num_experts{}; - int experts_per_token{}; - kernels::MOEParallelismConfig parallelism_config{}; - int64_t hidden{}; - int64_t inter{}; - int64_t group_size{}; - ActivationType actfn{}; - nvinfer1::DataType dtype{}; - nvinfer1::DataType wdtype{}; - tensorrt_llm::common::QuantMode quant_mode; - bool determinism_mode = false; - - bool operator==(GemmIDMoe const& id) const - { - return id.gemm_idx == gemm_idx && id.num_experts == num_experts && id.experts_per_token == experts_per_token - && id.parallelism_config == parallelism_config && id.hidden == hidden && id.inter == inter - && id.group_size == group_size && id.actfn == actfn && id.dtype == dtype && id.wdtype == wdtype - && id.quant_mode == quant_mode && id.determinism_mode == determinism_mode; - } - - friend std::ostream& operator<<(std::ostream& out, GemmIDMoe const& id) - { - out << "gemm idx, experts, experts_per_token, parallelism_config, hidden, inter, group_size, actfn, dtype, " - "weight " - "type, parallelism mode, determinism mode=" - - << id.gemm_idx << "," << id.num_experts << "," << id.experts_per_token << "," << id.parallelism_config - << "," << id.hidden << "," << id.inter << "," << id.group_size << "," << static_cast<int>(id.actfn) << "," - << static_cast<int>(id.dtype) << "," << static_cast<int>(id.wdtype) << "," << id.quant_mode.value() << "," - << id.determinism_mode; - return out; - } -}; - -// Hash of GemmIDMoe -struct GemmIDMoeHash -{ - std::size_t operator()(GemmIDMoe const& id) const - { - size_t hash = std::hash<int>{}(id.gemm_idx); - hash ^= std::hash<int>{}(id.num_experts); - hash ^= std::hash<int>{}(id.experts_per_token); - hash ^= std::hash<int>{}(id.parallelism_config.tp_size); - hash ^= std::hash<int>{}(id.parallelism_config.ep_size); - hash ^= std::hash<int>{}(id.parallelism_config.tp_rank); - hash ^= std::hash<int>{}(id.parallelism_config.ep_rank); - hash ^= std::hash<int>{}(id.hidden); - hash ^= std::hash<int>{}(id.inter); - hash ^= std::hash<int>{}(id.group_size); - hash ^= std::hash<int>{}(static_cast<int>(id.actfn)); - hash ^= std::hash<int>{}(static_cast<int>(id.dtype)); - hash ^= std::hash<int>{}(static_cast<int>(id.wdtype)); - hash ^= std::hash<int>{}(static_cast<int>(id.quant_mode.value())); - return hash; - } -}; - -class MixtureOfExpertsPlugin : public nvinfer1::IPluginV2DynamicExt -{ -public: - using LoraPluginProfilerPtr = std::shared_ptr<CublasLtGemmPluginProfiler>; - using LoraImplPtr = std::shared_ptr<tensorrt_llm::kernels::LoraImpl>; - MixtureOfExpertsPlugin() = delete; - MixtureOfExpertsPlugin(bool remove_input_padding, int number_of_experts, int experts_per_token, - int expert_hidden_size, int expert_inter_size, int groupwise_quant_algo, int group_size, - ActivationType activation_type, nvinfer1::DataType type, nvinfer1::DataType weight_type, - nvinfer1::DataType output_type, tensorrt_llm::common::QuantMode quant_mode, bool use_final_scales, - bool use_bias, int tp_size, int tp_rank, int ep_size, int ep_rank, bool force_determinism, int side_stream_id, - MixtureOfExpertsPluginProfilerPtr gemm_profiler_ptr, bool use_lora, nvinfer1::DataType lora_type, - LoraPluginProfilerPtr lora_profiler, int max_low_rank); - MixtureOfExpertsPlugin(void const* data, size_t length, MixtureOfExpertsPluginProfilerPtr gemm_profiler_ptr, - LoraPluginProfilerPtr lora_profiler); - MixtureOfExpertsPlugin(MixtureOfExpertsPlugin const&); - - void init(); - - ~MixtureOfExpertsPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - - int getNbOutputs() const noexcept override - { - return 1 + useSideStream(); - } - - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - void setPluginNamespace(char const* pluginNamespace) noexcept override; - char const* getPluginNamespace() const noexcept override; - -private: - friend class MixtureOfExpertsGemmProfiler; - std::unique_ptr<kernels::CutlassMoeFCRunnerInterface> mMOERunner{}; - int mNumExperts{}; - int mExpertsPerToken{}; - int64_t mExpertHiddenSize{}; - int64_t mExpertInterSize{}; - int64_t mGroupwiseQuantAlgo{}; - int64_t mGroupSize{}; - ActivationType mActivationType; - nvinfer1::DataType mType{}; - nvinfer1::DataType mWeightType{}; - nvinfer1::DataType mOutputType{}; - tensorrt_llm::common::QuantMode mQuantMode; - bool mUseFinalScales{}; - bool mUseBias{}; - MOEParallelismConfig mParallelismConfig{}; - - GemmDims mDims{}; - bool mUseDeterministicKernels = false; - int mSideStreamId = 0; - - int mDebugStallMain = 0; - int mDebugStallSide = 0; - - GemmIDMoe mGemmId1{}; - GemmIDMoe mGemmId2{}; - - MixtureOfExpertsPluginProfilerPtr mGemmProfiler; - - // lora related - bool mUseLora{}; - nvinfer1::DataType mLoraType{}; - int mMaxLowRank{}; - bool mRemoveInputPadding{}; - - LoraImplPtr mLoraImpl1; - LoraImplPtr mLoraImpl2; - - GemmIdCublas mLoraGemmId1{}; - GemmIdCublas mLoraGemmId2{}; - LoraPluginProfilerPtr mLoraProfiler; - - std::vector<void const*> mLoraExpandFC1WeightPtrs{}; - std::vector<void const*> mLoraExpandFC2WeightPtrs{}; - std::vector<void const*> mLoraExpandGatedWeightPtrs{}; - std::vector<int32_t> mLoraExpandFC1Ranks{}; - std::vector<int32_t> mLoraExpandFC2Ranks{}; - std::vector<int32_t> mLoraExpandGatedRanks{}; - - cudaEvent_t mMemcpyEvent; - nvinfer1::pluginInternal::SideStream* mSideStreamPtr; - - // The below are not serialised - std::string const mLayerName{}; - std::string mNamespace{}; - - struct WorkspaceInfo - { - void* workspace{}; - void* src_to_dest_map{}; - void* lora_workspace{}; - size_t size{}; - }; - - int64_t getNumTokens(nvinfer1::PluginTensorDesc const* input_tensor) const; - WorkspaceInfo setupWorkspace(void* base_ptr, int64_t num_tokens, int num_reqs = 0) const; - - MOEParallelismConfig getParallelismConfig() const; - QuantParams getQuantParams(nvinfer1::PluginTensorDesc const* inputDesc, void const* const* inputs, - int scale_1_idx = -1, int scale_2_idx = -1, int scale_3_idx = -1, int scale_4_idx = -1, int scale_5_idx = -1, - int scale_6_idx = -1, int scale_7_idx = -1, int scale_8_idx = -1) const; - - int getNumLoraRequests(nvinfer1::PluginTensorDesc const* input_tensor) const; - tensorrt_llm::kernels::LoraParams getLoraParams( - nvinfer1::PluginTensorDesc const* inputDesc, void const* const* inputs, void* workspace); - - enum class RequestType : int32_t - { - kCONTEXT = 0, - kGENERATION = 1 - }; - - using IndexType = std::int32_t; - - // Inputs - constexpr static IndexType getInputTensorIndex() - { - return 0; - } - - constexpr static IndexType getExpertWeights1Index() - { - return getInputTensorIndex() + 1; - } - - constexpr static IndexType getExpertWeights2Index() - { - return getExpertWeights1Index() + 1; - } - - constexpr static IndexType getTokenSelectedExpertsIndex() - { - return getExpertWeights2Index() + 1; - } - - // Conditional inputs, we only allocate a new index if actually used - bool hasBias() const - { - return mUseBias; - } - - bool hasFinalScales() const - { - return mUseFinalScales; - } - - bool hasExpertIntQuantScales() const - { - return mQuantMode.hasInt4Weights() || mQuantMode.hasInt8Weights(); - } - - bool hasExpertFp8QuantScales() const - { - return mQuantMode.hasFp8Qdq(); - } - - bool hasExpertFp8FinalQuantScales() const - { - return hasExpertFp8QuantScales() && mOutputType == nvinfer1::DataType::kFP8; - } - - bool hasFP4QuantScales() const - { - return mQuantMode.hasNvfp4(); - } - - bool hasGroupwiseIntQuantScales() const - { - return mGroupwiseQuantAlgo > 0; - } - - bool hasExpertWeightQuantZeros() const - { - return mGroupwiseQuantAlgo & GroupwiseQuantAlgo::ZERO; - } - - bool hasExpertPrequantScales() const - { - return mGroupwiseQuantAlgo & GroupwiseQuantAlgo::PRE_QUANT_SCALE; - } - - bool hasGroupwiseFp8Alpha() const - { - return mGroupwiseQuantAlgo & GroupwiseQuantAlgo::FP8_ALPHA; - } - - bool useSideStream() const - { - return mSideStreamId > 0; - } - - bool hasLora() const - { - return mUseLora; - } - - bool hasGatedLoraWeightsAndRanks() const - { - return mUseLora && isGatedActivation(mActivationType); - } - - IndexType getTokenFinalScalesIndex() const - { - return getTokenSelectedExpertsIndex() + hasFinalScales(); - } - - IndexType getExpertBias1Index() const - { - return getTokenFinalScalesIndex() + hasBias(); - } - - IndexType getExpertBias2Index() const - { - return getExpertBias1Index() + hasBias(); - } - - /* - * Weight-Only int quant scales - */ - IndexType getExpertIntQuantScale1Index() const - { - return getExpertBias2Index() + hasExpertIntQuantScales(); - } - - IndexType getExpertIntQuantScale2Index() const - { - return getExpertIntQuantScale1Index() + hasExpertIntQuantScales(); - } - - /* - * FP8 Quant Scales - */ - IndexType getExpertFP8Dequant1Index() const - { - return getExpertIntQuantScale2Index() + hasExpertFp8QuantScales(); - } - - IndexType getExpertFP8Quant2Index() const - { - return getExpertFP8Dequant1Index() + hasExpertFp8QuantScales(); - } - - IndexType getExpertFP8Dequant2Index() const - { - return getExpertFP8Quant2Index() + hasExpertFp8QuantScales(); - } - - IndexType getExpertFP8QuantFinalIndex() const - { - return getExpertFP8Dequant2Index() + hasExpertFp8FinalQuantScales(); - } - - IndexType getInputFP8DequantIndex() const - { - return getExpertFP8QuantFinalIndex() + (hasExpertFp8QuantScales() && hasLora()); - } - - /* - * FP4 Quant Scales - */ - IndexType getFP4GlobalActSF1Index() const - { - return getInputFP8DequantIndex() + hasFP4QuantScales(); - } - - IndexType getFP4WeightSF1Index() const - { - return getFP4GlobalActSF1Index() + hasFP4QuantScales(); - } - - IndexType getFP4GlobalSF1Index() const - { - return getFP4WeightSF1Index() + hasFP4QuantScales(); - } - - IndexType getFP4GlobalActSF2Index() const - { - return getFP4GlobalSF1Index() + hasFP4QuantScales(); - } - - IndexType getFP4WeightSF2Index() const - { - return getFP4GlobalActSF2Index() + hasFP4QuantScales(); - } - - IndexType getFP4GlobalSF2Index() const - { - return getFP4WeightSF2Index() + hasFP4QuantScales(); - } - - /* - * Groupwise Params - */ - IndexType getExpertPrequantScales1Index() const - { - return getFP4GlobalSF2Index() + hasExpertPrequantScales(); - } - - IndexType getExpertPrequantScales2Index() const - { - return getExpertPrequantScales1Index() + hasExpertPrequantScales(); - } - - IndexType getExpertIntQuantZeros1Index() const - { - return getExpertPrequantScales2Index() + hasExpertWeightQuantZeros(); - } - - IndexType getExpertIntQuantZeros2Index() const - { - return getExpertIntQuantZeros1Index() + hasExpertWeightQuantZeros(); - } - - IndexType getExpertFp8Alpha1Index() const - { - return getExpertIntQuantZeros2Index() + hasGroupwiseFp8Alpha(); - } - - IndexType getExpertFp8Alpha2Index() const - { - return getExpertFp8Alpha1Index() + hasGroupwiseFp8Alpha(); - } - - /* - * LoRA params - */ - IndexType getLoraFC1WeightPtrsIndex() const - { - return getExpertFp8Alpha2Index() + hasLora(); - } - - IndexType getLoraFC1RanksIndex() const - { - return getLoraFC1WeightPtrsIndex() + hasLora(); - } - - IndexType getLoraFC2WeightPtrsIndex() const - { - return getLoraFC1RanksIndex() + hasLora(); - } - - IndexType getLoraFC2RanksIndex() const - { - return getLoraFC2WeightPtrsIndex() + hasLora(); - } - - IndexType getLoraGatedWeightPtrsIndex() const - { - return getLoraFC2RanksIndex() + hasGatedLoraWeightsAndRanks(); - } - - IndexType getLoraGatedRanksIndex() const - { - return getLoraGatedWeightPtrsIndex() + hasGatedLoraWeightsAndRanks(); - } - - IndexType getHostRequestTypeIndex() const - { - return getLoraGatedRanksIndex() + hasLora(); - } - - IndexType getHostContextLengthIndex() const - { - return getHostRequestTypeIndex() + (mRemoveInputPadding && hasLora()); - } - - IndexType getInputDummyTensorIndex() const - { - return getHostContextLengthIndex() + useSideStream(); - } - - IndexType getNbInputs() const - { - return getInputDummyTensorIndex() + 1; - } - - // Outputs - constexpr static IndexType getOutputTensorIndex() - { - return 0; - } - - IndexType getOutputDummyTensorIndex() const - { - return getOutputTensorIndex() + useSideStream(); - } - - /** - * Get the index of the expert shape tuple that represents the inner dimension - */ - int getGemmShapeInnerDimIndex() const - { - // In weight only mode the shape is transposed - return hasExpertIntQuantScales() ? 1 : 2; - } - - /** - * Get the index of the expert shape tuple that represents the outer dimension - */ - int getGemmShapeOuterDimIndex() const - { - // In weight only mode the shape is transposed - return hasExpertIntQuantScales() ? 2 : 1; - } - - /** - * Get quantization dimension scaling factor - */ - std::pair<int, int> getWeightPackedElements() const - { - if (mGroupwiseQuantAlgo == 0) - { - return {1, mQuantMode.hasInt4Weights() ? 2 : 1}; - } - else - { - return {1, 4}; - } - } -}; - -class MixtureOfExpertsGemmProfiler - : public tensorrt_llm::plugins::GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - MixtureOfExpertsPlugin*, GemmIDMoe, GemmIDMoeHash> -{ -public: - MixtureOfExpertsGemmProfiler() - { - // NOTE: Do not access mPlugin here, since we are called from the constructor before all fields are init - } - - void setGemmToProfile(kernels::GemmProfilerBackend::GemmToProfile gemm_to_profile) - { - // Just set the backend directly. This will just be reused in checkInit(). - backend.mGemmToProfile = gemm_to_profile; - // We need to set the backend to reinitialise itself with the new GEMM - init_backend = false; - } - - void setMaxProfileM(int maxProfileM) - { - mMaxProfileM = maxProfileM; - } - - virtual int getMaxProfileM() const override - { - return mMaxProfileM; - } - -protected: - using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; - void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - std::vector<Config> getTactics(int m, int n, int k) const override; - void initTmpData(int maxM, int n, int k, char* workspace, size_t size, cudaStream_t stream) override; - - void checkInit(); - - bool init_backend = false; - kernels::GemmProfilerBackend backend{}; - -private: - int mMaxProfileM = 0; -}; - -class MixtureOfExpertsPluginCreator : public nvinfer1::IPluginCreator -{ -public: - MixtureOfExpertsPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - - void setPluginNamespace(char const* pluginNamespace) noexcept override; - - char const* getPluginNamespace() const noexcept override; - -private: - GemmPluginProfilerManager<MixtureOfExpertsGemmProfiler> moePluginProfiler; - GemmPluginProfilerManager<CublasLtGemmPluginProfiler> loraPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; - std::string mNamespace; -}; - -} // namespace tensorrt_llm::plugins - -#endif // TRT_MIXTURE_OF_EXPERTS_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/ncclPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.cpp b/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.cpp deleted file mode 100644 index 4825dd51bbab..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.cpp +++ /dev/null @@ -1,253 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "allgatherPlugin.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" - -#include <nccl.h> - -using namespace nvinfer1; -using tensorrt_llm::plugins::AllgatherPluginCreator; -using tensorrt_llm::plugins::AllgatherPlugin; - -static char const* ALLGATHER_PLUGIN_VERSION{"1"}; -static char const* ALLGATHER_PLUGIN_NAME{"AllGather"}; -PluginFieldCollection AllgatherPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> AllgatherPluginCreator::mPluginAttributes; - -AllgatherPlugin::AllgatherPlugin(std::set<int> group, nvinfer1::DataType type) - : mGroup(std::move(group)) - , mType(type) -{ -} - -// Parameterized constructor -AllgatherPlugin::AllgatherPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mType); - mGroup.clear(); - int groupItem = 0; - while (d != a + length) - { - read(d, groupItem); - mGroup.insert(groupItem); - } - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* AllgatherPlugin::clone() const noexcept -{ - auto* plugin = new AllgatherPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs AllgatherPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - auto ret = inputs[0]; - auto groupSize = exprBuilder.constant(mGroup.size()); - ret.d[0] = exprBuilder.operation(DimensionOperation::kPROD, *ret.d[0], *groupSize); - return ret; -} - -bool AllgatherPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); -} - -void AllgatherPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t AllgatherPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -int AllgatherPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - size_t size = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) - { - size *= inputDesc[0].dims.d[i]; - } - - TLLM_CHECK_WITH_INFO(mNcclComm.get() != nullptr, "mNcclComm should be initialized before used"); - NCCLCHECK(ncclAllGather(inputs[0], outputs[0], size, (*getDtypeMap())[inputDesc[0].type], *mNcclComm, stream)); - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType AllgatherPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - assert(index == 0); - return inputTypes[0]; -} - -// IPluginV2 Methods - -char const* AllgatherPlugin::getPluginType() const noexcept -{ - return ALLGATHER_PLUGIN_NAME; -} - -char const* AllgatherPlugin::getPluginVersion() const noexcept -{ - return ALLGATHER_PLUGIN_VERSION; -} - -int AllgatherPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int AllgatherPlugin::initialize() noexcept -{ - if (isBuilding()) - { - return 0; - } - TLLM_LOG_TRACE("%s start for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); - mNcclComm = getComm(mGroup); - TLLM_LOG_TRACE("%s stop for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); - return 0; -} - -void AllgatherPlugin::terminate() noexcept {} - -size_t AllgatherPlugin::getSerializationSize() const noexcept -{ - return sizeof(int) * mGroup.size() + sizeof(mType); -} - -void AllgatherPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - for (auto it = mGroup.begin(); it != mGroup.end(); ++it) - { - write(d, *it); - } - TLLM_CHECK(d == a + getSerializationSize()); -} - -void AllgatherPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -AllgatherPluginCreator::AllgatherPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("group", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* AllgatherPluginCreator::getPluginName() const noexcept -{ - return ALLGATHER_PLUGIN_NAME; -} - -char const* AllgatherPluginCreator::getPluginVersion() const noexcept -{ - return ALLGATHER_PLUGIN_VERSION; -} - -PluginFieldCollection const* AllgatherPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* AllgatherPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - std::set<int> group; - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "group")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - auto const* r = static_cast<int const*>(fields[i].data); - for (int j = 0; j < fields[i].length; ++j) - { - group.insert(*r); - ++r; - } - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - - try - { - auto* obj = new AllgatherPlugin(group, type); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* AllgatherPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call AllgatherPlugin::destroy() - try - { - auto* obj = new AllgatherPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.h b/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.h deleted file mode 100644 index 3d7810e6bd49..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class AllgatherPlugin : public BasePlugin -{ -public: - AllgatherPlugin(std::set<int> group, nvinfer1::DataType type); - - AllgatherPlugin(void const* data, size_t length); - - ~AllgatherPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - const std::string mLayerName; - std::set<int> mGroup; - nvinfer1::DataType mType; - std::shared_ptr<ncclComm_t> mNcclComm; -}; - -class AllgatherPluginCreator : public BaseCreator -{ -public: - AllgatherPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.cpp b/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.cpp deleted file mode 100644 index 24d9aff418f7..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.cpp +++ /dev/null @@ -1,986 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "allreducePlugin.h" - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/customAllReduceUtils.h" -#include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/common/nvmlWrapper.h" -#include "tensorrt_llm/kernels/customAllReduceKernels.h" -#include "tensorrt_llm/kernels/userbuffers/ub_interface.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" - -#include <nccl.h> - -#include <unordered_set> - -using namespace nvinfer1; -using tensorrt_llm::plugins::AllreducePluginCreator; -using tensorrt_llm::plugins::AllreducePlugin; -using tensorrt_llm::kernels::AllReduceFusionOp; -using tensorrt_llm::kernels::AllReduceStrategyType; -using tensorrt_llm::kernels::AllReduceStrategyConfig; -using tensorrt_llm::mpi::MpiTag; - -static char const* ALLREDUCE_PLUGIN_VERSION{"1"}; -static char const* ALLREDUCE_PLUGIN_NAME{"AllReduce"}; -PluginFieldCollection AllreducePluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> AllreducePluginCreator::mPluginAttributes; - -AllreducePlugin::AllreducePlugin(std::set<int> group, nvinfer1::DataType type, AllReduceStrategyType strategy, - AllReduceStrategyConfig config, AllReduceFusionOp op, int32_t counter, float eps, int8_t affine, int8_t bias, - int8_t scale) - : mGroup(std::move(group)) - , mType(type) - , mStrategy(strategy) - , mConfig(config) - , mOp(op) - , mEps(eps) - , mAffine(affine) - , mBias(bias) - , mScale(scale) -{ - check(); -} - -// Parameterized constructor -AllreducePlugin::AllreducePlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mType); - read(d, mStrategy); - read(d, mConfig); - read(d, mOp); - read(d, mEps); - read(d, mAffine); - read(d, mBias); - read(d, mScale); - mGroup.clear(); - int groupItem = 0; - while (d != a + length) - { - read(d, groupItem); - mGroup.insert(groupItem); - } - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); - check(); -} - -void AllreducePlugin::check() noexcept -{ - if (mStrategy != AllReduceStrategyType::UB) - { - TLLM_CHECK(mOp != AllReduceFusionOp::LAST_PROCESS_FOR_UB); - } -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* AllreducePlugin::clone() const noexcept -{ - auto* plugin = new AllreducePlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs AllreducePlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_NVFP4 && mStrategy == AllReduceStrategyType::UB && mScale) - { - if (outputIndex == 0) - { - DimsExprs ret; - ret.nbDims = inputs[0].nbDims; - for (int di = 0; di < ret.nbDims; ++di) - { - ret.d[di] = inputs[0].d[di]; - } - return ret; - } - else if (outputIndex == 2) - { - DimsExprs ret; - ret.nbDims = inputs[0].nbDims; - for (int di = 0; di < ret.nbDims; ++di) - { - ret.d[di] = inputs[0].d[di]; - } - auto dimM = exprBuilder.operation( - DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 2], *exprBuilder.constant(128)); - ret.d[ret.nbDims - 2] = exprBuilder.operation(DimensionOperation::kPROD, *dimM, *exprBuilder.constant(128)); - ret.d[ret.nbDims - 1] = exprBuilder.operation( - DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 1], *exprBuilder.constant(16)); - return ret; - } - } - return inputs[0]; -} - -bool AllreducePlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - int base_inputs = 0; - switch (mStrategy) - { - case AllReduceStrategyType::NCCL: - case AllReduceStrategyType::UB: - case AllReduceStrategyType::NCCL_SYMMETRIC: base_inputs = 1; break; - default: base_inputs = 2; break; - } - int fusion_op_extra_inputs = 0; - int scale_idx = 0; - if (mOp != AllReduceFusionOp::NONE) - { - ++fusion_op_extra_inputs; - if (mAffine) - { - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_PREPOST_NORM) - ++fusion_op_extra_inputs; - ++fusion_op_extra_inputs; - } - if (mBias) - { - ++fusion_op_extra_inputs; - } - if (mScale) - { - scale_idx = base_inputs + fusion_op_extra_inputs; - ++fusion_op_extra_inputs; - } - } - - TLLM_CHECK(nbInputs == (base_inputs + fusion_op_extra_inputs)); - - if (pos == 1) - { - switch (mStrategy) - { - case AllReduceStrategyType::NCCL: - case AllReduceStrategyType::UB: - case AllReduceStrategyType::NCCL_SYMMETRIC: break; - default: return (inOut[pos].type == nvinfer1::DataType::kINT64) && (inOut[pos].format == TensorFormat::kLINEAR); - } - } - if (mStrategy == AllReduceStrategyType::UB) - { - if (mScale && pos == scale_idx) - { - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_NVFP4) - { - if (pos == nbInputs) - { - return (inOut[pos].type == nvinfer1::DataType::kFP4) && (inOut[pos].format == TensorFormat::kLINEAR); - } - if (pos == (nbInputs + 2)) - { - return (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); - } - } - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_FP8) - { - if (pos == nbInputs) - { - return (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); - } - } - } - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); -} - -void AllreducePlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t AllreducePlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -AllReduceStrategyType AllreducePlugin::selectImplementation( - size_t messageSize, int worldSize, nvinfer1::DataType type) noexcept -{ - bool const isAuto = (mStrategy == AllReduceStrategyType::AUTO); - - bool const forceDeterministic = common::getEnvForceDeterministicAllReduce(); - if (!mIsP2PSupported) - { - if (!isAuto) - { - TLLM_LOG_INFO("Since Peer to Peer not supported, fallback to AllReduceStrategy: NCCL_SYMMETRIC"); - } - else if (forceDeterministic) - { - TLLM_LOG_WARNING( - "Since Peer to Peer not supported, fallback to AllReduceStrategy: NCCL_SYMMETRIC. NCCL_SYMMETRIC might " - "produce " - "non-deterministic results."); - } - return AllReduceStrategyType::NCCL_SYMMETRIC; - } - - if (isAuto && !mIsNVLINKSupported && !forceDeterministic) - { - return AllReduceStrategyType::NCCL_SYMMETRIC; - } - - auto const maxWorkspaceSize = utils::customAllReduceUtils::getMaxRequiredWorkspaceSize(worldSize); - - AllReduceStrategyType strat = AllReduceStrategyType::NCCL_SYMMETRIC; - auto const messageSizeBytes = messageSize * common::getDTypeSize(type); - - if (messageSizeBytes <= maxWorkspaceSize) - { - // In some instances, the two-shot strategy has exhibited significant performance issues. - // As a temporary measure, we have disabled the two-shot strategy. - // TODO: remove this WAR after https://nvbugspro.nvidia.com/bug/4718747 is fixed. - if (!isAuto) - { - strat = mStrategy; - } - else if (forceDeterministic) - { - strat = AllReduceStrategyType::ONESHOT; - } - else if (worldSize <= 2) - { - strat = AllReduceStrategyType::ONESHOT; - } - else if (worldSize <= 4) - { - if (messageSizeBytes < 1 * 1000 * 1000) - { - strat = AllReduceStrategyType::ONESHOT; - } - else - { - strat = AllReduceStrategyType::NCCL_SYMMETRIC; - } - } - else - { - if (messageSizeBytes < 500 * 1000) - { - strat = AllReduceStrategyType::ONESHOT; - } - else - { - strat = AllReduceStrategyType::NCCL_SYMMETRIC; - } - } - - if (!kernels::configurationSupported(strat, messageSize, worldSize, type)) - { - if (!isAuto) - { - TLLM_LOG_WARNING("Since not aligned, fallback to AllReduceStrategy: NCCL_SYMMETRIC"); - } - else if (forceDeterministic) - { - TLLM_LOG_WARNING( - "Since not aligned, fallback to AllReduceStrategy: NCCL_SYMMETRIC. NCCL_SYMMETRIC might produce " - "non-deterministic results."); - } - strat = AllReduceStrategyType::NCCL_SYMMETRIC; - } - } - else - { - if (!isAuto) - { - TLLM_LOG_WARNING("Since messageSize > maxWorkspace, fallback to AllReduceStrategy: NCCL_SYMMETRIC"); - } - else if (forceDeterministic) - { - TLLM_LOG_WARNING( - "Since messageSize > maxWorkspace, fallback to AllReduceStrategy: NCCL_SYMMETRIC. NCCL_SYMMETRIC might " - "produce " - "non-deterministic results."); - } - strat = AllReduceStrategyType::NCCL_SYMMETRIC; - } - - return strat; -} - -int AllreducePlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - size_t size = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) - { - size *= inputDesc[0].dims.d[i]; - } - - kernels::AllReduceStrategyType runtimeStrategy; - - static char* forceNcclAllReduceStrategyChar = std::getenv("FORCE_NCCL_ALL_REDUCE_STRATEGY"); - bool forceNcclAllReduceStrategy = (forceNcclAllReduceStrategyChar != nullptr); - if (forceNcclAllReduceStrategy || mStrategy == AllReduceStrategyType::NCCL) - { - runtimeStrategy = AllReduceStrategyType::NCCL; - } - else if (mStrategy == AllReduceStrategyType::NCCL_SYMMETRIC) - { - runtimeStrategy = AllReduceStrategyType::NCCL_SYMMETRIC; - } - else if (mStrategy == AllReduceStrategyType::UB) - { - runtimeStrategy = AllReduceStrategyType::UB; - } - else - { - runtimeStrategy = selectImplementation(size, mGroup.size(), mType); - } - - // Log runtime strategy - auto const rank = COMM_SESSION.getRank(); - switch (runtimeStrategy) - { - case AllReduceStrategyType::NCCL: - { - TLLM_LOG_DEBUG("AllReducePlugin strategy for rank %d: NCCL", rank); - break; - } - case AllReduceStrategyType::NCCL_SYMMETRIC: - { - TLLM_LOG_DEBUG("AllReducePlugin strategy for rank %d: NCCL_SYMMETRIC", rank); - break; - } - case AllReduceStrategyType::ONESHOT: - { - TLLM_LOG_DEBUG("AllReducePlugin strategy for rank %d: ONESHOT", rank); - break; - } - case AllReduceStrategyType::TWOSHOT: - { - TLLM_LOG_DEBUG("AllReducePlugin strategy for rank %d: TWOSHOT", rank); - break; - } - case AllReduceStrategyType::UB: - { - TLLM_LOG_DEBUG("AllReducePlugin strategy for rank %d: UB", rank); - break; - } - default: break; - } - - if (runtimeStrategy == AllReduceStrategyType::NCCL || runtimeStrategy == AllReduceStrategyType::NCCL_SYMMETRIC) - { - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM || mOp == AllReduceFusionOp::RESIDUAL_RMS_PREPOST_NORM) - { - NCCLCHECK(ncclAllReduce(inputs[0], outputs[1], size, (*getDtypeMap())[mType], ncclSum, *mNcclComm, stream)); - tensorrt_llm::kernels::AllReduceParams params; - int fusion_ptr_idx = 0; - if (mStrategy == AllReduceStrategyType::NCCL || mStrategy == AllReduceStrategyType::NCCL_SYMMETRIC) - { - fusion_ptr_idx = 1; - } - else - { - fusion_ptr_idx = 2; - } - params.fusion_params.bias_buffer = mBias ? inputs[fusion_ptr_idx++] : nullptr; - params.fusion_params.residual_buffer = inputs[fusion_ptr_idx++]; - params.fusion_params.weight_buffer = mAffine ? inputs[fusion_ptr_idx++] : nullptr; - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_PREPOST_NORM) - { - params.fusion_params.weight_buffer_pre_residual_norm = mAffine ? inputs[fusion_ptr_idx++] : nullptr; - } - params.local_output_buffer_ptr = outputs[0]; - params.elts_total = size; - params.fusion_params.hidden_size = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; - params.fusion_params.eps = mEps; - params.fusion_params.intermediate_buffer = outputs[1]; - TLLM_LOG_DEBUG("residualRmsNorm called"); - tensorrt_llm::kernels::residualRmsNorm(params, mType, stream, mOp); - } - else - { - NCCLCHECK(ncclAllReduce(inputs[0], outputs[0], size, (*getDtypeMap())[mType], ncclSum, *mNcclComm, stream)); - } - } - else if (runtimeStrategy == AllReduceStrategyType::UB) - { - TLLM_CHECK(!mBias); - - size_t dtype_size = tensorrt_llm::common::getDTypeSize(mType); - int hidden_size = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; - - TLLM_CHECK_WITH_INFO(tensorrt_llm::runtime::ub::ub_is_initialized(), "UserBuffer has not been initialized!"); - auto ub_buffer0 = tensorrt_llm::runtime::ub::ub_get(0); - auto ub_buffer1 = tensorrt_llm::runtime::ub::ub_get(1); - TLLM_CHECK(inputs[0] == ub_buffer0.addr); - auto ub_comm = tensorrt_llm::runtime::ub::ub_comm(); - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_FP8) - { - TLLM_CHECK(mAffine); - TLLM_CHECK(mScale); - TLLM_CHECK(outputs[0] == ub_buffer1.addr); - void* residual = const_cast<void*>(inputs[1]); - void* gamma = const_cast<void*>(inputs[2]); - float* scale = const_cast<float*>(reinterpret_cast<float const*>(inputs[3])); - tensorrt_llm::kernels::ub::allreduce2_userbuff_inplace_rmsnorm_quant_launcher(ub_buffer0.handle, 0, - ub_buffer1.handle, 0, size, hidden_size, nullptr, gamma, mEps, scale, residual, outputs[1], mType, - ub_comm, stream); - } - else if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_NVFP4) - { - auto ub_buffer2 = tensorrt_llm::runtime::ub::ub_get(2); - TLLM_CHECK(mAffine); - TLLM_CHECK(mScale); - TLLM_CHECK(outputs[0] == ub_buffer1.addr); - TLLM_CHECK(outputs[2] == ub_buffer2.addr); - void* residual = const_cast<void*>(inputs[1]); - void* gamma = const_cast<void*>(inputs[2]); - float* scale = const_cast<float*>(reinterpret_cast<float const*>(inputs[3])); - tensorrt_llm::kernels::ub::allreduce2_userbuff_inplace_rmsnorm_quant_fp4_launcher(ub_buffer0.handle, 0, - ub_buffer1.handle, 0, ub_buffer2.handle, 0, size, hidden_size, nullptr, gamma, mEps, scale, residual, - outputs[1], mType, ub_comm, stream); - } - else if (mOp == AllReduceFusionOp::LAST_PROCESS_FOR_UB) - { - TLLM_CHECK(outputs[1] == ub_buffer1.addr); - void* residual = const_cast<void*>(inputs[1]); - tensorrt_llm::kernels::ub::allreduce2_userbuff_inplace_launcher( - ub_buffer0.handle, 0, size, mType, ub_comm, stream); - tensorrt_llm::kernels::ub::allgather2_userbuff_residual_launcher( - ub_buffer1.handle, 0, size, hidden_size, residual, mType, ub_comm, stream); - TLLM_CUDA_CHECK( - cudaMemcpyAsync(outputs[0], ub_buffer0.addr, size * dtype_size, cudaMemcpyDeviceToDevice, stream)); - } - else if (mOp == AllReduceFusionOp::NONE) - { - tensorrt_llm::kernels::ub::allreduce2_userbuff_inplace_launcher( - ub_buffer0.handle, 0, size, mType, ub_comm, stream); - TLLM_CUDA_CHECK( - cudaMemcpyAsync(outputs[0], ub_buffer0.addr, size * dtype_size, cudaMemcpyDeviceToDevice, stream)); - } - else - { - TLLM_CHECK_WITH_INFO(false, "Unsupported UB allreduce fusion op"); - } - } - else - { - auto const tpSize = mGroup.size(); - int tpRank = 0; - for (auto const& currentRank : mGroup) - { - if (rank == currentRank) - break; - ++tpRank; - } - - int token_num = size / inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; - int hidden_size = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; - auto params = tensorrt_llm::kernels::AllReduceParams::deserialize( - reinterpret_cast<int64_t*>(const_cast<void*>(inputs[1])), tpSize, tpRank, mType, token_num, hidden_size, - mOp); - - params.local_output_buffer_ptr = outputs[0]; - params.local_input_buffer_ptr = inputs[0]; - params.elts_total = size; - - int fusion_ptr_idx = 2; - params.fusion_params.bias_buffer = mBias ? inputs[fusion_ptr_idx++] : nullptr; - params.fusion_params.residual_buffer = inputs[fusion_ptr_idx++]; - params.fusion_params.weight_buffer = mAffine ? inputs[fusion_ptr_idx++] : nullptr; - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_PREPOST_NORM) - params.fusion_params.weight_buffer_pre_residual_norm = mAffine ? inputs[fusion_ptr_idx++] : nullptr; - params.fusion_params.hidden_size = hidden_size; - params.fusion_params.eps = mEps; - params.fusion_params.intermediate_buffer = outputs[1]; - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM) - { - for (size_t i = 0; i < tpSize; ++i) - { - params.fusion_params.lamport_peer_comm_buffer_ptrs[i] - = reinterpret_cast<void**>(const_cast<void*>(inputs[1]))[tpSize * 4 + i]; - params.fusion_params.lamport_peer_comm_buffer_ptrs[i + tensorrt_llm::kernels::MAX_RANKS_PER_NODE] - = reinterpret_cast<void**>(const_cast<void*>(inputs[1]))[tpSize * 5 + i]; - params.fusion_params.lamport_peer_comm_buffer_ptrs[i + tensorrt_llm::kernels::MAX_RANKS_PER_NODE * 2] - = reinterpret_cast<void**>(const_cast<void*>(inputs[1]))[tpSize * 6 + i]; - } - } - TLLM_LOG_DEBUG("customAllReduce called"); - tensorrt_llm::kernels::customAllReduce(params, mType, runtimeStrategy, mConfig, mOp, stream); - } - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType AllreducePlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index < getNbOutputs()); - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_NVFP4) - { - if (index == 0) - { - return nvinfer1::DataType::kFP4; - } - else if (index == 2) - { - return nvinfer1::DataType::kFP8; - } - } - if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_FP8) - { - if (index == 0) - { - return nvinfer1::DataType::kFP8; - } - } - return inputTypes[0]; -} - -// IPluginV2 Methods - -char const* AllreducePlugin::getPluginType() const noexcept -{ - return ALLREDUCE_PLUGIN_NAME; -} - -char const* AllreducePlugin::getPluginVersion() const noexcept -{ - return ALLREDUCE_PLUGIN_VERSION; -} - -int AllreducePlugin::getNbOutputs() const noexcept -{ - if (mOp == AllReduceFusionOp::NONE) - { - return 1; - } - else if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_NVFP4) - { - return 3; - } - else - { - return 2; - } -} - -bool AllreducePlugin::isCustomAllReduceSupported(int ranks_per_node) const noexcept -{ - constexpr bool isCudaVersionSupported = -#if defined(CUDART_VERSION) && CUDART_VERSION >= 11020 - true; -#else - false; -#endif - - return isCudaVersionSupported && (ranks_per_node % 2 == 0) - && (static_cast<size_t>(ranks_per_node) <= kernels::MAX_RANKS_PER_NODE) && (ranks_per_node > 0); -} - -using tensorrt_llm::common::NvmlManager; -using tensorrt_llm::common::NVMLWrapper; - -std::set<int> getLocalGroup(std::set<int> const& group) -{ - auto const myRank = COMM_SESSION.getRank(); - auto const myLocalRank = LOCAL_COMM_SESSION.getRank(); - auto const localSize = LOCAL_COMM_SESSION.getSize(); - - std::vector<int32_t> ranks(localSize, 0); - std::vector<int32_t> localRanks(localSize, 0); - if (group.size() >= static_cast<size_t>(localSize)) - { - LOCAL_COMM_SESSION.allgather(&myRank, ranks.data(), 1, tensorrt_llm::mpi::MpiType::kINT32); - LOCAL_COMM_SESSION.allgather(&myLocalRank, localRanks.data(), 1, tensorrt_llm::mpi::MpiType::kINT32); - } - else - { - if (myRank == *group.begin()) - { - ranks.clear(); - int rank; - ranks.push_back(myRank); - for (auto it = std::next(std::begin(group), 1); it != group.end(); ++it) - { - COMM_SESSION.recvValue(rank, *it, MpiTag::kDefault); - ranks.push_back(rank); - } - for (auto it = std::next(std::begin(group), 1); it != group.end(); ++it) - { - COMM_SESSION.send(ranks.data(), localSize, tensorrt_llm::mpi::MpiType::kINT32, *it, MpiTag::kDefault); - } - - localRanks.clear(); - localRanks.push_back(myLocalRank); - for (auto it = std::next(std::begin(group), 1); it != group.end(); ++it) - { - COMM_SESSION.recvValue(rank, *it, MpiTag::kDefault); - localRanks.push_back(rank); - } - for (auto it = std::next(std::begin(group), 1); it != group.end(); ++it) - { - COMM_SESSION.send( - localRanks.data(), localSize, tensorrt_llm::mpi::MpiType::kINT32, *it, MpiTag::kDefault); - } - } - else - { - COMM_SESSION.sendValue(myRank, *group.begin(), MpiTag::kDefault); - COMM_SESSION.recv( - ranks.data(), localSize, tensorrt_llm::mpi::MpiType::kINT32, *group.begin(), MpiTag::kDefault); - - COMM_SESSION.sendValue(myLocalRank, *group.begin(), MpiTag::kDefault); - COMM_SESSION.recv( - localRanks.data(), localSize, tensorrt_llm::mpi::MpiType::kINT32, *group.begin(), MpiTag::kDefault); - } - } - - std::set<int> localGroup; - for (size_t i = 0; i < ranks.size(); ++i) - { - auto rank = ranks[i]; - if (group.find(rank) != group.end()) - { - localGroup.insert(localRanks[i]); - } - } - return localGroup; -} - -void AllreducePlugin::initGroupTopology() noexcept -{ - static std::map<std::set<int>, std::tuple<bool, bool>> cache; - if (cache.find(mGroup) != cache.end()) - { - auto [isNVLINKSupported, isP2PSupported] = cache[mGroup]; - mIsNVLINKSupported = isNVLINKSupported; - mIsP2PSupported = isP2PSupported; - return; - } - setGroupTopology(); - cache[mGroup] = {mIsNVLINKSupported, mIsP2PSupported}; -} - -void AllreducePlugin::setGroupTopology() noexcept -{ - auto const rank = COMM_SESSION.getRank(); - TLLM_LOG_INFO("Detecting local TP group for rank %d", rank); - std::set<int> localGroup = getLocalGroup(mGroup); - if (mGroup.size() != localGroup.size()) - { - mIsP2PSupported = false; - mIsNVLINKSupported = false; - TLLM_LOG_INFO("Found inter-node TP group for rank %d", rank); - return; - } - TLLM_LOG_INFO("TP group is intra-node for rank %d", rank); - - NvmlManager nvmlManager; - auto const& nvml = nvmlManager.sharedWrapper(); - std::unordered_set<int> visitedDevice; - mIsP2PSupported = true; - mIsNVLINKSupported = true; - - // Use cudaDeviceCanAccessPeer to determine whether p2p is supported, - // and use nvml to determine whether there are nvlink links between ranks. - for (int firstDeviceId : localGroup) - { - for (int secondDeviceId : localGroup) - { - if (firstDeviceId == secondDeviceId || visitedDevice.find(secondDeviceId) != visitedDevice.end()) - { - continue; - } - - int canAccessPeer = 0; - TLLM_CUDA_CHECK(cudaDeviceCanAccessPeer(&canAccessPeer, firstDeviceId, secondDeviceId)); - - if (!canAccessPeer) - { - mIsP2PSupported = false; - mIsNVLINKSupported = false; - - return; - } - - nvmlDevice_t firstDevice; - NVML_CHECK(nvml->nvmlDeviceGetHandleByIndex(firstDeviceId, &firstDevice)); - - bool isNVLINK = false; - - for (unsigned int link = 0; link < NVML_NVLINK_MAX_LINKS; link++) - { - nvmlPciInfo_t remotePciInfo; - if (nvml->nvmlDeviceGetNvLinkRemotePciInfo(firstDevice, link, &remotePciInfo) != NVML_SUCCESS) - { - continue; - } - - nvmlDevice_t remoteDevice; - auto const result = nvml->nvmlDeviceGetHandleByPciBusId(remotePciInfo.busId, &remoteDevice); - - if (result == NVML_SUCCESS) - { - // Two GPUs are connected directly through nvlink - unsigned int remoteDeviceId; - NVML_CHECK(nvml->nvmlDeviceGetIndex(remoteDevice, &remoteDeviceId)); - - if (remoteDeviceId == static_cast<unsigned int>(secondDeviceId)) - { - isNVLINK = true; - } - } - else if (result == NVML_ERROR_NOT_FOUND) - { - // Maybe Two GPUs are connected via nvswitch, - // now remotePciInfo represents the pci information of nvswitch, - // determine whether nvlink is supported by whether two GPUs are connected to the same nvswitch. - nvmlDevice_t secondDevice; - NVML_CHECK(nvml->nvmlDeviceGetHandleByIndex(secondDeviceId, &secondDevice)); - - for (unsigned int secondLink = 0; secondLink < NVML_NVLINK_MAX_LINKS; secondLink++) - { - nvmlPciInfo_t secondRemotePciInfo; - if (nvml->nvmlDeviceGetNvLinkRemotePciInfo(secondDevice, secondLink, &secondRemotePciInfo) - != NVML_SUCCESS) - { - continue; - } - - if (strcmp(remotePciInfo.busId, secondRemotePciInfo.busId) == 0) - { - isNVLINK = true; - break; - } - } - } - else - { - NVML_CHECK(result); - } - - if (isNVLINK) - { - break; - } - } - - mIsNVLINKSupported &= isNVLINK; - } - visitedDevice.insert(firstDeviceId); - } -} - -int AllreducePlugin::initialize() noexcept -{ - if (isBuilding()) - { - return 0; - } - - TLLM_LOG_TRACE("%s start for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); - mNcclComm = getComm(mGroup); - if (mStrategy != AllReduceStrategyType::NCCL) - { - initGroupTopology(); - } - - TLLM_LOG_TRACE("%s stop for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); - return 0; -} - -void AllreducePlugin::terminate() noexcept {} - -size_t AllreducePlugin::getSerializationSize() const noexcept -{ - return sizeof(int) * mGroup.size() + sizeof(mType) + sizeof(mStrategy) + sizeof(mConfig) + sizeof(mOp) - + sizeof(mEps) + sizeof(mAffine) + sizeof(mBias) + sizeof(mScale); -} - -void AllreducePlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - write(d, mStrategy); - write(d, mConfig); - write(d, mOp); - write(d, mEps); - write(d, mAffine); - write(d, mBias); - write(d, mScale); - for (auto it = mGroup.begin(); it != mGroup.end(); ++it) - { - write(d, *it); - } - TLLM_CHECK(d == a + getSerializationSize()); -} - -void AllreducePlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -AllreducePluginCreator::AllreducePluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("group", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("strategy", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("config", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("fusion_op", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("counter", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("eps", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("affine", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("bias", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("scale", nullptr, PluginFieldType::kINT8)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* AllreducePluginCreator::getPluginName() const noexcept -{ - return ALLREDUCE_PLUGIN_NAME; -} - -char const* AllreducePluginCreator::getPluginVersion() const noexcept -{ - return ALLREDUCE_PLUGIN_VERSION; -} - -PluginFieldCollection const* AllreducePluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* AllreducePluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - std::set<int> group; - nvinfer1::DataType type{}; - AllReduceStrategyType strategy{}; - AllReduceStrategyConfig config{}; - AllReduceFusionOp fusion_op{}; - int32_t counter{}; - float eps{}; - int8_t affine{}; - int8_t bias{}; - int8_t scale{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "group")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - auto const* r = static_cast<int const*>(fields[i].data); - for (int j = 0; j < fields[i].length; ++j) - { - group.insert(*r); - ++r; - } - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "strategy")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - strategy = static_cast<AllReduceStrategyType>(*static_cast<int8_t const*>(fields[i].data)); - } - else if (!strcmp(attrName, "config")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - config = static_cast<AllReduceStrategyConfig>(*static_cast<int8_t const*>(fields[i].data)); - } - else if (!strcmp(attrName, "fusion_op")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - fusion_op = static_cast<AllReduceFusionOp>(*static_cast<int8_t const*>(fields[i].data)); - } - else if (!strcmp(attrName, "counter")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - counter = *static_cast<int32_t const*>(fields[i].data); - } - else if (!strcmp(attrName, "eps")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - eps = *static_cast<float const*>(fields[i].data); - } - else if (!strcmp(attrName, "affine")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - affine = *static_cast<int8_t const*>(fields[i].data); - } - else if (!strcmp(attrName, "bias")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - bias = *static_cast<int8_t const*>(fields[i].data); - } - else if (!strcmp(attrName, "scale")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - scale = *static_cast<int8_t const*>(fields[i].data); - } - } - try - { - auto* obj = new AllreducePlugin(group, type, strategy, config, fusion_op, counter, eps, affine, bias, scale); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* AllreducePluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call AllreducePlugin::destroy() - try - { - auto* obj = new AllreducePlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.h b/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.h deleted file mode 100644 index 881fbf3b89a5..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.h +++ /dev/null @@ -1,114 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/kernels/customAllReduceKernels.h" -#include "tensorrt_llm/plugins/common/plugin.h" - -#include <cassert> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ -namespace tk = ::tensorrt_llm::kernels; - -class AllreducePlugin : public BasePlugin -{ -public: - AllreducePlugin(std::set<int> group, nvinfer1::DataType type, tk::AllReduceStrategyType strategy, - tk::AllReduceStrategyConfig config, tk::AllReduceFusionOp op, int32_t counter, float eps, int8_t affine, - int8_t bias, int8_t scale); - - AllreducePlugin(void const* data, size_t length); - - ~AllreducePlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - bool isCustomAllReduceSupported(int ranks_per_node) const noexcept; - void initGroupTopology() noexcept; - void setGroupTopology() noexcept; - tk::AllReduceStrategyType selectImplementation(size_t messageSize, int worldSize, nvinfer1::DataType type) noexcept; - void check() noexcept; - -private: - std::string const mLayerName; - std::set<int> mGroup; - bool mIsNVLINKSupported; - bool mIsP2PSupported; - nvinfer1::DataType mType; - tk::AllReduceStrategyType mStrategy; - tk::AllReduceStrategyConfig mConfig; - tk::AllReduceFusionOp mOp; - float mEps; - std::shared_ptr<ncclComm_t> mNcclComm; - int8_t mAffine; - int8_t mBias; - int8_t mScale; -}; - -class AllreducePluginCreator : public BaseCreator -{ -public: - AllreducePluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.cpp b/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.cpp deleted file mode 100644 index 089ed31175b2..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.cpp +++ /dev/null @@ -1,252 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "recvPlugin.h" - -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" - -#include <nccl.h> - -using namespace nvinfer1; -using tensorrt_llm::plugins::RecvPluginCreator; -using tensorrt_llm::plugins::RecvPlugin; -using tensorrt_llm::mpi::MpiTag; - -static char const* RECV_PLUGIN_VERSION{"1"}; -static char const* RECV_PLUGIN_NAME{"Recv"}; -PluginFieldCollection RecvPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> RecvPluginCreator::mPluginAttributes; - -RecvPlugin::RecvPlugin(int srcRank, nvinfer1::DataType type) - : mSrcRank(srcRank) - , mType(type) -{ -} - -// Parameterized constructor -RecvPlugin::RecvPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mType); - read(d, mSrcRank); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* RecvPlugin::clone() const noexcept -{ - auto* plugin = new RecvPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs RecvPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - return inputs[0]; -} - -bool RecvPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); -} - -void RecvPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t RecvPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -int RecvPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - size_t size = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) - { - size *= inputDesc[0].dims.d[i]; - } - TLLM_LOG_DEBUG("start ncclRecv with size %d", size); - NCCLCHECK(ncclRecv(outputs[0], size, (*getDtypeMap())[inputDesc[0].type], 0, mComm, stream)); - TLLM_LOG_DEBUG("end ncclRecv with size %d", size); - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType RecvPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - assert(index == 0); - return inputTypes[0]; -} - -// IPluginV2 Methods - -char const* RecvPlugin::getPluginType() const noexcept -{ - return RECV_PLUGIN_NAME; -} - -char const* RecvPlugin::getPluginVersion() const noexcept -{ - return RECV_PLUGIN_VERSION; -} - -int RecvPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int RecvPlugin::initialize() noexcept -{ - if (isBuilding()) - { - return 0; - } - ncclUniqueId id; - COMM_SESSION.recvValue(id, mSrcRank, MpiTag::kDefault); -// Need static connection initialization for accurate KV cache size estimation -#if defined(_WIN32) - if (getenv("NCCL_RUNTIME_CONNECT") == nullptr) - _putenv_s("NCCL_RUNTIME_CONNECT", "0"); -#else - setenv("NCCL_RUNTIME_CONNECT", "0", 0); -#endif // _WIN32 - NCCLCHECK(ncclCommInitRank(&mComm, 2, id, 1)); - return 0; -} - -void RecvPlugin::terminate() noexcept -{ - if (isBuilding()) - { - return; - } - NCCLCHECK(ncclCommDestroy(mComm)); -} - -size_t RecvPlugin::getSerializationSize() const noexcept -{ - return sizeof(mSrcRank) + sizeof(mType); -} - -void RecvPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - write(d, mSrcRank); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void RecvPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -RecvPluginCreator::RecvPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("src_rank", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* RecvPluginCreator::getPluginName() const noexcept -{ - return RECV_PLUGIN_NAME; -} - -char const* RecvPluginCreator::getPluginVersion() const noexcept -{ - return RECV_PLUGIN_VERSION; -} - -PluginFieldCollection const* RecvPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* RecvPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int srcRank{}; - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "src_rank")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - srcRank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - - try - { - auto* obj = new RecvPlugin(srcRank, type); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* RecvPluginCreator::deserializePlugin(char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call RecvPlugin::destroy() - try - { - auto* obj = new RecvPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.h b/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.h deleted file mode 100644 index 5c8eedfb5218..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.h +++ /dev/null @@ -1,90 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class RecvPlugin : public BasePlugin -{ -public: - RecvPlugin(int srcRank, nvinfer1::DataType type); - - RecvPlugin(void const* data, size_t length); - - ~RecvPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - ncclComm_t mComm; // TODO: Remove this - int mSrcRank; - nvinfer1::DataType mType; -}; - -class RecvPluginCreator : public BaseCreator -{ -public: - RecvPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.cpp b/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.cpp deleted file mode 100644 index fe17c44fc418..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.cpp +++ /dev/null @@ -1,252 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "reduceScatterPlugin.h" - -#include <cassert> -#include <nccl.h> - -using namespace nvinfer1; -using tensorrt_llm::plugins::ReduceScatterPluginCreator; -using tensorrt_llm::plugins::ReduceScatterPlugin; - -static char const* REDUCE_SCATTER_PLUGIN_VERSION{"1"}; -static char const* REDUCE_SCATTER_PLUGIN_NAME{"ReduceScatter"}; -PluginFieldCollection ReduceScatterPluginCreator::mFC{}; -std::vector<PluginField> ReduceScatterPluginCreator::mPluginAttributes; - -ReduceScatterPlugin::ReduceScatterPlugin(std::set<int> group, nvinfer1::DataType type) - : mGroup(std::move(group)) - , mType(type) -{ -} - -// Parameterized constructor -ReduceScatterPlugin::ReduceScatterPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mType); - mGroup.clear(); - int groupItem = 0; - while (d != a + length) - { - read(d, groupItem); - mGroup.insert(groupItem); - } - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* ReduceScatterPlugin::clone() const noexcept -{ - auto* plugin = new ReduceScatterPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs ReduceScatterPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - auto output = inputs[0]; - output.d[0] - = exprBuilder.operation(DimensionOperation::kFLOOR_DIV, *output.d[0], *exprBuilder.constant(mGroup.size())); - return output; -} - -bool ReduceScatterPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); -} - -void ReduceScatterPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t ReduceScatterPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -int ReduceScatterPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - size_t size = 1; - for (int i = 0; i < outputDesc[0].dims.nbDims; ++i) - { - size *= outputDesc[0].dims.d[i]; - } - - TLLM_CHECK_WITH_INFO(mNcclComm.get() != nullptr, "mNcclComm should be initialized before used"); - NCCLCHECK(ncclReduceScatter( - inputs[0], outputs[0], size, (*getDtypeMap())[inputDesc[0].type], ncclSum, *mNcclComm, stream)); - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType ReduceScatterPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - assert(index == 0); - return inputTypes[0]; -} - -// IPluginV2 Methods - -char const* ReduceScatterPlugin::getPluginType() const noexcept -{ - return REDUCE_SCATTER_PLUGIN_NAME; -} - -char const* ReduceScatterPlugin::getPluginVersion() const noexcept -{ - return REDUCE_SCATTER_PLUGIN_VERSION; -} - -int ReduceScatterPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int ReduceScatterPlugin::initialize() noexcept -{ - if (isBuilding()) - { - return 0; - } - mNcclComm = getComm(mGroup); - return 0; -} - -void ReduceScatterPlugin::terminate() noexcept {} - -size_t ReduceScatterPlugin::getSerializationSize() const noexcept -{ - return sizeof(int) * mGroup.size() + sizeof(mType); -} - -void ReduceScatterPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - for (auto it = mGroup.begin(); it != mGroup.end(); ++it) - { - write(d, *it); - } - TLLM_CHECK(d == a + getSerializationSize()); -} - -void ReduceScatterPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -ReduceScatterPluginCreator::ReduceScatterPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("group", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* ReduceScatterPluginCreator::getPluginName() const noexcept -{ - return REDUCE_SCATTER_PLUGIN_NAME; -} - -char const* ReduceScatterPluginCreator::getPluginVersion() const noexcept -{ - return REDUCE_SCATTER_PLUGIN_VERSION; -} - -PluginFieldCollection const* ReduceScatterPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* ReduceScatterPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - std::set<int> group; - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "group")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - auto const* r = static_cast<int const*>(fields[i].data); - for (int j = 0; j < fields[i].length; ++j) - { - group.insert(*r); - ++r; - } - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - - try - { - auto* obj = new ReduceScatterPlugin(group, type); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* ReduceScatterPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call ReduceScatterPlugin::destroy() - try - { - auto* obj = new ReduceScatterPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.h b/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.h deleted file mode 100644 index c630b57a2b98..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.h +++ /dev/null @@ -1,91 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class ReduceScatterPlugin : public BasePlugin -{ -public: - ReduceScatterPlugin(std::set<int> group, nvinfer1::DataType type); - - ReduceScatterPlugin(void const* data, size_t length); - - ~ReduceScatterPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - const std::string mLayerName; - std::set<int> mGroup; - nvinfer1::DataType mType; - std::shared_ptr<ncclComm_t> mNcclComm; -}; - -class ReduceScatterPluginCreator : public BaseCreator -{ -public: - ReduceScatterPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.cpp b/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.cpp deleted file mode 100644 index 81d66aa8211e..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.cpp +++ /dev/null @@ -1,255 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "sendPlugin.h" - -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" - -#include <cassert> -#include <nccl.h> - -using namespace nvinfer1; -using tensorrt_llm::plugins::SendPluginCreator; -using tensorrt_llm::plugins::SendPlugin; -using tensorrt_llm::mpi::MpiTag; - -static char const* SEND_PLUGIN_VERSION{"1"}; -static char const* SEND_PLUGIN_NAME{"Send"}; -PluginFieldCollection SendPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> SendPluginCreator::mPluginAttributes; - -SendPlugin::SendPlugin(int tgtRank, nvinfer1::DataType type) - : mTgtRank(tgtRank) - , mType(type) -{ -} - -// Parameterized constructor -SendPlugin::SendPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mType); - read(d, mTgtRank); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* SendPlugin::clone() const noexcept -{ - auto* plugin = new SendPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs SendPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - return inputs[0]; -} - -bool SendPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); -} - -void SendPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t SendPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -int SendPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - size_t size = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) - { - size *= inputDesc[0].dims.d[i]; - } - - TLLM_LOG_DEBUG("start ncclSend with size %d", size); - NCCLCHECK(ncclSend(inputs[0], size, (*getDtypeMap())[inputDesc[0].type], 1, mComm, stream)); - TLLM_LOG_DEBUG("end ncclSend with size %d", size); - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType SendPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - assert(index == 0); - return inputTypes[0]; -} - -// IPluginV2 Methods - -char const* SendPlugin::getPluginType() const noexcept -{ - return SEND_PLUGIN_NAME; -} - -char const* SendPlugin::getPluginVersion() const noexcept -{ - return SEND_PLUGIN_VERSION; -} - -int SendPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int SendPlugin::initialize() noexcept -{ - if (isBuilding()) - { - return 0; - } - - ncclUniqueId id; - ncclGetUniqueId(&id); - COMM_SESSION.sendValue(id, mTgtRank, MpiTag::kDefault); -// Need static connection initialization for accurate KV cache size estimation -#if defined(_WIN32) - if (getenv("NCCL_RUNTIME_CONNECT") == nullptr) - _putenv_s("NCCL_RUNTIME_CONNECT", "0"); -#else - setenv("NCCL_RUNTIME_CONNECT", "0", 0); -#endif // _WIN32 - NCCLCHECK(ncclCommInitRank(&mComm, 2, id, 0)); - return 0; -} - -void SendPlugin::terminate() noexcept -{ - if (isBuilding()) - { - return; - } - NCCLCHECK(ncclCommDestroy(mComm)); -} - -size_t SendPlugin::getSerializationSize() const noexcept -{ - return sizeof(mTgtRank) + sizeof(mType); -} - -void SendPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - write(d, mTgtRank); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void SendPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -SendPluginCreator::SendPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("tgt_rank", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* SendPluginCreator::getPluginName() const noexcept -{ - return SEND_PLUGIN_NAME; -} - -char const* SendPluginCreator::getPluginVersion() const noexcept -{ - return SEND_PLUGIN_VERSION; -} - -PluginFieldCollection const* SendPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* SendPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int tgtRank{}; - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "tgt_rank")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - tgtRank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - - try - { - auto* obj = new SendPlugin(tgtRank, type); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* SendPluginCreator::deserializePlugin(char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call SendPlugin::destroy() - try - { - auto* obj = new SendPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.h b/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.h deleted file mode 100644 index 0d36b0ebff28..000000000000 --- a/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.h +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class SendPlugin : public BasePlugin -{ -public: - SendPlugin(int tgtRank, nvinfer1::DataType type); - - SendPlugin(void const* data, size_t length); - - ~SendPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - ncclComm_t mComm; // TODO: Remove this - int mTgtRank; - nvinfer1::DataType mType; -}; - -class SendPluginCreator : public BaseCreator -{ -public: - SendPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/qserveGemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/qserveGemmPlugin/CMakeLists.txt deleted file mode 100755 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/qserveGemmPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.cpp b/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.cpp deleted file mode 100644 index 166f1cc32cbe..000000000000 --- a/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.cpp +++ /dev/null @@ -1,416 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "qserveGemmPlugin.h" -#include "tensorrt_llm/kernels/qserveGemm.h" -#include <cassert> -#include <numeric> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::QServeGemmPluginCreator; -using tensorrt_llm::plugins::QServeGemmPlugin; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; -using namespace tensorrt_llm::kernels::qserve; - -static char const* QSERVE_GEMM_PLUGIN_VERSION{"1"}; -static char const* QSERVE_GEMM_PLUGIN_NAME{"QServeGemm"}; - -PluginFieldCollection QServeGemmPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> QServeGemmPluginCreator::mPluginAttributes; - -namespace tensorrt_llm::plugins -{ - -QServeGemmPlugin::QServeGemmPlugin( - // QuantMode quantMode, - nvinfer1::DataType dtype, int groupSize) -{ - init(dtype, groupSize); -} - -QServeGemmPlugin::QServeGemmPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - - nvinfer1::DataType type; - unsigned int quantMode; - int groupSize; - - read(d, quantMode); - read(d, type); - read(d, groupSize); - - read(d, mDims); - - // mQuantMode = QuantMode(quantMode); - - init(type, groupSize); - - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -void QServeGemmPlugin::init(nvinfer1::DataType dtype, int groupSize) -{ - if (groupSize <= 0) - groupSize = -1; // Per-channel - mGroupSize = groupSize; - mType = dtype; - mRunner = std::make_shared<QServeGemmRunner>(); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* QServeGemmPlugin::clone() const noexcept -{ - auto* plugin = new QServeGemmPlugin(*this); - return plugin; -} - -nvinfer1::DimsExprs QServeGemmPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs == 6); - TLLM_CHECK(outputIndex == 0); - int const nbDimsA = inputs[0].nbDims; - TLLM_CHECK(nbDimsA >= 2); - DimsExprs ret; - ret.nbDims = nbDimsA; - for (int ii = 0; ii < nbDimsA - 1; ++ii) - { - ret.d[ii] = inputs[0].d[ii]; - } - ret.d[nbDimsA - 1] = inputs[1].d[0]; - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool QServeGemmPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - if (mGroupSize != -1) - { // Per-group - switch (pos) - { - case 0: - // activation - return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; - case 1: - // uint4 weights packed in int8 - return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; - case 2: - // int8 weight s2_zeros - return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; - case 3: - // int8 weight s2_scales - return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; - case 4: - // fp16 weight s1_scales - return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; - case 5: - // fp16 activation scales - return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; - case 6: - // fp16 output activation - return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; - default: return false; - } - } - - else - { // Per-channel - switch (pos) - { - case 0: - // activation - return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; - case 1: - // uint4 weights packed in int8 - return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; - case 2: - // fp16 s1_scales - return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; - case 3: - // fp16 s1_szeros - return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; - case 4: - // fp16 act_sums - return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; - case 5: - // fp16 act_scales - return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; - case 6: - // fp16 output activation - return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; - default: return false; - } - } -} - -void QServeGemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); - auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); - - int const maxK = in[0].max.d[in[0].max.nbDims - 1]; - int const maxN = in[1].max.d[0]; - int const minK = in[0].min.d[in[0].min.nbDims - 1]; - int const minN = in[1].min.d[0]; - - TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); - TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, maxN, maxK}; - } - m_workspaceMaxSize = mRunner->getWorkspaceSize(maxM, maxN, maxK); -} - -size_t QServeGemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return m_workspaceMaxSize; -} - -int QServeGemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept -{ - // inputs - - // Per group: - // activation [M, K] int8_t Quantized sint8 activations - // weights [N, K/2] int8_t Quantized uint4 weights (packed as int8_t) - // s2_zeros [K/group_size, N] int8_t Level-2 sint8 scaled zeros of weights - // s2_scales [K/group_size, N] int8_t Level-2 sint8 scales of weights - // s1_scales [N] half Level-1 fp16 scales of weights - // act_scales [M] half Scales of activations - - // Per channel: - // activation [M, K] int8_t Quantized sint8 activations - // weights [N, K/2] int8_t Quantized uint4 weights (packed as int8_t) - // s1_scales [N] half Level-1 scales of weights - // s1_szeros [N] half Level-1 scaled zeros of weights - // act_sums [M] half Per-token sums of activations - // act_scales [M] half Scales of activations - - // outputs - // mat [M(*), N] half - - int64_t m64 = 1; - for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) - { - m64 *= inputDesc[0].dims.d[ii]; - } - int const m = TLLM_INT32_CAST(m64); - int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[0]); - int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); - - // TODO: Implement optimized kernels if (m <= 4) - - if (mGroupSize != -1) - { - ParamsPerGroup params = {reinterpret_cast<int8_t const*>(inputs[0]), // A - reinterpret_cast<int8_t const*>(inputs[1]), // B - reinterpret_cast<int8_t const*>(inputs[2]), // s2_zeros - reinterpret_cast<int8_t const*>(inputs[3]), // s2_scales - reinterpret_cast<half const*>(inputs[4]), // s1_scales - reinterpret_cast<half const*>(inputs[5]), // act_scales - reinterpret_cast<half*>(outputs[0]), // C - m, n, k}; - mRunner->gemmPerGroup(params, stream); - } - else - { - ParamsPerChannel params = {reinterpret_cast<int8_t const*>(inputs[0]), // A - reinterpret_cast<int8_t const*>(inputs[1]), // B - reinterpret_cast<half const*>(inputs[2]), // s1_scales - reinterpret_cast<half const*>(inputs[3]), // s1_szeros - reinterpret_cast<half const*>(inputs[4]), // act_sums - reinterpret_cast<half const*>(inputs[5]), // act_scales - reinterpret_cast<half*>(outputs[0]), // C - m, n, k}; - mRunner->gemmPerChannel(params, stream); - } - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType QServeGemmPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* QServeGemmPlugin::getPluginType() const noexcept -{ - return QSERVE_GEMM_PLUGIN_NAME; -} - -char const* QServeGemmPlugin::getPluginVersion() const noexcept -{ - return QSERVE_GEMM_PLUGIN_VERSION; -} - -int QServeGemmPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int QServeGemmPlugin::initialize() noexcept -{ - configGemm(); - return 0; -} - -void QServeGemmPlugin::terminate() noexcept {} - -size_t QServeGemmPlugin::getSerializationSize() const noexcept -{ - return sizeof(mQuantMode) + // QuantMode - sizeof(mType) + // dtype - sizeof(mGroupSize) + // GroupSize - sizeof(mDims); // Dimensions -} - -void QServeGemmPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mQuantMode.value()); - write(d, mType); - write(d, mGroupSize); - write(d, mDims); - - TLLM_CHECK(d == a + getSerializationSize()); -} - -void QServeGemmPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -void QServeGemmPlugin::configGemm() {} - -/////////////// - -QServeGemmPluginCreator::QServeGemmPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.push_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.push_back(PluginField("group_size", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* QServeGemmPluginCreator::getPluginName() const noexcept -{ - return QSERVE_GEMM_PLUGIN_NAME; -} - -char const* QServeGemmPluginCreator::getPluginVersion() const noexcept -{ - return QSERVE_GEMM_PLUGIN_VERSION; -} - -PluginFieldCollection const* QServeGemmPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* QServeGemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - // We do not use any fields for now. - - PluginField const* fields = fc->fields; - - // bool perTokenScaling, perChannelScaling; - DataType dtype{}; - int group_size = -1; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - dtype = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - // Only supports fp16 for now. - assert(dtype == nvinfer1::DataType::kHALF); - } - else if (!strcmp(attrName, "group_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - group_size = *static_cast<int const*>(fields[i].data); - // Currently only support per-channel or g128. - assert(group_size == -1 || group_size == 128); - } - } - try - { - // QServeGemmPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - // auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); - // QuantMode quantMode = QuantMode::fromQuantAlgo("W4A8_QSERVE"); - auto* obj = new QServeGemmPlugin(dtype, group_size); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* QServeGemmPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call QServeGemmPlugin::destroy() - try - { - // Create plugin profiler with private tactics map which is read from the serialized engine - // auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); - auto* obj = new QServeGemmPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.h b/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.h deleted file mode 100644 index 086460863c4f..000000000000 --- a/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <memory> -#include <string> -#include <tensorrt_llm/kernels/qserveGemm.h> - -namespace tensorrt_llm::plugins -{ - -using QServeGemmRunnerPtr = std::shared_ptr<tensorrt_llm::kernels::qserve::QServeGemmRunner>; - -class QServeGemmPlugin : public BasePlugin -{ -public: - // using PluginProfilerPtr = std::shared_ptr<QServeGemmPluginProfiler>; - - QServeGemmPlugin(void const* data, size_t length); - - QServeGemmPlugin(nvinfer1::DataType dtype, int groupSize); - - ~QServeGemmPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - void init(nvinfer1::DataType dtype, int groupSize); - - void configGemm(); - - std::string const mLayerName; - - QServeGemmRunnerPtr mRunner; - - tensorrt_llm::common::QuantMode mQuantMode; // Not used for now - GemmDims mDims{}; - - size_t m_workspaceMaxSize; - - // Only supports fp16 output for now. - nvinfer1::DataType mType; - - int mGroupSize; -}; - -class QServeGemmPluginCreator : public BaseCreator -{ -public: - QServeGemmPluginCreator(); - - QServeGemmPluginCreator(void const* data, size_t length); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/CMakeLists.txt deleted file mode 100755 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.cpp b/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.cpp deleted file mode 100644 index 23d0b80390e3..000000000000 --- a/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.cpp +++ /dev/null @@ -1,353 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "quantizePerTokenPlugin.h" -#include "tensorrt_llm/kernels/quantization.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using namespace tensorrt_llm::kernels; -using tensorrt_llm::plugins::QuantizePerTokenPluginCreator; -using tensorrt_llm::plugins::QuantizePerTokenPlugin; - -static char const* QUANTIZE_PER_TOKEN_PLUGIN_VERSION{"1"}; -static char const* QUANTIZE_PER_TOKEN_PLUGIN_NAME{"QuantizePerToken"}; -PluginFieldCollection QuantizePerTokenPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> QuantizePerTokenPluginCreator::mPluginAttributes; - -QuantizePerTokenPlugin::QuantizePerTokenPlugin( - nvinfer1::DataType outputType, QuantMode quantMode, bool clampValEnabled, bool sumPerToken) - : mOutputType{outputType} - , mQuantMode{quantMode} - , mClampValEnabled{clampValEnabled} - , mSumPerToken{sumPerToken} -{ - TLLM_CHECK_WITH_INFO(mOutputType == nvinfer1::DataType::kINT8 || mOutputType == nvinfer1::DataType::kFP8, - "Only int8 or fp8 output type is allowed."); - // Check if the quant mode is valid. - TLLM_CHECK_WITH_INFO(mQuantMode.hasPerTokenScaling(), "The quant mode is not valid."); -} - -// Parameterized constructor -QuantizePerTokenPlugin::QuantizePerTokenPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mOutputType); - read(d, mQuantMode); - read(d, mClampValEnabled); - read(d, mSumPerToken); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* QuantizePerTokenPlugin::clone() const noexcept -{ - auto* plugin = new QuantizePerTokenPlugin(mOutputType, mQuantMode, mClampValEnabled, mSumPerToken); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs QuantizePerTokenPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs <= 2); - TLLM_CHECK(outputIndex <= 2); - if (outputIndex == 2) - { - // Per token sums. - TLLM_CHECK(mSumPerToken); - } - - if (outputIndex == 0) - { - // Quantized input - return inputs[0]; - } - - DimsExprs ret; - ret.nbDims = inputs[0].nbDims; - for (int ii = 0; ii < ret.nbDims - 1; ++ii) - { - ret.d[ii] = inputs[0].d[ii]; - } - ret.d[ret.nbDims - 1] = exprBuilder.constant(1); - // [M(*), 1] dynamic per token scales or sums - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool QuantizePerTokenPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - if (pos == 0) - { - // activation - return (inOut[pos].type == nvinfer1::DataType::kFLOAT || inOut[pos].type == nvinfer1::DataType::kHALF -#ifdef ENABLE_BF16 - || inOut[pos].type == nvinfer1::DataType::kBF16 -#endif - ) - && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (pos == 1 && mClampValEnabled) - { - // clamp_max_v - return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (pos == 1 + int(mClampValEnabled)) - { - // quantized activation - return inOut[pos].type == mOutputType && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (pos == 2 + int(mClampValEnabled)) - { - // scales - return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (pos == 3 + int(mClampValEnabled)) - { - TLLM_CHECK(mSumPerToken); - // per-token sums - return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; - } - - // Never should be here - assert(false); - return false; -} - -void QuantizePerTokenPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t QuantizePerTokenPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -template <typename T, typename QuantT> -void QuantizePerTokenPlugin::dispatchDataType(void* output, void const* input, void const* clampValPtr, void* scalePtr, - void* sumPtr, int dim0, int dim1, cudaStream_t stream) noexcept -{ - // inputs - // activation [dim0(*), dim1] - // clamp_value [2], contains min val, and max val (optional) - // outputs - // quant [dim0(*), dim1] - // scale_tokens [dim0(*), 1] - - invokePerTokenQuantization(reinterpret_cast<QuantT*>(output), reinterpret_cast<T const*>(input), dim0, dim1, - reinterpret_cast<float const*>(clampValPtr), reinterpret_cast<float*>(scalePtr), - reinterpret_cast<float*>(sumPtr), mQuantMode, stream); -} - -int QuantizePerTokenPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // activation [M(*), K] - // clamp_value [2], contains min val, and max val (optional) - // outputs - // quant [M(*), K] Quantized activations. - // scale_tokens [M(*), 1] Per-token scales. - // token_sums [M(*), 1] (Optional) Per-token sums of all the channels (before quantization). - - int64_t m = 1; - for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) - { - m *= inputDesc[0].dims.d[ii]; - } - int64_t const k = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; - - void const* clampValPtr = mClampValEnabled ? inputs[1] : nullptr; - void* sumPtr = mSumPerToken ? outputs[2] : nullptr; - - if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kINT8) - { - dispatchDataType<float, int8_t>(outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); - } -#ifdef ENABLE_FP8 - else if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kFP8) - { - dispatchDataType<float, __nv_fp8_e4m3>(outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); - } -#endif // ENABLE_FP8 - else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kINT8) - { - dispatchDataType<half, int8_t>(outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); - } -#ifdef ENABLE_FP8 - else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kFP8) - { - dispatchDataType<half, __nv_fp8_e4m3>(outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); - } -#endif // ENABLE_FP8 -#ifdef ENABLE_BF16 - else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kINT8) - { - dispatchDataType<__nv_bfloat16, int8_t>(outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); - } -#ifdef ENABLE_FP8 - else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kFP8) - { - dispatchDataType<__nv_bfloat16, __nv_fp8_e4m3>( - outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); - } -#endif // ENABLE_FP8 -#endif // ENABLE_BF16 - sync_check_cuda_error(stream); - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType QuantizePerTokenPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(nbInputs >= 1); - TLLM_CHECK(index <= 2); - if (index == 2) - { - // Per token sums. - TLLM_CHECK(mSumPerToken); - } - return index == 0 ? mOutputType : nvinfer1::DataType::kFLOAT; -} - -// IPluginV2 Methods - -char const* QuantizePerTokenPlugin::getPluginType() const noexcept -{ - return QUANTIZE_PER_TOKEN_PLUGIN_NAME; -} - -char const* QuantizePerTokenPlugin::getPluginVersion() const noexcept -{ - return QUANTIZE_PER_TOKEN_PLUGIN_VERSION; -} - -int QuantizePerTokenPlugin::getNbOutputs() const noexcept -{ - return 2 + static_cast<int>(mSumPerToken); -} - -int QuantizePerTokenPlugin::initialize() noexcept -{ - return 0; -} - -void QuantizePerTokenPlugin::terminate() noexcept {} - -size_t QuantizePerTokenPlugin::getSerializationSize() const noexcept -{ - return sizeof(mOutputType) + sizeof(mQuantMode) + sizeof(mClampValEnabled) + sizeof(mSumPerToken); -} - -void QuantizePerTokenPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mOutputType); - write(d, mQuantMode); - write(d, mClampValEnabled); - write(d, mSumPerToken); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void QuantizePerTokenPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -QuantizePerTokenPluginCreator::QuantizePerTokenPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("quant_mode", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("clamp_enabled", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("sum_per_token", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* QuantizePerTokenPluginCreator::getPluginName() const noexcept -{ - return QUANTIZE_PER_TOKEN_PLUGIN_NAME; -} - -char const* QuantizePerTokenPluginCreator::getPluginVersion() const noexcept -{ - return QUANTIZE_PER_TOKEN_PLUGIN_VERSION; -} - -PluginFieldCollection const* QuantizePerTokenPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* QuantizePerTokenPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginFieldParser p{fc->nbFields, fc->fields}; - try - { - auto* obj = new QuantizePerTokenPlugin(static_cast<nvinfer1::DataType>(p.getScalar<int32_t>("type_id").value()), - QuantMode(p.getScalar<int32_t>("quant_mode").value()), - static_cast<bool>(p.getScalar<int8_t>("clamp_enabled").value()), - static_cast<bool>(p.getScalar<int32_t>("sum_per_token").value())); - - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* QuantizePerTokenPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call QuantizePerTokenPlugin::destroy() - try - { - auto* obj = new QuantizePerTokenPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.h b/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.h deleted file mode 100644 index 47b218acfd28..000000000000 --- a/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.h +++ /dev/null @@ -1,104 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class QuantizePerTokenPlugin : public BasePlugin -{ -public: - QuantizePerTokenPlugin(nvinfer1::DataType outputType, tensorrt_llm::common::QuantMode quantMode, - bool clampValEnabled, bool sumPerToken); - - QuantizePerTokenPlugin(void const* data, size_t length); - - ~QuantizePerTokenPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - template <typename T, typename QuantT> - void dispatchDataType(void* output, void const* input, void const* clampValPtr, void* scalePtr, void* sumPtr, - int dim0, int dim1, cudaStream_t stream) noexcept; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - std::string const mLayerName; - // The quantized output data type. - nvinfer1::DataType mOutputType; - // The quantization mode. - tensorrt_llm::common::QuantMode mQuantMode; - // Do we clamp the input tensor ? - bool mClampValEnabled; - // Do we output the per-token sum? - bool mSumPerToken; -}; - -class QuantizePerTokenPluginCreator : public BaseCreator -{ -public: - QuantizePerTokenPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/CMakeLists.txt deleted file mode 100755 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.cpp b/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.cpp deleted file mode 100644 index cacb32b809bf..000000000000 --- a/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.cpp +++ /dev/null @@ -1,250 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "quantizeTensorPlugin.h" -#include "tensorrt_llm/kernels/quantization.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using tensorrt_llm::plugins::QuantizeTensorPluginCreator; -using tensorrt_llm::plugins::QuantizeTensorPlugin; - -static char const* QUANTIZE_TENSOR_PLUGIN_VERSION{"1"}; -static char const* QUANTIZE_TENSOR_PLUGIN_NAME{"QuantizeTensor"}; -PluginFieldCollection QuantizeTensorPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> QuantizeTensorPluginCreator::mPluginAttributes; - -QuantizeTensorPlugin::QuantizeTensorPlugin() {} - -// Parameterized constructor -QuantizeTensorPlugin::QuantizeTensorPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* QuantizeTensorPlugin::clone() const noexcept -{ - return new QuantizeTensorPlugin(*this); -} - -nvinfer1::DimsExprs QuantizeTensorPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs == 2); - TLLM_CHECK(outputIndex < 1); - // Quantized input - return inputs[0]; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool QuantizeTensorPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - switch (pos) - { - case 0: - // activation - return (inOut[pos].type == nvinfer1::DataType::kFLOAT || inOut[pos].type == nvinfer1::DataType::kHALF -#ifdef ENABLE_BF16 - || inOut[pos].type == nvinfer1::DataType::kBF16 -#endif - ) - && inOut[pos].format == TensorFormat::kLINEAR; - case 1: - // scales - return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; - case 2: - // quantized activation - return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; - default: - // Never should be here - TLLM_CHECK(false); - return false; - } -} - -void QuantizeTensorPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t QuantizeTensorPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -int QuantizeTensorPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // activation [M(*), K] - // scale [1, 1] - // outputs - // quant [M(*), K] - - int64_t numElts = 1; - for (int ii = 0; ii < inputDesc[0].dims.nbDims; ++ii) - { - numElts *= inputDesc[0].dims.d[ii]; - } - - if (inputDesc[0].type == DataType::kFLOAT) - { - invokeQuantization<float>(reinterpret_cast<int8_t*>(outputs[0]), reinterpret_cast<float const*>(inputs[0]), - numElts, reinterpret_cast<float const*>(inputs[1]), stream, mProp.maxGridSize[0]); - } - else if (inputDesc[0].type == DataType::kHALF) - { - invokeQuantization<half>(reinterpret_cast<int8_t*>(outputs[0]), reinterpret_cast<half const*>(inputs[0]), - numElts, reinterpret_cast<float const*>(inputs[1]), stream, mProp.maxGridSize[0]); - } -#ifdef ENABLE_BF16 - else if (inputDesc[0].type == DataType::kBF16) - { - invokeQuantization<__nv_bfloat16>(reinterpret_cast<int8_t*>(outputs[0]), - reinterpret_cast<__nv_bfloat16 const*>(inputs[0]), numElts, reinterpret_cast<float const*>(inputs[1]), - stream, mProp.maxGridSize[0]); - } -#endif - sync_check_cuda_error(stream); - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType QuantizeTensorPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(nbInputs == 2); - TLLM_CHECK(index == 0); - return nvinfer1::DataType::kINT8; -} - -// IPluginV2 Methods - -char const* QuantizeTensorPlugin::getPluginType() const noexcept -{ - return QUANTIZE_TENSOR_PLUGIN_NAME; -} - -char const* QuantizeTensorPlugin::getPluginVersion() const noexcept -{ - return QUANTIZE_TENSOR_PLUGIN_VERSION; -} - -int QuantizeTensorPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int QuantizeTensorPlugin::initialize() noexcept -{ - int deviceId = 0; - tensorrt_llm::common::check_cuda_error(cudaGetDevice(&deviceId)); - tensorrt_llm::common::check_cuda_error(cudaGetDeviceProperties(&mProp, deviceId)); - return 0; -} - -void QuantizeTensorPlugin::terminate() noexcept {} - -size_t QuantizeTensorPlugin::getSerializationSize() const noexcept -{ - return 0; -} - -void QuantizeTensorPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - TLLM_CHECK(d == a + getSerializationSize()); -} - -void QuantizeTensorPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -QuantizeTensorPluginCreator::QuantizeTensorPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* QuantizeTensorPluginCreator::getPluginName() const noexcept -{ - return QUANTIZE_TENSOR_PLUGIN_NAME; -} - -char const* QuantizeTensorPluginCreator::getPluginVersion() const noexcept -{ - return QUANTIZE_TENSOR_PLUGIN_VERSION; -} - -PluginFieldCollection const* QuantizeTensorPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* QuantizeTensorPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - try - { - auto* obj = new QuantizeTensorPlugin(); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* QuantizeTensorPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call QuantizeTensorPlugin::destroy() - try - { - auto* obj = new QuantizeTensorPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.h b/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.h deleted file mode 100644 index 6f1ce864ec35..000000000000 --- a/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class QuantizeTensorPlugin : public BasePlugin -{ -public: - QuantizeTensorPlugin(); - - QuantizeTensorPlugin(void const* data, size_t length); - - ~QuantizeTensorPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - const std::string mLayerName; - cudaDeviceProp mProp; -}; - -class QuantizeTensorPluginCreator : public BaseCreator -{ -public: - QuantizeTensorPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.cpp b/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.cpp deleted file mode 100644 index b5eaffeeda2a..000000000000 --- a/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.cpp +++ /dev/null @@ -1,301 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "quantizeToFP4Plugin.h" -#include "pluginUtils.h" -#include "tensorrt_llm/kernels/quantization.h" -#include <NvInferRuntimeBase.h> - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::QuantizeToFP4PluginCreator; -using tensorrt_llm::plugins::QuantizeToFP4Plugin; - -constexpr nvinfer1::DataType FP4_DTYPE = nvinfer1::DataType::kFP4; -constexpr nvinfer1::DataType FP8_DTYPE = nvinfer1::DataType::kFP8; - -static char const* QUANT_FP4_PLUGIN_VERSION{"1"}; -static char const* QUANT_FP4_PLUGIN_NAME{"QuantizeToFP4"}; -PluginFieldCollection QuantizeToFP4PluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> QuantizeToFP4PluginCreator::mPluginAttributes; - -QuantizeToFP4Plugin::QuantizeToFP4Plugin(){}; - -// Parameterized constructor -QuantizeToFP4Plugin::QuantizeToFP4Plugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* QuantizeToFP4Plugin::clone() const noexcept -{ - auto* plugin = new QuantizeToFP4Plugin(); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs QuantizeToFP4Plugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - // Quantized output in FP4 datatype. - if (outputIndex == 0) - { - DimsExprs ret; - ret.nbDims = inputs[0].nbDims; - for (int di = 0; di < ret.nbDims; ++di) - { - ret.d[di] = inputs[0].d[di]; - } - // // Div up by 16 as the storage type has 16 FP4 values per element. - // ret.d[ret.nbDims - 1] - // = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 1], - // *exprBuilder.constant(16)); - return ret; - } - // Scaling Factors in FP8. - else if (outputIndex == 1) - { - DimsExprs ret; - ret.nbDims = inputs[0].nbDims; - for (int di = 0; di < ret.nbDims; ++di) - { - ret.d[di] = inputs[0].d[di]; - } - // Sequence dimension or token dimension. - // Pad to multiple of 128. - auto dimM - = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 2], *exprBuilder.constant(128)); - ret.d[ret.nbDims - 2] = exprBuilder.operation(DimensionOperation::kPROD, *dimM, *exprBuilder.constant(128)); - // Hidden size dimension. - // Div (rounding up) by 16 since 16 elements share one SF and SF padded to k%4==0. - ret.d[ret.nbDims - 1] - = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 1], *exprBuilder.constant(16)); - return ret; - } - return DimsExprs{}; -} - -bool QuantizeToFP4Plugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - // half input + float global_sf + fp4 output (e2m1) + fp8 SF output. - int const totalPoses = 2 + 2; - TLLM_CHECK(0 <= pos && pos < totalPoses); - TLLM_CHECK(nbInputs == 2); - switch (pos) - { - case 0: - return (inOut[pos].type == nvinfer1::DataType::kHALF || inOut[pos].type == nvinfer1::DataType::kBF16 - || inOut[pos].type == nvinfer1::DataType::kFP8) - && (inOut[pos].format == TensorFormat::kLINEAR); - case 1: return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - case 2: return (inOut[pos].type == FP4_DTYPE) && (inOut[pos].format == TensorFormat::kLINEAR); - case 3: return (inOut[pos].type == FP8_DTYPE) && (inOut[pos].format == TensorFormat::kLINEAR); - default: break; - } - return false; -} - -void QuantizeToFP4Plugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t QuantizeToFP4Plugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -int QuantizeToFP4Plugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // input [M(*), N] half data type - // SF scale [1] float data type - // used to scale SF from input range to fp8 range (448.f / (MaxVal of input / 6.f)) - // outputs - // output [M(*), N] fp4 storage (E2M1) - // SF output [M, N / 16] fp8 storage (UE4M3) - - int64_t m64 = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims - 1; ++i) - { - m64 *= inputDesc[0].dims.d[i]; - } - int const m = TLLM_INT32_CAST(m64); - int const n = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); - - TLLM_CHECK_WITH_INFO(n % 16 == 0, "the N dimension must be multiple of 16."); - - float const* SFScale = static_cast<float const*>(inputs[1]); - int64_t* output = reinterpret_cast<int64_t*>(outputs[0]); - int32_t* SFoutput = reinterpret_cast<int32_t*>(outputs[1]); - - DataType inputDtype = inputDesc[0].type; - - switch (inputDtype) - { - case DataType::kHALF: - { - auto input = reinterpret_cast<half const*>(inputs[0]); - invokeFP4Quantization(1, m, n, input, SFScale, output, SFoutput, false, QuantizationSFLayout::SWIZZLED, - mMultiProcessorCount, stream); - break; - } - - case DataType::kBF16: - { - auto input = reinterpret_cast<__nv_bfloat16 const*>(inputs[0]); - invokeFP4Quantization(1, m, n, input, SFScale, output, SFoutput, false, QuantizationSFLayout::SWIZZLED, - mMultiProcessorCount, stream); - break; - } - - case DataType::kFP8: - { - auto input = reinterpret_cast<__nv_fp8_e4m3 const*>(inputs[0]); - invokeFP4Quantization(1, m, n, input, SFScale, output, SFoutput, false, QuantizationSFLayout::SWIZZLED, - mMultiProcessorCount, stream); - break; - } - - default: TLLM_LOG_ERROR("only half, bfloat16 and fp8 data type are supported."); break; - } - - // Use UE4M3 scales by default. - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType QuantizeToFP4Plugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - if (index == 0) - { - // Output 0 quantized output. - return FP4_DTYPE; - } - // Output 1 SF (scaling factors). - return FP8_DTYPE; -} - -// IPluginV2 Methods - -char const* QuantizeToFP4Plugin::getPluginType() const noexcept -{ - return QUANT_FP4_PLUGIN_NAME; -} - -char const* QuantizeToFP4Plugin::getPluginVersion() const noexcept -{ - return QUANT_FP4_PLUGIN_VERSION; -} - -int QuantizeToFP4Plugin::getNbOutputs() const noexcept -{ - return 2; -} - -int QuantizeToFP4Plugin::initialize() noexcept -{ - return 0; -} - -void QuantizeToFP4Plugin::terminate() noexcept {} - -size_t QuantizeToFP4Plugin::getSerializationSize() const noexcept -{ - return 0; -} - -void QuantizeToFP4Plugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - TLLM_CHECK(d == a + getSerializationSize()); -} - -void QuantizeToFP4Plugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -QuantizeToFP4PluginCreator::QuantizeToFP4PluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* QuantizeToFP4PluginCreator::getPluginName() const noexcept -{ - return QUANT_FP4_PLUGIN_NAME; -} - -char const* QuantizeToFP4PluginCreator::getPluginVersion() const noexcept -{ - return QUANT_FP4_PLUGIN_VERSION; -} - -PluginFieldCollection const* QuantizeToFP4PluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* QuantizeToFP4PluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - try - { - auto* obj = new QuantizeToFP4Plugin(); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* QuantizeToFP4PluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call QuantizeToFP4Plugin::destroy() - try - { - auto* obj = new QuantizeToFP4Plugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.h b/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.h deleted file mode 100644 index b584837a447a..000000000000 --- a/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.h +++ /dev/null @@ -1,90 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class QuantizeToFP4Plugin : public BasePlugin -{ -public: - QuantizeToFP4Plugin(); - - QuantizeToFP4Plugin(void const* data, size_t length); - - ~QuantizeToFP4Plugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - const std::string mLayerName; - int const mMultiProcessorCount = tensorrt_llm::common::getMultiProcessorCount(); -}; - -class QuantizeToFP4PluginCreator : public BaseCreator -{ -public: - QuantizeToFP4PluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/CMakeLists.txt deleted file mode 100755 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.cpp b/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.cpp deleted file mode 100644 index 16d0bf2dc356..000000000000 --- a/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.cpp +++ /dev/null @@ -1,452 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "rmsnormQuantizationPlugin.h" -#include "pluginUtils.h" -#include "tensorrt_llm/kernels/rmsnormKernels.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::RmsnormQuantizationPluginCreator; -using tensorrt_llm::plugins::RmsnormQuantizationPlugin; - -static char const* RMSNORM_QUANTIZATION_PLUGIN_VERSION{"1"}; -static char const* RMSNORM_QUANTIZATION_PLUGIN_NAME{"RmsnormQuantization"}; -PluginFieldCollection RmsnormQuantizationPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> RmsnormQuantizationPluginCreator::mPluginAttributes; - -RmsnormQuantizationPlugin::RmsnormQuantizationPlugin(float eps, bool dynamicActivationScaling, bool sumPerToken, - bool clampValEnabled, QuantMode quantMode, nvinfer1::DataType type, nvinfer1::DataType outputType) - : mEps(eps) - , mDynActScaling(dynamicActivationScaling) - , mType(type) - , mOutputType{outputType} - , mClampValEnabled{clampValEnabled} - , mQuantMode{quantMode} - , mSumPerToken(sumPerToken) -{ - TLLM_CHECK_WITH_INFO(mOutputType == nvinfer1::DataType::kINT8 || mOutputType == nvinfer1::DataType::kFP8, - "Only int8 or fp8 output type is allowed."); - // Check if the quant mode is valid. - TLLM_CHECK_WITH_INFO(mQuantMode.hasPerTokenScaling(), "The quant mode is not valid."); -} - -// Parameterized constructor -RmsnormQuantizationPlugin::RmsnormQuantizationPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mEps); - read(d, mDynActScaling); - read(d, mSumPerToken); - read(d, mClampValEnabled); - read(d, mQuantMode); - read(d, mType); - read(d, mOutputType); - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* RmsnormQuantizationPlugin::clone() const noexcept -{ - auto* plugin = new RmsnormQuantizationPlugin( - mEps, mDynActScaling, mSumPerToken, mClampValEnabled, mQuantMode, mType, mOutputType); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs RmsnormQuantizationPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - if (outputIndex == 0) - { - // Quantized output - return inputs[outputIndex]; - } - - // Dynamic scaling or per-token sum if enabled. - try - { - if (outputIndex == 1) - { - TLLM_CHECK(mDynActScaling); - } - else if (outputIndex == 2) - { - TLLM_CHECK(mSumPerToken); - } - else - { - TLLM_CHECK(false); - } - - DimsExprs ret; - ret.nbDims = inputs[0].nbDims; - for (int di = 0; di < ret.nbDims - 1; ++di) - { - ret.d[di] = inputs[0].d[di]; - } - ret.d[ret.nbDims - 1] = exprBuilder.constant(1); - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool RmsnormQuantizationPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - int const totalPoses - = 6 + static_cast<int>(mClampValEnabled) + static_cast<int>(mDynActScaling) + static_cast<int>(mSumPerToken); - TLLM_CHECK(0 <= pos && pos < totalPoses); - TLLM_CHECK(nbInputs == 4 + static_cast<int>(mClampValEnabled)); - if (pos < nbInputs) - { - if (pos < 3) - { - // activation, weight, bias - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == 3) - { - // scale - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == 4 && mClampValEnabled) - { - // clamp_max_v - return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; - } - } - else if (pos == 4 + int(mClampValEnabled)) - { - // Quantized output - return (inOut[pos].type == mOutputType) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == 5 + int(mClampValEnabled)) - { - // Dynamic scaling if enabled - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (pos == 6 + int(mClampValEnabled)) - { - // Per-token activation sum if enabled - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - - // Never should be here - TLLM_CHECK_WITH_INFO(false, "The input/output is not supported."); - return false; -} - -void RmsnormQuantizationPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t RmsnormQuantizationPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return 0; -} - -template <typename T, typename QuantT> -void RmsnormQuantizationPlugin::dispatchDataType(void* out, void const* input, void const* gamma, void const* beta, - float const eps, int const tokens, int const hidden_dim, cudaStream_t stream, void const* clampValPtr, - void const* scale, void* dynamic_scale, void* sum_per_token, void* normed_output_quant) noexcept -{ - // inputs - // activation [dim0(*), dim1] - // clamp_value [2], contains min val, and max val (optional) - // outputs - // quant [dim0(*), dim1] - // scale_tokens [dim0(*), 1] - - invokeGeneralRmsNorm(reinterpret_cast<T*>(out), reinterpret_cast<T const*>(input), - reinterpret_cast<T const*>(gamma), reinterpret_cast<T const*>(beta), eps, tokens, hidden_dim, mQuantMode, - stream, reinterpret_cast<float const*>(clampValPtr), reinterpret_cast<float const*>(scale), - reinterpret_cast<float*>(dynamic_scale), reinterpret_cast<float*>(sum_per_token), - reinterpret_cast<QuantT*>(normed_output_quant)); -} - -int RmsnormQuantizationPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // input [M(*), N] - // weight [N, ] - // bias [N, ] - // scale_to_int [1] - // clamp_value [2], contains min val, and max val (optional) - // outputs - // output [M(*), N] Normalized activations, potentially with quantization applied. - // dynamic_scaling [M(*), 1] (Optional) Per-token scales if quantization is enabled. - // token_sums [M(*), 1] (Optional) Per-token sums of all the channels (before quantization). - - int64_t m64 = 1; - for (int i = 0; i < inputDesc[0].dims.nbDims - 1; ++i) - { - m64 *= inputDesc[0].dims.d[i]; - } - int const m = TLLM_INT32_CAST(m64); - int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[0]); - - void const* input = inputs[0]; - void const* weight = inputs[1]; - void const* bias = inputs[2]; - void const* scale = inputs[3]; - void const* clampValPtr = mClampValEnabled ? inputs[4] : nullptr; - void* output = outputs[0]; - void* dynamic_scale = mDynActScaling ? outputs[1] : nullptr; - void* sum_per_token = mSumPerToken ? outputs[2] : nullptr; - - if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kINT8) - { - dispatchDataType<float, int8_t>( - nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#ifdef ENABLE_FP8 - else if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kFP8) - { - dispatchDataType<float, __nv_fp8_e4m3>( - nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#endif // ENABLE_FP8 - else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kINT8) - { - dispatchDataType<half, int8_t>( - nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#ifdef ENABLE_FP8 - else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kFP8) - { - dispatchDataType<half, __nv_fp8_e4m3>( - nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#endif // ENABLE_FP8 -#ifdef ENABLE_BF16 - else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kINT8) - { - dispatchDataType<__nv_bfloat16, int8_t>( - nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#ifdef ENABLE_FP8 - else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kFP8) - { - dispatchDataType<__nv_bfloat16, __nv_fp8_e4m3>( - nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); - } -#endif // ENABLE_FP8 -#endif // ENABLE_BF16 - sync_check_cuda_error(stream); - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType RmsnormQuantizationPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - assert(index <= 2); - - if (index == 0) - { - // Output 0 quantized output of layer norm - return mOutputType; - } - if (index == 1) - { - assert(mDynActScaling); - // Output 1 dynamic act scaling - return nvinfer1::DataType::kFLOAT; - } - // index == 2 - { - assert(mDynActScaling && mSumPerToken); - // Output 2 per token sum - return nvinfer1::DataType::kFLOAT; - } -} - -// IPluginV2 Methods - -char const* RmsnormQuantizationPlugin::getPluginType() const noexcept -{ - return RMSNORM_QUANTIZATION_PLUGIN_NAME; -} - -char const* RmsnormQuantizationPlugin::getPluginVersion() const noexcept -{ - return RMSNORM_QUANTIZATION_PLUGIN_VERSION; -} - -int RmsnormQuantizationPlugin::getNbOutputs() const noexcept -{ - return 1 + static_cast<int>(mDynActScaling) + static_cast<int>(mSumPerToken); -} - -int RmsnormQuantizationPlugin::initialize() noexcept -{ - return 0; -} - -void RmsnormQuantizationPlugin::terminate() noexcept {} - -size_t RmsnormQuantizationPlugin::getSerializationSize() const noexcept -{ - return sizeof(mOutputType) + sizeof(mClampValEnabled) + sizeof(mEps) + sizeof(mDynActScaling) + sizeof(mSumPerToken) - + sizeof(mType) + sizeof(mQuantMode); -} - -void RmsnormQuantizationPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mEps); - write(d, mDynActScaling); - write(d, mSumPerToken); - write(d, mClampValEnabled); - write(d, mQuantMode); - write(d, mType); - write(d, mOutputType); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void RmsnormQuantizationPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -RmsnormQuantizationPluginCreator::RmsnormQuantizationPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("eps", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("dyn_act_scaling", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("sum_per_token", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("clamp_enabled", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("quant_mode", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("out_type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* RmsnormQuantizationPluginCreator::getPluginName() const noexcept -{ - return RMSNORM_QUANTIZATION_PLUGIN_NAME; -} - -char const* RmsnormQuantizationPluginCreator::getPluginVersion() const noexcept -{ - return RMSNORM_QUANTIZATION_PLUGIN_VERSION; -} - -PluginFieldCollection const* RmsnormQuantizationPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* RmsnormQuantizationPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - nvinfer1::DataType outputType{}; - QuantMode quantMode; - bool clampValEnabled = false; - float eps{}; - nvinfer1::DataType type{}; - bool dynamicActivationScaling{}; - bool sumPerToken{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "quant_mode")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - quantMode = QuantMode(*(static_cast<int32_t const*>(fields[i].data))); - } - else if (!strcmp(attrName, "out_type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - outputType = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "clamp_enabled")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - clampValEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "eps")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - eps = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "dyn_act_scaling")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - dynamicActivationScaling = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "sum_per_token")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - sumPerToken = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - } - try - { - auto* obj = new RmsnormQuantizationPlugin( - eps, dynamicActivationScaling, sumPerToken, clampValEnabled, quantMode, type, outputType); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* RmsnormQuantizationPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call RmsnormQuantizationPlugin::destroy() - try - { - auto* obj = new RmsnormQuantizationPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.h b/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.h deleted file mode 100644 index 762a9bb8de1b..000000000000 --- a/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -class RmsnormQuantizationPlugin : public BasePlugin -{ -public: - RmsnormQuantizationPlugin(float eps, bool dynamicActivationScaling, bool sumPerToken, bool clampValEnabled, - tensorrt_llm::common::QuantMode quantMode, nvinfer1::DataType type, nvinfer1::DataType outputType); - - RmsnormQuantizationPlugin(void const* data, size_t length); - - ~RmsnormQuantizationPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - template <typename T, typename QuantT> - void dispatchDataType(void* out, void const* input, void const* gamma, void const* beta, float const eps, - int const tokens, int const hidden_dim, cudaStream_t stream, void const* clampValPtr, void const* scale, - void* dynamic_scale, void* normed_output_quant, void* act_sum) noexcept; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - float mEps; - bool mDynActScaling; - nvinfer1::DataType mType; - - std::string const mLayerName; - // The quantized output data type. - nvinfer1::DataType mOutputType; - // Do we clamp the input tensor ? - bool mClampValEnabled; - // The quantization mode. - tensorrt_llm::common::QuantMode mQuantMode; - // Should we output the sum of channels per-token? (Used by QServe GEMM) - bool mSumPerToken; -}; - -class RmsnormQuantizationPluginCreator : public BaseCreator -{ -public: - RmsnormQuantizationPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/selectiveScanPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/selectiveScanPlugin/CMakeLists.txt deleted file mode 100644 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/selectiveScanPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.cpp b/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.cpp deleted file mode 100644 index 3e60182f28c2..000000000000 --- a/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.cpp +++ /dev/null @@ -1,594 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "selectiveScanPlugin.h" -#include "tensorrt_llm/common/assert.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::SelectiveScanPluginCreator; -using tensorrt_llm::plugins::SelectiveScanPlugin; - -static char const* SELECTIVE_SCAN_PLUGIN_VERSION{"1"}; -static char const* SELECTIVE_SCAN_PLUGIN_NAME{"SelectiveScan"}; -PluginFieldCollection SelectiveScanPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> SelectiveScanPluginCreator::mPluginAttributes; - -SelectiveScanPlugin::SelectiveScanPlugin(int dim, int dstate, int dtRank, int nHeads, int nGroups, int chunkSize, - bool deltaSoftplus, nvinfer1::DataType type, bool removePadding, bool pagedState, bool zEnabled, bool isMamba2) - : mDim(dim) - , mDState(dstate) - , mDtRank(dtRank) - , mNHeads(nHeads) - , mNGroups(nGroups) - , mChunkSize(chunkSize) - , mDeltaSoftplus(deltaSoftplus) - , mType(type) - , mRemovePadding(removePadding) - , mPagedState(pagedState) - , mZEnabled(zEnabled) - , mIsMamba2(isMamba2) - , mDriver(tensorrt_llm::common::CUDADriverWrapper::getInstance()) -{ - TLLM_CHECK_WITH_INFO( - (mChunkSize == 256 || mChunkSize == 128) || (!mIsMamba2), "Only support CHUNK_SIZE 256 or 128"); - TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), - "Only support float, half, and bfloat16."); -} - -// Parameterized constructor -SelectiveScanPlugin::SelectiveScanPlugin(void const* data, size_t length) - : mDriver(tensorrt_llm::common::CUDADriverWrapper::getInstance()) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mDim); - read(d, mDState); - read(d, mDtRank); - read(d, mNHeads); - read(d, mNGroups); - read(d, mChunkSize); - read(d, mDeltaSoftplus); - read(d, mType); - read(d, mRemovePadding); - read(d, mPagedState); - read(d, mZEnabled); - read(d, mIsMamba2); - TLLM_CHECK(d == a + length); - TLLM_CHECK_WITH_INFO( - (mChunkSize == 256 || mChunkSize == 128) || (!mIsMamba2), "Only support CHUNK_SIZE 256 or 128"); - TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), - "Only support float, half, and bfloat16."); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* SelectiveScanPlugin::clone() const noexcept -{ - auto* plugin = new SelectiveScanPlugin(mDim, mDState, mDtRank, mNHeads, mNGroups, mChunkSize, mDeltaSoftplus, mType, - mRemovePadding, mPagedState, mZEnabled, mIsMamba2); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -// Outputs -// output_tensor: [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// state: [batch_size, dstate, dim] -nvinfer1::DimsExprs SelectiveScanPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - if (outputIndex == 0) - { - if (mIsMamba2) - { - auto ret = inputs[getInputTensorIdx()]; - ret.d[mRemovePadding ? 1 : 2] = exprBuilder.constant(mDim); - return ret; - } - else - { - return inputs[getInputTensorIdx()]; - } - } - return inputs[getStateIdx()]; -} - -bool SelectiveScanPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - if (pos == getHostRequestTypesIdx() || pos == getLastTokenIdsIdx() - || (mRemovePadding && pos == getHostContextLengthIdx()) || (mPagedState && pos == getSlotMappingIdx())) - { - return inOut[pos].type == nvinfer1::DataType::kINT32; - } - else if (pos == getAIdx() || pos == getDeltaBiasIdx() || pos == getDIdx()) - { - return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); - } - else if (mPagedState && pos == getStateIdx()) - { - return inOut[pos].type == nvinfer1::DataType::kINT64; - } - else - { - return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); - } -} - -void SelectiveScanPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t SelectiveScanPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - if (!mIsMamba2) - return 0; - - int const NUM_BUFFERS = 6; - size_t workspaces[NUM_BUFFERS]; - - if (mRemovePadding) - { - int B = inputs[getLastTokenIdsIdx()].dims.d[0]; - int BxL = inputs[getInputTensorIdx()].dims.d[0]; // num_tokens - int H = mNHeads; - int P = mDim / H; - int G = mNGroups; - int N = mDState; - int Q = mChunkSize; - int BxC = (BxL + Q - 1) / Q + B; - - workspaces[0] = long(BxC) * H * N * P * 2; // g_mxOs_ - workspaces[1] = long(BxC) * H * N * P * 4; // g_mxSt_ in float - workspaces[2] = long(BxC) * H * Q * 4; // g_mxdc_ in float - workspaces[3] = long(BxC) * H * Q * 4; // g_mxdA_ in float - workspaces[4] = long(BxC) * G * Q * Q * 2; // g_mxCB_ - workspaces[5] = 1024; // TMA descs - } - else - { - int B = inputs[getInputTensorIdx()].dims.d[0]; - int L = inputs[getInputTensorIdx()].dims.d[1]; - int H = mNHeads; - int P = mDim / H; - int G = mNGroups; - int N = mDState; - int Q = mChunkSize; - int C = (L + Q - 1) / Q; - - workspaces[0] = long(B * C) * H * N * P * 2; // g_mxOs_ - workspaces[1] = long(B * C) * H * N * P * 4; // g_mxSt_ in float - workspaces[2] = long(B * C) * H * Q * 4; // g_mxdc_ in float - workspaces[3] = long(B * C) * H * Q * 4; // g_mxdA_ in float - workspaces[4] = long(B * C) * G * Q * Q * 2; // g_mxCB_ - workspaces[5] = 1024; // TMA descs - } - - return calculateTotalWorkspaceSize(workspaces, NUM_BUFFERS); -} - -void SelectiveScanPlugin::setSSMParams(SSMParamsBase& params, const size_t batch, const size_t dim, - const size_t maxSeqLen, const size_t numTokens, const size_t dstate, const size_t dtRank, const size_t nHeads, - const size_t nGroups, const size_t chunkSize, void* statePtr, void const* x, void const* delta, - void const* deltaBias, void const* A, void const* BC, void const* D, void const* z, void* osPtr, void* stPtr, - void* dcPtr, void* dAPtr, void* cbPtr, void* descPtr, int const* lastTokenIds, int const* slotMapping, void* out, - bool deltaSoftplus, bool removePadding) -{ - // Reset the parameters - memset(¶ms, 0, sizeof(params)); - - params.batch = batch; - params.dim = dim; - params.max_seqlen = maxSeqLen; - params.num_tokens = numTokens; - params.dstate = dstate; - params.dt_rank = dtRank; - params.nheads = nHeads; - params.ngroups = nGroups; - params.chunk_size = chunkSize; - - params.delta_softplus = deltaSoftplus; - params.remove_padding = removePadding; - params.is_mamba2 = mIsMamba2; - - // Set the pointers and strides. - params.u_ptr = const_cast<void*>(x); - params.delta_ptr = const_cast<void*>(delta); - params.A_ptr = const_cast<void*>(A); - params.BC_ptr = const_cast<void*>(BC); - params.D_ptr = const_cast<void*>(D); - params.delta_bias_ptr = const_cast<void*>(deltaBias); - params.out_ptr = out; - params.x_ptr = statePtr; - params.z_ptr = const_cast<void*>(z); - params.Os_ptr = osPtr; - params.St_ptr = stPtr; - params.dc_ptr = dcPtr; - params.dA_ptr = dAPtr; - params.CB_ptr = cbPtr; - params.desc_ptr = descPtr; - params.last_token_ids_ptr = lastTokenIds; - params.slot_mapping_ptr = slotMapping; -} - -template <typename T> -int SelectiveScanPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) -{ - // inputs - // 0. input_tensor [batch_size, max_seq_len, dim] or [num_tokens, dim] - // 1. state mamba: [batch_size, dstate, dim] or host [1] containing only pointer for paged_state - // mamba2: [batch_size, nheads, dstate, dim] or host [1] containing only pointer for paged_state - // 2. delta, mamba: [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding - // mamba2: [batch_size, seq_len, nheads] or [num_tokens, nheads] for remove_input_padding - // 3. delta_bias, [dim] for mamba, [nheads] for mamba2 - // 4. A, [dstate, dim] for mamba, [nheads] for mamba2 - // 5. BC, mamba: [batch_size, seq_len, dstate * 2] or [num_tokens, dstate * 2] for remove_input_padding - // mamba2: [batch_size, seq_len, ngroups * dstate * 2] or [num_tokens, ngroups * dstate * 2] for - // remove_input_padding - // 6. D, [dim] for mamba, [nheads] for mamba2 - // 7. host_request_types [batch_size] int32. 0: context; 1: generation. - // 8. last_token_ids [batch_size] int32 - // 9. host_context_lengths [batch_size] int32, optional for remove_input_padding - // 10. state_slot_mapping [batch_size] int32, optional for paged state - // 11. z [batch_size, max_seq_len, dim] or [num_tokens, dim] - // outputs - // 0. output_tensor [batch_size, max_seq_len, dim] or [num_tokens, dim] - // 1. state, [batch_size, dstate, dim] for mamba, [batch_size, nheads, dstate, dim] for mamba2 - auto const batch_size = inputDesc[getHostRequestTypesIdx()].dims.d[0]; - int max_seq_len; - if (mRemovePadding) - { - int const* host_context_length = static_cast<int const*>(inputs[getHostContextLengthIdx()]); - max_seq_len = *std::max_element(host_context_length, host_context_length + batch_size); - } - else - { - max_seq_len = inputDesc[getInputTensorIdx()].dims.d[1]; - } - - // only support context or generation, not for both of them - RequestType const* reqTypes = static_cast<RequestType const*>(inputs[getHostRequestTypesIdx()]); - - SSMParamsBase ssm_params; - - int const* slotMapping = mPagedState ? static_cast<int const*>(inputs[getSlotMappingIdx()]) : nullptr; - void const* z = mZEnabled ? inputs[getZIdx()] : nullptr; - - void* statePtr = mPagedState ? *reinterpret_cast<void**>(const_cast<void*>(inputs[getStateIdx()])) : outputs[1]; - - // Workspace pointer shift - int8_t* workspace_byte_ptr = reinterpret_cast<int8_t*>(workspace); - size_t offset = 0; - - T* mxOs = nullptr; - float* mxSt = nullptr; - float* mxdc = nullptr; - float* mxdA = nullptr; - T* mxCB = nullptr; - void* descs = nullptr; - - if (!mIsMamba2 || reqTypes[0] == RequestType::kGENERATION) /* no workspace needed */ - ; - else if (mRemovePadding) - { - int B = inputDesc[getLastTokenIdsIdx()].dims.d[0]; - int BxL = inputDesc[getInputTensorIdx()].dims.d[0]; // num_tokens - int H = mNHeads; - int P = mDim / H; - int G = mNGroups; - int N = mDState; - int Q = mChunkSize; - int BxC = (BxL + Q - 1) / Q + B; - - mxOs = reinterpret_cast<T*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(BxC) * H * N * P * 2)); - mxSt = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(BxC) * H * N * P * 4)); - mxdc = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(BxC) * H * Q * 4)); - mxdA = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(BxC) * H * Q * 4)); - mxCB = reinterpret_cast<T*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(BxC) * G * Q * Q * 2)); - descs = nextWorkspacePtr(workspace_byte_ptr, offset, 1024); - } - else - { - int B = inputDesc[getInputTensorIdx()].dims.d[0]; - int L = inputDesc[getInputTensorIdx()].dims.d[1]; - int H = mNHeads; - int P = mDim / H; - int G = mNGroups; - int N = mDState; - int Q = mChunkSize; - int C = (L + Q - 1) / Q; - - mxOs = reinterpret_cast<T*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(B * C) * H * N * P * 2)); - mxSt = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(B * C) * H * N * P * 4)); - mxdc = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(B * C) * H * Q * 4)); - mxdA = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(B * C) * H * Q * 4)); - mxCB = reinterpret_cast<T*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(B * C) * G * Q * Q * 2)); - descs = nextWorkspacePtr(workspace_byte_ptr, offset, 1024); - } - - int numTokens = inputDesc[getInputTensorIdx()].dims.d[0]; - if (!mRemovePadding) - numTokens *= inputDesc[getInputTensorIdx()].dims.d[1]; - - setSSMParams(ssm_params, batch_size, mDim, max_seq_len, numTokens, mDState, mDtRank, mNHeads, mNGroups, mChunkSize, - statePtr, inputs[getInputTensorIdx()], inputs[getDeltaIdx()], inputs[getDeltaBiasIdx()], inputs[getAIdx()], - inputs[getBCIdx()], inputs[getDIdx()], z, mxOs, mxSt, mxdc, mxdA, mxCB, descs, - static_cast<int const*>(inputs[getLastTokenIdsIdx()]), slotMapping, outputs[0], mDeltaSoftplus, mRemovePadding); - - if (reqTypes[0] == RequestType::kCONTEXT) - { - if (mIsMamba2) - { - invokeChunkScan<T, float>(ssm_params, stream, mDriver.get()); - } - else - { - invokeSelectiveScan<T, float>(ssm_params, stream); - } - } - else if (reqTypes[0] == RequestType::kGENERATION) - { - invokeSelectiveScanUpdate<T, float>(ssm_params, stream); - } - sync_check_cuda_error(stream); - return 0; -} - -int SelectiveScanPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - if (isBuilding()) - { - return 0; - } - if (mType == DataType::kHALF) - { - return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (mType == DataType::kFLOAT) - { - return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#ifdef ENABLE_BF16 - else if (mType == DataType::kBF16) - { - return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#endif - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType SelectiveScanPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - if (index == 0) - { - return inputTypes[getInputTensorIdx()]; - } - else - { - return inputTypes[getStateIdx()]; - } -} - -// IPluginV2 Methods - -char const* SelectiveScanPlugin::getPluginType() const noexcept -{ - return SELECTIVE_SCAN_PLUGIN_NAME; -} - -char const* SelectiveScanPlugin::getPluginVersion() const noexcept -{ - return SELECTIVE_SCAN_PLUGIN_VERSION; -} - -int SelectiveScanPlugin::getNbOutputs() const noexcept -{ - return mPagedState ? 1 : 2; -} - -int SelectiveScanPlugin::initialize() noexcept -{ - return 0; -} - -void SelectiveScanPlugin::terminate() noexcept {} - -size_t SelectiveScanPlugin::getSerializationSize() const noexcept -{ - return sizeof(mDim) + sizeof(mDState) + sizeof(mDtRank) + sizeof(mNHeads) + sizeof(mNGroups) + sizeof(mChunkSize) - + sizeof(mDeltaSoftplus) + sizeof(mType) + sizeof(mRemovePadding) + sizeof(mPagedState) + sizeof(mZEnabled) - + sizeof(mIsMamba2); -} - -void SelectiveScanPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mDim); - write(d, mDState); - write(d, mDtRank); - write(d, mNHeads); - write(d, mNGroups); - write(d, mChunkSize); - write(d, mDeltaSoftplus); - write(d, mType); - write(d, mRemovePadding); - write(d, mPagedState); - write(d, mZEnabled); - write(d, mIsMamba2); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void SelectiveScanPlugin::destroy() noexcept -{ - delete this; -} - -/////////////// - -SelectiveScanPluginCreator::SelectiveScanPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("dim", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("dstate", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("dt_rank", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("nheads", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("ngroups", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("chunk_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("delta_softplus", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("remove_input_padding", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("paged_state", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("z_enabled", nullptr, PluginFieldType::kINT8)); - mPluginAttributes.emplace_back(PluginField("is_mamba2", nullptr, PluginFieldType::kINT8)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* SelectiveScanPluginCreator::getPluginName() const noexcept -{ - return SELECTIVE_SCAN_PLUGIN_NAME; -} - -char const* SelectiveScanPluginCreator::getPluginVersion() const noexcept -{ - return SELECTIVE_SCAN_PLUGIN_VERSION; -} - -PluginFieldCollection const* SelectiveScanPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* SelectiveScanPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int dim{}; - int dstate{}; - int dtRank{}; - int nHeads{}; - int nGroups{}; - int chunkSize{}; - bool deltaSoftplus{}; - bool removePadding{}; - bool pagedState{}; - bool zEnabled{}; - bool isMamab2{}; - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "dim")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - dim = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "dstate")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - dstate = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "dt_rank")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - dtRank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "nheads")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - nHeads = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "ngroups")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - nGroups = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "chunk_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - chunkSize = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "delta_softplus")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - deltaSoftplus = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "remove_input_padding")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - removePadding = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "paged_state")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - pagedState = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "z_enabled")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - zEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - else if (!strcmp(attrName, "is_mamba2")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); - isMamab2 = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); - } - } - try - { - auto* obj = new SelectiveScanPlugin(dim, dstate, dtRank, nHeads, nGroups, chunkSize, deltaSoftplus, type, - removePadding, pagedState, zEnabled, isMamab2); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* SelectiveScanPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call SelectiveScanPlugin::destroy() - try - { - auto* obj = new SelectiveScanPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.h b/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.h deleted file mode 100644 index 96cb86fc4cbb..000000000000 --- a/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.h +++ /dev/null @@ -1,218 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TRT_SELECTIVE_SCAN_PLUGIN_H -#define TRT_SELECTIVE_SCAN_PLUGIN_H -#include "tensorrt_llm/kernels/selectiveScan/selectiveScan.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> - -namespace tensorrt_llm::plugins -{ -// batch_size = num_ctx_requests or num_gen_requests -// num_ctx_requests = number of context requests (single sequence per request). -// num_gen_requests = number of generation requests (single sequences per request). -// can not support beam search - -// inputs -// 0. input_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// 1. state, mamba: [batch_size, dstate, dim] or host [1] containing only pointer for paged_state -// mamba2: [batch_size, nheads, dstate, dim] or host [1] containing only pointer for paged_state -// 2. delta, mamba: [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// mamba2: [batch_size, seq_len, nheads] or [num_tokens, nheads] for remove_input_padding -// 3. delta_bias, [dim] for mamba, [nheads] for mamba2 -// 4. A, [dstate, dim] for mamba, [nheads] for mamba2 -// 5. BC, mamba: [batch_size, seq_len, dstate * 2] or [num_tokens, dstate * 2] for remove_input_padding -// mamba2: [batch_size, seq_len, ngroups * dstate * 2] or [num_tokens, ngroups * dstate * 2] for -// remove_input_padding -// 6. D, [dim] for mamba, [nheads] for mamba2 -// 7. host_request_types [batch_size] int32. 0: context; 1: generation; 2: none. -// 8. last_token_ids [batch_size] int32 -// 9. host_context_lengths [batch_size] int32, optional for remove_input_padding -// 10. state_slot_mapping [batch_size] int32, optional for paged state -// 11. z [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// outputs -// 0. output_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding -// 1. state, [batch_size, dstate, dim] for mamba, [batch_size, nheads, dstate, dim] for mamba2 - -class SelectiveScanPlugin : public BasePlugin -{ -public: - SelectiveScanPlugin(int dim, int dstate, int dtRank, int nHeads, int nGroups, int chunkSize, bool deltaSoftplus, - nvinfer1::DataType type, bool removePadding, bool pagedState, bool zEnabled, bool isMamba2); - - SelectiveScanPlugin(void const* data, size_t length); - - ~SelectiveScanPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - template <typename T> - int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - - enum class RequestType : int32_t - { - kCONTEXT = 0, - kGENERATION = 1 - }; - -private: - using IndexType = std::int32_t; - - IndexType getInputTensorIdx() const - { - return 0; - }; - - IndexType getStateIdx() const - { - return 1; - }; - - IndexType getDeltaIdx() const - { - return 2; - }; - - IndexType getDeltaBiasIdx() const - { - return 3; - }; - - IndexType getAIdx() const - { - return 4; - }; - - IndexType getBCIdx() const - { - return 5; - }; - - IndexType getDIdx() const - { - return 6; - }; - - IndexType getHostRequestTypesIdx() const - { - return 7; - }; - - IndexType getLastTokenIdsIdx() const - { - return 8; - }; - - IndexType getHostContextLengthIdx() const - { - if (mRemovePadding) - return 9; - else - return 8; - }; - - IndexType getSlotMappingIdx() const - { - if (mPagedState) - return getHostContextLengthIdx() + 1; - else - return getHostContextLengthIdx(); - }; - - IndexType getZIdx() const - { - if (mZEnabled) - return getSlotMappingIdx() + 1; - else - return getSlotMappingIdx(); - }; - - void setSSMParams(tensorrt_llm::kernels::SSMParamsBase& params, - // sizes - const size_t batch, const size_t dim, const size_t maxSeqLen, const size_t numTokens, const size_t dstate, - const size_t dtRank, const size_t nHeads, const size_t nGroups, const size_t chunkSize, - // device pointers - void* statePtr, void const* x, void const* delta, void const* deltaBias, void const* A, void const* BC, - void const* D, void const* z, void* osPtr, void* stPtr, void* dcPtr, void* dAPtr, void* cbPtr, void* descs, - int const* lastTokenIds, int const* slotMapping, void* out, bool deltaSoftplus, bool removePadding); - -private: - int mDim; - int mDState; - int mDtRank; - int mNHeads; - int mNGroups; - int mChunkSize; - bool mDeltaSoftplus; - nvinfer1::DataType mType; - bool mRemovePadding = false; - bool mPagedState = false; - bool mZEnabled = true; - bool mIsMamba2 = false; - std::shared_ptr<tensorrt_llm::common::CUDADriverWrapper> mDriver; -}; - -class SelectiveScanPluginCreator : public BaseCreator -{ -public: - SelectiveScanPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins - -#endif // TRT_SELECTIVE_SCAN_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/CMakeLists.txt deleted file mode 100755 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.cpp b/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.cpp deleted file mode 100644 index 718d8b7e830d..000000000000 --- a/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.cpp +++ /dev/null @@ -1,431 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "smoothQuantGemmPlugin.h" -#include "tensorrt_llm/kernels/weightOnlyBatchedGemv/int8SQ.h" -#include <numeric> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using namespace tensorrt_llm::kernels::cutlass_kernels; -using tensorrt_llm::plugins::SmoothQuantGemmPluginCreator; -using tensorrt_llm::plugins::SmoothQuantGemmPlugin; -using tensorrt_llm::plugins::SmoothQuantGemmPluginProfiler; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; - -static char const* SQ_GEMM_PLUGIN_VERSION{"1"}; -static char const* SQ_GEMM_PLUGIN_NAME{"SmoothQuantGemm"}; -PluginFieldCollection SmoothQuantGemmPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> SmoothQuantGemmPluginCreator::mPluginAttributes; - -void SmoothQuantGemmPluginProfiler::runTactic(int m, int n, int k, SmoothQuantGemmPluginProfiler::Config const& tactic, - char* workspace, cudaStream_t const& stream) -{ - int8_t* aTmp = reinterpret_cast<int8_t*>(workspace); - int8_t* bTmp = nextWorkspacePtr(aTmp, m * k * sizeof(int8_t)); - void* cTmp = reinterpret_cast<void*>(nextWorkspacePtr(bTmp, n * k * sizeof(int8_t))); - float* alphaRowTmp = reinterpret_cast<float*>( - nextWorkspacePtr(reinterpret_cast<int8_t*>(cTmp), m * n * (mType == nvinfer1::DataType::kFLOAT ? 4 : 2))); - float* alphaColTmp - = reinterpret_cast<float*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(alphaRowTmp), m * sizeof(float))); - char* workspaceTmp - = reinterpret_cast<char*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(alphaColTmp), n * sizeof(float))); - - int const wsSize = mRunner->getWorkspaceSize(m, n, k); - - mRunner->gemm( - aTmp, bTmp, mQuantMode, alphaColTmp, alphaRowTmp, cTmp, m, n, k, tactic, workspaceTmp, wsSize, stream); -} - -void SmoothQuantGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - std::vector<size_t> workspaces = { - maxM * k * sizeof(int8_t), // A - n * k * sizeof(int8_t), // B - maxM * n * (mType == nvinfer1::DataType::kFLOAT ? 4u : 2u), // C - maxM * sizeof(float), // alphaRow - n * sizeof(float), // alphaCol - mRunner->getWorkspaceSize(maxM, n, k) // workspace - }; - size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); - setTmpWorkspaceSizeInBytes(bytes); -} - -std::vector<SmoothQuantGemmPluginProfiler::Config> SmoothQuantGemmPluginProfiler::getTactics(int m, int n, int k) const -{ - return mRunner->getConfigs(); -} - -SmoothQuantGemmPlugin::SmoothQuantGemmPlugin( - QuantMode quantMode, nvinfer1::DataType type, SmoothQuantGemmPlugin::PluginProfilerPtr const& pluginProfiler) - : mQuantMode(quantMode) - , mPluginProfiler(pluginProfiler) -{ - init(type); -} - -// Parameterized constructor -SmoothQuantGemmPlugin::SmoothQuantGemmPlugin( - void const* data, size_t length, SmoothQuantGemmPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - nvinfer1::DataType type; - unsigned int quantMode; - read(d, quantMode); - read(d, type); - read(d, mDims); - - mQuantMode = QuantMode(quantMode); - - init(type); - - mPluginProfiler->deserialize(d, mDims, mGemmId); - - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -void SmoothQuantGemmPlugin::init(nvinfer1::DataType type) -{ - mType = type; - if (mType == nvinfer1::DataType::kHALF) - { - m_sqGemmRunner = std::make_shared<CutlassInt8GemmRunner<half>>(); - } - else if (mType == nvinfer1::DataType::kFLOAT) - { - m_sqGemmRunner = std::make_shared<CutlassInt8GemmRunner<float>>(); - } - else if (mType == nvinfer1::DataType::kINT32) - { - m_sqGemmRunner = std::make_shared<CutlassInt8GemmRunner<int32_t>>(); - } -#ifdef ENABLE_BF16 - else if (mType == nvinfer1::DataType::kBF16) - { - m_sqGemmRunner = std::make_shared<CutlassInt8GemmRunner<__nv_bfloat16>>(); - } -#endif - - mPluginProfiler->setQuantMode(mQuantMode); - - mGemmId = GemmIdCore(mDims.n, mDims.k, mType); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* SmoothQuantGemmPlugin::clone() const noexcept -{ - auto* plugin = new SmoothQuantGemmPlugin(*this); - return plugin; -} - -nvinfer1::DimsExprs SmoothQuantGemmPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - try - { - TLLM_CHECK(nbInputs == 4); - TLLM_CHECK(outputIndex == 0); - int const nbDimsA = inputs[0].nbDims; - TLLM_CHECK(nbDimsA >= 2); - DimsExprs ret; - ret.nbDims = nbDimsA; - for (int ii = 0; ii < nbDimsA - 1; ++ii) - { - ret.d[ii] = inputs[0].d[ii]; - } - ret.d[nbDimsA - 1] = inputs[1].d[0]; - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool SmoothQuantGemmPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - switch (pos) - { - case 0: - // activation - return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; - case 1: - // weights - // Weights stored in checkpoint must have int8 type - return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; - case 2: - // scales channels - case 3: - // scales tokens - return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; - case 4: - // out - return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - default: - // Never should be here - assert(false); - return false; - } -} - -void SmoothQuantGemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); - auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); - - int const maxK = in[0].max.d[in[0].max.nbDims - 1]; - int const maxN = in[1].max.d[0]; - int const minK = in[0].min.d[in[0].min.nbDims - 1]; - int const minN = in[1].min.d[0]; - - TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); - TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, maxN, maxK}; - } - mGemmId = {maxN, maxK, mType}; - - m_workspaceMaxSize = m_sqGemmRunner->getWorkspaceSize(maxM, maxN, maxK); -} - -size_t SmoothQuantGemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return m_workspaceMaxSize; -} - -int SmoothQuantGemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // mat1 [M(*), K] - // mat2 [N, K] - // scale_tokens [M, 1] if has_per_token_scaling else [1, 1] - // scale_channels [1, N] if has_per_channel_scaling else [1, 1] - // outputs - // mat [M(*), N] - int64_t m64 = 1; - for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) - { - m64 *= inputDesc[0].dims.d[ii]; - } - int const m = TLLM_INT32_CAST(m64); - int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[0]); - int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); - int const wsSize = m_sqGemmRunner->getWorkspaceSize(m, n, k); - if (m <= 4) - { - tensorrt_llm::kernels::smooth_quant::Params params(reinterpret_cast<int8_t const*>(inputs[0]), - reinterpret_cast<int8_t const*>(inputs[1]), reinterpret_cast<float const*>(inputs[2]), - reinterpret_cast<float const*>(inputs[3]), reinterpret_cast<void*>(outputs[0]), m, n, k, mQuantMode); - if (mType == nvinfer1::DataType::kHALF) - { - tensorrt_llm::kernels::smooth_quant::int8_sq_launcher<half>(params, stream); - } - else if (mType == nvinfer1::DataType::kFLOAT) - { - tensorrt_llm::kernels::smooth_quant::int8_sq_launcher<float>(params, stream); - } -#ifdef ENABLE_BF16 - else if (mType == nvinfer1::DataType::kBF16) - { - tensorrt_llm::kernels::smooth_quant::int8_sq_launcher<__nv_bfloat16>(params, stream); - } -#endif - else if (mType == nvinfer1::DataType::kINT32) - { - tensorrt_llm::kernels::smooth_quant::int8_sq_launcher<int>(params, stream); - } - } - else - { - auto const& bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); - TLLM_CHECK_WITH_INFO(bestTactic, "No valid SQ GEMM tactic"); - m_sqGemmRunner->gemm(reinterpret_cast<int8_t const*>(inputs[0]), reinterpret_cast<int8_t const*>(inputs[1]), - mQuantMode, reinterpret_cast<float const*>(inputs[3]), reinterpret_cast<float const*>(inputs[2]), - reinterpret_cast<void*>(outputs[0]), m, n, k, *bestTactic, reinterpret_cast<char*>(workspace), wsSize, - stream); - } - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType SmoothQuantGemmPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* SmoothQuantGemmPlugin::getPluginType() const noexcept -{ - return SQ_GEMM_PLUGIN_NAME; -} - -char const* SmoothQuantGemmPlugin::getPluginVersion() const noexcept -{ - return SQ_GEMM_PLUGIN_VERSION; -} - -int SmoothQuantGemmPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int SmoothQuantGemmPlugin::initialize() noexcept -{ - configGemm(); - return 0; -} - -void SmoothQuantGemmPlugin::terminate() noexcept {} - -size_t SmoothQuantGemmPlugin::getSerializationSize() const noexcept -{ - return sizeof(unsigned int) + // QuantMode - sizeof(nvinfer1::DataType) + // dtype - sizeof(mDims) + // Dimensions - mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size -} - -void SmoothQuantGemmPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mQuantMode.value()); - write(d, mType); - write(d, mDims); - - mPluginProfiler->serialize(d, mGemmId); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void SmoothQuantGemmPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -void SmoothQuantGemmPlugin::configGemm() -{ - mPluginProfiler->profileTactics(m_sqGemmRunner, mType, mDims, mGemmId); -} - -/////////////// - -SmoothQuantGemmPluginCreator::SmoothQuantGemmPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("has_per_channel_scaling", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("has_per_token_scaling", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* SmoothQuantGemmPluginCreator::getPluginName() const noexcept -{ - return SQ_GEMM_PLUGIN_NAME; -} - -char const* SmoothQuantGemmPluginCreator::getPluginVersion() const noexcept -{ - return SQ_GEMM_PLUGIN_VERSION; -} - -PluginFieldCollection const* SmoothQuantGemmPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* SmoothQuantGemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - bool perTokenScaling{}; - bool perChannelScaling{}; - nvinfer1::DataType type{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "has_per_channel_scaling")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - perChannelScaling = static_cast<bool>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "has_per_token_scaling")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - perTokenScaling = static_cast<bool>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - try - { - // SmoothQuantGemmPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); - QuantMode quantMode = QuantMode::fromDescription(true, true, perTokenScaling, perChannelScaling, false, false, - false, false, false, false, false, false, false, false, false, false); - auto* obj = new SmoothQuantGemmPlugin(quantMode, type, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* SmoothQuantGemmPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call SmoothQuantGemmPlugin::destroy() - try - { - // Create plugin profiler with private tactics map which is read from the serialized engine - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); - auto* obj = new SmoothQuantGemmPlugin(serialData, serialLength, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.h b/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.h deleted file mode 100644 index 3cabf558076b..000000000000 --- a/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/kernels/cutlass_kernels/int8_gemm/int8_gemm.h" -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::plugins -{ - -using perfMapType = std::unordered_map<int, tensorrt_llm::cutlass_extensions::CutlassGemmConfig>; -using SqGemmRunnerPtr = std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassInt8GemmRunnerInterface>; - -class SmoothQuantGemmPluginProfiler : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - SqGemmRunnerPtr, GemmIdCore, GemmIdCoreHash> -{ -public: - using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; - - void setQuantMode(tensorrt_llm::common::QuantMode const& quantMode) - { - mQuantMode = quantMode; - } - -protected: - void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; - - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - - std::vector<Config> getTactics(int m, int n, int k) const override; - -private: - tensorrt_llm::common::QuantMode mQuantMode; -}; - -class SmoothQuantGemmPlugin : public BasePlugin -{ -public: - using PluginProfilerPtr = std::shared_ptr<SmoothQuantGemmPluginProfiler>; - - SmoothQuantGemmPlugin() = delete; - - SmoothQuantGemmPlugin( - tensorrt_llm::common::QuantMode quantMode, nvinfer1::DataType type, PluginProfilerPtr const& pluginProfiler); - - SmoothQuantGemmPlugin(void const* data, size_t length, PluginProfilerPtr const& pluginProfiler); - - ~SmoothQuantGemmPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - void init(nvinfer1::DataType type); - - void configGemm(); - -private: - const std::string mLayerName; - - SqGemmRunnerPtr m_sqGemmRunner; - tensorrt_llm::common::QuantMode mQuantMode; - size_t m_workspaceMaxSize; - - GemmDims mDims{}; - GemmIdCore mGemmId{}; - - PluginProfilerPtr mPluginProfiler; - - nvinfer1::DataType mType; -}; - -class SmoothQuantGemmPluginCreator : public BaseCreator -{ -public: - SmoothQuantGemmPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - GemmPluginProfilerManager<SmoothQuantGemmPluginProfiler> gemmPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/topkLastDimPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/topkLastDimPlugin/CMakeLists.txt deleted file mode 100644 index 6b4e3d8d9e0f..000000000000 --- a/cpp/tensorrt_llm/plugins/topkLastDimPlugin/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# - -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.cpp b/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.cpp deleted file mode 100644 index 072bfc9c8fc4..000000000000 --- a/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.cpp +++ /dev/null @@ -1,316 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "topkLastDimPlugin.h" -#include "tensorrt_llm/common/assert.h" - -using namespace nvinfer1; -using namespace tensorrt_llm::kernels; -using namespace tensorrt_llm::common; -using tensorrt_llm::plugins::TopkLastDimPluginCreator; -using tensorrt_llm::plugins::TopkLastDimPlugin; - -static char const* TOPK_LAST_DIM_PLUGIN_VERSION{"1"}; -static char const* TOPK_LAST_DIM_PLUGIN_NAME{"TopkLastDim"}; -PluginFieldCollection TopkLastDimPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> TopkLastDimPluginCreator::mPluginAttributes; - -TopkLastDimPlugin::TopkLastDimPlugin(nvinfer1::DataType type, int32_t k, bool is_largest) - : mType(type) - , mK(k) // To avoid data-dependent shape, enforce K to be non-dynamic - , mIsLargest(is_largest) -{ - TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF) - || (mType == DataType::kINT32), - "Only support int, float, half, and bfloat16."); -} - -// Parameterized constructor -TopkLastDimPlugin::TopkLastDimPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - read(d, mType); - read(d, mK); - read(d, mIsLargest); - TLLM_CHECK(d == a + length); - TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF) - || (mType == DataType::kINT32), - "Only support int, float, half, and bfloat16."); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* TopkLastDimPlugin::clone() const noexcept -{ - auto* plugin = new TopkLastDimPlugin(mType, mK, mIsLargest); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -// Outputs -// out_val or out_idx: [batch_size, K] -nvinfer1::DimsExprs TopkLastDimPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - TLLM_CHECK_WITH_INFO(outputIndex < 2, "Only 2 outputs."); - nvinfer1::DimsExprs output(inputs[0]); - int numDim = output.nbDims; - output.d[numDim - 1] = exprBuilder.constant(mK); - return output; -} - -bool TopkLastDimPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - bool res = inOut[pos].format == TensorFormat::kLINEAR; - if (pos < 2) // input and out_val tensor must be the same type as the plugin - { - res = res && inOut[pos].type == mType; - } - else if (pos == 2) // out_idx must be int32 - { - res = res && inOut[pos].type == DataType::kINT32; - } - return res; -} - -void TopkLastDimPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t TopkLastDimPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - // extract shape info and then call helper - auto const batchSize = inputs[getInputTensorIdx()].dims.d[0]; - auto const inputLength = inputs[getInputTensorIdx()].dims.d[1]; - size_t tempStorageBytes{}; - if (mType == DataType::kINT32) - { - tempStorageBytes = invokeComputeTopkLastDimWorkspaceSize<int>(batchSize, inputLength, mK, mIsLargest); - } - else if (mType == DataType::kHALF) - { - tempStorageBytes = invokeComputeTopkLastDimWorkspaceSize<half>(batchSize, inputLength, mK, mIsLargest); - } - else if (mType == DataType::kFLOAT) - { - tempStorageBytes = invokeComputeTopkLastDimWorkspaceSize<float>(batchSize, inputLength, mK, mIsLargest); - } -#ifdef ENABLE_BF16 - else if (mType == DataType::kBF16) - { - tempStorageBytes = invokeComputeTopkLastDimWorkspaceSize<__nv_bfloat16>(batchSize, inputLength, mK, mIsLargest); - } -#endif - return tempStorageBytes; -} - -template <typename T> -int TopkLastDimPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) -{ - // inputs - // 0. input_tensor [batch_size, inputLength] - // outputs - // 0. output_values [batch_size, k] - // 1. output_indices [batch_size, k] - auto const batchSize = inputDesc[getInputTensorIdx()].dims.d[0]; - auto const inputLength = inputDesc[getInputTensorIdx()].dims.d[1]; - if (batchSize == 0) - { - // nothing to do for empty tensor - return 0; - } - - invokeTopkLastDim<T>( - batchSize, inputLength, mK, mIsLargest, inputs[getInputTensorIdx()], outputs[0], outputs[1], workspace, stream); - - sync_check_cuda_error(stream); - return 0; -} - -int TopkLastDimPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - if (mType == DataType::kINT32) - { - return enqueueImpl<int>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (mType == DataType::kHALF) - { - return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (mType == DataType::kFLOAT) - { - return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#ifdef ENABLE_BF16 - else if (mType == DataType::kBF16) - { - return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } -#endif - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType TopkLastDimPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK_WITH_INFO(index < 2, "Only 2 outputs."); - nvinfer1::DataType data_type; - if (index == 1) - { - data_type = DataType::kINT32; - } - else - { - data_type = inputTypes[getInputTensorIdx()]; - } - return data_type; -} - -// IPluginV2 Methods - -char const* TopkLastDimPlugin::getPluginType() const noexcept -{ - return TOPK_LAST_DIM_PLUGIN_NAME; -} - -char const* TopkLastDimPlugin::getPluginVersion() const noexcept -{ - return TOPK_LAST_DIM_PLUGIN_VERSION; -} - -int TopkLastDimPlugin::getNbOutputs() const noexcept -{ - return 2; -} - -int TopkLastDimPlugin::initialize() noexcept -{ - return 0; -} - -void TopkLastDimPlugin::terminate() noexcept {} - -size_t TopkLastDimPlugin::getSerializationSize() const noexcept -{ - return sizeof(mType) + sizeof(mK) + sizeof(mIsLargest); -} - -void TopkLastDimPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - write(d, mK); - write(d, mIsLargest); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void TopkLastDimPlugin::destroy() noexcept -{ - delete this; -} - -/////////////// - -TopkLastDimPluginCreator::TopkLastDimPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("k", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("is_largest", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* TopkLastDimPluginCreator::getPluginName() const noexcept -{ - return TOPK_LAST_DIM_PLUGIN_NAME; -} - -char const* TopkLastDimPluginCreator::getPluginVersion() const noexcept -{ - return TOPK_LAST_DIM_PLUGIN_VERSION; -} - -PluginFieldCollection const* TopkLastDimPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* TopkLastDimPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - nvinfer1::DataType type{}; - int32_t k{}; - bool is_largest{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "k")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - k = static_cast<int32_t>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "is_largest")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - is_largest = static_cast<int32_t>(*(static_cast<int const*>(fields[i].data))) != 0; - } - } - try - { - auto* obj = new TopkLastDimPlugin(type, k, is_largest); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* TopkLastDimPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call TopkLastDimPlugin::destroy() - try - { - auto* obj = new TopkLastDimPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.h b/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.h deleted file mode 100644 index 0ca38ccfe105..000000000000 --- a/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.h +++ /dev/null @@ -1,99 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TRT_TOPK_LAST_DIM_PLUGIN_H -#define TRT_TOPK_LAST_DIM_PLUGIN_H - -#include "tensorrt_llm/kernels/topkLastDim.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include <cassert> - -namespace tensorrt_llm::plugins -{ -class TopkLastDimPlugin : public BasePlugin -{ -public: - TopkLastDimPlugin(nvinfer1::DataType type, int32_t k, bool largest); - TopkLastDimPlugin(void const* data, size_t length); - ~TopkLastDimPlugin() override = default; - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - template <typename T> - int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - using IndexType = std::int32_t; - - IndexType getInputTensorIdx() const - { - return 0; - }; - -private: - nvinfer1::DataType mType; - int32_t mK; - bool mIsLargest; -}; - -class TopkLastDimPluginCreator : public BaseCreator -{ -public: - TopkLastDimPluginCreator(); - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins - -#endif diff --git a/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/CMakeLists.txt deleted file mode 100755 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.cpp b/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.cpp deleted file mode 100644 index 85f0cf011293..000000000000 --- a/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.cpp +++ /dev/null @@ -1,657 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "weightOnlyGroupwiseQuantMatmulPlugin.h" - -#include <numeric> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using namespace tensorrt_llm::kernels::cutlass_kernels; -using tensorrt_llm::plugins::WeightOnlyGroupwiseQuantMatmulPluginCreator; -using tensorrt_llm::plugins::WeightOnlyGroupwiseQuantMatmulPlugin; -using tensorrt_llm::plugins::WeightOnlyGroupwiseQuantGemmPluginProfiler; -using tensorrt_llm::plugins::WeightOnlyGemmRunnerPtr; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; - -static char const* WOQ_GROUPWISE_MATMUL_PLUGIN_VERSION{"1"}; -static char const* WOQ_GROUPWISE_MATMUL_PLUGIN_NAME{"WeightOnlyGroupwiseQuantMatmul"}; -PluginFieldCollection WeightOnlyGroupwiseQuantMatmulPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> WeightOnlyGroupwiseQuantMatmulPluginCreator::mPluginAttributes; - -void WeightOnlyGroupwiseQuantGemmPluginProfiler::runTactic(int m, int n, int k, - WeightOnlyGroupwiseQuantGemmPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) -{ - // Quantized weights are packed in FP16 format (INT4*4 -> FP16, INT8*2 -> FP16) - int const originalN = mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT ? n * FP16_INT8_RATIO : n * FP16_INT4_RATIO; - half* actPtr = reinterpret_cast<half*>(workspace); - void* weightPtr = nextWorkspacePtr(reinterpret_cast<int8_t*>(actPtr), m * k * sizeof(half)); - half* inputScalesPtr - = reinterpret_cast<half*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(weightPtr), n * k * sizeof(float))); - half* zerosPtr = reinterpret_cast<half*>( - nextWorkspacePtr(reinterpret_cast<int8_t*>(inputScalesPtr), k * originalN * sizeof(half) / mGroupSize)); - half* biasesPtr = reinterpret_cast<half*>( - nextWorkspacePtr(reinterpret_cast<int8_t*>(zerosPtr), k * originalN * sizeof(half) / mGroupSize)); - half* outputPtr = reinterpret_cast<half*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(biasesPtr), n * sizeof(half))); - char* workspacePtr - = reinterpret_cast<char*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(outputPtr), m * originalN * sizeof(half))); - if ((mQuantAlgo & GroupwiseQuantAlgo::ZERO) == 0) - { - zerosPtr = nullptr; - } - if ((mQuantAlgo & GroupwiseQuantAlgo::BIAS) == 0) - { - biasesPtr = nullptr; - } - - if (tactic.enableCudaKernel) - { - // run CUDA kernel - void const* pre_quant_scale_ptr = nullptr; - bool apply_alpha_in_advance = false; - float alpha = 1.0; - tensorrt_llm::kernels::weight_only::Params params{actPtr, pre_quant_scale_ptr, weightPtr, inputScalesPtr, - zerosPtr, biasesPtr, outputPtr, alpha, m, originalN, k, mGroupSize, mCudaKernelType, - apply_alpha_in_advance}; - tensorrt_llm::kernels::weight_only::kernel_launcher(mArch, params, stream); - } - else - { - // run CUTLASS kernel - int const wsSize = mRunner->getWorkspaceSize(m, originalN, k); - if (mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT) - { - mRunner->gemm(actPtr, reinterpret_cast<int8_t*>(weightPtr), inputScalesPtr, zerosPtr, biasesPtr, outputPtr, - m, originalN, k, mGroupSize, tactic, workspacePtr, wsSize, stream); - } - else - { - mRunner->gemm(actPtr, reinterpret_cast<cutlass::uint4b_t*>(weightPtr), inputScalesPtr, zerosPtr, biasesPtr, - outputPtr, m, originalN, k, mGroupSize, tactic, workspacePtr, wsSize, stream); - } - } -} - -void WeightOnlyGroupwiseQuantGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - // Quantized weights are packed in FP16 format (INT4*4 -> FP16, INT8*2 -> FP16) - int const originalN = mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT ? n * FP16_INT8_RATIO : n * FP16_INT4_RATIO; - std::vector<size_t> workspaces = { - maxM * k * sizeof(half), // A - k * n * sizeof(float), // B - k * originalN * sizeof(half) / mGroupSize, // scales - k * originalN * sizeof(half) / mGroupSize, // zeros - originalN * sizeof(half), // biases - maxM * originalN * sizeof(half), // C - mRunner->getWorkspaceSize(maxM, originalN, k) // workspace - }; - size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); - setTmpWorkspaceSizeInBytes(bytes); -} - -std::vector<WeightOnlyGroupwiseQuantGemmPluginProfiler::Config> WeightOnlyGroupwiseQuantGemmPluginProfiler::getTactics( - int m, int n, int k) const -{ - return mRunner->getConfigs(); -} - -bool WeightOnlyGroupwiseQuantGemmPluginProfiler::checkTactic(int m, int n, int k, Config const& tactic) const -{ - // stop to profile Cuda kernel for m >= 16 - if (tactic.enableCudaKernel) - { - return m < 16; - } - return true; -} - -WeightOnlyGroupwiseQuantMatmulPlugin::WeightOnlyGroupwiseQuantMatmulPlugin(nvinfer1::DataType type, int quant_algo, - int group_size, float alpha, WeightOnlyGroupwiseQuantMatmulPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - init(type, quant_algo, group_size, alpha); -} - -// Parameterized constructor -WeightOnlyGroupwiseQuantMatmulPlugin::WeightOnlyGroupwiseQuantMatmulPlugin( - void const* data, size_t length, WeightOnlyGroupwiseQuantMatmulPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - nvinfer1::DataType type; - int quant_algo = 0; - int group_size = 0; - float alpha = 1.0f; - read(d, type); - read(d, quant_algo); - read(d, group_size); - read(d, alpha); - read(d, mDims); - - init(type, quant_algo, group_size, alpha); - - mPluginProfiler->deserialize(d, mDims, mGemmId); - - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -template <typename ActivationType, typename WeightType, typename OutputType, typename ScaleZeroType, - cutlass::WeightOnlyQuantOp QuantOp> -using GemmRunner = tensorrt_llm::kernels::cutlass_kernels::CutlassFpAIntBGemmRunner<ActivationType, WeightType, QuantOp, - ScaleZeroType, OutputType, OutputType>; - -template <typename ActivationType, typename WeightType, typename OutputType, typename ScaleZeroType = OutputType> -WeightOnlyGemmRunnerPtr selectGemmRunnerForZERO(int quant_algo) -{ - if (quant_algo & GroupwiseQuantAlgo::ZERO) - { - return std::make_shared<GemmRunner<ActivationType, WeightType, OutputType, ScaleZeroType, - cutlass::WeightOnlyQuantOp::FINEGRAINED_SCALE_AND_ZEROS>>(); - } - else - { - return std::make_shared<GemmRunner<ActivationType, WeightType, OutputType, ScaleZeroType, - cutlass::WeightOnlyQuantOp::FINEGRAINED_SCALE_ONLY>>(); - } -} - -template <typename ActivationType> -WeightOnlyGemmRunnerPtr selectGemmRunnerForWeightType(int quant_algo) -{ - if (quant_algo & GroupwiseQuantAlgo::INT8_WEIGHT) - { - return selectGemmRunnerForZERO<ActivationType, uint8_t, ActivationType>(quant_algo); - } - else - { - return selectGemmRunnerForZERO<ActivationType, cutlass::uint4b_t, ActivationType>(quant_algo); - } -} - -void WeightOnlyGroupwiseQuantMatmulPlugin::init(nvinfer1::DataType type, int quant_algo, int group_size, float alpha) -{ - mArch = tensorrt_llm::common::getSMVersion(); - mType = type; - mQuantAlgo = quant_algo; - mGroupSize = group_size; - - // quant_algo = int8_weight * 16 + fp8_alpha * 8 + pre_quant_scale * 4 + zero * 2 + bias - mPreQuantScaleInputIdx = (quant_algo & GroupwiseQuantAlgo::PRE_QUANT_SCALE) ? 1 : 0; - mWeightInputIdx = mPreQuantScaleInputIdx + 1; - mScalesInputIdx = mWeightInputIdx + 1; - mZerosInputIdx = (quant_algo & GroupwiseQuantAlgo::ZERO) ? mScalesInputIdx + 1 : mScalesInputIdx; - mBiasesInputIdx = (quant_algo & GroupwiseQuantAlgo::BIAS) ? mZerosInputIdx + 1 : mZerosInputIdx; - - if (mType == nvinfer1::DataType::kHALF) - { - // CUTLASS kernel selection - if (quant_algo & GroupwiseQuantAlgo::FP8_ALPHA) - { - mAlpha = alpha; - - // Ada & Hopper style kernels - if (mArch < 89) - { - TLLM_THROW("W4A(fp)8 kernel is unsupported on pre-Ada (sm<89) architectures!"); - } - assert(!(quant_algo & GroupwiseQuantAlgo::INT8_WEIGHT) && "W4A(fp)8 kernel requires INT4 weight!"); - m_weightOnlyGroupwiseGemmRunner - = selectGemmRunnerForZERO<__nv_fp8_e4m3, cutlass::uint4b_t, half>(quant_algo); - } - else - { - m_weightOnlyGroupwiseGemmRunner = selectGemmRunnerForWeightType<half>(quant_algo); - } - // CUDA kernel selection - if (quant_algo & GroupwiseQuantAlgo::INT8_WEIGHT) - { - // INT8 weight - mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( - mArch, tensorrt_llm::kernels::weight_only::KernelType::FP16Int8Groupwise); - mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::FP16Int8Groupwise; - } - else - { - // INT4 weight - mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( - mArch, tensorrt_llm::kernels::weight_only::KernelType::FP16Int4Groupwise); - mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::FP16Int4Groupwise; - } - } -#if defined(ENABLE_BF16) - else if (mType == nvinfer1::DataType::kBF16) - { - // CUTLASS kernel selection - if (quant_algo & GroupwiseQuantAlgo::FP8_ALPHA) - { - mAlpha = alpha; - - // FP8 requires at least sm89 devices - if (mArch < 89) - { - TLLM_THROW("W4A(fp)8 kernel is unsupported on pre-Ada (sm<89) architectures!"); - } - assert(!(quant_algo & GroupwiseQuantAlgo::INT8_WEIGHT) && "W4A(fp)8 kernel requires INT4 weight!"); - m_weightOnlyGroupwiseGemmRunner - = selectGemmRunnerForZERO<__nv_fp8_e4m3, cutlass::uint4b_t, __nv_bfloat16, half>(quant_algo); - } - else - { - m_weightOnlyGroupwiseGemmRunner = selectGemmRunnerForWeightType<__nv_bfloat16>(quant_algo); - } - // CUDA kernel selection - if (quant_algo & GroupwiseQuantAlgo::INT8_WEIGHT) - { - // INT8 weight - mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( - mArch, tensorrt_llm::kernels::weight_only::KernelType::BF16Int8Groupwise); - mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::BF16Int8Groupwise; - } - else - { - // INT4 weight - mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( - mArch, tensorrt_llm::kernels::weight_only::KernelType::BF16Int4Groupwise); - mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::BF16Int4Groupwise; - } - } -#endif - else - { - TLLM_THROW("Unsupported data type"); - } - mPluginProfiler->setQuantAlgo(mQuantAlgo); - mPluginProfiler->setGroupSize(mGroupSize); - if (mCudaKernelEnabled) - { - mPluginProfiler->setCudaKernelType(mCudaKernelType, mArch); - } - mGemmId = GemmIdCore(mDims.n, mDims.k, mType); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* WeightOnlyGroupwiseQuantMatmulPlugin::clone() const noexcept -{ - auto* plugin = new WeightOnlyGroupwiseQuantMatmulPlugin(*this); - return plugin; -} - -void WeightOnlyGroupwiseQuantMatmulPlugin::configGemm() -{ - mPluginProfiler->profileTactics(m_weightOnlyGroupwiseGemmRunner, mType, mDims, mGemmId, mCudaKernelEnabled); -} - -nvinfer1::DimsExprs WeightOnlyGroupwiseQuantMatmulPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - - // inputs - // 0 activations [M, K] - // 1 pre-quant scales [K] (optional) - // 2 weights [K, N/2] - // 3 scales [K // group_size, N] - // 4 zeros [K // group_size, N] (optional) - // 5 biases [N] (optional) - - try - { - TLLM_CHECK(nbInputs == mBiasesInputIdx + 1); - TLLM_CHECK(outputIndex == 0); - int const nbDimsA = inputs[0].nbDims; - int const nbDimsB = inputs[mWeightInputIdx].nbDims; - TLLM_CHECK(nbDimsA >= 2); - TLLM_CHECK(nbDimsB == 2); - DimsExprs ret; - ret.nbDims = nbDimsA; - for (int ii = 0; ii < nbDimsA - 1; ++ii) - { - ret.d[ii] = inputs[0].d[ii]; - } - - // int4/int8 weight only quant (INT4*4 -> FP16, INT8*2 -> FP16) - int const weight_multiplier = mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT ? FP16_INT8_RATIO : FP16_INT4_RATIO; - ret.d[nbDimsA - 1] = exprBuilder.constant(inputs[mWeightInputIdx].d[1]->getConstantValue() * weight_multiplier); - - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool WeightOnlyGroupwiseQuantMatmulPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - if (pos < nbInputs + 1) - { - return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - } - else - { - // Never should be here - assert(false); - return false; - } -} - -void WeightOnlyGroupwiseQuantMatmulPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); - auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); - int const maxK = in[0].max.d[in[0].max.nbDims - 1]; - - // Quantized weights are packed in FP16 format (INT4*4 -> FP16, INT8*2 -> FP16) - int const weight_multiplier = mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT ? FP16_INT8_RATIO : FP16_INT4_RATIO; - int const maxN = in[mWeightInputIdx].max.d[1] * weight_multiplier; - - auto const K = maxK; - auto const N = maxN / weight_multiplier; - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, N, K}; - } - mGemmId = {N, K, mType}; - - size_t smoothedActSize = static_cast<size_t>(maxM) * static_cast<size_t>(maxK) - * (in[0].desc.type == nvinfer1::DataType::kFLOAT ? sizeof(float) : sizeof(half)); - m_workspaceMaxSize = smoothedActSize + m_weightOnlyGroupwiseGemmRunner->getWorkspaceSize(maxM, maxN, maxK); -} - -size_t WeightOnlyGroupwiseQuantMatmulPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return m_workspaceMaxSize; -} - -template <typename ActType> -void pre_quant_scale_for_act(int const m, int const k, int const mQuantAlgo, int const mPreQuantScaleInputIdx, - void const* const* inputs, void* workspace, cudaStream_t stream) -{ - // Apply pre-quant per channel scale on activations - if (mQuantAlgo & GroupwiseQuantAlgo::FP8_ALPHA) - { - tensorrt_llm::kernels::apply_per_channel_scale_kernel_launcher<ActType, __nv_fp8_e4m3>( - reinterpret_cast<__nv_fp8_e4m3*>(workspace), reinterpret_cast<ActType const*>(inputs[0]), - reinterpret_cast<ActType const*>(inputs[mPreQuantScaleInputIdx]), m, k, nullptr, stream); - } - else - { - tensorrt_llm::kernels::apply_per_channel_scale_kernel_launcher<ActType, ActType>( - reinterpret_cast<ActType*>(workspace), reinterpret_cast<ActType const*>(inputs[0]), - reinterpret_cast<ActType const*>(inputs[mPreQuantScaleInputIdx]), m, k, nullptr, stream); - } -} - -int WeightOnlyGroupwiseQuantMatmulPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // 0 activations [M, K] - // 1 pre-quant scales [K] - // 2 weights [K, N/2] - // 3 scales [K // group_size, N] - // 4 zeros [K // group_size, N] - // 5 biases [N] - // outputs - // mat [M, N] - - int64_t m64 = 1; - for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) - { - m64 *= inputDesc[0].dims.d[ii]; - } - int const m = TLLM_INT32_CAST(m64); - int const n = TLLM_INT32_CAST(inputDesc[mWeightInputIdx].dims.d[1]); - int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); - - // get best tactic and check if CUDA kernel should be used - bool use_cuda_kernel = false; - auto const& bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); - TLLM_CHECK_WITH_INFO(bestTactic, - "No valid weight only groupwise GEMM tactic(It is usually caused by the failure to execute all " - "candidate configurations of the CUTLASS kernel, please pay attention to the warning information " - "when building the engine.)"); - use_cuda_kernel = bestTactic->enableCudaKernel; - - bool use_pre_quant_scale = mQuantAlgo & GroupwiseQuantAlgo::PRE_QUANT_SCALE; - half const* zeros_ptr - = (mQuantAlgo & GroupwiseQuantAlgo::ZERO) ? reinterpret_cast<half const*>(inputs[mZerosInputIdx]) : nullptr; - half const* biases_ptr - = (mQuantAlgo & GroupwiseQuantAlgo::BIAS) ? reinterpret_cast<half const*>(inputs[mBiasesInputIdx]) : nullptr; - half const* act_ptr = reinterpret_cast<half const*>(inputs[0]); - - if (use_pre_quant_scale && !use_cuda_kernel) - { - // Apply pre-quant per channel scale on activations - act_ptr = reinterpret_cast<half const*>(workspace); - if (mType == nvinfer1::DataType::kHALF) - { - pre_quant_scale_for_act<half>(m, k, mQuantAlgo, mPreQuantScaleInputIdx, inputs, workspace, stream); - } -#if defined(ENABLE_BF16) - else if (mType == nvinfer1::DataType::kBF16) - { - pre_quant_scale_for_act<__nv_bfloat16>(m, k, mQuantAlgo, mPreQuantScaleInputIdx, inputs, workspace, stream); - } -#endif - } - -#if defined(ENABLE_BF16) - TLLM_CHECK_WITH_INFO(mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kBF16, - "No valid weightOnlyGropwiseQuantMatmul configuration"); -#else - TLLM_CHECK_WITH_INFO(mType == nvinfer1::DataType::kHALF, "No valid weightOnlyGropwiseQuantMatmul configuration"); -#endif - - // Quantized weights are packed in FP16 format (INT4*4 -> FP16, INT8*2 -> FP16) - int real_n = mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT ? n * FP16_INT8_RATIO : n * FP16_INT4_RATIO; - - if (use_cuda_kernel) - { - // Apply CUDA kernel - void const* pre_quant_scale_ptr = nullptr; - if (use_pre_quant_scale) - pre_quant_scale_ptr = inputs[mPreQuantScaleInputIdx]; - void const* cuda_kernel_act_ptr = inputs[0]; - void const* cuda_kernel_weight_ptr = inputs[mWeightInputIdx]; - void const* cuda_kernel_scales_ptr = inputs[mScalesInputIdx]; - void* cuda_kernel_out_ptr = outputs[0]; - tensorrt_llm::kernels::weight_only::Params params{cuda_kernel_act_ptr, pre_quant_scale_ptr, - cuda_kernel_weight_ptr, cuda_kernel_scales_ptr, zeros_ptr, biases_ptr, cuda_kernel_out_ptr, mAlpha, m, - real_n, k, mGroupSize, mCudaKernelType, static_cast<bool>(mQuantAlgo & GroupwiseQuantAlgo::FP8_ALPHA)}; - tensorrt_llm::kernels::weight_only::kernel_launcher(mArch, params, stream); - } - else - { - // Apply CUTLASS kernel - int const ws_bytes = m_weightOnlyGroupwiseGemmRunner->getWorkspaceSize(m, real_n, k); - int32_t* weight_ptr = const_cast<int32_t*>(reinterpret_cast<int32_t const*>(inputs[mWeightInputIdx])); - m_weightOnlyGroupwiseGemmRunner->gemm(act_ptr, weight_ptr, inputs[mScalesInputIdx], zeros_ptr, biases_ptr, - mAlpha, outputs[0], m, real_n, k, mGroupSize, *bestTactic, - reinterpret_cast<char*>(workspace) + m * k * sizeof(half), ws_bytes, stream); - } - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType WeightOnlyGroupwiseQuantMatmulPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* WeightOnlyGroupwiseQuantMatmulPlugin::getPluginType() const noexcept -{ - return WOQ_GROUPWISE_MATMUL_PLUGIN_NAME; -} - -char const* WeightOnlyGroupwiseQuantMatmulPlugin::getPluginVersion() const noexcept -{ - return WOQ_GROUPWISE_MATMUL_PLUGIN_VERSION; -} - -int WeightOnlyGroupwiseQuantMatmulPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int WeightOnlyGroupwiseQuantMatmulPlugin::initialize() noexcept -{ - configGemm(); - return 0; -} - -void WeightOnlyGroupwiseQuantMatmulPlugin::terminate() noexcept {} - -size_t WeightOnlyGroupwiseQuantMatmulPlugin::getSerializationSize() const noexcept -{ - return sizeof(nvinfer1::DataType) + // mType - sizeof(int) + // mQuantAlgo - sizeof(int) + // mGroupSize - sizeof(float) + // mAlpha - sizeof(mDims) + // Dimensions - mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size -} - -void WeightOnlyGroupwiseQuantMatmulPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - write(d, mQuantAlgo); - write(d, mGroupSize); - write(d, mAlpha); - write(d, mDims); - - mPluginProfiler->serialize(d, mGemmId); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void WeightOnlyGroupwiseQuantMatmulPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -WeightOnlyGroupwiseQuantMatmulPluginCreator::WeightOnlyGroupwiseQuantMatmulPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("quant_algo", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("group_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("alpha", nullptr, PluginFieldType::kFLOAT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* WeightOnlyGroupwiseQuantMatmulPluginCreator::getPluginName() const noexcept -{ - return WOQ_GROUPWISE_MATMUL_PLUGIN_NAME; -} - -char const* WeightOnlyGroupwiseQuantMatmulPluginCreator::getPluginVersion() const noexcept -{ - return WOQ_GROUPWISE_MATMUL_PLUGIN_VERSION; -} - -PluginFieldCollection const* WeightOnlyGroupwiseQuantMatmulPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* WeightOnlyGroupwiseQuantMatmulPluginCreator::createPlugin( - char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - nvinfer1::DataType type{}; - int QuantAlgo{}; - int GroupSize{}; - float Alpha{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "quant_algo")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - QuantAlgo = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "group_size")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - GroupSize = static_cast<int>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - else if (!strcmp(attrName, "alpha")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); - Alpha = static_cast<float>(*(static_cast<float const*>(fields[i].data))); - } - } - try - { - // WeightOnlyGroupwiseQuantMatmulPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); - auto* obj = new WeightOnlyGroupwiseQuantMatmulPlugin(type, QuantAlgo, GroupSize, Alpha, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* WeightOnlyGroupwiseQuantMatmulPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call weightOnlyGroupwiseQuantMatmulPlugin::destroy() - try - { - // Create plugin profiler with private tactics map which is read from the serialized engine - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); - auto* obj = new WeightOnlyGroupwiseQuantMatmulPlugin(serialData, serialLength, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.h b/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.h deleted file mode 100644 index 94e98ce0f5c0..000000000000 --- a/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.h +++ /dev/null @@ -1,186 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/kernels/cutlass_kernels/fpA_intB_gemm/fpA_intB_gemm.h" -#include "tensorrt_llm/kernels/preQuantScaleKernel.h" -#include "tensorrt_llm/kernels/weightOnlyBatchedGemv//kernelLauncher.h" -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include "tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.h" - -#include <cutlass/numeric_types.h> - -#include <cassert> -#include <cuda_runtime.h> -#include <memory> -#include <set> -#include <string> -#include <vector> - -// The blank line here is to avoid clang-format -sort-includes option reordering these two cutlass header files and -// breaking dependencies -#include "cutlass/integer_subbyte.h" - -namespace tensorrt_llm::plugins -{ - -using WeightOnlyGemmRunner = tensorrt_llm::kernels::cutlass_kernels::CutlassFpAIntBGemmRunnerInterface; -using WeightOnlyGemmRunnerPtr = std::shared_ptr<WeightOnlyGemmRunner>; -using KernelType = tensorrt_llm::kernels::weight_only::KernelType; - -class WeightOnlyGroupwiseQuantGemmPluginProfiler - : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, WeightOnlyGemmRunnerPtr, - GemmIdCore, GemmIdCoreHash> -{ -public: - using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; - - void setQuantAlgo(int quantAlgo) - { - mQuantAlgo = quantAlgo; - } - - void setGroupSize(int groupSize) - { - mGroupSize = groupSize; - } - - void setCudaKernelType(KernelType cudaKernelType, int arch) - { - mCudaKernelType = cudaKernelType; - mArch = arch; - } - -protected: - void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; - - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - - std::vector<Config> getTactics(int m, int n, int k) const override; - - bool checkTactic(int m, int n, int k, Config const& tactic) const override; - -private: - int mQuantAlgo; - int mGroupSize; - KernelType mCudaKernelType; - int mArch; -}; - -class WeightOnlyGroupwiseQuantMatmulPlugin : public BasePlugin -{ -public: - using PluginProfilerPtr = std::shared_ptr<WeightOnlyGroupwiseQuantGemmPluginProfiler>; - - WeightOnlyGroupwiseQuantMatmulPlugin() = delete; - - WeightOnlyGroupwiseQuantMatmulPlugin( - nvinfer1::DataType type, int quant_algo, int group_size, float alpha, PluginProfilerPtr const& profiler); - - WeightOnlyGroupwiseQuantMatmulPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); - - ~WeightOnlyGroupwiseQuantMatmulPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - // group_size: 64, 128 - void init(nvinfer1::DataType type, int quant_algo, int group_size, float alpha); - - void configGemm(); - -private: - const std::string mLayerName; - - WeightOnlyGemmRunnerPtr m_weightOnlyGroupwiseGemmRunner; - size_t m_workspaceMaxSize; - nvinfer1::DataType mType; - bool mCudaKernelEnabled; - tensorrt_llm::kernels::weight_only::KernelType mCudaKernelType; - int mArch; - - // When M is smaller than this value, we trigger a fast path - // I.e. a tailored kernel instead of cutlass. - - int mQuantAlgo; - - int mGroupSize; - - float mAlpha = 1.0f; - - int mPreQuantScaleInputIdx; - int mWeightInputIdx; - int mScalesInputIdx; - int mZerosInputIdx; - int mBiasesInputIdx; - - GemmDims mDims{}; - GemmIdCore mGemmId{}; - - PluginProfilerPtr mPluginProfiler; -}; - -class WeightOnlyGroupwiseQuantMatmulPluginCreator : public BaseCreator -{ -public: - WeightOnlyGroupwiseQuantMatmulPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - GemmPluginProfilerManager<WeightOnlyGroupwiseQuantGemmPluginProfiler> gemmPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/CMakeLists.txt deleted file mode 100755 index 86876224fccd..000000000000 --- a/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -file(GLOB SRCS *.cpp) -set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) -set(PLUGIN_SOURCES - ${PLUGIN_SOURCES} - PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.cpp b/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.cpp deleted file mode 100644 index f3ed07fafaff..000000000000 --- a/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.cpp +++ /dev/null @@ -1,507 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & - * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "weightOnlyQuantMatmulPlugin.h" - -#include <numeric> - -using namespace nvinfer1; -using namespace tensorrt_llm::common; -using namespace tensorrt_llm::kernels::cutlass_kernels; -using tensorrt_llm::plugins::WeightOnlyQuantMatmulPluginCreator; -using tensorrt_llm::plugins::WeightOnlyQuantMatmulPlugin; -using tensorrt_llm::plugins::WeightOnlyQuantGemmPluginProfiler; -using tensorrt_llm::plugins::read; -using tensorrt_llm::plugins::write; - -static char const* WOQ_MATMUL_PLUGIN_VERSION{"1"}; -static char const* WOQ_MATMUL_PLUGIN_NAME{"WeightOnlyQuantMatmul"}; -PluginFieldCollection WeightOnlyQuantMatmulPluginCreator::mFC{}; -std::vector<nvinfer1::PluginField> WeightOnlyQuantMatmulPluginCreator::mPluginAttributes; - -void WeightOnlyQuantGemmPluginProfiler::runTactic(int m, int n, int k, - WeightOnlyQuantGemmPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) -{ - int const originalN = n * getWeightTypeMultiplier(mWeightTypeId); - half* actPtr = reinterpret_cast<half*>(workspace); - int8_t* weightPtr - = reinterpret_cast<int8_t*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(actPtr), m * k * sizeof(half))); - half* scalesPtr - = reinterpret_cast<half*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(weightPtr), n * k * sizeof(int8_t))); - half* outputPtr - = reinterpret_cast<half*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(scalesPtr), originalN * sizeof(half))); - char* workspacePtr - = reinterpret_cast<char*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(outputPtr), m * originalN * sizeof(half))); - - int const wsSize = mRunner->getWorkspaceSize(m, originalN, k); - - if (tactic.enableCudaKernel) - { - // run CUDA kernel - tensorrt_llm::kernels::weight_only::Params params{actPtr, nullptr, weightPtr, scalesPtr, nullptr, nullptr, - outputPtr, 1.f, m, originalN, k, 0, mCudaKernelType}; - tensorrt_llm::kernels::weight_only::kernel_launcher(mArch, params, stream); - } - else - { - // run CUTLASS kernel - if (mWeightTypeId == WeightTypeId::INT8) - { - mRunner->gemm( - actPtr, weightPtr, scalesPtr, outputPtr, m, originalN, k, tactic, workspacePtr, wsSize, stream); - } - else - { - mRunner->gemm(actPtr, reinterpret_cast<cutlass::uint4b_t*>(weightPtr), scalesPtr, outputPtr, m, originalN, - k, tactic, workspacePtr, wsSize, stream); - } - } -} - -void WeightOnlyQuantGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) -{ - int const originalN = n * getWeightTypeMultiplier(mWeightTypeId); - std::vector<size_t> workspaces = { - maxM * k * sizeof(half), // A - n * k * sizeof(int8_t), // B - originalN * sizeof(half), // scales - maxM * originalN * sizeof(half), // C - mRunner->getWorkspaceSize(maxM, originalN, k) // workspace - }; - size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); - setTmpWorkspaceSizeInBytes(bytes); -} - -std::vector<WeightOnlyQuantGemmPluginProfiler::Config> WeightOnlyQuantGemmPluginProfiler::getTactics( - int m, int n, int k) const -{ - return mRunner->getConfigs(); -} - -bool WeightOnlyQuantGemmPluginProfiler::checkTactic(int m, int n, int k, Config const& tactic) const -{ - // stop to profile Cuda kernel for m >= 16 - if (tactic.enableCudaKernel) - { - return m < 16; - } - return true; -} - -WeightOnlyQuantMatmulPlugin::WeightOnlyQuantMatmulPlugin(nvinfer1::DataType type, WeightTypeId weightTypeId, - WeightOnlyQuantMatmulPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - init(type, weightTypeId); -} - -// Parameterized constructor -WeightOnlyQuantMatmulPlugin::WeightOnlyQuantMatmulPlugin( - void const* data, size_t length, WeightOnlyQuantMatmulPlugin::PluginProfilerPtr const& pluginProfiler) - : mPluginProfiler(pluginProfiler) -{ - char const *d = reinterpret_cast<char const*>(data), *a = d; - nvinfer1::DataType type; - WeightTypeId weightTypeId; - read(d, type); - read(d, weightTypeId); - read(d, mDims); - - init(type, weightTypeId); - - mPluginProfiler->deserialize(d, mDims, mGemmId); - - TLLM_CHECK_WITH_INFO(d == a + length, - "Expected length (%d) != real length (%d). This is often " - "caused by using different TensorRT LLM version to build " - "engine and run engine.", - (int) length, (int) (d - a)); -} - -void WeightOnlyQuantMatmulPlugin::init(nvinfer1::DataType type, WeightTypeId weightTypeId) -{ - mArch = tensorrt_llm::common::getSMVersion(); - mType = type; - mWeightTypeId = weightTypeId; - - if (mWeightTypeId == WeightTypeId::INT8) - { - if (mType == nvinfer1::DataType::kHALF) - { - m_weightOnlyGemmRunner = std::make_shared< - CutlassFpAIntBGemmRunner<half, uint8_t, cutlass::WeightOnlyQuantOp::PER_COLUMN_SCALE_ONLY>>(); - mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( - mArch, tensorrt_llm::kernels::weight_only::KernelType::FP16Int8PerChannel); - mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::FP16Int8PerChannel; - } -#if defined(ENABLE_BF16) - else if (mType == nvinfer1::DataType::kBF16) - { - m_weightOnlyGemmRunner = std::make_shared< - CutlassFpAIntBGemmRunner<__nv_bfloat16, uint8_t, cutlass::WeightOnlyQuantOp::PER_COLUMN_SCALE_ONLY>>(); - mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( - mArch, tensorrt_llm::kernels::weight_only::KernelType::BF16Int8PerChannel); - mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::BF16Int8PerChannel; - } -#endif - else - { - TLLM_CHECK(false); - } - } - else if (mWeightTypeId == WeightTypeId::INT4) - { - if (mType == nvinfer1::DataType::kHALF) - { - m_weightOnlyGemmRunner = std::make_shared< - CutlassFpAIntBGemmRunner<half, cutlass::uint4b_t, cutlass::WeightOnlyQuantOp::PER_COLUMN_SCALE_ONLY>>(); - mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( - mArch, tensorrt_llm::kernels::weight_only::KernelType::FP16Int4PerChannel); - mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::FP16Int4PerChannel; - } -#if defined(ENABLE_BF16) - else if (mType == nvinfer1::DataType::kBF16) - { - m_weightOnlyGemmRunner = std::make_shared<CutlassFpAIntBGemmRunner<__nv_bfloat16, cutlass::uint4b_t, - cutlass::WeightOnlyQuantOp::PER_COLUMN_SCALE_ONLY>>(); - mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( - mArch, tensorrt_llm::kernels::weight_only::KernelType::BF16Int4PerChannel); - mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::BF16Int4PerChannel; - } -#endif - else - { - TLLM_CHECK(false); - } - } - else - { - TLLM_CHECK(false); - } - - mPluginProfiler->setWeightTypeId(mWeightTypeId); - if (mCudaKernelEnabled) - { - mPluginProfiler->setCudaKernelType(mCudaKernelType, mArch); - } - mGemmId = GemmIdCore(mDims.n, mDims.k, mType); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* WeightOnlyQuantMatmulPlugin::clone() const noexcept -{ - auto* plugin = new WeightOnlyQuantMatmulPlugin(*this); - return plugin; -} - -void WeightOnlyQuantMatmulPlugin::configGemm() -{ - mPluginProfiler->profileTactics(m_weightOnlyGemmRunner, mType, mDims, mGemmId, mCudaKernelEnabled); -} - -nvinfer1::DimsExprs WeightOnlyQuantMatmulPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - // input [m1, m2, m3, ... , k] - // weight [k, n] for int8, [k, n/2] for int4 - - try - { - TLLM_CHECK(nbInputs == 3); - TLLM_CHECK(outputIndex == 0); - int const nbDimsA = inputs[0].nbDims; - int const nbDimsB = inputs[1].nbDims; - TLLM_CHECK(nbDimsA >= 2); - TLLM_CHECK(nbDimsB == 2); - DimsExprs ret; - ret.nbDims = nbDimsA; - for (int ii = 0; ii < nbDimsA - 1; ++ii) - { - ret.d[ii] = inputs[0].d[ii]; - } - if (mWeightTypeId == WeightTypeId::INT8) - { - // int8 weight only quant - ret.d[nbDimsA - 1] = exprBuilder.constant(inputs[1].d[1]->getConstantValue()); - } - else - { - // int4 weight only quant - ret.d[nbDimsA - 1] = exprBuilder.constant(inputs[1].d[1]->getConstantValue() * INT8_INT4_RATIO); - } - return ret; - } - catch (std::exception const& e) - { - caughtError(e); - } - return DimsExprs{}; -} - -bool WeightOnlyQuantMatmulPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - switch (pos) - { - case 0: - // activation - return inOut[0].type == mType && inOut[0].format == TensorFormat::kLINEAR; - case 1: - // weights - // Weights are required to be int8, but will be reinterpreted as int4 in enqueue if required - // Weights stored in checkpoint should have int8/int4 type - return inOut[1].type == nvinfer1::DataType::kINT8 && inOut[1].format == TensorFormat::kLINEAR; - case 2: - // scales channels - return inOut[2].type == mType && inOut[2].format == TensorFormat::kLINEAR; - case 3: - // out - return inOut[3].type == mType && inOut[3].format == TensorFormat::kLINEAR; - default: - // Never should be here - assert(false); - return false; - } -} - -void WeightOnlyQuantMatmulPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ - auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); - auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); - - int const maxK = in[0].max.d[in[0].max.nbDims - 1]; - int const maxN = in[1].max.d[1] * getWeightTypeMultiplier(mWeightTypeId); - - auto const K = maxK; - auto const N = maxN / getWeightTypeMultiplier(mWeightTypeId); - - if (!mDims.isInitialized()) - { - mDims = {minM, maxM, N, K}; - } - - mGemmId = {N, K, mType}; - - m_workspaceMaxSize = m_weightOnlyGemmRunner->getWorkspaceSize(maxM, maxN, maxK); -} - -size_t WeightOnlyQuantMatmulPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - return m_workspaceMaxSize; -} - -int WeightOnlyQuantMatmulPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - // inputs - // mat1 [M1, M2,..., K] - // mat2 [K, N] for int8, [K, N/2] for int4 - // scale_channels [N] - // outputs - // mat [M, N] - - int64_t m64 = 1; - for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) - { - m64 *= inputDesc[0].dims.d[ii]; - } - int const m = TLLM_INT32_CAST(m64); - int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[1]); - int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); - - if (m == 0) - return 0; - -#if defined(ENABLE_BF16) - TLLM_CHECK_WITH_INFO(mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kBF16, - "No valid weightOnlyQuantMatmul configuration"); -#else - TLLM_CHECK_WITH_INFO(mType == nvinfer1::DataType::kHALF, "No valid weightOnlyQuantMatmul configuration"); -#endif - int real_n = mWeightTypeId == WeightTypeId::INT4 ? n * INT8_INT4_RATIO : n; - - // get best tactic and check if CUDA kernel should be used - bool use_cuda_kernel = false; - auto const& bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); - TLLM_CHECK_WITH_INFO(bestTactic, - "No valid weight only per-channel GEMM tactic(It is usually caused by the failure to execute all candidate " - "configurations of the CUTLASS kernel, please pay attention to the warning information when building the " - "engine.)"); - use_cuda_kernel = bestTactic->enableCudaKernel; - if (use_cuda_kernel) - { - void const* cuda_kernel_act_ptr = inputs[0]; - void const* cuda_kernel_weight_ptr = inputs[1]; - void const* cuda_kernel_scales_ptr = inputs[2]; - void* cuda_kernel_out_ptr = outputs[0]; - tensorrt_llm::kernels::weight_only::Params params(cuda_kernel_act_ptr, nullptr, cuda_kernel_weight_ptr, - cuda_kernel_scales_ptr, nullptr, nullptr, cuda_kernel_out_ptr, 1.f, m, real_n, k, 0, mCudaKernelType); - tensorrt_llm::kernels::weight_only::kernel_launcher(mArch, params, stream); - } - else - { - int const ws_size = m_weightOnlyGemmRunner->getWorkspaceSize(m, real_n, k); - - m_weightOnlyGemmRunner->gemm(inputs[0], inputs[1], inputs[2], outputs[0], m, real_n, k, *bestTactic, - reinterpret_cast<char*>(workspace), ws_size, stream); - } - - return 0; -} - -// IPluginV2Ext Methods -nvinfer1::DataType WeightOnlyQuantMatmulPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - TLLM_CHECK(index == 0); - return mType; -} - -// IPluginV2 Methods - -char const* WeightOnlyQuantMatmulPlugin::getPluginType() const noexcept -{ - return WOQ_MATMUL_PLUGIN_NAME; -} - -char const* WeightOnlyQuantMatmulPlugin::getPluginVersion() const noexcept -{ - return WOQ_MATMUL_PLUGIN_VERSION; -} - -int WeightOnlyQuantMatmulPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int WeightOnlyQuantMatmulPlugin::initialize() noexcept -{ - configGemm(); - return 0; -} - -void WeightOnlyQuantMatmulPlugin::terminate() noexcept {} - -size_t WeightOnlyQuantMatmulPlugin::getSerializationSize() const noexcept -{ - return sizeof(mWeightTypeId) + // mWeightTypeId - sizeof(nvinfer1::DataType) + // mType - sizeof(mDims) + // Dimensions - mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size -} - -void WeightOnlyQuantMatmulPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast<char*>(buffer), *a = d; - write(d, mType); - write(d, mWeightTypeId); - write(d, mDims); - - mPluginProfiler->serialize(d, mGemmId); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void WeightOnlyQuantMatmulPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -/////////////// - -WeightOnlyQuantMatmulPluginCreator::WeightOnlyQuantMatmulPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("weight_type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* WeightOnlyQuantMatmulPluginCreator::getPluginName() const noexcept -{ - return WOQ_MATMUL_PLUGIN_NAME; -} - -char const* WeightOnlyQuantMatmulPluginCreator::getPluginVersion() const noexcept -{ - return WOQ_MATMUL_PLUGIN_VERSION; -} - -PluginFieldCollection const* WeightOnlyQuantMatmulPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* WeightOnlyQuantMatmulPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - nvinfer1::DataType type{}; - WeightTypeId weightTypeId{}; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "weight_type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - weightTypeId = static_cast<WeightTypeId>(*(static_cast<int const*>(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); - type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); - } - } - try - { - // WeightOnlyGroupwiseQuantMatmulPluginCreator is unique and shared for an engine generation - // Create plugin profiler with shared tactics map - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); - auto* obj = new WeightOnlyQuantMatmulPlugin(type, weightTypeId, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} - -IPluginV2* WeightOnlyQuantMatmulPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call WeightOnlyQuantMatmulPlugin::destroy() - try - { - // Create plugin profiler with private tactics map which is read from the serialized engine - auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); - auto* obj = new WeightOnlyQuantMatmulPlugin(serialData, serialLength, pluginProfiler); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - caughtError(e); - } - return nullptr; -} diff --git a/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.h b/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.h deleted file mode 100644 index 3177d8297d2d..000000000000 --- a/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.h +++ /dev/null @@ -1,175 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/kernels/cutlass_kernels/fpA_intB_gemm/fpA_intB_gemm.h" -#include "tensorrt_llm/kernels/weightOnlyBatchedGemv/kernelLauncher.h" -#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" -#include "tensorrt_llm/plugins/common/plugin.h" - -#include <cassert> -#include <cutlass/numeric_types.h> -#include <memory> -#include <set> -#include <string> -#include <vector> - -// The blank line here is to avoid clang-format -sort-includes option reordering these two cutlass header files and -// breaking dependencies -#include "cutlass/integer_subbyte.h" - -namespace tensorrt_llm::plugins -{ -enum class WeightTypeId -{ - INT8 = 1, - INT4 = 2, -}; - -constexpr int32_t FP16_BITS = 16; -constexpr int32_t INT8_BITS = 8; -constexpr int32_t INT4_BITS = 4; -constexpr int32_t INT8_INT4_RATIO = INT8_BITS / INT4_BITS; -constexpr int32_t FP16_INT4_RATIO = FP16_BITS / INT4_BITS; -constexpr int32_t FP16_INT8_RATIO = FP16_BITS / INT8_BITS; - -inline int32_t getWeightTypeMultiplier(WeightTypeId weightTypeId) -{ - return weightTypeId == WeightTypeId::INT8 ? 1 : INT8_INT4_RATIO; -} - -using WeightOnlyGemmRunner = tensorrt_llm::kernels::cutlass_kernels::CutlassFpAIntBGemmRunnerInterface; -using WeightOnlyGemmRunnerPtr = std::shared_ptr<WeightOnlyGemmRunner>; -using KernelType = tensorrt_llm::kernels::weight_only::KernelType; - -class WeightOnlyQuantGemmPluginProfiler : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, - WeightOnlyGemmRunnerPtr, GemmIdCore, GemmIdCoreHash> -{ -public: - using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; - - void setWeightTypeId(WeightTypeId weightId) - { - mWeightTypeId = weightId; - } - - void setCudaKernelType(KernelType cudaKernelType, int arch) - { - mCudaKernelType = cudaKernelType; - mArch = arch; - } - -protected: - void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; - - void computeTmpSize(size_t maxM, size_t n, size_t k) override; - - std::vector<Config> getTactics(int m, int n, int k) const override; - - bool checkTactic(int m, int n, int k, Config const& tactic) const override; - -private: - WeightTypeId mWeightTypeId; - KernelType mCudaKernelType; - int mArch; -}; - -class WeightOnlyQuantMatmulPlugin : public BasePlugin -{ -public: - using PluginProfilerPtr = std::shared_ptr<WeightOnlyQuantGemmPluginProfiler>; - WeightOnlyQuantMatmulPlugin() = delete; - - WeightOnlyQuantMatmulPlugin(nvinfer1::DataType type, WeightTypeId weightTypeId, PluginProfilerPtr const& profiler); - - WeightOnlyQuantMatmulPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); - - ~WeightOnlyQuantMatmulPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - -private: - void init(nvinfer1::DataType type, WeightTypeId weightTypeId); - - void configGemm(); - -private: - const std::string mLayerName; - - WeightOnlyGemmRunnerPtr m_weightOnlyGemmRunner; - size_t m_workspaceMaxSize; - nvinfer1::DataType mType; - WeightTypeId mWeightTypeId; - bool mCudaKernelEnabled; - tensorrt_llm::kernels::weight_only::KernelType mCudaKernelType; - int mArch; - - GemmDims mDims{}; - GemmIdCore mGemmId{}; - - PluginProfilerPtr mPluginProfiler; -}; - -class WeightOnlyQuantMatmulPluginCreator : public BaseCreator -{ -public: - WeightOnlyQuantMatmulPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - -private: - GemmPluginProfilerManager<WeightOnlyQuantGemmPluginProfiler> gemmPluginProfileManager; - static nvinfer1::PluginFieldCollection mFC; - static std::vector<nvinfer1::PluginField> mPluginAttributes; -}; - -} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/runtime/CMakeLists.txt b/cpp/tensorrt_llm/runtime/CMakeLists.txt index ca81fbb0f6cd..11a9391c0e69 100644 --- a/cpp/tensorrt_llm/runtime/CMakeLists.txt +++ b/cpp/tensorrt_llm/runtime/CMakeLists.txt @@ -26,7 +26,6 @@ set(SRCS eagleBuffers.cpp explicitDraftTokensBuffers.cpp lookaheadBuffers.cpp - layerProfiler.cpp loraManager.cpp loraUtils.cpp loraModule.cpp @@ -51,9 +50,6 @@ set(SRCS promptTuningParams.cpp runtimeKernels.cu tllmBuffers.cpp - tllmRuntime.cpp - tllmStreamReaders.cpp - tllmLogger.cpp workerPool.cpp worldConfig.cpp virtualMemory.cpp) diff --git a/cpp/tensorrt_llm/runtime/bufferManager.cpp b/cpp/tensorrt_llm/runtime/bufferManager.cpp index 3de42a253158..58257516a387 100644 --- a/cpp/tensorrt_llm/runtime/bufferManager.cpp +++ b/cpp/tensorrt_llm/runtime/bufferManager.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tllmBuffers.h" #include <cstring> @@ -37,7 +38,7 @@ BufferManager::BufferManager(CudaStreamPtr stream, bool trimPool) mPool = CudaMemPool::getPrimaryPoolForDevice(mStream->getDevice()); } -BufferManager::IBufferPtr BufferManager::gpu(std::size_t size, nvinfer1::DataType type) const +BufferManager::IBufferPtr BufferManager::gpu(std::size_t size, tensorrt_llm::DataType type) const { if (auto vmAllocator = getVirtualMemoryAllocator()) { @@ -51,7 +52,7 @@ BufferManager::IBufferPtr BufferManager::gpu(std::size_t size, nvinfer1::DataTyp return gpuSync(size, type); } -BufferManager::ITensorPtr BufferManager::gpu(nvinfer1::Dims dims, nvinfer1::DataType type) const +BufferManager::ITensorPtr BufferManager::gpu(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) const { if (auto vmAllocator = getVirtualMemoryAllocator()) { @@ -65,7 +66,7 @@ BufferManager::ITensorPtr BufferManager::gpu(nvinfer1::Dims dims, nvinfer1::Data return gpuSync(dims, type); } -BufferManager::IBufferPtr BufferManager::gpuSync(std::size_t size, nvinfer1::DataType type) +BufferManager::IBufferPtr BufferManager::gpuSync(std::size_t size, tensorrt_llm::DataType type) { if (auto vmAllocator = getVirtualMemoryAllocator()) { @@ -74,7 +75,7 @@ BufferManager::IBufferPtr BufferManager::gpuSync(std::size_t size, nvinfer1::Dat return std::make_unique<StaticDeviceBuffer>(size, type, CudaAllocator{}); } -BufferManager::ITensorPtr BufferManager::gpuSync(nvinfer1::Dims dims, nvinfer1::DataType type) +BufferManager::ITensorPtr BufferManager::gpuSync(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) { if (auto vmAllocator = getVirtualMemoryAllocator()) { @@ -83,47 +84,48 @@ BufferManager::ITensorPtr BufferManager::gpuSync(nvinfer1::Dims dims, nvinfer1:: return std::make_unique<StaticDeviceTensor>(dims, type, CudaAllocator{}); } -BufferManager::IBufferPtr BufferManager::cpu(std::size_t size, nvinfer1::DataType type) +BufferManager::IBufferPtr BufferManager::cpu(std::size_t size, tensorrt_llm::DataType type) { return std::make_unique<HostBuffer>(size, type); } -BufferManager::ITensorPtr BufferManager::cpu(nvinfer1::Dims dims, nvinfer1::DataType type) +BufferManager::ITensorPtr BufferManager::cpu(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) { return std::make_unique<HostTensor>(dims, type); } -BufferManager::IBufferPtr BufferManager::pinned(std::size_t size, nvinfer1::DataType type) +BufferManager::IBufferPtr BufferManager::pinned(std::size_t size, tensorrt_llm::DataType type) { return std::make_unique<PinnedBuffer>(size, type); } -BufferManager::ITensorPtr BufferManager::pinned(nvinfer1::Dims dims, nvinfer1::DataType type) +BufferManager::ITensorPtr BufferManager::pinned(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) { return std::make_unique<PinnedTensor>(dims, type); } -BufferManager::IBufferPtr BufferManager::pinnedPool(std::size_t size, nvinfer1::DataType type) +BufferManager::IBufferPtr BufferManager::pinnedPool(std::size_t size, tensorrt_llm::DataType type) { return std::make_unique<PinnedPoolBuffer>(size, type); } -BufferManager::ITensorPtr BufferManager::pinnedPool(nvinfer1::Dims dims, nvinfer1::DataType type) +BufferManager::ITensorPtr BufferManager::pinnedPool(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) { return std::make_unique<PinnedPoolTensor>(dims, type); } -BufferManager::IBufferPtr BufferManager::managed(std::size_t size, nvinfer1::DataType type) +BufferManager::IBufferPtr BufferManager::managed(std::size_t size, tensorrt_llm::DataType type) { return std::make_unique<UVMBuffer>(size, type); } -BufferManager::ITensorPtr BufferManager::managed(nvinfer1::Dims dims, nvinfer1::DataType type) +BufferManager::ITensorPtr BufferManager::managed(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) { return std::make_unique<UVMTensor>(dims, type); } -BufferManager::ITensorPtr BufferManager::ipcNvls(std::set<int> ranks, nvinfer1::Dims dims, nvinfer1::DataType type) +BufferManager::ITensorPtr BufferManager::ipcNvls( + std::set<int> ranks, tensorrt_llm::Dims dims, tensorrt_llm::DataType type) { return std::make_unique<MulticastTensor>(dims, type, ranks); } @@ -187,7 +189,7 @@ void BufferManager::copy(IBuffer const& src, IBuffer& dst) const } BufferManager::IBufferPtr BufferManager::allocate( - MemoryType memoryType, std::size_t size, nvinfer1::DataType type) const + MemoryType memoryType, std::size_t size, tensorrt_llm::DataType type) const { switch (memoryType) { @@ -202,7 +204,7 @@ BufferManager::IBufferPtr BufferManager::allocate( } BufferManager::ITensorPtr BufferManager::allocate( - MemoryType memoryType, nvinfer1::Dims dims, nvinfer1::DataType type) const + MemoryType memoryType, tensorrt_llm::Dims dims, tensorrt_llm::DataType type) const { switch (memoryType) { diff --git a/cpp/tensorrt_llm/runtime/bufferView.h b/cpp/tensorrt_llm/runtime/bufferView.h index 236b89d7d455..a001d05f1f8b 100644 --- a/cpp/tensorrt_llm/runtime/bufferView.h +++ b/cpp/tensorrt_llm/runtime/bufferView.h @@ -17,6 +17,7 @@ #pragma once #include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/iBuffer.h" #include <string> @@ -70,7 +71,7 @@ class BufferView : virtual public IBuffer return mBuffer->getCapacity() - mOffset; } - [[nodiscard]] nvinfer1::DataType getDataType() const override + [[nodiscard]] tensorrt_llm::DataType getDataType() const override { return mBuffer->getDataType(); } diff --git a/cpp/tensorrt_llm/runtime/decoderState.cpp b/cpp/tensorrt_llm/runtime/decoderState.cpp index b5851dc1c2d2..83037b2431cb 100644 --- a/cpp/tensorrt_llm/runtime/decoderState.cpp +++ b/cpp/tensorrt_llm/runtime/decoderState.cpp @@ -16,6 +16,7 @@ #include "tensorrt_llm/runtime/decoderState.h" #include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/runtime/runtimeKernels.h" @@ -27,10 +28,10 @@ using TensorPtr = DecoderState::TensorPtr; BeamSearchBuffers::BeamSearchBuffers(BufferManager const& bufferManager) : mOutputBeamHypotheses{} - , mCumLogProbsTmp(bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kFLOAT)) + , mCumLogProbsTmp(bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT)) { mOutputBeamHypotheses.empty(bufferManager); - mCumLogProbsTmp = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kFLOAT); + mCumLogProbsTmp = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); int device; cudaGetDevice(&device); @@ -54,8 +55,8 @@ DecoderState::DecoderState() } void DecoderState::setup(SizeType32 maxNumSequences, SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow, - SizeType32 sinkTokenLength, SizeType32 maxSequenceLength, nvinfer1::DataType dtype, ModelConfig const& modelConfig, - WorldConfig const& worldConfig, BufferManager const& bufferManager) + SizeType32 sinkTokenLength, SizeType32 maxSequenceLength, tensorrt_llm::DataType dtype, + ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); setupBuffers(dtype, bufferManager); @@ -64,7 +65,7 @@ void DecoderState::setup(SizeType32 maxNumSequences, SizeType32 maxBeamWidth, Si TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); } -void DecoderState::setupBuffers(nvinfer1::DataType dtype, BufferManager const& bufferManager) +void DecoderState::setupBuffers(tensorrt_llm::DataType dtype, BufferManager const& bufferManager) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); auto constexpr nvTokenIdType = TRTDataType<TokenIdType>::value; @@ -114,7 +115,7 @@ void DecoderState::setupBuffers(nvinfer1::DataType dtype, BufferManager const& b } void DecoderState::setupSpeculativeDecoding(SpeculativeDecodingMode const& speculativeDecodingMode, - SizeType32 maxTokensPerEngineStep, nvinfer1::DataType dtype, ModelConfig const& modelConfig, + SizeType32 maxTokensPerEngineStep, tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); @@ -124,8 +125,8 @@ void DecoderState::setupSpeculativeDecoding(SpeculativeDecodingMode const& specu TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); } -void DecoderState::setupSpeculativeDecodingBuffers( - SpeculativeDecodingMode const speculativeDecodingMode, nvinfer1::DataType dtype, BufferManager const& bufferManager) +void DecoderState::setupSpeculativeDecodingBuffers(SpeculativeDecodingMode const speculativeDecodingMode, + tensorrt_llm::DataType dtype, BufferManager const& bufferManager) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); @@ -151,13 +152,13 @@ void DecoderState::setupSpeculativeDecodingBuffers( if (speculativeDecodingMode.predictsDraftTokens()) { speculativeDecodingOutputs.nextDraftTokens - = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); if (speculativeDecodingMode.variableDraftLength()) { speculativeDecodingOutputs.nextDraftTokensLen - = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); speculativeDecodingOutputs.prevDraftTokensLen - = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); } } if (speculativeDecodingMode.isLookaheadDecoding()) @@ -167,11 +168,11 @@ void DecoderState::setupSpeculativeDecodingBuffers( if (speculativeDecodingMode.needsKVCacheRewind()) { speculativeDecodingOutputs.acceptedTokensLen - = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); speculativeDecodingOutputs.acceptedLengthsCumSum - = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); speculativeDecodingOutputs.pathsOffsets - = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); } dOutput->speculativeDecodingOutputs = speculativeDecodingOutputs; diff --git a/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.cpp b/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.cpp index f8a4fa4e7467..c5098bf777e0 100644 --- a/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.cpp +++ b/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.cpp @@ -15,11 +15,12 @@ */ #include "tensorrt_llm/runtime/decodingLayerWorkspace.h" +#include "tensorrt_llm/common/tllmDataType.h" #include <utility> tensorrt_llm::runtime::DecodingLayerWorkspace::DecodingLayerWorkspace(std::shared_ptr<BufferManager> bufferManager, - tensorrt_llm::layers::DecoderDomain const& decoderDomain, nvinfer1::DataType logitsType, + tensorrt_llm::layers::DecoderDomain const& decoderDomain, tensorrt_llm::DataType logitsType, size_t workspaceBufferSizeInBytes) : mBufferManager(std::move(bufferManager)) , mBatchSlotsDevice( @@ -82,7 +83,8 @@ void tensorrt_llm::runtime::DecodingLayerWorkspace::resize(size_t minSize) } tensorrt_llm::runtime::DecodingLayerWorkspace::TensorPtr -tensorrt_llm::runtime::DecodingLayerWorkspace::getWorkspaceAsDeviceTensor(ITensor::Shape shape, nvinfer1::DataType type) +tensorrt_llm::runtime::DecodingLayerWorkspace::getWorkspaceAsDeviceTensor( + ITensor::Shape shape, tensorrt_llm::DataType type) { auto const sizeInBytes = ITensor::volume(shape) * BufferDataType(type).getSize(); return std::make_shared<GenericTensor<BorrowingAllocator<MemoryType::kGPU>>>( diff --git a/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.h b/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.h index c2688b51139f..68d3d54124f5 100644 --- a/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.h +++ b/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.h @@ -19,6 +19,7 @@ #include <memory> #include "tensorrt_llm/common/dataType.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/common/workspace.h" #include "tensorrt_llm/layers/decodingParams.h" #include "tensorrt_llm/runtime/bufferManager.h" @@ -39,7 +40,7 @@ class DecodingLayerWorkspace using BufferPtr = IBuffer::SharedPtr; DecodingLayerWorkspace(std::shared_ptr<BufferManager> bufferManager, layers::DecoderDomain const& decoderDomain, - nvinfer1::DataType logitsType, size_t workspaceBufferSizeInBytes); + tensorrt_llm::DataType logitsType, size_t workspaceBufferSizeInBytes); DecodingLayerWorkspace() = delete; @@ -71,7 +72,7 @@ class DecodingLayerWorkspace [[nodiscard]] TensorPtr getDeviceRuntimeLogits() const; ///@brief Gets a tensor with the given shape and type at the start of the device workspace. - TensorPtr getWorkspaceAsDeviceTensor(ITensor::Shape shape, nvinfer1::DataType type); + TensorPtr getWorkspaceAsDeviceTensor(ITensor::Shape shape, tensorrt_llm::DataType type); /// @brief A convenience function to copy the content of a standard vector to a device workspace. template <typename T, typename Alloc> @@ -112,7 +113,7 @@ class DecodingLayerWorkspace { size_t lastTensorOffset = 0; auto alignedSizeCalculator - = [&lastTensorOffset](std::pair<ITensor::Shape, nvinfer1::DataType> const& tensorDescriptor) + = [&lastTensorOffset](std::pair<ITensor::Shape, tensorrt_llm::DataType> const& tensorDescriptor) { auto const& [shape, type] = tensorDescriptor; auto const sizeInBytes = ITensor::volume(shape) * tensorrt_llm::common::getDTypeSize(type); diff --git a/cpp/tensorrt_llm/runtime/eagleBuffers.cpp b/cpp/tensorrt_llm/runtime/eagleBuffers.cpp index 097fd95f49aa..e0f2198c3e58 100644 --- a/cpp/tensorrt_llm/runtime/eagleBuffers.cpp +++ b/cpp/tensorrt_llm/runtime/eagleBuffers.cpp @@ -19,6 +19,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/speculativeDecoding/eagleDecodingKernels.h" #include "tensorrt_llm/kernels/speculativeDecoding/explicitDraftTokensKernels.h" #include "tensorrt_llm/runtime/common.h" @@ -41,50 +42,51 @@ void EagleBuffers::Inputs::create(SizeType32 maxNumSequences, BufferManager cons auto const numEagleLayers = speculativeDecodingModule.getMaxDraftPathLen(); auto constexpr TRTTokenIdType = runtime::TRTDataType<runtime::TokenIdType>::value; - temperatures = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kFLOAT); - randomDataSample = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kFLOAT); + temperatures = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kFLOAT); + randomDataSample = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kFLOAT); randomDataValidation - = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingTokens}), nvinfer1::DataType::kFLOAT); + = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingTokens}), tensorrt_llm::DataType::kFLOAT); draftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), TRTTokenIdType); - draftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + draftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); draftPaths - = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), nvinfer1::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), tensorrt_llm::DataType::kINT32); draftPathsHost = BufferManager::pinnedPool( - ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), nvinfer1::DataType::kINT32); - specDecodingGenerationLengths = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), tensorrt_llm::DataType::kINT32); + specDecodingGenerationLengths = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); specDecodingGenerationLengthsHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); specDecodingPackedMasks = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingTokens, common::ceilDiv(maxDecodingTokens, 32)}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); specDecodingPositionOffsets - = manager.gpu(ITensor::makeShape({maxNumSequences * maxDecodingTokens}), nvinfer1::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences * maxDecodingTokens}), tensorrt_llm::DataType::kINT32); eagleNetCtxRequestTypesHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); eagleNetCtxContextLengthsHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); eagleNetCtxPastKeyValueLengthsHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); eagleNetGenRequestTypesHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); eagleNetGenContextLengthsHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); eagleNetGenPastKeyValueLengthsHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); inputGenTokensHost = BufferManager::pinnedPool( - ITensor::makeShape({maxNumSequences * maxDecodingTokens}), nvinfer1::DataType::kINT32); - chunkedContextNextTokens = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); - useSpecDecoding = manager.cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxNumSequences * maxDecodingTokens}), tensorrt_llm::DataType::kINT32); + chunkedContextNextTokens = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + useSpecDecoding = manager.cpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); // Eagle-2 - useDynamicTreeHost = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - dynamicTreeMaxTopKHost = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - prevScores = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), nvinfer1::DataType::kFLOAT); + useDynamicTreeHost = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + dynamicTreeMaxTopKHost = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + prevScores + = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), tensorrt_llm::DataType::kFLOAT); currentExpandIndices = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), TRTTokenIdType); allLayersScores = manager.gpu( ITensor::makeShape({maxNumSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens}), - nvinfer1::DataType::kFLOAT); + tensorrt_llm::DataType::kFLOAT); allLayersDraftTokenIds = manager.gpu( ITensor::makeShape({maxNumSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens}), TRTTokenIdType); @@ -114,58 +116,63 @@ EagleBuffers::EagleBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, run auto constexpr TRTTokenIdType = runtime::TRTDataType<runtime::TokenIdType>::value; // input tensors - engineInputs.temperatures = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kFLOAT); - engineInputs.posteriorAlpha = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kFLOAT); - engineInputs.posteriorThreshold = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kFLOAT); - posteriorAlphaHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kFLOAT); - posteriorThresholdHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kFLOAT); - greedySamplingHost = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + engineInputs.temperatures = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); + engineInputs.posteriorAlpha = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); + engineInputs.posteriorThreshold = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); + posteriorAlphaHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kFLOAT); + posteriorThresholdHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kFLOAT); + greedySamplingHost = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); engineInputs.draftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), TRTTokenIdType); - engineInputs.draftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + engineInputs.draftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); engineInputs.draftPaths - = manager.gpu(ITensor::makeShape({maxNumSequences, numPaths, pathLen}), nvinfer1::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, numPaths, pathLen}), tensorrt_llm::DataType::kINT32); engineInputs.specDecodingGenerationLengths - = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); engineInputs.specDecodingPositionOffsets - = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineInputs.specDecodingPackedMasks = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineInputs.specDecodingPackedMasks + = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineInputs.randomDataSample = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kFLOAT); - engineInputs.randomDataValidation = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kFLOAT); + engineInputs.randomDataSample = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); + engineInputs.randomDataValidation = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); engineInputs.eagleNetCtxRequestTypesHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); engineInputs.eagleNetCtxContextLengthsHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); engineInputs.eagleNetCtxPastKeyValueLengthsHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); engineInputs.eagleNetGenRequestTypesHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); engineInputs.eagleNetGenContextLengthsHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); engineInputs.eagleNetGenPastKeyValueLengthsHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); - engineInputs.inputGenTokensHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); - engineInputs.chunkedContextNextTokens = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineInputs.useSpecDecoding = BufferManager::cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); + engineInputs.inputGenTokensHost + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); + engineInputs.chunkedContextNextTokens + = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineInputs.useSpecDecoding = BufferManager::cpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); bufferCast<SizeType32>(*engineInputs.useSpecDecoding)[0] = 1; - chunkedContextNextTokensHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); + chunkedContextNextTokensHost + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); // Eagle-2 - engineInputs.useDynamicTreeHost = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + engineInputs.useDynamicTreeHost + = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); engineInputs.dynamicTreeMaxTopKHost - = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); engineInputs.prevScores - = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), nvinfer1::DataType::kFLOAT); + = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), tensorrt_llm::DataType::kFLOAT); engineInputs.currentExpandIndices = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), TRTTokenIdType); engineInputs.allLayersScores = manager.gpu( ITensor::makeShape({maxNumSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens}), - nvinfer1::DataType::kFLOAT); + tensorrt_llm::DataType::kFLOAT); engineInputs.allLayersDraftTokenIds = manager.gpu( ITensor::makeShape({maxNumSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens}), TRTTokenIdType); @@ -176,24 +183,24 @@ EagleBuffers::EagleBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, run // output tensors engineOutputs.nextDraftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, numPaths, pathLen}), TRTTokenIdType); - engineOutputs.nextDraftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + engineOutputs.nextDraftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); engineOutputs.nextDraftPaths - = manager.gpu(ITensor::makeShape({maxNumSequences, numPaths, pathLen}), nvinfer1::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, numPaths, pathLen}), tensorrt_llm::DataType::kINT32); engineOutputs.acceptedTokens - = manager.gpu(ITensor::makeShape({maxNumSequences, pathLen}), nvinfer1::DataType::kINT32); - engineOutputs.acceptedLens = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); - engineOutputs.acceptedPaths = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, pathLen}), tensorrt_llm::DataType::kINT32); + engineOutputs.acceptedLens = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + engineOutputs.acceptedPaths = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); engineOutputs.chunkedContextNextTokens - = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); // helper tensors scanReduceTempStorageBytes = tksd::invokeScanReduceGenerationLengths( maxNumSequences, nullptr, nullptr, 0, nullptr, nullptr, manager.getStream().get()); scanReduceTempStorage = manager.gpu(scanReduceTempStorageBytes); - cumSumGenerationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - maxGenerationLength = manager.gpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + cumSumGenerationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + maxGenerationLength = manager.gpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); // pre-allocate empty tensors reshape(0, maxNumSequences, modelConfig); @@ -520,15 +527,15 @@ void EagleBuffers::setFromInputs(RequestVector const& contextRequests, RequestVe switch (dtype) { - case nvinfer1::DataType::kFLOAT: + case tensorrt_llm::DataType::kFLOAT: setFromInputs<float>( contextRequests, genRequests, vocabSizePadded, seqSlots, draftBuffers, *eagleModule, manager); break; - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: setFromInputs<half>( contextRequests, genRequests, vocabSizePadded, seqSlots, draftBuffers, *eagleModule, manager); break; - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: setFromInputs<__nv_bfloat16>( contextRequests, genRequests, vocabSizePadded, seqSlots, draftBuffers, *eagleModule, manager); break; diff --git a/cpp/tensorrt_llm/runtime/explicitDraftTokensBuffers.cpp b/cpp/tensorrt_llm/runtime/explicitDraftTokensBuffers.cpp index ed205ca0e117..89c74e6f9349 100644 --- a/cpp/tensorrt_llm/runtime/explicitDraftTokensBuffers.cpp +++ b/cpp/tensorrt_llm/runtime/explicitDraftTokensBuffers.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/speculativeDecoding/explicitDraftTokensKernels.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iBuffer.h" @@ -40,23 +41,24 @@ void ExplicitDraftTokensBuffers::Inputs::create(SizeType32 maxNumSequences, Buff auto constexpr TRTTokenIdType = runtime::TRTDataType<runtime::TokenIdType>::value; auto const dtype = modelConfig.getDataType(); - maxGenLengthHost = manager.pinned(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + maxGenLengthHost = manager.pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); temperatures = manager.gpu(ITensor::makeShape({maxNumSequences}), dtype); - positionIdsBase = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); - generationLengths = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); - generationLengthsHost = manager.pinned(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + positionIdsBase = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + generationLengths = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + generationLengthsHost = manager.pinned(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); randomDataSample = manager.gpu(ITensor::makeShape({maxNumSequences}), dtype); randomDataValidation = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxDraftPathLen}), dtype); draftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), TRTTokenIdType); draftIndices - = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), nvinfer1::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), tensorrt_llm::DataType::kINT32); draftProbs = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxDraftPathLen, vocabSizePadded}), dtype); packedMasks = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingTokens, common::ceilDiv(maxDecodingTokens, 32)}), - nvinfer1::DataType::kINT32); - positionIds = manager.gpu(ITensor::makeShape({maxNumSequences * maxDecodingTokens}), nvinfer1::DataType::kINT32); - useSpecDecoding = manager.cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); + positionIds + = manager.gpu(ITensor::makeShape({maxNumSequences * maxDecodingTokens}), tensorrt_llm::DataType::kINT32); + useSpecDecoding = manager.cpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); } ExplicitDraftTokensBuffers::ExplicitDraftTokensBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, @@ -81,52 +83,53 @@ ExplicitDraftTokensBuffers::ExplicitDraftTokensBuffers(SizeType32 maxBatchSize, auto const dtype = modelConfig.getDataType(); // input tensors - engineInputs.requestTypesDevice = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineInputs.requestTypesDevice = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); engineInputs.temperatures = manager.emptyTensor(runtime::MemoryType::kGPU, dtype); engineInputs.draftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), TRTTokenIdType); engineInputs.draftIndices - = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), nvinfer1::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), tensorrt_llm::DataType::kINT32); engineInputs.draftProbs = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamDraftLength, vocabSizePadded}), dtype); - engineInputs.generationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineInputs.positionIds = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineInputs.positionOffsets = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineInputs.packedMasks = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineInputs.generationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineInputs.positionIds = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineInputs.positionOffsets = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineInputs.packedMasks = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); engineInputs.randomDataSample = manager.emptyTensor(runtime::MemoryType::kGPU, dtype); engineInputs.randomDataValidation = manager.emptyTensor(runtime::MemoryType::kGPU, dtype); - engineInputs.positionIdsBase = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineInputs.useSpecDecoding = manager.cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + engineInputs.positionIdsBase = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineInputs.useSpecDecoding = manager.cpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); bufferCast<SizeType32>(*engineInputs.useSpecDecoding)[0] = 1; // output tensors engineOutputs.nextDraftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), TRTTokenIdType); engineOutputs.nextDraftIndices - = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), nvinfer1::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), tensorrt_llm::DataType::kINT32); engineOutputs.nextDraftProbs = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamDraftLength, vocabSizePadded}), dtype); - engineOutputs.maxGenToken = manager.gpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - engineOutputs.totalGenToken = manager.gpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + engineOutputs.maxGenToken = manager.gpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + engineOutputs.totalGenToken = manager.gpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); - engineOutputs.nextGenerationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineOutputs.nextPositionOffsets = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineOutputs.masks = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kBOOL); + engineOutputs.nextGenerationLengths + = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineOutputs.nextPositionOffsets = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineOutputs.masks = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kBOOL); engineOutputs.nextFlatTokens = manager.emptyTensor(runtime::MemoryType::kGPU, TRTTokenIdType); - engineOutputs.bestPathLengths = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineOutputs.bestPathIndices = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineOutputs.packedPositionIds = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineOutputs.bestPathLengths = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineOutputs.bestPathIndices = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineOutputs.packedPositionIds = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); // helper tensors auto const& stream = manager.getStream(); scanTempStorageBytes = tksd::invokeScanGenerationLengths(nullptr, 0, nullptr, nullptr, maxNumSequences, stream.get()); scanTempStorage = manager.gpu(scanTempStorageBytes); - cumSumGenerationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + cumSumGenerationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); // pre-allocate empty tensors reshape(0, maxNumSequences, modelConfig); @@ -295,15 +298,15 @@ void ExplicitDraftTokensBuffers::setFromInputs(SizeType32 numCtxSequences, SizeT switch (dtype) { - case nvinfer1::DataType::kFLOAT: + case tensorrt_llm::DataType::kFLOAT: setFromInputs<float>(numCtxSequences, numGenSequences, vocabSizePadded, seqSlots, draftBuffers, contextPositionIds, *explicitDraftTokensModule, stream); break; - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: setFromInputs<half>(numCtxSequences, numGenSequences, vocabSizePadded, seqSlots, draftBuffers, contextPositionIds, *explicitDraftTokensModule, stream); break; - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: setFromInputs<__nv_bfloat16>(numCtxSequences, numGenSequences, vocabSizePadded, seqSlots, draftBuffers, contextPositionIds, *explicitDraftTokensModule, stream); break; diff --git a/cpp/tensorrt_llm/runtime/gptDecoder.cpp b/cpp/tensorrt_llm/runtime/gptDecoder.cpp index 930877206462..e1ac1717af45 100644 --- a/cpp/tensorrt_llm/runtime/gptDecoder.cpp +++ b/cpp/tensorrt_llm/runtime/gptDecoder.cpp @@ -21,7 +21,7 @@ #include "tensorrt_llm/layers/dynamicDecodeLayer.h" #include "tensorrt_llm/runtime/decodingLayerWorkspace.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <memory> @@ -121,7 +121,7 @@ void GptDecoder<T>::disableLookahead( template <typename T> void GptDecoder<T>::setup(SamplingConfig const& samplingConfig, size_t batchSize, TensorConstPtr const& batchSlots, - std::optional<DecodingOutput> const& output, std::optional<nvinfer1::DataType> explicitDraftTokensDType, + std::optional<DecodingOutput> const& output, std::optional<tensorrt_llm::DataType> explicitDraftTokensDType, std::optional<std::vector<TensorConstPtr>> const& lookaheadPrompt, std::optional<std::vector<tle::LookaheadDecodingConfig>> const& lookaheadAlgoConfigs) { diff --git a/cpp/tensorrt_llm/runtime/gptDecoderBatched.cpp b/cpp/tensorrt_llm/runtime/gptDecoderBatched.cpp index c55d02093afc..7b3a12ed7a2c 100644 --- a/cpp/tensorrt_llm/runtime/gptDecoderBatched.cpp +++ b/cpp/tensorrt_llm/runtime/gptDecoderBatched.cpp @@ -22,6 +22,7 @@ #include "tensorrt_llm/batch_manager/decoderBuffers.h" #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/kernels/decodingKernels.h" #include "tensorrt_llm/runtime/bufferManager.h" @@ -74,7 +75,7 @@ void GptDecoderBatched::disableLookahead(RequestVector const& genRequests, Tenso } void GptDecoderBatched::setup(executor::DecodingMode const& mode, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - nvinfer1::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) + tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); TLLM_CHECK(maxNumSequences > 0); diff --git a/cpp/tensorrt_llm/runtime/gptJsonConfig.cpp b/cpp/tensorrt_llm/runtime/gptJsonConfig.cpp index 311f63eaf1e7..47310a9a1282 100644 --- a/cpp/tensorrt_llm/runtime/gptJsonConfig.cpp +++ b/cpp/tensorrt_llm/runtime/gptJsonConfig.cpp @@ -20,6 +20,7 @@ #include "modelConfig.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/eagleModule.h" #include "tensorrt_llm/runtime/explicitDraftTokensModule.h" #include "tensorrt_llm/runtime/jsonSerialization.h" @@ -80,14 +81,14 @@ std::optional<FieldType> parseJsonFieldOptional(Json const& json, std::string_vi return value; } -nvinfer1::DataType strToDType(std::string type) +tensorrt_llm::DataType strToDType(std::string type) { - static std::map<std::string, nvinfer1::DataType> const typeMap = {{"int64", nvinfer1::DataType::kINT64}, - {"int32", nvinfer1::DataType::kINT32}, {"int", nvinfer1::DataType::kINT32}, - {"float32", nvinfer1::DataType::kFLOAT}, {"bfloat16", nvinfer1::DataType::kBF16}, - {"float16", nvinfer1::DataType::kHALF}, {"bool", nvinfer1::DataType::kBOOL}, - {"uint8", nvinfer1::DataType::kUINT8}, {"int8", nvinfer1::DataType::kINT8}, {"fp8", nvinfer1::DataType::kFP8}, - {"int4", nvinfer1::DataType::kINT4}}; + static std::map<std::string, tensorrt_llm::DataType> const typeMap = {{"int64", tensorrt_llm::DataType::kINT64}, + {"int32", tensorrt_llm::DataType::kINT32}, {"int", tensorrt_llm::DataType::kINT32}, + {"float32", tensorrt_llm::DataType::kFLOAT}, {"bfloat16", tensorrt_llm::DataType::kBF16}, + {"float16", tensorrt_llm::DataType::kHALF}, {"bool", tensorrt_llm::DataType::kBOOL}, + {"uint8", tensorrt_llm::DataType::kUINT8}, {"int8", tensorrt_llm::DataType::kINT8}, + {"fp8", tensorrt_llm::DataType::kFP8}, {"int4", tensorrt_llm::DataType::kINT4}}; TLLM_CHECK_WITH_INFO(typeMap.count(type) > 0, type + " not found in strToDtype."); return typeMap.at(type); @@ -140,14 +141,14 @@ std::vector<ModelConfig::LayerType> buildLayerTypes( return result; } -ModelConfig parseMultimodalConfig(Json const& json, nvinfer1::DataType dataType) +ModelConfig parseMultimodalConfig(Json const& json, tensorrt_llm::DataType dataType) { return ModelConfig{128, 10, 10, 0, 1, 128, dataType}; // use dummy values because vision engines of multimodal models does not record this info in config } ModelConfig createModelConfig(Json const& json, bool engineVersionNone, SizeType32 tensorParallelism, - SizeType32 contextParallelism, nvinfer1::DataType dataType) + SizeType32 contextParallelism, tensorrt_llm::DataType dataType) { auto const& config = engineVersionNone ? json.at("builder_config") : json.at("pretrained_config"); auto const multiModalName = parseJsonFieldOptional<std::string>(config, "model_name"); @@ -248,14 +249,14 @@ ModelConfig createModelConfig(Json const& json, bool engineVersionNone, SizeType modelConfig.setLayerTypes(layerTypes); // Set logits datatype - auto logitsDtype = nvinfer1::DataType::kFLOAT; + auto logitsDtype = tensorrt_llm::DataType::kFLOAT; if (logitsDtypeStr == "float32") { - logitsDtype = nvinfer1::DataType::kFLOAT; + logitsDtype = tensorrt_llm::DataType::kFLOAT; } else if (logitsDtypeStr == "float16") { - logitsDtype = nvinfer1::DataType::kHALF; + logitsDtype = tensorrt_llm::DataType::kHALF; } else { @@ -490,15 +491,15 @@ GptJsonConfig parseJson(InputType&& input) { if (precision == "float32") { - return nvinfer1::DataType::kFLOAT; + return tensorrt_llm::DataType::kFLOAT; } if (precision == "float16") { - return nvinfer1::DataType::kHALF; + return tensorrt_llm::DataType::kHALF; } if (precision == "bfloat16") { - return nvinfer1::DataType::kBF16; + return tensorrt_llm::DataType::kBF16; } TLLM_THROW("Model data type '%s' not supported", precision.c_str()); }(); diff --git a/cpp/tensorrt_llm/runtime/iBuffer.cpp b/cpp/tensorrt_llm/runtime/iBuffer.cpp index 77707a0e4cf8..82574b658b39 100644 --- a/cpp/tensorrt_llm/runtime/iBuffer.cpp +++ b/cpp/tensorrt_llm/runtime/iBuffer.cpp @@ -20,6 +20,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferView.h" #include <cuda_runtime_api.h> @@ -48,7 +49,7 @@ IBuffer::UniquePtr IBuffer::slice(IBuffer::SharedPtr buffer, std::size_t offset, return std::make_unique<BufferView>(std::move(buffer), offset, size); } -IBuffer::UniquePtr IBuffer::wrap(void* data, nvinfer1::DataType type, std::size_t size, std::size_t capacity) +IBuffer::UniquePtr IBuffer::wrap(void* data, tensorrt_llm::DataType type, std::size_t size, std::size_t capacity) { TLLM_CHECK_WITH_INFO(size <= capacity, "Requested size is larger than capacity"); auto memoryType = IBuffer::memoryType(data); @@ -91,17 +92,17 @@ char const* IBuffer::getDataTypeName(DataType dataType) { switch (dataType) { - case nvinfer1::DataType::kINT64: return DataTypeTraits<nvinfer1::DataType::kINT64>::name; - case nvinfer1::DataType::kINT32: return DataTypeTraits<nvinfer1::DataType::kINT32>::name; - case nvinfer1::DataType::kFLOAT: return DataTypeTraits<nvinfer1::DataType::kFLOAT>::name; - case nvinfer1::DataType::kBF16: return DataTypeTraits<nvinfer1::DataType::kBF16>::name; - case nvinfer1::DataType::kHALF: return DataTypeTraits<nvinfer1::DataType::kHALF>::name; - case nvinfer1::DataType::kBOOL: return DataTypeTraits<nvinfer1::DataType::kBOOL>::name; - case nvinfer1::DataType::kUINT8: return DataTypeTraits<nvinfer1::DataType::kUINT8>::name; - case nvinfer1::DataType::kINT8: return DataTypeTraits<nvinfer1::DataType::kINT8>::name; - case nvinfer1::DataType::kFP8: return DataTypeTraits<nvinfer1::DataType::kFP8>::name; - case nvinfer1::DataType::kINT4: [[fallthrough]] /* do nothing */; - case nvinfer1::DataType::kFP4: [[fallthrough]] /* do nothing */; + case tensorrt_llm::DataType::kINT64: return DataTypeTraits<tensorrt_llm::DataType::kINT64>::name; + case tensorrt_llm::DataType::kINT32: return DataTypeTraits<tensorrt_llm::DataType::kINT32>::name; + case tensorrt_llm::DataType::kFLOAT: return DataTypeTraits<tensorrt_llm::DataType::kFLOAT>::name; + case tensorrt_llm::DataType::kBF16: return DataTypeTraits<tensorrt_llm::DataType::kBF16>::name; + case tensorrt_llm::DataType::kHALF: return DataTypeTraits<tensorrt_llm::DataType::kHALF>::name; + case tensorrt_llm::DataType::kBOOL: return DataTypeTraits<tensorrt_llm::DataType::kBOOL>::name; + case tensorrt_llm::DataType::kUINT8: return DataTypeTraits<tensorrt_llm::DataType::kUINT8>::name; + case tensorrt_llm::DataType::kINT8: return DataTypeTraits<tensorrt_llm::DataType::kINT8>::name; + case tensorrt_llm::DataType::kFP8: return DataTypeTraits<tensorrt_llm::DataType::kFP8>::name; + case tensorrt_llm::DataType::kINT4: [[fallthrough]] /* do nothing */; + case tensorrt_llm::DataType::kFP4: [[fallthrough]] /* do nothing */; default: TLLM_THROW("Unknown data type"); } } diff --git a/cpp/tensorrt_llm/runtime/iTensor.cpp b/cpp/tensorrt_llm/runtime/iTensor.cpp index f78b25fdb19a..70b31707130a 100644 --- a/cpp/tensorrt_llm/runtime/iTensor.cpp +++ b/cpp/tensorrt_llm/runtime/iTensor.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/stringUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/tensorView.h" #include "tensorrt_llm/runtime/tllmBuffers.h" @@ -72,22 +73,22 @@ ITensor::UniquePtr ITensor::slice(SharedPtr tensor, Shape const& offsetDims, ITe return std::make_unique<TensorView>(std::move(tensor), offset, volume(dims), dims); } -ITensor::UniquePtr ITensor::view(IBuffer::SharedPtr buffer, nvinfer1::Dims const& dims) +ITensor::UniquePtr ITensor::view(IBuffer::SharedPtr buffer, tensorrt_llm::Dims const& dims) { auto const size = buffer->getSize(); return std::make_unique<TensorView>(std::move(buffer), 0, size, dims); } -nvinfer1::Dims ITensor::makeShape(std::initializer_list<ITensor::DimType64> const& dims) +tensorrt_llm::Dims ITensor::makeShape(std::initializer_list<ITensor::DimType64> const& dims) { - TLLM_CHECK_WITH_INFO(dims.size() <= nvinfer1::Dims::MAX_DIMS, "Number of dimensions is too large"); - nvinfer1::Dims shape{}; + TLLM_CHECK_WITH_INFO(dims.size() <= tensorrt_llm::Dims::MAX_DIMS, "Number of dimensions is too large"); + tensorrt_llm::Dims shape{}; shape.nbDims = static_cast<decltype(Shape::nbDims)>(dims.size()); std::copy(dims.begin(), dims.end(), shape.d); return shape; } -std::string ITensor::toString(nvinfer1::Dims const& dims) +std::string ITensor::toString(tensorrt_llm::Dims const& dims) { if (dims.nbDims < 0) { @@ -103,7 +104,8 @@ std::string ITensor::toString(nvinfer1::Dims const& dims) } } -ITensor::UniquePtr ITensor::wrap(void* data, nvinfer1::DataType type, nvinfer1::Dims const& shape, std::size_t capacity) +ITensor::UniquePtr ITensor::wrap( + void* data, tensorrt_llm::DataType type, tensorrt_llm::Dims const& shape, std::size_t capacity) { auto const size = volumeNonNegative(shape); TLLM_CHECK_WITH_INFO(size <= capacity, "Requested size is larger than capacity"); @@ -230,18 +232,18 @@ std::ostream& tensorrt_llm::runtime::operator<<(std::ostream& out, ITensor const { switch (tensor.getDataType()) { - case nvinfer1::DataType::kFLOAT: printTensor<float>(tensor, out); break; - case nvinfer1::DataType::kHALF: printTensor<half, float>(tensor, out); break; - case nvinfer1::DataType::kBOOL: printTensor<bool>(tensor, out); break; - case nvinfer1::DataType::kINT8: printTensor<std::int8_t, std::int32_t>(tensor, out); break; - case nvinfer1::DataType::kINT32: printTensor<std::int32_t>(tensor, out); break; - case nvinfer1::DataType::kINT64: printTensor<std::int64_t>(tensor, out); break; - case nvinfer1::DataType::kUINT8: printTensor<std::uint8_t, std::int32_t>(tensor, out); break; + case tensorrt_llm::DataType::kFLOAT: printTensor<float>(tensor, out); break; + case tensorrt_llm::DataType::kHALF: printTensor<half, float>(tensor, out); break; + case tensorrt_llm::DataType::kBOOL: printTensor<bool>(tensor, out); break; + case tensorrt_llm::DataType::kINT8: printTensor<std::int8_t, std::int32_t>(tensor, out); break; + case tensorrt_llm::DataType::kINT32: printTensor<std::int32_t>(tensor, out); break; + case tensorrt_llm::DataType::kINT64: printTensor<std::int64_t>(tensor, out); break; + case tensorrt_llm::DataType::kUINT8: printTensor<std::uint8_t, std::int32_t>(tensor, out); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: printTensor<__nv_bfloat16, float>(tensor, out); break; + case tensorrt_llm::DataType::kBF16: printTensor<__nv_bfloat16, float>(tensor, out); break; #endif #ifdef ENABLE_FP8 - case nvinfer1::DataType::kFP8: printTensor<__nv_fp8_e4m3, float>(tensor, out); break; + case tensorrt_llm::DataType::kFP8: printTensor<__nv_fp8_e4m3, float>(tensor, out); break; #endif default: TLLM_THROW("Unsupported data type"); } diff --git a/cpp/tensorrt_llm/runtime/ipcUtils.cpp b/cpp/tensorrt_llm/runtime/ipcUtils.cpp index 23a7e28a4f27..48368844f850 100644 --- a/cpp/tensorrt_llm/runtime/ipcUtils.cpp +++ b/cpp/tensorrt_llm/runtime/ipcUtils.cpp @@ -20,7 +20,7 @@ #include "tensorrt_llm/common/workspace.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include <NvInferRuntimeBase.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cstddef> namespace tensorrt_llm::runtime @@ -83,7 +83,7 @@ void IpcMemory::allocateIpcMemory(std::size_t bufferSize, BufferManager const& m // IPC handles. If we want to support stream-ordered allocations here, we need to create another pool with the // correct handle type. auto const ipcAlignedBufferSize = common::alignSize(bufferSize, 1LU << 21); - mBuffer = BufferManager::gpuSync(ipcAlignedBufferSize, nvinfer1::DataType::kUINT8); + mBuffer = BufferManager::gpuSync(ipcAlignedBufferSize, tensorrt_llm::DataType::kUINT8); manager.setZero(*mBuffer); auto* bufferPtr = mBuffer->data(); @@ -149,7 +149,7 @@ AllReduceBuffers::AllReduceBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWi { auto const tpSize = worldConfig.getTensorParallelism(); mAllReduceCommPtrs = BufferManager::cpu( - ITensor::makeShape({static_cast<SizeType32>(7) * tpSize + 3}), nvinfer1::DataType::kINT64); + ITensor::makeShape({static_cast<SizeType32>(7) * tpSize + 3}), tensorrt_llm::DataType::kINT64); } else { @@ -178,7 +178,7 @@ AllReduceBuffers::AllReduceBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWi mAllReduceCommPtrs = BufferManager::cpu(ITensor::makeShape({static_cast<SizeType32>(mIpcMemoryHandles.size()) * tpSize + 3}), - nvinfer1::DataType::kINT64); + tensorrt_llm::DataType::kINT64); auto commPtrs = BufferRange<void*>(*mAllReduceCommPtrs); // Start from 1 since 0 represents released state for barrier at the beginning of the all_reduce. // The last element is the barrier flag counter. @@ -211,9 +211,9 @@ AllReduceBuffers::AllReduceBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWi void lamportInitializeAll(void* buffer_0, void* buffer_1, void* buffer_2, size_t size) { #if ENABLE_MULTI_DEVICE - tensorrt_llm::kernels::lamportInitialize(buffer_0, size / sizeof(half), nvinfer1::DataType::kHALF, 0); - tensorrt_llm::kernels::lamportInitialize(buffer_1, size / sizeof(half), nvinfer1::DataType::kHALF, 0); - tensorrt_llm::kernels::lamportInitialize(buffer_2, size / sizeof(half), nvinfer1::DataType::kHALF, 0); + tensorrt_llm::kernels::lamportInitialize(buffer_0, size / sizeof(half), tensorrt_llm::DataType::kHALF, 0); + tensorrt_llm::kernels::lamportInitialize(buffer_1, size / sizeof(half), tensorrt_llm::DataType::kHALF, 0); + tensorrt_llm::kernels::lamportInitialize(buffer_2, size / sizeof(half), tensorrt_llm::DataType::kHALF, 0); cudaDeviceSynchronize(); #endif } diff --git a/cpp/tensorrt_llm/runtime/layerProfiler.cpp b/cpp/tensorrt_llm/runtime/layerProfiler.cpp deleted file mode 100644 index 4c3c9779cedb..000000000000 --- a/cpp/tensorrt_llm/runtime/layerProfiler.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/runtime/layerProfiler.h" -#include <iomanip> -#include <iostream> -#include <numeric> -#include <sstream> - -using namespace tensorrt_llm::runtime; - -void LayerProfiler::reportLayerTime(char const* layerName, float timeMs) noexcept -{ - if (mIterator == mLayers.end()) - { - bool const first = !mLayers.empty() && mLayers.begin()->name == layerName; - mUpdatesCount += mLayers.empty() || first; - if (first) - { - mIterator = mLayers.begin(); - } - else - { - mLayers.emplace_back(); - mLayers.back().name = layerName; - mIterator = mLayers.end() - 1; - } - } - - mIterator->timeMs.push_back(timeMs); - ++mIterator; -} - -float LayerProfiler::getTotalTime() const noexcept -{ - auto const plusLayerTime = [](float accumulator, LayerProfile const& lp) - { return accumulator + std::accumulate(lp.timeMs.begin(), lp.timeMs.end(), 0.F, std::plus<float>()); }; - return std::accumulate(mLayers.begin(), mLayers.end(), 0.0F, plusLayerTime); -} - -std::string LayerProfiler::getLayerProfile() noexcept -{ - std::string const nameHdr(" Layer"); - std::string const timeHdr(" Time(ms)"); - - float const totalTimeMs = getTotalTime(); - - auto const timeLength = timeHdr.size(); - - std::unordered_map<std::string, float> layer2times; - std::vector<std::string> layer_order; - for (auto const& p : mLayers) - { - if (!layer2times.count(p.name)) - { - layer2times[p.name] = 0; - layer_order.push_back(p.name); - } - for (auto const& t : p.timeMs) - { - layer2times[p.name] += t; - } - } - - std::stringstream ss; - ss << "\n=== Per-layer Profile ===\n" << timeHdr << nameHdr << "\n"; - - for (auto const& name : layer_order) - { - if (layer2times[name] == 0.0f) - { - continue; - } - ss << std::setw(timeLength) << std::fixed << std::setprecision(2) << layer2times[name] << " " << name << "\n"; - } - - ss << std::setw(timeLength) << std::fixed << std::setprecision(2) << totalTimeMs << " Total\n"; - ss << "\n"; - - // clear data - mLayers.clear(); - - return ss.str(); -} diff --git a/cpp/tensorrt_llm/runtime/layerProfiler.h b/cpp/tensorrt_llm/runtime/layerProfiler.h deleted file mode 100644 index bcae1546de3d..000000000000 --- a/cpp/tensorrt_llm/runtime/layerProfiler.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/runtime/common.h" -#include <vector> - -#include <NvInfer.h> - -namespace tensorrt_llm::runtime -{ -struct LayerProfile -{ - std::string name; - std::vector<float> timeMs; -}; - -class LayerProfiler : public nvinfer1::IProfiler -{ - -public: - void reportLayerTime(char const* layerName, float timeMs) noexcept override; - - std::string getLayerProfile() noexcept; - -private: - [[nodiscard]] float getTotalTime() const noexcept; - - std::vector<LayerProfile> mLayers; - std::vector<LayerProfile>::iterator mIterator{mLayers.begin()}; - int32_t mUpdatesCount{0}; -}; -} // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/runtime/lookaheadBuffers.cpp b/cpp/tensorrt_llm/runtime/lookaheadBuffers.cpp index ef800ef218e4..5e77046c47e0 100644 --- a/cpp/tensorrt_llm/runtime/lookaheadBuffers.cpp +++ b/cpp/tensorrt_llm/runtime/lookaheadBuffers.cpp @@ -16,208 +16,23 @@ */ #include "tensorrt_llm/runtime/lookaheadBuffers.h" -#include "tensorrt_llm/layers/lookaheadDecodingUtils.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" namespace tensorrt_llm::runtime { LookaheadDecodingBuffers::LookaheadDecodingBuffers( SizeType32 maxNumSequences, SizeType32 maxTokensPerStep, BufferManager const& bufferManager) - : generationLengths(bufferManager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32)) + : generationLengths(bufferManager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32)) , positionOffsets( - bufferManager.gpu(ITensor::makeShape({maxNumSequences, maxTokensPerStep}), nvinfer1::DataType::kINT32)) + bufferManager.gpu(ITensor::makeShape({maxNumSequences, maxTokensPerStep}), tensorrt_llm::DataType::kINT32)) , packedMasks(bufferManager.gpu(ITensor::makeShape({maxNumSequences, maxTokensPerStep, static_cast<ITensor::DimType64>(common::divUp(maxTokensPerStep, 32))}), - nvinfer1::DataType::kINT32)) + tensorrt_llm::DataType::kINT32)) , positionIds( - bufferManager.gpu(ITensor::makeShape({maxNumSequences, maxTokensPerStep}), nvinfer1::DataType::kINT32)) + bufferManager.gpu(ITensor::makeShape({maxNumSequences, maxTokensPerStep}), tensorrt_llm::DataType::kINT32)) { } -LookaheadRuntimeBuffers::LookaheadRuntimeBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, - BufferManager const& manager, ModelConfig const& modelConfig, WorldConfig const& worldConfig, - executor::DecodingConfig const& /* decodingConfig */, TllmRuntime const& runtime) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - TLLM_CHECK_WITH_INFO(maxBeamWidth == 1, "Lookahead decoding does not support beam search"); - - auto const tokensPerStep = modelConfig.getMaxDecodingTokens(); - auto const numPackedMasks = static_cast<ITensor::DimType64>(tensorrt_llm::common::divUp(tokensPerStep, 32)); - - cumSumLength = manager.pinned(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - - packedMasksDevice - = manager.gpu(ITensor::makeShape({maxBatchSize * tokensPerStep, numPackedMasks}), nvinfer1::DataType::kINT32); - positionOffsetsDevice = manager.gpu(ITensor::makeShape({maxBatchSize, tokensPerStep}), nvinfer1::DataType::kINT32); - generationLengthsDevice = manager.gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - positionIdsDevice = manager.gpu(ITensor::makeShape({maxBatchSize, tokensPerStep}), nvinfer1::DataType::kINT32); - - packedMaskHost = manager.cpu(packedMasksDevice->getShape(), nvinfer1::DataType::kINT32); - positionOffsetsHost = manager.cpu(positionOffsetsDevice->getShape(), nvinfer1::DataType::kINT32); - generationLengthsHost = manager.cpu(generationLengthsDevice->getShape(), nvinfer1::DataType::kINT32); - positionIdsHost = manager.cpu(positionIdsDevice->getShape(), nvinfer1::DataType::kINT32); - - packedMaskHostCopy = manager.cpu(packedMasksDevice->getShape(), nvinfer1::DataType::kINT32); - positionOffsetsHostCopy = manager.cpu(positionOffsetsDevice->getShape(), nvinfer1::DataType::kINT32); - generationLengthsHostCopy = manager.cpu(generationLengthsDevice->getShape(), nvinfer1::DataType::kINT32); - positionIdsHostCopy = manager.cpu(positionIdsDevice->getShape(), nvinfer1::DataType::kINT32); - - batchSlotsHostCopy = manager.cpu(generationLengthsDevice->getShape(), nvinfer1::DataType::kINT32); - - useSpecDecoding = manager.cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - bufferCast<SizeType32>(*useSpecDecoding)[0] = 1; - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void LookaheadRuntimeBuffers::setFromInputs(SizeType32 numCtxSequences, SizeType32 numGenSequences, - ITensor const& requestTypes, ITensor const& seqSlots, LookaheadDecodingBuffers const& decoderLookaheadBuffers, - TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig) const -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const& manager = runtime.getBufferManager(); - - auto const tokensPerStep = modelConfig.getMaxDecodingTokens(); - - manager.copy(seqSlots, *batchSlotsHostCopy); - manager.copy(*decoderLookaheadBuffers.generationLengths, *generationLengthsHostCopy); - manager.copy(*decoderLookaheadBuffers.positionOffsets, *positionOffsetsHostCopy); - manager.copy(*decoderLookaheadBuffers.packedMasks, *packedMaskHostCopy); - manager.copy(*decoderLookaheadBuffers.positionIds, *positionIdsHostCopy); - - manager.getStream().synchronize(); - - BufferRange<SizeType32 const> batchSlotsRange(*batchSlotsHostCopy); - BufferRange<SizeType32> cumSumLengthRange(*cumSumLength); - - SizeType32 maxGenerationLength = 0; - for (SizeType32 bi = 0; bi < numGenSequences; bi++) - { - SizeType32 gbi = batchSlotsRange[bi + numCtxSequences]; - SizeType32 theLength = BufferRange<SizeType32>(*generationLengthsHostCopy)[gbi]; - maxGenerationLength = std::max(maxGenerationLength, theLength); - } - - auto positionOffsetShape = positionOffsetsHost->getShape(); - positionOffsetShape.d[1] = maxGenerationLength; - positionOffsetsHost->reshape(positionOffsetShape); - positionOffsetsDevice->reshape(positionOffsetShape); - - auto positionIdsShape = positionIdsHostCopy->getShape(); - auto positionIdsShape1D = ITensor::makeShape({ITensor::volume(positionIdsShape)}); - positionIdsHostCopy->reshape(positionIdsShape1D); - positionIdsHost->reshape(positionIdsShape1D); - - cumSumLengthRange[0] = 0; - for (SizeType32 bi = 0; bi < numGenSequences; bi++) - { - SizeType32 gbi = batchSlotsRange[bi + numCtxSequences]; - SizeType32 theLength = BufferRange<SizeType32>(*generationLengthsHostCopy)[gbi]; - - manager.copy(*ITensor::at(generationLengthsHostCopy, {gbi}), *ITensor::at(generationLengthsHost, {bi})); - - manager.copy(*ITensor::slice(positionOffsetsHostCopy, {gbi, 0}, theLength), - *ITensor::slice(positionOffsetsHost, {bi, 0}, theLength)); - - manager.copy(*ITensor::slice(packedMaskHostCopy, gbi * tokensPerStep, theLength), - *ITensor::slice(packedMaskHost, cumSumLengthRange[0], theLength)); - - manager.copy(*ITensor::slice(positionIdsHostCopy, gbi * tokensPerStep, theLength), - *ITensor::slice(positionIdsHost, cumSumLengthRange[0], theLength)); - - cumSumLengthRange[0] += theLength; - } - - positionIdsHostCopy->reshape(positionIdsShape); - positionIdsHost->reshape(positionIdsShape); - positionIdsDevice->reshape(positionIdsShape); - - manager.copy(*ITensor::slice(generationLengthsHost, 0, numGenSequences), - *ITensor::slice(generationLengthsDevice, 0, numGenSequences)); - manager.copy(*ITensor::slice(positionOffsetsHost, 0, numGenSequences), - *ITensor::slice(positionOffsetsDevice, 0, numGenSequences)); - manager.copy(*ITensor::slice(packedMaskHost, 0, numGenSequences * tokensPerStep), - *ITensor::slice(packedMasksDevice, 0, numGenSequences * tokensPerStep)); - manager.copy( - *ITensor::slice(positionIdsHost, 0, numGenSequences), *ITensor::slice(positionIdsDevice, 0, numGenSequences)); - positionIdsDevice->reshape(ITensor::makeShape({cumSumLengthRange[0]})); - - manager.getStream().synchronize(); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void LookaheadRuntimeBuffers::reshape(SizeType32 numCtxSequences, SizeType32 numGenSequences, SizeType32 tokensPerStep) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const numSequences = numGenSequences; - - auto packedMaskShape = packedMasksDevice->getShape(); - packedMaskShape.d[0] = numSequences * tokensPerStep; - packedMasksDevice->reshape(packedMaskShape); - packedMaskHost->reshape(packedMaskShape); - - auto generationLengthsShape = generationLengthsDevice->getShape(); - generationLengthsShape.d[0] = numSequences; - generationLengthsDevice->reshape(generationLengthsShape); - generationLengthsHost->reshape(generationLengthsShape); - - auto positionOffsetsShape = positionOffsetsDevice->getShape(); - positionOffsetsShape.d[0] = numSequences; - positionOffsetsDevice->reshape(positionOffsetsShape); - positionOffsetsHost->reshape(positionOffsetsShape); - - auto positionIdsShape = positionIdsDevice->getShape(); - positionIdsShape.d[0] = numSequences; - positionIdsDevice->reshape(positionIdsShape); - positionIdsHost->reshape(positionIdsShape); - - auto batchSlotsShape = batchSlotsHostCopy->getShape(); - batchSlotsShape.d[0] = numCtxSequences + numGenSequences; - batchSlotsHostCopy->reshape(batchSlotsShape); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void LookaheadRuntimeBuffers::enableLookaheadDecoding(SizeType32 maxBatchSize, SizeType32 tokensPerStep) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - auto const numPackedMasks = static_cast<ITensor::DimType64>(tensorrt_llm::common::divUp(tokensPerStep, 32)); - packedMasksDevice->reshape(ITensor::makeShape({maxBatchSize * tokensPerStep, numPackedMasks})); - generationLengthsDevice->reshape(ITensor::makeShape({maxBatchSize})); - positionOffsetsDevice->reshape(ITensor::makeShape({maxBatchSize, tokensPerStep})); - bufferCast<SizeType32>(*useSpecDecoding)[0] = 1; - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void LookaheadRuntimeBuffers::disableLookaheadDecoding() -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - packedMasksDevice->reshape(ITensor::makeShape({1, 1})); - generationLengthsDevice->reshape(ITensor::makeShape({1})); - positionOffsetsDevice->reshape(ITensor::makeShape({1, 1})); - bufferCast<SizeType32>(*useSpecDecoding)[0] = 0; - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void LookaheadRuntimeBuffers::insertInputTensors( - TensorMap& inputBuffers, TensorMap& /* outputBuffers */, WorldConfig const& /* worldConfig */) const -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - - inputBuffers.insert_or_assign("spec_decoding_packed_mask", packedMasksDevice); - inputBuffers.insert_or_assign("spec_decoding_generation_lengths", generationLengthsDevice); - inputBuffers.insert_or_assign("spec_decoding_position_offsets", positionOffsetsDevice); - inputBuffers.insert_or_assign("spec_decoding_use", useSpecDecoding); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - } // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/runtime/loraCache.cpp b/cpp/tensorrt_llm/runtime/loraCache.cpp index 3dbb814f058b..36fb0363816f 100644 --- a/cpp/tensorrt_llm/runtime/loraCache.cpp +++ b/cpp/tensorrt_llm/runtime/loraCache.cpp @@ -23,6 +23,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/loraUtils.h" #include <memory> #include <mutex> @@ -537,15 +538,15 @@ void LoraCache::splitTransposeCpu(ITensor& output, ITensor const& input, SizeTyp switch (input.getDataType()) { - case nvinfer1::DataType::kINT32: splitTransposeCpuInner<SizeType32>(output, input, tpSize, tpRank); break; - case nvinfer1::DataType::kFLOAT: splitTransposeCpuInner<float>(output, input, tpSize, tpRank); break; - case nvinfer1::DataType::kHALF: splitTransposeCpuInner<half>(output, input, tpSize, tpRank); break; - case nvinfer1::DataType::kINT8: splitTransposeCpuInner<int8_t>(output, input, tpSize, tpRank); break; + case tensorrt_llm::DataType::kINT32: splitTransposeCpuInner<SizeType32>(output, input, tpSize, tpRank); break; + case tensorrt_llm::DataType::kFLOAT: splitTransposeCpuInner<float>(output, input, tpSize, tpRank); break; + case tensorrt_llm::DataType::kHALF: splitTransposeCpuInner<half>(output, input, tpSize, tpRank); break; + case tensorrt_llm::DataType::kINT8: splitTransposeCpuInner<int8_t>(output, input, tpSize, tpRank); break; #ifdef ENABLE_FP8 - case nvinfer1::DataType::kFP8: splitTransposeCpuInner<__nv_fp8_e4m3>(output, input, tpSize, tpRank); break; + case tensorrt_llm::DataType::kFP8: splitTransposeCpuInner<__nv_fp8_e4m3>(output, input, tpSize, tpRank); break; #endif // ENABLE_FP8 #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: splitTransposeCpuInner<__nv_bfloat16>(output, input, tpSize, tpRank); break; + case tensorrt_llm::DataType::kBF16: splitTransposeCpuInner<__nv_bfloat16>(output, input, tpSize, tpRank); break; #endif // ENABLE_BF16 default: TLLM_CHECK_WITH_INFO(false, "data type not supported"); } diff --git a/cpp/tensorrt_llm/runtime/loraManager.cpp b/cpp/tensorrt_llm/runtime/loraManager.cpp index 8d7ebe389853..1d25ea20c8e4 100644 --- a/cpp/tensorrt_llm/runtime/loraManager.cpp +++ b/cpp/tensorrt_llm/runtime/loraManager.cpp @@ -26,8 +26,6 @@ #include "tensorrt_llm/runtime/utils/runtimeUtils.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include <NvInferRuntime.h> - namespace tensorrt_llm::runtime { diff --git a/cpp/tensorrt_llm/runtime/loraUtils.cpp b/cpp/tensorrt_llm/runtime/loraUtils.cpp index 3c5e95162474..da7f1475ddec 100644 --- a/cpp/tensorrt_llm/runtime/loraUtils.cpp +++ b/cpp/tensorrt_llm/runtime/loraUtils.cpp @@ -17,6 +17,7 @@ #include "tensorrt_llm/runtime/loraUtils.h" #include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/modelConfig.h" @@ -57,7 +58,7 @@ void loraValidateRequestTensorDims(std::optional<ITensor::SharedPtr> const& optR keys->getShape().d[0] == expectedBatchSize, "Expected batch dimension to be 1 for each lora request"); TLLM_CHECK_WITH_INFO(weights->getMemoryType() != MemoryType::kGPU, "Expected lora weights to be in CPU memory"); TLLM_CHECK_WITH_INFO(keys->getMemoryType() != MemoryType::kGPU, "Expected lora weights to be in CPU memory"); - TLLM_CHECK_WITH_INFO(keys->getDataType() == nvinfer1::DataType::kINT32, + TLLM_CHECK_WITH_INFO(keys->getDataType() == tensorrt_llm::DataType::kINT32, "Expected lora keys to have TYPE_INT32 but was " + std::string(keys->getDataTypeName())); TLLM_CHECK_WITH_INFO(keys->getShape().d[1] == weights->getShape().d[1], diff --git a/cpp/tensorrt_llm/runtime/ncclCommunicator.cpp b/cpp/tensorrt_llm/runtime/ncclCommunicator.cpp index b76efb75952a..7edfba42f935 100644 --- a/cpp/tensorrt_llm/runtime/ncclCommunicator.cpp +++ b/cpp/tensorrt_llm/runtime/ncclCommunicator.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/ipcNvlsMemory.h" #include "tensorrt_llm/runtime/utils/multiDeviceUtils.h" @@ -102,19 +103,19 @@ void initNcclCommProbeWithTimeout(ncclUniqueId const& id, int worldSize, int ran } } -ncclDataType_t toNcclType(nvinfer1::DataType dataType) +ncclDataType_t toNcclType(tensorrt_llm::DataType dataType) { switch (dataType) { - case nvinfer1::DataType::kFLOAT: return ncclFloat32; - case nvinfer1::DataType::kHALF: return ncclHalf; - case nvinfer1::DataType::kINT8: return ncclInt8; - case nvinfer1::DataType::kINT32: return ncclInt32; - case nvinfer1::DataType::kUINT8: return ncclUint8; - case nvinfer1::DataType::kINT64: return ncclInt64; - case nvinfer1::DataType::kFP8: return ncclUint8; + case tensorrt_llm::DataType::kFLOAT: return ncclFloat32; + case tensorrt_llm::DataType::kHALF: return ncclHalf; + case tensorrt_llm::DataType::kINT8: return ncclInt8; + case tensorrt_llm::DataType::kINT32: return ncclInt32; + case tensorrt_llm::DataType::kUINT8: return ncclUint8; + case tensorrt_llm::DataType::kINT64: return ncclInt64; + case tensorrt_llm::DataType::kFP8: return ncclUint8; #if ENABLE_BF16 - case nvinfer1::DataType::kBF16: return ncclBfloat16; + case tensorrt_llm::DataType::kBF16: return ncclBfloat16; #endif // ENABLE_BF16 default: TLLM_THROW("Unsupported data type: %d", static_cast<int>(dataType)); } @@ -123,7 +124,7 @@ ncclDataType_t toNcclType(nvinfer1::DataType dataType) } // namespace void NcclCommunicator::send( - void const* sendbuff, size_t count, nvinfer1::DataType dataType, int peer, CudaStream const& stream) const + void const* sendbuff, size_t count, tensorrt_llm::DataType dataType, int peer, CudaStream const& stream) const { #if ENABLE_MULTI_DEVICE TLLM_NCCL_CHECK(ncclSend(sendbuff, count, toNcclType(dataType), peer, mComm, stream.get())); @@ -133,7 +134,7 @@ void NcclCommunicator::send( } void NcclCommunicator::receive( - void* sendbuff, size_t count, nvinfer1::DataType dataType, int peer, CudaStream const& stream) const + void* sendbuff, size_t count, tensorrt_llm::DataType dataType, int peer, CudaStream const& stream) const { #if ENABLE_MULTI_DEVICE TLLM_NCCL_CHECK(ncclRecv(sendbuff, count, toNcclType(dataType), peer, mComm, stream.get())); diff --git a/cpp/tensorrt_llm/runtime/ncclCommunicator.h b/cpp/tensorrt_llm/runtime/ncclCommunicator.h index 76cce4beab8a..21d7f116e95b 100644 --- a/cpp/tensorrt_llm/runtime/ncclCommunicator.h +++ b/cpp/tensorrt_llm/runtime/ncclCommunicator.h @@ -16,6 +16,7 @@ #pragma once +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" @@ -57,9 +58,10 @@ class NcclCommunicator private: void send( - void const* sendbuff, size_t count, nvinfer1::DataType dataType, int peer, CudaStream const& stream) const; + void const* sendbuff, size_t count, tensorrt_llm::DataType dataType, int peer, CudaStream const& stream) const; - void receive(void* sendbuff, size_t count, nvinfer1::DataType dataType, int peer, CudaStream const& stream) const; + void receive( + void* sendbuff, size_t count, tensorrt_llm::DataType dataType, int peer, CudaStream const& stream) const; static ncclComm_t createComm(int worldSize, int rank, mpi::MpiComm const& mpiComm); diff --git a/cpp/tensorrt_llm/runtime/runtimeKernels.cu b/cpp/tensorrt_llm/runtime/runtimeKernels.cu index 3b3dbcac894a..b22d36052370 100644 --- a/cpp/tensorrt_llm/runtime/runtimeKernels.cu +++ b/cpp/tensorrt_llm/runtime/runtimeKernels.cu @@ -21,7 +21,7 @@ #include "tensorrt_llm/kernels/speculativeDecoding/kvCacheUpdateKernels.h" #include "tensorrt_llm/runtime/runtimeKernels.h" -#include <NvInferRuntimeBase.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_runtime.h> using namespace tensorrt_llm::runtime; @@ -333,13 +333,13 @@ void invokeFillBatch(IBuffer& buffer, IBuffer const& slotIndices, std::size_t sl { switch (buffer.getDataType()) { - case nvinfer1::DataType::kINT32: + case tensorrt_llm::DataType::kINT32: invokeFillBatch<std::int32_t>(buffer, slotIndices, slotStride, values, stream); break; - case nvinfer1::DataType::kINT8: + case tensorrt_llm::DataType::kINT8: invokeFillBatch<std::int8_t>(buffer, slotIndices, slotStride, values, stream); break; - case nvinfer1::DataType::kFLOAT: invokeFillBatch<float>(buffer, slotIndices, slotStride, values, stream); break; + case tensorrt_llm::DataType::kFLOAT: invokeFillBatch<float>(buffer, slotIndices, slotStride, values, stream); break; default: TLLM_THROW("data type not supported"); } } @@ -349,13 +349,15 @@ void invokeGatherBatch(IBuffer& buffer, IBuffer const& values, IBuffer const& sl { switch (buffer.getDataType()) { - case nvinfer1::DataType::kINT32: + case tensorrt_llm::DataType::kINT32: invokeGatherBatch<std::int32_t>(buffer, values, slotIndices, slotStride, stream); break; - case nvinfer1::DataType::kINT8: + case tensorrt_llm::DataType::kINT8: invokeGatherBatch<std::int8_t>(buffer, values, slotIndices, slotStride, stream); break; - case nvinfer1::DataType::kFLOAT: invokeGatherBatch<float>(buffer, values, slotIndices, slotStride, stream); break; + case tensorrt_llm::DataType::kFLOAT: + invokeGatherBatch<float>(buffer, values, slotIndices, slotStride, stream); + break; default: TLLM_THROW("data type not supported"); } } @@ -408,12 +410,12 @@ void scatterTensor(ITensor& output, ITensor const& input, SizeType32 beamWidth, { switch (input.getDataType()) { - case nvinfer1::DataType::kINT32: invokeScatterTensor<SizeType32>(output, input, beamWidth, stream); break; - case nvinfer1::DataType::kFLOAT: invokeScatterTensor<float>(output, input, beamWidth, stream); break; - case nvinfer1::DataType::kHALF: invokeScatterTensor<half>(output, input, beamWidth, stream); break; - case nvinfer1::DataType::kINT8: invokeScatterTensor<int8_t>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kINT32: invokeScatterTensor<SizeType32>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kFLOAT: invokeScatterTensor<float>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kHALF: invokeScatterTensor<half>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kINT8: invokeScatterTensor<int8_t>(output, input, beamWidth, stream); break; #ifdef ENABLE_FP8 - case nvinfer1::DataType::kFP8: invokeScatterTensor<__nv_fp8_e4m3>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kFP8: invokeScatterTensor<__nv_fp8_e4m3>(output, input, beamWidth, stream); break; #endif // ENABLE_FP8 default: TLLM_THROW("data type not supported"); } @@ -423,15 +425,15 @@ void tileTensor(ITensor& output, ITensor const& input, SizeType32 beamWidth, Cud { switch (input.getDataType()) { - case nvinfer1::DataType::kINT32: invokeTileTensor<SizeType32>(output, input, beamWidth, stream); break; - case nvinfer1::DataType::kFLOAT: invokeTileTensor<float>(output, input, beamWidth, stream); break; - case nvinfer1::DataType::kHALF: invokeTileTensor<half>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kINT32: invokeTileTensor<SizeType32>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kFLOAT: invokeTileTensor<float>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kHALF: invokeTileTensor<half>(output, input, beamWidth, stream); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: invokeTileTensor<__nv_bfloat16>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kBF16: invokeTileTensor<__nv_bfloat16>(output, input, beamWidth, stream); break; #endif // ENABLE_BF16 - case nvinfer1::DataType::kINT8: invokeTileTensor<int8_t>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kINT8: invokeTileTensor<int8_t>(output, input, beamWidth, stream); break; #ifdef ENABLE_FP8 - case nvinfer1::DataType::kFP8: invokeTileTensor<__nv_fp8_e4m3>(output, input, beamWidth, stream); break; + case tensorrt_llm::DataType::kFP8: invokeTileTensor<__nv_fp8_e4m3>(output, input, beamWidth, stream); break; #endif // ENABLE_FP8 default: TLLM_THROW("data type not supported"); } @@ -444,22 +446,22 @@ void mergeLogitsFragments(BufferManager const& bufferManager, ITensor& output, { switch (output.getDataType()) { - case nvinfer1::DataType::kFLOAT: + case tensorrt_llm::DataType::kFLOAT: invokeMergeLogitsFragments<float>(bufferManager, output, fragmentsVector, cachePointerDevice, cachePointerHost, firstBatchSlotIdx, microBatchSize, beamWidth, stream, stepOffset); break; - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: invokeMergeLogitsFragments<half>(bufferManager, output, fragmentsVector, cachePointerDevice, cachePointerHost, firstBatchSlotIdx, microBatchSize, beamWidth, stream, stepOffset); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: invokeMergeLogitsFragments<__nv_bfloat16>(bufferManager, output, fragmentsVector, cachePointerDevice, cachePointerHost, firstBatchSlotIdx, microBatchSize, beamWidth, stream, stepOffset); break; #endif // ENABLE_BF16 #ifdef ENABLE_FP8 - case nvinfer1::DataType::kFP8: + case tensorrt_llm::DataType::kFP8: invokeMergeLogitsFragments<__nv_fp8_e4m3>(bufferManager, output, fragmentsVector, cachePointerDevice, cachePointerHost, firstBatchSlotIdx, microBatchSize, beamWidth, stream, stepOffset); break; diff --git a/cpp/tensorrt_llm/runtime/tensorView.h b/cpp/tensorrt_llm/runtime/tensorView.h index 17e7fb719415..d9e65b0efe23 100644 --- a/cpp/tensorrt_llm/runtime/tensorView.h +++ b/cpp/tensorrt_llm/runtime/tensorView.h @@ -16,6 +16,7 @@ #pragma once +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferView.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -45,19 +46,19 @@ class TensorView : virtual public ITensor, public BufferView mDims.d[0] = size; } - TensorView(IBuffer::SharedPtr const& buffer, size_t offset, size_t size, nvinfer1::Dims const& dims) + TensorView(IBuffer::SharedPtr const& buffer, size_t offset, size_t size, tensorrt_llm::Dims const& dims) : BufferView{buffer, offset, size} , mDims{dims} { Base::resize(ITensor::volumeNonNegative(dims)); } - [[nodiscard]] nvinfer1::Dims const& getShape() const override + [[nodiscard]] tensorrt_llm::Dims const& getShape() const override { return mDims; } - void reshape(nvinfer1::Dims const& dims) override + void reshape(tensorrt_llm::Dims const& dims) override { Base::resize(ITensor::volumeNonNegative(dims)); mDims = dims; @@ -81,6 +82,6 @@ class TensorView : virtual public ITensor, public BufferView return shape.nbDims > 0 && shape.d[0] > 0 ? ITensor::volume(shape) / shape.d[0] : 0; } - nvinfer1::Dims mDims{}; + tensorrt_llm::Dims mDims{}; }; } // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/runtime/tllmBuffers.cpp b/cpp/tensorrt_llm/runtime/tllmBuffers.cpp index ff7ed04001d3..4876d5b87bf6 100644 --- a/cpp/tensorrt_llm/runtime/tllmBuffers.cpp +++ b/cpp/tensorrt_llm/runtime/tllmBuffers.cpp @@ -15,6 +15,7 @@ */ #include "tensorrt_llm/runtime/tllmBuffers.h" +#include "tensorrt_llm/common/tllmDataType.h" namespace tensorrt_llm::runtime { @@ -62,12 +63,12 @@ std::shared_ptr<MulticastBuffer> MulticastTensorView::lock() const /////////////////////////////////////// // MulticastTensorView ITensor methods /////////////////////////////////////// -nvinfer1::Dims const& MulticastTensorView::getShape() const +tensorrt_llm::Dims const& MulticastTensorView::getShape() const { return mDims; } -void MulticastTensorView::reshape(nvinfer1::Dims const& dims) +void MulticastTensorView::reshape(tensorrt_llm::Dims const& dims) { auto new_size = nonNegative(volume(dims)); if (new_size > getCapacity()) @@ -102,7 +103,7 @@ std::size_t MulticastTensorView::getCapacity() const return lock()->getCapacity(); } -nvinfer1::DataType MulticastTensorView::getDataType() const +tensorrt_llm::DataType MulticastTensorView::getDataType() const { return lock()->getDataType(); } diff --git a/cpp/tensorrt_llm/runtime/tllmBuffers.h b/cpp/tensorrt_llm/runtime/tllmBuffers.h index faed36537e5c..d023823de5b2 100644 --- a/cpp/tensorrt_llm/runtime/tllmBuffers.h +++ b/cpp/tensorrt_llm/runtime/tllmBuffers.h @@ -27,7 +27,7 @@ #include "tensorrt_llm/runtime/memoryCounters.h" #include "tensorrt_llm/runtime/virtualMemory.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_runtime_api.h> #include <algorithm> @@ -550,7 +550,7 @@ class GenericBuffer : virtual public IBuffer, TAllocator // Inherit from TAlloca //! //! \brief Construct an empty buffer. //! - explicit GenericBuffer(nvinfer1::DataType type, TAllocator allocator = {}) // NOLINT(*-pro-type-member-init) + explicit GenericBuffer(tensorrt_llm::DataType type, TAllocator allocator = {}) // NOLINT(*-pro-type-member-init) : GenericBuffer{0, type, std::move(allocator)} { } @@ -559,7 +559,7 @@ class GenericBuffer : virtual public IBuffer, TAllocator // Inherit from TAlloca //! \brief Construct a buffer with the specified allocation size in number of elements. //! explicit GenericBuffer( // NOLINT(*-pro-type-member-init) - std::size_t size, nvinfer1::DataType type, TAllocator allocator = {}) + std::size_t size, tensorrt_llm::DataType type, TAllocator allocator = {}) : GenericBuffer{size, size, type, std::move(allocator)} { } @@ -636,7 +636,7 @@ class GenericBuffer : virtual public IBuffer, TAllocator // Inherit from TAlloca //! //! \brief Returns the type of the buffer. //! - [[nodiscard]] nvinfer1::DataType getDataType() const override + [[nodiscard]] tensorrt_llm::DataType getDataType() const override { return mType; } @@ -687,7 +687,8 @@ class GenericBuffer : virtual public IBuffer, TAllocator // Inherit from TAlloca } protected: - explicit GenericBuffer(std::size_t size, std::size_t capacity, nvinfer1::DataType type, TAllocator allocator = {}) + explicit GenericBuffer( + std::size_t size, std::size_t capacity, tensorrt_llm::DataType type, TAllocator allocator = {}) : TAllocator{std::move(allocator)} , mSize{size} , mCapacity{capacity} @@ -700,14 +701,14 @@ class GenericBuffer : virtual public IBuffer, TAllocator // Inherit from TAlloca private: std::size_t mSize{0}, mCapacity{0}; - nvinfer1::DataType mType; + tensorrt_llm::DataType mType; void* mBuffer; }; class MulticastBuffer : virtual public IBuffer { public: - explicit MulticastBuffer(nvinfer1::DataType type, std::set<int> const& ranks) + explicit MulticastBuffer(tensorrt_llm::DataType type, std::set<int> const& ranks) : mSize(0) , mCapacity(0) , mType(type) @@ -716,7 +717,7 @@ class MulticastBuffer : virtual public IBuffer TLLM_CHECK(ranks.size() > 1); } - explicit MulticastBuffer(size_t size, nvinfer1::DataType type, std::set<int> const& ranks) + explicit MulticastBuffer(size_t size, tensorrt_llm::DataType type, std::set<int> const& ranks) : mSize(0) , mCapacity(0) , mType(type) @@ -817,7 +818,7 @@ class MulticastBuffer : virtual public IBuffer return mCapacity; } - [[nodiscard]] nvinfer1::DataType getDataType() const override + [[nodiscard]] tensorrt_llm::DataType getDataType() const override { return mType; } @@ -853,7 +854,7 @@ class MulticastBuffer : virtual public IBuffer private: std::size_t mSize = 0; std::size_t mCapacity = 0; - nvinfer1::DataType mType; + tensorrt_llm::DataType mType; std::set<int> mRanks; IpcNvlsHandle* mHandle; }; @@ -882,7 +883,7 @@ class GenericTensor : virtual public ITensor, public GenericBuffer<TAllocator> //! //! \brief Construct an empty tensor. //! - explicit GenericTensor(nvinfer1::DataType type, TAllocator allocator = {}) + explicit GenericTensor(tensorrt_llm::DataType type, TAllocator allocator = {}) : Base{type, std::move(allocator)} { mDims.nbDims = 0; @@ -891,14 +892,14 @@ class GenericTensor : virtual public ITensor, public GenericBuffer<TAllocator> //! //! \brief Construct a tensor with the specified allocation dimensions. //! - explicit GenericTensor(nvinfer1::Dims const& dims, nvinfer1::DataType type, TAllocator allocator = {}) + explicit GenericTensor(tensorrt_llm::Dims const& dims, tensorrt_llm::DataType type, TAllocator allocator = {}) : Base{nonNegative(volume(dims)), type, std::move(allocator)} , mDims{dims} { } explicit GenericTensor( - nvinfer1::Dims const& dims, std::size_t capacity, nvinfer1::DataType type, TAllocator allocator = {}) + tensorrt_llm::Dims const& dims, std::size_t capacity, tensorrt_llm::DataType type, TAllocator allocator = {}) : Base{nonNegative(volume(dims)), capacity, type, std::move(allocator)} , mDims{dims} { @@ -923,12 +924,12 @@ class GenericTensor : virtual public ITensor, public GenericBuffer<TAllocator> return *this; } - [[nodiscard]] nvinfer1::Dims const& getShape() const override + [[nodiscard]] tensorrt_llm::Dims const& getShape() const override { return mDims; } - void reshape(nvinfer1::Dims const& dims) override + void reshape(tensorrt_llm::Dims const& dims) override { Base::resize(nonNegative(volume(dims))); mDims = dims; @@ -946,7 +947,7 @@ class GenericTensor : virtual public ITensor, public GenericBuffer<TAllocator> } private: - nvinfer1::Dims mDims{}; + tensorrt_llm::Dims mDims{}; }; // Forward declaration @@ -971,9 +972,9 @@ class MulticastTensorView : virtual public ITensor ///////////////////// // ITensor methods ///////////////////// - [[nodiscard]] nvinfer1::Dims const& getShape() const override; + [[nodiscard]] tensorrt_llm::Dims const& getShape() const override; - void reshape(nvinfer1::Dims const& dims) override; + void reshape(tensorrt_llm::Dims const& dims) override; ///////////////////// // IBuffer methods @@ -983,7 +984,7 @@ class MulticastTensorView : virtual public ITensor [[nodiscard]] std::size_t getCapacity() const override; - [[nodiscard]] nvinfer1::DataType getDataType() const override; + [[nodiscard]] tensorrt_llm::DataType getDataType() const override; [[nodiscard]] MemoryType getMemoryType() const override; @@ -1016,7 +1017,7 @@ class MulticastTensorView : virtual public ITensor std::weak_ptr<MulticastTensor> mTensor; ViewType mViewType; - nvinfer1::Dims mDims{}; + tensorrt_llm::Dims mDims{}; }; class MulticastTensor : virtual public ITensor, @@ -1026,13 +1027,13 @@ class MulticastTensor : virtual public ITensor, public: using Base = MulticastBuffer; - explicit MulticastTensor(nvinfer1::DataType type, std::set<int> const& ranks) + explicit MulticastTensor(tensorrt_llm::DataType type, std::set<int> const& ranks) : Base(type, ranks) { mDims.nbDims = 0; } - explicit MulticastTensor(nvinfer1::Dims const& dims, nvinfer1::DataType type, std::set<int> const& ranks) + explicit MulticastTensor(tensorrt_llm::Dims const& dims, tensorrt_llm::DataType type, std::set<int> const& ranks) : Base(nonNegative(volume(dims)), type, ranks) , mDims(dims) { @@ -1068,12 +1069,12 @@ class MulticastTensor : virtual public ITensor, ///////////////////// // ITensor methods ///////////////////// - [[nodiscard]] nvinfer1::Dims const& getShape() const override + [[nodiscard]] tensorrt_llm::Dims const& getShape() const override { return mDims; } - void reshape(nvinfer1::Dims const& dims) override + void reshape(tensorrt_llm::Dims const& dims) override { Base::resize(nonNegative(volume(dims))); mDims = dims; @@ -1091,7 +1092,7 @@ class MulticastTensor : virtual public ITensor, } private: - nvinfer1::Dims mDims{}; + tensorrt_llm::Dims mDims{}; }; using DeviceTensor = GenericTensor<CudaAllocatorAsync>; diff --git a/cpp/tensorrt_llm/runtime/tllmLogger.cpp b/cpp/tensorrt_llm/runtime/tllmLogger.cpp deleted file mode 100644 index 586ab2f4ae95..000000000000 --- a/cpp/tensorrt_llm/runtime/tllmLogger.cpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "tllmLogger.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" - -using namespace tensorrt_llm::runtime; -namespace tc = tensorrt_llm::common; - -void TllmLogger::log(nvinfer1::ILogger::Severity severity, nvinfer1::AsciiChar const* msg) noexcept -{ - switch (severity) - { - case nvinfer1::ILogger::Severity::kINTERNAL_ERROR: - case nvinfer1::ILogger::Severity::kERROR: TLLM_LOG_ERROR(msg); break; - case nvinfer1::ILogger::Severity::kWARNING: TLLM_LOG_WARNING(msg); break; - case nvinfer1::ILogger::Severity::kINFO: TLLM_LOG_INFO(msg); break; - case nvinfer1::ILogger::Severity::kVERBOSE: TLLM_LOG_DEBUG(msg); break; - default: TLLM_LOG_TRACE(msg); break; - } -} - -nvinfer1::ILogger::Severity TllmLogger::getLevel() -{ - auto* const logger = tc::Logger::getLogger(); - switch (logger->getLevel()) - { - case tc::Logger::Level::ERROR: return nvinfer1::ILogger::Severity::kERROR; - case tc::Logger::Level::WARNING: return nvinfer1::ILogger::Severity::kWARNING; - case tc::Logger::Level::INFO: return nvinfer1::ILogger::Severity::kINFO; - case tc::Logger::Level::DEBUG: - case tc::Logger::Level::TRACE: return nvinfer1::ILogger::Severity::kVERBOSE; - default: return nvinfer1::ILogger::Severity::kINTERNAL_ERROR; - } -} - -void TllmLogger::setLevel(nvinfer1::ILogger::Severity level) -{ - auto* const logger = tc::Logger::getLogger(); - switch (level) - { - case nvinfer1::ILogger::Severity::kINTERNAL_ERROR: - case nvinfer1::ILogger::Severity::kERROR: logger->setLevel(tc::Logger::Level::ERROR); break; - case nvinfer1::ILogger::Severity::kWARNING: logger->setLevel(tc::Logger::Level::WARNING); break; - case nvinfer1::ILogger::Severity::kINFO: logger->setLevel(tc::Logger::Level::INFO); break; - case nvinfer1::ILogger::Severity::kVERBOSE: logger->setLevel(tc::Logger::Level::TRACE); break; - default: TLLM_THROW("Unsupported severity"); - } -} diff --git a/cpp/tensorrt_llm/runtime/tllmRuntime.cpp b/cpp/tensorrt_llm/runtime/tllmRuntime.cpp deleted file mode 100644 index 7c2ca4747213..000000000000 --- a/cpp/tensorrt_llm/runtime/tllmRuntime.cpp +++ /dev/null @@ -1,831 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "tllmRuntime.h" -#include "common.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/common/safetensors.h" -#include "tensorrt_llm/executor/tensor.h" -#include "tensorrt_llm/kernels/userbuffers/ub_interface.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "tllmLogger.h" -#include "tllmStreamReaders.h" - -#include "nlohmann/json.hpp" -#include <NvInferRuntime.h> - -#include <algorithm> -#include <cstddef> -#include <limits> -#include <memory> -#include <optional> -#include <string> -#include <type_traits> -#include <utility> -#include <vector> - -using namespace tensorrt_llm::runtime; -using TensorMap = StringPtrMap<ITensor>; - -namespace -{ -static_assert(std::is_signed<SizeType32>::value, "SizeType32 must be signed"); - -nvinfer1::Dims shapeToDims(std::vector<std::size_t> const& shape) -{ - TLLM_CHECK(shape.size() <= nvinfer1::Dims::MAX_DIMS); - nvinfer1::Dims dims; - auto constexpr dim_max = std::numeric_limits<ITensor::DimType64>::max(); - dims.nbDims = static_cast<std::int32_t>(shape.size()); - for (std::size_t i = 0; i < shape.size(); ++i) - { - // shape[i] >= 0 because it has unsigned type. Check upper bound: - TLLM_CHECK(shape[i] <= static_cast<std::size_t>(dim_max)); - dims.d[i] = static_cast<ITensor::DimType64>(shape[i]); - } - return dims; -} - -std::vector<std::size_t> dimsToShape(nvinfer1::Dims const& dims) -{ - TLLM_CHECK(dims.nbDims >= 0); - std::vector<std::size_t> shape(dims.nbDims); - for (std::int32_t i = 0; i < dims.nbDims; ++i) - { - TLLM_CHECK(dims.d[i] >= 0); - shape[i] = static_cast<std::size_t>(dims.d[i]); - } - return shape; -} - -tensorrt_llm::runtime::TllmLogger defaultLogger{}; - -void setWeightStreaming(nvinfer1::ICudaEngine& engine, float const gpuWeightsPercent) -{ - if (gpuWeightsPercent < 1) - { - int64_t streamableSize = engine.getStreamableWeightsSize(); - int64_t budget = gpuWeightsPercent * streamableSize; - TLLM_LOG_INFO("Set gpu weights percent to %f, which is %lld bytes. Valid range: %lld bytes - %lld bytes.", - gpuWeightsPercent, budget, 0, streamableSize); - engine.setWeightStreamingBudgetV2(budget); - } -} - -class LayerInfo -{ -public: - LayerInfo(std::optional<std::string> name, std::string type) - : name(std::move(name)) - , type(std::move(type)){}; - std::optional<std::string> name; - std::string type; -}; - -void assessLikelihoodOfRuntimeAllocation( - nvinfer1::ICudaEngine const& engine, nvinfer1::IEngineInspector const& engineInspector) - -{ - TLLM_LOG_INFO("Inspecting the engine to identify potential runtime issues..."); - auto const profilingVerbosity = engine.getProfilingVerbosity(); - if (profilingVerbosity != nvinfer1::ProfilingVerbosity::kDETAILED) - { - TLLM_LOG_INFO( - "The profiling verbosity of the engine does not allow this analysis to proceed. Re-build the engine with " - "'detailed' profiling verbosity to get more diagnostics."); - return; - } - auto const* const layerTypeKey = "LayerType"; - auto const* const nameKey = "Name"; - auto const numLayers = engine.getNbLayers(); - TLLM_LOG_INFO("Model has %i layers.", numLayers); - std::vector<SizeType32> indexes(numLayers); - std::iota(indexes.begin(), indexes.end(), 0); - std::vector<std::optional<LayerInfo>> layerInfos(numLayers); - std::transform(indexes.cbegin(), indexes.cend(), layerInfos.begin(), - [&](SizeType32 const idx) - { - auto const* const layerInfo - = engineInspector.getLayerInformation(idx, nvinfer1::LayerInformationFormat::kJSON); - - // Needs to be copied explicitly, see documentation of `getLayerInformation`. - auto const layerInfoCopy = std::string(layerInfo); - auto const jsonLayerInfo = nlohmann::json::parse(layerInfoCopy); - auto const layerJsonType = jsonLayerInfo.type(); - if (layerJsonType != nlohmann::detail::value_t::object) - { - return std::optional<LayerInfo>{}; - } - if (!jsonLayerInfo.contains(layerTypeKey)) - { - return std::optional<LayerInfo>{}; - } - auto const& typeJson = jsonLayerInfo.at(layerTypeKey); - if (typeJson.type() != nlohmann::detail::value_t::string) - { - return std::optional<LayerInfo>{}; - } - std::optional<std::string> name{}; - if (jsonLayerInfo.contains(nameKey)) - { - auto const& nameJson = jsonLayerInfo.at(nameKey); - auto const nameJsonType = nameJson.type(); - if (nameJsonType == nlohmann::detail::value_t::string) - { - name = nameJson.get<std::string>(); - } - } - return std::make_optional(LayerInfo{name, typeJson.get<std::string>()}); - }); - auto const layersWithInfoEnd = std::partition( - layerInfos.begin(), layerInfos.end(), [](std::optional<LayerInfo> const& info) { return info.has_value(); }); - if (layersWithInfoEnd == layerInfos.begin()) - { - TLLM_LOG_INFO("Engine layer infos could not be parsed into useful information."); - return; - } - auto const allocateLayersEnd = std::partition(layerInfos.begin(), layersWithInfoEnd, - [](std::optional<LayerInfo> const& info) { return info.value().type == "allocate"; }); - auto numWarnings = 0; - for (auto layerInfo = layerInfos.begin(); layerInfo != allocateLayersEnd; layerInfo++) - { - auto constexpr maxNumWarnings = 25; - if (numWarnings < maxNumWarnings) - { - auto const layerName = layerInfo->value().name.value_or(""); - TLLM_LOG_WARNING( - "Layer '%s' has type '%s', which could lead to large runtime memory allocations. Performance " - "might be degraded and / or you might run out of memory.", - layerName.c_str(), layerInfo->value().type.c_str()); - } - numWarnings++; - } - if (numWarnings > 0) - { - TLLM_LOG_WARNING( - "There were a total of %i layers with type 'allocate'. Some warnings might have been silenced to keep the " - "output concise.", - numWarnings); - } -} - -} // namespace - -TllmRuntime::TllmRuntime(RawEngine const& rawEngine, nvinfer1::ILogger* logger, bool useGpuDirectStorage, - float gpuWeightsPercent, bool useShapeInference) - : mStream(std::make_shared<CudaStream>()) - , mBufferManager{mStream, true} // Ensure to trim the memory pool on destruction. - , mRuntime{nvinfer1::createInferRuntime(static_cast<bool>(logger) ? *logger : defaultLogger)} - , mUseShapeInference{useShapeInference} - , mUserBufferEnabled{false} -{ - auto const startTime = std::chrono::high_resolution_clock::now(); - - switch (rawEngine.getType()) - { - case RawEngine::Type::FilePath: - { - if (useGpuDirectStorage) - { - TLLM_LOG_INFO("GDS is used to load the engine!"); - auto reader = GDSStreamReader(rawEngine.getPath()); - mEngine.reset(mRuntime->deserializeCudaEngine(reader)); - } - else - { - auto reader = StreamReader(rawEngine.getPath()); - mEngine.reset(mRuntime->deserializeCudaEngine(reader)); - } - break; - } - case RawEngine::Type::AddressWithSize: - mEngine.reset(mRuntime->deserializeCudaEngine(rawEngine.getAddress(), rawEngine.getSize())); - break; - case RawEngine::Type::HostMemory: - mEngine.reset( - mRuntime->deserializeCudaEngine(rawEngine.getHostMemory()->data(), rawEngine.getHostMemory()->size())); - break; - default: TLLM_THROW("Unsupported raw engine type."); - } - - auto const elapsedMs - = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - startTime); - - TLLM_LOG_INFO("Engine load time %lld ms", elapsedMs); - - TLLM_CHECK_WITH_INFO(mEngine != nullptr, "Failed to deserialize cuda engine."); - mEngineInspector.reset(mEngine->createEngineInspector()); - assessLikelihoodOfRuntimeAllocation(*mEngine, *mEngineInspector); - setWeightStreaming(getEngine(), gpuWeightsPercent); - auto const devMemorySize = mEngine->getDeviceMemorySizeV2(); - mEngineBuffer = mBufferManager.gpu(devMemorySize); - // Print context memory size for CI/CD to track. - TLLM_LOG_INFO("[MemUsageChange] Allocated %.2f MiB for execution context memory.", - static_cast<double>(devMemorySize) / 1048576.0); - - cacheTensorNames(); -} - -void TllmRuntime::cacheTensorNames() -{ - for (std::int32_t i = 0; i < mEngine->getNbIOTensors(); ++i) - { - auto const* const name = mEngine->getIOTensorName(i); - if (mEngine->getTensorIOMode(name) == nvinfer1::TensorIOMode::kINPUT) - { - mInputTensorNames.emplace_back(name); - } - else if (mEngine->getTensorIOMode(name) == nvinfer1::TensorIOMode::kOUTPUT) - { - mOutputTensorNames.emplace_back(name); - } - } -} - -nvinfer1::IExecutionContext& TllmRuntime::addContext(std::int32_t profileIndex) -{ - TLLM_CHECK(0 <= profileIndex && profileIndex < mEngine->getNbOptimizationProfiles()); - mContexts.emplace_back(mEngine->createExecutionContextWithoutDeviceMemory()); - if (!mContexts.back()) - { - if (mEngine->getStreamableWeightsSize() > 0) - { - TLLM_THROW("Failed to allocate memory for weights. Please try reducing --gpu_weights_percent."); - } - else - { - TLLM_THROW("Internal Error: Failed to create an execution context."); - } - } - auto& context = *mContexts.back(); - context.setDeviceMemoryV2(mEngineBuffer->data(), static_cast<int64_t>(mEngineBuffer->getCapacity())); - - if (tensorrt_llm::common::Logger::getLogger()->isEnabled(tensorrt_llm::common::Logger::TRACE) - && mContexts.size() == 1) - { - // Print engine information only once - printEngineInfo(); - } - - context.setOptimizationProfileAsync(profileIndex, mStream->get()); - // If nvtx verbosity is DETAILED, print an info about potential perf overhead. - if (context.getNvtxVerbosity() == nvinfer1::ProfilingVerbosity::kDETAILED) - { - TLLM_LOG_INFO( - "The engine was built with kDETAILED profiling verbosity, which may result in small overheads at runtime."); - } - return context; -} - -void TllmRuntime::printEngineInfo() -{ - auto& context = *(mContexts[0]); - int const nIO = mEngine->getNbIOTensors(); // Count of input / output tensor - int const nOP = mEngine->getNbOptimizationProfiles(); // Count of Optimization Profile - std::size_t maxNameWidth = 0; - std::size_t maxShapeWidth = 0; - - // Get information of engine input / output - std::vector<std::string> tensorNameList{}; - tensorNameList.reserve(nIO); - for (int i = 0; i < nIO; ++i) - { - tensorNameList.emplace_back(mEngine->getIOTensorName(i)); - } - std::vector<std::map<std::string, std::string>> tensorInfo(nIO); // Tensor Information Vector - std::vector<std::vector<std::vector<nvinfer1::Dims64>>> profileInfo(nIO); // Tensor Optimization Profile Vector - for (int i = 0; i < nIO; ++i) - { - auto const& name = tensorNameList[i]; - char const* nameC{name.c_str()}; // name of C-style - maxNameWidth = std::max(maxNameWidth, name.size()); - tensorInfo[i]["mode"] = mEngine->getTensorIOMode(nameC) == nvinfer1::TensorIOMode::kINPUT ? "I" : "O"; - tensorInfo[i]["location"] - = mEngine->getTensorLocation(nameC) == nvinfer1::TensorLocation::kDEVICE ? "GPU" : "CPU"; - tensorInfo[i]["data_type"] = dataTypeToString(mEngine->getTensorDataType(nameC)); - tensorInfo[i]["build_shape"] = shapeToString(mEngine->getTensorShape(nameC)); - maxShapeWidth = std::max(maxShapeWidth, tensorInfo[i]["build_shape"].size()); - if (tensorInfo[i]["mode"] == "I") - { - std::vector<std::vector<nvinfer1::Dims64>> topPerTensor(nOP); - for (int k = 0; k < nOP; ++k) - { - if (tensorInfo[i]["location"] == std::string("GPU")) - { - std::vector<nvinfer1::Dims64> top(3); - top[0] = mEngine->getProfileShape(nameC, k, nvinfer1::OptProfileSelector::kMIN); - top[1] = mEngine->getProfileShape(nameC, k, nvinfer1::OptProfileSelector::kOPT); - top[2] = mEngine->getProfileShape(nameC, k, nvinfer1::OptProfileSelector::kMAX); - topPerTensor[k] = top; - maxShapeWidth = std::max(maxShapeWidth, shapeToString(top[2]).size()); - } - else - { - // Shape input tensor, not used in TRT-LLM support yet - std::vector<nvinfer1::Dims64> top(3); - int const nDim = mEngine->getTensorShape(nameC).nbDims; - nvinfer1::Dims64 tensorShape{nDim, {-1}}; - int const* pos = nullptr; - pos = mEngine->getProfileTensorValues(nameC, k, nvinfer1::OptProfileSelector::kMIN); - std::copy(pos, pos + nDim, tensorShape.d); - top[0] = tensorShape; - pos = mEngine->getProfileTensorValues(nameC, k, nvinfer1::OptProfileSelector::kOPT); - std::copy(pos, pos + nDim, tensorShape.d); - top[1] = tensorShape; - pos = mEngine->getProfileTensorValues(nameC, k, nvinfer1::OptProfileSelector::kMAX); - std::copy(pos, pos + nDim, tensorShape.d); - top[2] = tensorShape; - topPerTensor[k] = top; - } - } - profileInfo[i] = topPerTensor; - } - else - { - profileInfo[i] = std::vector<std::vector<nvinfer1::Dims64>>(nOP); - } - } - // Set input shape to get output shape - for (int k = 0; k < nOP; ++k) - { - for (int j = 0; j < 3; ++j) // Min, Opt, Max - { - for (int i = 0; i < nIO; ++i) - { - auto const& name = tensorNameList[i]; - char const* nameC = name.c_str(); - if (tensorInfo[i]["mode"] == "I") - { - if (tensorInfo[i]["location"] == std::string("GPU")) - { - context.setInputShape(nameC, profileInfo[i][k][j]); - } - else - { - // Shape input tensor, not used in TRT-LLM support yet - context.setInputTensorAddress(nameC, profileInfo[i][k][j].d); - } - } - else - { - TLLM_CHECK_WITH_INFO(context.allInputDimensionsSpecified(), "Input dimensions not specified"); - TLLM_CHECK_WITH_INFO(context.allInputShapesSpecified(), "Input shapes not specified"); - if (tensorInfo[i]["location"] == std::string("GPU")) - { - profileInfo[i][k].push_back(context.getTensorShape(nameC)); - } - else - { - // Shape input tensor, not used in TRT-LLM support yet - int const nDim = mEngine->getTensorShape(nameC).nbDims; - nvinfer1::Dims64 tensorShape{nDim, {}}; - int const* pos = reinterpret_cast<int const*>(context.getTensorAddress(nameC)); - std::copy(pos, pos + nDim, tensorShape.d); - profileInfo[i][k].push_back(tensorShape); - } - } - } - } - } - - // Print information of engine input / output - std::string info; - TLLM_LOG_TRACE("Information of engine input / output."); - TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 24, '=')); - info = alignText("Name", maxNameWidth) + "|I/O|Location|DataType|" + alignText("Shape", maxShapeWidth) + "|"; - TLLM_LOG_TRACE(info.c_str()); - TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 24, '-')); - for (int i = 0; i < nIO; ++i) - { - info = alignText(tensorNameList[i], maxNameWidth, false) + "|"; - info += alignText(tensorInfo[i]["mode"], 3) + "|"; - info += alignText(tensorInfo[i]["location"], 8) + "|"; - info += alignText(tensorInfo[i]["data_type"], 8) + "|"; - info += alignText(tensorInfo[i]["build_shape"], maxShapeWidth) + "|"; - TLLM_LOG_TRACE(info.c_str()); - } - TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 24, '=')); - // Print information of optimization profile - TLLM_LOG_TRACE("Information of optimization profile."); - for (int k = 0; k < nOP; ++k) - { - TLLM_LOG_TRACE("Optimization Profile %d:", k); - TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth * 3 + 4, '=')); - info = alignText("Name", maxNameWidth) + "|"; - info += alignText("Min", maxShapeWidth) + "|"; - info += alignText("Opt", maxShapeWidth) + "|"; - info += alignText("Max", maxShapeWidth) + "|"; - TLLM_LOG_TRACE(info.c_str()); - TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth * 3 + 4, '-')); - for (int i = 0; i < nIO; ++i) - { - auto const& top = profileInfo[i][k]; - info = alignText(tensorNameList[i], maxNameWidth, false) + "|"; - info += alignText(shapeToString(top[0]), maxShapeWidth) + "|"; - info += alignText(shapeToString(top[1]), maxShapeWidth) + "|"; - info += alignText(shapeToString(top[2]), maxShapeWidth) + "|"; - TLLM_LOG_TRACE(info.c_str()); - } - TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth * 3 + 4, '=')); - } -} - -void TllmRuntime::printContextInfo(SizeType32 contextIndex) -{ - auto const& context = *(mContexts[contextIndex]); - int const nIO = mEngine->getNbIOTensors(); // Count of input / output tensor - std::size_t maxNameWidth = 0; - std::size_t maxShapeWidth = 0; - std::vector<std::tuple<std::string, bool, std::string>> tensorInfo(nIO); - for (int i = 0; i < nIO; ++i) - { - auto const name = std::string(mEngine->getIOTensorName(i)); - bool const isInput = mEngine->getTensorIOMode(name.c_str()) == nvinfer1::TensorIOMode::kINPUT; - auto const shape = shapeToString(context.getTensorShape(name.c_str())); - tensorInfo[i] = std::make_tuple(name, isInput, shape); - maxNameWidth = std::max(maxNameWidth, name.size()); - maxShapeWidth = std::max(maxShapeWidth, shape.size()); - // Shape input tensor is not considered in TRT-LLM yet - } - - TLLM_LOG_TRACE("Information of context input / output."); - TLLM_LOG_TRACE("Using Optimization Profile: %d", contextIndex); - TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 6, '=')); - std::string info = alignText("Name", maxNameWidth) + "|I/O|" + alignText("Shape", maxShapeWidth) + "|"; - TLLM_LOG_TRACE(info.c_str()); - TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 6, '-')); - for (int i = 0; i < nIO; ++i) - { - auto const& [name, isInput, shape] = tensorInfo[i]; - info = alignText(name, maxNameWidth, false) + "|"; - info += alignText(isInput ? "I" : "O", 3) + "|"; - info += alignText(shape, maxShapeWidth) + "|"; - TLLM_LOG_TRACE(info.c_str()); - } - TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 6, '=')); -} - -void TllmRuntime::clearContexts() -{ - for (auto& context : mContexts) - { - context.reset(); - } - mContexts.clear(); -} - -bool TllmRuntime::executeContext(SizeType32 contextIndex) const -{ - NVTX3_FUNC_RANGE(); - auto& context = getContext(contextIndex); - auto res = context.enqueueV3(mStream->get()); - sync_check_cuda_error(mStream->get()); - return res; -} - -void TllmRuntime::setInputTensorsImpl(SizeType32 contextIndex, TensorMap const& tensorMap, bool throwOnMiss) -{ - NVTX3_FUNC_RANGE(); - auto& context = getContext(contextIndex); - for (auto const& name : mInputTensorNames) - { - auto const pos = tensorMap.find(name); - if (pos == tensorMap.end()) - { - if (throwOnMiss) - { - auto expectedShape = mEngine->getTensorShape(name.c_str()); - TLLM_THROW("Input tensor '%s' not found; expected shape: %s", name.c_str(), - ITensor::toString(expectedShape).c_str()); - } - else - { - continue; - } - } - - auto const& tensor = pos->second; - auto const tensorDtype = tensor->getDataType(); - auto const engineDtype = mEngine->getTensorDataType(name.c_str()); - // WAR: TRT does not support mixed FP8 and FP16 input, so engine expects FP16 tensors. - TLLM_CHECK_WITH_INFO(tensorDtype == engineDtype - || (tensorDtype == nvinfer1::DataType::kFP8 && engineDtype == nvinfer1::DataType::kHALF), - "%s: expected type %d, provided type %d", name.c_str(), static_cast<std::int32_t>(engineDtype), - static_cast<std::int32_t>(tensorDtype)); - - auto tensorShape = tensor->getShape(); - - // Change shape of `cache_indirection` for Variable-Beam-Width-Search - // TODO: remove this hack if beamWidth of each request are passed into GptAttentionPlugin by input tensor - if (name == "cache_indirection" && mCurrentBeamWidths.size() > 0) - { - SizeType32 const beamWidth = getCurrentBeamWidth(); - if (tensorShape.d[1] != beamWidth) - { - tensorShape.d[1] = beamWidth; - TLLM_LOG_TRACE("Change shape of cache_indirection to %s", ITensor::toString(tensorShape).c_str()); - } - } - - auto const setInputShapeSuccess = context.setInputShape(name.c_str(), tensorShape); - if (!setInputShapeSuccess) - { - auto const minShape - = mEngine->getProfileShape(name.c_str(), contextIndex, nvinfer1::OptProfileSelector::kMIN); - auto const maxShape - = mEngine->getProfileShape(name.c_str(), contextIndex, nvinfer1::OptProfileSelector::kMAX); - - TLLM_THROW("Tensor '%s' has invalid shape %s, expected in range min %s, max %s", name.c_str(), - ITensor::toString(tensorShape).c_str(), ITensor::toString(minShape).c_str(), - ITensor::toString(maxShape).c_str()); - } - auto* const data = tensor->data(); - if (static_cast<bool>(data)) - { - context.setInputTensorAddress(name.c_str(), data); - } - else - { - TLLM_CHECK_WITH_INFO(tensor->getSize() == 0, std::string("Invalid data for tensor: ") + name); - // TensorRT runtime does not support nullptr. - if (!mDummyTensor) - { - mDummyTensor = mBufferManager.gpu(ITensor::makeShape({1})); - } - context.setInputTensorAddress(name.c_str(), mDummyTensor->data()); - } - } -} - -void TllmRuntime::setStaticInputTensors(TensorMap const& tensorMap) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_FUNC_RANGE(); - - TLLM_CHECK_WITH_INFO(getNbContexts() > 0, "Contexts should be created before calling setStaticInputTensors"); - for (auto contextIndex = 0; contextIndex < getNbContexts(); ++contextIndex) - { - setInputTensorsImpl(contextIndex, tensorMap, false); - } - - // move static input tensor names to separate vector - auto const begin = mInputTensorNames.begin(); - auto end = mInputTensorNames.end(); - for (auto const& [name, tensor] : tensorMap) - { - end = std::remove(begin, end, name); - } - mInputTensorNames.erase(end, mInputTensorNames.end()); - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TllmRuntime::setInputTensors(SizeType32 contextIndex, TensorMap const& tensorMap) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_FUNC_RANGE(); - setInputTensorsImpl(contextIndex, tensorMap, true); - - auto& context = getContext(contextIndex); - if (mUseShapeInference) - { - NVTX3_SCOPED_RANGE(infer_shapes); - char const* missing = nullptr; - auto const nbMissing = context.inferShapes(1, &missing); - if (nbMissing > 0) - { - TLLM_THROW("Input shape not specified: %s", missing); - } - else if (nbMissing < 0) - { - TLLM_THROW("Invalid input shape"); - } - } - - { - NVTX3_SCOPED_RANGE(final_checks); - TLLM_CHECK_WITH_INFO(context.allInputDimensionsSpecified(), "Input dimensions not specified"); - TLLM_CHECK_WITH_INFO(context.allInputShapesSpecified(), "Input shapes not specified"); - } - - // Print shape of input / output tensors for the TRT engine - if (tensorrt_llm::common::Logger::getLogger()->isEnabled(tensorrt_llm::common::Logger::TRACE)) - { - printContextInfo(contextIndex); - } - - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TllmRuntime::setOutputTensors(SizeType32 contextIndex, TensorMap& tensorMap) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_FUNC_RANGE(); - if (isUserBufferEnabled()) - { - // This function will identify the output tensors in the network that need to be bound as UB buffers - // and bind the corresponding buffers to them based on their names. - setUserBufferTensors(contextIndex, tensorMap); - } - - auto& context = getContext(contextIndex); - for (auto const& name : mOutputTensorNames) - { - auto const engineDtype = mEngine->getTensorDataType(name.c_str()); - auto const pos = tensorMap.find(name); - if (pos != tensorMap.end()) - { - auto const& tensor = pos->second; - auto const tensorDtype = tensor->getDataType(); - // WAR: TRT does not support mixed FP8 and FP16 input, so engine expects FP16 tensors. - TLLM_CHECK_WITH_INFO(tensorDtype == engineDtype - || (tensorDtype == nvinfer1::DataType::kFP8 && engineDtype == nvinfer1::DataType::kHALF), - "%s: expected type %d, provided type %d", name.c_str(), static_cast<std::int32_t>(engineDtype), - static_cast<std::int32_t>(tensorDtype)); - - if (mUseShapeInference) - { - auto const dims = context.getTensorShape(name.c_str()); - tensor->reshape(dims); - } - context.setTensorAddress(name.c_str(), tensor->data()); - } - else if (mUseShapeInference) - { - auto const dims = context.getTensorShape(name.c_str()); - auto tensor = ITensor::SharedPtr(mBufferManager.gpu(dims, engineDtype)); - tensorMap.insert(pos, std::make_pair(name, tensor)); - context.setTensorAddress(name.c_str(), tensor->data()); - } - else - { - TLLM_THROW("Tensor %s is not found in tensorMap and shape inference is not allowed", name.c_str()); - } - } - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} - -void TllmRuntime::setUserBufferTensors(SizeType32 contextIndex, TensorMap& tensorMap) -{ - auto startsWith = [](std::string const& str, std::string const& prefix) -> bool - { return str.size() > prefix.size() && str.compare(0, prefix.size(), prefix) == 0; }; - std::string const prefix(tensorrt_llm::runtime::ub::tensor_prefix); - auto& context = getContext(contextIndex); - for (auto const& name : mOutputTensorNames) - { - auto const pos = tensorMap.find(name); - if (pos != tensorMap.end() || !startsWith(name, prefix)) - { - continue; - } - auto const engineDtype = mEngine->getTensorDataType(name.c_str()); - auto const dims = context.getTensorShape(name.c_str()); - void* ubBuffer = nullptr; - if (name[prefix.size()] == '0') - { - ubBuffer = tensorrt_llm::runtime::ub::ub_get(0).addr; - } - else if (name[prefix.size()] == '1') - { - ubBuffer = tensorrt_llm::runtime::ub::ub_get(1).addr; - } - else if (name[prefix.size()] == '2') - { - ubBuffer = tensorrt_llm::runtime::ub::ub_get(2).addr; - } - else - { - TLLM_CHECK(false); - } - auto tensor = ITensor::SharedPtr(ITensor::wrap(ubBuffer, engineDtype, dims)); - tensorMap.insert(pos, std::make_pair(name, tensor)); - context.setTensorAddress(name.c_str(), ubBuffer); - } -} - -void TllmRuntime::initializeUserBuffer(tensorrt_llm::runtime::WorldConfig const& world_config, SizeType32 maxBatchSize, - SizeType32 maxBeamWidth, SizeType32 maxSequenceLength, SizeType32 hiddenSize, - std::optional<SizeType32> maxNumTokens) -{ - auto startsWith = [](std::string const& str, std::string const& prefix) -> bool - { return str.size() > prefix.size() && str.compare(0, prefix.size(), prefix) == 0; }; - std::string const prefix(tensorrt_llm::runtime::ub::tensor_prefix); - bool useNVFP4Model = false; - for (auto const& name : mOutputTensorNames) - { - if (startsWith(name, prefix)) - { - mUserBufferEnabled = true; - if (name[prefix.size()] == '2') - { - useNVFP4Model = true; - break; - } - } - } - if (!mUserBufferEnabled) - { - return; - } - // The hidden size returned by ModelConfig is the real hidden size divided by the TP size. - auto const tpSize = world_config.getTensorParallelism(); - size_t const realHiddenSize = hiddenSize * tpSize; - size_t const tokensNum = maxNumTokens.value_or(maxBatchSize * maxBeamWidth * maxSequenceLength); - TLLM_CHECK(tokensNum > 0); - size_t const elemNum = tokensNum * realHiddenSize; - TLLM_LOG_INFO("[UserBuffer] MaxBatchSize %d, maxBeamWidth %d, maxSequenceLength %d, maxNumTokens %d, select %lu", - maxBatchSize, maxBeamWidth, maxSequenceLength, maxNumTokens.has_value() ? maxNumTokens.value() : 0, tokensNum); - tensorrt_llm::runtime::ub::ub_initialize(world_config); - tensorrt_llm::runtime::ub::ub_allocate(elemNum * sizeof(half)); - tensorrt_llm::runtime::ub::ub_allocate(elemNum * sizeof(half)); - if (useNVFP4Model) - { - tensorrt_llm::runtime::ub::ub_allocate(elemNum * sizeof(uint8_t) / 16); - } -} - -CudaStream const& TllmRuntime::getStream() const -{ - return *mStream; -} - -bool TllmRuntime::hasLayerProfiler(SizeType32 contextId) const -{ - return mContexts[contextId]->getProfiler() != nullptr; -} - -void TllmRuntime::setLayerProfiler() -{ - mLayerProfiler = std::make_unique<LayerProfiler>(); - for (auto& context : mContexts) - { - context->setProfiler(mLayerProfiler.get()); - context->setEnqueueEmitsProfile(false); - } -} - -std::string TllmRuntime::getLayerProfileInfo() const -{ - TLLM_CHECK(mLayerProfiler); - return mLayerProfiler->getLayerProfile(); -} - -void TllmRuntime::reportToProfiler(SizeType32 contextId) -{ - mContexts[contextId]->reportToProfiler(); -} - -void TllmRuntime::loadManagedWeights(RawEngine const& rawEngine, int localRank) -{ - TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); - NVTX3_FUNC_RANGE(); - auto& engine = getEngine(); - auto& manager = getBufferManager(); - if (rawEngine.getManagedWeightsMapOpt().has_value()) - { - TLLM_LOG_DEBUG("Loading managed weights from raw engine"); - auto executorMap = rawEngine.getManagedWeightsMapOpt().value(); - for (auto const& [name, weight] : executorMap) - { - TLLM_LOG_DEBUG("Loading managed weight: %s", name.c_str()); - auto iTensor = tensorrt_llm::executor::detail::toITensor(weight); - auto weightsDevice = std::shared_ptr<ITensor>{manager.copyFrom(*iTensor, MemoryType::kGPU)}; - mManagedWeightsMap.insert(std::make_pair(name, weightsDevice)); - } - } - else - { - TLLM_LOG_DEBUG("Loading managed weights from file"); - auto const enginePath = rawEngine.getPathOpt(); - TLLM_CHECK_WITH_INFO(enginePath.has_value(), "Engine path is not set."); - auto weightPath - = enginePath->parent_path() / ("rank" + std::to_string(localRank) + "_managed_weights.safetensors"); - auto managed_weights = common::safetensors::ISafeTensor::open(weightPath.string().c_str()); - for (auto const& name : managed_weights->keys()) - { - TLLM_LOG_DEBUG("Loading managed weight: %s", name.c_str()); - auto const weight = managed_weights->getTensor(name.c_str()); - TLLM_CHECK(weight->dtype() == engine.getTensorDataType(name.c_str())); - auto weightsDevice - = std::shared_ptr<ITensor>{manager.allocate(MemoryType::kGPU, weight->trtDims(), weight->dtype())}; - manager.copy(weight->data(), *weightsDevice, MemoryType::kCPU); - mManagedWeightsMap.insert(std::make_pair(name, weightsDevice)); - } - } - setStaticInputTensors(mManagedWeightsMap); - TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); -} diff --git a/cpp/tensorrt_llm/runtime/tllmRuntime.h b/cpp/tensorrt_llm/runtime/tllmRuntime.h deleted file mode 100644 index dfef06d8b45f..000000000000 --- a/cpp/tensorrt_llm/runtime/tllmRuntime.h +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/layerProfiler.h" -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/worldConfig.h" -#include <NvInferRuntime.h> - -#include <cstdint> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace tensorrt_llm::runtime -{ -class TllmRuntime -{ -public: - using TensorMap = StringPtrMap<ITensor>; - - explicit TllmRuntime(RawEngine const& rawEngine, nvinfer1::ILogger* logger, bool useGpuDirectStorage = false, - float gpuWeightsPercent = 1.0f, bool useShapeInference = true); - - SizeType32 getNbContexts() const - { - return static_cast<SizeType32>(mContexts.size()); - } - - nvinfer1::IExecutionContext& getContext(SizeType32 contextIndex) const - { - return *mContexts.at(contextIndex); - } - - SizeType32 getNbProfiles() const - { - return static_cast<SizeType32>(mEngine->getNbOptimizationProfiles()); - } - - /// @brief If multiple TensorRT optimization profiles are built in the engine, this function selects the - /// corresponding profile that is going to be used based on the runtime shape, for now, TensorRT LLM only split - /// multiple profiles on the num_tokens dimension, hence the profile index is selected based on which profile - /// handles the actual num_tokens - /// @return The index of the selected TensorRT optimization profile - [[nodiscard]] SizeType32 getOptProfileId(int numTokens, std::vector<SizeType32> const& splitPoints) const - { - if (getNbProfiles() == 1) - { - return 0; - } - auto const it = std::lower_bound(splitPoints.begin(), splitPoints.end(), numTokens); - auto const optProfileId = std::distance(splitPoints.begin(), it); - return optProfileId; - } - - nvinfer1::IExecutionContext& addContext(std::int32_t profileIndex); - - void clearContexts(); - - /// @brief Set input tensors from tensorMap for all contexts. - /// @details The function can be used to set static input tensors for all iterations. If a tensor was set this way, - /// it doesn't need to included in calls to setInputTensors anymore. - void setStaticInputTensors(TensorMap const& tensorMap); - - /// @brief Set input tensors from tensorMap for context at contextIndex. - /// @details The function expects that all input tensors (excluding the ones set by setStaticInputTensors) are - /// contained in the tensorMap. If a tensor is missing, has a bad shape or type, it will throw. - void setInputTensors(SizeType32 contextIndex, TensorMap const& tensorMap); - - /// @brief Set output tensors from tensorMap for context at contextIndex. - /// @details The function expects that all output tensors are contained in the tensorMap. If a tensor is missing and - /// shape inference is enabled, it will allocate the tensor on GPU and insert it into the tensorMap. Otherwise it - /// will throw. - void setOutputTensors(SizeType32 contextIndex, TensorMap& tensorMap); - - bool executeContext(SizeType32 contextIndex) const; - - CudaStream const& getStream() const; - - BufferManager::CudaStreamPtr getStreamPtr() - { - return mStream; - } - - nvinfer1::ICudaEngine& getEngine() - { - return *mEngine; - } - - nvinfer1::ICudaEngine const& getEngine() const - { - return *mEngine; - } - - nvinfer1::IEngineInspector& getEngineInspector() - { - return *mEngineInspector; - } - - nvinfer1::IEngineInspector const& getEngineInspector() const - { - return *mEngineInspector; - } - - BufferManager& getBufferManager() - { - return mBufferManager; - } - - BufferManager const& getBufferManager() const - { - return mBufferManager; - } - - void setLayerProfiler(); - bool hasLayerProfiler(SizeType32 contextId) const; - std::string getLayerProfileInfo() const; - void reportToProfiler(SizeType32 contextId); - void loadManagedWeights(RawEngine const& rawEngine, int localRank); - void initializeUserBuffer(tensorrt_llm::runtime::WorldConfig const& world_config, SizeType32 maxBatchSize, - SizeType32 maxBeamWidth, SizeType32 maxSequenceLength, SizeType32 hiddenSize, - std::optional<SizeType32> maxNumTokens); - - bool isUserBufferEnabled() const - { - return mUserBufferEnabled; - } - - void setCurrentBeamWidths(std::vector<SizeType32> const& beamWidth) noexcept - { - mCurrentBeamWidths = beamWidth; - } - - [[nodiscard]] SizeType32 const& getCurrentBeamWidth() const noexcept - { - // At present, all requests of a batch must have the same beam width in one generation step (or they will not - // be batched together). So, the beam widths in `mCurrentBeamWidths` are the same. - // Corresponding changes must be done if Diverse-Beam-Width-Search (DBWS, requests with diverse beam width in - // a batch in one generation step) is supported in the future. - TLLM_CHECK_WITH_INFO(mCurrentBeamWidths.size() > 0, "`mCurrentBeamWidths` is empty."); - bool const isEqual = std::all_of(mCurrentBeamWidths.begin(), mCurrentBeamWidths.end(), - [&](int elem) { return elem == mCurrentBeamWidths.front(); }); - TLLM_CHECK_WITH_INFO(isEqual, "beam widths in `mCurrentBeamWidths` are not all equal."); - return mCurrentBeamWidths.front(); - } - -private: - void cacheTensorNames(); - - void setInputTensorsImpl(SizeType32 contextIndex, TensorMap const& tensorMap, bool throwOnMiss); - - void setUserBufferTensors(SizeType32 contextIndex, TensorMap& tensorMap); - - void printEngineInfo(); - - void printContextInfo(SizeType32 contextIndex); - - // Tool functions for `printEngineInfo()`. - static std::string shapeToString(nvinfer1::Dims64 const& dim) - { - std::string output("("); - if (dim.nbDims == 0) - { - return output + ")"; - } - for (int i = 0; i < dim.nbDims - 1; ++i) - { - output += std::to_string(dim.d[i]) + ", "; - } - output += std::to_string(dim.d[dim.nbDims - 1]) + ")"; - return output; - } - - static std::string dataTypeToString(nvinfer1::DataType type) - { - switch (type) - { - case nvinfer1::DataType::kINT64: return "INT64"; - case nvinfer1::DataType::kINT32: return "INT32"; - case nvinfer1::DataType::kFLOAT: return "FP32"; - case nvinfer1::DataType::kBF16: return "BF16"; - case nvinfer1::DataType::kHALF: return "FP16"; - case nvinfer1::DataType::kBOOL: return "BOOL"; - case nvinfer1::DataType::kUINT8: return "UINT8"; - case nvinfer1::DataType::kINT8: return "INT8"; - case nvinfer1::DataType::kFP8: return "FP8"; - case nvinfer1::DataType::kINT4: return "INT4"; - case nvinfer1::DataType::kFP4: return "FP4"; - default: return "UNKNOWN"; - } - return ""; - } - - static std::string alignText( - std::string const& text, int const width, bool const bCenter = true, char const blank = ' ') - { - int textLen = text.size(); - int padLeft = 0; - int padRight = 0; - padLeft = bCenter ? (width - textLen) / 2 : 0; - padRight = width - padLeft - textLen; - return std::string(padLeft, blank) + text + std::string(padRight, blank); - } - - BufferManager::CudaStreamPtr mStream; - BufferManager mBufferManager; - std::unique_ptr<nvinfer1::IRuntime> mRuntime; - std::unique_ptr<nvinfer1::ICudaEngine> mEngine; - BufferManager::IBufferPtr mEngineBuffer; - std::vector<std::unique_ptr<nvinfer1::IExecutionContext>> mContexts; - std::unique_ptr<ITensor> mDummyTensor; - std::unique_ptr<nvinfer1::IEngineInspector> mEngineInspector; - std::unique_ptr<LayerProfiler> mLayerProfiler; - bool mUseShapeInference; - TensorMap mManagedWeightsMap; - // List of input tensor names. - // Names of static tensors are removed from this list when setStaticInputTensors is called. - std::vector<std::string> mInputTensorNames; - std::vector<std::string> mOutputTensorNames; - - bool mUserBufferEnabled; - // For Variable-Beam-Width-Search - std::vector<SizeType32> mCurrentBeamWidths; -}; -} // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/runtime/tllmStreamReaders.cpp b/cpp/tensorrt_llm/runtime/tllmStreamReaders.cpp deleted file mode 100644 index 55440bbe714f..000000000000 --- a/cpp/tensorrt_llm/runtime/tllmStreamReaders.cpp +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tllmStreamReaders.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" - -#include <cufile.h> -#include <dlfcn.h> -#include <fcntl.h> -#include <filesystem> -#include <fstream> -#include <string> -#include <unistd.h> - -// Non-GDS StreamReader - -StreamReader::StreamReader(std::filesystem::path fp) -{ - mFile.open(fp.string(), std::ios::binary | std::ios::in); - TLLM_CHECK_WITH_INFO(mFile.good(), std::string("Error opening engine file: " + fp.string())); -} - -StreamReader::~StreamReader() -{ - if (mFile.is_open()) - { - mFile.close(); - } -} - -int64_t StreamReader::read(void* destination, int64_t nbBytes) -{ - if (!mFile.good()) - { - return -1; - } - - mFile.read(static_cast<char*>(destination), nbBytes); - - return mFile.gcount(); -} - -// StreamReader using GDS - -GDSStreamReader::GDSStreamReader(std::filesystem::path const& filePath) -{ - auto const start_time = std::chrono::high_resolution_clock::now(); - initializeDriver(); - auto const elapsed_ms - = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - start_time); - - TLLM_LOG_INFO("GDS driver initialization time %lld ms", elapsed_ms); - - open(filePath); -} - -bool GDSStreamReader::open(std::string const& filepath) -{ - if (!initializeDriver()) - { - TLLM_LOG_INFO("Failed to initialize cuFile driver"); - return false; - } - - int32_t const ret = ::open(filepath.c_str(), O_CREAT | O_RDWR | O_DIRECT, 0664); - - if (ret < 0) - { - TLLM_LOG_INFO("Failed to open engine file"); - return false; - } - - mFd = ret; - mFileSize = lseek(mFd, 0, SEEK_END); - lseek(mFd, 0, SEEK_SET); - - CUfileDescr_t fileDescr; - memset((void*) &fileDescr, 0, sizeof(fileDescr)); - fileDescr.handle.fd = mFd; - fileDescr.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD; - - CUfileError_t gdsStatus = cuFileHandleRegister(&mFileHandle, &fileDescr); - - if (gdsStatus.err != CU_FILE_SUCCESS) - { - TLLM_LOG_INFO("Failed to cuFileHandleRegister"); - ::close(mFd); - return false; - } - return true; -} - -void GDSStreamReader::close() -{ - if (mFd >= 0) - { - ::close(mFd); - mFd = -1; - } -} - -GDSStreamReader::~GDSStreamReader() -{ - if (mFileHandle) - { - cuFileHandleDeregister(mFileHandle); - mFileHandle = nullptr; - } - - if (mDriverInitialized) - { - cuFileDriverClose(); - } -} - -bool GDSStreamReader::seek(int64_t offset, nvinfer1::SeekPosition where) noexcept -{ - switch (where) - { - case nvinfer1::SeekPosition::kSET: mCursor = offset; return true; - case nvinfer1::SeekPosition::kCUR: mCursor += offset; return true; - case nvinfer1::SeekPosition::kEND: mCursor = -offset; return true; - default: return false; - } - return true; -} - -int64_t GDSStreamReader::read(void* dest, int64_t bytes, cudaStream_t stream) noexcept -{ - cudaPointerAttributes attributes{}; - if (cudaPointerGetAttributes(&attributes, dest) != cudaSuccess) - { - TLLM_LOG_INFO("cudaPointerGetAttributes failed"); - } - - off_t destOffset = 0; - void* destBase = dest; - - if (attributes.type == cudaMemoryTypeDevice) - { - CUdeviceptr cuDest = reinterpret_cast<CUdeviceptr>(dest); - CUdeviceptr cuBufBase = 0; - size_t cuBufSize = 0; - - cuMemGetAddressRange(&cuBufBase, &cuBufSize, cuDest); - destOffset += cuDest - cuBufBase; - destBase = reinterpret_cast<void*>(cuBufBase); - } - cuFileRead(this->mFileHandle, destBase, bytes, mCursor, destOffset); - - mCursor += bytes; - return bytes; -} - -void GDSStreamReader::reset() -{ - lseek(mFd, 0, SEEK_SET); - mCursor = 0; -} - -[[nodiscard]] bool GDSStreamReader::isOpen() const -{ - bool open = mFd >= 0; - return open; -} - -bool GDSStreamReader::initializeDriver() -{ - if (mDriverInitialized) - { - return true; - } - - mCuFileLibHandle = dlopen("libcufile.so", RTLD_LAZY | RTLD_GLOBAL); - if (!mCuFileLibHandle) - { - TLLM_LOG_INFO("Failed to dlopen libcufile.so"); - return false; - } - - // Load the required functions - *reinterpret_cast<void**>(&cuFileDriverOpen) = dlsym(mCuFileLibHandle, "cuFileDriverOpen"); - *reinterpret_cast<void**>(&cuFileHandleRegister) = dlsym(mCuFileLibHandle, "cuFileHandleRegister"); - *reinterpret_cast<void**>(&cuFileHandleDeregister) = dlsym(mCuFileLibHandle, "cuFileHandleDeregister"); - *reinterpret_cast<void**>(&cuFileDriverClose) = dlsym(mCuFileLibHandle, "cuFileDriverClose"); - *reinterpret_cast<void**>(&cuFileRead) = dlsym(mCuFileLibHandle, "cuFileRead"); - - if (!cuFileDriverOpen || !cuFileHandleRegister || !cuFileHandleDeregister || !cuFileDriverClose || !cuFileRead) - { - TLLM_LOG_INFO("Failed to dlsym libcufile.so"); - return false; - } - - CUfileError_t gdsStatus = cuFileDriverOpen(); - if (gdsStatus.err != CU_FILE_SUCCESS) - { - TLLM_LOG_INFO("cuFileDriverOpen failed"); - return false; - } - - mDriverInitialized = true; - return true; -} diff --git a/cpp/tensorrt_llm/runtime/tllmStreamReaders.h b/cpp/tensorrt_llm/runtime/tllmStreamReaders.h deleted file mode 100644 index 943f0bb3e32e..000000000000 --- a/cpp/tensorrt_llm/runtime/tllmStreamReaders.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include <NvInferRuntime.h> - -#include <cufile.h> -#include <filesystem> -#include <fstream> - -class StreamReader final : public nvinfer1::IStreamReader -{ -public: - StreamReader(std::filesystem::path fp); - - virtual ~StreamReader(); - - int64_t read(void* destination, int64_t nbBytes) final; - -private: - std::ifstream mFile; -}; - -class GDSStreamReader final : public nvinfer1::IStreamReaderV2 -{ -public: - explicit GDSStreamReader(std::filesystem::path const& filePath); - - virtual ~GDSStreamReader(); - - void close(); - - [[nodiscard]] bool isOpen() const; - - bool open(std::string const& filepath); - - int64_t read(void* dest, int64_t bytes, cudaStream_t stream) noexcept final; - - void reset(); - - bool seek(int64_t offset, nvinfer1::SeekPosition where) noexcept final; - -private: - bool initializeDriver(); - - void* mCuFileLibHandle{}; - CUfileHandle_t mFileHandle{nullptr}; - bool mDriverInitialized{false}; - int32_t mFd{-1}; - int64_t mCursor{0}; - int64_t mFileSize{0}; - - CUfileError_t (*cuFileDriverOpen)(){}; - CUfileError_t (*cuFileHandleRegister)(CUfileHandle_t*, CUfileDescr_t*){}; - CUfileError_t (*cuFileHandleDeregister)(CUfileHandle_t){}; - CUfileError_t (*cuFileDriverClose)(){}; - ssize_t (*cuFileRead)(CUfileHandle_t, void*, size_t, int64_t, int64_t){}; -}; diff --git a/cpp/tensorrt_llm/runtime/utils/debugUtils.cu b/cpp/tensorrt_llm/runtime/utils/debugUtils.cu index 661dacd9a7ac..d4aaa8244c11 100644 --- a/cpp/tensorrt_llm/runtime/utils/debugUtils.cu +++ b/cpp/tensorrt_llm/runtime/utils/debugUtils.cu @@ -26,6 +26,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include <cfloat> #include <string> @@ -167,7 +168,7 @@ template <typename T> bool tensorHasInvalid(ITensor const& tensor, BufferManager const& manager, std::string const& infoStr) { printLogitsKeyInfo<T>(tensor, infoStr); - auto foundInvalid = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + auto foundInvalid = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); auto foundInvalidPtr = bufferCast<int32_t>(*foundInvalid); foundInvalidPtr[0] = 0; auto const size = tensor.getSize(); @@ -184,24 +185,24 @@ template bool tensorHasInvalid<__nv_fp8_e4m3>( ITensor const& tensor, BufferManager const& manager, std::string const& infoStr); bool tensorHasInvalid( - size_t M, size_t K, nvinfer1::DataType type, void const* data, cudaStream_t stream, std::string const& infoStr) + size_t M, size_t K, tensorrt_llm::DataType type, void const* data, cudaStream_t stream, std::string const& infoStr) { auto tensorView = ITensor::wrap( const_cast<void*>(data), type, ITensor::makeShape({static_cast<int32_t>(M), static_cast<int32_t>(K)})); auto manager = BufferManager(std::make_shared<CudaStream>(stream)); - if (type == nvinfer1::DataType::kFLOAT) + if (type == tensorrt_llm::DataType::kFLOAT) { return tensorHasInvalid<float>(*tensorView, manager, infoStr); } - else if (type == nvinfer1::DataType::kHALF) + else if (type == tensorrt_llm::DataType::kHALF) { return tensorHasInvalid<half>(*tensorView, manager, infoStr); } - else if (type == nvinfer1::DataType::kBF16) + else if (type == tensorrt_llm::DataType::kBF16) { return tensorHasInvalid<__nv_bfloat16>(*tensorView, manager, infoStr); } - else if (type == nvinfer1::DataType::kFP8) + else if (type == tensorrt_llm::DataType::kFP8) { return tensorHasInvalid<__nv_fp8_e4m3>(*tensorView, manager, infoStr); } diff --git a/cpp/tensorrt_llm/runtime/utils/mpiUtils.cpp b/cpp/tensorrt_llm/runtime/utils/mpiUtils.cpp index 0f8f31082e96..ae508bbdbc3b 100644 --- a/cpp/tensorrt_llm/runtime/utils/mpiUtils.cpp +++ b/cpp/tensorrt_llm/runtime/utils/mpiUtils.cpp @@ -597,9 +597,16 @@ MpiComm::~MpiComm() noexcept #if ENABLE_MULTI_DEVICE if (mFreeComm && mComm) { - if (MPI_Comm_free(&mComm) != MPI_SUCCESS) + // Calling MPI_Comm_free after MPI has been finalized is undefined behavior. + // We need this check to prevent heap corruption during program exit when + // static MpiComm objects are created. + int finalized = 0; + if (MPI_Finalized(&finalized) == MPI_SUCCESS && !finalized) { - TLLM_LOG_ERROR("MPI_Comm_free failed"); + if (MPI_Comm_free(&mComm) != MPI_SUCCESS) + { + TLLM_LOG_ERROR("MPI_Comm_free failed"); + } } } #endif // ENABLE_MULTI_DEVICE diff --git a/cpp/tensorrt_llm/runtime/utils/numpyUtils.cpp b/cpp/tensorrt_llm/runtime/utils/numpyUtils.cpp index 6f95704455d3..931fbf0203da 100644 --- a/cpp/tensorrt_llm/runtime/utils/numpyUtils.cpp +++ b/cpp/tensorrt_llm/runtime/utils/numpyUtils.cpp @@ -20,9 +20,9 @@ #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/stringUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" -#include <NvInferRuntime.h> #include <sstream> #include <stdexcept> @@ -34,9 +34,9 @@ namespace tc = tensorrt_llm::common; namespace tensorrt_llm::runtime::utils { -std::string getNumpyTypeDesc(nvinfer1::DataType type) +std::string getNumpyTypeDesc(tensorrt_llm::DataType type) { - using dt = nvinfer1::DataType; + using dt = tensorrt_llm::DataType; static std::unordered_map<dt, std::string> const type_map{{dt::kBOOL, "?"}, {dt::kUINT8, "u1"}, {dt::kINT8, "i1"}, {dt::kINT32, "i4"}, {dt::kINT64, "i8"}, {dt::kHALF, "f2"}, {dt::kFLOAT, "f4"}}; @@ -51,11 +51,11 @@ std::string getNumpyTypeDesc(nvinfer1::DataType type) return type_map.count(type) > 0 ? type_map.at(type) : "x"; } -nvinfer1::DataType typeFromNumpyDesc(std::string const& type) +tensorrt_llm::DataType typeFromNumpyDesc(std::string const& type) { TLLM_LOG_DEBUG("numpy type: %s", type.c_str()); - using dt = nvinfer1::DataType; + using dt = tensorrt_llm::DataType; static std::unordered_map<std::string, dt> const type_map{{"?", dt::kBOOL}, {"u1", dt::kUINT8}, {"i1", dt::kINT8}, {"i4", dt::kINT32}, {"i8", dt::kINT64}, {"f2", dt::kHALF}, {"f4", dt::kFLOAT}}; TLLM_CHECK_WITH_INFO(type_map.count(type) > 0, "numpy data type '" + type + "' not supported"); @@ -102,7 +102,7 @@ void parseNpyIntro(FILE*& f_ptr, uint32_t& header_len, uint32_t& start_data) start_data = 8 + 2 * npy_major + header_len; } -int parseNpyHeader(FILE*& f_ptr, uint32_t header_len, nvinfer1::DataType& type, std::vector<size_t>& shapeVec) +int parseNpyHeader(FILE*& f_ptr, uint32_t header_len, tensorrt_llm::DataType& type, std::vector<size_t>& shapeVec) { char* header_c = (char*) malloc(header_len * sizeof(char)); TLLM_CHECK_WITH_INFO(header_c != nullptr, "Failed to allocate memory for npy header"); @@ -168,11 +168,11 @@ int parseNpyHeader(FILE*& f_ptr, uint32_t header_len, nvinfer1::DataType& type, uint32_t header_len, start_data; utils::parseNpyIntro(f_ptr, header_len, start_data); - nvinfer1::DataType type; + tensorrt_llm::DataType type; std::vector<size_t> shape; utils::parseNpyHeader(f_ptr, header_len, type, shape); - nvinfer1::Dims dims; + tensorrt_llm::Dims dims; dims.nbDims = shape.size(); std::copy(shape.begin(), shape.end(), dims.d); @@ -203,10 +203,10 @@ void saveNpy(BufferManager const& manager, ITensor const& tensor, std::string co auto const dtype = tensor.getDataType(); #ifdef ENABLE_BF16 - if (dtype == nvinfer1::DataType::kBF16) + if (dtype == tensorrt_llm::DataType::kBF16) { TLLM_CHECK(where == MemoryType::kGPU); - auto tensorFp32 = manager.gpu(shape, nvinfer1::DataType::kFLOAT); + auto tensorFp32 = manager.gpu(shape, tensorrt_llm::DataType::kFLOAT); auto dataFp32 = bufferCast<float>(*tensorFp32); auto dataBf16 = bufferCast<__nv_bfloat16 const>(tensor); tc::invokeCudaD2DcpyConvert(dataFp32, dataBf16, tensorSize); diff --git a/cpp/tensorrt_llm/runtime/utils/runtimeUtils.h b/cpp/tensorrt_llm/runtime/utils/runtimeUtils.h index f131ab3419bf..da94419eff64 100644 --- a/cpp/tensorrt_llm/runtime/utils/runtimeUtils.h +++ b/cpp/tensorrt_llm/runtime/utils/runtimeUtils.h @@ -25,7 +25,6 @@ namespace tensorrt_llm::runtime { -class TllmRuntime; namespace utils { diff --git a/cpp/tensorrt_llm/runtime/virtualMemory.cpp b/cpp/tensorrt_llm/runtime/virtualMemory.cpp index 0d08012a29d8..c2ca710db9ac 100644 --- a/cpp/tensorrt_llm/runtime/virtualMemory.cpp +++ b/cpp/tensorrt_llm/runtime/virtualMemory.cpp @@ -16,6 +16,7 @@ #include "tensorrt_llm/runtime/virtualMemory.h" #include "bufferManager.h" +#include "tensorrt_llm/common/tllmDataType.h" #include <forward_list> #include <shared_mutex> @@ -141,8 +142,8 @@ void OffloadConfigurator::teardown(CUmemGenericAllocationHandle, bool destructin { switch (mBackType) { - case MemoryType::kCPU: mBackedStorage = BufferManager::cpu(mSize, nvinfer1::DataType::kINT8); break; - case MemoryType::kPINNED: mBackedStorage = BufferManager::pinned(mSize, nvinfer1::DataType::kINT8); break; + case MemoryType::kCPU: mBackedStorage = BufferManager::cpu(mSize, tensorrt_llm::DataType::kINT8); break; + case MemoryType::kPINNED: mBackedStorage = BufferManager::pinned(mSize, tensorrt_llm::DataType::kINT8); break; default: TLLM_THROW("Unknown memory type: %d", static_cast<int32_t>(mBackType)); } } diff --git a/cpp/tensorrt_llm/testing/CMakeLists.txt b/cpp/tensorrt_llm/testing/CMakeLists.txt deleted file mode 100644 index 646302929817..000000000000 --- a/cpp/tensorrt_llm/testing/CMakeLists.txt +++ /dev/null @@ -1,38 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -include(FetchContent) - -set(SRCS modelSpec.cpp) - -include_directories(${API_INCLUDE_DIR}/tensorrt_llm/runtime) - -if(NOT WIN32) - # additional warnings - # - # Ignore overloaded-virtual warning. We intentionally change parameters of - # some methods in derived class. - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") - if(WARNING_IS_ERROR) - message(STATUS "Treating warnings as errors in GCC compilation") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") - endif() -else() # Windows - # warning level 4 - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") -endif() - -add_library(testing_src OBJECT ${SRCS}) -set_property(TARGET testing_src PROPERTY POSITION_INDEPENDENT_CODE ON) -set_property(TARGET testing_src PROPERTY CUDA_RESOLVE_DEVICE_SYMBOLS ON) diff --git a/cpp/tensorrt_llm/testing/modelSpec.cpp b/cpp/tensorrt_llm/testing/modelSpec.cpp deleted file mode 100644 index bc868a157a1c..000000000000 --- a/cpp/tensorrt_llm/testing/modelSpec.cpp +++ /dev/null @@ -1,303 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "modelSpec.h" -#include "tensorrt_llm/common/dataType.h" - -#include <numeric> - -namespace tensorrt_llm::testing -{ - -std::string ModelSpec::getQuantMethodString() const -{ - switch (mQuantMethod) - { - case QuantMethod::kNONE: - // Bypass here. - break; - case QuantMethod::kSMOOTH_QUANT: return "sq"; break; - default: throw std::runtime_error("Unsupported quant method"); break; - } - - return ""; -} - -std::string ModelSpec::getKVCacheTypeString() const -{ - switch (mKVCacheType) - { - case KVCacheType::kDISABLED: return "no-cache"; break; - case KVCacheType::kPAGED: return "paged"; break; - case KVCacheType::kCONTINUOUS: return "continuous"; break; - default: throw std::runtime_error("Unsupported KV cache type"); break; - } - - return ""; -} - -std::string ModelSpec::getSpeculativeDecodingModeString() const -{ - if (mSpecDecodingMode.isLookaheadDecoding()) - { - return "la-decoding"; - } - else if (mSpecDecodingMode.isDraftTokensExternal()) - { - return "draft-tokens"; - } - else if (mSpecDecodingMode.isNone()) - { - // Bypass here. - } - else if (mSpecDecodingMode.isExplicitDraftTokens()) - { - return "explicit-draft-tokens"; - } - else if (mSpecDecodingMode.isMedusa()) - { - return "medusa"; - } - else if (mSpecDecodingMode.isEagle()) - { - return "eagle"; - } - else - { - throw std::runtime_error("Unsupported decoding mode"); - } - - return ""; -} - -std::string ModelSpec::getCapacitySchedulerString() const -{ - if (mCapacitySchedulerPolicy) - { - if (mCapacitySchedulerPolicy.value() == tensorrt_llm::executor::CapacitySchedulerPolicy::kMAX_UTILIZATION) - { - return "MaxUtilization"; - } - else if (mCapacitySchedulerPolicy.value() - == tensorrt_llm::executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT) - { - return "GuaranteedNoEvict"; - } - else if (mCapacitySchedulerPolicy.value() == tensorrt_llm::executor::CapacitySchedulerPolicy::kSTATIC_BATCH) - { - return "StaticBatch"; - } - else - { - throw std::runtime_error("Unsupported capacity scheduler"); - } - } - return ""; -} - -std::string ModelSpec::getInputFile() const -{ - return mInputFile; -} - -std::string ModelSpec::getModelPath() const -{ - std::vector<std::string> ret; - - ret.emplace_back(getDtypeString()); - - if (mUseGptAttentionPlugin || mUseMambaPlugin) - { - if (mUseGptAttentionPlugin && mUseMambaPlugin) - { - throw std::runtime_error("Cannot use both GPT attention plugin and MAMBA plugin"); - } - - ret.emplace_back("plugin"); - } - else - { - ret.emplace_back("default"); - } - - if (mUsePackedInput) - { - ret.emplace_back("packed"); - } - - ret.emplace_back(getKVCacheTypeString()); - - if (mMaxInputLength) - { - ret.emplace_back("in" + std::to_string(mMaxInputLength)); - } - - ret.emplace_back(getSpeculativeDecodingModeString()); - - if (mUseLoraPlugin) - { - ret.emplace_back("lora"); - } - - ret.emplace_back(getQuantMethodString()); - - if (mUseMultipleProfiles) - { - ret.emplace_back("nprofiles"); - } - - if (mGatherLogits) - { - ret.emplace_back("gather"); - } - - auto finalRet = std::accumulate(ret.begin(), ret.end(), std::string(), - [](std::string& a, std::string& b) - { - if (a.empty()) - { - return b; - } - else - { - return b.empty() ? a : a + "_" + b; - } - }); - - return finalRet; -} - -std::string ModelSpec::getResultsFileInternal(OutputContentType outputContentType) const -{ - std::vector<std::string> ret; - - if (mInputFile == "input_tokens_long.npy") - { - ret.emplace_back("output_tokens_long"); - } - else - { - ret.emplace_back("output_tokens"); - } - - if (mMaxOutputLength) - { - ret.emplace_back("out" + std::to_string(mMaxOutputLength)); - } - - ret.emplace_back(getDtypeString()); - - if (mUseGptAttentionPlugin || mUseMambaPlugin) - { - if (mUseGptAttentionPlugin && mUseMambaPlugin) - { - throw std::runtime_error("Cannot use both GPT attention plugin and MAMBA plugin"); - } - ret.emplace_back("plugin"); - } - - if (mUsePackedInput) - { - ret.emplace_back("packed"); - } - - ret.emplace_back(getKVCacheTypeString()); - - ret.emplace_back(getQuantMethodString()); - - if (mGatherLogits) - { - ret.emplace_back("gather"); - } - - ret.emplace_back("tp" + std::to_string(mTPSize)); - - ret.emplace_back("pp" + std::to_string(mPPSize)); - - ret.emplace_back("cp" + std::to_string(mCPSize)); - - if (mEnableContextFMHAFp32Acc) - { - ret.emplace_back("fmhafp32acc"); - } - - switch (outputContentType) - { - case OutputContentType::kNONE: - // Bypass here. - break; - case OutputContentType::kCONTEXT_LOGITS: ret.emplace_back("logits_context"); break; - case OutputContentType::kGENERATION_LOGITS: ret.emplace_back("logits_generation"); break; - case OutputContentType::kLOG_PROBS: ret.emplace_back("log_probs"); break; - case OutputContentType::kCUM_LOG_PROBS: ret.emplace_back("cum_log_probs"); break; - default: throw std::runtime_error("Unsupported output content type"); break; - } - - auto finalRet = std::accumulate(ret.begin(), ret.end(), std::string(), - [](std::string& a, std::string& b) - { - if (a.empty()) - { - return b; - } - else - { - return b.empty() ? a : a + "_" + b; - } - }); - return finalRet + ".npy"; -} - -std::string ModelSpec::getResultsFile() const -{ - return mOtherModelSpecToCompare ? mOtherModelSpecToCompare->getResultsFileInternal(OutputContentType::kNONE) - : getResultsFileInternal(OutputContentType::kNONE); -} - -std::string ModelSpec::getGenerationLogitsFile() const -{ - return mOtherModelSpecToCompare - ? mOtherModelSpecToCompare->getResultsFileInternal(OutputContentType::kGENERATION_LOGITS) - : getResultsFileInternal(OutputContentType::kGENERATION_LOGITS); -} - -std::string ModelSpec::getContextLogitsFile() const -{ - return mOtherModelSpecToCompare - ? mOtherModelSpecToCompare->getResultsFileInternal(OutputContentType::kCONTEXT_LOGITS) - : getResultsFileInternal(OutputContentType::kCONTEXT_LOGITS); -} - -std::string ModelSpec::getCumLogProbsFile() const -{ - return mOtherModelSpecToCompare - ? mOtherModelSpecToCompare->getResultsFileInternal(OutputContentType::kCUM_LOG_PROBS) - : getResultsFileInternal(OutputContentType::kCUM_LOG_PROBS); -} - -std::string ModelSpec::getLogProbsFile() const -{ - return mOtherModelSpecToCompare ? mOtherModelSpecToCompare->getResultsFileInternal(OutputContentType::kLOG_PROBS) - : getResultsFileInternal(OutputContentType::kLOG_PROBS); -} - -std::string ModelSpec::getDtypeString() const -{ - return tensorrt_llm::common::getDtypeString(mDataType); -} - -} // namespace tensorrt_llm::testing diff --git a/cpp/tensorrt_llm/testing/modelSpec.h b/cpp/tensorrt_llm/testing/modelSpec.h deleted file mode 100644 index 5b6f88dcd135..000000000000 --- a/cpp/tensorrt_llm/testing/modelSpec.h +++ /dev/null @@ -1,342 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "NvInfer.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/speculativeDecodingMode.h" - -#include <filesystem> -#include <vector> - -namespace tensorrt_llm::testing -{ - -using tensorrt_llm::runtime::SizeType32; -using tensorrt_llm::runtime::SpeculativeDecodingMode; -using KVCacheType = tensorrt_llm::runtime::ModelConfig::KVCacheType; - -enum class QuantMethod -{ - kNONE, - kSMOOTH_QUANT, -}; - -enum class OutputContentType -{ - kNONE, - kCONTEXT_LOGITS, - kGENERATION_LOGITS, - kLOG_PROBS, - kCUM_LOG_PROBS -}; - -class ModelSpec -{ -public: - ModelSpec(std::string const& inputFile, nvinfer1::DataType dtype, - std::shared_ptr<ModelSpec> otherModelSpecToCompare = nullptr) - : mInputFile{std::move(inputFile)} - , mDataType{dtype} - , mOtherModelSpecToCompare(otherModelSpecToCompare) - { - } - - ModelSpec& setInputFile(std::string const& inputFile) - { - mInputFile = inputFile; - return *this; - } - - ModelSpec& useGptAttentionPlugin() - { - mUseGptAttentionPlugin = true; - return *this; - } - - ModelSpec& usePackedInput() - { - mUsePackedInput = true; - return *this; - } - - ModelSpec& setKVCacheType(KVCacheType kvCacheType) - { - mKVCacheType = kvCacheType; - return *this; - } - - ModelSpec& setKVCacheReuse(bool kvCacheReuse) - { - mKVCacheReuse = kvCacheReuse; - return *this; - } - - ModelSpec& useDecoderPerRequest() - { - mDecoderPerRequest = true; - return *this; - } - - ModelSpec& useTensorParallelism(int tensorParallelism) - { - mTPSize = tensorParallelism; - return *this; - } - - ModelSpec& usePipelineParallelism(int pipelineParallelism) - { - mPPSize = pipelineParallelism; - return *this; - } - - ModelSpec& useContextParallelism(int contextParallelism) - { - mCPSize = contextParallelism; - return *this; - } - - ModelSpec& setDraftTokens(SizeType32 maxDraftTokens) - { - mMaxDraftTokens = maxDraftTokens; - return *this; - } - - ModelSpec& useAcceptByLogits() - { - mAcceptDraftByLogits = true; - return *this; - } - - ModelSpec& useMambaPlugin() - { - mUseMambaPlugin = true; - return *this; - } - - ModelSpec& gatherLogits() - { - mGatherLogits = true; - return *this; - } - - ModelSpec& replaceLogits() - { - mReplaceLogits = true; - return *this; - } - - ModelSpec& returnLogProbs() - { - mReturnLogProbs = true; - return *this; - } - - ModelSpec& smokeTest() - { - mSmokeTest = true; - return *this; - } - - ModelSpec& useMedusa() - { - mSpecDecodingMode = SpeculativeDecodingMode::Medusa(); - return *this; - } - - ModelSpec& useEagle() - { - mSpecDecodingMode = SpeculativeDecodingMode::Eagle(); - return *this; - } - - ModelSpec& useLookaheadDecoding() - { - mSpecDecodingMode = SpeculativeDecodingMode::LookaheadDecoding(); - return *this; - } - - ModelSpec& useExplicitDraftTokensDecoding() - { - mSpecDecodingMode = SpeculativeDecodingMode::ExplicitDraftTokens(); - return *this; - } - - ModelSpec& useDraftTokensExternalDecoding() - { - mSpecDecodingMode = SpeculativeDecodingMode::DraftTokensExternal(); - return *this; - } - - [[nodiscard]] bool useLogits() const - { - return mGatherLogits || mReplaceLogits; - } - - ModelSpec& useMultipleProfiles() - { - mUseMultipleProfiles = true; - return *this; - } - - ModelSpec& enableContextFMHAFp32Acc() - { - mEnableContextFMHAFp32Acc = true; - return *this; - } - - [[nodiscard]] bool getEnableContextFMHAFp32Acc() const - { - return mEnableContextFMHAFp32Acc; - } - - ModelSpec& setMaxInputLength(SizeType32 maxInputLength) - { - mMaxInputLength = maxInputLength; - return *this; - } - - ModelSpec& setMaxOutputLength(SizeType32 maxOutputLength) - { - mMaxOutputLength = maxOutputLength; - return *this; - } - - ModelSpec& setQuantMethod(QuantMethod quantMethod) - { - mQuantMethod = quantMethod; - return *this; - } - - ModelSpec& useLoraPlugin() - { - mUseLoraPlugin = true; - return *this; - } - - ModelSpec& collectGenerationLogitsFile() - { - mCollectGenerationLogits = true; - return *this; - } - - ModelSpec& collectContextLogitsFile() - { - mCollectContextLogits = true; - return *this; - } - - ModelSpec& collectCumLogProbsFile() - { - mCollectCumLogProbs = true; - return *this; - } - - ModelSpec& collectLogProbsFile() - { - mCollectLogProbs = true; - return *this; - } - - ModelSpec& capacitySchedulerPolicy(tensorrt_llm::executor::CapacitySchedulerPolicy policy) - { - mCapacitySchedulerPolicy = policy; - return *this; - } - - friend std::ostream& operator<<(std::ostream& os, ModelSpec const& modelSpec) - { - return os << modelSpec.getModelPath(); - } - - // Computed properties - [[nodiscard]] std::string getInputFile() const; - - [[nodiscard]] std::string getModelPath() const; - - [[nodiscard]] std::string getResultsFileInternal( - OutputContentType outputContentType = OutputContentType::kNONE) const; - - [[nodiscard]] std::string getResultsFile() const; - [[nodiscard]] std::string getGenerationLogitsFile() const; - - [[nodiscard]] std::string getContextLogitsFile() const; - - [[nodiscard]] std::string getCumLogProbsFile() const; - - [[nodiscard]] std::string getLogProbsFile() const; - - [[nodiscard]] std::string getDtypeString() const; - - [[nodiscard]] std::string getQuantMethodString() const; - - [[nodiscard]] std::string getKVCacheTypeString() const; - - [[nodiscard]] std::string getSpeculativeDecodingModeString() const; - - [[nodiscard]] std::string getCapacitySchedulerString() const; - - static ModelSpec getDefaultModelSpec() - { - static ModelSpec modelSpec{"input_tokens.npy", nvinfer1::DataType::kHALF}; - modelSpec.useGptAttentionPlugin().setKVCacheType(KVCacheType::kPAGED).usePackedInput(); - - return modelSpec; - } - - std::string mInputFile; - nvinfer1::DataType mDataType; - - bool mUseGptAttentionPlugin{false}; - bool mUsePackedInput{false}; - KVCacheType mKVCacheType{KVCacheType::kCONTINUOUS}; - bool mKVCacheReuse{false}; - bool mDecoderPerRequest{false}; - int mPPSize{1}; - int mTPSize{1}; - int mCPSize{1}; - int mMaxDraftTokens{0}; - bool mAcceptDraftByLogits{false}; - bool mUseMambaPlugin{false}; - bool mGatherLogits{false}; - bool mReplaceLogits{false}; - bool mReturnLogProbs{false}; - bool mSmokeTest{false}; - bool mUseMultipleProfiles{false}; - int mMaxInputLength{0}; - int mMaxOutputLength{0}; - bool mUseLoraPlugin{false}; - bool mEnableContextFMHAFp32Acc{false}; - - // Flags to store whether model spec wants collect these outputs, you could call getXXXFile() if you need the name. - bool mCollectGenerationLogits{false}; - bool mCollectContextLogits{false}; - bool mCollectCumLogProbs{false}; - bool mCollectLogProbs{false}; - QuantMethod mQuantMethod{QuantMethod::kNONE}; - - SpeculativeDecodingMode mSpecDecodingMode{SpeculativeDecodingMode::None()}; - - std::optional<tensorrt_llm::executor::CapacitySchedulerPolicy> mCapacitySchedulerPolicy{std::nullopt}; - - // Sometimes, we need to compare with another model spec for golden results. - std::shared_ptr<ModelSpec> mOtherModelSpecToCompare{nullptr}; -}; - -}; // namespace tensorrt_llm::testing diff --git a/cpp/tensorrt_llm/thop/CMakeLists.txt b/cpp/tensorrt_llm/thop/CMakeLists.txt index dccfbbbf6a43..7af68a90f1c9 100644 --- a/cpp/tensorrt_llm/thop/CMakeLists.txt +++ b/cpp/tensorrt_llm/thop/CMakeLists.txt @@ -31,8 +31,11 @@ endif() add_library(th_utils STATIC thUtils.cpp) set_property(TARGET th_utils PROPERTY POSITION_INDEPENDENT_CODE ON) set_property(TARGET th_utils PROPERTY CUDA_RESOLVE_DEVICE_SYMBOLS ON) -target_link_libraries(th_utils PUBLIC ${TORCH_LIBRARIES} ${CUBLAS_LIB} - ${CURAND_LIB}) +# Declare the dependency on the main shared library explicitly so consumers +# (e.g. thUtilsTest) place it after th_utils on the link line; this was +# previously satisfied transitively via the removed TensorRT plugin target. +target_link_libraries(th_utils PUBLIC ${SHARED_TARGET} ${TORCH_LIBRARIES} + ${CUBLAS_LIB} ${CURAND_LIB}) # TODO This does not compile with internal cutlass MOE gemm add_library( @@ -75,6 +78,7 @@ add_library( kdaDecodeOp.cpp attnResOp.cpp fusedQKNormRopeOp.cpp + fusedAdaptiveLayerNormOp.cpp fusedDiTQKNormRopeOp.cpp fusedDiTSplitQKNormRopeOp.cpp fusedDiTSplitNormOp.cpp @@ -112,6 +116,7 @@ add_library( IndexerKCacheGatherOp.cpp IndexerKCacheScatterOp.cpp IndexerTopKOp.cpp + sparseKvCacheCompactOp.cpp mlaRopeInplaceOp.cpp ncclCommunicatorOp.cpp allocateOutput.cpp diff --git a/cpp/tensorrt_llm/thop/IndexerTopKOp.cpp b/cpp/tensorrt_llm/thop/IndexerTopKOp.cpp index 1981c417dbae..fc29041bbf93 100644 --- a/cpp/tensorrt_llm/thop/IndexerTopKOp.cpp +++ b/cpp/tensorrt_llm/thop/IndexerTopKOp.cpp @@ -19,7 +19,6 @@ #include "tensorrt_llm/kernels/IndexerTopK.h" -// #include <NvInferRuntime.h> // #include <c10/cuda/CUDAStream.h> // #include <cassert> // #include <set> diff --git a/cpp/tensorrt_llm/thop/allgatherOp.cpp b/cpp/tensorrt_llm/thop/allgatherOp.cpp index 0d92aa966901..5f7d4571d258 100644 --- a/cpp/tensorrt_llm/thop/allgatherOp.cpp +++ b/cpp/tensorrt_llm/thop/allgatherOp.cpp @@ -20,7 +20,6 @@ #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/pgUtils.h" -#include <NvInferRuntime.h> #include <c10/cuda/CUDAStream.h> #include <cassert> #include <set> diff --git a/cpp/tensorrt_llm/thop/allreduceOp.cpp b/cpp/tensorrt_llm/thop/allreduceOp.cpp index e29b6cf5a892..116df25d7a00 100644 --- a/cpp/tensorrt_llm/thop/allreduceOp.cpp +++ b/cpp/tensorrt_llm/thop/allreduceOp.cpp @@ -23,6 +23,7 @@ #include "tensorrt_llm/common/ncclUtils.h" #include "tensorrt_llm/common/nvmlWrapper.h" #include "tensorrt_llm/common/opUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h" #include "tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h" @@ -61,7 +62,6 @@ #include <limits> #include <unordered_set> -// using namespace nvinfer1; using tensorrt_llm::kernels::AllReduceFusionOp; using tensorrt_llm::kernels::AllReduceStrategyType; using tensorrt_llm::mpi::MpiTag; @@ -234,8 +234,8 @@ std::set<int> getLocalGroupTorch(std::set<int> const& group) class AllreduceOp { public: - AllreduceOp( - std::set<int> group, nvinfer1::DataType type, AllReduceStrategyType strategy, AllReduceFusionOp op, float eps) + AllreduceOp(std::set<int> group, tensorrt_llm::DataType type, AllReduceStrategyType strategy, AllReduceFusionOp op, + float eps) : mGroup(std::move(group)) , mIsNVLINKSupported(false) , mIsP2PSupported(false) @@ -248,7 +248,7 @@ class AllreduceOp } AllreduceOp(std::set<int> group, c10::intrusive_ptr<c10d::ProcessGroup> const& process_group_, - nvinfer1::DataType type, AllReduceStrategyType strategy, AllReduceFusionOp op, float eps) + tensorrt_llm::DataType type, AllReduceStrategyType strategy, AllReduceFusionOp op, float eps) : mGroup(std::move(group)) , mIsNVLINKSupported(false) , mIsP2PSupported(false) @@ -348,7 +348,7 @@ class AllreduceOp { TORCH_CHECK(norm_weight, "norm_weight is required for residual rms norm allreduce"); TORCH_CHECK(!bias, "bias is not supported for residual rms norm allreduce"); - TORCH_CHECK(mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kBF16); + TORCH_CHECK(mType == tensorrt_llm::DataType::kHALF || mType == tensorrt_llm::DataType::kBF16); auto [norm_out, ub_buffer1] = torch_ext::create_userbuffers_tensor(input.sizes(), input.scalar_type()); tensorrt_llm::kernels::ub::allreduce2_userbuff_rmsnorm_launcher(ub_buffer0.handle, 0, ub_buffer1.handle, 0, size, hidden_size, nullptr, norm_weight.value().data_ptr(), mEps, residual.value().data_ptr(), @@ -1461,7 +1461,7 @@ class AllreduceOp bool mIsNVLINKSupported; bool mIsP2PSupported; bool mIsMNNVLSupported; - nvinfer1::DataType mType; + tensorrt_llm::DataType mType; AllReduceStrategyType mStrategy; AllReduceFusionOp mOp; float mEps; diff --git a/cpp/tensorrt_llm/thop/attentionOp.cpp b/cpp/tensorrt_llm/thop/attentionOp.cpp index 848554c512d6..24d869f704dc 100644 --- a/cpp/tensorrt_llm/thop/attentionOp.cpp +++ b/cpp/tensorrt_llm/thop/attentionOp.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/attentionOp.h" #include "tensorrt_llm/common/attentionWorkspace.h" #include "tensorrt_llm/common/dataType.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/flashMLA/flash_mla.h" #include "tensorrt_llm/kernels/gptKernels.h" #include "tensorrt_llm/kernels/mlaKernels.h" @@ -28,7 +29,9 @@ #include "tensorrt_llm/thop/thUtils.h" #include <cstdint> #include <functional> +#include <memory> #include <torch/extension.h> +#include <tuple> #include <type_traits> #include <unordered_set> @@ -38,7 +41,7 @@ namespace torch_ext { using tensorrt_llm::common::op::AttentionOp; using tensorrt_llm::common::op::AttentionWorkspaceManager; -using tensorrt_llm::common::op::hash; +using tensorrt_llm::common::op::OpCustomHash; using tensorrt_llm::runtime::RequestType; namespace @@ -1037,6 +1040,32 @@ using RunnerPtr = std::shared_ptr<torch_ext::trtllm::attention::RunnerBase>; using torch_ext::trtllm::attention::Runner; using torch_ext::trtllm::attention::AttentionInputType; +static std::shared_ptr<AttentionOp> get_attention_op( + RunnerPtr const& runner, std::shared_ptr<AttentionOp>& op, int64_t local_layer_idx) +{ + auto cache_key = std::make_tuple(op->data(), runner->data()); + using CacheKey = decltype(cache_key); + static std::unordered_map<CacheKey, std::shared_ptr<AttentionOp>, OpCustomHash<CacheKey>> op_cache; + static std::shared_mutex op_cache_mutex; + + std::shared_lock<std::shared_mutex> read_lock{op_cache_mutex}; + + if (auto iter = op_cache.find(cache_key); iter != op_cache.end()) + { + TLLM_LOG_TRACE("Attention op for layer %lld is cached", local_layer_idx); + return iter->second; + } + + read_lock.unlock(); + TLLM_LOG_TRACE( + "Attention op for layer %lld is not cached, cache key: %s", local_layer_idx, to_string(cache_key).c_str()); + std::unique_lock<std::shared_mutex> lock{op_cache_mutex}; + op->initialize(); + runner->prepare(*op); + auto [iter, _] = op_cache.try_emplace(cache_key, op); + return iter->second; +} + void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<torch::Tensor> v, torch::Tensor& output, std::optional<torch::Tensor> output_sf, std::optional<torch::Tensor> workspace_, torch::Tensor sequence_length, torch::Tensor host_past_key_value_lengths, torch::Tensor host_total_kv_lens, torch::Tensor context_lengths, @@ -1084,7 +1113,8 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to std::optional<int64_t> compressed_kv_cache_pool_ptr, bool const is_cross, std::optional<torch::Tensor> cross_kv, std::optional<torch::Tensor> relative_attention_bias, int64_t relative_attention_max_distance, std::optional<int64_t> spec_decoding_target_max_draft_tokens, std::optional<torch::Tensor> quant_scale_qkv, - std::optional<torch::Tensor> dsv4_inv_rope_cos_sin_cache, bool enable_dsv4_epilogue_fusion) + std::optional<torch::Tensor> dsv4_inv_rope_cos_sin_cache, bool enable_dsv4_epilogue_fusion, + bool const force_prepare_spec_dec_tree_mask) { TLLM_LOG_TRACE("Attention op starts at layer %d", local_layer_idx); // Use these tensors to infer if the attention is using KV cache @@ -1123,7 +1153,7 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to bool const is_fp4_out = out_dtype == torch::kUInt8; RunnerPtr runner; - if (dtype == nvinfer1::DataType::kHALF) + if (dtype == tensorrt_llm::DataType::kHALF) { if (is_fp8_out) { @@ -1139,13 +1169,13 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to runner = std::make_shared<Runner<half>>(); } } - else if (dtype == nvinfer1::DataType::kFLOAT) + else if (dtype == tensorrt_llm::DataType::kFLOAT) { TLLM_CHECK(out_dtype == torch::kFloat32); runner = std::make_shared<Runner<float>>(); } #ifdef ENABLE_BF16 - else if (dtype == nvinfer1::DataType::kBF16) + else if (dtype == tensorrt_llm::DataType::kBF16) { if (is_fp8_out) { @@ -1168,7 +1198,7 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to auto op = std::make_shared<AttentionOp>(); op->mType = dtype; - op->mFMHAForceFP32Acc = dtype == nvinfer1::DataType::kBF16; + op->mFMHAForceFP32Acc = dtype == tensorrt_llm::DataType::kBF16; op->mLayerIdx = local_layer_idx; op->mNumHeads = num_heads; op->mNumKVHeads = num_kv_heads; @@ -1237,6 +1267,7 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to { op->mSpecDecodingTargetMaxGenLen = static_cast<int32_t>(spec_decoding_target_max_draft_tokens.value()) + 1; } + op->mForcePrepareSpecDecTreeMask = force_prepare_spec_dec_tree_mask; op->mUseSparseAttention = false; op->mUseTllmGenSparseAttentionPaged = false; @@ -1302,22 +1333,7 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to = chunked_prefill_buffer_batch_size.has_value() ? chunked_prefill_buffer_batch_size.value() : 1; } - auto cache_key = std::make_tuple(op->data(), runner->data()); - using CacheKey = decltype(cache_key); - static std::unordered_map<CacheKey, std::shared_ptr<AttentionOp>, hash<CacheKey>> op_cache; - if (auto it = op_cache.find(cache_key); it != op_cache.end()) - { - TLLM_LOG_TRACE("Attention op for layer %d is cached", local_layer_idx); - op = it->second; - } - else - { - TLLM_LOG_TRACE("Preparing new attention op for layer %d with cache key: %s", local_layer_idx, - to_string(cache_key).c_str()); - op->initialize(); - runner->prepare(*op); - op_cache[cache_key] = op; - } + op = get_attention_op(runner, op, local_layer_idx); int32_t const num_seqs = host_context_lengths.size(0); RequestType const* request_types = static_cast<RequestType const*>(host_request_types.data_ptr()); @@ -1431,7 +1447,7 @@ bool attention_supports_nvfp4_output(int64_t const num_heads, int64_t const num_ } auto op = std::make_shared<AttentionOp>(); - op->mType = nvinfer1::DataType::kHALF; + op->mType = tensorrt_llm::DataType::kHALF; op->mNumHeads = num_heads; op->mNumKVHeads = num_kv_heads; op->mHeadSize = head_size; @@ -1446,7 +1462,7 @@ bool attention_supports_nvfp4_output(int64_t const num_heads, int64_t const num_ auto cache_key = op->data(); using CacheKey = decltype(cache_key); - static std::unordered_map<CacheKey, bool, hash<CacheKey>> op_cache; + static std::unordered_map<CacheKey, bool, OpCustomHash<CacheKey>> op_cache; if (auto it = op_cache.find(cache_key); it != op_cache.end()) { TLLM_LOG_TRACE("Attention op runtime check is cached"); diff --git a/cpp/tensorrt_llm/thop/attentionOp.h b/cpp/tensorrt_llm/thop/attentionOp.h index b4f0e90bc9a7..f28209166e0d 100644 --- a/cpp/tensorrt_llm/thop/attentionOp.h +++ b/cpp/tensorrt_llm/thop/attentionOp.h @@ -95,7 +95,8 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to std::optional<torch::Tensor> relative_attention_bias = std::nullopt, int64_t relative_attention_max_distance = 0, std::optional<int64_t> spec_decoding_target_max_draft_tokens = std::nullopt, std::optional<torch::Tensor> quant_scale_qkv = std::nullopt, - std::optional<torch::Tensor> dsv4_inv_rope_cos_sin_cache = std::nullopt, bool enable_dsv4_epilogue_fusion = false); + std::optional<torch::Tensor> dsv4_inv_rope_cos_sin_cache = std::nullopt, bool enable_dsv4_epilogue_fusion = false, + bool const force_prepare_spec_dec_tree_mask = false); struct KvCachePoolPointers { diff --git a/cpp/tensorrt_llm/thop/cublasFp4ScaledMM.cpp b/cpp/tensorrt_llm/thop/cublasFp4ScaledMM.cpp index a9ad46ad8f04..8e4da99dbadc 100644 --- a/cpp/tensorrt_llm/thop/cublasFp4ScaledMM.cpp +++ b/cpp/tensorrt_llm/thop/cublasFp4ScaledMM.cpp @@ -16,7 +16,6 @@ #include "tensorrt_llm/common/cublasMMWrapper.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/plugins/common/plugin.h" #include "tensorrt_llm/thop/outputTensor.h" #include "tensorrt_llm/thop/thUtils.h" #include "userbuffersTensor.h" diff --git a/cpp/tensorrt_llm/thop/cublasScaledMM.cpp b/cpp/tensorrt_llm/thop/cublasScaledMM.cpp index dea6f51363e5..62f51f7b06f9 100644 --- a/cpp/tensorrt_llm/thop/cublasScaledMM.cpp +++ b/cpp/tensorrt_llm/thop/cublasScaledMM.cpp @@ -18,8 +18,6 @@ #include "tensorrt_llm/common/cublasMMWrapper.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/kernels/userbuffers/ub_interface.h" -#include "tensorrt_llm/plugins/common/plugin.h" -#include "tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h" #include "tensorrt_llm/runtime/torchUtils.h" #include "tensorrt_llm/thop/outputTensor.h" #include "tensorrt_llm/thop/thUtils.h" diff --git a/cpp/tensorrt_llm/thop/dynamicDecodeOp.cpp b/cpp/tensorrt_llm/thop/dynamicDecodeOp.cpp index 8e9e817bbb51..228b2c614ab7 100644 --- a/cpp/tensorrt_llm/thop/dynamicDecodeOp.cpp +++ b/cpp/tensorrt_llm/thop/dynamicDecodeOp.cpp @@ -16,6 +16,7 @@ #include "tensorrt_llm/thop/dynamicDecodeOp.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/runtime/bufferManager.h" @@ -54,7 +55,7 @@ FtDynamicDecode<T>::FtDynamicDecode(size_t const maxBatchSize, size_t const maxB auto bufferManager = std::make_shared<tensorrt_llm::runtime::BufferManager>(cudaStreamPtr); mFinishedSum = bufferManager->pinnedPool( - tr::ITensor::makeShape({static_cast<int32_t>(maxBatchSize)}), nvinfer1::DataType::kINT32); + tr::ITensor::makeShape({static_cast<int32_t>(maxBatchSize)}), tensorrt_llm::DataType::kINT32); mDynamicDecodeLayer = std::make_shared<tl::DynamicDecodeLayer<T>>(tle::DecodingMode::Auto(), decodingDomain, bufferManager); mBatchSlots = tr::getDefaultBatchSlots(maxBatchSize); diff --git a/cpp/tensorrt_llm/thop/fp8Quantize.cpp b/cpp/tensorrt_llm/thop/fp8Quantize.cpp index 35ba5c440e18..43eea8cff838 100644 --- a/cpp/tensorrt_llm/thop/fp8Quantize.cpp +++ b/cpp/tensorrt_llm/thop/fp8Quantize.cpp @@ -209,6 +209,38 @@ std::tuple<at::Tensor, at::Tensor> fp8_quantize_1x128_packed_ue8m0(at::Tensor co return {valueE4M3.slice(0, 0, m), packedScale}; } + +std::tuple<at::Tensor, at::Tensor> fp8_quantize_1x128_cutedsl_ue8m0(at::Tensor const& self) +{ + CHECK_TH_CUDA(self); + CHECK_CONTIGUOUS(self); + + TORCH_CHECK(self.scalar_type() == at::ScalarType::BFloat16, "Input matrix dtype must be BF16."); + TORCH_CHECK(self.dim() == 2, "input must be a matrix"); + TORCH_CHECK(tensorrt_llm::common::isSM100Family(), + "fp8_quantize_1x128_cutedsl_ue8m0 currently only supports SM100 (Blackwell)."); + + auto const m = self.sizes()[0]; + auto const k = self.sizes()[1]; + TORCH_CHECK(m <= std::numeric_limits<int32_t>::max(), "M must be within int32"); + TORCH_CHECK(k <= std::numeric_limits<int32_t>::max(), "K must be within int32"); + TORCH_CHECK(k % 128 == 0, "K must be divisible by the production FP8 block size 128, but got ", k); + + at::Tensor valueE4M3 + = at::detail::empty_cuda({m, k}, at::ScalarType::Float8_e4m3fn, self.device(), /* stride */ std::nullopt); + auto const paddedM = (m + 127) / 128 * 128; + auto const sfCols = (k / 32 + 3) / 4 * 4; + at::Tensor scaleE8M0 + = at::detail::empty_cuda({paddedM * sfCols}, at::ScalarType::Byte, self.device(), /* stride */ std::nullopt); + + auto stream = at::cuda::getCurrentCUDAStream(self.get_device()); + tensorrt_llm::kernels::fp8_blockscale_gemm::launch_fp8_quantize_1x128_cutedsl_bf16_e4m3( + reinterpret_cast<__nv_fp8_e4m3*>(valueE4M3.data_ptr()), scaleE8M0.data_ptr<uint8_t>(), + reinterpret_cast<__nv_bfloat16 const*>(self.data_ptr()), static_cast<int>(m), static_cast<int>(k), + static_cast<int>(paddedM), stream); + + return {valueE4M3, scaleE8M0}; +} } // namespace torch_ext TRTLLM_NAMESPACE_END @@ -218,6 +250,7 @@ TORCH_LIBRARY_FRAGMENT(trtllm, m) m.def("fp8_quantize_1x128(Tensor input, bool use_ue8m0=False) -> (Tensor, Tensor)"); m.def("fp8_batched_quantize_1x128_permute102(Tensor input) -> (Tensor, Tensor)"); m.def("fp8_quantize_1x128_packed_ue8m0(Tensor input) -> (Tensor, Tensor)"); + m.def("fp8_quantize_1x128_cutedsl_ue8m0(Tensor input) -> (Tensor, Tensor)"); } TORCH_LIBRARY_IMPL(trtllm, CUDA, m) @@ -225,4 +258,5 @@ TORCH_LIBRARY_IMPL(trtllm, CUDA, m) m.impl("fp8_quantize_1x128", &tensorrt_llm::torch_ext::fp8_quantize_1x128); m.impl("fp8_batched_quantize_1x128_permute102", &tensorrt_llm::torch_ext::fp8_batched_quantize_1x128_permute102); m.impl("fp8_quantize_1x128_packed_ue8m0", &tensorrt_llm::torch_ext::fp8_quantize_1x128_packed_ue8m0); + m.impl("fp8_quantize_1x128_cutedsl_ue8m0", &tensorrt_llm::torch_ext::fp8_quantize_1x128_cutedsl_ue8m0); } diff --git a/cpp/tensorrt_llm/thop/fusedAdaptiveLayerNormOp.cpp b/cpp/tensorrt_llm/thop/fusedAdaptiveLayerNormOp.cpp new file mode 100644 index 000000000000..01fa055f0ac8 --- /dev/null +++ b/cpp/tensorrt_llm/thop/fusedAdaptiveLayerNormOp.cpp @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/kernels/fusedAdaptiveLayerNormKernel.h" +#include "tensorrt_llm/thop/thUtils.h" + +TRTLLM_NAMESPACE_BEGIN + +namespace torch_ext +{ + +namespace +{ + +// Validate and fill common fields of AdaptiveLayerNormParams. +// Returns {M, D, has_ln_affine, has_modulation}. +struct ValidatedParams +{ + int64_t M; + int64_t D; + bool has_ln_affine; + bool has_modulation; +}; + +ValidatedParams validateInputs(at::Tensor const& x, std::optional<at::Tensor> const& ln_weight, + std::optional<at::Tensor> const& ln_bias, std::optional<at::Tensor> const& scale_msa, + std::optional<at::Tensor> const& shift_msa, int64_t seq_len_per_batch) +{ + CHECK_TH_CUDA(x); + TORCH_CHECK(x.is_contiguous(), "x must be contiguous"); + TORCH_CHECK(x.dim() == 2, "x must be 2D [M, D], got ", x.dim(), " dims"); + CHECK_TYPE(x, torch::kBFloat16); + + int64_t const M = x.size(0); + int64_t const D = x.size(1); + TORCH_CHECK(D == 5120, "fused_adaptive_layernorm only supports D=5120 (got ", D, ")"); + TORCH_CHECK(D % 16 == 0, "D must be divisible by 16 (NVFP4 group size)"); + + TORCH_CHECK(ln_weight.has_value() == ln_bias.has_value(), "ln_weight and ln_bias must both be provided together"); + TORCH_CHECK( + scale_msa.has_value() == shift_msa.has_value(), "scale_msa and shift_msa must both be provided together"); + + bool const has_ln_affine = ln_weight.has_value(); + bool const has_modulation = scale_msa.has_value(); + + TORCH_CHECK(!(has_ln_affine && has_modulation), + "fused_adaptive_layernorm: ln_weight/ln_bias and scale_msa/shift_msa are mutually exclusive"); + + if (has_ln_affine) + { + CHECK_TH_CUDA(ln_weight.value()); + CHECK_TH_CUDA(ln_bias.value()); + TORCH_CHECK(ln_weight->device() == x.device(), "ln_weight must be on the same device as x"); + TORCH_CHECK(ln_bias->device() == x.device(), "ln_bias must be on the same device as x"); + TORCH_CHECK(ln_weight->dim() == 1 && ln_weight->size(0) == D, "ln_weight must be 1D [D]"); + TORCH_CHECK(ln_bias->dim() == 1 && ln_bias->size(0) == D, "ln_bias must be 1D [D]"); + CHECK_TYPE(ln_weight.value(), torch::kBFloat16); + CHECK_TYPE(ln_bias.value(), torch::kBFloat16); + TORCH_CHECK(ln_weight->is_contiguous(), "ln_weight must be contiguous"); + TORCH_CHECK(ln_bias->is_contiguous(), "ln_bias must be contiguous"); + } + + if (has_modulation) + { + CHECK_TH_CUDA(scale_msa.value()); + CHECK_TH_CUDA(shift_msa.value()); + TORCH_CHECK(scale_msa->device() == x.device(), "scale_msa must be on the same device as x"); + TORCH_CHECK(shift_msa->device() == x.device(), "shift_msa must be on the same device as x"); + TORCH_CHECK(seq_len_per_batch > 0, "seq_len_per_batch must be positive when using AdaLN modulation"); + TORCH_CHECK( + M % seq_len_per_batch == 0, "M (", M, ") must be divisible by seq_len_per_batch (", seq_len_per_batch, ")"); + int64_t const B = M / seq_len_per_batch; + TORCH_CHECK(scale_msa->dim() == 2 && scale_msa->size(0) == B && scale_msa->size(1) == D, + "scale_msa must be [B, D] = [", B, ", ", D, "]"); + TORCH_CHECK(shift_msa->dim() == 2 && shift_msa->size(0) == B && shift_msa->size(1) == D, + "shift_msa must be [B, D] = [", B, ", ", D, "]"); + CHECK_TYPE(scale_msa.value(), torch::kBFloat16); + CHECK_TYPE(shift_msa.value(), torch::kBFloat16); + TORCH_CHECK(scale_msa->is_contiguous(), "scale_msa must be contiguous"); + TORCH_CHECK(shift_msa->is_contiguous(), "shift_msa must be contiguous"); + } + + return {M, D, has_ln_affine, has_modulation}; +} + +} // namespace + +at::Tensor fused_adaptive_layernorm(at::Tensor const& x, std::optional<at::Tensor> const& ln_weight, + std::optional<at::Tensor> const& ln_bias, std::optional<at::Tensor> const& scale_msa, + std::optional<at::Tensor> const& shift_msa, int64_t seq_len_per_batch, double eps) +{ + auto const v = validateInputs(x, ln_weight, ln_bias, scale_msa, shift_msa, seq_len_per_batch); + + at::Tensor out = torch::empty_like(x); + + auto stream = at::cuda::getCurrentCUDAStream(x.get_device()); + + tensorrt_llm::kernels::AdaptiveLayerNormParams params; + params.x = reinterpret_cast<__nv_bfloat16 const*>(x.data_ptr()); + params.ln_weight = v.has_ln_affine ? reinterpret_cast<__nv_bfloat16 const*>(ln_weight->data_ptr()) : nullptr; + params.ln_bias = v.has_ln_affine ? reinterpret_cast<__nv_bfloat16 const*>(ln_bias->data_ptr()) : nullptr; + params.scale_msa = v.has_modulation ? reinterpret_cast<__nv_bfloat16 const*>(scale_msa->data_ptr()) : nullptr; + params.shift_msa = v.has_modulation ? reinterpret_cast<__nv_bfloat16 const*>(shift_msa->data_ptr()) : nullptr; + params.out_bf16 = reinterpret_cast<__nv_bfloat16*>(out.data_ptr()); + params.out_fp4 = nullptr; + params.out_sf = nullptr; + params.sf_scale = nullptr; + params.M = static_cast<int>(v.M); + params.D = static_cast<int>(v.D); + params.seq_len_per_batch = static_cast<int>(seq_len_per_batch); + params.eps = static_cast<float>(eps); + + tensorrt_llm::kernels::launchFusedAdaptiveLayerNormKernel( + params, v.has_ln_affine, v.has_modulation, /*has_quant=*/false, stream); + + return out; +} + +std::tuple<at::Tensor, at::Tensor> fused_adaptive_layernorm_quant(at::Tensor const& x, + std::optional<at::Tensor> const& ln_weight, std::optional<at::Tensor> const& ln_bias, + std::optional<at::Tensor> const& scale_msa, std::optional<at::Tensor> const& shift_msa, at::Tensor const& sf_scale, + int64_t seq_len_per_batch, double eps) +{ + auto const v = validateInputs(x, ln_weight, ln_bias, scale_msa, shift_msa, seq_len_per_batch); + + CHECK_INPUT(sf_scale, torch::kFloat32); + TORCH_CHECK(sf_scale.device() == x.device(), "sf_scale must be on the same device as x"); + TORCH_CHECK(sf_scale.numel() == 1, "sf_scale must be a scalar tensor (1 element), got numel=", sf_scale.numel()); + + int64_t const M = v.M; + int64_t const D = v.D; + + // y_fp4: [M, D/2] uint8 (2 FP4 nibbles per byte). + auto const opt_u8 = torch::TensorOptions().dtype(FLOAT4_E2M1X2).device(x.device()); + at::Tensor y_fp4 = torch::empty({M, D / 2}, opt_u8); + + // sf_out: swizzled NVFP4 scale factors. The layout tiles are 128×4 in (tokens, sf_cols), + // so we round up both dimensions before allocating. + int64_t const sf_cols = D / 16; + int64_t const sfSize = ((M + 127) / 128 * 128) * ((sf_cols + 3) / 4 * 4); + at::Tensor sf_out = torch::empty({sfSize}, torch::TensorOptions().dtype(SF_DTYPE).device(x.device())); + + auto stream = at::cuda::getCurrentCUDAStream(x.get_device()); + + tensorrt_llm::kernels::AdaptiveLayerNormParams params; + params.x = reinterpret_cast<__nv_bfloat16 const*>(x.data_ptr()); + params.ln_weight = v.has_ln_affine ? reinterpret_cast<__nv_bfloat16 const*>(ln_weight->data_ptr()) : nullptr; + params.ln_bias = v.has_ln_affine ? reinterpret_cast<__nv_bfloat16 const*>(ln_bias->data_ptr()) : nullptr; + params.scale_msa = v.has_modulation ? reinterpret_cast<__nv_bfloat16 const*>(scale_msa->data_ptr()) : nullptr; + params.shift_msa = v.has_modulation ? reinterpret_cast<__nv_bfloat16 const*>(shift_msa->data_ptr()) : nullptr; + params.out_bf16 = nullptr; + params.out_fp4 = reinterpret_cast<uint32_t*>(y_fp4.data_ptr()); + params.out_sf = reinterpret_cast<uint32_t*>(sf_out.data_ptr()); + params.sf_scale = reinterpret_cast<float const*>(sf_scale.data_ptr()); + params.M = static_cast<int>(M); + params.D = static_cast<int>(D); + params.seq_len_per_batch = static_cast<int>(seq_len_per_batch); + params.eps = static_cast<float>(eps); + + tensorrt_llm::kernels::launchFusedAdaptiveLayerNormKernel( + params, v.has_ln_affine, v.has_modulation, /*has_quant=*/true, stream); + + return std::make_tuple(y_fp4, sf_out); +} + +} // namespace torch_ext + +TRTLLM_NAMESPACE_END + +// Register the ops with PyTorch. +TORCH_LIBRARY_FRAGMENT(trtllm, m) +{ + m.def( + "fused_adaptive_layernorm(Tensor x, Tensor? ln_weight, Tensor? ln_bias, " + "Tensor? scale_msa, Tensor? shift_msa, int seq_len_per_batch, float eps) -> Tensor"); + m.def( + "fused_adaptive_layernorm_quant(Tensor x, Tensor? ln_weight, Tensor? ln_bias, " + "Tensor? scale_msa, Tensor? shift_msa, Tensor sf_scale, int seq_len_per_batch, float eps) -> (Tensor, Tensor)"); +} + +TORCH_LIBRARY_IMPL(trtllm, CUDA, m) +{ + m.impl("fused_adaptive_layernorm", &tensorrt_llm::torch_ext::fused_adaptive_layernorm); + m.impl("fused_adaptive_layernorm_quant", &tensorrt_llm::torch_ext::fused_adaptive_layernorm_quant); +} diff --git a/cpp/tensorrt_llm/thop/groupRmsNormOp.cpp b/cpp/tensorrt_llm/thop/groupRmsNormOp.cpp index c408a8c286fb..c1f3f41c5a1c 100644 --- a/cpp/tensorrt_llm/thop/groupRmsNormOp.cpp +++ b/cpp/tensorrt_llm/thop/groupRmsNormOp.cpp @@ -17,6 +17,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/dataType.h" #include "tensorrt_llm/common/opUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h" #include "tensorrt_llm/runtime/torchUtils.h" #include "tensorrt_llm/thop/thUtils.h" @@ -100,9 +101,9 @@ void groupRMSNormBase(torch::TensorList const& inputs, torch::TensorList const& /* Handle dtype conversion */ \ switch (dtype) \ { \ - case torch::ScalarType::Half: params.dtype = nvinfer1::DataType::kHALF; break; \ - case torch::ScalarType::BFloat16: params.dtype = nvinfer1::DataType::kBF16; break; \ - case torch::ScalarType::Float: params.dtype = nvinfer1::DataType::kFLOAT; break; \ + case torch::ScalarType::Half: params.dtype = tensorrt_llm::DataType::kHALF; break; \ + case torch::ScalarType::BFloat16: params.dtype = tensorrt_llm::DataType::kBF16; break; \ + case torch::ScalarType::Float: params.dtype = tensorrt_llm::DataType::kFLOAT; break; \ default: TORCH_CHECK(false, "Unsupported data type"); \ } \ tensorrt_llm::kernels::group_rms_norm::GroupRMSNormBaseKernelLauncher<n>(params); \ @@ -181,9 +182,9 @@ void groupRMSNormLargeBatch(torch::TensorList const& inputs, torch::TensorList c // Handle dtype conversion switch (dtype) { - case torch::ScalarType::Half: params.dtype = nvinfer1::DataType::kHALF; break; - case torch::ScalarType::BFloat16: params.dtype = nvinfer1::DataType::kBF16; break; - case torch::ScalarType::Float: params.dtype = nvinfer1::DataType::kFLOAT; break; + case torch::ScalarType::Half: params.dtype = tensorrt_llm::DataType::kHALF; break; + case torch::ScalarType::BFloat16: params.dtype = tensorrt_llm::DataType::kBF16; break; + case torch::ScalarType::Float: params.dtype = tensorrt_llm::DataType::kFLOAT; break; default: TORCH_CHECK(false, "Unsupported data type"); } @@ -260,9 +261,9 @@ void groupRMSNormHeuristic(torch::TensorList const& inputs, torch::TensorList co /* Handle dtype conversion */ \ switch (dtype) \ { \ - case torch::ScalarType::Half: params.dtype = nvinfer1::DataType::kHALF; break; \ - case torch::ScalarType::BFloat16: params.dtype = nvinfer1::DataType::kBF16; break; \ - case torch::ScalarType::Float: params.dtype = nvinfer1::DataType::kFLOAT; break; \ + case torch::ScalarType::Half: params.dtype = tensorrt_llm::DataType::kHALF; break; \ + case torch::ScalarType::BFloat16: params.dtype = tensorrt_llm::DataType::kBF16; break; \ + case torch::ScalarType::Float: params.dtype = tensorrt_llm::DataType::kFLOAT; break; \ default: TORCH_CHECK(false, "Unsupported data type"); \ } \ \ diff --git a/cpp/tensorrt_llm/thop/loraOp.cpp b/cpp/tensorrt_llm/thop/loraOp.cpp index b35ca2608625..6957987be6b8 100644 --- a/cpp/tensorrt_llm/thop/loraOp.cpp +++ b/cpp/tensorrt_llm/thop/loraOp.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/cublasMMWrapper.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/opUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/cuda_graph_grouped_gemm.h" #include "tensorrt_llm/kernels/lora/lora.h" #include "tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.h" @@ -151,11 +152,11 @@ std::vector<th::Tensor> lora_grouped_gemm(th::Tensor const& input, th::Tensor co { outHiddenSizes[i] = output_hidden_sizes[i]; } - nvinfer1::DataType loraRuntimeDataType; + tensorrt_llm::DataType loraRuntimeDataType; switch (input.scalar_type()) { - case torch::kFloat16: loraRuntimeDataType = nvinfer1::DataType::kHALF; break; - case torch::kBFloat16: loraRuntimeDataType = nvinfer1::DataType::kBF16; break; + case torch::kFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kHALF; break; + case torch::kBFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kBF16; break; default: throw std::invalid_argument("Invalid dtype, only supports float16, bfloat16"); } @@ -221,11 +222,11 @@ void lora_grouped_gemm_cuda_graph(th::Tensor const& lora_in_sizes, // [layer_mod auto* splitk_offsets_gpu = reinterpret_cast<int64_t*>(const_cast<void*>(splitk_offsets.data_ptr())); // Get data type - nvinfer1::DataType loraRuntimeDataType; + tensorrt_llm::DataType loraRuntimeDataType; switch (dtype) { - case torch::kFloat16: loraRuntimeDataType = nvinfer1::DataType::kHALF; break; - case torch::kBFloat16: loraRuntimeDataType = nvinfer1::DataType::kBF16; break; + case torch::kFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kHALF; break; + case torch::kBFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kBF16; break; default: TORCH_CHECK(false, "Invalid dtype, only supports float16, bfloat16, got %s", c10::toString(dtype)); } @@ -301,11 +302,11 @@ void lora_group_gemm_param_fill_row_reorder_fusion(th::Tensor const& in_sizes, / int32_t const module_count = static_cast<int32_t>(in_sizes.size(0)); // Get data type info - nvinfer1::DataType loraRuntimeDataType; + tensorrt_llm::DataType loraRuntimeDataType; switch (dtype) { - case torch::kFloat16: loraRuntimeDataType = nvinfer1::DataType::kHALF; break; - case torch::kBFloat16: loraRuntimeDataType = nvinfer1::DataType::kBF16; break; + case torch::kFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kHALF; break; + case torch::kBFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kBF16; break; default: TORCH_CHECK(false, "Invalid dtype, only supports float16, bfloat16, got %s", c10::toString(dtype)); } diff --git a/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp b/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp index 373f936c4c5d..e985eb943ee1 100644 --- a/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp +++ b/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp @@ -15,6 +15,7 @@ */ #include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/thop/moeAlltoAllMeta.h" @@ -484,20 +485,20 @@ torch::Tensor moeA2ACombineOp(torch::Tensor const& payload, int64_t localNumToke TORCH_CHECK(epRank >= 0 && epRank < epSize, "epRank must be in the range [0, epSize)"); TORCH_CHECK(topK > 0 && topK <= kMaxTopK, "topK must be in the range (0, kMaxTopK]"); - // Map torch dtype to nvinfer1::DataType - nvinfer1::DataType nvDtype = nvinfer1::DataType::kFLOAT; + // Map torch dtype to tensorrt_llm::DataType + tensorrt_llm::DataType nvDtype = tensorrt_llm::DataType::kFLOAT; auto scalarType = payload.scalar_type(); if (scalarType == at::kHalf) { - nvDtype = nvinfer1::DataType::kHALF; + nvDtype = tensorrt_llm::DataType::kHALF; } else if (scalarType == at::kBFloat16) { - nvDtype = nvinfer1::DataType::kBF16; + nvDtype = tensorrt_llm::DataType::kBF16; } else if (scalarType == at::kFloat) { - nvDtype = nvinfer1::DataType::kFLOAT; + nvDtype = tensorrt_llm::DataType::kFLOAT; } else { diff --git a/cpp/tensorrt_llm/thop/moeOp.cpp b/cpp/tensorrt_llm/thop/moeOp.cpp index 4a938455488b..52ef700495bf 100644 --- a/cpp/tensorrt_llm/thop/moeOp.cpp +++ b/cpp/tensorrt_llm/thop/moeOp.cpp @@ -30,6 +30,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/dataType.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/common/workspace.h" #include "tensorrt_llm/kernels/cuda_graph_grouped_gemm.h" #include "tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_gemm.h" @@ -87,7 +88,8 @@ enum class MoeLoraRequestType : int32_t // --------------------------------------------------------------------------- inline void moeLoraGroupedGemmRunImpl(::tensorrt_llm::kernels::cutlass_kernels::MoeLoraGroupedGemmModule const& mod, int64_t num_permuted_tokens, int64_t in_hidden_size, int64_t max_lora_rank, int64_t dtype_bytes, - int64_t splitk_slices, void const* input_base, void* output_base, nvinfer1::DataType data_type, cudaStream_t stream) + int64_t splitk_slices, void const* input_base, void* output_base, tensorrt_llm::DataType data_type, + cudaStream_t stream) { TLLM_CHECK_WITH_INFO(mod.permuted_ranks_dev != nullptr, "Grouped-GEMM LoRA module is missing permuted ranks buffer (forgot to populate grouped_gemm?)."); @@ -551,7 +553,17 @@ class FusedMoeRunner : public torch::CustomClassHolder reinterpret_cast<float const*>(swiglu_beta.has_value() ? swiglu_beta.value().const_data_ptr() : nullptr), reinterpret_cast<float const*>(swiglu_limit.has_value() ? swiglu_limit.value().const_data_ptr() : nullptr)); - setRunnerProfiles(profile_ids); + // ===== Routed-expert LoRA activation flags ===== + // LoRA is activated by the per-request (fc1_lora_ranks) or slot-indexed + // (fc1_slot_lora_ranks) schema. Computed before tactic selection so a + // GEMM2 fused-finalize tactic can be excluded when LoRA is active (the + // routed-expert LoRA delta occupies the GEMM2 epilogue that the FINALIZE + // fusion would use). + bool const lora_per_request = fc1_lora_ranks.has_value(); + bool const lora_slot_indexed = fc1_slot_lora_ranks.has_value(); + bool const lora_active = lora_per_request || lora_slot_indexed; + + setRunnerProfiles(profile_ids, lora_active); auto stream = at::cuda::getCurrentCUDAStream(input.get_device()); @@ -571,10 +583,8 @@ class FusedMoeRunner : public torch::CustomClassHolder } // ===== Routed-expert LoRA setup ===== - // LoRA is activated by the per-request schema (fc1_lora_ranks). - bool const lora_per_request = fc1_lora_ranks.has_value(); - bool const lora_slot_indexed = fc1_slot_lora_ranks.has_value(); - bool const lora_active = lora_per_request || lora_slot_indexed; + // Activation flags (lora_per_request / lora_slot_indexed / lora_active) + // were computed above, before tactic selection. bool const is_gated_act = isGatedActivation(base_activation_type); if (lora_active) { @@ -1138,7 +1148,7 @@ class FusedMoeRunner : public torch::CustomClassHolder } } - void setRunnerProfiles(torch::optional<c10::ArrayRef<int64_t>> profile_ids) + void setRunnerProfiles(torch::optional<c10::ArrayRef<int64_t>> profile_ids, bool lora_active = false) { if (mUseDeepSeekFP8BlockScaling) { @@ -1161,6 +1171,26 @@ class FusedMoeRunner : public torch::CustomClassHolder best_gemm2_profile = profile_ids.value()[1] == -1 ? best_gemm2_profile : mGemm2Profiles.at(profile_ids.value()[1]); } + + // Routed-expert MoE LoRA is incompatible with the GEMM2 fused-finalize + // epilogue: the LoRA delta is applied in the GEMM2 epilogue that the + // FINALIZE fusion would otherwise occupy (see setupTmaWarpSpecializedInputs + // in moe_kernels.cu). The GEMM2 tactic autotuner profiles with LoRA off + // (runGemmProfile forces USE_LORA=false) and can therefore select a + // FINALIZE tactic, and a cached runner may still expose FINALIZE tactics + // if it was first constructed with fused finalize enabled. Downgrade the + // selected tactic to the equivalent non-fused (NONE) epilogue when LoRA + // is active. Every FINALIZE config is a copy of a valid non-FINALIZE + // config with the fusion flag flipped (see MoeGemmRunner::getConfigs), so + // clearing the flag yields a supported tactic with the same tile shape. + if (lora_active + && best_gemm2_profile.epilogue_fusion_type + == tensorrt_llm::cutlass_extensions::CutlassGemmConfig::EpilogueFusionType::FINALIZE) + { + best_gemm2_profile.epilogue_fusion_type + = tensorrt_llm::cutlass_extensions::CutlassGemmConfig::EpilogueFusionType::NONE; + } + mKernelRunner->setTactic(best_gemm1_profile, best_gemm2_profile); } @@ -1206,17 +1236,17 @@ class FusedMoeRunner : public torch::CustomClassHolder // ===== LoRA helpers ===== - // Map a torch dtype to the TRT-LLM nvinfer1::DataType used to size the + // Map a torch dtype to the TRT-LLM tensorrt_llm::DataType used to size the // grouped-GEMM low-rank scratch. Kept as a const member (not static) so the // FP8 case can read mOutputDtype to pick the fp16/bf16 LoRA compute dtype. - nvinfer1::DataType loraTypeFromActDtype(c10::ScalarType dtype) const + tensorrt_llm::DataType loraTypeFromActDtype(c10::ScalarType dtype) const { switch (dtype) { - case c10::ScalarType::Half: return nvinfer1::DataType::kHALF; - case c10::ScalarType::Float: return nvinfer1::DataType::kFLOAT; + case c10::ScalarType::Half: return tensorrt_llm::DataType::kHALF; + case c10::ScalarType::Float: return tensorrt_llm::DataType::kFLOAT; #ifdef ENABLE_BF16 - case c10::ScalarType::BFloat16: return nvinfer1::DataType::kBF16; + case c10::ScalarType::BFloat16: return tensorrt_llm::DataType::kBF16; #endif #ifdef ENABLE_FP8 case c10::ScalarType::Float8_e4m3fn: diff --git a/cpp/tensorrt_llm/thop/noAuxTcOp.cpp b/cpp/tensorrt_llm/thop/noAuxTcOp.cpp index e445206e1d78..4dfb20072734 100644 --- a/cpp/tensorrt_llm/thop/noAuxTcOp.cpp +++ b/cpp/tensorrt_llm/thop/noAuxTcOp.cpp @@ -20,7 +20,6 @@ #include "tensorrt_llm/kernels/noAuxTcKernels.h" -// #include <NvInferRuntime.h> // #include <c10/cuda/CUDAStream.h> // #include <cassert> // #include <set> diff --git a/cpp/tensorrt_llm/thop/reducescatterOp.cpp b/cpp/tensorrt_llm/thop/reducescatterOp.cpp index 40f89e40ff75..a50ca1862f76 100644 --- a/cpp/tensorrt_llm/thop/reducescatterOp.cpp +++ b/cpp/tensorrt_llm/thop/reducescatterOp.cpp @@ -20,7 +20,6 @@ #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/pgUtils.h" -#include <NvInferRuntime.h> #include <c10/cuda/CUDAStream.h> #include <torch/extension.h> #if ENABLE_MULTI_DEVICE diff --git a/cpp/tensorrt_llm/thop/sparseKvCacheCompactOp.cpp b/cpp/tensorrt_llm/thop/sparseKvCacheCompactOp.cpp new file mode 100644 index 000000000000..b0446b0ef574 --- /dev/null +++ b/cpp/tensorrt_llm/thop/sparseKvCacheCompactOp.cpp @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. + * All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/common/opUtils.h" +#include "tensorrt_llm/kernels/unfusedAttentionKernels.h" +#include "tensorrt_llm/runtime/torchUtils.h" + +#include <limits> +#include <optional> +#include <vector> + +namespace th = torch; +namespace tk = tensorrt_llm::kernels; + +TRTLLM_NAMESPACE_BEGIN + +namespace torch_ext +{ + +//! Compact one uniform group of KVCacheManagerV2 HND layer pools in one batched +//! launch (per request and KV head, moves are ascending and never overtake their +//! sources: the copy runs in place). +void sparseKvCacheCompactLayers(std::vector<th::Tensor> const& pools, th::Tensor const& poolPointers, + th::Tensor const& pageTable, th::Tensor const& sourceIndices, th::Tensor const& sourceOffsets, + th::Tensor const& destinationBases, std::optional<th::Tensor> const& sourceLayerIndices) +{ + TORCH_CHECK(!pools.empty(), "sparse_kv_cache_compact_layers: pools must be non-empty"); + + auto const& firstPool = pools.front(); + TORCH_CHECK(firstPool.is_cuda() && firstPool.dim() == 5 && firstPool.size(1) == 2 && firstPool.is_contiguous(), + "sparse_kv_cache_compact_layers: pools must be contiguous CUDA " + "[pages, 2, kv_heads, tokens_per_block, head_dim] tensors"); + TORCH_CHECK(pageTable.is_cuda() && pageTable.dim() == 2 && pageTable.scalar_type() == th::kInt32 + && pageTable.stride(1) == 1, + "sparse_kv_cache_compact_layers: K block offsets must be CUDA int32 " + "[batch, max_pages] tensors with a contiguous block dimension"); + + auto const device = firstPool.get_device(); + auto const dtype = firstPool.scalar_type(); + auto const numLayers = static_cast<int32_t>(pools.size()); + auto const numKvHeads = static_cast<int32_t>(firstPool.size(2)); + auto const tokensPerBlock = static_cast<int32_t>(firstPool.size(3)); + auto const headDim = static_cast<int32_t>(firstPool.size(4)); + auto const batchSize = static_cast<int32_t>(pageTable.size(0)); + auto const pageTableRequestStride = pageTable.stride(0); + + // Layer 0 defined the reference geometry in the firstPool checks above. + for (int32_t layer = 1; layer < numLayers; ++layer) + { + auto const& pool = pools[layer]; + TORCH_CHECK(pool.is_cuda() && pool.get_device() == device && pool.scalar_type() == dtype && pool.dim() == 5 + && pool.size(1) == 2 && pool.is_contiguous(), + "sparse_kv_cache_compact_layers: all pools must have one device, dtype, layout, and contiguous storage"); + TORCH_CHECK(pool.size(2) == numKvHeads && pool.size(3) == tokensPerBlock && pool.size(4) == headDim, + "sparse_kv_cache_compact_layers: all pools must share KV-head, block, and head-dimension geometry"); + } + TORCH_CHECK( + pageTable.get_device() == device, "sparse_kv_cache_compact_layers: block offsets must be on the pool device"); + + TORCH_CHECK(poolPointers.is_cuda() && poolPointers.get_device() == device + && poolPointers.scalar_type() == th::kInt64 && poolPointers.dim() == 1 && poolPointers.size(0) == numLayers + && poolPointers.is_contiguous(), + "sparse_kv_cache_compact_layers: pool_pointers must be contiguous CUDA int64 [num_layers]"); + + TORCH_CHECK(sourceIndices.is_cuda() && sourceIndices.get_device() == device + && sourceIndices.scalar_type() == th::kInt32 && sourceIndices.is_contiguous() + && (sourceIndices.dim() == 2 || sourceIndices.dim() == 3), + "sparse_kv_cache_compact_layers: source_indices must be contiguous CUDA int32 " + "[kv_heads, total] or [source_layers, kv_heads, total]"); + int64_t sourceLayerStride = 0; + int32_t const* sourceLayerPtr = nullptr; + if (sourceIndices.dim() == 2) + { + TORCH_CHECK(sourceIndices.size(0) == numKvHeads, + "sparse_kv_cache_compact_layers: source_indices KV-head dimension mismatch"); + TORCH_CHECK(!sourceLayerIndices.has_value(), + "sparse_kv_cache_compact_layers: source_layer_indices require 3-D per-layer source_indices"); + } + else + { + TORCH_CHECK(sourceIndices.size(0) > 0 && sourceIndices.size(1) == numKvHeads, + "sparse_kv_cache_compact_layers: per-layer source_indices geometry mismatch"); + TORCH_CHECK(sourceLayerIndices.has_value(), + "sparse_kv_cache_compact_layers: per-layer source_indices require source_layer_indices"); + sourceLayerStride = sourceIndices.stride(0); + } + if (sourceLayerIndices.has_value()) + { + auto const& layerIndices = *sourceLayerIndices; + TORCH_CHECK(layerIndices.is_cuda() && layerIndices.get_device() == device + && layerIndices.scalar_type() == th::kInt32 && layerIndices.is_contiguous() && layerIndices.dim() == 1 + && layerIndices.size(0) == numLayers, + "sparse_kv_cache_compact_layers: source_layer_indices must be contiguous CUDA int32 [num_layers]"); + sourceLayerPtr = layerIndices.data_ptr<int32_t>(); + } + + // source_offsets carve each request's move range; device-resident, the kernel trusts them. + TORCH_CHECK(sourceOffsets.is_cuda() && sourceOffsets.get_device() == device + && sourceOffsets.scalar_type() == th::kInt32 && sourceOffsets.is_contiguous() && sourceOffsets.dim() == 1 + && sourceOffsets.size(0) == batchSize + 1, + "sparse_kv_cache_compact_layers: source_offsets must be contiguous CUDA int32 [batch + 1]"); + // Per-request landing positions: one launch covers a cohort with mixed + // pinned-prompt lengths. Values live on device; the kernel trusts them. + TORCH_CHECK(destinationBases.is_cuda() && destinationBases.get_device() == device + && destinationBases.scalar_type() == th::kInt32 && destinationBases.dim() == 1 + && destinationBases.size(0) == batchSize && destinationBases.is_contiguous(), + "sparse_kv_cache_compact_layers: destination_bases must be contiguous CUDA int32 [batch]"); + + auto const stream = at::cuda::getCurrentCUDAStream(device); + auto const* bases = destinationBases.data_ptr<int32_t>(); + auto const sourceHeadStride = sourceIndices.size(-1); + if (dtype == th::kBFloat16) + { + tk::invokeSparseKvCacheCompactLayers<__nv_bfloat16>(poolPointers.data_ptr<int64_t>(), + pageTable.data_ptr<int32_t>(), numLayers, pageTableRequestStride, sourceIndices.data_ptr<int32_t>(), + sourceLayerPtr, sourceLayerStride, sourceHeadStride, sourceOffsets.data_ptr<int32_t>(), bases, batchSize, + numKvHeads, tokensPerBlock, headDim, stream); + } + else + { + TORCH_CHECK(false, + "sparse_kv_cache_compact_layers ships only the pipelined bf16 kernels (head size 64/128, page size " + "32/128 tokens); got pool dtype ", + dtype); + } +} + +} // namespace torch_ext + +TRTLLM_NAMESPACE_END + +TORCH_LIBRARY_FRAGMENT(trtllm, m) +{ + m.def( + "sparse_kv_cache_compact_layers(Tensor(a!)[] pools, Tensor pool_pointers, Tensor page_table, Tensor " + "source_indices, Tensor source_offsets, Tensor destination_bases, " + "Tensor? source_layer_indices=None) -> ()"); +} + +TORCH_LIBRARY_IMPL(trtllm, CUDA, m) +{ + m.impl("sparse_kv_cache_compact_layers", &tensorrt_llm::torch_ext::sparseKvCacheCompactLayers); +} diff --git a/cpp/tensorrt_llm/thop/thUtils.cpp b/cpp/tensorrt_llm/thop/thUtils.cpp index 97fe6acaab7b..c151414127fa 100644 --- a/cpp/tensorrt_llm/thop/thUtils.cpp +++ b/cpp/tensorrt_llm/thop/thUtils.cpp @@ -15,7 +15,7 @@ */ #include "tensorrt_llm/thop/thUtils.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <array> TRTLLM_NAMESPACE_BEGIN @@ -25,12 +25,12 @@ namespace torch_ext tensorrt_llm::runtime::ITensor::Shape convert_shape(torch::Tensor tensor) { - constexpr auto trtMaxDims = nvinfer1::Dims::MAX_DIMS; + constexpr auto trtMaxDims = tensorrt_llm::Dims::MAX_DIMS; auto const torchTensorNumDims = tensor.dim(); TLLM_CHECK_WITH_INFO(torchTensorNumDims <= trtMaxDims, "TensorRT supports at most %i tensor dimensions. Found a Torch tensor with %li dimensions.", trtMaxDims, torchTensorNumDims); - auto result = nvinfer1::Dims{}; + auto result = tensorrt_llm::Dims{}; result.nbDims = static_cast<int32_t>(torchTensorNumDims); for (int i = 0; i < torchTensorNumDims; i++) { diff --git a/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp b/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp index 5720951e2720..bc80d8446445 100644 --- a/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp +++ b/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/common/attentionOp.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/gptKernels.h" #include "tensorrt_llm/kernels/kvCacheUtils.h" #include "tensorrt_llm/kernels/unfusedAttentionKernels.h" @@ -401,16 +402,16 @@ trtllmGenContextPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, tor switch (qkvDtype) { - case nvinfer1::DataType::kFLOAT: + case tensorrt_llm::DataType::kFLOAT: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<float, KVBlockArray>&>(qkvParams), stream); break; - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<half, KVBlockArray>&>(qkvParams), stream); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<__nv_bfloat16, KVBlockArray>&>(qkvParams), stream); break; @@ -549,16 +550,16 @@ void trtllmGenContextPostprocess(torch::Tensor qkv_input, torch::Tensor workspac switch (qkvDtype) { - case nvinfer1::DataType::kFLOAT: + case tensorrt_llm::DataType::kFLOAT: tensorrt_llm::kernels::invokeKvCachePostprocessing( reinterpret_cast<QKVPreprocessingParams<float, KVBlockArray>&>(qkvParams), stream); break; - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: tensorrt_llm::kernels::invokeKvCachePostprocessing( reinterpret_cast<QKVPreprocessingParams<half, KVBlockArray>&>(qkvParams), stream); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: tensorrt_llm::kernels::invokeKvCachePostprocessing( reinterpret_cast<QKVPreprocessingParams<__nv_bfloat16, KVBlockArray>&>(qkvParams), stream); break; @@ -733,16 +734,16 @@ trtllmGenGenerationPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, switch (qkvDtype) { - case nvinfer1::DataType::kFLOAT: + case tensorrt_llm::DataType::kFLOAT: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<float, KVBlockArray>&>(qkvParams), stream); break; - case nvinfer1::DataType::kHALF: + case tensorrt_llm::DataType::kHALF: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<half, KVBlockArray>&>(qkvParams), stream); break; #ifdef ENABLE_BF16 - case nvinfer1::DataType::kBF16: + case tensorrt_llm::DataType::kBF16: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<__nv_bfloat16, KVBlockArray>&>(qkvParams), stream); break; diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 0a06d40ee85e..2c65dee55abe 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & # AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -25,8 +25,13 @@ include_directories( ${PROJECT_SOURCE_DIR}/include ${cutlass_source_dir}/include ${cutlass_source_dir}/tools/util/include - ${PROJECT_SOURCE_DIR}/tests/batch_manager - ${PROJECT_SOURCE_DIR}/tests/utils) + ${PROJECT_SOURCE_DIR}/tests/batch_manager) + +# Tests previously inherited the MPI include dirs transitively through the +# removed TensorRT plugin target's PUBLIC includes. +if(ENABLE_MULTI_DEVICE) + include_directories(${MPI_C_INCLUDE_DIRS}) +endif() set(TOP_LEVEL_DIR "${PROJECT_SOURCE_DIR}/..") @@ -34,17 +39,17 @@ add_custom_target(google-tests) function(add_gtest test_name test_src) set(options NO_GTEST_MAIN NO_TLLM_LINKAGE) + set(oneValueArgs TIMEOUT) cmake_parse_arguments(ARGS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_executable(${test_name} ${test_src}) - target_link_libraries(${test_name} PUBLIC gmock_main TensorRT::OnnxParser) + target_link_libraries(${test_name} PUBLIC gmock_main) if(NOT ARGS_NO_GTEST_MAIN) target_link_libraries(${test_name} PUBLIC gtest_main) endif() if(NOT ARGS_NO_TLLM_LINKAGE) - target_link_libraries(${test_name} PUBLIC ${SHARED_TARGET} - nvinfer_plugin_tensorrt_llm) + target_link_libraries(${test_name} PUBLIC ${SHARED_TARGET}) if(WIN32) target_link_libraries(${test_name} PRIVATE context_attention_src) endif() @@ -57,15 +62,20 @@ function(add_gtest test_name test_src) target_compile_definitions(${test_name} PUBLIC TOP_LEVEL_DIR="${TOP_LEVEL_DIR}") + set(test_properties ENVIRONMENT "CUDA_MODULE_LOADING=LAZY") + if(ARGS_TIMEOUT) + list(APPEND test_properties TIMEOUT "${ARGS_TIMEOUT}") + endif() + gtest_discover_tests( ${test_name} - PROPERTIES ENVIRONMENT "CUDA_MODULE_LOADING=LAZY" DISCOVERY_MODE + PROPERTIES ${test_properties} + DISCOVERY_MODE PRE_TEST # WAR for DLL discovery on windows. - DISCOVERY_TIMEOUT 30) # Longer timeout needed because discovery - # can be slow on Windows + DISCOVERY_TIMEOUT + 30) # Longer timeout needed because discovery can be slow on + # Windows add_dependencies(google-tests ${test_name}) endfunction() -add_subdirectory(utils) add_subdirectory(unit_tests) -add_subdirectory(e2e_tests) diff --git a/cpp/tests/e2e_tests/CMakeLists.txt b/cpp/tests/e2e_tests/CMakeLists.txt deleted file mode 100644 index f5deb048a180..000000000000 --- a/cpp/tests/e2e_tests/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -add_subdirectory(batch_manager) -add_subdirectory(executor) diff --git a/cpp/tests/e2e_tests/batch_manager/CMakeLists.txt b/cpp/tests/e2e_tests/batch_manager/CMakeLists.txt deleted file mode 100644 index 875e12eb975f..000000000000 --- a/cpp/tests/e2e_tests/batch_manager/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -# guidedDecoderTest requires model tokenizer info, so it's easier to run it with -# e2e tests instead of unit tests. -add_gtest(guidedDecoderTest guidedDecoderTest.cpp) -add_gtest(trtEncoderModelTest trtEncoderModelTest.cpp) -add_gtest(trtGptModelTest trtGptModelTest.cpp) -add_gtest(trtGptModelRealDecoderTest trtGptModelRealDecoderTest.cpp) -target_link_libraries(trtGptModelRealDecoderTest PRIVATE testingUtils) diff --git a/cpp/tests/e2e_tests/batch_manager/guidedDecoderTest.cpp b/cpp/tests/e2e_tests/batch_manager/guidedDecoderTest.cpp deleted file mode 100644 index 7b262cacb27d..000000000000 --- a/cpp/tests/e2e_tests/batch_manager/guidedDecoderTest.cpp +++ /dev/null @@ -1,227 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TOP_LEVEL_DIR -#error "Define TOP_LEVEL_DIR" -#endif - -#include <fstream> -#include <gtest/gtest.h> -#include <nlohmann/json.hpp> - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/batch_manager/decoderBuffers.h" -#include "tensorrt_llm/batch_manager/guidedDecoder.h" -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/executor/executor.h" - -using namespace tensorrt_llm::runtime; -using namespace tensorrt_llm::batch_manager; -namespace texec = tensorrt_llm::executor; - -namespace -{ -auto const TEST_RESOURCE_PATH = std::filesystem::path{TOP_LEVEL_DIR} / "cpp/tests/resources"; -auto const DATA_PATH = TEST_RESOURCE_PATH / "data"; -auto const GPT_XGRAMMAR_TOKENIZER_INFO_PATH = DATA_PATH / "gpt2" / "xgrammar_tokenizer_info.json"; -auto const LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH = DATA_PATH / "Llama-3.2-1B" / "xgrammar_tokenizer_info.json"; -} // namespace - -class GuidedDecoderTest : public ::testing::Test -{ -public: - using TensorPtr = ITensor::SharedPtr; - using VecTokens = std::vector<TokenIdType>; - using RequestIdType = std::uint64_t; - using RequestVector = std::vector<std::shared_ptr<LlmRequest>>; - - void SetUp() override - { - mStream = std::make_shared<CudaStream>(); - mRuntimeBufferManager = std::make_shared<BufferManager>(mStream); - } - - void TearDown() override {} - - void initData(std::filesystem::path tokenizerInfoPath, SizeType32 vocabSizePadded, VecTokens outputIds, - std::vector<int32_t> expectedNumRejected) - { - mLogitsDtype = nvinfer1::DataType::kFLOAT; - mMaxNumRequests = 16; - - mVocabSizePadded = vocabSizePadded; - auto const tokenizerInfo = nlohmann::json::parse(std::ifstream{tokenizerInfoPath}); - auto const encodedVocab = tokenizerInfo["encoded_vocab"].template get<std::vector<std::string>>(); - auto const tokenizerStr = tokenizerInfo["tokenizer_str"].template get<std::string>(); - auto const stopTokenIds = tokenizerInfo["stop_token_ids"].template get<std::vector<TokenIdType>>(); - texec::GuidedDecodingConfig guidedDecodingConfig( - texec::GuidedDecodingConfig::GuidedDecodingBackend::kXGRAMMAR, encodedVocab, tokenizerStr, stopTokenIds); - mGuidedDecoder = std::make_shared<GuidedDecoder>( - guidedDecodingConfig, mMaxNumRequests, mVocabSizePadded, mLogitsDtype, *mRuntimeBufferManager); - - mLogits.resize(mMaxNumRequests); - mLogitsHost.resize(mMaxNumRequests); - for (int i = 0; i < mMaxNumRequests; i++) - { - mLogits[i] = mRuntimeBufferManager->gpu(ITensor::makeShape({mVocabSizePadded}), mLogitsDtype); - mLogitsHost[i] = BufferManager::pinned(ITensor::makeShape({mVocabSizePadded}), mLogitsDtype); - } - - mOutputIds = outputIds; - mExpectedNumRejected = expectedNumRejected; - } - - void resetLogits() - { - for (int i = 0; i < mMaxNumRequests; i++) - { - auto logitsHostData = bufferCast<float>(*mLogitsHost[i]); - for (int j = 0; j < mVocabSizePadded; j++) - { - logitsHostData[j] = 0.0f; - } - mRuntimeBufferManager->copy(*(mLogitsHost[i]), *(mLogits[i])); - } - } - - void syncLogitsToHost() - { - for (int i = 0; i < mMaxNumRequests; i++) - { - mRuntimeBufferManager->copy(*(mLogits[i]), *(mLogitsHost[i])); - } - } - - int32_t countRejected(int i) - { - int32_t numRejected = 0; - for (int j = 0; j < mVocabSizePadded; j++) - { - auto logitsHostData = bufferCast<float>(*mLogitsHost[i]); - if (logitsHostData[j] < -1e6) - { - numRejected++; - } - } - return numRejected; - } - - void runTest() - { - auto llmReq1 = std::make_shared<LlmRequest>(1, 100, std::make_shared<VecTokens>(10), SamplingConfig(), false); - texec::GuidedDecodingParams guidedDecodingParams(texec::GuidedDecodingParams::GuideType::kJSON); - llmReq1->setGuidedDecodingParams(guidedDecodingParams); - llmReq1->mSeqSlot = 1; - - auto llmReq2 = std::make_shared<LlmRequest>(1, 100, std::make_shared<VecTokens>(10), SamplingConfig(), false); - llmReq2->mSeqSlot = 2; - - RequestVector contextRequests{llmReq1, llmReq2}; - RequestVector generationRequests{}; - ScheduledRequests scheduledRequests{contextRequests, generationRequests}; - DecoderInputBuffers decoderInputBuffers(mMaxNumRequests, 1, *mRuntimeBufferManager); - - for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) - { - for (auto const& llmReq : requests) - { - decoderInputBuffers.decoderRequests.push_back(llmReq); - } - } - decoderInputBuffers.decoderLogits = mLogits; - - // Context phase - resetLogits(); - mGuidedDecoder->build(scheduledRequests); - mGuidedDecoder->execute(decoderInputBuffers, *mRuntimeBufferManager); - syncLogitsToHost(); - mRuntimeBufferManager->getStream().synchronize(); - - // Move request to generation phase - contextRequests.pop_back(); - contextRequests.pop_back(); - llmReq1->setState(LlmRequestState::kGENERATION_IN_PROGRESS); - generationRequests.push_back(llmReq1); - llmReq2->setState(LlmRequestState::kGENERATION_IN_PROGRESS); - generationRequests.push_back(llmReq2); - - decoderInputBuffers.decoderRequests.clear(); - for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) - { - for (auto const& llmReq : requests) - { - decoderInputBuffers.decoderRequests.push_back(llmReq); - } - } - - EXPECT_EQ(countRejected(0), mExpectedNumRejected[0]); - EXPECT_EQ(countRejected(1), 0); - - // Generation phase - for (int i = 0; i < mOutputIds.size(); i++) - { - llmReq1->addNewToken(mOutputIds[i], 0); - llmReq2->addNewToken(mOutputIds[i], 0); - - resetLogits(); - mGuidedDecoder->build(scheduledRequests); - mGuidedDecoder->execute(decoderInputBuffers, *mRuntimeBufferManager); - syncLogitsToHost(); - mRuntimeBufferManager->getStream().synchronize(); - - EXPECT_EQ(countRejected(0), mExpectedNumRejected[i + 1]); - EXPECT_EQ(countRejected(1), 0); - } - } - -private: - SizeType32 mMaxNumRequests; - SizeType32 mVocabSizePadded; - nvinfer1::DataType mLogitsDtype; - - std::vector<TensorPtr> mLogits; // [mBatchSize, mVocabSizePadded] - std::vector<TensorPtr> mLogitsHost; // [mBatchSize, mVocabSizePadded] - - std::shared_ptr<BufferManager> mRuntimeBufferManager; - std::shared_ptr<CudaStream> mStream; - std::shared_ptr<GuidedDecoder> mGuidedDecoder; - - VecTokens mOutputIds; - std::vector<int32_t> mExpectedNumRejected; -}; - -TEST_F(GuidedDecoderTest, GptTokenizer) -{ - VecTokens outputIds{4895, 824, 312, 1298, 366, 27743, 7934, 49793, 1600, 366, 12961, 19703, 4668, 1298, 366, 54, - 4537, 17, 12, 17469, 7919, 1600, 366, 3903, 10394, 1298, 366, 1485, 405, 41022, 20662}; - std::vector<int32_t> expectedNumRejected{50251, 219, 219, 219, 48558, 219, 219, 219, 219, 50191, 219, 219, 219, 219, - 48558, 219, 219, 219, 219, 219, 219, 219, 50191, 219, 219, 219, 48558, 219, 219, 219, 219, 50256}; - initData(GPT_XGRAMMAR_TOKENIZER_INFO_PATH, 50257, outputIds, expectedNumRejected); - runTest(); -} - -TEST_F(GuidedDecoderTest, LlamaTokenizer) -{ - VecTokens outputIds{6377, 893, 333, 1115, 376, 27247, 6779, 7898, 545, 613, 376, 8926, 17830, 1115, 376, 29956, - 7228, 29906, 29899, 10399, 7734, 613, 376, 4980, 2103, 1115, 376, 29896, 29941, 29900, 29900, 341, 29890, 567, - 9092}; - std::vector<int32_t> expectedNumRejected{128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, - 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, - 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235}; - initData(LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH, 128256, outputIds, expectedNumRejected); - runTest(); -} diff --git a/cpp/tests/e2e_tests/batch_manager/trtEncoderModelTest.cpp b/cpp/tests/e2e_tests/batch_manager/trtEncoderModelTest.cpp deleted file mode 100644 index dce09539bd45..000000000000 --- a/cpp/tests/e2e_tests/batch_manager/trtEncoderModelTest.cpp +++ /dev/null @@ -1,219 +0,0 @@ - -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TOP_LEVEL_DIR -#error "Define TOP_LEVEL_DIR" -#endif - -#include "tensorrt_llm/batch_manager/trtEncoderModel.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/gptJsonConfig.h" -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/utils/numpyUtils.h" -#include "tensorrt_llm/runtime/utils/runtimeUtils.h" - -#include <gmock/gmock.h> -#include <gtest/gtest.h> - -#include <filesystem> -#include <vector> - -using namespace tensorrt_llm::runtime; -namespace fs = std::filesystem; - -using TensorPtr = ITensor::SharedPtr; - -namespace -{ -auto const TEST_RESOURCE_PATH = fs::path{TOP_LEVEL_DIR} / "cpp/tests/resources"; -auto const ENC_DEC_BASE = TEST_RESOURCE_PATH / "models/enc_dec/trt_engines"; -auto const ENC_DEC_ENGINE_BASE = TEST_RESOURCE_PATH / "models/enc_dec/trt_engines"; -auto const BART_TP1_PP1_ENCODER_RMPAD_DIR = "bart-large-cnn/1-gpu/float16/tp1/encoder"; -auto const BART_TP2_PP1_ENCODER_RMPAD_DIR = "bart-large-cnn/2-gpu/float16/tp2/encoder"; -auto const BART_TP2_PP2_ENCODER_RMPAD_DIR = "bart-large-cnn/4-gpu/float16/tp2/encoder"; -auto const T5_TP1_PP1_ENCODER_RMPAD_DIR = "t5-small/1-gpu/float16/tp1/encoder"; -auto const ENC_DEC_DATA_BASE = TEST_RESOURCE_PATH / "data/enc_dec"; -} // namespace - -namespace tensorrt_llm::batch_manager -{ - -class EncoderModelTestSingleGPU : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) -{ -protected: - EncoderModelTestSingleGPU(std::filesystem::path const& modelPath) - : mModelConfig(1, 2, 1, 1, 1, 1, nvinfer1::DataType::kFLOAT) - , mModelPath(modelPath) - { - } - - EncoderModelTestSingleGPU() - : EncoderModelTestSingleGPU(ENC_DEC_ENGINE_BASE / T5_TP1_PP1_ENCODER_RMPAD_DIR) - { - } - - void SetUp() override - { - std::filesystem::path trtEnginePath = mModelPath; - - mBeamWidth = 1; - - mLogger = std::make_shared<TllmLogger>(); - - initTrtLlmPlugins(mLogger.get()); - - auto const json = GptJsonConfig::parse(trtEnginePath / "config.json"); - mModelConfig = json.getModelConfig(); - mWorldConfig = WorldConfig::mpi(json.getGpusPerNode(), json.getTensorParallelism(), - json.getPipelineParallelism(), json.getContextParallelism()); - mVocabSizePadded = mModelConfig.getVocabSizePadded(mWorldConfig.getSize()); - - auto const enginePath = trtEnginePath / json.engineFilename(mWorldConfig); - auto const dtype = mModelConfig.getDataType(); - - ASSERT_TRUE(fs::exists(enginePath)); - mEngineBuffer = utils::loadEngine(enginePath.string()); - - mStream = std::make_unique<CudaStream>(); - mManager = std::make_unique<BufferManager>(mStream); - } - - void TearDown() override {} - - int32_t mMaxNumRequests; - int32_t mMaxSeqLen; - int32_t mBeamWidth; - int32_t mVocabSizePadded; - // SamplingConfig mSamplingConfig; - std::string mDataPath; - std::shared_ptr<nvinfer1::ILogger> mLogger; - ModelConfig mModelConfig; - WorldConfig mWorldConfig; - std::vector<std::uint8_t> mEngineBuffer; - std::unique_ptr<BufferManager> mManager; - BufferManager::CudaStreamPtr mStream; - std::filesystem::path mModelPath; -}; - -// test for TP2PP2 -class TrtEncoderModelTestMultiGPU : public EncoderModelTestSingleGPU -{ -protected: - TrtEncoderModelTestMultiGPU() - : EncoderModelTestSingleGPU(ENC_DEC_ENGINE_BASE / BART_TP2_PP2_ENCODER_RMPAD_DIR) - { - } -}; - -namespace -{ - -void runEncoderTest(std::unique_ptr<BufferManager>& bufferManager, ModelConfig const& modelConfig, - WorldConfig const& worldConfig, std::vector<std::uint8_t> const& engineBuffer, - std::shared_ptr<nvinfer1::ILogger>& logger) -{ - using VecTokens = LlmRequest::VecTokens; - using TokenIdType = LlmRequest::TokenIdType; - - auto inputsIdsHost - = utils::loadNpy(*bufferManager, (ENC_DEC_DATA_BASE / "input_ids.npy").string(), MemoryType::kCPU); - auto inputsIdsPtr = bufferCast<SizeType32>(*inputsIdsHost); - auto inputLengthsHost - = utils::loadNpy(*bufferManager, (ENC_DEC_DATA_BASE / "input_lengths.npy").string(), MemoryType::kCPU); - auto inputLengthsPtr = bufferCast<SizeType32>(*inputLengthsHost); - auto encoderOutput - = utils::loadNpy(*bufferManager, (ENC_DEC_DATA_BASE / "encoder_output.npy").string(), MemoryType::kCPU); - auto encoderOutputPrt = bufferCast<half>(*encoderOutput); - - SizeType32 const nbRequests = inputLengthsHost->getShape().d[0]; - SizeType32 const stride = inputsIdsHost->getShape().d[1]; - SizeType32 const hiddenSize = encoderOutput->getShape().d[1]; - ASSERT_EQ(nbRequests, inputsIdsHost->getShape().d[0]); - - // std::vector<std::shared_ptr<VecTokens>> inputIds(nbRequests); - RequestVector requestList; - for (SizeType32 i = 0; i < nbRequests; i++) - { - SizeType32 length = inputLengthsPtr[i]; - auto currentInputId = std::make_shared<VecTokens>(0); - currentInputId->insert(currentInputId->end(), inputsIdsPtr, inputsIdsPtr + length); - executor::Request req(*currentInputId, 1); - req.setEncoderInputTokenIds(*currentInputId); - auto request = std::make_shared<LlmRequest>(i, req); - inputsIdsPtr += stride; - requestList.push_back(request); - } - - tensorrt_llm::executor::ExecutorConfig executorConfig{}; - auto trtEncoderModel = std::make_shared<TrtEncoderModel>( - modelConfig, worldConfig, runtime::RawEngine(engineBuffer.data(), engineBuffer.size()), logger, executorConfig); - - trtEncoderModel->forward(requestList); - - if (worldConfig.isLastPipelineParallelRank() && worldConfig.getTensorParallelRank() == 0) - { - auto arrayEqual = [](auto it0, auto it1, SizeType32 length) - { - SizeType32 nbNotEqual = 0; - for (SizeType32 i = 0; i < length; i++) - { - auto v0 = static_cast<float>(*it0); - auto v1 = static_cast<float>(*it1); - if (std::abs(v0 - v1) > 1e-3) - { - nbNotEqual++; - } - it0++; - it1++; - } - return static_cast<double>(nbNotEqual) / length; - }; - ASSERT_EQ(requestList.size(), inputLengthsHost->getShape().d[0]); - { - auto curLengthPtr = inputLengthsPtr; - auto curOutPtr = encoderOutputPrt; - for (auto const& req : requestList) - { - ASSERT_TRUE(req->getEncoderOutputHost()) << "Encoder output is empty!"; - EXPECT_EQ(req->getState(), LlmRequestState::kCONTEXT_INIT); - auto actualOut = bufferCast<half>(*(req->getEncoderOutputHost())); - auto unequalFraction = arrayEqual(curOutPtr, actualOut, *curLengthPtr); - EXPECT_TRUE(unequalFraction == 0) - << "Req " << req->mRequestId << ": " << unequalFraction << " of outputs are different"; - curOutPtr += *curLengthPtr * hiddenSize; - curLengthPtr++; - } - } - } -} - -} // Anonymous namespace - -TEST_F(EncoderModelTestSingleGPU, Forward) -{ - runEncoderTest(mManager, mModelConfig, mWorldConfig, mEngineBuffer, mLogger); -} - -TEST_F(TrtEncoderModelTestMultiGPU, Forward) -{ - - runEncoderTest(mManager, mModelConfig, mWorldConfig, mEngineBuffer, mLogger); -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tests/e2e_tests/batch_manager/trtGptModelRealDecoderTest.cpp b/cpp/tests/e2e_tests/batch_manager/trtGptModelRealDecoderTest.cpp deleted file mode 100644 index f401a31305d3..000000000000 --- a/cpp/tests/e2e_tests/batch_manager/trtGptModelRealDecoderTest.cpp +++ /dev/null @@ -1,1741 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/trtGptModel.h" -#include "tensorrt_llm/batch_manager/trtGptModelFactory.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "tensorrt_llm/runtime/utils/numpyUtils.h" -#include "tensorrt_llm/testing/modelSpec.h" -#include "tests/utils/common.h" - -#include <gmock/gmock.h> -#include <gtest/gtest.h> - -#include <cstdint> -#include <cstdlib> -#include <memory> -#include <optional> -#include <vector> - -using namespace tensorrt_llm::testing; -using namespace tensorrt_llm::runtime; -using namespace tensorrt_llm::runtime::utils; -using namespace tensorrt_llm::batch_manager; -namespace fs = std::filesystem; -namespace tc = tensorrt_llm::common; -namespace texec = tensorrt_llm::executor; -using tensorrt_llm::testing::ModelSpec; -using tensorrt_llm::testing::KVCacheType; -using tensorrt_llm::testing::QuantMethod; - -namespace -{ -using TensorPtr = tensorrt_llm::runtime::ITensor::SharedPtr; - -auto constexpr GPT_MODEL_DIR = "gpt2"; -auto constexpr GPTJ_MODEL_DIR = "gpt-j-6b"; -auto constexpr LLAMA_MODEL_DIR = "Llama-3.2-1B"; -auto constexpr MEDUSA_MODEL_DIR = "vicuna-7b-medusa"; -auto constexpr EAGLE_MODEL_DIR = "vicuna-7b-eagle"; -auto constexpr MAMBA_MODEL_DIR = "mamba-2.8b-hf"; -auto constexpr RECURRENTGEMMA_MODEL_DIR = "recurrentgemma-2b"; -auto constexpr EXPLICIT_DRAFT_MODEL_DIR = "vicuna-7b-redrafter"; -auto constexpr CHATGLM_MODEL_DIR = "chatglm-6b"; -auto constexpr GLM_MODEL_DIR = "glm-10b"; - -auto constexpr FP8_GPT_ATTENTION_PLUGIN_IFB_PACKED_PATH = "fp8-plugin"; - -auto constexpr INPUT_FILE = "input_tokens.npy"; -auto constexpr INPUT_LLAMA_FILE = "input_tokens_llama.npy"; -auto constexpr INPUT_VICUNA_FILE = "input_vicuna.npy"; -auto constexpr LONG_INPUT_FILE = "input_tokens_long.npy"; -auto constexpr CHATGLM_INPUT_FILE = "input_tokens_chatglm-6b.npy"; -auto constexpr GLM_INPUT_FILE = "input_tokens_glm-10b.npy"; - -auto constexpr LLAMA_END_ID = 128001; -auto constexpr LLAMA_PAD_ID = 128001; - -struct ModelParams -{ - char const* baseDir; - ModelIds ids; - - friend std::ostream& operator<<(std::ostream& os, ModelParams const& modelParams) - { - return os << "baseDir: " << modelParams.baseDir << ", ids: (" << modelParams.ids.padId << "," - << modelParams.ids.endId << ")"; - } -}; - -} // namespace - -class TrtModelRealDecoderTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) -{ -protected: - TrtModelRealDecoderTest() {} - - void SetUp() override - { - mDeviceCount = tc::getDeviceCount(); - if (mDeviceCount == 0) - { - GTEST_SKIP() << "No GPUs found"; - } - - mLogger = std::make_shared<TllmLogger>(); - - initTrtLlmPlugins(mLogger.get()); - } - - void TearDown() override {} - - int mDeviceCount{}; - std::shared_ptr<nvinfer1::ILogger> mLogger{}; -}; - -enum class TrtGptModelIfbTestType -{ - BULK, - WAVEFRONT, - RANDOM -}; - -namespace -{ - -void verifyOutput(RequestList const& finishedRequestList, - std::unordered_map<SizeType32, TestData> const& beamWidthTestData, std::vector<SizeType32> const& givenInputLengths, - SizeType32 nbGivenInputs, ModelSpec const& modelSpec) -{ - auto const checkRawLogits = modelSpec.mOtherModelSpecToCompare ? false : modelSpec.mGatherLogits; - auto const smokeTest = modelSpec.mSmokeTest; - auto const returnLogProbs = modelSpec.mReturnLogProbs; - auto const checkAcceptedTokenLogits = modelSpec.mAcceptDraftByLogits; - - if (smokeTest) - { - return; - } - - for (auto const& llmReqPtr : finishedRequestList) - { - auto const& llmReq = *llmReqPtr; - auto const requestId = llmReq.mRequestId; - auto const [givenInputIdx, givenInputLength] - = getRequestGivenInputIdxLength(requestId, nbGivenInputs, givenInputLengths); - auto const reqBeamWidth = llmReq.mSamplingConfig.beamWidth; - auto const& testData = beamWidthTestData.at(reqBeamWidth); - auto const* const expectedOutputData = bufferCast<TokenIdType const>(*testData.expectedOutputIds); - auto const expectedOutputLengths = testData.expectedOutputLengths; - auto const acceptedDraftTokensLengths = testData.acceptedDraftTokensLengths; - auto const endId = testData.endIds[givenInputIdx]; - auto const maxSeqLen = testData.maxSeqLen; - auto const draftLogits = testData.draftLogits; - auto const expectedGenerationLogits = testData.expectedGenerationLogits; - auto const expectedContextLogits = testData.expectedContextLogits; - auto const expectedCumLogProbs = testData.expectedCumLogProbs; - auto const expectedLogProbs = testData.expectedLogProbs; - auto const draftTokens = llmReq.getDraftTokens(); - auto const isDraftTokensExternal = modelSpec.mSpecDecodingMode.isDraftTokensExternal(); - auto const inputLength = givenInputLength + static_cast<SizeType32>(isDraftTokensExternal); - - for (auto beam = 0; beam < reqBeamWidth; ++beam) - { - auto const expectedOutputLength = expectedOutputLengths[givenInputIdx * reqBeamWidth + beam]; - auto const predictedTokens = llmReq.getTokens(beam); - - auto numPredTokens = static_cast<SizeType32>(predictedTokens.size() - inputLength); - if (isDraftTokensExternal && !draftTokens->empty()) - { - numPredTokens - = std::min(numPredTokens, acceptedDraftTokensLengths[givenInputIdx * reqBeamWidth + beam] + 1); - } - if (modelSpec.mSpecDecodingMode.isMedusa() || modelSpec.mSpecDecodingMode.isLookaheadDecoding() - || modelSpec.mSpecDecodingMode.isExplicitDraftTokens() || modelSpec.mSpecDecodingMode.isEagle()) - { - // WAR to ensure bulk execution of spec decoding. - // We hope that no request in batch can finish 2x faster than any other request. - // For the cases when BS < 8, some predicted tokens are mismatched to reference data. - numPredTokens /= 2; - } - - if (modelSpec.mKVCacheType == KVCacheType::kDISABLED) - { - EXPECT_EQ(numPredTokens, 1) << "b: " << requestId << " beam: " << beam; - } - else - { - EXPECT_EQ(predictedTokens.size(), expectedOutputLength) << "b: " << requestId << " beam: " << beam; - } - - bool anyMismatch = false; - for (auto i = 0; i < numPredTokens; ++i) - { - // Use the expected data for that beamWidth - auto const expectIndex = tc::flat_index3(givenInputIdx, beam, inputLength + i, reqBeamWidth, maxSeqLen); - - auto const expectedToken = expectedOutputData[expectIndex]; - if (expectedToken == endId) - { - break; - } - auto const predictIndex = inputLength + i; - auto const predictedToken = predictedTokens.at(predictIndex); - EXPECT_EQ(predictedToken, expectedToken) << "b: " << requestId << " beam: " << beam << " i: " << i; - anyMismatch |= (predictedToken != expectedToken); - } - EXPECT_FALSE(anyMismatch) << "b: " << requestId << " beam: " << beam; - - if (returnLogProbs) - { - auto cumLogProbs = llmReq.getCumLogProbs(); - auto* const reqExpectedCumLogProbs = bufferCast<float>(*expectedCumLogProbs[requestId]); - EXPECT_TRUE(almostEqual(reqExpectedCumLogProbs[beam], cumLogProbs[beam])); - - auto logProbs = llmReq.getLogProbs(beam); - auto expectedLogProbsBeam = std::shared_ptr(ITensor::slice(expectedLogProbs[requestId], beam, 1)); - expectedLogProbsBeam->squeeze(0); - auto* const reqExpectedLogProbs = bufferCast<float>(*expectedLogProbsBeam); - - for (auto i = 0; i < numPredTokens; ++i) - { - EXPECT_TRUE(almostEqual(reqExpectedLogProbs[inputLength + i], logProbs[i], 5e-2, 5e-2)) - << "expectedLogProbs : " << reqExpectedLogProbs[inputLength + i] - << " logProbs : " << logProbs[i]; - } - } - - if (checkAcceptedTokenLogits && llmReq.hasDraftTokens()) - { - TLLM_CHECK_WITH_INFO(reqBeamWidth == 1, "speculative decoding only works for beam width == 1"); - - TensorPtr const& acceptedTokensLogits = llmReq.getGenerationLogitsHost(); - auto const acceptedTokensLogitsShape = acceptedTokensLogits->getShape(); - - EXPECT_EQ(acceptedTokensLogitsShape.nbDims, 3); - EXPECT_EQ(1, acceptedTokensLogitsShape.d[0]); - EXPECT_EQ(numPredTokens, acceptedTokensLogitsShape.d[1]); - - TensorPtr const& expectedLogits = ITensor::slice(expectedGenerationLogits[requestId], 1, numPredTokens); - - // For hyperparameters - // Greater tolerance for the accepted logits of the target model. - float atol = 0.f; - float rtol = 0.01f; - EXPECT_TRUE(compareLogits(*expectedLogits, *acceptedTokensLogits, atol, rtol)); - } - - if (checkRawLogits) - { - // Check generation logits - TensorPtr const& expectedGenerationLogitsSliced - = ITensor::slice(expectedGenerationLogits[requestId], 0, numPredTokens); - - TensorPtr const& llmReqGeneration = llmReq.getGenerationLogitsHost(); - auto llmReqGenerationShape = llmReqGeneration->getShape(); - - TensorPtr generationLogitsBeam = nullptr; - if (llmReq.isStreaming()) - { - // Expect generation logits shape: [outputLength, beamWidth, vocabSizePad] - EXPECT_EQ(reqBeamWidth, llmReqGenerationShape.d[1]); - EXPECT_EQ(reqBeamWidth, 1); // Streaming mode does not support beam > 1 - llmReqGeneration->squeeze(1); // [outputLength, vocabSizePad] - generationLogitsBeam = llmReqGeneration; - } - else - { - // Expect generation logits shape: [beamWidth, outputLength, vocabSizePad] - EXPECT_EQ(reqBeamWidth, llmReqGenerationShape.d[0]); - generationLogitsBeam - = std::shared_ptr(ITensor::slice(llmReqGeneration, beam, 1)); // [1, outputLength, vocabSizePad] - generationLogitsBeam->squeeze(0); // [outputLength, vocabSizePad] - } - TensorPtr const& generationLogitsSliced = ITensor::slice(generationLogitsBeam, 0, numPredTokens); - EXPECT_TRUE(compareLogits(*expectedGenerationLogitsSliced, *generationLogitsSliced)); - } - } - - if (checkRawLogits) - { - // Check context logits - TensorPtr const& llmReqContext = llmReq.getContextLogitsHost(); - auto llmReqContextShape = llmReqContext->getShape(); - EXPECT_EQ(llmReqContextShape.nbDims, 2); - EXPECT_EQ(llmReq.mPromptLen, llmReqContextShape.d[0]); - EXPECT_TRUE(compareLogits(*expectedContextLogits[requestId], *llmReqContext)); - } - } -} - -// Pick a different endId at random from one of the expected tokens -std::vector<TokenIdType> pickRandomEndIds(TestData const& testData, std::vector<SizeType32> const& givenInputLengths, - SizeType32 const maxNewTokens, bool replaceLogits) -{ - auto const nbGivenInputs = testData.nbGivenInputs; - auto const beamWidth = testData.beamWidth; - auto* const expectedOutputData = bufferCast<TokenIdType>(*testData.expectedOutputIds); - - std::vector<TokenIdType> endIds; - - // For IFB, pick one of the output tokens as endId - for (SizeType32 bi = 0; bi < nbGivenInputs; ++bi) - { - TokenIdType skippedEndId0 = 0; - TokenIdType skippedEndId1 = 0; - SizeType32 endIdIndex = 0; - TokenIdType endId = 0; - auto const endIdRow = bi; - auto const inputLength = givenInputLengths.at(endIdRow); - do - { - auto const endIdBeam = std::rand() % beamWidth; - auto const firstOutputIndex - = tc::flat_index3(endIdRow, endIdBeam, inputLength, beamWidth, testData.maxSeqLen); - // We do not use the 1st token for EndId because of Speculative Decoding test design - // We skip 1st token because minLength is 1 - auto const endIdCol = 2 + (std::rand() % std::max(maxNewTokens - 2, 1)); - endIdIndex = firstOutputIndex + endIdCol; - skippedEndId0 = expectedOutputData[firstOutputIndex]; - skippedEndId1 = expectedOutputData[firstOutputIndex + 1]; - endId = expectedOutputData[endIdIndex]; - } while (endId == skippedEndId0 || endId == skippedEndId1); - // Workaround: The first example has endIdIndex 14, where the generation logits are almost same at - // token ids 257 and 373, which causes unstable generation results. Hence, we use the one previous - // token as endId. - if (bi == 0 && !replaceLogits) - { - endId = expectedOutputData[endIdIndex - 1]; - } - endIds.push_back(endId); - } - - return endIds; -} - -TestData loadTestData(ModelSpec const& modelSpec, ModelIds const modelIds, BeamResult const& beamResult, - ITensor const& givenInput, SizeType32 const maxBeamWidth, bool const useRandomEndId, bool const replaceLogits, - BufferManager& manager) -{ - auto const [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(givenInput, modelIds.padId); - auto const& [beamWidth, resultsFile, contextLogitsFile, genLogitsFile, cumLogProbsFile, logProbsFile] = beamResult; - - TestData testData{nbGivenInputs, beamWidth}; - testData.expectedOutputIds = loadNpy(manager, resultsFile.string(), MemoryType::kCPU); - - auto* const expectedOutputData = bufferCast<TokenIdType>(*testData.expectedOutputIds); - - auto const& outputShape = testData.expectedOutputIds->getShape(); - EXPECT_EQ(outputShape.nbDims, 2); - EXPECT_EQ(nbGivenInputs * beamWidth, outputShape.d[0]); - testData.maxSeqLen = static_cast<SizeType32>(outputShape.d[1]); - EXPECT_LE(maxInputLength, testData.maxSeqLen); - EXPECT_LE(beamWidth, maxBeamWidth); - - auto const maxNewTokens = testData.maxSeqLen - maxInputLength; - - std::srand(42); - - if (useRandomEndId) - { - testData.endIds = pickRandomEndIds(testData, givenInputLengths, maxNewTokens, replaceLogits); - } - else - { - testData.endIds.insert(testData.endIds.end(), nbGivenInputs, modelIds.endId); - } - - if (modelSpec.useLogits()) - { - testData.loadContextLogits(contextLogitsFile, givenInputLengths, manager); - } - if (modelSpec.useLogits() || modelSpec.mAcceptDraftByLogits) - { - testData.loadGenerationLogits(genLogitsFile, manager); - } - if (modelSpec.mReturnLogProbs) - { - testData.loadLogProbs(cumLogProbsFile, logProbsFile, manager); - } - - for (SizeType32 bi = 0; bi < nbGivenInputs; ++bi) - { - auto const endId = testData.endIds[bi]; - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - SizeType32 expectedLen = givenInputLengths[bi] + maxNewTokens; - for (SizeType32 si = givenInputLengths[bi]; si < testData.maxSeqLen; ++si) - { - auto const expectIndex = tc::flat_index2((bi * beamWidth + beam), si, testData.maxSeqLen); - if (expectedOutputData[expectIndex] == endId) - { - expectedLen = si; - break; - } - } - // Fill new EOS token to the expected data - for (SizeType32 si = expectedLen; si < testData.maxSeqLen; ++si) - { - auto const expectIndex = tc::flat_index2((bi * beamWidth + beam), si, testData.maxSeqLen); - expectedOutputData[expectIndex] = endId; - } - - testData.expectedOutputLengths[bi * beamWidth + beam] = expectedLen; - } - } - - if (modelSpec.mMaxDraftTokens > 0) - { - testData.makeDraft( - modelSpec.mMaxDraftTokens, modelSpec.mAcceptDraftByLogits, genLogitsFile, givenInputLengths, manager); - } - - return testData; -} - -std::tuple<std::vector<SizeType32>, std::unordered_map<SizeType32, TestData>> loadTestData(ModelSpec const& modelSpec, - ModelIds const modelIds, BeamResults const& resultsFilesBeamWidths, ITensor const& givenInput, - SizeType32 const maxBeamWidth, bool const useRandomEndId, bool const replaceLogits, BufferManager& manager) -{ - // Map between beam width, and expected results for that beam width - std::unordered_map<SizeType32, TestData> beamWidthTestData; - std::vector<SizeType32> beamWidths; - - for (auto const& beamResult : resultsFilesBeamWidths) - { - auto const beamWidth = beamResult.beamWidth; - - EXPECT_EQ(std::find(beamWidths.begin(), beamWidths.end(), beamWidth), beamWidths.end()); - beamWidths.push_back(beamWidth); - - auto testData = loadTestData( - modelSpec, modelIds, beamResult, givenInput, maxBeamWidth, useRandomEndId, replaceLogits, manager); - beamWidthTestData.emplace(beamWidth, std::move(testData)); - } - - return {std::move(beamWidths), std::move(beamWidthTestData)}; -} - -RequestList runGptModelInference(std::shared_ptr<TrtGptModel>& trtGptModel, std::vector<SizeType32> const& beamWidths, - std::unordered_map<SizeType32, TestData> const& beamWidthTestData, SizeType32 batchSize, SizeType32 nbGivenInputs, - SizeType32 maxInputLength, SizeType32 padId, std::vector<SizeType32> const& givenInputLengths, - TokenIdType const* givenInputData, ModelSpec const& modelSpec, TrtGptModelIfbTestType testType, int maxReqPerStep, - bool prepopulateKVCache, bool enableStreamingMode, bool enableBlockReuse) -{ - // Fill the requests using givenInput - // requestList will have batchSize requests - RequestList requestList; - - SizeType32 requestId = 0; - RequestList finishedRequestList; - std::vector<SizeType32> reqVec; - // Advance the requests until they are all finished - if (COMM_SESSION.getRank() == 0) - { - SizeType32 numReq = 0; - while (numReq < batchSize) - { - // Add appropriate number of requests in each iteration. For WAVEFRONT, this is always 1. - // For RANDOM, it could be any integer <= maxReqPerStep including 0. - SizeType32 reqThisStep{0}; - switch (testType) - { - case TrtGptModelIfbTestType::WAVEFRONT: reqThisStep = 1; break; - case TrtGptModelIfbTestType::RANDOM: reqThisStep = rand() % (maxReqPerStep + 1); break; - case TrtGptModelIfbTestType::BULK: [[fallthrough]]; - default: reqThisStep = batchSize; break; - } - reqThisStep = std::min(reqThisStep, (batchSize - numReq)); - reqVec.push_back(reqThisStep); - numReq += reqThisStep; - } - } - COMM_SESSION.bcast(reqVec, 0); - - SizeType32 reqVecIdx = 0; - while (requestId < batchSize || !requestList.empty()) - { - SizeType32 reqThisStep = reqVecIdx < reqVec.size() ? reqVec[reqVecIdx++] : 0; - for (SizeType32 req = 0; req < reqThisStep; req++) - { - // Alternate between beamWidths - SizeType32 beamWidth = beamWidths.at(requestId % beamWidths.size()); - auto const& testData = beamWidthTestData.at(beamWidth); - auto const* const expectedOutputData = bufferCast<TokenIdType const>(*testData.expectedOutputIds); - auto const maxSeqLen = testData.maxSeqLen; - - SamplingConfig samplingConfig{beamWidth}; - samplingConfig.temperature = std::vector{1.0f}; - samplingConfig.minLength = std::vector{1}; - samplingConfig.randomSeed = std::vector{static_cast<uint64_t>(42ull)}; - samplingConfig.topK = std::vector{1}; - samplingConfig.topP = std::vector{0.0f}; - samplingConfig.draftAcceptanceThreshold = std::vector{0.3f}; - samplingConfig.noRepeatNgramSize = std::vector{1 << 30}; - - auto const [givenInputIdx, inputLength] - = getRequestGivenInputIdxLength(requestId, nbGivenInputs, givenInputLengths); - SizeType32 endId = testData.endIds[givenInputIdx]; - - auto maxNewTokens = maxSeqLen - maxInputLength; - // Run model only to produce a single token and prepopulate KV cache - if (prepopulateKVCache || modelSpec.mKVCacheType == KVCacheType::kDISABLED) - { - maxNewTokens = 1; - } - auto const* const seqBegin = givenInputData + givenInputIdx * maxInputLength; - auto tokens = std::make_shared<std::vector<int32_t>>(seqBegin, seqBegin + inputLength); - if (!prepopulateKVCache && modelSpec.mMaxDraftTokens > 0) - { - // Append the 1st predicted token to the prompt to get the match with prepopulated KV cache - auto const expectIndex = tc::flat_index3(givenInputIdx, 0, inputLength, 1, maxSeqLen); - auto expectedToken = expectedOutputData[expectIndex]; - tokens->push_back(expectedToken); - // subtract this token from maxNewTokens - maxNewTokens -= 1; - } - auto r = std::make_shared<LlmRequest>(requestId, maxNewTokens, tokens, samplingConfig, false, endId, padId); - - auto const& draftTokens = testData.draftTokens[givenInputIdx]; - auto draftLogits = modelSpec.mAcceptDraftByLogits - ? std::make_optional<ITensor::SharedPtr>(testData.draftLogits[givenInputIdx]) - : std::nullopt; - if (!prepopulateKVCache && !draftTokens.empty()) - { - r->setDraftTokens(std::make_shared<std::vector<TokenIdType>>(draftTokens)); - r->setDraftLogits(draftLogits); - } - - SizeType32 maxDraftTokens{0}; - if (trtGptModel->getModelConfig().hasSpeculativeDecodingModule()) - { - maxDraftTokens - = trtGptModel->getModelConfig().getSpeculativeDecodingModulePtr()->getMaxDecodingDraftTokens(); - } - r->validate(trtGptModel->getMaxInputLen(), trtGptModel->getMaxSequenceLen(), maxDraftTokens, - trtGptModel->getVocabSizePadded(), std::nullopt, enableBlockReuse); - - if (enableStreamingMode) - { - r->setReturnAllGeneratedTokens(true); // Test allGeneratedTokens in this test - r->setStreaming(true); - } - - auto const vocabSizePadded - = trtGptModel->getModelConfig().getVocabSizePadded(trtGptModel->getWorldConfig().getSize()); - auto const logitDatatype = trtGptModel->getLogitDataType(); - if (modelSpec.mGatherLogits) - { - r->setReturnContextLogits(true); - r->setReturnGenerationLogits(true); - r->allocContextLogitsHost(vocabSizePadded, logitDatatype); - r->allocGenerationLogitsHost(vocabSizePadded, logitDatatype); - } - - if (!prepopulateKVCache && modelSpec.mAcceptDraftByLogits && !draftTokens.empty()) - { - r->allocTargetModelAcceptedTokenLogitsHost(vocabSizePadded, logitDatatype); - r->setReturnGenerationLogits(true); - } - - if (modelSpec.mReplaceLogits) - { - LlmRequest::LogitsPostProcessor logitsCb - = [&testData](uint64_t rId, tensorrt_llm::runtime::ITensor::SharedPtr& logits, - LlmRequest::BeamTokens const& tokens, - tensorrt_llm::runtime::BufferManager::CudaStreamPtr streamPtr, std::optional<uint64_t> cId) - { - auto const expectedGenerationLogits = testData.expectedGenerationLogits[rId]; - auto const expectedContextLogits = testData.expectedContextLogits[rId]; - auto const acceptedDraftTokensLengths = testData.acceptedDraftTokensLengths[rId]; - - auto const beamWidth = tokens.size(); - TLLM_CHECK_WITH_INFO(beamWidth == 1, "Logits substitution is not supported for beam search"); - - auto const genLogitsOffset = tokens[0].size() - expectedContextLogits->getShape().d[0]; - // TODO: Avoid static cast in TRT 10.0 - auto const numLogits = static_cast<SizeType32>(logits->getShape().d[0]); - auto const numVerifyLogits = std::min(numLogits, acceptedDraftTokensLengths + 1); - - TensorPtr logitsSlice = ITensor::slice(logits, 0, numVerifyLogits); - - auto manager = BufferManager(streamPtr); - TensorPtr logitsHost = manager.copyFrom(*logitsSlice, MemoryType::kCPU); - manager.getStream().synchronize(); - - TensorPtr refLogitsHost - = ITensor::slice(expectedGenerationLogits, genLogitsOffset, numVerifyLogits); - - EXPECT_TRUE(compareLogits(*refLogitsHost, *logitsHost, 0.f, 1e-2)) << "reqId: " << rId; - - manager.copy(*refLogitsHost, *logitsSlice); - }; - - r->mLogitsPostProcessor = logitsCb; - } - - if (modelSpec.mReturnLogProbs) - { - r->setReturnLogProbs(true); - } - requestList.push_back(r); - ++requestId; - } - - // Advance all active requests by one step - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - - // Check which requests are done, move them out - for (auto it = requestList.cbegin(); it != requestList.cend();) - { - if ((*it)->isGenerationCompleteState()) - { - finishedRequestList.push_back(*it); - requestList.erase(it++); - } - else - { - ++it; - } - } - } - return finishedRequestList; -} - -void runIfbTest(fs::path const& modelPath, ModelSpec const& modelSpec, ModelIds const modelIds, - TrtGptModelType modelType, std::vector<int32_t> const& batchSizes, BeamResults const& resultsFilesBeamWidths, - TrtGptModelIfbTestType testType, int maxReqPerStep, texec::ExecutorConfig const& executorConfig, - bool enableStreamingMode, bool useRandomEndId) -{ - auto manager = BufferManager(std::make_shared<CudaStream>()); - auto const padId = modelIds.padId; - - // Load input data - ASSERT_TRUE(fs::exists(DATA_PATH)); - auto const inputPath = DATA_PATH / modelSpec.mInputFile; - auto const& givenInput = loadNpy(manager, inputPath.string(), MemoryType::kCPU); - auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, padId); - auto const* const givenInputData = bufferCast<TokenIdType const>(*givenInput); - - auto const& inputShape = givenInput->getShape(); - ASSERT_EQ(inputShape.nbDims, 2); - ASSERT_GT(inputShape.d[0], 0); - - auto const maxBeamWidth = executorConfig.getMaxBeamWidth(); - // Load expected outputs for each beam width value - auto [beamWidths, beamWidthTestData] = loadTestData(modelSpec, modelIds, resultsFilesBeamWidths, *givenInput, - maxBeamWidth, useRandomEndId, modelSpec.mReplaceLogits, manager); - - int const worldSize = modelSpec.mTPSize * modelSpec.mPPSize * modelSpec.mCPSize; - auto const worldConfig = WorldConfig::mpi(worldSize, modelSpec.mTPSize, modelSpec.mPPSize, modelSpec.mCPSize); - - ASSERT_TRUE(fs::exists(modelPath)); - - for (auto batchSize : batchSizes) - { - std::cout << "=== batchSize:" << batchSize << " ===\n"; - - auto trtGptModel = TrtGptModelFactory::create(modelPath, modelType, executorConfig, false); - - if (modelSpec.mKVCacheType == KVCacheType::kDISABLED) - { - ASSERT_FALSE(trtGptModel->hasKVCacheManager()); - } - - // Prepopulate KV cache for speculative decoding test - bool const prepopulateKVCache = modelSpec.mMaxDraftTokens > 0; - auto finishedRequestList = runGptModelInference(trtGptModel, beamWidths, beamWidthTestData, batchSize, - nbGivenInputs, maxInputLength, padId, givenInputLengths, givenInputData, modelSpec, testType, maxReqPerStep, - prepopulateKVCache, enableStreamingMode, modelSpec.mKVCacheReuse); - - if (prepopulateKVCache) - { - // Call the 2nd time with prefilled KV cache - finishedRequestList = runGptModelInference(trtGptModel, beamWidths, beamWidthTestData, batchSize, - nbGivenInputs, maxInputLength, padId, givenInputLengths, givenInputData, modelSpec, testType, - maxReqPerStep, false, enableStreamingMode, modelSpec.mKVCacheReuse); - } - - // WAR: disabled verification because of switched beams for different batch composition - if (worldConfig.isFirstPipelineParallelRank() - && (testType == TrtGptModelIfbTestType::BULK || maxBeamWidth == 1)) - { - bool shouldVerify = true; - - if (testType == TrtGptModelIfbTestType::BULK) - { - if (modelSpec.mKVCacheType == KVCacheType::kDISABLED && maxBeamWidth != 1) - { - // For disabled KV cache, only verify when maxBeamWidth is 1, the reason is we only compare with - // results with KV cache enabled case and usually, beams search results locate in last token while - // disabled KV cache only get exactly one new token. - shouldVerify = false; - } - } - - if (shouldVerify) - { - verifyOutput(finishedRequestList, beamWidthTestData, givenInputLengths, nbGivenInputs, modelSpec); - } - } - } -} - -struct BeamConfig -{ - SizeType32 maxBeamWidth; - std::vector<SizeType32> beamWidths; -}; - -} // namespace - -using ParamType = std::tuple<ModelParams, ModelSpec, TrtGptModelType, TrtGptModelIfbTestType, BeamConfig, // id: 0-4 - std::optional<int32_t>, // 5. maxTokensInPagedKvCache - std::optional<float>, // 6. freeGpuMemoryFraction - bool, // 7. enableTrtOverlap - bool, // 8. enableChunkedContext - bool, // 9. enableStreamingMode - bool, // 10. enableCudaGraphMode - std::optional<size_t>, // 11. hostCacheSize - bool, // 12. useRandomEndId - std::vector<SizeType32>, // 13. batchSizes - std::optional<SizeType32> // 14. maxNumTokens - >; - -std::string generateTestName(testing::TestParamInfo<ParamType> const& info) -{ - auto const modelSpec = std::get<1>(info.param); - std::string name; - switch (modelSpec.mDataType) - { - case nvinfer1::DataType::kFLOAT: name.append("Float"); break; - case nvinfer1::DataType::kHALF: name.append("Half"); break; - case nvinfer1::DataType::kINT8: name.append("Int8"); break; - case nvinfer1::DataType::kINT32: name.append("Int32"); - case nvinfer1::DataType::kBOOL: name.append("Bool"); break; - case nvinfer1::DataType::kUINT8: name.append("UInt8"); break; - case nvinfer1::DataType::kFP8: name.append("Float8"); break; - case nvinfer1::DataType::kBF16: name.append("BFloat16"); break; - case nvinfer1::DataType::kINT4: name.append("Int4"); break; - case nvinfer1::DataType::kFP4: name.append("Fp4"); break; - default: throw std::runtime_error("Unsupported DataType"); break; - } - - auto const modelType = std::get<2>(info.param); - switch (modelType) - { - case TrtGptModelType::InflightBatching: name.append("IbModel"); break; - case TrtGptModelType::InflightFusedBatching: name.append("FusedIbModel"); break; - default: name.append("DefaultModel"); break; - } - - switch (modelSpec.mKVCacheType) - { - case KVCacheType::kCONTINUOUS: name.append("ContinuousKVCache"); break; - case KVCacheType::kPAGED: name.append("PagedKVCache"); break; - case KVCacheType::kDISABLED: name.append("NoKVCache"); break; - default: throw std::runtime_error("Unknown KVCacheType"); break; - } - - auto const testType = std::get<3>(info.param); - switch (testType) - { - case TrtGptModelIfbTestType::BULK: name.append("Bulk"); break; - case TrtGptModelIfbTestType::WAVEFRONT: name.append("Wavefront"); break; - case TrtGptModelIfbTestType::RANDOM: name.append("Random"); break; - default: name.append("DefaultTest"); break; - } - BeamConfig const beamConfig = std::get<4>(info.param); - name.append("MaxBeamWidth" + std::to_string(beamConfig.maxBeamWidth)); - for (auto const beamWdith : beamConfig.beamWidths) - { - name.append("Bw" + std::to_string(beamWdith)); - } - - auto const maxTokensInPagedKvCache = std::get<5>(info.param); - if (maxTokensInPagedKvCache.has_value()) - { - name.append("KvCacheSize" + std::to_string(maxTokensInPagedKvCache.value())); - } - - auto const freeGpuMemoryFraction = std::get<6>(info.param); - if (freeGpuMemoryFraction.has_value()) - { - name.append("GpuFrac"); - } - - auto const enableTrtOverlap = std::get<7>(info.param); - if (enableTrtOverlap) - { - name.append("TrtOverlap"); - } - - auto const enableChunkedContext = std::get<8>(info.param); - if (enableChunkedContext) - { - name.append("Chunked"); - } - - if (modelSpec.mTPSize > 1) - { - name.append("TP" + std::to_string(modelSpec.mTPSize)); - } - - if (modelSpec.mPPSize > 1) - { - name.append("PP" + std::to_string(modelSpec.mPPSize)); - } - - if (modelSpec.mCPSize > 1) - { - name.append("CP" + std::to_string(modelSpec.mCPSize)); - } - - auto const useRandomEndId = std::get<12>(info.param); - if (useRandomEndId) - { - name.append("EndId"); - } - - if (modelSpec.mMaxDraftTokens > 0) - { - name.append("DraftTokens" + std::to_string(modelSpec.mMaxDraftTokens)); - } - - if (modelSpec.mAcceptDraftByLogits) - { - name.append("AcceptByLogits"); - } - - if (modelSpec.mCapacitySchedulerPolicy) - { - name.append(modelSpec.getCapacitySchedulerString()); - } - - auto const enableStreamingMode = std::get<9>(info.param); - if (enableStreamingMode) - { - name.append("Streaming"); - } - - auto const enableCudaGraphMode = std::get<10>(info.param); - if (enableCudaGraphMode) - { - name.append("CudaGraph"); - } - - auto const enableHostCache = std::get<11>(info.param); - if (enableHostCache) - { - name.append("SecondaryOffloading"); - } - - return name; -} - -class ParamTest : public TrtModelRealDecoderTest, public ::testing::WithParamInterface<ParamType> -{ -}; - -TEST_P(ParamTest, Test) -{ - - auto const& beamConfig = std::get<4>(GetParam()); - auto const& beamWidths = beamConfig.beamWidths; - - auto const modelParams = std::get<0>(GetParam()); - auto const modelIds = modelParams.ids; - auto const* const modelDir = modelParams.baseDir; - auto const modelSpec = std::get<1>(GetParam()); - - auto const useRandomEndId = std::get<12>(GetParam()); - - auto const batchSizes = std::get<13>(GetParam()); - - std::ostringstream gpuSizePath; - gpuSizePath << "tp" << modelSpec.mTPSize << "-pp" << modelSpec.mPPSize << "-cp" << modelSpec.mCPSize; - gpuSizePath << "-gpu"; - - auto const modelPath{ENGINE_PATH / modelDir / modelSpec.getModelPath() / gpuSizePath.str()}; - - auto const inputPath = DATA_PATH / modelSpec.mInputFile; - - BeamResults beamResults; - beamResults.reserve(beamWidths.size()); - for (auto beamWidth : beamWidths) - { - fs::path resultsPath - = DATA_PATH / modelDir / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - fs::path generationLogitsPath - = modelSpec.mCollectGenerationLogits ? (resultsPath / modelSpec.getGenerationLogitsFile()).string() : ""; - fs::path contextLogitsPath - = modelSpec.mCollectContextLogits ? (resultsPath / modelSpec.getContextLogitsFile()).string() : ""; - fs::path cumLogProbsPath - = modelSpec.mCollectCumLogProbs ? (resultsPath / modelSpec.getCumLogProbsFile()).string() : ""; - fs::path logProbsPath = modelSpec.mCollectLogProbs ? (resultsPath / modelSpec.getLogProbsFile()).string() : ""; - - beamResults.emplace_back(beamWidth, (resultsPath / modelSpec.getResultsFile()).string(), contextLogitsPath, - generationLogitsPath, cumLogProbsPath, logProbsPath); - } - - auto const modelType = std::get<2>(GetParam()); - auto const testType = std::get<3>(GetParam()); - auto const enableStreamingMode = std::get<9>(GetParam()); - auto const cudaGraphMode = std::get<10>(GetParam()); - - if (!(modelSpec.mUsePackedInput - && (modelSpec.mKVCacheType == KVCacheType::kPAGED || modelSpec.mKVCacheType == KVCacheType::kDISABLED))) - { - GTEST_SKIP() << "Inflight batching requires packed input and (paged KV cache or disabled KV cache)."; - } - - if (!modelSpec.mUsePackedInput && useRandomEndId) - { - GTEST_SKIP() << "Test does not support endId test with padded inputs"; - } - - for (auto beamWidth : beamWidths) - { - if (useRandomEndId && beamWidth > 1) - { - GTEST_SKIP() << "Test does not support endId test with beam search"; - } - - if (modelSpec.mMaxDraftTokens > 0 && beamWidth > 1) - { - GTEST_SKIP() << "Target model in speculative decoding does not support beam search"; - } - } - - auto executorConfig = texec::ExecutorConfig{}; - - auto const maxTokens = std::get<5>(GetParam()); - auto const enableBlockReuse = modelSpec.mMaxDraftTokens > 0 || modelSpec.mKVCacheReuse; - auto const freeGpuMemoryFraction = std::get<6>(GetParam()); - auto const hostCacheSize = std::get<11>(GetParam()); - auto const kvCacheConfig = texec::KvCacheConfig{ - enableBlockReuse, maxTokens, std::nullopt, std::nullopt, freeGpuMemoryFraction, hostCacheSize}; - executorConfig.setKvCacheConfig(kvCacheConfig); - - executorConfig.setEnableTrtOverlap(std::get<7>(GetParam())); - executorConfig.setEnableChunkedContext(std::get<8>(GetParam())); - auto const maxNumTokens = std::get<14>(GetParam()); - if (maxNumTokens.has_value()) - { - executorConfig.setMaxNumTokens(maxNumTokens.value()); - } - executorConfig.setNormalizeLogProbs(false); - executorConfig.setMaxBeamWidth(beamConfig.maxBeamWidth); - executorConfig.setGatherGenerationLogits(modelSpec.mCollectGenerationLogits); - auto extendedRuntimePerfKnobConfig = texec::ExtendedRuntimePerfKnobConfig{}; - extendedRuntimePerfKnobConfig.setCudaGraphMode(cudaGraphMode); - executorConfig.setExtendedRuntimePerfKnobConfig(extendedRuntimePerfKnobConfig); - - auto const capacitySchedulerPolicy - = modelSpec.mCapacitySchedulerPolicy.value_or(texec::CapacitySchedulerPolicy::kMAX_UTILIZATION); - executorConfig.setSchedulerConfig(texec::SchedulerConfig{capacitySchedulerPolicy}); - - if (modelSpec.mSpecDecodingMode == SpeculativeDecodingMode::LookaheadDecoding()) - { - auto decodingConfig = texec::DecodingConfig{}; - decodingConfig.setLookaheadDecodingConfig(texec::LookaheadDecodingConfig(5, 5, 5)); - executorConfig.setDecodingConfig(decodingConfig); - } - - for (auto beamWidth : beamWidths) - { - if (executorConfig.getEnableTrtOverlap() && beamWidth > 1) - { - GTEST_SKIP() << "TrtOverlap is not supported with beam search"; - } - } - - if (executorConfig.getEnableTrtOverlap() && modelSpec.mMaxDraftTokens > 0) - { - GTEST_SKIP() << "TrtOverlap is not supported with speculative decoding"; - } - - // Warning: This should be the last check before running the test. - // It will initialize MPI which can take significant time. - if (modelSpec.mTPSize * modelSpec.mPPSize * modelSpec.mCPSize != COMM_SESSION.getSize()) - { - GTEST_SKIP() << "Model's world size " << modelSpec.mPPSize * modelSpec.mTPSize * modelSpec.mCPSize - << " is not equal to the system world size"; - } - - runIfbTest(modelPath, modelSpec, modelIds, modelType, batchSizes, beamResults, testType, 2, executorConfig, - enableStreamingMode, useRandomEndId); -} - -auto constexpr gptModelParams = ModelParams{GPT_MODEL_DIR, ModelIds{50256, 50256}}; - -std::shared_ptr<ModelSpec> getGptDraftTestsCompareModelSpec() -{ - auto pModelSpec = std::make_shared<ModelSpec>(INPUT_FILE, nvinfer1::DataType::kHALF); - pModelSpec->useGptAttentionPlugin(); - pModelSpec->gatherLogits(); - pModelSpec->usePackedInput(); - pModelSpec->setKVCacheType(KVCacheType::kPAGED); - - return pModelSpec; -} - -std::shared_ptr<ModelSpec> getMedusaTestsCompareModelSpec() -{ - auto pModelSpec = std::make_shared<ModelSpec>(LONG_INPUT_FILE, nvinfer1::DataType::kHALF); - pModelSpec->useGptAttentionPlugin(); - pModelSpec->usePackedInput(); - pModelSpec->setKVCacheType(KVCacheType::kPAGED); - pModelSpec->setMaxOutputLength(128); - - return pModelSpec; -} - -std::shared_ptr<ModelSpec> getEagleTestsCompareModelSpec() -{ - auto pModelSpec = std::make_shared<ModelSpec>(LONG_INPUT_FILE, nvinfer1::DataType::kHALF); - pModelSpec->useGptAttentionPlugin(); - pModelSpec->usePackedInput(); - pModelSpec->setKVCacheType(KVCacheType::kPAGED); - pModelSpec->setMaxOutputLength(128); - - return pModelSpec; -} - -std::shared_ptr<ModelSpec> getGptChunkedContextTestsCompareModelSpec() -{ - auto pModelSpec = std::make_shared<ModelSpec>(LONG_INPUT_FILE, nvinfer1::DataType::kHALF); - pModelSpec->useGptAttentionPlugin(); - pModelSpec->usePackedInput(); - pModelSpec->setKVCacheType(KVCacheType::kPAGED); - pModelSpec->setMaxInputLength(128); - - return pModelSpec; -} - -INSTANTIATE_TEST_SUITE_P(GptTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values( - // - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput(), - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF, - []() -> std::shared_ptr<ModelSpec> - { - auto pModelSpec = std::make_shared<ModelSpec>(INPUT_FILE, nvinfer1::DataType::kHALF); - pModelSpec->useGptAttentionPlugin().setKVCacheType(KVCacheType::kPAGED).usePackedInput(); - return pModelSpec; - }()} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kDISABLED) - .usePackedInput()), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values( - // TODO: enable more tests when mixed beam width is supported - BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} - ), - testing::Values(std::nullopt, 1280), // maxTokensInPagedKvCache - testing::Values(std::nullopt, 0.4), // freeGpuMemoryFraction - testing::Values(true), // enableTrtOverlap - testing::Values(false), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptRandomEndIdTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values( - // - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput()), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values( - // TODO: enable more tests when mixed beam width is supported - BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} - ), - testing::Values(std::nullopt, 1280), // maxTokensInPagedKvCache - testing::Values(std::nullopt, 0.4), // freeGpuMemoryFraction - testing::Values(true), // enableTrtOverlap - testing::Values(false), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(true), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptKVOffloadingTest, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values( - // - ModelSpec{LONG_INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput() - .setKVCacheReuse(true)), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values(BeamConfig{1, {1}}), - testing::Values(256), // maxTokensInPagedKvCache - testing::Values(std::nullopt, 0.4), // freeGpuMemoryFraction - testing::Values(true), // enableTrtOverlap - testing::Values(false), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(100000000), // hostCacheSize - testing::Values(false, true), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptCudaGraphTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values( - // - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput() - .capacitySchedulerPolicy(texec::CapacitySchedulerPolicy::kSTATIC_BATCH), - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput() - .capacitySchedulerPolicy(texec::CapacitySchedulerPolicy::kMAX_UTILIZATION)), - testing::Values(TrtGptModelType::InflightBatching, TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values( - // TODO: enable more tests when mixed beam width is supported - BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} - ), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(true), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptSwitchBwTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values( - // - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput()), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values( - // TODO: enable more tests when mixed beam width is supported - BeamConfig{2, {1}} // , BeamConfig{2, {1, 2}} - ), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{4}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptNProfilesTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values(ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .useMultipleProfiles()), - testing::Values(TrtGptModelType::InflightFusedBatching), testing::Values(TrtGptModelIfbTestType::BULK), - testing::Values( - // TODO: enable more tests when mixed beam width is supported - BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} - ), - testing::Values(std::nullopt, 1280), // maxTokensInPagedKvCache - testing::Values(std::nullopt, 0.4), // freeGpuMemoryFraction - testing::Values(true), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(true), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptSqTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values(ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .setQuantMethod(QuantMethod::kSMOOTH_QUANT), - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF, - []() -> std::shared_ptr<ModelSpec> - { - auto pModelSpec = std::make_shared<ModelSpec>(INPUT_FILE, nvinfer1::DataType::kHALF); - pModelSpec->useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .setQuantMethod(QuantMethod::kSMOOTH_QUANT); - return pModelSpec; - }()} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kDISABLED) - .setQuantMethod(QuantMethod::kSMOOTH_QUANT)), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values( - // TODO: enable more tests when mixed beam width is supported - // FIXME: disabled flaky beam search tests (https://nvbugspro.nvidia.com/bug/4646234) - BeamConfig{1, {1}} //, BeamConfig{2, {2}} - ), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(false), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -// disabled because paused requests generate different tokens after resuming -INSTANTIATE_TEST_SUITE_P(DISABLED_GptChunkedContextTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values( - // - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF, getGptChunkedContextTestsCompareModelSpec()} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .setMaxInputLength(128)), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values(TrtGptModelIfbTestType::BULK), // TrtGptModelIfbTestType - testing::Values(BeamConfig{1, {1}}), // beam config - testing::Values(257), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptChunkedLongContextTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values( - // - ModelSpec{LONG_INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .setMaxInputLength(128), - ModelSpec{LONG_INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .useDraftTokensExternalDecoding() - .setDraftTokens(5)), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values(TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, - TrtGptModelIfbTestType::RANDOM), // TrtGptModelIfbTestType - testing::Values(BeamConfig{1, {1}}), // beam config - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(true), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(64) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptDraftTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values( - // - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF, getGptDraftTestsCompareModelSpec()} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .useDraftTokensExternalDecoding() - .setDraftTokens(5) - .replaceLogits() - .collectGenerationLogitsFile() - .collectContextLogitsFile(), - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF, getGptDraftTestsCompareModelSpec()} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .useDraftTokensExternalDecoding() - .setDraftTokens(5) - .useAcceptByLogits() - .replaceLogits() - .collectGenerationLogitsFile() - .collectContextLogitsFile()), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values(BeamConfig{1, {1}}), // beamConfig - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false, true), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptLogitsTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values( - // modelSpec - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .gatherLogits() - .collectGenerationLogitsFile() - .collectContextLogitsFile()), - testing::Values(TrtGptModelType::InflightBatching, TrtGptModelType::InflightFusedBatching), // modelType - testing::Values(TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, - TrtGptModelIfbTestType::RANDOM), // testType - testing::Values(BeamConfig{1, {1}}), // beamConfig - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false, true), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(true), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptLogProbsTests, ParamTest, - testing::Combine(testing::Values(gptModelParams), - testing::Values( - // modelSpec - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .returnLogProbs() - .collectCumLogProbsFile() - .collectLogProbsFile()), - testing::Values(TrtGptModelType::InflightFusedBatching), // modelType - testing::Values(TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, - TrtGptModelIfbTestType::RANDOM), // testType - testing::Values(BeamConfig{1, {1}}), // beamConfig - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptjTests, ParamTest, - testing::Combine(testing::Values(ModelParams{GPTJ_MODEL_DIR, {50256, 50256}}), - testing::Values( - // - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kCONTINUOUS) - .usePackedInput(), - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput() - - ), - testing::Values(TrtGptModelType::InflightFusedBatching), - // WAR: disable wavefront and random tests on because of switched beams - testing::Values(TrtGptModelIfbTestType::BULK - /* , TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM */), - testing::Values( - // TODO: enable more tests when mixed beam width is supported - BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} - ), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(false), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(MambaTests, ParamTest, - testing::Combine(testing::Values(ModelParams{MAMBA_MODEL_DIR, {0, 1}}), - testing::Values( - // - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kCONTINUOUS) - .usePackedInput(), - ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput() - - ), - testing::Values(TrtGptModelType::InflightBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values(BeamConfig{1, {1}}), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(false), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(RecurrentGemmaTests, ParamTest, - testing::Combine(testing::Values(ModelParams{RECURRENTGEMMA_MODEL_DIR, {0, 1}}), - testing::Values(ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput() - - ), - testing::Values(TrtGptModelType::InflightBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values(BeamConfig{1, {1}}), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(false), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(LlamaTests, ParamTest, - testing::Combine(testing::Values(ModelParams{LLAMA_MODEL_DIR, {LLAMA_END_ID, LLAMA_PAD_ID}}), - testing::Values( - // - ModelSpec{INPUT_LLAMA_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput(), - ModelSpec{INPUT_LLAMA_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .usePipelineParallelism(4), - ModelSpec{INPUT_LLAMA_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .useTensorParallelism(4), - ModelSpec{INPUT_LLAMA_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .usePipelineParallelism(2) - .useTensorParallelism(2) - - ), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values( - // TODO: enable more tests when mixed beam width is supported - BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} - ), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(false), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(ChatGlmTests, ParamTest, - testing::Combine(testing::Values(ModelParams{CHATGLM_MODEL_DIR, {130005, 3}}), - testing::Values( - // - ModelSpec{CHATGLM_INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED)), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values(BeamConfig{1, {1}}), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(false, true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -// ChatGlm0Tests is for glm-10b. -INSTANTIATE_TEST_SUITE_P(ChatGlm0Tests, ParamTest, - testing::Combine(testing::Values(ModelParams{GLM_MODEL_DIR, {50258, 50256}}), - testing::Values( - // - ModelSpec{GLM_INPUT_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED)), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values(BeamConfig{1, {1}}), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(false), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -// https://nvbugspro.nvidia.com/bug/4640177 -// WAVEFRONT and RANDOM are disabled because of the accuracy mismatch -INSTANTIATE_TEST_SUITE_P(MedusaTests, ParamTest, - testing::Combine(testing::Values(ModelParams{MEDUSA_MODEL_DIR, {2, 2}}), - testing::Values( - // - ModelSpec{INPUT_VICUNA_FILE, nvinfer1::DataType::kHALF, getMedusaTestsCompareModelSpec()} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .useMedusa()), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values(BeamConfig{1, {1}}), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(true, false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(EagleTests, ParamTest, - testing::Combine(testing::Values(ModelParams{EAGLE_MODEL_DIR, {2, 2}}), - testing::Values( - // - ModelSpec{INPUT_VICUNA_FILE, nvinfer1::DataType::kHALF, getEagleTestsCompareModelSpec()} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .useEagle()), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values(BeamConfig{1, {1}}), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(true, false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(LlamaLookaheadDecodingTests, ParamTest, - testing::Combine(testing::Values(ModelParams{LLAMA_MODEL_DIR, {LLAMA_END_ID, LLAMA_PAD_ID}}), - testing::Values( - // - ModelSpec{INPUT_LLAMA_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .useLookaheadDecoding()), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values(BeamConfig{1, {1}}), // beamConfig - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(false), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(true), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 16}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - - generateTestName); - -INSTANTIATE_TEST_SUITE_P(ExplicitDraftTokensDecodingTests, ParamTest, - testing::Combine(testing::Values(ModelParams{EXPLICIT_DRAFT_MODEL_DIR, {2, 2}}), - testing::Values( - // - ModelSpec{INPUT_VICUNA_FILE, nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .useExplicitDraftTokensDecoding() - .setMaxOutputLength(128)), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values(BeamConfig{1, {1}}), // beamConfig - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - - generateTestName); - -#ifdef ENABLE_FP8 -// Using IFB-enabled engine -INSTANTIATE_TEST_SUITE_P(GptjFP8Tests, ParamTest, - testing::Combine(testing::Values(ModelParams{GPTJ_MODEL_DIR, {50256, 50256}}), - testing::Values( - // - ModelSpec{INPUT_FILE, nvinfer1::DataType::kFP8} - .useGptAttentionPlugin() - .setKVCacheType(KVCacheType::kPAGED) - .usePackedInput() - - ), - testing::Values(TrtGptModelType::InflightFusedBatching), - testing::Values( - TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), - testing::Values( - // TODO: enable more tests when supported - BeamConfig{1, {1}} // , BeamConfig{2, {2}}, BeamConfig{2, {1, 2}} - ), - testing::Values(std::nullopt), // maxTokensInPagedKvCache - testing::Values(0.4), // freeGpuMemoryFraction - testing::Values(false), // enableTrtOverlap - testing::Values(true), // enableChunkedContext - testing::Values(false), // enableStreamingMode - testing::Values(false), // enableCudaGraphMode - testing::Values(std::nullopt), // hostCacheSize - testing::Values(false), // useRandomEndId - testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes - testing::Values(std::nullopt) // maxNumTokens - ), - generateTestName); - -#endif diff --git a/cpp/tests/e2e_tests/batch_manager/trtGptModelTest.cpp b/cpp/tests/e2e_tests/batch_manager/trtGptModelTest.cpp deleted file mode 100644 index 268e9bf9a238..000000000000 --- a/cpp/tests/e2e_tests/batch_manager/trtGptModelTest.cpp +++ /dev/null @@ -1,1328 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TOP_LEVEL_DIR -#error "Define TOP_LEVEL_DIR" -#endif - -#include "tensorrt_llm/batch_manager/trtGptModel.h" -#include "tensorrt_llm/batch_manager/kvCacheManager.h" -#include "tensorrt_llm/batch_manager/trtGptModelInflightBatching.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/gptJsonConfig.h" -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/testing/modelSpec.h" - -#include <gmock/gmock.h> -#include <gtest/gtest.h> - -#include <filesystem> -#include <memory> -#include <vector> - -using ::testing::ElementsAre; -using namespace tensorrt_llm::runtime; -namespace fs = std::filesystem; -using tensorrt_llm::testing::ModelSpec; -using tensorrt_llm::testing::KVCacheType; - -using TensorPtr = ITensor::SharedPtr; - -namespace -{ -auto const TEST_RESOURCE_PATH = fs::path{TOP_LEVEL_DIR} / "cpp/tests/resources"; -auto const ENGINE_PATH = TEST_RESOURCE_PATH / "models/rt_engine"; -auto const GPT_MODEL_PATH = ENGINE_PATH / "gpt2"; -auto const LLAMA_MODEL_PATH = ENGINE_PATH / "Llama-3.2-1B"; -} // namespace - -namespace tensorrt_llm::batch_manager -{ - -class TrtGptModelTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) -{ -protected: - TrtGptModelTest(std::filesystem::path const& modelPath) - : mModelConfig(1, 1, 1, 0, 1, 1, nvinfer1::DataType::kFLOAT) - , mModelPath(modelPath) - { - } - - TrtGptModelTest() - : TrtGptModelTest(GPT_MODEL_PATH / GetModelSpec().getModelPath() / "tp1-pp1-cp1-gpu") - { - } - - static ModelSpec& GetModelSpec() - { - static ModelSpec modelSpec{"input_tokens.npy", nvinfer1::DataType::kHALF}; - modelSpec.useGptAttentionPlugin().usePackedInput().setKVCacheType(KVCacheType::kPAGED); - return modelSpec; - } - - void SetUp() override - { - std::filesystem::path trtEnginePath = mModelPath; - - mBeamWidth = 1; - - mLogger = std::make_shared<TllmLogger>(); - - initTrtLlmPlugins(mLogger.get()); - - auto const json = GptJsonConfig::parse(trtEnginePath / "config.json"); - mModelConfig = json.getModelConfig(); - mMaxNumRequests = mModelConfig.getMaxBatchSize(); - mMaxSeqLen = mModelConfig.getMaxSequenceLen(); - mWorldConfig = WorldConfig::mpi(); - mVocabSizePadded = mModelConfig.getVocabSizePadded(mWorldConfig.getSize()); - - auto const enginePath = trtEnginePath / json.engineFilename(mWorldConfig); - auto const dtype = mModelConfig.getDataType(); - - mRawEngine.reset(new RawEngine(enginePath)); - - mSamplingConfig.temperature = std::vector{1.0f}; - mSamplingConfig.minLength = std::vector{1}; - mSamplingConfig.randomSeed = std::vector{static_cast<uint64_t>(42ul)}; - mSamplingConfig.topK = std::vector{0}; - mSamplingConfig.topP = std::vector{0.0f}; - mSamplingConfig.noRepeatNgramSize = std::vector{1 << 30}; - - mStream = std::make_unique<CudaStream>(); - mManager = std::make_unique<BufferManager>(mStream); - } - - void TearDown() override {} - - // Thin wrapper around the private TrtGptModelInflightBatching::changeBeamWidth(). - static void changeBeamWidth(std::shared_ptr<TrtGptModelInflightBatching> const& model, SizeType32 beamWidth) - { - model->changeBeamWidth(beamWidth); - } - - void forwardRequestsToCompletion( - std::shared_ptr<TrtGptModel> const& trtGptModel, RequestList& requestList, SizeType32 maxNumIterations) - { - SizeType32 numFinished = 0; - SizeType32 numIterations = 0; - while (numFinished < requestList.size() && numIterations < maxNumIterations) - { - if (numIterations > maxNumIterations) - { - FAIL() << "Iterations never finished"; - } - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - numFinished = 0; - for (auto& request : requestList) - { - if (request->isGenerationCompleteState()) - { - ++numFinished; - } - } - ++numIterations; - } - } - - int32_t mMaxNumRequests; - int32_t mMaxSeqLen; - int32_t mBeamWidth; - int32_t mVocabSizePadded; - SamplingConfig mSamplingConfig; - std::string mDataPath; - std::shared_ptr<nvinfer1::ILogger> mLogger; - ModelConfig mModelConfig; - WorldConfig mWorldConfig; - std::unique_ptr<RawEngine> mRawEngine; - std::unique_ptr<BufferManager> mManager; - BufferManager::CudaStreamPtr mStream; - std::filesystem::path mModelPath; -}; - -class TrtGptModelLoraTest : public TrtGptModelTest -{ -protected: - TrtGptModelLoraTest() - : TrtGptModelTest(GPT_MODEL_PATH / GetModelSpec().getModelPath() / "tp1-pp1-cp1-gpu") - { - } - - static ModelSpec& GetModelSpec() - { - static ModelSpec modelSpec{"input_tokens.npy", nvinfer1::DataType::kHALF}; - modelSpec.useGptAttentionPlugin().usePackedInput().setKVCacheType(KVCacheType::kPAGED).useLoraPlugin(); - return modelSpec; - } -}; - -TEST_F(TrtGptModelTest, Forward) -{ - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - int correlationId = 0; - auto maxNewTokens = 4; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - - RequestList requestList{llmRequest}; - - auto& manager = *mManager; - std::vector<int32_t> newTokensHost(mMaxNumRequests, 5); - TensorPtr const fakeNewTokens - = manager.copyFrom(newTokensHost, ITensor::makeShape({mMaxNumRequests, 1}), MemoryType::kGPU); - - std::vector<bool> finished(mMaxNumRequests, false); - - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - // Generate one token for the requests in request_table - // We need to sync with decoder - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - - EXPECT_EQ(requestList.size(), 1); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_IN_PROGRESS); - EXPECT_EQ(requestList.front()->getNumTokens(0), 5); - EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(1, 2, 3, 4, 2)); -} - -TEST_F(TrtGptModelTest, ChangeBeamWidthClearsCudaGraphCache) -{ - if (mModelConfig.getMaxBeamWidth() < 2) - { - GTEST_SKIP() << "Engine was built with max_beam_width < 2; cannot exercise changeBeamWidth()."; - } - - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - // Configure the executor for max beam width = 2 so we can transition between - // operating beam widths 1 and 2. - executorConfig.setMaxBeamWidth(2); - executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); - - auto extendedRuntimePerfKnobConfig = executor::ExtendedRuntimePerfKnobConfig{}; - extendedRuntimePerfKnobConfig.setCudaGraphMode(true); - extendedRuntimePerfKnobConfig.setCudaGraphCacheSize(8); - executorConfig.setExtendedRuntimePerfKnobConfig(extendedRuntimePerfKnobConfig); - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - // Run a single beam=1 request to completion. After at least one generation step - // the model captures and caches a CUDA graph for the subsequent batch state. - SamplingConfig samplingConfig; - samplingConfig.beamWidth = 1; - samplingConfig.temperature = std::vector{1.0f}; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); - auto llmRequest = std::make_shared<LlmRequest>( - /*requestId=*/0, /*maxNewTokens=*/4, tokens, samplingConfig, /*isStreaming=*/false); - RequestList requestList{llmRequest}; - - forwardRequestsToCompletion(trtGptModel, requestList, /*maxNumIterations=*/8); - - // Cache must have been populated by the captured generation graph(s). - EXPECT_GT(trtGptModel->numCachedCudaGraphs(), 0) - << "Expected the CUDA graph executor cache to be populated after running a " - "beam=1 request to completion."; - - // Drop the completed request before changing beam width (changeBeamWidth requires - // no in-flight requests). - requestList.clear(); - - // Switch operating beam width via the fixture's friend-access helper. - changeBeamWidth(trtGptModel, 2); - - EXPECT_EQ(trtGptModel->numCachedCudaGraphs(), 0) - << "changeBeamWidth() must invalidate the CUDA graph executor cache. Stale " - "cudaGraphExec_t instances captured against the previous decoder state " - "would otherwise be replayed against freshly allocated memory."; -} - -TEST_F(TrtGptModelLoraTest, Forward) -{ - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - int correlationId = 0; - auto maxNewTokens = 4; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - - RequestList requestList{llmRequest}; - - auto& manager = *mManager; - std::vector<int32_t> newTokensHost(mMaxNumRequests, 5); - TensorPtr const fakeNewTokens - = manager.copyFrom(newTokensHost, ITensor::makeShape({mMaxNumRequests, 1}), MemoryType::kGPU); - - std::vector<bool> finished(mMaxNumRequests, false); - - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - // Generate one token for the requests in request_table - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - - EXPECT_EQ(requestList.size(), 1); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_IN_PROGRESS); - EXPECT_EQ(requestList.front()->getNumTokens(0), 5); - EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(1, 2, 3, 4, 2)); -} - -TEST_F(TrtGptModelTest, ForwardMaxNewTokens) -{ - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig( - executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); - - executor::KvCacheConfig kvCacheConfig; - kvCacheConfig.setMaxTokens(10000); - executorConfig.setKvCacheConfig(kvCacheConfig); - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - int correlationId = 0; - auto maxNewTokens = 4; - auto tokens = std::make_shared<std::vector<int32_t>>(256); - std::iota(std::begin(*tokens), std::end(*tokens), 1); - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - - int correlationId2 = 2; - auto maxNewTokens2 = 8; - auto llmRequest2 = std::make_shared<LlmRequest>(correlationId2, maxNewTokens2, tokens, inSamplingConfig, false); - - RequestList requestList{llmRequest, llmRequest2}; - - auto& manager = *mManager; - std::vector<bool> finished(mMaxNumRequests, false); - - // Generate one token for the requests in request_table - // We call forward twice because the first call doesn't sync with decoder - SizeType32 maxNumIterations = 13; - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - - for (auto& request : requestList) - { - auto outputTokens = request->getTokens(0); - if (request->mRequestId == correlationId) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); - } - if (request->mRequestId == correlationId2) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens2); - } - } -} - -TEST_F(TrtGptModelTest, MaxNumTokensInChunked) -{ - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setEnableChunkedContext(true); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig( - executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); - - auto modelConfig = mModelConfig; - mModelConfig.setMaxNumTokens(200); - - auto trtGptModelIfb = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - std::vector<std::shared_ptr<TrtGptModel>> trtGptModels{trtGptModelIfb}; - - for (auto trtGptModel : trtGptModels) - { - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - int correlationId = 0; - auto maxNewTokens = 4; - auto tokens = std::make_shared<std::vector<int32_t>>(256); - std::iota(std::begin(*tokens), std::end(*tokens), 1); - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - - int correlationId2 = 2; - auto maxNewTokens2 = 8; - auto llmRequest2 = std::make_shared<LlmRequest>(correlationId2, maxNewTokens2, tokens, inSamplingConfig, false); - - RequestList requestList{llmRequest, llmRequest2}; - - auto& manager = *mManager; - std::vector<bool> finished(mMaxNumRequests, false); - - // Generate one token for the requests in request_table - // We call forward twice because the first call doesn't sync with decoder - SizeType32 maxNumIterations = 13; - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - - for (auto& request : requestList) - { - auto outputTokens = request->getTokens(0); - if (request->mRequestId == correlationId) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); - } - if (request->mRequestId == correlationId2) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens2); - } - } - } -} - -TEST_F(TrtGptModelTest, ForwardEndId) -{ - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig( - executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); - - executor::KvCacheConfig kvCacheConfig; - kvCacheConfig.setMaxTokens(10000); - executorConfig.setKvCacheConfig(kvCacheConfig); - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - int correlationId = 0; - auto maxNewTokens = 4; - auto endId = 107; - auto tokens = std::make_shared<std::vector<int32_t>>(256); - std::iota(std::begin(*tokens), std::end(*tokens), 1); - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, endId); - - int correlationId2 = 2; - auto maxNewTokens2 = 8; - auto llmRequest2 - = std::make_shared<LlmRequest>(correlationId2, maxNewTokens2, tokens, inSamplingConfig, false, endId); - - RequestList requestList{llmRequest, llmRequest2}; - - auto& manager = *mManager; - std::vector<bool> finished(mMaxNumRequests, false); - - // Generate one token for the requests in request_table - // We call forward twice because the first call doesn't sync with decoder - SizeType32 maxNumIterations = 13; - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - - for (auto& request : requestList) - { - auto outputTokens = request->getTokens(0); - // endId token is generated at 2nd iteration, so expect 1 output token - if (request->mRequestId == correlationId) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + 1); - } - if (request->mRequestId == correlationId2) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + 1); - } - } -} - -TEST_F(TrtGptModelTest, ForwardNoEoS) -{ - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kSTATIC_BATCH}); - - executor::KvCacheConfig kvCacheConfig; - kvCacheConfig.setMaxTokens(10000); - executorConfig.setKvCacheConfig(kvCacheConfig); - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - SamplingConfig inSamplingConfig; - inSamplingConfig.topP = {0.9}; - inSamplingConfig.temperature = {0.6}; - inSamplingConfig.minLength = {5}; - - auto tokens = std::make_shared<std::vector<int32_t>>(256); - std::iota(std::begin(*tokens), std::end(*tokens), 1); - - RequestList requestList; - for (auto requestIdx = 0; requestIdx < mMaxNumRequests; requestIdx++) - { - auto llmRequest = std::make_shared<LlmRequest>(requestIdx, 8, tokens, inSamplingConfig, false, -1); - requestList.push_back(llmRequest); - } - - auto& manager = *mManager; - std::vector<bool> finished(mMaxNumRequests, false); - - // Generate one token for the requests in request_table - // We call forward twice because the first call doesn't sync with decoder - SizeType32 maxNumIterations = 13; - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); -} - -TEST_F(TrtGptModelTest, ForwardFinished) -{ - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - int correlationId = 0; - auto maxNewTokens = 2; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{10, 9, 8, 7, 6}); - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - - RequestList requestList{llmRequest}; - - int mForwardCount = 0; - - auto& manager = *mManager; - std::vector<int32_t> newTokensHost(mMaxNumRequests, 5); - TensorPtr const fakeNewTokens - = manager.copyFrom(newTokensHost, ITensor::makeShape({mMaxNumRequests, 1}), MemoryType::kGPU); - - std::vector<int32_t> newTokensHost2(mMaxNumRequests, 4); - TensorPtr const fakeNewTokens2 - = manager.copyFrom(newTokensHost2, ITensor::makeShape({mMaxNumRequests, 1}), MemoryType::kGPU); - - // Below are only used if beam > 1 - // So we are just returning tensors with the correct shape, content is not important - std::vector<int32_t> outputIdsHost(mMaxNumRequests * (5 + 2), 5); - TensorPtr const fakeOutputIds - = manager.copyFrom(outputIdsHost, ITensor::makeShape({mMaxNumRequests, 1, 5 + 2}), MemoryType::kGPU); - - std::vector<bool> finishedFalse(mMaxNumRequests, false); - std::vector<bool> finishedTrue(mMaxNumRequests, true); - - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - // Generate one token for the requests in request_table - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - - EXPECT_EQ(requestList.size(), 1); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_IN_PROGRESS); - EXPECT_EQ(requestList.front()->getNumTokens(0), 6); - EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10)); - - // Generate one more token - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - - EXPECT_EQ(requestList.size(), 1); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); - EXPECT_EQ(requestList.front()->getNumTokens(0), 7); - EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 2); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6)); -} - -TEST_F(TrtGptModelTest, ForwardStopWords) -{ - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig( - executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); - - executor::KvCacheConfig kvCacheConfig; - kvCacheConfig.setMaxTokens(10000); - executorConfig.setKvCacheConfig(kvCacheConfig); - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - int correlationId = 0; - auto maxNewTokens = 4; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{10, 9, 8, 7, 6}); - std::optional<SizeType32> endId(std::nullopt); - std::optional<SizeType32> padId(std::nullopt); - std::optional<TensorPtr> embeddingBias(std::nullopt); - std::optional<TensorPtr> badWordsList(std::nullopt); - - auto& manager = *mManager; - // No stop words - { - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - RequestList requestList{llmRequest}; - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); - } - // With stop words - { - TensorPtr stopWordsList = manager.cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); - auto stopWordsPtr = bufferCast<int32_t>(*stopWordsList); - // make 10, 6 10 the tokens for the stop word: - stopWordsPtr[0] = 10; - stopWordsPtr[1] = 6; - stopWordsPtr[2] = 10; - stopWordsPtr[3] = 3; - stopWordsPtr[4] = -1; - stopWordsPtr[5] = -1; - - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, - endId, padId, embeddingBias, badWordsList, stopWordsList); - RequestList requestList{llmRequest}; - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10)); - } - - // With stop words - { - TensorPtr stopWordsList = manager.cpu(ITensor::makeShape({1, 2, 1}), nvinfer1::DataType::kINT32); - auto stopWordsPtr = bufferCast<int32_t>(*stopWordsList); - // make 10 is the token for the stop word: - stopWordsPtr[0] = 10; - stopWordsPtr[1] = 1; - - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, - endId, padId, embeddingBias, badWordsList, stopWordsList); - RequestList requestList{llmRequest}; - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10)); - } - - // Multiple requests, each with different stop words - { - // Request w/o stop words - auto llmRequest = std::make_shared<LlmRequest>(1, maxNewTokens, tokens, inSamplingConfig, false); - - TensorPtr stopWordsList2 = manager.cpu(ITensor::makeShape({1, 2, 1}), nvinfer1::DataType::kINT32); - { - auto stopWordsPtr = bufferCast<int32_t>(*stopWordsList2); - stopWordsPtr[0] = 10; - stopWordsPtr[1] = 1; - } - auto llmRequest2 = std::make_shared<LlmRequest>(2, maxNewTokens, tokens, inSamplingConfig, false, endId, padId, - embeddingBias, badWordsList, stopWordsList2); - - TensorPtr stopWordsList3 = manager.cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); - { - auto stopWordsPtr = bufferCast<int32_t>(*stopWordsList3); - stopWordsPtr[0] = 10; - stopWordsPtr[1] = 6; - stopWordsPtr[2] = 10; - stopWordsPtr[3] = 3; - stopWordsPtr[4] = -1; - stopWordsPtr[5] = -1; - } - auto llmRequest3 = std::make_shared<LlmRequest>(3, maxNewTokens, tokens, inSamplingConfig, false, endId, padId, - embeddingBias, badWordsList, stopWordsList3); - - RequestList requestList{llmRequest, llmRequest2, llmRequest3}; - - SizeType32 maxNumIterations(5); - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - - for (auto& request : requestList) - { - auto outputTokens = request->getTokens(0); - if (request->mRequestId == 1) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); - EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); - } - if (request->mRequestId == 2) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + 1); - EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10)); - } - if (request->mRequestId == 3) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + 3); - EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10)); - } - } - } -} - -TEST_F(TrtGptModelTest, ForwardBadWords) -{ - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig( - executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); - - executor::KvCacheConfig kvCacheConfig; - kvCacheConfig.setMaxTokens(10000); - executorConfig.setKvCacheConfig(kvCacheConfig); - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - int correlationId = 0; - auto maxNewTokens = 4; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{10, 9, 8, 7, 6}); - std::optional<SizeType32> endId(std::nullopt); - std::optional<SizeType32> padId(std::nullopt); - std::optional<TensorPtr> embeddingBias(std::nullopt); - std::optional<TensorPtr> stopWordsList(std::nullopt); - - auto& manager = *mManager; - // No bad words - { - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - RequestList requestList{llmRequest}; - - SizeType32 maxNumIterations = 5; - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); - } - // With bad words, multiple tokens - { - TensorPtr badWordsList = manager.cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); - auto badWordsPtr = bufferCast<int32_t>(*badWordsList); - // make 10, 6 10 the tokens for the bad word: - badWordsPtr[0] = 10; - badWordsPtr[1] = 6; - badWordsPtr[2] = 10; - badWordsPtr[3] = 3; - badWordsPtr[4] = -1; - badWordsPtr[5] = -1; - - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, - endId, padId, embeddingBias, badWordsList, stopWordsList); - RequestList requestList{llmRequest}; - SizeType32 maxNumIterations = 5; - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); - // Token at position 7 should be different than 10 - EXPECT_NE(requestList.front()->getTokens(0).at(7), 10); - } - - // With bad words single token - { - TensorPtr badWordsList = manager.cpu(ITensor::makeShape({1, 2, 1}), nvinfer1::DataType::kINT32); - auto badWordsPtr = bufferCast<int32_t>(*badWordsList); - // make 10 is the token for the bad word: - badWordsPtr[0] = 10; - badWordsPtr[1] = 1; - - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, - endId, padId, embeddingBias, badWordsList, stopWordsList); - RequestList requestList{llmRequest}; - SizeType32 maxNumIterations = 5; - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); - EXPECT_NE(requestList.front()->getTokens(0).at(5), 10); - } - - // Multiple requests, each with different bad words - { - // Request w/o bad words - auto llmRequest = std::make_shared<LlmRequest>(1, maxNewTokens, tokens, inSamplingConfig, false); - - TensorPtr badWordsList2 = manager.cpu(ITensor::makeShape({1, 2, 1}), nvinfer1::DataType::kINT32); - { - auto badWordsPtr = bufferCast<int32_t>(*badWordsList2); - badWordsPtr[0] = 10; - badWordsPtr[1] = 1; - } - auto llmRequest2 = std::make_shared<LlmRequest>(2, maxNewTokens, tokens, inSamplingConfig, false, endId, padId, - embeddingBias, badWordsList2, stopWordsList); - - TensorPtr badWordsList3 = manager.cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); - { - auto badWordsPtr = bufferCast<int32_t>(*badWordsList3); - badWordsPtr[0] = 10; - badWordsPtr[1] = 6; - badWordsPtr[2] = 10; - badWordsPtr[3] = 3; - badWordsPtr[4] = -1; - badWordsPtr[5] = -1; - } - auto llmRequest3 = std::make_shared<LlmRequest>(3, maxNewTokens, tokens, inSamplingConfig, false, endId, padId, - embeddingBias, badWordsList3, stopWordsList); - - RequestList requestList{llmRequest, llmRequest2, llmRequest3}; - - SizeType32 maxNumIterations(6); - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - - for (auto& request : requestList) - { - auto outputTokens = request->getTokens(0); - if (request->mRequestId == 1) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); - EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); - } - if (request->mRequestId == 2) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); - EXPECT_NE(request->getTokens(0).at(5), 10); - } - if (request->mRequestId == 3) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); - EXPECT_NE(request->getTokens(0).at(7), 10); - } - } - } -} - -TEST_F(TrtGptModelTest, ForwardEmbeddingBias) -{ - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig( - executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); - - executor::KvCacheConfig kvCacheConfig; - kvCacheConfig.setMaxTokens(10000); - executorConfig.setKvCacheConfig(kvCacheConfig); - - auto trtGptModelIfb = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - std::vector<std::shared_ptr<TrtGptModel>> trtGptModels{trtGptModelIfb}; - - for (auto& trtGptModel : trtGptModels) - { - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - int correlationId = 0; - auto maxNewTokens = 4; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{10, 9, 8, 7, 6}); - std::optional<SizeType32> endId(std::nullopt); - std::optional<SizeType32> padId(std::nullopt); - std::optional<TensorPtr> badWordsList(std::nullopt); - std::optional<TensorPtr> stopWordsList(std::nullopt); - - auto& manager = *mManager; - // No bad words - { - auto llmRequest - = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - RequestList requestList{llmRequest}; - - SizeType32 maxNumIterations = 5; - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); - } - // With embedding bias - { - TensorPtr embeddingBias - = manager.cpu(ITensor::makeShape({1, mVocabSizePadded}), nvinfer1::DataType::kFLOAT); - auto embeddingBiasPtr = bufferCast<float>(*embeddingBias); - for (SizeType32 vi = 0; vi < mVocabSizePadded; ++vi) - { - embeddingBiasPtr[vi] = 0.f; - } - // bias all words to the 10th token - embeddingBiasPtr[10] = std::numeric_limits<float>::max(); - - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, - endId, padId, embeddingBias, badWordsList, stopWordsList); - RequestList requestList{llmRequest}; - SizeType32 maxNumIterations = 5; - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); - // All tokens should become 10 after applying bias - EXPECT_EQ(requestList.front()->getTokens(0).at(5), 10); - EXPECT_EQ(requestList.front()->getTokens(0).at(6), 10); - EXPECT_EQ(requestList.front()->getTokens(0).at(7), 10); - EXPECT_EQ(requestList.front()->getTokens(0).at(8), 10); - } - - // Multiple requests, each with different bias - { - // Request w/o bias - auto llmRequest = std::make_shared<LlmRequest>(1, maxNewTokens, tokens, inSamplingConfig, false); - - TensorPtr embeddingBias1 - = manager.cpu(ITensor::makeShape({1, mVocabSizePadded}), nvinfer1::DataType::kFLOAT); - auto embeddingBias1Ptr = bufferCast<float>(*embeddingBias1); - for (SizeType32 vi = 0; vi < mVocabSizePadded; ++vi) - { - embeddingBias1Ptr[vi] = 0.f; - } - // bias all words to the 10th token - embeddingBias1Ptr[10] = std::numeric_limits<float>::max(); - - auto llmRequest2 = std::make_shared<LlmRequest>(2, maxNewTokens, tokens, inSamplingConfig, false, endId, - padId, embeddingBias1, badWordsList, stopWordsList); - - TensorPtr embeddingBias2 - = manager.cpu(ITensor::makeShape({1, mVocabSizePadded}), nvinfer1::DataType::kFLOAT); - auto embeddingBias2Ptr = bufferCast<float>(*embeddingBias2); - for (SizeType32 vi = 0; vi < mVocabSizePadded; ++vi) - { - embeddingBias2Ptr[vi] = 0.f; - } - // bias all words to the 100th token - embeddingBias2Ptr[100] = std::numeric_limits<float>::max(); - - auto llmRequest3 = std::make_shared<LlmRequest>(3, maxNewTokens, tokens, inSamplingConfig, false, endId, - padId, embeddingBias2, badWordsList, stopWordsList); - - RequestList requestList{llmRequest, llmRequest2, llmRequest3}; - - SizeType32 maxNumIterations(6); - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - - for (auto& request : requestList) - { - auto outputTokens = request->getTokens(0); - if (request->mRequestId == 1) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); - EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); - } - if (request->mRequestId == 2) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); - EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 10, 10, 10)); - } - if (request->mRequestId == 3) - { - EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); - EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 100, 100, 100, 100)); - } - } - } - } -} - -class TrtGptModelIfbHelper : public TrtGptModelInflightBatching -{ -public: - using TrtGptModelInflightBatching::TrtGptModelInflightBatching; - - [[nodiscard]] std::shared_ptr<kv_cache_manager::BaseKVCacheManager const> getKVCacheManager() const - { - return TrtGptModelInflightBatching::getKVCacheManager(); - } - - [[nodiscard]] SizeType32 getMaxAttentionWindow() const - { - return TrtGptModelInflightBatching::getMaxAttentionWindow(); - } -}; - -TEST_F(TrtGptModelTest, KVCacheReuseChunked) -{ - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setEnableChunkedContext(true); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig( - executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); - - executor::KvCacheConfig kvCacheConfig; - kvCacheConfig.setEnableBlockReuse(true); - executorConfig.setKvCacheConfig(kvCacheConfig); - - mModelConfig.setMaxNumTokens(384); - - for (int const numBlocksExpectedReused : {1, 2}) - { - auto trtGptModelIfb = std::make_shared<TrtGptModelIfbHelper>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - auto const cacheManager = trtGptModelIfb->getKVCacheManager(); - auto const tokensPerBlock = cacheManager->getTokensPerBlock(); - constexpr int numPrefillBlocks = 2; - - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - constexpr int correlationId = 0; - constexpr int maxNewTokens = 4; - - auto tokens = std::make_shared<std::vector<int32_t>>(tokensPerBlock * numPrefillBlocks); - std::iota(std::begin(*tokens), std::end(*tokens), 1); - auto subTokens = std::make_shared<std::vector<int32_t>>( - tokens->begin(), tokens->begin() + numBlocksExpectedReused * tokensPerBlock); - // Add new token to "start" a new block. - subTokens->push_back(0); - { - auto llmRequest - = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - RequestList requests{llmRequest}; - forwardRequestsToCompletion(trtGptModelIfb, requests, 6); - EXPECT_EQ(llmRequest->isGenerationCompleteState(), true); - } - for (size_t i = 1; i <= 2; ++i) - { - auto llmRequest - = std::make_shared<LlmRequest>(correlationId, maxNewTokens, subTokens, inSamplingConfig, false); - RequestList req{llmRequest}; - forwardRequestsToCompletion(trtGptModelIfb, req, 5); - EXPECT_EQ(cacheManager->getBlockManager().getNumReusedBlocks(), i * numBlocksExpectedReused); - } - } -} - -TEST_F(TrtGptModelTest, PauseRequestStats) -{ - SamplingConfig inSamplingConfig; - inSamplingConfig.temperature = std::vector{2.0f}; - int correlationId = 0; - auto maxNewTokens = 3; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); - auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, - std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, - std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, - std::nullopt, std::nullopt, std::nullopt, std::nullopt, false, false, false, std::nullopt, std::nullopt, false, - std::nullopt, false, std::nullopt, false, std::nullopt, executor::Request::kDefaultPriority, std::nullopt, - std::nullopt, std::nullopt, LlmRequestType::LLMREQUEST_TYPE_CONTEXT_AND_GENERATION, std::nullopt, 1, - std::nullopt, std::nullopt, true /* returnPerfMetrics */); - - RequestList requestList{llmRequest}; - - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - // Generate one token for the requests in request_table - // We need to sync with decoder - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - - EXPECT_EQ(requestList.size(), 1); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_IN_PROGRESS); - EXPECT_EQ(requestList.front()->getNumTokens(0), 5); - EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(1, 2, 3, 4, 2)); - - auto perfMetrics = requestList.front()->getPerfMetrics(); - auto zero = executor::RequestPerfMetrics::TimePoint{}; - - EXPECT_NE(perfMetrics.timingMetrics.arrivalTime, zero); - EXPECT_NE(perfMetrics.timingMetrics.firstScheduledTime, zero); - EXPECT_NE(perfMetrics.timingMetrics.firstTokenTime, zero); - EXPECT_EQ(perfMetrics.timingMetrics.lastTokenTime, zero); - EXPECT_EQ(perfMetrics.firstIter, 0); - EXPECT_EQ(perfMetrics.iter, 0); - EXPECT_EQ(perfMetrics.lastIter, std::nullopt); - - // Pause the request - trtGptModel->terminateRequest(llmRequest, true); - - // Resume work - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - - // Generate one more token - EXPECT_EQ(requestList.size(), 1); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_IN_PROGRESS); - EXPECT_EQ(requestList.front()->getNumTokens(0), 6); - EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(1, 2, 3, 4, 2, 4)); - - auto newPerfMetrics = requestList.front()->getPerfMetrics(); - EXPECT_EQ(newPerfMetrics.firstIter, 0); - EXPECT_EQ(newPerfMetrics.iter, 1); - EXPECT_EQ(newPerfMetrics.lastIter, std::nullopt); - - // Check that firstScheduledTime and firstTokenTime are the same - EXPECT_EQ(perfMetrics.timingMetrics.firstScheduledTime, newPerfMetrics.timingMetrics.firstScheduledTime); - EXPECT_EQ(perfMetrics.timingMetrics.firstTokenTime, newPerfMetrics.timingMetrics.firstTokenTime); - - // Pause the request - trtGptModel->terminateRequest(llmRequest, true); - - // Resume work - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - - // Generate last token - EXPECT_EQ(requestList.size(), 1); - EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); - EXPECT_EQ(requestList.front()->getNumTokens(0), 7); - EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); - EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(1, 2, 3, 4, 2, 4, 2)); - - auto endPerfMetrics = requestList.front()->getPerfMetrics(); - EXPECT_EQ(endPerfMetrics.firstIter, 0); - EXPECT_EQ(endPerfMetrics.iter, 2); - EXPECT_EQ(endPerfMetrics.lastIter, 2); - - // Check that firstScheduledTime and firstTokenTime are the same - EXPECT_EQ(perfMetrics.timingMetrics.firstScheduledTime, endPerfMetrics.timingMetrics.firstScheduledTime); - EXPECT_EQ(perfMetrics.timingMetrics.firstTokenTime, endPerfMetrics.timingMetrics.firstTokenTime); -} - -class TrtGptModelLogitsTest : public TrtGptModelTest -{ -protected: - TrtGptModelLogitsTest() - : TrtGptModelTest(GPT_MODEL_PATH / GetModelSpec().getModelPath() / "tp1-pp1-cp1-gpu") - { - } - - static ModelSpec& GetModelSpec() - { - static ModelSpec modelSpec{"input_tokens.npy", nvinfer1::DataType::kHALF}; - modelSpec.useGptAttentionPlugin().usePackedInput().setKVCacheType(KVCacheType::kPAGED).gatherLogits(); - return modelSpec; - } -}; - -TEST_F(TrtGptModelLogitsTest, ReturnContextLogitsWithChunkedContext) -{ - // General config - int correlationId = 0; - auto maxNewTokens = 4; - int const worldSize = 1; - auto const vocabSizePadded = mModelConfig.getVocabSizePadded(worldSize); - - SamplingConfig inSamplingConfig; - - // Different prompt length - for (int const promptLength : {10, 128, 200, 250, 256}) - { - RequestList finishList; - for (bool enableChunkedContext : {false, true}) - { - auto modelConfig = mModelConfig; - if (enableChunkedContext) - { - modelConfig.setMaxNumTokens(128); - } - - executor::ExecutorConfig executorConfig; - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(mBeamWidth); - executorConfig.setEnableChunkedContext(enableChunkedContext); - executorConfig.setSchedulerConfig( - executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); - - executor::KvCacheConfig kvCacheConfig; - kvCacheConfig.setEnableBlockReuse(true); - executorConfig.setKvCacheConfig(kvCacheConfig); - - auto trtGptModelIfb = std::make_shared<TrtGptModelIfbHelper>( - mLogger, modelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - // Prepare input tokens - std::vector<int32_t> input_ids; - for (int i = 1; i <= promptLength; i++) - { - input_ids.push_back(i); - } - auto tokens = std::make_shared<std::vector<int32_t>>(input_ids); - - auto llmRequest - = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - TensorPtr contextLogitsHost = BufferManager::cpu( - ITensor::makeShape({llmRequest->mPromptLen, vocabSizePadded}), nvinfer1::DataType::kFLOAT); - - llmRequest->setContextLogitsHost(contextLogitsHost); - llmRequest->setReturnContextLogits(true); - - RequestList requestList{llmRequest}; - forwardRequestsToCompletion(trtGptModelIfb, requestList, 6); - - finishList.push_back(llmRequest); - } - EXPECT_EQ(finishList.size(), 2); - - float const* const disableChunkedContextLogits - = bufferCast<float>(*(finishList.front()->getContextLogitsHost())); - float const* const enableChunkedContextLogits = bufferCast<float>(*(finishList.back()->getContextLogitsHost())); - - for (int tokenIdx = 0; tokenIdx < promptLength; tokenIdx++) - { - for (int vocabIdx = 0; vocabIdx < vocabSizePadded; vocabIdx++) - { - size_t idx = tokenIdx * vocabSizePadded + vocabIdx; - EXPECT_NEAR(disableChunkedContextLogits[idx], enableChunkedContextLogits[idx], 1e-0) - << "tokenIdx=" << tokenIdx << " vocabIdx=" << vocabIdx; - } - } - finishList.clear(); - } -} - -class LlamaModelLADTest : public TrtGptModelTest -{ -protected: - LlamaModelLADTest() - : TrtGptModelTest(LLAMA_MODEL_PATH / GetModelSpec().getModelPath() / "tp1-pp1-cp1-gpu") - { - } - - static ModelSpec& GetModelSpec() - { - static ModelSpec modelSpec = ModelSpec{"input_tokens.npy", nvinfer1::DataType::kHALF} - .useGptAttentionPlugin() - .usePackedInput() - .setKVCacheType(KVCacheType::kPAGED) - .useLookaheadDecoding(); - return modelSpec; - } -}; - -TEST_F(LlamaModelLADTest, SeamlessLookaheadDecoding) -{ - GTEST_SKIP() << "Will enable this test when we have a force LAD support."; - SizeType32 requestId = 0; - for (bool const initLADConfig : {true, false}) - { - RequestList requestList{}; - for (SizeType32 i = 0; i < 8; ++i) - { - SamplingConfig inSamplingConfig; - int correlationId = requestId; - auto maxNewTokens = 8; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); - auto llmRequest - = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - requestList.emplace_back(std::move(llmRequest)); - requestId += 1; - } - - executor::ExecutorConfig executorConfig; - executorConfig.setEnableChunkedContext(false); - executorConfig.setEnableTrtOverlap(false); - executorConfig.setMaxBeamWidth(1); - executorConfig.setSchedulerConfig( - executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); - if (initLADConfig) - { - executor::DecodingConfig decodingConfig; - decodingConfig.setLookaheadDecodingConfig(executor::LookaheadDecodingConfig(5, 5, 5)); - executorConfig.setDecodingConfig(decodingConfig); - } - - auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( - mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - - // Generate tokens for the requests in request_table - // We need to sync with decoder - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - EXPECT_EQ(trtGptModel->getSpeculativeDecodingMode().isLookaheadDecoding(), true); - - // Add new requests - for (SizeType32 i = 0; i < 4; ++i) - { - SamplingConfig inSamplingConfig; - int correlationId = requestId; - auto maxNewTokens = 8; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); - auto llmRequest - = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - requestList.emplace_back(std::move(llmRequest)); - requestId += 1; - } - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - EXPECT_EQ(trtGptModel->getSpeculativeDecodingMode().isLookaheadDecoding(), false); - - // Complete all of the requests - SizeType32 maxNumIterations = 8; - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - - // Run new requests with lookahead - requestList.clear(); - for (SizeType32 i = 0; i < 4; ++i) - { - SamplingConfig inSamplingConfig; - int correlationId = requestId; - auto maxNewTokens = 8; - auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); - auto llmRequest - = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); - requestList.emplace_back(std::move(llmRequest)); - requestId += 1; - } - trtGptModel->forwardAsync(requestList); - trtGptModel->forwardSync(); - EXPECT_EQ(trtGptModel->getSpeculativeDecodingMode().isLookaheadDecoding(), true); - forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); - requestList.clear(); - } -} - -TEST_F(TrtGptModelTest, ClampSeqLenToAttentionWindow) -{ - auto constexpr maxAttentionWindow = 65536; - auto constexpr maxSequenceLen = maxAttentionWindow + 1; - - executor::KvCacheConfig kvCacheConfig; - kvCacheConfig.setMaxAttentionWindowVec(std::vector<SizeType32>{maxAttentionWindow}); - kvCacheConfig.setFreeGpuMemoryFraction(0.0001); // minuscule amount of memory to force a clamp - - executor::ExecutorConfig executorConfig; - executorConfig.setKvCacheConfig(kvCacheConfig); - executorConfig.setMaxBeamWidth(mBeamWidth); - - auto modelConfig = mModelConfig; - modelConfig.setMaxSequenceLen(maxSequenceLen); - - auto trtGptModel = std::make_shared<TrtGptModelIfbHelper>( - mLogger, modelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); - EXPECT_LT(trtGptModel->getMaxAttentionWindow(), maxAttentionWindow); - EXPECT_EQ(trtGptModel->getMaxSequenceLen(), trtGptModel->getMaxAttentionWindow()); -} - -} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tests/e2e_tests/executor/CMakeLists.txt b/cpp/tests/e2e_tests/executor/CMakeLists.txt deleted file mode 100644 index 4813c92584fc..000000000000 --- a/cpp/tests/e2e_tests/executor/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -add_gtest(executorMockTest executorMockTest.cpp) -add_gtest(executorTest executorTest.cpp) -target_link_libraries(executorTest PRIVATE testingUtils) -add_gtest(encDecTest encDecTest.cpp) -target_link_libraries(encDecTest PRIVATE testingUtils) -add_gtest(disaggExecutorTest disaggExecutorTest.cpp) -target_link_libraries(disaggExecutorTest PRIVATE testingUtils) diff --git a/cpp/tests/e2e_tests/executor/disaggExecutor.h b/cpp/tests/e2e_tests/executor/disaggExecutor.h deleted file mode 100644 index 6b3a529ca16e..000000000000 --- a/cpp/tests/e2e_tests/executor/disaggExecutor.h +++ /dev/null @@ -1,840 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/stringUtils.h" -#include "tensorrt_llm/common/utils.h" -#include "tensorrt_llm/executor/dataTransceiverState.h" -#include "tensorrt_llm/executor/disaggServerUtil.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/requestWithId.h" -#include "tensorrt_llm/executor/serializeUtils.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" - -#include <gmock/gmock.h> -#include <gtest/gtest.h> -#include <nlohmann/json.hpp> - -#include <algorithm> -#include <chrono> -#include <cstddef> -#include <cstdint> -#include <memory> -#include <mutex> -#include <queue> -#include <string> -#include <thread> -#include <unordered_map> -#include <vector> - -using namespace tensorrt_llm::executor; -using namespace tensorrt_llm::executor::disagg_executor; -namespace su = tensorrt_llm::executor::serialize_utils; - -namespace tensorrt_llm::testing::disaggexecutor -{ - -constexpr int32_t kM_INSTANCE_ID_TAG{12024}; -constexpr int32_t kM_CONTROLLER_ID_TAG{22024}; -constexpr int32_t kM_INSTANCE_DATA_TAG{32024}; -constexpr int32_t kM_CONTROLLER_DATA_TAG{42024}; - -enum class MessageID : uint64_t -{ - PENDING_CONTEXT_REQUEST = 1, - PENDING_GENERATION_REQUEST = 2, - PENDING_FULL_REQUEST = 3, - CONTEXT_RESPONSE = 4, - GENERATION_RESPONSE = 5, - - TERMINATION = 6, -}; - -enum DisaggRole : uint32_t -{ - DISAGG_CONTEXT = 1, - DISAGG_GENERATION = 2, - DISAGG_MIXED = DISAGG_CONTEXT | DISAGG_GENERATION, - DISAGG_LEADER = 4, - DISAGG_CONTROLLER = 8, -}; - -struct RequestsData -{ - std::vector<RequestWithId> requests; -}; - -static std::vector<char> serializeResponseWithIds(std::vector<ResponseWithId> const& responseWithIds) -{ - size_t totalSize = 0; - totalSize += sizeof(size_t); - for (auto const& responseWithId : responseWithIds) - { - totalSize += su::serializedSize(responseWithId.gid); - totalSize += su::serializedSize(responseWithId.response); - } - - std::vector<char> buffer(totalSize); - std::stringbuf strbuf{std::ios_base::out | std::ios_base::in}; - strbuf.pubsetbuf(buffer.data(), static_cast<std::streamsize>(buffer.size())); - std::ostream ostream{&strbuf}; - - su::serialize(responseWithIds.size(), ostream); - for (auto const& responseWithId : responseWithIds) - { - su::serialize(responseWithId.gid, ostream); - su::serialize(responseWithId.response, ostream); - } - return buffer; -} - -static std::vector<ResponseWithId> deserializeResponseWithIds(std::vector<char>& buffer) -{ - std::vector<ResponseWithId> responseWithIds; - su::VectorWrapBuf<char> strbuf{buffer}; - std::istream istream{&strbuf}; - auto numReq = su::deserialize<std::int64_t>(istream); - for (int64_t req = 0; req < numReq; ++req) - { - auto const id = su::deserialize<std::uint64_t>(istream); - responseWithIds.emplace_back(ResponseWithId{Serialization::deserializeResponse(istream), id}); - } - return responseWithIds; -} - -struct ResponsesData -{ - std::vector<ResponseWithId> response; -}; - -using MessageData = std::variant<RequestsData, ResponsesData>; - -struct Message -{ - MessageID id; - MessageData data; -}; - -class MessageQueue -{ -public: - void push(Message&& message) - { - std::lock_guard<std::mutex> lock(mMutex); - mQueue.push(std::move(message)); - mCv.notify_one(); - } - - Message pop() - { - std::unique_lock<std::mutex> lock(mMutex); - mCv.wait(lock, [this] { return !mQueue.empty(); }); - Message message = std::move(mQueue.front()); - mQueue.pop(); - return message; - } - -private: - std::queue<Message> mQueue; - std::mutex mMutex; - std::condition_variable mCv; -}; - -class DisaggExecutorLeader -{ -public: - DisaggExecutorLeader(std::filesystem::path const& modelPath, ModelType modelType, - ExecutorConfig const& executorConfig, bool isController, bool isContext, bool isGeneration, int numRequests, - std::vector<int>& participatIds, std::vector<int> const& participantDeviceIdsThisInstance, int worldRank) - : mNumRequests(numRequests) - , mWorldRanksInstances(participatIds) - , mDeviceIdsThisInstance(participantDeviceIdsThisInstance) - , mWorldRank(worldRank) - , mShutdown(false) - , mWorldComm(tensorrt_llm::mpi::MpiComm::world()) - - { - -#if ENABLE_MULTI_DEVICE - - auto world_size = mWorldComm.getSize(); - mRolesPerRank.resize(world_size); - - if (isContext) - { - mRole |= DisaggRole::DISAGG_CONTEXT; - } - if (isGeneration) - { - mRole |= DisaggRole::DISAGG_GENERATION; - } - - if (!mWorldRanksInstances.empty() && mWorldRank == mWorldRanksInstances.front()) - { - mRole |= DisaggRole::DISAGG_LEADER; - } - - if (isController) - { - mRole |= DisaggRole::DISAGG_CONTROLLER; - } - - bool needExecutor = (std::find(mWorldRanksInstances.begin(), mWorldRanksInstances.end(), worldRank) - != mWorldRanksInstances.end()); - if (needExecutor) - { - ExecutorConfig executorConfigC = executorConfig; - - auto parallelConfig = executorConfigC.getParallelConfig().value_or(ParallelConfig{}); - std::vector<int> participantIds = mWorldRanksInstances; - - parallelConfig.setParticipantIds(participantIds); - TLLM_CHECK(parallelConfig.getCommunicationMode() == tensorrt_llm::executor::CommunicationMode::kLEADER); - parallelConfig.setCommunicationType(tensorrt_llm::executor::CommunicationType::kMPI); - parallelConfig.setDeviceIds(mDeviceIdsThisInstance); - executorConfigC.setParallelConfig(parallelConfig); - - mExecutor = std::make_unique<Executor>(modelPath, modelType, executorConfigC); - } - - TLLM_CHECK(mWorldRanksInstances.size() == mDeviceIdsThisInstance.size()); - - mWorldComm.allgather(&mRole, mRolesPerRank.data(), 1, tensorrt_llm::mpi::MpiType::kUINT32); - - generateRoles(); - - if (isController) - { - mControllerSendThread = std::thread(&DisaggExecutorLeader::ControllerSendThread, this); - mControllerRecvThread = std::thread(&DisaggExecutorLeader::ControllerRecvThread, this); - } - if (isLeaderInstance()) - { - mInstanceRecvThread = std::thread(&DisaggExecutorLeader::InstanceLeaderRecvThread, this); - mInstanceSendThread = std::thread(&DisaggExecutorLeader::InstanceLeaderSendThread, this); - mInstanceLoopThread = std::thread(&DisaggExecutorLeader::InstanceLeaderLoopThread, this); - } -#else - TLLM_THROW("DisaggExecutor only support being compiled with ENABLE_MULTI_DEVICE"); - -#endif - } - - bool isControllerRank() const - { - return mRole & DISAGG_CONTROLLER; - } - - bool isContextRank() const - { - return mRole & DISAGG_CONTEXT; - } - - bool isGenerationRank() const - { - return mRole & DISAGG_GENERATION; - } - - bool isLeaderInstance() const - { - return mRole & DISAGG_LEADER; - } - - std::vector<IdType> enqueueRequests(std::vector<Request> const& llmRequests) - - { - if (!isControllerRank()) - { - return {}; - } - - std::vector<RequestWithId> requestWithIds; - std::vector<RequestWithId> requestWithIdsFull; // full request, not disaggregated - std::vector<IdType> reqIds; - for (auto const& req : llmRequests) - { - IdType id = generatedControlId(); - reqIds.push_back(id); - - RequestWithId reqWithId{req, id}; - if (req.getRequestType() == RequestType::REQUEST_TYPE_CONTEXT_ONLY) - { - requestWithIds.push_back(std::move(reqWithId)); - } - else - { - TLLM_CHECK(req.getRequestType() == RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION); - requestWithIdsFull.push_back(std::move(reqWithId)); - } - - mRequestMap.insert(std::make_pair(id, req)); - } - - if (!requestWithIds.empty()) - { - Message message{MessageID::PENDING_CONTEXT_REQUEST, MessageData{RequestsData{requestWithIds}}}; - mControllerSendQueue.push(std::move(message)); - } - if (!requestWithIdsFull.empty()) - { - Message message{MessageID::PENDING_FULL_REQUEST, MessageData{RequestsData{requestWithIdsFull}}}; - mControllerSendQueue.push(std::move(message)); - } - - return reqIds; - } - - std::vector<Response> awaitResponses(std::optional<std::chrono::milliseconds> const& timeout) - { - // wait for responseQueue , modify reqid- - std::vector<Response> responses; - std::unique_lock<std::mutex> lck(mResponsesMtx); - auto pred = [&mShutdown = mShutdown, &resp = this->mResponses]() -> bool { return !resp.empty() || mShutdown; }; - auto storeResponses = [this, &resp = this->mResponses, &responses]() - { - for (auto it = resp.cbegin(); it != resp.cend();) - { - responses.insert(responses.end(), it->second.begin(), it->second.end()); - resp.erase(it++); - } - }; - - if (timeout) - { - if (mResponsesCv.wait_for(lck, timeout.value(), pred)) - { - storeResponses(); - } - } - else - { - mResponsesCv.wait(lck, pred); - storeResponses(); - } - return responses; - } - - std::deque<RequestStatsPerIteration> getLatestRequestStats() - { - if (mExecutor && mExecutor->canEnqueueRequests()) - { - return mExecutor->getLatestRequestStats(); - } - return {}; - } - - void shutDown() - { - if (mShutdown) - { - return; - } - - if (isControllerRank()) - { - std::call_once(mHasSendTerminFlag, - [&]() - { - MessageID terminationMessage = MessageID::TERMINATION; - std::vector<bool> isSend(mWorldComm.getSize(), false); - for (auto&& leaderRanks : {mContextLeaderRanks, mGenerationLeaderRanks}) - { - for (auto&& leaderRank : leaderRanks) - { - if (isSend[leaderRank]) - { - continue; - } - mWorldComm.sendRawTag(&terminationMessage, 1, tensorrt_llm::mpi::MpiType::kUINT64, - leaderRank, kM_CONTROLLER_ID_TAG); - isSend[leaderRank] = true; - } - } - - mWorldComm.sendRawTag(&terminationMessage, 1, tensorrt_llm::mpi::MpiType::kUINT64, mControllerRank, - kM_INSTANCE_ID_TAG); - }); - // end recv thread; - } - mShutdown = true; - - // end send thread - if (isControllerRank()) - { - mControllerSendQueue.push({MessageID::TERMINATION, {}}); - } - mInstanceSendQueue.push({MessageID::TERMINATION, {}}); - } - - ~DisaggExecutorLeader() - { - - if (isControllerRank()) - { - shutDown(); - } - - if (isLeaderInstance()) - { - if (mInstanceSendThread.joinable()) - { - mInstanceSendThread.join(); - } - if (mInstanceRecvThread.joinable()) - { - mInstanceRecvThread.join(); - } - if (mInstanceLoopThread.joinable()) - { - mInstanceLoopThread.join(); - } - } - - if (isControllerRank()) - { - if (mControllerSendThread.joinable()) - { - mControllerSendThread.join(); - } - if (mControllerRecvThread.joinable()) - { - mControllerRecvThread.join(); - } - } - - if (!isControllerRank()) - { - mExecutor->shutdown(); - } - if (isControllerRank() && isLeaderInstance()) - { - mExecutor->shutdown(); - } - - shutDown(); - } - -private: - tensorrt_llm::mpi::MpiComm const& mWorldComm; - std::unique_ptr<Executor> mExecutor; - std::thread mInstanceSendThread; - std::thread mInstanceRecvThread; - std::thread mInstanceLoopThread; - std::thread mControllerSendThread; - std::thread mControllerRecvThread; - int mNumRequests; - std::map<std::uint64_t, Request> mRequestMap; - std::map<IdType, DataTransceiverState> mGenIdToContextPhase; - std::unordered_map<IdType, IdType> mInstanceIdToGlobalId; - std::mutex mIdToGlbalMutex; - - std::vector<int> mWorldRanksInstances; - - int mWorldRank; - int mControllerRank = 0; - uint32_t mRole = 0; - std::vector<uint32_t> mRolesPerRank; - std::vector<int> mContextLeaderRanks; - std::vector<int> mGenerationLeaderRanks; - - IdType mLastId = 1; - MessageQueue mControllerSendQueue; - MessageQueue mInstanceSendQueue; - - std::atomic<bool> mShutdown; - - // Ready responses - std::unordered_map<IdType, std::vector<Response>> mResponses; - mutable std::mutex mResponsesMtx; - std::condition_variable mResponsesCv; - - std::vector<int> mDeviceIdsThisInstance; - std::once_flag mHasSendTerminFlag; - - void appendNewResponses(std::vector<ResponseWithId>& newResponses) - { - { - std::scoped_lock<std::mutex> lck(mResponsesMtx); - for (auto& responseWithId : newResponses) - { - // global id to Result - responseWithId.response = Response(responseWithId.gid, responseWithId.response.getResult()); - - mResponses[responseWithId.gid].emplace_back(responseWithId.response); - } - } - mResponsesCv.notify_all(); - } - - void generateRoles() - { - int contextNum = 0; - int genrationNum = 0; - int controllerNum = 0; - for (int rank = 0; rank < mRolesPerRank.size(); rank++) - { - uint32_t role = mRolesPerRank[rank]; - if (role & DISAGG_LEADER) - { - if (role & DISAGG_CONTEXT) - { - contextNum++; - mContextLeaderRanks.push_back(rank); - } - if (role & DISAGG_GENERATION) - { - genrationNum++; - mGenerationLeaderRanks.push_back(rank); - } - } - if (role & DISAGG_CONTROLLER) - { - controllerNum++; - mControllerRank = rank; - } - } - TLLM_CHECK_WITH_INFO(controllerNum == 1, "only one rank is controller but get %d controllerNum", controllerNum); - TLLM_LOG_INFO("leader ctx: %s, gen: %s", common::vec2str(mContextLeaderRanks).c_str(), - common::vec2str(mGenerationLeaderRanks).c_str()); - } - - IdType generatedControlId() - { - return (mLastId++ % UINT64_MAX); - } - - int selectContextLeaderRank() - { - static int leaderRank = 0; - leaderRank = (leaderRank + 1) % mContextLeaderRanks.size(); - return mContextLeaderRanks[leaderRank]; - } - - int selectGenerationLeaderRank() - { - - // TODO: for same reqId , need select specific generationLeader - static int leaderRank = 0; - leaderRank = (leaderRank + 1) % mGenerationLeaderRanks.size(); - return mGenerationLeaderRanks[leaderRank]; - } - - void ControllerSendThread() - { - // send request to context reqid - // and send context pahse to generation - - TLLM_CUDA_CHECK( - cudaSetDevice(mDeviceIdsThisInstance.at(COMM_SESSION.getRank() % (mDeviceIdsThisInstance.size())))); - tensorrt_llm::common::setThreadName("ControllerSendThread"); - - while (!mShutdown) - { - auto message = mControllerSendQueue.pop(); - if (message.id == MessageID::TERMINATION) - { - - TLLM_LOG_DEBUG("controller get termination message in sendQueue"); - break; - } - if (message.id == MessageID::PENDING_CONTEXT_REQUEST) - { - - auto& reqWithIds = std::get<RequestsData>(message.data); - auto packed = RequestWithId::serializeReqWithIds(reqWithIds.requests); - int contextRank = selectContextLeaderRank(); - - mWorldComm.sendRawTag( - &message.id, 1, tensorrt_llm::mpi::MpiType::kUINT64, contextRank, kM_CONTROLLER_ID_TAG); - - mWorldComm.sendRawTag(packed.data(), packed.size(), tensorrt_llm::mpi::MpiType::kCHAR, contextRank, - kM_CONTROLLER_DATA_TAG); - } - else if (message.id == MessageID::PENDING_GENERATION_REQUEST - || message.id == MessageID::PENDING_FULL_REQUEST) - { - - auto& reqWithIds = std::get<RequestsData>(message.data); - auto packed = RequestWithId::serializeReqWithIds(reqWithIds.requests); - int generationRank = selectGenerationLeaderRank(); - - mWorldComm.sendRawTag( - &message.id, 1, tensorrt_llm::mpi::MpiType::kUINT64, generationRank, kM_CONTROLLER_ID_TAG); - - mWorldComm.sendRawTag(packed.data(), packed.size(), tensorrt_llm::mpi::MpiType::kCHAR, generationRank, - kM_CONTROLLER_DATA_TAG); - } - else - { - TLLM_THROW("rank:%d, size:%d controller send Invalid message id:%ld", mWorldComm.getRank(), - mWorldComm.getSize(), static_cast<uint64_t>(message.id)); - } - } - } - - void ControllerRecvThread() - { -#if ENABLE_MULTI_DEVICE - tensorrt_llm::common::setThreadName("ControllerRecvThread"); - - // recv response from context and push to sendQueue - // recv response from generation and push to responseQueue and notify awaitResponse - TLLM_CUDA_CHECK( - cudaSetDevice(mDeviceIdsThisInstance.at(COMM_SESSION.getRank() % (mDeviceIdsThisInstance.size())))); - - while (!mShutdown) - { - - MPI_Message msg = nullptr; - MPI_Status status; - - mWorldComm.mprobeRawTag(MPI_ANY_SOURCE, kM_INSTANCE_ID_TAG, &msg, &status); - - auto sourceRank{status.MPI_SOURCE}; - int32_t count = 0; - MPICHECK(MPI_Get_count(&status, MPI_UINT64_T, &count)); - TLLM_CHECK(count == 1); - - MessageID messageId; - MPICHECK(MPI_Mrecv(&messageId, count, MPI_UINT64_T, &msg, &status)); - - if (messageId == MessageID::TERMINATION) - { - TLLM_LOG_DEBUG("controller received termination message***************\n"); - break; - } - if (messageId == MessageID::CONTEXT_RESPONSE) - { - mWorldComm.mprobeRawTag(sourceRank, kM_INSTANCE_DATA_TAG, &msg, &status); - MPICHECK(MPI_Get_count(&status, MPI_CHAR, &count)); - std::vector<char> buffer(count); - MPICHECK(MPI_Mrecv(buffer.data(), count, MPI_CHAR, &msg, &status)); - auto responseWithIds = deserializeResponseWithIds(buffer); - // enqueueTo sendQueue like enqueuRequest. . modify requestType and set ContextPhaseParams - // and push to sendQueue. - std::vector<RequestWithId> requestWithIds; - for (auto&& responseWithId : responseWithIds) - { - auto reqId = responseWithId.gid; - auto& request = mRequestMap.at(reqId); - - request.setRequestType(RequestType::REQUEST_TYPE_GENERATION_ONLY); - request.setContextPhaseParams(responseWithId.response.getResult().contextPhaseParams.value()); - requestWithIds.push_back(RequestWithId{request, reqId}); - } - mControllerSendQueue.push({MessageID::PENDING_GENERATION_REQUEST, RequestsData{requestWithIds}}); - } - - else if (messageId == MessageID::GENERATION_RESPONSE) - { - - mWorldComm.mprobeRawTag(sourceRank, kM_INSTANCE_DATA_TAG, &msg, &status); - MPICHECK(MPI_Get_count(&status, MPI_CHAR, &count)); - std::vector<char> buffer(count); - MPICHECK(MPI_Mrecv(buffer.data(), count, MPI_CHAR, &msg, &status)); - - auto responseWithIds = deserializeResponseWithIds(buffer); - appendNewResponses(responseWithIds); - } - else - { - TLLM_THROW("rank:%d, size:%d controller recv Invalid message id:%ld", mWorldComm.getRank(), - mWorldComm.getSize(), static_cast<uint64_t>(messageId)); - } - } -#endif - } - - void InstanceLeaderSendThread() - { - tensorrt_llm::common::setThreadName("InstanceLeaderSendThread"); - - TLLM_CUDA_CHECK( - cudaSetDevice(mDeviceIdsThisInstance.at(COMM_SESSION.getRank() % (mDeviceIdsThisInstance.size())))); - - // pop senQueue and send response to controller - - while (!mShutdown) - { - auto message = mInstanceSendQueue.pop(); - if (message.id == MessageID::CONTEXT_RESPONSE || message.id == MessageID::GENERATION_RESPONSE) - { - auto& responseWithIds = std::get<ResponsesData>(message.data); - auto packed = serializeResponseWithIds(responseWithIds.response); - - mWorldComm.sendRawTag( - &message.id, 1, tensorrt_llm::mpi::MpiType::kUINT64, mControllerRank, kM_INSTANCE_ID_TAG); - mWorldComm.sendRawTag(packed.data(), packed.size(), tensorrt_llm::mpi::MpiType::kCHAR, mControllerRank, - kM_INSTANCE_DATA_TAG); - } - else if (message.id == MessageID::TERMINATION) - { - // break; no send - TLLM_LOG_DEBUG( - "ranK:%d ,size:%d ,isContext:%d... Context or Generation leader get termination message in " - "sendQueue***************\n", - mWorldComm.getRank(), mWorldComm.getSize(), int(isContextRank())); - break; - } - else - { - TLLM_THROW("rank:%d, size:%d InstanceLeaderSendThread send Invalid message id:%ld", - mWorldComm.getRank(), mWorldComm.getSize(), static_cast<uint64_t>(message.id)); - } - } - } - - void InstanceLeaderRecvThread() - { - -#if ENABLE_MULTI_DEVICE - tensorrt_llm::common::setThreadName("InstanceLeaderRecvThread"); - - TLLM_CUDA_CHECK( - cudaSetDevice(mDeviceIdsThisInstance.at(COMM_SESSION.getRank() % (mDeviceIdsThisInstance.size())))); - - // recv request from controller and enqueRequest to executor - while (!mShutdown) - { - MPI_Message msg; - MPI_Status status; - auto sourceRank{mControllerRank}; - mWorldComm.mprobeRawTag(sourceRank, kM_CONTROLLER_ID_TAG, &msg, &status); - - int32_t count; - MPICHECK(MPI_Get_count(&status, MPI_UINT64_T, &count)); - TLLM_CHECK(count == 1); - - MessageID messageId; - MPICHECK(MPI_Mrecv(&messageId, count, MPI_UINT64_T, &msg, &status)); - - if (messageId == MessageID::TERMINATION) - { - TLLM_LOG_DEBUG( - "ranK:%d ,size:%d ,isContext:%d ... Context or Generation leader recv termination message in " - "InstanceLeaderRecvThread***************\n", - mWorldComm.getRank(), mWorldComm.getSize(), int(isContextRank())); - shutDown(); - break; - } - if (messageId == MessageID::PENDING_CONTEXT_REQUEST || messageId == MessageID::PENDING_GENERATION_REQUEST - || messageId == MessageID::PENDING_FULL_REQUEST) - { - mWorldComm.mprobeRawTag(sourceRank, kM_CONTROLLER_DATA_TAG, &msg, &status); - MPICHECK(MPI_Get_count(&status, MPI_CHAR, &count)); - std::vector<char> buffer(count); - MPICHECK(MPI_Mrecv(buffer.data(), count, MPI_CHAR, &msg, &status)); - auto requestWithIds = RequestWithId::deserializeReqWithIds(buffer); - for (auto&& requestWithId : requestWithIds) - { - - auto globalReqId = requestWithId.id; - if (isContextRank() && messageId == MessageID::PENDING_CONTEXT_REQUEST) - { - TLLM_CHECK(requestWithId.req.getRequestType() == RequestType::REQUEST_TYPE_CONTEXT_ONLY); - } - else if (isGenerationRank() - && (messageId == MessageID::PENDING_GENERATION_REQUEST - || messageId == MessageID::PENDING_FULL_REQUEST)) - { - if (messageId == MessageID::PENDING_GENERATION_REQUEST) - { - TLLM_CHECK(requestWithId.req.getRequestType() == RequestType::REQUEST_TYPE_GENERATION_ONLY); - } - else // PENDING_FULL_REQUEST - { - TLLM_CHECK( - requestWithId.req.getRequestType() == RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION); - } - } - else - { - TLLM_THROW("rank:%d, size:%d InstanceLeaderRecvThread recv Invalid message id:%ld", - mWorldComm.getRank(), mWorldComm.getSize(), static_cast<uint64_t>(messageId)); - } - auto reqId = mExecutor->enqueueRequest(requestWithId.req); - { - std::scoped_lock<std::mutex> lock{mIdToGlbalMutex}; - mInstanceIdToGlobalId[reqId] = globalReqId; - } - } - } - else - { - TLLM_THROW("rank:%d, size:%d InstanceLeaderRecvThread send Invalid message id:%ld", - mWorldComm.getRank(), mWorldComm.getSize(), static_cast<uint64_t>(messageId)); - } - } -#endif - } - - void InstanceLeaderLoopThread() - { - - tensorrt_llm::common::setThreadName("InstanceLeaderLoopThread"); - - TLLM_CUDA_CHECK( - cudaSetDevice(mDeviceIdsThisInstance.at(COMM_SESSION.getRank() % (mDeviceIdsThisInstance.size())))); - - // loop awaitResponse and enqueue into sendQueue - while (!mShutdown) - { - std::chrono::milliseconds waitTime(1); - - auto responses = mExecutor->awaitResponses(waitTime); - if (responses.empty()) - { - continue; - } - std::vector<ResponseWithId> responseWithIdsContext; - std::vector<ResponseWithId> responseWithIdsGeneration; - for (auto&& response : responses) - { - auto reqId = response.getRequestId(); - IdType globalId{0}; - { - std::scoped_lock<std::mutex> lock{mIdToGlbalMutex}; - globalId = mInstanceIdToGlobalId[reqId]; - } - TLLM_CHECK(globalId != 0); - auto const& result = response.getResult(); - if (result.contextPhaseParams.has_value()) - { - responseWithIdsContext.emplace_back(response, globalId); - } - else - { - responseWithIdsGeneration.emplace_back(response, globalId); - } - } - - if (isContextRank()) - { - mInstanceSendQueue.push({MessageID::CONTEXT_RESPONSE, ResponsesData{responseWithIdsContext}}); - } - if (isGenerationRank()) - { - mInstanceSendQueue.push({MessageID::GENERATION_RESPONSE, ResponsesData{responseWithIdsGeneration}}); - } - } - } -}; -} // namespace tensorrt_llm::testing::disaggexecutor diff --git a/cpp/tests/e2e_tests/executor/disaggExecutorTest.cpp b/cpp/tests/e2e_tests/executor/disaggExecutorTest.cpp deleted file mode 100644 index 0eb05d2cc807..000000000000 --- a/cpp/tests/e2e_tests/executor/disaggExecutorTest.cpp +++ /dev/null @@ -1,1437 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "disaggExecutor.h" -#include "executorTest.h" -#include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/runtime/utils/numpyUtils.h" -#include "tests/utils/common.h" - -#include <cstddef> -#include <unordered_set> - -namespace tr = tensorrt_llm::runtime; - -using namespace tensorrt_llm::testing; - -namespace -{ -auto constexpr LLAMA_INPUT_FILE = "input_tokens_llama.npy"; -auto constexpr LLAMA_VOCAB_SIZE_PADDED = 128256; -auto constexpr LLAMA_END_ID = 128001; -auto constexpr LLAMA_PAD_ID = 128001; - -using CondDisaggParamsType = std::tuple<std::string>; // modelName - -enum class InstanceRole : int -{ - kCONTEXT = 1, - kGENERATION = 0, - kMIXED = 2 -}; - -using DisaggParamsType = std::tuple< // - int, // processNum - std::vector<std::string>, // modelNames - std::vector<std::vector<int>>, // participantIdsEachInstance - std::vector<std::vector<int>>, // participantDeviceIdsEachInstance - std::vector<InstanceRole>, // instanceRoles - int // controllerRank - >; - -std::string convertToString(std::vector<std::vector<int>> const& vec) -{ - std::ostringstream oss; - oss << "XX"; - - for (size_t i = 0; i < vec.size(); ++i) - { - for (size_t j = 0; j < vec[i].size(); ++j) - { - oss << vec[i][j]; - if (j < vec[i].size() - 1) - { - oss << "_"; - } - } - if (i < vec.size() - 1) - { - oss << "X_X"; - } - } - - oss << "XX"; - return oss.str(); -}; - -std::string convertToString(std::vector<InstanceRole> const& vec) -{ - std::ostringstream oss; - oss << "XX"; - - for (size_t j = 0; j < vec.size(); ++j) - { - oss << static_cast<int>(vec[j]); - if (j < vec.size() - 1) - { - oss << "_"; - } - } - - oss << "XX"; - return oss.str(); -}; - -std::string generateTestNameDisaggParams(testing::TestParamInfo<DisaggParamsType> const& info) -{ - auto const processNum = std::get<0>(info.param); - auto const modelNames = std::get<1>(info.param); - auto const participantIdsEachInstance = std::get<2>(info.param); // std::vector<std::vector<int>> - auto const participantDeviceIdsEachInstance = std::get<3>(info.param); // std::vector<std::vector<int>>; - auto const instanceRoles = std::get<4>(info.param); // std::vector<int> ; //1 is context , 0 is generation - auto const controllerRank = std::get<5>(info.param); - - std::string name = "DisaggExecutorTest_"; - - name.append("ProcessNum_" + std::to_string(processNum)); - // name.append("_contextModel_" + contextModel + "_genModel_" + genModel); - name.append("_modelNames_"); - for (auto&& modelName : modelNames) - { - name.append(modelName).append("_"); - } - - name.append("_controllerRank_" + std::to_string(controllerRank)); - - name.append("_ranks_").append(convertToString(participantIdsEachInstance)); - name.append("_devices_").append(convertToString(participantDeviceIdsEachInstance)); - name.append("_roles_").append(convertToString(instanceRoles)); - name.append("_controllerRank_" + std::to_string(controllerRank)); - - return name; -} - -std::string generateTestNameCondDisaggParams(testing::TestParamInfo<CondDisaggParamsType> const& info) -{ - auto const modelName = std::get<0>(info.param); - return "Model_" + modelName; -} - -class DisaggParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<DisaggParamsType> -{ -}; - -class DisaggOrchestratorParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<DisaggParamsType> -{ -}; - -class ConditionalDisaggParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<CondDisaggParamsType> -{ -}; - -void verifyGenerateDistStats(std::deque<RequestStatsPerIteration> const& iterationStats) -{ - for (auto const& iteration : iterationStats) - { - for (auto const& requestStats : iteration.requestStats) - { - // exclude context only requests for mixed server - if (requestStats.stage == RequestStage::kGENERATION_COMPLETE && requestStats.numGeneratedTokens > 1) - { - EXPECT_TRUE(requestStats.disServingStats.has_value()); - EXPECT_GT(requestStats.disServingStats.value().kvCacheTransferMS, 0.0); - } - if (requestStats.stage != RequestStage::kQUEUED) - { - EXPECT_TRUE(requestStats.disServingStats.has_value()); - } - else - { - EXPECT_FALSE(requestStats.disServingStats.has_value()); - } - } - } -} -} // namespace - -void runDisaggTest(tensorrt_llm::testing::disaggexecutor::DisaggExecutorLeader& executor, - tensorrt_llm::runtime::BufferManager& manager, ITensor const& givenInput, ModelIds const& modelIds, - FlakyTestInfo const& flakyTestInfo, bool streaming, SizeType32 const vocabSizePadded, BeamResult const& beamResult, - OutputConfig const& outConfig, bool isSpeculativeDecoding, int maxWaitMs, BatchingType batchingType, - bool returnAllGeneratedTokens) -{ - - auto& comm = tensorrt_llm::mpi::MpiComm::world(); - auto const worldRank = comm.getRank(); - auto const worldSize = comm.getSize(); - auto const beamWidth = beamResult.beamWidth; - - std::unordered_map<IdType, SizeType32> reqIdToBatchId; - std::unordered_map<SizeType32, std::vector<BeamTokens>> tokens; - auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(givenInput, modelIds.padId); - auto const* const givenInputData = tr::bufferCast<TokenIdType const>(givenInput); - - auto const& inputShape = givenInput.getShape(); - ASSERT_EQ(inputShape.nbDims, 2); - ASSERT_GT(inputShape.d[0], 0); - - // Load expected outputs for each beam width value - auto testData = TestData::loadTestData(beamResult, givenInput, beamWidth, manager, outConfig, modelIds); - auto const maxSeqLen = testData.maxSeqLen; - - // Load expected outputs and inputs - SizeType32 numRequests = static_cast<SizeType32>(givenInputLengths.size()); - SizeType32 maxRequests = numRequests; - std::vector<Request> requests; - std::vector<SizeType32> reqMaxNewTokens; - SizeType32 const numReturnSequences = 1; - - for (SizeType32 req = 0; req < maxRequests; ++req) - { - SizeType32 inputLen = givenInputLengths.at(req); - auto maxNewTokens = maxSeqLen - maxInputLength; - reqMaxNewTokens.push_back(maxNewTokens); - SizeType32 endId = -1; - auto const* const seqBegin = givenInputData + req * maxInputLength; - VecTokens tokens(seqBegin, seqBegin + inputLen); - auto samplingConfig = tensorrt_llm::executor::SamplingConfig(beamWidth); - samplingConfig.setNumReturnSequences(numReturnSequences); - auto request = Request( - VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, samplingConfig, outConfig, endId); - request.setReturnAllGeneratedTokens(returnAllGeneratedTokens); - request.setRequestType(RequestType::REQUEST_TYPE_CONTEXT_ONLY); - requests.emplace_back(std::move(request)); - } - - if (executor.isControllerRank()) - { - std::vector<IdType> reqIds; - - for (int i = 0; i < requests.size(); ++i) - { - std::vector<BeamTokens> resultTokens; - resultTokens.reserve(numReturnSequences); - for (SizeType32 seqIdx = 0; seqIdx < numReturnSequences; ++seqIdx) - { - resultTokens.emplace_back(beamWidth); - } - auto retReqId = executor.enqueueRequests({requests[i]}); - reqIds.push_back(retReqId.front()); - tokens[i] = std::move(resultTokens); - reqIdToBatchId[retReqId.front()] = i; - } - - // Get the new tokens for each requests - int32_t numFinished = 0; - int iter = 0; - SizeType32 numResponses = 0; - while (numFinished < maxRequests && iter < maxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - numResponses++; - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - auto batchId = reqIdToBatchId.at(response.getRequestId()); - auto seqIdx = result.sequenceIndex; - - auto& contextLogits = result.contextLogits; - auto& genLogits = result.generationLogits; - auto& outputTokenIds = result.outputTokenIds; - - EXPECT_EQ(result.finishReasons.size(), beamWidth); - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - auto& newTokens = outputTokenIds.at(beam); - auto& reqTokens = tokens.at(batchId).at(seqIdx).at(beam); - - reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); - // FinishReason is only supported for bw=1 and inflight batching. - if (beamWidth == 1 && batchingType == BatchingType::kINFLIGHT) - { - EXPECT_EQ(result.finishReasons.at(beam), - result.isFinal ? FinishReason::kLENGTH : FinishReason::kNOT_FINISHED); - } - } - - auto& cumLogProbs = result.cumLogProbs; - auto& logProbs = result.logProbs; - auto& beamTokens = tokens.at(batchId).at(seqIdx); - testData.verifyLogProbs(outConfig.returnLogProbs, streaming, outConfig.excludeInputFromOutput, - givenInputLengths.at(batchId), beamWidth, beamTokens, cumLogProbs, logProbs, batchId, - flakyTestInfo); - - testData.validateContextLogits(outConfig.returnContextLogits, givenInputLengths.at(batchId), - beamWidth, contextLogits, vocabSizePadded, batchId); - testData.validateGenerationLogits(outConfig.returnGenerationLogits, result.isFinal, streaming, - outConfig.excludeInputFromOutput, givenInputLengths.at(batchId), reqMaxNewTokens.at(batchId), - beamWidth, beamTokens, genLogits, vocabSizePadded, batchId, returnAllGeneratedTokens); - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - } - ++iter; - } - EXPECT_LT(iter, maxWaitMs); - testData.verifyOutput(tokens, givenInputLengths, streaming, outConfig.excludeInputFromOutput, flakyTestInfo, - isSpeculativeDecoding, beamWidth, numReturnSequences, false); - } - comm.barrier(); - if (executor.isGenerationRank()) - { - verifyGenerateDistStats(executor.getLatestRequestStats()); - } -} - -void runDisaggTest(DisaggExecutorOrchestrator& executor, tensorrt_llm::runtime::BufferManager& manager, - ITensor const& givenInput, ModelIds const& modelIds, FlakyTestInfo const& flakyTestInfo, bool streaming, - SizeType32 const vocabSizePadded, BeamResult const& beamResult, OutputConfig const& outConfig, - bool isSpeculativeDecoding, int maxWaitMs, BatchingType batchingType, bool returnAllGeneratedTokens) -{ - - auto& comm = tensorrt_llm::mpi::MpiComm::world(); - auto const worldRank = comm.getRank(); - auto const worldSize = comm.getSize(); - auto const beamWidth = beamResult.beamWidth; - - std::unordered_map<IdType, SizeType32> reqIdToBatchId; - std::unordered_map<SizeType32, std::vector<BeamTokens>> tokens; - // std::unordered_map<IdType, IdType> gGenIdIdTogContextId; - auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(givenInput, modelIds.padId); - auto const* const givenInputData = tr::bufferCast<TokenIdType const>(givenInput); - - auto const& inputShape = givenInput.getShape(); - ASSERT_EQ(inputShape.nbDims, 2); - ASSERT_GT(inputShape.d[0], 0); - - // Load expected outputs for each beam width value - auto testData = TestData::loadTestData(beamResult, givenInput, beamWidth, manager, outConfig, modelIds); - auto const maxSeqLen = testData.maxSeqLen; - - // Load expected outputs and inputs - SizeType32 numRequests = static_cast<SizeType32>(givenInputLengths.size()); - SizeType32 maxRequests = numRequests; - std::vector<Request> requests; - std::vector<SizeType32> reqMaxNewTokens; - SizeType32 const numReturnSequences = 1; - - for (SizeType32 req = 0; req < maxRequests; ++req) - { - SizeType32 inputLen = givenInputLengths.at(req); - auto maxNewTokens = maxSeqLen - maxInputLength; - reqMaxNewTokens.push_back(maxNewTokens); - SizeType32 endId = -1; - auto const* const seqBegin = givenInputData + req * maxInputLength; - VecTokens tokens(seqBegin, seqBegin + inputLen); - auto samplingConfig = tensorrt_llm::executor::SamplingConfig(beamWidth); - samplingConfig.setNumReturnSequences(numReturnSequences); - auto request = Request( - VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, samplingConfig, outConfig, endId); - request.setReturnAllGeneratedTokens(returnAllGeneratedTokens); - request.setRequestType(RequestType::REQUEST_TYPE_CONTEXT_ONLY); - requests.emplace_back(std::move(request)); - } - - if (worldRank == 0) - { - std::vector<IdType> reqIds; - - for (int i = 0; i < requests.size(); ++i) - { - std::vector<BeamTokens> resultTokens; - resultTokens.reserve(numReturnSequences); - for (SizeType32 seqIdx = 0; seqIdx < numReturnSequences; ++seqIdx) - { - resultTokens.emplace_back(beamWidth); - } - auto retReqId = executor.enqueueContext({requests[i]}, std::nullopt); - reqIds.push_back(retReqId.front()); - tokens[i] = std::move(resultTokens); - reqIdToBatchId[retReqId.front()] = i; - } - - int32_t numContextFinished = 0; - int contextIter = 0; - while (numContextFinished < maxRequests && contextIter < maxWaitMs) - { - std::chrono::milliseconds waitTime(1); - - auto contextResponses = executor.awaitContextResponses(waitTime); - contextIter++; - numContextFinished += contextResponses.size(); - - for (auto&& responseWithId : contextResponses) - { - auto contextGid = responseWithId.gid; - int batchId = reqIdToBatchId[contextGid]; - auto&& request = requests[batchId]; - request.setRequestType(RequestType::REQUEST_TYPE_GENERATION_ONLY); - request.setContextPhaseParams(responseWithId.response.getResult().contextPhaseParams.value()); - executor.enqueueGeneration({request}, {responseWithId.gid}, std::nullopt); - } - } - // Get the new tokens for each requests - int32_t numFinished = 0; - int iter = 0; - SizeType32 numResponses = 0; - while (numFinished < maxRequests && iter < maxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitGenerationResponses(waitTime); - for (auto& responseWithId : responses) - { - numResponses++; - if (!responseWithId.response.hasError()) - { - auto result = responseWithId.response.getResult(); - numFinished += result.isFinal; - auto batchId = reqIdToBatchId.at(responseWithId.gid); - auto seqIdx = result.sequenceIndex; - - auto& contextLogits = result.contextLogits; - auto& genLogits = result.generationLogits; - auto& outputTokenIds = result.outputTokenIds; - - EXPECT_EQ(result.finishReasons.size(), beamWidth); - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - auto& newTokens = outputTokenIds.at(beam); - auto& reqTokens = tokens.at(batchId).at(seqIdx).at(beam); - - reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); - // FinishReason is only supported for bw=1 and inflight batching. - if (beamWidth == 1 && batchingType == BatchingType::kINFLIGHT) - { - EXPECT_EQ(result.finishReasons.at(beam), - result.isFinal ? FinishReason::kLENGTH : FinishReason::kNOT_FINISHED); - } - } - - auto& cumLogProbs = result.cumLogProbs; - auto& logProbs = result.logProbs; - auto& beamTokens = tokens.at(batchId).at(seqIdx); - testData.verifyLogProbs(outConfig.returnLogProbs, streaming, outConfig.excludeInputFromOutput, - givenInputLengths.at(batchId), beamWidth, beamTokens, cumLogProbs, logProbs, batchId, - flakyTestInfo); - - testData.validateContextLogits(outConfig.returnContextLogits, givenInputLengths.at(batchId), - beamWidth, contextLogits, vocabSizePadded, batchId); - testData.validateGenerationLogits(outConfig.returnGenerationLogits, result.isFinal, streaming, - outConfig.excludeInputFromOutput, givenInputLengths.at(batchId), reqMaxNewTokens.at(batchId), - beamWidth, beamTokens, genLogits, vocabSizePadded, batchId, returnAllGeneratedTokens); - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(responseWithId.gid) - + " has already been processed and was terminated."; - EXPECT_EQ(responseWithId.response.getErrorMsg(), err); - } - } - ++iter; - } - EXPECT_LT(iter, maxWaitMs); - testData.verifyOutput(tokens, givenInputLengths, streaming, outConfig.excludeInputFromOutput, flakyTestInfo, - isSpeculativeDecoding, beamWidth, numReturnSequences, false); - } - comm.barrier(); -} - -TEST_P(DisaggParamsTest, DisaggTokenComparison) -{ - -#if ENABLE_MULTI_DEVICE - - if (!(tensorrt_llm::common::getEnvUseUCXKvCache())) - { - setenv("UCX_TLS", "^cuda_ipc", 1); // disable cuda_ipc for testing for mpi - } - else - { - setenv("UCX_TCP_CM_REUSEADDR", "y", - 1); // tests creates and destroies ucxCacheCommunicatoers frequently, so listener ports must be reused - } - auto const processNum = std::get<0>(GetParam()); - auto const modelNames = std::get<1>(GetParam()); - auto const participantIdsEachInstance = std::get<2>(GetParam()); // std::vector<std::vector<int>> - auto const participantDeviceIdsEachInstance = std::get<3>(GetParam()); // std::vector<std::vector<int>>; - auto const instanceRoles - = std::get<4>(GetParam()); // std::vector<int> ; //1 is context , 0 is generation, 2 is mixed - auto const controllerRank = std::get<5>(GetParam()); - - // params_check - auto const& world_comm = tensorrt_llm::mpi::MpiComm::world(); - int const commRank = world_comm.getRank(); - int const commSize = world_comm.getSize(); - if (commSize != processNum) - { - GTEST_SKIP() << " need " << processNum << " processes but got " << commSize << " mpi processes, skip test."; - } - ASSERT_EQ(participantIdsEachInstance.size(), participantDeviceIdsEachInstance.size()); - SizeType32 instanceNum = participantIdsEachInstance.size(); - ASSERT_EQ(instanceNum, instanceRoles.size()); - ASSERT_EQ(instanceNum, modelNames.size()); - - std::unordered_set<int> deviceIdsSet; - for (auto const& ids : participantDeviceIdsEachInstance) - { - for (auto const& id : ids) - { - deviceIdsSet.insert(id); - } - } - if (mDeviceCount < deviceIdsSet.size()) - { - GTEST_SKIP() << " need " << deviceIdsSet.size() << " devices but got " << mDeviceCount - << " devices, skip test."; - } - - ASSERT_GE(controllerRank, 0); - ASSERT_LT(controllerRank, commSize); - int ranksNum = 0; - std::unordered_map<SizeType32, SizeType32> rankCounter; - std::unordered_map<SizeType32, SizeType32> deviceCounter; - SizeType32 deviceRuseNum = 1; - bool isContext = false; - bool isGeneration = false; - std::vector<int> participatntIds; - std::vector<int> deviceIds; - std::string modelName; - bool isController = (commRank == controllerRank); - for (SizeType32 i = 0; i < instanceNum; i++) - { - auto const& ranksThisInstance = participantIdsEachInstance[i]; - auto const& devicesThisInstance = participantDeviceIdsEachInstance[i]; - - ASSERT_EQ(ranksThisInstance.size(), devicesThisInstance.size()); - SizeType32 rankNumThisInstance = ranksThisInstance.size(); - ASSERT_GT(rankNumThisInstance, 0); - ranksNum += rankNumThisInstance; - for (SizeType32 j = 0; j < rankNumThisInstance; j++) - { - rankCounter[ranksThisInstance[j]]++; - deviceCounter[devicesThisInstance[j]]++; - ASSERT_GE(rankCounter[ranksThisInstance[j]], 1); - deviceRuseNum = std::max(deviceCounter[devicesThisInstance[j]], deviceRuseNum); - ASSERT_GE(ranksThisInstance[j], 0); - ASSERT_LT(ranksThisInstance[j], commSize); - - if (commRank == ranksThisInstance[j]) - { - participatntIds = ranksThisInstance; - deviceIds = devicesThisInstance; - isContext = instanceRoles[i] == InstanceRole::kCONTEXT || instanceRoles[i] == InstanceRole::kMIXED; - isGeneration - = instanceRoles[i] == InstanceRole::kGENERATION || instanceRoles[i] == InstanceRole::kMIXED; - // modelName = isContext ? contextModel : genModel; - modelName = modelNames[i]; - } - } - } - ASSERT_GE(ranksNum, commSize); - - OutputConfig outConfig; - int const beamWidth = 1; - BeamResult beamResult{beamWidth}; - - bool streaming = false; - int const maxBeamWidth = 1; - ASSERT_TRUE(fs::exists(DATA_PATH)); - - fs::path modelPath; - // set defaults and adjust if needed by different models - fs::path inputPath = DATA_PATH / "input_tokens.npy"; - ModelIds modelIds{50256, 50256}; - SizeType32 vocabSizePadded{50257}; // gpt vocabSizePadded - bool isSpeculativeDecoding{false}; - - // NOTE: This can be used to disable checks for certain prompt batch entries - FlakyTestInfo flakyTestInfo; - - if (modelName == "gpt") - { - auto const resultsPath - = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - if (outConfig.returnContextLogits || outConfig.returnGenerationLogits) - { - modelPath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"; - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); - beamResult.contextLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); - beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); - if (outConfig.returnLogProbs) - { - beamResult.cumLogProbsFile - = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_CUM_LOG_PROBS_FILE(); - beamResult.logProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_LOG_PROBS_FILE(); - } - } - else - { - modelPath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_FILE(); - if (outConfig.returnLogProbs) - { - beamResult.cumLogProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_FILE(); - beamResult.logProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_FILE(); - } - } - } - else if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1" - || modelName == "llama_tp1_pp2_cp1" || modelName == "llama_tp2_pp1_cp1" || modelName == "llama_tp1_pp1_cp1") - { - inputPath = DATA_PATH / LLAMA_INPUT_FILE; - vocabSizePadded = LLAMA_VOCAB_SIZE_PADDED; - - auto const resultsPath - = LLAMA_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - modelIds.padId = LLAMA_PAD_ID; - modelIds.endId = LLAMA_END_ID; - if (modelName == "llama_tp4_pp1_cp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP4_PP1_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp4_cp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP4_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp2_cp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP2_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp2-cp1-gpu"; - } - else if (modelName == "llama_tp2_pp1_cp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP1_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp1-cp1-gpu"; - } - else if (modelName == "llama_tp2_pp2_cp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp1_cp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - } - } - else - { - TLLM_THROW("Unrecognized modelName"); - } - - // Warning: This should be the last check before running the test. - // It will initialize MPI which can take significant time. - if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1" - || modelName == "llama_tp1_pp2_cp1" || modelName == "llama_tp2_pp1_cp1") - { - if (outConfig.returnLogProbs || outConfig.returnContextLogits || outConfig.returnGenerationLogits) - { - GTEST_SKIP() << "Skipping logits and log probs tests for mpi runs"; - } - } - - // Returning logits will bring higher latency - if (streaming && (outConfig.returnContextLogits || outConfig.returnGenerationLogits)) - { - mMaxWaitMs = 20000; - } - - auto executorConfig = ExecutorConfig(maxBeamWidth); - FloatType freeGpuMemoryFraction = 0.9f / (deviceRuseNum); // context and gen instance run on same device - KvCacheConfig kvCacheConfig{true, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction}; - executorConfig.setKvCacheConfig(kvCacheConfig); - executorConfig.setRequestStatsMaxIterations(1000); - executorConfig.setCacheTransceiverConfig( - texec::CacheTransceiverConfig(texec::CacheTransceiverConfig::BackendType::DEFAULT)); - auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); - auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); - auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); - world_comm.barrier(); - auto disaggExecutor = tensorrt_llm::testing::disaggexecutor::DisaggExecutorLeader(modelPath, - ModelType::kDECODER_ONLY, executorConfig, isController, isContext, isGeneration, givenInputLengths.size(), - participatntIds, deviceIds, commRank); - - runDisaggTest(disaggExecutor, manager, *givenInput, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, - outConfig, isSpeculativeDecoding, mMaxWaitMs, executorConfig.getBatchingType(), false); - -#else - - GTEST_SKIP() << "Skipping DisaggExecutor Test"; - -#endif -} - -TEST_P(DisaggOrchestratorParamsTest, DisaggTokenComparison) -{ - -#if ENABLE_MULTI_DEVICE - - if (!(tensorrt_llm::common::getEnvUseUCXKvCache())) - { - setenv("UCX_TLS", "^cuda_ipc", 1); // disable cuda_ipc for testing for mpi - } - else - { - setenv("UCX_TCP_CM_REUSEADDR", "y", - 1); // tests creates and destroies ucxCacheCommunicatoers frequently, so listener ports must be reused - } - auto const processNum = std::get<0>(GetParam()); - auto const modelNames = std::get<1>(GetParam()); - auto const participantIdsEachInstance = std::get<2>(GetParam()); // std::vector<std::vector<int>> - auto const participantDeviceIdsEachInstance = std::get<3>(GetParam()); // std::vector<std::vector<int>>; - auto const instanceRoles = std::get<4>(GetParam()); // std::vector<int> ; //1 is context , 0 is generation - auto const controllerRank = std::get<5>(GetParam()); - - // params_check - auto const& world_comm = tensorrt_llm::mpi::MpiComm::world(); - int const commRank = world_comm.getRank(); - int const commSize = world_comm.getSize(); - if (commSize != processNum) - { - GTEST_SKIP() << " need " << processNum << " processes but got " << commSize << " mpi processes, skip test."; - } - - bool spawnProcess = false; - if (commSize == 1) - { - spawnProcess = true; - if (mDeviceCount < 4) - { - GTEST_SKIP() << "DisaggExecutorTest requires at least 4 GPUs"; - } - ASSERT_TRUE(tensorrt_llm::common::getEnvUseUCXKvCache() || tensorrt_llm::common::getEnvUseNixlKvCache()); - } - - ASSERT_EQ(participantIdsEachInstance.size(), participantDeviceIdsEachInstance.size()); - SizeType32 instanceNum = participantIdsEachInstance.size(); - ASSERT_EQ(instanceNum, instanceRoles.size()); - ASSERT_EQ(instanceNum, modelNames.size()); - - std::unordered_set<int> deviceIdsSet; - for (auto const& ids : participantDeviceIdsEachInstance) - { - for (auto const& id : ids) - { - deviceIdsSet.insert(id); - } - } - if (mDeviceCount < deviceIdsSet.size()) - { - GTEST_SKIP() << " need " << deviceIdsSet.size() << " devices but got " << mDeviceCount - << " devices, skip test."; - } - - ASSERT_GE(controllerRank, 0); - ASSERT_LT(controllerRank, commSize); - std::string modelName = modelNames[0]; - bool isController = (commRank == controllerRank); - std::vector<fs::path> contextModels; - std::vector<fs::path> genModels; - - auto getModelPath = [=](std::string modelNN) - { - fs::path retPath; - if (modelNN == "llama_tp4_pp1") - { - retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; - } - else if (modelNN == "llama_tp1_pp4") - { - retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; - } - else if (modelNN == "llama_tp1_pp2") - { - retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp2-cp1-gpu"; - } - else if (modelNN == "llama_tp2_pp1") - { - retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp1-cp1-gpu"; - } - else if (modelNN == "llama_tp2_pp2") - { - retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; - } - else if (modelNN == "llama_tp1_pp1") - { - retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - } - return retPath; - }; - for (SizeType32 i = 0; i < instanceNum; i++) - { - if (instanceRoles[i] == InstanceRole::kCONTEXT) - { - contextModels.push_back(getModelPath(modelNames[i])); - } - else - { - genModels.push_back(getModelPath(modelNames[i])); - } - } - - OutputConfig outConfig; - int const beamWidth = 1; - BeamResult beamResult{beamWidth}; - - bool streaming = false; - int const maxBeamWidth = 1; - ASSERT_TRUE(fs::exists(DATA_PATH)); - - fs::path modelPath; - // set defaults and adjust if needed by different models - fs::path inputPath = DATA_PATH / "input_tokens.npy"; - ModelIds modelIds{50256, 50256}; - SizeType32 vocabSizePadded{50257}; // gpt vocabSizePadded - bool isSpeculativeDecoding{false}; - - // NOTE: This can be used to disable checks for certain prompt batch entries - FlakyTestInfo flakyTestInfo; - if (modelName == "llama_tp4_pp1" || modelName == "llama_tp1_pp4" || modelName == "llama_tp2_pp2" - || modelName == "llama_tp1_pp2" || modelName == "llama_tp2_pp1" || modelName == "llama_tp1_pp1") - { - inputPath = DATA_PATH / LLAMA_INPUT_FILE; - vocabSizePadded = LLAMA_VOCAB_SIZE_PADDED; - - auto const resultsPath - = LLAMA_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - modelIds.padId = LLAMA_PAD_ID; - modelIds.endId = LLAMA_END_ID; - if (modelName == "llama_tp4_pp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP4_PP1_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp4") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP4_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp2") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP2_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp2-cp1-gpu"; - } - else if (modelName == "llama_tp2_pp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP1_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp1-cp1-gpu"; - } - else if (modelName == "llama_tp2_pp2") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - } - } - - else - { - TLLM_THROW("Unrecognized modelName"); - } - - // Warning: This should be the last check before running the test. - // It will initialize MPI which can take significant time. - if (modelName == "llama_tp4_pp1" || modelName == "llama_tp1_pp4" || modelName == "llama_tp2_pp2" - || modelName == "llama_tp1_pp2" || modelName == "llama_tp2_pp1") - { - if (outConfig.returnLogProbs || outConfig.returnContextLogits || outConfig.returnGenerationLogits) - { - GTEST_SKIP() << "Skipping logits and log probs tests for mpi runs"; - } - } - - // Returning logits will bring higher latency - if (streaming && (outConfig.returnContextLogits || outConfig.returnGenerationLogits)) - { - mMaxWaitMs = 20000; - } - - auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); - auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); - auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); - world_comm.barrier(); - auto contextNum = contextModels.size(); - auto genNum = genModels.size(); - // int deviceCount = -1; - // TLLM_CUDA_CHECK(cudaGetDeviceCount(&deviceCount)); - bool isOrchestrator = commRank == 0; - std::vector<ExecutorConfig> ctxExecutorConfigs; - std::vector<ExecutorConfig> genExecutorConfigs; - for (int in = 0; in < instanceNum; in++) - { - tensorrt_llm::executor::SchedulerConfig schedulerConfig(CapacitySchedulerPolicy::kMAX_UTILIZATION); - KvCacheConfig kvCacheConfig{true, std::nullopt, std::nullopt, std::nullopt, 0.2}; - - tensorrt_llm::executor::ExecutorConfig executorConfig(maxBeamWidth, schedulerConfig, kvCacheConfig); - tensorrt_llm::executor::OrchestratorConfig orchestratorConfig{ - isOrchestrator, PathUtil::EXECUTOR_WORKER_PATH(), nullptr, spawnProcess}; - - tensorrt_llm::executor::ParallelConfig parallelConfig{tensorrt_llm::executor::CommunicationType::kMPI, - tensorrt_llm::executor::CommunicationMode::kORCHESTRATOR, participantDeviceIdsEachInstance.at(in), - spawnProcess ? std::nullopt : std::optional<std::vector<SizeType32>>(participantIdsEachInstance.at(in)), - orchestratorConfig}; - executorConfig.setParallelConfig(parallelConfig); - executorConfig.setCacheTransceiverConfig( - texec::CacheTransceiverConfig(texec::CacheTransceiverConfig::BackendType::DEFAULT)); - if (in < contextNum) - { - ctxExecutorConfigs.push_back(executorConfig); - } - else - { - genExecutorConfigs.push_back(executorConfig); - } - } - auto disaggExecutor - = DisaggExecutorOrchestrator(contextModels, genModels, ctxExecutorConfigs, genExecutorConfigs, true, true); - - runDisaggTest(disaggExecutor, manager, *givenInput, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, - outConfig, isSpeculativeDecoding, mMaxWaitMs, BatchingType::kINFLIGHT, false); - -#else - - GTEST_SKIP() << "Skipping DisaggExecutor Test"; - -#endif -} - -TEST_P(ConditionalDisaggParamsTest, DisaggTokenComparison) -{ -#if ENABLE_MULTI_DEVICE - if (!tensorrt_llm::common::getEnvUseUCXKvCache()) - { - setenv("UCX_TLS", "^cuda_ipc", 1); // disable cuda_ipc for testing for mpi - } - auto constexpr processNum = 2; - auto constexpr deviceNum = 2; - auto const& modelName = std::get<0>(GetParam()); - auto constexpr controllerRank = 0; - - // params_check - auto const& world_comm = tensorrt_llm::mpi::MpiComm::world(); - int const commRank = world_comm.getRank(); - int const commSize = world_comm.getSize(); - if (commSize != processNum) - { - GTEST_SKIP() << " need " << processNum << " processes but got " << commSize << " mpi processes, skip test."; - } - if (mDeviceCount < deviceNum) - { - GTEST_SKIP() << " need " << deviceNum << " devices but got " << mDeviceCount << " devices, skip test."; - } - - bool isContext = commRank == 0; - bool isGeneration = commRank == 1; - std::vector<int> participatntIds = {commRank}; - std::vector<int> deviceIds = {commRank}; - bool isController = (commRank == controllerRank); - - OutputConfig outConfig(false, false, false, false, false, false); - int const beamWidth = 1; - BeamResult beamResult{beamWidth}; - - bool streaming = false; - int const maxBeamWidth = 1; - ASSERT_TRUE(fs::exists(DATA_PATH)); - - fs::path modelPath; - // set defaults and adjust if needed by different models - fs::path inputPath = DATA_PATH / "input_tokens.npy"; - ModelIds modelIds{50256, 50256}; - SizeType32 vocabSizePadded{50257}; // gpt vocabSizePadded - bool isSpeculativeDecoding{false}; - - // NOTE: This can be used to disable checks for certain prompt batch entries - FlakyTestInfo flakyTestInfo; - - if (modelName == "gpt") - { - auto const resultsPath - = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - modelPath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_FILE(); - } - else if (modelName == "llama_tp1_pp1_cp1") - { - inputPath = DATA_PATH / LLAMA_INPUT_FILE; - vocabSizePadded = LLAMA_VOCAB_SIZE_PADDED; - - auto const resultsPath - = LLAMA_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - modelIds.padId = LLAMA_PAD_ID; - modelIds.endId = LLAMA_END_ID; - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP1_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - } - else - { - TLLM_THROW("Unrecognized modelName"); - } - - auto executorConfig = ExecutorConfig(maxBeamWidth); - FloatType freeGpuMemoryFraction = 0.9f; - KvCacheConfig kvCacheConfig{true, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction}; - executorConfig.setKvCacheConfig(kvCacheConfig); - executorConfig.setRequestStatsMaxIterations(1000); - executorConfig.setCacheTransceiverConfig( - texec::CacheTransceiverConfig(CacheTransceiverConfig::BackendType::DEFAULT)); - auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); - auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); - auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); - world_comm.barrier(); - auto executor = tensorrt_llm::testing::disaggexecutor::DisaggExecutorLeader(modelPath, ModelType::kDECODER_ONLY, - executorConfig, isController, isContext, isGeneration, givenInputLengths.size(), participatntIds, deviceIds, - commRank); - - std::unordered_map<IdType, SizeType32> reqIdToBatchId; - std::unordered_map<SizeType32, std::vector<BeamTokens>> tokens; - auto const* const givenInputData = tr::bufferCast<TokenIdType const>(*givenInput); - - auto const& inputShape = givenInput->getShape(); - ASSERT_EQ(inputShape.nbDims, 2); - ASSERT_GT(inputShape.d[0], 0); - - // Load expected outputs for each beam width value - auto testData = TestData::loadTestData(beamResult, *givenInput, beamWidth, manager, outConfig, modelIds); - auto const maxSeqLen = testData.maxSeqLen; - - // Load expected outputs and inputs - SizeType32 numRequests = static_cast<SizeType32>(givenInputLengths.size()); - SizeType32 maxRequests = numRequests; - std::vector<Request> requests; - std::vector<SizeType32> reqMaxNewTokens; - SizeType32 const numReturnSequences = 1; - - for (SizeType32 req = 0; req < maxRequests; ++req) - { - SizeType32 inputLen = givenInputLengths.at(req); - auto maxNewTokens = maxSeqLen - maxInputLength; - reqMaxNewTokens.push_back(maxNewTokens); - SizeType32 endId = -1; - auto const* const seqBegin = givenInputData + req * maxInputLength; - VecTokens tokens(seqBegin, seqBegin + inputLen); - auto samplingConfig = tensorrt_llm::executor::SamplingConfig(beamWidth); - samplingConfig.setNumReturnSequences(numReturnSequences); - auto request = Request( - VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, samplingConfig, outConfig, endId); - request.setReturnAllGeneratedTokens(false); - // setting request type to context/full by condition - if (req % 2 == 0) - { - request.setRequestType(RequestType::REQUEST_TYPE_CONTEXT_ONLY); - } - else - { - request.setRequestType(RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION); - } - requests.emplace_back(std::move(request)); - } - - if (isController) - { - std::vector<IdType> reqIds; - - for (int i = 0; i < requests.size(); ++i) - { - std::vector<BeamTokens> resultTokens; - resultTokens.reserve(numReturnSequences); - for (SizeType32 seqIdx = 0; seqIdx < numReturnSequences; ++seqIdx) - { - resultTokens.emplace_back(beamWidth); - } - auto retReqId = executor.enqueueRequests({requests[i]}); - reqIds.push_back(retReqId.front()); - tokens[i] = std::move(resultTokens); - reqIdToBatchId[retReqId.front()] = i; - } - - // Get the new tokens for each requests - int32_t numFinished = 0; - int iter = 0; - SizeType32 numResponses = 0; - while (numFinished < maxRequests && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - numResponses++; - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - auto batchId = reqIdToBatchId.at(response.getRequestId()); - auto seqIdx = result.sequenceIndex; - - auto& outputTokenIds = result.outputTokenIds; - - EXPECT_EQ(result.finishReasons.size(), beamWidth); - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - auto& newTokens = outputTokenIds.at(beam); - auto& reqTokens = tokens.at(batchId).at(seqIdx).at(beam); - - reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); - // FinishReason is only supported for bw=1 and inflight batching. - if (beamWidth == 1 && executorConfig.getBatchingType() == BatchingType::kINFLIGHT) - { - EXPECT_EQ(result.finishReasons.at(beam), - result.isFinal ? FinishReason::kLENGTH : FinishReason::kNOT_FINISHED); - } - } - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - testData.verifyOutput(tokens, givenInputLengths, streaming, outConfig.excludeInputFromOutput, flakyTestInfo, - isSpeculativeDecoding, beamWidth, numReturnSequences, false); - } - world_comm.barrier(); -#else - GTEST_SKIP() << "Skipping DisaggExecutor Test"; -#endif -} - -INSTANTIATE_TEST_SUITE_P(GptDisaggSymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(2), // processNum - testing::Values(std::vector<std::string>{"gpt", "gpt"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0, 1) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(GptDisaggSymmetricExecutorMixedTest, DisaggParamsTest, - testing::Combine( // - testing::Values(2), // processNum - testing::Values(std::vector<std::string>{"gpt", "gpt"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kMIXED, InstanceRole::kMIXED}), // instanceRoles - testing::Values(1) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(GptSingleDeviceDisaggSymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(2), // processNum - testing::Values(std::vector<std::string>{"gpt", "gpt"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0}, {0}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(GptSingleDeviceDisaggSymmetricExecutorMixedTest, DisaggParamsTest, - testing::Combine( // - testing::Values(2), // processNum - testing::Values(std::vector<std::string>{"gpt", "gpt"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0}, {0}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kMIXED, InstanceRole::kMIXED}), // instanceRoles - testing::Values(1) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(GptConditionalDisaggSymmetricExecutorTest, ConditionalDisaggParamsTest, - testing::Combine(testing::Values("gpt")), generateTestNameCondDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaConditionalDisaggSymmetricExecutorTest, ConditionalDisaggParamsTest, - testing::Combine(testing::Values("llama_tp1_pp1_cp1")), generateTestNameCondDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaTP2DisaggSymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(4), // processNum - testing::Values(std::vector<std::string>{"llama_tp2_pp1_cp1", "llama_tp2_pp1_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaPP2DisaggSymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(4), // processNum - testing::Values(std::vector<std::string>{"llama_tp1_pp2_cp1", "llama_tp1_pp2_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{1, 0}, {3, 2}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaTP2DisaggSymmetricExecutorMixedTest, DisaggParamsTest, - testing::Combine( // - testing::Values(2), // processNum - testing::Values(std::vector<std::string>{"llama_tp2_pp1_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0, 1}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0, 1}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kMIXED}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaPP2DisaggSymmetricExecutorMixedTest, DisaggParamsTest, - testing::Combine( // - testing::Values(2), // processNum - testing::Values(std::vector<std::string>{"llama_tp1_pp2_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0, 1}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0, 1}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kMIXED}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaTP2PP2DisaggSymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(8), // processNum - testing::Values(std::vector<std::string>{"llama_tp2_pp2_cp1", "llama_tp2_pp2_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0, 1, 2, 3}, {4, 5, 6, 7}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{2, 3, 0, 1}, {2, 3, 0, 1}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaConPP2GenTP2DisaggAsymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(4), // processNum - testing::Values(std::vector<std::string>{"llama_tp1_pp2_cp1", "llama_tp2_pp1_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}}), // (1,0) (2,3) // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{1, 0}, {2, 3}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaConTP2GenPP2DisaggAsymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(4), // processNum - testing::Values(std::vector<std::string>{"llama_tp2_pp1_cp1", "llama_tp1_pp2_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}}), // (0,1), (3,2)// participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0, 1}, {3, 2}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaConTP2PP2GenPP2DisaggAsymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(6), // processNum - testing::Values(std::vector<std::string>{"llama_tp2_pp2_cp1", "llama_tp1_pp2_cp1"}), // modelNames - testing::Values( - std::vector<std::vector<int>>{{0, 1, 2, 3}, {4, 5}}), // (2,3,0,1) , (5,4)// participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{2, 3, 0, 1}, {1, 0}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaConTP2PP2GenTP2DisaggAsymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(6), // processNum - testing::Values(std::vector<std::string>{"llama_tp2_pp2_cp1", "llama_tp2_pp1_cp1"}), // modelNames - testing::Values( - std::vector<std::vector<int>>{{0, 1, 2, 3}, {4, 5}}), // (2,3,0,1), (4,5)// participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{2, 3, 0, 1}, {0, 1}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); -INSTANTIATE_TEST_SUITE_P(LlamaConTP2PP1GenTP2PP2DisaggAsymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(6), // processNum - testing::Values(std::vector<std::string>{"llama_tp2_pp1_cp1", "llama_tp2_pp2_cp1"}), // modelNames - testing::Values( - std::vector<std::vector<int>>{{0, 1}, {2, 3, 4, 5}}), // (0,1) , (4,5,2,3)%4// participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0, 1}, {0, 1, 2, 3}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaConTP2GenPP4DisaggAsymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(6), // processNum - testing::Values(std::vector<std::string>{"llama_tp2_pp1_cp1", "llama_tp1_pp4_cp1"}), // modelNames - testing::Values( - std::vector<std::vector<int>>{{4, 5}, {0, 1, 2, 3}}), // (4,5) ,(3,2,1,0)// participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0, 1}, {3, 2, 1, 0}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaCon4TP1Gen1TP4DisaggAsymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(8), // processNum - testing::Values(std::vector<std::string>{"llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", - "llama_tp1_pp1_cp1", "llama_tp4_pp1_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2}, {3}, {4, 5, 6, 7}}), // participantIdsEachInstance - testing::Values( - std::vector<std::vector<int>>{{0}, {1}, {2}, {3}, {0, 1, 2, 3}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, - InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(4) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen2TP2AndPP2DisaggAsymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(6), // processNum - testing::Values(std::vector<std::string>{ - "llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", "llama_tp2_pp1_cp1", "llama_tp1_pp2_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2, 3}, {4, 5}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2, 3}, {1, 0}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, - InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen2PP2DisaggAsymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(6), // processNum - testing::Values(std::vector<std::string>{ - "llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", "llama_tp1_pp2_cp1", "llama_tp1_pp2_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2, 3}, {4, 5}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0}, {1}, {3, 2}, {1, 0}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, - InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaCon4TP1Gen1TP2PP2DisaggAsymmetricExecutorTest, DisaggParamsTest, - testing::Combine( // - testing::Values(8), // processNum - testing::Values(std::vector<std::string>{"llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", - "llama_tp1_pp1_cp1", "llama_tp2_pp2_cp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2}, {3}, {4, 5, 6, 7}}), // participantIdsEachInstance - testing::Values( - std::vector<std::vector<int>>{{0}, {1}, {2}, {3}, {2, 3, 0, 1}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, - InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(4) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen2TP2DisaaggOrchestrator, DisaggOrchestratorParamsTest, - testing::Combine( // - testing::Values(7), // processNum - testing::Values( - std::vector<std::string>{"llama_tp1_pp1", "llama_tp1_pp1", "llama_tp2_pp1", "llama_tp2_pp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{1}, {2}, {3, 4}, {5, 6}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2, 3}, {0, 1}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, - InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); -// for disaggOrchestrator 1->0, 2->1, 3->2, 4->3, 5->0, 6->1 - -INSTANTIATE_TEST_SUITE_P(LlamaCon2TP2Gen2TP1DisaaggOrchestrator, DisaggOrchestratorParamsTest, - testing::Combine( // - testing::Values(7), // processNum - testing::Values( - std::vector<std::string>{"llama_tp2_pp1", "llama_tp2_pp1", "llama_tp1_pp1", "llama_tp1_pp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{1, 2}, {3, 4}, {5}, {6}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}, {0}, {1}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, - InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen2PP2DisaaggOrchestrator, DisaggOrchestratorParamsTest, - testing::Combine( // - testing::Values(7), // processNum - testing::Values( - std::vector<std::string>{"llama_tp1_pp1", "llama_tp1_pp1", "llama_tp1_pp2", "llama_tp1_pp2"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{1}, {2}, {3, 4}, {5, 6}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0}, {1}, {3, 2}, {1, 0}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, - InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen1TP2PP2DisaaggOrchestrator, DisaggOrchestratorParamsTest, - testing::Combine( // - testing::Values(7), // processNum - testing::Values(std::vector<std::string>{"llama_tp1_pp1", "llama_tp1_pp1", "llama_tp2_pp2"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{1}, {2}, {3, 4, 5, 6}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0}, {1}, {0, 1, 2, 3}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{ - InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaCon2TP2Gen2TP1DisaggSpawnOrchestrator, DisaggOrchestratorParamsTest, - testing::Combine( // - testing::Values(1), // processNum - testing::Values( - std::vector<std::string>{"llama_tp2_pp1", "llama_tp2_pp1", "llama_tp1_pp1", "llama_tp1_pp1"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{1, 2}, {3, 4}, {5}, {6}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}, {0}, {1}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, - InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); - -INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen2PP2DisaggSpawnOrchestrator, DisaggOrchestratorParamsTest, - testing::Combine( // - testing::Values(1), // processNum - testing::Values( - std::vector<std::string>{"llama_tp1_pp1", "llama_tp1_pp1", "llama_tp1_pp2", "llama_tp1_pp2"}), // modelNames - testing::Values(std::vector<std::vector<int>>{{1}, {2}, {3, 4}, {5, 6}}), // participantIdsEachInstance - testing::Values(std::vector<std::vector<int>>{{0}, {1}, {3, 2}, {1, 0}}), // participantDeviceIdsEachInstance - testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, - InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles - testing::Values(0) // controllerRank - ), - generateTestNameDisaggParams); diff --git a/cpp/tests/e2e_tests/executor/encDecTest.cpp b/cpp/tests/e2e_tests/executor/encDecTest.cpp deleted file mode 100644 index 0095ae30ad40..000000000000 --- a/cpp/tests/e2e_tests/executor/encDecTest.cpp +++ /dev/null @@ -1,387 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "executorTest.h" - -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "tensorrt_llm/runtime/utils/numpyUtils.h" -#include "tensorrt_llm/testing/modelSpec.h" -#include "tests/utils/common.h" - -#include <gmock/gmock.h> -#include <gtest/gtest.h> -#include <nlohmann/json.hpp> - -#include <algorithm> -#include <chrono> -#include <memory> -#include <string> -#include <vector> - -namespace tr = tensorrt_llm::runtime; - -using namespace tensorrt_llm::testing; -using namespace tensorrt_llm::executor; -using namespace std::chrono_literals; -using tensorrt_llm::testing::KVCacheType; - -namespace -{ - -std::string getEncDecEnginePath(std::string const& modelName, SizeType32 tp, SizeType32 pp, SizeType32 cp) -{ - return modelName + '/' + std::to_string(tp * pp * cp) + "-gpu/float16"; -} - -TokenIdType getDecTokenFromJsonConfig(std::filesystem::path decEnginePath, std::string const& token_name) -{ - TokenIdType tokenId = 0; - try - { - std::ifstream decoderJsonConfigPath(decEnginePath / "config.json"); - auto const decoderPretrainedConfig - = nlohmann::json::parse(decoderJsonConfigPath, nullptr, true, true).at("pretrained_config"); - tokenId = decoderPretrainedConfig.at(token_name).template get<int32_t>(); - } - catch (nlohmann::json::out_of_range& e) - { - TLLM_LOG_ERROR( - "Parameter %s cannot be found from decoder config.json in pretrained_config. Using default id 0.", - token_name.c_str()); - } - catch (nlohmann::json::type_error const& e) - { - TLLM_LOG_ERROR( - "Parameter %s has a different type from decoder config.json in pretrained_config. Using default id 0.", - token_name.c_str()); - } - return tokenId; -} - -} // namespace - -using EncDecParamsType = std::tuple<std::string, SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, - std::vector<SizeType32>>; - -std::string generateTestNameEncDec(testing::TestParamInfo<EncDecParamsType> const& info) -{ - auto modelName = std::get<0>(info.param); - auto const beamWidth = std::get<1>(info.param); - auto const maxNewTokens = std::get<2>(info.param); - auto const tp = std::get<3>(info.param); - auto const pp = std::get<4>(info.param); - - // GTEST does not allow '-' in its test name - for (auto& c : modelName) - { - if (c == '-') - { - c = '_'; - } - } - - std::string name = "EncDecTest"; - name.append("_" + modelName); - name.append("_BeamWidth" + std::to_string(beamWidth)); - name.append("_MaxNewTokens" + std::to_string(maxNewTokens)); - name.append("_TP" + std::to_string(tp)); - name.append("_PP" + std::to_string(pp)); - return name; -} - -bool isLanguageAdapterName(std::string const& modelName) -{ - return modelName == LANGUAGE_ADAPTER_NAME; -} - -class EncDecParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<EncDecParamsType> -{ -}; - -TEST_P(EncDecParamsTest, validEncDecCtor) -{ - auto const modelName = std::get<0>(GetParam()); - SizeType32 const beamWidth = std::get<1>(GetParam()); - SizeType32 const maxNewTokens = std::get<2>(GetParam()); - SizeType32 const tp = std::get<3>(GetParam()); - SizeType32 const pp = std::get<4>(GetParam()); - SizeType32 const cp = std::get<5>(GetParam()); - - auto const enginePathName = getEncDecEnginePath(modelName, tp, pp, cp); - std::filesystem::path encEnginePath = ENC_DEC_ENGINE_BASE / enginePathName / "encoder"; - std::filesystem::path decEnginePath = ENC_DEC_ENGINE_BASE / enginePathName / "decoder"; - ExecutorConfig executorConfig{}; - FloatType freeGpuMemoryFraction = 0.4f; - FloatType crossKvCacheFraction = 0.4f; - KvCacheConfig kvCacheConfig{false, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction}; - kvCacheConfig.setCrossKvCacheFraction(crossKvCacheFraction); - executorConfig.setKvCacheConfig(kvCacheConfig); - auto executor = Executor(encEnginePath, decEnginePath, ModelType::kENCODER_DECODER, executorConfig); -} - -TEST_P(EncDecParamsTest, Forward) -{ - bool constexpr VERBOSE = false; - auto const modelName = std::get<0>(GetParam()); - SizeType32 const beamWidth = std::get<1>(GetParam()); - SizeType32 const maxNewTokens = std::get<2>(GetParam()); - SizeType32 const tp = std::get<3>(GetParam()); - SizeType32 const pp = std::get<4>(GetParam()); - SizeType32 const cp = std::get<5>(GetParam()); - - // Parameters for language adapter test - SizeType32 const numLanguages = std::get<6>(GetParam()); - std::vector<SizeType32> languageAdapterUids = std::get<7>(GetParam()); - - bool const streaming = false; - - auto const enginePathName = getEncDecEnginePath(modelName, tp, pp, cp); - std::filesystem::path encEnginePath = ENC_DEC_ENGINE_BASE / enginePathName / "encoder"; - std::filesystem::path decEnginePath = ENC_DEC_ENGINE_BASE / enginePathName / "decoder"; - - // load ground truth input & output data - auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); - auto inputsIdsHost - = tr::utils::loadNpy(manager, (ENC_DEC_DATA_BASE / "input_ids.npy").string(), tr::MemoryType::kCPU); - auto inputsIdsPtr = tr::bufferCast<TokenIdType>(*inputsIdsHost); - auto inputLengthsHost - = tr::utils::loadNpy(manager, (ENC_DEC_DATA_BASE / "input_lengths.npy").string(), tr::MemoryType::kCPU); - auto inputLengthsPtr = tr::bufferCast<SizeType32>(*inputLengthsHost); - auto encoderOutputHost - = tr::utils::loadNpy(manager, (ENC_DEC_DATA_BASE / "encoder_output.npy").string(), tr::MemoryType::kCPU); - auto encoderOutputPtr = tr::bufferCast<half>(*encoderOutputHost); - auto decoderOutputHost = tr::utils::loadNpy(manager, - (ENC_DEC_DATA_BASE / "output_ids_beam").string() + std::to_string(beamWidth) + ".npy", tr::MemoryType::kCPU); - auto decoderOutputPtr = tr::bufferCast<TokenIdType>(*decoderOutputHost); - - // Rank and size info - auto& comm = tensorrt_llm::mpi::MpiComm::world(); - auto const worldRank = comm.getRank(); - auto const worldSize = comm.getSize(); - - // create executor - BatchingType const batchingType = BatchingType::kINFLIGHT; - FloatType freeGpuMemoryFraction = 0.5f; - FloatType crossKvCacheFraction = 0.5f; - KvCacheConfig kvCacheConfig{false, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction}; - kvCacheConfig.setCrossKvCacheFraction(crossKvCacheFraction); - - ExecutorConfig executorConfig{beamWidth}; - executorConfig.setBatchingType(batchingType); - executorConfig.setKvCacheConfig(kvCacheConfig); - executorConfig.setNormalizeLogProbs(false); - - // TODO: OrchestratorMode test does not pass - bool const useOrchestratorMode = (tp * pp) > worldSize; - std::optional<OrchestratorConfig> orchestratorConfig = std::nullopt; - if (useOrchestratorMode) - { - orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); - } - auto parallelConfig = ParallelConfig(CommunicationType::kMPI, - useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::nullopt, std::nullopt, - orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - - auto executor = Executor(encEnginePath, decEnginePath, ModelType::kENCODER_DECODER, executorConfig); - - OutputConfig outConfig; - outConfig.excludeInputFromOutput = false; - outConfig.returnLogProbs = false; - outConfig.returnGenerationLogits = false; - outConfig.returnContextLogits = false; - outConfig.returnEncoderOutput = false; - - TokenIdType bosId = getDecTokenFromJsonConfig(decEnginePath, "bos_token_id"); - TokenIdType padId = getDecTokenFromJsonConfig(decEnginePath, "pad_token_id"); - TokenIdType eosId = getDecTokenFromJsonConfig(decEnginePath, "eos_token_id"); - TokenIdType decoderStartTokenId = getDecTokenFromJsonConfig(decEnginePath, "decoder_start_token_id"); - - bool const isLanguageAdapterTest = isLanguageAdapterName(modelName); - // create requests - SizeType32 const nbRequests = inputLengthsHost->getShape().d[0]; - std::vector<Request> requests; - for (int i = 0, cumInputLen = 0; i < nbRequests; i++) - { - auto encoderInput = VecTokens(&inputsIdsPtr[cumInputLen], - &inputsIdsPtr[cumInputLen] + inputLengthsPtr[i]); // assume inputIds is flattened / no-padding - cumInputLen += inputLengthsPtr[i]; - auto decoderInput = VecTokens{decoderStartTokenId}; - Request req(decoderInput, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, - eosId, padId); - req.setEncoderInputTokenIds(encoderInput); - if (isLanguageAdapterTest) - { - req.setLanguageAdapterUid(languageAdapterUids[i]); - } - requests.emplace_back(req); - } - - using namespace std::chrono; - - // enqueue requests - if (worldRank == 0) - { - auto tik = high_resolution_clock::now(); - std::vector<IdType> reqIds = executor.enqueueRequests(std::move(requests)); - - // get responses - milliseconds waitTime(5000); - auto responsesAll = executor.awaitResponses(reqIds, waitTime); - auto tok = high_resolution_clock::now(); - TLLM_LOG_DEBUG("TRT-LLM C++ E2E time %d ms", duration_cast<milliseconds>(tok - tik).count()); - TLLM_LOG_DEBUG("Number of responses: %d", responsesAll.size()); - - int32_t numFinished = 0; - int iter = 0; - SizeType32 numResponses = 0; - std::unordered_map<IdType, std::vector<VecTokens>> outputTokens; - for_each(reqIds.begin(), reqIds.end(), - [&outputTokens, &beamWidth](auto const& id) - { - TLLM_LOG_DEBUG("Request IDs: %d", id); - outputTokens[id] = {}; - for (int i = 0; i < beamWidth; i++) - { - outputTokens[id].emplace_back(VecTokens{}); - } - }); - for (int i = 0; i < reqIds.size(); i++) - { - auto& responses = responsesAll[i]; - for (auto& response : responses) - { - numResponses++; - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - for (int beam = 0; beam < beamWidth; beam++) - { - auto& resTokens = result.outputTokenIds.at(beam); - auto& outTokens = outputTokens.at(response.getRequestId()).at(beam); - outTokens.insert(outTokens.end(), std::make_move_iterator(resTokens.begin()), - std::make_move_iterator(resTokens.end())); - } - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - } - } - - // print output & check correctness with ground truth - for (auto const& [reqId, tokens] : outputTokens) - { - SizeType32 gtMaxLength = decoderOutputHost->getShape().d[1]; - auto gtOutput = decoderOutputPtr + (reqId - 1) * gtMaxLength; - - if constexpr (VERBOSE) - { - std::cout << ">>> Request ID: " << reqId << std::endl; - for (int beam = 0; beam < beamWidth; beam++) - { - std::cout << "output tokens, beam " << beam << ", output length " << tokens[beam].size() << ": " - << std::endl; - for_each(tokens[beam].begin(), tokens[beam].end(), - [](auto const& token) { std::cout << token << ", "; }); - std::cout << std::endl; - } - std::cout << "ground truth tokens: " << std::endl; - - SizeType32 gtLength = 0; - for (int i = 0; i < gtMaxLength; i++) - { - if (gtOutput[i] != eosId) - { - std::cout << gtOutput[i] << ", "; - gtLength++; - } - } - std::cout << std::endl; - std::cout << "ground truth length: " << gtLength << std::endl; - } - - // check token-by-token match between beam 0 & ground truth - ASSERT_TRUE(tokens.size() <= gtMaxLength) - << "Request ID " << reqId << "'s generated length is longer than ground truth length " << gtMaxLength; - for (int i = 0; i < gtMaxLength; i++) - { - if (outConfig.excludeInputFromOutput) - { - // if results exclude decoder start token, skip it in ground truth too - continue; - } - if (i < tokens[0].size()) - { - ASSERT_EQ(tokens[0][i], gtOutput[i]) - << "Generated token id: " << tokens[0][i] << " v.s. ground truth: " << gtOutput[i]; - } - else - { - ASSERT_EQ(gtOutput[i], eosId) << "Request ID " << reqId << "'s generated length " << tokens.size() - << " is shorter than ground truth length " << gtMaxLength; - } - } - } - } -} - -INSTANTIATE_TEST_SUITE_P(T5BasicTest, EncDecParamsTest, - testing::Combine(testing::Values(T5_NAME), testing::Values(1), testing::Values(64), testing::Values(1), - testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), - generateTestNameEncDec); - -INSTANTIATE_TEST_SUITE_P(T5Beam2Test, EncDecParamsTest, - testing::Combine(testing::Values(T5_NAME), testing::Values(2), testing::Values(64), testing::Values(1), - testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), - generateTestNameEncDec); - -INSTANTIATE_TEST_SUITE_P(T5MultiGPUTest, EncDecParamsTest, - testing::Combine(testing::Values(T5_NAME), testing::Values(1), testing::Values(64), testing::Values(4), - testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), - generateTestNameEncDec); - -INSTANTIATE_TEST_SUITE_P(BartBasicTest, EncDecParamsTest, - testing::Combine(testing::Values(BART_NAME), testing::Values(1), testing::Values(64), testing::Values(1), - testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), - generateTestNameEncDec); - -INSTANTIATE_TEST_SUITE_P(BartBeam2Test, EncDecParamsTest, - testing::Combine(testing::Values(BART_NAME), testing::Values(2), testing::Values(64), testing::Values(1), - testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), - generateTestNameEncDec); - -INSTANTIATE_TEST_SUITE_P(BartMultiGPUTest, EncDecParamsTest, - testing::Combine(testing::Values(BART_NAME), testing::Values(1), testing::Values(64), testing::Values(4), - testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), - generateTestNameEncDec); - -INSTANTIATE_TEST_SUITE_P(LanguageAdapterBasicTest, EncDecParamsTest, - testing::Combine(testing::Values(LANGUAGE_ADAPTER_NAME), testing::Values(1), testing::Values(64), - testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(4), - testing::Values(std::vector<SizeType32>{2, 3})), - generateTestNameEncDec); diff --git a/cpp/tests/e2e_tests/executor/executorMockTest.cpp b/cpp/tests/e2e_tests/executor/executorMockTest.cpp deleted file mode 100644 index 0f0176f1ed6b..000000000000 --- a/cpp/tests/e2e_tests/executor/executorMockTest.cpp +++ /dev/null @@ -1,1028 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "executorTest.h" - -#include "tensorrt_llm/batch_manager/trtGptModel.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "tensorrt_llm/testing/modelSpec.h" -#include "tests/utils/common.h" - -#include <gmock/gmock.h> -#include <gtest/gtest.h> -#include <nlohmann/json.hpp> - -#include <algorithm> -#include <chrono> -#include <memory> -#include <numeric> -#include <string> -#include <thread> -#include <vector> - -using ::testing::_; -using ::testing::Invoke; - -namespace tr = tensorrt_llm::runtime; -namespace tb = tensorrt_llm::batch_manager; - -using namespace tensorrt_llm::testing; -using namespace tensorrt_llm::executor; -using namespace std::chrono_literals; -using tensorrt_llm::testing::KVCacheType; - -class MockedModel : public Model -{ - using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; - using RequestList = std::list<LlmRequestPtr>; - -public: - MOCK_METHOD(void, forwardSync, (), ()); - MOCK_METHOD(void, forwardAsync, (RequestList const&), ()); - MOCK_METHOD(void, terminateRequest, (std::shared_ptr<tb::LlmRequest> const& llmRequest, bool pause), ()); - MOCK_METHOD( - void, terminateRequestSync, (std::shared_ptr<tb::LlmRequest> const& llmRequest, FinishReason finishReason), ()); - MOCK_METHOD(SizeType32, getMaxNumSequences, (), (const)); - MOCK_METHOD(SizeType32, getMaxInputLen, (), (const)); - MOCK_METHOD(SizeType32, getHiddenSize, (), (const)); - MOCK_METHOD(SizeType32, getMaxSequenceLen, (), (const)); - MOCK_METHOD(SizeType32, getVocabSizePadded, (), (const)); - MOCK_METHOD(SizeType32, getMaxDraftLen, (), (const)); - MOCK_METHOD(SizeType32, getNumMicroBatches, (), (const)); - MOCK_METHOD(SizeType32, getOperatingBeamWidth, (), (const)); - MOCK_METHOD(nvinfer1::DataType, getLogitDataType, (), (const)); - MOCK_METHOD(nvinfer1::DataType, getTensorDataType, (std::string const&), (const)); - MOCK_METHOD(nvinfer1::Dims, getTensorShape, (std::string const&), (const)); - MOCK_METHOD(void, getCurrentIterationStats, (IterationStats&), (const)); - MOCK_METHOD(void, getCurrentRequestStats, (RequestStatsPerIteration&), (const)); - MOCK_METHOD(DebugTensorsPerIteration, getCurrentDebugTensors, (), (const)); - MOCK_METHOD(tr::WorldConfig const&, getWorldConfig, (), (const)); - MOCK_METHOD(tr::ModelConfig const&, getModelConfig, (), (const)); - MOCK_METHOD(tr::BufferManager const&, getBufferManager, (), (const)); - MOCK_METHOD(tr::BufferManager::CudaStreamPtr, getRuntimeStreamPtr, (), (const)); - MOCK_METHOD(IterationType, getIterCounter, (), (const, noexcept)); - MOCK_METHOD(bool, hasSpeculativeDecodingFastLogits, (), (const, noexcept)); - MOCK_METHOD(bool, getGatherGenerationLogits, (), (const)); - MOCK_METHOD(void, updatePeftCache, (LlmRequestPtr const& llmReqeust), ()); - MOCK_METHOD(void, setLogitsPostProcessorBatched, (std::optional<LogitsPostProcessorBatched>), ()); - MOCK_METHOD(void, setReplicateLogitsPostProcessor, (bool), ()); - MOCK_METHOD(bool, getReplicateLogitsPostProcessor, (), (const)); - MOCK_METHOD(bool, hasGuidedDecoder, (), (const, noexcept)); - MOCK_METHOD(void, resetIterationStats, (), ()); - MOCK_METHOD( - std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager>, getKVCacheManager, (), ()); - MOCK_METHOD(std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager const>, - getKVCacheManager, (), (const)); - MOCK_METHOD(SizeType32, getMaxCapacityBatchSize, (SizeType32, SizeType32), (const)); -}; - -using ParamType = std::tuple<bool, bool, int>; - -std::string generateTestName(testing::TestParamInfo<ParamType> const& info) -{ - auto const streaming = std::get<0>(info.param); - auto const excludeInputFromOutput = std::get<1>(info.param); - auto const beamWidth = std::get<2>(info.param); - std::string name = "ExecutorTest"; - if (streaming) - { - name += "Streaming"; - } - if (excludeInputFromOutput) - { - name += "ExclInput"; - } - name.append("BW" + std::to_string(beamWidth)); - return name; -} - -class ParamTest : public GptExecutorTest, public ::testing::WithParamInterface<ParamType> -{ -}; - -TEST_P(ParamTest, MockedModel) -{ - using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; - using RequestList = std::list<LlmRequestPtr>; - - bool const streaming = std::get<0>(GetParam()); - bool const excludeInputFromOutput = std::get<1>(GetParam()); - auto const beamWidth = std::get<2>(GetParam()); - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - auto model = std::make_shared<MockedModel>(); - - EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); - EXPECT_CALL(*model, getVocabSizePadded()).Times(0); - EXPECT_CALL(*model, getLogitDataType()).Times(0); - tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); - EXPECT_CALL(*model, getModelConfig()) - .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); - SizeType32 callCount = 0; - EXPECT_CALL(*model, forwardAsync(_)) - .WillRepeatedly(Invoke( - [&](RequestList const& requestList) - { - for (auto const& llmReq : requestList) - { - // Don't add any tokens to simulate no output tokens - llmReq->addNewTokens(VecTokens(beamWidth, 1)); - llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); - if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) - { - llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); - } - } - callCount++; - })); - - EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxDraftLen()).WillRepeatedly(Invoke([&]() { return 0; })); - EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); - tr::WorldConfig const dummyWorldConfig; - EXPECT_CALL(*model, getWorldConfig()) - .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); - EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); - EXPECT_CALL(*model, getCurrentRequestStats(_)) - .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); - - ExecutorConfig const executorConfig(beamWidth); - auto executor = Executor(model, executorConfig); - - // Create the request - constexpr SizeType32 maxNewTokens = 5; - VecTokens const inputTokens{1, 2, 3, 4}; - auto request - = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - - // Enqueue the request - auto requestId = executor.enqueueRequest(request); - - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - auto const& result = response.getResult(); - done = result.isFinal; - } - ++iter; - } - - EXPECT_LT(iter, mMaxWaitMs); - EXPECT_EQ(callCount, maxNewTokens); -} - -TEST_F(GptExecutorTest, MockedModelMaxQueueSize) -{ - using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; - using RequestList = std::list<LlmRequestPtr>; - - auto model = std::make_shared<MockedModel>(); - - EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); - EXPECT_CALL(*model, terminateRequestSync(_, _)).Times(0); - EXPECT_CALL(*model, getVocabSizePadded()).Times(0); - EXPECT_CALL(*model, getLogitDataType()).Times(0); - - SizeType32 callCount = 0; - EXPECT_CALL(*model, forwardAsync(_)) - .WillRepeatedly(Invoke( - [&](RequestList const& requestList) - { - for (auto const& llmReq : requestList) - { - // Sleep to allow queue to fill up - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - // Don't add any tokens to simulate no output tokens - llmReq->addNewTokens({1}); - llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); - if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) - { - llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); - } - } - callCount++; - })); - - EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxDraftLen()).WillRepeatedly(Invoke([&]() { return 0; })); - EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); - tr::WorldConfig const dummyWorldConfig; - EXPECT_CALL(*model, getWorldConfig()) - .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); - EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); - EXPECT_CALL(*model, getCurrentRequestStats(_)) - .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); - tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); - EXPECT_CALL(*model, getModelConfig()) - .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); - SizeType32 maxQueueSize = 6; - ExecutorConfig executorConfig; - executorConfig.setMaxQueueSize(maxQueueSize); - - auto executor = Executor(model, executorConfig); - - // Create the request - SizeType32 const maxNewTokens = 5; - VecTokens const inputTokens{1, 2, 3, 4}; - auto request = Request(inputTokens, maxNewTokens); - - // Enqueue as many requests as the queue can manage - for (int i = 0; i < maxQueueSize; i++) - { - auto requestId = executor.enqueueRequest(request); - } - try - { - auto requestId = executor.enqueueRequest(request); - - FAIL() << "Expected TllmException"; - } - catch (std::exception const& e) - { - EXPECT_THAT(e.what(), testing::HasSubstr("Maximum queue size of 6 has been reached, please try again later")); - } - - // Wait for requests to get scheduled to free up space in queue - std::this_thread::sleep_for(std::chrono::milliseconds(maxQueueSize * 200)); - auto requestId = executor.enqueueRequest(request); - - try - { - auto samplingConfig = SamplingConfig(1); - samplingConfig.setNumReturnSequences(maxQueueSize); - auto request = Request(inputTokens, maxNewTokens, false, samplingConfig); - auto requestId = executor.enqueueRequest(request); - FAIL() << "Expected TllmException"; - } - catch (std::exception const& e) - { - EXPECT_THAT(e.what(), testing::HasSubstr("Maximum queue size of 6 has been reached, please try again later")); - } -} - -TEST_F(GptExecutorTest, MockedModelReqStatsBug) -{ - using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; - using RequestList = std::list<LlmRequestPtr>; - - bool streaming = false; - bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - auto model = std::make_shared<MockedModel>(); - - EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); - EXPECT_CALL(*model, getVocabSizePadded()).Times(0); - EXPECT_CALL(*model, getLogitDataType()).Times(0); - EXPECT_CALL(*model, updatePeftCache(_)).WillRepeatedly(Invoke([&]() { return; })); - - SizeType32 callCount = 0; - RequestList currentReq; - EXPECT_CALL(*model, forwardAsync(_)) - .WillRepeatedly(Invoke( - [&](RequestList const& requestList) - { - currentReq = requestList; - for (auto const& llmReq : requestList) - { - // Don't add any tokens to simulate no output tokens - llmReq->addNewTokens({1}); - llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); - } - callCount++; - })); - - EXPECT_CALL(*model, forwardSync()) - .WillRepeatedly(Invoke( - [&]() - { - for (auto const& llmReq : currentReq) - { - if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) - { - llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); - } - } - return; - })); - - EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxDraftLen()).WillRepeatedly(Invoke([&]() { return 0; })); - EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); - tr::WorldConfig const dummyWorldConfig; - EXPECT_CALL(*model, getWorldConfig()) - .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); - EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& stats) { return; })); - EXPECT_CALL(*model, getCurrentRequestStats(_)) - .WillRepeatedly(Invoke([&](RequestStatsPerIteration& stats) { return; })); - - SizeType32 beamWidth = 1; - ExecutorConfig executorConfig(beamWidth); - executorConfig.setRequestStatsMaxIterations(1000); - auto executor = Executor(model, executorConfig); - - // Create the request - SizeType32 maxNewTokens = 5; - VecTokens inputTokens{1, 2, 3, 4}; - int numRequests = 10000; - auto request - = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - - auto done = std::atomic<bool>{false}; - auto statsThreadDone = false; - // Spawn a thread that continuously get stats - auto statsThread = std::thread( - [&executor, &done, &statsThreadDone]() - { - while (!done) - { - auto reqStats = executor.getLatestRequestStats(); - std::this_thread::sleep_for(std::chrono::microseconds(10)); - } - statsThreadDone = true; - }); - - // Spawn a thread that enqueues the requests - std::vector<IdType> requestIds; - auto enqueueThread = std::thread( - [&executor, &requestIds, &request, &done, numRequests]() - { - for (int i = 0; i < numRequests; ++i) - { - requestIds.push_back(executor.enqueueRequest(request)); - } - done = true; - }); - enqueueThread.join(); - ASSERT_EQ(requestIds.size(), numRequests); - - // Wait for stats thread to be done, fail otherwise - int iter = 0; - while (!statsThreadDone && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - std::this_thread::sleep_for(std::chrono::milliseconds(waitTime)); - iter++; - } - ASSERT_TRUE(statsThreadDone); - statsThread.join(); -} - -TEST_F(GptExecutorTest, MockedModelEvictRestartValidityTest) -{ - using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; - using RequestList = std::list<LlmRequestPtr>; - - constexpr bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - auto model = std::make_shared<MockedModel>(); - - EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); - EXPECT_CALL(*model, getVocabSizePadded()).Times(0); - EXPECT_CALL(*model, getLogitDataType()).Times(0); - EXPECT_CALL(*model, updatePeftCache(_)).WillRepeatedly(Invoke([&]() { return; })); - tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); - EXPECT_CALL(*model, getModelConfig()) - .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); - SizeType32 callCount = 0; - RequestList currentReq; - EXPECT_CALL(*model, forwardAsync(_)) - .WillRepeatedly(Invoke( - [&](RequestList const& requestList) - { - currentReq = requestList; - for (auto const& llmReq : requestList) - { - // Don't add any tokens to simulate no output tokens - llmReq->addNewTokens({1}); - llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); - } - callCount++; - })); - - EXPECT_CALL(*model, forwardSync()) - .WillRepeatedly(Invoke( - [&]() - { - for (auto const& llmReq : currentReq) - { - if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) - { - llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); - } - } - return; - })); - - EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 6; })); - EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxDraftLen()).WillRepeatedly(Invoke([&]() { return 0; })); - EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); - tr::WorldConfig const dummyWorldConfig; - EXPECT_CALL(*model, getWorldConfig()) - .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); - EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); - EXPECT_CALL(*model, getCurrentRequestStats(_)) - .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); - - SizeType32 const beamWidth = 1; - ExecutorConfig executorConfig(beamWidth, - SchedulerConfig(CapacitySchedulerPolicy::kMAX_UTILIZATION)); // Condition 1 : MAX_UTILIZATION scheduling policy - executorConfig.setEnableChunkedContext(false); // Condition 2 : Chunked context disabled - executorConfig.setRequestStatsMaxIterations(1000); - auto executor = Executor(model, executorConfig); - - // Create the request - constexpr bool streaming = true; // Condition 3 : Streaming enabled - SizeType32 const maxNewTokens = 5; - VecTokens const tooLongInputTokens{1, 2, 3, 4, 5}; // Condition 4 : prompt input len + maxNewTokens > MaxInputLen - auto tooLongRequest = Request( - tooLongInputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - - // Enqueue the request - auto longRequestId = executor.enqueueRequest(tooLongRequest); - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(longRequestId, waitTime); - for (auto& response : responses) - { - EXPECT_EQ(response.hasError(), true); - EXPECT_THAT(response.getErrorMsg(), - testing::HasSubstr("sequence length is potentially greater than max input length")); - done = true; - } - ++iter; - } -} - -#if ENABLE_MULTI_DEVICE -// This test can be run manually to test multiGPU execution -// mpirun --allow-run-as-root -n 5 ./executorTest --gtest_filter="*MockedModelMultiGpu/ExecutorTest" -// Number of MPI ranks can be greater than tp - -TEST_P(ParamTest, MockedModelMultiGpu) -{ - auto const& world = tensorrt_llm::mpi::MpiComm::world(); - auto const worldRank = world.getRank(); - auto const worldSize = world.getSize(); - - // In this test, allow worldSize to be greater than tp = 4 - // If so, set participant ids to be the last 4 ranks - SizeType32 const tp = std::min(4, worldSize); - - using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; - using RequestList = std::list<LlmRequestPtr>; - - bool const streaming = std::get<0>(GetParam()); - bool const excludeInputFromOutput = std::get<1>(GetParam()); - auto const beamWidth = std::get<2>(GetParam()); - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - auto model = std::make_shared<MockedModel>(); - - // Create the request - constexpr SizeType32 maxNewTokens = 5; - VecTokens const inputTokens{1, 2, 3, 4}; - auto request - = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - - EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); - EXPECT_CALL(*model, getVocabSizePadded()).Times(0); - EXPECT_CALL(*model, getLogitDataType()).Times(0); - tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); - EXPECT_CALL(*model, getModelConfig()) - .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); - SizeType32 callCount = 0; - SizeType32 reqCallCount = 0; - EXPECT_CALL(*model, forwardAsync(_)) - .WillRepeatedly(Invoke( - [&](RequestList const& requestList) - { - for (auto const& llmReq : requestList) - { - EXPECT_EQ(llmReq->getTokens().size(), beamWidth); - // Verify that all MPI ranks get the expected request, even though only rank 0 actually gets the - // request - if (reqCallCount == 0) - { - EXPECT_EQ(llmReq->getOrigPromptLen(), request.getInputTokenIds().size()); - for (int i = 0; i < llmReq->getOrigPromptLen(); ++i) - { - EXPECT_EQ(llmReq->getTokens(beamWidth - 1).at(i), request.getInputTokenIds().at(i)); - } - } - EXPECT_EQ(llmReq->isStreaming(), request.getStreaming()); - EXPECT_EQ(llmReq->mMaxNewTokens, request.getMaxTokens()); - EXPECT_EQ( - llmReq->getTokens(beamWidth - 1).size(), request.getInputTokenIds().size() + reqCallCount); - - SizeType32 tokenId = 1; - COMM_SESSION.bcastValue(tokenId, 0); - // Don't add any tokens to simulate no output tokens - // Simulate leader rank communicating with comm session - VecTokens const newTokens(beamWidth, tokenId); - llmReq->addNewTokens(newTokens); - llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); - if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) - { - llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); - } - reqCallCount++; - } - callCount++; - })); - - EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); - EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); - EXPECT_CALL(*model, getCurrentRequestStats(_)) - .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); - - tr::WorldConfig dummyWorldConfig = tr::WorldConfig(tp, 1, 1, worldRank, tp); - EXPECT_CALL(*model, getWorldConfig()) - .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); - - ParallelConfig parallelConfig; - - // Set participant ids to be of size tp, starting at worldSize - 1 - std::vector<SizeType32> participantIds; - participantIds.reserve(tp); - for (int i = 0; i < tp; ++i) - { - participantIds.push_back(worldSize - tp + i); - } - bool const isLeader = (worldRank == participantIds.front()); - parallelConfig.setParticipantIds(participantIds); - - bool const isWorker = (std::find(participantIds.begin(), participantIds.end(), worldRank) != participantIds.end()); - - // Set device ids - std::vector<SizeType32> deviceIds(tp); - std::iota(deviceIds.begin(), deviceIds.end(), 0); - parallelConfig.setDeviceIds(deviceIds); - - ExecutorConfig executorConfig(beamWidth); - executorConfig.setParallelConfig(parallelConfig); - auto executor = Executor(model, executorConfig); - - EXPECT_EQ(isWorker, executor.isParticipant()); - - // Enqueue the request - IdType requestId = 0; - if (isLeader) - { - requestId = executor.enqueueRequest(request); - - SizeType32 numResponses{0}; - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - ++numResponses; - auto const& result = response.getResult(); - EXPECT_EQ(result.outputTokenIds.size(), beamWidth); - auto expectedSize = streaming ? (beamWidth > 1 ? numResponses : 1) - : (maxNewTokens + (excludeInputFromOutput ? 0 : inputTokens.size())); - EXPECT_EQ(result.outputTokenIds.at(beamWidth - 1).size(), expectedSize); - done = result.isFinal; - } - ++iter; - } - - EXPECT_LT(iter, mMaxWaitMs); - EXPECT_EQ(numResponses, streaming ? maxNewTokens : 1); - EXPECT_EQ(callCount, maxNewTokens); - } -} -#endif // ENABLE_MULTI_DEVICE - -TEST_F(GptExecutorTest, MockedModelWithError) -{ - using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; - using RequestList = std::list<LlmRequestPtr>; - - struct MockedModelParams - { - SizeType32 maxInputLen; - SizeType32 maxSeqLen; - SizeType32 expectedTerminateCnt; - SizeType32 expectedForwardCnt; - bool computeGenLogits; - bool computeContextLogits; - std::string expectedError; - }; - - std::vector<MockedModelParams> mockedModelParams; - // Mocked error in forward call - mockedModelParams.emplace_back(MockedModelParams{10, 20, 1, 1, true, true, "mocked error"}); - // prompt longer than maxInputLen - mockedModelParams.emplace_back(MockedModelParams{1, 20, 0, 0, true, true, "exceeds maximum input length"}); - // Model doesn't support context logits output - mockedModelParams.emplace_back( - MockedModelParams{10, 20, 0, 0, false, true, "gather_generation_logits must be enabled"}); - // Model doesn't support gen logits output - mockedModelParams.emplace_back( - MockedModelParams{10, 20, 0, 0, true, false, "need to build engine with gather_context"}); - - for (auto const& mockedModelParam : mockedModelParams) - { - auto model = std::make_shared<MockedModel>(); - SizeType32 beamWidth = 1; - - // One request should be terminated - EXPECT_CALL(*model, terminateRequest(_, _)).Times(mockedModelParam.expectedTerminateCnt); - EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 1024; })); - EXPECT_CALL(*model, getLogitDataType()).WillRepeatedly(Invoke([&]() { return nvinfer1::DataType::kFLOAT; })); - EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& stats) { return; })); - EXPECT_CALL(*model, getCurrentRequestStats(_)) - .WillRepeatedly(Invoke([&](RequestStatsPerIteration& stats) { return; })); - - SizeType32 callCount = 0; - EXPECT_CALL(*model, forwardAsync(_)) - .WillRepeatedly(Invoke( - [&](RequestList const&) - { - callCount++; - // There was a bug where we were missing a notify call when errors were encountered - // and this test was not catching it, probably because the error was reported - // before the first call to awaitResponses. So we add a sleep here to make sure - // the awaitResponses is called before the error is thrown - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - throw std::runtime_error("mocked error"); - })); - - EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return mockedModelParam.maxInputLen; })); - EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return mockedModelParam.maxSeqLen; })); - EXPECT_CALL(*model, getMaxDraftLen()).WillRepeatedly(Invoke([&]() { return 0; })); - tr::WorldConfig const dummyWorldConfig; - EXPECT_CALL(*model, getWorldConfig()) - .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); - tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); - dummyModelConfig.computeContextLogits(mockedModelParam.computeContextLogits); - dummyModelConfig.computeGenerationLogits(mockedModelParam.computeGenLogits); - EXPECT_CALL(*model, getModelConfig()) - .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); - EXPECT_CALL(*model, getGatherGenerationLogits()) - .WillRepeatedly(Invoke([&]() -> bool { return mockedModelParam.computeGenLogits; })); - EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& stats) { return; })); - EXPECT_CALL(*model, getCurrentRequestStats(_)) - .WillRepeatedly(Invoke([&](RequestStatsPerIteration& stats) { return; })); - EXPECT_CALL(*model, getIterCounter()).WillRepeatedly(Invoke([&]() -> IterationType { return 0; })); - - ExecutorConfig executorConfig(beamWidth); - auto executor = Executor(model, executorConfig); - - // Create the request - SizeType32 maxNewTokens = 5; - VecTokens inputTokens{1, 2, 3, 4}; - - OutputConfig outConfig; - outConfig.returnContextLogits = true; - outConfig.returnGenerationLogits = true; - - auto streaming = false; - auto request = Request( - inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - - // Enqueue the request - auto requestId = executor.enqueueRequest(std::move(request)); - - bool done = false; - auto responses = executor.awaitResponses(requestId); - for (auto& response : responses) - { - if (!response.hasError()) - { - FAIL() << "Expecting an error to be received"; - } - else - { - auto err = response.getErrorMsg(); - EXPECT_THAT(err, testing::HasSubstr(mockedModelParam.expectedError)); - done = true; - } - } - - EXPECT_TRUE(done); - EXPECT_EQ(callCount, mockedModelParam.expectedForwardCnt); - } -} - -TEST_F(GptExecutorTest, MockedModelCancelRequest) -{ - using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; - using RequestList = std::list<LlmRequestPtr>; - - constexpr bool streaming = true; - auto model = std::make_shared<MockedModel>(); - - std::unordered_map<IdType, tensorrt_llm::executor::FinishReason> reqIdsToTerminate; - // Two requests with one child request (3 in total) should be terminated - EXPECT_CALL(*model, terminateRequestSync(_, _)) - .Times(3) - .WillRepeatedly(Invoke([&](LlmRequestPtr const& llmRequest, FinishReason finishReason) - { reqIdsToTerminate.try_emplace(llmRequest->mRequestId, finishReason); })); - EXPECT_CALL(*model, terminateRequest(_, _)).Times(3); - EXPECT_CALL(*model, getVocabSizePadded()).Times(0); - EXPECT_CALL(*model, getLogitDataType()).Times(0); - tr::WorldConfig const dummyWorldConfig; - EXPECT_CALL(*model, getWorldConfig()) - .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); - EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); - EXPECT_CALL(*model, getCurrentRequestStats(_)) - .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); - tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); - EXPECT_CALL(*model, getModelConfig()) - .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); - - SizeType32 callCount = 0; - std::unordered_map<IdType, SizeType32> callCountPerSeq; - EXPECT_CALL(*model, forwardAsync(_)) - .WillRepeatedly(Invoke( - [&](RequestList const& requestList) - { - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - - for (auto const& llmReq : requestList) - { - if (llmReq->isGenerationCompleteState()) - { - continue; - } - // Don't add any tokens to simulate no output tokens - llmReq->addNewTokens({1}); - llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); - if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) - { - llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); - } - if (callCountPerSeq.find(llmReq->mRequestId) != callCountPerSeq.end()) - { - callCountPerSeq[llmReq->mRequestId]++; - } - else - { - callCountPerSeq[llmReq->mRequestId] = 1; - } - - if (reqIdsToTerminate.count(llmReq->mRequestId) != 0U) - { - if (!llmReq->isGenerationToCompleteState()) - { - model->terminateRequest(llmReq, false); - llmReq->finishByReason(reqIdsToTerminate[llmReq->mRequestId]); - llmReq->clearGeneratedTokens(); - } - reqIdsToTerminate.erase(llmReq->mRequestId); - } - } - callCount++; - })); - - EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 100; })); - EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 200; })); - EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); - - SizeType32 const beamWidth = 1; - ExecutorConfig const executorConfig(beamWidth); - auto executor = Executor(model, executorConfig); - - // Create the request - SizeType32 const maxNewTokens = 150; - VecTokens const inputTokens{1, 2, 3, 4}; - auto request = Request(inputTokens, maxNewTokens, streaming); - - // Enqueue the request - auto requestId = executor.enqueueRequest(std::move(request)); - - // Cancel the request - std::this_thread::sleep_for(std::chrono::milliseconds(30)); - executor.cancelRequest(requestId); - - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - - if (response.hasError()) - { - FAIL() << "Not expecting an error to be received"; - } - - auto const& result = response.getResult(); - done = result.isFinal; - if (done) - { - for (SizeType32 beamIdx = 0; beamIdx < beamWidth; ++beamIdx) - { - EXPECT_EQ(result.finishReasons[beamIdx], FinishReason::kCANCELLED); - } - } - } - ++iter; - } - - EXPECT_LT(iter, mMaxWaitMs); - // Expecting to receiving fewer tokens than maxNewTokens - EXPECT_LT(callCount, maxNewTokens); - - // Create the request having child requests. - auto samplingConfig2 = SamplingConfig(1); - samplingConfig2.setNumReturnSequences(2); - auto request2 = Request(inputTokens, maxNewTokens, streaming, samplingConfig2); - - // Reset call count. - callCount = 0; - callCountPerSeq.clear(); - - // Enqueue the request - auto requestId2 = executor.enqueueRequest(request2); - - // Cancel the request - std::this_thread::sleep_for(std::chrono::milliseconds(30)); - executor.cancelRequest(requestId2); - - done = false; - iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId2, waitTime); - for (auto& response : responses) - { - - if (response.hasError()) - { - FAIL() << "Not expecting an error to be received"; - } - - auto const& result = response.getResult(); - done = result.isFinal; - if (done) - { - EXPECT_EQ(result.finishReasons[0], FinishReason::kCANCELLED); - } - } - ++iter; - } - - EXPECT_LT(iter, mMaxWaitMs); - for (auto& [reqId, count] : callCountPerSeq) - { - // Expecting to receiving fewer tokens than maxNewTokens - EXPECT_LT(count, maxNewTokens) << "Failed at request id: " << reqId; - } -} - -TEST_F(GptExecutorTest, MockedModelNumReturns) -{ - using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; - using RequestList = std::list<LlmRequestPtr>; - - SizeType32 const maxBeamWidth = 4; - OutputConfig const outConfig; - auto model = std::make_shared<MockedModel>(); - - EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); - EXPECT_CALL(*model, getVocabSizePadded()).Times(0); - EXPECT_CALL(*model, getLogitDataType()).Times(0); - tr::WorldConfig const dummyWorldConfig; - EXPECT_CALL(*model, getWorldConfig()) - .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); - EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); - EXPECT_CALL(*model, getCurrentRequestStats(_)) - .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); - tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); - EXPECT_CALL(*model, getModelConfig()) - .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); - SizeType32 callCount = 0; - EXPECT_CALL(*model, forwardAsync(_)) - .WillRepeatedly(Invoke( - [&](RequestList const& requestList) - { - for (auto const& llmReq : requestList) - { - // Don't add any tokens to simulate no output tokens - auto numBeams = llmReq->mSamplingConfig.getNumReturnBeams(); - llmReq->addNewTokens(VecTokens(numBeams, 1)); - llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); - if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) - { - llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); - } - } - callCount++; - })); - - EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 10; })); - EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 20; })); - EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); - - ExecutorConfig const executorConfig(maxBeamWidth); - auto executor = Executor(model, executorConfig); - - // Create the request - SizeType32 const maxNewTokens = 5; - VecTokens const inputTokens{1, 2, 3, 4}; - constexpr bool streaming = false; - - auto samplingConfig1 = SamplingConfig(1); - samplingConfig1.setNumReturnSequences(3); - auto request1 = Request(inputTokens, maxNewTokens, streaming, samplingConfig1, outConfig); - auto samplingConfig2 = SamplingConfig(4); - auto request2 = Request(inputTokens, maxNewTokens, streaming, samplingConfig2, outConfig); - auto samplingConfig3 = SamplingConfig(4); - samplingConfig3.setNumReturnSequences(2); - auto request3 = Request(inputTokens, maxNewTokens, streaming, samplingConfig3, outConfig); - - // Enqueue the request - auto requestId1 = executor.enqueueRequest(request1); - auto requestId2 = executor.enqueueRequest(request2); - auto requestId3 = executor.enqueueRequest(request3); - - // Expecting one response in beam search. Instead, numReturnSequences limits the number of beams to return. - std::unordered_map<IdType, SizeType32> expectedNumResponses{{requestId1, 3}, {requestId2, 1}, {requestId3, 1}}; - std::unordered_map<IdType, SizeType32> const expectedNumBeams{{requestId1, 1}, {requestId2, 4}, {requestId3, 2}}; - - std::unordered_map<IdType, SizeType32> numResponses{{requestId1, 0}, {requestId2, 0}, {requestId3, 0}}; - std::unordered_map<IdType, SizeType32> numBeams{{requestId1, 0}, {requestId2, 0}, {requestId3, 0}}; - int numFinished = 0; - int iter = 0; - while (numFinished < 3 && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - auto const& result = response.getResult(); - auto reqId = response.getRequestId(); - numFinished += result.isFinal; - numResponses[reqId]++; - numBeams[reqId] = result.outputTokenIds.size(); - } - ++iter; - } - - EXPECT_LT(iter, mMaxWaitMs); - EXPECT_EQ(numFinished, 3); - for (auto& [reqId, numResp] : numResponses) - { - EXPECT_EQ(numResp, expectedNumResponses[reqId]); - } - for (auto& [reqId, numResp] : numResponses) - { - EXPECT_EQ(numResp, expectedNumResponses[reqId]); - } -} - -INSTANTIATE_TEST_SUITE_P(GptExecutorTest, ParamTest, - testing::Combine(testing::Values(false, true), // streaming - testing::Values(false, true), // excludeInputFromOutput - testing::Values(1, 2) // beamWidth - ), - generateTestName); diff --git a/cpp/tests/e2e_tests/executor/executorTest.cpp b/cpp/tests/e2e_tests/executor/executorTest.cpp deleted file mode 100644 index e1227970cb71..000000000000 --- a/cpp/tests/e2e_tests/executor/executorTest.cpp +++ /dev/null @@ -1,4671 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TOP_LEVEL_DIR -#error "Define TOP_LEVEL_DIR" -#endif - -#include "executorTest.h" - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/executor/dataTransceiverState.h" -#include "tensorrt_llm/executor/requestWithId.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/executor/version.h" -#include "tensorrt_llm/runtime/gptJsonConfig.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "tensorrt_llm/runtime/utils/numpyUtils.h" -#include "tensorrt_llm/testing/modelSpec.h" -#include "tests/utils/common.h" - -#include <gmock/gmock.h> -#include <gtest/gtest.h> -#include <nlohmann/json.hpp> - -#include <algorithm> -#include <chrono> -#include <cstddef> -#include <functional> -#include <memory> -#include <string> -#include <thread> -#include <vector> - -namespace tr = tensorrt_llm::runtime; -namespace tc = tensorrt_llm::common; - -using namespace tensorrt_llm::testing; -using namespace tensorrt_llm::executor; -using namespace std::chrono_literals; -namespace fs = std::filesystem; -using tensorrt_llm::testing::KVCacheType; -using tensorrt_llm::testing::ModelSpec; - -namespace -{ - -auto const LORA_DATA_PATH = DATA_PATH / "lora-test-weights-gpt2-tp1"; -auto const LORA_WEIGHTS_FILE = LORA_DATA_PATH / "source.npy"; -auto const LORA_CONFIG_FILE = LORA_DATA_PATH / "config.npy"; - -auto constexpr LLAMA_INPUT_FILE = "input_tokens_llama.npy"; -auto constexpr LLAMA_VOCAB_SIZE_PADDED = 128256; -auto constexpr LLAMA_PAD_ID = 128001; -auto constexpr LLAMA_END_ID = 128001; - -} // namespace - -void testInvalidCtor(std::filesystem::path const& enginePath, ModelType modelType, ExecutorConfig executorConfig, - std::string expectedErrMsg = "") -{ - try - { - auto executor = Executor(enginePath, modelType, executorConfig); - - FAIL() << "Expected TllmException"; - } - catch (std::exception const& e) - { - EXPECT_THAT(e.what(), testing::HasSubstr(expectedErrMsg)); - } -} - -TEST_F(GptExecutorTest, version) -{ - EXPECT_STRNE(kTensorRtLlmVersion, "@TRTLLM_VERSION@"); - EXPECT_STREQ(kTensorRtLlmVersion, version()); -} - -TEST_F(GptExecutorTest, validCtor) -{ - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); -} - -TEST_F(GptExecutorTest, invalidCtor) -{ - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - std::filesystem::path invalidPath{"Bla"}; - - // Invalid path - { - testInvalidCtor(invalidPath, ModelType::kDECODER_ONLY, executorConfig, "File does not exist"); - } -} - -TEST_F(GptExecutorTest, enqueueAfterShutdown) -{ - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - SizeType32 maxNewTokens = 5; - VecTokens inputTokens{1, 2, 3, 4}; - auto request = Request(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); - auto requestId = executor.enqueueRequest(request); - - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - FAIL(); - } - else - { - done = response.getResult().isFinal; - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - - executor.shutdown(); - - EXPECT_FALSE(executor.canEnqueueRequests()); - - std::string expErrMsg{"Shutdown called"}; - EXPECT_THAT([&]() { auto reqId = executor.enqueueRequest(request); }, - testing::Throws<tensorrt_llm::common::TllmException>( - testing::Property(&tensorrt_llm::common::TllmException::what, testing::HasSubstr(expErrMsg)))); - EXPECT_THAT([&]() { auto resp = executor.awaitResponses(); }, - testing::Throws<tensorrt_llm::common::TllmException>( - testing::Property(&tensorrt_llm::common::TllmException::what, testing::HasSubstr(expErrMsg)))); - EXPECT_THAT([&]() { auto stats = executor.getLatestIterationStats(); }, - testing::Throws<tensorrt_llm::common::TllmException>( - testing::Property(&tensorrt_llm::common::TllmException::what, testing::HasSubstr(expErrMsg)))); - EXPECT_THAT([&]() { auto stats = executor.getLatestRequestStats(); }, - testing::Throws<tensorrt_llm::common::TllmException>( - testing::Property(&tensorrt_llm::common::TllmException::what, testing::HasSubstr(expErrMsg)))); - EXPECT_THAT([&]() { executor.cancelRequest(requestId); }, - testing::Throws<tensorrt_llm::common::TllmException>( - testing::Property(&tensorrt_llm::common::TllmException::what, testing::HasSubstr(expErrMsg)))); -} - -TEST_F(GptExecutorTest, missingPeftTask) -{ - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_LORA_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - SizeType32 maxNewTokens = 5; - VecTokens inputTokens{1, 2, 3, 4}; - auto request = Request(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); - auto loraConfig = LoraConfig{10}; - request.setLoraConfig(loraConfig); - - auto requestId = executor.enqueueRequest(request); - - bool done = false; - std::chrono::milliseconds waitTime(mMaxWaitMs); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - auto err = response.getErrorMsg(); - EXPECT_EQ(err, std::string("LoRA task 10 not found in cache. Please send LoRA weights with request")); - done = true; - } - else - { - FAIL() << "Expects error due to missing Lora weights"; - } - } - EXPECT_TRUE(done); -} - -TEST_F(GptExecutorTest, ReturnAcceptedTokenLogits) -{ - SizeType32 constexpr beamWidth{1}; - SizeType32 constexpr vocabSizePadded{50257}; // gpt vocabSizePadded - - // Create executor config - auto executorConfig = ExecutorConfig(beamWidth); - executorConfig.setGatherGenerationLogits(true); - - // Enable kv cache reuse of executorConfig - bool enableBlockReuse = true; - FloatType freeGpuMemoryFraction = 0.4; - auto kvCacheConfig - = KvCacheConfig(enableBlockReuse, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction); - executorConfig.setKvCacheConfig(kvCacheConfig); - - // Create executor - auto trtEnginePath - = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DRAFT_TOKENS_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create request - SizeType32 maxNewTokens = 5; - VecTokens inputTokens{1, 2, 3, 4, 5, 6, 7, 8}; - - std::vector<bool> streamingOptions{false, true}; - - for (auto streaming : streamingOptions) - { - auto request = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth)); - - // Set draft tokens - auto draftTokens = VecTokens{9, 10, 11, 12, 13}; // draft tokens - auto draftLength = draftTokens.size(); - FloatType const acceptanceThreshold = 0.00001f; // Ensure the draft token can be accepted - auto externalDraftTokensConfig = ExternalDraftTokensConfig(draftTokens, std::nullopt, acceptanceThreshold); - request.setExternalDraftTokensConfig(externalDraftTokensConfig); - - // Set return accepted token logits for this request - OutputConfig outConfig; - outConfig.returnGenerationLogits = true; - request.setOutputConfig(outConfig); - - // Enqueue this request - auto requestId = executor.enqueueRequest(request); - - bool done = false; - int iter = 0; - while (!done && iter < 5000) - { - std::chrono::milliseconds waitTime(mMaxWaitMs); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - FAIL(); - } - else - { - auto result = response.getResult(); - done = result.isFinal; - auto& genLogits = result.generationLogits; - EXPECT_TRUE(genLogits.has_value()); - - // Expected shape: (1, numAcceptedDraftToken, vocabSizePadded) - auto const& acceptedTokenLogitsShape = genLogits->getShape(); - EXPECT_EQ(acceptedTokenLogitsShape.size(), 3); - EXPECT_EQ(acceptedTokenLogitsShape[0], 1); - EXPECT_LE(acceptedTokenLogitsShape[1], draftLength); // number of accepted tokens - EXPECT_EQ(acceptedTokenLogitsShape[2], vocabSizePadded); // vocabSizePadded - } - } - ++iter; - } - } -} - -TEST_F(GptExecutorTest, GenerationLogitsEarlyStop) -{ - SizeType32 constexpr beamWidth{1}; - SizeType32 constexpr vocabSizePadded{50257}; // gpt vocabSizePadded - auto constexpr streaming = false; - - ExtendedRuntimePerfKnobConfig perfKnobConfig = ExtendedRuntimePerfKnobConfig(); - - // Create executor config - auto executorConfig = ExecutorConfig(beamWidth); - executorConfig.setExtendedRuntimePerfKnobConfig(perfKnobConfig); - executorConfig.setGatherGenerationLogits(true); - - // Create executor - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - auto const inputPath = DATA_PATH / "input_tokens.npy"; - ModelIds modelIds{50256, 50256}; - - auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); - auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); - auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); - auto const* const givenInputData = tr::bufferCast<TokenIdType const>(*givenInput); - - auto const& inputShape = givenInput->getShape(); - ASSERT_EQ(inputShape.nbDims, 2); - ASSERT_GT(inputShape.d[0], 0); - - BeamResult beamResult{beamWidth}; - auto const resultsPath - = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); - beamResult.contextLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); - beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); - - // Set return generation logits for this request - OutputConfig outConfig; - outConfig.returnGenerationLogits = true; - outConfig.excludeInputFromOutput = true; - - // Load expected outputs for each beam width value - auto testData = TestData::loadTestData(beamResult, *givenInput, beamWidth, manager, outConfig, modelIds); - auto const maxSeqLen = testData.maxSeqLen; - - // Load expected outputs and inputs - std::vector<Request> requests; - std::vector<SizeType32> reqMaxNewTokens; - - auto constexpr reqIdx = 0; - SizeType32 inputLen = givenInputLengths.at(reqIdx); - auto maxNewTokens = maxSeqLen - maxInputLength; - reqMaxNewTokens.push_back(maxNewTokens); - auto const* const seqBegin = givenInputData + reqIdx * maxInputLength; - - auto request = Request(VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, modelIds.endId); - // copy request - auto request2 = request; - - auto const expectedOutputData = tr::BufferRange<TokenIdType const>(*testData.expectedOutputIds); - auto const expectedOutputLengths = testData.expectedOutputLengths; - auto const endPos = expectedOutputLengths[reqIdx] - 3; - auto const endIndex = tc::flat_index3(reqIdx, beamWidth - 1, endPos, beamWidth, maxSeqLen); - auto const endToken = expectedOutputData[endIndex]; - - // Set end id to stop early - request.setEndId(endToken); - requests.emplace_back(std::move(request)); - - // Set stop words to stop early - request2.setStopWords({{endToken}}); - requests.emplace_back(std::move(request2)); - - // Enqueue requests - auto requestIds = executor.enqueueRequests(requests); - - std::map<IdType, SizeType32> expectedNewTokens; - expectedNewTokens[requestIds.at(0)] = endPos - inputLen; - expectedNewTokens[requestIds.at(1)] = endPos - inputLen + 1; - - std::map<IdType, FinishReason> expectedFinishReason; - expectedFinishReason[requestIds.at(0)] = FinishReason::kEND_ID; - expectedFinishReason[requestIds.at(1)] = FinishReason::kSTOP_WORDS; - - std::map<IdType, bool> done; - std::for_each(requestIds.begin(), requestIds.end(), [&done](auto id) { done[id] = false; }); - int iter = 0; - while (!(std::all_of(done.begin(), done.end(), [](auto x) { return x.second; })) && iter < 5000) - { - std::chrono::milliseconds waitTime(mMaxWaitMs); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - FAIL(); - } - else - { - auto const reqId = response.getRequestId(); - auto const& result = response.getResult(); - EXPECT_TRUE(result.isFinal); - done.at(reqId) = result.isFinal; - - // only 1 beam - auto const& outputIds = result.outputTokenIds.at(0); - EXPECT_EQ(outputIds.size(), expectedNewTokens.at(reqId)) << "req " << reqId; - - auto const& finishReason = result.finishReasons.at(0); - EXPECT_EQ(finishReason, expectedFinishReason.at(reqId)) << "req " << reqId; - - auto const& genLogits = result.generationLogits; - EXPECT_TRUE(genLogits.has_value()); - - // Expected shape: (1, numAcceptedDraftToken, vocabSizePadded) - auto const& generationLogitsShape = genLogits->getShape(); - EXPECT_EQ(generationLogitsShape.size(), 3); - EXPECT_EQ(generationLogitsShape[0], 1); - EXPECT_LE(generationLogitsShape[1], maxNewTokens); - EXPECT_EQ(generationLogitsShape[2], vocabSizePadded); - - auto const genLogitsTensor = detail::toITensor(*genLogits); - genLogitsTensor->squeeze(0); // only 1 beam - - for (size_t outputIdx = 0; outputIdx < expectedNewTokens.at(reqId); ++outputIdx) - { - // logits argmax should be equal to tokenId - auto const genLogitsSlice = tr::ITensor::slice(genLogitsTensor, outputIdx, 1); - auto const genLogitsRange = tr::BufferRange<float>(*genLogitsSlice); - auto const* maxPos = std::max_element(genLogitsRange.begin(), genLogitsRange.end()); - auto const maxIdx = std::distance(genLogitsRange.begin(), maxPos); - - auto const tokenId = outputIds.at(outputIdx); - // Observed token mismatch at index 2 after building GPT engine with TRT builder optimization - // level 3. The testcase is sensitive to slight variation in kernel computation, so we skip checking - // for token id at index 2. - if (outputIdx != 2) - { - EXPECT_EQ(tokenId, maxIdx) << "req " << reqId << " outputIdx " << outputIdx; - } - } - } - } - ++iter; - } -} - -TEST_F(GptExecutorTest, GenerationChangeEndId) -{ - SizeType32 constexpr beamWidth{2}; - SizeType32 constexpr vocabSizePadded{50257}; // gpt vocabSizePadded - auto constexpr streaming = false; - - ExtendedRuntimePerfKnobConfig perfKnobConfig = ExtendedRuntimePerfKnobConfig(); - perfKnobConfig.setEnableContextFMHAFP32Acc(true); // use fmha fp32 acc for better accuracy - - // Create executor config - auto executorConfig = ExecutorConfig(beamWidth); - executorConfig.setExtendedRuntimePerfKnobConfig(perfKnobConfig); - - // Create executor - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - auto const inputPath = DATA_PATH / "input_tokens.npy"; - ModelIds modelIds{50256, 50256}; - - auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); - auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); - auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); - auto const* const givenInputData = tr::bufferCast<TokenIdType const>(*givenInput); - - auto const& inputShape = givenInput->getShape(); - ASSERT_EQ(inputShape.nbDims, 2); - ASSERT_GT(inputShape.d[0], 0); - - BeamResult beamResult{beamWidth}; - auto const resultsPath - = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_CONTEXTFMHAFP32ACC_RESULT_FILE(); - - // Just return tokens for check - OutputConfig outConfig; - outConfig.excludeInputFromOutput = true; - - // Load expected outputs for each beam width value - auto testData = TestData::loadTestData(beamResult, *givenInput, beamWidth, manager, outConfig, modelIds); - auto const maxSeqLen = testData.maxSeqLen; - - // Load expected outputs and inputs - std::vector<Request> requests; - std::vector<SizeType32> reqMaxNewTokens; - - // Only use the first request to test - auto constexpr reqIdx = 0; - SizeType32 inputLen = givenInputLengths.at(reqIdx); - auto maxNewTokens = maxSeqLen - maxInputLength; - reqMaxNewTokens.push_back(maxNewTokens); - auto const* const seqBegin = givenInputData + reqIdx * maxInputLength; - - // Use customized `EndId` to enqueue once - auto request = Request(VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, modelIds.endId); - - TokenIdType customizedEndId = *(seqBegin + 1); // Use a token appeared in ground-truth - request.setEndId(customizedEndId); - requests.emplace_back(std::move(request)); - - auto requestIds = executor.enqueueRequests(requests); - std::chrono::milliseconds waitTime(mMaxWaitMs); - auto responses = executor.awaitResponses(waitTime); - if (responses.at(0).hasError()) - { - FAIL(); - } - requests.clear(); - - // Change back to default `EndId` to enqueue again, and check the output - request = Request(VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, modelIds.endId); - - auto const expectedOutputData = tr::BufferRange<TokenIdType const>(*testData.expectedOutputIds); - auto const expectedOutputLengths = testData.expectedOutputLengths; - auto const endPos = expectedOutputLengths[reqIdx]; - auto const endIndex = tc::flat_index3(reqIdx, beamWidth, endPos, beamWidth, maxSeqLen); - auto const endToken = expectedOutputData[endIndex]; - - request.setEndId(endToken); - requests.emplace_back(std::move(request)); - requestIds = executor.enqueueRequests(requests); - auto const requestId = requestIds.at(0); - - std::map<IdType, SizeType32> expectedNewTokens; - expectedNewTokens[requestId] = endPos - inputLen; - - std::map<IdType, FinishReason> expectedFinishReason; - expectedFinishReason[requestId] = FinishReason::kLENGTH; - - std::map<IdType, bool> done; - std::for_each(requestIds.begin(), requestIds.end(), [&done](auto id) { done[id] = false; }); - int iter = 0; - while (!(std::all_of(done.begin(), done.end(), [](auto x) { return x.second; })) && iter < 5000) - { - std::chrono::milliseconds waitTime(mMaxWaitMs); - auto responses = executor.awaitResponses(waitTime); - auto& response = responses.at(0); - if (response.hasError()) - { - FAIL(); - } - else - { - auto const reqId = response.getRequestId(); - auto const& result = response.getResult(); - EXPECT_TRUE(result.isFinal); - done.at(reqId) = result.isFinal; - - bool anyMismatch = false; - for (int i = 0; i < result.outputTokenIds.size(); ++i) - { - auto const& outputIds = result.outputTokenIds.at(i); - EXPECT_EQ(outputIds.size(), expectedNewTokens.at(reqId)) << "req " << reqId; - anyMismatch |= outputIds.size() != expectedNewTokens.at(reqId); - - auto const& finishReason = result.finishReasons.at(i); - EXPECT_EQ(finishReason, expectedFinishReason.at(reqId)) << "req " << reqId; - anyMismatch |= finishReason != expectedFinishReason.at(reqId); - - if (anyMismatch) - { - break; - } - - for (int j = 0; j < outputIds.size(); ++j) - { - auto const resultToken = outputIds[j]; - auto const groundTruthToken = expectedOutputData[maxSeqLen * i + inputLen + j]; - EXPECT_EQ(resultToken, groundTruthToken); - anyMismatch |= resultToken != groundTruthToken; - } - } - EXPECT_FALSE(anyMismatch); - } - ++iter; - } -} - -// stream, excludeInputFromOutput, beamWidth -using ParamType = std::tuple<bool, bool, int>; -// useOrchestratorMode, beamWidth, modelName -using ParamCancelReqType = std::tuple<bool, int, std::string>; -// modelName -using LeaderApiUsageType = std::tuple<std::string>; -// iterStatsMaxIterations, useOrchestratorMode -using ParamStatsType = std::tuple<int, bool>; -// streaming, beamWidth, computeLogProbs, excludeInputInOutput, returnContextLogits, returnGenerationLogits, modelName, -// useOrchestratorMode, returnAllGeneratedTokens, numReturnSequences -using AllParamsType = std::tuple<bool, int, bool, bool, bool, bool, std::string, bool, bool, int>; -// modelName, batched, replicated -using LogitsProcParamsType = std::tuple<std::string, bool, bool>; -// modelName -using GuidedDecodingParamsType = std::tuple<std::string>; -// modelName, useOrchestratorMode, beamWidth -using TimeoutTestParamsType = std::tuple<std::string, bool, int>; - -std::string generateTestName(testing::TestParamInfo<ParamType> const& info) -{ - auto const streaming = std::get<0>(info.param); - auto const excludeInputFromOutput = std::get<1>(info.param); - auto const beamWidth = std::get<2>(info.param); - std::string name = "ExecutorTest"; - if (streaming) - { - name += "Streaming"; - } - if (excludeInputFromOutput) - { - name += "ExclInput"; - } - name.append("BW" + std::to_string(beamWidth)); - return name; -} - -std::string generateTestNameCancelReq(testing::TestParamInfo<ParamCancelReqType> const& info) -{ - auto const& useOrchestratorMode = std::get<0>(info.param); - auto const beamWidth = std::get<1>(info.param); - auto const modelName = std::get<2>(info.param); - std::string name = "ExecutorTest"; - name.append("BW" + std::to_string(beamWidth)); - name.append("_" + modelName + "_"); - - if (useOrchestratorMode) - { - name.append("OrchMode"); - } - else - { - name.append("LeaderMode"); - } - return name; -} - -std::string generateTestNameLeaderApiUsage(testing::TestParamInfo<LeaderApiUsageType> const& info) -{ - auto const modelName = std::get<0>(info.param); - std::string name = "ExecutorTest"; - name.append("_" + modelName); - return name; -} - -std::string generateTestNameLogitsProc(testing::TestParamInfo<LogitsProcParamsType> const& info) -{ - auto const modelName = std::get<0>(info.param); - bool const batched = std::get<1>(info.param); - bool const replicated = std::get<2>(info.param); - std::string name = "ExecutorTest"; - name.append("_" + modelName); - if (batched) - { - name.append("_Batched"); - } - if (replicated) - { - name.append("_Replicated"); - } - return name; -} - -std::string generateTestNameGuidedDecoding(testing::TestParamInfo<GuidedDecodingParamsType> const& info) -{ - auto const modelName = std::get<0>(info.param); - std::string name = "ExecutorTest"; - name.append("_" + modelName); - return name; -} - -std::string generateTestNameTimeoutTest(testing::TestParamInfo<TimeoutTestParamsType> const& info) -{ - auto const modelName = std::get<0>(info.param); - auto const& useOrchestratorMode = std::get<1>(info.param); - auto const beamWidth = std::get<2>(info.param); - - std::string name = "ExecutorTest"; - name.append("_" + modelName); - - if (useOrchestratorMode) - { - name.append("_OrchMode"); - } - else - { - name.append("_LeaderMode"); - } - name.append("_BW" + std::to_string(beamWidth)); - return name; -} - -std::string generateTestNameStats(testing::TestParamInfo<ParamStatsType> const& info) -{ - int iterStatsMaxIterations = std::get<0>(info.param); - auto const& useOrchestratorMode = std::get<1>(info.param); - std::string name = "ExecutorTest_"; - name.append(std::to_string(iterStatsMaxIterations) + "_"); - if (useOrchestratorMode) - { - name.append("OrchMode"); - } - else - { - name.append("LeaderMode"); - } - return name; -} - -std::string generateTestNameAllParams(testing::TestParamInfo<AllParamsType> const& info) -{ - auto const streaming = std::get<0>(info.param); - auto const& beamWidth = std::get<1>(info.param); - auto const& computeLogProbs = std::get<2>(info.param); - auto const& excludeInputInOutput = std::get<3>(info.param); - auto const& returnContextLogits = std::get<4>(info.param); - auto const& returnGenerationLogits = std::get<5>(info.param); - auto const modelName = std::get<6>(info.param); - auto const& useOrchestratorMode = std::get<7>(info.param); - auto const& returnAllGeneratedTokens = std::get<8>(info.param); - auto const& numReturnSequences = std::get<9>(info.param); - - std::string name = "ExecutorTest_"; - - if (streaming) - { - name += "Streaming"; - } - - name.append("_BW" + std::to_string(beamWidth)); - name.append("Nseq" + std::to_string(numReturnSequences)); - - if (computeLogProbs) - { - name.append("LogProbs"); - } - if (excludeInputInOutput) - { - name.append("ExcludeInput"); - } - if (returnContextLogits) - { - name.append("ContextLogits"); - } - if (returnGenerationLogits) - { - name.append("GenerationLogits"); - } - name.append("_" + modelName + "_"); - if (useOrchestratorMode) - { - name.append("OrchMode"); - } - else - { - name.append("LeaderMode"); - } - - if (returnAllGeneratedTokens) - { - name.append("returnAllGeneratedTokens"); - } - return name; -} - -class ParamTest : public GptExecutorTest, public ::testing::WithParamInterface<ParamType> -{ -}; - -class ParamStatsTest : public GptExecutorTest, public ::testing::WithParamInterface<ParamStatsType> -{ -}; - -class AllParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<AllParamsType> -{ -}; - -class ParamCancelReqTest : public GptExecutorTest, public ::testing::WithParamInterface<ParamCancelReqType> -{ -}; - -class LeaderApiUsageTest : public GptExecutorTest, public ::testing::WithParamInterface<LeaderApiUsageType> -{ -}; - -class LogitsProcParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<LogitsProcParamsType> -{ -}; - -class GuidedDecodingParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<GuidedDecodingParamsType> -{ -}; - -class TimeoutTest : public GptExecutorTest, public ::testing::WithParamInterface<TimeoutTestParamsType> -{ -}; - -TEST_F(GptExecutorTest, GetLatestStats) -{ - bool streaming = false; - bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - SizeType32 maxNewTokens = 5; - VecTokens inputTokens{1, 2, 3, 4}; - auto request - = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - auto requestId = executor.enqueueRequest(std::move(request)); - - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - FAIL(); - } - else - { - done = response.getResult().isFinal; - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - - // Expect 6 non-empty iterations - auto stats = executor.getLatestIterationStats(); - EXPECT_EQ(stats.size(), 6); - uint64_t currentIter = 0; - for (auto const& stat : stats) - { - EXPECT_EQ(stat.timestamp.size(), 26); - EXPECT_EQ(stat.iter, currentIter); - if (currentIter != 5) - { - EXPECT_EQ(stat.numActiveRequests, 1); - } - else - { - // For the last iteration the number of active requests - // should be zero. - EXPECT_EQ(stat.numActiveRequests, 0); - } - EXPECT_EQ(stat.maxNumActiveRequests, 64); - // Very loose check to make sure the memory stats are valid - EXPECT_GT(stat.gpuMemUsage, 16); - EXPECT_GT(stat.cpuMemUsage, 16); - EXPECT_GT(stat.pinnedMemUsage, 16); - - // Stats for KV cache - EXPECT_TRUE(stat.kvCacheStats.has_value()); - KvCacheStats const& kvStats = stat.kvCacheStats.value(); - EXPECT_GT(kvStats.maxNumBlocks, 0); - EXPECT_GT(kvStats.freeNumBlocks, 0); - EXPECT_EQ(kvStats.usedNumBlocks, currentIter == maxNewTokens ? 0 : 1); - EXPECT_GT(kvStats.tokensPerBlock, 0); - EXPECT_GT(kvStats.allocTotalBlocks, 0); - EXPECT_GT(kvStats.allocNewBlocks, 0); - EXPECT_GE(kvStats.reusedBlocks, 0); - EXPECT_GE(kvStats.missedBlocks, 0); - EXPECT_GE(kvStats.cacheHitRate, 0); - - // Stats for inflight batching - EXPECT_TRUE(stat.inflightBatchingStats.has_value() && !stat.staticBatchingStats.has_value()); - InflightBatchingStats const& modelStats = stat.inflightBatchingStats.value(); - EXPECT_EQ(modelStats.numScheduledRequests, currentIter == maxNewTokens ? 0 : 1); - EXPECT_EQ(modelStats.numContextRequests, currentIter == 0 ? 1 : 0); - EXPECT_EQ(modelStats.numGenRequests, currentIter == 0 || currentIter == maxNewTokens ? 0 : 1); - EXPECT_EQ(modelStats.numPausedRequests, 0); - EXPECT_EQ(modelStats.numCtxTokens, currentIter == 0 ? inputTokens.size() : 0); - EXPECT_EQ(modelStats.microBatchId, 0); - EXPECT_NEAR( - modelStats.avgNumDecodedTokensPerIter, currentIter == 0 || currentIter == maxNewTokens ? 0.f : 1.f, 1e-9f); - - auto jsonStr = JsonSerialization::toJsonStr(stat); - EXPECT_THAT(jsonStr, testing::HasSubstr("\"iter\":" + std::to_string(currentIter))); - EXPECT_THAT(jsonStr, testing::HasSubstr("\"staticBatchingStats\":null")); - EXPECT_THAT(jsonStr, testing::HasSubstr("\"numCtxTokens\":" + std::to_string(modelStats.numCtxTokens))); - EXPECT_THAT(jsonStr, testing::HasSubstr("\"numGenRequests\":" + std::to_string(modelStats.numGenRequests))); - - ++currentIter; - } -} - -TEST_F(GptExecutorTest, GetLatestStatsWithMultipleRequests) -{ - bool streaming = false; - bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the requests - SizeType32 const numRequests = 2; - std::vector<SizeType32> maxNewTokens{3, 5}; - std::vector<VecTokens> inputTokens{{1, 2, 3, 4}, {5, 6, 7}}; - std::vector<IdType> reqIds; - for (SizeType32 ireq = 0; ireq < numRequests; ++ireq) - { - auto request = Request(inputTokens[ireq], maxNewTokens[ireq], streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - auto requestId = executor.enqueueRequest(std::move(request)); - reqIds.emplace_back(requestId); - // sleep for 10 ms before sending the next request - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - } - - for (SizeType32 ireq = 0; ireq < numRequests; ++ireq) - { - auto requestId = reqIds[ireq]; - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - FAIL(); - } - else - { - done = response.getResult().isFinal; - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - } - - // NOTES: - // Expect at least max(maxNewTokens) i.e. 5 non-empty iterations - // 4th iteration should have numCompletedRequests to be 1. - // Depending on the timing, first iteration will either have: - // 2 active requests - // or - // 1 active requests and 1 queued requests - auto stats = executor.getLatestIterationStats(); - EXPECT_GT(stats.size(), 0); // make sure we have at least 1 stat before the accessing 0-th element - if (stats[0].numActiveRequests == 2) - { - // we cannot reliably check queue latency since both started in the same iteration - // there should be exactly 5 non-empty iterations - EXPECT_EQ(stats.size(), 5); - // only check numCompletedRequests in 4th iteration - EXPECT_EQ(stats[3].numCompletedRequests, 1); - // 1st iteration shall record all 2 requests queueing time; - EXPECT_EQ(stats[0].numNewActiveRequests, 2); - // all rest iterations shall not return any queueing time; - for (int i = 1; i < stats.size(); ++i) - { - EXPECT_EQ(stats[i].numNewActiveRequests, 0); - } - } - else - { - // there should be more than 5 non-empty iterations since 2nd request started after 1st iteration - EXPECT_GT(stats.size(), 5); - // 1st request's completion is at 4th iteration - EXPECT_EQ(stats[3].numCompletedRequests, 1); - // 1st iteration record 1 request's queueing time; - EXPECT_EQ(stats[0].numNewActiveRequests, 1); - // the iteration where 2nd request became active, queue latency must be > 0 - uint64_t currentIter = 0; - for (auto const& stat : stats) - { - // To check when 2nd request becomes active, we need to think about 2 cases: - // - it overlaps with first request - // => only check queue time in this case - // - it doesn't overlap with the first request (e.g. 1st request ended too fast) - // => little to no queue time, cannot check reliably - // so we only check for queue time when numActiveRequests > 1 i.e. overlap happened after first iteration - if (stat.numActiveRequests > 1) - { - EXPECT_GT(currentIter, 0); // it must be after 1st iteration - EXPECT_GT(stat.newActiveRequestsQueueLatencyMS, 0); - // 2nd request record queueing time in this iteration - EXPECT_EQ(stat.numNewActiveRequests, 1); - break; - } - ++currentIter; - } - } -} - -TEST_F(GptExecutorTest, GetLatestRequestStats) -{ - bool streaming = false; - bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - executorConfig.setRequestStatsMaxIterations(1000); - executorConfig.setEnableChunkedContext(true); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the requests - std::vector<std::pair<SizeType32, VecTokens>> requestParams = { - // {maxNewTokens, inputTokens} - {5, {1, 2, 3, 4}}, {4, {1, 1, 2, 3, 5}}, {1, {1}}, - {8, VecTokens(383, 1)} // Long enough to be chunked into multiple iterations - }; - std::vector<Request> requests; - for (auto requestParam : requestParams) - { - requests.emplace_back(requestParam.second, requestParam.first, streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - } - auto requestIdsVec = executor.enqueueRequests(std::move(requests)); - std::map<IdType, SizeType32> requestIdToIndex; - std::set<IdType> activeRequests; - for (SizeType32 i = 0; i < requestIdsVec.size(); ++i) - { - auto requestId = requestIdsVec[i]; - activeRequests.insert(requestId); - requestIdToIndex[requestId] = i; - } - - int iter = 0; - while (!activeRequests.empty() && iter < mMaxWaitMs) - { - for (auto i = activeRequests.begin(); i != activeRequests.end();) - { - auto requestId = *i; - bool thisDone = false; - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - else - { - thisDone = response.getResult().isFinal; - } - } - if (thisDone) - { - // Erase completed request and move to the next one - i = activeRequests.erase(i); - } - else - { - ++i; - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - - // Expect 5 non-empty iterations - // Note: The 6th iteration with the last finished request will be reported - // but might be unavailable when getLatestRequestStats is called since - // it could be updated after the final response has been sent. - auto stats = executor.getLatestRequestStats(); - EXPECT_GE(stats.size(), 5); - SizeType32 currentIter = 0; - auto invalidStart = std::numeric_limits<SizeType32>::max(); - std::vector<SizeType32> genStart(requestParams.size(), invalidStart); // The iteration index when generation started - std::set<IdType> completedRequests; - for (auto stat = stats.begin(); stat != stats.begin() + 5; ++stat) - { - auto jsonStrIter = JsonSerialization::toJsonStr(*stat); - EXPECT_EQ(stat->iter, currentIter); - EXPECT_THAT(jsonStrIter, testing::HasSubstr("\"iter\":" + std::to_string(currentIter))); - EXPECT_EQ(stat->requestStats.size() + completedRequests.size(), requestParams.size()); - for (auto rStat : stat->requestStats) - { - auto jsonStr = JsonSerialization::toJsonStr(rStat); - // Only a few requests here so all of them should be scheduled. A separate test - // GetLatestRequestStatsScheduling will target the scheduling stats. - if (rStat.stage != RequestStage::kGENERATION_COMPLETE) - { - EXPECT_TRUE(rStat.scheduled); - EXPECT_THAT(jsonStr, testing::HasSubstr("\"scheduled\":true")); - } - EXPECT_TRUE(!rStat.paused); - EXPECT_THAT(jsonStr, testing::HasSubstr("\"paused\":false")); - EXPECT_TRUE(requestIdToIndex.count(rStat.id)); - EXPECT_THAT(jsonStr, testing::HasSubstr("\"id\":" + std::to_string(rStat.id))); - auto requestIndex = requestIdToIndex[rStat.id]; - auto contextSize = requestParams[requestIndex].second.size(); - if (rStat.contextPrefillPosition == contextSize) // Check generation phase - { - bool firstIteration{false}; - // Context phase is done - EXPECT_TRUE(rStat.stage == RequestStage::kGENERATION_IN_PROGRESS - || rStat.stage == RequestStage::kGENERATION_COMPLETE); - EXPECT_THAT(jsonStr, testing::HasSubstr("\"stage\":\"GENERATION")); - if (genStart[requestIndex] == invalidStart) - { - // Just started generation - genStart[requestIndex] = currentIter; - firstIteration = true; - } - - // One token per iteration - EXPECT_TRUE(currentIter - genStart[requestIndex] == rStat.numGeneratedTokens); - EXPECT_NEAR(rStat.avgNumDecodedTokensPerIter, firstIteration ? 0.f : 1.0f, 1e-9); - if (rStat.stage == RequestStage::kGENERATION_COMPLETE) - { - EXPECT_TRUE(requestParams[requestIndex].first >= rStat.numGeneratedTokens); - completedRequests.insert(requestIndex); - } - else - { - EXPECT_FALSE(completedRequests.count(requestIndex)); - } - } - else if (rStat.contextPrefillPosition < contextSize) // Check context phase - { - // Must be chunked - SizeType32 const maxChunkSize = 128; - EXPECT_TRUE(rStat.contextPrefillPosition % maxChunkSize == 0); - // Context phase is on-going - EXPECT_TRUE(rStat.stage == RequestStage::kCONTEXT_IN_PROGRESS); - // No tokens are generated - EXPECT_TRUE(0 == rStat.numGeneratedTokens); - } - else - { - FAIL() << "Out-of-boundary contextPrefillPosition in stats: " << rStat.contextPrefillPosition - << " out of " << contextSize; - } - // Sanity check that disaggregated serving stats is not set in typical use case - EXPECT_FALSE(rStat.disServingStats.has_value()); - } - ++currentIter; - } - // We should have visited all requests. - // Take into consideration the last request has not been reported - EXPECT_EQ(completedRequests.size() + 1, requestParams.size()); -} - -TEST_F(GptExecutorTest, GetLatestRequestStatsScheduling) -{ - // Specifically test the case where there are too many requests to be scheduled for a iteration - bool streaming = false; - bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - executorConfig.setRequestStatsMaxIterations(1000); - executorConfig.setEnableChunkedContext(true); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create 100 requests. Note the max batch size for this model is 64 so some requests won't be scheduled right away. - std::vector<std::pair<SizeType32, VecTokens>> requestParams(100, {5, {1, 2, 3, 4}}); - std::vector<Request> requests; - requests.reserve(requestParams.size()); - for (auto requestParam : requestParams) - { - requests.emplace_back(requestParam.second, requestParam.first, streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - } - auto requestIdsVec = executor.enqueueRequests(std::move(requests)); - std::map<IdType, SizeType32> requestIdToIndex; - std::set<IdType> activeRequests; - for (SizeType32 i = 0; i < requestIdsVec.size(); ++i) - { - auto requestId = requestIdsVec[i]; - activeRequests.insert(requestId); - requestIdToIndex[requestId] = i; - } - - int iter = 0; - while (!activeRequests.empty() && iter < mMaxWaitMs) - { - for (auto i = activeRequests.begin(); i != activeRequests.end();) - { - auto requestId = *i; - bool thisDone = false; - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - else - { - thisDone = response.getResult().isFinal; - } - } - if (thisDone) - { - // Erase completed request and move to the next one - i = activeRequests.erase(i); - } - else - { - ++i; - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - - auto stats = executor.getLatestRequestStats(); - SizeType32 numFinished = 0; - SizeType32 const maxActiveSize = 64; // Decided by the model - - // The 6th iteration request stat may or may not be available when getLatestRequestStats - // is called. When there are no other active or inTransmission requests, there will be - // another request stats to properly reset all the statistics to zero. - for (auto stat = stats.begin(); stat != stats.begin() + 5; ++stat) - { - SizeType32 numReqs = 0; - SizeType32 numReqsActive = 0; - SizeType32 numReqsQueued = 0; - SizeType32 numReqsJustDone = 0; - for (auto rStat : stat->requestStats) - { - ++numReqs; - numReqsActive += rStat.scheduled ? 1 : 0; - numReqsQueued += rStat.stage == RequestStage::kQUEUED ? 1 : 0; - numReqsJustDone += rStat.stage == RequestStage::kGENERATION_COMPLETE ? 1 : 0; - } - EXPECT_EQ(numReqs, numReqsActive + numReqsQueued + numReqsJustDone); - EXPECT_EQ(numReqs + numFinished, requestParams.size()); // Should report all unfinished requests - EXPECT_TRUE(numReqsActive <= maxActiveSize); // Not all requests are active due to max active size limit. - numFinished += numReqsJustDone; - } -} - -TEST_F(GptExecutorTest, GetRequestStatsMultipleRequests) -{ - bool streaming = false; - bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - executorConfig.setRequestStatsMaxIterations(1000); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - auto sendRequestWaitForResponseFn = [&]() - { - Request request({1, 2, 3}, 5); - auto requestId = executor.enqueueRequest(request); - bool isFinalResponse = false; - while (!isFinalResponse) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto response : responses) - { - if (response.getResult().isFinal) - { - isFinalResponse = true; - break; - } - } - } - return requestId; - }; - - std::unordered_map<IdType, size_t> requestIdToGenerationComplete; - auto updateStats = [&]() - { - auto stats = executor.getLatestRequestStats(); - for (auto& stat : stats) - { - for (auto const& request : stat.requestStats) - { - // only check and aggregate results when request is completed - if (request.stage == RequestStage::kGENERATION_COMPLETE) - { - requestIdToGenerationComplete[request.id] += 1; - } - } - } - }; - - auto requestId = sendRequestWaitForResponseFn(); - requestIdToGenerationComplete[requestId] = 0; - updateStats(); - - requestId = sendRequestWaitForResponseFn(); - requestIdToGenerationComplete[requestId] = 0; - updateStats(); - - for (auto [key, value] : requestIdToGenerationComplete) - { - EXPECT_EQ(value, 1); - } -} - -TEST_F(GptExecutorTest, BatchSizeTuning) -{ - bool streaming = false; - bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - executorConfig.setRequestStatsMaxIterations(1000); - executorConfig.setEnableChunkedContext(true); - - DynamicBatchConfig dynamicBatchConfig(true, false, 1); // Set window size to 1 - SchedulerConfig schedulerConfig(CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT, std::nullopt, dynamicBatchConfig); - executorConfig.setSchedulerConfig(schedulerConfig); - - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - std::vector<SizeType32> tunerRecommendedBatchSizes; - - for (size_t i = 0; i <= 8; ++i) - { - auto inputLength = 1 << i; // Note that for this model max input len is 383 - Request request( - VecTokens(inputLength, 2), 5, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - auto requestId = executor.enqueueRequest(std::move(request)); - // Wait for current request to finish - while (true) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - bool done = false; - if (responses.size() != 0) - { - EXPECT_TRUE(responses.size() == 1); - auto response = responses[0]; - EXPECT_FALSE(response.hasError()); - if (response.getResult().isFinal) - { - break; - } - } - } - auto reqStats = executor.getLatestIterationStats(); - EXPECT_TRUE(reqStats.size() > 0); - auto lastStat = reqStats.back(); - tunerRecommendedBatchSizes.push_back(lastStat.maxBatchSizeTunerRecommended); - } - - EXPECT_TRUE(tunerRecommendedBatchSizes.size() > 0); - // It's supposed to be decreasing when input length increases - EXPECT_TRUE(*tunerRecommendedBatchSizes.begin() > *tunerRecommendedBatchSizes.rbegin()); -} - -TEST_F(GptExecutorTest, GetLatestDebugTensors) -{ - bool streaming = false; - bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - - SizeType32 maxNewTokens = 5; - - tensorrt_llm::executor::DebugConfig debugConfig; - debugConfig.setDebugTensorNames({{"sequence_length"}}); - debugConfig.setDebugTensorsMaxIterations(maxNewTokens); - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - executorConfig.setDebugConfig(debugConfig); - - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - VecTokens inputTokens{1, 2, 3, 4}; - auto request - = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - auto requestId = executor.enqueueRequest(request); - - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - FAIL(); - } - else - { - done = response.getResult().isFinal; - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - - auto stream = std::make_shared<tr::CudaStream>(); - - // Expect 5 non-empty iterations - auto debugTensors = executor.getLatestDebugTensors(); - EXPECT_EQ(debugTensors.size(), 5); - uint64_t currentIter = 0; - for (auto const& debugIteration : debugTensors) - { - EXPECT_EQ(debugIteration.iter, currentIter); - EXPECT_EQ(debugIteration.debugTensors.size(), 2); - - { - auto it = debugIteration.debugTensors.find("request_ids"); - EXPECT_NE(it, debugIteration.debugTensors.end()); - auto const& tensor = it->second; - auto const& shape = tensor.getShape(); - EXPECT_EQ(shape.size(), 1); - EXPECT_EQ(shape[0], 1); - EXPECT_EQ(tensor.getSize(), 1); - auto const* dataPtr = static_cast<SizeType32 const*>(tensor.getData()); - EXPECT_EQ(dataPtr[0], 1) << "currentIter " << currentIter; - } - { - auto it = debugIteration.debugTensors.find("sequence_length"); - EXPECT_NE(it, debugIteration.debugTensors.end()); - auto const& tensor = it->second; - auto const& shape = tensor.getShape(); - EXPECT_EQ(shape.size(), 1); - EXPECT_EQ(tensor.getSize(), 1); - auto tensorHost = tensor.copyToCpu(stream); - auto const* dataPtr = static_cast<SizeType32 const*>(tensorHost.getData()); - EXPECT_EQ(dataPtr[0], inputTokens.size() + currentIter); - } - - ++currentIter; - } -} - -TEST_P(ParamTest, SingleRequestDemo) -{ - bool const streaming = std::get<0>(GetParam()); - bool const excludeInputFromOutput = std::get<1>(GetParam()); - auto const beamWidth = std::get<2>(GetParam()); - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - SizeType32 maxNewTokens = 5; - VecTokens inputTokens{1, 2, 3, 4}; - auto request - = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - - // Enqueue the request - auto requestId = executor.enqueueRequest(request); - - // Get the new tokens - VecTokens tokens; - SizeType32 numResponses{0}; - bool done = false; - int iter = 0; - std::chrono::milliseconds waitTime(1); - while (!done && iter < mMaxWaitMs) - { - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - ++numResponses; - if (response.hasError()) - { - // This request failed for some reason, get error msg - std::string errStr - = "Request id " + std::to_string(requestId) + " failed with err " + response.getErrorMsg(); - FAIL(); - } - - auto result = response.getResult(); - done = result.isFinal; - auto& newTokens = result.outputTokenIds.at(beamWidth - 1); - auto const expectedSize = streaming ? (beamWidth > 1 ? numResponses : 1) - : (maxNewTokens + (excludeInputFromOutput ? 0 : inputTokens.size())); - EXPECT_EQ(newTokens.size(), expectedSize); - - if (streaming && beamWidth > 1) - { - // replace tokens - tokens = newTokens; - } - else - { - // Append tokens - tokens.insert(tokens.end(), newTokens.begin(), newTokens.end()); - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - EXPECT_EQ(numResponses, streaming ? maxNewTokens : 1); - EXPECT_EQ( - tokens.size(), streaming ? maxNewTokens : (excludeInputFromOutput ? 0 : inputTokens.size()) + maxNewTokens); - - // Expect awaitResponse to return error message because the request is already terminated (isFinal = True) - auto response = executor.awaitResponses(requestId, waitTime).at(0); - EXPECT_TRUE(response.hasError()); - std::string err - = "ReqId " + std::to_string(response.getRequestId()) + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); -} - -TEST_P(ParamTest, MultipleRequestDemo) -{ - bool const streaming = std::get<0>(GetParam()); - bool const excludeInputFromOutput = std::get<1>(GetParam()); - auto const beamWidth = std::get<2>(GetParam()); - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - SizeType32 numRequests = 20; - - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - SizeType32 maxPromptLen = 20; - SizeType32 maxMaxNewTokens = 20; - - SizeType32 endId = -1; - // Enqueue the requests - std::unordered_map<IdType, VecTokens> tokens; - std::unordered_map<IdType, SizeType32> expectedNumTokens; - std::unordered_map<IdType, SizeType32> expectedNumResponses; - for (SizeType32 req = 0; req < numRequests; ++req) - { - SizeType32 promptLen = rand() % maxPromptLen + 1; - SizeType32 maxNewTokens = rand() % maxMaxNewTokens + 1; - - auto request = Request(VecTokens(promptLen, 1), maxNewTokens, streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, endId); - auto reqId = executor.enqueueRequest(std::move(request)); - tokens[reqId] = {}; - expectedNumTokens[reqId] = ((streaming || excludeInputFromOutput) ? 0 : promptLen) + maxNewTokens; - expectedNumResponses[reqId] = streaming ? maxNewTokens : 1; - } - - // Get the new tokens for each requests - int32_t numFinished = 0; - int iter = 0; - std::unordered_map<IdType, SizeType32> numResponses; - while (numFinished < numRequests && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - auto reqId = response.getRequestId(); - ++numResponses[reqId]; - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - auto& newTokens = result.outputTokenIds.at(beamWidth - 1); - auto const expectedSize - = streaming ? (beamWidth > 1 ? numResponses[reqId] : 1) : expectedNumTokens[reqId]; - EXPECT_EQ(newTokens.size(), expectedSize); - - auto& reqTokens = tokens.at(response.getRequestId()); - if (streaming && beamWidth > 1) - { - reqTokens = newTokens; - } - else - { - reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); - } - - for (SizeType32 b = 0; b < beamWidth; ++b) - { - EXPECT_EQ(result.finishReasons.at(b), - result.isFinal ? FinishReason::kLENGTH : FinishReason::kNOT_FINISHED); - } - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - - // Check that number of tokens matches expectations - for (auto const& [reqId, numTokens] : expectedNumTokens) - { - EXPECT_EQ(expectedNumResponses[reqId], numResponses[reqId]) << "reqId " << reqId; - EXPECT_EQ(expectedNumTokens[reqId], tokens[reqId].size()) << "reqId " << reqId; - } -} - -TEST_P(ParamStatsTest, MultipleRequestStats) -{ - bool streaming = false; - bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - SizeType32 numRequests = 100; - auto iterStatsMaxIterations = std::get<0>(GetParam()); - bool useOrchestratorMode = std::get<1>(GetParam()); - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - executorConfig.setIterStatsMaxIterations(iterStatsMaxIterations); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - - std::optional<OrchestratorConfig> orchestratorConfig = std::nullopt; - if (useOrchestratorMode) - { - orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); - } - auto parallelConfig = ParallelConfig(CommunicationType::kMPI, - useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::nullopt, std::nullopt, - orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - SizeType32 maxPromptLen = 20; - SizeType32 maxMaxNewTokens = 20; - - SizeType32 endId = -1; - // Enqueue the requests - std::unordered_map<IdType, VecTokens> tokens; - std::unordered_map<IdType, SizeType32> expectedNumTokens; - for (SizeType32 req = 0; req < numRequests; ++req) - { - SizeType32 promptLen = rand() % maxPromptLen + 1; - SizeType32 maxNewTokens = rand() % maxMaxNewTokens + 1; - - auto request = Request(VecTokens(promptLen, 1), maxNewTokens, streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, endId); - auto reqId = executor.enqueueRequest(std::move(request)); - tokens[reqId] = {}; - expectedNumTokens[reqId] = (streaming ? 0 : (excludeInputFromOutput ? 0 : promptLen)) + maxNewTokens; - } - - std::atomic<bool> statsThreadDone = false; - std::atomic<int32_t> numFinished = 0; - std::deque<IterationStats> iterStatsReceived; - // Spawn a thread that continuously get stats - auto statsThread = std::thread( - [&executor, &numFinished, numRequests, &iterStatsReceived, &statsThreadDone]() - { - while (numFinished < numRequests) - { - auto reqStats = executor.getLatestIterationStats(); - iterStatsReceived.insert(iterStatsReceived.end(), std::make_move_iterator(reqStats.begin()), - std::make_move_iterator(reqStats.end())); - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - statsThreadDone = true; - }); - - // Get the new tokens for each requests - int iter = 0; - SizeType32 numResponses = 0; - while (numFinished < numRequests && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - numResponses++; - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - auto& newTokens = result.outputTokenIds.at(beamWidth - 1); - auto& reqTokens = tokens.at(response.getRequestId()); - reqTokens.insert(reqTokens.end(), std::make_move_iterator(newTokens.begin()), - std::make_move_iterator(newTokens.end())); - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - - // Check that number of tokens matches expectations - for (auto const& [reqId, numTokens] : expectedNumTokens) - { - EXPECT_EQ(expectedNumTokens[reqId], tokens[reqId].size()) << "reqId " << reqId; - } - - // Wait for stats thread to be done, fail otherwise - iter = 0; - while (!statsThreadDone && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - std::this_thread::sleep_for(std::chrono::milliseconds(waitTime)); - iter++; - } - ASSERT_TRUE(statsThreadDone); - if (iterStatsMaxIterations > 0) - { - ASSERT_GT(iterStatsReceived.size(), 1); - - for (auto stats : iterStatsReceived) - { - EXPECT_GT(stats.numActiveRequests, 0); - TLLM_LOG_INFO("%d %d", stats.iter, stats.numActiveRequests); - - EXPECT_TRUE(stats.inflightBatchingStats.has_value()); - if (stats.inflightBatchingStats.has_value()) - { - EXPECT_GT(stats.inflightBatchingStats.value().numScheduledRequests, 0); - } - } - } - - statsThread.join(); -} - -TEST_P(ParamTest, MultipleRequestBatchResponses) -{ - bool const streaming = std::get<0>(GetParam()); - bool const excludeInputFromOutput = std::get<1>(GetParam()); - auto const beamWidth = std::get<2>(GetParam()); - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - SizeType32 constexpr numRequests{20}; - - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - SizeType32 constexpr maxPromptLen{20}; - SizeType32 constexpr maxMaxNewTokens{20}; - - SizeType32 endId = -1; - // Enqueue the requests - std::unordered_map<IdType, VecTokens> tokens; - std::unordered_map<IdType, SizeType32> expectedNumTokens; - std::vector<IdType> requestIds; - for (SizeType32 req = 0; req < numRequests; ++req) - { - SizeType32 promptLen = rand() % maxPromptLen + 1; - SizeType32 maxNewTokens = rand() % maxMaxNewTokens + 1; - - auto request = Request(VecTokens(promptLen, 1), maxNewTokens, streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, endId); - auto reqId = executor.enqueueRequest(std::move(request)); - requestIds.push_back(reqId); - tokens[reqId] = {}; - expectedNumTokens[reqId] = (streaming ? 0 : (excludeInputFromOutput ? 0 : promptLen)) + maxNewTokens; - } - - // Get the new tokens for each requests - int32_t numFinished = 0; - int iter = 0; - SizeType32 numResponses = 0; - std::chrono::milliseconds waitTime(1); - while (numFinished < numRequests && iter < mMaxWaitMs) - { - auto idResponses = executor.awaitResponses(requestIds, waitTime); - for (unsigned i = 0; i < requestIds.size(); ++i) - { - auto& responses = idResponses[i]; - for (auto& response : responses) - { - numResponses++; - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - auto& newTokens = result.outputTokenIds.at(beamWidth - 1); - auto& reqTokens = tokens.at(response.getRequestId()); - if (streaming && beamWidth > 1) - { - reqTokens = newTokens; - } - else - { - reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); - } - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - - // Rerun awaitResponses again and we expect to only see terminated request id error. - auto idResponses = executor.awaitResponses(requestIds, waitTime); - for (auto const& responses : idResponses) - { - for (auto& response : responses) - { - EXPECT_TRUE(response.hasError()); - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - } - - // Check that number of tokens matches expectations - for (auto const& [reqId, numTokens] : expectedNumTokens) - { - EXPECT_EQ(expectedNumTokens[reqId], tokens[reqId].size()) << "reqId " << reqId; - } -} - -TEST_P(ParamTest, GetNumResponsesReadyTest) -{ - bool const streaming = std::get<0>(GetParam()); - bool const excludeInputFromOutput = std::get<1>(GetParam()); - auto const beamWidth = std::get<2>(GetParam()); - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - SizeType32 maxNumRequests = 50; - SizeType32 maxPromptLen = 20; - SizeType32 maxMaxNewTokens = 20; - - SizeType32 numRequests = rand() % maxNumRequests + 1; - SizeType32 numExpectedResponses = 0; - std::map<IdType, SizeType32> reqNumExpectedResponses; - std::vector<IdType> ids; - for (SizeType32 req = 0; req < numRequests; ++req) - { - SizeType32 promptLen = rand() % maxPromptLen + 1; - SizeType32 maxNewTokens = rand() % maxMaxNewTokens + 1; - - auto request = Request(VecTokens(promptLen, 1), maxNewTokens, streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - auto id = executor.enqueueRequest(std::move(request)); - ids.emplace_back(id); - reqNumExpectedResponses[id] = streaming ? maxNewTokens : 1; - numExpectedResponses += reqNumExpectedResponses.at(id); - } - - SizeType32 iter = 0; - SizeType32 numReady = 0; - while (numReady < numExpectedResponses && iter < mMaxWaitMs) - { - numReady = 0; - for (auto id : ids) - { - numReady += executor.getNumResponsesReady(id); - } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - // Expect one response per request - for (auto id : ids) - { - SizeType32 numReady = executor.getNumResponsesReady(id); - EXPECT_EQ(numReady, reqNumExpectedResponses.at(id)); - } - auto numResponsesReady = executor.getNumResponsesReady(); - EXPECT_EQ(numResponsesReady, numExpectedResponses); -} - -namespace -{ - -void runTest(Executor& executor, fs::path const& inputPath, ModelIds const& modelIds, - FlakyTestInfo const& flakyTestInfo, bool streaming, SizeType32 const vocabSizePadded, BeamResult const& beamResult, - OutputConfig const& outConfig, bool isSpeculativeDecoding, int maxWaitMs, bool returnAllGeneratedTokens, - SizeType32 const numReturnSequences, bool isNonGreedySampling, SizeType32 const modelParallelism) -{ - auto const beamWidth = beamResult.beamWidth; - - auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); - auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); - auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); - auto const* const givenInputData = tr::bufferCast<TokenIdType const>(*givenInput); - - auto const& inputShape = givenInput->getShape(); - ASSERT_EQ(inputShape.nbDims, 2); - ASSERT_GT(inputShape.d[0], 0); - - // Load expected outputs for each beam width value - auto testData = TestData::loadTestData(beamResult, *givenInput, beamWidth, manager, outConfig, modelIds); - auto const maxSeqLen = testData.maxSeqLen; - - // Load expected outputs and inputs - SizeType32 numRequests = static_cast<SizeType32>(givenInputLengths.size()); - SizeType32 maxRequests = numRequests; - std::vector<Request> requests; - std::vector<SizeType32> reqMaxNewTokens; - - auto samplingConfig = tensorrt_llm::executor::SamplingConfig(beamWidth); - // top-k will be set by a large number to test non-identical N sequences. - if (isNonGreedySampling) - { - samplingConfig.setTopK(32); - } - samplingConfig.setNumReturnSequences(numReturnSequences); - - for (SizeType32 req = 0; req < maxRequests; ++req) - { - SizeType32 inputLen = givenInputLengths.at(req); - auto maxNewTokens = maxSeqLen - maxInputLength; - reqMaxNewTokens.push_back(maxNewTokens); - SizeType32 endId = -1; - auto const* const seqBegin = givenInputData + req * maxInputLength; - VecTokens tokens(seqBegin, seqBegin + inputLen); - auto request = Request( - VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, samplingConfig, outConfig, endId); - request.setReturnAllGeneratedTokens(returnAllGeneratedTokens); - requests.emplace_back(std::move(request)); - } - - auto& comm = tensorrt_llm::mpi::MpiComm::world(); - auto const worldRank = comm.getRank(); - - // Expected return sizes. - auto const numSequences = beamWidth > 1 ? 1 : numReturnSequences; - auto const numReturnBeams = std::min(beamWidth, numReturnSequences); - - if (worldRank == 0) - { - auto const reqIds = executor.enqueueRequests(requests); - - std::unordered_map<SizeType32, std::vector<BeamTokens>> tokens; - std::unordered_map<IdType, SizeType32> reqIdToBatchId; - - for (SizeType32 req = 0; req < reqIds.size(); ++req) - { - std::vector<BeamTokens> resultTokens(numSequences, BeamTokens(numReturnBeams)); - tokens[req] = std::move(resultTokens); - reqIdToBatchId[reqIds.at(req)] = req; - } - - // Get the new tokens for each requests - int32_t numFinished = 0; - int iter = 0; - std::unordered_map<IdType, SizeType32> numResponses; - while (numFinished < maxRequests && iter < maxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - auto batchId = reqIdToBatchId.at(response.getRequestId()); - numResponses[batchId]++; - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - auto seqIdx = result.sequenceIndex; - - auto const& contextLogits = result.contextLogits; - auto const& genLogits = result.generationLogits; - auto const& outputTokenIds = result.outputTokenIds; - - EXPECT_EQ(result.finishReasons.size(), numReturnBeams); - for (SizeType32 beam = 0; beam < numReturnBeams; ++beam) - { - auto const& newTokens = outputTokenIds.at(beam); - auto& reqTokens = tokens.at(batchId).at(seqIdx).at(beam); - - if (!returnAllGeneratedTokens) - { - reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); - } - else - { - EXPECT_EQ(newTokens.size(), - (numResponses.at(batchId) + numReturnSequences - 1) / numReturnSequences); - reqTokens = newTokens; - } - // FinishReason is only supported for bw=1 and inflight batching. - if (beamWidth == 1) - { - EXPECT_EQ(result.finishReasons.at(beam), - result.isSequenceFinal ? FinishReason::kLENGTH : FinishReason::kNOT_FINISHED); - } - } - - auto const& cumLogProbs = result.cumLogProbs; - auto const& logProbs = result.logProbs; - auto const& beamTokens = tokens.at(batchId).at(seqIdx); - EXPECT_EQ(beamTokens.size(), numReturnBeams); - - if (!isNonGreedySampling) - { - float const logitsAtol = modelParallelism > 1 ? 1e-1 : 1e-2; - float const logitsRtol = modelParallelism > 1 ? 1e-2 : 1e-3; - - testData.verifyLogProbs(outConfig.returnLogProbs, streaming, outConfig.excludeInputFromOutput, - givenInputLengths.at(batchId), beamWidth, beamTokens, cumLogProbs, logProbs, batchId, - flakyTestInfo); - testData.validateContextLogits(outConfig.returnContextLogits, givenInputLengths.at(batchId), - beamWidth, contextLogits, vocabSizePadded, batchId, logitsAtol, logitsRtol); - testData.validateGenerationLogits(outConfig.returnGenerationLogits, result.isSequenceFinal, - streaming, outConfig.excludeInputFromOutput, givenInputLengths.at(batchId), - reqMaxNewTokens.at(batchId), beamWidth, beamTokens, genLogits, vocabSizePadded, batchId, - returnAllGeneratedTokens, logitsAtol, logitsRtol); - } - - // Ignore first iteration as it doesn't use draft tokens - if (outConfig.returnPerfMetrics && isSpeculativeDecoding - && result.requestPerfMetrics.value().iter > 0) - { - auto& specDecMetrics = result.requestPerfMetrics.value().speculativeDecoding; - // 4 draft tokens are used per step - EXPECT_EQ(specDecMetrics.totalDraftTokens, result.requestPerfMetrics.value().iter.value() * 4); - EXPECT_EQ(specDecMetrics.acceptanceRate, - static_cast<float>(specDecMetrics.totalAcceptedDraftTokens) - / specDecMetrics.totalDraftTokens); - } - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - } - ++iter; - } - EXPECT_LT(iter, maxWaitMs); - testData.verifyOutput(tokens, givenInputLengths, streaming, outConfig.excludeInputFromOutput, flakyTestInfo, - isSpeculativeDecoding, beamWidth, numSequences, isNonGreedySampling); - } -} - -void runTest(fs::path const& modelPath, ExecutorConfig const& executorConfig, fs::path const& inputPath, - ModelIds const& modelIds, FlakyTestInfo const& flakyTestInfo, bool streaming, SizeType32 const vocabSizePadded, - BeamResult const& beamResult, OutputConfig const& outConfig, bool isSpeculativeDecoding, int maxWaitMs, - bool returnAllGeneratedTokens, SizeType32 const numReturnSequences, bool isNonGreedySampling, - SizeType32 const modelParallelism) -{ - auto executor = Executor{modelPath, ModelType::kDECODER_ONLY, executorConfig}; - - runTest(executor, inputPath, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, outConfig, - isSpeculativeDecoding, maxWaitMs, returnAllGeneratedTokens, numReturnSequences, isNonGreedySampling, - modelParallelism); -} - -ExecutorConfig createExecutorConfig(SizeType32 maxBeamWidth, bool useOrchestratorMode, bool gatherGenerationLogits, - std::optional<std::vector<SizeType32>> deviceIds = std::nullopt, - std::optional<std::vector<SizeType32>> participantIds = std::nullopt) -{ - // Note: we reduce memory fraction for cases that return context/generation logits which require more free - // memory - FloatType constexpr freeGpuMemoryFraction{0.5F}; - KvCacheConfig kvCacheConfig(false, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction); - auto executorConfig = ExecutorConfig(maxBeamWidth); - executorConfig.setKvCacheConfig(kvCacheConfig); - executorConfig.setNormalizeLogProbs(false); - executorConfig.setGatherGenerationLogits(gatherGenerationLogits); - - std::optional<OrchestratorConfig> orchestratorConfig = std::nullopt; - if (useOrchestratorMode) - { - orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); - } - auto parallelConfig = ParallelConfig(CommunicationType::kMPI, - useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::move(deviceIds), - std::move(participantIds), orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - - return executorConfig; -} - -} // namespace - -TEST_P(AllParamsTest, TokenComparison) -{ - auto const streaming = std::get<0>(GetParam()); - auto const& beamWidth = std::get<1>(GetParam()); - OutputConfig outConfig; - outConfig.returnLogProbs = std::get<2>(GetParam()); - outConfig.excludeInputFromOutput = std::get<3>(GetParam()); - outConfig.returnContextLogits = std::get<4>(GetParam()); - outConfig.returnGenerationLogits = std::get<5>(GetParam()); - auto const modelName = std::get<6>(GetParam()); - auto const useOrchestratorMode = std::get<7>(GetParam()); - auto const returnAllGeneratedTokens = std::get<8>(GetParam()); - auto const numReturnSequences = std::get<9>(GetParam()); - if (returnAllGeneratedTokens && !streaming) - { - GTEST_SKIP() << "Test does not support returnAllGeneratedTokens without streaming"; - } - - std::optional<std::vector<SizeType32>> participantIds = std::nullopt; - - BeamResult beamResult{beamWidth}; - - ASSERT_TRUE(fs::exists(DATA_PATH)); - - fs::path modelPath; - // set defaults and adjust if needed by different models - fs::path inputPath = DATA_PATH / "input_tokens.npy"; - ModelIds modelIds{50256, 50256}; - bool isSpeculativeDecoding{false}; - - SizeType32 vocabSizePadded = 50257; - - // NOTE: This can be used to disable checks for certain prompt batch entries - FlakyTestInfo flakyTestInfo; - - if (modelName == "gpt") - { - auto const resultsPath - = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - if (outConfig.returnContextLogits || outConfig.returnGenerationLogits) - { - modelPath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"; - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); - beamResult.contextLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); - beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); - if (outConfig.returnLogProbs) - { - beamResult.cumLogProbsFile - = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_CUM_LOG_PROBS_FILE(); - beamResult.logProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_LOG_PROBS_FILE(); - } - } - else - { - modelPath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_FILE(); - if (outConfig.returnLogProbs) - { - beamResult.cumLogProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_FILE(); - beamResult.logProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_FILE(); - } - } - } - else if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1" - || modelName == "llama_tp1_pp2_cp1") - { - inputPath = DATA_PATH / LLAMA_INPUT_FILE; - modelIds.padId = LLAMA_PAD_ID; - modelIds.endId = LLAMA_END_ID; - - vocabSizePadded = LLAMA_VOCAB_SIZE_PADDED; - - auto const resultsPath - = LLAMA_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - if (modelName == "llama_tp4_pp1_cp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP4_PP1_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp4_cp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP4_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp2_cp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP2_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp2-cp1-gpu"; - } - else if (modelName == "llama_tp2_pp2_cp1") - { - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; - } - beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_TP4_PP1_FILE(); - if (outConfig.returnLogProbs) - { - beamResult.cumLogProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_TP4_PP1_FILE(); - beamResult.logProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_TP4_PP1_FILE(); - } - } - else if (modelName == "medusa") - { - TLLM_CHECK_WITH_INFO(beamWidth == 1, "Medusa does not support beam search."); - auto const resultsPath = MEDUSA_DATA_PATH / "sampling"; - auto modelSpec = ModelSpec::getDefaultModelSpec() - .useMedusa() - .setInputFile("input_tokens_long.npy") - .setMaxOutputLength(128); - beamResult.resultsFile = resultsPath / modelSpec.getResultsFile(); - modelPath = MEDUSA_MODEL_PATH / modelSpec.getModelPath() / "tp1-pp1-cp1-gpu"; - - inputPath = DATA_PATH / "input_vicuna.npy"; - modelIds.padId = 2; - modelIds.endId = 2; - isSpeculativeDecoding = true; - outConfig.returnPerfMetrics = true; - } - else if (modelName == "chatglm" || modelName == "chatglm2" || modelName == "chatglm3" || modelName == "glm") - { - fs::path resultsPath; - if (modelName == "chatglm") - { - resultsPath = CHATGLM_DATA_PATH; - modelPath = CHATGLM_MODEL_PATH; - } - else if (modelName == "chatglm2") - { - resultsPath = CHATGLM2_DATA_PATH; - modelPath = CHATGLM2_MODEL_PATH; - } - else if (modelName == "chatglm3") - { - resultsPath = CHATGLM3_DATA_PATH; - modelPath = CHATGLM3_MODEL_PATH; - } - else if (modelName == "glm") - { - resultsPath = GLM_DATA_PATH; - modelPath = GLM_MODEL_PATH; - } - resultsPath /= (beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth); - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_FILE(); - modelPath = modelPath / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - - char versionChatglm{0}; - if (size_t index = modelPath.string().find("chatglm"); index != std::string::npos) - { - versionChatglm = modelPath.string()[index + 7]; - std::string const vChatglmString - = (versionChatglm == '-') ? std::string("") : std::string(1, versionChatglm); - inputPath = DATA_PATH / ("input_tokens_chatglm" + vChatglmString + "-6b.npy"); - modelIds.padId = (versionChatglm == '-') ? 3 : 0; - modelIds.endId = (versionChatglm == '-') ? 130005 : 2; - } - else if (size_t index = modelPath.string().find("glm-10b"); index != std::string::npos) - { - inputPath = DATA_PATH / "input_tokens_glm-10b.npy"; - modelIds.padId = 50256; - modelIds.endId = 50258; - } - - if (versionChatglm != 0) - { - flakyTestInfo.batchIdBeams.insert(std::make_pair(1, 0)); - } - } - else - { - TLLM_THROW("Unrecognized modelName"); - } - - if (streaming && beamWidth > 1) - { - GTEST_SKIP() << "Test does not support streaming with beam search"; - } - - // Warning: This should be the last check before running the test. - // It will initialize MPI which can take significant time. - if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1" - || modelName == "llama_tp1_pp2_cp1") - { - // For llama model, only run for multiple GPUs - // This is detected by setting an env variable when running the test - char const* val = getenv("RUN_LLAMA_MULTI_GPU"); - if (val == nullptr) - { - GTEST_SKIP() << "Skipping Llama test"; - } - - if (outConfig.returnContextLogits) - { - GTEST_SKIP() << "Skipping context logits tests for mpi runs"; - } - - // Check that it was launched with right number of MPI ranks - if (!useOrchestratorMode && COMM_SESSION.getSize() != 4) - { - // No orchestrator, need worldSize to match TP*PP - FAIL() << "Leader mode and world size is not equal to 4"; - } - if (useOrchestratorMode && COMM_SESSION.getSize() != 1) - { - // No orchestrator, need worldSize to match TP*PP - FAIL() << "Orchestrator mode and World size is not equal to 1"; - } - } - auto decoderJsonConfig = tensorrt_llm::runtime::GptJsonConfig::parse(modelPath / "config.json"); - - auto const modelTP = decoderJsonConfig.getTensorParallelism(); - auto const modelPP = decoderJsonConfig.getPipelineParallelism(); - auto const modelParallelism = modelTP * modelPP; - int deviceCount = -1; - TLLM_CUDA_CHECK(cudaGetDeviceCount(&deviceCount)); - std::optional<std::vector<SizeType32>> deviceIds = std::vector<SizeType32>(modelParallelism); - for (auto i = 0; i < deviceIds->size(); i++) - { - deviceIds->at(i) = i % deviceCount; - } - if (modelName == "llama_tp1_pp2_cp1") - { - auto const& session = tensorrt_llm::mpi::MpiComm::world(); - if (session.getSize() != 4) - { - FAIL() << "Llama-tp1-pp2 is intended solely for testing coexisting engines within the same MPI world," - " which requires a session size of 4. However, the current session size is " - << session.getSize() << " ."; - } - if (session.getRank() / 2 == 0) - { - participantIds = std::vector<SizeType32>{0, 1}; - deviceIds = std::vector<SizeType32>{0, 1}; - } - else - { - participantIds = std::vector<SizeType32>{2, 3}; - deviceIds = std::vector<SizeType32>{2, 3}; - } - } - - if (modelPP > 1) - { - std::reverse(deviceIds->begin(), deviceIds->end()); - if (modelTP > 1) - { - for (SizeType32 ppRank = 0; ppRank < modelPP; ppRank++) - { - std::reverse(deviceIds->begin() + ppRank * modelTP, deviceIds->begin() + (ppRank + 1) * modelPP); - } - } - } - - // Returning logits will bring higher latency - if (streaming && (outConfig.returnContextLogits || outConfig.returnGenerationLogits)) - { - mMaxWaitMs = 20000; - } - - auto executorConfig = createExecutorConfig(beamWidth, useOrchestratorMode, outConfig.returnGenerationLogits, - std::move(deviceIds), std::move(participantIds)); - - runTest(modelPath, executorConfig, inputPath, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, - outConfig, isSpeculativeDecoding, mMaxWaitMs, returnAllGeneratedTokens, numReturnSequences, false, - modelParallelism); -} - -TEST_F(GptExecutorTest, ChangeBeamWidth) -{ - SizeType32 constexpr maxBeamWidth{2}; - auto executorConfig = ExecutorConfig(maxBeamWidth); - - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - SizeType32 constexpr beamWidth1{1}; - SizeType32 constexpr beamWidth2{2}; - SizeType32 constexpr maxNewTokens{2}; - VecTokens inputTokens{1, 2, 3, 4}; - - // Create requests with different beam widths - std::vector<Request> requests; - requests.emplace_back(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth1)); - requests.emplace_back(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth1)); - requests.emplace_back(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth2)); - requests.emplace_back(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth1)); - - auto requestIds = executor.enqueueRequests(requests); - - int numFinished = 0; - int iter = 0; - while (numFinished < 4 && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - auto err = response.getErrorMsg(); - std::cout << "err:" << err << std::endl; - FAIL() << "Should not get a response with error"; - } - else - { - auto result = response.getResult(); - numFinished += static_cast<int>(result.isFinal); - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - - auto stats = executor.getLatestIterationStats(); - uint64_t currentIter = 0; - for (auto const& stat : stats) - { - // TODO: enable this check when stats are cleaned - // EXPECT_EQ(stat.iter, currentIter); - if (stat.iter < 2) - { - // req 1 and 2 run with same beam width - EXPECT_EQ(stat.numActiveRequests, 2); - } - else if (stat.numActiveRequests != 0) // TODO: remove this check when stats are cleaned - { - // req 3 or 4 run width different beam width - EXPECT_EQ(stat.numActiveRequests, 1); - } - - ++currentIter; - } -} - -void doTokenComparisonChangeBeamWidth(bool enableReuse, SizeType32 maxWaitMs) -{ - SizeType32 constexpr maxBeamWidth{2}; - SizeType32 constexpr vocabSizePadded{50257}; // gpt vocabSizePadded - auto constexpr streaming = false; - - // Create executor config - auto kvCacheConfig = KvCacheConfig(enableReuse); - auto executorConfig = ExecutorConfig(maxBeamWidth, SchedulerConfig(), kvCacheConfig); - - // Create executor - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - auto const inputPath = DATA_PATH / "input_tokens.npy"; - ModelIds modelIds{50256, 50256}; - - OutputConfig outConfig; - FlakyTestInfo flakyTestInfo; - bool constexpr isSpeculativeDecoding{false}; - - for (SizeType32 beamWidth : {1, 2}) - { - TLLM_LOG_INFO("Running beam width: %d", beamWidth); - BeamResult beamResult{beamWidth}; - auto const resultsPath - = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); - beamResult.contextLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); - beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); - - auto const numReturnSequences = beamWidth; - - runTest(executor, inputPath, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, outConfig, - isSpeculativeDecoding, maxWaitMs, false, numReturnSequences, false, 1); - } -} - -TEST_F(GptExecutorTest, TokenComparisonChangeBeamWidth) -{ - doTokenComparisonChangeBeamWidth(false, mMaxWaitMs); -} - -TEST_F(GptExecutorTest, TokenComparisonChangeBeamWidthBlockReuse) -{ - doTokenComparisonChangeBeamWidth(true, mMaxWaitMs); -} - -TEST_F(GptExecutorTest, NReturnRandomness) -{ - SizeType32 constexpr maxBeamWidth{1}; - SizeType32 constexpr numReturnSequences{2}; - SizeType32 constexpr vocabSizePadded{50257}; // gpt vocabSizePadded - auto constexpr streaming = false; - - // Create executor config - auto executorConfig = ExecutorConfig(maxBeamWidth); - - // Create executor - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - auto const inputPath = DATA_PATH / "input_tokens.npy"; - ModelIds modelIds{50256, 50256}; - - OutputConfig outConfig; - FlakyTestInfo flakyTestInfo; - bool constexpr isSpeculativeDecoding{false}; - - BeamResult beamResult{maxBeamWidth}; - auto const resultsPath = GPT_DATA_PATH / "sampling"; - beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); - beamResult.contextLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); - beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); - - runTest(executor, inputPath, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, outConfig, - isSpeculativeDecoding, mMaxWaitMs, false, 1, true, 1); -} - -TEST_F(GptExecutorTest, TimedOut) -{ - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // No requests enqueued, expect no responses - auto numResponsesReady = executor.getNumResponsesReady(); - EXPECT_EQ(numResponsesReady, 0); - - std::chrono::milliseconds waitTime(10); - auto responses = executor.awaitResponses(waitTime); - EXPECT_EQ(responses.size(), 0); -} - -TEST_F(GptExecutorTest, MaxSeqIdleMicrosecondsError) -{ - auto executorConfig = ExecutorConfig(1); - // Request will time out - executorConfig.setMaxSeqIdleMicroseconds(1); - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - SizeType32 constexpr maxNewTokens{5}; - VecTokens inputTokens{1, 2, 3, 4}; - - std::vector<Request> requests; - requests.emplace_back(inputTokens, maxNewTokens, false); - - auto requestIds = executor.enqueueRequests(requests); - - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - auto err = response.getErrorMsg(); - std::cout << "err:" << err << std::endl; - EXPECT_THAT(err, testing::HasSubstr("Unable to get batch slot for request ID")); - done = true; - } - else - { - FAIL() << "Should get a response with error"; - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); -} - -void logitsProcessorMixedReqsTest(std::string const& modelDir, SizeType32 worldRank, SizeType32 maxWaitMs, - bool replicated, std::optional<std::vector<SizeType32>> deviceIds); - -TEST_P(LogitsProcParamsTest, All) -{ - auto const modelName = std::get<0>(GetParam()); - auto const batched = std::get<1>(GetParam()); - auto const replicated = std::get<2>(GetParam()); - - std::string modelDir; - int tp_size = 1, pp_size = 1, cp_size = 1; - std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; - - if (modelName == "llama_tp1_pp1_cp1") - { - modelDir = "tp1-pp1-cp1-gpu"; - } - else if (modelName == "llama_tp4_pp1_cp1") - { - modelDir = "tp4-pp1-cp1-gpu"; - tp_size = 4; - } - else if (modelName == "llama_tp1_pp4_cp1") - { - modelDir = "tp1-pp4-cp1-gpu"; - pp_size = 4; - deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; - } - else if (modelName == "llama_tp2_pp2_cp1") - { - modelDir = "tp2-pp2-cp1-gpu"; - tp_size = pp_size = 2; - deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; - } - else - { - TLLM_THROW("Unrecognized modelName"); - } - std::filesystem::path modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / modelDir; - - auto& comm = tensorrt_llm::mpi::MpiComm::world(); - auto const worldRank = comm.getRank(); - auto const worldSize = comm.getSize(); - - if (tp_size * pp_size * cp_size != 1) - { - // Run multi GPU test only when env variable is set - char const* val = getenv("RUN_LLAMA_MULTI_GPU"); - if (val == NULL) - { - GTEST_SKIP() << "Skipping multi-gpu logits post processor test"; - } - - if (worldSize != 4) - { - FAIL() << "Leader mode and world size is not equal to 4"; - } - } - else - { - // This has no effect for single-GPU tests - if (replicated) - { - GTEST_SKIP() << "Skipping single-gpu replicated logits post processor test"; - } - } - - // Configuration options - bool const streaming = false; - bool excludeInputFromOutput = false; - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - SizeType32 numRequests = 20; - IdType const kClientId = 1234; - - SizeType32 beamWidth = 1; - SizeType32 maxPromptLen = 20; - SizeType32 maxMaxNewTokens = 20; - - SizeType32 constexpr endId{2}; - SizeType32 constexpr vocabSizePadded{32000}; // llama-7b vocabSizePadded - // We just use tokenIdCalculator to generate a token_id based on request index, output position and max new tokens. - // Then LogitsPostProcessor set all other logits except the generated token_id to large negative value. - // So the output token should be the generated token by tokenIdCalculator. - auto tokenIdCalculator = [endId, vocabSizePadded](IdType req, SizeType32 pos) - { - SizeType32 tokenId = (req * 1000 + pos) % vocabSizePadded; - if (tokenId == endId) - { - tokenId = 0; - } - return tokenId; - }; - - std::unordered_map<IdType, VecTokens> tokens; - std::unordered_map<IdType, SizeType32> expectedNumTokens; - std::unordered_map<IdType, VecTokens> expectedOutputTokens; - - // Enqueue the requests - auto enqueueRequests = [&](Executor& executor, std::optional<std::string const> logitsProcessorName, - std::optional<LogitsPostProcessor> logitsProcessor = std::nullopt) - { - tokens.clear(); - expectedNumTokens.clear(); - expectedOutputTokens.clear(); - - for (SizeType32 req = 0; req < numRequests; ++req) - { - SizeType32 promptLen = rand() % maxPromptLen + 1; - SizeType32 maxNewTokens = rand() % maxMaxNewTokens + 1; - - auto request = Request(VecTokens(promptLen, 1), maxNewTokens, streaming, - tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, endId); - request.setClientId(kClientId); - if (logitsProcessorName) - { - request.setLogitsPostProcessorName(logitsProcessorName.value()); - } - else if (logitsProcessor) - { - request.setLogitsPostProcessor(logitsProcessor.value()); - } - auto reqId = executor.enqueueRequest(std::move(request)); - tokens[reqId] = {}; - expectedNumTokens[reqId] = (streaming ? 0 : (excludeInputFromOutput ? 0 : promptLen)) + maxNewTokens; - expectedOutputTokens[reqId] = {}; - if (!streaming && !excludeInputFromOutput) - { - expectedOutputTokens[reqId].resize(promptLen, 1); - } - for (SizeType32 outputPos = 0; outputPos < maxNewTokens; ++outputPos) - { - SizeType32 outputTokenId = tokenIdCalculator(reqId, outputPos + promptLen); - expectedOutputTokens[reqId].push_back(outputTokenId); - } - } - }; - - // Get the new tokens for each requests - auto collectResponses = [&](Executor& executor) - { - int32_t numFinished = 0; - int iter = 0; - SizeType32 numResponses = 0; - while (numFinished < numRequests && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - numResponses++; - if (!response.hasError()) - { - EXPECT_EQ(response.getClientId().value(), kClientId); - auto result = response.getResult(); - numFinished += result.isFinal; - auto& newTokens = result.outputTokenIds.at(beamWidth - 1); - auto& reqTokens = tokens.at(response.getRequestId()); - reqTokens.insert(reqTokens.end(), std::make_move_iterator(newTokens.begin()), - std::make_move_iterator(newTokens.end())); - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - }; - - // Check that tokens matches expectations - auto checkOutput = [&]() - { - for (auto const& [reqId, numTokens] : expectedNumTokens) - { - EXPECT_EQ(expectedNumTokens[reqId], tokens[reqId].size()) << "reqId " << reqId; - for (SizeType32 tokenPos = 0; - tokenPos < std::min<SizeType32>(expectedNumTokens[reqId], tokens[reqId].size()); ++tokenPos) - { - EXPECT_EQ(expectedOutputTokens[reqId][tokenPos], tokens[reqId][tokenPos]) - << "reqId=" << reqId << ", tokenPos=" << tokenPos; - } - } - }; - - // Test non-batched logits processor - std::string const logitsProcessorName = "SelectToken"; - - auto logitsPostProcessorFn = [&](IdType reqId, Tensor& logits, BeamTokens const& tokens, StreamPtr const& streamPtr, - std::optional<IdType> clientId) - { - if (replicated) - { - EXPECT_TRUE(worldRank <= tp_size - 1); - } - else - { - EXPECT_TRUE(worldRank == 0); - } - EXPECT_TRUE(clientId.value() == kClientId); - SizeType32 numTokens = tokens.at(0).size(); - SizeType32 pos = numTokens; - SizeType32 outputTokenId = tokenIdCalculator(reqId, pos); - auto logitsDataType = logits.getDataType(); - EXPECT_TRUE(logitsDataType == DataType::kFP16 || logitsDataType == DataType::kBF16 - || logitsDataType == DataType::kFP32); - // logits has shape [draftLength + 1, reqBeamWidth, vocabSize] - auto logitsCpu = tensorrt_llm::executor::Tensor::cpu(logitsDataType, logits.getShape()); - auto* dataPtr = logitsCpu.getData(); - auto eltSize = logitsCpu.getSizeInBytes() / logitsCpu.getSize(); - EXPECT_TRUE(eltSize == 2 || eltSize == 4); - if (eltSize == 2) - { - auto* dataPtrU16 = static_cast<uint16_t*>(dataPtr); - uint16_t hugeNegValue = logitsDataType == DataType::kFP16 ? 0xFBFF : 0xFF7F; // a huge negative value - for (size_t i = 0; i < logitsCpu.getSize(); ++i) - { - dataPtrU16[i] = hugeNegValue; - } - dataPtrU16[outputTokenId] = 0; - } - else - { - auto* dataPtrFloat = static_cast<float*>(dataPtr); - for (size_t i = 0; i < logitsCpu.getSize(); ++i) - { - dataPtrFloat[i] = -HUGE_VALF; - } - dataPtrFloat[outputTokenId] = 0.0f; - } - - logits.setFrom(logitsCpu, streamPtr); - }; - - if (!batched) - { - auto executorConfig = ExecutorConfig(beamWidth); - LogitsPostProcessorConfig logitsProcConfig{ - std::unordered_map<std::string, tensorrt_llm::executor::LogitsPostProcessor>{ - {logitsProcessorName, logitsPostProcessorFn}}, - std::nullopt, replicated}; - executorConfig.setLogitsPostProcessorConfig(logitsProcConfig); - if (deviceIds.has_value()) - { - auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); - parallelConfig.setDeviceIds(deviceIds.value()); - executorConfig.setParallelConfig(parallelConfig); - } - auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); - - if (worldRank == 0) - { - enqueueRequests(executor, logitsProcessorName); - collectResponses(executor); - checkOutput(); - - if (!replicated || tp_size == 1) - { - // Dynamic logits postprocessor must be used with replicate=false or no tensor parallelism. - enqueueRequests(executor, std::nullopt, logitsPostProcessorFn); - collectResponses(executor); - checkOutput(); - } - } - } - - // Test batched logits processor - auto logitsPostProcessorBatchedFn - = [logitsPostProcessorFn](std::vector<IdType> const& reqIdBatch, std::vector<Tensor>& logitsBatch, - std::vector<std::reference_wrapper<BeamTokens const>> const& tokensBatch, StreamPtr const& streamPtr, - std::vector<std::optional<IdType>> const& clientIdBatch) - { - for (int sample = 0; sample < reqIdBatch.size(); sample++) - { - logitsPostProcessorFn( - reqIdBatch[sample], logitsBatch[sample], tokensBatch[sample], streamPtr, clientIdBatch[sample]); - } - }; - - if (batched) - { - auto batchedExecutorConfig = ExecutorConfig(beamWidth); - if (deviceIds.has_value()) - { - auto parallelConfig = batchedExecutorConfig.getParallelConfig().value_or(ParallelConfig()); - - parallelConfig.setDeviceIds(deviceIds.value()); - batchedExecutorConfig.setParallelConfig(parallelConfig); - } - LogitsPostProcessorConfig logitsProcConfig{std::nullopt, logitsPostProcessorBatchedFn, replicated}; - batchedExecutorConfig.setLogitsPostProcessorConfig(logitsProcConfig); - - auto batchedExecutor = Executor(modelPath, ModelType::kDECODER_ONLY, batchedExecutorConfig); - - if (worldRank == 0) - { - enqueueRequests(batchedExecutor, Request::kBatchedPostProcessorName); - collectResponses(batchedExecutor); - checkOutput(); - } - } - - if (!batched) - { - logitsProcessorMixedReqsTest(modelDir, worldRank, mMaxWaitMs, replicated, std::move(deviceIds)); - } -} - -// Test for mixing requests with and without logits processor. -void logitsProcessorMixedReqsTest(std::string const& modelDir, SizeType32 worldRank, SizeType32 maxWaitMs, - bool replicated, std::optional<std::vector<SizeType32>> deviceIds) -{ - std::string const logitsProcessorName = "dummy"; - auto logitsPostProcessorFn = [&](IdType reqId, Tensor& logits, BeamTokens const& tokens, StreamPtr const& streamPtr, - std::optional<IdType> clientId) - { - // Dummy callback that does not modify logits - assert(!clientId.has_value()); - }; - - LogitsPostProcessorConfig logitsProcConfig{ - std::unordered_map<std::string, tensorrt_llm::executor::LogitsPostProcessor>{ - {logitsProcessorName, logitsPostProcessorFn}}, - std::nullopt, replicated}; - - // Create executor - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - executorConfig.setLogitsPostProcessorConfig(logitsProcConfig); - if (deviceIds.has_value()) - { - auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); - - parallelConfig.setDeviceIds(deviceIds.value()); - executorConfig.setParallelConfig(parallelConfig); - } - std::filesystem::path modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / modelDir; - auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); - - if (worldRank == 0) - { - SizeType32 numRequests = 2; - SizeType32 promptLen = 5; - - // First request with no LP and many output tokens - auto request1 = Request(VecTokens(promptLen, 1), 25); - // Second request with LP and few output tokens - auto request2 = Request(VecTokens(promptLen, 1), 5); - request2.setLogitsPostProcessorName(logitsProcessorName); - - // Enqueue requests - auto reqId1 = executor.enqueueRequest(request1); - auto reqId2 = executor.enqueueRequest(request2); - - // Wait for responses - int32_t numFinished = 0; - int iter = 0; - SizeType32 numResponses = 0; - while (numFinished < numRequests && iter < maxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - numResponses++; - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - EXPECT_EQ(response.getErrorMsg(), err); - } - } - ++iter; - } - EXPECT_LT(iter, maxWaitMs); - } -} - -TEST_F(GptExecutorTest, LogitsPostProcessorThrow) -{ - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - std::string const logitsProcessorName = "UnExistProcessor"; - - auto request - = Request(VecTokens(10, 1), 10, false, tensorrt_llm::executor::SamplingConfig(beamWidth), OutputConfig()); - request.setLogitsPostProcessorName(logitsProcessorName); - EXPECT_THROW({ auto reqId = executor.enqueueRequest(std::move(request)); }, tensorrt_llm::common::TllmException); -} - -static Response executeDraftRequest(Executor& executor) -{ - OutputConfig outputConfig; - outputConfig.returnGenerationLogits = true; - - // Create the request - SizeType32 maxNewTokens = 4; - VecTokens inputTokens{1, 2, 3, 4}; - - Request request{std::move(inputTokens), maxNewTokens}; - request.setOutputConfig(outputConfig); - - // Enqueue the request - auto requestId = executor.enqueueRequest(std::move(request)); - - // Wait for the response - auto responses = executor.awaitResponses(requestId); - - return responses.at(0); -} - -static Response executeTargetRequest(Executor& executor, Result const& draftResult) -{ - // Create the request - SizeType32 maxNewTokens = 5; - VecTokens inputTokens{1, 2, 3, 4}; - - Request request{std::move(inputTokens), maxNewTokens}; - - VecTokens const& outputTokenIds = draftResult.outputTokenIds.at(0); - VecTokens draftTokens(outputTokenIds.end() - 4, outputTokenIds.end()); - - auto const& logitsInfo = draftResult.specDecFastLogitsInfo.value(); - auto logitsTensor = logitsInfo.toTensor(); - - ExternalDraftTokensConfig draftTokensConfig( - std::move(draftTokens), logitsTensor, std::nullopt /* acceptance threshold */, true /* fastLogits */); - request.setExternalDraftTokensConfig(draftTokensConfig); - - // Enqueue the request - auto requestId = executor.enqueueRequest(std::move(request)); - - // Wait for the response - auto responses = executor.awaitResponses(requestId); - - return responses.at(0); -} - -class SpeculativeDecodingTest : public GptExecutorTest -{ -}; - -TEST_F(SpeculativeDecodingTest, SpecDecFastLogits) -{ - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtDraftEnginePath - = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"; - auto trtEnginePath - = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DRAFT_TOKENS_DIR() / "tp1-pp1-cp1-gpu"; - - FloatType freeGpuMemoryFraction = 0.3; - auto kvCacheConfig - = KvCacheConfig(true /* enableBlockReuse */, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction); - executorConfig.setKvCacheConfig(kvCacheConfig); - - tensorrt_llm::mpi::initialize(tensorrt_llm::mpi::MpiThreadSupport::THREAD_MULTIPLE); - int const worldSize = tensorrt_llm::mpi::MpiComm::world().getSize(); - ASSERT_EQ(worldSize, 3); - int const myRank = tensorrt_llm::mpi::MpiComm::world().getRank(); - bool const isOrchestrator = (myRank == 0); - - auto orchestratorConfig - = OrchestratorConfig(isOrchestrator, "" /* workerExecutablePath */, nullptr, false /* spawnPrcesses */); - auto parallelConfig = ParallelConfig( - CommunicationType::kMPI, CommunicationMode::kORCHESTRATOR, std::nullopt, std::nullopt, orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - - auto specDecConfig = SpeculativeDecodingConfig(true /* fastLogits */); - executorConfig.setSpecDecConfig(specDecConfig); - - std::unique_ptr<Executor> draftExecutor; - std::unique_ptr<Executor> targetExecutor; - - if (isOrchestrator) - { - auto executorConfigDraft = executorConfig; - parallelConfig.setParticipantIds({1}); - executorConfigDraft.setParallelConfig(parallelConfig); - - draftExecutor = std::make_unique<Executor>(trtDraftEnginePath, ModelType::kDECODER_ONLY, executorConfigDraft); - - parallelConfig.setParticipantIds({2}); - executorConfig.setParallelConfig(parallelConfig); - - targetExecutor = std::make_unique<Executor>(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - } - else if (myRank == 1) // draft model process - { - parallelConfig.setParticipantIds({1}); - parallelConfig.setDeviceIds({0}); - executorConfig.setParallelConfig(parallelConfig); - executorConfig.setGatherGenerationLogits(true); - draftExecutor = std::make_unique<Executor>(trtDraftEnginePath, ModelType::kDECODER_ONLY, executorConfig); - } - else if (myRank == 2) // target model process - { - parallelConfig.setParticipantIds({2}); - parallelConfig.setDeviceIds({0}); - executorConfig.setParallelConfig(parallelConfig); - draftExecutor = std::make_unique<Executor>(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - } - - if (isOrchestrator) - { - auto response = executeDraftRequest(*draftExecutor); - ASSERT_FALSE(response.hasError()); - response = executeTargetRequest(*targetExecutor, response.getResult()); - ASSERT_FALSE(response.hasError()); - } -} - -TEST_F(GptExecutorTest, OrchestratorMaxQueueSize) -{ - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - SizeType32 maxQueueSize = 6; - ExecutorConfig executorConfig; - executorConfig.setMaxQueueSize(maxQueueSize); - auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); - auto parallelConfig = ParallelConfig( - CommunicationType::kMPI, CommunicationMode::kORCHESTRATOR, std::nullopt, std::nullopt, orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - SizeType32 maxNewTokens = 100; - VecTokens inputTokens{1, 2, 3, 4}; - auto request = Request(inputTokens, maxNewTokens); - std::vector<IdType> requestIds; - auto numberOfRequests = maxQueueSize * 5; - requestIds.reserve(numberOfRequests); - - // Enqueue more requests than the queue can manage - for (int i = 0; i < numberOfRequests; i++) - { - auto requestId = executor.enqueueRequest(request); - requestIds.emplace_back(requestId); - } - - auto responseVectors = executor.awaitResponses(std::move(requestIds)); - bool failedWithFullQueue = false; - for (auto& responseVector : responseVectors) - { - for (auto& response : responseVector) - { - if (response.hasError()) - { - EXPECT_THAT(response.getErrorMsg(), - testing::HasSubstr("Maximum queue size of 6 has been reached, please try again later")); - failedWithFullQueue = true; - } - } - } - EXPECT_TRUE(failedWithFullQueue) << "Expected requests to fail due to maximum queue size reached"; - - // Wait for requests to get scheduled to free up space in queue - std::this_thread::sleep_for(std::chrono::milliseconds(maxQueueSize * 200)); - auto requestId = executor.enqueueRequest(std::move(request)); - auto responses = executor.awaitResponses(requestId); - for (auto& response : responses) - { - EXPECT_FALSE(response.hasError()); - } -} - -TEST_F(GptExecutorTest, SingleRequestInvalidInputs) -{ - bool streaming = true; - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - SizeType32 maxNewTokens = 5; - VecTokens inputTokens{1, 2, 3, 4}; - - std::vector<std::string> expectedErrMsgs; - std::vector<Request> requests; - - // Invalid embedding bias shape - { - requests.emplace_back(inputTokens, maxNewTokens, streaming); - auto embeddingBias = Tensor::cpu(DataType::kFP32, {1}); - requests.back().setEmbeddingBias(embeddingBias); - expectedErrMsgs.emplace_back("embedding bias shape is not as expected"); - } - - for (auto req = 0; req < requests.size(); ++req) - { - auto& request = requests.at(req); - auto const& expectedErrMsg = expectedErrMsgs.at(req); - - auto requestId = executor.enqueueRequest(std::move(request)); - - // Try to get the new tokens - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - - auto err = response.getErrorMsg(); - EXPECT_THAT(err, testing::HasSubstr(expectedErrMsg)); - done = true; - } - else - { - FAIL() << "Expected an err: " << expectedErrMsg; - } - } - ++iter; - } - EXPECT_EQ(done, true); - } -} - -TEST_F(GptExecutorTest, ExecutorKVCacheManager) -{ - - bool streaming = true; - int numRequests = 3; - - SizeType32 beamWidth = 1; - SizeType32 maxNewTokens = 5; - auto executorConfig = ExecutorConfig(beamWidth); - auto kvCacheConfig = KvCacheConfig(true, 128); - kvCacheConfig.setEventBufferMaxSize(1024); - executorConfig.setKvCacheConfig(kvCacheConfig); - - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - auto kvCacheManager = *executor.getKVCacheEventManager(); - - // Created event should be available before any requests. - auto events = kvCacheManager->getLatestEvents(std::chrono::seconds(1)); - EXPECT_EQ(events.size(), 1); - EXPECT_TRUE(std::holds_alternative<KVCacheCreatedData>(events.front().data)); - - // Create requests - std::vector<Request> requests; - for (int request = 0; request < 3; request++) - { - VecTokens inputTokens; - for (int i = 0; i < 63; i++) - { - inputTokens.emplace_back(i + request); - } - requests.emplace_back(inputTokens, maxNewTokens, streaming); - } - - for (auto req = 0; req < requests.size(); ++req) - { - auto& request = requests.at(req); - - auto requestId = executor.enqueueRequest(std::move(request)); - - // Get the new tokens - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - // This request failed for some reason, get error msg - std::string errStr - = "Request id " + std::to_string(requestId) + " failed with err " + response.getErrorMsg(); - FAIL(); - } - else - { - auto result = response.getResult(); - done = result.isFinal; - if (done) - { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - auto events = kvCacheManager->getLatestEvents(std::chrono::milliseconds(100)); - if (req == 0) - { - EXPECT_EQ(events.size(), 3); - - // Store the first context block - EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).parentHash, std::nullopt); - EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks.size(), 1); - events.pop_front(); - // Store the second (now completed) context block and the partial decode block. - EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks.size(), 1); - EXPECT_EQ(std::get<KVCacheStoredData>(events.back().data).blocks.size(), 1); - EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks[0].blockHash, - std::get<KVCacheStoredData>(events.back().data).parentHash); - } - else - { - EXPECT_EQ(events.size(), 5); - - // Remove a block to make room for the second context block. On the second request, we need - // to remove 2 blocks. - EXPECT_EQ(std::get<KVCacheRemovedData>(events.front().data).blockHashes.size(), req); - events.pop_front(); - // Store the first filled context block - EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks.size(), 1); - events.pop_front(); - // Remove a block for the decode phase - EXPECT_EQ(std::get<KVCacheRemovedData>(events.front().data).blockHashes.size(), 1); - events.pop_front(); - // Store the final context block and the decode block - EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks.size(), 1); - events.pop_front(); - EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks.size(), 1); - } - } - } - } - iter++; - } - EXPECT_EQ(done, true); - } -} - -TEST_F(GptExecutorTest, SingleRequestLora) -{ - bool streaming = true; - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Load lora weights, config - auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); - auto loraWeightsTensor - = std::shared_ptr(tr::utils::loadNpy(manager, LORA_WEIGHTS_FILE.string(), tr::MemoryType::kCPU)); - auto loraConfigTensor - = std::shared_ptr(tr::utils::loadNpy(manager, LORA_CONFIG_FILE.string(), tr::MemoryType::kCPU)); - - // Create the request - SizeType32 maxNewTokens = 5; - VecTokens inputTokens{1, 2, 3, 4}; - auto request = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig()); - auto loraConfig = LoraConfig(0, detail::ofITensor(loraWeightsTensor), detail::ofITensor(loraConfigTensor)); - request.setLoraConfig(loraConfig); - - // Enqueue the request - auto requestId = executor.enqueueRequest(std::move(request)); - - // Get the new tokens - VecTokens tokens; - bool done = false; - int iter = 0; - std::chrono::milliseconds waitTime(1); - while (!done && iter < mMaxWaitMs) - { - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - // This request failed for some reason, get error msg - std::string errStr - = "Request id " + std::to_string(requestId) + " failed with err " + response.getErrorMsg(); - FAIL(); - } - else - { - auto result = response.getResult(); - done = result.isFinal; - // Append tokens - auto& newTokens = result.outputTokenIds.at(beamWidth - 1); - tokens.insert( - tokens.end(), std::make_move_iterator(newTokens.begin()), std::make_move_iterator(newTokens.end())); - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); - EXPECT_EQ(tokens.size(), maxNewTokens); -} - -TEST_P(GuidedDecodingParamsTest, All) -{ - auto const modelName = std::get<0>(GetParam()); - std::filesystem::path enginePath; - std::filesystem::path tokenizerInfoPath; - int tp_size = 1, pp_size = 1, cp_size = 1; - std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; - - if (modelName == "gpt") - { - enginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - tokenizerInfoPath = GPT_XGRAMMAR_TOKENIZER_INFO_PATH; - } - else if (modelName == "llama_tp1_pp1_cp1") - { - enginePath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - tokenizerInfoPath = LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH; - } - else if (modelName == "llama_tp4_pp1_cp1") - { - enginePath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; - tokenizerInfoPath = LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH; - tp_size = 4; - } - else if (modelName == "llama_tp1_pp4_cp1") - { - enginePath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; - tokenizerInfoPath = LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH; - pp_size = 4; - deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; - } - else if (modelName == "llama_tp2_pp2_cp1") - { - enginePath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; - tokenizerInfoPath = LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH; - tp_size = 2; - pp_size = 2; - deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; - } - else - { - TLLM_THROW("Unrecognized modelName"); - } - - auto& comm = tensorrt_llm::mpi::MpiComm::world(); - auto const worldRank = comm.getRank(); - auto const worldSize = comm.getSize(); - - if (tp_size * pp_size * cp_size > 1) - { - // Run multi GPU test only when env variable is set - char const* val = getenv("RUN_LLAMA_MULTI_GPU"); - if (val == NULL) - { - GTEST_SKIP() << "Skipping multi-gpu guided decoding test"; - } - else - { - if (worldSize != 4) - { - FAIL() << "Leader mode and world size is not equal to 4"; - } - } - } - - bool streaming = false; - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - - auto const tokenizerInfo = nlohmann::json::parse(std::ifstream{tokenizerInfoPath}); - auto const encodedVocab = tokenizerInfo["encoded_vocab"].template get<std::vector<std::string>>(); - auto const tokenizerStr = tokenizerInfo["tokenizer_str"].template get<std::string>(); - auto const stopTokenIds = tokenizerInfo["stop_token_ids"].template get<std::vector<TokenIdType>>(); - GuidedDecodingConfig guidedDecodingConfig( - GuidedDecodingConfig::GuidedDecodingBackend::kXGRAMMAR, encodedVocab, tokenizerStr, stopTokenIds); - executorConfig.setGuidedDecodingConfig(guidedDecodingConfig); - - if (deviceIds.has_value()) - { - auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); - - parallelConfig.setDeviceIds(deviceIds.value()); - executorConfig.setParallelConfig(parallelConfig); - } - auto executor = Executor(enginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the requests - VecTokens inputTokens; - if (modelName == "gpt") - { - inputTokens = {2061, 318, 352, 10, 16, 30, 23998, 39559, 287, 257, 8633, 287, 33918, 5794, 25, 220}; - } - else // llama - { - inputTokens = { - 128000, 62, 3923, 7037, 62, 16, 10, 16, 30, 62, 16533, 87710, 1265, 4404, 5356, 1265, 9643, 9132, 25, 62}; - } - SizeType32 maxNewTokens = 10; - SamplingConfig samplingConfig{}; - OutputConfig outputConfig{false, false, false, true}; - - std::vector<Request> requests; - requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); - - requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); - requests.back().setGuidedDecodingParams(GuidedDecodingParams(GuidedDecodingParams::GuideType::kJSON)); - - requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); - std::string jsonSchema{ - R"({"properties": {"answer": {"title": "Answer", "type": "integer"}}, "required": ["answer"], "title": "Answer", "type": "object"})"}; - requests.back().setGuidedDecodingParams( - GuidedDecodingParams(GuidedDecodingParams::GuideType::kJSON_SCHEMA, jsonSchema)); - - requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); - std::string regex{R"(\d+)"}; - requests.back().setGuidedDecodingParams(GuidedDecodingParams(GuidedDecodingParams::GuideType::kREGEX, regex)); - - requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); - std::string ebnfGrammar{R"(root ::= [0-9]+)"}; - requests.back().setGuidedDecodingParams( - GuidedDecodingParams(GuidedDecodingParams::GuideType::kEBNF_GRAMMAR, ebnfGrammar)); - - std::vector<VecTokens> expectedOutputTokens; - if (modelName == "gpt") - { - expectedOutputTokens.push_back({1849, 7, 16, 10, 16, 8, 198, 16, 10, 16}); - expectedOutputTokens.push_back({90, 366, 3672, 1298, 366, 7554, 31780, 1600, 366, 12888}); - expectedOutputTokens.push_back({90, 366, 64, 77, 2032, 68, 81, 1, 1058, 352}); - expectedOutputTokens.push_back({25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645}); - expectedOutputTokens.push_back({25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645}); - } - else // llama - { - expectedOutputTokens.push_back({16, 10, 16, 28, 17, 198, 62, 3923, 7037, 62}); - expectedOutputTokens.push_back({5018, 16, 794, 330, 16, 498, 330, 17, 794, 330}); - expectedOutputTokens.push_back({5018, 9399, 794, 16, 92}); - expectedOutputTokens.push_back({16}); - expectedOutputTokens.push_back({16}); - } - - if (executor.canEnqueueRequests()) - { - // Enqueue the requests - auto reqIds = executor.enqueueRequests(std::move(requests)); - - // Get the responses - int numFinished = 0; - int iter = 0; - while (numFinished < 5 && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - auto reqId = response.getRequestId(); - if (response.hasError()) - { - // This request failed for some reason, get error msg - std::string errStr - = "Request id " + std::to_string(reqId) + " failed with err " + response.getErrorMsg(); - FAIL(); - } - else - { - auto result = response.getResult(); - auto& newTokens = result.outputTokenIds.at(0); - - int reqIdx = std::find(reqIds.begin(), reqIds.end(), reqId) - reqIds.begin(); - EXPECT_THAT(newTokens, ::testing::ElementsAreArray(expectedOutputTokens[reqIdx])); - } - numFinished++; - } - } - EXPECT_LT(iter, mMaxWaitMs); - EXPECT_EQ(numFinished, 5); - } -} - -TEST_F(GptExecutorTest, GuidedDecodingFailure) -{ - bool streaming = false; - - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - - std::vector<int> stopTokenIds{50256}; - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the requests - SizeType32 maxNewTokens = 10; - SamplingConfig samplingConfig{}; - OutputConfig outputConfig{false, false, false, true}; - VecTokens inputTokens{2061, 318, 352, 10, 16, 30, 23998, 39559, 287, 257, 8633, 287, 33918, 5794, 25, 220}; - - std::vector<Request> requests; - requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); - requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); - requests.back().setGuidedDecodingParams(GuidedDecodingParams(GuidedDecodingParams::GuideType::kJSON)); - - // Enqueue the requests - auto reqIds = executor.enqueueRequests(std::move(requests)); - - // Get the responses - int numFinished = 0; - int iter = 0; - while (numFinished < 2 && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - auto reqId = response.getRequestId(); - int reqIdx = std::find(reqIds.begin(), reqIds.end(), reqId) - reqIds.begin(); - if (reqIdx == 0) - { - EXPECT_FALSE(response.hasError()); - } - else - { - EXPECT_TRUE(response.hasError()); - } - numFinished++; - } - } - EXPECT_LT(iter, mMaxWaitMs); - EXPECT_EQ(numFinished, 2); -} - -TEST_P(ParamTest, SingleRequestCancelRequest) -{ - bool const streaming = std::get<0>(GetParam()); - bool const excludeInputFromOutput = std::get<1>(GetParam()); - auto const beamWidth = std::get<2>(GetParam()); - OutputConfig outConfig; - outConfig.excludeInputFromOutput = excludeInputFromOutput; - - auto executorConfig = ExecutorConfig(beamWidth); - auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - SizeType32 maxNewTokens = 300; - VecTokens inputTokens{1, 2, 3, 4}; - auto request - = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - - auto requestId = executor.enqueueRequest(std::move(request)); - - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - executor.cancelRequest(requestId); - - // Try to get the new tokens - bool done = false; - int iter = 0; - VecTokens tokens; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(requestId, waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - FAIL() << "Did not expect errors"; - } - else - { - auto result = response.getResult(); - done = result.isFinal; - // Append tokens - auto& newTokens = result.outputTokenIds.at(beamWidth - 1); - if (done) - { - for (SizeType32 beamIdx = 0; beamIdx < beamWidth; ++beamIdx) - { - EXPECT_EQ(result.finishReasons[beamIdx], FinishReason::kCANCELLED); - } - } - - if (streaming && beamWidth > 1) - { - tokens = newTokens; - } - else - { - tokens.insert(tokens.end(), newTokens.begin(), newTokens.end()); - } - } - } - ++iter; - } - EXPECT_EQ(done, true); - EXPECT_LT(iter, mMaxWaitMs); - auto expectedNumTokens - = streaming ? maxNewTokens : (excludeInputFromOutput ? 0 : inputTokens.size()) + maxNewTokens; - TLLM_LOG_INFO("num tokens: %d, expected %d", tokens.size(), expectedNumTokens); - EXPECT_LT(tokens.size(), expectedNumTokens); -} - -TEST_F(GptExecutorTest, orchModeFetchNewReqErr) -{ - SizeType32 beamWidth = 1; - auto executorConfig = ExecutorConfig(beamWidth); - - auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); - auto parallelConfig = ParallelConfig( - CommunicationType::kMPI, CommunicationMode::kORCHESTRATOR, std::nullopt, std::nullopt, orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Create a req with invalid parameters - SizeType32 maxNewTokens = 5; - // Create very long prompt which should result in error during request validate - VecTokens inputTokens(10000000); - - auto request = Request(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); - auto requestId = executor.enqueueRequest(request); - auto requestId2 = executor.enqueueRequest(request); - - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - auto err = response.getErrorMsg(); - EXPECT_THAT(err, testing::HasSubstr("exceeds maximum input length")); - EXPECT_THAT(err, testing::HasSubstr("Encountered an error when fetching new request:")); - done = true; - } - else - { - FAIL() << "Should get a response with error"; - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); -} - -TEST_F(GptExecutorTest, orchModeForwardError) -{ - SizeType32 constexpr maxBeamWidth{1}; - auto executorConfig = ExecutorConfig(maxBeamWidth); - - auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); - auto parallelConfig = ParallelConfig( - CommunicationType::kMPI, CommunicationMode::kORCHESTRATOR, std::nullopt, std::nullopt, orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - - // Setting request beam width to 2 which should cause failure - SizeType32 constexpr beamWidth{2}; - SizeType32 constexpr maxNewTokens{5}; - VecTokens inputTokens{1, 2, 3, 4}; - - auto request = Request(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); - auto requestId = executor.enqueueRequest(request); - auto requestId2 = executor.enqueueRequest(request); - - bool done = false; - int iter = 0; - while (!done && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - if (response.hasError()) - { - auto err = response.getErrorMsg(); - std::cout << "err:" << err << std::endl; - EXPECT_THAT( - err, testing::HasSubstr("Requested beam width 2 is larger than configured max beam width 1")); - done = true; - } - else - { - FAIL() << "Should get a response with error"; - } - } - ++iter; - } - EXPECT_LT(iter, mMaxWaitMs); -} - -TEST_P(ParamCancelReqTest, MultipleRequestsMultiGpuCancelRequest) -{ - auto const useOrchestratorMode = std::get<0>(GetParam()); - auto const beamWidth = std::get<1>(GetParam()); - auto const modelName = std::get<2>(GetParam()); - - std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; - - OutputConfig outConfig; - - auto executorConfig = ExecutorConfig(beamWidth); - std::filesystem::path modelPath; - if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1") - { - if (modelName == "llama_tp4_pp1_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp4_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; - deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; - } - else if (modelName == "llama_tp2_pp2_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; - deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; - } - } - - // For llama model, only run for multiple GPUs - // This is detected by setting an env variable when running the test - char const* val = getenv("RUN_LLAMA_MULTI_GPU"); - if (val == NULL) - { - GTEST_SKIP() << "Skipping Llama test"; - } - else - { - // Check that it was launched with right number of MPI ranks - if (!useOrchestratorMode && COMM_SESSION.getSize() != 4) - { - // No orchestrator, need worldSize to match TP*PP - FAIL() << "Leader mode and world size is not equal to 4"; - } - else if (useOrchestratorMode && COMM_SESSION.getSize() != 1) - { - // No orchestrator, need worldSize to match TP*PP - FAIL() << "Orchestrator mode and World size is not equal to 1"; - } - } - - if (useOrchestratorMode) - { - auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); - auto parallelConfig = ParallelConfig(CommunicationType::kMPI, - useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::nullopt, - std::nullopt, orchestratorConfig); - if (deviceIds.has_value()) - { - parallelConfig.setDeviceIds(deviceIds.value()); - } - executorConfig.setParallelConfig(parallelConfig); - } - else - { - if (deviceIds.has_value()) - { - auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); - parallelConfig.setDeviceIds(deviceIds.value()); - executorConfig.setParallelConfig(parallelConfig); - } - } - - auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - SizeType32 maxNewTokens = 50; - VecTokens inputTokens{1, 2, 3, 4}; - - std::vector<Request> requests; - for (auto streaming : {false, true}) - { - // Add two requests with numReturnSequences = 1 - auto samplingConfig = tensorrt_llm::executor::SamplingConfig(beamWidth); - requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outConfig); - requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outConfig); - // Add a request with numReturnSequences > 1 - auto samplingConfig2 = tensorrt_llm::executor::SamplingConfig(beamWidth); - auto constexpr numReturnSequences = 2; - samplingConfig2.setNumReturnSequences(numReturnSequences); - requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig2, outConfig); - } - std::vector<bool> cancelRequests{true, false, true, true, false, true}; - - if (executor.canEnqueueRequests()) - { - auto const requestIds = executor.enqueueRequests(requests); - - // Cancel the first and third requests - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - for (SizeType32 i = 0; i < requests.size(); i++) - { - if (cancelRequests.at(i)) - { - executor.cancelRequest(requestIds.at(i)); - } - } - - std::unordered_map<IdType, bool> isStreaming; - std::unordered_map<IdType, SizeType32> expectedNumTokens; - SizeType32 expectedNumResponses = 0; - for (SizeType32 i = 0; i < requests.size(); i++) - { - auto const& request = requests.at(i); - auto requestId = requestIds.at(i); - isStreaming[requestId] = request.getStreaming(); - expectedNumTokens[requestId] = (request.getStreaming() ? 0 : inputTokens.size()) + maxNewTokens; - auto const numResponses = request.getStreaming() ? expectedNumTokens[requestId] : 1; - auto const numReturnSequences = request.getSamplingConfig().getBeamWidth() > 1 - ? 1 - : request.getSamplingConfig().getNumReturnSequences().value_or(1); - expectedNumResponses += numResponses * numReturnSequences; - } - - std::unordered_map<IdType, std::unordered_map<SizeType32, VecTokens>> tokens; - - // Get the new tokens for each requests - int32_t numFinished = 0; - int iter = 0; - SizeType32 numResponses = 0; - while (numFinished < requests.size() && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - numResponses++; - if (!response.hasError()) - { - auto requestId = response.getRequestId(); - auto result = response.getResult(); - numFinished += result.isFinal; - auto seqIdx = result.sequenceIndex; - auto numSequences = result.outputTokenIds.size(); - auto& newTokens = result.outputTokenIds.at(numSequences - 1); - auto& reqResults = tokens[response.getRequestId()]; - auto& reqTokens = reqResults[seqIdx]; - if (isStreaming.at(requestId) && beamWidth > 1) - { - reqTokens = newTokens; - } - else - { - reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); - } - } - else - { - FAIL() << "Did not expect errors"; - } - } - ++iter; - } - - EXPECT_LE(numResponses, expectedNumResponses); - EXPECT_EQ(numFinished, requests.size()); - EXPECT_LT(iter, mMaxWaitMs); - - for (auto requestIdx = 0; requestIdx < requests.size(); requestIdx++) - { - auto const requestId = requestIds.at(requestIdx); - for (auto seqIdx = 0; seqIdx < tokens.at(requestId).size(); seqIdx++) - { - auto const& seqTokens = tokens.at(requestId).at(seqIdx); - if (cancelRequests.at(requestIdx)) - { - EXPECT_LT(seqTokens.size(), expectedNumTokens.at(requestId)); - } - else - { - EXPECT_EQ(seqTokens.size(), expectedNumTokens.at(requestId)); - } - } - } - } -} - -TEST_P(LeaderApiUsageTest, LeaderModeTest) -{ - auto const modelName = std::get<0>(GetParam()); - - SizeType32 beamWidth = 2; - OutputConfig outConfig; - std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; - - auto executorConfig = ExecutorConfig(beamWidth); - std::filesystem::path modelPath; - if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1") - { - if (modelName == "llama_tp4_pp1_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp4_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; - deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; - } - else if (modelName == "llama_tp2_pp2_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; - deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; - } - } - - // For llama model, only run for multiple GPUs - // This is detected by setting an env variable when running the test - char const* val = getenv("RUN_LLAMA_MULTI_GPU"); - if (val == NULL) - { - GTEST_SKIP() << "Skipping Llama test"; - } - else - { - // Check that it was launched with right number of MPI ranks - if (COMM_SESSION.getSize() != 4) - { - // No orchestrator, need worldSize to match TP*PP - FAIL() << "Leader mode and world size is not equal to 4"; - } - } - - if (deviceIds.has_value()) - { - auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); - - parallelConfig.setDeviceIds(deviceIds.value()); - executorConfig.setParallelConfig(parallelConfig); - } - auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); - - // Since this is leader mode, all ranks should participate - EXPECT_TRUE(executor.isParticipant()); - - // Create the request - SizeType32 maxNewTokens = 50; - VecTokens inputTokens{1, 2, 3, 4}; - auto request - = Request(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - auto requestStreaming - = Request(inputTokens, maxNewTokens, true, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); - - // Leader enqueues requests and wait for responses - if (executor.canEnqueueRequests()) - { - auto requestId = executor.enqueueRequest(request); - auto requestId2 = executor.enqueueRequest(request); - auto requestId3 = executor.enqueueRequest(requestStreaming); - auto requestId4 = executor.enqueueRequest(requestStreaming); - - int32_t numFinished = 0; - int iter = 0; - SizeType32 numResponses = 0; - while (numFinished < 4 && iter < mMaxWaitMs) - { - std::chrono::milliseconds waitTime(1); - auto responses = executor.awaitResponses(waitTime); - for (auto& response : responses) - { - numResponses++; - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - } - else - { - FAIL() << "Did not expect errors"; - } - } - ++iter; - } - EXPECT_EQ(numFinished, 4); - EXPECT_LT(iter, mMaxWaitMs); - } - else - { - // Check that non-leader cannot enqueue requests - EXPECT_THROW({ auto reqId = executor.enqueueRequest(request); }, tensorrt_llm::common::TllmException); - EXPECT_THROW({ auto responses = executor.awaitResponses(); }, tensorrt_llm::common::TllmException); - EXPECT_THROW({ auto numResp = executor.getNumResponsesReady(); }, tensorrt_llm::common::TllmException); - EXPECT_THROW({ executor.cancelRequest(1); }, tensorrt_llm::common::TllmException); - EXPECT_THROW({ auto stats = executor.getLatestIterationStats(); }, tensorrt_llm::common::TllmException); - EXPECT_THROW({ auto stats = executor.getLatestRequestStats(); }, tensorrt_llm::common::TllmException); - } -} - -TEST_F(GptExecutorTest, validateParallelConfig) -{ - - auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); - { - auto executorConfig = ExecutorConfig(); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - } - - { - std::string expectedErrMsg = "OrchestratorConfig must be set"; - try - { - auto executorConfig = ExecutorConfig(); - auto parallelConfig = ParallelConfig(CommunicationType::kMPI, CommunicationMode::kORCHESTRATOR); - executorConfig.setParallelConfig(parallelConfig); - auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); - FAIL() << "Expected TllmException"; - } - catch (tc::TllmException& e) - { - EXPECT_THAT(e.what(), testing::HasSubstr(expectedErrMsg)); - } - catch (std::exception const& e) - { - FAIL() << "Expected TllmException"; - } - } -} - -TEST_P(TimeoutTest, TimeoutStreamingTest) -{ - auto const modelName = std::get<0>(GetParam()); - auto const useOrchestratorMode = std::get<1>(GetParam()); - auto const beamWidth = std::get<2>(GetParam()); - - auto executorConfig = ExecutorConfig(beamWidth); - std::filesystem::path modelPath; - bool isMultiGpu{false}; - std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; - - if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1") - { - isMultiGpu = true; - if (modelName == "llama_tp4_pp1_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp4_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; - deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; - } - else if (modelName == "llama_tp2_pp2_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; - deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; - } - } - if (modelName == "llama_tp1_pp1_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - } - // For llama model, only run for multiple GPUs - // This is detected by setting an env variable when running the test - char const* val = getenv("RUN_LLAMA_MULTI_GPU"); - if (val == NULL && isMultiGpu) - { - GTEST_SKIP() << "Skipping MultiGpu tests"; - } - if (val != NULL && !isMultiGpu) - { - GTEST_SKIP() << "Skipping SingleGpu tests"; - } - if (val != NULL && isMultiGpu) - { - // Check that it was launched with right number of MPI ranks - if (!useOrchestratorMode && COMM_SESSION.getSize() != 4) - { - // No orchestrator, need worldSize to match TP*PP - FAIL() << "Leader mode and world size is not equal to 4"; - } - if (useOrchestratorMode && COMM_SESSION.getSize() != 1) - { - // No orchestrator, need worldSize to match TP*PP - FAIL() << "Orchestrator mode and World size is not equal to 1"; - } - } - - if (useOrchestratorMode) - { - auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); - auto parallelConfig = ParallelConfig(CommunicationType::kMPI, - useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::nullopt, - std::nullopt, orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - if (deviceIds.has_value()) - { - parallelConfig.setDeviceIds(deviceIds.value()); - } - executorConfig.setParallelConfig(parallelConfig); - } - else - { - if (deviceIds.has_value()) - { - auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); - parallelConfig.setDeviceIds(deviceIds.value()); - executorConfig.setParallelConfig(parallelConfig); - } - } - auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); - - SizeType32 constexpr maxNewTokens = 10; - // create 1 request that times out immediately - // momentarily we don't cancel requests before forwardAsync so it will get scheduled for at least 1 forward - VecTokens immediateCancelTokens{1, 2, 3, 4}; - auto immediateCancelRequest - = Request(immediateCancelTokens, maxNewTokens, true, tensorrt_llm::executor::SamplingConfig(beamWidth)); - immediateCancelRequest.setReturnAllGeneratedTokens(true); - immediateCancelRequest.setAllottedTimeMs(std::chrono::milliseconds(0)); - SizeType32 constexpr immediateCancelMinLength = 0; - SizeType32 constexpr immediateCancelMaxLength = 1; - - // create 1 request that times out during the first forward - VecTokens oneForwardTokens{11, 12, 13, 14}; - auto oneForwardRequest - = Request(oneForwardTokens, maxNewTokens, true, tensorrt_llm::executor::SamplingConfig(beamWidth)); - oneForwardRequest.setReturnAllGeneratedTokens(true); - oneForwardRequest.setAllottedTimeMs(std::chrono::milliseconds(1)); - SizeType32 constexpr oneForwardlMinLength = 0; - SizeType32 constexpr oneForwardlMaxLength = 1; - - // Create the request that finishes by the number of tokens - VecTokens finishedTokens{101, 102, 103, 104}; - auto finishedRequest - = Request(finishedTokens, maxNewTokens, true, tensorrt_llm::executor::SamplingConfig(beamWidth)); - finishedRequest.setReturnAllGeneratedTokens(true); - finishedRequest.setAllottedTimeMs(std::chrono::milliseconds(5000)); - SizeType32 constexpr finishedMinLength = 5; - SizeType32 constexpr finishedMaxLength = maxNewTokens; - - std::vector<FinishReason> referenceFinishReasons - = {FinishReason::kTIMED_OUT, FinishReason::kTIMED_OUT, FinishReason::kLENGTH}; - std::vector<SizeType32> minLengths = {immediateCancelMinLength, oneForwardlMinLength, finishedMinLength}; - std::vector<SizeType32> maxLengths = {immediateCancelMaxLength, oneForwardlMaxLength, finishedMaxLength}; - // workaround because the last response will be empty, but we want to have at least *some* responses surpass the - // minLength - std::vector<SizeType32> achievedLength = {0, 0, 0}; - SizeType32 itNr{0}; - - if (executor.canEnqueueRequests()) - { - - std::vector<Request> requests = {immediateCancelRequest, oneForwardRequest, finishedRequest}; - auto requestIds = executor.enqueueRequests(requests); - - auto numFinished = 0; - - while (numFinished < static_cast<SizeType32>(requests.size())) - { - itNr++; - std::chrono::milliseconds waitTime(mMaxWaitMs); - auto responses = executor.awaitResponses(requestIds, waitTime); - for (auto const& response : responses) - { - for (auto const& responseIt : response) - { - auto const reqId = responseIt.getRequestId(); - if (responseIt.hasError()) - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err - = "ReqId " + std::to_string(reqId) + " has already been processed and was terminated."; - if (responseIt.getErrorMsg() != err) - { - TLLM_THROW("Request id %lu encountered error: %s", reqId, responseIt.getErrorMsg().c_str()); - } - continue; - } - - auto const& result = responseIt.getResult(); - if (result.isFinal) - { - requestIds.erase(std::remove(requestIds.begin(), requestIds.end(), reqId), requestIds.end()); - numFinished++; - } - - auto const finishReason = result.finishReasons; - auto const actualResponse = result.outputTokenIds; - TLLM_LOG_DEBUG("reqId %d finished %d", reqId, result.isFinal); - TLLM_LOG_DEBUG("actual response:"); - - for (auto const& beam : actualResponse) - { - std::string tokenStr; - for (auto tok : beam) - { - tokenStr += std::to_string(tok) + " "; - } - TLLM_LOG_DEBUG("%s", tokenStr.c_str()); - } - - TLLM_LOG_DEBUG( - "beams' length must be in range [%d, %d]", minLengths[reqId - 1], maxLengths[reqId - 1]); - - if (result.isFinal) - { - TLLM_LOG_DEBUG("finishReason"); - std::string reasonStr; - for (auto const reason : finishReason) - { - // cast for easier visibility during debugging - EXPECT_EQ(static_cast<int>(reason), static_cast<int>(referenceFinishReasons[reqId - 1])); - reasonStr += std::to_string(static_cast<int>(reason)) + " "; - } - TLLM_LOG_DEBUG("%s", reasonStr.c_str()); - } - - EXPECT_EQ(beamWidth, actualResponse.size()); - for (int beam = 0; beam < beamWidth; beam++) - { - EXPECT_LE(actualResponse.at(beam).size(), maxLengths[reqId - 1]) << "for request " << reqId; - achievedLength[reqId - 1] = std::max( - achievedLength[reqId - 1], static_cast<SizeType32>(actualResponse.at(beam).size())); - } - } - } - } - - for (int reqIt = 0; reqIt < achievedLength.size(); ++reqIt) - { - EXPECT_GE(achievedLength[reqIt], minLengths[reqIt]) - << "request " << reqIt + 1 << " has not achieved min lengths"; - } - } -} - -TEST_P(TimeoutTest, TimeoutNonstreamingTest) -{ - auto const modelName = std::get<0>(GetParam()); - auto const useOrchestratorMode = std::get<1>(GetParam()); - auto const beamWidth = std::get<2>(GetParam()); - - std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; - - auto executorConfig = ExecutorConfig(beamWidth); - std::filesystem::path modelPath; - bool isMultiGpu{false}; - if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1") - { - isMultiGpu = true; - if (modelName == "llama_tp4_pp1_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; - } - else if (modelName == "llama_tp1_pp4_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; - deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; - } - else if (modelName == "llama_tp2_pp2_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; - deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; - } - } - if (modelName == "llama_tp1_pp1_cp1") - { - modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; - } - // For llama model, only run for multiple GPUs - // This is detected by setting an env variable when running the test - char const* val = getenv("RUN_LLAMA_MULTI_GPU"); - if (val == NULL && isMultiGpu) - { - GTEST_SKIP() << "Skipping MultiGpu tests"; - } - if (val != NULL && !isMultiGpu) - { - GTEST_SKIP() << "Skipping SingleGpu tests"; - } - if (val != NULL && isMultiGpu) - { - // Check that it was launched with right number of MPI ranks - if (!useOrchestratorMode && COMM_SESSION.getSize() != 4) - { - // No orchestrator, need worldSize to match TP*PP - FAIL() << "Leader mode and world size is not equal to 4"; - } - if (useOrchestratorMode && COMM_SESSION.getSize() != 1) - { - // No orchestrator, need worldSize to match TP*PP - FAIL() << "Orchestrator mode and World size is not equal to 1"; - } - } - - if (useOrchestratorMode) - { - auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); - auto parallelConfig = ParallelConfig(CommunicationType::kMPI, - useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::nullopt, - std::nullopt, orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - if (deviceIds.has_value()) - { - parallelConfig.setDeviceIds(deviceIds.value()); - } - executorConfig.setParallelConfig(parallelConfig); - } - else - { - if (deviceIds.has_value()) - { - auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); - parallelConfig.setDeviceIds(deviceIds.value()); - executorConfig.setParallelConfig(parallelConfig); - } - } - auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); - - SizeType32 constexpr maxNewTokens = 5; - // create 1 request that times out immediately - // momentarily we don't cancel requests before forwardAsync so it will get scheduled for at least 1 forward - VecTokens immediateCancelTokens{1, 2, 3, 4}; - auto immediateCancelRequest - = Request(immediateCancelTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); - immediateCancelRequest.setAllottedTimeMs(std::chrono::milliseconds(0)); - std::vector<std::vector<int>> immediateCancelResponse = {immediateCancelTokens, immediateCancelTokens}; - - // create 1 request that times out during the first forward - VecTokens oneForwardTokens{11, 12, 13, 14}; - auto oneForwardRequest - = Request(oneForwardTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); - oneForwardRequest.setAllottedTimeMs(std::chrono::milliseconds(1)); - std::vector<std::vector<int>> oneForwardResponse = {oneForwardTokens, oneForwardTokens}; - - // Create the request that finishes by the number of tokens - VecTokens finishedTokens{101, 102, 103, 104}; - auto finishedRequest - = Request(finishedTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); - finishedRequest.setAllottedTimeMs(std::chrono::milliseconds(6000)); - std::vector<std::vector<int>> finishedReponse - = {{101, 102, 103, 104, 49849, 225, 49849, 232, 55742}, {101, 102, 103, 104, 49849, 225, 49849, 232, 29082}}; - - // assume responses will come in FIFO order - std::vector<BeamTokens> refResponses = {immediateCancelResponse, oneForwardResponse, finishedReponse}; - std::vector<FinishReason> referenceFinishReasons - = {FinishReason::kTIMED_OUT, FinishReason::kTIMED_OUT, FinishReason::kLENGTH}; - if (executor.canEnqueueRequests()) - { - - std::vector<Request> requests = {immediateCancelRequest, oneForwardRequest, finishedRequest}; - auto requestIds = executor.enqueueRequests(requests); - - std::chrono::milliseconds waitTime(mMaxWaitMs); - auto responses = executor.awaitResponses(requestIds, waitTime); - for (auto const& response : responses) - { - for (auto const& responseIt : response) - { - auto const reqId = responseIt.getRequestId(); - if (responseIt.hasError()) - { - TLLM_THROW("Request id %lu encountered error: %s", reqId, responseIt.getErrorMsg().c_str()); - } - - auto const& result = responseIt.getResult(); - - auto const finishReason = result.finishReasons; - auto const actualResponse = result.outputTokenIds; - TLLM_LOG_DEBUG("reqId %d finished %d", reqId, result.isFinal); - TLLM_LOG_DEBUG("actual response:"); - - for (auto const& beam : actualResponse) - { - std::string tokenStr; - for (auto tok : beam) - { - tokenStr += std::to_string(tok) + " "; - } - TLLM_LOG_DEBUG("%s", tokenStr.c_str()); - } - - TLLM_LOG_DEBUG("reference:"); - auto referenceResponse = refResponses[reqId - 1]; - for (auto const& beam : referenceResponse) - { - std::string tokenStr; - for (auto tok : beam) - { - tokenStr += std::to_string(tok) + " "; - } - TLLM_LOG_DEBUG("%s", tokenStr.c_str()); - } - - if (result.isFinal) - { - TLLM_LOG_DEBUG("finishReason"); - std::string reasonStr; - for (auto const reason : finishReason) - { - // cast for easier visibility during debugging - EXPECT_EQ(static_cast<int>(reason), static_cast<int>(referenceFinishReasons[reqId - 1])); - reasonStr += std::to_string(static_cast<int>(reason)) + " "; - } - TLLM_LOG_DEBUG("%s", reasonStr.c_str()); - } - - EXPECT_EQ(beamWidth, actualResponse.size()); - for (int beam = 0; beam < beamWidth; beam++) - { - EXPECT_EQ(referenceResponse.at(beam).size(), actualResponse.at(beam).size()); - EXPECT_THAT(actualResponse.at(beam), testing::ElementsAreArray(referenceResponse.at(beam))); - } - } - } - } -} - -INSTANTIATE_TEST_SUITE_P(GptExecutorTest, ParamTest, - testing::Combine( // - testing::Values(false, true), // streaming - testing::Values(false, true), // excludeInputFromOutput - testing::Values(1, 2) // beamWidth - ), - generateTestName); - -INSTANTIATE_TEST_SUITE_P(GptExecutorTest, ParamStatsTest, - testing::Combine( // - testing::Values(0, 1000), // iterStatsMaxIterations - testing::Values(false, true) // useOrchestratorMode - ), - generateTestNameStats); - -INSTANTIATE_TEST_SUITE_P(LlamaExecutorTest, ParamCancelReqTest, - testing::Combine( // - testing::Values(false, true), // useOrchestratorMode - testing::Values(1, 2), // beamWidth - testing::Values("llama_tp1_pp4_cp1", "llama_tp4_pp1_cp1", "llama_tp2_pp2_cp1") // modelName - ), - generateTestNameCancelReq); - -INSTANTIATE_TEST_SUITE_P(LlamaExecutorTest, TimeoutTest, - testing::Combine( // - testing::Values("llama_tp1_pp4_cp1", "llama_tp4_pp1_cp1", "llama_tp1_pp1_cp1"), // modelName - testing::Values(false, true), // useOrchestratorMode - testing::Values(2) // beamWidth - ), - generateTestNameTimeoutTest); - -INSTANTIATE_TEST_SUITE_P(LlamaExecutorTest, LeaderApiUsageTest, - testing::Combine( // - testing::Values("llama_tp1_pp4_cp1", "llama_tp4_pp1_cp1", "llama_tp2_pp2_cp1") // modelName - ), - generateTestNameLeaderApiUsage); - -INSTANTIATE_TEST_SUITE_P(GptExecutorTest, AllParamsTest, - testing::Combine( // - testing::Values(false, true), // streaming - testing::Values(1, 2), // beamWidth - testing::Values(true), // computeLogProbs - testing::Values(false, true), // excludeInputInOutput - testing::Values(true), // returnContextLogits - testing::Values(true), // returnGenerationLogits - testing::Values("gpt"), // modelName - testing::Values(false, true), // useOrchestratorMode - testing::Values(false, true), // returnAllGeneratedTokens - testing::Values(1, 2) // numReturnSequences - ), - generateTestNameAllParams); - -INSTANTIATE_TEST_SUITE_P(LlamaExecutorTest, AllParamsTest, - testing::Combine( // - testing::Values(false, true), // streaming - testing::Values(1, 2), // beamWidth - testing::Values(true), // computeLogProbs - testing::Values(false, true), // excludeInputInOutput - testing::Values(false), // returnContextLogits - testing::Values(true), // returnGenerationLogits - testing::Values("llama_tp1_pp4_cp1", "llama_tp4_pp1_cp1", "llama_tp2_pp2_cp1"), // modelName - testing::Values(false, true), // useOrchestratorMode - testing::Values(false), // returnAllGeneratedTokens - testing::Values(1) // numReturnSequences - ), - generateTestNameAllParams); - -INSTANTIATE_TEST_SUITE_P(LlamaMultiExecutorTest, AllParamsTest, - testing::Combine( // - testing::Values(false, true), // streaming - testing::Values(1, 2), // beamWidth - testing::Values(false), // computeLogProbs - testing::Values(false, true), // excludeInputInOutput - testing::Values(false), // returnContextLogits - testing::Values(false), // returnGenerationLogits - testing::Values("llama_tp1_pp2_cp1"), // modelName - testing::Values(false), // useOrchestratorMode - testing::Values(false), // returnAllGeneratedTokens - testing::Values(1) // numReturnSequences - ), - generateTestNameAllParams); - -INSTANTIATE_TEST_SUITE_P(MedusaExecutorTest, AllParamsTest, - testing::Combine( // - testing::Values(false, true), // streaming - testing::Values(1), // beamWidth - testing::Values(false), // computeLogProbs - testing::Values(false, true), // excludeInputInOutput - testing::Values(false), // returnContextLogits - testing::Values(false), // returnGenerationLogits - testing::Values("medusa"), // modelName - testing::Values(false, true), // useOrchestratorMode - testing::Values(false), // returnAllGeneratedTokens - testing::Values(1) // numReturnSequences - ), - generateTestNameAllParams); - -// Disable some of ChatGLM's tests since they are the same as gpt's. -INSTANTIATE_TEST_SUITE_P(ChatGlmExecutorTest, AllParamsTest, - testing::Combine( // - testing::Values(false), // streaming - testing::Values(1, 2), // beamWidth - testing::Values(false), // computeLogProbs - testing::Values(false), // excludeInputInOutput - testing::Values(false), // returnContextLogits - testing::Values(false), // returnGenerationLogits - testing::Values("chatglm"), // modelName - testing::Values(false), // useOrchestratorMode - testing::Values(false), // returnAllGeneratedTokens - testing::Values(1, 2) // numReturnSequences - ), - generateTestNameAllParams); - -// ChatGlm0 Test is for glm-10b. -INSTANTIATE_TEST_SUITE_P(ChatGlm0ExecutorTest, AllParamsTest, - testing::Combine( // - testing::Values(false), // streaming - testing::Values(1), // beamWidth - testing::Values(false), // computeLogProbs - testing::Values(false), // excludeInputInOutput - testing::Values(false), // returnContextLogits - testing::Values(false), // returnGenerationLogits - testing::Values("glm"), // modelName - testing::Values(false), // useOrchestratorMode - testing::Values(false), // returnAllGeneratedTokens - testing::Values(1) // numReturnSequences - ), - generateTestNameAllParams); - -INSTANTIATE_TEST_SUITE_P(ChatGlm2ExecutorTest, AllParamsTest, - testing::Combine( // - testing::Values(false), // streaming - testing::Values(1), // beamWidth - testing::Values(false), // computeLogProbs - testing::Values(false), // excludeInputInOutput - testing::Values(false), // returnContextLogits - testing::Values(false), // returnGenerationLogits - testing::Values("chatglm2"), // modelName - testing::Values(false), // useOrchestratorMode - testing::Values(false), // returnAllGeneratedTokens - testing::Values(1) // numReturnSequences - ), - generateTestNameAllParams); - -INSTANTIATE_TEST_SUITE_P(ChatGlm3ExecutorTest, AllParamsTest, - testing::Combine( // - testing::Values(false), // streaming - testing::Values(1), // beamWidth - testing::Values(false), // computeLogProbs - testing::Values(false), // excludeInputInOutput - testing::Values(false), // returnContextLogits - testing::Values(false), // returnGenerationLogits - testing::Values("chatglm3"), // modelName - testing::Values(false), // useOrchestratorMode - testing::Values(false), // returnAllGeneratedTokens - testing::Values(1) // numReturnSequences - ), - generateTestNameAllParams); - -INSTANTIATE_TEST_SUITE_P(LlamaExecutorTest, LogitsProcParamsTest, - testing::Combine( // - testing::Values( - "llama_tp1_pp1_cp1", "llama_tp4_pp1_cp1", "llama_tp2_pp2_cp1", "llama_tp1_pp4_cp1"), // modelName - testing::Values(false, true), // batched - testing::Values(false, true) // replicated - ), - generateTestNameLogitsProc); - -INSTANTIATE_TEST_SUITE_P(GptExecutorGuidedDecodingTest, GuidedDecodingParamsTest, - testing::Combine(testing::Values("gpt")), generateTestNameGuidedDecoding); - -INSTANTIATE_TEST_SUITE_P(LlamaExecutorGuidedDecodingTest, GuidedDecodingParamsTest, - testing::Combine( - testing::Values("llama_tp1_pp1_cp1", "llama_tp4_pp1_cp1", "llama_tp2_pp2_cp1", "llama_tp1_pp4_cp1")), - generateTestNameGuidedDecoding); diff --git a/cpp/tests/e2e_tests/executor/executorTest.h b/cpp/tests/e2e_tests/executor/executorTest.h deleted file mode 100644 index 7866a6992266..000000000000 --- a/cpp/tests/e2e_tests/executor/executorTest.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tests/utils/common.h" - -#include <gmock/gmock.h> -#include <gtest/gtest.h> - -#include <memory> - -namespace tensorrt_llm::testing -{ - -class GptExecutorTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) -{ -public: - using SizeType32 = tensorrt_llm::testing::SizeType32; - -protected: - void SetUp() override - { - mDeviceCount = tensorrt_llm::common::getDeviceCount(); - if (mDeviceCount == 0) - { - GTEST_SKIP() << "No GPUs found"; - } - - mLogger = std::make_shared<tensorrt_llm::runtime::TllmLogger>(); - initTrtLlmPlugins(mLogger.get()); - } - - void TearDown() override {} - - int mDeviceCount{}; - std::shared_ptr<nvinfer1::ILogger> mLogger{}; - SizeType32 mMaxWaitMs = 300000; - SizeType32 mTrigWarnMs = 10000; -}; - -} // namespace tensorrt_llm::testing diff --git a/cpp/tests/resources/scripts/build_chatglm_engines.py b/cpp/tests/resources/scripts/build_chatglm_engines.py deleted file mode 100644 index abe187307604..000000000000 --- a/cpp/tests/resources/scripts/build_chatglm_engines.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import platform -import shutil -import sys -import typing -from pathlib import Path -from typing import Optional - -from build_engines_utils import run_command, wincopy - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - -resources_dir = Path(__file__).parent.resolve().parent -model_dir = resources_dir / "models" -chatglm_example_dir = Path("examples/chatglm") -bCopyModel = True # "False" to remove redundant copy of model from model_cache - - -def convert_ckpt(model_dir: str, output_dir: str, world_size: int): - if os.path.exists(output_dir): - print('Skip ckpt convert - output already exists') - return - - convert_cmd = [ - sys.executable, - str(chatglm_example_dir / "convert_checkpoint.py"), "--dtype=float16", - f"--model_dir={model_dir}", f"--output_dir={output_dir}", - f"--tp_size={world_size}" - ] - run_command(convert_cmd) - - -def build_engine(ckpt_dir: str, - engine_dir: str, - is_ifb: bool = False, - is_chatglm_6b_or_glm_10b: bool = False): - if os.path.exists(engine_dir): - print('Skip engine build - output already exists') - return - - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - "--log_level=error", - "--max_batch_size=8", - "--max_beam_width=2", - "--max_input_len=256", - "--max_seq_len=384", - "--gpt_attention_plugin=float16", - "--gemm_plugin=float16", - ] - if is_ifb: - build_cmd.extend([ - "--remove_input_padding=enable", - "--paged_kv_cache=enable", - "--context_fmha=enable", - "--use_paged_context_fmha=enable", - ]) - else: - build_cmd.extend([ - "--remove_input_padding=disable", - "--paged_kv_cache=disable", - ]) - - if is_chatglm_6b_or_glm_10b: - print("Disable Context FMHA for ChatGLM-6B and GLM-10B") - build_cmd.extend(["--context_fmha=disable"]) - - run_command(build_cmd) - - -def build_engines(model_cache: typing.Optional[str] = None, - world_size: int = 1, - clean: Optional[bool] = False): - - for model_name in [ - "chatglm-6b", "chatglm2-6b", "chatglm3-6b", "glm-10b", "glm-4-9b", - "chatglm3-6b-32k" - ]: - is_chatglm_6b_or_glm_10b = model_name in ["chatglm-6b", "glm-10b"] - if model_cache and (Path(model_cache) / model_name).is_dir(): - model_cache_dir = Path(model_cache) / model_name - if bCopyModel or model_name == "chatglm-6b": - print("Copy model from model_cache") - hf_dir = model_dir / model_name - if platform.system() == "Windows": - wincopy(source=str(model_cache_dir), - dest=model_name, - isdir=True, - cwd=model_dir) - else: - run_command(["rsync", "-rlptD", - str(model_cache_dir), "."], - cwd=model_dir) - else: - print("Use model from model_cache directly except ChatGLM-6B") - hf_dir = Path(model_cache) - - else: - hf_dir = model_dir / model_name - if not hf_dir.is_dir(): - print("Clone model from HF") - run_command( - [ - "git", "clone", - f"https://huggingface.co/THUDM/{model_name}", model_name - ], - cwd=model_dir, - ) - - # Build engines - print(f"Building {model_name}") - ckpt_dir = Path(model_dir) / "c-model" / model_name - if clean: - print('clean up ckpt folder ', ckpt_dir) - if ckpt_dir.is_dir(): - shutil.rmtree(ckpt_dir, ignore_errors=True) - - # Fix HF error for ChatGLM-6B / GLM-4-9B / ChatGLM2-6B / ChatGLM3-6B-32K, hope to remove this in the future - if model_name in [ - "chatglm-6b", "glm-4-9b", "chatglm2-6b", "chatglm3-6b-32k" - ]: - shutil.copy( - chatglm_example_dir / f"{model_name}/tokenization_chatglm.py", - hf_dir, - ) - - convert_ckpt(hf_dir, ckpt_dir, world_size) - - model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.CONTINUOUS) - model_spec_obj.use_gpt_plugin() - engine_dir = Path( - model_dir - ) / "rt_engine" / model_name / model_spec_obj.get_model_path( - ) / "tp1-pp1-cp1-gpu" - if clean: - print('clean up engine folder ', engine_dir) - if engine_dir.is_dir(): - shutil.rmtree(engine_dir, ignore_errors=True) - build_engine(ckpt_dir, engine_dir, False, is_chatglm_6b_or_glm_10b) - - model_spec_obj.use_packed_input() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - engine_dir = Path( - model_dir - ) / "rt_engine" / model_name / model_spec_obj.get_model_path( - ) / "tp1-pp1-cp1-gpu" - if clean: - print('clean up engine folder ', engine_dir) - if engine_dir.is_dir(): - shutil.rmtree(engine_dir, ignore_errors=True) - build_engine(ckpt_dir, engine_dir, True, is_chatglm_6b_or_glm_10b) - - print("Done") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--model_cache", - type=str, - help="Directory where models are stored") - - parser.add_argument('--world_size', - type=int, - default=1, - help='world size, only support tensor parallelism now') - - parser.add_argument('--clean', - action='store_true', - default=False, - help='Clean target folders before building engines') - - build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_eagle_engines.py b/cpp/tests/resources/scripts/build_eagle_engines.py deleted file mode 100755 index 8b10698a603b..000000000000 --- a/cpp/tests/resources/scripts/build_eagle_engines.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import pathlib as _pl -import platform as _pf -import sys as _sys - -from build_engines_utils import run_command, wincopy - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def build_engine(base_model_dir: _pl.Path, eagle_model_dir: _pl.Path, - engine_dir: _pl.Path, build_base_model: bool, *args): - - if build_base_model: - checkpoint_path = "examples/models/core/llama/convert_checkpoint.py" - else: - checkpoint_path = "examples/eagle/convert_checkpoint.py" - - covert_cmd = [_sys.executable, checkpoint_path] + ( - ['--model_dir', str(base_model_dir)] if base_model_dir else []) + [ - '--output_dir', str(engine_dir), '--dtype=float16' - ] + list(args) - - if not build_base_model: - covert_cmd += [ - '--eagle_model_dir', - str(eagle_model_dir), '--num_eagle_layers=4', '--max_draft_len=63' - ] - - run_command(covert_cmd) - - build_args = ["trtllm-build"] + ( - ['--checkpoint_dir', str(engine_dir)] if engine_dir else []) + [ - '--output_dir', - str(engine_dir), - '--gemm_plugin=float16', - '--max_batch_size=8', - '--max_input_len=12', - '--max_seq_len=140', - '--log_level=error', - '--paged_kv_cache=enable', - '--remove_input_padding=enable', - '--use_paged_context_fmha=enable', - ] - - if not build_base_model: - build_args += ['--speculative_decoding_mode=eagle'] - - run_command(build_args) - - -def build_engines(model_cache: str): - resources_dir = _pl.Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - model_name = 'vicuna-7b-eagle' - base_model_name = 'vicuna-7b-v1.3' - eagle_model_name = 'EAGLE-Vicuna-7B-v1.3' - - if model_cache: - print(f"Copy model from {model_cache}") - base_model_cache_dir = _pl.Path(model_cache) / base_model_name - eagle_cache_dir = _pl.Path(model_cache) / eagle_model_name - assert base_model_cache_dir.is_dir(), base_model_cache_dir - assert eagle_cache_dir.is_dir(), eagle_cache_dir - - if _pf.system() == "Windows": - wincopy(source=str(base_model_cache_dir), - dest=base_model_name, - isdir=True, - cwd=models_dir) - wincopy(source=str(eagle_cache_dir), - dest=eagle_model_name, - isdir=True, - cwd=models_dir) - else: - run_command(["rsync", "-rlptD", - str(base_model_cache_dir), "."], - cwd=models_dir) - run_command(["rsync", "-rlptD", - str(eagle_cache_dir), "."], - cwd=models_dir) - - base_model_dir = models_dir / base_model_name - eagle_model_dir = models_dir / eagle_model_name - assert base_model_dir.is_dir() - assert eagle_model_dir.is_dir() - - eagle_engine_dir = models_dir / 'rt_engine' / model_name - base_engine_dir = models_dir / 'rt_engine' / base_model_name - - model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - - base_full_engine_path = base_engine_dir / model_spec_obj.get_model_path( - ) / 'tp1-pp1-cp1-gpu' - print(f"\nBuilding fp16 engine at {str(base_full_engine_path)}") - build_engine(base_model_dir, - eagle_model_dir, - base_full_engine_path, - build_base_model=True) - - model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - model_spec_obj.use_eagle() - eagle_full_engine_path = eagle_engine_dir / model_spec_obj.get_model_path( - ) / 'tp1-pp1-cp1-gpu' - print(f"\nBuilding fp16 engine at {str(eagle_full_engine_path)}") - build_engine(base_model_dir, - eagle_model_dir, - eagle_full_engine_path, - build_base_model=False) - - print("Done.") - - -if __name__ == "__main__": - parser = _arg.ArgumentParser() - parser.add_argument("--model_cache", - type=str, - help="Directory where models are stored") - - build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_enc_dec_engines.py b/cpp/tests/resources/scripts/build_enc_dec_engines.py deleted file mode 100644 index 7079916267d9..000000000000 --- a/cpp/tests/resources/scripts/build_enc_dec_engines.py +++ /dev/null @@ -1,187 +0,0 @@ -import os.path -from argparse import ArgumentParser -from dataclasses import dataclass, fields -from subprocess import run -from sys import stderr, stdout -from typing import List, Literal, Union - -split = os.path.split -join = os.path.join -dirname = os.path.dirname - - -@dataclass -class Arguments: - download: bool = False - dtype: Literal['float16', 'float32', 'bfloat16'] = 'float16' - - hf_repo_name: Literal[ - 'facebook/bart-large-cnn', 't5-small', - 'language_adapter-enc_dec_language_adapter'] = 'facebook/bart-large-cnn' - - model_cache: str = '/llm-models' - - tp: int = 1 - pp: int = 1 - - beams: str = '1' - gpus_per_node: int = 4 - debug: bool = False - - rm_pad: bool = True - gemm: bool = True - - max_new_tokens: int = 64 - - @property - def beams_tuple(self): - return eval(f'tuple([{self.beams}])') - - @property - def max_beam(self): - return max(self.beams_tuple) - - @property - def ckpt(self): - return self.hf_repo_name.split('/')[-1] - - @property - def base_dir(self): - return dirname(dirname(__file__)) - - @property - def data_dir(self): - return join(self.base_dir, 'data/enc_dec') - - @property - def models_dir(self): - return join(self.base_dir, 'models/enc_dec') - - @property - def hf_models_dir(self): - return join(self.model_cache, self.ckpt) - - @property - def trt_models_dir(self): - return join(self.models_dir, 'trt_models', self.ckpt) - - @property - def engines_dir(self): - return join(self.models_dir, 'trt_engines', self.ckpt, - f'{self.tp * self.pp}-gpu', self.dtype) - - @property - def model_type(self): - return self.ckpt.split('-')[0] - - def __post_init__(self): - parser = ArgumentParser() - for k in fields(self): - k = k.name - v = getattr(self, k) - if isinstance(v, bool): - parser.add_argument(f'--{k}', action='store_true') - else: - parser.add_argument(f'--{k}', default=v, type=type(v)) - - args = parser.parse_args() - for k, v in args._get_kwargs(): - setattr(self, k, v) - - -@dataclass -class RunCMDMixin: - args: Arguments - - def command(self) -> Union[str, List[str]]: - raise NotImplementedError - - def run(self): - cmd = self.command() - if cmd: - cmd = ' '.join(cmd) if isinstance(cmd, list) else cmd - print('+ ' + cmd) - run(cmd, shell='bash', stdout=stdout, stderr=stderr, check=True) - - -class DownloadHF(RunCMDMixin): - - def command(self): - args = self.args - return [ - 'git', 'clone', f'https://huggingface.co/{args.hf_repo_name}', - args.hf_models_dir - ] if args.download and args.model_type != 'language_adapter' else '' - - -class Convert(RunCMDMixin): - - def command(self): - args = self.args - return [ - f'python examples/models/core/enc_dec/convert_checkpoint.py', - f'--model_type {args.model_type}', - f'--model_dir {args.hf_models_dir}', - f'--output_dir {args.trt_models_dir}', - f'--tp_size {args.tp} --pp_size {args.pp}' - ] - - -class Build(RunCMDMixin): - - def command(self): - args = self.args - engine_dir = args.engines_dir - weight_dir = args.trt_models_dir - encoder_build = [ - f"trtllm-build --checkpoint_dir {join(weight_dir, 'encoder')}", - f"--output_dir {join(engine_dir, 'encoder')}", - f'--paged_kv_cache disable', - f'--max_beam_width {args.max_beam}', - f'--max_batch_size 8', - f'--max_input_len 512', - f'--gemm_plugin {args.dtype}', - f'--bert_attention_plugin {args.dtype}', - f'--gpt_attention_plugin {args.dtype}', - f'--remove_input_padding enable', - ] - - decoder_build = [ - f"trtllm-build --checkpoint_dir {join(weight_dir, 'decoder')}", - f"--output_dir {join(engine_dir, 'decoder')}", - f'--paged_kv_cache enable', - f'--max_beam_width {args.max_beam}', - f'--max_batch_size 8', - f'--max_seq_len 201', - f'--max_encoder_input_len 512', - f'--gemm_plugin {args.dtype}', - f'--bert_attention_plugin {args.dtype}', - f'--gpt_attention_plugin {args.dtype}', - f'--remove_input_padding enable', - '--max_input_len 1', - ] - - # t5 model with relative attention cannot use context_fmha - encoder_build.append(f'--context_fmha disable') - decoder_build.append(f'--context_fmha disable') - - # language adapter plugin leverages MOE plugin for static expert selection - if args.model_type == 'language_adapter': - encoder_build.append(f'--moe_plugin auto') - decoder_build.append(f'--moe_plugin auto') - else: - encoder_build.append(f'--moe_plugin disable') - decoder_build.append(f'--moe_plugin disable') - - encoder_build = ' '.join(encoder_build) - decoder_build = ' '.join(decoder_build) - ret = ' && '.join((encoder_build, decoder_build)) - return ret - - -if __name__ == "__main__": - # TODO: add support for more models / setup - args = Arguments() - DownloadHF(args).run() - Convert(args).run() - Build(args).run() diff --git a/cpp/tests/resources/scripts/build_engines_utils.py b/cpp/tests/resources/scripts/build_engines_utils.py deleted file mode 100644 index ad8525217e3a..000000000000 --- a/cpp/tests/resources/scripts/build_engines_utils.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging as _log -import os as _os -import pathlib as _pl -import subprocess as _sp -import typing as _tp - - -def run_command(command: _tp.Sequence[str], - *, - cwd=None, - timeout=None, - **kwargs) -> None: - _log.info("Running: cd %s && %s", str(cwd), " ".join(command)) - override_timeout = int(_os.environ.get("CPP_TEST_TIMEOUT_OVERRIDDEN", "-1")) - if override_timeout > 0 and (timeout is None or override_timeout > timeout): - _log.info("Overriding the command timeout: %s (before) and %s (after)", - timeout, override_timeout) - timeout = override_timeout - _sp.check_call(command, cwd=cwd, timeout=timeout, **kwargs) - - -# We can't use run_command() because robocopy (Robust Copy, rsync equivalent on Windows) -# for some reason uses nonzero return codes even on *successful* copies, so we need to check it manually. -# Also, robocopy only accepts dirs, not individual files, so we need a separate command for the -# single-file case. -def wincopy(source: str, dest: str, isdir: bool, cwd=None) -> None: - if not isdir: # Single-file copy - run_command(["cmd", "/c", "copy", - str(_pl.Path(source)), f".\\{dest}"], - cwd=cwd) - else: # Directory sync - copy_cmd = ["robocopy", source, f"./{dest}", "/mir", "/e"] - print(f"Running: cd %s && %s" % - (str(cwd or _pl.Path.cwd()), " ".join(copy_cmd))) - - # Run the command from the specified directory - result = _sp.run(copy_cmd, cwd=cwd) - - # Check for valid exit code - if result.returncode < 8: - print("ROBOCOPY completed successfully.") - else: - print( - "ROBOCOPY failure. Displaying error. See https://ss64.com/nt/robocopy-exit.html for exit code info." - ) - raise _sp.CalledProcessError(returncode=result.returncode, - cmd=copy_cmd, - output=result.stderr) diff --git a/cpp/tests/resources/scripts/build_gpt_engines.py b/cpp/tests/resources/scripts/build_gpt_engines.py deleted file mode 100755 index fa089d773dc0..000000000000 --- a/cpp/tests/resources/scripts/build_gpt_engines.py +++ /dev/null @@ -1,292 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import platform -import shutil -import sys -from pathlib import Path -from typing import Optional - -from build_engines_utils import run_command, wincopy - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec, QuantMethod - - -def convert_ckpt(model_dir: str, - output_dir: str, - *args, - world_size: int = 1, - dtype: str = 'float16'): - convert_cmd = [ - sys.executable, "examples/models/core/gpt/convert_checkpoint.py", - f"--model_dir={model_dir}", f"--output_dir={output_dir}", - f"--dtype={dtype}", f"--tp_size={world_size}" - ] + list(args) - run_command(convert_cmd) - - -def build_engine( - checkpoint_dir: str, - engine_dir: str, - *args, - max_input_len: int = 256, - max_seq_len: int = 384, -): - - build_cmd = [ - "trtllm-build", - '--log_level=error', - f'--checkpoint_dir={checkpoint_dir}', - f'--output_dir={engine_dir}', - '--max_batch_size=64', - f'--max_input_len={max_input_len}', - f'--max_seq_len={max_seq_len}', - '--max_beam_width=2', - '--kv_cache_type=continuous', - ] - legacy_args = [ - "--gpt_attention_plugin=disable", - "--context_fmha=disable", - "--remove_input_padding=disable", - ] - build_cmd = build_cmd + legacy_args + list(args) - run_command(build_cmd) - - -def build_engines(model_cache: Optional[str] = None, - world_size: int = 1, - clean: Optional[bool] = False): - # TODO add support of Pipeline parallelism to GPT - tp_size = world_size - pp_size = 1 - cp_size = 1 - - resources_dir = Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - model_name = 'gpt2' - - # Clone or update the model directory without lfs - hf_dir = models_dir / model_name - if hf_dir.exists(): - assert hf_dir.is_dir() - run_command(["git", "pull"], cwd=hf_dir) - else: - if platform.system() == "Windows": - url_prefix = "" - else: - url_prefix = "file://" - - model_url = url_prefix + str( - Path(model_cache) / - model_name) if model_cache else "https://huggingface.co/gpt2" - run_command([ - "git", "clone", model_url, "--single-branch", "--no-local", - model_name - ], - cwd=hf_dir.parent, - env={ - **os.environ, "GIT_LFS_SKIP_SMUDGE": "1" - }) - - assert hf_dir.is_dir() - - # Download the model file - model_file_name = "pytorch_model.bin" - if model_cache: - if platform.system() == "Windows": - wincopy(source=str( - Path(model_cache) / model_name / model_file_name), - dest=model_file_name, - isdir=False, - cwd=hf_dir) - else: - run_command([ - "rsync", "-rlptD", - str(Path(model_cache) / model_name / model_file_name), "." - ], - cwd=hf_dir) - else: - run_command(["git", "lfs", "pull", "--include", model_file_name], - cwd=hf_dir) - - safetensor_file = hf_dir / "model.safetensors" - has_safetensor = safetensor_file.exists() - if has_safetensor: - safetensor_file.rename(str(safetensor_file) + ".bak") - - assert (hf_dir / model_file_name).is_file() - - ckpt_dir = models_dir / 'c-model' / model_name - engine_dir = models_dir / 'rt_engine' / model_name - - if clean: - target_dir = Path(engine_dir) - print('clean up target folder ', target_dir) - if target_dir.is_dir(): - shutil.rmtree(target_dir, ignore_errors=True) - - tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu" - tp_dir = f"{world_size}-gpu" - - print("\nConverting to fp16") - fp16_ckpt_dir = ckpt_dir / 'fp16' / tp_dir - convert_ckpt(str(hf_dir), - str(fp16_ckpt_dir), - world_size=tp_size, - dtype='float16') - - print("\nBuilding fp16 engines") - - input_file = 'input_tokens.npy' - # this engine can be use for in-flight batching - ifb_base_args = [ - '--gpt_attention_plugin=float16', - '--remove_input_padding=enable', - '--context_fmha=enable', - '--max_num_tokens=10000', - '--use_paged_context_fmha=enable', - ] - - paged_kv_cache_args = ['--kv_cache_type=paged'] - - no_kv_cache_args = ['--kv_cache_type=disabled'] - - def get_ifb_args(kv_cache_type): - if kv_cache_type == _tb.KVCacheType.DISABLED: - return ifb_base_args + no_kv_cache_args - elif kv_cache_type == _tb.KVCacheType.PAGED: - return ifb_base_args + paged_kv_cache_args - else: - assert False, f"Unsupported kv_cache_type: {kv_cache_type}" - - model_spec_obj = ModelSpec(input_file, _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - - model_spec_current = model_spec_obj.__copy__() - - for kv_cache_type in [_tb.KVCacheType.DISABLED, _tb.KVCacheType.PAGED]: - model_spec_current.set_kv_cache_type(kv_cache_type) - build_engine( - str(fp16_ckpt_dir), - str(engine_dir / model_spec_current.get_model_path() / - tp_pp_cp_dir), *get_ifb_args(kv_cache_type)) - - model_spec_current = model_spec_obj.__copy__() - max_draft_tokens = 5 - model_spec_current.use_draft_tokens_external_decoding() - model_spec_current.set_draft_tokens(max_draft_tokens) - - build_engine( - str(fp16_ckpt_dir), - str(engine_dir / model_spec_current.get_model_path() / tp_pp_cp_dir), - f'--max_draft_len={max_draft_tokens}', - '--speculative_decoding_mode=draft_tokens_external', - *get_ifb_args(_tb.KVCacheType.PAGED)) - - model_spec_current = model_spec_obj.__copy__() - model_spec_current.use_multiple_profiles() - - build_engine( - str(fp16_ckpt_dir), - str(engine_dir / model_spec_current.get_model_path() / tp_pp_cp_dir), - '--multiple_profiles=enable', *get_ifb_args(_tb.KVCacheType.PAGED)) - - model_spec_current = model_spec_obj.__copy__() - max_input_len = 128 - model_spec_current.set_max_input_length(max_input_len) - - build_engine(str(fp16_ckpt_dir), - str(engine_dir / model_spec_current.get_model_path() / - tp_pp_cp_dir), - *get_ifb_args(_tb.KVCacheType.PAGED), - max_input_len=max_input_len) - - # We build almost the same engine twice. But this engine has gather_context_logits - # to extract logits from python runtime and uses context FMHA for generation to match draft model executions, - # which uses context FMHA for draft tokens prediction. - # Currently the gather_context_logits is not supported with target model of speculative decoding - model_spec_current = model_spec_obj.__copy__() - model_spec_current.gather_logits() - - build_engine( - str(fp16_ckpt_dir), - str(engine_dir / model_spec_current.get_model_path() / tp_pp_cp_dir), - '--gather_context_logits', *get_ifb_args(_tb.KVCacheType.PAGED)) - - # build engine with lora enabled - model_spec_current = model_spec_obj.__copy__() - model_spec_current.use_lora_plugin() - build_engine( - str(fp16_ckpt_dir), - str(engine_dir / model_spec_current.get_model_path() / tp_pp_cp_dir), - "--lora_target_modules=attn_qkv", '--lora_plugin=float16', - *get_ifb_args(_tb.KVCacheType.PAGED)) - - if model_cache: - llm_datasets_root = Path(model_cache) / "datasets" - calib_dataset = llm_datasets_root / "cimec/lambada/" - else: - calib_dataset = "lambada" - print("\nConverting to fp16 SQ") - fp16_sq_ckpt_dir = ckpt_dir / 'fp16-sq' / tp_dir - convert_ckpt(str(hf_dir), - str(fp16_sq_ckpt_dir), - "--smoothquant=0.5", - f"--calib_dataset={calib_dataset}", - world_size=tp_size, - dtype='float16') - - print("\nBuilding fp16 SQ engines") - model_spec_current = ModelSpec(input_file, _tb.DataType.HALF) - model_spec_current.use_gpt_plugin() - model_spec_current.use_packed_input() - model_spec_current.set_quant_method(QuantMethod.SMOOTH_QUANT) - - for kv_cache_type in [_tb.KVCacheType.DISABLED, _tb.KVCacheType.PAGED]: - model_spec_current.set_kv_cache_type(kv_cache_type) - build_engine( - str(fp16_sq_ckpt_dir), - str(engine_dir / model_spec_current.get_model_path() / - tp_pp_cp_dir), *get_ifb_args(kv_cache_type)) - - if has_safetensor: - Path(str(safetensor_file) + ".bak").rename(safetensor_file) - - print("Done.") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--model_cache", - type=str, - help="Directory where models are stored") - - parser.add_argument('--world_size', - type=int, - default=1, - help='World size, only support tensor parallelism now') - - parser.add_argument('--clean', - action='store_true', - default=False, - help='Clean target folders before building engines') - - build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_gptj_engines.py b/cpp/tests/resources/scripts/build_gptj_engines.py deleted file mode 100755 index bfab97e0ec11..000000000000 --- a/cpp/tests/resources/scripts/build_gptj_engines.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import os as _os -import pathlib as _pl -import platform as _pf -import sys as _sys -import typing as _tp - -from build_engines_utils import run_command, wincopy - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def get_ckpt_without_quatization(model_dir, output_dir): - build_args = [ - _sys.executable, "examples/models/contrib/gpt/convert_checkpoint.py" - ] + [ - '--model_dir={}'.format(model_dir), - '--output_dir={}'.format(output_dir), - ] - run_command(build_args) - - -def get_ckpt_with_modelopt_quant(model_dir, output_dir, model_cache): - build_args = [_sys.executable, "examples/quantization/quantize.py"] + [ - '--model_dir={}'.format(model_dir), - '--output_dir={}'.format(output_dir), '--qformat=fp8', - '--kv_cache_dtype=fp8', - f'--calib_dataset={model_cache}/datasets/cnn_dailymail' - ] - run_command(build_args) - - -def build_engine(checkpoint_dir: _pl.Path, engine_dir: _pl.Path, *args): - build_args = ["trtllm-build"] + ( - ['--checkpoint_dir', str(checkpoint_dir)] if checkpoint_dir else []) + [ - '--output_dir', - str(engine_dir), - '--logits_dtype=float16', - '--gemm_plugin=float16', - '--max_batch_size=32', - '--max_input_len=40', - '--max_seq_len=60', - '--max_beam_width=2', - '--log_level=error', - ] + list(args) - run_command(build_args) - - -def build_engines(model_cache: _tp.Optional[str] = None, only_fp8=False): - resources_dir = _pl.Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - model_name = 'gpt-j-6b' - - # Clone or update the model directory without lfs - hf_dir = models_dir / model_name - if hf_dir.exists(): - assert hf_dir.is_dir() - run_command(["git", "pull"], cwd=hf_dir) - else: - if _pf.system() == "Windows": - url_prefix = "" - else: - url_prefix = "file://" - model_url = url_prefix + str( - _pl.Path(model_cache) / model_name - ) if model_cache else "https://huggingface.co/EleutherAI/gpt-j-6b" - run_command([ - "git", "clone", model_url, "--single-branch", "--no-local", - model_name - ], - cwd=hf_dir.parent, - env={ - **_os.environ, "GIT_LFS_SKIP_SMUDGE": "1" - }) - - assert (hf_dir.is_dir()) - - # Download the model file - model_file_name = "pytorch_model.bin" - if model_cache: - if _pf.system() == "Windows": - wincopy(source=str( - _pl.Path(model_cache) / model_name / model_file_name), - dest=model_file_name, - isdir=False, - cwd=hf_dir) - else: - run_command([ - "rsync", "-rlptD", - str(_pl.Path(model_cache) / model_name / model_file_name), "." - ], - cwd=hf_dir) - else: - run_command(["git", "lfs", "pull", "--include", model_file_name], - cwd=hf_dir) - - assert ((hf_dir / model_file_name).is_file()) - - engine_dir = models_dir / 'rt_engine' / model_name - - # TODO add Tensor and Pipeline parallelism to GPT-J - tp_size = 1 - pp_size = 1 - cp_size = 1 - tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu" - input_file = 'input_tokens.npy' - - if only_fp8: - # with ifb, new plugin - print( - "\nBuilding fp8-plugin engine using gpt_attention_plugin with inflight-batching, packed" - ) - # TODO: use dummy scales atm; to re-enable when data is uploaded to the model cache - # quantized_fp8_model_arg = '--quantized_fp8_model_path=' + \ - # str(_pl.Path(model_cache) / 'fp8-quantized-modelopt' / 'gptj_tp1_rank0.npz') - fp8_ckpt_path = engine_dir / 'fp8' / tp_pp_cp_dir - get_ckpt_with_modelopt_quant(hf_dir, fp8_ckpt_path, model_cache) - model_spec_obj = ModelSpec(input_file, _tb.DataType.FP8) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - build_engine( - fp8_ckpt_path, - engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - '--gpt_attention_plugin=float16', - '--paged_kv_cache=enable', - '--remove_input_padding=enable', - '--use_paged_context_fmha=enable', - ) - else: - fp16_ckpt_path = engine_dir / 'fp16' / tp_pp_cp_dir - get_ckpt_without_quatization(hf_dir, fp16_ckpt_path) - print("\nBuilding fp16-plugin engine") - model_spec_obj = ModelSpec(input_file, _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.CONTINUOUS) - - build_engine( - fp16_ckpt_path, - engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - '--gpt_attention_plugin=float16', '--paged_kv_cache=disable', - '--remove_input_padding=disable', "--context_fmha=disable") - - print("\nBuilding fp16-plugin-packed engine") - model_spec_obj.use_packed_input() - build_engine( - fp16_ckpt_path, - engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - '--gpt_attention_plugin=float16', '--paged_kv_cache=disable', - '--remove_input_padding=enable', "--context_fmha=disable") - - print("\nBuilding fp16-plugin-packed-paged engine") - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - build_engine( - fp16_ckpt_path, - engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - '--gpt_attention_plugin=float16', '--paged_kv_cache=enable', - '--remove_input_padding=enable', "--context_fmha=disable") - print("Done.") - - -if __name__ == "__main__": - parser = _arg.ArgumentParser() - parser.add_argument("--model_cache", - type=str, - help="Directory where models are stored") - parser.add_argument( - "--only_fp8", - action="store_true", - help="Build engines for only FP8 tests. Implemented for H100 runners.") - - build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_llama_engines.py b/cpp/tests/resources/scripts/build_llama_engines.py deleted file mode 100644 index dbac12621c73..000000000000 --- a/cpp/tests/resources/scripts/build_llama_engines.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import pathlib as _pl -import platform as _pf -import sys as _sys -import time - -from build_engines_utils import run_command, wincopy - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def build_engine(weight_dir: _pl.Path, engine_dir: _pl.Path, convert_extra_args, - build_extra_args): - - ckpt_dir = engine_dir / 'ckpt' - - convert_cmd = [ - _sys.executable, "examples/models/core/llama/convert_checkpoint.py" - ] + ([f'--model_dir={weight_dir}'] if weight_dir else []) + [ - f'--output_dir={ckpt_dir}', - '--dtype=float16', - ] + convert_extra_args - - run_command(convert_cmd) - - build_args = [ - 'trtllm-build', - f'--checkpoint_dir={ckpt_dir}', - f'--output_dir={engine_dir}', - '--gpt_attention_plugin=float16', - '--gemm_plugin=float16', - '--max_batch_size=32', - '--max_input_len=40', - '--max_seq_len=60', - '--max_beam_width=2', - '--log_level=error', - '--paged_kv_cache=enable', - '--remove_input_padding=enable', - ] + build_extra_args - - run_command(build_args) - - -def build_engines(model_cache: str, only_multi_gpu: bool): - resources_dir = _pl.Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - model_name = 'Llama-3.2-1B' - - if model_cache: - print("Copy model from model_cache") - model_cache_dir = _pl.Path( - model_cache) / 'llama-3.2-models' / model_name - assert (model_cache_dir.is_dir()), model_cache_dir - - if _pf.system() == "Windows": - wincopy(source=str(model_cache_dir), - dest=model_name, - isdir=True, - cwd=models_dir) - else: - run_command(["rsync", "-rlptD", - str(model_cache_dir), "."], - cwd=models_dir) - - hf_dir = models_dir / model_name - assert hf_dir.is_dir(), f"testing {hf_dir}" - - engine_dir = models_dir / 'rt_engine' / model_name - - model_spec_obj = ModelSpec('input_tokens_llama.npy', _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - - tp_pp_cp_sizes = [(1, 1, 1)] - if only_multi_gpu: - tp_pp_cp_sizes = [(1, 4, 1), (4, 1, 1), (1, 2, 1), (2, 2, 1), (2, 1, 1), - (1, 1, 2), (2, 1, 2)] - for tp_size, pp_size, cp_size in tp_pp_cp_sizes: - print(f"\nBuilding fp16 tp{tp_size} pp{pp_size} cp{cp_size} engine") - start_time = time.time() - - tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu" - model_spec_obj.use_tensor_parallelism(tp_size) - model_spec_obj.use_pipeline_parallelism(pp_size) - model_spec_obj.use_context_parallelism(cp_size) - - build_engine( - hf_dir, engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - [ - f'--tp_size={tp_size}', f'--pp_size={pp_size}', - f'--cp_size={cp_size}' - ], ['--use_paged_context_fmha=disable']) - - duration = time.time() - start_time - print( - f"Building fp16 tp{tp_size} pp{pp_size} cp{cp_size} engine took {duration} seconds" - ) - - if not only_multi_gpu: - print(f"\nBuilding lookahead engine") - start_time = time.time() - - model_spec_obj.use_tensor_parallelism(1) - model_spec_obj.use_pipeline_parallelism(1) - model_spec_obj.use_context_parallelism(1) - model_spec_obj.use_lookahead_decoding() - build_engine( - hf_dir, - engine_dir / model_spec_obj.get_model_path() / 'tp1-pp1-cp1-gpu', - [], [ - '--max_draft_len=39', - '--speculative_decoding_mode=lookahead_decoding' - ]) - - duration = time.time() - start_time - print(f"Building lookahead engine took {duration} seconds") - - print("Done.") - - -if __name__ == "__main__": - parser = _arg.ArgumentParser() - parser.add_argument("--model_cache", - type=str, - help="Directory where models are stored") - parser.add_argument( - "--only_multi_gpu", - action="store_true", - help="Flag to build only for Tensor and Pipeline parallelism") - - build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_mamba_engines.py b/cpp/tests/resources/scripts/build_mamba_engines.py deleted file mode 100644 index 6b10a5b03531..000000000000 --- a/cpp/tests/resources/scripts/build_mamba_engines.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import os as _os -import pathlib as _pl -import platform as _pf -import sys as _sys -import typing as _tp - -from build_engines_utils import run_command, wincopy - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def build_engine(weight_dir: _pl.Path, ckpt_dir: _pl.Path, engine_dir: _pl.Path, - *args): - convert_args = [ - _sys.executable, "examples/models/core/mamba/convert_checkpoint.py" - ] + (['--model_dir', str(weight_dir)] if weight_dir else []) + [ - '--output_dir', - str(ckpt_dir), - '--dtype=float16', - ] - run_command(convert_args) - build_args = ["trtllm-build"] + ['--checkpoint_dir', - str(ckpt_dir)] + [ - '--output_dir', - str(engine_dir), - '--gpt_attention_plugin=disable', - '--paged_kv_cache=disable', - '--gemm_plugin=disable', - '--max_batch_size=8', - '--max_input_len=924', - '--max_seq_len=1024', - '--max_beam_width=1', - ] + list(args) - run_command(build_args) - - -def build_engines(model_cache: _tp.Optional[str] = None): - resources_dir = _pl.Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - model_name = 'mamba-2.8b-hf' - - if model_cache: - print("Copy model from model_cache") - model_cache_dir = _pl.Path(model_cache) / 'mamba' / model_name - if _pf.system() == "Windows": - wincopy(source=str(model_cache_dir), - dest=model_name, - isdir=True, - cwd=models_dir) - else: - run_command(["rsync", "-rlptD", - str(model_cache_dir), "."], - cwd=models_dir) - else: - print("Clone model from HF") - hf_dir = _pl.Path(models_dir) / model_name - run_command( - [ - "git", "clone", - "https://huggingface.co/state-spaces/mamba-2.8b-hf", model_name - ], - cwd=models_dir, - ) - hf_dir = models_dir / model_name - assert (hf_dir.is_dir()) - - # Clone or update the tokenizer directory without lfs - tokenizer_name = 'gpt-neox-20b' - tokenizer_hf_dir = models_dir / tokenizer_name - if tokenizer_hf_dir.exists(): - assert tokenizer_hf_dir.is_dir() - run_command(["git", "pull"], cwd=tokenizer_hf_dir) - else: - if _pf.system() == "Windows": - url_prefix = "" - else: - url_prefix = "file://" - tokenizer_url = url_prefix + str( - _pl.Path(model_cache) / tokenizer_name - ) if model_cache else "https://huggingface.co/EleutherAI/gpt-neox-20b" - run_command([ - "git", "clone", tokenizer_url, "--single-branch", "--no-local", - tokenizer_name - ], - cwd=tokenizer_hf_dir.parent, - env={ - **_os.environ, "GIT_LFS_SKIP_SMUDGE": "1" - }) - - tp_size = 1 - pp_size = 1 - cp_size = 1 - tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu" - - ckpt_dir = models_dir / 'rt_ckpt' / model_name - engine_dir = models_dir / 'rt_engine' / model_name - model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.CONTINUOUS) - model_spec_obj.use_tensor_parallelism(tp_size) - model_spec_obj.use_pipeline_parallelism(pp_size) - model_spec_obj.use_context_parallelism(cp_size) - - print("\nBuilding fp16 engine") - build_engine(hf_dir, - ckpt_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - '--remove_input_padding=disable', '--paged_state=disable', - '--mamba_conv1d_plugin=disable') - print("\nBuilding fp16-plugin engine") - model_spec_obj.use_mamba_plugin() - build_engine(hf_dir, - ckpt_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - '--remove_input_padding=disable', '--paged_state=disable') - print("\nBuilding fp16-plugin-packed engine") - model_spec_obj.use_packed_input() - build_engine(hf_dir, - ckpt_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - '--remove_input_padding=enable', '--paged_state=disable') - print("\nBuilding fp16-plugin-packed-paged engine") - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - build_engine(hf_dir, - ckpt_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - '--remove_input_padding=enable', '--paged_state=enable') - print("Done.") - - -if __name__ == "__main__": - parser = _arg.ArgumentParser() - parser.add_argument("--model_cache", - type=str, - help="Directory where models are stored") - - build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_medusa_engines.py b/cpp/tests/resources/scripts/build_medusa_engines.py deleted file mode 100755 index cf9c74f8779f..000000000000 --- a/cpp/tests/resources/scripts/build_medusa_engines.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import pathlib as _pl -import platform as _pf -import sys as _sys - -from build_engines_utils import run_command, wincopy - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def build_engine(base_model_dir: _pl.Path, medusa_model_dir: _pl.Path, - engine_dir: _pl.Path, *args): - - covert_cmd = [_sys.executable, "examples/medusa/convert_checkpoint.py"] + ( - ['--model_dir', str(base_model_dir)] if base_model_dir else []) + [ - '--medusa_model_dir', str(medusa_model_dir), \ - '--output_dir', str(engine_dir), '--dtype=float16', '--num_medusa_heads=4' - ] + list(args) - - run_command(covert_cmd) - - build_args = ["trtllm-build"] + ( - ['--checkpoint_dir', str(engine_dir)] if engine_dir else []) + [ - '--output_dir', - str(engine_dir), - '--gemm_plugin=float16', - '--max_batch_size=8', - '--max_input_len=12', - '--max_seq_len=140', - '--log_level=error', - '--paged_kv_cache=enable', - '--use_paged_context_fmha=enable', - '--remove_input_padding=enable', - '--speculative_decoding_mode=medusa', - ] - - run_command(build_args) - - -def build_engines(model_cache: str): - resources_dir = _pl.Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - model_name = 'vicuna-7b-medusa' - base_model_name = 'vicuna-7b-v1.3' - medusa_model_name = 'medusa-vicuna-7b-v1.3' - - if model_cache: - print(f"Copy model from {model_cache}") - base_model_cache_dir = _pl.Path(model_cache) / base_model_name - medusa_head_cache_dir = _pl.Path(model_cache) / medusa_model_name - assert base_model_cache_dir.is_dir(), base_model_cache_dir - assert medusa_head_cache_dir.is_dir(), medusa_head_cache_dir - - if _pf.system() == "Windows": - wincopy(source=str(base_model_cache_dir), - dest=base_model_name, - isdir=True, - cwd=models_dir) - wincopy(source=str(medusa_head_cache_dir), - dest=medusa_model_name, - isdir=True, - cwd=models_dir) - else: - run_command(["rsync", "-rlptD", - str(base_model_cache_dir), "."], - cwd=models_dir) - run_command(["rsync", "-rlptD", - str(medusa_head_cache_dir), "."], - cwd=models_dir) - - base_model_dir = models_dir / base_model_name - medusa_model_dir = models_dir / medusa_model_name - assert base_model_dir.is_dir() - assert medusa_model_dir.is_dir() - - engine_dir = models_dir / 'rt_engine' / model_name - - model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - model_spec_obj.use_medusa() - - full_engine_path = engine_dir / model_spec_obj.get_model_path( - ) / 'tp1-pp1-cp1-gpu' - print(f"\nBuilding fp16 engine at {str(full_engine_path)}") - build_engine(base_model_dir, medusa_model_dir, full_engine_path) - - print("Done.") - - -if __name__ == "__main__": - parser = _arg.ArgumentParser() - parser.add_argument("--model_cache", - type=str, - help="Directory where models are stored") - - build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_recurrentgemma_engines.py b/cpp/tests/resources/scripts/build_recurrentgemma_engines.py deleted file mode 100644 index 293aab101d38..000000000000 --- a/cpp/tests/resources/scripts/build_recurrentgemma_engines.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import os as _os -import pathlib as _pl -import platform as _pf -import sys as _sys -import typing as _tp - -from build_engines_utils import run_command, wincopy - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def build_engine(weight_dir: _pl.Path, ckpt_dir: _pl.Path, engine_dir: _pl.Path, - *args): - convert_args = [ - _sys.executable, - "examples/models/core/recurrentgemma/convert_checkpoint.py" - ] + (['--model_dir', str(weight_dir)] if weight_dir else []) + [ - '--output_dir', - str(ckpt_dir), - '--ckpt_type=hf', - '--dtype=float16', - ] - run_command(convert_args) - build_args = ["trtllm-build"] + ['--checkpoint_dir', - str(ckpt_dir)] + [ - '--output_dir', - str(engine_dir), - '--gpt_attention_plugin=float16', - '--paged_kv_cache=enable', - '--gemm_plugin=float16', - '--max_batch_size=8', - '--max_input_len=924', - '--max_seq_len=1024', - '--max_beam_width=1', - ] + list(args) - run_command(build_args) - - -def build_engines(model_cache: _tp.Optional[str] = None): - resources_dir = _pl.Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - model_name = 'recurrentgemma-2b' - hf_dir = models_dir / model_name - - # Clone or update the model directory without lfs - if model_cache: - print("Copy model from model_cache") - model_cache_dir = _pl.Path(model_cache) / 'recurrentgemma' / model_name - print(model_cache_dir) - assert (model_cache_dir.is_dir()) - if _pf.system() == "Windows": - wincopy(source=str(model_cache_dir), - dest=model_name, - isdir=True, - cwd=models_dir) - else: - run_command(["rsync", "-rlptD", - str(model_cache_dir), "."], - cwd=models_dir) - else: - if not hf_dir.is_dir(): - if _pf.system() == "Windows": - url_prefix = "" - else: - url_prefix = "file://" - model_url = "https://huggingface.co/google/recurrentgemma-2b" - run_command([ - "git", "clone", model_url, "--single-branch", "--no-local", - model_name - ], - cwd=models_dir, - env={ - **_os.environ, "GIT_LFS_SKIP_SMUDGE": "1" - }) - - assert (hf_dir.is_dir()) - - # Download the model file - model_file_name = "*" - if not model_cache: - run_command(["git", "lfs", "pull", "--include", model_file_name], - cwd=hf_dir) - - tp_size = 1 - pp_size = 1 - cp_size = 1 - tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu" - - ckpt_dir = models_dir / 'rt_ckpt' / model_name - engine_dir = models_dir / 'rt_engine' / model_name - - python_exe = _sys.executable - run_command([python_exe, "-m", "pip", "install", "transformers>=4.40.0"], - env=_os.environ, - timeout=300) - input_file = 'input_tokens.npy' - model_spec_obj = ModelSpec(input_file, _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.use_packed_input() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - - print("\nBuilding fp16-plugin-packed-paged engine") - build_engine(hf_dir, - ckpt_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, - '--remove_input_padding=enable', '--paged_state=enable') - - print("Done.") - - -if __name__ == "__main__": - parser = _arg.ArgumentParser() - parser.add_argument("--model_cache", - type=str, - help="Directory where models are stored") - - build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_redrafter_engines.py b/cpp/tests/resources/scripts/build_redrafter_engines.py deleted file mode 100755 index cdf3e889ac35..000000000000 --- a/cpp/tests/resources/scripts/build_redrafter_engines.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import pathlib as _pl -import platform as _pf -import sys as _sys - -from build_engines_utils import run_command, wincopy - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def build_engine(base_model_dir: _pl.Path, drafter_model_dir: _pl.Path, - engine_dir: _pl.Path, *args): - - base_ckpt_dir = f'{base_model_dir}-ckpt' - covert_cmd_base = [ - _sys.executable, "examples/models/core/llama/convert_checkpoint.py" - ] + (['--model_dir', str(base_model_dir)] if base_model_dir else []) + [ - '--output_dir', str(base_ckpt_dir), '--dtype=float16' - ] + list(args) - - run_command(covert_cmd_base) - - covert_cmd = [ - _sys.executable, "examples/redrafter/convert_checkpoint.py"] + ( - ['--base_model_checkpoint_dir', str(base_ckpt_dir)] if base_model_dir else []) + [ - '--drafter_model_dir', str(drafter_model_dir), \ - '--output_dir', str(engine_dir), '--dtype=float16', - '--redrafter_num_beams=5', '--redrafter_draft_len_per_beam=5' - ] + list(args) - - run_command(covert_cmd) - - build_args = ["trtllm-build"] + ( - ['--checkpoint_dir', str(engine_dir)] if engine_dir else []) + [ - '--output_dir', - str(engine_dir), - '--gemm_plugin=float16', - '--max_batch_size=8', - '--max_input_len=64', - '--max_seq_len=1024', - '--log_level=error', - '--paged_kv_cache=enable', - '--use_paged_context_fmha=enable', - '--remove_input_padding=enable', - '--speculative_decoding_mode=explicit_draft_tokens', - ] - - run_command(build_args) - - -def build_engines(model_cache: str): - resources_dir = _pl.Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - model_name = 'vicuna-7b-redrafter' - base_model_name = 'vicuna-7b-v1.3' - drafter_model_name = 'redrafter-vicuna-7b-v1.3' - - if model_cache: - print(f"Copy model from {model_cache}") - base_model_cache_dir = _pl.Path(model_cache) / base_model_name - drafter_cache_dir = _pl.Path(model_cache) / drafter_model_name - assert base_model_cache_dir.is_dir(), base_model_cache_dir - assert drafter_cache_dir.is_dir(), drafter_cache_dir - - if _pf.system() == "Windows": - wincopy(source=str(base_model_cache_dir), - dest=base_model_name, - isdir=True, - cwd=models_dir) - wincopy(source=str(drafter_cache_dir), - dest=drafter_model_name, - isdir=True, - cwd=models_dir) - else: - run_command(["rsync", "-rlptD", - str(base_model_cache_dir), "."], - cwd=models_dir) - run_command(["rsync", "-rlptD", - str(drafter_cache_dir), "."], - cwd=models_dir) - - base_model_dir = models_dir / base_model_name - drafter_model_dir = models_dir / drafter_model_name - assert base_model_dir.is_dir() - assert drafter_model_dir.is_dir() - - engine_dir = models_dir / 'rt_engine' / model_name - - model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - model_spec_obj.use_explicit_draft_tokens_decoding() - - full_engine_path = engine_dir / model_spec_obj.get_model_path( - ) / 'tp1-pp1-cp1-gpu' - print(f"\nBuilding fp16 engine at {str(full_engine_path)}") - build_engine(base_model_dir, drafter_model_dir, full_engine_path) - - print("Done.") - - -if __name__ == "__main__": - parser = _arg.ArgumentParser() - parser.add_argument("--model_cache", - type=str, - help="Directory where models are stored") - - build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/generate_expected_chatglm_output.py b/cpp/tests/resources/scripts/generate_expected_chatglm_output.py deleted file mode 100755 index 416f76938700..000000000000 --- a/cpp/tests/resources/scripts/generate_expected_chatglm_output.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from pathlib import Path - -import numpy as np - -# isort: off -import run -# isort: on - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - -resources_dir = Path(__file__).parent.resolve().parent -model_path = resources_dir / "models" - - -def generate_output( - model_name: str = "", - num_beams: int = 1, - max_output_len: int = 8, - output_logits: bool = False, - output_cum_log_probs: bool = False, - output_log_probs: bool = False, -): - hf_path = model_path / model_name - tp_size = 1 - pp_size = 1 - cp_size = 1 - tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu/" - input_file = f"input_tokens_{model_name}.npy" - - data_input_file_name = resources_dir / "data" / input_file - if num_beams == 1: - output_dir = resources_dir / "data" / model_name / "sampling" - else: - output_dir = resources_dir / "data" / model_name / f"beam_search_{num_beams}" - output_dir.mkdir(exist_ok=True, parents=True) - - model_spec_obj_list = [ - ModelSpec(input_file, - _tb.DataType.HALF).use_gpt_plugin().set_kv_cache_type( - _tb.KVCacheType.CONTINUOUS), - ModelSpec(input_file, _tb.DataType.HALF).use_gpt_plugin(). - use_packed_input().set_kv_cache_type(_tb.KVCacheType.PAGED), - ] - - for model_spec_obj in model_spec_obj_list: - engine_dir = model_path / 'rt_engine' / model_name / model_spec_obj.get_model_path( - ) / tp_pp_cp_dir - base_output_name = os.path.splitext( - model_spec_obj.get_results_file())[0] - output_npy_file_name = output_dir / f'{base_output_name}.npy' - output_csv_file_name = output_dir / f'{base_output_name}.csv' - - args_list = [ - '--engine_dir', - str(engine_dir), - '--tokenizer_dir', - str(hf_path), - '--input_file', - str(data_input_file_name), - '--output_npy', - str(output_npy_file_name), - '--output_csv', - str(output_csv_file_name), - '--max_output_len', - str(max_output_len), - '--num_beams', - str(num_beams), - '--use_py_session', - ] - - if output_logits: - file_name = str(output_npy_file_name)[:-4] + "_logits.npy" - args_list.extend(['--output_logits_npy', file_name]) - - if output_cum_log_probs: - file_name = str(output_npy_file_name)[:-4] + "_cum_log_probs.npy" - args_list.extend(['--output_cum_log_probs_npy', file_name]) - - if output_log_probs: - file_name = str(output_npy_file_name)[:-4] + "_log_probs.npy" - args_list.extend(['--output_log_probs_npy', file_name]) - - args = run.parse_arguments(args_list) - run.main(args) - - # Convert pad_id to end_id in .npy out put file - data = np.load(str(output_npy_file_name)) - if model_name == 'chatglm-6b': - data[data == 3] = 130005 - elif model_name == 'chatglm2-6b' or model_name == 'chatglm3-6b': - data[data == 0] = 2 - elif model_name == 'glm-10b': - data[data == 50256] = 50258 - else: - raise NameError('bad model name') - - np.save(str(output_npy_file_name), data) - - -if __name__ == '__main__': - generate_output(model_name='chatglm-6b', num_beams=1) - generate_output(model_name='chatglm-6b', num_beams=2) - generate_output(model_name='chatglm2-6b', num_beams=1) - generate_output(model_name='chatglm2-6b', num_beams=2) - generate_output(model_name='chatglm3-6b', num_beams=1) - generate_output(model_name='chatglm3-6b', num_beams=2) - generate_output(model_name='glm-10b', num_beams=1) - print("Done") diff --git a/cpp/tests/resources/scripts/generate_expected_eagle_output.py b/cpp/tests/resources/scripts/generate_expected_eagle_output.py deleted file mode 100755 index 253a98beaf4e..000000000000 --- a/cpp/tests/resources/scripts/generate_expected_eagle_output.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import os -from pathlib import Path - -# isort: off -import run -# isort: on - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def generate_output(engine: str, - model_spec_obj: ModelSpec, - max_output_len: int = 8): - - model = 'vicuna-7b-v1.3' - model_eagle = 'vicuna-7b-eagle' - hf_model = 'vicuna-7b-v1.3' - resources_dir = Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - hf_dir = models_dir / hf_model - tp_pp_cp_dir = 'tp1-pp1-cp1-gpu/' - engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir - - data_dir = resources_dir / 'data' - input_file = data_dir / 'input_vicuna.npy' - model_data_dir = data_dir / model_eagle - output_dir = model_data_dir / 'sampling' - - base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] - - args = run.parse_arguments([ - '--engine_dir', - str(engine_dir), '--input_file', - str(input_file), '--tokenizer_dir', - str(hf_dir), '--output_npy', - str(output_dir / (base_output_name + '.npy')), '--output_csv', - str(output_dir / (base_output_name + '.csv')), '--max_output_len', - str(max_output_len), '--use_py_session', '--temperature', '1.0' - ]) - run.main(args) - print(f"Output saved at {str(output_dir / base_output_name)}.[npy|csv]") - - -def generate_outputs(): - print(f'Generating outputs for Vicuna 7B v1.3 FP16') - max_output_len = 128 - model_spec_obj = ModelSpec('input_tokens_long.npy', _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_max_output_length(max_output_len) - model_spec_obj.use_packed_input() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - - generate_output(engine=model_spec_obj.get_model_path(), - model_spec_obj=model_spec_obj, - max_output_len=max_output_len) - - -if __name__ == '__main__': - parser = _arg.ArgumentParser() - parser.add_argument( - "--only_multi_gpu", - action="store_true", - help="Generate data with Pipeline and Tensor Parallelism") - - args = parser.parse_args() - - generate_outputs() - print("Done") diff --git a/cpp/tests/resources/scripts/generate_expected_enc_dec_output.py b/cpp/tests/resources/scripts/generate_expected_enc_dec_output.py deleted file mode 100644 index fc3dc615c918..000000000000 --- a/cpp/tests/resources/scripts/generate_expected_enc_dec_output.py +++ /dev/null @@ -1,30 +0,0 @@ -from build_enc_dec_engines import Arguments, RunCMDMixin - - -class Run(RunCMDMixin): - - def command(self): - args = self.args - world_size = args.tp * args.pp - mpi_run = f'mpirun --allow-run-as-root -np {world_size}' if world_size > 1 else '' - ret = [] - for beam in args.beams_tuple: - ret.append(( - mpi_run, - f'python3 examples/models/core/enc_dec/run.py --engine_dir {args.engines_dir}', - f'--engine_name {args.ckpt}', - f'--model_name "{args.hf_models_dir}"', - f'--max_new_tokens={args.max_new_tokens}', - f'--num_beams={beam}', - f'--compare_hf_fp32', - f'--output_npy={args.data_dir}', - "--debug_mode" if args.debug else "", - )) - ret = [' '.join(x) for x in ret] - ret = ' && '.join(ret) - return ret - - -if __name__ == '__main__': - args = Arguments() - Run(args).run() diff --git a/cpp/tests/resources/scripts/generate_expected_gpt_output.py b/cpp/tests/resources/scripts/generate_expected_gpt_output.py deleted file mode 100755 index 16fa5cc8db64..000000000000 --- a/cpp/tests/resources/scripts/generate_expected_gpt_output.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -from pathlib import Path - -# isort: off -import run -# isort: on - -import os -import shutil - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec, QuantMethod - - -def get_model_data_dir(): - resources_dir = Path(__file__).parent.resolve().parent - data_dir = resources_dir / 'data' - return data_dir / 'gpt2' - - -def generate_output(engine: str, - num_beams: int, - input_name: str, - model_spec_obj: ModelSpec, - max_output_len: int = 8, - output_logits: bool = False, - output_cum_log_probs: bool = False, - output_log_probs: bool = False): - tp_size = 1 - pp_size = 1 - cp_size = 1 - model = 'gpt2' - resources_dir = Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - tp_pp_cp_dir = 'tp' + str(tp_size) + '-pp' + str(pp_size) + '-cp' + str( - cp_size) + '-gpu/' - engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir - - data_dir = resources_dir / 'data' - input_file = data_dir / input_name - model_data_dir = get_model_data_dir() - if num_beams <= 1: - output_dir = model_data_dir / 'sampling' - else: - output_dir = model_data_dir / ('beam_search_' + str(num_beams)) - - model_spec_obj.use_tensor_parallelism(tp_size).use_pipeline_parallelism( - pp_size).use_context_parallelism(cp_size) - - base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] - - args_list = [ - f'--engine_dir={engine_dir}', - f'--input_file={input_file}', - f'--tokenizer_dir={models_dir / model}', - f'--output_npy={output_dir / (base_output_name + ".npy")}', - f'--output_csv={output_dir / (base_output_name + ".csv")}', - f'--max_output_len={max_output_len}', - f'--num_beams={num_beams}', - '--use_py_session', - ] - - if output_logits: - args_list.extend([ - f'--output_logits_npy={output_dir / (base_output_name + "_logits.npy")}', - '--output_generation_logits', - ]) - - # Generate context_fmha_fp32_acc enabled results for GptExecutorTest.GenerationLogitsEarlyStop - if model_spec_obj.get_enable_context_fmha_fp32_acc(): - args_list.extend(["--enable_context_fmha_fp32_acc"]) - - if output_cum_log_probs: - args_list.extend([ - f'--output_cum_log_probs_npy={output_dir / model_spec_obj.get_cum_log_probs_file()}' - ]) - - if output_log_probs: - args_list.extend([ - f'--output_log_probs_npy={output_dir / model_spec_obj.get_log_probs_file()}' - ]) - - args = run.parse_arguments(args_list) - run.main(args) - - -def generate_outputs(num_beams): - input_name = 'input_tokens.npy' - input_name_long = 'input_tokens_long.npy' - - print('Generating GPT2 FP16 outputs') - model_spec_obj = ModelSpec(input_name, _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.use_packed_input() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.gather_logits() - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name, - model_spec_obj=model_spec_obj, - output_logits=True, - output_log_probs=True, - output_cum_log_probs=True) - # GptExecutorTest.GenerationLogitsEarlyStop and several tests require to use context_fmha_fp32_acc flag in runtime - model_spec_obj.enable_context_fmha_fp32_acc() - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name, - model_spec_obj=model_spec_obj, - output_logits=True, - output_log_probs=True, - output_cum_log_probs=True) - - model_spec_obj = ModelSpec(input_name, _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name, - model_spec_obj=model_spec_obj, - output_logits=False, - output_log_probs=True, - output_cum_log_probs=True) - model_spec_obj.enable_context_fmha_fp32_acc() - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name, - model_spec_obj=model_spec_obj, - output_logits=False, - output_log_probs=True, - output_cum_log_probs=True) - model_spec_obj.set_max_output_length(128) - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name, - model_spec_obj=model_spec_obj, - output_logits=False, - max_output_len=128) - - model_spec_obj = ModelSpec(input_name_long, _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.use_packed_input() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name_long, - model_spec_obj=model_spec_obj, - output_logits=False) - - model_spec_obj = ModelSpec(input_name, _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.use_packed_input() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.set_quant_method(QuantMethod.SMOOTH_QUANT) - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name, - model_spec_obj=model_spec_obj, - output_logits=False) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--clean', - action='store_true', - default=False, - help='Clean target folders before building engines') - args = parser.parse_args() - if args.clean: - model_data_dir = get_model_data_dir() - print(f'Cleaning target folder {model_data_dir}') - shutil.rmtree(model_data_dir, ignore_errors=True) - generate_outputs(num_beams=1) - generate_outputs(num_beams=2) diff --git a/cpp/tests/resources/scripts/generate_expected_gptj_output.py b/cpp/tests/resources/scripts/generate_expected_gptj_output.py deleted file mode 100755 index 8d650d6bfc31..000000000000 --- a/cpp/tests/resources/scripts/generate_expected_gptj_output.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import os -from pathlib import Path - -# isort: off -import run -# isort: on - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def generate_output(engine: str, - num_beams: int, - model_spec_obj: ModelSpec, - max_output_len: int = 4): - - tp_size = 1 - pp_size = 1 - cp_size = 1 - model = 'gpt-j-6b' - resources_dir = Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - hf_dir = models_dir / model - tp_pp_cp_dir = 'tp' + str(tp_size) + '-pp' + str(pp_size) + '-cp' + str( - cp_size) + '-gpu/' - engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir - - data_dir = resources_dir / 'data' - input_file = data_dir / 'input_tokens.npy' - model_data_dir = data_dir / model - if num_beams <= 1: - output_dir = model_data_dir / 'sampling' - else: - output_dir = model_data_dir / ('beam_search_' + str(num_beams)) - - base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] - - args = run.parse_arguments([ - '--engine_dir', - str(engine_dir), '--input_file', - str(input_file), '--tokenizer_dir', - str(hf_dir), '--output_npy', - str(output_dir / (base_output_name + '.npy')), '--output_csv', - str(output_dir / (base_output_name + '.csv')), '--max_output_len', - str(max_output_len), '--num_beams', - str(num_beams), '--use_py_session' - ]) - run.main(args) - - -def generate_outputs(only_fp8, num_beams): - input_file = 'input_tokens.npy' - if only_fp8 and num_beams == 1: - model_spec_obj = ModelSpec(input_file, _tb.DataType.FP8) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - - print('Generating GPT-J FP8-kv-cache outputs') - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - model_spec_obj=model_spec_obj) - elif not only_fp8: - print('Generating GPT-J FP16 outputs') - model_spec_obj = ModelSpec(input_file, _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.CONTINUOUS) - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - model_spec_obj=model_spec_obj) - - model_spec_obj.use_packed_input() - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - model_spec_obj=model_spec_obj) - - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - model_spec_obj=model_spec_obj) - - -if __name__ == '__main__': - parser = _arg.ArgumentParser() - parser.add_argument( - "--only_fp8", - action="store_true", - help="Generate data for only FP8 tests. Implemented for H100 runners.") - - generate_outputs(**vars(parser.parse_args()), num_beams=1) - generate_outputs(**vars(parser.parse_args()), num_beams=2) diff --git a/cpp/tests/resources/scripts/generate_expected_llama_output.py b/cpp/tests/resources/scripts/generate_expected_llama_output.py deleted file mode 100644 index 74916e77d053..000000000000 --- a/cpp/tests/resources/scripts/generate_expected_llama_output.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import os -import time -from pathlib import Path - -from mpi4py.MPI import COMM_WORLD - -# isort: off -import run -# isort: on - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def generate_output(engine: str, - num_beams: int, - model_spec_obj: ModelSpec, - tp_size: int = 1, - pp_size: int = 1, - cp_size: int = 1, - max_output_len: int = 8, - output_logits: bool = False, - output_cum_log_probs: bool = False, - output_log_probs: bool = False): - - model = 'Llama-3.2-1B' - resources_dir = Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - tp_pp_cp_dir = 'tp' + str(tp_size) + '-pp' + str(pp_size) + '-cp' + str( - cp_size) + '-gpu/' - engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir - - data_dir = resources_dir / 'data' - input_file = data_dir / 'input_tokens_llama.npy' - model_data_dir = data_dir / model - if num_beams <= 1: - output_dir = model_data_dir / 'sampling' - else: - output_dir = model_data_dir / ('beam_search_' + str(num_beams)) - - base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] - - args_list = [ - f'--engine_dir={engine_dir}', - f'--input_file={input_file}', - f'--tokenizer_dir={models_dir / model}', - f'--output_npy={output_dir / (base_output_name + ".npy")}', - f'--output_csv={output_dir / (base_output_name + ".csv")}', - f'--max_output_len={max_output_len}', - f'--num_beams={num_beams}', - '--use_py_session', - ] - - if output_logits: - args_list.extend([ - f'--output_logits_npy={output_dir / (base_output_name + "_logits.npy")}', - '--output_generation_logits', - ]) - - if output_cum_log_probs: - args_list.extend([ - f'--output_cum_log_probs_npy={output_dir / model_spec_obj.get_cum_log_probs_file()}' - ]) - - if output_log_probs: - args_list.extend([ - f'--output_log_probs_npy={output_dir / model_spec_obj.get_log_probs_file()}' - ]) - - args = run.parse_arguments(args_list) - run.main(args) - - -def generate_outputs(num_beams, only_multi_gpu=False): - if not only_multi_gpu: - tp_pp_cp_sizes = [(1, 1, 1)] - elif COMM_WORLD.size == 4: - tp_pp_cp_sizes = [(4, 1, 1), (2, 2, 1), (1, 4, 1)] - elif COMM_WORLD.size == 2: - tp_pp_cp_sizes = [(1, 2, 1), (2, 1, 1)] - else: - raise RuntimeError( - f"The world size of MPI {COMM_WORLD.size} is not equal to 1, 2, or 4." - ) - model_spec_obj = ModelSpec('input_tokens_llama.npy', _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - - for tp_size, pp_size, cp_size in tp_pp_cp_sizes: - print( - f'Generating outputs for Llama FP16 with TP={tp_size}, PP={pp_size}, CP={cp_size}, BW={num_beams}' - ) - start_time = time.time() - - output_logits = False - output_log_probs = False - output_cum_log_probs = False - if tp_size == 4 and pp_size == 1: - output_logits = True - output_log_probs = True - output_cum_log_probs = True - - model_spec_obj.use_tensor_parallelism(tp_size) - model_spec_obj.use_pipeline_parallelism(pp_size) - model_spec_obj.use_context_parallelism(cp_size) - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - tp_size=tp_size, - pp_size=pp_size, - cp_size=cp_size, - model_spec_obj=model_spec_obj, - output_logits=output_logits, - output_log_probs=output_log_probs, - output_cum_log_probs=output_cum_log_probs) - - duration = time.time() - start_time - print( - f"Generating outputs for Llama FP16 with TP={tp_size}, PP={pp_size}, CP={cp_size}, BW={num_beams} took {duration} seconds" - ) - - -if __name__ == '__main__': - parser = _arg.ArgumentParser() - parser.add_argument( - "--only_multi_gpu", - action="store_true", - help="Generate data with Pipeline and Tensor Parallelism") - - args = parser.parse_args() - - generate_outputs(num_beams=1, only_multi_gpu=args.only_multi_gpu) - generate_outputs(num_beams=2, only_multi_gpu=args.only_multi_gpu) - print("Done") diff --git a/cpp/tests/resources/scripts/generate_expected_mamba_output.py b/cpp/tests/resources/scripts/generate_expected_mamba_output.py deleted file mode 100644 index 16779c434775..000000000000 --- a/cpp/tests/resources/scripts/generate_expected_mamba_output.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from pathlib import Path - -# isort: off -import run -# isort: on - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def generate_output(engine: str, - num_beams: int, - input_name: str, - model_spec_obj: ModelSpec, - max_output_len: int = 8, - output_logits: bool = False): - tp_size = 1 - pp_size = 1 - cp_size = 1 - model = 'mamba-2.8b-hf' - resources_dir = Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - tp_pp_cp_dir = 'tp' + str(tp_size) + '-pp' + str(pp_size) + '-cp' + str( - cp_size) + '-gpu/' - engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir - - data_dir = resources_dir / 'data' - input_file = data_dir / input_name - model_data_dir = data_dir / model - if num_beams <= 1: - output_dir = model_data_dir / 'sampling' - else: - output_dir = model_data_dir / ('beam_search_' + str(num_beams)) - - base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] - - output_logits_npy = None - if output_logits: - output_logits_npy = str(output_dir / - (base_output_name + '_logits' + '.npy')) - - args = run.parse_arguments([ - '--engine_dir', - str(engine_dir), '--input_file', - str(input_file), '--tokenizer_dir', - str(models_dir / 'gpt-neox-20b'), '--output_npy', - str(output_dir / (base_output_name + '.npy')), '--output_csv', - str(output_dir / (base_output_name + '.csv')), '--max_output_len', - str(max_output_len), '--num_beams', - str(num_beams), '--output_logits_npy', - str(output_logits_npy), '--use_py_session' - ]) - run.main(args) - - -def generate_outputs(num_beams): - print('Generating Mamba FP16 outputs') - input_name = 'input_tokens.npy' - model_spec_obj = ModelSpec(input_name, _tb.DataType.HALF) - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.CONTINUOUS) - - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name, - model_spec_obj=model_spec_obj) - - print('Generating Mamba FP16-plugin outputs') - model_spec_obj.use_gpt_plugin() - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name, - model_spec_obj=model_spec_obj) - - print('Generating Mamba FP16-plugin-packed outputs') - model_spec_obj.use_packed_input() - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name, - model_spec_obj=model_spec_obj) - - print('Generating Mamba FP16-plugin-packed-paged outputs') - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_name, - model_spec_obj=model_spec_obj) - - -if __name__ == '__main__': - generate_outputs(num_beams=1) diff --git a/cpp/tests/resources/scripts/generate_expected_medusa_output.py b/cpp/tests/resources/scripts/generate_expected_medusa_output.py deleted file mode 100755 index e1cbc20c051b..000000000000 --- a/cpp/tests/resources/scripts/generate_expected_medusa_output.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import os -from pathlib import Path - -# isort: off -import run -# isort: on - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def generate_output(engine: str, - model_spec_obj: ModelSpec, - max_output_len: int = 8): - - model = 'vicuna-7b-medusa' - hf_model = 'vicuna-7b-v1.3' - resources_dir = Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - hf_dir = models_dir / hf_model - tp_pp_dir = 'tp1-pp1-cp1-gpu/' - engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_dir - - data_dir = resources_dir / 'data' - input_file = data_dir / 'input_vicuna.npy' - model_data_dir = data_dir / model - output_dir = model_data_dir / 'sampling' - - base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] - - args = run.parse_arguments([ - '--engine_dir', - str(engine_dir), '--input_file', - str(input_file), '--tokenizer_dir', - str(hf_dir), '--output_npy', - str(output_dir / (base_output_name + '.npy')), '--output_csv', - str(output_dir / (base_output_name + '.csv')), '--max_output_len', - str(max_output_len), '--use_py_session', - '--medusa_choices=[[0], [0, 0], [1], [0, 1], [2], [0, 0, 0], [1, 0], [0, 2], [3], [0, 3], [4], [0, 4], [2, 0], [0, 5], [0, 0, 1], [5], [0, 6], [6], [0, 7], [0, 1, 0], [1, 1], [7], [0, 8], [0, 0, 2], [3, 0], [0, 9], [8], [9], [1, 0, 0], [0, 2, 0], [1, 2], [0, 0, 3], [4, 0], [2, 1], [0, 0, 4], [0, 0, 5], [0, 0, 0, 0], [0, 1, 1], [0, 0, 6], [0, 3, 0], [5, 0], [1, 3], [0, 0, 7], [0, 0, 8], [0, 0, 9], [6, 0], [0, 4, 0], [1, 4], [7, 0], [0, 1, 2], [2, 0, 0], [3, 1], [2, 2], [8, 0], [0, 5, 0], [1, 5], [1, 0, 1], [0, 2, 1], [9, 0], [0, 6, 0], [0, 0, 0, 1], [1, 6], [0, 7, 0]]', - '--temperature', '1.0' - ]) - run.main(args) - print(f"Output saved at {str(output_dir / base_output_name)}.[npy|csv]") - - -def generate_outputs(): - print(f'Generating outputs for Medusa FP16') - max_output_len = 128 - model_spec_obj = ModelSpec('input_tokens_long.npy', _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_max_output_length(max_output_len) - model_spec_obj.use_packed_input() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_medusa() - - generate_output(engine=model_spec_obj.get_model_path(), - model_spec_obj=model_spec_obj, - max_output_len=max_output_len) - - -if __name__ == '__main__': - parser = _arg.ArgumentParser() - parser.add_argument( - "--only_multi_gpu", - action="store_true", - help="Generate data with Pipeline and Tensor Parallelism") - - args = parser.parse_args() - - generate_outputs() - print("Done") diff --git a/cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py b/cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py deleted file mode 100644 index 0ef4cc4509fd..000000000000 --- a/cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from pathlib import Path - -# isort: off -import run -# isort: on - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def generate_output(engine: str, - num_beams: int, - input_name: str, - model_spec_obj: ModelSpec, - max_output_len: int = 8, - output_logits: bool = False): - tp_size = 1 - pp_size = 1 - cp_size = 1 - model = 'recurrentgemma-2b' - resources_dir = Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - tp_pp_cp_dir = 'tp' + str(tp_size) + '-pp' + str(pp_size) + '-cp' + str( - cp_size) + '-gpu/' - engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir - - data_dir = resources_dir / 'data' - input_file = data_dir / input_name - model_data_dir = data_dir / model - if num_beams <= 1: - output_dir = model_data_dir / 'sampling' - else: - output_dir = model_data_dir / ('beam_search_' + str(num_beams)) - - base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] - - output_logits_npy = None - if output_logits: - output_logits_npy = str(output_dir / - (base_output_name + '_logits' + '.npy')) - - args = run.parse_arguments([ - '--engine_dir', - str(engine_dir), '--input_file', - str(input_file), '--tokenizer_dir', - str(models_dir / model), '--output_npy', - str(output_dir / (base_output_name + '.npy')), '--output_csv', - str(output_dir / (base_output_name + '.csv')), '--max_output_len', - str(max_output_len), '--num_beams', - str(num_beams), '--output_logits_npy', - str(output_logits_npy), '--use_py_session' - ]) - run.main(args) - - -def generate_outputs(num_beams): - input_file = 'input_tokens.npy' - model_spec_obj = ModelSpec(input_file, _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_packed_input() - - print('Generating RecurrentGemma FP16-plugin-packed-paged outputs') - generate_output(engine=model_spec_obj.get_model_path(), - num_beams=num_beams, - input_name=input_file, - model_spec_obj=model_spec_obj) - - -if __name__ == '__main__': - generate_outputs(num_beams=1) diff --git a/cpp/tests/resources/scripts/generate_expected_redrafter_output.py b/cpp/tests/resources/scripts/generate_expected_redrafter_output.py deleted file mode 100644 index 989e029a5ab1..000000000000 --- a/cpp/tests/resources/scripts/generate_expected_redrafter_output.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse as _arg -import os -from pathlib import Path - -# isort: off -import run -# isort: on - -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - - -def generate_output(engine: str, - model_spec_obj: ModelSpec, - max_output_len: int = 8): - - model = 'vicuna-7b-redrafter' - hf_model = 'vicuna-7b-v1.3' - resources_dir = Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - hf_dir = models_dir / hf_model - tp_pp_dir = 'tp1-pp1-cp1-gpu/' - engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_dir - - data_dir = resources_dir / 'data' - input_filename = model_spec_obj.get_input_file() - input_file = data_dir / input_filename - model_data_dir = data_dir / model - output_dir = model_data_dir / 'sampling' - - base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] - - args = run.parse_arguments([ - '--engine_dir', - str(engine_dir), - '--input_file', - str(input_file), - '--tokenizer_dir', - str(hf_dir), - '--output_npy', - str(output_dir / (base_output_name + '.npy')), - '--output_csv', - str(output_dir / (base_output_name + '.csv')), - '--max_output_len', - str(max_output_len), - '--use_py_session', - ]) - run.main(args) - print(f"Output saved at {str(output_dir / base_output_name)}.[npy|csv]") - - -def generate_outputs(): - print(f'Generating outputs for ReDrafter FP16') - max_output_len = 128 - model_spec_obj = ModelSpec('input_vicuna.npy', _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin() - model_spec_obj.set_max_output_length(max_output_len) - model_spec_obj.use_packed_input() - model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - model_spec_obj.use_explicit_draft_tokens_decoding() - - generate_output(engine=model_spec_obj.get_model_path(), - model_spec_obj=model_spec_obj, - max_output_len=max_output_len) - - -if __name__ == '__main__': - parser = _arg.ArgumentParser() - args = parser.parse_args() - - generate_outputs() - print("Done") diff --git a/cpp/tests/resources/scripts/generate_hf_gpt_output.py b/cpp/tests/resources/scripts/generate_hf_gpt_output.py deleted file mode 100755 index a40ada8cb455..000000000000 --- a/cpp/tests/resources/scripts/generate_hf_gpt_output.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path - -import run_hf - - -def generate_hf_output(data_type: str, - output_name: str, - max_output_len: int = 8): - - model = 'gpt2' - resources_dir = Path(__file__).parent.resolve().parent - models_dir = resources_dir / 'models' - model_dir = models_dir / model - - data_dir = resources_dir / 'data' - input_file = data_dir / 'input_tokens.npy' - output_dir = data_dir / model / 'huggingface' - - run_hf.generate(model_dir=str(model_dir), - data_type=data_type, - input_file=str(input_file), - output_npy=str(output_dir / (output_name + '.npy')), - output_csv=str(output_dir / (output_name + '.csv')), - max_output_len=max_output_len) - - -def generate_hf_outputs(): - generate_hf_output(data_type='fp32', - output_name='output_tokens_fp32_huggingface') - generate_hf_output(data_type='fp16', - output_name='output_tokens_fp16_huggingface') - - -if __name__ == '__main__': - generate_hf_outputs() diff --git a/cpp/tests/resources/scripts/io_converter.py b/cpp/tests/resources/scripts/io_converter.py deleted file mode 100755 index 0ed6413c5ac4..000000000000 --- a/cpp/tests/resources/scripts/io_converter.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import csv -import os - -import numpy as np - - -def csv_to_npy(input_file, output_file, pad_id, verbose): - data = [] - with open(input_file, newline='') as csvfile: - csv_reader = csv.reader(csvfile, delimiter=',') - for line in csv_reader: - data.append([int(e) for e in line]) - max_input_length = max([len(x) for x in data]) - data = [row + [pad_id] * (max_input_length - len(row)) for row in data] - data = np.array(data, dtype='int32') - if (verbose): - print(data, data.dtype) - np.save(output_file, data) - - -def npy_to_csv(input_file, output_file, verbose): - data = np.load(input_file) - if (verbose): - print(data, data.dtype) - np.savetxt(output_file, data, delimiter=",", fmt='%i') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - 'input_file', - type=str, - help='Read token ids from this file. Must be csv or npy.') - parser.add_argument('output_file', - type=str, - help='Write token ids this file. Must be csv or npy.') - parser.add_argument( - '-p', - '--pad_id', - type=int, - help= - 'Token id used for padding csv input with different sequence lengths.', - default=-1) - parser.add_argument('-v', '--verbose', action="store_true") - args = parser.parse_args() - - _, input_ext = os.path.splitext(args.input_file) - _, output_ext = os.path.splitext(args.output_file) - - if (input_ext == '.csv' and output_ext == '.npy'): - print('Converting csv to npy') - csv_to_npy(args.input_file, args.output_file, args.pad_id, args.verbose) - elif (input_ext == '.npy' and output_ext == '.csv'): - print('Converting npy to csv') - npy_to_csv(args.input_file, args.output_file, args.verbose) - else: - print('unknown file extensions') diff --git a/cpp/tests/unit_tests/CMakeLists.txt b/cpp/tests/unit_tests/CMakeLists.txt index 034de457bb78..9d22bd03b52a 100644 --- a/cpp/tests/unit_tests/CMakeLists.txt +++ b/cpp/tests/unit_tests/CMakeLists.txt @@ -27,4 +27,3 @@ add_subdirectory(multi_gpu) add_subdirectory(layers) add_subdirectory(runtime) add_subdirectory(thop) -add_subdirectory(utils) diff --git a/cpp/tests/unit_tests/batch_manager/CMakeLists.txt b/cpp/tests/unit_tests/batch_manager/CMakeLists.txt index f4e5e9fb2be0..02f214986712 100644 --- a/cpp/tests/unit_tests/batch_manager/CMakeLists.txt +++ b/cpp/tests/unit_tests/batch_manager/CMakeLists.txt @@ -20,16 +20,28 @@ add_gtest(cacheTransBufferTest cacheTransBufferTest.cpp) add_gtest(bufferIndexHolderTest bufferIndexHolderTest.cpp) add_gtest(capacitySchedulerTest capacitySchedulerTest.cpp) add_gtest(contextProgressTest contextProgressTest.cu) +add_gtest(contextTransferCoordinatorTest contextTransferCoordinatorTest.cpp) add_gtest(evictionPolicyTest evictionPolicyTest.cpp) add_gtest(kvCacheManagerTest kvCacheManagerTest.cpp) add_gtest(kvCacheManagerFabricMemoryTest kvCacheManagerFabricMemoryTest.cpp) +add_gtest(kvCacheManagerV2TypedIndexTest kvCacheManagerV2TypedIndexTest.cpp) +target_include_directories( + kvCacheManagerV2TypedIndexTest + PRIVATE ${PROJECT_SOURCE_DIR}/tensorrt_llm/batch_manager) +add_gtest(kvCacheManagerV2HostMemTest kvCacheManagerV2HostMemTest.cpp) +target_include_directories( + kvCacheManagerV2HostMemTest + PRIVATE ${PROJECT_SOURCE_DIR}/tensorrt_llm/batch_manager) +add_gtest(kvCacheManagerV2StatsTest kvCacheManagerV2StatsTest.cpp) +target_include_directories( + kvCacheManagerV2StatsTest + PRIVATE ${PROJECT_SOURCE_DIR}/tensorrt_llm/batch_manager + ${PROJECT_SOURCE_DIR}/tensorrt_llm/common/sha256) add_gtest(kvCacheUtilsTest kvCacheUtilsTest.cpp) add_gtest(llmRequestTest llmRequestTest.cpp) add_gtest(microBatchSchedulerTest microBatchSchedulerTest.cpp) add_gtest(peftCacheManagerTest peftCacheManagerTest.cpp) add_gtest(staticThreadPoolTest staticThreadPoolTest.cpp) add_gtest(rnnCacheFormatterTest rnnCacheFormatterTest.cpp) -add_gtest(cudaGraphExecutorCacheTest cudaGraphExecutorCacheTest.cpp) add_gtest(agentTreeTest agentTreeTest.cpp) add_gtest(truncateBlocksTest truncateBlocksTest.cpp) -add_gtest(encDecBeamSearchTest encDecBeamSearchTest.cpp) diff --git a/cpp/tests/unit_tests/batch_manager/bufferIndexHolderTest.cpp b/cpp/tests/unit_tests/batch_manager/bufferIndexHolderTest.cpp index 00664f4aeb4e..31400831a5bb 100644 --- a/cpp/tests/unit_tests/batch_manager/bufferIndexHolderTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/bufferIndexHolderTest.cpp @@ -18,7 +18,7 @@ #include "tensorrt_llm/batch_manager/baseTransBuffer.h" #include "tensorrt_llm/batch_manager/cacheTransBuffer.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <gtest/gtest.h> #include <memory> #include <optional> @@ -95,7 +95,7 @@ class BufferIndexHolderLifecycleTest : public ::testing::TestWithParam<HolderCas mKv = std::make_unique<KVCacheManager>(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, std::vector<BlockManager::SizeType32>{kvMaxNumTokens}, - nvinfer1::DataType::kFLOAT, sinkTokenLength, stream, kvMaxNumTokens, kvMaxNumTokens, + tensorrt_llm::DataType::kFLOAT, sinkTokenLength, stream, kvMaxNumTokens, kvMaxNumTokens, /*enableBlockReuse=*/true, CacheType::kSELF, std::nullopt, nullptr, true); mKv->allocatePools(false); mTrans = std::make_unique<ObservableTransBufferManager>(mKv.get(), std::optional<size_t>{kvMaxNumTokens}); diff --git a/cpp/tests/unit_tests/batch_manager/cacheTransBufferTest.cpp b/cpp/tests/unit_tests/batch_manager/cacheTransBufferTest.cpp index 8150c6fa5406..2fa0477d2352 100644 --- a/cpp/tests/unit_tests/batch_manager/cacheTransBufferTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/cacheTransBufferTest.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/batch_manager/cacheTransBuffer.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -51,7 +52,7 @@ class CacheTransBufferTest : public ::testing::Test auto constexpr blocksInSecondaryPool = 0; auto constexpr enableBlockReuse = true; - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {totalNumBlocks, blocksInSecondaryPool}}}; diff --git a/cpp/tests/unit_tests/batch_manager/capacitySchedulerTest.cpp b/cpp/tests/unit_tests/batch_manager/capacitySchedulerTest.cpp index 7bb87c91e361..2c9757b847b2 100644 --- a/cpp/tests/unit_tests/batch_manager/capacitySchedulerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/capacitySchedulerTest.cpp @@ -31,7 +31,7 @@ #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/testing/kvCacheManagerTestUtil.h" -#include <NvInferPlugin.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cstdlib> #include <functional> @@ -132,7 +132,7 @@ class CapacitySchedulerTest : public ::testing::Test // NOLINT(cppcoreguidelines auto const nbKvHeads = 10; auto constexpr sizePerHead = 1; auto const maxNumBlocks = tc::divUp(maxNumTokens, tokensPerBlock); - auto const kvDtype = nvinfer1::DataType::kHALF; + auto const kvDtype = tensorrt_llm::DataType::kHALF; CudaStreamPtr streamPtr = std::make_shared<tensorrt_llm::runtime::CudaStream>(); using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; diff --git a/cpp/tests/unit_tests/batch_manager/contextTransferCoordinatorTest.cpp b/cpp/tests/unit_tests/batch_manager/contextTransferCoordinatorTest.cpp new file mode 100644 index 000000000000..7b0f7aba3fc4 --- /dev/null +++ b/cpp/tests/unit_tests/batch_manager/contextTransferCoordinatorTest.cpp @@ -0,0 +1,143 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/contextTransferCoordinator.h" + +#include <gtest/gtest.h> + +namespace tensorrt_llm::batch_manager +{ +namespace +{ + +TEST(ContextTransferVoteReducerTest, WaitsForEveryParticipant) +{ + ContextTransferVoteReducer reducer(4); + reducer.recordVote(0, 17, ContextTransferVote::kCompleted); + reducer.recordVote(1, 17, ContextTransferVote::kCompleted); + reducer.recordVote(2, 17, ContextTransferVote::kCompleted); + EXPECT_TRUE(reducer.takeReady().completedRequestIds.empty()); + + reducer.recordVote(3, 17, ContextTransferVote::kCompleted); + auto const result = reducer.takeReady(); + EXPECT_EQ(result.completedRequestIds, std::unordered_set<std::uint64_t>{17}); + EXPECT_TRUE(result.failedRequestIds.empty()); + EXPECT_TRUE(result.timedOutRequestIds.empty()); +} + +TEST(ContextTransferVoteReducerTest, FailureWinsAfterEveryParticipantVotes) +{ + ContextTransferVoteReducer reducer(4); + reducer.recordVote(0, 23, ContextTransferVote::kCompleted); + reducer.recordVote(1, 23, ContextTransferVote::kFailed); + reducer.recordVote(2, 23, ContextTransferVote::kCompleted); + reducer.recordVote(3, 23, ContextTransferVote::kCompleted); + + auto const result = reducer.takeReady(); + EXPECT_TRUE(result.completedRequestIds.empty()); + EXPECT_EQ(result.failedRequestIds, std::unordered_set<std::uint64_t>{23}); +} + +TEST(ContextTransferVoteReducerTest, RejectsChangedTerminalVote) +{ + ContextTransferVoteReducer reducer(2); + reducer.recordVote(0, 29, ContextTransferVote::kCompleted); + reducer.recordVote(0, 29, ContextTransferVote::kCompleted); + EXPECT_ANY_THROW(reducer.recordVote(0, 29, ContextTransferVote::kFailed)); +} + +TEST(ContextTransferVoteReducerTest, AccumulatesInterleavedRequestsIndependently) +{ + ContextTransferVoteReducer reducer(2); + reducer.recordVote(0, 31, ContextTransferVote::kCompleted); + reducer.recordVote(1, 37, ContextTransferVote::kFailed); + reducer.recordVote(1, 31, ContextTransferVote::kCompleted); + + auto result = reducer.takeReady(); + EXPECT_EQ(result.completedRequestIds, std::unordered_set<std::uint64_t>{31}); + EXPECT_TRUE(result.failedRequestIds.empty()); + + reducer.recordVote(0, 37, ContextTransferVote::kCompleted); + result = reducer.takeReady(); + EXPECT_TRUE(result.completedRequestIds.empty()); + EXPECT_EQ(result.failedRequestIds, std::unordered_set<std::uint64_t>{37}); +} + +TEST(ContextTransferVoteReducerTest, RejectsInvalidInput) +{ + EXPECT_ANY_THROW(ContextTransferVoteReducer(0)); + ContextTransferVoteReducer reducer(2); + EXPECT_ANY_THROW(reducer.recordVote(-1, 41, ContextTransferVote::kCompleted)); + EXPECT_ANY_THROW(reducer.recordVote(2, 41, ContextTransferVote::kCompleted)); + EXPECT_ANY_THROW(reducer.recordVote(0, 41, static_cast<ContextTransferVote>(99))); +} + +TEST(ContextTransferVoteReducerTest, TimeoutIsVisibleBeforeTerminalFailure) +{ + ContextTransferVoteReducer reducer(2); + reducer.recordTimeout(43); + + auto result = reducer.takeReady(); + EXPECT_EQ(result.timedOutRequestIds, std::unordered_set<std::uint64_t>{43}); + EXPECT_TRUE(result.completedRequestIds.empty()); + EXPECT_TRUE(result.failedRequestIds.empty()); + + result = reducer.takeReady(); + EXPECT_TRUE(result.timedOutRequestIds.empty()); + EXPECT_TRUE(result.failedRequestIds.empty()); + + reducer.recordVote(0, 43, ContextTransferVote::kCompleted); + reducer.recordVote(1, 43, ContextTransferVote::kCompleted); + result = reducer.takeReady(); + EXPECT_EQ(result.failedRequestIds, std::unordered_set<std::uint64_t>{43}); + EXPECT_TRUE(result.completedRequestIds.empty()); + EXPECT_TRUE(result.timedOutRequestIds.empty()); +} + +TEST(ContextTransferVoteReducerTest, DuplicateTimeoutIsIdempotent) +{ + ContextTransferVoteReducer reducer(2); + reducer.recordTimeout(47); + reducer.recordTimeout(47); + + auto result = reducer.takeReady(); + EXPECT_EQ(result.timedOutRequestIds, std::unordered_set<std::uint64_t>{47}); + + reducer.recordTimeout(47); + result = reducer.takeReady(); + EXPECT_TRUE(result.timedOutRequestIds.empty()); +} + +TEST(ContextTransferVoteReducerTest, TimeoutAfterPartialTerminalVotesStillWins) +{ + ContextTransferVoteReducer reducer(3); + reducer.recordVote(2, 53, ContextTransferVote::kCompleted); + reducer.recordTimeout(53); + reducer.recordVote(0, 53, ContextTransferVote::kCompleted); + + auto result = reducer.takeReady(); + EXPECT_EQ(result.timedOutRequestIds, std::unordered_set<std::uint64_t>{53}); + EXPECT_TRUE(result.failedRequestIds.empty()); + + reducer.recordVote(1, 53, ContextTransferVote::kCompleted); + result = reducer.takeReady(); + EXPECT_EQ(result.failedRequestIds, std::unordered_set<std::uint64_t>{53}); + EXPECT_TRUE(result.completedRequestIds.empty()); +} + +} // namespace +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tests/unit_tests/batch_manager/cudaGraphExecutorCacheTest.cpp b/cpp/tests/unit_tests/batch_manager/cudaGraphExecutorCacheTest.cpp deleted file mode 100644 index f8ae0a9db64f..000000000000 --- a/cpp/tests/unit_tests/batch_manager/cudaGraphExecutorCacheTest.cpp +++ /dev/null @@ -1,161 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h" - -#include <gtest/gtest.h> - -#include <memory> - -namespace tb = tensorrt_llm::batch_manager; -namespace tbu = tensorrt_llm::batch_manager::utils; -using SizeType32 = tensorrt_llm::runtime::SizeType32; - -namespace -{ -// A default-constructed CudaGraphExecutor holds mInstance == nullptr, so its destructor -// is a no-op and these tests do not require an active CUDA context. -std::shared_ptr<tbu::CudaGraphExecutor> makeDummyExecutor() -{ - return std::make_shared<tbu::CudaGraphExecutor>(); -} - -tb::BatchState makeBatchState(SizeType32 numTokens) -{ - return tb::BatchState{/*numCtxRequests=*/0, /*numGenRequests=*/1, numTokens, /*maxKvCacheLength=*/256}; -} -} // namespace - -class CudaGraphExecutorCacheTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) -{ -}; - -TEST_F(CudaGraphExecutorCacheTest, EmptyByDefault) -{ - tbu::CudaGraphExecutorCache cache(/*capacity=*/4); - EXPECT_EQ(cache.size(), 0); - EXPECT_FALSE(cache.get(makeBatchState(1)).has_value()); -} - -TEST_F(CudaGraphExecutorCacheTest, PutAndGetReturnsSameInstance) -{ - tbu::CudaGraphExecutorCache cache(/*capacity=*/4); - - auto bs = makeBatchState(1); - auto exec = makeDummyExecutor(); - cache.put(bs, exec); - - ASSERT_EQ(cache.size(), 1); - auto got = cache.get(bs); - ASSERT_TRUE(got.has_value()); - EXPECT_EQ(got->get(), exec.get()); -} - -TEST_F(CudaGraphExecutorCacheTest, PutWithExistingKeyReplaces) -{ - tbu::CudaGraphExecutorCache cache(/*capacity=*/4); - - auto bs = makeBatchState(1); - auto first = makeDummyExecutor(); - auto second = makeDummyExecutor(); - - cache.put(bs, first); - cache.put(bs, second); - - EXPECT_EQ(cache.size(), 1); - auto got = cache.get(bs); - ASSERT_TRUE(got.has_value()); - EXPECT_EQ(got->get(), second.get()); -} - -TEST_F(CudaGraphExecutorCacheTest, EvictsLeastRecentlyUsedAtCapacity) -{ - tbu::CudaGraphExecutorCache cache(/*capacity=*/2); - - auto bsA = makeBatchState(1); - auto bsB = makeBatchState(2); - auto bsC = makeBatchState(3); - - auto execA = makeDummyExecutor(); - auto execB = makeDummyExecutor(); - auto execC = makeDummyExecutor(); - - cache.put(bsA, execA); - cache.put(bsB, execB); - ASSERT_EQ(cache.size(), 2); - - // Access A so that B becomes the LRU entry. - EXPECT_TRUE(cache.get(bsA).has_value()); - - // Inserting C must evict B (the LRU), not A (just touched). - cache.put(bsC, execC); - EXPECT_EQ(cache.size(), 2); - EXPECT_TRUE(cache.get(bsA).has_value()); - EXPECT_FALSE(cache.get(bsB).has_value()); - EXPECT_TRUE(cache.get(bsC).has_value()); -} - -TEST_F(CudaGraphExecutorCacheTest, ClearDropsAllEntries) -{ - tbu::CudaGraphExecutorCache cache(/*capacity=*/4); - - auto bsA = makeBatchState(1); - auto bsB = makeBatchState(2); - auto bsC = makeBatchState(3); - - cache.put(bsA, makeDummyExecutor()); - cache.put(bsB, makeDummyExecutor()); - cache.put(bsC, makeDummyExecutor()); - ASSERT_EQ(cache.size(), 3); - - cache.clear(); - - EXPECT_EQ(cache.size(), 0); - EXPECT_FALSE(cache.get(bsA).has_value()); - EXPECT_FALSE(cache.get(bsB).has_value()); - EXPECT_FALSE(cache.get(bsC).has_value()); - - // After clearing, the cache must remain functional (i.e. clear() must not - // leave it in a broken state). - auto execA2 = makeDummyExecutor(); - cache.put(bsA, execA2); - EXPECT_EQ(cache.size(), 1); - auto got = cache.get(bsA); - ASSERT_TRUE(got.has_value()); - EXPECT_EQ(got->get(), execA2.get()); -} - -TEST_F(CudaGraphExecutorCacheTest, ClearReleasesExecutorOwnership) -{ - tbu::CudaGraphExecutorCache cache(/*capacity=*/4); - - auto exec = makeDummyExecutor(); - std::weak_ptr<tbu::CudaGraphExecutor> weak = exec; - - cache.put(makeBatchState(1), exec); - exec.reset(); - // The cache still owns one strong reference at this point. - ASSERT_FALSE(weak.expired()); - - cache.clear(); - - // After clear(), no strong references should remain. This guarantees that - // ~CudaGraphExecutor (which calls cudaGraphExecDestroy) actually runs for - // every cached entry - exactly what changeBeamWidth() relies on. - EXPECT_TRUE(weak.expired()); -} diff --git a/cpp/tests/unit_tests/batch_manager/encDecBeamSearchTest.cpp b/cpp/tests/unit_tests/batch_manager/encDecBeamSearchTest.cpp deleted file mode 100644 index 8e96c6a52ee3..000000000000 --- a/cpp/tests/unit_tests/batch_manager/encDecBeamSearchTest.cpp +++ /dev/null @@ -1,154 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/batch_manager/kvCacheManager.h" -#include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/batch_manager/runtimeBuffers.h" -#include "tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h" -#include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/kernels/kvCacheIndex.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/cudaStream.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/samplingConfig.h" -#include "gtest/gtest.h" -#include <memory> - -using namespace tensorrt_llm::batch_manager; -using namespace tensorrt_llm::batch_manager::kv_cache_manager; -namespace tr = tensorrt_llm::runtime; -namespace tc = tensorrt_llm::common; -namespace tk = tensorrt_llm::kernels; -using SizeType32 = tr::SizeType32; - -// Verify that copyGenerationLogits correctly assembles the host logits buffer -// using the real kernel merge path, and that two back-to-back calls (simulating -// two requests flushing in the same batch) use distinct fragmentPointerDevice -// slots so their pointer arrays do not clobber each other. -TEST(CopyGenerationLogitsTest, KernelMergePathProducesCorrectHostLayoutAndSlotsAreIsolated) -{ - SizeType32 constexpr beamWidth = 2; - SizeType32 constexpr numSteps = RuntimeBuffers::GenerationLogitsCache::kCACHE_LENGTH; // full flush - SizeType32 constexpr vocabSize = 8; - SizeType32 constexpr promptLen = 1; - SizeType32 constexpr maxBatchSize = 4; // must be >= 2 to test slot isolation - - auto stream = std::make_shared<tr::CudaStream>(); - tr::BufferManager bufferMgr{stream}; - - // Build a real GenerationLogitsCache so that transposedLogits, - // fragmentPointerDevice and fragmentPointerHost are all properly allocated. - // cache.logits uses pinned memory so the test can fill it from the CPU while - // the GPU kernel can still read from it via DMA. - RuntimeBuffers::GenerationLogitsCache cache; - cache.logits = tr::BufferManager::pinnedPool( - tr::ITensor::makeShape({numSteps, maxBatchSize * beamWidth, vocabSize}), nvinfer1::DataType::kFLOAT); - cache.transposedLogits - = bufferMgr.gpu(tr::ITensor::makeShape({beamWidth, numSteps, vocabSize}), nvinfer1::DataType::kFLOAT); - cache.fragmentPointerDevice - = bufferMgr.gpu(tr::ITensor::makeShape({maxBatchSize, numSteps}), nvinfer1::DataType::kINT64); - cache.fragmentPointerHost - = tr::BufferManager::pinnedPool(tr::ITensor::makeShape({maxBatchSize, numSteps}), nvinfer1::DataType::kINT64); - - // Helper: build one LlmRequest that has numSteps fragments pointing into - // cache.logits[0..numSteps-1][logitsIndex:logitsIndex+beamWidth]. - // Each fragment is filled with sentinel value (step*100 + beam + reqOffset). - auto makeRequest = [&](RequestIdType reqId, SizeType32 logitsIndex, float reqOffset) -> std::shared_ptr<LlmRequest> - { - auto tokens = std::make_shared<VecTokens>(promptLen, 0); - tr::SamplingConfig sc{beamWidth}; - auto req = std::make_shared<LlmRequest>(reqId, numSteps, tokens, sc, false); - - LlmRequest::BeamTokens gen(beamWidth, VecTokens(numSteps, 1)); - req->setGeneratedTokens(gen); - req->allocGenerationLogitsHost(vocabSize, nvinfer1::DataType::kFLOAT); - - // Write known values into the logits cache slots for this request and - // create matching fragment slice views. - for (SizeType32 step = 0; step < numSteps; ++step) - { - // cache.logits shape: [numSteps, maxBatchSize*beamWidth, vocabSize] - // Slice to [1, maxBS*bw, vocab], squeeze to [maxBS*bw, vocab]. - tr::ITensor::SharedPtr slot = tr::ITensor::slice(cache.logits, step, 1); - slot->squeeze(0); // [maxBS*bw, vocab] - auto* slotPtr = tr::bufferCast<float>(*slot); - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - float const val = reqOffset + static_cast<float>(step * 100 + beam); - for (SizeType32 v = 0; v < vocabSize; ++v) - { - slotPtr[(logitsIndex + beam) * vocabSize + v] = val; - } - } - - // Fragment matches HandleGenerationLogits: slice [logitsIndex:logitsIndex+beamWidth] - // from the step slot, then unsqueeze(0) → [1, beamWidth, vocab]. - tr::ITensor::SharedPtr fragView = tr::ITensor::slice(slot, logitsIndex, beamWidth); - fragView->unsqueeze(0); // [1, beamWidth, vocab] - req->addGenerationLogitsFragment(fragView); - } - return req; - }; - - // Request 0 occupies logitsIndex=0 in the batch slot. - auto req0 = makeRequest(1, /*logitsIndex=*/0, /*reqOffset=*/0.0f); - // Request 1 occupies logitsIndex=beamWidth in the batch slot. - auto req1 = makeRequest(2, /*logitsIndex=*/beamWidth, /*reqOffset=*/1000.0f); - - // Flush request 0 — uses workIdx=0. - utils::copyGenerationLogits(cache, bufferMgr, *req0, /*beforeDecoder=*/false, {}); - // Flush request 1 — uses workIdx=1 (different slot → no pointer clobbering). - utils::copyGenerationLogits(cache, bufferMgr, *req1, /*beforeDecoder=*/false, {}); - - ASSERT_EQ(cudaStreamSynchronize(stream->get()), cudaSuccess); - - // Verify req0 host buffer: host[beam, step, v] == step*100 + beam - auto const* host0 = tr::bufferCast<float>(*req0->getGenerationLogitsHost()); - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - for (SizeType32 step = 0; step < numSteps; ++step) - { - float const expected = static_cast<float>(step * 100 + beam); - for (SizeType32 v = 0; v < vocabSize; ++v) - { - SizeType32 const idx = (beam * numSteps + step) * vocabSize + v; - EXPECT_FLOAT_EQ(host0[idx], expected) << "req0 host[beam=" << beam << ",step=" << step << ",v=" << v - << "]=" << host0[idx] << " expected " << expected; - } - } - } - - // Verify req1 host buffer: host[beam, step, v] == 1000 + step*100 + beam - auto const* host1 = tr::bufferCast<float>(*req1->getGenerationLogitsHost()); - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - for (SizeType32 step = 0; step < numSteps; ++step) - { - float const expected = 1000.0f + static_cast<float>(step * 100 + beam); - for (SizeType32 v = 0; v < vocabSize; ++v) - { - SizeType32 const idx = (beam * numSteps + step) * vocabSize + v; - EXPECT_FLOAT_EQ(host1[idx], expected) << "req1 host[beam=" << beam << ",step=" << step << ",v=" << v - << "]=" << host1[idx] << " expected " << expected; - } - } - } - - // Both requests must have had their fragments cleared. - EXPECT_EQ(req0->getGenerationLogitsFragmentsSize(), 0); - EXPECT_EQ(req1->getGenerationLogitsFragmentsSize(), 0); -} diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheManagerFabricMemoryTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheManagerFabricMemoryTest.cpp index d23cbabe4144..4d41d751476c 100644 --- a/cpp/tests/unit_tests/batch_manager/kvCacheManagerFabricMemoryTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/kvCacheManagerFabricMemoryTest.cpp @@ -20,6 +20,7 @@ #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/samplingConfig.h" #include "tensorrt_llm/testing/kvCacheManagerTestUtil.h" @@ -108,7 +109,7 @@ TEST_F(KVCacheManagerFabricMemoryTest, AllocatePoolsFallbackWhenFabricUnsupporte BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, 0); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, 0); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -147,7 +148,7 @@ TEST_F(KVCacheManagerFabricMemoryTest, AllocatePoolsWithFabricMemory) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, 0); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, 0); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getMaxNumBlocks(), blocksInPrimaryPool); @@ -191,7 +192,7 @@ TEST_F(KVCacheManagerFabricMemoryTest, OffloadOnboardRoundTripWithFabricPrimary) BlockManager blockManager(std::vector<BlockManager::SizeType32>(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, 0); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, 0); blockManager.allocatePools(false); auto primaryPoolPtr = blockManager.getPrimaryPool(0); @@ -292,7 +293,7 @@ TEST_F(KVCacheManagerFabricMemoryTest, ReleasePoolsClearsFabricMemory) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, 0); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, 0); size_t freeBefore = 0; size_t freeAfterAlloc = 0; diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp index 34cf8bc82def..7bacaf9a2575 100644 --- a/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp @@ -26,6 +26,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/transferAgent.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/kernels/kvCacheIndex.h" @@ -174,7 +175,8 @@ TEST_F(KVCacheManagerTest, BlockManagerTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, + maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -310,7 +312,7 @@ void writePatternToOffloadedBlocksGDS( } } -template <typename T, nvinfer1::DataType type, int mask, KvCacheTransferMode transferMode> +template <typename T, tensorrt_llm::DataType type, int mask, KvCacheTransferMode transferMode> void runPartialCopyTest() { auto constexpr numLayers = 12; @@ -521,59 +523,59 @@ void runPartialCopyTest() TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyINT64) { - runPartialCopyTest<std::uint64_t, nvinfer1::DataType::kINT64, -1, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint64_t, nvinfer1::DataType::kINT64, -1, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint64_t, tensorrt_llm::DataType::kINT64, -1, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint64_t, tensorrt_llm::DataType::kINT64, -1, KvCacheTransferMode::GDS>(); } TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyINT32) { - runPartialCopyTest<std::uint32_t, nvinfer1::DataType::kINT32, -1, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint32_t, nvinfer1::DataType::kINT32, -1, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint32_t, tensorrt_llm::DataType::kINT32, -1, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint32_t, tensorrt_llm::DataType::kINT32, -1, KvCacheTransferMode::GDS>(); } TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyFLOAT) { - runPartialCopyTest<std::uint32_t, nvinfer1::DataType::kFLOAT, -1, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint32_t, nvinfer1::DataType::kFLOAT, -1, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint32_t, tensorrt_llm::DataType::kFLOAT, -1, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint32_t, tensorrt_llm::DataType::kFLOAT, -1, KvCacheTransferMode::GDS>(); } #ifdef ENABLE_BF16 TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyBF16) { - runPartialCopyTest<std::uint16_t, nvinfer1::DataType::kBF16, 65535, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint16_t, nvinfer1::DataType::kBF16, 65535, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint16_t, tensorrt_llm::DataType::kBF16, 65535, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint16_t, tensorrt_llm::DataType::kBF16, 65535, KvCacheTransferMode::GDS>(); } #endif TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyHALF) { - runPartialCopyTest<std::uint16_t, nvinfer1::DataType::kHALF, 65535, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint16_t, nvinfer1::DataType::kHALF, 65535, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint16_t, tensorrt_llm::DataType::kHALF, 65535, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint16_t, tensorrt_llm::DataType::kHALF, 65535, KvCacheTransferMode::GDS>(); } TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyBOOL) { - runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kBOOL, 255, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kBOOL, 255, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kBOOL, 255, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kBOOL, 255, KvCacheTransferMode::GDS>(); } TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyUINT8) { - runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kUINT8, 255, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kUINT8, 255, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kUINT8, 255, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kUINT8, 255, KvCacheTransferMode::GDS>(); } TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyINT8) { - runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kINT8, 255, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kINT8, 255, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kINT8, 255, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kINT8, 255, KvCacheTransferMode::GDS>(); } #ifdef ENABLE_FP8 TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyFP8) { - runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kFP8, 255, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kFP8, 255, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kFP8, 255, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kFP8, 255, KvCacheTransferMode::GDS>(); } #endif @@ -731,8 +733,8 @@ TEST_F(KVCacheManagerTest, FindBlocksInReuseTreeByBlockKeysTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, false, stream, - maxAttentionWindow, maxAttentionWindow, true); + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, false, + stream, maxAttentionWindow, maxAttentionWindow, true); // Add sequence [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16] (17 tokens, three blocks) auto inputTokens = std::make_shared<VecTokens>(VecTokens{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}); @@ -783,8 +785,8 @@ TEST_F(KVCacheManagerTest, FP4BlockScaleManagementTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kFP4, false, stream, - maxAttentionWindow, maxAttentionWindow, true); + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kFP4, false, + stream, maxAttentionWindow, maxAttentionWindow, true); kvCacheManager.allocatePools(/*useUvm=*/false); @@ -823,14 +825,14 @@ TEST_F(KVCacheManagerTest, FP4AttentionWithHalfRecurrentStatesPoolTest) {maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}, }; auto const poolConfigurations = std::vector<PoolConfiguration>{ - {recurrentStatesWindow, sizePerHead, nvinfer1::DataType::kHALF}, - {maxAttentionWindow, sizePerHead, nvinfer1::DataType::kFP4}, + {recurrentStatesWindow, sizePerHead, tensorrt_llm::DataType::kHALF}, + {maxAttentionWindow, sizePerHead, tensorrt_llm::DataType::kFP4}, }; auto const stream = std::make_shared<tr::CudaStream>(); KVCacheManager kvCacheManager(std::vector<SizeType32>{0, numKvHeads}, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, std::vector<SizeType32>{recurrentStatesWindow, maxAttentionWindow}, - nvinfer1::DataType::kFP4, + tensorrt_llm::DataType::kFP4, /*sinkTokenLength=*/0, stream, maxAttentionWindow, /*chunkSize=*/0, /*enableBlockReuse=*/false, CacheType::kSELF, std::nullopt, nullptr, /*enablePartialReuse=*/false, /*copyOnPartialReuse=*/true, nullptr, /*enableIndexerKCache=*/false, /*indexerKCacheQuantBlockSize=*/128, /*indexerKCacheIndexHeadDim=*/0, @@ -838,13 +840,13 @@ TEST_F(KVCacheManagerTest, FP4AttentionWithHalfRecurrentStatesPoolTest) kvCacheManager.allocatePools(/*useUvm=*/false); auto const& blockManager = kvCacheManager.getBlockManager(); - EXPECT_EQ(blockManager.getDataTypeForWindow(recurrentStatesWindow), nvinfer1::DataType::kHALF); - EXPECT_EQ(blockManager.getDataTypeForWindow(maxAttentionWindow), nvinfer1::DataType::kFP4); + EXPECT_EQ(blockManager.getDataTypeForWindow(recurrentStatesWindow), tensorrt_llm::DataType::kHALF); + EXPECT_EQ(blockManager.getDataTypeForWindow(maxAttentionWindow), tensorrt_llm::DataType::kFP4); auto const& recurrentStatesPool = blockManager.getRecurrentStatesPool(); ASSERT_NE(recurrentStatesPool.primaryPtr, nullptr); - EXPECT_EQ(recurrentStatesPool.primaryPtr->getDataType(), nvinfer1::DataType::kHALF); - auto const recurrentStatesElementsPerBlock = recurrentStatesBytes / tc::getDTypeSize(nvinfer1::DataType::kHALF); + EXPECT_EQ(recurrentStatesPool.primaryPtr->getDataType(), tensorrt_llm::DataType::kHALF); + auto const recurrentStatesElementsPerBlock = recurrentStatesBytes / tc::getDTypeSize(tensorrt_llm::DataType::kHALF); EXPECT_EQ(recurrentStatesPool.blockSize, recurrentStatesElementsPerBlock); SizeType32 numRecurrentScalePools = 0; @@ -905,7 +907,8 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, + maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -1240,7 +1243,8 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, + maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -1475,7 +1479,8 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithMultimodalHashTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, + maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -1679,7 +1684,8 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithLoraTaskIdTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, + maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -1972,7 +1978,8 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdAndLoraTaskIdTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, + maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -2241,7 +2248,8 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithCacheSaltTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, + maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -2473,7 +2481,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerPerRequestStatsTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -2530,7 +2538,8 @@ TEST_F(KVCacheManagerTest, BlockManagerBlockPriorityTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, + maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -2668,7 +2677,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerDecodeBlockPriorityTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -2775,7 +2784,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerTimedEvictionTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -2847,7 +2856,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerDecodeTimedEvictionTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); { @@ -2940,7 +2949,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerSecondaryBlockPrimaryChildTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -3037,7 +3046,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerStoreContextBlocksUsesMaterializedConte auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, /*chunkSize*/ 0, true); kvCacheManager.allocatePools(false); @@ -3081,7 +3090,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerReleaseBlocksUsesMaterializedContextExt auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, /*chunkSize*/ 0, true); kvCacheManager.allocatePools(false); @@ -3123,7 +3132,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerLeafBlockTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -3208,7 +3217,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerLeafBlockWithDependentTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -3312,7 +3321,7 @@ TEST_P(KVCacheManagerTest, DISABLED_KVCacheManagerAllocationTest) auto constexpr maxNumSequences = 8; auto constexpr maxBeamWidth = 4; auto constexpr sinkTokenLength = 0; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr maxSequenceLength = tokensPerBlock * maxBlocksPerSeq; @@ -3335,11 +3344,12 @@ TEST_P(KVCacheManagerTest, DISABLED_KVCacheManagerAllocationTest) KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(std::vector<KVCacheManager::SizeType32>(numLayers, numHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, - nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); + tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, + enableBlockReuse); auto const& blockManager = kvCacheManager.getBlockManager(); auto const& bufferManager = blockManager.getBufferManager(theOnlyWindowSize(kvCacheManager)); @@ -3410,10 +3420,10 @@ TEST_P(KVCacheManagerTest, KVCacheManagerTest) KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(numHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -3580,11 +3590,12 @@ TEST_P(KVCacheManagerTest, KVCacheManagerRewindTokensTest) KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(std::vector<KVCacheManager::SizeType32>(numLayers, numHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, - nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); + tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, + enableBlockReuse); kvCacheManager.allocatePools(false); EXPECT_EQ(kvCacheManager.getTokensPerBlock(), tokensPerBlock); @@ -3688,10 +3699,10 @@ TEST_P(KVCacheManagerTest, KVCacheManagerMaxAttentionWindowTest) KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(numHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -3812,7 +3823,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerMaxAttentionWindowSmallerThanBlockSizeT auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -3906,7 +3917,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStream) auto constexpr blocksInPrimaryPool = 8; auto constexpr blocksInSecondaryPool = 2; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr beamWidth = 1; @@ -4090,7 +4101,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerMaxAttentionWindowWithReuseTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, /*chunkSize=*/tokensPerBlock, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -4217,7 +4228,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerSWAInvalidateReuseTest) auto constexpr maxNumSequences = 8; auto constexpr maxBeamWidth = 1; auto constexpr sinkTokenLength = 0; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr maxSequenceLength = 128; SizeType32 constexpr maxNewTokens = 40; @@ -4300,7 +4311,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerVariableWindowAttentionWithReuseTest) auto constexpr maxNumSequences = 8; auto constexpr maxBeamWidth = 1; auto constexpr sinkTokenLength = 0; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr maxSequenceLength = 128; @@ -4426,7 +4437,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStreamOverflow) auto constexpr blocksInPrimaryPool = 8; auto constexpr blocksInSecondaryPool = 2; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr beamWidth = 1; @@ -4486,7 +4497,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStreamPriority) auto constexpr blocksInPrimaryPool = 8; auto constexpr blocksInSecondaryPool = 2; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr beamWidth = 1; @@ -4563,7 +4574,7 @@ TEST_F(KVCacheManagerTest, GetPriorityByBlockId) auto constexpr maxAttentionWindow = 32; auto constexpr maxNumSequences = 4; auto constexpr beamWidth = 1; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens = 4; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -4629,7 +4640,7 @@ TEST_F(KVCacheManagerTest, CommitAndGetBlockHashesForRequest) auto constexpr maxNumSequences = 4; auto constexpr beamWidth = 1; auto constexpr beamIdx = 0; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens = 8; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -4755,7 +4766,7 @@ TEST_F(KVCacheManagerTest, CommitAndGetBlockHashesFrontRunsTrailingFullBlock) auto constexpr maxNumSequences = 4; auto constexpr beamWidth = 1; auto constexpr beamIdx = 0; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens = 8; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -4871,7 +4882,7 @@ TEST_F(KVCacheManagerTest, PinAndUnpinBlocksById) BlocksPerWindow const blocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxAttentionWindow, maxAttentionWindow, true); kvCacheManager.allocatePools(false); @@ -4923,7 +4934,7 @@ TEST_F(KVCacheManagerTest, StoreBlocksForReuseWithPinDoesNotCreateGhostFreeBlock BlocksPerWindow const blocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxAttentionWindow, maxAttentionWindow, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); @@ -4995,7 +5006,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStreamBlocking) auto constexpr blocksInPrimaryPool = 8; auto constexpr blocksInSecondaryPool = 2; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr beamWidth = 1; @@ -5015,7 +5026,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStreamBlocking) EXPECT_EQ(getEvents(kvCacheManagerTest).size(), 0); KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true, CacheType::kSELF, std::nullopt, std::make_unique<tlk::KVCacheEventManager>(1024)); @@ -5050,7 +5061,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStreamWindowSize) auto blocksInPool = std::vector<SizeType32>{8, 2}; auto blocksInSlidingWindowPool = std::vector<SizeType32>{4, 2}; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr beamWidth = 1; @@ -5109,10 +5120,11 @@ TEST_F(KVCacheManagerTest, KVCacheTransferManagerConcurrencyTest) auto pool = KVCacheBlockPool(0, 2, 0, 0, 0); - pool.primaryPtr = bufferManager.gpu(tr::ITensor::makeShape({1, blockSize}), nvinfer1::DataType::kFLOAT); + pool.primaryPtr = bufferManager.gpu(tr::ITensor::makeShape({1, blockSize}), tensorrt_llm::DataType::kFLOAT); bufferManager.setZero(*pool.primaryPtr); - pool.secondaryPtr = tr::BufferManager::pinned(tr::ITensor::makeShape({1, blockSize}), nvinfer1::DataType::kFLOAT); + pool.secondaryPtr + = tr::BufferManager::pinned(tr::ITensor::makeShape({1, blockSize}), tensorrt_llm::DataType::kFLOAT); // Write some specific data into the cpu blocks. for (int i = 0; i < blockSize; i++) @@ -5149,11 +5161,11 @@ TEST_F(KVCacheManagerTest, KVCacheTransferManagerPendingTransfersDistinguishPrim auto pool = KVCacheBlockPool(0, 2, 0, 0, 0); pool.primaryPtr - = bufferManager.gpu(tr::ITensor::makeShape({kNumSlotsPerPool, kBlockSize}), nvinfer1::DataType::kFLOAT); + = bufferManager.gpu(tr::ITensor::makeShape({kNumSlotsPerPool, kBlockSize}), tensorrt_llm::DataType::kFLOAT); bufferManager.setZero(*pool.primaryPtr); - pool.secondaryPtr - = tr::BufferManager::pinned(tr::ITensor::makeShape({kNumSlotsPerPool, kBlockSize}), nvinfer1::DataType::kFLOAT); + pool.secondaryPtr = tr::BufferManager::pinned( + tr::ITensor::makeShape({kNumSlotsPerPool, kBlockSize}), tensorrt_llm::DataType::kFLOAT); auto primarySlot0 = std::make_shared<KVCacheBlock>(0, tk::KVCacheIndex(0, false)); auto primarySlot1 = std::make_shared<KVCacheBlock>(1, tk::KVCacheIndex(1, false)); @@ -5240,10 +5252,10 @@ TEST_P(KVCacheManagerTest, DISABLED_KVCacheManagerSinkTokenLengthTest) auto const maxSequenceLength = tokensPerBlock * maxBlocksPerSeq; KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(numHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -5402,10 +5414,10 @@ TEST_P(KVCacheManagerTest, KVCacheManagerBatchTest) KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(numHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -5542,12 +5554,12 @@ void testNeededBlocksOneStep(bool kv_cache_block_reuse, int beamWidth, int draft KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, - sinkTokenLength, stream, maxSequenceLength, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, + tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, /*chunkSize=*/tokensPerBlock, kv_cache_block_reuse) : KVCacheManager(numHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, - sinkTokenLength, stream, maxSequenceLength, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, + tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, /*chunkSize=*/tokensPerBlock, kv_cache_block_reuse); kvCacheManager.allocatePools(false); @@ -5750,7 +5762,7 @@ struct KvCacheManagerInstantiationParameters SizeType32 maxNumTokens; bool kvCacheBlockReuse; std::vector<SizeType32> maxAttentionWindowVec = {maxAttentionWindow}; - nvinfer1::DataType dtype = nvinfer1::DataType::kFLOAT; + tensorrt_llm::DataType dtype = tensorrt_llm::DataType::kFLOAT; }; BlocksPerWindow blocksAndWindow(SizeType32 numPrimaryBlocks, SizeType32 windowSize) @@ -7246,7 +7258,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventRemovedBatchedWithinWindow) auto constexpr maxNumSequences = 4; auto constexpr maxAttentionWindow = 32; auto constexpr beamWidth = 1; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens{0}; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -7324,7 +7336,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventRemovedOrderedBeforeStore) auto constexpr maxNumSequences = 4; auto constexpr maxAttentionWindow = 32; auto constexpr beamWidth = 1; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens{0}; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -7418,7 +7430,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStoreForDifferentWindowDoesNotFlus auto constexpr blocksInSecondaryPool = 0; auto constexpr maxNumSequences = 4; auto constexpr beamWidth = 1; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens{0}; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -7531,7 +7543,8 @@ void testBlockManagerLinearAttention_ContextNoReuse(int beamWidth, int numTokens BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{linearWindowSizeCode, maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, + std::vector<BlockManager::SizeType32>{linearWindowSizeCode, maxAttentionWindow}, tensorrt_llm::DataType::kHALF, + 0, /*chunkSize*/ 0, CacheType::kSELF, std::nullopt, nullptr, false, true, nullptr, std::nullopt, false, 128, 0, false, linearAttentionMetadata); blockManager.allocatePools(false); @@ -7676,7 +7689,8 @@ void testBlockManagerLinearAttention_ContextReuse(int beamWidth, int numTokens0, BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{linearWindowSizeCode, maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, + std::vector<BlockManager::SizeType32>{linearWindowSizeCode, maxAttentionWindow}, tensorrt_llm::DataType::kHALF, + 0, /*chunkSize*/ 0, CacheType::kSELF, std::nullopt, nullptr, false, true, nullptr, std::nullopt, false, 128, 0, false, linearAttentionMetadata); blockManager.allocatePools(false); @@ -7900,7 +7914,7 @@ void testKVCacheManagerLinearAttention_DecodingBlockGrowth( {linearWindowSizeCode, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, beamWidth, std::vector<BlockManager::SizeType32>{linearWindowSizeCode}, - /*dtype*/ nvinfer1::DataType::kHALF, + /*dtype*/ tensorrt_llm::DataType::kHALF, /*sinkTokenLen*/ sinkTokenLen, /*stream*/ stream, /*maxSequenceLength*/ maxAttentionWindow, @@ -8012,7 +8026,7 @@ void testKVCacheManagerLinearAttention_BlockCopying( {linearWindowSizeCode, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, beamWidth, std::vector<BlockManager::SizeType32>{linearWindowSizeCode, maxAttentionWindow}, - nvinfer1::DataType::kHALF, sinkTokenLen, stream, maxAttentionWindow, /*chunkSize*/ 0, enableContextReuse, + tensorrt_llm::DataType::kHALF, sinkTokenLen, stream, maxAttentionWindow, /*chunkSize*/ 0, enableContextReuse, CacheType::kSELF, std::nullopt, nullptr, false, true, nullptr, false, 128, 0, false, linearAttentionMetadata); kvCacheManager.allocatePools(false); @@ -8338,7 +8352,7 @@ TEST_F(KVCacheManagerTest, StaticLinearHybridAllocationTest) // Static-hybrid path requires block reuse to be disabled. tle::KvCacheConfig const kvCacheConfigDisabledReuse{/*enableBlockReuse=*/false}; auto const blocksPerWindow - = KVCacheManager::calculateMaxNumBlocks(kvCacheConfigDisabledReuse, nvinfer1::DataType::kHALF, + = KVCacheManager::calculateMaxNumBlocks(kvCacheConfigDisabledReuse, tensorrt_llm::DataType::kHALF, numKvHeadsPerLayer, sizePerHead, tokensPerBlock, worldConfig, windowSizeToLayers, allottedPrimaryMemBytes, allottedSecondaryMemBytes, extraCostMemory, kvFactor, maxBatchSize, linearAttentionMetadata); @@ -8357,7 +8371,7 @@ TEST_F(KVCacheManagerTest, StaticLinearHybridAllocationTest) // so the linear pool falls back to memory-budget-based sizing rather than maxBatchSize. tle::KvCacheConfig const kvCacheConfigEnabledReuse{/*enableBlockReuse=*/true}; auto const dynamicBlocksPerWindow - = KVCacheManager::calculateMaxNumBlocks(kvCacheConfigEnabledReuse, nvinfer1::DataType::kHALF, + = KVCacheManager::calculateMaxNumBlocks(kvCacheConfigEnabledReuse, tensorrt_llm::DataType::kHALF, numKvHeadsPerLayer, sizePerHead, tokensPerBlock, worldConfig, windowSizeToLayers, allottedPrimaryMemBytes, allottedSecondaryMemBytes, extraCostMemory, kvFactor, maxBatchSize, linearAttentionMetadata); EXPECT_NE(std::get<0>(dynamicBlocksPerWindow.at(linearWindowSizeCode)), maxBatchSize); @@ -8390,7 +8404,7 @@ static auto makeBatchTestKVCacheManager(std::shared_ptr<tensorrt_llm::runtime::C auto mgr = std::make_unique<KVCacheManager>(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, - nvinfer1::DataType::kHALF, 0, stream, maxAttentionWindow, /*chunkSize=*/maxAttentionWindow, + tensorrt_llm::DataType::kHALF, 0, stream, maxAttentionWindow, /*chunkSize=*/maxAttentionWindow, /*enableBlockReuse=*/true, CacheType::kSELF, /*secondaryOffloadMinPriority=*/std::nullopt, /*eventManager=*/nullptr, @@ -8719,7 +8733,7 @@ TEST_F(KVCacheManagerTest, BatchAddSequence_NonLeafCopySourceTightPool) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxAttentionWindow, /*chunkSize=*/maxAttentionWindow, /*enableBlockReuse=*/true, CacheType::kSELF, /*secondaryOffloadMinPriority=*/std::nullopt, /*eventManager=*/nullptr, @@ -8876,7 +8890,7 @@ std::unique_ptr<KVCacheManager> makePriorityEvictionManager( auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, 0}}}; auto mgr = std::make_unique<KVCacheManager>(kPE_NUM_LAYERS, kPE_NUM_HEADS, kPE_SIZE_PER_HEAD, kPE_TOKENS_PER_BLOCK, blocksPerWindow, kPE_MAX_NUM_SEQUENCES, kPE_BEAM_WIDTH, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxAttentionWindow, /*chunkSize=*/maxAttentionWindow, /*enableBlockReuse=*/true); mgr->allocatePools(false); return mgr; @@ -9210,7 +9224,7 @@ std::unique_ptr<KVCacheManager> makeVSWAManager( { auto const blocksPerWindow = BlocksPerWindow{{kVSWA_ATTENTION_WINDOW, {blocksInPrimaryPool, 0}}}; auto mgr = std::make_unique<KVCacheManager>(2, 2, 64, kVSWA_TOKENS_PER_BLOCK, blocksPerWindow, 8, kVSWA_BEAM_WIDTH, - std::vector<SizeType32>{kVSWA_ATTENTION_WINDOW}, nvinfer1::DataType::kHALF, 0, stream, + std::vector<SizeType32>{kVSWA_ATTENTION_WINDOW}, tensorrt_llm::DataType::kHALF, 0, stream, kVSWA_MAX_SEQUENCE_LENGTH, /*chunkSize=*/kVSWA_MAX_SEQUENCE_LENGTH, enableBlockReuse); mgr->allocatePools(false); return mgr; @@ -9229,7 +9243,7 @@ std::unique_ptr<KVCacheManager> makeSmallWindowManager( SizeType32 constexpr kSmallMaxSeqLen = 128; auto const blocksPerWindow = BlocksPerWindow{{kSmallWindow, {blocksInPrimaryPool, 0}}}; auto mgr = std::make_unique<KVCacheManager>(2, 2, 64, kSmallTpb, blocksPerWindow, 8, kVSWA_BEAM_WIDTH, - std::vector<SizeType32>{kSmallWindow}, nvinfer1::DataType::kHALF, 0, stream, kSmallMaxSeqLen, + std::vector<SizeType32>{kSmallWindow}, tensorrt_llm::DataType::kHALF, 0, stream, kSmallMaxSeqLen, /*chunkSize=*/kSmallMaxSeqLen, /*enableBlockReuse=*/true); mgr->allocatePools(false); return mgr; @@ -9952,7 +9966,7 @@ TEST_F(KVCacheManagerTest, VSWAEvictedPlaceholderAnchorAllowsTrailingReuse) auto const blocksPerWindow = BlocksPerWindow{{window, {blocksInPrimaryPool, 0}}}; KVCacheManager kvCacheManager(2, 2, 64, tpb, blocksPerWindow, 8, kVSWA_BEAM_WIDTH, std::vector<SizeType32>{window}, - nvinfer1::DataType::kHALF, 0, stream, + tensorrt_llm::DataType::kHALF, 0, stream, /*maxSequenceLength=*/128, /*chunkSize=*/128, /*enableBlockReuse=*/true); kvCacheManager.allocatePools(false); auto const& blockManager = kvCacheManager.getBlockManager(); @@ -10088,7 +10102,7 @@ std::unique_ptr<KVCacheManager> makeConnectorTestKVCacheManager( auto mgr = std::make_unique<KVCacheManager>(std::vector<SizeType32>(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, - /*dtype*/ nvinfer1::DataType::kHALF, + /*dtype*/ tensorrt_llm::DataType::kHALF, /*sinkTokenLength*/ 0, stream, /*maxSequenceLength*/ maxAttentionWindow, /*chunkSize*/ maxAttentionWindow, @@ -10282,7 +10296,7 @@ TEST_F(KVCacheManagerTest, BlockManagerTestPerWindowFallback) auto constexpr maxBeamWidth = 1; auto constexpr smallWindow = 1024; auto constexpr largeWindow = 4096; - auto constexpr scalarDtype = nvinfer1::DataType::kHALF; + auto constexpr scalarDtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto const maxAttentionWindowVec = std::vector<SizeType32>{smallWindow, largeWindow}; auto const blocksPerWindow = BlocksPerWindow{ @@ -10330,7 +10344,7 @@ TEST(BaseKVCacheManagerCalculateMaxNumBlocks, PerWindowOverrideDivergesByteBudge uint64_t const allottedPrimaryMemBytes = static_cast<uint64_t>(1) << 30; // 1 GiB uint64_t const allottedSecondaryMemBytes = static_cast<uint64_t>(1) << 30; size_t const extraCostMemory = 0; - auto const dtype = nvinfer1::DataType::kHALF; + auto const dtype = tensorrt_llm::DataType::kHALF; tensorrt_llm::executor::KvCacheConfig const config{}; tensorrt_llm::runtime::WorldConfig const worldConfig{}; @@ -10384,7 +10398,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerSWAEvictionCountPerWindow) auto constexpr maxNumSequences = 4; auto constexpr maxBeamWidth = 1; auto constexpr sinkTokenLength = 0; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr maxSequenceLength = 128; auto constexpr maxNewTokens = 40; @@ -10474,7 +10488,7 @@ TEST_F(KVCacheManagerTest, GenerationRequestClearCacheBlocksPerWindowResetsOnlyT BlockManager blockManager(std::vector<BlockManager::SizeType32>(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, /*maxSequenceLength=*/fullWindow, maxBeamWidth, maxAttentionWindowVec, - nvinfer1::DataType::kHALF, /*sinkBubbleLength=*/0, /*chunkSize=*/0); + tensorrt_llm::DataType::kHALF, /*sinkBubbleLength=*/0, /*chunkSize=*/0); blockManager.allocatePools(/*useUvm=*/false); auto constexpr requestId = 7; @@ -10527,7 +10541,7 @@ TEST_F(KVCacheManagerTest, VswaMixedHeadDimReuseSmoke) auto constexpr smallSizePerHead = 256; auto constexpr largeSizePerHead = 512; auto constexpr sinkTokenLength = 0; - auto constexpr dtype = nvinfer1::DataType::kHALF; + auto constexpr dtype = tensorrt_llm::DataType::kHALF; auto constexpr maxSequenceLength = 64; auto constexpr maxNewTokens = 0; auto const stream = std::make_shared<tr::CudaStream>(); @@ -10605,12 +10619,58 @@ TEST_F(KVCacheManagerTest, VswaMixedHeadDimReuseSmoke) } } +TEST_F(KVCacheManagerTest, HybridDisaggUsesAttentionPoolDtype) +{ + auto constexpr numKvHeads = 2; + auto constexpr sizePerHead = 16; + auto constexpr tokensPerBlock = 4; + auto constexpr blocksInPrimaryPool = 4; + auto constexpr blocksInSecondaryPool = 0; + auto constexpr maxNumSequences = 2; + auto constexpr maxBeamWidth = 1; + auto constexpr maxAttentionWindow = 16; + auto constexpr recurrentStatesBytes = 64; + SizeType32 constexpr recurrentStatesWindow = LinearAttentionMetadata::LinearCacheType::kRecurrentStates; + + LinearAttentionMetadata const linearAttentionMetadata{ + .linearLayerIndices = {0}, + .cacheType = recurrentStatesWindow, + .allRecurrentStatesBytes = recurrentStatesBytes, + }; + auto const blocksPerWindow = BlocksPerWindow{ + {recurrentStatesWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}, + {maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}, + }; + auto const poolConfigurations = std::vector<PoolConfiguration>{ + {recurrentStatesWindow, sizePerHead, tensorrt_llm::DataType::kHALF}, + {maxAttentionWindow, sizePerHead, tensorrt_llm::DataType::kFP8}, + }; + auto const stream = std::make_shared<tr::CudaStream>(); + + auto kvCacheManager = std::make_unique<KVCacheManager>(std::vector<SizeType32>{0, numKvHeads}, sizePerHead, + tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, + std::vector<SizeType32>{recurrentStatesWindow, maxAttentionWindow}, tensorrt_llm::DataType::kFP8, + /*sinkTokenLength=*/0, stream, maxAttentionWindow, /*chunkSize=*/0, /*enableBlockReuse=*/false, + CacheType::kSELF, std::nullopt, nullptr, /*enablePartialReuse=*/false, /*copyOnPartialReuse=*/true, nullptr, + /*enableIndexerKCache=*/false, /*indexerKCacheQuantBlockSize=*/128, /*indexerKCacheIndexHeadDim=*/0, + /*indexerKCacheUseFp4=*/false, linearAttentionMetadata, poolConfigurations); + kvCacheManager->allocatePools(/*useUvm=*/false); + + CacheTransBufferManager cacheTransBufferManager(kvCacheManager.get(), /*maxNumTokens=*/tokensPerBlock); + EXPECT_EQ(cacheTransBufferManager.getDataType(), tensorrt_llm::DataType::kFP8); + + auto const bufferId = cacheTransBufferManager.assignBufferIndexForSend(); + ASSERT_TRUE(bufferId.has_value()); + EXPECT_EQ(cacheTransBufferManager.getSendBuffer(bufferId)->getDataType(), tensorrt_llm::DataType::kFP8); + cacheTransBufferManager.freeBufferIndexForSend(bufferId); +} + // A6: VSWA + disagg dtype mismatch must fire the A4 guard. // -// The constructor of CacheTransBufferManager picks pool 0's dtype as canonical for -// the wire transport. When a KVCacheManager hosts pools with differing dtypes -// (mixed-precision per-window), that silent coercion would corrupt the wire format. -// The guard added in cacheTransBuffer.cpp must throw at construction time. +// CacheTransBufferManager uses a single dtype for the wire transport. When a +// KVCacheManager hosts attention pools with differing dtypes (mixed-precision +// per-window), that silent coercion would corrupt the wire format. The guard in +// cacheTransBuffer.cpp must throw at construction time. // // This test only exercises the helper / construction path that runs the guard; it // does not stand up a full disaggregated transfer (out of scope at unit-test @@ -10633,11 +10693,12 @@ TEST_F(KVCacheManagerTest, VswaDisaggDtypeMismatchTriggersGuard) auto const maxAttentionWindowVec = std::vector<SizeType32>{smallWindow, largeWindow}; auto const blocksPerWindow = BlocksPerWindow{ {smallWindow, {blocksInPrimary, blocksInSecondary}}, {largeWindow, {blocksInPrimary, blocksInSecondary}}}; - auto const poolConfigurations = std::vector<PoolConfiguration>{ - {smallWindow, sizePerHead, nvinfer1::DataType::kHALF}, {largeWindow, sizePerHead, nvinfer1::DataType::kBF16}}; + auto const poolConfigurations + = std::vector<PoolConfiguration>{{smallWindow, sizePerHead, tensorrt_llm::DataType::kHALF}, + {largeWindow, sizePerHead, tensorrt_llm::DataType::kBF16}}; auto kvCacheManager = std::make_unique<KVCacheManager>(numLayers, numKvHeads, sizePerHead, tokensPerBlock, - blocksPerWindow, maxNumSequences, maxBeamWidth, maxAttentionWindowVec, /*dtype=*/nvinfer1::DataType::kHALF, + blocksPerWindow, maxNumSequences, maxBeamWidth, maxAttentionWindowVec, /*dtype=*/tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, /*chunkSize=*/0, /*enableBlockReuse=*/false, CacheType::kSELF, /*secondaryOffloadMinPriority=*/std::nullopt, /*eventManager=*/nullptr, @@ -10649,14 +10710,23 @@ TEST_F(KVCacheManagerTest, VswaDisaggDtypeMismatchTriggersGuard) kvCacheManager->allocatePools(/*useUvm=*/false); // Sanity: the manager really does host KV pools with two different dtypes. - auto const numKvPools = kvCacheManager->getBlockManager().getNumPools( - /*includeBlockScalePools=*/false, /*includeIndexerKCachePools=*/false); - ASSERT_GE(numKvPools, 2); - auto const dtype0 = kvCacheManager->getPrimaryPool(0)->getDataType(); + auto const& blockManager = kvCacheManager->getBlockManager(); + ASSERT_GE(blockManager.getNumPools(/*includeBlockScalePools=*/false, /*includeIndexerKCachePools=*/false), 2); + std::optional<tensorrt_llm::DataType> dtype0; bool foundMismatch = false; - for (SizeType32 i = 1; i < numKvPools; ++i) + for (SizeType32 poolIdx = 0; poolIdx < blockManager.getNumPools(); ++poolIdx) { - if (kvCacheManager->getPrimaryPool(i)->getDataType() != dtype0) + auto const& pool = blockManager.getPool(poolIdx); + if (pool.containsBlockScales || pool.containsIndexerKCache) + { + continue; + } + auto const dataType = blockManager.getPrimaryPool(poolIdx)->getDataType(); + if (!dtype0.has_value()) + { + dtype0 = dataType; + } + else if (dataType != dtype0.value()) { foundMismatch = true; break; diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheManagerV2HostMemTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheManagerV2HostMemTest.cpp new file mode 100644 index 000000000000..82ba4c049f86 --- /dev/null +++ b/cpp/tests/unit_tests/batch_manager/kvCacheManagerV2HostMemTest.cpp @@ -0,0 +1,206 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/exceptions.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/hostMem.h" + +#include <cuda_runtime_api.h> +#include <gtest/gtest.h> +#include <sys/mman.h> + +#include <algorithm> +#include <cerrno> +#include <cstdlib> +#include <cstring> +#include <optional> +#include <string> +#include <system_error> +#include <vector> + +namespace +{ + +using namespace tensorrt_llm::batch_manager::kv_cache_manager_v2; + +class ScopedEnv +{ +public: + ScopedEnv(char const* name, std::optional<std::string> value) + : mName(name) + { + if (char const* oldValue = std::getenv(name); oldValue != nullptr) + { + mOldValue = oldValue; + } + if (value.has_value()) + { + if (::setenv(name, value->c_str(), /*overwrite=*/1) != 0) + { + throw std::system_error(errno, std::generic_category(), "setenv failed"); + } + } + else if (::unsetenv(name) != 0) + { + throw std::system_error(errno, std::generic_category(), "unsetenv failed"); + } + } + + ~ScopedEnv() + { + if (mOldValue.has_value()) + { + ::setenv(mName.c_str(), mOldValue->c_str(), /*overwrite=*/1); + } + else + { + ::unsetenv(mName.c_str()); + } + } + +private: + std::string mName; + std::optional<std::string> mOldValue; +}; + +int gMadviseErrno = 0; +int gCapturedAdvice = 0; +int gMemsetCalls = 0; + +int captureMadvise(void*, size_t, int advice) +{ + gCapturedAdvice = advice; + return 0; +} + +int failMadvise(void*, size_t, int) +{ + errno = gMadviseErrno; + return -1; +} + +void* countMemset(void* ptr, int value, size_t size) +{ + ++gMemsetCalls; + return std::memset(ptr, value, size); +} + +TEST(KvCacheManagerV2HostMemTest, SelectsConfiguredPageMode) +{ + hostMadvisePageMode(MemAddress{1}, HostMem::kAlignment, true, captureMadvise); + EXPECT_EQ(gCapturedAdvice, MADV_HUGEPAGE); + hostMadvisePageMode(MemAddress{1}, HostMem::kAlignment, false, captureMadvise); + EXPECT_EQ(gCapturedAdvice, MADV_NOHUGEPAGE); + + ScopedEnv defaultThp("TLLM_KV_CACHE_MANAGER_V2_THP", std::nullopt); + EXPECT_TRUE(hostUseThp()); + { + ScopedEnv disableThp("TLLM_KV_CACHE_MANAGER_V2_THP", "0"); + EXPECT_FALSE(hostUseThp()); + } +} + +TEST(KvCacheManagerV2HostMemTest, ReadsPrefaultThreadConfiguration) +{ + ScopedEnv disablePrefault("TLLM_KV_CACHE_MANAGER_V2_PREFAULT_THREADS", "0"); + EXPECT_EQ(hostPrefaultThreads(), 0); + { + ScopedEnv threeThreads("TLLM_KV_CACHE_MANAGER_V2_PREFAULT_THREADS", "3"); + EXPECT_EQ(hostPrefaultThreads(), 3); + } +} + +class PrefaultFallbackTest : public testing::TestWithParam<int> +{ +}; + +TEST_P(PrefaultFallbackTest, TouchesMemory) +{ + std::vector<unsigned char> data(HostMem::kAlignment, 0xFF); + gMadviseErrno = GetParam(); + gMemsetCalls = 0; + hostPrefaultChunk(reinterpret_cast<MemAddress>(data.data()), data.size(), failMadvise, countMemset); + EXPECT_EQ(gMemsetCalls, 1); + EXPECT_TRUE(std::all_of(data.begin(), data.end(), [](unsigned char value) { return value == 0; })); +} + +INSTANTIATE_TEST_SUITE_P(UnsupportedPopulateWrite, PrefaultFallbackTest, testing::Values(EINVAL, ENOSYS)); + +TEST(KvCacheManagerV2HostMemTest, ConvertsPrefaultEnomem) +{ + std::vector<unsigned char> data(HostMem::kAlignment); + gMadviseErrno = ENOMEM; + EXPECT_THROW(hostPrefaultChunk(reinterpret_cast<MemAddress>(data.data()), data.size(), failMadvise, countMemset), + HostOOMError); +} + +TEST(KvCacheManagerV2HostMemTest, PropagatesOtherPrefaultErrors) +{ + std::vector<unsigned char> data(HostMem::kAlignment); + gMadviseErrno = EIO; + try + { + hostPrefaultChunk(reinterpret_cast<MemAddress>(data.data()), data.size(), failMadvise, countMemset); + FAIL() << "Expected std::system_error"; + } + catch (std::system_error const& error) + { + EXPECT_EQ(error.code().value(), EIO); + } +} + +class HostMemPageModeTest : public testing::TestWithParam<char const*> +{ +}; + +TEST_P(HostMemPageModeTest, RegistersAndResizesWithPrefaultDisabled) +{ + ScopedEnv thp("TLLM_KV_CACHE_MANAGER_V2_THP", std::string(GetParam())); + ScopedEnv prefault("TLLM_KV_CACHE_MANAGER_V2_PREFAULT_THREADS", "0"); + ASSERT_EQ(cudaSetDevice(0), cudaSuccess); + HostMem memory(HostMem::kAlignment); + EXPECT_NE(memory.address(), 0); + EXPECT_EQ(memory.size(), HostMem::kAlignment); + memory.resize(2 * HostMem::kAlignment); + EXPECT_EQ(memory.size(), 2 * HostMem::kAlignment); +} + +INSTANTIATE_TEST_SUITE_P(ThpModes, HostMemPageModeTest, testing::Values("0", "1")); + +TEST(KvCacheManagerV2HostMemTest, PrefaultedAllocationSupportsGpuRoundTrip) +{ + ScopedEnv thp("TLLM_KV_CACHE_MANAGER_V2_THP", "1"); + ScopedEnv prefault("TLLM_KV_CACHE_MANAGER_V2_PREFAULT_THREADS", "2"); + ASSERT_EQ(cudaSetDevice(0), cudaSuccess); + + constexpr size_t kSize = 4 << 20; + HostMem memory(kSize); + std::memset(reinterpret_cast<void*>(memory.address()), 0x5A, kSize); + + void* devicePtr = nullptr; + ASSERT_EQ(cudaMalloc(&devicePtr, kSize), cudaSuccess); + ASSERT_EQ( + cudaMemcpy(devicePtr, reinterpret_cast<void*>(memory.address()), kSize, cudaMemcpyHostToDevice), cudaSuccess); + std::memset(reinterpret_cast<void*>(memory.address()), 0, kSize); + ASSERT_EQ( + cudaMemcpy(reinterpret_cast<void*>(memory.address()), devicePtr, kSize, cudaMemcpyDeviceToHost), cudaSuccess); + + auto const* bytes = reinterpret_cast<unsigned char const*>(memory.address()); + EXPECT_TRUE(std::all_of(bytes, bytes + kSize, [](unsigned char value) { return value == 0x5A; })); + EXPECT_EQ(cudaFree(devicePtr), cudaSuccess); +} + +} // namespace diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheManagerV2StatsTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheManagerV2StatsTest.cpp new file mode 100644 index 000000000000..cd91444e6028 --- /dev/null +++ b/cpp/tests/unit_tests/batch_manager/kvCacheManagerV2StatsTest.cpp @@ -0,0 +1,371 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/blockRadixTree.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/config.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCache.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/kvCacheManager.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/pendingStats.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/stats.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/storageManager.h" + +#include <cuda_runtime_api.h> +#include <gtest/gtest.h> + +#include <limits> +#include <memory> + +namespace +{ + +using namespace tensorrt_llm::batch_manager::kv_cache_manager_v2; + +KVCacheManagerConfig makeConfig(bool enableStats = true) +{ + KVCacheManagerConfig config; + config.tokensPerBlock = 4; + config.cacheTiers.emplace_back(GpuCacheTierConfig{4 << 20}); + AttentionLayerConfig layer; + layer.layerId = 0; + layer.buffers.push_back(BufferConfig{"key", 4096, std::nullopt}); + config.layers.emplace_back(std::move(layer)); + config.enableStats = enableStats; + return config; +} + +KVCacheManagerConfig makeTieredConfig() +{ + KVCacheManagerConfig config; + config.tokensPerBlock = 4; + config.cacheTiers.emplace_back(GpuCacheTierConfig{4 << 20}); + config.cacheTiers.emplace_back(HostCacheTierConfig{4 << 20}); + AttentionLayerConfig layer; + layer.layerId = 0; + layer.buffers.push_back(BufferConfig{"key", 2 << 20, std::nullopt}); + config.layers.emplace_back(std::move(layer)); + return config; +} + +TEST(KvCacheManagerV2StatsTest, StatsDeltaArithmetic) +{ + KVCacheStatsDelta stats{4, 3, 2, 1}; + KVCacheStatsDelta const delta{1, 2, 3, 4}; + stats.add(delta); + EXPECT_EQ(stats.allocTotalBlocks, 5); + EXPECT_EQ(stats.allocNewBlocks, 5); + EXPECT_EQ(stats.reusedBlocks, 5); + EXPECT_EQ(stats.missedBlocks, 5); + + KVCacheStatsDelta const copy = stats.copy(); + stats.subtract(delta); + EXPECT_EQ(stats.allocTotalBlocks, 4); + EXPECT_EQ(copy.allocTotalBlocks, 5); + stats.clear(); + EXPECT_TRUE(stats.empty()); +} + +TEST(KvCacheManagerV2StatsTest, IterationStatsDeltaArithmeticAndHitRate) +{ + KVCacheIterationStatsDelta stats; + stats.iterReusedBlocks = 3; + stats.iterFullReusedBlocks = 2; + stats.iterPartialReusedBlocks = 1; + stats.iterMissedBlocks = 1; + stats.iterOnboardBytes = 1024; + EXPECT_DOUBLE_EQ(stats.iterCacheHitRate(), 0.75); + + KVCacheIterationStatsDelta delta = stats.copy(); + stats.add(delta); + EXPECT_EQ(stats.iterReusedBlocks, 6); + EXPECT_EQ(stats.iterOnboardBytes, 2048); + stats.subtract(delta); + EXPECT_EQ(stats.iterReusedBlocks, 3); + stats.clear(); + EXPECT_TRUE(stats.empty()); + EXPECT_DOUBLE_EQ(stats.iterCacheHitRate(), 0.0); +} + +TEST(KvCacheManagerV2StatsTest, PendingAllocationRangesAreReversibleAndScoped) +{ + PendingStats pending; + EXPECT_TRUE(pending.recordAllocationRange( + LifeCycleId{0}, BlockOrdinal{0}, BlockOrdinal{3}, /*beamWidth=*/2, /*countAsMissed=*/true)); + EXPECT_TRUE(pending.recordAllocationRange(LifeCycleId{1}, BlockOrdinal{3}, BlockOrdinal{5}, + /*beamWidth=*/1, /*countAsMissed=*/false, /*countAsGeneration=*/true)); + + EXPECT_EQ(pending.globalStats().allocTotalBlocks, 8); + EXPECT_EQ(pending.globalStats().allocNewBlocks, 8); + EXPECT_EQ(pending.globalStats().missedBlocks, 6); + EXPECT_EQ(pending.requestStats().allocTotalBlocks, 8); + ASSERT_EQ(pending.iterationStatsByLifeCycle().size(), 2); + EXPECT_EQ(pending.iterationStatsByLifeCycle().at(LifeCycleId{0}).iterMissedBlocks, 6); + EXPECT_EQ(pending.iterationStatsByLifeCycle().at(LifeCycleId{1}).iterGenAllocBlocks, 2); + + EXPECT_TRUE(pending.subtractAllocationRange(BlockOrdinal{2}, BlockOrdinal{5})); + EXPECT_EQ(pending.globalStats().allocTotalBlocks, 4); + EXPECT_EQ(pending.globalStats().missedBlocks, 4); + ASSERT_EQ(pending.iterationStatsByLifeCycle().size(), 1); + EXPECT_EQ(pending.iterationStatsByLifeCycle().at(LifeCycleId{0}).iterAllocTotalBlocks, 4); + + EXPECT_TRUE(pending.subtractAllocationRange(BlockOrdinal{0}, BlockOrdinal{2})); + EXPECT_TRUE(pending.empty()); +} + +TEST(KvCacheManagerV2StatsTest, PendingReuseSurvivesAllocationRollbackUntilClear) +{ + PendingStats pending; + EXPECT_TRUE(pending.recordAllocationRange( + LifeCycleId{0}, BlockOrdinal{0}, BlockOrdinal{1}, /*beamWidth=*/1, /*countAsMissed=*/true)); + EXPECT_TRUE(pending.recordReuse(LifeCycleId{0}, /*fullReusedBlocks=*/2, /*partialReusedBlocks=*/1)); + + EXPECT_TRUE(pending.subtractAllocationRange(BlockOrdinal{0}, BlockOrdinal{1})); + EXPECT_EQ(pending.globalStats().allocTotalBlocks, 0); + EXPECT_EQ(pending.globalStats().reusedBlocks, 3); + auto const& iteration = pending.iterationStatsByLifeCycle().at(LifeCycleId{0}); + EXPECT_EQ(iteration.iterReusedBlocks, 3); + EXPECT_EQ(iteration.iterFullReusedBlocks, 2); + EXPECT_EQ(iteration.iterPartialReusedBlocks, 1); + + pending.clear(); + EXPECT_TRUE(pending.empty()); +} + +TEST(KvCacheManagerV2StatsTest, ManagerCommitResetAndRequestIdTracking) +{ + ASSERT_EQ(cudaSetDevice(0), cudaSuccess); + auto manager = std::make_shared<KvCacheManager>(makeConfig()); + + KVCacheStatsDelta globalStats{4, 3, 2, 1}; + KVCacheIterationStatsDelta iterationStats; + iterationStats.iterAllocTotalBlocks = 4; + iterationStats.iterReusedBlocks = 2; + manager->commitStats(globalStats, {{LifeCycleId{0}, iterationStats}}); + + EXPECT_EQ(manager->getCommittedStats().allocTotalBlocks, 4); + auto firstIteration = manager->getAndResetIterationStats(); + ASSERT_EQ(firstIteration.size(), 1); + EXPECT_EQ(firstIteration.at(LifeCycleId{0}).iterReusedBlocks, 2); + EXPECT_TRUE(manager->getAndResetIterationStats().empty()); + + manager->markStatsDirty(11); + manager->markStatsDirty(std::nullopt); + EXPECT_EQ(manager->getDirtyStatsKvCacheIds().count(11), 1); + manager->markStatsExcluded(11); + EXPECT_TRUE(manager->isStatsExcluded(11)); + EXPECT_TRUE(manager->getDirtyStatsKvCacheIds().empty()); + manager->clearStatsExcluded(11); + EXPECT_FALSE(manager->isStatsExcluded(11)); + + auto cache = manager->createKvCache({}, {}, 17, {}, 8); + manager->markStatsDirty(17); + EXPECT_TRUE(cache->commitPendingStats().empty()); + EXPECT_TRUE(manager->getDirtyStatsKvCacheIds().empty()); + cache->close(); + + RequestIdType const cudaGraphDummyRequestId = std::numeric_limits<RequestIdType>::max(); + auto dummyCache = manager->createKvCache({}, {}, cudaGraphDummyRequestId); + ASSERT_TRUE(dummyCache->id.has_value()); + EXPECT_EQ(*dummyCache->id, cudaGraphDummyRequestId); + manager->markStatsDirty(cudaGraphDummyRequestId); + EXPECT_EQ(manager->getDirtyStatsKvCacheIds(), std::unordered_set<RequestIdType>{cudaGraphDummyRequestId}); + manager->markStatsExcluded(cudaGraphDummyRequestId); + EXPECT_TRUE(manager->isStatsExcluded(cudaGraphDummyRequestId)); + EXPECT_TRUE(manager->getDirtyStatsKvCacheIds().empty()); + dummyCache->close(); +} + +TEST(KvCacheManagerV2StatsTest, DisabledStatsSuppressManagerCommit) +{ + ASSERT_EQ(cudaSetDevice(0), cudaSuccess); + auto manager = std::make_shared<KvCacheManager>(makeConfig(false)); + manager->commitStats(KVCacheStatsDelta{4, 3, 2, 1}); + EXPECT_TRUE(manager->getCommittedStats().empty()); + EXPECT_TRUE(manager->getAndResetIterationStats().empty()); +} + +TEST(KvCacheManagerV2StatsTest, PeakBlockStatsResetStartsNextIntervalFromCurrentSnapshot) +{ + ASSERT_EQ(cudaSetDevice(0), cudaSuccess); + auto manager = std::make_shared<KvCacheManager>(makeTieredConfig()); + auto& storage = manager->storage(); + LifeCycleId const lifeCycle{0}; + + TypedVec<LifeCycleId, SlotCount> twoSlots(LifeCycleId{1}, 2); + auto gpuSlots = storage.newGpuSlots(twoSlots); + manager->commitStats({}); + + RootBlock& root = manager->radixTree().addOrGetExisting({}); + std::vector<SharedPtr<Page>> pages; + NodeBase* previous = &root; + int token = 0; + for (auto& slot : gpuSlots[lifeCycle]) + { + std::vector<TokenIdExt> tokens; + for (int i = 0; i < manager->tokensPerBlock(); ++i) + { + tokens.emplace_back(TokenId{token++}); + } + auto block = addOrGetExistingBlock(previous, LifeCycleId{1}, std::move(tokens)); + auto page = makeShared<CommittedPage>(&storage, block, lifeCycle, kGpuLevel, kPriorityDefault); + page->setSlot(slot); + block->storage[lifeCycle] = page.get(); + storage.scheduleForEviction(*page); + pages.push_back(page); + previous = block.get(); + } + manager->commitStats({}); + + TypedVec<LifeCycleId, SlotCount> oneSlot(LifeCycleId{1}, 1); + auto hostSlots = storage.newSlots(CacheLevel{1}, oneSlot); + manager->commitStats({}); + storage.releaseSlot(lifeCycle, CacheLevel{1}, std::move(hostSlots[lifeCycle].front())); + manager->clearReusableBlocks(); + pages.clear(); + + auto primaryPeak = manager->getAndResetIterationPeakBlockStats(kGpuLevel); + auto secondaryPeak = manager->getAndResetIterationPeakBlockStats(CacheLevel{1}); + ASSERT_EQ(primaryPeak.size(), PoolGroupIndex{1}); + ASSERT_EQ(secondaryPeak.size(), PoolGroupIndex{1}); + EXPECT_EQ(primaryPeak[PoolGroupIndex{0}].available, 2); + EXPECT_EQ(primaryPeak[PoolGroupIndex{0}].unavailable, 2); + EXPECT_EQ(primaryPeak[PoolGroupIndex{0}].evictable, 2); + EXPECT_EQ(secondaryPeak[PoolGroupIndex{0}].available, 2); + EXPECT_EQ(secondaryPeak[PoolGroupIndex{0}].unavailable, 1); + EXPECT_EQ(secondaryPeak[PoolGroupIndex{0}].evictable, 0); + + primaryPeak = manager->getAndResetIterationPeakBlockStats(kGpuLevel); + secondaryPeak = manager->getAndResetIterationPeakBlockStats(CacheLevel{1}); + EXPECT_EQ(primaryPeak[PoolGroupIndex{0}].available, 2); + EXPECT_EQ(primaryPeak[PoolGroupIndex{0}].unavailable, 0); + EXPECT_EQ(primaryPeak[PoolGroupIndex{0}].evictable, 0); + EXPECT_EQ(secondaryPeak[PoolGroupIndex{0}].available, 2); + EXPECT_EQ(secondaryPeak[PoolGroupIndex{0}].unavailable, 0); + EXPECT_EQ(secondaryPeak[PoolGroupIndex{0}].evictable, 0); + + auto nextIntervalSlots = storage.newSlots(kGpuLevel, oneSlot); + manager->commitStats({}); + storage.releaseSlot(lifeCycle, kGpuLevel, std::move(nextIntervalSlots[lifeCycle].front())); + primaryPeak = manager->getAndResetIterationPeakBlockStats(kGpuLevel); + EXPECT_EQ(primaryPeak[PoolGroupIndex{0}].available, 2); + EXPECT_EQ(primaryPeak[PoolGroupIndex{0}].unavailable, 1); + EXPECT_EQ(primaryPeak[PoolGroupIndex{0}].evictable, 0); +} + +TEST(KvCacheManagerV2StatsTest, MigrationAndLastTierDropRecordersReceiveExactPages) +{ + ASSERT_EQ(cudaSetDevice(0), cudaSuccess); + auto manager = std::make_shared<KvCacheManager>(makeTieredConfig()); + auto& storage = manager->storage(); + LifeCycleId const lifeCycle{0}; + ASSERT_EQ(storage.getStatistics(kGpuLevel).total, 2); + ASSERT_EQ(storage.getStatistics(CacheLevel{1}).total, 2); + + int offloaded = 0; + int onboarded = 0; + int dropped = 0; + MigrationRecorder const migrationRecorder + = [&](std::vector<SharedPtr<Page>> const& pages, std::vector<Slot> const& slots, CacheLevel srcLevel, + CacheLevel dstLevel) + { + EXPECT_EQ(pages.size(), slots.size()); + if (srcLevel == kGpuLevel && dstLevel == CacheLevel{1}) + { + offloaded += static_cast<int>(pages.size()); + } + else if (srcLevel == CacheLevel{1} && dstLevel == kGpuLevel) + { + onboarded += static_cast<int>(pages.size()); + } + }; + DropRecorder const dropRecorder = [&](std::vector<SharedPtr<Page>> const& pages, CacheLevel level) + { + EXPECT_EQ(level, CacheLevel{1}); + dropped += static_cast<int>(pages.size()); + }; + + RootBlock& root = manager->radixTree().addOrGetExisting({}); + int tokenBase = 0; + auto makeCommittedPages = [&](std::vector<Slot> slots) + { + std::vector<SharedPtr<Page>> pages; + NodeBase* previous = &root; + for (auto& slot : slots) + { + std::vector<TokenIdExt> tokens; + for (int i = 0; i < manager->tokensPerBlock(); ++i) + { + tokens.emplace_back(TokenId{tokenBase++}); + } + auto block = addOrGetExistingBlock(previous, LifeCycleId{1}, std::move(tokens)); + auto page = makeShared<CommittedPage>(&storage, block, lifeCycle, kGpuLevel, kPriorityDefault); + page->setSlot(slot); + block->storage[lifeCycle] = page.get(); + storage.scheduleForEviction(*page); + pages.push_back(page); + previous = block.get(); + } + return pages; + }; + + TypedVec<LifeCycleId, SlotCount> twoSlots(LifeCycleId{1}, 2); + auto initialSlots = storage.newGpuSlots(twoSlots); + auto firstPages = makeCommittedPages(std::move(initialSlots[lifeCycle])); + + auto temporarySlots = storage.newGpuSlots(twoSlots, migrationRecorder, dropRecorder); + EXPECT_EQ(offloaded, 2); + EXPECT_EQ(onboarded, 0); + EXPECT_EQ(dropped, 0); + for (auto& slot : temporarySlots[lifeCycle]) + { + storage.releaseSlot(lifeCycle, kGpuLevel, std::move(slot)); + } + + auto cache = manager->createKvCache(); + std::vector<BatchedLockTarget> targets; + for (BlockOrdinal ordinal{0}; ordinal < BlockOrdinal{2}; ++ordinal) + { + auto const& page = firstPages[toSizeT(ordinal)]; + ASSERT_TRUE(page->scheduledForEviction()); + storage.excludeFromEviction(*page); + targets.push_back({page, kDefaultBeamIndex, ordinal, lifeCycle}); + } + storage.batchedMigrateToGpu(targets, *cache, migrationRecorder); + EXPECT_EQ(onboarded, 2); + for (auto const& page : firstPages) + { + storage.scheduleForEviction(*page); + } + + temporarySlots = storage.newGpuSlots(twoSlots, migrationRecorder, dropRecorder); + EXPECT_EQ(offloaded, 4); + auto secondPages = makeCommittedPages(std::move(temporarySlots[lifeCycle])); + (void) secondPages; + firstPages.clear(); + targets.clear(); + + auto finalSlots = storage.newGpuSlots(twoSlots, migrationRecorder, dropRecorder); + EXPECT_EQ(offloaded, 6); + EXPECT_EQ(onboarded, 2); + EXPECT_EQ(dropped, 2); + for (auto& slot : finalSlots[lifeCycle]) + { + storage.releaseSlot(lifeCycle, kGpuLevel, std::move(slot)); + } + cache->close(); +} + +} // namespace diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheManagerV2TypedIndexTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheManagerV2TypedIndexTest.cpp new file mode 100644 index 000000000000..ac94dc063e72 --- /dev/null +++ b/cpp/tests/unit_tests/batch_manager/kvCacheManagerV2TypedIndexTest.cpp @@ -0,0 +1,333 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/common.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/lifeCycleRegistry.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/config.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/storage/core.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/storageManager.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/math.h" +#include "tensorrt_llm/batch_manager/kv_cache_manager_v2/utils/typedIndex.h" + +#include <gtest/gtest.h> + +#include <cstddef> +#include <cstdint> +#include <type_traits> +#include <utility> + +namespace +{ + +using namespace tensorrt_llm::batch_manager::kv_cache_manager_v2; + +template <typename Container, typename Index, typename = void> +struct HasIndexOperator : std::false_type +{ +}; + +template <typename Container, typename Index> +struct HasIndexOperator<Container, Index, std::void_t<decltype(std::declval<Container&>()[std::declval<Index>()])>> + : std::true_type +{ +}; + +template <typename Range, typename Index, typename = void> +struct HasContains : std::false_type +{ +}; + +template <typename Range, typename Index> +struct HasContains<Range, Index, std::void_t<decltype(std::declval<Range const&>().contains(std::declval<Index>()))>> + : std::true_type +{ +}; + +template <typename Lhs, typename Rhs, typename = void> +struct HasEqual : std::false_type +{ +}; + +template <typename Lhs, typename Rhs> +struct HasEqual<Lhs, Rhs, std::void_t<decltype(std::declval<Lhs>() == std::declval<Rhs>())>> : std::true_type +{ +}; + +template <typename Lhs, typename Rhs, typename = void> +struct HasNotEqual : std::false_type +{ +}; + +template <typename Lhs, typename Rhs> +struct HasNotEqual<Lhs, Rhs, std::void_t<decltype(std::declval<Lhs>() != std::declval<Rhs>())>> : std::true_type +{ +}; + +template <typename Lhs, typename Rhs, typename = void> +struct HasLess : std::false_type +{ +}; + +template <typename Lhs, typename Rhs> +struct HasLess<Lhs, Rhs, std::void_t<decltype(std::declval<Lhs>() < std::declval<Rhs>())>> : std::true_type +{ +}; + +template <typename Lhs, typename Rhs, typename = void> +struct HasGreater : std::false_type +{ +}; + +template <typename Lhs, typename Rhs> +struct HasGreater<Lhs, Rhs, std::void_t<decltype(std::declval<Lhs>() > std::declval<Rhs>())>> : std::true_type +{ +}; + +template <typename Lhs, typename Rhs, typename = void> +struct HasLessEqual : std::false_type +{ +}; + +template <typename Lhs, typename Rhs> +struct HasLessEqual<Lhs, Rhs, std::void_t<decltype(std::declval<Lhs>() <= std::declval<Rhs>())>> : std::true_type +{ +}; + +template <typename Lhs, typename Rhs, typename = void> +struct HasGreaterEqual : std::false_type +{ +}; + +template <typename Lhs, typename Rhs> +struct HasGreaterEqual<Lhs, Rhs, std::void_t<decltype(std::declval<Lhs>() >= std::declval<Rhs>())>> : std::true_type +{ +}; + +TEST(KvCacheManagerV2TypedIndexTest, StrongIndexSupportsIntegerArithmetic) +{ + LifeCycleId lc{3}; + + EXPECT_EQ((lc + 2).value(), 5); + EXPECT_EQ((2 + lc).value(), 5); + EXPECT_EQ((lc - 2).value(), 1); + EXPECT_EQ((lc + std::size_t{2}).value(), 5); + EXPECT_EQ((std::int64_t{2} + lc).value(), 5); + EXPECT_EQ((lc - std::int64_t{2}).value(), 1); + + LifeCycleId begin{4}; + LifeCycleId end{10}; + static_assert(std::is_same<decltype(end - begin), int>::value, "index difference must be an integer"); + EXPECT_EQ(end - begin, 6); + + lc += 4; + EXPECT_EQ(lc.value(), 7); + + lc -= 2; + EXPECT_EQ(lc.value(), 5); + + EXPECT_EQ((++lc).value(), 6); + EXPECT_EQ((lc++).value(), 6); + EXPECT_EQ(lc.value(), 7); + EXPECT_EQ((--lc).value(), 6); + EXPECT_EQ((lc--).value(), 6); + EXPECT_EQ(lc.value(), 5); +} + +TEST(KvCacheManagerV2TypedIndexTest, StrongIndexSupportsValueTypeUpperBoundComparison) +{ + static_assert(HasEqual<LifeCycleId, LifeCycleId>::value, "matching strong index equality must work"); + static_assert(HasNotEqual<LifeCycleId, LifeCycleId>::value, "matching strong index inequality must work"); + static_assert(HasLess<LifeCycleId, LifeCycleId>::value, "matching strong index ordering must work"); + static_assert(HasGreater<LifeCycleId, LifeCycleId>::value, "matching strong index ordering must work"); + static_assert(HasLessEqual<LifeCycleId, LifeCycleId>::value, "matching strong index ordering must work"); + static_assert(HasGreaterEqual<LifeCycleId, LifeCycleId>::value, "matching strong index ordering must work"); + + static_assert(HasLess<LifeCycleId, LifeCycleId::ValueType>::value, + "strong index must support upper-bound comparison against its value type"); + static_assert(HasLess<SlotId, SlotCount>::value, "slot id must support upper-bound comparison against slot counts"); + static_assert(HasGreaterEqual<LifeCycleId, LifeCycleId::ValueType>::value, + "strong index must support upper-bound rejection against its value type"); + static_assert( + HasGreaterEqual<SlotId, SlotCount>::value, "slot id must support upper-bound rejection against slot counts"); + + static_assert( + !HasEqual<LifeCycleId, LifeCycleId::ValueType>::value, "strong index must not compare equal to raw value type"); + static_assert( + !HasEqual<LifeCycleId::ValueType, LifeCycleId>::value, "raw value type must not compare equal to strong index"); + static_assert(!HasNotEqual<LifeCycleId, LifeCycleId::ValueType>::value, + "strong index must not compare unequal to raw value type"); + static_assert(!HasNotEqual<LifeCycleId::ValueType, LifeCycleId>::value, + "raw value type must not compare unequal to strong index"); + static_assert( + !HasLess<LifeCycleId::ValueType, LifeCycleId>::value, "raw value type must not order against strong index"); + static_assert(!HasGreater<LifeCycleId, LifeCycleId::ValueType>::value, + "strong index must not use raw value type for greater-than comparison"); + static_assert( + !HasGreater<LifeCycleId::ValueType, LifeCycleId>::value, "raw value type must not order against strong index"); + static_assert(!HasLessEqual<LifeCycleId, LifeCycleId::ValueType>::value, + "strong index must not use raw value type for less-equal comparison"); + static_assert(!HasLessEqual<LifeCycleId::ValueType, LifeCycleId>::value, + "raw value type must not order against strong index"); + static_assert(!HasGreaterEqual<LifeCycleId::ValueType, LifeCycleId>::value, + "raw value type must not order against strong index"); +} + +TEST(KvCacheManagerV2TypedIndexTest, StrongIndexDefaultsMatchSentinels) +{ + static_assert(CacheLevel{}.value() == kGpuLevel.value(), "CacheLevel default should name the GPU level"); + static_assert(BeamIndex{}.value() == kDefaultBeamIndex.value(), "BeamIndex default should name the default beam"); + static_assert(BlockOrdinal{}.value() == kBadBlockOrdinal.value(), "BlockOrdinal default should be invalid"); + static_assert(PageIndex{}.value() == kBadPageIndex.value(), "PageIndex default should be invalid"); +} + +TEST(KvCacheManagerV2TypedIndexTest, SlotCapacityAccessorsUsePlainSlotCount) +{ + static_assert(std::is_same<decltype(std::declval<SlotAllocator const&>().numSlots()), SlotCount>::value, + "SlotAllocator::numSlots must return a plain slot count"); + static_assert(std::is_same<decltype(std::declval<PoolGroupBase const&>().numSlots()), SlotCount>::value, + "PoolGroupBase::numSlots must return a plain slot count"); + static_assert( + std::is_same<decltype(std::declval<CacheLevelStorage const&>().numSlots(PoolGroupIndex{0})), SlotCount>::value, + "CacheLevelStorage::numSlots must return a plain slot count"); + static_assert( + std::is_same<decltype(std::declval<StorageManager const&>().numSlots(PoolGroupIndex{0})), SlotCount>::value, + "StorageManager::numSlots must return a plain slot count"); +} + +TEST(KvCacheManagerV2TypedIndexTest, SlotIdUsesInt64) +{ + static_assert(std::is_same<SlotId::ValueType, std::int64_t>::value, "SlotId must use int64_t"); + static_assert(std::is_same<SlotCount, std::int64_t>::value, "SlotCount must use int64_t"); +} + +TEST(KvCacheManagerV2TypedIndexTest, SlotCountsUsePlainSlotCount) +{ + static_assert(std::is_same<decltype(std::declval<SlotAllocator const&>().numFreeSlots()), SlotCount>::value, + "SlotAllocator::numFreeSlots must return a plain slot count"); + static_assert(std::is_same<decltype(std::declval<SlotAllocator const&>().numOccupiedSlots()), SlotCount>::value, + "SlotAllocator::numOccupiedSlots must return a plain slot count"); + static_assert(std::is_same<decltype(std::declval<SlotAllocator const&>().numOverflowSlots()), SlotCount>::value, + "SlotAllocator::numOverflowSlots must return a plain slot count"); + static_assert(std::is_same<decltype(std::declval<PoolGroupBase const&>().numFreeSlots()), SlotCount>::value, + "PoolGroupBase::numFreeSlots must return a plain slot count"); + static_assert(std::is_same<decltype(std::declval<CacheLevelStorage const&>().numFreeSlots(PoolGroupIndex{0})), + SlotCount>::value, + "CacheLevelStorage::numFreeSlots must return a plain slot count"); + static_assert(std::is_same<decltype(std::declval<PrioritizedEvictionPolicy const&>().size()), SlotCount>::value, + "PrioritizedEvictionPolicy::size must return a plain count"); + static_assert(std::is_same<decltype(std::declval<StorageStatistics const&>().total), SlotCount>::value, + "StorageStatistics::total must be a plain count"); + static_assert(std::is_same<decltype(std::declval<StorageStatistics const&>().available()), SlotCount>::value, + "StorageStatistics::available must return a plain count"); + static_assert(std::is_same<decltype(std::declval<StorageStatistics const&>().unavailable()), SlotCount>::value, + "StorageStatistics::unavailable must return a plain count"); +} + +TEST(KvCacheManagerV2TypedIndexTest, AllocationRequestsUsePlainSlotCount) +{ + using SlotAllocatorAllocateMultiple = std::vector<Slot> (SlotAllocator::*)(SlotCount); + static_assert(std::is_same<decltype(&SlotAllocator::allocateMultiple), SlotAllocatorAllocateMultiple>::value, + "SlotAllocator::allocateMultiple must take a plain slot count"); + static_assert(!std::is_invocable<decltype(&SlotAllocator::allocateMultiple), SlotAllocator&, SlotId>::value, + "SlotAllocator::allocateMultiple must not take a slot id"); + + using StorageManagerNewGpuSlots = TypedVec<LifeCycleId, std::vector<Slot>> (StorageManager::*)( + TypedVec<LifeCycleId, SlotCount> const&, MigrationRecorder const&, DropRecorder const&); + static_assert(std::is_same<decltype(&StorageManager::newGpuSlots), StorageManagerNewGpuSlots>::value, + "StorageManager::newGpuSlots must take plain slot counts"); + + using StorageManagerNewSlotsForPoolGroup = std::vector<Slot> (StorageManager::*)( + CacheLevel, PoolGroupIndex, SlotCount, MigrationRecorder const&, DropRecorder const&); + static_assert( + std::is_same<decltype(&StorageManager::newSlotsForPoolGroup), StorageManagerNewSlotsForPoolGroup>::value, + "StorageManager::newSlotsForPoolGroup must take a plain slot count"); + static_assert( + std::is_same<decltype(std::declval<PerLevelEvictionController const&>().numEvictablePages(PoolGroupIndex{0})), + SlotCount>::value, + "PerLevelEvictionController::numEvictablePages must return a plain count"); +} + +TEST(KvCacheManagerV2TypedIndexTest, SlotAllocatorRejectsOutOfRangeSlotId) +{ + SlotAllocator allocator(1); + Slot invalidSlot; + invalidSlot.setSlotId(SlotId{1}); + + EXPECT_THROW(allocator.release(std::move(invalidSlot)), LogicError); +} + +TEST(KvCacheManagerV2TypedIndexTest, HalfOpenRangeCarriesIndexType) +{ + using BlockRange = HalfOpenRange<BlockOrdinal>; + + static_assert(std::is_same<typename BlockRange::IndexType, BlockOrdinal>::value, + "range endpoints must carry the requested index type"); + static_assert(std::is_same<decltype(std::declval<BlockRange>().length()), int>::value, + "range length must remain an integer count"); + static_assert(HasContains<BlockRange, BlockOrdinal>::value, "matching index type must work"); + static_assert(!HasContains<BlockRange, LifeCycleId>::value, "different strong index must not work"); + static_assert(!HasContains<BlockRange, int>::value, "raw int must not be accepted by typed contains"); + + BlockRange empty; + EXPECT_EQ(empty.beg, BlockOrdinal{0}); + EXPECT_EQ(empty.end, BlockOrdinal{0}); + EXPECT_EQ(empty.length(), 0); + EXPECT_FALSE(empty); + + BlockRange range{2, 5}; + EXPECT_EQ(range.beg, BlockOrdinal{2}); + EXPECT_EQ(range.end, BlockOrdinal{5}); + EXPECT_EQ(range.length(), 3); + EXPECT_TRUE(range.contains(BlockOrdinal{2})); + EXPECT_FALSE(range.contains(BlockOrdinal{5})); + + BlockRange overlap = intersect(range, BlockRange{4, 8}); + EXPECT_EQ(overlap.beg, BlockOrdinal{4}); + EXPECT_EQ(overlap.end, BlockOrdinal{5}); +} + +TEST(KvCacheManagerV2TypedIndexTest, TypedVecRequiresMatchingIndexType) +{ + using LifeCycleVector = TypedVec<LifeCycleId, int>; + + static_assert(HasIndexOperator<LifeCycleVector, LifeCycleId>::value, "matching index type must work"); + static_assert(!HasIndexOperator<LifeCycleVector, PoolGroupIndex>::value, "different strong index must not work"); + static_assert(!HasIndexOperator<LifeCycleVector, int>::value, "raw int must not index typed vector"); + + LifeCycleVector values(LifeCycleId{2}, 0); + values[LifeCycleId{0}] = 11; + values[LifeCycleId{1}] = 17; + + EXPECT_EQ(values.size(), LifeCycleId{2}); + EXPECT_EQ(values[LifeCycleId{0}], 11); + EXPECT_EQ(values.at(LifeCycleId{1}), 17); +} + +TEST(KvCacheManagerV2TypedIndexTest, TypedVecSupportsVectorInterop) +{ + TypedVec<PoolGroupIndex, int> values; + values.reserve(PoolGroupIndex{2}); + values.push_back(3); + values.push_back(5); + + EXPECT_EQ(values.stdSize(), 2); + EXPECT_EQ(values.size(), PoolGroupIndex{2}); + EXPECT_EQ(values.raw().at(0), 3); + EXPECT_EQ(values[PoolGroupIndex{1}], 5); +} + +} // namespace diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheUtilsTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheUtilsTest.cpp index 2047a885d98c..a7b15e5ae8e0 100644 --- a/cpp/tests/unit_tests/batch_manager/kvCacheUtilsTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/kvCacheUtilsTest.cpp @@ -21,6 +21,7 @@ #include <gtest/gtest.h> #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" namespace tc = tensorrt_llm::common; namespace tr = tensorrt_llm::runtime; @@ -48,7 +49,7 @@ TEST_F(BlockIteratorTest, BasicTest) auto constexpr mNumLayers = 5; auto constexpr mBlockSize = 32; auto const cacheShape = tr::ITensor::makeShape({mNumPrimaryBlocks, mNumLayers, 2, mBlockSize}); - constexpr nvinfer1::DataType dtype{tr::TRTDataType<DataType>::value}; + constexpr tensorrt_llm::DataType dtype{tr::TRTDataType<DataType>::value}; tr::ITensor::SharedPtr pool = tr::BufferManager::cpu(cacheShape, dtype); std::vector<SizeType32> blockIds(mNumPrimaryBlocks); std::iota(blockIds.begin(), blockIds.end(), 0); @@ -75,7 +76,7 @@ TEST_F(BlockIteratorTest, BasicTest) TEST_F(BlockIteratorTest, CacheManagerTest) { - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; auto constexpr numLayers = 12; auto constexpr numKvHeads = 6; auto constexpr sizePerHead = 16; diff --git a/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp b/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp index 120263b01af1..4a897beec48d 100644 --- a/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +16,7 @@ */ #include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/executor/types.h" @@ -97,7 +98,7 @@ TEST_F(LlmRequestTest, fromExecutorRequest) EXPECT_TRUE(llmReq.getStopWordsList().has_value()); { auto badWordsTensor = llmReq.getBadWordsList().value(); - EXPECT_EQ(badWordsTensor->getDataType(), nvinfer1::DataType::kINT32); + EXPECT_EQ(badWordsTensor->getDataType(), tensorrt_llm::DataType::kINT32); EXPECT_EQ(badWordsTensor->getShape().nbDims, 3); EXPECT_EQ(badWordsTensor->getShape().d[0], 1); EXPECT_EQ(badWordsTensor->getShape().d[1], 2); @@ -119,7 +120,7 @@ TEST_F(LlmRequestTest, fromExecutorRequest) { auto stopWordsTensor = llmReq.getStopWordsList().value(); - EXPECT_EQ(stopWordsTensor->getDataType(), nvinfer1::DataType::kINT32); + EXPECT_EQ(stopWordsTensor->getDataType(), tensorrt_llm::DataType::kINT32); EXPECT_EQ(stopWordsTensor->getShape().nbDims, 3); EXPECT_EQ(stopWordsTensor->getShape().d[0], 1); EXPECT_EQ(stopWordsTensor->getShape().d[1], 2); @@ -151,7 +152,7 @@ TEST_F(LlmRequestTest, fromExecutorRequest) EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getShape().d[0], 1); EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getShape().d[1], vocabSize); EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getShape().d[2], hiddenSize); - EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getDataType(), nvinfer1::DataType::kFLOAT); + EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getDataType(), tensorrt_llm::DataType::kFLOAT); EXPECT_EQ(llmReq.getPromptVocabSize().value(), vocabSize); VecUniqueTokens uniqueTokens; for (size_t i = 0; i < inputTokens.size(); ++i) @@ -373,7 +374,7 @@ TEST_F(LlmRequestTest, testAllocateLogitsBuffer) EXPECT_EQ(llmReq.mPromptLen, 5); SizeType32 vocabSizePadded = 32000; - nvinfer1::DataType logitsDataType = nvinfer1::DataType::kFLOAT; + tensorrt_llm::DataType logitsDataType = tensorrt_llm::DataType::kFLOAT; // Test the allocation of context logits EXPECT_EQ(llmReq.getContextLogitsHost(), nullptr); @@ -462,7 +463,7 @@ TEST_F(LlmRequestTest, testCreateRequests) SizeType32 maxNewTokens{60}; tb::LlmRequest::RequestIdType requestId{77}; SizeType32 vocabSize{32}; - nvinfer1::DataType dtype{nvinfer1::DataType::kHALF}; + tensorrt_llm::DataType dtype{tensorrt_llm::DataType::kHALF}; tr::SamplingConfig samplingConfig(1); samplingConfig.randomSeed = std::vector<texec::RandomSeedType>{7}; @@ -789,6 +790,41 @@ TEST_F(LlmRequestTest, createResultDisaggContextComplete) EXPECT_TRUE(response->isSequenceFinal); } +TEST_F(LlmRequestTest, generationOnlyRequestAdoptsContextPhaseDraftTokens) +{ + VecTokens inputTokens{1, 2, 3, 4, 5}; + VecTokens firstGenTokens{100}; + VecTokens draftTokens{101, 102, 103}; + SizeType32 maxNewTokens{10}; + texec::IdType requestId{42}; + + texec::Request execReq(inputTokens, maxNewTokens); + execReq.setRequestType(texec::RequestType::REQUEST_TYPE_GENERATION_ONLY); + execReq.setContextPhaseParams( + texec::ContextPhaseParams{firstGenTokens, requestId, static_cast<void*>(nullptr), draftTokens}); + auto const expectedContextPhaseTokens = static_cast<SizeType32>(firstGenTokens.size() + draftTokens.size()); + auto const expectedDraftTokens = static_cast<SizeType32>(draftTokens.size()); + + tb::LlmRequest llmReq(requestId, execReq); + + EXPECT_TRUE(llmReq.isGenerationOnlyRequest()); + EXPECT_EQ(llmReq.getNumDraftTokens(), expectedDraftTokens); + EXPECT_EQ(*llmReq.getDraftTokens(), draftTokens); + EXPECT_EQ(llmReq.getNumContextPhaseGenerationTokens(), expectedContextPhaseTokens); + + texec::Request lateExecReq(inputTokens, maxNewTokens); + lateExecReq.setRequestType(texec::RequestType::REQUEST_TYPE_GENERATION_ONLY); + tb::LlmRequest lateLlmReq(requestId, lateExecReq); + EXPECT_EQ(lateLlmReq.getNumDraftTokens(), 0); + + lateLlmReq.setContextPhaseParams( + texec::ContextPhaseParams{firstGenTokens, requestId, static_cast<void*>(nullptr), draftTokens}); + + EXPECT_EQ(lateLlmReq.getNumDraftTokens(), expectedDraftTokens); + EXPECT_EQ(*lateLlmReq.getDraftTokens(), draftTokens); + EXPECT_EQ(lateLlmReq.getNumContextPhaseGenerationTokens(), expectedContextPhaseTokens); +} + INSTANTIATE_TEST_SUITE_P(LlmRequestTest, ParamTest, testing::Combine( // TODO: Support and add coverage for streamLLM diff --git a/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp b/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp index c2c308b37dd9..c392b05dccf7 100644 --- a/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp @@ -23,6 +23,7 @@ #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/batch_manager/microBatchScheduler.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/testing/kvCacheManagerTestUtil.h" #include <numeric> @@ -59,7 +60,7 @@ class MicroBatchSchedulerTest : public ::testing::Test // NOLINT(cppcoreguidelin { draftTokens = std::make_shared<std::vector<int32_t>>(draftTokensLen, 2); draftLogits = BufferManager::cpu( - ITensor::makeShape({draftTokensLen, /* vocabSizePadded*/ 42}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({draftTokensLen, /* vocabSizePadded*/ 42}), tensorrt_llm::DataType::kFLOAT); } return std::make_shared<LlmRequest>(reqId, maxNewTokens, inputTokens, samplingConfig, /*isStreaming=*/false, @@ -1246,7 +1247,7 @@ class CombinedSchedulerTest : public ::testing::Test return std::make_shared<kv_cache_manager::KVCacheManager>( /*numLayers=*/10, /*nbKvHeads=*/10, /*sizePerHead=*/1, tokensPerBlock, blocksPerWindow, maxNumRequests, - /*maxBeamWidth=*/1, std::vector<SizeType32>{maxNumTokensPerSeq}, nvinfer1::DataType::kHALF, + /*maxBeamWidth=*/1, std::vector<SizeType32>{maxNumTokensPerSeq}, tensorrt_llm::DataType::kHALF, /*sinkTokenLength=*/0, stream, maxNumTokensPerSeq, /*chunkSize=*/maxNumTokensPerSeq, enableReuse); } @@ -2053,19 +2054,19 @@ class ForceChunkTest : public MicroBatchSchedulerTest } }; -TEST_F(ForceChunkTest, Basic) +TEST_F(ForceChunkTest, NoSnapshotPointsUsesRemainingContext) { - // A single request with prompt_len > chunk_unit_size is chunked to unit_size. + // A request without snapshot points is not split at chunkUnitSize. auto reqs = initRequests({30}); MicroBatchScheduler::setCtxRequestsChunkSize(reqs, Policy::kFORCE_CHUNK, /*ctxTokensCapacity=*/std::nullopt, /*chunkUnitSize=*/10, /*maxContextLength=*/std::nullopt); - EXPECT_EQ(reqs[0]->getContextChunkSize(), 10); + EXPECT_EQ(reqs[0]->getContextChunkSize(), 30); } TEST_F(ForceChunkTest, PromptSmallerThanUnit) { - // When prompt_len < chunk_unit_size, chunk_size = prompt_len (min). + // Without snapshot points, a short prompt is consumed in full. auto reqs = initRequests({8}); MicroBatchScheduler::setCtxRequestsChunkSize(reqs, Policy::kFORCE_CHUNK, std::nullopt, 20, std::nullopt); @@ -2074,7 +2075,7 @@ TEST_F(ForceChunkTest, PromptSmallerThanUnit) TEST_F(ForceChunkTest, ExactUnitSize) { - // When prompt_len == chunk_unit_size, chunk_size = prompt_len. + // Without snapshot points, an exact-unit prompt is consumed in full. auto reqs = initRequests({10}); MicroBatchScheduler::setCtxRequestsChunkSize(reqs, Policy::kFORCE_CHUNK, std::nullopt, 10, std::nullopt); @@ -2083,31 +2084,36 @@ TEST_F(ForceChunkTest, ExactUnitSize) TEST_F(ForceChunkTest, MultipleRequests) { - // Each request independently gets min(remaining, unit_size). + // Requests without snapshot points independently consume their remaining contexts. auto reqs = initRequests({25, 15, 5}); MicroBatchScheduler::setCtxRequestsChunkSize(reqs, Policy::kFORCE_CHUNK, std::nullopt, 10, std::nullopt); - EXPECT_EQ(reqs[0]->getContextChunkSize(), 10); - EXPECT_EQ(reqs[1]->getContextChunkSize(), 10); - EXPECT_EQ(reqs[2]->getContextChunkSize(), 5); // min(5, 10) = 5 + EXPECT_EQ(reqs[0]->getContextChunkSize(), 25); + EXPECT_EQ(reqs[1]->getContextChunkSize(), 15); + EXPECT_EQ(reqs[2]->getContextChunkSize(), 5); } TEST_F(ForceChunkTest, CapacityLimits) { - // When capacity is limited, later requests get chunk_size=0. + // Budget truncation is unit-aligned; later requests with less than one + // unit available are delayed. auto reqs = initRequests({30, 30}); MicroBatchScheduler::setCtxRequestsChunkSize( reqs, Policy::kFORCE_CHUNK, /*ctxTokensCapacity=*/15, /*chunkUnitSize=*/10, std::nullopt); - // req0 gets 10, req1 would push total to 20 > 15 → 0 + // req0 is budget-truncated to 10; only 5 remain, so req1 gets 0. EXPECT_EQ(reqs[0]->getContextChunkSize(), 10); EXPECT_EQ(reqs[1]->getContextChunkSize(), 0); } TEST_F(ForceChunkTest, CapacityExactFit) { - // When capacity exactly accommodates all chunks. + // Capacity exactly accommodates both requested snapshot chunks. auto reqs = initRequests({30, 30}); + for (auto const& req : reqs) + { + req->setExpectedSnapshotPoints({10}); + } MicroBatchScheduler::setCtxRequestsChunkSize( reqs, Policy::kFORCE_CHUNK, /*ctxTokensCapacity=*/20, /*chunkUnitSize=*/10, std::nullopt); @@ -2115,11 +2121,38 @@ TEST_F(ForceChunkTest, CapacityExactFit) EXPECT_EQ(reqs[1]->getContextChunkSize(), 10); } +TEST_F(ForceChunkTest, ExpectedChunkingPoints) +{ + // Expected snapshot points are absolute context positions. + auto reqs = initRequests({30}); + reqs[0]->setExpectedSnapshotPoints({12, 25}); + + chunkIteration(reqs, 10); + expectPositions(reqs, {12}, "iter 1"); + + chunkIteration(reqs, 10); + expectPositions(reqs, {25}, "iter 2"); + + chunkIteration(reqs, 10); + expectPositions(reqs, {30}, "iter 3"); +} + +TEST_F(ForceChunkTest, CapacityRoundsExpectedChunkDownToUnit) +{ + auto reqs = initRequests({50}); + reqs[0]->setExpectedSnapshotPoints({30}); + + MicroBatchScheduler::setCtxRequestsChunkSize( + reqs, Policy::kFORCE_CHUNK, /*ctxTokensCapacity=*/25, /*chunkUnitSize=*/10, std::nullopt); + + EXPECT_EQ(reqs[0]->getContextChunkSize(), 20); +} + TEST_F(ForceChunkTest, MultiIteration) { - // A request with prompt_len=25 and chunk_unit_size=10 processes in 3 iterations: - // chunk 1: 10, chunk 2: 10, chunk 3: 5. + // Snapshot points at 10 and 20 split a 25-token prompt into three iterations. auto reqs = initRequests({25}); + reqs[0]->setExpectedSnapshotPoints({10, 20}); // Iteration 1 chunkIteration(reqs, 10); @@ -2137,8 +2170,10 @@ TEST_F(ForceChunkTest, MultiIteration) TEST_F(ForceChunkTest, MultiRequestMultiIteration) { // Two requests with different lengths processed over multiple iterations. - // prompt_len={25, 12}, chunk_unit_size=10. + // Their expected snapshot points determine each boundary. auto reqs = initRequests({25, 12}); + reqs[0]->setExpectedSnapshotPoints({10, 20}); + reqs[1]->setExpectedSnapshotPoints({10}); // Iteration 1: both get 10 chunkIteration(reqs, 10); @@ -2156,8 +2191,12 @@ TEST_F(ForceChunkTest, MultiRequestMultiIteration) TEST_F(ForceChunkTest, CapacityAcrossIterations) { // With limited capacity, some requests may be delayed to later iterations. - // prompt_len={25, 25}, chunk_unit_size=10, capacity=15. + // Both requests have snapshot points at 10 and 20; capacity is 15. auto reqs = initRequests({25, 25}); + for (auto const& req : reqs) + { + req->setExpectedSnapshotPoints({10, 20}); + } // Iteration 1: req0=10, req1=0 (10+10=20 > 15) chunkIteration(reqs, 10, /*ctxTokensCapacity=*/15); @@ -2180,10 +2219,10 @@ TEST_F(ForceChunkTest, CapacityAcrossIterations) expectPositions(reqs, {25, 25}, "iter 5"); } -TEST_F(ForceChunkTest, FullSchedulerPath) +TEST_F(ForceChunkTest, FullSchedulerWithoutSnapshotPoints) { - // Test via MicroBatchScheduler::operator() — FORCE_CHUNK always re-chunks - // even when all contexts fit within the token budget. + // Test via MicroBatchScheduler::operator(): without snapshot points, a + // context that fits is not split at chunkUnitSize. batch_scheduler::ContextChunkingConfig chunkConfig; chunkConfig.chunkingPolicy = Policy::kFORCE_CHUNK; chunkConfig.chunkUnitSize = 10; @@ -2200,9 +2239,33 @@ TEST_F(ForceChunkTest, FullSchedulerPath) auto const [contextRequests, genRequests] = (*scheduler)(activeRequests, inflightReqIds, maxBatchSize, maxNumTokens); - // Despite budget=100 >> prompt=30, FORCE_CHUNK limits chunk to unit_size=10. ASSERT_EQ(contextRequests.size(), 1); - EXPECT_EQ(contextRequests[0]->getContextChunkSize(), 10); + EXPECT_EQ(contextRequests[0]->getContextChunkSize(), 30); + EXPECT_EQ(genRequests.size(), 0); +} + +TEST_F(ForceChunkTest, FullSchedulerUsesExpectedChunkingPoints) +{ + batch_scheduler::ContextChunkingConfig chunkConfig; + chunkConfig.chunkingPolicy = Policy::kFORCE_CHUNK; + chunkConfig.chunkUnitSize = 10; + + auto scheduler = std::make_shared<MicroBatchScheduler>(chunkConfig); + + constexpr SizeType32 maxBatchSize = 4; + constexpr SizeType32 maxNumTokens = 100; + + RequestVector activeRequests; + auto request = createRequest(/*promptLen=*/30, /*maxNewTokens=*/1, /*reqId=*/0); + request->setExpectedSnapshotPoints({12, 25}); + activeRequests.push_back(request); + + ReqIdsSet inflightReqIds; + auto const [contextRequests, genRequests] + = (*scheduler)(activeRequests, inflightReqIds, maxBatchSize, maxNumTokens); + + ASSERT_EQ(contextRequests.size(), 1); + EXPECT_EQ(contextRequests[0]->getContextChunkSize(), 12); EXPECT_EQ(genRequests.size(), 0); } @@ -2234,8 +2297,8 @@ TEST_F(ForceChunkTest, FullSchedulerMultipleRequests) { chunks[req->mRequestId] = req->getContextChunkSize(); } - EXPECT_EQ(chunks[0], 10); - EXPECT_EQ(chunks[1], 10); + EXPECT_EQ(chunks[0], 25); + EXPECT_EQ(chunks[1], 15); EXPECT_EQ(chunks[2], 5); } @@ -2267,6 +2330,7 @@ TEST_F(ForceChunkTest, FullSchedulerWithGeneration) EXPECT_EQ(genRequests.size(), 1); ASSERT_EQ(contextRequests.size(), 1); - // Budget remaining = 15 - 1 (gen) = 14; chunk = min(30, 10) = 10 + // Budget remaining is 14, so the context is rounded down to one + // 10-token chunk-unit boundary. EXPECT_EQ(contextRequests[0]->getContextChunkSize(), 10); } diff --git a/cpp/tests/unit_tests/batch_manager/peftCacheManagerTest.cpp b/cpp/tests/unit_tests/batch_manager/peftCacheManagerTest.cpp index 49adfe5a6cb6..ef513894bfb6 100644 --- a/cpp/tests/unit_tests/batch_manager/peftCacheManagerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/peftCacheManagerTest.cpp @@ -32,7 +32,7 @@ #include "tensorrt_llm/runtime/utils/numpyUtils.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_runtime.h> #include <gmock/gmock-matchers.h> @@ -78,7 +78,7 @@ class PeftCacheManagerTest : public ::testing::Test // NOLINT(cppcoreguidelines- void SetUp() override { - mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); + mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); mModelConfig->setMlpHiddenSize(32); mWorldConfig = std::make_unique<WorldConfig>(2, 1, 1, 0); std::vector<LoraModule> modules{ @@ -285,7 +285,7 @@ TEST_F(PeftCacheManagerTest, gptManagerSim) auto peftManager = std::make_unique<PeftCacheManager>(config, *mModelConfig, *mWorldConfig, *mManager); auto pageConfig = LoraCachePageManagerConfig( - runtime::MemoryType::kCPU, nvinfer1::DataType::kFLOAT, 128, 128, 2 * 8 * 64, 4 * 16, 1); + runtime::MemoryType::kCPU, tensorrt_llm::DataType::kFLOAT, 128, 128, 2 * 8 * 64, 4 * 16, 1); auto loraCache = std::make_unique<LoraCache>(pageConfig, *mModelConfig, *mWorldConfig, *mManager); std::map<uint64_t, std::pair<TensorPtr, TensorPtr>> loras; @@ -505,7 +505,7 @@ TEST_F(PeftCacheManagerTest, getMaxNumSlots) config.numHostModuleLayer = 8192 * 8; config.numDeviceModuleLayer = 8292 * 2; auto [hostSlots, deviceSlots] - = PeftCacheManager::getMaxNumSlots(config, nvinfer1::DataType::kHALF, 256, 4 * 256, *mManager); + = PeftCacheManager::getMaxNumSlots(config, tensorrt_llm::DataType::kHALF, 256, 4 * 256, *mManager); EXPECT_EQ(262144, hostSlots); EXPECT_EQ(66336, deviceSlots); @@ -516,13 +516,13 @@ TEST_F(PeftCacheManagerTest, getMaxNumSlots) config.maxPagesPerBlockDevice = 8; std::tie(hostSlots, deviceSlots) - = PeftCacheManager::getMaxNumSlots(config, nvinfer1::DataType::kHALF, 256, 4 * 256, *mManager); + = PeftCacheManager::getMaxNumSlots(config, tensorrt_llm::DataType::kHALF, 256, 4 * 256, *mManager); EXPECT_EQ(195, hostSlots); EXPECT_EQ(66336, deviceSlots); std::tie(hostSlots, deviceSlots) - = PeftCacheManager::getMaxNumSlots(config, nvinfer1::DataType::kFLOAT, 384, 4 * 1024, *mManager); + = PeftCacheManager::getMaxNumSlots(config, tensorrt_llm::DataType::kFLOAT, 384, 4 * 1024, *mManager); config.hostCacheSize = 100000000; config.numHostModuleLayer = 8291 * 2; @@ -539,7 +539,7 @@ TEST_F(PeftCacheManagerTest, getPageManagerConfig) auto [hostCfg, deviceCfg] = PeftCacheManager::getPageManagerConfig(config, *mModelConfig, *mWorldConfig, *mManager); EXPECT_EQ(runtime::MemoryType::kCPU, hostCfg.getMemoryType()); - EXPECT_EQ(nvinfer1::DataType::kFLOAT, hostCfg.getDataType()); + EXPECT_EQ(tensorrt_llm::DataType::kFLOAT, hostCfg.getDataType()); EXPECT_EQ(456, hostCfg.getTotalNumPages()); EXPECT_EQ(24, hostCfg.getMaxPagesPerBlock()); EXPECT_EQ(288, hostCfg.getSlotsPerPage()); @@ -547,7 +547,7 @@ TEST_F(PeftCacheManagerTest, getPageManagerConfig) EXPECT_FALSE(hostCfg.getInitToZero()); EXPECT_EQ(runtime::MemoryType::kGPU, deviceCfg.getMemoryType()); - EXPECT_EQ(nvinfer1::DataType::kFLOAT, deviceCfg.getDataType()); + EXPECT_EQ(tensorrt_llm::DataType::kFLOAT, deviceCfg.getDataType()); EXPECT_EQ(116, deviceCfg.getTotalNumPages()); EXPECT_EQ(8, deviceCfg.getMaxPagesPerBlock()); EXPECT_EQ(288, deviceCfg.getSlotsPerPage()); @@ -563,7 +563,7 @@ TEST_F(PeftCacheManagerTest, getPageManagerConfig) = PeftCacheManager::getPageManagerConfig(config, *mModelConfig, *mWorldConfig, *mManager); EXPECT_EQ(runtime::MemoryType::kCPU, hostCfg.getMemoryType()); - EXPECT_EQ(nvinfer1::DataType::kFLOAT, hostCfg.getDataType()); + EXPECT_EQ(tensorrt_llm::DataType::kFLOAT, hostCfg.getDataType()); EXPECT_EQ(3617, hostCfg.getTotalNumPages()); EXPECT_EQ(4, hostCfg.getMaxPagesPerBlock()); EXPECT_EQ(288, hostCfg.getSlotsPerPage()); @@ -571,7 +571,7 @@ TEST_F(PeftCacheManagerTest, getPageManagerConfig) EXPECT_FALSE(hostCfg.getInitToZero()); EXPECT_EQ(runtime::MemoryType::kGPU, deviceCfg.getMemoryType()); - EXPECT_EQ(nvinfer1::DataType::kFLOAT, deviceCfg.getDataType()); + EXPECT_EQ(tensorrt_llm::DataType::kFLOAT, deviceCfg.getDataType()); EXPECT_EQ(116, deviceCfg.getTotalNumPages()); EXPECT_EQ(8, deviceCfg.getMaxPagesPerBlock()); EXPECT_EQ(288, deviceCfg.getSlotsPerPage()); @@ -586,7 +586,7 @@ class PeftCacheManagerPrefetchTest : public ::testing::Test // NOLINT(cppcoregui void SetUp() override { - mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); + mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); mModelConfig->setMlpHiddenSize(32); mWorldConfig = std::make_unique<WorldConfig>(2, 1, 1, 0); std::vector<LoraModule> modules{ diff --git a/cpp/tests/unit_tests/batch_manager/rnnCacheFormatterTest.cpp b/cpp/tests/unit_tests/batch_manager/rnnCacheFormatterTest.cpp index 8840130df1ea..7dc4cea6e9af 100644 --- a/cpp/tests/unit_tests/batch_manager/rnnCacheFormatterTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/rnnCacheFormatterTest.cpp @@ -7,6 +7,7 @@ #include "tensorrt_llm/batch_manager/cacheFormatter.h" #include "tensorrt_llm/batch_manager/rnnCacheFormatter.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include <random> @@ -27,12 +28,12 @@ class RnnTargetIRanksTest : public ::testing::Test std::vector<SizeType32> kvLayersPerPP(pp, 0); // No attention layers auto state = texec::kv_cache::CacheState( /*nbAttentionLayers=*/0, /*nbKvHeads=*/1, /*sizePerHead=*/64, /*tokensPerBlock=*/32, tp, pp, - /*contextParallelism=*/1, kvLayersPerPP, nvinfer1::DataType::kFLOAT); + /*contextParallelism=*/1, kvLayersPerPP, tensorrt_llm::DataType::kFLOAT); texec::kv_cache::CacheState::RnnModelConfig rnnModelConfig{/*mDState=*/16, /*mDConv=*/4, /*mHiddenSize=*/256, /*mHeadDim=*/64, /*mConvDimSize=*/128, /*mNGroups=*/1, /*mNumLayers=*/numLayers, /*mNumHeads=*/4}; - state.setRnnConfig(rnnModelConfig, layersPerPP, nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kFLOAT); + state.setRnnConfig(rnnModelConfig, layersPerPP, tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kFLOAT); return state; } }; @@ -216,7 +217,7 @@ class HybridModelCounterpartsTest : public ::testing::Test SizeType32 tokensPerBlock = 32) { return texec::kv_cache::CacheState(numLayers, numHeads, sizePerHead, tokensPerBlock, tp, pp, - /*contextParallelism=*/1, layersPerPP, nvinfer1::DataType::kFLOAT, + /*contextParallelism=*/1, layersPerPP, tensorrt_llm::DataType::kFLOAT, texec::kv_cache::CacheState::AttentionType::kDEFAULT, /*kvFactor=*/2, /*enableAttentionDP=*/false, /*DPrank=*/0, /*DPsize=*/1); } @@ -228,7 +229,8 @@ class HybridModelCounterpartsTest : public ::testing::Test { auto state = makeKvCacheState(kvNumLayers, tp, pp, kvLayersPerPP, numHeads, sizePerHead, tokensPerBlock); texec::kv_cache::CacheState::RnnModelConfig rnnModelConfig{16, 4, 256, 64, 128, 1, rnnNumLayers, 4}; - state.setRnnConfig(rnnModelConfig, rnnLayersPerPP, nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kFLOAT); + state.setRnnConfig( + rnnModelConfig, rnnLayersPerPP, tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kFLOAT); return state; } @@ -471,7 +473,7 @@ class AttentionOnlyModelTest : public ::testing::Test SizeType32 tokensPerBlock = 32) { return texec::kv_cache::CacheState(numLayers, numHeads, sizePerHead, tokensPerBlock, tp, pp, - /*contextParallelism=*/1, layersPerPP, nvinfer1::DataType::kFLOAT, + /*contextParallelism=*/1, layersPerPP, tensorrt_llm::DataType::kFLOAT, texec::kv_cache::CacheState::AttentionType::kDEFAULT, /*kvFactor=*/2, /*enableAttentionDP=*/false, /*DPrank=*/0, /*DPsize=*/1); } diff --git a/cpp/tests/unit_tests/batch_manager/truncateBlocksTest.cpp b/cpp/tests/unit_tests/batch_manager/truncateBlocksTest.cpp index d0dce8eb71c5..85d4ac112244 100644 --- a/cpp/tests/unit_tests/batch_manager/truncateBlocksTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/truncateBlocksTest.cpp @@ -13,6 +13,7 @@ #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/samplingConfig.h" #include "tensorrt_llm/testing/kvCacheManagerTestUtil.h" @@ -89,7 +90,7 @@ TEST_F(TruncateBlocksTest, MultiTurnConversationTruncation) // Create KVCacheManager with block reuse enabled KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength /* chunkSize */, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); @@ -219,7 +220,7 @@ TEST_F(TruncateBlocksTest, SharedPrefixTruncation) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength /* chunkSize */, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); @@ -317,7 +318,7 @@ TEST_F(TruncateBlocksTest, CompleteTruncation) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength /* chunkSize */, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); @@ -377,7 +378,7 @@ TEST_F(TruncateBlocksTest, NonExistentTokensTruncation) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength /* chunkSize */, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); @@ -453,7 +454,7 @@ TEST_F(TruncateBlocksTest, ComplexMultiTurnConversationTruncation) // Create KVCacheManager with block reuse enabled KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength /* chunkSize */, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); diff --git a/cpp/tests/unit_tests/common/loggerTest.cpp b/cpp/tests/unit_tests/common/loggerTest.cpp index 8aaea6863fb9..6fa28a69b5f8 100644 --- a/cpp/tests/unit_tests/common/loggerTest.cpp +++ b/cpp/tests/unit_tests/common/loggerTest.cpp @@ -26,7 +26,7 @@ using namespace tensorrt_llm::common; TEST(LoggerModuleTest, FormatModuleNoTrailingSpaces) { for (auto const* raw : {"batch_manager", "common", "cutlass_extensions", "deep_ep", "deep_gemm", "executor", - "executor_worker", "flash_mla", "kernels", "layers", "nanobind", "plugins", "runtime", "testing", "thop"}) + "flash_mla", "kernels", "layers", "nanobind", "runtime", "testing", "thop"}) { auto const fmt = formatModule(raw); EXPECT_FALSE(fmt.empty()); diff --git a/cpp/tests/unit_tests/executor/CMakeLists.txt b/cpp/tests/unit_tests/executor/CMakeLists.txt index a51baa6ed00f..6d851ebd4544 100644 --- a/cpp/tests/unit_tests/executor/CMakeLists.txt +++ b/cpp/tests/unit_tests/executor/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & +# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & # AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -19,24 +19,15 @@ add_gtest(decodingConfigTest decodingConfigTest.cpp) add_gtest(requestTest requestTest.cpp) add_gtest(responseTest responseTest.cpp) -add_gtest(executorTestSmall executorTestSmall.cpp) -target_link_libraries(executorTestSmall PRIVATE testingUtils) - -add_gtest(executorTestSmallArbitraryOutputTensors - executorTestSmallArbitraryOutputTensors.cpp) -target_link_libraries(executorTestSmallArbitraryOutputTensors - PRIVATE testingUtils) - add_gtest(executorConfigTest executorConfigTest.cpp) add_gtest(executorTensorTest tensorTest.cpp) add_gtest(serializeUtilsTest serializeUtilsTest.cpp) add_gtest(requestWithIdTest requestWithIdTest.cpp) add_gtest(loraConfigTest loraConfigTest.cpp) -add_gtest(intervalSetTest intervalSetTest.cpp) -add_gtest(dynamicBatchTunerTest dynamicBatchTunerTest.cpp) +add_gtest(coalesceTest coalesceTest.cpp) add_gtest(genUniqueAgentNameTest genUniqueAgentNameTest.cpp) target_link_libraries(genUniqueAgentNameTest PRIVATE ${Python3_LIBRARIES}) -add_gtest(ucxCommTest ucxCommTest.cpp) +add_gtest(ucxCommTest ucxCommTest.cpp TIMEOUT 60) target_link_libraries(ucxCommTest PRIVATE ${Python3_LIBRARIES}) target_link_libraries(serializeUtilsTest PRIVATE ${Python3_LIBRARIES}) @@ -56,10 +47,6 @@ if(NIXL_ROOT OR (MOONCAKE_ROOT AND NOT IS_ROCKY8)) ${Python3_LIBRARIES}) target_compile_definitions(transferAgentTest PRIVATE TEST_NIXL_BACKEND=1) target_compile_definitions(agentCommTest PRIVATE TEST_NIXL_BACKEND=1) - - add_gtest(coalesceTest coalesceTest.cpp) - target_link_libraries(coalesceTest PRIVATE tensorrt_llm_nixl_wrapper - NIXL::nixl) endif() if(MOONCAKE_ROOT) diff --git a/cpp/tests/unit_tests/executor/agentCommTest.cpp b/cpp/tests/unit_tests/executor/agentCommTest.cpp index 194d5267c6b3..89488ba373e8 100644 --- a/cpp/tests/unit_tests/executor/agentCommTest.cpp +++ b/cpp/tests/unit_tests/executor/agentCommTest.cpp @@ -15,6 +15,7 @@ * limitations under the License. */ +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" #include <gtest/gtest.h> @@ -120,7 +121,7 @@ class AgentCommTest : public ::testing::TestWithParam<std::string> auto constexpr blocksInSecondaryPool = 0; auto constexpr enableBlockReuse = true; - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; BlocksPerWindow const blocksPerWindow diff --git a/cpp/tests/unit_tests/executor/coalesceTest.cpp b/cpp/tests/unit_tests/executor/coalesceTest.cpp index 8675d93fc3b4..f115d0fd06db 100644 --- a/cpp/tests/unit_tests/executor/coalesceTest.cpp +++ b/cpp/tests/unit_tests/executor/coalesceTest.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,154 +15,60 @@ * limitations under the License. */ -#include "tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.h" +#include "tensorrt_llm/executor/transferAgent.h" #include <gtest/gtest.h> using namespace tensorrt_llm::executor::kv_cache; -// ==================== coalesceMemoryDescs tests ==================== - -TEST(CoalesceMemoryDescsTest, EmptyInput) -{ - MemoryDescs descs{MemoryType::kVRAM, {}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - EXPECT_EQ(result.getDescs().size(), 0); -} - -TEST(CoalesceMemoryDescsTest, SingleEntry) -{ - MemoryDescs descs{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - ASSERT_EQ(result.getDescs().size(), 1); - EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); - EXPECT_EQ(result.getDescs()[0].getLen(), 256); - EXPECT_EQ(result.getDescs()[0].getDeviceId(), 0); -} - -TEST(CoalesceMemoryDescsTest, TwoContiguous) -{ - // [0x1000, 256) then [0x1100, 256) — adjacent, should merge into one - MemoryDescs descs{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - ASSERT_EQ(result.getDescs().size(), 1); - EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); - EXPECT_EQ(result.getDescs()[0].getLen(), 512); - EXPECT_EQ(result.getDescs()[0].getDeviceId(), 0); -} - -TEST(CoalesceMemoryDescsTest, TwoNonContiguous) +namespace { - // Gap between blocks — should stay as two - MemoryDescs descs{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x2000, 256, 0}}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - ASSERT_EQ(result.getDescs().size(), 2); - EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); - EXPECT_EQ(result.getDescs()[0].getLen(), 256); - EXPECT_EQ(result.getDescs()[1].getAddr(), 0x2000); - EXPECT_EQ(result.getDescs()[1].getLen(), 256); -} +VramRegionMap const kEmptyMap; -TEST(CoalesceMemoryDescsTest, ThreeContiguous) +// Merging requires region metadata: an address that misses its region map is never coalesced. +// Tests exercising merge behavior therefore provide maps covering their addresses; a flat +// region (chunkSize=0) imposes no chunk boundaries within it. +VramRegionMap flatRegion(uintptr_t base, size_t len) { - MemoryDescs descs{ - MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x1200, 256, 0}}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - ASSERT_EQ(result.getDescs().size(), 1); - EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); - EXPECT_EQ(result.getDescs()[0].getLen(), 768); + VramRegionMap map; + map[base] = {len, 0}; + return map; } -TEST(CoalesceMemoryDescsTest, UnsortedInput) +std::pair<MemoryDescs, MemoryDescs> run(TransferDescs const& src, TransferDescs const& dst, + VramRegionMap const& localMap = kEmptyMap, VramRegionMap const& remoteMap = kEmptyMap) { - // Same three contiguous blocks but in reverse order — sorting should fix it - MemoryDescs descs{ - MemoryType::kVRAM, {MemoryDesc{0x1200, 256, 0}, MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - ASSERT_EQ(result.getDescs().size(), 1); - EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); - EXPECT_EQ(result.getDescs()[0].getLen(), 768); + return VmmDescSplitter::splitAndCoalesceTransferDescs(src, dst, localMap, remoteMap); } +} // namespace -TEST(CoalesceMemoryDescsTest, DifferentDevices) -{ - // Contiguous addresses but different devices — should NOT merge - MemoryDescs descs{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 1}}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - ASSERT_EQ(result.getDescs().size(), 2); -} +// ==================== coalescing within flat regions (chunkSize=0) ==================== -TEST(CoalesceMemoryDescsTest, MixedContiguousAndGaps) -{ - // First two are contiguous, then a gap before the third - MemoryDescs descs{ - MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x3000, 128, 0}}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - ASSERT_EQ(result.getDescs().size(), 2); - EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); - EXPECT_EQ(result.getDescs()[0].getLen(), 512); - EXPECT_EQ(result.getDescs()[1].getAddr(), 0x3000); - EXPECT_EQ(result.getDescs()[1].getLen(), 128); -} - -TEST(CoalesceMemoryDescsTest, MultipleDevicesEachContiguous) -{ - // Two contiguous on device 0, two contiguous on device 1 - MemoryDescs descs{MemoryType::kVRAM, - {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x2000, 128, 1}, - MemoryDesc{0x2080, 128, 1}}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - ASSERT_EQ(result.getDescs().size(), 2); - EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); - EXPECT_EQ(result.getDescs()[0].getLen(), 512); - EXPECT_EQ(result.getDescs()[0].getDeviceId(), 0); - EXPECT_EQ(result.getDescs()[1].getAddr(), 0x2000); - EXPECT_EQ(result.getDescs()[1].getLen(), 256); - EXPECT_EQ(result.getDescs()[1].getDeviceId(), 1); -} - -TEST(CoalesceMemoryDescsTest, PreservesMemoryType) -{ - MemoryDescs descs{MemoryType::kDRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - EXPECT_EQ(result.getType(), MemoryType::kDRAM); -} - -TEST(CoalesceMemoryDescsTest, AllSeparate) -{ - // Nothing can be merged — all have gaps - MemoryDescs descs{MemoryType::kVRAM, - {MemoryDesc{0x1000, 100, 0}, MemoryDesc{0x2000, 100, 0}, MemoryDesc{0x3000, 100, 0}, - MemoryDesc{0x4000, 100, 0}}}; - auto result = NixlHelper::coalesceMemoryDescs(descs); - ASSERT_EQ(result.getDescs().size(), 4); -} - -// ==================== coalesceTransferDescs tests ==================== - -TEST(CoalesceTransferDescsTest, EmptyInput) +TEST(SplitAndCoalesceTest, EmptyInput) { TransferDescs src{MemoryType::kVRAM, {}}; TransferDescs dst{MemoryType::kVRAM, {}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + auto [resSrc, resDst] = run(src, dst); EXPECT_EQ(resSrc.getDescs().size(), 0); EXPECT_EQ(resDst.getDescs().size(), 0); } -TEST(CoalesceTransferDescsTest, SinglePair) +TEST(SplitAndCoalesceTest, SinglePair) { TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + auto [resSrc, resDst] = run(src, dst); ASSERT_EQ(resSrc.getDescs().size(), 1); ASSERT_EQ(resDst.getDescs().size(), 1); + EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x1000); + EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x5000); } -TEST(CoalesceTransferDescsTest, BothSidesContiguous) +TEST(SplitAndCoalesceTest, BothSidesContiguous) { - // src contiguous AND dst contiguous — should merge into one transfer + // src contiguous AND dst contiguous within known regions — should merge into one transfer TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x1000)); ASSERT_EQ(resSrc.getDescs().size(), 1); ASSERT_EQ(resDst.getDescs().size(), 1); EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x1000); @@ -171,98 +77,79 @@ TEST(CoalesceTransferDescsTest, BothSidesContiguous) EXPECT_EQ(resDst.getDescs()[0].getLen(), 512); } -TEST(CoalesceTransferDescsTest, SrcContiguousDstNot) +TEST(SplitAndCoalesceTest, SrcContiguousDstNot) { - // src is contiguous but dst has a gap — can't merge TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x6000, 256, 1}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x2000)); ASSERT_EQ(resSrc.getDescs().size(), 2); ASSERT_EQ(resDst.getDescs().size(), 2); } -TEST(CoalesceTransferDescsTest, DstContiguousSrcNot) +TEST(SplitAndCoalesceTest, DstContiguousSrcNot) { - // dst is contiguous but src has a gap — can't merge TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x2000, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x2000), flatRegion(0x5000, 0x1000)); ASSERT_EQ(resSrc.getDescs().size(), 2); ASSERT_EQ(resDst.getDescs().size(), 2); } -TEST(CoalesceTransferDescsTest, DifferentDevicesOnSrc) +TEST(SplitAndCoalesceTest, DifferentDevicesOnSrc) { - // src addresses look contiguous but are on different devices — can't merge TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 1}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 0}, MemoryDesc{0x5100, 256, 0}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x1000)); ASSERT_EQ(resSrc.getDescs().size(), 2); ASSERT_EQ(resDst.getDescs().size(), 2); } -TEST(CoalesceTransferDescsTest, DifferentDevicesOnDst) +TEST(SplitAndCoalesceTest, DifferentDevicesOnDst) { - // dst addresses look contiguous but are on different devices — can't merge TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 0}, MemoryDesc{0x5100, 256, 1}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x1000)); ASSERT_EQ(resSrc.getDescs().size(), 2); ASSERT_EQ(resDst.getDescs().size(), 2); } -TEST(CoalesceTransferDescsTest, MismatchedSizes) -{ - // src has 2 entries, dst has 1 — sizes don't match, return as-is - TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); - ASSERT_EQ(resSrc.getDescs().size(), 2); - ASSERT_EQ(resDst.getDescs().size(), 1); -} - -TEST(CoalesceTransferDescsTest, ThreePairsAllContiguous) +TEST(SplitAndCoalesceTest, ThreePairsAllContiguous) { TransferDescs src{ MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x1200, 256, 0}}}; TransferDescs dst{ MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}, MemoryDesc{0x5200, 256, 1}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x1000)); ASSERT_EQ(resSrc.getDescs().size(), 1); ASSERT_EQ(resDst.getDescs().size(), 1); EXPECT_EQ(resSrc.getDescs()[0].getLen(), 768); EXPECT_EQ(resDst.getDescs()[0].getLen(), 768); } -TEST(CoalesceTransferDescsTest, PartialMerge) +TEST(SplitAndCoalesceTest, PartialMerge) { - // First two pairs: both sides contiguous — merge - // Third pair: src contiguous but dst has gap — stays separate + // First two pairs merge; third pair's dst has a gap — stays separate TransferDescs src{ MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x1200, 256, 0}}}; TransferDescs dst{ MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}, MemoryDesc{0x9000, 256, 1}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x8000)); ASSERT_EQ(resSrc.getDescs().size(), 2); ASSERT_EQ(resDst.getDescs().size(), 2); - // Merged pair EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x1000); EXPECT_EQ(resSrc.getDescs()[0].getLen(), 512); EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x5000); EXPECT_EQ(resDst.getDescs()[0].getLen(), 512); - // Separate pair EXPECT_EQ(resSrc.getDescs()[1].getAddr(), 0x1200); - EXPECT_EQ(resSrc.getDescs()[1].getLen(), 256); EXPECT_EQ(resDst.getDescs()[1].getAddr(), 0x9000); - EXPECT_EQ(resDst.getDescs()[1].getLen(), 256); } -TEST(CoalesceTransferDescsTest, UnsortedInput) +TEST(SplitAndCoalesceTest, UnsortedInput) { - // Same as BothSidesContiguous but in reverse order — sorting should fix it + // Same as BothSidesContiguous but in reverse order — sorting by src addr should fix it TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x1000, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5100, 256, 1}, MemoryDesc{0x5000, 256, 1}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x1000)); ASSERT_EQ(resSrc.getDescs().size(), 1); ASSERT_EQ(resDst.getDescs().size(), 1); EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x1000); @@ -271,11 +158,259 @@ TEST(CoalesceTransferDescsTest, UnsortedInput) EXPECT_EQ(resDst.getDescs()[0].getLen(), 512); } -TEST(CoalesceTransferDescsTest, PreservesMemoryType) +TEST(SplitAndCoalesceTest, NonVramPassthrough) { - TransferDescs src{MemoryType::kDRAM, {MemoryDesc{0x1000, 256, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 0}}}; - auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + // Non-kVRAM descs pass through unchanged: no region info exists to bound a merge + TransferDescs src{MemoryType::kDRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; + TransferDescs dst{MemoryType::kDRAM, {MemoryDesc{0x5000, 256, 0}, MemoryDesc{0x5100, 256, 0}}}; + auto [resSrc, resDst] = run(src, dst); + ASSERT_EQ(resSrc.getDescs().size(), 2); + ASSERT_EQ(resDst.getDescs().size(), 2); EXPECT_EQ(resSrc.getType(), MemoryType::kDRAM); - EXPECT_EQ(resDst.getType(), MemoryType::kVRAM); +} + +// ==================== unknown regions never merge ==================== + +TEST(SplitAndCoalesceTest, NoMergeWithoutRegionMetadata) +{ + // Contiguous on both sides, but neither map covers the addresses: two unknown regions are + // indistinguishable (both look up as a miss), so merging is disabled and pairs stay separate. + TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}}}; + auto [resSrc, resDst] = run(src, dst); + ASSERT_EQ(resSrc.getDescs().size(), 2); + ASSERT_EQ(resDst.getDescs().size(), 2); +} + +TEST(SplitAndCoalesceTest, NoMergeWhenRemoteRegionUnknown) +{ + // Local map covers src, but the remote side sent no region info (e.g. an older peer): + // dst lookups miss, so nothing merges even though both sides are contiguous. + TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}}}; + auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000)); + ASSERT_EQ(resSrc.getDescs().size(), 2); + ASSERT_EQ(resDst.getDescs().size(), 2); +} + +TEST(SplitAndCoalesceTest, NoMergeWhenLocalRegionUnknown) +{ + // Remote map covers dst, but src addresses miss the local map: no merging. + TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}}}; + auto [resSrc, resDst] = run(src, dst, kEmptyMap, flatRegion(0x5000, 0x1000)); + ASSERT_EQ(resSrc.getDescs().size(), 2); + ASSERT_EQ(resDst.getDescs().size(), 2); +} + +// ==================== chunk-boundary-constrained coalescing ==================== + +TEST(SplitAndCoalesceTest, MergeStopsAtSrcChunkBoundary) +{ + // Local VMM region: base=0x100000, 4MB total, 2MB chunks → boundary at 0x300000. + VramRegionMap localMap; + localMap[0x100000] = {0x400000, 0x200000}; + + // Four contiguous 1MB pairs covering 4MB on both sides; dst is one flat remote region. + std::vector<MemoryDesc> srcVec, dstVec; + for (size_t i = 0; i < 4; ++i) + { + srcVec.emplace_back(0x100000 + i * 0x100000, 0x100000, 0); + dstVec.emplace_back(0x900000 + i * 0x100000, 0x100000, 1); + } + TransferDescs src{MemoryType::kVRAM, srcVec}; + TransferDescs dst{MemoryType::kVRAM, dstVec}; + + auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x400000)); + // Merged per src chunk: two 2MB transfers, split exactly at the chunk boundary. + ASSERT_EQ(resSrc.getDescs().size(), 2); + ASSERT_EQ(resDst.getDescs().size(), 2); + EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x100000); + EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x200000); + EXPECT_EQ(resSrc.getDescs()[1].getAddr(), 0x300000); + EXPECT_EQ(resSrc.getDescs()[1].getLen(), 0x200000); + EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x900000); + EXPECT_EQ(resDst.getDescs()[0].getLen(), 0x200000); + EXPECT_EQ(resDst.getDescs()[1].getAddr(), 0xB00000); + EXPECT_EQ(resDst.getDescs()[1].getLen(), 0x200000); +} + +TEST(SplitAndCoalesceTest, MergeStopsAtDstChunkBoundary) +{ + // Remote VMM region: base=0x800000, 1MB chunks → boundary at 0x900000. + VramRegionMap remoteMap; + remoteMap[0x800000] = {0x400000, 0x100000}; + + // Two contiguous pairs whose dst junction sits exactly on the remote chunk boundary. + auto localMap = flatRegion(0x1000, 0x100000); + TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 0x80000, 0}, MemoryDesc{0x81000, 0x80000, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x880000, 0x80000, 1}, MemoryDesc{0x900000, 0x80000, 1}}}; + + // With a flat remote region they would merge into one transfer... + { + auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x800000, 0x400000)); + ASSERT_EQ(resSrc.getDescs().size(), 1); + ASSERT_EQ(resDst.getDescs().size(), 1); + } + // ...but the dst chunk boundary must block the merge. + auto [resSrc, resDst] = run(src, dst, localMap, remoteMap); + ASSERT_EQ(resSrc.getDescs().size(), 2); + ASSERT_EQ(resDst.getDescs().size(), 2); + EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x880000); + EXPECT_EQ(resDst.getDescs()[1].getAddr(), 0x900000); +} + +TEST(SplitAndCoalesceTest, SplitPiecesAreNotRemerged) +{ + // A single pair spanning two src chunks stays split even though the pieces are contiguous. + VramRegionMap localMap; + localMap[0x100000] = {0x400000, 0x100000}; + + TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x100000, 0x200000, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x900000, 0x200000, 1}}}; + + auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x400000)); + ASSERT_EQ(resSrc.getDescs().size(), 2); + ASSERT_EQ(resDst.getDescs().size(), 2); + EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x100000); + EXPECT_EQ(resSrc.getDescs()[1].getLen(), 0x100000); +} + +TEST(SplitAndCoalesceTest, UnalignedRegionBaseBoundary) +{ + // Chunk boundaries are relative to the region base, not absolute alignment. + // base=0x180000, 1MB chunks → boundaries at 0x280000, 0x380000, ... + VramRegionMap localMap; + localMap[0x180000] = {0x300000, 0x100000}; + + // Three contiguous 512KB pairs: first two share the first chunk, third starts a new chunk. + std::vector<MemoryDesc> srcVec, dstVec; + for (size_t i = 0; i < 3; ++i) + { + srcVec.emplace_back(0x180000 + i * 0x80000, 0x80000, 0); + dstVec.emplace_back(0x900000 + i * 0x80000, 0x80000, 1); + } + TransferDescs src{MemoryType::kVRAM, srcVec}; + TransferDescs dst{MemoryType::kVRAM, dstVec}; + + auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x300000)); + ASSERT_EQ(resSrc.getDescs().size(), 2); + EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x180000); + EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x100000); + EXPECT_EQ(resSrc.getDescs()[1].getAddr(), 0x280000); + EXPECT_EQ(resSrc.getDescs()[1].getLen(), 0x80000); +} + +TEST(SplitAndCoalesceTest, NoMergeAcrossRegions) +{ + // Two VA-adjacent but distinct local regions (cudaMalloc-style, chunkSize=0): + // contiguous descs must not merge across the region boundary. + VramRegionMap localMap; + localMap[0x100000] = {0x100000, 0}; + localMap[0x200000] = {0x100000, 0}; + + TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x100000, 0x100000, 0}, MemoryDesc{0x200000, 0x100000, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x900000, 0x100000, 1}, MemoryDesc{0xA00000, 0x100000, 1}}}; + + auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x200000)); + ASSERT_EQ(resSrc.getDescs().size(), 2); + ASSERT_EQ(resDst.getDescs().size(), 2); +} + +TEST(SplitAndCoalesceTest, MergeWithinSingleRegion) +{ + // Control for NoMergeAcrossRegions: same layout as one region (chunkSize=0) merges freely. + VramRegionMap localMap; + localMap[0x100000] = {0x200000, 0}; + + TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x100000, 0x100000, 0}, MemoryDesc{0x200000, 0x100000, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x900000, 0x100000, 1}, MemoryDesc{0xA00000, 0x100000, 1}}}; + + auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x200000)); + ASSERT_EQ(resSrc.getDescs().size(), 1); + EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x200000); + EXPECT_EQ(resDst.getDescs()[0].getLen(), 0x200000); +} + +TEST(SplitAndCoalesceTest, NoMergeAcrossRemoteRegions) +{ + // The remote side registered two discrete but VA-adjacent buffers (chunkSize=0 each): + // contiguous dst descs must not merge across the remote registration boundary. + VramRegionMap remoteMap; + remoteMap[0x900000] = {0x100000, 0}; + remoteMap[0xA00000] = {0x100000, 0}; + + TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x100000, 0x100000, 0}, MemoryDesc{0x200000, 0x100000, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x900000, 0x100000, 1}, MemoryDesc{0xA00000, 0x100000, 1}}}; + + auto [resSrc, resDst] = run(src, dst, flatRegion(0x100000, 0x200000), remoteMap); + ASSERT_EQ(resSrc.getDescs().size(), 2); + ASSERT_EQ(resDst.getDescs().size(), 2); +} + +TEST(SplitAndCoalesceTest, CoalesceDisabledSplitsOnly) +{ + // With coalescing disabled, contiguous pairs stay separate and chunk splitting still applies. + VramRegionMap localMap; + localMap[0x100000] = {0x400000, 0x100000}; + + // Two contiguous 512KB pairs within one chunk plus one pair spanning a chunk boundary. + TransferDescs src{MemoryType::kVRAM, + {MemoryDesc{0x100000, 0x80000, 0}, MemoryDesc{0x180000, 0x80000, 0}, MemoryDesc{0x200000, 0x200000, 0}}}; + TransferDescs dst{MemoryType::kVRAM, + {MemoryDesc{0x900000, 0x80000, 1}, MemoryDesc{0x980000, 0x80000, 1}, MemoryDesc{0xA00000, 0x200000, 1}}}; + + auto [resSrc, resDst] + = VmmDescSplitter::splitAndCoalesceTransferDescs(src, dst, localMap, kEmptyMap, /*enableCoalesce=*/false); + // No merging: pair 1, pair 2, and pair 3 split into two chunk pieces → 4 descs. + ASSERT_EQ(resSrc.getDescs().size(), 4); + ASSERT_EQ(resDst.getDescs().size(), 4); + EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x80000); + EXPECT_EQ(resSrc.getDescs()[1].getLen(), 0x80000); + EXPECT_EQ(resSrc.getDescs()[2].getLen(), 0x100000); + EXPECT_EQ(resSrc.getDescs()[3].getLen(), 0x100000); +} + +TEST(SplitAndCoalesceTest, CoalesceDisabledPreservesInputOrder) +{ + // With coalescing disabled there is no sorting either: descs come out in input order, + // matching the historical split-only behavior. + TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x2000, 256, 0}, MemoryDesc{0x1000, 256, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x6000, 256, 1}, MemoryDesc{0x5000, 256, 1}}}; + + auto [resSrc, resDst] + = VmmDescSplitter::splitAndCoalesceTransferDescs(src, dst, kEmptyMap, kEmptyMap, /*enableCoalesce=*/false); + ASSERT_EQ(resSrc.getDescs().size(), 2); + EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x2000); + EXPECT_EQ(resSrc.getDescs()[1].getAddr(), 0x1000); + EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x6000); + EXPECT_EQ(resDst.getDescs()[1].getAddr(), 0x5000); +} + +TEST(SplitAndCoalesceTest, SplitAndMergeScatteredBlocks) +{ + // Scattered input order + chunked src region: blocks are sorted, merged per chunk. + // base=0x100000, 1MB chunks; four 512KB blocks given out of order. + VramRegionMap localMap; + localMap[0x100000] = {0x400000, 0x100000}; + + std::vector<size_t> perm{2, 0, 3, 1}; + std::vector<MemoryDesc> srcVec, dstVec; + for (size_t i : perm) + { + srcVec.emplace_back(0x100000 + i * 0x80000, 0x80000, 0); + dstVec.emplace_back(0x900000 + i * 0x80000, 0x80000, 1); + } + TransferDescs src{MemoryType::kVRAM, srcVec}; + TransferDescs dst{MemoryType::kVRAM, dstVec}; + + auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x400000)); + // 2MB of contiguous data over two 1MB chunks → one transfer per chunk. + ASSERT_EQ(resSrc.getDescs().size(), 2); + EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x100000); + EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x100000); + EXPECT_EQ(resSrc.getDescs()[1].getAddr(), 0x200000); + EXPECT_EQ(resSrc.getDescs()[1].getLen(), 0x100000); + EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x900000); + EXPECT_EQ(resDst.getDescs()[1].getAddr(), 0xA00000); } diff --git a/cpp/tests/unit_tests/executor/dynamicBatchTunerTest.cpp b/cpp/tests/unit_tests/executor/dynamicBatchTunerTest.cpp deleted file mode 100644 index 04ce10393b67..000000000000 --- a/cpp/tests/unit_tests/executor/dynamicBatchTunerTest.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/executor/dynamicBatchTuner.h" -#include "tensorrt_llm/common/tllmException.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" -#include <gmock/gmock.h> -#include <gtest/gtest.h> - -using ::testing::_; -using ::testing::Invoke; - -using namespace tensorrt_llm::executor; -using namespace tensorrt_llm::common; - -TEST(DynamicBatchTunerTest, Stats) -{ - // moving average window size is 3 - DynamicBatchConfig dynamicBatchConfig(true, true, 3); - DynamicBatchTuner dynamicBatchTuner(dynamicBatchConfig); - - // check no division by zero issue - EXPECT_EQ(dynamicBatchTuner.getAverageInputLength(), 0); - EXPECT_EQ(dynamicBatchTuner.getAverageOutputLength(), 0); - - dynamicBatchTuner.updateStats(1, 2); - EXPECT_EQ(dynamicBatchTuner.getAverageInputLength(), 1); - EXPECT_EQ(dynamicBatchTuner.getAverageOutputLength(), 2); - - dynamicBatchTuner.updateStats(2, 3); - EXPECT_EQ(dynamicBatchTuner.getAverageInputLength(), 1.5); - EXPECT_EQ(dynamicBatchTuner.getAverageOutputLength(), 2.5); - - dynamicBatchTuner.updateStats(3, 4); - EXPECT_EQ(dynamicBatchTuner.getAverageInputLength(), 2); - EXPECT_EQ(dynamicBatchTuner.getAverageOutputLength(), 3); - - // check that the first element is removed from the moving average window - dynamicBatchTuner.updateStats(4, 5); - EXPECT_EQ(dynamicBatchTuner.getAverageInputLength(), 3); - EXPECT_EQ(dynamicBatchTuner.getAverageOutputLength(), 4); -} - -TEST(DynamicBatchConfig, RuntimeBatchSize) -{ - // moving average window size is 3 - DynamicBatchConfig dynamicBatchConfig(true, true, 3); - DynamicBatchTuner dynamicBatchTuner(dynamicBatchConfig); - // check runtime batch size computation - EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(143), 128); - EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(335), 256); - EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(671), 512); - EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(831), 768); - EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(1279), 1024); - EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(1663), 1536); - // fall back - EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(2049), 2048); - EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(1665), 1665); -} - -TEST(DynamicBatchConfig, RuntimeMaxNumTokens) -{ - // moving average window size is 1 - DynamicBatchConfig dynamicBatchConfig(true, true, 1); - DynamicBatchTuner dynamicBatchTuner(dynamicBatchConfig); - - // context heavy - dynamicBatchTuner.updateStats(100, 2); - EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(1), 8192); - // context heavy fall back - EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(256), 16384); - - // balanced - dynamicBatchTuner.updateStats(100, 100); - EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(1), 4096); - // balanced fall back - EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(4000), 8192); - - // gen heavy - dynamicBatchTuner.updateStats(2, 256); - EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(1), 2048); - // gen heavy fall back - EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(4000), 4096); -} diff --git a/cpp/tests/unit_tests/executor/executorTestSmall.cpp b/cpp/tests/unit_tests/executor/executorTestSmall.cpp deleted file mode 100644 index 2987509f16ac..000000000000 --- a/cpp/tests/unit_tests/executor/executorTestSmall.cpp +++ /dev/null @@ -1,289 +0,0 @@ -#include "tensorrt_llm/batch_manager/trtGptModelInflightBatching.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tests/utils/common.h" -#include "tests/utils/engines.h" -#include "tests/utils/executorUtils.h" - -#include "gtest/gtest.h" - -#include <random> -#include <tuple> -#include <unordered_map> - -namespace tensorrt_llm::testing -{ - -struct TrivialConstantDecoderTestParameters -{ - using TupleT = std::tuple<runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, - runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, runtime::SizeType32>; - runtime::SizeType32 randomSeed; - runtime::SizeType32 vocabSize; - runtime::SizeType32 maxNumTokens; - runtime::SizeType32 maxBeamWidth; - runtime::SizeType32 maxBatchSize; - runtime::SizeType32 numRequests; - runtime::SizeType32 promptLength; - runtime::SizeType32 maxOutputLength; - - // Constructor that takes a tuple - TrivialConstantDecoderTestParameters( // NOLINT: implicit to allow gtest to convert from tuple generated by - // 'combine' - TupleT t) - : randomSeed(std::get<0>(t)) - , vocabSize(std::get<1>(t)) - , maxNumTokens(std::get<2>(t)) - , maxBeamWidth(std::get<3>(t)) - , maxBatchSize(std::get<4>(t)) - , numRequests(std::get<5>(t)) - , promptLength(std::get<6>(t)) - , maxOutputLength(std::get<7>(t)) - { - } -}; - -template <typename TLogits> -struct DecoderTestShared -{ - static constexpr runtime::SizeType32 kNumTokensPerBlock = 64; - static constexpr runtime::SizeType32 kKvCacheMaxTokens = 2048 * 8; - - DecoderTestShared(std::shared_ptr<runtime::TllmLogger> logger, std::mt19937 rng, - std::shared_ptr<executor::Executor> executor, std::vector<TLogits> randomLogits) - : logger(std::move(logger)) - , rng(rng) - , executor(std::move(executor)) - , randomLogits(std::move(randomLogits)){}; - std::shared_ptr<runtime::TllmLogger> logger; - std::mt19937 rng; - std::shared_ptr<executor::Executor> executor; - std::vector<TLogits> randomLogits; -}; - -template <typename TLogits> -std::unique_ptr<DecoderTestShared<TLogits>> SetupDecoderTest(TrivialConstantDecoderTestParameters const& params) -{ - auto logger = std::make_shared<runtime::TllmLogger>(); - auto rng = std::mt19937(params.randomSeed); - auto randomLogits = tensorrt_llm::testing::randomLogits<std::mt19937, TLogits>(params.vocabSize, &rng); - auto const decoderParameters = tensorrt_llm::testing::utils::engines::ConstantTrivialDecoderParameters<TLogits>{ - tensorrt_llm::testing::utils::engines::TrivialDecoderParameters{params.vocabSize, params.maxBatchSize, - params.maxNumTokens, DecoderTestShared<TLogits>::kNumTokensPerBlock, params.maxBeamWidth, false}, - randomLogits}; - auto engineHostMemory - = tensorrt_llm::testing::utils::engines::createConstantTrivialDecoder<TLogits>(decoderParameters, logger); - auto const engine = runtime::RawEngine(engineHostMemory.release()); - auto const dtype = runtime::TRTDataType<TLogits>::value; - auto modelConfig = runtime::ModelConfig(params.vocabSize, 1, 1, 0, 1, 1, dtype); - modelConfig.useGptAttentionPlugin(true); - modelConfig.setModelVariant(runtime::ModelConfig::ModelVariant::kGpt); - modelConfig.usePackedInput(true); - modelConfig.setKVCacheType(runtime::ModelConfig::KVCacheType::kPAGED); - modelConfig.setMaxNumTokens(params.maxNumTokens); - modelConfig.setMaxBatchSize(params.maxBatchSize); - modelConfig.setMaxBeamWidth(params.maxBeamWidth); - modelConfig.setMaxSequenceLen(params.maxNumTokens); - modelConfig.setMaxInputLen(params.maxNumTokens); - modelConfig.setLayerTypes({runtime::ModelConfig::LayerType::kATTENTION}); - modelConfig.setTokensPerBlock(DecoderTestShared<TLogits>::kNumTokensPerBlock); - modelConfig.setPagedContextFMHA(true); - - auto const worldConfig = runtime::WorldConfig(); - auto kvCacheConfig = executor::KvCacheConfig{}; - kvCacheConfig.setMaxTokens(DecoderTestShared<TLogits>::kKvCacheMaxTokens); - - auto const executorConfig - = tensorrt_llm::executor::ExecutorConfig(params.maxBeamWidth, executor::SchedulerConfig(), kvCacheConfig, true, - true, 1, 1, executor::BatchingType::kINFLIGHT, params.maxBatchSize, params.maxNumTokens, std::nullopt, - std::nullopt, std::nullopt, std::nullopt, false, 1, std::nullopt, executor::ExtendedRuntimePerfKnobConfig(), - std::nullopt, 0, executor::ExecutorConfig::kDefaultMaxSeqIdleMicroseconds, std::nullopt, std::nullopt); - - auto model = std::make_shared<batch_manager::TrtGptModelInflightBatching>( - logger, modelConfig, worldConfig, engine, false, executorConfig, false); - - return std::make_unique<DecoderTestShared<TLogits>>( - logger, rng, std::make_shared<executor::Executor>(model, executorConfig), randomLogits); -} - -template <typename TLogits> -class DecoderTest : public ::testing::Test, public ::testing::WithParamInterface<TrivialConstantDecoderTestParameters> -{ -protected: - std::unique_ptr<DecoderTestShared<TLogits>> state; - - DecoderTest() - { - auto const params = GetParam(); - state = SetupDecoderTest<TLogits>(params); - } - - void runDecoderTest(TrivialConstantDecoderTestParameters const& parameters) - { - auto const requestTokens = createConsecutiveTokenSequence(parameters.promptLength, parameters.vocabSize, 0); - auto requests = std::vector<executor::Request>{}; - requests.reserve(static_cast<std::size_t>(parameters.numRequests)); - for (auto i = 0; i < parameters.numRequests; i++) - { - requests.emplace_back(requestTokens, parameters.maxOutputLength, false, executor::SamplingConfig{}, - executor::OutputConfig{false, false, false, true, false, false}); - } - auto const accumulatedResponses - = runThroughRequests(*state->executor, requests, std::chrono::duration<float, std::milli>(3600000)); - ASSERT_EQ(accumulatedResponses.size(), parameters.numRequests); - - std::sort(state->randomLogits.begin(), state->randomLogits.end()); - std::reverse(state->randomLogits.begin(), state->randomLogits.end()); - for (auto const& [requestId, responses] : accumulatedResponses) - { - for (auto const& response : responses) - { - ASSERT_FALSE(response.hasError()); - auto const& tokensByBeam = response.getResult().outputTokenIds; - ASSERT_EQ(tokensByBeam.size(), 1); - for (auto const& tokensForBeam : tokensByBeam) - { - ASSERT_EQ(tokensForBeam.size(), parameters.maxOutputLength); - } - } - } - } -}; - -namespace -{ -constexpr runtime::SizeType32 kRandomSeed1 = 45; -auto const randomSeeds = ::testing::Values(kRandomSeed1); - -constexpr runtime::SizeType32 kMinVocabSize = 16; -auto const vocabSizes = ::testing::Values(kMinVocabSize); - -constexpr runtime::SizeType32 kMinMaxNumTokens = 2048; -auto const maxNumTokenses = ::testing::Values(kMinMaxNumTokens); - -constexpr runtime::SizeType32 kMinBeamWidth = 1; -auto const beamWidths = ::testing::Values(kMinBeamWidth); - -constexpr runtime::SizeType32 kMinMaxBatchSize = 2048; -auto const maxBatchSizes = ::testing::Values(kMinMaxBatchSize); - -constexpr runtime::SizeType32 kMinNumRequests = 64; -auto const numRequestses = ::testing::Values(kMinNumRequests); - -constexpr runtime::SizeType32 kMinPromptLength = 32; -auto const promptLengths = ::testing::Values(kMinPromptLength); - -constexpr runtime::SizeType32 kMinMaxOutputLength = 16; -auto const maxOutputLengths = ::testing::Values(kMinMaxOutputLength); - -auto const paramGenerator - = ::testing::ConvertGenerator<TrivialConstantDecoderTestParameters::TupleT>(::testing::Combine(randomSeeds, - vocabSizes, maxNumTokenses, beamWidths, maxBatchSizes, numRequestses, promptLengths, maxOutputLengths)); -} // namespace - -using DecoderFloatTest = DecoderTest<float>; - -TEST_P(DecoderFloatTest, TestSizeAndValues) -{ - runDecoderTest(GetParam()); -} - -INSTANTIATE_TEST_SUITE_P(Float, DecoderFloatTest, paramGenerator, - [](::testing::TestParamInfo<TrivialConstantDecoderTestParameters> const& info) -> std::string - { - std::stringstream nameStringStream; - nameStringStream << "_maxBatchSize_" << info.param.maxBatchSize << "_vocabSize_" << info.param.vocabSize - << "_maxBeamWidth_" << info.param.maxBeamWidth << "_maxNumTokens_" << info.param.maxNumTokens - << "_maxOutputLength_" << info.param.maxOutputLength << "_numRequests_" - << info.param.numRequests << "_promptLength_" << info.param.promptLength << "_randomSeed_" - << info.param.randomSeed; - return nameStringStream.str(); - }); - -// Helper function to test calculateCacheSizePerToken with given parameters. -std::map<runtime::SizeType32, runtime::SizeType32> calculateCacheSizePerTokenHelper( - std::vector<runtime::SizeType32> const& maxAttentionWindowVec, runtime::SizeType32 kvFactor = 2, - runtime::SizeType32 vocabSize = 32, runtime::SizeType32 nbLayers = 4, runtime::SizeType32 nbAttentionLayers = 4, - runtime::SizeType32 nbRnnLayers = 0, runtime::SizeType32 nbHeads = 8, runtime::SizeType32 hiddenSize = 512, - bool isCrossAttention = false) -{ - // Create minimal ModelConfig for testing. - auto modelConfig = runtime::ModelConfig( - vocabSize, nbLayers, nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, nvinfer1::DataType::kFLOAT); - modelConfig.useGptAttentionPlugin(true); - modelConfig.setModelVariant(runtime::ModelConfig::ModelVariant::kGpt); - modelConfig.setKVCacheType(runtime::ModelConfig::KVCacheType::kPAGED); - - auto const worldConfig = runtime::WorldConfig(); - - return batch_manager::TrtGptModelInflightBatching::calculateCacheSizePerTokenForDisagg( - modelConfig, worldConfig, maxAttentionWindowVec, isCrossAttention, kvFactor); -} - -// Test for TrtGptModelInflightBatching::calculateCacheSizePerToken function with different layer types. -TEST(TrtInflightBatchingTest, CalculateCacheSizePerTokenForDisagg) -{ - // Common parameters. - constexpr runtime::SizeType32 nbLayers = 5; - constexpr runtime::SizeType32 hiddenSize = 512; - constexpr runtime::SizeType32 kvFactor = 2; - constexpr runtime::SizeType32 vocabSize = 32; - constexpr runtime::SizeType32 nbHeads = 8; - // Test case 1: Single attention window size - attention layers only. - { - std::vector<runtime::SizeType32> maxAttentionWindowVec = {128}; - constexpr runtime::SizeType32 nbAttentionLayers = 5; - constexpr runtime::SizeType32 numBytesPerFloatElement = 4; - constexpr runtime::SizeType32 nbRnnLayers = 0; - auto result = calculateCacheSizePerTokenHelper(maxAttentionWindowVec, kvFactor, vocabSize, nbLayers, - nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, false); - EXPECT_EQ(result.size(), 1); - EXPECT_EQ(result.at(128), nbAttentionLayers * kvFactor * hiddenSize * numBytesPerFloatElement); - } - - // Test case 2: Multiple attention window sizes - attention layers only. - { - std::vector<runtime::SizeType32> maxAttentionWindowVec = {128, 256}; - constexpr runtime::SizeType32 nbAttentionLayers = 5; - constexpr runtime::SizeType32 numBytesPerFloatElement = 4; - constexpr runtime::SizeType32 nbRnnLayers = 0; - auto result = calculateCacheSizePerTokenHelper(maxAttentionWindowVec, kvFactor, vocabSize, nbLayers, - nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, false); - EXPECT_EQ(result.size(), 2); - auto const nbAttentionLayersIn128Window = 3; - auto const nbAttentionLayersIn256Window = 2; - EXPECT_EQ(result.at(128), nbAttentionLayersIn128Window * kvFactor * hiddenSize * numBytesPerFloatElement); - EXPECT_EQ(result.at(256), nbAttentionLayersIn256Window * kvFactor * hiddenSize * numBytesPerFloatElement); - } - - // Test case 3: Single attention window size - attention and rnn layers. - { - std::vector<runtime::SizeType32> maxAttentionWindowVec = {128}; - constexpr runtime::SizeType32 nbAttentionLayers = 3; - constexpr runtime::SizeType32 numBytesPerFloatElement = 4; - constexpr runtime::SizeType32 nbRnnLayers = 2; - auto result = calculateCacheSizePerTokenHelper(maxAttentionWindowVec, kvFactor, vocabSize, nbLayers, - nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, false); - EXPECT_EQ(result.size(), 1); - EXPECT_EQ(result.at(128), nbAttentionLayers * kvFactor * hiddenSize * numBytesPerFloatElement); - } - - // Test case 4: Multiple attention window sizes - attention and rnn layers. - { - std::vector<runtime::SizeType32> maxAttentionWindowVec = {128, 256}; - constexpr runtime::SizeType32 nbAttentionLayers = 3; - constexpr runtime::SizeType32 numBytesPerFloatElement = 4; - constexpr runtime::SizeType32 nbRnnLayers = 2; - auto result = calculateCacheSizePerTokenHelper(maxAttentionWindowVec, kvFactor, vocabSize, nbLayers, - nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, false); - EXPECT_EQ(result.size(), 2); - auto const nbAttentionLayersIn128Window = 2; - auto const nbAttentionLayersIn256Window = 1; - EXPECT_EQ(result.at(128), nbAttentionLayersIn128Window * kvFactor * hiddenSize * numBytesPerFloatElement); - EXPECT_EQ(result.at(256), nbAttentionLayersIn256Window * kvFactor * hiddenSize * numBytesPerFloatElement); - } -} - -} // namespace tensorrt_llm::testing diff --git a/cpp/tests/unit_tests/executor/executorTestSmallArbitraryOutputTensors.cpp b/cpp/tests/unit_tests/executor/executorTestSmallArbitraryOutputTensors.cpp deleted file mode 100644 index b64bd775fe30..000000000000 --- a/cpp/tests/unit_tests/executor/executorTestSmallArbitraryOutputTensors.cpp +++ /dev/null @@ -1,491 +0,0 @@ -#include "include/tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/batch_manager/trtGptModelInflightBatching.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/modelConfig.h" -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/worldConfig.h" -#include "tests/utils/common.h" -#include "tests/utils/engines.h" -#include "tests/utils/executorUtils.h" - -#include "gtest/gtest.h" -#include <NvInfer.h> -#include <NvInferRuntime.h> -#include <NvInferRuntimeBase.h> -#include <gmock/gmock.h> - -#include <algorithm> -#include <chrono> -#include <cstddef> -#include <memory> -#include <optional> -#include <random> -#include <ratio> -#include <utility> -#include <vector> - -namespace tensorrt_llm::testing -{ - -struct TrivialConstantDecoderWithTopKLogitsTestParameters -{ - using TupleT = std::tuple<runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, - runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, bool>; - runtime::SizeType32 randomSeed; - runtime::SizeType32 vocabSize; - runtime::SizeType32 maxNumTokens; - runtime::SizeType32 maxBeamWidth; - runtime::SizeType32 maxBatchSize; - runtime::SizeType32 numTopKLogits; - runtime::SizeType32 numRequests; - runtime::SizeType32 promptLength; - runtime::SizeType32 maxOutputLength; - bool gatherContext; - - // Constructor that takes a tuple - TrivialConstantDecoderWithTopKLogitsTestParameters( // NOLINT: implicit to allow gtest to convert from tuple - // generated by 'combine' - TupleT t) - : randomSeed(std::get<0>(t)) - , vocabSize(std::get<1>(t)) - , maxNumTokens(std::get<2>(t)) - , maxBeamWidth(std::get<3>(t)) - , maxBatchSize(std::get<4>(t)) - , numTopKLogits(std::get<5>(t)) - , numRequests(std::get<6>(t)) - , promptLength(std::get<7>(t)) - , maxOutputLength(std::get<8>(t)) - , gatherContext(std::get<9>(t)) - { - } -}; - -template <typename TLogits> -struct DecoderTestShared -{ - static constexpr runtime::SizeType32 kNumTokensPerBlock = 64; - static constexpr runtime::SizeType32 kKvCacheMaxTokens = 2048 * 8; - static constexpr auto kTopKTensorName = "topKLogits"; - - DecoderTestShared(std::shared_ptr<runtime::TllmLogger> logger, std::mt19937 rng, - std::shared_ptr<executor::Executor> executor, std::vector<TLogits> randomLogits) - : logger(std::move(logger)) - , rng(rng) - , executor(std::move(executor)) - , randomLogits(std::move(randomLogits)){}; - std::shared_ptr<runtime::TllmLogger> logger; - std::mt19937 rng; - std::shared_ptr<executor::Executor> executor; - std::vector<TLogits> randomLogits; -}; - -template <typename TLogits> -std::unique_ptr<DecoderTestShared<TLogits>> SetupDecoderTest( - TrivialConstantDecoderWithTopKLogitsTestParameters const& params) -{ - auto logger = std::make_shared<runtime::TllmLogger>(); - auto rng = std::mt19937(params.randomSeed); - auto randomLogits = tensorrt_llm::testing::randomLogits<std::mt19937, TLogits>(params.vocabSize, &rng); - auto const decoderParameters = tensorrt_llm::testing::utils::engines::ConstantTrivialDecoderParameters<TLogits>{ - tensorrt_llm::testing::utils::engines::TrivialDecoderParameters{params.vocabSize, params.maxBatchSize, - params.maxNumTokens, DecoderTestShared<TLogits>::kNumTokensPerBlock, params.maxBeamWidth, - params.gatherContext}, - randomLogits}; - auto engineHostMemory = tensorrt_llm::testing::utils::engines::createConstantTrivialDecoderWithTopKLogits<TLogits>( - decoderParameters, params.numTopKLogits, DecoderTestShared<TLogits>::kTopKTensorName, logger); - auto const engine = runtime::RawEngine(engineHostMemory.release()); - - auto const dtype = runtime::TRTDataType<TLogits>::value; - auto modelConfig = runtime::ModelConfig(params.vocabSize, 1, 1, 0, 1, 1, dtype); - modelConfig.useGptAttentionPlugin(true); - modelConfig.setModelVariant(runtime::ModelConfig::ModelVariant::kGpt); - modelConfig.usePackedInput(true); - modelConfig.setKVCacheType(runtime::ModelConfig::KVCacheType::kPAGED); - modelConfig.setMaxNumTokens(params.maxNumTokens); - modelConfig.setMaxBatchSize(params.maxBatchSize); - modelConfig.setMaxBeamWidth(params.maxBeamWidth); - modelConfig.setMaxSequenceLen(params.maxNumTokens); - modelConfig.setMaxInputLen(params.maxNumTokens); - modelConfig.setLayerTypes({runtime::ModelConfig::LayerType::kATTENTION}); - modelConfig.setTokensPerBlock(DecoderTestShared<TLogits>::kNumTokensPerBlock); - modelConfig.setPagedContextFMHA(true); - modelConfig.computeContextLogits(params.gatherContext); - - auto const worldConfig = runtime::WorldConfig(); - - auto kvCacheConfig = executor::KvCacheConfig{}; - kvCacheConfig.setMaxTokens(DecoderTestShared<TLogits>::kKvCacheMaxTokens); - - auto const executorConfig - = executor::ExecutorConfig(params.maxBeamWidth, executor::SchedulerConfig(), kvCacheConfig, true, true, 1, 1, - executor::BatchingType::kINFLIGHT, params.maxBatchSize, params.maxNumTokens, std::nullopt, std::nullopt, - std::nullopt, std::nullopt, false, 1, std::nullopt, executor::ExtendedRuntimePerfKnobConfig(), std::nullopt, - 0, executor::ExecutorConfig::kDefaultMaxSeqIdleMicroseconds, std::nullopt, std::nullopt, - std::vector<executor::AdditionalModelOutput>{ - executor::AdditionalModelOutput{DecoderTestShared<TLogits>::kTopKTensorName, params.gatherContext}}); - - auto model = std::make_shared<batch_manager::TrtGptModelInflightBatching>( - logger, modelConfig, worldConfig, engine, false, executorConfig, false); - - return std::make_unique<DecoderTestShared<TLogits>>( - logger, rng, std::make_shared<executor::Executor>(model, executorConfig), randomLogits); -} - -template <typename TLogits> -class DecoderTopKGenerationLogitsTest - : public ::testing::Test, - public ::testing::WithParamInterface<TrivialConstantDecoderWithTopKLogitsTestParameters> -{ -protected: - std::unique_ptr<DecoderTestShared<TLogits>> state; - - DecoderTopKGenerationLogitsTest() - { - auto const params = GetParam(); - state = SetupDecoderTest<TLogits>(params); - } - - void runTopKGenerationLogitsTest(TrivialConstantDecoderWithTopKLogitsTestParameters const& parameters) - { - auto const requestTokens = createConsecutiveTokenSequence(parameters.promptLength, parameters.vocabSize, 0); - auto requests = std::vector<executor::Request>{}; - requests.reserve(static_cast<std::size_t>(parameters.numRequests)); - for (auto i = 0; i < parameters.numRequests; i++) - { - std::vector<executor::AdditionalModelOutput> additionalOutputs{ - executor::AdditionalModelOutput{DecoderTestShared<TLogits>::kTopKTensorName}}; - requests.emplace_back(requestTokens, parameters.maxOutputLength, false, executor::SamplingConfig{}, - executor::OutputConfig{false, false, false, true, false, false, additionalOutputs}); - } - auto const accumulatedResponses - = runThroughRequests(*state->executor, requests, std::chrono::duration<float, std::milli>(100000)); - ASSERT_EQ(accumulatedResponses.size(), parameters.numRequests); - - std::sort(state->randomLogits.begin(), state->randomLogits.end()); - std::reverse(state->randomLogits.begin(), state->randomLogits.end()); - for (auto const& [requestId, responses] : accumulatedResponses) - { - for (auto const& response : responses) - { - ASSERT_FALSE(response.hasError()); - auto const& tokensByBeam = response.getResult().outputTokenIds; - auto const& additionalOutputs = response.getResult().additionalOutputs; - ASSERT_EQ(additionalOutputs.size(), 1); - auto const& topKLogits = additionalOutputs.front(); - auto const expectedOutputSize = parameters.maxOutputLength * parameters.numTopKLogits; - ASSERT_EQ(topKLogits.output.getSize(), expectedOutputSize); - auto const* topKLogitsData = reinterpret_cast<TLogits const*>(topKLogits.output.getData()); - for (auto i = 0; i < parameters.numTopKLogits; i++) - { - EXPECT_TRUE(almostEqual(topKLogitsData[i], state->randomLogits[i], 1e-5)) - << "requestId " << requestId << " i " << i << ": " << topKLogitsData[i] - << " != " << state->randomLogits[i]; - } - ASSERT_EQ(tokensByBeam.size(), 1); - for (auto const& tokensForBeam : tokensByBeam) - { - ASSERT_EQ(tokensForBeam.size(), parameters.maxOutputLength); - } - } - } - } -}; - -template <typename TLogits> -class DecoderTopKGenerationLogitsStreamingTest - : public ::testing::Test, - public ::testing::WithParamInterface<TrivialConstantDecoderWithTopKLogitsTestParameters> -{ -protected: - std::unique_ptr<DecoderTestShared<TLogits>> state; - - DecoderTopKGenerationLogitsStreamingTest() - { - auto const params = GetParam(); - state = SetupDecoderTest<TLogits>(params); - } - - void runTopKGenerationLogitsStreamingTest(TrivialConstantDecoderWithTopKLogitsTestParameters const& parameters) - { - auto const requestTokens = createConsecutiveTokenSequence(parameters.promptLength, parameters.vocabSize, 0); - auto requests = std::vector<executor::Request>{}; - requests.reserve(static_cast<std::size_t>(parameters.numRequests)); - for (auto i = 0; i < parameters.numRequests; i++) - { - std::vector<executor::AdditionalModelOutput> additionalOutputs{ - executor::AdditionalModelOutput{DecoderTestShared<TLogits>::kTopKTensorName}}; - requests.emplace_back(requestTokens, parameters.maxOutputLength, true, executor::SamplingConfig{}, - executor::OutputConfig{false, false, false, true, false, false, additionalOutputs}); - } - auto const accumulatedResponses - = runThroughRequests(*state->executor, requests, std::chrono::duration<float, std::milli>(100000)); - ASSERT_EQ(accumulatedResponses.size(), parameters.numRequests); - - std::sort(state->randomLogits.begin(), state->randomLogits.end()); - std::reverse(state->randomLogits.begin(), state->randomLogits.end()); - for (auto const& idResponsesKvp : accumulatedResponses) - { - auto const& [requestId, responses] = idResponsesKvp; - auto numTokensForRequest = 0; - for (auto const& response : responses) - { - ASSERT_FALSE(response.hasError()); - auto const& tokensByBeam = response.getResult().outputTokenIds; - auto const& additionalOutputs = response.getResult().additionalOutputs; - ASSERT_EQ(additionalOutputs.size(), 1); - auto const& topKLogits = additionalOutputs.front(); - auto const expectedOutputSize = parameters.maxOutputLength * parameters.numTopKLogits; - ASSERT_EQ(topKLogits.output.getSize(), expectedOutputSize); - auto const* topKLogitsData = reinterpret_cast<TLogits const*>(topKLogits.output.getData()); - for (auto i = 0; i < parameters.numTopKLogits; i++) - { - EXPECT_TRUE(almostEqual(topKLogitsData[i], state->randomLogits[i], 1e-5)) - << "requestId " << requestId << " i " << i << ": " << topKLogitsData[i] - << " != " << state->randomLogits[i]; - } - ASSERT_EQ(tokensByBeam.size(), 1); - for (auto const& tokensForBeam : tokensByBeam) - { - numTokensForRequest += tokensForBeam.size(); - } - } - ASSERT_EQ(numTokensForRequest, parameters.maxOutputLength); - } - } -}; - -template <typename TLogits> -class DecoderTopKContextLogitsStreamingTest - : public ::testing::Test, - public ::testing::WithParamInterface<TrivialConstantDecoderWithTopKLogitsTestParameters> -{ -protected: - std::unique_ptr<DecoderTestShared<TLogits>> state; - - DecoderTopKContextLogitsStreamingTest() - { - auto const params = GetParam(); - state = SetupDecoderTest<TLogits>(params); - } - - void runTopKContextLogitsTest(TrivialConstantDecoderWithTopKLogitsTestParameters const& parameters) - { - auto requests = std::vector<executor::Request>{}; - requests.reserve(static_cast<std::size_t>(parameters.numRequests)); - for (auto i = 0; i < parameters.numRequests; i++) - { - // create different sequence for each request to avoid KV cache reuse - auto const requestTokens = createConsecutiveTokenSequence(parameters.promptLength, parameters.vocabSize, i); - std::vector<executor::AdditionalModelOutput> additionalOutputs{ - executor::AdditionalModelOutput{DecoderTestShared<TLogits>::kTopKTensorName, true}}; - requests.emplace_back(requestTokens, parameters.maxOutputLength, true, executor::SamplingConfig{}, - executor::OutputConfig{false, false, false, true, false, false, additionalOutputs}); - } - auto const& accumulatedResponses - = runThroughRequests(*state->executor, requests, std::chrono::duration<float, std::milli>(100000)); - ASSERT_EQ(accumulatedResponses.size(), parameters.numRequests); - - std::sort(state->randomLogits.begin(), state->randomLogits.end()); - std::reverse(state->randomLogits.begin(), state->randomLogits.end()); - std::string const expectedAdditionalOutputName - = std::string("context_") + DecoderTestShared<TLogits>::kTopKTensorName; - for (auto const& idResponsesKvp : accumulatedResponses) - { - auto const& [requestId, responses] = idResponsesKvp; - std::size_t numTokensForRequest{0}; - for (auto const& response : responses) - { - ASSERT_FALSE(response.hasError()); - auto const& tokensByBeam = response.getResult().outputTokenIds; - auto const& additionalOutputs = response.getResult().additionalOutputs; - ASSERT_EQ(additionalOutputs.size(), 2); - auto const contextTopKLogitsPtr = std::find_if(additionalOutputs.cbegin(), additionalOutputs.cend(), - [&expectedAdditionalOutputName](auto const& ao) - { return ao.name == expectedAdditionalOutputName; }); - auto const expectedOutputSize = parameters.promptLength * parameters.numTopKLogits; - ASSERT_EQ(contextTopKLogitsPtr->output.getSize(), expectedOutputSize); - auto const* topKLogitsData = reinterpret_cast<TLogits const*>(contextTopKLogitsPtr->output.getData()); - for (auto i = 0; i < parameters.numTopKLogits; i++) - { - EXPECT_TRUE(almostEqual(topKLogitsData[i], state->randomLogits[i], 1e-5)) - << "requestId " << requestId << " i " << i << ": " << topKLogitsData[i] - << " != " << state->randomLogits[i]; - } - ASSERT_EQ(tokensByBeam.size(), 1); - for (auto const& tokensForBeam : tokensByBeam) - { - numTokensForRequest += static_cast<std::size_t>(tokensForBeam.size()); - } - } - ASSERT_EQ(numTokensForRequest, parameters.maxOutputLength); - } - } -}; - -template <typename TLogits> -class DecoderTopKContextLogitsTest - : public ::testing::Test, - public ::testing::WithParamInterface<TrivialConstantDecoderWithTopKLogitsTestParameters> -{ -protected: - std::unique_ptr<DecoderTestShared<TLogits>> state; - - DecoderTopKContextLogitsTest() - { - auto const params = GetParam(); - state = SetupDecoderTest<TLogits>(params); - } - - void runTopKContextLogitsTest(TrivialConstantDecoderWithTopKLogitsTestParameters const& parameters) - { - auto requests = std::vector<executor::Request>{}; - requests.reserve(static_cast<std::size_t>(parameters.numRequests)); - for (auto i = 0; i < parameters.numRequests; i++) - { - // create different sequence for each request to avoid KV cache reuse - auto const requestTokens = createConsecutiveTokenSequence(parameters.promptLength, parameters.vocabSize, i); - std::vector<executor::AdditionalModelOutput> additionalOutputs{ - executor::AdditionalModelOutput{DecoderTestShared<TLogits>::kTopKTensorName, true}}; - requests.emplace_back(requestTokens, parameters.maxOutputLength, false, executor::SamplingConfig{}, - executor::OutputConfig{false, false, false, true, false, false, additionalOutputs}); - } - auto const accumulatedResponses - = runThroughRequests(*state->executor, requests, std::chrono::duration<float, std::milli>(100000)); - ASSERT_EQ(accumulatedResponses.size(), parameters.numRequests); - - std::sort(state->randomLogits.begin(), state->randomLogits.end()); - std::reverse(state->randomLogits.begin(), state->randomLogits.end()); - std::string const expectedAdditionalOutputName - = std::string("context_") + DecoderTestShared<TLogits>::kTopKTensorName; - for (auto const& idResponsesKvp : accumulatedResponses) - { - auto const& [requestId, responses] = idResponsesKvp; - for (auto const& response : responses) - { - ASSERT_FALSE(response.hasError()); - auto const& tokensByBeam = response.getResult().outputTokenIds; - auto const& additionalOutputs = response.getResult().additionalOutputs; - ASSERT_EQ(additionalOutputs.size(), 2); - auto const contextTopKLogitsPtr = std::find_if(additionalOutputs.cbegin(), additionalOutputs.cend(), - [&expectedAdditionalOutputName](auto const& ao) - { return ao.name == expectedAdditionalOutputName; }); - auto const expectedOutputSize = parameters.promptLength * parameters.numTopKLogits; - ASSERT_EQ(contextTopKLogitsPtr->output.getSize(), expectedOutputSize); - auto const* topKLogitsData = reinterpret_cast<TLogits const*>(contextTopKLogitsPtr->output.getData()); - for (auto i = 0; i < parameters.numTopKLogits; i++) - { - EXPECT_TRUE(almostEqual(topKLogitsData[i], state->randomLogits[i], 1e-5)) - << "requestId " << requestId << " i " << i << ": " << topKLogitsData[i] - << " != " << state->randomLogits[i]; - } - ASSERT_EQ(tokensByBeam.size(), 1); - for (auto const& tokensForBeam : tokensByBeam) - { - ASSERT_EQ(tokensForBeam.size(), parameters.maxOutputLength); - } - } - } - } -}; - -namespace -{ -constexpr runtime::SizeType32 kRandomSeed1 = 45; -auto const randomSeeds = ::testing::Values(kRandomSeed1); - -constexpr runtime::SizeType32 kMinVocabSize = 64; -constexpr runtime::SizeType32 kMaxVocabSize = 2048; -auto const vocabSizes = ::testing::Values(kMinVocabSize); - -constexpr runtime::SizeType32 kMinMaxNumTokens = 2048; -auto const maxNumTokenses = ::testing::Values(kMinMaxNumTokens); - -constexpr runtime::SizeType32 kMinBeamWidth = 1; -auto const beamWidths = ::testing::Values(kMinBeamWidth); - -constexpr runtime::SizeType32 kMinMaxBatchSize = 2048; -auto const batchSizes = ::testing::Values(kMinMaxBatchSize); - -constexpr runtime::SizeType32 kMinNumTopKLogits = 4; -constexpr runtime::SizeType32 kMaxNumTopKLogits = 32; -auto const numTopKLogitses = ::testing::Values(kMinNumTopKLogits, kMaxNumTopKLogits); - -constexpr runtime::SizeType32 kMinNumRequests = 16; -constexpr runtime::SizeType32 kMaxNumRequests = 2048; -auto const numRequestses = ::testing::Values(kMinNumRequests); - -constexpr runtime::SizeType32 kMinPromptLength = 4; -constexpr runtime::SizeType32 kMaxPromptLength = 512; -auto const promptLengths = ::testing::Values(kMinPromptLength, kMaxPromptLength); - -constexpr runtime::SizeType32 kMinMaxOutputLength = 4; -constexpr runtime::SizeType32 kMaxMaxOutputLength = 256; -auto const maxOutputLengths = ::testing::Values(kMinMaxOutputLength, kMaxMaxOutputLength); - -auto const gatherContext = ::testing::Values(false, true); -auto const alwaysGatherContext = ::testing::Values(true); - -auto const paramGenerator = ::testing::ConvertGenerator<TrivialConstantDecoderWithTopKLogitsTestParameters::TupleT>( - ::testing::Combine(randomSeeds, vocabSizes, maxNumTokenses, beamWidths, batchSizes, numTopKLogitses, numRequestses, - promptLengths, maxOutputLengths, gatherContext)); - -auto const paramGeneratorGatherContext - = ::testing::ConvertGenerator<TrivialConstantDecoderWithTopKLogitsTestParameters::TupleT>( - ::testing::Combine(randomSeeds, vocabSizes, maxNumTokenses, beamWidths, batchSizes, numTopKLogitses, - numRequestses, promptLengths, maxOutputLengths, alwaysGatherContext)); - -auto const nameSuffixGenerator - = [](::testing::TestParamInfo<TrivialConstantDecoderWithTopKLogitsTestParameters> const& info) -> std::string -{ - std::stringstream nameStringStream; - nameStringStream << "gatherContext_" << info.param.gatherContext << "_maxBatchSize_" << info.param.maxBatchSize - << "_vocabSize_" << info.param.vocabSize << "_maxBeamWidth_" << info.param.maxBeamWidth - << "_maxNumTokens_" << info.param.maxNumTokens << "_maxOutputLength_" << info.param.maxOutputLength - << "_numRequests_" << info.param.numRequests << "_numTopKLogits_" << info.param.numTopKLogits - << "_promptLength_" << info.param.promptLength << "_randomSeed_" << info.param.randomSeed; - return nameStringStream.str(); -}; - -} // namespace - -using DecoderTopKGenerationLogitsFloatTest = DecoderTopKGenerationLogitsTest<float>; - -TEST_P(DecoderTopKGenerationLogitsFloatTest, TestSizeAndValues) -{ - runTopKGenerationLogitsTest(GetParam()); -} - -INSTANTIATE_TEST_SUITE_P(Float, DecoderTopKGenerationLogitsFloatTest, paramGenerator, nameSuffixGenerator); - -using DecoderTopKGenerationLogitsStreamingFloatTest = DecoderTopKGenerationLogitsStreamingTest<float>; - -TEST_P(DecoderTopKGenerationLogitsStreamingFloatTest, TestSizeAndValues) -{ - runTopKGenerationLogitsStreamingTest(GetParam()); -} - -INSTANTIATE_TEST_SUITE_P(Float, DecoderTopKGenerationLogitsStreamingFloatTest, paramGenerator, nameSuffixGenerator); - -using DecoderTopKContextLogitsStreamingFloatTest = DecoderTopKContextLogitsStreamingTest<float>; - -TEST_P(DecoderTopKContextLogitsStreamingFloatTest, TestSizeAndValues) -{ - runTopKContextLogitsTest(GetParam()); -} - -INSTANTIATE_TEST_SUITE_P( - Float, DecoderTopKContextLogitsStreamingFloatTest, paramGeneratorGatherContext, nameSuffixGenerator); - -using DecoderTopKContextLogitsFloatTest = DecoderTopKContextLogitsTest<float>; - -TEST_P(DecoderTopKContextLogitsFloatTest, TestSizeAndValues) -{ - runTopKContextLogitsTest(GetParam()); -} - -INSTANTIATE_TEST_SUITE_P(Float, DecoderTopKContextLogitsFloatTest, paramGeneratorGatherContext, nameSuffixGenerator); - -} // namespace tensorrt_llm::testing diff --git a/cpp/tests/unit_tests/executor/intervalSetTest.cpp b/cpp/tests/unit_tests/executor/intervalSetTest.cpp deleted file mode 100644 index a2bb0a8f7532..000000000000 --- a/cpp/tests/unit_tests/executor/intervalSetTest.cpp +++ /dev/null @@ -1,224 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/executor/intervalSet.h" - -#include <gtest/gtest.h> - -using tensorrt_llm::executor::IntervalSet; -using tensorrt_llm::executor::IdType; - -class IntervalSetTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) -{ -protected: - void SetUp() override {} - - void TearDown() override {} - - IntervalSet<IdType> mIntervalSet; -}; - -namespace -{ - -TEST_F(IntervalSetTest, testPublicAPI) -{ - EXPECT_FALSE(mIntervalSet.contains(0)); - EXPECT_EQ(mIntervalSet.getNumElements(), 0); - mIntervalSet.insert(0); - mIntervalSet.insert(1); - mIntervalSet.insert(4); - mIntervalSet.insert(6); - EXPECT_TRUE(mIntervalSet.contains(0)); - EXPECT_TRUE(mIntervalSet.contains(4)); - EXPECT_FALSE(mIntervalSet.contains(2125)); - EXPECT_EQ(mIntervalSet.getNumElements(), 4); - mIntervalSet.insert(6); - mIntervalSet.insert(4); - mIntervalSet.insert(1); - mIntervalSet.insert(0); - EXPECT_EQ(mIntervalSet.getNumElements(), 4); - EXPECT_TRUE(mIntervalSet.contains(0)); - EXPECT_TRUE(mIntervalSet.contains(4)); - EXPECT_FALSE(mIntervalSet.contains(9)); - EXPECT_FALSE(mIntervalSet.contains(3)); - EXPECT_FALSE(mIntervalSet.contains(11)); - mIntervalSet.clear(); - EXPECT_EQ(mIntervalSet.getNumElements(), 0); -} - -TEST_F(IntervalSetTest, testClear) -{ - for (int i = 0; i < 100; i++) - { - if (i % 2 == 0) - { - EXPECT_FALSE(mIntervalSet.contains(i)); - mIntervalSet.insert(i); - EXPECT_TRUE(mIntervalSet.contains(i)); - mIntervalSet.clear(); - EXPECT_EQ(mIntervalSet.getNumElements(), 0); - } - EXPECT_FALSE(mIntervalSet.contains(i)); - } - for (int i = 0; i < 100; i++) - { - EXPECT_FALSE(mIntervalSet.contains(i)); - } - mIntervalSet.clear(); - EXPECT_EQ(mIntervalSet.getNumElements(), 0); - mIntervalSet.clear(); - EXPECT_EQ(mIntervalSet.getNumElements(), 0); - - for (int i = 19; i >= 10; i--) - { - if (i % 2 == 0) - { - EXPECT_FALSE(mIntervalSet.contains(i)); - mIntervalSet.insert(i); - EXPECT_TRUE(mIntervalSet.contains(i)); - } - else - { - EXPECT_FALSE(mIntervalSet.contains(i)); - } - } - EXPECT_EQ(mIntervalSet.getNumElements(), 5); - mIntervalSet.clear(); - EXPECT_EQ(mIntervalSet.getNumElements(), 0); - mIntervalSet.clear(); - EXPECT_EQ(mIntervalSet.getNumElements(), 0); -} - -TEST_F(IntervalSetTest, testRandomInsert) -{ - mIntervalSet.insert(4); - mIntervalSet.insert(8); - EXPECT_EQ(mIntervalSet.getNumElements(), 2); - std::vector<int> idToAdd{9, 7, 5, 1, 6, 0, 2}; - for (auto id : idToAdd) - { - mIntervalSet.insert(id); - } - for (int i = 0; i < 10; i++) - { - if (i != 3) - { - EXPECT_TRUE(mIntervalSet.contains(i)); - } - else - { - EXPECT_FALSE(mIntervalSet.contains(i)); - } - } - EXPECT_EQ(mIntervalSet.getNumElements(), 9); - mIntervalSet.insert(3); - for (int i = 0; i < 10; i++) - { - EXPECT_TRUE(mIntervalSet.contains(i)); - } - EXPECT_EQ(mIntervalSet.getNumElements(), 10); - mIntervalSet.clear(); - EXPECT_EQ(mIntervalSet.getNumElements(), 0); -} - -TEST_F(IntervalSetTest, testTerminatedReqIdIntervals) -{ - mIntervalSet.insert(4); - mIntervalSet.insert(8); - EXPECT_EQ(mIntervalSet.getNumElements(), 2); - // terminatedReqIdIntervals is [[4, 4], [8, 8]] - EXPECT_EQ(mIntervalSet.getIntervals().size(), 2); - mIntervalSet.insert(3); - mIntervalSet.insert(5); - // terminatedReqIdIntervals is [[3, 5], [8, 8]] - EXPECT_EQ(mIntervalSet.getNumElements(), 4); - EXPECT_EQ(mIntervalSet.getIntervals().size(), 2); - mIntervalSet.insert(9); - mIntervalSet.insert(7); - // terminatedReqIdIntervals is [[3, 5], [7, 9]] - EXPECT_EQ(mIntervalSet.getNumElements(), 6); - EXPECT_EQ(mIntervalSet.getIntervals().size(), 2); - mIntervalSet.insert(6); - // terminatedReqIdIntervals is [[3, 9]] - EXPECT_EQ(mIntervalSet.getNumElements(), 7); - EXPECT_EQ(mIntervalSet.getIntervals().size(), 1); - mIntervalSet.insert(1); - // terminatedReqIdIntervals is [[1, 1], [3, 9]] - EXPECT_EQ(mIntervalSet.getNumElements(), 8); - EXPECT_EQ(mIntervalSet.getIntervals().size(), 2); - mIntervalSet.insert(0); - // terminatedReqIdIntervals is [[0, 1], [3, 9]] - EXPECT_EQ(mIntervalSet.getNumElements(), 9); - EXPECT_EQ(mIntervalSet.getIntervals().size(), 2); - mIntervalSet.insert(2); - // terminatedReqIdIntervals is [[0, 9]] - EXPECT_EQ(mIntervalSet.getNumElements(), 10); - EXPECT_EQ(mIntervalSet.getIntervals().size(), 1); - for (int i = 0; i < 10; i++) - { - mIntervalSet.insert(i); - // terminatedReqIdIntervals is always [[0, 9]] - EXPECT_EQ(mIntervalSet.getNumElements(), 10); - EXPECT_EQ(mIntervalSet.getIntervals().size(), 1); - } - mIntervalSet.clear(); - EXPECT_EQ(mIntervalSet.getNumElements(), 0); - EXPECT_EQ(mIntervalSet.getIntervals().size(), 0); - - // Insert continuous decreasing numbers. Interval size is always one. - for (int i = 19; i >= 10; i--) - { - mIntervalSet.insert(i); - EXPECT_TRUE(mIntervalSet.contains(i)); - EXPECT_EQ(mIntervalSet.getIntervals().size(), 1); - } - mIntervalSet.clear(); - EXPECT_EQ(mIntervalSet.getNumElements(), 0); - EXPECT_EQ(mIntervalSet.getIntervals().size(), 0); - - // Insert 50 disjoint even numbers - for (int i = 0; i < 100; i++) - { - if (i % 2 == 0) - { - mIntervalSet.insert(i); - EXPECT_EQ(mIntervalSet.getNumElements(), (i / 2) + 1); - EXPECT_EQ(mIntervalSet.getIntervals().size(), (i / 2) + 1); - } - } - - // Insert 50 disjoint odd numbers. Interval size should go down as the intervals are merged. - for (int i = 0; i < 100; i++) - { - if (i % 2 != 0) - { - mIntervalSet.insert(i); - EXPECT_EQ(mIntervalSet.getNumElements(), 50 + (i + 1) / 2); - if (i != 99) - { - EXPECT_EQ(mIntervalSet.getIntervals().size(), 50 - (i + 1) / 2); - } - else - { - EXPECT_EQ(mIntervalSet.getIntervals().size(), 1); - } - } - } -} - -} // namespace diff --git a/cpp/tests/unit_tests/executor/requestTest.cpp b/cpp/tests/unit_tests/executor/requestTest.cpp index f44f55b5141d..ec43d088657f 100644 --- a/cpp/tests/unit_tests/executor/requestTest.cpp +++ b/cpp/tests/unit_tests/executor/requestTest.cpp @@ -26,7 +26,9 @@ using ::testing::_; using ::testing::Invoke; using namespace tensorrt_llm::executor; -using namespace tensorrt_llm::common; +// Not a namespace-wide import: common also exports DataType/Dims, which would +// make the unqualified DataType (= executor::DataType) below ambiguous. +using tensorrt_llm::common::TllmException; TEST(RequestTest, validInputs) { diff --git a/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp b/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp index a39756bf7243..3c78659c87c4 100644 --- a/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp +++ b/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp @@ -18,6 +18,7 @@ #include "tensorrt_llm/executor/serializeUtils.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/executor.h" @@ -751,7 +752,8 @@ TEST(SerializeUtilsTest, ContextPhaseParams) { auto state = std::make_unique<texec::DataTransceiverState>(); state->setCommState(texec::kv_cache::CommState{12, "127.0.0.1"}); - state->setCacheState(texec::kv_cache::CacheState{10, 12, 128, 128, 8, 8, 8, {4}, nvinfer1::DataType::kFLOAT}); + state->setCacheState( + texec::kv_cache::CacheState{10, 12, 128, 128, 8, 8, 8, {4}, tensorrt_llm::DataType::kFLOAT}); auto stats = texec::ContextPhaseParams({10, 20, 30, 40, 50, 60}, 0, state.release(), VecTokens{10, 20}); auto stats2 = serializeDeserialize(stats); EXPECT_EQ(stats, stats2); @@ -1553,7 +1555,7 @@ TEST(SerializeUtilsTest, CacheStateIndexerKCache) texec::SizeType32 pp = 1; texec::SizeType32 cp = 1; std::vector<texec::SizeType32> attentionLayerNumPerPP{static_cast<texec::SizeType32>(nbKvHeadsPerLayer.size())}; - auto dataType = nvinfer1::DataType::kFLOAT; + auto dataType = tensorrt_llm::DataType::kFLOAT; auto attentionType = CacheState::AttentionType::kDEFAULT; int kvFactor = 2; bool enableAttentionDP = false; diff --git a/cpp/tests/unit_tests/executor/transferAgentTest.cpp b/cpp/tests/unit_tests/executor/transferAgentTest.cpp index b915ec3bb9c4..dcf6dd5a6e1e 100644 --- a/cpp/tests/unit_tests/executor/transferAgentTest.cpp +++ b/cpp/tests/unit_tests/executor/transferAgentTest.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -702,8 +702,7 @@ TEST(VmmDescSplitterTest, SplitTransferDescsDifferentChunkSizes) MemoryDescs srcInput{MemoryType::kVRAM, srcDescs}; MemoryDescs dstInput{MemoryType::kVRAM, dstDescs}; - auto [splitSrc, splitDst] - = VmmDescSplitter::splitTransferDescsWithRegionMaps(srcInput, dstInput, localMap, remoteMap); + auto [splitSrc, splitDst] = VmmDescSplitter::splitAndCoalesceTransferDescs(srcInput, dstInput, localMap, remoteMap); // dst has smaller chunks (512KB), so we get 4 pieces: 512K, 512K, 512K, 512K ASSERT_EQ(splitSrc.getDescs().size(), 4); @@ -735,8 +734,7 @@ TEST(VmmDescSplitterTest, SplitTransferDescsUnalignedBothSides) MemoryDescs srcInput{MemoryType::kVRAM, srcDescs}; MemoryDescs dstInput{MemoryType::kVRAM, dstDescs}; - auto [splitSrc, splitDst] - = VmmDescSplitter::splitTransferDescsWithRegionMaps(srcInput, dstInput, localMap, remoteMap); + auto [splitSrc, splitDst] = VmmDescSplitter::splitAndCoalesceTransferDescs(srcInput, dstInput, localMap, remoteMap); // src has 4MB chunks starting at srcBase → 1 piece from src side // dst has 2MB chunks starting at dstBase → 2 pieces from dst side @@ -760,7 +758,7 @@ TEST(VmmDescSplitterTest, SplitTransferDescsNoDstRegion) MemoryDescs dstInput{MemoryType::kVRAM, dstDescs}; auto [splitSrc, splitDst] - = VmmDescSplitter::splitTransferDescsWithRegionMaps(srcInput, dstInput, localMap, emptyRemoteMap); + = VmmDescSplitter::splitAndCoalesceTransferDescs(srcInput, dstInput, localMap, emptyRemoteMap); // Only src boundaries: 2 pieces of 1MB each ASSERT_EQ(splitSrc.getDescs().size(), 2); diff --git a/cpp/tests/unit_tests/executor/ucxCommTest.cpp b/cpp/tests/unit_tests/executor/ucxCommTest.cpp index 51d1d84d2676..d3695144cc92 100644 --- a/cpp/tests/unit_tests/executor/ucxCommTest.cpp +++ b/cpp/tests/unit_tests/executor/ucxCommTest.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,17 +36,21 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/mpi_utils/connection.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "gtest/gtest.h" +#include <atomic> +#include <chrono> #include <csignal> #include <cstddef> #include <cstdint> #include <cstdio> #include <cstdlib> +#include <future> #include <gmock/gmock.h> #include <memory> #include <random> @@ -90,6 +94,42 @@ class UcxCommTest : public ::testing::Test using DataContext = tensorrt_llm::executor::kv_cache::DataContext; using TransceiverTag = tensorrt_llm::batch_manager::TransceiverTag; +TEST_F(UcxCommTest, recvConnectCancellation) +{ + try + { + auto connectionManager = makeOneUcxConnectionManager(); + ASSERT_NE(connectionManager, nullptr); + + std::atomic<bool> transferTerminate{false}; + TransceiverTag::Id id; + auto receiveFuture = std::async(std::launch::async, + [&]() { + return connectionManager->recvConnect( + DataContext{TransceiverTag::kID_TAG, transferTerminate}, &id, sizeof(id)); + }); + + constexpr auto kReceiveStartPeriod = std::chrono::milliseconds{100}; + constexpr auto kCancellationTimeout = std::chrono::seconds{5}; + ASSERT_EQ(receiveFuture.wait_for(kReceiveStartPeriod), std::future_status::timeout); + + transferTerminate.store(true, std::memory_order_relaxed); + ASSERT_EQ(receiveFuture.wait_for(kCancellationTimeout), std::future_status::ready); + EXPECT_EQ(receiveFuture.get(), nullptr); + } + catch (std::exception const& e) + { + std::string error = e.what(); + if (error.find("UCX wrapper library is not open correctly") != std::string::npos + || error.find("Unable to load UCX wrapper library symbol") != std::string::npos) + { + GTEST_SKIP() << "UCX wrapper library is not open correctly. Skip this test case."; + } + + throw e; + } +} + TEST_F(UcxCommTest, Basic) { @@ -132,11 +172,11 @@ TEST_F(UcxCommTest, Basic) tensorrt_llm::runtime::BufferManager bufferManager{std::make_shared<tensorrt_llm::runtime::CudaStream>()}; // Create and fill source CUDA buffer with random data - auto srcBuffer = bufferManager.gpu(buffer.size(), nvinfer1::DataType::kINT8); + auto srcBuffer = bufferManager.gpu(buffer.size(), tensorrt_llm::DataType::kINT8); bufferManager.copy(buffer.data(), *srcBuffer); bufferManager.getStream().synchronize(); - auto dstBuffer = bufferManager.gpu(buffer.size(), nvinfer1::DataType::kINT8); + auto dstBuffer = bufferManager.gpu(buffer.size(), tensorrt_llm::DataType::kINT8); // Send CUDA buffer using connection1 connection1->send(DataContext{0x75}, srcBuffer->data(), srcBuffer->getSizeInBytes()); @@ -204,14 +244,14 @@ TEST_F(UcxCommTest, multiSend) tensorrt_llm::runtime::BufferManager bufferManager{std::make_shared<tensorrt_llm::runtime::CudaStream>()}; - auto srcBuffer1 = bufferManager.gpu(buffer1.size(), nvinfer1::DataType::kINT8); - auto srcBuffer2 = bufferManager.gpu(buffer2.size(), nvinfer1::DataType::kINT8); + auto srcBuffer1 = bufferManager.gpu(buffer1.size(), tensorrt_llm::DataType::kINT8); + auto srcBuffer2 = bufferManager.gpu(buffer2.size(), tensorrt_llm::DataType::kINT8); bufferManager.copy(buffer1.data(), *srcBuffer1); bufferManager.copy(buffer2.data(), *srcBuffer2); bufferManager.getStream().synchronize(); - auto dstBuffer1 = bufferManager.gpu(buffer1.size(), nvinfer1::DataType::kINT8); - auto dstBuffer2 = bufferManager.gpu(buffer2.size(), nvinfer1::DataType::kINT8); + auto dstBuffer1 = bufferManager.gpu(buffer1.size(), tensorrt_llm::DataType::kINT8); + auto dstBuffer2 = bufferManager.gpu(buffer2.size(), tensorrt_llm::DataType::kINT8); connection1Peer->send(DataContext{0x75}, srcBuffer1->data(), srcBuffer1->getSizeInBytes()); connection2Peer->send(DataContext{0x75}, srcBuffer2->data(), srcBuffer2->getSizeInBytes()); diff --git a/cpp/tests/unit_tests/kernels/CMakeLists.txt b/cpp/tests/unit_tests/kernels/CMakeLists.txt index e593ce4b76b8..5f121d2406a9 100644 --- a/cpp/tests/unit_tests/kernels/CMakeLists.txt +++ b/cpp/tests/unit_tests/kernels/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & +# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & # AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -35,9 +35,14 @@ if(USING_OSS_CUTLASS_MOE_GEMM) target_compile_definitions(mixtureOfExpertsTest PUBLIC USING_OSS_CUTLASS_MOE_GEMM) - add_gtest(mixtureOfExpertsInternalTest mixtureOfExpertsTest.cu) - remove_compile_definition(mixtureOfExpertsInternalTest - USING_OSS_CUTLASS_MOE_GEMM) + # The internal-path variant includes headers (quantization.h) that only ship + # with the internal cutlass kernels sources; the prebuilt tarball provides the + # library and a reduced header set only. + if(INTERNAL_CUTLASS_KERNELS_PATH) + add_gtest(mixtureOfExpertsInternalTest mixtureOfExpertsTest.cu) + remove_compile_definition(mixtureOfExpertsInternalTest + USING_OSS_CUTLASS_MOE_GEMM) + endif() endif() add_gtest(ropeTest ropeTest.cu) diff --git a/cpp/tests/unit_tests/kernels/banRepeatNGramsKernelsTest.cpp b/cpp/tests/unit_tests/kernels/banRepeatNGramsKernelsTest.cpp index 567cb95e8e44..83e94ea2d6ad 100644 --- a/cpp/tests/unit_tests/kernels/banRepeatNGramsKernelsTest.cpp +++ b/cpp/tests/unit_tests/kernels/banRepeatNGramsKernelsTest.cpp @@ -15,6 +15,7 @@ */ #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/banRepeatNgram.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/runtimeKernels.h" @@ -52,24 +53,25 @@ class BanRepeatNgramKernelsTest : public testing::Test SizeType32 const batchSize = outputIds.size(); auto const maxBatchSize = 2 * batchSize; - mLogits = BufferManager::pinned(ITensor::makeShape({batchSize, mVocabSizePadded}), nvinfer1::DataType::kFLOAT); + mLogits + = BufferManager::pinned(ITensor::makeShape({batchSize, mVocabSizePadded}), tensorrt_llm::DataType::kFLOAT); mSequenceLengths - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mBeamWidth}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mBeamWidth}), tensorrt_llm::DataType::kINT32); mFinished = BufferManager::pinned( ITensor::makeShape({maxBatchSize, mBeamWidth}), TRTDataType<tk::FinishedState::UnderlyingType>::value); mOutputIds = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxBatchSize, mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); mOutputIdsPtr = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mBeamWidth}), ptrType); mParentIds = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxBatchSize, mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); mParentIdsPtr = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mBeamWidth}), ptrType); - mNGramSizes = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mNGramSizes = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); auto batchSlotsPtr = bufferCast<int32_t>(*mBatchSlots); for (SizeType32 bi = 0; bi < batchSize; ++bi) diff --git a/cpp/tests/unit_tests/kernels/cudaCoreGemm/cudaCoreGemmKernelTest.cpp b/cpp/tests/unit_tests/kernels/cudaCoreGemm/cudaCoreGemmKernelTest.cpp index 05247e2d27a4..f7dabf98a93e 100644 --- a/cpp/tests/unit_tests/kernels/cudaCoreGemm/cudaCoreGemmKernelTest.cpp +++ b/cpp/tests/unit_tests/kernels/cudaCoreGemm/cudaCoreGemmKernelTest.cpp @@ -1,4 +1,3 @@ -#include <NvInferRuntime.h> #include <cublasLt.h> #include <cuda_fp8.h> #include <cuda_profiler_api.h> diff --git a/cpp/tests/unit_tests/kernels/decodingKernelTest.cpp b/cpp/tests/unit_tests/kernels/decodingKernelTest.cpp index 4b94e67cb5a1..a458f783ef65 100644 --- a/cpp/tests/unit_tests/kernels/decodingKernelTest.cpp +++ b/cpp/tests/unit_tests/kernels/decodingKernelTest.cpp @@ -15,6 +15,7 @@ */ #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/kernels/decodingKernels.h" #include "tensorrt_llm/kernels/speculativeDecoding/externalDraftTokensKernels.h" @@ -216,13 +217,15 @@ class TestBeamHypothesesCopy : public ::testing::Test srcBeams.empty(*mBufferManager); srcBeams.reshape(batchSize, beamWidth, maxSeqLen); - mSrcCumLogProbs = mBufferManager->gpu(ITensor::makeShape({batchSize, beamWidth}), nvinfer1::DataType::kFLOAT); + mSrcCumLogProbs + = mBufferManager->gpu(ITensor::makeShape({batchSize, beamWidth}), tensorrt_llm::DataType::kFLOAT); setBuffers(srcBeams, mSrcCumLogProbs, 2); dstBeams.empty(*mBufferManager); dstBeams.reshape(batchSize, beamWidth, maxSeqLen); - mDstCumLogProbs = mBufferManager->gpu(ITensor::makeShape({batchSize, beamWidth}), nvinfer1::DataType::kFLOAT); + mDstCumLogProbs + = mBufferManager->gpu(ITensor::makeShape({batchSize, beamWidth}), tensorrt_llm::DataType::kFLOAT); setBuffers(dstBeams, mDstCumLogProbs, 1); } @@ -544,7 +547,7 @@ class TestGatherTree : public ::testing::Test SizeType32 constexpr nbRnnLayers{0}; SizeType32 constexpr nbHeads{16}; SizeType32 constexpr hiddenSize{1024}; - nvinfer1::DataType constexpr dtype{nvinfer1::DataType::kFLOAT}; + tensorrt_llm::DataType constexpr dtype{tensorrt_llm::DataType::kFLOAT}; ModelConfig modelConfig{ vocabSize, nbAttentionLayers + nbRnnLayers, nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, dtype}; @@ -1139,32 +1142,35 @@ class DecodingKernelsTest : public testing::Test auto const ptrType = TRTDataType<T*>::value; mDraftTokens = mBufferManager->pinnedPool( - ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqlen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqlen}), tensorrt_llm::DataType::kINT32); mTargetTokens = mBufferManager->pinnedPool( - ITensor::makeShape({mMaxBatchSize, mMaxTargetSeqlen}), nvinfer1::DataType::kINT32); - mOutputTokens - = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxTargetSeqlen}), tensorrt_llm::DataType::kINT32); + mOutputTokens = mBufferManager->pinnedPool( + ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); mNumsDraftTokens = mBufferManager->pinnedPool( - ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqPerStep}), nvinfer1::DataType::kINT32); - mSequenceLengths = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mAcceptedLengths = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mContextLengths = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqPerStep}), tensorrt_llm::DataType::kINT32); + mSequenceLengths + = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mAcceptedLengths + = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mContextLengths + = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mFinishedSteps = mBufferManager->pinnedPool(ITensor::makeShape({mMaxDraftTokens + 1, mMaxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); mFinishedFinal = mBufferManager->pinnedPool( ITensor::makeShape({mMaxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mFinishedSum = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mFinishedSum = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mPaths = mBufferManager->pinnedPool( - ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqPerStep, mMaxDraftTokens}), nvinfer1::DataType::kINT32); - mEndIds = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqPerStep, mMaxDraftTokens}), tensorrt_llm::DataType::kINT32); + mEndIds = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mBatchSlots = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mBatchSlots = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); auto batchSlotsRange = BufferRange<SizeType32>(*mBatchSlots); std::iota(batchSlotsRange.begin(), batchSlotsRange.end(), 0); mCurandStates = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, sizeof(curandState_t)}), nvinfer1::DataType::kINT8); + ITensor::makeShape({mMaxBatchSize, sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); mAcceptedLen.resize(mMaxBatchSize); mOutputLen.resize(mMaxBatchSize); @@ -1194,8 +1200,9 @@ class DecodingKernelsTest : public testing::Test mMedusaInputLogitsPtrs = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize, mMaxNumHeads}), ptrType); mTokensPerStep - = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mBestPaths = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mBestPaths + = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); } } diff --git a/cpp/tests/unit_tests/kernels/eaglePackDataTest.cpp b/cpp/tests/unit_tests/kernels/eaglePackDataTest.cpp index bdf74efb59be..8ce24f813e65 100644 --- a/cpp/tests/unit_tests/kernels/eaglePackDataTest.cpp +++ b/cpp/tests/unit_tests/kernels/eaglePackDataTest.cpp @@ -25,9 +25,8 @@ #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/runtimeKernels.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include <NvInferRuntimeBase.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <algorithm> #include <cstdint> @@ -132,81 +131,81 @@ class EaglePackDataTest : public ::testing::Test { // inputs mBatchSlots = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); mInputTemperatures = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kFLOAT); mInputRandomDataSample = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kFLOAT); mInputRandomDataValidation = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kFLOAT); + tensorrt_llm::DataType::kFLOAT); mInputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mInputNextDraftPaths = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mInputSpecDecodingGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mInputSpecDecodingPositionOffsets = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); auto const numPackedMasks = static_cast<SizeType32>(tensorrt_llm::common::divUp(mSamplingParams.getMaxDecodingTokens(), 32)); mInputSpecDecodingPackedMasks = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), numPackedMasks}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); // outputs mOutputTemperatures = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kFLOAT); mOutputRandomDataSample = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kFLOAT); mOutputRandomDataValidation = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kFLOAT); + tensorrt_llm::DataType::kFLOAT); mOutputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mOutputNextDraftLens = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); mOutputNextDraftPaths = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mOutputSpecDecodingGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); mOutputSpecDecodingPositionOffsets = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mOutputSpecDecodingPackedMasks = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens(), numPackedMasks}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); // workspace - mMaxGenerationLength = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + mMaxGenerationLength = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); mCumSumGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize() + 1}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getBatchSize() + 1}), tensorrt_llm::DataType::kINT32); mScanReduceTempStorageBytes = tksd::invokeScanReduceGenerationLengths( mSamplingParams.getBatchSize(), nullptr, nullptr, 0, nullptr, nullptr, mStream->get()); diff --git a/cpp/tests/unit_tests/kernels/mixtureOfExpertsTest.cu b/cpp/tests/unit_tests/kernels/mixtureOfExpertsTest.cu index 01cd1c4d792d..61c182dfd1ea 100644 --- a/cpp/tests/unit_tests/kernels/mixtureOfExpertsTest.cu +++ b/cpp/tests/unit_tests/kernels/mixtureOfExpertsTest.cu @@ -32,6 +32,7 @@ #endif #include "tensorrt_llm/kernels/cutlass_kernels/include/cutlass_kernel_selector.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include <tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h> @@ -2508,31 +2509,31 @@ constexpr static auto typeToDtypeID() { if constexpr (std::is_same_v<T, SafeFP8>) { - return nvinfer1::DataType::kFP8; + return tensorrt_llm::DataType::kFP8; } else if constexpr (std::is_same_v<T, SafeFP4>) { - return nvinfer1::DataType::kFP4; + return tensorrt_llm::DataType::kFP4; } else if constexpr (std::is_same_v<T, uint8_t>) { - return nvinfer1::DataType::kINT8; + return tensorrt_llm::DataType::kINT8; } else if constexpr (std::is_same_v<T, cutlass::uint4b_t>) { - return nvinfer1::DataType::kINT4; + return tensorrt_llm::DataType::kINT4; } else if constexpr (std::is_same_v<T, nv_bfloat16>) { - return nvinfer1::DataType::kBF16; + return tensorrt_llm::DataType::kBF16; } else if constexpr (std::is_same_v<T, half>) { - return nvinfer1::DataType::kHALF; + return tensorrt_llm::DataType::kHALF; } else if constexpr (std::is_same_v<T, float>) { - return nvinfer1::DataType::kFLOAT; + return tensorrt_llm::DataType::kFLOAT; } else { @@ -2602,14 +2603,16 @@ TEST_F(MixtureOfExpertsProfilerTest, TestGeneratedProfilerDistribution) for (int ep : {1, 4, 8}) { #ifdef USING_OSS_CUTLASS_MOE_GEMM - backend.init(this->mMoERunner, GemmProfilerBackend::GemmToProfile::GEMM_1, nvinfer1::DataType::kHALF, - nvinfer1::DataType::kHALF, nvinfer1::DataType::kHALF, num_experts, k, 1024, 1024, 4096, mGroupSize, {}, - false, mUseLora, /*min_latency_mode=*/false, /*need_weights=*/true, MOEParallelismConfig{1, 0, ep, 0}, + backend.init(this->mMoERunner, GemmProfilerBackend::GemmToProfile::GEMM_1, tensorrt_llm::DataType::kHALF, + tensorrt_llm::DataType::kHALF, tensorrt_llm::DataType::kHALF, num_experts, k, 1024, 1024, 4096, + mGroupSize, {}, false, mUseLora, /*min_latency_mode=*/false, /*need_weights=*/true, + MOEParallelismConfig{1, 0, ep, 0}, /*enable_alltoall=*/false); #else - backend.init(this->mMoERunner, GemmProfilerBackend::GemmToProfile::GEMM_1, nvinfer1::DataType::kHALF, - nvinfer1::DataType::kHALF, nvinfer1::DataType::kHALF, num_experts, k, 1024, 4096, mGroupSize, {}, false, - mUseLora, /*min_latency_mode=*/false, /*need_weights=*/true, MOEParallelismConfig{1, 0, ep, ep - 1}); + backend.init(this->mMoERunner, GemmProfilerBackend::GemmToProfile::GEMM_1, tensorrt_llm::DataType::kHALF, + tensorrt_llm::DataType::kHALF, tensorrt_llm::DataType::kHALF, num_experts, k, 1024, 4096, mGroupSize, + {}, false, mUseLora, /*min_latency_mode=*/false, /*need_weights=*/true, + MOEParallelismConfig{1, 0, ep, ep - 1}); #endif auto ws_size = backend.getWorkspaceSize(num_tokens); diff --git a/cpp/tests/unit_tests/kernels/mlaChunkedPrefillTest.cu b/cpp/tests/unit_tests/kernels/mlaChunkedPrefillTest.cu index 3e4e9a1da0a3..c1fa77239729 100644 --- a/cpp/tests/unit_tests/kernels/mlaChunkedPrefillTest.cu +++ b/cpp/tests/unit_tests/kernels/mlaChunkedPrefillTest.cu @@ -8,6 +8,7 @@ #include "tensorrt_llm/kernels/kvCacheUtils.h" #include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/mlaChunkedPrefill.cuh" #include "tensorrt_llm/runtime/cudaStream.h" #include <cstring> @@ -429,18 +430,18 @@ protected: using tensorrt_llm::runtime::ITensor; using tensorrt_llm::runtime::bufferCast; - auto dtype = nvinfer1::DataType::kHALF; + auto dtype = tensorrt_llm::DataType::kHALF; if constexpr (std::is_same_v<DataType, float>) { - dtype = nvinfer1::DataType::kFLOAT; + dtype = tensorrt_llm::DataType::kFLOAT; } else if constexpr (std::is_same_v<DataType, half>) { - dtype = nvinfer1::DataType::kHALF; + dtype = tensorrt_llm::DataType::kHALF; } else if constexpr (std::is_same_v<DataType, __nv_bfloat16>) { - dtype = nvinfer1::DataType::kBF16; + dtype = tensorrt_llm::DataType::kBF16; } else { @@ -449,11 +450,11 @@ protected: auto cacheType = dtype; if constexpr (std::is_same_v<TCache, __nv_fp8_e4m3>) { - cacheType = nvinfer1::DataType::kFP8; + cacheType = tensorrt_llm::DataType::kFP8; this->h_kv_scale_quant_orig - = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); - this->d_kv_scale_quant_orig - = tensorrt_llm::runtime::BufferManager::gpuSync(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); + = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); + this->d_kv_scale_quant_orig = tensorrt_llm::runtime::BufferManager::gpuSync( + ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); auto* kv_scale_quant_orig_ptr = bufferCast<float>(*(this->h_kv_scale_quant_orig)); float kv_scale_orig_quant = 2.0F; kv_scale_quant_orig_ptr[0] = 1.0 / kv_scale_orig_quant; @@ -463,13 +464,13 @@ protected: // cu lens this->h_cu_kv_seq_lens = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mBatchSize + 1}), nvinfer1::DataType::kINT64); + ITensor::makeShape({this->mBatchSize + 1}), tensorrt_llm::DataType::kINT64); this->h_cu_q_seq_lens = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mBatchSize + 1}), nvinfer1::DataType::kINT64); + ITensor::makeShape({this->mBatchSize + 1}), tensorrt_llm::DataType::kINT64); this->d_cu_kv_seq_lens = tensorrt_llm::runtime::BufferManager::gpuSync( - this->h_cu_kv_seq_lens->getShape(), nvinfer1::DataType::kINT64); + this->h_cu_kv_seq_lens->getShape(), tensorrt_llm::DataType::kINT64); this->d_cu_q_seq_lens = tensorrt_llm::runtime::BufferManager::gpuSync( - this->h_cu_q_seq_lens->getShape(), nvinfer1::DataType::kINT64); + this->h_cu_q_seq_lens->getShape(), tensorrt_llm::DataType::kINT64); { this->mMaxSeqLen = 0; this->mMaxQSeqLen = 0; @@ -512,14 +513,14 @@ protected: int const total_cached_kv_len = this->mTotalKVLen - this->mTotalQLen; int const chunked_loop_num = (total_cached_kv_len + total_chunk_size - 1) / total_chunk_size; this->h_cu_chunk_lens = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize + 1}), nvinfer1::DataType::kINT64); + ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize + 1}), tensorrt_llm::DataType::kINT64); this->h_chunked_ld_global_offset = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize}), nvinfer1::DataType::kINT64); + ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize}), tensorrt_llm::DataType::kINT64); this->memsetZeroHost(this->h_chunked_ld_global_offset); this->d_cu_chunk_lens = tensorrt_llm::runtime::BufferManager::gpuSync( - this->h_cu_chunk_lens->getShape(), nvinfer1::DataType::kINT64); + this->h_cu_chunk_lens->getShape(), tensorrt_llm::DataType::kINT64); this->d_chunked_ld_global_offset = tensorrt_llm::runtime::BufferManager::gpuSync( - this->h_chunked_ld_global_offset->getShape(), nvinfer1::DataType::kINT64); + this->h_chunked_ld_global_offset->getShape(), tensorrt_llm::DataType::kINT64); // kv cache this->mMaxBlockPerSeq = (this->mMaxSeqLen + this->mTokensPerBlock - 1) / this->mTokensPerBlock; @@ -539,13 +540,13 @@ protected: this->mLoraSize + this->mRopeSize}), cacheType); this->h_compressed_offset_tensor = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mBatchSize, 2, this->mMaxBlockPerSeq + 1}), nvinfer1::DataType::kINT32); + ITensor::makeShape({this->mBatchSize, 2, this->mMaxBlockPerSeq + 1}), tensorrt_llm::DataType::kINT32); this->d_kv_cache_tensor = tensorrt_llm::runtime::BufferManager::gpuSync(this->h_kv_cache_tensor->getShape(), dtype); this->d_compressed_kv_cache_tensor = tensorrt_llm::runtime::BufferManager::gpuSync(this->h_compressed_kv_cache_tensor->getShape(), cacheType); this->d_compressed_offset_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( - this->h_compressed_offset_tensor->getShape(), nvinfer1::DataType::kINT32); + this->h_compressed_offset_tensor->getShape(), tensorrt_llm::DataType::kINT32); { auto* compressed_kv_cache_ptr = bufferCast<TCache>(*(this->h_compressed_kv_cache_tensor)); @@ -601,15 +602,15 @@ protected: this->m_h_output_tensor = tensorrt_llm::runtime::BufferManager::pinned( ITensor::makeShape({this->mTotalQLen, this->mNumHeads, this->mNopeSize}), dtype); this->m_h_softmax_sum_tensor = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({2, this->mTotalQLen, this->mNumHeads}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({2, this->mTotalQLen, this->mNumHeads}), tensorrt_llm::DataType::kFLOAT); this->m_h_softmax_sum_accum_tensor = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({2, this->mTotalQLen, this->mNumHeads}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({2, this->mTotalQLen, this->mNumHeads}), tensorrt_llm::DataType::kFLOAT); this->m_h_output_tensor_ref = tensorrt_llm::runtime::BufferManager::pinned( ITensor::makeShape({this->mTotalQLen, this->mNumHeads, this->mNopeSize}), dtype); this->m_h_output_tensor_accum = tensorrt_llm::runtime::BufferManager::pinned( ITensor::makeShape({this->mTotalQLen, this->mNumHeads, this->mNopeSize}), dtype); this->m_h_merge_op = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize}), nvinfer1::DataType::kINT64); + ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize}), tensorrt_llm::DataType::kINT64); this->m_d_q_tensor = tensorrt_llm::runtime::BufferManager::gpuSync(this->m_h_q_tensor->getShape(), dtype); this->m_d_kv_full_tensor = tensorrt_llm::runtime::BufferManager::gpuSync(this->m_h_kv_full_tensor->getShape(), dtype); @@ -618,13 +619,13 @@ protected: this->m_d_output_tensor = tensorrt_llm::runtime::BufferManager::gpuSync(this->m_h_output_tensor->getShape(), dtype); this->m_d_softmax_sum_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( - this->m_h_softmax_sum_tensor->getShape(), nvinfer1::DataType::kFLOAT); + this->m_h_softmax_sum_tensor->getShape(), tensorrt_llm::DataType::kFLOAT); this->m_d_softmax_sum_accum_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( - this->m_h_softmax_sum_accum_tensor->getShape(), nvinfer1::DataType::kFLOAT); + this->m_h_softmax_sum_accum_tensor->getShape(), tensorrt_llm::DataType::kFLOAT); this->m_d_output_tensor_accum = tensorrt_llm::runtime::BufferManager::gpuSync(this->m_h_output_tensor_accum->getShape(), dtype); - this->m_d_merge_op - = tensorrt_llm::runtime::BufferManager::gpuSync(this->m_h_merge_op->getShape(), nvinfer1::DataType::kINT64); + this->m_d_merge_op = tensorrt_llm::runtime::BufferManager::gpuSync( + this->m_h_merge_op->getShape(), tensorrt_llm::DataType::kINT64); { auto* q_ptr = bufferCast<DataType>(*(this->m_h_q_tensor)); diff --git a/cpp/tests/unit_tests/kernels/mlaPreprocessTest.cu b/cpp/tests/unit_tests/kernels/mlaPreprocessTest.cu index f2c0863779bc..3fc249a2f24a 100644 --- a/cpp/tests/unit_tests/kernels/mlaPreprocessTest.cu +++ b/cpp/tests/unit_tests/kernels/mlaPreprocessTest.cu @@ -23,6 +23,7 @@ #include "tensorrt_llm/kernels/kvCacheUtils.h" #include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/mlaKernels.h" #include <random> @@ -232,18 +233,18 @@ protected: using tensorrt_llm::runtime::ITensor; using tensorrt_llm::runtime::bufferCast; - auto dtype = nvinfer1::DataType::kHALF; + auto dtype = tensorrt_llm::DataType::kHALF; if constexpr (std::is_same_v<DataType, float>) { - dtype = nvinfer1::DataType::kFLOAT; + dtype = tensorrt_llm::DataType::kFLOAT; } else if constexpr (std::is_same_v<DataType, half>) { - dtype = nvinfer1::DataType::kHALF; + dtype = tensorrt_llm::DataType::kHALF; } else if constexpr (std::is_same_v<DataType, __nv_bfloat16>) { - dtype = nvinfer1::DataType::kBF16; + dtype = tensorrt_llm::DataType::kBF16; } else { @@ -252,15 +253,15 @@ protected: auto cache_dtype = dtype; if constexpr (std::is_same_v<TCache, __nv_fp8_e4m3>) { - cache_dtype = nvinfer1::DataType::kFP8; + cache_dtype = tensorrt_llm::DataType::kFP8; this->h_kv_scale_orig_quant - = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); - this->d_kv_scale_orig_quant - = tensorrt_llm::runtime::BufferManager::gpuSync(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); + = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); + this->d_kv_scale_orig_quant = tensorrt_llm::runtime::BufferManager::gpuSync( + ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); this->h_kv_scale_quant_orig - = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); - this->d_kv_scale_quant_orig - = tensorrt_llm::runtime::BufferManager::gpuSync(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); + = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); + this->d_kv_scale_quant_orig = tensorrt_llm::runtime::BufferManager::gpuSync( + ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); auto* kv_scale_orig_quant_ptr = bufferCast<float>(*(this->h_kv_scale_orig_quant)); auto* kv_scale_quant_orig_ptr = bufferCast<float>(*(this->h_kv_scale_quant_orig)); float kv_scale_orig_quant = 2.0f; @@ -276,13 +277,13 @@ protected: static_assert(std::is_same_v<DataType, TCache>, "TCache must be the same type as DataType"); } this->h_cu_seq_lens = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mNumRequests + 1}), nvinfer1::DataType::kINT64); + ITensor::makeShape({this->mNumRequests + 1}), tensorrt_llm::DataType::kINT64); this->h_cu_ctx_cached_kv_lens = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mNumRequests + 1}), nvinfer1::DataType::kINT64); + ITensor::makeShape({this->mNumRequests + 1}), tensorrt_llm::DataType::kINT64); this->d_cu_seq_lens = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({this->mNumRequests + 1}), nvinfer1::DataType::kINT64); + ITensor::makeShape({this->mNumRequests + 1}), tensorrt_llm::DataType::kINT64); this->d_cu_ctx_cached_kv_lens = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({this->mNumRequests + 1}), nvinfer1::DataType::kINT64); + ITensor::makeShape({this->mNumRequests + 1}), tensorrt_llm::DataType::kINT64); { // set random sequence length auto* cu_seq_lens_temp_ptr = bufferCast<int64_t>(*(this->h_cu_seq_lens)); @@ -333,9 +334,9 @@ protected: this->mTokensPerBlock, this->mLoraSize + this->mRopeSize}), cache_dtype); this->h_offset_tensor = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), nvinfer1::DataType::kINT32); + ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), tensorrt_llm::DataType::kINT32); this->h_compressed_offset_tensor = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), nvinfer1::DataType::kINT32); + ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), tensorrt_llm::DataType::kINT32); this->d_kv_cache_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq, this->mNumHeadsUncompressed, this->mTokensPerBlock, this->mUncompressedHeadSize + this->mRopeSize}), @@ -349,9 +350,9 @@ protected: this->mTokensPerBlock, this->mLoraSize + this->mRopeSize}), cache_dtype); this->d_offset_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), nvinfer1::DataType::kINT32); + ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), tensorrt_llm::DataType::kINT32); this->d_compressed_offset_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), nvinfer1::DataType::kINT32); + ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), tensorrt_llm::DataType::kINT32); { auto* kv_cache_ptr = bufferCast<DataType>(*(this->h_kv_cache_tensor)); auto* kv_cache_ref_ptr = bufferCast<DataType>(*(this->h_kv_cache_tensor_ref)); diff --git a/cpp/tests/unit_tests/kernels/prepareCustomMaskTest.cpp b/cpp/tests/unit_tests/kernels/prepareCustomMaskTest.cpp index 61617934f236..6616c9c47668 100644 --- a/cpp/tests/unit_tests/kernels/prepareCustomMaskTest.cpp +++ b/cpp/tests/unit_tests/kernels/prepareCustomMaskTest.cpp @@ -25,6 +25,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaKernels.h" #include "tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaRunnerParams.h" #include "tensorrt_llm/kernels/trtllmGenKernels/fmha/prepareCustomMask.h" @@ -293,8 +294,8 @@ class PrepareCustomMaskTest : public ::testing::Test int64_t totalMaskSize = static_cast<int64_t>(batchSize) * maxNumTilesQ * maxNumCustomMaskTilesKv * numInstsQ * numInstsKv * (tileSizeQ * tileSizeKvPadded) / 32; - auto customMaskOffsetsDevice = mBufferManager->gpu(batchSize, nvinfer1::DataType::kINT64); - auto customMaskDevice = mBufferManager->gpu(totalMaskSize, nvinfer1::DataType::kINT32); + auto customMaskOffsetsDevice = mBufferManager->gpu(batchSize, tensorrt_llm::DataType::kINT64); + auto customMaskDevice = mBufferManager->gpu(totalMaskSize, tensorrt_llm::DataType::kINT32); // Clear GPU buffers to ensure no stale data from previous tests cudaMemsetAsync(bufferCast<int64_t>(*customMaskOffsetsDevice), 0, batchSize * sizeof(int64_t), mStream->get()); diff --git a/cpp/tests/unit_tests/kernels/ropeTest.cu b/cpp/tests/unit_tests/kernels/ropeTest.cu index 517b006e4fde..36c91481722f 100644 --- a/cpp/tests/unit_tests/kernels/ropeTest.cu +++ b/cpp/tests/unit_tests/kernels/ropeTest.cu @@ -15,11 +15,13 @@ */ #include <gtest/gtest.h> +#include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" +#include "tensorrt_llm/kernels/gptKernels.h" #include "tensorrt_llm/kernels/kvCacheUtils.h" #include "tensorrt_llm/kernels/unfusedAttentionKernels.h" -#include "tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h" #include "tensorrt_llm/runtime/bufferManager.h" #include <random> @@ -29,6 +31,7 @@ #include <cuda_fp4.h> #endif +using namespace tensorrt_llm::common; using namespace tensorrt_llm::runtime; using namespace tensorrt_llm::kernels; @@ -502,26 +505,27 @@ protected: { auto const cu_seqlens_size = batch_size + 1; - cu_q_seqlens_tensor = mBufferManager->pinned(ITensor::makeShape({cu_seqlens_size}), nvinfer1::DataType::kINT32); + cu_q_seqlens_tensor + = mBufferManager->pinned(ITensor::makeShape({cu_seqlens_size}), tensorrt_llm::DataType::kINT32); cu_kv_seqlens_tensor - = mBufferManager->pinned(ITensor::makeShape({cu_seqlens_size}), nvinfer1::DataType::kINT32); - padding_offset_tensor - = mBufferManager->pinned(ITensor::makeShape({batch_size, input_seq_length}), nvinfer1::DataType::kINT32); - encoder_padding_offset_tensor - = mBufferManager->pinned(ITensor::makeShape({batch_size, cross_qkv_length}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({cu_seqlens_size}), tensorrt_llm::DataType::kINT32); + padding_offset_tensor = mBufferManager->pinned( + ITensor::makeShape({batch_size, input_seq_length}), tensorrt_llm::DataType::kINT32); + encoder_padding_offset_tensor = mBufferManager->pinned( + ITensor::makeShape({batch_size, cross_qkv_length}), tensorrt_llm::DataType::kINT32); fmha_tile_counter_ptr_tensor - = mBufferManager->pinned(ITensor::makeShape({mEnableContextFMHA ? 1 : 0}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({mEnableContextFMHA ? 1 : 0}), tensorrt_llm::DataType::kINT32); rotary_inv_freq_buf_tensor = mBufferManager->pinned( - ITensor::makeShape({batch_size, mRotaryEmbeddingDim / 2}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({batch_size, mRotaryEmbeddingDim / 2}), tensorrt_llm::DataType::kFLOAT); int const max_num_tokens = batch_size * input_seq_length; tokens_info_tensor - = mBufferManager->pinned(ITensor::makeShape({max_num_tokens, 2}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({max_num_tokens, 2}), tensorrt_llm::DataType::kINT32); #ifdef ENABLE_FP4 if constexpr (std::is_same_v<KVCacheType, __nv_fp4_e2m1>) { - global_scale_tensor = mBufferManager->pinned(ITensor::makeShape({2}), nvinfer1::DataType::kFLOAT); + global_scale_tensor = mBufferManager->pinned(ITensor::makeShape({2}), tensorrt_llm::DataType::kFLOAT); } #endif } @@ -584,7 +588,7 @@ protected: // // Rotary cos sin cache buffer to avoid re-computing. SizeType32 maxOutputSize{generateRandomSizeSmallerThan(1024)}; rotary_cos_sin_tensor = this->mBufferManager->pinned( - ITensor::makeShape({mRotaryEmbeddingMaxPositions, mRotaryEmbeddingDim}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({mRotaryEmbeddingMaxPositions, mRotaryEmbeddingDim}), tensorrt_llm::DataType::kFLOAT); rotary_fill_help = bufferCast<float>(*(rotary_cos_sin_tensor)); // createCosSinBuf(rotary_fill_help, mRotaryEmbeddingMaxPositions, mRotaryEmbeddingDim); //currently broken // fillWithOnesAndZerosInterleaved(rotary_fill_help, mRotaryEmbeddingMaxPositions* @@ -594,7 +598,7 @@ protected: batch_size = generateRandomSizeSmallerThan(12); - q_seq_lengths_tensor = mBufferManager->pinned(ITensor::makeShape({batch_size}), nvinfer1::DataType::kINT32); + q_seq_lengths_tensor = mBufferManager->pinned(ITensor::makeShape({batch_size}), tensorrt_llm::DataType::kINT32); q_seq_lengths = bufferCast<int32_t>(*(q_seq_lengths_tensor)); for (SizeType32 ii = 0; ii < batch_size; ++ii) diff --git a/cpp/tests/unit_tests/kernels/routing/routingDeepSeekTest.cpp b/cpp/tests/unit_tests/kernels/routing/routingDeepSeekTest.cpp index 78598fa4c417..be5c8f3c48d1 100644 --- a/cpp/tests/unit_tests/kernels/routing/routingDeepSeekTest.cpp +++ b/cpp/tests/unit_tests/kernels/routing/routingDeepSeekTest.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "tensorrt_llm/common/tllmDataType.h" #include "tests/unit_tests/kernels/routing/routingTest.h" namespace tk = tensorrt_llm::kernels; @@ -156,8 +157,8 @@ class RoutingDeepSeekKernelTest : public RoutingKernelTest<T> { RoutingKernelTest<T>::allocateBuffers(param); int64_t scoresSize = param.numTokens * param.numExperts; - this->mPtrScoresHost = mBufferManager->pinned(ITensor::makeShape({scoresSize}), nvinfer1::DataType::kFLOAT); - this->mPtrScoresDevice = mBufferManager->gpu(ITensor::makeShape({scoresSize}), nvinfer1::DataType::kFLOAT); + this->mPtrScoresHost = mBufferManager->pinned(ITensor::makeShape({scoresSize}), tensorrt_llm::DataType::kFLOAT); + this->mPtrScoresDevice = mBufferManager->gpu(ITensor::makeShape({scoresSize}), tensorrt_llm::DataType::kFLOAT); this->mPtrRoutingBiasHost = mBufferManager->pinned(ITensor::makeShape({param.numExperts}), TRTDataType<T>::value); @@ -430,9 +431,9 @@ TYPED_TEST(RoutingDeepSeekKernelTest, ClusterLevelWithFloat32Bias) // the GPU kernel (using fp32 bias) and the host reference (using T-typed bias) // observe numerically equivalent inputs. auto float32BiasHost - = this->mBufferManager->pinned(ITensor::makeShape({param.numExperts}), nvinfer1::DataType::kFLOAT); + = this->mBufferManager->pinned(ITensor::makeShape({param.numExperts}), tensorrt_llm::DataType::kFLOAT); auto float32BiasDevice - = this->mBufferManager->gpu(ITensor::makeShape({param.numExperts}), nvinfer1::DataType::kFLOAT); + = this->mBufferManager->gpu(ITensor::makeShape({param.numExperts}), tensorrt_llm::DataType::kFLOAT); auto fp32BiasPtr = bufferCast<float>(*float32BiasHost); auto tBiasPtr = bufferCast<TypeParam>(*this->mPtrRoutingBiasHost); for (int i = 0; i < param.numExperts; i++) diff --git a/cpp/tests/unit_tests/kernels/routing/routingTest.cpp b/cpp/tests/unit_tests/kernels/routing/routingTest.cpp index ba5c020ade9e..c71510dc319c 100644 --- a/cpp/tests/unit_tests/kernels/routing/routingTest.cpp +++ b/cpp/tests/unit_tests/kernels/routing/routingTest.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ #include "tests/unit_tests/kernels/routing/routingTest.h" +#include "tensorrt_llm/common/tllmDataType.h" namespace tensorrt_llm::tests::kernels::routing { @@ -54,25 +55,25 @@ void RoutingKernelTest<T>::allocateBuffers(RoutingKernelTestParam const& param) { countsSize = 2 * 256; } - mPtrExpertCountsHost = mBufferManager->pinned(ITensor::makeShape({countsSize}), nvinfer1::DataType::kINT32); - mPtrExpertCountsDevice = mBufferManager->gpu(ITensor::makeShape({countsSize}), nvinfer1::DataType::kINT32); + mPtrExpertCountsHost = mBufferManager->pinned(ITensor::makeShape({countsSize}), tensorrt_llm::DataType::kINT32); + mPtrExpertCountsDevice = mBufferManager->gpu(ITensor::makeShape({countsSize}), tensorrt_llm::DataType::kINT32); int64_t permIdxSize = 1; - mPtrPermutedIdxSizeHost = mBufferManager->pinned(ITensor::makeShape({permIdxSize}), nvinfer1::DataType::kINT32); - mPtrPermutedIdxSizeDevice = mBufferManager->gpu(ITensor::makeShape({permIdxSize}), nvinfer1::DataType::kINT32); + mPtrPermutedIdxSizeHost = mBufferManager->pinned(ITensor::makeShape({permIdxSize}), tensorrt_llm::DataType::kINT32); + mPtrPermutedIdxSizeDevice = mBufferManager->gpu(ITensor::makeShape({permIdxSize}), tensorrt_llm::DataType::kINT32); int64_t expIdxToPermIdxSize = numTokens * topK; mPtrExpandedIdxToPermutedIdxHost - = mBufferManager->pinned(ITensor::makeShape({expIdxToPermIdxSize}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({expIdxToPermIdxSize}), tensorrt_llm::DataType::kINT32); mPtrExpandedIdxToPermutedIdxDevice - = mBufferManager->gpu(ITensor::makeShape({expIdxToPermIdxSize}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({expIdxToPermIdxSize}), tensorrt_llm::DataType::kINT32); // int64_t permIdxToTokenIdxSize = (numTokens * topK + (numExperts << paddingLog2) - numExperts); int64_t permIdxToTokenIdxSize = (numTokens * topK + (numExperts * tileTokensDim) - numExperts); mPtrPermutedIdxToTokenIdxHost - = mBufferManager->pinned(ITensor::makeShape({permIdxToTokenIdxSize}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({permIdxToTokenIdxSize}), tensorrt_llm::DataType::kINT32); mPtrPermutedIdxToTokenIdxDevice - = mBufferManager->gpu(ITensor::makeShape({permIdxToTokenIdxSize}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({permIdxToTokenIdxSize}), tensorrt_llm::DataType::kINT32); int64_t expWeightsSize = numTokens * topK; mPtrTopKWeightsHost = mBufferManager->pinned(ITensor::makeShape({expWeightsSize}), TRTDataType<T>::value); @@ -81,8 +82,8 @@ void RoutingKernelTest<T>::allocateBuffers(RoutingKernelTestParam const& param) if (useTopKAsInput) { int64_t topKIdsSize = numTokens * topK; - mPtrTopKIdsHost = mBufferManager->pinned(ITensor::makeShape({topKIdsSize}), nvinfer1::DataType::kINT32); - mPtrTopKIdsDevice = mBufferManager->gpu(ITensor::makeShape({topKIdsSize}), nvinfer1::DataType::kINT32); + mPtrTopKIdsHost = mBufferManager->pinned(ITensor::makeShape({topKIdsSize}), tensorrt_llm::DataType::kINT32); + mPtrTopKIdsDevice = mBufferManager->gpu(ITensor::makeShape({topKIdsSize}), tensorrt_llm::DataType::kINT32); } else { @@ -91,23 +92,26 @@ void RoutingKernelTest<T>::allocateBuffers(RoutingKernelTestParam const& param) } int64_t ctaIdxSize = numTokens * topK; - mPtrCtaIdxXyToBatchIdxHost = mBufferManager->pinned(ITensor::makeShape({ctaIdxSize}), nvinfer1::DataType::kINT32); - mPtrCtaIdxXyToBatchIdxDevice = mBufferManager->gpu(ITensor::makeShape({ctaIdxSize}), nvinfer1::DataType::kINT32); + mPtrCtaIdxXyToBatchIdxHost + = mBufferManager->pinned(ITensor::makeShape({ctaIdxSize}), tensorrt_llm::DataType::kINT32); + mPtrCtaIdxXyToBatchIdxDevice + = mBufferManager->gpu(ITensor::makeShape({ctaIdxSize}), tensorrt_llm::DataType::kINT32); - mPtrCtaIdxXyToMnLimitHost = mBufferManager->pinned(ITensor::makeShape({ctaIdxSize}), nvinfer1::DataType::kINT32); - mPtrCtaIdxXyToMnLimitDevice = mBufferManager->gpu(ITensor::makeShape({ctaIdxSize}), nvinfer1::DataType::kINT32); + mPtrCtaIdxXyToMnLimitHost + = mBufferManager->pinned(ITensor::makeShape({ctaIdxSize}), tensorrt_llm::DataType::kINT32); + mPtrCtaIdxXyToMnLimitDevice = mBufferManager->gpu(ITensor::makeShape({ctaIdxSize}), tensorrt_llm::DataType::kINT32); int64_t numNonExitingCtasSize = 1; mPtrNumNonExitingCtasHost - = mBufferManager->pinned(ITensor::makeShape({numNonExitingCtasSize}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({numNonExitingCtasSize}), tensorrt_llm::DataType::kINT32); mPtrNumNonExitingCtasDevice - = mBufferManager->gpu(ITensor::makeShape({numNonExitingCtasSize}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({numNonExitingCtasSize}), tensorrt_llm::DataType::kINT32); int64_t idxSize = numTokens * topK * sizeof(PackedType); - mPtrTopKPackedHost = mBufferManager->pinned(ITensor::makeShape({idxSize}), nvinfer1::DataType::kINT8); - mPtrTopKPackedDevice = mBufferManager->gpu(ITensor::makeShape({idxSize}), nvinfer1::DataType::kINT8); + mPtrTopKPackedHost = mBufferManager->pinned(ITensor::makeShape({idxSize}), tensorrt_llm::DataType::kINT8); + mPtrTopKPackedDevice = mBufferManager->gpu(ITensor::makeShape({idxSize}), tensorrt_llm::DataType::kINT8); mCurandStatesDevice - = mBufferManager->gpu(ITensor::makeShape({numTokens, sizeof(curandState_t)}), nvinfer1::DataType::kINT8); + = mBufferManager->gpu(ITensor::makeShape({numTokens, sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); } template <typename T> @@ -127,19 +131,19 @@ void RoutingKernelTest<T>::computePermutation(RoutingKernelTestParam const& para PackedType* expIdxHostPtr = reinterpret_cast<PackedType*>(bufferCast<int8_t>(*this->mPtrTopKPackedHost)); auto tokenToExpertHost - = mBufferManager->pinned(ITensor::makeShape({param.numTokens * param.topK}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({param.numTokens * param.topK}), tensorrt_llm::DataType::kINT32); auto tokenToExpertHostPtr = bufferCast<int32_t>(*tokenToExpertHost); auto tokenToIdxInExpertHost - = mBufferManager->pinned(ITensor::makeShape({param.numTokens * param.topK}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({param.numTokens * param.topK}), tensorrt_llm::DataType::kINT32); auto tokenToIdxInExpertHostPtr = bufferCast<int32_t>(*tokenToIdxInExpertHost); auto expertScanCountsHost - = mBufferManager->pinned(ITensor::makeShape({param.numExperts + 1}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({param.numExperts + 1}), tensorrt_llm::DataType::kINT32); auto expertScanCountsHostPtr = bufferCast<int32_t>(*expertScanCountsHost); auto ctaScanCountsHost - = mBufferManager->pinned(ITensor::makeShape({param.numExperts + 1}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({param.numExperts + 1}), tensorrt_llm::DataType::kINT32); auto ctaScanCountsHostPtr = bufferCast<int32_t>(*ctaScanCountsHost); for (int ie = 0; ie < param.numExperts + 1; ++ie) @@ -407,7 +411,7 @@ void RoutingKernelTest<T>::runTest(RoutingKernelTestParam const& param) // Retrieve the workspace size of the routing kernel. auto const workspaceSize = getDeviceWorkspaceSize(param); TensorPtr workspaceDevice - = mBufferManager->gpu(ITensor::makeShape({static_cast<int64_t>(workspaceSize)}), nvinfer1::DataType::kINT8); + = mBufferManager->gpu(ITensor::makeShape({static_cast<int64_t>(workspaceSize)}), tensorrt_llm::DataType::kINT8); // Call tested function routing callTestedFunction(param, workspaceDevice); // Verify results diff --git a/cpp/tests/unit_tests/kernels/routing/routingTest.h b/cpp/tests/unit_tests/kernels/routing/routingTest.h index 630cd72a5fcb..8b24ee3aa24e 100644 --- a/cpp/tests/unit_tests/kernels/routing/routingTest.h +++ b/cpp/tests/unit_tests/kernels/routing/routingTest.h @@ -23,7 +23,6 @@ #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/runtimeKernels.h" -#include "tensorrt_llm/runtime/tllmLogger.h" #include <chrono> #include <cmath> #include <memory> //@todo check the usage of this diff --git a/cpp/tests/unit_tests/kernels/sampling/samplingPenaltyTest.cpp b/cpp/tests/unit_tests/kernels/sampling/samplingPenaltyTest.cpp index 8896dd005cf7..a188abf6bc31 100644 --- a/cpp/tests/unit_tests/kernels/sampling/samplingPenaltyTest.cpp +++ b/cpp/tests/unit_tests/kernels/sampling/samplingPenaltyTest.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/penaltyTypes.h" #include "tests/unit_tests/kernels/sampling/samplingTest.h" @@ -161,14 +162,14 @@ class TemperaturePenaltyTest : public SamplingKernelTest<T> mLogitsPtrs = BufferManager::pinned(ITensor::makeShape({mBatchSize}), ptrType); mPenaltyWorkspaceDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep, mVocabSize * 2}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep, mVocabSize * 2}), tensorrt_llm::DataType::kINT32); - mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mBiasHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, mVocabSizePadded}), dataType); mBiasDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mVocabSizePadded}), dataType); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); trk::invokeFill(*mLogitsRefHost, T{0.0f}, *mStream); trk::invokeFill(*mOutLogitsDevice, T{0.0f}, *mStream); @@ -204,7 +205,7 @@ class TemperaturePenaltyTest : public SamplingKernelTest<T> ASSERT_EQ(param.temperaturesSize, mMaxBatchSize) << "Invalid test configuration."; mTemperaturesDevice - = mBufferManager->gpu(ITensor::makeShape({param.temperaturesSize}), nvinfer1::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({param.temperaturesSize}), tensorrt_llm::DataType::kFLOAT); mBufferManager->copy(*param.temperatures, *mTemperaturesDevice); } @@ -281,7 +282,8 @@ TYPED_TEST(TemperaturePenaltyTest, NoPenalty) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + TensorPtr temperaturesHost + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 1.0f; @@ -297,7 +299,8 @@ TYPED_TEST(TemperaturePenaltyTest, LessThanOne) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + TensorPtr temperaturesHost + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 0.53f; @@ -313,7 +316,8 @@ TYPED_TEST(TemperaturePenaltyTest, GreaterThaneOne) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + TensorPtr temperaturesHost + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 2.01f; @@ -329,7 +333,8 @@ TYPED_TEST(TemperaturePenaltyTest, Mixed) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + TensorPtr temperaturesHost + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 0.53f + 0.2f * i; @@ -345,7 +350,8 @@ TYPED_TEST(TemperaturePenaltyTest, LargeVocab) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + TensorPtr temperaturesHost + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 0.53f + 0.2f * i; @@ -361,7 +367,8 @@ TYPED_TEST(TemperaturePenaltyTest, LargeVocabTokensPerStep) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + TensorPtr temperaturesHost + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 1.f; // 0.53f + 0.2f * i; @@ -541,25 +548,25 @@ class RepetitionPenaltyTest : public SamplingKernelTest<T> mLogitsPtrs = BufferManager::pinned(ITensor::makeShape({mBatchSize}), ptrType); mPenaltyWorkspaceDevice = mBufferManager->gpu( - ITensor::makeShape({mBatchSize, mMaxTokensPerStep, mVocabSize * 2}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mBatchSize, mMaxTokensPerStep, mVocabSize * 2}), tensorrt_llm::DataType::kINT32); - mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mOutputIdsHost - = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, mSequenceLength}), nvinfer1::DataType::kINT32); + mOutputIdsHost = BufferManager::pinned( + ITensor::makeShape({mMaxBatchSize, mSequenceLength}), tensorrt_llm::DataType::kINT32); mOutputIdsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mSequenceLength}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mSequenceLength}), tensorrt_llm::DataType::kINT32); - mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mIdsPtrHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), ptrType); mIdsPtrDevice = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), ptrType); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); auto batchSlotsPtr = bufferCast<int32_t>(*mBatchSlots); for (SizeType32 bi = 0; bi < mBatchSize; ++bi) @@ -606,13 +613,13 @@ class RepetitionPenaltyTest : public SamplingKernelTest<T> ASSERT_EQ(param.frequencyPenaltiesSize, mMaxBatchSize) << "Invalid test configuration."; ASSERT_EQ(param.promptIgnoreLengthsSize, mMaxBatchSize) << "Invalid test configuration."; mRepetitionPenaltiesDevice - = mBufferManager->gpu(ITensor::makeShape({param.repetitionPenaltiesSize}), nvinfer1::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({param.repetitionPenaltiesSize}), tensorrt_llm::DataType::kFLOAT); mPresencePenaltiesDevice - = mBufferManager->gpu(ITensor::makeShape({param.presencePenaltiesSize}), nvinfer1::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({param.presencePenaltiesSize}), tensorrt_llm::DataType::kFLOAT); mFrequencyPenaltiesDevice - = mBufferManager->gpu(ITensor::makeShape({param.frequencyPenaltiesSize}), nvinfer1::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({param.frequencyPenaltiesSize}), tensorrt_llm::DataType::kFLOAT); mPromptIgnoreLengthsDevice - = mBufferManager->gpu(ITensor::makeShape({param.promptIgnoreLengthsSize}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({param.promptIgnoreLengthsSize}), tensorrt_llm::DataType::kINT32); mBufferManager->copy(*param.repetitionPenalties, *mRepetitionPenaltiesDevice); mBufferManager->copy(*param.presencePenalties, *mPresencePenaltiesDevice); mBufferManager->copy(*param.frequencyPenalties, *mFrequencyPenaltiesDevice); @@ -740,13 +747,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchNoPenalty) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -773,13 +780,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchRepetitionLessThanOne) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53f; @@ -806,13 +813,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchRepetitionGreaterThaneOne) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 2.01f; @@ -839,13 +846,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchRepetitionMixed) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -872,13 +879,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchPresenceMixed) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -905,13 +912,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchPresenceHasDefaultValueZero2) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -938,13 +945,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchFrequencyMixed) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -971,13 +978,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchFrequencyHasDefaultValueZero2) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -1004,13 +1011,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeRepetitionPresence) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1037,13 +1044,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeRepetitionFrequency) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1070,13 +1077,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypePresenceFrequency) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -1103,13 +1110,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeFull) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1136,13 +1143,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeFullTokensPerStep) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1170,13 +1177,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeFullWithPartialPromptIgnore) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1203,13 +1210,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeFullTokensPerStepWithFullPromptIgno int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1333,23 +1340,23 @@ class MinLengthPenaltyTest : public SamplingKernelTest<T> mLogitsPtrs = BufferManager::pinned(ITensor::makeShape({mBatchSize}), ptrType); mPenaltyWorkspaceDevice = mBufferManager->gpu( - ITensor::makeShape({mBatchSize, mMaxTokensPerStep, mVocabSize}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mBatchSize, mMaxTokensPerStep, mVocabSize}), tensorrt_llm::DataType::kINT32); - mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mMinLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mMinLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mMinLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mMinLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mEndIdsHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mEndIdsHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); auto batchSlotsPtr = bufferCast<int32_t>(*mBatchSlots); for (SizeType32 bi = 0; bi < mBatchSize; ++bi) @@ -1551,30 +1558,30 @@ class MinLengthPenaltyOOBSafetyTest : public SamplingKernelTest<T> } // Defines currentStep. - mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); initConstant<int>(bufferCast<int32_t>(*mSeqLengthHost), mMaxBatchSize, 3); - mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mBufferManager->copy(*mSeqLengthHost, *mSeqLengthDevice); // Defines inputLength. - mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); initConstant<int>(bufferCast<int32_t>(*mContextLengthHost), mMaxBatchSize, 2); - mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mBufferManager->copy(*mContextLengthHost, *mContextLengthDevice); // Defines minLength. - mMinLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mMinLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); initConstant<int>(bufferCast<int32_t>(*mMinLengthHost), mMaxBatchSize, 10); - mMinLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mMinLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mBufferManager->copy(*mMinLengthHost, *mMinLengthDevice); // Defines endIds. - mEndIdsHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mEndIdsHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); initConstant<int>(bufferCast<int32_t>(*mEndIdsHost), mMaxBatchSize, -1); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mBufferManager->copy(*mEndIdsHost, *mEndIdsDevice); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); auto batchSlotsPtr = bufferCast<int32_t>(*mBatchSlots); for (SizeType32 bi = 0; bi < mBatchSize; ++bi) { diff --git a/cpp/tests/unit_tests/kernels/sampling/samplingTest.cpp b/cpp/tests/unit_tests/kernels/sampling/samplingTest.cpp index 90da247f5203..1d4f19c45ad2 100644 --- a/cpp/tests/unit_tests/kernels/sampling/samplingTest.cpp +++ b/cpp/tests/unit_tests/kernels/sampling/samplingTest.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ #include "tests/unit_tests/kernels/sampling/samplingTest.h" +#include "tensorrt_llm/common/tllmDataType.h" namespace tensorrt_llm::tests::kernels::sampling { @@ -51,75 +52,78 @@ void SamplingKernelTest<T>::allocateBuffers(SamplingKernelTestParam const& param auto const ptrType = TRTDataType<T*>::value; // Allocate GPU data - mSeqLengthsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mSeqLengthsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); mFinishedHost = BufferManager::pinned( ITensor::makeShape({maxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); mFinishedDevice = mBufferManager->gpu( ITensor::makeShape({maxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mOutputIdsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kINT32); - mOutputIdsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kINT32); + mOutputIdsHost + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + mOutputIdsDevice + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); mProbsHost = BufferManager::pinned(ITensor::makeShape({batchSize, maxTokensPerStep, vocabSize}), dataType); mProbsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize, maxTokensPerStep, vocabSize}), dataType); mProbsPtrsDevice - = BufferManager::pinned(ITensor::makeShape({batchSize, maxTokensPerStep}), nvinfer1::DataType::kINT64); + = BufferManager::pinned(ITensor::makeShape({batchSize, maxTokensPerStep}), tensorrt_llm::DataType::kINT64); - mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); if (param.returnAllSelectedTokens) { SizeType32 maxTopK = param.topK == 0 ? vocabSize : param.topK; mOutputLogProbsDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, maxTopK}), nvinfer1::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, maxTopK}), tensorrt_llm::DataType::kFLOAT); } else { mOutputLogProbsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxSeqLen, maxBatchSize}), nvinfer1::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({mMaxSeqLen, maxBatchSize}), tensorrt_llm::DataType::kFLOAT); } mZeroParentIdsDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), tensorrt_llm::DataType::kINT32); mLogitsHost = BufferManager::pinned(ITensor::makeShape({batchSize, maxTokensPerStep, vocabSize}), dataType); mLogProbsHost = BufferManager::pinned(ITensor::makeShape({batchSize, maxTokensPerStep, vocabSize}), dataType); mIdsPtrHost = BufferManager::pinned(ITensor::makeShape({2 * maxBatchSize}), ptrType); - mEndIdsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mEndIdsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); - mTopPsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); - mTopPsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + mTopPsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + mTopPsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); - mTopKsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - mTopKsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mTopKsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mTopKsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); - mSkipDecodeHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kBOOL); - mSkipDecodeDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kBOOL); + mSkipDecodeHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kBOOL); + mSkipDecodeDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kBOOL); - mTokensPerStep = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mTokensPerStep = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); - mExpectedCumLogProbsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + mExpectedCumLogProbsHost + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); if (param.returnAllSelectedTokens) { SizeType32 maxTopK = param.topK == 0 ? vocabSize : param.topK; mExpectedLogProbsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, maxTopK}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, maxTopK}), tensorrt_llm::DataType::kFLOAT); } else { mExpectedLogProbsHost - = BufferManager::pinned(ITensor::makeShape({mMaxSeqLen, maxBatchSize}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({mMaxSeqLen, maxBatchSize}), tensorrt_llm::DataType::kFLOAT); } mCurandStatesDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), nvinfer1::DataType::kINT8); + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); } template <typename T> @@ -494,7 +498,7 @@ void SamplingKernelTest<T>::runTest(SamplingKernelTestParam const& param) // Retrieve the workspace size of the sampling kernel. auto const workspaceSize = getWorkspaceSize(param); TensorPtr workspaceDevice - = mBufferManager->gpu(ITensor::makeShape({static_cast<int32_t>(workspaceSize)}), nvinfer1::DataType::kINT8); + = mBufferManager->gpu(ITensor::makeShape({static_cast<int32_t>(workspaceSize)}), tensorrt_llm::DataType::kINT8); // Call tested function sampling callTestedFunction(param, workspaceDevice); diff --git a/cpp/tests/unit_tests/kernels/sampling/samplingTest.h b/cpp/tests/unit_tests/kernels/sampling/samplingTest.h index 0c7f52ba369e..74268bee092c 100644 --- a/cpp/tests/unit_tests/kernels/sampling/samplingTest.h +++ b/cpp/tests/unit_tests/kernels/sampling/samplingTest.h @@ -27,7 +27,6 @@ #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/runtimeKernels.h" -#include "tensorrt_llm/runtime/tllmLogger.h" namespace tensorrt_llm::tests::kernels::sampling { diff --git a/cpp/tests/unit_tests/kernels/sampling/samplingUtilsTest.cu b/cpp/tests/unit_tests/kernels/sampling/samplingUtilsTest.cu index 71a6d767171c..f0f7690257e7 100644 --- a/cpp/tests/unit_tests/kernels/sampling/samplingUtilsTest.cu +++ b/cpp/tests/unit_tests/kernels/sampling/samplingUtilsTest.cu @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/gptDecoder.h" #include "tests/unit_tests/kernels/sampling/samplingTest.h" #include <random> @@ -57,7 +58,8 @@ TEST_F(SamplingUtilsKernelTest, CurandInitialize) sync_check_cuda_error(this->mStream->get()); // Generate random numbers using initialized curand states.MemoryType - auto randValsDevice = this->mBufferManager->gpu(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + auto randValsDevice + = this->mBufferManager->gpu(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); generateRandomNumber<<<1, batchSize, 0, this->mStream->get()>>>( bufferCast<int32_t>(*randValsDevice), batchSlotsPtr, curandStates, batchSize); auto randValsHost = this->mBufferManager->copyFrom(*randValsDevice, MemoryType::kCPU); @@ -97,7 +99,7 @@ TEST_F(SamplingUtilsKernelTest, CurandBatchInitialize) curandState_t* curandStates; cudaMalloc(&curandStates, sizeof(curandState_t) * 2 * batchSize); - auto randomSeedsHost = mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT64); + auto randomSeedsHost = mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT64); auto randomSeedsHostPtr = bufferCast<int64_t>(*randomSeedsHost); size_t const periodSize = 3; for (size_t i = 0; i < batchSize; ++i) @@ -106,7 +108,7 @@ TEST_F(SamplingUtilsKernelTest, CurandBatchInitialize) } auto randomSeedsDevice = mBufferManager->copyFrom(*randomSeedsHost, MemoryType::kGPU); - auto batchSlots = mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + auto batchSlots = mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); auto batchSlotsPtr = bufferCast<SizeType32>(*batchSlots); for (SizeType32 bi = 0; bi < batchSize; ++bi) @@ -120,7 +122,7 @@ TEST_F(SamplingUtilsKernelTest, CurandBatchInitialize) sync_check_cuda_error(mStream->get()); // Generate random numbers using initialized curand states. - auto randValsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + auto randValsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); generateRandomNumber<<<1, batchSize, 0, this->mStream->get()>>>( bufferCast<SizeType32>(*randValsDevice), batchSlotsPtr, curandStates, batchSize); auto const randValsHost = mBufferManager->copyFrom(*randValsDevice, MemoryType::kCPU); @@ -166,25 +168,26 @@ public: ITensor::makeShape({batchSize, maxBeamWidth, vocabSizePadded}), dataType); ITensor::SharedPtr logitsHostPtrs = this->mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), ptrType); auto refLogitsHost = this->mBufferManager->pinnedPool( - ITensor::makeShape({batchSize, maxBeamWidth, vocabSizePadded}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({batchSize, maxBeamWidth, vocabSizePadded}), tensorrt_llm::DataType::kFLOAT); auto refEntropyHost = this->mBufferManager->pinnedPool( - ITensor::makeShape({maxBatchSize, maxBeamWidth}), nvinfer1::DataType::kFLOAT); - auto entropyDevice - = this->mBufferManager->gpu(ITensor::makeShape({maxBatchSize, maxBeamWidth}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({maxBatchSize, maxBeamWidth}), tensorrt_llm::DataType::kFLOAT); + auto entropyDevice = this->mBufferManager->gpu( + ITensor::makeShape({maxBatchSize, maxBeamWidth}), tensorrt_llm::DataType::kFLOAT); auto biasHost = this->mBufferManager->pinnedPool(ITensor::makeShape({vocabSize}), dataType); auto temperatureHost - = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); auto endIdsHost - = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); auto beamWidthsHost - = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); ITensor::SharedPtr finishedHost = this->mBufferManager->pinnedPool( ITensor::makeShape({maxBeamWidth, maxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - auto batchSlots = this->mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + auto batchSlots + = this->mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); auto batchSlotsPtr = bufferCast<int32_t>(*batchSlots); auto beamWidthsHostPtr = bufferCast<SizeType32>(*beamWidthsHost); diff --git a/cpp/tests/unit_tests/kernels/shiftKCacheKernelTest.cu b/cpp/tests/unit_tests/kernels/shiftKCacheKernelTest.cu index b1b3bd6234bf..75fede666434 100644 --- a/cpp/tests/unit_tests/kernels/shiftKCacheKernelTest.cu +++ b/cpp/tests/unit_tests/kernels/shiftKCacheKernelTest.cu @@ -2,6 +2,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decoderMaskedMultiheadAttentionUtils.h" #include "tensorrt_llm/kernels/gptKernels.h" #include "tensorrt_llm/kernels/kvCacheUtils.h" @@ -197,37 +198,37 @@ public: std::vector<int32_t> const& tokenSeqIdxs) { // allocate buffer - mSeqLengthsHost = mBufferManager->pinned(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); - mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + mSeqLengthsHost = mBufferManager->pinned(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); - mInputLengthsHost = mBufferManager->pinned(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); - mInputLengthsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + mInputLengthsHost = mBufferManager->pinned(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + mInputLengthsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); - mKScaleQuantOrigDevice = mBufferManager->gpu(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); + mKScaleQuantOrigDevice = mBufferManager->gpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); mTokenReadIdxsHost = mBufferManager->pinned( - ITensor::makeShape({static_cast<int>(tokenReadIdxs.size())}), nvinfer1::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenReadIdxs.size())}), tensorrt_llm::DataType::kINT32); mTokenReadIdxsDevice = mBufferManager->gpu( - ITensor::makeShape({static_cast<int>(tokenReadIdxs.size())}), nvinfer1::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenReadIdxs.size())}), tensorrt_llm::DataType::kINT32); mTokenWriteIdxsHost = mBufferManager->pinned( - ITensor::makeShape({static_cast<int>(tokenWriteIdxs.size())}), nvinfer1::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenWriteIdxs.size())}), tensorrt_llm::DataType::kINT32); mTokenWriteIdxsDevice = mBufferManager->gpu( - ITensor::makeShape({static_cast<int>(tokenWriteIdxs.size())}), nvinfer1::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenWriteIdxs.size())}), tensorrt_llm::DataType::kINT32); mTokenPosIdxsHost = mBufferManager->pinned( - ITensor::makeShape({static_cast<int>(tokenPosIdxs.size())}), nvinfer1::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenPosIdxs.size())}), tensorrt_llm::DataType::kINT32); mTokenPosIdxsDevice = mBufferManager->gpu( - ITensor::makeShape({static_cast<int>(tokenPosIdxs.size())}), nvinfer1::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenPosIdxs.size())}), tensorrt_llm::DataType::kINT32); mTokenSeqIdxsHost = mBufferManager->pinned( - ITensor::makeShape({static_cast<int>(tokenSeqIdxs.size())}), nvinfer1::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenSeqIdxs.size())}), tensorrt_llm::DataType::kINT32); mTokenSeqIdxsDevice = mBufferManager->gpu( - ITensor::makeShape({static_cast<int>(tokenSeqIdxs.size())}), nvinfer1::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenSeqIdxs.size())}), tensorrt_llm::DataType::kINT32); - // nvinfer1::DataType dataType = nvinfer1::DataType::kHALF - // nvinfer1::DataType::kHALF - // nvinfer1::DataType::kBF16 + // tensorrt_llm::DataType dataType = tensorrt_llm::DataType::kHALF + // tensorrt_llm::DataType::kHALF + // tensorrt_llm::DataType::kBF16 int32_t batchBeam = batchSize * beamWidth; if (pagedKvCache) { diff --git a/cpp/tests/unit_tests/kernels/sparseAttentionKernelsTest.cpp b/cpp/tests/unit_tests/kernels/sparseAttentionKernelsTest.cpp index 508d157e1f61..14aa58f04df5 100644 --- a/cpp/tests/unit_tests/kernels/sparseAttentionKernelsTest.cpp +++ b/cpp/tests/unit_tests/kernels/sparseAttentionKernelsTest.cpp @@ -1,5 +1,6 @@ #include <gtest/gtest.h> +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/sparseAttentionKernels.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" @@ -40,28 +41,29 @@ TEST_F(sparseAttentionKernelsTest, GatherKvPageOffsetsKernelTest) constexpr int total_sparse_tokens = 14; // Create input buffers - auto kv_page_offsets - = mBufferManager->gpu(ITensor::makeShape({batch_size, 2, max_num_pages_per_seq}), nvinfer1::DataType::kINT32); - auto seq_lengths = mBufferManager->gpu(ITensor::makeShape({batch_size}), nvinfer1::DataType::kINT32); + auto kv_page_offsets = mBufferManager->gpu( + ITensor::makeShape({batch_size, 2, max_num_pages_per_seq}), tensorrt_llm::DataType::kINT32); + auto seq_lengths = mBufferManager->gpu(ITensor::makeShape({batch_size}), tensorrt_llm::DataType::kINT32); // Shape: [num_head_kv, total_sparse_tokens] - flattened across all batches auto sparse_indices - = mBufferManager->gpu(ITensor::makeShape({num_head_kv, total_sparse_tokens}), nvinfer1::DataType::kINT32); - auto sparse_indices_offsets = mBufferManager->gpu(ITensor::makeShape({batch_size + 1}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({num_head_kv, total_sparse_tokens}), tensorrt_llm::DataType::kINT32); + auto sparse_indices_offsets + = mBufferManager->gpu(ITensor::makeShape({batch_size + 1}), tensorrt_llm::DataType::kINT32); // Create output buffers auto output_kv_page_offsets = mBufferManager->gpu( - ITensor::makeShape({num_head_kv, batch_size, 2, max_num_pages_per_seq}), nvinfer1::DataType::kINT32); + ITensor::makeShape({num_head_kv, batch_size, 2, max_num_pages_per_seq}), tensorrt_llm::DataType::kINT32); auto output_seq_lengths - = mBufferManager->gpu(ITensor::makeShape({num_head_kv, batch_size}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({num_head_kv, batch_size}), tensorrt_llm::DataType::kINT32); // Create pinned host buffers for data initialization auto kv_page_offsets_host = mBufferManager->pinned( - ITensor::makeShape({batch_size, 2, max_num_pages_per_seq}), nvinfer1::DataType::kINT32); - auto seq_lengths_host = mBufferManager->pinned(ITensor::makeShape({batch_size}), nvinfer1::DataType::kINT32); - auto sparse_indices_host - = mBufferManager->pinned(ITensor::makeShape({num_head_kv, total_sparse_tokens}), nvinfer1::DataType::kINT32); + ITensor::makeShape({batch_size, 2, max_num_pages_per_seq}), tensorrt_llm::DataType::kINT32); + auto seq_lengths_host = mBufferManager->pinned(ITensor::makeShape({batch_size}), tensorrt_llm::DataType::kINT32); + auto sparse_indices_host = mBufferManager->pinned( + ITensor::makeShape({num_head_kv, total_sparse_tokens}), tensorrt_llm::DataType::kINT32); auto sparse_indices_offsets_host - = mBufferManager->pinned(ITensor::makeShape({batch_size + 1}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({batch_size + 1}), tensorrt_llm::DataType::kINT32); // Initialize test data auto kv_page_offsets_ptr = bufferCast<int32_t>(*kv_page_offsets_host); @@ -143,9 +145,9 @@ TEST_F(sparseAttentionKernelsTest, GatherKvPageOffsetsKernelTest) // Copy results back to host for verification auto output_kv_page_offsets_host = mBufferManager->pinned( - ITensor::makeShape({num_head_kv, batch_size, 2, max_num_pages_per_seq}), nvinfer1::DataType::kINT32); + ITensor::makeShape({num_head_kv, batch_size, 2, max_num_pages_per_seq}), tensorrt_llm::DataType::kINT32); auto output_seq_lengths_host - = mBufferManager->pinned(ITensor::makeShape({num_head_kv, batch_size}), nvinfer1::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({num_head_kv, batch_size}), tensorrt_llm::DataType::kINT32); mBufferManager->copy(*output_kv_page_offsets, *output_kv_page_offsets_host); mBufferManager->copy(*output_seq_lengths, *output_seq_lengths_host); diff --git a/cpp/tests/unit_tests/kernels/stopCriteriaKernelsTest.cpp b/cpp/tests/unit_tests/kernels/stopCriteriaKernelsTest.cpp index 2fefae39552e..9fe4737e82c0 100644 --- a/cpp/tests/unit_tests/kernels/stopCriteriaKernelsTest.cpp +++ b/cpp/tests/unit_tests/kernels/stopCriteriaKernelsTest.cpp @@ -16,6 +16,7 @@ #include "tensorrt_llm/kernels/stopCriteriaKernels.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/runtime/bufferManager.h" @@ -60,34 +61,35 @@ class StopCriteriaKernelsTest : public testing::Test std::uniform_int_distribution<SizeType32> tokensPerStepDistr(1, mMaxTokensPerStep); mSequenceLengths - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), nvinfer1::DataType::kINT32); - mSequenceLengthLimits = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), tensorrt_llm::DataType::kINT32); + mSequenceLengthLimits + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); mFinished = BufferManager::pinned( ITensor::makeShape({maxBatchSize, beamWidth}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mFinishedSum = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mFinishedSum = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); mOutputIds = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); mOutputIdsPtr - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), nvinfer1::DataType::kINT64); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), tensorrt_llm::DataType::kINT64); mParentIds = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); mParentIdsPtr - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), nvinfer1::DataType::kINT64); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), tensorrt_llm::DataType::kINT64); mRefOutputIds = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); - mStopWords - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, 2, maxStopWordsLen}), nvinfer1::DataType::kINT32); - mStopWordsPtr = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT64); - mStopWordsLen = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mStopWords = BufferManager::pinned( + ITensor::makeShape({maxBatchSize, 2, maxStopWordsLen}), tensorrt_llm::DataType::kINT32); + mStopWordsPtr = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT64); + mStopWordsLen = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); - mEndIds = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - mTokensPerStep = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mEndIds = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mTokensPerStep = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); auto batchSlotsPtr = bufferCast<SizeType32>(*mBatchSlots); for (SizeType32 bi = 0; bi < batchSize; ++bi) diff --git a/cpp/tests/unit_tests/layers/baseSamplingLayerTest.cpp b/cpp/tests/unit_tests/layers/baseSamplingLayerTest.cpp index 8bdacb2e9f6c..7886a6b54e2b 100644 --- a/cpp/tests/unit_tests/layers/baseSamplingLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/baseSamplingLayerTest.cpp @@ -15,6 +15,7 @@ */ #include "tests/unit_tests/layers/baseSamplingLayerTest.h" +#include "tensorrt_llm/common/tllmDataType.h" namespace tensorrt_llm::tests::layers::sampling { @@ -67,26 +68,26 @@ void BaseSamplingLayerTest<T>::setup(uint64_t seed, TestSamplingParams const& pa BaseSamplingLayerTest::mMaxOutputLen * params.beamWidth, mVocabSize); } - mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), nvinfer1::DataType::kINT32); - mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), nvinfer1::DataType::kINT32); + mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), tensorrt_llm::DataType::kINT32); + mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), tensorrt_llm::DataType::kINT32); mFinishedDevice = params.isExternalDraftTokensLayerTest ? mBufferManager->gpu(ITensor::makeShape({mMaxTokensPerEngineStep, maxBatchSize()}), TRTDataType<tk::FinishedState::UnderlyingType>::value) : mBufferManager->gpu( ITensor::makeShape({maxBatchSize()}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mOutputIdsDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), nvinfer1::DataType::kINT32); + mOutputIdsDevice = mBufferManager->gpu( + ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), tensorrt_llm::DataType::kINT32); mIdsPtrHost = mBufferManager->pinned(ITensor::makeShape({maxBatchSize()}), ptrType); - mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), nvinfer1::DataType::kFLOAT); + mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), tensorrt_llm::DataType::kFLOAT); mOutputLogProbsDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize(), mMaxSeqLen}), nvinfer1::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize(), mMaxSeqLen}), tensorrt_llm::DataType::kFLOAT); mBatchSlots - = mBufferManager->pinned(ITensor::makeShape({mBatchSize + mBatchSizeBadPad}), nvinfer1::DataType::kINT32); - mCurandStatesDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize(), sizeof(curandState_t)}), nvinfer1::DataType::kINT8); + = mBufferManager->pinned(ITensor::makeShape({mBatchSize + mBatchSizeBadPad}), tensorrt_llm::DataType::kINT32); + mCurandStatesDevice = mBufferManager->gpu( + ITensor::makeShape({maxBatchSize(), sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); auto const workspaceSize = mSamplingLayer->getWorkspaceSize(); @@ -152,11 +153,11 @@ void BaseSamplingLayerTest<T>::setup(uint64_t seed, TestSamplingParams const& pa setupParams = samplingSetupParams; mSrcCacheIndirection = mBufferManager->gpu( - ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); mTgtCacheIndirection = mBufferManager->gpu( - ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); mParentIds = mBufferManager->gpu( - ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); auto constexpr nvTokenIdType = TRTDataType<TokenIdType>::value; auto constexpr nvSizeType = TRTDataType<SizeType32>::value; diff --git a/cpp/tests/unit_tests/layers/baseSamplingLayerTest.h b/cpp/tests/unit_tests/layers/baseSamplingLayerTest.h index 5a375000a5e8..2bf782a5f5e8 100644 --- a/cpp/tests/unit_tests/layers/baseSamplingLayerTest.h +++ b/cpp/tests/unit_tests/layers/baseSamplingLayerTest.h @@ -34,7 +34,6 @@ #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/runtimeKernels.h" -#include "tensorrt_llm/runtime/tllmLogger.h" #include "tensorrt_llm/common/tllmException.h" diff --git a/cpp/tests/unit_tests/layers/dynamicDecodeLayerTest.cpp b/cpp/tests/unit_tests/layers/dynamicDecodeLayerTest.cpp index a3c2d56de16e..cc3b9c411f5b 100644 --- a/cpp/tests/unit_tests/layers/dynamicDecodeLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/dynamicDecodeLayerTest.cpp @@ -15,6 +15,7 @@ */ #include "tests/unit_tests/layers/dynamicDecodeLayerTest.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/runtime/runtimeKernels.h" #include <algorithm> @@ -150,43 +151,43 @@ void DynamicDecodeLayerTest<T>::allocateData(TestSamplingParams const& params, T mRuntimeLogitsHost = BufferManager::pinned(ITensor::makeShape({mBatchSize, mBeamWidth, mVocabSizePadded}), dataType); - mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mFinishedDevice = mBufferManager->gpu( ITensor::makeShape({mMaxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mFinishedSumDevice = BufferManager::pinned(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); - mOutputIdsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); + mFinishedSumDevice = BufferManager::pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); + mOutputIdsDevice = mBufferManager->gpu( + ITensor::makeShape({mMaxBatchSize, mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); mNewTokens - = BufferManager::pinned(ITensor::makeShape({mMaxTokensPerStep, mMaxBatchSize}), nvinfer1::DataType::kINT32); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({mMaxTokensPerStep, mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mEmbeddingBiasHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, mVocabSizePadded}), dataType); mEmbeddingBiasDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mVocabSizePadded}), dataType); mRefLogProbsHost - = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kFLOAT); mOutputLogProbsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kFLOAT); mOutputLogProbsTiledDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxSeqLen, mMaxBatchSize}), nvinfer1::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({mMaxSeqLen, mMaxBatchSize}), tensorrt_llm::DataType::kFLOAT); - mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kFLOAT); + mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kFLOAT); mMaxBadWordsLen = getMaxWordsLen(params.badWords); mMaxStopWordsLen = getMaxWordsLen(params.stopWords); - mBadWords - = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, 2, mMaxBadWordsLen}), nvinfer1::DataType::kINT32); - mBadWordsLens = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mBadWordsPtrs = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT64); + mBadWords = BufferManager::pinned( + ITensor::makeShape({mMaxBatchSize, 2, mMaxBadWordsLen}), tensorrt_llm::DataType::kINT32); + mBadWordsLens = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mBadWordsPtrs = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT64); - mStopWords - = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, 2, mMaxStopWordsLen}), nvinfer1::DataType::kINT32); - mStopWordsLens = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mStopWordsPtrs = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT64); + mStopWords = BufferManager::pinned( + ITensor::makeShape({mMaxBatchSize, 2, mMaxStopWordsLen}), tensorrt_llm::DataType::kINT32); + mStopWordsLens = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mStopWordsPtrs = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT64); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); if (mDecodingMode.isMedusa()) { @@ -204,19 +205,19 @@ void DynamicDecodeLayerTest<T>::allocateMedusaData(TestSamplingParams const& par auto const dataType = TRTDataType<T>::value; mMaxMedusaHeads = params.maxNumMedusaHeads.value(); mPathsDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep, mMaxMedusaHeads + 1}), nvinfer1::DataType::kINT32); - mAcceptedLengths = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep, mMaxMedusaHeads + 1}), tensorrt_llm::DataType::kINT32); + mAcceptedLengths = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mMedusaLogitsDevice = BufferManager::pinned( ITensor::makeShape({mMaxMedusaHeads, mMaxBatchSize, mMaxTokensPerStep, mVocabSizePadded}), dataType); - mNextDraftTokensDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep - 1}), nvinfer1::DataType::kINT32); - mTokensPerStepDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mTreeIdsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep - 1}), nvinfer1::DataType::kINT32); + mNextDraftTokensDevice = mBufferManager->gpu( + ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep - 1}), tensorrt_llm::DataType::kINT32); + mTokensPerStepDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mTreeIdsDevice = mBufferManager->gpu( + ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep - 1}), tensorrt_llm::DataType::kINT32); mAcceptedLengthCumSumDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize + 1}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize + 1}), tensorrt_llm::DataType::kINT32); mPackedPathsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize * mMaxMedusaHeads}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize * mMaxMedusaHeads}), tensorrt_llm::DataType::kINT32); } template <typename T> @@ -604,7 +605,7 @@ template <typename T> void DynamicDecodeLayerTest<T>::batchCopy(SizeType32 step) { auto const logitsHost = ITensor::wrap(mTestLogitsInit.data() + step * mVocabSizePadded, - std::is_same_v<T, float> ? nvinfer1::DataType::kFLOAT : nvinfer1::DataType::kHALF, + std::is_same_v<T, float> ? tensorrt_llm::DataType::kFLOAT : tensorrt_llm::DataType::kHALF, ITensor::makeShape({mMaxTokensPerStep, mVocabSizePadded})); for (SizeType32 bi = 0; bi < mBatchSize; ++bi) { diff --git a/cpp/tests/unit_tests/layers/eagleLayerTest.cpp b/cpp/tests/unit_tests/layers/eagleLayerTest.cpp index bdb53f15618f..47bca93b47b4 100644 --- a/cpp/tests/unit_tests/layers/eagleLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/eagleLayerTest.cpp @@ -22,9 +22,8 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/runtimeKernels.h" #include "tensorrt_llm/runtime/speculativeDecodingModule.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include <NvInferRuntimeBase.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <algorithm> #include <cstdint> @@ -555,126 +554,126 @@ void EagleDecodingLayerTest<T>::allocateBuffers() // outputs mOutputIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxSeqLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mSeqLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mOutputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mOutputUnpackedNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mAcceptedLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mNextPosIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mPrevDraftLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mNextDraftLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mNextGenerationLengths - = mBufferManager->gpu(ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mNextGenerationLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mAcceptedLengthCumSum = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize() + 1}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize() + 1}), tensorrt_llm::DataType::kINT32); mPathsOffsets = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize() * mSamplingParams.getMaxDraftPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mPackedMasks = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), static_cast<SizeType32>(divUp(mSamplingParams.getMaxDecodingTokens(), 32))}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mRandomDataSample = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kFLOAT); mRandomDataValidation = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kFLOAT); + tensorrt_llm::DataType::kFLOAT); mOutputTemperatures = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kFLOAT); mOutputNextDraftPaths = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mEagleNetCtxRequestTypesHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mEagleNetCtxContextLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mEagleNetCtxPastKeyValueLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mEagleNetGenRequestTypesHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mEagleNetGenContextLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mEagleNetGenPastKeyValueLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); // inputs - mBatchSlots - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); - mEndIds - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mEndIds = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); mInputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); - mInputNextDraftLens - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mInputNextDraftLens = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); mInputNextDraftPaths = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mInputLastDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); - mInputLastDraftLens - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mInputLastDraftLens = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); mInputLastDraftPaths = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mInputAcceptedTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); - mInputAcceptedLens - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mInputAcceptedLens = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); - mInputAcceptedPathIds - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mInputAcceptedPathIds = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); - mChunkedContextNextTokens - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mChunkedContextNextTokens = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); mDecodingWorkspace = std::make_shared<tensorrt_llm::runtime::DecodingLayerWorkspace>(mBufferManager, decodingDomain, TRTDataType<float>::value, mSamplingParams.getMaxBatchSize() * sizeof(curandState_t)); diff --git a/cpp/tests/unit_tests/layers/explicitDraftTokensLayerTest.cpp b/cpp/tests/unit_tests/layers/explicitDraftTokensLayerTest.cpp index e7831b57f77f..04d05e0d16a9 100644 --- a/cpp/tests/unit_tests/layers/explicitDraftTokensLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/explicitDraftTokensLayerTest.cpp @@ -22,9 +22,8 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/runtimeKernels.h" #include "tensorrt_llm/runtime/speculativeDecodingModule.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include <NvInferRuntimeBase.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <algorithm> #include <cstdint> @@ -655,29 +654,29 @@ void ExplicitDraftTokensLayerTest<T>::allocateBuffers() // outputs mOutputIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxSeqLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mSeqLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mAcceptedLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mNextDraftLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mPrevDraftLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mAcceptedLengthCumSum = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize() + 1}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize() + 1}), tensorrt_llm::DataType::kINT32); mOutputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mOutputPositionIdsBase = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mRandomDataSample = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), dataType); @@ -689,21 +688,21 @@ void ExplicitDraftTokensLayerTest<T>::allocateBuffers() mPackedMasks = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), static_cast<SizeType32>(divUp(mSamplingParams.getMaxDecodingTokens(), 32))}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mNextPosIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mOutputUnpackedNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mOutputUnpackedNextDraftIndices = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mOutputDraftProbs = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), @@ -713,68 +712,68 @@ void ExplicitDraftTokensLayerTest<T>::allocateBuffers() mOutputTemperatures = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), dataType); mOutputGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mOutputGenerationLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); - mMaxGenLengthHost = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + mMaxGenLengthHost = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); // inputs - mBatchSlots - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); mTokensPerStep = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mPathsOffsets = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize() * mSamplingParams.getMaxDraftPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mMasks = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kBOOL); + tensorrt_llm::DataType::kBOOL); mInputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mLastDraftTokens = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mPackedPosIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mBestPathLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mBestPathIndices = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mSpecDecodingGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mNextFlatTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize() * mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mInputPositionIdsBase = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); mNextDraftIndices = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mLastDraftIndices = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mNextDraftProbs = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), @@ -782,20 +781,20 @@ void ExplicitDraftTokensLayerTest<T>::allocateBuffers() dataType); mEndIds = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); - mMaxGenLengthDevice = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + mMaxGenLengthDevice = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); // Packed inputs - mMaxGenerationLength = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - mCumSumGenerationLengths - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mMaxGenerationLength = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + mCumSumGenerationLengths = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); // Packed outputs - mPackedPositionIdsBase - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); - mPackedGenerationLengths - = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mPackedPositionIdsBase = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mPackedGenerationLengths = BufferManager::pinnedPool( + ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); mPackedRandomDataSample = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), dataType); mPackedRandomDataVerification = BufferManager::pinnedPool( ITensor::makeShape( @@ -804,21 +803,21 @@ void ExplicitDraftTokensLayerTest<T>::allocateBuffers() mPackedNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mPackedNextDraftIndices = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mPackedPackedMasks = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens(), static_cast<SizeType32>(divUp(mSamplingParams.getMaxDecodingTokens(), 32))}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mPackedPositionOffsets = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mPackedPackedPosIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - nvinfer1::DataType::kINT32); + tensorrt_llm::DataType::kINT32); mPackedDraftProbs = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxDraftPathLen(), mSamplingParams.getVocabSize()}), @@ -1564,7 +1563,6 @@ class FillRandDataTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro- void SetUp() override { - mLogger = std::make_shared<TllmLogger>(); mStream = std::make_shared<tensorrt_llm::runtime::CudaStream>(); mBufferManager = std::make_shared<tensorrt_llm::runtime::BufferManager>(mStream); } @@ -1576,12 +1574,12 @@ class FillRandDataTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro- { SizeType32* batchSlotsPtr{nullptr}; - auto curandState = mBufferManager->gpu(ITensor::makeShape({batchSize, 48}), nvinfer1::DataType::kUINT8); + auto curandState = mBufferManager->gpu(ITensor::makeShape({batchSize, 48}), tensorrt_llm::DataType::kUINT8); auto* curandStatePtr = reinterpret_cast<curandState_t*>(bufferCast<uint8_t>(*curandState)); if (batchInit) { - auto randomSeeds = mBufferManager->gpu(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT64); + auto randomSeeds = mBufferManager->gpu(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT64); trk::invokeFill(*randomSeeds, static_cast<int64_t>(randomSeed), *mStream); auto* randomSeedsPtr = bufferCast<uint64_t>(*randomSeeds); tk::invokeCurandBatchInitialize(curandStatePtr, batchSlotsPtr, batchSize, randomSeedsPtr, mStream->get()); @@ -1625,7 +1623,6 @@ class FillRandDataTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro- } private: - std::shared_ptr<nvinfer1::ILogger> mLogger; std::shared_ptr<tensorrt_llm::runtime::CudaStream> mStream; std::shared_ptr<tensorrt_llm::runtime::BufferManager> mBufferManager; }; diff --git a/cpp/tests/unit_tests/layers/externalDraftTokensLayerTest.cpp b/cpp/tests/unit_tests/layers/externalDraftTokensLayerTest.cpp index c216a76bc2b8..cdc913d3b3aa 100644 --- a/cpp/tests/unit_tests/layers/externalDraftTokensLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/externalDraftTokensLayerTest.cpp @@ -15,6 +15,7 @@ */ #include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/runtime/iBuffer.h" #include "tests/unit_tests/layers/baseSamplingLayerTest.h" @@ -88,7 +89,7 @@ class ExternalDraftTokensLayerTest : public BaseSamplingLayerTest<T> dataType); mDraftTokenIds = this->mBufferManager->gpu( - ITensor::makeShape({this->maxBatchSize(), mMaxDraftLen}), nvinfer1::DataType::kINT32); + ITensor::makeShape({this->maxBatchSize(), mMaxDraftLen}), tensorrt_llm::DataType::kINT32); mUseDraftLogits = this->mBufferManager->gpu(ITensor::makeShape({this->maxBatchSize()}), TRTDataType<bool>::value); mUseDraftLogitsHost diff --git a/cpp/tests/unit_tests/layers/lookaheadAlgorithmTest.cpp b/cpp/tests/unit_tests/layers/lookaheadAlgorithmTest.cpp index cd1dc4799e6c..7f4791a85ba4 100644 --- a/cpp/tests/unit_tests/layers/lookaheadAlgorithmTest.cpp +++ b/cpp/tests/unit_tests/layers/lookaheadAlgorithmTest.cpp @@ -17,6 +17,7 @@ #include <tuple> #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/layers/lookaheadAlgorithm.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" @@ -87,24 +88,25 @@ TEST_P(LookaheadAlgorithmTest, predict) auto shape = ITensor::makeShape({maxTokensPerStep}); auto shape2d = ITensor::makeShape({maxTokensPerStep, maxTokensPerStep}); auto shapeSingle = ITensor::makeShape({1}); - TensorPtr posidMax = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); - TensorPtr attentionMaskMax = BufferManager::cpu(shape2d, nvinfer1::DataType::kBOOL); - TensorPtr inputLengthPtr = BufferManager::cpu(shapeSingle, nvinfer1::DataType::kINT32); + TensorPtr posidMax = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); + TensorPtr attentionMaskMax = BufferManager::cpu(shape2d, tensorrt_llm::DataType::kBOOL); + TensorPtr inputLengthPtr = BufferManager::cpu(shapeSingle, tensorrt_llm::DataType::kINT32); auto& inputLength(*BufferRange<SizeType32>(*inputLengthPtr).begin()); - TensorPtr outputMax = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); - TensorPtr endIdPtr = BufferManager::cpu(shapeSingle, nvinfer1::DataType::kINT32); + TensorPtr outputMax = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); + TensorPtr endIdPtr = BufferManager::cpu(shapeSingle, tensorrt_llm::DataType::kINT32); auto& endId(*BufferRange<TokenIdType>(*endIdPtr).begin()); endId = ascii->getEndToken(); - TensorPtr acceptedMax = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); - TensorPtr acceptedOffsetsMax = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); - TensorPtr acceptedLengthPtr = BufferManager::cpu(shapeSingle, nvinfer1::DataType::kINT32); + TensorPtr acceptedMax = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); + TensorPtr acceptedOffsetsMax = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); + TensorPtr acceptedLengthPtr = BufferManager::cpu(shapeSingle, tensorrt_llm::DataType::kINT32); auto& acceptedLength(*BufferRange<SizeType32>(*acceptedLengthPtr).begin()); - TensorPtr sequence = BufferManager::cpu(ITensor::makeShape({maxSeqLen + maxDraftLen}), nvinfer1::DataType::kINT32); + TensorPtr sequence + = BufferManager::cpu(ITensor::makeShape({maxSeqLen + maxDraftLen}), tensorrt_llm::DataType::kINT32); BufferRange<TokenIdType> sequenceRange(*sequence); - TensorPtr sequenceLengthPtr = BufferManager::cpu(shapeSingle, nvinfer1::DataType::kINT32); + TensorPtr sequenceLengthPtr = BufferManager::cpu(shapeSingle, tensorrt_llm::DataType::kINT32); auto& sequenceLength(*bufferCast<SizeType32>(*sequenceLengthPtr)); std::copy(promptRange.begin(), promptRange.end(), sequenceRange.begin()); @@ -224,13 +226,13 @@ TEST(LookaheadAlgorithmTest, treeEncodeTest) auto shape = inputTokens->getShape(); auto shape2d = ITensor::makeShape({shape.d[0], shape.d[0]}); - TensorPtr inputMasks = BufferManager::cpu(shape2d, nvinfer1::DataType::kBOOL); + TensorPtr inputMasks = BufferManager::cpu(shape2d, tensorrt_llm::DataType::kBOOL); LookaheadAlgorithm::posIdsToMask(inputMasks, inputPosIds); - TensorPtr outputTokens = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); - TensorPtr outputPosIds = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); - TensorPtr encodeMap = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); - TensorPtr outputMasks = BufferManager::cpu(shape2d, nvinfer1::DataType::kBOOL); + TensorPtr outputTokens = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); + TensorPtr outputPosIds = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); + TensorPtr encodeMap = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); + TensorPtr outputMasks = BufferManager::cpu(shape2d, tensorrt_llm::DataType::kBOOL); // auto len = LookaheadAlgorithm::treeEncode(outputTokens, outputPosIds, outputMasks, inputTokens, inputPosIds, // inputMasks, '$', 9); diff --git a/cpp/tests/unit_tests/layers/lookaheadDecodingLayerTest.cpp b/cpp/tests/unit_tests/layers/lookaheadDecodingLayerTest.cpp index 414e6f101743..917f6dbdca55 100644 --- a/cpp/tests/unit_tests/layers/lookaheadDecodingLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/lookaheadDecodingLayerTest.cpp @@ -23,6 +23,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/layers/decodingParams.h" #include "tensorrt_llm/layers/lookaheadDecodingLayer.h" @@ -320,7 +321,7 @@ void LookaheadDecodingLayerTest::allocateBuffers() mLlm[gbi] = std::make_shared<LookaheadRandomLlm>(mAscii, mOracle[gbi], gbi); mScoreBoard[gbi] = std::ostringstream(); - mHistogram[gbi] = BufferManager::cpu(ITensor::makeShape({mTestParam.n + 1}), nvinfer1::DataType::kINT32); + mHistogram[gbi] = BufferManager::cpu(ITensor::makeShape({mTestParam.n + 1}), tensorrt_llm::DataType::kINT32); } switch (mTestParam.batchType) { @@ -348,48 +349,50 @@ void LookaheadDecodingLayerTest::allocateBuffers() auto maxBatchShape1D = ITensor::makeShape({maxBatchSize}); - mAlgoConfigBatch = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, 3}), nvinfer1::DataType::kINT32); + mAlgoConfigBatch = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, 3}), tensorrt_llm::DataType::kINT32); - mEndIds = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); - mTokensPerStep = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); + mEndIds = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + mTokensPerStep = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - mOutputIds = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, maxBeamSize, mMaxSeqLen + mMaxTokensPerStep}), nvinfer1::DataType::kINT32); - mSequenceLengths = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); + mOutputIds + = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, maxBeamSize, mMaxSeqLen + mMaxTokensPerStep}), + tensorrt_llm::DataType::kINT32); + mSequenceLengths = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); mProbs = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, mMaxTokensPerStep, vocabSize}), nvinfer1::DataType::kFLOAT); + ITensor::makeShape({maxBatchSize, mMaxTokensPerStep, vocabSize}), tensorrt_llm::DataType::kFLOAT); mGoldenSampledTokens - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), nvinfer1::DataType::kINT32); - mInputTokensBatch - = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), nvinfer1::DataType::kINT32); - mPositionIdsBatch - = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), nvinfer1::DataType::kINT32); + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), tensorrt_llm::DataType::kINT32); + mInputTokensBatch = BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), tensorrt_llm::DataType::kINT32); + mPositionIdsBatch = BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), tensorrt_llm::DataType::kINT32); mNewTokens = BufferManager::pinnedPool( - ITensor::makeShape({mMaxTokensPerStep, maxBatchSize, 1}), nvinfer1::DataType::kINT32); - mNumNewTokens = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); - mDraftLengths = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); - mPrevDraftLengths = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); + ITensor::makeShape({mMaxTokensPerStep, maxBatchSize, 1}), tensorrt_llm::DataType::kINT32); + mNumNewTokens = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + mDraftLengths = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + mPrevDraftLengths = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); mDraftTokens - = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, maxDraftLen}), nvinfer1::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, maxDraftLen}), tensorrt_llm::DataType::kINT32); auto packedMaskShape = ITensor::makeShape( {maxBatchSize, mMaxTokensPerStep, static_cast<ITensor::DimType64>(common::divUp(mMaxTokensPerStep, 32))}); - mPackedMasks = BufferManager::pinnedPool(packedMaskShape, nvinfer1::DataType::kINT32); + mPackedMasks = BufferManager::pinnedPool(packedMaskShape, tensorrt_llm::DataType::kINT32); mPackedMasksBool = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, mMaxTokensPerStep, mMaxTokensPerStep}), nvinfer1::DataType::kBOOL); - mNumNewTokensCumSum = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize + 1}), nvinfer1::DataType::kINT32); + ITensor::makeShape({maxBatchSize, mMaxTokensPerStep, mMaxTokensPerStep}), tensorrt_llm::DataType::kBOOL); + mNumNewTokensCumSum + = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize + 1}), tensorrt_llm::DataType::kINT32); mPathsOffsets = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), nvinfer1::DataType::kINT32); - mGenerationLengths = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); - mPositionOffsets - = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), nvinfer1::DataType::kINT32); - mPositionIds - = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), nvinfer1::DataType::kINT32); - mAttentionPackedMask = BufferManager::pinnedPool(packedMaskShape, nvinfer1::DataType::kINT32); - - mBatchSlotsMax = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); + ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), tensorrt_llm::DataType::kINT32); + mGenerationLengths = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + mPositionOffsets = BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), tensorrt_llm::DataType::kINT32); + mPositionIds = BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), tensorrt_llm::DataType::kINT32); + mAttentionPackedMask = BufferManager::pinnedPool(packedMaskShape, tensorrt_llm::DataType::kINT32); + + mBatchSlotsMax = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); auto const batchSize = 0; auto batchShape1D = ITensor::makeShape({batchSize}); diff --git a/cpp/tests/unit_tests/layers/lookaheadRandomLlmTest.cpp b/cpp/tests/unit_tests/layers/lookaheadRandomLlmTest.cpp index 2cc2523d08cb..4ca7206c3bc8 100644 --- a/cpp/tests/unit_tests/layers/lookaheadRandomLlmTest.cpp +++ b/cpp/tests/unit_tests/layers/lookaheadRandomLlmTest.cpp @@ -15,6 +15,7 @@ */ #include <gtest/gtest.h> +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/samplingTopKKernels.h" #include "tensorrt_llm/layers/lookaheadAlgorithm.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" @@ -48,7 +49,7 @@ TEST(LookaheadRandomllm, forward) std::string str("hello world!"); TensorPtr logits = BufferManager::cpu(ITensor::makeShape({static_cast<SizeType32>(str.size()), ascii->getVocabSize()}), - nvinfer1::DataType::kFLOAT); + tensorrt_llm::DataType::kFLOAT); ascii->stringToLogits(logits, str); auto result = ascii->logitsToString(logits); EXPECT_EQ(result, str); @@ -67,7 +68,7 @@ TEST(LookaheadRandomllm, forward) std::vector<TokenIdType> positionIdVec({22, 23, 24, 23, 24, 25, 24, 25, 26, 25, 26, 27, 26, 27, 28}); TensorPtr positionIds = ITensor::wrap(positionIdVec, ITensor::makeShape({len})); TensorPtr outputLogits - = BufferManager::cpu(ITensor::makeShape({len, ascii->getVocabSize()}), nvinfer1::DataType::kFLOAT); + = BufferManager::cpu(ITensor::makeShape({len, ascii->getVocabSize()}), tensorrt_llm::DataType::kFLOAT); llm.forward(outputLogits, inputTokens, positionIds); @@ -123,29 +124,29 @@ TEST(LookaheadRandomllm, gpuSampling) SizeType32 workspaceSize = tensorrt_llm::kernels::getTopKWorkspaceSize<float>(maxBatchSize, maxTokensPerStep, mMaxTopK, vocabSizePadded); - TensorPtr workspaceDevice - = mBufferManager->pinned(ITensor::makeShape({static_cast<int32_t>(workspaceSize)}), nvinfer1::DataType::kINT8); + TensorPtr workspaceDevice = mBufferManager->pinned( + ITensor::makeShape({static_cast<int32_t>(workspaceSize)}), tensorrt_llm::DataType::kINT8); auto const dataType = TRTDataType<float>::value; auto const ptrType = TRTDataType<float*>::value; // Allocate GPU data - TensorPtr mSeqLengths = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kINT32); + TensorPtr mSeqLengths = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kINT32); TensorPtr mFinished = BufferManager::pinned(maxBatchShape1D, TRTDataType<tk::FinishedState::UnderlyingType>::value); - TensorPtr mEndIds = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kINT32); - TensorPtr mTopPs = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kFLOAT); - TensorPtr mTopKs = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kINT32); - TensorPtr mSkipDecode = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kBOOL); - TensorPtr mTokensPerStep = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kINT32); - - TensorPtr mCurandStates - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), nvinfer1::DataType::kINT8); + TensorPtr mEndIds = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + TensorPtr mTopPs = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kFLOAT); + TensorPtr mTopKs = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + TensorPtr mSkipDecode = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kBOOL); + TensorPtr mTokensPerStep = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + + TensorPtr mCurandStates = BufferManager::pinned( + ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); TensorPtr mOutputIds - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); TensorPtr mProbs = BufferManager::pinned(maxBatchShape3D, dataType); - TensorPtr mBatchSlots = BufferManager::pinned(batchShape1D, nvinfer1::DataType::kINT32); + TensorPtr mBatchSlots = BufferManager::pinned(batchShape1D, tensorrt_llm::DataType::kINT32); ///////////////////////////////////// std::copy(batchSlotsVec.begin(), batchSlotsVec.end(), BufferRange<SizeType32>(*mBatchSlots).begin()); diff --git a/cpp/tests/unit_tests/layers/medusaDecodeLayerTest.cpp b/cpp/tests/unit_tests/layers/medusaDecodeLayerTest.cpp index 37a479eb4214..a93955d02c9a 100644 --- a/cpp/tests/unit_tests/layers/medusaDecodeLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/medusaDecodeLayerTest.cpp @@ -15,6 +15,7 @@ */ #include "medusaDecodeLayerTest.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/runtime/medusaModule.h" #include "tensorrt_llm/runtime/runtimeKernels.h" @@ -138,35 +139,36 @@ void MedusaDecodingLayerTest<T>::allocateBuffers() mFinishedDevice = mBufferManager->gpu( ITensor::makeShape({mMaxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mOutputIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kINT32); + mOutputIdsDevice + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mPathsDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens, mMaxDraftPathLen + 1}), nvinfer1::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens, mMaxDraftPathLen + 1}), tensorrt_llm::DataType::kINT32); - mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mAcceptedLengths = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mAcceptedLengths = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mTreeIdsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens - 1}), nvinfer1::DataType::kINT32); + mTreeIdsDevice = mBufferManager->gpu( + ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens - 1}), tensorrt_llm::DataType::kINT32); mMedusaLogitsDevice = mBufferManager->gpu( ITensor::makeShape({mMaxDraftPathLen, mMaxBatchSize, mMaxDecodingTokens, mVocabSizePadded}), dataType); - mNextDraftTokensDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens - 1}), nvinfer1::DataType::kINT32); + mNextDraftTokensDevice = mBufferManager->gpu( + ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens - 1}), tensorrt_llm::DataType::kINT32); - mTokensPerStepDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mTokensPerStepDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); mAcceptedLengthCumSumDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize + 1}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize + 1}), tensorrt_llm::DataType::kINT32); mPackedPathsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize * mMaxDraftPathLen}), nvinfer1::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize * mMaxDraftPathLen}), tensorrt_llm::DataType::kINT32); for (int32_t bi = 0; bi < mBatchSize; ++bi) { @@ -215,7 +217,7 @@ void MedusaDecodingLayerTest<T>::setup(SamplingParams& params) for (SizeType32 bi = 0; bi < mBatchSize; ++bi) { auto const draftIdsHost = ITensor::wrap(reinterpret_cast<TokenIdType*>(params.draftIds[bi].data()), - nvinfer1::DataType::kINT32, ITensor::makeShape({1, mMaxDecodingTokens - 1})); + tensorrt_llm::DataType::kINT32, ITensor::makeShape({1, mMaxDecodingTokens - 1})); auto draftIdsDeviceSlice = ITensor::slice(mNextDraftTokensDevice, batchSlotsPtr[bi], 1); mBufferManager->copy(*draftIdsHost, *draftIdsDeviceSlice); } @@ -224,7 +226,7 @@ void MedusaDecodingLayerTest<T>::setup(SamplingParams& params) { auto& path = params.paths[bi]; auto const numPaths = static_cast<SizeType32>(params.paths[bi].size() / (mMaxDraftPathLen + 1)); - auto const pathsHost = ITensor::wrap(reinterpret_cast<SizeType32*>(path.data()), nvinfer1::DataType::kINT32, + auto const pathsHost = ITensor::wrap(reinterpret_cast<SizeType32*>(path.data()), tensorrt_llm::DataType::kINT32, ITensor::makeShape({1, numPaths, mMaxDraftPathLen + 1})); TensorPtr pathsDeviceSlice = ITensor::slice(mPathsDevice, batchSlotsPtr[bi], 1); pathsDeviceSlice->squeeze(0); diff --git a/cpp/tests/unit_tests/layers/randomLlm.cpp b/cpp/tests/unit_tests/layers/randomLlm.cpp index 632cf8765c20..63aa85eaad16 100644 --- a/cpp/tests/unit_tests/layers/randomLlm.cpp +++ b/cpp/tests/unit_tests/layers/randomLlm.cpp @@ -16,6 +16,7 @@ #include "tests/unit_tests/layers/randomLlm.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/common.h" @@ -34,7 +35,7 @@ TensorPtr initTensor(std::string str, std::optional<ITensor::Shape> shape) { TLLM_CHECK(ITensor::volume(shape1d) == ITensor::volume(shape.value())); } - TensorPtr tensor = BufferManager::cpu(shape.value_or(shape1d), nvinfer1::DataType::kINT32); + TensorPtr tensor = BufferManager::cpu(shape.value_or(shape1d), tensorrt_llm::DataType::kINT32); auto tensorRange = BufferRange<TokenIdType>(*tensor); std::copy(str.begin(), str.end(), tensorRange.begin()); return tensor; @@ -42,7 +43,7 @@ TensorPtr initTensor(std::string str, std::optional<ITensor::Shape> shape) TensorConstPtr RandomTokenLogits::tokenToLogits(TokenIdType token) const { - TensorPtr logits = BufferManager::cpu(mVocabulary->getShape(), nvinfer1::DataType::kFLOAT); + TensorPtr logits = BufferManager::cpu(mVocabulary->getShape(), tensorrt_llm::DataType::kFLOAT); tokenToLogits(logits, token); return logits; } @@ -152,7 +153,7 @@ void RandomTokenLogits::logitsToTensor(TensorPtr const& tokens, TensorConstPtr c TensorConstPtr RandomTokenLogits::logitsToTensor(TensorConstPtr const& logits) const { auto len = logits->getShape().d[0]; - TensorPtr result = BufferManager::cpu(ITensor::makeShape({len}), nvinfer1::DataType::kINT32); + TensorPtr result = BufferManager::cpu(ITensor::makeShape({len}), tensorrt_llm::DataType::kINT32); logitsToTensor(result, logits); return result; } @@ -209,7 +210,7 @@ bool RandomLlm::verify(SizeType32 const offset, TensorConstPtr const& script) co void RandomLlm::forward(TensorPtr const& output, runtime::SizeType32 startId, TensorConstPtr const& input, TensorConstPtr const& offsets, TensorConstPtr const mask) const { - TensorPtr posIds = BufferManager::cpu(input->getShape(), nvinfer1::DataType::kINT32); + TensorPtr posIds = BufferManager::cpu(input->getShape(), tensorrt_llm::DataType::kINT32); BufferRange<SizeType32> idRange(*posIds); BufferRange<SizeType32 const> offsetRange(*offsets); for (auto i = 0; i < idRange.size(); i++) @@ -226,7 +227,7 @@ void RandomLlm::forward(TensorPtr const& output, TensorConstPtr const& input, Te TLLM_CHECK(ITensor::volume(input->getShape()) == ITensor::volume(position->getShape())); TLLM_CHECK(ITensor::volume(output->getShape()) == ITensor::volume(input->getShape()) * mTable->getVocabSize()); - TensorPtr tokens = BufferManager::cpu(input->getShape(), nvinfer1::DataType::kINT32); + TensorPtr tokens = BufferManager::cpu(input->getShape(), tensorrt_llm::DataType::kINT32); foretell(tokens, input, position, mask); // foretellOld(tokens, input, position); mTable->tensorToLogits(output, tokens); @@ -247,7 +248,7 @@ void LookaheadRandomLlm::foretell(TensorPtr const& output, TensorConstPtr const& TLLM_CHECK(mask->getShape().d[1] >= len); } - TensorPtr maskRebuilt = BufferManager::cpu(ITensor::makeShape({len, len}), nvinfer1::DataType::kBOOL); + TensorPtr maskRebuilt = BufferManager::cpu(ITensor::makeShape({len, len}), tensorrt_llm::DataType::kBOOL); posIdsToMask(maskRebuilt, position); auto outputRange = BufferRange<TokenIdType>(*output); diff --git a/cpp/tests/unit_tests/layers/randomLlm.h b/cpp/tests/unit_tests/layers/randomLlm.h index a6e898baedf5..b0f0564944dc 100644 --- a/cpp/tests/unit_tests/layers/randomLlm.h +++ b/cpp/tests/unit_tests/layers/randomLlm.h @@ -18,6 +18,7 @@ #include <gtest/gtest.h> #include <list> +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/runtimeKernels.h" @@ -77,7 +78,7 @@ class AsciiRandomTokenLogits : public RandomTokenLogits : RandomTokenLogits( []() { - auto vocab = BufferManager::cpu(ITensor::makeShape({128}), nvinfer1::DataType::kINT32); + auto vocab = BufferManager::cpu(ITensor::makeShape({128}), tensorrt_llm::DataType::kINT32); auto vocabRange = BufferRange<TokenIdType>(*vocab); TokenIdType token{0}; std::for_each(vocabRange.begin(), vocabRange.end(), [&token](auto& v) { v = token++; }); diff --git a/cpp/tests/unit_tests/multi_gpu/cacheTransceiverTest.cpp b/cpp/tests/unit_tests/multi_gpu/cacheTransceiverTest.cpp index 401cdbfd5d44..1f216578f669 100644 --- a/cpp/tests/unit_tests/multi_gpu/cacheTransceiverTest.cpp +++ b/cpp/tests/unit_tests/multi_gpu/cacheTransceiverTest.cpp @@ -31,10 +31,12 @@ #include "tensorrt_llm/batch_manager/cacheFormatter.h" #include "tensorrt_llm/batch_manager/cacheTransceiver.h" +#include "tensorrt_llm/batch_manager/contextTransferCoordinator.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" #include "tensorrt_llm/executor/cache_transmission/mpi_utils/connection.h" #include "tensorrt_llm/executor/dataTransceiverState.h" @@ -42,6 +44,7 @@ #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/testing/kvCacheManagerTestUtil.h" +#include <chrono> #include <csignal> #include <cstddef> #include <cstdint> @@ -53,6 +56,7 @@ #include <tensorrt_llm/batch_manager/cacheTransBuffer.h> #include <tensorrt_llm/batch_manager/mlaCacheFormatter.h> #include <tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h> +#include <thread> #include "gtest/gtest.h" #include <gmock/gmock.h> @@ -89,6 +93,95 @@ T serializeDeserialize(T const& val) } // namespace +TEST(ContextTransferCoordinatorTest, CommitsStaggeredSuccessAndFailureWithoutCollectivePolling) +{ + auto& world = tensorrt_llm::mpi::MpiComm::world(); + if (world.getSize() < 2) + { + GTEST_SKIP() << "mpirun with at least two processes is required to run this test."; + } + + auto comm = std::make_shared<CacheTransceiverComm>(std::addressof(world)); + { + ContextTransferCoordinator coordinator(comm); + auto waitForOutcome = [&](std::uint64_t const requestId, bool const expectFailure) + { + bool observed = false; + auto const deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!observed && std::chrono::steady_clock::now() < deadline) + { + auto result = coordinator.poll(); + observed = expectFailure ? result.failedRequestIds.count(requestId) != 0 + : result.completedRequestIds.count(requestId) != 0; + if (!observed) + { + std::this_thread::yield(); + } + } + EXPECT_TRUE(observed); + }; + auto waitForTimeout = [&](std::uint64_t const requestId) + { + bool observed = false; + auto const deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!observed && std::chrono::steady_clock::now() < deadline) + { + auto const result = coordinator.poll(); + observed = result.timedOutRequestIds.count(requestId) != 0; + if (!observed) + { + std::this_thread::yield(); + } + } + EXPECT_TRUE(observed); + }; + + constexpr std::uint64_t kCompletedRequestId = 644815201; + world.barrier(); + if (world.getRank() == world.getSize() - 1) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + coordinator.publishLocalOutcome(kCompletedRequestId, /*failed=*/false); + if (world.getRank() != world.getSize() - 1) + { + auto const earlyResult = coordinator.poll(); + EXPECT_TRUE(earlyResult.completedRequestIds.empty()); + EXPECT_TRUE(earlyResult.failedRequestIds.empty()); + } + waitForOutcome(kCompletedRequestId, /*expectFailure=*/false); + + constexpr std::uint64_t kFailedRequestId = 644815202; + world.barrier(); + coordinator.publishLocalOutcome(kFailedRequestId, /*failed=*/world.getRank() == 0); + waitForOutcome(kFailedRequestId, /*expectFailure=*/true); + + constexpr std::uint64_t kTimedOutRequestId = 644815203; + world.barrier(); + if (world.getRank() == 0) + { + coordinator.publishTimeout(kTimedOutRequestId); + coordinator.publishTimeout(kTimedOutRequestId); + } + waitForTimeout(kTimedOutRequestId); + coordinator.publishLocalOutcome(kTimedOutRequestId, /*failed=*/false); + waitForOutcome(kTimedOutRequestId, /*expectFailure=*/true); + + constexpr std::uint64_t kInvalidOrderingRequestId = 644815204; + world.barrier(); + coordinator.publishLocalOutcome(kInvalidOrderingRequestId, /*failed=*/false); + EXPECT_ANY_THROW(coordinator.publishTimeout(kInvalidOrderingRequestId)); + waitForOutcome(kInvalidOrderingRequestId, /*expectFailure=*/false); + + // Exercise asymmetric but orderly teardown after every decision has committed. + if (world.getRank() == 0) + { + std::this_thread::sleep_for(std::chrono::milliseconds(25)); + } + } + world.barrier(); +} + class RequestInfoTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) { public: @@ -105,7 +198,7 @@ TEST_F(RequestInfoTest, Basic) } auto state = std::make_unique<texec::DataTransceiverState>(); state->setCommState(texec::kv_cache::CommState{12, "127.0.0.1"}); - state->setCacheState(texec::kv_cache::CacheState{10, 12, 128, 128, 8, 8, 8, {10}, nvinfer1::DataType::kFLOAT}); + state->setCacheState(texec::kv_cache::CacheState{10, 12, 128, 128, 8, 8, 8, {10}, tensorrt_llm::DataType::kFLOAT}); RequestInfo info{1, *state}; auto info2 = serializeDeserialize(info); EXPECT_EQ(info, info2); @@ -135,7 +228,7 @@ TEST_F(CacheConfigTest, EqualTo) constexpr SizeType32 nbRnnLayers{2}; constexpr SizeType32 nbHeads{12}; constexpr SizeType32 hiddenSize{768}; - constexpr nvinfer1::DataType dtype{nvinfer1::DataType::kFLOAT}; + constexpr tensorrt_llm::DataType dtype{tensorrt_llm::DataType::kFLOAT}; constexpr SizeType32 tokensPerBlock{64}; constexpr SizeType32 tensorParallelism{8}; constexpr SizeType32 pipelineParallelism{2}; @@ -193,7 +286,7 @@ class SymmetricalCacheTest : public ::testing::Test // NOLINT(cppcoreguidelines- return mWorldSize; } - void setUpCacheManager() + void setUpCacheManager(bool enableBlockReuse = false) { auto constexpr numLayers = 4; auto constexpr numHeads = 2; @@ -215,8 +308,7 @@ class SymmetricalCacheTest : public ::testing::Test // NOLINT(cppcoreguidelines- auto totalNumBlocks = mMaxNumSequences * numBlocksPerSeq; auto constexpr blocksInSecondaryPool = 0; - auto constexpr enableBlockReuse = false; - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {totalNumBlocks, blocksInSecondaryPool}}}; @@ -329,6 +421,21 @@ class SymmetricalCacheTest : public ::testing::Test // NOLINT(cppcoreguidelines- return std::make_unique<LlmRequest>(mRequestId++, std::move(request)); } + // Generation-only request whose DataTransceiverState carries the arbitrary-transfer + // provenance marker, as getSerializedDataTransceiverState would produce. + auto makeArbitraryLlmRequest(SizeType32 length, LlmRequest::RequestIdType arbitraryId) + { + constexpr SizeType32 maxNewTokens{1}; + texec::Request request{VecTokens(length, length), maxNewTokens}; + auto state = std::make_unique<texec::DataTransceiverState>(); + state->setCommState(*mContextCommState); + state->setCacheState(*mCacheState); + state->setIsArbitraryTransferState(true); + auto stats = texec::ContextPhaseParams({}, arbitraryId, state.release(), std::nullopt); + request.setContextPhaseParams(std::move(stats)); + return std::make_unique<LlmRequest>(arbitraryId, std::move(request)); + } + void addRequestAndTransportCache(std::shared_ptr<LlmRequest> const& llmRequest) { auto constexpr beamIdx{0}; @@ -426,10 +533,83 @@ TEST_F(SymmetricalCacheTest, SimpleTest) } } +TEST_F(SymmetricalCacheTest, ArbitraryTransferTest) +{ + auto worldSize = setUpCommunicator(); + if (worldSize != 2) + { + GTEST_SKIP() << "mpirun 2 processes is required to run this test."; + } + setUpCacheManager(/*enableBlockReuse=*/true); + setUpCacheTransceiver(); + + // 3 full blocks (tokensPerBlock = 8) so every requested chunk fully matches a stored block. + constexpr SizeType32 promptLen = 24; + constexpr SizeType32 missPromptLen = 40; + constexpr LlmRequest::RequestIdType arbitraryId = 4242; + auto constexpr beamIdx{0}; + auto constexpr beamWidth{1}; + + if (isSender) + { + // Store a patterned request in the reuse tree; no LlmRequest exists on the + // sender for the transfers below. + auto request = makeLlmRequest(promptLen); + mManager->addSequenceBatch( + {{{request->mRequestId, request->getNumTokens(beamIdx), beamWidth}}}, {std::ref(*request)}); + auto blockRange = BlockRange::fromAllBlockIds(*mManager, request->mRequestId); + for (auto const& windowSize : blockRange.getWindowSizes()) + { + auto blockRangeForWindow = blockRange.getBlockRangeForWindow(windowSize); + for (auto it = blockRangeForWindow.begin(); it != blockRangeForWindow.end(); ++it) + { + TLLM_CUDA_CHECK(cudaMemset(it->data(), request->getPromptLen(), it->getSizeInBytes())); + } + } + tensorrt_llm::testing::KvCacheManagerTestUtil::simulatePrefillCompletion(*request); + // A completed context request carries one generated token beyond the prompt; + // without it, storeBlocksForReuse drops the trailing prompt token and the + // final block never enters the reuse tree. + request->addNewToken(0, beamIdx); + mManager->removeSequence(request->mRequestId, request); + } + else + { + // Hit: the requested tokens are stored in the sender's reuse tree. + std::shared_ptr<LlmRequest> request = makeArbitraryLlmRequest(promptLen, arbitraryId); + mManager->addSequenceBatch( + {{{request->mRequestId, request->getNumTokens(beamIdx), beamWidth}}}, {std::ref(*request)}); + auto future = mRequester->receiveAsync(request); + future.get(); + TLLM_CUDA_CHECK(cudaDeviceSynchronize()); + auto blockRange = BlockRange::fromAllBlockIds(*mManager, request->mRequestId); + for (auto const& windowSize : blockRange.getWindowSizes()) + { + auto blockRangeForWindow = blockRange.getBlockRangeForWindow(windowSize); + for (auto it = blockRangeForWindow.begin(); it != blockRangeForWindow.end(); ++it) + { + std::vector<uint8_t> bytes(it->getSizeInBytes()); + TLLM_CUDA_CHECK(cudaMemcpy(bytes.data(), it->data(), it->getSizeInBytes(), cudaMemcpyDeviceToHost)); + EXPECT_TRUE(std::all_of(bytes.begin(), bytes.end(), [](uint8_t i) { return i == (promptLen & 0xff); })); + } + } + + // Miss: tokens never stored on the sender are rejected; the rejection + // surfaces as an exception on the receive future. + std::shared_ptr<LlmRequest> missRequest = makeArbitraryLlmRequest(missPromptLen, arbitraryId + 1); + mManager->addSequenceBatch( + {{{missRequest->mRequestId, missRequest->getNumTokens(beamIdx), beamWidth}}}, {std::ref(*missRequest)}); + auto missFuture = mRequester->receiveAsync(missRequest); + EXPECT_THROW(missFuture.get(), tensorrt_llm::common::TllmException); + } + // The sender must not tear down while the receiver's transfers are in flight. + tensorrt_llm::mpi::MpiComm::world().barrier(); +} + #if ENABLE_MULTI_DEVICE -using AsymmetricTestParam = std::tuple<int, int, int, int, int, int, int, int, int, int, nvinfer1::DataType, int, bool, - bool, bool, bool, bool, int, int>; +using AsymmetricTestParam = std::tuple<int, int, int, int, int, int, int, int, int, int, tensorrt_llm::DataType, int, + bool, bool, bool, bool, bool, int, int>; // CPMetaData struct to hold CP-specific information struct CPMetaData @@ -579,7 +759,7 @@ class AsymmetricalCacheTest : public ::testing::TestWithParam<AsymmetricTestPara } void setUpCacheManager(int numLayers, int numHeads, int sizePerHead, int tokensPerBlock, - nvinfer1::DataType dataType, int kvFactor = 2, bool isMLA = false, bool enableDPAttention = false, + tensorrt_llm::DataType dataType, int kvFactor = 2, bool isMLA = false, bool enableDPAttention = false, bool isWindow = false, bool isIndexerKCache = true, int indexerDimPerHead = 0, int indexerKCacheQuantBlockSize = 128) { @@ -790,15 +970,18 @@ class AsymmetricalCacheTest : public ::testing::TestWithParam<AsymmetricTestPara = [this, bufferManagers]() { return createCacheFormatter(mManager.get(), bufferManagers, mIsMLA); }; TLLM_LOG_DEBUG("setUpCacheTransceiver makeFormatter"); + // Generate a per-instance ID so each ctx/gen instance writes to + // its own CSV files (mirrors CacheTransceiver behaviour). + auto instanceId = "test_" + std::to_string(tensorrt_llm::mpi::MpiComm::world().getRank()); if (mIsContext) { - mSender = std::make_unique<CacheSender>( - mConnectionManager.get(), mRankInInstance, CacheTransferLayer(*mCacheState, makeFormatter())); + mSender = std::make_unique<CacheSender>(mConnectionManager.get(), mRankInInstance, + CacheTransferLayer(*mCacheState, makeFormatter()), instanceId); } else { - mRequester = std::make_unique<CacheReceiver>( - mConnectionManager.get(), mRankInInstance, CacheTransferLayer(*mCacheState, makeFormatter())); + mRequester = std::make_unique<CacheReceiver>(mConnectionManager.get(), mRankInInstance, + CacheTransferLayer(*mCacheState, makeFormatter()), instanceId); } TLLM_LOG_DEBUG("setUpCacheTransceiver mSender"); @@ -1248,7 +1431,7 @@ class AsymmetricalCacheTest : public ::testing::TestWithParam<AsymmetricTestPara } std::variant<double, float, int16_t, int8_t, uint8_t> generateExpectedValue(size_t initial, int windowSize, - int tokenId, int layerId, int headId, int hiddenId, bool key, nvinfer1::DataType dataType) + int tokenId, int layerId, int headId, int hiddenId, bool key, tensorrt_llm::DataType dataType) { size_t seed = 0; std::size_t hashValue = std::hash<size_t>{}(initial); @@ -1324,7 +1507,7 @@ TEST_P(AsymmetricalCacheTest, TestCase) int numHeads = std::get<7>(param); int sizePerHead = std::get<8>(param); int tokensPerBlock = std::get<9>(param); - nvinfer1::DataType dataType = std::get<10>(param); + tensorrt_llm::DataType dataType = std::get<10>(param); int kvFactor = std::get<11>(param); bool isMLA = std::get<12>(param); @@ -1439,7 +1622,7 @@ TEST_P(AsymmetricalCacheTestWithDP, TestCase) int numHeads = std::get<7>(param); int sizePerHead = std::get<8>(param); int tokensPerBlock = std::get<9>(param); - nvinfer1::DataType dataType = std::get<10>(param); + tensorrt_llm::DataType dataType = std::get<10>(param); int kvFactor = std::get<11>(param); bool isMLA = std::get<12>(param); @@ -1576,7 +1759,7 @@ TEST_P(UnexpectedTerminationRaceTest, UnexpectedTerminationRaceTest) int numHeads = std::get<7>(param); int sizePerHead = std::get<8>(param); int tokensPerBlock = std::get<9>(param); - nvinfer1::DataType dataType = std::get<10>(param); + tensorrt_llm::DataType dataType = std::get<10>(param); int kvFactor = std::get<11>(param); bool isMLA = std::get<12>(param); @@ -1723,87 +1906,87 @@ TEST_P(UnexpectedTerminationRaceTest, UnexpectedTerminationRaceTest) INSTANTIATE_TEST_CASE_P(UnexpectedTerminationRaceTest, UnexpectedTerminationRaceTest, testing::Combine(testing::Values(2), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(4), testing::Values(4), testing::Values(16), - testing::Values(nvinfer1::DataType::kFLOAT), testing::Values(2), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), - testing::Values(128))); + testing::Values(tensorrt_llm::DataType::kFLOAT), testing::Values(2), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), + testing::Values(0), testing::Values(128))); // Waive off isWindow test for now INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0, AsymmetricalCacheTest, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(4), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), - testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(/*true,*/ false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), + testing::Values(/*true,*/ false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithWindow, AsymmetricalCacheTest, testing::Combine(testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(5), testing::Values(4), testing::Values(4), testing::Values(8), - testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(true), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1, AsymmetricalCacheTest, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(8), testing::Values(4), testing::Values(4), testing::Values(8), - testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false /*, true*/), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1EvenLayer, AsymmetricalCacheTest, testing::Combine(testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(10), testing::Values(4), testing::Values(4), testing::Values(8), - testing::Values(nvinfer1::DataType::kFLOAT), testing::Values(2), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), - testing::Values(128))); + testing::Values(tensorrt_llm::DataType::kFLOAT), testing::Values(2), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), + testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest2EvenLayer, AsymmetricalCacheTest, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(10), testing::Values(4), testing::Values(4), testing::Values(8), - testing::Values(nvinfer1::DataType::kFLOAT), testing::Values(2), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), - testing::Values(128))); + testing::Values(tensorrt_llm::DataType::kFLOAT), testing::Values(2), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), + testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest2, AsymmetricalCacheTest, testing::Combine(testing::Values(1), testing::Values(2), testing::Values(1), testing::Values(1), testing::Values(1, 4), testing::Values(1), testing::Values(16), testing::Values(16), testing::Values(4), - testing::Values(8), testing::Values(nvinfer1::DataType::kFLOAT), testing::Values(2), testing::Values(false), + testing::Values(8), testing::Values(tensorrt_llm::DataType::kFLOAT), testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0ForMLA, AsymmetricalCacheTest, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), - testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1ForMLA, AsymmetricalCacheTest, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(8), - testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), + testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1ForMLAEvenLayer, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(10), testing::Values(1), testing::Values(4), testing::Values(8), - testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), + testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(false, true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest2ForMLAEvenLayer, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(10), testing::Values(1), testing::Values(4), testing::Values(8), - testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), + testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(false, true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0ForMLAWithIndexerKCache, AsymmetricalCacheTest, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), - testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(false), - testing::Values(true), testing::Values(256), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(true), testing::Values(256), testing::Values(128))); // Tests cases where there's non-trivial TP and PP on context side but only CP on gen side. INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0WithCPForMLA, AsymmetricalCacheTest, @@ -1817,7 +2000,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0WithCPForMLA, AsymmetricalCacheTest, /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -1839,7 +2022,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1WithCPForMLA, AsymmetricalCacheTest, /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -1861,7 +2044,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0WithCPForGQA, AsymmetricalCacheTest, /*numHeads*/ testing::Values(4), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(2), /*isMLA*/ testing::Values(false), /*contextDP*/ testing::Values(false), @@ -1880,7 +2063,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1WithCPForGQA, AsymmetricalCacheTest, /*numHeads*/ testing::Values(4), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(2), /*isMLA*/ testing::Values(false), /*contextDP*/ testing::Values(false), @@ -1899,7 +2082,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0WithCPForMLAUnevenLayer, Asymmetrical /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -1921,7 +2104,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1WithCPForMLAUnevenLayer, Asymmetrical /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -1943,7 +2126,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest2WithCPForMLAUnevenLayer, Asymmetrical /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -1965,7 +2148,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithCPAndDPForMLA0, AsymmetricalCacheT /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -1987,7 +2170,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithCPAndDPForMLA1, AsymmetricalCacheT /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(true), @@ -2009,7 +2192,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithCPAndDPForGQA0, AsymmetricalCacheT /*numHeads*/ testing::Values(4), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(2), /*isMLA*/ testing::Values(false), /*contextDP*/ testing::Values(false), @@ -2028,7 +2211,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithCPAndDPForGQA1, AsymmetricalCacheT /*numHeads*/ testing::Values(4), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), + /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), /*kvFactor*/ testing::Values(2), /*isMLA*/ testing::Values(false), /*contextDP*/ testing::Values(true), @@ -2038,97 +2221,97 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithCPAndDPForGQA1, AsymmetricalCacheT INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForMLA1, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), - testing::Values(true), testing::Values(true), testing::Values(true), testing::Values(false), + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(1), testing::Values(true), testing::Values(true), testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForMLA2, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), - testing::Values(true), testing::Values(true), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(1), testing::Values(true), testing::Values(true), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForMLA3, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), - testing::Values(true), testing::Values(false), testing::Values(true), testing::Values(false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(true), + testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForMLA4, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(2), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(16), - testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), + testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForMLA5, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(2), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(16), - testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), + testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLA, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(4), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), - testing::Values(false), testing::Values(true), testing::Values(true), testing::Values(false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(2), testing::Values(false), testing::Values(true), testing::Values(true), + testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLA1, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(4), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), - testing::Values(false), testing::Values(true), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(2), testing::Values(false), testing::Values(true), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLA2, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(4), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), - testing::Values(false), testing::Values(false), testing::Values(true), testing::Values(false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(true), + testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate0, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(2), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), - testing::Values(false), testing::Values(true, false), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(2), testing::Values(false), testing::Values(true, false), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate0EvenLayer, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(5), testing::Values(2), testing::Values(4), testing::Values(16), - testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(2), testing::Values(false), testing::Values(true, false), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate1, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(2), testing::Values(2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), - testing::Values(false), testing::Values(true, false), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(2), testing::Values(false), testing::Values(true, false), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate2, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4, 2), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(2), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), - testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate3, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(2), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(2), testing::Values(4), testing::Values(16), - testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate4, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(1, 2), testing::Values(2), testing::Values(1), testing::Values(4), testing::Values(1, 2), testing::Values(4), - testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), - testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); #endif @@ -2139,7 +2322,7 @@ TEST(targetTest, CacheStateNODP) int const numHeads = 2; int const sizePerHead = 64; int const tokensPerBlock = 64; - auto const dataType = nvinfer1::DataType::kFLOAT; + auto const dataType = tensorrt_llm::DataType::kFLOAT; bool const isMLA = true; int const kvFactor = 2; @@ -2429,7 +2612,7 @@ TEST(targetTest, CacheStateNODPForGQAWithCP) int const numHeads = 4; int const sizePerHead = 64; int const tokensPerBlock = 64; - auto const dataType = nvinfer1::DataType::kFLOAT; + auto const dataType = tensorrt_llm::DataType::kFLOAT; bool const isMLA = false; int const kvFactor = 2; @@ -2645,7 +2828,7 @@ TEST(targetTest, CacheStateContextDP) int const numHeads = 2; int const sizePerHead = 64; int const tokensPerBlock = 64; - auto const dataType = nvinfer1::DataType::kFLOAT; + auto const dataType = tensorrt_llm::DataType::kFLOAT; bool const isMLA = true; int const kvFactor = 2; diff --git a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceFusionTest.cu b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceFusionTest.cu index 4b9c7af29a46..e95c84d7d05a 100644 --- a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceFusionTest.cu +++ b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceFusionTest.cu @@ -25,6 +25,7 @@ #include <random> #include <vector> +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceWorkspace.h" #include "tensorrt_llm/kernels/quantization.h" @@ -235,28 +236,28 @@ template <> struct DTypeTraits<half> { static constexpr ncclDataType_t kNCCLDataType = ncclFloat16; - static constexpr nvinfer1::DataType kTRTDataType = nvinfer1::DataType::kHALF; + static constexpr tensorrt_llm::DataType kTRTDataType = tensorrt_llm::DataType::kHALF; }; template <> struct DTypeTraits<__nv_bfloat16> { static constexpr ncclDataType_t kNCCLDataType = ncclBfloat16; - static constexpr nvinfer1::DataType kTRTDataType = nvinfer1::DataType::kBF16; + static constexpr tensorrt_llm::DataType kTRTDataType = tensorrt_llm::DataType::kBF16; }; template <> struct DTypeTraits<float> { static constexpr ncclDataType_t kNCCLDataType = ncclFloat32; - static constexpr nvinfer1::DataType kTRTDataType = nvinfer1::DataType::kFLOAT; + static constexpr tensorrt_llm::DataType kTRTDataType = tensorrt_llm::DataType::kFLOAT; }; template <typename DType, ar_fusion::AllReduceFusionPattern Pattern> class TestRunner { static constexpr ncclDataType_t kNCCLDataType = DTypeTraits<DType>::kNCCLDataType; - static constexpr nvinfer1::DataType kTRTDataType = DTypeTraits<DType>::kTRTDataType; + static constexpr tensorrt_llm::DataType kTRTDataType = DTypeTraits<DType>::kTRTDataType; static constexpr bool kFP4QuantOutSupport = !std::is_same_v<DType, float>; static_assert(kFP4QuantOutSupport || Pattern != ar_fusion::AllReduceFusionPattern::kARResidualRMSNormFP4Quant, "kARResidualRMSNormFP4Quant is not supported for float dtype"); diff --git a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceKernelTest.cu b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceKernelTest.cu index b3d120e7015c..78747373accf 100644 --- a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceKernelTest.cu +++ b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceKernelTest.cu @@ -32,6 +32,7 @@ #include <type_traits> #include <vector> +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/customAllReduceKernels.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" @@ -212,7 +213,7 @@ public: { } - void set_params(AllReduceParams& params, nvinfer1::DataType dataType, int token_num, int hidden_size, + void set_params(AllReduceParams& params, tensorrt_llm::DataType dataType, int token_num, int hidden_size, AllReduceFusionOp op) const { int world_size = world_config.getSize(); @@ -316,7 +317,7 @@ bool test(Workspace const& workspace, int token_num, int hidden_size, bool has_b in.copy_from(input_buffer.data()); AllReduceParams params; - workspace.set_params(params, nvinfer1::DataType::kHALF, token_num, hidden_size, fusion_op); + workspace.set_params(params, tensorrt_llm::DataType::kHALF, token_num, hidden_size, fusion_op); params.ranks_per_node = world_size; params.local_rank = rank; params.local_output_buffer_ptr = out.data(); @@ -334,21 +335,21 @@ bool test(Workspace const& workspace, int token_num, int hidden_size, bool has_b cudaEventCreate(&begin); cudaEventCreate(&end); lamportInitialize( - params.fusion_params.lamport_peer_comm_buffer_ptrs[rank], message_size, nvinfer1::DataType::kHALF, s); + params.fusion_params.lamport_peer_comm_buffer_ptrs[rank], message_size, tensorrt_llm::DataType::kHALF, s); lamportInitialize(params.fusion_params.lamport_peer_comm_buffer_ptrs[rank + MAX_RANKS_PER_NODE], message_size, - nvinfer1::DataType::kHALF, s); + tensorrt_llm::DataType::kHALF, s); lamportInitialize(params.fusion_params.lamport_peer_comm_buffer_ptrs[rank + MAX_RANKS_PER_NODE * 2], message_size, - nvinfer1::DataType::kHALF, s); + tensorrt_llm::DataType::kHALF, s); cudaDeviceSynchronize(); comm.barrier(); for (int i = 0; i < warmup; ++i) { - customAllReduce(params, nvinfer1::DataType::kHALF, runtime_strategy, config, fusion_op, s); + customAllReduce(params, tensorrt_llm::DataType::kHALF, runtime_strategy, config, fusion_op, s); } cudaEventRecord(begin, s); for (int i = 0; i < iter; ++i) { - customAllReduce(params, nvinfer1::DataType::kHALF, runtime_strategy, config, fusion_op, s); + customAllReduce(params, tensorrt_llm::DataType::kHALF, runtime_strategy, config, fusion_op, s); } cudaEventRecord(end, s); cudaEventSynchronize(end); @@ -462,7 +463,7 @@ bool test_prepostnorm(Workspace const& workspace, int token_num, int hidden_size in.copy_from(input_buffer.data()); AllReduceParams params; - workspace.set_params(params, nvinfer1::DataType::kHALF, token_num, hidden_size, fusion_op); + workspace.set_params(params, tensorrt_llm::DataType::kHALF, token_num, hidden_size, fusion_op); params.ranks_per_node = world_size; params.local_rank = rank; params.local_output_buffer_ptr = out.data(); @@ -484,12 +485,12 @@ bool test_prepostnorm(Workspace const& workspace, int token_num, int hidden_size comm.barrier(); for (int i = 0; i < warmup; ++i) { - customAllReduce(params, nvinfer1::DataType::kHALF, runtime_strategy, config, fusion_op, s); + customAllReduce(params, tensorrt_llm::DataType::kHALF, runtime_strategy, config, fusion_op, s); } cudaEventRecord(begin, s); for (int i = 0; i < iter; ++i) { - customAllReduce(params, nvinfer1::DataType::kHALF, runtime_strategy, config, fusion_op, s); + customAllReduce(params, tensorrt_llm::DataType::kHALF, runtime_strategy, config, fusion_op, s); } cudaEventRecord(end, s); cudaEventSynchronize(end); diff --git a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/gemmAllReduceTest.cu b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/gemmAllReduceTest.cu index 415a3c32da81..2ff2c8c37b9c 100644 --- a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/gemmAllReduceTest.cu +++ b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/gemmAllReduceTest.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2022-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,12 +25,11 @@ #else #include "allreduce_gemm_runner.h" #endif -#include "common.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/userbuffers/ub_interface.h" #include "tensorrt_llm/runtime/ipcNvlsMemory.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include <NvInferRuntime.h> #include "cute/tensor.hpp" #include "cutlass/cutlass.h" @@ -53,8 +52,59 @@ #include "cutlass/util/reference/host/gett.hpp" #include "cutlass/util/reference/host/tensor_fill.h" +namespace tensorrt_llm::testing +{ + +/** + * GPU timer for recording the elapsed time across kernel(s) launched in GPU stream + */ +struct GpuTimer +{ + cudaStream_t _stream_id; + cudaEvent_t _start; + cudaEvent_t _stop; + + /// Constructor + GpuTimer() + : _stream_id(0) + { + TLLM_CUDA_CHECK(cudaEventCreate(&_start)); + TLLM_CUDA_CHECK(cudaEventCreate(&_stop)); + } + + /// Destructor + ~GpuTimer() + { + TLLM_CUDA_CHECK(cudaEventDestroy(_start)); + TLLM_CUDA_CHECK(cudaEventDestroy(_stop)); + } + + /// Start the timer for a given stream (defaults to the default stream) + void start(cudaStream_t stream_id = 0) + { + _stream_id = stream_id; + TLLM_CUDA_CHECK(cudaEventRecord(_start, _stream_id)); + } + + /// Stop the timer + void stop() + { + TLLM_CUDA_CHECK(cudaEventRecord(_stop, _stream_id)); + } + + /// Return the elapsed time (in milliseconds) + float elapsed_millis() + { + float elapsed = 0.0; + TLLM_CUDA_CHECK(cudaEventSynchronize(_stop)); + TLLM_CUDA_CHECK(cudaEventElapsedTime(&elapsed, _start, _stop)); + return elapsed; + } +}; + +} // namespace tensorrt_llm::testing + using namespace cutlass; -using namespace nvinfer1; using namespace tensorrt_llm::mpi; using namespace tensorrt_llm::runtime; using namespace tensorrt_llm::common; @@ -238,7 +288,7 @@ struct ToType template <> struct ToType<cutlass::bfloat16_t> { - nvinfer1::DataType trt_value = nvinfer1::DataType::kBF16; + tensorrt_llm::DataType trt_value = tensorrt_llm::DataType::kBF16; ncclDataType_t nccl_value = ncclBfloat16; char const* str_value = "bf16"; }; @@ -246,7 +296,7 @@ struct ToType<cutlass::bfloat16_t> template <> struct ToType<cutlass::half_t> { - nvinfer1::DataType trt_value = nvinfer1::DataType::kHALF; + tensorrt_llm::DataType trt_value = tensorrt_llm::DataType::kHALF; ncclDataType_t nccl_value = ncclFloat16; char const* str_value = "fp16"; }; @@ -254,7 +304,7 @@ struct ToType<cutlass::half_t> template <> struct ToType<cutlass::float_e4m3_t> { - nvinfer1::DataType trt_value = nvinfer1::DataType::kFP8; + tensorrt_llm::DataType trt_value = tensorrt_llm::DataType::kFP8; ncclDataType_t nccl_value = ncclFloat8e4m3; char const* str_value = "fp8_e4m3"; }; diff --git a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/moeAllReduceFusionTest.cu b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/moeAllReduceFusionTest.cu index 8abccf214b7a..4b81a4133fa9 100644 --- a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/moeAllReduceFusionTest.cu +++ b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/moeAllReduceFusionTest.cu @@ -24,6 +24,7 @@ #include <random> #include <vector> +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceWorkspace.h" #include "tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h" #include "tensorrt_llm/kernels/quantization.h" @@ -392,8 +393,8 @@ class MoEARFuseTestRunner { static_assert(std::is_same_v<DType, half> || std::is_same_v<DType, __nv_bfloat16>); static constexpr ncclDataType_t kNCCLDataType = std::is_same_v<DType, half> ? ncclFloat16 : ncclBfloat16; - static constexpr nvinfer1::DataType kTRTDataType - = std::is_same_v<DType, half> ? nvinfer1::DataType::kHALF : nvinfer1::DataType::kBF16; + static constexpr tensorrt_llm::DataType kTRTDataType + = std::is_same_v<DType, half> ? tensorrt_llm::DataType::kHALF : tensorrt_llm::DataType::kBF16; public: MoEARFuseTestRunner(int max_token_num, int hidden_dim, int max_expert_num) diff --git a/cpp/tests/unit_tests/multi_gpu/mpiUtilsTest.cpp b/cpp/tests/unit_tests/multi_gpu/mpiUtilsTest.cpp index 221cd98b5f02..941d7cb53ed6 100644 --- a/cpp/tests/unit_tests/multi_gpu/mpiUtilsTest.cpp +++ b/cpp/tests/unit_tests/multi_gpu/mpiUtilsTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2022-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,11 +20,12 @@ #include "tensorrt_llm/runtime/utils/mpiUtils.h" #if ENABLE_MULTI_DEVICE -#include "tensorrt_llm/plugins/common/plugin.h" #include <nccl.h> #endif // ENABLE_MULTI_DEVICE #include <algorithm> +#include <cstdint> +#include <thread> namespace mpi = tensorrt_llm::mpi; namespace tr = tensorrt_llm::runtime; @@ -47,6 +48,29 @@ TEST(MPIUtils, WorldRankAndSize) EXPECT_LE(rank, size); } +#if ENABLE_MULTI_DEVICE +TEST(MPIUtils, AsyncSendCanBePolled) +{ + auto& comm = mpi::MpiComm::world(); + auto const rank = comm.getRank(); + auto const size = comm.getSize(); + auto const destination = (rank + 1) % size; + auto const source = (rank + size - 1) % size; + std::uint64_t const sentValue = static_cast<std::uint64_t>(rank); + std::uint64_t receivedValue = 0; + + auto request + = comm.sendAsync(&sentValue, 1, mpi::MpiType::kUINT64, destination, mpi::MpiTag::kContextTransferEvent); + static_cast<void>(comm.recv(&receivedValue, 1, mpi::MpiType::kUINT64, source, mpi::MpiTag::kContextTransferEvent)); + while (!request->isCompleted()) + { + std::this_thread::yield(); + } + + EXPECT_EQ(receivedValue, static_cast<std::uint64_t>(source)); +} +#endif // ENABLE_MULTI_DEVICE + template <typename T> void testBroadcast() { @@ -92,11 +116,6 @@ TEST(MPIUtils, BroadcastNcclId) EXPECT_TRUE(std::any_of( id.internal, id.internal + sizeof(id.internal) / sizeof(id.internal[0]), [](auto x) { return x != 0; })); } - -TEST(MPIUtils, GlobalSessionHandle) -{ - EXPECT_EQ(tensorrt_llm::plugins::getCommSessionHandle(), &COMM_SESSION); -} #endif // ENABLE_MULTI_DEVICE template <typename T> diff --git a/cpp/tests/unit_tests/runtime/CMakeLists.txt b/cpp/tests/unit_tests/runtime/CMakeLists.txt index c022ba31ebca..3a171ee39877 100644 --- a/cpp/tests/unit_tests/runtime/CMakeLists.txt +++ b/cpp/tests/unit_tests/runtime/CMakeLists.txt @@ -35,7 +35,6 @@ add_gtest(samplingConfigTest samplingConfigTest.cpp) add_gtest(samplingTest samplingTest.cpp) add_gtest(sanitizerTest sanitizerTest.cpp) add_gtest(tllmBuffersTest tllmBuffersTest.cpp) -add_gtest(tllmRuntimeTest tllmRuntimeTest.cpp) add_gtest(transposeKVKernelTest transposeKVKernelTest.cpp) add_gtest(utilsTest utilsTest.cpp) add_gtest(virtualMemoryTest virtualMemoryTest.cpp) diff --git a/cpp/tests/unit_tests/runtime/bufferManagerTest.cpp b/cpp/tests/unit_tests/runtime/bufferManagerTest.cpp index 194cf88b6765..8bdc6d0352a0 100644 --- a/cpp/tests/unit_tests/runtime/bufferManagerTest.cpp +++ b/cpp/tests/unit_tests/runtime/bufferManagerTest.cpp @@ -17,6 +17,7 @@ #include <gtest/gtest.h> #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaMemPool.h" @@ -121,7 +122,7 @@ TEST_F(BufferManagerTest, Pointers) static_assert(std::is_same_v<decltype(trtPointerType), BufferDataType const>); static_assert(trtPointerType.isPointer()); static_assert(trtPointerType.getDataType() == TRTDataType<cppBaseType>::value); - static_assert(static_cast<nvinfer1::DataType>(trtPointerType) == BufferDataType::kTrtPointerType); + static_assert(static_cast<tensorrt_llm::DataType>(trtPointerType) == BufferDataType::kTrtPointerType); static_assert(trtPointerType == BufferDataType::kTrtPointerType); // uses implicit type conversion // The C++ type corresponding to the TensorRT type for storing pointers (int64_t) using cppStorageType = DataTypeTraits<trtPointerType>::type; diff --git a/cpp/tests/unit_tests/runtime/decodingLayerWorkspaceTest.cpp b/cpp/tests/unit_tests/runtime/decodingLayerWorkspaceTest.cpp index bb6ce6410ad5..74f8faa37c87 100644 --- a/cpp/tests/unit_tests/runtime/decodingLayerWorkspaceTest.cpp +++ b/cpp/tests/unit_tests/runtime/decodingLayerWorkspaceTest.cpp @@ -16,6 +16,7 @@ #include "tensorrt_llm/runtime/decodingLayerWorkspace.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/common/workspace.h" #include <gtest/gtest.h> #include <random> @@ -121,8 +122,9 @@ auto const tensorDataTypesTuples = testing::Combine(tensorDataTypes, tensorDataT auto const tensorShapeTuples = testing::Combine(tensorDimensions, tensorDimensions, tensorDimensions); auto const mirrorInWorkspaceParams = testing::Combine(tensorDataTypesTuples, tensorShapeTuples, randomSeeds); -using MirrorInWorkspaceParamType = std::tuple<std::tuple<nvinfer1::DataType, nvinfer1::DataType, nvinfer1::DataType>, - std::tuple<std::int32_t, std::int32_t, std::int32_t>, std::uint64_t>; +using MirrorInWorkspaceParamType + = std::tuple<std::tuple<tensorrt_llm::DataType, tensorrt_llm::DataType, tensorrt_llm::DataType>, + std::tuple<std::int32_t, std::int32_t, std::int32_t>, std::uint64_t>; class MirrorInWorkspaceTest : public testing::TestWithParam<MirrorInWorkspaceParamType> { diff --git a/cpp/tests/unit_tests/runtime/gptDecoderBatchedTest.cpp b/cpp/tests/unit_tests/runtime/gptDecoderBatchedTest.cpp index 15476899fce3..a979c9a4699f 100644 --- a/cpp/tests/unit_tests/runtime/gptDecoderBatchedTest.cpp +++ b/cpp/tests/unit_tests/runtime/gptDecoderBatchedTest.cpp @@ -17,10 +17,11 @@ #include "tensorrt_llm/runtime/gptDecoderBatched.h" #include "tensorrt_llm/batch_manager/createNewDecoderRequests.h" #include "tensorrt_llm/batch_manager/decoderBuffers.h" -#include "tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h" +#include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/common.h" @@ -47,6 +48,62 @@ using TensorPtr = ITensor::SharedPtr; namespace { +// Local copy of the former MakeDecodingBatchInputOutput::createDecoderBatchInputs +// helper, which was removed with the TensorRT-engine execution path. The decoder +// under test is backend-agnostic; this builds its step-batched inputs directly. +void createDecoderBatchInputs(tb::DecoderInputBuffers& inputBuffers, std::vector<SizeType32> const& activeSlots, + decoder::DecoderState const& decoderState) +{ + auto const& numDecodingEngineTokens = decoderState.getNumDecodingEngineTokens(); + auto const& maxDecodingEngineTokens = decoderState.getMaxDecodingEngineTokens(); + auto const& maxDecodingDecoderTokens = decoderState.getMaxDecodingDecoderTokens(); + auto const maxDecoderSteps = tc::ceilDiv(maxDecodingEngineTokens, maxDecodingDecoderTokens); + + auto& batchSlots = inputBuffers.forwardBatchSlots; + auto& decoderLogits = inputBuffers.decoderLogits; + + for (SizeType32 step = 0; step < maxDecoderSteps; ++step) + { + batchSlots.at(step)->resize(activeSlots.size()); + } + + auto constexpr singleRequest = 1; + + std::vector<SizeType32> batchSizes(maxDecoderSteps); + std::vector<std::vector<ITensor::SharedConstPtr>> batchLogits(maxDecoderSteps); + auto maxActiveDecoderSteps = 1; + for (size_t batchIdx = 0; batchIdx < activeSlots.size(); ++batchIdx) + { + auto const slot = activeSlots.at(batchIdx); + auto const& logits = decoderLogits.at(batchIdx); + + auto const numDecoderSteps = tc::ceilDiv(numDecodingEngineTokens.at(slot), maxDecodingDecoderTokens); + maxActiveDecoderSteps = std::max(maxActiveDecoderSteps, numDecoderSteps); + for (SizeType32 step = 0; step < numDecoderSteps; ++step) + { + auto batchSlotsRange = BufferRange<SizeType32>(*batchSlots.at(step)); + batchSlotsRange[batchSizes[step]] = slot; + batchSizes[step]++; + auto logitsSlice = ITensor::slice(logits, step, singleRequest); + batchLogits[step].emplace_back(std::move(logitsSlice)); + } + } + + for (SizeType32 step = 0; step < maxDecoderSteps; ++step) + { + batchSlots.at(step)->resize(batchSizes[step]); + } + batchLogits.resize(maxActiveDecoderSteps); + + inputBuffers.maxDecoderSteps = maxActiveDecoderSteps; + inputBuffers.batchLogits = batchLogits; +} + +} // namespace + +namespace +{ + std::shared_ptr<tb::LlmRequest> createLlmRequest(SizeType32 batchSlot, SizeType32 inputLengths, SizeType32 generatedTokensPerSteps, SizeType32 acceptedTokensPerStep, TokenIdType inputTokenId, TokenIdType expectedTokenId, SizeType32 maxNewTokens, SamplingConfig const& samplingConfig, TokenIdType endId) @@ -93,7 +150,7 @@ std::vector<std::shared_ptr<tb::LlmRequest>> createLlmRequests(std::vector<SizeT } void newRequests(std::vector<std::shared_ptr<tb::LlmRequest>> const& requests, TensorPtr const& batchSlots, - nvinfer1::DataType logitsType, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + tensorrt_llm::DataType logitsType, ModelConfig const& modelConfig, WorldConfig const& worldConfig, tle::DecodingConfig const& decodingConfig, GptDecoderBatched& decoder, CudaStream const& runtimeStream, SizeType32 maxSequenceLength, tb::DecoderInputBuffers& inputBuffers, decoder::DecoderState& decoderState) { @@ -125,7 +182,7 @@ void newRequests(std::vector<std::shared_ptr<tb::LlmRequest>> const& requests, T } void createDecoderInputs(tb::DecoderInputBuffers& inputBuffers, SizeType32 batchSize, SizeType32 vocabSizePadded, - nvinfer1::DataType dataType, std::vector<SamplingConfig>& samplingConfigs, + tensorrt_llm::DataType dataType, std::vector<SamplingConfig>& samplingConfigs, std::vector<SizeType32> const& generatedTokensPerSteps, bool computeLogProbs, BufferManager& manager) { auto& logits = inputBuffers.decoderLogits; @@ -242,8 +299,8 @@ void verifyResults(BufferManager& manager, decoder::DecoderState const& decoderS } } -void testDecoder(nvinfer1::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, SizeType32 maxBeamWidth, - bool computeLogProbs) +void testDecoder(tensorrt_llm::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, + SizeType32 maxBeamWidth, bool computeLogProbs) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); SizeType32 constexpr tensorParallelism{1}; @@ -345,7 +402,7 @@ void testDecoder(nvinfer1::DataType const dtype, std::vector<SamplingConfig>& sa auto activeSlots = std::vector<SizeType32>(batchSize); std::iota(activeSlots.begin(), activeSlots.end(), 0); - tb::MakeDecodingBatchInputOutput::createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); + createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); decoder.forward(decoderState, inputBuffers); checkSequenceLengths(*decoderState.getSequenceLengths(), expectedLengths, manager); @@ -375,7 +432,7 @@ void testDecoder(nvinfer1::DataType const dtype, std::vector<SamplingConfig>& sa EXPECT_FALSE(getFinished(*decoderState.getFinishedSum(), samplingConfigs, manager)[0]); } -void testDecoderWavefront(nvinfer1::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, +void testDecoderWavefront(tensorrt_llm::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, SizeType32 maxBeamWidth, bool computeLogProbs) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); @@ -474,7 +531,7 @@ void testDecoderWavefront(nvinfer1::DataType const dtype, std::vector<SamplingCo auto activeSlots = std::vector<SizeType32>(batchIdx + 1); std::iota(activeSlots.begin(), activeSlots.end(), 0); - tb::MakeDecodingBatchInputOutput::createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); + createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); decoder.forward(decoderState, inputBuffers); advanceSequenceLengths( @@ -496,7 +553,7 @@ void testDecoderWavefront(nvinfer1::DataType const dtype, std::vector<SamplingCo auto finishedVec = getFinished(*decoderState.getFinishedSum(), samplingConfigs, manager); while (!std::all_of(expectedFinished.begin(), expectedFinished.end(), [](bool finish) { return finish; })) { - tb::MakeDecodingBatchInputOutput::createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); + createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); decoder.forward(decoderState, inputBuffers); finishedVec = getFinished(*decoderState.getFinishedSum(), samplingConfigs, manager); @@ -526,7 +583,7 @@ void testDecoderWavefront(nvinfer1::DataType const dtype, std::vector<SamplingCo maxSeqLength, inputTokenId, expectedTokenId, endId); } -void testDecoderDraft(nvinfer1::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, +void testDecoderDraft(tensorrt_llm::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, SizeType32 maxBeamWidth, std::vector<SizeType32> const& generatedTokensPerSteps, std::vector<SizeType32> const& acceptedTokensPerStep, SizeType32 maxGeneratedTokensPerStep) { @@ -631,7 +688,7 @@ void testDecoderDraft(nvinfer1::DataType const dtype, std::vector<SamplingConfig auto activeSlots = std::vector<SizeType32>(batchSize); std::iota(activeSlots.begin(), activeSlots.end(), 0); - tb::MakeDecodingBatchInputOutput::createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); + createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); decoder.forward(decoderState, inputBuffers); checkSequenceLengths(*decoderState.getSequenceLengths(), expectedLengths, manager); EXPECT_THAT(getFinished(*decoderState.getFinishedSum(), samplingConfigs, manager), ::testing::Each(false)); @@ -648,11 +705,11 @@ struct BeamConfig std::vector<SizeType32> beamWidths; }; -using ParamType = std::tuple<nvinfer1::DataType, BeamConfig, bool>; +using ParamType = std::tuple<tensorrt_llm::DataType, BeamConfig, bool>; std::string generateTestName(testing::TestParamInfo<ParamType> const& info) { - std::string name{std::get<0>(info.param) == nvinfer1::DataType::kFLOAT ? "Float" : "Half"}; + std::string name{std::get<0>(info.param) == tensorrt_llm::DataType::kFLOAT ? "Float" : "Half"}; BeamConfig const beamConfig = std::get<1>(info.param); name.append("MaxBeamWidth" + std::to_string(beamConfig.maxBeamWidth)); for (auto const beamWdith : beamConfig.beamWidths) @@ -673,7 +730,7 @@ class ParamTest : public ::testing::TestWithParam<ParamType> TEST_P(ParamTest, Test) { - nvinfer1::DataType const dtype{std::get<0>(GetParam())}; + tensorrt_llm::DataType const dtype{std::get<0>(GetParam())}; BeamConfig const beamConfig{std::get<1>(GetParam())}; bool const computeLogProbs{std::get<2>(GetParam())}; std::vector<SamplingConfig> samplingConfigs; @@ -686,7 +743,7 @@ TEST_P(ParamTest, Test) } INSTANTIATE_TEST_SUITE_P(DecoderBwTest, ParamTest, - testing::Combine(testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF), + testing::Combine(testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF), testing::Values(BeamConfig{1, {1, 1, 1}}, BeamConfig{3, {3, 3, 3, 3}}, BeamConfig{4, {4, 4, 4}}, BeamConfig{10, {10, 10, 10}}), testing::Values(false, true)), @@ -698,7 +755,7 @@ class ParamWavefrontTest : public ::testing::TestWithParam<ParamType> TEST_P(ParamWavefrontTest, Test) { - nvinfer1::DataType const dtype{std::get<0>(GetParam())}; + tensorrt_llm::DataType const dtype{std::get<0>(GetParam())}; BeamConfig const beamConfig{std::get<1>(GetParam())}; bool const computeLogProbs{std::get<2>(GetParam())}; bool const normalizeLogProbs{true}; @@ -712,7 +769,7 @@ TEST_P(ParamWavefrontTest, Test) } INSTANTIATE_TEST_SUITE_P(DecoderBwTest, ParamWavefrontTest, - testing::Combine(testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF), + testing::Combine(testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF), testing::Values(BeamConfig{1, {1, 1, 1}}, BeamConfig{3, {3, 3, 3, 3}}, BeamConfig{4, {4, 4, 4}}, BeamConfig{10, {10, 10, 10}}), testing::Values(false, true)), @@ -725,7 +782,7 @@ struct DraftConfig std::vector<SizeType32> acceptedTokensPerStep; }; -using DraftTestParamType = std::tuple<nvinfer1::DataType, BeamConfig, DraftConfig>; +using DraftTestParamType = std::tuple<tensorrt_llm::DataType, BeamConfig, DraftConfig>; class ParamDraftTest : public ::testing::TestWithParam<DraftTestParamType> { @@ -733,7 +790,7 @@ class ParamDraftTest : public ::testing::TestWithParam<DraftTestParamType> TEST_P(ParamDraftTest, Test) { - nvinfer1::DataType const dtype{std::get<0>(GetParam())}; + tensorrt_llm::DataType const dtype{std::get<0>(GetParam())}; BeamConfig const beamConfig{std::get<1>(GetParam())}; DraftConfig const draftConfig{std::get<2>(GetParam())}; @@ -751,7 +808,7 @@ TEST_P(ParamDraftTest, Test) } INSTANTIATE_TEST_SUITE_P(DecoderTest, ParamDraftTest, - testing::Combine(testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF), + testing::Combine(testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF), testing::Values(BeamConfig{1, {1, 1, 1}}), testing::Values( // DraftConfig{2, {1, 1, 1}, {0, 0, 0}}, DraftConfig{2, {2, 2, 2}, {1, 1, 1}}, @@ -760,7 +817,7 @@ INSTANTIATE_TEST_SUITE_P(DecoderTest, ParamDraftTest, )), [](testing::TestParamInfo<DraftTestParamType> const& info) { - std::string name{std::get<0>(info.param) == nvinfer1::DataType::kFLOAT ? "Float" : "Half"}; + std::string name{std::get<0>(info.param) == tensorrt_llm::DataType::kFLOAT ? "Float" : "Half"}; BeamConfig const beamConfig = std::get<1>(info.param); DraftConfig const draftConfig = std::get<2>(info.param); name.append("MaxBeamWidth" + std::to_string(beamConfig.maxBeamWidth)); diff --git a/cpp/tests/unit_tests/runtime/gptDecoderTest.cpp b/cpp/tests/unit_tests/runtime/gptDecoderTest.cpp index e1fed49293bd..5f620aa4d9c4 100644 --- a/cpp/tests/unit_tests/runtime/gptDecoderTest.cpp +++ b/cpp/tests/unit_tests/runtime/gptDecoderTest.cpp @@ -17,6 +17,7 @@ #include <gtest/gtest.h> #include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/gptDecoder.h" @@ -68,7 +69,7 @@ bool forwardAndSync(std::unique_ptr<IGptDecoder> const& decoder, DecodingOutput& } } -void testDecoder(nvinfer1::DataType const dtype, SamplingConfig const& samplingConfig) +void testDecoder(tensorrt_llm::DataType const dtype, SamplingConfig const& samplingConfig) { SizeType32 constexpr tensorParallelism{1}; SizeType32 constexpr pipelineParallelism{1}; @@ -140,21 +141,21 @@ void testDecoder(nvinfer1::DataType const dtype, SamplingConfig const& samplingC if (beamWidth > 1) { auto srcCacheIndirection = std::shared_ptr( - manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), nvinfer1::DataType::kINT32)); + manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), tensorrt_llm::DataType::kINT32)); manager.setZero(*srcCacheIndirection); inputs.cacheIndirection = srcCacheIndirection; } // set up outputs auto outputIds = std::shared_ptr( - manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), nvinfer1::DataType::kINT32)); + manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), tensorrt_llm::DataType::kINT32)); manager.setZero(*outputIds); auto gatheredOutputIds = std::shared_ptr( - manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), nvinfer1::DataType::kINT32)); + manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), tensorrt_llm::DataType::kINT32)); manager.setZero(*gatheredOutputIds); DecodingOutput outputs{outputIds, gatheredOutputIds}; auto newTokens - = std::shared_ptr(manager.gpu(ITensor::makeShape({batchSize, beamWidth}), nvinfer1::DataType::kINT32)); + = std::shared_ptr(manager.gpu(ITensor::makeShape({batchSize, beamWidth}), tensorrt_llm::DataType::kINT32)); manager.setZero(*newTokens); outputs.newTokens = newTokens; @@ -165,7 +166,7 @@ void testDecoder(nvinfer1::DataType const dtype, SamplingConfig const& samplingC TRTDataType<tensorrt_llm::kernels::FinishedState::UnderlyingType>::value); inputs.finishReasons = ITensor::view(outputs.finishReasons); manager.setZero(*outputs.finishReasons); - outputs.finishedSum = BufferManager::pinnedPool(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + outputs.finishedSum = BufferManager::pinnedPool(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); auto finishedSumHost = bufferCast<std::int32_t>(*outputs.finishedSum); for (SizeType32 bi = 0; bi < batchSize; ++bi) { @@ -175,17 +176,17 @@ void testDecoder(nvinfer1::DataType const dtype, SamplingConfig const& samplingC if (beamWidth > 1) { auto tgtCacheIndirection = std::shared_ptr( - manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), nvinfer1::DataType::kINT32)); + manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), tensorrt_llm::DataType::kINT32)); manager.setZero(*tgtCacheIndirection); outputs.cacheIndirection = tgtCacheIndirection; auto cumLogProbs - = std::shared_ptr(manager.gpu(ITensor::makeShape({batchSize, beamWidth}), nvinfer1::DataType::kFLOAT)); + = std::shared_ptr(manager.gpu(ITensor::makeShape({batchSize, beamWidth}), tensorrt_llm::DataType::kFLOAT)); manager.setZero(*cumLogProbs); outputs.cumLogProbs = cumLogProbs; auto parentIds = std::shared_ptr( - manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), nvinfer1::DataType::kINT32)); + manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), tensorrt_llm::DataType::kINT32)); manager.setZero(*parentIds); outputs.parentIds = parentIds; } @@ -245,13 +246,13 @@ void testDecoder(nvinfer1::DataType const dtype, SamplingConfig const& samplingC } // namespace -class ParamTest : public ::testing::TestWithParam<std::tuple<nvinfer1::DataType, SizeType32>> +class ParamTest : public ::testing::TestWithParam<std::tuple<tensorrt_llm::DataType, SizeType32>> { }; TEST_P(ParamTest, Test) { - nvinfer1::DataType const dtype{std::get<0>(GetParam())}; + tensorrt_llm::DataType const dtype{std::get<0>(GetParam())}; SizeType32 const beamWidth{std::get<1>(GetParam())}; SamplingConfig const samplingConfig{beamWidth}; @@ -259,10 +260,11 @@ TEST_P(ParamTest, Test) } INSTANTIATE_TEST_SUITE_P(DecoderTest, ParamTest, - testing::Combine(testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF), testing::Values(1, 3)), + testing::Combine( + testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF), testing::Values(1, 3)), [](testing::TestParamInfo<ParamTest::ParamType> const& info) { - std::string name{std::get<0>(info.param) == nvinfer1::DataType::kFLOAT ? "Float" : "Half"}; + std::string name{std::get<0>(info.param) == tensorrt_llm::DataType::kFLOAT ? "Float" : "Half"}; auto const beamWidth = std::get<1>(info.param); name.append(beamWidth == 1 ? "Sampling" : "BeamWidth" + std::to_string(beamWidth)); return name; diff --git a/cpp/tests/unit_tests/runtime/iTensorTest.cpp b/cpp/tests/unit_tests/runtime/iTensorTest.cpp index 54ba8aa3beec..4637474f72c7 100644 --- a/cpp/tests/unit_tests/runtime/iTensorTest.cpp +++ b/cpp/tests/unit_tests/runtime/iTensorTest.cpp @@ -17,6 +17,7 @@ #include <gmock/gmock.h> #include <gtest/gtest.h> +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -25,7 +26,7 @@ using namespace tensorrt_llm::runtime; TEST(ITensorTest, SqueezeTensor) { auto dims = ITensor::makeShape({16, 1, 4}); - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; ITensor::SharedPtr tensor{BufferManager::cpu(dims, dataType)}; auto squeezeDim = 0; @@ -102,7 +103,7 @@ TEST(ITensorTest, UnsqueezeTensor) auto oldShape = ITensor::makeShape({2, 3, 4, 5}); { - auto tensor = BufferManager::cpu(oldShape, nvinfer1::DataType::kINT32); + auto tensor = BufferManager::cpu(oldShape, tensorrt_llm::DataType::kINT32); tensor->unsqueeze(0); auto shape = tensor->getShape(); @@ -114,7 +115,7 @@ TEST(ITensorTest, UnsqueezeTensor) EXPECT_EQ(shape.d[4], 5); } { - auto tensor = BufferManager::cpu(oldShape, nvinfer1::DataType::kINT32); + auto tensor = BufferManager::cpu(oldShape, tensorrt_llm::DataType::kINT32); tensor->unsqueeze(1); auto shape = tensor->getShape(); @@ -127,7 +128,7 @@ TEST(ITensorTest, UnsqueezeTensor) } { - auto tensor = BufferManager::cpu(oldShape, nvinfer1::DataType::kINT32); + auto tensor = BufferManager::cpu(oldShape, tensorrt_llm::DataType::kINT32); tensor->unsqueeze(4); auto shape = tensor->getShape(); @@ -144,7 +145,7 @@ TEST(ITensorTest, UnsqueezeTensor) { try { - auto tensor = BufferManager::cpu(oldShape, nvinfer1::DataType::kINT32); + auto tensor = BufferManager::cpu(oldShape, tensorrt_llm::DataType::kINT32); tensor->unsqueeze(invalidDim); FAIL() << "Expected failure"; } @@ -162,7 +163,7 @@ TEST(ITensorTest, UnsqueezeTensor) TEST(ITensorTest, TensorView) { auto const dims = ITensor::makeShape({16, 1, 4}); - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; ITensor::SharedPtr tensor = BufferManager::cpu(dims, dataType); auto const viewDims = ITensor::makeShape({16, 1, 2}); @@ -180,7 +181,7 @@ TEST(ITensorTest, TensorView) TEST(ITensorTest, TensorSlice) { auto dims = ITensor::makeShape({16, 8, 4}); - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; ITensor::SharedPtr tensor{BufferManager::cpu(dims, dataType)}; auto offset = dims.d[0] / 4; auto slice = ITensor::slice(tensor, offset); @@ -221,7 +222,7 @@ TEST(ITensorTest, TensorSlice) TEST(ITensorTest, TensorDimsSliceAtManual) { auto shape = ITensor::makeShape({5, 5, 5, 5, 5}); - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; ITensor::SharedPtr tensor(BufferManager::cpu(shape, dataType)); auto offsetDims = ITensor::makeShape({4, 3, 3}); auto sizeDim = 2; @@ -282,7 +283,7 @@ TEST(ITensorTest, TensorDimsSliceAtManual) TEST(ITensorTest, TensorDimsSliceAtExtrame) { - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; { auto shape = ITensor::makeShape({5, 5, 5, 5, 5}); ITensor::SharedPtr tensor(BufferManager::cpu(shape, dataType)); @@ -540,7 +541,7 @@ TEST(ShapeRange, test) TEST(ITensorTest, TensorDimsSliceAt) { auto shape = ITensor::makeShape({5, 5, 5, 5}); - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; ITensor::SharedPtr tensor(BufferManager::cpu(shape, dataType)); auto verify = [&shape, &tensor, &dataType](ITensor::Shape const& index) @@ -657,7 +658,7 @@ TEST(ITensorTest, TensorDimsSliceAt) TEST(BufferRangeTest, ConstType) { auto shape = ITensor::makeShape({5, 5, 5, 5, 5}); - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; ITensor::SharedPtr tensor(BufferManager::cpu(shape, dataType)); ITensor::SharedConstPtr tensorConst = tensor; @@ -694,7 +695,7 @@ TEST(BufferRangeTest, ConstType) TEST(ITensorTest, GetDimension) { auto shape = ITensor::makeShape({10, 11, 12}); - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; ITensor::SharedPtr tensor(BufferManager::cpu(shape, dataType)); auto firstDimensionFromStart = tensor->getDimension<0>(); diff --git a/cpp/tests/unit_tests/runtime/loraCacheTest.cpp b/cpp/tests/unit_tests/runtime/loraCacheTest.cpp index 4d4dc86dc824..6a91d11df3db 100644 --- a/cpp/tests/unit_tests/runtime/loraCacheTest.cpp +++ b/cpp/tests/unit_tests/runtime/loraCacheTest.cpp @@ -28,7 +28,7 @@ #include "tensorrt_llm/runtime/utils/numpyUtils.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <gmock/gmock-matchers.h> #include <gmock/gmock.h> @@ -78,7 +78,7 @@ class LoraCacheTest : public ::testing::Test, void SetUp() override { - mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); + mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); mModelConfig->setMlpHiddenSize(32); mWorldConfig = std::make_unique<WorldConfig>(2, 1, 1, 0); std::vector<LoraModule> modules{ @@ -101,7 +101,7 @@ class LoraCacheTest : public ::testing::Test, mManager = std::make_unique<BufferManager>(mStream); auto pageConfig = LoraCachePageManagerConfig( - runtime::MemoryType::kCPU, nvinfer1::DataType::kFLOAT, 2 * 8, 6, 64, 4 * 16, 1); + runtime::MemoryType::kCPU, tensorrt_llm::DataType::kFLOAT, 2 * 8, 6, 64, 4 * 16, 1); pageConfig.setInitToZero(true); auto pageConfig2 = pageConfig; pageConfig2.setInitToZero(true); @@ -125,7 +125,7 @@ TEST_F(LoraCacheTest, LoraCachePageManagerTest) auto pageShape = ITensor::makeShape({maxAdapterSize, maxAdapterWeights}); LoraCachePageManagerConfig config( - runtime::MemoryType::kCPU, nvinfer1::DataType::kFLOAT, 8, 6, maxAdapterSize, maxAdapterWeights, 1); + runtime::MemoryType::kCPU, tensorrt_llm::DataType::kFLOAT, 8, 6, maxAdapterSize, maxAdapterWeights, 1); LoraCachePageManager manager(config, *mManager); auto block0 = manager.blockPtr(0); @@ -182,11 +182,11 @@ TEST_F(LoraCacheTest, LoraCachePageManagerTest) TEST_F(LoraCacheTest, determineNumPages) { - ModelConfig modelConfig(0, 2, 2, 0, 1, 4, nvinfer1::DataType::kFLOAT); + ModelConfig modelConfig(0, 2, 2, 0, 1, 4, tensorrt_llm::DataType::kFLOAT); modelConfig.setLoraModules(LoraModule::createLoraModules({"attn_dense", "attn_qkv"}, 4, 4, 1, 1, 2, 2, 0)); WorldConfig worldConfig(1, 1, 1, 0); - LoraCachePageManagerConfig pageConfig(MemoryType::kCPU, nvinfer1::DataType::kFLOAT, 12393, 40, 80, 16, 1); + LoraCachePageManagerConfig pageConfig(MemoryType::kCPU, tensorrt_llm::DataType::kFLOAT, 12393, 40, 80, 16, 1); LoraCache cache(pageConfig, modelConfig, worldConfig, *mManager); @@ -374,7 +374,7 @@ TEST_F(LoraCacheTest, basicPutGet) TEST_F(LoraCacheTest, splitTransposeCpu) { - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); auto worldConfig = WorldConfig(2, 1, 1, 0); SizeType32 const split{2}; @@ -391,8 +391,8 @@ TEST_F(LoraCacheTest, splitTransposeCpu) auto const outputShape = ITensor::makeShape({batchSize, inputLength / split}); auto inputTensor = mManager->copyFrom(input, inputShape, MemoryType::kCPU); - auto outputTensorRank0 = mManager->cpu(outputShape, nvinfer1::DataType::kINT32); - auto outputTensorRank1 = mManager->cpu(outputShape, nvinfer1::DataType::kINT32); + auto outputTensorRank0 = mManager->cpu(outputShape, tensorrt_llm::DataType::kINT32); + auto outputTensorRank1 = mManager->cpu(outputShape, tensorrt_llm::DataType::kINT32); mManager->setZero(*outputTensorRank0); mManager->setZero(*outputTensorRank1); @@ -416,8 +416,8 @@ TEST_F(LoraCacheTest, splitTransposeCpu) auto const outputShape = ITensor::makeShape({batchSize, inputLength / split}); auto inputTensor = mManager->copyFrom(input, inputShape, MemoryType::kCPU); - auto outputTensorRank0 = mManager->cpu(outputShape, nvinfer1::DataType::kINT32); - auto outputTensorRank1 = mManager->cpu(outputShape, nvinfer1::DataType::kINT32); + auto outputTensorRank0 = mManager->cpu(outputShape, tensorrt_llm::DataType::kINT32); + auto outputTensorRank1 = mManager->cpu(outputShape, tensorrt_llm::DataType::kINT32); mManager->setZero(*outputTensorRank0); mManager->setZero(*outputTensorRank1); @@ -438,7 +438,7 @@ TEST_F(LoraCacheTest, splitTransposeCpu) TEST_P(LoraCacheTest, copyToPages_tp1) { bool const isDora = GetParam(); - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); modelConfig.setMlpHiddenSize(32); auto worldConfig = WorldConfig(1, 1, 1, 0); std::vector<LoraModule> modules{ @@ -501,7 +501,7 @@ TEST_P(LoraCacheTest, copyToPages_tp1) TEST_P(LoraCacheTest, copyToPages_tp2_rank0) { bool const isDora = GetParam(); - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); modelConfig.setMlpHiddenSize(32); auto worldConfig = WorldConfig(2, 1, 1, 0); std::vector<LoraModule> modules{ @@ -562,7 +562,7 @@ TEST_P(LoraCacheTest, copyToPages_tp2_rank0) TEST_P(LoraCacheTest, copyToPages_tp2_rank1) { bool const isDora = GetParam(); - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); modelConfig.setMlpHiddenSize(32); auto worldConfig = WorldConfig(2, 1, 1, 1); std::vector<LoraModule> modules{ diff --git a/cpp/tests/unit_tests/runtime/loraManagerTest.cpp b/cpp/tests/unit_tests/runtime/loraManagerTest.cpp index 6910719da76f..11c19d22efb0 100644 --- a/cpp/tests/unit_tests/runtime/loraManagerTest.cpp +++ b/cpp/tests/unit_tests/runtime/loraManagerTest.cpp @@ -32,6 +32,7 @@ #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/worldConfig.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/utils/numpyUtils.h" #include <gtest/gtest.h> @@ -66,7 +67,7 @@ class LoraManagerTest { protected: LoraManagerTest() - : mModelConfig(1, 2, 2, 0, 1, 4, nvinfer1::DataType::kFLOAT) + : mModelConfig(1, 2, 2, 0, 1, 4, tensorrt_llm::DataType::kFLOAT) { } @@ -87,7 +88,7 @@ class LoraManagerTest PeftTable getPeftTable(SizeType32 tpRank = 0) { - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); modelConfig.setMlpHiddenSize(32); auto worldConfig = WorldConfig(2, 2, 1, 3); std::vector<LoraModule> modules{ @@ -102,7 +103,7 @@ class LoraManagerTest }; modelConfig.setLoraModules(modules); auto pageConfig = LoraCachePageManagerConfig( - runtime::MemoryType::kCPU, nvinfer1::DataType::kFLOAT, 2 * 8, 6, 64, 4 * 16, 1); + runtime::MemoryType::kCPU, tensorrt_llm::DataType::kFLOAT, 2 * 8, 6, 64, 4 * 16, 1); pageConfig.setInitToZero(true); LoraCache loraCache(pageConfig, modelConfig, worldConfig, *mManager); @@ -213,7 +214,7 @@ static void checkLoraTensors(LoraManager const& loraManager, std::vector<int64_t auto expectedTensor = expectedTensors.find(fieldName)->second; auto actualTensor = inputTensors.find(fieldName)->second; ITensor::shapeEquals(expectedTensor->getShape(), actualTensor->getShape()); - if (expectedTensor->getDataType() == nvinfer1::DataType::kINT64) + if (expectedTensor->getDataType() == tensorrt_llm::DataType::kINT64) { auto expT = bufferCast<int64_t>(*expectedTensor); auto actT = bufferCast<int64_t>(*actualTensor); @@ -308,7 +309,7 @@ TEST_P(LoraManagerTest, fillInputTensors) bool const isDora = GetParam(); LoraManager loraManager; - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); modelConfig.setMlpHiddenSize(32); auto worldConfig = WorldConfig(1, 1, 1, 0); std::vector<LoraModule> modules{ @@ -332,9 +333,9 @@ TEST_P(LoraManagerTest, fillInputTensors) auto numLayers = static_cast<SizeType32>(modelConfig.getNbAttentionLayers()); SizeType32 numSeqs = 4; TensorPtr weightsPtrs - = mManager->cpu(ITensor::makeShape({numModules, numLayers, numSeqs, 3}), nvinfer1::DataType::kINT64); + = mManager->cpu(ITensor::makeShape({numModules, numLayers, numSeqs, 3}), tensorrt_llm::DataType::kINT64); TensorPtr adapterSizes - = mManager->cpu(ITensor::makeShape({numModules, numLayers, numSeqs}), nvinfer1::DataType::kINT32); + = mManager->cpu(ITensor::makeShape({numModules, numLayers, numSeqs}), tensorrt_llm::DataType::kINT32); mManager->setZero(*weightsPtrs); mManager->setZero(*adapterSizes); diff --git a/cpp/tests/unit_tests/runtime/loraUtilsTest.cpp b/cpp/tests/unit_tests/runtime/loraUtilsTest.cpp index a14fa7bb8c47..994a77acf818 100644 --- a/cpp/tests/unit_tests/runtime/loraUtilsTest.cpp +++ b/cpp/tests/unit_tests/runtime/loraUtilsTest.cpp @@ -25,7 +25,7 @@ #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <algorithm> #include <optional> @@ -53,7 +53,7 @@ class LoraUtilsTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-typ TEST_F(LoraUtilsTest, null_values) { std::optional<TensorPtr> optReqLoraWeights = std::nullopt; - std::optional<TensorPtr> optReqLoraConfig = mManager->emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kHALF); + std::optional<TensorPtr> optReqLoraConfig = mManager->emptyTensor(MemoryType::kCPU, tensorrt_llm::DataType::kHALF); EXPECT_THAT([&]() { loraValidateRequestTensorDims(optReqLoraWeights, optReqLoraConfig); }, testing::Throws<std::runtime_error>()); @@ -66,33 +66,35 @@ TEST_F(LoraUtilsTest, null_values) TEST_F(LoraUtilsTest, dims_mem_type) { - std::optional<TensorPtr> optReqLoraWeights = mManager->cpu(ITensor::makeShape({1, 2}), nvinfer1::DataType::kHALF); + std::optional<TensorPtr> optReqLoraWeights + = mManager->cpu(ITensor::makeShape({1, 2}), tensorrt_llm::DataType::kHALF); std::optional<TensorPtr> optReqLoraConfig - = mManager->cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); + = mManager->cpu(ITensor::makeShape({1, 2, 3}), tensorrt_llm::DataType::kINT32); EXPECT_THAT([&]() { loraValidateRequestTensorDims(optReqLoraWeights, optReqLoraConfig); }, testing::Throws<std::runtime_error>()); - std::optional<TensorPtr> optGpuWeights = mManager->gpu(ITensor::makeShape({1, 2, 50}), nvinfer1::DataType::kHALF); + std::optional<TensorPtr> optGpuWeights + = mManager->gpu(ITensor::makeShape({1, 2, 50}), tensorrt_llm::DataType::kHALF); EXPECT_THAT([&]() { loraValidateRequestTensorDims(optGpuWeights, optReqLoraConfig); }, testing::Throws<std::runtime_error>()); - optReqLoraWeights = mManager->cpu(ITensor::makeShape({1, 2, 50}), nvinfer1::DataType::kHALF); - optReqLoraConfig = mManager->cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); + optReqLoraWeights = mManager->cpu(ITensor::makeShape({1, 2, 50}), tensorrt_llm::DataType::kHALF); + optReqLoraConfig = mManager->cpu(ITensor::makeShape({1, 2, 3}), tensorrt_llm::DataType::kINT32); loraValidateRequestTensorDims(optReqLoraWeights, optReqLoraConfig); } TEST_F(LoraUtilsTest, loraValidateRequestTensors) { - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 4, nvinfer1::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 4, tensorrt_llm::DataType::kFLOAT); auto worldConfig = WorldConfig(); std::optional<TensorPtr> optReqLoraWeights - = mManager->cpu(ITensor::makeShape({1, 2, 32}), nvinfer1::DataType::kFLOAT); + = mManager->cpu(ITensor::makeShape({1, 2, 32}), tensorrt_llm::DataType::kFLOAT); std::optional<TensorPtr> optReqLoraConfig - = mManager->cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); + = mManager->cpu(ITensor::makeShape({1, 2, 3}), tensorrt_llm::DataType::kINT32); std::vector<int32_t> config{1, 0, 4, 1, 1, 4}; diff --git a/cpp/tests/unit_tests/runtime/medusaModuleTest.cpp b/cpp/tests/unit_tests/runtime/medusaModuleTest.cpp index 3cba1bf2994d..8e7ff6c75459 100644 --- a/cpp/tests/unit_tests/runtime/medusaModuleTest.cpp +++ b/cpp/tests/unit_tests/runtime/medusaModuleTest.cpp @@ -20,7 +20,7 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/utils/speculativeChoicesUtils.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <gmock/gmock-matchers.h> #include <gmock/gmock.h> @@ -58,14 +58,16 @@ class MedusaModuleTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro- auto const tokensPerStep = medusaModule.getMaxDecodingTokens(); // batch size = 1 here. - TensorPtr medusaGenerationLengthsHost = mManager->pinned(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + TensorPtr medusaGenerationLengthsHost + = mManager->pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); TensorPtr medusaPositionOffsetsHost - = mManager->pinned(ITensor::makeShape({tokensPerStep}), nvinfer1::DataType::kINT32); - TensorPtr medusaTreeIdsHost = mManager->pinned(ITensor::makeShape({tokensPerStep}), nvinfer1::DataType::kINT32); + = mManager->pinned(ITensor::makeShape({tokensPerStep}), tensorrt_llm::DataType::kINT32); + TensorPtr medusaTreeIdsHost + = mManager->pinned(ITensor::makeShape({tokensPerStep}), tensorrt_llm::DataType::kINT32); TensorPtr medusaPathsHost - = mManager->pinned(ITensor::makeShape({tokensPerStep, medusaHeads + 1}), nvinfer1::DataType::kINT32); + = mManager->pinned(ITensor::makeShape({tokensPerStep, medusaHeads + 1}), tensorrt_llm::DataType::kINT32); TensorPtr attentionPackedMaskHost - = mManager->pinned(ITensor::makeShape({tokensPerStep, numPackedMasks}), nvinfer1::DataType::kINT32); + = mManager->pinned(ITensor::makeShape({tokensPerStep, numPackedMasks}), tensorrt_llm::DataType::kINT32); std::vector<SizeType32> topKs; utils::initTensorsFromChoices(medusaModule, choices, topKs, medusaGenerationLengthsHost, diff --git a/cpp/tests/unit_tests/runtime/runtimeKernelTest.cpp b/cpp/tests/unit_tests/runtime/runtimeKernelTest.cpp index dd517c5de5a4..58372a6bd479 100644 --- a/cpp/tests/unit_tests/runtime/runtimeKernelTest.cpp +++ b/cpp/tests/unit_tests/runtime/runtimeKernelTest.cpp @@ -23,7 +23,7 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/runtimeKernels.h" -#include <NvInferRuntime.h> +#include "tensorrt_llm/common/tllmDataType.h" #include <gtest/gtest.h> #include <algorithm> @@ -85,7 +85,7 @@ TEST_F(RuntimeKernelTest, FillBufferInt8) { for (auto size : {123LLU, 1025LLU, 1LLU << 32}) { - auto buffer = mManager->gpu(size, nvinfer1::DataType::kINT8); + auto buffer = mManager->gpu(size, tensorrt_llm::DataType::kINT8); testFill<std::int8_t>(*buffer, *mManager, *mStream); } } @@ -94,7 +94,7 @@ TEST_F(RuntimeKernelTest, FillTensorInt8) { for (auto size : {123, 1025, std::numeric_limits<int32_t>::max()}) { - auto tensor = mManager->gpu(tr::ITensor::makeShape({size, 2}), nvinfer1::DataType::kINT8); + auto tensor = mManager->gpu(tr::ITensor::makeShape({size, 2}), tensorrt_llm::DataType::kINT8); testFill<std::int8_t>(*tensor, *mManager, *mStream); } } @@ -111,7 +111,7 @@ TEST_F(RuntimeKernelTest, ScatterHalf) auto const outputShape = tr::ITensor::makeShape({batchSize * beamWidth, inputLength}); auto inputTensor = mManager->copyFrom(input, inputShape, tr::MemoryType::kGPU); - auto outputTensor = mManager->gpu(outputShape, nvinfer1::DataType::kHALF); + auto outputTensor = mManager->gpu(outputShape, tensorrt_llm::DataType::kHALF); mManager->setZero(*outputTensor); tr::kernels::scatterTensor(*outputTensor, *inputTensor, beamWidth, *mStream); @@ -174,7 +174,7 @@ TEST_F(RuntimeKernelTest, TileInt32) auto const outputShape = tr::ITensor::makeShape({batchSize * beamWidth, inputLength}); auto inputTensor = mManager->copyFrom(input, inputShape, tr::MemoryType::kGPU); - auto outputTensor = mManager->gpu(outputShape, nvinfer1::DataType::kINT32); + auto outputTensor = mManager->gpu(outputShape, tensorrt_llm::DataType::kINT32); tr::kernels::tileTensor(*outputTensor, *inputTensor, beamWidth, *mStream); @@ -194,7 +194,7 @@ TEST_F(RuntimeKernelTest, TileHalf) auto const outputShape = tr::ITensor::makeShape({batchSize * beamWidth, inputLength}); auto inputTensor = mManager->copyFrom(input, inputShape, tr::MemoryType::kGPU); - auto outputTensor = mManager->gpu(outputShape, nvinfer1::DataType::kHALF); + auto outputTensor = mManager->gpu(outputShape, tensorrt_llm::DataType::kHALF); tr::kernels::tileTensor(*outputTensor, *inputTensor, beamWidth, *mStream); @@ -228,11 +228,11 @@ TEST_F(RuntimeKernelTest, TileInt8Large) // Scope the allocated tensors to ensure they are de-allocated before the test ends. { - auto inputTensor = mManager->gpu(inputShape, nvinfer1::DataType::kINT8); + auto inputTensor = mManager->gpu(inputShape, tensorrt_llm::DataType::kINT8); tr::kernels::invokeFill(*inputTensor, value, *mStream); mStream->synchronize(); - auto outputTensor = mManager->gpu(outputShape, nvinfer1::DataType::kINT8); + auto outputTensor = mManager->gpu(outputShape, tensorrt_llm::DataType::kINT8); tr::kernels::tileTensor(*outputTensor, *inputTensor, beamWidth, *mStream); mStream->synchronize(); @@ -257,11 +257,11 @@ void testCopyBatch(tr::SizeType64 stride, tr::BufferManager& manager, tr::CudaSt auto const bufferShape = tr::ITensor::makeShape({rows, stride}); auto const indicesShape = tr::ITensor::makeShape({numIndices}); - auto srcBufferHost = tr::BufferManager::cpu(bufferShape, nvinfer1::DataType::kINT32); - auto dstBufferDevice = manager.gpu(bufferShape, nvinfer1::DataType::kINT32); - auto srcOffsets = tr::BufferManager::pinned(indicesShape, nvinfer1::DataType::kINT64); - auto dstOffsets = tr::BufferManager::pinned(indicesShape, nvinfer1::DataType::kINT64); - auto sizes = tr::BufferManager::pinned(indicesShape, nvinfer1::DataType::kINT64); + auto srcBufferHost = tr::BufferManager::cpu(bufferShape, tensorrt_llm::DataType::kINT32); + auto dstBufferDevice = manager.gpu(bufferShape, tensorrt_llm::DataType::kINT32); + auto srcOffsets = tr::BufferManager::pinned(indicesShape, tensorrt_llm::DataType::kINT64); + auto dstOffsets = tr::BufferManager::pinned(indicesShape, tensorrt_llm::DataType::kINT64); + auto sizes = tr::BufferManager::pinned(indicesShape, tensorrt_llm::DataType::kINT64); tr::kernels::invokeFill(*dstBufferDevice, 0, stream); auto* srcBufferHostPtr = tr::bufferCast<std::int32_t>(*srcBufferHost); diff --git a/cpp/tests/unit_tests/runtime/samplingTest.cpp b/cpp/tests/unit_tests/runtime/samplingTest.cpp index dad99323164e..bb93478cf8ff 100644 --- a/cpp/tests/unit_tests/runtime/samplingTest.cpp +++ b/cpp/tests/unit_tests/runtime/samplingTest.cpp @@ -14,12 +14,12 @@ * limitations under the License. */ +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/layers/dynamicDecodeLayer.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/gptDecoder.h" -#include "tensorrt_llm/runtime/tllmLogger.h" #include <gtest/gtest.h> @@ -39,14 +39,11 @@ class SamplingTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type if (mDeviceCount == 0) GTEST_SKIP() << "No GPUs found"; - - mLogger = std::make_shared<TllmLogger>(); } void TearDown() override {} int mDeviceCount; - std::shared_ptr<nvinfer1::ILogger> mLogger; }; std::shared_ptr<tl::BaseDecodingOutputs> dynamicDecodeTest(std::shared_ptr<BufferManager> manager, size_t vocabSize, @@ -67,10 +64,10 @@ std::shared_ptr<tl::BaseDecodingOutputs> dynamicDecodeTest(std::shared_ptr<Buffe tk::FinishedState::UnderlyingType* gpuFinished = nullptr; - ITensor::SharedPtr gpuEndIds = manager->gpu(ITensor::makeShape({signedBatchSize}), nvinfer1::DataType::kINT32); + ITensor::SharedPtr gpuEndIds = manager->gpu(ITensor::makeShape({signedBatchSize}), tensorrt_llm::DataType::kINT32); manager->copy(cpuEndIds.data(), *gpuEndIds, MemoryType::kCPU); ITensor::SharedPtr gpuOutputIds = manager->gpu( - ITensor::makeShape({signedBatchSize, signedBeamWidth, signedMaxSeqLength}), nvinfer1::DataType::kINT32); + ITensor::makeShape({signedBatchSize, signedBeamWidth, signedMaxSeqLength}), tensorrt_llm::DataType::kINT32); manager->copy(cpuOutputIds.data(), *gpuOutputIds, MemoryType::kCPU); auto const decodingMode = beamWidth == 1 ? tle::DecodingMode::TopKTopP() : tle::DecodingMode::BeamSearch(); @@ -92,7 +89,7 @@ std::shared_ptr<tl::BaseDecodingOutputs> dynamicDecodeTest(std::shared_ptr<Buffe auto forwardParams = std::make_shared<tl::SamplingInputs>(gpuEndIds, batchSlots, step, ite, localBatchSize); auto logitsShape = ITensor::makeShape({signedBatchSize, static_cast<int64_t>(beamWidth), static_cast<int64_t>(vocabSizePadded)}); - ITensor::SharedPtr inputLogits = manager->gpu(logitsShape, nvinfer1::DataType::kFLOAT); + ITensor::SharedPtr inputLogits = manager->gpu(logitsShape, tensorrt_llm::DataType::kFLOAT); forwardParams->logits = inputLogits; manager->copy(cpuLogits.data(), *inputLogits, MemoryType::kCPU); @@ -101,10 +98,10 @@ std::shared_ptr<tl::BaseDecodingOutputs> dynamicDecodeTest(std::shared_ptr<Buffe forwardParams->stopCriteriaInputs = std::make_shared<tl::StopCriteriaDecodingInputs>(localBatchSize); auto outputParams = std::make_shared<tl::BaseDecodingOutputs>(gpuOutputIds); - outputParams->sequenceLength = manager->gpu(ITensor::makeShape({signedBatchSize}), nvinfer1::DataType::kINT32); + outputParams->sequenceLength = manager->gpu(ITensor::makeShape({signedBatchSize}), tensorrt_llm::DataType::kINT32); manager->copy(cpuSequenceLengths.data(), *outputParams->sequenceLength.value(), MemoryType::kCPU); outputParams->newTokens - = manager->gpu(ITensor::makeShape({signedBatchSize, signedBeamWidth}), nvinfer1::DataType::kINT32); + = manager->gpu(ITensor::makeShape({signedBatchSize, signedBeamWidth}), tensorrt_llm::DataType::kINT32); outputParams->finished = manager->gpu( ITensor::makeShape({signedBatchSize, signedBeamWidth}), TRTDataType<tk::FinishedState::UnderlyingType>::value); diff --git a/cpp/tests/unit_tests/runtime/tllmBuffersTest.cpp b/cpp/tests/unit_tests/runtime/tllmBuffersTest.cpp index c901061695bb..4080a0f29e9e 100644 --- a/cpp/tests/unit_tests/runtime/tllmBuffersTest.cpp +++ b/cpp/tests/unit_tests/runtime/tllmBuffersTest.cpp @@ -19,6 +19,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/stringUtils.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaMemPool.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -230,7 +231,7 @@ TEST_F(TllmBuffersTest, DeviceBuffer) { CudaAllocatorAsync allocator{mStream, mMemPool}; { - DeviceBuffer buffer{size, nvinfer1::DataType::kFLOAT, allocator}; + DeviceBuffer buffer{size, tensorrt_llm::DataType::kFLOAT, allocator}; testBuffer(buffer, sizeof(float)); } streamPtr->synchronize(); @@ -242,7 +243,7 @@ TEST_F(TllmBuffersTest, DeviceBuffer) { CudaAllocator allocator{}; { - StaticDeviceBuffer buffer{size, nvinfer1::DataType::kFLOAT, allocator}; + StaticDeviceBuffer buffer{size, tensorrt_llm::DataType::kFLOAT, allocator}; testBuffer(buffer, sizeof(float)); } streamPtr->synchronize(); @@ -263,10 +264,10 @@ TEST_F(TllmBuffersTest, DeviceTensor) GTEST_SKIP() << noPoolSkipReason; } auto streamPtr = std::make_shared<CudaStream>(); - nvinfer1::Dims constexpr dims{3, 16, 8, 4}; + tensorrt_llm::Dims constexpr dims{3, 16, 8, 4}; CudaAllocatorAsync allocator{streamPtr, mMemPool}; { - DeviceTensor tensor{dims, nvinfer1::DataType::kFLOAT, allocator}; + DeviceTensor tensor{dims, tensorrt_llm::DataType::kFLOAT, allocator}; EXPECT_EQ(tensor.getSize(), ITensor::volume(dims)); testBuffer(tensor, sizeof(float)); EXPECT_EQ(tensor.getSize(), ITensor::volume(tensor.getShape())); @@ -281,7 +282,7 @@ TEST_F(TllmBuffersTest, BufferSlice) { auto constexpr size = 1024; HostAllocator allocator{}; - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; auto buffer = std::make_shared<HostBuffer>(size, dataType, allocator); auto offset = size / 8; auto slice = IBuffer::slice(buffer, offset); @@ -319,7 +320,7 @@ TEST_F(TllmBuffersTest, BufferOutput) CudaAllocatorAsync allocator{streamPtr, mMemPool}; for (std::size_t size : {0, 16}) { - DeviceBuffer buffer{size, nvinfer1::DataType::kFLOAT, allocator}; + DeviceBuffer buffer{size, tensorrt_llm::DataType::kFLOAT, allocator}; TLLM_CUDA_CHECK(cudaMemsetAsync(buffer.data(), 0, buffer.getSizeInBytes(), streamPtr->get())); streamPtr->synchronize(); std::stringstream ss; @@ -343,11 +344,11 @@ TEST_F(TllmBuffersTest, TensorOutput) } auto streamPtr = std::make_shared<CudaStream>(); - nvinfer1::Dims constexpr dims{3, 16, 8, 4}; + tensorrt_llm::Dims constexpr dims{3, 16, 8, 4}; CudaAllocatorAsync allocator{streamPtr, mMemPool}; - for (auto dataType : - {nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF, nvinfer1::DataType::kBOOL, nvinfer1::DataType::kINT8, - nvinfer1::DataType::kINT32, nvinfer1::DataType::kINT64, nvinfer1::DataType::kUINT8}) + for (auto dataType : {tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF, tensorrt_llm::DataType::kBOOL, + tensorrt_llm::DataType::kINT8, tensorrt_llm::DataType::kINT32, tensorrt_llm::DataType::kINT64, + tensorrt_llm::DataType::kUINT8}) { DeviceTensor tensor{dims, dataType, allocator}; TLLM_CUDA_CHECK(cudaMemsetAsync(tensor.data(), 0, tensor.getSizeInBytes(), streamPtr->get())); @@ -483,8 +484,8 @@ TEST_F(TllmBuffersTest, PinnedPoolAllocator) EXPECT_EQ(segments.size(), 0); { - auto a = BufferManager::pinnedPool(ITensor::makeShape({512, 4, 4}), nvinfer1::DataType::kFLOAT); - auto b = BufferManager::pinnedPool(ITensor::makeShape({512, 10}), nvinfer1::DataType::kHALF); + auto a = BufferManager::pinnedPool(ITensor::makeShape({512, 4, 4}), tensorrt_llm::DataType::kFLOAT); + auto b = BufferManager::pinnedPool(ITensor::makeShape({512, 10}), tensorrt_llm::DataType::kHALF); pool.logSegments(); auto it = std::begin(segments); EXPECT_NE(it->tag, nullptr); @@ -512,7 +513,7 @@ TEST_F(TllmBuffersTest, PinnedPoolAllocator) std::size_t secondChunkSize; { // Test creating a new chunk - auto c = BufferManager::pinnedPool(ITensor::makeShape({initChunkSize + 1}), nvinfer1::DataType::kUINT8); + auto c = BufferManager::pinnedPool(ITensor::makeShape({initChunkSize + 1}), tensorrt_llm::DataType::kUINT8); pool.logSegments(); auto it = std::begin(segments); EXPECT_EQ(it->tag, nullptr); diff --git a/cpp/tests/unit_tests/runtime/tllmRuntimeTest.cpp b/cpp/tests/unit_tests/runtime/tllmRuntimeTest.cpp deleted file mode 100644 index b3a6f99146c2..000000000000 --- a/cpp/tests/unit_tests/runtime/tllmRuntimeTest.cpp +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TOP_LEVEL_DIR -#error "Define TOP_LEVEL_DIR" -#endif - -#include <NvInfer.h> -#include <NvOnnxParser.h> -#include <gtest/gtest.h> - -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/runtime/rawEngine.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/runtime/tllmRuntime.h" - -#include <algorithm> -#include <array> -#include <filesystem> -#include <memory> -#include <vector> - -namespace fs = std::filesystem; -namespace trt = nvinfer1; - -namespace -{ -auto const TEST_RESOURCE_DIR = fs::path{TOP_LEVEL_DIR} / "cpp/tests/resources"; -auto const MNIST_MODEL_PATH = TEST_RESOURCE_DIR / "models/mnist.onnx"; - -template <typename T> -std::unique_ptr<T> makeUnique(T* ptr) -{ - EXPECT_NE(ptr, nullptr); - return std::unique_ptr<T>(ptr); -} - -std::unique_ptr<trt::IHostMemory> buildMnistEngine(trt::ILogger& logger) -{ - EXPECT_TRUE(fs::exists(MNIST_MODEL_PATH)); - auto builder = makeUnique(trt::createInferBuilder(logger)); - auto const explicitBatch = 1U << static_cast<uint32_t>(trt::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH); - auto network = makeUnique(builder->createNetworkV2(explicitBatch)); - auto parser = makeUnique(nvonnxparser::createParser(*network, logger)); - auto const parsingSuccess = parser->parseFromFile( - MNIST_MODEL_PATH.string().c_str(), static_cast<int32_t>(trt::ILogger::Severity::kWARNING)); - EXPECT_TRUE(parsingSuccess); - auto config = makeUnique(builder->createBuilderConfig()); - return makeUnique(builder->buildSerializedNetwork(*network, *config)); -} -} // namespace - -using namespace tensorrt_llm::runtime; -namespace tc = tensorrt_llm::common; - -class TllmRuntimeTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) -{ -protected: - void SetUp() override - { - mDeviceCount = tc::getDeviceCount(); - - if (mDeviceCount == 0) - GTEST_SKIP(); - - mLogger.setLevel(trt::ILogger::Severity::kINFO); - mSerializedEngine = buildMnistEngine(mLogger); - ASSERT_NE(mSerializedEngine, nullptr); - } - - void TearDown() override {} - - int mDeviceCount; - TllmLogger mLogger{}; - std::unique_ptr<trt::IHostMemory> mSerializedEngine; -}; - -TEST_F(TllmRuntimeTest, SinglePass) -{ - EXPECT_TRUE(mSerializedEngine); - TllmRuntime rt{RawEngine(mSerializedEngine.get()), &mLogger, false, 1.0F}; - auto& engine = rt.getEngine(); - EXPECT_FALSE(engine.hasImplicitBatchDimension()); - EXPECT_EQ(rt.getNbProfiles(), engine.getNbOptimizationProfiles()); - EXPECT_EQ(rt.getNbContexts(), 0); - auto const nbIoTensors = engine.getNbIOTensors(); - EXPECT_EQ(nbIoTensors, 2); - rt.addContext(0); - EXPECT_EQ(rt.getNbContexts(), 1); - - auto constexpr dataType = trt::DataType::kFLOAT; - - auto const inputName = engine.getIOTensorName(0); - EXPECT_EQ(engine.getTensorIOMode(inputName), trt::TensorIOMode::kINPUT); - auto const inputDims = engine.getTensorShape(inputName); - std::array constexpr inputDimsExpected = {1, 1, 28, 28}; - EXPECT_EQ(inputDims.nbDims, inputDimsExpected.size()); - for (int i = 0; i < inputDims.nbDims; ++i) - { - EXPECT_EQ(inputDims.d[i], inputDimsExpected[i]); - } - EXPECT_EQ(engine.getTensorDataType(inputName), dataType); - - auto const outputName = engine.getIOTensorName(1); - EXPECT_EQ(engine.getTensorIOMode(outputName), trt::TensorIOMode::kOUTPUT); - auto const outputDims = engine.getTensorShape(outputName); - std::array constexpr outputDimsExpected = {1, 10}; - EXPECT_EQ(outputDims.nbDims, outputDimsExpected.size()); - for (int i = 0; i < outputDims.nbDims; ++i) - { - EXPECT_EQ(outputDims.d[i], outputDimsExpected[i]); - } - EXPECT_EQ(engine.getTensorDataType(outputName), dataType); - - auto& allocator = rt.getBufferManager(); - TllmRuntime::TensorMap tensorMap{}; - auto inputBuffer = std::shared_ptr<ITensor>{allocator.gpu(inputDims, dataType)}; - allocator.setZero(*inputBuffer); - tensorMap.insert(std::make_pair(inputName, inputBuffer)); - rt.setInputTensors(0, tensorMap); - rt.setOutputTensors(0, tensorMap); - ASSERT_NE(tensorMap.find(outputName), tensorMap.end()); - auto outputBuffer = tensorMap.at(outputName); - allocator.setZero(*outputBuffer); - rt.executeContext(0); - - std::vector<float> output(outputBuffer->getSize()); - allocator.copy(*outputBuffer, output.data()); - rt.getStream().synchronize(); - auto min = std::min_element(output.begin(), output.end()); - EXPECT_NEAR(*min, -0.126409f, 1e-5f); - auto max = std::max_element(output.begin(), output.end()); - EXPECT_NEAR(*max, 0.140218f, 1e-5f); -} diff --git a/cpp/tests/unit_tests/runtime/torchTest.cpp b/cpp/tests/unit_tests/runtime/torchTest.cpp index 4aa498de8d25..4ca36d875d44 100644 --- a/cpp/tests/unit_tests/runtime/torchTest.cpp +++ b/cpp/tests/unit_tests/runtime/torchTest.cpp @@ -17,6 +17,7 @@ #include <gmock/gmock.h> #include <gtest/gtest.h> +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/torch.h" #include "tensorrt_llm/runtime/torchView.h" @@ -52,7 +53,7 @@ class TorchTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-me namespace { -template <nvinfer1::DataType DType> +template <tensorrt_llm::DataType DType> void checkFilled(IBuffer& buffer, int fillValue) { if (DType == buffer.getDataType()) @@ -79,13 +80,13 @@ TEST_F(TorchTest, Aten) } auto constexpr fillValue = 1; - auto tensorHostBase = manager.allocate(MemoryType::kPINNED, shapeTllm, nvinfer1::DataType::kINT64); + auto tensorHostBase = manager.allocate(MemoryType::kPINNED, shapeTllm, tensorrt_llm::DataType::kINT64); for (auto memoryType : {MemoryType::kCPU, MemoryType::kGPU, MemoryType::kPINNED}) { - for (auto dtype : {nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF, nvinfer1::DataType::kINT8, - nvinfer1::DataType::kUINT8, nvinfer1::DataType::kINT32, nvinfer1::DataType::kINT64, - nvinfer1::DataType::kBF16, nvinfer1::DataType::kFP8, nvinfer1::DataType::kBOOL}) + for (auto dtype : {tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF, tensorrt_llm::DataType::kINT8, + tensorrt_llm::DataType::kUINT8, tensorrt_llm::DataType::kINT32, tensorrt_llm::DataType::kINT64, + tensorrt_llm::DataType::kBF16, tensorrt_llm::DataType::kFP8, tensorrt_llm::DataType::kBOOL}) { ITensor::SharedPtr tensorTllm{manager.allocate(memoryType, shapeTllm, dtype)}; @@ -98,20 +99,20 @@ TEST_F(TorchTest, Aten) EXPECT_THAT(tensorAten.sizes(), ::testing::ElementsAreArray(shapeAten)); EXPECT_EQ(tensorAten.data_ptr(), tensorTllm->data()); - if (dtype != nvinfer1::DataType::kFP8) + if (dtype != tensorrt_llm::DataType::kFP8) { tensorAten.fill_(c10::Scalar(fillValue)); auto tensorHost = ITensor::wrap(tensorHostBase->data(), dtype, shapeTllm); manager.copy(*tensorTllm, *tensorHost); mStream->synchronize(); - checkFilled<nvinfer1::DataType::kFLOAT>(*tensorHost, fillValue); - checkFilled<nvinfer1::DataType::kHALF>(*tensorHost, fillValue); - checkFilled<nvinfer1::DataType::kINT8>(*tensorHost, fillValue); - checkFilled<nvinfer1::DataType::kUINT8>(*tensorHost, fillValue); - checkFilled<nvinfer1::DataType::kINT32>(*tensorHost, fillValue); - checkFilled<nvinfer1::DataType::kINT64>(*tensorHost, fillValue); - checkFilled<nvinfer1::DataType::kBF16>(*tensorHost, fillValue); - checkFilled<nvinfer1::DataType::kBOOL>(*tensorHost, fillValue); + checkFilled<tensorrt_llm::DataType::kFLOAT>(*tensorHost, fillValue); + checkFilled<tensorrt_llm::DataType::kHALF>(*tensorHost, fillValue); + checkFilled<tensorrt_llm::DataType::kINT8>(*tensorHost, fillValue); + checkFilled<tensorrt_llm::DataType::kUINT8>(*tensorHost, fillValue); + checkFilled<tensorrt_llm::DataType::kINT32>(*tensorHost, fillValue); + checkFilled<tensorrt_llm::DataType::kINT64>(*tensorHost, fillValue); + checkFilled<tensorrt_llm::DataType::kBF16>(*tensorHost, fillValue); + checkFilled<tensorrt_llm::DataType::kBOOL>(*tensorHost, fillValue); } // Conversion back to TRT-LLM tensor diff --git a/cpp/tests/unit_tests/runtime/utilsTest.cpp b/cpp/tests/unit_tests/runtime/utilsTest.cpp index 8b69070c03d8..58882824a7dc 100644 --- a/cpp/tests/unit_tests/runtime/utilsTest.cpp +++ b/cpp/tests/unit_tests/runtime/utilsTest.cpp @@ -18,6 +18,7 @@ #error "Define TOP_LEVEL_DIR" #endif +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -71,7 +72,7 @@ TEST_F(UtilsTest, LoadNpy) TEST_F(UtilsTest, LoadStoreNpy) { auto dims = ITensor::makeShape({2, 3, 4}); - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; ITensor::SharedPtr tensor{BufferManager::cpu(dims, dataType)}; auto tensorRange = BufferRange<float>(*tensor); std::iota(tensorRange.begin(), tensorRange.end(), 0); @@ -96,7 +97,7 @@ TEST_F(UtilsTest, LoadStoreNpy) TEST_F(UtilsTest, LoadStoreNpyGPU) { auto dims = ITensor::makeShape({2, 3, 4}); - auto constexpr dataType = nvinfer1::DataType::kFLOAT; + auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; ITensor::SharedPtr tensor{BufferManager::cpu(dims, dataType)}; auto tensorRange = BufferRange<float>(*tensor); std::iota(tensorRange.begin(), tensorRange.end(), 0); diff --git a/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp b/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp index f2045e7659d1..159a07770694 100644 --- a/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp +++ b/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp @@ -19,6 +19,7 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/nvmlWrapper.h" +#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/tllmBuffers.h" #include "tensorrt_llm/runtime/virtualMemory.h" @@ -1502,7 +1503,7 @@ TEST_F(VirtualMemoryManagerTest, TestCudaVirtualMemoryAllocator) // Create a buffer using the virtual address allocator auto buffer = std::make_unique<VirtualAddressDeviceBuffer>( - size, nvinfer1::DataType::kINT8, CudaVirtualMemoryAllocator{config}); + size, tensorrt_llm::DataType::kINT8, CudaVirtualMemoryAllocator{config}); auto memoryAfterAllocation = getCurrentProcessMemoryInfo(); if (memoryInfoAvailable()) @@ -1513,7 +1514,7 @@ TEST_F(VirtualMemoryManagerTest, TestCudaVirtualMemoryAllocator) // Test that we can access the buffer data ASSERT_NE(buffer->data(), nullptr) << "Buffer data should not be null"; ASSERT_EQ(buffer->getSize(), size) << "Buffer size should match requested size"; - ASSERT_EQ(buffer->getDataType(), nvinfer1::DataType::kINT8) << "Buffer data type should be INT8"; + ASSERT_EQ(buffer->getDataType(), tensorrt_llm::DataType::kINT8) << "Buffer data type should be INT8"; ASSERT_EQ(buffer->getMemoryType(), MemoryType::kGPU) << "Buffer memory type should be GPU"; // Test memory access by setting memory to a known pattern @@ -1574,7 +1575,7 @@ TEST_F(VirtualMemoryManagerTest, TestCudaVirtualMemoryAllocatorUnalignedSize) // Create a buffer using the virtual address allocator auto buffer = std::make_unique<VirtualAddressDeviceBuffer>( - size, nvinfer1::DataType::kINT8, CudaVirtualMemoryAllocator{config}); + size, tensorrt_llm::DataType::kINT8, CudaVirtualMemoryAllocator{config}); auto memoryAfterAllocation = getCurrentProcessMemoryInfo(); if (memoryInfoAvailable()) diff --git a/cpp/tests/unit_tests/utils/CMakeLists.txt b/cpp/tests/unit_tests/utils/CMakeLists.txt deleted file mode 100644 index 2d7e9145c817..000000000000 --- a/cpp/tests/unit_tests/utils/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -add_gtest(testUtilsTest utilsTest.cpp) diff --git a/cpp/tests/unit_tests/utils/utilsTest.cpp b/cpp/tests/unit_tests/utils/utilsTest.cpp deleted file mode 100644 index 38587120ef84..000000000000 --- a/cpp/tests/unit_tests/utils/utilsTest.cpp +++ /dev/null @@ -1,55 +0,0 @@ - -#include "common.h" -#include "tensorrt_llm/runtime/common.h" - -#include <gtest/gtest.h> - -#include <cstdint> -#include <numeric> - -struct RandomLogitsTestParameters -{ - using TupleT = std::tuple<int32_t, tensorrt_llm::runtime::SizeType32>; - - int32_t randomSeed; - tensorrt_llm::runtime::SizeType32 vocabSize; - - // Constructor that takes a tuple - RandomLogitsTestParameters( // NOLINT: implicit to allow gtest to convert from tuple generated by - // 'combine' - TupleT t) - : randomSeed(std::get<0>(t)) - , vocabSize(std::get<1>(t)) - { - } -}; - -class RandomLogits : public ::testing::Test, public ::testing::WithParamInterface<RandomLogitsTestParameters> -{ -protected: - static constexpr int randomSeed = 2345; -}; - -namespace -{ -constexpr int32_t kRandomSeed1 = 45; -constexpr int32_t kRandomSeed2 = 567; -auto const randomSeeds = ::testing::Values(kRandomSeed1, kRandomSeed2); - -constexpr tensorrt_llm::runtime::SizeType32 kMinVocabSize = 16; -constexpr tensorrt_llm::runtime::SizeType32 kMaxVocabSize = 100000; -auto const vocabSizes = ::testing::Values(kMinVocabSize, kMaxVocabSize); - -auto const paramGenerator - = ::testing::ConvertGenerator<RandomLogitsTestParameters::TupleT>(::testing::Combine(randomSeeds, vocabSizes)); -} // namespace - -TEST_P(RandomLogits, FloatSumToOne) -{ - auto rng = std::mt19937(randomSeed); - auto const randomLogits = tensorrt_llm::testing::randomLogits<std::mt19937, float>(456, &rng); - auto const sum = std::reduce(randomLogits.begin(), randomLogits.end()); - ASSERT_DOUBLE_EQ(sum, 1.0); -} - -INSTANTIATE_TEST_SUITE_P(Float, RandomLogits, paramGenerator); diff --git a/cpp/tests/utils/CMakeLists.txt b/cpp/tests/utils/CMakeLists.txt deleted file mode 100644 index 0123ac1c4600..000000000000 --- a/cpp/tests/utils/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -add_library(testingUtils common.cpp engines.cpp executorUtils.cpp) -target_link_libraries(testingUtils PUBLIC gtest_main ${SHARED_TARGET}) -target_include_directories(testingUtils PRIVATE ${MPI_C_INCLUDE_DIRS}) -target_compile_definitions(testingUtils PUBLIC TOP_LEVEL_DIR="${TOP_LEVEL_DIR}") diff --git a/cpp/tests/utils/common.cpp b/cpp/tests/utils/common.cpp deleted file mode 100644 index 5640cef7b495..000000000000 --- a/cpp/tests/utils/common.cpp +++ /dev/null @@ -1,684 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "common.h" - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/utils/numpyUtils.h" -#include "tensorrt_llm/testing/modelSpec.h" -#include "tests/utils/common.h" - -#include <gtest/gtest.h> - -#include <algorithm> -#include <vector> - -namespace tensorrt_llm::testing -{ -namespace fs = std::filesystem; -namespace tr = tensorrt_llm::runtime; -namespace tc = tensorrt_llm::common; - -std::string PathUtil::FP16_GPT_ATTENTION_PACKED_DIR() -{ - return ModelSpec::getDefaultModelSpec().setKVCacheType(KVCacheType::kCONTINUOUS).getModelPath(); -} - -std::string PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() -{ - return ModelSpec::getDefaultModelSpec().getModelPath(); -} - -std::string PathUtil::FP16_GPT_LORA_DIR() -{ - return ModelSpec::getDefaultModelSpec().useLoraPlugin().getModelPath(); -} - -std::string PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DRAFT_TOKENS_DIR() -{ - return ModelSpec::getDefaultModelSpec().useDraftTokensExternalDecoding().getModelPath(); -} - -std::string PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() -{ - return ModelSpec::getDefaultModelSpec().gatherLogits().getModelPath(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_FILE() -{ - return ModelSpec::getDefaultModelSpec().getResultsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_LONG_RESULT_FILE() -{ - return ModelSpec::getDefaultModelSpec().setInputFile("input_tokens_long.npy").getResultsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE() -{ - return ModelSpec::getDefaultModelSpec().gatherLogits().getResultsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE() -{ - return ModelSpec::getDefaultModelSpec().gatherLogits().getGenerationLogitsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE() -{ - return ModelSpec::getDefaultModelSpec().gatherLogits().getContextLogitsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_FILE() -{ - return ModelSpec::getDefaultModelSpec().getCumLogProbsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_CUM_LOG_PROBS_FILE() -{ - return ModelSpec::getDefaultModelSpec().gatherLogits().getCumLogProbsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_FILE() -{ - return ModelSpec::getDefaultModelSpec().getLogProbsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_LOG_PROBS_FILE() -{ - return ModelSpec::getDefaultModelSpec().gatherLogits().getLogProbsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP1_FILE() -{ - return ModelSpec::getDefaultModelSpec().getResultsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP4_PP1_FILE() -{ - return ModelSpec::getDefaultModelSpec().useTensorParallelism(4).getResultsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE() -{ - return ModelSpec::getDefaultModelSpec().useTensorParallelism(2).usePipelineParallelism(2).getResultsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP4_FILE() -{ - return ModelSpec::getDefaultModelSpec().usePipelineParallelism(4).getResultsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP2_FILE() -{ - return ModelSpec::getDefaultModelSpec().usePipelineParallelism(2).getResultsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP1_FILE() -{ - return ModelSpec::getDefaultModelSpec().useTensorParallelism(2).getResultsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_TP4_PP1_FILE() -{ - return ModelSpec::getDefaultModelSpec().useTensorParallelism(4).getContextLogitsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_TP4_PP1_FILE() -{ - return ModelSpec::getDefaultModelSpec().useTensorParallelism(4).getGenerationLogitsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_TP4_PP1_FILE() -{ - return ModelSpec::getDefaultModelSpec().useTensorParallelism(4).getCumLogProbsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_TP4_PP1_FILE() -{ - return ModelSpec::getDefaultModelSpec().useTensorParallelism(4).getLogProbsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_CONTEXTFMHAFP32ACC_RESULT_FILE() -{ - return ModelSpec::getDefaultModelSpec().gatherLogits().enableContextFMHAFp32Acc().getResultsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXTFMHAFP32ACC_GENERATION_LOGITS_FILE() -{ - return ModelSpec::getDefaultModelSpec().gatherLogits().enableContextFMHAFp32Acc().getGenerationLogitsFile(); -} - -std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXTFMHAFP32ACC_CONTEXT_LOGITS_FILE() -{ - return ModelSpec::getDefaultModelSpec().gatherLogits().enableContextFMHAFp32Acc().getContextLogitsFile(); -} - -void TestData::loadLogProbs( - fs::path const& cumLogProbsFile, fs::path const& logProbsFile, tr::BufferManager const& manager) -{ - TLLM_CHECK_WITH_INFO( - cumLogProbsFile != "", "Testing return log probs, but missing the expected cum log probs results file."); - auto expectedCumLogProbsPtr - = std::shared_ptr(tr::utils::loadNpy(manager, cumLogProbsFile.string(), MemoryType::kCPU)); - - TLLM_CHECK_WITH_INFO( - logProbsFile != "", "Testing return log probs, but missing the expected log probs results file."); - auto expectedLogProbsPtr = std::shared_ptr(tr::utils::loadNpy(manager, logProbsFile.string(), MemoryType::kCPU)); - - for (SizeType32 inputIdx = 0; inputIdx < nbGivenInputs; ++inputIdx) - { - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - auto expectedCumLogProbsBatchSlice = std::shared_ptr(ITensor::slice(expectedCumLogProbsPtr, inputIdx, 1)); - expectedCumLogProbsBatchSlice->squeeze(0); // bs - expectedCumLogProbs[inputIdx] = expectedCumLogProbsBatchSlice; // shape: [beamWidth] - - auto expectedLogProbsBatchSlice = std::shared_ptr(ITensor::slice(expectedLogProbsPtr, inputIdx, 1)); - expectedLogProbsBatchSlice->squeeze(0); // bs - expectedLogProbs[inputIdx] = expectedLogProbsBatchSlice; // shape: [beamWidth, numOutputTokens] - } - } -} - -void TestData::loadContextLogits(fs::path const& contextLogitsFile, std::vector<SizeType32> const& givenInputLengths, - tr::BufferManager const& manager) -{ - TLLM_CHECK_WITH_INFO(contextLogitsFile != "", - "Testing with gather or replace logits, but missing the expected context logits results file."); - auto expectedContextLogitsPtr - = std::shared_ptr(tr::utils::loadNpy(manager, contextLogitsFile.string(), MemoryType::kCPU)); - - int promptOffset = 0; - for (SizeType32 bi = 0; bi < nbGivenInputs; ++bi) - { - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - auto expectedContextLogitBatchSlice - = std::shared_ptr(ITensor::slice(expectedContextLogitsPtr, promptOffset, givenInputLengths.at(bi))); - expectedContextLogits.at(bi) = expectedContextLogitBatchSlice; // shape: [prompt_length, vocab_size] - } - promptOffset += givenInputLengths.at(bi); - } -} - -void TestData::loadGenerationLogits(fs::path const& genLogitsFile, tr::BufferManager const& manager) -{ - TLLM_CHECK_WITH_INFO(genLogitsFile != "", - "Testing with gather or replace logits, but missing the expected generation logits results file."); - auto expectedGenerationLogitsPtr - = std::shared_ptr(tr::utils::loadNpy(manager, genLogitsFile.string(), MemoryType::kCPU)); - - for (SizeType32 bi = 0; bi < nbGivenInputs; ++bi) - { - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - auto expectedGenerationLogitBatchSlice - = std::shared_ptr(ITensor::slice(expectedGenerationLogitsPtr, bi, 1)); - expectedGenerationLogitBatchSlice->squeeze(0); // bs - expectedGenerationLogitBatchSlice->squeeze(0); // beam - expectedGenerationLogits.at(bi) = expectedGenerationLogitBatchSlice; // shape: [max_output_len, vocab_size] - } - } -} - -void TestData::makeDraft(SizeType32 maxDraftTokens, bool acceptDraftByLogits, fs::path const& genLogitsFile, - std::vector<SizeType32> const& givenInputLengths, tr::BufferManager const& manager) -{ - TLLM_CHECK(beamWidth == 1); - - ITensor::SharedPtr expectedGenerationLogitsPtr; - if (acceptDraftByLogits) - { - TLLM_CHECK_WITH_INFO( - genLogitsFile != "", "Testing Draft token, but missing the expected generation logits results file."); - expectedGenerationLogitsPtr - = std::shared_ptr(tr::utils::loadNpy(manager, genLogitsFile.string(), MemoryType::kCPU)); - } - - std::vector<SizeType32> draftLengths(givenInputLengths.size()); - // first draft length stays 0 - std::transform(givenInputLengths.begin() + 1, givenInputLengths.end(), draftLengths.begin() + 1, - [this, &maxDraftTokens](auto inputLength) - { return std::rand() % std::min((maxSeqLen - (inputLength + 1)), maxDraftTokens) + 1; }); - - auto* const expectedOutputData = tr::bufferCast<TokenIdType>(*expectedOutputIds); - for (SizeType32 bi = 0; bi < nbGivenInputs; ++bi) - { - SizeType32 constexpr beamIdx{0}; - auto const endId = endIds.at(bi); - auto const draftLen = draftLengths.at(bi); - auto acceptedLen = draftLen > 0 ? std::rand() % draftLen : 0; - - if (acceptDraftByLogits && draftLen > 0) - { - auto expectedLogitBatchSlice = std::shared_ptr(ITensor::slice(expectedGenerationLogitsPtr, bi, 1)); - expectedLogitBatchSlice->squeeze(0); // bs - expectedLogitBatchSlice->squeeze(0); // beam - auto expectedLogitBatchStepSlice = std::shared_ptr(ITensor::slice(expectedLogitBatchSlice, 1, draftLen)); - auto expectedLogitBatchStepView = ITensor::view(expectedLogitBatchStepSlice, - ITensor::makeShape({draftLen, 1, 1, expectedLogitBatchStepSlice->getShape().d[1]})); - draftLogits.at(bi) = manager.copyFrom(*expectedLogitBatchStepView, MemoryType::kCPU); - } - - for (SizeType32 si = 0; si < draftLen; ++si) - { - auto const draftIndex - = tc::flat_index3(bi, beamIdx, givenInputLengths.at(bi) + si + 1, beamWidth, maxSeqLen); - auto draftToken = expectedOutputData[draftIndex]; - if (draftToken == endId) - { - acceptedLen = std::min(acceptedLen, si); - } - if (si >= acceptedLen) - { - draftToken = -1; - if (acceptDraftByLogits) - { - auto vocabSizePadded = expectedGenerationLogitsPtr->getShape().d[3]; - auto* draftLogitsPtr = tr::bufferCast<float>(*draftLogits.at(bi)); - for (SizeType32 vi = 0; vi < vocabSizePadded; ++vi) - { - draftLogitsPtr[si * vocabSizePadded + vi] = 0.f; - } - } - } - draftTokens.at(bi).push_back(draftToken); - } - acceptedDraftTokensLengths.at(bi) = acceptedLen; - - auto const expectedLen = expectedOutputLengths.at(bi * beamWidth + beamIdx); - TLLM_CHECK(expectedLen > 0); - expectedOutputLengths[bi * beamWidth + beamIdx] - = draftLen > 0 ? std::min(expectedLen, (givenInputLengths.at(bi) + 1) + acceptedLen + 1) : expectedLen; - } -} - -template <typename T> -bool invokeCompareLogits(ITensor const& groundTruthLogits, ITensor const& outputLogits, float atol, float rtol) -{ - bool allMatch = true; - T const* const gtLogitsPtr = tr::bufferCast<T>(groundTruthLogits); - T const* const outputLogitsPtr = tr::bufferCast<T>(outputLogits); - - size_t outputSize = outputLogits.getSize(); - int errorNumber = 0; - - for (size_t i = 0; i < outputSize; i++) - { - if (!almostEqual(outputLogitsPtr[i], gtLogitsPtr[i], atol, rtol)) - { - TLLM_LOG_DEBUG("Mismatch value. Position of logits: %d, expected value: %f, output value: %f", i, - gtLogitsPtr[i], outputLogitsPtr[i]); - allMatch = false; - errorNumber++; - if (errorNumber == 10) - { - break; - } - } - } - return allMatch; -} - -bool compareLogits(ITensor const& groundTruthLogits, ITensor const& outputLogits, float atol, float rtol) -{ - EXPECT_EQ(groundTruthLogits.getDataType(), outputLogits.getDataType()); - switch (groundTruthLogits.getDataType()) - { - case nvinfer1::DataType::kFLOAT: return invokeCompareLogits<float>(groundTruthLogits, outputLogits, atol, rtol); - case nvinfer1::DataType::kHALF: return invokeCompareLogits<half>(groundTruthLogits, outputLogits, atol, rtol); - default: TLLM_THROW("Unsupported data type"); - } -} - -std::tuple<SizeType32, SizeType32> getRequestGivenInputIdxLength( - std::uint64_t requestId, SizeType32 nbGivenInputs, std::vector<SizeType32> const& givenInputLengths) -{ - auto const givenInputIdx = requestId % nbGivenInputs; - auto const inputLength = givenInputLengths.at(givenInputIdx); - return {givenInputIdx, inputLength}; -} - -std::tuple<std::vector<SizeType32>, SizeType32, SizeType32> getGivenInputLengths( - ITensor const& givenInput, SizeType32 padId) -{ - auto const& inputShape = givenInput.getShape(); - auto const nbGivenInputs = static_cast<SizeType32>(inputShape.d[0]); - auto const maxInputLength = static_cast<SizeType32>(inputShape.d[1]); - auto const* const givenInputData = tr::bufferCast<TokenIdType const>(givenInput); - - std::vector<SizeType32> givenInputLengths(nbGivenInputs); - for (SizeType32 i = 0; i < nbGivenInputs; ++i) - { - auto const* const seqBegin = givenInputData + i * maxInputLength; - auto const* const it = std::find(seqBegin, seqBegin + maxInputLength, padId); - givenInputLengths[i] = std::distance(seqBegin, it); - } - - return {givenInputLengths, nbGivenInputs, maxInputLength}; -} - -std::vector<executor::TokenIdType> createConsecutiveTokenSequence( - tr::SizeType32 length, tr::SizeType32 vocabSize, tr::TokenIdType firstTokenId) -{ - auto result = std::vector<executor::TokenIdType>(static_cast<size_t>(length), 0); - std::iota(result.begin(), result.end(), firstTokenId); - std::transform(result.begin(), result.end(), result.begin(), [&](auto const i) { return i % vocabSize; }); - return result; -} - -TestData TestData::loadTestData(BeamResult const& beamResults, ITensor const& givenInput, SizeType32 const maxBeamWidth, - tr::BufferManager& manager, executor::OutputConfig const& outConfig, ModelIds const& modelIds) -{ - auto const [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(givenInput, modelIds.padId); - auto const& [beamWidth, resultsFile, contextLogitsFile, genLogitsFile, cumLogProbsFile, logProbsFile] = beamResults; - - TestData testData{nbGivenInputs, beamWidth}; - testData.expectedOutputIds = tr::utils::loadNpy(manager, resultsFile.string(), tr::MemoryType::kCPU); - - auto const& outputShape = testData.expectedOutputIds->getShape(); - EXPECT_EQ(outputShape.nbDims, 2); - EXPECT_EQ(nbGivenInputs * beamWidth, outputShape.d[0]); - testData.maxSeqLen = static_cast<SizeType32>(outputShape.d[1]); - EXPECT_LE(maxInputLength, testData.maxSeqLen); - EXPECT_LE(beamWidth, maxBeamWidth); - - auto const maxNewTokens = testData.maxSeqLen - maxInputLength; - - testData.endIds.insert(testData.endIds.end(), nbGivenInputs, modelIds.endId); - - if (outConfig.returnContextLogits && beamWidth == 1) - { - testData.loadContextLogits(contextLogitsFile, givenInputLengths, manager); - } - if (outConfig.returnGenerationLogits && beamWidth == 1) - { - testData.loadGenerationLogits(genLogitsFile, manager); - } - if (outConfig.returnLogProbs && beamWidth == 1) - { - testData.loadLogProbs(cumLogProbsFile, logProbsFile, manager); - } - - for (SizeType32 inputIdx = 0; inputIdx < nbGivenInputs; ++inputIdx) - { - for (SizeType32 beam = 0; beam < beamWidth; ++beam) - { - SizeType32 expectedLen = givenInputLengths[inputIdx] + maxNewTokens; - testData.expectedOutputLengths[inputIdx * beamWidth + beam] = expectedLen; - } - } - - return testData; -} - -void TestData::verifyOutput(std::unordered_map<SizeType32, std::vector<executor::BeamTokens>> const& resultTokens, - std::vector<SizeType32> const& givenInputLengths, bool streaming, bool excludeInputFromOutput, - FlakyTestInfo flakyTestInfo, bool isSpeculativeDecoding, SizeType32 reqBeamWidth, SizeType32 numReturnSequences, - bool isNonGreedySampling) -{ - for (auto const& [batchId, beamTokens] : resultTokens) - { - for (auto seqIdx = 0; seqIdx < numReturnSequences; seqIdx++) - { - auto const& tokens = beamTokens.at(seqIdx); - auto const inputLength = givenInputLengths.at(batchId); - SizeType32 const numReturnBeams = tokens.size(); - auto const* const expectedOutputData = tr::bufferCast<TokenIdType const>(*this->expectedOutputIds); - auto const expectedOutputLengths = this->expectedOutputLengths; - auto const endId = this->endIds[batchId]; - auto const maxSeqLen = this->maxSeqLen; - - for (SizeType32 beam = 0; beam < numReturnBeams; ++beam) - { - bool isFlaky = flakyTestInfo.batchIdBeams.count(std::make_pair(batchId, beam)); - if (isFlaky) - { - TLLM_LOG_WARNING("Disabling token comparison for batchId %d beam %d, test if flaky", batchId, beam); - } - - auto const expectInputOutputLength - = expectedOutputLengths[batchId * reqBeamWidth + beam]; // Ground truth output length - auto expectedOutputLength - = expectInputOutputLength - inputLength; // Number of new generated output tokens - - bool inputNotIncluded = (streaming || excludeInputFromOutput); - bool anyMismatch = false; - auto predictedTokens = tokens.at(beam); - // Remove the prompt - if (!inputNotIncluded) - { - predictedTokens.erase(predictedTokens.begin(), predictedTokens.begin() + inputLength); - } - - if (!isNonGreedySampling) - { - EXPECT_EQ(predictedTokens.size(), expectedOutputLength) - << "b: " << batchId << " seq: " << seqIdx << " beam: " << beam; - } - - auto numPredTokens = static_cast<SizeType32>(predictedTokens.size()); - - if (isSpeculativeDecoding) - { - // WAR to ensure bulk execution of spec decoding. - // We hope that no request in batch can finish 2x faster than any other request. - // For the cases when BS < 8, some predicted tokens are mismatched to reference data. - numPredTokens /= 2; - } - - for (auto i = 0; i < numPredTokens; ++i) - { - // Use the expected data for that beamWidth - auto const expectIndex = tc::flat_index3(batchId, beam, inputLength + i, reqBeamWidth, maxSeqLen); - auto const expectedToken = expectedOutputData[expectIndex]; - if (expectedToken == endId) - { - // TODO: can not find the error when (expectedToken == endId) && (predictedToken != endId) - break; - } - auto const predictedToken = predictedTokens.at(i); - if (!isFlaky && !isNonGreedySampling) - { - EXPECT_EQ(predictedToken, expectedToken) - << "b: " << batchId << " seq: " << seqIdx << " beam: " << beam << " i: " << i; - } - anyMismatch |= (predictedToken != expectedToken); - } - if (!isFlaky && !isNonGreedySampling) - { - EXPECT_FALSE(anyMismatch) << "b: " << batchId << " seq: " << seqIdx << " beam: " << beam; - } - else if (isNonGreedySampling) - { - EXPECT_TRUE(anyMismatch) << "b: " << batchId << " seq: " << seqIdx << " beam: " << beam; - } - } - } - } -} - -void TestData::verifyLogProbs(bool computeLogProbs, bool streaming, bool excludeInputFromOutput, SizeType32 inputLength, - SizeType32 beamWidth, executor::BeamTokens const& beamTokens, - std::optional<executor::VecLogProbs> const& cumLogProbs, - std::optional<std::vector<executor::VecLogProbs>> const& logProbs, SizeType32 batchId, FlakyTestInfo flakyTestInfo) -{ - auto expectedCumLogProbs = this->expectedCumLogProbs[batchId]; - auto expectedLogProbs = this->expectedLogProbs[batchId]; - auto const expectedOutputLengths = this->expectedOutputLengths; - auto const numReturnBeams = beamTokens.size(); - - if (computeLogProbs) - { - EXPECT_TRUE(cumLogProbs.has_value()) << "bid: " << batchId; - EXPECT_TRUE(logProbs.has_value()) << "bid: " << batchId; - EXPECT_EQ(cumLogProbs.value().size(), numReturnBeams) << "bid: " << batchId; - EXPECT_EQ(logProbs.value().size(), numReturnBeams) << "bid: " << batchId; - - bool removeInput = !excludeInputFromOutput && !streaming; - - for (SizeType32 beam = 0; beam < numReturnBeams; ++beam) - { - bool isFlaky = flakyTestInfo.batchIdBeams.count(std::make_pair(batchId, beam)); - if (isFlaky) - { - TLLM_LOG_WARNING("Disabling token comparison for batchId %d beam %d, test if flaky", batchId, beam); - } - - auto expectedOutputLength = expectedOutputLengths[batchId * beamWidth + beam]; - expectedOutputLength -= inputLength; - - auto numPredTokens = logProbs.value().at(beam).size(); - // Check shape - EXPECT_EQ(numPredTokens, beamTokens.at(beam).size() - (removeInput ? inputLength : 0)) - << "bid: " << batchId << " beam: " << beam; - - // If beamWidth == 1, compare log probs against python runtime - if (beamWidth == 1) - { - auto* const reqExpectedCumLogProbs = tr::bufferCast<float>(*expectedCumLogProbs); - // Only check cumLogProbs for the last generated token - if (numPredTokens == expectedOutputLength && !isFlaky) - { - EXPECT_TRUE(almostEqual(reqExpectedCumLogProbs[beam], cumLogProbs.value().at(beam), 2e-1, 5e-2)) - << "expectedCumLogProbs : " << reqExpectedCumLogProbs[beam] - << " cumlogProbs : " << cumLogProbs.value().at(beam); - } - - auto expectedLogProbsBeam = std::shared_ptr(tr::ITensor::slice(expectedLogProbs, beam, 1)); - expectedLogProbsBeam->squeeze(0); - auto* const reqExpectedLogProbs = tr::bufferCast<float>(*expectedLogProbsBeam); - for (auto i = 0; i < numPredTokens; ++i) - { - if (!isFlaky) - { - EXPECT_TRUE( - almostEqual(reqExpectedLogProbs[inputLength + i], logProbs.value()[beam][i], 5e-2, 5e-2)) - << "expectedLogProbs : " << reqExpectedLogProbs[inputLength + i] - << " logProbs : " << logProbs.value()[beam][i]; - } - } - } - } - } - else - { - EXPECT_FALSE(cumLogProbs.has_value()) << "bid: " << batchId; - EXPECT_FALSE(logProbs.has_value()) << "bid: " << batchId; - } -} - -void TestData::validateContextLogits(bool getContextLogits, SizeType32 inputLength, SizeType32 beamWidth, - std::optional<executor::Tensor> const& contextLogits, SizeType32 vocabSizePadded, SizeType32 batchId, float atol, - float rtol) -{ - if (getContextLogits) - { - EXPECT_TRUE(contextLogits.has_value()) << "bid: " << batchId; - EXPECT_EQ(contextLogits.value().getShape().size(), 2); - EXPECT_EQ(contextLogits.value().getShape()[0], inputLength); - EXPECT_EQ(contextLogits.value().getShape()[1], vocabSizePadded); - auto const expectedContextLogits = this->expectedContextLogits[batchId]; - - if (beamWidth == 1) - { - cudaDeviceSynchronize(); // Make sure the logits copy is complete. - EXPECT_TRUE(compareLogits( - *expectedContextLogits, *(executor::detail::toITensor(contextLogits.value())), atol, rtol)); - } - } - else - { - EXPECT_FALSE(contextLogits.has_value()) << "bid: " << batchId; - } -} - -void TestData::validateGenerationLogits(bool getGenLogits, bool isFinal, bool streaming, bool excludeInputFromOutput, - SizeType32 inputLength, SizeType32 maxOutputLen, SizeType32 beamWidth, executor::BeamTokens const& beamTokens, - std::optional<executor::Tensor> const& genLogits, SizeType32 vocabSizePadded, SizeType32 batchId, - bool const returnAllGeneratedTokens, float atol, float rtol) -{ - auto const numReturnBeams = beamTokens.size(); - - if (getGenLogits) - { - EXPECT_TRUE(genLogits.has_value()) << "bid: " << batchId; - EXPECT_EQ(genLogits.value().getShape().size(), 3); - - // Expected generation logits - auto const& expectedGenerationLogits - = this->expectedGenerationLogits[batchId]; // [maxOutputLen, vocabSizePadded] - // Output generation logits - // 1. non-streaming: [beamWidth, maxOutputLen, vocabSizePadded] - // 2. streaming: [maxOutputLen (or 1), beamWidth, vocabSizePadded] - auto const& outputGenerationLogits = executor::detail::toITensor(genLogits.value()); - - if (streaming) - { - EXPECT_EQ(genLogits.value().getShape()[1], numReturnBeams); - EXPECT_EQ(beamWidth, 1); // Only support streaming && beamWidth == 1 - - SizeType32 const beamIdx = 0; - bool removeInput = !excludeInputFromOutput && !streaming; - // If returnAllGeneratedTokens, will contain duplicate tokens - auto const& numPredTokens = beamTokens.at(beamIdx).size() - (removeInput ? inputLength : 0); - - SizeType32 numGeneratedToken = genLogits.value().getShape()[0]; - if (returnAllGeneratedTokens) - { - EXPECT_EQ(numGeneratedToken, numPredTokens); - } - else - { - EXPECT_EQ(numGeneratedToken, 1); - } - SizeType32 sliceOffset = returnAllGeneratedTokens ? 0 : numPredTokens - 1; - - auto const& expectedGenerationLogitsSlice - = std::shared_ptr(ITensor::slice(expectedGenerationLogits, sliceOffset, - numGeneratedToken)); // [numGeneratedToken, vocabSizePadded] - - cudaDeviceSynchronize(); // Make sure the logits copy is complete. - EXPECT_TRUE(compareLogits(*expectedGenerationLogitsSlice, *outputGenerationLogits, atol, rtol)); - } - else - { - // Non-streaming - EXPECT_EQ(genLogits.value().getShape()[0], numReturnBeams); - EXPECT_EQ(genLogits.value().getShape()[1], maxOutputLen); - - if (isFinal && beamWidth == 1) - { - cudaDeviceSynchronize(); // Make sure the logits copy is complete. - EXPECT_TRUE(compareLogits(*expectedGenerationLogits, *outputGenerationLogits, atol, rtol)); - } - } - EXPECT_EQ(genLogits.value().getShape()[2], vocabSizePadded); - } - else - { - EXPECT_FALSE(genLogits.has_value()) << "bid: " << batchId; - } -} - -} // namespace tensorrt_llm::testing diff --git a/cpp/tests/utils/common.h b/cpp/tests/utils/common.h deleted file mode 100644 index f7b73a9acea4..000000000000 --- a/cpp/tests/utils/common.h +++ /dev/null @@ -1,352 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#ifndef TOP_LEVEL_DIR -#error "Define TOP_LEVEL_DIR" -#endif - -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/iTensor.h" - -#include <cmath> -#include <filesystem> -#include <random> -#include <set> -#include <string> -#include <utility> -#include <vector> - -namespace tensorrt_llm::testing -{ -namespace fs = std::filesystem; -namespace tr = tensorrt_llm::runtime; - -using tr::SizeType32; -using tr::TokenIdType; -using tr::ITensor; -using tr::MemoryType; - -auto const TEST_RESOURCE_PATH = fs::path{TOP_LEVEL_DIR} / "cpp/tests/resources"; - -auto const ENGINE_PATH = TEST_RESOURCE_PATH / "models/rt_engine"; -auto const GPT_MODEL_PATH = ENGINE_PATH / "gpt2"; -auto const LLAMA_MODEL_PATH = ENGINE_PATH / "Llama-3.2-1B"; -auto const MEDUSA_MODEL_PATH = ENGINE_PATH / "vicuna-7b-medusa"; -auto const CHATGLM_MODEL_PATH = ENGINE_PATH / "chatglm-6b"; -auto const CHATGLM2_MODEL_PATH = ENGINE_PATH / "chatglm2-6b"; -auto const CHATGLM3_MODEL_PATH = ENGINE_PATH / "chatglm3-6b"; -auto const GLM_MODEL_PATH = ENGINE_PATH / "glm-10b"; -auto const ENC_DEC_ENGINE_BASE = TEST_RESOURCE_PATH / "models/enc_dec/trt_engines"; - -auto const DATA_PATH = TEST_RESOURCE_PATH / "data"; -auto const GPT_DATA_PATH = DATA_PATH / "gpt2"; -auto const GPT_XGRAMMAR_TOKENIZER_INFO_PATH = GPT_DATA_PATH / "xgrammar_tokenizer_info.json"; -auto const LLAMA_DATA_PATH = DATA_PATH / "Llama-3.2-1B"; -auto const LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH = LLAMA_DATA_PATH / "xgrammar_tokenizer_info.json"; -auto const MEDUSA_DATA_PATH = DATA_PATH / "vicuna-7b-medusa"; -auto const CHATGLM_DATA_PATH = DATA_PATH / "chatglm-6b"; -auto const CHATGLM2_DATA_PATH = DATA_PATH / "chatglm2-6b"; -auto const CHATGLM3_DATA_PATH = DATA_PATH / "chatglm3-6b"; -auto const GLM_DATA_PATH = DATA_PATH / "glm-10b"; -auto const ENC_DEC_DATA_BASE = DATA_PATH / "enc_dec"; - -auto constexpr T5_NAME = "t5-small"; -auto constexpr BART_NAME = "bart-large-cnn"; -auto constexpr LANGUAGE_ADAPTER_NAME = "language_adapter-enc_dec_language_adapter"; - -class PathUtil -{ -public: - static std::string EXECUTOR_WORKER_PATH() - { - return (std::filesystem::path{TOP_LEVEL_DIR} / "cpp/build/tensorrt_llm/executor_worker/executorWorker") - .string(); - } - - // model paths - static std::string FP16_GPT_ATTENTION_PACKED_DIR(); - static std::string FP16_GPT_ATTENTION_PACKED_PAGED_DIR(); - static std::string FP16_GPT_LORA_DIR(); - static std::string FP16_GPT_ATTENTION_PACKED_PAGED_DRAFT_TOKENS_DIR(); - static std::string FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR(); - static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_LONG_RESULT_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); - // logits - static std::string FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_GATHER_CUM_LOG_PROBS_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_GATHER_LOG_PROBS_FILE(); - // results - static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP1_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP4_PP1_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP4_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP2_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP1_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_TP4_PP1_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_TP4_PP1_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_TP4_PP1_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_TP4_PP1_FILE(); - // GptExecutorTest.GenerationLogitsEarlyStop requires to use context_fmha_fp32_acc flag in runtime for better - // accuracy - static std::string FP16_PLUGIN_PACKED_PAGED_GATHER_CONTEXTFMHAFP32ACC_RESULT_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_CONTEXTFMHAFP32ACC_GENERATION_LOGITS_FILE(); - static std::string FP16_PLUGIN_PACKED_PAGED_CONTEXTFMHAFP32ACC_CONTEXT_LOGITS_FILE(); -}; - -class ModelIds -{ -public: - ModelIds() = default; - - constexpr ModelIds(TokenIdType endId, TokenIdType padId) - : endId{endId} - , padId{padId} - { - } - - TokenIdType endId{}; - TokenIdType padId{}; -}; - -class BeamResult -{ -public: - explicit BeamResult(SizeType32 beamWidth) - : beamWidth{beamWidth} {}; - - BeamResult(SizeType32 beamWidth, fs::path resultsFile, fs::path contextLogitsFile, fs::path genLogitsFile, - fs::path cumLogProbsFile, fs::path logProbsFile) - : beamWidth{beamWidth} - , resultsFile{std::move(resultsFile)} - , contextLogitsFile{std::move(contextLogitsFile)} - , genLogitsFile{std::move(genLogitsFile)} - , cumLogProbsFile{std::move(cumLogProbsFile)} - , logProbsFile{std::move(logProbsFile)} {}; - - SizeType32 beamWidth; - fs::path resultsFile; - - fs::path contextLogitsFile; - fs::path genLogitsFile; - - fs::path cumLogProbsFile; - fs::path logProbsFile; -}; - -using BeamResults = std::vector<BeamResult>; - -struct FlakyTestInfo -{ - // Pair of batch ID + beam which are flaky - std::set<std::pair<SizeType32, SizeType32>> batchIdBeams; -}; - -class TestData -{ -public: - explicit TestData(SizeType32 nbGivenInputs, SizeType32 beamWidth) - : nbGivenInputs{nbGivenInputs} - , beamWidth{beamWidth} - { - expectedOutputLengths.resize(nbGivenInputs * beamWidth); - - draftTokens.resize(nbGivenInputs); - draftLogits.resize(nbGivenInputs); - acceptedDraftTokensLengths.resize(nbGivenInputs); - expectedGenerationLogits.resize(nbGivenInputs); - expectedContextLogits.resize(nbGivenInputs); - expectedCumLogProbs.resize(nbGivenInputs); - expectedLogProbs.resize(nbGivenInputs); - } - - void loadLogProbs(fs::path const& cumLogProbsFile, fs::path const& logProbsFile, tr::BufferManager const& manager); - - void loadContextLogits(fs::path const& contextLogitsFile, std::vector<SizeType32> const& givenInputLengths, - tr::BufferManager const& manager); - void loadGenerationLogits(fs::path const& genLogitsFile, tr::BufferManager const& manager); - - void makeDraft(SizeType32 maxDraftTokens, bool acceptDraftByLogits, fs::path const& genLogitsFile, - std::vector<SizeType32> const& givenInputLengths, tr::BufferManager const& manager); - - static TestData loadTestData(BeamResult const& beamResults, ITensor const& givenInput, SizeType32 maxBeamWidth, - tr::BufferManager& manager, executor::OutputConfig const& outConfig, ModelIds const& modelIds); - - void verifyOutput(std::unordered_map<SizeType32, std::vector<executor::BeamTokens>> const& resultTokens, - std::vector<SizeType32> const& givenInputLengths, bool streaming, bool excludeInputFromOutput, - FlakyTestInfo flakyTestInfo, bool isSpeculativeDecoding, SizeType32 reqBeamWidth, SizeType32 numReturnSequences, - bool isNonGreedySampling); - - void verifyLogProbs(bool computeLogProbs, bool streaming, bool excludeInputFromOutput, SizeType32 inputLength, - SizeType32 beamWidth, executor::BeamTokens const& beamTokens, - std::optional<executor::VecLogProbs> const& cumLogProbs, - std::optional<std::vector<executor::VecLogProbs>> const& logProbs, SizeType32 batchId, - FlakyTestInfo flakyTestInfo); - - void validateContextLogits(bool getContextLogits, SizeType32 inputLength, SizeType32 beamWidth, - std::optional<executor::Tensor> const& contextLogits, SizeType32 vocabSizePadded, SizeType32 batchId, - float atol = 1e-2, float rtol = 1e-3); - - void validateGenerationLogits(bool getGenLogits, bool isFinal, bool streaming, bool excludeInputFromOutput, - SizeType32 inputLength, SizeType32 maxOutputLen, SizeType32 beamWidth, executor::BeamTokens const& beamTokens, - std::optional<executor::Tensor> const& genLogits, SizeType32 vocabSizePadded, SizeType32 batchId, - bool returnAllGeneratedTokens, float atol = 1e-2, float rtol = 1e-3); - - SizeType32 nbGivenInputs{}; - SizeType32 beamWidth{}; - SizeType32 maxSeqLen{}; - ITensor::SharedPtr expectedOutputIds; - std::vector<SizeType32> expectedOutputLengths; - std::vector<TokenIdType> endIds; - std::vector<tensorrt_llm::executor::VecTokens> draftTokens; - std::vector<ITensor::SharedPtr> draftLogits; - std::vector<SizeType32> acceptedDraftTokensLengths; - std::vector<ITensor::SharedPtr> expectedGenerationLogits; - std::vector<ITensor::SharedPtr> expectedContextLogits; - std::vector<ITensor::SharedPtr> expectedCumLogProbs; - std::vector<ITensor::SharedPtr> expectedLogProbs; -}; - -inline bool almostEqual(float a, float b, float atol = 1e-2, float rtol = 1e-3) -{ - // Params: a = value to compare and b = reference - // This function follows implementation of numpy.isclose(), which checks - // abs(a - b) <= (atol + rtol * abs(b)). - // Note that the inequality above is asymmetric where b is considered as - // a reference value. To account into both absolute/relative errors, it - // uses absolute tolerance and relative tolerance at the same time. The - // default values of atol and rtol borrowed from numpy.isclose(). For the - // case of nan value, the result will be true. - if (std::isnan(a) && std::isnan(b)) - { - return true; - } - return fabs(a - b) <= (atol + rtol * fabs(b)); -} - -bool compareLogits(ITensor const& groundTruthLogits, ITensor const& outputLogits, float atol = 1e-2, float rtol = 1e-3); - -std::tuple<SizeType32, SizeType32> getRequestGivenInputIdxLength( - std::uint64_t requestId, SizeType32 nbGivenInputs, std::vector<SizeType32> const& givenInputLengths); - -std::tuple<std::vector<SizeType32>, SizeType32, SizeType32> getGivenInputLengths( - ITensor const& givenInput, SizeType32 padId); - -/// @brief Generates a vector of floating point values summing to 1, that can be used as logits. -/// -/// @tparam TEngine The type of the random engine. -/// @tparam TLogits The type of floating point values. -/// @param vocabSize The vocabulary size, i.e. the size of the vector. -/// @param engine A random engine. -/// @return std::vector<TLogits> A vector of floating point values, summing to 1. -template <typename TEngine, typename TLogits> -std::vector<TLogits> randomLogits(runtime::SizeType32 vocabSize, TEngine* engine) -{ - if constexpr (std::disjunction_v<std::is_floating_point<TLogits>, std::is_same<TLogits, half>>) - { - // This algorithm ensures the resulting values sum to 1 by: - // 1. Sampling in the interval 0..1 - // 2. Sorting the sampled values and adding a last value equal to 1 - // 3. Calculating the adjacent differences of the sorted values - // Since the values are sorted and the last value is 1, we get that all the differences are positive and must - // sum to 1. It can be proven recursively by seeing that the first value sums to itself, and the n-1 first - // values must sum to the value at n, minus the difference between the n-th and n-1-th values. - // It is also helpful to convince yourself of it with a quick drawing. - auto distribution = std::uniform_real_distribution<float>(0, 1); - std::vector<float> samples(vocabSize); - samples.back() = 1.0; - std::transform(samples.begin(), samples.end() - 1, samples.begin(), - [&](auto const /*i*/) { return distribution(*engine); }); - std::sort(samples.begin(), samples.end() - 1); - std::vector<float> result(vocabSize); - std::adjacent_difference(samples.begin(), samples.end(), result.begin()); - if constexpr (std::is_same_v<TLogits, float>) - { - return result; - } - - if constexpr (std::is_same_v<TLogits, half>) - { - std::vector<half> halfResults(vocabSize); - std::transform( - result.begin(), result.end(), halfResults.begin(), [&](auto const f) { return __float2half(f); }); - return halfResults; - } - } - TLLM_THROW("Unsupported logits type."); -} - -std::vector<tensorrt_llm::executor::TokenIdType> createConsecutiveTokenSequence( - tr::SizeType32 length, tr::SizeType32 vocabSize, tr::TokenIdType firstTokenId); - -/** - * GPU timer for recording the elapsed time across kernel(s) launched in GPU stream - */ -struct GpuTimer -{ - cudaStream_t _stream_id; - cudaEvent_t _start; - cudaEvent_t _stop; - - /// Construct`or - GpuTimer() - : _stream_id(0) - { - TLLM_CUDA_CHECK(cudaEventCreate(&_start)); - TLLM_CUDA_CHECK(cudaEventCreate(&_stop)); - } - - /// Destructor - ~GpuTimer() - { - TLLM_CUDA_CHECK(cudaEventDestroy(_start)); - TLLM_CUDA_CHECK(cudaEventDestroy(_stop)); - } - - /// Start the timer for a given stream (defaults to the default stream) - void start(cudaStream_t stream_id = 0) - { - _stream_id = stream_id; - TLLM_CUDA_CHECK(cudaEventRecord(_start, _stream_id)); - } - - /// Stop the timer - void stop() - { - TLLM_CUDA_CHECK(cudaEventRecord(_stop, _stream_id)); - } - - /// Return the elapsed time (in milliseconds) - float elapsed_millis() - { - float elapsed = 0.0; - TLLM_CUDA_CHECK(cudaEventSynchronize(_stop)); - TLLM_CUDA_CHECK(cudaEventElapsedTime(&elapsed, _start, _stop)); - return elapsed; - } -}; - -} // namespace tensorrt_llm::testing diff --git a/cpp/tests/utils/engines.cpp b/cpp/tests/utils/engines.cpp deleted file mode 100644 index 28f0d9b15935..000000000000 --- a/cpp/tests/utils/engines.cpp +++ /dev/null @@ -1,97 +0,0 @@ -#include "engines.h" - -#include "tensorrt_llm/batch_manager/transformerBuffers.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/tllmLogger.h" - -#include <NvInfer.h> -#include <algorithm> -#include <memory> - -nvinfer1::ITensor& tensorrt_llm::testing::utils::engines::details::addInputIds( - EngineBuildState& buildState, runtime::SizeType32 maxNumTokens) -{ - auto* input_ids = buildState.networkDefinition->addInput(batch_manager::RuntimeBuffers::kInputIdsTensorName, - nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({-1})); - buildState.tensors.push_back(input_ids); - buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kInputIdsTensorName, - nvinfer1::OptProfileSelector::kMAX, runtime::ITensor::makeShape({maxNumTokens})); - buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kInputIdsTensorName, - nvinfer1::OptProfileSelector::kOPT, runtime::ITensor::makeShape({maxNumTokens / 2})); - buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kInputIdsTensorName, - nvinfer1::OptProfileSelector::kMIN, runtime::ITensor::makeShape({1})); - return *input_ids; -} - -nvinfer1::ITensor* tensorrt_llm::testing::utils::engines::details::addLastTokenIds( - EngineBuildState& buildState, runtime::SizeType32 maxBatchSize, runtime::SizeType32 maxBeamWidth) -{ - auto* last_token_ids - = buildState.networkDefinition->addInput(batch_manager::RuntimeBuffers::kLastTokenIdsTensorName, - nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({-1})); - buildState.tensors.push_back(last_token_ids); - buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kLastTokenIdsTensorName, - nvinfer1::OptProfileSelector::kMAX, runtime::ITensor::makeShape({maxBatchSize * maxBeamWidth})); - buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kLastTokenIdsTensorName, - nvinfer1::OptProfileSelector::kOPT, runtime::ITensor::makeShape({maxBatchSize * maxBeamWidth / 2})); - buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kLastTokenIdsTensorName, - nvinfer1::OptProfileSelector::kMIN, runtime::ITensor::makeShape({1})); - return last_token_ids; -} - -nvinfer1::ITensor& tensorrt_llm::testing::utils::engines::details::addKvCacheOffsets(EngineBuildState& buildState, - runtime::SizeType32 numPools, runtime::SizeType32 tokensPerBlock, runtime::SizeType32 maxBatchSize, - runtime::SizeType32 maxNumTokens, runtime::SizeType32 maxBeamWidth) -{ - auto* kvCacheOffsets = buildState.networkDefinition->addInput( - batch_manager::TransformerBuffers::kKvCacheBlockOffsetsTensorName, nvinfer1::DataType::kINT32, - runtime::ITensor::makeShape({numPools, -1, 2, -1})); // [numPools, maxBatch * maxBeamWidth, 2, maxBlocksPerSeq] - buildState.tensors.push_back(kvCacheOffsets); - auto const maxBlocksPerSeq = maxNumTokens / tokensPerBlock; - buildState.profile->setDimensions(batch_manager::TransformerBuffers::kKvCacheBlockOffsetsTensorName, - nvinfer1::OptProfileSelector::kMAX, - runtime::ITensor::makeShape({numPools, maxBatchSize * maxBeamWidth, 2, maxBlocksPerSeq})); - buildState.profile->setDimensions(batch_manager::TransformerBuffers::kKvCacheBlockOffsetsTensorName, - nvinfer1::OptProfileSelector::kOPT, - runtime::ITensor::makeShape({numPools, maxBatchSize * maxBeamWidth / 2, 2, maxBlocksPerSeq / 2})); - buildState.profile->setDimensions(batch_manager::TransformerBuffers::kKvCacheBlockOffsetsTensorName, - nvinfer1::OptProfileSelector::kMIN, runtime::ITensor::makeShape({numPools, 1, 2, 1})); - return *kvCacheOffsets; -} - -tensorrt_llm::testing::utils::engines::details::EngineBuildState -tensorrt_llm::testing::utils::engines::initializeEngineBuild(std::shared_ptr<runtime::TllmLogger> const& logger) -{ - auto* builder = nvinfer1::createInferBuilder(*logger); - auto* profile = builder->createOptimizationProfile(); - auto* network = builder->createNetworkV2( - 1U << static_cast<uint32_t>(nvinfer1::NetworkDefinitionCreationFlag::kSTRONGLY_TYPED)); - nvinfer1::IBuilderConfig* config = builder->createBuilderConfig(); - return {builder, network, profile, config}; -} - -nvinfer1::ITensor& tensorrt_llm::testing::utils::engines::details::addSingleOutputLayer( - tensorrt_llm::testing::utils::engines::details::EngineBuildState& buildState, nvinfer1::ILayer* layer) -{ - buildState.layers.push_back(layer); - auto* output = layer->getOutput(0); - buildState.tensors.push_back(output); - TLLM_LOG_INFO("Adding layer %s with output shape %s.", layer->getName(), - tensorrt_llm::runtime::ITensor::toString(output->getDimensions()).c_str()); - - return *output; -} - -tensorrt_llm::common::OptionalRef<nvinfer1::ITensor> tensorrt_llm::testing::utils::engines::details::getTensorByName( - tensorrt_llm::testing::utils::engines::details::EngineBuildState& buildState, std::string_view name) -{ - auto result = std::find_if(buildState.tensors.begin(), buildState.tensors.end(), - [name](auto const tensor) { return tensor->getName() == name; }); - if (result == buildState.tensors.end()) - { - return tensorrt_llm::common::OptionalRef<nvinfer1::ITensor>{}; - } - return **result; -} diff --git a/cpp/tests/utils/engines.h b/cpp/tests/utils/engines.h deleted file mode 100644 index 54b53e5c6097..000000000000 --- a/cpp/tests/utils/engines.h +++ /dev/null @@ -1,356 +0,0 @@ -#ifndef CA1B91B5_DF64_4CF8_948F_5AFF243A2555 -#define CA1B91B5_DF64_4CF8_948F_5AFF243A2555 - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/optionalRef.h" -#include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/runtime/tllmLogger.h" -#include <NvInfer.h> -#include <NvInferRuntime.h> -#include <algorithm> -#include <cstddef> -#include <memory> -#include <tensorrt_llm/batch_manager/runtimeBuffers.h> -#include <utility> -#include <vector> - -namespace tensorrt_llm::testing::utils::engines -{ - -namespace details -{ - -struct EngineBuildResource -{ - EngineBuildResource() = default; - virtual ~EngineBuildResource() = default; - EngineBuildResource(EngineBuildResource const& vector) = default; - EngineBuildResource& operator=(EngineBuildResource const& vector) = default; - EngineBuildResource(EngineBuildResource&& vector) noexcept = default; - EngineBuildResource& operator=(EngineBuildResource&& vector) noexcept = default; -}; - -template <typename TValue> -struct Vector : public EngineBuildResource -{ - explicit Vector(std::vector<TValue> values) - : values(std::move(values)){}; - Vector(Vector const& vector) = default; - Vector& operator=(Vector const& vector) = default; - Vector(Vector&& vector) noexcept = default; - Vector& operator=(Vector&& vector) noexcept = default; - ~Vector() override = default; - std::vector<TValue> values; -}; - -template <typename TValue, size_t Size> -struct Array : public EngineBuildResource -{ - explicit Array(std::array<TValue, Size> values) - : values(std::move(values)){}; - Array(Array const& vector) = default; - Array& operator=(Array const& vector) = default; - Array(Array&& vector) noexcept = default; - Array& operator=(Array&& vector) noexcept = default; - ~Array() override = default; - std::array<TValue, Size> values; -}; - -struct EngineBuildState -{ - EngineBuildState(nvinfer1::IBuilder* builder, nvinfer1::INetworkDefinition* networkDefinition, - nvinfer1::IOptimizationProfile* profile, nvinfer1::IBuilderConfig* builderConfig) - : builder(builder) - , networkDefinition(networkDefinition) - , profile(profile) - , builderConfig(builderConfig){}; - EngineBuildState(EngineBuildState const& vector) = delete; - EngineBuildState& operator=(EngineBuildState const& vector) = delete; - EngineBuildState(EngineBuildState&& vector) noexcept = default; - EngineBuildState& operator=(EngineBuildState&& vector) noexcept = default; - std::unique_ptr<nvinfer1::IBuilder> builder; - std::unique_ptr<nvinfer1::INetworkDefinition> networkDefinition; - nvinfer1::IOptimizationProfile* profile; - - // While building the engine, one might need some data for weights and such. Turns out, TensorRT does not keep a - // copy of those, so if you create them as temporaries and pass them to the TRT APIs, you will get UB. So we need - // some place where we can keep those things. - std::unique_ptr<nvinfer1::IBuilderConfig> builderConfig; - std::vector<std::unique_ptr<EngineBuildResource>> resources; - std::vector<nvinfer1::ITensor*> tensors; - std::vector<nvinfer1::ILayer*> layers; - - ~EngineBuildState() - { - // Builder needs to be deleteds last. - networkDefinition.reset(); - builderConfig.reset(); - builder.reset(); - } -}; - -common::OptionalRef<nvinfer1::ITensor> getTensorByName(EngineBuildState& buildState, std::string_view name); - -nvinfer1::ITensor& addSingleOutputLayer(EngineBuildState& buildState, nvinfer1::ILayer* layer); - -template <typename TResource> -TResource& addResource(EngineBuildState& buildState, TResource resource) -{ - return *dynamic_cast<TResource*>( - buildState.resources.emplace_back(std::make_unique<TResource>(std::move(resource))).get()); -} - -template <typename TValue> -Vector<TValue>& addSingleConstantVectorResource(EngineBuildState& buildState, TValue value, std::size_t length) -{ - std::vector<TValue> weights(length); - std::fill(weights.begin(), weights.end(), value); - return addResource(buildState, Vector<TValue>{weights}); -} - -template <typename TValue> -Vector<TValue>& addConstantVectorResource(EngineBuildState& buildState, std::vector<TValue> values) -{ - return addResource(buildState, Vector<TValue>{values}); -} - -template <typename TValue> -Array<TValue, 1>& addConstantScalarResource(EngineBuildState& buildState, TValue value) -{ - return addResource(buildState, Array<TValue, 1>{{value}}); -} - -nvinfer1::ITensor& addInputIds(EngineBuildState& buildState, runtime::SizeType32 maxNumTokens); -nvinfer1::ITensor* addLastTokenIds( - EngineBuildState& buildState, runtime::SizeType32 maxBatchSize, runtime::SizeType32 maxBeamWidth); -nvinfer1::ITensor& addKvCacheOffsets(EngineBuildState& buildState, runtime::SizeType32 numPools, - runtime::SizeType32 tokensPerBlock, runtime::SizeType32 maxBatchSize, runtime::SizeType32 maxNumTokens, - runtime::SizeType32 maxBeamWidth); - -template <typename TValue> -nvinfer1::ITensor& addSingleConstantVector(EngineBuildState& buildState, TValue value, runtime::SizeType32 length) -{ - auto& resourceWeights = addSingleConstantVectorResource(buildState, value, length); - auto const trtDatatype = runtime::TRTDataType<TValue>::value; - auto* layer = buildState.networkDefinition->addConstant(runtime::ITensor::makeShape({length}), - {trtDatatype, resourceWeights.values.data(), static_cast<runtime::ITensor::DimType64>(length)}); - return addSingleOutputLayer(buildState, layer); -} - -template <typename TValue> -nvinfer1::ITensor& addSingleConstantTensor(EngineBuildState& buildState, TValue value, runtime::SizeType32 length) -{ - auto& resourceWeights = addSingleConstantVectorResource(buildState, value, length); - auto const trtDatatype = runtime::TRTDataType<TValue>::value; - auto* layer = buildState.networkDefinition->addConstant(runtime::ITensor::makeShape({1, length}), - {trtDatatype, resourceWeights.values.data(), static_cast<runtime::ITensor::DimType64>(length)}); - return addSingleOutputLayer(buildState, layer); -} - -template <typename TValue> -nvinfer1::ITensor& addConstantVector(EngineBuildState& buildState, std::vector<TValue> values) -{ - auto& resourceWeights = addConstantVectorResource(buildState, values); - auto const trtDatatype = runtime::TRTDataType<TValue>::value; - auto const length = static_cast<runtime::ITensor::DimType64>(values.size()); - auto* layer = buildState.networkDefinition->addConstant( - runtime::ITensor::makeShape({length}), {trtDatatype, resourceWeights.values.data(), length}); - return addSingleOutputLayer(buildState, layer); -} - -template <typename TValue> -nvinfer1::ITensor& addConstantTensor( - EngineBuildState& buildState, std::vector<TValue> values, runtime::ITensor::Shape shape) -{ - auto& resourceWeights = addConstantVectorResource(buildState, values); - auto const trtDatatype = runtime::TRTDataType<TValue>::value; - auto const count = runtime::ITensor::volume(shape); - auto* layer = buildState.networkDefinition->addConstant(shape, {trtDatatype, resourceWeights.values.data(), count}); - return addSingleOutputLayer(buildState, layer); -} - -template <typename TValue> -nvinfer1::ITensor& addSingleConstantTensor(EngineBuildState& buildState, TValue value, runtime::ITensor::Shape shape) -{ - auto const count = runtime::ITensor::volume(shape); - auto& resourceWeights = addSingleConstantVectorResource(buildState, value, count); - auto const trtDatatype = runtime::TRTDataType<TValue>::value; - auto* layer = buildState.networkDefinition->addConstant(shape, {trtDatatype, resourceWeights.values.data(), count}); - return addSingleOutputLayer(buildState, layer); -} - -template <typename TValue> -nvinfer1::ITensor& addConstantScalar(EngineBuildState& buildState, TValue value) -{ - auto& resourceWeights = addConstantScalarResource<TValue>(buildState, value); - auto const trtDatatype = runtime::TRTDataType<TValue>::value; - auto* layer = buildState.networkDefinition->addConstant( - runtime::ITensor::makeShape({}), {trtDatatype, resourceWeights.values.data(), 1}); - return addSingleOutputLayer(buildState, layer); -} - -template <typename TValue> -nvinfer1::ITensor& oneHotEncode( - EngineBuildState& buildState, nvinfer1::ITensor& inputIds, runtime::SizeType32 vocabSize) -{ - auto const trtValueType = runtime::TRTDataType<TValue>::value; - auto& oneHotValues = addConstantVector<TValue>(buildState, {0, 1}); - auto& oneHotDepth = addConstantScalar(buildState, vocabSize); - auto* oneHotLayer = buildState.networkDefinition->addOneHot(inputIds, oneHotValues, oneHotDepth, 0); - return addSingleOutputLayer(buildState, oneHotLayer); -} -} // namespace details - -struct TrivialDecoderParameters -{ - TrivialDecoderParameters(runtime::SizeType32 vocabSize, runtime::SizeType32 maxBatchSize, - runtime::SizeType32 maxNumTokens, runtime::SizeType32 tokensPerBlock, runtime::SizeType32 maxBeamWidth, - bool gatherContextLogits) - : vocabSize(vocabSize) - , maxBatchSize(maxBatchSize) - , maxNumTokens(maxNumTokens) - , tokensPerBlock(tokensPerBlock) - , maxBeamWidth(maxBeamWidth) - , gatherContextLogits(gatherContextLogits){}; - runtime::SizeType32 vocabSize; - runtime::SizeType32 maxBatchSize; - runtime::SizeType32 maxNumTokens; - runtime::SizeType32 tokensPerBlock; - runtime::SizeType32 maxBeamWidth; - bool gatherContextLogits; -}; - -details::EngineBuildState initializeEngineBuild(std::shared_ptr<runtime::TllmLogger> const& logger); - -template <typename TLogits> -std::unique_ptr<nvinfer1::IHostMemory> createTrivialDecoder( - TrivialDecoderParameters parameters, std::shared_ptr<runtime::TllmLogger> const& logger) -{ - auto const trtLogitsType = runtime::TRTDataType<TLogits>::value; - auto buildState = initializeEngineBuild(logger); - auto* builder = buildState.builder.get(); - auto* profile = buildState.profile; - auto* network = buildState.networkDefinition.get(); - auto& inputIds = details::addInputIds(buildState, parameters.maxNumTokens); - auto& kvCacheOffsets = details::addKvCacheOffsets(buildState, 1, parameters.tokensPerBlock, parameters.maxBatchSize, - parameters.maxNumTokens, parameters.maxBeamWidth); - - auto& oneHotLayerOutput = details::oneHotEncode<TLogits>(buildState, inputIds, parameters.vocabSize); - oneHotLayerOutput.setName(batch_manager::RuntimeBuffers::kLogitsTensorName); - network->markOutput(oneHotLayerOutput); - - buildState.builderConfig->addOptimizationProfile(profile); - buildState.builderConfig->setProfilingVerbosity(nvinfer1::ProfilingVerbosity::kDETAILED); - auto* engine = builder->buildSerializedNetwork(*network, *buildState.builderConfig); - return std::unique_ptr<nvinfer1::IHostMemory>(engine); -} - -template <typename TLogits> -struct ConstantTrivialDecoderParameters -{ - ConstantTrivialDecoderParameters(TrivialDecoderParameters trivialDecoderParameters, std::vector<TLogits> logits) - : trivialDecoderParameters(trivialDecoderParameters) - , logits(logits) - { - auto const sizeTypeVocabSize = static_cast<std::size_t>(trivialDecoderParameters.vocabSize); - auto const logitsSize = logits.size(); - TLLM_CHECK_WITH_INFO(static_cast<std::size_t>(trivialDecoderParameters.vocabSize) == logits.size(), - "The size of the constant logits (%lu) has to be equal to the vocabulary size (%lu).", logitsSize, - sizeTypeVocabSize); - }; - - TrivialDecoderParameters trivialDecoderParameters; - std::vector<TLogits> logits; -}; - -template <typename TLogits> -details::EngineBuildState createConstantTrivialDecoderBase( - ConstantTrivialDecoderParameters<TLogits> parameters, std::shared_ptr<runtime::TllmLogger> const& logger) -{ - auto const trtLogitsType = runtime::TRTDataType<TLogits>::value; - auto buildState = initializeEngineBuild(logger); - auto* builder = buildState.builder.get(); - auto* profile = buildState.profile; - auto* network = buildState.networkDefinition.get(); - auto& inputIds = details::addInputIds(buildState, parameters.trivialDecoderParameters.maxNumTokens); - nvinfer1::ITensor* lastTokenIds = nullptr; - if (!parameters.trivialDecoderParameters.gatherContextLogits) - { - lastTokenIds = details::addLastTokenIds(buildState, parameters.trivialDecoderParameters.maxBatchSize, - parameters.trivialDecoderParameters.maxBeamWidth); - } - auto& kvCacheOffsets = details::addKvCacheOffsets(buildState, 1, parameters.trivialDecoderParameters.tokensPerBlock, - parameters.trivialDecoderParameters.maxBatchSize, parameters.trivialDecoderParameters.maxNumTokens, - parameters.trivialDecoderParameters.maxBeamWidth); - - auto const vocabSize = static_cast<runtime::ITensor::DimType64>(parameters.logits.size()); - - auto& constantLogitsPerToken = details::addConstantTensor<TLogits>( - buildState, parameters.logits, runtime::ITensor::makeShape({vocabSize, 1})); - auto& oneHotLayerOutput - = details::oneHotEncode<TLogits>(buildState, inputIds, parameters.trivialDecoderParameters.vocabSize); - auto& ones = details::addSingleConstantTensor<TLogits>(buildState, 1, runtime::ITensor::makeShape({1, vocabSize})); - auto* intermediateLayer1 = network->addMatrixMultiply( - ones, nvinfer1::MatrixOperation::kNONE, oneHotLayerOutput, nvinfer1::MatrixOperation::kNONE); - auto* intermediateLayer1Output = intermediateLayer1->getOutput(0); - - nvinfer1::ITensor* gatherLayerOutput = nullptr; - if (!parameters.trivialDecoderParameters.gatherContextLogits) - { - auto& one = details::addSingleConstantTensor<int32_t>(buildState, 1, runtime::ITensor::makeShape({1})); - auto* lastTokenIdsMinus1Layer - = network->addElementWise(*lastTokenIds, one, nvinfer1::ElementWiseOperation::kSUB); - auto* gatherLayer = network->addGather(*intermediateLayer1Output, *lastTokenIdsMinus1Layer->getOutput(0), 1); - gatherLayerOutput = gatherLayer->getOutput(0); - } - else - { - gatherLayerOutput = intermediateLayer1Output; - } - - auto* constLogitsLayer = network->addMatrixMultiply(*gatherLayerOutput, nvinfer1::MatrixOperation::kTRANSPOSE, - constantLogitsPerToken, nvinfer1::MatrixOperation::kTRANSPOSE); - auto* outputLogits = constLogitsLayer->getOutput(0); - network->markOutput(*outputLogits); - outputLogits->setName(batch_manager::RuntimeBuffers::kLogitsTensorName); - buildState.tensors.push_back(outputLogits); - return buildState; -} - -template <typename TLogits> -std::unique_ptr<nvinfer1::IHostMemory> createConstantTrivialDecoder( - ConstantTrivialDecoderParameters<TLogits> parameters, std::shared_ptr<runtime::TllmLogger> const& logger) -{ - auto buildState = createConstantTrivialDecoderBase<TLogits>(parameters, logger); - buildState.builderConfig->addOptimizationProfile(buildState.profile); - buildState.builderConfig->setProfilingVerbosity(nvinfer1::ProfilingVerbosity::kDETAILED); - auto* engine = buildState.builder->buildSerializedNetwork(*buildState.networkDefinition, *buildState.builderConfig); - return std::unique_ptr<nvinfer1::IHostMemory>(engine); -} - -template <typename TLogits> -std::unique_ptr<nvinfer1::IHostMemory> createConstantTrivialDecoderWithTopKLogits( - ConstantTrivialDecoderParameters<TLogits> parameters, runtime::SizeType32 numTopLogits, std::string_view outputName, - std::shared_ptr<runtime::TllmLogger> const& logger) -{ - auto buildState = createConstantTrivialDecoderBase<TLogits>(parameters, logger); - auto logits = details::getTensorByName(buildState, batch_manager::RuntimeBuffers::kLogitsTensorName); - TLLM_CHECK_WITH_INFO(static_cast<bool>(logits), - "You can only add topk logits on top of a network which contains a tensor named %s", - batch_manager::RuntimeBuffers::kLogitsTensorName); - auto* topKLayer = buildState.networkDefinition->addTopK( - logits.value(), nvinfer1::TopKOperation::kMAX, numTopLogits, 1UL << 1UL); - auto* topKLayerOutput = topKLayer->getOutput(0); - topKLayerOutput->setName(outputName.data()); - buildState.networkDefinition->markOutput(*topKLayerOutput); - auto* profile = buildState.profile; - buildState.builderConfig->addOptimizationProfile(profile); - buildState.builderConfig->setProfilingVerbosity(nvinfer1::ProfilingVerbosity::kDETAILED); - auto* engine = buildState.builder->buildSerializedNetwork(*buildState.networkDefinition, *buildState.builderConfig); - return std::unique_ptr<nvinfer1::IHostMemory>(engine); -} -} // namespace tensorrt_llm::testing::utils::engines - -#endif /* CA1B91B5_DF64_4CF8_948F_5AFF243A2555 */ diff --git a/cpp/tests/utils/executorUtils.cpp b/cpp/tests/utils/executorUtils.cpp deleted file mode 100644 index 6dbd6223ef19..000000000000 --- a/cpp/tests/utils/executorUtils.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include "executorUtils.h" -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include <exception> -#include <future> - -std::unordered_map<tensorrt_llm::batch_manager::RequestIdType, std::vector<tensorrt_llm::executor::Response>> -tensorrt_llm::testing::runThroughRequests(executor::Executor& executor, std::vector<executor::Request> const& requests, - std::chrono::duration<float, std::milli> timeout) -{ - std::unordered_map<batch_manager::RequestIdType, std::vector<executor::Response>> accumulatedResponses; - auto responseReadFuture = std::async(std::launch::async, - [&]() -> std::optional<std::exception> - { - auto remainingRequests = requests.size(); - try - { - while (remainingRequests > 0) - { - auto const responses = executor.awaitResponses(); - for (auto const& response : responses) - { - auto const requestId = response.getRequestId(); - if (response.hasError()) - { - TLLM_LOG_ERROR("Error response received for request: %lu", requestId); - TLLM_THROW(response.getErrorMsg()); - } - auto const isFinal = response.hasError() || response.getResult().isFinal; - accumulatedResponses[requestId].emplace_back(response); - if (isFinal) - { - TLLM_LOG_DEBUG("Final response received for request: %lu", requestId); - --remainingRequests; - } - } - } - - return std::nullopt; - } - catch (std::exception const& e) - { - TLLM_LOG_EXCEPTION(e); - return e; - } - }); - auto const requestIds = executor.enqueueRequests(requests); - responseReadFuture.wait_for(timeout); - auto const readResult = responseReadFuture.get(); - if (readResult.has_value()) - { - throw std::exception(readResult.value()); - } - return accumulatedResponses; -} diff --git a/cpp/tests/utils/executorUtils.h b/cpp/tests/utils/executorUtils.h deleted file mode 100644 index 97f154ca275d..000000000000 --- a/cpp/tests/utils/executorUtils.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef A073F2DA_315E_434B_B811_D420F0A59DF3 -#define A073F2DA_315E_434B_B811_D420F0A59DF3 - -#include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/executor/executor.h" -#include <ratio> -#include <unordered_map> - -namespace tensorrt_llm::testing -{ - -std::unordered_map<batch_manager::RequestIdType, std::vector<executor::Response>> runThroughRequests( - executor::Executor& executor, std::vector<executor::Request> const& requests, - std::chrono::duration<float, std::milli> timeout); - -} // namespace tensorrt_llm::testing - -#endif /* A073F2DA_315E_434B_B811_D420F0A59DF3 */ diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index fa01610fafd1..828859395043 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -31,7 +31,6 @@ FROM base AS devel # NB: PyTorch requires this to be < 1.0 ENV PYTORCH_ALLOC_CONF="garbage_collection_threshold:0.99999" -ARG TRT_VER ARG CUDA_VER ARG CUDNN_VER ARG NCCL_VER @@ -45,18 +44,16 @@ RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \ echo "Using Python version: ${PYTHON_VERSION}" && \ GITHUB_MIRROR=${GITHUB_MIRROR} \ PYTHON_VERSION=${PYTHON_VERSION} \ - TRT_VER=${TRT_VER} CUDA_VER=${CUDA_VER} CUDNN_VER=${CUDNN_VER} \ + CUDA_VER=${CUDA_VER} CUDNN_VER=${CUDNN_VER} \ NCCL_VER=${NCCL_VER} CUBLAS_VER=${CUBLAS_VER} \ TORCH_INSTALL_TYPE=${TORCH_INSTALL_TYPE} \ bash /opt/docker/common/install.sh --base --cmake --ccache --cuda_toolkit \ - --tensorrt --polygraphy --mpi4py --pytorch + --cuda_libs --polygraphy --mpi4py --pytorch # Install constraints after install.sh so cleanup() doesn't delete the file mid-RUN COPY constraints.txt /tmp/constraints.txt RUN --mount=type=cache,target=/root/.cache/pip \ - # WAR: uninstall dependencies that has vulnerability or need upgrading. - apt remove python3-wheel \ - pip3 uninstall -y tornado black nbconvert pillow nvidia-cutlass-dsl nvidia-cutlass-dsl-libs-base numpy || true && \ + pip3 uninstall -y tornado black nbconvert pillow nvidia-cutlass-dsl nvidia-cutlass-dsl-libs-base numpy wandb || true && \ # Remove any leftover namespace dirs or dist-info that pip missed rm -rf $(python3 -c "import site; print(site.getsitepackages()[0])")/nvidia_cutlass_dsl* && \ pip3 install -r /tmp/constraints.txt && \ @@ -107,7 +104,6 @@ RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \ FROM ${DEVEL_IMAGE} AS wheel WORKDIR /src/tensorrt_llm -COPY benchmarks benchmarks COPY cpp cpp COPY docker docker COPY scripts scripts @@ -119,7 +115,7 @@ COPY .gitmodules setup.py requirements.txt requirements-dev.txt constraints.txt ENV CCACHE_DIR=/root/.cache/ccache # Build the TRT-LLM wheel ARG GITHUB_MIRROR="" -ARG BUILD_WHEEL_ARGS="--clean --benchmarks" +ARG BUILD_WHEEL_ARGS="--clean" ARG BUILD_WHEEL_SCRIPT="scripts/build_wheel.py" RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=${CCACHE_DIR} \ GITHUB_MIRROR=$GITHUB_MIRROR python3 ${BUILD_WHEEL_SCRIPT} ${BUILD_WHEEL_ARGS} @@ -128,15 +124,15 @@ FROM ${DEVEL_IMAGE} AS release WORKDIR /app/tensorrt_llm RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,from=wheel,source=/src/tensorrt_llm/build,target=/tmp/wheel \ - pip install /tmp/wheel/tensorrt_llm*.whl + TRTLLM_WHEEL=$(find /tmp/wheel -maxdepth 1 -name 'tensorrt_llm*.whl' -print -quit) && \ + test -n "${TRTLLM_WHEEL}" && \ + pip install "${TRTLLM_WHEEL}[mx]" RUN --mount=type=bind,source=README.md,target=/mnt/ctx/README.md \ --mount=type=bind,source=docs,target=/mnt/ctx/docs \ --mount=type=bind,source=cpp/include,target=/mnt/ctx/include \ --mount=type=bind,source=examples,target=/mnt/ctx/examples \ --mount=type=bind,from=wheel,source=/src/tensorrt_llm/build,target=/mnt/wheel \ - --mount=type=bind,from=wheel,source=/src/tensorrt_llm/benchmarks,target=/mnt/benchmarks \ - --mount=type=bind,from=wheel,source=/src/tensorrt_llm/cpp/build/benchmarks,target=/mnt/cpp_benchmarks \ # Copy build context files cp /mnt/ctx/README.md ./ && \ cp -r /mnt/ctx/docs ./docs && \ @@ -145,25 +141,12 @@ RUN --mount=type=bind,source=README.md,target=/mnt/ctx/README.md \ chmod -R a+w examples && \ # Copy wheel stage outputs cp /mnt/wheel/tensorrt_llm*.whl ./ && \ - cp -r /mnt/benchmarks ./benchmarks && \ - mkdir -p benchmarks/cpp && \ - cp /mnt/cpp_benchmarks/bertBenchmark \ - /mnt/cpp_benchmarks/gptManagerBenchmark \ - /mnt/cpp_benchmarks/disaggServerBenchmark \ - benchmarks/cpp/ && \ - rm -v \ - benchmarks/cpp/bertBenchmark.cpp \ - benchmarks/cpp/gptManagerBenchmark.cpp \ - benchmarks/cpp/disaggServerBenchmark.cpp \ - benchmarks/cpp/CMakeLists.txt && \ - # Create symlinks to installed package binaries and libraries - ln -sv $(python3 -c 'import site; print(f"{site.getsitepackages()[0]}/tensorrt_llm/bin")') bin && \ - test -f bin/executorWorker && \ + # Create a symlink to installed package libraries ln -sv $(python3 -c 'import site; print(f"{site.getsitepackages()[0]}/tensorrt_llm/libs")') lib && \ - test -f lib/libnvinfer_plugin_tensorrt_llm.so && \ + test -f lib/libtensorrt_llm.so && \ echo "/app/tensorrt_llm/lib" > /etc/ld.so.conf.d/tensorrt_llm.conf && \ ldconfig && \ - ! ( ldd -v bin/executorWorker | grep tensorrt_llm | grep -q "not found" ) && \ + ! ( ldd -v lib/libth_common.so | grep tensorrt_llm | grep -q "not found" ) && \ # Clean up caches and CVE workarounds rm -rf /root/.cache/uv/archive-v0 && \ # WAR against https://github.com/advisories/GHSA-58pv-8j8x-9vj2 diff --git a/docker/Makefile b/docker/Makefile index 0ca78eef80a0..7c8568ffe54c 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -47,7 +47,6 @@ CUDA_VERSION ?= CUDNN_VERSION ?= NCCL_VERSION ?= CUBLAS_VERSION ?= -TRT_VERSION ?= GIT_COMMIT ?= $(shell git rev-parse HEAD) TRT_LLM_VERSION ?= $(shell grep '^__version__' ../tensorrt_llm/version.py | grep -o '=.*' | tr -d '= "') GITHUB_MIRROR ?= @@ -99,7 +98,6 @@ base_pull: $(if $(CUDNN_VERSION), --build-arg CUDNN_VER="$(CUDNN_VERSION)") \ $(if $(NCCL_VERSION), --build-arg NCCL_VER="$(NCCL_VERSION)") \ $(if $(CUBLAS_VERSION), --build-arg CUBLAS_VER="$(CUBLAS_VERSION)") \ - $(if $(TRT_VERSION), --build-arg TRT_VER="$(TRT_VERSION)") \ $(if $(TRT_LLM_VERSION), --build-arg TRT_LLM_VER="$(TRT_LLM_VERSION)") \ $(if $(DEVEL_IMAGE), --build-arg DEVEL_IMAGE="$(DEVEL_IMAGE)") \ $(if $(GIT_COMMIT), --build-arg GIT_COMMIT="$(GIT_COMMIT)") \ diff --git a/docker/common/install.sh b/docker/common/install.sh index 0d12d812a354..5f0dc91c8ed8 100755 --- a/docker/common/install.sh +++ b/docker/common/install.sh @@ -11,7 +11,7 @@ base=0 cmake=0 ccache=0 cuda_toolkit=0 -tensorrt=0 +cuda_libs=0 polygraphy=0 mpi4py=0 pytorch=0 @@ -34,8 +34,8 @@ while [[ $# -gt 0 ]]; do cuda_toolkit=1 shift 1 ;; - --tensorrt) - tensorrt=1 + --cuda_libs) + cuda_libs=1 shift 1 ;; --polygraphy) @@ -55,7 +55,7 @@ while [[ $# -gt 0 ]]; do cmake=1 ccache=1 cuda_toolkit=1 - tensorrt=1 + cuda_libs=1 polygraphy=1 mpi4py=1 pytorch=1 @@ -92,10 +92,9 @@ if [ $cuda_toolkit -eq 1 ]; then bash $SCRIPT_DIR/install_cuda_toolkit.sh fi -if [ $tensorrt -eq 1 ]; then - echo "Installing TensorRT..." - bash $SCRIPT_DIR/install_tensorrt.sh \ - --TRT_VER=${TRT_VER} \ +if [ $cuda_libs -eq 1 ]; then + echo "Installing CUDA libraries (cuDNN/NCCL/cuBLAS)..." + bash $SCRIPT_DIR/install_cuda_libs.sh \ --CUDA_VER=${CUDA_VER} \ --CUDNN_VER=${CUDNN_VER} \ --NCCL_VER=${NCCL_VER} \ diff --git a/docker/common/install_base.sh b/docker/common/install_base.sh index 499edc16b7d0..1141bd9d1531 100644 --- a/docker/common/install_base.sh +++ b/docker/common/install_base.sh @@ -95,6 +95,10 @@ init_ubuntu() { wget \ pigz \ libzmq3-dev + # WAR against CVE-2026-45447: upgrade OpenSSL runtime libraries + apt-get install -y --no-install-recommends \ + libssl3t64=3.0.13-0ubuntu3.11 \ + openssl=3.0.13-0ubuntu3.11 if ! command -v mpirun &> /dev/null; then DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openmpi-bin libopenmpi-dev fi @@ -107,6 +111,9 @@ init_ubuntu() { # used by cutlass. Use --ignore-installed to avoid failing on system setuptools # that lack a RECORD file (installed via apt without pip metadata). pip3 install --ignore-installed "setuptools<80" + # WAR: uninstall dependencies that has vulnerability or need upgrading. + # pip and wheel are already installed to /usr/local/ above, so removing + # the apt packages leaves pip3 functional. echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH' >> "${ENV}" # Remove previous TRT installation diff --git a/docker/common/install_tensorrt.sh b/docker/common/install_cuda_libs.sh similarity index 81% rename from docker/common/install_tensorrt.sh rename to docker/common/install_cuda_libs.sh index 0c762e0a3ad3..e5a8566c62d3 100644 --- a/docker/common/install_tensorrt.sh +++ b/docker/common/install_cuda_libs.sh @@ -2,7 +2,6 @@ set -ex -TRT_VER="10.16.1.11" # Align with the pre-installed cuDNN / cuBLAS / NCCL versions from # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 CUDA_VER="13.2" # 13.2.1 @@ -19,7 +18,6 @@ CUDA_DRIVER_VERSION="595.58.03-1.el8" for i in "$@"; do case $i in - --TRT_VER=?*) TRT_VER="${i#*=}";; --CUDA_VER=?*) CUDA_VER="${i#*=}";; --CUDNN_VER=?*) CUDNN_VER="${i#*=}";; --NCCL_VER=?*) NCCL_VER="${i#*=}";; @@ -166,47 +164,14 @@ install_rockylinux_requirements() { ldconfig } -install_tensorrt() { - PY_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[0:2])))') - PARSED_PY_VERSION=$(echo "${PY_VERSION//./}") - - TRT_CUDA_VERSION=${CUDA_VER} - TRT_VER_SHORT=$(echo $TRT_VER | cut -d. -f1-3) - - if [ -z "$RELEASE_URL_TRT" ];then - ARCH=${TRT_TARGETARCH} - if [ -z "$ARCH" ];then ARCH=$(uname -m);fi - if [ "$ARCH" = "arm64" ];then ARCH="aarch64";fi - if [ "$ARCH" = "amd64" ];then ARCH="x86_64";fi - RELEASE_URL_TRT="https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/${TRT_VER_SHORT}/tars/TensorRT-${TRT_VER}.Linux.${ARCH}-gnu.cuda-${TRT_CUDA_VERSION}.tar.gz" - fi - - wget --retry-connrefused --timeout=180 --tries=10 --continue ${RELEASE_URL_TRT} -O /tmp/TensorRT.tar - tar -xf /tmp/TensorRT.tar -C /usr/local/ - mv /usr/local/TensorRT-${TRT_VER} /usr/local/tensorrt - pip3 install --no-cache-dir /usr/local/tensorrt/python/tensorrt-*-cp${PARSED_PY_VERSION}-*.whl - rm -rf /tmp/TensorRT.tar - echo 'export LD_LIBRARY_PATH=/usr/local/tensorrt/lib:$LD_LIBRARY_PATH' >> "${ENV}" - - rm -f /usr/local/tensorrt/lib/libnvinfer_vc_plugin_static.a \ - /usr/local/tensorrt/lib/libnvinfer_plugin_static.a \ - /usr/local/tensorrt/lib/libnvinfer_static.a \ - /usr/local/tensorrt/lib/libnvinfer_dispatch_static.a \ - /usr/local/tensorrt/lib/libnvinfer_lean_static.a \ - /usr/local/tensorrt/lib/libnvonnxparser_static.a \ - /usr/local/tensorrt/lib/libnvinfer_builder_resource_win.so.* -} - # Install base packages depending on the base OS ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') case "$ID" in ubuntu) install_ubuntu_requirements - install_tensorrt ;; rocky) install_rockylinux_requirements - install_tensorrt ;; *) echo "Unable to determine OS..." diff --git a/docker/common/install_etcd.sh b/docker/common/install_etcd.sh index 663650dff438..ffe734d4d89e 100644 --- a/docker/common/install_etcd.sh +++ b/docker/common/install_etcd.sh @@ -2,7 +2,7 @@ set -ex -ETCD_VER=v3.6.9 +ETCD_VER=v3.7.1 # choose either URL DOWNLOAD_URL=https://storage.googleapis.com/etcd diff --git a/docs/source/_ext/llmapi_config_telemetry.py b/docs/source/_ext/llmapi_config_telemetry.py index 01509c74a628..0b0bf9e63427 100644 --- a/docs/source/_ext/llmapi_config_telemetry.py +++ b/docs/source/_ext/llmapi_config_telemetry.py @@ -76,19 +76,16 @@ def _table(rows: list[dict]) -> str: def generate_telemetry_reference(repo_root: Path | str, output_path: Path | str) -> None: repo_root = Path(repo_root) golden = json.loads((repo_root / _GOLDEN_REL).read_text()) - content = [_REFERENCE_PREAMBLE] - for args_class in ("TorchLlmArgs", "TrtLlmArgs"): - rows = golden.get(args_class, []) - content.extend( - [ - f"### `{args_class}`", - "", - f"{len(rows)} captured fields.", - "", - _table(rows), - "", - ] - ) + rows = golden.get("TorchLlmArgs", []) + content = [ + _REFERENCE_PREAMBLE, + "### `TorchLlmArgs`", + "", + f"{len(rows)} captured fields.", + "", + _table(rows), + "", + ] output = Path(output_path) output.parent.mkdir(parents=True, exist_ok=True) output.write_text("\n".join(content)) diff --git a/docs/source/_static/config_db.json b/docs/source/_static/config_db.json index 7671934a6911..8af081380d9c 100644 --- a/docs/source/_static/config_db.json +++ b/docs/source/_static/config_db.json @@ -179,6 +179,30 @@ "model_display_name": "MiniMax-M3 (MXFP8)", "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8", "scenario": "Max Throughput" + }, + { + "command": "trtllm-serve deepseek-ai/DeepSeek-V4-Pro --config ${TRTLLM_DIR}/examples/configs/curated/deepseek-v4-pro-latency.yaml", + "config_filename": "deepseek-v4-pro-latency.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/deepseek-v4-pro-latency.yaml", + "config_path": "examples/configs/curated/deepseek-v4-pro-latency.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/curated/deepseek-v4-pro-latency.yaml", + "gpu_compatibility": "B200", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model_display_name": "deepseek-ai/DeepSeek-V4-Pro", + "model_url": "", + "scenario": "Min Latency" + }, + { + "command": "trtllm-serve deepseek-ai/DeepSeek-V4-Pro --config ${TRTLLM_DIR}/examples/configs/curated/deepseek-v4-pro-throughput.yaml", + "config_filename": "deepseek-v4-pro-throughput.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/deepseek-v4-pro-throughput.yaml", + "config_path": "examples/configs/curated/deepseek-v4-pro-throughput.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/curated/deepseek-v4-pro-throughput.yaml", + "gpu_compatibility": "B200", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model_display_name": "deepseek-ai/DeepSeek-V4-Pro", + "model_url": "", + "scenario": "Max Throughput" } ], "entries": [ @@ -1321,6 +1345,82 @@ "osl": 1024, "performance_profile": "Max Throughput" }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", + "concurrency": 4, + "config_filename": "1k1k_tp4_conc4.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 1024, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Min Latency", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", + "concurrency": 8, + "config_filename": "1k1k_tp4_conc8.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 1024, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", + "concurrency": 16, + "config_filename": "1k1k_tp4_conc16.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 1024, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", + "concurrency": 256, + "config_filename": "1k1k_tp4_conc256.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 1024, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Max Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp4_conc2048.yaml", "concurrency": 2048, @@ -1338,6 +1438,82 @@ "osl": 8192, "performance_profile": "High Throughput" }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", + "concurrency": 4, + "config_filename": "8k1k_tp4_conc4.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 8192, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Min Latency", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", + "concurrency": 8, + "config_filename": "8k1k_tp4_conc8.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 8192, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Low Latency", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", + "concurrency": 16, + "config_filename": "8k1k_tp4_conc16.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 8192, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", + "concurrency": 256, + "config_filename": "8k1k_tp4_conc256.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 8192, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc1024.yaml", "concurrency": 1024, @@ -1353,7 +1529,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 4, "osl": 1024, - "performance_profile": "Min Latency" + "performance_profile": "High Throughput" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc2048.yaml", @@ -1421,7 +1597,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Low Latency" + "performance_profile": "Low Latency", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml", @@ -1472,7 +1650,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Balanced" + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml", @@ -1489,7 +1669,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Balanced" + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml", @@ -1506,7 +1688,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "High Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml", @@ -1523,7 +1707,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "High Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc512.yaml", @@ -1812,7 +1998,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Low Latency" + "performance_profile": "Low Latency", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml", @@ -1880,7 +2068,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "High Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml", @@ -1897,7 +2087,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "High Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml", @@ -1914,7 +2106,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "High Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc512.yaml", @@ -3532,6 +3726,10 @@ "display_name": "DeepSeek-R1", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528" }, + "deepseek-ai/DeepSeek-V4-Pro": { + "display_name": "deepseek-ai/DeepSeek-V4-Pro", + "url": "" + }, "nvidia/DeepSeek-R1-0528-FP4-v2": { "display_name": "DeepSeek-R1 (NVFP4)", "url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2" diff --git a/docs/source/_static/config_selector.css b/docs/source/_static/config_selector.css index ca84bb991f2a..43c5a4aa7bc6 100644 --- a/docs/source/_static/config_selector.css +++ b/docs/source/_static/config_selector.css @@ -1,3 +1,6 @@ +/* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ +/* SPDX-License-Identifier: Apache-2.0 */ + .trtllm-config-selector { border: 1px solid var(--pst-color-shadow); border-radius: 14px; @@ -29,6 +32,10 @@ align-items: start; } +.trtllm-config-selector__field[hidden] { + display: none; +} + .trtllm-config-selector__label { display: flex; align-items: center; diff --git a/docs/source/_static/config_selector.js b/docs/source/_static/config_selector.js index e8ef62550a6b..ca053655923a 100644 --- a/docs/source/_static/config_selector.js +++ b/docs/source/_static/config_selector.js @@ -1,14 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + (function () { "use strict"; let dbPromise = null; let widgetId = 0; - const GROUP_ORDER = ["model", "topology", "islOsl", "concurrency"]; + const GROUP_ORDER = ["model", "topology", "islOsl", "concurrency", "profile"]; const GROUP_LABELS = { model: "Model", topology: "GPU(s)", islOsl: "ISL / OSL", concurrency: "Concurrency", + profile: "Profile", }; function $(root, sel) { @@ -58,6 +62,7 @@ state.concurrency != null && state.concurrency !== "" ? String(state.concurrency) : "", + profile: state.profile || "", }; } @@ -138,9 +143,22 @@ function profileLabel(profile) { const text = String(profile || "").trim(); + const labels = { + latency: "Latency", + balanced: "Balanced", + throughput: "Throughput", + }; + if (labels[text]) return labels[text]; return text || "Unknown Profile"; } + function profileOption(profile) { + return { + value: profile, + label: profileLabel(profile), + }; + } + function formatProfileSummary(profiles) { const labels = uniqBy(profiles.map((profile) => profileLabel(profile)), (label) => label); if (!labels.length) { @@ -194,7 +212,12 @@ .map((group) => concurrencyOption(group)) .sort((a, b) => sortNums(a.concurrency, b.concurrency)); - return { model, topology, islOsl, concurrency }; + const profile = uniqBy( + entries.filter((entry) => entry.profile).map((entry) => profileOption(entry.profile)), + (option) => option.value + ); + + return { model, topology, islOsl, concurrency, profile }; } function filterEntriesByState(entries, state) { @@ -214,6 +237,9 @@ if (normalizedState.concurrency) { if (String(entry.concurrency) !== normalizedState.concurrency) return false; } + if (normalizedState.profile && entry.profile !== normalizedState.profile) { + return false; + } return true; }); } @@ -362,7 +388,8 @@ normalizedState.model && normalizedState.topology && normalizedState.islOsl && - normalizedState.concurrency + normalizedState.concurrency && + (!groups.profile.options.length || normalizedState.profile) ) { return "Selection did not resolve to a single configuration."; } @@ -389,6 +416,12 @@ }; } + function validatedCommitUrl(commit) { + const normalized = String(commit || "").trim().toLowerCase(); + if (!/^[0-9a-f]{40}$/.test(normalized)) return ""; + return `https://github.com/NVIDIA/TensorRT-LLM/commit/${normalized}`; + } + function isFileProtocol() { return window.location.protocol === "file:"; } @@ -608,14 +641,14 @@ : allCurated ).map(normalizeEntry); - // curatedIndex lives outside normalizeState's scope — it is preserved - // across Object.assign(state, view.state) because normalizeState only - // touches the four filter keys. + // curatedIndex lives outside normalizeState's scope and is preserved + // across Object.assign(state, view.state). const state = { model: "", topology: "", islOsl: "", concurrency: "", + profile: "", curatedIndex: null, }; @@ -685,6 +718,7 @@ const selTopo = mkOptionGroup("GPU(s)", 2); const selSeq = mkOptionGroup("ISL / OSL", 3); const selConc = mkSelectField("Concurrency", `trtllm-conc-${id}`, 4); + const selProfile = mkSelectField("Profile", `trtllm-profile-${id}`, 5); form.appendChild(selModel.wrap); @@ -707,6 +741,7 @@ form.appendChild(selTopo.wrap); form.appendChild(selSeq.wrap); form.appendChild(selConc.wrap); + form.appendChild(selProfile.wrap); const output = el("div", { class: "trtllm-config-selector__output" }); const cmdPre = el("pre", { class: "trtllm-config-selector__cmd" }, [ @@ -835,7 +870,10 @@ return { type: "model-select" }; } if (activeEl === selConc.select) { - return { type: "select" }; + return { type: "select", key: "concurrency" }; + } + if (activeEl === selProfile.select) { + return { type: "select", key: "profile" }; } if ( activeEl.classList && @@ -857,7 +895,8 @@ return; } if (descriptor.type === "select") { - selConc.select.focus(); + if (descriptor.key === "profile") selProfile.select.focus(); + else selConc.select.focus(); return; } if (descriptor.type !== "button") return; @@ -997,8 +1036,8 @@ selectEl.dataset.status = (selectedOption && selectedOption.status) || "idle"; } - function setSelectOptions(selectEl, group) { - const previousValue = state.concurrency || ""; + function setSelectOptions(selectEl, group, stateKey, placeholder) { + const previousValue = state[stateKey] || ""; const visibleOptions = group.options.filter( (option) => option.status !== "incompatible" ); @@ -1006,7 +1045,7 @@ selectEl.appendChild( el("option", { value: "", - text: visibleOptions.length ? "Select concurrency" : "No concurrency available", + text: visibleOptions.length ? `Select ${placeholder}` : `No ${placeholder} available`, }) ); for (const option of visibleOptions) { @@ -1077,7 +1116,12 @@ setOptionButtons(selTopo.options, "topology", view.groups.topology); setOptionButtons(selSeq.options, "islOsl", view.groups.islOsl); - setSelectOptions(selConc.select, view.groups.concurrency); + setSelectOptions(selConc.select, view.groups.concurrency, "concurrency", "concurrency"); + const hasProfileChoices = + Boolean(view.state.concurrency) && + view.groups.profile.options.some((option) => option.status !== "incompatible"); + selProfile.wrap.hidden = !hasProfileChoices; + setSelectOptions(selProfile.select, view.groups.profile, "profile", "profile"); const code = cmdPre.querySelector("code"); if (curatedSelected) { @@ -1132,6 +1176,26 @@ } else { meta.appendChild(el("span", { text: e.config_path || "" })); } + if (e.validated_trtllm_version) { + meta.appendChild( + el("span", { + text: ` \u00b7 Validated with TensorRT-LLM ${e.validated_trtllm_version}`, + }) + ); + } + const commitUrl = validatedCommitUrl(e.validated_trtllm_commit); + if (commitUrl) { + meta.appendChild(el("span", { text: " \u00b7 Commit: " })); + meta.appendChild( + el("a", { + class: "trtllm-config-selector__configLink", + href: commitUrl, + target: "_blank", + rel: "noopener", + text: e.validated_trtllm_commit.slice(0, 12), + }) + ); + } currentEntry = e; resetYamlPanel(); @@ -1161,6 +1225,12 @@ selConc.select.addEventListener("change", () => { state.concurrency = selConc.select.value; + state.profile = ""; + render(); + }); + + selProfile.select.addEventListener("change", () => { + state.profile = selProfile.select.value; render(); }); @@ -1218,6 +1288,7 @@ formatCuratedCommand: formatCommand, nextStateAfterSelection, normalizeState, + validatedCommitUrl, }; } diff --git a/docs/source/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.md b/docs/source/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.md index 22d4688c503d..91d89d70cacf 100644 --- a/docs/source/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.md +++ b/docs/source/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.md @@ -92,7 +92,7 @@ Here we set `LOCAL_USER=1` argument to set up the local user instead of root acc Here we compile the source inside the container: ``` bash -python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --cuda_architectures "90-real;100-real" --python_bindings --clean +python3 ./scripts/build_wheel.py --cuda_architectures "90-real;100-real" --clean ``` You can set the cuda_architectures to "100-real" if targeting Blackwell only, and "90-real" to target Hopper only to save some build time. diff --git a/docs/source/blogs/media/tech_blog26_agentperf_closed_loop_workflow.svg b/docs/source/blogs/media/tech_blog26_agentperf_closed_loop_workflow.svg new file mode 100644 index 000000000000..f9d0003051bd --- /dev/null +++ b/docs/source/blogs/media/tech_blog26_agentperf_closed_loop_workflow.svg @@ -0,0 +1,156 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +--> +<svg xmlns="http://www.w3.org/2000/svg" width="1800" height="815" viewBox="0 0 1800 815" role="img" aria-labelledby="title desc"> + <title id="title">AA-AgentPerf closed-loop request workflow + A simulated agent sends the next recorded turn with its accumulated conversation. The inference deployment routes the request using conversation or cache affinity, reuses matching KV blocks, prefills only the uncached suffix, optionally retrieves the KV cache on the generation side in disaggregated serving, and streams the generated response. The client drains the complete response, simulates tool time when needed, advances the trajectory, and sends the next turn. + + + + + + + + + + + + + + + + + + + + + + + + + + CLIENT / AGENT HARNESS + + + + + INFERENCE DEPLOYMENT + + + + + 1 + Send recorded turn + Accumulated conversation + and the next request + + + + + 8 + Advance trajectory + Next recorded turn or + next assigned trajectory + + + + + 7 + Simulate tool time + Only for turns with a tool call + Client-side delay · no LLM compute + + + + + 6 + Receive complete response + Drain the stream and assemble chunks + Record TTFT and output speed + + + + + + + + + + 2 + Route + prefix lookup + Conversation-aware or cache-aware + worker and rank selection + Reuse matching resident KV blocks + + + + + 3 + Prefill uncached suffix + Reuse removes the cached prefix + from new prefill computation + Context side produces the first token + + + + + 4 + Retrieve KV cache + Returned context metadata locates + the KV cache for the generation side + Disaggregated serving only + + + + + 5 + Generate + stream + Continue decoding the response + on the same worker or a + separate generation worker + + + + + + + + + Disaggregated path + + + + Aggregated serving · same worker + + + + + + + Conversation affinity helps preserve reuse across turns + Later turns prefer the worker and rank that hold the accumulated conversation prefix. + + + + + Inference work + + + Client-simulated tool time + + + Conversation affinity and prefix reuse + + + diff --git a/docs/source/blogs/media/tech_blog26_deepseek_v4_hybrid_attention.png b/docs/source/blogs/media/tech_blog26_deepseek_v4_hybrid_attention.png new file mode 100644 index 000000000000..8238521bcb40 --- /dev/null +++ b/docs/source/blogs/media/tech_blog26_deepseek_v4_hybrid_attention.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b9b5b49bba503ffcd6b0fb7993cf25e2c6537098ad4ffc232e9a89890fc218f +size 13509614 diff --git a/docs/source/blogs/media/tech_blog26_deepseek_v4_mhc_moe.png b/docs/source/blogs/media/tech_blog26_deepseek_v4_mhc_moe.png new file mode 100644 index 000000000000..32a68bbe5038 --- /dev/null +++ b/docs/source/blogs/media/tech_blog26_deepseek_v4_mhc_moe.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a034c944f943400df7b42b5a1f2321f7d3ee87eb80bd5c509dba5066cec5eee4 +size 1854626 diff --git a/docs/source/blogs/media/tech_blog26_dsv4_performance_evolution_8k1k.png b/docs/source/blogs/media/tech_blog26_dsv4_performance_evolution_8k1k.png new file mode 100644 index 000000000000..3fbaede6ebe4 Binary files /dev/null and b/docs/source/blogs/media/tech_blog26_dsv4_performance_evolution_8k1k.png differ diff --git a/docs/source/blogs/media/tech_blog26_dsv4_scratch_swa.png b/docs/source/blogs/media/tech_blog26_dsv4_scratch_swa.png new file mode 100644 index 000000000000..37e806c18c77 Binary files /dev/null and b/docs/source/blogs/media/tech_blog26_dsv4_scratch_swa.png differ diff --git a/docs/source/blogs/media/tech_blog26_two_level_routing.svg b/docs/source/blogs/media/tech_blog26_two_level_routing.svg new file mode 100644 index 000000000000..f032ffafa1ba --- /dev/null +++ b/docs/source/blogs/media/tech_blog26_two_level_routing.svg @@ -0,0 +1,167 @@ + + + + Two-level routing for context locality in disaggregated serving + An agentic request enters the TensorRT LLM disaggregated front-end. Instance-level routing first selects a context server, then the selected server's attention data-parallel router performs rank-level routing. Each level can use conversation-aware affinity or KV-cache-aware scoring. Full conversation-prefix reuse requires a later turn to return to both the context server and the rank that hold the corresponding KV blocks. The selected context rank transfers KV cache to a generation server, which decodes and streams the response. + + + + + + + + + + + + + + + + + + + + + + + + + + + + AGENT CLIENT + Next turn + + Conversation ID + + Accumulated prompt + + Cache salt (optional) + + + + request + + + + + TRTLLM-SERVE DISAGGREGATED + Front-end orchestrator + + LEVEL 1 · INSTANCE-LEVEL ROUTING + Use either routing policy to select a context server + + + + CONVERSATION-AWARE + Conversation ID → CTX server + Least-loaded first placement, then explicit affinity + + + + KV-CACHE-AWARE + Prompt blocks → cache view + Matched blocks plus active-request load + Content-derived affinity protects later turns + + + Selected destination: CTX server 1 + + + + + CONTEXT (CTX) TIER + + + + CTX server 0 + other rank-local caches + + CTX server 2 + other rank-local caches + + + + + SELECTED + CTX server 1 + + + + Attention data-parallel router + + LEVEL 2 · RANK-LEVEL ROUTING + Again, use either routing policy inside the selected server + + + CONVERSATION-AWARE + Conversation ID → rank + First turn: round-robin + + + KV-CACHE-AWARE + Rank-local prefix probe + Post-reuse work plus load + + + + + + rank-local reuse lost + + + Rank 0 + KV cache + + + Rank 1 + + prefix KV + + + Rank 2 + KV cache + + + Rank 3 + KV cache + + + + same server + + server-local + reuse lost + + + + + GEN SERVER + Generation + + Receive KV cache + + Continue decode + + Stream response + + + CTX → GEN KV transfer + + + + streamed response + diff --git a/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md b/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md new file mode 100644 index 000000000000..260b94448f42 --- /dev/null +++ b/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md @@ -0,0 +1,490 @@ + + +# DeepSeek-V4 on NVIDIA Blackwell: Model-Specific and Agentic-Workload Optimizations in TensorRT LLM + +By NVIDIA TensorRT LLM Team + +## Table of Contents + +- [Introduction](#introduction) +- [Part I. DeepSeek-V4 Model Support and Optimizations](#part-i-deepseek-v4-model-support-and-optimizations) + - [Building a Production-Ready DeepSeek-V4 Stack](#building-a-production-ready-deepseek-v4-stack) + - [Model support and parallelism](#model-support-and-parallelism) + - [Hybrid attention: SWA, CSA, and HCA](#hybrid-attention-swa-csa-and-hca) + - [Compressor implementation](#compressor-implementation) + - [Beyond attention: mHC, MoE, and speculative decoding](#beyond-attention-mhc-moe-and-speculative-decoding) + - [Parallelism and deployment notation](#parallelism-and-deployment-notation) + - [Cache management and runtime features](#cache-management-and-runtime-features) + - [Precision strategy and accuracy](#precision-strategy-and-accuracy) + - [DeepSeek-V4 Performance Optimizations](#deepseek-v4-performance-optimizations) + - [Optimize the CSA/HCA hot path](#optimize-the-csahca-hot-path) + - [Compressor and mHC](#compressor-and-mhc) + - [Compressor optimizations](#compressor-optimizations) + - [mHC optimizations](#mhc-optimizations) + - [MoE optimizations](#moe-optimizations) + - [Runtime optimization](#runtime-optimization) + - [DeepSeek-V4 Performance Evolution](#deepseek-v4-performance-evolution) + - [Experimental setup](#experimental-setup) + - [From Baseline to the Latest Measured Curve](#from-baseline-to-the-latest-measured-curve) +- [Part II. Agentic-Workload Optimizations](#part-ii-agentic-workload-optimizations) + - [AgentPerf Workflow](#agentperf-workflow) + - [Lessons from DeepSeek-V3.2 Agentic Workload Optimization](#lessons-from-deepseek-v32-agentic-workload-optimization) + - [End-to-End Optimizations for Agentic Serving](#end-to-end-optimizations-for-agentic-serving) + - [Preserve locality with two-level routing](#preserve-locality-with-two-level-routing) + - [Instance-level routing across context servers](#instance-level-routing-across-context-servers) + - [Rank-level routing within a context server](#rank-level-routing-within-a-context-server) + - [Routing policy selection](#routing-policy-selection) + - [KV cache reuse optimizations](#kv-cache-reuse-optimizations) + - [Optimize scheduling and remove end-to-end overhead](#optimize-scheduling-and-remove-end-to-end-overhead) + - [Optimize host overhead for CTX and GEN](#optimize-host-overhead-for-ctx-and-gen) + - [Optimize scheduling, orchestration, and protocol handling](#optimize-scheduling-orchestration-and-protocol-handling) + - [AgentPerf Results and External Validation](#agentperf-results-and-external-validation) +- [Reproduction and Future Work](#reproduction-and-future-work) + - [How to reproduce](#how-to-reproduce) + - [Future Work](#future-work) +- [Conclusion and Acknowledgments](#conclusion-and-acknowledgments) + +## Introduction + +DeepSeek-V4's hybrid attention, online compression, mHC, and MoE design make production inference a model-and-system co-design problem. This blog follows TensorRT LLM's optimization journey on NVIDIA Blackwell in two parts: first building and optimizing a production-ready DeepSeek-V4 model stack, then extending the optimization boundary to routing, KV reuse, host efficiency, and control-plane efficiency for agentic workloads. On GB300, the latest measured fixed-shape sweep increased peak output throughput from 984 to 1,618 tokens/s/GPU, a 64.5% improvement, while NVIDIA's GB300 AA-AgentPerf configurations, run and verified by Artificial Analysis, reached 57.5 concurrency per GPU (CPG) at SLO20 and 19.2 CPG at SLO60. + +## Part I. DeepSeek-V4 Model Support and Optimizations + +### Building a Production-Ready DeepSeek-V4 Stack + +Supporting DeepSeek-V4 required more than registering a new model class. Relative to DeepSeek-V3.2, the model replaces homogeneous sparse attention with a hybrid scheme that interleaves three attention modes across layers, adds an online sequence Compressor with its own persistent state, wraps every attention and MoE block in manifold-constrained hyper-connections (mHC), and updates the MoE routing and checkpoint layout. All of this must compose with the production features users expect from TensorRT LLM: Multi-Token Prediction (MTP), parallel execution, chunked prefill, KV-cache reuse, CUDA Graphs, the overlap scheduler, and disaggregated serving. + +TensorRT LLM implements DeepSeek-V4 as a dedicated PyTorch-backend model, `DeepseekV4ForCausalLM`, together with a DeepSeek-V4 sparse-attention backend and a specialized `DeepseekV4CacheManager`, targeting NVIDIA Blackwell GPUs (SM100+). This section describes that foundation. The performance optimizations built on top of it are covered in [**DeepSeek-V4 Performance Optimizations**](#deepseek-v4-performance-optimizations). + +#### Model support and parallelism + +DeepSeek-V4 is released in two model scales, each with Base and Instruct variants. TensorRT LLM loads both directly from their checkpoint metadata. + +| Checkpoint family | Total parameters | Activated parameters | Published precision | +| :--- | ---: | ---: | :--- | +| DeepSeek-V4-Flash-Base / -Flash | 284B | 13B | FP8 mixed / FP4 + FP8 mixed | +| DeepSeek-V4-Pro-Base / -Pro | 1.6T | 49B | FP8 mixed / FP4 + FP8 mixed | + +The implementation reads the attention layout from the checkpoint rather than hard-coding serving assumptions: the per-layer compression-ratio list (Flash begins `[0, 0, 4, 128, ...]`, while Pro begins `[128, 128, 4, 128, ...]`, and 0 marks a sliding-window-only layer), the 128-token sliding window, and the Indexer configuration including its Top-K (512 for Flash and 1024 for Pro). + +##### Hybrid attention: SWA, CSA, and HCA + +DeepSeek-V4's largest architectural change is its attention. DeepSeek-V3.2 introduced DeepSeek Sparse Attention (DSA), in which a learned Indexer scores the KV history and a Top-K selector supplies token indices to sparse MLA (see our [DeepSeek-V3.2 optimization blog](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog15_Optimizing_DeepSeek_V32_on_NVIDIA_Blackwell_GPUs.md)). TensorRT LLM implemented DSA on a general [sparse-attention framework](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog17_Sparse_Attention_in_TensorRT-LLM.md) that separates the algorithm-specific selection logic from the attention backend that consumes the selected indices. DeepSeek-V4 builds on the same separation but adds online compression and interleaves three layer types, determined by each layer's compression ratio: + +| Attention mode | Ratio | History visible to each query | Selection and computation | +| :--- | ---: | :--- | :--- | +| Sliding Window Attention (SWA) | 0 | The latest 128 raw tokens | Dense attention inside the window. No Compressor or Indexer. | +| Compressed Sparse Attention (CSA) | 4 | Latest 128 raw tokens + 4× compressed entries from completed compression groups | The Indexer scores the compressed entries. Sparse MLA attends to the window plus the Top-K selected entries. | +| Heavily Compressed Attention (HCA) | 128 | Latest 128 raw tokens + all 128× compressed entries from completed compression groups | Dense attention over all completed compressed entries. No Indexer or Top-K stage. | + +
+
+ DeepSeek-V4 hybrid attention architecture. Panel a shows SWA attending over the latest 128 raw tokens. Panel b shows CSA combining the sliding window with 4× compressed KV entries selected by an Indexer and Top-K stage. Panel c shows HCA combining the sliding window with all 128× compressed KV entries without an Indexer. +
+
+

Figure 1. DeepSeek-V4 hybrid attention. (a) SWA attends densely within the 128-token sliding window. (b) CSA augments that window with Top-K entries selected from the 4× compressed history. (c) HCA attends to the window and the complete 128× compressed history without an Indexer.

+ +At the kernel boundary, the history is represented as two cache pools, one for the sliding window and one for compressed entries, and a unified dual-pool MLA operator serves all three modes: it processes the sliding window first, then the selected CSA entries or the full HCA compressed stream, and combines both in one online-softmax reduction. An SWA-only layer simply passes an empty second pool. Selected compressed positions are converted from cache-page-local offsets to token-level global addresses before the attention kernel consumes them. Because KV compression makes the attention workload vary by query, TensorRT LLM passes a per-query `topk_lens` vector (`sparse_mla_topk_lens` in the implementation) to the FMHA kernel. It identifies the active SWA-plus-compressed index span within the fixed-width index buffer used for CUDA Graph compatibility. + +Checkpoint-provided attention-sink logits are also remapped and supplied to the backend as an additional softmax sink. This dual-pool design lets the heterogeneous layer types share one attention backend, and extends the sparse-attention framework from DSA's fine-grained token selection to DeepSeek-V4's combination of local attention, compressed memory, and optional sparse selection. + +##### Compressor implementation + +The Compressor produces the compressed KV entries that CSA and HCA attend to. It is a token-level, softmax-gated pooling ([DeepSeek-V4 paper](https://arxiv.org/abs/2606.19348)): each token's projected KV entry carries a learned compression-weight vector, a learnable per-position bias is added, and a softmax over the compression window reduces the group to one entry as a weighted sum. For CSA ($m{=}4$), each output pools projected entries from two adjacent $m$-token groups, giving a $2m$-token receptive field. Consecutive outputs overlap by $m$ raw-token positions through separate projection branches. HCA instead uses disjoint $m'$-token windows ($m'{=}128$). On CSA layers the same compression operation is applied a second time to produce the compressed Indexer keys that the Indexer scores for Top-K selection, so a CSA layer maintains two compressed streams: the attention KV entries and the Indexer keys. + +TensorRT LLM implements the Compressor as a fused `wkv_gate` projection plus dedicated CUDA kernels for prefill reduction, paged decode updates, and cache-write postprocessing. Tokens arrive incrementally: one at a time in decode, and chunk by chunk in chunked prefill, where chunk boundaries rarely align with compression windows. A compression window is often still incomplete when a forward pass ends. TensorRT LLM persists each token's KV and score projection outputs in FP32 Compressor-state buffers for the still-open windows. Once a window completes, the compression kernels finalize its entry from this state, preserving the semantics of compressing the whole sequence in one pass. Compressed entries are written in the dtype their consumer expects: BF16 or per-tensor FP8 for the main attention cache, and blockwise FP8 or packed MXFP4 for the Indexer-K stream. The kernel fusion work on this path is covered in [**Compressor and mHC**](#compressor-and-mhc). + +##### Beyond attention: mHC, MoE, and speculative decoding + +**mHC hyper-connections** ([mHC paper](https://arxiv.org/abs/2512.24880)). mHC widens the residual stream by a factor of 4 and mixes it around every sublayer through three dynamically generated mappings: a **pre-mapping** that reads the sublayer input from the expanded residual, a **residual-mixing** matrix projected onto the manifold of doubly stochastic matrices through 20 Sinkhorn-Knopp iterations, and a **post-mapping** that writes the sublayer output back into the residual streams. TensorRT LLM instantiates two mHC modules per layer (one around attention and one around the MoE block), plus an **HC head** that collapses the expanded residual to a single stream before the language-model head. It fuses adjacent post- and pre-mappings at the attention-to-MoE boundary and, where possible, across consecutive transformer layers. + +**MoE.** Both model scales activate six routed experts and one shared expert per token, selecting from 256 routed experts in Flash and 384 in Pro. The first three layers use hash routing, where a checkpoint-provided token-ID-to-expert table maps each input token directly to its experts. Later layers use a learned gate with Sqrt-Softplus affinity scores and a score-correction bias. TensorRT LLM implements both routing modes through a shared interface that supplies the selected experts and routing weights to its configurable MoE backends. TRTLLM-Gen, DeepGEMM, and the standard CuTe DSL fused MoE backend cover the corresponding DeepSeek-V4 precision paths and preserve the checkpoint-defined `swiglu_limit` activation clamp, including the uniform clamp used by the NVFP4 CuTe DSL path. DeepSeek-V4 is also registered with TensorRT LLM's expert parallel load balancer, allowing supported expert-parallel deployments to rebalance physical expert placement without changing the model's routing semantics. + +
+
+ DeepSeek-V4 mHC architecture and MoE routing. Panel a shows the repeated transformer stack with mHC pre- and post-mappings around the attention and MoE blocks, followed by the HC head and language-model head. Panel b shows hash routing through a token-ID embedding table and learned routing through Sqrt-Softplus scores, correction bias, and Top-K expert selection. +
+
+

Figure 2. DeepSeek-V4 components beyond attention. (a) mHC wraps the attention and MoE sublayers with pre- and post-mappings, then uses an HC head to collapse the expanded residual stream before the language-model head. (b) The MoE router supports checkpoint-defined hash routing in the first three layers (left) and learned Sqrt-Softplus scoring with correction bias and Top-6 selection in later layers (right).

+ +**Speculative decoding.** DeepSeek-V4 checkpoints with next-token prediction layers run MTP speculative decoding through the Eagle-style one-model path. Each MTP module consumes the expanded residual state from the target model or previous MTP module together with the next-token embedding, then runs the same mHC-wrapped attention-and-MoE structure as a target-model layer, using SWA for its attention. TensorRT LLM remaps the MTP weights, shares the embedding and language-model head with the target model, and extends the attention and cache metadata for the configured draft depth. The speculative-decoding performance results reported in this blog use this MTP path. DeepSeek has released dedicated [DSpark](https://arxiv.org/abs/2607.05147) checkpoints for DeepSeek-V4. DSpark adds a lightweight sequential head to a parallel draft backbone to model dependencies within a draft block, while performance tuning for this path remains in progress. + +##### Parallelism and deployment notation + +DeepSeek-V4 adopts the same parallel strategies as DeepSeek-R1 and DeepSeek-V3.2: attention data parallelism (ADP) combined with expert parallelism (EP) for throughput-oriented serving, and tensor parallelism (TP) for latency-oriented deployments, with pipeline parallelism available for fitting the largest checkpoints. See [Tech Blog 3](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog03_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.md) for the performance rationale and [Tech Blog 4](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog04_Scaling_Expert_Parallelism_in_TensorRT-LLM.md)/[8](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog08_Scaling_Expert_Parallelism_in_TensorRT-LLM_part2.md)/[14](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.md) for large-scale expert parallelism. Note that with ADP, `max_batch_size` is a per-rank limit, so benchmark concurrency must be scaled to populate all ranks. + +The rest of this blog uses a compact notation for these combinations. `TEP` shards both attention (TP) and experts (EP) across `N` ranks. `DEP` keeps attention data-parallel (ADP) while distributing experts across `N` ranks. Both compose with disaggregated serving, where context (CTX) and generation (GEN) servers are sized independently. For example, `DEP4 (CTX) + TEP8 (GEN)` runs a four-rank DEP context instance alongside an eight-rank TEP generation instance. + +#### Cache management and runtime features + +DeepSeek-V4's hybrid attention turns the "KV cache" into a collection of states with different shapes and lifetimes. Each attention mode owns a different set of logical cache layers: + +- an **SWA** layer owns only its sliding-window KV. +- a **CSA** layer adds the 4× compressed attention cache, Compressor KV/score state, compressed Indexer-K cache, and Indexer-Compressor KV/score state. +- an **HCA** layer adds the 128× compressed attention cache and Compressor KV/score state, with no Indexer-side cache. + +These tensors differ in dtype, bytes per entry, production rate, and lifetime. Treating them as a uniform KV tensor would either waste memory or give the scheduler an incorrect view of capacity. For the rest of this blog, we group them into two lifecycle categories. The **sliding-window cache** contains SWA's sliding-window KV and the short-lived Compressor KV/score state used by CSA, HCA, and the CSA Indexer. These buffers are needed only for bounded windows over recent raw-token positions, so older pages can be recycled. The **persistent compressed caches** contain the finalized 4× and 128× compressed attention entries and CSA's compressed Indexer-K entries. They represent retained history and remain available for attention and prefix reuse, so their storage grows with sequence length at the corresponding compression rate. These terms describe lifecycle categories rather than a one-to-one mapping to physical pools. `DeepseekV4CacheManager` describes every state as a buffer role with an explicit lifecycle and page-indexing mode, then groups compatible buffers into physical memory pools. Sliding-window cache buffers use per-layer page indices. Persistent compressed caches use shared indices across layers with the same compression layout. + +The Compressor breaks two assumptions made by a conventional paged KV cache: its intermediate state is short-lived, and its persistent output grows more slowly than the raw token sequence. TensorRT LLM maps both cases onto existing KV Cache Manager V2 abstractions. + +**Short-lived Compressor state.** A raw token's intermediate state is needed only while a compression window that contains it remains open. Once the last such window is finalized, that token's intermediate state can be recycled. This lifetime is naturally represented as a sliding window: 128 raw-token positions for HCA, and 8 for CSA because its overlapping Compressor keeps two 4-token groups live. The manager can therefore reuse the existing window-eviction and block-recycling machinery. + +**Slow-growing compressed caches.** A compressed stream produces one entry for every $r$ raw tokens. Rather than introduce a second coordinate system, the manager keeps page allocation in raw-token coordinates. For a page covering `tokens_per_block` raw positions, its compressed buffer stores `tokens_per_block / r` entries. The page still represents the same interval of the original sequence, but its storage cost reflects the 4× or 128× compression ratio. Allocation, block-table construction, prefix reuse, and capacity accounting can consequently share one page model across all streams. + +The physical pool split is workload-dependent. Fresh prefill creates the greatest pressure on the sliding-window cache, whereas persistent compressed caches grow with sequence length and decode concurrency. When `kv_cache_config.pool_ratio` is not specified, KV Cache Manager V2 derives an initial split from a representative mixed batch: one context request and up to `max_batch_size - 1` generation requests. `kv_cache_config.avg_seq_len` controls the representative total sequence length of a typical decode request, while additional constraints reserve enough capacity for a maximum-length decode request and a chunked-prefill step. Advanced deployments can override the initial split with `pool_ratio` (specified in pool-group order and summing to 1.0). An opt-in beta rebalancer, enabled with `enable_kv_pool_rebalance`, can later adjust the split from runtime statistics. It is disabled by default and currently targets a narrower set of aggregated-serving configurations. + +DeepSeek-V4 also enables SWA scratch reuse by default. During prefill, transient sliding-window pages can be recycled across compatible layers instead of being retained as independent long-lived allocations. This reduces the prefill peak that would otherwise dictate the entire pool split. Its performance impact is discussed in [**Runtime optimization**](#runtime-optimization). + +With this state model in place, the same attention implementation composes with the runtime features expected in production: + +- **Chunked prefill.** The Compressor and Indexer carry compressed lengths and open-window state across chunks, preserving the semantics of processing the prompt in one pass. +- **KV-cache reuse.** Prefix reuse restores the attention, Compressor, and Indexer state owned by each layer type, rather than treating the main attention KV as sufficient. +- **CUDA Graphs.** Top-K and Compressor workspaces use graph-safe buffers whose captured shapes remain stable for a given graph batch size. +- **Overlap scheduling.** Cache updates and sparse-attention preparation are designed to avoid host-device synchronizations on the critical path, allowing request preparation to overlap GPU execution. +- **Disaggregated serving.** The V2 transfer path describes cache contents by pool, layer, and buffer role, allowing context workers to transfer every DeepSeek-V4 cache type and generation workers to rebuild their local block tables consistently. + +Production support also extends to the API boundary. For Instruct checkpoints, selecting the `deepseek_v4` tokenizer wrapper applies the reference chat format, including thinking controls, tool definitions, and DSML, DeepSeek's special-token-delimited format for tool calls. Tool results are inserted with `` tags. Matching reasoning and tool parsers turn generated thinking and DSML tool-call blocks back into OpenAI-compatible response fields. Base checkpoints remain completion models and should receive raw prompts without this wrapper. + +#### Precision strategy and accuracy + +DeepSeek-V4 uses three checkpoint-level precision recipes, each mapping low precision to the components where it reduces weight or cache bandwidth while keeping higher precision for stateful reductions and numerically sensitive paths: + +| Component | FP8 (Base) | MXFP4 (Instruct) | NVFP4 (requantized) | +| :--- | :--- | :--- | :--- | +| MoE routed experts GEMM | Blockwise FP8 | MXFP8 activations × MXFP4 weights | NVFP4 | +| MoE shared expert GEMM | Blockwise FP8 | MXFP8 | MXFP8 | +| Attention QKV/O GEMM | Blockwise FP8 | MXFP8 | MXFP8 | +| Indexer Q projection | Blockwise FP8 | MXFP8 | MXFP8 | +| Indexer weights projection | BF16 | BF16 | BF16 | +| Compressor KV/score linears | BF16 | BF16 | BF16 | +| Main attention KV cache | BF16 or per-tensor FP8 | BF16 or per-tensor FP8 | BF16 or per-tensor FP8 | +| Indexer K cache | Blockwise FP8 or MXFP4 | Blockwise FP8 or MXFP4 | Blockwise FP8 or MXFP4 | +| Compressor state cache | FP32 | FP32 | FP32 | + +Apart from the main-transformer routed experts, the NVFP4 checkpoint keeps its tensors identical to the MXFP4 Instruct source. In particular, the MTP subtree retains the Instruct precision recipe. The two FP4 routed-expert formats are distinct recipes served by different kernels. The Instruct checkpoints publish packed MXFP4 routed-expert weights executed with MXFP8 activations and MXFP4 weights, while the main-layer experts in requantized checkpoints such as `nvidia/DeepSeek-V4-Pro-NVFP4` run through the NVFP4 MoE path. On the cache side, FP4 approximately halves the Indexer-K data payload and is TensorRT LLM's default Indexer-K format for DeepSeek-V4 on Blackwell. The attention projections follow the same philosophy of staying in low precision across kernel boundaries. For eligible context-only sparse MLA batches, query quantization is fused into the Q RMSNorm and RoPE kernels, which write the FP8 Q buffer consumed by attention. Generation and mixed batches use the unfused Q path. For eligible DeepSeek-V4 Pro configurations, inverse RoPE and FP8 quantization are further fused into the FMHA epilogue, while other configurations use the optimized standalone inverse-RoPE and quantization kernel before `o_a_proj`. These kernel fusions are discussed further in [**Optimize the CSA/HCA hot path**](#optimize-the-csahca-hot-path). + +Accuracy was validated across Flash and Pro checkpoints, the FP8 (Base), MXFP4 (Instruct), and NVFP4 recipes, MTP on and off, FP8 and BF16 KV cache, and aggregated and disaggregated serving, comparing GPQA-Diamond and LiveCodeBench scores against the paper baselines. In this blog, MTP-1 and MTP-3 denote speculative draft lengths of one and three tokens, respectively. Representative GPQA-Diamond results with MTP-3 and FP8 KV cache unless noted: + +| Configuration | GPQA-Diamond | +| :--- | ---: | +| DeepSeek-V4-Flash | 88.8 | +| DeepSeek-V4-Pro | 89.08 | +| DeepSeek-V4-Pro, disaggregated | 89.39 | +| DeepSeek-V4-Pro-NVFP4 | 89.39 | + +Production readiness also required testing well beyond single-feature correctness: the validation matrix spans CUDA Graph capture, chunked prefill with cached history, MTP, long contexts, high concurrency, disaggregated transfer and shutdown, and memory pressure during loading and autotuning, with model-specific CI covering aggregated and disaggregated serving on Blackwell. This coverage is what turned the initial functional implementation into a stack the subsequent performance work could safely optimize. + +### DeepSeek-V4 Performance Optimizations + +Building on the production-ready execution stack described above, TensorRT LLM optimizes DeepSeek-V4 across four areas: the CSA/HCA attention hot path, Compressor and mHC, MoE, and runtime execution and memory management. The following sections explain the key techniques in each area, while [**DeepSeek-V4 Performance Evolution**](#deepseek-v4-performance-evolution) presents their cumulative end-to-end impact under a consistent model workload. Performance numbers cited here are scoped operator measurements or A/B end-to-end comparisons, not cumulative speedups. + +#### Optimize the CSA/HCA hot path + +CSA and HCA share a dual-pool attention kernel over the 128-token sliding window and compressed history, while CSA additionally runs an Indexer and Top-K selection. TensorRT LLM optimizes this hot path through attention-kernel improvements, low-precision fusion, multi-stream scheduling, and Top-K optimization. + +**Sparse MLA kernel optimization.** The dual-pool kernel processes the 128-token sliding window first, then gathers selected CSA entries or streams all HCA entries from the compressed pool. The optimized TRTLLM-Gen kernel skips redundant dense and sliding-window softmax masking on full tiles and issues sparse V loads with less coordinate setup and register spill/reload pressure. On B200 with an FP8 KV cache, the measured FMHA kernel was up to 1.31× faster for CSA prefill and 1.21× faster for HCA generation in the tested shapes. These are operator-level results, and the model-level gain depends on the model's mix of SWA, CSA, and HCA layers. + +**Kernel fusion and low-precision dataflow.** The functional path initially relied on framework or generic operators around attention. TensorRT LLM adds a DeepSeek-V4-specific Q RMSNorm CUDA kernel and extends the MLA RoPE/assignment and FMHA epilogue kernels so that normalization, rotation, and quantization can be fused into their producers. The resulting optimizations are: + +- **Fused Q RMSNorm, RoPE, and FP8 quantization.** The initial context path used three kernels for Q RMSNorm, RoPE and assignment, and full-Q FP8 quantization. For eligible context-only sparse MLA batches with an FP8 KV cache, TensorRT LLM folds quantization of the 448-dimensional non-RoPE segment into the Q RMSNorm kernel, then folds quantization of the remaining 64-dimensional RoPE segment and BMM-scale generation into the RoPE and assignment kernel. These two producer kernels write directly into the same FP8 Q buffer, eliminating the standalone quantization kernel and the full higher-precision normalized Q tensor. Generation and mixed batches continue to use the unfused path. +- **Standalone inverse-RoPE and FP8 quantization.** DeepSeek-V4 first establishes an FP8-native input path for `o_a_proj` by combining inverse RoPE and 1×128 blockwise FP8 quantization in a standalone operator. This operator consumes the BF16 output from FMHA and writes E4M3 activations and FP32 scales directly in the layout required by the Blackwell `o_a_proj` BMM. Replacing the original Triton implementation with a shape-specialized CUDA kernel made the standalone operator 1.6–2.3× faster in kernel-only measurements across 1K–32K tokens on B200. This path remains the fallback when FMHA epilogue fusion is not applicable. +- **FMHA epilogue fusion.** Building on the same FP8 `o_a_proj` input contract, TensorRT LLM moves inverse RoPE and 1×128 E4M3 quantization into the FMHA correction epilogue for validated DeepSeek-V4 Pro configurations using the FP8 KV cache, sparse MLA, and attention data parallelism. FMHA then writes the E4M3 output and FP32 scales directly in the layout consumed by the `o_a_proj` BMM, eliminating both the standalone operator launch and the intermediate BF16 output write and read. The `o_a_proj` BMM remains a separate kernel. This fusion supports context-only and generation-only batches, while mixed batches and unsupported configurations use the standalone path. Relative to the combined time of standard FMHA and the standalone inverse-RoPE and quantization operator, the fused FMHA kernel was up to 1.62× faster for prefill and 1.34× faster for generation in the tested shapes. +- **Others.** For CSA layers with BF16, bias-free `q_b` projection weights, a shape-autotuned CuTe DSL GEMM replaces the generic linear path on supported SM100f systems and falls back when CuTe DSL is unavailable. The optimized attention path also removes a device-to-device copy after FMHA, and FP8 quantization kernels initialize their own scale-buffer padding instead of requiring a separate host-launched `zero_()` operation. + +**Top-K optimizations.** The CSA Indexer selects hundreds of entries per query, making Top-K particularly visible during long-context decode. TensorRT LLM accelerates it in two complementary ways: + +- **Top-K kernel optimization.** TensorRT LLM makes the standard Top-K path shape-aware and device-aware. Depending on the compressed sequence length and row count, the dispatcher uses insertion selection, a single-CTA radix kernel, or a multi-CTA split-and-merge implementation. The multi-CTA path exposes enough parallel work for small decode batches, while its launch policy uses the row count, compressed sequence length, configured split threshold, and the GPU's actual SM count. On B200 with FP32 logits, 196,608 columns, `next_n=1`, and no previous indices, the device-aware policy reduced the K=512 latency at batch size 148 from 384 µs to 112 µs. +- **GVR Top-K.** GVR, introduced in our [GVR technical blog](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog21_Temporal_Correlation_Meets_Sparse_Attention.md), exploits the overlap between the entries selected by adjacent decode steps. It uses the previous step's indices as candidates, verifies them against current scores, and refines or falls back when the fast-path conditions are not met. DeepSeek-V4 extends GVR to the Indexer's 4× compressed coordinate space and to checkpoint-defined Top-K values of 512 for Flash and 1024 for Pro. In kernel-only B300 tests, GVR was 1.40–2.17× faster than the radix-selection baseline for the tested Flash and Pro shapes. An end-to-end Flash test on eight B300 SXM6 GPUs with a 65K-token input, batch size 4, and MTP-3 reported 6.4% higher request throughput. + +**Multi-stream attention scheduling.** The CSA attention prologue contains the main-attention Compressor, the `q_b` projection, and an Indexer with only a few true dependencies between them. The main-attention Compressor and the query-independent part of the Indexer can start directly from the layer input, while the `q_b` projection and Q RMSNorm wait for the shared projection and `q_a` normalization. TensorRT LLM progressively exposes this parallelism at two levels: + +- **Initial outer overlap.** The first dependency-aware schedule runs the main-attention Compressor concurrently with the `q_b` projection and Q RMSNorm. It then adds a dedicated stream for the complete Indexer branch, allowing all three branches to make progress before sparse MLA consumes their outputs. +- **Nested Indexer overlap.** Inside the Indexer, query projection, RoPE, and Q quantization form one branch, while the query-independent weight projection, Indexer Compressor, and K-cache update form another. Events synchronize them only when Q scales are needed for weight scaling and when the logits and Top-K kernels require the updated cache. +- **Final schedule with earlier launches.** The refined schedule pre-launches the main-attention Compressor and the query-independent half of the Indexer before the shared `kv_a` projection and Q/KV normalizations. Once normalized Q is available, the query-dependent Indexer work runs on its dedicated stream, while the `q_b` projection and Q RMSNorm are queued after the main-attention Compressor on the Compressor stream. This is the final stream assignment, replacing the earlier three-way outer schedule. + +Correctness across streams requires explicit tensor-lifetime tracking. TensorRT LLM calls `record_stream()` on the precomputed Indexer tensors, normalized Q output, and Top-K indices when they move to the consuming stream, preventing the PyTorch caching allocator from reusing their storage before the downstream work completes. + +#### Compressor and mHC + +The Compressor and mHC introduce repeated chains of small, stateful, and reduction-heavy operations around attention and MoE. The Compressor turns projected KV and scores into persistent paged state and compressed cache entries, while mHC transforms the residual streams at each sublayer boundary. TensorRT LLM optimizes both in two layers: first fuse adjacent operations to remove launches and intermediate tensors, then specialize the fused kernels for DeepSeek-V4 shapes and execution regimes. + +##### Compressor optimizations + +The optimized Compressor pipeline uses fusion at three boundaries: + +- **KV and score projection.** A single `wkv_gate` projection produces both the KV vectors and their compression scores. +- **Compression and state update.** The prefill and decode kernels write new KV/score state into paged storage and perform the online-softmax reduction when a compression window completes, avoiding separate state-update and reduction passes. +- **Postprocessing and cache write.** One kernel applies RMSNorm, RoPE, the optional Hadamard transform, output quantization, and scatter into the compressed paged cache. + +**Kernel optimization.** The fused Compressor kernels support the different dtype and cache-layout requirements of both the main-attention Compressor and the Compressor inside the Indexer, including BF16, FP8, and MXFP4 paths. They accept BF16 projection output directly while retaining FP32 state updates and online-softmax accumulation where required, avoiding a full FP32 KV/score intermediate. The reduction kernels are specialized for the 4×-compression decode and MTP regime and the much longer 128×-compression prefill regime, with the latter also reducing newly arrived tokens while writing their paged state to avoid a later readback. + +##### mHC optimizations + +DeepSeek-V4 places mHC around both attention and MoE, making the boundary between consecutive sublayers the primary fusion opportunity. + +**Kernel fusion.** TensorRT LLM fuses the previous sublayer's post-mapping with the next sublayer's pre-mapping into shape-specialized CUDA kernels, with both two-kernel and all-in-one tactics for different token counts. The next sublayer's input RMSNorm is further folded into the fused mHC epilogue, eliminating a separate launch and an HBM round trip. Because mHC owns the residual connection at the attention-to-MoE boundary, the MoE path also skips separate residual handling. + +**Kernel optimization.** No single mHC kernel is optimal across the full range of token counts `M`. TensorRT LLM provides FMA implementations for small-`M` workloads and higher-throughput MMA implementations as `M` grows, with two-kernel and all-in-one variants, multiple tile configurations, and Split-K support. The autotuner evaluates the valid tactics for each workload and selects the best-performing implementation, while pruning choices that are not competitive in the corresponding `M` regime and caching the selected tactic for reuse. + +#### MoE optimizations + +Building on the routing design [described above](#beyond-attention-mhc-moe-and-speculative-decoding), TensorRT LLM optimizes DeepSeek-V4's MoE path through a custom router GEMM, the MXFP8 × MXFP4 DeepGEMM MegaMoE backend, streamlined input preparation, and ongoing NVFP4 MegaMoE integration. These efforts preserve the model's routing metadata, precision recipe, and checkpoint-defined `swiglu_limit` activation clamp. + +**MoE router GEMM optimization.** For the 256-expert Flash shape, the score-based router uses a custom GEMM when it receives one to 16 token rows. The kernel consumes BF16 activations and weights and produces FP32 logits directly, avoiding the full FP32 input and weight casts required by the original linear path. Unsupported shapes, including the 384-expert Pro router, continue to use cuBLAS. In the reported small-shape measurements, router latency decreased from approximately 8 µs to 3 µs. + +**MXFP8 × MXFP4 DeepGEMM MegaMoE.** For the W4A8 recipe with MXFP8 activations and MXFP4 weights, the DeepGEMM MegaMoE backend moves dispatch, the first expert GEMM, SwiGLU, the second expert GEMM, and combine into a fused-communication path that uses symmetric memory and in-kernel synchronization. In a 500-request DeepSeek-V4 Flash test on B200 with 1K-token inputs and outputs, TP4, EP4, and attention data parallelism, switching from the TensorRT LLM MoE backend to DeepGEMM MegaMoE increased throughput by 15.3% and reduced latency by 12.7%. TensorRT LLM also fuses the input preparation for this backend: instead of separately quantizing BF16 activations to MXFP8, converting selected expert indices, and copying activations, scales, indices, and routing weights into symmetric buffers, one CUDA kernel performs the quantization and writes all four outputs directly to their final buffers, reducing six GPU operations to one. No separate end-to-end gain is attributed to the preparation change. + +**NVFP4 MegaMoE (WIP).** For native NVFP4 checkpoints, TensorRT LLM is integrating the ported `Sm100MegaMoEKernel` as a CuTe DSL execution backend on the SM100f path. The integration covers BF16-to-NVFP4 input quantization, dispatch, the two expert GEMMs, SwiGLU, per-route combine writes with native NVFP4 weights, and a separate reduction of the Top-K route contributions. It also includes symmetric-memory exchange for multi-rank expert parallelism and workload-specific tactic selection through the TensorRT LLM autotuner. The NVFP4 and DeepGEMM MegaMoE paths have separate hardware, quantization, and topology requirements. This precision-specific backend remains under integration and validation. + +#### Runtime optimization + +After the major kernels are optimized, runtime overhead shifts to the boundaries between kernels, repeated metadata and data movement, and transient memory growth during long-context prefill. TensorRT LLM addresses these costs through earlier dependent launches, streamlined input preparation, and workload-aware memory control. + +**Programmatic Dependent Launch.** PDL allows a dependent consumer grid to launch before its producer grid fully retires, wait at an explicit dependency point, and continue as soon as the required producer state is ready. TensorRT LLM configures PDL globally for supported DeepGEMM operations and also applies it to the packed 1×128 FP8 quantization, MLA RoPE, sparse-index conversion, sparse FMHA, and fused mHC paths used by DeepSeek-V4. This shortens producer-to-consumer boundaries without changing the model's dependency graph. + +**Metadata preparation.** TensorRT LLM streamlines DeepSeek-V4 attention-metadata preparation by reusing tensors and consolidating the host- and device-side updates required for each step, reducing Python work, kernel launches, and data transfers. The cache manager also builds DeepSeek-V4 block tables on the GPU and uses a dedicated CUDA operator instead of compiling the earlier tensor implementation, reducing host-memory pressure while keeping block-table generation on the device. + +**SWA scratch reuse.** A long prefill step produces KV entries for every token in the input chunk, but sliding-window attention retains only the most recent window after the step. Among the new blocks allocated for the current chunk, blocks that will already be outside the final non-rewindable window are needed only by the currently executing layer. TensorRT LLM maps this scratch-eligible range through per-layer block tables to coalesced scratch subpages. Because these per-layer lifetimes do not overlap, the same physical scratch storage can be reused as execution advances across layers. Pre-existing history from earlier chunks remains in the normal KV cache and is never overwritten by scratch reuse. New blocks in the active window also remain in the normal cache, and the tail that MTP may rewind is excluded from scratch storage. As a result, the cache does not retain the full current prefill chunk independently for every compatible SWA layer, reducing long-context prefill memory consumption and leaving more capacity for larger chunks, longer prompts, or higher concurrency. + +
+
+ SWA scratch reuse during long-context prefill. Scratch-eligible KV blocks produced by the current prefill chunk and falling outside the final retained window reuse the same physical storage across SWA layers, while blocks inside the sliding window remain in the normal KV cache. +
+
+

Figure 3. SWA scratch reuse during long-context prefill. Scratch-eligible blocks from the current prefill chunk reuse the same physical storage across SWA layers, while the final sliding-window state remains in the normal KV cache.

+ +**Bounded Indexer workspace.** During long-context prefill, the Indexer's MQA-logits transient grows with both the query chunk and compressed history. TensorRT LLM controls this allocation at two levels. An outer heuristic reduces the configured prefill chunk when the compressed KV history crosses long-context thresholds. Within each chunk, query tiling imposes a hard per-call element budget so that only a bounded slice of the logits matrix is materialized at a time. Because logits and Top-K are computed independently for each query row, tiling preserves the exact selection result while enabling very long prompts without reducing the model-wide token budget. + +Together, these runtime optimizations help carry the preceding kernel gains through to the full model. Dependent work starts earlier, recurring host-side metadata work is reduced, and long-context memory follows a bounded working set instead of unchecked temporary growth. The next section, [**DeepSeek-V4 Performance Evolution**](#deepseek-v4-performance-evolution), shows how the kernel and runtime improvements accumulate at the full-model level. + +### DeepSeek-V4 Performance Evolution + +The preceding [**DeepSeek-V4 Performance Optimizations**](#deepseek-v4-performance-optimizations) section described the individual attention, Compressor, mHC, MoE, and runtime optimizations. Their real value, however, is whether they move the end-to-end serving frontier after the full system is re-tuned. We therefore tracked a sequence of complete TensorRT LLM Pareto sweeps rather than multiplying speedups from isolated operator benchmarks. Under the same DeepSeek-V4 Pro 8K/1K workload, the maximum observed output throughput increased from 984 to 1,618 tokens/s/GPU between the first validated functional baseline and the latest fully refreshed May 28 snapshot, a **64.5% improvement**. Additional optimizations have since been integrated into the newer DeepSeek-V4 stack, but have not yet been included in a new full Pareto sweep. + +#### Experimental setup + +The primary evaluation uses the InferenceX fixed-length 8K/1K workload, with exactly 8K input tokens and 1K output tokens per request. This shape stresses both sides of disaggregated serving: the context workers must process a substantial prompt, while the generation workers execute enough decode steps for sparse attention, MTP, and MoE efficiency to materially affect the result. + +| Item | Configuration | +| :--- | :--- | +| Model | DeepSeek-V4 Pro, 1.6T total parameters | +| Hardware | NVIDIA GB300 | +| Precision | Mixed-precision MXFP4 Instruct recipe: MXFP8 activations × MXFP4 weights for routed-expert GEMMs, with component-specific precision elsewhere | +| Workload | InferenceX, 8K input tokens and 1K output tokens per request | +| Serving mode | End-to-end disaggregated serving with independently sized context and generation workers | +| Sweep | TEP4/TEP8 for the latency-oriented end; DEP8/DEP16/DEP32 for the throughput-oriented end; MTP-3 for most latency and mid-frontier points and MTP-1 at the highest-concurrency points | +| Metrics | Output throughput per concurrent request stream (tokens/s/user) and output throughput per GPU (tokens/s/GPU) | + +Each curve in Figure 4 is a newly measured system Pareto sweep. A point can therefore change concurrency, MTP depth, parallel strategy, expert placement, and the context-to-generation resource ratio. The curve movement is the meaningful comparison; the peak values should not be interpreted as a component-by-component A/B test at an identical configuration. + +![Four DeepSeek-V4 Pro 8K/1K disaggregated-serving Pareto curves, showing peak output throughput increasing from 984 to 1,618 tokens/s/GPU](../media/tech_blog26_dsv4_performance_evolution_8k1k.png) + +

Figure 4. DeepSeek-V4 Pro 8K/1K disaggregated serving on GB300, using the mixed-precision MXFP4 Instruct recipe with MXFP8 × MXFP4 routed-expert GEMMs. Each line is a complete TensorRT LLM Pareto sweep from that milestone.

+ +#### From Baseline to the Latest Measured Curve + +The four curves capture three distinct phases of optimization. The table reports the highest measured tokens/s/GPU on each sweep; because the maximizing configuration changes as the frontier moves, these values summarize system evolution rather than a strict single-configuration waterfall. + +| Milestone | Peak output throughput (tokens/s/GPU) | Gain from previous | Gain from baseline | Representative changes in this stage | +| :--- | ---: | ---: | ---: | :--- | +| April 28: validated baseline | 984 | — | — | Validated end-to-end disaggregated stack under the common model and workload definition | +| May 8: Top-K and Compressor | 1,167 | +18.6% | +18.6% | Small-batch exact Top-K, BF16 Compressor input and fusion, and disaggregated memory/correctness fixes | +| May 18: MegaMoE and EPLB | 1,505 | +29.0% | +53.0% | MegaMoE, EPLB-enabled DEP placements, fused mHC and RMSNorm, and expanded multi-stream attention | +| May 28: attention and runtime refinements | 1,618 | +7.5% | +64.5% | Deeper attention overlap, the optimized inverse-RoPE and FP8 `o_a_proj` path, PDL, and GPU-side scale-buffer and block-table cleanup | + +**The first measured curve establishes the validated functional baseline.** It represents the complete execution path after functional bring-up: hybrid SWA/CSA/HCA attention, persistent Compressor state, MTP, KV Cache Manager V2, and disaggregated cache transfer. We treat all work required to reach this point as part of the baseline rather than assigning it a performance gain. Every subsequent curve uses the same model, precision recipe, and fixed-shape workload definition. The reported evolution therefore measures optimization of a working system rather than the addition of missing model features. + +**Exact Indexer Top-K and Compressor improvements move the May 8 frontier.** The small-batch multi-CTA path increased GPU utilization for exact Top-K, while BF16 Compressor input and additional Compressor fusion removed conversion, launch, and intermediate-buffer overhead. Runtime fixes were equally important: correcting sliding-window over-allocation improved usable generation-side KV capacity, while the launcher fix ensured that the configured NUMA policy was applied to multi-process disaggregated runs. Together, the full re-tuned sweep raised the peak by 18.6%, from 984 to 1,167 tokens/s/GPU. + +**MegaMoE and EPLB drive the mid-May step.** The largest movement in the system curve came between May 8 and May 18. MegaMoE fused expert dispatch, the two expert GEMMs, activation, and combine into a communication-aware path using symmetric memory and in-kernel synchronization. At the throughput-oriented end of the curve, the May 18 sweep also moved to EPLB-enabled DEP placements to improve expert balance across ranks. In parallel, mHC absorbed the adjacent RMSNorm, and the Indexer and attention paths gained expanded dependency-aware overlap. Together, these changes produced a 29.0% stage-over-stage increase and a 53.0% increase over the initial baseline. Because the maximizing configuration changed across the complete sweeps, this movement is not a single-configuration decomposition of the individual optimizations. + +**Attention overlap and runtime cleanup extend the late-May frontier.** By May 28, deeper Indexer, Compressor, and Q-path overlap reduced exposed attention work. PDL, producer-side FP8 scale-buffer initialization, the optimized inverse-RoPE and FP8 `o_a_proj` input path, and cheaper GPU-side block-table preparation removed additional synchronization, launch, and data-movement overhead. The May 28 curve is roughly 7–8% above May 18 across most throughput-oriented DEP points, lifting the observed peak to 1,618 tokens/s/GPU. This broad movement is more informative than any single microbenchmark: it shows that the optimizations survived full-model scheduling, cache management, and context/generation rate matching. + +May 28 is the latest measured curve, not the current performance ceiling. The newer DeepSeek-V4 stack subsequently integrated sparse-MLA softmax and V-load improvements, fused MegaMoE input preparation, the optimized 128× Compressor prefill reduction, and later attention-epilogue fusion. These changes target attention, MoE preparation, and Compressor overhead, but the complete GB300 8K/1K Pareto sweep has not been rerun with the newer stack. None of their gains is included in the reported 64.5% improvement, and we do not extrapolate a new end-to-end number. We expect a refreshed curve to move further once it is measured under the same methodology. + +The latest measured fixed-shape InferenceX curve establishes the clean model/runtime baseline for [Part II](#part-ii-agentic-workload-optimizations). Agentic workloads change the optimization boundary: dynamic multi-turn prompts, very high KV reuse, conversation locality, routing, and serialization become part of the critical path. The next section, [**AgentPerf Workflow**](#agentperf-workflow), introduces that wider optimization boundary without re-counting the model-specific gains above. + + +## Part II. Agentic-Workload Optimizations + +### AgentPerf Workflow + +[AA-AgentPerf](https://artificialanalysis.ai/methodology/agentperf) measures how many active coding agents an inference deployment can support while meeting a model-specific service level objective (SLO). This [NVIDIA Developer Blog](https://developer.nvidia.com/blog/nvidia-achieves-leading-agentic-coding-performance-on-first-agentic-ai-benchmark/) introduces the benchmark and explains why prerecorded trajectories are needed to represent agentic inference. Unlike the fixed-shape InferenceX workload used in Part I, AA-AgentPerf replays multi-turn coding sessions with interleaved reasoning and tool calls. Each simulated agent advances sequentially and waits for the current response and any client-simulated tool time before sending its next request. + +| Property | Fixed-shape InferenceX | AA-AgentPerf | +| :--- | :--- | :--- | +| Unit of load | Independent request | Active agent following a trajectory | +| Request shape | Fixed input and output lengths | Growing input context and variable output lengths | +| Inter-request dependency | Requests are independent | Each turn waits for the response and any client-simulated tool time | +| Prefix reuse | No accumulated cross-turn history | Accumulated history enables reuse across turns | +| Objective | Throughput and latency at a fixed shape | Highest concurrency satisfying model-specific SLO thresholds | + +The following figure summarizes the closed-loop request path for aggregated and disaggregated serving. + +
+
+ AA-AgentPerf closed-loop workflow. A simulated agent sends the next recorded turn with its accumulated conversation. The inference deployment routes the request, reuses matching KV blocks, prefills the uncached suffix, and streams the generated response. In disaggregated serving, the generation side retrieves the KV cache using context metadata. The client drains the complete response, simulates tool time when needed, advances the trajectory, and sends the next turn only after the current cycle completes. +
+
+

Figure 5. AA-AgentPerf closed-loop request workflow. Repeated conversation prefixes can be reused through the KV cache. Aggregated serving performs prefill and generation on the same worker. In disaggregated serving, context metadata allows the generation worker to retrieve the KV cache and continue decoding. Tool time is simulated by the client and consumes no LLM compute.

+ +This closed loop makes the complete request cycle part of every agent's progress. Repeated prefixes make KV-cache reuse important because later turns can avoid recomputing accumulated history. Growing contexts and variable output lengths also exercise cache capacity and scheduling differently from fixed-shape requests. Simulated tool delays introduce idle periods between turns, while concurrent agents maintain sustained load on the deployment. + +AA-AgentPerf reports the highest concurrency that satisfies both the P25 of per-request output speed and the P95 of per-request time to first token (TTFT). Because the benchmark is continuously updated, this blog uses the [AA methodology](https://artificialanalysis.ai/methodology/agentperf) current at publication for the exact thresholds. We refer to the current DeepSeek-V4 Pro SLO #1 and SLO #2 tiers as **SLO20** and **SLO60**. SLO20 requires at least 20 tokens/s and P95 TTFT at most 10 seconds. SLO60 requires at least 60 tokens/s and P95 TTFT at most 5 seconds. SLO20 emphasizes serving capacity, while SLO60 preserves a faster interactive experience. + +For engineering comparisons, we use concurrency per GPU (CPG). It is the supported concurrency divided by the total number of GPUs in the deployment, including both context and generation GPUs. + +### Lessons from DeepSeek-V3.2 Agentic Workload Optimization + +Before DeepSeek-V4, we optimized DeepSeek-V3.2 for long, multi-turn agentic workloads in both aggregated and disaggregated serving. The durable value of that work was not a particular benchmark result or deployment configuration. It was a set of system-level rules that guided the DeepSeek-V4 effort: + +1. **High reuse creates a different bottleneck from raw GPU throughput.** High prefix reuse can leave only a small uncached suffix to compute while the accumulated conversation still occupies substantial KV-cache capacity. Scheduling and capacity planning should therefore consider the remaining compute tokens and resident KV footprint rather than raw prompt length alone. +2. **Amortize host overhead without pausing generation.** Two complementary approaches addressed this overhead. First, CUDA Graph capture and kernel fusion reduced recurring host and launch work. **Piecewise CUDA Graph capture** covered graphable regions in prefill and mixed prefill-decode iterations when the full iteration could not be captured as one graph. Generation-only iterations continued to use the regular monolithic CUDA Graph path. Second, context-only **Delay batching** briefly held small context requests so newly arriving context requests could join a larger batch. Generation requests continued to run during this wait. +3. **Treat routing as a trajectory-level balance between locality and load.** Later turns benefit from returning to the workers and ranks that hold their conversation prefix. That affinity must be balanced against load so that preserving reuse does not create persistent rank imbalance. +4. **Measure disaggregated serving end to end.** Front-end preprocessing and KV-cache movement can limit throughput even when context and generation GPUs appear underutilized. Asynchronous preprocessing, parallel data movement, and per-stage observability are therefore part of serving performance. + +DeepSeek-V4 changed the model path, cache layout, and serving control plane, so the earlier configuration could not simply be reused. We carried forward these rules rather than the exact parallel strategy, cache settings, batching parameters, or transfer settings. The following section applies them to DeepSeek-V4 routing, KV lifecycle, host overhead, and scheduling. + +### End-to-End Optimizations for Agentic Serving + +Starting from the final model/runtime stack in [Part I](#part-i-deepseek-v4-model-support-and-optimizations), this section applies the measurement methodology and system-level principles learned from [DeepSeek-V3.2](#lessons-from-deepseek-v32-agentic-workload-optimization) to DeepSeek-V4's AgentPerf-specific request path, without counting the model-specific gains again. In this workload, a routing miss can force a long conversation prefix to be recomputed, erasing gains from faster kernels. Hashing that same prefix token by token in Python can also serialize the orchestrator's request path. These bottlenecks expand the optimization boundary beyond GPU kernels. The remaining work spans three coupled areas: preserving locality through routing, keeping reusable KV blocks resident through lifecycle and capacity management, and removing overhead across the worker host paths, scheduler, orchestrator, and CTX-to-GEN protocol once reuse reduces the GPU work per turn. + +#### Preserve locality with two-level routing + +The DeepSeek-V3.2 study showed the central routing tradeoff: affinity without load control creates imbalance, while load balancing that overrides affinity destroys reuse. DeepSeek-V4 applies that lesson through two consecutive placement decisions. The front-end orchestrator first selects a context (CTX) server instance. The attention data-parallel (ADP) router then selects a rank within that instance. At either level, the router can use an explicit conversation identifier or infer locality from KV-cache contents. Reuse is preserved only when both decisions return a later turn to the server and rank that own its KV blocks. Instance-level affinity cannot compensate for a rank-level miss, and the correct rank is unreachable after the wrong server has been selected. + +
+
+ Two-level routing for context locality in disaggregated serving. The front-end orchestrator first selects a context server, then the selected server's attention data-parallel router selects a rank. At each level, the router can use either conversation-aware affinity or KV-cache-aware scoring. Full conversation-prefix reuse requires both decisions to return to the context server and rank that own the corresponding KV blocks. The selected context rank transfers the KV cache to a generation server for decoding. +
+
+

Figure 6. Two-level routing for context locality in disaggregated serving. The front-end orchestrator first selects a CTX server, and that server's ADP router then selects a rank. Conversation-aware routing uses explicit conversation affinity, while KV-cache-aware routing infers placement from reusable prefix blocks and load. Full conversation-prefix reuse requires both decisions to return the request to the CTX server and ADP rank that hold the corresponding KV blocks.

+ +##### Instance-level routing across context servers + +The first routing level distributes conversations across CTX server instances. It must preserve locality for later turns while balancing new conversations across servers. + +**Conversation-aware routing.** When the client provides a stable conversation identifier, the orchestrator assigns the first turn to the least-loaded CTX server, using round-robin among equally loaded candidates, and records the binding. For each later turn, the orchestrator uses that binding to send the request back to the same server, preserving reuse of the conversation prefix cached there as long as the server remains available. The affinity decision then becomes a session-table lookup, avoiding full-prompt tokenization and block-hash computation on the front-end event loop. + +**KV-cache-aware routing.** When a reliable conversation identifier is unavailable, the orchestrator infers locality by tokenizing the prompt, deriving its block keys, estimating the reusable prefix on each CTX server, and balancing reuse against current load. We made three improvements to keep this path efficient and stable. First, incremental tokenization reuses the token IDs for the stable prompt prefix and encodes only the appended suffix. The orchestrator passes those token IDs to the worker and applies the same tool definitions and chat-template arguments, avoiding repeated tokenization while ensuring that both sides derive the same block keys. Second, we refined candidate scoring, which selects a CTX server by weighing its reusable prefix against its current load. Instead of normalizing the cache match by the full prompt length, the new score uses the absolute number of matched blocks and a separate active-request penalty. This prevents the long system and tool prefix shared across requests from diluting the score advantage contributed by conversation-specific blocks. Third, we strengthened affinity when temporary cache eviction weakens the cache-match signal. A bounded mapping derived from stable early conversation content remembers the selected CTX server and continues to direct later turns to it even if eviction temporarily lowers the server's cache-match score. + +Cache-aware routing also requires the orchestrator to maintain a compatible and sufficiently current view of each worker's cache. Three design choices make that view both reliable and inexpensive to maintain. First, the orchestrator and workers use the same hash algorithm and cache namespace. An optional cache salt acts as a namespace identifier, allowing KV reuse only between requests that carry the same value. Second, the orchestrator adopts the worker's block size by default. When it tracks cache at a finer supported granularity, completed-request backfill adds the routed block keys to the selected server's cache view after successful execution, which lets the orchestrator track newly cached prefixes at its own granularity without translating between incompatible event boundaries. Third, cache metadata updates are batched, and worker events are refreshed in the background to keep this maintenance off the routing critical path. Together, these changes keep cache tracking useful without turning it into a new routing bottleneck. + +##### Rank-level routing within a context server + +After a CTX server has been selected, the second routing level places the request on one of its ADP ranks. It must preserve the exact rank that owns the reusable prefix while keeping new conversations distributed across the instance. + +**Conversation-aware routing.** The ADP router records a bounded `conversation ID → rank` mapping. It assigns the first turn of each new conversation by round-robin among ranks below a loose fair-share target, then pins later turns to the recorded rank. A returning conversation may exceed that soft target because moving it would discard rank-local reuse. Affinity is relaxed only when the assigned rank reaches the hard active-request limit. An overflow request does not overwrite the original binding, which allows a later turn to return when the rank has capacity again. + +**KV-cache-aware routing.** Without a reliable conversation ID, the ADP router must infer rank locality from the KV blocks that are actually present. For each new request, every rank probes its local radix tree to measure the reusable prefix without reserving any cache blocks. The probe uses the request's cache salt so that it searches the same cache namespace as the KV cache manager. The ranks then exchange their prefix-match lengths and current loads. For each candidate rank, the router estimates the remaining prefill work as the input length minus the matched prefix, then adds a normalized load penalty. A meaningful prefix match therefore favors the rank that can reuse it, while a weak match caused only by the shared system and tool scaffold is ignored in favor of load balancing. A loose fair-share cap provides a second guard against concentrating too many requests on a few warm ranks. When several related requests arrive in the same scheduling batch, the router groups them by their early prefix and places longer requests first. After each assignment, it updates the selected rank's load using only the tokens that still require computation. The next request is therefore routed against the work already placed in that batch rather than a stale load snapshot. + +**Protect rank balance.** A new conversation contains the shared system and tool scaffold but little conversation-specific history. Without explicit cold-start handling, the first few ranks that cache this common prefix can attract more new conversations, while the remaining ranks stay cold. Conversation-aware routing prevents this feedback loop by distributing first turns with round-robin before pinning later turns. KV-cache-aware routing can instead use an optional warmup phase that sends initial requests to ranks not yet selected, allowing every rank to cache the shared prefix before normal scoring begins. + +##### Routing policy selection + +Our evaluation results show that, with completed-request backfill, the cache-aware path achieved a cache hit rate comparable to conversation-aware routing, but required additional preprocessing and cache tracking. The two approaches therefore offer different ways to preserve locality: one uses explicit conversation metadata, while the other infers locality from cache contents. + +#### KV cache reuse optimizations + +Two-level routing keeps successive turns of the same conversation on the CTX server and ADP rank that hold the reusable prefix. Placement alone is not enough. The prefix must also remain resident until the next turn arrives. Analysis of early DeepSeek-V4 AgentPerf runs showed substantial optimization headroom in cache memory management. DeepSeek-V4 divides cache memory among the sliding-window cache and several persistent compressed caches with different storage costs and demand profiles, so reuse depends on both the cache lifecycle and how memory is budgeted across pools. We first added per-pool cache metrics to locate pressure, then used those signals to optimize the sliding-window cache lifecycle and tune cache pool ratios. + +**Add per-pool cache metrics.** An overall cache hit rate shows how much reuse was achieved, but it does not reveal which pool was constrained or whether the cause was sustained pressure or a short-lived capacity spike. We added per-pool telemetry for occupancy, allocation and commit volume, cached KV length, and data movement between HBM and host memory. Reuse counters distinguish fully reused, partially reused, and missed blocks. Peak watermarks and complete iteration histories capture transient pressure that periodic summaries can miss, while memory probes expose the HBM headroom before and after model loading. Host-tier gauges further distinguish evictable blocks from blocks dropped before reuse. Together, these signals identify the constrained pool, distinguish incorrect lifecycle retention from insufficient pool capacity, and provide the evidence for the next two optimizations. + +**Optimize the sliding-window cache lifecycle.** As described in [**Cache management and runtime features**](#cache-management-and-runtime-features), DeepSeek-V4's sliding-window cache includes SWA KV and short-lived Compressor KV/score state. These buffers should remain bounded by their active windows rather than grow with the full prompt. The previous reuse policy retained SWA KV across the full prompt, even though a later agent turn needs only the blocks within the active window. We introduced the `per_request` policy to select the sliding-window state needed by the next turn when the current prefill request completes, allowing older pages to be recycled. The same retention issue appeared at the CTX-to-GEN handoff in disaggregated serving. Before this optimization, GEN allocated sliding-window cache for the full transferred prompt, including blocks outside the active window. We changed GEN initialization to allocate cache only for blocks within the active window. Together, these changes keep the sliding-window cache bounded during prefill and disaggregated handoff. Persistent compressed caches still retain the long-term prefix and continue to grow with sequence length. + +**Tune cache pool ratios.** Correcting the cache lifecycle removes stale allocations, but reusable prefixes can still be evicted when the available HBM is divided poorly across pools. The two cache categories respond differently to workload shape. Sliding-window cache pressure is driven by the active-request count, while persistent compressed caches grow with retained sequence length and the number of concurrent conversations. Each allocation must be satisfied by its own pool group, so free pages in one group cannot cover a shortage in another. A split tuned for a different request-length distribution or serving role can therefore exhaust one pool and evict reusable blocks even while another pool has unused capacity. To address this imbalance, we introduced configurable pool-ratio controls for DeepSeek-V4. On CTX servers, prefix reuse makes pool demand highly dependent on the workload's prefix-reuse pattern, which the runtime cannot infer reliably. Users should therefore tune `pool_ratio` manually using the per-pool metrics described above. On GEN servers, users can set `avg_seq_len` to the workload's average total sequence length, allowing KV Cache Manager V2 to derive the initial pool ratio automatically. These controls direct capacity to the pools that need it and avoid evicting reusable history while memory remains unused elsewhere. An opt-in beta rebalancer can also adjust the split from runtime statistics. It is disabled by default and currently supports a narrower set of aggregated-serving configurations. Extending this mechanism to broader deployments remains ongoing work. + +**Host offloading and incremental transfer.** We also evaluated host KV offloading and incremental CTX-to-GEN transfer. Compared with models that retain full K/V history for every token, DeepSeek-V4 stores long-term history in compressed attention and Indexer caches while keeping raw-token KV and Compressor state within bounded sliding windows. Its KV-cache footprint per input token is therefore relatively small. This compact representation keeps more reusable KV blocks resident, leaving less additional reuse for host offloading to recover. It also reduces the CTX-to-GEN payload, limiting the data movement that incremental transfer can overlap. Neither mechanism therefore produced a meaningful end-to-end improvement for this AgentPerf configuration. The evaluation is still useful for workloads with larger KV-cache footprints or higher transfer volumes. + +#### Optimize scheduling and remove end-to-end overhead + +Routing and cache management make later turns with reusable prefixes much cheaper on the GPU. That saving reaches the user only if the rest of the serving path also avoids processing the full long prompt again. Profiling showed substantial host work during CTX prefill and a persistent gap between GEN inter-token latency and the speed-of-light (SOL) estimate. Long token arrays were still being hashed, copied, tokenized, serialized, and broadcast at several boundaries. We addressed this gap inside both the workers and the serving control plane. First, we removed host work from the CTX and GEN critical paths. We then optimized scheduling, orchestration, and protocol handling. + +##### Optimize host overhead for CTX and GEN + +Effective prefix reuse reduces the GPU work to the uncached suffix, but the surrounding host path can still scale with the full prompt. This is especially costly when the work runs in Python or is repeated on every distributed rank. We therefore moved per-token loops out of Python where possible and skipped host operations that could not produce useful GPU work. + +**CTX host path.** Profiling showed that CTX was paying host work at the wrong granularity. Block-key construction paid a conversion and SHA-256 update for every token, turning a long prompt into many small operations before prefill could begin. Distributed coordination was paid once per iteration, even when no new request was available. Those empty iterations could still enter prefix-gather and request-broadcast collectives. The first cost grew with the full prompt, while the second could create host bubbles without producing any GPU work. We changed each operation to follow the unit of useful work. Block-key construction now packs one block of tokens into a contiguous byte buffer and updates the hash once per block. All ranks perform a lightweight request-count probe before entering the larger collectives. If no new request exists, they take the same fast path and skip both prefix gathering and request broadcast. + +**GEN host path.** On GEN, profiling did not point to one dominant function. Instead, host time accumulated through three prompt-proportional costs as each long-prompt request crossed the CTX-to-GEN handoff, entered GEN, and was distributed across the ranks. First, GEN could repeat tokenization already completed by CTX. Second, request admission and routing repeatedly copied or materialized the full token list. Third, interprocess request transfer serialized token IDs as individual Python objects before rebuilding the full list on the receiving side. The latter two costs occurred on request-ingress or executor-loop paths while holding the Python GIL, so they could delay the next generation iteration. To reduce this accumulated host time, we introduced a set of targeted optimizations, each addressing one of these costs directly. (a) CTX now returns the prompt token IDs that it already computed, allowing GEN to bypass tokenization. (b) During admission, enqueue no longer deep-copies the token list, while routing obtains the token count without materializing the full list. (c) Serialization and C++ request construction use a compact token representation, avoiding per-token Python objects and full-list reconstruction. + +##### Optimize scheduling, orchestration, and protocol handling + +After reducing host work inside CTX and GEN, latency could still accumulate before a request entered a worker and while data moved between workers. We treated this serving control plane as three related problems. Scheduling determines which ready request runs next. The orchestrator performs routing and relays data between CTX and GEN. The protocol must minimize redundant payload while preserving request identity when retries occur. + +**Scheduling.** Under high concurrency, a GEN worker may have to choose between newly admitted requests waiting for their first token and requests that have already begun decoding. When the batch or token budget is tight, placing both groups in the same queue can leave a new request behind many ongoing conversations for several scheduling iterations. We added an opt-in policy for disaggregated GEN that gives first-token requests priority while preserving FIFO order within both groups. This can move a newly admitted request into an earlier feasible batch and reduce the queueing delay before its first decode step without adding GPU work. The policy is designed to improve first-token responsiveness and tail-latency SLO compliance rather than raw throughput. + +**Orchestrator.** Profiling exposed two sources of per-request overhead. First, **native block-key hashing** removed the routing hot spot. The instance-level KV router originally hashed every token ID in a Python loop on its single asyncio thread. This held the Global Interpreter Lock (GIL) and serialized concurrent requests before they could reach CTX. Replacing the loop with a native C++ block-key hasher preserved the cache-key format while moving prompt-length hashing out of the Python event loop. Second, **efficient relay serialization** reduced request-body processing when a request crossed the HTTP boundaries to CTX and GEN. The default path previously materialized a JSON-compatible Python object containing the full prompt-token list, after which the HTTP client traversed that object again to encode the request body. We changed this path to use `pydantic-core` to emit the JSON body directly. For deployments where request-body processing remains a bottleneck, an opt-in `msgspec` MessagePack transport provides an alternative internal orchestrator-to-worker path that removes JSON encoding and parsing from this hop. The external OpenAI API and the default direct-JSON path remain unchanged. + +**CTX-to-GEN protocol.** The CTX-to-GEN protocol had two issues: it could carry more request state than GEN needed, and a retried request could leave CTX and GEN with inconsistent identifiers. In supported text-only, non-Harmony deployments, an opt-in path removes earlier chat history from the GEN request while preserving the prompt token IDs, final message, tool definitions, and generation settings that GEN still needs. This reduces copying, serialization, and transfer without changing the request semantics supported by that path. We also hardened the retry path so that known transient connection failures do not leave CTX and GEN with inconsistent request identifiers. Together, these changes reduce protocol overhead while preserving consistency across the disaggregated handoff. + +Across the host and serving-control-plane paths, these changes reduce prompt-proportional processing, redundant serialization, and avoidable synchronization between GPU iterations. Together with routing and cache management, they allow prefix reuse to translate into end-to-end performance. The next section, [**AgentPerf Results and External Validation**](#agentperf-results-and-external-validation), presents the combined end-to-end results under frozen configurations and evaluates them against the SLO20 and SLO60 objectives. + +### AgentPerf Results and External Validation + +The previous section described how routing, KV-cache policy, host-path cleanup, scheduling, and orchestration removed distinct end-to-end bottlenecks. This section presents the DeepSeek-V4 Pro AgentPerf evaluation setup and the results reported by Artificial Analysis. We keep these agent-workload results separate from the model-specific results in [Part I](#part-i-deepseek-v4-model-support-and-optimizations) and report the best-performing configurations we developed for those AA-AgentPerf measurements. + +The GB300 NVL72 configurations served DeepSeek-V4 Pro at maximum thinking effort using TensorRT LLM with disaggregated serving. Several context (CTX) workers fed one generation (GEN) worker through NIXL KV-cache transfer. Each CTX worker used four GPUs with attention data parallelism and MoE expert parallelism. + +To select the end-to-end topology, we followed the speed-of-light (SOL) rate-matching workflow introduced in the [TensorRT LLM disaggregated serving tech blog](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog05_Disaggregated_Serving_in_TensorRT-LLM.md#measurement-methodology). We first measured CTX request throughput across candidate parallel mappings and batch settings that could satisfy the TTFT target, and independently measured GEN output throughput and per-agent output speed across mappings, batch sizes, and concurrency levels. We then matched the aggregate service rates of the two tiers to construct an idealized SOL Pareto curve. The final SLO20 configuration used DEP8 with a larger GEN batch to maximize capacity. The final SLO60 configuration used DEP16 with a smaller GEN batch to preserve higher per-agent decode speed. SOL assumes perfect rate matching and excludes practical end-to-end costs such as KV-cache transfer, routing, host work, and workload burstiness, so it served as a configuration-selection reference rather than a reported AgentPerf result. + +The same rate-matching calculation provided the starting `xPyD` ratio, where `xPyD` denotes `x` CTX/prefill instances feeding `y` GEN/decode instances. For each selected GEN operating point, we converted its output-token throughput into a request-equivalent service rate using the workload's output-length distribution, then compared it with the measured per-instance CTX request rate to estimate the ideal number of CTX instances. Because this ratio can be fractional while a deployment requires whole workers, we evaluated the neighboring integer `xPyD` topologies that fit the GPU budget. We selected the topology that provided the best end-to-end balance while satisfying both the output-speed and TTFT requirements, then swept concurrency to find its maximum passing load. + +The configurations used DEP4 CTX workers with `max_batch_size: 128` and the TensorRT LLM MoE backend. The capacity-oriented SLO20 configuration used a DEP8 GEN worker with `max_batch_size: 192`. The higher-speed SLO60 configuration used a DEP16 GEN worker with `max_batch_size: 32` and an offline EPLB map. Both GEN workers used the DeepGEMM MegaMoE backend. Both deployments used the MXFP4 recipe, MTP-3, FP8 KV cache, conversation-affinity routing at the orchestrator and ADP-rank levels, and heuristic GVR Top-K. + +As defined in [**AgentPerf Workflow**](#agentperf-workflow), we use concurrency per GPU (CPG) to compare deployment configurations. The GPU count here includes both the CTX and GEN tiers, and only points that pass the target SLO are reported. The Artificial Analysis article, [First results from AA-AgentPerf: the hardware benchmark for the agent era](https://artificialanalysis.ai/articles/aa-agentperf/), reports the GB300 configurations provided by NVIDIA and run and verified by Artificial Analysis. These results validate the complete workload path rather than an isolated TensorRT LLM microbenchmark. Artificial Analysis replayed the agent trajectories and applied its SLO search and steady-state methodology. The associated [serving-configuration browser](https://artificialanalysis.ai/benchmarks/hardware/configs) discloses the precision, topology, worker YAML, routing mode, MTP setup, cache policy, transfer backend, and launch commands. + +The GB300 results reached 57.5 CPG at SLO20 and 19.2 CPG at SLO60: + +| Target | CTX/GEN topology | Passing concurrency | GPUs used | CPG | Provenance | +| :--- | :--- | ---: | ---: | ---: | :--- | +| SLO20 | 6 × CTX DEP4 + 1 × GEN DEP8 | 1,840 | 32 | 57.5 | NVIDIA configuration, run and verified by AA | +| SLO60 | 6 × CTX DEP4 + 1 × GEN DEP16 | 768 | 40 | 19.2 | NVIDIA configuration, run and verified by AA | + +These optimizations are now also available in [NVIDIA Dynamo](https://github.com/ai-dynamo/dynamo) for readers looking to deploy in production. + +## Reproduction and Future Work + +### How to reproduce + +TensorRT LLM already provides the common operational steps for launching disaggregated serving. Follow the [disaggregated serving tech blog](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog05_Disaggregated_Serving_in_TensorRT-LLM.md) and the [disaggregated serving guide](https://nvidia.github.io/TensorRT-LLM/features/disagg-serving.html) for environment setup, CTX and GEN worker startup, NIXL KV-cache transfer, and orchestrator launch. Both result sets below build on this foundation but use different workload-specific references. The InferenceX path uses the linked GB300 recipes to reproduce the fixed-shape 8K/1K Pareto sweeps. The AA-AgentPerf path starts from the current TensorRT LLM `main` branch to evaluate the latest optimized stack and uses the linked AA configuration entries and methodology for the submitted topology, settings, and SLO evaluation. + +**InferenceX 8K/1K.** Reproduce the fixed-shape results from [**DeepSeek-V4 Performance Evolution**](#deepseek-v4-performance-evolution) with the [GB300 MXFP4 ISL8K/OSL1K MTP recipes](https://github.com/NVIDIA/srt-slurm/tree/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP). The directory contains the CTX/GEN topology sweep, MTP settings, benchmark concurrencies, and worker configurations used for the 8K/1K measurements. + +**AA-AgentPerf.** For the results reported above, use the corresponding entries in the [Artificial Analysis configuration browser](https://artificialanalysis.ai/benchmarks/hardware/configs). The 57.5-CPG SLO20 result uses the [GB300 DEP8 configuration](https://artificialanalysis.ai/benchmarks/hardware/configs#config-60987f21-a7db-4a8e-8d3a-810a7527de2f), while the 19.2-CPG SLO60 result uses the [GB300 DEP16 configuration](https://artificialanalysis.ai/benchmarks/hardware/configs#config-8e07c398-605a-457f-96c4-bb002691de79). Each entry provides the CTX and GEN worker YAML, disaggregated server configuration, topology, routing settings, and launch commands needed to reproduce the deployment setup. Use the [AA-AgentPerf methodology](https://artificialanalysis.ai/methodology/agentperf) for the benchmark procedure and SLO evaluation. + +### Future Work + +Our next phase continues to optimize DeepSeek-V4 as a complete serving system: improve the low-precision GPU execution path, reduce host and control-plane overhead, broaden hardware coverage, and expand speculative decoding support. These directions need to advance together, because faster model execution raises end-to-end capacity only when cache management, routing, and orchestration can keep the GPUs supplied efficiently. The following work is already underway. + +**Advance the NVFP4 execution path.** We are moving the performance path to the NVFP4 DeepSeek-V4 checkpoint, continuing to tune the NVFP4 MegaMoE backend, and evaluating a more aggressive quantization recipe that applies NVFP4 to additional linear layers. This work treats the checkpoint, kernel coverage, and quantization recipe as one end-to-end path. + +**Fuse more small kernels.** We will continue combining adjacent small operations to reduce intermediate memory traffic and the number of kernel launches. These fusions target both GPU execution time and the host launch overhead that remains visible in small effective batches. + +**Move the KV Cache Manager V2 hot path to C++.** KV Cache Manager V2 provides the flexible multi-pool model required by DeepSeek-V4, but its performance-critical control path is still primarily implemented in Python. Migrating this path to C++ will reduce GIL contention and recurring host overhead, which become especially visible when high cache reuse leaves only a small amount of GPU work per turn. + +**Optimize and scale the serving control plane.** Conversation-aware and KV-aware routing still have room to reduce TTFT and CPU overhead while preserving cache locality and balanced placement. At larger deployment scales, a multiprocess orchestrator with a coordinator-and-worker architecture is being developed to address the point at which a single orchestrator becomes the bottleneck. We will optimize these paths as one control plane, resolve the current multiprocess TTFT issues, and validate the complete design under high-concurrency, multi-turn workloads. + +**Add DeepSeek-V4 DSpark support.** We are integrating the DSpark speculative-decoding path for DeepSeek-V4 and validating it against the MTP baseline. The remaining work includes measuring acceptance behavior, tuning the runtime configuration, and quantifying the resulting end-to-end gain. + +**Extend DeepSeek-V4 support to Hopper.** The current performance work is centered on Blackwell. We are extending the model and runtime paths required for complete DeepSeek-V4 execution on Hopper, with platform-appropriate kernels and precision support. + +## Conclusion and Acknowledgments + +DeepSeek-V4 optimization in TensorRT LLM is an end-to-end systems effort. Building a production-ready model stack required correctness across hybrid attention, mHC, MTP, MoE, quantization, cache management, and disaggregated execution. Moving the performance frontier then required coordinated work from GPU kernels and execution overlap through KV reuse, host efficiency, routing, orchestration, configuration tuning, and benchmark methodology. The result is a stack that improves both fixed-shape inference and long-running agentic serving, demonstrating that delivered performance comes from optimizing the model and the serving system together. + +This work is the result of close collaboration across many teams and every layer of the inference stack. We sincerely thank everyone who contributed to model bring-up, accuracy validation, benchmark optimization, and infrastructure support. The results shared here came from repeated cycles of analysis, implementation, profiling, debugging, validation, and configuration tuning. We are grateful for the expertise, persistence, and teamwork that turned DeepSeek-V4 into a production-ready, high-performance TensorRT LLM stack. diff --git a/docs/source/commands/trtllm-bench.rst b/docs/source/commands/trtllm-bench.rst index fee60a9ab70c..309422df3fba 100644 --- a/docs/source/commands/trtllm-bench.rst +++ b/docs/source/commands/trtllm-bench.rst @@ -20,10 +20,10 @@ Syntax Dataset preparation ------------------ -prepare_dataset.py -^^^^^^^^^^^^^^^^^^ +prepare-dataset +^^^^^^^^^^^^^^^ -trtllm-bench is designed to work with the `prepare_dataset.py `_ script, which generates benchmark datasets in the required format. The prepare_dataset script supports: +trtllm-bench ships a ``prepare-dataset`` subcommand which generates benchmark datasets in the required format. It supports: **Dataset Types:** @@ -38,17 +38,17 @@ trtllm-bench is designed to work with the `prepare_dataset.py prepare-dataset [OPTIONS] **Options** @@ -60,12 +60,10 @@ prepare_dataset * - Option - Description - * - ``--tokenizer`` - - Tokenizer directory or HuggingFace model name (required) * - ``--output`` - Output JSON filename (default: preprocessed_dataset.json) * - ``--stdout`` - - Print output to stdout with JSON dataset entry on each line (**required for trtllm-bench**) + - Print output to stdout with a JSON dataset entry on each line instead of writing a file * - ``--random-seed`` - Random seed for token generation (default: 420) * - ``--task-id`` @@ -77,14 +75,14 @@ prepare_dataset * - ``--log-level`` - Logging level: info or debug (default: info) -dataset -""""""" +real-dataset +"""""""""""" Process real datasets from various sources. .. code-block:: bash - python prepare_dataset.py dataset [OPTIONS] + trtllm-bench --model prepare-dataset real-dataset [OPTIONS] **Options** @@ -108,14 +106,14 @@ Process real datasets from various sources. - Input format: json, jsonl, csv, or txt (default: auto-detect) -token_norm_dist +token-norm-dist """"""""""""""" Generate synthetic datasets with normal token distribution. .. code-block:: bash - python prepare_dataset.py token_norm_dist [OPTIONS] + trtllm-bench --model prepare-dataset token-norm-dist [OPTIONS] **Options** @@ -139,14 +137,14 @@ Generate synthetic datasets with normal token distribution. - Normal distribution standard deviation for output tokens (required) -token_unif_dist +token-unif-dist """"""""""""""" Generate synthetic datasets with uniform token distribution .. code-block:: bash - python prepare_dataset.py token_unif_dist [OPTIONS] + trtllm-bench --model prepare-dataset token-unif-dist [OPTIONS] **Options** diff --git a/docs/source/developer-guide/ci-overview.md b/docs/source/developer-guide/ci-overview.md index 908315c3e3c1..a272a0018df3 100644 --- a/docs/source/developer-guide/ci-overview.md +++ b/docs/source/developer-guide/ci-overview.md @@ -98,8 +98,8 @@ Each line contains the fully qualified test name followed by an optional specific hardware family. Example: ```text -examples/test_openai.py::test_llm_openai_triton_1gpu SKIP (https://nvbugspro.nvidia.com/bug/4963654) -full:GH200/examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (arm is not supported) +accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] SKIP (https://nvbugs/6120535) +full:A100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) ``` Changes to `waives.txt` should include a bug link or brief explanation so other @@ -109,8 +109,17 @@ developers understand why the test is disabled. ### Triggering Post-merge tests -When you only need to verify a handful of post-merge tests, avoid the heavy -`/bot run --post-merge` command. Instead, specify exactly which stages to run: +Full `/bot run --post-merge` runs require the `ci: post-merge approved` PR +label because they can consume substantial shared GPU resources. The label is +intended to be applied by an active member of the +`NVIDIA/trt-llm-ci-approvers` GitHub team. A GitHub workflow validates the label +actor and normally removes invalid approvals. This is a best-effort resource +governance guard, not a strict authorization boundary. The label remains in +place when new commits are pushed and can be removed manually when the approval +no longer applies. + +When you only need to verify a handful of post-merge tests, specify exactly +which stages to run: ```bash /bot run --stage-list "stage-A,stage-B" @@ -123,8 +132,16 @@ default pre-merge set: /bot run --extra-stage "stage-A,stage-B" ``` -Both options accept any stage name defined in `jenkins/L0_Test.groovy`. Being -selective keeps CI turnaround fast and conserves hardware resources. +Both options accept stage names and wildcard patterns defined in +`jenkins/L0_Test.groovy`. Explicit Post-merge stage test doesn't need +`ci: post-merge approved` PR label to run. However, the `"*"`, +`"*Post-Merge*"`, and `"*PerfSanity*"` selectors require the same approval +label, including when they appear in a comma-separated list. Equivalent escaped +or repeated-star forms are treated the same. Other stage selectors, including +explicit stage names and other limited wildcard patterns, retain their existing +behavior. + +Being selective keeps CI turnaround fast and conserves hardware resources. ### Avoiding unnecessary `--disable-fail-fast` usage diff --git a/docs/source/developer-guide/overview.md b/docs/source/developer-guide/overview.md index af7f44f139cf..d8fe31631612 100644 --- a/docs/source/developer-guide/overview.md +++ b/docs/source/developer-guide/overview.md @@ -101,7 +101,6 @@ Module names longer than 8 characters are abbreviated to fit the fixed-width tag | `deep_ep` | `deep_ep ` | | `deep_gemm` | `deepgemm` | | `executor` | `executor` | -| `executor_worker` | `exec_wkr` | | `flash_mla` | `flashmla` | | `kernels` | `kernels ` | | `layers` | `layers ` | diff --git a/docs/source/developer-guide/perf-benchmarking.md b/docs/source/developer-guide/perf-benchmarking.md index 4eb04eef4d13..5903c12e32ea 100644 --- a/docs/source/developer-guide/perf-benchmarking.md +++ b/docs/source/developer-guide/perf-benchmarking.md @@ -171,7 +171,7 @@ can simply read a line and assume a complete entry. When creating a dataset, be JSON entry is on every line. ``` -In order to prepare a synthetic dataset, you can use the provided script in the `benchmarks/cpp` +In order to prepare a synthetic dataset, you can use the provided script in the `benchmarks` directory. For example, to generate a synthetic dataset of 1000 requests with a uniform ISL/OSL of 128/128 for [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B), run: diff --git a/docs/source/developer-guide/perf-overview.md b/docs/source/developer-guide/perf-overview.md index 223ac5e8e92b..58de0107b3d8 100644 --- a/docs/source/developer-guide/perf-overview.md +++ b/docs/source/developer-guide/perf-overview.md @@ -268,7 +268,7 @@ Testing was performed using the PyTorch backend - this workflow does not require | Stage | Description | Command | | :- | - | - | -| [Dataset](#preparing-a-dataset) | Create a synthetic dataset | `python benchmarks/cpp/prepare_dataset.py --tokenizer=$model_name --stdout token-norm-dist --num-requests=$num_requests --input-mean=$isl --output-mean=$osl --input-stdev=0 --output-stdev=0 > $dataset_file` | +| [Dataset](#preparing-a-dataset) | Create a synthetic dataset | `trtllm-bench --model $model_name prepare-dataset --output $dataset_file token-norm-dist --num-requests=$num_requests --input-mean=$isl --output-mean=$osl --input-stdev=0 --output-stdev=0` | | [Run](#running-the-benchmark) | Run a benchmark with a dataset | `trtllm-bench --model $model_name throughput --dataset $dataset_file --backend pytorch --config $llm_options` | ### Variables @@ -281,18 +281,18 @@ Testing was performed using the PyTorch backend - this workflow does not require | `$pp_size` | Pipeline parallel mapping degree to run the benchmark with | | `$ep_size` | Expert parallel mapping degree to run the benchmark with | | `$model_name` | HuggingFace model name eg. meta-llama/Llama-2-7b-hf or use the path to a local weights directory | -| `$dataset_file` | Location of the dataset file generated by `prepare_dataset.py` | +| `$dataset_file` | Location of the dataset file generated by `trtllm-bench prepare-dataset` | | `$num_requests` | The number of requests to generate for dataset generation | | `$seq_len` | A sequence length of ISL + OSL | | `$llm_options` | (optional) A yaml file containing additional options for the LLM API | ### Preparing a Dataset -In order to prepare a dataset, you can use the provided [script](source:benchmarks/cpp/prepare_dataset.py). +In order to prepare a dataset, use the `trtllm-bench prepare-dataset` subcommand. To generate a synthetic dataset, run the following command: ```shell -python benchmarks/cpp/prepare_dataset.py --tokenizer=$model_name --stdout token-norm-dist --num-requests=$num_requests --input-mean=$isl --output-mean=$osl --input-stdev=0 --output-stdev=0 > $dataset_file +trtllm-bench --model $model_name prepare-dataset --output $dataset_file token-norm-dist --num-requests=$num_requests --input-mean=$isl --output-mean=$osl --input-stdev=0 --output-stdev=0 ``` The command will generate a text file located at the path specified `$dataset_file` where all requests are of the same diff --git a/docs/source/developer-guide/telemetry.md b/docs/source/developer-guide/telemetry.md index fc0e1fc8bbe1..90d9f3b9c7d8 100644 --- a/docs/source/developer-guide/telemetry.md +++ b/docs/source/developer-guide/telemetry.md @@ -28,7 +28,7 @@ unset or when the safety sanitizer rejects the runtime value. ### `TorchLlmArgs` -261 captured fields. +270 captured fields. | Captured key | Annotation | Kind | Converter | Allowed values | |--------------|------------|------|-----------|----------------| @@ -43,6 +43,7 @@ unset or when the safety sanitizer rejects the runtime value. | `attention_dp_config.kv_cache_routing_load_balance_weight` | `` | `value` | | | | `attention_dp_config.kv_cache_routing_match_rate_threshold` | `` | `value` | | | | `attention_dp_config.kv_cache_routing_max_sessions` | `` | `value` | | | +| `attention_dp_config.kv_cache_routing_new_conv_placement` | `Literal['round_robin', 'least_queued']` | `categorical` | | `round_robin`, `least_queued` | | `attention_dp_config.timeout_iters` | `` | `value` | | | | `attn_backend` | `` | `categorical` | allowlist | `VANILLA`, `TRTLLM`, `FLASHINFER`, `FLASHINFER_STAR_ATTENTION` | | `backend` | `Literal['pytorch']` | `categorical` | | `pytorch` | @@ -71,6 +72,7 @@ unset or when the safety sanitizer rejects the runtime value. | `cuda_graph_config.mode` | `Literal['decode']` | `categorical` | | `decode`, `encode` | | `cuda_graph_config.num_tokens` | `Optional[List[Annotated[int, Gt(gt=0)]]]` | `value` | | | | `cuda_graph_config.seq_lens` | `Optional[List[Annotated[int, Gt(gt=0)]]]` | `value` | | | +| `disable_mm_encoder` | `` | `value` | | | | `disable_overlap_scheduler` | `` | `value` | | | | `dtype` | `` | `categorical` | allowlist | `auto`, `float16`, `bfloat16`, `float32` | | `dwdp_config.contention_opt` | `` | `value` | | | @@ -104,7 +106,7 @@ unset or when the safety sanitizer rejects the runtime value. | `iter_stats_max_iterations` | `Optional[int]` | `value` | | | | `kv_cache_config.attention_dp_events_gather_period_ms` | `` | `value` | | | | `kv_cache_config.avg_seq_len` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | -| `kv_cache_config.block_reuse_policy` | `Literal['all_reusable', 'per_request']` | `categorical` | | `all_reusable`, `per_request` | +| `kv_cache_config.block_reuse_policy` | `Literal['all_reusable', 'per_request', 'per_conversation']` | `categorical` | | `all_reusable`, `per_request`, `per_conversation` | | `kv_cache_config.copy_on_partial_reuse` | `` | `value` | | | | `kv_cache_config.cross_kv_cache_fraction` | `Optional[float]` | `value` | | | | `kv_cache_config.disk_cache_size` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | @@ -122,7 +124,7 @@ unset or when the safety sanitizer rejects the runtime value. | `kv_cache_config.mamba_ssm_cache_dtype` | `Literal['auto', 'float16', 'bfloat16', 'float32']` | `categorical` | | `auto`, `float16`, `bfloat16`, `float32` | | `kv_cache_config.mamba_ssm_philox_rounds` | `` | `value` | | | | `kv_cache_config.mamba_ssm_stochastic_rounding` | `` | `value` | | | -| `kv_cache_config.mamba_state_cache_interval` | `` | `value` | | | +| `kv_cache_config.mamba_state_config.periodic_snapshot_interval` | `` | `value` | | | | `kv_cache_config.max_attention_window` | `Optional[List[int]]` | `value` | | | | `kv_cache_config.max_gpu_total_bytes` | `` | `value` | | | | `kv_cache_config.max_tokens` | `Optional[int]` | `value` | | | @@ -131,7 +133,7 @@ unset or when the safety sanitizer rejects the runtime value. | `kv_cache_config.secondary_offload_min_priority` | `Optional[int]` | `value` | | | | `kv_cache_config.sink_token_length` | `Optional[int]` | `value` | | | | `kv_cache_config.tokens_per_block` | `` | `value` | | | -| `kv_cache_config.use_kv_cache_manager_v2` | `` | `value` | | | +| `kv_cache_config.use_kv_cache_manager_v2` | `Union[bool, Literal['auto']]` | `value` | | `auto` | | `kv_cache_config.use_uvm` | `` | `value` | | | | `kv_connector_config.connector` | `Optional[str]` | `categorical` | allowlist | `lmcache`, `lmcache-mp`, `kvbm` | | `layer_wise_benchmarks_config.calibration_layer_indices` | `Optional[List[int]]` | `value` | | | @@ -156,11 +158,13 @@ unset or when the safety sanitizer rejects the runtime value. | `moe_config.use_low_precision_moe_combine` | `` | `value` | | | | `moe_expert_parallel_size` | `Optional[int]` | `value` | | | | `moe_tensor_parallel_size` | `Optional[int]` | `value` | | | +| `multimodal_config.encoder_cache_max_bytes` | `` | `value` | | | | `multimodal_config.encoder_side_stream_max_ahead` | `` | `value` | | | | `multimodal_config.video_pruning_rate` | `Optional[float]` | `value` | | | | `mx_config.preshard_strategy` | `` | `categorical` | allowlist | `per_module` | | `mx_config.server_query_timeout_s` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | | `num_postprocess_workers` | `` | `value` | | | +| `num_serve_frontends` | `` | `value` | | | | `nvfp4_gemm_config.allowed_backends` | `List[Literal['cutlass', 'cublaslt', 'cutedsl', 'cuda_core', 'marlin']]` | `value` | | `cutlass`, `cublaslt`, `cutedsl`, `cuda_core`, `marlin` | | `orchestrator_type` | `Optional[Literal['rpc', 'ray']]` | `categorical` | | `rpc`, `ray` | | `peft_cache_config.device_cache_percent` | `` | `value` | | | @@ -208,6 +212,7 @@ unset or when the safety sanitizer rejects the runtime value. | `sparse_attention_config.algorithm` | `Literal['dsa']` | `categorical` | | `dsa`, `deepseek_v4`, `minimax_m3`, `rocket`, `skip_softmax` | | `sparse_attention_config.compress_ratios` | `List[int]` | `value` | | | | `sparse_attention_config.enable_heuristic_topk` | `` | `value` | | | +| `sparse_attention_config.implementation` | `Literal['triton', 'msa']` | `categorical` | | `triton`, `msa` | | `sparse_attention_config.index_head_dim` | `Optional[int]` | `value` | | | | `sparse_attention_config.index_n_heads` | `Optional[int]` | `value` | | | | `sparse_attention_config.index_topk` | `Optional[int]` | `value` | | | @@ -216,6 +221,8 @@ unset or when the safety sanitizer rejects the runtime value. | `sparse_attention_config.indexer_rope_interleave` | `` | `value` | | | | `sparse_attention_config.kernel_size` | `Optional[int]` | `value` | | | | `sparse_attention_config.kt_cache_dtype` | `Optional[str]` | `categorical` | allowlist | `bfloat16`, `float8_e5m2` | +| `sparse_attention_config.num_attention_heads` | `Optional[int]` | `value` | | | +| `sparse_attention_config.num_key_value_heads` | `Optional[int]` | `value` | | | | `sparse_attention_config.page_size` | `Optional[int]` | `value` | | | | `sparse_attention_config.prompt_budget` | `Optional[int]` | `value` | | | | `sparse_attention_config.q_split_threshold` | `` | `value` | | | @@ -238,7 +245,8 @@ unset or when the safety sanitizer rejects the runtime value. | `speculative_config.acceptance_rate_window_size` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | | `speculative_config.allow_advanced_sampling` | `` | `value` | | | | `speculative_config.begin_thinking_phase_token` | `` | `value` | | | -| `speculative_config.decoding_type` | `Literal['AUTO']` | `categorical` | | `AUTO`, `DFlash`, `Draft_Target`, `Eagle3`, `Eagle`, `Lookahead`, `MTP`, `Medusa`, `NGram`, `PARD`, `SA`, `SaveState`, `User_Provided` | +| `speculative_config.block_size` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `speculative_config.decoding_type` | `Literal['AUTO']` | `categorical` | | `AUTO`, `DFlash`, `DSpark`, `Draft_Target`, `Eagle3`, `Eagle`, `Lookahead`, `MTP`, `Medusa`, `NGram`, `PARD`, `SA`, `SaveState`, `User_Provided` | | `speculative_config.dynamic_tree_max_topK` | `Optional[int]` | `value` | | | | `speculative_config.eagle3_layers_to_capture` | `Optional[Set[int]]` | `value` | | | | `speculative_config.eagle3_model_arch` | `Literal['llama3', 'mistral_large3']` | `categorical` | | `llama3`, `mistral_large3` | @@ -251,6 +259,8 @@ unset or when the safety sanitizer rejects the runtime value. | `speculative_config.is_keep_all` | `` | `value` | | | | `speculative_config.is_public_pool` | `` | `value` | | | | `speculative_config.is_use_oldest` | `` | `value` | | | +| `speculative_config.markov_head_type` | `Optional[Literal['vanilla', 'gated', 'rnn']]` | `categorical` | | `vanilla`, `gated`, `rnn` | +| `speculative_config.markov_rank` | `Optional[int]` | `value` | | | | `speculative_config.mask_token_id` | `Optional[int]` | `value` | | | | `speculative_config.max_concurrency` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | | `speculative_config.max_draft_len` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | @@ -292,290 +302,3 @@ unset or when the safety sanitizer rejects the runtime value. | `use_cute_dsl_bf16_gemm` | `` | `value` | | | | `use_cute_dsl_blockscaling_bmm` | `` | `value` | | | | `use_cute_dsl_blockscaling_mm` | `` | `value` | | | - -### `TrtLlmArgs` - -280 captured fields. - -| Captured key | Annotation | Kind | Converter | Allowed values | -|--------------|------------|------|-----------|----------------| -| `backend` | `Optional[str]` | `categorical` | allowlist | `pytorch`, `tensorrt`, `_autodeploy` | -| `batching_type` | `Optional[tensorrt_llm.llmapi.llm_args.BatchingType]` | `categorical` | | `STATIC`, `INFLIGHT` | -| `build_config.dry_run` | `` | `value` | | | -| `build_config.enable_debug_output` | `` | `value` | | | -| `build_config.force_num_profiles` | `Optional[int]` | `value` | | | -| `build_config.gather_context_logits` | `` | `value` | | | -| `build_config.gather_generation_logits` | `` | `value` | | | -| `build_config.kv_cache_type` | `Optional[tensorrt_llm.llmapi.kv_cache_type.KVCacheType]` | `categorical` | | `continuous`, `paged`, `disabled` | -| `build_config.lora_config.lora_ckpt_source` | `Literal['hf', 'nemo']` | `categorical` | | `hf`, `nemo` | -| `build_config.lora_config.max_cpu_loras` | `Optional[int]` | `value` | | | -| `build_config.lora_config.max_lora_rank` | `` | `value` | | | -| `build_config.lora_config.max_loras` | `Optional[int]` | `value` | | | -| `build_config.lora_config.swap_gate_up_proj_lora_b_weight` | `` | `value` | | | -| `build_config.max_batch_size` | `` | `value` | | | -| `build_config.max_beam_width` | `` | `value` | | | -| `build_config.max_draft_len` | `` | `value` | | | -| `build_config.max_encoder_input_len` | `` | `value` | | | -| `build_config.max_input_len` | `` | `value` | | | -| `build_config.max_num_tokens` | `` | `value` | | | -| `build_config.max_prompt_embedding_table_size` | `` | `value` | | | -| `build_config.max_seq_len` | `Optional[int]` | `value` | | | -| `build_config.monitor_memory` | `` | `value` | | | -| `build_config.opt_batch_size` | `` | `value` | | | -| `build_config.opt_num_tokens` | `Optional[int]` | `value` | | | -| `build_config.plugin_config.bert_attention_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.bert_context_fmha_fp32_acc` | `` | `value` | | | -| `build_config.plugin_config.context_fmha` | `` | `value` | | | -| `build_config.plugin_config.dora_plugin` | `` | `value` | | | -| `build_config.plugin_config.fp8_rowwise_gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.fuse_fp4_quant` | `` | `value` | | | -| `build_config.plugin_config.gemm_allreduce_plugin` | `Optional[Literal['float16', 'bfloat16', None]]` | `categorical` | | `float16`, `bfloat16`, `None` | -| `build_config.plugin_config.gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', 'fp8', 'nvfp4', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `fp8`, `nvfp4`, `None` | -| `build_config.plugin_config.gemm_swiglu_plugin` | `Optional[Literal['fp8', None]]` | `categorical` | | `fp8`, `None` | -| `build_config.plugin_config.gpt_attention_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.identity_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.layernorm_quantization_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.lora_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.low_latency_gemm_plugin` | `Optional[Literal['fp8', None]]` | `categorical` | | `fp8`, `None` | -| `build_config.plugin_config.low_latency_gemm_swiglu_plugin` | `Optional[Literal['fp8', None]]` | `categorical` | | `fp8`, `None` | -| `build_config.plugin_config.mamba_conv1d_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.manage_weights` | `` | `value` | | | -| `build_config.plugin_config.moe_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.multiple_profiles` | `` | `value` | | | -| `build_config.plugin_config.nccl_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.norm_quant_fusion` | `` | `value` | | | -| `build_config.plugin_config.paged_kv_cache` | `Optional[bool]` | `value` | | | -| `build_config.plugin_config.paged_state` | `` | `value` | | | -| `build_config.plugin_config.pp_reduce_scatter` | `` | `value` | | | -| `build_config.plugin_config.qserve_gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.quantize_per_token_plugin` | `` | `value` | | | -| `build_config.plugin_config.quantize_tensor_plugin` | `` | `value` | | | -| `build_config.plugin_config.reduce_fusion` | `` | `value` | | | -| `build_config.plugin_config.remove_input_padding` | `` | `value` | | | -| `build_config.plugin_config.rmsnorm_quantization_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.smooth_quant_gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.smooth_quant_plugins` | `` | `value` | | | -| `build_config.plugin_config.streamingllm` | `` | `value` | | | -| `build_config.plugin_config.tokens_per_block` | `` | `value` | | | -| `build_config.plugin_config.use_fp8_context_fmha` | `` | `value` | | | -| `build_config.plugin_config.use_fused_mlp` | `` | `value` | | | -| `build_config.plugin_config.use_paged_context_fmha` | `` | `value` | | | -| `build_config.plugin_config.user_buffer` | `` | `value` | | | -| `build_config.plugin_config.weight_only_groupwise_quant_matmul_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.plugin_config.weight_only_quant_matmul_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | -| `build_config.speculative_decoding_mode` | `` | `categorical` | | `NONE`, `DRAFT_TOKENS_EXTERNAL`, `MEDUSA`, `LOOKAHEAD_DECODING`, `EXPLICIT_DRAFT_TOKENS`, `EAGLE`, `NGRAM`, `USER_PROVIDED`, `SAVE_HIDDEN_STATES`, `AUTO` | -| `build_config.strongly_typed` | `` | `value` | | | -| `build_config.use_mrope` | `` | `value` | | | -| `build_config.use_refit` | `` | `value` | | | -| `build_config.use_strip_plan` | `` | `value` | | | -| `build_config.weight_sparsity` | `` | `value` | | | -| `build_config.weight_streaming` | `` | `value` | | | -| `cache_transceiver_config.backend` | `Optional[Literal['DEFAULT', 'UCX', 'NIXL', 'MOONCAKE', 'MPI']]` | `categorical` | | `DEFAULT`, `UCX`, `NIXL`, `MOONCAKE`, `MPI` | -| `cache_transceiver_config.kv_cache_bounce_size_mb` | `` | `value` | | | -| `cache_transceiver_config.kv_transfer_poll_interval_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | -| `cache_transceiver_config.kv_transfer_sender_future_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | -| `cache_transceiver_config.kv_transfer_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | -| `cache_transceiver_config.max_tokens_in_buffer` | `Optional[int]` | `value` | | | -| `cache_transceiver_config.transceiver_runtime` | `Optional[Literal['CPP', 'PYTHON', 'auto']]` | `categorical` | | `CPP`, `PYTHON`, `auto` | -| `calib_config.calib_batch_size` | `` | `value` | | | -| `calib_config.calib_batches` | `` | `value` | | | -| `calib_config.calib_max_seq_length` | `` | `value` | | | -| `calib_config.device` | `Literal['cuda', 'cpu']` | `categorical` | | `cuda`, `cpu` | -| `calib_config.random_seed` | `` | `value` | | | -| `calib_config.tokenizer_max_seq_length` | `` | `value` | | | -| `context_parallel_size` | `` | `value` | | | -| `cp_config.block_size` | `Optional[int]` | `value` | | | -| `cp_config.cp_anchor_size` | `Optional[int]` | `value` | | | -| `cp_config.cp_type` | `` | `categorical` | | `ULYSSES`, `STAR`, `RING`, `HELIX` | -| `cp_config.fifo_version` | `Optional[int]` | `value` | | | -| `cp_config.tokens_per_block` | `Optional[int]` | `value` | | | -| `cp_config.use_nccl_for_alltoall` | `Optional[bool]` | `value` | | | -| `dtype` | `` | `categorical` | allowlist | `auto`, `float16`, `bfloat16`, `float32` | -| `embedding_parallel_mode` | `Literal['NONE', 'SHARDING_ALONG_VOCAB', 'SHARDING_ALONG_HIDDEN']` | `categorical` | | `NONE`, `SHARDING_ALONG_VOCAB`, `SHARDING_ALONG_HIDDEN` | -| `enable_attention_dp` | `` | `value` | | | -| `enable_build_cache.max_cache_storage_gb` | `` | `value` | | | -| `enable_build_cache.max_records` | `` | `value` | | | -| `enable_chunked_prefill` | `` | `value` | | | -| `enable_energy_metrics` | `` | `value` | | | -| `enable_lm_head_tp_in_adp` | `` | `value` | | | -| `enable_lora` | `` | `value` | | | -| `enable_prompt_adapter` | `` | `value` | | | -| `enable_tqdm` | `` | `value` | | | -| `extended_runtime_perf_knob_config.cuda_graph_cache_size` | `` | `value` | | | -| `extended_runtime_perf_knob_config.cuda_graph_mode` | `` | `value` | | | -| `extended_runtime_perf_knob_config.enable_context_fmha_fp32_acc` | `` | `value` | | | -| `extended_runtime_perf_knob_config.multi_block_mode` | `` | `value` | | | -| `fail_fast_on_attention_window_too_large` | `` | `value` | | | -| `fast_build` | `` | `value` | | | -| `gather_generation_logits` | `` | `value` | | | -| `gpus_per_node` | `Optional[int]` | `value` | | | -| `guided_decoding_backend` | `Optional[Literal['xgrammar', 'llguidance']]` | `categorical` | | `xgrammar`, `llguidance` | -| `iter_stats_max_iterations` | `Optional[int]` | `value` | | | -| `kv_cache_config.attention_dp_events_gather_period_ms` | `` | `value` | | | -| `kv_cache_config.avg_seq_len` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | -| `kv_cache_config.block_reuse_policy` | `Literal['all_reusable', 'per_request']` | `categorical` | | `all_reusable`, `per_request` | -| `kv_cache_config.copy_on_partial_reuse` | `` | `value` | | | -| `kv_cache_config.cross_kv_cache_fraction` | `Optional[float]` | `value` | | | -| `kv_cache_config.disk_cache_size` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | -| `kv_cache_config.disk_prefetch_num_reqs` | `` | `value` | | | -| `kv_cache_config.dtype` | `` | `categorical` | allowlist | `auto`, `float16`, `bfloat16`, `float32`, `fp8`, `nvfp4` | -| `kv_cache_config.enable_block_reuse` | `` | `value` | | | -| `kv_cache_config.enable_kv_pool_rebalance` | `` | `value` | | | -| `kv_cache_config.enable_partial_reuse` | `` | `value` | | | -| `kv_cache_config.enable_swa_scratch_reuse` | `` | `value` | | | -| `kv_cache_config.event_buffer_max_size` | `` | `value` | | | -| `kv_cache_config.free_gpu_memory_fraction` | `Optional[float]` | `value` | | | -| `kv_cache_config.host_cache_size` | `Optional[int]` | `value` | | | -| `kv_cache_config.iteration_stats_interval` | `` | `value` | | | -| `kv_cache_config.kv_cache_event_hash_algo` | `Literal['auto', 'v1_block_key', 'v2_sha256', 'v2_sha256_64']` | `categorical` | | `auto`, `v1_block_key`, `v2_sha256`, `v2_sha256_64` | -| `kv_cache_config.mamba_ssm_cache_dtype` | `Literal['auto', 'float16', 'bfloat16', 'float32']` | `categorical` | | `auto`, `float16`, `bfloat16`, `float32` | -| `kv_cache_config.mamba_ssm_philox_rounds` | `` | `value` | | | -| `kv_cache_config.mamba_ssm_stochastic_rounding` | `` | `value` | | | -| `kv_cache_config.mamba_state_cache_interval` | `` | `value` | | | -| `kv_cache_config.max_attention_window` | `Optional[List[int]]` | `value` | | | -| `kv_cache_config.max_gpu_total_bytes` | `` | `value` | | | -| `kv_cache_config.max_tokens` | `Optional[int]` | `value` | | | -| `kv_cache_config.max_util_for_resume` | `` | `value` | | | -| `kv_cache_config.pool_ratio` | `Optional[List[float]]` | `value` | | | -| `kv_cache_config.secondary_offload_min_priority` | `Optional[int]` | `value` | | | -| `kv_cache_config.sink_token_length` | `Optional[int]` | `value` | | | -| `kv_cache_config.tokens_per_block` | `` | `value` | | | -| `kv_cache_config.use_kv_cache_manager_v2` | `` | `value` | | | -| `kv_cache_config.use_uvm` | `` | `value` | | | -| `load_format` | `Literal['auto', 'dummy']` | `categorical` | | `auto`, `dummy` | -| `lora_config.lora_ckpt_source` | `Literal['hf', 'nemo']` | `categorical` | | `hf`, `nemo` | -| `lora_config.max_cpu_loras` | `Optional[int]` | `value` | | | -| `lora_config.max_lora_rank` | `` | `value` | | | -| `lora_config.max_loras` | `Optional[int]` | `value` | | | -| `lora_config.swap_gate_up_proj_lora_b_weight` | `` | `value` | | | -| `max_batch_size` | `Optional[int]` | `value` | | | -| `max_beam_width` | `Optional[int]` | `value` | | | -| `max_input_len` | `Optional[int]` | `value` | | | -| `max_num_tokens` | `Optional[int]` | `value` | | | -| `max_prompt_adapter_token` | `` | `value` | | | -| `max_seq_len` | `Optional[int]` | `value` | | | -| `moe_cluster_parallel_size` | `Optional[int]` | `value` | | | -| `moe_expert_parallel_size` | `Optional[int]` | `value` | | | -| `moe_tensor_parallel_size` | `Optional[int]` | `value` | | | -| `normalize_log_probs` | `` | `value` | | | -| `num_postprocess_workers` | `` | `value` | | | -| `orchestrator_type` | `Optional[Literal['rpc', 'ray']]` | `categorical` | | `rpc`, `ray` | -| `peft_cache_config.device_cache_percent` | `` | `value` | | | -| `peft_cache_config.host_cache_size` | `` | `value` | | | -| `peft_cache_config.max_adapter_size` | `` | `value` | | | -| `peft_cache_config.max_pages_per_block_device` | `` | `value` | | | -| `peft_cache_config.max_pages_per_block_host` | `` | `value` | | | -| `peft_cache_config.num_copy_streams` | `` | `value` | | | -| `peft_cache_config.num_device_module_layer` | `` | `value` | | | -| `peft_cache_config.num_ensure_workers` | `` | `value` | | | -| `peft_cache_config.num_host_module_layer` | `` | `value` | | | -| `peft_cache_config.num_put_workers` | `` | `value` | | | -| `peft_cache_config.optimal_adapter_size` | `` | `value` | | | -| `perf_metrics_max_requests` | `` | `value` | | | -| `pipeline_parallel_size` | `` | `value` | | | -| `pp_partition` | `Optional[List[int]]` | `value` | | | -| `prometheus_metrics_config.e2e_request_latency_buckets` | `Optional[List[float]]` | `value` | | | -| `prometheus_metrics_config.request_decode_time_buckets` | `Optional[List[float]]` | `value` | | | -| `prometheus_metrics_config.request_inference_time_buckets` | `Optional[List[float]]` | `value` | | | -| `prometheus_metrics_config.request_prefill_time_buckets` | `Optional[List[float]]` | `value` | | | -| `prometheus_metrics_config.request_queue_time_buckets` | `Optional[List[float]]` | `value` | | | -| `prometheus_metrics_config.time_per_output_token_buckets` | `Optional[List[float]]` | `value` | | | -| `prometheus_metrics_config.time_to_first_token_buckets` | `Optional[List[float]]` | `value` | | | -| `quant_config.clamp_val` | `Optional[List[float]]` | `value` | | | -| `quant_config.group_size` | `Optional[int]` | `value` | | | -| `quant_config.has_zero_point` | `` | `value` | | | -| `quant_config.kv_cache_quant_algo` | `Optional[tensorrt_llm.quantization.mode.QuantAlgo]` | `categorical` | | `W8A16`, `W4A16`, `W4A16_AWQ`, `W4A8_AWQ`, `W8A16_GPTQ`, `W4A16_GPTQ`, `W8A8_SQ_PER_CHANNEL`, `W8A8_SQ_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN`, `W4A8_QSERVE_PER_GROUP`, `W4A8_QSERVE_PER_CHANNEL`, `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `FP8_BLOCK_SCALES`, `INT8`, `MIXED_PRECISION`, `NVFP4`, `W4A8_NVFP4_FP8`, `W4A8_MXFP4_FP8`, `W4A8_MXFP4_MXFP8`, `W4A16_MXFP4`, `MXFP8`, `W4A16_NVFP4`, `NVFP4_AWQ`, `NVFP4_ARC`, `NO_QUANT` | -| `quant_config.mamba_ssm_philox_rounds` | `` | `value` | | | -| `quant_config.mamba_ssm_stochastic_rounding` | `` | `value` | | | -| `quant_config.pre_quant_scale` | `` | `value` | | | -| `quant_config.quant_algo` | `Optional[tensorrt_llm.quantization.mode.QuantAlgo]` | `categorical` | | `W8A16`, `W4A16`, `W4A16_AWQ`, `W4A8_AWQ`, `W8A16_GPTQ`, `W4A16_GPTQ`, `W8A8_SQ_PER_CHANNEL`, `W8A8_SQ_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN`, `W4A8_QSERVE_PER_GROUP`, `W4A8_QSERVE_PER_CHANNEL`, `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `FP8_BLOCK_SCALES`, `INT8`, `MIXED_PRECISION`, `NVFP4`, `W4A8_NVFP4_FP8`, `W4A8_MXFP4_FP8`, `W4A8_MXFP4_MXFP8`, `W4A16_MXFP4`, `MXFP8`, `W4A16_NVFP4`, `NVFP4_AWQ`, `NVFP4_ARC`, `NO_QUANT` | -| `quant_config.smoothquant_val` | `` | `value` | | | -| `quant_config.use_meta_recipe` | `` | `value` | | | -| `reasoning_parser` | `Optional[str]` | `categorical` | allowlist | `auto`, `deepseek-r1`, `laguna`, `qwen3`, `qwen3_5`, `minimax_m2`, `minimax_m2_append_think`, `nano-v3`, `gemma4`, `kimi_k2`, `kimi_k25` | -| `request_stats_max_iterations` | `Optional[int]` | `value` | | | -| `return_perf_metrics` | `` | `value` | | | -| `scheduler_config.capacity_scheduler_policy` | `` | `categorical` | | `MAX_UTILIZATION`, `GUARANTEED_NO_EVICT`, `STATIC_BATCH` | -| `scheduler_config.context_chunking_policy` | `Optional[tensorrt_llm.llmapi.llm_args.ContextChunkingPolicy]` | `categorical` | | `FIRST_COME_FIRST_SERVED`, `EQUAL_PROGRESS`, `FORCE_CHUNK` | -| `scheduler_config.dynamic_batch_config.dynamic_batch_moving_average_window` | `` | `value` | | | -| `scheduler_config.dynamic_batch_config.enable_batch_size_tuning` | `` | `value` | | | -| `scheduler_config.dynamic_batch_config.enable_max_num_tokens_tuning` | `` | `value` | | | -| `scheduler_config.enable_prefix_aware_scheduling` | `` | `value` | | | -| `scheduler_config.use_python_scheduler` | `` | `value` | | | -| `scheduler_config.waiting_queue_policy` | `` | `categorical` | | `fcfs`, `priority` | -| `skip_tokenizer_init` | `` | `value` | | | -| `sparse_attention_config.algorithm` | `Literal['dsa']` | `categorical` | | `dsa`, `deepseek_v4`, `minimax_m3`, `rocket`, `skip_softmax` | -| `sparse_attention_config.compress_ratios` | `List[int]` | `value` | | | -| `sparse_attention_config.enable_heuristic_topk` | `` | `value` | | | -| `sparse_attention_config.index_head_dim` | `Optional[int]` | `value` | | | -| `sparse_attention_config.index_n_heads` | `Optional[int]` | `value` | | | -| `sparse_attention_config.index_topk` | `Optional[int]` | `value` | | | -| `sparse_attention_config.indexer_k_dtype` | `Literal['fp8', 'fp4']` | `categorical` | | `fp8`, `fp4` | -| `sparse_attention_config.indexer_max_chunk_size` | `Optional[int]` | `value` | | | -| `sparse_attention_config.indexer_rope_interleave` | `` | `value` | | | -| `sparse_attention_config.kernel_size` | `Optional[int]` | `value` | | | -| `sparse_attention_config.kt_cache_dtype` | `Optional[str]` | `categorical` | allowlist | `bfloat16`, `float8_e5m2` | -| `sparse_attention_config.page_size` | `Optional[int]` | `value` | | | -| `sparse_attention_config.prompt_budget` | `Optional[int]` | `value` | | | -| `sparse_attention_config.q_split_threshold` | `` | `value` | | | -| `sparse_attention_config.seq_len_threshold` | `Optional[int]` | `value` | | | -| `sparse_attention_config.skip_indexer_for_short_seqs` | `` | `value` | | | -| `sparse_attention_config.sparse_block_size` | `` | `value` | | | -| `sparse_attention_config.sparse_disable_index_value` | `` | `value` | | | -| `sparse_attention_config.sparse_index_dim` | `` | `value` | | | -| `sparse_attention_config.sparse_init_blocks` | `` | `value` | | | -| `sparse_attention_config.sparse_local_blocks` | `` | `value` | | | -| `sparse_attention_config.sparse_num_index_heads` | `` | `value` | | | -| `sparse_attention_config.sparse_score_type` | `Literal['max']` | `categorical` | | `max` | -| `sparse_attention_config.sparse_topk_blocks` | `` | `value` | | | -| `sparse_attention_config.topk` | `Optional[int]` | `value` | | | -| `sparse_attention_config.topr` | `Union[int, float, NoneType]` | `value` | | | -| `sparse_attention_config.use_cute_dsl_paged_mqa_logits` | `` | `value` | | | -| `sparse_attention_config.use_cute_dsl_topk` | `` | `value` | | | -| `sparse_attention_config.window_size` | `` | `value` | | | -| `speculative_config.acceptance_rate_threshold` | `Optional[float]` | `value` | | | -| `speculative_config.acceptance_rate_window_size` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | -| `speculative_config.allow_advanced_sampling` | `` | `value` | | | -| `speculative_config.begin_thinking_phase_token` | `` | `value` | | | -| `speculative_config.decoding_type` | `Literal['AUTO']` | `categorical` | | `AUTO`, `DFlash`, `Draft_Target`, `Eagle3`, `Eagle`, `Lookahead`, `MTP`, `Medusa`, `NGram`, `PARD`, `SA`, `SaveState`, `User_Provided` | -| `speculative_config.dynamic_tree_max_topK` | `Optional[int]` | `value` | | | -| `speculative_config.eagle3_layers_to_capture` | `Optional[Set[int]]` | `value` | | | -| `speculative_config.eagle3_model_arch` | `Literal['llama3', 'mistral_large3']` | `categorical` | | `llama3`, `mistral_large3` | -| `speculative_config.eagle3_one_model` | `Optional[bool]` | `value` | | | -| `speculative_config.eagle_choices` | `Optional[List[List[int]]]` | `value` | | | -| `speculative_config.enable_global_pool` | `` | `value` | | | -| `speculative_config.end_thinking_phase_token` | `` | `value` | | | -| `speculative_config.global_pool_size` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | -| `speculative_config.greedy_sampling` | `Optional[bool]` | `value` | | | -| `speculative_config.is_keep_all` | `` | `value` | | | -| `speculative_config.is_public_pool` | `` | `value` | | | -| `speculative_config.is_use_oldest` | `` | `value` | | | -| `speculative_config.mask_token_id` | `Optional[int]` | `value` | | | -| `speculative_config.max_concurrency` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | -| `speculative_config.max_draft_len` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | -| `speculative_config.max_matching_ngram_size` | `` | `value` | | | -| `speculative_config.max_ngram_size` | `` | `value` | | | -| `speculative_config.max_non_leaves_per_layer` | `Optional[int]` | `value` | | | -| `speculative_config.max_total_draft_tokens` | `Optional[int]` | `value` | | | -| `speculative_config.max_verification_set_size` | `` | `value` | | | -| `speculative_config.max_window_size` | `` | `value` | | | -| `speculative_config.medusa_choices` | `Optional[List[List[int]]]` | `value` | | | -| `speculative_config.mtp_eagle_one_model` | `` | `value` | | | -| `speculative_config.num_eagle_layers` | `Optional[int]` | `value` | | | -| `speculative_config.num_medusa_heads` | `Optional[int]` | `value` | | | -| `speculative_config.num_nextn_predict_layers` | `Optional[int]` | `value` | | | -| `speculative_config.posterior_threshold` | `Optional[float]` | `value` | | | -| `speculative_config.relaxed_delta` | `` | `value` | | | -| `speculative_config.relaxed_topk` | `` | `value` | | | -| `speculative_config.sa_config.enable_global_pool` | `` | `value` | | | -| `speculative_config.sa_config.threshold` | `` | `value` | | | -| `speculative_config.target_layer_ids` | `Optional[List[int]]` | `value` | | | -| `speculative_config.use_dynamic_tree` | `Optional[bool]` | `value` | | | -| `speculative_config.use_mtp_vanilla` | `` | `value` | | | -| `speculative_config.use_rejection_sampling` | `` | `value` | | | -| `speculative_config.use_relaxed_acceptance_for_thinking` | `` | `value` | | | -| `speculative_config.write_interval` | `` | `value` | | | -| `telemetry_config.disabled` | `` | `value` | | | -| `telemetry_config.usage_context` | `` | `categorical` | | `unknown`, `llm_class`, `cli_serve`, `cli_bench`, `cli_eval` | -| `tensor_parallel_size` | `` | `value` | | | -| `tokenizer_mode` | `Literal['auto', 'slow']` | `categorical` | | `auto`, `slow` | -| `trust_remote_code` | `` | `value` | | | diff --git a/docs/source/examples/customization.md b/docs/source/examples/customization.md index 4c357554f504..3a7aa5b3f9e8 100644 --- a/docs/source/examples/customization.md +++ b/docs/source/examples/customization.md @@ -2,70 +2,51 @@ ## Quantization -TensorRT LLM can quantize the Hugging Face model automatically. By setting the appropriate flags in the `LLM` instance. For example, to perform an Int4 AWQ quantization, the following code triggers the model quantization. Please refer to complete list of [supported flags](https://nvidia.github.io/TensorRT-LLM/_modules/tensorrt_llm/quantization/mode.html#QuantAlgo) and acceptable values. +TensorRT LLM runs quantized models from pre-quantized checkpoints. Use a checkpoint quantized with [NVIDIA TensorRT Model Optimizer](https://github.com/NVIDIA/TensorRT-Model-Optimizer) (for example, the ready-made FP8/NVFP4 checkpoints published on the [NVIDIA Hugging Face hub](https://huggingface.co/nvidia)), and the quantization configuration is detected automatically when the model loads: ``` python -from tensorrt_llm.llmapi import QuantConfig, QuantAlgo +from tensorrt_llm import LLM -quant_config = QuantConfig(quant_algo=QuantAlgo.W4A16_AWQ) - -llm = LLM(, quant_config=quant_config) +llm = LLM("nvidia/Llama-3.1-8B-Instruct-FP8") ``` +Refer to the [quantization feature documentation](../features/quantization.md) for the supported formats per GPU architecture and instructions on quantizing your own model. + ## Sampling -SamplingParams can customize the sampling strategy to control LLM generated responses, such as beam search, temperature, and [others](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/llmapi/utils.py#L55-L76). +SamplingParams can customize the sampling strategy to control LLM generated responses, such as beam search, temperature, and many others. -As an example, to enable beam search with a beam size of 4, set the `sampling_params` as follows: +As an example, to enable beam search with a beam width of 4, configure the engine limit with `max_beam_width` and request beam search through `SamplingParams`: ```python -from tensorrt_llm.llmapi import LLM, SamplingParams, BuildConfig +from tensorrt_llm import LLM, SamplingParams -build_config = BuildConfig() -build_config.max_beam_width = 4 - -llm = LLM(, build_config=build_config) +llm = LLM(, max_beam_width=4) # Let the LLM object generate text with the default sampling strategy, or # you can create a SamplingParams object as well with several fields set manually -sampling_params = SamplingParams(beam_width=4) # current limitation: beam_width should be equal to max_beam_width +sampling_params = SamplingParams(n=4, use_beam_search=True) for output in llm.generate(, sampling_params=sampling_params): print(output) ``` -`SamplingParams` manages and dispatches fields to C++ classes including: - -* [SamplingConfig](https://nvidia.github.io/TensorRT-LLM/_cpp_gen/runtime.html#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE) -* [OutputConfig](https://nvidia.github.io/TensorRT-LLM/_cpp_gen/executor.html#_CPPv4N12tensorrt_llm8executor12OutputConfigE) - -Refer to the [class documentation](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html#tensorrt_llm.llmapi.SamplingParams) for more details. - -## Build Configuration - -Apart from the arguments mentioned above, you can also customize the build configuration with the `build_config` class and other arguments borrowed from the trtllm-build CLI. These build configuration options provide flexibility in building engines for the target hardware and use cases. Refer to the following example: - -```python -llm = LLM(, - build_config=BuildConfig( - max_num_tokens=4096, - max_batch_size=128, - max_beam_width=4)) -``` -Refer to the [buildconfig documentation](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/builder.py#L470-L501) for more details. +Refer to the [class documentation](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html#tensorrt_llm.llmapi.SamplingParams) for the complete list of fields. ## Runtime Customization -Similar to `build_config`, you can also customize the runtime configuration with the `runtime_config`, `peft_cache_config` or other [arguments](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/llmapi/llm_utils.py#L186-L223) borrowed from the Executor APIs. These runtime configuration options provide additional flexibility with respect to KV cache management, GPU memory allocation and so on. Refer to the following example: - +Runtime behavior such as KV cache management and GPU memory allocation can be customized with dedicated configuration classes like `kv_cache_config` and `peft_cache_config` passed to the `LLM` constructor. Refer to the following example: ```python -from tensorrt_llm.llmapi import LLM, KvCacheConfig +from tensorrt_llm import LLM +from tensorrt_llm.llmapi import KvCacheConfig llm = LLM(, kv_cache_config=KvCacheConfig( free_gpu_memory_fraction=0.8)) ``` +Refer to the [LLM API reference](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) for all available configuration classes. + ## Tokenizer Customization By default, the LLM API uses transformers’ `AutoTokenizer`. You can override it with your own tokenizer by passing it when creating the LLM object. Refer to the following example: @@ -90,8 +71,8 @@ for output in llm.generate([32, 12]): For performance considerations, you can disable the tokenizer by passing `skip_tokenizer_init=True` when creating `LLM`. In this case, `LLM.generate` and `LLM.generate_async` will expect prompt token ids as input. Refer to the following example: ```python -llm = LLM() -for output in llm.generate([[32, 12]], skip_tokenizer_init=True): +llm = LLM(, skip_tokenizer_init=True) +for output in llm.generate([[32, 12]]): print(output) ``` diff --git a/docs/source/features/auto_deploy/advanced/testing_strategy.md b/docs/source/features/auto_deploy/advanced/testing_strategy.md index d65841862ee6..b60eada7e722 100644 --- a/docs/source/features/auto_deploy/advanced/testing_strategy.md +++ b/docs/source/features/auto_deploy/advanced/testing_strategy.md @@ -117,8 +117,8 @@ Format: `path/to/test_file.py::test_function_name[param_id]` Example from `l0_a30.yml`: ```yaml -- accuracy/test_cli_flow.py::TestLlama3_1_8BInstruct::test_medusa_fp8_prequantized -- examples/test_multimodal.py::test_llm_multimodal_general[Qwen2-VL-7B-Instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:4] +- accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_auto_dtype +- unittest/_torch/modeling -k "modeling_llama" ``` ### Example: Adding an Accuracy Test diff --git a/docs/source/features/checkpoint-loading.md b/docs/source/features/checkpoint-loading.md index 6caa3a0fcd95..fa617b43dbdf 100644 --- a/docs/source/features/checkpoint-loading.md +++ b/docs/source/features/checkpoint-loading.md @@ -83,6 +83,15 @@ Currently, HF checkpoint loader is the primary built-in format, supporting: - **Configuration parser** - Parsing HF stored configuration information to TRTLLM `ModelConfig` object - **Weights Mapping** - Converting HF weights into TRTLLM compatible representation +### ModelExpress (MX) Loading Path + +The PyTorch backend can use ModelExpress (MX) for peer-to-peer weight transfer +from a running TensorRT-LLM source instance before falling back to Hugging Face +checkpoint loading. Selecting MX does not require an MX-specific on-disk +checkpoint or conversion of the Hugging Face checkpoint. For installation, MX +service deployment, and configuration details, see +[ModelExpress (MX) Checkpoint Loading](./model-express.md). + ## Using Checkpoint Loaders ### Basic Usage diff --git a/docs/source/features/disagg-serving.md b/docs/source/features/disagg-serving.md index f38e3c818f57..9ed621158a26 100644 --- a/docs/source/features/disagg-serving.md +++ b/docs/source/features/disagg-serving.md @@ -94,11 +94,19 @@ The optimizations required for KV cache transmission vary depending on whether i ### Unique Global Request ID -A disaggregated-serving request can provide a unique global request ID via `DisaggregatedParams.disagg_request_id`. -When this field is a positive integer, the context and generation requests share that value as their internal request ID, which enables end-to-end tracking. -To avoid collisions with worker-local or warm-up requests, it is recommended to use a value larger than `1 << 42 = 4398046511104`. -If the field is unset or non-positive, the context and generation requests instead receive separate local sequence IDs, rotating within the range `(0, 1<<42)`, assigned by the respective workers. When `disagg_request_id` is specified, do not route the context and generation requests to the same worker. -This field is optional at present; however, some forthcoming features will depend on this unique identifier. +The context and generation phases of one request must share a single request ID: the ctx↔gen KV-cache transfer is keyed by it, so a collision (two in-flight requests with the same ID) corrupts the transfer. This shared ID is carried on `DisaggregatedParams.disagg_request_id`. + +The disaggregated server generates this ID itself as a **snowflake** — a self-contained 64-bit positive integer that is unique without any cross-process coordination. The bit layout is: + +``` +[ 0 (1 bit) | timestamp_ms (39 bits) | node_id (8 bits) | process_id (6 bits) | counter (10 bits) ] +``` + +- `node_id` (0–255) identifies the node (defaults to a hash of the MAC address; overridable via `node_id` in the disaggregated config). +- `process_id` (0–63) identifies the orchestrator process on that node. In a [coordinator + worker fleet](#coordinator-and-worker-fleet) each fleet worker receives a distinct value, so co-located workers never emit the same ID in the same millisecond. It is set from the `TRTLLM_DISAGG_WORKER_PROCESS_ID` environment variable (assigned automatically per worker by the launcher). +- The `(node_id, process_id)` pair therefore makes the ID unique across all orchestrator processes without a shared counter or an extra network round trip — each worker mints its own IDs locally. + +Global disaggregated IDs occupy the range `[1 << 40, 2**63)`; worker-local and warm-up request IDs occupy the disjoint range `[0, 1 << 40)`, so the two never collide. If a client supplies its own positive `disagg_request_id`, that value is used verbatim and must be globally unique; when unset, the server mints a snowflake ID as above. ## Usage @@ -201,7 +209,7 @@ When routing requests to the context servers, the disaggregated server will mark when routing requests to the generation servers, the disaggregated server will mark the requests as "generation-only" to skip the context phase. Clients can then send requests to the disaggregated server at `localhost:8000`, which is an OpenAI-compatible endpoint. For example, you can send requests to the disaggregated server using curl: -```bash +``` curl http://localhost:8000/v1/completions \ -H "Content-Type: application/json" \ -d '{ @@ -233,6 +241,95 @@ Example (two-node deployment): - **Client entrypoint** - Send requests or use a load balancer forwarding to `node-a:8000` and `node-b:8000` +### Coordinator and Worker Fleet + +A single disaggregated server process is itself a single-threaded orchestrator and can become a throughput bottleneck (it terminates every client connection, runs routing, and proxies the ctx→gen hop). To scale the orchestrator on one node without standing up multiple independent instances, `trtllm-serve disaggregated` can run a **fleet** of stateless disaggregated-server worker processes behind a shared **coordinator**. + +The two roles split as follows: + +- **Coordinator** — a single process that owns all cluster state: the ctx/gen routers, worker readiness, and (for the KV-cache-aware router) the single ZMQ event-ingest endpoint. It exposes an internal coordination API (`/select`, `/finish`, `/cluster_info`, `/health`). +- **Fleet workers** — `num_workers` stateless disaggregated servers that share the public port via `SO_REUSEPORT` (each worker is its own process binding the same port, so the kernel load-balances incoming connections across them by 4-tuple hash). Each holds a lightweight delegating client: it computes the routing key locally (e.g. block hashes) and delegates the placement decision to the coordinator over HTTP. Workers own no routing state, so routing stays globally consistent no matter which worker terminates a connection. Each worker also gets a distinct `process_id` for the [global request ID](#unique-global-request-id). + +This is controlled by two fields in the disaggregated config: + +- `num_workers` (int, default `1`) — number of disaggregated-server worker processes to run on the public port. +- `disagg_coordinator_url` (str, optional) — URL of an already-running coordinator. When set, this process starts **no** coordinator and its fleet delegates to that external one. + +The three resulting topologies: + +| `num_workers` | `disagg_coordinator_url` | Behavior | +|---------------|--------------------------|----------| +| `1` | unset | Single self-contained server with an in-process coordinator (the default; unchanged from earlier examples). | +| `> 1` | unset | An **implicit** coordinator starts in this process (on `port - 1`) and a fleet of `num_workers` delegating servers runs on the public port. | +| any | set | **No** coordinator starts here; a fleet of `num_workers` delegating servers points at the external `disagg_coordinator_url`. | + +```{note} +The fleet is most useful with a *stateful* router (`kv_cache_aware`, `conversation`) where placement must be globally consistent — that decision is delegated to the coordinator. With a *stateless* router (`round_robin`, `load_balancing`) each worker simply places locally and no coordinator round-trip occurs. +``` + +#### Example: implicit coordinator + 4-worker fleet + +Extend the `disagg_config.yaml` from the [trtllm-serve](#trtllm-serve) example with `num_workers` and a router type: + +```yaml +hostname: localhost +port: 8000 +backend: pytorch +# Run 4 stateless disaggregated-server workers on port 8000, with an implicit +# coordinator started in-process on port 7999 (port - 1). +num_workers: 4 +context_servers: + num_instances: 2 + urls: + - "localhost:8001" + - "localhost:8002" + router: + type: kv_cache_aware +generation_servers: + num_instances: 1 + urls: + - "localhost:8003" + router: + type: kv_cache_aware +``` + +Launch it exactly as before — the coordinator and fleet are started for you: + +```bash +trtllm-serve disaggregated -c disagg_config.yaml +``` + +Clients still send requests to the public endpoint (`localhost:8000`); the fleet transparently delegates routing to the coordinator. + +#### Example: external coordinator + +To point a fleet at a coordinator already running elsewhere (for example, one shared across nodes), set `disagg_coordinator_url` and omit the coordinator from this process: + +```yaml +hostname: localhost +port: 8000 +backend: pytorch +num_workers: 4 +disagg_coordinator_url: "http://coordinator-host:7999" +context_servers: + num_instances: 2 + urls: + - "localhost:8001" + - "localhost:8002" + router: + type: kv_cache_aware +generation_servers: + num_instances: 1 + urls: + - "localhost:8003" + router: + type: kv_cache_aware +``` + +```{note} +A fleet worker fails fast if its coordinator is unreachable: on startup it probes the coordinator's `/cluster_info` with bounded retry (up to `--server_start_timeout` seconds) and exits with an error rather than coming up and returning `Cluster is not ready` for every request. +``` + ## Environment Variables TRT-LLM uses some environment variables to control the behavior of disaggregated service. diff --git a/docs/source/features/feature-combination-matrix.md b/docs/source/features/feature-combination-matrix.md index b56c1b219af9..d91322fc8c11 100644 --- a/docs/source/features/feature-combination-matrix.md +++ b/docs/source/features/feature-combination-matrix.md @@ -14,7 +14,7 @@ | Speculative Decoding — Linear | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | --- | | | | | | | | | | | Speculative Decoding — Dynamic Trees | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | No | --- | | | | | | | | | | Speculative Decoding — Legacy Path (NGram, user-provided) | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | No | No | --- | | | | | | | | -| Torch Sampler | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | --- | | | | | | | +| Torch Sampler | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes (MTP dynamic tree: greedy only) | Yes | --- | | | | | | | | TLLM C++ Sampler | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | No | No | No | --- | | | | | | | KV Cache Reuse | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | --- | | | | | | Sliding Window Attention | Yes | Yes | Yes | Yes | Yes | Untested | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | --- | | | | diff --git a/docs/source/features/kvcache.md b/docs/source/features/kvcache.md index a44a6a866904..b8b395abdc4d 100644 --- a/docs/source/features/kvcache.md +++ b/docs/source/features/kvcache.md @@ -74,12 +74,57 @@ scheduler_config: enable_prefix_aware_scheduling: false ``` +### Mamba Snapshot Boundaries + +Hybrid Mamba models must retain the recurrent Mamba state together with the +attention KV prefix. Snapshot policy is grouped under +`kv_cache_config.mamba_state_config`. `periodic_snapshot_interval` controls +periodic boundaries. They are disabled by default; set the interval to a +positive value to enable them. The deprecated +`kv_cache_config.mamba_state_cache_interval` alias remains accepted for +compatibility and is copied to the nested field during validation. New code and +configuration files should use the nested field. The prototype +`additional_snapshot_offsets_from_start` and +`additional_snapshot_offsets_from_end` options add fixed boundaries. Start +offsets count tokens from the beginning of the prompt. End offsets count +backward from the prompt end, and an end offset of `0` selects the final +prompt boundary. The `per_conversation` block reuse policy disables periodic +Mamba snapshots, so configure one or more explicit stable boundaries (usually +an end offset of `0`) when using it with a hybrid Mamba model. For example: + +```yaml +kv_cache_config: + enable_block_reuse: true + use_kv_cache_manager_v2: true + avg_seq_len: 2048 + mamba_state_config: + periodic_snapshot_interval: 0 + additional_snapshot_offsets_from_start: [128] + additional_snapshot_offsets_from_end: [0, 32] +``` + +This retains snapshots after the first 128 tokens, at the end of the prompt, +and before the final 32 prompt tokens. Positions outside a particular prompt +are ignored. Set `avg_seq_len` to the workload's average total sequence length +so V2 can size the attention KV and Mamba state pools in the right proportion. +If neither `avg_seq_len` nor an explicit `pool_ratio` is configured, hybrid +Mamba models warn and fall back to half of `max_seq_len`, which can produce a +suboptimal pool split. Exact explicit boundaries currently require +`MambaHybridCacheManagerV2`, `max_beam_width=1`, and no KV connector. Hybrid +Mamba models select V2 by default when +`use_kv_cache_manager_v2: auto`; set it to `false` to select the V1 C++ +compatibility manager. In disaggregated serving, V2 Mamba requires the Python +NIXL transceiver (`transceiver_runtime: PYTHON`); V1 routes support periodic +snapshots only. + ### KV Cache Salting for Secure Reuse KV cache salting provides a security mechanism to control which requests can reuse cached KV states. When a `cache_salt` parameter is provided with a request, the KV cache system will only allow reuse of cached blocks given the same cache salt value. This prevents potential security issues such as prompt theft attacks, where malicious users might try to infer information from cached states of other users' requests. To use cache salting, specify the `cache_salt` parameter as a string when creating requests. Only requests with matching cache salt values can share cached KV blocks. The salt value can be any non-empty string, such as a user ID, tenant ID, or hash string. +This isolation is enforced entirely by the block-key hash: the salt is mixed into the hashed input and prefix matching is decided by digest equality alone (blocks are not re-compared token-by-token). The block-key hash is therefore required to be a cryptographic hash with strong collision resistance and a 256-bit digest (SHA-256 provides ~128-bit collision resistance, which is ample here); substituting a non-cryptographic hash would allow crafted collisions to bypass salt isolation and must not be done. + ### Multimodal UUID Support for Cache Identification When working with multimodal models (e.g., vision-language models), the KV cache system needs to identify which cached blocks correspond to which multimodal inputs (images, videos, etc.). By default, the system uses content-based hashing to generate unique identifiers for each multimodal input. However, this approach has limitations for cache management across sessions, as the same content must be re-processed to generate the same hash. diff --git a/docs/source/features/model-express.md b/docs/source/features/model-express.md new file mode 100644 index 000000000000..3007b1402c4a --- /dev/null +++ b/docs/source/features/model-express.md @@ -0,0 +1,152 @@ + + +# ModelExpress (MX) Checkpoint Loading + +The MX checkpoint-loading integration is intended to reduce repeated disk +reads when multiple TensorRT LLM workers load the same model. A worker that +loads from disk can publish its weights as an MX source, and later workers can +receive those weights directly through MX. + +TensorRT LLM can use ModelExpress (MX) as a checkpoint-loading path for +PyTorch backend deployments. `checkpoint_format="MX"` selects this loading +path; it does not identify an MX-specific on-disk checkpoint format, and no +checkpoint conversion is required. TensorRT LLM attempts to fetch compatible +weights from another running TensorRT LLM instance through the MX server. If +no compatible source is available, or if MX transfer fails, loading falls back +to the provided Hugging Face checkpoint. + +## Current Support Scope + +The post-transform MX receive path currently supports only +`LlamaForCausalLM` with transform protocol version 1. TensorRT LLM publishes +post-transform weights together with source-identity and layout metadata. A +receiver whose model family is not allow-listed does not consume those bytes; +it falls back to the standard Hugging Face checkpoint path. + +Loads that require a separately loaded draft model also fall back to the +standard checkpoint path. Target-plus-draft post-transform transfer remains +disabled until layout state is tracked and qualified independently for each +submodel. + +### Adding a Model Family + +Support for another model family requires a focused qualification change: + +1. Audit every post-load hook in the family and its nested modules. Move + structural wiring to `setup_aliases()`, one-time tensor-layout changes to + `transform_weights()`, and process-local derived state to + `cache_derived_state()`. +2. Verify that every one-time transform is guarded by `_weights_transformed` + and that the staged receiver can skip `transform_weights()` without + changing aliases, derived state, tensor layout, or outputs. +3. Add the model class and transform protocol version to the MX staged-receiver + allow-list only after full-load and staged-load equivalence tests pass. +4. Cover compatible transfer, source-identity mismatch, unsupported layout or + protocol, and non-allow-listed fallback. Keep target-plus-draft loading + disabled unless that combination has its own mixed-layout tests. +5. Run a real ModelExpress donor/receiver test with the model configurations + being claimed, including the supported quantization and TP/PP/EP layouts. + Compare deterministic output token IDs with the standard Hugging Face load + path before documenting the family as supported. + +## Installation + +The official TensorRT LLM release container includes the MX Python client. No +additional Python package installation is required in that container. MX +remains opt-in at runtime: TensorRT LLM uses the client only when the MX +checkpoint-loading path and a server URL are configured. Installing the client +does not expand the model support scope described above. + +For pip installations outside the official release container, install the MX +Python client through the optional `mx` extra: + +```bash +pip install "tensorrt-llm[mx]" +``` + +The extra pins the ModelExpress client to version `0.4.1`, matching the client +API qualified by this integration. Deploy a compatible MX server version. +The extra can be added to an existing TensorRT LLM installation. If the MX +loading path is configured but the client cannot be imported, TensorRT LLM +fails with an actionable installation message instead of silently loading from +the Hugging Face checkpoint. Source discovery and transfer failures continue to +use the Hugging Face fallback described above. + +## Deploy the MX Service + +Deploy the MX server and its Redis metadata backend independently of +TensorRT LLM. One MX service can be shared by multiple TensorRT LLM launches, +provided every instance can reach the MX endpoint. TensorRT LLM does not start, +stop, or otherwise manage either service. + +The following commands illustrate a standalone Docker deployment. Production +deployments should manage service lifecycle, persistence, networking, and +security according to their environment. + +```bash +docker network create modelexpress +docker run -d --name modelexpress-redis \ + --network modelexpress \ + redis:8-alpine +docker run -d --name modelexpress-server \ + --network modelexpress \ + -p 8001:8001 \ + -e MODEL_EXPRESS_SERVER_PORT=8001 \ + -e MODEL_EXPRESS_LOG_LEVEL=info \ + -e MX_METADATA_BACKEND=redis \ + -e REDIS_URL=redis://modelexpress-redis:6379 \ + nvcr.io/nvidia/ai-dynamo/modelexpress-server:0.4.1 +``` + +## Configure TensorRT LLM + +Select the MX checkpoint-loading path and provide the MX server URL in a +`trtllm-serve` config. The model argument remains a standard Hugging Face model +ID or checkpoint path: + +```yaml +checkpoint_format: MX +mx_config: + server_url: http://mx-server.example.com:8001 +``` + +```bash +trtllm-serve /path/to/model --config config.yaml +``` + +The `MODEL_EXPRESS_URL` environment variable can also provide the server URL +when `mx_config.server_url` is not set. + +Multiple TensorRT LLM launches can use the same configuration. A worker that +does not find a compatible source loads from Hugging Face storage and publishes +its weights through MX. Later compatible workers can receive those weights by +P2P transfer. + +If neither `mx_config.server_url` nor `MODEL_EXPRESS_URL` is set, MX transfer is +not attempted and checkpoint loading falls back to the standard Hugging Face +path. + +## Configuration + +| Field | Default | Description | +|-------|---------|-------------| +| `mx_config.server_url` | `null` | URL of the separately managed MX server. | +| `mx_config.server_query_timeout_s` | `null` | Timeout for MX source discovery. When unset, TensorRT LLM uses a short fallback cap when no source exists and otherwise lets MX wait for long donor loads. | + +## Notes and Limitations + +- Post-transform MX reception is currently limited to the Llama model family. + Other model families safely fall back to Hugging Face loading until they are + explicitly qualified and added to the staged-receiver allow-list. +- The MX server and Redis lifecycle is external to TensorRT LLM. Every + TensorRT LLM instance must be able to reach the configured MX server URL. +- The MX server coordinates source discovery but does not store model weights. + A source TensorRT LLM process must remain running and network-reachable until + receiver transfers finish. +- The first worker may still load weights from disk if no compatible MX source + is already registered. +- This page describes the MX checkpoint-loading path only. GPU Memory Service + (GMS) integration is configured separately. diff --git a/docs/source/features/sampling.md b/docs/source/features/sampling.md index d5a9eaef0685..3bee98bb87e7 100644 --- a/docs/source/features/sampling.md +++ b/docs/source/features/sampling.md @@ -13,6 +13,7 @@ The PyTorch backend supports a wide variety of features, listed below: | | Rejection sampling (composable) | Return Logits | | | | Return LogProbs | | | | TopK LogProbs | +| | | Penalties | ## General usage @@ -75,19 +76,24 @@ llm.generate(["Hello, my name is", * The sampling is controlled via `SamplingParams`. -* By default (`temperature = top_p = top_k = None`), greedy sampling is used. +* By default (`temperature = top_p = top_k = None`), greedy sampling is used + (unless min-p or top-p decay is active, see below). -* If either `temperature = 0`, `top_p = 0`, and/or `top_k = 1`, is specified, sampling is greedy, - irrespective of the values of the remaining parameters. +* If either `temperature = 0`, `top_p = 0`, `top_k = 1`, and/or `min_p = 1`, is specified, + sampling is greedy, irrespective of the values of the remaining parameters. * Otherwise, sampling proceeds according to the specified sampling parameter values and any - unspecified parameters default to `top_k = 0`, `top_p = 1`, `temperature = 1.0`: + unspecified parameters default to `top_k = 0`, `top_p = 1`, `min_p = 0`, `temperature = 1.0`: * The logits are scaled by `1/temperature` before applying softmax to compute probabilities. Sampling is performed according to these probabilities. - * If `top_k = 0` (or `top_k = vocab_size`) and `top_p = 1`, the output tokens are sampled - from the entire vocabulary. + * If `top_k = 0` (or `top_k = vocab_size`), `top_p = 1` and `min_p = 0`, the output tokens + are sampled from the entire vocabulary. + + * If `0 < min_p < 1` is specified, the sampling is restricted to the tokens whose probability + is at least `min_p` times the probability of the most likely token ("min-p sampling"). + When combined with `top_k` and/or `top_p`, `min_p` is applied first. * If `1 < top_k < vocab_size` is specified, the sampling is restricted to the `top_k` highest-probability tokens. @@ -101,6 +107,53 @@ llm.generate(["Hello, my name is", * The implementation does not guarantee any particular treatment of tied probabilities. +* Top-P decay is supported: if `top_p_decay < 1` is specified, the effective `top_p` is + multiplied by `top_p_decay` after every sampled token, bounded from below by `top_p_min` + (default `1e-6`), and reset to the initial `top_p` whenever the token `top_p_reset_ids` + is sampled (default `-1`, which never matches a token). Out-of-range values + (`top_p_decay` or `top_p_min` outside `(0, 1]`, negative `top_p_reset_ids`) are rejected. + + * An active top-p decay implies top-p sampling even if `top_p` is unspecified or `top_p = 1` + (the initial `top_p` then defaults to 1). However, explicitly requested greedy sampling + (`temperature = 0`, `top_p = 0`, and/or `top_k = 1`) takes precedence over top-p decay. + + * Top-P decay is not supported in combination with beam search or with speculative decoding + modes that route draft tokens through the Torch Sampler; such requests are rejected. + +* Positive Min-P is not supported in combination with one-model speculative decoding. Such + requests are rejected at admission. + +* Occurrence penalties are supported: `repetition_penalty`, `presence_penalty` and + `frequency_penalty` discourage (or encourage) the model from reusing tokens it has + already seen. All three rewrite the logits before temperature scaling, driven by the + occurrence history of the prompt plus everything generated so far. Writing `c` for the + number of times a token has occurred in that history: + + * `repetition_penalty` (default `1.0`) rescales the logit of every token with `c > 0`: + the logit is divided by the penalty when it is non-negative and multiplied by it when + it is negative. The two branches move a positive and a negative logit the same way, so + a value `> 1` always pushes a seen token down, and a value `< 1` always pulls it up. + Must be `> 0`. + + * `presence_penalty` (default `0.0`) subtracts the penalty itself from every token with + `c > 0`. The amount does not depend on `c`, so it controls whether a token reappears, + not how often. + + * `frequency_penalty` (default `0.0`) subtracts the penalty multiplied by `c`, so the + more often a token has already been produced, the harder it is pushed down. + + * `prompt_ignore_length` (default `0`) excludes the first N prompt tokens from the + presence and frequency counts. Those ignored tokens still count for + `repetition_penalty`. Values `<= 0` have no effect, and values larger than the prompt + are clamped to the prompt length. + + * Occurrence penalties are not supported in combination with beam search; such requests + are rejected. + +* If `no_repeat_ngram_size = n` is specified, any token that would recreate an `n`-gram already + present in the sequence (prompt included) is excluded from sampling. `None` or `0` disables + the restriction. + ### Performance The Torch Sampler leverages the optimized sampling kernels provided by @@ -115,6 +168,49 @@ Moreover, Torch Sampler internally batches requests with compatible sampling par can greatly reduce the overall latency of the sampling step when request batches are comprised of requests with very heterogeneous sampling strategies (e.g. a mix of requests using greedy and top-p-after-top-k sampling). +## Advanced sampling mode (speculative decoding) + +For one-model speculative decoding (e.g. MTP-Eagle one-model), the per-request +advanced sampler applies a `top_k` mask, a temperature softmax, and a `top_p` +filter before sampling each draft/target token. When a deployment fixes its +sampling configuration such that a filter is always disabled (`top_k = 0` / +`top_k = vocab_size`, or `top_p = 1`), that filter's kernel is pure overhead. + +`advanced_sampling_mode` (on `DecodingBaseConfig`, so it is available to any +speculative config) lets you skip those redundant kernels for a fixed deploy +config. The output is identical to `FULL` whenever the skipped filter is already +disabled, so this is a lossless throughput optimization for advanced use cases: + +| Mode | `top_k` kernel | `top_p` kernel | +|---|---|---| +| `full` (default) | applied | applied | +| `no_topk` | **skipped** | applied | +| `no_topp` | applied | **skipped** | +| `no_topk_no_topp` | **skipped** | **skipped** | + +Notes: + +* `full` is the default and always safe; the specialization is opt-in. +* `advanced_sampling_mode` and `use_rejection_sampling` are independent: every mode + works with rejection sampling on or off; the flag no longer gates the mode choice. +* `no_topp` and `no_topk_no_topp` disable `top_p`, switching the sampler from the + fused `top_p_sampling_from_probs` to the cheaper `sampling_from_probs`; `no_topk` + keeps `top_p`. +* Greedy requests are handled natively (via a sentinel temperature that makes the + softmax collapse to a one-hot argmax), so any mode supports mixed greedy + + sampling batches without a special case. +* `advanced_sampling_mode` is a deploy-time choice; it is *not* part of the CUDA + graph key, so it adds no extra warmup graphs. + +```python +from tensorrt_llm.llmapi import MTPDecodingConfig + +spec_config = MTPDecodingConfig( + max_draft_len=3, + advanced_sampling_mode="no_topk_no_topp", # temperature-only deploy config +) +``` + ## Beam search Beam search is a decoding strategy that maintains multiple candidate sequences (beams) during text generation, exploring different possible continuations to find higher quality outputs. Unlike greedy decoding or sampling, beam search considers multiple hypotheses simultaneously. diff --git a/docs/source/index.rst b/docs/source/index.rst index 011d222644fd..15cfe320bd53 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -80,6 +80,7 @@ Welcome to TensorRT LLM's Documentation! features/guided-decoding.md features/speculative-decoding.md features/checkpoint-loading.md + features/model-express.md features/auto_deploy/auto-deploy.md features/auto_deploy/transforms.rst features/ray-orchestrator.md diff --git a/docs/source/legacy/performance/perf-benchmarking.md b/docs/source/legacy/performance/perf-benchmarking.md index 4fc460596f22..8d1567aee020 100644 --- a/docs/source/legacy/performance/perf-benchmarking.md +++ b/docs/source/legacy/performance/perf-benchmarking.md @@ -202,7 +202,7 @@ can simply read a line and assume a complete entry. When creating a dataset, be JSON entry is on every line. ``` -In order to prepare a synthetic dataset, you can use the provided script in the `benchmarks/cpp` +In order to prepare a synthetic dataset, you can use the provided script in the `benchmarks` directory. For example, to generate a synthetic dataset of 1000 requests with a uniform ISL/OSL of 128/128 for [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B), run: @@ -481,7 +481,7 @@ The PyTorch workflow supports benchmarking with LoRA (Low-Rank Adaptation) adapt Use `prepare_dataset.py` with LoRA-specific options to generate requests with LoRA metadata: ```shell -python3 benchmarks/cpp/prepare_dataset.py \ +python3 benchmarks/prepare_dataset.py \ --stdout \ --rand-task-id 0 1 \ --tokenizer /path/to/tokenizer \ @@ -555,7 +555,7 @@ To benchmark multi-modal models with PyTorch workflow, you can follow the simila First, prepare the dataset: ``` -python ./benchmarks/cpp/prepare_dataset.py \ +python ./benchmarks/prepare_dataset.py \ --tokenizer Qwen/Qwen2-VL-2B-Instruct \ --stdout \ dataset \ @@ -846,7 +846,7 @@ The following table summarizes the commands needed for running benchmarks: | Scenario | Phase | Command | | - | - | - | -| Dataset | Preparation | `python benchmarks/cpp/prepare_dataset.py --stdout --tokenizer $HF_MODEL token-norm-dist --input-mean $ISL --output-mean $OSL --input-stdev 0 --output-stdev 0 --num-requests $NUM_REQUESTS > $DATASET_PATH` | +| Dataset | Preparation | `python benchmarks/prepare_dataset.py --stdout --tokenizer $HF_MODEL token-norm-dist --input-mean $ISL --output-mean $OSL --input-stdev 0 --output-stdev 0 --num-requests $NUM_REQUESTS > $DATASET_PATH` | | Throughput | Build | `trtllm-bench --model $HF_MODEL build --dataset $DATASET_PATH` | | Throughput | Benchmark | `trtllm-bench --model $HF_MODEL throughput --dataset $DATASET_PATH --engine_dir $ENGINE_DIR` | | Latency | Build | See [section about building low latency engines](#low-latency-tensorrt-llm-engine-for-llama-3-70b) | diff --git a/docs/source/legacy/performance/performance-tuning-guide/benchmarking-default-performance.md b/docs/source/legacy/performance/performance-tuning-guide/benchmarking-default-performance.md index 17cb9aef45ba..7277b0afa5b0 100644 --- a/docs/source/legacy/performance/performance-tuning-guide/benchmarking-default-performance.md +++ b/docs/source/legacy/performance/performance-tuning-guide/benchmarking-default-performance.md @@ -86,7 +86,7 @@ The README in the examples folder for supported models walks through building en `trtllm-bench` expects to be passed in a dataset of requests to run through the model. This guide creates a dummy dataset of 1000 requests with every request having input and output sequence length of 2048. TensorRT-LLM provides the `prepare_dataset.py` script to produce the dataset. To use it clone the TensorRT-LLM Repo and run the following command: -`python benchmarks/cpp/prepare_dataset.py --stdout --tokenizer /path/to/hf/Llama-3.3-70B-Instruct/ token-norm-dist --input-mean 2048 --output-mean 2048 --input-stdev 0 --output-stdev 0 --num-requests 1000 > synthetic_2048_2048.txt` +`python benchmarks/prepare_dataset.py --stdout --tokenizer /path/to/hf/Llama-3.3-70B-Instruct/ token-norm-dist --input-mean 2048 --output-mean 2048 --input-stdev 0 --output-stdev 0 --num-requests 1000 > synthetic_2048_2048.txt` `trtllm-bench` can also take in real data, see [`trtllm-bench` documentation](../perf-benchmarking.md) for more details on the required format. diff --git a/docs/source/models/encoder-decoder.md b/docs/source/models/encoder-decoder.md new file mode 100644 index 000000000000..5675eed4d742 --- /dev/null +++ b/docs/source/models/encoder-decoder.md @@ -0,0 +1,658 @@ + + +# Use encoder-decoder models with the PyTorch backend + +TensorRT LLM can run supported Hugging Face encoder-decoder checkpoints directly +with the PyTorch backend. You do not need to convert the checkpoint or build a +TensorRT engine. The LLM API treats the supplied prompt as the encoder input and +automatically starts the decoder with the checkpoint's +`decoder_start_token_id` or `bos_token_id`. + +This guide covers text-to-text generation and speech-to-text transcription with +the following Hugging Face architectures: + +| Hugging Face architecture | Model families and examples | +| --- | --- | +| `T5ForConditionalGeneration` | T5, Flan-T5, and ByT5, for example `google/flan-t5-small` | +| `BartForConditionalGeneration` | BART checkpoints | +| `MBartForConditionalGeneration` | mBART checkpoints | +| `WhisperForConditionalGeneration` | Whisper automatic speech recognition (ASR), for example `openai/whisper-large-v3` | + +mBART architecture loading is available. When a BART or mBART checkpoint +defines `forced_bos_token_id`, the PyTorch backend seeds that token in the +decoder prefix. Source- and target-language selection remains +checkpoint-specific, so validate the configured language tokens before +deployment. Refer to [Understand BART and mBART decoder tokens](#understand-bart-and-mbart-decoder-tokens) +for BOS, EOS, and output-limit behavior. + +Whisper consumes audio rather than a text prompt. The sections up to +[Transcribe audio with Whisper](#transcribe-audio-with-whisper) describe the +text-to-text models; the runtime configuration, KV cache manager, beam search, +CUDA graph, and parallelism guidance in this guide applies to all architectures. + +## Feature support + +The following table describes the supported and recommended configurations. + +| Feature | Support | Notes | +| --- | --- | --- | +| KV cache manager V1 | Yes; recommended | This is the default. It supports greedy decoding, beam search, batching, the overlap scheduler, decoder CUDA graphs, and tensor parallelism. | +| KV cache manager V2 | Yes | Set `use_kv_cache_manager_v2=True`. It currently requires `max_beam_width=1`, so use greedy or sampling with a single sequence rather than beam search. | +| Greedy decoding | Yes | Set `temperature=0.0`. | +| Beam search | Yes with V1 | Configure `max_beam_width` when constructing `LLM`, then set `use_beam_search=True` in `SamplingParams`. | +| Attention backend | `TRTLLM` | Use this backend for encoder-decoder models. It is required when `tensor_parallel_size > 1`. | +| Decoder CUDA graphs | Yes, except in FP32 | `CudaGraphConfig` captures decoder work. V1 supports greedy and beam search; V2 supports its single-beam path. FP32 encoder-decoder models decline capture at engine init and log a warning instead of failing. | +| Encoder CUDA graphs | No | `EncodeCudaGraphConfig` is disabled for encoder-decoder models. The encoder runs eagerly. | +| Overlap scheduler | Yes | Enabled by default. V1 supports greedy decoding and beam search; V2 remains limited to `max_beam_width=1`. | +| Tensor parallelism | Yes | Use `tensor_parallel_size > 1` with `attn_backend="TRTLLM"`. Attention head counts must be divisible by the TP size. | +| Pipeline parallelism | No | Keep `pipeline_parallel_size=1`. | +| Context parallelism | No | Keep `context_parallel_size=1`. | +| Attention data parallelism | No | Keep `enable_attention_dp=False`. | +| Chunked prefill | Not supported for the encoder phase | Set `enable_chunked_prefill=False`. The complete encoder input must fit in the iteration token budget. | +| Piecewise CUDA graph | No | Do not set `torch_compile_config.enable_piecewise_cuda_graph=True`. | + +BF16 is the recommended model dtype. Validate accuracy with your checkpoint and +task before deploying a different precision or quantization configuration. + +## Choose the attention backend + +Use `attn_backend="TRTLLM"` for encoder-decoder models. T5 self-attention needs +this backend to apply relative attention bias, and tensor parallel +encoder-decoder execution explicitly requires it. + +The `TRTLLM` backend can internally select optimized kernels when the hardware +and request are eligible. For example, compatible operations on Blackwell can +use FlashInfer TRTLLM-Gen kernels. This internal selection is different from +setting `attn_backend="FLASHINFER"`. Cases such as T5 relative attention bias or +beam-expanded self-attention can fall back to another kernel within the +`TRTLLM` backend; no customer-side backend change is needed. + +## Run basic generation + +Install TensorRT LLM using the [installation guide](../installation/installation-guide.md) +and make sure the checkpoint is accessible either from the Hugging Face Hub or +from a local directory. + +The following example uses KV cache manager V1, greedy decoding, and the overlap +scheduler: + +```python +from tensorrt_llm.llmapi import LLM, KvCacheConfig, SamplingParams, SchedulerConfig + + +model = "google/flan-t5-small" + +with LLM( + model=model, + backend="pytorch", + max_batch_size=4, + max_input_len=512, + max_num_tokens=2048, + max_seq_len=512, + kv_cache_config=KvCacheConfig( + enable_block_reuse=False, + free_gpu_memory_fraction=0.8, + cross_kv_cache_fraction=0.5, + use_kv_cache_manager_v2=False, + ), + scheduler_config=SchedulerConfig(use_python_scheduler=True), +) as llm: + sampling_params = SamplingParams( + max_tokens=64, + temperature=0.0, + ) + result = llm.generate( + "translate English to German: The house is wonderful.", + sampling_params=sampling_params, + use_tqdm=False, + ) + print(result.outputs[0].text) +``` + +Use the task format expected by the checkpoint. For example, T5 translation +checkpoints commonly expect a task prefix such as `translate English to +German:`, while a summarization checkpoint expects the source document. + +The LLM API performs these encoder-decoder-specific steps automatically: + +1. Tokenizes the supplied string as the encoder input. +2. Runs the encoder once and retains its output for cross-attention. +3. Initializes the decoder from the checkpoint's decoder start token. +4. Generates decoder tokens and returns the detokenized decoder output. + +Do not prepend a decoder start token to the source prompt. If you pass token IDs +instead of text, pass only the encoder-side token IDs: + +```python +from transformers import AutoTokenizer + + +tokenizer = AutoTokenizer.from_pretrained(model) +source_text = "translate English to German: The house is wonderful." +source_token_ids = tokenizer.encode(source_text, add_special_tokens=True) +result = llm.generate(source_token_ids, sampling_params=sampling_params) +``` + +### Configure an mBART tokenizer + +mBART tokenization depends on the source language. Create the Hugging Face +tokenizer with `src_lang` and pass that tokenizer to `LLM` so string prompts +receive the correct source-language token: + +```python +from transformers import AutoTokenizer + +from tensorrt_llm.llmapi import LLM, KvCacheConfig, SamplingParams + + +model = "/path/to/mbart-large-50-many-to-one-mmt" +tokenizer = AutoTokenizer.from_pretrained(model, src_lang="ro_RO") + +with LLM( + model=model, + tokenizer=tokenizer, + backend="pytorch", + attn_backend="TRTLLM", + dtype="bfloat16", + enable_chunked_prefill=False, + kv_cache_config=KvCacheConfig(cross_kv_cache_fraction=0.5), +) as llm: + result = llm.generate( + "Şeful ONU spune că nu există o soluţie militară în Siria.", + sampling_params=SamplingParams(max_tokens=64, temperature=0.0), + use_tqdm=False, + ) + print(result.outputs[0].text) +``` + +For this many-to-one checkpoint, `generation_config.json` selects English with +the `en_XX` forced BOS token. For other mBART checkpoints, confirm that +`decoder_start_token_id`, `forced_bos_token_id`, `eos_token_id`, and the +tokenizer language settings select the source and target languages you intend +to serve. + +### Understand BART and mBART decoder tokens + +When a BART or mBART checkpoint defines `forced_bos_token_id`, the PyTorch +backend initializes the decoder with the following internal prefix: + +```text +[decoder_start_token_id, forced_bos_token_id] +``` + +For example, BART-large-CNN uses `[2, 0]`. Customers provide only the encoder +input; do not prepend either decoder token. By default, the returned token IDs +exclude `decoder_start_token_id` but include `forced_bos_token_id`, so the +BART-large-CNN output begins with token ID 0. + +The forced BOS token counts against `SamplingParams.max_tokens`. Consequently, +a request using this prefix requires `max_tokens` to be at least 2. The runtime +uses the remaining token budget for model-selected tokens. This behavior is the +same for greedy decoding and beam search and does not require a customer logits +processor. + +EOS is a stopping token rather than a forced final token. The runtime uses +`SamplingParams.end_id`, which defaults to the tokenizer's `eos_token_id`. If +the model generates EOS before the output limit, the returned token IDs include +EOS and `finish_reason` is `"stop"`. Set `ignore_eos=True` to continue decoding +past EOS. + +The runtime does not inject `forced_eos_token_id` when a sequence reaches +`max_tokens`. It preserves the model-selected final token and reports +`finish_reason="length"`. + +## Transcribe audio with Whisper + +Whisper is an audio encoder-decoder model for speech transcription and +translation. It differs from the text-to-text models in this guide in two ways: +the encoder consumes audio instead of a text prompt, and the text prompt, when +supplied, sets the decoder task prefix rather than the encoder input. + +Pass one audio clip per request through `multi_modal_data["audio"]`: + +```python +import soundfile + +from tensorrt_llm.llmapi import LLM, KvCacheConfig, SamplingParams, SchedulerConfig + + +model = "openai/whisper-large-v3" +wave, sample_rate = soundfile.read("utterance.wav") + +with LLM( + model=model, + backend="pytorch", + attn_backend="TRTLLM", + max_batch_size=4, + # Cross-KV pool capacity. The default (1024) is smaller than the 1500 + # encoder positions that every Whisper request produces. + max_input_len=1500, + max_num_tokens=3000, + enable_chunked_prefill=False, + kv_cache_config=KvCacheConfig( + enable_block_reuse=False, + free_gpu_memory_fraction=0.8, + cross_kv_cache_fraction=0.5, + ), + scheduler_config=SchedulerConfig(use_python_scheduler=True), +) as llm: + result = llm.generate( + {"prompt": "", "multi_modal_data": {"audio": [(wave, sample_rate)]}}, + sampling_params=SamplingParams(max_tokens=96, temperature=0.0), + use_tqdm=False, + ) + print(result.outputs[0].text) +``` + +Set `max_input_len` to at least 1500. Every Whisper request produces 1500 encoder +positions regardless of clip length, and the cross-KV pool is sized from `max_input_len`. + +The audio item accepts a file path or URL, an `(array, sample_rate)` tuple, or a +`{"array": ..., "sampling_rate": ...}` mapping. Supply exactly one clip per +request, at the checkpoint's sampling rate, which is 16 kHz for the published +Whisper checkpoints. Clips shorter than the 30-second window are zero-padded; +longer clips and other sampling rates are rejected rather than silently +truncated or resampled. Long-form chunked transcription is not part of this +path. + +### Select the language and task + +An empty text prompt selects the checkpoint default, which is English +transcription (`<|startoftranscript|>[<|en|>][<|transcribe|>]<|notimestamps|>`). +A non-empty text prompt replaces that decoder prefix verbatim and is how you +override the language or switch to translation. It must begin with +`<|startoftranscript|>`: + +```python +prompt = "<|startoftranscript|><|de|><|transcribe|><|notimestamps|>" +result = llm.generate( + {"prompt": prompt, "multi_modal_data": {"audio": [(wave, sample_rate)]}}, + sampling_params=SamplingParams(max_tokens=96, temperature=0.0), +) +``` + +The prefix counts against the decoder position table, so `SamplingParams.max_tokens` +is capped to the space remaining in it. A request asking for more logs a warning +and proceeds with the capped budget. Pre-tokenized `prompt_token_ids` are not +consumed on this path. + +## Run a batch + +Pass a list of strings to batch inputs. The strings can have different tokenized +lengths: + +```python +sources = [ + "translate English to German: The house is wonderful.", + "translate English to German: The book is on the table.", +] + +results = llm.generate(sources, sampling_params=sampling_params, use_tqdm=False) +for source, result in zip(sources, results): + print(f"source={source!r} output={result.outputs[0].text!r}") +``` + +`max_num_tokens` must cover the encoder tokens admitted in an iteration as well +as decoder work. Increase it for larger batches or longer source sequences. + +## Choose KV cache manager V1 or V2 + +Encoder-decoder execution uses two KV cache pools: + +- The self-attention pool stores decoder-side KV states. +- The cross-attention pool stores encoder-derived K/V states used by every + decoder layer. + +`cross_kv_cache_fraction` is required for every encoder-decoder model. It divides +the configured KV cache memory budget between the two pools. A value of `0.5` +is a reasonable starting point: + +```python +kv_cache_config = KvCacheConfig( + free_gpu_memory_fraction=0.8, + cross_kv_cache_fraction=0.5, + use_kv_cache_manager_v2=False, +) +``` + +Increase `cross_kv_cache_fraction` when long encoder inputs exhaust the cross +pool. Decrease it when long decoder outputs or wide beams exhaust the +self-attention pool. The two fractions are related as follows: + +```text +cross-attention pool = total KV cache budget * cross_kv_cache_fraction +self-attention pool = total KV cache budget * (1 - cross_kv_cache_fraction) +``` + +V1 is the default and should be the first choice for production deployments. +To evaluate V2, change only the manager selection and keep beam width equal to +one: + +```python +kv_cache_config = KvCacheConfig( + free_gpu_memory_fraction=0.8, + cross_kv_cache_fraction=0.5, + use_kv_cache_manager_v2=True, +) + +llm = LLM( + model=model, + backend="pytorch", + attn_backend="TRTLLM", + max_beam_width=1, + kv_cache_config=kv_cache_config, +) +``` + +KV cache manager V2 is a prototype feature and rejects configurations with a +maximum beam width greater than one. + +## Use beam search + +Beam search requires KV cache manager V1. The maximum beam width is a runtime +capacity setting and must be specified when constructing `LLM`: + +```python +beam_width = 4 + +with LLM( + model="/path/to/bart-large-cnn", + backend="pytorch", + attn_backend="TRTLLM", + dtype="bfloat16", + max_beam_width=beam_width, + enable_chunked_prefill=False, + kv_cache_config=KvCacheConfig( + free_gpu_memory_fraction=0.8, + cross_kv_cache_fraction=0.5, + use_kv_cache_manager_v2=False, + ), +) as llm: + beam_params = SamplingParams( + best_of=beam_width, + max_tokens=64, + n=beam_width, + temperature=0.0, + use_beam_search=True, + ) + result = llm.generate( + "The engineering team released a faster inference service on Monday. " + "The update improves batching, lowers latency, and adds detailed " + "monitoring for operators.", + sampling_params=beam_params, + use_tqdm=False, + ) + + for hypothesis in result.outputs: + print(hypothesis.text) +``` + +`best_of` sets the beam width and must not exceed `LLM.max_beam_width`. `n` +sets the number of returned hypotheses and must not exceed `best_of`. Set `n=1` +to return only the best hypothesis. + +Beam search expands decoder-side cache and compute requirements. Include this +expansion when sizing the self-attention KV pool and CUDA graph batch sizes. + +## Enable decoder CUDA graphs + +Pass `CudaGraphConfig` to capture and replay decoder iterations: + +```python +from tensorrt_llm.llmapi import CudaGraphConfig + + +llm = LLM( + model=model, + backend="pytorch", + attn_backend="TRTLLM", + max_batch_size=8, + cuda_graph_config=CudaGraphConfig( + max_batch_size=8, + enable_padding=True, + ), + kv_cache_config=KvCacheConfig( + free_gpu_memory_fraction=0.8, + cross_kv_cache_fraction=0.5, + ), +) +``` + +This configuration captures decoder work only; the encoder continues to run +eagerly. With beam search, graph batch sizes must cover the active decoder +sequences after beam expansion. Padding lets nearby runtime batch sizes reuse a +captured graph. + +Do not use `EncodeCudaGraphConfig` for an encoder-decoder model. The runtime +warns and disables it. Piecewise CUDA graphs through `TorchCompileConfig` are +also unsupported for this model type. + +## Control the overlap scheduler + +The PyTorch backend enables the overlap scheduler by default. The examples set +`disable_overlap_scheduler=False` explicitly to make that choice visible: + +```python +llm = LLM( + model=model, + backend="pytorch", + disable_overlap_scheduler=False, + kv_cache_config=KvCacheConfig(cross_kv_cache_fraction=0.5), +) +``` + +Overlap is not restricted to KV cache manager V1. Both V1 and V2 enter the same +overlap executor loop, and that loop contains V2-specific resource handling. +V2 remains limited to `max_beam_width=1`. Set +`disable_overlap_scheduler=True` when debugging. + +## Use tensor parallelism + +Set `tensor_parallel_size` to the number of GPUs over which to shard the model: + +```python +with LLM( + model=model, + backend="pytorch", + attn_backend="TRTLLM", + tensor_parallel_size=2, + pipeline_parallel_size=1, + context_parallel_size=1, + enable_attention_dp=False, + enable_chunked_prefill=False, + kv_cache_config=KvCacheConfig( + free_gpu_memory_fraction=0.8, + cross_kv_cache_fraction=0.5, + use_kv_cache_manager_v2=False, + ), +) as llm: + result = llm.generate(source_text, sampling_params=sampling_params) +``` + +For single-node execution through the LLM API, do not add an `mpirun` prefix. +TensorRT LLM starts the worker processes. The selected TP size must divide the +encoder and decoder attention head counts. Cross-attention KV head duplication +is not supported, so its KV head count must also be divisible by the TP size. + +Tensor parallelism currently requires `attn_backend="TRTLLM"`. Pipeline +parallelism, context parallelism, and attention DP are rejected for +encoder-decoder models. + +## Serve an encoder-decoder model + +The following configuration starts a greedy Flan-T5 service with the PyTorch +backend. Save it as `enc-dec-config.yaml`: + +```yaml +attn_backend: TRTLLM +dtype: bfloat16 +disable_overlap_scheduler: false +enable_chunked_prefill: false +max_beam_width: 1 +max_input_len: 512 +max_num_tokens: 2048 +max_seq_len: 512 +kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + cross_kv_cache_fraction: 0.5 + use_kv_cache_manager_v2: false +scheduler_config: + use_python_scheduler: true +``` + +Start the server: + +```bash +trtllm-serve google/flan-t5-small \ + --backend pytorch \ + --max_batch_size 4 \ + --config enc-dec-config.yaml +``` + +Send the source text through the completions endpoint: + +```bash +curl http://localhost:8000/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "google/flan-t5-small", + "prompt": "translate English to German: The house is wonderful.", + "max_tokens": 64, + "temperature": 0.0 + }' +``` + +To serve beam search, use KV cache manager V1, restart the server with +`max_beam_width` set to the desired maximum, and add the following request +fields: + +```json +{ + "use_beam_search": true, + "best_of": 4, + "n": 1 +} +``` + +For tensor parallel serving, add `--tp_size ` to `trtllm-serve` and keep the +attention backend set to `TRTLLM`. + +## Size the runtime + +Use these guidelines as a starting point: + +- Set `max_input_len` to at least the maximum tokenized encoder input length. +- Set `max_seq_len` to at least the larger of the maximum encoder input length + and maximum decoded sequence length. The current encoder-decoder runtime uses + this value while sizing both phases. +- Set `max_num_tokens` high enough for all encoder tokens admitted together and + for the active decoder tokens. This is especially important for mixed-length + batches. +- Increase `max_batch_size` for more concurrent requests. Beam width multiplies + the number of active decoder sequences but not the number of source requests. +- Tune `free_gpu_memory_fraction` first, then tune + `cross_kv_cache_fraction` based on whether the cross-attention or + self-attention pool is exhausted. + +## Performance + +The following benchmarks compare the PyTorch backend with the legacy TensorRT +encoder-decoder path for large-batch inference. The measurements use BF16 on +one H100 80 GB GPU with greedy decoding, an output limit of 128 tokens, and +mixed encoder input lengths from 260 to 440 tokens. The Flan-T5-XL results are +the average of ten timed runs after three warmup runs. The BART results are the +average of 20 timed runs after five warmup runs. Executed-token throughput +includes the terminal EOS token when a sequence emits it. + +The PyTorch configuration uses the `TRTLLM` attention backend, the overlap +scheduler, the Python scheduler, decoder CUDA graphs with padding, KV cache +manager V1, `max_input_len=512`, `max_seq_len=1024`, and +`max_num_tokens=65536`. Block reuse and chunked prefill are disabled. The KV +cache uses `free_gpu_memory_fraction=0.3` and +`cross_kv_cache_fraction=0.5`. + +The legacy TensorRT configuration uses separate BF16 encoder and decoder +engines built for batch size 128 and beam width 1. The encoder supports 512 +input tokens and 65,536 tokens per iteration; the decoder supports a sequence +length of 129. The benchmark runs these engines through `ModelRunnerCpp` with +greedy `top_k=1` decoding and the same KV cache fractions. For BART, the legacy +TensorRT benchmark starts the decoder with token IDs `[2, 0]` and generates at +most 127 more tokens. The PyTorch LLM API applies the same decoder prefix +internally and counts token ID 0 as the first output token; customers do not +need to provide the decoder prefix. Both paths use token ID 2 as EOS and stop +when the model generates it naturally. If a sequence reaches the output limit, +it retains the model-selected final token and reports a length stop instead of +forcing EOS. This setup also lets beam search begin after the shared decoder +prefix without a per-step Python logits processor. + +### Flan-T5-XL + +For Flan-T5-XL, the PyTorch backend performs on par with the legacy TensorRT +path, with slightly lower latency and higher executed-token throughput across +the tested batch sizes. + +| Batch size | Legacy TensorRT latency | PyTorch latency | PyTorch latency improvement over legacy TensorRT | Legacy TensorRT executed tokens/s | PyTorch executed tokens/s | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 32 | 727.6 ms | 706.1 ms | 3.0% | 3,153 | 3,312 | +| 64 | 1,225.0 ms | 1,136.7 ms | 7.2% | 3,863 | 4,184 | +| 128 | 2,056.8 ms | 1,999.3 ms | 2.8% | 4,601 | 4,768 | + +### BART-large-CNN + +For BART-large-CNN, the PyTorch backend has 21.9% to 36.0% higher latency than +the legacy TensorRT path across the tested batch sizes. + +| Batch size | Legacy TensorRT latency | PyTorch latency | PyTorch latency difference | Legacy TensorRT executed tokens/s | PyTorch executed tokens/s | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 32 | 229.9 ms | 280.2 ms | 21.9% slower | 12,611 | 10,662 | +| 64 | 252.2 ms | 343.0 ms | 36.0% slower | 22,007 | 16,209 | +| 128 | 352.3 ms | 472.7 ms | 34.2% slower | 31,544 | 23,518 | + +Performance depends on the model, request distribution, decoding settings, and +GPU configuration. Benchmark with a representative workload before deployment. + +## Troubleshooting + +### `cross_kv_cache_fraction` is required + +Every encoder-decoder runtime needs a cross-attention KV pool. Add +`KvCacheConfig(cross_kv_cache_fraction=...)`; `0.5` is a reasonable initial +value. Do not set this field for a decoder-only model. + +### `decoder_start_token_id` is required + +The checkpoint must define `decoder_start_token_id` or `bos_token_id` in its +Hugging Face model or generation configuration. Use a checkpoint with a complete +`config.json` and, when applicable, `generation_config.json`. + +### KV cache manager V2 fails with beam search + +V2 currently requires `max_beam_width=1`. Select V1 by setting +`use_kv_cache_manager_v2=False` before enabling beam search. + +### Tensor parallel initialization is rejected + +Check all of the following: + +- `attn_backend` is `TRTLLM`. +- Encoder, decoder, and cross-attention head counts are divisible by the TP + size. +- `pipeline_parallel_size=1` and `context_parallel_size=1`. +- `enable_attention_dp=False`. + +### CUDA graphs do not capture the encoder + +This is expected. `CudaGraphConfig` accelerates decoder iterations only. The +encoder path runs eagerly, and `EncodeCudaGraphConfig` is disabled for +encoder-decoder models. + +### Output quality differs from the Hugging Face example + +Confirm that the source uses the task prefix and language settings expected by +the checkpoint. Also compare the same model dtype, beam width, length penalty, +EOS stopping behavior, and forced BOS configuration. Small numerical +differences can change lower-ranked beam hypotheses when scores are close. diff --git a/docs/source/models/supported-models.md b/docs/source/models/supported-models.md index 46f90a9df31c..4f4db5525b60 100644 --- a/docs/source/models/supported-models.md +++ b/docs/source/models/supported-models.md @@ -33,6 +33,7 @@ The following is a table of supported models for the PyTorch backend: | `LagunaForCausalLM` | Laguna-XS | `poolside/laguna-XS.2` | | `LlamaForCausalLM` | Llama 3.1, Llama 3, Llama 2, LLaMA | `meta-llama/Meta-Llama-3.1-70B` | | `Llama4ForConditionalGeneration` | Llama 4 | `meta-llama/Llama-4-Scout-17B-16E-Instruct` | +| `MiniCPMV4_6ForConditionalGeneration` [^14]| MiniCPM-V 4.6 | `openbmb/MiniCPM-V-4.6` | | `MiniMaxM2ForCausalLM` [^5] | MiniMax M2/M2.1/M2.7 | `MiniMaxAI/MiniMax-M2.7` | | `MiniMaxM3SparseForConditionalGeneration` [^12]| MiniMax-M3 | `MiniMaxAI/MiniMax-M3` | | `MistralForCausalLM` | Mistral | `mistralai/Mistral-7B-v0.1` | @@ -80,8 +81,8 @@ Note: Support for other models may vary. Features marked "N/A" are not applicabl | `Step3p7ForConditionalGeneration`| Yes | Yes | Yes | Untested | Untested | Yes | No | No | No | Yes | Untested | Untested | Yes | Untested | Untested | | `MiniMaxM3SparseForConditionalGeneration` [^12] | Yes | Yes | Yes | Untested | Untested | No | No | No | No | Yes | Untested | No | N/A | Untested | Untested | -[^1]: Chunked Prefill for MLA can only be enabled on SM100/SM103. -[^2]: KV cache reuse for MLA can only be enabled on SM90/SM100/SM103 and in BF16/FP8 KV cache dtype. +[^1]: Chunked Prefill for MLA can only be enabled on SM90/SM100/SM103/SM120. +[^2]: KV cache reuse for MLA can only be enabled on SM90/SM100/SM103/SM120/SM121 and in BF16/FP8 KV cache dtype. [^3]: Qwen3-Next-80B-A3B exhibits relatively low accuracy on the SciCode-AA-v2 benchmark. [^5]: Supported via the [AutoDeploy](../features/auto_deploy/auto-deploy.md) backend. See [AD Configs](../../../examples/auto_deploy/model_registry/configs). [^6]: Also supports text-only inference via the [AutoDeploy](../features/auto_deploy/auto-deploy.md) backend. @@ -92,6 +93,7 @@ Note: Support for other models may vary. Features marked "N/A" are not applicabl [^11]: DeepSeek-V4 is only supported on Blackwell GPUs (`SM100+`). See the [DeepSeek-V4 example README](../../../examples/models/core/deepseek_v4/README.md) for setup and parallelism. [^12]: Supports text, image, and video inputs over the block-sparse attention path. The published MXFP8 checkpoint is dequantized on load so the runtime sees an effectively BF16 model. The text decoder is also usable standalone (text-only) via the `MiniMaxM3SparseForCausalLM` architecture. KV cache reuse and MTP are not supported on the sparse-attention path in this release. [^13]: The Cosmos 3 family also supports visual generation through the VisualGen API. See [Visual Generation Models](#visual-generation-models). +[^14]: Requires `transformers>=5.7.0`: MiniCPM-V 4.6 was upstreamed into transformers as a native model type (`minicpmv4_6`) and the checkpoint ships no remote code (`auto_map`) to fall back on. The Qwen3.5-hybrid text tower runs in BF16. Image, video, and text inputs are supported in this release (video reuses the same NaViT-packed vision path as image via `MiniCPMV4_6InputProcessor`). # Multimodal Feature Support Matrix (PyTorch Backend) @@ -105,6 +107,7 @@ Note: Support for other models may vary. Features marked "N/A" are not applicabl | `LlavaLlamaModel (VILA)` | Yes | Yes | No | Yes | Yes | No | Yes | No | L + I + V | | `LlavaNextForConditionalGeneration` | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | L + I | | `Llama4ForConditionalGeneration` | Yes | Yes | No | Yes | Yes | No | Yes | No | L + I | +| `MiniCPMV4_6ForConditionalGeneration` [^14] | Yes | Untested | Untested | Yes | Untested | Untested | Untested | No | L + I + V | | `Mistral3ForConditionalGeneration` | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | L + I | | `NemotronH_Nano_VL_V2` | Yes | Yes | Yes | Yes | Yes | N/A | Yes | Yes | L + I + V + A [^10] | | `Phi4MMForCausalLM` | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | L + I + A | @@ -127,22 +130,31 @@ Note: ## Multimodal Encoder Optimizations The following optimizations are available to models that implement -`MultimodalModelMixin`. Currently, only `Mistral3ForConditionalGeneration` supports them. +`MultimodalModelMixin`. | Model Architecture | Multimodal Encoder Side Stream | Multimodal Embeddings Cache | | ------------------ | ------------------------------ | --------------------------- | +| `Gemma4ForConditionalGeneration` | Yes | Yes | +| `Gemma4UnifiedForConditionalGeneration` | Yes | Yes | | `Mistral3ForConditionalGeneration` | Yes | Yes | +| `Qwen3VLForConditionalGeneration` | Yes | Yes | +| `Qwen3VLMoeForConditionalGeneration` | Yes | Yes | +| `Qwen3_5ForConditionalGeneration` | Yes | Yes | +| `Qwen3_5MoeForConditionalGeneration` | Yes | Yes | - **Multimodal encoder side stream** prefetches encoder work for pending requests on a separate CUDA stream, allowing it to overlap with work on the main stream. Set `multimodal_config.encoder_side_stream_max_ahead` to a positive value to enable it; the value limits the number of prefetched requests that can be ahead of admission. This option is mutually - exclusive with `multimodal_config.encoder_cuda_graph` and can increase peak GPU memory use. + exclusive with `multimodal_config.encoder_cuda_graph` and can increase peak GPU memory use. It + can be combined with the multimodal embeddings cache so side-stream cache hits skip encoder work + and misses populate the cache. - **Multimodal embeddings cache** is a per-model, cross-request LRU cache of encoder embeddings. Set `multimodal_config.encoder_cache_max_bytes` to its capacity (for example, `"512MiB"`), or `0` to disable it. Entries are cached per multimodal item, but a request reuses cached embeddings only when all of its items hit the cache. At present, only single-modality requests are cacheable; - mixed-modality requests bypass the cache. + mixed-modality requests bypass the cache. When combined with side-stream prefetch, peak memory is + the cache capacity plus any in-flight prefetched encoder inputs and outputs. # Visual Generation Models @@ -165,8 +177,12 @@ For full documentation, see the [Visual Generation](./visual-generation.md) page | `Lightricks/LTX-2` | Text-to-Video (with Audio), Image-to-Video (with Audio) | | `Qwen/Qwen-Image` | Text-to-Image | | `Qwen/Qwen-Image-2512` | Text-to-Image | +| `Qwen/Qwen-Image-Layered` | Image-to-Image | +| `Qwen/Qwen-Image-Edit-2511` | Image Editing (text+images-to-image) | | `nvidia/Cosmos3-Nano` | Text-to-Image, Text-to-Video, Image-to-Video | | `nvidia/Cosmos3-Super` | Text-to-Image, Text-to-Video, Image-to-Video | +| `nvidia/Cosmos3-Super-Text2Image-4Step` | Text-to-Image (DMD2-distilled, fixed 4-step schedule) | +| `nvidia/Cosmos3-Super-Image2Video-4Step` | Image-to-Video (DMD2-distilled, fixed 4-step schedule) | ### Feature Matrix @@ -177,7 +193,10 @@ For full documentation, see the [Visual Generation](./visual-generation.md) page | **Wan 2.1** | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | | **Wan 2.2** | Yes | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | | **LTX-2** | Yes | Yes | No | Yes | Yes | No | No | Yes | Yes | Yes | Yes | No | -| **Qwen-Image** [^2] | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| **Qwen-Image** | Yes | Yes | No | Yes | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| **Qwen-Image-Layered** [^3] | No | No | No | No | No | No | Yes | Yes | No | No | No | No | +| **Qwen-Image-Edit-2511** | Yes | Yes | No | Yes | No | No | Yes | Yes | No | No | No | No | | **Cosmos3** | Yes | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | [^vg1]: FLUX models use embedded guidance and do not have a separate negative prompt path, so CFG parallelism is not applicable. +[^3]: Qwen-Image-Layered supports baseline BF16 image-conditioned layer decomposition. FP8 blockwise, NVFP4, `trtllm-serve` image-edit routing, and attention-parallel backends are not enabled yet. diff --git a/docs/source/models/visual-generation.md b/docs/source/models/visual-generation.md index d1e376b9a13e..01ddf2944a95 100644 --- a/docs/source/models/visual-generation.md +++ b/docs/source/models/visual-generation.md @@ -37,8 +37,12 @@ TensorRT-LLM **VisualGen** provides a unified inference stack for diffusion mode | `Lightricks/LTX-2` | Text-to-Video (with Audio), Image-to-Video (with Audio) | | `Qwen/Qwen-Image` | Text-to-Image | | `Qwen/Qwen-Image-2512` | Text-to-Image | +| `Qwen/Qwen-Image-Layered` | Image-to-Image | +| `Qwen/Qwen-Image-Edit-2511` | Image Editing (text+images-to-image) | | `nvidia/Cosmos3-Nano` | Text-to-Image, Text-to-Video, Image-to-Video | | `nvidia/Cosmos3-Super` | Text-to-Image, Text-to-Video, Image-to-Video | +| `nvidia/Cosmos3-Super-Text2Image-4Step` | Text-to-Image (DMD2-distilled, fixed 4-step schedule) | +| `nvidia/Cosmos3-Super-Image2Video-4Step` | Image-to-Video (DMD2-distilled, fixed 4-step schedule) | Models are auto-detected from the checkpoint directory. Diffusers-format models are detected via `model_index.json`; LTX-2 monolithic safetensors checkpoints are detected via embedded metadata. The `AutoPipeline` registry selects the appropriate pipeline class automatically. @@ -52,7 +56,9 @@ Models are auto-detected from the checkpoint directory. Diffusers-format models | **Wan 2.1 VSA** [^2] | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | | **Wan 2.2** | Yes | Yes | Yes [^3] | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | | **LTX-2** | Yes | Yes | Yes [^4] | Yes | Yes | Yes | No | No | Yes | Yes | Yes | Yes | No | No | -| **Qwen-Image** [^5] | Yes | Yes | No | No | No | Yes | No | Yes | Yes | Yes | Yes | Yes | No | No | +| **Qwen-Image** | Yes | Yes | No | No | Yes | Yes | No | Yes | Yes | Yes | Yes | Yes | No | No | +| **Qwen-Image-Layered** [^6] | No | No | No | No | No | No | No | Yes | Yes | No | No | No | No | No | +| **Qwen-Image-Edit-2511** | Yes | Yes | No | No | Yes | No | No | Yes | Yes | No | No | No | No | No | | **Cosmos3** | Yes | Yes | No | No | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | No | [^1]: FLUX models use embedded guidance and do not have a separate negative prompt path, so CFG parallelism is not applicable. @@ -63,7 +69,7 @@ Models are auto-detected from the checkpoint directory. Diffusers-format models [^4]: LTX-2 has no built-in TeaCache coefficient table in TRT-LLM; set `teacache.coefficients` explicitly when enabling TeaCache. -[^5]: Qwen-Image ships a native BF16 implementation with per-module numerical parity against `diffusers.QwenImagePipeline` (cosine similarity >= 0.999 on the full 20B transformer) and supports `trtllm-serve` / `/v1/images/generations`. VisualGen supports FP8 blockwise and NVFP4 dynamic quantization from BF16 checkpoints, as well as direct loading of statically quantized FP8 and NVFP4 ModelOpt checkpoints. +[^6]: Qwen-Image-Layered supports baseline BF16 image-conditioned layer decomposition and returns the generated RGBA layer stack as a saveable image grid. FP8 blockwise, NVFP4, cache acceleration, attention-parallel/Sage/VSA backends, Tensor Parallelism, and `trtllm-serve` image-edit routing are not enabled for this pipeline yet. ## Quick Start diff --git a/examples/auto_deploy/paragraf/README.md b/examples/auto_deploy/paragraf/README.md index 55600b7e0ede..07d96ae6fc10 100644 --- a/examples/auto_deploy/paragraf/README.md +++ b/examples/auto_deploy/paragraf/README.md @@ -64,10 +64,22 @@ In standalone mode the package uses the PyTorch, Triton, and FlashInfer kernel p ### Run the bundled tests +The generated repository contains every AutoDeploy test selected by the +TensorRT-LLM test-name classifier. Tests that need TensorRT-LLM skip cleanly +when its optional wheel is absent. + ```bash pytest tests/ ``` +To include tests that exercise TensorRT-LLM kernels, runtime, or test helpers, +install the optional wheel and explicitly enable the Paragraf redirect: + +```bash +uv pip install -e ".[dev,trtllm]" +TRTLLM_REDIRECT_AD_TO_PARAGRAF=true pytest tests/ +``` + ______________________________________________________________________ ## Building a custom inference pipeline diff --git a/examples/auto_deploy/paragraf/create_standalone_package.py b/examples/auto_deploy/paragraf/create_standalone_package.py index ccd919c61303..d2e75e5cbf39 100644 --- a/examples/auto_deploy/paragraf/create_standalone_package.py +++ b/examples/auto_deploy/paragraf/create_standalone_package.py @@ -41,6 +41,7 @@ import os import re import shutil +import subprocess import sys import textwrap @@ -65,8 +66,8 @@ # Test source directories AD_TESTS_DIR = os.path.join(REPO_ROOT, "tests", "unittest", "auto_deploy") -AD_UTILS_TEST_DIR = os.path.join(AD_TESTS_DIR, "_utils_test") AD_TORCH_TESTS_DIR = os.path.join(REPO_ROOT, "tests", "unittest", "_torch", "auto_deploy") +AD_INTEGRATION_TESTS_DIR = os.path.join(REPO_ROOT, "tests", "integration", "defs") # Example/e2e harness sources (Tier-1 e2e: build_and_run_ad.py + model registry). # These ship with the package so the standalone install can run e2e models via @@ -141,14 +142,50 @@ "pytest-xdist", "pytest-cov", "pytest-mock", + "pytest-asyncio", "parameterized", + "cloudpickle", + "mpi4py", + "openai", + "requests", + "scipy", ] -# Test directories to exclude from the standalone package (require TRT-LLM runtime) -EXCLUDE_TEST_DIRS = {"smoke", "shim", "standalone"} +# These tests validate the generator and its output. They belong to the source +# repository rather than the generated package. +SOURCE_ONLY_TEST_DIRS = {"standalone"} + +# Source test names retain their AutoDeploy identity inside TensorRT-LLM. Rename +# them only in the generated repository, where they exercise Paragraf through +# the optional TensorRT-LLM integration. +PARAGRAF_TRTLLM_TEST_RENAMES = { + "test_llm_api_autodeploy.py": "test_llm_api_paragraf_trtllm.py", + "test_ad_disagg.py": "test_paragraf_trtllm_disagg.py", + "test_ad_disagg_trtllm_serve.py": "test_paragraf_trtllm_disagg_serve.py", + "test_ad_guided_decoding.py": "test_paragraf_trtllm_guided_decoding.py", + "test_ad_speculative_decoding.py": "test_paragraf_trtllm_speculative_decoding.py", + "test_ad_dist_strategies.py": "test_paragraf_trtllm_dist_strategies.py", + "test_ad_allreduce_strategies.py": "test_paragraf_trtllm_allreduce_strategies.py", + "test_ad_build_small_multi.py": "test_paragraf_trtllm_build_small_multi.py", + "test_ad_moe_op.py": "test_paragraf_trtllm_moe_op.py", + "test_ad_executor_swa_eviction.py": "test_paragraf_trtllm_executor_swa_eviction.py", + "test_create_ad_executor.py": "test_create_paragraf_trtllm_executor.py", + "test_ad_build_small_single.py": "test_paragraf_trtllm_build_small_single.py", + "test_ad_guided_decoding_regex.py": "test_paragraf_trtllm_guided_decoding_regex.py", + "test_ad_trtllm_bench.py": "test_paragraf_trtllm_bench.py", + "test_ad_trtllm_sampler.py": "test_paragraf_trtllm_sampler.py", + "test_ad_trtllm_serve.py": "test_paragraf_trtllm_serve.py", +} +SOURCE_TEST_NAMES_BY_GENERATED_NAME = { + generated_name: source_name + for source_name, generated_name in PARAGRAF_TRTLLM_TEST_RENAMES.items() +} -# Individual test files to exclude (require TRT-LLM runtime/kernels or external scripts) -EXCLUDE_TEST_FILES = { +# Tests in this set are copied, but only collected when the optional TensorRT-LLM +# wheel is enabled through TRTLLM_REDIRECT_AD_TO_PARAGRAF. Keeping this explicit +# also covers indirect dependencies where a rewritten ``paragraf`` import loads +# a module that depends on TensorRT-LLM. +OPTIONAL_TRTLLM_TEST_FILES = { # TRT-LLM kernel tests "test_trtllm_moe.py", "test_trtllm_attention_op.py", @@ -177,8 +214,6 @@ "test_triton_moe.py", # Require onnx (optional dep) "test_export_fp8_linear_to_onnx.py", - # Uses hardcoded TRT-LLM repo path - "test_mrope_delta_cache.py", # Depend on TRT-LLM mamba/fla kernels (relative imports beyond auto_deploy) "test_mamba_rms_norm.py", "test_triton_rms_norm.py", @@ -228,22 +263,10 @@ "test_trtllm_quant_mxfp4_trtllm_gen_moe.py", } -# Single-GPU smoke tests. Some run with Paragraf alone; others exercise the optional -# TensorRT-LLM wheel/CLI path and are gated by TRTLLM_REDIRECT_AD_TO_PARAGRAF. -STANDALONE_SINGLEGPU_SMOKE_TEST_FILES = ( - "smoke/test_ad_build_small_single.py", - "smoke/test_ad_guided_decoding_regex.py", - "smoke/test_ad_speculative_decoding.py", - "smoke/test_ad_trtllm_bench.py", - "smoke/test_ad_trtllm_sampler.py", - "smoke/test_ad_trtllm_serve.py", - "smoke/test_disagg.py", -) - -# Multi-GPU tests that exercise only AutoDeploy and its standalone dependencies. -# Keep this list explicit: the remaining multi-GPU tests depend on TensorRT-LLM -# runtime components, kernels, or test infrastructure that Paragraf does not ship. -STANDALONE_MULTIGPU_TEST_FILES = ( +# Multi-GPU tests known to run without the optional TensorRT-LLM wheel. Other +# AutoDeploy multi-GPU tests are still copied, but receive the optional-wheel +# collection guard. +PURE_STANDALONE_MULTIGPU_TEST_FILES = { "custom_ops/test_dist.py", "custom_ops/test_sharded_rmsnorm.py", "smoke/test_ad_build_small_multi.py", @@ -254,15 +277,31 @@ "transformations/library/test_sharding_num_correctness.py", "transformations/library/test_step3p7_sharding_ir.py", "transformations/library/test_tp_sharding.py", -) +} -# Pytest support files required by the allowlisted multi-GPU tests. -STANDALONE_MULTIGPU_SUPPORT_FILES = ("transformations/library/conftest.py",) +# AutoDeploy integration files selected by the same classifier used by CI. +AUTODEPLOY_TEST_RE = re.compile(r"auto_?deploy|_ad_", re.IGNORECASE) +TEST_FILE_RE = re.compile(r"(?:^test_.*|.*_test)\.py$") + +# Support files needed by the selected integration tests. The source integration +# conftest is intentionally not copied because it pulls in the complete TRT-LLM +# CI harness; a focused replacement is generated below. +INTEGRATION_SUPPORT_FILES = ( + "__init__.py", + "common.py", + "trt_test_alternative.py", + "accuracy/__init__.py", + "accuracy/accuracy_core.py", + "accuracy/video_mme.py", + "disaggregated/disagg_test_utils.py", +) +INTEGRATION_SUPPORT_DIRS = ("accuracy/references",) -# Newer AD unit tests live under tests/unittest/_torch/auto_deploy. Copy only -# tracked tests whose dependencies are available in Paragraf plus the optional -# TensorRT-LLM wheel. -STANDALONE_TORCH_UNIT_TEST_FILES = ("unit/singlegpu/models/test_gpt_oss_modeling.py",) +# Additional source-tree test helpers imported by copied AutoDeploy tests. +TORCH_TEST_SUPPORT_FILES = ( + "tests/unittest/_torch/__init__.py", + "tests/unittest/_torch/helpers.py", +) # Import path rewrite: old -> new (applied to test files only). _IMPORT_REWRITE = "tensorrt_llm._torch.auto_deploy" @@ -273,9 +312,10 @@ r"(?:tensorrt_llm(?:\.|\b)|paragraf\.models\.custom\.modeling_gpt_oss(?:\.|\b))" ) _PARAGRAF_OPTIONAL_TRTLLM_GUARD = """ -_trtllm_redirect_value = os.environ.get("TRTLLM_REDIRECT_AD_TO_PARAGRAF") +_trtllm_environ = __import__("os").environ +_trtllm_redirect_value = _trtllm_environ.get("TRTLLM_REDIRECT_AD_TO_PARAGRAF") if _trtllm_redirect_value is None: - _trtllm_redirect_value = os.environ.get("TRTLLM_REDIRECT_AD_TO_LLMC", "") + _trtllm_redirect_value = _trtllm_environ.get("TRTLLM_REDIRECT_AD_TO_LLMC", "") _trtllm_redirect_value = _trtllm_redirect_value.lower() if _trtllm_redirect_value not in {"1", "true", "yes", "on"}: pytest.skip( @@ -295,6 +335,8 @@ # Remove the package directory produced before the Paragraf rename. "llmc", "tests", + # Remove the source-shaped test-data path produced by older generators. + "examples/auto_deploy", "runners", "pyproject.toml", "README.md", @@ -316,10 +358,50 @@ def _should_copy(filepath: str) -> bool: for pattern in EXCLUDE_PATTERNS: if pattern in filepath: return False + basename = os.path.basename(filepath) + if ".bak." in basename or basename.endswith("~"): + return False _, ext = os.path.splitext(filepath) return ext in COPY_EXTENSIONS +def _tracked_files_under(directory: str) -> list[str]: + """Return tracked files below a source directory, with an archive fallback.""" + relative_directory = os.path.relpath(directory, REPO_ROOT) + try: + result = subprocess.run( + ["git", "-C", REPO_ROOT, "ls-files", "--", relative_directory], + check=True, + capture_output=True, + text=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError): + discovered_files = [] + for root, dirs, files in os.walk(directory): + dirs[:] = [directory_name for directory_name in dirs if directory_name != "__pycache__"] + discovered_files.extend(os.path.join(root, filename) for filename in files) + return discovered_files + + return [os.path.join(REPO_ROOT, path) for path in result.stdout.splitlines()] + + +def _copy_file(src_path: str, dst_path: str) -> int: + if not os.path.isfile(src_path) or not _should_copy(src_path): + return 0 + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + shutil.copy2(src_path, dst_path) + return 1 + + +def _copy_tracked_tree(src_dir: str, dst_dir: str) -> int: + """Copy tracked files from a tree, or all files in a source archive.""" + count = 0 + for src_path in _tracked_files_under(src_dir): + rel_path = os.path.relpath(src_path, src_dir) + count += _copy_file(src_path, os.path.join(dst_dir, rel_path)) + return count + + def _copy_tree(src_dir: str, dst_dir: str) -> int: """Copy files from src_dir to dst_dir, preserving directory structure.""" count = 0 @@ -337,7 +419,62 @@ def _copy_tree(src_dir: str, dst_dir: str) -> int: return count -def _rewrite_imports_in_file(filepath: str, *, optional_trtllm_guards: bool = True) -> int: +def _rewrite_generated_test_layout(filepath: str, content: str) -> str: + """Rewrite TensorRT-LLM source-tree paths in selected generated tests.""" + + def replace_required(old: str, new: str) -> None: + nonlocal content + if old not in content: + raise ValueError(f"Expected standalone layout pattern not found in {filepath}: {old}") + content = content.replace(old, new) + + def substitute_required(pattern: str, replacement: str) -> None: + nonlocal content + content, count = re.subn(pattern, replacement, content) + if count == 0: + raise ValueError( + f"Expected standalone layout pattern not found in {filepath}: {pattern}" + ) + + filename = os.path.basename(filepath) + model_registry_pattern = ( + r"""["']examples["']\s*/\s*["']auto_deploy["']\s*/\s*""" + r"""["']model_registry["']""" + ) + runner_model_registry = '"runners" / "trtllm" / "model_registry"' + + if filename == "test_llm_api_paragraf_trtllm.py": + substitute_required(model_registry_pattern, runner_model_registry) + for config_name in ("nano_v3.yaml", "super_v3.yaml"): + substitute_required( + rf"""["']examples["']\s*/\s*["']auto_deploy["']\s*/\s*["']{config_name}["']""", + f'{runner_model_registry} / "configs" / "{config_name}"', + ) + elif filename == "test_mrope_delta_cache.py": + replace_required( + "return Path(__file__).resolve().parents[6]", + "return Path(__file__).resolve().parents[4]", + ) + replace_required( + '_repo_root() / "tensorrt_llm" / "_torch" / "auto_deploy" / "config"', + '_repo_root() / "paragraf" / "config"', + ) + substitute_required(model_registry_pattern, runner_model_registry) + elif filename == "test_example_configs.py": + replace_required( + '_AD_EXAMPLES_DIR = _REPO_ROOT / "examples" / "auto_deploy"', + '_AD_EXAMPLES_DIR = _REPO_ROOT / "runners" / "trtllm" / "model_registry"', + ) + + return content + + +def _rewrite_imports_in_file( + filepath: str, + *, + optional_trtllm_guards: bool = True, + force_optional_trtllm_guard: bool = False, +) -> int: """Rewrite imports in a copied test file for standalone mode. Source files inside ``tensorrt_llm/_torch/auto_deploy`` already use @@ -356,6 +493,21 @@ def _rewrite_imports_in_file(filepath: str, *, optional_trtllm_guards: bool = Tr original = content content = content.replace(_IMPORT_REWRITE, _IMPORT_TARGET) + content = content.replace("from auto_deploy.", "from ") + content = re.sub( + r"(?m)^from (test_[A-Za-z0-9_]+|disagg_test_utils) import ", + r"from .\1 import ", + content, + ) + content = content.replace( + 'pytest_plugins = ["disagg_test_utils"]', + 'pytest_plugins = ["integration.defs.disaggregated.disagg_test_utils"]', + ) + content = content.replace( + "_REPO_ROOT = pathlib.Path(__file__).resolve().parents[5]", + "_REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]", + ) + content = _rewrite_generated_test_layout(filepath, content) def ensure_imports(before_pos: int, *imports: str) -> None: nonlocal content @@ -394,20 +546,15 @@ def insert_optional_trtllm_guard() -> None: ensure_imports(build_import_pos, "os", "pytest") insert_optional_trtllm_guard() content = content.replace(_BUILD_AND_RUN_AD_IMPORT, _PARAGRAF_TRTLLM_RUNNER_IMPORT) + elif optional_trtllm_guards and force_optional_trtllm_guard: + ensure_imports(len(content), "os", "pytest") + insert_optional_trtllm_guard() elif optional_trtllm_guards: trtllm_import = _TRTLLM_IMPORT_RE.search(content) if trtllm_import is not None: ensure_imports(trtllm_import.start(), "os", "pytest") insert_optional_trtllm_guard() - if optional_trtllm_guards: - # The standalone package can rely on the installed trtllm-bench entrypoint, - # but it does not ship TensorRT-LLM's source-tree benchmarks/cpp directory. - content = content.replace( - ' script_dir = Path(root_dir, "benchmarks", "cpp")\n', - " script_dir = Path(temp_dir)\n", - ) - replacements = sum(1 for a, b in zip(original, content) if a != b) # rough count if content != original: with open(filepath, "w") as f: @@ -418,6 +565,29 @@ def insert_optional_trtllm_guard() -> None: return replacements +def _requires_optional_trtllm_guard(filepath: str, tests_dir: str) -> bool: + relative_path = os.path.relpath(filepath, tests_dir).replace("\\", "/") + if not TEST_FILE_RE.fullmatch(os.path.basename(filepath)): + return False + generated_basename = os.path.basename(filepath) + source_basename = SOURCE_TEST_NAMES_BY_GENERATED_NAME.get( + generated_basename, generated_basename + ) + path_parts = relative_path.split("/") + if relative_path.startswith("integration/"): + return True + if "shim" in path_parts: + return True + if source_basename in OPTIONAL_TRTLLM_TEST_FILES: + return True + if relative_path.startswith("multigpu/"): + multigpu_parts = relative_path.removeprefix("multigpu/").split("/") + multigpu_parts[-1] = source_basename + multigpu_path = "/".join(multigpu_parts) + return multigpu_path not in PURE_STANDALONE_MULTIGPU_TEST_FILES + return False + + def _rewrite_imports_in_dir(directory: str, *, optional_trtllm_guards: bool = True) -> int: """Rewrite imports in all .py files in a directory tree.""" total = 0 @@ -427,6 +597,10 @@ def _rewrite_imports_in_dir(directory: str, *, optional_trtllm_guards: bool = Tr total += _rewrite_imports_in_file( os.path.join(root, filename), optional_trtllm_guards=optional_trtllm_guards, + force_optional_trtllm_guard=( + optional_trtllm_guards + and _requires_optional_trtllm_guard(os.path.join(root, filename), directory) + ), ) return total @@ -471,89 +645,80 @@ def _resolve_dependencies(dep_names: list, pinned: dict) -> list: # --------------------------------------------------------------------------- # Test copying # --------------------------------------------------------------------------- -def _should_exclude_test(filepath: str) -> bool: - """Check if a test file should be excluded from the standalone package.""" - basename = os.path.basename(filepath) - if basename in EXCLUDE_TEST_FILES: - return True - parts = filepath.replace("\\", "/").split("/") - return any(d in EXCLUDE_TEST_DIRS for d in parts) - - def _copy_tests(output_dir: str) -> int: """Copy auto_deploy test files to the standalone package tests/ directory.""" tests_dst = os.path.join(output_dir, "tests") count = 0 - # Copy singlegpu tests (excluding TRT-LLM-only dirs/files) - singlegpu_src = os.path.join(AD_TESTS_DIR, "singlegpu") - if os.path.isdir(singlegpu_src): - for root, dirs, files in os.walk(singlegpu_src): - # Skip excluded directories - dirs[:] = [d for d in dirs if d not in EXCLUDE_TEST_DIRS and d != "__pycache__"] - - for filename in files: - src_path = os.path.join(root, filename) - if not _should_copy(src_path) or _should_exclude_test(src_path): - continue - rel_path = os.path.relpath(src_path, AD_TESTS_DIR) - dst_path = os.path.join(tests_dst, rel_path) - os.makedirs(os.path.dirname(dst_path), exist_ok=True) - shutil.copy2(src_path, dst_path) - count += 1 - - # Copy all single-GPU smoke tests. Tests that require the optional - # TensorRT-LLM wheel are guarded during import rewriting. - singlegpu_smoke_files = STANDALONE_SINGLEGPU_SMOKE_TEST_FILES - for rel_path in singlegpu_smoke_files: - src_path = os.path.join(singlegpu_src, rel_path) - if not os.path.isfile(src_path): - raise FileNotFoundError( - f"Allowlisted single-GPU smoke test file does not exist: {src_path}" - ) - dst_path = os.path.join(tests_dst, "singlegpu", rel_path) - os.makedirs(os.path.dirname(dst_path), exist_ok=True) - shutil.copy2(src_path, dst_path) - count += 1 + # Copy every tracked legacy AutoDeploy unit-test file except the tests of + # this generator itself. Optional TensorRT-LLM dependencies are handled by + # collection guards after the files are copied. + for src_path in _tracked_files_under(AD_TESTS_DIR): + rel_path = os.path.relpath(src_path, AD_TESTS_DIR) + if any(part in SOURCE_ONLY_TEST_DIRS for part in rel_path.split(os.sep)): + continue + generated_name = PARAGRAF_TRTLLM_TEST_RENAMES.get( + os.path.basename(rel_path), os.path.basename(rel_path) + ) + generated_rel_path = os.path.join(os.path.dirname(rel_path), generated_name) + count += _copy_file(src_path, os.path.join(tests_dst, generated_rel_path)) + + # Copy every tracked test from the newer unit-test tree. This avoids an + # allowlist that silently misses tests added alongside new AutoDeploy code. + for src_path in _tracked_files_under(AD_TORCH_TESTS_DIR): + rel_path = os.path.relpath(src_path, AD_TORCH_TESTS_DIR) + path_parts = rel_path.split(os.sep) + if path_parts[0] == "unit": + path_parts = path_parts[1:] + generated_rel_path = os.path.join(*path_parts) + generated_path = os.path.join(tests_dst, generated_rel_path) + if os.path.exists(generated_path): + raise FileExistsError(f"Generated test path collision for {src_path}: {generated_path}") + count += _copy_file( + src_path, + generated_path, + ) - # Copy only the explicitly standalone-compatible multi-GPU tests. Unlike - # singlegpu/, this tree contains several tests that require TensorRT-LLM. - multigpu_src = os.path.join(AD_TESTS_DIR, "multigpu") - multigpu_files = STANDALONE_MULTIGPU_TEST_FILES + STANDALONE_MULTIGPU_SUPPORT_FILES - for rel_path in multigpu_files: - src_path = os.path.join(multigpu_src, rel_path) - if not os.path.isfile(src_path): - raise FileNotFoundError(f"Allowlisted multi-GPU test file does not exist: {src_path}") - dst_path = os.path.join(tests_dst, "multigpu", rel_path) - os.makedirs(os.path.dirname(dst_path), exist_ok=True) - shutil.copy2(src_path, dst_path) - count += 1 + # The CI classifier also finds a small number of AutoDeploy integration + # tests outside the unit-test roots. Copy those tests and their focused + # support modules without bringing in the complete TensorRT-LLM CI suite. + integration_tests_root = os.path.join(REPO_ROOT, "tests", "integration") + for src_path in _tracked_files_under(AD_INTEGRATION_TESTS_DIR): + rel_from_tests = os.path.relpath(src_path, integration_tests_root) + if not TEST_FILE_RE.fullmatch(os.path.basename(src_path)): + continue + if not AUTODEPLOY_TEST_RE.search(rel_from_tests.replace("\\", "/")): + continue + generated_name = PARAGRAF_TRTLLM_TEST_RENAMES.get( + os.path.basename(rel_from_tests), os.path.basename(rel_from_tests) + ) + generated_rel_path = os.path.join(os.path.dirname(rel_from_tests), generated_name) + count += _copy_file( + src_path, + os.path.join(tests_dst, "integration", generated_rel_path), + ) - # Copy standalone-compatible tests from the newer _torch/auto_deploy unit tree. - for rel_path in STANDALONE_TORCH_UNIT_TEST_FILES: - src_path = os.path.join(AD_TORCH_TESTS_DIR, rel_path) - if not os.path.isfile(src_path): - raise FileNotFoundError( - f"Allowlisted _torch AutoDeploy unit test does not exist: {src_path}" - ) - dst_path = os.path.join(tests_dst, "_torch", "auto_deploy", rel_path) - os.makedirs(os.path.dirname(dst_path), exist_ok=True) - shutil.copy2(src_path, dst_path) - count += 1 + for rel_path in INTEGRATION_SUPPORT_FILES: + count += _copy_file( + os.path.join(AD_INTEGRATION_TESTS_DIR, rel_path), + os.path.join(tests_dst, "integration", "defs", rel_path), + ) + for rel_path in INTEGRATION_SUPPORT_DIRS: + count += _copy_tracked_tree( + os.path.join(AD_INTEGRATION_TESTS_DIR, rel_path), + os.path.join(tests_dst, "integration", "defs", rel_path), + ) - # Copy test utilities - if os.path.isdir(AD_UTILS_TEST_DIR): - utils_dst = os.path.join(tests_dst, "_utils_test") - for filename in os.listdir(AD_UTILS_TEST_DIR): - src_path = os.path.join(AD_UTILS_TEST_DIR, filename) - if os.path.isfile(src_path) and _should_copy(src_path): - dst_path = os.path.join(utils_dst, filename) - os.makedirs(utils_dst, exist_ok=True) - shutil.copy2(src_path, dst_path) - count += 1 + for rel_path in TORCH_TEST_SUPPORT_FILES: + count += _copy_file( + os.path.join(REPO_ROOT, rel_path), + os.path.join(tests_dst, os.path.relpath(rel_path, "tests/unittest")), + ) # Create conftest.py for test discovery and imports + _create_test_package_init_files(tests_dst) _create_test_conftest(tests_dst) + _create_integration_conftest(tests_dst) # Create a stub for test_common.llm_data (used by some model tests) _create_test_common_stub(tests_dst) @@ -562,6 +727,23 @@ def _copy_tests(output_dir: str) -> int: return count +def _create_test_package_init_files(tests_dir: str) -> None: + """Give copied test directories stable package-qualified module names.""" + content = ( + "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION" + " & AFFILIATES. All rights reserved.\n" + "# SPDX-License-Identifier: Apache-2.0\n" + ) + for root, dirs, _ in os.walk(tests_dir): + dirs[:] = [directory for directory in dirs if directory != "__pycache__"] + if root == tests_dir: + continue + init_path = os.path.join(root, "__init__.py") + if not os.path.exists(init_path): + with open(init_path, "w") as f: + f.write(content) + + def _create_test_conftest(tests_dir: str) -> None: """Create a conftest.py that configures the test environment for standalone mode.""" content = textwrap.dedent("""\ @@ -604,11 +786,14 @@ def _create_test_conftest(tests_dir: str) -> None: _tests_dir = os.path.dirname(__file__) _package_root = os.path.dirname(_tests_dir) + _integration_tests_dir = os.path.join(_tests_dir, "integration") # Add generated package/test roots to the Python path so tests can import - # local paragraf, runners, and _utils_test even under safe-path settings. + # local paragraf, runners, integration helpers, and _utils_test even + # under safe-path settings. sys.path.insert(0, _package_root) sys.path.insert(0, _tests_dir) + sys.path.insert(0, _integration_tests_dir) sys.path.insert(0, os.path.join(_tests_dir, "_utils_test")) @@ -623,6 +808,84 @@ def llm_root(): f.write(content) +def _create_integration_conftest(tests_dir: str) -> None: + """Create the focused helpers needed by copied AutoDeploy integration tests.""" + defs_dir = os.path.join(tests_dir, "integration", "defs") + os.makedirs(defs_dir, exist_ok=True) + content = textwrap.dedent("""\ + # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # SPDX-License-Identifier: Apache-2.0 + + import os + from pathlib import Path + + import pytest + import torch + + + def get_llm_root(): + return str(Path(__file__).resolve().parents[3]) + + + def llm_models_root(): + models_root = os.environ.get("LLM_MODELS_ROOT") + if not models_root: + pytest.skip("LLM_MODELS_ROOT is required for AutoDeploy integration tests") + return models_root + + + def get_sm_version(): + if not torch.cuda.is_available(): + return 0 + major, minor = torch.cuda.get_device_capability(0) + return major * 10 + minor + + + def get_device_count(): + return torch.cuda.device_count() + + + def get_device_memory(): + if not torch.cuda.is_available(): + return 0 + return torch.cuda.get_device_properties(0).total_memory // (1024 * 1024) + + + def check_device_contain(keyword_list): + if not torch.cuda.is_available(): + return False + device_name = torch.cuda.get_device_name(0) + return any(keyword in device_name for keyword in keyword_list) + + + skip_pre_ada = pytest.mark.skipif( + get_sm_version() < 89, + reason="This test is not supported in pre-Ada architecture", + ) + skip_pre_hopper = pytest.mark.skipif( + get_sm_version() < 90, + reason="This test is not supported in pre-Hopper architecture", + ) + skip_pre_blackwell = pytest.mark.skipif( + get_sm_version() < 100, + reason="This test is not supported in pre-Blackwell architecture", + ) + + + @pytest.fixture(autouse=True) + def _apply_resource_markers(request): + device_marker = request.node.get_closest_marker("skip_less_device") + if device_marker and get_device_count() < device_marker.args[0]: + pytest.skip(f"Test requires {device_marker.args[0]} GPUs") + + for memory_marker in request.node.iter_markers("skip_less_device_memory"): + if get_device_memory() < memory_marker.args[0]: + pytest.skip(f"Test requires {memory_marker.args[0]} MiB of GPU memory") + """) + with open(os.path.join(defs_dir, "conftest.py"), "w") as f: + f.write(content) + + def _create_test_common_stub(tests_dir: str) -> None: """Create a stub for test_common.llm_data (provides HF model path resolution). @@ -730,10 +993,45 @@ def _sm_version() -> int: _sm_version() < 90, reason="This test is not supported in pre-Hopper architecture", ) + skip_no_hopper = pytest.mark.skipif( + _sm_version() != 90, + reason="This test is only supported in Hopper architecture", + ) + skip_pre_blackwell = pytest.mark.skipif( + _sm_version() < 100, + reason="This test is not supported in pre-Blackwell architecture", + ) """) with open(os.path.join(utils_dir, "util.py"), "w") as f: f.write(content) + cpp_paths_content = textwrap.dedent("""\ + # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # SPDX-License-Identifier: Apache-2.0 + + from pathlib import Path + + import pytest + + + @pytest.fixture(scope="module") + def llm_root(): + return Path(__file__).resolve().parents[2] + """) + with open(os.path.join(utils_dir, "cpp_paths.py"), "w") as f: + f.write(cpp_paths_content) + + llm_data_content = textwrap.dedent("""\ + # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # SPDX-License-Identifier: Apache-2.0 + + from test_common.llm_data import llm_models_root + + __all__ = ["llm_models_root"] + """) + with open(os.path.join(utils_dir, "llm_data.py"), "w") as f: + f.write(llm_data_content) + # --------------------------------------------------------------------------- # Example / e2e harness copying @@ -794,6 +1092,7 @@ def _create_pyproject_toml(output_dir: str, dependencies: list, dev_dependencies "]\n" "\n" "[project.optional-dependencies]\n" + 'trtllm = ["tensorrt-llm"]\n' "dev = [\n" f"{dev_deps_lines}\n" "]\n" @@ -805,6 +1104,8 @@ def _create_pyproject_toml(output_dir: str, dependencies: list, dev_dependencies 'testpaths = ["tests"]\n' "markers = [\n" ' "threadleak(enabled): configure thread-leak checks (inert in standalone tests)",\n' + ' "skip_less_device(count): require at least count GPUs",\n' + ' "skip_less_device_memory(mib): require at least mib MiB on one GPU",\n' "]\n" ) @@ -852,7 +1153,7 @@ def create_standalone_package(output_dir: str) -> None: # 2. Copy and rewrite tests (tests use absolute self-imports by design). test_count = _copy_tests(output_dir) rewrite_count = _rewrite_imports_in_dir(os.path.join(output_dir, "tests")) - print(f" Copied {test_count} test files to tests/ ({rewrite_count} import rewrites)") + print(f" Copied {test_count} test/support files ({rewrite_count} import rewrites)") # 2b. Copy the Tier-1 e2e harness into runners/ (build_and_run_paragraf_trtllm.py # + model_registry) and rewrite its imports auto_deploy -> paragraf. YAML is @@ -920,6 +1221,9 @@ def create_standalone_package(output_dir: str) -> None: print(" source .venv/bin/activate") print(" uv pip install -e '.[dev]'") print("\nTo run tests: pytest tests/") + print("To run optional TensorRT-LLM tests:") + print(" uv pip install -e '.[dev,trtllm]'") + print(" TRTLLM_REDIRECT_AD_TO_PARAGRAF=true pytest tests/") print( 'To verify: python -c "from paragraf._compat import TRTLLM_AVAILABLE; print(TRTLLM_AVAILABLE)"' ) diff --git a/examples/bindings/executor/README.md b/examples/bindings/executor/README.md deleted file mode 100644 index df44568fd4f1..000000000000 --- a/examples/bindings/executor/README.md +++ /dev/null @@ -1,76 +0,0 @@ -# Python Bindings Example - -This example shows how to use the python bindings interface to generate tokens -using a TensorRT engine. - -## Setup - -Build a TensorRT engine for one of the supported TensorRT LLM model following -instructions in the corresponding `examples` folder. - -## Usage - -### Basic example - -Run `example_basic.py`, passing in the directory where the TensorRT engine was generated. For example: - -``` -cd examples/bindings -python3 example_basic.py --model_path=../llama/tmp/7B/trt_engines/fp16/1-gpu/ -``` - -### Debug example - -This example shows how you can define which engine IO tensors should be kept or dumped to numpy files. -Run `example_debug.py`, passing in the directory where the TensorRT engine was generated. For example: - -``` -cd examples/bindings -python3 example_debug.py --model_path=../llama/tmp/7B/trt_engines/fp16/1-gpu/ -``` - -### Advanced example - -This example shows how you can use the python bindings to generate tokens for a larger number of requests concurrently and demonstrate how tokens can be returned in a streaming fashion. - -The full list of supported input parameters can be obtained with: -``` -pytho3 example_advanced.py -h -``` - -For example, assuming a CSV file named `input_tokens.csv` exist which contains the following input tokens: -``` -1, 2, 3, 4, 5, 6 -1, 2, 3, 4 -1, 2, 3, 4, 5, 6, 7, 8, 9, 10 -``` -one can generate output tokens for those 3 prompts with: -``` -python3 example_advanced.py --model_path --input_tokens_csv_file input_tokens.csv -``` -Upon successful completion, the output tokens will be written to file `output_tokens.csv`. - -### Multi-GPU Example - -To run the two examples for models requiring more than one gpu, you can run the example with MPI. - -For example, the basic example can be run as follows: -``` -mpirun -n 4 --allow-run-as-root python3 example_basic.py --model_path=../llama/tmp/7B/trt_engines/fp16/4gpu_tp4_pp1/ -``` - -The advanced example can also be run using the ORCHESTRATOR mode, where the additional processes needed for multi-GPU runs will automatically be spawned. -This can be done by running: -``` -python3 example_advanced.py --model_path=../llama/tmp/7B/trt_engines/fp16/4gpu_tp4_pp1/ --use_orchestrator_mode -``` - -### Logits post processor example - -This example shows how to generate JSON structured output using LogitsPostProcessor API. - -``` -python3 example_logits_processor.py -t -e --batch_size 8 -``` - -LogitsPostProcessorBatched, which fuses logits processing for all samples in a batch into a single callback, is enabled by `--lpp_batched` diff --git a/examples/bindings/executor/example_advanced.py b/examples/bindings/executor/example_advanced.py deleted file mode 100644 index 25f063a855e2..000000000000 --- a/examples/bindings/executor/example_advanced.py +++ /dev/null @@ -1,166 +0,0 @@ -import argparse -import csv -import datetime -from pathlib import Path - -import tensorrt_llm - -trtllm_package_dir = Path(tensorrt_llm.__file__).parent -executor_worker_path = trtllm_package_dir / 'bin' / 'executorWorker' - -import tensorrt_llm.bindings.executor as trtllm - - -# Read input tokens from csv file -def read_input_tokens(input_tokens_csv_file: str) -> list[int]: - - input_tokens = [] - with open(input_tokens_csv_file, mode='r') as file: - csvFile = csv.reader(file) - for lines in csvFile: - input_tokens.append([int(item) for item in lines]) - return input_tokens - - -# Prepare and enqueue the requests -def enqueue_requests(args: argparse.Namespace, - executor: trtllm.Executor) -> None: - - output_config = trtllm.OutputConfig() - output_config.exclude_input_from_output = args.exclude_input_from_output - sampling_config = trtllm.SamplingConfig(args.beam_width) - input_tokens = read_input_tokens(args.input_tokens_csv_file) - - request_ids = [] - for tokens in input_tokens: - req = trtllm.Request(input_token_ids=tokens, - max_tokens=args.max_tokens, - streaming=args.streaming, - sampling_config=sampling_config, - output_config=output_config) - req_id = executor.enqueue_request(req) - request_ids.append(req_id) - - return request_ids - - -# Wait for responses and store output tokens -def wait_for_responses(args: argparse.Namespace, request_ids: list[int], - executor: trtllm.Executor) -> dict[dict[list[int]]]: - - output_tokens = { - req_id: { - beam: [] - for beam in range(args.beam_width) - } - for req_id in request_ids - } - num_finished = 0 - iter = 0 - while (num_finished < len(request_ids) and iter < args.timeout_ms): - responses = executor.await_responses( - datetime.timedelta(milliseconds=args.timeout_ms)) - for response in responses: - req_id = response.request_id - if not response.has_error(): - result = response.result - num_finished += 1 if result.is_final else 0 - for beam, outTokens in enumerate(result.output_token_ids): - output_tokens[req_id][beam].extend(outTokens) - else: - raise RuntimeError( - str(req_id) + " encountered error:" + response.error_msg) - - return output_tokens - - -# Write the output tokens to file -def write_output_tokens(output_tokens_csv_file: str, request_ids: list[int], - output_tokens: dict[dict[list[int]]], - beam_width: int) -> None: - - with open(output_tokens_csv_file, 'w') as csvfile: - - writer = csv.writer(csvfile) - for req_id in request_ids: - out_tokens = output_tokens[req_id] - for beam in range(args.beam_width): - beam_tokens = out_tokens[beam] - writer.writerow(beam_tokens) - - print("Output tokens written to:", output_tokens_csv_file) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Executor Bindings Example") - parser.add_argument("--model_path", - type=str, - required=True, - help="Directory containing model engine") - parser.add_argument("--input_tokens_csv_file", - type=str, - required=True, - help="CSV file containing the input tokens") - parser.add_argument("--output_tokens_csv_file", - type=str, - required=False, - default="output_tokens.csv", - help="CSV file where to write output tokens") - parser.add_argument("--beam_width", - type=int, - required=False, - default=1, - help="The beam width") - parser.add_argument("--streaming", - default=False, - action="store_true", - help="Operate in streaming mode") - - parser.add_argument("--use_orchestrator_mode", - default=False, - action="store_true", - help="Operate in orchestrator mode for multi-GPU runs") - - parser.add_argument( - "--exclude_input_from_output", - default=False, - action="store_true", - help= - "Exclude input token when writing output tokens. Only has effect for streaming=False since in streaming mode, input tokens are never included in output." - ) - parser.add_argument("--max_tokens", - type=int, - required=False, - default=10, - help="The max number of tokens to be generated") - parser.add_argument( - "--timeout_ms", - type=int, - required=False, - default=10000, - help="The maximum time to wait for all responses, in milliseconds") - - args = parser.parse_args() - executor_config = trtllm.ExecutorConfig(args.beam_width) - - if args.use_orchestrator_mode: - orchestrator_config = trtllm.OrchestratorConfig( - True, str(executor_worker_path)) - executor_config.parallel_config = trtllm.ParallelConfig( - trtllm.CommunicationType.MPI, trtllm.CommunicationMode.ORCHESTRATOR, - None, None, orchestrator_config) - - # Create the executor. - executor = trtllm.Executor(args.model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - if executor.can_enqueue_requests(): - # Enqueue the requests - request_ids = enqueue_requests(args, executor) - - # Wait for the responses - output_tokens = wait_for_responses(args, request_ids, executor) - - # Write the output tokens - write_output_tokens(args.output_tokens_csv_file, request_ids, - output_tokens, args.beam_width) diff --git a/examples/bindings/executor/example_basic.py b/examples/bindings/executor/example_basic.py deleted file mode 100644 index 3c71bde594e1..000000000000 --- a/examples/bindings/executor/example_basic.py +++ /dev/null @@ -1,34 +0,0 @@ -import argparse - -import tensorrt_llm.bindings.executor as trtllm - -# This example hows to use the python bindings to create an executor, enqueue a -# request, and get the generated tokens. - -# First, follow the steps in README.md to generate the engines. - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Executor Bindings Example") - parser.add_argument("--model_path", - type=str, - required=True, - help="Directory containing model engine") - args = parser.parse_args() - - # Create the executor. - executor = trtllm.Executor(args.model_path, trtllm.ModelType.DECODER_ONLY, - trtllm.ExecutorConfig(1)) - - if executor.can_enqueue_requests(): - # Create the request. - request = trtllm.Request(input_token_ids=[1, 2, 3, 4], max_tokens=10) - - # Enqueue the request. - request_id = executor.enqueue_request(request) - - # Wait for the new tokens. - responses = executor.await_responses(request_id) - output_tokens = responses[0].result.output_token_ids - - # Print tokens. - print(output_tokens) diff --git a/examples/bindings/executor/example_debug.py b/examples/bindings/executor/example_debug.py deleted file mode 100644 index f7c0669b1254..000000000000 --- a/examples/bindings/executor/example_debug.py +++ /dev/null @@ -1,67 +0,0 @@ -import argparse -import pathlib as pl - -import numpy as np - -import tensorrt_llm.bindings.executor as trtllm - -# This example hows to use the python bindings to create an executor, enqueue a -# request, and get the generated tokens. - -# First, follow the steps in README.md to generate the engines. - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Executor Bindings Example") - parser.add_argument("--model_path", - type=str, - required=True, - help="Directory containing model engine") - parser.add_argument("--dump_tensors", - action="store_true", - help="Dump debug tensors to files") - args = parser.parse_args() - - max_tokens = 2 - - # Select which tensors should be kept or dumped - debug_config = trtllm.DebugConfig( - debug_tensor_names=["sequence_length"], - debug_tensors_max_iterations=0 if args.dump_tensors else max_tokens) - - # Create the executor. - executor = trtllm.Executor( - args.model_path, trtllm.ModelType.DECODER_ONLY, - trtllm.ExecutorConfig(1, debug_config=debug_config)) - - if executor.can_enqueue_requests(): - # Create the request. - request = trtllm.Request(input_token_ids=[1, 2, 3, 4], - max_tokens=max_tokens) - - # Enqueue the request. - request_id = executor.enqueue_request(request) - - # Wait for the new tokens. - responses = executor.await_responses(request_id) - output_tokens = responses[0].result.output_token_ids - - # Print tokens. - print(output_tokens) - - if args.dump_tensors: - print("debug tensors from files:") - debug_dir = pl.Path("/tmp/tllm_debug/PP_1/TP_1") - if debug_dir.is_dir(): - for iter_dir in [x for x in debug_dir.iterdir() if x.is_dir()]: - print(iter_dir.name) - for file in [x for x in iter_dir.iterdir() if x.is_file()]: - print(file.name, np.load(file)) - else: - print("debug dir not found") - else: - print("debug tensors from queue:") - debug_tensors = executor.get_latest_debug_tensors() - for debug_iter in debug_tensors: - print(f"iteration {debug_iter.iter}") - for [name, tensor] in debug_iter.debug_tensors.items(): - print(name, tensor) diff --git a/examples/bindings/executor/example_logits_processor.py b/examples/bindings/executor/example_logits_processor.py deleted file mode 100644 index 6cb1a751a6da..000000000000 --- a/examples/bindings/executor/example_logits_processor.py +++ /dev/null @@ -1,212 +0,0 @@ -import argparse -import datetime -import typing as _tp - -import torch as _tor -from lmformatenforcer import (JsonSchemaParser, TokenEnforcer, - TokenEnforcerTokenizerData) -from pydantic import BaseModel -from transformers import AutoTokenizer - -import tensorrt_llm.bindings.executor as trtllm - - -def _build_regular_tokens_list( - tokenizer) -> _tp.List[_tp.Tuple[int, str, bool]]: - token_0 = [tokenizer.encode("0")[-1]] - regular_tokens = [] - vocab_size = tokenizer.vocab_size - for token_idx in range(vocab_size): - if token_idx in tokenizer.all_special_ids: - continue - # We prepend token 0 and skip the first letter of the result to get a space if the token is a start word. - tensor_after_0 = _tor.tensor(token_0 + [token_idx], dtype=_tor.long) - decoded_after_0 = tokenizer.decode(tensor_after_0)[1:] - decoded_regular = tokenizer.decode(token_0) - is_word_start_token = len(decoded_after_0) > len(decoded_regular) - regular_tokens.append((token_idx, decoded_after_0, is_word_start_token)) - return regular_tokens - - -def build_token_enforcer(tokenizer, character_level_parser): - """ - Build logits processor for feeding it into generate function (use_py_session should be True) - """ - regular_tokens = _build_regular_tokens_list(tokenizer) - - def _decode(tokens: _tp.List[int]) -> str: - tensor = _tor.tensor(tokens, dtype=_tor.long) - return tokenizer.decode(tensor) - - tokenizer_data = TokenEnforcerTokenizerData(regular_tokens, _decode, - tokenizer.eos_token_id) - return TokenEnforcer(tokenizer_data, character_level_parser) - - -# Prepare and enqueue the requests -def enqueue_requests(args: argparse.Namespace, - executor: trtllm.Executor) -> None: - - sampling_config = trtllm.SamplingConfig(args.beam_width) - - request_ids = [] - for iter_id in range(args.batch_size): - # Create the request. - request = trtllm.Request(input_token_ids=prompt, - max_tokens=25, - end_id=tokenizer.eos_token_id, - sampling_config=sampling_config, - client_id=iter_id % 2) - request.logits_post_processor_name = request.BATCHED_POST_PROCESSOR_NAME if args.lpp_batched else "my_logits_pp" - - # Enqueue the request. - req_id = executor.enqueue_request(request) - request_ids.append(req_id) - - return request_ids - - -# Wait for responses and store output tokens -def wait_for_responses(args: argparse.Namespace, request_ids: list[int], - executor: trtllm.Executor) -> dict[dict[list[int]]]: - - output_tokens = { - req_id: { - beam: [] - for beam in range(args.beam_width) - } - for req_id in request_ids - } - num_finished = 0 - iter = 0 - while (num_finished < len(request_ids) and iter < args.timeout_ms): - responses = executor.await_responses( - datetime.timedelta(milliseconds=args.timeout_ms)) - for response in responses: - req_id = response.request_id - if not response.has_error(): - result = response.result - num_finished += 1 if result.is_final else 0 - for beam, outTokens in enumerate(result.output_token_ids): - output_tokens[req_id][beam].extend(outTokens) - else: - raise RuntimeError( - str(req_id) + " encountered error:" + response.error_msg) - - return output_tokens - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Executor Bindings Example") - parser.add_argument("--tokenizer_path", - "-t", - type=str, - required=True, - help="Directory containing model tokenizer") - parser.add_argument("--engine_path", - "-e", - type=str, - required=True, - help="Directory containing model engine") - parser.add_argument("--beam_width", - type=int, - required=False, - default=1, - help="The beam width") - parser.add_argument("--batch_size", - type=int, - required=False, - default=1, - help="The batch size") - parser.add_argument( - "--timeout_ms", - type=int, - required=False, - default=10000, - help="The maximum time to wait for all responses, in milliseconds") - parser.add_argument("--lpp_batched", - action="store_true", - default=False, - help="Enable batched logits post processor") - - args = parser.parse_args() - - tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_path) - - class AnswerFormat(BaseModel): - last_name: str - year_of_birth: int - - parser = JsonSchemaParser(AnswerFormat.model_json_schema()) - token_enforcer = build_token_enforcer(tokenizer, parser) - - def get_allowed_tokens(ids, client_id): - if client_id is None or client_id == 0: return [42] - - def _trim(ids): - return [x for x in ids if x != tokenizer.eos_token_id] - - allowed = token_enforcer.get_allowed_tokens(_trim(ids[0])) - return allowed - - def logits_post_processor(req_id: int, logits: _tor.Tensor, - ids: _tp.List[_tp.List[int]], stream_ptr: int, - client_id: _tp.Optional[int]): - mask = _tor.full_like(logits, fill_value=float("-inf"), device="cpu") - allowed = get_allowed_tokens(ids, client_id) - mask[:, :, allowed] = 0 - - with _tor.cuda.stream(_tor.cuda.ExternalStream(stream_ptr)): - mask = mask.to(logits.device, non_blocking=True) - logits += mask - - def logits_post_processor_batched( - req_ids_batch: _tp.List[int], logits_batch: _tp.List[_tor.Tensor], - ids_batch: _tp.List[_tp.List[_tp.List[int]]], stream_ptr, - client_ids_batch: _tp.List[_tp.Optional[int]]): - masks = [] - for req_id, logits, ids, client_id in zip(req_ids_batch, logits_batch, - ids_batch, client_ids_batch): - del req_id - mask = _tor.full_like(logits, - fill_value=float("-inf"), - device="cpu") - allowed = get_allowed_tokens(ids, client_id) - mask[:, :, allowed] = 0 - masks.append(mask) - - with _tor.cuda.stream(_tor.cuda.ExternalStream(stream_ptr)): - for logits, mask in zip(logits_batch, masks): - logits += mask.to(logits.device, non_blocking=True) - - # Create the executor. - executor_config = trtllm.ExecutorConfig(args.beam_width) - logits_proc_config = trtllm.LogitsPostProcessorConfig() - if not args.lpp_batched: - logits_proc_config.processor_map = { - "my_logits_pp": logits_post_processor - } - else: - logits_proc_config.processor_batched = logits_post_processor_batched - executor_config.logits_post_processor_config = logits_proc_config - executor = trtllm.Executor(args.engine_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - input = "Please give me information about Michael Jordan. You MUST answer using the following json schema: " - prompt = tokenizer.encode(input) - print(f"Input text: {input}\n") - - if executor.can_enqueue_requests(): - request_ids = enqueue_requests(args, executor) - output_tokens = wait_for_responses(args, request_ids, executor) - - # Print output - for req_id in request_ids: - for beam_id in range(args.beam_width): - result = tokenizer.decode( - output_tokens[req_id][beam_id][len(prompt):]) - generated_tokens = len( - output_tokens[req_id][beam_id]) - len(prompt) - print( - f"Request {req_id} Beam {beam_id} ({generated_tokens} tokens): {result}" - ) diff --git a/examples/configs/database/database.py b/examples/configs/database/database.py index 3dcdd13c0300..af33a4dab41b 100644 --- a/examples/configs/database/database.py +++ b/examples/configs/database/database.py @@ -13,13 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. - import logging +import re +from collections import defaultdict from pathlib import Path -from typing import Any, Dict, Iterator, List, Tuple +from typing import Any, Iterator, Literal import yaml -from pydantic import BaseModel, Field, RootModel, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PositiveInt, + RootModel, + field_validator, + model_validator, +) logger = logging.getLogger(__name__) @@ -30,11 +39,22 @@ LOW_LATENCY_CONCURRENCY_THRESHOLD = 8 HIGH_THROUGHPUT_CONCURRENCY_THRESHOLD = 32 KEY_PROFILES = {"Min Latency", "Balanced", "Max Throughput"} +PROFILE_DISPLAY_NAMES = { + "latency": "Min Latency", + "balanced": "Balanced", + "throughput": "Max Throughput", +} +PROFILE_ORDER = {profile: idx for idx, profile in enumerate(PROFILE_DISPLAY_NAMES)} +VALIDATED_COMMIT_PATTERN = re.compile(r"^[0-9a-f]{40}$") +VALIDATED_VERSION_PATTERN = re.compile(r"^[0-9A-Za-z][0-9A-Za-z._+-]*$") +Profile = Literal["latency", "balanced", "throughput"] class CuratedRecipe(BaseModel): """A curated (hand-tuned) recipe entry.""" + model_config = ConfigDict(extra="forbid") + model: str = Field(description="HuggingFace model ID") arch: str = Field(description="Model architecture class name") config_path: str = Field(description="Relative path to YAML config") @@ -51,7 +71,7 @@ def _validate_config_path(cls, v: str) -> str: return v -class CuratedRecipeList(RootModel[List[CuratedRecipe]]): +class CuratedRecipeList(RootModel[list[CuratedRecipe]]): """Validated list of curated recipe entries.""" @classmethod @@ -75,15 +95,58 @@ def __len__(self) -> int: class Recipe(BaseModel): """Recipe record for scenario list.""" - model: str = Field(description="Model name") - gpu: str = Field(description="GPU name") - isl: int = Field(description="Input sequence length") - osl: int = Field(description="Output sequence length") - concurrency: int = Field(description="Concurrency") - config_path: str = Field(description="Configuration path") - num_gpus: int = Field(description="Number of GPUs") - - def load_config(self) -> Dict[str, Any]: + model_config = ConfigDict(extra="forbid") + + model: str = Field(min_length=1, description="Model name") + arch: str = Field(min_length=1, description="Model architecture class name") + gpu: str = Field(min_length=1, description="GPU name") + isl: PositiveInt = Field(description="Input sequence length") + osl: PositiveInt = Field(description="Output sequence length") + concurrency: PositiveInt = Field(description="Concurrency") + config_path: str = Field(min_length=1, description="Configuration path") + num_gpus: PositiveInt = Field(description="Number of GPUs") + profile: Profile | None = Field( + default=None, + description="Profile discriminator used only when the workload key has multiple configs", + ) + validated_trtllm_commit: str | None = Field( + default=None, + description="Full TensorRT-LLM commit SHA against which the recipe was validated", + ) + validated_trtllm_version: str | None = Field( + default=None, + description="TensorRT-LLM release version reported by the validation source", + ) + + @field_validator("validated_trtllm_commit") + @classmethod + def _validate_commit(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip().lower() + if not VALIDATED_COMMIT_PATTERN.fullmatch(normalized): + raise ValueError("validated_trtllm_commit must be a full 40-character Git SHA") + return normalized + + @field_validator("validated_trtllm_version") + @classmethod + def _validate_version(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip() + if not VALIDATED_VERSION_PATTERN.fullmatch(normalized): + raise ValueError("validated_trtllm_version must be a release-tag-safe version") + return normalized + + @model_validator(mode="after") + def _validate_provenance_pair(self) -> "Recipe": + if bool(self.validated_trtllm_commit) != bool(self.validated_trtllm_version): + raise ValueError( + "validated_trtllm_commit and validated_trtllm_version must be provided together" + ) + return self + + def load_config(self) -> dict[str, Any]: """Load and return the YAML config at config_path.""" config_relative_path = Path(self.config_path) # Ensure config path is within the repo root @@ -96,7 +159,41 @@ def load_config(self) -> Dict[str, Any]: return yaml.safe_load(f) -class RecipeList(RootModel[List[Recipe]]): +class RecipeList(RootModel[list[Recipe]]): + @model_validator(mode="after") + def _validate_conflict_profiles(self) -> "RecipeList": + groups = defaultdict(list) + for recipe in self.root: + key = ( + recipe.model, + recipe.gpu, + recipe.num_gpus, + recipe.isl, + recipe.osl, + recipe.concurrency, + ) + groups[key].append(recipe) + + required_profiles = {"latency", "throughput"} + for key, recipes in groups.items(): + profiles = [recipe.profile for recipe in recipes] + if len(recipes) == 1: + if profiles[0] is not None: + raise ValueError(f"profile is only allowed for conflicting workload key {key}") + continue + profile_set = set(profiles) + if ( + None in profile_set + or len(profile_set) != len(profiles) + or not required_profiles.issubset(profile_set) + ): + raise ValueError( + "conflicting workload key " + f"{key} must have exactly one latency and throughput profile, " + "with an optional balanced profile" + ) + return self + @classmethod def from_yaml(cls, yaml_path: Path) -> "RecipeList": """Load and validate recipe list from YAML file.""" @@ -132,12 +229,15 @@ def assign_profile(num_recipes: int, idx: int, concurrency: int) -> str: return "High Throughput" -def select_key_recipes(recipes: List[Recipe]) -> List[Tuple[Recipe, str]]: +def select_key_recipes(recipes: list[Recipe]) -> list[tuple[Recipe, str]]: """Select key recipes (min latency, balanced, max throughput) from a list of recipes.""" if not recipes: return [] - sorted_recipes = sorted(recipes, key=lambda r: r.concurrency) + sorted_recipes = sorted( + recipes, + key=lambda r: (r.concurrency, PROFILE_ORDER.get(r.profile, -1)), + ) n = len(sorted_recipes) result = [] diff --git a/examples/configs/database/lookup.yaml b/examples/configs/database/lookup.yaml index 523cda14c412..d773fa361596 100644 --- a/examples/configs/database/lookup.yaml +++ b/examples/configs/database/lookup.yaml @@ -558,6 +558,8 @@ concurrency: 4 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -582,6 +584,8 @@ concurrency: 32 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -590,6 +594,8 @@ concurrency: 64 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -598,6 +604,8 @@ concurrency: 128 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -606,6 +614,8 @@ concurrency: 256 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -750,6 +760,8 @@ concurrency: 4 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -782,6 +794,8 @@ concurrency: 64 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -790,6 +804,8 @@ concurrency: 128 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -798,6 +814,8 @@ concurrency: 256 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -1566,3 +1584,83 @@ concurrency: 1536 config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc1536.yaml num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 256 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 256 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..c9986a111b5b --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml @@ -0,0 +1,14 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_seq_len: 8192 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml new file mode 100644 index 000000000000..bb43a3699209 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml @@ -0,0 +1,24 @@ +max_batch_size: 64 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +speculative_config: + decoding_type: MTP + max_draft_len: 1 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +max_num_tokens: 1216 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..ade4e5b1de21 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml @@ -0,0 +1,19 @@ +max_batch_size: 4 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +speculative_config: + decoding_type: MTP + max_draft_len: 3 +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_num_tokens: 1152 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..7f49059c11b1 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml @@ -0,0 +1,19 @@ +max_batch_size: 8 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +speculative_config: + decoding_type: MTP + max_draft_len: 3 +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_num_tokens: 1152 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml index ea4b03e17984..3db82c4881df 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml @@ -1,24 +1,19 @@ -max_batch_size: 512 +max_batch_size: 128 cuda_graph_config: enable_padding: true - max_batch_size: 32 -enable_attention_dp: true + max_batch_size: 128 print_iter_log: true kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.8 stream_interval: 10 moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 + backend: TRTLLM speculative_config: decoding_type: MTP - max_draft_len: 1 + max_draft_len: 3 tensor_parallel_size: 8 -moe_expert_parallel_size: 8 +moe_expert_parallel_size: 1 trust_remote_code: true -max_num_tokens: 3072 -max_seq_len: 2068 +max_num_tokens: 1600 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml index 9031655d28fd..9bc761a79868 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml @@ -1,7 +1,6 @@ -max_batch_size: 512 cuda_graph_config: enable_padding: true - max_batch_size: 256 + max_batch_size: 64 enable_attention_dp: true print_iter_log: true kv_cache_config: @@ -14,11 +13,7 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 -speculative_config: - decoding_type: MTP - max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 2112 -max_seq_len: 2068 +max_seq_len: 8192 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml index 5d7f74990a3a..03c9e6b27901 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -max_batch_size: 512 +max_batch_size: 32 cuda_graph_config: enable_padding: true max_batch_size: 32 @@ -15,5 +15,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 3136 -max_seq_len: 2068 +max_num_tokens: 1216 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml index 68eab71800d9..cf0bb6a44862 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -max_batch_size: 512 +max_batch_size: 4 cuda_graph_config: enable_padding: true max_batch_size: 4 @@ -13,7 +13,7 @@ speculative_config: decoding_type: MTP max_draft_len: 3 tensor_parallel_size: 8 -moe_expert_parallel_size: 8 +moe_expert_parallel_size: 1 trust_remote_code: true -max_num_tokens: 3136 -max_seq_len: 2068 +max_num_tokens: 1152 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml index c239238e8f0d..3348357986a6 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml @@ -1,7 +1,7 @@ +max_batch_size: 64 cuda_graph_config: enable_padding: true - max_batch_size: 16 -enable_attention_dp: true + max_batch_size: 64 print_iter_log: true kv_cache_config: dtype: fp8 @@ -9,14 +9,11 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 speculative_config: decoding_type: MTP - max_draft_len: 1 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_seq_len: 2068 +max_num_tokens: 1344 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..283c8ccd18a3 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml @@ -0,0 +1,19 @@ +max_batch_size: 16 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +speculative_config: + decoding_type: MTP + max_draft_len: 3 +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml new file mode 100644 index 000000000000..aa75a4c8d4c5 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml @@ -0,0 +1,24 @@ +max_batch_size: 64 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +speculative_config: + decoding_type: MTP + max_draft_len: 1 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +max_num_tokens: 8384 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..594aae2eaba4 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml @@ -0,0 +1,19 @@ +max_batch_size: 4 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +speculative_config: + decoding_type: MTP + max_draft_len: 3 +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..2dbc0435be33 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml @@ -0,0 +1,19 @@ +max_batch_size: 8 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +speculative_config: + decoding_type: MTP + max_draft_len: 3 +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml index baaa644ca539..143fe4bba30b 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml @@ -1,4 +1,4 @@ -max_batch_size: 256 +max_batch_size: 32 cuda_graph_config: enable_padding: true max_batch_size: 32 @@ -20,5 +20,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 8768 -max_seq_len: 9416 +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml index 93df61d6fa56..251ed48facac 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml @@ -1,4 +1,4 @@ -max_batch_size: 256 +max_batch_size: 64 cuda_graph_config: enable_padding: true max_batch_size: 64 @@ -20,5 +20,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 8768 -max_seq_len: 9416 +max_num_tokens: 8384 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml index 6a6b9fb25c63..97ce51217d82 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -max_batch_size: 512 +max_batch_size: 4 cuda_graph_config: enable_padding: true max_batch_size: 4 @@ -13,7 +13,7 @@ speculative_config: decoding_type: MTP max_draft_len: 3 tensor_parallel_size: 8 -moe_expert_parallel_size: 8 +moe_expert_parallel_size: 1 trust_remote_code: true -max_num_tokens: 10304 -max_seq_len: 9416 +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml index b39f07478407..6729a6fa5c6d 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -max_batch_size: 256 +max_batch_size: 16 cuda_graph_config: enable_padding: true max_batch_size: 16 @@ -20,5 +20,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 8768 -max_seq_len: 9416 +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/constraints.txt b/examples/constraints.txt index 4141980256e5..1aef1aa12fab 100644 --- a/examples/constraints.txt +++ b/examples/constraints.txt @@ -1,3 +1,3 @@ -tensorrt_llm==1.3.0rc21 +tensorrt_llm==1.3.0rc24 evaluate~=0.4.1 rouge_score~=0.1.2 diff --git a/examples/cpp/executor/CMakeLists.txt b/examples/cpp/executor/CMakeLists.txt deleted file mode 100644 index b448667e8d75..000000000000 --- a/examples/cpp/executor/CMakeLists.txt +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. cmake needs this line - -cmake_minimum_required(VERSION 3.27) - -set(TRTLLM_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../..") -list(APPEND CMAKE_MODULE_PATH "${TRTLLM_DIR}/cpp/cmake/modules") - -if(NOT TRTLLM_BUILD_DIR) - set(TRTLLM_BUILD_DIR "${TRTLLM_DIR}/cpp/build") -endif() -set(TRTLLM_LIB_PATH "${TRTLLM_BUILD_DIR}/tensorrt_llm/libtensorrt_llm.so") -if(NOT EXISTS ${TRTLLM_LIB_PATH}) - message(FATAL_ERROR "Cannot find ${TRTLLM_LIB_PATH}") -endif() - -set(TRTLLM_PLUGIN_PATH - "${TRTLLM_BUILD_DIR}/tensorrt_llm/plugins/libnvinfer_plugin_tensorrt_llm.so" -) -set(TRTLLM_INCLUDE_DIR "${TRTLLM_DIR}/cpp/include") - -option( - ENABLE_MULTI_DEVICE - "Enable multi device/instance examples building (requires MPI headers/libs)" - ON) - -# Determine CXX11 ABI compatibility -execute_process( - COMMAND bash -c "nm -f posix -D ${TRTLLM_LIB_PATH} | grep __cxx11" - RESULT_VARIABLE GLIB_CXX11_FOUND - OUTPUT_QUIET) -if(GLIB_CXX11_FOUND EQUAL 0) - set(USE_CXX11_ABI 1) -else() - set(USE_CXX11_ABI 0) -endif() -message(STATUS "Use CXX11 ABI: ${USE_CXX11_ABI}") -add_compile_options("-D_GLIBCXX_USE_CXX11_ABI=${USE_CXX11_ABI}") - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED TRUE) -set(CMAKE_VERBOSE_MAKEFILE 1) - -# Define project name -project(executorExamples) - -# Compile options $ ? -if(ENABLE_MULTI_DEVICE) - set(EMD 1) -else() - set(EMD 0) -endif() -set(CMAKE_CXX_FLAGS "-Wall -pthread -lstdc++ -DENABLE_MULTI_DEVICE=${EMD} ") -set(CMAKE_CXX_FLAGS_RELEASE "-O3") -set(CMAKE_BUILD_TYPE release) - -find_package(CUDAToolkit REQUIRED COMPONENTS cuda_driver cudart_static nvml) -message(STATUS "CUDA library status:") -message(STATUS " version: ${CUDAToolkit_VERSION}") -message(STATUS " libraries: ${CUDAToolkit_LIBRARY_DIR}") -message(STATUS " include path: ${CUDAToolkit_INCLUDE_DIRS}") - -# TRT dependencies -find_package(TensorRT 10 REQUIRED) - -if(${CUDAToolkit_VERSION} VERSION_GREATER_EQUAL "11") - add_definitions("-DENABLE_BF16") - message( - STATUS - "CUDA_VERSION ${CUDA_VERSION} is greater or equal than 11.0, enable -DENABLE_BF16 flag" - ) -endif() - -if(${CUDAToolkit_VERSION} VERSION_GREATER_EQUAL "11.8") - add_definitions("-DENABLE_FP8") - message( - STATUS - "CUDA_VERSION ${CUDA_VERSION} is greater or equal than 11.8, enable -DENABLE_FP8 flag" - ) -endif() - -add_subdirectory(${TRTLLM_DIR}/3rdparty 3rdparty) -FetchContent_MakeAvailable(cxxopts) - -# tensorrt_llm shared lib -add_library(tensorrt_llm SHARED IMPORTED) -set_property(TARGET tensorrt_llm PROPERTY IMPORTED_LOCATION ${TRTLLM_LIB_PATH}) -set_property( - TARGET tensorrt_llm PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES - CUDA::cuda_driver CUDA::cudart_static CUDA::nvml) - -# nvinfer_plugin_tensorrt_llm shared lib -add_library(nvinfer_plugin_tensorrt_llm SHARED IMPORTED) -set_property(TARGET nvinfer_plugin_tensorrt_llm PROPERTY IMPORTED_LOCATION - ${TRTLLM_PLUGIN_PATH}) -set_property(TARGET nvinfer_plugin_tensorrt_llm - PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES tensorrt_llm) - -include_directories(${TRTLLM_INCLUDE_DIR} ${CUDAToolkit_INCLUDE_DIRS}) - -# Basic -add_executable(executorExampleBasic executorExampleBasic.cpp) -target_link_libraries(executorExampleBasic nvinfer_plugin_tensorrt_llm) - -add_executable(executorExampleDebug executorExampleDebug.cpp) -target_link_libraries(executorExampleDebug nvinfer_plugin_tensorrt_llm) - -add_executable(executorExampleKvEvents executorExampleKvEvents.cpp) -target_link_libraries(executorExampleKvEvents nvinfer_plugin_tensorrt_llm - cxxopts::cxxopts) - -add_executable(executorExampleLogitsProcessor - executorExampleLogitsProcessor.cpp) -target_link_libraries(executorExampleLogitsProcessor - nvinfer_plugin_tensorrt_llm) - -# Advanced -if(NOT TARGET cxxopts::cxxopts) - add_subdirectory(${CMAKE_BINARY_DIR}/_deps/cxxopts-src - ${CMAKE_CURRENT_BINARY_DIR}/cxxopts) -endif() - -add_executable(executorExampleAdvanced executorExampleAdvanced.cpp) -target_link_libraries(executorExampleAdvanced nvinfer_plugin_tensorrt_llm - cxxopts::cxxopts) - -# MultiInstance -if(ENABLE_MULTI_DEVICE) - find_package(MPI REQUIRED) - message(STATUS "Using MPI_C_INCLUDE_DIRS: ${MPI_C_INCLUDE_DIRS}") - message(STATUS "Using MPI_C_LIBRARIES: ${MPI_C_LIBRARIES}") - include_directories(${MPI_C_INCLUDE_DIRS}) - - add_executable(executorExampleAdvancedMultiInstances - executorExampleAdvancedMultiInstances.cpp) - target_link_libraries( - executorExampleAdvancedMultiInstances nvinfer_plugin_tensorrt_llm - cxxopts::cxxopts ${MPI_C_LIBRARIES}) - - # FastLogits - add_executable(executorExampleFastLogits executorExampleFastLogits.cpp) - target_link_libraries(executorExampleFastLogits nvinfer_plugin_tensorrt_llm - cxxopts::cxxopts ${MPI_C_LIBRARIES}) - - add_executable(executorExampleDisaggregated executorExampleDisaggregated.cpp) - target_link_libraries( - executorExampleDisaggregated nvinfer_plugin_tensorrt_llm cxxopts::cxxopts - ${MPI_C_LIBRARIES}) -endif() diff --git a/examples/cpp/executor/README.md b/examples/cpp/executor/README.md deleted file mode 100644 index 597a38effe01..000000000000 --- a/examples/cpp/executor/README.md +++ /dev/null @@ -1,135 +0,0 @@ -# Executor API examples - -This directory contains several examples that demonstrate how to use the `Executor` API: -- The example defined in `executorExampleBasic.cpp` shows how you can generate output tokens for a single prompt in only a few lines of code. -- The example defined in `executorExampleAdvanced.cpp` supports more options such as providing an arbitrary number of input requests with arbitrary tokens per request and running in streaming mode. -- The example defined in `executorExampleLogitsProcessor.cpp` shows how to use `LogitsPostProcessor` to control output tokens. -- The example defined in `executorExampleFastLogits.cpp` shows how to use `ExternalDraftTokensConfig` for speculative decoding and optionally use the fast logits feature. -- The example defined in `executorExampleKvEvents.cpp` shows how to use the KV cache event API. -- The example defined in `executorExampleDisaggregated.cpp` shows how to use the disaggregated executor API. - -## Building the examples - -To build the examples, you first need to build the TensorRT LLM C++ shared libraries (`libtensorrt_llm.so` and `libnvinfer_plugin_tensorrt_llm.so`) using the [`build_wheel.py`](source:scripts/build_wheel.py) script. Alternatively, if you have already build the TensorRT LLM libraries, you can modify the provided `CMakeLists.txt` such that the `libtensorrt_llm.so` and `libnvinfer_plugin_tensorrt_llm.so` are imported properly. - -Once the TensorRT LLM libraries are built, you can run - -``` -mkdir build -cd build -cmake .. -make -j -``` -from the `./examples/cpp/executor/` folder to build the basic and advanced examples. - -## Preparing the TensorRT LLM engine(s) - -Before you run the examples, please make sure that you have already built engine(s) using the TensorRT LLM API. - -Use `trtllm-build` to build the TRT-LLM engine. - -## Running the examples - -### executorExampleBasic - -From the `examples/cpp/executor/build` folder, you can get run the `executorExampleBasic` example with: - -``` -./executorExampleBasic -``` -where `` is the path to the directly containing the TensorRT engine files. - -### executorExampleDebug - -This example shows how you can define which engine IO tensors should be dumped to numpy files. -From the `examples/cpp/executor/build` folder, you can get run the `executorExampleDebug` example with: - -``` -./executorExampleDebug -``` -where `` is the path to the directly containing the TensorRT engine files. - -### executorExampleAdvanced - -From the `examples/cpp/executor/build` folder, you can also run the `executorExampleAdvanced` example. To get the full list of supported input arguments, type - -``` -./executorExampleAdvanced -h -``` - -For example, you can run: - -``` -./executorExampleAdvanced --engine_dir --input_tokens_csv_file ../inputTokens.csv -``` - -to run with the provided dummy input tokens from `inputTokens.csv`. Upon successful completion, you should see the following in the logs: -``` -[TensorRT-LLM][INFO] Creating request with 6 input tokens -[TensorRT-LLM][INFO] Creating request with 4 input tokens -[TensorRT-LLM][INFO] Creating request with 10 input tokens -[TensorRT-LLM][INFO] Got 20 tokens for beam 0 for requestId 3 -[TensorRT-LLM][INFO] Request id 3 is completed. -[TensorRT-LLM][INFO] Got 14 tokens for beam 0 for requestId 2 -[TensorRT-LLM][INFO] Request id 2 is completed. -[TensorRT-LLM][INFO] Got 16 tokens for beam 0 for requestId 1 -[TensorRT-LLM][INFO] Request id 1 is completed. -[TensorRT-LLM][INFO] Writing output tokens to outputTokens.csv -[TensorRT-LLM][INFO] Exiting. -``` - -#### Multi-GPU run - -To run the `executorExampleAdvanced` on models that require multiple GPUs, you can run the example using MPI as follows: - -``` -mpirun -n --allow-run-as-root ./executorExampleAdvanced --engine_dir --input_tokens_csv_file ../inputTokens.csv -``` -where `` must equal to `tp*pp` for the TensorRT engine. By default GPU device IDs `[0...(num_ranks-1)]` will be used. - -Alternatively, it's also possible to run multi-GPU model by using the so-called `Orchestrator` communication mode, where the `Executor` instance will automatically spawn additional processes to run the model on multiple GPUs. To use the `Orchestrator` communication mode, you can run the example with: - -``` -./executorExampleAdvanced --engine_dir --input_tokens_csv_file ../inputTokens.csv --use_orchestrator_mode --worker_executable_path -``` -where `` is the absolute path to the stand-alone executor worker executable, located at`cpp/build/tensorrt_llm/executor_worker/executorWorker` by default. - - -### executorExampleFastLogits - -To run the `executorExampleFastLogits`, you need two GPUs (one for the draft model and one for the target model). You can run it as follows: - -``` -mpirun -n 3 --allow-run-as-root ./executorExampleFastLogits --engine_dir --draft_engine_dir --num_draft_tokens=3 -``` - -The examples uses 3 MPI ranks (one for the orchestrator, one for the draft model and one for the target model). - -Use `--fast_logits=false` to disable the fast logits feature. - -### executorExampleKvEvents - -From the `examples/cpp/executor/build` folder, you can get run the `executorExampleKvEvents` example with: - -``` -./executorExampleKvEvents --engine_dir -``` -where `` is the path to the directly containing the TensorRT engine files. - -This example shows how the KV Cache Event API can be used to reconstruct the state of TRT-LLM's internal radix tree. This can be used in applications such as smart routing to route requests between multiple executor instances to maximize KV Cache reuse. Events are emitted when blocks are stored, removed, or updated in the radix tree. - -### executorExampleDisaggregated - -From the `examples/cpp/executor/build` folder, you can also run the `executorExampleDisaggregated` example. To get the full list of supported input arguments, type -``` -./executorExampleDisaggregated -h -``` -Note setting `TRTLLM_USE_UCX_KVCACHE=1` is required to run disaggregated executor. -For example, you can run : -``` -export TRTLLM_USE_UCX_KVCACHE=1 - -mpirun -n --allow-run-as-root --oversubscribe ./executorExampleDisaggregated --context_engine_dir --context_rank_size --generation_engine_dir --generation_rank_size --input_tokens_csv_file ../inputTokens.csv - -``` -where `` must equal to `tp*pp` for the context engine, and `` must equal to `tp*pp` for the generation engine,the context engine and generation engine can be heterogeneous in parallelism. `` must equal to `++1`, the additional rank is used as orchestrator process. diff --git a/examples/cpp/executor/executorExampleAdvanced.cpp b/examples/cpp/executor/executorExampleAdvanced.cpp deleted file mode 100644 index e2fdf489f63d..000000000000 --- a/examples/cpp/executor/executorExampleAdvanced.cpp +++ /dev/null @@ -1,368 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include - -namespace tle = tensorrt_llm::executor; - -namespace fs = std::filesystem; - -struct RuntimeOptions -{ - std::string trtEnginePath; - std::string inputTokensCsvFile; - std::string outputTokensCsvFile; - - bool streaming; - bool excludeInputFromOutput; - tle::SizeType32 maxNewTokens; - tle::SizeType32 beamWidth; - std::optional numReturnSequences; - tle::SizeType32 timeoutMs; - - bool useOrchestratorMode; - std::string workerExecutablePath; -}; - -// Utility function to parse input arguments -RuntimeOptions parseArgs(int argc, char* argv[]); - -// Function that enqueues requests -std::vector enqueueRequests(RuntimeOptions const& runtimeOpts, tle::Executor& executor); - -// Function that waits for responses and stores output tokens -std::unordered_map waitForResponses( - RuntimeOptions const& runtimeOpts, std::vector const& requestIds, tle::Executor& executor); - -// Utility function to read input tokens from csv file -std::vector readInputTokens(std::string const& path); - -// Utility function to write output tokens from csv file -void writeOutputTokens(std::string const& path, std::vector& requestIds, - std::unordered_map const& outputTokens, tle::SizeType32 beamWidth); - -tle::SizeType32 getNumSequencesPerRequest(RuntimeOptions const& runtimeOpts); - -// Main -int main(int argc, char* argv[]) -{ - // Register the TRT-LLM plugins - initTrtLlmPlugins(); - - auto runtimeOpts = parseArgs(argc, argv); - - // Create the executor for this engine - auto executorConfig = tle::ExecutorConfig(runtimeOpts.beamWidth); - - if (runtimeOpts.useOrchestratorMode) - { - auto orchestratorConfig = tle::OrchestratorConfig(true, runtimeOpts.workerExecutablePath); - auto parallelConfig = tle::ParallelConfig(tle::CommunicationType::kMPI, tle::CommunicationMode::kORCHESTRATOR, - std::nullopt, std::nullopt, orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - } - - auto executor = tle::Executor(runtimeOpts.trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); - - if (executor.canEnqueueRequests()) - { - // Create the requests - auto requestIds = enqueueRequests(runtimeOpts, executor); - - // Wait for responses and store output tokens - auto outputTokens = waitForResponses(runtimeOpts, requestIds, executor); - - // Write output tokens csv file - TLLM_LOG_INFO("Writing output tokens to %s", runtimeOpts.outputTokensCsvFile.c_str()); - auto numSequences = getNumSequencesPerRequest(runtimeOpts); - writeOutputTokens(runtimeOpts.outputTokensCsvFile, requestIds, outputTokens, numSequences); - } - TLLM_LOG_INFO("Exiting."); - return 0; -} - -RuntimeOptions parseArgs(int argc, char* argv[]) -{ - RuntimeOptions runtimeOpts; - - cxxopts::Options options(argv[0], "Example that demonstrates how to use the Executor API"); - options.add_options()("h,help", "Print usage"); - options.add_options()("engine_dir", "Directory that store the engines.", cxxopts::value()); - options.add_options()("beam_width", "The beam width", cxxopts::value()->default_value("1")); - options.add_options()( - "num_return_sequences", "The number of return sequences per request.", cxxopts::value>()); - options.add_options()("streaming", "Operate in streaming mode", cxxopts::value()->default_value("false")); - options.add_options()("exclude_input_from_output", - "Exclude input tokens when writing output tokens. Only has effect for streaming = false. For streaming = true, " - "output tokens are not included.", - cxxopts::value()->default_value("false")); - options.add_options()( - "max_new_tokens", "The maximum number of tokens to generate", cxxopts::value()->default_value("10")); - options.add_options()( - "input_tokens_csv_file", "Path to a csv file that contains input tokens", cxxopts::value()); - options.add_options()("output_tokens_csv_file", "Path to a csv file that will contain the output tokens", - cxxopts::value()->default_value("outputTokens.csv")); - options.add_options()("timeout_ms", "The maximum time to wait for all responses, in milliseconds.", - cxxopts::value()->default_value("10000")); - options.add_options()("use_orchestrator_mode", "Use orchestrator communication mode.", - cxxopts::value()->default_value("false")); - options.add_options()("worker_executable_path", "The location of the worker executable.", - cxxopts::value()->default_value("")); - - auto parsedOptions = options.parse(argc, argv); - - // Argument: help - if (parsedOptions.count("help")) - { - TLLM_LOG_ERROR(options.help()); - exit(0); - } - - // Argument: Engine directory - if (!parsedOptions.count("engine_dir")) - { - TLLM_LOG_ERROR(options.help()); - TLLM_LOG_ERROR("Please specify engine directory."); - exit(1); - } - runtimeOpts.trtEnginePath = parsedOptions["engine_dir"].as(); - if (!fs::exists(runtimeOpts.trtEnginePath) || !fs::is_directory(runtimeOpts.trtEnginePath)) - { - TLLM_LOG_ERROR("Engine directory doesn't exist."); - exit(1); - } - - // Argument: Input tokens csv file - if (!parsedOptions.count("input_tokens_csv_file")) - { - TLLM_LOG_ERROR(options.help()); - TLLM_LOG_ERROR("Please specify input_tokens_csv_file"); - exit(1); - } - runtimeOpts.inputTokensCsvFile = parsedOptions["input_tokens_csv_file"].as(); - runtimeOpts.streaming = parsedOptions["streaming"].as(); - runtimeOpts.excludeInputFromOutput = parsedOptions["exclude_input_from_output"].as(); - runtimeOpts.maxNewTokens = parsedOptions["max_new_tokens"].as(); - runtimeOpts.beamWidth = parsedOptions["beam_width"].as(); - if (parsedOptions.count("num_return_sequences") > 0) - { - runtimeOpts.numReturnSequences = parsedOptions["num_return_sequences"].as>(); - } - runtimeOpts.timeoutMs = parsedOptions["timeout_ms"].as(); - runtimeOpts.outputTokensCsvFile = parsedOptions["output_tokens_csv_file"].as(); - - runtimeOpts.useOrchestratorMode = parsedOptions["use_orchestrator_mode"].as(); - runtimeOpts.workerExecutablePath = parsedOptions["worker_executable_path"].as(); - - return runtimeOpts; -} - -std::vector enqueueRequests(RuntimeOptions const& runtimeOpts, tle::Executor& executor) -{ - tle::OutputConfig outputConfig; - outputConfig.excludeInputFromOutput = runtimeOpts.excludeInputFromOutput; - tle::SamplingConfig samplingConfig(runtimeOpts.beamWidth); - if (runtimeOpts.numReturnSequences && runtimeOpts.beamWidth == 1) - { - samplingConfig.setTopP(0.9); - } - samplingConfig.setNumReturnSequences(runtimeOpts.numReturnSequences); - - TLLM_LOG_INFO("Reading input tokens from %s", runtimeOpts.inputTokensCsvFile.c_str()); - auto inputTokens = readInputTokens(runtimeOpts.inputTokensCsvFile); - TLLM_LOG_INFO("Number of requests: %d", inputTokens.size()); - - std::vector requests; - for (auto& tokens : inputTokens) - { - TLLM_LOG_INFO("Creating request with %d input tokens", tokens.size()); - requests.emplace_back( - std::move(tokens), runtimeOpts.maxNewTokens, runtimeOpts.streaming, samplingConfig, outputConfig); - } - - // Enqueue the requests - auto requestIds = executor.enqueueRequests(std::move(requests)); - - return requestIds; -} - -std::unordered_map waitForResponses( - RuntimeOptions const& runtimeOpts, std::vector const& requestIds, tle::Executor& executor) -{ - // Map that will be used to store output tokens for requests - std::unordered_map outputTokens; - auto numSequences = getNumSequencesPerRequest(runtimeOpts); - for (auto requestId : requestIds) - { - outputTokens[requestId] = tle::BeamTokens(numSequences); - } - - tle::SizeType32 numFinished{0}; - tle::SizeType32 iter{0}; - - // Get the new tokens for each request - while (numFinished < static_cast(requestIds.size()) && iter < runtimeOpts.timeoutMs) - { - std::chrono::milliseconds waitTime(1); - // Wait for any response - auto responses = executor.awaitResponses(waitTime); - - auto insertResponseTokens - = [&outputTokens](tle::IdType requestId, tle::SizeType32 seqIdx, tle::VecTokens const& respTokens) - { - TLLM_LOG_INFO("Got %d tokens for seqIdx %d for requestId %d", respTokens.size(), seqIdx, requestId); - - // Store the output tokens for that request id - auto& outTokens = outputTokens.at(requestId).at(seqIdx); - outTokens.insert(outTokens.end(), std::make_move_iterator(respTokens.begin()), - std::make_move_iterator(respTokens.end())); - }; - - // Loop over the responses - for (auto const& response : responses) - { - auto requestId = response.getRequestId(); - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - if (runtimeOpts.beamWidth > 1) - { - for (tle::SizeType32 beam = 0; beam < numSequences; ++beam) - { - insertResponseTokens(requestId, beam, result.outputTokenIds.at(beam)); - } - } - else - { - insertResponseTokens(requestId, result.sequenceIndex, result.outputTokenIds.at(0)); - } - if (result.isFinal) - { - TLLM_LOG_INFO("Request id %lu is completed.", requestId); - } - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - if (response.getErrorMsg() != err) - { - TLLM_THROW("Request id %lu encountered error: %s", requestId, response.getErrorMsg().c_str()); - } - } - } - ++iter; - } - if (iter == runtimeOpts.timeoutMs) - { - TLLM_THROW("Timeout exceeded."); - } - - return outputTokens; -} - -std::vector readInputTokens(std::string const& path) -{ - std::vector data; - std::ifstream file(path); - - if (!file.is_open()) - { - auto const err = std::string{"Failed to open file: "} + path; - TLLM_LOG_ERROR(err); - TLLM_THROW(err); - } - - std::string line; - while (std::getline(file, line)) - { - std::vector row; - std::stringstream ss(line); - std::string token; - - while (std::getline(ss, token, ',')) - { - try - { - row.push_back(std::stoi(token)); - } - catch (std::invalid_argument const& e) - { - TLLM_LOG_ERROR("Invalid argument: %s", e.what()); - } - catch (std::out_of_range const& e) - { - TLLM_LOG_ERROR("Out of range: %s", e.what()); - } - } - - data.push_back(row); - } - - file.close(); - return data; -} - -void writeOutputTokens(std::string const& path, std::vector& requestIds, - std::unordered_map const& outputTokens, tle::SizeType32 beamWidth) -{ - std::ofstream file(path); - - if (!file.is_open()) - { - TLLM_LOG_ERROR("Failed to open file %s", path.c_str()); - return; - } - - for (auto requestId : requestIds) - { - auto const& outTokens = outputTokens.at(requestId); - for (tle::SizeType32 beam = 0; beam < beamWidth; ++beam) - { - auto const& beamTokens = outTokens.at(beam); - for (size_t i = 0; i < beamTokens.size(); ++i) - { - file << beamTokens[i]; - if (i < beamTokens.size() - 1) - { - file << ", "; - } - } - file << "\n"; - } - } - - file.close(); -} - -tle::SizeType32 getNumSequencesPerRequest(RuntimeOptions const& runtimeOpts) -{ - auto numReturnSequences = runtimeOpts.numReturnSequences.value_or(runtimeOpts.beamWidth); - return runtimeOpts.beamWidth > 1 ? std::min(numReturnSequences, runtimeOpts.beamWidth) : numReturnSequences; -} diff --git a/examples/cpp/executor/executorExampleAdvancedMultiInstances.cpp b/examples/cpp/executor/executorExampleAdvancedMultiInstances.cpp deleted file mode 100644 index f967661ccd27..000000000000 --- a/examples/cpp/executor/executorExampleAdvancedMultiInstances.cpp +++ /dev/null @@ -1,381 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include - -namespace tle = tensorrt_llm::executor; - -namespace fs = std::filesystem; - -struct RuntimeOptions -{ - std::string trtEnginePath; - std::string inputTokensCsvFile; - std::string outputTokensCsvFile; - - bool streaming; - bool excludeInputFromOutput; - tle::SizeType32 maxNewTokens; - tle::SizeType32 beamWidth; - tle::SizeType32 timeoutMs; - - bool useOrchestratorMode; - std::string workerExecutablePath; - bool spawnProcesses; -}; - -// Utility function to parse input arguments -RuntimeOptions parseArgs(int argc, char* argv[]); - -// Function that enqueues requests -std::vector> enqueueRequests( - RuntimeOptions const& runtimeOpts, std::deque& executors); - -// Function that waits for responses and stores output tokens -std::map, tle::BeamTokens> waitForResponses(RuntimeOptions const& runtimeOpts, - std::vector> const& instanceRequestIds, std::deque& executors); - -// Utility function to read input tokens from csv file -std::vector readInputTokens(std::string const& path); - -// Utility function to write output tokens from csv file -void writeOutputTokens(std::string const& path, std::vector>& requestIds, - std::map, tle::BeamTokens> const& outputTokens, tle::SizeType32 beamWidth); - -// Main -int main(int argc, char* argv[]) -{ - // Register the TRT-LLM plugins - initTrtLlmPlugins(); - - auto runtimeOpts = parseArgs(argc, argv); - - // Create the executor for this engine - auto executorConfig = tle::ExecutorConfig(runtimeOpts.beamWidth); - - tle::KvCacheConfig kvCacheConfig{false, 10000}; - executorConfig.setKvCacheConfig(kvCacheConfig); - - bool isOrchestrator = true; - if (!runtimeOpts.spawnProcesses) - { - tensorrt_llm::mpi::initialize(tensorrt_llm::mpi::MpiThreadSupport::THREAD_MULTIPLE); - int myRank = tensorrt_llm::mpi::MpiComm::world().getRank(); - isOrchestrator = (myRank == 0); - } - - auto orchestratorConfig = tle::OrchestratorConfig( - isOrchestrator, runtimeOpts.workerExecutablePath, nullptr, runtimeOpts.spawnProcesses); - auto parallelConfig = tle::ParallelConfig(tle::CommunicationType::kMPI, tle::CommunicationMode::kORCHESTRATOR, - std::nullopt, std::nullopt, orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - - int numInstances = 3; - if (!runtimeOpts.spawnProcesses) - { - // Keep one rank for orchestrator - numInstances = tensorrt_llm::mpi::MpiComm::world().getSize() - 1; - } - std::deque executors; - for (int instanceId = 0; instanceId < numInstances; ++instanceId) - { - auto executorConfigTmp = executorConfig; - // Set the rank id participating in each model instance - if (!runtimeOpts.spawnProcesses) - { - parallelConfig.setParticipantIds({instanceId + 1}); - } - executorConfigTmp.setParallelConfig(parallelConfig); - executors.emplace_back(runtimeOpts.trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfigTmp); - } - - // Only orchestrator rank (rank 0) will enter - if (isOrchestrator) - { - // Create the requests - auto instanceRequestIds = enqueueRequests(runtimeOpts, executors); - - // Wait for responses and store output tokens - auto outputTokens = waitForResponses(runtimeOpts, instanceRequestIds, executors); - - // Write output tokens csv file - TLLM_LOG_INFO("Writing output tokens to %s", runtimeOpts.outputTokensCsvFile.c_str()); - writeOutputTokens(runtimeOpts.outputTokensCsvFile, instanceRequestIds, outputTokens, runtimeOpts.beamWidth); - } - TLLM_LOG_INFO("Exiting."); - return 0; -} - -RuntimeOptions parseArgs(int argc, char* argv[]) -{ - RuntimeOptions runtimeOpts; - - cxxopts::Options options(argv[0], "Example that demonstrates how to use the Executor API"); - options.add_options()("h,help", "Print usage"); - options.add_options()("engine_dir", "Directory that store the engines.", cxxopts::value()); - options.add_options()("beam_width", "The beam width", cxxopts::value()->default_value("1")); - options.add_options()("streaming", "Operate in streaming mode", cxxopts::value()->default_value("false")); - options.add_options()("exclude_input_from_output", - "Exclude input tokens when writing output tokens. Only has effect for streaming = false. For streaming = true, " - "output tokens are not included.", - cxxopts::value()->default_value("false")); - options.add_options()( - "max_new_tokens", "The maximum number of tokens to generate", cxxopts::value()->default_value("10")); - options.add_options()( - "input_tokens_csv_file", "Path to a csv file that contains input tokens", cxxopts::value()); - options.add_options()("output_tokens_csv_file", "Path to a csv file that will contain the output tokens", - cxxopts::value()->default_value("outputTokens.csv")); - options.add_options()("timeout_ms", "The maximum time to wait for all responses, in milliseconds.", - cxxopts::value()->default_value("10000")); - options.add_options()("worker_executable_path", "The location of the worker executable.", - cxxopts::value()->default_value("")); - options.add_options()("spawn_processes", - "Flag that controls if MPI_Comm_spawn should be used to spawn worker processes, or if they have been launched " - "with mpi already.", - cxxopts::value()->default_value("true")); - - auto parsedOptions = options.parse(argc, argv); - - // Argument: help - if (parsedOptions.count("help")) - { - TLLM_LOG_ERROR(options.help()); - exit(0); - } - - // Argument: Engine directory - if (!parsedOptions.count("engine_dir")) - { - TLLM_LOG_ERROR(options.help()); - TLLM_LOG_ERROR("Please specify engine directory."); - exit(1); - } - runtimeOpts.trtEnginePath = parsedOptions["engine_dir"].as(); - if (!fs::exists(runtimeOpts.trtEnginePath) || !fs::is_directory(runtimeOpts.trtEnginePath)) - { - TLLM_LOG_ERROR("Engine directory doesn't exist."); - exit(1); - } - - // Argument: Input tokens csv file - if (!parsedOptions.count("input_tokens_csv_file")) - { - TLLM_LOG_ERROR(options.help()); - TLLM_LOG_ERROR("Please specify input_tokens_csv_file"); - exit(1); - } - runtimeOpts.inputTokensCsvFile = parsedOptions["input_tokens_csv_file"].as(); - runtimeOpts.streaming = parsedOptions["streaming"].as(); - runtimeOpts.excludeInputFromOutput = parsedOptions["exclude_input_from_output"].as(); - runtimeOpts.maxNewTokens = parsedOptions["max_new_tokens"].as(); - runtimeOpts.beamWidth = parsedOptions["beam_width"].as(); - runtimeOpts.timeoutMs = parsedOptions["timeout_ms"].as(); - runtimeOpts.outputTokensCsvFile = parsedOptions["output_tokens_csv_file"].as(); - - runtimeOpts.workerExecutablePath = parsedOptions["worker_executable_path"].as(); - runtimeOpts.spawnProcesses = parsedOptions["spawn_processes"].as(); - - return runtimeOpts; -} - -std::vector> enqueueRequests( - RuntimeOptions const& runtimeOpts, std::deque& executors) -{ - tle::OutputConfig outputConfig; - outputConfig.excludeInputFromOutput = runtimeOpts.excludeInputFromOutput; - tle::SamplingConfig samplingConfig(runtimeOpts.beamWidth); - - TLLM_LOG_INFO("Reading input tokens from %s", runtimeOpts.inputTokensCsvFile.c_str()); - auto inputTokens = readInputTokens(runtimeOpts.inputTokensCsvFile); - TLLM_LOG_INFO("Number of requests: %d", inputTokens.size()); - - std::vector requests; - for (auto& tokens : inputTokens) - { - TLLM_LOG_INFO("Creating request with %d input tokens", tokens.size()); - requests.emplace_back( - std::move(tokens), runtimeOpts.maxNewTokens, runtimeOpts.streaming, samplingConfig, outputConfig); - } - - // Enqueue the requests - // Round robin over instances - std::vector> instanceRequestIds; - for (size_t req = 0; req < requests.size(); ++req) - { - auto instanceId = req % executors.size(); - TLLM_LOG_INFO("Enqueuing request %d for instance %d", req, instanceId); - auto requestId = executors.at(instanceId).enqueueRequest(requests[req]); - instanceRequestIds.emplace_back(instanceId, requestId); - } - TLLM_LOG_INFO("Enqueued %d requests", instanceRequestIds.size()); - return instanceRequestIds; -} - -std::map, tle::BeamTokens> waitForResponses(RuntimeOptions const& runtimeOpts, - std::vector> const& instanceRequestIds, std::deque& executors) -{ - // Map that will be used to store output tokens for requests - int numRequests = 0; - std::map, tle::BeamTokens> outputTokens; - for (auto instanceRequestId : instanceRequestIds) - { - outputTokens[instanceRequestId] = tle::BeamTokens(runtimeOpts.beamWidth); - numRequests++; - } - - tle::SizeType32 numFinished{0}; - tle::SizeType32 iter{0}; - - // Get the new tokens for each request - while (numFinished < numRequests && iter < runtimeOpts.timeoutMs) - { - std::chrono::milliseconds waitTime(1); - for (size_t instanceId = 0; instanceId < executors.size(); ++instanceId) - { - // Wait for any response for given instance - auto responses = executors.at(instanceId).awaitResponses(waitTime); - // Loop over the responses - for (auto const& response : responses) - { - auto requestId = response.getRequestId(); - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - TLLM_LOG_INFO("Number of finished requests: %d", numFinished); - - for (tle::SizeType32 beam = 0; beam < runtimeOpts.beamWidth; ++beam) - { - auto& respTokens = result.outputTokenIds.at(beam); - - TLLM_LOG_INFO("Got %d tokens for beam %d for requestId %d", respTokens.size(), beam, requestId); - - // Store the output tokens for that request id - auto& outTokens = outputTokens.at(std::make_pair(instanceId, requestId)).at(beam); - outTokens.insert(outTokens.end(), std::make_move_iterator(respTokens.begin()), - std::make_move_iterator(respTokens.end())); - } - if (result.isFinal) - { - TLLM_LOG_INFO("Request id %lu is completed.", requestId); - } - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "ReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - if (response.getErrorMsg() != err) - { - TLLM_THROW("Request id %lu encountered error: %s", requestId, response.getErrorMsg().c_str()); - } - } - } - } - ++iter; - } - if (iter == runtimeOpts.timeoutMs) - { - TLLM_THROW("Timeout exceeded."); - } - - return outputTokens; -} - -std::vector readInputTokens(std::string const& path) -{ - std::vector data; - std::ifstream file(path); - - if (!file.is_open()) - { - auto const err = std::string{"Failed to open file: "} + path; - TLLM_LOG_ERROR(err); - TLLM_THROW(err); - } - - std::string line; - while (std::getline(file, line)) - { - std::vector row; - std::stringstream ss(line); - std::string token; - - while (std::getline(ss, token, ',')) - { - try - { - row.push_back(std::stoi(token)); - } - catch (std::invalid_argument const& e) - { - TLLM_LOG_ERROR("Invalid argument: %s", e.what()); - } - catch (std::out_of_range const& e) - { - TLLM_LOG_ERROR("Out of range: %s", e.what()); - } - } - - data.push_back(row); - } - - file.close(); - return data; -} - -void writeOutputTokens(std::string const& path, std::vector>& instanceRequestIds, - std::map, tle::BeamTokens> const& outputTokens, tle::SizeType32 beamWidth) -{ - std::ofstream file(path); - - if (!file.is_open()) - { - TLLM_LOG_ERROR("Failed to open file %s", path.c_str()); - return; - } - - for (auto instanceRequestId : instanceRequestIds) - { - auto const& outTokens = outputTokens.at(instanceRequestId); - for (tle::SizeType32 beam = 0; beam < beamWidth; ++beam) - { - auto const& beamTokens = outTokens.at(beam); - for (size_t i = 0; i < beamTokens.size(); ++i) - { - file << beamTokens[i]; - if (i < beamTokens.size() - 1) - { - file << ", "; - } - } - file << "\n"; - } - } - - file.close(); -} diff --git a/examples/cpp/executor/executorExampleBasic.cpp b/examples/cpp/executor/executorExampleBasic.cpp deleted file mode 100644 index b3ae3328392c..000000000000 --- a/examples/cpp/executor/executorExampleBasic.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" - -namespace tlc = tensorrt_llm::common; -namespace tle = tensorrt_llm::executor; - -int main(int argc, char* argv[]) -{ - // Register the TRT-LLM plugins - initTrtLlmPlugins(); - - if (argc != 2) - { - TLLM_LOG_ERROR("Usage: %s ", argv[0]); - return 1; - } - - // Create the executor for this engine - tle::SizeType32 beamWidth = 1; - auto executorConfig = tle::ExecutorConfig(beamWidth); - auto trtEnginePath = argv[1]; - auto executor = tle::Executor(trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - tle::SizeType32 maxNewTokens = 5; - tle::VecTokens inputTokens{1, 2, 3, 4}; - auto request = tle::Request(inputTokens, maxNewTokens); - - // Enqueue the request - auto requestId = executor.enqueueRequest(request); - - // Wait for the response - auto responses = executor.awaitResponses(requestId); - - // Get outputTokens - auto outputTokens = responses.at(0).getResult().outputTokenIds.at(beamWidth - 1); - - TLLM_LOG_INFO("Output tokens: %s", tlc::vec2str(outputTokens).c_str()); - - return 0; -} diff --git a/examples/cpp/executor/executorExampleDebug.cpp b/examples/cpp/executor/executorExampleDebug.cpp deleted file mode 100644 index d0af1a8140b5..000000000000 --- a/examples/cpp/executor/executorExampleDebug.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" - -namespace tlc = tensorrt_llm::common; -namespace tle = tensorrt_llm::executor; - -int main(int argc, char* argv[]) -{ - // Register the TRT-LLM plugins - initTrtLlmPlugins(); - - if (argc != 2) - { - TLLM_LOG_ERROR("Usage: %s ", argv[0]); - return 1; - } - - // Create the executor for this engine - tle::SizeType32 beamWidth = 1; - auto executorConfig = tle::ExecutorConfig(beamWidth); - // Select which tensors should be dumped - auto debugConfig = tle::DebugConfig(); - debugConfig.setDebugTensorNames({"host_request_types"}); - executorConfig.setDebugConfig(debugConfig); - - auto trtEnginePath = argv[1]; - auto executor = tle::Executor(trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - tle::SizeType32 maxNewTokens = 2; - tle::VecTokens inputTokens{1, 2, 3, 4}; - auto request = tle::Request(inputTokens, maxNewTokens); - - // Enqueue the request - auto requestId = executor.enqueueRequest(request); - - // Wait for the response - auto responses = executor.awaitResponses(requestId); - - // Get outputTokens - auto outputTokens = responses.at(0).getResult().outputTokenIds.at(beamWidth - 1); - - TLLM_LOG_INFO("Output tokens: %s", tlc::vec2str(outputTokens).c_str()); - - return 0; -} diff --git a/examples/cpp/executor/executorExampleDisaggregated.cpp b/examples/cpp/executor/executorExampleDisaggregated.cpp deleted file mode 100644 index ef9ff85c5e3b..000000000000 --- a/examples/cpp/executor/executorExampleDisaggregated.cpp +++ /dev/null @@ -1,441 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" - -#include - -namespace tle = tensorrt_llm::executor; - -namespace fs = std::filesystem; - -struct RuntimeOptions -{ - std::string trtContextEnginePath; - std::string trtGenerationEnginePath; - std::string inputTokensCsvFile; - std::string outputTokensCsvFile; - - bool streaming; - bool excludeInputFromOutput; - int contextRankSize; - int generationRankSize; - tle::SizeType32 maxNewTokens; - tle::SizeType32 beamWidth; - std::optional numReturnSequences; - tle::SizeType32 timeoutMs; -}; - -RuntimeOptions parseArgs(int argc, char* argv[]); - -// Function that enqueues requests into context executor and generation executor -std::unordered_map enqueueRequests( - RuntimeOptions const& runtimeOpts, tle::Executor& contextExecutor, tle::Executor& generationExecutor); - -// Function that waits for gen responses and stores output tokens -std::unordered_map waitForGenResponses(RuntimeOptions const& runtimeOpts, - std::unordered_map const& genRequestIdToContextRequestId, - tle::Executor& generationExecutor); - -// Utility function to read input tokens from csv file -std::vector readInputTokens(std::string const& path); - -// Utility function to write output tokens from csv file -void writeOutputTokens(std::string const& path, - std::unordered_map& genRequestIdToContextRequestId, - std::unordered_map const& outputTokens, tle::SizeType32 beamWidth); - -int main(int argc, char* argv[]) -{ - - // Register the TRT-LLM plugins - initTrtLlmPlugins(); - - auto runtimeOpts = parseArgs(argc, argv); - TLLM_CHECK_WITH_INFO(runtimeOpts.beamWidth == 1, "Only support beamWidth =1"); - TLLM_CHECK_WITH_INFO( - runtimeOpts.numReturnSequences.has_value() == false || runtimeOpts.numReturnSequences.value() == 1, - "Only support numReturnSequences =1"); - // Create the executor for this engine - auto contextExecutorConfig = tle::ExecutorConfig(runtimeOpts.beamWidth); - auto generationExecutorConfig = tle::ExecutorConfig(runtimeOpts.beamWidth); - bool isOrchestrator = (tensorrt_llm::mpi::MpiComm::world().getRank() == 0); - auto orchestratorConfig = tle::OrchestratorConfig(isOrchestrator, "", nullptr, false); - int contextRankSize = runtimeOpts.contextRankSize; - int generationRankSize = runtimeOpts.generationRankSize; - TLLM_CHECK_WITH_INFO(tensorrt_llm::mpi::MpiComm::world().getSize() >= contextRankSize + generationRankSize + 1, - " MPI should launch at least [contextRankSize+generationRankSize+1]: %d processes", - contextRankSize + generationRankSize + 1); - int deviceCount = -1; - TLLM_CHECK(cudaGetDeviceCount(&deviceCount) == cudaSuccess); - - std::vector contextRankIds(contextRankSize); - std::vector contextDeviceIds(contextRankSize); - std::vector generationRankIds(generationRankSize); - std::vector generationDeviceIds(generationRankSize); - for (int i = 0; i < contextRankSize; i++) - { - contextRankIds[i] = i + 1; - contextDeviceIds[i] = i % deviceCount; - TLLM_LOG_INFO("context Rank %d on device %d", contextRankIds[i], contextDeviceIds[i]); - } - tle::ParallelConfig contextParallelConfig{tensorrt_llm::executor::CommunicationType::kMPI, - tensorrt_llm::executor::CommunicationMode::kORCHESTRATOR, contextDeviceIds, contextRankIds, orchestratorConfig}; - - for (int i = 0; i < generationRankSize; i++) - { - generationRankIds[i] = i + 1 + contextRankSize; - generationDeviceIds[i] = (i + contextRankSize) % deviceCount; - TLLM_LOG_INFO("generation Rank %d on device %d", generationRankIds[i], generationDeviceIds[i]); - } - tle::ParallelConfig generationParallelConfig{tensorrt_llm::executor::CommunicationType::kMPI, - tensorrt_llm::executor::CommunicationMode::kORCHESTRATOR, generationDeviceIds, generationRankIds, - orchestratorConfig}; - - contextExecutorConfig.setParallelConfig(contextParallelConfig); - generationExecutorConfig.setParallelConfig(generationParallelConfig); - - auto contextExecutor - = tle::Executor(runtimeOpts.trtContextEnginePath, tle::ModelType::kDECODER_ONLY, contextExecutorConfig); - auto generationExecutor - = tle::Executor(runtimeOpts.trtGenerationEnginePath, tle::ModelType::kDECODER_ONLY, generationExecutorConfig); - tensorrt_llm::mpi::MpiComm::world().barrier(); - - if (tensorrt_llm::mpi::MpiComm::world().getRank() == 0) - { - - TLLM_CHECK_WITH_INFO(contextExecutor.canEnqueueRequests(), "contextExecutor can't enqueue requests"); - TLLM_CHECK_WITH_INFO(generationExecutor.canEnqueueRequests(), "generationExecutor can't enqueue requests"); - auto genRequestIdsToContextRequestIds = enqueueRequests(runtimeOpts, contextExecutor, generationExecutor); - auto outputTokens = waitForGenResponses(runtimeOpts, genRequestIdsToContextRequestIds, generationExecutor); - TLLM_LOG_INFO("Writing output tokens to %s", runtimeOpts.outputTokensCsvFile.c_str()); - writeOutputTokens( - runtimeOpts.outputTokensCsvFile, genRequestIdsToContextRequestIds, outputTokens, runtimeOpts.beamWidth); - } - tensorrt_llm::mpi::MpiComm::world().barrier(); - - TLLM_LOG_INFO("Exiting."); - return 0; -} - -RuntimeOptions parseArgs(int argc, char* argv[]) -{ - - RuntimeOptions runtimeOpts; - - cxxopts::Options options(argv[0], "Example that demonstrates how to use the Executor Disaggregated API"); - options.add_options()("h,help", "Print usage"); - options.add_options()( - "context_engine_dir", "Directory that store the context engine.", cxxopts::value()); - options.add_options()( - "generation_engine_dir", "Directory that store the generation engine.", cxxopts::value()); - options.add_options()( - "context_rank_size", "The number of ranks for the context engine", cxxopts::value()->default_value("1")); - options.add_options()("generation_rank_size", "The number of ranks for the generation engine", - cxxopts::value()->default_value("1")); - options.add_options()("beam_width", "The beam width", cxxopts::value()->default_value("1")); - options.add_options()( - "num_return_sequences", "The number of return sequences per request.", cxxopts::value>()); - options.add_options()("streaming", "Operate in streaming mode", cxxopts::value()->default_value("false")); - options.add_options()("exclude_input_from_output", - "Exclude input tokens when writing output tokens. Only has effect for streaming = false. For streaming = true, " - "output tokens are not included.", - cxxopts::value()->default_value("false")); - options.add_options()( - "max_new_tokens", "The maximum number of tokens to generate", cxxopts::value()->default_value("10")); - options.add_options()( - "input_tokens_csv_file", "Path to a csv file that contains input tokens", cxxopts::value()); - options.add_options()("output_tokens_csv_file", "Path to a csv file that will contain the output tokens", - cxxopts::value()->default_value("outputTokens.csv")); - options.add_options()("timeout_ms", "The maximum time to wait for all responses, in milliseconds.", - cxxopts::value()->default_value("10000")); - - auto parsedOptions = options.parse(argc, argv); - - // Argument: help - if (parsedOptions.count("help")) - { - TLLM_LOG_ERROR(options.help()); - exit(0); - } - - runtimeOpts.trtContextEnginePath = parsedOptions["context_engine_dir"].as(); - if (!fs::exists(runtimeOpts.trtContextEnginePath) || !fs::is_directory(runtimeOpts.trtContextEnginePath)) - { - TLLM_LOG_ERROR("Context engine directory doesn't exist."); - exit(1); - } - - runtimeOpts.trtGenerationEnginePath = parsedOptions["generation_engine_dir"].as(); - if (!fs::exists(runtimeOpts.trtGenerationEnginePath) || !fs::is_directory(runtimeOpts.trtGenerationEnginePath)) - { - TLLM_LOG_ERROR("Generation engine directory doesn't exist."); - exit(1); - } - // Argument: Input tokens csv file - if (!parsedOptions.count("input_tokens_csv_file")) - { - TLLM_LOG_ERROR(options.help()); - TLLM_LOG_ERROR("Please specify input_tokens_csv_file"); - exit(1); - } - - runtimeOpts.inputTokensCsvFile = parsedOptions["input_tokens_csv_file"].as(); - runtimeOpts.streaming = parsedOptions["streaming"].as(); - runtimeOpts.excludeInputFromOutput = parsedOptions["exclude_input_from_output"].as(); - runtimeOpts.maxNewTokens = parsedOptions["max_new_tokens"].as(); - runtimeOpts.beamWidth = parsedOptions["beam_width"].as(); - runtimeOpts.contextRankSize = parsedOptions["context_rank_size"].as(); - runtimeOpts.generationRankSize = parsedOptions["generation_rank_size"].as(); - if (parsedOptions.count("num_return_sequences") > 0) - { - runtimeOpts.numReturnSequences = parsedOptions["num_return_sequences"].as>(); - } - runtimeOpts.timeoutMs = parsedOptions["timeout_ms"].as(); - runtimeOpts.outputTokensCsvFile = parsedOptions["output_tokens_csv_file"].as(); - - return runtimeOpts; -} - -std::unordered_map enqueueRequests( - RuntimeOptions const& runtimeOpts, tle::Executor& contextExecutor, tle::Executor& generationExecutor) -{ - - tle::OutputConfig outputConfig; - outputConfig.excludeInputFromOutput = runtimeOpts.excludeInputFromOutput; - tle::SamplingConfig samplingConfig(runtimeOpts.beamWidth); - std::unordered_map genRequestIdToContextRequestId; - if (runtimeOpts.numReturnSequences && runtimeOpts.beamWidth == 1) - { - samplingConfig.setTopP(0.9); - } - samplingConfig.setNumReturnSequences(runtimeOpts.numReturnSequences); - - TLLM_LOG_INFO("Reading input tokens from %s", runtimeOpts.inputTokensCsvFile.c_str()); - auto inputTokens = readInputTokens(runtimeOpts.inputTokensCsvFile); - TLLM_LOG_INFO("Number of requests: %d", inputTokens.size()); - - std::vector requests; - for (auto& tokens : inputTokens) - { - TLLM_LOG_INFO("Creating request with %d input tokens", tokens.size()); - requests.emplace_back( - std::move(tokens), runtimeOpts.maxNewTokens, runtimeOpts.streaming, samplingConfig, outputConfig); - requests.back().setRequestType(tensorrt_llm::executor::RequestType::REQUEST_TYPE_CONTEXT_ONLY); - } - - auto contextRequestIds = contextExecutor.enqueueRequests(requests); - - for (size_t i = 0; i < requests.size(); i++) - { - - TLLM_LOG_INFO("waiting response for Context request id: %lu,", contextRequestIds[i]); - auto response = contextExecutor.awaitResponses(contextRequestIds[i]); - TLLM_LOG_INFO("response received for Context request id: %lu", contextRequestIds[i]); - TLLM_CHECK(response.size() == 1); - TLLM_CHECK(response.back().getResult().contextPhaseParams.has_value()); - requests.at(i).setContextPhaseParams(response.back().getResult().contextPhaseParams.value()); - requests.at(i).setRequestType(tensorrt_llm::executor::RequestType::REQUEST_TYPE_GENERATION_ONLY); - auto genRequestId = generationExecutor.enqueueRequest(requests.at(i)); - genRequestIdToContextRequestId[genRequestId] = contextRequestIds[i]; - - TLLM_LOG_INFO("enqueuing generation request for Context request id: %lu, generation request id: %lu", - contextRequestIds[i], genRequestId); - } - - return genRequestIdToContextRequestId; -} - -std::unordered_map waitForGenResponses(RuntimeOptions const& runtimeOpts, - std::unordered_map const& genRequestIdToContextRequestId, - tle::Executor& generationExecutor) -{ - - // Map that will be used to store output tokens for requests - std::unordered_map outputTokens; - std::vector contextRequestIds{}; - std::vector genRequestIds{}; - for (auto const& [key, value] : genRequestIdToContextRequestId) - { - genRequestIds.push_back(key); - contextRequestIds.push_back(value); - } - for (auto contextRequestId : contextRequestIds) - { - outputTokens[contextRequestId] = tle::BeamTokens(runtimeOpts.beamWidth); - } - - tle::SizeType32 numFinished{0}; - tle::SizeType32 iter{0}; - - // Get the new tokens for each request - while (numFinished < static_cast(genRequestIds.size()) && iter < runtimeOpts.timeoutMs) - { - std::chrono::milliseconds waitTime(1); - // Wait for any response - auto responses = generationExecutor.awaitResponses(waitTime); - - auto insertResponseTokens = [&outputTokens, &genRequestIdToContextRequestId](tle::IdType genRequestId, - tle::SizeType32 seqIdx, tle::VecTokens const& respTokens) - { - TLLM_LOG_INFO("Got %d tokens for seqIdx %d for genRequestId %d,contextRequestId %d", respTokens.size(), - seqIdx, genRequestId, genRequestIdToContextRequestId.at(genRequestId)); - - // Store the output tokens for that request id - auto& outTokens = outputTokens.at(genRequestIdToContextRequestId.at(genRequestId)).at(seqIdx); - outTokens.insert(outTokens.end(), std::make_move_iterator(respTokens.begin()), - std::make_move_iterator(respTokens.end())); - }; - - // Loop over the responses - for (auto const& response : responses) - { - auto genRequestId = response.getRequestId(); - if (!response.hasError()) - { - auto result = response.getResult(); - numFinished += result.isFinal; - if (runtimeOpts.beamWidth > 1) - { - for (tle::SizeType32 beam = 0; beam < runtimeOpts.beamWidth; ++beam) - { - insertResponseTokens(genRequestId, beam, result.outputTokenIds.at(beam)); - } - } - else - { - insertResponseTokens(genRequestId, result.sequenceIndex, result.outputTokenIds.at(0)); - } - if (result.isFinal) - { - TLLM_LOG_INFO("genRequest id %lu ,contextRequestId %lu is completed.", genRequestId, - genRequestIdToContextRequestId.at(genRequestId)); - } - } - else - { - // Allow response with error only if awaitResponse processed a terminated request id - std::string err = "genReqId " + std::to_string(response.getRequestId()) - + " has already been processed and was terminated."; - if (response.getErrorMsg() != err) - { - TLLM_THROW("GenRequest id %lu encountered error: %s", genRequestId, response.getErrorMsg().c_str()); - } - } - } - ++iter; - } - if (iter == runtimeOpts.timeoutMs) - { - TLLM_THROW("Timeout exceeded."); - } - - return outputTokens; -} - -std::vector readInputTokens(std::string const& path) -{ - std::vector data; - std::ifstream file(path); - - if (!file.is_open()) - { - auto const err = std::string{"Failed to open file: "} + path; - TLLM_LOG_ERROR(err); - TLLM_THROW(err); - } - - std::string line; - while (std::getline(file, line)) - { - std::vector row; - std::stringstream ss(line); - std::string token; - - while (std::getline(ss, token, ',')) - { - try - { - row.push_back(std::stoi(token)); - } - catch (std::invalid_argument const& e) - { - TLLM_LOG_ERROR("Invalid argument: %s", e.what()); - } - catch (std::out_of_range const& e) - { - TLLM_LOG_ERROR("Out of range: %s", e.what()); - } - } - - data.push_back(row); - } - - file.close(); - return data; -} - -void writeOutputTokens(std::string const& path, - std::unordered_map& genRequestIdToContextRequestId, - std::unordered_map const& outputTokens, tle::SizeType32 beamWidth) -{ - std::ofstream file(path); - - if (!file.is_open()) - { - TLLM_LOG_ERROR("Failed to open file %s", path.c_str()); - return; - } - std::vector requestIds; - for (auto const& [key, value] : genRequestIdToContextRequestId) - { - requestIds.push_back(value); - } - std::sort(requestIds.begin(), requestIds.end()); - - for (auto requestId : requestIds) - { - auto const& outTokens = outputTokens.at(requestId); - for (tle::SizeType32 beam = 0; beam < beamWidth; ++beam) - { - auto const& beamTokens = outTokens.at(beam); - for (size_t i = 0; i < beamTokens.size(); ++i) - { - file << beamTokens[i]; - if (i < beamTokens.size() - 1) - { - file << ", "; - } - } - file << "\n"; - } - } - - file.close(); -} diff --git a/examples/cpp/executor/executorExampleFastLogits.cpp b/examples/cpp/executor/executorExampleFastLogits.cpp deleted file mode 100644 index 3611a1bb73d7..000000000000 --- a/examples/cpp/executor/executorExampleFastLogits.cpp +++ /dev/null @@ -1,264 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include - -namespace tlc = tensorrt_llm::common; -namespace tle = tensorrt_llm::executor; - -namespace fs = std::filesystem; - -struct RuntimeOptions -{ - std::string trtDraftEnginePath; - std::string trtEnginePath; - bool fastLogits; - tle::SizeType32 numDraftTokens; -}; - -// Utility function to parse input arguments -RuntimeOptions parseArgs(int argc, char* argv[]); - -// Runs a draft request -tle::Result executeDraftRequest(tle::Executor& executor, RuntimeOptions const& runtimeOpts); - -// Runs a target request -tle::Result executeTargetRequest( - tle::Executor& executor, tle::Result const& draftResult, RuntimeOptions const& runtimeOpts); - -// Main -int main(int argc, char* argv[]) -{ - // Register the TRT-LLM plugins - initTrtLlmPlugins(); - - auto runtimeOpts = parseArgs(argc, argv); - - // Create the executor for this engine - auto executorConfig = tle::ExecutorConfig(); - - tensorrt_llm::mpi::initialize(tensorrt_llm::mpi::MpiThreadSupport::THREAD_MULTIPLE); - int const myRank = tensorrt_llm::mpi::MpiComm::world().getRank(); - bool const isOrchestrator = (myRank == 0); - - auto kvCacheConfig = tle::KvCacheConfig(true /* enableBlockReuse */); - executorConfig.setKvCacheConfig(kvCacheConfig); - - auto orchestratorConfig - = tle::OrchestratorConfig(isOrchestrator, "" /* workerExecutablePath */, nullptr, false /* spawnPrcesses */); - auto parallelConfig = tle::ParallelConfig(tle::CommunicationType::kMPI, tle::CommunicationMode::kORCHESTRATOR, - std::nullopt, std::nullopt, orchestratorConfig); - executorConfig.setParallelConfig(parallelConfig); - - auto specDecConfig = tle::SpeculativeDecodingConfig(runtimeOpts.fastLogits); - executorConfig.setSpecDecConfig(specDecConfig); - - std::unique_ptr draftExecutor; - std::unique_ptr targetExecutor; - - if (isOrchestrator) - { - auto executorConfigDraft = executorConfig; - parallelConfig.setParticipantIds({1}); - executorConfigDraft.setParallelConfig(parallelConfig); - - draftExecutor = std::make_unique( - runtimeOpts.trtDraftEnginePath, tle::ModelType::kDECODER_ONLY, executorConfigDraft); - - parallelConfig.setParticipantIds({2}); - executorConfig.setParallelConfig(parallelConfig); - - targetExecutor - = std::make_unique(runtimeOpts.trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); - } - else if (myRank == 1) // draft model process - { - parallelConfig.setParticipantIds({1}); - parallelConfig.setDeviceIds({0}); - executorConfig.setParallelConfig(parallelConfig); - draftExecutor = std::make_unique( - runtimeOpts.trtDraftEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); - } - else if (myRank == 2) // target model process - { - parallelConfig.setParticipantIds({2}); - parallelConfig.setDeviceIds({1}); - executorConfig.setParallelConfig(parallelConfig); - targetExecutor - = std::make_unique(runtimeOpts.trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); - ; - } - - // Only orchestrator rank (rank 0) will enter - if (isOrchestrator) - { - auto draftResult = executeDraftRequest(*draftExecutor, runtimeOpts); - - executeTargetRequest(*targetExecutor, draftResult, runtimeOpts); - } - TLLM_LOG_INFO("Exiting."); - return 0; -} - -RuntimeOptions parseArgs(int argc, char* argv[]) -{ - RuntimeOptions runtimeOpts; - - cxxopts::Options options(argv[0], "Example that demonstrates how to use the Executor API"); - options.add_options()("h,help", "Print usage"); - options.add_options()("engine_dir", "Directory that store the engine.", cxxopts::value()); - options.add_options()("draft_engine_dir", "Directory that store the draft engine.", cxxopts::value()); - options.add_options()( - "fast_logits", "Use speculative decoding fast logits feature", cxxopts::value()->default_value("true")); - options.add_options()( - "num_draft_tokens", "Number of draft tokens to use", cxxopts::value()->default_value("5")); - - auto parsedOptions = options.parse(argc, argv); - - // Argument: help - if (parsedOptions.count("help")) - { - TLLM_LOG_ERROR(options.help()); - exit(0); - } - - // Argument: Engine directory - if (!parsedOptions.count("engine_dir")) - { - TLLM_LOG_ERROR(options.help()); - TLLM_LOG_ERROR("Please specify engine directory."); - exit(1); - } - runtimeOpts.trtEnginePath = parsedOptions["engine_dir"].as(); - if (!fs::exists(runtimeOpts.trtEnginePath) || !fs::is_directory(runtimeOpts.trtEnginePath)) - { - TLLM_LOG_ERROR("Engine directory doesn't exist."); - exit(1); - } - - // Argument: Draft engine directory - if (!parsedOptions.count("draft_engine_dir")) - { - TLLM_LOG_ERROR(options.help()); - TLLM_LOG_ERROR("Please specify draft engine directory."); - exit(1); - } - runtimeOpts.trtDraftEnginePath = parsedOptions["draft_engine_dir"].as(); - if (!fs::exists(runtimeOpts.trtDraftEnginePath) || !fs::is_directory(runtimeOpts.trtDraftEnginePath)) - { - TLLM_LOG_ERROR("Draft engine directory doesn't exist."); - exit(1); - } - - runtimeOpts.fastLogits = parsedOptions["fast_logits"].as(); - runtimeOpts.numDraftTokens = parsedOptions["num_draft_tokens"].as(); - - return runtimeOpts; -} - -tle::Result executeDraftRequest(tle::Executor& executor, RuntimeOptions const& runtimeOpts) -{ - tle::OutputConfig outputConfig; - outputConfig.returnGenerationLogits = true; - - // Create the request - tle::SizeType32 maxNewTokens = runtimeOpts.numDraftTokens; - tle::VecTokens inputTokens{1, 2, 3, 4}; - - tle::Request request{std::move(inputTokens), maxNewTokens}; - request.setOutputConfig(outputConfig); - - // Enqueue the request - auto requestId = executor.enqueueRequest(std::move(request)); - - // Wait for the response - auto responses = executor.awaitResponses(requestId); - - if (responses.at(0).hasError()) - { - TLLM_LOG_ERROR(responses.at(0).getErrorMsg()); - exit(1); - } - - auto outputTokens = responses.at(0).getResult().outputTokenIds.at(0); - - TLLM_LOG_INFO("[DRAFT] Output tokens: %s", tlc::vec2str(outputTokens).c_str()); - - return responses.at(0).getResult(); -} - -tle::Result executeTargetRequest( - tle::Executor& executor, tle::Result const& draftResult, RuntimeOptions const& runtimeOpts) -{ - // Create the request - tle::SizeType32 maxNewTokens = runtimeOpts.numDraftTokens + 1; - tle::VecTokens inputTokens{1, 2, 3, 4}; - - tle::Request request{std::move(inputTokens), maxNewTokens}; - - tle::VecTokens const& outputTokenIds = draftResult.outputTokenIds.at(0); - tle::VecTokens draftTokens(outputTokenIds.end() - runtimeOpts.numDraftTokens, outputTokenIds.end()); - TLLM_LOG_INFO("[DRAFT] Draft tokens: %s", tlc::vec2str(draftTokens).c_str()); - - tle::Tensor logitsTensor; - - if (runtimeOpts.fastLogits) - { - auto const& logitsInfo = draftResult.specDecFastLogitsInfo.value(); - logitsTensor = logitsInfo.toTensor(); - } - else - { - auto generationLogits = draftResult.generationLogits.value(); - auto logitsShape = generationLogits.getShape(); - TLLM_CHECK(logitsShape[0] == 1); - logitsTensor = tle::Tensor::cpu(generationLogits.getDataType(), {logitsShape[1], logitsShape[2]}); - std::memcpy(logitsTensor.getData(), generationLogits.getData(), generationLogits.getSizeInBytes()); - } - - tle::ExternalDraftTokensConfig draftTokensConfig( - std::move(draftTokens), logitsTensor, std::nullopt /* acceptance threshold */, runtimeOpts.fastLogits); - request.setExternalDraftTokensConfig(draftTokensConfig); - - // Enqueue the request - auto requestId = executor.enqueueRequest(std::move(request)); - - // Wait for the response - auto responses = executor.awaitResponses(requestId); - - if (responses.at(0).hasError()) - { - TLLM_LOG_ERROR(responses.at(0).getErrorMsg()); - exit(1); - } - - auto outputTokens = responses.at(0).getResult().outputTokenIds.at(0); - - TLLM_LOG_INFO("[TARGET] Output tokens: %s", tlc::vec2str(outputTokens).c_str()); - - return responses.at(0).getResult(); -} diff --git a/examples/cpp/executor/executorExampleKvEvents.cpp b/examples/cpp/executor/executorExampleKvEvents.cpp deleted file mode 100644 index ea1923294382..000000000000 --- a/examples/cpp/executor/executorExampleKvEvents.cpp +++ /dev/null @@ -1,341 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include - -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" -#include - -namespace tlc = tensorrt_llm::common; -namespace tle = tensorrt_llm::executor; -namespace fs = std::filesystem; - -struct RuntimeOptions -{ - std::string trtEnginePath; - tle::SizeType32 numSysPrompts; - - tle::SizeType32 sysPromptTokens; - tle::SizeType32 contextTokens; - - tle::SizeType32 maxTokensMean; - tle::SizeType32 maxTokensStddev; - - tle::SizeType32 numRequests; - - size_t hostCacheSize; - size_t maxTokensInPagedKvCache; -}; - -struct KVCacheBlock -{ - KVCacheBlock(size_t hash, int cacheLevel, int priority, std::optional loraId = std::nullopt, - std::shared_ptr prevBlock = nullptr, std::optional cacheSalt = std::nullopt); - - size_t hash; - int cacheLevel; - int priority; - - std::optional loraId; - std::optional cacheSalt; - - std::shared_ptr prevBlock; - std::unordered_map> nextBlocks; -}; - -class RadixTree -{ -public: - explicit RadixTree(tle::Executor& executor); - // Check the executor for new events. - void pollEvents(); - -private: - std::shared_ptr mCacheEventManager; - // The root block of the radix tree - std::shared_ptr root; - // A table mapping block hashes to their pointers - std::unordered_map> blockTable; - // Event counter - size_t eventCounter; -}; - -// Utility function to parse input arguments -RuntimeOptions parseArgs(int argc, char* argv[]); - -// Create a tle::Request -tle::Request makeRequest(int sysPromptTokens, int contextTokens, std::uniform_int_distribution sysPromptSelector, - std::normal_distribution maxNumTokensSelector); - -std::default_random_engine gen; - -int main(int argc, char* argv[]) -{ - // Register the TRT-LLM plugins - initTrtLlmPlugins(); - - auto runtimeOpts = parseArgs(argc, argv); - - // Create the executor for this engine - auto executorConfig = tle::ExecutorConfig(1); // Beam width 1 is required for cache block reuse - auto kvCacheConfig = tle::KvCacheConfig(true, - runtimeOpts.maxTokensInPagedKvCache ? std::optional(runtimeOpts.maxTokensInPagedKvCache) - : std::nullopt); // Enable cache block reuse - kvCacheConfig.setHostCacheSize(runtimeOpts.hostCacheSize); - kvCacheConfig.setEventBufferMaxSize(32768); - executorConfig.setKvCacheConfig(kvCacheConfig); - - auto executor = tle::Executor(runtimeOpts.trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); - - auto radixTree = RadixTree(executor); - - auto activeRequests = runtimeOpts.numRequests; - - std::uniform_int_distribution sysPromptSelector( - 1, runtimeOpts.numSysPrompts); // Select a system prompt between 1 and `runtimeOpts.numSysPrompts` - std::normal_distribution maxNumTokensSelector(runtimeOpts.maxTokensMean, runtimeOpts.maxTokensStddev); - - // Create and enqueue the requests - for (int i = 0; i < runtimeOpts.numRequests; i++) - { - std::ignore = executor.enqueueRequest(makeRequest( - runtimeOpts.sysPromptTokens, runtimeOpts.contextTokens, sysPromptSelector, maxNumTokensSelector)); - } - - while (activeRequests > 0) - { - auto responses = executor.awaitResponses(std::chrono::milliseconds(20)); - for (auto const& response : responses) - { - if (response.getResult().isFinal) - activeRequests--; - } - // Only call pollEvents once every 20ms. Events are only added to the queue once per iteration, so no need to - // poll faster than this. - radixTree.pollEvents(); - } - - return 0; -} - -RuntimeOptions parseArgs(int argc, char* argv[]) -{ - RuntimeOptions runtimeOpts; - - cxxopts::Options options(argv[0], "Example that demonstrates how to use the ExecutorKVCacheManager API"); - options.add_options()("h,help", "Print usage"); - options.add_options()("engine_dir", "Directory that store the engines.", cxxopts::value()); - options.add_options()("num_sys_prompts", "Amount of unique simulated system prompts to use", - cxxopts::value()->default_value("10")); - options.add_options()( - "sys_prompt_tokens", "Size of the simulated system prompts", cxxopts::value()->default_value("256")); - options.add_options()("context_tokens", "Amount of varying context tokens coming after the system prompts", - cxxopts::value()->default_value("128")); - options.add_options()( - "max_tokens_mean", "Mean number of max output tokens", cxxopts::value()->default_value("128")); - options.add_options()( - "max_tokens_stddev", "Standard deviation of max output tokens", cxxopts::value()->default_value("32")); - options.add_options()( - "num_requests", "Amount of requests to send to the engine", cxxopts::value()->default_value("100")); - options.add_options()("host_cache_size", "Size of the KV Cache in host memory in bytes", - cxxopts::value()->default_value("0")); - options.add_options()("max_tokens_in_paged_kv_cache", "Amount of tokens in the kv cache", - cxxopts::value()->default_value("0")); - - auto parsedOptions = options.parse(argc, argv); - - // Argument: help - if (parsedOptions.count("help")) - { - TLLM_LOG_ERROR(options.help()); - exit(0); - } - - // Argument: Engine directory - if (!parsedOptions.count("engine_dir")) - { - TLLM_LOG_ERROR(options.help()); - TLLM_LOG_ERROR("Please specify engine directory."); - exit(1); - } - runtimeOpts.trtEnginePath = parsedOptions["engine_dir"].as(); - if (!fs::exists(runtimeOpts.trtEnginePath) || !fs::is_directory(runtimeOpts.trtEnginePath)) - { - TLLM_LOG_ERROR("Engine directory doesn't exist."); - exit(1); - } - - runtimeOpts.numSysPrompts = parsedOptions["num_sys_prompts"].as(); - runtimeOpts.sysPromptTokens = parsedOptions["sys_prompt_tokens"].as(); - runtimeOpts.contextTokens = parsedOptions["context_tokens"].as(); - runtimeOpts.maxTokensMean = parsedOptions["max_tokens_mean"].as(); - runtimeOpts.maxTokensStddev = parsedOptions["max_tokens_stddev"].as(); - runtimeOpts.numRequests = parsedOptions["num_requests"].as(); - runtimeOpts.hostCacheSize = parsedOptions["host_cache_size"].as(); - runtimeOpts.maxTokensInPagedKvCache = parsedOptions["max_tokens_in_paged_kv_cache"].as(); - - return runtimeOpts; -} - -KVCacheBlock::KVCacheBlock(size_t hash, int cacheLevel, int priority, std::optional loraId, - std::shared_ptr prevBlock, std::optional cacheSalt) - : hash{hash} - , cacheLevel{cacheLevel} - , priority{priority} - , loraId{loraId} - , cacheSalt{std::move(cacheSalt)} - , prevBlock{prevBlock} - , nextBlocks{} -{ -} - -RadixTree::RadixTree(tle::Executor& executor) - : mCacheEventManager(*executor.getKVCacheEventManager()) - , eventCounter{1} -{ - // Use id=-1 for the root block. Doesn't matter what exact id is used, just that it is unique. - root = std::make_shared(-1, -1, -1); - blockTable[-1] = root; - - // Wait for the `CREATED` event to be emitted. - while (true) - { - auto events = mCacheEventManager->getLatestEvents(); - if (events.size() == 1) - { - auto const& eventData = std::get(events.front().data); - TLLM_LOG_INFO("Event ID %d: KV Cache Manager initialized with blocks per level of: %s", - events.front().eventId, tlc::vec2str(eventData.numBlocksPerCacheLevel).c_str()); - break; - } - } -}; - -void RadixTree::pollEvents() -{ - auto events = mCacheEventManager->getLatestEvents(std::chrono::milliseconds(20)); - for (tle::KVCacheEvent const& event : events) - { - TLLM_CHECK(event.eventId == eventCounter++); - if (std::holds_alternative(event.data)) - { - // Blocks have been stored into the radix tree - auto const& eventData = std::get(event.data); - auto prevBlock = blockTable[eventData.parentHash.value_or(-1)]; - - // This block should be in the tree - TLLM_CHECK(blockTable.find(prevBlock->hash) != blockTable.end()); - - for (auto& block : eventData.blocks) - { - - TLLM_LOG_INFO("Event ID %d: Block %04x was inserted into the radix tree with parent %04x.", - event.eventId, block.blockHash, prevBlock->hash); - - // This block shouldn't already exist in the tree, and should have tokens associated with it - TLLM_CHECK(blockTable.find(block.blockHash) == blockTable.end()); - TLLM_CHECK(block.tokens.size() > 0); - - auto thisBlock = std::make_shared( - block.blockHash, block.cacheLevel, block.priority, block.loraId, prevBlock, block.cacheSalt); - - blockTable[block.blockHash] = thisBlock; - // Link the parent to the new block - prevBlock->nextBlocks[block.blockHash] = thisBlock; - - prevBlock = thisBlock; - } - } - else if (std::holds_alternative(event.data)) - { - auto const& eventData = std::get(event.data); - - for (auto const& hash : eventData.blockHashes) - { - - TLLM_LOG_INFO("Event ID %d: Block %04x was removed from the radix tree.", event.eventId, hash); - - // This block should exist in the tree - TLLM_CHECK(blockTable.find(hash) != blockTable.end()); - - auto& block = blockTable[hash]; - - // Check that the block has no children, and that the parent has the block listed as a child - TLLM_CHECK(block->nextBlocks.size() == 0); - TLLM_CHECK(block->prevBlock->nextBlocks.find(block->hash) != block->prevBlock->nextBlocks.end()); - - // Remove the block from it's parent, and remove the entry in the block table - block->prevBlock->nextBlocks.erase(block->hash); - blockTable.erase(hash); - } - } - else if (std::holds_alternative(event.data)) - { - auto const& eventData = std::get(event.data); - - if (eventData.priority.has_value()) - { - // The block priority was updated - TLLM_LOG_INFO("Event ID %d: Block %04x priority was changed from %d to %d", event.eventId, - eventData.blockHash, eventData.priority->oldValue, eventData.priority->newValue); - - TLLM_CHECK(blockTable[eventData.blockHash]->priority == eventData.priority->oldValue); - blockTable[eventData.blockHash]->priority = eventData.priority->newValue; - } - - if (eventData.cacheLevel.has_value()) - { - // The block cache level was updated - TLLM_LOG_INFO("Event ID %d: Block %04x cache level was changed from %d to %d", event.eventId, - eventData.blockHash, eventData.cacheLevel->oldValue, eventData.cacheLevel->newValue); - - TLLM_CHECK(blockTable[eventData.blockHash]->cacheLevel == eventData.cacheLevel->oldValue); - blockTable[eventData.blockHash]->cacheLevel = eventData.cacheLevel->newValue; - } - } - else - { - TLLM_LOG_ERROR("Unsupported event type. This shouldn't happen!"); - } - } -} - -tle::Request makeRequest(int sysPromptTokens, int contextTokens, std::uniform_int_distribution sysPromptSelector, - std::normal_distribution maxNumTokensSelector) -{ - int sysPromptVersion = sysPromptSelector(gen); - tle::VecTokens inputTokens; - - // Add `sysPromptTokens` tokens. Add the version to the token ids to create a unique system prompt - for (int i = 0; i < sysPromptTokens; i++) - { - inputTokens.emplace_back(sysPromptVersion + i); - } - // Add random context tokens - for (int i = 0; i < contextTokens; i++) - { - inputTokens.emplace_back(rand() % 1000); - } - - return tle::Request(inputTokens, maxNumTokensSelector(gen)); -} diff --git a/examples/cpp/executor/executorExampleLogitsProcessor.cpp b/examples/cpp/executor/executorExampleLogitsProcessor.cpp deleted file mode 100644 index 0913b77b1775..000000000000 --- a/examples/cpp/executor/executorExampleLogitsProcessor.cpp +++ /dev/null @@ -1,91 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/plugins/api/tllmPlugin.h" - -namespace tlc = tensorrt_llm::common; -namespace tle = tensorrt_llm::executor; - -int main(int argc, char* argv[]) -{ - // Register the TRT-LLM plugins - initTrtLlmPlugins(); - - if (argc != 2) - { - TLLM_LOG_ERROR("Usage: %s ", argv[0]); - return 1; - } - - int constexpr sentinels[] = {42, 29}; - int step = 0; - - auto logitsPostProcessorFn - = [&step, &sentinels](tle::IdType reqId, tle::Tensor& logits, tle::BeamTokens const& tokens, - tle::StreamPtr const& streamPtr, std::optional clientId) - { - auto logitsDataType = logits.getDataType(); - auto logitsCpu = tensorrt_llm::executor::Tensor::cpu(logitsDataType, logits.getShape()); - auto* dataPtr = logitsCpu.getData(); - auto* dataPtrFloat = static_cast(dataPtr); - for (size_t i = 0; i < logitsCpu.getSize(); ++i) - { - dataPtrFloat[i] = -1.0e20; - } - dataPtrFloat[sentinels[step]] = 0.0f; - - logits.setFrom(logitsCpu, streamPtr); - step = (1 - step); - }; - - std::string logitsPostProcessorName = "MyLogitsPP"; - - // Create the executor for this engine - tle::SizeType32 beamWidth = 1; - auto executorConfig = tle::ExecutorConfig(beamWidth); - - auto logitsProcConfig = tle::LogitsPostProcessorConfig(); - logitsProcConfig.setProcessorMap(std::unordered_map{ - {logitsPostProcessorName, logitsPostProcessorFn}}); - executorConfig.setLogitsPostProcessorConfig(logitsProcConfig); - - auto trtEnginePath = argv[1]; - auto executor = tle::Executor(trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); - - // Create the request - tle::SizeType32 maxNewTokens = 5; - tle::VecTokens inputTokens{1, 2, 3, 4}; - auto request = tle::Request(inputTokens, maxNewTokens); - request.setLogitsPostProcessorName(logitsPostProcessorName); - - // Enqueue the request - auto requestId = executor.enqueueRequest(std::move(request)); - - // Wait for the response - auto responses = executor.awaitResponses(requestId); - - // Get outputTokens - auto outputTokens = responses.at(0).getResult().outputTokenIds.at(beamWidth - 1); - - TLLM_LOG_INFO("Output tokens: %s", tlc::vec2str(outputTokens).c_str()); - - return 0; -} diff --git a/examples/cpp/executor/inputTokens.csv b/examples/cpp/executor/inputTokens.csv deleted file mode 100644 index 4cb3974a91b5..000000000000 --- a/examples/cpp/executor/inputTokens.csv +++ /dev/null @@ -1,3 +0,0 @@ -1, 2, 3, 4, 5, 6 -1, 2, 3, 4 -1, 2, 3, 4, 5, 6, 7, 8, 9, 10 diff --git a/examples/cpp_library/CMakeLists.txt b/examples/cpp_library/CMakeLists.txt deleted file mode 100644 index c60ff48d4cde..000000000000 --- a/examples/cpp_library/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. cmake needs this line -cmake_minimum_required(VERSION 3.1) -# cmake_minimum_required(VERSION 2.8) - -# Enable C++11 -set(CMAKE_CXX_STANDARD 14) -set(CMAKE_CXX_STANDARD_REQUIRED TRUE) - -# Define project name -set(TARGET_NAME trt_llm_plugins_cpp_load_example) -project(${TARGET_NAME}) - -set(CMAKE_VERBOSE_MAKEFILE 1) - -# Compile options -set(CMAKE_C_FLAGS "-Wall -pthread ") -set(CMAKE_C_FLAGS_DEBUG "-g -O0") -set(CMAKE_C_FLAGS_RELEASE "-O2") -set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -lstdc++") -set(CMAKE_CXX_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG}) -set(CMAKE_CXX_FLAGS_RELEASE ${CMAKE_C_FLAGS_RELEASE}) - -set(CMAKE_BUILD_TYPE release) -# set(CMAKE_BUILD_TYPE debug) - -find_package(CUDA REQUIRED) -message(STATUS "CUDA library status:") -message(STATUS " config: ${CUDA_DIR}") -message(STATUS " version: ${CUDA_VERSION}") -message(STATUS " libraries: ${CUDA_LIBRARIES}") -message(STATUS " include path: ${CUDA_INCLUDE_DIRS}") - -# Declare the executable target built from your sources -add_executable(${TARGET_NAME} main.cpp) - -# Link your application with CUDA libraries -target_link_libraries(${TARGET_NAME} LINK_PRIVATE ${CUDA_LIBRARIES}) -target_link_libraries(${TARGET_NAME} LINK_PRIVATE cudnn) -target_link_libraries(${TARGET_NAME} LINK_PRIVATE nvinfer) -target_link_libraries(${TARGET_NAME} LINK_PRIVATE nvinfer_plugin_tensorrt_llm) - -target_include_directories(${TARGET_NAME} PUBLIC /usr/local/cuda/include) diff --git a/examples/cpp_library/build.sh b/examples/cpp_library/build.sh deleted file mode 100755 index a384e5cad4a0..000000000000 --- a/examples/cpp_library/build.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -BUILD_DIR="build" -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -rm -rf ${BUILD_DIR} && mkdir -p ${BUILD_DIR} - -pushd ${BUILD_DIR} - -cmake \ - -DCMAKE_BUILD_TYPE=Release \ - .. - -make -j"$(grep -c ^processor /proc/cpuinfo)" - -export LD_LIBRARY_PATH="${SCRIPT_DIR}:${LD_LIBRARY_PATH}" - -# Test Lib -echo -echo "--------------------------------------------------------------------" -./trt_llm_plugins_cpp_load_example -echo "--------------------------------------------------------------------" -echo - -popd diff --git a/examples/cpp_library/main.cpp b/examples/cpp_library/main.cpp deleted file mode 100644 index 7613a75a140c..000000000000 --- a/examples/cpp_library/main.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include - -#include "tensorrt_llm_libutils.h" - -int main(int argc, char* argv[]) -{ - class TRTLogger : public nvinfer1::ILogger - { - public: - void log(nvinfer1::ILogger::Severity severity, char const* msg) noexcept override - { - if (severity <= nvinfer1::ILogger::Severity::kERROR) - std::cerr << "[TensorRT LLM ERR]: " << msg << std::endl; - else if (severity == nvinfer1::ILogger::Severity::kWARNING) - std::cerr << "[TensorRT LLM WARNING]: " << msg << std::endl; - else - std::cout << "[TensorRT LLM LOG]: " << msg << std::endl; - } - }; - - TRTLogger* trtLogger = new TRTLogger(); - - std::string libname = "libtensorrt_llm_plugin.so"; - - /* =============== initLibNvInferPlugins =============== */ - - typedef bool (*initLibNvInferPlugins_sig)(void*, void const*); - - auto initLibNvInferPlugins = getTrtLLMFunction( - /*libFileSoName=*/libname, - /*symbol=*/"initLibNvInferPlugins"); - - std::cout << std::endl; - - std::string libNamespace = "tensorrt_llm"; - char const* libNamespace_cstr = libNamespace.data(); - - bool status1 = initLibNvInferPlugins(trtLogger, libNamespace_cstr); - std::cout << "Success Status: " << status1 << std::endl << std::endl; - - bool status2 = initLibNvInferPlugins(trtLogger, libNamespace_cstr); - std::cout << "Success Status: " << status2 << std::endl; - - /* =============== getInferLibVersion =============== */ - - std::cout << std::endl; - std::cout << "--------------------------------------------------------------------" << std::endl; - - typedef int32_t (*getInferLibVersion_sig)(); - - auto getInferLibVersion = getTrtLLMFunction( - /*libFileSoName=*/libname, - /*symbol=*/"getInferLibVersion"); - - std::cout << std::endl; - - int32_t version = getInferLibVersion(); - std::cout << "Version: " << version << std::endl; - - return 0; -} diff --git a/examples/cpp_library/tensorrt_llm_libutils.h b/examples/cpp_library/tensorrt_llm_libutils.h deleted file mode 100644 index aa60444eefa3..000000000000 --- a/examples/cpp_library/tensorrt_llm_libutils.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#if !defined(_WIN32) -#include -#endif // !defined(_WIN32) -#include -#include -#include - -#include "NvInfer.h" - -template -tSymbolSignature getTrtLLMFunction(std::string libFileSoName, std::string symbol) -{ -#if !defined(_WIN32) - std::cout << "Trying to load " << libFileSoName << " ..." << std::endl; - - // 1. Defining a handle to the library - void* handle = dlopen(libFileSoName.c_str(), RTLD_LAZY | RTLD_GLOBAL); - - // 2. Check for errors - char const* dl_error1 = dlerror(); - if (!handle) - { - throw std::runtime_error("Cannot open library: " + std::string(dl_error1)); - } - - // 3. Load actual queried `symbol` - std::cout << "Loading symbol `" << symbol << "` ..." << std::endl; - - tSymbolSignature symbolFctn = nullptr; - *(void**) (&symbolFctn) = dlsym(handle, symbol.c_str()); - - // 4. Check for errors - char const* dl_error2 = dlerror(); - if (dl_error2) - { - dlclose(handle); - throw std::runtime_error("Cannot load symbol '" + symbol + "': " + std::string(dl_error2)); - } - - return symbolFctn; -#else // on windows - throw std::runtime_error( - "`tSymbolSignature getTrtLLMFunction(std::string, std::string)` is not implemented on Windows."); - return nullptr; -#endif // !defined(_WIN32) -} diff --git a/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm b/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm index 275016c255dd..d73430ba5820 100644 --- a/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm +++ b/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm @@ -101,7 +101,7 @@ elif [ -d "${trtllm_repo}" ]; then if [ "${build_wheel}" = "true" ]; then echo "Building TensorRT-LLM wheel on one node..." - build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache --clean" + build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --use_ccache --clean" if [ -n "${cuda_architectures:-}" ]; then build_command="${build_command} --cuda_architectures \"${cuda_architectures}\"" fi diff --git a/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm b/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm index 8835399d6c50..07015e6473a0 100644 --- a/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm +++ b/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm @@ -101,7 +101,7 @@ elif [ -d "${trtllm_repo}" ]; then if [ "${build_wheel}" = "true" ]; then echo "Building TensorRT-LLM wheel on one node..." - build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache --clean" + build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --use_ccache --clean" if [ -n "${cuda_architectures:-}" ]; then build_command="${build_command} --cuda_architectures \"${cuda_architectures}\"" fi diff --git a/examples/disaggregated/slurm/benchmark/run_benchmark_aiperf.sh b/examples/disaggregated/slurm/benchmark/run_benchmark_aiperf.sh index 1f2d19b18417..385936d67b8b 100755 --- a/examples/disaggregated/slurm/benchmark/run_benchmark_aiperf.sh +++ b/examples/disaggregated/slurm/benchmark/run_benchmark_aiperf.sh @@ -92,11 +92,5 @@ for concurrency in ${concurrency_list}; do echo "Benchmark with concurrency ${concurrency} done" done -# Fetch perf metrics from disagg server -echo "Fetching perf metrics from http://${hostname}:${port}/perf_metrics ..." -curl -s "http://${hostname}:${port}/perf_metrics" > ${log_path}/perf_metrics.json 2>&1 || true -if [ -s "${log_path}/perf_metrics.json" ]; then - echo "Perf metrics saved to ${log_path}/perf_metrics.json" -else - echo "Warning: perf_metrics response was empty or endpoint not available" -fi +# Configure perf_metrics_output_dir on each server to persist per-request JSONL. +echo "Per-request metrics are available in the configured server-side JSONL output." diff --git a/examples/disaggregated/slurm/benchmark/start_worker.sh b/examples/disaggregated/slurm/benchmark/start_worker.sh index bda3dd4aefb1..0c8653893958 100644 --- a/examples/disaggregated/slurm/benchmark/start_worker.sh +++ b/examples/disaggregated/slurm/benchmark/start_worker.sh @@ -11,9 +11,16 @@ numa_bind=${5} log_dir=${6} enable_nsys=${7} config_file=${8} +cuda_devices=${9} # CUDA_VISIBLE_DEVICES selection: # - Default packing (no gpu_map file): each node is dedicated to one -# worker, so SLURM_LOCALID maps directly to the physical GPU id. +# worker, so every rank on the node is given the node's full GPU list +# (passed as ${9} by submit.py) and binds to its own device via +# mapping.local_rank (= rank % gpus_per_node). Exposing the whole node +# is required for intra-node TP custom all-reduce (attention_dp=false / +# TEP): its cudaDeviceCanAccessPeer() topology check must be able to see +# the peer GPUs. Pinning a single GPU per rank only works for DEP +# (attention_dp=true), which has no intra-node TP all-reduce. # - Compact packing (gpu_map file emitted by submit.py): two workers may # share a node and would both see LOCALID=0, so look up the per-worker # gpu_map " " by SLURM_PROCID. srun @@ -28,7 +35,7 @@ if [ -f "${gpu_map_file}" ]; then fi export CUDA_VISIBLE_DEVICES=${gpu_id} else - export CUDA_VISIBLE_DEVICES=${SLURM_LOCALID} + export CUDA_VISIBLE_DEVICES=${cuda_devices} fi # Container runtimes (pyxis/enroot) reset image-defined variables like PATH diff --git a/examples/disaggregated/slurm/benchmark/submit.py b/examples/disaggregated/slurm/benchmark/submit.py index 75c21ffe79cd..ab9b332d20cc 100644 --- a/examples/disaggregated/slurm/benchmark/submit.py +++ b/examples/disaggregated/slurm/benchmark/submit.py @@ -622,12 +622,22 @@ def submit_job(config, log_dir, dry_run): hf.write(f"{host}\n") gm.write(f"{rank} {host} {gpu}\n") rank += 1 + # Compact packing derives CUDA_VISIBLE_DEVICES from gpu_map. + cuda_devices = "none" else: # Default packing: each node is dedicated to one worker, so - # SLURM_LOCALID directly maps to the physical GPU id in - # start_worker.sh (no hostfile/gpu_map needed). + # every rank on that node is given the node's full GPU list and + # binds to its own device via mapping.local_rank + # (= rank % gpus_per_node). Exposing the whole node is required + # for intra-node TP custom all-reduce (attention_dp=false / + # TEP), whose cudaDeviceCanAccessPeer() topology check must see + # the peer GPUs; pinning one GPU per rank only works for DEP. node_list = list(allocation["nodes"].keys()) num_nodes = len(node_list) + # Whole-node ownership means every node carries the same GPU + # layout, so the first node's list applies to all ranks. + gpu_ids = sorted(list(allocation["nodes"].values())[0]) + cuda_devices = ','.join(map(str, gpu_ids)) worker_env = build_worker_environment( worker_config=worker_config, @@ -670,6 +680,7 @@ def submit_job(config, log_dir, dry_run): log_dir, str(profiling_config['nsys_on']).lower(), server_cfg['config_path'], + cuda_devices, f"&> {log_dir}/3_output_{server_type}_{server_id}.log &", ] start_server_cmds.append(" ".join(cmd)) diff --git a/examples/disaggregated/slurm/cache_transceiver_test/README.md b/examples/disaggregated/slurm/cache_transceiver_test/README.md index 14619e4d387c..e99c57dc4fe3 100644 --- a/examples/disaggregated/slurm/cache_transceiver_test/README.md +++ b/examples/disaggregated/slurm/cache_transceiver_test/README.md @@ -64,7 +64,7 @@ resolved_config.json # the validated config the job ran with logs/ctt-.log # batch-level log (stdout+stderr merged) logs/install.log # TensorRT-LLM install log logs/sweep__rank*.log # per-rank logs: transfer START/DONE+verify, UCX_PROTO_INFO -csv//ctx|gen/rank_*_{send,recv}.csv # C++ transceiver timing (Bandwidth(Gbps)) +csv//ctx|gen/_*_{send,recv}.csv # C++ transceiver timing (Bandwidth(Gbps)); renamed to _*_{send,recv}__c.csv per combination csv//ctx/py_*_*.csv # Python transceiver perf log (throughput_mbs) status/sweep_.jsonl # PASS / MISMATCH / TRANSFER_ERROR / TIMEOUT results.json # full results, grouped per combination (longest req_len) @@ -96,7 +96,7 @@ deliverable for tuning your cluster. | Transceiver | Env enabling timing | File | Column (native) | |---|---|---|---| -| C++ (UCX/NIXL) | `TRTLLM_KVCACHE_TIME_OUTPUT_PATH` (set by the driver) | `rank_*_recv.csv` | `Bandwidth(Gbps)` | +| C++ (UCX/NIXL) | `TRTLLM_KVCACHE_TIME_OUTPUT_PATH` (set by the driver) | `_*_recv.csv` (renamed `_*_recv__c.csv` per combination) | `Bandwidth(Gbps)` | | Python (NIXL) | `TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO=1`, `TLLM_KV_TRANSFER_PERF_LOG_FILE` (set by the driver) | `py_*_*.csv` | `throughput_mbs` (MiB/s) | `report.py` normalizes both to **per-GPU GB/s** (bytes, ÷1e9): C++ diff --git a/examples/disaggregated/slurm/cache_transceiver_test/report.py b/examples/disaggregated/slurm/cache_transceiver_test/report.py index 70d2658eab6b..d1cf66203a6d 100644 --- a/examples/disaggregated/slurm/cache_transceiver_test/report.py +++ b/examples/disaggregated/slurm/cache_transceiver_test/report.py @@ -184,7 +184,7 @@ def _mean(values): def _parse_cpp_recv_csvs(csv_dir): - """Return {rid: [per_rank_GBps, ...]} from C++ rank_*_recv.csv files. + """Return {rid: [per_rank_GBps, ...]} from C++ *_recv.csv files. Each rank writes one row per request with a repeating Bandwidth(Gbps) column per transmission; we take the mean transmission bandwidth as that rank's @@ -194,9 +194,10 @@ def _parse_cpp_recv_csvs(csv_dir): per-rank rates with unequal durations would overstate the real throughput. """ per_rid = {} # rid -> list[per-rank mean bw] - # The driver renames each combination's rank_*_recv.csv to rank_*_recv__cl
  • .csv + # C++ writes "__recv.csv" (instanceId is a runtime UUID); + # the driver renames each combination's *_recv.csv to *_recv__cl
  • .csv # (and the un-renamed name may exist for the last iteration). Match both. - for path in glob.glob(os.path.join(csv_dir, "rank_*_recv*.csv")): + for path in glob.glob(os.path.join(csv_dir, "*_recv*.csv")): with open(path) as f: reader = csv.reader(f) header = next(reader, None) @@ -226,7 +227,16 @@ def _parse_cpp_recv_csvs(csv_dir): def _parse_python_csvs(csv_dir): - """Return {rid: [per_rank_GBps, ...]} from perf_logger py_*_*.csv files. + """Return {rid: [per_rank_GBps, ...]} from Python perf_logger task CSVs. + + PerfLogManager names its per-task CSV by priority of the env vars it sees: + with TRTLLM_KVCACHE_TIME_OUTPUT_PATH set (which this harness always sets + for the C++ transceiver), files are "{dir}/{instanceUuid}_{rank}.csv"; + only the legacy TLLM_KV_TRANSFER_PERF_LOG_FILE fallback produces the old + "py_{instanceUuid}_{rank}.csv" prefix. So identify Python task CSVs by + their header columns (unique_rid + throughput_mbs) rather than by file + name -- C++ send/recv CSVs and the gen_transfer_summary CSV have neither + column and are skipped. `throughput_mbs` is recorded for send tasks in MiB/s (perf_logger divides by 1024*1024). We convert MiB/s -> GB/s (`* 1024^2 / 1e9`) so it matches the @@ -234,14 +244,17 @@ def _parse_python_csvs(csv_dir): `_parse_cpp_recv_csvs`. """ per_rid = {} # rid -> list[per-rank throughput GB/s] - for path in glob.glob(os.path.join(csv_dir, "py_*_*.csv")): + for path in glob.glob(os.path.join(csv_dir, "*.csv")): with open(path) as f: reader = csv.DictReader(f) + fields = reader.fieldnames or [] + if "unique_rid" not in fields or "throughput_mbs" not in fields: + continue # not a Python perf_logger task CSV (e.g. C++ send/recv) for row in reader: - if "unique_rid" not in row or "throughput_mbs" not in row: - continue - if "Send" not in row.get("task_type", ""): - continue # throughput is meaningful only for send tasks + if row.get("task_type") != "KVSendTask": + continue # only KV sends carry meaningful throughput; + # AuxSendTask rows are tiny metadata sends that would + # drag the median down, KVRecvTask rows have none try: rid = int(float(row["unique_rid"])) mbs = float(row["throughput_mbs"]) diff --git a/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py b/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py index a54d7b91f3c5..94142ab1274e 100644 --- a/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py +++ b/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py @@ -24,8 +24,9 @@ deterministic, rank-specific pattern, sends it, and the gen side verifies the received blocks regenerate to the same pattern. Bandwidth is emitted by the transceivers themselves into per-rank CSVs (parsed later by report.py): - C++ -> TRTLLM_KVCACHE_TIME_OUTPUT_PATH (rank_*_send.csv / rank_*_recv.csv) - Py -> TLLM_KV_TRANSFER_PERF_LOG_FILE (py_*_*.csv, throughput_mbs) + C++ -> TRTLLM_KVCACHE_TIME_OUTPUT_PATH (_*_send.csv / _*_recv.csv) + Py -> same env var (PerfLogManager gives it top priority): + _.csv, throughput_mbs column This driver mirrors the single-process test (tests/unittest/others/ test_kv_cache_transceiver.py) and the multi-process Python test @@ -468,15 +469,19 @@ def _preserve_cpp_csvs(csv_dir, ci, rank): request lengths of a combination and appends a row per request, so we move the whole combination's output aside (rid encodes req_len). - Each rank touches ONLY its own files: all ranks share `csv_dir`, so a glob - over `rank_*` would race -- multiple ranks renaming the same file, leaving - some with FileNotFoundError, crashing those ranks and deadlocking the rest on - the next case's collective KVCacheManager allreduce. + C++ names files "__.csv" (instanceId is a runtime + UUID). Each rank touches ONLY files carrying its own "__.csv" + suffix: all ranks share `csv_dir`, so matching a broader pattern would race + -- multiple ranks renaming the same file, leaving some with + FileNotFoundError, crashing those ranks and deadlocking the rest on the next + case's collective KVCacheManager allreduce. """ for tag in ("send", "recv"): - path = os.path.join(csv_dir, f"rank_{rank}_{tag}.csv") - if os.path.exists(path): - os.replace(path, os.path.join(csv_dir, f"rank_{rank}_{tag}__c{ci}.csv")) + suffix = f"_{rank}_{tag}.csv" + for name in os.listdir(csv_dir): + if name.endswith(suffix) and "__c" not in name: + base = name[: -len(".csv")] + os.replace(os.path.join(csv_dir, name), os.path.join(csv_dir, f"{base}__c{ci}.csv")) def main(): @@ -514,12 +519,12 @@ def main(): work_dir = cfg["environment"]["work_dir"] csv_dir = os.path.join(work_dir, "csv", str(sweep), role) os.makedirs(csv_dir, exist_ok=True) + # One env var drives both transceivers: C++ caches it on first read, and + # Python's PerfLogManager gives it top priority (enabling perf logging and + # writing task CSVs as _.csv in csv_dir; the + # per-transceiver UUID avoids cross-combination collisions, and report.py + # identifies these files by header columns, not name). os.environ["TRTLLM_KVCACHE_TIME_OUTPUT_PATH"] = csv_dir - # Python perf logging (singleton reads these once; C++ ignores them). Set at - # startup so it is enabled regardless of combination ordering. Per-transceiver UUID - # filenames (py__.csv) avoid cross-combination collisions. - os.environ["TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO"] = "1" - os.environ["TLLM_KV_TRANSFER_PERF_LOG_FILE"] = os.path.join(csv_dir, "py") status_dir = os.path.join(work_dir, "status") os.makedirs(status_dir, exist_ok=True) diff --git a/examples/dora/README.md b/examples/dora/README.md deleted file mode 100644 index 21f70768a257..000000000000 --- a/examples/dora/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# DoRA - -This document shows how to run a model using DoRA adapters. -DoRA is a PEFT strategy extending LoRA. It is fully supported in the Huggingface `peft` library. For a more detailed description please refer to the DoRA [paper](https://arxiv.org/abs/2402.09353) or official [repo](https://github.com/NVlabs/DoRA). - -## Support Matrix - * FP16/BF16 (over arbitrary precision of the base model). - * Supports adapters from Huggingface `peft` or from the official NVlabs [checkpoints](https://huggingface.co/sliuau/DoRA-weights). - * Multiple adapters (+ mixed LoRA/DoRA setups). - * inflight loading of new adapters to a preloaded base model. - * C++ and python runtime. - * Tensor parallelism and Pipeline parallelism. - -## Usage -Using DoRA is almost exactly the same as using LoRA in TRTLLM, with an additional preprocessing step. -While the official DoRA paper describes the magnitude normalization as part of the execution flow, it can be performed once beforehand to boost inference performance. - -Start by obtaining a local copy of your desired DoRA adapter **and** your base model. We'll use the official NVlabs checkpoint for LLaMA3-8B as an example: - -``` bash -git clone https://huggingface.co/sliuau/DoRA-weights -git clone https://huggingface.co/meta-llama/Meta-Llama-3-8B -``` - -Next, use the [normalize_weights.py](./normalize_weights.py) script to normalize the DoRA magnitude vectors in the adapter checkpoint. -The script requires access to both the local adapter weights and the local base model weights: - -``` bash -export NORMALIZED_DORA_ADAPTER=path/to/normalized/adapter/ckpt - -python ./normalize_weights.py -i DoRA-weights/llama_dora_commonsense_checkpoints/LLama3-8B/dora_r32 -b Meta-Llama-3-8B -o $NORMALIZED_DORA_ADAPTER -``` - -The script will create a new adapter checkpoint, with normalized DoRA vectors, in the provided path. - -Now we may convert our Llama checkpoint and build our TRT engine as described in the Llama [examples](../models/core/llama/README.md). When doing so, ensure you pass `--dora_plugin=enable` to the `trtllm-build` command, as well as enabling the lora plugin: - -``` bash -export CHECKPOINT_DIR=path/to/trtllm/ckpt -export ENGIRE_DIR=path/to/trtllm/engine - -python ../models/core/llama/convert_checkpoint.py --model_dir Meta-Llama-3-8B \ - --output_dir $CHECKPOINT_DIR \ - --dtype float16 - -trtllm-build --checkpoint_dir $CHECKPOINT_DIR \ - --output_dir $ENGINE_DIR \ - --gemm_plugin=auto \ - --lora_plugin=auto \ - --dora_plugin=enable \ - --lora_dir $NORMALIZED_DORA_ADAPTER -``` - -If you wish, you may provide additional LoRA / DoRA adapters to `trtllm-build`. - -**NOTE**: if you omit `--dora_plugin=enable`, you will not receive any warning even if you provide a DoRA adapter to `--lora_dir`. In such a case the DoRA magnitudes will simply be ignored during inference and you may receive wrong output. - -Proceed to execute the engine as you would a normal LoRA engine: - -``` bash -python ../run.py --engine_dir $ENGINE_DIR --tokenizer_dir Meta-Llama-3-8B --lora_task_uids 0 --max_output_len 32 --input_text ... -``` - -## Usage with Triton Server -Using DoRA over Triton is the same as using LoRA, but before using [hf_lora_convert.py](../hf_lora_convert.py), make sure you call [normalize_weights.py](./normalize_weights.py) and use the resulting normalized adapter. diff --git a/examples/dora/normalize_weights.py b/examples/dora/normalize_weights.py deleted file mode 100755 index 40d5ae172431..000000000000 --- a/examples/dora/normalize_weights.py +++ /dev/null @@ -1,289 +0,0 @@ -#! /usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This script applies preprocessing to the DoRA magnitude vector to speed up inference. -DoRA applies columnwise normalization and scaling to the LoRA output. -By applying the normalization to the scaling vector, we can skip calculating the normalization vector at inference time. -""" -import abc -import enum -import json -from pathlib import Path - -import numpy as np -import safetensors.torch as st -import torch - -StateDict = dict[str, torch.Tensor] - -PEFT_MODULE_PREFIX = "base_model.model." -PEFT_MODULE_SUFFIXES = [".lora_A.weight", ".lora_B.weight"] -DORA_VECTOR_SUFFIXES = [ - ".lora_magnitude_vector", # HF peft - ".weight_m_wdecomp.weight" # NVLabs -] - - -class HFWeightsReader(abc.ABC): - - @abc.abstractmethod - def __init__(self, model_dir: str) -> None: - ... - - @abc.abstractmethod - def get_weight(self, weight_name: str) -> torch.Tensor: - ... - - @abc.abstractmethod - def get_all(self) -> StateDict: - ... - - -class HFSafeTensorsReader(HFWeightsReader): - - def __init__(self, model_dir: str) -> None: - self.model_dir = Path(model_dir) - - self._fds = [ - st.safe_open(f, framework="torch") - for f in self.model_dir.glob("*.safetensors") - ] - - self._weight_to_fd = {} - for f in self._fds: - for k in f.keys(): - self._weight_to_fd[k] = f - - def get_weight(self, weight_name: str) -> torch.Tensor: - return self._weight_to_fd[weight_name].get_tensor(weight_name) - - def get_all(self) -> StateDict: - return {k: self.get_weight(k) for k in self._weight_to_fd.keys()} - - -class HFBinWeightsReader(HFWeightsReader): - - def __init__(self, model_dir: str) -> None: - self.model_dir = Path(model_dir) - - self._weights = {} - - for f in self.model_dir.glob("*.bin"): - self._weights.update( - torch.load(f, weights_only=True, mmap=True, map_location="cpu")) - - def get_weight(self, weight_name: str) -> torch.Tensor: - weight_name = f"{weight_name}.weight" - return self._weights[weight_name] - - def get_all(self) -> StateDict: - return self._weights - - -class WeightsFormat(enum.Enum): - BINARY = enum.auto() - SAFETENSORS = enum.auto() - UNKNOWN = enum.auto() - - def __str__(self) -> str: - if self == self.BINARY: - return "bin" - elif self == self.SAFETENSORS: - return "safetensors" - return "unknown" - - -def deduce_weights_format(model_dir: str) -> WeightsFormat: - model_dir_p = Path(model_dir) - - if any(model_dir_p.glob("*.safetensors")): - return WeightsFormat.SAFETENSORS - elif any(model_dir_p.glob("*.bin")): - return WeightsFormat.BINARY - return WeightsFormat.UNKNOWN - - -def get_weights_reader(model_dir: str) -> HFWeightsReader: - model_dir_p = Path(model_dir) - - if not model_dir_p.is_dir(): - raise ValueError( - f"{model_dir} is not a valid model directory: not found") - - weights_format = deduce_weights_format(model_dir) - - if weights_format == WeightsFormat.SAFETENSORS: - return HFSafeTensorsReader(model_dir) - elif weights_format == WeightsFormat.BINARY: - return HFBinWeightsReader(model_dir) - else: - raise ValueError( - f"{model_dir} does not contain .safetensors or .bin weights") - - -def normalize_hf_peft_module_name(module_name: str) -> str: - """ - Remove parts of the module name in the peft adapter to derive the module name of the base model. - """ - - if module_name.startswith(PEFT_MODULE_PREFIX): - module_name = module_name[len(PEFT_MODULE_PREFIX):] - - for suffix in PEFT_MODULE_SUFFIXES + DORA_VECTOR_SUFFIXES: - if module_name.endswith(suffix): - module_name = module_name[:-len(suffix)] - - return module_name - - -def get_peft_module_names(base_module_name: str) -> tuple[str, ...]: - """ - Convert the name of a base module to the names of its LoRA A and LoRA B weights. - """ - return tuple([ - f"{PEFT_MODULE_PREFIX}{base_module_name}{suffix}" - for suffix in PEFT_MODULE_SUFFIXES - ]) - - -def get_dora_magnitude_names(base_module_name: str) -> tuple[str, ...]: - """ - Convert the name of a base module to the potential names of its DoRA magnitude vectors. - """ - return tuple([ - f"{PEFT_MODULE_PREFIX}{base_module_name}{suffix}" - for suffix in DORA_VECTOR_SUFFIXES - ]) - - -def normalize_dora_vector(W: torch.Tensor, A: torch.Tensor, B: torch.Tensor, - mag: torch.Tensor, scale: float) -> torch.Tensor: - return mag / torch.linalg.norm(W + scale * B @ A, dim=1).to(W.dtype) - - -def normalize_dora_scales(lora_sd: StateDict, - weights_reader: HFWeightsReader, - alpha: float, - use_rslora: bool, - strip: bool = False) -> StateDict: - out_sd = {} - - while lora_sd: - # take some lora weight name - module_name = next(iter(lora_sd.keys())) - base_module_name = normalize_hf_peft_module_name(module_name) - A_name, B_name = get_peft_module_names(base_module_name) - magnitude_names = get_dora_magnitude_names(base_module_name) - - if module_name not in [A_name, B_name] + list(magnitude_names): - raise ValueError(f"Encountered unknown weight: {module_name}") - - # get lora weights - A = lora_sd.pop(A_name) - B = lora_sd.pop(B_name) - for name in magnitude_names: - if name in lora_sd: - mag_name = name - mag = lora_sd.pop(mag_name).view(-1) - break - else: - mag_name = "" - mag = None - - out_sd[A_name] = A.contiguous() - out_sd[B_name] = B.contiguous() - - if mag is not None and not strip: - # get base weight and normalize - W = weights_reader.get_weight(base_module_name + ".weight") - - adapter_size = A.size(0) - - if use_rslora: - scale = alpha / np.sqrt(adapter_size) - else: - scale = alpha / adapter_size - - mag = normalize_dora_vector(W, A, B, mag, scale) - out_sd[mag_name] = mag.contiguous() - - return out_sd - - -def save_state_dict(out_file: str, sd: StateDict) -> None: - out_path = Path(out_file) - - if out_path.suffix == ".safetensors": - st.save_file(sd, out_path) - elif out_path.suffix == ".bin": - torch.save(sd, out_path) - else: - raise ValueError(f"Unregornized weights format: {out_path.suffix}") - - -def normalize_peft_ckpt(model_dir: str, - base_model_dir: str, - out_dir: str, - strip: bool = False) -> None: - out_path = Path(out_dir) - out_path.mkdir(parents=True, exist_ok=True) - - weights_reader = get_weights_reader(base_model_dir) - lora_sd = get_weights_reader(model_dir).get_all() - - adapter_config_path = Path(f"{model_dir}/adapter_config.json") - with adapter_config_path.open() as f: - adapter_config = json.load(f) - - alpha = adapter_config["lora_alpha"] - use_rslora = adapter_config.get("use_rslora", False) - - new_sd = normalize_dora_scales(lora_sd, weights_reader, alpha, use_rslora, - strip) - - with (out_path / "adapter_config.json").open("w") as f: - json.dump(adapter_config, f) - - weights_format = deduce_weights_format(model_dir) - save_state_dict( - (out_path / f"adapter_model.{str(weights_format)}").as_posix(), new_sd) - - -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser() - parser.add_argument( - '--out-dir', - '-o', - type=Path, - help='path to output adapter weights with normalized DoRA vectors', - required=True) - parser.add_argument('--in-dir', - '-i', - type=Path, - help='path to input lora checkpoint file', - required=True) - parser.add_argument("--base-model", - "-b", - help="Path to base model", - required=True) - parser.add_argument("--strip", - action="store_true", - help="remove DoRA vectors entirely") - - args = parser.parse_args() - - normalize_peft_ckpt(args.in_dir, args.base_model, args.out_dir, args.strip) diff --git a/examples/eval_long_context.py b/examples/eval_long_context.py deleted file mode 100644 index 90b7ef2dd270..000000000000 --- a/examples/eval_long_context.py +++ /dev/null @@ -1,331 +0,0 @@ -# MIT License - -# Copyright (c) 2023 OpenBMB - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# reference: https://github.com/OpenBMB/InfiniteBench/blob/main/src/eval_yarn_mistral.py - -import argparse -import ast -import json -from pathlib import Path - -import torch -from infinitebench.compute_scores import compute_scores -from infinitebench.eval_utils import (DATA_NAME_TO_MAX_NEW_TOKENS, - create_prompt, dump_jsonl, get_answer, - load_data) -from utils import (DEFAULT_HF_MODEL_DIRS, DEFAULT_PROMPT_TEMPLATES, - add_common_args, load_tokenizer, read_model_name) - -import tensorrt_llm -import tensorrt_llm.profiler as profiler -from tensorrt_llm.logger import logger -from tensorrt_llm.runtime import PYTHON_BINDINGS, ModelRunner - -if PYTHON_BINDINGS: - from tensorrt_llm.runtime import ModelRunnerCpp - -MAX_POSITION_ID = 128 * 1024 # Determined by the model -TRUNCATE_LEN = 128 * 1024 - - -def parse_arguments(args=None): - parser = argparse.ArgumentParser() - parser.add_argument('--batch_size', type=int, default=1) - parser.add_argument('--max_input_length', type=int, default=923) - parser.add_argument('--output_log_probs_npy', - type=str, - help='Numpy file where the log_probs are stored', - default=None) - - parser.add_argument('--output_cum_log_probs_npy', - type=str, - help='Numpy file where the cum_log_probs are stored', - default=None) - - parser.add_argument( - "--task", - type=str, - choices=['passkey', 'kv_retrieval'], - required=True, - help= - "Which task to use. Note that \"all\" can only be used in `compute_scores.py`.", # noqa - ) - parser.add_argument('--data_dir', - type=str, - default='./', - help="The directory of data.") - parser.add_argument("--output_dir", - type=str, - default=None, - help="Where to dump the prediction results.") # noqa - parser.add_argument( - "--start_idx", - type=int, - default=0, - help= - "The index of the first example to infer on. This is used if you want to evaluate on a (contiguous) subset of the data." - ) # noqa - parser.add_argument( - "--stop_idx", - type=int, - help= - "The index of the last example to infer on. This is used if you want to evaluate on a (contiguous) subset of the data. Defaults to the length of dataset." - ) # noqa - parser.add_argument('--tensorrt_llm_accuracy_threshold', - type=float, - default=99) - parser = add_common_args(parser) - - return parser.parse_args(args=args) - - -def parse_input(tokenizer, - input_text=None, - prompt_template=None, - add_special_tokens=True, - max_input_length=923, - pad_id=None, - num_prepend_vtokens=[], - model_name=None, - model_version=None): - if pad_id is None: - pad_id = tokenizer.pad_token_id - - batch_input_ids = [] - for curr_text in input_text: - if prompt_template is not None: - curr_text = prompt_template.format(input_text=curr_text) - input_ids = tokenizer.encode(curr_text, - add_special_tokens=add_special_tokens, - truncation=True, - max_length=max_input_length) - batch_input_ids.append(input_ids) - - if num_prepend_vtokens: - assert len(num_prepend_vtokens) == len(batch_input_ids) - base_vocab_size = tokenizer.vocab_size - len( - tokenizer.special_tokens_map.get('additional_special_tokens', [])) - for i, length in enumerate(num_prepend_vtokens): - batch_input_ids[i] = list( - range(base_vocab_size, - base_vocab_size + length)) + batch_input_ids[i] - - if 'GLM' in model_name and model_version == 'glm': - for ids in batch_input_ids: - ids.append(tokenizer.sop_token_id) - - batch_input_ids = [ - torch.tensor(x, dtype=torch.int32) for x in batch_input_ids - ] - return batch_input_ids - - -def main(args): - # model_name = "yarn-mistral" - runtime_rank = tensorrt_llm.mpi_rank() - logger.set_level(args.log_level) - - print(json.dumps(vars(args), indent=4)) - data_name = args.task - - # Model - max_tokens = DATA_NAME_TO_MAX_NEW_TOKENS[data_name] - - model_name, model_version = read_model_name(args.engine_dir) - if args.tokenizer_dir is None: - logger.warning( - "tokenizer_dir is not specified. Try to infer from model_name, but this may be incorrect." - ) - args.tokenizer_dir = DEFAULT_HF_MODEL_DIRS[model_name] - - tokenizer, pad_id, end_id = load_tokenizer( - tokenizer_dir=args.tokenizer_dir, - vocab_file=args.vocab_file, - model_name=model_name, - model_version=model_version, - tokenizer_type=args.tokenizer_type, - ) - - if not PYTHON_BINDINGS and not args.use_py_session: - logger.warning( - "Python bindings of C++ session is unavailable, fallback to Python session." - ) - args.use_py_session = True - if args.debug_mode and not args.use_py_session: - logger.warning( - "Debug mode is not supported in C++ session for now, fallback to Python session." - ) - args.use_py_session = True - runner_cls = ModelRunner if args.use_py_session else ModelRunnerCpp - runner_kwargs = dict( - engine_dir=args.engine_dir, - lora_dir=args.lora_dir, - rank=runtime_rank, - debug_mode=args.debug_mode, - lora_ckpt_source=args.lora_ckpt_source, - gpu_weights_percent=args.gpu_weights_percent, - ) - if args.medusa_choices is not None: - args.medusa_choices = ast.literal_eval(args.medusa_choices) - assert args.temperature == 1.0, "Medusa should use temperature == 1.0" - assert args.num_beams == 1, "Medusa should use num_beams == 1" - runner_kwargs.update(medusa_choices=args.medusa_choices) - if not args.use_py_session: - runner_kwargs.update( - max_batch_size=args.batch_size, - max_input_len=args.max_input_length, - max_output_len=max_tokens, - max_beam_width=args.num_beams, - max_attention_window_size=args.max_attention_window_size, - sink_token_length=args.sink_token_length, - max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, - kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, - kv_cache_free_gpu_memory_fraction=args. - kv_cache_free_gpu_memory_fraction, - enable_chunked_context=args.enable_chunked_context, - ) - runner = runner_cls.from_dir(**runner_kwargs) - - # Data - examples = load_data(data_name, data_dir=args.data_dir) - if args.stop_idx is None: - args.stop_idx = len(examples) - - output_path = None - if runtime_rank == 0: - if args.output_dir is not None: - result_dir = Path(args.output_dir, model_name) - result_dir.mkdir(exist_ok=True, parents=True) - - if args.stop_idx is None: - output_path = (result_dir / f"preds_{data_name}.jsonl") - else: - output_path = ( - result_dir / - f"preds_{data_name}_{args.start_idx}-{args.stop_idx}.jsonl" # noqa - ) - - prompt_template = None - if args.use_prompt_template and model_name in DEFAULT_PROMPT_TEMPLATES: - prompt_template = DEFAULT_PROMPT_TEMPLATES[model_name] - - if runtime_rank == 0: - preds = [] - logger.info("==== Evaluation ====") - logger.info(f"# examples: {len(examples)}") - logger.info(f"Start index: {args.start_idx}") - logger.info(f"Stop index: {args.stop_idx}") - logger.info(f"Max tokens: {max_tokens}") - assert args.batch_size == 1 - profiler.start('Evaluation') - for i in range(args.start_idx, args.stop_idx): - eg = examples[i] - input_text = [create_prompt(eg, data_name, args.data_dir)] - batch_input_ids = parse_input( - tokenizer=tokenizer, - input_text=input_text, - prompt_template=prompt_template, - add_special_tokens=args.add_special_tokens, - max_input_length=args.max_input_length, - pad_id=pad_id, - num_prepend_vtokens=args.num_prepend_vtokens, - model_name=model_name, - model_version=model_version) - input_lengths = [x.size(0) for x in batch_input_ids] - - if runtime_rank == 0: - logger.debug(f"====== Example {i} ======") - logger.debug(f"input_lengths: {input_lengths}") - logger.debug(f"input_text: {input_text}") - logger.debug(f"answer: {get_answer(eg, data_name)}") - outputs = runner.generate( - batch_input_ids, - max_new_tokens=max_tokens, - max_attention_window_size=args.max_attention_window_size, - sink_token_length=args.sink_token_length, - end_id=end_id, - pad_id=pad_id, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - length_penalty=args.length_penalty, - early_stopping=args.early_stopping, - beam_width_array=args.beam_width_array, - repetition_penalty=args.repetition_penalty, - presence_penalty=args.presence_penalty, - frequency_penalty=args.frequency_penalty, - prompt_ignore_length=args.prompt_ignore_length, - # stop_words_list=stop_words_list, - # bad_words_list=bad_words_list, - output_cum_log_probs=(args.output_cum_log_probs_npy != None), - output_log_probs=(args.output_log_probs_npy != None), - lora_uids=args.lora_task_uids, - prompt_table=args.prompt_table_path, - prompt_tasks=args.prompt_tasks, - streaming=args.streaming, - output_sequence_lengths=True, - return_dict=True, - medusa_choices=args.medusa_choices) - torch.cuda.synchronize() - if runtime_rank == 0: - output_ids = outputs['output_ids'] - output_beams_list = [ - tokenizer.batch_decode(output_ids[batch_idx, :, - input_lengths[batch_idx]:], - skip_special_tokens=True) - for batch_idx in range(args.batch_size) - ] - - logger.debug(f"preds: {output_beams_list[0]}") - preds.append({ - "id": i, - "prediction": output_beams_list[0][0], - "ground_truth": get_answer(eg, data_name), - "input_lengths": input_lengths, - }) - if output_path is not None: - dump_jsonl(preds, output_path) - profiler.stop('Evaluation') - - if runtime_rank == 0: - logger.info( - f'Evaluation takes: {profiler.elapsed_time_in_sec("Evaluation")} sec.' - ) - logger.info("Compute the score") - acc = compute_scores(preds, args.task) * 100 - logger.info(f"{args.task} accuracy: {acc:.2f} ({len(preds)})") - - if args.tensorrt_llm_accuracy_threshold is not None: - assert acc >= args.tensorrt_llm_accuracy_threshold, f"acc ({acc}) < tensorrt_llm_accuracy_threshold ({args.tensorrt_llm_accuracy_threshold})" - - -if __name__ == "__main__": - args = parse_arguments() - main(args) diff --git a/examples/generate_checkpoint_config.py b/examples/generate_checkpoint_config.py deleted file mode 100644 index a11104eeba68..000000000000 --- a/examples/generate_checkpoint_config.py +++ /dev/null @@ -1,149 +0,0 @@ -import argparse -import json -import os - -from tensorrt_llm.quantization import KV_CACHE_QUANT_ALGO_LIST, QUANT_ALGO_LIST - - -def parse_arguments(): - parser = argparse.ArgumentParser() - - parser.add_argument( - '--output_path', - type=str, - default='config.json', - help='The path to save the TensorRT LLM checkpoint config.json file') - parser.add_argument('--architecture', type=str, default='GPTForCausalLM') - parser.add_argument('--dtype', - type=str, - default='float16', - choices=['float32', 'bfloat16', 'float16']) - parser.add_argument('--vocab_size', type=int, default=32000) - parser.add_argument('--max_position_embeddings', type=int, default=1024) - parser.add_argument('--hidden_size', type=int, default=768) - parser.add_argument('--intermediate_size', type=int, default=None) - parser.add_argument('--num_hidden_layers', type=int, default=12) - parser.add_argument('--num_attention_heads', type=int, default=12) - parser.add_argument('--num_key_value_heads', type=int, default=None) - parser.add_argument('--hidden_act', type=str, default='gelu') - parser.add_argument('--norm_epsilon', type=float, default=1e-5) - parser.add_argument('--position_embedding_type', - type=str, - default='learned_absolute') - parser.add_argument( - '--use_parallel_embedding', - action='store_true', - default=False, - help= - 'By default embedding parallelism is disabled. By setting this flag, embedding parallelism is enabled' - ) - parser.add_argument( - '--embedding_sharding_dim', - type=int, - default=0, - choices=[0, 1], - help= - 'By default the embedding lookup table is sharded along vocab dimension (embedding_sharding_dim=0). ' - 'To shard it along hidden dimension, set embedding_sharding_dim=1' - 'Note: embedding sharing is only enabled when embedding_sharding_dim = 0' - ) - - parser.add_argument('--tp_size', - type=int, - default=1, - help='N-way tensor parallelism size') - parser.add_argument('--pp_size', - type=int, - default=1, - help='N-way pipeline parallelism size') - - parser.add_argument('--quant_algo', - type=str, - default=None, - choices=[None] + QUANT_ALGO_LIST) - parser.add_argument('--kv_cache_quant_algo', - type=str, - default=None, - choices=[None] + KV_CACHE_QUANT_ALGO_LIST) - parser.add_argument('--group_size', type=int, default=64) - parser.add_argument('--smoothquant_val', type=float, default=None) - parser.add_argument('--has_zero_point', default=False, action='store_true') - parser.add_argument('--pre_quant_scale', default=False, action='store_true') - parser.add_argument('--exclude_modules', nargs='+', default=None) - - parser.add_argument('--bias', default=False, action='store_true') - parser.add_argument('--apply_query_key_layer_scaling', - default=False, - action='store_true') - parser.add_argument('--rotary_pct', type=float, default=1.0) - parser.add_argument('--rotary_base', type=float, default=10000.0) - parser.add_argument('--rotary_scaling', nargs=2, type=str, default=None) - - args = parser.parse_args() - return args - - -if __name__ == '__main__': - args = parse_arguments() - world_size = args.tp_size * args.pp_size - - assert args.output_path.endswith('.json') - output_dir = os.path.dirname(args.output_path) - if output_dir and not os.path.exists(output_dir): - os.makedirs(output_dir) - - config = { - 'architecture': args.architecture, - 'dtype': args.dtype, - 'vocab_size': args.vocab_size, - 'max_position_embeddings': args.max_position_embeddings, - 'hidden_size': args.hidden_size, - 'intermediate_size': args.intermediate_size, - 'num_hidden_layers': args.num_hidden_layers, - 'num_attention_heads': args.num_attention_heads, - 'num_key_value_heads': args.num_key_value_heads, - 'hidden_act': args.hidden_act, - 'norm_epsilon': args.norm_epsilon, - 'position_embedding_type': args.position_embedding_type, - 'use_parallel_embedding': args.use_parallel_embedding, - 'embedding_sharding_dim': args.embedding_sharding_dim, - 'quantization': { - 'quant_algo': args.quant_algo, - 'kv_cache_quant_algo': args.kv_cache_quant_algo, - 'exclude_modules': args.exclude_modules, - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': args.tp_size, - 'pp_size': args.pp_size, - }, - 'bias': args.bias, - 'apply_query_key_layer_scaling': args.apply_query_key_layer_scaling, - 'rotary_pct': args.rotary_pct, - 'rotary_base': args.rotary_base, - 'rotary_scaling': args.rotary_scaling, - } - - if args.intermediate_size is None: - config['intermediate_size'] = args.hidden_size * 4 - if args.num_key_value_heads is None: - config['num_key_value_heads'] = args.num_attention_heads - - if args.quant_algo is not None: - if 'AWQ' in args.quant_algo or 'GPTQ' in args.quant_algo: - config['quantization'].update({ - 'group_size': - args.group_size, - 'has_zero_point': - args.has_zero_point, - 'pre_quant_scale': - args.pre_quant_scale, - }) - if 'SQ' in args.quant_algo: - config['quantization'].update({ - 'smoothquant_val': - args.smoothquant_val, - }) - - with open(args.output_path, 'w') as f: - json.dump(config, f, indent=4) diff --git a/examples/generate_xgrammar_tokenizer_info.py b/examples/generate_xgrammar_tokenizer_info.py deleted file mode 100644 index 67e05eb9c968..000000000000 --- a/examples/generate_xgrammar_tokenizer_info.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import json -import os -from pathlib import Path - -from transformers import AutoTokenizer - -from tensorrt_llm.llmapi.tokenizer import _xgrammar_tokenizer_info - - -def generate_xgrammar_tokenizer_info(args): - - tokenizer = AutoTokenizer.from_pretrained(str(args.model_dir)) - tokenizer_info = _xgrammar_tokenizer_info(tokenizer) - - os.makedirs(args.output_dir, exist_ok=True) - with open(str(args.output_dir / "xgrammar_tokenizer_info.json"), 'w') as f: - json.dump(tokenizer_info, f) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--model_dir', - type=Path, - default=None, - required=True, - help="HF model directory") - parser.add_argument( - '--output_dir', - type=Path, - default=None, - required=True, - help="File path to save xgrammar's info. in json format") - args = parser.parse_args() - generate_xgrammar_tokenizer_info(args) diff --git a/examples/hf_lora_convert.py b/examples/hf_lora_convert.py deleted file mode 100755 index 019d78a48563..000000000000 --- a/examples/hf_lora_convert.py +++ /dev/null @@ -1,266 +0,0 @@ -#! /usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import datetime -import json -import logging -import re -from collections import defaultdict -from pathlib import Path - -import numpy as np -import torch - -from tensorrt_llm._utils import str_dtype_to_torch, torch_to_numpy -from tensorrt_llm.lora_manager import LoraManager -from tensorrt_llm.models.convert_utils import get_model_path, load_state_dict - -log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s" -logging.basicConfig(format=log_format) -LOGGER = logging.getLogger(__name__) - - -def save_val(val, dir, key, tp_num=None, write_npy=False): - ext = "npy" if write_npy else "bin" - suffix = ext if tp_num is None else f"{tp_num}.{ext}" - if write_npy: - np.save(dir / f"model.{key}.{suffix}", val) - else: - val.tofile(dir / f"model.{key}.{suffix}") - - -def get_all_lora_weights(lora_weights): - all_weights = defaultdict(lambda: defaultdict(dict)) - pattern = re.compile( - r'(.*\.layers\.([0-9]+)\.(self_attn|mlp)\.([a-z_]+))\.(?:lora_(?:(A|B)\.weight|(magnitude)_vector)|weight_(m_wdecomp).weight).*' - ) - moe_pattern = re.compile( - r'(.*\.layers\.([0-9]+)\.(block_sparse_moe)\.((experts)\.([0-9]+)\.|)([a-zA-Z0-9_]+))\.(?:lora_(?:(A|B)\.weight|(magnitude)_vector)|weight_(m_wdecomp).weight).*' - ) - for key, weights in lora_weights.items(): - m = pattern.match(key) - m_moe = moe_pattern.match(key) - if m: - layer_idx = int(m.group(2)) - hf_module = m.group(4) - inout = m.group(5) - dora_magnitude = m.group(6) or m.group(7) - - if inout: - inout = "in" if inout == "A" else "out" - all_weights[layer_idx][hf_module][inout] = weights - elif dora_magnitude: - LOGGER.warning( - "Detected DoRA magnitude vector, make sure it was preprocessed and normalized using the proper base model weights" - ) - all_weights[layer_idx][hf_module]["magnitude"] = weights.view( - -1) - - elif m_moe: - layer_idx = int(m_moe.group(2)) - hf_module = m_moe.group(7) - inout = m_moe.group(8) - dora_magnitude = m_moe.group(9) or m.group(10) - - if inout: - inout = "in" if inout == "A" else "out" - all_weights[layer_idx][hf_module][inout] = weights - elif dora_magnitude: - LOGGER.warning( - "Detected DoRA magnitude vector, make sure it was preprocessed and normalized using the proper base model weights" - ) - all_weights[layer_idx][hf_module]["magnitude"] = weights.view( - -1) - else: - print(f"no match {key}") - continue - return all_weights - - -def preprocess_lora_weights(lora_model): - # Swap weights of gate_up_proj - for key, value in lora_model.items(): - if "gate_up_proj.lora_B.weight" in key: - print("Swap {}".format(key)) - original_weights = value.contiguous().clone() - half_split = original_weights.shape[0] // 2 - first_half = original_weights[:half_split, :] - second_half = original_weights[half_split:, :] - value = torch.cat((second_half, first_half), dim=0) - lora_model[key] = value - return lora_model - - -hf_modules_to_trtllm_modules = { - "q_proj": "attn_q", - "v_proj": "attn_v", - "k_proj": "attn_k", - "qkv_proj": "attn_qkv", - "query_key_value": "attn_qkv", - "o_proj": "attn_dense", - "dense": "attn_dense", - "gate_proj": "mlp_h_to_4h", - "down_proj": "mlp_4h_to_h", - "up_proj": "mlp_gate", - "gate_up_proj": "mlp_h_to_4h", - "c_fc": "mlp_h_to_4h", - "c_proj": "mlp_4h_to_h", - "w1": "moe_h_to_4h", - "w2": "moe_4h_to_h", - "w3": "moe_gate", - "gate": "moe_router", -} # lora modules on llama -hf_modules_to_module_id = { - k: LoraManager.LORA_MODULE_IDS[v] - for k, v in hf_modules_to_trtllm_modules.items() -} - - -def convert_hf_model(model_dir, dtype, out_dir): - saved_dir = Path(out_dir) - saved_dir.mkdir(parents=True, exist_ok=True) - with open(f"{model_dir}/adapter_config.json", "r") as f: - config = json.load(f) - - alpha = config.get("lora_alpha") - use_rslora = config.get("use_rslora", False) - - lora_model = load_state_dict(get_model_path(model_dir, "adapter_model")) - lora_model = preprocess_lora_weights(lora_model) - all_weights = get_all_lora_weights(lora_model) - converted_weights = [] - converted_config = [] - - def derive_adapter_size(inout_weight: torch.Tensor) -> int: - assert len(inout_weight.shape) == 2 - dim0, dim1 = inout_weight.shape - # assume the hidden dim is the larger of the 2 - adapter_size = min(dim0, dim1) - return adapter_size - - def derive_weights_scale(adapter_size: int, alpha: float, - use_rslora: bool) -> float: - if use_rslora: - return alpha / np.sqrt(adapter_size) - return alpha / adapter_size - - for layer_idx, layer_weights in all_weights.items(): - for hf_module, module_weights in layer_weights.items(): - in_weights = module_weights['in'] - out_weights = module_weights['out'] - magnitude = module_weights.get("magnitude", None) - is_dora = magnitude is not None - - processed_weights = [] - - assert len(in_weights.shape) == 2 - assert len(out_weights.shape) == 2 - assert not is_dora or len(magnitude.shape) == 1 - - adapter_size = derive_adapter_size(in_weights) - assert adapter_size == derive_adapter_size( - out_weights), "adapter size of A mismatches adapter size of B" - scale = derive_weights_scale(adapter_size, alpha, use_rslora) - - for w, inout in ((in_weights, "in"), (out_weights, "out")): - dim0 = w.shape[0] - dim1 = w.shape[1] - # in_weights should have shape [adaper_size, hidden] - if dim1 < dim0 and inout == "in": - w = w.transpose(1, 0) - # out_weights should have shape [hidden, adapter_size] - elif dim0 < dim1 and inout == "out": - w = w.transpose(1, 0) - if inout == "out": - w = w * scale - w = w.contiguous().flatten().to(dtype=str_dtype_to_torch(dtype)) - processed_weights.append(w) - - if is_dora: - processed_weights.append(magnitude.contiguous().flatten().to( - dtype=str_dtype_to_torch(dtype))) - - processed_weights = torch.concatenate(processed_weights).flatten() - converted_weights.append(processed_weights) - converted_config.append([ - hf_modules_to_module_id[hf_module], layer_idx, adapter_size, - 1 if is_dora else 0 - ]) - max_row_size = 0 - for t in converted_weights: - max_row_size = max(max_row_size, t.shape[0]) - for i in range(len(converted_weights)): - converted_weights[i] = torch.nn.functional.pad( - converted_weights[i], - (0, max_row_size - converted_weights[i].shape[0])).unsqueeze(0) - converted_weights = torch_to_numpy( - torch.concatenate( - converted_weights, - dim=0).unsqueeze(0).to(dtype=str_dtype_to_torch(dtype)).cpu()) - converted_config = torch.tensor(converted_config, - dtype=torch.int32, - device='cpu').unsqueeze(0).numpy() - - save_val(converted_weights, - saved_dir, - "lora_weights", - tp_num=None, - write_npy=True) - save_val(converted_config, - saved_dir, - "lora_config", - tp_num=None, - write_npy=True) - - -def main(args): - start_time = datetime.datetime.now() - convert_hf_model(args.in_file, args.storage_type, args.out_dir) - - LOGGER.info("Spent %s (h:m:s) to convert the prompt model", - datetime.datetime.now() - start_time) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - '--out-dir', - '-o', - type=Path, - help='path to output embedding table file in the .npy format', - required=True) - parser.add_argument('--in-file', - '-i', - type=Path, - help='path to input lora checkpoint file', - required=True) - parser.add_argument("--verbose", - action="store_true", - help="Provide verbose messages") - parser.add_argument("--storage-type", - type=str, - default="float16", - choices=["float32", "float16", "bfloat16"]) - args = parser.parse_args() - - LOGGER.setLevel(logging.DEBUG if args.verbose else logging.INFO) - - print("\n=============== Argument ===============") - for key in vars(args): - print(f"{key}: {vars(args)[key]}") - print("========================================") - - main(args) diff --git a/examples/kimi_k3/README.md b/examples/kimi_k3/README.md index 1151fa1cdf9c..323aa56d3796 100644 --- a/examples/kimi_k3/README.md +++ b/examples/kimi_k3/README.md @@ -193,6 +193,26 @@ decoding requires the default cache manager, which cannot reuse blocks. ## Current limitations - Pipeline parallelism is not supported. +- **KV cache size estimation is temporarily skipped whenever speculative + decoding is enabled** (TRTLLM-14903; see the note in + `py_executor_creator.py`; non-speculative runs keep the normal + estimation behavior and the `TRTLLM_SKIP_KV_CACHE_ESTIMATION` gate). + Background: with speculative decoding and a self-spawned MPI session, + the estimation executor's warmup hangs indefinitely while exercising the + q>1 generation-path attention kernels that only its spec-mode dummy + requests reach; the pre-merge branch tip (`ec52c6418b`) passes the + identical run. While skipped, the cache is sized analytically from + `kv_cache_config.free_gpu_memory_fraction` (or explicitly via + `kv_cache_config.max_tokens`) without a measurement forward pass, which + can be slightly more conservative than the estimated size. The override + is removed once TRTLLM-14903 is fixed. +- **Known performance regression at DEP16 saturation** (attention-DP + + EP16 throughput recipe): relative to the pre-merge branch tip + (`ec52c6418b`), the 8K/1K serving sweep reproducibly loses ~15% output + throughput at concurrency 1024 and 4–5% at concurrency 128–256; + concurrency ≤ 64 and the TEP16/TEP8 latency recipes are at parity + (within ±2%). Developers investigating can A/B against `ec52c6418b`, + which is the last commit before this merge. Tracked as TRTLLM-14904. - FP8 KV cache (`kv_cache_config.dtype: fp8`) is supported and forces the trtllm-gen MLA generation backend (the default cute-dsl backend does not accept fp8 KV device scales); accuracy matches the bf16 KV baseline on diff --git a/examples/kimi_k3/eval_extra_llm_options_reuse.yaml b/examples/kimi_k3/eval_extra_llm_options_reuse.yaml index e8d411e0c17b..7fa88a5adc39 100644 --- a/examples/kimi_k3/eval_extra_llm_options_reuse.yaml +++ b/examples/kimi_k3/eval_extra_llm_options_reuse.yaml @@ -10,3 +10,10 @@ kv_cache_config: # allocations OOM during executor creation on GB300 (executor_extra with # only ~32 GiB free after the KV pool). free_gpu_memory_fraction: 0.5 + # Restore the pre-rework snapshot cadence: upstream deprecated + # mamba_state_cache_interval (default 256) into + # mamba_state_config.periodic_snapshot_interval (default 0 = snapshots + # off), and hybrid models only expose reusable prefixes at snapshot + # boundaries — without this, block reuse silently never engages. + mamba_state_config: + periodic_snapshot_interval: 256 diff --git a/examples/kimi_k3/perf_sweep/perf_sweep.sbatch b/examples/kimi_k3/perf_sweep/perf_sweep.sbatch index 17c29ebbe56a..1670ad74bd56 100644 --- a/examples/kimi_k3/perf_sweep/perf_sweep.sbatch +++ b/examples/kimi_k3/perf_sweep/perf_sweep.sbatch @@ -183,6 +183,7 @@ srun --mpi=pmix --kill-on-bad-exit=1 \ export HF_HOME=\${HF_HOME:-$CACHE_DIR/hf_home} mkdir -p \"\$FLASHINFER_WORKSPACE_BASE\" \"\$FLASHINFER_CUBIN_DIR\" \"\$HF_HOME\" export PATH=\"$REPO/.venv-3.12/bin:\$PATH\" + export PYTHONPATH=\"$REPO\${PYTHONPATH:+:\$PYTHONPATH}\" exec '$REPO/tensorrt_llm/llmapi/trtllm-llmapi-launch' trtllm-serve \ \"$MODEL\" \ --backend pytorch --host 0.0.0.0 --port $PORT \ diff --git a/examples/language_adapter/README.md b/examples/language_adapter/README.md deleted file mode 100755 index 8487c8ab42a0..000000000000 --- a/examples/language_adapter/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# Language-Adapter - -This document shows how to build and run a model with Language-Adapter plugin in TensorRT LLM on NVIDIA GPUs. - -## Overview -The concept of Language Adapter during inference time was introduced in [MAD-X: An Adapter-Based Framework for Multi-Task Cross-Lingual Transfer -](https://arxiv.org/pdf/2005.00052): -> we can simply replace a language-specific adapter trained for English with a language-specific adapter trained for Quechua at inference time. - -The implementation is done with MOE plugin with static expert selection passed during runtime as a parameter in request. - -For instance, encoder-decoder model may leverage language adapter for language-specific translation tasks when each of the language-adapter is trained for a specific language, this language adapter plugin achieves the language switching within one session only by passing in the `language_task_uid` to the plugin. - -The model checkpoint here is not publicly available. Please leverage `layers/language_adapter.py` in your own model. - -### Engine Preparation (convert and build) -``` -MODEL_DIR="dummy_model" # model not publicly available -INFERENCE_PRECISION="float16" -TP_SIZE=1 -PP_SIZE=1 -WORLD_SIZE=1 -MODEL_TYPE=language_adapter -MODEL_NAME=$MODEL_TYPE -CKPT_DIR=/scratch/tmp/trt_models/${MODEL_NAME}/${WORLD_SIZE}-gpu/${INFERENCE_PRECISION} -ENGINE_DIR=/scratch/tmp/trt_engines/${MODEL_NAME}/${WORLD_SIZE}-gpu/${INFERENCE_PRECISION} - -max_beam=5 -max_batch=32 -max_input_len=1024 -max_output_len=1024 - -python ../enc_dec/convert_checkpoint.py --model_type ${MODEL_TYPE} \ - --model_dir ${MODEL_DIR} \ - --output_dir $CKPT_DIR \ - --tp_size ${TP_SIZE} \ - --pp_size ${PP_SIZE} \ - --dtype ${INFERENCE_PRECISION} \ - --workers 1 - -trtllm-build --checkpoint_dir $CKPT_DIR/encoder \ - --output_dir $ENGINE_DIR/encoder \ - --paged_kv_cache disable \ - --moe_plugin auto \ - --bert_attention_plugin ${INFERENCE_PRECISION} \ - --gpt_attention_plugin ${INFERENCE_PRECISION} \ - --gemm_plugin ${INFERENCE_PRECISION} \ - --remove_input_padding enable \ - --max_input_len ${max_input_len} \ - --max_beam_width ${max_beam} \ - --max_batch_size ${max_batch} - -trtllm-build --checkpoint_dir $CKPT_DIR/decoder \ - --output_dir $ENGINE_DIR/decoder \ - --paged_kv_cache enable \ - --moe_plugin auto \ - --bert_attention_plugin ${INFERENCE_PRECISION} \ - --gpt_attention_plugin ${INFERENCE_PRECISION} \ - --gemm_plugin ${INFERENCE_PRECISION} \ - --remove_input_padding enable \ - --max_input_len 1 \ - --max_beam_width ${max_beam} \ - --max_batch_size ${max_batch} \ - --max_seq_len ${max_output_len} -``` - -### CPP runtime -A list `language_task_uids` that includes the language_task_uid for each input prompt is required: -``` -# translate 2 sentence, 1 to France (language_task_uid=3) 1 to Spanish (language_task_uid=2). -# language_task_uids = [3, 2] - -TEXT="Where is the nearest restaurant? Wikipedia is a free online encyclopedia written and maintained by a community of volunteers (called Wikis) through open collaboration and the use of MediaWiki, a wiki-based editing system." - -python3 ../run.py --engine_dir $ENGINE_DIR --tokenizer_type "language_adapter" --max_input_length 512 --max_output_len 512 --num_beams 1 --input_file input_ids.npy --tokenizer_dir $MODEL_DIR --language_task_uids 3 2 - -# Input [Text 0]: "" -# Output [Text 0 Beam 0]: "Où se trouve le restaurant le plus proche ? Wikipédia est une encyclopédie en ligne gratuite écrite et maintenue par une communauté de bénévoles (appelés Wikis) grâce à une collaboration ouverte et à l'utilisation de MediaWiki, un système d'édition basé sur wiki." -# Input [Text 1]: "" -# Output [Text 1 Beam 0]: "¿Dónde está el restaurante más cercano? Wikipedia es una enciclopedia en línea gratuita escrita y mantenida por una comunidad de voluntarios (llamada Wikis) a través de la colaboración abierta y el uso de MediaWiki, un sistema de edición basado en wiki." - -``` - -### Python runtime -Currently Python runtime does not support beam_width > 1. - -For Python runtime, full routing information of length [num_tokens, 1] is required for both encoder and decoder, which stacks routing information for each token in a batch of requests. -``` -# language_adapter_routing = get_language_adapter_routings(language_task_uid, input_ids) - -TEXT="Where is the nearest restaurant? Wikipedia is a free online encyclopedia written and maintained by a community of volunteers (called Wikis) through open collaboration and the use of MediaWiki, a wiki-based editing system." - -python3 ../enc_dec/run.py --engine_dir $ENGINE_DIR --engine_name ${MODEL_NAME} --model_name $MODEL_DIR --max_new_token=64 --num_beams=1 - -# in the run.py, 2 input prompts and 2 language task uids are provided. The two task uid represent the language of the input prompts to be translated to. - -# TRT-LLM output text: ['¿Dónde está el restaurante más cercano? Wikipedia es una enciclopedia en línea gratuita escrita y mantenida por una comunidad de voluntarios (llamada Wikis) a través de la colaboración abierta y el uso de MediaWiki, un sistema de edición basado en wiki.', "Où se trouve le restaurant le plus proche ? Wikipédia est une encyclopédie en ligne gratuite é -crite et maintenue par une communauté de bénévoles (appelés Wikis) grâce à une collaboration ouverte et à l'utilisation de MediaWiki, un système d'édition basé sur wiki."] -``` diff --git a/examples/layer_wise_benchmarks/sample_performance_alignment.sh b/examples/layer_wise_benchmarks/sample_performance_alignment.sh index 812fe46f0024..c79b91a1b93d 100755 --- a/examples/layer_wise_benchmarks/sample_performance_alignment.sh +++ b/examples/layer_wise_benchmarks/sample_performance_alignment.sh @@ -14,17 +14,20 @@ export TLLM_AUTOTUNER_CACHE_PATH="$PROFILE_DIR/sample_performance_alignment_cach mkdir -p -- "$PROFILE_DIR" mkdir -p -- "$(dirname -- "$TLLM_AUTOTUNER_CACHE_PATH")" -python3 ../../benchmarks/cpp/prepare_dataset.py \ - --tokenizer "$MODEL" \ - --stdout \ +# Write to a file via --output rather than redirecting --stdout: trtllm-bench +# prints an import-time banner on stdout that would otherwise corrupt line 1 of +# the JSONL dataset. +trtllm-bench \ + --model "$MODEL" \ + prepare-dataset \ + --output /tmp/dataset.jsonl \ --random-seed 42 \ token-norm-dist \ --num-requests $((BATCH_SIZE * NP)) \ --input-mean 2048 \ --input-stdev 0 \ --output-mean 256 \ - --output-stdev 0 \ - >/tmp/dataset.jsonl + --output-stdev 0 # Step 1 diff --git a/examples/llm-api/llm_speculative_decoding.py b/examples/llm-api/llm_speculative_decoding.py index af24a78cd4ea..b7af29dc01da 100644 --- a/examples/llm-api/llm_speculative_decoding.py +++ b/examples/llm-api/llm_speculative_decoding.py @@ -5,7 +5,8 @@ import click -from tensorrt_llm import LLM, SamplingParams +from tensorrt_llm import LLM, SamplingParams, logger +from tensorrt_llm._utils import get_sm_version from tensorrt_llm.llmapi import (Eagle3DecodingConfig, KvCacheConfig, MTPDecodingConfig, NGramDecodingConfig) @@ -16,6 +17,14 @@ def run_MTP(model: Optional[str] = None): + sm_version = get_sm_version() + if sm_version < 90: + logger.warning( + f"Skipping the MTP example: it requires the DeepSeek MLA " + "generation FMHA kernel, which is only available on " + f"Hopper+ (SM>=90). Detected SM{sm_version}.") + return + spec_config = MTPDecodingConfig(use_relaxed_acceptance_for_thinking=True, relaxed_topk=10, relaxed_delta=0.01) diff --git a/examples/llm-api/quickstart_advanced.py b/examples/llm-api/quickstart_advanced.py index 718d4a410e2c..bb06009bdc33 100644 --- a/examples/llm-api/quickstart_advanced.py +++ b/examples/llm-api/quickstart_advanced.py @@ -194,6 +194,11 @@ def add_llm_args(parser): parser.add_argument('--apply_chat_template', default=False, action='store_true') + parser.add_argument('--custom_tokenizer', + type=str, + default=None, + help='Override the tokenizer. Accepts a built-in alias ' + " or a fully-qualified class import path.") # Sampling parser.add_argument("--max_tokens", type=int, default=64) @@ -310,6 +315,9 @@ def setup_llm(args, **kwargs): relaxed_topk=args.relaxed_topk, relaxed_delta=args.relaxed_delta, mtp_eagle_one_model=args.use_one_model, + use_dynamic_tree=args.use_dynamic_tree, + dynamic_tree_max_topK=args.dynamic_tree_max_topK, + max_total_draft_tokens=args.max_total_draft_tokens, speculative_model=args.model_dir) elif spec_decode_algo == "EAGLE3": spec_config = Eagle3DecodingConfig( @@ -387,6 +395,7 @@ def setup_llm(args, **kwargs): gather_generation_logits=args.return_generation_logits, max_beam_width=args.max_beam_width, orchestrator_type=args.orchestrator_type, + custom_tokenizer=args.custom_tokenizer, **kwargs) use_beam_search = args.max_beam_width > 1 diff --git a/examples/llm-eval/lm-eval-harness/README.md b/examples/llm-eval/lm-eval-harness/README.md deleted file mode 100644 index c3854654dd0d..000000000000 --- a/examples/llm-eval/lm-eval-harness/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Evaluation scripts for LLM tasks - -This folder includes code to use the [LM-Eval-Harness](https://github.com/EleutherAI/lm-evaluation-harness), a unified framework to test generative language models on a large number of different evaluation tasks. The supported eval tasks are [here](https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks). - -The following instructions show how to evaluate TRT-LLM engines with the benchmark. - -## Instructions - -### TRT-LLM API - -Build the TRT-LLM engine using `trtllm-build`. - -Install the `lm_eval` package in the `requirements.txt` file in this folder. - -Run the evaluation script with the following command: - -```sh -python lm_eval_tensorrt_llm.py --model trt-llm \ - --model_args tokenizer=,model=,chunk_size= \ - --tasks -``` - -In the LM-Eval-Harness, model args are submitted as a comma-separated list of the form `arg=value`. The `trt-llm` model supports the following `model_args`: - -| Name | Description | Default Value | -|--------------------------|-------------------------------------------------------------------|----------------| -| tokenizer | directory containing the HF tokenizer. | | -| model | directory containing the TRTLLM engine or torch model. | | -| max_gen_toks | max number of tokens to generate (if not specified in gen_kwargs) | 256 | -| chunk_size | number of async requests to send at once to the engine | 200 | -| max_tokens_kv_cache | max tokens in paged KV cache | None | -| free_gpu_memory_fraction | KV cache free GPU memory fraction | 0.9 | -| trust_remote_code | trust remote code; use if necessary to set up the tokenizer | False | -| tp | tensor parallel size (for torch backend) | no. of workers | -| use_cuda_graph | enable CUDA graph | True | -| max_context_length | maximum context length for evaluation | None | -| moe_expert_parallel_size | expert parallel size for MoE models | None | -| moe_backend | backend for MoE models (e.g., "TRTLLM") | "TRTLLM" | - -### Torch backend - -Install the `lm_eval` package in the `requirements.txt` file in this folder. - -Run the evaluation script with the same command as above, but include `backend=torch` in the `model_args`. For example: - -```sh -python lm_eval_tensorrt_llm.py --model trt-llm \ - --model_args model=,backend=torch,chunk_size= \ - --tasks -``` - -### trtllm-serve - -Build the TRT-LLM engine using `trtllm-build` and deploy with `trtllm-serve`. - -Install the `lm_eval` package in the `requirements.txt` file in this folder. - -Run the evaluation script with the following command: - -```sh -python lm_eval_tensorrt_llm.py --model local-completions \ - --model_args base_url=http://${HOST_NAME}:8001/v1/completions,model=,tokenizer= \ - --tasks \ - --batch_size <#> -``` - -Because `trtllm-serve` is OpenAI API compatible, we can use the `local-completions` model built in to `lm_eval`, which supports [these model_args](https://github.com/EleutherAI/lm-evaluation-harness/blob/v0.4.7/lm_eval/models/openai_completions.py#L12). diff --git a/examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py b/examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py deleted file mode 100644 index 1738242267d9..000000000000 --- a/examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py +++ /dev/null @@ -1,317 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import gc -import json -import logging -import os -import signal -import threading -import time -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -import torch -import torch.nn.functional as F -import transformers -from lm_eval.__main__ import cli_evaluate -from lm_eval.api.model import TemplateLM -from lm_eval.api.registry import register_model -from packaging.version import parse -from tqdm import tqdm - -import tensorrt_llm -from tensorrt_llm import LLM as TORCH_LLM -from tensorrt_llm._tensorrt_engine import LLM as TRT_LLM -from tensorrt_llm.bindings.executor import DecodingConfig -from tensorrt_llm.llmapi import KvCacheConfig as TRT_KvCacheConfig -from tensorrt_llm.llmapi import RequestOutput, SamplingParams -from tensorrt_llm.llmapi.llm_args import MoeConfig - -logger = logging.getLogger(__name__) - - -@register_model("trt-llm") -class TRTLLMEvalBase(TemplateLM): - - def __init__( - self, - model: str, - tokenizer: Optional[str] = None, - tp: int = 0, # tensor_parallel_size - max_gen_toks: int = 256, - chunk_size: int = 200, - max_tokens_kv_cache: Optional[int] = None, - free_gpu_memory_fraction: float = 0.9, - trust_remote_code: bool = False, - use_cuda_graph: bool = True, - backend: str = 'trt', - max_context_length: Optional[int] = None, - moe_expert_parallel_size: Optional[int] = None, - moe_backend: Optional[str] = "TRTLLM", - enable_chunked_prefill: bool = False, - max_num_tokens: Optional[int] = None, - **kwargs, - ): - # initialize TemplateLM, copied from TemplateAPI - super().__init__() - assert isinstance(model, str) - assert parse(tensorrt_llm.__version__) >= parse("0.15.0") - - self.max_gen_toks = max_gen_toks - self.chunk_size = chunk_size - self.backend = backend - self.max_context_length = max_context_length - self.moe_expert_parallel_size = moe_expert_parallel_size - self.moe_backend = moe_backend - trt_kv_cache_config = TRT_KvCacheConfig(enable_block_reuse=False) - trt_kv_cache_config.free_gpu_memory_fraction = free_gpu_memory_fraction - if max_tokens_kv_cache is not None: - trt_kv_cache_config.max_tokens = max_tokens_kv_cache - - if tokenizer is None: - # Assume the tokenizer is stored in the model_dir if not specified. - tokenizer = model - logger.info(f"Tokenizer: {tokenizer}") - self.tokenizer = transformers.AutoTokenizer.from_pretrained( - tokenizer, trust_remote_code=trust_remote_code) - - if self.tokenizer.pad_token_id is None: - self.tokenizer.pad_token_id = self.tokenizer.eos_token_id - - if self.backend == 'torch': - kwargs.pop('batch_size') - if tp < 1: - tp = torch.cuda.device_count() - - pytorch_config_params = { - 'cuda_graph_config': {} if use_cuda_graph else None, - "print_iter_log": False, - 'moe_config': MoeConfig(backend=self.moe_backend) - } - - # stop words not currently supported by torch backend - self.use_stop_words = False - - self.llm = TORCH_LLM( - model=model, - tensor_parallel_size=tp, - trust_remote_code=trust_remote_code, - enable_chunked_prefill=enable_chunked_prefill, - max_num_tokens=max_num_tokens, - **pytorch_config_params, - tokenizer=self.tokenizer, - kv_cache_config=trt_kv_cache_config, - moe_expert_parallel_size=self.moe_expert_parallel_size, - **kwargs) - logger.info("Loaded TRT-LLM Torch engine") - else: - with open(Path(model) / "config.json", "r") as engine_config_file: - engine_config = json.load(engine_config_file) - build_config = engine_config["build_config"] - world_size = (engine_config.get("pretrained_config", {}).get( - "mapping", {}).get("world_size", 1)) - if max_tokens_kv_cache is None: - max_tokens_kv_cache = build_config[ - "max_seq_len"] * build_config["max_batch_size"] - self.gather_context_logits = build_config.get( - "gather_context_logits", False) - - medusa_choices = kwargs[ - 'medusa_choices'] if 'medusa_choices' in kwargs else None - kwargs = {} - if medusa_choices is not None: - decoding_config = DecodingConfig() - decoding_config.medusa_choices = medusa_choices - kwargs["decoding_config"] = decoding_config - assert world_size == 1, "decoding_config does not support multi TP in HLAPI." - - self.llm = TRT_LLM(model=model, - tokenizer=self.tokenizer, - kv_cache_config=trt_kv_cache_config, - **kwargs) - self.max_length = build_config['max_seq_len'] - 1 - logger.info("Loaded TRT-LLM engine") - - @property - def eot_token_id(self) -> int: - return self.llm.tokenizer.eos_token_id - - def tok_encode(self, string, add_special_tokens=False, **kwargs): - return self.llm.tokenizer.encode(string, - add_special_tokens=add_special_tokens, - **kwargs) - - def _loglikelihood_tokens( - self, - requests: List[Any], - disable_tqdm: bool = False) -> List[Tuple[float, bool]]: - """Compute the log likelihood of the continuation given the context.""" - if self.backend == 'torch': - raise NotImplementedError( - 'Torch backend does not return context logits yet') - - num_r = len(requests) - desc = "Processing loglikelihood requests" - sampling_params = SamplingParams(max_tokens=1, - return_context_logits=True) - - # process requests - futures: Dict[int, RequestOutput] = {} - results = [] - for i, request in tqdm(enumerate(requests), - desc=desc, - total=num_r, - disable=disable_tqdm): - # asynchronously submit a chunk of requests ahead of time... - if i % self.chunk_size == 0: - for j in range(i, min(i + self.chunk_size, num_r)): - prompt_ids = requests[j][1] + requests[j][2] - futures[j] = self.llm.generate_async( - prompt_ids, sampling_params) - - # process the output of the request i - r_out: RequestOutput = futures.pop(i).result() - - # check continuation portion of the prompt - # NOTE: context_logits are offset by 1 since they predict future token - ctxlen = len(request[1]) - token_ids_cont = request[2] - logits_cont = r_out.context_logits[ctxlen - 1:-1] # [sl, vocab] - logprobs_cont = F.log_softmax(logits_cont, dim=-1) # [sl, vocab] - top_tokens_cont = logprobs_cont.argmax(dim=-1).tolist() # [sl] - - # compute logprob and check for greedy - logprob_sum = sum(logprobs_cont[list(range(len(logprobs_cont))), - token_ids_cont]).item() - is_greedy = top_tokens_cont == token_ids_cont - - results.append((logprob_sum, is_greedy)) - - # clear response - del r_out - - return results - - def loglikelihood_rolling(self, requests, disable_tqdm: bool = False): - raise NotImplementedError - - def generate_until(self, - requests: List[Any], - disable_tqdm: bool = False) -> List[str]: - # some book-keeping and parameters... - num_r = len(requests) - desc = "Processing generate requests" - - if self.max_context_length is not None: - """ - Create updated_requests to contain qualified requests with the context length <= max_context_length. - Unqualified requests cannot simply be dropped as lm-eval library requires the number of requests to be the same. - - Note: The final score will drop if disqualified requests exist. - """ - request_idx_to_replace = [] - qualified_requests = [] - updated_requests = [] - for i, request in enumerate(requests): - context, gen_kwargs = request.args - if len(self.tok_encode(context)) > self.max_context_length: - request_idx_to_replace.append(i) - else: - qualified_requests.append(request) - - assert len( - qualified_requests - ) > 1, "No requests with context length <= max_context_length. Cannot run the evaluation." - if len(request_idx_to_replace) > 0: - print( - f"Warning: {len(request_idx_to_replace)} requests with context length > max_context_length will be replaced. The final score will drop." - ) - - for i, request in enumerate(requests): - if i in request_idx_to_replace: - # Replace the requests with context length > max_context_length with the qualified requests - updated_requests.append( - qualified_requests[i % len(qualified_requests)]) - else: - updated_requests.append(request) - assert len( - updated_requests - ) == num_r, "Number of updated requests does not match the number of requests." - requests = updated_requests - - def _get_sp(gen_kwargs): - k_mapping = { - "temperature": "temperature", - "top_p": "top_p", - "max_gen_toks": "max_tokens", - "until": "stop", - } - kwargs_mapped = { - k_sp: gen_kwargs[k_gen] - for k_gen, k_sp in k_mapping.items() if k_gen in gen_kwargs - } - if "max_tokens" not in kwargs_mapped: - kwargs_mapped["max_tokens"] = self.max_gen_toks - return SamplingParams(**kwargs_mapped) - - # process requests - futures: Dict[int, RequestOutput] = {} - future_stop_words: Dict[int, RequestOutput] = {} - results = [] - for i, _ in tqdm(enumerate(requests), - desc=desc, - total=num_r, - disable=disable_tqdm): - # asynchronously submit a chunk of requests ahead of time... - if i % self.chunk_size == 0: - for j in range(i, min(i + self.chunk_size, num_r)): - context, gen_kwargs = requests[j].args - prompt_ids = self.tok_encode(context) - if self.max_context_length is not None: - assert len( - prompt_ids - ) <= self.max_context_length, f"Prompt length > {self.max_context_length}, {len(prompt_ids)}, should be filtered out." - kwargs_mapped = _get_sp(gen_kwargs) - futures[j] = self.llm.generate_async( - prompt_ids, kwargs_mapped) - del kwargs_mapped - future_stop_words[j] = gen_kwargs["until"] - - # process the output of the request i - r_out: RequestOutput = futures.pop(i).result() - stop_words = future_stop_words.pop(i) - txt = r_out.outputs[0].text - if stop_words: - for word in stop_words: - word_index = txt.find(word) - if word_index >= 0: - txt = txt[:word_index] - results.append(txt) - - return results - - -if __name__ == "__main__": - cli_evaluate() - # Force clean up the LLM instance and void hanging. - gc.collect() - - # Force terminate in case gc.collect() is not enough. - def _terminate(): - time.sleep(10) - os.kill(os.getpid(), signal.SIGTERM) - - termination_thread = threading.Thread(target=_terminate, daemon=True) - termination_thread.start() diff --git a/examples/llm-eval/lm-eval-harness/requirements.txt b/examples/llm-eval/lm-eval-harness/requirements.txt deleted file mode 100644 index 44a3383f612d..000000000000 --- a/examples/llm-eval/lm-eval-harness/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -lm_eval[api]==0.4.7 diff --git a/examples/mmlu.py b/examples/mmlu.py deleted file mode 100644 index 82bb9a7ec9ad..000000000000 --- a/examples/mmlu.py +++ /dev/null @@ -1,479 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020 Dan Hendrycks -# SPDX-FileCopyrightText: Copyright (c) 2023 Deep Cognition and Language Research (DeCLaRe) Lab -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 and MIT -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Adapted from https://github.com/declare-lab/instruct-eval -Helper script to compare TRTLLM and HF models on the MMLU dataset. -Example usage: - mkdir data; wget https://people.eecs.berkeley.edu/~hendrycks/data.tar -O data/mmlu.tar - tar -xf data/mmlu.tar -C data && mv data/data data/mmlu - - python mmlu.py --hf_model_dir --engine_dir --test_trt_llm - python mmlu.py --hf_model_dir --engine_dir --test_hf -""" - -import argparse -import os -import random - -import numpy as np -import pandas as pd -import torch -import torch.nn as nn -from tqdm import tqdm -from transformers import (AutoConfig, AutoModel, AutoModelForCausalLM, - AutoModelForSeq2SeqLM, AutoTokenizer, - GenerationConfig) -from utils import (add_common_args, load_tokenizer, prepare_enc_dec_inputs, - read_is_enc_dec, read_model_name) - -import tensorrt_llm -from tensorrt_llm.runtime import PYTHON_BINDINGS, ModelRunner - -if PYTHON_BINDINGS: - from tensorrt_llm.runtime import ModelRunnerCpp - -os.environ["TOKENIZERS_PARALLELISM"] = "false" - -DTYPE_STR_MAPPING = { - "fp32": torch.float32, - "fp16": torch.float16, - "bf16": torch.bfloat16, - "float32": torch.float32, - "float16": torch.float16, - "bfloat16": torch.bfloat16, -} -RAND_SEED = 1234 - - -def get_choices(): - return ["A", "B", "C", "D"] - - -def get_subcategories(): - return { - "abstract_algebra": ["math"], - "anatomy": ["health"], - "astronomy": ["physics"], - "business_ethics": ["business"], - "clinical_knowledge": ["health"], - "college_biology": ["biology"], - "college_chemistry": ["chemistry"], - "college_computer_science": ["computer science"], - "college_mathematics": ["math"], - "college_medicine": ["health"], - "college_physics": ["physics"], - "computer_security": ["computer science"], - "conceptual_physics": ["physics"], - "econometrics": ["economics"], - "electrical_engineering": ["engineering"], - "elementary_mathematics": ["math"], - "formal_logic": ["philosophy"], - "global_facts": ["other"], - "high_school_biology": ["biology"], - "high_school_chemistry": ["chemistry"], - "high_school_computer_science": ["computer science"], - "high_school_european_history": ["history"], - "high_school_geography": ["geography"], - "high_school_government_and_politics": ["politics"], - "high_school_macroeconomics": ["economics"], - "high_school_mathematics": ["math"], - "high_school_microeconomics": ["economics"], - "high_school_physics": ["physics"], - "high_school_psychology": ["psychology"], - "high_school_statistics": ["math"], - "high_school_us_history": ["history"], - "high_school_world_history": ["history"], - "human_aging": ["health"], - "human_sexuality": ["culture"], - "international_law": ["law"], - "jurisprudence": ["law"], - "logical_fallacies": ["philosophy"], - "machine_learning": ["computer science"], - "management": ["business"], - "marketing": ["business"], - "medical_genetics": ["health"], - "miscellaneous": ["other"], - "moral_disputes": ["philosophy"], - "moral_scenarios": ["philosophy"], - "nutrition": ["health"], - "philosophy": ["philosophy"], - "prehistory": ["history"], - "professional_accounting": ["other"], - "professional_law": ["law"], - "professional_medicine": ["health"], - "professional_psychology": ["psychology"], - "public_relations": ["politics"], - "security_studies": ["politics"], - "sociology": ["culture"], - "us_foreign_policy": ["politics"], - "virology": ["health"], - "world_religions": ["philosophy"], - } - - -def get_categories(): - return { - "STEM": [ - "physics", - "chemistry", - "biology", - "computer science", - "math", - "engineering", - ], - "humanities": ["history", "philosophy", "law"], - "social sciences": [ - "politics", - "culture", - "economics", - "geography", - "psychology", - ], - "other (business, health, misc.)": ["other", "business", "health"], - } - - -def format_subject(subject): - line = subject.split("_") - s = "" - for entry in line: - s += " " + entry - return s - - -def format_example(df, idx, include_answer=True): - prompt = df.iloc[idx, 0] - k = df.shape[1] - 2 - for j in range(k): - prompt += "\n{}. {}".format(get_choices()[j], df.iloc[idx, j + 1]) - prompt += "\nAnswer:" - if include_answer: - prompt += " {}\n\n".format(df.iloc[idx, k + 1]) - return prompt - - -def gen_prompt(train_df, subject, k=-1): - prompt = "The following are multiple choice questions (with answers) about {}.\n\n".format( - format_subject(subject)) - if k == -1: - k = train_df.shape[0] - for i in range(k): - prompt += format_example(train_df, i) - return prompt - - -def evaluate(args, subject, pipeline, dev_df, test_df): - rank = tensorrt_llm.mpi_rank() - cors = [] - all_probs = [] - for i in range(test_df.shape[0]): - if i >= args.max_ite: - break - # get prompt and make sure it fits - k = args.ntrain - prompt_end = format_example(test_df, i, include_answer=False) - train_prompt = gen_prompt(dev_df, subject, k) - prompt = train_prompt + prompt_end - - while not pipeline.check_valid_length(prompt) and k > 0: - k -= 1 - train_prompt = gen_prompt(dev_df, subject, k) - prompt = train_prompt + prompt_end - - label = test_df.iloc[i, test_df.shape[1] - 1] - pred = pipeline(prompt) - - if rank == 0: - probs = [0 for _ in get_choices()] - cor = pred.strip().startswith(label) - cors.append(cor) - all_probs.append(probs) - - if rank == 0: - acc = np.mean(cors) - cors = np.array(cors) - - all_probs = np.array(all_probs) - print("Average accuracy {:.3f} - {}".format(acc, subject)) - - return cors, acc, all_probs - else: - return None, 0, None - - -def get_tokenizer(ckpt_path, max_seq_len): - print(f"Initializing tokenizer from {ckpt_path}") - tokenizer = AutoTokenizer.from_pretrained( - ckpt_path, - model_max_length=max_seq_len, - padding_side="left", - trust_remote_code=True, - ) - tokenizer.pad_token = tokenizer.eos_token - - return tokenizer - - -class Pipeline: - - def __init__(self, tokenizer, model, model_name, pad_id, end_id, - max_attention_window_size, is_enc_dec, hf_model_dir, - engine_dir): - self.tokenizer = tokenizer - self.model = model - self.model_name = model_name - self.pad_id = pad_id - self.end_id = end_id - self.max_attention_window_size = max_attention_window_size - self.output_len = 2 - self.is_enc_dec = is_enc_dec - self.decoder_start_token_id = None - self.engine_dir = engine_dir - if self.is_enc_dec: - self.decoder_start_token_id = AutoConfig.from_pretrained( - hf_model_dir).decoder_start_token_id - - def __call__(self, prompt): - rank = tensorrt_llm.mpi_rank() - # Run the model in batch size 1 and beam size 1 - inputs = self.tokenizer.encode(prompt, return_tensors="pt").squeeze(0) - batch_input_ids = [inputs] - - # For multi-choice tasks like MMLU, we don't need to adjust following parameters - output_len = self.output_len - top_k = 1 - top_p = 0.0 - - input_lengths = [x.size(0) for x in batch_input_ids] - - with torch.no_grad(): - if isinstance(self.model, nn.Module): - # Left padding for HF - max_length = max(input_lengths) - paddings = [ - torch.ones(max_length - l, dtype=torch.int32) * self.pad_id - for l in input_lengths - ] - batch_input_ids = [ - torch.cat([pad, x]) - for x, pad in zip(batch_input_ids, paddings) - ] - batch_input_ids = torch.stack(batch_input_ids) - batch_input_ids = batch_input_ids.cuda() - if self.is_enc_dec: - batch_decoder_input_ids = torch.IntTensor( - [[self.decoder_start_token_id]]).to('cuda') - batch_decoder_input_ids = batch_decoder_input_ids.repeat( - (batch_input_ids.shape[0], 1)) - - with torch.no_grad(): - # Use default temperature and top_k - outputs = self.model.generate( - batch_input_ids, - max_new_tokens=output_len, - top_k=top_k, - decoder_input_ids=batch_decoder_input_ids - if self.is_enc_dec else None) - if not self.is_enc_dec: - output_ids = outputs[0, input_lengths[0]:] - else: - output_ids = outputs[0] - - elif isinstance(self.model, ModelRunnerCpp) or isinstance( - self.model, ModelRunner): - if self.is_enc_dec: - encoder_input_ids, encoder_input_features, encoder_output_lengths, decoder_input_ids = prepare_enc_dec_inputs( - batch_input_ids, self.model_name, self.engine_dir, None) - - outputs = self.model.generate( - batch_input_ids=decoder_input_ids - if self.is_enc_dec else batch_input_ids, - encoder_input_ids=encoder_input_ids - if self.is_enc_dec else None, - encoder_input_features=encoder_input_features - if self.is_enc_dec else None, - encoder_output_lengths=encoder_output_lengths - if self.is_enc_dec else None, - max_new_tokens=output_len, - max_attention_window_size=self.max_attention_window_size, - end_id=self.end_id, - pad_id=self.pad_id, - top_k=top_k, - top_p=top_p, - ) - torch.cuda.synchronize() - if rank == 0: - if not self.is_enc_dec: - output_ids = outputs[0, 0, input_lengths[0]:] - else: - output_ids = outputs[0, 0] - if rank == 0: - return self.tokenizer.decode(output_ids, skip_special_tokens=True) - else: - return None - - def check_valid_length(self, prompt): - if isinstance(self.model, nn.Module): - return True - input_len = len(self.tokenizer.encode(prompt)) - return input_len <= self.model.max_input_len and input_len + self.output_len <= self.model.max_seq_len - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--data_dir", - type=str, - default="data/mmlu", - help=("Path to the data directory. If not available, " - "download https://people.eecs.berkeley.edu/~hendrycks/data.tar"), - ) - parser.add_argument("--ntrain", type=int, default=5) - parser.add_argument("--max_input_length", type=int, default=2048) - parser.add_argument("--test_trt_llm", action="store_true") - parser.add_argument("--test_hf", action="store_true") - parser.add_argument('--check_accuracy', action='store_true') - parser.add_argument('--accuracy_threshold', type=float, default=30) - parser.add_argument('--max_ite', type=int, default=10000000) - parser = add_common_args(parser) - - args = parser.parse_args() - - return args - - -def main(): - args = parse_args() - if args.tokenizer_dir is None: - args.tokenizer_dir = args.hf_model_dir - random.seed(RAND_SEED) - np.random.seed(RAND_SEED) - runtime_rank = tensorrt_llm.mpi_rank() - - os.path.dirname(os.path.abspath(__file__)) - data_fullpath = os.path.join(args.data_dir, "test") - - subjects = sorted([ - f.split("_test.csv")[0] for f in os.listdir(data_fullpath) - if "_test.csv" in f - ]) - - all_cors = [] - subcat_cors = { - subcat: [] - for subcat_lists in get_subcategories().values() - for subcat in subcat_lists - } - cat_cors = {cat: [] for cat in get_categories()} - - # different handling if encoder-decoder models - is_enc_dec = read_is_enc_dec( - args.engine_dir if not args.test_hf else args.hf_model_dir, - args.test_hf) - - model_name, model_version = read_model_name( - (args.engine_dir if not is_enc_dec else os.path.join( - args.engine_dir, 'encoder')) - if not args.test_hf else args.hf_model_dir, args.test_hf) - - tokenizer, pad_id, end_id = load_tokenizer( - tokenizer_dir=args.tokenizer_dir, - vocab_file=args.vocab_file, - model_name=model_name, - model_version=model_version, - ) - - if args.test_trt_llm: - assert not args.test_hf, "Cannot test both TRT-LLM and HF" - runner_cls = ModelRunner if not PYTHON_BINDINGS else ModelRunnerCpp - runner_kwargs = {} - if PYTHON_BINDINGS: - runner_kwargs.update(max_beam_width=1) - runner_kwargs.update( - is_enc_dec=is_enc_dec, - max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, - kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, - kv_cache_free_gpu_memory_fraction=args. - kv_cache_free_gpu_memory_fraction, - cross_kv_cache_fraction=args.cross_kv_cache_fraction - if is_enc_dec else None, - enable_chunked_context=args.enable_chunked_context, - multi_block_mode=args.multi_block_mode) - model = runner_cls.from_dir(engine_dir=args.engine_dir, - rank=runtime_rank, - **runner_kwargs) - else: - assert args.test_hf, "Must test either TRT-LLM or HF" - if 'GLM' in model_name and model_version == 'glm': - auto_model_cls = AutoModelForSeq2SeqLM - elif 'GLM' in model_name and model_version == 'chatglm': - auto_model_cls = AutoModel - elif is_enc_dec: - auto_model_cls = AutoModelForSeq2SeqLM - else: - auto_model_cls = AutoModelForCausalLM - model = auto_model_cls.from_pretrained( - args.hf_model_dir, - trust_remote_code=True, - dtype=DTYPE_STR_MAPPING[args.hf_data_type], - device_map="auto" if args.hf_device_map_auto else None, - ) - if not args.hf_device_map_auto: - model.cuda() - if model_name == "qwen": - model.generation_config = GenerationConfig.from_pretrained( - args.hf_model_dir, trust_remote_code=True) - - pipeline = Pipeline(tokenizer, model, model_name, pad_id, end_id, - args.max_attention_window_size, is_enc_dec, - args.hf_model_dir, args.engine_dir) - - for subject in tqdm(subjects): - dev_df = pd.read_csv(os.path.join(args.data_dir, "dev", - subject + "_dev.csv"), - header=None)[:args.ntrain] - test_df = pd.read_csv(os.path.join(args.data_dir, "test", - subject + "_test.csv"), - header=None) - - cors, acc, probs = evaluate(args, subject, pipeline, dev_df, test_df) - subcats = get_subcategories()[subject] - for subcat in subcats: - subcat_cors[subcat].append(cors) - for key in get_categories().keys(): - if subcat in get_categories()[key]: - cat_cors[key].append(cors) - all_cors.append(cors) - - if runtime_rank == 0: - for subcat in subcat_cors: - acc = np.mean(np.concatenate(subcat_cors[subcat])) * 100 - print(f"Average accuracy {acc:.2f} - {subcat}") - - for cat in cat_cors: - acc = np.mean(np.concatenate(cat_cors[cat])) * 100 - print(f"Average accuracy {acc:.2f} - {cat}") - - weighted_acc = np.mean(np.concatenate(all_cors)) * 100 - print(f"MMLU weighted average accuracy: {weighted_acc:.2f}") - - if args.check_accuracy: - assert weighted_acc >= args.accuracy_threshold, f"Expected accuracy >= {args.accuracy_threshold} while got {weighted_acc}" - return weighted_acc - - -if __name__ == "__main__": - main() diff --git a/examples/models/contrib/arctic/README.md b/examples/models/contrib/arctic/README.md deleted file mode 100644 index d346a3644517..000000000000 --- a/examples/models/contrib/arctic/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# Arctic - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document shows how to build and run a [Arctic](https://huggingface.co/Snowflake/snowflake-arctic-instruct) model in TensorRT-LLM. - -The TensorRT LLM Arctic implementation is based on the LLaMA model, with Mixture of Experts (MoE) enabled. The implementation can -be found in [llama/model.py](../../../../tensorrt_llm/models/llama/model.py). -See the LLaMA example [`examples/models/core/llama`](../../core/llama) for details. - -- [Arctic](#arctic) - - [Download model checkpoints](#download-model-checkpoints) - - [TensorRT LLM workflow](#tensorrt-llm-workflow) - - [Apply FP8 PTQ](#apply-fp8-ptq) - - [Build TensorRT engine](#build-tensorrt-engine) - - [Run Engine](#run-engine) - - [OOTB](#ootb) - -## Download model checkpoints - -First, download the HuggingFace BF16 checkpoints of Arctic model. - -**CAVEAT: this model is a pretty large Mixture-of-Experts (MoE) model, which has nearly 500B parameters and requires around 900GB disk space for storage. Please make sure you have enough space before proceeding.** - -```bash -HF_MODEL="arctic" -git clone https://huggingface.co/Snowflake/snowflake-arctic-instruct tmp/hf_checkpoints/${HF_MODEL} - -``` - -## TensorRT LLM workflow -Next, we use the general quantization script `quantize.py` to convert the checkpoints in FP8, and build the model with `trtllm-build` on multi-GPUs. In the example below, we use Tensor Parallelism (TP) across 8 GPUs. - -**Note: for such large model, it is deemed necessary to apply Post-Training Quantization (PTQ) methods on the model weights to deploy it on a cluster node, e.g., 8xH100 GPUs. In this example, we demonstrate the FP8 quantization workflow, which is supported on Hopper-and-next GPU architectures. For instructions of other PTQ methods other than FP8, please refer to the LLaMA or Mixtral examples.** - - -Set environment variables and necessary directory: - -```bash -PREC_RAW="bfloat16" -PREC_QUANT="fp8" -TP=8 -ENGINE="${HF_MODEL}_${PREC_QUANT}_tp${TP}" - -mkdir -p tmp/trt_engines -``` - -### Apply FP8 PTQ - -Notes: -- currently quantize.py does not support for Expert Parallelism (EP) mode yet. User should use `../../core/llama/convert_checkpoint.py` and specify `--moe_ep_size 1` instead, if needed. -- TensorRT LLM uses static quantization methods, which is expected to be faster at runtime as compared to dynamic quantization methods. This comes at a cost of an offline calibration step during quantization. `batch_size` and `calib_size` can be adjusted to shorten the calibration time. Please refer to ../quantization/README.md for explanation. -- **due to the large model size and the calibration step (which has to load the HuggingFace model and run forward passes), it is likely that you will need more number of GPUs during quantization step than the number of GPUs for engine building and final deployment. For example, using 16xH100 or 8xH200 for quantization & 8xH100 for deployment.** - -```bash -python ../../../quantization/quantize.py --model_dir tmp/hf_checkpoints/${HF_MODEL} \ - --dtype ${PREC_RAW} \ - --qformat ${PREC_QUANT} \ - --kv_cache_dtype ${PREC_QUANT} \ - --output_dir tmp/tllm_checkpoints/${ENGINE} \ - --batch_size 1 \ - --calib_size 128 \ - --tp_size ${TP} |& tee tmp/trt_engines/${ENGINE}_quantize.log - -``` - -### Build TensorRT engine -```bash -# Enable fp8 context fmha to get further acceleration by setting `--use_fp8_context_fmha enable` -# Use --workers to enable parallel build -trtllm-build --checkpoint_dir ./tmp/tllm_checkpoints/${ENGINE} \ - --output_dir ./tmp/trt_engines/${ENGINE} \ - --gpt_attention_plugin ${PREC_RAW} \ - --gemm_plugin ${PREC_RAW} \ - --workers ${TP} |& tee tmp/trt_engines/${ENGINE}_build.log -``` - -### Run Engine -Test your engine with the [run.py](../../../run.py) script: - -```bash -mpirun -n ${TP} --allow-run-as-root python ../../../run.py --engine_dir ./tmp/trt_engines/${ENGINE} --tokenizer_dir tmp/hf_checkpoints/${HF_MODEL} --max_output_len 20 --input_text "The future of AI is" |& tee tmp/trt_engines/${ENGINE}_run.log -``` - -For more examples see [`examples/models/core/llama/README.md`](../../core/llama/README.md) - - -### OOTB - -Arctic supports OOTB operation without the plugin, however this comes at a significant performance cost. Users should prefer using the plugin path whenever possible. diff --git a/examples/models/contrib/blip2/README.md b/examples/models/contrib/blip2/README.md deleted file mode 100644 index 0e80ed4fd81b..000000000000 --- a/examples/models/contrib/blip2/README.md +++ /dev/null @@ -1,4 +0,0 @@ -This example has been moved to [`../multimodal`](../../../multimodal) and merged with other multimodal examples. -Please follow [`../multimodal/README.md`](../../../multimodal/README.md) for new instructions. - -**NOTICE:** This folder will be removed in v1.0 release. diff --git a/examples/models/contrib/chatglm-6b/README.md b/examples/models/contrib/chatglm-6b/README.md deleted file mode 100644 index 00a8c2424665..000000000000 --- a/examples/models/contrib/chatglm-6b/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# ChatGLM - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document explains how to build the [ChatGLM-6B](https://huggingface.co/THUDM/chatglm-6b) models using TensorRT LLM and run on a single GPU, a single node with multiple GPUs or multiple nodes with multiple GPUs. - -- [ChatGLM](#chatglm) - - [Overview](#overview) - - [Support Matrix](#support-matrix) - - [Model comparison](#model-comparison) - - [Tokenizer and special tokens comparison](#tokenizer-and-special-tokens-comparison) - - [Usage](#usage) - - [1. Download repo and weights from HuggingFace Transformers](#1-download-repo-and-weights-from-huggingface-transformers) - - [2. Convert weights from HF Transformers to TensorRT LLM format](#2-convert-weights-from-hf-transformers-to-tensorrt-llm-format) - - [3. Build TensorRT engine(s)](#3-build-tensorrt-engines) - - [Enable plugins](#enable-plugins) - - [In-flight batching](#in-flight-batching) - - [4. Run inference](#4-run-inference) - - [Single node, single GPU](#single-node-single-gpu) - - [Single node, multi GPU](#single-node-multi-gpu) - - [5. Run summarization task](#5-run-summarization-task) - - [Weight Only quantization](#weight-only-quantization) - - [Smooth Quantization (SQ)](#smooth-quantization-sq) - - [Activation-aware Weight Quantization (AWQ)](#activation-aware-weight-quantization-awq) - - [FP8 Quantization](#fp8-quantization) - - [Benchmark](#benchmark) - - -## Overview - -The TensorRT LLM ChatGLM implementation can be found in [`tensorrt_llm/models/chatglm/model.py`](../../tensorrt_llm/models/chatglm/model.py). -The TensorRT LLM ChatGLM example code is located in [`examples/models/contrib/chatglm-6b`](./). There is one main file: - -* [`examples/models/core/glm-4-9b/convert_checkpoint.py`](../../../glm-4-9b/convert_checkpoint.py) to convert a checkpoint from the [HuggingFace (HF) Transformers](https://github.com/huggingface/transformers) format to the TensorRT LLM format. - -In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: - -* [`../../../run.py`](../../../run.py) to run the inference on an input text; -* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. - -## Support Matrix - -| Model Name | FP16 | FMHA | WO | SQ | AWQ | FP8 | TP | PP | ST | C++ | benchmark | IFB | -| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :-------: | :---: | -| chatglm_6b | Y | | Y | | | | Y | | Y | Y | Y | Y | -| glm_10b | Y | Y | Y | | Y | Y | Y | | Y | Y | Y | Y | - -* Model Name: the name of the model, the same as the name on HuggingFace -* FMHA: Fused MultiHead Attention (see introduction below) -* WO: Weight Only Quantization (int8 / int4) -* SQ: Smooth Quantization (int8) -* AWQ: Activation Aware Weight Quantization (int4) -* FP8: FP8 Quantization -* TP: Tensor Parallel -* PP: Pipeline Parallel -* ST: Strongly Typed -* C++: C++ Runtime -* benchmark: benchmark by python / C++ Runtime -* IFB: In-flight Batching (see introduction below) - -## Model comparison - -| Name | nL | nAH | nKH | nHW | nH | nF | nMSL | nV | bP2D | bBQKV | bBDense | Comments | -| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :----: | :---: | :---: | :-----: | :----------------------------------------------------------------- | -| chatglm_6b | 28 | 32 | 32 | 128 | 4096 | 16384 | 2048 | 130528 | Y | Y | Y | | -| glm_10b | 48 | 64 | 32 | 64 | 4096 | 16384 | 1024 | 50304 | Y | Y | Y | | - -* nL: number of layers -* nAH: number of attention heads -* nKH: number of kv heads (less than nAH if multi_query_attention is used) -* nHW: head width -* nH: hidden size -* nF: FFN hidden size -* nMSL: max sequence length (input + output) -* nV: vocabulary size -* bP2D: use position_encoding_2d (Y: Yes, N: No) -* bBQKV: use bias for QKV multiplication in self-attention -* bBDense: use bias for Dense multiplication in self-attention - -## Tokenizer and special tokens comparison - -| Name | Tokenizer | bos | eos | pad | cls | startofpiece | endofpiece | mask | smask | gmask | -| :--------------: | :--------------: | :----: | :----: | :---: | :---: | :----------: | :--------: | :----: | :---: | :----: | -| chatglm_6b | ChatGLMTokenizer | 130004 | 130005 | 3 | | 130004 | 130005 | 130000 | | 130001 | -| glm_10b | GLMGPT2Tokenizer | 50257 | 50256 | 50256 | 50259 | 50257 | 50258 | 50260 | 50264 | 50263 | - -## Usage - -The next section describe how to build the engine and run the inference demo. - -### 1. Download repo and weights from HuggingFace Transformers - -```bash -pip install -r requirements.txt -apt-get update -apt-get install git-lfs -rm -rf chatglm* - -# clone one or more models we want to build -git clone https://huggingface.co/THUDM/chatglm-6b chatglm_6b -git clone https://huggingface.co/THUDM/glm-10b glm_10b - -# replace tokenization file if using transformers-4.36.1 for model ChatGLM-6B (this might be needless in the future) -cp chatglm_6b/tokenization_chatglm.py chatglm_6b/tokenization_chatglm.py-backup -cp tokenization_chatglm.py chatglm_6b -``` - -For more example codes, please refer to the [examples/models/core/glm-4-9b/README.md](../../../glm-4-9b/README.md). diff --git a/examples/models/contrib/chatglm-6b/requirements.txt b/examples/models/contrib/chatglm-6b/requirements.txt deleted file mode 100644 index cdc65bf2bb38..000000000000 --- a/examples/models/contrib/chatglm-6b/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.0.dev0 -datasets==3.1.0 -evaluate -protobuf -rouge_score -sentencepiece -tiktoken diff --git a/examples/models/contrib/chatglm-6b/tokenization_chatglm.py b/examples/models/contrib/chatglm-6b/tokenization_chatglm.py deleted file mode 100755 index 8ae124909e98..000000000000 --- a/examples/models/contrib/chatglm-6b/tokenization_chatglm.py +++ /dev/null @@ -1,467 +0,0 @@ -"""Tokenization classes for ChatGLM.""" -import os -from typing import Dict, List, Optional, Union - -import numpy as np -import sentencepiece as spm -from transformers.tokenization_utils import PreTrainedTokenizer -from transformers.tokenization_utils_base import BatchEncoding, EncodedInput -from transformers.utils import PaddingStrategy, logging - -logger = logging.get_logger(__name__) - -PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - "THUDM/chatglm-6b": 2048, -} - - -class TextTokenizer: - - def __init__(self, model_path): - self.sp = spm.SentencePieceProcessor() - self.sp.Load(model_path) - self.num_tokens = self.sp.vocab_size() - - def encode(self, text): - return self.sp.EncodeAsIds(text) - - def decode(self, ids: List[int]): - return self.sp.DecodeIds(ids) - - def tokenize(self, text): - return self.sp.EncodeAsPieces(text) - - def convert_tokens_to_string(self, tokens): - return self.sp.DecodePieces(tokens) - - def convert_tokens_to_ids(self, tokens): - return [self.sp.PieceToId(token) for token in tokens] - - def convert_token_to_id(self, token): - return self.sp.PieceToId(token) - - def convert_id_to_token(self, idx): - return self.sp.IdToPiece(idx) - - def __len__(self): - return self.num_tokens - - -class SPTokenizer: - - def __init__( - self, - vocab_file, - num_image_tokens=20000, - max_blank_length=80, - byte_fallback=True, - ): - assert vocab_file is not None - self.vocab_file = vocab_file - self.num_image_tokens = num_image_tokens - self.special_tokens = [ - "[MASK]", "[gMASK]", "[sMASK]", "", "", "", - "", "" - ] - self.max_blank_length = max_blank_length - self.byte_fallback = byte_fallback - self.text_tokenizer = TextTokenizer(vocab_file) - - def _get_text_tokenizer(self): - return self.text_tokenizer - - @staticmethod - def get_blank_token(length: int): - assert length >= 2 - return f"<|blank_{length}|>" - - @staticmethod - def get_tab_token(): - return f"<|tab|>" - - @property - def num_text_tokens(self): - return self.text_tokenizer.num_tokens - - @property - def num_tokens(self): - return self.num_image_tokens + self.num_text_tokens - - @staticmethod - def _encode_whitespaces(text: str, max_len: int = 80): - text = text.replace("\t", SPTokenizer.get_tab_token()) - for i in range(max_len, 1, -1): - text = text.replace(" " * i, SPTokenizer.get_blank_token(i)) - return text - - def _preprocess(self, text: str, linebreak=True, whitespaces=True): - if linebreak: - text = text.replace("\n", "") - if whitespaces: - text = self._encode_whitespaces(text, max_len=self.max_blank_length) - return text - - def encode(self, - text: str, - linebreak=True, - whitespaces=True, - add_dummy_prefix=True) -> List[int]: - """ - @param text: Text to encode. - @param linebreak: Whether to encode newline (\n) in text. - @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding. - @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text. - @param add_dummy_prefix: Whether to add dummy blank space in the beginning. - """ - text = self._preprocess(text, linebreak, whitespaces) - if not add_dummy_prefix: - text = "" + text - tmp = self._get_text_tokenizer().encode(text) - tokens = [x + self.num_image_tokens for x in tmp] - return tokens if add_dummy_prefix else tokens[2:] - - def postprocess(self, text): - text = text.replace("", "\n") - text = text.replace(SPTokenizer.get_tab_token(), "\t") - for i in range(2, self.max_blank_length + 1): - text = text.replace(self.get_blank_token(i), " " * i) - return text - - def decode(self, text_ids: List[int]) -> str: - ids = [int(_id) - self.num_image_tokens for _id in text_ids] - ids = [_id for _id in ids if _id >= 0] - text = self._get_text_tokenizer().decode(ids) - text = self.postprocess(text) - return text - - def decode_tokens(self, tokens: List[str]) -> str: - text = self._get_text_tokenizer().convert_tokens_to_string(tokens) - text = self.postprocess(text) - return text - - def tokenize(self, - text: str, - linebreak=True, - whitespaces=True, - add_dummy_prefix=True) -> List[str]: - """ - @param text: Text to encode. - @param linebreak: Whether to encode newline (\n) in text. - @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding. - @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text. - @param add_dummy_prefix: Whether to add dummy blank space in the beginning. - """ - text = self._preprocess(text, linebreak, whitespaces) - if not add_dummy_prefix: - text = "" + text - tokens = self._get_text_tokenizer().tokenize(text) - return tokens if add_dummy_prefix else tokens[2:] - - def __getitem__(self, x: Union[int, str]): - if isinstance(x, int): - if x < self.num_image_tokens: - return "".format(x) - else: - return self.text_tokenizer.convert_id_to_token( - x - self.num_image_tokens) - elif isinstance(x, str): - if x.startswith("") and x[7:-1].isdigit(): - return int(x[7:-1]) - else: - return self.text_tokenizer.convert_token_to_id( - x) + self.num_image_tokens - else: - raise ValueError("The key should be str or int.") - - -class ChatGLMTokenizer(PreTrainedTokenizer): - """ - Construct a ChatGLM tokenizer. Based on byte-level Byte-Pair-Encoding. - - Args: - vocab_file (`str`): - Path to the vocabulary file. - """ - - vocab_files_names = {"vocab_file": "ice_text.model"} - max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES - model_input_names = ["input_ids", "attention_mask", "position_ids"] - - def __init__(self, - vocab_file, - do_lower_case=False, - remove_space=False, - bos_token='', - eos_token='', - end_token='', - mask_token='[MASK]', - gmask_token='[gMASK]', - padding_side="left", - pad_token="", - unk_token="", - num_image_tokens=20000, - **kwargs) -> None: # Fix for new transformers - - self.do_lower_case = do_lower_case - self.remove_space = remove_space - self.vocab_file = vocab_file - - self.bos_token = bos_token - self.eos_token = eos_token - self.end_token = end_token - self.mask_token = mask_token - self.gmask_token = gmask_token - - self.sp_tokenizer = SPTokenizer(vocab_file, - num_image_tokens=num_image_tokens) - - super().__init__( - do_lower_case=do_lower_case, # Fix for new transformers - remove_space=remove_space, - padding_side=padding_side, - bos_token=bos_token, - eos_token=eos_token, - end_token=end_token, - mask_token=mask_token, - gmask_token=gmask_token, - pad_token=pad_token, - unk_token=unk_token, - num_image_tokens=num_image_tokens, - **kwargs) - """ Initialization """ - - @property - def gmask_token_id(self) -> Optional[int]: - if self.gmask_token is None: - return None - return self.convert_tokens_to_ids(self.gmask_token) - - @property - def end_token_id(self) -> Optional[int]: - """ - `Optional[int]`: Id of the end of context token in the vocabulary. Returns `None` if the token has not been - set. - """ - if self.end_token is None: - return None - return self.convert_tokens_to_ids(self.end_token) - - @property - def vocab_size(self): - """ Returns vocab size """ - return self.sp_tokenizer.num_tokens - - def get_vocab(self): - """ Returns vocab as a dict """ - vocab = { - self._convert_id_to_token(i): i - for i in range(self.vocab_size) - } - vocab.update(self.added_tokens_encoder) - return vocab - - def preprocess_text(self, inputs): - if self.remove_space: - outputs = " ".join(inputs.strip().split()) - else: - outputs = inputs - - if self.do_lower_case: - outputs = outputs.lower() - - return outputs - - def _tokenize(self, text, **kwargs): - """ Returns a tokenized string. """ - text = self.preprocess_text(text) - - seq = self.sp_tokenizer.tokenize(text) - - return seq - - def convert_tokens_to_string(self, tokens: List[str]) -> str: - return self.sp_tokenizer.decode_tokens(tokens) - - def _decode(self, token_ids: Union[int, List[int]], **kwargs) -> str: - if isinstance(token_ids, int): - token_ids = [token_ids] - if len(token_ids) == 0: - return "" - if self.pad_token_id in token_ids: # remove pad - token_ids = list(filter((self.pad_token_id).__ne__, token_ids)) - return super()._decode(token_ids, **kwargs) - - def _convert_token_to_id(self, token): - """ Converts a token (str) in an id using the vocab. """ - return self.sp_tokenizer[token] - - def _convert_id_to_token(self, index): - """Converts an index (integer) in a token (str) using the vocab.""" - return self.sp_tokenizer[index] - - def save_vocabulary(self, save_directory, filename_prefix=None): - """ - Save the vocabulary and special tokens file to a directory. - - Args: - save_directory (`str`): - The directory in which to save the vocabulary. - filename_prefix (`str`, *optional*): - An optional prefix to add to the named of the saved files. - - Returns: - `Tuple(str)`: Paths to the files saved. - """ - if os.path.isdir(save_directory): - vocab_file = os.path.join(save_directory, - self.vocab_files_names["vocab_file"]) - else: - vocab_file = save_directory - - with open(self.vocab_file, 'rb') as fin: - proto_str = fin.read() - - with open(vocab_file, "wb") as writer: - writer.write(proto_str) - - return (vocab_file, ) - - def build_inputs_with_special_tokens( - self, - token_ids_0: List[int], - token_ids_1: Optional[List[int]] = None) -> List[int]: - """ - Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and - adding special tokens. A BERT sequence has the following format: - - - single sequence: `[CLS] X [SEP]` - - pair of sequences: `[CLS] A [SEP] B [SEP]` - - Args: - token_ids_0 (`List[int]`): - List of IDs to which the special tokens will be added. - token_ids_1 (`List[int]`, *optional*): - Optional second list of IDs for sequence pairs. - - Returns: - `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. - """ - gmask_id = self.sp_tokenizer[self.gmask_token] - eos_id = self.sp_tokenizer[self.eos_token] - token_ids_0 = token_ids_0 + [ - gmask_id, self.sp_tokenizer[self.bos_token] - ] - if token_ids_1 is not None: - token_ids_0 = token_ids_0 + token_ids_1 + [eos_id] - return token_ids_0 - - def _pad( - self, - encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], - max_length: Optional[int] = None, - padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, - pad_to_multiple_of: Optional[int] = None, - return_attention_mask: Optional[bool] = None, - padding_side: str = "left", # Fix for new transformers - ) -> dict: - """ - Pad encoded inputs (on left/right and up to predefined length or max length in the batch) - - Args: - encoded_inputs: - Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). - max_length: maximum length of the returned list and optionally padding length (see below). - Will truncate by taking into account the special tokens. - padding_strategy: PaddingStrategy to use for padding. - - - PaddingStrategy.LONGEST Pad to the longest sequence in the batch - - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) - - PaddingStrategy.DO_NOT_PAD: Do not pad - The tokenizer padding sides are defined in self.padding_side: - - - 'left': pads on the left of the sequences - - 'right': pads on the right of the sequences - pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. - This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability - `>= 7.5` (Volta). - return_attention_mask: - (optional) Set to False to avoid returning attention mask (default: set to model specifics) - """ - # Load from model defaults - bos_token_id = self.sp_tokenizer[self.bos_token] - mask_token_id = self.sp_tokenizer[self.mask_token] - gmask_token_id = self.sp_tokenizer[self.gmask_token] - assert self.padding_side == "left" - - required_input = encoded_inputs[self.model_input_names[0]] - seq_length = len(required_input) - - if padding_strategy == PaddingStrategy.LONGEST: - max_length = len(required_input) - - if max_length is not None and pad_to_multiple_of is not None and ( - max_length % pad_to_multiple_of != 0): - max_length = ( - (max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of - - needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len( - required_input) != max_length - - # Initialize attention mask if not present. - if max_length is not None: - if "attention_mask" not in encoded_inputs: - if bos_token_id in required_input: - context_length = required_input.index(bos_token_id) - else: - context_length = seq_length - attention_mask = np.ones((1, seq_length, seq_length)) - attention_mask = np.tril(attention_mask) - attention_mask[:, :, :context_length] = 1 - attention_mask = np.bool_(attention_mask < 0.5) - encoded_inputs["attention_mask"] = attention_mask - - if "position_ids" not in encoded_inputs: - if bos_token_id in required_input: - context_length = required_input.index(bos_token_id) - else: - context_length = seq_length - position_ids = np.arange(seq_length, dtype=np.int64) - mask_token = mask_token_id if mask_token_id in required_input else gmask_token_id - if mask_token in required_input: - mask_position = required_input.index(mask_token) - position_ids[context_length:] = mask_position - block_position_ids = np.concatenate([ - np.zeros(context_length, dtype=np.int64), - np.arange(1, - seq_length - context_length + 1, - dtype=np.int64) - ]) - encoded_inputs["position_ids"] = np.stack( - [position_ids, block_position_ids], axis=0) - - if needs_to_be_padded: - difference = max_length - len(required_input) - - if "attention_mask" in encoded_inputs: - encoded_inputs["attention_mask"] = np.pad( - encoded_inputs["attention_mask"], - pad_width=[(0, 0), (difference, 0), (difference, 0)], - mode='constant', - constant_values=True) - if "token_type_ids" in encoded_inputs: - encoded_inputs["token_type_ids"] = [ - self.pad_token_type_id - ] * difference + encoded_inputs["token_type_ids"] - if "special_tokens_mask" in encoded_inputs: - encoded_inputs["special_tokens_mask"] = [ - 1 - ] * difference + encoded_inputs["special_tokens_mask"] - if "position_ids" in encoded_inputs: - encoded_inputs["position_ids"] = np.pad( - encoded_inputs["position_ids"], - pad_width=[(0, 0), (difference, 0)]) - encoded_inputs[self.model_input_names[ - 0]] = [self.pad_token_id] * difference + required_input - - return encoded_inputs diff --git a/examples/models/contrib/chatglm2-6b/README.md b/examples/models/contrib/chatglm2-6b/README.md deleted file mode 100644 index c25818e3f95c..000000000000 --- a/examples/models/contrib/chatglm2-6b/README.md +++ /dev/null @@ -1,108 +0,0 @@ -# ChatGLM - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document explains how to build the [ChatGLM2-6B](https://huggingface.co/THUDM/chatglm2-6b), [ChatGLM2-6B-32k](https://huggingface.co/THUDM/chatglm2-6b-32k) models using TensorRT LLM and run on a single GPU, a single node with multiple GPUs or multiple nodes with multiple GPUs. - -- [ChatGLM](#chatglm) - - [Overview](#overview) - - [Support Matrix](#support-matrix) - - [Model comparison](#model-comparison) - - [Tokenizer and special tokens comparison](#tokenizer-and-special-tokens-comparison) - - [Usage](#usage) - - [1. Download repo and weights from HuggingFace Transformers](#1-download-repo-and-weights-from-huggingface-transformers) - - [2. Convert weights from HF Transformers to TensorRT LLM format](#2-convert-weights-from-hf-transformers-to-tensorrt-llm-format) - - [3. Build TensorRT engine(s)](#3-build-tensorrt-engines) - - [Enable plugins](#enable-plugins) - - [In-flight batching](#in-flight-batching) - - [4. Run inference](#4-run-inference) - - [Single node, single GPU](#single-node-single-gpu) - - [Single node, multi GPU](#single-node-multi-gpu) - - [5. Run summarization task](#5-run-summarization-task) - - [Weight Only quantization](#weight-only-quantization) - - [Smooth Quantization (SQ)](#smooth-quantization-sq) - - [Activation-aware Weight Quantization (AWQ)](#activation-aware-weight-quantization-awq) - - [FP8 Quantization](#fp8-quantization) - - [Benchmark](#benchmark) - - -## Overview - -The TensorRT LLM ChatGLM implementation can be found in [`tensorrt_llm/models/chatglm/model.py`](../../tensorrt_llm/models/chatglm/model.py). -The TensorRT LLM ChatGLM example code is located in [`examples/models/contrib/chatglm2-6b`](./). There is one main file: - -* [`examples/models/core/glm-4-9b/convert_checkpoint.py`](../../../glm-4-9b/convert_checkpoint.py) to convert a checkpoint from the [HuggingFace (HF) Transformers](https://github.com/huggingface/transformers) format to the TensorRT LLM format. - -In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: - -* [`../../../run.py`](../../../run.py) to run the inference on an input text; -* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. - -## Support Matrix - -| Model Name | FP16 | FMHA | WO | SQ | AWQ | FP8 | TP | PP | ST | C++ | benchmark | IFB | -| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :-------: | :---: | -| chatglm2_6b | Y | Y | Y | Y | Y | Y | Y | | Y | Y | Y | Y | -| chatglm2_6b_32k | Y | Y | Y | | Y | Y | Y | | Y | Y | Y | Y | - -* Model Name: the name of the model, the same as the name on HuggingFace -* FMHA: Fused MultiHead Attention (see introduction below) -* WO: Weight Only Quantization (int8 / int4) -* SQ: Smooth Quantization (int8) -* AWQ: Activation Aware Weight Quantization (int4) -* FP8: FP8 Quantization -* TP: Tensor Parallel -* PP: Pipeline Parallel -* ST: Strongly Typed -* C++: C++ Runtime -* benchmark: benchmark by python / C++ Runtime -* IFB: In-flight Batching (see introduction below) - -## Model comparison - -| Name | nL | nAH | nKH | nHW | nH | nF | nMSL | nV | bP2D | bBQKV | bBDense | Comments | -| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :----: | :---: | :---: | :-----: | :----------------------------------------------------------------- | -| chatglm2_6b | 28 | 32 | 2 | 128 | 4096 | 13696 | 32768 | 65024 | N | Y | N | Multi_query_attention, RMSNorm rather than LayerNorm in chatglm_6b | -| chatglm2_6b_32k | 28 | 32 | 2 | 128 | 4096 | 13696 | 32768 | 65024 | N | Y | N | RoPE base=160000 rather than 10000 in chatglm2_6b | - -* nL: number of layers -* nAH: number of attention heads -* nKH: number of kv heads (less than nAH if multi_query_attention is used) -* nHW: head width -* nH: hidden size -* nF: FFN hidden size -* nMSL: max sequence length (input + output) -* nV: vocabulary size -* bP2D: use position_encoding_2d (Y: Yes, N: No) -* bBQKV: use bias for QKV multiplication in self-attention -* bBDense: use bias for Dense multiplication in self-attention - -## Tokenizer and special tokens comparison - -| Name | Tokenizer | bos | eos | pad | cls | startofpiece | endofpiece | mask | smask | gmask | -| :--------------: | :--------------: | :----: | :----: | :---: | :---: | :----------: | :--------: | :----: | :---: | :----: | -| chatglm2_6b | ChatGLMTokenizer | 1 | 2 | 0 | | | | | | | -| chatglm2_6b_32k | ChatGLMTokenizer | 1 | 2 | 0 | | | | | | | - -## Usage - -The next section describe how to build the engine and run the inference demo. - -### 1. Download repo and weights from HuggingFace Transformers - -```bash -pip install -r requirements.txt -apt-get update -apt-get install git-lfs -rm -rf chatglm* - -# clone one or more models we want to build -git clone https://huggingface.co/THUDM/chatglm2-6b chatglm2_6b -git clone https://huggingface.co/THUDM/chatglm2-6b-32k chatglm2_6b_32k -``` - -For more example codes, please refer to the [examples/models/core/glm-4-9b/README.md](../../../glm-4-9b/README.md). diff --git a/examples/models/contrib/chatglm2-6b/requirements.txt b/examples/models/contrib/chatglm2-6b/requirements.txt deleted file mode 100644 index cdc65bf2bb38..000000000000 --- a/examples/models/contrib/chatglm2-6b/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.0.dev0 -datasets==3.1.0 -evaluate -protobuf -rouge_score -sentencepiece -tiktoken diff --git a/examples/models/contrib/chatglm2-6b/tokenization_chatglm.py b/examples/models/contrib/chatglm2-6b/tokenization_chatglm.py deleted file mode 100644 index dea5fdc7c7dc..000000000000 --- a/examples/models/contrib/chatglm2-6b/tokenization_chatglm.py +++ /dev/null @@ -1,282 +0,0 @@ -import os -from typing import Dict, List, Optional, Union - -from sentencepiece import SentencePieceProcessor -from transformers import PreTrainedTokenizer -from transformers.tokenization_utils_base import BatchEncoding, EncodedInput -from transformers.utils import PaddingStrategy - - -class SPTokenizer: - - def __init__(self, model_path: str): - # reload tokenizer - assert os.path.isfile(model_path), model_path - self.sp_model = SentencePieceProcessor(model_file=model_path) - - # BOS / EOS token IDs - self.n_words: int = self.sp_model.vocab_size() - self.bos_id: int = self.sp_model.bos_id() - self.eos_id: int = self.sp_model.eos_id() - self.pad_id: int = self.sp_model.unk_id() - assert self.sp_model.vocab_size() == self.sp_model.get_piece_size() - - special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"] - self.special_tokens = {} - self.index_special_tokens = {} - for token in special_tokens: - self.special_tokens[token] = self.n_words - self.index_special_tokens[self.n_words] = token - self.n_words += 1 - - def tokenize(self, s: str): - return self.sp_model.EncodeAsPieces(s) - - def encode(self, s: str, bos: bool = False, eos: bool = False) -> List[int]: - assert type(s) is str - t = self.sp_model.encode(s) - if bos: - t = [self.bos_id] + t - if eos: - t = t + [self.eos_id] - return t - - def decode(self, t: List[int]) -> str: - return self.sp_model.decode(t) - - def decode_tokens(self, tokens: List[str]) -> str: - text = self.sp_model.DecodePieces(tokens) - return text - - def convert_token_to_id(self, token): - """ Converts a token (str) in an id using the vocab. """ - if token in self.special_tokens: - return self.special_tokens[token] - return self.sp_model.PieceToId(token) - - def convert_id_to_token(self, index): - """Converts an index (integer) in a token (str) using the vocab.""" - if index in self.index_special_tokens or index in [ - self.eos_id, self.bos_id, self.pad_id - ] or index < 0: - return "" - return self.sp_model.IdToPiece(index) - - -class ChatGLMTokenizer(PreTrainedTokenizer): - vocab_files_names = {"vocab_file": "tokenizer.model"} - - model_input_names = ["input_ids", "attention_mask", "position_ids"] - - def __init__(self, - vocab_file, - padding_side="left", - clean_up_tokenization_spaces=False, - **kwargs): - self.name = "GLMTokenizer" - - self.vocab_file = vocab_file - self.tokenizer = SPTokenizer(vocab_file) - self.special_tokens = { - "": self.tokenizer.bos_id, - "": self.tokenizer.eos_id, - "": self.tokenizer.pad_id - } - super().__init__( - padding_side=padding_side, - clean_up_tokenization_spaces=clean_up_tokenization_spaces, - **kwargs) - - def get_command(self, token): - if token in self.special_tokens: - return self.special_tokens[token] - assert token in self.tokenizer.special_tokens, f"{token} is not a special token for {self.name}" - return self.tokenizer.special_tokens[token] - - @property - def unk_token(self) -> str: - return "" - - @property - def pad_token(self) -> str: - return "" - - @property - def pad_token_id(self): - return self.get_command("") - - @property - def eos_token(self) -> str: - return "" - - @property - def eos_token_id(self): - return self.get_command("") - - @property - def vocab_size(self): - return self.tokenizer.n_words - - def get_vocab(self): - """ Returns vocab as a dict """ - vocab = { - self._convert_id_to_token(i): i - for i in range(self.vocab_size) - } - vocab.update(self.added_tokens_encoder) - return vocab - - def _tokenize(self, text, **kwargs): - return self.tokenizer.tokenize(text) - - def _convert_token_to_id(self, token): - """ Converts a token (str) in an id using the vocab. """ - return self.tokenizer.convert_token_to_id(token) - - def _convert_id_to_token(self, index): - """Converts an index (integer) in a token (str) using the vocab.""" - return self.tokenizer.convert_id_to_token(index) - - def convert_tokens_to_string(self, tokens: List[str]) -> str: - return self.tokenizer.decode_tokens(tokens) - - def save_vocabulary(self, save_directory, filename_prefix=None): - """ - Save the vocabulary and special tokens file to a directory. - - Args: - save_directory (`str`): - The directory in which to save the vocabulary. - filename_prefix (`str`, *optional*): - An optional prefix to add to the named of the saved files. - - Returns: - `Tuple(str)`: Paths to the files saved. - """ - if os.path.isdir(save_directory): - vocab_file = os.path.join(save_directory, - self.vocab_files_names["vocab_file"]) - else: - vocab_file = save_directory - - with open(self.vocab_file, 'rb') as fin: - proto_str = fin.read() - - with open(vocab_file, "wb") as writer: - writer.write(proto_str) - - return (vocab_file, ) - - def get_prefix_tokens(self): - prefix_tokens = [self.get_command("[gMASK]"), self.get_command("sop")] - return prefix_tokens - - def build_prompt(self, query, history=None): - if history is None: - history = [] - prompt = "" - for i, (old_query, response) in enumerate(history): - prompt += "[Round {}]\n\n问：{}\n\n答：{}\n\n".format( - i + 1, old_query, response) - prompt += "[Round {}]\n\n问：{}\n\n答：".format(len(history) + 1, query) - return prompt - - def build_inputs_with_special_tokens( - self, - token_ids_0: List[int], - token_ids_1: Optional[List[int]] = None) -> List[int]: - """ - Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and - adding special tokens. A BERT sequence has the following format: - - - single sequence: `[CLS] X [SEP]` - - pair of sequences: `[CLS] A [SEP] B [SEP]` - - Args: - token_ids_0 (`List[int]`): - List of IDs to which the special tokens will be added. - token_ids_1 (`List[int]`, *optional*): - Optional second list of IDs for sequence pairs. - - Returns: - `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. - """ - prefix_tokens = self.get_prefix_tokens() - token_ids_0 = prefix_tokens + token_ids_0 - if token_ids_1 is not None: - token_ids_0 = token_ids_0 + token_ids_1 + [ - self.get_command("") - ] - return token_ids_0 - - def _pad( - self, - encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], - max_length: Optional[int] = None, - padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, - pad_to_multiple_of: Optional[int] = None, - return_attention_mask: Optional[bool] = None, - padding_side: str = "left", # Fix for new transformers - ) -> dict: - """ - Pad encoded inputs (on left/right and up to predefined length or max length in the batch) - - Args: - encoded_inputs: - Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). - max_length: maximum length of the returned list and optionally padding length (see below). - Will truncate by taking into account the special tokens. - padding_strategy: PaddingStrategy to use for padding. - - - PaddingStrategy.LONGEST Pad to the longest sequence in the batch - - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) - - PaddingStrategy.DO_NOT_PAD: Do not pad - The tokenizer padding sides are defined in self.padding_side: - - - 'left': pads on the left of the sequences - - 'right': pads on the right of the sequences - pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. - This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability - `>= 7.5` (Volta). - return_attention_mask: - (optional) Set to False to avoid returning attention mask (default: set to model specifics) - """ - # Load from model defaults - assert self.padding_side == "left" - - required_input = encoded_inputs[self.model_input_names[0]] - seq_length = len(required_input) - - if padding_strategy == PaddingStrategy.LONGEST: - max_length = len(required_input) - - if max_length is not None and pad_to_multiple_of is not None and ( - max_length % pad_to_multiple_of != 0): - max_length = ( - (max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of - - needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len( - required_input) != max_length - - # Initialize attention mask if not present. - if "attention_mask" not in encoded_inputs: - encoded_inputs["attention_mask"] = [1] * seq_length - - if "position_ids" not in encoded_inputs: - encoded_inputs["position_ids"] = list(range(seq_length)) - - if needs_to_be_padded: - difference = max_length - len(required_input) - - if "attention_mask" in encoded_inputs: - encoded_inputs["attention_mask"] = [ - 0 - ] * difference + encoded_inputs["attention_mask"] - if "position_ids" in encoded_inputs: - encoded_inputs["position_ids"] = [ - 0 - ] * difference + encoded_inputs["position_ids"] - encoded_inputs[self.model_input_names[ - 0]] = [self.pad_token_id] * difference + required_input - - return encoded_inputs diff --git a/examples/models/contrib/chatglm3-6b-32k/README.md b/examples/models/contrib/chatglm3-6b-32k/README.md deleted file mode 100644 index 0636cba34f9c..000000000000 --- a/examples/models/contrib/chatglm3-6b-32k/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# ChatGLM - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document explains how to build the [ChatGLM3-6B](https://huggingface.co/THUDM/chatglm3-6b), [ChatGLM3-6B-Base](https://huggingface.co/THUDM/chatglm3-6b-base), [ChatGLM3-6B-32k](https://huggingface.co/THUDM/chatglm3-6b-32k) models using TensorRT LLM and run on a single GPU, a single node with multiple GPUs or multiple nodes with multiple GPUs. - -- [ChatGLM](#chatglm) - - [Overview](#overview) - - [Support Matrix](#support-matrix) - - [Model comparison](#model-comparison) - - [Tokenizer and special tokens comparison](#tokenizer-and-special-tokens-comparison) - - [Usage](#usage) - - [1. Download repo and weights from HuggingFace Transformers](#1-download-repo-and-weights-from-huggingface-transformers) - - [2. Convert weights from HF Transformers to TensorRT LLM format](#2-convert-weights-from-hf-transformers-to-tensorrt-llm-format) - - [3. Build TensorRT engine(s)](#3-build-tensorrt-engines) - - [Enable plugins](#enable-plugins) - - [In-flight batching](#in-flight-batching) - - [4. Run inference](#4-run-inference) - - [Single node, single GPU](#single-node-single-gpu) - - [Single node, multi GPU](#single-node-multi-gpu) - - [5. Run summarization task](#5-run-summarization-task) - - [Weight Only quantization](#weight-only-quantization) - - [Smooth Quantization (SQ)](#smooth-quantization-sq) - - [Activation-aware Weight Quantization (AWQ)](#activation-aware-weight-quantization-awq) - - [FP8 Quantization](#fp8-quantization) - - [Benchmark](#benchmark) - - -## Overview - -The TensorRT LLM ChatGLM implementation can be found in [`tensorrt_llm/models/chatglm/model.py`](../../tensorrt_llm/models/chatglm/model.py). -The TensorRT LLM ChatGLM example code is located in [`examples/models/contrib/chatglm3-6b-32k`](./). There is one main file: - -* [`examples/models/core/glm-4-9b/convert_checkpoint.py`](../../../glm-4-9b/convert_checkpoint.py) to convert a checkpoint from the [HuggingFace (HF) Transformers](https://github.com/huggingface/transformers) format to the TensorRT LLM format. - -In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: - -* [`../../../run.py`](../../../run.py) to run the inference on an input text; -* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. - -## Support Matrix - -| Model Name | FP16 | FMHA | WO | SQ | AWQ | FP8 | TP | PP | ST | C++ | benchmark | IFB | -| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :-------: | :---: | -| chatglm3_6b | Y | Y | Y | Y | Y | Y | Y | | Y | Y | Y | Y | -| chatglm3_6b_base | Y | Y | Y | Y | Y | Y | Y | | Y | Y | Y | Y | -| chatglm3_6b_32k | Y | Y | Y | Y | Y | Y | Y | | Y | Y | Y | Y | - -* Model Name: the name of the model, the same as the name on HuggingFace -* FMHA: Fused MultiHead Attention (see introduction below) -* WO: Weight Only Quantization (int8 / int4) -* SQ: Smooth Quantization (int8) -* AWQ: Activation Aware Weight Quantization (int4) -* FP8: FP8 Quantization -* TP: Tensor Parallel -* PP: Pipeline Parallel -* ST: Strongly Typed -* C++: C++ Runtime -* benchmark: benchmark by python / C++ Runtime -* IFB: In-flight Batching (see introduction below) - -## Model comparison - -| Name | nL | nAH | nKH | nHW | nH | nF | nMSL | nV | bP2D | bBQKV | bBDense | Comments | -| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :----: | :---: | :---: | :-----: | :----------------------------------------------------------------- | -| chatglm3_6b | 28 | 32 | 2 | 128 | 4096 | 13696 | 8192 | 65024 | N | Y | N | Different in preprocess and postprocess than chatglm2_6b | -| chatglm3_6b_base | 28 | 32 | 2 | 128 | 4096 | 13696 | 32768 | 65024 | N | Y | N | | -| chatglm3_6b_32k | 28 | 32 | 2 | 128 | 4096 | 13696 | 32768 | 65024 | N | Y | N | RoPE base=500000 rather than 10000 in chatglm3_6b | - -* nL: number of layers -* nAH: number of attention heads -* nKH: number of kv heads (less than nAH if multi_query_attention is used) -* nHW: head width -* nH: hidden size -* nF: FFN hidden size -* nMSL: max sequence length (input + output) -* nV: vocabulary size -* bP2D: use position_encoding_2d (Y: Yes, N: No) -* bBQKV: use bias for QKV multiplication in self-attention -* bBDense: use bias for Dense multiplication in self-attention - -## Tokenizer and special tokens comparison - -| Name | Tokenizer | bos | eos | pad | cls | startofpiece | endofpiece | mask | smask | gmask | -| :--------------: | :--------------: | :----: | :----: | :---: | :---: | :----------: | :--------: | :----: | :---: | :----: | -| chatglm3_6b | ChatGLMTokenizer | 1 | 2 | 0 | | | | 130000 | | | -| chatglm3_6b_base | ChatGLMTokenizer | 1 | 2 | 0 | | | | 130000 | | | -| chatglm3_6b_32k | ChatGLMTokenizer | 1 | 2 | 0 | | | | 130000 | | | - -## Usage - -The next section describe how to build the engine and run the inference demo. - -### 1. Download repo and weights from HuggingFace Transformers - -```bash -pip install -r requirements.txt -apt-get update -apt-get install git-lfs -rm -rf chatglm* - -# clone one or more models we want to build -git clone https://huggingface.co/THUDM/chatglm3-6b chatglm3_6b -git clone https://huggingface.co/THUDM/chatglm3-6b-base chatglm3_6b_base -git clone https://huggingface.co/THUDM/chatglm3-6b-32k chatglm3_6b_32k -``` - -For more example codes, please refer to the [examples/models/core/glm-4-9b/README.md](../../../glm-4-9b/README.md). diff --git a/examples/models/contrib/chatglm3-6b-32k/requirements.txt b/examples/models/contrib/chatglm3-6b-32k/requirements.txt deleted file mode 100644 index cdc65bf2bb38..000000000000 --- a/examples/models/contrib/chatglm3-6b-32k/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.0.dev0 -datasets==3.1.0 -evaluate -protobuf -rouge_score -sentencepiece -tiktoken diff --git a/examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py b/examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py deleted file mode 100644 index 67a3d52442a6..000000000000 --- a/examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py +++ /dev/null @@ -1,313 +0,0 @@ -import json -import os -from typing import Dict, List, Optional, Union - -from sentencepiece import SentencePieceProcessor -from transformers import PreTrainedTokenizer -from transformers.tokenization_utils_base import BatchEncoding, EncodedInput -from transformers.utils import PaddingStrategy - - -class SPTokenizer: - - def __init__(self, model_path: str): - # reload tokenizer - assert os.path.isfile(model_path), model_path - self.sp_model = SentencePieceProcessor(model_file=model_path) - - # BOS / EOS token IDs - self.n_words: int = self.sp_model.vocab_size() - self.bos_id: int = self.sp_model.bos_id() - self.eos_id: int = self.sp_model.eos_id() - self.pad_id: int = self.sp_model.unk_id() - assert self.sp_model.vocab_size() == self.sp_model.get_piece_size() - - special_tokens = [ - "[MASK]", "[gMASK]", "[sMASK]", "sop", "eop", "<|system|>", - "<|user|>", "<|assistant|>", "<|observation|>" - ] - self.special_tokens = {} - self.index_special_tokens = {} - for token in special_tokens: - self.special_tokens[token] = self.n_words - self.index_special_tokens[self.n_words] = token - self.n_words += 1 - - def tokenize(self, s: str): - return self.sp_model.EncodeAsPieces(s) - - def encode(self, s: str, bos: bool = False, eos: bool = False) -> List[int]: - assert type(s) is str - t = self.sp_model.encode(s) - if bos: - t = [self.bos_id] + t - if eos: - t = t + [self.eos_id] - return t - - def decode(self, t: List[int]) -> str: - text, buffer = "", [] - for token in t: - if token in self.index_special_tokens: - if buffer: - text += self.sp_model.decode(buffer) - buffer = [] - text += self.index_special_tokens[token] - else: - buffer.append(token) - if buffer: - text += self.sp_model.decode(buffer) - return text - - def decode_tokens(self, tokens: List[str]) -> str: - text = self.sp_model.DecodePieces(tokens) - return text - - def convert_token_to_id(self, token): - """ Converts a token (str) in an id using the vocab. """ - if token in self.special_tokens: - return self.special_tokens[token] - return self.sp_model.PieceToId(token) - - def convert_id_to_token(self, index): - """Converts an index (integer) in a token (str) using the vocab.""" - if index in self.index_special_tokens: - return self.index_special_tokens[index] - if index in [self.eos_id, self.bos_id, self.pad_id] or index < 0: - return "" - return self.sp_model.IdToPiece(index) - - -class ChatGLMTokenizer(PreTrainedTokenizer): - vocab_files_names = {"vocab_file": "tokenizer.model"} - - model_input_names = ["input_ids", "attention_mask", "position_ids"] - - def __init__(self, - vocab_file, - padding_side="left", - clean_up_tokenization_spaces=False, - **kwargs): - self.name = "GLMTokenizer" - - self.vocab_file = vocab_file - self.tokenizer = SPTokenizer(vocab_file) - self.special_tokens = { - "": self.tokenizer.bos_id, - "": self.tokenizer.eos_id, - "": self.tokenizer.pad_id - } - super().__init__( - padding_side=padding_side, - clean_up_tokenization_spaces=clean_up_tokenization_spaces, - **kwargs) - - def get_command(self, token): - if token in self.special_tokens: - return self.special_tokens[token] - assert token in self.tokenizer.special_tokens, f"{token} is not a special token for {self.name}" - return self.tokenizer.special_tokens[token] - - @property - def unk_token(self) -> str: - return "" - - @property - def pad_token(self) -> str: - return "" - - @property - def pad_token_id(self): - return self.get_command("") - - @property - def eos_token(self) -> str: - return "" - - @property - def eos_token_id(self): - return self.get_command("") - - @property - def vocab_size(self): - return self.tokenizer.n_words - - def get_vocab(self): - """ Returns vocab as a dict """ - vocab = { - self._convert_id_to_token(i): i - for i in range(self.vocab_size) - } - vocab.update(self.added_tokens_encoder) - return vocab - - def _tokenize(self, text, **kwargs): - return self.tokenizer.tokenize(text) - - def _convert_token_to_id(self, token): - """ Converts a token (str) in an id using the vocab. """ - return self.tokenizer.convert_token_to_id(token) - - def _convert_id_to_token(self, index): - """Converts an index (integer) in a token (str) using the vocab.""" - return self.tokenizer.convert_id_to_token(index) - - def convert_tokens_to_string(self, tokens: List[str]) -> str: - return self.tokenizer.decode_tokens(tokens) - - def save_vocabulary(self, save_directory, filename_prefix=None): - """ - Save the vocabulary and special tokens file to a directory. - - Args: - save_directory (`str`): - The directory in which to save the vocabulary. - filename_prefix (`str`, *optional*): - An optional prefix to add to the named of the saved files. - - Returns: - `Tuple(str)`: Paths to the files saved. - """ - if os.path.isdir(save_directory): - vocab_file = os.path.join(save_directory, - self.vocab_files_names["vocab_file"]) - else: - vocab_file = save_directory - - with open(self.vocab_file, 'rb') as fin: - proto_bytes = fin.read() - - with open(vocab_file, "wb") as writer: - writer.write(proto_bytes) - - return (vocab_file, ) - - def get_prefix_tokens(self): - prefix_tokens = [self.get_command("[gMASK]"), self.get_command("sop")] - return prefix_tokens - - def build_single_message(self, role, metadata, message): - assert role in ["system", "user", "assistant", "observation"], role - role_tokens = [self.get_command(f"<|{role}|>") - ] + self.tokenizer.encode(f"{metadata}\n") - message_tokens = self.tokenizer.encode(message) - tokens = role_tokens + message_tokens - return tokens - - def build_chat_input(self, query, history=None, role="user"): - if history is None: - history = [] - input_ids = [] - for item in history: - content = item["content"] - if item["role"] == "system" and "tools" in item: - content = content + "\n" + json.dumps( - item["tools"], indent=4, ensure_ascii=False) - input_ids.extend( - self.build_single_message(item["role"], - item.get("metadata", ""), content)) - input_ids.extend(self.build_single_message(role, "", query)) - input_ids.extend([self.get_command("<|assistant|>")]) - return self.batch_encode_plus([input_ids], - return_tensors="pt", - is_split_into_words=True) - - def build_inputs_with_special_tokens( - self, - token_ids_0: List[int], - token_ids_1: Optional[List[int]] = None) -> List[int]: - """ - Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and - adding special tokens. A BERT sequence has the following format: - - - single sequence: `[CLS] X [SEP]` - - pair of sequences: `[CLS] A [SEP] B [SEP]` - - Args: - token_ids_0 (`List[int]`): - List of IDs to which the special tokens will be added. - token_ids_1 (`List[int]`, *optional*): - Optional second list of IDs for sequence pairs. - - Returns: - `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. - """ - prefix_tokens = self.get_prefix_tokens() - token_ids_0 = prefix_tokens + token_ids_0 - if token_ids_1 is not None: - token_ids_0 = token_ids_0 + token_ids_1 + [ - self.get_command("") - ] - return token_ids_0 - - def _pad( - self, - encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], - max_length: Optional[int] = None, - padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, - pad_to_multiple_of: Optional[int] = None, - return_attention_mask: Optional[bool] = None, - padding_side: str = "left", # Fix for new transformers - ) -> dict: - """ - Pad encoded inputs (on left/right and up to predefined length or max length in the batch) - - Args: - encoded_inputs: - Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). - max_length: maximum length of the returned list and optionally padding length (see below). - Will truncate by taking into account the special tokens. - padding_strategy: PaddingStrategy to use for padding. - - - PaddingStrategy.LONGEST Pad to the longest sequence in the batch - - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) - - PaddingStrategy.DO_NOT_PAD: Do not pad - The tokenizer padding sides are defined in self.padding_side: - - - 'left': pads on the left of the sequences - - 'right': pads on the right of the sequences - pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. - This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability - `>= 7.5` (Volta). - return_attention_mask: - (optional) Set to False to avoid returning attention mask (default: set to model specifics) - """ - # Load from model defaults - assert self.padding_side == "left" - - required_input = encoded_inputs[self.model_input_names[0]] - seq_length = len(required_input) - - if padding_strategy == PaddingStrategy.LONGEST: - max_length = len(required_input) - - if max_length is not None and pad_to_multiple_of is not None and ( - max_length % pad_to_multiple_of != 0): - max_length = ( - (max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of - - needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len( - required_input) != max_length - - # Initialize attention mask if not present. - if "attention_mask" not in encoded_inputs: - encoded_inputs["attention_mask"] = [1] * seq_length - - if "position_ids" not in encoded_inputs: - encoded_inputs["position_ids"] = list(range(seq_length)) - - if needs_to_be_padded: - difference = max_length - len(required_input) - - if "attention_mask" in encoded_inputs: - encoded_inputs["attention_mask"] = [ - 0 - ] * difference + encoded_inputs["attention_mask"] - if "position_ids" in encoded_inputs: - encoded_inputs["position_ids"] = [ - 0 - ] * difference + encoded_inputs["position_ids"] - encoded_inputs[self.model_input_names[ - 0]] = [self.pad_token_id] * difference + required_input - - return encoded_inputs diff --git a/examples/models/contrib/internlm/.gitignore b/examples/models/contrib/internlm/.gitignore deleted file mode 100644 index 7ce339719a3e..000000000000 --- a/examples/models/contrib/internlm/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -internlm* -tokenizer.model diff --git a/examples/models/contrib/internlm/README.md b/examples/models/contrib/internlm/README.md deleted file mode 100644 index f44180f01aad..000000000000 --- a/examples/models/contrib/internlm/README.md +++ /dev/null @@ -1,320 +0,0 @@ -# InternLM - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document shows how to build and run InternLM 7B / 20B models in TensorRT LLM on both single GPU, single node multi-GPU and multi-node multi-GPU. - -- [InternLM](#internlm) - - [Overview](#overview) - - [Support Matrix](#support-matrix) - - [Usage](#usage) - - [Build TensorRT engine(s)](#build-tensorrt-engines) - - [INT8 weight only + INT8 KV cache](#int8-weight-only--int8-kv-cache) - - [SmoothQuant](#smoothquant) - - [Run](#run) - - [Summarization using the InternLM model](#summarization-using-the-internlm-model) - -## Overview - -The TensorRT LLM InternLM implementation is based on the LLaMA model. The implementation can -be found in [tensorrt_llm/models/llama/model.py](../../../../tensorrt_llm/models/llama/model.py). -The TensorRT LLM InternLM example code lies in [`examples/models/contrib/internlm`](./): - -* [`convert_checkpoint.py`](../../core/llama/convert_checkpoint.py) converts the Huggingface Model of InternLM into TensorRT LLM checkpoint. -* [`convert_checkpoint.py`] to to convert a checkpoint from the [HuggingFace (HF) Transformers](https://github.com/huggingface/transformers) format to the TensorRT LLM format - -In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: - -* [`../../../run.py`](../../../run.py) to run the inference on an input text; -* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. - -## Support Matrix - * FP16 / BF16 - * INT8 & INT4 Weight-Only - * Smooth Quant - * INT8 KV Cache - * Tensor Parallel & Pipeline Parallel - -## Usage - -The TensorRT LLM InternLM example code locates at [examples/models/contrib/internlm](./). It takes HF weights as input, and builds the corresponding TensorRT engines. The number of TensorRT engines depends on the number of GPUs used to run inference. - -### Build TensorRT engine(s) - -Please install required packages first: - -```bash -pip install -r requirements.txt -``` - -TensorRT LLM InternLM builds TensorRT engine(s) from HF checkpoint. If no checkpoint directory is specified, TensorRT LLM will build engine(s) with dummy weights. - -InternLM has released several checkpoints of different size or capabilities under https://huggingface.co/internlm. Users can pick any one repository and follow instructions to prepare the checkpoint. - -Below examples use [internlm-chat-7b](https://huggingface.co/internlm/internlm-chat-7b) and [internlm-chat-20b](https://huggingface.co/internlm/internlm-chat-20b) and assume these repositories are cloned or linked under this directory, for example `./internlm-chat-7b/`. - -Normally `trtllm-build` only requires single GPU, but if you've already got all the GPUs needed for inference, you could enable parallel building to make the engine building process faster by adding `--workers` argument. Please note that currently `--workers` feature only supports single node. - -Here're some examples: - -```bash -# Build a single-GPU float16 engine from HF weights. -# gpt_attention_plugin is necessary in InternLM. -# Try use_gemm_plugin to prevent accuracy issue. -cd examples/models/core/llama - -# Convert the InternLM 7B model using a single GPU and FP16. -python convert_checkpoint.py --model_dir ./internlm-chat-7b/ \ - --dtype float16 \ - --output_dir ./internlm-chat-7b/trt_engines/fp16/1-gpu/ -# Note: setting `--dtype bfloat16` to use bfloat16 precision. - -# BUild the InternLM 7B model using a single GPU -trtllm-build --checkpoint_dir ./internlm-chat-7b/trt_engines/fp16/1-gpu/ \ - --output_dir ./engine_outputs \ - --gemm_plugin float16 - -# Convert the InternLM 7B model using a single GPU and apply INT8 weight-only quantization.. -python convert_checkpoint.py --model_dir ./internlm-chat-7b/ \ - --dtype float16 \ - --output_dir ./internlm-chat-7b/trt_engines/int8/1-gpu/ \ - --use_weight_only \ - --weight_only_precision int8 - -trtllm-build --checkpoint_dir ./internlm-chat-7b/trt_engines/int8/1-gpu/ \ - --output_dir ./engine_outputs \ - --gemm_plugin float16 - -# Note: setting `--weight_only_precision int4` to use INT4 weight-only quantization - -# Build InternLM 7B using 2-way tensor parallelism. -python convert_checkpoint.py --model_dir ./internlm-chat-7b/ \ - --dtype float16 \ - --output_dir ./internlm-chat-7b/trt_engines/fp16/2-gpu/ \ - --tp_size 2 - -trtllm-build --checkpoint_dir ./internlm-chat-7b/trt_engines/fp16/2-gpu/ \ - --output_dir ./engine_outputs \ - --gemm_plugin float16 - -# Build InternLM 20B using 2-way tensor parallelism. -python convert_checkpoint.py --model_dir ./internlm-chat-20b/ \ - --dtype bfloat16 \ - --output_dir ./internlm-chat-20b/trt_engines/bf16/2-gpu/ \ - --tp_size 2 --workers 2 - -trtllm-build --checkpoint_dir ./internlm-chat-7b/trt_engines/bf16/2-gpu/ \ - --output_dir ./engine_outputs \ - --gpt_attention_plugin bfloat16 \ - --gemm_plugin bfloat16 -``` - -#### INT8 weight only + INT8 KV cache - -For INT8 KV cache, [`convert_checkpoint.py`](./convert_checkpoint.py) features a -`--int8_kv_cache` option. Setting `--int8_kv_cache` will calibrate the model, -and then export the scaling factors needed for INT8 KV cache inference. - - -Example: - -```bash -cd examples/models/core/llama - -# For 7B models -python convert_checkpoint.py --model_dir ./internlm-chat-7b \ - --output_dir ./internlm-chat-7b/smooth_internlm/int8_kv_cache/ \ - --dtype float16 \ - --use_weight_only \ - --weight_only_precision int8 \ - --int8_kv_cache - -# Build 7B model with both INT8 weight-only and INT8 KV cache enabled -trtllm-build --checkpoint_dir ./internlm-chat-7b/smooth_internlm/int8_kv_cache/ \ - --output_dir ./engine_outputs \ - --gemm_plugin float16 \ -``` - - -```bash -cd examples/models/core/llama - -# For 20B models -python convert_checkpoint.py --model_dir ./internlm-chat-20b \ - --output_dir ./internlm-chat-20b/smooth_internlm/int8_kv_cache/ \ - --dtype float16 \ - --use_weight_only \ - --weight_only_precision int8 \ - --int8_kv_cache - -# Build 20B model with both INT8 weight-only and INT8 KV cache enabled -trtllm-build --checkpoint_dir ./internlm-chat-20b/smooth_internlm/int8_kv_cache/ \ - --output_dir ./engine_outputs \ - --gemm_plugin float16 \ -``` - - -Test with `../../../run.py` or `../../../summarize.py`: - -```bash -python ../../../run.py --max_output_len=120 \ - --input_text 'Tell me about yourself.' \ - --tokenizer_dir ./internlm-chat-7b/ \ - --engine_dir ./internlm-chat-7b/trt_engines/int8_kv_cache_weight_only/1-gpu - -python ../../../run.py --max_output_len=120 \ - --input_text 'Tell me about yourself.' \ - --tokenizer_dir ./internlm-chat-20b/ \ - --engine_dir ./internlm-chat-20b/trt_engines/int8_kv_cache_weight_only/1-gpu - -python ../../../summarize.py --test_trt_llm --test_hf \ - --hf_model_dir ./internlm-chat-7b \ - --data_type fp16 \ - --engine_dir ./internlm-chat-7b/trt_engines/int8_kv_cache_weight_only/1-gpu - -python ../../../summarize.py --test_trt_llm --test_hf \ - --hf_model_dir ./internlm-chat-20b \ - --data_type fp16 \ - --engine_dir ./internlm-chat-20b/trt_engines/int8_kv_cache_weight_only/1-gpu -``` - -#### SmoothQuant - -Unlike the FP16 build where the HF weights are processed and loaded into the TensorRT LLM directly, the SmoothQuant needs to load INT8 weights which should be pre-processed before building an engine. - -Example: -```bash -cd examples/models/core/llama - -# For 7B models -python convert_checkpoint.py --model_dir ./internlm-chat-7b --output_dir ./internlm-chat-7b/smooth_internlm/sq0.5/ --dtype float16 --smoothquant 0.5 -# Build the engine -trtllm-build --checkpoint_dir ./internlm-chat-7b/smooth_internlm/sq0.5/ \ - --output_dir ./engine_outputs \ - --gemm_plugin float16 - -# For 20B models -cd examples/models/core/llama - -python convert_checkpoint.py --model_dir ./internlm-chat-20b --output_dir ./internlm-chat-20b/smooth_internlm/sq0.5/ --dtype float16 --smoothquant 0.5 -trtllm-build --checkpoint_dir ./internlm-chat-20b/smooth_internlm/sq0.5/ \ - --output_dir ./engine_outputs \ - --gemm_plugin float16 -``` - -[`convert_checkpoint.py`](./convert_checkpoint.py) add new options for the support of INT8 inference of SmoothQuant models. - -`--smoothquant` is the starting point of INT8 inference. By default, it -will run the model in the _per-tensor_ mode. - -Then, you can add any combination of `--per-token` and `--per-channel` to get the corresponding behaviors. - -Examples of build invocations: - -```bash -# Build model for SmoothQuant in the _per_token_ + _per_channel_ mode -cd examples/models/core/llama - -# 7B model -python convert_checkpoint.py --model_dir ./internlm-chat-7b --output_dir ./internlm-chat-7b/smooth_internlm/sq0.5/ --dtype float16 --smoothquant 0.5 --per_channel --per_token - -# 20B model -python convert_checkpoint.py --model_dir ./internlm-chat-20b --output_dir ./internlm-chat-20b/smooth_internlm/sq0.5/ --dtype float16 --smoothquant 0.5 --per_channel --per_token -``` - - -Test with `../../../run.py` or `../../../summarize.py`: - -```bash -python ../../../run.py --max_output_len=120 \ - --input_text 'Tell me about yourself.' \ - --tokenizer_dir ./internlm-chat-7b/ \ - --engine_dir ./internlm-chat-7b/smooth_internlm/sq0.5/ - -python ../../../run.py --max_output_len=120 \ - --input_text 'Tell me about yourself.' \ - --tokenizer_dir ./internlm-chat-20b/ \ - --engine_dir ./internlm-chat-20b/smooth_internlm/sq0.5/ - -python ../../../summarize.py --test_trt_llm --test_hf \ - --hf_model_dir ./internlm-chat-7b \ - --data_type fp16 \ - --engine_dir ./internlm-chat-7b/smooth_internlm/sq0.5/ - -python ../../../summarize.py --test_trt_llm --test_hf \ - --hf_model_dir ./internlm-chat-20b \ - --data_type fp16 \ - --engine_dir ./internlm-chat-20b/smooth_internlm/sq0.5/ -``` - -### Run - -To run a TensorRT LLM InternLM model using the engines generated by `trtllm-build` - -```bash -# InternLM 7B with fp16 -python ../../../run.py --max_output_len=120 \ - --input_text 'Tell me about yourself.' \ - --tokenizer_dir ./internlm-chat-7b/ \ - --engine_dir=./internlm-chat-7b/trt_engines/fp16/1-gpu/ - -# InternLM 7B with bf16 -python ../../../run.py --max_output_len=120 \ - --input_text 'Tell me about yourself.' \ - --tokenizer_dir ./internlm-chat-7b/ \ - --engine_dir=./internlm-chat-7b/trt_engines/bf16/1-gpu/ - -# InternLM 7B with int8 weight only quantization -python ../../../run.py --max_output_len=120 \ - --input_text 'Tell me about yourself.' \ - --tokenizer_dir ./internlm-chat-7b/ \ - --engine_dir=./internlm-chat-7b/trt_engines/weight_only/1-gpu/ - -# InternLM 7B with fp16 and tensor parallelism -mpirun -n 2 --allow-run-as-root \ - python ../../../run.py --max_output_len=120 \ - --input_text 'Tell me about yourself.' \ - --tokenizer_dir ./internlm-chat-7b/ \ - --engine_dir=./internlm-chat-7b/trt_engines/fp16/2-gpu/ - -# InternLM 20B with fp16 and tensor parallelism and pipeline parallelism -mpirun -n 4 --allow-run-as-root \ - python ../../../run.py --max_output_len=120 \ - --input_text 'Tell me about yourself.' \ - --tokenizer_dir ./internlm-chat-7b/ \ - --engine_dir=./internlm-chat-7b/trt_engines/bf16/4-gpu/ -``` - -### Summarization using the InternLM model - -```bash -# Run summarization using the InternLM 7B model in FP16. -python ../../../summarize.py --test_trt_llm --test_hf \ - --hf_model_dir ./internlm-chat-7b/ \ - --data_type fp16 \ - --engine_dir ./engine_outputs - -# Run summarization using the InternLM 7B model quantized to INT8. -python ../../../summarize.py --test_trt_llm --test_hf \ - --hf_model_dir ./internlm-chat-7b/ \ - --data_type fp16 \ - --engine_dir ./engine_outputs - -# Run summarization using the InternLM 7B model in FP16 using two GPUs. -mpirun -n 2 --allow-run-as-root \ - python ../../../summarize.py --test_trt_llm --test_hf \ - --hf_model_dir ./internlm-chat-7b/ \ - --data_type fp16 \ - --engine_dir ./internlm-chat-7b/trt_engines/fp16/2-gpu/ - -# Run summarization using the InternLM 20B model in BF16 using 4 GPUs. -mpirun -n 4 --allow-run-as-root \ - python ../../../summarize.py --test_trt_llm --test_hf \ - --hf_model_dir ./internlm-chat-20b/ \ - --data_type bf16 \ - --engine_dir ./internlm-chat-20b/trt_engines/bf16/4-gpu/ -``` diff --git a/examples/models/contrib/internlm/requirements.txt b/examples/models/contrib/internlm/requirements.txt deleted file mode 100644 index d9354a133c65..000000000000 --- a/examples/models/contrib/internlm/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.0.dev0 -datasets==3.1.0 -rouge_score -sentencepiece>=0.1.99 -evaluate diff --git a/examples/models/contrib/jais/README.md b/examples/models/contrib/jais/README.md deleted file mode 100644 index d15cfd116962..000000000000 --- a/examples/models/contrib/jais/README.md +++ /dev/null @@ -1,125 +0,0 @@ -# Jais - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document elaborates how to build Jais model to runnable engines on multi-GPU node and perform a summarization task using these engines. - -Currently it has been tested on -- [Jais-13b-chat](https://huggingface.co/core42/jais-13b-chat) -- [Jais-30b-chat-v3](https://huggingface.co/core42/jais-30b-chat-v3) - - -- [Jais](#jais) - - [Overview](#overview) - - [Support Matrix](#support-matrix) - - [Usage](#usage) - - [Build TensorRT engine(s)](#build-tensorrt-engines) - - [Run inference](#run) - -## Overview - -The TensorRT LLM support for Jais is based on the GPT model, the implementation can be found in [tensorrt_llm/models/gpt/model.py](../../../../tensorrt_llm/models/gpt/model.py). Jais model resembles GPT very much except it uses alibi embedding, embedding scale, swiglu, and logits scale, we therefore reuse the [GPT example code](../../../gpt) for Jais, - -* [`convert_checkpoint.py`](../../../gpt/convert_checkpoint.py) to convert the Jais model into TensorRT LLM checkpoint format. - -In addition, there are two shared files in the parent folder [`examples`](../) for inference and evaluation: - -* [`../../../run.py`](../../../run.py) to run the inference on an input text; -* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. - -## Support Matrix -The tested configurations are: - * FP16 - * FP8 - * Inflight Batching - * Tensor Parallel - -## Usage - -This section gives a whole process where we convert HF models, build TensorRT LLM engines and ultimately perform summarization. - -### Build TensorRT engine(s) - -Run the following commands and TRT-LLM will first transforms a HF model into its own checkpoint format, then builds a TRT engine based on the checkpoint - -```bash -# single gpu, dtype float16 for jais-13b-chat -python3 ../../../gpt/convert_checkpoint.py --model_dir core42/jais-13b-chat \ - --dtype float16 \ - --output_dir jais-13b-chat/trt_ckpt/fp16/1-gpu - -# 2-way tensor parallelism for jais-30b-chat-v3 -python3 ../../../gpt/convert_checkpoint.py --model_dir core42/jais-30b-chat-v3 \ - --dtype float16 \ - --tp_size 2 \ - --output_dir jais-30b-chat-v3/trt_ckpt/fp16/2-gpu -``` - -```bash -# Build a single-GPU float16 engine from TensorRT LLM checkpoint for jais-13b-chat -# Enable the special TensorRT LLM GPT Attention plugin (--gpt_attention_plugin) to increase runtime performance. -# It is recommend to use --remove_input_padding along with --gpt_attention_plugin for better performance -trtllm-build --checkpoint_dir jais-13b-chat/trt_ckpt/fp16/1-gpu \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --output_dir jais-13b-chat/trt_engines/fp16/1-gpu - -# Build 2-way tensor parallelism engines from TensorRT LLM checkpoint for jais-30b-chat-v3 -trtllm-build --checkpoint_dir jais-30b-chat-v3/trt_ckpt/fp16/2-gpu \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --output_dir jais-30b-chat-v3/trt_engines/fp16/2-gpu -``` - - -### Run - -The [`../../../run.py`](../../../run.py) script can be used to run inference with the built engine(s). - -```bash -python3 ../../../run.py --engine_dir jais-13b-chat/trt_engines/fp16/1-gpu \ - --tokenizer_dir core42/jais-13b-chat \ - --max_output_len 10 -``` - -If the engines are run successfully, you will see output like: -``` -...... -Input [Text 0]: "Born in north-east France, Soyer trained as a" -Output [Text 0 Beam 0]: " chef in Paris before moving to England in 1816" -``` - -```bash -python3 ../../../run.py --engine_dir jais-13b-chat/trt_engines/fp16/1-gpu \ - --tokenizer_dir core42/jais-13b-chat \ - --max_output_len 8 \ - --input_text "ولد في 1304 ميلادياً ابن بطوطه, لقد ذهب" -``` - -If the engines are run successfully, you will see output like: -``` -..... -Input [Text 0]: "ولد في 1304 ميلادياً ابن بطوطه, لقد ذهب" -Output [Text 0 Beam 0]: " في جميع أنحاء العالم المعروف في ذلك الوقت" -``` - - -To run a 2 TP model you can do the following -```bash -mpirun -np 2 \ - python3 ../../../run.py --engine_dir jais-30b-chat-v3/trt_engines/fp16/2-gpu \ - --tokenizer_dir core42/jais-30b-chat-v3 \ - --max_output_len 30 -``` - -If the engines are run successfully, you will see output like: -``` -Input [Text 0]: "Born in north-east France, Soyer trained as a" -Output [Text 0 Beam 0]: " chef, working in a series of high-end establishments. - -Soyer's career took him to work in a number of establishments across Europe," -``` diff --git a/examples/models/contrib/jais/requirements.txt b/examples/models/contrib/jais/requirements.txt deleted file mode 100644 index 592e01e5ba6d..000000000000 --- a/examples/models/contrib/jais/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.0.dev0 -datasets==3.1.0 -evaluate -rouge_score -SentencePiece>=0.1.99 diff --git a/examples/models/contrib/sdxl/README.md b/examples/models/contrib/sdxl/README.md deleted file mode 100644 index 05d874f5f27a..000000000000 --- a/examples/models/contrib/sdxl/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Stable Diffusion XL - -This document showcases how to build and run the [Stable Diffusion XL (SDXL)](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) model on multiple GPUs using TensorRT-LLM. The community-contributed SDXL example in TRT-LLM is intended solely to showcase distributed inference for high-resolution use cases. For an optimized single-GPU setup in Stable Diffusion inference, please refer to the [TensorRT DemoDiffusion example](https://github.com/NVIDIA/TensorRT/tree/main/demo/Diffusion). - -The design of distributed parallel inference comes from the CVPR 2024 paper [DistriFusion](https://github.com/mit-han-lab/distrifuser) from [MIT HAN Lab](https://hanlab.mit.edu/). To simplify the implementation, all communications in this example are handled synchronously. - -## Usage - -### 1. Build TensorRT Engine - -```bash -# 1 gpu -python build_sdxl_unet.py --size 1024 - -# 2 gpus -mpirun -n 2 --allow-run-as-root python build_sdxl_unet.py --size 1024 -``` - -### 2. Generate images using the engine - - -```bash -# 1 gpu -python run_sdxl.py --size 1024 --prompt "flowers, rabbit" - -# 2 gpus -mpirun -n 2 --allow-run-as-root python run_sdxl.py --size 1024 --prompt "flowers, rabbit" -``` - -## Latency Benchmark -This benchmark is provided as reference points and should not be considered as the peak inference speed that can be delivered by TensorRT-LLM. - -| Framework | Resolution | n_gpu | A100 latency (s) | A100 speedup | H100 latency (s) | H100 speedup | -|:---------:|:----------:|:-----:|:---------------:|:-------------:|:---------------:|:-------------:| -| Torch | 1024x1024 | 1 | 6.280 | 1 | 5.820 | 1 | -| TRT-LLM | 1024x1024 | 2 | 2.803 | **2.24x** | 1.719 | **3.39x** | -| TRT-LLM | 1024x1024 | 4 | 2.962 | **2.12x** | 2.592 | **2.25x** | -| Torch | 2048x2048 | 1 | 27.865 | 1 | 18.330 | 1 | -| TRT-LLM | 2048x2048 | 2 | 13.152 | **2.12x** | 7.943 | **2.31x** | -| TRT-LLM | 2048x2048 | 4 | 9.781 | **2.85x** | 7.596 | **2.41x** | - -torch v2.5.0. TRT-LLM v0.15.0.dev2024102900, `--num-warmup-runs=5; --avg-runs=20`. All communications are synchronous. diff --git a/examples/models/contrib/sdxl/build_sdxl_unet.py b/examples/models/contrib/sdxl/build_sdxl_unet.py deleted file mode 100755 index e2893859f731..000000000000 --- a/examples/models/contrib/sdxl/build_sdxl_unet.py +++ /dev/null @@ -1,148 +0,0 @@ -import argparse -import os - -import tensorrt as trt -import torch -from diffusers import DiffusionPipeline - -import tensorrt_llm -from tensorrt_llm.builder import Builder -from tensorrt_llm.mapping import Mapping -from tensorrt_llm.models.unet.pp.unet_pp import DistriUNetPP -from tensorrt_llm.models.unet.unet_2d_condition import UNet2DConditionModel -from tensorrt_llm.models.unet.weights import load_from_hf_unet -from tensorrt_llm.network import net_guard - -parser = argparse.ArgumentParser(description='build the UNet TensorRT engine.') -parser.add_argument('--model_dir', - type=str, - default='stabilityai/stable-diffusion-xl-base-1.0') -parser.add_argument('--size', type=int, default=1024, help='image size') -parser.add_argument('--output_dir', - type=str, - default=None, - help='output directory') - -args = parser.parse_args() - -model_dir = args.model_dir -size = args.size -sample_size = size // 8 - -world_size = tensorrt_llm.mpi_world_size() -rank = tensorrt_llm.mpi_rank() -output_dir = f'sdxl_s{size}_w{world_size}' if args.output_dir is None else args.output_dir -if rank == 0 and not os.path.exists(output_dir): - os.makedirs(output_dir) - -device_per_batch = world_size // 2 if world_size > 1 else 1 -batch_group = 2 if world_size > 1 else 1 - -# Use tp_size to indicate the size of patch parallelism -# Use pp_size to indicate the size of batch parallelism -mapping = Mapping(world_size=world_size, - rank=rank, - tp_size=device_per_batch, - pp_size=batch_group) - -torch.cuda.set_device(tensorrt_llm.mpi_rank()) - -tensorrt_llm.logger.set_level('verbose') -builder = Builder() -builder_config = builder.create_builder_config( - name='UNet2DConditionModel', - precision='float16', - timing_cache='model.cache', - profiling_verbosity='detailed', - tensor_parallel=world_size, - precision_constraints= - None, # do not use obey or the precision error will be too large -) - -pipeline = DiffusionPipeline.from_pretrained(model_dir, - torch_dtype=torch.float16) -model = UNet2DConditionModel( - sample_size=sample_size, - in_channels=4, - out_channels=4, - center_input_sample=False, - flip_sin_to_cos=True, - freq_shift=0, - down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", - "CrossAttnDownBlock2D"), - up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), - block_out_channels=(320, 640, 1280), - layers_per_block=2, - downsample_padding=1, - mid_block_scale_factor=1.0, - act_fn="silu", - norm_num_groups=32, - norm_eps=1e-5, - cross_attention_dim=2048, - attention_head_dim=[5, 10, 20], - addition_embed_type="text_time", - addition_time_embed_dim=256, - projection_class_embeddings_input_dim=2816, - transformer_layers_per_block=[1, 2, 10], - use_linear_projection=True, - dtype=trt.float16, -) - -load_from_hf_unet(pipeline.unet, model) -model = DistriUNetPP(model, mapping) - -# Module -> Network -network = builder.create_network() -network.plugin_config.to_legacy_setting() -if mapping.world_size > 1: - network.plugin_config.set_nccl_plugin('float16') - -with net_guard(network): - # Prepare - network.set_named_parameters(model.named_parameters()) - - # Forward - sample = tensorrt_llm.Tensor( - name='sample', - dtype=trt.float16, - shape=[2, 4, sample_size, sample_size], - ) - timesteps = tensorrt_llm.Tensor( - name='timesteps', - dtype=trt.float16, - shape=[ - 1, - ], - ) - encoder_hidden_states = tensorrt_llm.Tensor( - name='encoder_hidden_states', - dtype=trt.float16, - shape=[2, 77, 2048], - ) - text_embeds = tensorrt_llm.Tensor( - name='text_embeds', - dtype=trt.float16, - shape=[2, 1280], - ) - time_ids = tensorrt_llm.Tensor( - name='time_ids', - dtype=trt.float16, - shape=[2, 6], - ) - - output = model(sample, timesteps, encoder_hidden_states, text_embeds, - time_ids) - - # Mark outputs - output_dtype = trt.float16 - output.mark_output('pred', output_dtype) - -# Network -> Engine -engine = builder.build_engine(network, builder_config) -assert engine is not None, 'Failed to build engine.' - -engine_name = f'sdxl_unet_s{size}_w{world_size}_r{rank}.engine' -engine_path = os.path.join(output_dir, engine_name) -with open(engine_path, 'wb') as f: - f.write(engine) -builder.save_config(builder_config, os.path.join(output_dir, 'config.json')) diff --git a/examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py b/examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py deleted file mode 100755 index 018f3fad783f..000000000000 --- a/examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py +++ /dev/null @@ -1,1365 +0,0 @@ -# Copyright 2023 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -import json -import os -from typing import Any, Callable, Dict, List, Optional, Tuple, Union - -import tensorrt as trt -import torch -from diffusers.image_processor import PipelineImageInput, VaeImageProcessor -from diffusers.loaders import (FromSingleFileMixin, IPAdapterMixin, - StableDiffusionXLLoraLoaderMixin, - TextualInversionLoaderMixin) -from diffusers.models import AutoencoderKL, UNet2DConditionModel -from diffusers.models.attention_processor import (AttnProcessor2_0, - LoRAAttnProcessor2_0, - LoRAXFormersAttnProcessor, - XFormersAttnProcessor) -from diffusers.models.lora import adjust_lora_scale_text_encoder -from diffusers.pipelines.pipeline_utils import DiffusionPipeline -from diffusers.pipelines.stable_diffusion_xl.pipeline_output import \ - StableDiffusionXLPipelineOutput -from diffusers.schedulers import KarrasDiffusionSchedulers -from diffusers.utils import (USE_PEFT_BACKEND, deprecate, - is_invisible_watermark_available, - is_torch_xla_available, logging, - replace_example_docstring, scale_lora_layers, - unscale_lora_layers) -from diffusers.utils.torch_utils import randn_tensor -from transformers import (CLIPImageProcessor, CLIPTextModel, - CLIPTextModelWithProjection, CLIPTokenizer, - CLIPVisionModelWithProjection) - -import tensorrt_llm -from tensorrt_llm.runtime import Session, TensorInfo - -if is_invisible_watermark_available(): - from diffusers.pipelines.stable_diffusion_xl.watermark import \ - StableDiffusionXLWatermarker - -if is_torch_xla_available(): - import torch_xla.core.xla_model as xm - - XLA_AVAILABLE = True -else: - XLA_AVAILABLE = False - -logger = logging.get_logger(__name__) # pylint: disable=invalid-name - -EXAMPLE_DOC_STRING = """ - Examples: - ```py - >>> import torch - >>> from diffusers import StableDiffusionXLPipeline - - >>> pipe = StableDiffusionXLPipeline.from_pretrained( - ... "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 - ... ) - >>> pipe = pipe.to("cuda") - - >>> prompt = "a photo of an astronaut riding a horse on mars" - >>> image = pipe(prompt).images[0] - ``` -""" - - -# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg -def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): - """ - Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and - Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 - """ - std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), - keepdim=True) - std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) - # rescale the results from guidance (fixes overexposure) - noise_pred_rescaled = noise_cfg * (std_text / std_cfg) - # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images - noise_cfg = guidance_rescale * noise_pred_rescaled + \ - (1 - guidance_rescale) * noise_cfg - return noise_cfg - - -# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps -def retrieve_timesteps( - scheduler, - num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - **kwargs, -): - """ - Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles - custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. - - Args: - scheduler (`SchedulerMixin`): - The scheduler to get timesteps from. - num_inference_steps (`int`): - The number of diffusion steps used when generating samples with a pre-trained model. If used, - `timesteps` must be `None`. - device (`str` or `torch.device`, *optional*): - The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): - Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default - timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` - must be `None`. - - Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the - second element is the number of inference steps. - """ - if timesteps is not None: - accepts_timesteps = "timesteps" in set( - inspect.signature(scheduler.set_timesteps).parameters.keys()) - if not accepts_timesteps: - raise ValueError( - f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" - f" timestep schedules. Please check whether you are using the correct scheduler." - ) - scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) - timesteps = scheduler.timesteps - num_inference_steps = len(timesteps) - else: - scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) - timesteps = scheduler.timesteps - return timesteps, num_inference_steps - - -class StableDiffusionXLPipeline( - DiffusionPipeline, - FromSingleFileMixin, - StableDiffusionXLLoraLoaderMixin, - TextualInversionLoaderMixin, - IPAdapterMixin, -): - r""" - Pipeline for text-to-image generation using Stable Diffusion XL. - - This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the - library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) - - In addition the pipeline inherits the following loading methods: - - *LoRA*: [`loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] - - *Ckpt*: [`loaders.FromSingleFileMixin.from_single_file`] - - as well as the following saving methods: - - *LoRA*: [`loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] - - Args: - vae ([`AutoencoderKL`]): - Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. - text_encoder ([`CLIPTextModel`]): - Frozen text-encoder. Stable Diffusion XL uses the text portion of - [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically - the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. - text_encoder_2 ([` CLIPTextModelWithProjection`]): - Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of - [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), - specifically the - [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) - variant. - tokenizer (`CLIPTokenizer`): - Tokenizer of class - [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). - tokenizer_2 (`CLIPTokenizer`): - Second Tokenizer of class - [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). - unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. - scheduler ([`SchedulerMixin`]): - A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of - [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. - force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): - Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of - `stabilityai/stable-diffusion-xl-base-1-0`. - add_watermarker (`bool`, *optional*): - Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to - watermark output images. If not defined, it will default to True if the package is installed, otherwise no - watermarker will be used. - """ - - model_cpu_offload_seq = "text_encoder->text_encoder_2->unet->vae" - _optional_components = [ - "tokenizer", - "tokenizer_2", - "text_encoder", - "text_encoder_2", - "image_encoder", - "feature_extractor", - ] - _callback_tensor_inputs = [ - "latents", - "prompt_embeds", - "negative_prompt_embeds", - "add_text_embeds", - "add_time_ids", - "negative_pooled_prompt_embeds", - "negative_add_time_ids", - ] - - def __init__( - self, - vae: AutoencoderKL, - text_encoder: CLIPTextModel, - text_encoder_2: CLIPTextModelWithProjection, - tokenizer: CLIPTokenizer, - tokenizer_2: CLIPTokenizer, - unet: UNet2DConditionModel, - scheduler: KarrasDiffusionSchedulers, - image_encoder: CLIPVisionModelWithProjection = None, - feature_extractor: CLIPImageProcessor = None, - force_zeros_for_empty_prompt: bool = True, - add_watermarker: Optional[bool] = None, - ): - super().__init__() - - self.register_modules( - vae=vae, - text_encoder=text_encoder, - text_encoder_2=text_encoder_2, - tokenizer=tokenizer, - tokenizer_2=tokenizer_2, - unet=unet, - scheduler=scheduler, - image_encoder=image_encoder, - feature_extractor=feature_extractor, - ) - self.register_to_config( - force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) - self.vae_scale_factor = 2**(len(self.vae.config.block_out_channels) - 1) - self.image_processor = VaeImageProcessor( - vae_scale_factor=self.vae_scale_factor) - - self.default_sample_size = self.unet.config.sample_size - - add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available( - ) - - if add_watermarker: - self.watermark = StableDiffusionXLWatermarker() - else: - self.watermark = None - - self.execution_device = torch.device('cpu') - self.engine = {} - - def to( - self, - torch_device: Optional[Union[str, torch.device]] = None, - torch_dtype: Optional[torch.dtype] = None, - silence_dtype_warnings: bool = False, - ): - super().to(torch_device) - if isinstance(torch_device, str): - torch_device = torch.device(torch_device) - self.execution_device = torch_device - return self - - def prepare(self, path, size): - self.unet.cpu() - torch.cuda.empty_cache() - - def trt_dtype_to_torch(dtype): - if dtype == trt.float16: - return torch.float16 - elif dtype == trt.float32: - return torch.float32 - elif dtype == trt.int32: - return torch.int32 - else: - raise TypeError("%s is not supported" % dtype) - - config_path = os.path.join(path, 'config.json') - with open(config_path, 'r') as f: - config = json.load(f) - config['builder_config']['precision'] - world_size = config['builder_config']['tensor_parallel'] - - runtime_world_size = tensorrt_llm.mpi_world_size() - assert world_size == runtime_world_size, f'Engine world size ({world_size}) != Runtime world size ({runtime_world_size})' - runtime_rank = tensorrt_llm.mpi_rank() if world_size > 1 else 0 - torch.cuda.set_device(runtime_rank) - - serialize_file = f'sdxl_unet_s{size}_w{world_size}_r{runtime_rank}.engine' - serialize_path = os.path.join(path, serialize_file) - self.stream = torch.cuda.current_stream().cuda_stream - print(f'Loading engine from {serialize_path}') - with open(serialize_path, 'rb') as f: - engine_buffer = f.read() - print(f'Creating session from engine') - self.session = Session.from_serialized_engine(engine_buffer) - - output_info = self.session.infer_shapes([ - TensorInfo('sample', trt.DataType.HALF, - [2, 4, size // 8, size // 8]), - TensorInfo('timesteps', trt.DataType.HALF, [ - 1, - ]), - TensorInfo('encoder_hidden_states', trt.DataType.HALF, - [2, 77, 2048]), - TensorInfo('text_embeds', trt.DataType.HALF, [2, 1280]), - TensorInfo('time_ids', trt.DataType.HALF, [2, 6]), - ]) - self.outputs = { - t.name: - torch.empty(tuple(t.shape), - dtype=trt_dtype_to_torch(t.dtype), - device='cuda') - for t in output_info - } - - # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_slicing - def enable_vae_slicing(self): - r""" - Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to - compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. - """ - self.vae.enable_slicing() - - # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_slicing - def disable_vae_slicing(self): - r""" - Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to - computing decoding in one step. - """ - self.vae.disable_slicing() - - # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_tiling - def enable_vae_tiling(self): - r""" - Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to - compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow - processing larger images. - """ - self.vae.enable_tiling() - - # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_tiling - def disable_vae_tiling(self): - r""" - Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to - computing decoding in one step. - """ - self.vae.disable_tiling() - - def encode_prompt( - self, - prompt: str, - prompt_2: Optional[str] = None, - device: Optional[torch.device] = None, - num_images_per_prompt: int = 1, - do_classifier_free_guidance: bool = True, - negative_prompt: Optional[str] = None, - negative_prompt_2: Optional[str] = None, - prompt_embeds: Optional[torch.FloatTensor] = None, - negative_prompt_embeds: Optional[torch.FloatTensor] = None, - pooled_prompt_embeds: Optional[torch.FloatTensor] = None, - negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, - lora_scale: Optional[float] = None, - clip_skip: Optional[int] = None, - ): - r""" - Encodes the prompt into text encoder hidden states. - - Args: - prompt (`str` or `List[str]`, *optional*): - prompt to be encoded - prompt_2 (`str` or `List[str]`, *optional*): - The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is - used in both text-encoders - device: (`torch.device`): - torch device - num_images_per_prompt (`int`): - number of images that should be generated per prompt - do_classifier_free_guidance (`bool`): - whether to use classifier free guidance or not - negative_prompt (`str` or `List[str]`, *optional*): - The prompt or prompts not to guide the image generation. If not defined, one has to pass - `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is - less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): - The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and - `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders - prompt_embeds (`torch.FloatTensor`, *optional*): - Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not - provided, text embeddings will be generated from `prompt` input argument. - negative_prompt_embeds (`torch.FloatTensor`, *optional*): - Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt - weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input - argument. - pooled_prompt_embeds (`torch.FloatTensor`, *optional*): - Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. - If not provided, pooled text embeddings will be generated from `prompt` input argument. - negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): - Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt - weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` - input argument. - lora_scale (`float`, *optional*): - A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. - clip_skip (`int`, *optional*): - Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that - the output of the pre-final layer will be used for computing the prompt embeddings. - """ - device = self.execution_device - - # set lora scale so that monkey patched LoRA - # function of text encoder can correctly access it - if lora_scale is not None and isinstance( - self, StableDiffusionXLLoraLoaderMixin): - self._lora_scale = lora_scale - - # dynamically adjust the LoRA scale - if self.text_encoder is not None: - if not USE_PEFT_BACKEND: - adjust_lora_scale_text_encoder(self.text_encoder, - lora_scale) - else: - scale_lora_layers(self.text_encoder, lora_scale) - - if self.text_encoder_2 is not None: - if not USE_PEFT_BACKEND: - adjust_lora_scale_text_encoder(self.text_encoder_2, - lora_scale) - else: - scale_lora_layers(self.text_encoder_2, lora_scale) - - prompt = [prompt] if isinstance(prompt, str) else prompt - - if prompt is not None: - batch_size = len(prompt) - else: - batch_size = prompt_embeds.shape[0] - - # Define tokenizers and text encoders - tokenizers = [self.tokenizer, self.tokenizer_2 - ] if self.tokenizer is not None else [self.tokenizer_2] - text_encoders = ([ - self.text_encoder, self.text_encoder_2 - ] if self.text_encoder is not None else [self.text_encoder_2]) - - if prompt_embeds is None: - prompt_2 = prompt_2 or prompt - prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 - - # textual inversion: procecss multi-vector tokens if necessary - prompt_embeds_list = [] - prompts = [prompt, prompt_2] - for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, - text_encoders): - if isinstance(self, TextualInversionLoaderMixin): - prompt = self.maybe_convert_prompt(prompt, tokenizer) - - text_inputs = tokenizer( - prompt, - padding="max_length", - max_length=tokenizer.model_max_length, - truncation=True, - return_tensors="pt", - ) - - text_input_ids = text_inputs.input_ids - untruncated_ids = tokenizer(prompt, - padding="longest", - return_tensors="pt").input_ids - - if untruncated_ids.shape[ - -1] >= text_input_ids.shape[-1] and not torch.equal( - text_input_ids, untruncated_ids): - removed_text = tokenizer.batch_decode( - untruncated_ids[:, tokenizer.model_max_length - 1:-1]) - logger.warning( - "The following part of your input was truncated because CLIP can only handle sequences up to" - f" {tokenizer.model_max_length} tokens: {removed_text}") - - prompt_embeds = text_encoder(text_input_ids.to(device), - output_hidden_states=True) - - # We are only ALWAYS interested in the pooled output of the final text encoder - pooled_prompt_embeds = prompt_embeds[0] - if clip_skip is None: - prompt_embeds = prompt_embeds.hidden_states[-2] - else: - # "2" because SDXL always indexes from the penultimate layer. - prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + - 2)] - - prompt_embeds_list.append(prompt_embeds) - - prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) - - # get unconditional embeddings for classifier free guidance - zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt - if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: - negative_prompt_embeds = torch.zeros_like(prompt_embeds) - negative_pooled_prompt_embeds = torch.zeros_like( - pooled_prompt_embeds) - elif do_classifier_free_guidance and negative_prompt_embeds is None: - negative_prompt = negative_prompt or "" - negative_prompt_2 = negative_prompt_2 or negative_prompt - - # normalize str to list - negative_prompt = batch_size * \ - [negative_prompt] if isinstance( - negative_prompt, str) else negative_prompt - negative_prompt_2 = (batch_size * [negative_prompt_2] if isinstance( - negative_prompt_2, str) else negative_prompt_2) - - uncond_tokens: List[str] - if prompt is not None and type(prompt) is not type(negative_prompt): - raise TypeError( - f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" - f" {type(prompt)}.") - elif batch_size != len(negative_prompt): - raise ValueError( - f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" - f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" - " the batch size of `prompt`.") - else: - uncond_tokens = [negative_prompt, negative_prompt_2] - - negative_prompt_embeds_list = [] - for negative_prompt, tokenizer, text_encoder in zip( - uncond_tokens, tokenizers, text_encoders): - if isinstance(self, TextualInversionLoaderMixin): - negative_prompt = self.maybe_convert_prompt( - negative_prompt, tokenizer) - - max_length = prompt_embeds.shape[1] - uncond_input = tokenizer( - negative_prompt, - padding="max_length", - max_length=max_length, - truncation=True, - return_tensors="pt", - ) - - negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), - output_hidden_states=True, - ) - # We are only ALWAYS interested in the pooled output of the final text encoder - negative_pooled_prompt_embeds = negative_prompt_embeds[0] - negative_prompt_embeds = negative_prompt_embeds.hidden_states[ - -2] - - negative_prompt_embeds_list.append(negative_prompt_embeds) - - negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, - dim=-1) - - if self.text_encoder_2 is not None: - prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, - device=device) - else: - prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, - device=device) - - bs_embed, seq_len, _ = prompt_embeds.shape - # duplicate text embeddings for each generation per prompt, using mps friendly method - prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) - prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, - seq_len, -1) - - if do_classifier_free_guidance: - # duplicate unconditional embeddings for each generation per prompt, using mps friendly method - seq_len = negative_prompt_embeds.shape[1] - - if self.text_encoder_2 is not None: - negative_prompt_embeds = negative_prompt_embeds.to( - dtype=self.text_encoder_2.dtype, device=device) - else: - negative_prompt_embeds = negative_prompt_embeds.to( - dtype=self.unet.dtype, device=device) - - negative_prompt_embeds = negative_prompt_embeds.repeat( - 1, num_images_per_prompt, 1) - negative_prompt_embeds = negative_prompt_embeds.view( - batch_size * num_images_per_prompt, seq_len, -1) - - pooled_prompt_embeds = pooled_prompt_embeds.repeat( - 1, num_images_per_prompt).view(bs_embed * num_images_per_prompt, -1) - if do_classifier_free_guidance: - negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat( - 1, num_images_per_prompt).view(bs_embed * num_images_per_prompt, - -1) - - if self.text_encoder is not None: - if isinstance( - self, - StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: - # Retrieve the original scale by scaling back the LoRA layers - unscale_lora_layers(self.text_encoder, lora_scale) - - if self.text_encoder_2 is not None: - if isinstance( - self, - StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: - # Retrieve the original scale by scaling back the LoRA layers - unscale_lora_layers(self.text_encoder_2, lora_scale) - - return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds - - # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image - def encode_image(self, image, device, num_images_per_prompt): - dtype = next(self.image_encoder.parameters()).dtype - - if not isinstance(image, torch.Tensor): - image = self.feature_extractor(image, - return_tensors="pt").pixel_values - - image = image.to(device=device, dtype=dtype) - image_embeds = self.image_encoder(image).image_embeds - image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, - dim=0) - - uncond_image_embeds = torch.zeros_like(image_embeds) - return image_embeds, uncond_image_embeds - - # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs - def prepare_extra_step_kwargs(self, generator, eta): - # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature - # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. - # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 - # and should be between [0, 1] - - accepts_eta = "eta" in set( - inspect.signature(self.scheduler.step).parameters.keys()) - extra_step_kwargs = {} - if accepts_eta: - extra_step_kwargs["eta"] = eta - - # check if the scheduler accepts generator - accepts_generator = "generator" in set( - inspect.signature(self.scheduler.step).parameters.keys()) - if accepts_generator: - extra_step_kwargs["generator"] = generator - return extra_step_kwargs - - def check_inputs( - self, - prompt, - prompt_2, - height, - width, - callback_steps, - negative_prompt=None, - negative_prompt_2=None, - prompt_embeds=None, - negative_prompt_embeds=None, - pooled_prompt_embeds=None, - negative_pooled_prompt_embeds=None, - callback_on_step_end_tensor_inputs=None, - ): - if height % 8 != 0 or width % 8 != 0: - raise ValueError( - f"`height` and `width` have to be divisible by 8 but are {height} and {width}." - ) - - if callback_steps is not None and (not isinstance(callback_steps, int) - or callback_steps <= 0): - raise ValueError( - f"`callback_steps` has to be a positive integer but is {callback_steps} of type" - f" {type(callback_steps)}.") - - if callback_on_step_end_tensor_inputs is not None and not all( - k in self._callback_tensor_inputs - for k in callback_on_step_end_tensor_inputs): - raise ValueError( - f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" - ) - - if prompt is not None and prompt_embeds is not None: - raise ValueError( - f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" - " only forward one of the two.") - elif prompt_2 is not None and prompt_embeds is not None: - raise ValueError( - f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" - " only forward one of the two.") - elif prompt is None and prompt_embeds is None: - raise ValueError( - "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." - ) - elif prompt is not None and (not isinstance(prompt, str) - and not isinstance(prompt, list)): - raise ValueError( - f"`prompt` has to be of type `str` or `list` but is {type(prompt)}" - ) - elif prompt_2 is not None and (not isinstance(prompt_2, str) - and not isinstance(prompt_2, list)): - raise ValueError( - f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}" - ) - - if negative_prompt is not None and negative_prompt_embeds is not None: - raise ValueError( - f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" - f" {negative_prompt_embeds}. Please make sure to only forward one of the two." - ) - elif negative_prompt_2 is not None and negative_prompt_embeds is not None: - raise ValueError( - f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" - f" {negative_prompt_embeds}. Please make sure to only forward one of the two." - ) - - if prompt_embeds is not None and negative_prompt_embeds is not None: - if prompt_embeds.shape != negative_prompt_embeds.shape: - raise ValueError( - "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" - f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" - f" {negative_prompt_embeds.shape}.") - - if prompt_embeds is not None and pooled_prompt_embeds is None: - raise ValueError( - "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." - ) - - if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: - raise ValueError( - "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." - ) - - # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents - def prepare_latents(self, - batch_size, - num_channels_latents, - height, - width, - dtype, - device, - generator, - latents=None): - shape = (batch_size, num_channels_latents, - height // self.vae_scale_factor, - width // self.vae_scale_factor) - if isinstance(generator, list) and len(generator) != batch_size: - raise ValueError( - f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" - f" size of {batch_size}. Make sure the batch size matches the length of the generators." - ) - - if latents is None: - latents = randn_tensor(shape, - generator=generator, - device=device, - dtype=dtype) - else: - latents = latents.to(device) - - # scale the initial noise by the standard deviation required by the scheduler - latents = latents * self.scheduler.init_noise_sigma - return latents - - def _get_add_time_ids(self, - original_size, - crops_coords_top_left, - target_size, - dtype, - text_encoder_projection_dim=None): - add_time_ids = list(original_size + crops_coords_top_left + target_size) - - passed_add_embed_dim = ( - self.unet.config.addition_time_embed_dim * len(add_time_ids) + - text_encoder_projection_dim) - expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features - - if expected_add_embed_dim != passed_add_embed_dim: - raise ValueError( - f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." - ) - - add_time_ids = torch.tensor([add_time_ids], dtype=dtype) - return add_time_ids - - # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae - def upcast_vae(self): - dtype = self.vae.dtype - self.vae.to(dtype=torch.float32) - use_torch_2_0_or_xformers = isinstance( - self.vae.decoder.mid_block.attentions[0].processor, - ( - AttnProcessor2_0, - XFormersAttnProcessor, - LoRAXFormersAttnProcessor, - LoRAAttnProcessor2_0, - ), - ) - # if xformers or torch_2_0 is used attention block does not need - # to be in float32 which can save lots of memory - if use_torch_2_0_or_xformers: - self.vae.post_quant_conv.to(dtype) - self.vae.decoder.conv_in.to(dtype) - self.vae.decoder.mid_block.to(dtype) - - # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_freeu - def enable_freeu(self, s1: float, s2: float, b1: float, b2: float): - r"""Enables the FreeU mechanism as in https://arxiv.org/abs/2309.11497. - - The suffixes after the scaling factors represent the stages where they are being applied. - - Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of the values - that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. - - Args: - s1 (`float`): - Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to - mitigate "oversmoothing effect" in the enhanced denoising process. - s2 (`float`): - Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to - mitigate "oversmoothing effect" in the enhanced denoising process. - b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. - b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. - """ - if not hasattr(self, "unet"): - raise ValueError("The pipeline must have `unet` for using FreeU.") - self.unet.enable_freeu(s1=s1, s2=s2, b1=b1, b2=b2) - - # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_freeu - def disable_freeu(self): - """Disables the FreeU mechanism if enabled.""" - self.unet.disable_freeu() - - # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding - def get_guidance_scale_embedding(self, - w, - embedding_dim=512, - dtype=torch.float32): - """ - See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 - - Args: - timesteps (`torch.Tensor`): - generate embedding vectors at these timesteps - embedding_dim (`int`, *optional*, defaults to 512): - dimension of the embeddings to generate - dtype: - data type of the generated embeddings - - Returns: - `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` - """ - assert len(w.shape) == 1 - w = w * 1000.0 - - half_dim = embedding_dim // 2 - emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) - emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) - emb = w.to(dtype)[:, None] * emb[None, :] - emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) - if embedding_dim % 2 == 1: # zero pad - emb = torch.nn.functional.pad(emb, (0, 1)) - assert emb.shape == (w.shape[0], embedding_dim) - return emb - - @property - def guidance_scale(self): - return self._guidance_scale - - @property - def guidance_rescale(self): - return self._guidance_rescale - - @property - def clip_skip(self): - return self._clip_skip - - # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) - # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` - # corresponds to doing no classifier free guidance. - @property - def do_classifier_free_guidance(self): - return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None - - @property - def cross_attention_kwargs(self): - return self._cross_attention_kwargs - - @property - def denoising_end(self): - return self._denoising_end - - @property - def num_timesteps(self): - return self._num_timesteps - - @torch.no_grad() - @replace_example_docstring(EXAMPLE_DOC_STRING) - def __call__( - self, - prompt: Union[str, List[str]] = None, - prompt_2: Optional[Union[str, List[str]]] = None, - height: Optional[int] = None, - width: Optional[int] = None, - num_inference_steps: int = 50, - timesteps: List[int] = None, - denoising_end: Optional[float] = None, - guidance_scale: float = 5.0, - negative_prompt: Optional[Union[str, List[str]]] = None, - negative_prompt_2: Optional[Union[str, List[str]]] = None, - num_images_per_prompt: Optional[int] = 1, - eta: float = 0.0, - generator: Optional[Union[torch.Generator, - List[torch.Generator]]] = None, - latents: Optional[torch.FloatTensor] = None, - prompt_embeds: Optional[torch.FloatTensor] = None, - negative_prompt_embeds: Optional[torch.FloatTensor] = None, - pooled_prompt_embeds: Optional[torch.FloatTensor] = None, - negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, - ip_adapter_image: Optional[PipelineImageInput] = None, - output_type: Optional[str] = "pil", - return_dict: bool = True, - cross_attention_kwargs: Optional[Dict[str, Any]] = None, - guidance_rescale: float = 0.0, - original_size: Optional[Tuple[int, int]] = None, - crops_coords_top_left: Tuple[int, int] = (0, 0), - target_size: Optional[Tuple[int, int]] = None, - negative_original_size: Optional[Tuple[int, int]] = None, - negative_crops_coords_top_left: Tuple[int, int] = (0, 0), - negative_target_size: Optional[Tuple[int, int]] = None, - clip_skip: Optional[int] = None, - callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, - callback_on_step_end_tensor_inputs: List[str] = ["latents"], - **kwargs, - ): - r""" - Function invoked when calling the pipeline for generation. - - Args: - prompt (`str` or `List[str]`, *optional*): - The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. - instead. - prompt_2 (`str` or `List[str]`, *optional*): - The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is - used in both text-encoders - height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): - The height in pixels of the generated image. This is set to 1024 by default for the best results. - Anything below 512 pixels won't work well for - [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) - and checkpoints that are not specifically fine-tuned on low resolutions. - width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): - The width in pixels of the generated image. This is set to 1024 by default for the best results. - Anything below 512 pixels won't work well for - [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) - and checkpoints that are not specifically fine-tuned on low resolutions. - num_inference_steps (`int`, *optional*, defaults to 50): - The number of denoising steps. More denoising steps usually lead to a higher quality image at the - expense of slower inference. - timesteps (`List[int]`, *optional*): - Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument - in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is - passed will be used. Must be in descending order. - denoising_end (`float`, *optional*): - When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be - completed before it is intentionally prematurely terminated. As a result, the returned sample will - still retain a substantial amount of noise as determined by the discrete timesteps selected by the - scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a - "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image - Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) - guidance_scale (`float`, *optional*, defaults to 5.0): - Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). - `guidance_scale` is defined as `w` of equation 2. of [Imagen - Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > - 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, - usually at the expense of lower image quality. - negative_prompt (`str` or `List[str]`, *optional*): - The prompt or prompts not to guide the image generation. If not defined, one has to pass - `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is - less than `1`). - negative_prompt_2 (`str` or `List[str]`, *optional*): - The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and - `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders - num_images_per_prompt (`int`, *optional*, defaults to 1): - The number of images to generate per prompt. - eta (`float`, *optional*, defaults to 0.0): - Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to - [`schedulers.DDIMScheduler`], will be ignored for others. - generator (`torch.Generator` or `List[torch.Generator]`, *optional*): - One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) - to make generation deterministic. - latents (`torch.FloatTensor`, *optional*): - Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image - generation. Can be used to tweak the same generation with different prompts. If not provided, a latents - tensor will ge generated by sampling using the supplied random `generator`. - prompt_embeds (`torch.FloatTensor`, *optional*): - Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not - provided, text embeddings will be generated from `prompt` input argument. - negative_prompt_embeds (`torch.FloatTensor`, *optional*): - Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt - weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input - argument. - pooled_prompt_embeds (`torch.FloatTensor`, *optional*): - Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. - If not provided, pooled text embeddings will be generated from `prompt` input argument. - negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): - Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt - weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` - input argument. - ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. - output_type (`str`, *optional*, defaults to `"pil"`): - The output format of the generate image. Choose between - [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. - return_dict (`bool`, *optional*, defaults to `True`): - Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead - of a plain tuple. - cross_attention_kwargs (`dict`, *optional*): - A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under - `self.processor` in - [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - guidance_rescale (`float`, *optional*, defaults to 0.0): - Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are - Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of - [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). - Guidance rescale factor should fix overexposure when using zero terminal SNR. - original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): - If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. - `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as - explained in section 2.2 of - [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): - `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position - `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting - `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of - [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): - For most cases, `target_size` should be set to the desired height and width of the generated image. If - not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in - section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). - negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): - To negatively condition the generation process based on a specific image resolution. Part of SDXL's - micro-conditioning as explained in section 2.2 of - [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more - information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): - To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's - micro-conditioning as explained in section 2.2 of - [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more - information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): - To negatively condition the generation process based on a target image resolution. It should be as same - as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of - [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more - information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. - callback_on_step_end (`Callable`, *optional*): - A function that calls at the end of each denoising steps during the inference. The function is called - with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, - callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by - `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*): - The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list - will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the - `._callback_tensor_inputs` attribute of your pipeline class. - - Examples: - - Returns: - [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`: - [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a - `tuple`. When returning a tuple, the first element is a list with the generated images. - """ - - callback = kwargs.pop("callback", None) - callback_steps = kwargs.pop("callback_steps", None) - - if callback is not None: - deprecate( - "callback", - "1.0.0", - "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", - ) - if callback_steps is not None: - deprecate( - "callback_steps", - "1.0.0", - "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", - ) - - # 0. Default height and width to unet - height = height or self.default_sample_size * self.vae_scale_factor - width = width or self.default_sample_size * self.vae_scale_factor - - original_size = original_size or (height, width) - target_size = target_size or (height, width) - - # 1. Check inputs. Raise error if not correct - self.check_inputs( - prompt, - prompt_2, - height, - width, - callback_steps, - negative_prompt, - negative_prompt_2, - prompt_embeds, - negative_prompt_embeds, - pooled_prompt_embeds, - negative_pooled_prompt_embeds, - callback_on_step_end_tensor_inputs, - ) - - self._guidance_scale = guidance_scale - self._guidance_rescale = guidance_rescale - self._clip_skip = clip_skip - self._cross_attention_kwargs = cross_attention_kwargs - self._denoising_end = denoising_end - - # 2. Define call parameters - if prompt is not None and isinstance(prompt, str): - batch_size = 1 - elif prompt is not None and isinstance(prompt, list): - batch_size = len(prompt) - else: - batch_size = prompt_embeds.shape[0] - - device = self.execution_device - - # 3. Encode input prompt - lora_scale = (self.cross_attention_kwargs.get("scale", None) - if self.cross_attention_kwargs is not None else None) - - ( - prompt_embeds, - negative_prompt_embeds, - pooled_prompt_embeds, - negative_pooled_prompt_embeds, - ) = self.encode_prompt( - prompt=prompt, - prompt_2=prompt_2, - device=device, - num_images_per_prompt=num_images_per_prompt, - do_classifier_free_guidance=self.do_classifier_free_guidance, - negative_prompt=negative_prompt, - negative_prompt_2=negative_prompt_2, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - pooled_prompt_embeds=pooled_prompt_embeds, - negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, - lora_scale=lora_scale, - clip_skip=self.clip_skip, - ) - - # 4. Prepare timesteps - timesteps, num_inference_steps = retrieve_timesteps( - self.scheduler, num_inference_steps, device, timesteps) - - # 5. Prepare latent variables - num_channels_latents = self.unet.config.in_channels - latents = self.prepare_latents( - batch_size * num_images_per_prompt, - num_channels_latents, - height, - width, - prompt_embeds.dtype, - device, - generator, - latents, - ) - - # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline - extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) - - # 7. Prepare added time ids & embeddings - add_text_embeds = pooled_prompt_embeds - if self.text_encoder_2 is None: - text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) - else: - text_encoder_projection_dim = self.text_encoder_2.config.projection_dim - - add_time_ids = self._get_add_time_ids( - original_size, - crops_coords_top_left, - target_size, - dtype=prompt_embeds.dtype, - text_encoder_projection_dim=text_encoder_projection_dim, - ) - if negative_original_size is not None and negative_target_size is not None: - negative_add_time_ids = self._get_add_time_ids( - negative_original_size, - negative_crops_coords_top_left, - negative_target_size, - dtype=prompt_embeds.dtype, - text_encoder_projection_dim=text_encoder_projection_dim, - ) - else: - negative_add_time_ids = add_time_ids - - if self.do_classifier_free_guidance: - prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], - dim=0) - add_text_embeds = torch.cat( - [negative_pooled_prompt_embeds, add_text_embeds], dim=0) - add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], - dim=0) - - prompt_embeds = prompt_embeds.to(device) - add_text_embeds = add_text_embeds.to(device) - add_time_ids = add_time_ids.to(device).repeat( - batch_size * num_images_per_prompt, 1) - - if ip_adapter_image is not None: - image_embeds, negative_image_embeds = self.encode_image( - ip_adapter_image, device, num_images_per_prompt) - if self.do_classifier_free_guidance: - image_embeds = torch.cat([negative_image_embeds, image_embeds]) - image_embeds = image_embeds.to(device) - - # 8. Denoising loop - num_warmup_steps = max( - len(timesteps) - num_inference_steps * self.scheduler.order, 0) - - # 8.1 Apply denoising_end - if (self.denoising_end is not None - and isinstance(self.denoising_end, float) - and self.denoising_end > 0 and self.denoising_end < 1): - discrete_timestep_cutoff = int( - round(self.scheduler.config.num_train_timesteps - - (self.denoising_end * - self.scheduler.config.num_train_timesteps))) - num_inference_steps = len( - list( - filter(lambda ts: ts >= discrete_timestep_cutoff, - timesteps))) - timesteps = timesteps[:num_inference_steps] - - # 9. Optionally get Guidance Scale Embedding - timestep_cond = None - if self.unet.config.time_cond_proj_dim is not None: - guidance_scale_tensor = torch.tensor(self.guidance_scale - - 1).repeat( - batch_size * - num_images_per_prompt) - timestep_cond = self.get_guidance_scale_embedding( - guidance_scale_tensor, - embedding_dim=self.unet.config.time_cond_proj_dim).to( - device=device, dtype=latents.dtype) - - self._num_timesteps = len(timesteps) - with self.progress_bar(total=num_inference_steps) as progress_bar: - for i, t in enumerate(timesteps): - # expand the latents if we are doing classifier free guidance - latent_model_input = torch.cat( - [latents] * - 2) if self.do_classifier_free_guidance else latents - - latent_model_input = self.scheduler.scale_model_input( - latent_model_input, t) - - # predict the noise residual - added_cond_kwargs = { - "text_embeds": add_text_embeds, - "time_ids": add_time_ids - } - if ip_adapter_image is not None: - added_cond_kwargs["image_embeds"] = image_embeds - - t = t.to(latent_model_input.dtype) - feed_dict = { - 'sample': latent_model_input, - 'timesteps': t.unsqueeze(0), - 'encoder_hidden_states': prompt_embeds, - 'text_embeds': add_text_embeds, - 'time_ids': add_time_ids, - } - ok = self.session.run(feed_dict, self.outputs, self.stream) - assert ok, "Runtime execution failed" - noise_pred = self.outputs['pred'] - torch.cuda.synchronize() - - # perform guidance - if self.do_classifier_free_guidance: - noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) - noise_pred = noise_pred_uncond + self.guidance_scale * \ - (noise_pred_text - noise_pred_uncond) - - if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: - # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf - noise_pred = rescale_noise_cfg( - noise_pred, - noise_pred_text, - guidance_rescale=self.guidance_rescale) - - # compute the previous noisy sample x_t -> x_t-1 - latents = self.scheduler.step(noise_pred, - t, - latents, - **extra_step_kwargs, - return_dict=False)[0] - - if callback_on_step_end is not None: - callback_kwargs = {} - for k in callback_on_step_end_tensor_inputs: - callback_kwargs[k] = locals()[k] - callback_outputs = callback_on_step_end( - self, i, t, callback_kwargs) - - latents = callback_outputs.pop("latents", latents) - prompt_embeds = callback_outputs.pop( - "prompt_embeds", prompt_embeds) - negative_prompt_embeds = callback_outputs.pop( - "negative_prompt_embeds", negative_prompt_embeds) - add_text_embeds = callback_outputs.pop( - "add_text_embeds", add_text_embeds) - negative_pooled_prompt_embeds = callback_outputs.pop( - "negative_pooled_prompt_embeds", - negative_pooled_prompt_embeds) - add_time_ids = callback_outputs.pop("add_time_ids", - add_time_ids) - negative_add_time_ids = callback_outputs.pop( - "negative_add_time_ids", negative_add_time_ids) - - # call the callback, if provided - if i == len(timesteps) - 1 or ( - (i + 1) > num_warmup_steps and - (i + 1) % self.scheduler.order == 0): - progress_bar.update() - if callback is not None and i % callback_steps == 0: - step_idx = i // getattr(self.scheduler, "order", 1) - callback(step_idx, t, latents) - - if XLA_AVAILABLE: - xm.mark_step() - - if not output_type == "latent": - # make sure the VAE is in float32 mode, as it overflows in float16 - needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast - - if needs_upcasting: - self.upcast_vae() - latents = latents.to( - next(iter(self.vae.post_quant_conv.parameters())).dtype) - - image = self.vae.decode(latents / self.vae.config.scaling_factor, - return_dict=False)[0] - - # cast back to fp16 if needed - if needs_upcasting: - self.vae.to(dtype=torch.float16) - else: - image = latents - - if not output_type == "latent": - # apply watermark if available - if self.watermark is not None: - image = self.watermark.apply_watermark(image) - - image = self.image_processor.postprocess(image, - output_type=output_type) - - # Offload all models - self.maybe_free_model_hooks() - - if not return_dict: - return (image, ) - - return StableDiffusionXLPipelineOutput(images=image) diff --git a/examples/models/contrib/sdxl/run_sdxl.py b/examples/models/contrib/sdxl/run_sdxl.py deleted file mode 100755 index 077b93f9ab7e..000000000000 --- a/examples/models/contrib/sdxl/run_sdxl.py +++ /dev/null @@ -1,96 +0,0 @@ -import argparse -import time - -import numpy as np -import torch -from pipeline_stable_diffusion_xl import StableDiffusionXLPipeline - -import tensorrt_llm - -world_size = tensorrt_llm.mpi_world_size() -rank = tensorrt_llm.mpi_rank() - - -def parseArgs(): - parser = argparse.ArgumentParser( - description='run SDXL with the UNet TensorRT engine.') - parser.add_argument('--model_dir', - type=str, - default='stabilityai/stable-diffusion-xl-base-1.0') - parser.add_argument('--size', type=int, default=1024) - parser.add_argument('--seed', type=int, default=233) - parser.add_argument('--num_inference_steps', type=int, default=50) - parser.add_argument( - '--prompt', - type=str, - default= - "masterpiece, gouache painting, 1girl, distant view, lone boat, willow trees" - ) - parser.add_argument('--engine_dir', - type=str, - default=None, - help='engine directory') - parser.add_argument('--num-warmup-runs', type=int, default=3) - parser.add_argument('--avg-runs', type=int, default=10) - parser.add_argument("--ignore_ratio", - type=float, - default=0.2, - help="Ignored ratio of the slowest and fastest steps") - parser.add_argument("--output", - type=str, - default="output.png", - help="Output file name") - return parser.parse_args() - - -if __name__ == "__main__": - args = parseArgs() - model_dir = args.model_dir - size = args.size - seed = args.seed - prompt = args.prompt - num_inference_steps = args.num_inference_steps - engine_dir = f'sdxl_s{size}_w{world_size}' if args.engine_dir is None else args.engine_dir - num_warmup_runs = args.num_warmup_runs - avg_runs = args.avg_runs - output_file = args.output - - pipeline = StableDiffusionXLPipeline.from_pretrained( - model_dir, - torch_dtype=torch.float16, - use_safetensors=True, - ) - pipeline.set_progress_bar_config(disable=rank != 0) - pipeline.prepare(engine_dir, size) - pipeline.to('cuda') - - # warm up - for i in range(num_warmup_runs): - image = pipeline( - num_inference_steps=num_inference_steps, - prompt=prompt, - generator=torch.Generator(device="cuda").manual_seed(seed), - height=size, - width=size).images[0] - - latency_list = [] - for i in range(avg_runs): - st = time.time() - image = pipeline( - num_inference_steps=num_inference_steps, - prompt=prompt, - generator=torch.Generator(device="cuda").manual_seed(seed), - height=size, - width=size, - ).images[0] - ed = time.time() - latency_list.append(ed - st) - - latency_list = sorted(latency_list) - ignored_count = int(args.ignore_ratio * len(latency_list) / 2) - if ignored_count > 0: - latency_list = latency_list[ignored_count:-ignored_count] - - if rank == 0: - print(f"Avg latency: {np.sum(latency_list) / len(latency_list):.5f} s") - image.save(output_file) diff --git a/examples/models/contrib/skywork/README.md b/examples/models/contrib/skywork/README.md deleted file mode 100644 index 59b22a861987..000000000000 --- a/examples/models/contrib/skywork/README.md +++ /dev/null @@ -1,113 +0,0 @@ -# Skywork - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document elaborates how to build the [Skywork](https://huggingface.co/Skywork/) model to runnable engines on single GPU node and perform a summarization task using these engines. - -## Overview -The TensorRT LLM Skywork implementation is based on the LLaMA model. The implementation can -be found in [tensorrt_llm/models/llama/model.py](../../../../tensorrt_llm/models/llama/model.py). -The TensorRT LLM Skywork example code lies in [`examples/models/contrib/skywork`](./): - -* [`convert_checkpoint.py`](../../core/llama/convert_checkpoint.py) converts the Huggingface Model of Skywork into TensorRT LLM checkpoint. - -In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: - -* [`../../../run.py`](../../../run.py) to run the inference on an input text; -* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. - -## Support Matrix - * FP16 & BF16 - -## Usage - -This section gives a whole process where we convert HF models, build TensorRT LLM engines and ultimately perform summarization. - -### 1. Clone Code and Weights from Huggingface - -To download checkpoints from HF, you need to have `git-lfs` installed in your machine: - -```bash -pip install -r requirements.txt && sudo apt-get install git-lfs -``` - -Then clone the HF repository with: - -```bash -# Skywork 13B Base Model -git clone https://huggingface.co/Skywork/Skywork-13B-base -``` - -### 2. Convert HF Model to TRT Checkpoint - -```bash -cd examples/models/core/llama - -# fp16 model -python3 convert_checkpoint.py --model_dir ./Skywork-13B-base \ - --dtype float16 \ - --output_dir ./skywork-13b-base/trt_ckpt/fp16 - -# bf16 model -python3 convert_checkpoint.py --model_dir ./Skywork-13B-base \ - --dtype bfloat16 \ - --output_dir ./skywork-13b-base/trt_ckpt/bf16 -``` - -### 3. Build TensorRT Engine(s) - -```bash -# fp16 -trtllm-build --checkpoint_dir ./skywork-13b-base/trt_ckpt/fp16 \ - --gemm_plugin float16 \ - --gpt_attention_plugin float16 \ - --context_fmha enable \ - --max_batch_size 32 \ - --max_input_len 512 \ - --max_seq_len 1024 \ - --output_dir ./skywork-13b-base/trt_engine/fp16 - -# bf16 -trtllm-build --checkpoint_dir ./skywork-13b-base/trt_ckpt/bf16 \ - --gemm_plugin bfloat16 \ - --gpt_attention_plugin bfloat16 \ - --context_fmha enable \ - --max_batch_size 32 \ - --max_input_len 512 \ - --max_seq_len 1024 \ - --output_dir ./skywork-13b-base/trt_engine/bf16 -``` - -### 4. Summarization using the Engines - -After building TRT engines, we can use them to perform various tasks. TensorRT LLM provides handy code to run summarization on [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset and get [ROUGE](https://en.wikipedia.org/wiki/ROUGE_(metric)) scores. The `ROUGE-1` score can be used to validate model implementations. - -```bash -# fp16 -python ../../../summarize.py --hf_model_dir ./Skywork-13B-base \ - --test_hf \ - --batch_size 32 \ - --max_input_length 512 \ - --output_len 512 \ - --test_trt_llm \ - --engine_dir ./skywork-13b-base/trt_engine/fp16 \ - --data_type fp16 \ - --check_accuracy \ - --tensorrt_llm_rouge1_threshold=14 - -# bf16 -python ../../../summarize.py --hf_model_dir ./Skywork-13B-base \ - --test_hf \ - --batch_size 32 \ - --max_input_length 512 \ - --output_len 512 \ - --test_trt_llm \ - --engine_dir ./skywork-13b-base/trt_engine/bf16 \ - --data_type bf16 \ - --check_accuracy \ - --tensorrt_llm_rouge1_threshold=14 -``` diff --git a/examples/models/contrib/skywork/requirements.txt b/examples/models/contrib/skywork/requirements.txt deleted file mode 100644 index 88232baef811..000000000000 --- a/examples/models/contrib/skywork/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.0.dev0 -datasets==3.1.0 -evaluate -rouge_score -sentencepiece>=0.1.99 diff --git a/examples/models/contrib/smaug/README.md b/examples/models/contrib/smaug/README.md deleted file mode 100644 index 2d96da5854e1..000000000000 --- a/examples/models/contrib/smaug/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Smaug - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document elaborates how to build the [Smaug-72B-v0.1](https://huggingface.co/abacusai/Smaug-72B-v0.1) model to runnable engines on multi-GPU node and perform a summarization task using these engines. - -## Overview - -The TensorRT LLM support for Smaug-72B-v0.1 is based on the LLaMA model, the implementation can be found in [tensorrt_llm/models/llama/model.py](../../../../tensorrt_llm/models/llama/model.py). Smaug model resembles LLaMA very much except it uses bias term in its attention module, we therefore reuse the [LLaMA example code](../../core/llama) for Smaug, - -* [`convert_checkpoint.py`](./convert_checkpoint.py) to convert the LLaMA model into TensorRT LLM checkpoint format. - -In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: - -* [`../../../run.py`](../../../run.py) to run the inference on an input text; -* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. - -## Support Matrix - -* FP16 - -## Usage - -This section gives a whole process where we convert HF models, build TensorRT LLM engines and ultimately perform summarization. - -### Build TensorRT engine(s) - -Run the following commands and TRT-LLM will first transforms a HF model into its own checkpoint format, then builds a TRT engine based on the checkpoint - -```bash -python ../../../llama/convert_checkpoint.py \ - --model_dir ./Smaug-72B-v0.1 \ - --output_dir ./tllm_checkpoint_8gpu_tp8 \ - --dtype float16 \ - --tp_size 8 - -trtllm-build --checkpoint_dir ./tllm_checkpoint_8gpu_tp8 \ - --output_dir ./Smaug_72B_tp8 \ - --gemm_plugin float16 \ - --gpt_attention_plugin float16 \ - --context_fmha=enable \ - --max_batch_size 64 \ - --remove_input_padding=enable -``` - -### Run Summarization - -After building TRT engine, we can use it to perform various tasks. TensorRT LLM provides handy code to run summarization on [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset and get [ROUGE](https://en.wikipedia.org/wiki/ROUGE_(metric)) scores. The `ROUGE-1` score can be used to validate model implementations. - -```bash -mpirun -n 8 -allow-run-as-root python ../../../summarize.py \ - --hf_model_dir ../Smaug-72B-v0.1 \ - --engine_dir ./Smaug_72B_tp8 \ - --data_type fp16 \ - --test_hf \ - --hf_device_map_auto \ - --test_trt_llm -``` diff --git a/examples/models/contrib/smaug/requirements.txt b/examples/models/contrib/smaug/requirements.txt deleted file mode 100644 index 88232baef811..000000000000 --- a/examples/models/contrib/smaug/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.0.dev0 -datasets==3.1.0 -evaluate -rouge_score -sentencepiece>=0.1.99 diff --git a/examples/models/core/deepseek_v3/README.md b/examples/models/core/deepseek_v3/README.md index 6c9774b83624..6f3af476866a 100644 --- a/examples/models/core/deepseek_v3/README.md +++ b/examples/models/core/deepseek_v3/README.md @@ -863,10 +863,10 @@ echo "All processes completed!" The converted checkpoint could be used as `` and consumed by other commands. ### KV Cache Reuse -KV cache reuse is supported for MLA on SM90, SM100 and SM120. It is enabled by default. Due to extra operations like memcpy and GEMMs, GPU memory consumption may be higher and the E2E performance may have regression in some cases. Users could pass `KvCacheConfig(enable_block_reuse=False)` to LLM API to disable it. +KV cache reuse is supported for MLA on SM90, SM100, SM103, SM120 and SM121. It is enabled by default. Due to extra operations like memcpy and GEMMs, GPU memory consumption may be higher and the E2E performance may have regression in some cases. Users could pass `KvCacheConfig(enable_block_reuse=False)` to LLM API to disable it. ### Chunked Prefill -Chunked Prefill is supported for MLA only on SM90 and SM100 currently. You should add `--enable_chunked_prefill` to enable it. The GPU memory consumption is highly correlated with `max_num_tokens` and `max_batch_size`. If encountering out-of-memory errors, you may make these values smaller. (`max_num_tokens` must be divisible by kv cache's `tokens_per_block`) +Chunked Prefill is supported for MLA on SM90, SM100, SM103 and SM120. You should add `--enable_chunked_prefill` to enable it. The GPU memory consumption is highly correlated with `max_num_tokens` and `max_batch_size`. If encountering out-of-memory errors, you may make these values smaller. (`max_num_tokens` must be divisible by kv cache's `tokens_per_block`) More specifically, we can imitate what we did in the [Quick Start](#quick-start): diff --git a/examples/models/core/granite/README.md b/examples/models/core/granite/README.md deleted file mode 100644 index b6cf34c50c5f..000000000000 --- a/examples/models/core/granite/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Granite - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document shows how to build and run a [Granite 3.0](https://huggingface.co/collections/ibm-granite/granite-30-language-models-66fdb59bbb54785c3512114f) model in TensorRT-LLM. - -The TensorRT LLM Granite implementation is based on the LLaMA model, with Mixture of Experts (MoE) enabled. The implementation can be found in [`llama/model.py`](../../../../tensorrt_llm/models/llama/model.py). See the LLaMA example [`examples/models/core/llama`](../llama) for details. - -- [Granite 3.0](#Granite) - - [Download model checkpoints](#download-model-checkpoints) - - [Convert weights from HF Transformers to TensorRT LLM format](#Convert-weights-from-HF-Transformers-to-TensorRT-LLM-format) - - [Build TensorRT engine](#build-tensorrt-engine) - - [Run Engine](#run-engine) - -## Download model checkpoints - -First, download the HuggingFace BF16 checkpoints of Granite 3.0 model. - -```bash -HF_MODEL="granite-3.0-8b-instruct" # or granite-3.0-3b-a800m-instruct -# clone the model we want to build -git clone https://huggingface.co/ibm-granite/${HF_MODEL} tmp/hf_checkpoints/${HF_MODEL} -``` - -## Convert weights from HF Transformers to TensorRT LLM format -Set environment variables and necessary directory: - -```bash -PREC_RAW="bfloat16" -TP=1 -mkdir -p tmp/trt_engines -``` - -### BF16 -Convert the weights using the `convert_checkpoint.py` script: - -```bash -ENGINE="${HF_MODEL}_${PREC_RAW}_tp${TP}" -export TRTLLM_DISABLE_UNIFIED_CONVERTER=1 # The current checkpoint conversion code requires legacy path -python3 ../llama/convert_checkpoint.py --model_dir tmp/hf_checkpoints/${HF_MODEL} \ - --output_dir tmp/tllm_checkpoints/${ENGINE} \ - --dtype ${PREC_RAW} \ - --tp_size ${TP} \ - --use_embedding_sharing - - -``` -### FP8 PTQ -Notes: -- Currently quantize.py does not support Expert Parallelism (EP) mode yet. User should use `../llama/convert_checkpoint.py` and specify `--moe_ep_size 1` instead, if needed. -- TensorRT LLM uses static quantization methods, which is expected to be faster at runtime as compared to dynamic quantization methods. This comes at a cost of an offline calibration step during quantization. `batch_size` and `calib_size` can be adjusted to shorten the calibration time. Please refer to `../../../quantization/README.md` for explanation. - -```bash -PREC_QUANT="fp8" -ENGINE="${HF_MODEL}_${PREC_QUANT}_tp${TP}" -python ../../../quantization/quantize.py --model_dir tmp/hf_checkpoints/${HF_MODEL} \ - --dtype ${PREC_RAW} \ - --qformat ${PREC_QUANT} \ - --kv_cache_dtype ${PREC_QUANT} \ - --output_dir tmp/tllm_checkpoints/${ENGINE} \ - --batch_size 1 \ - --calib_size 128 \ - --tp_size ${TP} - -``` - -## Build TensorRT engine -```bash -# Enable fp8 context fmha to get further acceleration by setting `--use_fp8_context_fmha enable` -# Use --workers to enable parallel build -trtllm-build --checkpoint_dir ./tmp/tllm_checkpoints/${ENGINE} \ - --output_dir ./tmp/trt_engines/${ENGINE} \ - --gpt_attention_plugin ${PREC_RAW} \ - --gemm_plugin ${PREC_RAW} \ - --workers ${TP} -``` - -## Run Engine -Test your engine with the [run.py](../../../run.py) script: - -```bash -mpirun -n ${TP} --allow-run-as-root python ../../../run.py --engine_dir ./tmp/trt_engines/${ENGINE} --tokenizer_dir tmp/hf_checkpoints/${HF_MODEL} --max_output_len 20 --input_text "The future of AI is" -``` - -For more usage examples see [`examples/models/core/llama/README.md`](../llama/README.md) diff --git a/examples/models/core/mixtral/README.md b/examples/models/core/mixtral/README.md deleted file mode 100644 index 079753f88c92..000000000000 --- a/examples/models/core/mixtral/README.md +++ /dev/null @@ -1,219 +0,0 @@ -# Mixtral - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document shows how to build and run a Mixtral model in TensorRT LLM on both single GPU, single node multi-GPU and -multi-node multi-GPU. Mixtral 8x22B is also supported and can be replace Mixtral 8x7B below as long as GPU memory is -sufficient. - -## Overview - -The TensorRT LLM Mixtral implementation is based on the LLaMA model, with Mixture of Experts enabled. The implementation can -be found in [tensorrt_llm/models/llama/model.py](../../../../tensorrt_llm/models/llama/model.py). -See the LLaMA example [`examples/models/core/llama`](../llama) for details. - -### Build TensorRT engine(s) - -#### Download Mixtral 8x7b weights -Get the weights by downloading from HF https://huggingface.co/mistralai/Mixtral-8x7B-v0.1. -See also https://huggingface.co/docs/transformers/main/en/model_doc/mixtral - -```bash -git lfs install -git clone https://huggingface.co/mistralai/Mixtral-8x7B-v0.1 -``` - -#### Download Mixtral 8x22b weights -Get the weights by downloading from HF https://huggingface.co/mistralai/Mixtral-8x22B-v0.1. -See also https://huggingface.co/docs/transformers/main/en/model_doc/mixtral - -```bash -git lfs install -git clone https://huggingface.co/mistralai/Mixtral-8x22B-v0.1 -``` - -We use the LLaMA `convert_checkpoint.py` script to convert and build the model. TensorRT LLM LLaMA builds TensorRT engine(s) from HF checkpoint provided by `--model_dir`. -If no checkpoint directory is specified, TensorRT LLM will build engine(s) with dummy weights. - -`trtllm-build` uses one GPU by default, but if you have already more GPUs available at build time, -you may enable parallel builds to make the engine building process faster by adding the `--workers` argument. - -Here are some examples: - -```bash -# Build Mixtral8x7B with pipeline parallelism -python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x7B-v0.1 \ - --output_dir ./tllm_checkpoint_mixtral_2gpu \ - --dtype float16 \ - --pp_size 2 -trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_2gpu \ - --output_dir ./trt_engines/mixtral/pp2 \ - --gemm_plugin float16 - -``` - -```bash -# Build Mixtral8x7B with tensor parallelism -python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x7B-v0.1 \ - --output_dir ./tllm_checkpoint_mixtral_2gpu \ - --dtype float16 \ - --tp_size 2 \ - --moe_tp_size 2 -trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_2gpu \ - --output_dir ./trt_engines/mixtral/tp2 \ - --gemm_plugin float16 - - -# Build Mixtral8x22B with tensor parallelism and expert parallelism -python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x22B-v0.1 \ - --output_dir ./tllm_checkpoint_mixtral_8gpu \ - --dtype float16 \ - --tp_size 8 \ - --moe_tp_size 2 \ - --moe_ep_size 4 -trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_8gpu \ - --output_dir ./trt_engines/mixtral/tp2ep4 \ - --gemm_plugin float16 -``` - -Then, you can test your engine with the [run.py](../../../run.py) script: - -```bash -mpirun -n 2 python3 ../../../run.py --engine_dir ./trt_engines/mixtral/tp2 --tokenizer_dir ./Mixtral-8x7B-v0.1 --max_output_len 8 --input_text "I love french quiche" -``` - -For more examples see [`examples/models/core/llama/README.md`](../llama/README.md) - -### Parallelism Modes - -Mixture of Experts supports 3 parallelism modes, these are Expert Parallelism (EP), Tensor Parallelism (TP), and the hybrid of the two (TP+EP). - -In TP mode (default) expert weight matrices are sliced evenly between all GPUs, so that all GPUs work together to calculate the result for each expert. - -In EP mode each GPU is assigned a subset of the expert weights matrices, so each GPU works independently to calculate the result for its assigned experts. This may cause load balancing issues where some GPUs have more work than others, thus increasing latency. - -In TP+EP mode, both strategies are used simultaneously. This means each GPU handles a portion of the expert weights matrices (as in EP mode) and these weights are further sliced across multiple GPUs (as in TP mode). This hybrid approach aims to balance the workload more evenly across GPUs, enhancing efficiency and reducing the likelihood of bottlenecks associated with EP mode alone. - -You can enable Expert Parallel or hybrid parallel by setting `--moe_tp_size` and `--moe_ep_size` when calling `convert_coneckpoint.py`. If only `--moe_tp_size` is provided, TRT-LLM will use Tensor Parallel for the MoE model; if only `--moe_ep_size` is provided, TRT-LLM will use Expert Parallel; if both are provided, the hybrid parallel will be used. - -Be sure that the product of `moe_tp_size` and `moe_ep_size` should equal to `tp_size`, since the total number of MoE parallelism across all GPUs must match the total number of parallelism in other parts of the model. - -```bash -# Build Mixtral8x7B with Expert Parallelism -python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x7B-v0.1 \ - --output_dir ./tllm_checkpoint_mixtral_2gpu \ - --dtype float16 \ - --tp_size 2 \ - --moe_ep_size 2 -trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_2gpu \ - --output_dir ./trt_engines/mixtral/ep2 \ - --gemm_plugin float16 - -# Build Mixtral8x7B with Expert Parallelism and Tensor Parallelism -python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x7B-v0.1 \ - --output_dir ./tllm_checkpoint_mixtral_4gpu \ - --dtype float16 \ - --tp_size 4 \ - --moe_tp_size 2 \ - --moe_ep_size 2 -trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_4gpu \ - --output_dir ./trt_engines/mixtral/tp2ep2 \ - --gemm_plugin float16 -``` - -### Normalization Modes - -MOE Supports different normalization modes which influence how the scales are calculated for the final weighted sum in -of the different top-k values. - -- 0 (NONE) corresponds to: `scales = topk(softmax(routing values))` -- 1 (RENORM) corresponds to: `scales = softmax(topk(routing values))` -- 2 (SPARSE_MIXER) corresponds to: `scales = sparsemixer(routing values)` - -Mixtral uses `RENORM` mode, this is set as the default. To use a different mode use the `--moe_normalization_mode` flag. -See [tensorrt_llm/layers/moe.py](../../../../tensorrt_llm/layers/moe.py#L56) for available values - - -## Quantization - -### Weight-only Quantization - -Mixtral supports weight only quantization - -```bash -# Build Mixtral8x7B with weight only -python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x7B-v0.1 \ - --output_dir ./tllm_checkpoint_mixtral_2gpu \ - --dtype float16 \ - --tp_size 2 \ - --use_weight_only \ - --weight_only_precision int8 -trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_2gpu \ - --output_dir ./trt_engines/mixtral/tp2 \ - --gemm_plugin float16 -``` - -### FP8 Post-Training Quantization - -Mixtral supports FP8 quantization, using Modelopt. See [`examples/models/core/llama/README.md`](../llama/README.md#fp8-post-training-quantization) for full details on installing Modelopt - -```bash -# Quantize HF Mixtral into FP8 and export trtllm checkpoint -python ../../../quantization/quantize.py --model_dir ./Mixtral-8x7B-v0.1 \ - --dtype float16 \ - --qformat fp8 \ - --kv_cache_dtype fp8 \ - --output_dir ./tllm_checkpoint_mixtral_2gpu \ - --calib_size 512 \ - --tp_size 2 - -# Build trtllm engines from the trtllm checkpoint -# Enable fp8 context fmha to get further acceleration by setting `--use_fp8_context_fmha enable` -trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_2gpu \ - --output_dir ./engine_outputs \ - --workers 2 -``` - -### AWQ Quantization - -Mixtral supports AWQ quantization using [AutoAWQ](https://github.com/casper-hansen/AutoAWQ). - -```bash -# Convert AutoAWQ HF checkpoints into TRT-LLM checkpoint -python ../llama/convert_checkpoint.py --model_dir ./tmp/mixtral-8x7b-v0.1-AWQ/ \ - --output_dir ./tllm_checkpoint_mixtral_awq_1gpu - -# Build trtllm engines from the trtllm checkpoint -trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_awq_1gpu \ - --output_dir ./engine_outputs -``` - -You may found `quant_algo = W4A16_GPTQ` in the configuration file of the converted checkpoints, and that's because AutoAWQ is using exactly the same components as GPTQ. - -### NVFP4 Post-Training Quantization - -Mixtral supports NVFP4 quantization. - -```bash -# Quantize HF Mixtral into FP8 and export trtllm checkpoint -python ../../../quantization/quantize.py --model_dir ./Mixtral-8x7B-v0.1 \ - --dtype float16 \ - --qformat nvfp4 \ - --kv_cache_dtype fp8 \ - --output_dir ./tllm_checkpoint_mixtral_nvfp4_1gpu \ - --calib_size 512 \ - --tp_size 1 - -# Build trtllm engines from the trtllm checkpoint -# Enable fp8 context fmha to get further acceleration by setting `--use_fp8_context_fmha enable` -trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_nvfp4_1gpu \ - --output_dir ./engine_outputs -``` - -## OOTB - -Mixtral supports OOTB operation without the plugin, however this comes at a significant performance cost. Users should prefer using the plugin path whenever possible diff --git a/examples/models/core/mixtral/requirements.txt b/examples/models/core/mixtral/requirements.txt deleted file mode 100644 index c1b98e5d3f7a..000000000000 --- a/examples/models/core/mixtral/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.0.dev0 -transformers==4.56.0 -accelerate==0.25.0 diff --git a/examples/models/core/multimodal/README.md b/examples/models/core/multimodal/README.md index 159e26d4c7d8..924561126cfb 100644 --- a/examples/models/core/multimodal/README.md +++ b/examples/models/core/multimodal/README.md @@ -1,1169 +1,17 @@ # Multi-Modal -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. +The engine-build multimodal workflow that used to live here +(`build_multimodal_engine.py` / `run.py` / `eval.py` on top of +`trtllm-build`) was removed together with the legacy TensorRT backend. -This document shows how to run multimodal pipelines with TensorRT-LLM, e.g. from image+text input modalities to text output. +Multimodal models are supported on the PyTorch backend. See: -Multimodal models' LLM part has an additional parameter `--max_multimodal_len` compared to LLM-only build commands. Under the hood, `max_multimodal_len` and `max_prompt_embedding_table_size` are effectively the same concept, i.e., prepended/concatenated embeddings (either multimodal feature embeddings or prompt tuning embeddings) to the LLM input embeddings. The multimodal features from the visual encoder of shape `[batch_size, num_visual_features, visual_hidden_dim]` is flattened as `[batch_size * num_visual_features, visual_hidden_dim]` and passed like a prompt embedding table. - -We first describe three runtime modes for running multimodal models and how to run each model on a single GPU. We then provide general guidelines on using tensor parallelism for the LLM part of the pipeline. - -- [Runtime Mode](#runtime-modes) -- [BLIP2](#blip2) -- [CogVLM](#cogvlm) -- [Deplot](#deplot) -- [Fuyu](#fuyu) -- [Gemma3](#gemma3) -- [InternLM-XComposer2](#internlm-xcomposer2) -- [InternVL2](#internvl2) -- [Kosmos-2](#kosmos-2) -- [LLaVA, LLaVa-NeXT, LLaVA-OneVision and VILA](#llava-llava-next-llava-onevision-and-vila) -- [MLLaMA](#mllama) -- [NeVA](#neva) -- [Nougat](#nougat) -- [Phi-3-vision](#phi-3-vision) -- [Phi-4-multimodal](#phi-4-multimodal) -- [Qwen2-VL](#qwen2-vl) -- [Qwen-Image-Bench Evaluator](#qwen-image-bench-evaluator) -- [Video NeVA](#video-neva) -- [Dataset Evaluation](#dataset-evaluation) -- [Enabling Tensor Parallelism for multi-GPU](#enabling-tensor-parallelism-for-multi-gpu) -- [Enabling Embedding Table Offloading](#enabling-embedding-table-offloading) - -## Runtime Modes -TensorRT LLM supports three runtime modes for running multimodal models. -- `cpp_llm_only` (default): vision engine runs in python runtime, LLM in pybind C++ runtime -- `python`: everything runs in python runtime -- `cpp`: everything runs in C++ runtime - -This can be specified by the `--session RUNTIME_MODE` argument in `run.py` (see instructions of each model below). -Not all models supports end-to-end `cpp` mode, the checked ones below are supported. See footnotes for reasons models are unsupported -- [ ] BLIP-2-T5 [^1] -- [x] BLIP-2-OPT -- [x] CogVLM -- [ ] Deplot [^1] -- [ ] Pix2Struct [^1] -- [x] Fuyu -- [x] InternVL2-2b -- [x] Kosmos-2 -- [x] LLaVA -- [ ] LLaVA-NeXT / OneVision [^2] -- [x] VILA [^3] -- [ ] Mllama [^1] -- [x] NeVA -- [ ] Nougat [^1] -- [ ] Phi-3-Vision [^2] -- [ ] Phi-4-multimodal -- [ ] Qwen2-VL [^4] -- [x] Video-NeVA - - -[^1]: Model uses cross attention to feed visiual features to LLM decoder, which is not supported -[^2]: Model requires post processing its encoder output features, which is not supported -[^3]: Currently C++ runtime only supports single image per request (VILA mode 2) -[^4]: Vision encoder requires additional inputs not supported by the C++ runtime - -## BLIP2 - -This BLIP section covers both BLIP2-OPT and BLIP2-T5, with minor changes needed when switching the LLM backbone. - -1. Download Huggingface weights and convert original checkpoint to TRT-LLM checkpoint format - following example in `examples/models/contrib/opt/README.md` and `examples/models/core/enc_dec/README.md`. - - ```bash - export MODEL_NAME="blip2-opt-2.7b" # options: blip2-opt-6.7b, blip2-flan-t5-xl, blip2-flan-t5-xxl - git clone https://huggingface.co/Salesforce/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - ``` - - For BLIP2-OPT family, - ```bash - python ../../contrib/opt/convert_checkpoint.py --model_type blip2 \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --dtype float16 - ``` - - For BLIP2-T5 family, - ```bash - python ../enc_dec/convert_checkpoint.py --model_type blip2 \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/bfloat16 \ - --tp_size 1 \ - --pp_size 1 \ - --dtype bfloat16 - ``` - -2. Build TRT-LLM engine from TRT-LLM checkpoint - - For BLIP2-OPT family, - ```bash - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gemm_plugin float16 \ - --max_beam_width 1 \ - --max_batch_size 8 \ - --max_seq_len 1024 \ - --max_input_len 924 \ - --max_multimodal_len 256 # 8 (max_batch_size) * 32 (num_visual_features) - ``` - - For BLIP2-T5 family, - ```bash - trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/bfloat16/encoder \ - --output_dir tmp/trt_engines/${MODEL_NAME}/bfloat16/llm/encoder \ - --paged_kv_cache disable \ - --moe_plugin disable \ - --gemm_plugin bfloat16 \ - --bert_attention_plugin bfloat16 \ - --gpt_attention_plugin bfloat16 \ - --remove_input_padding enable \ - --context_fmha disable \ - --max_beam_width 1 \ - --max_batch_size 8 \ - --max_input_len 924 \ - --max_multimodal_len 256 # 8 (max_batch_size) * 32 (num_visual_features) - - trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/bfloat16/decoder \ - --output_dir tmp/trt_engines/${MODEL_NAME}/bfloat16/llm/decoder \ - --paged_kv_cache disable \ - --moe_plugin disable \ - --gemm_plugin bfloat16 \ - --bert_attention_plugin bfloat16 \ - --gpt_attention_plugin bfloat16 \ - --remove_input_padding enable \ - --context_fmha disable \ - --max_beam_width 1 \ - --max_batch_size 8 \ - --max_seq_len 1024 \ - --max_encoder_input_len 924 \ - --max_input_len 1 # Same command for decoder but don't set --max_multimodal_len - ``` - - **NOTE**: `max_multimodal_len = max_batch_size * num_visual_features`, so if you change max_batch_size, max multimodal length **MUST** be changed accordingly. - -3. Build TensorRT engines for vision encoders - - ```bash - python build_multimodal_engine.py --model_type blip2 --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/bfloat16/vision --max_batch_size 8 - ``` - - The built engines are located in `tmp/trt_engines/${MODEL_NAME}/bfloat16/vision` for BLIP2-T5, similarly for BLIP-OPT. - - To run the BLIP2 pipeline with batch size > 1, change `--max_batch_size` argument to `build_multimodal_engine.py` accordingly. - -4. Assemble everything into BLIP2 pipeline - - For BLIP2-OPT family, - ```bash - python run.py \ - --max_new_tokens 30 \ - --input_text "Question: which city is this? Answer:" \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu - ``` - - For BLIP2-T5 family, - ```bash - python run.py \ - --max_new_tokens 30 \ - --input_text "Question: which city is this? Answer:" \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/bfloat16 - ``` - -5. (Optional) INT8/INT4 weight-only quantization for OPT can be enabled using commands as follows (take `INT4` as an example, while `INT8` is the default precision for weight-only quantization): - ```bash - python ../../contrib/opt/convert_checkpoint.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --dtype float16 \ - --output_dir tmp/trt_models/${MODEL_NAME}/int4_weightonly/1-gpu \ - --use_weight_only \ - --weight_only_precision int4 - - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/int4_weightonly/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/int4_weightonly/1-gpu/llm \ - --gemm_plugin float16 \ - --max_beam_width 1 \ - --max_batch_size 8 \ - --max_multimodal_len 256 \ - --max_input_len 924 \ - --max_seq_len 1024 - ``` - - The built OPT engines lie in `tmp/trt_engines/${MODEL_NAME}/int4_weightonly/1-gpu/llm`. - You should use this directory without the `llm` part as `--engine_dir` argument to `run.py` - - **NOTE:** INT8/INT4 option is not supported for BLIP2-T5, because quantization support has not been - added for encoder-decoder models yet. - -## CogVLM - -Currently, CogVLM only support bfloat16 precision. - -1. Download Huggingface weights - - ```bash - export MODEL_NAME="cogvlm-chat-hf" - git clone https://huggingface.co/THUDM/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - export TOKENIZER_NAME="vicuna-7b-v1.5" - git clone https://huggingface.co/lmsys/${TOKENIZER_NAME} tmp/hf_models/${TOKENIZER_NAME} - ``` - - Because currently onnx doesn't support `xops.memory_efficient_attention`, we need to modify some source code of the huggingface CogVLM. - ``` - cd tmp/hf_models/${MODEL_NAME} - sed -i '4s/.*//;40s/.*/ out = self.attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)).transpose(1, 2).contiguous()/;41s/.*//;42s/.*//' visual.py # It will replace memory_efficient_attention with some basic ops - ``` - -2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/cogvlm` - - CogVLM uses a Vit encoder as LLM encoder and a modified Llama as decoder. - - ```bash - python ../../contrib/cogvlm/convert_checkpoint.py --model_dir tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_models/${MODEL_NAME} --dtype bfloat16 --use_prompt_tuning - - trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME} \ - --output_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu/llm \ - --gemm_plugin bfloat16 \ - --gpt_attention_plugin bfloat16 \ - --remove_input_padding enable \ - --max_batch_size 48 \ - --max_input_len 2048 \ - --max_seq_len 3076 \ - --paged_kv_cache enable \ - --bert_attention_plugin disable \ - --moe_plugin disable \ - --max_multimodal_len 61440 # 48 (max_batch_size) * 1280 (max_num_visual_features) - ``` - -3. Generate TensorRT engines for visual components and combine everything into final pipeline. - - ```bash - python build_multimodal_engine.py --model_type cogvlm --model_path tmp/hf_models/${MODEL_NAME} --max_batch_size 48 --output_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu/vision - - python run.py \ - --max_new_tokens 1000 \ - --input_text " [INST] please describe this image in detail [/INST] " \ - --hf_model_dir tmp/hf_models/${TOKENIZER_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu \ - --batch_size 1 \ - --top_p 0.4 \ - --top_k 1 \ - --temperature 0.2 \ - --repetition_penalty 1.2 \ - --enable_context_fmha_fp32_acc - - CogVLM uses model_runner_cpp for its LLM decoder by default. To switch to model_runner, set `--session python` in the command mentioned above. - ``` - -## Deplot - -1. Download Huggingface weights and convert original checkpoint to TRT-LLM checkpoint format - following example in `examples/models/core/enc_dec/README.md`. - - ```bash - export MODEL_NAME="deplot" - git clone https://huggingface.co/google/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - - python ../enc_dec/convert_checkpoint.py --model_type pix2struct \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/float16 \ - --tp_size 1 \ - --pp_size 1 \ - --dtype float16 - ``` - -2. Build TRT-LLM engine from TRT-LLM checkpoint - - ```bash - trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/float16/decoder \ - --output_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/float16/llm/decoder \ - --paged_kv_cache disable \ - --moe_plugin disable \ - --gemm_plugin float16 \ - --bert_attention_plugin float16 \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --context_fmha disable \ - --max_beam_width 1 \ - --max_batch_size 8 \ - --max_seq_len 2558 \ - --max_encoder_input_len 2048 \ - --max_input_len 1 - ``` - - The built deplot engines are located in `tmp/trt_engines/${MODEL_NAME}/1-gpu/float16`. - -3. Build TensorRT engines for visual components - - ```bash - python build_multimodal_engine.py --model_type pix2struct --model_path tmp/hf_models/${MODEL_NAME} --max_batch_size 8 --output_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/float16/vision - ``` - - The built visual engines are located in `tmp/trt_engines/${MODEL_NAME}/1-gpu/float16/vision`. - - To run the deplot pipeline with batch size > 1, change `--max_batch_size` argument to `build_multimodal_engine.py` accordingly. - -4. Assemble everything into deplot pipeline - - ```bash - python run.py \ - --max_new_tokens 100 \ - --input_text "" \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/float16 - ``` - -## Fuyu - -1. Download Huggingface weights - - ```bash - export MODEL_NAME="fuyu-8b" - git clone https://huggingface.co/adept/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - ``` - -2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/models/core/gpt`. - The LLM portion of Fuyu uses a Persimmon model - ```bash - python ../gpt/convert_checkpoint.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --dtype float16 \ - --gpt_variant persimmon - - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gemm_plugin float16 \ - --use_fused_mlp=enable \ - --max_batch_size 1 \ - --max_input_len 2048 \ - --max_seq_len 2560 \ - --max_multimodal_len 2048 - ``` - -3. Generate TensorRT engines for visual components and combine everything into final pipeline. - - ```bash - python build_multimodal_engine.py --model_type fuyu --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision - - python run.py \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu - ``` - -## Gemma3 - -**NOTE: We only support Gemma3 VLMs in Pytorch workflow.** - -Gemma3VL decoder requires a custom attention mask while processing images. During the context phase: -- Text tokens attend to other tokens in a causal fashion (standard autoregressive behavior) -- Image tokens attend to other tokens in a causal fashion AND attend to other tokens from the same image in a bidirectional manner - -**Reference:** [Gemma3 Model Documentation](https://huggingface.co/docs/transformers/en/model_doc/gemma3) - -We support this custom mask with FlashInfer attention backend. - -### Requirements - -To ensure expected behavior with Gemma3VL, the following configurations are **required**: -- **Attention Backend**: Use the FlashInfer attention backend -- **Chunked Prefill**: Must be disabled -- **KV Cache Reuse**: Must be disabled - -### Quick Start - -#### 1. Download Model Weights - -```bash -export MODEL_NAME="gemma-3-27b-it" -git clone https://huggingface.co/google/${MODEL_NAME} -``` - -#### 2. Interactive Testing - -Use the `quickstart_multimodal.py` script for quick testing: - -```bash -python3 examples/llm-api/quickstart_multimodal.py \ - --model_dir ${MODEL_NAME}/ \ - --modality image \ - --image_format pil \ - --attention_backend FLASHINFER \ - --disable_kv_cache_reuse -``` - -#### 3. Model Serving - -Serve the model using `trtllm-serve` with the required llmapi arguments mentioned in a yaml file: - -```bash -# Create the configuration file -cat > extra-llm-api-options.yaml << 'EOF' -cuda_graph_config: null -attn_backend: "FLASHINFER" -enable_chunked_prefill: false -kv_cache_config: - enable_block_reuse: false -EOF - -# Serve the model -trtllm-serve ${MODEL_NAME}/ \ - --backend pytorch \ - --tp_size 1 \ - --port 8000 \ - --max_batch_size 4 \ - --config extra-llm-api-options.yaml -``` - -### Supported Model Variants - -Currently supported Gemma3 variants: 4B, 12B, 27B - - -## InternLM-XComposer2 - -**NOTE: We only support InternLM-XComposer-VL-7b for now** - -Firstly, please install transformers with 4.45.2 -```bash - pip install -r requirements-internlm-xcomposer2.txt -``` - -1. Convert Huggingface weights to TRT-LLM checkpoint format using `examples/models/contrib/internlm/README.md`. - -2. Use `trtllm-build` command to build TRT-LLM engine for OPT. - -3. The full list of commands is as follows: - - ```bash - export MODEL_NAME=internlm-xcomposer2-vl-7b - git lfs clone https://huggingface.co/internlm/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - - python ../internlm2/convert_checkpoint.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --dtype float16 \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu - - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gemm_plugin float16 \ - --lora_plugin float16 \ - --lora_dir . \ - --max_lora_rank 256 \ - --max_input_len 1536 \ - --max_batch_size 48 \ - --max_multimodal_len 58800 # 58800 = 1225(visual token/img) * 48 (max_batch_size), as each image corresponds to 1225 visual tokens in the ViT here - - python build_multimodal_engine.py \ - --model_type internlm-xcomposer2 \ - --model_path tmp/hf_models/${MODEL_NAME} \ - --output_dir trt_engines/${MODEL_NAME}/fp16/1-gpu/vision \ - --max_batch_size 48 - - python run.py \ - --max_new_tokens 200 \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir trt_engines/${MODEL_NAME}/fp16/1-gpu \ - --batch_size 1 - ``` - -## InternVL2 - -[InternVL Family](https://github.com/OpenGVLab/InternVL): Closing the Gap to Commercial Multimodal Models with Open-Source Suites —— A Pioneering Open-Source Alternative to GPT-4o. Here we show how to deploy InternVL2‑1B/InternVL2‑2B/InternVL2‑4B/InternVL2‑8B/InternVL2‑26B in TensorRT-LLM. - -Firstly, please install transformers with 4.37.2 -```bash - pip install transformers==4.37.2 -``` - -1. Download Huggingface weights - - For InternVL2-1B - ```bash - export MODEL_NAME="InternVL2-1B" - git clone https://huggingface.co/OpenGVLab/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - export LLM_MODEL_NAME="qwen" - ``` - - - For InternVL2-2B/InternVL2‑8B/InternVL2‑26B - ```bash - export MODEL_NAME="InternVL2-2B" # or InternVL2‑8B, InternVL2‑26B - git clone https://huggingface.co/OpenGVLab/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - export LLM_MODEL_NAME="internlm2" - ``` - - - For InternVL2-4B - ```bash - export MODEL_NAME="InternVL2-4B" - git clone https://huggingface.co/OpenGVLab/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - export LLM_MODEL_NAME="phi" - ``` - -2. Convert Huggingface weights into TRT-LLM checkpoints - ```bash - python ../${LLM_MODEL_NAME}/convert_checkpoint.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --dtype float16 - ``` - -3. Build TRT engines - ```bash - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gemm_plugin auto \ - --max_batch_size 1 \ - --max_input_len 4096 \ - --max_seq_len 4608 \ - --max_multimodal_len 3328 - ``` - -4. Generate TensorRT engines for visual components and combine everything into final pipeline. - ```bash - python build_multimodal_engine.py --model_type internvl --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision - python run.py \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/ \ - --image_path tmp/hf_models/${MODEL_NAME}/examples/image1.jpg - ``` - -5. (Optional) FP8 and INT8 SmoothQuant quantization is supported for the InternVL2-4B variant (LLM model only). - - ```bash - # FP8 quantization - python ../../../quantization/quantize.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp8/1-gpu \ - --dtype bfloat16 \ - --qformat fp8 \ - --kv_cache_dtype fp8 - - # INT8 SmoothQuant quantization - python ../../../quantization/quantize.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/int8/1-gpu \ - --dtype bfloat16 \ - --qformat int8_sq - ``` - - Then follow the same `trtllm-build`, `build_multimodal_engine.py` and `run.py` steps as before. - - -## Kosmos-2 - -1. Download Huggingface weights - - ```bash - export MODEL_NAME="kosmos-2" - git clone https://huggingface.co/microsoft/kosmos-2-patch14-224 tmp/hf_models/${MODEL_NAME} - ``` - -2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/models/core/gpt`. - ```bash - python ../gpt/convert_checkpoint.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --dtype float16 \ - --gpt_variant ${MODEL_NAME} - - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gpt_attention_plugin float16 \ - --gemm_plugin float16 \ - --max_batch_size 1 \ - --max_input_len 512 \ - --max_seq_len 1024 \ - --max_multimodal_len 64 # 1 (max_batch_size) * 64 (num_visual_features) - ``` - -3. Generate TensorRT engines for visual components and combine everything into final pipeline. - - ```bash - python build_multimodal_engine.py --model_type kosmos-2 --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision - - python run.py \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu - ``` - -## LLaVA, LLaVa-NeXT, LLaVA-OneVision and VILA - -[LLaVA](https://github.com/haotian-liu/LLaVA) and [VILA](https://github.com/Efficient-Large-Model/VILA) are both visual language models (VLM) that can be deployed in TensorRT LLM with many quantization options. [LLaVA-NeXT](https://huggingface.co/collections/llava-hf/llava-next-65f75c4afac77fd37dbbe6cf) is an extension of LLaVA. TRT-LLM currently supports [Mistral-7b](https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf) and [ Nous-Hermes-2-Yi-34B](https://huggingface.co/llava-hf/llava-v1.6-34b-hf) variant of LLaVA-NeXT. [LLaVA-OneVision](https://huggingface.co/collections/llava-hf/llava-onevision-66bb1e9ce8856e210a7ed1fe) is another extension of LLaVA. - -1. Download Huggingface model weights. These models have both visual and LLM components - unlike BLIP2 example which downloads only LLM components from Huggingface. - - For LLaVA, - - ```bash - export MODEL_NAME="llava-1.5-7b-hf" # also llava-1.5-13b-hf - git clone https://huggingface.co/llava-hf/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - ``` - For LLaVA-NeXT, - - ```bash - export MODEL_NAME="llava-v1.6-mistral-7b-hf" #for 34b variant "llava-v1.6-34b-hf" - git clone https://huggingface.co/llava-hf/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - ``` - - For LLaVA-OneVision, - - ```bash - export MODEL_NAME="llava-onevision-qwen2-7b-ov-hf" # also llava-onevision-qwen2-0.5b-ov-hf, llava-onevision-qwen2-72b-ov-hf, etc - git clone https://huggingface.co/llava-hf/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - ``` - - For VILA, we need a few more steps until it is added to HF model zoo - - ```bash - # install the following dependency - pip install -r requirements-vila.txt - - # clone original VILA repo - export VILA_PATH="tmp/hf_models/VILA" - git clone https://github.com/Efficient-Large-Model/VILA.git ${VILA_PATH} - - # download VILA checkpoints - export MODEL_NAME="vila1.5-3b" # NOTE: name must contain vila or VILA! it's used to identify whether we need to register the non-HF VILA codebase in HF Auto class - git clone https://huggingface.co/Efficient-Large-Model/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - ``` - -2. Generate TRT-LLM engine for LLaMA following example in `examples/models/core/llama/README.md` and `examples/models/core/qwen/README.md` - - ```bash - python ../llama/convert_checkpoint.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --dtype float16 - - # for LLaVA - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gemm_plugin float16 \ - --use_fused_mlp=enable \ - --max_batch_size 1 \ - --max_input_len 2048 \ - --max_seq_len 2560 \ - --max_multimodal_len 576 # 1 (max_batch_size) * 576 (num_visual_features) - - # for LLaVA-NeXT - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gpt_attention_plugin float16 \ - --gemm_plugin float16 \ - --use_fused_mlp=enable \ - --max_batch_size 1 \ - --max_input_len 4096 \ - --max_seq_len 5120 \ - --max_num_tokens 4096 \ - --max_multimodal_len 4096 # 1 (max_batch_size) * 4096 (max_input_len) - - # for VILA - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gemm_plugin float16 \ - --use_fused_mlp=enable \ - --max_batch_size 1 \ - --max_input_len 2048 \ - --max_seq_len 2560 \ - --max_multimodal_len 196 # 1 (max_batch_size) * 196 (num_visual_features) - - # for LLaVA-OneVision - python ../qwen/convert_checkpoint.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --dtype float16 - - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gemm_plugin float16 \ - --use_fused_mlp=enable \ - --max_batch_size 1 \ - --max_input_len 7228 \ - --max_seq_len 7328 \ - --max_multimodal_len 7128 # max_batch_size * num_visual_features(depends on the image size or the specified video num frame) - ``` - -3. Build TensorRT engines for visual components - - ```bash - python build_multimodal_engine.py --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision --model_type llava # for LLaVA - - python build_multimodal_engine.py --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision --model_type llava_next --max_batch_size 5 # 1 (max_batch_size) * 5 (because LLAVA-NeXT visual encoder can have at most 5 patches) # for LLaVA-NeXT - - python build_multimodal_engine.py --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision --model_type llava_onevision --max_batch_size 32 # max_batch_size * patch for image or frame for video # for LLaVA-OneVision - - python build_multimodal_engine.py --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision --model_type vila --vila_path ${VILA_PATH} # for VILA - ``` - - ```bash - python run.py \ - --max_new_tokens 30 \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ - --input_text "\n Which city is this?" # for LLaVA and for LLaVA-NeXT - - python run.py \ - --max_new_tokens 30 \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ - --image_path=https://github.com/Efficient-Large-Model/VILA/raw/main/demo_images/av.png,https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png \ - --input_text "\n Please elaborate what you see in the images?","\n Which city is this?" \ - --batch_size=2 # for LLaVA - ``` - - Note that Llava can support N pairs inference batching, `--batch_size=N` should be used. There should be N images listed under `--image_path` and N text prompts listed under `--input_text`. Don't forget to set the `--max_batch_size` and `--max_multimodal_len` during engine building. - - For LLaVA-OneVision, you can use either image or video as inputs. - ```bash - python run.py \ - --max_new_tokens 30 \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ - --input_text "What is shown in this image?" \ - --image_path image.png - - python run.py \ - --max_new_tokens 30 \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ - --input_text "Why is this video funny?" \ - --video_path video.mp4 - --video_num_frames 8 # sample uniformly 8 frames from the video, up to 32 frames - ``` - - For VILA, you can use either local file or web url as input images. - Suppose you have a local image `av.png` downloaded from `https://github.com/Efficient-Large-Model/VILA/blob/main/demo_trt_llm/av.png` and the url of `merlion.png` - ```bash - wget -O av.png https://raw.githubusercontent.com/Efficient-Large-Model/VILA/main/demo_images/av.png - - python run.py \ - --max_new_tokens 30 \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ - --image_path=av.png,https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png \ - --input_text="\n\n Please elaborate what you see in the images?" \ - --batch_size=1 # for VILA mode 1 - - python run.py \ - --max_new_tokens 30 \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ - --image_path=av.png,https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png \ - --input_text="\n Please elaborate what you see in the images?","\n Which city is this?" \ - --batch_size=2 \ - --check_accuracy # for VILA mode 2 - ``` - - Note that VILA can support different modes in terms of batching: - - Mode 1: if you want to query N images as a whole using a prompt, `--batch_size=1` should be used (which is the default value). Example is given above. - - Mode 2: if you want to query N pairs, `--batch_size=N` should be used. There should be N images listed under `--image_path` and N text prompts listed under `--input_text`. Don't forget to set the `--max_batch_size` and `--max_multimodal_len` during engine building. - - Note: use `--run_profiling` for performance measurement, use `--check_accuracy` for accuracy check. - -4. (Optional) Different quantization methods supported in LLaMA and Qwen can be applied to LLaVA/VILA/LLaVA-OneVision as well, such as INT4/INT8 weight-only, SmoothQuant, and INT4 Activation-Aware Quantization (AWQ). Detailed instructions can be found in LLaMA [README](../llama/README.md) and Qwen [README](../qwen/README.md). - - For example, - - ```bash - # INT4 weight only - python ../llama/convert_checkpoint.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --dtype float16 \ - --output_dir tmp/trt_models/${MODEL_NAME}/int4_weightonly/1-gpu \ - --use_weight_only \ - --weight_only_precision int4 - - # INT4 AWQ - python ../../../quantization/quantize.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/int4_awq/1-gpu \ - --dtype float16 \ - --qformat int4_awq \ - --calib_size 32 - ``` - - Then follow the same `trtllm-build` and `run.py` steps as before. NOTE: for `trtllm-build` command, do not use `--use_fused_mlp=enable` in these quantization modes. - -## MLLaMA - -This section shows how to build and run a LLaMA-3.2 Vision model in TensorRT-LLM. We use [Llama-3.2-11B-Vision/](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision) as an example. - -For LLaMA-3.2 text model, please refer to the [examples/models/core/llama/README.md](../llama/README.md) because it shares the model architecture of llama. - -### Support data types - * BF16 - * Tensor Parallel - * INT8 & INT4 Weight-Only - * FP8 - -### Build and run vision model - -* build engine of vision encoder model - -```bash -python examples/models/core/multimodal/build_multimodal_engine.py --model_type mllama \ - --model_path Llama-3.2-11B-Vision/ \ - --output_dir /tmp/mllama/trt_engines/vision/ -``` - -* build engine of decoder model - -```bash -python examples/models/core/mllama/convert_checkpoint.py --model_dir Llama-3.2-11B-Vision/ \ - --output_dir /tmp/mllama/trt_ckpts \ - --dtype bfloat16 - -trtllm-build --checkpoint_dir /tmp/mllama/trt_ckpts \ - --output_dir /tmp/mllama/trt_engines/llm/ \ - --max_num_tokens 4096 \ - --max_seq_len 2048 \ - --workers 1 \ - --gemm_plugin auto \ - --max_batch_size 4 \ - --max_encoder_input_len 4100 \ - --input_timing_cache model.cache -``` - -Note that for instruct Vision model, please set the `max_encoder_input_len` as `6404`. - -* Run test on multimodal/run.py with C++ runtime (LLM part only) - -```bash -python3 examples/models/core/multimodal/run.py --engine_dir /tmp/mllama/trt_engines/ \ - --hf_model_dir Llama-3.2-11B-Vision/ \ - --image_path https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg \ - --input_text "<|image|><|begin_of_text|>If I had to write a haiku for this one" \ - --max_new_tokens 50 \ - --batch_size 2 - -Use model_runner_cpp by default. To switch to model_runner, set `--session python` in the command mentioned above. - -python3 examples/models/core/multimodal/eval.py \ - --engine_dir /tmp/mllama/trt_engines/ \ - --hf_model_dir Llama-3.2-11B-Vision/ \ - --test_trtllm \ - --accuracy_threshold 65 \ - --eval_task lmms-lab/ai2d -``` - -### Run MLLaMA decoder part by FP8 - -```bash -# install modelopt 0.21.0 -pip install nvidia-modelopt[torch]~=0.21.0 - -python ./examples/quantization/quantize.py --model_dir Llama-3.2-11B-Vision/ \ - --dtype bfloat16 \ - --qformat fp8 \ - --output_dir /tmp/llama-3.2-11B-Vision/fp8/ \ - --kv_cache_dtype fp8 \ - --calib_size 512 \ - --calib_dataset scienceqa - -trtllm-build --checkpoint_dir /tmp/llama-3.2-11B-Vision/fp8/ \ - --output_dir /tmp/trt_engines/llama-3.2-11B-Vision/fp8/llm \ - --max_num_tokens 4096 \ - --max_seq_len 2048 \ - --workers 1 \ - --gemm_plugin auto \ - --max_batch_size 4 \ - --max_encoder_input_len 4100 \ - --input_timing_cache model.cache \ - --use_paged_context_fmha enable \ - --use_fp8_context_fmha enable - -# copy visiual engine directory `/tmp/mllama/trt_engines/vision/` to fp8 engine directory `/tmp/trt_engines/llama-3.2-11B-Vision/fp8/vision` - -python3 examples/models/core/multimodal/run.py --engine_dir /tmp/trt_engines/llama-3.2-11B-Vision/fp8/ \ - --hf_model_dir Llama-3.2-11B-Vision/ \ - --image_path https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg \ - --input_text "<|image|><|begin_of_text|>If I had to write a haiku for this one" \ - --max_new_tokens 50 \ - --batch_size 2 - -python3 examples/models/core/multimodal/eval.py --engine_dir /tmp/trt_engines/llama-3.2-11B-Vision/fp8/ \ - --hf_model_dir Llama-3.2-11B-Vision/ \ - --test_trtllm \ - --accuracy_threshold 65 \ - --eval_task lmms-lab/ai2d -``` - -Note that for instruct Vision model, please set the `max_encoder_input_len` as `6404`. - -## NeVA - -[NeVA](https://docs.nvidia.com/nemo-framework/user-guide/24.12/nemotoolkit/multimodal/mllm/neva.html) is a groundbreaking addition to the NeMo Multimodal ecosystem. This model seamlessly integrates large language-centric models with a vision encoder, that can be deployed in TensorRT-LLM. - -1. Generate TRT-LLM engine for NVGPT following example in `examples/models/core/gpt/README.md`. To adhere to the NVGPT conventions of the conversion script, some layer keys have to be remapped using `--nemo_rename_key`. - - ```bash - export MODEL_NAME="neva" - python ../gpt/convert_checkpoint.py \ - --nemo_ckpt_path ./${MODEL_NAME}.nemo \ - --dtype bfloat16 \ - --output_dir tmp/trt_models/${MODEL_NAME} \ - --nemo_rename_key model:model.language_model \ - attention.linear_qkv.layer_norm_bias:input_layernorm.bias \ - attention.linear_qkv.layer_norm_weight:input_layernorm.weight \ - mlp.linear_fc1.layer_norm_bias:post_attention_layernorm.bias \ - mlp.linear_fc1.layer_norm_weight:post_attention_layernorm.weight \ - linear_qkv:query_key_value \ - linear_fc1:dense_h_to_4h \ - linear_fc2:dense_4h_to_h \ - linear_proj:dense \ - decoder:encoder - - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME} \ - --output_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu/llm \ - --gpt_attention_plugin bfloat16 \ - --gemm_plugin bfloat16 \ - --max_batch_size 1 \ - --max_input_len 2048 \ - --max_seq_len 2560 \ - --max_multimodal_len 729 # 1 (max_batch_size) * 729 (num_visual_features) - ``` - -2. Build TensorRT engines for visual components - - ```bash - python build_multimodal_engine.py --model_path ./${MODEL_NAME}.nemo --model_type neva --output_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu/vision - ``` - - ```bash - python run.py \ - --max_new_tokens 30 \ - --hf_model_dir tmp/trt_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu \ - --input_text "Question: which city is this? Answer:" - ``` - - Note: use `--run_profiling` for performance measurement, use `--check_accuracy` for accuracy check. - -## Nougat - -1. Download Huggingface weights - - ```bash - export MODEL_NAME="nougat-base" # also nougat-small - git clone https://huggingface.co/facebook/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - ``` - -2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/models/core/enc_dec` - - Nougat uses mBART architecture but replaces the LLM encoder with a Swin Transformer encoder. - To achieve this, we add an extra `--nougat` flag (over mBART example) to - `convert_checkpoint.py` in `examples/models/core/enc_dec` and `trtllm-build`. - - ```bash - python ../enc_dec/convert_checkpoint.py --model_type bart \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/bfloat16 \ - --tp_size 1 \ - --pp_size 1 \ - --dtype bfloat16 \ - --nougat - - trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/bfloat16/decoder \ - --output_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/bfloat16/llm/decoder \ - --paged_kv_cache disable \ - --moe_plugin disable \ - --gemm_plugin bfloat16 \ - --bert_attention_plugin bfloat16 \ - --gpt_attention_plugin bfloat16 \ - --remove_input_padding enable \ - --max_beam_width 1 \ - --max_batch_size 1 \ - --max_seq_len 101 \ - --max_input_len 1 \ - --max_encoder_input_len 588 # 1 (max_batch_size) * 588 (num_visual_features) - ``` - -3. Generate TensorRT engines for visual components and combine everything into final pipeline. - - ```bash - python build_multimodal_engine.py --model_type nougat --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/bfloat16/vision - - python run.py \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/bfloat16 - ``` - - Note: Nougat models usually do not need a text prompt. - - -## Phi-3-vision - -1. Download Huggingface weights - - ```bash - export MODEL_NAME="Phi-3-vision-128k-instruct" # or Phi-3.5-vision-instruct - git clone https://huggingface.co/microsoft/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - ``` - -2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/models/core/phi`. - ```bash - python ../phi/convert_checkpoint.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --dtype float16 - - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gpt_attention_plugin float16 \ - --gemm_plugin float16 \ - --max_batch_size 1 \ - --max_input_len 4096 \ - --max_seq_len 4608 \ - --max_multimodal_len 4096 - ``` - -3. Generate TensorRT engines for visual components and combine everything into final pipeline. - - ```bash - python build_multimodal_engine.py --model_type phi-3-vision --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision - - python run.py \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --kv_cache_free_gpu_memory_fraction 0.7 \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/ \ - --image_path=https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png - ``` -## Phi-4-multimodal -Navigate to the folder `TensorRT-LLM/examples/models/core/multimodal` - -1. Download Huggingface weights - - ```bash - export MODEL_NAME="Phi-4-multimodal-instruct" - export HF_DIR="tmp/hf_models/${MODEL_NAME}" - export CKPT_DIR="tmp/trt_models/${MODEL_NAME}/fp16/1-gpu" - export ENGINE_DIR="tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu" - git clone https://huggingface.co/microsoft/${MODEL_NAME} ${HF_DIR} - - ``` - -2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/models/core/phi`. - ```bash - python ../phi/convert_checkpoint.py \ - --model_dir ${HF_DIR} \ - --output_dir ${CKPT_DIR} \ - --dtype float16 - - trtllm-build \ - --checkpoint_dir ${CKPT_DIR} \ - --output_dir ${ENGINE_DIR} \ - --gpt_attention_plugin float16 \ - --gemm_plugin float16 \ - --max_batch_size 1 \ - --max_input_len 4096 \ - --max_seq_len 4608 \ - --max_multimodal_len 4096 - ``` - -3. Generate TensorRT engines for visual components and combine everything into final pipeline. -*Note: the encoders are not the TRT engines but are pure Pytorch ones* - - ```bash - python build_multimodal_engine.py --model_type phi-4-multimodal --model_path ${HF_DIR} --output_dir ${ENGINE_DIR} - - python run.py \ - --hf_model_dir ${HF_DIR} \ - --kv_cache_free_gpu_memory_fraction 0.7 \ - --engine_dir ${ENGINE_DIR} \ - --image_path=https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png - --audio_path=${HF_DIR}/examples/what_is_shown_in_this_image.wav - ``` -## Qwen2-VL -[Qwen2-VL Family](https://github.com/QwenLM/Qwen2-VL): is the latest version of the vision language models in the Qwen model families. Here we show how to deploy Qwen2-VL 2B and 7B in TensorRT-LLM. - -Firstly, please install transformers and qwen-vl-utils -```bash -pip install -r requirements-qwen2vl.txt -``` -### Support data types - * FP16 - * FP8 - -### Build and run vision model -* Download Huggingface weights - ```bash - export MODEL_NAME="Qwen2-VL-7B-Instruct" # or Qwen2-VL-2B-Instruct - git clone https://huggingface.co/Qwen/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} - ``` -* Build engine of decoder model - - ```bash - python3 ../qwen/convert_checkpoint.py \ - --model_dir=tmp/hf_models/${MODEL_NAME} \ - --output_dir=tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --dtype float16 - - trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ - --gemm_plugin=float16 \ - --gpt_attention_plugin=float16 \ - --max_batch_size=4 \ - --max_input_len=2048 \ - --max_seq_len=3072 \ - --max_multimodal_len=1296 #(max_batch_size) * 324 (num_visual_features), this's for image_shape=[504,504] - ``` - -* Build engine of vision encoder model - ```bash - python build_multimodal_engine.py --model_type qwen2_vl --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision - ``` - -* Run test on multimodal/run.py with C++ runtime (LLM part only) - ```bash - python3 run.py \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/ - ``` -### Run Qwen2-VL decoder part by FP8 -* Build engine - ```bash - python ./examples/quantization/quantize.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --dtype float16 \ - --qformat fp8 \ - --kv_cache_dtype fp8 \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp8/1-gpu \ - --calib_size 512 - - trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp8/1-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp8/1-gpu/llm \ - --max_input_len=2048 \ - --max_seq_len 3072 \ - --gemm_plugin auto \ - --max_batch_size 4 \ - --max_multimodal_len=1296 - - # copy visiual engine directory `tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision/` to fp8 engine directory `tmp/trt_engines/${MODEL_NAME}/fp8/1-gpu/vision` - ``` -* Run test on multimodal/run.py with C++ runtime (LLM part only) - ```bash - python3 run.py \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp8/1-gpu/ - ``` +- [Supported models](https://nvidia.github.io/TensorRT-LLM/models/supported-models.html) +- [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) and the + [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) +- Multimodal serving examples under `examples/llm-api/` and + `examples/serve/` ## Qwen-Image-Bench Evaluator @@ -1183,153 +31,3 @@ python examples/models/core/multimodal/qwen_image_bench_eval.py \ The script evaluates all five Qwen-Image-Bench level-1 dimensions by default: Quality, Aesthetics, Alignment, Real-world Fidelity, and Creative Generation. - -## Video NeVA - -[Video NeVA](https://github.com/NVIDIA/NeMo/blob/main/docs/source/multimodal/mllm/video_neva.rst) is a groundbreaking addition to the NeMo Multimodal ecosystem that could work with video modality. This model seamlessly integrates large language-centric models with a vision encoder, that can be deployed in TensorRT-LLM. - -1. Generate TRT-LLM engine for Nemotron model following example in `examples/models/core/nemotron/README.md`. To adhere to the NVGPT conventions of the conversion script. This will be used as our base LM for inference. - - ```bash - pip install decord # used for loading video - - python3 ../../../quantization/quantize.py \ - --nemo_ckpt_path /path/to/nemotron/model.nemo \ - --dtype bfloat16 \ - --batch_size 64 \ - --qformat full_prec \ - --output_dir nemotron-3/trt_ckpt/bf16/1-gpu - - - trtllm-build \ - --checkpoint_dir nemotron-3/trt_ckpt/bf16/1-gpu \ - --output_dir tmp/trt_engines/nemotron-3/bf16/1-gpu/llm \ - --gpt_attention_plugin bfloat16 \ - --gemm_plugin bfloat16 \ - --max_batch_size 1 \ - --max_input_len 4096 \ - --max_seq_len 4352 \ - --max_multimodal_len 3072 # 1 (max_batch_size) * (12 num_frames) * (256 image_token_len) - ``` - -2. Build TensorRT engines for visual components - - ```bash - python build_multimodal_engine.py --model_path /path/to/video/neva/projector.nemo --model_type video-neva --output_dir tmp/trt_engines/nemotron-3/visual_encoder --output_dir tmp/trt_engines/nemotron-3/bf16/1-gpu/vision - ``` - - ```bash - python run.py \ - --max_new_tokens 30 \ - --hf_model_dir nemotron-3/trt_ckpt/bf16/1-gpu \ - --engine_dir tmp/trt_engines/nemotron-3/bf16/1-gpu \ - --input_text "Question: what is in the video? Answer:" \ - --video_path /path/to/your/local/video/file - ``` - - Note: use `--run_profiling` for performance measurement, use `--check_accuracy` for accuracy check. - -## Dataset Evaluation - -This section explains how to evaluate datasets using our provided script, including supported models and configurations. - -### Evaluation Command -To run an evaluation, use the following command: - -```bash -python ./examples/models/core/multimodal/eval.py \ - --model_type \ - --engine_dir \ - --hf_model_dir \ - --dataset_dir \ - --test_trtllm (or --test_hf, or both) \ - --accuracy_threshold \ - --eval_task \ - --max_ite 20 \ - --visual_engine_name -``` - -### Parameters -- `--model_type`: Specify the model type to evaluate. -- `--engine_dir`: Path to the model engines directory. -- `--hf_model_dir`: Path to the Hugging Face model directory. -- `--dataset_dir`: Path to the dataset directory. If not specified, will load the dataset from HF with the `--eval_task` as dataset tag. -- `--test_trtllm` or `--test_hf`: Specify which evaluation framework to use. Both can be used simultaneously. -- `--accuracy_threshold`: Set the accuracy threshold for evaluation. -- `--eval_task`: Specify the evaluation task. Supported tasks: `['lmms-lab/ai2d', 'lmms-lab/VQAv2', 'lmms-lab/MME']`. Default to `'lmms-lab/VQAv2'`. -- `--max_ite`: Maximum number of iterations, default to 20. -- `--visual_engine_name`: Name of the visual engine. - -### Supported Evaluation Tasks -The following evaluation tasks are supported: -- `lmms-lab/ai2d` -- `lmms-lab/VQAv2` -- `lmms-lab/MME` - -### Supported Model Types -The script supports the following model types: -- `blip2` -- `fuyu` -- `kosmos-2` -- `llava` -- `llava_next` -- `llava_onevision` -- `phi-3-vision` -- `qwen2_vl` -- `mllama` -- `vila` -- `cogvlm` -- `neva` -- `internvl` - -**Note:** The models `vila`, `cogvlm`, `neva`, and `internvl` do not support the `--test_hf` evaluation framework. - -## Enabling Tensor Parallelism for multi-GPU - -The LLM part of the pipeline can be run on multiple GPUs using tensor parallelism. -The visual encoder will be replicated on each GPU and operate in a data parallel fashion. - -To enable tensor parallelism, both weight conversion step (from Huggingface to FT format) -and engine building step should use additional arguments. Finally `run.py` should be prefixed -with `mpirun -n NUM_GPUS --allow-run-as-root`. - -The full set of commands to enable 2-way tensor parallelism for LLaVA is: - - ```bash - export MODEL_NAME="llava-1.5-7b-hf" - - python ../llama/convert_checkpoint.py \ - --model_dir tmp/hf_models/${MODEL_NAME} \ - --output_dir tmp/trt_models/${MODEL_NAME}/fp16/2-gpu \ - --dtype float16 --tp_size 2 - - trtllm-build \ - --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/2-gpu \ - --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/2-gpu/llm \ - --gemm_plugin float16 \ - --max_batch_size 1 \ - --max_input_len 2048 \ - --max_seq_len 2560 \ - --max_multimodal_len 576 - - python build_multimodal_engine.py --model_type llava --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/2-gpu/vision - - mpirun -n 2 --allow-run-as-root \ - python run.py \ - --max_new_tokens 30 \ - --hf_model_dir tmp/hf_models/${MODEL_NAME} \ - --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/2-gpu \ - ``` -## Enabling Embedding Table Offloading - -Embedding Table Offloading is a memory optimization technique that helps manage large embedding tables more efficiently. It offloads the embedding table to CPU memory and uses a chunked prefetching mechanism during processing. This approach is only available when operating in context chunk mode. - -To enable this feature, use the `--mm_embedding_offloading` argument: -```bash -python run.py \ - --enable_chunked_context \ - --mm_embedding_offloading true \ - --hf_model_dir ${HF_MODEL_PATH} \ - --engine_dir ${ENGINE_PATH} -``` -When not explicitly specified, this feature automatically enables if you're using a multimodal model along with context chunking enabled. diff --git a/examples/models/core/multimodal/__init__.py b/examples/models/core/multimodal/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/examples/models/core/multimodal/build_multimodal_engine.py b/examples/models/core/multimodal/build_multimodal_engine.py deleted file mode 100644 index 59e8bb4ffa25..000000000000 --- a/examples/models/core/multimodal/build_multimodal_engine.py +++ /dev/null @@ -1,12 +0,0 @@ -import argparse - -from tensorrt_llm.tools.multimodal_builder import (MultimodalEngineBuilder, - add_multimodal_arguments) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser = add_multimodal_arguments(parser) - args = parser.parse_args() - - builder = MultimodalEngineBuilder(args) - builder.build() diff --git a/examples/models/core/multimodal/eval.py b/examples/models/core/multimodal/eval.py deleted file mode 100644 index 01804d290dee..000000000000 --- a/examples/models/core/multimodal/eval.py +++ /dev/null @@ -1,317 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os - -import aiohttp -import datasets -import torch -from transformers import AutoProcessor -from utils import add_common_args - -import tensorrt_llm -import tensorrt_llm.profiler as profiler -from tensorrt_llm import logger -from tensorrt_llm.runtime import MultimodalModelRunner - -SUPPORTED_MODEL_TYPES = { - 'blip2': 'Blip2ForConditionalGeneration', - 'fuyu': 'FuyuForCausalLM', - 'kosmos-2': 'Kosmos2ForConditionalGeneration', - 'llava': 'LlavaForConditionalGeneration', - 'llava_next': 'LlavaNextForConditionalGeneration', - 'llava_onevision': 'LlavaOnevisionForConditionalGeneration', - 'phi-3-vision': 'AutoModelForCausalLM', - 'qwen2_vl': 'Qwen2VLForConditionalGeneration', # not tested for TRT-LLM yet - 'mllama': 'MllamaForConditionalGeneration', - 'vila': None, - 'cogvlm': None, # not tested for TRT-LLM yet - 'neva': None, # not tested for TRT-LLM yet - 'internvl': None, -} -EVAL_TASKS = ['lmms-lab/ai2d', 'lmms-lab/VQAv2', 'lmms-lab/MME'] - - -def parse_arguments(args=None): - parser = argparse.ArgumentParser() - parser = add_common_args(parser) - parser.add_argument('--test_trtllm', - action='store_true', - default=None, - help="Evaluate the TensorRT-LLM.") - parser.add_argument('--test_hf', - action='store_true', - default=None, - help="Evaluate the Huggingface.") - parser.add_argument('--max_ite', type=int, default=20) - parser.add_argument('--eval_task', - type=str, - choices=EVAL_TASKS, - default='lmms-lab/VQAv2') - parser.add_argument('--model_type', - type=str, - default=None, - choices=SUPPORTED_MODEL_TYPES.keys()) - parser.add_argument( - '--accuracy_threshold', - type=float, - default=None, - help= - 'used to check the accuracy of test_trtllm. Should be between 0 and 100.' - ) - parser.add_argument( - '--dataset_dir', - type=str, - default=None, - help="The local directory of the dataset for evaluation; " - "will download the dataset from huggingface hub if not specified.") - parser.add_argument( - '--dataset_cache_dir', - type=str, - default=None, - help="The local cache directory for dataset; " - "will use `~/.cache/huggingface/datasets` if not specified.") - return parser.parse_args(args=args) - - -def load_dataset(args) -> datasets.Dataset: - split_name = 'validation' if 'VQAv2' in args.eval_task else 'test' - - if args.dataset_dir is not None and os.path.exists( - os.path.join(args.dataset_dir, "dataset_info.json")): - logger.info(f"load dataset by load_from_disk from {args.dataset_dir}") - dataset = datasets.load_from_disk(args.dataset_dir) - - else: - logger.info( - f"load dataset by load_dataset from {args.dataset_dir or args.eval_task}" - ) - dataset = datasets.load_dataset( - args.dataset_dir or args.eval_task, - cache_dir=args.dataset_cache_dir, - split=split_name, - storage_options={ - 'client_kwargs': { - 'timeout': aiohttp.ClientTimeout(total=3600) - } - }, - trust_remote_code=True, - ) - return dataset - - -def load_hf_model(args): - if SUPPORTED_MODEL_TYPES[args.model_type] is None: - raise ValueError(f"Unsupported HF model_type: {args.model_type}") - profiler.start('load HF model') - model_class = getattr(__import__('transformers'), - SUPPORTED_MODEL_TYPES[args.model_type]) - hf_model = model_class.from_pretrained(args.hf_model_dir, - dtype=torch.float16, - device_map="cuda:0", - trust_remote_code=True) - profiler.stop('load HF model') - - logger.info( - f'Load HF model takes: {profiler.elapsed_time_in_sec("load HF model")} sec' - ) - return hf_model - - -def load_trtllm_model(args): - profiler.start('load TensorRT LLM model') - trtllm_model = MultimodalModelRunner(args) - profiler.stop('load TensorRT LLM model') - logger.info( - f'Load TensorRT LLM model takes: {profiler.elapsed_time_in_sec("load TensorRT LLM model")} sec' - ) - return trtllm_model - - -def prepare_prompts(task, data, model_type, processor) -> str: - prompts = None - question = data['question'] - if question[-1] != '?': - question += '?' - - if task == 'lmms-lab/ai2d': - for j, option in enumerate(data['options']): - question += f" ({j}) {option}" - - if model_type in ['blip2', 'neva']: - prompts = f"Question: {question} Answer: " - elif model_type == 'fuyu': - prompts = f"Answer the following {task} question based on the image: {question}" - elif model_type == 'kosmos-2': - prompts = f" Question: {question} Answer: " - elif model_type == 'cogvlm': - prompts = f"A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {question} ASSISTANT:" - elif model_type in ['llava', 'llava_next', 'llava_onevision']: - conversation = [ - { - "role": - "user", - "content": [ - { - "type": "image" - }, - { - "type": "text", - "text": question - }, - ], - }, - ] - prompts = processor.apply_chat_template(conversation, - add_generation_prompt=True) - elif model_type in ['vila', 'internvl']: - prompts = f"\n{question}" - elif model_type == 'phi-3-vision': - messages = [ - { - "role": "user", - "content": f"<|image_1|>\n{question}" - }, - ] - prompts = processor.tokenizer.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True) - elif model_type == 'qwen2_vl': - conversation = [{ - "role": - "user", - "content": [{ - "type": "image", - }, { - "type": "text", - "text": question - }] - }] - prompts = processor.apply_chat_template(conversation, - add_generation_prompt=True) - elif model_type == 'mllama': - prompts = processor.apply_chat_template(images=data['image'], - text=question + "; answer: ") - else: - raise ValueError(f"Unsupported model_type: {model_type}") - - return prompts - - -def eval(output, task, data) -> bool: - output = output.strip().lower() - if task == 'lmms-lab/VQAv2': - return any(answer['answer'] in output for answer in data['answers']) - else: - return data['answer'].lower() in output - - -os.environ["TOKENIZERS_PARALLELISM"] = "false" -args = parse_arguments() -if args.model_type not in SUPPORTED_MODEL_TYPES: - raise ValueError(f"Unsupported model_type: {args.model_type}") - -logger.set_level(args.log_level) - -runtime_rank = tensorrt_llm.mpi_rank() -dataset = load_dataset(args) -hf_model = load_hf_model(args) if args.test_hf else None -trtllm_model = load_trtllm_model(args) if args.test_trtllm else None -if SUPPORTED_MODEL_TYPES[args.model_type] is None: - hf_processor = None -else: - hf_processor = AutoProcessor.from_pretrained(args.hf_model_dir, - trust_remote_code=True) -hf_correct = trtllm_correct = 0 - -if args.model_type == 'mllama': - from tensorrt_llm.runtime.processor_wrapper import MllamaProcessorWrapper - hf_processor = MllamaProcessorWrapper(hf_processor, logger) - -torch.random.manual_seed(0) -profiler.start('evaluation') -if args.test_trtllm or args.test_hf: - for i in range(args.max_ite): - logger.debug(f"Ite: {i:3d}") - data = dataset[i] - if i > len(dataset): - break - prompts = prepare_prompts(args.eval_task, data, args.model_type, - hf_processor) - image = data['image'] - - if args.test_hf: - assert hf_model is not None, f"Unsupported HF model_type: {args.model_type}" - profiler.start('hf') - inputs = hf_processor( - images=image, - text=prompts, - return_tensors="pt", - ).to(hf_model.device, - torch.float16) # add torch.float16 for llava-onevision - input_length = inputs.input_ids.shape[-1] - hf_output = hf_model.generate(**inputs, - max_new_tokens=args.max_new_tokens) - hf_result = (hf_processor.batch_decode( - hf_output, skip_special_tokens=True)[0] if args.model_type in [ - 'blip2' - ] else hf_processor.decode(hf_output[0][input_length:], - skip_special_tokens=True)) - hf_correct += eval(hf_result, args.eval_task, data) - profiler.stop('hf') - - if args.test_trtllm: - profiler.start('tensorrt_llm') - _, output_text = trtllm_model.run( - input_text=prompts, - input_image=image, - input_audio=None, - max_new_tokens=args.max_new_tokens) - if runtime_rank == 0: - trtllm_result = output_text[0][0] - trtllm_correct += eval(trtllm_result, args.eval_task, data) - profiler.stop('tensorrt_llm') - - if runtime_rank == 0: - if args.eval_task == 'lmms-lab/VQAv2': - answer = data['answers'] - else: - answer = data['answer'] - logger.debug(f"prompts: {prompts}") - logger.debug(f"reference answer: {answer}") - if args.test_hf: - logger.debug(f"HF's answer: {hf_result}") - if args.test_trtllm: - logger.debug(f"TRT-LLM's answer: {trtllm_result}") - - if runtime_rank == 0: - logger.info(f"total iterations: {args.max_ite}") - if args.test_hf: - logger.info( - f"HF's accuracy: {100 * hf_correct / args.max_ite:4.2f}%") - if args.test_trtllm: - logger.info( - f"TRT-LLM's accuracy: {100 * trtllm_correct / args.max_ite:4.2f}%" - ) - # check if the accuracy is above the threshold - if args.accuracy_threshold is not None and args.test_trtllm: - assert trtllm_correct / args.max_ite >= args.accuracy_threshold / 100, \ - f"TRT-LLM's accuracy is below the threshold: {args.accuracy_threshold}%." -else: - logger.info("Neither enable test_trtllm nor enable test_hf") - -profiler.stop('evaluation') -logger.info( - f'Evaluation takes: {profiler.elapsed_time_in_sec("evaluation")} sec') diff --git a/examples/models/core/multimodal/requirements-eclair.txt b/examples/models/core/multimodal/requirements-eclair.txt deleted file mode 100644 index 281c8ae93a91..000000000000 --- a/examples/models/core/multimodal/requirements-eclair.txt +++ /dev/null @@ -1 +0,0 @@ -timm diff --git a/examples/models/core/multimodal/requirements-internlm-xcomposer2.txt b/examples/models/core/multimodal/requirements-internlm-xcomposer2.txt deleted file mode 100644 index 5d27a97ebb68..000000000000 --- a/examples/models/core/multimodal/requirements-internlm-xcomposer2.txt +++ /dev/null @@ -1 +0,0 @@ -transformers==4.56.0 diff --git a/examples/models/core/multimodal/requirements-llava_onevision.txt b/examples/models/core/multimodal/requirements-llava_onevision.txt deleted file mode 100644 index 126cbda08e15..000000000000 --- a/examples/models/core/multimodal/requirements-llava_onevision.txt +++ /dev/null @@ -1,4 +0,0 @@ -git+https://github.com/LLaVA-VL/LLaVA-NeXT.git -transformers>=4.56.0 -einops -av diff --git a/examples/models/core/multimodal/requirements-qwen2vl.txt b/examples/models/core/multimodal/requirements-qwen2vl.txt deleted file mode 100644 index 50f14d1d8095..000000000000 --- a/examples/models/core/multimodal/requirements-qwen2vl.txt +++ /dev/null @@ -1,2 +0,0 @@ -accelerate -qwen-vl-utils==0.0.8 # 0.0.9 has bug https://github.com/QwenLM/Qwen2-VL/pull/673, rollback until a newer version is released diff --git a/examples/models/core/multimodal/requirements-vila.txt b/examples/models/core/multimodal/requirements-vila.txt deleted file mode 100644 index 00775aa0cdfa..000000000000 --- a/examples/models/core/multimodal/requirements-vila.txt +++ /dev/null @@ -1,2 +0,0 @@ -git+https://github.com/bfshi/scaling_on_scales.git -transformers==4.56.0 diff --git a/examples/models/core/multimodal/run.py b/examples/models/core/multimodal/run.py deleted file mode 100644 index fb13554848e7..000000000000 --- a/examples/models/core/multimodal/run.py +++ /dev/null @@ -1,128 +0,0 @@ -import argparse -import os - -from utils import add_common_args, compute_str_match_rate - -import tensorrt_llm -import tensorrt_llm.profiler as profiler -from tensorrt_llm import logger -from tensorrt_llm.runtime import MultimodalModelRunner - - -def print_result(model, input_text, output_text, args): - logger.info("---------------------------------------------------------") - if model.model_type != 'nougat': - logger.info(f"\n[Q] {input_text}") - for i in range(len(output_text)): - logger.info(f"\n[A]: {output_text[i]}") - - if args.num_beams == 1: - output_ids = model.tokenizer(output_text[0][0], - add_special_tokens=False)['input_ids'] - logger.info(f"Generated {len(output_ids)} tokens") - - if args.check_accuracy: - if model.model_type != 'nougat': - if model.model_type == "vila": - for i in range(len(args.image_path.split(args.path_sep))): - if i % 2 == 0: - assert output_text[i][0].lower( - ) == "the image captures a bustling city intersection teeming with life. from the perspective of a car's dashboard camera, we see" - else: - assert output_text[i][0].lower( - ) == "the image captures the iconic merlion statue in singapore, a renowned worldwide landmark. the merlion, a mythical" - elif model.model_type == "llava": - for i in range(len(args.image_path.split(args.path_sep))): - assert output_text[i][0].lower() == 'singapore' - elif model.model_type == 'fuyu': - assert output_text[0][0].lower() == '4' - elif model.model_type == "pix2struct": - assert "characteristic | cat food, day | cat food, wet | cat treats" in output_text[ - 0][0].lower() - elif model.model_type in [ - 'blip2', 'neva', 'phi-3-vision', 'llava_next', - 'phi-4-multimodal', 'pixtral' - ]: - assert 'singapore' in output_text[0][0].lower() - elif model.model_type == 'video-neva': - assert 'robot' in output_text[0][0].lower() - elif model.model_type == 'kosmos-2': - assert 'snowman' in output_text[0][0].lower() - elif model.model_type == "mllama": - if "If I had to write a haiku for this one" in input_text: - ref_1 = ", it would be:.\\nPeter Rabbit is a rabbit.\\nHe lives in a cozy little house.\\nHe's a very good rabbit.\\" - ref_2 = "Here is a haiku for the image:\n\n" - - elif "Answer:" in input_text: - ref_1 = "2,173. A 1 2 3 4 5 6 Date Income 2005-12-17" - ref_2 = "Answer: 2,173. 1 2 3 4 5 6 Date Income 2005-12-17" - - elif "The key to life is" in input_text: - ref_1 = "to find your passion and pursue it with all your heart. For me, that passion is photography. I love capturing the beauty of the world around me" - ref_2 = "not to be found in the external world," - output = output_text[0][0] - match_rate = max(compute_str_match_rate(ref_1, output), - compute_str_match_rate(ref_2, output)) - logger.info(f"match rate: {match_rate}") - assert match_rate >= 50, \ - f"expected results: '{ref_1}' or '{ref_2}', generated results: '{output}'" - - elif model.model_type == 'llava_onevision': - if args.video_path is None: - assert 'singapore' in output_text[0][0].lower() - else: - assert 'the video is funny because the child\'s actions are' in output_text[ - 0][0].lower() - elif model.model_type == "qwen2_vl": - assert 'dog' in output_text[0][0].lower() - else: - assert output_text[0][0].lower() == 'singapore' - - if args.run_profiling: - msec_per_batch = lambda name: 1000 * profiler.elapsed_time_in_sec( - name) / args.profiling_iterations - logger.info('Latencies per batch (msec)') - logger.info('e2e generation: %.1f' % (msec_per_batch('Generate'))) - logger.info(' ' * 2 + 'Preprocessing: %.1f' % - (msec_per_batch('Preprocess'))) - logger.info(' ' * 4 + 'Vision encoder: %.1f' % - (msec_per_batch('Vision encoder'))) - if profiler.elapsed_time_in_sec('Feature transform') is not None: - logger.info(' ' * 4 + 'Feature transform: %.1f' % - (msec_per_batch('Feature transform'))) - logger.info(' ' * 2 + 'LLM generate: %.1f' % (msec_per_batch('LLM'))) - logger.info(' ' * 2 + 'Tokenizer decode: %.1f' % - (msec_per_batch('Tokenizer decode'))) - - logger.info("---------------------------------------------------------") - - -if __name__ == '__main__': - os.environ["TOKENIZERS_PARALLELISM"] = "false" - parser = argparse.ArgumentParser() - parser = add_common_args(parser) - args = parser.parse_args() - logger.set_level(args.log_level) - - model = MultimodalModelRunner(args) - visual_data = model.load_test_data(args.image_path, args.video_path) - audio_data = model.load_test_audio(args.audio_path) - - if args.run_profiling: - num_warmup_iters = 3 # Multiple iterations to load both vision and LLM engines into memory - for _ in range(num_warmup_iters): - input_text, output_text = model.run(args.input_text, visual_data, - audio_data, args.max_new_tokens) - profiler.reset() - - num_iters = args.profiling_iterations if args.run_profiling else 1 - - for _ in range(num_iters): - input_text, output_text = model.run(args.input_text, visual_data, - audio_data, args.max_new_tokens) - - runtime_rank = tensorrt_llm.mpi_rank() - if runtime_rank == 0: - print_result(model, input_text, output_text, args) - -# TODO: raise error if VILA mode 1 with C++ runtime diff --git a/examples/models/core/multimodal/utils.py b/examples/models/core/multimodal/utils.py deleted file mode 100644 index 6d13fb600f40..000000000000 --- a/examples/models/core/multimodal/utils.py +++ /dev/null @@ -1,163 +0,0 @@ -def add_common_args(parser): - parser.add_argument('--max_new_tokens', type=int, default=128) - parser.add_argument('--batch_size', type=int, default=1) - parser.add_argument('--log_level', type=str, default='info') - parser.add_argument('--engine_dir', - type=str, - default=None, - help='Directory containing visual and LLM TRT engines') - parser.add_argument('--visual_engine_name', - type=str, - default='model.engine', - help='Name of visual TRT engine') - parser.add_argument('--audio_engine_name', - type=str, - default='model.engine', - help='Name of audio TRT engine') - parser.add_argument('--hf_model_dir', - type=str, - default=None, - help="Directory containing tokenizer") - parser.add_argument('--input_text', - type=str, - nargs='+', - default=None, - help='Text prompt to LLM') - parser.add_argument('--num_beams', - type=int, - help="Use beam search if num_beams >1", - default=1) - parser.add_argument('--top_k', type=int, default=1) - parser.add_argument('--top_p', type=float, default=0.0) - parser.add_argument('--temperature', type=float, default=1.0) - parser.add_argument('--repetition_penalty', type=float, default=1.0) - parser.add_argument('--run_profiling', - action='store_true', - help='Profile runtime over several iterations') - parser.add_argument('--profiling_iterations', - type=int, - help="Number of iterations to run profiling", - default=20) - parser.add_argument('--check_accuracy', - action='store_true', - help='Check correctness of text output') - parser.add_argument( - '--video_path', - type=str, - default=None, - help= - 'Path to your local video file, using \'llava-onevision-accuracy\' to check the Llava-OneVision model accuracy' - ) - parser.add_argument( - '--video_num_frames', - type=int, - help= - "The number of frames sampled from the video in the Llava-OneVision model.", - default=None) - parser.add_argument("--image_path", - type=str, - nargs='+', - default=None, - help='List of input image paths, separated by symbol') - parser.add_argument("--audio_path", - type=str, - default=None, - help='input audio path') - parser.add_argument("--path_sep", - type=str, - default=",", - help='Path separator symbol') - parser.add_argument("--prompt_sep", - type=str, - default=",", - help="Prompt separator symbol") - parser.add_argument('--enable_context_fmha_fp32_acc', - action='store_true', - default=None, - help="Enable FMHA runner FP32 accumulation.") - parser.add_argument( - '--enable_chunked_context', - action='store_true', - help='Enables chunked context (only available with cpp session).', - ) - parser.add_argument( - '--mm_embedding_offloading', - type=lambda s: s.lower() == "true", - default=None, - help= - 'Enable position table offloading. When not specified, defaults to True if using a multimodal model with chunked context.' - ) - parser.add_argument( - '--session', - default='cpp_llm_only', - type=str, - choices=['python', 'cpp_llm_only', 'cpp'], - help= - 'Rumtime used to run the models. \n`cpp_llm_only`: vision engine run in python runtime, but LLM in pybind cpp runtime\n`python`: everything runs in python runtime\n`cpp`: everything runs in C++ runtime' - ) - parser.add_argument( - '--kv_cache_free_gpu_memory_fraction', - default=0.7, - type=float, - help='Specify the free gpu memory fraction.', - ) - parser.add_argument( - '--cross_kv_cache_fraction', - default=0.5, - type=float, - help= - 'Specify the kv cache fraction reserved for cross attention. Only applicable for encoder-decoder models. By default 0.5 for self and 0.5 for cross.', - ) - parser.add_argument( - '--multi_block_mode', - type=lambda s: s.lower() in - ("yes", "true", "t", "1" - ), # custom boolean function to convert input string to boolean - default=True, - help= - "Distribute the work across multiple CUDA thread-blocks on the GPU for masked MHA kernel." - ) - parser.add_argument( - '--lora_task_uids', - type=str, - default=None, - nargs="+", - help="The list of LoRA task uids; use -1 to disable the LoRA module") - parser.add_argument('--debug_mode', - default=False, - action='store_true', - help="Whether or not to turn on the debug mode") - parser.add_argument( - '--trust_remote_code', - action='store_true', - default=False, - help='Allow loading models with custom remote code from HuggingFace Hub. ' - 'Only enable this for models from trusted sources.') - return parser - - -def levenshtein_distance(s1, s2): - if len(s1) < len(s2): - return levenshtein_distance(s2, s1) - - if len(s2) == 0: - return len(s1) - - previous_row = range(len(s2) + 1) - for i, c1 in enumerate(s1): - current_row = [i + 1] - for j, c2 in enumerate(s2): - insertions = previous_row[j + 1] + 1 - deletions = current_row[j] + 1 - substitutions = previous_row[j] + (c1 != c2) - current_row.append(min(insertions, deletions, substitutions)) - previous_row = current_row - - return previous_row[-1] - - -def compute_str_match_rate(s1, s2): - distance = levenshtein_distance(s1, s2) - max_length = max(len(s1), len(s2)) - match_rate = (1 - distance / max_length) * 100 - return match_rate diff --git a/examples/models/core/nemotron/README_nemotron-3.md b/examples/models/core/nemotron/README_nemotron-3.md deleted file mode 100644 index 0816eb995fd8..000000000000 --- a/examples/models/core/nemotron/README_nemotron-3.md +++ /dev/null @@ -1,214 +0,0 @@ -# Nemotron-3 - -This document demonstrates how to build the Nemotron models using TensorRT LLM and run on a single GPU or multiple GPUs. - -- [Nemotron](#nemotron) - - [Overview](#overview) - - [Support Matrix](#support-matrix) - - [Usage](#usage) - - [Download weights from HuggingFace Transformers](#download-weights-from-huggingface-transformers) - - [Build TensorRT engine(s)](#build-tensorrt-engines) - - [FP8 Quantization](#fp8-quantization) - - [INT4 AWQ Quantization](#int4-awq-quantization) - - [Run Inference](#run-inference) - -## Overview - -The TensorRT LLM Nemotron implementation is based on the GPT model, which can be found in [`tensorrt_llm/models/gpt/model.py`](../../../../tensorrt_llm/models/gpt/model.py). The TensorRT LLM Nemotron example is located in [`examples/models/core/nemotron`](./). - -In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: - -* [`run.py`](../../../run.py) to run the inference on an input text; -* [`summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. - -## Support Matrix - * FP16/BF16 - * FP8 - * INT4 AWQ - * Tensor Parallel - * Pipeline Parallel - * Inflight Batching - * PAGED_KV_CACHE - * STRONGLY TYPED - * checkpoint type: Nemo, Huggingface (HF) - -## Nemo checkpoint - Usage - -### Download weights from HuggingFace Transformers - - -Install the dependencies and setup `git-lfs`. - -```bash -# Install dependencies -pip install -r requirements.txt - -# Setup git-lfs -git lfs install -``` - -Download one or more Nemotron models that you would like to build to TensorRT LLM engines. You can download from the [HuggingFace](https://huggingface.co) hub: - -```bash -# Download nemotron-3-8b-base-4k -git clone https://huggingface.co/nvidia/nemotron-3-8b-base-4k - -# Download nemotron-3-8b-chat-4k-sft -git clone https://huggingface.co/nvidia/nemotron-3-8b-chat-4k-sft - -# Download nemotron-3-8b-chat-4k-rlhf -git clone https://huggingface.co/nvidia/nemotron-3-8b-chat-4k-rlhf -``` - -### Build TensorRT engine(s) -The [`examples/quantization/quantize.py`](../../../quantization/quantize.py) script can quantize the Nemotron models and export to TensorRT LLM checkpoints. You may optionally skip the quantization step by specifying `--qformat full_prec` and thus export float16 or bfloat16 TensorRT LLM checkpoints. - -The `trtllm-build` command builds TensorRT LLM engines from TensorRT LLM checkpoints. The number of engine files is same to the number of GPUs used to run inference. Normally, `trtllm-build` uses one GPU by default, but if you have already more GPUs available at build time, you may enable parallel builds to make the engine building process faster by adding the `--workers` argument. - -Here are some examples: - -```bash -# single gpu, dtype bfloat16 -python3 ../../../quantization/quantize.py \ - --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ - --dtype bfloat16 \ - --batch_size 64 \ - --qformat full_prec \ - --output_dir nemotron-3-8b/trt_ckpt/bf16/1-gpu - -trtllm-build --checkpoint_dir nemotron-3-8b/trt_ckpt/bf16/1-gpu \ - --gpt_attention_plugin bfloat16 \ - --gemm_plugin bfloat16 \ - --output_dir nemotron-3-8b/trt_engines/bf16/1-gpu -``` - -```bash -# 2-way tensor parallelism -python3 ../../../quantization/quantize.py \ - --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ - --dtype bfloat16 \ - --batch_size 64 \ - --qformat full_prec \ - --tp_size 2 \ - --output_dir nemotron-3-8b/trt_ckpt/bf16/tp2 - -trtllm-build --checkpoint_dir nemotron-3-8b/trt_ckpt/bf16/tp2 \ - --gpt_attention_plugin bfloat16 \ - --gemm_plugin bfloat16 \ - --workers 2 \ - --output_dir nemotron-3-8b/trt_engines/bf16/tp2 -``` - -```bash -# 2-way tensor parallelism for both calibration and inference -mpirun -np 2 \ - python3 ../../../quantization/quantize.py \ - --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ - --dtype bfloat16 \ - --batch_size 64 \ - --qformat full_prec \ - --calib_tp_size 2 \ - --tp_size 2 \ - --output_dir nemotron-3-8b/trt_ckpt/bf16/tp2 - -trtllm-build --checkpoint_dir nemotron-3-8b/trt_ckpt/bf16/tp2 \ - --gpt_attention_plugin bfloat16 \ - --gemm_plugin bfloat16 \ - --workers 2 \ - --output_dir nemotron-3-8b/trt_engines/bf16/tp2 -``` - -#### FP8 Quantization - -Quantize the Nemotron models to FP8 by specifying `--qformat fp8` to `quantize.py`. - -```bash -# single gpu, fp8 quantization -python3 ../../../quantization/quantize.py \ - --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ - --dtype bfloat16 \ - --batch_size 64 \ - --qformat fp8 \ - --output_dir nemotron-3-8b/trt_ckpt/fp8/1-gpu - -trtllm-build --checkpoint_dir nemotron-3-8b/trt_ckpt/fp8/1-gpu \ - --gpt_attention_plugin bfloat16 \ - --output_dir nemotron-3-8b/trt_engines/fp8/1-gpu -``` - -#### INT4 AWQ Quantization - -Quantize the Nemotron models using INT4 AWQ by specifying `--qformat int4_awq` to `quantize.py`. - -```bash -# single gpu, int4 awq quantization -python3 ../../../quantization/quantize.py \ - --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ - --dtype bfloat16 \ - --batch_size 64 \ - --qformat int4_awq \ - --output_dir nemotron-3-8b/trt_ckpt/int4_awq/1-gpu - -trtllm-build --checkpoint_dir nemotron-3-8b/trt_ckpt/int4_awq/1-gpu \ - --gpt_attention_plugin bfloat16 \ - --output_dir nemotron-3-8b/trt_engines/int4_awq/1-gpu -``` - -### Run Inference - -The `summarize.py` script can run the built engines to summarize the articles from the -[cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. - -```bash -# single gpu -python3 ../../../summarize.py --test_trt_llm \ - --no_add_special_tokens \ - --engine_dir nemotron-3-8b/trt_engines/bf16/1-gpu \ - --vocab_file nemotron-3-8b/trt_ckpt/bf16/1-gpu/tokenizer.model - -# multiple gpus -mpirun -np 2 \ - python3 ../../../summarize.py --test_trt_llm \ - --no_add_special_tokens \ - --engine_dir nemotron-3-8b/trt_engines/bf16/tp2 \ - --vocab_file nemotron-3-8b/trt_ckpt/bf16/tp2/tokenizer.model -``` - -If the engines are run successfully, you will see output like: -``` -...... -[04/23/2024-09:55:54] [TRT-LLM] [I] TensorRT LLM (total latency: 14.926485538482666 sec) -[04/23/2024-09:55:54] [TRT-LLM] [I] TensorRT LLM (total output tokens: 2000) -[04/23/2024-09:55:54] [TRT-LLM] [I] TensorRT LLM (tokens per second: 133.99001357980129) -[04/23/2024-09:55:54] [TRT-LLM] [I] TensorRT LLM beam 0 result -[04/23/2024-09:55:54] [TRT-LLM] [I] rouge1 : 19.48743720965424 -[04/23/2024-09:55:54] [TRT-LLM] [I] rouge2 : 6.272381295466071 -[04/23/2024-09:55:54] [TRT-LLM] [I] rougeL : 15.011005943152721 -[04/23/2024-09:55:54] [TRT-LLM] [I] rougeLsum : 17.76145734406502 -``` - -## HF checkpoint - Usage -Support for Nemotron models was added with transformers 4.44.0 release. - -```bash -# install transformers library -pip install transformers>=4.44.0 -# Download hf minitron model -git clone https://huggingface.co/nvidia/Minitron-4B-Base - -# Convert to TensorRT LLM checkpoint -python3 ../gpt/convert_checkpoint.py --model_dir Minitron-4B-Base \ - --dtype bfloat16 \ - --output_dir minitron/trt_ckpt/bf16/1-gpu - -# Build TensorRT LLM engines -trtllm-build --checkpoint_dir minitron/trt_ckpt/bf16/1-gpu \ - --gemm_plugin auto \ - --output_dir minitron/trt_engines/bf16/1-gpu - -# Run inference -python3 ../../../run.py --engine_dir minitron/trt_engines/bf16/1-gpu \ - --tokenizer_dir Minitron-4B-Base \ - --input_text "def print_hello_world():" \ - --max_output_len 20 -``` diff --git a/examples/models/core/nemotron/requirements.txt b/examples/models/core/nemotron/requirements.txt deleted file mode 100644 index f8d1f8e1abf4..000000000000 --- a/examples/models/core/nemotron/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.0.dev0 -nemo-toolkit[all]==2.0.0rc1 -megatron-core @ git+https://github.com/NVIDIA/Megatron-LM@core_r0.8.0 -datasets==3.1.0 -evaluate -rouge_score diff --git a/examples/models/core/qwen2audio/README.md b/examples/models/core/qwen2audio/README.md deleted file mode 100644 index 92115345077e..000000000000 --- a/examples/models/core/qwen2audio/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# Guide to Qwen2-Audio deployment pipeline - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -1. Download the Qwen2-Audio model. - ```bash - git lfs install - export MODEL_PATH="tmp/Qwen2-Audio-7B-Instruct" - git clone https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct $MODEL_PATH - ``` -2. Generate the TensorRT engine of audio encoder. - ```bash - export ENGINE_DIR="./trt_engines/qwen2audio/fp16" - python3 ../multimodal/build_multimodal_engine.py --model_type qwen2_audio --model_path $MODEL_PATH --max_batch_size 32 --output_dir ${ENGINE_DIR}/audio - ``` - - The TensorRT engine will be generated under `${ENGINE_DIR}/audio`. - -3. Build Qwen2 LLM TensorRT engine. -- Convert checkpoint - 1. Install packages - ```bash - pip install -r requirements.txt - ``` - 2. Convert - 2.1 FP16 checkpoint - ```bash - python3 ../qwen/convert_checkpoint.py --model_dir=$MODEL_PATH \ - --dtype=float16 \ - --output_dir=./tllm_checkpoint_1gpu_fp16 - ``` - 2.2 (Optional) INT8 Weight Only checkpoint - ```bash - python3 ../qwen/convert_checkpoint.py --model_dir=$MODEL_PATH \ - --dtype=float16 \ - --use_weight_only \ - --weight_only_precision=int8 \ - --output_dir=./tllm_checkpoint_1gpu_fp16_wo8 - ``` - -- Build TensorRT LLM engine - - NOTE: `max_prompt_embedding_table_size = query_token_num * max_batch_size`, therefore, if you change `max_batch_size`, `--max_prompt_embedding_table_size` must be reset accordingly. - ```bash - trtllm-build --checkpoint_dir=./tllm_checkpoint_1gpu_fp16 \ - --gemm_plugin=float16 --gpt_attention_plugin=float16 \ - --max_batch_size=1 --max_prompt_embedding_table_size=4096 \ - --output_dir=${ENGINE_DIR}/llm - ``` - The built Qwen engines are located in `${ENGINE_DIR}/llm`. - - You can replace the `--checkpoint_dir` with INT8 Weight Only checkpoint to build INT8 Weight Only engine as well. - For more information about Qwen, refer to the README.md in [`example/models/core/qwen`](../qwen). - -4. Assemble everything into the Qwen2-Audio pipeline. - - 4.1 Run with FP16 LLM engine - ```bash - python3 run.py \ - --tokenizer_dir=$MODEL_PATH \ - --engine_dir=${ENGINE_DIR}/llm \ - --audio_engine_path=${ENGINE_DIR}/audio/model.engine \ - --audio_url='./audio/glass-breaking-151256.mp3' - ``` - 4.2 (Optional) For multiple rounds of dialogue, you can run: - ```bash - python3 run_chat.py \ - --tokenizer_dir=$MODEL_PATH \ - --engine_dir=${ENGINE_DIR}/llm \ - --audio_engine_path=${ENGINE_DIR}/audio/model.engine \ - --max_new_tokens=256 - ``` - - Note: - - This example supports reusing the KV Cache for audio segments by assigning unique audio IDs. - - To further optimize performance, users can also cache the audio features (encoder output) to bypass the audio encoder if the original audio data remains unchanged. diff --git a/examples/models/core/qwen2audio/audio/glass-breaking-151256.mp3 b/examples/models/core/qwen2audio/audio/glass-breaking-151256.mp3 deleted file mode 100644 index 150e1080f2e5..000000000000 Binary files a/examples/models/core/qwen2audio/audio/glass-breaking-151256.mp3 and /dev/null differ diff --git a/examples/models/core/qwen2audio/requirements.txt b/examples/models/core/qwen2audio/requirements.txt deleted file mode 100644 index 1d6d844e4b64..000000000000 --- a/examples/models/core/qwen2audio/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.dev0 -datasets==3.1.0 -evaluate -rouge_score -transformers>=4.45.0 -transformers-stream-generator -sentencepiece>=0.1.99 -tiktoken -einops diff --git a/examples/models/core/qwen2audio/run.py b/examples/models/core/qwen2audio/run.py deleted file mode 100644 index a0b0a68fb1c9..000000000000 --- a/examples/models/core/qwen2audio/run.py +++ /dev/null @@ -1,640 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import os -from io import BytesIO -from urllib.request import urlopen - -import librosa -import tensorrt as trt -import torch -from transformers import AutoConfig, AutoProcessor, AutoTokenizer -from utils import add_common_args - -import tensorrt_llm -import tensorrt_llm.profiler as profiler -from tensorrt_llm import logger -from tensorrt_llm.llmapi.kv_cache_type import KVCacheType -from tensorrt_llm.quantization import QuantMode -from tensorrt_llm.runtime import (PYTHON_BINDINGS, ModelConfig, ModelRunner, - SamplingConfig, Session, TensorInfo) - -if PYTHON_BINDINGS: - from tensorrt_llm.runtime import ModelRunnerCpp - - -def get_engine_name(rank): - return "rank{}.engine".format(rank) - - -def trt_dtype_to_torch(dtype): - if dtype == trt.float16: - return torch.float16 - elif dtype == trt.float32: - return torch.float32 - elif dtype == trt.int32: - return torch.int32 - else: - raise TypeError("%s is not supported" % dtype) - - -class QWenInfer(object): - - def __init__(self, - audio_engine_path, - tokenizer_dir, - engine_dir, - log_level, - output_csv, - output_npy, - num_beams, - gpu_id=0): - self.audio_engine_path = audio_engine_path - self.tokenizer_dir = tokenizer_dir - self.engine_dir = engine_dir - self.log_level = log_level - self.max_seq_len = 0 - self.runner = None - self.hf_audio_tower = None - self.tokenizer = None - self.config = None - self.sampling_config = None - self.output_csv = output_csv - self.output_npy = output_npy - self.num_beams = num_beams - self.model_config = None - self.gpu_device = torch.device("cuda", gpu_id) - - def get_model(self): - # --load the tokenizer and engines # - tokenizer = AutoTokenizer.from_pretrained( - self.tokenizer_dir, - legacy=False, - trust_remote_code=True, - ) - processor = AutoProcessor.from_pretrained(self.tokenizer_dir, - trust_remote_code=True) - config_path = os.path.join(self.engine_dir, "config.json") - with open(config_path, "r") as f: - config = json.load(f) - self.max_seq_len = config["build_config"]["max_seq_len"] - assert self.max_seq_len > 0, "max_seq_len must be positive" - - gen_config_path = os.path.join(self.tokenizer_dir, - "generation_config.json") - with open(gen_config_path, "r") as f: - gen_config = json.load(f) - top_k = gen_config["top_k"] - top_p = gen_config["top_p"] - eos_token_id = tokenizer.pad_token_id - pad_token_id = tokenizer.pad_token_id - - use_gpt_attention_plugin = config["build_config"]["plugin_config"][ - "gpt_attention_plugin"] - remove_input_padding = config["build_config"]["plugin_config"][ - "remove_input_padding"] - dtype = config["pretrained_config"]["dtype"] - tp_size = config["pretrained_config"]["mapping"]["tp_size"] - pp_size = config["pretrained_config"]["mapping"]["pp_size"] - world_size = tp_size * pp_size - assert ( - world_size == tensorrt_llm.mpi_world_size() - ), f"Engine world size ({world_size}) != Runtime world size ({tensorrt_llm.mpi_world_size()})" - num_heads = config["pretrained_config"][ - "num_attention_heads"] // world_size - max_batch_size = config["build_config"]["max_batch_size"] - hidden_size = config["pretrained_config"]["hidden_size"] // world_size - vocab_size = config["pretrained_config"]["vocab_size"] - num_layers = config["pretrained_config"]["num_hidden_layers"] - num_kv_heads = config["pretrained_config"].get("num_key_value_heads", - num_heads) - if "kv_cache_type" in config["build_config"]: - kv_cache_type = KVCacheType(config["build_config"]["kv_cache_type"]) - else: - kv_cache_type = KVCacheType.CONTINUOUS - - tokens_per_block = config["build_config"]["plugin_config"][ - "tokens_per_block"] - max_prompt_embedding_table_size = config["build_config"].get( - "max_prompt_embedding_table_size", 0) - quant_mode = QuantMode.from_quant_algo( - config["pretrained_config"]["quantization"]["quant_algo"], - config["pretrained_config"]["quantization"]["kv_cache_quant_algo"], - ) - if config["pretrained_config"].get("multi_query_mode", False): - tensorrt_llm.logger.warning( - "`multi_query_mode` config is deprecated. Please rebuild the engine." - ) - num_kv_heads = 1 - - runtime_rank = tensorrt_llm.mpi_rank() - runtime_mapping = tensorrt_llm.Mapping(world_size=world_size, - rank=runtime_rank, - tp_size=tp_size, - pp_size=pp_size) - - model_config = ModelConfig( - max_batch_size=max_batch_size, - num_heads=num_heads, - num_kv_heads=num_kv_heads, - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - gpt_attention_plugin=use_gpt_attention_plugin, - kv_cache_type=kv_cache_type, - tokens_per_block=tokens_per_block, - remove_input_padding=remove_input_padding, - dtype=dtype, - quant_mode=quant_mode, - max_prompt_embedding_table_size=max_prompt_embedding_table_size, - max_beam_width=self.num_beams, - ) - sampling_config = SamplingConfig( - end_id=eos_token_id, - pad_id=pad_token_id, - num_beams=self.num_beams, - top_k=top_k, - top_p=top_p, - temperature=1.0, - ) - - engine_name = get_engine_name(runtime_rank) - serialize_path = os.path.join(self.engine_dir, engine_name) - print(f"Loading engine from {serialize_path}") - return ( - model_config, - sampling_config, - runtime_mapping, - runtime_rank, - serialize_path, - tokenizer, - processor, - eos_token_id, - pad_token_id, - ) - - def qwen_model_init(self, args): - logger.info(f"Loading audio engine from {self.audio_engine_path}") - with open(self.audio_engine_path, "rb") as f: - engine_buffer = f.read() - logger.info(f"Creating session from engine {self.audio_engine_path}") - self.session_audio = Session.from_serialized_engine(engine_buffer) - - self.config, _ = AutoConfig.from_pretrained( - self.tokenizer_dir, - return_unused_kwargs=True, - trust_remote_code=True, - ) - - ( - model_config, - sampling_config, - runtime_mapping, - runtime_rank, - serialize_path, - tokenizer, - processor, - eos_token_id, - pad_token_id, - ) = self.get_model() - runner_cls = ModelRunner if args.use_py_session else ModelRunnerCpp - runner_kwargs = dict( - engine_dir=args.engine_dir, - lora_dir=args.lora_dir, - rank=runtime_rank, - debug_mode=args.debug_mode, - lora_ckpt_source=args.lora_ckpt_source, - gpu_weights_percent=args.gpu_weights_percent, - max_output_len=args.max_new_tokens, - ) - if not args.use_py_session: - runner_kwargs.update( - is_enc_dec=False, - max_batch_size=model_config.max_batch_size, - max_input_len=self.max_seq_len - args.max_new_tokens, - max_beam_width=model_config.max_beam_width, - max_attention_window_size=args.max_attention_window_size, - sink_token_length=args.sink_token_length, - max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, - kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, - kv_cache_free_gpu_memory_fraction=args. - kv_cache_free_gpu_memory_fraction, - cross_kv_cache_fraction=None, - enable_chunked_context=args.enable_chunked_context, - multi_block_mode=args.multi_block_mode, - cuda_graph_mode=args.cuda_graph_mode, - device_ids=[args.gpu_id]) - runner_kwargs.update( - enable_context_fmha_fp32_acc=args.enable_context_fmha_fp32_acc) - self.runner = runner_cls.from_dir(**runner_kwargs) - self.tokenizer = tokenizer - self.processor = processor - self.sampling_config = sampling_config - self.model_config = model_config - - def ptuning_setup(self, prompt_table, dtype, hidden_size, tasks, input_ids): - if prompt_table is not None: - task_vocab_size = torch.tensor([prompt_table.shape[0]], - dtype=torch.int32, - device=self.gpu_device) - prompt_table = prompt_table.to( - dtype=tensorrt_llm._utils.str_dtype_to_torch(dtype), - device=self.gpu_device) - else: - prompt_table = torch.empty([1, hidden_size], device=self.gpu_device) - task_vocab_size = torch.zeros([1], device=self.gpu_device) - - if tasks is not None: - tasks = torch.tensor([int(t) for t in tasks.split(",")], - dtype=torch.int32, - device=self.gpu_device) - assert (tasks.shape[0] == input_ids.shape[0] - ), "Number of supplied tasks must match input batch size" - else: - tasks = torch.zeros([input_ids.size(0)], - dtype=torch.int32, - device=self.gpu_device) - - return [prompt_table, tasks, task_vocab_size] - - def build_user_input(self, audio=None, text=None): - assert isinstance(audio, str) or isinstance( - text, str), "audio or text must be provided as user input" - content = [] - if audio: - content.append({'type': 'audio', 'audio_url': audio}) - if text: - content.append({'type': 'text', 'text': text}) - user_input = {'role': 'user', 'content': content} - return user_input - - def get_raw_audios(self, audio_url): - audios = [] - for url in audio_url: - if os.path.isfile(url): - audio_data, _ = librosa.load( - url, sr=self.processor.feature_extractor.sampling_rate) - else: - audio_data, _ = librosa.load( - BytesIO(urlopen(url).read()), - sr=self.processor.feature_extractor.sampling_rate) - audios.append(audio_data) - return audios - - def audio_tower(self, audios, mask, stream, run_time=1): - audios = audios.to(self.gpu_device) - mask = mask.to(self.gpu_device) - audio_inputs = {"input": audios.float(), "mask": mask} - audio_output_info = self.session_audio.infer_shapes([ - TensorInfo("input", trt.DataType.FLOAT, audios.shape), - TensorInfo("mask", trt.DataType.HALF, mask.shape) - ]) - audio_outputs = { - t.name: - torch.empty(tuple(t.shape), - dtype=trt_dtype_to_torch(t.dtype), - device=self.gpu_device) - for t in audio_output_info - } - profiler.start("Audio") - for _ in range(run_time): - ok = self.session_audio.run(audio_inputs, audio_outputs, - stream.cuda_stream) - stream.synchronize() - audio_time = profiler.stop("Audio") / run_time - logger.info(f"TensorRT LLM Audio latency: {audio_time:3f} sec ") - - assert ok, "Runtime execution failed for audio session" - - audio_features = audio_outputs["output"] - - return audio_features - - def generate_for_qwen_audio( - self, - input_tokens, - args, - prompt_table=None, - extra_ids=None, - run_time=1, - ): - input_ids = torch.as_tensor(input_tokens, - device=self.gpu_device, - dtype=torch.int32) - input_lengths = torch.tensor([input_ids.size(1)], - device=self.gpu_device, - dtype=torch.int32) - max_input_length = torch.max(input_lengths).item() - max_new_tokens = min(args.max_new_tokens, - self.max_seq_len - max_input_length) - - prompt_table = prompt_table.unsqueeze(0) - profiler.start("QWen") - for _ in range(run_time): - outputs = self.runner.generate( - batch_input_ids=input_ids, - max_new_tokens=max_new_tokens, - max_attention_window_size=args.max_attention_window_size, - sink_token_length=args.sink_token_length, - end_id=self.sampling_config.end_id, - pad_id=self.sampling_config.pad_id, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - num_return_sequences=args.num_return_sequences, - length_penalty=args.length_penalty, - early_stopping=args.early_stopping, - repetition_penalty=args.repetition_penalty, - presence_penalty=args.presence_penalty, - frequency_penalty=args.frequency_penalty, - stop_words_list=[[[151643], [151645]]], - bad_words_list=self.sampling_config.bad_words_list, - random_seed=args.random_seed, - lora_uids=args.lora_task_uids, - prompt_table=prompt_table, - prompt_tasks="0", - output_sequence_lengths=True, - no_repeat_ngram_size=args.no_repeat_ngram_size, - return_dict=True, - return_all_generated_tokens=False, - input_token_extra_ids=extra_ids) - output_ids = outputs['output_ids'] - torch.cuda.synchronize() - Qwen_time = profiler.stop("QWen") / run_time - - return output_ids, Qwen_time - - def get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor): - """ - Computes the output length of the convolutional layers and the output length of the audio encoder - """ - input_lengths = (input_lengths - 1) // 2 + 1 - output_lengths = (input_lengths - 2) // 2 + 1 - return input_lengths, output_lengths - - def qwen_infer(self, - input_text, - audios, - audio_ids, - args, - stream, - history=None, - past_audio_features=None, - run_time=1): - assert input_text, "input_text must be provided" - assert torch.cuda.is_available(), "no gpu available" - # preprocess on CPU maybe faster - device = torch.device("cpu") - if isinstance(history, list): - history.append(input_text) - full_text = self.processor.apply_chat_template( - history, add_generation_prompt=True, tokenize=False) - else: - full_text = input_text - inputs = self.processor( - text=full_text, - audios=audios, - return_tensors="pt", - padding=True, - sampling_rate=self.processor.feature_extractor.sampling_rate) - inputs = inputs.to(device) - input_ids = inputs.input_ids - - if hasattr(inputs, - 'input_features') and inputs.input_features is not None: - # audio tower - batch_size, _, max_mel_seq_len = inputs.input_features.shape - feature_attention_mask = inputs.feature_attention_mask - - audio_feat_lengths, num_audio_tokens = self.get_feat_extract_output_lengths( - feature_attention_mask.sum(-1)) - - max_seq_len = (max_mel_seq_len - 2) // 2 + 1 - # Create a sequence tensor of shape (batch_size, max_seq_len) - seq_range = (torch.arange(0, - max_seq_len, - dtype=audio_feat_lengths.dtype, - device=device).unsqueeze(0).expand( - batch_size, max_seq_len)) - lengths_expand = audio_feat_lengths.unsqueeze(1).expand( - batch_size, max_seq_len) - # Create mask - padding_mask = seq_range >= lengths_expand - - audio_attention_mask_ = padding_mask.view( - batch_size, 1, 1, max_seq_len).expand(batch_size, 1, - max_seq_len, max_seq_len) - audio_attention_mask = audio_attention_mask_.to(dtype=torch.float16, - device=device) - audio_attention_mask[audio_attention_mask_] = float("-inf") - - audio_features = self.audio_tower(inputs.input_features, - audio_attention_mask, stream, - run_time) - - # merge audio features and input ids - num_audios, max_audio_tokens, embed_dim = audio_features.shape - audio_features_mask = torch.arange( - max_audio_tokens, device=device).expand( - num_audios, - max_audio_tokens) < num_audio_tokens.unsqueeze(1) - masked_audio_features = audio_features[audio_features_mask].view( - -1, embed_dim) - batch_size, _ = input_ids.shape - - # 1. Create a mask to know where special audio tokens are - special_audio_token_mask = input_ids == self.config.audio_token_index - special_audio_token_num = special_audio_token_mask.sum().item() - if past_audio_features is not None: - assert isinstance(past_audio_features, - list), f'past_audio_features should be a list' - assert ( - special_audio_token_num == len(past_audio_features) + - num_audios - ), f'special_audio_token_num {special_audio_token_num} should be equal to len(past_audio_features) + num_audios ({len(past_audio_features)} + {num_audios})' - # split to get current audio features - cur_audio_features = torch.split(masked_audio_features, - num_audio_tokens.tolist()) - if len(past_audio_features) > 0: - # concat past and current audio features - masked_audio_features = torch.cat( - (torch.cat(past_audio_features).to( - masked_audio_features.device), - masked_audio_features)) - # get past audio tokens number - past_num_audio_tokens = torch.tensor([ - past_feat.size(0) for past_feat in past_audio_features - ]) - # concat past and current audio tokens number - num_audio_tokens = torch.cat( - (past_num_audio_tokens.to(num_audio_tokens.device), - num_audio_tokens)) - # extend past audio features, cache them in CPU memory - past_audio_features.extend( - [cur_feat.cpu() for cur_feat in cur_audio_features]) - - batch_indices, non_audio_indices = torch.where( - input_ids != self.config.audio_token_index) - - # 2. Fill the final input ids based on the mask. - batch_indices, audio_indices = torch.where( - input_ids == self.config.audio_token_index) - - vocab_size = self.config.vocab_size - fake_prompt_id = torch.arange(vocab_size, - vocab_size + num_audio_tokens.sum(), - device=device) - - input_ids[batch_indices, audio_indices] = fake_prompt_id - input_lengths = torch.tensor(input_ids.size(1), - dtype=torch.int32, - device=self.gpu_device) - dtype = self.model_config.dtype - prompt_table, tasks, task_vocab_size = self.ptuning_setup( - masked_audio_features, dtype, embed_dim, None, input_ids) - - # build extra ids - assert isinstance(audio_ids, list), "audio_ids must be a list" - assert ( - len(audio_ids) == num_audio_tokens.size(0) - ), f"audio_ids length doesn't match with num_audio_tokens ({len(audio_ids)} != {num_audio_tokens.size(0)})" - for i in audio_ids: - assert isinstance( - i, int - ) and i > 0, "audio_id should be an integer greater than 0" - extra_ids = torch.zeros_like(input_ids, - dtype=torch.int64, - device=device) - seq_extra_ids = torch.cat([ - torch.full((n, ), audio_ids[i], dtype=torch.int64) - for i, n in enumerate(num_audio_tokens) - ]).to(device) - extra_ids[batch_indices, audio_indices] = seq_extra_ids - extra_ids = extra_ids.tolist() - else: - input_ids = input_ids.to(dtype=torch.int32, device=self.gpu_device) - input_lengths = torch.tensor(input_ids.size(1), - dtype=torch.int32, - device=self.gpu_device) - dtype = self.model_config.dtype - prompt_table, tasks, task_vocab_size = self.ptuning_setup( - None, dtype, self.model_config.hidden_size, None, input_ids) - extra_ids = torch.zeros_like(input_ids, dtype=torch.int64).tolist() - - # print(f"extra_ids: {extra_ids}") - output_ids, Qwen_time = self.generate_for_qwen_audio( - input_ids, args, prompt_table, extra_ids, run_time) - - runtime_rank = tensorrt_llm.mpi_rank() - input_lengths = torch.tensor([input_ids.size(1)], - device=self.gpu_device, - dtype=torch.int32) - effective_output_token = 0 - if runtime_rank == 0: - if self.output_csv is None and self.output_npy is None: - for b in range(input_lengths.size(0)): - inputs = input_ids[b] - if self.num_beams <= 1: - outputs = output_ids[b][0, len(inputs):].tolist() - try: - effective_output_token = (effective_output_token + - outputs.index(151643)) - except: - effective_output_token = 1 - output_text = self.tokenizer.decode( - outputs, skip_special_tokens=True) - print(f'Output: "{output_text}"') - else: - for beam in range(self.num_beams): - outputs = output_ids[b][beam, len(inputs):].tolist() - output_text = self.tokenizer.decode( - outputs, skip_special_tokens=True) - print(f'Output(beam: {beam}): "{output_text}"') - logger.info(f"Input length={input_lengths[b]}") - logger.info(f"Output length={output_ids.shape}") - logger.info(f"TensorRT LLM QWen time: {Qwen_time:3f} sec ") - if isinstance(history, list): - history.append({'role': 'assistant', 'content': output_text}) - return output_text, past_audio_features - - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--max_new_tokens", type=int, default=10) - parser.add_argument( - "--audio_engine_path", - type=str, - default="plan/audio_encoder/audio_encoder_fp16.plan", - ) - parser.add_argument( - "--input_text", - type=str, - default= - "<|audio_bos|><|AUDIO|><|audio_eos|>Generate the caption in English:") - parser.add_argument( - "--audio_url", - nargs="+", - type=str, - default=["./audio/glass-breaking-151256.mp3"], - ) - parser.add_argument( - "--input_tokens", - dest="input_file", - type=str, - help= - "CSV or Numpy file containing tokenized input. Alternative to text input.", - default=None, - ) - parser.add_argument( - "--output_csv", - type=str, - help="CSV file where the tokenized output is stored.", - default=None, - ) - parser.add_argument( - "--output_npy", - type=str, - help="Numpy file where the tokenized output is stored.", - default=None, - ) - parser.add_argument( - "--gpu_id", - type=int, - help= - "Specify GPU device index for running. Should be the index seen by torch, not original index", - default=0, - ) - parser = add_common_args(parser) - - return parser.parse_args() - - -if __name__ == "__main__": - args = parse_arguments() - tensorrt_llm.logger.set_level(args.log_level) - - # use cudaSetDevice before loading audio engine - torch.cuda.set_device(args.gpu_id) - qinfer = QWenInfer(args.audio_engine_path, args.tokenizer_dir, - args.engine_dir, args.log_level, args.output_csv, - args.output_npy, args.num_beams, args.gpu_id) - qinfer.qwen_model_init(args) - - audios = qinfer.get_raw_audios(args.audio_url) - gpu_device = torch.device("cuda", args.gpu_id) - stream = torch.cuda.current_stream(device=gpu_device) - qinfer.qwen_infer(args.input_text, audios, [1], args, stream, None, None, 1) diff --git a/examples/models/core/qwen2audio/run_chat.py b/examples/models/core/qwen2audio/run_chat.py deleted file mode 100644 index 00d58cdc862b..000000000000 --- a/examples/models/core/qwen2audio/run_chat.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# isort: off -import torch -from run import QWenInfer, parse_arguments - -import tensorrt_llm -# isort: on - -if __name__ == '__main__': - args = parse_arguments() - stream = torch.cuda.current_stream() - tensorrt_llm.logger.set_level(args.log_level) - qinfer = QWenInfer( - args.audio_engine_path, - args.tokenizer_dir, - args.engine_dir, - args.log_level, - args.output_csv, - args.output_npy, - args.num_beams, - ) - qinfer.qwen_model_init(args) - - run_i = 0 - history = [] - audios = None - global_audio_id = 1 - audio_ids = [] - - while True: - input_text = None - try: - input_text = input( - "Text (type 'q' to quit, or 'audio_url:[url]' to input audio): " - ) - except: - continue - - if input_text == "clear history": - history = [] - audios = None - continue - - if input_text.lower() == 'q': - break - print('\n') - - if input_text.startswith('audio_url:'): - audio_url = input_text[len('audio_url:'):].strip() - if isinstance(audios, list): - audios.extend(qinfer.get_raw_audios([audio_url])) - else: - audios = qinfer.get_raw_audios([audio_url]) - user_input = qinfer.build_user_input(audio=audio_url) - audio_ids.append(global_audio_id) - global_audio_id += 1 - else: - user_input = qinfer.build_user_input(text=input_text) - - qinfer.qwen_infer( - user_input, - audios, - audio_ids, - args, - stream, - history, - ) diff --git a/examples/models/core/qwen2audio/utils.py b/examples/models/core/qwen2audio/utils.py deleted file mode 100644 index 3252beebbf7d..000000000000 --- a/examples/models/core/qwen2audio/utils.py +++ /dev/null @@ -1,130 +0,0 @@ -from argparse import BooleanOptionalAction - - -def add_common_args(parser): - # sampling arguments - parser.add_argument('--num_beams', - type=int, - help="Use beam search if num_beams > 1", - default=1) - parser.add_argument('--num_return_sequences', - type=int, - help="Number of sequences to generate for each input.", - default=None) - parser.add_argument('--temperature', type=float, default=1.0) - parser.add_argument('--top_k', type=int, default=1) - parser.add_argument('--top_p', type=float, default=0.0) - parser.add_argument('--length_penalty', type=float, default=1.0) - parser.add_argument('--repetition_penalty', type=float, default=1.0) - parser.add_argument('--presence_penalty', type=float, default=0.0) - parser.add_argument('--frequency_penalty', type=float, default=0.0) - parser.add_argument('--random_seed', type=int, default=0) - parser.add_argument('--early_stopping', - type=int, - help='Use early stopping if num_beams > 1, ' - '1 for early-stopping, 0 for non-early-stopping' - 'other values for stopping by length', - default=1) - parser.add_argument('--no_repeat_ngram_size', type=int, default=None) - - # common runtime arguments - parser.add_argument('--sink_token_length', - type=int, - default=None, - help='The sink token length.') - parser.add_argument( - '--max_attention_window_size', - type=int, - default=None, - nargs="+", - help= - 'The attention window size that controls the sliding window attention kv cache behavior' - ) - parser.add_argument( - '--multi_block_mode', - type=lambda s: s.lower() in - ("yes", "true", "t", "1" - ), # custom boolean function to convert input string to boolean - default=True, - help= - "Distribute the work across multiple CUDA thread-blocks on the GPU for masked MHA kernel." - ) - parser.add_argument('--enable_context_fmha_fp32_acc', - action='store_true', - help="Enable FMHA runner FP32 accumulation.") - parser.add_argument('--cuda_graph_mode', - action='store_true', - help="Enable cuda graphs in the inference.") - parser.add_argument( - '--log_level', - type=str, - choices=['verbose', 'info', 'warning', 'error', 'internal_error'], - default='info') - parser.add_argument('--use_py_session', - default=False, - action='store_true', - help="Whether or not to use Python runtime session") - parser.add_argument('--debug_mode', - default=False, - action='store_true', - help="Whether or not to turn on the debug mode") - parser.add_argument('--lora_dir', - type=str, - default=None, - nargs="+", - help="The directory of LoRA weights") - parser.add_argument('--lora_ckpt_source', - type=str, - default="hf", - choices=["hf", "nemo"], - help="The source of lora checkpoint.") - parser.add_argument( - '--lora_task_uids', - type=str, - default=None, - nargs="+", - help="The list of LoRA task uids; use -1 to disable the LoRA module") - - # model arguments - parser.add_argument('--engine_dir', type=str, default='engine_outputs') - parser.add_argument('--hf_model_dir', '--model_dir', type=str, default=None) - parser.add_argument( - '--tokenizer_dir', - default=None, - help='tokenizer path; defaults to hf_model_dir if left unspecified') - - # memory argument - parser.add_argument( - '--gpu_weights_percent', - default=1, - type=float, - help= - 'Specify the percentage of weights that reside on GPU instead of CPU and streaming load during runtime.', - ) - parser.add_argument( - '--max_tokens_in_paged_kv_cache', - default=None, - type=int, - help= - 'Specify the maximum number of tokens in a kv cache page (only available with cpp session).', - ) - parser.add_argument( - '--kv_cache_enable_block_reuse', - default=True, - action=BooleanOptionalAction, - help= - 'Enables block reuse in kv cache (only available with cpp session).', - ) - parser.add_argument( - '--kv_cache_free_gpu_memory_fraction', - default=0.9, - type=float, - help='Specify the free gpu memory fraction.', - ) - parser.add_argument( - '--enable_chunked_context', - action='store_true', - help='Enables chunked context (only available with cpp session).', - ) - - return parser diff --git a/examples/models/core/qwenvl/README.md b/examples/models/core/qwenvl/README.md deleted file mode 100644 index 27fc5d1dfce0..000000000000 --- a/examples/models/core/qwenvl/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# Guide to Qwen-VL deployment pipeline - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -1. Download the Qwen vision-language model (Qwen-VL). - ```bash - git lfs install - git clone https://huggingface.co/Qwen/Qwen-VL-Chat - ``` -2. Generate the Vision Transformer (ViT) ONNX model and the TensorRT engine. -- If you don't have ONNX file, run: - ```bash - python3 vit_onnx_trt.py --pretrained_model_path ./Qwen-VL-Chat - ``` - The ONNX and TensorRT engine will be generated under `./onnx/visual_encoder` and `./plan/visual_encoder` respectively. - -- If you already have an ONNX file under `./onnx/visual_encoder` and want to build a TensorRT engine with it, run: - ```bash - python3 vit_onnx_trt.py --pretrained_model_path ./Qwen-VL-Chat --only_trt - ``` - This command saves the test image tensor to `image.pt` for later pipeline inference. - -3. Build Qwen TensorRT engine. -- Convert checkpoint - 1. Install packages - ```bash - pip install -r requirements.txt - ``` - 2. Convert - ```bash - python3 ./examples/models/core/qwen/convert_checkpoint.py --model_dir=./Qwen-VL-Chat \ - --output_dir=./tllm_checkpoint_1gpu \ - --dtype float16 - ``` - -- Build TensorRT LLM engine - - NOTE: `max_prompt_embedding_table_size = query_token_num * max_batch_size`, therefore, if you change `max_batch_size`, `--max_prompt_embedding_table_size` must be reset accordingly. - ```bash - trtllm-build --checkpoint_dir=./tllm_checkpoint_1gpu \ - --gemm_plugin=float16 --gpt_attention_plugin=float16 \ - --max_input_len=2048 --max_seq_len=3072 \ - --max_batch_size=8 --max_prompt_embedding_table_size=2048 \ - --remove_input_padding=enable \ - --output_dir=./trt_engines/Qwen-VL-7B-Chat - ``` - The built Qwen engines are located in `./trt_engines/Qwen-VL-7B-Chat`. - For more information about Qwen, refer to the README.md in [`example/qwen`](../qwen). - -4. Assemble everything into the Qwen-VL pipeline. - - 4.1 Run with INT4 GPTQ weight-only quantization engine - ```bash - python3 run.py \ - --tokenizer_dir=./Qwen-VL-Chat \ - --qwen_engine_dir=./trt_engines/Qwen-VL-7B-Chat \ - --vit_engine_path=./plan/visual_encoder/visual_encoder_fp16.plan \ - --images_path='{"image": "./pics/demo.jpeg"}' - ``` - 4.2 (Optional) For multiple rounds of dialogue, you can run: - ```bash - python3 run_chat.py \ - --tokenizer_dir=./Qwen-VL-Chat \ - --qwen_engine_dir=./trt_engines/Qwen-VL-7B-Chat \ - --vit_engine_path=./plan/visual_encoder/visual_encoder_fp16.plan \ - --images_path='{"image": "./pics/demo.jpeg"}' - ``` - 4.3 (Optional) To show the bounding box result in the demo picture, install OpenCV, ZMQ, and request: - ```bash - pip install opencv-python==4.5.5.64 - pip install opencv-python-headless==4.5.5.64 - pip install zmq - pip install request - ``` - -   4.3.1 If the current program is executed on a remote machine, run the following command on a local machine: - - ```bash - python3 show_pic.py --ip=127.0.0.1 --port=8006 - ``` - -   Replace the `ip` and `port` values, where `ip` is your remote machine IP address. - -   Run the following command on the remote machine: - - ```bash - python3 run_chat.py \ - --tokenizer_dir=./Qwen-VL-Chat \ - --qwen_engine_dir=./trt_engines/Qwen-VL-7B-Chat \ - --vit_engine_path=./plan/visual_encoder/visual_encoder_fp16.plan \ - --display \ - --port=8006 - ``` - -   Replace the `port` value. - -   4.3.2 If the current program is executed on the local machine, run the following command: - - ```bash - python3 run_chat.py \ - --tokenizer_dir=./Qwen-VL-Chat \ - --qwen_engine_dir=./trt_engines/Qwen-VL-7B-Chat \ - --vit_engine_path=./plan/visual_encoder/visual_encoder_fp16.plan \ - --display \ - --local_machine - ``` - -   The question "Print the bounding box of the girl" is displayed. You should see the following image: - - ![image](./pics/1.png) diff --git a/examples/models/core/qwenvl/pics/1.png b/examples/models/core/qwenvl/pics/1.png deleted file mode 100644 index 09e872ac90da..000000000000 Binary files a/examples/models/core/qwenvl/pics/1.png and /dev/null differ diff --git a/examples/models/core/qwenvl/pics/demo.jpeg b/examples/models/core/qwenvl/pics/demo.jpeg deleted file mode 100644 index 9fdc04005062..000000000000 Binary files a/examples/models/core/qwenvl/pics/demo.jpeg and /dev/null differ diff --git a/examples/models/core/qwenvl/requirements.txt b/examples/models/core/qwenvl/requirements.txt deleted file mode 100644 index 9257ddb9545a..000000000000 --- a/examples/models/core/qwenvl/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ --c ../../../constraints.txt -tensorrt_llm>=0.0.0.dev0 -datasets==3.1.0 -evaluate -rouge_score -transformers-stream-generator -sentencepiece>=0.1.99 -tiktoken -einops -matplotlib -torchvision diff --git a/examples/models/core/qwenvl/run.py b/examples/models/core/qwenvl/run.py deleted file mode 100644 index c996b20fd643..000000000000 --- a/examples/models/core/qwenvl/run.py +++ /dev/null @@ -1,549 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import os -from typing import List, Tuple - -import tensorrt as trt -import torch -from transformers import AutoConfig, AutoTokenizer -from vit_onnx_trt import Preprocss - -import tensorrt_llm -import tensorrt_llm.profiler as profiler -from tensorrt_llm import logger -from tensorrt_llm._deprecation import emit_engine_arch_deprecation -from tensorrt_llm.llmapi.kv_cache_type import KVCacheType -from tensorrt_llm.quantization import QuantMode -from tensorrt_llm.runtime import (ModelConfig, SamplingConfig, Session, - TensorInfo) - - -def get_engine_name(rank): - return "rank{}.engine".format(rank) - - -def trt_dtype_to_torch(dtype): - if dtype == trt.float16: - return torch.float16 - elif dtype == trt.float32: - return torch.float32 - elif dtype == trt.int32: - return torch.int32 - else: - raise TypeError("%s is not supported" % dtype) - - -class QWenInfer(object): - - def __init__( - self, - tokenizer_dir, - qwen_engine_dir, - log_level, - output_csv, - output_npy, - num_beams, - ): - self.tokenizer_dir = tokenizer_dir - self.qwen_engine_dir = qwen_engine_dir - self.log_level = log_level - self.global_max_input_len = 2048 - self.decoder = None - self.tokenizer = None - self.config = None - self.sampling_config = None - self.output_csv = output_csv - self.output_npy = output_npy - self.num_beams = num_beams - self.model_config = None - - def get_model(self): - # --load the tokenizer and engine # - tokenizer = AutoTokenizer.from_pretrained( - self.tokenizer_dir, - legacy=False, - trust_remote_code=True, - ) - config_path = os.path.join(self.qwen_engine_dir, "config.json") - with open(config_path, "r") as f: - config = json.load(f) - gen_config_path = os.path.join(self.tokenizer_dir, - "generation_config.json") - with open(gen_config_path, "r") as f: - gen_config = json.load(f) - top_k = gen_config["top_k"] - top_p = gen_config["top_p"] - chat_format = gen_config["chat_format"] - if chat_format == "raw": - eos_token_id = gen_config["eos_token_id"] - pad_token_id = gen_config["pad_token_id"] - elif chat_format == "chatml": - pad_token_id = eos_token_id = tokenizer.im_end_id - else: - raise Exception("unknown chat format ", chat_format) - - use_gpt_attention_plugin = config["build_config"]["plugin_config"][ - "gpt_attention_plugin"] - gemm_allreduce_plugin = config["build_config"]["plugin_config"][ - "gemm_allreduce_plugin"] - - remove_input_padding = config["build_config"]["plugin_config"][ - "remove_input_padding"] - dtype = config["pretrained_config"]["dtype"] - tp_size = config["pretrained_config"]["mapping"]["tp_size"] - pp_size = config["pretrained_config"]["mapping"]["pp_size"] - world_size = tp_size * pp_size - assert ( - world_size == tensorrt_llm.mpi_world_size() - ), f"Engine world size ({world_size}) != Runtime world size ({tensorrt_llm.mpi_world_size()})" - num_heads = config["pretrained_config"][ - "num_attention_heads"] // world_size - max_batch_size = config["build_config"]["max_batch_size"] - hidden_size = config["pretrained_config"]["hidden_size"] // world_size - vocab_size = config["pretrained_config"]["vocab_size"] - num_layers = config["pretrained_config"]["num_hidden_layers"] - num_kv_heads = config["pretrained_config"].get("num_key_value_heads", - num_heads) - if "kv_cache_type" in config["build_config"]: - kv_cache_type = KVCacheType(config["build_config"]["kv_cache_type"]) - else: - kv_cache_type = KVCacheType.CONTINUOUS - - tokens_per_block = config["build_config"]["plugin_config"][ - "tokens_per_block"] - max_prompt_embedding_table_size = config["build_config"].get( - "max_prompt_embedding_table_size", 0) - quant_mode = QuantMode.from_quant_algo( - config["pretrained_config"]["quantization"]["quant_algo"], - config["pretrained_config"]["quantization"]["kv_cache_quant_algo"], - ) - if config["pretrained_config"].get("multi_query_mode", False): - tensorrt_llm.logger.warning( - "`multi_query_mode` config is deprecated. Please rebuild the engine." - ) - num_kv_heads = 1 - - runtime_rank = tensorrt_llm.mpi_rank() - runtime_mapping = tensorrt_llm.Mapping(world_size=world_size, - rank=runtime_rank, - tp_size=tp_size, - pp_size=pp_size) - torch.cuda.set_device(runtime_rank % runtime_mapping.gpus_per_node) - - model_config = ModelConfig( - max_batch_size=max_batch_size, - num_heads=num_heads, - num_kv_heads=num_kv_heads, - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - gpt_attention_plugin=use_gpt_attention_plugin, - gemm_allreduce_plugin=gemm_allreduce_plugin, - kv_cache_type=kv_cache_type, - tokens_per_block=tokens_per_block, - remove_input_padding=remove_input_padding, - dtype=dtype, - quant_mode=quant_mode, - max_prompt_embedding_table_size=max_prompt_embedding_table_size, - max_beam_width=self.num_beams, - ) - sampling_config = SamplingConfig( - end_id=eos_token_id, - pad_id=pad_token_id, - num_beams=self.num_beams, - top_k=top_k, - top_p=top_p, - temperature=1.0, - ) - - engine_name = get_engine_name(runtime_rank) - serialize_path = os.path.join(self.qwen_engine_dir, engine_name) - print(f"Loading engine from {serialize_path}") - return ( - model_config, - sampling_config, - runtime_mapping, - runtime_rank, - serialize_path, - tokenizer, - eos_token_id, - pad_token_id, - ) - - def qwen_model_init(self): - ( - model_config, - sampling_config, - runtime_mapping, - runtime_rank, - serialize_path, - tokenizer, - eos_token_id, - pad_token_id, - ) = self.get_model() - with open(serialize_path, "rb") as f: - engine_buffer = f.read() - self.decoder = tensorrt_llm.runtime.GenerationSession( - model_config, - engine_buffer, - runtime_mapping, - ) - self.tokenizer = tokenizer - self.sampling_config = sampling_config - self.model_config = model_config - self.config, _ = AutoConfig.from_pretrained( - self.tokenizer_dir, - return_unused_kwargs=True, - trust_remote_code=True, - ) - - def ptuning_setup(self, prompt_table, dtype, hidden_size, tasks, input_ids): - if prompt_table is not None: - task_vocab_size = torch.tensor([prompt_table.shape[1]], - dtype=torch.int32, - device="cuda") - prompt_table = prompt_table.view( - (prompt_table.shape[0] * prompt_table.shape[1], - prompt_table.shape[2])) - prompt_table = prompt_table.cuda().to( - dtype=tensorrt_llm._utils.str_dtype_to_torch(dtype)) - else: - prompt_table = torch.empty([1, hidden_size]).cuda() - task_vocab_size = torch.zeros([1]).cuda() - - if tasks is not None: - tasks = torch.tensor([int(t) for t in tasks.split(",")], - dtype=torch.int32, - device="cuda") - assert (tasks.shape[0] == input_ids.shape[0] - ), "Number of supplied tasks must match input batch size" - else: - tasks = torch.zeros([input_ids.size(0)], dtype=torch.int32).cuda() - - return [prompt_table, tasks, task_vocab_size] - - def make_context( - self, - query: str, - history: List[Tuple[str, str]] = None, - system: str = "You are a helpful assistant.", - max_window_size: int = 6144, - ): - if history is None: - history = [] - - im_start, im_end = "<|im_start|>", "<|im_end|>" - im_start_tokens = [self.tokenizer.im_start_id] # 151644 - im_end_tokens = [self.tokenizer.im_end_id] # [151645] - nl_tokens = self.tokenizer.encode("\n") - - def _tokenize_str(role, content): - return f"{role}\n{content}", self.tokenizer.encode( - role, allowed_special=set(self.tokenizer.IMAGE_ST) - ) + nl_tokens + self.tokenizer.encode( - content, allowed_special=set(self.tokenizer.IMAGE_ST)) - - system_text, system_tokens_part = _tokenize_str("system", system) - system_tokens = im_start_tokens + system_tokens_part + im_end_tokens - - raw_text = "" - context_tokens = [] - - for turn_query, turn_response in reversed(history): - query_text, query_tokens_part = _tokenize_str("user", turn_query) - query_tokens = im_start_tokens + query_tokens_part + im_end_tokens - if turn_response is not None: - response_text, response_tokens_part = _tokenize_str( - "assistant", turn_response) - response_tokens = im_start_tokens + response_tokens_part + im_end_tokens - - next_context_tokens = (nl_tokens + query_tokens + nl_tokens + - response_tokens) - prev_chat = f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}" - else: - next_context_tokens = nl_tokens + query_tokens + nl_tokens - prev_chat = f"\n{im_start}{query_text}{im_end}\n" - - current_context_size = (len(system_tokens) + - len(next_context_tokens) + - len(context_tokens)) - if current_context_size < max_window_size: - context_tokens = next_context_tokens + context_tokens - raw_text = prev_chat + raw_text - else: - break - - context_tokens = system_tokens + context_tokens - raw_text = f"{im_start}{system_text}{im_end}" + raw_text - context_tokens += (nl_tokens + im_start_tokens + - _tokenize_str("user", query)[1] + im_end_tokens + - nl_tokens + im_start_tokens + - self.tokenizer.encode("assistant") + nl_tokens) - raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n" - - return raw_text, context_tokens - - def generate_for_qwenvl( - self, - input_tokens, - max_new_tokens: int, - prompt_table=None, - tasks=None, - task_vocab_size=None, - num_beams=1, - ): - input_ids = None - input_lengths = None - input_ids = torch.as_tensor(input_tokens, - device="cuda", - dtype=torch.int32) - input_lengths = torch.tensor([input_ids.size(1)], - device="cuda", - dtype=torch.int32) - max_input_length = torch.max(input_lengths).item() - max_new_tokens = min(max_new_tokens, - self.global_max_input_len - max_input_length) - - profiler.start("QWen") - run_time = 10 - for _ in range(run_time): - self.decoder.setup( - batch_size=input_lengths.size(0), - max_context_length=max_input_length, - max_new_tokens=max_new_tokens, - beam_width=num_beams, - ) - output_ids = self.decoder.decode( - input_ids, - input_lengths, - self.sampling_config, - prompt_table, - tasks, - task_vocab_size, - ) - torch.cuda.synchronize() - profiler.stop("QWen") - Qwen_time = profiler.elapsed_time_in_sec("QWen") / run_time - - return output_ids, Qwen_time - - def qwen_infer( - self, - input_vit, - images_path, - input_text, - max_new_tokens, - num_beams=1, - history=None, - ): - if images_path is None: - content_list = [] - else: - content_list = images_path - if history is None: - history = [] - content_list.append({"text": input_text}) - query = self.tokenizer.from_list_format(content_list) - raw_text, context_tokens = self.make_context(query, history=history) - # context_tokens = self.tokenizer.encode(query) - input_ids = torch.tensor([context_tokens]).to("cuda") - bos_pos = torch.where(input_ids == self.config.visual["image_start_id"]) - eos_pos = torch.where( - input_ids == self.config.visual["image_start_id"] + 1) - assert (bos_pos[0] == eos_pos[0]).all() - img_pos = torch.stack((bos_pos[0], bos_pos[1], eos_pos[1]), dim=1) - vocab_size = self.config.vocab_size - fake_prompt_id = torch.arange( - vocab_size, - vocab_size + input_vit.shape[0] * input_vit.shape[1], - device="cuda", - ) - fake_prompt_id = fake_prompt_id.reshape(input_vit.shape[0], - input_vit.shape[1]) - for idx, (i, a, b) in enumerate(img_pos): - input_ids[i][a + 1:b] = fake_prompt_id[idx] - input_ids = input_ids.contiguous().to(torch.int32).cuda() - input_lengths = torch.tensor(input_ids.size(1), - dtype=torch.int32).cuda() - dtype = self.model_config.dtype - prompt_table, tasks, task_vocab_size = self.ptuning_setup( - input_vit, dtype, self.config.hidden_size, None, input_ids) - - output_ids, Qwen_time = self.generate_for_qwenvl( - input_ids, max_new_tokens, prompt_table, tasks, task_vocab_size, - num_beams) - - runtime_rank = tensorrt_llm.mpi_rank() - input_lengths = torch.tensor([input_ids.size(1)], - device="cuda", - dtype=torch.int32) - effective_output_token = 0 - if runtime_rank == 0: - if self.output_csv is None and self.output_npy is None: - for b in range(input_lengths.size(0)): - inputs = input_ids[b] - if content_list is not None: - print(f'Input: "{content_list}"') - print("\n") - if self.num_beams <= 1: - outputs = output_ids[b][0, len(inputs):].tolist() - try: - effective_output_token = (effective_output_token + - outputs.index(151643)) - except: - effective_output_token = 1 - output_text = self.tokenizer.decode( - outputs, skip_special_tokens=True) - print(f'Output: "{output_text}"') - print("\n") - else: - for beam in range(self.num_beams): - outputs = output_ids[b][beam, len(inputs):].tolist() - output_text = self.tokenizer.decode( - outputs, skip_special_tokens=True) - print(f'Output(beam: {beam}): "{output_text}"') - logger.info(f"Input length={input_lengths[b]}") - logger.info(f"Output length={output_ids.shape}") - logger.info(f"TensorRT LLM QWen time: {Qwen_time:3f} sec ") - history.append((query, output_text)) - return output_text - - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--max_new_tokens", type=int, default=200) - parser.add_argument("--log_level", type=str, default="info") - parser.add_argument( - "--vit_engine_path", - type=str, - default="plan/visual_encoder/visual_encoder_fp16.plan", - ) - parser.add_argument( - "--qwen_engine_dir", - type=str, - default="qwen_outputs", - ) - parser.add_argument( - "--tokenizer_dir", - type=str, - default=".", - help="Directory containing the tokenizer.model.", - ) - parser.add_argument("--input_text", - type=str, - default="Describe the picture") - parser.add_argument( - "--images_path", - nargs="+", - type=json.loads, - default=[{ - "image": "./pics/demo.jpeg" - }], - ) - parser.add_argument( - "--input_tokens", - dest="input_file", - type=str, - help= - "CSV or Numpy file containing tokenized input. Alternative to text input.", - default=None, - ) - parser.add_argument( - "--output_csv", - type=str, - help="CSV file where the tokenized output is stored.", - default=None, - ) - parser.add_argument( - "--output_npy", - type=str, - help="Numpy file where the tokenized output is stored.", - default=None, - ) - parser.add_argument("--num_beams", - type=int, - help="Use beam search if num_beams >1", - default=1) - parser.add_argument("--display", default=False, action='store_true') - parser.add_argument('--port', type=str, default='8006') - parser.add_argument("--local_machine", default=False, action='store_true') - - return parser.parse_args() - - -def vit_process(image_path, vit_engine_path, stream): - img_processor = Preprocss(448) - logger.info(f"Loading engine from {vit_engine_path}") - with open(vit_engine_path, "rb") as f: - engine_buffer = f.read() - logger.info(f"Creating session from engine {vit_engine_path}") - session_vit = Session.from_serialized_engine(engine_buffer) - device = torch.device("cuda") if torch.cuda.is_available() else "cpu" - image_path_list = [] - for item in image_path: - image_path_list.append(next(iter(item.values()))) - images = img_processor.encode(image_path_list).to(device) - batch_size = images.size(0) - images = images.expand(batch_size, -1, -1, -1).contiguous() - visual_inputs = {"input": images.float()} - visual_output_info = session_vit.infer_shapes( - [TensorInfo("input", trt.DataType.FLOAT, images.shape)]) - visual_outputs = { - t.name: - torch.empty(tuple(t.shape), - dtype=trt_dtype_to_torch(t.dtype), - device="cuda") - for t in visual_output_info - } - profiler.start("ViT") - - run_time = 10 - for _ in range(run_time): - ok = session_vit.run(visual_inputs, visual_outputs, stream) - profiler.stop("ViT") - Vit_time = profiler.elapsed_time_in_sec("ViT") / run_time - logger.info(f"TensorRT LLM ViT latency: {Vit_time:3f} sec ") - - assert ok, "Runtime execution failed for vit session" - - image_embeds = visual_outputs["output"] - return image_embeds - - -if __name__ == "__main__": - emit_engine_arch_deprecation("run.py") - args = parse_arguments() - stream = torch.cuda.current_stream().cuda_stream - tensorrt_llm.logger.set_level(args.log_level) - image_embeds = vit_process(args.images_path, args.vit_engine_path, stream) - qinfer = QWenInfer( - args.tokenizer_dir, - args.qwen_engine_dir, - args.log_level, - args.output_csv, - args.output_npy, - args.num_beams, - ) - qinfer.qwen_model_init() - qinfer.qwen_infer( - image_embeds, - args.images_path, - args.input_text, - args.max_new_tokens, - args.num_beams, - history=[], - ) diff --git a/examples/models/core/qwenvl/run_chat.py b/examples/models/core/qwenvl/run_chat.py deleted file mode 100644 index 1f1ba6fb6faf..000000000000 --- a/examples/models/core/qwenvl/run_chat.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re - -# isort: off -import torch -from run import QWenInfer, parse_arguments, vit_process -# isort: on - - -def make_display(port=8006): - import cv2 - import zmq - context = zmq.Context() - socket = context.socket(zmq.REP) - socket.bind(f"tcp://*:{port}") - - def func(image): - data = cv2.imencode(".jpg", image)[1].tobytes() - socket.recv() - socket.send(data) - - return func - - -def show_pic(image_path, port): - import cv2 - image = cv2.imread(image_path) - display_obj = make_display(port) - display_obj(image) - - -def show_pic_local(image_path): - import cv2 - import matplotlib.pyplot as plt - image = cv2.imread(image_path) - image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - - plt.imshow(image_rgb) - plt.pause(0.1) - - -def cooridinate_extract_show(input, history, tokenizer, local_machine, port): - pattern = r"\((\d+),(\d+)\)" - coordinates = re.findall(pattern, input) - result = "Box({},{})".format(coordinates[0][0], - coordinates[0][1]) - result += ",({},{})".format(coordinates[1][0], coordinates[1][1]) - - image = tokenizer.draw_bbox_on_latest_picture(result, history) - if image: - image.save('1.png') - if local_machine: - show_pic_local('1.png') - else: - show_pic('1.png', port) - else: - print("======No bounding boxes are detected!") - - -def exist_cooridinate(input): - pattern = r"\((\d+),(\d+)\)" - match = re.search(pattern, input) - if match: - return True - else: - return False - - -if __name__ == '__main__': - args = parse_arguments() - stream = torch.cuda.current_stream().cuda_stream - image_embeds = vit_process(args.images_path, args.vit_engine_path, stream) - qinfer = QWenInfer(args.tokenizer_dir, args.qwen_engine_dir, args.log_level, - args.output_csv, args.output_npy, args.num_beams) - qinfer.qwen_model_init() - - run_i = 0 - history = [] - if args.display: - if args.local_machine: - show_pic_local("./pics/demo.jpeg") - else: - show_pic("./pics/demo.jpeg", args.port) - - while True: - input_text = None - try: - input_text = input("Text (or 'q' to quit): ") - except: - continue - - if input_text == "clear history": - history = [] - continue - - if input_text.lower() == 'q': - break - print('\n') - - content_list = args.images_path - content_list.append({'text': input_text}) - - if run_i == 0: - query = qinfer.tokenizer.from_list_format(content_list) - else: - query = input_text - - run_i = run_i + 1 - output_text = qinfer.qwen_infer(image_embeds, None, query, - args.max_new_tokens, args.num_beams, - history) - if args.display: - if exist_cooridinate(output_text): - cooridinate_extract_show(output_text, history, qinfer.tokenizer, - args.local_machine, args.port) diff --git a/examples/models/core/qwenvl/show_pic.py b/examples/models/core/qwenvl/show_pic.py deleted file mode 100644 index f390ec860f58..000000000000 --- a/examples/models/core/qwenvl/show_pic.py +++ /dev/null @@ -1,34 +0,0 @@ -import argparse - -import cv2 -import numpy as np -import zmq - -context = zmq.Context() -socket = context.socket(zmq.REQ) - - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument('--ip', type=str, default='127.0.0.1') - parser.add_argument('--port', type=str, default='8006') - return parser.parse_args() - - -args = parse_arguments() -ip_addr = "tcp://" + args.ip + ":" + args.port -socket.connect(ip_addr) - -while True: - socket.send(b"a") - message = socket.recv() - if len(message) == 1 and message == b'x': - break - image = np.frombuffer(message, dtype=np.uint8) - image = cv2.imdecode(image, 1) - image = cv2.resize(image, dsize=(512, 384)) - cv2.imshow("image", image) - key = cv2.waitKey(1) & 0xFF - - if key == ord('q'): - break diff --git a/examples/models/core/qwenvl/vit_onnx_trt.py b/examples/models/core/qwenvl/vit_onnx_trt.py deleted file mode 100644 index ba21fc93ef0a..000000000000 --- a/examples/models/core/qwenvl/vit_onnx_trt.py +++ /dev/null @@ -1,196 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import os -import time -from typing import List - -import requests -import tensorrt as trt -import torch -from PIL import Image -from torchvision import transforms -from torchvision.transforms import InterpolationMode -from transformers import AutoModelForCausalLM - -from tensorrt_llm._utils import release_gc, str_dtype_to_torch - - -class Preprocss: - - def __init__(self, image_size: int): - mean = (0.48145466, 0.4578275, 0.40821073) - std = (0.26862954, 0.26130258, 0.27577711) - self.image_transform = transforms.Compose([ - transforms.Resize((image_size, image_size), - interpolation=InterpolationMode.BICUBIC), - transforms.ToTensor(), - transforms.Normalize(mean=mean, std=std), - ]) - - def encode(self, image_paths: List[str]): - images = [] - for image_path in image_paths: - if image_path.startswith("http://") or image_path.startswith( - "https://"): - image = Image.open(requests.get(image_path, stream=True).raw) - else: - image = Image.open(image_path) - image = image.convert("RGB") - images.append(self.image_transform(image)) - images = torch.stack(images, dim=0) - return images - - -class ONNX_TRT: - - def __init__(self, image_size): - self.image_size = image_size - - def export_onnx(self, onnx_file_path, pretrained_model_path, image_url): - print("Start converting ONNX model!") - image_pre_obj = Preprocss(self.image_size) - torch_dtype = str_dtype_to_torch("float16") - model = AutoModelForCausalLM.from_pretrained( - pretrained_model_path, - device_map="cuda", - dtype=torch_dtype, - fp16=True, - trust_remote_code=True, - ).eval() - device = torch.device("cuda") if torch.cuda.is_available() else "cpu" - image = image_pre_obj.encode(image_url).to(device) - if not os.path.exists("image.pt"): - torch.save(image, "image.pt") - - model_visual = model.transformer.visual - model_visual.eval() - del model # To save GPU memory - - torch.onnx.export( - model_visual, - image.to("cuda"), - onnx_file_path, - opset_version=17, - input_names=["input"], - output_names=["output"], - dynamic_axes={"input": { - 0: "batch" - }}, - # Required for pytorch>=2.9.0 as dynamo becomes the default and introduces bugs as it does not support opset_version=17 natively - dynamo=False) - release_gc() # Further release memory - print( - f"Export to ONNX file successfully! The ONNX file stays in {onnx_file_path}" - ) - - def generate_trt_engine(self, - onnxFile, - planFile, - minBS=1, - optBS=2, - maxBS=4): - print("Start converting TRT engine!") - logger = trt.Logger(trt.Logger.VERBOSE) - builder = trt.Builder(logger) - network = builder.create_network( - 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) - profile = builder.create_optimization_profile() - config = builder.create_builder_config() - config.set_flag(trt.BuilderFlag.FP16) - parser = trt.OnnxParser(network, logger) - - with open(onnxFile, "rb") as model: - if not parser.parse(model.read(), "/".join(onnxFile.split("/"))): - print("Failed parsing %s" % onnxFile) - for error in range(parser.num_errors): - print(parser.get_error(error)) - print("Succeeded parsing %s" % onnxFile) - - nBS = -1 - nMinBS = minBS - nOptBS = optBS - nMaxBS = maxBS - inputT = network.get_input(0) - inputT.shape = [nBS, 3, self.image_size, self.image_size] - profile.set_shape( - inputT.name, - [nMinBS, 3, self.image_size, self.image_size], - [nOptBS, 3, self.image_size, self.image_size], - [nMaxBS, 3, self.image_size, self.image_size], - ) - - config.add_optimization_profile(profile) - - t0 = time.time() - engineString = builder.build_serialized_network(network, config) - t1 = time.time() - if engineString is None: - print("Failed building %s" % planFile) - else: - print("Succeeded building %s in %d s" % (planFile, t1 - t0)) - with open(planFile, "wb") as f: - f.write(engineString) - - -def parse_arguments(): - parser = argparse.ArgumentParser() - # onnx/visual_encoder - parser.add_argument("--onnxFile", - type=str, - default="visual_encoder/visual_encoder.onnx", - help="") - parser.add_argument("--pretrained_model_path", - type=str, - default="Qwen-VL-Chat", - help="") - parser.add_argument( - "--planFile", - type=str, - default="plan/visual_encoder/visual_encoder_fp16.plan", - help="", - ) - parser.add_argument( - "--only_trt", - action="store_true", - help="Run only convert the onnx to TRT engine.", - ) - parser.add_argument("--minBS", type=int, default=1) - parser.add_argument("--optBS", type=int, default=1) - parser.add_argument("--maxBS", type=int, default=4) - parser.add_argument("--image_url", nargs="+", default=["./pics/demo.jpeg"]) - args = parser.parse_args() - return args - - -if __name__ == "__main__": - args = parse_arguments() - onnx_file_dir = os.path.dirname(args.onnxFile) - if not onnx_file_dir == "" and not os.path.exists(onnx_file_dir): - os.makedirs(onnx_file_dir) - plan_file_dir = os.path.dirname(args.planFile) - if not os.path.exists(plan_file_dir): - os.makedirs(plan_file_dir) - - onnx_trt_obj = ONNX_TRT(448) # or ONNX_TRT(config.visual['image_size']) - - if args.only_trt: - onnx_trt_obj.generate_trt_engine(args.onnxFile, args.planFile, - args.minBS, args.optBS, args.maxBS) - else: - onnx_trt_obj.export_onnx(args.onnxFile, args.pretrained_model_path, - args.image_url) - onnx_trt_obj.generate_trt_engine(args.onnxFile, args.planFile, - args.minBS, args.optBS, args.maxBS) diff --git a/examples/ngram/README.md b/examples/ngram/README.md index 1e4dc8792b08..4b5a00ee43ac 100644 --- a/examples/ngram/README.md +++ b/examples/ngram/README.md @@ -1,39 +1,32 @@ # NGram Speculative Decoding -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -This document shows how to build and run a model using NGram speculative decoding (supported as `ASSISTED_GENERATION` in transformers and vLLM, source: [GitHub](https://github.com/apoorvumang/prompt-lookup-decoding/tree/main)) in TensorRT LLM on single GPU, or single node multiple GPU. +This document shows how to run a model with NGram speculative decoding +(supported as `ASSISTED_GENERATION` in transformers and vLLM, source: +[GitHub](https://github.com/apoorvumang/prompt-lookup-decoding/tree/main)) +in TensorRT LLM. ## Overview -We provide two styles of workflow to run NGram (named V1 and V2 respectively) now. V1 is in TRT workflow and similar to the Draft-Target-Model workflow, running in orchestrator mode and calling `runner.generate()` multiple times to get outputs, which is more flexible for customizing but slightly more overhead. V2 is in pytorch workflow and similar to the Look-Ahead workflow, running in leader mode and calling `runner.generate()` only one time to get outputs, which provides higher performance but fixed process. - -The NGram has 3 additional hyperparameters that you need to specify to control the process of generation: -- `max_draft_len`: the maximum number of tokens provided as draft tokens in one iteration, which is usually from 4 to 10 in common usage (default value: 4). Empirically, the larger the value is, the higher acceptance rate but higher overhead is expected at the same time, so the right balance based on the models and application scenarios needs to be found. -- `max_matching_ngram_size`: the maximum number of tokens extracted from the tail of the input prompt or generated output as a pattern, which is used to search corresponding draft tokens (default value: 2). Empirically, the larger the value is, the more precise context can be matched from the existed sequence, indicating higher acceptance rate, but the higher probability of miss-match and higher overhead appear, which fall back to normal generation (one token per iteration). -- `device_list`: the index list of device(s) to run the model in V1 workflow. The length of it must be the same as the TP size of the draft model engine. For instances, `device_list=[0]` means using tp_size=1 and GPU 0 for the model, `device_list=[4,5,6,7]` means using tp=4 and GPU from 4 to 7 for the model. This parameter is neddless in V2 workflow. - -+ For example, the process of getting draft tokens using `max_draft_len=2` and `max_matching_ngram_size=4` with a sentence `prefix=[..., t1, t2, t3, t4]` is like below: - -```Python -pattern = prefix[:-2] # pattern=[t3, t4] (length=2) -if pattern in pool and len(pool[pattern]) == 4: # assuming it is {(t3, t4): (t5, t6, t7, t8)} - return pool[pattern] # draft token = [t5, t6, t7, t8] -elif pattern in pool and len(pool[pattern]) == <4: # assuming it is {(t3, t4): (t9, t10, t11)} - return pool[pattern] # draft token = [t9, t10, t11] -pattern = prefix[:-1] # Try shorter pattern if no candidate of length=2 exists, pattern=[t4] (length=1) -if pattern in pool and len(pool[pattern]) == 4: # The same process as above - return pool[pattern] -elif pattern in pool and len(pool[pattern]) == <4: - return pool[pattern] -return None # No any candidate exists -``` +NGram builds a pattern pool from the prompt and previously generated tokens +and proposes draft tokens by matching the tail of the current sequence +against that pool. It has 2 hyperparameters that control the process of +generation: + +- `max_draft_len`: the maximum number of tokens provided as draft tokens in + one iteration, which is usually from 4 to 10 in common usage (default + value: 4). Empirically, the larger the value is, the higher acceptance rate + but higher overhead is expected at the same time, so the right balance + based on the models and application scenarios needs to be found. +- `max_matching_ngram_size`: the maximum number of tokens extracted from the + tail of the input prompt or generated output as a pattern, which is used to + search corresponding draft tokens (default value: 2). Empirically, the + larger the value is, the more precise context can be matched from the + existed sequence, indicating higher acceptance rate, but the higher + probability of miss-match and higher overhead appear, which fall back to + normal generation (one token per iteration). ## Support Matrix + * GPU Compute Capability >= 8.0 (Ampere or newer) * FP16 / BF16 / FP8 * Paged KV Cache @@ -41,59 +34,6 @@ return None # No any candidate exists ## Usage -### V1 workflow - -+ We use an open-source `llama-v2-13B` models in this example. -+ `--use_paged_context_fmha=enable` must be specified since we need KVcache reuse in this approach. -+ `--speculative_decoding_mode=draft_tokens_external` must be specified. -+ `--max_draft_len` must be specified as the length maximum of the draft tokens. -+ `--ngram_config` is corresponding configuration of NGram, we can see its usage in [util.py](../util.py). - + As an example, `[10,2,[0]]` means `max_draft_len=10`, `max_matching_ngram_size=2`, and device of target model is `GPU0`. -+ `--kv_cache_enable_block_reuse` must be specified for this approach. -+ Only CPP session is supported, so `--use_py_session` must not be specified. -+ `--num_beams` can not be specified as larger than 1 since beam search is not supported in this approach yet. - -```bash -# Build engine -python3 examples/models/core/llama/convert_checkpoint.py \ - --model_dir \ - --output_dir ./ckpt-target \ - --dtype float16 - -trtllm-build \ - --checkpoint_dir ./ckpt-target \ - --output_dir ./target-engine \ - --gemm_plugin float16 \ - --use_paged_context_fmha enable \ - --speculative_decoding_mode draft_tokens_external \ - --max_draft_len 10 \ - --max_batch_size 4 \ - --max_input_len 3200 \ - --max_seq_len 4800 - -# Run decoding -python3 examples/run.py \ - --tokenizer_dir \ - --engine_dir ./target-engine \ - --ngram_config "[10,2,[0]]" \ - --max_output_len 256 \ - --kv_cache_enable_block_reuse \ - --input_text "How does Draft-Sampling work?" - -# Run summarization tasks -python examples/summarize.py \ - --test_hf \ - --test_trt_llm \ - --check_accuracy \ - --hf_model_dir \ - --engine_dir ./target-engine \ - --batch_size 1 \ - --ngram_config "[10,2,[0]]" \ - --kv_cache_enable_block_reuse -``` - -### V2 workflow - ```bash python3 examples/llm-api/quickstart_advanced.py \ --spec_decode_max_draft_len 4 \ @@ -101,3 +41,8 @@ python3 examples/llm-api/quickstart_advanced.py \ --disable_overlap_scheduler \ --disable_kv_cache_reuse ``` + +With the LLM API, configure NGram through `NGramDecodingConfig` +(`speculative_config`). See the +[speculative decoding documentation](https://nvidia.github.io/TensorRT-LLM/features/speculative-decoding.html) +for details. diff --git a/examples/ngram/run_dtm_ngram.py b/examples/ngram/run_dtm_ngram.py deleted file mode 100644 index d0cd8687ef86..000000000000 --- a/examples/ngram/run_dtm_ngram.py +++ /dev/null @@ -1,382 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import ast - -import numpy as np -import torch -from ordered_set import OrderedSet - -from tensorrt_llm.logger import logger -from tensorrt_llm.runtime import ModelRunnerCpp - - -class NgramPool: # Ngrams pool for Ngram - - def __init__( - self, - input_batch_size: int, - max_draft_len: int, - max_matching_ngram_size: int, - end_id: int, - max_seq_len: list[int], - is_keep_all: bool = True, - is_use_oldest: bool = True, - ): - self.input_batch_size = input_batch_size - self.max_draft_len = max_draft_len - self.max_matching_ngram_size = max_matching_ngram_size - self.end_id = end_id - self.max_seq_len = max_seq_len - self.is_keep_all = is_keep_all - self.is_use_oldest = is_use_oldest - self.pool = [{} for _ in range(input_batch_size)] - self.start_index = [0 for _ in range(input_batch_size)] - - assert self.max_draft_len > 0, f"max_draft_len must be greater than 0, but got {self.max_draft_len}" - assert self.max_matching_ngram_size > 0, f"max_matching_ngram_size must be greater than 0, but got {self.max_matching_ngram_size}" - - def print_pool(self): - """ - For debug - """ - logger.info(f"Batch size = {self.input_batch_size}") - for i, map in enumerate(self.pool): - logger.info(f"Slot {i}, size = {len(map)}") - for key, values in map.items(): - logger.info(f" {key}->{values}") - - def get_draft_tokens(self, prefix: list[torch.Tensor], - batch_slot: list[int]): - """ - Get draft tokens from a batch of requests - modified from `transformers/generation/candidate_generator.py` - """ - batch_size = len(prefix) - prefix_len = [len(prefix[bi]) for bi in range(batch_size)] - draft_tokens = [] # `logits` is useless yet - for bi in range(batch_size): - gbi = batch_slot[bi] # Global index in the input batch - chosen_ids = [self.end_id] - # Skip search if prefix is length of `max_length - 1` - if prefix_len[bi] >= self.max_seq_len[gbi] - 1: - draft_tokens.append(chosen_ids) - continue - - # Update pool - sequence = prefix[bi][self.start_index[gbi]:].tolist() - for size in range( - min(self.max_matching_ngram_size, prefix_len[bi] - 1), 0, - -1): - # Find each possible key-value combination, and use tuple for hash - for l in range(len(sequence) - size): - r = min(l + size + self.max_draft_len, len(sequence)) - key = tuple(sequence[l:l + size]) - value = tuple(sequence[l + size:r]) - if key not in self.pool[gbi] or not self.is_keep_all or \ - len(self.pool[gbi][key][0]) < self.max_draft_len: - # Update the value if - # 1. the key does not exist - # 2. we only keep the newest one value for each key (MRU) - # 3. the length of the value saved before is less than `max_draft_len` - self.pool[gbi][key] = OrderedSet((value, )) - elif value not in self.pool[gbi][key]: - # Extend the value if the key is already existed but count of values is not enough - self.pool[gbi][key].add(value) - - # Find match - for size in range( - min(self.max_matching_ngram_size, prefix_len[bi] - 1), 0, - -1): - pattern = tuple(prefix[bi][-size:].tolist()) - if pattern not in self.pool[gbi]: - continue - if self.is_use_oldest: - # Always choose the oldest match, aligned with HF - chosen_ids = self.pool[gbi][pattern][0] - else: - # Always choose the newest match - chosen_ids = self.pool[gbi][pattern][-1] - break - draft_tokens.append(chosen_ids) - self.start_index[gbi] = max( - 0, prefix_len[bi] - - (self.max_draft_len + self.max_matching_ngram_size - 1)) - - return draft_tokens, None - - -def run_dtm_ngram(batch_input_ids, - args, - runtime_rank, - end_id, - pad_id, - stop_words_list, - bad_words_list, - vocab_size, - *, - target_runner=None): - # `dtm` for Draft-Target-Model, `ngram` for NGram - is_dtm = (args.draft_target_model_config is not None) - is_ngram = (args.ngram_config is not None) - assert is_dtm ^ is_ngram, "`--draft_target_model_config` and `--ngram_config` can not be specified at the same time." - if is_dtm: - assert args.draft_engine_dir is not None, "`--draft_engine_dir` must be specified in Draft-Target-Model." - draft_len, draft_device_list, target_device_list, use_logits = ast.literal_eval( - args.draft_target_model_config) - logger.info(f"Using Draft-Target-Model speculative decoding") - logger.info(f"draft_len: {draft_len}") - logger.info(f"Device(s) for draft model: {draft_device_list}") - logger.info(f"Device(s) for target model: {target_device_list}") - logger.info(f"Use logits to accept tokens: {use_logits}") - if is_ngram: - logger.info(f"Using NGram speculative decoding V1 workflow") - max_draft_len, max_matching_ngram_size, target_device_list = ast.literal_eval( - args.ngram_config) - logger.info(f"max_draft_len: {max_draft_len}") - logger.info(f"max_matching_ngram_size: {max_matching_ngram_size}") - logger.info(f"Device(s) for the model: {target_device_list}") - use_logits = False # `logits` is useless in this approach yet - - # Variables keeping constant during decoding - input_batch_size = len(batch_input_ids) # Note as `BS` - beam_width = args.num_beams # Note as `BW` - is_compute_acceptance_ratio = logger.level == 'verbose' # Only for verbose - input_len = [len(p) for p in batch_input_ids] - max_seq_len = [i + args.max_output_len for i in input_len] - # Variables changing during decoding - n_iteration = 0 - prefix = batch_input_ids # Input for each iteration - batch_slot = list(range(input_batch_size)) # Index of requests - if is_compute_acceptance_ratio: - n_draft_token = [0 for _ in range(input_batch_size)] - n_accept_token = [0 for _ in range(input_batch_size)] - - if is_ngram: - ngram_pool = NgramPool(input_batch_size, max_draft_len, - max_matching_ngram_size, end_id, max_seq_len) - - # Repack the output like the output of function `generate` - outputs = {} - outputs["output_ids"] = torch.full( - [input_batch_size, beam_width, - max(max_seq_len)], - end_id, - dtype=torch.int32) - for bi in range(input_batch_size): - outputs["output_ids"][bi, :, :input_len[bi]] = batch_input_ids[bi] - outputs["sequence_lengths"] = torch.full([input_batch_size, beam_width], - 0, - dtype=torch.int32) - outputs["context_logits"] = None - outputs["generation_logits"] = torch.full( - [input_batch_size, beam_width, - max(max_seq_len), vocab_size], - 0, - dtype=torch.float16) - outputs['cum_log_probs'] = None - outputs['log_probs'] = None - - # Model runner - common_runner_kwargs = dict( - lora_dir=args.lora_dir, - rank=runtime_rank, - debug_mode=args.debug_mode, - lora_ckpt_source=args.lora_ckpt_source, - gpu_weights_percent=args.gpu_weights_percent, - max_output_len=args.max_output_len, - is_enc_dec=False, - max_batch_size=input_batch_size, - max_input_len=max(input_len) + args.max_output_len, - max_beam_width=beam_width, - max_attention_window_size=args.max_attention_window_size, - sink_token_length=args.sink_token_length, - max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, - kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, - kv_cache_free_gpu_memory_fraction=args. - kv_cache_free_gpu_memory_fraction, - cross_kv_cache_fraction=None, - enable_chunked_context=args.enable_chunked_context, - multi_block_mode=args.multi_block_mode, - cuda_graph_mode=args.cuda_graph_mode, - enable_context_fmha_fp32_acc=args.enable_context_fmha_fp32_acc, - is_orchestrator_mode=True, - ) - - if is_dtm: - draft_runner_kwargs = common_runner_kwargs.copy() - draft_runner_kwargs.update(engine_dir=args.draft_engine_dir, - device_ids=draft_device_list) - draft_runner = ModelRunnerCpp.from_dir(**draft_runner_kwargs) - - if target_runner is None: # Skip this constructor if we have prepared the runner before - target_runner_kwargs = common_runner_kwargs.copy() - target_runner_kwargs.update(engine_dir=args.engine_dir, - device_ids=target_device_list) - target_runner = ModelRunnerCpp.from_dir(**target_runner_kwargs) - - if is_dtm and use_logits: - assert draft_runner.gather_generation_logits and target_runner.gather_generation_logits, "`--gather_generation_logits` must be specified while building draft/target models for using logits to accept" - - common_generaion_kwargs = dict( - max_attention_window_size=args.max_attention_window_size, - sink_token_length=args.sink_token_length, - end_id=end_id, - pad_id=pad_id, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=beam_width, - num_return_sequences=args.num_return_sequences, - length_penalty=args.length_penalty, - early_stopping=args.early_stopping, - beam_width_array=None, - repetition_penalty=args.repetition_penalty, - presence_penalty=args.presence_penalty, - frequency_penalty=args.frequency_penalty, - min_p=args.min_p, - stop_words_list=stop_words_list, - bad_words_list=bad_words_list, - random_seed=args.random_seed, - lora_uids=args.lora_task_uids, - prompt_table=args.prompt_table_path, - prompt_tasks=args.prompt_tasks, - streaming=False, - output_sequence_lengths=True, - no_repeat_ngram_size=args.no_repeat_ngram_size, - return_dict=True, - return_all_generated_tokens=args.return_all_generated_tokens, - ) - - while True: - n_iteration += 1 - # Dynamic batch_size, decreases if some requests finish - batch_size = len(prefix) - prefix_len = [len(prefix[i]) for i in range(batch_size)] - # Get draft tokens - # `d_*` means variables from draft - # `d_seq_len` includes input part, but `d_len` doesn't - if is_dtm: - draft_generation_kwargs = common_generaion_kwargs.copy() - draft_generation_kwargs.update( - batch_input_ids=prefix, - max_new_tokens=draft_len, - streaming=False, - output_sequence_lengths=True, - return_dict=True, - ) - draft = draft_runner.generate(**draft_generation_kwargs) - torch.cuda.synchronize() - - # draft["output_ids"].shape -> [BS, BW, maxSL] - # draft["sequence_lengths"].shape -> [BS, BW] - # draft["generation_logits"].shape -> [BS, BW, draft_len, vocab_size] - d_ids = [[end_id]] * batch_size - d_logits = [None] * batch_size if use_logits else None - d_seq_len = draft["sequence_lengths"][:, 0].tolist() - d_len = [d_seq_len[bi] - prefix_len[bi] for bi in range(batch_size)] - for bi in range(batch_size): - l, r = prefix_len[bi], d_seq_len[bi] - if l >= r: # No useful draft tokens - continue - d_ids[bi] = draft["output_ids"][bi, 0, l:r].tolist() - if use_logits: - d_logits[bi] = draft["generation_logits"][bi, 0, - -d_len[bi]:, :] - if is_ngram: - d_ids, d_logits = ngram_pool.get_draft_tokens(prefix, batch_slot) - d_len = [len(i) for i in d_ids] - - # Run target model - # `t_*` means variables from target model - # `t_seq_len` and `t_seq_ids` include input part, but `t_len` or `t_ids` don't - target_generation_kwargs = common_generaion_kwargs.copy() - target_generation_kwargs.update(batch_input_ids=prefix, - draft_tokens_list=d_ids, - draft_logits_list=d_logits) - if is_dtm: - max_new_tokens = draft_len + 1 - if is_ngram: - max_new_tokens = max_draft_len + 1 - target_generation_kwargs.update(max_new_tokens=max_new_tokens) - target = target_runner.generate(**target_generation_kwargs) - torch.cuda.synchronize() - - t_ids = [None] * batch_size - t_seq_ids = [None] * batch_size - t_seq_len = target["sequence_lengths"][:, 0].tolist() - t_len = [t_seq_len[bi] - prefix_len[bi] for bi in range(batch_size)] - - # Update output and tokens for next iteration - for bi in range(batch_size): - gbi = batch_slot[bi] # Global index in the input batch - l = prefix_len[bi] - r = min(t_seq_len[bi], max_seq_len[gbi]) - t_ids[bi] = target["output_ids"][bi, 0, l:r].tolist() - t_seq_ids[bi] = target["output_ids"][bi, 0, :r] - outputs["output_ids"][gbi, 0, l:r] = torch.IntTensor(t_ids[bi]) - outputs["sequence_lengths"][gbi, 0] = r - if use_logits: - outputs["generation_logits"][gbi, 0, (l - input_len[bi]):(r - input_len[bi])] = \ - target["generation_logits"][bi][0,:(r-l)].detach().cpu() - if is_compute_acceptance_ratio: - n_draft_token[gbi] += d_len[bi] - length = min(d_len[bi], t_len[bi], - max_seq_len[gbi] - prefix_len[bi]) - res = [d_ids[bi][i] == t_ids[bi][i] for i in range(length)] - n_accept_token[gbi] += \ - ((~torch.BoolTensor(res)).cumsum(axis=-1) < 1).sum() - - # Yield output if using streaming - if args.streaming and not n_iteration % args.streaming_interval: - yield outputs - - # Evaluate stop criteria and prepare inputs for next iteration - prefix_next = [] - batch_slot_next = [] - for bi in range(batch_size): - gbi = batch_slot[bi] # Global index in the input batch - # Stop due to output length - if len(t_seq_ids[bi]) >= max_seq_len[gbi]: - continue # No need to update for the stopped requests - # Stop due to the same output. Normally target should return 1 more token. - # if (d_ids is not None and np.array_equal(d_ids[bi], t_ids[bi])): - # continue - # Stop due to no change (hit early stopping) - if np.array_equal(t_seq_ids[bi].cpu().numpy(), - prefix[bi].cpu().numpy()): - continue - # Stop due to end words - if end_id in t_seq_ids[bi][prefix_len[bi]:]: - continue - # TODO: Check bad words and stop words criteria - prefix_next.append(t_seq_ids[bi]) - batch_slot_next.append(gbi) - prefix = prefix_next - batch_slot = batch_slot_next - if len(prefix) == 0: # Leave while loop if no request remained - break - - if is_compute_acceptance_ratio: - logger.debug(f"Count of iteration(s): {n_iteration}") - logger.debug(f"Acceptance ratio:") - for i, (a, d) in enumerate(zip(n_accept_token, n_draft_token)): - logger.debug(f"Request {i}: {a / d * 100 :6.2f}%") - - # Return runner in No-Streaming mode - if args.streaming: - yield outputs - else: - yield outputs, target_runner diff --git a/examples/openai_triton/README.md b/examples/openai_triton/README.md deleted file mode 100644 index b5f39d105974..000000000000 --- a/examples/openai_triton/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Integration for OpenAI Triton - -The typical approach to integrate a kernel into TensorRT LLM is to create TensorRT plugins. -Specially for integrating OpenAI Triton kernels, there are two methods: - -1. Creating TensorRT plugin manually, you can refer to [manual plugin example](./manual_plugin/) for details, -2. Generate the TensorRT plugins automatically, please refer to [automatic plugin example](./plugin_autogen/) for details. diff --git a/examples/openai_triton/manual_plugin/CMakeLists.txt b/examples/openai_triton/manual_plugin/CMakeLists.txt deleted file mode 100644 index bec14231511e..000000000000 --- a/examples/openai_triton/manual_plugin/CMakeLists.txt +++ /dev/null @@ -1,113 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -cmake_minimum_required(VERSION 3.1) - -# Enable C++ -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED TRUE) - -# Define project name -set(TARGET_NAME trt_llm_custom_plugins) -project(${TARGET_NAME}) - -set(CMAKE_VERBOSE_MAKEFILE 1) - -# Compile options -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -pthread ") -set(CMAKE_C_FLAGS_DEBUG "-g -O0") -set(CMAKE_C_FLAGS_RELEASE "-O2") -set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -lstdc++") -set(CMAKE_CXX_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG}) -set(CMAKE_CXX_FLAGS_RELEASE ${CMAKE_C_FLAGS_RELEASE}) - -set(CMAKE_BUILD_TYPE release) - -find_package(CUDA REQUIRED) -message(STATUS "CUDA library status:") -message(STATUS " config: ${CUDA_DIR}") -message(STATUS " version: ${CUDA_VERSION}") -message(STATUS " libraries: ${CUDA_LIBRARIES}") -message(STATUS " include path: ${CUDA_INCLUDE_DIRS}") - -if(NOT DEFINED TRT_INCLUDE_DIR) - set(TRT_INCLUDE_DIR "/usr/local/tensorrt/include") - if(NOT EXISTS ${TRT_INCLUDE_DIR}) - # In case of TensorRT installed from a deb package. - set(TRT_INCLUDE_DIR "/usr/include/x86_64-linux-gnu") - endif() -endif() -message(STATUS "tensorrt include path: ${TRT_INCLUDE_DIR}") -if(DEFINED TRT_LLM_INCLUDE_DIR) - message( - STATUS "openai_triton/manual_plugin example has been self-contained " - "and TRT_LLM_INCLUDE_DIR is now unnecessary to specify the path of " - "C++ runtime source files.") -endif() - -if(NOT DEFINED TRT_LIB_DIR) - set(TRT_LIB_DIR "/usr/local/tensorrt/lib") - if(NOT EXISTS ${TRT_INCLUDE_DIR}) - # In case of TensorRT installed from a deb package. - set(TRT_LIB_DIR "/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu") - endif() -endif() -find_library( - TRT_LIB_PATH nvinfer - HINTS ${TRT_LIB_DIR} - NO_DEFAULT_PATH) -find_library(TRT_LIB_PATH nvinfer REQUIRED) -message(STATUS "TRT_LIB_DIR: ${TRT_LIB_DIR}") -message(STATUS "Found nvinfer library: ${TRT_LIB_PATH}") - -if(NOT DEFINED TRT_LLM_LIB_DIR) - # Find at tensorrt_llm/libs. - execute_process( - COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${PYTHONPATH}" "python" "-c" - "import tensorrt_llm; print(f'{tensorrt_llm.__path__[0]}/libs')" - OUTPUT_VARIABLE TRT_LLM_LIB_DIR - OUTPUT_STRIP_TRAILING_WHITESPACE) - # Find /tensorrt_llm/libs. - list(APPEND TRT_LLM_LIB_DIR "../../../tensorrt_llm/libs") -endif() -find_library(TRT_LLM_LIB_PATH nvinfer_plugin_tensorrt_llm - HINTS ${TRT_LLM_LIB_DIR} NO_DEFAULT_PATH) -find_library(TRT_LLM_LIB_PATH nvinfer_plugin_tensorrt_llm REQUIRED) -message(STATUS "Found nvinfer_plugin_tensorrt_llm library: ${TRT_LLM_LIB_PATH}") - -find_library(TRT_LLM_COMMON_LIB_PATH th_common HINTS ${TRT_LLM_LIB_DIR} - NO_DEFAULT_PATH) -find_library(TRT_LLM_COMMON_LIB_PATH th_common REQUIRED) -message(STATUS "Found th_common library: ${TRT_LLM_COMMON_LIB_PATH}") - -# Declare the target library. -add_library( - ${TARGET_NAME} SHARED - tritonPlugins.cpp - TritonFlashAttentionPlugin.cpp - aot/fmha_kernel_fp16.c - aot/fmha_kernel_fp32.c - aot/fp16/fmha_kernel_d64_fp16.fbf0f274_0d1d2d3d4d5d6789.c - aot/fp32/fmha_kernel_d64_fp32.f30323ef_0d1d2d3d4d5d6789.c) - -target_link_libraries( - ${TARGET_NAME} PUBLIC cuda ${CUDA_LIBRARIES} ${TRT_LLM_LIB_PATH} - ${TRT_LLM_COMMON_LIB_PATH} ${TRT_LIB_PATH}) - -if(NOT MSVC) - set_property(TARGET ${TARGET_NAME} PROPERTY LINK_FLAGS "-Wl,--no-undefined") -endif() - -target_include_directories(${TARGET_NAME} PUBLIC /usr/local/cuda/include) -target_include_directories(${TARGET_NAME} PUBLIC ${TRT_INCLUDE_DIR}) diff --git a/examples/openai_triton/manual_plugin/README.md b/examples/openai_triton/manual_plugin/README.md deleted file mode 100644 index 5c8b5d481d52..000000000000 --- a/examples/openai_triton/manual_plugin/README.md +++ /dev/null @@ -1,177 +0,0 @@ -# OpenAI Triton Plugin in TensorRT-LLM - -This document describes how to build and run a custom plugin leveraging [OpenAI Triton](https://github.com/openai/triton) in TensorRT-LLM. -The workflow can be summarized as follows. - 1. Implement a kernel using Triton in Python. - 2. Compile that kernel using Triton AoT (Ahead-of-Time) compilation tool to generate C files. - 3. Implement a custom TensorRT LLM plugin to execute the compiled kernel. - 4. Build the TensorRT engine. - 5. It is ready to be executed by TensorRT. - -In this example, we show how to create a TensorRT LLM plugin to wrap a [Fused Attention]((fmha_triton.py)) kernel implemented in OpenAI Triton. -As a prerequisite, it is necessary to have the TensorRT LLM C++ runtime library. -The instructions to build that library can be found [here](../../README.md#build-from-source). - -## 1. Triton AoT Preparation - -OpenAI Triton offers an Ahead-of-Time (AoT) compilation tool to generate C files that wrap compiled GPU kernel. -To use the AoT feature, you need a Triton version posterior to the [d0c35b3](https://github.com/openai/triton/commit/d0c35b3b7d6badf0c0d56a821dddab7ace73b4de) commit -and this example has been tested on the [b43c28f](https://github.com/openai/triton/tree/b43c28fdd7a2f95b2e87180cba5d984732120d5c) commit. -```bash -git clone https://github.com/openai/triton -cd triton/python/ -git checkout d4644d6cb3ae674e1f15932cac1f28104795744f -pip install cmake && pip install . -cd - -``` - -For AoT compilation, it is necessary to provide a kernel signature and specify the values of `tl.constexpr` parameters in a comma-separated format. -Details can be found in the [compile.py](https://github.com/openai/triton/blob/main/python/triton/tools/compile.py) file in the Triton project. - -Here are examples of kernel AOT compilations for the [Fused Attention](fmha_triton.py) kernel. -```bash -# Kernel for data type=float16, BLOCK_M=128, BLOCK_DMODEL=64, BLOCK_N=128 -export TRITON_ROOT=$(pip show triton | grep Location | cut -d' ' -f2) -rm -rf aot -mkdir -p aot/fp16 -python ${TRITON_ROOT}/triton/tools/compile.py \ - fmha_triton.py \ - -n fused_attention_kernel \ - -o aot/fp16/fmha_kernel_d64_fp16 \ - --out-name fmha_d64_fp16 \ - -w 4 \ - -ns 2 \ - -s "*fp16:16, *fp32:16, *fp32:16, *fp16:16, *fp16:16, *fp16:16, fp32, i32, i32, i32, 128, 64, 128" \ - -g "(seq_len + 127) / 128, batch_size * num_heads, 1" -# Kernel for data type=float32, BLOCK_M=64, BLOCK_DMODEL=64, BLOCK_N=64 -mkdir -p aot/fp32 -python ${TRITON_ROOT}/triton/tools/compile.py \ - fmha_triton.py \ - -n fused_attention_kernel \ - -o aot/fp32/fmha_kernel_d64_fp32 \ - --out-name fmha_d64_fp32 \ - -w 4 \ - -ns 2 \ - -s "*fp32:16, *fp32:16, *fp32:16, *fp32:16, *fp32:16, *fp32:16, fp32, i32, i32, i32, 64, 64, 64" \ - -g "(seq_len + 63) / 64, batch_size * num_heads, 1" - -# Link generated headers and create dispatchers. -python ${TRITON_ROOT}/triton/tools/link.py aot/fp16/*.h -o aot/fmha_kernel_fp16 -python ${TRITON_ROOT}/triton/tools/link.py aot/fp32/*.h -o aot/fmha_kernel_fp32 -``` -The tool will generate .c and .h files to launch the GPU kernel. -Note that it is necessary to specify the kernel name using the --out-name option, it allows to define dispatcher names for the different data types. -The above invocations will generate `aot/fmha_kernel_{fp16|fp32}.{c|h}` files that contain three functions: - - the `load_fmha_d64_{fp16|fp32}` function to load the code of the GPU kernel, - - the `fmha_d64_{fp16|fp32}` function to launch the kernel, - - the `unload_fmha_d64_{fp16|fp32}` function to unload the GPU kernel. - -If GPU resources are limited, it is recommended to adjust the number of stages or warps accordingly. For example, on the V100, the aforementioned arguments might fail due to insufficient shared memory of the GPU. This can be mitigated by reducing the number of stages by one, using `-ns 1`. - - -## 2. Implement a Custom TensorRT Plugin - -This section describes how to implement a custom plugin for TensorRT LLM to execute the Triton kernel created in the previous section. -We provide an example of plugin implementation. - - TritonFlashAttentionPlugin([.cpp](TritonFlashAttentionPlugin.cpp), [.h](TritonFlashAttentionPlugin.h)): TensorRT plugin. - - [plugin.py](plugin.py): Python wrapper. - -`TritonFlashAttentionPlugin` is a TensorRT plugin that integrates a Triton kernel generated with the AoT compiler. -The `initialize` and `terminate` functions show how to initialize and terminate the TensorRT plugin. -The `enqueue` member function shows how to call the generated Triton kernel on the GPU. -Note that the name of the Triton kernel depends on the function's signature, meaning that different types or specialization leads a different kernel name. -Thus, if you change an option during AoT compilation like `-s `, you also have to update file names in CMakeLists.txt in order to match the names generated by the AoT compiler. - -To build a shared library for the custom Triton plugin, run: -```bash -mkdir -p build && cd build -cmake .. && make -cd .. -``` -As mentioned in the previous section, it is necessary to have the TensorRT LLM C++ runtime library. -If you want to specify the library paths, run: -```bash -cmake -DTRT_LIB_DIR= -DTRT_INCLUDE_DIR= -DTRT_LLM_LIB_DIR= .. -``` -If the build is successful, you should be able to find a shared library for the custom plugin at `build/libtrt_llm_custom_plugins.so`. - -A Python wrapper of the Fused Multihead Attention (FMHA) operator and the corresponding TensorRT LLM layer are implemented in [plugin.py](plugin.py). -It is similar to other TensorRT LLM operators and layers implemented in [functional.py](../../tensorrt_llm/functional.py) and [layers](../../tensorrt_llm/layers), respectively. -That FMHA operator uses the custom plugin that wraps the functions generated from the Triton kernel. - -## 3. Build and Run the TensorRT Engine - -We are now ready to build and run the TensorRT engine that uses the Triton kernel. -Here are the two commands to build and run the engine: -```bash -python build.py --num_heads 32 --head_size 64 --max_batch_size 8 --max_seq_len 512 --dtype float16 -python run.py --num_heads 32 --head_size 64 --batch_size 8 --seq_len 512 --log_level verbose --benchmark -``` - -## 4. Known Issues - -### 1. A generated dispatcher might not execute a kernel without raising an error due to a missing branch. - -The kernel dispatcher written by `link.py` has a missing branch, which can result in returning without executing a kernel. -For instance, in our example, the generated dispatcher looks like this: -```c++ -CUresult fmha_d64_fp16(CUstream stream, unsigned int gX, unsigned int gY, unsigned int gZ, CUdeviceptr Out, CUdeviceptr L, CUdeviceptr M, CUdeviceptr Q, CUdeviceptr K, CUdeviceptr V, float sm_scale, int32_t seq_len){ - if ((Out % 16 == 0) && (L % 16 == 0) && (M % 16 == 0) && (Q % 16 == 0) && (K % 16 == 0) && (V % 16 == 0)) - return fmha_d64_fp16_0eb6b090_0d1d2d3d4d5d67(stream, gX, gY, gZ, Out, L, M, Q, K, V, sm_scale, seq_len); -} -``` -It is recommended to manually update the generated functions by `link.py` to return a proper error for proper error handling. - - -### 2. The shared memory required by a generated kernel may exceed the hardware limitation. - -The AoT compiler does not verify the limitations of shared memory size during compilation time, which could potentially lead to the out-of-resource errors during runtime. -It would be helpful to verify if the requirement of the dynamic shared memory size in a generated kernel exceeds the hardware limitation. -You can find the number at the line of `cuLaunchKernel` call in the generated `.c` file. -For instance, the shared memory size is 114690 bytes in our example. -```c++ -CUresult fmha_d64_fp16_0eb6b090_0d1d2d3d4d5d67(CUstream stream, unsigned int gX, unsigned int gY, unsigned int gZ, CUdeviceptr Out, CUdeviceptr L, CUdeviceptr M, CUdeviceptr Q, CUdeviceptr K, CUdeviceptr V, float sm_scale, int32_t seq_len) { - if (fmha_d64_fp16_0eb6b090_0d1d2d3d4d5d67_func == NULL) - load_fmha_d64_fp16_0eb6b090_0d1d2d3d4d5d67(); - void *args[8] = { &Out, &L, &M, &Q, &K, &V, &sm_scale, &seq_len }; - // TODO: shared memory - if(gX * gY * gZ > 0) - return cuLaunchKernel(fmha_d64_fp16_0eb6b090_0d1d2d3d4d5d67_func, gX, gY, gZ, 4 * 32, 1, 1, 114690, stream, args, NULL); -} -``` -It may be resolved by reduing the block size. - - -### 3. AttributeError: module 'triton' has no attribute 'jit' - -This problem may arise if Triton is installed in editable mode. To resolve this issue, please install Triton using the non-editable mode. Refer https://github.com/openai/triton/issues/1693. - -### 4. Unload the same module more than once while building the engine -When the plugin is used more than once within a model, the function cuModuleUnload() will be invoked multiple times during the engine building stage. Related code is generated by Openai Triton and can be found in the folder `examples/openai_triton/manual_plugin/aot/`. One example is: - -```c++ -void unload_fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789(void) { - CUDA_CHECK(cuModuleUnload(fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789_mod)); -} -``` - -As the generated code didn't check the value of the module object, this function might unload the same module multiple times, which will cause an error as follows: - -``` -Triton Error [CUDA]: invalid resource handle\n/opt/rapids/src/cudf/cpp/build/_deps/arrow-src/cpp/src/arrow/filesystem/s3fs.cc:2904:  arrow::fs::FinalizeS3 was not called even though S3 was initialized.  This could lead to a segmentation fault at exit -``` - -The error message is ambiguous. If we use compute-sanitizer to help debug, we can get the following information: -``` -========= Program hit CUDA_ERROR_INVALID_HANDLE (error 400) due to "invalid resource handle" on CUDA API call to cuModuleUnload. -``` - -So we need to modify the above generated code as follows to avoid the above error. -```c++ -void unload_fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789(void) { - if(fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789_mod){ - CUDA_CHECK(cuModuleUnload(fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789_mod)); - } - fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789_mod=NULL; -} -``` diff --git a/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.cpp b/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.cpp deleted file mode 100644 index 198d8be1ca16..000000000000 --- a/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.cpp +++ /dev/null @@ -1,386 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "TritonFlashAttentionPlugin.h" - -// Import a generated header to use generated triton kernels. -extern "C" -{ -#include "aot/fmha_kernel_fp16.h" -#include "aot/fmha_kernel_fp32.h" -} - -#include -#include -#include -#include - -using namespace nvinfer1; -using openai_triton::plugin::TritonFlashAttentionPluginCreator; -using openai_triton::plugin::TritonFlashAttentionPlugin; - -static char const* TRITON_FLASH_ATTENTION_PLUGIN_VERSION{"1"}; -static char const* TRITON_FLASH_ATTENTION_PLUGIN_NAME{"TritonFlashAttention"}; -PluginFieldCollection TritonFlashAttentionPluginCreator::mFC{}; -std::vector TritonFlashAttentionPluginCreator::mPluginAttributes; - -namespace openai_triton::plugin -{ - -// Write values into buffer -template -void writeArg(char*& buffer, T const& val) -{ - std::memcpy(buffer, &val, sizeof(T)); - buffer += sizeof(T); -} - -// Read values from buffer -template -void readArg(char const*& buffer, T& val) -{ - std::memcpy(&val, buffer, sizeof(T)); - buffer += sizeof(T); -} - -std::uintptr_t constexpr kCudaMemAlign = 128; - -int8_t* nextWorkspacePtr(int8_t* ptr, uintptr_t previousWorkspaceSize) -{ - uintptr_t addr = (uintptr_t) ptr; - addr += previousWorkspaceSize; - if (addr % kCudaMemAlign) - { - addr += kCudaMemAlign - addr % kCudaMemAlign; - } - return (int8_t*) addr; -} - -TritonFlashAttentionPlugin::TritonFlashAttentionPlugin( - int numHeads, int headSize, float softmaxScale, nvinfer1::DataType type) - : mNumHeads(numHeads) - , mHeadSize(headSize) - , mSoftmaxScale(softmaxScale) - , mType(type) -{ -} - -// Parameterized constructor -TritonFlashAttentionPlugin::TritonFlashAttentionPlugin(void const* data, size_t length) -{ - char const *d = reinterpret_cast(data), *a = d; - readArg(d, mNumHeads); - readArg(d, mHeadSize); - readArg(d, mSoftmaxScale); - readArg(d, mType); - TLLM_CHECK(d == a + length); -} - -// IPluginV2DynamicExt Methods -nvinfer1::IPluginV2DynamicExt* TritonFlashAttentionPlugin::clone() const noexcept -{ - auto* plugin = new TritonFlashAttentionPlugin(*this); - plugin->setPluginNamespace(mNamespace.c_str()); - return plugin; -} - -nvinfer1::DimsExprs TritonFlashAttentionPlugin::getOutputDimensions( - int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept -{ - // Output shape. - // output tensor [batchSize, seqLen, mNumHeads, head_size] - assert(outputIndex == 0); - return inputs[outputIndex]; -} - -bool TritonFlashAttentionPlugin::supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept -{ - // In this example, inputs: Q, K, V, outputs: Out - assert(nbInputs + nbOutputs == 4); - assert(0 <= pos && pos < nbInputs + nbOutputs); - - bool is_valid = false; - if (0 <= pos && pos < 3) // Q, K, V - { - is_valid = inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - } - else if (pos == nbInputs) // Out - { - is_valid = inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; - } - return is_valid; -} - -void TritonFlashAttentionPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept -{ -} - -size_t TritonFlashAttentionPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept -{ - // Set workspace size if needed. In this example, we need for L and m buffers. - auto const Q = inputs[0]; - int const batchSize = Q.dims.d[0]; - int const seqLen = Q.dims.d[2]; - int const numBuffers = 2; - size_t workspaces[numBuffers]; - workspaces[0] = sizeof(float) * batchSize * mNumHeads * seqLen; - workspaces[1] = sizeof(float) * batchSize * mNumHeads * seqLen; - - size_t total = 0; - for (int i = 0; i < numBuffers; i++) - { - total += workspaces[i]; - if (workspaces[i] % kCudaMemAlign) - { - total += kCudaMemAlign - (workspaces[i] % kCudaMemAlign); - } - } - return total; -} - -template -int TritonFlashAttentionPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) -{ - assert(inputDesc[0].dims.d[1] == mNumHeads && inputDesc[0].dims.d[3] == mHeadSize); - assert(inputDesc[1].dims.d[1] == mNumHeads && inputDesc[1].dims.d[3] == mHeadSize); - assert(inputDesc[2].dims.d[1] == mNumHeads && inputDesc[2].dims.d[3] == mHeadSize); - - int batchSize = inputDesc[0].dims.d[0]; - int seqLen = inputDesc[0].dims.d[2]; - - T* Out = reinterpret_cast(outputs[0]); - - const size_t bufSize = sizeof(float) * batchSize * mNumHeads * seqLen; - float* L = reinterpret_cast(workspace); - float* M = reinterpret_cast(nextWorkspacePtr(reinterpret_cast(L), bufSize)); - - T const* Q = reinterpret_cast(inputs[0]); - T const* K = reinterpret_cast(inputs[1]); - T const* V = reinterpret_cast(inputs[2]); - - // Launch a cuda kernel generated by Triton AoT. - int res = 0; - if (std::is_same::value) - { - res = fmha_d64_fp32_default(stream, reinterpret_cast(Out), reinterpret_cast(L), - reinterpret_cast(M), reinterpret_cast(Q), reinterpret_cast(K), - reinterpret_cast(V), mSoftmaxScale, batchSize, mNumHeads, seqLen); - } - else - { - res = fmha_d64_fp16_default(stream, reinterpret_cast(Out), reinterpret_cast(L), - reinterpret_cast(M), reinterpret_cast(Q), reinterpret_cast(K), - reinterpret_cast(V), mSoftmaxScale, batchSize, mNumHeads, seqLen); - } - return res; -} - -int TritonFlashAttentionPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, - nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, - cudaStream_t stream) noexcept -{ - int res = 1; - if (mType == DataType::kHALF) - { - res = enqueueImpl(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - else if (mType == DataType::kFLOAT) - { - res = enqueueImpl(inputDesc, outputDesc, inputs, outputs, workspace, stream); - } - sync_check_cuda_error(); - return res; -} - -// IPluginV2Ext Methods -nvinfer1::DataType TritonFlashAttentionPlugin::getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept -{ - assert(index == 0); - return inputTypes[0]; -} - -// IPluginV2 Methods - -char const* TritonFlashAttentionPlugin::getPluginType() const noexcept -{ - return TRITON_FLASH_ATTENTION_PLUGIN_NAME; -} - -char const* TritonFlashAttentionPlugin::getPluginVersion() const noexcept -{ - return TRITON_FLASH_ATTENTION_PLUGIN_VERSION; -} - -int TritonFlashAttentionPlugin::getNbOutputs() const noexcept -{ - return 1; -} - -int TritonFlashAttentionPlugin::initialize() noexcept -{ - // Load kernels generated by Triton AoT. - load_fmha_d64_fp32(); - load_fmha_d64_fp16(); - return 0; -} - -void TritonFlashAttentionPlugin::terminate() noexcept -{ - // Unload kernels generated by Triton AoT. - unload_fmha_d64_fp32(); - unload_fmha_d64_fp16(); -} - -size_t TritonFlashAttentionPlugin::getSerializationSize() const noexcept -{ - return sizeof(mNumHeads) + sizeof(mHeadSize) + sizeof(mSoftmaxScale) + sizeof(mType); -} - -void TritonFlashAttentionPlugin::serialize(void* buffer) const noexcept -{ - char *d = static_cast(buffer), *a = d; - writeArg(d, mNumHeads); - writeArg(d, mHeadSize); - writeArg(d, mSoftmaxScale); - writeArg(d, mType); - TLLM_CHECK(d == a + getSerializationSize()); -} - -void TritonFlashAttentionPlugin::destroy() noexcept -{ - // This gets called when the network containing plugin is destroyed - delete this; -} - -void TritonFlashAttentionPlugin::setPluginNamespace(char const* libNamespace) noexcept -{ - mNamespace = libNamespace; -} - -char const* TritonFlashAttentionPlugin::getPluginNamespace() const noexcept -{ - return mNamespace.c_str(); -} - -/////////////// - -TritonFlashAttentionPluginCreator::TritonFlashAttentionPluginCreator() -{ - // Fill PluginFieldCollection with PluginField arguments metadata - mPluginAttributes.clear(); - mPluginAttributes.emplace_back(PluginField("num_heads", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("head_size", nullptr, PluginFieldType::kINT32)); - mPluginAttributes.emplace_back(PluginField("softmax_scale", nullptr, PluginFieldType::kFLOAT32)); - mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); - mFC.nbFields = mPluginAttributes.size(); - mFC.fields = mPluginAttributes.data(); -} - -char const* TritonFlashAttentionPluginCreator::getPluginName() const noexcept -{ - return TRITON_FLASH_ATTENTION_PLUGIN_NAME; -} - -char const* TritonFlashAttentionPluginCreator::getPluginVersion() const noexcept -{ - return TRITON_FLASH_ATTENTION_PLUGIN_VERSION; -} - -PluginFieldCollection const* TritonFlashAttentionPluginCreator::getFieldNames() noexcept -{ - return &mFC; -} - -IPluginV2* TritonFlashAttentionPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept -{ - PluginField const* fields = fc->fields; - int numHeads = 0; - int headSize = 0; - float softmaxScale = 1.0f; - nvinfer1::DataType type; - // Read configurations from each fields - for (int i = 0; i < fc->nbFields; ++i) - { - char const* attrName = fields[i].name; - if (!strcmp(attrName, "num_heads")) - { - assert(fields[i].type == PluginFieldType::kINT32); - numHeads = static_cast(*(static_cast(fields[i].data))); - } - else if (!strcmp(attrName, "head_size")) - { - assert(fields[i].type == PluginFieldType::kINT32); - headSize = static_cast(*(static_cast(fields[i].data))); - } - else if (!strcmp(attrName, "softmax_scale")) - { - assert(fields[i].type == PluginFieldType::kFLOAT32); - softmaxScale = static_cast(*(static_cast(fields[i].data))); - } - else if (!strcmp(attrName, "type_id")) - { - assert(fields[i].type == PluginFieldType::kINT32); - type = static_cast(*(static_cast(fields[i].data))); - } - } - try - { - auto* obj = new TritonFlashAttentionPlugin(numHeads, headSize, softmaxScale, type); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - std::cerr << "Caught exception: " << e.what() << std::endl; - } - return nullptr; -} - -IPluginV2* TritonFlashAttentionPluginCreator::deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept -{ - // This object will be deleted when the network is destroyed, which will - // call TritonFlashAttentionPlugin::destroy() - try - { - auto* obj = new TritonFlashAttentionPlugin(serialData, serialLength); - obj->setPluginNamespace(mNamespace.c_str()); - return obj; - } - catch (std::exception const& e) - { - std::cerr << "Caught exception: " << e.what() << std::endl; - } - return nullptr; -} - -void TritonFlashAttentionPluginCreator::setPluginNamespace(char const* libNamespace) noexcept -{ - mNamespace = libNamespace; -} - -char const* TritonFlashAttentionPluginCreator::getPluginNamespace() const noexcept -{ - return mNamespace.c_str(); -} - -} // namespace openai_triton::plugin diff --git a/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.h b/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.h deleted file mode 100644 index cd95eb48ec2a..000000000000 --- a/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.h +++ /dev/null @@ -1,113 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include - -#include -#include -#include -#include - -#include -#include - -namespace openai_triton::plugin -{ - -class TritonFlashAttentionPlugin : public nvinfer1::IPluginV2DynamicExt -{ -public: - TritonFlashAttentionPlugin(int numHeads, int headSize, float softmaxScale, nvinfer1::DataType type); - - TritonFlashAttentionPlugin(void const* data, size_t length); - - ~TritonFlashAttentionPlugin() override = default; - - // IPluginV2DynamicExt Methods - nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; - nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, - nvinfer1::IExprBuilder& exprBuilder) noexcept override; - bool supportsFormatCombination( - int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; - void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, - nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; - size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, - nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; - int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; - - template - int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, - void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); - - // IPluginV2Ext Methods - nvinfer1::DataType getOutputDataType( - int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; - - // IPluginV2 Methods - char const* getPluginType() const noexcept override; - char const* getPluginVersion() const noexcept override; - int getNbOutputs() const noexcept override; - int initialize() noexcept override; - void terminate() noexcept override; - size_t getSerializationSize() const noexcept override; - void serialize(void* buffer) const noexcept override; - void destroy() noexcept override; - void setPluginNamespace(char const* pluginNamespace) noexcept override; - char const* getPluginNamespace() const noexcept override; - -private: - const std::string mLayerName; - std::string mNamespace; - - int mNumHeads; - int mHeadSize; - float mSoftmaxScale; - nvinfer1::DataType mType; - - CUmodule mModule; - CUfunction mKernel; -}; - -class TritonFlashAttentionPluginCreator : public nvinfer1::IPluginCreator -{ -public: - TritonFlashAttentionPluginCreator(); - - char const* getPluginName() const noexcept override; - - char const* getPluginVersion() const noexcept override; - - nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; - - nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; - - nvinfer1::IPluginV2* deserializePlugin( - char const* name, void const* serialData, size_t serialLength) noexcept override; - - void setPluginNamespace(char const* pluginNamespace) noexcept override; - - char const* getPluginNamespace() const noexcept override; - -private: - static nvinfer1::PluginFieldCollection mFC; - static std::vector mPluginAttributes; - std::string mNamespace; -}; - -} // namespace openai_triton::plugin diff --git a/examples/openai_triton/manual_plugin/build.py b/examples/openai_triton/manual_plugin/build.py deleted file mode 100644 index 12b3ca883e7c..000000000000 --- a/examples/openai_triton/manual_plugin/build.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import math -import time -from pathlib import Path - -import tensorrt as trt -from plugin import LAYER_NAME, FmhaLayer, get_engine_name - -import tensorrt_llm -from tensorrt_llm.builder import Builder, BuilderConfig -from tensorrt_llm.logger import logger -from tensorrt_llm.network import net_guard - - -def build_engine(builder: Builder, builder_config: BuilderConfig, - engine_name: str, args: argparse.Namespace) -> trt.IHostMemory: - ''' - - @brief: Build a TensorRT engine. - @param args: The cmd line arguments. - @return: The built or refitted engine. - ''' - - # Initialize Module - softmax_scale = 1.0 / math.sqrt(args.head_size) - layer = FmhaLayer(args.num_heads, args.head_size, softmax_scale, args.dtype) - - # Module -> Network - network = builder.create_network() - network.trt_network.name = engine_name - network.plugin_config.to_legacy_setting() - with net_guard(network): - # Prepare - inputs = layer.prepare_inputs(args.max_batch_size, args.max_seq_len) - # Forward - logger.debug(f'model inputs: {inputs}') - out = layer(*inputs) - out.trt_tensor.name = 'out' - - # Network -> Engine - engine = builder.build_engine(network, builder_config) - config_path = Path(args.output_dir) / 'config.json' - builder.save_config(builder_config, str(config_path)) - return engine - - -def build(args): - tensorrt_llm.logger.set_level(args.log_level) - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - builder = Builder() - cache = None - builder_config = builder.create_builder_config( - name=LAYER_NAME, - precision=args.dtype, - timing_cache=args.timing_cache if cache is None else cache, - profiling_verbosity=args.profiling_verbosity) - - engine_name = get_engine_name(args.head_size, args.dtype) - engine = build_engine(builder, builder_config, engine_name, args) - assert engine is not None - - engine_path = output_dir / engine_name - logger.info(f'Serializing engine to {str(engine_path)}...') - tik = time.time() - with engine_path.open('wb') as f: - f.write(engine) - tok = time.time() - t = time.strftime('%H:%M:%S', time.gmtime(tok - tik)) - logger.info(f'Engine serialized. Total time: {t}') - - ok = builder.save_timing_cache(builder_config, - Path(args.output_dir) / "model.cache") - assert ok, "Failed to save timing cache." - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--max_batch_size', type=int, default=4) - parser.add_argument('--max_seq_len', type=int, default=256) - parser.add_argument('--num_heads', type=int, default=8) - parser.add_argument('--head_size', type=int, default=64) - parser.add_argument('--dtype', - type=str, - default='float16', - choices=['float16', 'float32']) - parser.add_argument( - '--timing_cache', - type=str, - default='model.cache', - help='The path of to read timing cache from, will be ignored ' - 'if the file does not exist') - parser.add_argument( - '--profiling_verbosity', - type=str, - default='layer_names_only', - choices=['layer_names_only', 'detailed', 'none'], - help= - 'The profiling verbosity for the generated TRT engine. Set to detailed can inspect tactic choices and kernel parameters.' - ) - parser.add_argument('--log_level', type=str, default='info') - parser.add_argument( - '--output_dir', - type=str, - default='outputs', - help='The path to save the serialized engine files, timing cache ' - 'file and model configs') - args = parser.parse_args() - - logger.set_level(args.log_level) - logger.info('Parameters'.center(40, '=')) - for k, v in vars(args).items(): - logger.info(f' - {k.ljust(15, ".")}: {v}') - logger.info(''.center(40, '=')) - - tik = time.time() - logger.info('Build TensorRT engine.') - build(args) - tok = time.time() - t = time.strftime('%H:%M:%S', time.gmtime(tok - tik)) - logger.info(f'Total time of building TRT engine: {t}') diff --git a/examples/openai_triton/manual_plugin/fmha_triton.py b/examples/openai_triton/manual_plugin/fmha_triton.py deleted file mode 100644 index 3e47dff263f0..000000000000 --- a/examples/openai_triton/manual_plugin/fmha_triton.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Fused attention from triton tutorial. -Modified from the original implementation - - https://github.com/openai/triton/blob/main/python/tutorials/06-fused-attention.py -=============== - -This is a Triton implementation of the Flash Attention algorithm -(see: Dao et al., https://arxiv.org/pdf/2205.14135v2.pdf; Rabe and Staats https://arxiv.org/pdf/2112.05682v2.pdf) -""" - -import torch -import triton -import triton.language as tl - - -# yapf: disable -@triton.jit -def fused_attention_kernel( - Out, L, M, # outputs - Q, K, V, - sm_scale, - batch_size, num_heads, seq_len, - BLOCK_M: tl.constexpr, BLOCK_DMODEL: tl.constexpr, - BLOCK_N: tl.constexpr, -): - start_m = tl.program_id(0) - off_hz = tl.program_id(1) - stride_h = BLOCK_DMODEL * seq_len - - # initialize offsets - offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) - offs_n = tl.arange(0, BLOCK_N) - offs_d = tl.arange(0, BLOCK_DMODEL) - off_q = off_hz * stride_h + offs_m[:, None] * BLOCK_DMODEL + offs_d[None, :] - off_k = off_hz * stride_h + offs_n[None, :] * BLOCK_DMODEL + offs_d[:, None] - off_v = off_hz * stride_h + offs_n[:, None] * BLOCK_DMODEL + offs_d[None, :] - # Initialize pointers to Q, K, V - q_ptrs = Q + off_q - k_ptrs = K + off_k - v_ptrs = V + off_v - # initialize pointer to m and l - m_prev = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") - l_prev = tl.zeros([BLOCK_M], dtype=tl.float32) - acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32) - # load q: it will stay in SRAM throughout - q = tl.load(q_ptrs) - # loop over k, v and update accumulator - for start_n in range(0, (start_m + 1) * BLOCK_M, BLOCK_N): - # -- compute qk ---- - k = tl.load(k_ptrs) - qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) - qk += tl.dot(q, k) - qk *= sm_scale - qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk, float("-inf")) - # compute new m - m_curr = tl.maximum(tl.max(qk, 1), m_prev) - # correct old l - l_prev *= tl.exp(m_prev - m_curr) - # attention weights - p = tl.exp(qk - m_curr[:, None]) - l_curr = tl.sum(p, 1) + l_prev - # rescale operands of matmuls - l_rcp = 1. / l_curr - p *= l_rcp[:, None] - acc *= (l_prev * l_rcp)[:, None] - # update acc - p = p.to(Q.dtype.element_ty) - v = tl.load(v_ptrs) - acc += tl.dot(p, v) - # update m_i and l_i - l_prev = l_curr - m_prev = m_curr - # update pointers - k_ptrs += BLOCK_N * BLOCK_DMODEL - v_ptrs += BLOCK_N * BLOCK_DMODEL - # rematerialize offsets to save registers - start_m = tl.program_id(0) - offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) - # write back l and m - l_ptrs = L + off_hz * seq_len + offs_m - m_ptrs = M + off_hz * seq_len + offs_m - tl.store(l_ptrs, l_prev) - tl.store(m_ptrs, m_prev) - # initialize pointers to output - offs_n = tl.arange(0, BLOCK_DMODEL) - off_o = off_hz * stride_h + offs_m[:, None] * BLOCK_DMODEL + offs_n[None, :] - out_ptrs = Out + off_o - tl.store(out_ptrs, acc) - - -def fused_attention(q, k, v, sm_scale, o_buf=None, l_buf=None, m_buf=None): - BLOCK = 128 if q.dtype == torch.float16 else 64 - # shape constraints - Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1] - assert Lq == Lk and Lk == Lv - assert Lk in {16, 32, 64, 128} - o = torch.empty_like(q) if o_buf is None else o_buf - grid = (triton.cdiv(q.shape[2], BLOCK), q.shape[0] * q.shape[1], 1) - shape = (q.shape[0] * q.shape[1], q.shape[2]) - L = torch.empty(shape, device=q.device, dtype=torch.float32) if l_buf is None else l_buf - m = torch.empty(shape, device=q.device, dtype=torch.float32) if m_buf is None else m_buf - - num_warps = 4 if Lk <= 64 else 8 - # Adjust num_stages for limited resource cases. - num_stages = 2 if torch.cuda.get_device_capability() >= (8, 0) else 1 - - fused_attention_kernel[grid]( - o, L, m, - q, k, v, - sm_scale, - q.shape[0], q.shape[1], q.shape[2], - # tl.constexpr - BLOCK_M=BLOCK, - BLOCK_N=BLOCK, - BLOCK_DMODEL=Lk, - num_warps=num_warps, - num_stages=num_stages, - ) - - return o -# yapf: enable diff --git a/examples/openai_triton/manual_plugin/plugin.py b/examples/openai_triton/manual_plugin/plugin.py deleted file mode 100644 index 7009caaeb680..000000000000 --- a/examples/openai_triton/manual_plugin/plugin.py +++ /dev/null @@ -1,133 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import ctypes -from collections import OrderedDict -from pathlib import Path -from typing import List - -import numpy as np -import tensorrt as trt - -from tensorrt_llm._common import default_trtnet -from tensorrt_llm._utils import str_dtype_to_trt -from tensorrt_llm.functional import Tensor, _create_tensor -from tensorrt_llm.module import Module - -TRT_LLM_PLUGIN_NAMESPACE = 'tensorrt_llm' -LAYER_NAME = 'TritonFlashAttentionLayer' -FMHA_KERNEL_BLOCK_SIZE = 128 - - -def _load_triton_plugin_lib(): - triton_plugin_dir = Path(__file__).parent.absolute() - plugin_lib = triton_plugin_dir / 'build/libtrt_llm_custom_plugins.so' - handle = ctypes.CDLL(plugin_lib, mode=ctypes.RTLD_GLOBAL) - if handle is None: - raise ImportError('TensorRT LLM Triton Plugin is unavailable') - handle.initOpenAiTritonPlugins.argtypes = [ctypes.c_void_p, ctypes.c_char_p] - handle.initOpenAiTritonPlugins.restype = ctypes.c_bool - assert handle.initOpenAiTritonPlugins( - None, TRT_LLM_PLUGIN_NAMESPACE.encode('utf-8')) - - -_load_triton_plugin_lib() - - -def flash_attention_op(num_heads: int, head_size: int, softmax_scale: float, - inputs: List[trt.ITensor]) -> Tensor: - # Create a plugin instance. - plugin_creator = trt.get_plugin_registry().get_plugin_creator( - 'TritonFlashAttention', '1', TRT_LLM_PLUGIN_NAMESPACE) - assert plugin_creator is not None - - pfc = trt.PluginFieldCollection([ - trt.PluginField("num_heads", np.array([num_heads], np.int32), - trt.PluginFieldType.INT32), - trt.PluginField("head_size", np.array([head_size], np.int32), - trt.PluginFieldType.INT32), - trt.PluginField("softmax_scale", np.array([softmax_scale], np.float32), - trt.PluginFieldType.FLOAT32), - trt.PluginField("type_id", np.array([int(inputs[0].dtype)], np.int32), - trt.PluginFieldType.INT32) - ]) - plugin = plugin_creator.create_plugin("flash_attention", pfc) - layer = default_trtnet().add_plugin_v2(inputs, plugin) - return _create_tensor(layer.get_output(0), layer) - - -class FmhaLayer(Module): - - def __init__(self, num_heads: int, head_size: int, softmax_scale: float, - dtype: str): - super().__init__() - self.num_heads = num_heads - self.head_size = head_size - self.softmax_scale = softmax_scale - self.dtype = str_dtype_to_trt(dtype) - - def forward(self, Q: Tensor, K: Tensor, V: Tensor): - inputs = [Q, K, V] - out = flash_attention_op(num_heads=self.num_heads, - head_size=self.head_size, - softmax_scale=self.softmax_scale, - inputs=[p.trt_tensor for p in inputs]) - out.mark_output('out', self.dtype) - return out - - def prepare_inputs(self, max_batch_size: int, max_len: int) -> List[Tensor]: - ''' - - @brief: Prepare inputs Tensors for the model, the given sizes are used to - determine the ranges of the dimensions of when using TRT dynamic shapes. - - @return: a list contains values which can be fed into the self.forward() - ''' - - bs_range = [1, (max_batch_size + 1) // 2, max_batch_size] - max_len_range = [1, (max_len + 1) // 2, max_len] - - dynamic_shape = [-1, self.num_heads, -1, self.head_size] - Q = Tensor(name='Q', - dtype=self.dtype, - shape=dynamic_shape, - dim_range=OrderedDict([ - ('batch_size', [bs_range]), - ('num_heads', [self.num_heads]), - ('seq_len', [max_len_range]), - ('head_size', [self.head_size]), - ])) - K = Tensor(name='K', - dtype=self.dtype, - shape=dynamic_shape, - dim_range=OrderedDict([ - ('batch_size', [bs_range]), - ('num_heads', [self.num_heads]), - ('seq_len', [max_len_range]), - ('head_size', [self.head_size]), - ])) - V = Tensor(name='V', - dtype=self.dtype, - shape=dynamic_shape, - dim_range=OrderedDict([ - ('batch_size', [bs_range]), - ('num_heads', [self.num_heads]), - ('seq_len', [max_len_range]), - ('head_size', [self.head_size]), - ])) - return [Q, K, V] - - -def get_engine_name(head_size, dtype): - return f'{LAYER_NAME}_{FMHA_KERNEL_BLOCK_SIZE}_d{head_size}_{dtype}.engine' diff --git a/examples/openai_triton/manual_plugin/run.py b/examples/openai_triton/manual_plugin/run.py deleted file mode 100644 index ec7cf4dd5600..000000000000 --- a/examples/openai_triton/manual_plugin/run.py +++ /dev/null @@ -1,170 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import math -from pathlib import Path - -import torch -from fmha_triton import fused_attention -from plugin import get_engine_name - -from tensorrt_llm import profiler -from tensorrt_llm._deprecation import emit_engine_arch_deprecation -from tensorrt_llm._utils import (str_dtype_to_torch, str_dtype_to_trt, - trt_dtype_to_torch) -from tensorrt_llm.logger import logger -from tensorrt_llm.runtime.session import Session, TensorInfo - - -def run(engine_dir, - batch_size, - seq_len, - num_heads, - head_size, - do_benchmark=False): - # Load trt engine. - engine_dir = Path(engine_dir) - config_path = engine_dir / 'config.json' - with config_path.open('r') as f: - config = json.load(f) - dtype = config['builder_config']['precision'] - serialize_path = engine_dir / get_engine_name(head_size, dtype) - - with open(serialize_path, 'rb') as f: - session = Session.from_serialized_engine(f.read()) - - # Prepare input tensors. - torch_dtype = str_dtype_to_torch(dtype) if isinstance(dtype, str) else dtype - shape = (batch_size, num_heads, seq_len, head_size) - q = torch.normal(mean=0.1, - std=0.2, - size=shape, - dtype=torch_dtype, - device='cuda') - k = torch.normal(mean=0.4, - std=0.2, - size=shape, - dtype=torch_dtype, - device='cuda') - v = torch.normal(mean=0.3, - std=0.2, - size=shape, - dtype=torch_dtype, - device='cuda') - inputs = {'Q': q, 'K': k, 'V': v} - - # Prepare output tensors. - output_info = session.infer_shapes([ - TensorInfo(name, str_dtype_to_trt(dtype), tensor.shape) - for name, tensor in inputs.items() - ]) - logger.debug(f'output info {output_info}') - outputs = { - t.name: - torch.empty(tuple(t.shape), - dtype=trt_dtype_to_torch(t.dtype), - device='cuda') - for t in output_info - } - - # Execute model inference - stream = torch.cuda.Stream() - ok = session.run(inputs=inputs, outputs=outputs, stream=stream.cuda_stream) - assert ok, 'Engine execution failed' - - # Sanity check - stream.synchronize() - sm_scale = 1.0 / math.sqrt(head_size) - ref = fused_attention(q, k, v, sm_scale) - out = outputs["out"] - logger.debug( - f'Out: vals: {out.view(1, -1)} abs_sum: {out.float().abs().sum()}') - logger.debug( - f'Ref: vals: {ref.view(1, -1)} abs_sum: {ref.float().abs().sum()}') - torch.testing.assert_close(out, ref) - - if do_benchmark: - n_repeats = 10 - - # For fair comparison, pre-allocate buffers as trt plugin does. - shape = (q.shape[0] * q.shape[1], q.shape[2]) - L = torch.empty(shape, device=q.device, dtype=torch.float32) - m = torch.empty(shape, device=q.device, dtype=torch.float32) - o = torch.empty_like(q) - - # Triton warm-up - fused_attention(q, k, v, sm_scale, l_buf=L, m_buf=m, o_buf=o) - stream.synchronize() - for _ in range(n_repeats): - profiler.start('Triton') - fused_attention(q, k, v, sm_scale, l_buf=L, m_buf=m, o_buf=o) - stream.synchronize() - profiler.stop('Triton') - - # TRT warm-up - stream.synchronize() - ok = session.run(inputs=inputs, - outputs=outputs, - stream=stream.cuda_stream) - stream.synchronize() - for _ in range(n_repeats): - profiler.start('TRT Plugin') - ok = session.run(inputs=inputs, - outputs=outputs, - stream=stream.cuda_stream) - stream.synchronize() - profiler.stop('TRT Plugin') - assert ok - profiler.summary() - - -if __name__ == '__main__': - emit_engine_arch_deprecation("run.py") - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--batch_size', type=int, default=4) - parser.add_argument('--seq_len', type=int, default=128) - parser.add_argument('--num_heads', type=int, default=8) - parser.add_argument('--head_size', type=int, default=64) - parser.add_argument('--log_level', type=str, default='info') - parser.add_argument( - '--engine_dir', - type=Path, - default='outputs', - help='The directory where serialized engine files locate.') - parser.add_argument( - '--benchmark', - action='store_true', - help='Do performance benchmark compared to triton baseline.') - args = parser.parse_args() - - logger.set_level(args.log_level) - logger.info('Parameters'.center(40, '=')) - for k, v in vars(args).items(): - logger.info(f' - {k.ljust(15, ".")}: {v}') - logger.info(''.center(40, '=')) - - assert args.engine_dir.exists(), \ - f"Engine file {str(args.engine_dir)} doesn't exists." - - logger.info('Inference using the built TensorRT engine.') - run(args.engine_dir, - args.batch_size, - args.seq_len, - args.num_heads, - args.head_size, - do_benchmark=args.benchmark) - logger.info('Done.') diff --git a/examples/openai_triton/manual_plugin/tritonPlugins.cpp b/examples/openai_triton/manual_plugin/tritonPlugins.cpp deleted file mode 100644 index 27b1ece08448..000000000000 --- a/examples/openai_triton/manual_plugin/tritonPlugins.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "NvInferRuntime.h" -#include "TritonFlashAttentionPlugin.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace -{ - -// This singleton ensures that each plugin is only registered once for a given -// namespace and type, and attempts of duplicate registration are ignored. -class TritonPluginCreatorRegistry -{ -public: - static TritonPluginCreatorRegistry& getInstance() - { - static TritonPluginCreatorRegistry instance; - return instance; - } - - template - void addPluginCreator(void* logger, char const* libNamespace) - { - // Make accesses to the plugin creator registry thread safe - std::lock_guard lock(mRegistryLock); - - std::string errorMsg; - std::string verboseMsg; - - std::unique_ptr pluginCreator{new CreatorType{}}; - pluginCreator->setPluginNamespace(libNamespace); - - nvinfer1::ILogger* trtLogger = static_cast(logger); - std::string pluginType = std::string{pluginCreator->getPluginNamespace()} - + "::" + std::string{pluginCreator->getPluginName()} + " version " - + std::string{pluginCreator->getPluginVersion()}; - - if (mRegistryList.find(pluginType) == mRegistryList.end()) - { - bool status = getPluginRegistry()->registerCreator(*pluginCreator, libNamespace); - if (status) - { - mRegistry.push(std::move(pluginCreator)); - mRegistryList.insert(pluginType); - verboseMsg = "Registered plugin creator - " + pluginType; - } - else - { - errorMsg = "Could not register plugin creator - " + pluginType; - } - } - else - { - verboseMsg = "Plugin creator already registered - " + pluginType; - } - - if (trtLogger) - { - if (!errorMsg.empty()) - { - trtLogger->log(nvinfer1::ILogger::Severity::kERROR, errorMsg.c_str()); - } - - if (!verboseMsg.empty()) - { - trtLogger->log(nvinfer1::ILogger::Severity::kVERBOSE, verboseMsg.c_str()); - } - } - } - - ~TritonPluginCreatorRegistry() - { - std::lock_guard lock(mRegistryLock); - - // Release pluginCreators in LIFO order of registration. - while (!mRegistry.empty()) - { - mRegistry.pop(); - } - mRegistryList.clear(); - } - -private: - TritonPluginCreatorRegistry() {} - - std::mutex mRegistryLock; - std::stack> mRegistry; - std::unordered_set mRegistryList; - -public: - TritonPluginCreatorRegistry(TritonPluginCreatorRegistry const&) = delete; - void operator=(TritonPluginCreatorRegistry const&) = delete; -}; - -template -void initializeTritonPlugin(void* logger, char const* libNamespace) -{ - TritonPluginCreatorRegistry::getInstance().addPluginCreator(logger, libNamespace); -} - -} // namespace - -// New Plugin APIs - -extern "C" -{ - bool initOpenAiTritonPlugins(void* logger, char const* libNamespace) - { - initializeTritonPlugin(logger, libNamespace); - return true; - } -} // extern "C" diff --git a/examples/openai_triton/plugin_autogen/README.md b/examples/openai_triton/plugin_autogen/README.md deleted file mode 100644 index 0c046330fa0b..000000000000 --- a/examples/openai_triton/plugin_autogen/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Integrating Triton Kernel with TensorRT Plugin Generator - -In the previous [OpenAI Triton Plugin in TensorRT-LLM](../../openai_triton/README.md) tutorial, it is demonstrated how to integrate a Triton kernel by manually writing a TensorRT plugin in C++ as well as a Python wrapper. In the latest TensorRT-LLM, we now have an end-to-end tool called PluginGen that simplifies this process. All you need to do is providing a plugin configuration. - -In this example, we will introduce the usage of the PluginGen tool and demonstrate the integration of the [Fused Attention](../openai_triton/fmha_triton.py) kernel. - - -To use the feature, you need a Triton version posterior to the [d0c35b3](https://github.com/openai/triton/commit/d0c35b3b7d6badf0c0d56a821dddab7ace73b4de) commit -and this example has been tested on the [d4644d6](https://github.com/openai/triton/tree/d4644d6cb3ae674e1f15932cac1f28104795744f) commit. - -## Introduction to the PluginGen Toolkit - -The PluginGen script can be found at `tensorrt_llm/tools/triton_integration/plugin_gen.py`. Its usage is as follows: - -```sh -usage: plugin_gen.py [-h] --workspace WORKSPACE --kernel_config KERNEL_CONFIG [--tensorrt_llm_include_path TENSORRT_LLM_INCLUDE_PATH] -``` - -There are three command-line arguments: - -1. `workspace`: This is the root directory to hold the temporary generation files. PluginGen should not alter anything outside of the workspace, -2. `kernel_config`: This is a Python file that holds a variable called `KERNELS` of type `List[KernelMetaData]`. PluginGen can process one or more kernels at a time, -3. `tensorrt_llm_include_path`: This is the path to the TensorRT LLM include directory. It is used to include the TensorRT LLM header files in the generated plugin. - -You can refer to [./kernel_config.py](./kernel_config.py) for an example of `KernelMetaData` for the Fused Attention kernel. It contains several fields: - -1. `ios` (short for "input and outputs"): This holds all the metadata of the inputs and outputs of the Triton kernel, including the data type, shape, and the name of the tensor. There are several kinds of arguments: - - `InputArg`: A common variable for this kernel. - - `OutputArg`: An output of the kernel. - - `ParamArg`: A special input that is a constant; it will be mapped to a PluginField in the generated plugin. - - `DimSizeArg`: A special input that is an expression of the input tensors' shape size; it requires an inference rule to compute the value. -2. `shape_infer_rules`: This field contains two types of rules: - a) Rules for deducing the shape of the output tensors from the input tensors. The syntax is like `input0[dim_names], input1[dim_names] -> output0[dim_names]`. - b) Rules for inferring `DimSizeArg`. The syntax is like `input0[dim_names]: some_dim_expression -> arg_name`. - -The user should provide the kernel configurations as well as the Triton kernel script, and the PluginGen toolkit will handle the following steps: - -1. Trigger the Triton AOT tool to obtain the necessary C files. -2. Generate the C++ code for a TensorRT plugin. -3. Generate the CMAKE code for compiling all the C/C++ files. -4. Perform the compilation and generate `libtriton_plugins.so`. -5. Generate a `functional.py` containing a Python wrapper for this plugin. - -After the generation, you should have `libtriton_plugins.so` and `functional.py` in the workspace. You can use them to integrate the Triton kernel by simply using the corresponding Python methods in the generated `functional.py` during the model-building stage, just like other layers located in the TensorRT LLM built-in `functional.py`. - -## End-to-End Example for FHMA Kernel Integration - -In this section, we will demonstrate the integration of the Fused Attention kernel. The steps are as follows: - -### Pre-Stage: Install Triton with a Specific Version - -In case the Triton AOT tool's update breaks compatibility, we recommend installing a specific version of Triton. The commit we tested is [d4644d6](https://github.com/openai/triton/tree/d4644d6cb3ae674e1f15932cac1f28104795744f). - -Install Triton with the following commands: - -```sh -git clone https://github.com/openai/triton -cd triton/python/ -pip install cmake && pip install . -cd - -``` - -### Step 1: Prepare the Configuration for FHMA - -To instruct the PluginGen toolkit on how to generate the plugin, please provide a Python file containing the metadata of the kernels. You can refer to [./kernel_config.py](./kernel_config.py) for an example of preparing `KernelMetaData` for the Fused Attention kernel. - -### Step 2: Run the PluginGen Tool and Generate the Plugin - -```sh -python3 {GIT_ROOT_DIR}/tensorrt_llm/tools/plugin_gen/plugin_gen.py --workspace ./tmp --kernel_config ./kernel_config.py -``` - -PluginGen will generate all the necessary files within the `./tmp` directory. The final output will be located in the `./tmp/output` directory, where you should ideally find two files: - -``` --rw-r--r-- 1 1001 1001 2163 Sep 21 17:13 functional.py --rwxr-xr-x 1 1001 1001 3748464 Sep 21 17:13 libtriton_plugins.so -``` - -### Post-Stage: Use the Plugin - -To use the plugin in a TensorRT LLM model, please refer to the generated `output/functional.py`. It should contain Python wrappers for all the plugins. To use the plugins, first import `functional.py` and then use the corresponding Python methods to build the model. - -For an example of using the Fused Attention plugin in a model, please refer to [build_engine.py](./build_engine.py) for building the TensorRT engine and [run_engine.py](./run_engine.py) for running the engine in the runtime. - -To run the example, you can use the following commands: - -```sh -# copy the triton script to the current directory -cp ../manual_plugin/fmha_triton.py . - -# build the TensorRT engine -python3 build_engine.py - -# run the engine -python3 run_engine.py -``` diff --git a/examples/openai_triton/plugin_autogen/build_engine.py b/examples/openai_triton/plugin_autogen/build_engine.py deleted file mode 100644 index 23b829f0b7e4..000000000000 --- a/examples/openai_triton/plugin_autogen/build_engine.py +++ /dev/null @@ -1,206 +0,0 @@ -import argparse -import math -# include plugins -# yapf: disable -import os -import sys -import time -from pathlib import Path -from typing import List, OrderedDict - -import tensorrt as trt - -# from plugin import LAYER_NAME, FmhaLayer, get_engine_name -import tensorrt_llm -from tensorrt_llm import Module, str_dtype_to_trt -from tensorrt_llm.builder import Builder, BuilderConfig -from tensorrt_llm.functional import Tensor -from tensorrt_llm.logger import logger -from tensorrt_llm.network import net_guard - -sys.path.append(os.environ.get('PLUGIN_GEN_WORKSPACE', './tmp')) -from functional import fused_attention_kernel # isort:skip -# yapf: enable - - -def get_engine_name(head_size: int, dtype: str) -> str: - return f'fmha_{head_size}_{dtype}.engine' - - -class FmhaLayer(Module): - - def __init__(self, num_heads: int, head_size: int, softmax_scale: float): - super().__init__() - self.num_heads = num_heads - self.head_size = head_size - self.softmax_scale = softmax_scale - self.dtype = str_dtype_to_trt('float16') - - def forward(self, Q: Tensor, K: Tensor, V: Tensor): - inputs = [Q, K, V] - Out, L, M = fused_attention_kernel(self.softmax_scale, self.num_heads, - *[p.trt_tensor for p in inputs]) - Out.mark_output('out', self.dtype) - L.mark_output('L', self.dtype) - M.mark_output('M', self.dtype) - return Out, L, M - - def prepare_inputs(self, max_batch_size: int, max_len: int) -> List[Tensor]: - ''' - - @brief: Prepare inputs Tensors for the model, the given sizes are used to - determine the ranges of the dimensions of when using TRT dynamic shapes. - - @return: a list contains values which can be fed into the self.forward() - ''' - - bs_range = [1, (max_batch_size + 1) // 2, max_batch_size] - max_len_range = [1, (max_len + 1) // 2, max_len] - - dynamic_shape = [-1, self.num_heads, -1, self.head_size] - Q = Tensor(name='Q', - dtype=trt.float16, - shape=dynamic_shape, - dim_range=OrderedDict([ - ('batch_size', [bs_range]), - ('num_heads', [self.num_heads]), - ('seq_len', [max_len_range]), - ('head_size', [self.head_size]), - ])) - K = Tensor(name='K', - dtype=trt.float16, - shape=dynamic_shape, - dim_range=OrderedDict([ - ('batch_size', [bs_range]), - ('num_heads', [self.num_heads]), - ('seq_len', [max_len_range]), - ('head_size', [self.head_size]), - ])) - V = Tensor(name='V', - dtype=trt.float16, - shape=dynamic_shape, - dim_range=OrderedDict([ - ('batch_size', [bs_range]), - ('num_heads', [self.num_heads]), - ('seq_len', [max_len_range]), - ('head_size', [self.head_size]), - ])) - return [Q, K, V] - - -def build_engine(builder: Builder, builder_config: BuilderConfig, - engine_name: str, args: argparse.Namespace) -> trt.IHostMemory: - ''' - @brief: Build a TensorRT engine. - @param args: The cmd line arguments. - @return: The built or refitted engine. - ''' - - # Initialize Module - softmax_scale = 1.0 / math.sqrt(args.head_size) - layer = FmhaLayer(args.num_heads, args.head_size, softmax_scale) - - # Module -> Network - network = builder.create_network() - network.trt_network.name = engine_name - network.plugin_config.to_legacy_setting() - with net_guard(network): - # Prepare - inputs = layer.prepare_inputs(args.max_batch_size, args.max_seq_len) - # Forward - logger.debug(f'model inputs: {inputs}') - layer(*inputs) - - print('dot:') - print(network.to_dot()) - - layer = network.get_layer_by_name(next( - network.get_layers()).name).as_layer() - print('layer', layer.plugin.plugin_type) - print('layer', layer.plugin.plugin_version) - print('layer', layer.plugin.plugin_namespace) - - # Network -> Engine - engine = builder.build_engine(network, builder_config) - config_path = Path(args.output_dir) / 'config.json' - builder.save_config(builder_config, str(config_path)) - return engine - - -def build(args): - tensorrt_llm.logger.set_level(args.log_level) - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - builder = Builder() - cache = None - builder_config = builder.create_builder_config( - name='fmha_triton', - precision=args.dtype, - timing_cache=args.timing_cache if cache is None else cache, - profiling_verbosity=args.profiling_verbosity) - - engine_name = get_engine_name(args.head_size, args.dtype) - engine = build_engine(builder, builder_config, engine_name, args) - assert engine is not None - - engine_path = output_dir / engine_name - logger.info(f'Serializing engine to {str(engine_path)}...') - tik = time.time() - with engine_path.open('wb') as f: - f.write(engine) - tok = time.time() - t = time.strftime('%H:%M:%S', time.gmtime(tok - tik)) - logger.info(f'Engine serialized. Total time: {t}') - - ok = builder.save_timing_cache(builder_config, - Path(args.output_dir) / "model.cache") - assert ok, "Failed to save timing cache." - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--max_batch_size', type=int, default=4) - parser.add_argument('--max_seq_len', type=int, default=256) - parser.add_argument('--num_heads', type=int, default=8) - parser.add_argument('--head_size', type=int, default=64) - parser.add_argument('--dtype', - type=str, - default='float16', - choices=['float16', 'float32']) - parser.add_argument( - '--timing_cache', - type=str, - default='model.cache', - help='The path of to read timing cache from, will be ignored ' - 'if the file does not exist') - parser.add_argument( - '--profiling_verbosity', - type=str, - default='layer_names_only', - choices=['layer_names_only', 'detailed', 'none'], - help= - 'The profiling verbosity for the generated TRT engine. Set to detailed can inspect tactic choices and kernel parameters.' - ) - parser.add_argument('--log_level', type=str, default='info') - parser.add_argument( - '--output_dir', - type=str, - default='outputs', - help='The path to save the serialized engine files, timing cache ' - 'file and model configs') - args = parser.parse_args() - - logger.set_level(args.log_level) - logger.info('Parameters'.center(40, '=')) - for k, v in vars(args).items(): - logger.info(f' - {k.ljust(15, ".")}: {v}') - logger.info(''.center(40, '=')) - - tik = time.time() - logger.info('Build TensorRT engine.') - build(args) - tok = time.time() - t = time.strftime('%H:%M:%S', time.gmtime(tok - tik)) - logger.info(f'Total time of building TRT engine: {t}') diff --git a/examples/openai_triton/plugin_autogen/kernel_config.py b/examples/openai_triton/plugin_autogen/kernel_config.py deleted file mode 100644 index 93d64c187d61..000000000000 --- a/examples/openai_triton/plugin_autogen/kernel_config.py +++ /dev/null @@ -1,55 +0,0 @@ -import os - -import torch - -from tensorrt_llm.tools.plugin_gen.core import * - -openai_triton_example_root = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "..", "manual_plugin") - - -def get_fmha_kernel_meta_data(): - block_size = 128 - num_stages = 2 if torch.cuda.get_device_capability() >= (8, 0) else 1 - - return KernelMetaData( - kernel_name='fused_attention_kernel', - ios=[ - # outputs - OutputArg('Out', Type('tensor[fp16]'), hints=['16', '16']), - OutputArg('L', Type('tensor[fp32]'), hints=['16', '16']), - OutputArg('M', Type('tensor[fp32]'), hints=['16', '16']), - # inputs - InputArg('Q', Type('tensor[fp16]'), hints=['16', '16']), - InputArg('K', Type('tensor[fp16]'), hints=['16', '16']), - InputArg('V', Type('tensor[fp16]'), hints=['16', '16']), - ParamArg('sm_scale', Type('fp32')), - DimSizeArg('batch_size'), - ParamArg('num_heads', Type('i32')), - DimSizeArg('seq_len', hints=['', '16']), - # constexprs - Constexpr(block_size), - Constexpr(64), - Constexpr(block_size), - ], - shape_infer_rules=[ - # The following rules helps to deduce the shapes of the output tensors - "Q[*] -> Out[*]", - "Q[m,n,k,*] -> L[m,n,k]", - "Q[m,n,k,*] -> M[m,n,k]", - - # The following rules helps to deduce both DimSizeArgs: batch_size and seq_len - "Q[m,n,k,*] : m -> batch_size", - "Q[m,n,k,*] : k -> seq_len", - ], - version=0, - kernel_file=f'{openai_triton_example_root}/fmha_triton.py', - num_warps=4, - num_stages=num_stages, - grid_dims=(f"(seq_len + {block_size-1}) / {block_size}", - "batch_size * num_heads", "1")) - - -KERNELS = [ - get_fmha_kernel_meta_data(), -] diff --git a/examples/openai_triton/plugin_autogen/run_engine.py b/examples/openai_triton/plugin_autogen/run_engine.py deleted file mode 100644 index 9a438d1d8d87..000000000000 --- a/examples/openai_triton/plugin_autogen/run_engine.py +++ /dev/null @@ -1,170 +0,0 @@ -import argparse -import json -import math -# include plugins -# yapf: disable -import sys -from pathlib import Path - -import torch -from fmha_triton import fused_attention - -from tensorrt_llm import profiler -from tensorrt_llm._utils import (str_dtype_to_torch, str_dtype_to_trt, - trt_dtype_to_torch) -from tensorrt_llm.logger import logger -from tensorrt_llm.runtime.session import Session, TensorInfo - -# from tensorrt_llm.plugin import get_engine_name - - -sys.path.append('./tmp') -from functional import fused_attention_kernel # isort:skip -# yapf: enable - - -def get_engine_name(head_size, dtype): - return f'fmha_{head_size}_{dtype}.engine' - - -def run(engine_dir, - batch_size, - seq_len, - num_heads, - head_size, - do_benchmark=False): - # Load trt engine. - engine_dir = Path(engine_dir) - config_path = engine_dir / 'config.json' - with config_path.open('r') as f: - config = json.load(f) - dtype = config['builder_config']['precision'] - serialize_path = engine_dir / get_engine_name(head_size, dtype) - - with open(serialize_path, 'rb') as f: - session = Session.from_serialized_engine(f.read()) - - # Prepare input tensors. - torch_dtype = str_dtype_to_torch(dtype) if isinstance(dtype, str) else dtype - shape = (batch_size, num_heads, seq_len, head_size) - q = torch.normal(mean=0.1, - std=0.2, - size=shape, - dtype=torch_dtype, - device='cuda') - k = torch.normal(mean=0.4, - std=0.2, - size=shape, - dtype=torch_dtype, - device='cuda') - v = torch.normal(mean=0.3, - std=0.2, - size=shape, - dtype=torch_dtype, - device='cuda') - batch_size = q.shape[0] - seq_len = q.shape[2] - - inputs = {'Q': q, 'K': k, 'V': v} - - # Prepare output tensors. - output_info = session.infer_shapes([ - TensorInfo(name, str_dtype_to_trt(dtype), tensor.shape) - for name, tensor in inputs.items() - ]) - logger.debug(f'output info {output_info}') - outputs = { - t.name: - torch.empty(tuple(t.shape), - dtype=trt_dtype_to_torch(t.dtype), - device='cuda') - for t in output_info - } - - # Execute model inference - stream = torch.cuda.current_stream() - ok = session.run(inputs=inputs, outputs=outputs, stream=stream.cuda_stream) - assert ok, 'Engine execution failed' - - # Sanity check - stream.synchronize() - sm_scale = 1.0 / math.sqrt(head_size) - ref = fused_attention(q, k, v, sm_scale) - out = outputs["out"] - logger.debug( - f'Out: vals: {out.view(1, -1)} abs_sum: {out.float().abs().sum()}') - logger.debug( - f'Ref: vals: {ref.view(1, -1)} abs_sum: {ref.float().abs().sum()}') - torch.testing.assert_close(out, ref) - - if do_benchmark: - n_repeats = 10 - - # For fair comparison, pre-allocate buffers as trt plugin does. - shape = (q.shape[0] * q.shape[1], q.shape[2]) - L = torch.empty(shape, device=q.device, dtype=torch.float32) - m = torch.empty(shape, device=q.device, dtype=torch.float32) - o = torch.empty_like(q) - - # Triton warm-up - fused_attention(q, k, v, sm_scale, l_buf=L, m_buf=m, o_buf=o) - stream.synchronize() - for _ in range(n_repeats): - profiler.start('Triton') - fused_attention(q, k, v, sm_scale, l_buf=L, m_buf=m, o_buf=o) - stream.synchronize() - profiler.stop('Triton') - - # TRT warm-up - stream.synchronize() - ok = session.run(inputs=inputs, - outputs=outputs, - stream=stream.cuda_stream) - stream.synchronize() - for _ in range(n_repeats): - profiler.start('TRT Plugin') - ok = session.run(inputs=inputs, - outputs=outputs, - stream=stream.cuda_stream) - stream.synchronize() - profiler.stop('TRT Plugin') - assert ok - profiler.summary() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--batch_size', type=int, default=4) - parser.add_argument('--seq_len', type=int, default=128) - parser.add_argument('--num_heads', type=int, default=8) - parser.add_argument('--head_size', type=int, default=64) - parser.add_argument('--log_level', type=str, default='info') - parser.add_argument( - '--engine_dir', - type=Path, - default='outputs', - help='The directory where serialized engine files locate.') - parser.add_argument( - '--benchmark', - action='store_true', - help='Do performance benchmark compared to triton baseline.') - args = parser.parse_args() - - logger.set_level(args.log_level) - logger.info('Parameters'.center(40, '=')) - for k, v in vars(args).items(): - logger.info(f' - {k.ljust(15, ".")}: {v}') - logger.info(''.center(40, '=')) - - assert args.engine_dir.exists(), \ - f"Engine file {str(args.engine_dir)} doesn't exists." - - logger.info('Inference using the built TensorRT engine.') - run(args.engine_dir, - args.batch_size, - args.seq_len, - args.num_heads, - args.head_size, - do_benchmark=args.benchmark) - logger.info('Done.') diff --git a/examples/python_plugin/README.md b/examples/python_plugin/README.md deleted file mode 100644 index 8079d381109a..000000000000 --- a/examples/python_plugin/README.md +++ /dev/null @@ -1,120 +0,0 @@ -# TensorRT LLM Python Plugin - -TensorRT LLM provides a Python plugin interface to integrate TensorRT LLM with pure Python. - -+ `openai_triton_plugin`: plugin package -+ `build_lookup.py`: Build a TensorRT engine with TensorRT LLM Python plugin -+ `run_lookup.py`: Run the engine and compare the result with PyTorch - -## Plugin Definition - -The following code shows how to create a look-up plugin. -We only need to do a few things to define a TensorRT LLM plugin. - -1. Inherit the `PluginBase`. -2. Register the plugin class to TensorRT LLM by using `@trtllm_plugin("your_plugin_name")`. -3. Define an `__init__` function and initialize the base class. -4. Define a shape and dtype inference function. -5. Define the compute flow. - -```python -@trtllm_plugin("TritonLookUp") -class LookUpPlugin(PluginBase): - - def __init__(self, use_torch_tensor, fp32_output): - super().__init__() - self.use_torch_tensor = use_torch_tensor - self.fp32_output = fp32_output - - def shape_dtype_inference(self, inputs: Sequence[SymTensor]) -> SymTensor: - shape = inputs[1].shape - shape[0] = inputs[0].shape[0] + inputs[1].shape[0] - inputs[1].shape[0] - return SymTensor( - inputs[1].dtype if not self.fp32_output else torch.float32, shape) - - def forward(self, inputs: Sequence[TensorWrapper], - outputs: Sequence[TensorWrapper]): - assert len(inputs) == 2 - assert inputs[0].dtype in [torch.int32 or torch.int64] - assert inputs[1].dtype in [torch.float32, torch.float16, torch.bfloat16] - assert (self.fp32_output and outputs[0].dtype - == torch.float32) or outputs[0].dtype == inputs[1].dtype - - x = inputs[0] - y = inputs[1] - z = outputs[0] - if self.use_torch_tensor: - x = convert_to_torch_tensor(x) - y = convert_to_torch_tensor(y) - z = convert_to_torch_tensor(z) - MAX_BLOCK_NUM = 65536 - MAX_BLOCK_SIZE = 512 - grid = lambda meta: (min(MAX_BLOCK_NUM, x.shape[0]) * min( - MAX_BLOCK_SIZE, y.shape[1]), ) - lookup_kernel[grid](x, y, z, y.shape[0], y.shape[1], x.shape[0]) - -``` - -## Adding a TensorRT LLM Plugin to a Network - -You only need an instance of the plugin object and then call it with `tensorrt_llm.Tensor` as input arguments. - -```python -builder = tensorrt_llm.Builder() -network = builder.create_network() -with tensorrt_llm.net_guard(network): - x = Tensor(name='x', - shape=index_shape, - dtype=tensorrt_llm.str_dtype_to_trt('int32')) - y = Tensor(name='y', - shape=(vocab_size, n_embed), - dtype=torch_dtype_to_trt(dtype)) - - def lookup(x, y): - lookup_plugin = LookUpPlugin(False) - return lookup_plugin(x, y) - - output = lookup(x, y) - output.mark_output('output', torch_dtype_to_str(dtype)) -``` - -## Plugin Code Structure - -Because TensorRT LLM performs plugin registration when importing the custom TensorRT LLM plugin, there are some code structure conventions to register the plugin at runtime. - -```text -plugin_lib -├──__init__.py -├──lookup_plugin.py -└──lookup_kernel.py -``` - -The `__init__.py` file imports all the plugins in the plugin package. -With this convention, users only need to import the plugin package to register the plugins and do not need to manually import them. - -```python -# __init__.py -from .lookup_plugin import LookUpPlugin - -__all__ = ["LookUpPlugin"] -``` - -## Deserialize an Engine with TensorRT LLM Plugin - -During deserialization, TensorRT needs to find the user-defined plugin. Thus, we need to import the plugin once to register them. If the plugin follows the code structure convention, users only need to import that package to register all the custom plugins. - -```python -from tensorrt_llm.runtime.session import Session, TensorInfo - -import openai_triton_plugin # isort: skip - -if __name__ == "__main__": - - def run_engine(dtype): - output_dir = Path('tmp') / torch_dtype_to_str(dtype) - - engine_path = output_dir / "lookup.engine" - - with engine_path.open('rb') as f: - session = Session.from_serialized_engine(f.read()) -``` diff --git a/examples/python_plugin/build_lookup.py b/examples/python_plugin/build_lookup.py deleted file mode 100644 index 48cccd58882e..000000000000 --- a/examples/python_plugin/build_lookup.py +++ /dev/null @@ -1,61 +0,0 @@ -from pathlib import Path - -import torch -from plugin_lib import LookUpPlugin - -import tensorrt_llm -from tensorrt_llm import Tensor -from tensorrt_llm._utils import torch_dtype_to_str, torch_dtype_to_trt - -if __name__ == "__main__": - - # meta data - batch_size = 10 - vocab_size = 1000 - n_embed = 1024 - - # test data - ## input index - index_shape = (batch_size, ) - index_data = torch.randint(0, vocab_size, index_shape, - dtype=torch.int32).cuda() - - def test(dtype): - builder = tensorrt_llm.Builder() - builder.strongly_typed = True - network = builder.create_network() - with tensorrt_llm.net_guard(network): - x = Tensor( - name="x", - shape=index_shape, - dtype=tensorrt_llm.str_dtype_to_trt("int32"), - ) - y = Tensor(name="y", - shape=(vocab_size, n_embed), - dtype=torch_dtype_to_trt(dtype)) - - def lookup(x, y): - lookup_plugin = LookUpPlugin(False, True) - return lookup_plugin(x, y) - - output = lookup(x, y) - - output.mark_output("output", torch_dtype_to_str(torch.float32)) - - builder_config = builder.create_builder_config("float32") - engine = builder.build_engine(network, builder_config) - assert engine is not None - - output_dir = Path("tmp") / torch_dtype_to_str(dtype) - output_dir.mkdir(parents=True, exist_ok=True) - - engine_path = output_dir / "lookup.engine" - config_path = output_dir / "config.json" - - with engine_path.open("wb") as f: - f.write(engine) - builder.save_config(builder_config, str(config_path)) - - test(torch.bfloat16) - test(torch.float16) - test(torch.float32) diff --git a/examples/python_plugin/plugin_lib/__init__.py b/examples/python_plugin/plugin_lib/__init__.py deleted file mode 100644 index f27e0ded3a1c..000000000000 --- a/examples/python_plugin/plugin_lib/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .lookup_plugin import LookUpPlugin - -__all__ = ["LookUpPlugin"] diff --git a/examples/python_plugin/plugin_lib/lookup_kernel.py b/examples/python_plugin/plugin_lib/lookup_kernel.py deleted file mode 100644 index 25cf66704b96..000000000000 --- a/examples/python_plugin/plugin_lib/lookup_kernel.py +++ /dev/null @@ -1,17 +0,0 @@ -import triton -import triton.language as tl - - -@triton.jit -def lookup_kernel(X, Y, Z, vocab_size, hidden_size, token_num): - pid = tl.program_id(axis=0) - while pid < token_num * hidden_size: - row_idx = pid // hidden_size - col_idx = pid % hidden_size - word_idx = tl.load(X + row_idx) - embedding = tl.load( - Y + word_idx * hidden_size + col_idx, - mask=word_idx < vocab_size, - ) - tl.store(Z + pid, embedding) - pid += tl.num_programs(0) diff --git a/examples/python_plugin/plugin_lib/lookup_plugin.py b/examples/python_plugin/plugin_lib/lookup_plugin.py deleted file mode 100644 index 612ee7a4db1e..000000000000 --- a/examples/python_plugin/plugin_lib/lookup_plugin.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Sequence - -import torch - -from tensorrt_llm import PluginBase -from tensorrt_llm._utils import TensorWrapper, convert_to_torch_tensor -from tensorrt_llm.python_plugin import SymTensor, trtllm_plugin - -from .lookup_kernel import lookup_kernel - - -@trtllm_plugin("TritonLookUp") -class LookUpPlugin(PluginBase): - - def __init__(self, use_torch_tensor, fp32_output): - super().__init__() - self.use_torch_tensor = use_torch_tensor - self.fp32_output = fp32_output - - def shape_dtype_inference(self, inputs: Sequence[SymTensor]) -> SymTensor: - shape = inputs[1].shape - shape[0] = inputs[0].shape[0] + inputs[1].shape[0] - inputs[1].shape[0] - return SymTensor( - inputs[1].dtype if not self.fp32_output else torch.float32, shape) - - def forward(self, inputs: Sequence[TensorWrapper], - outputs: Sequence[TensorWrapper]): - assert len(inputs) == 2 - assert inputs[0].dtype in [torch.int32 or torch.int64] - assert inputs[1].dtype in [torch.float32, torch.float16, torch.bfloat16] - assert (self.fp32_output and outputs[0].dtype - == torch.float32) or outputs[0].dtype == inputs[1].dtype - - x = inputs[0] - y = inputs[1] - z = outputs[0] - if self.use_torch_tensor: - x = convert_to_torch_tensor(x) - y = convert_to_torch_tensor(y) - z = convert_to_torch_tensor(z) - MAX_BLOCK_NUM = 65536 - MAX_BLOCK_SIZE = 512 - grid = lambda meta: (min(MAX_BLOCK_NUM, x.shape[0]) * min( - MAX_BLOCK_SIZE, y.shape[1]), ) - lookup_kernel[grid](x, y, z, y.shape[0], y.shape[1], x.shape[0]) diff --git a/examples/python_plugin/run_lookup.py b/examples/python_plugin/run_lookup.py deleted file mode 100644 index 055e86008c07..000000000000 --- a/examples/python_plugin/run_lookup.py +++ /dev/null @@ -1,65 +0,0 @@ -from pathlib import Path - -import torch - -from tensorrt_llm import logger -from tensorrt_llm._utils import (torch_dtype_to_str, torch_dtype_to_trt, - trt_dtype_to_torch) -from tensorrt_llm.runtime.session import Session, TensorInfo - -import plugin_lib # isort: skip - -if __name__ == "__main__": - - def run_engine(dtype): - output_dir = Path('tmp') / torch_dtype_to_str(dtype) - - engine_path = output_dir / "lookup.engine" - - with engine_path.open('rb') as f: - session = Session.from_serialized_engine(f.read()) - - # meta data - batch_size = 10 - vocab_size = 1000 - n_embed = 1024 - - # test data - ## input index - index_shape = (batch_size, ) - index_data = torch.randint(0, - vocab_size, - index_shape, - dtype=torch.int32).cuda() - weight_data = torch.rand(vocab_size, n_embed, dtype=dtype).cuda() - - inputs = {"x": index_data, "y": weight_data} - - output_info = session.infer_shapes([ - TensorInfo(name, torch_dtype_to_trt(tensor.dtype), tensor.shape) - for name, tensor in inputs.items() - ]) - logger.debug(f'output info {output_info}') - outputs = { - t.name: - torch.empty(tuple(t.shape), - dtype=trt_dtype_to_torch(t.dtype), - device='cuda') - for t in output_info - } - - stream = torch.cuda.Stream() - ok = session.run(inputs=inputs, - outputs=outputs, - stream=stream.cuda_stream) - assert ok, 'Engine execution failed' - - embedding = torch.nn.Embedding.from_pretrained(weight_data) - torch_out = embedding(index_data).to(torch.float32) - trt_out = outputs['output'] - - torch.testing.assert_close(trt_out, torch_out) - - run_engine(torch.bfloat16) - run_engine(torch.float16) - run_engine(torch.float32) diff --git a/examples/quantization/README.md b/examples/quantization/README.md index b3b2e35b20ff..23bfa5d40c8e 100644 --- a/examples/quantization/README.md +++ b/examples/quantization/README.md @@ -1,249 +1,19 @@ -# TensorRT LLM Quantization Toolkit Installation Guide +# Model Quantization -## Introduction +To run quantized models with TensorRT LLM: -This document introduces: +- Use a pre-quantized Hugging Face checkpoint (for example the FP8/NVFP4 + checkpoints published on the [NVIDIA Hugging Face hub](https://huggingface.co/nvidia)). + Quantization settings are detected automatically when the model loads. +- To quantize your own model, use the + [NVIDIA TensorRT Model Optimizer](https://github.com/NVIDIA/TensorRT-Model-Optimizer) + Hugging Face export flow (`examples/llm_ptq` in that repository). -- The steps to install the TensorRT LLM quantization toolkit. -- The Python APIs to quantize the models. +See the [quantization feature documentation](https://nvidia.github.io/TensorRT-LLM/features/quantization.html) +for supported formats per GPU architecture. -The detailed LLM quantization recipe is distributed to the README.md of the corresponding model examples. +## Mixed-precision MoE checkpoints -## Installation - -The NVIDIA Model Optimizer quantization toolkit is installed automatically as a dependency of TensorRT-LLM. - -```bash -# Install the additional requirements -cd examples/quantization -pip install -r requirements.txt -``` - -## Usage - -```bash -# FP8 quantization. -python quantize.py --model_dir $MODEL_PATH --qformat fp8 --kv_cache_dtype fp8 --output_dir $OUTPUT_PATH - -# INT4_AWQ tp4 quantization. -python quantize.py --model_dir $MODEL_PATH --qformat int4_awq --awq_block_size 64 --tp_size 4 --output_dir $OUTPUT_PATH - -# INT8 SQ with INT8 kv cache. -python quantize.py --model_dir $MODEL_PATH --qformat int8_sq --kv_cache_dtype int8 --output_dir $OUTPUT_PATH - -# Auto quantization(e.g. fp8 + int4_awq + w4a8_awq) using average weights bits 5 -python quantize.py --model_dir $MODEL_PATH --autoq_format fp8,int4_awq,w4a8_awq --output_dir $OUTPUT_PATH --auto_quantize_bits 5 --tp_size 2 - -# FP8 quantization for NeMo model. -python quantize.py --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ - --dtype bfloat16 \ - --batch_size 64 \ - --qformat fp8 \ - --output_dir nemotron-3-8b/trt_ckpt/fp8/1-gpu - -# FP8 quantization for Medusa model. -python quantize.py --model_dir $MODEL_PATH\ - --dtype float16 \ - --qformat fp8 \ - --kv_cache_dtype fp8 \ - --output_dir $OUTPUT_PATH \ - --calib_size 512 \ - --tp_size 1 \ - --medusa_model_dir /path/to/medusa_head/ \ - --num_medusa_heads 4 -``` -Checkpoint saved in `output_dir` can be directly passed to `trtllm-build`. - -### Quantization Arguments: - -- model_dir: Hugging Face model path. -- qformat: Specify the quantization algorithm applied to the checkpoint. - - nvfp4: Weights are quantized to NVFP4 block-wise with size 16. Activation global scale are calibrated. - - fp8: Weights are quantized to FP8 tensor wise. Activation ranges are calibrated tensor wise. - - fp8_pc_pt: Weights are quantized to FP8 per-channel. Activation ranges are calibrated and quantized per-token. - - int8_sq: Weights are smoothed and quantized to INT8 channel wise. Activation ranges are calibrated tensor wise. - - int4_awq: Weights are re-scaled and block-wise quantized to INT4. Block size is specified by `awq_block_size`. - - w4a8_awq: Weights are re-scaled and block-wise quantized to INT4. Block size is specified by `awq_block_size`. Activation ranges are calibrated tensor wise. - - int8_wo: Actually nothing is applied to weights. Weights are quantized to INT8 channel wise when TRTLLM building the engine. - - int4_wo: Same as int8_wo but in INT4. - - full_prec: No quantization. -- autoq_format: Specific quantization algorithms are searched in auto quantization. The algorithm must in ['fp8', 'int4_awq', 'w4a8_awq', 'int8_sq'] and you can use ',' to separate more than one quantization algorithms, such as `--autoq_format fp8,int4_awq,w4a8_awq`. Please attention that using int8_sq and fp8 together is not supported. -- auto_quantize_bits: Effective bits constraint for auto quantization. If not set, regular quantization without auto quantization search is applied. Note: it must be set within correct range otherwise it will be set by lowest value if possible. For example, the weights of LLMs have 16 bits defaultly and it results in a weight compression rate of 40% if we set `auto_quantize_bits` to 9.6 (9.6 / 16 = 0.6), which means the average bits of the weights are 9.6 but not 16. However, which format to choose is determined by solving an optimization problem, so you need to generate the according checkpoint manually if you want to customize your checkpoint formats. The format of mixed precision checkpoint is described in detail below. -- output_dir: Path to save the quantized checkpoint. -- dtype: Specify data type of model when loading from Hugging Face. -- kv_cache_dtype: Specify kv cache data type. - - int8: Use int8 kv cache. - - fp8: Use FP8 kv cache. - - None (default): Use kv cache as model dtype. -- batch_size: Batch size for calibration. Default is 1. -- calib_size: Number of samples. Default is 512. -- calib_max_seq_length: Max sequence length of calibration samples. Default is 512. -- tp_size: Checkpoint is tensor paralleled by tp_size. Default is 1. -- pp_size: Checkpoint is pipeline paralleled by pp_size. Default is 1. -- awq_block_size: AWQ algorithm specific parameter. Indicate the block size when quantizing weights. 64 and 128 are supported by TRTLLM. -- quantize_lm_head: Enable quantization of lm_head layer. This is only supported for FP8 quantization. Default is false. - -#### NeMo model specific arguments: - -- nemo_ckpt_path: NeMo checkpoint path. -- calib_tp_size: TP size for NeMo checkpoint calibration. -- calib_pp_size: PP size for NeMo checkpoint calibration. - -#### Medusa specific arguments: - -- medusa_model_dir: Model path of medusa. -- quant_medusa_head: Whether to quantize the weights of medusa heads. -- num_medusa_heads: Number of medusa heads. -- num_medusa_layers: Number of medusa layers. -- max_draft_len: Max length of draft. -- medusa_hidden_act: Activation function of medusa. - -### Building Arguments: - -There are several arguments for the building stage which relate to quantization. -- use_fp8_context_fmha: This is Hopper-only feature. Use FP8 Gemm to calculate the attention operation. - -```python -qkv scale = 1.0 -FP_O = quantize(softmax(FP8_Q * FP8_K), scale=1.0) * FP8_V -FP_O * output_scale = FP8_O -``` - -### Checkpoint Conversion Arguments (not supported by all models) - -- FP8 - - use_fp8_rowwise: Enable FP8 per-token per-channel quantization for linear layer. (FP8 from `quantize.py` is per-tensor). -- INT8 - - smoothquant: Enable INT8 quantization for linear layer. Set the α parameter (see https://arxiv.org/pdf/2211.10438.pdf) to Smoothquant the model, and output int8 weights. A good first try is 0.5. Must be in [0, 1]. - - per_channel: Using per-channel quantization for weight when `smoothquant` is enabled. - - per_token: Using per-token quantization for activation when `smoothquant` is enabled. -- Weight-Only - - use_weight_only: Weights are quantized to INT4 or INT8 channel wise. - - weight_only_precision: Indicate `int4` or `int8` when `use_weight_only` is enabled. Or `int4_gptq` when `quant_ckpt_path` is provided which means checkpoint is for GPTQ. - - quant_ckpt_path: Path of a GPTQ quantized model checkpoint in `.safetensors` format. - - group_size: Group size used in GPTQ quantization. - - per_group: Should be enabled when load from GPTQ. -- KV Cache - - int8_kv_cache: By default, we use dtype for KV cache. int8_kv_cache chooses int8 quantization for KV cache. - - fp8_kv_cache: By default, we use dtype for KV cache. fp8_kv_cache chooses fp8 quantization for KV cache. - -### Format of Mixed Precision Checkpoints - -ModelOpt can produce a mixed precision TensorRT LLM checkpoint. After producing the quantized checkpoint, you can build engine directly by `trtllm-build` command: -```bash -trtllm-build --checkpoint_dir --output_dir $OUTPUT_PATH -``` -If you have some special needs about the model weights, such as int4 for MLP and int8 for the rest, you need to generate the checkpoint and config files by yourself. - -The `trtllm-build` command consumes the same format of weights, which is presented in [TensorRT LLM checkpoint formats](https://nvidia.github.io/TensorRT-LLM/architecture/checkpoint.html), but has different quantization method for every linear. Therefore, each layer, such as layer30.mlp.fc, layer30.attention.dense, and so on, keeps the same model weights according to the quantization formats in TensorRT LLM checkpoint. What's more, the `quantization` field in `config.json` will be like this: -``` - "quantization": { - "quant_algo": "MIXED_PRECISION", - "kv_cache_quant_algo": "FP8" // The quant_algo of KV cache may change - }, -``` -There will be another file about per-layer quantization information named `quant_cfg.json` in the same directory, the format of it is like: -``` -{ - "quant_algo": "MIXED_PRECISION", - "kv_cache_quant_algo": "FP8", - "quantized_layers": { // one more filed presents per-layer's information - "transformer.layers.0.attention.qkv": { - "quant_algo": "FP8" // specific algorithm for each linear - }, - "transformer.layers.0.attention.dense": { - "quant_algo": "FP8" - }, - "transformer.layers.0.mlp.fc": { - "quant_algo": "W4A16_AWQ", - "group_size": 128, - "has_zero_point": false, - "pre_quant_scale": true - }, - "transformer.layers.0.mlp.proj": { - "quant_algo": "W8A8_SQ_PER_CHANNEL" - }, - ... - "transformer.layers.31.mlp.proj": { - "quant_algo": "FP8" - } - } -} -``` - -TensorRT LLM will automatically read `quant_cfg.json` after recogniziong the `MIXED_PRECISION` quantization method in `config.json`. All the specific algorithm keeps the same as what in `quantization` field before. If some layers are not listed, they'll be treated as no quantization. - -## APIs - -[`quantize.py`](./quantize.py) uses the quantization toolkit to calibrate the PyTorch models and export TensorRT LLM checkpoints. Each TensorRT LLM checkpoint contains a config file (in .json format) and one or several rank weight files (in .safetensors format). It will produce one another quantization config for per-layer's information when setting auto quantization. The checkpoints can be directly used by `trtllm-build` command to build TensorRT LLM engines. See this [`doc`](../../docs/source/architecture/checkpoint.md) for more details on the TensorRT LLM checkpoint format. - -> *This quantization step may take a long time to finish and requires large GPU memory. Please use a server grade GPU if a GPU out-of-memory error occurs* - -> *If the model is trained with multi-GPU with tensor parallelism, the PTQ calibration process requires the same amount of GPUs as the training time too.* - - -### PTQ (Post Training Quantization) - -PTQ can be achieved with simple calibration on a small set of training or evaluation data (typically 128-512 samples) after converting a regular PyTorch model to a quantized model. - -```python -import torch -from torch.utils.data import DataLoader -from transformers import AutoModelForCausalLM -import modelopt.torch.quantization as mtq -import modelopt.torch.utils.dataset_utils as dataset_utils - -model = AutoModelForCausalLM.from_pretrained(...) - -# Select the quantization config, for example, FP8 -config = mtq.FP8_DEFAULT_CFG - -# Prepare the calibration set and define a forward loop -calib_dataloader = DataLoader(...) -calibrate_loop = dataset_utils.create_forward_loop( - calib_dataloader, dataloader=calib_dataloader -) - -# PTQ with in-place replacement to quantized modules -with torch.no_grad(): - mtq.quantize(model, config, forward_loop=calibrate_loop) - -# or PTQ with auto quantization -with torch.no_grad(): - model, search_history = mtq.auto_quantize( - model, - data_loader=calib_dataloader, - loss_func=lambda output, batch: output.loss, - constraints={"effective_bits": auto_quantize_bits}, # The average bits of quantized weights - forward_step=lambda model, batch: model(**batch), - quantization_formats=[quant_algo1, quant_algo2,...] + [None], - num_score_steps=min( - num_calib_steps=len(calib_dataloader), - len(calib_dataloader), 128 // batch_size - ), # Limit the number of score steps to avoid long calibration time - verbose=True, - ) -``` - -### Export Quantized Model - -After the model is quantized, it can be exported to a TensorRT LLM checkpoint, which includes - -- One json file recording the model structure and metadata, and -- One or several rank weight files storing quantized model weights and scaling factors. - -The export API is - -```python -from modelopt.torch.export import export_tensorrt_llm_checkpoint - -with torch.inference_mode(): - export_tensorrt_llm_checkpoint( - model, # The quantized model. - decoder_type, # The type of the model as str, e.g gptj, llama or gptnext. - dtype, # The exported weights data type as torch.dtype. - export_dir, # The directory where the exported files will be stored. - inference_tensor_parallel=tp_size, # The tensor parallelism size for inference. - inference_pipeline_parallel=pp_size, # The pipeline parallelism size for inference. - ) -``` +[`quantize_mixed_precision_moe.py`](quantize_mixed_precision_moe.py) builds a +mixed-precision MoE checkpoint from separately quantized checkpoints; see the +script's argparse help for usage. diff --git a/examples/quantization/quantize.py b/examples/quantization/quantize.py deleted file mode 100644 index 29c2fc5ca179..000000000000 --- a/examples/quantization/quantize.py +++ /dev/null @@ -1,208 +0,0 @@ -import argparse - -import torch.multiprocessing as mp - -from tensorrt_llm.quantization import (quantize_and_export, - quantize_nemo_and_export) - -if __name__ == "__main__": - mp.set_start_method("spawn", force=True) - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--model_dir", - help="Specify where the HuggingFace model is", - default=None) - parser.add_argument('--nemo_ckpt_path', - help="Specify where the NeMo checkpoint is", - default=None) - parser.add_argument( - '--decoder_type', - type=str, - default='gptnext', - choices=['gptnext', 'llama'], - help="Decoder type; effective for NeMo checkpoint only.") - parser.add_argument( - '--device', - help= - "The device to run calibration; effective for HuggingFace model only.", - default='cuda', - choices=['cuda', 'cpu']) - parser.add_argument( - "--device_map", - help="How to map the model on the devices", - default="auto", - choices=["auto", "sequential", "cpu", "gpu"], - ) - parser.add_argument( - '--calib_dataset', - type=str, - default='cnn_dailymail', - help= - "The huggingface dataset name or the local directory of the dataset for calibration." - ) - parser.add_argument( - '--calib_tp_size', - type=int, - default=1, - help= - "Tensor parallel size for calibration; effective for NeMo checkpoint only." - ) - parser.add_argument( - '--calib_pp_size', - type=int, - default=1, - help= - "Pipeline parallel size for calibration; effective for NeMo checkpoint only." - ) - - parser.add_argument( - '--dtype', - type=str, - default='auto', - choices=['auto', 'float16', 'bfloat16', 'float32'], - help= - "The data type for the model weights and activations of the non-quantized part, e.g., embedding and lm_head. " - "If 'auto', the data type is automatically inferred from the source model; " - "however, if the source dtype is float32, it is converted to float16.") - parser.add_argument( - "--qformat", - help="Quantization format.", - default="full_prec", - choices=[ - "nvfp4", - "fp8", - "fp8_pc_pt", - "int8_sq", - "int4_awq", - "w4a8_awq", - "int8_wo", - "int4_wo", - "full_prec", - ], - ) - parser.add_argument( - "--seed", - help="Seed the generate random numbers, the value will be used to call" - "random.seed(value) and numpy.random.seed(value)", - type=int, - default=1234) - parser.add_argument("--tokenizer_max_seq_length", - help="Max sequence length to init the tokenizers", - type=int, - default=2048) - - parser.add_argument("--batch_size", - help="Batch size for calibration.", - type=int, - default=1) - parser.add_argument("--calib_size", - help="Number of samples for calibration.", - type=int, - default=512) - parser.add_argument("--calib_max_seq_length", - help="Max sequence length for calibration", - type=int, - default=512) - parser.add_argument("--output_dir", default="exported_model") - parser.add_argument("--tp_size", type=int, default=1) - parser.add_argument("--pp_size", type=int, default=1) - parser.add_argument("--cp_size", type=int, default=1) - parser.add_argument("--awq_block_size", type=int, default=128) - parser.add_argument("--kv_cache_dtype", - help="KV Cache dtype.", - default=None, - choices=["int8", "fp8", None]) - parser.add_argument("--quantize_lm_head", - action='store_true', - default=False) - # Medusa - parser.add_argument('--num_medusa_heads', type=int, default=4) - parser.add_argument('--num_medusa_layers', type=int, default=1) - parser.add_argument('--max_draft_len', type=int, default=63) - parser.add_argument('--medusa_hidden_act', type=str, default="silu") - parser.add_argument('--medusa_model_dir', type=str, default=None) - parser.add_argument('--quant_medusa_head', - default=False, - action='store_true', - help="whether to quantize the weights of medusa heads") - - # auto quantization - parser.add_argument( - '--autoq_format', - default=None, - type=str, - help= - "Specific quantization algorithms will be searched in auto quantization." - "The algorithm must in ['fp8', 'int4_awq', 'w4a8_awq', 'int8_sq']." - "You can use ',' to separate more than one quantization algorithms(e.g. --autoq_format fp8,int4_awq,w4a8_awq)." - "Notice: fp8 and int8_sq can't be used at the same time.") - parser.add_argument( - '--auto_quantize_bits', - type=float, - default=None, - help="Effective bits constraint for auto quantization. If not set, " - "regular quantization without auto quantization search will be applied." - "You can't set it lower than the num_bits of most aggressive quantization format." - "For example, if 'int4_awq' is in autoq_format, it can't be lower than 4.0." - ) - - args = parser.parse_args() - - # auto_quantize_bits check - if args.autoq_format: - lower_bound, upper_bound = 4 if '4' in args.autoq_format else 8, 16 - if args.auto_quantize_bits is None or args.auto_quantize_bits < lower_bound or args.auto_quantize_bits > upper_bound: - print( - f"invalid auto_quantize_bits value, will be set to {lower_bound}" - ) - args.auto_quantize_bits = lower_bound - - if args.model_dir is not None: - quantize_and_export( - model_dir=args.model_dir, - device=args.device, - calib_dataset=args.calib_dataset, - dtype=args.dtype, - qformat=args.qformat - if args.auto_quantize_bits is None else args.autoq_format, - kv_cache_dtype=args.kv_cache_dtype, - calib_size=args.calib_size, - batch_size=args.batch_size, - calib_max_seq_length=args.calib_max_seq_length, - awq_block_size=args.awq_block_size, - output_dir=args.output_dir, - tp_size=args.tp_size, - pp_size=args.pp_size, - cp_size=args.cp_size, - seed=args.seed, - tokenizer_max_seq_length=args.tokenizer_max_seq_length, - num_medusa_heads=args.num_medusa_heads, - num_medusa_layers=args.num_medusa_layers, - max_draft_len=args.max_draft_len, - medusa_hidden_act=args.medusa_hidden_act, - medusa_model_dir=args.medusa_model_dir, - quant_medusa_head=args.quant_medusa_head, - auto_quantize_bits=args.auto_quantize_bits, - device_map=args.device_map, - quantize_lm_head=args.quantize_lm_head) - elif args.nemo_ckpt_path is not None: - quantize_nemo_and_export(nemo_ckpt_path=args.nemo_ckpt_path, - decoder_type=args.decoder_type, - calib_dataset=args.calib_dataset, - calib_tp_size=args.calib_tp_size, - calib_pp_size=args.calib_pp_size, - dtype=args.dtype, - qformat=args.qformat, - kv_cache_dtype=args.kv_cache_dtype, - calib_size=args.calib_size, - batch_size=args.batch_size, - calib_max_seq_length=args.calib_max_seq_length, - awq_block_size=args.awq_block_size, - output_dir=args.output_dir, - tp_size=args.tp_size, - pp_size=args.pp_size, - cp_size=args.cp_size, - seed=args.seed) - else: - raise ValueError( - "One of source checkpoint (model_dir, nemo_ckpt_path) must be specified" - ) diff --git a/examples/run.py b/examples/run.py deleted file mode 100755 index 7ce36bbe9848..000000000000 --- a/examples/run.py +++ /dev/null @@ -1,710 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import ast -import csv -import os -from pathlib import Path -from typing import List, Optional - -import numpy as np -import torch -from utils import (DEFAULT_HF_MODEL_DIRS, DEFAULT_PROMPT_TEMPLATES, - add_common_args, get_beam_width_array, load_tokenizer, - prepare_enc_dec_inputs, read_model_name, - supports_inflight_batching, throttle_generator) - -import tensorrt_llm -import tensorrt_llm.profiler -from tensorrt_llm.logger import logger -from tensorrt_llm.runtime import PYTHON_BINDINGS, ModelRunner - -if PYTHON_BINDINGS: - from tensorrt_llm.runtime import ModelRunnerCpp - -from ngram.run_dtm_ngram import run_dtm_ngram - - -def parse_arguments(args=None): - # see `add_common_args` for extended list of arguments - parser = argparse.ArgumentParser() - parser.add_argument('--max_input_length', type=int, default=923) - parser.add_argument('--max_output_len', type=int, required=True) - parser.add_argument( - '--draft_engine_dir', - type=str, - default=None, - help='Path to engine of draft model in Draft-Target-Model mode.') - parser.add_argument( - '--input_text', - type=str, - nargs='+', - default=["Born in north-east France, Soyer trained as a"]) - parser.add_argument( - '--input_file', - type=str, - help= - 'CSV or Numpy file containing tokenized input. Alternative to text input.', - default=None) - parser.add_argument('--multimodal_input_file', - type=str, - help='Path to multimodal input file.') - parser.add_argument( - '--input_token_extra_ids', - type=int, - nargs='+', - help= - 'Input token extra ids for using p-tuning and KV Cache reuse together (only available with cpp session).', - default=None) - parser.add_argument( - '--input_token_extra_ids_file', - type=str, - help= - 'CSV or Numpy file containing input token extra ids file. Alternative to text input (only available with cpp session).', - default=None) - parser.add_argument('--output_csv', - type=str, - help='CSV file where the tokenized output is stored.', - default=None) - parser.add_argument('--output_npy', - type=str, - help='Numpy file where the tokenized output is stored.', - default=None) - parser.add_argument('--output_generation_logits', - default=False, - action='store_true', - help="Enable gathering generation logits.") - parser.add_argument( - '--output_logits_npy', - type=str, - help= - 'Numpy file where the generation logits are stored. Use only when num_beams==1', - default=None) - parser.add_argument('--output_log_probs_npy', - type=str, - help='Numpy file where the log_probs are stored', - default=None) - parser.add_argument('--output_cum_log_probs_npy', - type=str, - help='Numpy file where the cum_log_probs are stored', - default=None) - parser.add_argument( - '--run_profiling', - default=False, - action='store_true', - help="Run several 10 iterations to profile the inference latencies.") - parser.add_argument( - '--fail_fast_on_attention_window_too_large', - action='store_true', - default=False, - help= - 'Exit with runtime error when attention window is too large to fit even a single sequence in the KV cache.' - ) - - parser = add_common_args(parser) - - return parser.parse_args(args=args) - - -def parse_input(tokenizer, - input_text=None, - prompt_template=None, - input_file=None, - add_special_tokens=True, - max_input_length=923, - pad_id=None, - num_prepend_vtokens=[], - model_name=None, - model_version=None): - if pad_id is None: - pad_id = tokenizer.pad_token_id - - batch_input_ids = [] - if input_file is None: - if 'whisper' in model_name.lower(): - batch_input_ids.append(tokenizer.prefix_tokens) - else: - for curr_text in input_text: - if prompt_template is not None: - curr_text = prompt_template.format(input_text=curr_text) - input_ids = tokenizer.encode( - curr_text, - add_special_tokens=add_special_tokens, - truncation=True, - max_length=max_input_length) - batch_input_ids.append(input_ids) - else: - if input_file.endswith('.csv'): - with open(input_file, 'r') as csv_file: - csv_reader = csv.reader(csv_file, delimiter=',') - for line in csv_reader: - input_ids = np.array(line, dtype='int32') - batch_input_ids.append(input_ids[-max_input_length:]) - elif input_file.endswith('.npy'): - inputs = np.load(input_file) - for row in inputs: - input_ids = row[row != pad_id] - batch_input_ids.append(input_ids[-max_input_length:]) - - elif input_file.endswith('.txt'): - with open(input_file, 'r', encoding='utf-8', - errors='replace') as txt_file: - input_text = txt_file.readlines() - batch_input_ids = tokenizer( - input_text, - add_special_tokens=add_special_tokens, - truncation=True, - max_length=max_input_length)["input_ids"] - else: - print('Input file format not supported.') - raise SystemExit - - if num_prepend_vtokens: - assert len(num_prepend_vtokens) == len(batch_input_ids) - base_vocab_size = tokenizer.vocab_size - for i, length in enumerate(num_prepend_vtokens): - batch_input_ids[i] = list( - range(base_vocab_size, - base_vocab_size + length)) + batch_input_ids[i] - - if input_file is None and 'GLM' in model_name and model_version == 'glm': - for ids in batch_input_ids: - ids.append(tokenizer.sop_token_id) - - batch_input_ids = [ - torch.tensor(x, dtype=torch.int32) for x in batch_input_ids - ] - - logger.debug(f"Input token ids (batch_size = {len(batch_input_ids)}):") - for i, input_ids in enumerate(batch_input_ids): - logger.debug(f"Request {i}: {input_ids.tolist()}") - - return batch_input_ids - - -def parse_input_token_extra_ids(prompt_table_path, kv_cache_enable_block_reuse, - input_token_extra_ids, - input_token_extra_ids_file, max_input_length): - batch_extra_ids = None - if prompt_table_path and kv_cache_enable_block_reuse: - assert input_token_extra_ids or input_token_extra_ids_file, \ - "Input token extra ids must be provided when p-tuning and KV Cache reuse are both enabled" - batch_extra_ids = [] - if input_token_extra_ids_file: - if input_token_extra_ids_file.endswith('.csv'): - with open(input_token_extra_ids_file, 'r') as csv_file: - csv_reader = csv.reader(csv_file, delimiter=',') - for line in csv_reader: - extra_ids = [int(num) for num in line] - batch_extra_ids.append(extra_ids[-max_input_length:]) - elif input_token_extra_ids_file.endswith('.npy'): - inputs = np.load(input_token_extra_ids_file) - for extra_ids in inputs: - batch_extra_ids.append(extra_ids[-max_input_length:]) - else: - print('Input file format not supported.') - raise SystemExit - else: - batch_extra_ids.append(input_token_extra_ids) - return batch_extra_ids - - -def print_output(tokenizer, - output_ids: torch.Tensor, - input_lengths: List[int], - sequence_lengths: torch.Tensor, - output_csv: Optional[str] = None, - output_npy: Optional[str] = None, - context_logits: Optional[torch.Tensor] = None, - generation_logits: Optional[torch.Tensor] = None, - cum_log_probs: Optional[torch.Tensor] = None, - log_probs: Optional[torch.Tensor] = None, - output_logits_npy: Optional[str] = None, - output_cum_log_probs_npy: Optional[str] = None, - output_log_probs_npy: Optional[str] = None): - num_output_sents, num_beams, _ = output_ids.size() - batch_size = len(input_lengths) - num_return_sequences = num_output_sents // batch_size - - if output_csv is None and output_npy is None and tokenizer is not None: - for i in range(batch_size * num_return_sequences): - batch_idx = i // num_return_sequences - seq_idx = i % num_return_sequences - inputs = output_ids[i][0][:input_lengths[batch_idx]].tolist() - input_text = tokenizer.decode(inputs) - if seq_idx == 0: - print(f'Input [Text {batch_idx}]: \"{input_text}\"') - - for beam in range(num_beams): - output_begin = input_lengths[batch_idx] - output_end = sequence_lengths[i][beam] - outputs = output_ids[i][beam][output_begin:output_end].tolist() - output_text = tokenizer.decode(outputs) - index_str = (f'Text {batch_idx} Seq {seq_idx} Beam {beam}' - if num_return_sequences > 1 else - f'Text {batch_idx} Beam {beam}') - print(f'Output [{index_str}]: \"{output_text}\"') - logger.debug(str(outputs)) - - output_ids = output_ids.reshape((-1, output_ids.size(2))) - - if output_csv is not None: - output_file = Path(output_csv) - output_file.parent.mkdir(exist_ok=True, parents=True) - outputs = output_ids.tolist() - with open(output_file, 'w') as csv_file: - writer = csv.writer(csv_file, delimiter=',') - writer.writerows(outputs) - - if output_npy is not None: - output_file = Path(output_npy) - output_file.parent.mkdir(exist_ok=True, parents=True) - outputs = np.array(output_ids.cpu().contiguous(), dtype='int32') - np.save(output_file, outputs) - - # Save context logits - if context_logits is not None and output_logits_npy is not None: - context_logits = torch.cat(context_logits, axis=0) - vocab_size_padded = context_logits.shape[-1] - context_logits = context_logits.reshape([1, -1, vocab_size_padded]) - - output_context_logits_npy = output_logits_npy.split( - '.npy')[0] + "_context" - output_context_logits_file = Path(output_context_logits_npy) - context_outputs = np.array( - context_logits.squeeze(0).cpu().contiguous(), - dtype='float32') # [promptLengthSum, vocabSize] - np.save(output_context_logits_file, context_outputs) - - # Save generation logits - if generation_logits is not None and output_logits_npy is not None and num_beams == 1: - output_generation_logits_npy = output_logits_npy.split( - '.npy')[0] + "_generation" - output_generation_logits_file = Path(output_generation_logits_npy) - generation_outputs = np.array(generation_logits.cpu().contiguous(), - dtype='float32') - np.save(output_generation_logits_file, generation_outputs) - - # Save cum log probs - if cum_log_probs is not None and output_cum_log_probs_npy is not None: - cum_log_probs_file = Path(output_cum_log_probs_npy) - cum_log_probs_outputs = np.array(cum_log_probs.cpu().contiguous(), - dtype='float32') - np.save(cum_log_probs_file, cum_log_probs_outputs) - - # Save cum log probs - if log_probs is not None and output_log_probs_npy is not None: - log_probs_file = Path(output_log_probs_npy) - log_probs_outputs = np.array(log_probs.cpu().contiguous(), - dtype='float32') - np.save(log_probs_file, log_probs_outputs) - - -def main(args): - runtime_rank = tensorrt_llm.mpi_rank() - logger.set_level(args.log_level) - - # different handling if encoder-decoder models - is_enc_dec = {'encoder', 'decoder'}.issubset({ - name - for name in os.listdir(args.engine_dir) - if os.path.isdir(os.path.join(args.engine_dir, name)) - }) - if is_enc_dec: - logger.warning( - "This path is an encoder-decoder model. Using different handling.") - assert not args.use_py_session, "Encoder-decoder models don't have a unified python runtime, please use its own examples/models/core/enc_dec/run.py instead." - - model_name, model_version = read_model_name( - args.engine_dir if not is_enc_dec else os.path. - join(args.engine_dir, 'encoder')) - - if args.tokenizer_dir is None and model_name in DEFAULT_HF_MODEL_DIRS: - logger.warning( - "tokenizer_dir is not specified. Try to infer from model_name, but this may be incorrect." - ) - args.tokenizer_dir = DEFAULT_HF_MODEL_DIRS[model_name] - - tokenizer, pad_id, end_id = load_tokenizer( - tokenizer_dir=args.tokenizer_dir, - vocab_file=args.vocab_file, - model_name=model_name, - model_version=model_version, - tokenizer_type=args.tokenizer_type, - ) - - if args.end_id: - end_id = args.end_id - - prompt_template = None - if args.use_prompt_template and model_name in DEFAULT_PROMPT_TEMPLATES: - prompt_template = DEFAULT_PROMPT_TEMPLATES[model_name] - - batch_input_ids = parse_input(tokenizer=tokenizer, - input_text=args.input_text, - prompt_template=prompt_template, - input_file=args.input_file, - add_special_tokens=args.add_special_tokens, - max_input_length=args.max_input_length, - pad_id=pad_id, - num_prepend_vtokens=args.num_prepend_vtokens, - model_name=model_name, - model_version=model_version) - - stop_words_list = None - if args.stop_words: - stop_words_list = tensorrt_llm.runtime.decode_words_list( - args.stop_words, tokenizer) - if model_version == 'glm4': # add default stop token ids for GLM-4 - glm4_stop_ids = [[151329], [151336], [151338]] - if stop_words_list is None: - stop_words_list = [glm4_stop_ids] * len(batch_input_ids) - else: - for req_stop_words_list in stop_words_list: - req_stop_words_list.extend(glm4_stop_ids) - - bad_words_list = None - if args.bad_words: - bad_words_list = tensorrt_llm.runtime.decode_words_list( - args.bad_words, tokenizer) - - if is_enc_dec: - encoder_input_ids, encoder_input_features, encoder_output_lengths, decoder_input_ids = prepare_enc_dec_inputs( - batch_input_ids, model_name, args.engine_dir, - args.multimodal_input_file) - - input_token_extra_ids = parse_input_token_extra_ids( - args.prompt_table_path, args.kv_cache_enable_block_reuse, - args.input_token_extra_ids, args.input_token_extra_ids_file, - args.max_input_length) - - input_lengths = [x.size(0) for x in decoder_input_ids - ] if is_enc_dec else [x.size(0) for x in batch_input_ids] - - encoder_input_lengths = [ - x.size(0) for x in (encoder_input_features or encoder_input_ids) - ] if is_enc_dec else None - - if args.beam_width_array is not None: - logger.info("Enable Variable-Beam-Width-Search (VBWS)") - assert not args.use_py_session, "`--use_py_session` is not supported in VBWS." - args.beam_width_array, args.num_beams = get_beam_width_array( - args.beam_width_array) - - if not args.use_py_session and not supports_inflight_batching( - os.path.join(args.engine_dir, "decoder") if is_enc_dec else args. - engine_dir): - logger.warning( - "The given engine does not support in-flight batching, fallback to python session" - ) - args.use_py_session = True - - if not PYTHON_BINDINGS and not args.use_py_session: - logger.warning( - "Python bindings of C++ session is unavailable, fallback to Python session." - ) - args.use_py_session = True - if args.debug_mode and not args.use_py_session: - logger.warning( - "Debug mode is not supported in C++ session for now, fallback to Python session." - ) - args.use_py_session = True - if args.return_all_generated_tokens and args.use_py_session: - raise ValueError( - "Returning all the generated tokens at each step is not supported in the Python session, use C++ session instead." - ) - if (not args.return_all_generated_tokens) and args.streaming and ( - args.num_beams > 1): - logger.warning( - "Setting return_all_generated_tokens to True since streaming AND beam search are done simultaneously. " - "Returning the full beams at each streaming step is needed because beam search + streaming can change previous outputs. " - "WARNING: using this option may increase network usage significantly (quadratically w.r.t output length)." - ) - args.return_all_generated_tokens = True - - logger.info(f"Using {'Python' if args.use_py_session else 'C++'} session") - - if args.draft_target_model_config is not None or args.ngram_config is not None: - # Speculative-Decoding of Draft-Target-Model (DTM) and NGram - # If the parameters of `runner_kwargs` and `runner.generate()` in the "else" branch change, the same change should be done for `examples/ngram/run_dtm_ngram.py` - assert args.kv_cache_enable_block_reuse, "`--kv_cache_enable_block_reuse` must be specified in speculative decoding." - assert not args.use_py_session, "`--use_py_session` is not supported in Speculative decoding." - assert not is_enc_dec, "Encoder-Decoder model is not supported in Speculative decoding." - assert args.num_beams == 1, "`--num_beams>1` is not supported in Speculative decoding." - - outputs = run_dtm_ngram(batch_input_ids, args, runtime_rank, end_id, - pad_id, stop_words_list, bad_words_list, - len(tokenizer)) - if not args.streaming: # Unpack runner from the return value in No-Streaming mode - outputs, runner = list(outputs)[0] - - else: # Normal run - runner_cls = ModelRunner if args.use_py_session else ModelRunnerCpp - runner_kwargs = dict( - engine_dir=args.engine_dir, - lora_dir=args.lora_dir, - rank=runtime_rank, - debug_mode=args.debug_mode, - lora_ckpt_source=args.lora_ckpt_source, - gpu_weights_percent=args.gpu_weights_percent, - max_output_len=args.max_output_len, - enable_context_fmha_fp32_acc=args.enable_context_fmha_fp32_acc, - fail_fast_on_attention_window_too_large=args. - fail_fast_on_attention_window_too_large, - ) - if args.medusa_choices is not None: - args.medusa_choices = ast.literal_eval(args.medusa_choices) - assert args.temperature == 1.0, "Medusa should use temperature == 1.0" - assert args.num_beams == 1, "Medusa should use num_beams == 1" - runner_kwargs.update(medusa_choices=args.medusa_choices) - if args.eagle_choices is not None or args.eagle_posterior_threshold is not None or args.eagle_use_dynamic_tree: - assert args.num_beams == 1, "Eagle should use num_beams == 1" - assert not args.use_py_session, "Eagle does not support py session" - if args.eagle_choices is not None and not args.eagle_use_dynamic_tree: - args.eagle_choices = ast.literal_eval(args.eagle_choices) - runner_kwargs.update(eagle_choices=args.eagle_choices) - if args.eagle_posterior_threshold is not None: - runner_kwargs.update( - eagle_posterior_threshold=args.eagle_posterior_threshold) - if args.eagle_use_dynamic_tree: - runner_kwargs.update( - eagle_use_dynamic_tree=args.eagle_use_dynamic_tree) - assert args.eagle_dynamic_tree_max_top_k is not None and args.eagle_dynamic_tree_max_top_k > 0 - runner_kwargs.update(eagle_dynamic_tree_max_top_k=args. - eagle_dynamic_tree_max_top_k) - if args.lookahead_config is not None: - args.lookahead_config = ast.literal_eval(args.lookahead_config) - assert len( - args.lookahead_config - ) == 3, "Lookahead needs [max_window_size, max_ngram_size, max_verification_set_size]" - runner_kwargs.update(lookahead_config=args.lookahead_config) - if not args.use_py_session: - runner_kwargs.update( - is_enc_dec=is_enc_dec, - max_batch_size=len(batch_input_ids), - max_input_len=max( - encoder_input_lengths if is_enc_dec else input_lengths), - max_beam_width=args.num_beams, - max_attention_window_size=args.max_attention_window_size, - sink_token_length=args.sink_token_length, - max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, - kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, - kv_cache_free_gpu_memory_fraction=args. - kv_cache_free_gpu_memory_fraction, - cross_kv_cache_fraction=args.cross_kv_cache_fraction - if is_enc_dec else None, - enable_chunked_context=args.enable_chunked_context, - multi_block_mode=args.multi_block_mode, - cuda_graph_mode=args.cuda_graph_mode, - gather_generation_logits=args.output_generation_logits, - use_variable_beam_width_search=(args.beam_width_array - is not None), - ) - runner = runner_cls.from_dir(**runner_kwargs) - - with torch.no_grad(): - outputs = runner.generate( - batch_input_ids=decoder_input_ids - if is_enc_dec else batch_input_ids, - encoder_input_ids=encoder_input_ids if is_enc_dec else None, - encoder_input_features=encoder_input_features - if is_enc_dec else None, - encoder_output_lengths=encoder_output_lengths - if is_enc_dec else None, - max_new_tokens=args.max_output_len, - max_attention_window_size=args.max_attention_window_size, - sink_token_length=args.sink_token_length, - end_id=end_id, - pad_id=pad_id, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - num_return_sequences=args.num_return_sequences, - length_penalty=args.length_penalty, - early_stopping=args.early_stopping, - beam_width_array=args.beam_width_array, - repetition_penalty=args.repetition_penalty, - presence_penalty=args.presence_penalty, - frequency_penalty=args.frequency_penalty, - prompt_ignore_length=args.prompt_ignore_length, - min_p=args.min_p, - stop_words_list=stop_words_list, - bad_words_list=bad_words_list, - output_cum_log_probs=(args.output_cum_log_probs_npy != None), - output_log_probs=(args.output_log_probs_npy != None), - random_seed=args.random_seed, - lora_uids=args.lora_task_uids, - prompt_table=args.prompt_table_path, - prompt_tasks=args.prompt_tasks, - streaming=args.streaming, - output_sequence_lengths=True, - output_generation_logits=args.output_generation_logits, - no_repeat_ngram_size=args.no_repeat_ngram_size, - return_dict=True, - medusa_choices=args.medusa_choices, - eagle_choices=args.eagle_choices, - return_all_generated_tokens=args.return_all_generated_tokens, - input_token_extra_ids=input_token_extra_ids, - fail_fast_on_attention_window_too_large=args. - fail_fast_on_attention_window_too_large, - language_adapter_uids=args.language_task_uids) - torch.cuda.synchronize() - - # Receive output, print to screen or save to file - if args.streaming: - for curr_outputs in throttle_generator(outputs, - args.streaming_interval): - if runtime_rank == 0: - output_ids = curr_outputs['output_ids'] - sequence_lengths = curr_outputs['sequence_lengths'] - cum_log_probs = None - log_probs = None - if args.output_cum_log_probs_npy is not None: - cum_log_probs = curr_outputs['cum_log_probs'] - if args.output_log_probs_npy is not None: - log_probs = curr_outputs['log_probs'] - print_output( - tokenizer, - output_ids, - input_lengths, - sequence_lengths, - output_csv=args.output_csv, - output_npy=args.output_npy, - cum_log_probs=cum_log_probs, - log_probs=log_probs, - output_cum_log_probs_npy=args.output_cum_log_probs_npy, - output_log_probs_npy=args.output_log_probs_npy) - else: - if runtime_rank == 0: - output_ids = outputs['output_ids'] - sequence_lengths = outputs['sequence_lengths'] - context_logits = None - generation_logits = None - cum_log_probs = None - log_probs = None - if runner.gather_context_logits: - context_logits = outputs['context_logits'] - if runner.gather_generation_logits or args.output_generation_logits: - generation_logits = outputs['generation_logits'] - if args.output_cum_log_probs_npy is not None: - cum_log_probs = outputs['cum_log_probs'] - if args.output_log_probs_npy is not None: - log_probs = outputs['log_probs'] - print_output(tokenizer, - output_ids, - input_lengths, - sequence_lengths, - output_csv=args.output_csv, - output_npy=args.output_npy, - context_logits=context_logits, - generation_logits=generation_logits, - output_logits_npy=args.output_logits_npy, - cum_log_probs=cum_log_probs, - log_probs=log_probs, - output_cum_log_probs_npy=args.output_cum_log_probs_npy, - output_log_probs_npy=args.output_log_probs_npy) - - # Profiling - if args.run_profiling: - ite = 10 - # warmup - for _ in range(ite): - with torch.no_grad(): - outputs = runner.generate( - batch_input_ids, - max_new_tokens=args.max_output_len, - max_attention_window_size=args.max_attention_window_size, - end_id=end_id, - pad_id=pad_id, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - length_penalty=args.length_penalty, - early_stopping=args.early_stopping, - beam_width_array=args.beam_width_array, - repetition_penalty=args.repetition_penalty, - presence_penalty=args.presence_penalty, - frequency_penalty=args.frequency_penalty, - prompt_ignore_length=args.prompt_ignore_length, - min_p=args.min_p, - stop_words_list=stop_words_list, - bad_words_list=bad_words_list, - output_cum_log_probs=(args.output_cum_log_probs_npy - is not None), - output_log_probs=(args.output_log_probs_npy is not None), - random_seed=args.random_seed, - lora_uids=args.lora_task_uids, - lookahead_config=args.lookahead_config, - prompt_table=args.prompt_table_path, - prompt_tasks=args.prompt_tasks, - streaming=args.streaming, - output_sequence_lengths=True, - return_dict=True, - return_all_generated_tokens=args. - return_all_generated_tokens, - input_token_extra_ids=input_token_extra_ids) - torch.cuda.synchronize() - - tensorrt_llm.profiler.start("tmp") - for _ in range(ite): - with torch.no_grad(): - outputs = runner.generate( - batch_input_ids, - max_new_tokens=args.max_output_len, - max_attention_window_size=args.max_attention_window_size, - end_id=end_id, - pad_id=pad_id, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - length_penalty=args.length_penalty, - early_stopping=args.early_stopping, - beam_width_array=args.beam_width_array, - repetition_penalty=args.repetition_penalty, - presence_penalty=args.presence_penalty, - frequency_penalty=args.frequency_penalty, - prompt_ignore_length=args.prompt_ignore_length, - stop_words_list=stop_words_list, - bad_words_list=bad_words_list, - output_cum_log_probs=(args.output_cum_log_probs_npy - != None), - output_log_probs=(args.output_log_probs_npy != None), - random_seed=args.random_seed, - lora_uids=args.lora_task_uids, - prompt_table=args.prompt_table_path, - prompt_tasks=args.prompt_tasks, - streaming=args.streaming, - output_sequence_lengths=True, - return_dict=True, - return_all_generated_tokens=args. - return_all_generated_tokens, - input_token_extra_ids=input_token_extra_ids, - fail_fast_on_attention_window_too_large=args. - fail_fast_on_attention_window_too_large) - torch.cuda.synchronize() - tensorrt_llm.profiler.stop("tmp") - - print( - f"batch_size: {len(batch_input_ids)}, avg latency of {ite} iterations: : {tensorrt_llm.profiler.elapsed_time_in_sec('tmp') / ite} sec" - ) - - -if __name__ == '__main__': - args = parse_arguments() - main(args) diff --git a/examples/sample_weight_stripping/README.md b/examples/sample_weight_stripping/README.md deleted file mode 100644 index cb3c04404902..000000000000 --- a/examples/sample_weight_stripping/README.md +++ /dev/null @@ -1,275 +0,0 @@ -# Sample Weight-Stripping - -> [!WARNING] -> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described -> below is **legacy** and will not receive new features. New projects should use -> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) -> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. - -## Table Of Contents - -- [Overview](#overview) - * [Build Weights Stripped Engine](#build-weights-stripped-engine) - * [Engine Refitter](#engine-refitter) -- [Prerequisites](#prerequisites) -- [Weight-Stripping Workflow Example](#weight-stripping-workflow-example) - * [GPT-J](#gpt-j) - * [Llama-7b INT4](#llama-7b-int4) - * [Llama-7b FP16 + WoQ INT8](#llama-7b-fp16-woq-int8) - * [Llama2-70b FP8 with TP=2](#llama2-70b-fp8-with-tp2) -- [Engine Plan File Size Results](#engine-plan-file-size-results) -- [Prototype](#prototype) - * [Checkpoint Pruner](#checkpoint-pruner) - * [Pruning a TensorRT LLM Checkpoint](#pruning-a-tensorrt-llm-checkpoint) - -## Overview - -This workflow introduces a new script `trtllm-refit`. `trtllm-refit` allows you to refit the generated engine with weights from any TensorRT LLM checkpoint matching the same architecture, so long as you build the engine as refittable or stripped. - -### Build Weights Stripped Engine -TensorRT can generate refittable engines with the same performance as the non-refittable ones when TensorRT builder optimize under the assumption that the engine will be refitted with weights identical to those provide at build time. Those refittable weights can be stripped to reduce the engine plan file size, with the option to subsequently supply them via the refit interface. - -New option `--strip_plan` is introduced in `trtllm-build` - -```bash -trtllm-build --strip_plan --checkpoint_dir ${CHECKPOINT_DIR} --output_dir ${ENGINE_DIR} ... -``` - -### Engine Refitter -The refitter allows you to refit an engine with weights in a TensorRT LLM checkpoint. It does this by doing a textual match between engine and checkpoint weight names. In order for the refitter to work, the engine must be built with refitting enabled. This can be accomplished by passing `--strip_plan` to `trtllm-build`. - -After building a stripped engine via `trtllm-build`, run - -```bash -trtllm-refit --checkpoint_dir ${CHECKPOINT_DIR} --engine_dir ${ENGINE_DIR} -``` - - -## Prerequisites - -Install [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM/blob/main/README.md) either through [pip](https://github.com/NVIDIA/TensorRT-LLM/blob/main/README.md#installation) or [from the source](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/installation/build-from-source.md). - -## Weight-Stripping Workflow Example - -### GPT-J - -1. Download the weights. -```bash -# 1. Weights & config -git clone https://huggingface.co/EleutherAI/gpt-j-6b -pushd gpt-j-6b && \ - rm -f pytorch_model.bin && \ - wget https://huggingface.co/EleutherAI/gpt-j-6b/resolve/main/pytorch_model.bin && \ -popd - -# 2. Vocab and merge table -wget https://huggingface.co/EleutherAI/gpt-j-6b/resolve/main/vocab.json -wget https://huggingface.co/EleutherAI/gpt-j-6b/resolve/main/merges.txt -``` - -2. Convert the Hugging Face checkpoint into TensorRT LLM format. -Run below command lines in [`examples/models/contrib/gpt`](../gptj) directory. -```bash -# Build a float16 checkpoint using HF weights. -python convert_checkpoint.py --model_dir ./gpt-j-6b \ - --dtype float16 \ - --output_dir ./trt_ckpt/gptj_fp16_tp1/ - -# Build an int8 weight-only checkpoint using HF weights. -python convert_checkpoint.py --model_dir ./gpt-j-6b \ - --dtype float16 \ - --use_weight_only \ - --weight_only_precision int8 \ - --output_dir ./trt_ckpt/gptj_int8_tp1/ - -``` - -3. Build the weights stripped engine. -```bash -# Build with --strip_plan. Requires TRT>=10.0.0 -trtllm-build --checkpoint_dir ./trt_ckpt/gptj_fp16_tp1/ \ - --output_dir ./trt_engines/gptj_fp16_tp1/ \ - --gemm_plugin float16 \ - --max_batch_size=32 \ - --max_input_len=1919 \ - --max_seq_len=2047 \ - --strip_plan -``` - -4. Refit the engine. The refit engine lives at `${ENGINE_DIR}.refit`. -```bash -# --checkpoint_dir points to the path of the weights you want refit, in this case the original weights. -trtllm-refit --checkpoint_dir ./trt_ckpt/gptj_fp16_tp1/ --engine_dir ./trt_engines/gptj_fp16_tp1/ --output_dir ./trt_engines/gptj_fp16_tp1.refit/ -``` - -5. Verify the engine. -```bash -# Run the summarization task. -python3 ../summarize.py --engine_dir ./trt_engines/gptj_fp16_tp1.refit \ - --hf_model_dir ./gpt-j-6b \ - --batch_size 1 \ - --test_trt_llm \ - --tensorrt_llm_rouge1_threshold 14 \ - --data_type fp16 \ - --check_accuracy -``` - -### Llama-7b INT4 - -1. Download the llama-7b-hf checkpoint and saved in /llm-models/llama-models/llama-7b-hf/. - -2. Calibrate the checkpoint and convert into TensorRT LLM format. -Run below command lines in [`examples/models/core/llama`](../models/core/llama) directory. -```bash -# Calibrate INT4 using AMMO. -python ../quantization/quantize.py --model_dir /llm-models/llama-models/llama-7b-hf/ \ - --dtype float16 \ - --qformat int4_awq \ - --awq_block_size 128 \ - --output_dir ./quantized_int4-awq \ - --calib_size 32 -``` - -3. Build the weights stripped engine. -```bash -# Build with --strip_plan. Requires TRT>=10.0.0 -trtllm-build --checkpoint_dir ./quantized_int4-awq \ - --strip_plan \ - --gemm_plugin float16 \ - --output_dir trt_int4_AWQ -``` - -4. Refit the engine. -```bash -trtllm-refit --checkpoint_dir ./quantized_int4-awq \ - --engine_dir trt_int4_AWQ \ - --output_dir trt_int4_AWQ_full_from_wtless -``` - -5. Verify the engine. -```bash -python3 ../summarize.py --engine_dir trt_int4_AWQ_full_from_wtless \ - --hf_model_dir /llm-models/llama-models/llama-7b-hf/ \ - --batch_size 1 \ - --test_trt_llm \ - --check_accuracy -``` - -### Llama-7b FP16 + WoQ INT8 - -1. Download the llama-7b-hf checkpoint and saved in /llm-models/llama-models/llama-7b-hf/. - -2. Convert the checkpoint into TensorRT LLM format. -Run below command lines in [`examples/models/core/llama`](../models/core/llama) directory. -```bash -python3 convert_checkpoint.py --model_dir /llm-models/llama-models/llama-7b-hf/ \ - --output_dir ./llama-7b-hf-fp16-woq \ - --dtype float16 \ - --use_weight_only \ - --weight_only_precision int8 -``` - -3. Build the weights stripped engine. -```bash -# Build with --strip_plan. Requires TRT>=10.0.0 -trtllm-build --checkpoint_dir ./llama-7b-hf-fp16-woq \ - --output_dir ./engines/llama-7b-hf-fp16-woq-1gpu-wtless \ - --strip_plan \ - --gemm_plugin float16 -``` - -4. Refit the engine. -```bash -trtllm-refit --checkpoint_dir ./llama-7b-hf-fp16-woq \ - --engine_dir ./engines/llama-7b-hf-fp16-woq-1gpu-wtless \ - --output_dir ./engines/llama-7b-hf-fp16-woq-1gpu-wtless-to-full -``` - -5. Verify the engine. -```bash -python3 ../summarize.py --engine_dir ./engines/llama-7b-hf-fp16-woq-1gpu-wtless-to-full \ - --hf_model_dir /llm-models/llama-models/llama-7b-hf/ \ - --batch_size 1 \ - --test_trt_llm \ - --check_accuracy -``` - - -### Llama2-70b FP8 with TP=2 - -1. Download the llama-v2-70b-hf checkpoint and saved in /llm-models/llama-models-v2/llama-v2-70b-hf/. - -2. Calibrate the checkpoint and convert into TensorRT LLM format. -Run below command lines in [`examples/models/core/llama`](../models/core/llama) directory. -```bash -# Calibrate FP8 using AMMO. -python ../quantization/quantize.py --model_dir /llm-models/llama-models-v2/llama-v2-70b-hf/ \ - --dtype float16 \ - --qformat fp8 \ - --kv_cache_dtype fp8 \ - --output_dir ./llama2-70b-hf-fp8-tp2 \ - --calib_size 512 \ - --tp_size 2 -``` - -3. Build the weights stripped engine. -```bash -trtllm-build --checkpoint_dir ./llama2-70b-hf-fp8-tp2 \ - --output_dir engines/llama2-70b-hf-fp8-tp2 \ - --gemm_plugin float16 \ - --workers 2 -``` - -4. Refit the engine. -```bash -trtllm-refit --checkpoint_dir ./llama2-70b-hf-fp8-tp2 \ - --engine_dir engines/llama2-70b-hf-fp8-tp2 \ - --output_dir engines/llama2-70b-hf-fp8-tp2.refit -``` - -5. Verify the engine. -```bash -python3 ../summarize.py --engine_dir engines/llama2-70b-hf-fp8-tp2.refit \ - --hf_model_dir /llm-models/llama-models-v2/llama-v2-70b-hf/ \ - --batch_size 1 \ - --test_trt_llm \ - --check_accuracy -``` - - -## Engine Plan File Size Results - -| **Model** | **Full Engine Plan Size** | **Weight-Stripped Engine Plan Size** | -|:---------:|:----------:|:----:| -|llama-7b INT4 | 3.7GB | 5.3MB | -|llama-7b FP16 + WoQ INT8 | 6.54GB | 28.69MB | -|llama2-70b FP8 + TP=2 | 64.78GB | 60.61MB | - -## Prototype -### Checkpoint Pruner -The checkpoint pruner allows you to strip `Conv` and `Gemm` weights out of a TensorRT LLM [checkpoint](https://nvidia.github.io/TensorRT-LLM/0.21.0/architecture/checkpoint.html). Since these make up the vast majority of weights, the pruner will decrease the size of your checkpoint up to 99%. - -When building an engine with a pruned checkpoint, TensorRT LLM fills in the missing weights with random ones. These weights should later be [refit](#engine-refitter) with the original weights to preserve the intended behavior. - -Building an engine from a pruned checkpoint will also allow the engine to be [refit](#engine-refitter). - -#### Pruning a TensorRT LLM Checkpoint - -1. Install [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM/blob/main/README.md) either through [pip](https://github.com/NVIDIA/TensorRT-LLM/blob/main/README.md#installation) or [from the source](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/installation/build-from-source.md). -2. Download a model of your choice and convert it to a TensorRT LLM checkpoint ([llama instructions](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/models/core/llama/README.md#usage)). -3. (Optional) Run the `trtllm-prune` command. -```bash -# Prunes the TRT-LLM checkpoint at ${CHECKPOINT_DIR}, and stores it in the directory ${CHECKPOINT_DIR}.pruned -trtllm-prune --checkpoint_dir ${CHECKPOINT_DIR} -``` - -The pruned checkpoint lives at `${CHECKPOINT_DIR}.pruned` by default, however, this can be overridden by issuing the `--out_dir` flag. - -4. Build the stripped engine. - -```bash -# From pruned checkpoint. -trtllm-build --checkpoint_dir ${CHECKPOINT_DIR}.pruned \ - --output_dir ${ENGINE_OUT_DIR} \ - ${EXTRA_ARGS} -``` diff --git a/examples/summarize.py b/examples/summarize.py deleted file mode 100644 index 1f6f8979bb7b..000000000000 --- a/examples/summarize.py +++ /dev/null @@ -1,944 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import ast -import os -from pathlib import Path - -import evaluate -import numpy as np -import torch -from datasets import load_dataset -from transformers import (AutoModel, AutoModelForCausalLM, - AutoModelForSeq2SeqLM, GenerationConfig) -from utils import (DEFAULT_HF_MODEL_DIRS, add_common_args, get_beam_width_array, - load_tokenizer, read_model_name, supports_inflight_batching) - -import tensorrt_llm -import tensorrt_llm.profiler as profiler -from tensorrt_llm._utils import mpi_broadcast, str_dtype_to_torch -from tensorrt_llm.builder import EngineConfig -from tensorrt_llm.functional import RopeEmbeddingUtils, RotaryScalingType -from tensorrt_llm.layers import MropeParams -from tensorrt_llm.logger import logger -from tensorrt_llm.models.qwen.utils import make_context -from tensorrt_llm.runtime import PYTHON_BINDINGS, ModelRunner -from tensorrt_llm.tools.ppl import ppl - -if PYTHON_BINDINGS: - from tensorrt_llm.runtime import ModelRunnerCpp - -from ngram.run_dtm_ngram import run_dtm_ngram - - -def ensemble_mrope_params(batch_input_ids, max_position_embeddings, - rotary_embedding_dim, theta): - mrope_params = MropeParams() - batch_size = len(batch_input_ids) - - _, rotary_cos_sin = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( - num_pos=max_position_embeddings, - dim=rotary_embedding_dim, - theta=1000000.0, - scale_type=RotaryScalingType.mrope, - ) - rotary_cos_sin = torch.tensor(rotary_cos_sin).to(batch_input_ids[0].device) - rotary_cos_sin = rotary_cos_sin.reshape(max_position_embeddings, - int(rotary_embedding_dim / 2), 2) - - cos_ori = rotary_cos_sin[:, :, 0] - sin_ori = rotary_cos_sin[:, :, 1] - - mrope_position_ids_padding = torch.zeros( - (batch_size, max_position_embeddings), dtype=torch.int32) - for i in range(batch_size): - seq_len = batch_input_ids[i].shape[-1] - mrope_position_ids_padding[i, :seq_len] = torch.arange( - seq_len, device=batch_input_ids[i].device) - - cos = cos_ori[mrope_position_ids_padding].unsqueeze(-1) - sin = sin_ori[mrope_position_ids_padding].unsqueeze(-1) - - mrope_params.mrope_rotary_cos_sin = torch.concatenate( - (cos, sin), axis=-1).reshape(batch_size, -1) - mrope_params.mrope_position_deltas = torch.zeros( - [batch_size, 1], device=batch_input_ids[0].device) - - return mrope_params - - -def main(args): - is_integration_test = os.getenv('INTEGRATION_TEST', '0') == '1' - if is_integration_test: - logger.info( - "Running in integration test mode - will only run one batch and skip accuracy checks" - ) - logger.info( - "Setting max_ite=1 and check_accuracy=False for integration test") - args.max_ite = 1 - args.check_accuracy = False - - runtime_rank = tensorrt_llm.mpi_rank() - logger.set_level(args.log_level) - - test_hf = args.test_hf and runtime_rank == 0 # only run hf on rank 0 - test_trt_llm = args.test_trt_llm - model_name, model_version = read_model_name( - args.engine_dir if not test_hf else args.hf_model_dir, test_hf) - if args.hf_model_dir is None: - logger.warning( - "hf_model_dir is not specified. Try to infer from model_name, but this may be incorrect." - ) - if model_name in DEFAULT_HF_MODEL_DIRS: - args.hf_model_dir = DEFAULT_HF_MODEL_DIRS[model_name] - else: - args.hf_model_dir = None - if args.tokenizer_dir is None: - args.tokenizer_dir = args.hf_model_dir - - profiler.start('load tokenizer') - tokenizer, pad_id, end_id = load_tokenizer( - tokenizer_dir=args.tokenizer_dir, - vocab_file=args.vocab_file, - model_name=model_name, - model_version=model_version, - tokenizer_type=args.tokenizer_type, - ) - profiler.stop('load tokenizer') - logger.info( - f'Load tokenizer takes: {profiler.elapsed_time_in_sec("load tokenizer")} sec' - ) - - if args.eval_task == 'code_completion': - dataset_name = "openai_humaneval" - dataset_revision = None - dataset_input_key = 'prompt' - dataset_output_key = 'canonical_solution' - dataset_split = 'test' - elif args.eval_task == 'summarize': - dataset_name = "ccdv/cnn_dailymail" - dataset_revision = "3.0.0" - dataset_input_key = 'article' - dataset_output_key = 'highlights' - dataset_split = 'test' - elif args.eval_task == 'summarize_long': - dataset_name = "tau/zero_scrolls" - dataset_revision = 'squality' - dataset_input_key = 'input' - dataset_output_key = 'output' - dataset_split = 'validation' # only this split contains reference strings - elif args.eval_task == "eval_context_ppl": - dataset_name = "SlimPajama-6B" - dataset_revision = None - dataset_input_key = 'text' - dataset_output_key = 'text' - dataset_split = 'test' - args.output_len = 1 # Only want to compute the ppl of context - args.eval_ppl = True - logger.warning( - f"Run task '{args.eval_task}', setting 'output_len' to 1, and enable 'eval_ppl'." - ) - if args.dataset_dir is not None and isinstance(args.dataset_dir, str): - args.dataset_dir = args.dataset_dir.rstrip('/') - if args.dataset_dir.endswith(dataset_name): - dataset_name = args.dataset_dir - else: - dataset_name = f"{args.dataset_dir}/{dataset_name}" - dataset = load_dataset(dataset_name, - dataset_revision, - cache_dir=args.dataset_cache_dir, - split=dataset_split, - trust_remote_code=True) - dataset = dataset.shuffle(args.random_seed) - - max_batch_size = args.batch_size - - # runtime parameters - top_k = args.top_k - top_p = args.top_p - output_len = args.output_len - test_token_num = args.max_input_length - max_attention_window_size = args.max_attention_window_size - sink_token_length = args.sink_token_length - - if args.end_id: - end_id = args.end_id - - stop_words_list = None - if args.stop_words: - stop_words_list = tensorrt_llm.runtime.decode_words_list( - args.stop_words, tokenizer) - if model_version == 'glm4': # add default stop token ids for GLM-4 - glm4_stop_ids = [[151329], [151336], [151338]] - if stop_words_list is None: - stop_words_list = [glm4_stop_ids] * args.batch_size - else: - for req_stop_words_list in stop_words_list: - req_stop_words_list.extend(glm4_stop_ids) - - bad_words_list = None - if args.bad_words: - bad_words_list = tensorrt_llm.runtime.decode_words_list( - args.bad_words, tokenizer) - - if args.beam_width_array is not None: - logger.info("Use Variable-Beam-Width-Search") - args.beam_width_array, args.num_beams = get_beam_width_array( - args.beam_width_array) - - num_beams = args.num_beams - num_return_sequences = args.num_return_sequences - num_sequences = args.num_return_sequences or num_beams - assert num_beams == 1 or num_sequences <= num_beams - - temperature = args.temperature - length_penalty = args.length_penalty - early_stopping = args.early_stopping - beam_width_array = args.beam_width_array - repetition_penalty = args.repetition_penalty - presence_penalty = args.presence_penalty - frequency_penalty = args.frequency_penalty - prompt_ignore_length = args.prompt_ignore_length - random_seed = args.random_seed - torch.manual_seed(random_seed) - - output_dir = Path(args.output_dir) if args.output_dir else None - if output_dir is not None: - output_dir.mkdir(exist_ok=True, parents=True) - if test_trt_llm: - with (output_dir / 'trtllm.out').open('w') as f: - f.write(f'Engine path: {args.engine_dir}\n') - f.write(f'Tokenizer path: {args.tokenizer_dir}\n') - if test_hf: - with (output_dir / 'hf.out').open('w') as f: - f.write(f'Model path: {args.hf_model_dir}\n') - f.write(f'Tokenizer path: {args.tokenizer_dir}\n') - - rouge_dir = args.rouge_dir if args.rouge_dir and os.path.exists( - args.rouge_dir) else "rouge" - metric_tensorrt_llm = [ - evaluate.load(rouge_dir) for _ in range(num_sequences) - ] - metric_hf = [evaluate.load(rouge_dir) for _ in range(num_sequences)] - for i in range(num_sequences): - metric_tensorrt_llm[i].seed = 0 - metric_hf[i].seed = 0 - ppls_trt_llm = [[] for _ in range(num_sequences)] - ppls_hf = [[] for _ in range(num_sequences)] - - def _prepare_inputs(batch_input_texts, - eval_task='summarize', - add_special_tokens=True, - min_input_length=0): - batch_size = len(batch_input_texts) - append_str = ' TL;DR: ' if eval_task == 'summarize' else '' - batch_input_ids = [] - for i in range(batch_size): - curr_text = batch_input_texts[i] + append_str - curr_text = curr_text.strip().replace(" n't", "n't") - - # TODO: The below lines are used to be compatible with the original code; may need fix - if 'GLM' in model_name and model_version in ('chatglm2', - 'chatglm3'): - input_ids = tokenizer.encode(curr_text, - return_tensors='pt').squeeze(0) - input_ids = input_ids[:test_token_num] - elif 'qwen' in model_name.lower() and model_version == 'qwen': - # use make_content to generate prompt - system_prompt = "You are a useful assistant, please directly output the corresponding summary according to the article entered by the user." - _, input_id_list = make_context( - tokenizer=tokenizer, - query=curr_text, - history=[], - system=system_prompt, - max_input_length=test_token_num, - ) - input_ids = torch.tensor(input_id_list) - else: - if 'qwen' in model_name.lower() and 'qwen2' in model_version: - messages = [{ - "role": - "system", - "content": - "You are a helpful assistant, please summarize the article entered by the user with one or two sentences." - }, { - "role": "user", - "content": curr_text - }] - curr_text = tokenizer.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True) - input_ids = tokenizer.encode( - curr_text, - return_tensors='pt', - add_special_tokens=add_special_tokens, - truncation=True, - max_length=test_token_num).squeeze(0) - - if input_ids.numel() > min_input_length: - batch_input_ids.append(input_ids) - return batch_input_ids - - def eval_trt_llm(datapoint, - eval_task='summarize', - eval_ppl=False, - add_special_tokens=True, - min_input_length=0, - runner=None): - batch_size = len(datapoint[dataset_input_key]) - batch_input_ids = _prepare_inputs(datapoint[dataset_input_key], - eval_task=eval_task, - add_special_tokens=add_special_tokens, - min_input_length=min_input_length) - # Generate mrope params for qwen model - engine_config = EngineConfig.from_json_file( - f"{args.engine_dir}/config.json") - pretrain_config = engine_config.pretrained_config - mrope_params = None - if 'qwen' in model_name.lower(): - mrope_params = ensemble_mrope_params( - batch_input_ids, - max_position_embeddings=pretrain_config.max_position_embeddings, - rotary_embedding_dim=pretrain_config.rotary_embedding_dim, - theta=pretrain_config.rotary_base, - ) - - if batch_size == 0 or len(batch_input_ids) == 0: - return [], [], [], {} - input_lengths = [x.size(0) for x in batch_input_ids] - - if args.ngram_config is not None: - # Speculative decoding of NGram - outputs = run_dtm_ngram(batch_input_ids, - args, - runtime_rank, - end_id, - pad_id, - stop_words_list, - bad_words_list, - tokenizer.vocab_size, - target_runner=runner) - if not args.streaming: # Unpack runner from the return value in No-Streaming mode - outputs, runner = list(outputs)[0] - else: # Normal run - with torch.no_grad(): - outputs = runner.generate( - batch_input_ids, - max_new_tokens=output_len, - max_attention_window_size=max_attention_window_size, - sink_token_length=sink_token_length, - end_id=end_id, - pad_id=pad_id, - temperature=temperature, - top_k=top_k, - top_p=top_p, - stop_words_list=stop_words_list, - bad_words_list=bad_words_list, - num_beams=num_beams, - num_return_sequences=num_return_sequences, - length_penalty=length_penalty, - early_stopping=early_stopping, - beam_width_array=beam_width_array, - repetition_penalty=repetition_penalty, - presence_penalty=presence_penalty, - frequency_penalty=frequency_penalty, - prompt_ignore_length=prompt_ignore_length, - lora_uids=args.lora_task_uids, - lookahead_config=args.lookahead_config, - output_sequence_lengths=True, - output_generation_logits=eval_ppl, - return_dict=True, - random_seed=random_seed, - medusa_choices=args.medusa_choices, - eagle_choices=args.eagle_choices, - mrope_params=mrope_params) - torch.cuda.synchronize() - - # Extract a list of tensors of shape beam_width x output_ids. - if runtime_rank == 0: - output_ids = outputs['output_ids'] - output_beams_list = [ - tokenizer.batch_decode(beam_tokens[:, input_lengths[i]:], - skip_special_tokens=True) - for i, beam_tokens in enumerate(output_ids) - ] - output_ids_list = [ - beam_tokens[:, input_lengths[i]:] - for i, beam_tokens in enumerate(output_ids) - ] - - ppls = [[] for _ in range(batch_size)] - lengths_info = { - 'input_lengths': input_lengths, - 'seq_lengths': outputs["sequence_lengths"].cpu().tolist(), - } - if eval_ppl: - seq_lengths = outputs['sequence_lengths'] - context_logits = outputs['context_logits'] - # Remove the first generation logits which are same to last - # context logits. - generation_logits = outputs['generation_logits'][:, :, 1:] - for batch_idx in range(batch_size): - # [batch, beam, step] - for beam_idx in range(num_sequences): - curr_len = seq_lengths[batch_idx, beam_idx] - curr_ctx_len = input_lengths[batch_idx] - curr_gen_len = curr_len - curr_ctx_len - - curr_ids = output_ids[batch_idx, beam_idx, 1:curr_len] - curr_logits = torch.cat([ - context_logits[batch_idx], - generation_logits[batch_idx, - beam_idx, :curr_gen_len - 1] - ], - dim=0) - curr_ppl = ppl(curr_logits, curr_ids) - logger.debug(f"TensorRT LLM PPL: {curr_ppl:.3f} | " - f"Generation length: {curr_gen_len}") - ppls[batch_idx].append(curr_ppl) - return output_beams_list, output_ids_list, ppls, lengths_info - return [], [], [], {} - - def eval_hf(datapoint, - eval_task='summarize', - eval_ppl=False, - add_special_tokens=True, - min_input_length=0): - batch_size = len(datapoint[dataset_input_key]) - if batch_size > 1: - logger.warning( - f"HF does not support batch_size > 1 to verify correctness due to padding. Current batch size is {batch_size}" - ) - batch_input_ids = _prepare_inputs(datapoint[dataset_input_key], - eval_task=eval_task, - add_special_tokens=add_special_tokens, - min_input_length=min_input_length) - batch_size = len(batch_input_ids) - if batch_size == 0: - return [], [], [], [[] for _ in range(batch_size)] - input_lengths = [x.size(0) for x in batch_input_ids] - # Left padding for HF - max_length = max(input_lengths) - paddings = [ - torch.ones(max_length - l, dtype=torch.int32) * pad_id - for l in input_lengths - ] - batch_input_ids = [ - torch.cat([pad, x]) for x, pad in zip(batch_input_ids, paddings) - ] - batch_input_ids = torch.stack(batch_input_ids) - batch_input_ids = batch_input_ids.cuda() - - # specialization for HF - if early_stopping in [0, 1]: - local_early_stopping = bool(early_stopping) - else: - local_early_stopping = "never" - - with torch.no_grad(): - hf_config = {} - if num_beams == 1: - hf_config.update({ - "top_k": top_k, - "top_p": top_p, - "do_sample": True, - }) - else: - hf_config.update({ - "num_beams": num_beams, - "early_stopping": local_early_stopping, - }) - - outputs = model.generate(batch_input_ids, - max_new_tokens=output_len, - num_return_sequences=num_sequences, - temperature=temperature, - eos_token_id=end_id, - pad_token_id=pad_id, - length_penalty=length_penalty, - output_scores=True, - return_dict_in_generate=True, - **hf_config) - if eval_ppl and batch_size == 1: - # model.generate cannot return context logits? - # Will cause additional latency - context_outputs = model(batch_input_ids) - - output_ids = outputs['sequences'] - tokens_list = output_ids[:, max_length:].tolist() - output_ids = output_ids.reshape([batch_size, num_sequences, -1]) - output_lines_list = [ - tokenizer.batch_decode(output_ids[:, i, max_length:], - skip_special_tokens=True) - for i in range(num_sequences) - ] - - ppls = [[] for _ in range(batch_size)] - if eval_ppl and batch_size == 1: - # Only for batch size of 1 - seq_lens = (output_ids - != end_id).logical_and(output_ids != pad_id).sum(dim=-1) - context_logits = context_outputs['logits'] - # Remove the first generation logits which are same to last context logits - generation_logits = outputs['scores'][1:] - # When output_len is 1, generation_logits would be () and lead to error if we do torch.stack - if len(generation_logits) == 0: - generation_logits = torch.empty( - [context_logits.shape[0], 0, context_logits.shape[-1]], - device=context_logits.device) - else: - generation_logits = torch.stack(generation_logits, dim=1) - _, max_gen_len, voc_size = generation_logits.size() - generation_logits = generation_logits.view(batch_size, num_beams, - max_gen_len, voc_size) - for batch_idx in range(batch_size): - for beam_idx in range(num_sequences): - curr_len = seq_lens[batch_idx, beam_idx] - curr_ctx_len = input_lengths[batch_idx] - curr_gen_len = curr_len - curr_ctx_len - - curr_ids = output_ids[batch_idx, beam_idx, 1:curr_len] - curr_logits = torch.cat([ - context_logits[batch_idx], - generation_logits[batch_idx, - beam_idx, :curr_gen_len - 1] - ], - dim=0) - curr_ppl = ppl(curr_logits, curr_ids) - logger.debug( - f"HF PPL: {curr_ppl:.3f} | Generation length: {curr_gen_len}" - ) - ppls[batch_idx].append(curr_ppl) - - return output_lines_list, tokens_list, ppls - - if test_trt_llm: - if not supports_inflight_batching(args.engine_dir): - logger.warning( - "The given engine does not support in-flight batching, fallback to python session" - ) - args.use_py_session = True - - if not PYTHON_BINDINGS and not args.use_py_session: - logger.warning( - "Python bindings of C++ session is unavailable, fallback to Python session." - ) - args.use_py_session = True - if args.return_all_generated_tokens: - raise ValueError( - "Returning all the generated tokens at each step is not supported in summarize.py" - ) - - logger.info( - f"Using {'Python' if args.use_py_session else 'C++'} session") - - runner_cls = ModelRunner if args.use_py_session else ModelRunnerCpp - runner_kwargs = dict( - engine_dir=args.engine_dir, - rank=runtime_rank, - debug_mode=args.debug_mode, - gpu_weights_percent=args.gpu_weights_percent, - enable_context_fmha_fp32_acc=args.enable_context_fmha_fp32_acc, - ) - if not args.use_py_session: - runner_kwargs.update( - lora_dir=args.lora_dir, - lora_ckpt_source=args.lora_ckpt_source, - max_batch_size=max_batch_size, - max_input_len=test_token_num, - max_output_len=output_len, - max_beam_width=num_beams, - max_attention_window_size=max_attention_window_size, - sink_token_length=sink_token_length, - max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, - kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, - kv_cache_free_gpu_memory_fraction=args. - kv_cache_free_gpu_memory_fraction, - enable_chunked_context=args.enable_chunked_context, - multi_block_mode=args.multi_block_mode, - cuda_graph_mode=args.cuda_graph_mode, - gather_generation_logits=args.eval_ppl, - use_gpu_direct_storage=args.use_gpu_direct_storage, - ) - - if args.medusa_choices is not None: - args.medusa_choices = ast.literal_eval(args.medusa_choices) - assert args.temperature == 1.0, "Medusa should use temperature == 1.0" - assert args.num_beams == 1, "Medusa should use num_beams == 1" - runner_kwargs.update(medusa_choices=args.medusa_choices) - if args.eagle_choices is not None or args.eagle_posterior_threshold is not None or args.eagle_use_dynamic_tree: - assert args.num_beams == 1, "Eagle should use num_beams == 1" - if args.eagle_choices is not None and not args.eagle_use_dynamic_tree: - args.eagle_choices = ast.literal_eval(args.eagle_choices) - runner_kwargs.update(eagle_choices=args.eagle_choices) - if args.eagle_posterior_threshold is not None: - runner_kwargs.update( - eagle_posterior_threshold=args.eagle_posterior_threshold) - if args.eagle_use_dynamic_tree: - runner_kwargs.update( - eagle_use_dynamic_tree=args.eagle_use_dynamic_tree) - assert args.eagle_dynamic_tree_max_top_k is not None and args.eagle_dynamic_tree_max_top_k > 0 - runner_kwargs.update(eagle_dynamic_tree_max_top_k=args. - eagle_dynamic_tree_max_top_k) - if args.lookahead_config is not None: - args.lookahead_config = ast.literal_eval(args.lookahead_config) - assert len( - args.lookahead_config - ) == 3, "Lookahead needs [max_window_size, max_ngram_size, max_verification_set_size]" - runner_kwargs.update(lookahead_config=args.lookahead_config) - if args.ngram_config is not None: - assert args.kv_cache_enable_block_reuse, "`--kv_cache_enable_block_reuse` must be specified in speculative decoding." - assert not args.use_py_session, "`--use_py_session` is not supported in Speculative decoding." - assert args.num_beams == 1, "`--num_beams>1` is not supported in Speculative decoding." - max_draft_len, _, target_device_list = ast.literal_eval( - args.ngram_config) - args.max_output_len = output_len # Specialization for NGram - runner_kwargs.update(is_orchestrator_mode=True, - device_ids=target_device_list, - max_input_len=test_token_num + max_draft_len + - output_len) - - runner = runner_cls.from_dir(**runner_kwargs) - assert not (args.eval_ppl and not runner.gather_context_logits), \ - "PPL evaluation requires engine built with gather_context_logits enabled" - - datapoint = dataset[0:1] - output, *_ = eval_trt_llm(datapoint, - eval_task=args.eval_task, - eval_ppl=args.eval_ppl, - add_special_tokens=args.add_special_tokens, - min_input_length=args.min_input_length, - runner=runner) - if runtime_rank == 0 and args.eval_task != "eval_context_ppl": - logger.info( - "---------------------------------------------------------") - logger.info("TensorRT LLM Generated: ") - logger.info(f" Input: {datapoint[dataset_input_key]}") - logger.info(f"\n Reference: {datapoint[dataset_output_key]}") - logger.info(f"\n Output: {output}") - logger.info( - "---------------------------------------------------------") - - ite_count = 0 - data_point_idx = 0 - total_output_token_count_trt_llm = 0 # only valid for runtime_rank == 0 - while (data_point_idx < len(dataset)) and (ite_count < args.max_ite): - if runtime_rank == 0: - logger.debug( - f"run data_point {data_point_idx} ~ {data_point_idx + max_batch_size}" - ) - datapoint = dataset[data_point_idx:(data_point_idx + - max_batch_size)] - - profiler.start('tensorrt_llm') - output_tensorrt_llm, output_ids_trt_llm, curr_ppls_trt_llm, lengths_info = eval_trt_llm( - datapoint, - eval_task=args.eval_task, - eval_ppl=args.eval_ppl, - add_special_tokens=args.add_special_tokens, - min_input_length=args.min_input_length, - runner=runner) - profiler.stop('tensorrt_llm') - - empty_batch = runtime_rank == 0 and len(output_tensorrt_llm) == 0 - empty_batch = mpi_broadcast(empty_batch, 0) - if empty_batch: - # No valid samples in the current batch, skip this iteration - data_point_idx += max_batch_size - continue - - if runtime_rank == 0: - input_lengths = lengths_info['input_lengths'] - seq_lengths = lengths_info['seq_lengths'] - output_token_count_trt_llm = sum( - beam_len - input_lengths[batch_idx] - for batch_idx, beam_lens in enumerate(seq_lengths) - for beam_len in beam_lens) - total_output_token_count_trt_llm += output_token_count_trt_llm - for batch_idx, output_beams in enumerate(output_tensorrt_llm): - reference = datapoint[dataset_output_key][batch_idx] - for beam_idx, output_beam in enumerate(output_beams): - metric_tensorrt_llm[beam_idx].add_batch( - predictions=[output_beam], references=[reference]) - if args.eval_ppl: - ppls_trt_llm[beam_idx].append( - curr_ppls_trt_llm[batch_idx][beam_idx]) - if output_dir is not None: - for i in range(len(output_tensorrt_llm[0])): - for beam_idx in range(num_sequences): - with (output_dir / 'trtllm.out').open('a') as f: - f.write( - f'[{data_point_idx + i}] [Beam {beam_idx}] {output_tensorrt_llm[beam_idx][i]}\n' - ) - - logger.debug('-' * 100) - logger.debug(f"Input: {datapoint[dataset_input_key]}") - logger.debug(f'TensorRT LLM Output: {output_tensorrt_llm}') - logger.debug(f"Reference: {datapoint[dataset_output_key]}") - - data_point_idx += max_batch_size - ite_count += 1 - del runner - - if test_hf and runtime_rank == 0: - profiler.start('load HF model') - dtype_alias_mapping = { - 'fp32': 'float32', - 'fp16': 'float16', - 'bf16': 'bfloat16' - } - args.hf_data_type = dtype_alias_mapping.get(args.hf_data_type, - args.hf_data_type) - if 'GLM' in model_name and model_version == 'glm': - auto_model_cls = AutoModelForSeq2SeqLM - elif 'GLM' in model_name and model_version == 'chatglm': - auto_model_cls = AutoModel - else: - auto_model_cls = AutoModelForCausalLM - # TODO: args.hf_device_map_auto is not being correctly set - # remove in future version - if model_name == 'DeepseekV2ForCausalLM': - args.hf_device_map_auto = True - model = auto_model_cls.from_pretrained( - args.hf_model_dir, - trust_remote_code=True, - dtype=str_dtype_to_torch(args.hf_data_type), - device_map='auto' if args.hf_device_map_auto else None) - try: - model.to_bettertransformer() - except Exception as e: - logger.warning( - f'Fail to call model.to_bettertransformer(), exception:\n{str(e)}' - ) - if not args.hf_device_map_auto: - model.cuda() - if model_name == 'qwen': - model.generation_config = GenerationConfig.from_pretrained( - args.hf_model_dir, trust_remote_code=True) - profiler.stop('load HF model') - logger.info( - f'Load HF model takes: {profiler.elapsed_time_in_sec("load HF model")} sec' - ) - - datapoint = dataset[0:1] - output, *_ = eval_hf(datapoint, - eval_task=args.eval_task, - eval_ppl=args.eval_ppl, - add_special_tokens=args.add_special_tokens, - min_input_length=args.min_input_length) - if runtime_rank == 0 and args.eval_task != "eval_context_ppl": - logger.info( - "---------------------------------------------------------") - logger.info("HF Generated: ") - logger.info(f" Input: {datapoint[dataset_input_key]}") - logger.info(f"\n Reference: {datapoint[dataset_output_key]}") - logger.info(f"\n Output: {output}") - logger.info( - "---------------------------------------------------------") - - ite_count = 0 - data_point_idx = 0 - total_output_token_count_hf = 0 # only valid for runtime_rank == 0 - while (data_point_idx < len(dataset)) and (ite_count < args.max_ite): - if runtime_rank == 0: - logger.debug( - f"run data_point {data_point_idx} ~ {data_point_idx + max_batch_size}" - ) - datapoint = dataset[data_point_idx:(data_point_idx + - max_batch_size)] - - profiler.start('hf') - output_hf, token_list, curr_ppls_hf = eval_hf( - datapoint, - eval_task=args.eval_task, - eval_ppl=args.eval_ppl, - add_special_tokens=args.add_special_tokens, - min_input_length=args.min_input_length) - profiler.stop('hf') - - # HF model runs on rank 0 only - empty_batch = len(output_hf) == 0 - if empty_batch: - # No valid samples in the current batch, skip this iteration - data_point_idx += max_batch_size - continue - - if runtime_rank == 0: - seq_lengths = [len(tokens) for tokens in token_list] - total_output_token_count_hf += sum(seq_lengths) - for beam_idx in range(num_sequences): - for batch_idx in range(len(output_hf[beam_idx])): - metric_hf[beam_idx].add_batch( - predictions=[output_hf[beam_idx][batch_idx]], - references=[ - datapoint[dataset_output_key][batch_idx] - ]) - if args.eval_ppl and args.batch_size == 1: - ppls_hf[beam_idx].append( - curr_ppls_hf[batch_idx][beam_idx]) - if output_dir is not None: - for i in range(len(output_hf[0])): - for beam_idx in range(num_sequences): - with (output_dir / 'hf.out').open('a') as f: - f.write( - f'[{data_point_idx + i}] [Beam {beam_idx}] {output_hf[beam_idx][i]}\n' - ) - - logger.debug('-' * 100) - logger.debug(f"Input: {datapoint[dataset_input_key]}") - logger.debug(f'HF Output: {output_hf}') - logger.debug(f"Reference: {datapoint[dataset_output_key]}") - - data_point_idx += max_batch_size - ite_count += 1 - del model - - if runtime_rank == 0 and args.max_ite > 0: - if test_trt_llm: - np.random.seed(0) # rouge score use sampling to compute the score - logger.info( - f'TensorRT LLM (total latency: {profiler.elapsed_time_in_sec("tensorrt_llm")} sec)' - ) - - logger.info( - f'TensorRT LLM (total output tokens: {total_output_token_count_trt_llm})' - ) - logger.info( - f'TensorRT LLM (tokens per second: {total_output_token_count_trt_llm / profiler.elapsed_time_in_sec("tensorrt_llm")})' - ) - for beam_idx in range(num_sequences): - logger.info(f"TensorRT LLM beam {beam_idx} result") - if args.eval_task != "eval_context_ppl": - if args.estimate_accuracy_std_dev: - computed_metrics_tensorrt_llm = metric_tensorrt_llm[ - beam_idx].compute(use_aggregator=False) - computed_std_dev_tensorrt_llm = { - key: np.std(scores) - for key, scores in - computed_metrics_tensorrt_llm.items() - } - computed_metrics_tensorrt_llm = { - key: np.mean(scores) - for key, scores in - computed_metrics_tensorrt_llm.items() - } - for key in computed_metrics_tensorrt_llm.keys(): - logger.info( - f" {key}: {computed_metrics_tensorrt_llm[key]*100} ({computed_std_dev_tensorrt_llm[key]*100})" - ) - else: - computed_metrics_tensorrt_llm = metric_tensorrt_llm[ - beam_idx].compute() - for key in computed_metrics_tensorrt_llm.keys(): - logger.info( - f" {key}: {computed_metrics_tensorrt_llm[key]*100}" - ) - if args.check_accuracy and beam_idx == 0: - rouge1 = computed_metrics_tensorrt_llm['rouge1'] * 100 - assert rouge1 > args.tensorrt_llm_rouge1_threshold, f"[FAILED] rouge1 ({rouge1}) is smaller than threshold ({args.tensorrt_llm_rouge1_threshold})." - if args.eval_ppl: - logger.info( - f" Per-token perplexity: {np.mean(ppls_trt_llm[beam_idx])}" - ) - if args.check_accuracy and beam_idx == 0: - avg_ppl = np.mean(ppls_trt_llm[beam_idx]) - assert avg_ppl < args.tensorrt_llm_ppl_threshold, f"[FAILED] average PPL ({avg_ppl}) is larger than threshold ({args.tensorrt_llm_ppl_threshold})." - if test_hf: - np.random.seed(0) # rouge score use sampling to compute the score - logger.info( - f'Hugging Face (total latency: {profiler.elapsed_time_in_sec("hf")} sec)' - ) - logger.info( - f'Hugging Face (total output tokens: {total_output_token_count_hf})' - ) - logger.info( - f'Hugging Face (tokens per second: {total_output_token_count_hf / profiler.elapsed_time_in_sec("hf")})' - ) - - for beam_idx in range(num_sequences): - logger.info(f"HF beam {beam_idx} result") - computed_metrics_hf = metric_hf[beam_idx].compute() - if args.eval_task != "eval_context_ppl": - for key in computed_metrics_hf.keys(): - logger.info(f' {key}: {computed_metrics_hf[key]*100}') - if args.eval_ppl and args.batch_size == 1: - logger.info( - f" Per-token perplexity: {np.mean(ppls_hf[beam_idx])}") - - -if __name__ == '__main__': - # see `add_common_args` for extended list of arguments - parser = argparse.ArgumentParser() - parser.add_argument('--test_hf', action='store_true') - parser.add_argument('--test_trt_llm', action='store_true') - parser.add_argument('--eval_task', - type=str, - default='summarize', - choices=[ - 'summarize', 'summarize_long', 'code_completion', - 'eval_context_ppl' - ]) - parser.add_argument('--check_accuracy', action='store_true') - parser.add_argument('--estimate_accuracy_std_dev', action='store_true') - parser.add_argument('--tensorrt_llm_rouge1_threshold', - type=float, - default=15.0) - parser.add_argument('--eval_ppl', action='store_true') - parser.add_argument('--tensorrt_llm_ppl_threshold', - type=float, - default=15.0) - parser.add_argument( - '--dataset_dir', - type=str, - default=None, - help="The local directory of the dataset for evaluation; " - "will download the dataset from huggingface hub if not specified.") - parser.add_argument( - '--dataset_cache_dir', - type=str, - default=None, - help="The local cache directory for dataset; " - "will use `~/.cache/huggingface/datasets` if not specified.") - parser.add_argument('--batch_size', type=int, default=1) - parser.add_argument('--max_ite', type=int, default=20) - parser.add_argument('--output_len', type=int, default=100) - parser.add_argument('--max_input_length', type=int, default=923) - parser.add_argument( - '--min_input_length', - type=int, - default=0, - help='skip the sentences which are shorter than min_input_length.') - parser.add_argument( - '--output_dir', - type=str, - default=None, - help="Directory where to save output sentences. 'trtllm.out' for " - "TensorRT LLM outputs, and 'hf.out' for HF outputs. If None, do not " - "save outputs.") - parser.add_argument( - '--rouge_dir', - default=None, - type=str, - help= - "evaluate.load('rouge') will attempt to pull rouge package from HF. Use cached rouge can avoid network outage of host or HF." - ) - parser.add_argument("--use_gpu_direct_storage", - default=False, - action="store_true", - help="Use GPUDirect Storage (GDS) to load the engine") - parser = add_common_args(parser) - args = parser.parse_args() - - main(args) diff --git a/examples/utils.py b/examples/utils.py deleted file mode 100644 index c75cd255850f..000000000000 --- a/examples/utils.py +++ /dev/null @@ -1,658 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import ast -import json -import os -import subprocess -import sys -from argparse import BooleanOptionalAction -from functools import partial -from pathlib import Path -from typing import List, Optional - -import torch -from transformers import AutoTokenizer, LlamaTokenizer - -from tensorrt_llm._utils import supports_inflight_batching # noqa -from tensorrt_llm._utils import (mpi_barrier, mpi_rank, mpi_world_size, - str_dtype_to_torch) -from tensorrt_llm.builder import get_engine_version - - -class SentencePieceTokenizer: - """Minimal SentencePiece-backed tokenizer with a transformers-like API. - - transformers v5 replaced the pure-Python SentencePiece backend of - ``T5Tokenizer`` / ``LlamaTokenizer`` with the Rust ``tokenizers`` backend, - so instantiating them with a raw SentencePiece ``.model`` vocab file no - longer reads the underlying vocabulary. This wrapper preserves the old - behavior for NEMO-style checkpoints (e.g. gpt-next) by delegating to - ``sentencepiece.SentencePieceProcessor`` directly. - """ - - def __init__(self, - vocab_file: str, - padding_side: str = 'left', - truncation_side: str = 'left'): - import sentencepiece as spm - sp = spm.SentencePieceProcessor() - sp.Load(vocab_file) - self.sp_model = sp - self.padding_side = padding_side - self.truncation_side = truncation_side - self.vocab_size = sp.GetPieceSize() - - def _opt(i: int) -> int | None: - return i if i >= 0 else None - - self.pad_token_id = _opt(sp.pad_id()) - self.eos_token_id = _opt(sp.eos_id()) - self.bos_token_id = _opt(sp.bos_id()) - self.unk_token_id = _opt(sp.unk_id()) - - def encode(self, - text: str, - return_tensors: Optional[str] = None, - add_special_tokens: bool = True, - truncation: bool = False, - max_length: Optional[int] = None, - **kwargs): - ids = self.sp_model.EncodeAsIds(text) - if add_special_tokens and self.bos_token_id is not None: - ids = [self.bos_token_id] + ids - if truncation and max_length is not None and len(ids) > max_length: - ids = ids[ - -max_length:] if self.truncation_side == 'left' else ids[: - max_length] - if return_tensors == 'pt': - return torch.tensor([ids], dtype=torch.long) - return ids - - def decode(self, ids, skip_special_tokens: bool = False, **kwargs) -> str: - if isinstance(ids, torch.Tensor): - ids = ids.tolist() - if skip_special_tokens: - special = {self.pad_token_id, self.eos_token_id, self.bos_token_id} - ids = [t for t in ids if t not in special] - return self.sp_model.DecodeIds(list(ids)) - - def batch_decode(self, - sequences, - skip_special_tokens: bool = False, - **kwargs) -> List[str]: - if isinstance(sequences, torch.Tensor): - sequences = sequences.tolist() - return [ - self.decode(seq, skip_special_tokens=skip_special_tokens) - for seq in sequences - ] - - -DEFAULT_HF_MODEL_DIRS = { - 'BaichuanForCausalLM': 'baichuan-inc/Baichuan-13B-Chat', - 'BaiChuanForCausalLM': 'baichuan-inc/Baichuan-13B-Chat', - 'BloomForCausalLM': 'bigscience/bloom-560m', - 'GLMModel': 'THUDM/glm-10b', - 'ChatGLMModel': 'THUDM/chatglm3-6b', - 'ChatGLMForCausalLM': 'THUDM/chatglm3-6b', - 'RWForCausalLM': 'tiiuae/falcon-rw-1b', - 'FalconForCausalLM': 'tiiuae/falcon-rw-1b', - 'GPT2LMHeadModel': 'gpt2', - 'GPT2LMHeadCustomModel': 'gpt2', - 'Starcoder2ForCausalLM': 'bigcode/starcoder2-3b', - 'GPTForCausalLM': 'gpt2', - 'GPTJForCausalLM': 'EleutherAI/gpt-j-6b', - 'GPTNeoXForCausalLM': 'EleutherAI/gpt-neox-20b', - 'InternLMForCausalLM': 'internlm/internlm-chat-7b', - 'InternLM2ForCausalLM': 'internlm/internlm2-chat-7b', - 'LlamaForCausalLM': 'meta-llama/Llama-2-7b-hf', - 'MPTForCausalLM': 'mosaicml/mpt-7b', - 'PhiForCausalLM': 'microsoft/phi-2', - 'OPTForCausalLM': 'facebook/opt-350m', - 'QWenLMHeadModel': 'Qwen/Qwen-7B', - 'QWenForCausalLM': 'Qwen/Qwen-7B', - 'Qwen2ForCausalLM': 'Qwen/Qwen1.5-7B', - 'Qwen2MoeForCausalLM': 'Qwen/Qwen1.5-MoE-A2.7B', - 'RecurrentGemmaForCausalLM': 'google/recurrentgemma-2b', -} - -INTERNLM_META_INSTRUCTION = """You are an AI assistant whose name is InternLM (书生·浦语). -- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless. -- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文. -""" - -QWEN_PROMPT_TEMPLATE = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{input_text}<|im_end|>\n<|im_start|>assistant\n" - -DEFAULT_PROMPT_TEMPLATES = { - 'InternLMForCausalLM': "<|User|>:{input_text}\n<|Bot|>:", - 'InternLM2ForCausalLM': "<|im_start|>system\n" + INTERNLM_META_INSTRUCTION + - "<|im_end|>\n<|im_start|>user\n{input_text}<|im_end|>\n<|im_start|>assistant\n", - 'QWenLMHeadModel': QWEN_PROMPT_TEMPLATE, - 'QWenForCausalLM': QWEN_PROMPT_TEMPLATE, - 'Qwen2ForCausalLM': QWEN_PROMPT_TEMPLATE, - 'Qwen2MoeForCausalLM': QWEN_PROMPT_TEMPLATE, -} - - -def read_decoder_start_token_id(engine_dir): - with open(Path(engine_dir) / "config.json", 'r') as f: - config = json.load(f) - return config['pretrained_config']['decoder_start_token_id'] - - -def read_is_enc_dec(engine_dir: str, is_hf: bool = False): - if is_hf: - with open(Path(engine_dir) / "config.json", 'r') as f: - config = json.load(f) - is_enc_dec = config.get('is_encoder_decoder', False) - else: - is_enc_dec = {'encoder', 'decoder'}.issubset({ - name - for name in os.listdir(engine_dir) - if os.path.isdir(os.path.join(engine_dir, name)) - }) - return is_enc_dec - - -def read_model_name(engine_dir: str, is_hf: bool = False): - with open(Path(engine_dir) / "config.json", 'r') as f: - config = json.load(f) - - if is_hf: - model_arch = config['architectures'][0] - model_version = config.get('model_type', None) - return model_arch, model_version - - engine_version = get_engine_version(engine_dir) - if engine_version is None: - return config['builder_config']['name'], None - - model_arch = config['pretrained_config']['architecture'] - model_version = None - if 'GLM' in model_arch: - model_version = config['pretrained_config']['chatglm_version'] - if 'qwen' in model_arch.lower(): - model_version = config['pretrained_config']['qwen_type'] - return model_arch, model_version - - -def throttle_generator(generator, stream_interval): - for i, out in enumerate(generator): - if not i % stream_interval: - yield out - - if i % stream_interval: - yield out - - -# Load tokenizer impl, it will be called in external wrapper to avoid loading tokenizer bug under MPI env. -def _load_tokenizer(tokenizer_dir: Optional[str] = None, - vocab_file: Optional[str] = None, - model_name: str = 'GPTForCausalLM', - model_version: Optional[str] = None, - tokenizer_type: Optional[str] = None): - if vocab_file is None: - if 'whisper' in model_name.lower(): - tokenizer = AutoTokenizer.from_pretrained( - tokenizer_dir or 'openai/whisper-large-v3', - language='english', - task='transcribe', - predict_timestamps=False, - ) - elif tokenizer_type == 'language_adapter': - tokenizer = None - else: - use_fast = True - if tokenizer_type is not None and tokenizer_type == "llama": - use_fast = False - # Should set both padding_side and truncation_side to be 'left' - tokenizer = AutoTokenizer.from_pretrained( - tokenizer_dir, - legacy=False, - padding_side='left', - truncation_side='left', - trust_remote_code=True, - tokenizer_type=tokenizer_type, - use_fast=use_fast) - elif model_name == 'GemmaForCausalLM' or model_name == 'RecurrentGemmaForCausalLM': - from transformers import GemmaTokenizer - - # Initialize tokenizer from vocab file. - tokenizer = GemmaTokenizer(vocab_file=vocab_file, - padding_side='left', - truncation_side='left', - legacy=False) - elif model_name == 'Grok1ModelForCausalLM': - tokenizer = LlamaTokenizer(vocab_file=vocab_file, - padding_side='left', - truncation_side='left', - legacy=False, - use_fast=False) - else: - # For gpt-next, directly load from the SentencePiece ``tokenizer.model`` - # file. transformers v5 removed the pure-Python SentencePiece backend, - # so ``T5Tokenizer(vocab_file=...)`` no longer reads the vocabulary and - # reports vocab_size=104 with all tokens decoding to . - tokenizer = SentencePieceTokenizer(vocab_file=vocab_file, - padding_side='left', - truncation_side='left') - if 'qwen' in model_name.lower() and model_version == 'qwen': - with open(Path(tokenizer_dir) / "generation_config.json") as f: - gen_config = json.load(f) - pad_id = gen_config['pad_token_id'] - end_id = gen_config['eos_token_id'] - elif 'GLM' in model_name and model_version == 'glm': - pad_id = tokenizer.pad_token_id - end_id = tokenizer.eop_token_id - elif tokenizer_type == 'language_adapter': - pad_id = 0 - end_id = 2 - else: - if tokenizer.pad_token_id is None: - tokenizer.pad_token_id = tokenizer.eos_token_id - pad_id = tokenizer.pad_token_id - end_id = tokenizer.eos_token_id - - return tokenizer, pad_id, end_id - - -def load_tokenizer(tokenizer_dir: Optional[str] = None, - vocab_file: Optional[str] = None, - model_name: str = 'GPTForCausalLM', - model_version: Optional[str] = None, - tokenizer_type: Optional[str] = None): - func = partial(_load_tokenizer, tokenizer_dir, vocab_file, model_name, - model_version, tokenizer_type) - if mpi_world_size() > 1: - # Under MPI env, load tokenizer will result in multiple processes to download the same file to the same folder. - # This will result some random bug. Force loading on rank0 to warmup the tokenizer to avoid this issue. - if mpi_rank() == 0: - func() - mpi_barrier() - return func() - - -def prepare_enc_dec_inputs(batch_input_ids: List[torch.Tensor], model_name: str, - engine_dir: str, - multimodal_input_file: Optional[str]): - encoder_input_features = None - encoder_input_ids = None - if 'whisper' in model_name.lower(): - # cannot directly import whisper due to name collision - sys.path.append(f"{os.path.dirname(__file__)}/models/core/whisper") - from whisper_utils import log_mel_spectrogram - - config_path = os.path.join(engine_dir, 'encoder', 'config.json') - with open(config_path, 'r') as f: - config = json.load(f) - n_mels = config['pretrained_config']['n_mels'] - dtype = config['pretrained_config']['dtype'] - - # download mel filters file - subprocess.run([ - "wget", "-nc", f"--directory-prefix={engine_dir}", - "https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz" - ], - check=True) - - mel, total_duration = log_mel_spectrogram(multimodal_input_file, - n_mels, - return_duration=True, - mel_filters_dir=engine_dir) - mel = mel.type(str_dtype_to_torch(dtype)) # [featureDim, seqLen] - decoder_input_ids = batch_input_ids - encoder_input_features = [torch.einsum('DL->LD', mel)] - encoder_output_lengths = [encoder_input_features[0].shape[0] // 2] - else: - encoder_input_ids = batch_input_ids - decoder_start_token_id = read_decoder_start_token_id( - os.path.join(engine_dir, "decoder")) - decoder_input_ids = [ - torch.tensor([decoder_start_token_id], dtype=torch.int32) - for _ in batch_input_ids - ] - encoder_output_lengths = None - return encoder_input_ids, encoder_input_features, encoder_output_lengths, decoder_input_ids - - -def get_beam_width_array(bwa: str = None): - bwa = ast.literal_eval(bwa) # Short for "beam_width_array" - if isinstance(bwa, str): - bwa = ast.literal_eval(bwa) # parse again for string - - def parse_one_bwa(row): - assert isinstance(row, list), f"Beam width array must be a list." - assert len( - row - ) <= 8, "Length of beam width array must not be greater than 8 now." - assert all([isinstance(beam, int) for beam in row - ]), "Numbers in beam width array must be integer." - bwa_tensor = torch.zeros([8], dtype=torch.int32) - for j in range(len(row)): - bwa_tensor[j] = row[j] - bwa_tensor[len(row):] = row[-1] - return bwa_tensor, max(row) - - if isinstance(bwa, list): # Only one BWA - bwa_tensor, max_beam_width = parse_one_bwa(bwa) - elif isinstance(bwa, tuple): # BWA for respective requests - bwa_tensor_list = [] - max_beam_width = 0 - for row in bwa: - bwa_tensor, beam_width = parse_one_bwa(row) - bwa_tensor_list.append(bwa_tensor) - max_beam_width = max(max_beam_width, beam_width) - bwa_tensor = torch.stack(bwa_tensor_list, dim=0) - else: - raise ValueError(f"Invalid beam width array: {bwa}") - - return bwa_tensor.tolist(), max_beam_width - - -def add_common_args(parser): - # sampling arguments - parser.add_argument('--num_beams', - type=int, - help="Use beam search if num_beams > 1", - default=1) - parser.add_argument('--num_return_sequences', - type=int, - help="Number of sequences to generate for each input.", - default=None) - parser.add_argument('--temperature', type=float, default=1.0) - parser.add_argument('--top_k', type=int, default=1) - parser.add_argument('--top_p', type=float, default=0.0) - parser.add_argument('--length_penalty', type=float, default=1.0) - parser.add_argument('--repetition_penalty', type=float, default=1.0) - parser.add_argument('--presence_penalty', type=float, default=0.0) - parser.add_argument('--frequency_penalty', type=float, default=0.0) - parser.add_argument('--prompt_ignore_length', type=int, default=0) - parser.add_argument('--min_p', type=float, default=0.0) - parser.add_argument('--beam_search_diversity_rate', type=float, default=0.0) - parser.add_argument('--random_seed', type=int, default=0) - parser.add_argument('--early_stopping', - type=int, - help='Use early stopping if num_beams > 1, ' - '1 for early-stopping, 0 for non-early-stopping' - 'other values for stopping by length', - default=1) - parser.add_argument( - '--beam_width_array', - type=str, - default=None, - help= - 'Beam width array for each step. E.g.: --beam_width_array="[2,4,6,8]"', - ) - parser.add_argument( - '--end_id', - default=None, - type=int, - help="Override tokenizer end_id to stop on given end_id token.") - parser.add_argument( - '--stop_words', - default=None, - type=str, - nargs="+", - action='append', - help= - 'Set stop words for a batch. Successive invocations of --stop_words set stop words for other batches.' - ' E.g.: --stop_words " London" " chef" --stop_words "eventually became" "was not"', - ) - parser.add_argument( - '--bad_words', - default=None, - type=str, - nargs="+", - action='append', - help= - 'Set bad words for a batch. Successive invocations of --bad_words set bad words for other batches.' - ' E.g.: --bad_words " London" " chef" --bad_words "eventually became" "was not"', - ) - parser.add_argument('--no_repeat_ngram_size', type=int, default=None) - - # common runtime arguments - parser.add_argument('--sink_token_length', - type=int, - default=None, - help='The sink token length.') - parser.add_argument( - '--max_attention_window_size', - type=int, - default=None, - nargs="+", - help= - 'The attention window size that controls the sliding window attention kv cache behavior' - ) - parser.add_argument( - '--multi_block_mode', - type=lambda s: s.lower() in - ("yes", "true", "t", "1" - ), # custom boolean function to convert input string to boolean - default=True, - help= - "Distribute the work across multiple CUDA thread-blocks on the GPU for masked MHA kernel." - ) - parser.add_argument('--enable_context_fmha_fp32_acc', - action='store_true', - help="Enable FMHA runner FP32 accumulation.") - parser.add_argument('--cuda_graph_mode', - action='store_true', - help="Enable cuda graphs in the inference.") - parser.add_argument( - '--log_level', - type=str, - choices=['verbose', 'info', 'warning', 'error', 'internal_error'], - default='info') - parser.add_argument( - '--no_prompt_template', - dest='use_prompt_template', - default=True, - action='store_false', - help= - "Whether or not to use default prompt template to wrap the input text.") - parser.add_argument('--use_py_session', - default=False, - action='store_true', - help="Whether or not to use Python runtime session") - parser.add_argument('--debug_mode', - default=False, - action='store_true', - help="Whether or not to turn on the debug mode") - parser.add_argument('--streaming', default=False, action='store_true') - parser.add_argument('--streaming_interval', - type=int, - help="How often to return tokens when streaming.", - default=5) - parser.add_argument( - '--prompt_table_path', - type=str, - help="Path to .npy file, exported by nemo_prompt_convert.py") - parser.add_argument( - '--prompt_tasks', - help="Comma-separated list of tasks for prompt tuning, e.g., 0,3,1,0") - parser.add_argument('--lora_dir', - type=str, - default=None, - nargs="+", - help="The directory of LoRA weights") - parser.add_argument('--lora_ckpt_source', - type=str, - default="hf", - choices=["hf", "nemo"], - help="The source of lora checkpoint.") - parser.add_argument( - '--lora_task_uids', - type=str, - default=None, - nargs="+", - help="The list of LoRA task uids; use -1 to disable the LoRA module") - parser.add_argument( - '--num_prepend_vtokens', - nargs="+", - type=int, - help="Number of (default) virtual tokens to prepend to each sentence." - " For example, '--num_prepend_vtokens=10' will prepend the tokens" - " [vocab_size, vocab_size + 1, ..., vocab_size + 9] to the sentence.") - parser.add_argument( - '--draft_target_model_config', - type=str, - default=None, - help= - "Configuration of Draft-Target-Model decoding, see `examples/draft_target_model/README.md` for more information." - " E.g.: [4, [0], [1], False] for [draft_len, draft_model_device_list, target_model_device_list, use_logits]." - ) - parser.add_argument( - '--ngram_config', - type=str, - default=None, - help= - "Configuration of NGram decoding, see `examples/ngram/README.md` for more information." - " E.g.: [10,2,[0]] for [max_draft_len, max_matching_ngram_size, device_list].", - ) - parser.add_argument( - '--medusa_choices', - type=str, - default=None, - help="Configuration of Medusa decoding." - " E.g.: [[0, 0, 0, 0], [0, 1, 0], [1, 0], [1, 1]] for 9 medusa tokens." - ) - parser.add_argument( - '--eagle_choices', - type=str, - default=None, - help="Configuration of Eagle-1 decoding." - " E.g.: [[0, 0, 0, 0], [0, 1, 0], [1, 0], [1, 1]] for 9 draft tokens." - ) - parser.add_argument( - '--eagle_posterior_threshold', - type=float, - default=None, - help="Minimum token probability threshold for typical acceptance. " - "Enables typical acceptance in Eagle. " - "Corresponds to epsilon in https://arxiv.org/pdf/2401.10774.") - parser.add_argument('--eagle_use_dynamic_tree', - action='store_true', - help="Whether to use Ealge-2") - parser.add_argument( - '--eagle_dynamic_tree_max_top_k', - default=None, - type=int, - help= - "The maximum number of draft tokens to expand for each node in Eagle-2." - ) - parser.add_argument( - '--lookahead_config', - type=str, - default=None, - help="Configuration of executor and request lookahead decoding." - " E.g.: [5, 6, 7] for [max_window_size, max_ngram_size, max_verification_set_size]." - ) - # model arguments - parser.add_argument('--engine_dir', type=str, default='engine_outputs') - parser.add_argument( - '--tokenizer_type', - help= - 'Specify that argument when providing a .model file as the tokenizer_dir. ' - 'It allows AutoTokenizer to instantiate the correct tokenizer type.') - parser.add_argument('--vocab_file', - help="Used for sentencepiece tokenizers") - parser.add_argument('--no_add_special_tokens', - dest='add_special_tokens', - default=True, - action='store_false', - help="Whether or not to add special tokens") - parser.add_argument('--hf_model_dir', '--model_dir', type=str, default=None) - parser.add_argument( - '--tokenizer_dir', - default=None, - help='tokenizer path; defaults to hf_model_dir if left unspecified') - - # memory argument - parser.add_argument( - '--gpu_weights_percent', - default=1, - type=float, - help= - 'Specify the percentage of weights that reside on GPU instead of CPU and streaming load during runtime.', - ) - parser.add_argument( - '--max_tokens_in_paged_kv_cache', - default=None, - type=int, - help= - 'Specify the maximum number of tokens in a kv cache page (only available with cpp session).', - ) - parser.add_argument( - '--kv_cache_enable_block_reuse', - default=True, - action=BooleanOptionalAction, - help= - 'Enables block reuse in kv cache (only available with cpp session).', - ) - parser.add_argument( - '--kv_cache_free_gpu_memory_fraction', - default=0.9, - type=float, - help='Specify the free gpu memory fraction.', - ) - parser.add_argument( - '--cross_kv_cache_fraction', - default=0.5, - type=float, - help= - 'Specify the kv cache fraction reserved for cross attention. Only applicable for encoder-decoder models. By default 0.5 for self and 0.5 for cross.', - ) - parser.add_argument( - '--enable_chunked_context', - action='store_true', - help='Enables chunked context (only available with cpp session).', - ) - - # hf model argument (if use hf model) - parser.add_argument( - '--hf_data_type', - '--data_type', - type=str, - choices=['fp32', 'fp16', 'bf16', 'float32', 'float16', 'bfloat16'], - default='fp16', - help="The data type for hf model.") - parser.add_argument( - '--hf_device_map_auto', - action='store_true', - help="Use device map 'auto' to load a pretrained HF model. This may " - "help to test a large model that cannot fit into a singlue GPU.") - - parser.add_argument( - "--return_all_generated_tokens", - default=False, - action="store_true", - help="This option changes the token output only for streaming. " - "If not specified, return only generated tokens at each step. " - "If specified, return the full beams/outputs at each step. " - "It is automatically enabled for num_beams>1 (only available with cpp session). " - "WARNING: using this option may increase network usage significantly (quadratically w.r.t output length)." - ) - - parser.add_argument( - '--language_task_uids', - type=int, - nargs='+', - default=None, - help= - "language task id indicating which adapter to use in language adapter. Please include 1 locale per input text" - ) - parser.add_argument('--backend', type=str, default=None) - - return parser diff --git a/examples/visual_gen/README.md b/examples/visual_gen/README.md index f1750cb10e3d..51e59f0a2eb4 100644 --- a/examples/visual_gen/README.md +++ b/examples/visual_gen/README.md @@ -23,6 +23,8 @@ python models/flux1.py python models/flux2.py python models/cosmos3_ti2v.py --prompt "A robot arm picks fruit in a grocery store" python models/qwen_image.py +python models/qwen_image_layered.py --image /path/to/image.png +python models/qwen_image_edit.py --image /path/to/source.png --prompt "Make the image look like a watercolor painting" # With engine config (quant, parallelism, etc.) python models/wan_t2v.py --visual_gen_args configs/wan2.2-t2v-fp4-1gpu.yaml @@ -32,6 +34,8 @@ python models/flux1.py --visual_gen_args configs/flux1-dev-fp4-1gpu.yaml python models/flux2.py --visual_gen_args configs/flux2-dev-fp4-1gpu.yaml python models/cosmos3_ti2v.py --visual_gen_args configs/cosmos3-nano-1gpu.yaml --prompt "A robot arm picks fruit in a grocery store" python models/qwen_image.py --visual_gen_args configs/qwen-image-fp8-1gpu.yaml +python models/qwen_image_layered.py --visual_gen_args configs/qwen-image-layered-1gpu.yaml --image /path/to/image.png +python models/qwen_image_edit.py --visual_gen_args configs/qwen-image-edit-2511-fp4-1gpu.yaml --image /path/to/source.png --prompt "Make the image look like a watercolor painting" ``` Install deps from the repo root: `pip install -r requirements-dev.txt`. diff --git a/examples/visual_gen/configs/cosmos3-t2i-1gpu.yaml b/examples/visual_gen/configs/cosmos3-t2i-1gpu.yaml new file mode 100644 index 000000000000..6ea32adaebf7 --- /dev/null +++ b/examples/visual_gen/configs/cosmos3-t2i-1gpu.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1-GPU Cosmos3 text-to-image deployment (base or distilled T2I checkpoints). +# Model: nvidia/Cosmos3-Super-Text2Image or nvidia/Cosmos3-Super-Text2Image-4Step +# Shared by offline examples (--visual_gen_args) and trtllm-serve. +# +# Warmup expresses the deployed workload: warms the 1024x1024 single-frame +# shape instead of the omni default (720p x 189-frame video). Requests should +# pass output_type="image". +attention_config: + backend: VANILLA +parallel_config: + cfg_size: 1 + ulysses_size: 1 +compilation_config: + resolutions: [[1024, 1024]] + num_frames: [1] diff --git a/examples/visual_gen/configs/qwen-image-edit-2511-fp8-1gpu.yaml b/examples/visual_gen/configs/qwen-image-edit-2511-fp8-1gpu.yaml new file mode 100644 index 000000000000..8437ba47567e --- /dev/null +++ b/examples/visual_gen/configs/qwen-image-edit-2511-fp8-1gpu.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1-GPU Qwen-Image-Edit-2511 with FP8 blockwise dynamic quantization. +# Shared by offline examples (--visual_gen_args) and trtllm-serve. +quant_config: + quant_algo: FP8_BLOCK_SCALES + dynamic: true +attention_config: + backend: VANILLA +parallel_config: + cfg_size: 1 + ulysses_size: 1 +cuda_graph_config: + enable: false diff --git a/examples/visual_gen/configs/qwen-image-layered-1gpu.yaml b/examples/visual_gen/configs/qwen-image-layered-1gpu.yaml new file mode 100644 index 000000000000..44f03440d49a --- /dev/null +++ b/examples/visual_gen/configs/qwen-image-layered-1gpu.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1-GPU Qwen-Image-Layered baseline. +# Load with VisualGenArgs.from_yaml() or example scripts that accept --visual_gen_args. +attention_config: + backend: VANILLA +parallel_config: + cfg_size: 1 + ulysses_size: 1 +cuda_graph_config: + enable: false diff --git a/examples/visual_gen/models/cosmos3/README.md b/examples/visual_gen/models/cosmos3/README.md index 69be21fe4880..4d17cea608c8 100644 --- a/examples/visual_gen/models/cosmos3/README.md +++ b/examples/visual_gen/models/cosmos3/README.md @@ -13,6 +13,8 @@ Pass the Hub ID or local path via `--model`: - [`nvidia/Cosmos3-Nano`](https://huggingface.co/nvidia/Cosmos3-Nano) - [`nvidia/Cosmos3-Super`](https://huggingface.co/nvidia/Cosmos3-Super) +- [`nvidia/Cosmos3-Super-Text2Image-4Step`](https://huggingface.co/nvidia/Cosmos3-Super-Text2Image-4Step) — DMD2-distilled text-to-image: fixed 4-step schedule with classifier-free guidance baked into the weights. Steps/guidance are read from the checkpoint; conflicting request values are rejected. Use with `configs/cosmos3-t2i-1gpu.yaml`. +- [`nvidia/Cosmos3-Super-Image2Video-4Step`](https://huggingface.co/nvidia/Cosmos3-Super-Image2Video-4Step) — DMD2-distilled image-to-video: same fixed 4-step, guidance-baked-in contract. The default omni video shape (720p × 189 frames) is the deployed shape, so no dedicated config is needed. This checkpoint declares `default_use_system_prompt: true` in its `model_index.json`, which the pipeline applies automatically (override with `--use_system_prompt` / `--no-use_system_prompt`). ## Guardrails @@ -36,6 +38,7 @@ See `examples/visual_gen/configs/`: - `cosmos3-nano-1gpu.yaml` — 1 GPU - `cosmos3-super-4gpu.yaml` — 4 GPU, CFG + Ulysses + parallel VAE +- `cosmos3-t2i-1gpu.yaml` — 1 GPU, text-to-image deployments (base or distilled): warms the deployed 1024×1024 single-frame shape instead of the omni video shape. Example prompts live under `prompts/` (mirroring `cosmos3-internal/inputs/omni`). @@ -70,6 +73,22 @@ python cosmos3.py --model nvidia/Cosmos3-Nano \ --visual_gen_args ../configs/cosmos3-nano-1gpu.yaml \ --output_path output.png +# T2I, distilled 4-step checkpoint (use the T2I config so warmup runs the +# image shape; steps/guidance come from the checkpoint automatically) +python cosmos3.py --model nvidia/Cosmos3-Super-Text2Image-4Step \ + --prompt_file prompts/t2i.json \ + --visual_gen_args ../../configs/cosmos3-t2i-1gpu.yaml \ + --output_type image \ + --output_path output.png + +# I2V, distilled 4-step checkpoint (steps/guidance and the system-prompt +# default come from the checkpoint automatically; defaults are the deployed +# 720p x 189-frame shape, so no config is required) +python cosmos3.py --model nvidia/Cosmos3-Super-Image2Video-4Step \ + --prompt "The camera slowly pans right across the scene" \ + --image_path https://example.com/frame.jpg \ + --output_path output.mp4 + # Inline prompt (--prompt or a JSON file path) python cosmos3.py --model nvidia/Cosmos3-Nano \ --prompt "A cute puppy playing with a ball in a park" \ diff --git a/examples/visual_gen/models/cosmos3/cosmos3.py b/examples/visual_gen/models/cosmos3/cosmos3.py index de9e9e5010ad..9c1f45be1abd 100644 --- a/examples/visual_gen/models/cosmos3/cosmos3.py +++ b/examples/visual_gen/models/cosmos3/cosmos3.py @@ -136,7 +136,13 @@ def main(): help="Disable resolution metadata template (enabled by default, matching cosmos-framework CLI)", ) parser.add_argument( - "--use_system_prompt", action="store_true", help="Use system prompt in prompt" + "--use_system_prompt", + action=argparse.BooleanOptionalAction, + default=None, + help=( + "Prepend the Cosmos3 system prompt (--no-use_system_prompt to disable). " + "When omitted, the checkpoint's declared default applies." + ), ) parser.add_argument("--enable_audio", action="store_true", help="Enable audio generation") parser.add_argument( @@ -181,7 +187,8 @@ def main(): params.extra_params["use_duration_template"] = False if args.disable_resolution_template: params.extra_params["use_resolution_template"] = False - params.extra_params["use_system_prompt"] = args.use_system_prompt + if args.use_system_prompt is not None: + params.extra_params["use_system_prompt"] = args.use_system_prompt params.extra_params["enable_audio"] = enable_audio params.extra_params["use_guardrails"] = not args.disable_guardrails params.extra_params["output_type"] = output_type diff --git a/examples/visual_gen/models/flux2.py b/examples/visual_gen/models/flux2.py index 3b48a17923bd..a39044448b2f 100644 --- a/examples/visual_gen/models/flux2.py +++ b/examples/visual_gen/models/flux2.py @@ -13,11 +13,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""FLUX.2 text-to-image generation. +"""FLUX.2 text-to-image and reference-image generation. Usage: python flux2.py python flux2.py --visual_gen_args ../configs/flux2-dev-fp4-1gpu.yaml + python flux2.py --image subject.png + python flux2.py --image subject.png --image style.png """ import argparse @@ -35,7 +37,9 @@ def _output_paths(output_path: str, num_images: int) -> str | list[str]: def main(): - parser = argparse.ArgumentParser(description="FLUX.2 Text-to-Image example") + parser = argparse.ArgumentParser( + description="FLUX.2 text-to-image and reference-image generation example" + ) parser.add_argument( "--model", type=str, @@ -61,6 +65,42 @@ def main(): default=1, help="Number of images to generate for the prompt", ) + parser.add_argument( + "--image", + action="append", + default=None, + help="Reference image path; repeat for a shared set of images", + ) + parser.add_argument( + "--height", + type=int, + default=None, + help="Output height; with references, omitted uses the first processed image", + ) + parser.add_argument( + "--width", + type=int, + default=None, + help="Output width; with references, omitted uses the first processed image", + ) + parser.add_argument( + "--num_inference_steps", + type=int, + default=None, + help="Number of denoising steps; omitted uses the model default", + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=None, + help="Embedded guidance scale; omitted uses the model default", + ) + parser.add_argument( + "--seed", + type=int, + default=None, + help="Random seed; omitted selects a fresh random seed", + ) parser.add_argument( "--output_path", type=str, @@ -75,10 +115,25 @@ def main(): extra_args = VisualGenArgs.from_yaml(args.visual_gen_args) if args.visual_gen_args else None visual_gen = VisualGen(model=args.model, args=extra_args) - # --- Model-specific: T2I request construction --- - # Start from per-model defaults (resolution, steps, guidance, seed, etc.) and set image count. + # Start from per-model defaults and override only user-provided request fields. params = visual_gen.default_params params.num_images_per_prompt = args.num_images_per_prompt + params.image = args.image + if args.image: + # Let FLUX.2 derive omitted dimensions from the first processed reference. + params.height = args.height + params.width = args.width + else: + if args.height is not None: + params.height = args.height + if args.width is not None: + params.width = args.width + if args.num_inference_steps is not None: + params.num_inference_steps = args.num_inference_steps + if args.guidance_scale is not None: + params.guidance_scale = args.guidance_scale + if args.seed is not None: + params.seed = args.seed output = visual_gen.generate(inputs=args.prompt, params=params) diff --git a/examples/visual_gen/models/qwen_image_edit.py b/examples/visual_gen/models/qwen_image_edit.py new file mode 100644 index 000000000000..43f37a73d1c4 --- /dev/null +++ b/examples/visual_gen/models/qwen_image_edit.py @@ -0,0 +1,74 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Qwen-Image-Edit-2511 image editing. + +Usage: + python qwen_image_edit.py --image input.png --prompt "Make the image look like a watercolor painting" + python qwen_image_edit.py \ + --visual_gen_args ../configs/qwen-image-edit-2511-fp8-1gpu.yaml \ + --image input.png +""" + +import argparse + +from tensorrt_llm import VisualGen, VisualGenArgs + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", + default="Qwen/Qwen-Image-Edit-2511", + help="Hugging Face model id or local checkpoint path.", + ) + parser.add_argument( + "--visual_gen_args", + dest="visual_gen_args", + type=str, + default=None, + help="Path to YAML config (same as trtllm-serve --visual_gen_args)", + ) + parser.add_argument( + "--image", + nargs="+", + required=True, + help="One or more input image paths or URLs.", + ) + parser.add_argument( + "--prompt", + default="Make the image look like a watercolor painting while preserving the main subject.", + help="Text edit instruction.", + ) + parser.add_argument( + "--output_path", + default="qwen_image_edit_output.png", + help="Edited image output path.", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + extra_args = VisualGenArgs.from_yaml(args.visual_gen_args) if args.visual_gen_args else None + visual_gen = VisualGen(model=args.model, args=extra_args) + params = visual_gen.default_params + params.image = args.image if len(args.image) > 1 else args.image[0] + output = visual_gen.generate(inputs=args.prompt, params=params) + saved = output.save(args.output_path) + print(f"Saved edited image to {saved}") + + +if __name__ == "__main__": + main() diff --git a/examples/visual_gen/models/qwen_image_layered.py b/examples/visual_gen/models/qwen_image_layered.py new file mode 100644 index 000000000000..6fbcaf9e9f47 --- /dev/null +++ b/examples/visual_gen/models/qwen_image_layered.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Qwen-Image-Layered image decomposition. + +Usage: + python qwen_image_layered.py --image input.png + python qwen_image_layered.py --visual_gen_args ../configs/qwen-image-layered-1gpu.yaml \ + --image input.png +""" + +import argparse + +from tensorrt_llm import VisualGen, VisualGenArgs + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", + default="Qwen/Qwen-Image-Layered", + help="Hugging Face model id or local checkpoint path.", + ) + parser.add_argument( + "--visual_gen_args", + "--extra_visual_gen_options", + dest="visual_gen_args", + help="Optional VisualGenArgs YAML file.", + ) + parser.add_argument( + "--image", + required=True, + help="Input image path.", + ) + parser.add_argument( + "--prompt", + default="", + help="Optional text prompt. Empty prompt enables image captioning.", + ) + parser.add_argument( + "--output_path", + default="qwen_image_layered_output.png", + help="Path to save the layer grid image.", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + extra_args = VisualGenArgs.from_yaml(args.visual_gen_args) if args.visual_gen_args else None + visual_gen = VisualGen(model=args.model, args=extra_args) + + params = visual_gen.default_params + params.image = args.image + + output = visual_gen.generate(inputs=args.prompt, params=params) + saved = output.save(args.output_path) + print(f"Saved image to {saved}") + + +if __name__ == "__main__": + main() diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index a5d3966bb5c8..273e13bb116b 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -33,7 +33,7 @@ AARCH64_TRIPLE = "aarch64-linux-gnu" LLM_DOCKER_IMAGE = env.dockerImage // Always use x86_64 image for agent -AGENT_IMAGE = env.dockerImage.replace("aarch64", "x86_64") +AGENT_IMAGE = env.dockerImage.replace("aarch64", "x86_64").replace("sbsa", "x86_64") POD_TIMEOUT_SECONDS_BUILD = env.podTimeoutSeconds ? env.podTimeoutSeconds : "43200" @@ -135,7 +135,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64") def nodeLabelPrefix = "" def archSuffix = arch == "arm64" ? "arm" : "amd" - def jnlpImage = "urm.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" + def jnlpImage = "artifactory.pdx.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" switch(type) { @@ -384,7 +384,7 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) sh "ccache -sv" sh "rm -rf **/*.xml *.tar.gz" - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) if (env.alternativeTRT) { sh "cd ${LLM_ROOT} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" } @@ -421,24 +421,12 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) def buildJobs = buildFlags[BUILD_JOBS_FOR_CONFIG] ?: BUILD_JOBS withCredentials([usernamePassword(credentialsId: "urm-artifactory-creds", usernameVariable: 'CONAN_LOGIN_USERNAME', passwordVariable: 'CONAN_PASSWORD')]) { - sh "cd ${LLM_ROOT} && python3 scripts/build_wheel.py --use_ccache -G Ninja -j ${buildJobs} -a '${buildFlags[WHEEL_ARCHS]}' ${buildFlags[WHEEL_EXTRA_ARGS]} --benchmarks" + sh "cd ${LLM_ROOT} && python3 scripts/build_wheel.py --use_ccache -G Ninja -j ${buildJobs} -a '${buildFlags[WHEEL_ARCHS]}' ${buildFlags[WHEEL_EXTRA_ARGS]}" } - if (is_linux_x86_64) { - sh "cd ${LLM_ROOT} && python3 scripts/build_cpp_examples.py" - } - // Step 3: packaging wheels into tarfile sh "cp ${LLM_ROOT}/build/tensorrt_llm-*.whl TensorRT-LLM/" - // Step 4: packaging benchmark and required cpp dependencies into tarfile - sh "mkdir -p TensorRT-LLM/benchmarks/cpp" - sh "cp ${LLM_ROOT}/cpp/build/benchmarks/bertBenchmark TensorRT-LLM/benchmarks/cpp" - sh "cp ${LLM_ROOT}/cpp/build/benchmarks/gptManagerBenchmark TensorRT-LLM/benchmarks/cpp" - sh "cp ${LLM_ROOT}/cpp/build/benchmarks/disaggServerBenchmark TensorRT-LLM/benchmarks/cpp" - sh "cp ${LLM_ROOT}/cpp/build/tensorrt_llm/libtensorrt_llm.so TensorRT-LLM/benchmarks/cpp" - sh "cp ${LLM_ROOT}/cpp/build/tensorrt_llm/plugins/libnvinfer_plugin_tensorrt_llm.so TensorRT-LLM/benchmarks/cpp" - - // Step 5: packaging attribution files into tarfile when they exist + // Step 4: packaging attribution files into tarfile when they exist sh "mkdir -p TensorRT-LLM/attribution" sh "cp ${LLM_ROOT}/cpp/build/attribution/missing_files.json TensorRT-LLM/attribution/ || true" sh "cp ${LLM_ROOT}/cpp/build/attribution/import_payload.json TensorRT-LLM/attribution/ || true" @@ -471,7 +459,7 @@ def buildWheelInContainer(pipeline, libraries=[], triple=X86_64_TRIPLE, clean=fa sh "cat ${CCACHE_DIR}/ccache.conf" // Step 1: cloning tekit source code - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) if (env.alternativeTRT) { trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, cpver) sh "cd ${LLM_ROOT} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" @@ -571,7 +559,7 @@ def launchStages(pipeline, cpu_arch, enableFailFast, globalVars) stage(key) { stage("[${key}] Run") { echoNodeAndGpuInfo(pipeline, key) - buildWheelInContainer(pipeline, [], X86_64_TRIPLE, false, false, "cp312", "-a '90-real' -b Debug --benchmarks --micro_benchmarks") + buildWheelInContainer(pipeline, [], X86_64_TRIPLE, false, false, "cp312", "-a '90-real' -b Debug --micro_benchmarks") } } }) diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index c41b999b8920..8c9f005f63d0 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -144,7 +144,7 @@ def createKubernetesPodConfig(type, arch = "amd64", build_wheel = false) } def archSuffix = arch == "arm64" ? "arm" : "amd" - def jnlpImage = "urm.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" + def jnlpImage = "artifactory.pdx.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" switch(type) { @@ -300,7 +300,7 @@ def buildImage(config, imageKeyToTag) stage (config.stageName) { // Step 1: Clone TRT-LLM source codes // If using a forked repo, svc_tensorrt needs to have the access to the forked repo. - trtllm_utils.checkoutSource(LLM_REPO, LLM_COMMIT_OR_BRANCH, LLM_ROOT, false, true) + trtllm_utils.checkoutSource(LLM_REPO, LLM_COMMIT_OR_BRANCH, LLM_ROOT, true, true) } // Step 2: Build the images @@ -453,15 +453,26 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { ] def release_action = params.action + def stageNames = [ + internalReleaseX86: "Build Internal release (x86_64 trtllm)", + internalReleaseSBSA: "Build Internal release (SBSA trtllm)", + ciImageX86: "Build CI Image (x86_64 tritondevel)", + ciImageSBSA: "Build CI Image (SBSA tritondevel)", + ciImageRockyPy310: "Build CI Image (RockyLinux8 Python310)", + ciImageRockyPy312: "Build CI Image (RockyLinux8 Python312)", + ciImageSBSAUbuntu: "Build CI Image (SBSA Ubuntu24.04 Python312)", + ngcReleaseX86: "Build NGC devel And release (x86_64)", + ngcReleaseSBSA: "Build NGC devel And release (SBSA)", + ] def buildConfigs = [ - "Build Internal release (x86_64 trtllm)": [ + (stageNames.internalReleaseX86): [ target: "trtllm", action: release_action, customTag: LLM_BRANCH_TAG + "-x86_64", build_wheel: true, dockerfileStage: "release", ], - "Build Internal release (SBSA trtllm)": [ + (stageNames.internalReleaseSBSA): [ target: "trtllm", action: release_action, customTag: LLM_BRANCH_TAG + "-sbsa", @@ -469,27 +480,27 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { arch: "arm64", dockerfileStage: "release", ], - "Build CI Image (x86_64 tritondevel)": [:], - "Build CI Image (SBSA tritondevel)": [ + (stageNames.ciImageX86): [:], + (stageNames.ciImageSBSA): [ arch: "arm64", ], - "Build CI Image (RockyLinux8 Python310)": [ + (stageNames.ciImageRockyPy310): [ target: "rockylinux8", args: "PYTHON_VERSION=3.10.12", postTag: "-py310", ], - "Build CI Image (RockyLinux8 Python312)": [ + (stageNames.ciImageRockyPy312): [ target: "rockylinux8", args: "PYTHON_VERSION=3.12.3", postTag: "-py312", ], - "Build CI Image (SBSA Ubuntu24.04 Python312)": [ + (stageNames.ciImageSBSAUbuntu): [ arch: "arm64", target: "ubuntu24", args: "PYTHON_VERSION=3.12.3", postTag: "-py312", ], - "Build NGC devel And release (x86_64)": [ + (stageNames.ngcReleaseX86): [ target: "ngc-release", action: release_action, args: "DOCKER_BUILD_OPTS='--load --platform linux/amd64'", @@ -500,7 +511,7 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { ], dockerfileStage: "release", ], - "Build NGC devel And release (SBSA)": [ + (stageNames.ngcReleaseSBSA): [ target: "ngc-release", action: release_action, args: "DOCKER_BUILD_OPTS='--load --platform linux/arm64'", @@ -513,6 +524,19 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { dockerfileStage: "release", ], ] + def enabledStages = [] + if (params.buildInternalRelease) { + enabledStages += [stageNames.internalReleaseX86, stageNames.internalReleaseSBSA] + } + if (params.buildCiImage) { + enabledStages += [stageNames.ciImageX86, stageNames.ciImageSBSA, stageNames.ciImageRockyPy310, stageNames.ciImageRockyPy312, stageNames.ciImageSBSAUbuntu] + } + if (params.buildNgcRelease) { + enabledStages += [stageNames.ngcReleaseX86, stageNames.ngcReleaseSBSA] + } + buildConfigs = buildConfigs.findAll { key, config -> key in enabledStages } + echo "Running stages: ${buildConfigs.keySet()}" + // Override all fields in build config with default values buildConfigs.each { key, config -> defaultBuildConfig.each { defaultKey, defaultValue -> @@ -581,6 +605,21 @@ pipeline { choices: ["build", "push"], description: "Docker image generation action. build: only perform image build step; push: build docker image and push it to artifacts" ) + booleanParam( + name: "buildInternalRelease", + defaultValue: true, + description: "Build internal release images (x86_64 and SBSA trtllm)" + ) + booleanParam( + name: "buildCiImage", + defaultValue: true, + description: "Build CI images (tritondevel and OS variant images)" + ) + booleanParam( + name: "buildNgcRelease", + defaultValue: true, + description: "Build NGC devel and release images (x86_64 and SBSA)" + ) } options { // Check the valid options at: https://www.jenkins.io/doc/book/pipeline/syntax/ diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 57bc4123c2d5..4b1d7376cf42 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -186,7 +186,7 @@ def globalVars = [ (CACHED_CHANGED_FILE_LIST): null, (ACTION_INFO): gitlabParamsFromBot.get('action_info', null), (IMAGE_KEY_TO_TAG): [:], - (TARGET_BRANCH): gitlabParamsFromBot.get('target_branch', null), + (TARGET_BRANCH): gitlabParamsFromBot.get('target_branch', 'main'), ] // If not running all test stages in the L0 pre-merge, we will not update the GitLab status at the end. @@ -212,7 +212,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64") def nodeLabelPrefix = "" def archSuffix = arch == "arm64" ? "arm" : "amd" - def jnlpImage = "urm.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" + def jnlpImage = "artifactory.pdx.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" switch(type) { @@ -325,10 +325,10 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) // NB: getContainerURIs reads files in ${LLM_ROOT}/jenkins/ if (env.gitlabMergeRequestLastCommit) { env.gitlabCommit = env.gitlabMergeRequestLastCommit - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) } else { branch = env.gitlabBranch ? env.gitlabBranch : "main" - trtllm_utils.checkoutSource(LLM_REPO, branch, LLM_ROOT, false, true) + trtllm_utils.checkoutSource(LLM_REPO, branch, LLM_ROOT, true, true) checkoutCommit = sh (script: "cd ${LLM_ROOT} && git rev-parse HEAD",returnStdout: true).trim() env.gitlabCommit = checkoutCommit } @@ -451,7 +451,7 @@ def launchReleaseCheck(pipeline, globalVars) sh "pip3 config set global.break-system-packages true" sh "git config --global --add safe.directory \"*\"" // Step 1: Clone TRT-LLM source codes - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) sh "cd ${LLM_ROOT} && git config --unset-all core.hooksPath" // Step 2: Run guardwords scan @@ -774,6 +774,9 @@ def getCbtsResult(pipeline, testFilter, globalVars) // pyyaml is needed by main.py's blocks.py to parse test-db YAMLs. sh "apt-get update -qq && apt-get install -y -qq python3-yaml" + // Shadow audit: download the latest merged touch DB and log its health + HEAD coverage gap (diagnostic only). + _cbtsCoverageAudit(pipeline) + // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py --list-needed-diffs", @@ -842,6 +845,33 @@ def getCbtsResult(pipeline, testFilter, globalVars) } } +// Download the latest merged touch DB and run coverage_audit.py on it; best-effort, never changes the CBTS decision. +def _cbtsCoverageAudit(pipeline) +{ + try { + def covDir = "${LLM_ROOT}/cbts_cov" + def url = sh( + script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py --print-url || true", + returnStdout: true, + ).trim() + if (!url) { + pipeline.echo("CBTS audit: no coverage DB artifact found — skipping") + return + } + sh "mkdir -p ${covDir}" + // wget the tarball (retrying) and extract the sqlite. + trtllm_utils.llmExecStepWithRetry(pipeline, script: + "wget -nv '${url}' -O ${covDir}/cbts_pystart_report.tar.gz && " + + "tar xzf ${covDir}/cbts_pystart_report.tar.gz -C ${covDir}") + sh "python3 ${LLM_ROOT}/jenkins/scripts/cbts/tools/coverage_audit.py " + + "--db ${covDir}/cbts_touchmap.sqlite" + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + pipeline.echo("CBTS audit: skipped (non-fatal): ${e.message}") + } +} + // Post one CBTS decision record to OpenSearch (best-effort; never blocks CI). // decisionJson null for deferred; reason used only then. Context/creds via env. def _cbtsReportDecision(pipeline, globalVars, String status, String reason, String decisionJson) @@ -954,7 +984,6 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "cpp/include/tensorrt_llm/runtime/worldConfig.h", "cpp/tensorrt_llm/batch_manager/", "cpp/tensorrt_llm/executor/", - "cpp/tensorrt_llm/executor_worker/", "cpp/tensorrt_llm/kernels/communicationKernels/", "cpp/tensorrt_llm/kernels/customAllReduceKernels.cu", "cpp/tensorrt_llm/kernels/customAllReduceKernels.h", @@ -969,13 +998,6 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "cpp/tensorrt_llm/kernels/userbuffers/", "cpp/tensorrt_llm/kernels/xqaDispatcher.cpp", "cpp/tensorrt_llm/kernels/xqaDispatcher.h", - "cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.cpp", - "cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.h", - "cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.cpp", - "cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h", - "cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.cpp", - "cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.h", - "cpp/tensorrt_llm/plugins/ncclPlugin/", "cpp/tensorrt_llm/nanobind/", "cpp/tensorrt_llm/runtime/ipcUtils.cpp", "cpp/tensorrt_llm/runtime/ncclCommunicator.cpp", @@ -985,8 +1007,6 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "cpp/tensorrt_llm/thop/allgatherOp.cpp", "cpp/tensorrt_llm/thop/allreduceOp.cpp", "cpp/tensorrt_llm/thop/reducescatterOp.cpp", - "cpp/tests/e2e_tests/batch_manager/", - "cpp/tests/e2e_tests/executor/", "cpp/tests/unit_tests/multi_gpu/", "jenkins/L0_Test.groovy", "tensorrt_llm/_ipc_utils.py", @@ -1047,22 +1067,18 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node2_gpu8.yml", - "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node4_gpu16.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node4_gpu16.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml", - "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx2_node1_gpu4_gen1_node4_gpu16.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb300.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml", - "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml", - "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node4_gpu16.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_node2_gpu8.yml", @@ -1197,16 +1213,7 @@ def collectTestResults(pipeline, testFilter, globalVars) echo "Result File Number: ${resultFileNumber}, Downloaded: ${resultFileDownloadedNumber}" sh "find . -name results-\\*.tar.gz -type f -exec tar -zxvf {} \\; || true" - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) - if (testFilter[(IS_POST_MERGE)]) { - try { - sh "python3 llm/scripts/generate_duration.py --duration-file=new_test_duration.json" - trtllm_utils.uploadArtifacts("new_test_duration.json", "${UPLOAD_PATH}/test-results/") - } catch (Exception e) { - // No need to fail the stage if the duration file generation fails - echo "An error occurred while generating or uploading the duration file: ${e.toString()}" - } - } + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) junit(testResults: '**/results*.xml', allowEmptyResults : true) @@ -1498,6 +1505,10 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Skipping x86_64 tests (GenPostMergeBuilds mode: builds only)" return } + if (testFilter[(TEST_STAGE_LIST)]?.contains("NGC-Container-Scaning")) { + echo "Skipping x86_64 tests (PLC container scanning)" + return + } testStageName = "[Test-x86_64-Single-GPU] Remote Run" def singleGpuTestFailed = false @@ -1611,6 +1622,11 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) return } + if (testFilter[(TEST_STAGE_LIST)]?.contains("NGC-Container-Scaning")) { + echo "Skipping SBSA tests (PLC container scanning)" + return + } + testStageName = "[Test-SBSA-Single-GPU] Remote Run" def singleGpuTestFailed = false stage(testStageName) { @@ -1757,6 +1773,88 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Build-Docker-Images job is set explicitly. Both x86_64-Linux and SBSA-Linux sub-pipelines will be disabled." } + def plcContainerScanningJob = [ + "PLC Container Scanning": { + script { + stage("[Build-Release-Docker-Images] Remote Run") { + try { + def branch = env.gitlabBranch ? env.gitlabBranch : "main" + if (globalVars[GITHUB_PR_API_URL]) { + branch = "github-pr-" + globalVars[GITHUB_PR_API_URL].split('/').last() + } + + // Force the image tag suffix to be this L0_MergeRequest BUILD_NUMBER + // instead of the BuildDockerImages helper job's own counter. + def shortCommit = env.gitlabCommit ? env.gitlabCommit.substring(0, 7) : "undefined" + def branchTag = branch.replaceAll('/', '_') + def defaultTag = "${shortCommit}-${branchTag}-${env.BUILD_NUMBER}" + + def additionalParameters = [ + 'branch': branch, + 'action': "push", + 'triggerType': "post-merge", + 'runSanityCheck': false, + 'defaultTag': defaultTag, + 'buildInternalRelease': false, + 'buildCiImage': false, + 'artifactPath': ARTIFACT_PATH, + 'nspect_id': "", + 'uploadPath': UPLOAD_PATH + ] + launchJob(pipeline, "/LLM/helpers/BuildDockerImages", false, enableFailFast, globalVars, "x86_64", additionalParameters) + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + if (BUILD_CHECK_CHOICE == STAGE_CHOICE_IGNORE) { + catchError( + buildResult: 'SUCCESS', + stageResult: 'FAILURE') { + error "Build-Docker-Images job failed but ignored due to Jenkins configuration" + } + } else { + throw e + } + } + } + stage("[NGC-Container-Compliance-Check] Run") { + echo "Triggering OSS Compliance (PLC) container scan for ref: " + try { + def params = [ + string(name: 'postMergePipelineName', value: env.JOB_NAME), + string(name: 'postMergeBuildNumber', value: env.BUILD_NUMBER), + string(name: 'scanMode', value: 'pre_merge'), + string(name: 'runSourceCodeScanning', value: 'false'), + string(name: 'runContainerScanning', value: 'true'), + string(name: 'runSonarQube', value: 'false'), + ] + def logger = new Logger(pipeline) + def handle = build( + job: "/LLM/helpers/PLCScanningSetup", + parameters: params, + propagate: false + ) + if (handle.result != "SUCCESS") { + catchError(buildResult: currentBuild.result ?: 'SUCCESS', stageResult: 'UNSTABLE') { + error "Risks detected on NGC Containers" + } + } + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { + error "OSS Compliance Check failed: ${e.getMessage()}" + } + } + } + } + } + ] + if (testFilter[(TEST_STAGE_LIST)]?.contains("NGC-Container-Scaning")) { + stages += plcContainerScanningJob + testFilter[(TEST_STAGE_LIST)]?.remove("NGC-Container-Scanning") + echo "Will run job to build ngc containers and running in-pipeline scanning for them" + } + parallelJobs = stages.collectEntries{key, value -> [key, { script { stage(key) { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 3f72edbf9e93..42903e069668 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -59,6 +59,7 @@ linuxPkgName = ( env.targetArch == AARCH64_TRIPLE ? "tensorrt-llm-sbsa-release-s // available tags can be found in: https://urm.nvidia.com/artifactory/sw-tensorrt-docker/tensorrt-llm/ // [base_image_name]-[arch]-[os](-[python_version])-[trt_version]-[torch_install_type]-[stage]-[date]-[mr_id] LLM_DOCKER_IMAGE = env.dockerImage +X86_64_DOCKER_IMAGE = LLM_DOCKER_IMAGE.replace("aarch64", "x86_64").replace("sbsa", "x86_64") LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE = env.wheelDockerImagePy310 LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE = env.wheelDockerImagePy312 LLM_WHEEL_DOCKER_IMAGE = env.wheelDockerImage @@ -151,12 +152,34 @@ SLURM_INFRA_RETRY_MAX = 1 // to avoid nesting with the inner SLURM retry. K8S_INFRA_RETRY_MAX = 1 +// Per-stage override of the above: set `infraRetryMax` in a stage's opts map (the +// 3rd element of its parallel-jobs config tuple, alongside singleAttempt) to cap +// or disable stage-level infra retries for resource-scarce hardware pools -- +// `infraRetryMax: 0` disables retries entirely (1 attempt). It may only reduce the +// budget: values above the scope global are clamped down to it (resolveInfraRetryMax), +// so it can never increase retries past these caps. It applies to whichever +// stage-level retry the stage uses: the SLURM retry (runLLMTestlistOnSlurm) for +// dispatcher pods, or the K8s pod retry (runKubernetesPodWithInfraRetry) for regular +// test pods. It does NOT touch the dispatcher-pod launch-retry (relaunching a cheap +// Blossom pod doesn't tax the scarce hardware). Null/absent = use the globals above. + // Fallback discriminator for SLURM timeouts. // If we can't reach the SLURM node for an authoritative reason, // we apply a heuristic: if the job needed more than this of its budget // to fail, we treat it as a timeout. SLURM_TIMEOUT_RETRY_FRACTION = 0.9 +// SLURM states in which the job is still alive (no terminal verdict yet). When a +// monitor/agent exception surfaces while the job is in one of these, the failure +// is a transient infra blip (lost SSH/agent) -- not a test result -- so the stage +// should retry rather than defer an opaque exception to the classifier (which +// could mistake it for a test failure and not retry). Mirrors the active-state +// set the sbatch resubmit guard reuses an existing job on. +SLURM_NON_TERMINAL_STATES = [ + "RUNNING", "PENDING", "CONFIGURING", "COMPLETING", + "REQUEUED", "RESIZING", "SUSPENDED", "SIGNALING", "STOPPED", +] + // Typed-exception hierarchy and FailureClassifier (PATTERN_CATALOG, classify(), // flattenThrowable) live in trtllm-jenkins-shared-lib under src/trtllm/. They // were originally inline here, but the Jenkins script-security sandbox @@ -224,11 +247,63 @@ def echoRemoteLogTail(def pipeline, Map remote, String remotePath, int lines = 2 } } +// Scrape the SLURM job output log for a device / driver / interconnect fault +// signature and return the matched signature itself, or "" for no match. +// +// Device faults (CUDA/NVLink/ECC/driver) print into job-output.log but never +// reach the stage exception chain -- the tracker squashes a failed job to +// `exit 1` -- so classify() otherwise sees only a generic failure and cannot +// steer the retry off the bad node. This is a GATE only: the returned signature +// is folded into a fresh exception so FailureClassifier.PATTERN_CATALOG (the +// authoritative list) makes the real retry/severity decision. A signature the +// catalog does not recognize simply falls through to a normal rethrow. +// App-induced CUDA errors (illegal memory access, unspecified launch failure, +// OOM) are deliberately excluded -- the OpenSearch stage data shows those are +// overwhelmingly code regressions, not node faults, and must not trigger a +// node-avoiding retry. +// +// grep -o returns only the matched signature (not the whole line), so a long +// log line cannot truncate the signature out of the result before it reaches +// classify(). Each alternative must therefore be catalog-exact: it must match +// (via `.` wildcards for shell-hostile chars) the full catalog substring, so +// grep -o emits text that still contains the catalog pattern. +def scrapeSlurmLogForDeviceFault(def pipeline, Map remote, String remoteLogPath) { + def deviceFaultRegex = "cudaErrorMapBufferObjectFailed|mapping of buffer object failed|" + + "uncorrectable NVLink error|cudaErrorNvlinkUncorrectable|CUDA_ERROR_SYSTEM_NOT_READY|" + + "uncorrectable ECC error|CUDA_ERROR_ECC_UNCORRECTABLE|has fallen off the bus|GPU is lost|" + + "Unable to determine the device handle for GPU|RmInitAdapter failed|Failed to initialize NVML|" + + "could... communicate with the NVIDIA driver|CUDA_ERROR_DEVICE_UNAVAILABLE|" + + "no CUDA-capable device is detected|CUDA_ERROR_UNKNOWN: 999|CUDA unknown error|" + + "CUDA-capable device.s. is/are busy or unavailable" + try { + // Wrap the body in `bash -c` so it is shell-agnostic: cluster login shells + // are often csh/tcsh, which can't parse this bash test/pipe/redirection + // syntax. The login shell only has to run `bash -c ''`. + return Utils.exec( + pipeline, + script: Utils.sshUserCmd(remote, + "\"bash -c 'if [ -f \\\"${remoteLogPath}\\\" ]; then grep -aioE \\\"${deviceFaultRegex}\\\" \\\"${remoteLogPath}\\\" 2>/dev/null | tail -n 1 | cut -c1-500; fi'\""), + returnStdout: true, + numRetries: 1, + )?.trim() + } catch (InterruptedException e) { + throw e + } catch (Exception scrapeEx) { + pipeline.echo("Ignorable warning: could not scrape ${remoteLogPath} for device faults on ${remote.host}: ${scrapeEx.message}") + return "" + } +} + // `postTag` uniquifies the uploaded tar filename, the Artifactory guard key and // the locally-staged result XMLs when the same stageName is uploaded more than // once in a build (e.g. SLURM infra-failure retries). First attempt passes "". -def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String nodeName, String stageName, Boolean stageIsInterrupted, String postTag="") { - CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote -> +def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String nodeName, String stageName, Boolean stageIsInterrupted, String postTag="", boolean suppressTestReporting=false) { + CloudManager.withSlurmSshCredentialRemotes(pipeline, clusterName, cluster) { remotes -> + // Pin one reachable frontend for the whole collect: every download targets + // the same node workspace (/home/svc_tensorrt/bloom/scripts/${nodeName}), + // so the find + scps must all hit the login node that holds those files. + // No whole-closure failover here -- uploadArtifacts/junit must not re-run. + def remote = CloudManager.selectReachableSlurmRemote(pipeline, remotes) def hasTimeoutTest = false def downloadResultSucceed = false def downloadPerfResultSucceed = false @@ -338,8 +413,10 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String } } - if (hasTimeoutTest || downloadResultSucceed) { + if ((hasTimeoutTest || downloadResultSucceed) && !suppressTestReporting) { junit(allowEmptyResults: true, testResults: "${stageName}/results*.xml") + } else if (suppressTestReporting) { + echo "[INFRA-RETRY] ${stageName}${postTag}: suppressing junit() because a retry is still planned" } } } @@ -599,7 +676,7 @@ def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, du } def cleanUpSlurmResources(def pipeline, SlurmCluster cluster, String clusterName, String jobUID){ - CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote -> + CloudManager.withSlurmFrontendFailover(pipeline, clusterName, cluster) { remote -> def jobWorkspace = "/home/svc_tensorrt/bloom/scripts/${jobUID}" Utils.exec(pipeline, script: "echo Sleeping to allow Slurm job completion; sleep 30") @@ -642,7 +719,7 @@ def cleanUpSlurmResources(def pipeline, SlurmCluster cluster, String clusterName pipeline, script: Utils.sshUserCmd( remote, - "\"${cleanupCommands}\"" + Utils.bashWrappedRemoteCmd(cleanupCommands) ) ) @@ -658,7 +735,7 @@ def cleanUpNodeResources(def pipeline, SlurmCluster cluster, String clusterName, Utils.exec(pipeline, script: "echo Sleeping to allow node destruction; sleep 30") - CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote -> + CloudManager.withSlurmFrontendFailover(pipeline, clusterName, cluster) { remote -> Utils.exec(pipeline, script: "echo Slurm job ID: ${slurmJobID}") Utils.exec( @@ -683,7 +760,7 @@ def cleanUpNodeResources(def pipeline, SlurmCluster cluster, String clusterName, pipeline, script: Utils.sshUserCmd( remote, - "\"${cleanupCommands}\"" + Utils.bashWrappedRemoteCmd(cleanupCommands) ) ) @@ -691,6 +768,114 @@ def cleanUpNodeResources(def pipeline, SlurmCluster cluster, String clusterName, } } +// ---- Off-pod SLURM resource reconciliation -------------------------------- +// A SLURM stage runs inside a K8s dispatcher pod that ssh-drives the job on the +// login node. If that pod dies mid-run (eviction, container error, agent +// offline), the in-pod cleanup can no longer reach the controller, so the SLURM +// job and any Jenkins agent node leak. slurmResourceRegistry records the live +// resources per stage -- the dispatcher pod spec (from the pod wrapper) and the +// SLURM job / Jenkins node identity (from the stage body, re-registered per +// inner attempt). Deregistering after a successful cleanup clears only the +// per-attempt job/node identity and keeps the stage-level podSpec, so a *later* +// attempt's dispatcher-pod death can still be reconciled off-pod (the pod spec +// is registered once, before the inner retry loop). Only serializable +// primitives are stored (no SlurmCluster/Throwable) so pipeline persistence is +// unaffected; the cluster is rebuilt from clusterName. Plain maps: pipeline +// Groovy runs single-threaded under CPS (parallel branches interleave at step +// boundaries, never execute Groovy concurrently) and each stage writes its own +// key, so no concurrent map corruption -- and the Jenkins script sandbox forbids +// `new ConcurrentHashMap`. Keyed by stageName. +@Field def slurmResourceRegistry = [:] + +void registerSlurmResource(String stageName, Map fields) { + if (!stageName) { + return + } + def entry = slurmResourceRegistry.get(stageName) + if (entry == null) { + entry = [:] + slurmResourceRegistry.put(stageName, entry) + } + // Skip absent fields. Writers for a given stage run sequentially (pod + // wrapper, then stage body), so no merge race. + fields.each { k, v -> if (v != null) { entry.put(k, v) } } +} + +// Called once an attempt's resources are actually torn down: drop the per-attempt +// job/node identity but keep the stage's podSpec (needed to launch a cleanup pod +// for a later attempt). An entry left with only a podSpec is inert -- finalize +// and the post-build sweep both skip entries with no job/node. +void deregisterSlurmResource(String stageName) { + if (!stageName) { + return + } + def entry = slurmResourceRegistry.get(stageName) + if (entry != null) { + ["clusterName", "jobUID", "nodeName", "slurmJobId", "usedSbatch"].each { entry.remove(it) } + } +} + +// Reconcile one orphaned SLURM entry off the (dead) dispatcher pod: launch a +// fresh short-lived pod and run the normal cleanup from there (scancel the job, +// clean the workspace, and -- agent path -- drop the leaked Jenkins node), then +// deregister. Best-effort: a finalizer failure is logged and the entry is left +// for the post-build sweep to retry; it never masks the stage's own failure. +def finalizeSlurmResourceEntry(pipeline, String stageName, def entry, def podSpecOverride = null) { + if (entry == null) { + return + } + // Pod died before any job/node was provisioned: nothing to reconcile. + if (!entry.jobUID && !entry.nodeName) { + deregisterSlurmResource(stageName) + return + } + def podSpec = podSpecOverride ?: entry.podSpec + def cluster = entry.clusterName ? SlurmConfig.clusterConfig[entry.clusterName] : null + if (!podSpec || !cluster) { + echo "[SLURM-FINALIZER] ${stageName}: cannot reconcile off-pod (missing pod spec or unknown cluster " + + "'${entry.clusterName}'); SLURM job=${entry.slurmJobId ?: entry.jobUID ?: 'unknown'} " + + "node=${entry.nodeName ?: 'n/a'} may need manual cleanup." + return + } + try { + echo "[SLURM-FINALIZER] ${stageName}: reconciling orphaned SLURM resources off-pod " + + "(job=${entry.slurmJobId ?: entry.jobUID}, node=${entry.nodeName ?: 'n/a'})." + trtllm_utils.launchKubernetesPod(pipeline, podSpec, entry.containerName ?: "trt-llm", { + if (entry.usedSbatch) { + cleanUpSlurmResources(pipeline, cluster, entry.clusterName, entry.jobUID) + } else { + cleanUpNodeResources(pipeline, cluster, entry.clusterName, entry.nodeName, entry.slurmJobId) + } + }) + deregisterSlurmResource(stageName) + echo "[SLURM-FINALIZER] ${stageName}: off-pod reconciliation complete." + } catch (Exception e) { + echo "[SLURM-FINALIZER] ${stageName}: off-pod reconciliation failed (${e.toString()}); leaving entry for post-build sweep." + } +} + +def finalizeOrphanedSlurmResource(pipeline, String stageName, def podSpecOverride = null) { + finalizeSlurmResourceEntry(pipeline, stageName, slurmResourceRegistry.get(stageName), podSpecOverride) +} + +// Post-build backstop: reconcile any SLURM resources still registered at the end +// of the build (a dispatcher-pod death whose in-catch finalize also failed, or a +// failure mode the catch never saw). Runs off-pod from a fresh cleanup pod. +def sweepOrphanedSlurmResources(pipeline) { + def orphans = slurmResourceRegistry.keySet().collect { it }.findAll { + def e = slurmResourceRegistry.get(it) + e != null && (e.jobUID || e.nodeName) + } + if (orphans.isEmpty()) { + return + } + echo "[SLURM-FINALIZER] post-build sweep: ${orphans.size()} SLURM resource(s) were not cleaned up " + + "in-stage; reconciling off-pod: ${orphans.join(', ')}" + orphans.each { sName -> + finalizeSlurmResourceEntry(pipeline, sName, slurmResourceRegistry.get(sName)) + } +} + // Authoritative timeout signal: ask the SLURM controller (via sacct on the // cluster login node) for a job's terminal state. Returns the uppercased // primary state token -- e.g. "TIMEOUT", "COMPLETED", "FAILED", "NODE_FAIL", @@ -704,7 +889,7 @@ def querySlurmJobState(def pipeline, SlurmCluster cluster, String clusterName, S } String state = null try { - CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote -> + CloudManager.withSlurmFrontendFailover(pipeline, clusterName, cluster) { remote -> // -X: allocation row only (skip .batch/.extern steps). -Pn: // parsable, no header. First line's first token is the job state; // SLURM renders cancellations as "CANCELLED by ", so we keep @@ -724,7 +909,11 @@ def querySlurmJobState(def pipeline, SlurmCluster cluster, String clusterName, S return state } -def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, skipInstallWheel=false, cpver="cp312", String postTag="", boolean useClusterDurations=false, Map placementContext=null) +boolean isNonTerminalSlurmState(String state) { + return state != null && SLURM_NON_TERMINAL_STATES.contains(state.toUpperCase(java.util.Locale.ROOT)) +} + +def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, skipInstallWheel=false, cpver="cp312", String postTag="", boolean useClusterDurations=false, Map placementContext=null, Map retryContext=null) { SlurmPartition partition = SlurmConfig.resolvePlatform(platform) SlurmCluster cluster = SlurmConfig.clusterConfig[partition.clusterName] @@ -748,7 +937,7 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, try { // Run ssh command to start node in desired cluster via SLURM - CloudManager.withSlurmSshCredentials(pipeline, partition.clusterName, cluster) { remote -> + CloudManager.withSlurmFrontendFailover(pipeline, partition.clusterName, cluster) { remote -> stage('Request Node Via Slurm') { println("Selected Cluster: ${cluster.name}") @@ -802,6 +991,11 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, slurmJobID = jobIDs ? jobIDs[-1] : null + // Record the live SLURM job + Jenkins node so a dispatcher-pod death + // can be reconciled off-pod (the in-pod cleanup can't reach the login + // node once the pod is gone). Deregistered when cleanup actually runs. + registerSlurmResource(stageName, [clusterName: partition.clusterName, nodeName: nodeName, slurmJobId: slurmJobID, usedSbatch: false]) + if (!slurmJobID || !slurmJobID.isNumber()) { echo "Slurm job did not submit successfully. No job ID found.\nSubmission output:\n${slurmSubmitOutput}" } @@ -817,12 +1011,14 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, def jobRunningStartMs = null stage('Check If Node Is Online') { - CloudManager.withSlurmSshCredentials(pipeline, partition.clusterName, cluster) { remote -> + CloudManager.withSlurmSshCredentialRemotes(pipeline, partition.clusterName, cluster) { remotes -> // Check the SLURM job once; if it is no longer active, raise a typed // InfraFailure(SLURM) so the retry layer routes it via instanceof (scope=SLURM). def checkSlurmJobActive = { try { - SlurmConfig.checkJobStatus(pipeline, cluster, slurmJobID, remote) + CloudManager.withSlurmFrontendFailover(pipeline, remotes) { statusRemote -> + SlurmConfig.checkJobStatus(pipeline, cluster, slurmJobID, statusRemote) + } } catch (InterruptedException e) { throw e } catch (Exception e) { @@ -845,8 +1041,8 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, // which overflowed the per-stage step cap). Release the held job every 10 // iterations (~30 min). 300 iterations * 3 min = 15h budget. // Exit codes: 0 = job RUNNING, 3 = job no longer active, 4 = timed out. - def sacctStateCmd = Utils.sshUserCmd(remote, "\"sacct -j ${slurmJobID} --format=State -Pn --allocations\"") - def releaseCmd = Utils.sshUserCmd(remote, "\"scontrol release ${slurmJobID} || true\"") + def sacctStateCmd = CloudManager.sshUserCmdWithSlurmFrontendFailover(remotes, "\"sacct -j ${slurmJobID} --format=State -Pn --allocations\"") + def releaseCmd = CloudManager.sshUserCmdWithSlurmFrontendFailover(remotes, "\"scontrol release ${slurmJobID} || true\"") def waitRc = pipeline.sh(returnStatus: true, script: """ set +e counter=0 @@ -967,7 +1163,7 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, def setupLogPath = "/home/svc_tensorrt/slurm-logs/slurm-${slurmJobID}-${nodeName}.out" def enrootLog = Utils.exec( pipeline, - script: Utils.sshUserCmd(remote, "\"grep '\\[ENROOT\\]' ${setupLogPath} 2>/dev/null || true\""), + script: CloudManager.sshUserCmdWithSlurmFrontendFailover(remotes, Utils.bashWrappedRemoteCmd("grep '\\[ENROOT\\]' ${setupLogPath} 2>/dev/null || true")), returnStdout: true, numRetries: 3 ).trim() @@ -993,7 +1189,7 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, } long executeStartMs = System.currentTimeMillis() try { - executeLLMTestOnSlurm(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, slurmRunner, postTag, useClusterDurations) + executeLLMTestOnSlurm(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, slurmRunner, postTag, useClusterDurations, retryContext) } catch (InterruptedException e) { throw e } catch (Exception e) { @@ -1024,6 +1220,14 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, e) } + if (isNonTerminalSlurmState(slurmState)) { + throw new InfraFailure( + "SLURM job ${slurmJobID} for ${stageName} is still in non-terminal state ${slurmState} " + + "(${elapsedMin}min of ${walltimeMin}min walltime); the monitor lost contact while the job was " + + "alive (transient infra), so this is not a test failure. Original failure: ${e.message}", + e, InfraFailure.TRANSIENT, InfraFailure.SLURM, "") + } + echo "[INFRA-RETRY] ${stageName}: SLURM job ${slurmJobID} terminal state=${slurmState ?: 'unknown'}, " + "ran ${elapsedMin}min of ${walltimeMin}min walltime; deferring to failure classifier." throw e @@ -1043,11 +1247,14 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, } } } + // Cleanup ran on the live pod; drop the registry entry so the off-pod + // finalizer/sweep does not reconcile already-freed resources. + deregisterSlurmResource(stageName) } } } -def executeLLMTestOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", runner, String postTag="", boolean useClusterDurations=false) +def executeLLMTestOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", runner, String postTag="", boolean useClusterDurations=false, Map retryContext=null) { runner { // TODO: refactor the finallyRunner to reuse within slurm or nonslurm job. @@ -1074,7 +1281,7 @@ def executeLLMTestOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p // Copy CPP test result sh "cp ${llmSrc}/cpp/build_backup/*.xml ${stageName} || true" sh "ls -al ${stageName}/" - }, false, postTag) + }, false, postTag, true, retryContext) } } // End of Methods to run Slurm job with Jenkins Agent @@ -1120,6 +1327,13 @@ def getPytestBaseCommandLine( extraInternalEnv += " NCCL_DEBUG=INFO" // Pass stage name to perf sanity tests for OpenSearch tracking extraInternalEnv += " stageName=${stageName}" + // Persist the AutoTuner profiling cache to a CONTAINER-LOCAL, volatile path so + // that repeated tactic profiling is reused across testcases within one stage. + // /tmp lives on the container overlay (srun --no-container-mount-home / fresh + // pod), so the cache is never written to the host and vanishes when the stage + // container is destroyed. Never point this at a bind-mounted / shared path: + // the AutoTuner cache uses fcntl.lockf, which is unreliable over NFS. + extraInternalEnv += " TLLM_AUTOTUNER_CACHE_PATH=/tmp/trtllm_autotuner_cache/autotuner_cache.json" // CBTS stages put cbts_plugin on PYTHONPATH (via ${VAR:-} for set -u safety) plus the marker/config env vars sitecustomize.py reads in subprocesses. if (cbtsMode) { def cbtsScriptDir = "${llmSrc}/jenkins/scripts/cbts/coverage_utils" @@ -1181,6 +1395,9 @@ def getPytestBaseCommandLine( } def unittestMarkExpr = (stageName.startsWith("CPU-")) ? "cpu_only and not disabled" : "not cpu_only" testCmdLine += ["--unittest-markexpr='${unittestMarkExpr}'"] + if (ENABLE_UPLOAD_TEST_RESULTS) { + testCmdLine += ["-o console_output_style=progress-even-when-capture-no"] + } if (extraArgs) { testCmdLine += extraArgs } @@ -1224,7 +1441,7 @@ def getMountListForSlurmTest(SlurmCluster cluster, boolean useSbatch = false) return mounts } -def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, skipInstallWheel=false, cpver="cp312", String postTag="", boolean useClusterDurations=false, Map placementContext=null) +def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, skipInstallWheel=false, cpver="cp312", String postTag="", boolean useClusterDurations=false, Map placementContext=null, Map retryContext=null) { SlurmPartition partition = SlurmConfig.resolvePlatform(platform) SlurmCluster cluster = SlurmConfig.clusterConfig[partition.clusterName] @@ -1245,6 +1462,10 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG Utils.exec(pipeline, script: "env | sort && pwd && ls -alh") def stageIsInterrupted = false + // Captured so the finally can suppress this attempt's junit when the failure is + // a retryable infra failure (a retry follows) -- otherwise a stage that fails + // an intermediate attempt and passes on retry leaves the build UNSTABLE. + def caughtStageError = null try { // Run ssh command to start node in desired cluster via SLURM @@ -1252,7 +1473,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), ]) { - CloudManager.withSlurmSshCredentials(pipeline, partition.clusterName, cluster) { remote -> + CloudManager.withSlurmFrontendFailover(pipeline, partition.clusterName, cluster) { remote -> def tarName = BUILD_CONFIGS[config][TARNAME] def llmTarfile = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/${tarName}" def llmPath = sh (script: "realpath .", returnStdout: true).trim() @@ -1323,11 +1544,20 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def makoOptsJson = transformMakoArgsToJson(["Mako options:"] + makoArgs) String clusterNameForDurations = useClusterDurations ? partition.clusterName.replaceAll('[^a-zA-Z0-9]', '_') : null def testListPathLocal = renderTestDB(pipeline, testList, llmSrcLocal, stageName, makoOptsJson, clusterNameForDurations) + // Copy the test list atomically. A retry that reuses a still-active job + // re-copies over ${testListPathNode} while that job may be reading it via + // --test-list; scp truncates-then-streams, so a concurrent read could see a + // partial list and silently run a subset. Stage to a temp path and mv into + // place (same-dir rename is atomic) so a reader sees the whole old or new file. Utils.copyFileToRemoteHost( pipeline, remote, testListPathLocal, - testListPathNode + "${testListPathNode}.tmp" + ) + Utils.exec( + pipeline, + script: Utils.sshUserCmd(remote, "\"mv -f ${testListPathNode}.tmp ${testListPathNode}\"") ) // Download and Merge waives.txt @@ -1406,7 +1636,10 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "--s3-upload-path=${uploadPath}/${stageName}", ] if (ENABLE_S3_ECHO_STDOUT) { - extraArgs += ["--s3-echo-stdout"] + extraArgs += [ + "--s3-echo-stdout", + "--s3-capture-mode=timestamped", + ] } } def pytestCommand = getPytestBaseCommandLine( @@ -1626,7 +1859,8 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG --script-prefix ${scriptLaunchPrefixPathLocal} \\ --srun-args ${scriptLaunchSrunArgsPathLocal} \\ --split-group ${splitId} \\ - --stage-name ${stageName} + --stage-name ${stageName} \\ + --cluster-name ${partition.clusterName} """ } else { if(nodeCount > 1) { @@ -1671,13 +1905,26 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG set -xEeuo pipefail trap 'rc=\$?; echo "Error in file \${BASH_SOURCE[0]} on line \$LINENO: \$BASH_COMMAND (exit \$rc)"; exit \$rc' ERR - # Clean up previous job intermediate files so that retry can work + # Reuse an already-active job after an ambiguous frontend disconnect. if [ -f "${jobWorkspace}/slurm_job_id.txt" ]; then previous_job_id=\$(cat "${jobWorkspace}/slurm_job_id.txt") echo "Found previous Slurm job ID: \${previous_job_id}" - scancel "\${previous_job_id}" || true - # Wait for 120 seconds to ensure the previous job is canceled - sleep 120 + previous_state=\$(sacct -j "\${previous_job_id}" --format=State -Pn --allocations 2>/dev/null | head -1 | cut -d'|' -f1 | awk '{print \$1}' || true) + if [ -z "\${previous_state}" ]; then + previous_state=\$(scontrol show job "\${previous_job_id}" 2>/dev/null | tr ' ' '\\n' | sed -n 's/^JobState=//p' | head -1 || true) + fi + case "\${previous_state}" in + RUNNING|PENDING|CONFIGURING|COMPLETING|REQUEUED|RESIZING|SUSPENDED|SIGNALING|STOPPED) + echo "Reusing active Slurm job \${previous_job_id} in state \${previous_state}" + exit 0 + ;; + *) + echo "Previous Slurm job \${previous_job_id} is not active (state='\${previous_state:-UNKNOWN}'). Cleaning it up before resubmission." + scancel "\${previous_job_id}" || true + # Wait for 120 seconds to ensure the previous job is canceled + sleep 120 + ;; + esac fi # Clean up workspace: remove all files/dirs not in the keep list @@ -1706,14 +1953,15 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG } stage("[${stageName}] Run Pytest") { - // Submit the Slurm job + // Submit the Slurm job. Submit/metadata/track all run on the one + // frontend the enclosing withSlurmFrontendFailover pinned, so they + // share the job workspace (slurm_job_id.txt, scripts) on that login + // node; a frontend disconnect fails the whole closure over to a fresh + // frontend as a unit (the submit script reuses an active job). Utils.exec( pipeline, timeout: false, - script: Utils.sshUserCmd( - remote, - scriptSubmitPathNode - ), + script: Utils.sshUserCmd(remote, scriptSubmitPathNode), numRetries: 3 ) @@ -1722,16 +1970,17 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG if (!slurmJobId) { slurmJobId = Utils.exec( pipeline, - script: Utils.sshUserCmd( - remote, - "\"cat ${jobWorkspace}/slurm_job_id.txt\"" - ), + script: Utils.sshUserCmd(remote, "\"cat ${jobWorkspace}/slurm_job_id.txt\""), returnStdout: true, numRetries: 3 ).trim() recordSlurmPlacementContext(placementContext, slurmJobId, null, stageName) } Utils.exec(pipeline, script: "echo Slurm job ID: ${slurmJobId}") + // Record the live SLURM job so a dispatcher-pod death can be reconciled + // off-pod (the in-pod cleanup can't reach the login node once the pod is + // gone). Deregistered when cleanup actually runs. + registerSlurmResource(stageName, [clusterName: partition.clusterName, jobUID: jobUID, slurmJobId: slurmJobId, usedSbatch: true]) def scriptTrack = """#!/bin/bash set -xEeuo pipefail @@ -1795,15 +2044,18 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG echo "Slurm job \$jobId nodelist: \${NODE_LIST:-UNKNOWN}" printf '%s\n' "\$NODE_LIST" > "${jobWorkspace}/slurm_node_list.txt" + # Record the verdict and always exit 0: a re-run can't change a + # terminal state, so numRetries should only fire on transport loss. + printf '%s %s\n' "\$STATUS" "\$EXIT_CODE" > "${jobWorkspace}/slurm_job_result.txt" if [[ "\$STATUS" == "COMPLETED" && \$EXIT_CODE -eq 0 ]]; then echo "Pytest succeed in Slurm job \$jobId" - echo "Status: \$STATUS | Exit_code \$EXIT_CODE" - exit 0 else echo "Pytest failed in Slurm job \$jobId" - echo "Status: \$STATUS | Exit_code \$EXIT_CODE" - exit 1 + echo "Full test output (logs not shown above) is uploaded after stage teardown to:" + echo " https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/test-results/results-${stageName}${postTag}.tar.gz" fi + echo "Status: \$STATUS | Exit_code \$EXIT_CODE" + exit 0 """.replaceAll("(?m)^\\s*", "").trim() pipeline.writeFile(file: scriptTrackPathLocal, text: scriptTrack) @@ -1816,53 +2068,93 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG true ) - // Track the Slurm job - try { - Utils.exec( - pipeline, - timeout: false, - script: Utils.sshUserCmd( - remote, - scriptTrackPathNode - ), - numRetries: 3 + // Monitor the job. The track script always exits 0 once it records a + // verdict, so a re-run can't change a terminal state. A frontend lost + // mid-monitor is recovered by the enclosing withSlurmFrontendFailover + // -- it fails the closure over to another frontend and the submit guard + // reuses the still-active job -- so the monitor needs no same-frontend + // retries of its own. + Utils.exec( + pipeline, + timeout: false, + script: Utils.sshUserCmd( + remote, + scriptTrackPathNode ) - } catch (InterruptedException e) { - throw e - } catch (Exception e) { - // The track script squashes the job's terminal SLURM state to - // exit 0/1, so a walltime kill is indistinguishable here from a - // real test failure. Re-query sacct for the allocation-level - // state (SLURM already aggregates it across nodes; TIMEOUT if - // any node hit the walltime) and, when it is TIMEOUT, raise a - // typed UserFailure so neither the SLURM retry loop nor the - // outer K8s pod retry re-runs a job that would just time out - // again. srun --kill-on-bad-exit=1 means a genuine test failure - // surfaces as FAILED, not TIMEOUT, so this stays unambiguous. - def slurmState = querySlurmJobState(pipeline, cluster, partition.clusterName, slurmJobId) + ) + + // Verdict: " "; success is COMPLETED + exit 0. + def jobResult = readSlurmWorkspaceFile(pipeline, remote, "${jobWorkspace}/slurm_job_result.txt", stageName, 3) + def resultFields = jobResult ? jobResult.tokenize(' ') : [] + def jobState = resultFields ? resultFields[0] : null + def jobExit = resultFields.size() > 1 ? resultFields[1] : null + if (jobState != "COMPLETED" || jobExit != "0") { + // Verdict unreadable: fall back to an authoritative sacct query. + def slurmState = jobState ?: querySlurmJobState(pipeline, cluster, partition.clusterName, slurmJobId) + // ... and re-confirm success, so a transient read blip on a job + // that actually passed doesn't fail the stage. + if (jobState == null && slurmState == "COMPLETED") { + echo "[INFRA-RETRY] ${stageName}: verdict unreadable but sacct reports COMPLETED for ${slurmJobId}; treating as success." + return + } + // TIMEOUT is a walltime kill -- typed UserFailure so neither + // retry layer re-runs a job that would just time out again. if (slurmState == "TIMEOUT") { throw new UserFailure( "SLURM job ${slurmJobId} for ${stageName} ended in state TIMEOUT " + - "(hit partition walltime ${partition?.time}min); treating as a test timeout, not retrying. " + - "Original failure: ${e.message}", - e) + "(hit partition walltime ${partition?.time}min); not retrying.", + null) } - echo "[INFRA-RETRY] ${stageName}: SLURM job ${slurmJobId} terminal state=${slurmState ?: 'unknown'}; " + - "deferring to failure classifier." - throw e + // Verdict unreadable but the job is still alive: a transport blip + // dropped the monitor while the job kept running, so this is infra, + // not a test failure. + if (isNonTerminalSlurmState(slurmState)) { + throw new InfraFailure( + "SLURM job ${slurmJobId} for ${stageName} is still in non-terminal state ${slurmState}; " + + "the monitor lost contact while the job was alive (transient infra), so this is not a " + + "test failure.", + null, InfraFailure.TRANSIENT, InfraFailure.SLURM, "") + } + // A terminal FAILED state may be a node/device fault whose signature + // (CUDA/NVLink/ECC/driver) printed only into the SLURM job output log, + // never into this verdict. Scrape the log and, on a hit, surface the + // matched line into a fresh exception so the authoritative catalog + // (FailureClassifier.classify at the runLLMTestlistWithSbatch caller) + // can match it and steer the retry off the bad node. A miss falls + // through to the plain "Pytest failed" rethrow below. + if (slurmState == "FAILED") { + def deviceHit = scrapeSlurmLogForDeviceFault(pipeline, remote, slurmJobLogPath) + if (deviceHit) { + echo "[INFRA-RETRY] ${stageName}: device-fault signature in SLURM job ${slurmJobId} log; " + + "surfacing to classifier: ${deviceHit}" + throw new Exception( + "Device/interconnect fault on SLURM node during job ${slurmJobId} for ${stageName}: ${deviceHit}") + } + } + echo "[INFRA-RETRY] ${stageName}: SLURM job ${slurmJobId} state=${slurmState ?: 'unknown'}, exit=${jobExit ?: 'unknown'}; deferring to classifier." + throw new Exception("Pytest failed in SLURM job ${slurmJobId} for ${stageName}") } } echo "Finished test stage execution." - } // end CloudManager.withSlurmSshCredentials + } // end CloudManager.withSlurmFrontendFailover } // end withCredentials } catch (InterruptedException e) { stageIsInterrupted = true throw e + } catch (Exception e) { + caughtStageError = e + throw e } finally { // Resource cleanup must run even if metadata capture or result upload is interrupted. try { captureSlurmJobNodeList(pipeline, cluster, partition.clusterName, placementContext?.slurmJobId ?: null, placementContext, stageName, jobWorkspace) - uploadResults(pipeline, cluster, partition.clusterName, jobUID, stageName, stageIsInterrupted, postTag) + // Suppress this attempt's junit when a retry is still planned (a retryable + // infra failure with budget), so a retried-and-passed stage doesn't leave + // the build UNSTABLE from an intermediate attempt's results. A genuine test + // failure classifies as UserFailure -> not suppressed -> reported. + boolean suppressTestReporting = (caughtStageError != null && retryContext != null) && + retryContextAllowsRetry(null, retryContext, caughtStageError, false) + uploadResults(pipeline, cluster, partition.clusterName, jobUID, stageName, stageIsInterrupted, postTag, suppressTestReporting) } finally { stage("Clean Up Slurm Resource") { // Workaround to handle the interruption during clean up SLURM resources @@ -1874,6 +2166,9 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG } } } + // Cleanup ran on the live pod; drop the registry entry so the off-pod + // finalizer/sweep does not reconcile already-freed resources. + deregisterSlurmResource(stageName) } } } @@ -1907,10 +2202,34 @@ def cbtsResizeSplits(configs) { return resized } -def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, runWithSbatch=false, skipInstallWheel=false, cpver="cp312", String outerAttemptTag="", boolean useClusterDurations=false) +// True when an exception indicates the K8s dispatcher pod this SLURM stage runs +// inside died mid-run -- kubelet eviction, container termination, or the JNLP +// agent otherwise going offline. Retrying inside such a pod is futile (every +// step runs on the dead agent and fails immediately) and its in-pod cleanup can +// no longer reach the SLURM controller, so callers stop retrying in place and +// reconcile the orphaned SLURM job / Jenkins node off-pod. Matches the flattened +// cause chain so the signal is still recognized when wrapped by the cleanup's +// AbortException (e.g. "Error during clean up SLURM resources: ... marked +// offline: Pod failed (Reason: Evicted ...)"). +boolean isDispatcherPodFailure(Throwable e) { + def text = FailureClassifier.flattenThrowable(e).collect { it.toString() }.join(" ").toLowerCase() + return [ + "pod failed (reason:", + "pod just failed", + "pod failed because container terminated", + "unable to create live filepath", + ].any { text.contains(it) } +} + +def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, runWithSbatch=false, skipInstallWheel=false, cpver="cp312", String outerAttemptTag="", boolean useClusterDurations=false, Integer infraRetryMax=null) { echo "Run Slurm job with native sbatch: $runWithSbatch" + // Per-stage override of the SLURM infra-retry budget (from opts.infraRetryMax, + // threaded via the dispatcher's retryContext). Lets resource-scarce pools cap + // or disable stage-level retries (0 = no retry). Null falls back to the global. + int slurmInfraRetryMax = resolveInfraRetryMax(InfraFailure.SLURM, infraRetryMax) + def attempt = 0 // Avoided SLURM nodes keyed by cluster. The platform can resolve to a // different cluster on each attempt (auto: platforms pick one at random), and @@ -1927,7 +2246,7 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p ] try { if (attempt > 1) { - echo "[INFRA-RETRY] ${stageName}: Starting attempt ${attempt} of ${SLURM_INFRA_RETRY_MAX + 1}" + echo "[INFRA-RETRY] ${stageName}: Starting attempt ${attempt} of ${slurmInfraRetryMax + 1}" if (!avoidedSlurmNodeListsByCluster.isEmpty()) { echo "[INFRA-RETRY] ${stageName}: avoiding prior SLURM node list(s): " + avoidedSlurmNodeListsByCluster.collect { c, ns -> "${c}: ${ns.join(' ')}" }.join('; ') @@ -1948,10 +2267,20 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p def innerSuffix = (attempt == 1) ? "" : "-attempt-${attempt}" def postTag = "${outerAttemptTag}${innerSuffix}" + // Describes this attempt so the stage body can suppress its junit when a + // retryable infra failure means another attempt will follow (mirrors the K8s + // path's retryContext). scope=SLURM so classification/budget match this loop. + def slurmRetryContext = [ + scope: InfraFailure.SLURM, + stageName: stageName, + attempt: attempt, + backoffMs: 60L * 1000L, + ] + if (nodeCount > 1 || runWithSbatch) { - runLLMTestlistWithSbatch(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, nodeCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext) + runLLMTestlistWithSbatch(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, nodeCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext) } else { - runLLMTestlistWithAgent(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext) + runLLMTestlistWithAgent(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext) } // Job succeeded @@ -1964,6 +2293,15 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p // User abort / pipeline timeout -- never retry throw e } catch (Exception e) { + // If the K8s dispatcher pod this stage runs inside died mid-run, every + // retry attempt would execute on that same dead pod and fail immediately, + // and the in-pod cleanup can no longer reach the SLURM controller. Stop + // retrying in place and propagate so the pod-level wrapper reconciles the + // orphaned SLURM job / Jenkins node off-pod (fail closed). + if (isDispatcherPodFailure(e)) { + echo "[INFRA-RETRY] ${stageName}: dispatcher pod died mid-run; not retrying on the dead pod (${e.toString()}). Failing closed for off-pod reconciliation." + throw e + } // classify() handles FlowInterruptedException + exit-code-143 + // typed throws + cause-chain pattern matching, returning one of // PipelineInterruption / InfraFailure / UserFailure. Scope=SLURM @@ -1974,7 +2312,7 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p rememberAvoidedSlurmNodeLists(avoidedSlurmNodeListsByCluster, attemptPlacementContext.lastSlurmClusterName, attemptPlacementContext.lastSlurmNodeList, stageName) - def effectiveMax = (c.severity == InfraFailure.PERSISTENT) ? 1 : SLURM_INFRA_RETRY_MAX + def effectiveMax = (c.severity == InfraFailure.PERSISTENT) ? Math.min(1, slurmInfraRetryMax) : slurmInfraRetryMax if (attempt > effectiveMax) { echo "[INFRA-RETRY] ${stageName}: Infrastructure failure (${c.detectedPattern}) " + @@ -2157,12 +2495,27 @@ long retrySafetyMarginMs(String scope) return scope == InfraFailure.SLURM ? 20L * 60L * 1000L : 15L * 60L * 1000L } -int retryMaxForFailure(String scope, InfraFailure failure) +// Resolve a per-stage infra-retry override (opts.infraRetryMax) against its +// scope's global budget. The override may only CAP or DISABLE retries, never +// exceed the global (which bounds CI time / worst-case attempts), so it is +// clamped to [0, scopeDefault]; e.g. with the default of 1, infraRetryMax=2 +// still yields 1, and infraRetryMax=0 disables. Null falls back to the default. +int resolveInfraRetryMax(String scope, Integer override) { - if (failure.severity == InfraFailure.PERSISTENT) { - return 1 + int scopeDefault = (scope == InfraFailure.SLURM) ? SLURM_INFRA_RETRY_MAX : K8S_INFRA_RETRY_MAX + if (override == null) { + return scopeDefault } - return scope == InfraFailure.SLURM ? SLURM_INFRA_RETRY_MAX : K8S_INFRA_RETRY_MAX + return Math.max(0, Math.min(override, scopeDefault)) +} + +int retryMaxForFailure(String scope, InfraFailure failure, Integer infraRetryMax=null) +{ + // Per-stage override caps the scope default (see resolveInfraRetryMax); a + // PERSISTENT failure is still capped to at most one retry, so infraRetryMax=0 + // disables retries for every severity. + int base = resolveInfraRetryMax(scope, infraRetryMax) + return (failure.severity == InfraFailure.PERSISTENT) ? Math.min(1, base) : base } boolean hasBudgetForInfraRetry(def pipeline, String stageName, String scope, InfraFailure failure, int attempt, int effectiveMax, long backoffMs, boolean logDecision) @@ -2194,7 +2547,7 @@ boolean retryContextAllowsRetry(def pipeline, Map retryContext, Throwable error, } Long parsedAttempt = trtllm_utils.parseCiBudgetLong(retryContext.attempt) int attempt = parsedAttempt != null ? parsedAttempt as int : 1 - int effectiveMax = retryMaxForFailure(scope, classified) + int effectiveMax = retryMaxForFailure(scope, classified, retryContext.infraRetryMax as Integer) Long parsedBackoffMs = trtllm_utils.parseCiBudgetLong(retryContext.backoffMs) long backoffMs = parsedBackoffMs != null ? parsedBackoffMs : 60L * 1000L return hasBudgetForInfraRetry(pipeline, retryContext.stageName ?: "Unknown", scope, classified, attempt, effectiveMax, backoffMs, logDecision) @@ -2245,7 +2598,7 @@ def readSlurmWorkspaceFile(def pipeline, Map remote, String path, String stageNa pipeline, script: Utils.sshUserCmd( remote, - "\"cat ${path} 2>/dev/null || true\"" + Utils.bashWrappedRemoteCmd("cat ${path} 2>/dev/null || true") ), returnStdout: true, numRetries: numRetries @@ -2254,6 +2607,13 @@ def readSlurmWorkspaceFile(def pipeline, Map remote, String path, String stageNa } catch (InterruptedException e) { throw e } catch (Exception e) { + // A dead frontend must propagate so the enclosing withSlurmFrontendFailover + // fails over to another remote; swallowing it as "" would strand the stage on + // the unreachable frontend. Any other read failure (missing file, transient) + // is non-fatal -- the metadata is best-effort, so return "" and carry on. + if (CloudManager.isSlurmFrontendConnectionFailure(e)) { + throw e + } echo "[INFRA-RETRY] ${stageName}: unable to read SLURM metadata file ${path}: ${e.toString()}" return "" } @@ -2308,7 +2668,7 @@ def captureSlurmJobNodeList(def pipeline, SlurmCluster cluster, String clusterNa def capturedJobID = slurmJobID def nodeList = null try { - CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote -> + CloudManager.withSlurmFrontendFailover(pipeline, clusterName, cluster) { remote -> def metadata = captureSlurmWorkspaceMetadata(pipeline, remote, jobWorkspace, placementContext, stageName) capturedJobID = capturedJobID ?: metadata.slurmJobId nodeList = metadata.nodeList @@ -2507,7 +2867,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod def extraDeviceEnv = "" def archSuffix = arch == "arm64" ? "arm" : "amd" - def jnlpImage = "urm.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" + def jnlpImage = "artifactory.pdx.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" switch(type) { @@ -2820,7 +3180,7 @@ def runLLMDocBuild(pipeline, config) sh "pwd && ls -alh" sh "env | sort" // allow to checkout from forked repo, svc_tensorrt needs to have access to the repo, otherwise clone will fail - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) sh "mkdir TensorRT-LLM" sh "cp -r ${LLM_ROOT}/ TensorRT-LLM/src/" trtllm_utils.llmExecStepWithRetry(pipeline, script: "git config --global --add safe.directory \"*\"") @@ -3028,10 +3388,6 @@ def getMakoArgsFromStageName(stageName, parseSysinfo=false) { // If stageName contains "-PyTorch-", add "backend=pytorch" to makoArgs // At this point, only tests with backend=pytorch or unspecified backend will be run makoArgs += ["backend=pytorch"] - } else if (stageName.contains("-TensorRT-")) { - // If stageName contains "-TensorRT-", add "backend=tensorrt" to makoArgs - // At this point, only tests with backend=tensorrt or unspecified backend will be run - makoArgs += ["backend=tensorrt"] } else if (stageName.contains("-CPP-")) { // If stageName contains "-CPP-", add "backend=cpp" to makoArgs // At this point, only tests with backend=cpp or unspecified backend will be run @@ -3056,7 +3412,7 @@ def getMakoArgsFromStageName(stageName, parseSysinfo=false) { // At this point, only tests with backend=verl or unspecified backend will be run makoArgs += ["backend=verl"] } else { - // If stageName does not contain "-PyTorch-", "-TensorRT-", "-CPP-", "-Triton-", "-FMHA-", "-AutoDeploy-", or "-Verl-", do not add any backend + // If stageName does not contain "-PyTorch-", "-CPP-", "-Triton-", "-FMHA-", "-AutoDeploy-", or "-Verl-", do not add any backend // At this point, all tests will be run // For cases where backend is not specified in makoArgs, we will match all types of backends and tests without specified backend } @@ -3887,7 +4243,10 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO "--s3-upload-path=${uploadPath}/${stageName}", ] if (ENABLE_S3_ECHO_STDOUT) { - extraArgs += ["--s3-echo-stdout"] + extraArgs += [ + "--s3-echo-stdout", + "--s3-capture-mode=timestamped", + ] } } def pytestCommand = getPytestBaseCommandLine( @@ -3918,11 +4277,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO withEnv(["LD_LIBRARY_PATH=${containerLD_LIBRARY_PATH}"]) { withCredentials([ string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), - usernamePassword( - credentialsId: 'svc_tensorrt_gitlab_read_api_token', - usernameVariable: 'GITLAB_API_USER', - passwordVariable: 'GITLAB_API_TOKEN' - ), string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), string(credentialsId: 'llm_evaltool_repo_url', variable: 'EVALTOOL_REPO_URL') ]) { @@ -4244,7 +4598,7 @@ def runLLMBuild( sh "env | sort" sh "ccache -sv" - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, "tensorrt_llm", false, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, "tensorrt_llm", true, true) if (env.alternativeTRT) { sh "cd tensorrt_llm/ && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" } @@ -4387,9 +4741,12 @@ def runPackageSanityCheck(pipeline, wheel_path, reinstall_dependencies=false, cp trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget -nv ${pkgUrl}") sh "tar -zvxf ${linuxPkgName}" - trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core/gpt && python3 ../../../generate_checkpoint_config.py --architecture GPTForCausalLM --dtype float16'") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core//gpt && trtllm-build --model_config config.json --log_level verbose'") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core/gpt && python3 ../../../run.py --max_output_len 4 --end_id -1'") + // TODO: The steps below drove the removed TensorRT engine flow (trtllm-build / examples/run.py). + // When re-enabling this sanity check, use PyTorch backend test samples instead + // (e.g. examples/llm-api/quickstart_example.py). + // trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core/gpt && python3 ../../../generate_checkpoint_config.py --architecture GPTForCausalLM --dtype float16'") + // trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core//gpt && trtllm-build --model_config config.json --log_level verbose'") + // trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core/gpt && python3 ../../../run.py --max_output_len 4 --end_id -1'") } def checkStageNameSet(stageNames, jobKeys, paramName) { @@ -4509,6 +4866,15 @@ def runInKubernetes(pipeline, podSpec, containerName) def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerName, String stageName, Closure runner) { boolean singleAttempt = opts.singleAttempt ?: false + // SLURM dispatcher pods opt in to off-pod resource reconciliation: on a + // mid-run pod death their SLURM job / Jenkins node would otherwise leak. + boolean slurmDispatcher = opts.slurmDispatcher ?: false + // Per-stage override of the K8s pod-level infra-retry budget (opts.infraRetryMax, + // 0 = no retry) so resource-scarce pools can cap or disable stage retries. Applies + // to the outer test-pod retry loop below; the singleAttempt launch-retry keeps the + // global (relaunching a dispatcher pod doesn't tax the scarce test hardware). SLURM + // dispatchers carry the same opt through to their inner SLURM retry via retryContext. + int k8sInfraRetryMax = resolveInfraRetryMax(InfraFailure.K8S, opts.infraRetryMax as Integer) // DEBUG_MODE preserves the existing 2-hour-input human-inspection workflow // inside runLLMTestlistOnPlatform's finallyRunner: a single attempt only. @@ -4536,6 +4902,11 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN echo "[INFRA-RETRY] ${stageName}: relaunching pod (attempt ${launchAttempt}), avoiding prior host node(s): ${avoidedKubernetesHostNodes.join(', ')}" } def attemptPodSpec = trtllm_utils.withKubernetesHostNodeExclusion(podSpec, avoidedKubernetesHostNodes) + if (slurmDispatcher) { + // Record the dispatcher pod spec so the off-pod finalizer/sweep can + // launch a fresh cleanup pod if this pod dies mid-run. + registerSlurmResource(stageName, [podSpec: attemptPodSpec, containerName: containerName]) + } trtllm_utils.launchKubernetesPodWithPlacement(pipeline, attemptPodSpec, containerName, attemptPlacementContext, { attemptPlacementContext.runnerStarted = true runner("", true, null) @@ -4545,8 +4916,18 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN throw e } catch (Exception e) { // Once the runner has started, this is an execution failure the - // inner retry owns -- honor singleAttempt and do not re-run. + // inner retry owns -- honor singleAttempt and do not re-run. But if + // the dispatcher pod itself died mid-run, its in-pod cleanup could not + // reach the login node, leaking the SLURM job / Jenkins node. We are + // back on the parent context here, so reconcile them from a fresh + // cleanup pod before failing closed. if (attemptPlacementContext.runnerStarted) { + if (slurmDispatcher && isDispatcherPodFailure(e)) { + // Pass this attempt's pod spec explicitly so the finalizer + // never depends on the registry entry's podSpec surviving a + // prior attempt's deregister. + finalizeOrphanedSlurmResource(pipeline, stageName, attemptPodSpec) + } throw e } def c = FailureClassifier.classify(e, InfraFailure.K8S) @@ -4586,7 +4967,7 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN Map attemptPlacementContext = [:] try { if (attempt > 1) { - echo "[INFRA-RETRY] ${stageName}: Starting attempt ${attempt} of ${K8S_INFRA_RETRY_MAX + 1}" + echo "[INFRA-RETRY] ${stageName}: Starting attempt ${attempt} of ${k8sInfraRetryMax + 1}" if (!avoidedKubernetesHostNodes.isEmpty()) { echo "[INFRA-RETRY] ${stageName}: avoiding prior Kubernetes host node(s): ${avoidedKubernetesHostNodes.join(', ')}" } @@ -4607,13 +4988,14 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN // cacheErrorAndUploadResult does not suppress synthetic stage-fail // XML / junit() on what would otherwise look (to it) like just // another intermediate attempt. - def effectiveMaxThisAttempt = (lastSeverity == InfraFailure.PERSISTENT) ? 1 : K8S_INFRA_RETRY_MAX + def effectiveMaxThisAttempt = (lastSeverity == InfraFailure.PERSISTENT) ? Math.min(1, k8sInfraRetryMax) : k8sInfraRetryMax boolean isFinalAttempt = (attempt > effectiveMaxThisAttempt) def retryContext = [ scope: InfraFailure.K8S, stageName: stageName, attempt: attempt, backoffMs: 60L * 1000L, + infraRetryMax: opts.infraRetryMax, excludedKubernetesHostNodes: avoidedKubernetesHostNodes.collect(), ] def attemptPodSpec = trtllm_utils.withKubernetesHostNodeExclusion(podSpec, avoidedKubernetesHostNodes) @@ -4639,7 +5021,7 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN rememberAvoidedKubernetesHostNodes(avoidedKubernetesHostNodes, attemptPlacementContext.lastHostNode, stageName) - def effectiveMax = (c.severity == InfraFailure.PERSISTENT) ? 1 : K8S_INFRA_RETRY_MAX + def effectiveMax = (c.severity == InfraFailure.PERSISTENT) ? Math.min(1, k8sInfraRetryMax) : k8sInfraRetryMax if (attempt > effectiveMax) { echo "[INFRA-RETRY] ${stageName}: Infrastructure failure (${c.detectedPattern}) " + @@ -4695,12 +5077,11 @@ def launchTestJobs(pipeline, testFilter) x86TestConfigs = [ "CPU-Generic-x86-1": ["cpu", "l0_cpu_x86", 1, 1], "DGX_H100-4_GPUs-CPP-1": ["dgx-h100-x4", "l0_dgx_h100", 1, 1, 4], - "A10-PyTorch-1": ["a10", "l0_a10", 1, 2], - "A10-PyTorch-2": ["a10", "l0_a10", 2, 2], - "A10-TensorRT-1": ["a10", "l0_a10", 1, 1], + "A10-PyTorch-1": ["a10", "l0_a10", 1, 3], + "A10-PyTorch-2": ["a10", "l0_a10", 2, 3], + "A10-PyTorch-3": ["a10", "l0_a10", 3, 3], "A30-PyTorch-1": ["a30", "l0_a30", 1, 2], "A30-PyTorch-2": ["a30", "l0_a30", 2, 2], - "A10-CPP-1": ["a10", "l0_a10", 1, 1], "A30-CPP-1": ["a30", "l0_a30", 1, 1], "A30-AutoDeploy-1": ["a30", "l0_a30", 1, 1], "A100X-PyTorch-1": ["a100x", "l0_a100", 1, 1], @@ -4709,56 +5090,29 @@ def launchTestJobs(pipeline, testFilter) "H100_PCIe-PyTorch-Ray-1": ["h100-cr", "l0_h100", 1, 1], "H100_PCIe-AutoDeploy-1": ["h100-cr", "l0_h100", 1, 1], "H100_PCIe-CPP-1": ["h100-cr", "l0_h100", 1, 1], - "H100_PCIe-TensorRT-1": ["h100-cr", "l0_h100", 1, 1], "RTX5090-PyTorch-1": ["rtx-5090", "l0_gb202", 1, 1], - "RTX5080-TensorRT-1": ["rtx-5080", "l0_gb203", 1, 2], - "RTX5080-TensorRT-2": ["rtx-5080", "l0_gb203", 2, 2], + "RTX5080-PyTorch-1": ["rtx-5080", "l0_gb203", 1, 2], + "RTX5080-PyTorch-2": ["rtx-5080", "l0_gb203", 2, 2], // Currently post-merge test stages only run tests with "stage: post_merge" mako // in the test-db. This behavior may change in the future. - "A10-PyTorch-Post-Merge-1": ["a10", "l0_a10", 1, 1], - "A10-TensorRT-Post-Merge-1": ["a10", "l0_a10", 1, 3], - "A10-TensorRT-Post-Merge-2": ["a10", "l0_a10", 2, 3], - "A10-TensorRT-Post-Merge-3": ["a10", "l0_a10", 3, 3], + "A10-PyTorch-Post-Merge-1": ["a10", "l0_a10", 1, 4], + "A10-PyTorch-Post-Merge-2": ["a10", "l0_a10", 2, 4], + "A10-PyTorch-Post-Merge-3": ["a10", "l0_a10", 3, 4], + "A10-PyTorch-Post-Merge-4": ["a10", "l0_a10", 4, 4], "A10-FMHA-Post-Merge-1": ["a10", "l0_a10", 1, 1], - // "A30-TensorRT-Post-Merge-1": ["a30", "l0_a30", 1, 6], - // "A30-TensorRT-Post-Merge-2": ["a30", "l0_a30", 2, 6], - // "A30-TensorRT-Post-Merge-3": ["a30", "l0_a30", 3, 6], - // "A30-TensorRT-Post-Merge-4": ["a30", "l0_a30", 4, 6], - // "A30-TensorRT-Post-Merge-5": ["a30", "l0_a30", 5, 6], - // "A30-TensorRT-Post-Merge-6": ["a30", "l0_a30", 6, 6], "A30-CPP-Post-Merge-1": ["a30", "l0_a30", 1, 2], "A30-CPP-Post-Merge-2": ["a30", "l0_a30", 2, 2], // "A30-Triton-Post-Merge-1": ["a30", "l0_a30", 1, 2], // "A30-Triton-Post-Merge-2": ["a30", "l0_a30", 2, 2], - // "A100X-TensorRT-Post-Merge-1": ["a100x", "l0_a100", 1, 6], - // "A100X-TensorRT-Post-Merge-2": ["a100x", "l0_a100", 2, 6], - // "A100X-TensorRT-Post-Merge-3": ["a100x", "l0_a100", 3, 6], - // "A100X-TensorRT-Post-Merge-4": ["a100x", "l0_a100", 4, 6], - // "A100X-TensorRT-Post-Merge-5": ["a100x", "l0_a100", 5, 6], - // "A100X-TensorRT-Post-Merge-6": ["a100x", "l0_a100", 6, 6], - // "L40S-TensorRT-Post-Merge-1": ["l40s", "l0_l40s", 1, 5], - // "L40S-TensorRT-Post-Merge-2": ["l40s", "l0_l40s", 2, 5], - // "L40S-TensorRT-Post-Merge-3": ["l40s", "l0_l40s", 3, 5], - // "L40S-TensorRT-Post-Merge-4": ["l40s", "l0_l40s", 4, 5], - // "L40S-TensorRT-Post-Merge-5": ["l40s", "l0_l40s", 5, 5], + "A100X-PyTorch-Post-Merge-1": ["a100x", "l0_a100", 1, 1], + "L40S-PyTorch-Post-Merge-1": ["l40s", "l0_l40s", 1, 1], "L40S-FMHA-Post-Merge-1": ["l40s", "l0_l40s", 1, 1], "H100_PCIe-AutoDeploy-Post-Merge-1": ["h100-cr", "l0_h100", 1, 1], - "H100_PCIe-CPP-Post-Merge-1": ["h100-cr", "l0_h100", 1, 1], - // "H100_PCIe-TensorRT-Post-Merge-1": ["h100-cr", "l0_h100", 1, 5], - // "H100_PCIe-TensorRT-Post-Merge-2": ["h100-cr", "l0_h100", 2, 5], - // "H100_PCIe-TensorRT-Post-Merge-3": ["h100-cr", "l0_h100", 3, 5], - // "H100_PCIe-TensorRT-Post-Merge-4": ["h100-cr", "l0_h100", 4, 5], - // "H100_PCIe-TensorRT-Post-Merge-5": ["h100-cr", "l0_h100", 5, 5], "H100_PCIe-FMHA-Post-Merge-1": ["h100-cr", "l0_h100", 1, 1], - // "B200_PCIe-TensorRT-Post-Merge-1": ["b100-ts2", "l0_b200", 1, 2], - // "B200_PCIe-TensorRT-Post-Merge-2": ["b100-ts2", "l0_b200", 2, 2], "H100_PCIe-PyTorch-Perf-1": ["h100-cr", "l0_perf", 1, 1], "DGX_H200-8_GPUs-PyTorch-Post-Merge-1": ["dgx-h200-x8", "l0_dgx_h200", 1, 1, 8], "DGX_H200-4_GPUs-PyTorch-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 1, 4], "DGX_H200-8_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["dgx-h200-x8", "l0_dgx_h200_perf_sanity", 1, 1, 8], - // "DGX_H200-4_GPUs-TensorRT-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 3, 4], - // "DGX_H200-4_GPUs-TensorRT-Post-Merge-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 3, 4], - // "DGX_H200-4_GPUs-TensorRT-Post-Merge-3": ["dgx-h200-x4", "l0_dgx_h200", 3, 3, 4], // Disable RTXPro6000 stages due to nodes will be offline temporarily. // [TODO] Split tests between RTXPro6000 and RTXPro6000D and move reasonable mount of tests to pre-merge. // "RTXPro6000-PyTorch-Post-Merge-1": ["rtx-pro-6000", "l0_rtx_pro_6000", 1, 1], @@ -4843,28 +5197,38 @@ def launchTestJobs(pipeline, testFilter) "DGX_B300-4_GPUs-PyTorch-Post-Merge-2": ["auto:dgx-b300-flex", "l0_dgx_b300", 2, 2, 4, 1, true], // VisualGen PerfSanity post-merge test "DGX_B200-8_GPUs-PyTorch-VisualGen-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_visual_gen_perf_sanity", 1, 1, 8, 1, true], + // Single-GPU Gemma4 PerfSanity regression gate and baseline + "DGX_B200-PyTorch-PerfSanity-1": ["auto:dgx-b200-flex", "l0_b200_perf_sanity", 1, 1, 1, 1, true], // PerfSanity post-merge tests - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 1, 6, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 2, 6, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 3, 6, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 4, 6, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 5, 6, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 6, 6, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 1, 4, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 2, 4, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 3, 4, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 4, 4, 8, 1, true], ] + // B200 PerfSanity pre-merge disaggregated (functional-only: perf regressions do not fail CI) + // 2 Nodes + x86SlurmTestConfigs += buildStageConfigs( + "DGX_B200-16_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE1-GPU8", + "auto:dgx-b200-flex", + "l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8", + 1, + 16, + 2 + ) // B200 PerfSanity post-merge disaggregated // 2 Nodes x86SlurmTestConfigs += buildStageConfigs( "DGX_B200-16_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE1-GPU8-Post-Merge", "auto:dgx-b200-flex", "l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8", - 6, + 2, 16, 2 ) x86SlurmTestConfigs = cbtsResizeSplits(x86SlurmTestConfigs) fullSet += x86SlurmTestConfigs.keySet() - parallelSlurmJobs = x86SlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE.replace("aarch64", "x86_64"), "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> + parallelSlurmJobs = x86SlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(X86_64_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> // attemptTag comes from runKubernetesPodWithInfraRetry for the outer // dispatcher pod (when retry is enabled — see opts below) and is // threaded into runLLMTestlistOnSlurm so a future re-enable of outer @@ -4878,8 +5242,8 @@ def launchTestJobs(pipeline, testFilter) if (key.contains("llvm")) { config = LLVM_CONFIG } - runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag) - }, [singleAttempt: true]]]} + runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag, false, retryContext?.infraRetryMax) + }, [singleAttempt: true, slurmDispatcher: true]]]} // SLURM dispatcher pods run their own inner retry loop // (runLLMTestlistOnSlurm with SLURM_INFRA_RETRY_MAX). Disabling the outer // K8s pod retry (singleAttempt:true) here caps total attempts at @@ -4893,7 +5257,7 @@ def launchTestJobs(pipeline, testFilter) // SBSA machines from the Blossom machine pool SBSATestConfigs = [ "CPU-Generic-arm-1": ["cpu", "l0_cpu_arm", 1, 1], - "GH200-TensorRT-Post-Merge-1": ["gh200", "l0_gh200", 1, 1], + "GH200-PyTorch-Post-Merge-1": ["gh200", "l0_gh200", 1, 1], // DGX Spark is also named as GB10 Grace Blackwell Superchip. "GB10-PyTorch-1": ["gb10x", "l0_gb10", 1, 1], ] @@ -4919,15 +5283,10 @@ def launchTestJobs(pipeline, testFilter) "GB200-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 1, 2, 4], "GB200-4_GPUs-PyTorch-PerfSanity-2": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 2, 2, 4], // PerfSanity post-merge tests - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 1, 9, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 2, 9, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 3, 9, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 4, 9, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 5, 9, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 6, 9, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-7": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 7, 9, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-8": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 8, 9, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-9": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 9, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 1, 4, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 2, 4, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 3, 4, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 4, 4, 4], "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 3, 4], "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 3, 4], "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 3, 3, 4], @@ -4954,28 +5313,30 @@ def launchTestJobs(pipeline, testFilter) 8, 2 ) - // PerfSanity post-merge disaggregated + // PerfSanity pre-merge disaggregated (functional-only: perf regressions do not fail CI) // 2 Nodes multiNodesSBSAConfigs += buildStageConfigs( - "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU2-Post-Merge", + "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4", "auto:gb200-flex", - "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2", - 4, + "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4", + 1, 8, 2 ) + // PerfSanity post-merge disaggregated + // 2 Nodes multiNodesSBSAConfigs += buildStageConfigs( - "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4-Post-Merge", + "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU2-Post-Merge", "auto:gb200-flex", - "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4", - 7, + "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2", + 1, 8, 2 ) multiNodesSBSAConfigs += buildStageConfigs( - "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE1-GPU4-Post-Merge", + "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4-Post-Merge", "auto:gb200-flex", - "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4", + "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4", 5, 8, 2 @@ -4993,7 +5354,7 @@ def launchTestJobs(pipeline, testFilter) "GB200-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", - 8, + 3, 12, 3 ) @@ -5016,14 +5377,6 @@ def launchTestJobs(pipeline, testFilter) 5 ) // 6 Nodes - multiNodesSBSAConfigs += buildStageConfigs( - "GB200-24_GPUs-6_Nodes-PyTorch-Disagg-PerfSanity-CTX2-NODE1-GPU4-GEN1-NODE4-GPU16-Post-Merge", - "auto:gb200-flex", - "l0_gb200_multi_nodes_perf_sanity_ctx2_node1_gpu4_gen1_node4_gpu16", - 2, - 24, - 6 - ) multiNodesSBSAConfigs += buildStageConfigs( "GB200-24_GPUs-6_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE2-GPU8-GEN1-NODE4-GPU16-Post-Merge", "auto:gb200-flex", @@ -5037,45 +5390,36 @@ def launchTestJobs(pipeline, testFilter) "GB200-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE8-GPU32-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32", - 12, + 8, 36, 9 ) - // 10 Nodes - multiNodesSBSAConfigs += buildStageConfigs( - "GB200-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE2-GPU8-GEN1-NODE8-GPU32-Post-Merge", - "auto:gb200-flex", - "l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32", - 1, - 40, - 10 - ) // GB300 PerfSanity post-merge aggregated // 2 Nodes multiNodesSBSAConfigs += buildStageConfigs( "GB300-8_GPUs-2_Nodes-PyTorch-PerfSanity-Node2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_node2_gpu8", - 3, + 2, 8, 2 ) // GB300 PerfSanity post-merge disaggregated - // 2 Nodes + // 3 Nodes (pre-merge, functional-only) multiNodesSBSAConfigs += buildStageConfigs( - "GB300-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE1-GPU4-Post-Merge", + "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8", "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4", - 3, - 8, - 2 + "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", + 1, + 12, + 3 ) // 3 Nodes multiNodesSBSAConfigs += buildStageConfigs( "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", - 5, + 2, 12, 3 ) @@ -5093,26 +5437,26 @@ def launchTestJobs(pipeline, testFilter) "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE8-GPU32-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32", - 2, + 1, 36, 9 ) // GB300 GLM-5 disaggregated (ctx DEP2) - // 2 Nodes + // 3 Nodes (pre-merge, functional-only) multiNodesSBSAConfigs += buildStageConfigs( - "GB300-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE1-GPU4-Post-Merge", + "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8", "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4", + "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8", 1, - 8, - 2 + 12, + 3 ) // 3 Nodes multiNodesSBSAConfigs += buildStageConfigs( "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8", - 5, + 2, 12, 3 ) @@ -5121,10 +5465,46 @@ def launchTestJobs(pipeline, testFilter) "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE8-GPU32-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32", + 1, + 36, + 9 + ) + // 9 Nodes: ctx1 (1 node, 4 GPUs) + gen4 (2 nodes, 8 GPUs each) = 36 GPUs + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN4-NODE2-GPU8-Post-Merge", + "auto:gb300-flex", + "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8", 2, 36, 9 ) + // 10 Nodes: ctx6 (1 node, 4 GPUs each) + gen1 (4 nodes, 16 GPUs) = 40 GPUs + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX6-NODE1-GPU4-GEN1-NODE4-GPU16-Post-Merge", + "auto:gb300-flex", + "l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16", + 2, + 40, + 10 + ) + // 11 Nodes: ctx3 (1 node, 4 GPUs each) + gen1 (8 nodes, 32 GPUs) = 44 GPUs + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-44_GPUs-11_Nodes-PyTorch-Disagg-PerfSanity-CTX3-NODE1-GPU4-GEN1-NODE8-GPU32-Post-Merge", + "auto:gb300-flex", + "l0_gb300_multi_nodes_perf_sanity_ctx3_node1_gpu4_gen1_node8_gpu32", + 2, + 44, + 11 + ) + // 14 Nodes: ctx12 (1 node, 4 GPUs each) + gen1 (2 nodes, 8 GPUs) = 56 GPUs + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-56_GPUs-14_Nodes-PyTorch-Disagg-PerfSanity-CTX12-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge", + "auto:gb300-flex", + "l0_gb300_multi_nodes_perf_sanity_ctx12_node1_gpu4_gen1_node2_gpu8", + 2, + 56, + 14 + ) multiNodesSBSAConfigs = cbtsResizeSplits(multiNodesSBSAConfigs) fullSet += multiNodesSBSAConfigs.keySet() @@ -5137,7 +5517,7 @@ def launchTestJobs(pipeline, testFilter) // singleAttempt:true disables the outer K8s pod retry; see the x86 // SLURM closure above for the full rationale (cap nested retry budget // so consistently-timing-out tests don't burn ~36h on retry cascades). - parallelSlurmJobs = SBSASlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE.replace("aarch64", "x86_64"), "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> + parallelSlurmJobs = SBSASlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(X86_64_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> // attemptTag is threaded into runLLMTestlistOnSlurm as the outer // dispatcher pod's tag so the inner SLURM retry's postTag can't // collide with a previous dispatcher pod's upload. See the x86 @@ -5149,13 +5529,13 @@ def launchTestJobs(pipeline, testFilter) if (key.contains("llvm")) { config = LLVM_CONFIG } - runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false) - }, [singleAttempt: true]]]} + runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false, retryContext?.infraRetryMax) + }, [singleAttempt: true, slurmDispatcher: true]]]} parallelJobs += parallelSlurmJobs // Add SBSA multi node Slurm jobs // singleAttempt:true disables the outer K8s pod retry; see above. - parallelMultiNodesSBSAJobs = multiNodesSBSAConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE.replace("aarch64", "x86_64"), "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> + parallelMultiNodesSBSAJobs = multiNodesSBSAConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(X86_64_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> def config = LINUX_AARCH64_CONFIG if (key.contains("single-device")) { config = SINGLE_DEVICE_CONFIG @@ -5163,8 +5543,8 @@ def launchTestJobs(pipeline, testFilter) if (key.contains("llvm")) { config = LLVM_CONFIG } - runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 2, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false) - }, [singleAttempt: true]]]} + runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 2, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false, retryContext?.infraRetryMax) + }, [singleAttempt: true, slurmDispatcher: true]]]} parallelJobs += parallelMultiNodesSBSAJobs } @@ -5335,7 +5715,7 @@ def launchTestJobs(pipeline, testFilter) trtllm_utils.llmExecStepWithRetry(pipeline, script: 'rm -rf $(python3 -c "import site; print(site.getsitepackages()[0])")/nvidia_cutlass_dsl*') } trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y python3-pip git rsync curl wget") - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 config set global.break-system-packages true") trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install requests") trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 uninstall -y tensorrt") @@ -5477,7 +5857,6 @@ def launchTestJobs(pipeline, testFilter) def backendMode = testFilter[(TEST_BACKEND)].collect { it.toLowerCase() } def changeMap = [ "pytorch": "-PyTorch-", - "tensorrt": "-TensorRT-", "cpp": "-CPP-", "triton": "-Triton-", "fmha": "-FMHA-", @@ -5504,9 +5883,9 @@ def launchTestJobs(pipeline, testFilter) } else { echo "ONLY_ONE_GROUP_CHANGED mode is true. The group is: ${testFilter[(ONLY_ONE_GROUP_CHANGED)]}." def excludedBackends = new HashMap() - excludedBackends["PyTorch"] = ["-CPP-", "-TensorRT-", "-FMHA-"] // Only pytorch file change also need to run triton tests - excludedBackends["Triton"] = ["-PyTorch-", "-CPP-", "-TensorRT-", "-FMHA-"] - excludedBackends["FMHA"] = ["-PyTorch-", "-CPP-", "-TensorRT-", "-Triton-"] + excludedBackends["PyTorch"] = ["-CPP-", "-FMHA-"] // Only pytorch file change also need to run triton tests + excludedBackends["Triton"] = ["-PyTorch-", "-CPP-", "-FMHA-"] + excludedBackends["FMHA"] = ["-PyTorch-", "-CPP-", "-Triton-"] def group = testFilter[(ONLY_ONE_GROUP_CHANGED)] if (excludedBackends.containsKey(group)) { parallelJobsFiltered = parallelJobsFiltered.findAll { key, value -> @@ -5590,7 +5969,15 @@ def launchTestJobs(pipeline, testFilter) // for SLURM dispatcher pods to disable nested pod retry). def opts = (values.size() >= 3 && values[2] instanceof Map) ? values[2] : [:] runKubernetesPodWithInfraRetry(opts, pipeline, values[0], "trt-llm", key, { attemptTag, isFinalAttempt, retryContext = null -> - values[1](attemptTag, isFinalAttempt, retryContext) + // Carry a per-stage infra-retry override (opts.infraRetryMax) to the + // inner runner via retryContext -- the SLURM dispatcher's runner reads + // it and passes it to runLLMTestlistOnSlurm. Preserve the null default + // when no override is set so non-SLURM runners are unaffected. + def innerRetryContext = retryContext + if (opts.infraRetryMax != null) { + innerRetryContext = (retryContext ?: [:]) + [infraRetryMax: opts.infraRetryMax] + } + values[1](attemptTag, isFinalAttempt, innerRetryContext) }) } else { values() @@ -5747,64 +6134,75 @@ pipeline { stage("Test") { steps { script { - if (env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/) { - parallelJobs = launchTestJobsForImagesSanityCheck(this, globalVars) - } else { - parallelJobs = launchTestJobs(this, testFilter) - } + try { + if (env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/) { + parallelJobs = launchTestJobsForImagesSanityCheck(this, globalVars) + } else { + parallelJobs = launchTestJobs(this, testFilter) + } - singleGpuJobs = parallelJobs - dgxJobs = [:] + singleGpuJobs = parallelJobs + dgxJobs = [:] - def testPhase2StageName = env.testPhase2StageName - if (testPhase2StageName) { - def multiGpuPattern = /\d+_GPUs/ - singleGpuJobs = parallelJobs.findAll{!(it.key =~ multiGpuPattern)} - dgxJobs = parallelJobs.findAll{it.key =~ multiGpuPattern} - } + def testPhase2StageName = env.testPhase2StageName + if (testPhase2StageName) { + def multiGpuPattern = /\d+_GPUs/ + singleGpuJobs = parallelJobs.findAll{!(it.key =~ multiGpuPattern)} + dgxJobs = parallelJobs.findAll{it.key =~ multiGpuPattern} + } - if (env.JOB_NAME ==~ /.*Single-GPU.*/) { - echo "Only run single-GPU tests." - if (dgxJobs.size() > 0) { - if (globalVars[ACTION_INFO]['parents'].size() > 0) { - // We add a special marker to the parent job's description. - // This will be used to decide whether to run multi-GPU test stage. - def parentJob = globalVars[ACTION_INFO]['parents'][-2] - def archStr = (env.targetArch == X86_64_TRIPLE) ? "x86_64" : (env.targetArch == AARCH64_TRIPLE ? "SBSA" : "Unknown") - trtllm_utils.appendBuildDescription(this, parentJob['name'], parentJob['build_number'], "====Require ${archStr} Multi-GPU Testing====
    ") + if (env.JOB_NAME ==~ /.*Single-GPU.*/) { + echo "Only run single-GPU tests." + if (dgxJobs.size() > 0) { + if (globalVars[ACTION_INFO]['parents'].size() > 0) { + // We add a special marker to the parent job's description. + // This will be used to decide whether to run multi-GPU test stage. + def parentJob = globalVars[ACTION_INFO]['parents'][-2] + def archStr = (env.targetArch == X86_64_TRIPLE) ? "x86_64" : (env.targetArch == AARCH64_TRIPLE ? "SBSA" : "Unknown") + trtllm_utils.appendBuildDescription(this, parentJob['name'], parentJob['build_number'], "====Require ${archStr} Multi-GPU Testing====
    ") + } else { + echo "No parent job found to add the special marker for executing multi-GPU test stage." + } } else { - echo "No parent job found to add the special marker for executing multi-GPU test stage." + echo "Skip multi-GPU testing. No test to run." } - } else { - echo "Skip multi-GPU testing. No test to run." - } - if (singleGpuJobs.size() > 0) { - singleGpuJobs.failFast = params.enableFailFast - parallel singleGpuJobs - } else { - echo "Skip single-GPU testing. No test to run." - } - } else if (env.JOB_NAME ==~ /.*Multi-GPU.*/) { - echo "Only run multi-GPU tests." - if (dgxJobs.size() > 0) { - dgxJobs.failFast = params.enableFailFast - parallel dgxJobs - } else { - error "Skip multi-GPU testing. No test to run." - } - } else { - if (singleGpuJobs.size() > 0) { - singleGpuJobs.failFast = params.enableFailFast - parallel singleGpuJobs - } else { - echo "Skip single-GPU testing. No test to run." - } - - if (dgxJobs.size() > 0) { - stage(testPhase2StageName) { + if (singleGpuJobs.size() > 0) { + singleGpuJobs.failFast = params.enableFailFast + parallel singleGpuJobs + } else { + echo "Skip single-GPU testing. No test to run." + } + } else if (env.JOB_NAME ==~ /.*Multi-GPU.*/) { + echo "Only run multi-GPU tests." + if (dgxJobs.size() > 0) { dgxJobs.failFast = params.enableFailFast parallel dgxJobs + } else { + error "Skip multi-GPU testing. No test to run." } + } else { + if (singleGpuJobs.size() > 0) { + singleGpuJobs.failFast = params.enableFailFast + parallel singleGpuJobs + } else { + echo "Skip single-GPU testing. No test to run." + } + + if (dgxJobs.size() > 0) { + stage(testPhase2StageName) { + dgxJobs.failFast = params.enableFailFast + parallel dgxJobs + } + } + } + } finally { + // Backstop: reclaim any SLURM job / Jenkins node left orphaned + // by a dispatcher-pod death whose in-catch finalize did not run + // or failed. Best-effort; never fails the build. + try { + sweepOrphanedSlurmResources(this) + } catch (Exception sweepErr) { + echo "[SLURM-FINALIZER] post-build sweep error: ${sweepErr}" } } } diff --git a/jenkins/TensorRT_LLM_PLC.groovy b/jenkins/TensorRT_LLM_PLC.groovy index dc3c1735e2d1..651e431ea96f 100644 --- a/jenkins/TensorRT_LLM_PLC.groovy +++ b/jenkins/TensorRT_LLM_PLC.groovy @@ -140,7 +140,7 @@ def checkoutSource () def LLM_REPO = getLLMRepo() sh "git config --global --add safe.directory ${env.WORKSPACE}" def ref = params.ref - trtllm_utils.checkoutSource(LLM_REPO, ref, env.WORKSPACE, false, true) + trtllm_utils.checkoutSource(LLM_REPO, ref, env.WORKSPACE, true, true) } def getPulseToken(serviceId, scopes) { diff --git a/jenkins/UpdateTestDurations.groovy b/jenkins/UpdateTestDurations.groovy new file mode 100644 index 000000000000..2d2749c045aa --- /dev/null +++ b/jenkins/UpdateTestDurations.groovy @@ -0,0 +1,236 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ + +LLM_ROOT = "llm" + +UBUNTU_24_04_IMAGE = "urm.nvidia.com/docker/ubuntu:24.04" +DURATION_FILE_PATH = "tests/integration/defs/.test_durations" +// Target repository the updated duration file is committed straight back into. +TARGET_REPO = "NVIDIA/TensorRT-LLM" +LLM_REPO = "https://github.com/${TARGET_REPO}.git" + +def createKubernetesPodConfig(image, arch = "amd64") +{ + def archSuffix = arch == "arm64" ? "arm" : "amd" + def jnlpImage = "artifactory.pdx.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" + + def podConfig = [ + cloud: "kubernetes-cpu", + namespace: "sw-tensorrt", + yaml: """ + apiVersion: v1 + kind: Pod + spec: + nodeSelector: + nvidia.com/node_type: builder + kubernetes.io/os: linux + containers: + - name: trt-llm + image: ${image} + command: ['cat'] + volumeMounts: + - name: sw-tensorrt-pvc + mountPath: "/mnt/sw-tensorrt-pvc" + readOnly: false + tty: true + resources: + requests: + cpu: 2 + memory: 5Gi + ephemeral-storage: 25Gi + limits: + cpu: 2 + memory: 5Gi + ephemeral-storage: 25Gi + imagePullPolicy: Always + - name: jnlp + image: ${jnlpImage} + args: ['\$(JENKINS_SECRET)', '\$(JENKINS_NAME)'] + resources: + requests: + cpu: '2' + memory: 5Gi + ephemeral-storage: 25Gi + limits: + cpu: '2' + memory: 5Gi + ephemeral-storage: 25Gi + qosClass: Guaranteed + volumes: + - name: sw-tensorrt-pvc + persistentVolumeClaim: + claimName: sw-tensorrt-pvc + """.stripIndent(), + ] + + return podConfig +} + +pipeline { + agent { + kubernetes createKubernetesPodConfig(UBUNTU_24_04_IMAGE) + } + options { + timestamps() + timeout(time: 1, unit: 'HOURS') + } + triggers { + cron('H 2 * * 1') + } + parameters { + string( + name: 'DAYS', + defaultValue: '7', + description: 'Number of days to look back in OpenSearch for test durations (e.g. 3, 7, 14). ') + string( + name: 'SOURCE_REPO', + defaultValue: 'NVIDIA/TensorRT-LLM', + description: 'GitHub repo to checkout scripts from (e.g. EmmaQiaoCh/TensorRT-LLM for testing).') + string( + name: 'TARGET_BRANCH', + defaultValue: 'main', + description: 'Branch of the target repo to commit the updated duration file to.') + booleanParam( + name: 'DRY_RUN', + defaultValue: false, + description: 'When true, generate the duration file but skip the commit/push.') + } + environment { + OPEN_SEARCH_DB_BASE_URL = credentials('open_search_db_base_url') + } + stages { + stage('Setup') { + steps { + container('trt-llm') { + sh """ + apt-get update -qq && \ + apt-get install -y -qq git python3-pip curl && \ + pip3 install --quiet --break-system-packages requests pyyaml + """ + } + } + } // stage Setup + + stage('Checkout') { + steps { + container('trt-llm') { + script { + def sourceRepo = "https://github.com/${params.SOURCE_REPO}.git" + trtllm_utils.checkoutSource(sourceRepo, params.TARGET_BRANCH, LLM_ROOT, false, false) + } + } + } + } // stage Checkout + + stage('Generate Duration File') { + steps { + container('trt-llm') { + sh """ + cd ${LLM_ROOT} + python3 jenkins/scripts/generate_duration.py \ + --days ${params.DAYS} \ + --duration-file new_test_durations.json + echo "Generated file size: \$(wc -l < new_test_durations.json) lines" + echo "Sample output (first 5 lines):" + head -5 new_test_durations.json + + """ + + // Always archive the freshly generated file so the user can download + // it and upload manually if the job later refuses to auto-commit. + archiveArtifacts( + artifacts: "${LLM_ROOT}/new_test_durations.json", + fingerprint: true) + + // Sanity gate: if the new file diverges too much from the one in use, + // fail the job instead of committing a possibly-broken duration file. + script { + def countItems = { path -> + sh(script: "python3 -c \"import json; print(len(json.load(open('${path}'))))\"", + returnStdout: true).trim() as Integer + } + def oldCount = countItems("${LLM_ROOT}/${DURATION_FILE_PATH}") + def newCount = countItems("${LLM_ROOT}/new_test_durations.json") + echo "Duration-file item counts -> old: ${oldCount}, new: ${newCount}" + if (oldCount == 0) { + error("Existing duration file is empty or missing; aborting.") + } + def diffPct = Math.abs(newCount - oldCount) * 100.0 / oldCount + echo "Item-count difference: ${String.format('%.1f', diffPct)}%" + if (diffPct >= 50.0) { + error("Item-count difference ${String.format('%.1f', diffPct)}% " + + ">= 50%; refusing to auto-commit. Download the archived " + + "new_test_durations.json, review, and upload manually.") + } + } + } + } + } // stage Generate Duration File + + stage('Commit and Push') { + when { + expression { !params.DRY_RUN } + } + steps { + container('trt-llm') { + script { + // Overwrite the checked-in duration file with the freshly generated one. + sh """ + cd ${LLM_ROOT} + git config --global --add safe.directory \$(pwd) + git config user.email "90828364+tensorrt-cicd@users.noreply.github.com" + git config user.name "TensorRT LLM" + cp new_test_durations.json ${DURATION_FILE_PATH} + """ + + def changeCount = sh( + script: "cd ${LLM_ROOT} && git status --porcelain ${DURATION_FILE_PATH} | wc -l", + returnStdout: true).trim() + echo "Changed duration-file count: ${changeCount}" + if (changeCount == "0") { + echo "No update to the duration file; nothing to commit." + return + } + + sh """ + cd ${LLM_ROOT} + git add ${DURATION_FILE_PATH} + git commit -s -m "[None][infra] Auto-update test durations from OpenSearch (last ${params.DAYS} days)" + """ + + withCredentials([usernamePassword( + credentialsId: 'github-cred-trtllm-ci', + usernameVariable: 'NOT_IN_USE', + passwordVariable: 'GITHUB_API_TOKEN')]) { + def authedUrl = LLM_REPO.replaceFirst( + 'https://', "https://svc_tensorrt:${GITHUB_API_TOKEN}@") + // Rebase onto the latest target branch before pushing to avoid + // clobbering commits landed since checkout. + sh """ + cd ${LLM_ROOT} + git remote set-url origin ${authedUrl} + git fetch origin ${params.TARGET_BRANCH} + git rebase origin/${params.TARGET_BRANCH} + git push origin HEAD:${params.TARGET_BRANCH} + """ + } + } + } + } + } // stage Commit and Push + } // stages +} // pipeline diff --git a/jenkins/controlCCache.groovy b/jenkins/controlCCache.groovy index 74ebebdeb032..6a61b25955fc 100644 --- a/jenkins/controlCCache.groovy +++ b/jenkins/controlCCache.groovy @@ -6,7 +6,7 @@ DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.10-py3 def createKubernetesPodConfig(image, arch = "amd64") { def archSuffix = arch == "arm64" ? "arm" : "amd" - def jnlpImage = "urm.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" + def jnlpImage = "artifactory.pdx.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" def podConfig = [ cloud: "kubernetes-cpu", diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index 504fd8f234a8..cf7ca75d2b50 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm -LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202607151440-16194 -LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202607151440-16194 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-trt10.16.1.11-skip-tritondevel-202607151440-16194 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-trt10.16.1.11-skip-tritondevel-202607151440-16194 -LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-trt10.16.1.11-skip-tritondevel-202607151440-16194 +LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-skip-tritondevel-202607271403-16694 +LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-skip-tritondevel-202607271403-16694 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202607271403-16694 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202607271403-16694 +LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202607271403-16694 diff --git a/jenkins/license_cpp.json b/jenkins/license_cpp.json index bba247b97f50..408a1217408c 100644 --- a/jenkins/license_cpp.json +++ b/jenkins/license_cpp.json @@ -44,6 +44,12 @@ "tensorrt_llm/kernels/causalConv1d/causalConv1d.h": "dual license", "tensorrt_llm/thop/causalConv1dOp.cpp": "dual license", "tensorrt_llm/common/vec_dtypes.cuh": "dual license", + "tensorrt_llm/common/sha256/attributes.h": "external (Bitcoin Core MIT)", + "tensorrt_llm/common/sha256/sha256.cpp": "external (Bitcoin Core MIT)", + "tensorrt_llm/common/sha256/sha256.h": "external (Bitcoin Core MIT)", + "tensorrt_llm/common/sha256/sha256_arm_shani.cpp": "external (Bitcoin Core MIT)", + "tensorrt_llm/common/sha256/sha256_endian.h": "external (Bitcoin Core MIT)", + "tensorrt_llm/common/sha256/sha256_x86_shani.cpp": "external (Bitcoin Core MIT)", "_": "don't remove, for trailing comma" } } diff --git a/jenkins/runPerfSanityTriage.groovy b/jenkins/runPerfSanityTriage.groovy index 86591da57cc7..cf725a544213 100644 --- a/jenkins/runPerfSanityTriage.groovy +++ b/jenkins/runPerfSanityTriage.groovy @@ -11,7 +11,7 @@ LLM_ROOT = "llm" def createKubernetesPodConfig(image, arch = "amd64") { def archSuffix = arch == "arm64" ? "arm" : "amd" - def jnlpImage = "urm.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" + def jnlpImage = "artifactory.pdx.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" def podConfig = [ cloud: "kubernetes-cpu", @@ -89,7 +89,7 @@ pipeline { container("trt-llm") { script { sh "pwd && ls -alh" - trtllm_utils.checkoutSource(LLM_REPO, params.BRANCH, LLM_ROOT, false, false) + trtllm_utils.checkoutSource(LLM_REPO, params.BRANCH, LLM_ROOT, true, false) def commandsBase64 = params.COMMANDS.bytes.encodeBase64().toString() sh """ cd ${LLM_ROOT}/jenkins/scripts/perf && python3 perf_sanity_triage.py \ diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index 3fc34edda9af..9ce689073682 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -437,7 +437,6 @@ def _classify_map_var(var_name: str) -> Optional[str]: # jenkins/L0_Test.groovy (line ~2079). IMPORTANT: keep this list in sync. _BACKEND_PATTERNS = [ ("-PyTorch-", "pytorch"), - ("-TensorRT-", "tensorrt"), ("-CPP-", "cpp"), ("-Triton-", "triton"), ("-FMHA-", "fmha"), diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py new file mode 100644 index 000000000000..74bb0159fffb --- /dev/null +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Resolve and fetch the latest merged CBTS touch DB from Artifactory. + +The tarball is uploaded per post-merge run to +`//cbts-coverage/cbts_pystart_report.tar.gz` (sqlite at +the tar root plus `cbts_report/`). + +`latest_tarball_url()` reads the newest build number from the Jenkins REST API, +then walks builds down, probing Artifactory with a 1-byte ranged GET until it +finds one whose tarball exists. + +Two entry points for the Groovy wiring: + * `--print-url` — resolve and print the tarball URL only (no download). + * `--dest DIR` — download + extract, printing the local sqlite path. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +import tarfile +import urllib.error +import urllib.request +from pathlib import Path +from typing import Optional + +# Merged-artifact base for the main-branch L0_PostMerge job. +ARTIFACT_BASE = "sw-tensorrt-generic/llm-artifacts/LLM/main/L0_PostMerge" +TARBALL_NAME = "cbts_pystart_report.tar.gz" +SQLITE_NAME = "cbts_touchmap.sqlite" + +_URM = "https://urm.nvidia.com/artifactory" +_JENKINS_BASE = "https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/main/job/L0_PostMerge" +# Max builds to walk back when recent builds have no tarball. +_MAX_PROBE = 10 +# Per-request timeout in seconds. +_TIMEOUT = 15 + + +def _get(url: str) -> tuple[Optional[int], Optional[bytes]]: + try: + with urllib.request.urlopen(url, timeout=_TIMEOUT) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as e: + return e.code, None + except OSError as e: + print(f"[artifact] error fetching {url}: {e}", file=sys.stderr) + return None, None + + +def _exists(url: str) -> bool: + """True if the artifact exists — a 1-byte ranged GET; 200/206 means present.""" + req = urllib.request.Request(url, headers={"Range": "bytes=0-0"}) + try: + with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp: + return resp.status in (200, 206) + except urllib.error.HTTPError: + return False + except OSError as e: + print(f"[artifact] error probing {url}: {e}", file=sys.stderr) + return False + + +def latest_build_number(jenkins_base: str = _JENKINS_BASE) -> Optional[int]: + """Newest build number via the Jenkins REST API (lastBuild, then lastCompletedBuild).""" + for kind in ("lastBuild", "lastCompletedBuild"): + status, data = _get(f"{jenkins_base}/{kind}/api/json") + if status == 200 and data: + try: + return int(json.loads(data)["number"]) + except (json.JSONDecodeError, KeyError, ValueError): + pass + return None + + +def tarball_url(build: int, artifact_base: str = ARTIFACT_BASE) -> str: + return f"{_URM}/{artifact_base}/{build}/cbts-coverage/{TARBALL_NAME}" + + +def latest_tarball_url( + artifact_base: str = ARTIFACT_BASE, + jenkins_base: str = _JENKINS_BASE, + max_probe: int = _MAX_PROBE, +) -> Optional[str]: + """URL of the newest build whose coverage tarball actually exists, or None.""" + build = latest_build_number(jenkins_base) + if build is None: + print("[artifact] could not resolve latest build number", file=sys.stderr) + return None + floor = max(0, build - max_probe) + while build > floor: + url = tarball_url(build, artifact_base) + if _exists(url): + return url + print(f"[artifact] build {build} has no tarball, trying {build - 1}", file=sys.stderr) + build -= 1 + print(f"[artifact] no tarball in the last {max_probe} builds", file=sys.stderr) + return None + + +def extract_touch_db(tarball: Path | str, dest_dir: Path | str) -> Optional[Path]: + """Extract `cbts_touchmap.sqlite` from a downloaded tarball; return its path.""" + dest_dir = Path(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + with tarfile.open(tarball) as tf: + member = next((m for m in tf.getmembers() if m.name.endswith(SQLITE_NAME)), None) + if member is None: + return None + member.name = SQLITE_NAME + tf.extract(member, dest_dir) + return dest_dir / SQLITE_NAME + + +def fetch_latest_touch_db(dest_dir: Path | str, url: Optional[str] = None) -> Optional[Path]: + """Download + extract the latest post-merge touch DB; return local sqlite Path or None. + + `url` pins an explicit tarball (skips latest-build resolution); any failure returns None. + """ + dest_dir = Path(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + url = url or latest_tarball_url() + if url is None: + return None + tarball = dest_dir / TARBALL_NAME + try: + with urllib.request.urlopen(url, timeout=_TIMEOUT) as resp, open(tarball, "wb") as f: + shutil.copyfileobj(resp, f) + return extract_touch_db(tarball, dest_dir) + except OSError as e: + print(f"[artifact] download/extract failed {url}: {e}", file=sys.stderr) + return None + + +def main(argv: Optional[list[str]] = None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--dest", help="download + extract into DIR; prints the local sqlite path") + ap.add_argument( + "--print-url", action="store_true", help="resolve and print the tarball URL only" + ) + ap.add_argument( + "--build", type=int, default=None, help="pin a build number (skip auto-resolve)" + ) + args = ap.parse_args(argv) + + url = tarball_url(args.build) if args.build is not None else None + + if args.print_url: + url = url or latest_tarball_url() + if url is None: + return 1 + print(url) + return 0 + + if args.dest: + path = fetch_latest_touch_db(args.dest, url=url) + if path is None: + return 1 + print(path) + return 0 + + ap.error("one of --print-url or --dest is required") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/jenkins/scripts/cbts/coverage_selection/touch_db.py b/jenkins/scripts/cbts/coverage_selection/touch_db.py new file mode 100644 index 000000000000..d6ff12abc686 --- /dev/null +++ b/jenkins/scripts/cbts/coverage_selection/touch_db.py @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Read-only accessor for the merged CBTS touch DB (`cbts_touchmap.sqlite`). + +Implements the TOUCH_DB_CONTRACT.md queries. Two real-data details: + - the `test` column is `/` (stage-prefixed); + - unit tests are recorded wrapped as + `test_unittests.py::test_unittests_v2[]`, while test-db YAML lists + the bare `` entry. +""" + +from __future__ import annotations + +import re +import sqlite3 +from pathlib import Path +from typing import Optional + +from blocks import normalize_test_id + +# Canonicalize an absolute co_filename to the DB `file` form (`tensorrt_llm/...`). +_CANON_RE = re.compile(r"(tensorrt_llm/.*)$") + +# A DB test value is `/`; unit tests wrap the inner entry. +_UNITTEST_WRAP_RE = re.compile(r"::test_unittests_v2\[(?P.+)\]$") + +# Completeness-heuristic constants consumed by `untrusted_tests()`. +_WORKER_SENTINEL = "tensorrt_llm/_torch/pyexecutor/py_executor.py" +_LAUNCH_MARKERS: tuple[tuple[str, str], ...] = ( + ("tensorrt_llm/llmapi/llm.py", "generate"), + ("tensorrt_llm/executor/executor.py", "GenerationExecutor.generate"), +) +_SERVING_PATH_MARKERS: tuple[str, ...] = ("disaggregated/",) +_MIN_FUNCS = 30 + + +def canon(path: str) -> str: + """Canonicalize a path to the DB's `file` form (`tensorrt_llm/...`).""" + m = _CANON_RE.search(path) + return m.group(1) if m else path + + +def split_stage(test: str) -> tuple[str, str]: + """Split a DB `test` value `/` into `(stage, nodeid)`; `("", test)` if no `/`.""" + stage, sep, nodeid = test.partition("/") + return (stage, nodeid) if sep else ("", test) + + +def unwrap_unittest(nodeid: str) -> Optional[str]: + """Return the inner `unittest/...` entry of a wrapped unittest nodeid, else None. + + `test_unittests.py::test_unittests_v2[unittest/x.py -m "part0"]` + -> `unittest/x.py -m "part0"` + """ + m = _UNITTEST_WRAP_RE.search(nodeid) + return m.group("inner") if m else None + + +def db_key(entry: str) -> Optional[str]: + """Map a test-db YAML `tests:` entry to the DB nodeid form, or None if not 1:1. + + Unit tests wrap as `test_unittests.py::test_unittests_v2[]`; a `-k` + keyword entry expands to many nodeids (no single DB key) -> None. + """ + e = normalize_test_id(entry) + if e.startswith("unittest/"): + return f"test_unittests.py::test_unittests_v2[{e}]" + if " -k " in e: + return None + return e + + +class TouchDB: + """Read-only view over a merged `cbts_touchmap.sqlite`.""" + + def __init__(self, conn: sqlite3.Connection) -> None: + self._conn = conn + + @classmethod + def open(cls, sqlite_path: Path | str) -> "TouchDB": + """Open the DB read-only (`mode=ro`) and verify the `touch` schema.""" + uri = f"file:{Path(sqlite_path).resolve()}?mode=ro" + conn = sqlite3.connect(uri, uri=True) + cols = {row[1] for row in conn.execute("PRAGMA table_info(touch)")} + if not {"test", "file", "qualname"} <= cols: + conn.close() + raise ValueError(f"unexpected touch schema, columns={sorted(cols)}") + return cls(conn) + + def close(self) -> None: + self._conn.close() + + def __enter__(self) -> "TouchDB": + return self + + def __exit__(self, *_exc) -> None: + self.close() + + # -- meta (every key optional; read with a default) -- + + def meta(self, key: str, default: Optional[str] = None) -> Optional[str]: + try: + row = self._conn.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone() + except sqlite3.OperationalError: + return default + return row[0] if row is not None else default + + def schema_version(self) -> Optional[str]: + return self.meta("schema_version") + + def collection_commit(self) -> Optional[str]: + """Commit the DB was collected at, or None if not recorded.""" + return self.meta("commit") or self.meta("collection_commit") + + # -- reverse lookup (the core of selection); always `test != ''` -- + + def tests_touching_file(self, file: str) -> set[str]: + """Stage-prefixed tests that entered any function in `file` (file-level).""" + return { + row[0] + for row in self._conn.execute( + "SELECT DISTINCT test FROM touch WHERE file=? AND test!=''", (file,) + ) + } + + def tests_touching_func(self, file: str, qualname: str) -> set[str]: + """Stage-prefixed tests that entered `qualname` in `file` (function-level).""" + return { + row[0] + for row in self._conn.execute( + "SELECT DISTINCT test FROM touch WHERE file=? AND qualname=? AND test!=''", + (file, qualname), + ) + } + + def file_has_touch_rows(self, file: str) -> bool: + """True iff any instrumented test entered a function in `file`.""" + row = self._conn.execute( + "SELECT 1 FROM touch WHERE file=? AND test!='' LIMIT 1", (file,) + ).fetchone() + return row is not None + + # -- universe / per-stage -- + + def known_tests(self) -> set[str]: + """Every stage-prefixed test with coverage data.""" + return { + row[0] for row in self._conn.execute("SELECT DISTINCT test FROM touch WHERE test!=''") + } + + def per_test_footprint(self) -> dict[str, int]: + """`{test -> functions entered}` over all stage-prefixed tests.""" + return { + row[0]: row[1] + for row in self._conn.execute( + "SELECT test, COUNT(*) FROM touch WHERE test!='' GROUP BY test" + ) + } + + def instrumented_stages(self) -> set[str]: + """Stage names the DB has data for — the stages coverage may narrow.""" + return {stage for stage, _ in map(split_stage, self.known_tests()) if stage} + + def known_by_stage(self) -> dict[str, set[str]]: + """`{stage -> {bare nodeid, ...}}` over all known tests.""" + out: dict[str, set[str]] = {} + for test in self.known_tests(): + stage, nodeid = split_stage(test) + if stage: + out.setdefault(stage, set()).add(nodeid) + return out + + # -- forward lookup (debug / explain-why) -- + + def files_touched_by(self, test: str) -> list[tuple[str, str]]: + """`(file, qualname)` rows for a stage-prefixed `test`.""" + return [ + (row[0], row[1]) + for row in self._conn.execute("SELECT file, qualname FROM touch WHERE test=?", (test,)) + ] + + # -- coverage-completeness heuristic -- + + def untrusted_tests( + self, + worker_file: str, + launch_markers: tuple[tuple[str, str], ...], + serving_path_markers: tuple[str, ...], + min_funcs: int, + ) -> set[str]: + """Stage-prefixed tests whose per-test capture looks incomplete (must always run). + + Flags a test that drove execution/serving but is missing `worker_file` — + matched by a `launch_markers` `(file, qualname_substring)` call or a + `serving_path_markers` nodeid substring — or that entered fewer than + `min_funcs` functions total. + """ + drove_execution: set[str] = set() + for file, qual_substr in launch_markers: + drove_execution |= { + row[0] + for row in self._conn.execute( + "SELECT DISTINCT test FROM touch WHERE file=? AND qualname LIKE ? AND test!=''", + (file, f"%{qual_substr}%"), + ) + } + if serving_path_markers: + drove_execution |= { + test + for test in self.known_tests() + if any(marker in test for marker in serving_path_markers) + } + missing_worker = drove_execution - self.tests_touching_file(worker_file) + tiny = { + row[0] + for row in self._conn.execute( + "SELECT test FROM touch WHERE test!='' GROUP BY test HAVING COUNT(*) < ?", + (min_funcs,), + ) + } + return missing_worker | tiny diff --git a/jenkins/scripts/cbts/tools/coverage_audit.py b/jenkins/scripts/cbts/tools/coverage_audit.py new file mode 100644 index 000000000000..b9e2309b6152 --- /dev/null +++ b/jenkins/scripts/cbts/tools/coverage_audit.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Audit a CBTS touch DB (`cbts_touchmap.sqlite`) — format, scale, and coverage completeness. + +Reports the format (stage prefix, schema_version, collection commit), scale, +per-stage known counts, the per-test footprint distribution, and the tests +whose capture looks incomplete (the same heuristic the selector uses). + +Example:: + + python3 jenkins/scripts/cbts/tools/coverage_audit.py \\ + --db cbts_touchmap.sqlite --list-untrusted +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +THIS = Path(__file__).resolve() +CBTS = THIS.parent.parent +sys.path.insert(0, str(CBTS)) +sys.path.insert(0, str(CBTS / "coverage_selection")) + +from blocks import YAMLIndex, block_matches_stage, parse_stages_from_groovy # noqa: E402 +from touch_db import ( # noqa: E402 + _LAUNCH_MARKERS, + _MIN_FUNCS, + _SERVING_PATH_MARKERS, + _WORKER_SENTINEL, + TouchDB, + db_key, + split_stage, +) + +_DEFAULT_TEST_DB = CBTS.parents[2] / "tests/integration/test_lists/test-db" +_DEFAULT_GROOVY = CBTS.parents[2] / "jenkins/L0_Test.groovy" + + +def _fmt_pct(n: int, d: int) -> str: + return f"{100.0 * n / d:.0f}%" if d else "n/a" + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--db", required=True, help="path to cbts_touchmap.sqlite") + ap.add_argument("--list-untrusted", action="store_true", help="print every untrusted test") + ap.add_argument( + "--min-funcs", type=int, default=_MIN_FUNCS, help=f"near-empty floor (default {_MIN_FUNCS})" + ) + ap.add_argument( + "--test-db", + default=str(_DEFAULT_TEST_DB), + help="test-db dir to diff against the DB (HEAD coverage gap); '' to skip", + ) + ap.add_argument( + "--groovy", default=str(_DEFAULT_GROOVY), help="Groovy file to parse stage defs from" + ) + ap.add_argument( + "--list-not-in-db", action="store_true", help="print every gap case (default: first 15)" + ) + args = ap.parse_args(argv) + + db = TouchDB.open(args.db) + known = db.known_tests() + stages = db.known_by_stage() + footprint = db.per_test_footprint() + + print(f"=== CBTS coverage DB audit: {args.db} ===\n") + + # -- Format -- + print("## Format") + if stages: + print(f" test field: stage-prefixed ({len(stages)} instrumented stage(s) derivable)") + elif known: + print( + " test field: BARE nodeid !! WARNING: no stage prefix -> per-stage narrowing impossible" + ) + sv = db.schema_version() + commit = db.collection_commit() + print(f" schema_version: {sv or 'MISSING (selector cannot hard-fail on format drift)'}") + print( + f" collection commit: {commit or 'MISSING (no staleness gating; zero-touch lever stays off)'}" + ) + + # -- Scale -- + print("\n## Scale") + print( + f" known tests: {len(known)} | meta: tests={db.meta('tests')} files={db.meta('files')} " + f"functions={db.meta('functions')}" + ) + fr, qr = db.meta("file_rate_pct"), db.meta("func_rate_pct") + if fr or qr: + print(f" coverage rate: files {fr}% functions {qr}%") + + # -- Per-stage -- + print("\n## Instrumented stages") + for stage in sorted(stages): + print(f" {stage}: {len(stages[stage])} known") + + # -- Completeness -- + untrusted = db.untrusted_tests( + _WORKER_SENTINEL, _LAUNCH_MARKERS, _SERVING_PATH_MARKERS, args.min_funcs + ) + + def reason(test: str) -> str: + if any(m in test for m in _SERVING_PATH_MARKERS): + return "disagg-path (servers uninstrumented)" + if footprint[test] < args.min_funcs: + return f"near-empty (<{args.min_funcs} funcs)" + return "worker-lost (drove inference, no py_executor)" + + print("\n## Coverage completeness") + if footprint: + print( + f" per-test footprint (functions entered): min={min(footprint.values())} " + f"max={max(footprint.values())} (few funcs => likely lost subprocess capture)" + ) + else: + print(" per-test footprint: none (no test != '' rows — no usable per-test coverage)") + trusted_fp = [footprint[t] for t in known if t not in untrusted] + untrusted_fp = [footprint[t] for t in untrusted] + if trusted_fp and untrusted_fp: + print( + f" footprint gap: untrusted max={max(untrusted_fp)} | trusted min={min(trusted_fp)}" + ) + print( + f" UNTRUSTED (incomplete capture): {len(untrusted)}/{len(known)} ({_fmt_pct(len(untrusted), len(known))})" + ) + by_reason: dict[str, int] = {} + by_stage: dict[str, int] = {} + for t in untrusted: + by_reason[reason(t)] = by_reason.get(reason(t), 0) + 1 + by_stage[split_stage(t)[0]] = by_stage.get(split_stage(t)[0], 0) + 1 + for r, n in sorted(by_reason.items(), key=lambda kv: -kv[1]): + print(f" - {r}: {n}") + print(f" by stage: {dict(sorted(by_stage.items()))}") + print( + f"\n TRUSTED skippable universe: {len(known) - len(untrusted)}/{len(known)} " + f"(only these may ever be skipped)" + ) + + if args.list_untrusted: + print("\n## Untrusted tests") + for t in sorted(untrusted): + print(f" [{footprint[t]:>5} funcs] {t}\n -> {reason(t)}") + + # -- HEAD coverage gap: cases on an instrumented stage with no DB row -- + if args.test_db and Path(args.test_db).is_dir() and Path(args.groovy).is_file(): + yaml_index = YAMLIndex.load(Path(args.test_db)) + all_stages = parse_stages_from_groovy(Path(args.groovy), include_post_merge=True) + bare_known = {split_stage(t)[1] for t in known} + per_stage: dict[str, set[str]] = {} + for name in sorted(set(all_stages) & set(stages)): + stage = all_stages[name] + missing = { + entry + for block in yaml_index.blocks + if block.yaml_stem == stage.yaml_stem and block_matches_stage(block, stage) + for entry in block.tests + if (k := db_key(entry)) is not None and k not in bare_known + } + if missing: + per_stage[name] = missing + gap = sorted(set().union(*per_stage.values())) if per_stage else [] + print("\n## HEAD coverage gap on instrumented stages") + print( + f" {len(gap)} unique case(s) render on an instrumented single-GPU stage but have " + f"NO DB row -> always must-run (new/renamed or never captured)" + ) + for name in sorted(per_stage, key=lambda n: -len(per_stage[n])): + print(f" {name}: {len(per_stage[name])} not-in-DB") + preview = gap if args.list_not_in_db else gap[:15] + if preview: + print(" cases:") + for t in preview: + print(f" - {t}") + if not args.list_not_in_db and len(gap) > 15: + print(f" ... (+{len(gap) - 15}; use --list-not-in-db)") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/jenkins/scripts/generate_duration.py b/jenkins/scripts/generate_duration.py new file mode 100644 index 000000000000..74f9e1687c7b --- /dev/null +++ b/jenkins/scripts/generate_duration.py @@ -0,0 +1,237 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import glob +import json +import os +import re +import time + +OPENSEARCH_INDEX = "df-swdl-trtllm-infra-ci-prod-test_info-*" + +# Test-list YAML entries may carry trailing turtle directives that are NOT part +# of the test identity and never appear in an OpenSearch s_turtle_name: +# - "TIMEOUT (90)" (also the rare no-space form "TIMEOUT(60)") +# - "ISOLATION" +# They can be chained, e.g. "... TIMEOUT (90) ISOLATION". pytest-native flags +# such as -k / -m DO change what runs and MUST be preserved. No node-id +# parameter list contains a space, so anchoring to the end is safe. +_TURTLE_DIRECTIVE_RE = re.compile(r"(?:\s+(?:TIMEOUT\s*\(\d+\)|ISOLATION))+\s*$") + +# Default location of the turtle test-db lists, relative to the repo root. +# This file lives at /jenkins/scripts/, so go up three levels. +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +DEFAULT_TEST_LIST_DIR = os.path.join(_REPO_ROOT, "tests", "integration", "test_lists", "test-db") + + +def normalize_test_spec(name): + """Strip trailing turtle directives (TIMEOUT/ISOLATION), keep -k/-m flags.""" + if not name: + return name + return _TURTLE_DIRECTIVE_RE.sub("", name).strip() + + +def load_test_list_specs(test_list_dir): + """Collect the set of normalized test specs declared in the test-db YAMLs.""" + import yaml + + specs = set() + yml_files = sorted(glob.glob(os.path.join(test_list_dir, "*.yml"))) + for path in yml_files: + with open(path) as f: + try: + data = yaml.safe_load(f) or {} + except yaml.YAMLError as e: + print(f" Warning: failed to parse {path}: {e}") + continue + for value in data.values(): + if not isinstance(value, list): + continue + for block in value: + if not isinstance(block, dict): + continue + for test in block.get("tests", []) or []: + if isinstance(test, str): + specs.add(normalize_test_spec(test)) + return specs, yml_files + + +def query_opensearch_durations(days): + import sys + + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import requests + from open_search_db import DEFAULT_RETRY_COUNT, OPEN_SEARCH_DB_BASE_URL, QUERY_TIMEOUT_SECONDS + + since_ms = int((time.time() - days * 86400) * 1000) + search_url = f"{OPEN_SEARCH_DB_BASE_URL}/opensearch/{OPENSEARCH_INDEX}/_search" + headers = {"Content-Type": "application/json", "Accept-Charset": "UTF-8"} + + test_durations = {} + after_key = None + page = 0 + + while True: + composite_agg = { + "size": 1000, + "sources": [{"test_name": {"terms": {"field": "s_turtle_name"}}}], + } + if after_key is not None: + composite_agg["after"] = after_key + + query = { + "size": 0, + "query": { + "bool": { + "must": [ + {"term": {"s_status": "PASSED"}}, + {"range": {"ts_created": {"gte": since_ms}}}, + ], + "must_not": [{"term": {"s_turtle_name": "Stage Failed"}}], + } + }, + "aggs": { + "by_test": { + "composite": composite_agg, + "aggs": {"avg_duration_ms": {"avg": {"field": "l_e2e_time_ms"}}}, + } + }, + } + + query_str = json.dumps(query) + res = None + for attempt in range(DEFAULT_RETRY_COUNT): + res = requests.get( + search_url, data=query_str, headers=headers, timeout=QUERY_TIMEOUT_SECONDS + ) + if res.status_code in (200, 201, 202): + break + print( + f" Warning: OpenSearch returned {res.status_code}, attempt {attempt + 1}/{DEFAULT_RETRY_COUNT}" + ) + else: + raise RuntimeError( + f"OpenSearch query failed after {DEFAULT_RETRY_COUNT} attempts: " + f"{res.status_code} {res.text[:200]}" + ) + + data = res.json() + buckets = data["aggregations"]["by_test"]["buckets"] + page += 1 + print(f" Page {page}: got {len(buckets)} buckets") + + for bucket in buckets: + test_name = bucket["key"]["test_name"] + avg_ms = bucket["avg_duration_ms"]["value"] + if avg_ms is not None: + test_durations[test_name] = avg_ms / 1000.0 + + after_key = data["aggregations"]["by_test"].get("after_key") + if not after_key or len(buckets) == 0: + break + + return test_durations + + +def main(): + # Parse command-line arguments + parser = argparse.ArgumentParser(description="Generate test duration file.") + parser.add_argument( + "--duration-file", + type=str, + default="new_test_duration.json", + help="Path to the output duration file (default: new_test_duration.json)", + ) + parser.add_argument( + "--cluster", + type=str, + default=None, + help="Cluster name (e.g. 'aws_dfw'). When set, writes " + "tests/integration/defs/.test_durations_ relative to the " + "repo root instead of --duration-file.", + ) + parser.add_argument( + "--days", + type=int, + default=7, + help="Number of days to look back in OpenSearch (default: 7).", + ) + parser.add_argument( + "--test-list-dir", + type=str, + default=DEFAULT_TEST_LIST_DIR, + help="Directory of turtle test-db YAML lists used to filter OpenSearch " + "results. Only turtle names present in these lists are written " + f"(default: {DEFAULT_TEST_LIST_DIR}).", + ) + parser.add_argument( + "--no-filter", + action="store_true", + default=False, + help="Skip filtering OpenSearch results against the test-db lists.", + ) + args = parser.parse_args() + + # Resolve output path + if args.cluster: + NEW_TEST_DURATION = os.path.join( + _REPO_ROOT, "tests", "integration", "defs", f".test_durations_{args.cluster}" + ) + else: + NEW_TEST_DURATION = args.duration_file + + print(f"Querying OpenSearch for last {args.days} day(s)...") + test_durations = query_opensearch_durations(args.days) + raw_count = len(test_durations) + + # Filter against the turtle test-db lists: an aggregated turtle name may be + # a stale entry or a subtest that is no longer scheduled. Keep only names + # that still appear in the checked-in lists. + dropped = 0 + if args.no_filter: + print("Filtering disabled (--no-filter); writing all turtle names.") + specs = None + else: + specs, yml_files = load_test_list_specs(args.test_list_dir) + print( + f"Loaded {len(specs)} test specs from {len(yml_files)} list(s) in {args.test_list_dir}" + ) + if not specs: + print(" Warning: no test specs loaded; writing all turtle names unfiltered.") + else: + filtered = { + name: dur + for name, dur in test_durations.items() + if normalize_test_spec(name) in specs + } + dropped = raw_count - len(filtered) + test_durations = filtered + + with open(NEW_TEST_DURATION, "w") as file: + json.dump(test_durations, file, indent=3) + file.write("\n") + + print("\nSummary:") + print(f" OpenSearch index : {OPENSEARCH_INDEX}") + print(f" Days looked back : {args.days}") + print(f" Turtle names from query: {raw_count}") + print(f" Dropped (not in lists) : {dropped}") + print(f" Unique tests in output : {len(test_durations)}") + print(f" Output written to : {NEW_TEST_DURATION}") + + +if __name__ == "__main__": + main() diff --git a/jenkins/scripts/perf/cluster_env.py b/jenkins/scripts/perf/cluster_env.py new file mode 100644 index 000000000000..cbc342aa2d24 --- /dev/null +++ b/jenkins/scripts/perf/cluster_env.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Cluster+GPU-aware environment settings for PerfSanity SLURM jobs. + +UCX transport selection depends primarily on the cluster's network fabric +(IB vs RoCE vs TCP-only), not just the GPU model. The same CI stage can land +on different clusters (frontend "auto:*" platforms are load-balanced across +backend clusters in bloom's SlurmConfig), so the rules below are keyed on +(cluster name, GPU type) instead of GPU type alone. + +Cluster names follow SlurmPartition.clusterName in the bloom Jenkins shared +library (src/com/nvidia/bloom/SlurmConfig.groovy), e.g. "gcp-nrt", "aws-cmh", +"aws-dfw", "oci-hsg", "nsc-svg", "dlcluster", "computelabSC01". In CI, +L0_Test.groovy passes the resolved cluster via --cluster-name; for local +submission it can be given explicitly or is best-effort detected from the +Slurm frontend. Slurm's own ClusterName carries a deployment suffix (e.g. +bloom "oci-nrt" -> slurm.conf "oci-nrt-cs-001"), so rules use prefix +wildcards to match both forms; pass --cluster-name explicitly if a cluster +breaks this naming convention. +""" + +from fnmatch import fnmatch + +# Applied on every cluster before any cluster-specific export: clear settings +# that may leak in from the outer environment and break UCX transport +# auto-selection. +BASE_UCX_UNSET = "unset UCX_CUDA_IPC_ENABLE_MNNVL UCX_TLS UCX_NET_DEVICES" + +# (cluster_pattern, gpu_pattern, extra_export) — evaluated in order, first +# match wins; the matched export is appended after BASE_UCX_UNSET (empty +# string = base unset only). Patterns are shell-style wildcards, matched +# case-insensitively. Cluster patterns are prefix wildcards so they match +# both the bloom name (CI, e.g. "aws-cmh") and the cluster's own slurm.conf +# ClusterName (local detection, e.g. "nsc-svg" -> "nsc-svg-slurm-1"). +UCX_ENV_RULES = [ + # gcp-nrt: RoCE fabric; pin the usable rocep ports and set the RoCE GID / + # QoS parameters required on this fabric. + ( + "gcp-nrt*", + "*", + "export UCX_NET_DEVICES=" + "rocep145s0:1,rocep146s0:1,rocep152s0:1,rocep153s0:1," + "rocep198s0:1,rocep199s0:1,rocep205s0:1,rocep206s0:1" + " UCX_IB_GID_INDEX=auto UCX_IB_TRAFFIC_CLASS=52 UCX_IB_SL=0", + ), + # oci-aga: avoid transports that fail on this VF fabric, disable DEVX to + # avoid UAR allocation failures, and pin the GPU-connected rail VFs. + ( + "oci-aga*", + "*", + "export UCX_TLS=^tcp,rc_gda,gga UCX_IB_MLX5_DEVX=n " + "UCX_NET_DEVICES=" + "rdma_vf_rail0:1,rdma_vf_rail1:1,rdma_vf_rail2:1,rdma_vf_rail3:1 " + "UCX_IB_TRAFFIC_CLASS=96 TRTLLM_NIXL_NUM_THREADS=1", + ), + # nsc-svg: UCX picks wrong RDMA devices; pin the usable mlx5 ports. + ( + "nsc-svg*", + "*", + "export UCX_NET_DEVICES=" + "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1", + ), + # aws-cmh: UCX transport auto-selection hangs on this fabric; pin the + # working transport set explicitly. + ("aws-cmh*", "*", "export UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp"), + # aws-dfw: gdr_copy is broken on this cluster; exclude it. + ("aws-dfw*", "*", "export UCX_TLS=^gdr_copy"), + # Default: base unset only. + ("*", "*", ""), +] + +# Ordered so composite names win over their substrings (GB200 before B200, +# GB300 before B300). +KNOWN_GPU_TYPES = ("GB300", "GB200", "GB10X", "B300", "B200", "H200", "H100", "A100") + + +def gpu_type_from_stage_name(stage_name): + """Extract the GPU type token from a CI stage name. + + Scans for the first match in KNOWN_GPU_TYPES (ordered longest-first to + avoid substring collisions, e.g. GB200 before B200). + + Args: + stage_name: CI stage name string, e.g. + "DGX_B200-8_GPUs-PyTorch-PerfSanity-1". None or empty string + is accepted and returns "". + + Returns: + A GPU type token such as "B200" or "GB300", or "" if no known GPU + type is found in the stage name. + """ + upper = (stage_name or "").upper() + for gpu in KNOWN_GPU_TYPES: + if gpu in upper: + return gpu + return "" + + +def gpu_type_from_supported_gpus(supported_gpus): + """Pick the GPU type from a config yaml's metadata.supported_gpus list. + + Args: + supported_gpus: List of GPU type strings from the config yaml + ``metadata.supported_gpus`` field. None or empty list returns "". + + Returns: + The first matching GPU type token from KNOWN_GPU_TYPES, or "" if + none of the known types appear in the list. + """ + gpus = {str(gpu).upper() for gpu in supported_gpus or []} + for gpu in KNOWN_GPU_TYPES: + if gpu in gpus: + return gpu + return "" + + +def get_ucx_tls_cmd(cluster_name, gpu_type): + """Return the shell prefix that sets UCX env vars for (cluster, GPU). + + Evaluates UCX_ENV_RULES in order, matching cluster_name and gpu_type + against shell-style wildcard patterns (case-insensitive). Cluster + patterns are prefix wildcards that match both the bloom CI name (e.g. + "aws-cmh") and the cluster's own slurm.conf ClusterName (e.g. + "aws-cmh-cs-001"). The first matching rule wins. + + Args: + cluster_name: Cluster name string (bloom CI name or detected via + scontrol). None or empty string matches only the catch-all "*" + rule. + gpu_type: GPU type token such as "B200" or "GB300". None or empty + string matches only the catch-all "*" rule. + + Returns: + A shell command prefix string that unsets leaking UCX env vars and + optionally exports cluster-specific overrides, ending with "&&" so + it can be prepended directly to the worker command. Example: + ``"unset UCX_CUDA_IPC_ENABLE_MNNVL UCX_TLS UCX_NET_DEVICES && + export UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp &&"``. + """ + cluster = (cluster_name or "").lower() + gpu = (gpu_type or "").upper() + extra = "" + for cluster_pat, gpu_pat, cmd in UCX_ENV_RULES: + if fnmatch(cluster, cluster_pat.lower()) and fnmatch(gpu, gpu_pat.upper()): + extra = cmd + break + if extra: + return f"{BASE_UCX_UNSET} && {extra} &&" + return f"{BASE_UCX_UNSET} &&" diff --git a/jenkins/scripts/perf/disaggregated/slurm_ct_precheck_gate.sh b/jenkins/scripts/perf/disaggregated/slurm_ct_precheck_gate.sh new file mode 100644 index 000000000000..e3b02f88741d --- /dev/null +++ b/jenkins/scripts/perf/disaggregated/slurm_ct_precheck_gate.sh @@ -0,0 +1,180 @@ +# Cache-transceiver precheck gate for the disaggregated perf-sanity launch +# script. submit.py splices this file into the generated launch script ahead +# of slurm_launch_draft.sh, which calls run_cache_transceiver_precheck after +# computing the per-server node slices. Kept as functions in a separate file +# so the gate logic can be sourced and exercised standalone. +# +# Expects the launch-script globals: srunArgs, numGenServers, numCtxServers, +# nodesPerGenServer/nodesPerCtxServer, gpusPerNodePerGenServer/ +# gpusPerNodePerCtxServer, genNodeLists/ctxNodeLists, testOutputDir, +# jobWorkspace, pytestCommandCTXPrecheck/pytestCommandGENPrecheck, +# precheckRunScript, ctPrecheckEnabled, ctPrecheckTimeout, stageName, +# and the cleanup_on_failure function. + +# Escape text for embedding in XML; junit parsers also choke on raw control +# bytes that MPI/UCX logs may contain, so strip those too. +ct_xml_escape() { + sed -e 's/&/\&/g' -e 's//\>/g' -e 's/"/\"/g' \ + | tr -d '\000-\010\013\014\016-\037' +} + +# First few root-cause-shaped lines of a step log — the tail alone can miss +# the real error when it happened early and retry spam follows. +ct_first_errors() { + grep -m 5 -nE "Traceback \(most recent call last\)|MPI_ABORT|MPIR_Err|srun: error|Segmentation fault|CUDA error|RuntimeError|AssertionError|INIT_ERROR|TRANSFER_ERROR" \ + "$1" 2>/dev/null || true +} + +# Shared verdict predicate + failing-log excerpt, so the console summary and +# the junit xml cannot drift apart. +ct_step_passed() { + local statusFile="$precheckDir/status/$1.status" + [ -f "$statusFile" ] && grep -q "^PASS" "$statusFile" +} + +ct_step_excerpt() { + local stepLog="$precheckDir/logs/$1.log" + echo "First error lines (line-numbered):" + ct_first_errors "$stepLog" + echo "Log tail ($stepLog):" + tail -n 60 "$stepLog" 2>/dev/null || true +} + +# Console summary for a failed precheck: per-instance verdicts, first error +# lines + tail of each failing step log, and UCX red-flag lines. +# Uses: precheckDir, precheckNames. +ct_print_failure_summary() { + echo "====================================================================" + echo "CACHE TRANSCEIVER PRECHECK FAILED - the disaggregated test will NOT run" + echo "Instance verdicts:" + cat "$precheckDir"/status/*.status 2>/dev/null \ + || echo "(no status files - steps died before transceiver setup; see logs below)" + echo "" + echo "Failing step logs:" + for k in "${!precheckNames[@]}"; do + ct_step_passed "${precheckNames[$k]}" && continue + echo "----- ${precheckNames[$k]} -----" + ct_step_excerpt "${precheckNames[$k]}" + done + echo "" + echo "UCX red flags (host-staged tcp fallback / UCX errors), if any:" + grep -hE "sw-emul|UCX +(ERROR|WARN)" "$precheckDir"/logs/*.log 2>/dev/null | sort -u | head -20 || true + echo "Full artifacts: $precheckDir (status/*.json, logs/, csv/)" + echo "====================================================================" +} + +# Synthetic junit result so the failure shows up as a test entry in the +# Jenkins test report: uploadResults scps $jobWorkspace/results*.xml back and +# junit() ingests them. Best-effort — callers must not let this mask the real +# failure path. Uses: precheckDir, precheckNames, jobWorkspace, stageName. +ct_write_junit_xml() { + local junitXml="$jobWorkspace/results-ct-precheck.xml" + local suiteName + suiteName="$(printf '%s' "${stageName:-${SLURM_JOB_NAME:-disagg_perf_sanity}}" | ct_xml_escape)" + local junitFailures=0 + local junitCases="" + local name verdict detail + for k in "${!precheckNames[@]}"; do + name="${precheckNames[$k]}" + if ct_step_passed "$name"; then + junitCases+=""$'\n' + continue + fi + junitFailures=$((junitFailures + 1)) + verdict="$( (head -n 1 "$precheckDir/status/$name.status" 2>/dev/null \ + || echo "NO_STATUS: step died before writing a verdict (see log)") | ct_xml_escape)" + detail="$(ct_step_excerpt "$name" | ct_xml_escape)" + junitCases+="" + junitCases+="$detail"$'\n' + done + if { + echo '' + echo "" + printf '%s' "$junitCases" + echo "" + } > "$junitXml" 2>/dev/null; then + echo "Synthetic junit result written to $junitXml (will appear in the Jenkins test report)" + else + echo "WARNING: could not write synthetic junit xml to $junitXml (non-fatal)" + fi +} + +# Run one precheck srun per ctx/gen server instance with the same node +# slices, MPI topology, and UCX env as the real server steps +# (pytestCommand{CTX,GEN}Precheck embed the same ucx_tls_cmd + worker env var +# strings). On failure: console summary + synthetic junit, then the stage +# aborts via cleanup_on_failure — before any model bring-up. +run_cache_transceiver_precheck() { + if [ "${ctPrecheckEnabled:-0}" != "1" ] || [ "${TRTLLM_DISAGG_BENCHMARK_GEN_ONLY:-0}" = "1" ]; then + return 0 + fi + echo "Starting cache transceiver precheck..." + precheckDir="$testOutputDir/cache_transceiver_precheck" + mkdir -p "$precheckDir/logs" + # A reused work dir (Slurm requeue reruns this batch script with the same + # directories) may hold a previous run's rendezvous/status/csv/abort files: + # stale addr files would point gen leaders at dead ports, stale status files + # would pollute the verdict aggregation below, a stale precheck.abort would + # fail-fast-skip the whole rerun (a requeued job keeps its SLURM_JOB_ID, so + # the driver's job-id stamp cannot tell it apart), and stale bandwidth CSVs + # (the Python transceiver's perf__.csv are per-run and appended) + # would make parse_python_bandwidth_gbps median over two runs' samples. The + # driver also job-id-stamps addr files as a second line of defense. + rm -f "$precheckDir"/rendezvous/*.addr "$precheckDir"/status/*.status \ + "$precheckDir"/status/*.json "$precheckDir"/precheck.abort 2>/dev/null || true + rm -rf "$precheckDir"/csv 2>/dev/null || true + precheckPids=() + precheckNames=() + # ct_launch_step + ct_launch_step() { + local role=$1 i=$2 nodes=$3 gpusPerNode=$4 nodeList=$5 pytestCmd=$6 + export DISAGG_SERVING_TYPE="${role^^}_PRECHECK_$i" + export pytestCommand="$pytestCmd --server-idx $i" + timeout -k 60 "${ctPrecheckTimeout:-900}" \ + srun "${srunArgs[@]}" --mpi=pmix --kill-on-bad-exit=1 \ + -N "$nodes" \ + -w "$nodeList" \ + --ntasks=$((nodes * gpusPerNode)) \ + --ntasks-per-node="$gpusPerNode" \ + bash $precheckRunScript &> "$precheckDir/logs/${role}_$i.log" & + precheckPids+=($!) + precheckNames+=("${role}_$i") + sleep 5 # Wait for pyxis container namespace initialization to avoid race condition + } + local i + for i in $(seq 0 $((numGenServers - 1))); do + ct_launch_step gen "$i" "$nodesPerGenServer" "$gpusPerNodePerGenServer" \ + "${genNodeLists[$i]}" "$pytestCommandGENPrecheck" + done + for i in $(seq 0 $((numCtxServers - 1))); do + ct_launch_step ctx "$i" "$nodesPerCtxServer" "$gpusPerNodePerCtxServer" \ + "${ctxNodeLists[$i]}" "$pytestCommandCTXPrecheck" + done + + local precheckFailed=0 k rc + for k in "${!precheckPids[@]}"; do + if wait "${precheckPids[$k]}"; then + echo "Precheck step ${precheckNames[$k]} passed" + else + rc=$? + echo "Precheck step ${precheckNames[$k]} FAILED (exit $rc; 124 = external timeout)" + precheckFailed=1 + fi + done + + if [ "$precheckFailed" -eq 1 ]; then + ct_print_failure_summary + ct_write_junit_xml + cleanup_on_failure "Cache transceiver precheck failed. See summary above and $precheckDir" + fi + # No status files means every step took the skip path (e.g. the yaml has no + # cache_transceiver_config.backend): the run wrote no verdicts, so report it + # as SKIPPED rather than an empty "PASSED" that reads like real validation. + if ls "$precheckDir"/status/*.status >/dev/null 2>&1; then + echo "Cache transceiver precheck PASSED:" + cat "$precheckDir"/status/*.status 2>/dev/null || true + else + echo "Cache transceiver precheck SKIPPED: not applicable for this config" \ + "(no verdicts written; see $precheckDir/logs)" + fi +} diff --git a/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh b/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh index a3712ff6d114..c698cd32b436 100644 --- a/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh +++ b/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh @@ -17,18 +17,67 @@ if ! srun "${srunArgs[@]}" $installScript &> $jobWorkspace/install.log; then fi echo "Installation completed on all nodes" +# Deterministic node slices per server: gen servers take the first nodes, +# then ctx servers (same order the steps are started in). Both the cache +# transceiver precheck and the real server steps pin to these slices with +# `srun -w`, so the precheck exercises exactly the node pairs / NICs the +# real disaggregated test will use. +mapfile -t allNodes < <(scontrol show hostnames "$SLURM_JOB_NODELIST") +nodeCursor=0 +genNodeLists=() +for i in $(seq 0 $((numGenServers - 1))); do + slice=("${allNodes[@]:$nodeCursor:$nodesPerGenServer}") + genNodeLists+=("$(IFS=,; echo "${slice[*]}")") + nodeCursor=$((nodeCursor + nodesPerGenServer)) +done +ctxNodeLists=() +if [ "${TRTLLM_DISAGG_BENCHMARK_GEN_ONLY:-0}" != "1" ]; then + for i in $(seq 0 $((numCtxServers - 1))); do + slice=("${allNodes[@]:$nodeCursor:$nodesPerCtxServer}") + ctxNodeLists+=("$(IFS=,; echo "${slice[*]}")") + nodeCursor=$((nodeCursor + nodesPerCtxServer)) + done +fi +if [ "$nodeCursor" -gt "${#allNodes[@]}" ]; then + cleanup_on_failure "Node slicing needs $nodeCursor nodes but the job only has ${#allNodes[@]} ($SLURM_JOB_NODELIST)" +fi + +# Cache transceiver network precheck: same instance count / node slices / +# MPI topology / UCX env as the real ctx+gen server steps. On failure the +# stage aborts HERE, with per-instance verdicts + a synthetic junit entry, +# before any model bring-up. Functions come from slurm_ct_precheck_gate.sh, +# spliced in above this draft by submit.py. No-op unless ctPrecheckEnabled=1. +run_cache_transceiver_precheck + # Start gen servers echo "Starting gen servers..." for i in $(seq 0 $((numGenServers - 1))); do gen_world_size=$((nodesPerGenServer * gpusPerNodePerGenServer)) export DISAGG_SERVING_TYPE="GEN_$i" export pytestCommand="$pytestCommandGENWorker" - srun "${srunArgs[@]}" --mpi=pmix --kill-on-bad-exit=1 \ - -N $nodesPerGenServer \ - --ntasks=$gen_world_size \ - --ntasks-per-node=$gpusPerNodePerGenServer \ - $runScript &> $testOutputDir/gen_server_$i.log & - echo "Started gen server $i" + # End-of-write sentinel: gen_server_$i.log is the srun's &> aggregate of + # every gen-worker rank (the per-iter prev_device_step_time lines the + # benchmark parses live only here, not in trtllm-serve.GEN_*.log). The + # file descriptor is owned by this srun, so the log is only guaranteed + # fully flushed once the srun is reaped. Run srun in the foreground of a + # backgrounded subshell and touch gen_server_$i.done immediately after it + # returns: the benchmark srun blocks on that sentinel before parsing, so + # it never reads a truncated / not-yet-flushed log (nvbugs 6487036 / + # 6487040). A stale sentinel from a re-run output dir is removed first. + # Note: srun is foreground inside the subshell (not `srun ... &` + a + # `kill -0` poll) so `touch` runs strictly after reap, with no + # late-zombie race that could either skip or prematurely fire the signal. + rm -f "$testOutputDir/gen_server_$i.done" + ( + srun "${srunArgs[@]}" --mpi=pmix --kill-on-bad-exit=1 \ + -N $nodesPerGenServer \ + -w "${genNodeLists[$i]}" \ + --ntasks=$gen_world_size \ + --ntasks-per-node=$gpusPerNodePerGenServer \ + $runScript &> $testOutputDir/gen_server_$i.log + touch "$testOutputDir/gen_server_$i.done" + ) & + echo "Started gen server $i on ${genNodeLists[$i]}" sleep 5 # Wait for pyxis container namespace initialization to avoid race condition done @@ -41,10 +90,11 @@ if [ "${TRTLLM_DISAGG_BENCHMARK_GEN_ONLY:-0}" != "1" ]; then export pytestCommand="$pytestCommandCTXWorker" srun "${srunArgs[@]}" --mpi=pmix --kill-on-bad-exit=1 \ -N $nodesPerCtxServer \ + -w "${ctxNodeLists[$i]}" \ --ntasks=$ctx_world_size \ --ntasks-per-node=$gpusPerNodePerCtxServer \ $runScript &> $testOutputDir/ctx_server_$i.log & - echo "Started ctx server $i" + echo "Started ctx server $i on ${ctxNodeLists[$i]}" sleep 5 # Wait for pyxis container namespace initialization to avoid race condition done else diff --git a/jenkins/scripts/perf/disaggregated/slurm_precheck_run.sh b/jenkins/scripts/perf/disaggregated/slurm_precheck_run.sh new file mode 100644 index 000000000000..1dbafd5c51bf --- /dev/null +++ b/jenkins/scripts/perf/disaggregated/slurm_precheck_run.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Per-rank container entrypoint for the disagg cache-transceiver PRECHECK +# step (launched by slurm_launch_draft.sh BEFORE the real ctx/gen servers). +# +# It intentionally mirrors slurm_run.sh's runtime environment: the same +# slurm_env_setup.sh is sourced (LD_LIBRARY_PATH, the `unset UCX_TLS=tcp` +# fixup, PMIX_MCA_gds), and $pytestCommand carries the same +# `unset/export UCX_TLS ...` prefix and worker env vars as the real ctx/gen +# worker steps (built from the same strings by jenkins/scripts/perf/submit.py), +# so a precheck PASS/FAIL is representative of the network environment the +# real test will run in. Unlike slurm_run.sh it skips coverage/perf-report +# handling -- the precheck is a plain MPI program, not a pytest run. + +set -xEeuo pipefail +trap 'rc=$?; echo "Error in file ${BASH_SOURCE[0]} on line $LINENO: $BASH_COMMAND (exit $rc)"; exit $rc' ERR + +# CI (jenkins/L0_Test.groovy) exports resourcePathNode with the /tmp +# extraction layout; the local flow (jenkins/scripts/perf/local/submit.py) +# exports llmSrcNode pointing at the repo directly and has no +# resourcePathNode. Honor whichever layout is present. +if [ -n "${resourcePathNode:-}" ]; then + cd "$resourcePathNode" + llmSrcNode=$resourcePathNode/TensorRT-LLM/src +fi +: "${llmSrcNode:?either resourcePathNode or llmSrcNode must be exported}" + +source "$llmSrcNode/jenkins/scripts/slurm_env_setup.sh" +slurm_setup_runtime_env + +echo "Precheck rank ${SLURM_PROCID:-?} (${DISAGG_SERVING_TYPE:-unknown}) command: $pytestCommand" + +set +e +eval $pytestCommand +precheck_exit_code=$? +set -e + +echo "Rank${SLURM_PROCID:-?} cache-transceiver precheck finished with exit code $precheck_exit_code" +exit $precheck_exit_code diff --git a/jenkins/scripts/perf/local/configs/example.conf b/jenkins/scripts/perf/local/configs/example.conf index c11e768d02ce..fc7371a41a78 100644 --- a/jenkins/scripts/perf/local/configs/example.conf +++ b/jenkins/scripts/perf/local/configs/example.conf @@ -18,6 +18,13 @@ account="${YOUR_SLURM_ACCOUNT:-your_account}" job_name="disagg_test" time_limit="04:00:00" +# Cluster name for cluster-specific env settings (UCX_TLS etc.) — see +# jenkins/scripts/perf/cluster_env.py for the rule table. Use the CI (bloom) +# name, e.g. gcp-nrt, aws-cmh, aws-dfw, oci-hsg. If unset, submit.py falls +# back to detecting the Slurm ClusterName from the frontend (prefix-matched +# against the rules, e.g. detected "oci-nrt-cs-001" matches "oci-nrt*"). +# cluster_name="gcp-nrt" + # Docker image — pick ONE of two modes. The script supports both; the rule is: # # ┌──────────────────────┬──────────────────────┬────────────────────────────┐ diff --git a/jenkins/scripts/perf/local/run_disagg.sh b/jenkins/scripts/perf/local/run_disagg.sh index 2f773e038b0c..50006181775b 100755 --- a/jenkins/scripts/perf/local/run_disagg.sh +++ b/jenkins/scripts/perf/local/run_disagg.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 # Generate slurm_launch.sh for a local disaggregated perf-sanity run # and submit it via sbatch. Run this on a SLURM login node. # @@ -67,6 +69,7 @@ source "$config_file" : "${build_wheel_flag:=}" : "${capture_nsys_flag:=}" : "${time_limit:=02:00:00}" +: "${cluster_name:=}" # Normalize test list: prefer 'test_ids' bash array if set, else fall back to # legacy single 'test_id'. Either declares a non-empty list at this point. @@ -231,6 +234,7 @@ for idx in "${!test_ids[@]}"; do --mounts "$mounts" \ --llm-models-root "$llm_models_path" \ --time "$time_limit" \ + ${cluster_name:+--cluster-name "$cluster_name"} \ "${install_args[@]}" \ $build_wheel_flag \ $capture_nsys_flag; then diff --git a/jenkins/scripts/perf/local/slurm_install.sh b/jenkins/scripts/perf/local/slurm_install.sh index 91bb7d664e62..2524c7c8ac79 100755 --- a/jenkins/scripts/perf/local/slurm_install.sh +++ b/jenkins/scripts/perf/local/slurm_install.sh @@ -21,7 +21,7 @@ slurm_build_wheel() { fi echo "Building wheel on node ${SLURM_NODEID:-0}, task ${SLURM_LOCALID:-0}" - retry_command bash -c "cd $llmSrcNode && rm -rf .venv-3.12 && python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache --cuda_architectures '100-real' --clean -c" + retry_command bash -c "cd $llmSrcNode && rm -rf .venv-3.12 && python3 ./scripts/build_wheel.py --use_ccache --cuda_architectures '100-real' --clean -c" cd $jobWorkspace echo "(Writing build wheel lock) Lock file: $build_lock_file" diff --git a/jenkins/scripts/perf/local/submit.py b/jenkins/scripts/perf/local/submit.py index f71a825936c3..2aab15795daf 100755 --- a/jenkins/scripts/perf/local/submit.py +++ b/jenkins/scripts/perf/local/submit.py @@ -6,10 +6,28 @@ import re import shutil import subprocess +import sys from datetime import datetime import yaml +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from cluster_env import get_ucx_tls_cmd, gpu_type_from_supported_gpus # noqa: E402 + + +def _import_precheck_config(llm_src): + """Import the pure-stdlib precheck config module from the repo tree. + + It is the single owner of the gate's enable policy and timeout formulas. + """ + path = os.path.join(llm_src, "tests", "scripts", "perf-sanity", "cache_transceiver_precheck") + if path not in sys.path: + sys.path.insert(0, path) + import precheck_config + + return precheck_config + + AGG_CONFIG_FOLDER = os.environ.get("AGG_CONFIG_FOLDER", "tests/scripts/perf-sanity/aggregated") DISAGG_CONFIG_FOLDER = os.environ.get( "DISAGG_CONFIG_FOLDER", "tests/scripts/perf-sanity/disaggregated" @@ -250,25 +268,43 @@ def get_hardware_config(config, runtime_mode, benchmark_mode, test_name=None): } +def _join_env(*parts): + """Space-join non-empty env-var strings (drops falsy entries).""" + return " ".join(p for p in parts if p) + + def get_env_config(config, runtime_mode, benchmark_mode=None, server_name=None): """Get worker / server / benchmark env vars from the yaml. Aggregated yaml stores env vars per server config under `server_configs[i].server_env_var`. Disaggregated yaml stores them at the - top-level `environment.{worker,server,benchmark}_env_var`. + top-level `environment.{worker,server,benchmark}_env_var`, plus optional + `environment.{ctx,gen}_worker_env_var` for role-specific extras (appended + to the shared `worker_env_var`). ctx_only is a hybrid: the launch path is aggregated, but the yaml is the disagg one, so the agg launch's "server_env_var" comes from - `environment.worker_env_var`. + `environment.worker_env_var` (merged with ctx-side extras when present). - Returns: {worker_env_var, server_env_var, benchmark_env_var}. + Returns: {worker_env_var (shared, back-compat), + ctx_worker_env_var, gen_worker_env_var, + server_env_var, benchmark_env_var}. """ env = config.get("environment", {}) or {} + common = env.get("worker_env_var", "") or "" + ctx_extra = env.get("ctx_worker_env_var", "") or "" + gen_extra = env.get("gen_worker_env_var", "") or "" + ctx_env = _join_env(common, ctx_extra) + gen_env = _join_env(common, gen_extra) if runtime_mode == "aggregated": if benchmark_mode == "ctx_only": return { - "worker_env_var": env.get("worker_env_var", "") or "", - "server_env_var": env.get("worker_env_var", "") or "", + "worker_env_var": common, + "ctx_worker_env_var": ctx_env, + "gen_worker_env_var": gen_env, + # ctx_only launches through the aggregated single-pytest path; + # the ctx-merged env is what actually runs. + "server_env_var": ctx_env, "benchmark_env_var": env.get("benchmark_env_var", "") or "", } agg_server_env_var = "" @@ -278,11 +314,15 @@ def get_env_config(config, runtime_mode, benchmark_mode=None, server_name=None): break return { "worker_env_var": "", + "ctx_worker_env_var": "", + "gen_worker_env_var": "", "server_env_var": agg_server_env_var, "benchmark_env_var": "", } return { - "worker_env_var": env.get("worker_env_var", "") or "", + "worker_env_var": common, + "ctx_worker_env_var": ctx_env, + "gen_worker_env_var": gen_env, "server_env_var": env.get("server_env_var", "") or "", "benchmark_env_var": env.get("benchmark_env_var", "") or "", } @@ -302,6 +342,25 @@ def get_benchmark_config(config, benchmark_mode): } +def get_benchmark_request_queue_size(config, concurrency): + """Cap the gen-only fill target to the GEN executor's active capacity.""" + gen_config = (config.get("worker_config", {}) or {}).get("gen", {}) or {} + concurrency = int(concurrency) + max_batch_size = int(gen_config.get("max_batch_size", concurrency)) + enable_attention_dp = gen_config.get("enable_attention_dp", False) + tp_size = int(gen_config.get("tensor_parallel_size", 1)) + max_capacity = max_batch_size * tp_size if enable_attention_dp else max_batch_size + queue_size = min(max_capacity, concurrency) + if queue_size < concurrency: + print( + "[WARNING] TLLM_BENCHMARK_REQ_QUEUES_SIZE capped to " + f"{queue_size} (max_batch_size={max_batch_size}, tp_size={tp_size}, " + f"attention_dp={enable_attention_dp}) instead of concurrency={concurrency}. " + "The fill loop cannot reach a target above the GEN executor capacity." + ) + return queue_size + + def partition_has_gpu_gres(partition): """Return True if the Slurm partition reports GPU GRES (e.g. 'gpu:4'), False if null/absent.""" try: @@ -320,6 +379,27 @@ def partition_has_gpu_gres(partition): return False +def detect_cluster_name(): + """Best-effort Slurm cluster name detection on the submission frontend.""" + name = os.environ.get("SLURM_CLUSTER_NAME", "") + if name: + return name + try: + config_out = subprocess.check_output( + ["scontrol", "show", "config"], + stderr=subprocess.DEVNULL, + text=True, + timeout=10, + ) + for line in config_out.splitlines(): + key, separator, value = line.partition("=") + if key.strip() == "ClusterName" and separator: + return value.strip() + except (OSError, subprocess.SubprocessError): + pass + return "" + + def generate_sbatch_params(args, hardware_config, work_dir): """Generate #SBATCH parameters.""" total_nodes = hardware_config["total_nodes"] @@ -569,6 +649,14 @@ def main(): default=8000, help="Port the disagg server listens on (exported as DISAGG_SERVER_PORT)", ) + parser.add_argument( + "--cluster-name", + default="", + help="Cluster name used with the GPU type to pick UCX env settings " + "(bloom SlurmPartition.clusterName, e.g. gcp-nrt, aws-cmh). If not " + "set, best-effort detected from SLURM_CLUSTER_NAME / scontrol; note " + "Slurm's own ClusterName may differ from the bloom name.", + ) args = parser.parse_args() @@ -630,10 +718,14 @@ def main(): with open(config_yaml, "r") as f: config = yaml.safe_load(f) - # Detect GPU type from config metadata - supported_gpus = config.get("metadata", {}).get("supported_gpus", []) - is_b200 = "B200" in supported_gpus - is_gb300 = "GB300" in supported_gpus + # Detect GPU type and cluster only for disaggregated UCX selection. + if runtime_mode == "disaggregated": + supported_gpus = config.get("metadata", {}).get("supported_gpus", []) + gpu_type = gpu_type_from_supported_gpus(supported_gpus) + cluster_name = args.cluster_name or detect_cluster_name() + else: + gpu_type = "" + cluster_name = "" # Create timestamp timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") @@ -786,19 +878,22 @@ def main(): server_env_vars = "" benchmark_env_var = "" if runtime_mode == "disaggregated": - # Build worker env vars (split into ctx and gen for role-specific settings) - common_worker_env_var = env_config.get("worker_env_var", "") + # Build worker env vars (split into ctx and gen for role-specific + # settings). get_env_config already merged the shared worker_env_var + # with any per-role ctx_worker_env_var / gen_worker_env_var from yaml. + ctx_worker_env_var = env_config.get("ctx_worker_env_var", "") + gen_worker_env_var = env_config.get("gen_worker_env_var", "") ctx_worker_env_vars = ( f"TLLM_PROFILE_START_STOP='{ctx_tllm_profile_start_stop}' " f"FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${{SLURM_LOCALID}} " f"HF_HOME=/tmp/hf_home " - f"{common_worker_env_var}" + f"{ctx_worker_env_var}" ) gen_worker_env_vars = ( f"TLLM_PROFILE_START_STOP='{gen_tllm_profile_start_stop}' " f"FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${{SLURM_LOCALID}} " f"HF_HOME=/tmp/hf_home " - f"{common_worker_env_var}" + f"{gen_worker_env_var}" ) server_env_vars = env_config.get("server_env_var", "") benchmark_env_var = env_config.get("benchmark_env_var", "") @@ -810,20 +905,17 @@ def main(): srun_args_lines.append("--container-env=TRTLLM_DISAGG_BENCHMARK_GEN_ONLY") elif "gen_only" in bm_config.get("mode", ""): concurrency = bm_config.get("concurrency", 1) + queue_size = get_benchmark_request_queue_size(config, concurrency) ctx_worker_env_vars = ( f"TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1 {ctx_worker_env_vars}" ) gen_worker_env_vars = ( f"TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1 " - f"TLLM_BENCHMARK_REQ_QUEUES_SIZE={concurrency} {gen_worker_env_vars}" + f"TLLM_BENCHMARK_REQ_QUEUES_SIZE={queue_size} {gen_worker_env_vars}" ) - if is_gb300: - ucx_tls_cmd = "export UCX_TLS=cuda_copy,cuda_ipc,sm,self,tcp &&" - elif is_b200: - ucx_tls_cmd = "export UCX_TLS=^ib &&" - else: - ucx_tls_cmd = "unset UCX_TLS UCX_NET_DEVICES &&" + ucx_tls_cmd = get_ucx_tls_cmd(cluster_name, gpu_type) + print(f"UCX env: cluster={cluster_name!r} gpu={gpu_type!r} -> {ucx_tls_cmd!r}") script_prefix_lines.extend( [ f'export CTX_WORKER_ENV_VARS="{ctx_worker_env_vars}"', @@ -865,6 +957,24 @@ def main(): ] ) + # Cache-transceiver network precheck (same wiring as jenkins/scripts/ + # perf/submit.py): reuses the exact ucx_tls_cmd + worker env strings + # of the real worker steps; enable policy and timeouts come from + # precheck_config (single owner). + pcfg = _import_precheck_config(llm_src) + script_prefix_lines.extend( + pcfg.precheck_prefix_lines( + config, + benchmark_mode, + config_path_expr="$configYamlPath", + ucx_tls_cmd=ucx_tls_cmd, + max_world=max( + hardware_config.get("gpus_per_ctx_server", 0) or 0, + hardware_config.get("gpus_per_gen_server", 0) or 0, + ), + ) + ) + # Add srun args for disagg srun_args_lines.extend( [ @@ -961,9 +1071,15 @@ def main(): draft_launch_lines = remove_whitespace_lines(draft_launch_lines) draft_launch_content = "\n".join(draft_launch_lines) + # The disagg draft calls run_cache_transceiver_precheck; splice in the gate + # function library ahead of it (single owner: precheck_config). + gate_content = "" + if runtime_mode == "disaggregated": + gate_content = pcfg.gate_library_content(draft_launch_sh, llm_src) + # Combine and write launch script script_prefix = "\n".join(script_prefix_lines) - final_script = f"{script_prefix}\n\n{srun_args}\n\n{draft_launch_content}" + final_script = f"{script_prefix}\n\n{srun_args}\n\n{gate_content}{draft_launch_content}" with open(launch_sh, "w") as f: f.write(final_script) diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py index a621d4ca1f8c..88df4076f255 100755 --- a/jenkins/scripts/perf/submit.py +++ b/jenkins/scripts/perf/submit.py @@ -21,8 +21,24 @@ import math import os import re +import sys import yaml +from cluster_env import get_ucx_tls_cmd, gpu_type_from_stage_name + + +def _import_precheck_config(llm_src): + """Import the pure-stdlib precheck config module from the repo tree. + + It is the single owner of the gate's enable policy and timeout formulas. + """ + path = os.path.join(llm_src, "tests", "scripts", "perf-sanity", "cache_transceiver_precheck") + if path not in sys.path: + sys.path.insert(0, path) + import precheck_config + + return precheck_config + AGG_CONFIG_FOLDER = "tests/scripts/perf-sanity/aggregated" DISAGG_CONFIG_FOLDER = "tests/scripts/perf-sanity/disaggregated" @@ -219,25 +235,43 @@ def get_hardware_config(config, runtime_mode, benchmark_mode, server_name): } +def _join_env(*parts): + """Space-join non-empty env-var strings (drops falsy entries).""" + return " ".join(p for p in parts if p) + + def get_env_config(config, runtime_mode, benchmark_mode, server_name): """Get worker / server / benchmark env vars from the yaml. Aggregated yaml stores env vars per server config under `server_configs[i].server_env_var`. Disaggregated yaml stores them at the - top-level `environment.{worker,server,benchmark}_env_var`. + top-level `environment.{worker,server,benchmark}_env_var`, plus optional + `environment.{ctx,gen}_worker_env_var` for role-specific extras (appended + to the shared `worker_env_var`). ctx_only is a hybrid: the launch path is aggregated, but the yaml is the disagg one, so the agg launch's "server_env_var" comes from - `environment.worker_env_var`. + `environment.worker_env_var` (merged with ctx-side extras when present). - Returns: {worker_env_var, server_env_var, benchmark_env_var}. + Returns: {worker_env_var (shared, back-compat), + ctx_worker_env_var, gen_worker_env_var, + server_env_var, benchmark_env_var}. """ env = config.get("environment", {}) or {} + common = env.get("worker_env_var", "") or "" + ctx_extra = env.get("ctx_worker_env_var", "") or "" + gen_extra = env.get("gen_worker_env_var", "") or "" + ctx_env = _join_env(common, ctx_extra) + gen_env = _join_env(common, gen_extra) if runtime_mode == "aggregated": if benchmark_mode == "ctx_only": return { - "worker_env_var": env.get("worker_env_var", "") or "", - "server_env_var": env.get("worker_env_var", "") or "", + "worker_env_var": common, + "ctx_worker_env_var": ctx_env, + "gen_worker_env_var": gen_env, + # ctx_only launches through the aggregated single-pytest path; + # the ctx-merged env is what actually runs. + "server_env_var": ctx_env, "benchmark_env_var": env.get("benchmark_env_var", "") or "", } agg_server_env_var = "" @@ -247,11 +281,15 @@ def get_env_config(config, runtime_mode, benchmark_mode, server_name): break return { "worker_env_var": "", + "ctx_worker_env_var": "", + "gen_worker_env_var": "", "server_env_var": agg_server_env_var, "benchmark_env_var": "", } return { - "worker_env_var": env.get("worker_env_var", "") or "", + "worker_env_var": common, + "ctx_worker_env_var": ctx_env, + "gen_worker_env_var": gen_env, "server_env_var": env.get("server_env_var", "") or "", "benchmark_env_var": env.get("benchmark_env_var", "") or "", } @@ -267,6 +305,25 @@ def get_benchmark_config(config): } +def get_benchmark_request_queue_size(config, concurrency): + """Cap the gen-only fill target to the GEN executor's active capacity.""" + gen_config = (config.get("worker_config", {}) or {}).get("gen", {}) or {} + concurrency = int(concurrency) + max_batch_size = int(gen_config.get("max_batch_size", concurrency)) + enable_attention_dp = gen_config.get("enable_attention_dp", False) + tp_size = int(gen_config.get("tensor_parallel_size", 1)) + max_capacity = max_batch_size * tp_size if enable_attention_dp else max_batch_size + queue_size = min(max_capacity, concurrency) + if queue_size < concurrency: + print( + "[WARNING] TLLM_BENCHMARK_REQ_QUEUES_SIZE capped to " + f"{queue_size} (max_batch_size={max_batch_size}, tp_size={tp_size}, " + f"attention_dp={enable_attention_dp}) instead of concurrency={concurrency}. " + "The fill loop cannot reach a target above the GEN executor capacity." + ) + return queue_size + + # --------------------------------------------------------------------------- # # pytestCommand splitting # --------------------------------------------------------------------------- # @@ -434,6 +491,13 @@ def main(): help="1-indexed split group id. Selects the N-th test from the test list.", ) parser.add_argument("--stage-name", default="", help="Stage name (for logging / GPU detect)") + parser.add_argument( + "--cluster-name", + default="", + help="Slurm cluster name as resolved by the Jenkins pipeline " + "(bloom SlurmPartition.clusterName, e.g. gcp-nrt, aws-cmh). " + "Used with the GPU type to pick UCX env settings.", + ) args = parser.parse_args() @@ -478,8 +542,7 @@ def main(): ) = get_pytest_commands(script_prefix_lines, runtime_mode) test_output_dir = get_test_output_dir(script_prefix_lines, test_case_name) - is_gb300 = "GB300" in args.stage_name.upper() - is_b200 = "B200" in args.stage_name.upper() and "GB200" not in args.stage_name.upper() + gpu_type = gpu_type_from_stage_name(args.stage_name) if runtime_mode == "aggregated": # Aggregated (incl. ctx_only): single pytestCommand built from the @@ -513,13 +576,13 @@ def main(): srun_args_lines.append("--container-env=pytestCommand") else: # Disaggregated (e2e or gen_only). - base_worker_env_vars = ( - f"FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${{SLURM_LOCALID}} " - f"HF_HOME=/tmp/hf_home " - f"{env_config['worker_env_var']}" + base_prefix = ( + "FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${SLURM_LOCALID} HF_HOME=/tmp/hf_home" ) - ctx_worker_env_vars = base_worker_env_vars - gen_worker_env_vars = base_worker_env_vars + # ctx / gen env vars: shared worker_env_var + optional per-role extras + # from the yaml. get_env_config already merged them. + ctx_worker_env_vars = f"{base_prefix} {env_config['ctx_worker_env_var']}".rstrip() + gen_worker_env_vars = f"{base_prefix} {env_config['gen_worker_env_var']}".rstrip() server_env_vars = env_config["server_env_var"] # gen_only_no_context comes from yaml's benchmark.mode, not the test @@ -532,20 +595,17 @@ def main(): srun_args_lines.append("--container-env=TRTLLM_DISAGG_BENCHMARK_GEN_ONLY") elif benchmark_mode == "gen_only": concurrency = benchmark_config.get("concurrency", 1) + queue_size = get_benchmark_request_queue_size(config, concurrency) ctx_worker_env_vars = ( f"TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1 {ctx_worker_env_vars}" ) gen_worker_env_vars = ( f"TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1 " - f"TLLM_BENCHMARK_REQ_QUEUES_SIZE={concurrency} {gen_worker_env_vars}" + f"TLLM_BENCHMARK_REQ_QUEUES_SIZE={queue_size} {gen_worker_env_vars}" ) - if is_gb300: - ucx_tls_cmd = "export UCX_TLS=cuda_copy,cuda_ipc,sm,self,tcp &&" - elif is_b200: - ucx_tls_cmd = "export UCX_TLS=^ib &&" - else: - ucx_tls_cmd = "unset UCX_TLS UCX_NET_DEVICES &&" + ucx_tls_cmd = get_ucx_tls_cmd(args.cluster_name, gpu_type) + print(f"UCX env: cluster={args.cluster_name!r} gpu={gpu_type!r} -> {ucx_tls_cmd!r}") ucx_tls_server_cmd = ucx_tls_cmd pytest_common_vars = "" @@ -584,6 +644,27 @@ def main(): f"export testOutputDir={test_output_dir}", ] ) + + # Cache-transceiver network precheck: runs BEFORE the real ctx/gen + # servers with the same instance topology, and reuses the exact + # $ucx_tls_cmd / $CTX_WORKER_ENV_VARS / $GEN_WORKER_ENV_VARS strings + # of the worker steps so the UCX environment matches by construction. + # Enable/kill-switch policy and timeouts live in precheck_config + # (single owner, shared with the local flow). + pcfg = _import_precheck_config(args.llm_src) + script_prefix_lines.extend( + pcfg.precheck_prefix_lines( + config, + benchmark_mode, + config_path_expr=f"$llmSrcNode/{os.path.relpath(config_yaml, args.llm_src)}", + ucx_tls_cmd=ucx_tls_cmd, + max_world=max( + hardware_config["gpus_per_ctx_server"], + hardware_config["gpus_per_gen_server"], + ), + stage_name=args.stage_name, + ) + ) srun_args_lines.extend( [ "--container-env=DISAGG_SERVING_TYPE", @@ -603,8 +684,14 @@ def main(): draft_launch_lines = remove_whitespace_lines(draft_launch_content.split("\n")) draft_launch_content = "\n".join(draft_launch_lines) + # The disagg draft calls run_cache_transceiver_precheck; splice in the gate + # function library ahead of it (single owner: precheck_config). + gate_content = "" + if runtime_mode == "disaggregated": + gate_content = pcfg.gate_library_content(args.draft_launch_sh, args.llm_src) + with open(args.launch_sh, "w") as f: - f.write(f"{script_prefix}\n{srun_args}\n{draft_launch_content}") + f.write(f"{script_prefix}\n{srun_args}\n{gate_content}{draft_launch_content}") print(f"Launch script generated at: {args.launch_sh}") print(f"Launch script:\n{script_prefix}\n{srun_args}\n{draft_launch_content}") diff --git a/jenkins/scripts/slurm_env_setup.sh b/jenkins/scripts/slurm_env_setup.sh new file mode 100644 index 000000000000..6b51c7c1b8fe --- /dev/null +++ b/jenkins/scripts/slurm_env_setup.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Shared runtime-environment setup for SLURM test steps. Sourced by +# slurm_run.sh (real test steps) AND by +# jenkins/scripts/perf/disaggregated/slurm_precheck_run.sh (the cache +# transceiver precheck), so both run with IDENTICAL library paths and +# UCX/PMIx fixups -- the precheck must observe exactly the network +# environment the real disaggregated workers will use. Keep any change here +# valid for both callers. + +slurm_setup_runtime_env() { + # Prepend the installed tensorrt_llm wheel's libs to LD_LIBRARY_PATH. + local containerPipLLMLibPath + containerPipLLMLibPath=$(pip3 show tensorrt_llm | grep "Location" | awk -F ":" '{ gsub(/ /, "", $2); print $2"/tensorrt_llm/libs"}') + containerPipLLMLibPath=$(echo "$containerPipLLMLibPath" | sed 's/[[:space:]]+/_/g') + local containerLDLibPath=$LD_LIBRARY_PATH + containerLDLibPath=$(echo "$containerLDLibPath" | sed 's/[[:space:]]+/_/g') + if [[ "$containerLDLibPath" != *"$containerPipLLMLibPath"* ]]; then + containerLDLibPath="$containerPipLLMLibPath:$containerLDLibPath" + containerLDLibPath="${containerLDLibPath%:}" + fi + export LD_LIBRARY_PATH=$containerLDLibPath + + # Slurm ENROOT/pyxis may inject UCX_TLS=tcp from the host MPI stack + # (intended for host-only MPI jobs). That disables CUDA transports and + # breaks NIXL GPU memory registration. Unset it so UCX can auto-select. + if [ "${UCX_TLS:-}" = "tcp" ]; then + unset UCX_TLS + echo "Unset UCX_TLS (cluster injected UCX_TLS=tcp)" + fi + + # Force PMIx to use the in-memory hash GDS instead of ds12/ds21 + # shared-memory. Under `srun --mpi=pmix` with the DLFW 26.04 OpenMPI + # build, the shared-memory GDS modes can fail to publish UCX worker + # addresses across nodes, producing: + # pml_ucx.c:178 Error: Failed to receive UCX worker address: Not found (-13) + # pml_ucx.c:482 Error: Failed to resolve UCX endpoint for rank N + # See https://github.com/open-mpi/ompi/issues/6981. Setting this is a + # no-op when PMIx isn't used. + export PMIX_MCA_gds=hash +} diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index e7d5f1a56d2c..25622daff391 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -45,32 +45,11 @@ else sleep 30 fi -containerPipLLMLibPath=$(pip3 show tensorrt_llm | grep "Location" | awk -F ":" '{ gsub(/ /, "", $2); print $2"/tensorrt_llm/libs"}') -containerPipLLMLibPath=$(echo "$containerPipLLMLibPath" | sed 's/[[:space:]]+/_/g') -containerLDLibPath=$LD_LIBRARY_PATH -containerLDLibPath=$(echo "$containerLDLibPath" | sed 's/[[:space:]]+/_/g') -if [[ "$containerLDLibPath" != *"$containerPipLLMLibPath"* ]]; then - containerLDLibPath="$containerPipLLMLibPath:$containerLDLibPath" - containerLDLibPath="${containerLDLibPath%:}" -fi -export LD_LIBRARY_PATH=$containerLDLibPath - -# Slurm ENROOT/pyxis may inject UCX_TLS=tcp from the host MPI stack (intended for -# host-only MPI jobs). That disables CUDA transports and breaks NIXL GPU memory -# registration. Unset it so UCX can auto-select. -if [ "${UCX_TLS:-}" = "tcp" ]; then - unset UCX_TLS - echo "Unset UCX_TLS (cluster injected UCX_TLS=tcp)" -fi - -# Force PMIx to use the in-memory hash GDS instead of ds12/ds21 shared-memory. -# Under `srun --mpi=pmix` with the DLFW 26.04 OpenMPI build, the shared-memory -# GDS modes can fail to publish UCX worker addresses across nodes, producing: -# pml_ucx.c:178 Error: Failed to receive UCX worker address: Not found (-13) -# pml_ucx.c:482 Error: Failed to resolve UCX endpoint for rank N -# See https://github.com/open-mpi/ompi/issues/6981. Setting this is a no-op -# when PMIx isn't used. -export PMIX_MCA_gds=hash +# Library path + UCX/PMIx fixups shared with the disagg cache-transceiver +# precheck (slurm_precheck_run.sh) -- keeping them in one place guarantees the +# precheck observes the same network environment as the real test steps. +source "$llmSrcNode/jenkins/scripts/slurm_env_setup.sh" +slurm_setup_runtime_env echo "Library Path:" echo "$LD_LIBRARY_PATH" env | sort diff --git a/legacy-files.txt b/legacy-files.txt index d6c8b23308aa..29648dcdbcff 100644 --- a/legacy-files.txt +++ b/legacy-files.txt @@ -1,14 +1,6 @@ .devcontainer/make_env.py .github/scripts/label_community_user.py .github/scripts/pr_checklist_check.py -benchmarks/cpp/__init__.py -benchmarks/cpp/prepare_dataset.py -benchmarks/cpp/utils/__init__.py -benchmarks/cpp/utils/convert_nemo_dataset.py -benchmarks/cpp/utils/generate_rand_loras.py -benchmarks/cpp/utils/prepare_real_data.py -benchmarks/cpp/utils/prepare_synthetic_data.py -benchmarks/cpp/utils/utils.py cpp/conanfile.py cpp/kernels/fmha_v2/conftest.py cpp/kernels/fmha_v2/fmha_test.py @@ -33,58 +25,17 @@ cpp/micro_benchmarks/gen-moe-benchmark-file.py cpp/tensorrt_llm/deep_ep/strip_nvshmem_helper.py cpp/tensorrt_llm/kernels/cutlass_kernels/python/generate_kernels.py cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/copy_cu.py -cpp/tests/resources/scripts/build_chatglm_engines.py -cpp/tests/resources/scripts/build_eagle_engines.py -cpp/tests/resources/scripts/build_enc_dec_engines.py -cpp/tests/resources/scripts/build_engines_utils.py -cpp/tests/resources/scripts/build_gpt_engines.py -cpp/tests/resources/scripts/build_gptj_engines.py -cpp/tests/resources/scripts/build_llama_engines.py -cpp/tests/resources/scripts/build_mamba_engines.py -cpp/tests/resources/scripts/build_medusa_engines.py -cpp/tests/resources/scripts/build_recurrentgemma_engines.py -cpp/tests/resources/scripts/build_redrafter_engines.py -cpp/tests/resources/scripts/generate_expected_chatglm_output.py -cpp/tests/resources/scripts/generate_expected_eagle_output.py -cpp/tests/resources/scripts/generate_expected_enc_dec_output.py -cpp/tests/resources/scripts/generate_expected_gpt_output.py -cpp/tests/resources/scripts/generate_expected_gptj_output.py -cpp/tests/resources/scripts/generate_expected_llama_output.py -cpp/tests/resources/scripts/generate_expected_mamba_output.py -cpp/tests/resources/scripts/generate_expected_medusa_output.py -cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py -cpp/tests/resources/scripts/generate_expected_redrafter_output.py -cpp/tests/resources/scripts/generate_hf_gpt_output.py cpp/tests/resources/scripts/generate_test_lora_weights.py -cpp/tests/resources/scripts/io_converter.py docs/source/conf.py docs/source/helper.py examples/apps/chat.py examples/apps/fastapi_server.py -examples/bindings/executor/example_advanced.py -examples/bindings/executor/example_basic.py -examples/bindings/executor/example_debug.py -examples/bindings/executor/example_logits_processor.py examples/disaggregated/clients/disagg_client.py examples/disaggregated/slurm/benchmark/submit.py -examples/dora/normalize_weights.py -examples/eagle/convert_checkpoint.py -examples/eval_long_context.py -examples/generate_checkpoint_config.py -examples/generate_xgrammar_tokenizer_info.py -examples/hf_lora_convert.py examples/infinitebench/args.py examples/infinitebench/compute_scores.py examples/infinitebench/construct_synthetic_dataset.py examples/infinitebench/eval_utils.py -examples/llm-api/_tensorrt_engine/llm_eagle2_decoding.py -examples/llm-api/_tensorrt_engine/llm_eagle_decoding.py -examples/llm-api/_tensorrt_engine/llm_inference_customize.py -examples/llm-api/_tensorrt_engine/llm_inference_kv_events.py -examples/llm-api/_tensorrt_engine/llm_lookahead_decoding.py -examples/llm-api/_tensorrt_engine/llm_medusa_decoding.py -examples/llm-api/_tensorrt_engine/llm_quantization.py -examples/llm-api/_tensorrt_engine/quickstart_example.py examples/llm-api/llm_guided_decoding.py examples/llm-api/llm_inference.py examples/llm-api/llm_inference_async.py @@ -104,122 +55,17 @@ examples/llm-api/quickstart_advanced.py examples/llm-api/quickstart_example.py examples/llm-api/quickstart_multimodal.py examples/llm-api/star_attention.py -examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py examples/longbench/eval_longbench_v1.py -examples/medusa/convert_checkpoint.py -examples/mmlu.py -examples/models/contrib/baichuan/convert_checkpoint.py -examples/models/contrib/bloom/convert_checkpoint.py -examples/models/contrib/chatglm-6b/tokenization_chatglm.py -examples/models/contrib/chatglm2-6b/tokenization_chatglm.py -examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py -examples/models/contrib/cogvlm/convert_checkpoint.py -examples/models/contrib/dbrx/convert_checkpoint.py -examples/models/contrib/deepseek_v1/__init__.py -examples/models/contrib/deepseek_v1/convert_checkpoint.py -examples/models/contrib/deepseek_v2/convert_checkpoint.py -examples/models/contrib/dit/convert_checkpoint.py -examples/models/contrib/dit/diffusion.py -examples/models/contrib/dit/sample.py -examples/models/contrib/dit/utils_modelopt.py -examples/models/contrib/dit/vae_decoder_trt.py -examples/models/contrib/falcon/convert_checkpoint.py -examples/models/contrib/gptj/convert_checkpoint.py -examples/models/contrib/gptneox/convert_checkpoint.py -examples/models/contrib/grok/convert_checkpoint.py -examples/models/contrib/mmdit/convert_checkpoint.py -examples/models/contrib/mmdit/sample.py -examples/models/contrib/mpt/convert_checkpoint.py -examples/models/contrib/opt/convert_checkpoint.py -examples/models/contrib/sdxl/build_sdxl_unet.py -examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py -examples/models/contrib/sdxl/run_sdxl.py -examples/models/contrib/stdit/aspect.py -examples/models/contrib/stdit/convert_checkpoint.py -examples/models/contrib/stdit/pipeline_tllm.py -examples/models/contrib/stdit/sample.py -examples/models/contrib/stdit/scheduler.py -examples/models/contrib/stdit/text_encoder.py -examples/models/contrib/stdit/utils.py -examples/models/contrib/stdit/vae.py -examples/models/contrib/stdit/video_transforms.py -examples/models/core/bert/__init__.py -examples/models/core/bert/convert_checkpoint.py -examples/models/core/bert/run.py -examples/models/core/bert/utils.py -examples/models/core/commandr/convert_checkpoint.py -examples/models/core/enc_dec/__init__.py -examples/models/core/enc_dec/convert_checkpoint.py -examples/models/core/enc_dec/helper.py -examples/models/core/enc_dec/run.py -examples/models/core/gemma/convert_checkpoint.py -examples/models/core/glm-4-9b/convert_checkpoint.py -examples/models/core/glm-4-9b/tokenization_chatglm.py -examples/models/core/gpt/convert_checkpoint.py -examples/models/core/gpt/merge_ptuning_tables.py -examples/models/core/gpt/nemo_lora_convert.py -examples/models/core/gpt/nemo_prompt_convert.py -examples/models/core/gpt/run_hf.py examples/models/core/gpt_oss/openai_chat_client_function_calling.py -examples/models/core/internlm2/convert_checkpoint.py examples/models/core/kimi_k2/kimi_k2_tool_calling_example.py -examples/models/core/llama/convert_checkpoint.py -examples/models/core/llama/summarize_long.py -examples/models/core/mamba/convert_checkpoint.py -examples/models/core/mllama/convert_checkpoint.py -examples/models/core/multimodal/__init__.py -examples/models/core/multimodal/build_multimodal_engine.py -examples/models/core/multimodal/eval.py -examples/models/core/multimodal/run.py -examples/models/core/multimodal/utils.py -examples/models/core/nemotron_nas/calibration_utils.py -examples/models/core/nemotron_nas/convert_checkpoint.py -examples/models/core/phi/convert_checkpoint.py -examples/models/core/qwen/convert_checkpoint.py -examples/models/core/qwen2audio/run.py -examples/models/core/qwen2audio/run_chat.py -examples/models/core/qwen2audio/utils.py -examples/models/core/qwenvl/run.py -examples/models/core/qwenvl/run_chat.py -examples/models/core/qwenvl/show_pic.py -examples/models/core/qwenvl/vit_onnx_trt.py -examples/models/core/recurrentgemma/convert_checkpoint.py -examples/models/core/vit/convert_checkpoint.py -examples/models/core/whisper/convert_checkpoint.py -examples/models/core/whisper/distil_whisper/convert_from_distil_whisper.py -examples/models/core/whisper/run.py -examples/models/core/whisper/tokenizer.py -examples/models/core/whisper/whisper_utils.py -examples/ngram/run_dtm_ngram.py -examples/openai_triton/manual_plugin/build.py -examples/openai_triton/manual_plugin/fmha_triton.py -examples/openai_triton/manual_plugin/plugin.py -examples/openai_triton/manual_plugin/run.py -examples/openai_triton/plugin_autogen/build_engine.py -examples/openai_triton/plugin_autogen/kernel_config.py -examples/openai_triton/plugin_autogen/run_engine.py -examples/python_plugin/build_lookup.py -examples/python_plugin/plugin_lib/__init__.py -examples/python_plugin/plugin_lib/lookup_kernel.py -examples/python_plugin/plugin_lib/lookup_plugin.py -examples/python_plugin/run_lookup.py -examples/quantization/quantize.py examples/quantization/quantize_mixed_precision_moe.py examples/ray_orchestrator/llm_inference_async_ray.py examples/ray_orchestrator/llm_inference_distributed_ray.py -examples/redrafter/convert_checkpoint.py -examples/run.py examples/scaffolding/contrib/AsyncGeneration/stream_generation_controller.py examples/scaffolding/contrib/DeepConf/run_generation.py examples/scaffolding/contrib/Dynasor/scaffolding_dynasor_run.py examples/scaffolding/contrib/TreeInference/run_mcts_example.py examples/scaffolding/contrib/TreeInference/run_tot_example.py -examples/scaffolding/contrib/mcp/e2b/e2bserver.py -examples/scaffolding/contrib/mcp/e2b/main.py -examples/scaffolding/contrib/mcp/mcptest.py -examples/scaffolding/contrib/mcp/weather/weather.py -examples/scaffolding/contrib/mcp/websearch/main.py -examples/scaffolding/contrib/mcp/websearch/websearch.py examples/scaffolding/run_basic_generation.py examples/scaffolding/run_best_of_n_with_reward.py examples/scaffolding/run_majority_vote_aime24.py @@ -229,8 +75,6 @@ examples/serve/openai_chat_client_for_multimodal.py examples/serve/openai_completion_client.py examples/serve/openai_completion_client_for_lora.py examples/serve/openai_completion_client_json_schema.py -examples/summarize.py -examples/utils.py examples/wide_ep/ep_load_balancer/generate_eplb_config.py examples/wide_ep/ep_load_balancer/report_load_statistics.py examples/wide_ep/ep_load_balancer/utils.py @@ -238,12 +82,10 @@ examples/wide_ep/slurm_scripts/process_gen_iterlog.py jenkins/scripts/mergeWaiveList.py jenkins/scripts/open_search_db.py jenkins/scripts/test_rerun.py -scripts/build_cpp_examples.py scripts/build_wheel.py scripts/check_test_list.py scripts/dco_check.py scripts/format_test_list.py -scripts/generate_duration.py scripts/generate_lock_file.py scripts/get_wheel_from_package.py scripts/git_replace.py @@ -254,7 +96,6 @@ scripts/test_to_stage_mapping.py setup.py tensorrt_llm/__init__.py tensorrt_llm/_ray_utils.py -tensorrt_llm/_tensorrt_engine/__init__.py tensorrt_llm/_torch/__init__.py tensorrt_llm/_torch/attention_backend/__init__.py tensorrt_llm/_torch/attention_backend/flashinfer.py @@ -458,7 +299,6 @@ tensorrt_llm/_torch/pyexecutor/grammar_matcher.py tensorrt_llm/_torch/pyexecutor/guided_decoder.py tensorrt_llm/_torch/pyexecutor/handle_additional_outputs.py tensorrt_llm/_torch/pyexecutor/handle_logits.py -tensorrt_llm/_torch/pyexecutor/kv_cache_connector.py tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py tensorrt_llm/_torch/pyexecutor/layerwise_nvtx_marker.py tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -496,11 +336,6 @@ tensorrt_llm/bench/benchmark/utils/__init__.py tensorrt_llm/bench/benchmark/utils/asynchronous.py tensorrt_llm/bench/benchmark/utils/general.py tensorrt_llm/bench/benchmark/utils/processes.py -tensorrt_llm/bench/build/__init__.py -tensorrt_llm/bench/build/build.py -tensorrt_llm/bench/build/dataclasses.py -tensorrt_llm/bench/build/tuning.py -tensorrt_llm/bench/build/utils.py tensorrt_llm/bench/dataclasses/__init__.py tensorrt_llm/bench/dataclasses/configuration.py tensorrt_llm/bench/dataclasses/engine.py @@ -510,13 +345,9 @@ tensorrt_llm/bench/dataclasses/reporting.py tensorrt_llm/bench/dataclasses/statistics.py tensorrt_llm/bench/utils/__init__.py tensorrt_llm/bench/utils/data.py -tensorrt_llm/builder.py tensorrt_llm/commands/__init__.py tensorrt_llm/commands/bench.py -tensorrt_llm/commands/build.py tensorrt_llm/commands/eval.py -tensorrt_llm/commands/prune.py -tensorrt_llm/commands/refit.py tensorrt_llm/commands/serve.py tensorrt_llm/evaluate/__init__.py tensorrt_llm/evaluate/cnn_dailymail.py @@ -552,23 +383,7 @@ tensorrt_llm/inputs/evs.py tensorrt_llm/inputs/multimodal.py tensorrt_llm/inputs/registry.py tensorrt_llm/inputs/utils.py -tensorrt_llm/layers/__init__.py -tensorrt_llm/layers/activation.py -tensorrt_llm/layers/attention.py -tensorrt_llm/layers/cast.py -tensorrt_llm/layers/conv.py -tensorrt_llm/layers/embedding.py -tensorrt_llm/layers/language_adapter.py -tensorrt_llm/layers/linear.py -tensorrt_llm/layers/lora.py -tensorrt_llm/layers/mlp.py -tensorrt_llm/layers/moe.py -tensorrt_llm/layers/normalization.py -tensorrt_llm/layers/pooling.py -tensorrt_llm/layers/recurrent.py -tensorrt_llm/layers/ssm.py tensorrt_llm/llmapi/__init__.py -tensorrt_llm/llmapi/build_cache.py tensorrt_llm/llmapi/disagg_utils.py tensorrt_llm/llmapi/kv_cache_type.py tensorrt_llm/llmapi/llm.py @@ -591,179 +406,17 @@ tensorrt_llm/metrics/collector.py tensorrt_llm/metrics/enums.py tensorrt_llm/models/__init__.py tensorrt_llm/models/automodel.py -tensorrt_llm/models/baichuan/__init__.py -tensorrt_llm/models/baichuan/config.py -tensorrt_llm/models/baichuan/convert.py -tensorrt_llm/models/baichuan/model.py -tensorrt_llm/models/bert/__init__.py -tensorrt_llm/models/bert/config.py -tensorrt_llm/models/bert/convert.py -tensorrt_llm/models/bert/model.py -tensorrt_llm/models/bloom/__init__.py -tensorrt_llm/models/bloom/model.py -tensorrt_llm/models/chatglm/__init__.py -tensorrt_llm/models/chatglm/config.py -tensorrt_llm/models/chatglm/convert.py -tensorrt_llm/models/chatglm/model.py -tensorrt_llm/models/clip/__init__.py -tensorrt_llm/models/clip/model.py -tensorrt_llm/models/cogvlm/__init__.py -tensorrt_llm/models/cogvlm/config.py -tensorrt_llm/models/cogvlm/convert.py -tensorrt_llm/models/cogvlm/model.py -tensorrt_llm/models/commandr/__init__.py -tensorrt_llm/models/commandr/config.py -tensorrt_llm/models/commandr/model.py tensorrt_llm/models/convert_utils.py -tensorrt_llm/models/dbrx/__init__.py -tensorrt_llm/models/dbrx/config.py -tensorrt_llm/models/dbrx/model.py -tensorrt_llm/models/deepseek_v1/__init__.py -tensorrt_llm/models/deepseek_v1/config.py -tensorrt_llm/models/deepseek_v1/convert.py -tensorrt_llm/models/deepseek_v1/model.py -tensorrt_llm/models/deepseek_v2/__init__.py -tensorrt_llm/models/deepseek_v2/config.py -tensorrt_llm/models/deepseek_v2/convert.py -tensorrt_llm/models/deepseek_v2/model.py -tensorrt_llm/models/dit/__init__.py -tensorrt_llm/models/dit/model.py -tensorrt_llm/models/eagle/__init__.py -tensorrt_llm/models/eagle/config.py -tensorrt_llm/models/eagle/model.py -tensorrt_llm/models/enc_dec/__init__.py -tensorrt_llm/models/enc_dec/model.py -tensorrt_llm/models/falcon/__init__.py -tensorrt_llm/models/falcon/config.py -tensorrt_llm/models/falcon/convert.py -tensorrt_llm/models/falcon/model.py -tensorrt_llm/models/gemma/__init__.py -tensorrt_llm/models/gemma/config.py -tensorrt_llm/models/gemma/convert.py -tensorrt_llm/models/gemma/model.py -tensorrt_llm/models/gemma/smoothquant.py -tensorrt_llm/models/gemma/utils/__init__.py -tensorrt_llm/models/gemma/utils/layers.py -tensorrt_llm/models/gemma/utils/modules.py -tensorrt_llm/models/gemma/utils/params.py -tensorrt_llm/models/gemma/utils/positional_embeddings.py -tensorrt_llm/models/gemma/utils/sampler.py -tensorrt_llm/models/gemma/utils/transformer.py -tensorrt_llm/models/gemma/weight.py -tensorrt_llm/models/generation_mixin.py -tensorrt_llm/models/gpt/__init__.py -tensorrt_llm/models/gpt/config.py -tensorrt_llm/models/gpt/convert.py -tensorrt_llm/models/gpt/model.py -tensorrt_llm/models/gptj/__init__.py -tensorrt_llm/models/gptj/config.py -tensorrt_llm/models/gptj/convert.py -tensorrt_llm/models/gptj/model.py -tensorrt_llm/models/gptneox/__init__.py -tensorrt_llm/models/gptneox/model.py -tensorrt_llm/models/grok/__init__.py -tensorrt_llm/models/grok/convert.py -tensorrt_llm/models/grok/model.py -tensorrt_llm/models/grok/weight.py -tensorrt_llm/models/llama/__init__.py -tensorrt_llm/models/llama/config.py -tensorrt_llm/models/llama/convert.py -tensorrt_llm/models/llama/model.py -tensorrt_llm/models/mamba/__init__.py -tensorrt_llm/models/mamba/config.py -tensorrt_llm/models/mamba/convert.py -tensorrt_llm/models/mamba/model.py -tensorrt_llm/models/medusa/__init__.py -tensorrt_llm/models/medusa/config.py -tensorrt_llm/models/medusa/model.py -tensorrt_llm/models/medusa/weight.py -tensorrt_llm/models/mllama/__init__.py -tensorrt_llm/models/mllama/config.py -tensorrt_llm/models/mllama/model.py -tensorrt_llm/models/mmdit_sd3/__init__.py -tensorrt_llm/models/mmdit_sd3/config.py -tensorrt_llm/models/mmdit_sd3/model.py -tensorrt_llm/models/model_weights_loader.py tensorrt_llm/models/modeling_utils.py -tensorrt_llm/models/mpt/__init__.py -tensorrt_llm/models/mpt/model.py -tensorrt_llm/models/multimodal_encoders/__init__.py -tensorrt_llm/models/multimodal_encoders/config.py -tensorrt_llm/models/multimodal_encoders/model.py -tensorrt_llm/models/nemotron_nas/__init__.py -tensorrt_llm/models/nemotron_nas/config.py -tensorrt_llm/models/nemotron_nas/convert.py -tensorrt_llm/models/nemotron_nas/layer_config.py -tensorrt_llm/models/nemotron_nas/model.py -tensorrt_llm/models/opt/__init__.py -tensorrt_llm/models/opt/model.py -tensorrt_llm/models/phi/__init__.py -tensorrt_llm/models/phi/config.py -tensorrt_llm/models/phi/convert.py -tensorrt_llm/models/phi/model.py -tensorrt_llm/models/phi3/__init__.py -tensorrt_llm/models/phi3/config.py -tensorrt_llm/models/phi3/convert.py -tensorrt_llm/models/phi3/model.py -tensorrt_llm/models/phi3/split_weights.py -tensorrt_llm/models/qwen/__init__.py -tensorrt_llm/models/qwen/config.py -tensorrt_llm/models/qwen/convert.py -tensorrt_llm/models/qwen/model.py -tensorrt_llm/models/qwen/utils.py -tensorrt_llm/models/recurrentgemma/__init__.py -tensorrt_llm/models/recurrentgemma/model.py -tensorrt_llm/models/redrafter/__init__.py -tensorrt_llm/models/redrafter/drafter.py -tensorrt_llm/models/redrafter/model.py -tensorrt_llm/models/redrafter/redrafter_helper.py -tensorrt_llm/models/stdit/__init__.py -tensorrt_llm/models/stdit/config.py -tensorrt_llm/models/stdit/model.py -tensorrt_llm/models/unet/__init__.py -tensorrt_llm/models/unet/attention.py -tensorrt_llm/models/unet/embeddings.py -tensorrt_llm/models/unet/pp/__init__.py -tensorrt_llm/models/unet/pp/attention.py -tensorrt_llm/models/unet/pp/conv2d.py -tensorrt_llm/models/unet/pp/groupnorm.py -tensorrt_llm/models/unet/pp/unet_pp.py -tensorrt_llm/models/unet/resnet.py -tensorrt_llm/models/unet/unet_2d_blocks.py -tensorrt_llm/models/unet/unet_2d_condition.py -tensorrt_llm/models/unet/weights.py -tensorrt_llm/network.py -tensorrt_llm/parameter.py -tensorrt_llm/plugin/__init__.py -tensorrt_llm/plugin/plugin.py tensorrt_llm/quantization/__init__.py tensorrt_llm/quantization/functional.py -tensorrt_llm/quantization/image_processing.py -tensorrt_llm/quantization/layers.py tensorrt_llm/quantization/mode.py -tensorrt_llm/quantization/quantize.py -tensorrt_llm/quantization/quantize_by_modelopt.py tensorrt_llm/quantization/utils/__init__.py tensorrt_llm/quantization/utils/fp4_utils.py tensorrt_llm/quantization/utils/fp8_utils.py tensorrt_llm/ray_stub.py tensorrt_llm/runtime/__init__.py -tensorrt_llm/runtime/enc_dec_model_runner.py -tensorrt_llm/runtime/generation.py -tensorrt_llm/runtime/kv_cache_manager.py -tensorrt_llm/runtime/medusa_utils.py tensorrt_llm/runtime/memory_pools/__init__.py -tensorrt_llm/runtime/memory_pools/memory_pools_allocator.py -tensorrt_llm/runtime/memory_pools/pool.py -tensorrt_llm/runtime/memory_pools/pools_kv_cache_manager.py -tensorrt_llm/runtime/model_runner.py -tensorrt_llm/runtime/model_runner_cpp.py -tensorrt_llm/runtime/multimodal_model_runner.py -tensorrt_llm/runtime/processor_wrapper/__init__.py -tensorrt_llm/runtime/processor_wrapper/mllama_processor_wrapper.py -tensorrt_llm/runtime/processor_wrapper/processor_wrapper.py -tensorrt_llm/runtime/redrafter_utils.py -tensorrt_llm/runtime/session.py tensorrt_llm/scaffolding/__init__.py tensorrt_llm/scaffolding/benchmark.py tensorrt_llm/scaffolding/contrib/AsyncGeneration/stream_generation.py @@ -818,12 +471,7 @@ tensorrt_llm/serve/tool_parser/utils.py tensorrt_llm/tokenizer/tokenizer.py tensorrt_llm/tools/__init__.py tensorrt_llm/tools/importlib_utils.py -tensorrt_llm/tools/multimodal_builder.py -tensorrt_llm/tools/onnx_utils.py tensorrt_llm/tools/plugin_gen/__init__.py -tensorrt_llm/tools/plugin_gen/core.py -tensorrt_llm/tools/plugin_gen/plugin_gen.py -tensorrt_llm/tools/plugin_gen/shape_infer.py tensorrt_llm/tools/ppl.py tensorrt_llm/tools/profiler/nsys_profile_tools/gputrc2graph.py tensorrt_llm/version.py @@ -834,7 +482,6 @@ tests/integration/defs/accuracy/scripts/collect_evaluated_accuracies.py tests/integration/defs/accuracy/scripts/compute_theta_and_thresholds.py tests/integration/defs/accuracy/test_cli_flow.py tests/integration/defs/accuracy/test_disaggregated_serving.py -tests/integration/defs/accuracy/test_llm_api.py tests/integration/defs/accuracy/test_llm_api_autodeploy.py tests/integration/defs/accuracy/test_llm_api_pytorch.py tests/integration/defs/accuracy/test_llm_api_pytorch_ray.py @@ -843,63 +490,29 @@ tests/integration/defs/common.py tests/integration/defs/conftest.py tests/integration/defs/cpp/conftest.py tests/integration/defs/cpp/cpp_common.py -tests/integration/defs/cpp/test_e2e.py tests/integration/defs/cpp/test_multi_gpu.py tests/integration/defs/cpp/test_unit_tests.py -tests/integration/defs/deterministic/mixtral_deterministic.py -tests/integration/defs/deterministic/test_mixtral_deterministic.py tests/integration/defs/disaggregated/test_auto_scaling.py tests/integration/defs/disaggregated/test_disaggregated.py tests/integration/defs/disaggregated/test_disaggregated_etcd.py tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py tests/integration/defs/disaggregated/test_workers.py -tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py tests/integration/defs/examples/run_llm_quickstart_atexit.py tests/integration/defs/examples/serve/test_serve.py tests/integration/defs/examples/serve/test_serve_negative.py tests/integration/defs/examples/test_ad_guided_decoding.py -tests/integration/defs/examples/test_bert.py -tests/integration/defs/examples/test_bindings.py -tests/integration/defs/examples/test_chatglm.py -tests/integration/defs/examples/test_commandr.py -tests/integration/defs/examples/test_draft_target_model.py -tests/integration/defs/examples/test_eagle.py -tests/integration/defs/examples/test_enc_dec.py -tests/integration/defs/examples/test_exaone.py -tests/integration/defs/examples/test_gemma.py tests/integration/defs/examples/test_gpt.py -tests/integration/defs/examples/test_gptj.py -tests/integration/defs/examples/test_granite.py -tests/integration/defs/examples/test_internlm.py -tests/integration/defs/examples/test_llama.py tests/integration/defs/examples/test_llm_api_with_mpi.py -tests/integration/defs/examples/test_mamba.py -tests/integration/defs/examples/test_medusa.py -tests/integration/defs/examples/test_mistral.py -tests/integration/defs/examples/test_mixtral.py -tests/integration/defs/examples/test_multimodal.py -tests/integration/defs/examples/test_nemotron.py -tests/integration/defs/examples/test_nemotron_nas.py -tests/integration/defs/examples/test_ngram.py -tests/integration/defs/examples/test_openai.py tests/integration/defs/examples/test_phi.py -tests/integration/defs/examples/test_qwen.py -tests/integration/defs/examples/test_qwen2audio.py -tests/integration/defs/examples/test_qwenvl.py tests/integration/defs/examples/test_ray.py -tests/integration/defs/examples/test_recurrentgemma.py -tests/integration/defs/examples/test_redrafter.py -tests/integration/defs/examples/test_whisper.py tests/integration/defs/llmapi/__init__.py tests/integration/defs/llmapi/_run_llmapi_llm.py tests/integration/defs/llmapi/test_llm_api_connector.py tests/integration/defs/llmapi/test_llm_api_qa.py -tests/integration/defs/llmapi/test_llm_e2e.py tests/integration/defs/llmapi/test_llm_examples.py tests/integration/defs/local_venv.py tests/integration/defs/perf/__init__.py tests/integration/defs/perf/allowed_configs.py -tests/integration/defs/perf/build.py tests/integration/defs/perf/create_perf_comparison_report.py tests/integration/defs/perf/data.py tests/integration/defs/perf/data_export.py @@ -920,38 +533,21 @@ tests/integration/defs/test_e2e.py tests/integration/defs/test_fmha.py tests/integration/defs/test_list_parser.py tests/integration/defs/test_list_validation.py -tests/integration/defs/test_mlpf_results.py tests/integration/defs/test_sanity.py tests/integration/defs/test_unittests.py tests/integration/defs/triton_server/__init__.py -tests/integration/defs/triton_server/build_engines.py tests/integration/defs/triton_server/common.py tests/integration/defs/triton_server/conftest.py -tests/integration/defs/triton_server/local_venv.py -tests/integration/defs/triton_server/rcca/bug_4323566/inflight_batcher_llm_client_with_end_id.py -tests/integration/defs/triton_server/runner_interface.py tests/integration/defs/triton_server/test_list_parser.py -tests/integration/defs/triton_server/test_triton.py -tests/integration/defs/triton_server/test_triton_llm.py -tests/integration/defs/triton_server/test_triton_memleak.py -tests/integration/defs/triton_server/test_triton_multi_node.py -tests/integration/defs/triton_server/test_triton_rcca.py tests/integration/defs/triton_server/trt_test_alternative.py tests/integration/defs/trt_test_alternative.py tests/integration/defs/utils/__init__.py tests/integration/defs/utils/periodic_junit.py tests/integration/defs/utils/timeout_manager.py tests/microbenchmarks/all_reduce.py -tests/microbenchmarks/build_time_benchmark.py -tests/microbenchmarks/build_time_dashboard.py tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py tests/scripts/allreduce_perf/allreduce_perf_viz.py tests/scripts/iteration_log_parser.py -tests/scripts/perf-sanity/parse_benchmark_results.py -tests/scripts/perf-sanity/run_benchmark_serve.py -tests/unittest/_torch/attention/sparse/test_dsa_indexer.py -tests/unittest/_torch/attention/sparse/test_flash_mla.py -tests/unittest/_torch/attention/sparse/test_rocketkv.py tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py tests/unittest/_torch/attention/test_attention.py tests/unittest/_torch/attention/test_attention_mla.py @@ -972,7 +568,6 @@ tests/unittest/_torch/misc/test_share_tensor.py tests/unittest/_torch/misc/test_virtual_memory.py tests/unittest/_torch/modeling/test_modeling_bert.py tests/unittest/_torch/modeling/test_modeling_clip.py -tests/unittest/_torch/modeling/test_modeling_exaone4.py tests/unittest/_torch/modeling/test_modeling_gemma3.py tests/unittest/_torch/modeling/test_modeling_gpt_oss.py tests/unittest/_torch/modeling/test_modeling_llama.py @@ -996,8 +591,6 @@ tests/unittest/_torch/modules/test_moe_load_balancer.py tests/unittest/_torch/modules/test_moe_routing.py tests/unittest/_torch/modules/test_rotary_embedding.py tests/unittest/_torch/modules/test_triton_linear.py -tests/unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py -tests/unittest/_torch/modules/tests_lora_modules/test_lora_plugin_vs_lora_op.py tests/unittest/_torch/multi_gpu/test_allreduce.py tests/unittest/_torch/multi_gpu/test_alltoall.py tests/unittest/_torch/multi_gpu/test_ar_residual_norm.py @@ -1024,24 +617,10 @@ tests/unittest/_torch/ray_orchestrator/single_gpu/test_cache_transceiver_comm.py tests/unittest/_torch/sampler/test_beam_search.py tests/unittest/_torch/sampler/test_best_of_n.py tests/unittest/_torch/sampler/test_trtllm_sampler.py -tests/unittest/_torch/speculative/test_draft_target.py -tests/unittest/_torch/speculative/test_draft_token_tree_sampling.py -tests/unittest/_torch/speculative/test_draft_token_tree_verification.py -tests/unittest/_torch/speculative/test_dynamic_spec_decode.py tests/unittest/_torch/speculative/test_eagle3.py -tests/unittest/_torch/speculative/test_kv_cache_reuse.py -tests/unittest/_torch/speculative/test_mtp.py -tests/unittest/_torch/speculative/test_ngram.py -tests/unittest/_torch/speculative/test_save_state.py -tests/unittest/_torch/speculative/test_spec_gate.py -tests/unittest/_torch/speculative/test_torch_rejection_sampling.py -tests/unittest/_torch/speculative/test_user_provided.py tests/unittest/_torch/test_connector.py tests/unittest/_torch/test_torch_multi_arange.py tests/unittest/_torch/thop/parallel/deep_gemm_tests.py -tests/unittest/_torch/thop/parallel/test_causal_conv1d_op.py -tests/unittest/_torch/thop/parallel/test_cublas_mm.py -tests/unittest/_torch/thop/parallel/test_custom_ops.py tests/unittest/_torch/thop/parallel/test_dsv3_fused_a_gemm.py tests/unittest/_torch/thop/parallel/test_dsv3_router_gemm.py tests/unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py @@ -1055,11 +634,6 @@ tests/unittest/_torch/thop/parallel/test_fp8_linear.py tests/unittest/_torch/thop/parallel/test_fp8_per_tensor_scale_tllmg_gemm.py tests/unittest/_torch/thop/parallel/test_fp8_quantize.py tests/unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py -tests/unittest/_torch/thop/parallel/test_fused_qk_norm_rope.py -tests/unittest/_torch/thop/parallel/test_logits_bitmask_op.py -tests/unittest/_torch/thop/parallel/test_mamba2_chunk_ss_update.py -tests/unittest/_torch/thop/parallel/test_mamba_conv1d_op.py -tests/unittest/_torch/thop/parallel/test_noaux_tc.py tests/unittest/_torch/thop/parallel/test_scaled_mm.py tests/unittest/_torch/thop/parallel/test_selective_scan_op.py tests/unittest/_torch/thop/parallel/test_tinygemm2.py @@ -1073,7 +647,6 @@ tests/unittest/_torch/thop/serial/test_moe.py tests/unittest/_torch/thop/serial/test_moe_alltoall.py tests/unittest/api_stability/api_stability_core.py tests/unittest/api_stability/test_llm_api.py -tests/unittest/bindings/binding_test_utils.py tests/unittest/bindings/test_bindings_moe.py tests/unittest/bindings/test_bindings_ut.py tests/unittest/bindings/test_executor_bindings.py @@ -1104,12 +677,10 @@ tests/unittest/llmapi/apps/_test_openai_chat_guided_decoding.py tests/unittest/llmapi/apps/_test_openai_chat_harmony.py tests/unittest/llmapi/apps/_test_openai_chat_multimodal.py tests/unittest/llmapi/apps/_test_openai_completions.py -tests/unittest/llmapi/apps/_test_openai_consistent_chat.py tests/unittest/llmapi/apps/_test_openai_lora.py tests/unittest/llmapi/apps/_test_openai_metrics.py tests/unittest/llmapi/apps/_test_openai_misc.py tests/unittest/llmapi/apps/_test_openai_mmencoder.py -tests/unittest/llmapi/apps/_test_openai_multi_chat.py tests/unittest/llmapi/apps/_test_openai_multi_gpu.py tests/unittest/llmapi/apps/_test_openai_multi_nodes.py tests/unittest/llmapi/apps/_test_openai_perf_metrics.py @@ -1132,15 +703,12 @@ tests/unittest/llmapi/run_llm.py tests/unittest/llmapi/run_llm_exit.py tests/unittest/llmapi/run_llm_with_postproc.py tests/unittest/llmapi/test_additional_model_outputs.py -tests/unittest/llmapi/test_build_cache.py tests/unittest/llmapi/test_executor.py tests/unittest/llmapi/test_gc_utils.py tests/unittest/llmapi/test_llm.py tests/unittest/llmapi/test_llm_args.py tests/unittest/llmapi/test_llm_download.py tests/unittest/llmapi/test_llm_kv_cache_events.py -tests/unittest/llmapi/test_llm_models.py -tests/unittest/llmapi/test_llm_multi_gpu.py tests/unittest/llmapi/test_llm_multi_gpu_pytorch.py tests/unittest/llmapi/test_llm_pytorch.py tests/unittest/llmapi/test_llm_quant.py @@ -1151,25 +719,14 @@ tests/unittest/llmapi/test_reasoning_parser.py tests/unittest/llmapi/test_serialization.py tests/unittest/llmapi/test_utils.py tests/unittest/others/__init__.py -tests/unittest/others/test_builder.py tests/unittest/others/test_convert_spec_decoding_mask_to_packed_mask.py -tests/unittest/others/test_debugging_api.py tests/unittest/others/test_exception.py tests/unittest/others/test_export.py -tests/unittest/others/test_graph_rewriter.py -tests/unittest/others/test_kv_cache_manager.py tests/unittest/others/test_kv_cache_transceiver.py tests/unittest/others/test_kv_cache_update.py -tests/unittest/others/test_layer.py -tests/unittest/others/test_leak.py tests/unittest/others/test_mapping.py -tests/unittest/others/test_model_dtype.py -tests/unittest/others/test_module.py tests/unittest/others/test_multimodal_registry.py -tests/unittest/others/test_plugins.py -tests/unittest/others/test_precision_control.py tests/unittest/others/test_pretrained_config.py -tests/unittest/others/test_session.py tests/unittest/others/test_time_breakdown.py tests/unittest/profile_utils.py tests/unittest/scaffolding/__init__.py @@ -1178,141 +735,14 @@ tests/unittest/scaffolding/test_parallel_process.py tests/unittest/scaffolding/test_scaffolding.py tests/unittest/scaffolding/test_task_collection.py tests/unittest/scaffolding/test_worker.py -tests/unittest/test_model_runner_cpp.py tests/unittest/test_pip_install.py tests/unittest/tools/__init__.py -tests/unittest/tools/plugin_gen/__init__.py -tests/unittest/tools/plugin_gen/kernel_config.py -tests/unittest/tools/plugin_gen/test_core.py -tests/unittest/tools/plugin_gen/test_plugin_gen.py -tests/unittest/tools/plugin_gen/test_shape_infer.py tests/unittest/tools/test_prepare_dataset.py tests/unittest/tools/test_test_to_stage_mapping.py -tests/unittest/trt/__init__.py -tests/unittest/trt/attention/test_bert_attention.py -tests/unittest/trt/attention/test_gpt_attention.py -tests/unittest/trt/attention/test_gpt_attention_IFB.py -tests/unittest/trt/attention/test_gpt_attention_no_cache.py -tests/unittest/trt/attention/test_sage_attention.py -tests/unittest/trt/functional/__init__.py -tests/unittest/trt/functional/test_alibi.py -tests/unittest/trt/functional/test_allreduce_norm.py -tests/unittest/trt/functional/test_allreduce_prepost_residual_norm.py -tests/unittest/trt/functional/test_arange.py -tests/unittest/trt/functional/test_argmax.py -tests/unittest/trt/functional/test_assertion.py -tests/unittest/trt/functional/test_avg_pool2d.py -tests/unittest/trt/functional/test_cast.py -tests/unittest/trt/functional/test_conv2d.py -tests/unittest/trt/functional/test_conv3d.py -tests/unittest/trt/functional/test_cos.py -tests/unittest/trt/functional/test_cumsum.py -tests/unittest/trt/functional/test_dora.py -tests/unittest/trt/functional/test_einsum.py -tests/unittest/trt/functional/test_embedding_single_gpu.py -tests/unittest/trt/functional/test_exp.py -tests/unittest/trt/functional/test_expand.py -tests/unittest/trt/functional/test_flatten.py -tests/unittest/trt/functional/test_flip.py -tests/unittest/trt/functional/test_fp4_gemm.py -tests/unittest/trt/functional/test_fp4_gemm_ootb.py -tests/unittest/trt/functional/test_gather.py -tests/unittest/trt/functional/test_gather_nd.py -tests/unittest/trt/functional/test_geglu.py -tests/unittest/trt/functional/test_gelu.py -tests/unittest/trt/functional/test_gemm_swiglu.py -tests/unittest/trt/functional/test_group_norm.py -tests/unittest/trt/functional/test_identity.py -tests/unittest/trt/functional/test_index_select.py -tests/unittest/trt/functional/test_interpolate.py -tests/unittest/trt/functional/test_logsoftmax.py -tests/unittest/trt/functional/test_lora.py -tests/unittest/trt/functional/test_low_latency_gemm.py -tests/unittest/trt/functional/test_mamba_conv1d.py -tests/unittest/trt/functional/test_masked_scatter.py -tests/unittest/trt/functional/test_masked_select.py -tests/unittest/trt/functional/test_matmul.py -tests/unittest/trt/functional/test_meshgrid2d.py -tests/unittest/trt/functional/test_moe.py -tests/unittest/trt/functional/test_nccl.py -tests/unittest/trt/functional/test_nonzero.py -tests/unittest/trt/functional/test_outer.py -tests/unittest/trt/functional/test_pad.py -tests/unittest/trt/functional/test_permute.py -tests/unittest/trt/functional/test_pp_reduce_scatter.py -tests/unittest/trt/functional/test_quant.py -tests/unittest/trt/functional/test_rearrange.py -tests/unittest/trt/functional/test_repeat.py -tests/unittest/trt/functional/test_repeat_interleave.py -tests/unittest/trt/functional/test_rg_lru.py -tests/unittest/trt/functional/test_sample.py -tests/unittest/trt/functional/test_scatter.py -tests/unittest/trt/functional/test_scatter_nd.py -tests/unittest/trt/functional/test_select.py -tests/unittest/trt/functional/test_selective_scan.py -tests/unittest/trt/functional/test_sigmoid.py -tests/unittest/trt/functional/test_silu.py -tests/unittest/trt/functional/test_sin.py -tests/unittest/trt/functional/test_slice.py -tests/unittest/trt/functional/test_softplus.py -tests/unittest/trt/functional/test_split.py -tests/unittest/trt/functional/test_squeeze.py -tests/unittest/trt/functional/test_swiglu.py -tests/unittest/trt/functional/test_topk.py -tests/unittest/trt/functional/test_transpose.py -tests/unittest/trt/functional/test_unbind.py -tests/unittest/trt/functional/test_unsqueeze.py -tests/unittest/trt/functional/test_view.py -tests/unittest/trt/functional/test_where.py -tests/unittest/trt/model/__init__.py -tests/unittest/trt/model/eagle/test_decode_draft_tokens_plugin.py -tests/unittest/trt/model/eagle/test_prepare_drafter_inputs_plugin.py -tests/unittest/trt/model/eagle/test_sample_accept_draft_tokens_plugin.py -tests/unittest/trt/model/redrafter/test_beams2tree.py -tests/unittest/trt/model/redrafter/test_draft_token.py -tests/unittest/trt/model/redrafter/test_draft_token_indices.py -tests/unittest/trt/model/redrafter/test_gather_beams.py -tests/unittest/trt/model/redrafter/test_mask.py -tests/unittest/trt/model/redrafter/test_packed_position_ids.py -tests/unittest/trt/model/redrafter/test_prefix_match_indices.py -tests/unittest/trt/model/redrafter/test_prepare_input.py -tests/unittest/trt/model/redrafter/test_process_logits.py -tests/unittest/trt/model/redrafter/test_top1.py -tests/unittest/trt/model/redrafter/test_unpack_gen_data.py -tests/unittest/trt/model/redrafter/test_validate.py -tests/unittest/trt/model/test_gpt.py -tests/unittest/trt/model/test_gpt_e2e.py -tests/unittest/trt/model/test_llama.py -tests/unittest/trt/model/test_mamba.py -tests/unittest/trt/model/test_mistral.py -tests/unittest/trt/model/test_nemotron_nas.py -tests/unittest/trt/model/test_phi.py -tests/unittest/trt/model/test_unet.py -tests/unittest/trt/model_api/test_model_api_multi_gpu.py -tests/unittest/trt/model_api/test_model_level_api.py -tests/unittest/trt/model_api/test_model_quantization.py -tests/unittest/trt/python_plugin/plugin_wrapper_utils.py -tests/unittest/trt/python_plugin/test_plugin_wrapper.py -tests/unittest/trt/quantization/__init__.py -tests/unittest/trt/quantization/_utils.py -tests/unittest/trt/quantization/test_fp8_quantization.py -tests/unittest/trt/quantization/test_fp8_rowwise_gemm.py -tests/unittest/trt/quantization/test_functional.py -tests/unittest/trt/quantization/test_mode.py -tests/unittest/trt/quantization/test_moe_weight_only_quant_matmul.py -tests/unittest/trt/quantization/test_qserve_gemm.py -tests/unittest/trt/quantization/test_quant.py -tests/unittest/trt/quantization/test_quant_layer.py -tests/unittest/trt/quantization/test_smooth_quant_gemm.py -tests/unittest/trt/quantization/test_smooth_quant_layer_norm.py -tests/unittest/trt/quantization/test_smooth_quant_rms_norm.py -tests/unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py -tests/unittest/trt/quantization/test_weight_only_quant_matmul.py tests/unittest/utils/__init__.py tests/unittest/utils/cpp_paths.py tests/unittest/utils/llm_data.py tests/unittest/utils/runtime_defaults.py -tests/unittest/utils/test_medusa_utils.py tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py tests/unittest/utils/test_util.py tests/unittest/utils/torch_ref.py diff --git a/pyproject.toml b/pyproject.toml index 99b65e954d00..a72d3c711bf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ column_limit = 80 [tool.codespell] skip = ".git,3rdparty,triton_kernels,tests/integration/test_input_files**,**.jsonl,**.json" -ignore-words-list = "rouge,inout,atleast,strat,nd,subtile,thrid,improbe,NotIn,te,iteract,anythin,tru,Tracin,vEw,dOut,indext,asend,medias" +ignore-words-list = "rouge,inout,atleast,strat,nd,subtile,thrid,improbe,NotIn,te,iteract,anythin,tru,Tracin,vEw,dOut,indext,asend,medias,thw" [tool.autoflake] in-place = true @@ -58,14 +58,6 @@ exclude = [ ".devcontainer/make_env.py", ".github/scripts/label_community_user.py", ".github/scripts/pr_checklist_check.py", - "benchmarks/cpp/__init__.py", - "benchmarks/cpp/prepare_dataset.py", - "benchmarks/cpp/utils/__init__.py", - "benchmarks/cpp/utils/convert_nemo_dataset.py", - "benchmarks/cpp/utils/generate_rand_loras.py", - "benchmarks/cpp/utils/prepare_real_data.py", - "benchmarks/cpp/utils/prepare_synthetic_data.py", - "benchmarks/cpp/utils/utils.py", "cpp/conanfile.py", "cpp/kernels/fmha_v2/conftest.py", "cpp/kernels/fmha_v2/fmha_test.py", @@ -90,58 +82,17 @@ exclude = [ "cpp/tensorrt_llm/deep_ep/strip_nvshmem_helper.py", "cpp/tensorrt_llm/kernels/cutlass_kernels/python/generate_kernels.py", "cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/copy_cu.py", - "cpp/tests/resources/scripts/build_chatglm_engines.py", - "cpp/tests/resources/scripts/build_eagle_engines.py", - "cpp/tests/resources/scripts/build_enc_dec_engines.py", - "cpp/tests/resources/scripts/build_engines_utils.py", - "cpp/tests/resources/scripts/build_gpt_engines.py", - "cpp/tests/resources/scripts/build_gptj_engines.py", - "cpp/tests/resources/scripts/build_llama_engines.py", - "cpp/tests/resources/scripts/build_mamba_engines.py", - "cpp/tests/resources/scripts/build_medusa_engines.py", - "cpp/tests/resources/scripts/build_recurrentgemma_engines.py", - "cpp/tests/resources/scripts/build_redrafter_engines.py", - "cpp/tests/resources/scripts/generate_expected_chatglm_output.py", - "cpp/tests/resources/scripts/generate_expected_eagle_output.py", - "cpp/tests/resources/scripts/generate_expected_enc_dec_output.py", - "cpp/tests/resources/scripts/generate_expected_gpt_output.py", - "cpp/tests/resources/scripts/generate_expected_gptj_output.py", - "cpp/tests/resources/scripts/generate_expected_llama_output.py", - "cpp/tests/resources/scripts/generate_expected_mamba_output.py", - "cpp/tests/resources/scripts/generate_expected_medusa_output.py", - "cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py", - "cpp/tests/resources/scripts/generate_expected_redrafter_output.py", - "cpp/tests/resources/scripts/generate_hf_gpt_output.py", "cpp/tests/resources/scripts/generate_test_lora_weights.py", - "cpp/tests/resources/scripts/io_converter.py", "docs/source/conf.py", "docs/source/helper.py", "examples/apps/chat.py", "examples/apps/fastapi_server.py", - "examples/bindings/executor/example_advanced.py", - "examples/bindings/executor/example_basic.py", - "examples/bindings/executor/example_debug.py", - "examples/bindings/executor/example_logits_processor.py", "examples/disaggregated/clients/disagg_client.py", "examples/disaggregated/slurm/benchmark/submit.py", - "examples/dora/normalize_weights.py", - "examples/eagle/convert_checkpoint.py", - "examples/eval_long_context.py", - "examples/generate_checkpoint_config.py", - "examples/generate_xgrammar_tokenizer_info.py", - "examples/hf_lora_convert.py", "examples/infinitebench/args.py", "examples/infinitebench/compute_scores.py", "examples/infinitebench/construct_synthetic_dataset.py", "examples/infinitebench/eval_utils.py", - "examples/llm-api/_tensorrt_engine/llm_eagle2_decoding.py", - "examples/llm-api/_tensorrt_engine/llm_eagle_decoding.py", - "examples/llm-api/_tensorrt_engine/llm_inference_customize.py", - "examples/llm-api/_tensorrt_engine/llm_inference_kv_events.py", - "examples/llm-api/_tensorrt_engine/llm_lookahead_decoding.py", - "examples/llm-api/_tensorrt_engine/llm_medusa_decoding.py", - "examples/llm-api/_tensorrt_engine/llm_quantization.py", - "examples/llm-api/_tensorrt_engine/quickstart_example.py", "examples/llm-api/llm_guided_decoding.py", "examples/llm-api/llm_inference.py", "examples/llm-api/llm_inference_async.py", @@ -161,122 +112,17 @@ exclude = [ "examples/llm-api/quickstart_example.py", "examples/llm-api/quickstart_multimodal.py", "examples/llm-api/star_attention.py", - "examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py", "examples/longbench/eval_longbench_v1.py", - "examples/medusa/convert_checkpoint.py", - "examples/mmlu.py", - "examples/models/contrib/baichuan/convert_checkpoint.py", - "examples/models/contrib/bloom/convert_checkpoint.py", - "examples/models/contrib/chatglm-6b/tokenization_chatglm.py", - "examples/models/contrib/chatglm2-6b/tokenization_chatglm.py", - "examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py", - "examples/models/contrib/cogvlm/convert_checkpoint.py", - "examples/models/contrib/dbrx/convert_checkpoint.py", - "examples/models/contrib/deepseek_v1/__init__.py", - "examples/models/contrib/deepseek_v1/convert_checkpoint.py", - "examples/models/contrib/deepseek_v2/convert_checkpoint.py", - "examples/models/contrib/dit/convert_checkpoint.py", - "examples/models/contrib/dit/diffusion.py", - "examples/models/contrib/dit/sample.py", - "examples/models/contrib/dit/utils_modelopt.py", - "examples/models/contrib/dit/vae_decoder_trt.py", - "examples/models/contrib/falcon/convert_checkpoint.py", - "examples/models/contrib/gptj/convert_checkpoint.py", - "examples/models/contrib/gptneox/convert_checkpoint.py", - "examples/models/contrib/grok/convert_checkpoint.py", - "examples/models/contrib/mmdit/convert_checkpoint.py", - "examples/models/contrib/mmdit/sample.py", - "examples/models/contrib/mpt/convert_checkpoint.py", - "examples/models/contrib/opt/convert_checkpoint.py", - "examples/models/contrib/sdxl/build_sdxl_unet.py", - "examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py", - "examples/models/contrib/sdxl/run_sdxl.py", - "examples/models/contrib/stdit/aspect.py", - "examples/models/contrib/stdit/convert_checkpoint.py", - "examples/models/contrib/stdit/pipeline_tllm.py", - "examples/models/contrib/stdit/sample.py", - "examples/models/contrib/stdit/scheduler.py", - "examples/models/contrib/stdit/text_encoder.py", - "examples/models/contrib/stdit/utils.py", - "examples/models/contrib/stdit/vae.py", - "examples/models/contrib/stdit/video_transforms.py", - "examples/models/core/bert/__init__.py", - "examples/models/core/bert/convert_checkpoint.py", - "examples/models/core/bert/run.py", - "examples/models/core/bert/utils.py", - "examples/models/core/commandr/convert_checkpoint.py", - "examples/models/core/enc_dec/__init__.py", - "examples/models/core/enc_dec/convert_checkpoint.py", - "examples/models/core/enc_dec/helper.py", - "examples/models/core/enc_dec/run.py", - "examples/models/core/gemma/convert_checkpoint.py", - "examples/models/core/glm-4-9b/convert_checkpoint.py", - "examples/models/core/glm-4-9b/tokenization_chatglm.py", - "examples/models/core/gpt/convert_checkpoint.py", - "examples/models/core/gpt/merge_ptuning_tables.py", - "examples/models/core/gpt/nemo_lora_convert.py", - "examples/models/core/gpt/nemo_prompt_convert.py", - "examples/models/core/gpt/run_hf.py", "examples/models/core/gpt_oss/openai_chat_client_function_calling.py", - "examples/models/core/internlm2/convert_checkpoint.py", "examples/models/core/kimi_k2/kimi_k2_tool_calling_example.py", - "examples/models/core/llama/convert_checkpoint.py", - "examples/models/core/llama/summarize_long.py", - "examples/models/core/mamba/convert_checkpoint.py", - "examples/models/core/mllama/convert_checkpoint.py", - "examples/models/core/multimodal/__init__.py", - "examples/models/core/multimodal/build_multimodal_engine.py", - "examples/models/core/multimodal/eval.py", - "examples/models/core/multimodal/run.py", - "examples/models/core/multimodal/utils.py", - "examples/models/core/nemotron_nas/calibration_utils.py", - "examples/models/core/nemotron_nas/convert_checkpoint.py", - "examples/models/core/phi/convert_checkpoint.py", - "examples/models/core/qwen/convert_checkpoint.py", - "examples/models/core/qwen2audio/run.py", - "examples/models/core/qwen2audio/run_chat.py", - "examples/models/core/qwen2audio/utils.py", - "examples/models/core/qwenvl/run.py", - "examples/models/core/qwenvl/run_chat.py", - "examples/models/core/qwenvl/show_pic.py", - "examples/models/core/qwenvl/vit_onnx_trt.py", - "examples/models/core/recurrentgemma/convert_checkpoint.py", - "examples/models/core/vit/convert_checkpoint.py", - "examples/models/core/whisper/convert_checkpoint.py", - "examples/models/core/whisper/distil_whisper/convert_from_distil_whisper.py", - "examples/models/core/whisper/run.py", - "examples/models/core/whisper/tokenizer.py", - "examples/models/core/whisper/whisper_utils.py", - "examples/ngram/run_dtm_ngram.py", - "examples/openai_triton/manual_plugin/build.py", - "examples/openai_triton/manual_plugin/fmha_triton.py", - "examples/openai_triton/manual_plugin/plugin.py", - "examples/openai_triton/manual_plugin/run.py", - "examples/openai_triton/plugin_autogen/build_engine.py", - "examples/openai_triton/plugin_autogen/kernel_config.py", - "examples/openai_triton/plugin_autogen/run_engine.py", - "examples/python_plugin/build_lookup.py", - "examples/python_plugin/plugin_lib/__init__.py", - "examples/python_plugin/plugin_lib/lookup_kernel.py", - "examples/python_plugin/plugin_lib/lookup_plugin.py", - "examples/python_plugin/run_lookup.py", - "examples/quantization/quantize.py", "examples/quantization/quantize_mixed_precision_moe.py", "examples/ray_orchestrator/llm_inference_async_ray.py", "examples/ray_orchestrator/llm_inference_distributed_ray.py", - "examples/redrafter/convert_checkpoint.py", - "examples/run.py", "examples/scaffolding/contrib/AsyncGeneration/stream_generation_controller.py", "examples/scaffolding/contrib/DeepConf/run_generation.py", "examples/scaffolding/contrib/Dynasor/scaffolding_dynasor_run.py", "examples/scaffolding/contrib/TreeInference/run_mcts_example.py", "examples/scaffolding/contrib/TreeInference/run_tot_example.py", - "examples/scaffolding/contrib/mcp/e2b/e2bserver.py", - "examples/scaffolding/contrib/mcp/e2b/main.py", - "examples/scaffolding/contrib/mcp/mcptest.py", - "examples/scaffolding/contrib/mcp/weather/weather.py", - "examples/scaffolding/contrib/mcp/websearch/main.py", - "examples/scaffolding/contrib/mcp/websearch/websearch.py", "examples/scaffolding/run_basic_generation.py", "examples/scaffolding/run_best_of_n_with_reward.py", "examples/scaffolding/run_majority_vote_aime24.py", @@ -286,8 +132,6 @@ exclude = [ "examples/serve/openai_completion_client.py", "examples/serve/openai_completion_client_for_lora.py", "examples/serve/openai_completion_client_json_schema.py", - "examples/summarize.py", - "examples/utils.py", "examples/wide_ep/ep_load_balancer/generate_eplb_config.py", "examples/wide_ep/ep_load_balancer/report_load_statistics.py", "examples/wide_ep/ep_load_balancer/utils.py", @@ -295,12 +139,10 @@ exclude = [ "jenkins/scripts/mergeWaiveList.py", "jenkins/scripts/open_search_db.py", "jenkins/scripts/test_rerun.py", - "scripts/build_cpp_examples.py", "scripts/build_wheel.py", "scripts/check_test_list.py", "scripts/dco_check.py", "scripts/format_test_list.py", - "scripts/generate_duration.py", "scripts/generate_lock_file.py", "scripts/get_wheel_from_package.py", "scripts/git_replace.py", @@ -311,7 +153,6 @@ exclude = [ "setup.py", "tensorrt_llm/__init__.py", "tensorrt_llm/_ray_utils.py", - "tensorrt_llm/_tensorrt_engine/__init__.py", "tensorrt_llm/_torch/__init__.py", "tensorrt_llm/_torch/attention_backend/__init__.py", "tensorrt_llm/_torch/attention_backend/flashinfer.py", @@ -515,7 +356,6 @@ exclude = [ "tensorrt_llm/_torch/pyexecutor/guided_decoder.py", "tensorrt_llm/_torch/pyexecutor/handle_additional_outputs.py", "tensorrt_llm/_torch/pyexecutor/handle_logits.py", - "tensorrt_llm/_torch/pyexecutor/kv_cache_connector.py", "tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py", "tensorrt_llm/_torch/pyexecutor/layerwise_nvtx_marker.py", "tensorrt_llm/_torch/pyexecutor/llm_request.py", @@ -553,11 +393,6 @@ exclude = [ "tensorrt_llm/bench/benchmark/utils/asynchronous.py", "tensorrt_llm/bench/benchmark/utils/general.py", "tensorrt_llm/bench/benchmark/utils/processes.py", - "tensorrt_llm/bench/build/__init__.py", - "tensorrt_llm/bench/build/build.py", - "tensorrt_llm/bench/build/dataclasses.py", - "tensorrt_llm/bench/build/tuning.py", - "tensorrt_llm/bench/build/utils.py", "tensorrt_llm/bench/dataclasses/__init__.py", "tensorrt_llm/bench/dataclasses/configuration.py", "tensorrt_llm/bench/dataclasses/engine.py", @@ -567,13 +402,9 @@ exclude = [ "tensorrt_llm/bench/dataclasses/statistics.py", "tensorrt_llm/bench/utils/__init__.py", "tensorrt_llm/bench/utils/data.py", - "tensorrt_llm/builder.py", "tensorrt_llm/commands/__init__.py", "tensorrt_llm/commands/bench.py", - "tensorrt_llm/commands/build.py", "tensorrt_llm/commands/eval.py", - "tensorrt_llm/commands/prune.py", - "tensorrt_llm/commands/refit.py", "tensorrt_llm/commands/serve.py", "tensorrt_llm/evaluate/__init__.py", "tensorrt_llm/evaluate/cnn_dailymail.py", @@ -609,23 +440,7 @@ exclude = [ "tensorrt_llm/inputs/multimodal.py", "tensorrt_llm/inputs/registry.py", "tensorrt_llm/inputs/utils.py", - "tensorrt_llm/layers/__init__.py", - "tensorrt_llm/layers/activation.py", - "tensorrt_llm/layers/attention.py", - "tensorrt_llm/layers/cast.py", - "tensorrt_llm/layers/conv.py", - "tensorrt_llm/layers/embedding.py", - "tensorrt_llm/layers/language_adapter.py", - "tensorrt_llm/layers/linear.py", - "tensorrt_llm/layers/lora.py", - "tensorrt_llm/layers/mlp.py", - "tensorrt_llm/layers/moe.py", - "tensorrt_llm/layers/normalization.py", - "tensorrt_llm/layers/pooling.py", - "tensorrt_llm/layers/recurrent.py", - "tensorrt_llm/layers/ssm.py", "tensorrt_llm/llmapi/__init__.py", - "tensorrt_llm/llmapi/build_cache.py", "tensorrt_llm/llmapi/disagg_utils.py", "tensorrt_llm/llmapi/kv_cache_type.py", "tensorrt_llm/llmapi/llm.py", @@ -648,179 +463,17 @@ exclude = [ "tensorrt_llm/metrics/enums.py", "tensorrt_llm/models/__init__.py", "tensorrt_llm/models/automodel.py", - "tensorrt_llm/models/baichuan/__init__.py", - "tensorrt_llm/models/baichuan/config.py", - "tensorrt_llm/models/baichuan/convert.py", - "tensorrt_llm/models/baichuan/model.py", - "tensorrt_llm/models/bert/__init__.py", - "tensorrt_llm/models/bert/config.py", - "tensorrt_llm/models/bert/convert.py", - "tensorrt_llm/models/bert/model.py", - "tensorrt_llm/models/bloom/__init__.py", - "tensorrt_llm/models/bloom/model.py", - "tensorrt_llm/models/chatglm/__init__.py", - "tensorrt_llm/models/chatglm/config.py", - "tensorrt_llm/models/chatglm/convert.py", - "tensorrt_llm/models/chatglm/model.py", - "tensorrt_llm/models/clip/__init__.py", - "tensorrt_llm/models/clip/model.py", - "tensorrt_llm/models/cogvlm/__init__.py", - "tensorrt_llm/models/cogvlm/config.py", - "tensorrt_llm/models/cogvlm/convert.py", - "tensorrt_llm/models/cogvlm/model.py", - "tensorrt_llm/models/commandr/__init__.py", - "tensorrt_llm/models/commandr/config.py", - "tensorrt_llm/models/commandr/model.py", "tensorrt_llm/models/convert_utils.py", - "tensorrt_llm/models/dbrx/__init__.py", - "tensorrt_llm/models/dbrx/config.py", - "tensorrt_llm/models/dbrx/model.py", - "tensorrt_llm/models/deepseek_v1/__init__.py", - "tensorrt_llm/models/deepseek_v1/config.py", - "tensorrt_llm/models/deepseek_v1/convert.py", - "tensorrt_llm/models/deepseek_v1/model.py", - "tensorrt_llm/models/deepseek_v2/__init__.py", - "tensorrt_llm/models/deepseek_v2/config.py", - "tensorrt_llm/models/deepseek_v2/convert.py", - "tensorrt_llm/models/deepseek_v2/model.py", - "tensorrt_llm/models/dit/__init__.py", - "tensorrt_llm/models/dit/model.py", - "tensorrt_llm/models/eagle/__init__.py", - "tensorrt_llm/models/eagle/config.py", - "tensorrt_llm/models/eagle/model.py", - "tensorrt_llm/models/enc_dec/__init__.py", - "tensorrt_llm/models/enc_dec/model.py", - "tensorrt_llm/models/falcon/__init__.py", - "tensorrt_llm/models/falcon/config.py", - "tensorrt_llm/models/falcon/convert.py", - "tensorrt_llm/models/falcon/model.py", - "tensorrt_llm/models/gemma/__init__.py", - "tensorrt_llm/models/gemma/config.py", - "tensorrt_llm/models/gemma/convert.py", - "tensorrt_llm/models/gemma/model.py", - "tensorrt_llm/models/gemma/smoothquant.py", - "tensorrt_llm/models/gemma/utils/__init__.py", - "tensorrt_llm/models/gemma/utils/layers.py", - "tensorrt_llm/models/gemma/utils/modules.py", - "tensorrt_llm/models/gemma/utils/params.py", - "tensorrt_llm/models/gemma/utils/positional_embeddings.py", - "tensorrt_llm/models/gemma/utils/sampler.py", - "tensorrt_llm/models/gemma/utils/transformer.py", - "tensorrt_llm/models/gemma/weight.py", - "tensorrt_llm/models/generation_mixin.py", - "tensorrt_llm/models/gpt/__init__.py", - "tensorrt_llm/models/gpt/config.py", - "tensorrt_llm/models/gpt/convert.py", - "tensorrt_llm/models/gpt/model.py", - "tensorrt_llm/models/gptj/__init__.py", - "tensorrt_llm/models/gptj/config.py", - "tensorrt_llm/models/gptj/convert.py", - "tensorrt_llm/models/gptj/model.py", - "tensorrt_llm/models/gptneox/__init__.py", - "tensorrt_llm/models/gptneox/model.py", - "tensorrt_llm/models/grok/__init__.py", - "tensorrt_llm/models/grok/convert.py", - "tensorrt_llm/models/grok/model.py", - "tensorrt_llm/models/grok/weight.py", - "tensorrt_llm/models/llama/__init__.py", - "tensorrt_llm/models/llama/config.py", - "tensorrt_llm/models/llama/convert.py", - "tensorrt_llm/models/llama/model.py", - "tensorrt_llm/models/mamba/__init__.py", - "tensorrt_llm/models/mamba/config.py", - "tensorrt_llm/models/mamba/convert.py", - "tensorrt_llm/models/mamba/model.py", - "tensorrt_llm/models/medusa/__init__.py", - "tensorrt_llm/models/medusa/config.py", - "tensorrt_llm/models/medusa/model.py", - "tensorrt_llm/models/medusa/weight.py", - "tensorrt_llm/models/mllama/__init__.py", - "tensorrt_llm/models/mllama/config.py", - "tensorrt_llm/models/mllama/model.py", - "tensorrt_llm/models/mmdit_sd3/__init__.py", - "tensorrt_llm/models/mmdit_sd3/config.py", - "tensorrt_llm/models/mmdit_sd3/model.py", - "tensorrt_llm/models/model_weights_loader.py", "tensorrt_llm/models/modeling_utils.py", - "tensorrt_llm/models/mpt/__init__.py", - "tensorrt_llm/models/mpt/model.py", - "tensorrt_llm/models/multimodal_encoders/__init__.py", - "tensorrt_llm/models/multimodal_encoders/config.py", - "tensorrt_llm/models/multimodal_encoders/model.py", - "tensorrt_llm/models/nemotron_nas/__init__.py", - "tensorrt_llm/models/nemotron_nas/config.py", - "tensorrt_llm/models/nemotron_nas/convert.py", - "tensorrt_llm/models/nemotron_nas/layer_config.py", - "tensorrt_llm/models/nemotron_nas/model.py", - "tensorrt_llm/models/opt/__init__.py", - "tensorrt_llm/models/opt/model.py", - "tensorrt_llm/models/phi/__init__.py", - "tensorrt_llm/models/phi/config.py", - "tensorrt_llm/models/phi/convert.py", - "tensorrt_llm/models/phi/model.py", - "tensorrt_llm/models/phi3/__init__.py", - "tensorrt_llm/models/phi3/config.py", - "tensorrt_llm/models/phi3/convert.py", - "tensorrt_llm/models/phi3/model.py", - "tensorrt_llm/models/phi3/split_weights.py", - "tensorrt_llm/models/qwen/__init__.py", - "tensorrt_llm/models/qwen/config.py", - "tensorrt_llm/models/qwen/convert.py", - "tensorrt_llm/models/qwen/model.py", - "tensorrt_llm/models/qwen/utils.py", - "tensorrt_llm/models/recurrentgemma/__init__.py", - "tensorrt_llm/models/recurrentgemma/model.py", - "tensorrt_llm/models/redrafter/__init__.py", - "tensorrt_llm/models/redrafter/drafter.py", - "tensorrt_llm/models/redrafter/model.py", - "tensorrt_llm/models/redrafter/redrafter_helper.py", - "tensorrt_llm/models/stdit/__init__.py", - "tensorrt_llm/models/stdit/config.py", - "tensorrt_llm/models/stdit/model.py", - "tensorrt_llm/models/unet/__init__.py", - "tensorrt_llm/models/unet/attention.py", - "tensorrt_llm/models/unet/embeddings.py", - "tensorrt_llm/models/unet/pp/__init__.py", - "tensorrt_llm/models/unet/pp/attention.py", - "tensorrt_llm/models/unet/pp/conv2d.py", - "tensorrt_llm/models/unet/pp/groupnorm.py", - "tensorrt_llm/models/unet/pp/unet_pp.py", - "tensorrt_llm/models/unet/resnet.py", - "tensorrt_llm/models/unet/unet_2d_blocks.py", - "tensorrt_llm/models/unet/unet_2d_condition.py", - "tensorrt_llm/models/unet/weights.py", - "tensorrt_llm/network.py", - "tensorrt_llm/parameter.py", - "tensorrt_llm/plugin/__init__.py", - "tensorrt_llm/plugin/plugin.py", "tensorrt_llm/quantization/__init__.py", "tensorrt_llm/quantization/functional.py", - "tensorrt_llm/quantization/image_processing.py", - "tensorrt_llm/quantization/layers.py", "tensorrt_llm/quantization/mode.py", - "tensorrt_llm/quantization/quantize.py", - "tensorrt_llm/quantization/quantize_by_modelopt.py", "tensorrt_llm/quantization/utils/__init__.py", "tensorrt_llm/quantization/utils/fp4_utils.py", "tensorrt_llm/quantization/utils/fp8_utils.py", "tensorrt_llm/ray_stub.py", "tensorrt_llm/runtime/__init__.py", - "tensorrt_llm/runtime/enc_dec_model_runner.py", - "tensorrt_llm/runtime/generation.py", - "tensorrt_llm/runtime/kv_cache_manager.py", - "tensorrt_llm/runtime/medusa_utils.py", "tensorrt_llm/runtime/memory_pools/__init__.py", - "tensorrt_llm/runtime/memory_pools/memory_pools_allocator.py", - "tensorrt_llm/runtime/memory_pools/pool.py", - "tensorrt_llm/runtime/memory_pools/pools_kv_cache_manager.py", - "tensorrt_llm/runtime/model_runner.py", - "tensorrt_llm/runtime/model_runner_cpp.py", - "tensorrt_llm/runtime/multimodal_model_runner.py", - "tensorrt_llm/runtime/processor_wrapper/__init__.py", - "tensorrt_llm/runtime/processor_wrapper/mllama_processor_wrapper.py", - "tensorrt_llm/runtime/processor_wrapper/processor_wrapper.py", - "tensorrt_llm/runtime/redrafter_utils.py", - "tensorrt_llm/runtime/session.py", "tensorrt_llm/scaffolding/__init__.py", "tensorrt_llm/scaffolding/benchmark.py", "tensorrt_llm/scaffolding/contrib/AsyncGeneration/stream_generation.py", @@ -875,12 +528,7 @@ exclude = [ "tensorrt_llm/tokenizer/tokenizer.py", "tensorrt_llm/tools/__init__.py", "tensorrt_llm/tools/importlib_utils.py", - "tensorrt_llm/tools/multimodal_builder.py", - "tensorrt_llm/tools/onnx_utils.py", "tensorrt_llm/tools/plugin_gen/__init__.py", - "tensorrt_llm/tools/plugin_gen/core.py", - "tensorrt_llm/tools/plugin_gen/plugin_gen.py", - "tensorrt_llm/tools/plugin_gen/shape_infer.py", "tensorrt_llm/tools/ppl.py", "tensorrt_llm/tools/profiler/nsys_profile_tools/gputrc2graph.py", "tensorrt_llm/version.py", @@ -891,7 +539,6 @@ exclude = [ "tests/integration/defs/accuracy/scripts/compute_theta_and_thresholds.py", "tests/integration/defs/accuracy/test_cli_flow.py", "tests/integration/defs/accuracy/test_disaggregated_serving.py", - "tests/integration/defs/accuracy/test_llm_api.py", "tests/integration/defs/accuracy/test_llm_api_autodeploy.py", "tests/integration/defs/accuracy/test_llm_api_pytorch.py", "tests/integration/defs/accuracy/test_llm_api_pytorch_ray.py", @@ -900,63 +547,29 @@ exclude = [ "tests/integration/defs/conftest.py", "tests/integration/defs/cpp/conftest.py", "tests/integration/defs/cpp/cpp_common.py", - "tests/integration/defs/cpp/test_e2e.py", "tests/integration/defs/cpp/test_multi_gpu.py", "tests/integration/defs/cpp/test_unit_tests.py", - "tests/integration/defs/deterministic/mixtral_deterministic.py", - "tests/integration/defs/deterministic/test_mixtral_deterministic.py", "tests/integration/defs/disaggregated/test_auto_scaling.py", "tests/integration/defs/disaggregated/test_disaggregated.py", "tests/integration/defs/disaggregated/test_disaggregated_etcd.py", "tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py", "tests/integration/defs/disaggregated/test_workers.py", - "tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py", "tests/integration/defs/examples/run_llm_quickstart_atexit.py", "tests/integration/defs/examples/serve/test_serve.py", "tests/integration/defs/examples/serve/test_serve_negative.py", "tests/integration/defs/examples/test_ad_guided_decoding.py", - "tests/integration/defs/examples/test_bert.py", - "tests/integration/defs/examples/test_bindings.py", - "tests/integration/defs/examples/test_chatglm.py", - "tests/integration/defs/examples/test_commandr.py", - "tests/integration/defs/examples/test_draft_target_model.py", - "tests/integration/defs/examples/test_eagle.py", - "tests/integration/defs/examples/test_enc_dec.py", - "tests/integration/defs/examples/test_exaone.py", - "tests/integration/defs/examples/test_gemma.py", "tests/integration/defs/examples/test_gpt.py", - "tests/integration/defs/examples/test_gptj.py", - "tests/integration/defs/examples/test_granite.py", - "tests/integration/defs/examples/test_internlm.py", - "tests/integration/defs/examples/test_llama.py", "tests/integration/defs/examples/test_llm_api_with_mpi.py", - "tests/integration/defs/examples/test_mamba.py", - "tests/integration/defs/examples/test_medusa.py", - "tests/integration/defs/examples/test_mistral.py", - "tests/integration/defs/examples/test_mixtral.py", - "tests/integration/defs/examples/test_multimodal.py", - "tests/integration/defs/examples/test_nemotron.py", - "tests/integration/defs/examples/test_nemotron_nas.py", - "tests/integration/defs/examples/test_ngram.py", - "tests/integration/defs/examples/test_openai.py", "tests/integration/defs/examples/test_phi.py", - "tests/integration/defs/examples/test_qwen.py", - "tests/integration/defs/examples/test_qwen2audio.py", - "tests/integration/defs/examples/test_qwenvl.py", "tests/integration/defs/examples/test_ray.py", - "tests/integration/defs/examples/test_recurrentgemma.py", - "tests/integration/defs/examples/test_redrafter.py", - "tests/integration/defs/examples/test_whisper.py", "tests/integration/defs/llmapi/__init__.py", "tests/integration/defs/llmapi/_run_llmapi_llm.py", "tests/integration/defs/llmapi/test_llm_api_connector.py", "tests/integration/defs/llmapi/test_llm_api_qa.py", - "tests/integration/defs/llmapi/test_llm_e2e.py", "tests/integration/defs/llmapi/test_llm_examples.py", "tests/integration/defs/local_venv.py", "tests/integration/defs/perf/__init__.py", "tests/integration/defs/perf/allowed_configs.py", - "tests/integration/defs/perf/build.py", "tests/integration/defs/perf/create_perf_comparison_report.py", "tests/integration/defs/perf/data.py", "tests/integration/defs/perf/data_export.py", @@ -977,38 +590,21 @@ exclude = [ "tests/integration/defs/test_fmha.py", "tests/integration/defs/test_list_parser.py", "tests/integration/defs/test_list_validation.py", - "tests/integration/defs/test_mlpf_results.py", "tests/integration/defs/test_sanity.py", "tests/integration/defs/test_unittests.py", "tests/integration/defs/triton_server/__init__.py", - "tests/integration/defs/triton_server/build_engines.py", "tests/integration/defs/triton_server/common.py", "tests/integration/defs/triton_server/conftest.py", - "tests/integration/defs/triton_server/local_venv.py", - "tests/integration/defs/triton_server/rcca/bug_4323566/inflight_batcher_llm_client_with_end_id.py", - "tests/integration/defs/triton_server/runner_interface.py", "tests/integration/defs/triton_server/test_list_parser.py", - "tests/integration/defs/triton_server/test_triton.py", - "tests/integration/defs/triton_server/test_triton_llm.py", - "tests/integration/defs/triton_server/test_triton_memleak.py", - "tests/integration/defs/triton_server/test_triton_multi_node.py", - "tests/integration/defs/triton_server/test_triton_rcca.py", "tests/integration/defs/triton_server/trt_test_alternative.py", "tests/integration/defs/trt_test_alternative.py", "tests/integration/defs/utils/__init__.py", "tests/integration/defs/utils/periodic_junit.py", "tests/integration/defs/utils/timeout_manager.py", "tests/microbenchmarks/all_reduce.py", - "tests/microbenchmarks/build_time_benchmark.py", - "tests/microbenchmarks/build_time_dashboard.py", "tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py", "tests/scripts/allreduce_perf/allreduce_perf_viz.py", "tests/scripts/iteration_log_parser.py", - "tests/scripts/perf-sanity/parse_benchmark_results.py", - "tests/scripts/perf-sanity/run_benchmark_serve.py", - "tests/unittest/_torch/attention/sparse/test_dsa_indexer.py", - "tests/unittest/_torch/attention/sparse/test_flash_mla.py", - "tests/unittest/_torch/attention/sparse/test_rocketkv.py", "tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py", "tests/unittest/_torch/attention/test_attention.py", "tests/unittest/_torch/attention/test_attention_mla.py", @@ -1029,7 +625,6 @@ exclude = [ "tests/unittest/_torch/misc/test_virtual_memory.py", "tests/unittest/_torch/modeling/test_modeling_bert.py", "tests/unittest/_torch/modeling/test_modeling_clip.py", - "tests/unittest/_torch/modeling/test_modeling_exaone4.py", "tests/unittest/_torch/modeling/test_modeling_gemma3.py", "tests/unittest/_torch/modeling/test_modeling_gpt_oss.py", "tests/unittest/_torch/modeling/test_modeling_llama.py", @@ -1053,8 +648,6 @@ exclude = [ "tests/unittest/_torch/modules/test_moe_routing.py", "tests/unittest/_torch/modules/test_rotary_embedding.py", "tests/unittest/_torch/modules/test_triton_linear.py", - "tests/unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py", - "tests/unittest/_torch/modules/tests_lora_modules/test_lora_plugin_vs_lora_op.py", "tests/unittest/_torch/multi_gpu/test_allreduce.py", "tests/unittest/_torch/multi_gpu/test_alltoall.py", "tests/unittest/_torch/multi_gpu/test_ar_residual_norm.py", @@ -1081,24 +674,10 @@ exclude = [ "tests/unittest/_torch/sampler/test_beam_search.py", "tests/unittest/_torch/sampler/test_best_of_n.py", "tests/unittest/_torch/sampler/test_trtllm_sampler.py", - "tests/unittest/_torch/speculative/test_draft_target.py", - "tests/unittest/_torch/speculative/test_draft_token_tree_sampling.py", - "tests/unittest/_torch/speculative/test_draft_token_tree_verification.py", - "tests/unittest/_torch/speculative/test_dynamic_spec_decode.py", "tests/unittest/_torch/speculative/test_eagle3.py", - "tests/unittest/_torch/speculative/test_kv_cache_reuse.py", - "tests/unittest/_torch/speculative/test_mtp.py", - "tests/unittest/_torch/speculative/test_ngram.py", - "tests/unittest/_torch/speculative/test_save_state.py", - "tests/unittest/_torch/speculative/test_spec_gate.py", - "tests/unittest/_torch/speculative/test_torch_rejection_sampling.py", - "tests/unittest/_torch/speculative/test_user_provided.py", "tests/unittest/_torch/test_connector.py", "tests/unittest/_torch/test_torch_multi_arange.py", "tests/unittest/_torch/thop/parallel/deep_gemm_tests.py", - "tests/unittest/_torch/thop/parallel/test_causal_conv1d_op.py", - "tests/unittest/_torch/thop/parallel/test_cublas_mm.py", - "tests/unittest/_torch/thop/parallel/test_custom_ops.py", "tests/unittest/_torch/thop/parallel/test_dsv3_fused_a_gemm.py", "tests/unittest/_torch/thop/parallel/test_dsv3_router_gemm.py", "tests/unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py", @@ -1112,11 +691,6 @@ exclude = [ "tests/unittest/_torch/thop/parallel/test_fp8_per_tensor_scale_tllmg_gemm.py", "tests/unittest/_torch/thop/parallel/test_fp8_quantize.py", "tests/unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py", - "tests/unittest/_torch/thop/parallel/test_fused_qk_norm_rope.py", - "tests/unittest/_torch/thop/parallel/test_logits_bitmask_op.py", - "tests/unittest/_torch/thop/parallel/test_mamba2_chunk_ss_update.py", - "tests/unittest/_torch/thop/parallel/test_mamba_conv1d_op.py", - "tests/unittest/_torch/thop/parallel/test_noaux_tc.py", "tests/unittest/_torch/thop/parallel/test_scaled_mm.py", "tests/unittest/_torch/thop/parallel/test_selective_scan_op.py", "tests/unittest/_torch/thop/parallel/test_tinygemm2.py", @@ -1130,7 +704,6 @@ exclude = [ "tests/unittest/_torch/thop/serial/test_moe_alltoall.py", "tests/unittest/api_stability/api_stability_core.py", "tests/unittest/api_stability/test_llm_api.py", - "tests/unittest/bindings/binding_test_utils.py", "tests/unittest/bindings/test_bindings_moe.py", "tests/unittest/bindings/test_bindings_ut.py", "tests/unittest/bindings/test_executor_bindings.py", @@ -1161,12 +734,10 @@ exclude = [ "tests/unittest/llmapi/apps/_test_openai_chat_harmony.py", "tests/unittest/llmapi/apps/_test_openai_chat_multimodal.py", "tests/unittest/llmapi/apps/_test_openai_completions.py", - "tests/unittest/llmapi/apps/_test_openai_consistent_chat.py", "tests/unittest/llmapi/apps/_test_openai_lora.py", "tests/unittest/llmapi/apps/_test_openai_metrics.py", "tests/unittest/llmapi/apps/_test_openai_misc.py", "tests/unittest/llmapi/apps/_test_openai_mmencoder.py", - "tests/unittest/llmapi/apps/_test_openai_multi_chat.py", "tests/unittest/llmapi/apps/_test_openai_multi_gpu.py", "tests/unittest/llmapi/apps/_test_openai_multi_nodes.py", "tests/unittest/llmapi/apps/_test_openai_perf_metrics.py", @@ -1189,15 +760,12 @@ exclude = [ "tests/unittest/llmapi/run_llm_exit.py", "tests/unittest/llmapi/run_llm_with_postproc.py", "tests/unittest/llmapi/test_additional_model_outputs.py", - "tests/unittest/llmapi/test_build_cache.py", "tests/unittest/llmapi/test_executor.py", "tests/unittest/llmapi/test_gc_utils.py", "tests/unittest/llmapi/test_llm.py", "tests/unittest/llmapi/test_llm_args.py", "tests/unittest/llmapi/test_llm_download.py", "tests/unittest/llmapi/test_llm_kv_cache_events.py", - "tests/unittest/llmapi/test_llm_models.py", - "tests/unittest/llmapi/test_llm_multi_gpu.py", "tests/unittest/llmapi/test_llm_multi_gpu_pytorch.py", "tests/unittest/llmapi/test_llm_pytorch.py", "tests/unittest/llmapi/test_llm_quant.py", @@ -1208,25 +776,14 @@ exclude = [ "tests/unittest/llmapi/test_serialization.py", "tests/unittest/llmapi/test_utils.py", "tests/unittest/others/__init__.py", - "tests/unittest/others/test_builder.py", "tests/unittest/others/test_convert_spec_decoding_mask_to_packed_mask.py", - "tests/unittest/others/test_debugging_api.py", "tests/unittest/others/test_exception.py", "tests/unittest/others/test_export.py", - "tests/unittest/others/test_graph_rewriter.py", - "tests/unittest/others/test_kv_cache_manager.py", "tests/unittest/others/test_kv_cache_transceiver.py", "tests/unittest/others/test_kv_cache_update.py", - "tests/unittest/others/test_layer.py", - "tests/unittest/others/test_leak.py", "tests/unittest/others/test_mapping.py", - "tests/unittest/others/test_model_dtype.py", - "tests/unittest/others/test_module.py", "tests/unittest/others/test_multimodal_registry.py", - "tests/unittest/others/test_plugins.py", - "tests/unittest/others/test_precision_control.py", "tests/unittest/others/test_pretrained_config.py", - "tests/unittest/others/test_session.py", "tests/unittest/others/test_time_breakdown.py", "tests/unittest/profile_utils.py", "tests/unittest/scaffolding/__init__.py", @@ -1235,141 +792,14 @@ exclude = [ "tests/unittest/scaffolding/test_scaffolding.py", "tests/unittest/scaffolding/test_task_collection.py", "tests/unittest/scaffolding/test_worker.py", - "tests/unittest/test_model_runner_cpp.py", "tests/unittest/test_pip_install.py", "tests/unittest/tools/__init__.py", - "tests/unittest/tools/plugin_gen/__init__.py", - "tests/unittest/tools/plugin_gen/kernel_config.py", - "tests/unittest/tools/plugin_gen/test_core.py", - "tests/unittest/tools/plugin_gen/test_plugin_gen.py", - "tests/unittest/tools/plugin_gen/test_shape_infer.py", "tests/unittest/tools/test_prepare_dataset.py", "tests/unittest/tools/test_test_to_stage_mapping.py", - "tests/unittest/trt/__init__.py", - "tests/unittest/trt/attention/test_bert_attention.py", - "tests/unittest/trt/attention/test_gpt_attention.py", - "tests/unittest/trt/attention/test_gpt_attention_IFB.py", - "tests/unittest/trt/attention/test_gpt_attention_no_cache.py", - "tests/unittest/trt/attention/test_sage_attention.py", - "tests/unittest/trt/functional/__init__.py", - "tests/unittest/trt/functional/test_alibi.py", - "tests/unittest/trt/functional/test_allreduce_norm.py", - "tests/unittest/trt/functional/test_allreduce_prepost_residual_norm.py", - "tests/unittest/trt/functional/test_arange.py", - "tests/unittest/trt/functional/test_argmax.py", - "tests/unittest/trt/functional/test_assertion.py", - "tests/unittest/trt/functional/test_avg_pool2d.py", - "tests/unittest/trt/functional/test_cast.py", - "tests/unittest/trt/functional/test_conv2d.py", - "tests/unittest/trt/functional/test_conv3d.py", - "tests/unittest/trt/functional/test_cos.py", - "tests/unittest/trt/functional/test_cumsum.py", - "tests/unittest/trt/functional/test_dora.py", - "tests/unittest/trt/functional/test_einsum.py", - "tests/unittest/trt/functional/test_embedding_single_gpu.py", - "tests/unittest/trt/functional/test_exp.py", - "tests/unittest/trt/functional/test_expand.py", - "tests/unittest/trt/functional/test_flatten.py", - "tests/unittest/trt/functional/test_flip.py", - "tests/unittest/trt/functional/test_fp4_gemm.py", - "tests/unittest/trt/functional/test_fp4_gemm_ootb.py", - "tests/unittest/trt/functional/test_gather.py", - "tests/unittest/trt/functional/test_gather_nd.py", - "tests/unittest/trt/functional/test_geglu.py", - "tests/unittest/trt/functional/test_gelu.py", - "tests/unittest/trt/functional/test_gemm_swiglu.py", - "tests/unittest/trt/functional/test_group_norm.py", - "tests/unittest/trt/functional/test_identity.py", - "tests/unittest/trt/functional/test_index_select.py", - "tests/unittest/trt/functional/test_interpolate.py", - "tests/unittest/trt/functional/test_logsoftmax.py", - "tests/unittest/trt/functional/test_lora.py", - "tests/unittest/trt/functional/test_low_latency_gemm.py", - "tests/unittest/trt/functional/test_mamba_conv1d.py", - "tests/unittest/trt/functional/test_masked_scatter.py", - "tests/unittest/trt/functional/test_masked_select.py", - "tests/unittest/trt/functional/test_matmul.py", - "tests/unittest/trt/functional/test_meshgrid2d.py", - "tests/unittest/trt/functional/test_moe.py", - "tests/unittest/trt/functional/test_nccl.py", - "tests/unittest/trt/functional/test_nonzero.py", - "tests/unittest/trt/functional/test_outer.py", - "tests/unittest/trt/functional/test_pad.py", - "tests/unittest/trt/functional/test_permute.py", - "tests/unittest/trt/functional/test_pp_reduce_scatter.py", - "tests/unittest/trt/functional/test_quant.py", - "tests/unittest/trt/functional/test_rearrange.py", - "tests/unittest/trt/functional/test_repeat.py", - "tests/unittest/trt/functional/test_repeat_interleave.py", - "tests/unittest/trt/functional/test_rg_lru.py", - "tests/unittest/trt/functional/test_sample.py", - "tests/unittest/trt/functional/test_scatter.py", - "tests/unittest/trt/functional/test_scatter_nd.py", - "tests/unittest/trt/functional/test_select.py", - "tests/unittest/trt/functional/test_selective_scan.py", - "tests/unittest/trt/functional/test_sigmoid.py", - "tests/unittest/trt/functional/test_silu.py", - "tests/unittest/trt/functional/test_sin.py", - "tests/unittest/trt/functional/test_slice.py", - "tests/unittest/trt/functional/test_softplus.py", - "tests/unittest/trt/functional/test_split.py", - "tests/unittest/trt/functional/test_squeeze.py", - "tests/unittest/trt/functional/test_swiglu.py", - "tests/unittest/trt/functional/test_topk.py", - "tests/unittest/trt/functional/test_transpose.py", - "tests/unittest/trt/functional/test_unbind.py", - "tests/unittest/trt/functional/test_unsqueeze.py", - "tests/unittest/trt/functional/test_view.py", - "tests/unittest/trt/functional/test_where.py", - "tests/unittest/trt/model/__init__.py", - "tests/unittest/trt/model/eagle/test_decode_draft_tokens_plugin.py", - "tests/unittest/trt/model/eagle/test_prepare_drafter_inputs_plugin.py", - "tests/unittest/trt/model/eagle/test_sample_accept_draft_tokens_plugin.py", - "tests/unittest/trt/model/redrafter/test_beams2tree.py", - "tests/unittest/trt/model/redrafter/test_draft_token.py", - "tests/unittest/trt/model/redrafter/test_draft_token_indices.py", - "tests/unittest/trt/model/redrafter/test_gather_beams.py", - "tests/unittest/trt/model/redrafter/test_mask.py", - "tests/unittest/trt/model/redrafter/test_packed_position_ids.py", - "tests/unittest/trt/model/redrafter/test_prefix_match_indices.py", - "tests/unittest/trt/model/redrafter/test_prepare_input.py", - "tests/unittest/trt/model/redrafter/test_process_logits.py", - "tests/unittest/trt/model/redrafter/test_top1.py", - "tests/unittest/trt/model/redrafter/test_unpack_gen_data.py", - "tests/unittest/trt/model/redrafter/test_validate.py", - "tests/unittest/trt/model/test_gpt.py", - "tests/unittest/trt/model/test_gpt_e2e.py", - "tests/unittest/trt/model/test_llama.py", - "tests/unittest/trt/model/test_mamba.py", - "tests/unittest/trt/model/test_mistral.py", - "tests/unittest/trt/model/test_nemotron_nas.py", - "tests/unittest/trt/model/test_phi.py", - "tests/unittest/trt/model/test_unet.py", - "tests/unittest/trt/model_api/test_model_api_multi_gpu.py", - "tests/unittest/trt/model_api/test_model_level_api.py", - "tests/unittest/trt/model_api/test_model_quantization.py", - "tests/unittest/trt/python_plugin/plugin_wrapper_utils.py", - "tests/unittest/trt/python_plugin/test_plugin_wrapper.py", - "tests/unittest/trt/quantization/__init__.py", - "tests/unittest/trt/quantization/_utils.py", - "tests/unittest/trt/quantization/test_fp8_quantization.py", - "tests/unittest/trt/quantization/test_fp8_rowwise_gemm.py", - "tests/unittest/trt/quantization/test_functional.py", - "tests/unittest/trt/quantization/test_mode.py", - "tests/unittest/trt/quantization/test_moe_weight_only_quant_matmul.py", - "tests/unittest/trt/quantization/test_qserve_gemm.py", - "tests/unittest/trt/quantization/test_quant.py", - "tests/unittest/trt/quantization/test_quant_layer.py", - "tests/unittest/trt/quantization/test_smooth_quant_gemm.py", - "tests/unittest/trt/quantization/test_smooth_quant_layer_norm.py", - "tests/unittest/trt/quantization/test_smooth_quant_rms_norm.py", - "tests/unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py", - "tests/unittest/trt/quantization/test_weight_only_quant_matmul.py", "tests/unittest/utils/__init__.py", "tests/unittest/utils/cpp_paths.py", "tests/unittest/utils/llm_data.py", "tests/unittest/utils/runtime_defaults.py", - "tests/unittest/utils/test_medusa_utils.py", "tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py", "tests/unittest/utils/test_util.py", "tests/unittest/utils/torch_ref.py", @@ -1502,6 +932,7 @@ module = [ "tests.*", "_torch.*", # tests/unittest/_torch/*, cf. mypy_path "test_torch_sampler", # tests/unittest/_torch/sampler/..., cf. mypy_path + "test_token_ban", # tests/unittest/_torch/sampler/..., cf. mypy_path "test_beam_search", # tests/unittest/_torch/sampler/..., cf. mypy_path "test_beam_search_util", # tests/unittest/_torch/sampler/..., cf. mypy_path ] diff --git a/requirements-dev.txt b/requirements-dev.txt index 8c6766a590eb..d5dcc3a2bf7e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,8 @@ -r requirements.txt +# Pin the exact diffusers version VisualGen LPIPS tests run against; install it +# before pytest collection so already-imported Diffusers modules match the +# package files. 0.39.0 matches the runtime floor in requirements.txt. +diffusers==0.39.0 boto3 einops lpips diff --git a/requirements.txt b/requirements.txt index 9555bf2ac80c..236a4e3d365d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,14 +4,13 @@ accelerate>=1.7.0 build colored cuda-python>=13 -diffusers>=0.37.1 +# FlowMatchEuler respects a supplied generator starting in 0.39.0 (huggingface/diffusers#13678). +diffusers>=0.39.0 ftfy lark lazy_loader~=0.5 mpi4py numpy>=2.0.0,<2.4 # numba 0.63.1 requires numpy<2.4 -onnx>=1.21.0 -onnx_graphsurgeon>=0.5.2 graphviz openai polygraphy @@ -22,14 +21,14 @@ pandas h5py==3.12.1 StrEnum sentencepiece>=0.1.99 -tensorrt~=10.16.1 # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 uses 2.12.0a0. torch>=2.11.0,<=2.13.0a0 torchvision -nvidia-modelopt[torch]~=0.37.0 # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 uses 2.30.4 # torch 2.11.0+cu130 depends on nvidia-nccl-cu13==2.28.9 nvidia-nccl-cu13>=2.28.9,<=2.30.4 +# NcclEP uses nccl4py's nccl.ep package without changing the NCCL wheel constraint. +nccl4py>=0.3.1,<0.4 nvidia-cuda-nvrtc transformers==5.5.4 prometheus_client @@ -58,7 +57,7 @@ ordered-set peft>=0.18.1,<0.19.0 patchelf einops -flashinfer-python==0.6.14 +flashinfer-python==0.6.15 xgrammar==0.1.32 llguidance==0.7.29 jsonschema @@ -76,10 +75,12 @@ blobfile openai-harmony==0.0.4 nvidia-cutlass-dsl[cu13]==4.5.0; python_version >= "3.10" nvidia-matmul-heuristics==0.1.0.27; python_version >= "3.10" # analytic GEMM heuristics for CuTe DSL autotuner tactic pruning +quack-kernels>=0.2.10; python_version >= "3.10" # required for MinimaxM3 MSA +jinja2 # required for MinimaxM3 MSA plotly numexpr partial_json_parser -mcp +mcp<2.0.0 # 2.0.0 removed mcp.server.fastmcp; scaffolding still uses the 1.x APIs apache-tvm-ffi==0.1.6 # used for reduce nvidia-cutlass-dsl host overhead torch-c-dlpack-ext==0.1.3 # used for reduce nvidia-cutlass-dsl host overhead, optional package for improved torch tensor calling perf flash-attn-4==4.0.0b11 diff --git a/ruff-legacy-baseline.json b/ruff-legacy-baseline.json index ee17a68362c2..eae44689bafc 100644 --- a/ruff-legacy-baseline.json +++ b/ruff-legacy-baseline.json @@ -1,22 +1,12 @@ { "_meta": { "generated_by": "scripts/legacy_utils.py lint-update-violations", - "total_violations": 2917, - "total_files": 303 + "total_violations": 2096, + "total_files": 224 }, ".github/scripts/label_community_user.py": { "D212": 1 }, - "benchmarks/cpp/utils/convert_nemo_dataset.py": { - "E741": 1 - }, - "benchmarks/cpp/utils/prepare_real_data.py": { - "D202": 1, - "D205": 1, - "D410": 7, - "D411": 8, - "D415": 1 - }, "cpp/kernels/fmha_v2/setup.py": { "E712": 27, "E731": 1 @@ -80,23 +70,6 @@ "D212": 1, "D300": 1 }, - "examples/bindings/executor/example_advanced.py": { - "E402": 1 - }, - "examples/bindings/executor/example_logits_processor.py": { - "D200": 1, - "D212": 1, - "D415": 1 - }, - "examples/dora/normalize_weights.py": { - "D200": 3, - "D202": 1, - "D205": 1, - "D212": 4 - }, - "examples/eval_long_context.py": { - "E711": 2 - }, "examples/infinitebench/compute_scores.py": { "D200": 1, "D205": 1, @@ -144,111 +117,10 @@ "D212": 1, "F821": 1 }, - "examples/mmlu.py": { - "D205": 1, - "D415": 1, - "E741": 1 - }, - "examples/models/contrib/chatglm-6b/tokenization_chatglm.py": { - "D205": 4, - "D210": 4, - "D212": 7, - "D301": 2, - "D415": 4 - }, - "examples/models/contrib/chatglm2-6b/tokenization_chatglm.py": { - "D205": 1, - "D210": 3, - "D212": 3, - "D415": 3 - }, - "examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py": { - "D205": 1, - "D210": 3, - "D212": 3, - "D415": 3 - }, - "examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py": { - "D202": 1, - "D205": 6, - "D212": 10, - "D414": 1, - "D415": 2 - }, - "examples/models/core/multimodal/run.py": { - "E731": 1 - }, - "examples/models/core/qwen2audio/run.py": { - "D200": 1, - "D212": 1, - "D415": 1, - "E722": 1 - }, - "examples/models/core/qwen2audio/run_chat.py": { - "E722": 1 - }, - "examples/models/core/qwenvl/run.py": { - "E722": 1 - }, - "examples/models/core/qwenvl/run_chat.py": { - "E722": 1 - }, - "examples/ngram/run_dtm_ngram.py": { - "D200": 1, - "D205": 1, - "D212": 2, - "D415": 2, - "E741": 3 - }, - "examples/openai_triton/manual_plugin/build.py": { - "D202": 1, - "D205": 1, - "D212": 1, - "D300": 1 - }, - "examples/openai_triton/manual_plugin/fmha_triton.py": { - "D205": 1, - "D212": 1, - "D415": 1 - }, - "examples/openai_triton/manual_plugin/plugin.py": { - "D202": 1, - "D205": 1, - "D212": 1, - "D300": 1 - }, - "examples/openai_triton/plugin_autogen/build_engine.py": { - "D202": 2, - "D205": 2, - "D212": 2, - "D300": 2 - }, - "examples/openai_triton/plugin_autogen/kernel_config.py": { - "F403": 1, - "F405": 22 - }, - "examples/python_plugin/plugin_lib/lookup_plugin.py": { - "E731": 1 - }, "examples/quantization/quantize_mixed_precision_moe.py": { "E741": 1, "F601": 2 }, - "examples/run.py": { - "E711": 4 - }, - "examples/scaffolding/contrib/mcp/e2b/e2bserver.py": { - "D202": 1, - "D205": 1, - "D411": 1 - }, - "examples/scaffolding/contrib/mcp/websearch/websearch.py": { - "D205": 1, - "D415": 1 - }, - "examples/summarize.py": { - "E741": 1 - }, "jenkins/scripts/open_search_db.py": { "D205": 1, "D212": 7 @@ -257,9 +129,6 @@ "D212": 1, "E741": 1 }, - "scripts/check_test_list.py": { - "D212": 1 - }, "scripts/dco_check.py": { "D212": 1 }, @@ -292,39 +161,21 @@ "D212": 1, "E402": 20 }, - "tensorrt_llm/_torch/attention_backend/sparse/dsa.py": { - "F821": 2 - }, "tensorrt_llm/_torch/attention_backend/sparse/kernel.py": { "E731": 3 }, "tensorrt_llm/_torch/attention_backend/sparse/rocket.py": { "E712": 2 }, - "tensorrt_llm/_torch/attention_backend/sparse/utils.py": { - "F821": 4 - }, "tensorrt_llm/_torch/attention_backend/trtllm.py": { "E712": 1 }, - "tensorrt_llm/_torch/attention_backend/utils.py": { - "F821": 2 - }, - "tensorrt_llm/_torch/compilation/patterns/ar_residual_norm.py": { - "E402": 1 - }, "tensorrt_llm/_torch/compilation/patterns/residual_add_norm.py": { "E402": 1 }, "tensorrt_llm/_torch/compilation/piecewise_optimizer.py": { "E711": 2 }, - "tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py": { - "E741": 11 - }, - "tensorrt_llm/_torch/custom_ops/torch_custom_ops.py": { - "F821": 2 - }, "tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_persistent.py": { "E741": 3 }, @@ -332,10 +183,6 @@ "E712": 1, "E731": 1 }, - "tensorrt_llm/_torch/model_config.py": { - "F811": 1, - "F821": 4 - }, "tensorrt_llm/_torch/models/checkpoints/auto_mapper.py": { "F821": 1 }, @@ -366,12 +213,6 @@ "tensorrt_llm/_torch/models/modeling_nemotron.py": { "E731": 1 }, - "tensorrt_llm/_torch/models/modeling_qwen2vl.py": { - "F811": 1 - }, - "tensorrt_llm/_torch/models/modeling_qwen3_next.py": { - "F821": 1 - }, "tensorrt_llm/_torch/models/modeling_siglip.py": { "F811": 1 }, @@ -379,9 +220,6 @@ "E731": 1, "F821": 5 }, - "tensorrt_llm/_torch/modules/attention.py": { - "E731": 2 - }, "tensorrt_llm/_torch/modules/fused_moe/deep_ep_utils.py": { "F821": 2 }, @@ -415,18 +253,6 @@ "tensorrt_llm/_torch/modules/mamba/ssd_state_passing.py": { "E731": 1 }, - "tensorrt_llm/_torch/pyexecutor/_util.py": { - "F811": 1 - }, - "tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py": { - "F821": 1 - }, - "tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py": { - "F821": 1 - }, - "tensorrt_llm/_torch/pyexecutor/model_engine.py": { - "F821": 1 - }, "tensorrt_llm/_torch/pyexecutor/model_loader.py": { "F811": 1 }, @@ -434,24 +260,15 @@ "E712": 1, "E721": 1 }, - "tensorrt_llm/_torch/pyexecutor/resource_manager.py": { - "F821": 2 - }, "tensorrt_llm/_torch/pyexecutor/seq_slot_manager.py": { "F821": 1 }, "tensorrt_llm/_torch/speculative/auto_heuristic.py": { "F821": 1 }, - "tensorrt_llm/_torch/speculative/interface.py": { - "F821": 1 - }, "tensorrt_llm/_torch/speculative/ngram.py": { "E741": 1 }, - "tensorrt_llm/_torch/speculative/speculation_gate.py": { - "F821": 1 - }, "tensorrt_llm/_utils.py": { "E722": 1, "F821": 1 @@ -464,18 +281,6 @@ "D411": 1, "D415": 1 }, - "tensorrt_llm/bench/build/build.py": { - "D205": 1, - "D210": 2, - "D411": 1 - }, - "tensorrt_llm/bench/build/dataclasses.py": { - "D210": 3 - }, - "tensorrt_llm/bench/build/tuning.py": { - "D205": 2, - "D210": 2 - }, "tensorrt_llm/bench/dataclasses/reporting.py": { "D200": 1, "D212": 1 @@ -535,9 +340,7 @@ "D202": 1, "D205": 1, "D208": 3, - "D210": 1, "D212": 1, - "D300": 1, "E722": 1 }, "tensorrt_llm/executor/ray_executor.py": { @@ -578,7 +381,6 @@ }, "tensorrt_llm/executor/rpc_proxy.py": { "D205": 1, - "D212": 1, "D415": 1 }, "tensorrt_llm/executor/rpc_worker.py": { @@ -589,24 +391,6 @@ "D300": 6, "F811": 1 }, - "tensorrt_llm/functional.py": { - "D200": 42, - "D202": 6, - "D205": 20, - "D207": 1, - "D208": 6, - "D210": 2, - "D212": 142, - "D214": 3, - "D300": 144, - "D301": 2, - "D411": 8, - "D415": 9 - }, - "tensorrt_llm/inputs/evs.py": { - "D205": 1, - "D212": 2 - }, "tensorrt_llm/inputs/multimodal.py": { "D415": 1 }, @@ -627,11 +411,11 @@ "E731": 2 }, "tensorrt_llm/llmapi/disagg_utils.py": { - "D205": 2, + "D205": 1, "D210": 1, - "D212": 2, - "D403": 2, - "D415": 2, + "D212": 1, + "D403": 1, + "D415": 1, "F822": 2 }, "tensorrt_llm/llmapi/llm.py": { @@ -639,8 +423,7 @@ "D205": 4 }, "tensorrt_llm/llmapi/llm_args.py": { - "D205": 11, - "D212": 1 + "D205": 10 }, "tensorrt_llm/llmapi/llm_utils.py": { "D200": 1 @@ -657,11 +440,7 @@ }, "tensorrt_llm/llmapi/mpi_session.py": { "D200": 1, - "D205": 2, - "D210": 7, - "D212": 3, - "D300": 10, - "D411": 1, + "D205": 1, "D415": 1 }, "tensorrt_llm/llmapi/reasoning_parser.py": { @@ -703,36 +482,6 @@ "E731": 1, "F821": 1 }, - "tensorrt_llm/models/modeling_utils.py": { - "D200": 3, - "D202": 1, - "D205": 4, - "D208": 4, - "D210": 3, - "D212": 2, - "D300": 5, - "D415": 4, - "E721": 2, - "E722": 1 - }, - "tensorrt_llm/quantization/functional.py": { - "D205": 4, - "D212": 4, - "D300": 4, - "D411": 2, - "D415": 2 - }, - "tensorrt_llm/quantization/quantize_by_modelopt.py": { - "D200": 1, - "D205": 1, - "D208": 2, - "D212": 2, - "D300": 1, - "D415": 2, - "E722": 1, - "F601": 1, - "F821": 2 - }, "tensorrt_llm/quantization/utils/fp4_utils.py": { "D200": 2, "D202": 1, @@ -791,18 +540,9 @@ "tensorrt_llm/scaffolding/result.py": { "F821": 1 }, - "tensorrt_llm/scaffolding/scaffolding_llm.py": { - "PLE0302": 1 - }, "tensorrt_llm/scaffolding/task.py": { "F821": 1 }, - "tensorrt_llm/scaffolding/task_collection.py": { - "F821": 1 - }, - "tensorrt_llm/scaffolding/worker.py": { - "PLE0302": 1 - }, "tensorrt_llm/serve/disagg_auto_scaling.py": { "D205": 2, "D212": 2 @@ -819,19 +559,14 @@ "F811": 1 }, "tensorrt_llm/serve/openai_server.py": { - "D205": 1, - "D212": 2, "F821": 3 }, "tensorrt_llm/serve/responses_utils.py": { "D200": 2, - "D205": 4, - "D212": 11 + "D212": 7 }, "tensorrt_llm/serve/router.py": { "D205": 1, - "D212": 1, - "D300": 1, "D415": 7 }, "tensorrt_llm/serve/scripts/benchmark_dataset.py": { @@ -896,20 +631,10 @@ "D403": 9, "D415": 11 }, - "tests/integration/defs/accuracy/accuracy_core.py": { - "D205": 1, - "D212": 1 - }, "tests/integration/defs/accuracy/test_disaggregated_serving.py": { "D212": 1, "F601": 1 }, - "tests/integration/defs/accuracy/test_llm_api_autodeploy.py": { - "D205": 1, - "D209": 1, - "D415": 1, - "F811": 2 - }, "tests/integration/defs/accuracy/test_llm_api_pytorch.py": { "D212": 1, "D300": 3, @@ -921,9 +646,9 @@ "D205": 3, "D210": 2, "D212": 5, - "D300": 7, - "D403": 6, - "D415": 12 + "D300": 6, + "D403": 5, + "D415": 11 }, "tests/integration/defs/conftest.py": { "D200": 3, @@ -932,9 +657,9 @@ "D209": 1, "D212": 5, "D214": 1, - "D300": 92, + "D300": 91, "D403": 51, - "D415": 92, + "D415": 91, "E722": 2, "E741": 1 }, @@ -946,8 +671,7 @@ "E741": 2 }, "tests/integration/defs/disaggregated/test_disaggregated.py": { - "D205": 3, - "D209": 1, + "D205": 2, "D212": 5, "D415": 1 }, @@ -967,79 +691,17 @@ "D415": 15, "E722": 1 }, - "tests/integration/defs/examples/test_bert.py": { - "D300": 1, - "D415": 1 - }, - "tests/integration/defs/examples/test_bindings.py": { - "D300": 1, - "D415": 1 - }, - "tests/integration/defs/examples/test_chatglm.py": { - "D300": 1 - }, - "tests/integration/defs/examples/test_commandr.py": { - "D300": 1 - }, "tests/integration/defs/examples/test_gpt.py": { "D202": 3, "D300": 5, "D403": 1, "D415": 4 }, - "tests/integration/defs/examples/test_granite.py": { - "D202": 1, - "D300": 1 - }, - "tests/integration/defs/examples/test_internlm.py": { - "D300": 1, - "D415": 1 - }, - "tests/integration/defs/examples/test_llama.py": { - "D202": 2, - "D300": 4, - "D403": 2, - "D415": 2 - }, - "tests/integration/defs/examples/test_mamba.py": { - "D300": 2, - "D415": 2 - }, - "tests/integration/defs/examples/test_mistral.py": { - "D202": 1 - }, - "tests/integration/defs/examples/test_mixtral.py": { - "D300": 1, - "D403": 1 - }, - "tests/integration/defs/examples/test_multimodal.py": { - "D202": 1, - "D300": 2, - "D415": 2 - }, - "tests/integration/defs/examples/test_openai.py": { - "D300": 3, - "D403": 1, - "D415": 3 - }, "tests/integration/defs/examples/test_phi.py": { "D202": 1, "D300": 2, "D415": 2 }, - "tests/integration/defs/examples/test_qwen.py": { - "D202": 1, - "D300": 2, - "D403": 1 - }, - "tests/integration/defs/examples/test_qwen2audio.py": { - "D300": 2, - "D415": 1 - }, - "tests/integration/defs/examples/test_qwenvl.py": { - "D300": 1, - "D415": 1 - }, "tests/integration/defs/llmapi/test_llm_api_qa.py": { "D200": 1, "D212": 1, @@ -1159,16 +821,6 @@ "E741": 2, "F811": 1 }, - "tests/integration/defs/test_mlpf_results.py": { - "D200": 1, - "D205": 1, - "D208": 4, - "D212": 2, - "D415": 2 - }, - "tests/integration/defs/triton_server/common.py": { - "E402": 1 - }, "tests/integration/defs/triton_server/conftest.py": { "D200": 3, "D202": 1, @@ -1215,16 +867,6 @@ "tests/integration/defs/utils/timeout_manager.py": { "D212": 9 }, - "tests/microbenchmarks/build_time_benchmark.py": { - "D200": 1, - "D300": 1, - "D415": 1 - }, - "tests/microbenchmarks/build_time_dashboard.py": { - "D200": 1, - "D300": 1, - "D415": 1 - }, "tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py": { "D212": 1, "E712": 1 @@ -1245,11 +887,7 @@ "D415": 6 }, "tests/unittest/bindings/test_executor_bindings.py": { - "D202": 1, - "E712": 54, - "F403": 2, - "F405": 3, - "F811": 1 + "E712": 51 }, "tests/unittest/conftest.py": { "D200": 3, @@ -1329,10 +967,6 @@ "D205": 1, "D212": 1 }, - "tests/unittest/llmapi/test_llm_utils.py": { - "F403": 2, - "F405": 14 - }, "tests/unittest/llmapi/test_mpi_session.py": { "D200": 1, "D212": 1, @@ -1345,12 +979,6 @@ "tests/unittest/others/test_kv_cache_transceiver.py": { "D212": 1 }, - "tests/unittest/others/test_leak.py": { - "D200": 1, - "D210": 1, - "D300": 1, - "D415": 1 - }, "tests/unittest/others/test_time_breakdown.py": { "D212": 1, "D415": 1 diff --git a/ruff-legacy.toml b/ruff-legacy.toml index 2f9d1bd00745..7e4897f0c374 100644 --- a/ruff-legacy.toml +++ b/ruff-legacy.toml @@ -18,14 +18,6 @@ include = [ ".devcontainer/make_env.py", ".github/scripts/label_community_user.py", ".github/scripts/pr_checklist_check.py", - "benchmarks/cpp/__init__.py", - "benchmarks/cpp/prepare_dataset.py", - "benchmarks/cpp/utils/__init__.py", - "benchmarks/cpp/utils/convert_nemo_dataset.py", - "benchmarks/cpp/utils/generate_rand_loras.py", - "benchmarks/cpp/utils/prepare_real_data.py", - "benchmarks/cpp/utils/prepare_synthetic_data.py", - "benchmarks/cpp/utils/utils.py", "cpp/conanfile.py", "cpp/kernels/fmha_v2/conftest.py", "cpp/kernels/fmha_v2/fmha_test.py", @@ -50,58 +42,17 @@ include = [ "cpp/tensorrt_llm/deep_ep/strip_nvshmem_helper.py", "cpp/tensorrt_llm/kernels/cutlass_kernels/python/generate_kernels.py", "cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/copy_cu.py", - "cpp/tests/resources/scripts/build_chatglm_engines.py", - "cpp/tests/resources/scripts/build_eagle_engines.py", - "cpp/tests/resources/scripts/build_enc_dec_engines.py", - "cpp/tests/resources/scripts/build_engines_utils.py", - "cpp/tests/resources/scripts/build_gpt_engines.py", - "cpp/tests/resources/scripts/build_gptj_engines.py", - "cpp/tests/resources/scripts/build_llama_engines.py", - "cpp/tests/resources/scripts/build_mamba_engines.py", - "cpp/tests/resources/scripts/build_medusa_engines.py", - "cpp/tests/resources/scripts/build_recurrentgemma_engines.py", - "cpp/tests/resources/scripts/build_redrafter_engines.py", - "cpp/tests/resources/scripts/generate_expected_chatglm_output.py", - "cpp/tests/resources/scripts/generate_expected_eagle_output.py", - "cpp/tests/resources/scripts/generate_expected_enc_dec_output.py", - "cpp/tests/resources/scripts/generate_expected_gpt_output.py", - "cpp/tests/resources/scripts/generate_expected_gptj_output.py", - "cpp/tests/resources/scripts/generate_expected_llama_output.py", - "cpp/tests/resources/scripts/generate_expected_mamba_output.py", - "cpp/tests/resources/scripts/generate_expected_medusa_output.py", - "cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py", - "cpp/tests/resources/scripts/generate_expected_redrafter_output.py", - "cpp/tests/resources/scripts/generate_hf_gpt_output.py", "cpp/tests/resources/scripts/generate_test_lora_weights.py", - "cpp/tests/resources/scripts/io_converter.py", "docs/source/conf.py", "docs/source/helper.py", "examples/apps/chat.py", "examples/apps/fastapi_server.py", - "examples/bindings/executor/example_advanced.py", - "examples/bindings/executor/example_basic.py", - "examples/bindings/executor/example_debug.py", - "examples/bindings/executor/example_logits_processor.py", "examples/disaggregated/clients/disagg_client.py", "examples/disaggregated/slurm/benchmark/submit.py", - "examples/dora/normalize_weights.py", - "examples/eagle/convert_checkpoint.py", - "examples/eval_long_context.py", - "examples/generate_checkpoint_config.py", - "examples/generate_xgrammar_tokenizer_info.py", - "examples/hf_lora_convert.py", "examples/infinitebench/args.py", "examples/infinitebench/compute_scores.py", "examples/infinitebench/construct_synthetic_dataset.py", "examples/infinitebench/eval_utils.py", - "examples/llm-api/_tensorrt_engine/llm_eagle2_decoding.py", - "examples/llm-api/_tensorrt_engine/llm_eagle_decoding.py", - "examples/llm-api/_tensorrt_engine/llm_inference_customize.py", - "examples/llm-api/_tensorrt_engine/llm_inference_kv_events.py", - "examples/llm-api/_tensorrt_engine/llm_lookahead_decoding.py", - "examples/llm-api/_tensorrt_engine/llm_medusa_decoding.py", - "examples/llm-api/_tensorrt_engine/llm_quantization.py", - "examples/llm-api/_tensorrt_engine/quickstart_example.py", "examples/llm-api/llm_guided_decoding.py", "examples/llm-api/llm_inference.py", "examples/llm-api/llm_inference_async.py", @@ -121,122 +72,17 @@ include = [ "examples/llm-api/quickstart_example.py", "examples/llm-api/quickstart_multimodal.py", "examples/llm-api/star_attention.py", - "examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py", "examples/longbench/eval_longbench_v1.py", - "examples/medusa/convert_checkpoint.py", - "examples/mmlu.py", - "examples/models/contrib/baichuan/convert_checkpoint.py", - "examples/models/contrib/bloom/convert_checkpoint.py", - "examples/models/contrib/chatglm-6b/tokenization_chatglm.py", - "examples/models/contrib/chatglm2-6b/tokenization_chatglm.py", - "examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py", - "examples/models/contrib/cogvlm/convert_checkpoint.py", - "examples/models/contrib/dbrx/convert_checkpoint.py", - "examples/models/contrib/deepseek_v1/__init__.py", - "examples/models/contrib/deepseek_v1/convert_checkpoint.py", - "examples/models/contrib/deepseek_v2/convert_checkpoint.py", - "examples/models/contrib/dit/convert_checkpoint.py", - "examples/models/contrib/dit/diffusion.py", - "examples/models/contrib/dit/sample.py", - "examples/models/contrib/dit/utils_modelopt.py", - "examples/models/contrib/dit/vae_decoder_trt.py", - "examples/models/contrib/falcon/convert_checkpoint.py", - "examples/models/contrib/gptj/convert_checkpoint.py", - "examples/models/contrib/gptneox/convert_checkpoint.py", - "examples/models/contrib/grok/convert_checkpoint.py", - "examples/models/contrib/mmdit/convert_checkpoint.py", - "examples/models/contrib/mmdit/sample.py", - "examples/models/contrib/mpt/convert_checkpoint.py", - "examples/models/contrib/opt/convert_checkpoint.py", - "examples/models/contrib/sdxl/build_sdxl_unet.py", - "examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py", - "examples/models/contrib/sdxl/run_sdxl.py", - "examples/models/contrib/stdit/aspect.py", - "examples/models/contrib/stdit/convert_checkpoint.py", - "examples/models/contrib/stdit/pipeline_tllm.py", - "examples/models/contrib/stdit/sample.py", - "examples/models/contrib/stdit/scheduler.py", - "examples/models/contrib/stdit/text_encoder.py", - "examples/models/contrib/stdit/utils.py", - "examples/models/contrib/stdit/vae.py", - "examples/models/contrib/stdit/video_transforms.py", - "examples/models/core/bert/__init__.py", - "examples/models/core/bert/convert_checkpoint.py", - "examples/models/core/bert/run.py", - "examples/models/core/bert/utils.py", - "examples/models/core/commandr/convert_checkpoint.py", - "examples/models/core/enc_dec/__init__.py", - "examples/models/core/enc_dec/convert_checkpoint.py", - "examples/models/core/enc_dec/helper.py", - "examples/models/core/enc_dec/run.py", - "examples/models/core/gemma/convert_checkpoint.py", - "examples/models/core/glm-4-9b/convert_checkpoint.py", - "examples/models/core/glm-4-9b/tokenization_chatglm.py", - "examples/models/core/gpt/convert_checkpoint.py", - "examples/models/core/gpt/merge_ptuning_tables.py", - "examples/models/core/gpt/nemo_lora_convert.py", - "examples/models/core/gpt/nemo_prompt_convert.py", - "examples/models/core/gpt/run_hf.py", "examples/models/core/gpt_oss/openai_chat_client_function_calling.py", - "examples/models/core/internlm2/convert_checkpoint.py", "examples/models/core/kimi_k2/kimi_k2_tool_calling_example.py", - "examples/models/core/llama/convert_checkpoint.py", - "examples/models/core/llama/summarize_long.py", - "examples/models/core/mamba/convert_checkpoint.py", - "examples/models/core/mllama/convert_checkpoint.py", - "examples/models/core/multimodal/__init__.py", - "examples/models/core/multimodal/build_multimodal_engine.py", - "examples/models/core/multimodal/eval.py", - "examples/models/core/multimodal/run.py", - "examples/models/core/multimodal/utils.py", - "examples/models/core/nemotron_nas/calibration_utils.py", - "examples/models/core/nemotron_nas/convert_checkpoint.py", - "examples/models/core/phi/convert_checkpoint.py", - "examples/models/core/qwen/convert_checkpoint.py", - "examples/models/core/qwen2audio/run.py", - "examples/models/core/qwen2audio/run_chat.py", - "examples/models/core/qwen2audio/utils.py", - "examples/models/core/qwenvl/run.py", - "examples/models/core/qwenvl/run_chat.py", - "examples/models/core/qwenvl/show_pic.py", - "examples/models/core/qwenvl/vit_onnx_trt.py", - "examples/models/core/recurrentgemma/convert_checkpoint.py", - "examples/models/core/vit/convert_checkpoint.py", - "examples/models/core/whisper/convert_checkpoint.py", - "examples/models/core/whisper/distil_whisper/convert_from_distil_whisper.py", - "examples/models/core/whisper/run.py", - "examples/models/core/whisper/tokenizer.py", - "examples/models/core/whisper/whisper_utils.py", - "examples/ngram/run_dtm_ngram.py", - "examples/openai_triton/manual_plugin/build.py", - "examples/openai_triton/manual_plugin/fmha_triton.py", - "examples/openai_triton/manual_plugin/plugin.py", - "examples/openai_triton/manual_plugin/run.py", - "examples/openai_triton/plugin_autogen/build_engine.py", - "examples/openai_triton/plugin_autogen/kernel_config.py", - "examples/openai_triton/plugin_autogen/run_engine.py", - "examples/python_plugin/build_lookup.py", - "examples/python_plugin/plugin_lib/__init__.py", - "examples/python_plugin/plugin_lib/lookup_kernel.py", - "examples/python_plugin/plugin_lib/lookup_plugin.py", - "examples/python_plugin/run_lookup.py", - "examples/quantization/quantize.py", "examples/quantization/quantize_mixed_precision_moe.py", "examples/ray_orchestrator/llm_inference_async_ray.py", "examples/ray_orchestrator/llm_inference_distributed_ray.py", - "examples/redrafter/convert_checkpoint.py", - "examples/run.py", "examples/scaffolding/contrib/AsyncGeneration/stream_generation_controller.py", "examples/scaffolding/contrib/DeepConf/run_generation.py", "examples/scaffolding/contrib/Dynasor/scaffolding_dynasor_run.py", "examples/scaffolding/contrib/TreeInference/run_mcts_example.py", "examples/scaffolding/contrib/TreeInference/run_tot_example.py", - "examples/scaffolding/contrib/mcp/e2b/e2bserver.py", - "examples/scaffolding/contrib/mcp/e2b/main.py", - "examples/scaffolding/contrib/mcp/mcptest.py", - "examples/scaffolding/contrib/mcp/weather/weather.py", - "examples/scaffolding/contrib/mcp/websearch/main.py", - "examples/scaffolding/contrib/mcp/websearch/websearch.py", "examples/scaffolding/run_basic_generation.py", "examples/scaffolding/run_best_of_n_with_reward.py", "examples/scaffolding/run_majority_vote_aime24.py", @@ -246,8 +92,6 @@ include = [ "examples/serve/openai_completion_client.py", "examples/serve/openai_completion_client_for_lora.py", "examples/serve/openai_completion_client_json_schema.py", - "examples/summarize.py", - "examples/utils.py", "examples/wide_ep/ep_load_balancer/generate_eplb_config.py", "examples/wide_ep/ep_load_balancer/report_load_statistics.py", "examples/wide_ep/ep_load_balancer/utils.py", @@ -255,12 +99,10 @@ include = [ "jenkins/scripts/mergeWaiveList.py", "jenkins/scripts/open_search_db.py", "jenkins/scripts/test_rerun.py", - "scripts/build_cpp_examples.py", "scripts/build_wheel.py", "scripts/check_test_list.py", "scripts/dco_check.py", "scripts/format_test_list.py", - "scripts/generate_duration.py", "scripts/generate_lock_file.py", "scripts/get_wheel_from_package.py", "scripts/git_replace.py", @@ -271,7 +113,6 @@ include = [ "setup.py", "tensorrt_llm/__init__.py", "tensorrt_llm/_ray_utils.py", - "tensorrt_llm/_tensorrt_engine/__init__.py", "tensorrt_llm/_torch/__init__.py", "tensorrt_llm/_torch/attention_backend/__init__.py", "tensorrt_llm/_torch/attention_backend/flashinfer.py", @@ -475,7 +316,6 @@ include = [ "tensorrt_llm/_torch/pyexecutor/guided_decoder.py", "tensorrt_llm/_torch/pyexecutor/handle_additional_outputs.py", "tensorrt_llm/_torch/pyexecutor/handle_logits.py", - "tensorrt_llm/_torch/pyexecutor/kv_cache_connector.py", "tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py", "tensorrt_llm/_torch/pyexecutor/layerwise_nvtx_marker.py", "tensorrt_llm/_torch/pyexecutor/llm_request.py", @@ -513,11 +353,6 @@ include = [ "tensorrt_llm/bench/benchmark/utils/asynchronous.py", "tensorrt_llm/bench/benchmark/utils/general.py", "tensorrt_llm/bench/benchmark/utils/processes.py", - "tensorrt_llm/bench/build/__init__.py", - "tensorrt_llm/bench/build/build.py", - "tensorrt_llm/bench/build/dataclasses.py", - "tensorrt_llm/bench/build/tuning.py", - "tensorrt_llm/bench/build/utils.py", "tensorrt_llm/bench/dataclasses/__init__.py", "tensorrt_llm/bench/dataclasses/configuration.py", "tensorrt_llm/bench/dataclasses/engine.py", @@ -527,13 +362,9 @@ include = [ "tensorrt_llm/bench/dataclasses/statistics.py", "tensorrt_llm/bench/utils/__init__.py", "tensorrt_llm/bench/utils/data.py", - "tensorrt_llm/builder.py", "tensorrt_llm/commands/__init__.py", "tensorrt_llm/commands/bench.py", - "tensorrt_llm/commands/build.py", "tensorrt_llm/commands/eval.py", - "tensorrt_llm/commands/prune.py", - "tensorrt_llm/commands/refit.py", "tensorrt_llm/commands/serve.py", "tensorrt_llm/evaluate/__init__.py", "tensorrt_llm/evaluate/cnn_dailymail.py", @@ -569,23 +400,7 @@ include = [ "tensorrt_llm/inputs/multimodal.py", "tensorrt_llm/inputs/registry.py", "tensorrt_llm/inputs/utils.py", - "tensorrt_llm/layers/__init__.py", - "tensorrt_llm/layers/activation.py", - "tensorrt_llm/layers/attention.py", - "tensorrt_llm/layers/cast.py", - "tensorrt_llm/layers/conv.py", - "tensorrt_llm/layers/embedding.py", - "tensorrt_llm/layers/language_adapter.py", - "tensorrt_llm/layers/linear.py", - "tensorrt_llm/layers/lora.py", - "tensorrt_llm/layers/mlp.py", - "tensorrt_llm/layers/moe.py", - "tensorrt_llm/layers/normalization.py", - "tensorrt_llm/layers/pooling.py", - "tensorrt_llm/layers/recurrent.py", - "tensorrt_llm/layers/ssm.py", "tensorrt_llm/llmapi/__init__.py", - "tensorrt_llm/llmapi/build_cache.py", "tensorrt_llm/llmapi/disagg_utils.py", "tensorrt_llm/llmapi/kv_cache_type.py", "tensorrt_llm/llmapi/llm.py", @@ -608,179 +423,17 @@ include = [ "tensorrt_llm/metrics/enums.py", "tensorrt_llm/models/__init__.py", "tensorrt_llm/models/automodel.py", - "tensorrt_llm/models/baichuan/__init__.py", - "tensorrt_llm/models/baichuan/config.py", - "tensorrt_llm/models/baichuan/convert.py", - "tensorrt_llm/models/baichuan/model.py", - "tensorrt_llm/models/bert/__init__.py", - "tensorrt_llm/models/bert/config.py", - "tensorrt_llm/models/bert/convert.py", - "tensorrt_llm/models/bert/model.py", - "tensorrt_llm/models/bloom/__init__.py", - "tensorrt_llm/models/bloom/model.py", - "tensorrt_llm/models/chatglm/__init__.py", - "tensorrt_llm/models/chatglm/config.py", - "tensorrt_llm/models/chatglm/convert.py", - "tensorrt_llm/models/chatglm/model.py", - "tensorrt_llm/models/clip/__init__.py", - "tensorrt_llm/models/clip/model.py", - "tensorrt_llm/models/cogvlm/__init__.py", - "tensorrt_llm/models/cogvlm/config.py", - "tensorrt_llm/models/cogvlm/convert.py", - "tensorrt_llm/models/cogvlm/model.py", - "tensorrt_llm/models/commandr/__init__.py", - "tensorrt_llm/models/commandr/config.py", - "tensorrt_llm/models/commandr/model.py", "tensorrt_llm/models/convert_utils.py", - "tensorrt_llm/models/dbrx/__init__.py", - "tensorrt_llm/models/dbrx/config.py", - "tensorrt_llm/models/dbrx/model.py", - "tensorrt_llm/models/deepseek_v1/__init__.py", - "tensorrt_llm/models/deepseek_v1/config.py", - "tensorrt_llm/models/deepseek_v1/convert.py", - "tensorrt_llm/models/deepseek_v1/model.py", - "tensorrt_llm/models/deepseek_v2/__init__.py", - "tensorrt_llm/models/deepseek_v2/config.py", - "tensorrt_llm/models/deepseek_v2/convert.py", - "tensorrt_llm/models/deepseek_v2/model.py", - "tensorrt_llm/models/dit/__init__.py", - "tensorrt_llm/models/dit/model.py", - "tensorrt_llm/models/eagle/__init__.py", - "tensorrt_llm/models/eagle/config.py", - "tensorrt_llm/models/eagle/model.py", - "tensorrt_llm/models/enc_dec/__init__.py", - "tensorrt_llm/models/enc_dec/model.py", - "tensorrt_llm/models/falcon/__init__.py", - "tensorrt_llm/models/falcon/config.py", - "tensorrt_llm/models/falcon/convert.py", - "tensorrt_llm/models/falcon/model.py", - "tensorrt_llm/models/gemma/__init__.py", - "tensorrt_llm/models/gemma/config.py", - "tensorrt_llm/models/gemma/convert.py", - "tensorrt_llm/models/gemma/model.py", - "tensorrt_llm/models/gemma/smoothquant.py", - "tensorrt_llm/models/gemma/utils/__init__.py", - "tensorrt_llm/models/gemma/utils/layers.py", - "tensorrt_llm/models/gemma/utils/modules.py", - "tensorrt_llm/models/gemma/utils/params.py", - "tensorrt_llm/models/gemma/utils/positional_embeddings.py", - "tensorrt_llm/models/gemma/utils/sampler.py", - "tensorrt_llm/models/gemma/utils/transformer.py", - "tensorrt_llm/models/gemma/weight.py", - "tensorrt_llm/models/generation_mixin.py", - "tensorrt_llm/models/gpt/__init__.py", - "tensorrt_llm/models/gpt/config.py", - "tensorrt_llm/models/gpt/convert.py", - "tensorrt_llm/models/gpt/model.py", - "tensorrt_llm/models/gptj/__init__.py", - "tensorrt_llm/models/gptj/config.py", - "tensorrt_llm/models/gptj/convert.py", - "tensorrt_llm/models/gptj/model.py", - "tensorrt_llm/models/gptneox/__init__.py", - "tensorrt_llm/models/gptneox/model.py", - "tensorrt_llm/models/grok/__init__.py", - "tensorrt_llm/models/grok/convert.py", - "tensorrt_llm/models/grok/model.py", - "tensorrt_llm/models/grok/weight.py", - "tensorrt_llm/models/llama/__init__.py", - "tensorrt_llm/models/llama/config.py", - "tensorrt_llm/models/llama/convert.py", - "tensorrt_llm/models/llama/model.py", - "tensorrt_llm/models/mamba/__init__.py", - "tensorrt_llm/models/mamba/config.py", - "tensorrt_llm/models/mamba/convert.py", - "tensorrt_llm/models/mamba/model.py", - "tensorrt_llm/models/medusa/__init__.py", - "tensorrt_llm/models/medusa/config.py", - "tensorrt_llm/models/medusa/model.py", - "tensorrt_llm/models/medusa/weight.py", - "tensorrt_llm/models/mllama/__init__.py", - "tensorrt_llm/models/mllama/config.py", - "tensorrt_llm/models/mllama/model.py", - "tensorrt_llm/models/mmdit_sd3/__init__.py", - "tensorrt_llm/models/mmdit_sd3/config.py", - "tensorrt_llm/models/mmdit_sd3/model.py", - "tensorrt_llm/models/model_weights_loader.py", "tensorrt_llm/models/modeling_utils.py", - "tensorrt_llm/models/mpt/__init__.py", - "tensorrt_llm/models/mpt/model.py", - "tensorrt_llm/models/multimodal_encoders/__init__.py", - "tensorrt_llm/models/multimodal_encoders/config.py", - "tensorrt_llm/models/multimodal_encoders/model.py", - "tensorrt_llm/models/nemotron_nas/__init__.py", - "tensorrt_llm/models/nemotron_nas/config.py", - "tensorrt_llm/models/nemotron_nas/convert.py", - "tensorrt_llm/models/nemotron_nas/layer_config.py", - "tensorrt_llm/models/nemotron_nas/model.py", - "tensorrt_llm/models/opt/__init__.py", - "tensorrt_llm/models/opt/model.py", - "tensorrt_llm/models/phi/__init__.py", - "tensorrt_llm/models/phi/config.py", - "tensorrt_llm/models/phi/convert.py", - "tensorrt_llm/models/phi/model.py", - "tensorrt_llm/models/phi3/__init__.py", - "tensorrt_llm/models/phi3/config.py", - "tensorrt_llm/models/phi3/convert.py", - "tensorrt_llm/models/phi3/model.py", - "tensorrt_llm/models/phi3/split_weights.py", - "tensorrt_llm/models/qwen/__init__.py", - "tensorrt_llm/models/qwen/config.py", - "tensorrt_llm/models/qwen/convert.py", - "tensorrt_llm/models/qwen/model.py", - "tensorrt_llm/models/qwen/utils.py", - "tensorrt_llm/models/recurrentgemma/__init__.py", - "tensorrt_llm/models/recurrentgemma/model.py", - "tensorrt_llm/models/redrafter/__init__.py", - "tensorrt_llm/models/redrafter/drafter.py", - "tensorrt_llm/models/redrafter/model.py", - "tensorrt_llm/models/redrafter/redrafter_helper.py", - "tensorrt_llm/models/stdit/__init__.py", - "tensorrt_llm/models/stdit/config.py", - "tensorrt_llm/models/stdit/model.py", - "tensorrt_llm/models/unet/__init__.py", - "tensorrt_llm/models/unet/attention.py", - "tensorrt_llm/models/unet/embeddings.py", - "tensorrt_llm/models/unet/pp/__init__.py", - "tensorrt_llm/models/unet/pp/attention.py", - "tensorrt_llm/models/unet/pp/conv2d.py", - "tensorrt_llm/models/unet/pp/groupnorm.py", - "tensorrt_llm/models/unet/pp/unet_pp.py", - "tensorrt_llm/models/unet/resnet.py", - "tensorrt_llm/models/unet/unet_2d_blocks.py", - "tensorrt_llm/models/unet/unet_2d_condition.py", - "tensorrt_llm/models/unet/weights.py", - "tensorrt_llm/network.py", - "tensorrt_llm/parameter.py", - "tensorrt_llm/plugin/__init__.py", - "tensorrt_llm/plugin/plugin.py", "tensorrt_llm/quantization/__init__.py", "tensorrt_llm/quantization/functional.py", - "tensorrt_llm/quantization/image_processing.py", - "tensorrt_llm/quantization/layers.py", "tensorrt_llm/quantization/mode.py", - "tensorrt_llm/quantization/quantize.py", - "tensorrt_llm/quantization/quantize_by_modelopt.py", "tensorrt_llm/quantization/utils/__init__.py", "tensorrt_llm/quantization/utils/fp4_utils.py", "tensorrt_llm/quantization/utils/fp8_utils.py", "tensorrt_llm/ray_stub.py", "tensorrt_llm/runtime/__init__.py", - "tensorrt_llm/runtime/enc_dec_model_runner.py", - "tensorrt_llm/runtime/generation.py", - "tensorrt_llm/runtime/kv_cache_manager.py", - "tensorrt_llm/runtime/medusa_utils.py", "tensorrt_llm/runtime/memory_pools/__init__.py", - "tensorrt_llm/runtime/memory_pools/memory_pools_allocator.py", - "tensorrt_llm/runtime/memory_pools/pool.py", - "tensorrt_llm/runtime/memory_pools/pools_kv_cache_manager.py", - "tensorrt_llm/runtime/model_runner.py", - "tensorrt_llm/runtime/model_runner_cpp.py", - "tensorrt_llm/runtime/multimodal_model_runner.py", - "tensorrt_llm/runtime/processor_wrapper/__init__.py", - "tensorrt_llm/runtime/processor_wrapper/mllama_processor_wrapper.py", - "tensorrt_llm/runtime/processor_wrapper/processor_wrapper.py", - "tensorrt_llm/runtime/redrafter_utils.py", - "tensorrt_llm/runtime/session.py", "tensorrt_llm/scaffolding/__init__.py", "tensorrt_llm/scaffolding/benchmark.py", "tensorrt_llm/scaffolding/contrib/AsyncGeneration/stream_generation.py", @@ -835,12 +488,7 @@ include = [ "tensorrt_llm/tokenizer/tokenizer.py", "tensorrt_llm/tools/__init__.py", "tensorrt_llm/tools/importlib_utils.py", - "tensorrt_llm/tools/multimodal_builder.py", - "tensorrt_llm/tools/onnx_utils.py", "tensorrt_llm/tools/plugin_gen/__init__.py", - "tensorrt_llm/tools/plugin_gen/core.py", - "tensorrt_llm/tools/plugin_gen/plugin_gen.py", - "tensorrt_llm/tools/plugin_gen/shape_infer.py", "tensorrt_llm/tools/ppl.py", "tensorrt_llm/tools/profiler/nsys_profile_tools/gputrc2graph.py", "tensorrt_llm/version.py", @@ -851,7 +499,6 @@ include = [ "tests/integration/defs/accuracy/scripts/compute_theta_and_thresholds.py", "tests/integration/defs/accuracy/test_cli_flow.py", "tests/integration/defs/accuracy/test_disaggregated_serving.py", - "tests/integration/defs/accuracy/test_llm_api.py", "tests/integration/defs/accuracy/test_llm_api_autodeploy.py", "tests/integration/defs/accuracy/test_llm_api_pytorch.py", "tests/integration/defs/accuracy/test_llm_api_pytorch_ray.py", @@ -860,63 +507,29 @@ include = [ "tests/integration/defs/conftest.py", "tests/integration/defs/cpp/conftest.py", "tests/integration/defs/cpp/cpp_common.py", - "tests/integration/defs/cpp/test_e2e.py", "tests/integration/defs/cpp/test_multi_gpu.py", "tests/integration/defs/cpp/test_unit_tests.py", - "tests/integration/defs/deterministic/mixtral_deterministic.py", - "tests/integration/defs/deterministic/test_mixtral_deterministic.py", "tests/integration/defs/disaggregated/test_auto_scaling.py", "tests/integration/defs/disaggregated/test_disaggregated.py", "tests/integration/defs/disaggregated/test_disaggregated_etcd.py", "tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py", "tests/integration/defs/disaggregated/test_workers.py", - "tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py", "tests/integration/defs/examples/run_llm_quickstart_atexit.py", "tests/integration/defs/examples/serve/test_serve.py", "tests/integration/defs/examples/serve/test_serve_negative.py", "tests/integration/defs/examples/test_ad_guided_decoding.py", - "tests/integration/defs/examples/test_bert.py", - "tests/integration/defs/examples/test_bindings.py", - "tests/integration/defs/examples/test_chatglm.py", - "tests/integration/defs/examples/test_commandr.py", - "tests/integration/defs/examples/test_draft_target_model.py", - "tests/integration/defs/examples/test_eagle.py", - "tests/integration/defs/examples/test_enc_dec.py", - "tests/integration/defs/examples/test_exaone.py", - "tests/integration/defs/examples/test_gemma.py", "tests/integration/defs/examples/test_gpt.py", - "tests/integration/defs/examples/test_gptj.py", - "tests/integration/defs/examples/test_granite.py", - "tests/integration/defs/examples/test_internlm.py", - "tests/integration/defs/examples/test_llama.py", "tests/integration/defs/examples/test_llm_api_with_mpi.py", - "tests/integration/defs/examples/test_mamba.py", - "tests/integration/defs/examples/test_medusa.py", - "tests/integration/defs/examples/test_mistral.py", - "tests/integration/defs/examples/test_mixtral.py", - "tests/integration/defs/examples/test_multimodal.py", - "tests/integration/defs/examples/test_nemotron.py", - "tests/integration/defs/examples/test_nemotron_nas.py", - "tests/integration/defs/examples/test_ngram.py", - "tests/integration/defs/examples/test_openai.py", "tests/integration/defs/examples/test_phi.py", - "tests/integration/defs/examples/test_qwen.py", - "tests/integration/defs/examples/test_qwen2audio.py", - "tests/integration/defs/examples/test_qwenvl.py", "tests/integration/defs/examples/test_ray.py", - "tests/integration/defs/examples/test_recurrentgemma.py", - "tests/integration/defs/examples/test_redrafter.py", - "tests/integration/defs/examples/test_whisper.py", "tests/integration/defs/llmapi/__init__.py", "tests/integration/defs/llmapi/_run_llmapi_llm.py", "tests/integration/defs/llmapi/test_llm_api_connector.py", "tests/integration/defs/llmapi/test_llm_api_qa.py", - "tests/integration/defs/llmapi/test_llm_e2e.py", "tests/integration/defs/llmapi/test_llm_examples.py", "tests/integration/defs/local_venv.py", "tests/integration/defs/perf/__init__.py", "tests/integration/defs/perf/allowed_configs.py", - "tests/integration/defs/perf/build.py", "tests/integration/defs/perf/create_perf_comparison_report.py", "tests/integration/defs/perf/data.py", "tests/integration/defs/perf/data_export.py", @@ -937,38 +550,21 @@ include = [ "tests/integration/defs/test_fmha.py", "tests/integration/defs/test_list_parser.py", "tests/integration/defs/test_list_validation.py", - "tests/integration/defs/test_mlpf_results.py", "tests/integration/defs/test_sanity.py", "tests/integration/defs/test_unittests.py", "tests/integration/defs/triton_server/__init__.py", - "tests/integration/defs/triton_server/build_engines.py", "tests/integration/defs/triton_server/common.py", "tests/integration/defs/triton_server/conftest.py", - "tests/integration/defs/triton_server/local_venv.py", - "tests/integration/defs/triton_server/rcca/bug_4323566/inflight_batcher_llm_client_with_end_id.py", - "tests/integration/defs/triton_server/runner_interface.py", "tests/integration/defs/triton_server/test_list_parser.py", - "tests/integration/defs/triton_server/test_triton.py", - "tests/integration/defs/triton_server/test_triton_llm.py", - "tests/integration/defs/triton_server/test_triton_memleak.py", - "tests/integration/defs/triton_server/test_triton_multi_node.py", - "tests/integration/defs/triton_server/test_triton_rcca.py", "tests/integration/defs/triton_server/trt_test_alternative.py", "tests/integration/defs/trt_test_alternative.py", "tests/integration/defs/utils/__init__.py", "tests/integration/defs/utils/periodic_junit.py", "tests/integration/defs/utils/timeout_manager.py", "tests/microbenchmarks/all_reduce.py", - "tests/microbenchmarks/build_time_benchmark.py", - "tests/microbenchmarks/build_time_dashboard.py", "tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py", "tests/scripts/allreduce_perf/allreduce_perf_viz.py", "tests/scripts/iteration_log_parser.py", - "tests/scripts/perf-sanity/parse_benchmark_results.py", - "tests/scripts/perf-sanity/run_benchmark_serve.py", - "tests/unittest/_torch/attention/sparse/test_dsa_indexer.py", - "tests/unittest/_torch/attention/sparse/test_flash_mla.py", - "tests/unittest/_torch/attention/sparse/test_rocketkv.py", "tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py", "tests/unittest/_torch/attention/test_attention.py", "tests/unittest/_torch/attention/test_attention_mla.py", @@ -989,7 +585,6 @@ include = [ "tests/unittest/_torch/misc/test_virtual_memory.py", "tests/unittest/_torch/modeling/test_modeling_bert.py", "tests/unittest/_torch/modeling/test_modeling_clip.py", - "tests/unittest/_torch/modeling/test_modeling_exaone4.py", "tests/unittest/_torch/modeling/test_modeling_gemma3.py", "tests/unittest/_torch/modeling/test_modeling_gpt_oss.py", "tests/unittest/_torch/modeling/test_modeling_llama.py", @@ -1013,8 +608,6 @@ include = [ "tests/unittest/_torch/modules/test_moe_routing.py", "tests/unittest/_torch/modules/test_rotary_embedding.py", "tests/unittest/_torch/modules/test_triton_linear.py", - "tests/unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py", - "tests/unittest/_torch/modules/tests_lora_modules/test_lora_plugin_vs_lora_op.py", "tests/unittest/_torch/multi_gpu/test_allreduce.py", "tests/unittest/_torch/multi_gpu/test_alltoall.py", "tests/unittest/_torch/multi_gpu/test_ar_residual_norm.py", @@ -1041,24 +634,10 @@ include = [ "tests/unittest/_torch/sampler/test_beam_search.py", "tests/unittest/_torch/sampler/test_best_of_n.py", "tests/unittest/_torch/sampler/test_trtllm_sampler.py", - "tests/unittest/_torch/speculative/test_draft_target.py", - "tests/unittest/_torch/speculative/test_draft_token_tree_sampling.py", - "tests/unittest/_torch/speculative/test_draft_token_tree_verification.py", - "tests/unittest/_torch/speculative/test_dynamic_spec_decode.py", "tests/unittest/_torch/speculative/test_eagle3.py", - "tests/unittest/_torch/speculative/test_kv_cache_reuse.py", - "tests/unittest/_torch/speculative/test_mtp.py", - "tests/unittest/_torch/speculative/test_ngram.py", - "tests/unittest/_torch/speculative/test_save_state.py", - "tests/unittest/_torch/speculative/test_spec_gate.py", - "tests/unittest/_torch/speculative/test_torch_rejection_sampling.py", - "tests/unittest/_torch/speculative/test_user_provided.py", "tests/unittest/_torch/test_connector.py", "tests/unittest/_torch/test_torch_multi_arange.py", "tests/unittest/_torch/thop/parallel/deep_gemm_tests.py", - "tests/unittest/_torch/thop/parallel/test_causal_conv1d_op.py", - "tests/unittest/_torch/thop/parallel/test_cublas_mm.py", - "tests/unittest/_torch/thop/parallel/test_custom_ops.py", "tests/unittest/_torch/thop/parallel/test_dsv3_fused_a_gemm.py", "tests/unittest/_torch/thop/parallel/test_dsv3_router_gemm.py", "tests/unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py", @@ -1072,11 +651,6 @@ include = [ "tests/unittest/_torch/thop/parallel/test_fp8_per_tensor_scale_tllmg_gemm.py", "tests/unittest/_torch/thop/parallel/test_fp8_quantize.py", "tests/unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py", - "tests/unittest/_torch/thop/parallel/test_fused_qk_norm_rope.py", - "tests/unittest/_torch/thop/parallel/test_logits_bitmask_op.py", - "tests/unittest/_torch/thop/parallel/test_mamba2_chunk_ss_update.py", - "tests/unittest/_torch/thop/parallel/test_mamba_conv1d_op.py", - "tests/unittest/_torch/thop/parallel/test_noaux_tc.py", "tests/unittest/_torch/thop/parallel/test_scaled_mm.py", "tests/unittest/_torch/thop/parallel/test_selective_scan_op.py", "tests/unittest/_torch/thop/parallel/test_tinygemm2.py", @@ -1090,7 +664,6 @@ include = [ "tests/unittest/_torch/thop/serial/test_moe_alltoall.py", "tests/unittest/api_stability/api_stability_core.py", "tests/unittest/api_stability/test_llm_api.py", - "tests/unittest/bindings/binding_test_utils.py", "tests/unittest/bindings/test_bindings_moe.py", "tests/unittest/bindings/test_bindings_ut.py", "tests/unittest/bindings/test_executor_bindings.py", @@ -1121,12 +694,10 @@ include = [ "tests/unittest/llmapi/apps/_test_openai_chat_harmony.py", "tests/unittest/llmapi/apps/_test_openai_chat_multimodal.py", "tests/unittest/llmapi/apps/_test_openai_completions.py", - "tests/unittest/llmapi/apps/_test_openai_consistent_chat.py", "tests/unittest/llmapi/apps/_test_openai_lora.py", "tests/unittest/llmapi/apps/_test_openai_metrics.py", "tests/unittest/llmapi/apps/_test_openai_misc.py", "tests/unittest/llmapi/apps/_test_openai_mmencoder.py", - "tests/unittest/llmapi/apps/_test_openai_multi_chat.py", "tests/unittest/llmapi/apps/_test_openai_multi_gpu.py", "tests/unittest/llmapi/apps/_test_openai_multi_nodes.py", "tests/unittest/llmapi/apps/_test_openai_perf_metrics.py", @@ -1149,15 +720,12 @@ include = [ "tests/unittest/llmapi/run_llm_exit.py", "tests/unittest/llmapi/run_llm_with_postproc.py", "tests/unittest/llmapi/test_additional_model_outputs.py", - "tests/unittest/llmapi/test_build_cache.py", "tests/unittest/llmapi/test_executor.py", "tests/unittest/llmapi/test_gc_utils.py", "tests/unittest/llmapi/test_llm.py", "tests/unittest/llmapi/test_llm_args.py", "tests/unittest/llmapi/test_llm_download.py", "tests/unittest/llmapi/test_llm_kv_cache_events.py", - "tests/unittest/llmapi/test_llm_models.py", - "tests/unittest/llmapi/test_llm_multi_gpu.py", "tests/unittest/llmapi/test_llm_multi_gpu_pytorch.py", "tests/unittest/llmapi/test_llm_pytorch.py", "tests/unittest/llmapi/test_llm_quant.py", @@ -1168,25 +736,14 @@ include = [ "tests/unittest/llmapi/test_serialization.py", "tests/unittest/llmapi/test_utils.py", "tests/unittest/others/__init__.py", - "tests/unittest/others/test_builder.py", "tests/unittest/others/test_convert_spec_decoding_mask_to_packed_mask.py", - "tests/unittest/others/test_debugging_api.py", "tests/unittest/others/test_exception.py", "tests/unittest/others/test_export.py", - "tests/unittest/others/test_graph_rewriter.py", - "tests/unittest/others/test_kv_cache_manager.py", "tests/unittest/others/test_kv_cache_transceiver.py", "tests/unittest/others/test_kv_cache_update.py", - "tests/unittest/others/test_layer.py", - "tests/unittest/others/test_leak.py", "tests/unittest/others/test_mapping.py", - "tests/unittest/others/test_model_dtype.py", - "tests/unittest/others/test_module.py", "tests/unittest/others/test_multimodal_registry.py", - "tests/unittest/others/test_plugins.py", - "tests/unittest/others/test_precision_control.py", "tests/unittest/others/test_pretrained_config.py", - "tests/unittest/others/test_session.py", "tests/unittest/others/test_time_breakdown.py", "tests/unittest/profile_utils.py", "tests/unittest/scaffolding/__init__.py", @@ -1195,141 +752,14 @@ include = [ "tests/unittest/scaffolding/test_scaffolding.py", "tests/unittest/scaffolding/test_task_collection.py", "tests/unittest/scaffolding/test_worker.py", - "tests/unittest/test_model_runner_cpp.py", "tests/unittest/test_pip_install.py", "tests/unittest/tools/__init__.py", - "tests/unittest/tools/plugin_gen/__init__.py", - "tests/unittest/tools/plugin_gen/kernel_config.py", - "tests/unittest/tools/plugin_gen/test_core.py", - "tests/unittest/tools/plugin_gen/test_plugin_gen.py", - "tests/unittest/tools/plugin_gen/test_shape_infer.py", "tests/unittest/tools/test_prepare_dataset.py", "tests/unittest/tools/test_test_to_stage_mapping.py", - "tests/unittest/trt/__init__.py", - "tests/unittest/trt/attention/test_bert_attention.py", - "tests/unittest/trt/attention/test_gpt_attention.py", - "tests/unittest/trt/attention/test_gpt_attention_IFB.py", - "tests/unittest/trt/attention/test_gpt_attention_no_cache.py", - "tests/unittest/trt/attention/test_sage_attention.py", - "tests/unittest/trt/functional/__init__.py", - "tests/unittest/trt/functional/test_alibi.py", - "tests/unittest/trt/functional/test_allreduce_norm.py", - "tests/unittest/trt/functional/test_allreduce_prepost_residual_norm.py", - "tests/unittest/trt/functional/test_arange.py", - "tests/unittest/trt/functional/test_argmax.py", - "tests/unittest/trt/functional/test_assertion.py", - "tests/unittest/trt/functional/test_avg_pool2d.py", - "tests/unittest/trt/functional/test_cast.py", - "tests/unittest/trt/functional/test_conv2d.py", - "tests/unittest/trt/functional/test_conv3d.py", - "tests/unittest/trt/functional/test_cos.py", - "tests/unittest/trt/functional/test_cumsum.py", - "tests/unittest/trt/functional/test_dora.py", - "tests/unittest/trt/functional/test_einsum.py", - "tests/unittest/trt/functional/test_embedding_single_gpu.py", - "tests/unittest/trt/functional/test_exp.py", - "tests/unittest/trt/functional/test_expand.py", - "tests/unittest/trt/functional/test_flatten.py", - "tests/unittest/trt/functional/test_flip.py", - "tests/unittest/trt/functional/test_fp4_gemm.py", - "tests/unittest/trt/functional/test_fp4_gemm_ootb.py", - "tests/unittest/trt/functional/test_gather.py", - "tests/unittest/trt/functional/test_gather_nd.py", - "tests/unittest/trt/functional/test_geglu.py", - "tests/unittest/trt/functional/test_gelu.py", - "tests/unittest/trt/functional/test_gemm_swiglu.py", - "tests/unittest/trt/functional/test_group_norm.py", - "tests/unittest/trt/functional/test_identity.py", - "tests/unittest/trt/functional/test_index_select.py", - "tests/unittest/trt/functional/test_interpolate.py", - "tests/unittest/trt/functional/test_logsoftmax.py", - "tests/unittest/trt/functional/test_lora.py", - "tests/unittest/trt/functional/test_low_latency_gemm.py", - "tests/unittest/trt/functional/test_mamba_conv1d.py", - "tests/unittest/trt/functional/test_masked_scatter.py", - "tests/unittest/trt/functional/test_masked_select.py", - "tests/unittest/trt/functional/test_matmul.py", - "tests/unittest/trt/functional/test_meshgrid2d.py", - "tests/unittest/trt/functional/test_moe.py", - "tests/unittest/trt/functional/test_nccl.py", - "tests/unittest/trt/functional/test_nonzero.py", - "tests/unittest/trt/functional/test_outer.py", - "tests/unittest/trt/functional/test_pad.py", - "tests/unittest/trt/functional/test_permute.py", - "tests/unittest/trt/functional/test_pp_reduce_scatter.py", - "tests/unittest/trt/functional/test_quant.py", - "tests/unittest/trt/functional/test_rearrange.py", - "tests/unittest/trt/functional/test_repeat.py", - "tests/unittest/trt/functional/test_repeat_interleave.py", - "tests/unittest/trt/functional/test_rg_lru.py", - "tests/unittest/trt/functional/test_sample.py", - "tests/unittest/trt/functional/test_scatter.py", - "tests/unittest/trt/functional/test_scatter_nd.py", - "tests/unittest/trt/functional/test_select.py", - "tests/unittest/trt/functional/test_selective_scan.py", - "tests/unittest/trt/functional/test_sigmoid.py", - "tests/unittest/trt/functional/test_silu.py", - "tests/unittest/trt/functional/test_sin.py", - "tests/unittest/trt/functional/test_slice.py", - "tests/unittest/trt/functional/test_softplus.py", - "tests/unittest/trt/functional/test_split.py", - "tests/unittest/trt/functional/test_squeeze.py", - "tests/unittest/trt/functional/test_swiglu.py", - "tests/unittest/trt/functional/test_topk.py", - "tests/unittest/trt/functional/test_transpose.py", - "tests/unittest/trt/functional/test_unbind.py", - "tests/unittest/trt/functional/test_unsqueeze.py", - "tests/unittest/trt/functional/test_view.py", - "tests/unittest/trt/functional/test_where.py", - "tests/unittest/trt/model/__init__.py", - "tests/unittest/trt/model/eagle/test_decode_draft_tokens_plugin.py", - "tests/unittest/trt/model/eagle/test_prepare_drafter_inputs_plugin.py", - "tests/unittest/trt/model/eagle/test_sample_accept_draft_tokens_plugin.py", - "tests/unittest/trt/model/redrafter/test_beams2tree.py", - "tests/unittest/trt/model/redrafter/test_draft_token.py", - "tests/unittest/trt/model/redrafter/test_draft_token_indices.py", - "tests/unittest/trt/model/redrafter/test_gather_beams.py", - "tests/unittest/trt/model/redrafter/test_mask.py", - "tests/unittest/trt/model/redrafter/test_packed_position_ids.py", - "tests/unittest/trt/model/redrafter/test_prefix_match_indices.py", - "tests/unittest/trt/model/redrafter/test_prepare_input.py", - "tests/unittest/trt/model/redrafter/test_process_logits.py", - "tests/unittest/trt/model/redrafter/test_top1.py", - "tests/unittest/trt/model/redrafter/test_unpack_gen_data.py", - "tests/unittest/trt/model/redrafter/test_validate.py", - "tests/unittest/trt/model/test_gpt.py", - "tests/unittest/trt/model/test_gpt_e2e.py", - "tests/unittest/trt/model/test_llama.py", - "tests/unittest/trt/model/test_mamba.py", - "tests/unittest/trt/model/test_mistral.py", - "tests/unittest/trt/model/test_nemotron_nas.py", - "tests/unittest/trt/model/test_phi.py", - "tests/unittest/trt/model/test_unet.py", - "tests/unittest/trt/model_api/test_model_api_multi_gpu.py", - "tests/unittest/trt/model_api/test_model_level_api.py", - "tests/unittest/trt/model_api/test_model_quantization.py", - "tests/unittest/trt/python_plugin/plugin_wrapper_utils.py", - "tests/unittest/trt/python_plugin/test_plugin_wrapper.py", - "tests/unittest/trt/quantization/__init__.py", - "tests/unittest/trt/quantization/_utils.py", - "tests/unittest/trt/quantization/test_fp8_quantization.py", - "tests/unittest/trt/quantization/test_fp8_rowwise_gemm.py", - "tests/unittest/trt/quantization/test_functional.py", - "tests/unittest/trt/quantization/test_mode.py", - "tests/unittest/trt/quantization/test_moe_weight_only_quant_matmul.py", - "tests/unittest/trt/quantization/test_qserve_gemm.py", - "tests/unittest/trt/quantization/test_quant.py", - "tests/unittest/trt/quantization/test_quant_layer.py", - "tests/unittest/trt/quantization/test_smooth_quant_gemm.py", - "tests/unittest/trt/quantization/test_smooth_quant_layer_norm.py", - "tests/unittest/trt/quantization/test_smooth_quant_rms_norm.py", - "tests/unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py", - "tests/unittest/trt/quantization/test_weight_only_quant_matmul.py", "tests/unittest/utils/__init__.py", "tests/unittest/utils/cpp_paths.py", "tests/unittest/utils/llm_data.py", "tests/unittest/utils/runtime_defaults.py", - "tests/unittest/utils/test_medusa_utils.py", "tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py", "tests/unittest/utils/test_util.py", "tests/unittest/utils/torch_ref.py", diff --git a/scripts/attribution/data/dependency_metadata.yml b/scripts/attribution/data/dependency_metadata.yml index dd4dc9135911..2b8485cb1383 100644 --- a/scripts/attribution/data/dependency_metadata.yml +++ b/scripts/attribution/data/dependency_metadata.yml @@ -23,9 +23,9 @@ cutlass/v4.3.0: deep_ep/5be51b228a7c82dbdb213ea58e77bffd12b38af8: license: 452b3ef002dc6ec283fb723f0dd84997 source: https://github.com/deepseek-ai/DeepEP/tree/5be51b228a7c82dbdb213ea58e77bffd12b38af8 -deepgemm/245dc5d6a5fe344c61505fe71011d203141d4479: +deepgemm/f8e8fb5830fa5cda6e4ea73d360bb3f21f87a3ca: license: 452b3ef002dc6ec283fb723f0dd84997 - source: https://github.com/deepseek-ai/DeepGEMM/tree/245dc5d6a5fe344c61505fe71011d203141d4479 + source: https://github.com/deepseek-ai/DeepGEMM/tree/f8e8fb5830fa5cda6e4ea73d360bb3f21f87a3ca dlpack/v1.0: license: cd9881918c97ec7b4962691660bb733e flashmla/1408756a88e52a25196b759eaf8db89d2b51b5a1: diff --git a/scripts/attribution/data/files_to_dependency.yml b/scripts/attribution/data/files_to_dependency.yml index 0db1c72cb8dd..7a5c25002735 100644 --- a/scripts/attribution/data/files_to_dependency.yml +++ b/scripts/attribution/data/files_to_dependency.yml @@ -1369,8 +1369,10 @@ cuda/13.1: cutlass/v4.2.1: - 3e779fe045aa4070e15b9fb12a0acf68 - 43a52c6b5623577f0a4a179af2d15590 +- 4b848740a80061bb4cc89e04022b769d - 7ea7f6702c01bc759785b34bb36239c7 - 7ecdb376ce00b384f0760f57b0c622d8 +- 936cca8a380e4dc810d99527d3af4751 - 94ce61ecff6c7e352003b05ac67f4905 - a8403f6f46a29b7e076bac9f9d4fe72c - adfa6fe7cde72db7d7b4188d6e0d785f @@ -2091,57 +2093,75 @@ deep_ep/5be51b228a7c82dbdb213ea58e77bffd12b38af8: - dbc9ea8cf83b20e6ce4c6f383b700f29 - e4024308dd534f83de92752e1d7cd9a8 - f41ae95dbbafe6107dd98bf66af018ea -deepgemm/245dc5d6a5fe344c61505fe71011d203141d4479: -- 010f3165163e4130374deadad3064bbb -- 0277845f3f5cbb5d22c6f6e046588e26 -- 0a4d6d91c996f54e24eea24fb8799c21 -- 0bb3bdbdbdc771ea4a77990048f46f86 -- 0ee6135be6853d56d700e71b4e813ebc -- 0f91858e77bfd9c164da55c3a9153968 -- 13d5a8edec348cba2bb29520b12dbee6 +deepgemm/f8e8fb5830fa5cda6e4ea73d360bb3f21f87a3ca: +- 07488ed395a262f652b63d0b1c1bb3a8 +- 1101099cb0a9c0489f9e4e49a719941a +- 111cfba37978c3bad26617cb42407970 +- 13852897dd05c1c29f2e098679875722 +- 1bedb45cd3faaa1beb45315299a4ecf6 - 21f12ada209fe9ad6c6009f561a0cd11 - 22d84d61c015ce63906f0646f4846acd -- 2e4c8c5460348a30b6553faafb7b57a5 -- 48892eb37bd139524e38d20fdb33ae80 -- 4ee346a64e951b3f76494801cd84eb0f -- 52959b03a674ef11e8b87c1411f6bcb1 +- 237d15df60fc034aca77d3c5b7f03598 +- 26d0a704cd6c7330a4daff026b2b7107 +- 2dcd97c72e333cd78191bc23d805abc7 +- 30234ac527d75a73256c5ed42ee519ee +- 361e84a339ee6b1484fd9fafc71fb16b +- 36e0a4342ea5ab161980d2615cc52b5c +- 39542d4f0093a3e36cb9271009e6b2e3 +- 3aa5495413dc1ef0867363ac9e8a456f +- 3e396b9c2dd15153095501aca1eeaf57 +- 4ab2bf20ab6672b94a9c2e65d238c17e - 5484c97b9f01cb095f53181bdc466ebc +- 5c074791db3d3688a19e2c199714abd0 +- 5f44dd92c5e5991465da86be94e3ff02 - 6274de12f364a929804285f8eaf01433 - 653b0b4e85486967a02c2162029cd21e +- 66f195fbe870a55634655b189db2d3c8 - 67a95a5fc3d2b87b6b365ed8efe7b24c +- 6c6222a36229587144260fa0ab77519c +- 6d9ca4deed2b000a813f6ed12f5cfc20 +- 72d393793b34c5722d08afd3fbc0f3c9 - 77759e9f29e4634768de17c5abdb83bb +- 7bdd11ae0471191b4a9f84581b9a1900 +- 7fcfb4f0f4f510830ff3bc9a14ef78fa +- 812f76ad364acc9035708909d4cd264b +- 83f6251c3d09c75dabef188fd4d3795a - 8436f7068d022ade5b2fc2f6fda4f6b7 - 87692b0010a14ad2df1db438f08998b5 - 885ab51cee88892f338f29c549e3b12a -- 88e4652508b73d182fb1d51e9026243e -- 9b6717ab7ff49cefc25ce1d3038005ab +- 8fd9a6344f2ba9855586c2a589100c70 +- 95b8e1175d420f5a05bc9fb0d0c55214 - 9bf6bbf8da71d31836279a87ffafe108 -- a23ac3ea1e92017854317062423f4ee7 +- 9e16e23b6894db65b377f93673dc733f +- a359f41b0ec5d67c1d9fda93dc65c5b5 +- a3645795bcf4bad7c333094975c558b6 - a775f6a60d47cd428f4c04289d4e8cb3 - a9f87d66fb89c05e1ed20a9459c54f68 -- afaecd9932c4cc84d10399adfb7ca3b8 -- afe2fecfac610c0f43c1e8574f6df20f - b2bc25c5c46087cd2fbb47808aad63c6 - b3758659dcee781c211c3b2468ac0da6 +- b6842c60f77e3a698126c3ef07bf01ac - bb67f82c43f17a0ab39f7c3a1077deab -- be2148d2c15ea64f74b34013faf30c1f -- c2800ca60c791a6d7f916a834d319c42 -- c82e69d0ed6bfa624c098542dcdef6a1 -- cce10c1227254a2067218a2f1ca1a0a7 -- d47ce388c77f9905e976fb73387d8c85 -- d72cc371017f747f0acb8897d9fb9f58 +- bcf2b578070dc8844a45e2a7dc3bdb28 +- bd6e5d6f717b238a42cae95c8ce49fcc +- c5cbc0c2bcf948b9ecebf3b7032bea38 +- c9c14f27fef150d389fc1481be3750b8 +- c9c4b55720e913abc707719aaf620548 +- cc1d7eb45036bdc86b2332f9e9131445 +- d0efeea676b9b4ee3c0fc9b9f902144c +- d10548c385eb50670ddd74b639f1f306 - d811f727a66051ba59548a8fd8a89aef - daca7269dfe56d06b9283885405d3375 - dbb93717b3e1535725e0becf51fd4573 - df608f12b8310e2ed4a00e8c6b921f6e -- e153309968237289f0071028ea41a464 - e2c811caa193a5af88c09eb531b6246e -- e413f583433fad2476c8546b9ab57661 -- e57cebe57da8e151279c1f5fe1ce767f - ea07df16c9a083277f55ae219c85f39d -- ee6355a9aae64339a97883f2e53d0862 +- ef5544cabdf0490063f2b2959f62a8cf - ef712bc72e01afbfdbceef82c6b49174 -- f611e187214aebd4da3f46cfdd5f8d76 +- f43084b87a14bcf56920e1e3b1ad2dba +- f5a3009221d096c818cf26fbdb4d9693 +- f5d68cc5860baa1de18f4aee4ccb0cc2 +- f7f27b18dae31db0aad429bfaaf4615e +- fdce0d8a612fe46f833812da73fb2961 dlpack/v1.0: - 86d243199750679d643dd53b75c7d6ab flashmla/1408756a88e52a25196b759eaf8db89d2b51b5a1: diff --git a/scripts/attribution/scan/metadata/msa.yml b/scripts/attribution/scan/metadata/msa.yml new file mode 100644 index 000000000000..ab212fc1b664 --- /dev/null +++ b/scripts/attribution/scan/metadata/msa.yml @@ -0,0 +1,5 @@ +name: msa +description: MiniMax Sparse Attention (fmha_sm100) kernels for SM100 sparse attention +source: submodule +directory_matches: +- 3rdparty/MSA diff --git a/scripts/build_cpp_examples.py b/scripts/build_cpp_examples.py deleted file mode 100644 index cb2591acfea2..000000000000 --- a/scripts/build_cpp_examples.py +++ /dev/null @@ -1,88 +0,0 @@ -import argparse -import contextlib -import logging -import os -import platform -import shutil -import subprocess -from os import PathLike -from pathlib import Path - - -@contextlib.contextmanager -def working_directory(path: PathLike): - """Changes working directory and returns to previous on exit.""" - prev_cwd = Path.cwd() - os.chdir(path) - try: - yield - finally: - os.chdir(prev_cwd) - - -def build_cpp_examples(build_dir: PathLike, trt_dir: PathLike, - enable_multi_device: str, loglevel: int) -> None: - logging.basicConfig(level=loglevel, - format='%(asctime)s - %(levelname)s - %(message)s') - # Convert input paths to pathlib.Path objects - build_dir = Path(build_dir) - trt_dir = Path(trt_dir) - - assert trt_dir.is_dir() - - def cmake_parse(path: PathLike) -> str: - return str(path).replace("\\", "/") - - # Remove the build directory if it exists - if build_dir.exists(): - logging.info(f"Removed directory: {build_dir}") - shutil.rmtree(build_dir) - - # Create the build directory - build_dir.mkdir(parents=True, exist_ok=True) - - # Change to the build directory - with working_directory(build_dir): - # Run CMake with the specified TensorRT directories - generator = ["-GNinja"] if platform.system() == "Windows" else [] - generate_command = [ - 'cmake', - '-S', - '..', - '-B', - '.', - f'-DTensorRT_ROOT={cmake_parse(trt_dir)}', - f'-DENABLE_MULTI_DEVICE={enable_multi_device}', - ] + generator - logging.info(f"Executing {generate_command}") - subprocess.run(generate_command, check=True) - - # Build the project using make - build_command = ["cmake", "--build", ".", "--config", "Release"] - logging.info(f"Executing {build_command}") - subprocess.run(build_command, check=True) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Build C++ examples') - parser.add_argument('--build-dir', - default='examples/cpp/executor/build', - help='Build directory path') - parser.add_argument('--trt-dir', - default='/usr/local/tensorrt', - help='TensorRT directory path') - parser.add_argument('--enable-multi-device', - default='ON', - help='Enable multi device support (requires MPI)') - parser.add_argument('-v', - '--verbose', - help="verbose", - action="store_const", - dest="loglevel", - const=logging.DEBUG, - default=logging.INFO) - cli = parser.parse_args() - - args = vars(cli) - print(args) # Log on Jenkins instance. - build_cpp_examples(**args) diff --git a/scripts/build_wheel.py b/scripts/build_wheel.py index 263f061d14b6..a38a1ddb686e 100755 --- a/scripts/build_wheel.py +++ b/scripts/build_wheel.py @@ -264,9 +264,18 @@ def setup_conan(scripts_dir, venv_python): return venv_conan +def _fmha_generation_stamp(fmha_v2_cu_dir: Path) -> Path: + # Written as the last step of generate_fmha_cu; its absence means a + # previous generation was interrupted and the directory contents cannot + # be trusted (the bare directory exists from the moment generation + # starts). + return fmha_v2_cu_dir / ".generation_complete" + + def generate_fmha_cu(project_dir, venv_python): fmha_v2_cu_dir = project_dir / "cpp/tensorrt_llm/kernels/contextFusedMultiHeadAttention/fmha_v2_cu" fmha_v2_cu_dir.mkdir(parents=True, exist_ok=True) + _fmha_generation_stamp(fmha_v2_cu_dir).unlink(missing_ok=True) fmha_v2_dir = project_dir / "cpp/kernels/fmha_v2" @@ -316,6 +325,11 @@ def move_if_updated(src, dst): move_if_updated(cu_file, dst_file) generated_files.add(str(dst_file.resolve())) + if not generated_files: + raise RuntimeError( + f"FMHA generation produced no *_sm*.cu files in {fmha_v2_cu_dir}; " + "generation may have failed silently.") + # Remove extra files for root, _, files in os.walk(fmha_v2_cu_dir): for file in files: @@ -323,6 +337,8 @@ def move_if_updated(src, dst): if file_path not in generated_files: os.remove(file_path) + _fmha_generation_stamp(fmha_v2_cu_dir).touch() + def create_cuda_stub_links(cuda_stub_dir: str, missing_libs: list[str]) -> str: """ @@ -388,6 +404,8 @@ def generate_python_stubs_linux(venv_python: Path, deep_ep: bool, binding_lib_name: str): build_run(f"\"{venv_python}\" -m pip install nanobind") build_run(f"\"{venv_python}\" -m pip install pybind11-stubgen") + nanobind_stubgen_patterns = get_project_dir( + ) / "scripts" / "nanobind_stubgen.patterns" env_stub_gen = os.environ.copy() cuda_home_dir = env_stub_gen.get("CUDA_HOME") or env_stub_gen.get( @@ -405,8 +423,10 @@ def generate_python_stubs_linux(venv_python: Path, deep_ep: bool, link_dir = None try: - build_run(f"\"{venv_python}\" -m nanobind.stubgen -m bindings -r -O .", - env=env_stub_gen) + build_run( + f"\"{venv_python}\" -m nanobind.stubgen -m bindings -r -O . " + f"-p \"{nanobind_stubgen_patterns}\" -q", + env=env_stub_gen) # Pre-import torch so deep_gemm_cpp_tllm's FP4 scalar-type registration # succeeds; CLI args after `-c ...` land in sys.argv[1:] for argparse. build_run( @@ -489,7 +509,6 @@ def main(*, job_count: int = None, extra_cmake_vars: Sequence[str] = tuple(), extra_make_targets: str = "", - trt_root: str = '/usr/local/tensorrt', nccl_root: str = None, nixl_root: str = None, mooncake_root: str = None, @@ -505,7 +524,6 @@ def main(*, install: bool = False, skip_building_wheel: bool = False, linking_install_binary: bool = False, - benchmarks: bool = False, micro_benchmarks: bool = False, nvtx: bool = False, skip_stubs: bool = False, @@ -542,21 +560,6 @@ def main(*, no_venv, yes=yes) - # Ensure base TRT is installed (check inside the venv) - try: - check_output([str(venv_python), "-m", "pip", "show", "tensorrt"]) - except CalledProcessError: - error_msg = "TensorRT was not installed properly." - if on_windows: - error_msg += ( - " Please download the TensorRT zip file manually," - " install it and relaunch build_wheel.py." - " See https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html#installing-zip for more details." - ) - else: - error_msg += f" Please install tensorrt into the venv using \"`{venv_python}` -m pip install tensorrt\" and relaunch build_wheel.py" - raise RuntimeError(error_msg) - if cuda_architectures is not None: if "70-real" in cuda_architectures: raise RuntimeError("Volta architecture is deprecated support.") @@ -612,9 +615,6 @@ def main(*, # Don't include duplicate conditions cmake_def_args.extend(set(extra_cmake_vars)) - if trt_root is not None: - cmake_def_args.append(f"-DTensorRT_ROOT={trt_root}") - if nccl_root is not None: cmake_def_args.append(f"-DNCCL_ROOT={nccl_root}") @@ -670,7 +670,7 @@ def main(*, "-- BOLT: Forcing NVRTC_DYNAMIC_LINKING=ON (static NVIDIA libs lack relocations)" ) - targets = ["tensorrt_llm", "nvinfer_plugin_tensorrt_llm"] + targets = ["tensorrt_llm"] if cpp_only: build_pyt = "OFF" @@ -687,9 +687,6 @@ def main(*, build_deep_gemm = "ON" build_flash_mla = "ON" - if benchmarks: - targets.append("benchmarks") - if micro_benchmarks: targets.append("micro_benchmarks") build_micro_benchmarks = "ON" @@ -698,13 +695,11 @@ def main(*, disable_nvtx = "OFF" if nvtx else "ON" - if not on_windows: - targets.append("executorWorker") - source_dir = get_source_dir() fmha_v2_cu_dir = project_dir / "cpp/tensorrt_llm/kernels/contextFusedMultiHeadAttention/fmha_v2_cu" - if clean or generate_fmha or not fmha_v2_cu_dir.exists(): + if (clean or generate_fmha + or not _fmha_generation_stamp(fmha_v2_cu_dir).exists()): generate_fmha_cu(project_dir, venv_python) with working_directory(build_dir): @@ -776,14 +771,26 @@ def main(*, if cache_dir.exists(): clear_folder(cache_dir) - install_file = copy + def safe_copy(src, dst): + """Copy a file, replacing a destination symlink with a real file.""" + src_path = Path(src) + dst_path = Path(dst) + if dst_path.is_dir(): + dst_path = dst_path / src_path.name + if dst_path.is_symlink(): + dst_path.unlink() + return copy(src_path, dst_path) + + install_file = safe_copy # Wrapper for copytree that checks if source and destination are the same def safe_copytree(src, dst, dirs_exist_ok=True): """Copy tree, but skip if source and destination resolve to the same directory.""" src_path = Path(src).resolve() - dst_path = Path(dst).resolve() - if src_path == dst_path: + dst_path = Path(dst) + if dst_path.is_symlink(): + dst_path.unlink() + elif src_path == dst_path.resolve(): # Source and destination are the same, skip copying return if dst_path.exists() and dirs_exist_ok: @@ -798,7 +805,7 @@ def symlink_remove_dst(src, dst): dst = os.path.abspath(dst) if os.path.isdir(dst): dst = os.path.join(dst, os.path.basename(src)) - if os.path.exists(dst): + if os.path.lexists(dst): os.remove(dst) os.symlink(src, dst) @@ -807,7 +814,7 @@ def symlink_remove_dst(src, dst): def symlink_remove_dst_tree(src, dst, dirs_exist_ok=True): src = os.path.abspath(src) dst = os.path.abspath(dst) - if dirs_exist_ok and os.path.exists(dst): + if dirs_exist_ok and os.path.lexists(dst): os.remove(dst) os.symlink(src, dst) @@ -932,18 +939,11 @@ def copy_resolving_symlink(src_path, dst_path): lib_dir / "tensorrt_llm.dll") install_file(build_dir / f"tensorrt_llm/thop/th_common.dll", lib_dir / "th_common.dll") - install_file( - build_dir / f"tensorrt_llm/plugins/nvinfer_plugin_tensorrt_llm.dll", - lib_dir / "nvinfer_plugin_tensorrt_llm.dll") else: install_file(build_dir / "tensorrt_llm/libtensorrt_llm.so", lib_dir / "libtensorrt_llm.so") install_file(build_dir / "tensorrt_llm/thop/libth_common.so", lib_dir / "libth_common.so") - install_file( - build_dir / - "tensorrt_llm/plugins/libnvinfer_plugin_tensorrt_llm.so", - lib_dir / "libnvinfer_plugin_tensorrt_llm.so") if os.path.exists( build_dir / "tensorrt_llm/executor/cache_transmission/ucx_utils/libtensorrt_llm_ucx_wrapper.so" @@ -1028,23 +1028,9 @@ def copy_resolving_symlink(src_path, dst_path): clear_folder(deep_gemm_dir) deep_gemm_dir.rmdir() - bin_dir = pkg_dir / "bin" - if bin_dir.exists(): - clear_folder(bin_dir) - bin_dir.mkdir(parents=True, exist_ok=True) - - if not on_windows: - install_file(build_dir / "tensorrt_llm/executor_worker/executorWorker", - bin_dir / "executorWorker") - scripts_dir = pkg_dir / "scripts" if scripts_dir.exists(): clear_folder(scripts_dir) - scripts_dir.mkdir(parents=True, exist_ok=True) - - if not on_windows: - install_file(project_dir / "docker/common/install_tensorrt.sh", - scripts_dir / "install_tensorrt.sh") if not cpp_only: @@ -1273,10 +1259,6 @@ def add_arguments(parser: ArgumentParser): help="Additional make targets to build. Example: \"target_1 target_2\"", nargs="+", default=[]) - parser.add_argument( - "--trt_root", - default="/usr/local/tensorrt", - help="Directory containing TensorRT headers and libraries") parser.add_argument("--nccl_root", help="Directory containing NCCL headers and libraries") parser.add_argument("--nixl_root", @@ -1313,9 +1295,6 @@ def add_arguments(parser: ArgumentParser): help= "Install the built binary by creating symbolic links instead of copying files" ) - parser.add_argument("--benchmarks", - action="store_true", - help="Build the benchmarks for the C++ runtime") parser.add_argument("--micro_benchmarks", action="store_true", help="Build the micro benchmarks for C++ components") diff --git a/scripts/generate_config_database_tests.py b/scripts/generate_config_database_tests.py index 10fdfbc66e26..a7958ec43fb8 100644 --- a/scripts/generate_config_database_tests.py +++ b/scripts/generate_config_database_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,6 +28,17 @@ import yaml + +class _DoubleQuoted(str): + """str subclass that yaml.dump emits with double quotes.""" + + +def _double_quoted_representer(dumper, data): + return dumper.represent_scalar("tag:yaml.org,2002:str", str(data), style='"') + + +yaml.add_representer(_DoubleQuoted, _double_quoted_representer) + SCRIPT_DIR = Path(__file__).parent.resolve() _REPO_ROOT = SCRIPT_DIR.parent @@ -42,7 +53,7 @@ ) REPO_ROOT = _REPO_ROOT -PERF_SANITY_DIR = REPO_ROOT / "tests" / "scripts" / "perf-sanity" +PERF_SANITY_DIR = REPO_ROOT / "tests" / "scripts" / "perf-sanity" / "aggregated" TEST_LIST_PATH = ( REPO_ROOT / "tests" / "integration" / "test_lists" / "qa" / "llm_config_database.yml" ) @@ -71,12 +82,17 @@ def generate_server_name(recipe: Recipe) -> str: """Generate a unique server name from recipe.""" model_slug = recipe.model.replace("/", "_").replace("-", "_").replace(".", "_") name = f"{model_slug}_{recipe.isl}_{recipe.osl}_conc{recipe.concurrency}_gpu{recipe.num_gpus}" + if recipe.profile: + name = f"{name}_{recipe.profile}" return name def generate_client_name(recipe: Recipe) -> str: """Generate client config name.""" - return f"con{recipe.concurrency}_isl{recipe.isl}_osl{recipe.osl}" + name = f"con{recipe.concurrency}_isl{recipe.isl}_osl{recipe.osl}" + if recipe.profile: + name = f"{name}_{recipe.profile}" + return name def recipe_to_server_config(recipe: Recipe, llm_api_config: dict) -> dict: @@ -85,8 +101,22 @@ def recipe_to_server_config(recipe: Recipe, llm_api_config: dict) -> dict: if not model_name: raise ValueError(f"Model not found in MODEL_NAME_MAPPING: {recipe.model}") + # Force a fixed number of accepted speculative-decoding tokens so perf + # measurements are deterministic across recipes that enable spec decoding. + max_draft_len = (llm_api_config.get("speculative_config") or {}).get("max_draft_len") + spec_env = ( + { + "server_env_var": _DoubleQuoted( + f"TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS={max_draft_len}" + ) + } + if max_draft_len is not None + else {} + ) + server_config = { "name": generate_server_name(recipe), + **spec_env, "model_name": model_name, "gpus": recipe.num_gpus, # Enable scenario-only matching for baseline comparison diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index 70fa8145c55c..fc73c29d8d90 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -32,6 +32,8 @@ from examples.configs.database.database import ( # noqa: E402 CURATED_LIST_PATH, DATABASE_LIST_PATH, + PROFILE_DISPLAY_NAMES, + PROFILE_ORDER, CuratedRecipeList, RecipeList, assign_profile, @@ -90,9 +92,9 @@ "display_name": "Kimi-K2-Thinking (NVFP4)", "url": "https://huggingface.co/nvidia/Kimi-K2-Thinking-NVFP4", }, - "MiniMaxAI/MiniMax-M3": { - "display_name": "MiniMax-M3 (BF16)", - "url": "https://huggingface.co/MiniMaxAI/MiniMax-M3", + "MiniMaxAI/MiniMax-M3-MXFP8": { + "display_name": "MiniMax-M3 (MXFP8)", + "url": "https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8", }, } @@ -114,6 +116,9 @@ class RecipeRow: config_filename: str config_github_url: str config_raw_url: str + profile: str | None + validated_trtllm_commit: str | None + validated_trtllm_version: str | None @dataclass(frozen=True) @@ -167,7 +172,7 @@ def build_curated_rows(yaml_path: Path) -> list[CuratedRow]: return rows -def build_rows(yaml_path) -> list[RecipeRow]: +def build_rows(yaml_path: Path) -> list[RecipeRow]: recipe_list = RecipeList.from_yaml(Path(yaml_path)) model_groups = defaultdict(lambda: defaultdict(list)) @@ -189,7 +194,7 @@ def build_rows(yaml_path) -> list[RecipeRow]: for key in sorted_keys: entries = subgroups[key] - entries.sort(key=lambda x: x.concurrency) + entries.sort(key=lambda x: (x.concurrency, PROFILE_ORDER.get(x.profile, -1))) for idx, entry in enumerate(entries): gpu = entry.gpu @@ -200,7 +205,11 @@ def build_rows(yaml_path) -> list[RecipeRow]: conc = entry.concurrency config_path = entry.config_path - profile = assign_profile(len(entries), idx, conc) + performance_profile = ( + PROFILE_DISPLAY_NAMES[entry.profile] + if entry.profile + else assign_profile(len(entries), idx, conc) + ) command = f"trtllm-serve {model} --config ${{TRTLLM_DIR}}/{config_path}" @@ -224,18 +233,23 @@ def build_rows(yaml_path) -> list[RecipeRow]: concurrency=conc, config_path=config_path, gpu_display=gpu_display, - performance_profile=profile, + performance_profile=performance_profile, command=command, config_filename=config_filename, config_github_url=config_github_url, config_raw_url=config_raw_url, + profile=entry.profile, + validated_trtllm_commit=entry.validated_trtllm_commit, + validated_trtllm_version=entry.validated_trtllm_version, ) ) return rows -def generate_json(yaml_path: Path, output_file: Path, curated_yaml_path: Path | None = None): +def generate_json( + yaml_path: Path, output_file: Path, curated_yaml_path: Path | None = None +) -> None: rows = build_rows(yaml_path) source_path = Path(yaml_path) @@ -266,7 +280,9 @@ def generate_json(yaml_path: Path, output_file: Path, curated_yaml_path: Path | payload = { "source": source, "models": models, - "entries": [asdict(r) for r in rows], + "entries": [ + {key: value for key, value in asdict(row).items() if value is not None} for row in rows + ], "curated_entries": curated_entries, } diff --git a/scripts/generate_duration.py b/scripts/generate_duration.py deleted file mode 100644 index fec429c305e6..000000000000 --- a/scripts/generate_duration.py +++ /dev/null @@ -1,75 +0,0 @@ -import argparse -import glob -import json -import os - -# Parse command-line arguments -parser = argparse.ArgumentParser(description="Generate test duration file.") -parser.add_argument( - "--duration-file", - type=str, - default="new_test_duration.json", - help="Path to the output duration file (default: new_test_duration.json)") -parser.add_argument( - "--cluster", - type=str, - default=None, - help="Cluster name (e.g. 'aws_dfw'). When set, writes " - "tests/integration/defs/.test_durations_ relative to the " - "repo root instead of --duration-file.") -args = parser.parse_args() - -# Define the directory containing the test result folders -TEST_RESULTS_DIR = os.getcwd() - -# Define the output file paths -FULL_RESULT_LOG = "full_result.log" -if args.cluster: - _repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - NEW_TEST_DURATION = os.path.join(_repo_root, "tests", "integration", "defs", - f".test_durations_{args.cluster}") -else: - NEW_TEST_DURATION = args.duration_file - -# Step 1: Prepare full_result.log -with open(FULL_RESULT_LOG, 'w') as full_result_file: - print(f"TEST_RESULTS_DIR: {TEST_RESULTS_DIR}") - for report_csv in glob.glob(os.path.join(TEST_RESULTS_DIR, '*/report.csv')): - print(f"Processing {report_csv}...") - with open(report_csv, 'r') as csv_file: - for line in csv_file: - if 'passed' in line: - full_result_file.write(line) - -# Step 2: Generate new_test_duration.json -test_durations = {} - -# Read the full_result.log file line by line -with open(FULL_RESULT_LOG, 'r') as file: - for line in file: - # Extract the first column and the last column - columns = line.strip().split(',') - first_column = columns[0] - last_column = columns[-1] - - # Remove from left to first '/' in the first column - test_name = first_column.split('/', 1)[-1] - # Replace \"\" with \" and ]\" with ] in case we got these in names from report.csv - # which will broken the json parse - test_name = test_name.replace(']\"', ']').replace('\"\"', '\"') - - try: - last_column = float(last_column) - except ValueError: - print( - f"Warning: Could not convert {last_column} to float. Skipping.") - continue - - # Add to the test duration dictionary - test_durations[test_name] = last_column - -# Write the test durations to the new test duration file -with open(NEW_TEST_DURATION, 'w') as file: - json.dump(test_durations, file, indent=3) - -print(f"Test durations have been written to {NEW_TEST_DURATION}") diff --git a/scripts/get_wheel_from_package.py b/scripts/get_wheel_from_package.py index cb604482c27b..f8dc16652361 100644 --- a/scripts/get_wheel_from_package.py +++ b/scripts/get_wheel_from_package.py @@ -78,27 +78,11 @@ def get_wheel_from_package(arch, artifact_path, timeout): build_dir = llm_root / "build" build_dir.mkdir(parents=True, exist_ok=True) - benchmarks_dir = llm_root / "cpp" / "build" / "benchmarks" - benchmarks_dir.mkdir(parents=True, exist_ok=True) - wheel_files = glob.glob(str(tmp_dir / "tensorrt_llm*.whl")) for wheel_file in wheel_files: shutil.move(wheel_file, str(build_dir)) print(f"Moved wheel file: {wheel_file} -> {build_dir}") - benchmark_files = [ - "bertBenchmark", "gptManagerBenchmark", "disaggServerBenchmark" - ] - - for benchmark in benchmark_files: - src_path = tmp_dir / "benchmarks" / "cpp" / benchmark - if src_path.exists(): - dst_path = benchmarks_dir / benchmark - shutil.copy2(src_path, dst_path) - print(f"Copied benchmark file: {src_path} -> {dst_path}") - else: - print(f"Warning: Benchmark file not found: {src_path}") - shutil.rmtree(tmp_dir) if os.path.exists(tarfile_name): diff --git a/scripts/nanobind_stubgen.patterns b/scripts/nanobind_stubgen.patterns new file mode 100644 index 000000000000..278dc4b92207 --- /dev/null +++ b/scripts/nanobind_stubgen.patterns @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +^bindings\.internal\.batch_manager\.kv_cache_manager_v2\.CachedCudaEvent\.NULL$: + \from typing import ClassVar + NULL: ClassVar[CachedCudaEvent] diff --git a/scripts/test_to_stage_mapping.py b/scripts/test_to_stage_mapping.py index 04626131b658..24f84c458f51 100644 --- a/scripts/test_to_stage_mapping.py +++ b/scripts/test_to_stage_mapping.py @@ -47,7 +47,7 @@ def _load_tests_file(path: str) -> List[str]: # Regex to parse Jenkins stage configurations from Groovy files -# Matches patterns like: "Stage-Name": ["platform", "yaml_file", split_id, split_count, gpu_count] +# Matches patterns like: "Stage-Name": ["platform", "yaml_file", split_id, split_count, gpu_count, node_count, runWithSbatch] # # Pattern breakdown: # "(?P[^"]+)" - Captures stage name in quotes (group 'stage') @@ -56,10 +56,12 @@ def _load_tests_file(path: str) -> List[str]: # "[^"]+" - Matches platform string in quotes (ignored) # ,\s* - Matches comma with optional whitespace # "(?P[^"]+)" - Captures yaml filename in quotes (group 'yml') -# (?:,\s*\d+)* - Matches zero or more comma-separated numbers (split_id, split_count, gpu_count) +# (?:,\s*(?:\d+|true|false))* - Matches zero or more comma-separated numbers or +# booleans (split_id, split_count, gpu_count, node_count, runWithSbatch) # \s*\] - Matches closing bracket with optional whitespace _STAGE_RE = re.compile( - r'"(?P[^"]+)"\s*:\s*\["[^"]+",\s*"(?P[^"]+)"(?:,\s*\d+)*\s*\]') + r'"(?P[^"]+)"\s*:\s*\["[^"]+",\s*"(?P[^"]+)"(?:,\s*(?:\d+|true|false))*\s*\]' +) def _extract_terms(entry): @@ -85,6 +87,8 @@ def _parse_stage_mapping(path): yaml_to_stages = defaultdict(list) with open(path, 'r') as f: for line in f: + if line.lstrip().startswith('//'): + continue m = _STAGE_RE.search(line) if m: stage = m.group('stage') diff --git a/security_scanning/docs/poetry.lock b/security_scanning/docs/poetry.lock index 9f572444dc2f..596368e25524 100644 --- a/security_scanning/docs/poetry.lock +++ b/security_scanning/docs/poetry.lock @@ -33,14 +33,14 @@ files = [ [[package]] name = "annotated-types" -version = "0.7.0" +version = "0.8.0" description = "Reusable constraint types to use with typing.Annotated" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, - {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, + {file = "annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0"}, + {file = "annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7"}, ] [[package]] @@ -127,14 +127,14 @@ test = ["pytest (>=8.0)"] [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -828,14 +828,14 @@ files = [ [[package]] name = "soupsieve" -version = "2.8.4" +version = "2.9.1" description = "A modern CSS selector implementation for Beautiful Soup." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65"}, - {file = "soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e"}, + {file = "soupsieve-2.9.1-py3-none-any.whl", hash = "sha256:4f4477399246b7a0c720a88ca2454b11cd6bb9ae4c9d170140786e916776c14c"}, + {file = "soupsieve-2.9.1.tar.gz", hash = "sha256:c33e6605bbc71dd628b00c632d58ae607c22bade247e52553928f83bbb75b4ba"}, ] [[package]] @@ -1025,14 +1025,14 @@ test = ["flake8", "mypy", "pytest"] [[package]] name = "sphinxcontrib-mermaid" -version = "2.0.3" +version = "2.1.0" description = "Mermaid diagrams in your Sphinx-powered docs" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "sphinxcontrib_mermaid-2.0.3-py3-none-any.whl", hash = "sha256:f001ed36a55c108f6221a2d656a441c487ee30651b54db72b7c752a20c7a66e8"}, - {file = "sphinxcontrib_mermaid-2.0.3.tar.gz", hash = "sha256:a6865ef6b65b225c5403a3170de63a04a07227cada11a4a71a6b87b4f9ed185a"}, + {file = "sphinxcontrib_mermaid-2.1.0-py3-none-any.whl", hash = "sha256:417cd144ec4b28852f46ba653f02ce8e538881c812111671a4c30344e87f2112"}, + {file = "sphinxcontrib_mermaid-2.1.0.tar.gz", hash = "sha256:13c5f9ac395cb6abf403eca34e228dc9fb3a30c9d960dbf3e40e9a8cef969549"}, ] [package.dependencies] @@ -1198,4 +1198,4 @@ packaging = ">=24.0" [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "6229137ad4354a8ad47953bd64635fd210251b2104fc49baad0d65705acc73cc" +content-hash = "7af684263a6aff0d081aa130bca302515fac11c985e02a678c589f234cd97194" diff --git a/security_scanning/docs/pyproject.toml b/security_scanning/docs/pyproject.toml index 30472f07fd21..7bddd72b848b 100644 --- a/security_scanning/docs/pyproject.toml +++ b/security_scanning/docs/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "sphinx-copybutton (>=0.5.2,<0.6.0)", "autodoc-pydantic (>=2.2.0,<3.0.0)", "sphinx-togglebutton (>=0.4.5,<0.5.0)", - "sphinxcontrib-mermaid (>=2.0.3,<3.0.0)" + "sphinxcontrib-mermaid (>=2.1.0,<3.0.0)" ] diff --git a/security_scanning/examples/apps/poetry.lock b/security_scanning/examples/apps/poetry.lock index cbf25147aac4..d3e68f07d867 100644 --- a/security_scanning/examples/apps/poetry.lock +++ b/security_scanning/examples/apps/poetry.lock @@ -2,14 +2,14 @@ [[package]] name = "annotated-types" -version = "0.7.0" +version = "0.8.0" description = "Reusable constraint types to use with typing.Annotated" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, - {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, + {file = "annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0"}, + {file = "annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7"}, ] [[package]] @@ -34,14 +34,14 @@ trio = ["trio (>=0.32.0)"] [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -278,14 +278,14 @@ files = [ [[package]] name = "openai" -version = "2.45.0" +version = "2.49.0" description = "The official Python library for the openai API" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "openai-2.45.0-py3-none-any.whl", hash = "sha256:5df105f5f8c9b711fcb9d06d2d3888cebc82506db216484c14a4e53cdf651777"}, - {file = "openai-2.45.0.tar.gz", hash = "sha256:10d34ca9c5643bce775852fddbfc172505cb1d4de1ccd101696c3ecff358765d"}, + {file = "openai-2.49.0-py3-none-any.whl", hash = "sha256:b694201eaa42a1ccf2aa125fe29458150108fb22df1abfb55d7188599da81d8c"}, + {file = "openai-2.49.0.tar.gz", hash = "sha256:80f934333b5b83cef2fde9af7151dacaa72e150f43f92b7675f7647ca6157f48"}, ] [package.dependencies] @@ -299,9 +299,10 @@ tqdm = ">4" typing-extensions = ">=4.14,<5" [package.extras] -aiohttp = ["aiohttp", "httpx-aiohttp (>=0.1.9)"] -bedrock = ["botocore (>=1.40.0,<1.43) ; python_version < \"3.10\"", "botocore (>=1.40.0,<2) ; python_version >= \"3.10\""] +aiohttp = ["aiohttp (>=3.14.1)", "httpx-aiohttp (>=0.1.9)"] +bedrock = ["botocore (>=1.40.0,<2)"] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +httpx2 = ["anyio (>=4.10.0,<5)", "httpx (>=0.25.1,<1)", "httpx2 (>=2.7.0,<3)"] realtime = ["websockets (>=13,<16)"] voice-helpers = ["numpy (>=2.0.2)", "sounddevice (>=0.5.1)"] @@ -474,14 +475,14 @@ files = [ [[package]] name = "tqdm" -version = "4.68.4" +version = "4.70.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, + {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, + {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, ] [package.dependencies] diff --git a/security_scanning/examples/auto_deploy/poetry.lock b/security_scanning/examples/auto_deploy/poetry.lock index 3b450ff0c25c..06ea70c2fb9d 100644 --- a/security_scanning/examples/auto_deploy/poetry.lock +++ b/security_scanning/examples/auto_deploy/poetry.lock @@ -110,131 +110,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.1" +version = "3.14.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32"}, + {file = "aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7"}, + {file = "aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19"}, + {file = "aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71"}, + {file = "aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf"}, + {file = "aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7"}, + {file = "aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc"}, ] [package.dependencies] @@ -354,14 +354,14 @@ files = [ [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -543,14 +543,14 @@ all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == [[package]] name = "cuda-pathfinder" -version = "1.5.6" +version = "1.6.0" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, + {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, ] [[package]] @@ -799,14 +799,14 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth [[package]] name = "filelock" -version = "3.29.7" +version = "3.32.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, + {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, + {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, ] [[package]] @@ -1030,38 +1030,30 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.1" +version = "1.5.2" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "(sys_platform == \"linux\" or sys_platform == \"win32\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\")" files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, + {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, ] [package.extras] @@ -1116,14 +1108,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.23.0" +version = "1.25.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, + {file = "huggingface_hub-1.25.1-py3-none-any.whl", hash = "sha256:004d4e70350517e24c68a7dbb7dc5e40b2b6aefef8f94bf7a85f6f9835102ea5"}, + {file = "huggingface_hub-1.25.1.tar.gz", hash = "sha256:21129595ca7a753be479b319913e22cc8808361ac118bd76cc413db831b28a99"}, ] [package.dependencies] @@ -1984,6 +1976,7 @@ description = "Fast numerical expression evaluator for NumPy" optional = false python-versions = ">=3.10" groups = ["main"] +markers = "python_version == \"3.10\"" files = [ {file = "numexpr-2.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d0fab3fd06a04f6b86102552b26aa5d85e20ac7d8296c15764c726eeabae6cc8"}, {file = "numexpr-2.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:64ae5dfd62d74a3ef82fe0b37f80527247f3626171ad82025900f46ffca4b39a"}, @@ -2047,6 +2040,66 @@ files = [ [package.dependencies] numpy = ">=1.23.0" +[[package]] +name = "numexpr" +version = "2.14.2" +description = "Fast numerical expression evaluator for NumPy" +optional = false +python-versions = ">=3.11" +groups = ["main"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "numexpr-2.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2aa65ddc2243f19c6915f34ee0978b4a2df20f297230a793c4ee6d55f3472599"}, + {file = "numexpr-2.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf959e6df6cb603611c034b6cba7b03a361be0ad0b80b73f163fab95f5ccbb7f"}, + {file = "numexpr-2.14.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d534ecb456a4ae3995f99c8a5deb469bfff05d4ec610a7885c175c881d12f710"}, + {file = "numexpr-2.14.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f41170e9d0dbba76851e35d80cfa9f4ca5fe78628c5bf24d941cf3364940ab7a"}, + {file = "numexpr-2.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6acafb2fdbeaaa6681a8f1a1d8b3f7dcd33704baace7057b950754b258be7c43"}, + {file = "numexpr-2.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7ca9e71195b36cc7aeafe97347549e1e1c1e889ff700238782ef6447651ec26d"}, + {file = "numexpr-2.14.2-cp311-cp311-win32.whl", hash = "sha256:779129d50974e7d6d6581d322f75b8f8375e96215b6861a2d5460347997ef649"}, + {file = "numexpr-2.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:2f132777d7d425471c458af5617e023402f13f5006301eacf8a1a6e7118ea70c"}, + {file = "numexpr-2.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:f1de5c88515ed9fbcad42699a0e2b5821b4d0f0adb0da6fb7e009e5cb19d8493"}, + {file = "numexpr-2.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:606ceaf5722e295ef965ca591736fc26d9e5f13ad950a479e64cead1947f8a3d"}, + {file = "numexpr-2.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:790da022539fe7c37dc893acf530a91c2ca6964d7ba11f464131383729d058f3"}, + {file = "numexpr-2.14.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:327be9ee62251c173236dc620147ff2d0e732a32f5bad918d78a10082f502f63"}, + {file = "numexpr-2.14.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6a5d8fc7016bf6f6e1808b011510aa7c3bd75ec1407f7650874ec591db59f5e"}, + {file = "numexpr-2.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4b1ff261c3e69c4c59578d3a9ca6132603619d38ae1abe73325563bed3b9bbaf"}, + {file = "numexpr-2.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b8384592c49cb15a91caa54e2cd84d1ce18edb7af030bb76cd29b52e5dc155d"}, + {file = "numexpr-2.14.2-cp312-cp312-win32.whl", hash = "sha256:41cdeacf1b4e51c1143983ea61fcee68139ca47222b55a9265b4fa73826c4260"}, + {file = "numexpr-2.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:8fc55d14bcf17b3fe69213bea14f999451892b4690717008c66f2edfd6a085ce"}, + {file = "numexpr-2.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:806a4471310fe20aa7cb1b2816a6f5e508073a1ad1c2e18041b83e57066fad6a"}, + {file = "numexpr-2.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0741efbd75c284e709b0fd430c85c31982b44c9962922ba8a9cbbea1bf413321"}, + {file = "numexpr-2.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92b00c78664070e3af155c6be713a0a5d75d598647ce32a5609adb79a8f961d3"}, + {file = "numexpr-2.14.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:149ab5744a5222f07b1d60455c4021c754d395e44938944ac7c7c2495f7feb54"}, + {file = "numexpr-2.14.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fd2f5882a66a7792aa6614c68831aa20085b499d41422aedd001080624ebb14c"}, + {file = "numexpr-2.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:375d8bee15be42dab22100a0a3de05fe6689a2de853eca012858768a9a7e02ab"}, + {file = "numexpr-2.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c1ffaf805d8636c3f95d0996517ecf9684c9ac62d768030ca78d1d00af2b3504"}, + {file = "numexpr-2.14.2-cp313-cp313-win32.whl", hash = "sha256:449a57fb9d38de136e742b1fc429572b42f29778f1d695c3fe50ffec9d3c9a71"}, + {file = "numexpr-2.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:dd905922d7dce457947d54b84c7ac345cef37332b724445e159a5a1a2080ce2b"}, + {file = "numexpr-2.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:b02738853b9b5b8a995f6c680f8f6ef33e8f419395b8fa380e38690495fdb911"}, + {file = "numexpr-2.14.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:76e87c7bd70d721ce4d418e81f4fb7ecf9e7e67d7cea8102527b07fd3d3facf9"}, + {file = "numexpr-2.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:939c89f613b814e64bb568859397dc9f99b219c3ef681a72fb99a86e435262f9"}, + {file = "numexpr-2.14.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b20c1c55aba7812ff2f2c6a50006425d02282fabb1eaf8d75fe638ffcf6deb02"}, + {file = "numexpr-2.14.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bac00898930f962f360c3d763a8e2273fc931f65a1759ff1bf64b3cf13d65aee"}, + {file = "numexpr-2.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:022e61a3d5dbf5807746264b62126d1c2c24057ad90052478a4d4482ab2555c2"}, + {file = "numexpr-2.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1d4593e2c6fa060cd7441e8b6ef25c16321a6be2144b3c82d1e00885f1fb6e94"}, + {file = "numexpr-2.14.2-cp314-cp314-win32.whl", hash = "sha256:66f3b125b1104241322811de87918724d6709bf082dc0703722d0cecb7b29e82"}, + {file = "numexpr-2.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:ef576a1cded27ba2f3129bc3c42df452a1c498072680d560793f98b0024cd7e6"}, + {file = "numexpr-2.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8274c51ae1842948f3ae7fe6951a23dcf4ddcbeeaff3737e978e7740b754662d"}, + {file = "numexpr-2.14.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:f3526699350f94c6277fb16863773a1af9defd95a6f78bbd69b1f0338fd94756"}, + {file = "numexpr-2.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:91e7928435f14fcb351c0157000bce65122b897cc8b0df6bcc48251f25850a6d"}, + {file = "numexpr-2.14.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c66925deb968f0b5280f723e2bb5918c11e6be2ca60e9e1530006286ab44031d"}, + {file = "numexpr-2.14.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a404c9a55902572eec810068d06b79a7c99e96f0400f5a7d73f39dff5ec5e371"}, + {file = "numexpr-2.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:44dc6b1dfa9abcbfc9917297f0d2af7c87c16b6ecd45747a8e70f54399a3a2f9"}, + {file = "numexpr-2.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:93233040f4bed3bce5abb0c2d20aeb1074511f29cbaa9c14828f86bcfa44d321"}, + {file = "numexpr-2.14.2-cp314-cp314t-win32.whl", hash = "sha256:2aceefa08f8f86317fa6e8fe9f6dc20d24ab8365d715be4a26306acf406d2dbe"}, + {file = "numexpr-2.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:cd684ac9daa539fcdac3437678834797b29d7780cfaad71111745132d466d51f"}, + {file = "numexpr-2.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:2ef72de3d3dd466cb0c435cae7141c99b0f8091b1eae9d03dcb38690f56c3f79"}, + {file = "numexpr-2.14.2.tar.gz", hash = "sha256:e7144e83ea9e581f2273e0304f15836736c4e470e2bd2e378ce617662a1ca278"}, +] + +[package.dependencies] +numpy = ">=1.26.0" + [[package]] name = "numpy" version = "2.2.6" @@ -2495,61 +2548,55 @@ xml = ["lxml (>=4.9.2)"] [[package]] name = "pandas" -version = "3.0.3" +version = "3.0.5" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.11" groups = ["main"] markers = "python_version >= \"3.11\"" files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, + {file = "pandas-3.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2946e77e4a53cd248cbde631a12f0e51c8324ce354c3eba4d20147c1ad6f4282"}, + {file = "pandas-3.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71ecc8fb7ed1a7aa4392316b5309a6347e8e7f832f38fd897846b3a1457a9298"}, + {file = "pandas-3.0.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b173f5951ff6b8b0ec7675e20dff3c97b7e7a57dfcce387c2d7c5afe87cb7899"}, + {file = "pandas-3.0.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c0cf1dd9b55a22d105fc46c1b489af3bd42264fcba7c66297bf47a9a1d9c78a"}, + {file = "pandas-3.0.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0fac0010c75e4efb6b99e249c183a8993ce0dc95c240f9b120a5e67c727b7928"}, + {file = "pandas-3.0.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:08d24fe11a17dc33bd6e937dc9c665f9cba08fbdc9f657f405713515febe300d"}, + {file = "pandas-3.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b1261758dfb6cf12c3cff8300e21cefad30e7ec709abb4c24ac7318e6a52462a"}, + {file = "pandas-3.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:679f4e85b30ddb1515458ab1e788d3e260eae369b1f78da7a3aa4cac8ebf4a2a"}, + {file = "pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d"}, + {file = "pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc"}, + {file = "pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc"}, + {file = "pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34"}, + {file = "pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6"}, + {file = "pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7"}, + {file = "pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce"}, + {file = "pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41"}, + {file = "pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49"}, + {file = "pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b"}, + {file = "pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3"}, + {file = "pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b"}, + {file = "pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be"}, + {file = "pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58"}, + {file = "pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee"}, + {file = "pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6"}, + {file = "pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e"}, + {file = "pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36"}, + {file = "pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c"}, + {file = "pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0"}, + {file = "pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b"}, + {file = "pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94"}, + {file = "pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da"}, + {file = "pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92"}, + {file = "pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85"}, + {file = "pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca"}, + {file = "pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da"}, + {file = "pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a"}, + {file = "pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040"}, + {file = "pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd"}, + {file = "pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c"}, + {file = "pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea"}, + {file = "pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c"}, + {file = "pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712"}, ] [package.dependencies] @@ -2579,7 +2626,7 @@ postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psyc pyarrow = ["pyarrow (>=13.0.0)"] spss = ["pyreadstat (>=1.2.8)"] sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] +test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4,<9.1)", "pytest-xdist (>=3.6.1)"] timezone = ["pytz (>=2020.1)"] xml = ["lxml (>=5.3.0)"] @@ -2699,18 +2746,18 @@ psutil = "*" [[package]] name = "prompt-toolkit" -version = "3.0.52" +version = "3.0.53" description = "Library for building powerful interactive command lines in Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955"}, - {file = "prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855"}, + {file = "prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2"}, + {file = "prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6"}, ] [package.dependencies] -wcwidth = "*" +wcwidth = ">=0.1.4" [[package]] name = "propcache" @@ -3043,14 +3090,14 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2026.2" +version = "2026.3.post1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" groups = ["main"] files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, + {file = "pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815"}, + {file = "pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d"}, ] [[package]] @@ -3170,126 +3217,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.10" +version = "2026.7.19" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, + {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, + {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, + {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, + {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, + {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, + {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, + {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, + {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, + {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, + {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, + {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, + {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, + {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, + {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, + {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, + {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, + {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, + {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, + {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, + {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, + {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, + {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, ] [[package]] @@ -4097,14 +4144,14 @@ pyyaml = ["pyyaml"] [[package]] name = "tqdm" -version = "4.68.4" +version = "4.70.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, + {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, + {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, ] [package.dependencies] @@ -4153,14 +4200,14 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.17.0,<1.19)", "pre-commit", "pytest [[package]] name = "transformers" -version = "5.14.0" +version = "5.14.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.0-py3-none-any.whl", hash = "sha256:288287da4fdcdae0b611be32b5be20d4bcbf5caaa5f269292927fb1ff888d19d"}, - {file = "transformers-5.14.0.tar.gz", hash = "sha256:9e1a2b345c195d216afd10082394b915a82e339fa09e9013c32094ff835ca9db"}, + {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, + {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, ] [package.dependencies] @@ -4599,116 +4646,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.2" +version = "1.24.5" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, + {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, + {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, + {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, + {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, + {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, + {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, + {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, + {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, + {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, + {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, + {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, + {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, + {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, + {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, ] [package.dependencies] diff --git a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock deleted file mode 100644 index e64bad8d25d0..000000000000 --- a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock +++ /dev/null @@ -1,4529 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "accelerate" -version = "1.14.0" -description = "Accelerate" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "accelerate-1.14.0-py3-none-any.whl", hash = "sha256:e94390c2863b873be18f623f9df48a0d8fe5eff13ea7f1a00092b0a7904888c6"}, - {file = "accelerate-1.14.0.tar.gz", hash = "sha256:41b9c4377a54e0b460a959b0defa1b736e4ca0a2373252d9a539964c2afe3c8d"}, -] - -[package.dependencies] -huggingface_hub = ">=0.21.0" -numpy = ">=1.17" -packaging = ">=20.0" -psutil = "*" -pyyaml = "*" -safetensors = ">=0.4.3" -torch = ">=2.0.0" - -[package.extras] -deepspeed = ["deepspeed"] -dev = ["bitsandbytes", "datasets", "diffusers", "evaluate", "parameterized", "pytest (>=7.2.0)", "pytest-order", "pytest-subtests", "pytest-xdist", "rich", "ruff (==0.13.1)", "scikit-learn", "scipy", "timm", "torchdata (>=0.8.0)", "torchpippy (>=0.2.0)", "tqdm", "transformers"] -quality = ["ruff (==0.13.1)"] -rich = ["rich"] -sagemaker = ["sagemaker"] -test-dev = ["bitsandbytes", "datasets", "diffusers", "evaluate", "scikit-learn", "scipy", "timm", "torchdata (>=0.8.0)", "torchpippy (>=0.2.0)", "tqdm", "transformers"] -test-fp8 = ["torchao"] -test-prod = ["parameterized", "pytest (>=7.2.0)", "pytest-order", "pytest-subtests", "pytest-xdist"] -test-trackers = ["dvclive", "matplotlib", "swanlab[dashboard]", "tensorboard", "trackio", "wandb"] -testing = ["bitsandbytes", "datasets", "diffusers", "evaluate", "parameterized", "pytest (>=7.2.0)", "pytest-order", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "timm", "torchdata (>=0.8.0)", "torchpippy (>=0.2.0)", "tqdm", "transformers"] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "annotated-doc" -version = "0.0.4" -description = "Document parameters, class attributes, return types, and variables inline, with Annotated." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320"}, - {file = "annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4"}, -] - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "chardet" -version = "6.0.0.post1" -description = "Universal encoding detector for Python 3" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "chardet-6.0.0.post1-py3-none-any.whl", hash = "sha256:c894a36800549adf7bb5f2af47033281b75fdfcd2aa0f0243be0ad22a52e2dcb"}, - {file = "chardet-6.0.0.post1.tar.gz", hash = "sha256:6b78048c3c97c7b2ed1fbad7a18f76f5a6547f7d34dbab536cc13887c9a92fa4"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "cuda-bindings" -version = "13.3.1" -description = "Python bindings for CUDA" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, - {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, - {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, - {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, - {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, - {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, - {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, - {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, - {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, - {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, - {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, - {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, - {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, - {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, - {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, - {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, - {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, - {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, -] - -[package.dependencies] -cuda-pathfinder = ">=1.4.2" - -[package.extras] -all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] - -[[package]] -name = "cuda-pathfinder" -version = "1.5.6" -description = "Pathfinder for CUDA components" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, -] - -[[package]] -name = "cuda-toolkit" -version = "13.0.3" -description = "CUDA Toolkit meta-package" -optional = false -python-versions = "*" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f"}, -] - -[package.dependencies] -nvidia-cublas = {version = "==13.1.1.3.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cublas\" or extra == \"cusolver\")"} -nvidia-cuda-cupti = {version = "==13.0.85.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cupti\""} -nvidia-cuda-nvrtc = {version = "==13.0.88.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cublas\" or extra == \"nvrtc\")"} -nvidia-cuda-runtime = {version = "==13.0.96.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cudart\""} -nvidia-cufft = {version = "==12.0.0.61.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cufft\""} -nvidia-cufile = {version = "==1.15.1.6.*", optional = true, markers = "sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cufile\""} -nvidia-curand = {version = "==10.4.0.35.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"curand\""} -nvidia-cusolver = {version = "==12.0.4.66.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cusolver\""} -nvidia-cusparse = {version = "==12.6.3.3.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cusolver\" or extra == \"cusparse\")"} -nvidia-nvjitlink = {version = ">=13.0.88,<14", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cufft\" or extra == \"cusolver\" or extra == \"cusparse\" or extra == \"nvjitlink\")"} -nvidia-nvtx = {version = "==13.0.85.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"nvtx\""} - -[package.extras] -all = ["nvidia-cublas (==13.1.1.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-cccl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-culibos (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")", "nvidia-cuda-cupti (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-cuxxfilt (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvcc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-opencl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-profiler-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cufft (==12.0.0.61.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cufile (==1.15.1.6.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")", "nvidia-curand (==10.4.0.35.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusolver (==12.0.4.66.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-npp (==13.0.1.2.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvfatbin (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjpeg (==13.0.1.86.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvml-dev (==13.0.87.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvptxcompiler (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvtx (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cccl = ["nvidia-cuda-cccl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -crt = ["nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cublas = ["nvidia-cublas (==13.1.1.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cudart = ["nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cufft = ["nvidia-cufft (==12.0.0.61.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cufile = ["nvidia-cufile (==1.15.1.6.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")"] -culibos = ["nvidia-cuda-culibos (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")"] -cupti = ["nvidia-cuda-cupti (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -curand = ["nvidia-curand (==10.4.0.35.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cusolver = ["nvidia-cublas (==13.1.1.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusolver (==12.0.4.66.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cusparse = ["nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cuxxfilt = ["nvidia-cuda-cuxxfilt (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -npp = ["nvidia-npp (==13.0.1.2.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvcc = ["nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvcc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvfatbin = ["nvidia-nvfatbin (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvjitlink = ["nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvjpeg = ["nvidia-nvjpeg (==13.0.1.86.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvml = ["nvidia-nvml-dev (==13.0.87.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvptxcompiler = ["nvidia-nvptxcompiler (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvrtc = ["nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvtx = ["nvidia-nvtx (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvvm = ["nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -opencl = ["nvidia-cuda-opencl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -profiler = ["nvidia-cuda-profiler-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -sanitizer = ["nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] - -[[package]] -name = "dataproperty" -version = "1.1.1" -description = "Python library for extract property from data." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "dataproperty-1.1.1-py3-none-any.whl", hash = "sha256:cf026aa002dbd6c57c619ec6741ffd61ae7bf2f20481951d8af2dff44480340e"}, - {file = "dataproperty-1.1.1.tar.gz", hash = "sha256:a83af82a234edda5378a36fb092bc90dd554646c5e58202a310acf468ae81bc8"}, -] - -[package.dependencies] -mbstrdecoder = ">=1.0.0,<2" -typepy = {version = ">=1.3.2,<3", extras = ["datetime"]} - -[package.extras] -logging = ["loguru (>=0.4.1,<1)"] -test = ["pytest (>=6.0.1)", "pytest-md-report (>=0.6.2)", "tcolorpy (>=0.1.2)"] - -[[package]] -name = "datasets" -version = "5.0.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "datasets-5.0.0-py3-none-any.whl", hash = "sha256:7dd34927a0fd7046e98aad5cb9430e699c373238a15befa7b9bf22b991a7fee6"}, - {file = "datasets-5.0.0.tar.gz", hash = "sha256:83dbbbdb07a33b82192b8c419deb18739b138ee2ce1a322d55ce6b100954ec1a"}, -] - -[package.dependencies] -dill = ">=0.3.0,<0.4.2" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2026.4.0", extras = ["http"]} -httpx = "<1.0.0" -huggingface-hub = ">=0.25.0,<2.0" -multiprocess = "<0.70.20" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=21.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["torch (>=2.8.0)", "torchcodec (>=0.6.0)"] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "aiohttp", "decorator", "elasticsearch (>=7.17.12,<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "h5py", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark ; python_version < \"3.14\"", "lz4 ; python_version < \"3.14\"", "moto[server]", "nibabel (>=5.3.1)", "numba (>=0.56.4) ; python_version < \"3.14\"", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyiceberg[pyarrow,sql-sqlite]", "pylance", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "sqlalchemy", "teich (==0.1.1a76)", "tensorflow (>=2.16.0) ; python_version >= \"3.10\" and sys_platform != \"win32\" and python_version < \"3.14\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\" and sys_platform != \"win32\"", "tiktoken", "torch", "torch (>=2.8.0)", "torchcodec (>=0.7.0) ; python_version < \"3.14\"", "torchdata", "transformers", "transformers (>=4.42.0)", "trimesh (>=4.10.0)", "zstandard"] -docs = ["tensorflow (>=2.6.0)", "torch", "transformers"] -iceberg = ["pyiceberg (>=0.7.0)"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -mesh = ["trimesh (>=4.10.0)"] -nibabel = ["ipyniivue (==2.4.2)", "nibabel (>=5.3.2)"] -pdfs = ["pdfplumber (>=0.11.4)"] -quality = ["ruff (>=0.3.0)"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "aiohttp", "decorator", "elasticsearch (>=7.17.12,<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "h5py", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark ; python_version < \"3.14\"", "lz4 ; python_version < \"3.14\"", "moto[server]", "nibabel (>=5.3.1)", "numba (>=0.56.4) ; python_version < \"3.14\"", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyiceberg[pyarrow,sql-sqlite]", "pylance", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "sqlalchemy", "teich (==0.1.1a76)", "tensorflow (>=2.16.0) ; python_version >= \"3.10\" and sys_platform != \"win32\" and python_version < \"3.14\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\" and sys_platform != \"win32\"", "tiktoken", "torch (>=2.8.0)", "torchcodec (>=0.7.0) ; python_version < \"3.14\"", "torchdata", "transformers (>=4.42.0)", "trimesh (>=4.10.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "aiohttp", "decorator", "elasticsearch (>=7.17.12,<8.0.0)", "h5py", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark ; python_version < \"3.14\"", "lz4 ; python_version < \"3.14\"", "moto[server]", "nibabel (>=5.3.1)", "numba (>=0.56.4) ; python_version < \"3.14\"", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyiceberg[pyarrow,sql-sqlite]", "pylance", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "sqlalchemy", "teich (==0.1.1a76)", "tiktoken", "torch (>=2.8.0)", "torchcodec (>=0.7.0) ; python_version < \"3.14\"", "torchdata", "transformers (>=4.42.0)", "trimesh (>=4.10.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.4.1" -description = "serialize all of Python" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d"}, - {file = "dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2026.4.0" -description = "File-system specification" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2"}, - {file = "fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff (>=0.5)"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs (>2024.2.0)", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs (>2024.2.0)", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs (>2024.2.0)"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs (>2024.2.0)"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "backports-zstd ; python_version < \"3.14\"", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas (<3.0.0)", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard ; python_version < \"3.14\""] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\")" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "jinja2" -version = "3.1.6" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, - {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "jsonlines" -version = "4.0.0" -description = "Library with helpers for the jsonlines file format" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "jsonlines-4.0.0-py3-none-any.whl", hash = "sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55"}, - {file = "jsonlines-4.0.0.tar.gz", hash = "sha256:0c6d2c09117550c089995247f605ae4cf77dd1533041d366351f6f298822ea74"}, -] - -[package.dependencies] -attrs = ">=19.2.0" - -[[package]] -name = "lm-eval" -version = "0.4.7" -description = "A framework for evaluating language models" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "lm_eval-0.4.7-py3-none-any.whl", hash = "sha256:d84a52580468fdc1d812e511db36e86679b69ee27f5a5e3dbd50f233d0bec69f"}, - {file = "lm_eval-0.4.7.tar.gz", hash = "sha256:dcbef8722f363f58cfba36b6d783fc6bb17924b24b8da1684bf1ac835866208d"}, -] - -[package.dependencies] -accelerate = ">=0.26.0" -aiohttp = {version = "*", optional = true, markers = "extra == \"api\""} -datasets = ">=2.16.0" -dill = "*" -evaluate = ">=0.4.0" -jsonlines = "*" -more_itertools = "*" -numexpr = "*" -peft = ">=0.2.0" -pybind11 = ">=2.6.2" -pytablewriter = "*" -requests = {version = "*", optional = true, markers = "extra == \"api\""} -rouge-score = ">=0.0.4" -sacrebleu = ">=1.5.0" -scikit-learn = ">=0.24.1" -sqlitedict = "*" -tenacity = {version = "*", optional = true, markers = "extra == \"api\""} -tiktoken = {version = "*", optional = true, markers = "extra == \"api\""} -torch = ">=1.8" -tqdm = {version = "*", optional = true, markers = "extra == \"api\""} -tqdm-multiprocess = "*" -transformers = ">=4.1" -word2number = "*" -zstandard = "*" - -[package.extras] -all = ["lm_eval[anthropic]", "lm_eval[deepsparse]", "lm_eval[dev]", "lm_eval[gptq]", "lm_eval[hf-transfer]", "lm_eval[ibm-watsonx-ai]", "lm_eval[ifeval]", "lm_eval[japanese-leaderboard]", "lm_eval[mamba]", "lm_eval[math]", "lm_eval[multilingual]", "lm_eval[openai]", "lm_eval[promptsource]", "lm_eval[sentencepiece]", "lm_eval[sparseml]", "lm_eval[testing]", "lm_eval[vllm]", "lm_eval[wandb]", "lm_eval[zeno]"] -api = ["aiohttp", "requests", "tenacity", "tiktoken", "tqdm"] -deepsparse = ["deepsparse-nightly[llm] (>=1.8.0.20240404)"] -dev = ["mypy", "pre-commit", "pytest", "pytest-cov", "pytest-xdist"] -gptq = ["auto-gptq[triton] (>=0.6.0)"] -gptqmodel = ["gptqmodel (>=1.0.9)"] -hf-transfer = ["hf_transfer"] -ibm-watsonx-ai = ["ibm_watsonx_ai (>=1.1.22)"] -ifeval = ["immutabledict", "langdetect", "nltk (>=3.9.1)"] -japanese-leaderboard = ["emoji (==2.14.0)", "fugashi[unidic-lite]", "neologdn (==0.5.3)", "rouge_score (>=0.1.2)"] -mamba = ["causal-conv1d (==1.0.2)", "mamba_ssm"] -math = ["antlr4-python3-runtime (==4.11)", "sympy (>=1.12)"] -multilingual = ["jieba (>=0.42.1)", "nagisa (>=0.2.7)", "pycountry"] -neuronx = ["optimum[neuronx]"] -optimum = ["optimum[openvino]"] -promptsource = ["promptsource (>=0.2.3)"] -sentencepiece = ["sentencepiece (>=0.1.98)"] -sparseml = ["sparseml-nightly[llm] (>=1.8.0.20240404)"] -testing = ["pytest", "pytest-cov", "pytest-xdist"] -vllm = ["vllm (>=0.4.2)"] -wandb = ["numpy", "pandas", "wandb (>=0.16.3)"] -zeno = ["pandas", "zeno-client"] - -[[package]] -name = "lxml" -version = "6.1.1" -description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "lxml-6.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:09dd5b7075dc2f7709654a46543ba1ea3c2e217b2ed8fbd413a8a945a0f40f60"}, - {file = "lxml-6.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f6ac4ef4d82dff54670227a69c67782ae0b811b5cf6b17954f1e8f7502fc0d1d"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:556e94a63c9b04716f8e4de2abb65775061f846e89331b6c5be79183a24f98ea"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5c6bf403fbb3b3e348a561a5f4f0b9961835657981c802a1df03653eef8a9074"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1dde6131244bba38a17c745836ba190bc753fd73c9291666287fd0a3fa3dcf30"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98fc784c2c1440667aeedf8465bdfe10208acf0ead656a2c68627299f546b315"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux_2_28_i686.whl", hash = "sha256:add8cf6ddf9a65116119a28ece0f7886e30af27ba724a7594305f1d1b58a92a1"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:cf9d57306d848218f3601fee7601fab1a327c942d56e2e97610583cb4dd74206"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88136950da4d13c318bde414ce10219931937851327f44328f2df4d2c4614067"}, - {file = "lxml-6.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cecdd5dfdc87b1fd87dbf81d4b037a544f47f4c744200a67013771682d67686a"}, - {file = "lxml-6.1.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cd312b9692e831d2ffcad61eab31d91d4b4655a962e61de8fb410472cbcd37aa"}, - {file = "lxml-6.1.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:5b7328b46d49fc9477d91ae8f6d55340347d827b7734ba3ea33faae0efef1383"}, - {file = "lxml-6.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37a58976370f36d9329d118ad0b953c5aeb9119ac9c6a4e258942a225d0573a1"}, - {file = "lxml-6.1.1-cp310-cp310-win32.whl", hash = "sha256:cea3f4c1af79af13cdb2da0c028111d8f8522d4f22a000c82385535f24e5cf3a"}, - {file = "lxml-6.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:3abf332af33a74288675d936fe861fd4344da0dd6622193fbc4f2bfbb35536b5"}, - {file = "lxml-6.1.1-cp310-cp310-win_arm64.whl", hash = "sha256:8dadbe5b217ff35b6a8d16610dd710219b59b76d13f0e3f0d9f36786206e4485"}, - {file = "lxml-6.1.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:53b7d2b7a10b1c35c0a5e21e9224accf60c1bbfba523990732e521b2b73adef2"}, - {file = "lxml-6.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ff3f333630ab480244a1bff72043e511a91eb22e7595dead8653ee5612dd8f3d"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a4bbea04c97f6d78a48e3fbc1cb9116d2780b1b39e03a23f6eb9b603fd61f510"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db1d75f6617a49c1c01bc7023713e0ff59ab32c9579ae62a7674c0e34f3b0b0a"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a12689be69a28ddaa0ab99a5a1137da2afd5f8f16df7b5680b66f616d3eda1d"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b73c339ae29b90fd2d06e58ebd555a751bde9cd6bbd36cc0281b9a2c94e9d8"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:752d3bbfe874715ccd0aec7f88d7fc623c0f1fd7aa7b3238a084e017bad2a009"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:6b1761fbf9ec984e2e9d9c589ef5f5fd684b7c19f92aadd567a26c5224958db6"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d680fbcb768404c601ecb43519ecd8461f6954cb11c06a78962f666832ccfca8"}, - {file = "lxml-6.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:162af1091cd785f2f27e62d3547ae9bc58ec5c86dd314d67021fd02463708d83"}, - {file = "lxml-6.1.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:e9308ff8241c532df3f3e570f9a5aeed6c853f888512ba4b75638d7c11c95ef6"}, - {file = "lxml-6.1.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5f6994074ebae6ffb04447268e37dc16edc304f9859cf91acb86e0af6c1b395c"}, - {file = "lxml-6.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80c2dfadb855da477cf73373ad29a333535dedb9b12bad02c9814c8e2b43bf08"}, - {file = "lxml-6.1.1-cp311-cp311-win32.whl", hash = "sha256:30a89d3ac8faec007453fb541f3f46807eeec88edd5826f6e3fe001752a2c621"}, - {file = "lxml-6.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:abbefa31eee84842140f67acef1c828e28bba8bbf0c3bc6e5492a9af88152c28"}, - {file = "lxml-6.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:dcb292aa7fe485ceff7af4f92e46c5af397daec5dff64871a528f0fc47a3cc5b"}, - {file = "lxml-6.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:104c09bda8d2a562824c0e319d0768ce26a779b7601e0931d33b09b53c392ef7"}, - {file = "lxml-6.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:25c6997a9a534e016695a0ba06b2f07945de682731ff01065b6d5a4474179da1"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c921ba5c51e4e9f63b8b00267d06566e1f63407408a0496da2d1d0bfc819c7fc"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:54a7f95e4de5fb94e2f9f4b9055c6ba33bf3d628fd77a1d647c5923caa2cdcdc"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f2ec43df44b1f76249ee0a615334f9b5b060e1c8bd90e706dad2d14d02f383"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:70ef8a7e102a1508f8121aae5b0867abd663f72c14f0a9c937e6554cb4587b7b"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ebe6af670449830d6d9b752c256a983291c766a1365ba5d5460048f9e33a7818"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:27acc820660aaffa4f7c087f29120e12980f7779d56d8492d263170111284740"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:1db753c9115ec7100d073b744d17e25e88a8f90f5c39b2f5dd878149af59671f"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4f469aebd783bb741c2ecb2a681008fd26bfe5c16a9a72ed5467f834e810df2"}, - {file = "lxml-6.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:766b010012d59470072c1816b5b6c69f1d243e5db36ea5968e94accf430a4635"}, - {file = "lxml-6.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b8d812c6011c08b8111a15e54dd990b8923692d80adf35488bee34026c35accf"}, - {file = "lxml-6.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:fe0306bd29505a9177aac19f1877174b0e7422c222a59f70b2cd41633448c3dc"}, - {file = "lxml-6.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5ba186ad207446c65d3bb3d3e0412b032b1d9f595e59861e2354798c5703d955"}, - {file = "lxml-6.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aa366a1e55b8ebfe8ca8ddc3cfe75c8ebade181aeb0f661d0cb05986b647f72a"}, - {file = "lxml-6.1.1-cp312-cp312-win32.whl", hash = "sha256:126c93f7f56f0eda92f6d8c619edc463a4f23d9252f1c9d0405a76f25fa9f11a"}, - {file = "lxml-6.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:26e6eda8d38c1fcab1090dd196ee87cbd13788e531937610e2589085de074e77"}, - {file = "lxml-6.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:6540377fbd53fe1b629172288c464fb18db11ce1fa7dc15891da10aa9dcc3e7f"}, - {file = "lxml-6.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736"}, - {file = "lxml-6.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c"}, - {file = "lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b"}, - {file = "lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2"}, - {file = "lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5"}, - {file = "lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785"}, - {file = "lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947"}, - {file = "lxml-6.1.1-cp313-cp313-win32.whl", hash = "sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca"}, - {file = "lxml-6.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660"}, - {file = "lxml-6.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc"}, - {file = "lxml-6.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0"}, - {file = "lxml-6.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5"}, - {file = "lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462"}, - {file = "lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465"}, - {file = "lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a"}, - {file = "lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590"}, - {file = "lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb"}, - {file = "lxml-6.1.1-cp314-cp314-win32.whl", hash = "sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603"}, - {file = "lxml-6.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137"}, - {file = "lxml-6.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf"}, - {file = "lxml-6.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee"}, - {file = "lxml-6.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e"}, - {file = "lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072"}, - {file = "lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52"}, - {file = "lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b"}, - {file = "lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2"}, - {file = "lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e"}, - {file = "lxml-6.1.1-cp314-cp314t-win32.whl", hash = "sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1"}, - {file = "lxml-6.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e"}, - {file = "lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c"}, - {file = "lxml-6.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6689e828a94eee4f139408c337bb198e014724bb8a8c26d3cfac49d119ed69a6"}, - {file = "lxml-6.1.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bdebcc8a75d38c7598dfb2c9ed852d7a9eb4a10d6e2d0764b919b802bf32ac88"}, - {file = "lxml-6.1.1-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8be8ad51249698103d24b0571df35a10990fbe93dd043b6c024172189485f5e3"}, - {file = "lxml-6.1.1-cp38-cp38-manylinux_2_28_i686.whl", hash = "sha256:76447f65250ed2501ead1a1552f5ce8edff159a86f308348e6a9c4acb5e1f1b4"}, - {file = "lxml-6.1.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:ffecec8eb889b58ba9be5b95fb1cc78e22ea8eedea38e8736a1568fe1979250e"}, - {file = "lxml-6.1.1-cp38-cp38-win32.whl", hash = "sha256:c674693f055fa2495de12292cb45e9944199d8eaef5a2dec45175c7c61cb73e3"}, - {file = "lxml-6.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:55b03549819867ea141c0202242c4816c82e52ec36e7e648db9d8da5a3dc3ed6"}, - {file = "lxml-6.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c9f79d5325907f13e1be0b3e4dacc1049d1dffc4aeee3c995284bea5fe0fab7d"}, - {file = "lxml-6.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:83b6b30eb131da7a75b601f28c5d6971e6ed3e887919bf6b6a1ad3c2df289080"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:441dd227fa0690eb9fc81edabc63cdcefc212bba99b906dcf6e32cc1a9d3e533"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e07c65f443c887bbcf31cc1771d932ecc192a5273943589b3c7572b749f1ffb2"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5bec7d03d78d853597d6107854c2310ce3f761fd218fe9fe91d5101fcf6c2efe"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f76acfb5f68ba982635a53fd985a8044be98a35b43232c2a1ee235ffab3e1dd"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux_2_28_i686.whl", hash = "sha256:8d43ca737b20e106e4aebc42b2f3ae19f00ba63d7eb731698ee083d72d15646f"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:32ab449a5486f6c758e849bb86710d0e45edc24a04e250c01555f8f5653958f8"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:53c909b62a0532183542fed00c5a7218258c56292d409bc789886fe1cb04c438"}, - {file = "lxml-6.1.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:640f97d43d867bcb9c75b3af013b64850756b746cb6bce8ace83b70da3abba9d"}, - {file = "lxml-6.1.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:469e3618338bd7ab5beb412d2439825479fcf0dab99e394ca563dbc4eaf6c834"}, - {file = "lxml-6.1.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:aae97dfdb60715c164419ac2532a76d013c3918a665eb6cb7288098b5f349aaf"}, - {file = "lxml-6.1.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c9a4b821dc7055bf9e05ff5719e18ec501f75c0f0bbfabd573b277559780833d"}, - {file = "lxml-6.1.1-cp39-cp39-win32.whl", hash = "sha256:639f6c857d91d9be29bd7502348d6736dab168b54b5158cd899abf11684dc186"}, - {file = "lxml-6.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:34c2d737beabfe35baada43941ed519251e9a12e779031496bcd5d539fcfd730"}, - {file = "lxml-6.1.1-cp39-cp39-win_arm64.whl", hash = "sha256:07a4a68e286ee7a1ed7dfb8af83e615757c0ccfe9f18c6b4ea6771388d9ba8c9"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:31033dc34636ea6b7d5cc11b1ddbda78a14de858ba9d3e1ed4b69a3085bc521e"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3893c14c4b6ac5b2d54ba8cf03e99fe5104e592de491f19bd6b82756c09f8004"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c07da4cebf6889f03ebac8d238f62318e29f495de0aa18a51ea14e61ae907e2e"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6f0ce10945fab9c4c06ce14e22af9059d1a87493a9af4501a5b0b9187e21cf2"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f8844cd288697c6425c9beba919302241e3278871dc6519515e72b04e987abcf"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:ed21202aec73cda4d55d1ce57b389aadb90ffb044e6cd1080b8347efe1b1ec84"}, - {file = "lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40"}, -] - -[package.extras] -cssselect = ["cssselect (>=0.7)"] -html-clean = ["lxml_html_clean"] -html5 = ["html5lib"] -htmlsoup = ["BeautifulSoup4"] - -[[package]] -name = "markdown-it-py" -version = "4.2.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a"}, - {file = "markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "markdown-it-pyrs", "mistletoe (>=1.0,<2.0)", "mistune (>=3.0,<4.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins (>=0.5.0)"] -profiling = ["gprof2dot"] -rtd = ["ipykernel", "jupyter_sphinx", "mdit-py-plugins (>=0.5.0)", "myst-parser", "pyyaml", "sphinx", "sphinx-book-theme (>=1.0,<2.0)", "sphinx-copybutton", "sphinx-design"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions", "pytest-timeout", "requests"] - -[[package]] -name = "markupsafe" -version = "3.0.3" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, - {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1"}, - {file = "markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa"}, - {file = "markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8"}, - {file = "markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1"}, - {file = "markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad"}, - {file = "markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a"}, - {file = "markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19"}, - {file = "markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01"}, - {file = "markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c"}, - {file = "markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e"}, - {file = "markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b"}, - {file = "markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d"}, - {file = "markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c"}, - {file = "markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f"}, - {file = "markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795"}, - {file = "markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12"}, - {file = "markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed"}, - {file = "markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5"}, - {file = "markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485"}, - {file = "markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73"}, - {file = "markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287"}, - {file = "markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe"}, - {file = "markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe"}, - {file = "markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9"}, - {file = "markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581"}, - {file = "markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4"}, - {file = "markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab"}, - {file = "markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa"}, - {file = "markupsafe-3.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26"}, - {file = "markupsafe-3.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d"}, - {file = "markupsafe-3.0.3-cp39-cp39-win32.whl", hash = "sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7"}, - {file = "markupsafe-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e"}, - {file = "markupsafe-3.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8"}, - {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, -] - -[[package]] -name = "mbstrdecoder" -version = "1.1.5" -description = "mbstrdecoder is a Python library for multi-byte character string decoder" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "mbstrdecoder-1.1.5-py3-none-any.whl", hash = "sha256:4a50fe113d4abecfd86e8f716b2e413cce03d63af83ec3c7cdbe81dec0e519ed"}, - {file = "mbstrdecoder-1.1.5.tar.gz", hash = "sha256:8cbfba26938befd8a35e3cc06ca0632f61320b7b2be7df32550b895e1725b1ce"}, -] - -[package.dependencies] -chardet = ">=3.0.4,<7" - -[package.extras] -test = ["Faker (>=1.0.2)", "pytest (>=6.0.1)", "pytest-md-report (>=0.6.2)"] - -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - -[[package]] -name = "more-itertools" -version = "11.1.0" -description = "More routines for operating on iterables, beyond itertools" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192"}, - {file = "more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d"}, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -description = "Python library for arbitrary-precision floating-point arithmetic" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, - {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, -] - -[package.extras] -develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] -docs = ["sphinx"] -gmpy = ["gmpy2 (>=2.1.0a4) ; platform_python_implementation != \"PyPy\""] -tests = ["pytest (>=4.6)"] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.19" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.19-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:02e5c35d7d6cd2bdc89c1858867f7bde4012837411023a4696c148c1bdd7c80e"}, - {file = "multiprocess-0.70.19-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:79576c02d1207ec405b00cabf2c643c36070800cca433860e14539df7818b2aa"}, - {file = "multiprocess-0.70.19-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c6b6d78d43a03b68014ca1f0b7937d965393a670c5de7c29026beb2258f2f896"}, - {file = "multiprocess-0.70.19-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1bbf1b69af1cf64cd05f65337d9215b88079ec819cd0ea7bac4dab84e162efe7"}, - {file = "multiprocess-0.70.19-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5be9ec7f0c1c49a4f4a6fd20d5dda4aeabc2d39a50f4ad53720f1cd02b3a7c2e"}, - {file = "multiprocess-0.70.19-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1c3dce098845a0db43b32a0b76a228ca059a668071cfeaa0f40c36c0b1585d45"}, - {file = "multiprocess-0.70.19-pp39-pypy39_pp73-macosx_10_13_arm64.whl", hash = "sha256:e5e7dc3e3e1732e88c07aaec17eeb9917f9ed1107d9e60d5ab985cdc14bac43a"}, - {file = "multiprocess-0.70.19-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:e6c0674d34b8adac22533f6786576b3de4e396aaeda9e0c15378af9b8ada2702"}, - {file = "multiprocess-0.70.19-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d6db91ca6391eebc139c352f34578cea382df6bfa03d3b4146ed12b18b01cc14"}, - {file = "multiprocess-0.70.19-py310-none-any.whl", hash = "sha256:97404393419dcb2a8385910864eedf47a3cadf82c66345b44f036420eb0b5d87"}, - {file = "multiprocess-0.70.19-py311-none-any.whl", hash = "sha256:928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c"}, - {file = "multiprocess-0.70.19-py312-none-any.whl", hash = "sha256:3a56c0e85dd5025161bac5ce138dcac1e49174c7d8e74596537e729fd5c53c28"}, - {file = "multiprocess-0.70.19-py313-none-any.whl", hash = "sha256:8d5eb4ec5017ba2fab4e34a747c6d2c2b6fecfe9e7236e77988db91580ada952"}, - {file = "multiprocess-0.70.19-py314-none-any.whl", hash = "sha256:e8cc7fbdff15c0613f0a1f1f8744bef961b0a164c0ca29bdff53e9d2d93c5e5f"}, - {file = "multiprocess-0.70.19-py39-none-any.whl", hash = "sha256:0d4b4397ed669d371c81dcd1ef33fd384a44d6c3de1bd0ca7ac06d837720d3c5"}, - {file = "multiprocess-0.70.19.tar.gz", hash = "sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897"}, -] - -[package.dependencies] -dill = ">=0.4.1" - -[[package]] -name = "narwhals" -version = "2.24.0" -description = "Extremely lightweight compatibility layer between dataframe libraries" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489"}, - {file = "narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d"}, -] - -[package.extras] -cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] -dask = ["dask[dataframe] (>=2024.8)"] -duckdb = ["duckdb (>=1.1)"] -ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)"] -modin = ["modin (>=0.22.0)"] -pandas = ["pandas (>=1.3.4)"] -polars = ["polars (>=0.20.4)"] -pyarrow = ["pyarrow (>=13.0.0)"] -pyspark = ["pyspark (>=3.5.0)"] -pyspark-connect = ["pyspark[connect] (>=3.5.0)"] -sql = ["narwhals[duckdb]", "sqlparse (>=0.5.5)"] -sqlframe = ["sqlframe (>=3.22.0,!=3.39.3)"] - -[[package]] -name = "networkx" -version = "3.4.2" -description = "Python package for creating and manipulating graphs and networks" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, - {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, -] - -[package.extras] -default = ["matplotlib (>=3.7)", "numpy (>=1.24)", "pandas (>=2.0)", "scipy (>=1.10,!=1.11.0,!=1.11.1)"] -developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] -doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.15)", "sphinx (>=7.3)", "sphinx-gallery (>=0.16)", "texext (>=0.6.7)"] -example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy (>=0.7.2)", "osmnx (>=1.9)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] -extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] -test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] - -[[package]] -name = "networkx" -version = "3.6.1" -description = "Python package for creating and manipulating graphs and networks" -optional = false -python-versions = "!=3.14.1,>=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762"}, - {file = "networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509"}, -] - -[package.extras] -benchmarking = ["asv", "virtualenv"] -default = ["matplotlib (>=3.8)", "numpy (>=1.25)", "pandas (>=2.0)", "scipy (>=1.11.2)"] -developer = ["mypy (>=1.15)", "pre-commit (>=4.1)"] -doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=10)", "pydata-sphinx-theme (>=0.16)", "sphinx (>=8.0)", "sphinx-gallery (>=0.18)", "texext (>=0.6.7)"] -example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "iplotx (>=0.9.0)", "momepy (>=0.7.2)", "osmnx (>=2.0.0)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] -extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] -release = ["build (>=0.10)", "changelist (==0.5)", "twine (>=4.0)", "wheel (>=0.40)"] -test = ["pytest (>=7.2)", "pytest-cov (>=4.0)", "pytest-xdist (>=3.0)"] -test-extras = ["pytest-mpl", "pytest-randomly"] - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numexpr" -version = "2.14.1" -description = "Fast numerical expression evaluator for NumPy" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "numexpr-2.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d0fab3fd06a04f6b86102552b26aa5d85e20ac7d8296c15764c726eeabae6cc8"}, - {file = "numexpr-2.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:64ae5dfd62d74a3ef82fe0b37f80527247f3626171ad82025900f46ffca4b39a"}, - {file = "numexpr-2.14.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:955c92b064f9074d2970cf3138f5e3b965be673b82024962ed526f39bc25a920"}, - {file = "numexpr-2.14.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:75440c54fc01e130396650fdf307aa9d41a67dc06ddbfb288971b591c13a395b"}, - {file = "numexpr-2.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dde9fa47ed319e1e1728940a539df3cb78326b7754bc7c6ab3152afc91808f9b"}, - {file = "numexpr-2.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76db0bc6267e591ab9c4df405ffb533598e4c88239db7338d11ae9e4b368a85a"}, - {file = "numexpr-2.14.1-cp310-cp310-win32.whl", hash = "sha256:0d1dcbdc4d0374c0d523cee2f94f06b001623cbc1fd163612841017a3495427c"}, - {file = "numexpr-2.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:823cd82c8e7937981339f634e7a9c6a92cb2d0b9d0a5cf627a5e394fffc05377"}, - {file = "numexpr-2.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2d03fcb4644a12f70a14d74006f72662824da5b6128bf1bcd10cc3ed80e64c34"}, - {file = "numexpr-2.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2773ee1133f77009a1fc2f34fe236f3d9823779f5f75450e183137d49f00499f"}, - {file = "numexpr-2.14.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebe4980f9494b9f94d10d2e526edc29e72516698d3bf95670ba79415492212a4"}, - {file = "numexpr-2.14.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a381e5e919a745c9503bcefffc1c7f98c972c04ec58fc8e999ed1a929e01ba6"}, - {file = "numexpr-2.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d08856cfc1b440eb1caaa60515235369654321995dd68eb9377577392020f6cb"}, - {file = "numexpr-2.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03130afa04edf83a7b590d207444f05a00363c9b9ea5d81c0f53b1ea13fad55a"}, - {file = "numexpr-2.14.1-cp311-cp311-win32.whl", hash = "sha256:db78fa0c9fcbaded3ae7453faf060bd7a18b0dc10299d7fcd02d9362be1213ed"}, - {file = "numexpr-2.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:e9b2f957798c67a2428be96b04bce85439bed05efe78eb78e4c2ca43737578e7"}, - {file = "numexpr-2.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:91ebae0ab18c799b0e6b8c5a8d11e1fa3848eb4011271d99848b297468a39430"}, - {file = "numexpr-2.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:47041f2f7b9e69498fb311af672ba914a60e6e6d804011caacb17d66f639e659"}, - {file = "numexpr-2.14.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d686dfb2c1382d9e6e0ee0b7647f943c1886dba3adbf606c625479f35f1956c1"}, - {file = "numexpr-2.14.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee6d4fbbbc368e6cdd0772734d6249128d957b3b8ad47a100789009f4de7083"}, - {file = "numexpr-2.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3a2839efa25f3c8d4133252ea7342d8f81226c7c4dda81f97a57e090b9d87a48"}, - {file = "numexpr-2.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9f9137f1351b310436662b5dc6f4082a245efa8950c3b0d9008028df92fefb9b"}, - {file = "numexpr-2.14.1-cp312-cp312-win32.whl", hash = "sha256:36f8d5c1bd1355df93b43d766790f9046cccfc1e32b7c6163f75bcde682cda07"}, - {file = "numexpr-2.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:fdd886f4b7dbaf167633ee396478f0d0aa58ea2f9e7ccc3c6431019623e8d68f"}, - {file = "numexpr-2.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:09078ba73cffe94745abfbcc2d81ab8b4b4e9d7bfbbde6cac2ee5dbf38eee222"}, - {file = "numexpr-2.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dce0b5a0447baa7b44bc218ec2d7dcd175b8eee6083605293349c0c1d9b82fb6"}, - {file = "numexpr-2.14.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:06855053de7a3a8425429bd996e8ae3c50b57637ad3e757e0fa0602a7874be30"}, - {file = "numexpr-2.14.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f9366d23a2e991fd5a8b5e61a17558f028ba86158a4552f8f239b005cdf83c"}, - {file = "numexpr-2.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c5f1b1605695778896534dfc6e130d54a65cd52be7ed2cd0cfee3981fd676bf5"}, - {file = "numexpr-2.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a4ba71db47ea99c659d88ee6233fa77b6dc83392f1d324e0c90ddf617ae3f421"}, - {file = "numexpr-2.14.1-cp313-cp313-win32.whl", hash = "sha256:638dce8320f4a1483d5ca4fda69f60a70ed7e66be6e68bc23fb9f1a6b78a9e3b"}, - {file = "numexpr-2.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:9fdcd4735121658a313f878fd31136d1bfc6a5b913219e7274e9fca9f8dac3bb"}, - {file = "numexpr-2.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:557887ad7f5d3c2a40fd7310e50597045a68e66b20a77b3f44d7bc7608523b4b"}, - {file = "numexpr-2.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:af111c8fe6fc55d15e4c7cab11920fc50740d913636d486545b080192cd0ad73"}, - {file = "numexpr-2.14.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33265294376e7e2ae4d264d75b798a915d2acf37b9dd2b9405e8b04f84d05cfc"}, - {file = "numexpr-2.14.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83647d846d3eeeb9a9255311236135286728b398d0d41d35dedb532dca807fe9"}, - {file = "numexpr-2.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6e575fd3ad41ddf3355d0c7ef6bd0168619dc1779a98fe46693cad5e95d25e6e"}, - {file = "numexpr-2.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:67ea4771029ce818573b1998f5ca416bd255156feea017841b86176a938f7d19"}, - {file = "numexpr-2.14.1-cp313-cp313t-win32.whl", hash = "sha256:15015d47d3d1487072d58c0e7682ef2eb608321e14099c39d52e2dd689483611"}, - {file = "numexpr-2.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:94c711f6d8f17dfb4606842b403699603aa591ab9f6bf23038b488ea9cfb0f09"}, - {file = "numexpr-2.14.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ede79f7ff06629f599081de644546ce7324f1581c09b0ac174da88a470d39c21"}, - {file = "numexpr-2.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2eac7a5a2f70b3768c67056445d1ceb4ecd9b853c8eda9563823b551aeaa5082"}, - {file = "numexpr-2.14.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5aedf38d4c0c19d3cecfe0334c3f4099fb496f54c146223d30fa930084bc8574"}, - {file = "numexpr-2.14.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439ec4d57b853792ebe5456e3160312281c3a7071ecac5532ded3278ede614de"}, - {file = "numexpr-2.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e23b87f744e04e302d82ac5e2189ae20a533566aec76a46885376e20b0645bf8"}, - {file = "numexpr-2.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:44f84e0e5af219dbb62a081606156420815890e041b87252fbcea5df55214c4c"}, - {file = "numexpr-2.14.1-cp314-cp314-win32.whl", hash = "sha256:1f1a5e817c534539351aa75d26088e9e1e0ef1b3a6ab484047618a652ccc4fc3"}, - {file = "numexpr-2.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:587c41509bc373dfb1fe6086ba55a73147297247bedb6d588cda69169fc412f2"}, - {file = "numexpr-2.14.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:ec368819502b64f190c3f71be14a304780b5935c42aae5bf22c27cc2cbba70b5"}, - {file = "numexpr-2.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7e87f6d203ac57239de32261c941e9748f9309cbc0da6295eabd0c438b920d3a"}, - {file = "numexpr-2.14.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd72d8c2a165fe45ea7650b16eb8cc1792a94a722022006bb97c86fe51fd2091"}, - {file = "numexpr-2.14.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70d80fcb418a54ca208e9a38e58ddc425c07f66485176b261d9a67c7f2864f73"}, - {file = "numexpr-2.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:edea2f20c2040df8b54ee8ca8ebda63de9545b2112872466118e9df4d0ae99f3"}, - {file = "numexpr-2.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:790447be6879a6c51b9545f79612d24c9ea0a41d537a84e15e6a8ddef0b6268e"}, - {file = "numexpr-2.14.1-cp314-cp314t-win32.whl", hash = "sha256:538961096c2300ea44240209181e31fae82759d26b51713b589332b9f2a4117e"}, - {file = "numexpr-2.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:a40b350cd45b4446076fa11843fa32bbe07024747aeddf6d467290bf9011b392"}, - {file = "numexpr-2.14.1.tar.gz", hash = "sha256:4be00b1086c7b7a5c32e31558122b7b80243fe098579b170967da83f3152b48b"}, -] - -[package.dependencies] -numpy = ">=1.23.0" - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "nvidia-cublas" -version = "13.1.1.3" -description = "CUBLAS native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5"}, - {file = "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436"}, - {file = "nvidia_cublas-13.1.1.3-py3-none-win_amd64.whl", hash = "sha256:b6cdce694e47ff6aadf0a69df1cab6628d696f5ff56e8d16af50309d855fa20f"}, -] - -[package.dependencies] -nvidia-cuda-nvrtc = "*" - -[[package]] -name = "nvidia-cublas" -version = "13.6.0.2" -description = "CUBLAS native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform != \"linux\" or platform_machine != \"aarch64\" and platform_machine != \"x86_64\") and platform_system == \"Linux\" and (sys_platform != \"linux\" and sys_platform != \"win32\" or platform_machine != \"x86_64\")" -files = [ - {file = "nvidia_cublas-13.6.0.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:946f6a252b1cc72d8de912c75975fd6d8ba44f67d4e5044fe764ddb909f4a688"}, - {file = "nvidia_cublas-13.6.0.2-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b82c80c886cea6da6e149a5c3bdba274f12b7e4ec4b00a050b916b0446fb4153"}, - {file = "nvidia_cublas-13.6.0.2-py3-none-win_amd64.whl", hash = "sha256:3b5bcd6bfb6f65010ebf195851bcb9b2aa34b9fe08479432002991c1fe84b67d"}, -] - -[package.dependencies] -nvidia-cuda-nvrtc = "*" - -[[package]] -name = "nvidia-cuda-cupti" -version = "13.0.85" -description = "CUDA profiling tools runtime libs." -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151"}, - {file = "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8"}, - {file = "nvidia_cuda_cupti-13.0.85-py3-none-win_amd64.whl", hash = "sha256:683f58d301548deeefcb8f6fac1b8d907691b9d8b18eccab417f51e362102f00"}, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.0.88" -description = "NVRTC native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575"}, - {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b"}, - {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-win_amd64.whl", hash = "sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872"}, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.3.33" -description = "NVRTC native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform != \"linux\" or platform_machine != \"aarch64\" and platform_machine != \"x86_64\") and platform_system == \"Linux\" and (sys_platform != \"linux\" and sys_platform != \"win32\" or platform_machine != \"x86_64\")" -files = [ - {file = "nvidia_cuda_nvrtc-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:82530788b8c6164a54d3fd9ae8bcca8893d397c4aeb998861982a03bbe41e204"}, - {file = "nvidia_cuda_nvrtc-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7b05ecda494c6dabc44231a608b060a71008a730d9dfda932cc508e6d29159e0"}, - {file = "nvidia_cuda_nvrtc-13.3.33-py3-none-win_amd64.whl", hash = "sha256:7d2af818851c0c224d5f92221e9226e51ee23c236df4b51f9194563979c888be"}, -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.0.96" -description = "CUDA Runtime native Libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55"}, - {file = "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548"}, - {file = "nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492"}, -] - -[[package]] -name = "nvidia-cudnn-cu13" -version = "9.20.0.48" -description = "cuDNN runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1"}, - {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304"}, - {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-win_amd64.whl", hash = "sha256:af8139732b99c0118be65ea5aac97f0d46018f8c552889e49d2fb0c6261a4a24"}, -] - -[package.dependencies] -nvidia-cublas = "*" - -[[package]] -name = "nvidia-cufft" -version = "12.0.0.61" -description = "CUFFT native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5"}, - {file = "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3"}, - {file = "nvidia_cufft-12.0.0.61-py3-none-win_amd64.whl", hash = "sha256:2abce5b39d2f5ae12730fb7e5db6696533e36c26e2d3e8fd1750bdd2853364eb"}, -] - -[package.dependencies] -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cufile" -version = "1.15.1.6" -description = "cuFile GPUDirect libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "sys_platform == \"linux\" and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44"}, - {file = "nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1"}, -] - -[[package]] -name = "nvidia-curand" -version = "10.4.0.35" -description = "CURAND native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a"}, - {file = "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc"}, - {file = "nvidia_curand-10.4.0.35-py3-none-win_amd64.whl", hash = "sha256:65b1710aa6961d326b411e314b374290904c5ddf41dc3f766ebc3f1d7d4ca69f"}, -] - -[[package]] -name = "nvidia-cusolver" -version = "12.0.4.66" -description = "CUDA solver native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2"}, - {file = "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112"}, - {file = "nvidia_cusolver-12.0.4.66-py3-none-win_amd64.whl", hash = "sha256:16515bd33a8e76bb54d024cfa068fa68d30e80fc34b9e1090813ea9362e0cb65"}, -] - -[package.dependencies] -nvidia-cublas = "*" -nvidia-cusparse = "*" -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cusparse" -version = "12.6.3.3" -description = "CUSPARSE native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c"}, - {file = "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b"}, - {file = "nvidia_cusparse-12.6.3.3-py3-none-win_amd64.whl", hash = "sha256:cbcf42feb737bd7ec15b4c0a63e62351886bd3f975027b8815d7f720a2b5ea79"}, -] - -[package.dependencies] -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cusparse" -version = "12.8.2.51" -description = "CUSPARSE native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "" -files = [ - {file = "nvidia_cusparse-12.8.2.51-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:00469fcf62c4d464a1225abd9b20864ecff35e3fbc9fb992572e83d358927755"}, - {file = "nvidia_cusparse-12.8.2.51-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65cbcc4e37a34fca4ee7df2fd57da103593842cda1bbb4a144664ecfe59873a5"}, - {file = "nvidia_cusparse-12.8.2.51-py3-none-win_amd64.whl", hash = "sha256:2ee59291cd362038f3d40d57c7cd09b26d689f3873ae5c94b31c3270772d41b8"}, -] - -[package.dependencies] -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cusparselt-cu13" -version = "0.8.1" -description = "NVIDIA cuSPARSELt" -optional = false -python-versions = "*" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f"}, - {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0"}, - {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-win_amd64.whl", hash = "sha256:dccbd362f91a7b9024d1f55ee9f548ac065027ff15d8c8b0db889ab3a8f31215"}, -] - -[[package]] -name = "nvidia-nccl-cu13" -version = "2.29.7" -description = "NVIDIA Collective Communication Library (NCCL) Runtime" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5"}, - {file = "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d"}, -] - -[[package]] -name = "nvidia-nvjitlink" -version = "13.3.33" -description = "Nvidia JIT LTO Library" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5"}, - {file = "nvidia_nvjitlink-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ce48b37dfeb3cb1eae4cf85adacb47d7a6539ea2272870c9a3628ce275c2037e"}, - {file = "nvidia_nvjitlink-13.3.33-py3-none-win_amd64.whl", hash = "sha256:4297ee49639b4f2e07255a1d69b3acc7ab2d011bb892b403e91ac98368962e3b"}, -] - -[[package]] -name = "nvidia-nvshmem-cu13" -version = "3.4.5" -description = "NVSHMEM creates a global address space that provides efficient and scalable communication for NVIDIA GPU clusters." -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9"}, - {file = "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80"}, -] - -[[package]] -name = "nvidia-nvtx" -version = "13.0.85" -description = "NVIDIA Tools Extension" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4"}, - {file = "nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6"}, - {file = "nvidia_nvtx-13.0.85-py3-none-win_amd64.whl", hash = "sha256:d66ea44254dd3c6eacc300047af6e1288d2269dd072b417e0adffbf479e18519"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "pathvalidate" -version = "3.3.1" -description = "pathvalidate is a Python library to sanitize/validate a string such as filenames/file-paths/etc." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "pathvalidate-3.3.1-py3-none-any.whl", hash = "sha256:5263baab691f8e1af96092fa5137ee17df5bdfbd6cff1fcac4d6ef4bc2e1735f"}, - {file = "pathvalidate-3.3.1.tar.gz", hash = "sha256:b18c07212bfead624345bb8e1d6141cdcf15a39736994ea0b94035ad2b1ba177"}, -] - -[package.extras] -docs = ["Sphinx (>=2.4)", "sphinx_rtd_theme (>=1.2.2)", "urllib3 (<2)"] -readme = ["path (>=13,<18)", "readmemaker (>=1.2.0)"] -test = ["Faker (>=1.0.8)", "allpairspy (>=2)", "click (>=6.2)", "pytest (>=6.0.1)", "pytest-md-report (>=0.6.2)"] - -[[package]] -name = "peft" -version = "0.19.1" -description = "Parameter-Efficient Fine-Tuning (PEFT)" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "peft-0.19.1-py3-none-any.whl", hash = "sha256:2113f72a81621b5913ef28f9022204c742df111890c5f49d812716a4a301e356"}, - {file = "peft-0.19.1.tar.gz", hash = "sha256:0d97542fe96dcdaa20d3b81c06f26f988618f416a73544ab23c3618ccb674a40"}, -] - -[package.dependencies] -accelerate = ">=0.21.0" -huggingface_hub = ">=0.25.0" -numpy = ">=1.17" -packaging = ">=20.0" -psutil = "*" -pyyaml = "*" -safetensors = "*" -torch = ">=1.13.0" -tqdm = "*" -transformers = "*" - -[package.extras] -dev = ["black", "black", "hf-doc-builder", "hf-doc-builder", "requests", "ruff (>=0.12.8,<0.13.0)"] -docs-specific = ["black", "hf-doc-builder", "requests"] -quality = ["black", "hf-doc-builder", "ruff (>=0.12.8,<0.13.0)"] -test = ["black", "black", "datasets", "diffusers", "hf-doc-builder", "hf-doc-builder", "parameterized", "protobuf", "pytest", "pytest-cov", "pytest-xdist", "requests", "ruff (>=0.12.8,<0.13.0)", "scikit-learn", "scipy", "sentencepiece", "torchvision"] - -[[package]] -name = "portalocker" -version = "3.2.0" -description = "Wraps the portalocker recipe for easy usage" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968"}, - {file = "portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac"}, -] - -[package.dependencies] -pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} - -[package.extras] -docs = ["portalocker[tests]"] -redis = ["redis"] -tests = ["coverage-conditional-plugin (>=0.9.0)", "portalocker[redis]", "pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-rerunfailures (>=15.0)", "pytest-timeout (>=2.1.0)", "sphinx (>=6.0.0)", "types-pywin32 (>=310.0.0.20250429)", "types-redis"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "psutil" -version = "7.2.2" -description = "Cross-platform lib for process and system monitoring." -optional = false -python-versions = ">=3.6" -groups = ["main"] -files = [ - {file = "psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b"}, - {file = "psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea"}, - {file = "psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63"}, - {file = "psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312"}, - {file = "psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b"}, - {file = "psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9"}, - {file = "psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00"}, - {file = "psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9"}, - {file = "psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a"}, - {file = "psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf"}, - {file = "psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1"}, - {file = "psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841"}, - {file = "psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486"}, - {file = "psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979"}, - {file = "psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9"}, - {file = "psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e"}, - {file = "psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8"}, - {file = "psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc"}, - {file = "psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988"}, - {file = "psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee"}, - {file = "psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372"}, -] - -[package.extras] -dev = ["abi3audit", "black", "check-manifest", "colorama ; os_name == \"nt\"", "coverage", "packaging", "psleak", "pylint", "pyperf", "pypinfo", "pyreadline3 ; os_name == \"nt\"", "pytest", "pytest-cov", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "validate-pyproject[all]", "virtualenv", "vulture", "wheel", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] -test = ["psleak", "pytest", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "setuptools", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "pybind11" -version = "3.0.4" -description = "Seamless operability between C++11 and Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "pybind11-3.0.4-py3-none-any.whl", hash = "sha256:961720ee652da51d531b7b2451a6bd2bc042b0106e6d9baa48ecb7d58034ce63"}, - {file = "pybind11-3.0.4.tar.gz", hash = "sha256:3286b59c8a774b9ee650169302dd5a4eedc30a8617905a0560dd8ee44775130c"}, -] - -[package.extras] -global = ["pybind11-global (==3.0.4)"] - -[[package]] -name = "pygments" -version = "2.20.0" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, - {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, -] - -[package.extras] -windows-terminal = ["colorama (>=0.4.6)"] - -[[package]] -name = "pytablewriter" -version = "1.2.1" -description = "pytablewriter is a Python library to write a table in various formats: AsciiDoc / CSV / Elasticsearch / HTML / JavaScript / JSON / LaTeX / LDJSON / LTSV / Markdown / MediaWiki / NumPy / Excel / Pandas / Python / reStructuredText / SQLite / TOML / TSV / YAML." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "pytablewriter-1.2.1-py3-none-any.whl", hash = "sha256:e906ff7ff5151d70a5f66e0f7b75642a7f2dce8d893c265b79cc9cf6bc04ddb4"}, - {file = "pytablewriter-1.2.1.tar.gz", hash = "sha256:7bd0f4f397e070e3b8a34edcf1b9257ccbb18305493d8350a5dbc9957fced959"}, -] - -[package.dependencies] -DataProperty = ">=1.1.0,<2" -mbstrdecoder = ">=1.0.0,<2" -pathvalidate = ">=2.3.0,<4" -setuptools = ">=38.3.0" -tabledata = ">=1.3.1,<2" -tcolorpy = ">=0.0.5,<1" -typepy = {version = ">=1.3.2,<2", extras = ["datetime"]} - -[package.extras] -all = ["PyYAML (>=3.11,<7)", "SimpleSQLite (>=1.3.2,<2)", "XlsxWriter (>=0.9.6,<4)", "dominate (>=2.1.5,<3)", "elasticsearch (>=8.0.1,<9)", "loguru (>=0.4.1,<1)", "pandas (>=0.25.3,<3)", "pytablereader (>=0.31.3,<2)", "pytablewriter-altcol-theme (>=0.1.0,<1)", "pytablewriter-altrow-theme (>=0.2.0,<1)", "simplejson (>=3.8.1,<4)", "toml (>=0.9.3,<1)", "xlwt"] -docs = ["PyYAML (>=3.11,<7)", "SimpleSQLite (>=1.3.2,<2)", "Sphinx (>=2.4)", "XlsxWriter (>=0.9.6,<4)", "dominate (>=2.1.5,<3)", "elasticsearch (>=8.0.1,<9)", "loguru (>=0.4.1,<1)", "pandas (>=0.25.3,<3)", "pytablereader (>=0.31.3,<2)", "pytablewriter-altcol-theme (>=0.1.0,<1)", "pytablewriter-altrow-theme (>=0.2.0,<1)", "simplejson (>=3.8.1,<4)", "sphinx_rtd_theme (>=1.2.2)", "toml (>=0.9.3,<1)", "xlwt"] -es = ["elasticsearch (>=8.0.1,<9)"] -es8 = ["elasticsearch (>=8.0.1,<9)"] -excel = ["XlsxWriter (>=0.9.6,<4)", "xlwt"] -from = ["pytablereader (>=0.31.3,<2)"] -html = ["dominate (>=2.1.5,<3)"] -logging = ["loguru (>=0.4.1,<1)"] -pandas = ["pandas (>=0.25.3,<3)"] -sqlite = ["SimpleSQLite (>=1.3.2,<2)"] -test = ["PyYAML (>=3.11,<7)", "SimpleSQLite (>=1.3.2,<2)", "XlsxWriter (>=0.9.6,<4)", "beautifulsoup4 (>=4.10)", "dominate (>=2.1.5,<3)", "elasticsearch (>=8.0.1,<9)", "loguru (>=0.4.1,<1)", "pandas (>=0.25.3,<3)", "pytablereader (>=0.31.3,<2)", "pytablereader[excel,sqlite] (>=0.31.3)", "pytablewriter-altcol-theme (>=0.1.0,<1)", "pytablewriter-altrow-theme (>=0.2.0,<1)", "pytest (>=6.0.1)", "pytest-md-report (>=0.6.2)", "simplejson (>=3.8.1,<4)", "sqliteschema (>=2)", "tablib (>=3.2.0)", "toml (>=0.9.3,<1)", "xlwt"] -theme = ["pytablewriter-altcol-theme (>=0.1.0,<1)", "pytablewriter-altrow-theme (>=0.2.0,<1)"] -toml = ["toml (>=0.9.3,<1)"] -yaml = ["PyYAML (>=3.11,<7)"] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pywin32" -version = "312" -description = "Python for Windows Extensions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "pywin32-312-cp310-cp310-win32.whl", hash = "sha256:772235332b5d1024c696f11cea1ae4be7930f0a8b894bb43db14e3f435f1ff7e"}, - {file = "pywin32-312-cp310-cp310-win_amd64.whl", hash = "sha256:5dbc35d2b5320dc07f25fa31269cfb767471002b17de5eb067d03da68c7cb2db"}, - {file = "pywin32-312-cp310-cp310-win_arm64.whl", hash = "sha256:3020656e34f1cf7faeb7bccd2b84653a607c6ff0c55ada85e6487d61716deabd"}, - {file = "pywin32-312-cp311-cp311-win32.whl", hash = "sha256:17948aeadbdb091f0ced6ef0841620794e68327b94ee415571c1203594b7215c"}, - {file = "pywin32-312-cp311-cp311-win_amd64.whl", hash = "sha256:d11417d84412f859b722fad0841b3614459ed0047f7542d8362e77884f6b6e8a"}, - {file = "pywin32-312-cp311-cp311-win_arm64.whl", hash = "sha256:b2200a054ca6d6625c4842fc56a4976a4b47f96b73dbe5538c3f813a80359f47"}, - {file = "pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b"}, - {file = "pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc"}, - {file = "pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950"}, - {file = "pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c"}, - {file = "pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9"}, - {file = "pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831"}, - {file = "pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b"}, - {file = "pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e"}, - {file = "pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa"}, - {file = "pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed"}, - {file = "pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5"}, - {file = "pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9"}, - {file = "pywin32-312-cp39-cp39-win32.whl", hash = "sha256:d620900033cc7531e50727c3c8333091df5dd3ffe6d68cdca38c03f5821408d5"}, - {file = "pywin32-312-cp39-cp39-win_amd64.whl", hash = "sha256:dc90147579a905b8635e1b0ec6514967dcb07e6e0d9c42f1477feef14cac23bb"}, - {file = "pywin32-312-cp39-cp39-win_arm64.whl", hash = "sha256:02ebca0f0242b75292e218065004310d6a477407c09fa449bfe4f6022bc0c0fc"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rich" -version = "15.0.0" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false -python-versions = ">=3.9.0" -groups = ["main"] -files = [ - {file = "rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb"}, - {file = "rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36"}, -] - -[package.dependencies] -markdown-it-py = ">=2.2.0" -pygments = ">=2.13.0,<3.0.0" - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "sacrebleu" -version = "2.6.0" -description = "Hassle-free computation of shareable, comparable, and reproducible BLEU, chrF, and TER scores" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sacrebleu-2.6.0-py3-none-any.whl", hash = "sha256:3edc1531575cfe4ad04ce53491a9307e234af1c3f805a1f491cbec844229a8a8"}, - {file = "sacrebleu-2.6.0.tar.gz", hash = "sha256:91499b6cd46138d95154fff1e863c2f9be57e82f0c719d8dd718d0006cf6c566"}, -] - -[package.dependencies] -colorama = "*" -lxml = "*" -numpy = ">=1.17" -portalocker = "*" -regex = "*" -tabulate = ">=0.8.9" - -[package.extras] -dev = ["lxml-stubs", "mypy", "pytest", "setuptools", "types-tabulate", "wheel"] -ja = ["ipadic (>=1.0,<2.0)", "mecab-python3 (>=1.0.9,<2.0.0)"] -ko = ["mecab-ko (>=1.0.2,<2.0.0)", "mecab-ko-dic (>=1.0,<2.0)"] - -[[package]] -name = "safetensors" -version = "0.8.0" -description = "" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0"}, - {file = "safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846"}, - {file = "safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d"}, - {file = "safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f"}, - {file = "safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452"}, - {file = "safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d"}, -] - -[package.extras] -all = ["safetensors[convert]", "safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] -convert = ["huggingface-hub (>=1.4)", "safetensors[torch]"] -dev = ["safetensors[all]", "safetensors[pinned-tf]"] -jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "safetensors[numpy]"] -mlx = ["mlx (>=0.0.9)"] -numpy = ["numpy (>=1.24.6)"] -paddlepaddle = ["paddlepaddle (>=2.4.1)", "safetensors[numpy]"] -pinned-tf = ["safetensors[numpy]", "tensorflow (==2.18.0)"] -quality = ["ruff"] -tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] -testing = ["fsspec (>=2024.6.0)", "h5py (>=3.7.0)", "hypothesis (>=6.70.2)", "pytest (>=9.0)", "pytest-benchmark (>=5.2)", "s3fs (>=2024.6.0)", "safetensors[numpy]", "setuptools-rust (>=1.12.0)"] -tf-nightly = ["safetensors[numpy]", "tf-nightly"] -torch = ["safetensors[numpy]", "torch (>=2.4)"] - -[[package]] -name = "scikit-learn" -version = "1.7.2" -description = "A set of python modules for machine learning and data mining" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "scikit_learn-1.7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b33579c10a3081d076ab403df4a4190da4f4432d443521674637677dc91e61f"}, - {file = "scikit_learn-1.7.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:36749fb62b3d961b1ce4fedf08fa57a1986cd409eff2d783bca5d4b9b5fce51c"}, - {file = "scikit_learn-1.7.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7a58814265dfc52b3295b1900cfb5701589d30a8bb026c7540f1e9d3499d5ec8"}, - {file = "scikit_learn-1.7.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a847fea807e278f821a0406ca01e387f97653e284ecbd9750e3ee7c90347f18"}, - {file = "scikit_learn-1.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:ca250e6836d10e6f402436d6463d6c0e4d8e0234cfb6a9a47835bd392b852ce5"}, - {file = "scikit_learn-1.7.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7509693451651cd7361d30ce4e86a1347493554f172b1c72a39300fa2aea79e"}, - {file = "scikit_learn-1.7.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:0486c8f827c2e7b64837c731c8feff72c0bd2b998067a8a9cbc10643c31f0fe1"}, - {file = "scikit_learn-1.7.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:89877e19a80c7b11a2891a27c21c4894fb18e2c2e077815bcade10d34287b20d"}, - {file = "scikit_learn-1.7.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8da8bf89d4d79aaec192d2bda62f9b56ae4e5b4ef93b6a56b5de4977e375c1f1"}, - {file = "scikit_learn-1.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:9b7ed8d58725030568523e937c43e56bc01cadb478fc43c042a9aca1dacb3ba1"}, - {file = "scikit_learn-1.7.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8d91a97fa2b706943822398ab943cde71858a50245e31bc71dba62aab1d60a96"}, - {file = "scikit_learn-1.7.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:acbc0f5fd2edd3432a22c69bed78e837c70cf896cd7993d71d51ba6708507476"}, - {file = "scikit_learn-1.7.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e5bf3d930aee75a65478df91ac1225ff89cd28e9ac7bd1196853a9229b6adb0b"}, - {file = "scikit_learn-1.7.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4d6e9deed1a47aca9fe2f267ab8e8fe82ee20b4526b2c0cd9e135cea10feb44"}, - {file = "scikit_learn-1.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:6088aa475f0785e01bcf8529f55280a3d7d298679f50c0bb70a2364a82d0b290"}, - {file = "scikit_learn-1.7.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0b7dacaa05e5d76759fb071558a8b5130f4845166d88654a0f9bdf3eb57851b7"}, - {file = "scikit_learn-1.7.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:abebbd61ad9e1deed54cca45caea8ad5f79e1b93173dece40bb8e0c658dbe6fe"}, - {file = "scikit_learn-1.7.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:502c18e39849c0ea1a5d681af1dbcf15f6cce601aebb657aabbfe84133c1907f"}, - {file = "scikit_learn-1.7.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a4c328a71785382fe3fe676a9ecf2c86189249beff90bf85e22bdb7efaf9ae0"}, - {file = "scikit_learn-1.7.2-cp313-cp313-win_amd64.whl", hash = "sha256:63a9afd6f7b229aad94618c01c252ce9e6fa97918c5ca19c9a17a087d819440c"}, - {file = "scikit_learn-1.7.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:9acb6c5e867447b4e1390930e3944a005e2cb115922e693c08a323421a6966e8"}, - {file = "scikit_learn-1.7.2-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:2a41e2a0ef45063e654152ec9d8bcfc39f7afce35b08902bfe290c2498a67a6a"}, - {file = "scikit_learn-1.7.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:98335fb98509b73385b3ab2bd0639b1f610541d3988ee675c670371d6a87aa7c"}, - {file = "scikit_learn-1.7.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:191e5550980d45449126e23ed1d5e9e24b2c68329ee1f691a3987476e115e09c"}, - {file = "scikit_learn-1.7.2-cp313-cp313t-win_amd64.whl", hash = "sha256:57dc4deb1d3762c75d685507fbd0bc17160144b2f2ba4ccea5dc285ab0d0e973"}, - {file = "scikit_learn-1.7.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fa8f63940e29c82d1e67a45d5297bdebbcb585f5a5a50c4914cc2e852ab77f33"}, - {file = "scikit_learn-1.7.2-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:f95dc55b7902b91331fa4e5845dd5bde0580c9cd9612b1b2791b7e80c3d32615"}, - {file = "scikit_learn-1.7.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9656e4a53e54578ad10a434dc1f993330568cfee176dff07112b8785fb413106"}, - {file = "scikit_learn-1.7.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96dc05a854add0e50d3f47a1ef21a10a595016da5b007c7d9cd9d0bffd1fcc61"}, - {file = "scikit_learn-1.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:bb24510ed3f9f61476181e4db51ce801e2ba37541def12dc9333b946fc7a9cf8"}, - {file = "scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda"}, -] - -[package.dependencies] -joblib = ">=1.2.0" -numpy = ">=1.22.0" -scipy = ">=1.8.0" -threadpoolctl = ">=3.1.0" - -[package.extras] -benchmark = ["matplotlib (>=3.5.0)", "memory_profiler (>=0.57.0)", "pandas (>=1.4.0)"] -build = ["cython (>=3.0.10)", "meson-python (>=0.17.1)", "numpy (>=1.22.0)", "scipy (>=1.8.0)"] -docs = ["Pillow (>=8.4.0)", "matplotlib (>=3.5.0)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.4.0)", "plotly (>=5.14.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pydata-sphinx-theme (>=0.15.3)", "scikit-image (>=0.19.0)", "seaborn (>=0.9.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.5.0)", "sphinx-design (>=0.6.0)", "sphinx-gallery (>=0.17.1)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)", "towncrier (>=24.8.0)"] -examples = ["matplotlib (>=3.5.0)", "pandas (>=1.4.0)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.19.0)", "seaborn (>=0.9.0)"] -install = ["joblib (>=1.2.0)", "numpy (>=1.22.0)", "scipy (>=1.8.0)", "threadpoolctl (>=3.1.0)"] -maintenance = ["conda-lock (==3.0.1)"] -tests = ["matplotlib (>=3.5.0)", "mypy (>=1.15)", "numpydoc (>=1.2.0)", "pandas (>=1.4.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pyamg (>=4.2.1)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.11.7)", "scikit-image (>=0.19.0)"] - -[[package]] -name = "scikit-learn" -version = "1.9.0" -description = "A set of python modules for machine learning and data mining" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "scikit_learn-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9db6f4d34e68c8899e4cab27fdf8eafe6ed21f2ba52ceb25ea250cd237f8e47b"}, - {file = "scikit_learn-1.9.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f401448645a3e7bc115aa3c094097865155b34bff1cba8101857d9104e99074c"}, - {file = "scikit_learn-1.9.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fd3a8ef0c758555a3b23c03adaa858af32f7736785ded50ad5991f59c4ed03fa"}, - {file = "scikit_learn-1.9.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7e254636164090da847715a27f8e5478feb98c40a9e0ee90cbd277de9e5ceb8"}, - {file = "scikit_learn-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:5dc1818c77575d149e25fce9ef82dd7b7263ae372f03494158668ad632a69759"}, - {file = "scikit_learn-1.9.0-cp311-cp311-win_arm64.whl", hash = "sha256:366652351f092b219c248f1e72821e841960a63d8f358f1dcfd54dc1cbdbbc28"}, - {file = "scikit_learn-1.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2bd41b0d201bc81575531b96b713d3eb5e5f50fb0b82101ff0f92294fdc236ac"}, - {file = "scikit_learn-1.9.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5be45aa4a42a68a533913a6ed736cf309de2226411c79ef8d609a5456f1939b1"}, - {file = "scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e50ed4da51974e86e940690e9a3d82e729b62b5a49f7c9bac534d515d39d86f"}, - {file = "scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:056c92bb67ad4c28463c2f2653d9701449201e7e7a9e94e321be0f71c4fef2b8"}, - {file = "scikit_learn-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:4306775fad04cc4b472a1b15af1ae9cede1540fbfcc17fbce3767cd8dc7ae283"}, - {file = "scikit_learn-1.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:26e22435f63bcdcf396b574273f29f13dd531f5ea035801f5be10ba1540a4e60"}, - {file = "scikit_learn-1.9.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:80746d63bd4b6eaca54d36fe5feaf4d28bb38dc6f9470f81c7cad7c40155f119"}, - {file = "scikit_learn-1.9.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5b934c45c252844a91d69fda3a34cff5e7307e1db10d77cb10a3980312c74713"}, - {file = "scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:38c3dcb9a1ffb85505ec53d54c7b4aea0cff70050425a7760c2af661ac85df05"}, - {file = "scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da76d09304a4706db7cc1e3ebaa3b6b98a67365cc11d2996c4f1e58ba47df714"}, - {file = "scikit_learn-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:5808d98f15c6bf6d9d96d2348c1997392a5888ce7097e664105f930c4bca1277"}, - {file = "scikit_learn-1.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:d77f54c017633791bc0225a43e2f8d03745fdcfe4880268fcc4df15f505dec2e"}, - {file = "scikit_learn-1.9.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9656acd4e93f74e0b66c8a36c88830a99252dfa900044d36bc2212ae89a47162"}, - {file = "scikit_learn-1.9.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:24360002ae845e7866522b0a5bbf690802e7bc388cac8663502e78aa98598aa2"}, - {file = "scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5162ad10a418c8a282dde04c9aa06965de3e9a65f33c1440c0ae69bb1a09d913"}, - {file = "scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fea2cc5677ab49d6f5bade978c866da44957b712d92e9635e8b4f723013c3cb"}, - {file = "scikit_learn-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:64fa347efc1c839c487433e40c5144d38c336e8a2b59c81aa8660373945c2673"}, - {file = "scikit_learn-1.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:1b944b6db288f6b926e3650026ddafb988929de95d11fc2cc5fa117773c9ba42"}, - {file = "scikit_learn-1.9.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4ccacf04ca5f4b492158a5f28afe0ace43f81b2571e4b9a66d34848b46128949"}, - {file = "scikit_learn-1.9.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ee1a8db2c18c08e34c7412d4b10be1cac214cd4ea7dc9715a6a327eb49a37c96"}, - {file = "scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:147e9329ef0e39f75d4cffa02b2aa48d827832684926cd5210d9a2cb5c57246b"}, - {file = "scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bad8f8b9950321b54c965fdcbac6c6c55e79e16646b49977bcf3668d3870a1a"}, - {file = "scikit_learn-1.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:78fc56eafd4edb9575d2d8950d1dd152061abb573341a1cb7e099fc40f6c6666"}, - {file = "scikit_learn-1.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:051075bda8b7aab87b1906ab3d4740a1e1224a19d7b3781a576736edc94e76aa"}, - {file = "scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557"}, -] - -[package.dependencies] -joblib = ">=1.4.0" -narwhals = ">=2.0.1" -numpy = ">=1.24.1" -scipy = ">=1.10.0" -threadpoolctl = ">=3.5.0" - -[package.extras] -benchmark = ["matplotlib (>=3.6.1)", "memory_profiler (>=0.57.0)", "pandas (>=1.5.0)"] -build = ["cython (>=3.1.2)", "meson-python (>=0.17.1)", "numpy (>=1.24.1)", "scipy (>=1.10.0)"] -docs = ["Pillow (>=12.1.1)", "matplotlib (>=3.6.1)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.5.0)", "plotly (>=5.22.0)", "polars (>=0.20.30)", "pooch (>=1.8.0)", "pydata-sphinx-theme (>=0.15.3)", "rich (>=14.1.0)", "scikit-image (>=0.22.0)", "seaborn (>=0.13.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.6.0)", "sphinx-gallery (>=0.17.1)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)", "towncrier (>=24.8.0)"] -examples = ["matplotlib (>=3.6.1)", "pandas (>=1.5.0)", "plotly (>=5.22.0)", "pooch (>=1.8.0)", "rich (>=14.1.0)", "scikit-image (>=0.22.0)", "seaborn (>=0.13.0)"] -install = ["joblib (>=1.4.0)", "narwhals (>=2.0.1)", "numpy (>=1.24.1)", "scipy (>=1.10.0)", "threadpoolctl (>=3.5.0)"] -maintenance = ["conda-lock (==3.0.1)"] -tests = ["matplotlib (>=3.6.1)", "mypy (>=1.15)", "numpydoc (>=1.2.0)", "pandas (>=1.5.0)", "polars (>=0.20.30)", "pooch (>=1.8.0)", "pyamg (>=5.0.0)", "pyarrow (>=13.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "rich (>=14.1.0)", "ruff (>=0.12.2)"] - -[[package]] -name = "scipy" -version = "1.15.3" -description = "Fundamental algorithms for scientific computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c"}, - {file = "scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253"}, - {file = "scipy-1.15.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:aef683a9ae6eb00728a542b796f52a5477b78252edede72b8327a886ab63293f"}, - {file = "scipy-1.15.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:1c832e1bd78dea67d5c16f786681b28dd695a8cb1fb90af2e27580d3d0967e92"}, - {file = "scipy-1.15.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:263961f658ce2165bbd7b99fa5135195c3a12d9bef045345016b8b50c315cb82"}, - {file = "scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40"}, - {file = "scipy-1.15.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed7284b21a7a0c8f1b6e5977ac05396c0d008b89e05498c8b7e8f4a1423bba0e"}, - {file = "scipy-1.15.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5380741e53df2c566f4d234b100a484b420af85deb39ea35a1cc1be84ff53a5c"}, - {file = "scipy-1.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:9d61e97b186a57350f6d6fd72640f9e99d5a4a2b8fbf4b9ee9a841eab327dc13"}, - {file = "scipy-1.15.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:993439ce220d25e3696d1b23b233dd010169b62f6456488567e830654ee37a6b"}, - {file = "scipy-1.15.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34716e281f181a02341ddeaad584205bd2fd3c242063bd3423d61ac259ca7eba"}, - {file = "scipy-1.15.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b0334816afb8b91dab859281b1b9786934392aa3d527cd847e41bb6f45bee65"}, - {file = "scipy-1.15.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6db907c7368e3092e24919b5e31c76998b0ce1684d51a90943cb0ed1b4ffd6c1"}, - {file = "scipy-1.15.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:721d6b4ef5dc82ca8968c25b111e307083d7ca9091bc38163fb89243e85e3889"}, - {file = "scipy-1.15.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39cb9c62e471b1bb3750066ecc3a3f3052b37751c7c3dfd0fd7e48900ed52982"}, - {file = "scipy-1.15.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:795c46999bae845966368a3c013e0e00947932d68e235702b5c3f6ea799aa8c9"}, - {file = "scipy-1.15.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18aaacb735ab38b38db42cb01f6b92a2d0d4b6aabefeb07f02849e47f8fb3594"}, - {file = "scipy-1.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:ae48a786a28412d744c62fd7816a4118ef97e5be0bee968ce8f0a2fba7acf3bb"}, - {file = "scipy-1.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6ac6310fdbfb7aa6612408bd2f07295bcbd3fda00d2d702178434751fe48e019"}, - {file = "scipy-1.15.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:185cd3d6d05ca4b44a8f1595af87f9c372bb6acf9c808e99aa3e9aa03bd98cf6"}, - {file = "scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:05dc6abcd105e1a29f95eada46d4a3f251743cfd7d3ae8ddb4088047f24ea477"}, - {file = "scipy-1.15.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:06efcba926324df1696931a57a176c80848ccd67ce6ad020c810736bfd58eb1c"}, - {file = "scipy-1.15.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05045d8b9bfd807ee1b9f38761993297b10b245f012b11b13b91ba8945f7e45"}, - {file = "scipy-1.15.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271e3713e645149ea5ea3e97b57fdab61ce61333f97cfae392c28ba786f9bb49"}, - {file = "scipy-1.15.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cfd56fc1a8e53f6e89ba3a7a7251f7396412d655bca2aa5611c8ec9a6784a1e"}, - {file = "scipy-1.15.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ff17c0bb1cb32952c09217d8d1eed9b53d1463e5f1dd6052c7857f83127d539"}, - {file = "scipy-1.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:52092bc0472cfd17df49ff17e70624345efece4e1a12b23783a1ac59a1b728ed"}, - {file = "scipy-1.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c620736bcc334782e24d173c0fdbb7590a0a436d2fdf39310a8902505008759"}, - {file = "scipy-1.15.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:7e11270a000969409d37ed399585ee530b9ef6aa99d50c019de4cb01e8e54e62"}, - {file = "scipy-1.15.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c9ed3ba2c8a2ce098163a9bdb26f891746d02136995df25227a20e71c396ebb"}, - {file = "scipy-1.15.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0bdd905264c0c9cfa74a4772cdb2070171790381a5c4d312c973382fc6eaf730"}, - {file = "scipy-1.15.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79167bba085c31f38603e11a267d862957cbb3ce018d8b38f79ac043bc92d825"}, - {file = "scipy-1.15.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9deabd6d547aee2c9a81dee6cc96c6d7e9a9b1953f74850c179f91fdc729cb7"}, - {file = "scipy-1.15.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dde4fc32993071ac0c7dd2d82569e544f0bdaff66269cb475e0f369adad13f11"}, - {file = "scipy-1.15.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f77f853d584e72e874d87357ad70f44b437331507d1c311457bed8ed2b956126"}, - {file = "scipy-1.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:b90ab29d0c37ec9bf55424c064312930ca5f4bde15ee8619ee44e69319aab163"}, - {file = "scipy-1.15.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3ac07623267feb3ae308487c260ac684b32ea35fd81e12845039952f558047b8"}, - {file = "scipy-1.15.3-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6487aa99c2a3d509a5227d9a5e889ff05830a06b2ce08ec30df6d79db5fcd5c5"}, - {file = "scipy-1.15.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:50f9e62461c95d933d5c5ef4a1f2ebf9a2b4e83b0db374cb3f1de104d935922e"}, - {file = "scipy-1.15.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14ed70039d182f411ffc74789a16df3835e05dc469b898233a245cdfd7f162cb"}, - {file = "scipy-1.15.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a769105537aa07a69468a0eefcd121be52006db61cdd8cac8a0e68980bbb723"}, - {file = "scipy-1.15.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db984639887e3dffb3928d118145ffe40eff2fa40cb241a306ec57c219ebbbb"}, - {file = "scipy-1.15.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:40e54d5c7e7ebf1aa596c374c49fa3135f04648a0caabcb66c52884b943f02b4"}, - {file = "scipy-1.15.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5e721fed53187e71d0ccf382b6bf977644c533e506c4d33c3fb24de89f5c3ed5"}, - {file = "scipy-1.15.3-cp313-cp313t-win_amd64.whl", hash = "sha256:76ad1fb5f8752eabf0fa02e4cc0336b4e8f021e2d5f061ed37d6d264db35e3ca"}, - {file = "scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf"}, -] - -[package.dependencies] -numpy = ">=1.23.5,<2.5" - -[package.extras] -dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] -doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.19.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.0.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] -test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] - -[[package]] -name = "scipy" -version = "1.17.1" -description = "Fundamental algorithms for scientific computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec"}, - {file = "scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696"}, - {file = "scipy-1.17.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee"}, - {file = "scipy-1.17.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd"}, - {file = "scipy-1.17.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c"}, - {file = "scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4"}, - {file = "scipy-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444"}, - {file = "scipy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082"}, - {file = "scipy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff"}, - {file = "scipy-1.17.1-cp311-cp311-win_arm64.whl", hash = "sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d"}, - {file = "scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8"}, - {file = "scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76"}, - {file = "scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086"}, - {file = "scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b"}, - {file = "scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21"}, - {file = "scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458"}, - {file = "scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb"}, - {file = "scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea"}, - {file = "scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87"}, - {file = "scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3"}, - {file = "scipy-1.17.1-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c"}, - {file = "scipy-1.17.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f"}, - {file = "scipy-1.17.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d"}, - {file = "scipy-1.17.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b"}, - {file = "scipy-1.17.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6"}, - {file = "scipy-1.17.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464"}, - {file = "scipy-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950"}, - {file = "scipy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369"}, - {file = "scipy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448"}, - {file = "scipy-1.17.1-cp313-cp313-win_arm64.whl", hash = "sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87"}, - {file = "scipy-1.17.1-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a"}, - {file = "scipy-1.17.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0"}, - {file = "scipy-1.17.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce"}, - {file = "scipy-1.17.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6"}, - {file = "scipy-1.17.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e"}, - {file = "scipy-1.17.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475"}, - {file = "scipy-1.17.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50"}, - {file = "scipy-1.17.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca"}, - {file = "scipy-1.17.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c"}, - {file = "scipy-1.17.1-cp313-cp313t-win_arm64.whl", hash = "sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49"}, - {file = "scipy-1.17.1-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717"}, - {file = "scipy-1.17.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9"}, - {file = "scipy-1.17.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b"}, - {file = "scipy-1.17.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866"}, - {file = "scipy-1.17.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350"}, - {file = "scipy-1.17.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118"}, - {file = "scipy-1.17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068"}, - {file = "scipy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118"}, - {file = "scipy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19"}, - {file = "scipy-1.17.1-cp314-cp314-win_arm64.whl", hash = "sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293"}, - {file = "scipy-1.17.1-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6"}, - {file = "scipy-1.17.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1"}, - {file = "scipy-1.17.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39"}, - {file = "scipy-1.17.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca"}, - {file = "scipy-1.17.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad"}, - {file = "scipy-1.17.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a"}, - {file = "scipy-1.17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4"}, - {file = "scipy-1.17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2"}, - {file = "scipy-1.17.1-cp314-cp314t-win_amd64.whl", hash = "sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484"}, - {file = "scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21"}, - {file = "scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0"}, -] - -[package.dependencies] -numpy = ">=1.26.4,<2.7" - -[package.extras] -dev = ["click (<8.3.0)", "cython-lint (>=0.12.2)", "mypy (==1.10.0)", "pycodestyle", "ruff (>=0.12.0)", "spin", "types-psutil", "typing_extensions"] -doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.19.1)", "jupytext", "linkify-it-py", "matplotlib (>=3.5)", "myst-nb (>=1.2.0)", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.2.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)", "tabulate"] -test = ["Cython", "array-api-strict (>=2.3.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest (>=8.0.0)", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] - -[[package]] -name = "scipy" -version = "1.18.0" -description = "Fundamental algorithms for scientific computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "scipy-1.18.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7bd21faaf5a1a3b2eff922d02db5f191b99a6518db9078a8fb23169f6d22259a"}, - {file = "scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:265915e79107de9f946b855e50d7470d5893ec3f54b342e1aa6201cbdcd8bb6b"}, - {file = "scipy-1.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9ab7b758be6940954a713ee466e2043e9f6e2ed965c1fce5c91039f4be3d90a9"}, - {file = "scipy-1.18.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:97b6cddaaee0a779ef6b5ca83c9604b27cc16b2b8fc22c142652df8793319fb8"}, - {file = "scipy-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52a96e21517c7292375c0e27dd796a811f03fcea5fd4d108fdfea8145dcf17ab"}, - {file = "scipy-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55797419e16e7f30cf88ffb3113ce0467f00cfe3f70d5c281730b21769bfc2"}, - {file = "scipy-1.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ad033410e2e0672ffdc1042110cef20e1c46f8fd0616cee1d44d8d58fad8fc11"}, - {file = "scipy-1.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a55985d54c769c872e64b7f4c8a81cc30ef700cc04296abbbf3705439c126de"}, - {file = "scipy-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:71ccc8faa2dd16ac310233203474a8b5cb67f10dedd54a3116d34943f4b19132"}, - {file = "scipy-1.18.0-cp312-cp312-win_arm64.whl", hash = "sha256:d88363fd9d8fbd3511bd273f1a49efb2a540773ddf92a91d57498ce7dd7f3e76"}, - {file = "scipy-1.18.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:09143f676d157d9f546d663504ef9c1becb819824f1afc018814176411942446"}, - {file = "scipy-1.18.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5efe260f69417b97ddae455bfb5a95e8359f7f66ad7fa9522a60feb66f169520"}, - {file = "scipy-1.18.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:68363b7eaacd8b5dd426df56d782cc156468ac79a127a1b87ca597d6e2e82197"}, - {file = "scipy-1.18.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:c5557d8be5da8e41353fcd4d21491fdbab83b062fc579e94dc09a7c8ab4f669b"}, - {file = "scipy-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d13bca67c096d89fb95ced0d8921807300fce0275643aef9533cc63a0773468"}, - {file = "scipy-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a46f9273dbd0eb1cefba61c9b8648b4dfe3cbc14a080176f9a73e44b8336dc7f"}, - {file = "scipy-1.18.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5aba46108853ddfc77906b6557aac839d2b52e900c1d72a1180adaaab58d265f"}, - {file = "scipy-1.18.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b6f758e35f12757b5d95c00bc6de2438e229c2664b7a92e96f205959d9f2dfa4"}, - {file = "scipy-1.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:1afac4a847207c7ff8efd321734a50b06d0280b3b2a2c0fc2f413101747ad7c7"}, - {file = "scipy-1.18.0-cp313-cp313-win_arm64.whl", hash = "sha256:c5dbddf60e58c2312316d097271a8e73d40eaf2eabfa4d95ed7d3695bbf2ce7b"}, - {file = "scipy-1.18.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4c256ee70c0d1a8a2ace807e199ccd4e3f57037433842abb3fb36bc17eaa9578"}, - {file = "scipy-1.18.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:2ef3abc54a4ffc53765374b0d5728532dfdd2585ed23f6b11c206a1f0b1b9af8"}, - {file = "scipy-1.18.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f2a6af57bd9e4a75d70e4117e78a1bbee84f79ae3fbb6d0111005d6ebcc4cb8d"}, - {file = "scipy-1.18.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:3f1ac564d3bf6c03d861d2cd87a1bea0da2887136f7fb1bf519c05a8971452d6"}, - {file = "scipy-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40395a5fcd1abee49a5c7aaa98c29db393eedc835138560a588c47ec16156690"}, - {file = "scipy-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ca01e8ae69f1b18e9a58d91afead31be3cef0dd905a10249dac559ee15460a0"}, - {file = "scipy-1.18.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7a7f3b01647384dbc3a711e8c6778e0aabbe93959249fef5c7393396bcac0867"}, - {file = "scipy-1.18.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6aa94e78ec192a30063a5e72e561c28af769dc311190b24fe91774eff1969709"}, - {file = "scipy-1.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:2d8bbdc6c817f5b4006a54d799d4f5bab6f910193cbb9a1ff310833d4d270f61"}, - {file = "scipy-1.18.0-cp314-cp314-win_arm64.whl", hash = "sha256:18e9575f1569b2c54174e6159d32942e03731177f63dce7975f0a0c88d102f5b"}, - {file = "scipy-1.18.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f351e0dd702687d12a402b867a1b4146a256923e1c38317cbc472f6372b94707"}, - {file = "scipy-1.18.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7c7a51b33ce387193c97f228320cf8e87361daa1bba750638677729598b3e677"}, - {file = "scipy-1.18.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:84031d7b052a54fae2f8632e0ec802073d385476eb9a63079bce6e23ef9283d4"}, - {file = "scipy-1.18.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:56abf29a7c067dde59be8b9a22d606a4ea1b2f2a4b756d9d903c62818f5dacce"}, - {file = "scipy-1.18.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ad44305cfa24b1ba5803cbbebf033590ccbac1aa5d612d727b785325ab408b0"}, - {file = "scipy-1.18.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:945c1761b93f38d7f99ae81ae80c63e621471608c7eeead563f6df025585cd58"}, - {file = "scipy-1.18.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a4441f15d620578772a49e5ab48c0ee1f7a0220e387110283062729136b2553"}, - {file = "scipy-1.18.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9aac6192fac56bf2ca534389d24623f07b39ff83317d58287285e7fbd622ff76"}, - {file = "scipy-1.18.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e40baea28ae7f5475c779741e2d90b1247c78531207b49c7030e698ff81cee3f"}, - {file = "scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8"}, - {file = "scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378"}, -] - -[package.dependencies] -numpy = ">=2.0.0,<2.8" - -[package.extras] -dev = ["click (<8.3.0)", "cython-lint (>=0.12.2)", "mypy (==1.19.1)", "pycodestyle", "pyrefly (==0.63.0)", "ruff (>=0.12.0)", "spin", "types-psutil", "typing_extensions"] -doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.19.1)", "jupytext", "linkify-it-py", "matplotlib (>=3.5)", "myst-nb (>=1.2.0)", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.2.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)", "tabulate"] -test = ["Cython", "array-api-strict (>=2.3.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest (>=8.0.0)", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "scipy-doctest (>=2.0.0)", "threadpoolctl"] - -[[package]] -name = "setuptools" -version = "83.0.0" -description = "Most extensible Python build backend with support for C/C++ extension modules" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3"}, - {file = "setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef"}, -] - -[package.extras] -check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.13.0) ; sys_platform != \"cygwin\""] -core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -enabler = ["pytest-enabler (>=3.4)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.18.*)", "pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] - -[[package]] -name = "shellingham" -version = "1.5.4" -description = "Tool to Detect Surrounding Shell" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, - {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, -] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "sqlitedict" -version = "2.1.0" -description = "Persistent dict in Python, backed up by sqlite3 and pickle, multithread-safe." -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "sqlitedict-2.1.0.tar.gz", hash = "sha256:03d9cfb96d602996f1d4c2db2856f1224b96a9c431bdd16e78032a72940f9e8c"}, -] - -[[package]] -name = "sympy" -version = "1.14.0" -description = "Computer algebra system (CAS) in Python" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5"}, - {file = "sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517"}, -] - -[package.dependencies] -mpmath = ">=1.1.0,<1.4" - -[package.extras] -dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"] - -[[package]] -name = "tabledata" -version = "1.3.5" -description = "tabledata is a Python library to represent tabular data. Used for pytablewriter/pytablereader/SimpleSQLite/etc." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "tabledata-1.3.5-py3-none-any.whl", hash = "sha256:a1e57afc4767b51bef551114c0df31f205d712dbb75e3caf9be7834a79f23136"}, - {file = "tabledata-1.3.5.tar.gz", hash = "sha256:98c64d0ad6b520846b41000fb3f5b2f42fa7ca2675c2c669e5ccab6b93082a36"}, -] - -[package.dependencies] -DataProperty = ">=1.0.2,<2" -typepy = ">=1.2.0,<3" - -[package.extras] -logging = ["loguru (>=0.4.1,<1)"] -test = ["pytablewriter (>=0.46)", "pytest"] - -[[package]] -name = "tabulate" -version = "0.10.0" -description = "Pretty-print tabular data" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3"}, - {file = "tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d"}, -] - -[package.extras] -widechars = ["wcwidth"] - -[[package]] -name = "tcolorpy" -version = "0.1.7" -description = "tcolopy is a Python library to apply true color for terminal text." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tcolorpy-0.1.7-py3-none-any.whl", hash = "sha256:26a59d52027e175a37e0aba72efc99dda43f074db71f55b316d3de37d3251378"}, - {file = "tcolorpy-0.1.7.tar.gz", hash = "sha256:0fbf6bf238890bbc2e32662aa25736769a29bf6d880328f310c910a327632614"}, -] - -[package.extras] -test = ["pytest (>=6.0.1)", "pytest-md-report (>=0.6.2)"] - -[[package]] -name = "tenacity" -version = "9.1.4" -description = "Retry code until it succeeds" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55"}, - {file = "tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a"}, -] - -[package.extras] -doc = ["reno", "sphinx"] -test = ["pytest", "tornado (>=4.5)", "typeguard"] - -[[package]] -name = "threadpoolctl" -version = "3.6.0" -description = "threadpoolctl" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb"}, - {file = "threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e"}, -] - -[[package]] -name = "tiktoken" -version = "0.13.0" -description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tiktoken-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:47b1df8d73390a24f94980c75158cdd5c56d256f16d55f30cb49c230caba9ba4"}, - {file = "tiktoken-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7d40c6c5aab171dcd6eb8455bc567bde404bb9def60cdb8c1299cc782b242bb9"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:9b842981fa91accdffd48ff6408a977b7a91c3fbda55d353c3c68114d5c9d69e"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:ed5a30027cb4d8c7ca8b273d4766f3db3cf58fad9e9f3b1a68a351ffb54873d5"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7ab10f4a21c2999846940113f6dbd72e0fa06a24119feddd74cc47e85818e06d"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a2937ad042d49d50eac6e1ba07c5661d4bd3942a5b1e0c0d08475c4df83676e1"}, - {file = "tiktoken-0.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:44733b99bfd72b590cd0936b1c01b3b4dd73122db2d544bc1ceeb18a7678c910"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7bfe1849caa65d1e1d9871817170ec497bbb7984e182012e1bdce72f66608cdb"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:91c180fe255bd5a86d8316210d2833a1d4d33d026cd86a67812f4773743c8d26"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:059c8ecf554eb5b41e6e054ba467b871b03277d267dee7244380aca4359747d4"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:36217497eaffc158607a3b26f065300db2aefd43b115263f3b9688ce38146173"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:303f7d91b4fce3baddbcde05c139091d4caa5026ac7214c1dc7ff7a71ee429ff"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5d48843bee149630eb735a99e1f4a85b47308d21868ea63163f6e87768d3cfed"}, - {file = "tiktoken-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:fc1c44cd37b43fc46bae593129164f4f281e82ea116b57a85aa81bda57eafc94"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91"}, - {file = "tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615"}, - {file = "tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f"}, - {file = "tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51"}, - {file = "tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424"}, - {file = "tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:35e1ea1e0631c04f551297284a1ab7e1f65a3c55a9a48728d5e0f66b4527c04a"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a3b536c55802fe42f4b4644d2be4f04bf788506b48de0a0a658cb58f8bce232"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b8ac2d6420ff05841a89ba5205c6d45f56c4f6843454f3c884b7eb1a2a8dddb2"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:477c9a38e20d0ed248090509acf1e839ad3967a4f00b4b0f958210049f656dee"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:da86f8c96ac1c235d7a3b3eebff1eacfdbcfb8ad792706943268d4d2938fbafe"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9b8858b29804b3a0add25ce9e62fb00f89f621dc754d75d03ca419d17e8ddf67"}, - {file = "tiktoken-0.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:b967dfb9d0adf9a631953b1b40717684f04478270fc51bbccdd2f838d67a2f00"}, - {file = "tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1"}, -] - -[package.dependencies] -regex = "*" -requests = "*" - -[package.extras] -blobfile = ["blobfile (>=3)"] - -[[package]] -name = "tokenizers" -version = "0.22.2" -description = "" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c"}, - {file = "tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5"}, - {file = "tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92"}, - {file = "tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48"}, - {file = "tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113"}, - {file = "tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917"}, -] - -[package.dependencies] -huggingface-hub = ">=0.16.4,<2.0" - -[package.extras] -dev = ["tokenizers[testing]"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] -testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", "ty"] - -[[package]] -name = "torch" -version = "2.13.0" -description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d"}, - {file = "torch-2.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0ab4b69f3ee03a62a002cfbf77b1ca5e88aceb4ea64cb4388bb28f638ddbb045"}, - {file = "torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c78b7b4d04461855a764cf01bae9a462bb88bc93defcfa11235cbc8fdf3e12c4"}, - {file = "torch-2.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:2bd30b6b730d987fa386ce3898933762c5cb8cc82eb0535211d787cc3ce2dfeb"}, - {file = "torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8"}, - {file = "torch-2.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:092790c696a760c729fd5722835f50b9d81fd7c8f141571f3f3cf4081a8f664c"}, - {file = "torch-2.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:60fcdcb2f3876e21146cb4524ef06397d727ca9ad5f020818547e25075fe3cb7"}, - {file = "torch-2.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a0d8b11f16a48d60e2015d8213aa0390744cbebb98e58b62b3514dddc656e330"}, - {file = "torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027"}, - {file = "torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4"}, - {file = "torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b"}, - {file = "torch-2.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:024c6cc0c1b085f2f91f20a3dc27b0471d021c31ce84b81be3afdc39f791fd9d"}, - {file = "torch-2.13.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:33449899ce5496c1b84b4853179d94fd102028ae1407314d9fb956bb79e70d09"}, - {file = "torch-2.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1e09d6a722504957c694faceca843acde562786df1144ebcc5a74075ec7f6005"}, - {file = "torch-2.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a3a9a21312872af8a26950b2c15680335a386a1f56ed03e780653d78b9607e9e"}, - {file = "torch-2.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:49b58f1e2c52440abb6f17c28f0335fe6c6d01ad1a7f55b0183b81e4b34d64e6"}, - {file = "torch-2.13.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d849b390e07d8d333ce8ecaf91b273c656c598379a19c9acf1318a883f6b391c"}, - {file = "torch-2.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a3893dc2da0a972a8ca5d698c85a9f967559ac5f8ee1797b77408aa8734d073c"}, - {file = "torch-2.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:49f1ea385c754e54919408a9bb3b5a72b0b755bbe2c916c1d6f70afbec4908a2"}, - {file = "torch-2.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f8573e3ce9ebcd53fe922f01077a6085ccdfbe5f12fd215883a9d87d7a744fd"}, - {file = "torch-2.13.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c28def70706c2f9ecc752574766e8ae4da9b810ab6676b611166761a78a9f1e1"}, - {file = "torch-2.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:31061ff56ed8fbf26c749806905aeb749ebeb819810fd5d52508aa5afd90dddc"}, - {file = "torch-2.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:cc26eead4cf51d0b544e31e364dcf000846549c273bd148936fe9d24d29acb92"}, - {file = "torch-2.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a7de8a313090dc5c7d7ba4bfe5c3be222528f9a4dba1acc83bddb1157360c4b8"}, -] - -[package.dependencies] -cuda-bindings = {version = ">=13.0.3,<14", markers = "platform_system == \"Linux\" and python_version < \"3.15\""} -cuda-toolkit = {version = "13.0.3", extras = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], markers = "platform_system == \"Linux\""} -filelock = "*" -fsspec = ">=0.8.5" -jinja2 = "*" -networkx = ">=2.5.1" -nvidia-cudnn-cu13 = {version = "9.20.0.48", markers = "platform_system == \"Linux\""} -nvidia-cusparselt-cu13 = {version = "0.8.1", markers = "platform_system == \"Linux\""} -nvidia-nccl-cu13 = {version = "2.29.7", markers = "platform_system == \"Linux\""} -nvidia-nvshmem-cu13 = {version = "3.4.5", markers = "platform_system == \"Linux\""} -setuptools = ">=77.0.3" -sympy = ">=1.13.3" -triton = {version = "3.7.1", markers = "platform_system == \"Linux\" and python_version < \"3.15\""} -typing-extensions = ">=4.10.0" - -[package.extras] -opt-einsum = ["opt-einsum (>=3.3)"] -optree = ["optree (>=0.13.0)"] -pyyaml = ["pyyaml"] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "tqdm-multiprocess" -version = "0.0.11" -description = "Easy multiprocessing with tqdm and logging redirected to main process." -optional = false -python-versions = ">=3.6" -groups = ["main"] -files = [ - {file = "tqdm-multiprocess-0.0.11.tar.gz", hash = "sha256:a74002a1222ea9cbe8cdc9bd460108c6009be359621fbee9b92d0515d4d180f7"}, - {file = "tqdm_multiprocess-0.0.11-py3-none-any.whl", hash = "sha256:3ebdf03e7a675150fa0bbceaa9c3c64b8cb556e9ffafa4fe6c078e51820524aa"}, -] - -[package.dependencies] -colorama = "*" -tqdm = "*" - -[package.extras] -dev = ["twine"] - -[[package]] -name = "transformers" -version = "5.14.0" -description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "transformers-5.14.0-py3-none-any.whl", hash = "sha256:288287da4fdcdae0b611be32b5be20d4bcbf5caaa5f269292927fb1ff888d19d"}, - {file = "transformers-5.14.0.tar.gz", hash = "sha256:9e1a2b345c195d216afd10082394b915a82e339fa09e9013c32094ff835ca9db"}, -] - -[package.dependencies] -huggingface-hub = ">=1.5.0,<2.0" -numpy = ">=1.17" -packaging = ">=20.0" -pyyaml = ">=5.1" -regex = ">=2025.10.22" -safetensors = ">=0.8.0" -tokenizers = ">=0.22.0,<=0.23.0" -tqdm = ">=4.60" -typer = "*" - -[package.extras] -accelerate = ["accelerate (>=1.1.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.11.5)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] -audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -benchmark = ["optimum-benchmark (>=0.3.0)"] -chat-template = ["jinja2 (>=3.1.0)"] -codecarbon = ["codecarbon (>=2.8.1)"] -deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.11.5)", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] -docs = ["hf-doc-builder"] -integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.15.2,<0.16)", "optuna", "ray[tune] (>=2.7.0)"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] -kernels = ["kernels (>=0.15.2,<0.16)"] -mistral-common = ["mistral-common[image] (>=1.11.5)"] -num2words = ["num2words"] -optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] -ray = ["ray[tune] (>=2.7.0)"] -retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] -sagemaker = ["sagemaker (>=2.31.0)"] -sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] -sklearn = ["scikit-learn"] -testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -tiktoken = ["blobfile", "tiktoken"] -timm = ["timm (>=1.0.23)"] -torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] -video = ["av"] -vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] - -[[package]] -name = "triton" -version = "3.7.1" -description = "A language and compiler for custom Deep Learning operations" -optional = false -python-versions = "<3.15,>=3.10" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "triton-3.7.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3daf64305d6cea88d3334c65ebc9bcd0c64c9564a977084366aa768d57cbcf64"}, - {file = "triton-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee89fbf782ec2ad50391dd1cf26cbea4f4467154c37f4773026da8fc31c0f58e"}, - {file = "triton-3.7.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a0e1cd4c4a76370ed74a8432a53cea28716827d19e40ffc732233e35ceb3f6"}, - {file = "triton-3.7.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6744957e9fd610a29680ec2346057d0c86948ed3812468670719f391e94b44a5"}, - {file = "triton-3.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1"}, - {file = "triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728"}, - {file = "triton-3.7.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cdbfc09d9ec58bc5e68321525653220de7515c199e7a8097a97c85e62b52cd0a"}, - {file = "triton-3.7.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58c0e131da05134a2a4788ccbcc0c1105cf0f54c8e98f19e34cd465396dc15eb"}, - {file = "triton-3.7.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe4ea396a06171f1f1f58cbd39c70b09294398f7dd7c620939bab54ad6f934fa"}, - {file = "triton-3.7.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2020153b08280415ec0da6607834e79166442147e78e144df06b508c75b186d2"}, - {file = "triton-3.7.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c58e4c61f0c73b5dba3b5d19b4a7093c32f90dc18b2a7f121a7c16ccd31107b7"}, - {file = "triton-3.7.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10ba85fa2cca4a2fbdeb36bf1cb082f2c252bda55bf9fccd74f65ec5bc647e68"}, -] - -[package.extras] -build = ["cmake (>=3.20,<4.0)", "lit"] -tests = ["autopep8", "isort", "llnl-hatchet", "numpy", "pytest", "pytest-forked", "pytest-xdist", "scipy (>=1.7.1)"] -tutorials = ["matplotlib", "pandas", "tabulate"] - -[[package]] -name = "typepy" -version = "1.3.5" -description = "typepy is a Python library for variable type checker/validator/converter at a run time." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typepy-1.3.5-py3-none-any.whl", hash = "sha256:de361b59609c7503efc2edbe9d7a4e053ae71307bf90ae1678ec4d6bcd807922"}, - {file = "typepy-1.3.5.tar.gz", hash = "sha256:a1c5f54c41860f89bab175f512b11e8c9a57cfe7b8b3d5ae5d52d828b756b6dd"}, -] - -[package.dependencies] -mbstrdecoder = ">=1.0.0,<2" -packaging = {version = "*", optional = true, markers = "extra == \"datetime\""} -python-dateutil = {version = ">=2.8.0,<3.0.0", optional = true, markers = "extra == \"datetime\""} -pytz = {version = ">=2018.9", optional = true, markers = "extra == \"datetime\""} - -[package.extras] -datetime = ["packaging", "python-dateutil (>=2.8.0,<3.0.0)", "pytz (>=2018.9)"] -test = ["packaging", "pytest (>=6.0.1)", "python-dateutil (>=2.8.0,<3.0.0)", "pytz (>=2018.9)", "tcolorpy"] - -[[package]] -name = "typer" -version = "0.27.0" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, -] - -[package.dependencies] -annotated-doc = ">=0.0.2" -colorama = {version = "*", markers = "platform_system == \"Windows\""} -rich = ">=13.8.0" -shellingham = ">=1.3.0" - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "python_version == \"3.10\" or sys_platform == \"win32\" or sys_platform == \"emscripten\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "word2number" -version = "1.1" -description = "Convert number words eg. three hundred and forty two to numbers (342)." -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "word2number-1.1.zip", hash = "sha256:70e27a5d387f67b04c71fbb7621c05930b19bfd26efd6851e6e0f9969dcde7d0"}, -] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[[package]] -name = "zstandard" -version = "0.25.0" -description = "Zstandard bindings for Python" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "zstandard-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd"}, - {file = "zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7"}, - {file = "zstandard-0.25.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550"}, - {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d"}, - {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b"}, - {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0"}, - {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74"}, - {file = "zstandard-0.25.0-cp310-cp310-win32.whl", hash = "sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa"}, - {file = "zstandard-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e"}, - {file = "zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c"}, - {file = "zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f"}, - {file = "zstandard-0.25.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431"}, - {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a"}, - {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc"}, - {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6"}, - {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7"}, - {file = "zstandard-0.25.0-cp311-cp311-win32.whl", hash = "sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4"}, - {file = "zstandard-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2"}, - {file = "zstandard-0.25.0-cp311-cp311-win_arm64.whl", hash = "sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137"}, - {file = "zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b"}, - {file = "zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00"}, - {file = "zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64"}, - {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea"}, - {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb"}, - {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a"}, - {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa"}, - {file = "zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd"}, - {file = "zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01"}, - {file = "zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9"}, - {file = "zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94"}, - {file = "zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1"}, - {file = "zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f"}, - {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea"}, - {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e"}, - {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551"}, - {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf"}, - {file = "zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09"}, - {file = "zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5"}, - {file = "zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049"}, - {file = "zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3"}, - {file = "zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f"}, - {file = "zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c"}, - {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439"}, - {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043"}, - {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859"}, - {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0"}, - {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7"}, - {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2"}, - {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344"}, - {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c"}, - {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088"}, - {file = "zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12"}, - {file = "zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2"}, - {file = "zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d"}, - {file = "zstandard-0.25.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b9af1fe743828123e12b41dd8091eca1074d0c1569cc42e6e1eee98027f2bbd0"}, - {file = "zstandard-0.25.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4b14abacf83dfb5c25eb4e4a79520de9e7e205f72c9ee7702f91233ae57d33a2"}, - {file = "zstandard-0.25.0-cp39-cp39-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:a51ff14f8017338e2f2e5dab738ce1ec3b5a851f23b18c1ae1359b1eecbee6df"}, - {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3b870ce5a02d4b22286cf4944c628e0f0881b11b3f14667c1d62185a99e04f53"}, - {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:05353cef599a7b0b98baca9b068dd36810c3ef0f42bf282583f438caf6ddcee3"}, - {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:19796b39075201d51d5f5f790bf849221e58b48a39a5fc74837675d8bafc7362"}, - {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:53e08b2445a6bc241261fea89d065536f00a581f02535f8122eba42db9375530"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1f3689581a72eaba9131b1d9bdbfe520ccd169999219b41000ede2fca5c1bfdb"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d8c56bb4e6c795fc77d74d8e8b80846e1fb8292fc0b5060cd8131d522974b751"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:53f94448fe5b10ee75d246497168e5825135d54325458c4bfffbaafabcc0a577"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c2ba942c94e0691467ab901fc51b6f2085ff48f2eea77b1a48240f011e8247c7"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:07b527a69c1e1c8b5ab1ab14e2afe0675614a09182213f21a0717b62027b5936"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:51526324f1b23229001eb3735bc8c94f9c578b1bd9e867a0a646a3b17109f388"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89c4b48479a43f820b749df49cd7ba2dbc2b1b78560ecb5ab52985574fd40b27"}, - {file = "zstandard-0.25.0-cp39-cp39-win32.whl", hash = "sha256:1cd5da4d8e8ee0e88be976c294db744773459d51bb32f707a0f166e5ad5c8649"}, - {file = "zstandard-0.25.0-cp39-cp39-win_amd64.whl", hash = "sha256:37daddd452c0ffb65da00620afb8e17abd4adaae6ce6310702841760c2c26860"}, - {file = "zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b"}, -] - -[package.extras] -cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and python_version < \"3.14\"", "cffi (>=2.0.0b0) ; platform_python_implementation != \"PyPy\" and python_version >= \"3.14\""] - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "9b21a24bd320b638cb4343da8c7beeda0064fec50232199eb2a2796b011deac6" diff --git a/security_scanning/examples/llm-eval/lm-eval-harness/pyproject.toml b/security_scanning/examples/llm-eval/lm-eval-harness/pyproject.toml deleted file mode 100644 index 3c6eca5cb5ee..000000000000 --- a/security_scanning/examples/llm-eval/lm-eval-harness/pyproject.toml +++ /dev/null @@ -1,16 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "lm-eval[api] (==0.4.7)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock b/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock deleted file mode 100644 index 0871b91e2b22..000000000000 --- a/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock +++ /dev/null @@ -1,2487 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "datasets" -version = "3.1.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, - {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, -] - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.9" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} -huggingface-hub = ">=0.23.0" -multiprocess = "<0.70.17" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=15.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] -docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -quality = ["ruff (>=0.3.0)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2024.9.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, - {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.16" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, - {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, - {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, -] - -[package.dependencies] -dill = ">=0.3.8" - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "protobuf" -version = "7.35.1" -description = "" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6"}, - {file = "protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799"}, - {file = "protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4"}, - {file = "protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4"}, - {file = "protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30"}, - {file = "protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87"}, - {file = "protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9"}, - {file = "protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a"}, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "sentencepiece" -version = "0.2.2" -description = "Unsupervised text tokenizer and detokenizer." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_arm64.whl", hash = "sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d"}, - {file = "sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6"}, -] - -[package.extras] -numpy = ["numpy"] -protobuf = ["protobuf"] -test = ["numpy", "protobuf", "pytest"] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "tiktoken" -version = "0.13.0" -description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tiktoken-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:47b1df8d73390a24f94980c75158cdd5c56d256f16d55f30cb49c230caba9ba4"}, - {file = "tiktoken-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7d40c6c5aab171dcd6eb8455bc567bde404bb9def60cdb8c1299cc782b242bb9"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:9b842981fa91accdffd48ff6408a977b7a91c3fbda55d353c3c68114d5c9d69e"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:ed5a30027cb4d8c7ca8b273d4766f3db3cf58fad9e9f3b1a68a351ffb54873d5"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7ab10f4a21c2999846940113f6dbd72e0fa06a24119feddd74cc47e85818e06d"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a2937ad042d49d50eac6e1ba07c5661d4bd3942a5b1e0c0d08475c4df83676e1"}, - {file = "tiktoken-0.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:44733b99bfd72b590cd0936b1c01b3b4dd73122db2d544bc1ceeb18a7678c910"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7bfe1849caa65d1e1d9871817170ec497bbb7984e182012e1bdce72f66608cdb"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:91c180fe255bd5a86d8316210d2833a1d4d33d026cd86a67812f4773743c8d26"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:059c8ecf554eb5b41e6e054ba467b871b03277d267dee7244380aca4359747d4"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:36217497eaffc158607a3b26f065300db2aefd43b115263f3b9688ce38146173"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:303f7d91b4fce3baddbcde05c139091d4caa5026ac7214c1dc7ff7a71ee429ff"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5d48843bee149630eb735a99e1f4a85b47308d21868ea63163f6e87768d3cfed"}, - {file = "tiktoken-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:fc1c44cd37b43fc46bae593129164f4f281e82ea116b57a85aa81bda57eafc94"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91"}, - {file = "tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615"}, - {file = "tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f"}, - {file = "tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51"}, - {file = "tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424"}, - {file = "tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:35e1ea1e0631c04f551297284a1ab7e1f65a3c55a9a48728d5e0f66b4527c04a"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a3b536c55802fe42f4b4644d2be4f04bf788506b48de0a0a658cb58f8bce232"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b8ac2d6420ff05841a89ba5205c6d45f56c4f6843454f3c884b7eb1a2a8dddb2"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:477c9a38e20d0ed248090509acf1e839ad3967a4f00b4b0f958210049f656dee"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:da86f8c96ac1c235d7a3b3eebff1eacfdbcfb8ad792706943268d4d2938fbafe"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9b8858b29804b3a0add25ce9e62fb00f89f621dc754d75d03ca419d17e8ddf67"}, - {file = "tiktoken-0.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:b967dfb9d0adf9a631953b1b40717684f04478270fc51bbccdd2f838d67a2f00"}, - {file = "tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1"}, -] - -[package.dependencies] -regex = "*" -requests = "*" - -[package.extras] -blobfile = ["blobfile (>=3)"] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "70a344e7439e1e5bb7c72d8e7a0c914a17731571bd09853de3da5a8b5d14251b" diff --git a/security_scanning/examples/models/contrib/chatglm-6b/pyproject.toml b/security_scanning/examples/models/contrib/chatglm-6b/pyproject.toml deleted file mode 100644 index 6350395dec0e..000000000000 --- a/security_scanning/examples/models/contrib/chatglm-6b/pyproject.toml +++ /dev/null @@ -1,21 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "datasets (==3.1.0)", - "evaluate (>=0.4.6,<0.5.0)", - "protobuf (>=7.35.1,<8.0.0)", - "rouge-score (>=0.1.2,<0.2.0)", - "sentencepiece (>=0.2.2,<0.3.0)", - "tiktoken (>=0.13.0,<0.14.0)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock b/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock deleted file mode 100644 index 0871b91e2b22..000000000000 --- a/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock +++ /dev/null @@ -1,2487 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "datasets" -version = "3.1.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, - {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, -] - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.9" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} -huggingface-hub = ">=0.23.0" -multiprocess = "<0.70.17" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=15.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] -docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -quality = ["ruff (>=0.3.0)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2024.9.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, - {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.16" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, - {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, - {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, -] - -[package.dependencies] -dill = ">=0.3.8" - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "protobuf" -version = "7.35.1" -description = "" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6"}, - {file = "protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799"}, - {file = "protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4"}, - {file = "protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4"}, - {file = "protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30"}, - {file = "protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87"}, - {file = "protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9"}, - {file = "protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a"}, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "sentencepiece" -version = "0.2.2" -description = "Unsupervised text tokenizer and detokenizer." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_arm64.whl", hash = "sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d"}, - {file = "sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6"}, -] - -[package.extras] -numpy = ["numpy"] -protobuf = ["protobuf"] -test = ["numpy", "protobuf", "pytest"] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "tiktoken" -version = "0.13.0" -description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tiktoken-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:47b1df8d73390a24f94980c75158cdd5c56d256f16d55f30cb49c230caba9ba4"}, - {file = "tiktoken-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7d40c6c5aab171dcd6eb8455bc567bde404bb9def60cdb8c1299cc782b242bb9"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:9b842981fa91accdffd48ff6408a977b7a91c3fbda55d353c3c68114d5c9d69e"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:ed5a30027cb4d8c7ca8b273d4766f3db3cf58fad9e9f3b1a68a351ffb54873d5"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7ab10f4a21c2999846940113f6dbd72e0fa06a24119feddd74cc47e85818e06d"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a2937ad042d49d50eac6e1ba07c5661d4bd3942a5b1e0c0d08475c4df83676e1"}, - {file = "tiktoken-0.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:44733b99bfd72b590cd0936b1c01b3b4dd73122db2d544bc1ceeb18a7678c910"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7bfe1849caa65d1e1d9871817170ec497bbb7984e182012e1bdce72f66608cdb"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:91c180fe255bd5a86d8316210d2833a1d4d33d026cd86a67812f4773743c8d26"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:059c8ecf554eb5b41e6e054ba467b871b03277d267dee7244380aca4359747d4"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:36217497eaffc158607a3b26f065300db2aefd43b115263f3b9688ce38146173"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:303f7d91b4fce3baddbcde05c139091d4caa5026ac7214c1dc7ff7a71ee429ff"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5d48843bee149630eb735a99e1f4a85b47308d21868ea63163f6e87768d3cfed"}, - {file = "tiktoken-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:fc1c44cd37b43fc46bae593129164f4f281e82ea116b57a85aa81bda57eafc94"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91"}, - {file = "tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615"}, - {file = "tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f"}, - {file = "tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51"}, - {file = "tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424"}, - {file = "tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:35e1ea1e0631c04f551297284a1ab7e1f65a3c55a9a48728d5e0f66b4527c04a"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a3b536c55802fe42f4b4644d2be4f04bf788506b48de0a0a658cb58f8bce232"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b8ac2d6420ff05841a89ba5205c6d45f56c4f6843454f3c884b7eb1a2a8dddb2"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:477c9a38e20d0ed248090509acf1e839ad3967a4f00b4b0f958210049f656dee"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:da86f8c96ac1c235d7a3b3eebff1eacfdbcfb8ad792706943268d4d2938fbafe"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9b8858b29804b3a0add25ce9e62fb00f89f621dc754d75d03ca419d17e8ddf67"}, - {file = "tiktoken-0.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:b967dfb9d0adf9a631953b1b40717684f04478270fc51bbccdd2f838d67a2f00"}, - {file = "tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1"}, -] - -[package.dependencies] -regex = "*" -requests = "*" - -[package.extras] -blobfile = ["blobfile (>=3)"] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "70a344e7439e1e5bb7c72d8e7a0c914a17731571bd09853de3da5a8b5d14251b" diff --git a/security_scanning/examples/models/contrib/chatglm2-6b/pyproject.toml b/security_scanning/examples/models/contrib/chatglm2-6b/pyproject.toml deleted file mode 100644 index 6350395dec0e..000000000000 --- a/security_scanning/examples/models/contrib/chatglm2-6b/pyproject.toml +++ /dev/null @@ -1,21 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "datasets (==3.1.0)", - "evaluate (>=0.4.6,<0.5.0)", - "protobuf (>=7.35.1,<8.0.0)", - "rouge-score (>=0.1.2,<0.2.0)", - "sentencepiece (>=0.2.2,<0.3.0)", - "tiktoken (>=0.13.0,<0.14.0)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock b/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock deleted file mode 100644 index 0871b91e2b22..000000000000 --- a/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock +++ /dev/null @@ -1,2487 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "datasets" -version = "3.1.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, - {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, -] - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.9" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} -huggingface-hub = ">=0.23.0" -multiprocess = "<0.70.17" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=15.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] -docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -quality = ["ruff (>=0.3.0)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2024.9.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, - {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.16" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, - {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, - {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, -] - -[package.dependencies] -dill = ">=0.3.8" - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "protobuf" -version = "7.35.1" -description = "" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6"}, - {file = "protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799"}, - {file = "protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4"}, - {file = "protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4"}, - {file = "protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30"}, - {file = "protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87"}, - {file = "protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9"}, - {file = "protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a"}, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "sentencepiece" -version = "0.2.2" -description = "Unsupervised text tokenizer and detokenizer." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_arm64.whl", hash = "sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d"}, - {file = "sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6"}, -] - -[package.extras] -numpy = ["numpy"] -protobuf = ["protobuf"] -test = ["numpy", "protobuf", "pytest"] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "tiktoken" -version = "0.13.0" -description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tiktoken-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:47b1df8d73390a24f94980c75158cdd5c56d256f16d55f30cb49c230caba9ba4"}, - {file = "tiktoken-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7d40c6c5aab171dcd6eb8455bc567bde404bb9def60cdb8c1299cc782b242bb9"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:9b842981fa91accdffd48ff6408a977b7a91c3fbda55d353c3c68114d5c9d69e"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:ed5a30027cb4d8c7ca8b273d4766f3db3cf58fad9e9f3b1a68a351ffb54873d5"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7ab10f4a21c2999846940113f6dbd72e0fa06a24119feddd74cc47e85818e06d"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a2937ad042d49d50eac6e1ba07c5661d4bd3942a5b1e0c0d08475c4df83676e1"}, - {file = "tiktoken-0.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:44733b99bfd72b590cd0936b1c01b3b4dd73122db2d544bc1ceeb18a7678c910"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7bfe1849caa65d1e1d9871817170ec497bbb7984e182012e1bdce72f66608cdb"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:91c180fe255bd5a86d8316210d2833a1d4d33d026cd86a67812f4773743c8d26"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:059c8ecf554eb5b41e6e054ba467b871b03277d267dee7244380aca4359747d4"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:36217497eaffc158607a3b26f065300db2aefd43b115263f3b9688ce38146173"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:303f7d91b4fce3baddbcde05c139091d4caa5026ac7214c1dc7ff7a71ee429ff"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5d48843bee149630eb735a99e1f4a85b47308d21868ea63163f6e87768d3cfed"}, - {file = "tiktoken-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:fc1c44cd37b43fc46bae593129164f4f281e82ea116b57a85aa81bda57eafc94"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91"}, - {file = "tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615"}, - {file = "tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f"}, - {file = "tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51"}, - {file = "tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424"}, - {file = "tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:35e1ea1e0631c04f551297284a1ab7e1f65a3c55a9a48728d5e0f66b4527c04a"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a3b536c55802fe42f4b4644d2be4f04bf788506b48de0a0a658cb58f8bce232"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b8ac2d6420ff05841a89ba5205c6d45f56c4f6843454f3c884b7eb1a2a8dddb2"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:477c9a38e20d0ed248090509acf1e839ad3967a4f00b4b0f958210049f656dee"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:da86f8c96ac1c235d7a3b3eebff1eacfdbcfb8ad792706943268d4d2938fbafe"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9b8858b29804b3a0add25ce9e62fb00f89f621dc754d75d03ca419d17e8ddf67"}, - {file = "tiktoken-0.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:b967dfb9d0adf9a631953b1b40717684f04478270fc51bbccdd2f838d67a2f00"}, - {file = "tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1"}, -] - -[package.dependencies] -regex = "*" -requests = "*" - -[package.extras] -blobfile = ["blobfile (>=3)"] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "70a344e7439e1e5bb7c72d8e7a0c914a17731571bd09853de3da5a8b5d14251b" diff --git a/security_scanning/examples/models/contrib/chatglm3-6b-32k/pyproject.toml b/security_scanning/examples/models/contrib/chatglm3-6b-32k/pyproject.toml deleted file mode 100644 index 6350395dec0e..000000000000 --- a/security_scanning/examples/models/contrib/chatglm3-6b-32k/pyproject.toml +++ /dev/null @@ -1,21 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "datasets (==3.1.0)", - "evaluate (>=0.4.6,<0.5.0)", - "protobuf (>=7.35.1,<8.0.0)", - "rouge-score (>=0.1.2,<0.2.0)", - "sentencepiece (>=0.2.2,<0.3.0)", - "tiktoken (>=0.13.0,<0.14.0)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock index ba30259eb5f4..15cae9ae5808 100644 --- a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock +++ b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock @@ -22,14 +22,14 @@ trio = ["trio (>=0.32.0)"] [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -97,14 +97,14 @@ all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == [[package]] name = "cuda-pathfinder" -version = "1.5.6" +version = "1.6.0" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, + {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, ] [[package]] @@ -199,14 +199,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.29.7" +version = "3.32.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, + {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, + {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, ] [[package]] @@ -263,38 +263,30 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.1" +version = "1.5.2" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\")" files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, + {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, ] [package.extras] @@ -349,14 +341,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.23.0" +version = "1.25.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, + {file = "huggingface_hub-1.25.1-py3-none-any.whl", hash = "sha256:004d4e70350517e24c68a7dbb7dc5e40b2b6aefef8f94bf7a85f6f9835102ea5"}, + {file = "huggingface_hub-1.25.1.tar.gz", hash = "sha256:21129595ca7a753be479b319913e22cc8808361ac118bd76cc413db831b28a99"}, ] [package.dependencies] @@ -1290,14 +1282,14 @@ scipy = ["scipy"] [[package]] name = "tqdm" -version = "4.68.4" +version = "4.70.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, + {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, + {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/internlm/poetry.lock b/security_scanning/examples/models/contrib/internlm/poetry.lock deleted file mode 100644 index 66374311f86f..000000000000 --- a/security_scanning/examples/models/contrib/internlm/poetry.lock +++ /dev/null @@ -1,2395 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "datasets" -version = "3.1.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, - {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, -] - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.9" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} -huggingface-hub = ">=0.23.0" -multiprocess = "<0.70.17" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=15.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] -docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -quality = ["ruff (>=0.3.0)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2024.9.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, - {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.16" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, - {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, - {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, -] - -[package.dependencies] -dill = ">=0.3.8" - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "sentencepiece" -version = "0.2.2" -description = "Unsupervised text tokenizer and detokenizer." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_arm64.whl", hash = "sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d"}, - {file = "sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6"}, -] - -[package.extras] -numpy = ["numpy"] -protobuf = ["protobuf"] -test = ["numpy", "protobuf", "pytest"] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "24cdc2fcd697c95206d488c7e95a21f441d72dafce3a8fa5015c51273af83c00" diff --git a/security_scanning/examples/models/contrib/internlm/pyproject.toml b/security_scanning/examples/models/contrib/internlm/pyproject.toml deleted file mode 100644 index 717cb32e3b33..000000000000 --- a/security_scanning/examples/models/contrib/internlm/pyproject.toml +++ /dev/null @@ -1,19 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "datasets (==3.1.0)", - "rouge-score (>=0.1.2,<0.2.0)", - "sentencepiece (>=0.1.99)", - "evaluate (>=0.4.6,<0.5.0)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/contrib/jais/poetry.lock b/security_scanning/examples/models/contrib/jais/poetry.lock deleted file mode 100644 index 0f548f4c56af..000000000000 --- a/security_scanning/examples/models/contrib/jais/poetry.lock +++ /dev/null @@ -1,2395 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "datasets" -version = "3.1.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, - {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, -] - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.9" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} -huggingface-hub = ">=0.23.0" -multiprocess = "<0.70.17" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=15.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] -docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -quality = ["ruff (>=0.3.0)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2024.9.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, - {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.16" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, - {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, - {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, -] - -[package.dependencies] -dill = ">=0.3.8" - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "sentencepiece" -version = "0.2.2" -description = "Unsupervised text tokenizer and detokenizer." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_arm64.whl", hash = "sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d"}, - {file = "sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6"}, -] - -[package.extras] -numpy = ["numpy"] -protobuf = ["protobuf"] -test = ["numpy", "protobuf", "pytest"] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "09a90aee92909874a9141a92032b89a82cef48f8fcd27e5513d3157ab9f87989" diff --git a/security_scanning/examples/models/contrib/jais/pyproject.toml b/security_scanning/examples/models/contrib/jais/pyproject.toml deleted file mode 100644 index 97e0a7d76e0e..000000000000 --- a/security_scanning/examples/models/contrib/jais/pyproject.toml +++ /dev/null @@ -1,19 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "datasets (==3.1.0)", - "evaluate (>=0.4.6,<0.5.0)", - "rouge-score (>=0.1.2,<0.2.0)", - "sentencepiece (>=0.1.99)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/contrib/skywork/poetry.lock b/security_scanning/examples/models/contrib/skywork/poetry.lock deleted file mode 100644 index 0f548f4c56af..000000000000 --- a/security_scanning/examples/models/contrib/skywork/poetry.lock +++ /dev/null @@ -1,2395 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "datasets" -version = "3.1.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, - {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, -] - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.9" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} -huggingface-hub = ">=0.23.0" -multiprocess = "<0.70.17" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=15.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] -docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -quality = ["ruff (>=0.3.0)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2024.9.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, - {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.16" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, - {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, - {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, -] - -[package.dependencies] -dill = ">=0.3.8" - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "sentencepiece" -version = "0.2.2" -description = "Unsupervised text tokenizer and detokenizer." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_arm64.whl", hash = "sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d"}, - {file = "sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6"}, -] - -[package.extras] -numpy = ["numpy"] -protobuf = ["protobuf"] -test = ["numpy", "protobuf", "pytest"] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "09a90aee92909874a9141a92032b89a82cef48f8fcd27e5513d3157ab9f87989" diff --git a/security_scanning/examples/models/contrib/skywork/pyproject.toml b/security_scanning/examples/models/contrib/skywork/pyproject.toml deleted file mode 100644 index 97e0a7d76e0e..000000000000 --- a/security_scanning/examples/models/contrib/skywork/pyproject.toml +++ /dev/null @@ -1,19 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "datasets (==3.1.0)", - "evaluate (>=0.4.6,<0.5.0)", - "rouge-score (>=0.1.2,<0.2.0)", - "sentencepiece (>=0.1.99)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/contrib/smaug/poetry.lock b/security_scanning/examples/models/contrib/smaug/poetry.lock deleted file mode 100644 index 0f548f4c56af..000000000000 --- a/security_scanning/examples/models/contrib/smaug/poetry.lock +++ /dev/null @@ -1,2395 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "datasets" -version = "3.1.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, - {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, -] - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.9" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} -huggingface-hub = ">=0.23.0" -multiprocess = "<0.70.17" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=15.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] -docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -quality = ["ruff (>=0.3.0)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2024.9.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, - {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.16" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, - {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, - {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, -] - -[package.dependencies] -dill = ">=0.3.8" - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "sentencepiece" -version = "0.2.2" -description = "Unsupervised text tokenizer and detokenizer." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_arm64.whl", hash = "sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d"}, - {file = "sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6"}, -] - -[package.extras] -numpy = ["numpy"] -protobuf = ["protobuf"] -test = ["numpy", "protobuf", "pytest"] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "09a90aee92909874a9141a92032b89a82cef48f8fcd27e5513d3157ab9f87989" diff --git a/security_scanning/examples/models/contrib/smaug/pyproject.toml b/security_scanning/examples/models/contrib/smaug/pyproject.toml deleted file mode 100644 index 97e0a7d76e0e..000000000000 --- a/security_scanning/examples/models/contrib/smaug/pyproject.toml +++ /dev/null @@ -1,19 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "datasets (==3.1.0)", - "evaluate (>=0.4.6,<0.5.0)", - "rouge-score (>=0.1.2,<0.2.0)", - "sentencepiece (>=0.1.99)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/core/mixtral/poetry.lock b/security_scanning/examples/models/core/mixtral/poetry.lock deleted file mode 100644 index 90ddd0e3ce42..000000000000 --- a/security_scanning/examples/models/core/mixtral/poetry.lock +++ /dev/null @@ -1,1510 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "accelerate" -version = "0.25.0" -description = "Accelerate" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "accelerate-0.25.0-py3-none-any.whl", hash = "sha256:c7bb817eb974bba0ff3ea1ba0f24d55afb86d50e3d4fe98d6922dc69cf2ccff1"}, - {file = "accelerate-0.25.0.tar.gz", hash = "sha256:ecf55b0ab278a1dac8539dde0d276977aff04683f07ede73eaf02478538576a1"}, -] - -[package.dependencies] -huggingface-hub = "*" -numpy = ">=1.17" -packaging = ">=20.0" -psutil = "*" -pyyaml = "*" -safetensors = ">=0.3.1" -torch = ">=1.10.0" - -[package.extras] -dev = ["bitsandbytes", "black (>=23.1,<24.0)", "datasets", "deepspeed", "evaluate", "hf-doc-builder (>=0.3.0)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "rich", "ruff (>=0.0.241)", "scikit-learn", "scipy", "timm", "tqdm", "transformers", "urllib3 (<2.0.0)"] -quality = ["black (>=23.1,<24.0)", "hf-doc-builder (>=0.3.0)", "ruff (>=0.0.241)", "urllib3 (<2.0.0)"] -rich = ["rich"] -sagemaker = ["sagemaker"] -test-dev = ["bitsandbytes", "datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "timm", "tqdm", "transformers"] -test-prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] -test-trackers = ["comet-ml", "dvclive", "tensorboard", "wandb"] -testing = ["bitsandbytes", "datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "timm", "tqdm", "transformers"] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "cuda-bindings" -version = "13.3.1" -description = "Python bindings for CUDA" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, - {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, - {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, - {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, - {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, - {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, - {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, - {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, - {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, - {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, - {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, - {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, - {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, - {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, - {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, - {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, - {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, - {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, -] - -[package.dependencies] -cuda-pathfinder = ">=1.4.2" - -[package.extras] -all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] - -[[package]] -name = "cuda-pathfinder" -version = "1.5.6" -description = "Pathfinder for CUDA components" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, -] - -[[package]] -name = "cuda-toolkit" -version = "13.0.3" -description = "CUDA Toolkit meta-package" -optional = false -python-versions = "*" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f"}, -] - -[package.dependencies] -nvidia-cublas = {version = "==13.1.1.3.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cublas\" or extra == \"cusolver\")"} -nvidia-cuda-cupti = {version = "==13.0.85.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cupti\""} -nvidia-cuda-nvrtc = {version = "==13.0.88.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cublas\" or extra == \"nvrtc\")"} -nvidia-cuda-runtime = {version = "==13.0.96.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cudart\""} -nvidia-cufft = {version = "==12.0.0.61.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cufft\""} -nvidia-cufile = {version = "==1.15.1.6.*", optional = true, markers = "sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cufile\""} -nvidia-curand = {version = "==10.4.0.35.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"curand\""} -nvidia-cusolver = {version = "==12.0.4.66.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cusolver\""} -nvidia-cusparse = {version = "==12.6.3.3.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cusolver\" or extra == \"cusparse\")"} -nvidia-nvjitlink = {version = ">=13.0.88,<14", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cufft\" or extra == \"cusolver\" or extra == \"cusparse\" or extra == \"nvjitlink\")"} -nvidia-nvtx = {version = "==13.0.85.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"nvtx\""} - -[package.extras] -all = ["nvidia-cublas (==13.1.1.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-cccl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-culibos (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")", "nvidia-cuda-cupti (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-cuxxfilt (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvcc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-opencl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-profiler-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cufft (==12.0.0.61.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cufile (==1.15.1.6.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")", "nvidia-curand (==10.4.0.35.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusolver (==12.0.4.66.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-npp (==13.0.1.2.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvfatbin (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjpeg (==13.0.1.86.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvml-dev (==13.0.87.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvptxcompiler (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvtx (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cccl = ["nvidia-cuda-cccl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -crt = ["nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cublas = ["nvidia-cublas (==13.1.1.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cudart = ["nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cufft = ["nvidia-cufft (==12.0.0.61.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cufile = ["nvidia-cufile (==1.15.1.6.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")"] -culibos = ["nvidia-cuda-culibos (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")"] -cupti = ["nvidia-cuda-cupti (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -curand = ["nvidia-curand (==10.4.0.35.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cusolver = ["nvidia-cublas (==13.1.1.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusolver (==12.0.4.66.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cusparse = ["nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cuxxfilt = ["nvidia-cuda-cuxxfilt (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -npp = ["nvidia-npp (==13.0.1.2.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvcc = ["nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvcc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvfatbin = ["nvidia-nvfatbin (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvjitlink = ["nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvjpeg = ["nvidia-nvjpeg (==13.0.1.86.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvml = ["nvidia-nvml-dev (==13.0.87.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvptxcompiler = ["nvidia-nvptxcompiler (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvrtc = ["nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvtx = ["nvidia-nvtx (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvvm = ["nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -opencl = ["nvidia-cuda-opencl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -profiler = ["nvidia-cuda-profiler-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -sanitizer = ["nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "fsspec" -version = "2026.6.0" -description = "File-system specification" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1"}, - {file = "fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a"}, -] - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff (>=0.5)"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs (>2024.2.0)", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs (>2024.2.0)", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs (>2024.2.0)"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs (>2024.2.0)"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "backports-zstd ; python_version < \"3.14\"", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas (<3.0.0)", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr (<3.2.0)", "zstandard ; python_version < \"3.14\""] -tqdm = ["tqdm"] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\")" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "huggingface-hub" -version = "0.36.2" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-0.36.2-py3-none-any.whl", hash = "sha256:48f0c8eac16145dfce371e9d2d7772854a4f591bcb56c9cf548accf531d54270"}, - {file = "huggingface_hub-0.36.2.tar.gz", hash = "sha256:1934304d2fb224f8afa3b87007d58501acfda9215b334eed53072dd5e815ff7a"}, -] - -[package.dependencies] -filelock = "*" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.1.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -packaging = ">=20.9" -pyyaml = ">=5.1" -requests = "*" -tqdm = ">=4.42.1" -typing-extensions = ">=3.7.4.3" - -[package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0) ; python_version >= \"3.9\"", "mypy (>=1.14.1,<1.15.0) ; python_version == \"3.8\"", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0) ; python_version >= \"3.9\"", "mypy (>=1.14.1,<1.15.0) ; python_version == \"3.8\"", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -hf-transfer = ["hf_transfer (>=0.1.4)"] -hf-xet = ["hf-xet (>=1.1.2,<2.0.0)"] -inference = ["aiohttp"] -mcp = ["aiohttp", "mcp (>=1.8.0)", "typer"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0) ; python_version >= \"3.9\"", "mypy (>=1.14.1,<1.15.0) ; python_version == \"3.8\"", "ruff (>=0.9.0)", "ty"] -tensorflow = ["graphviz", "pydot", "tensorflow"] -tensorflow-testing = ["keras (<3.0)", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "jinja2" -version = "3.1.6" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, - {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "markupsafe" -version = "3.0.3" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, - {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1"}, - {file = "markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa"}, - {file = "markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8"}, - {file = "markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1"}, - {file = "markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad"}, - {file = "markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a"}, - {file = "markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19"}, - {file = "markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01"}, - {file = "markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c"}, - {file = "markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e"}, - {file = "markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b"}, - {file = "markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d"}, - {file = "markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c"}, - {file = "markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f"}, - {file = "markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795"}, - {file = "markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12"}, - {file = "markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed"}, - {file = "markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5"}, - {file = "markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485"}, - {file = "markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73"}, - {file = "markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287"}, - {file = "markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe"}, - {file = "markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe"}, - {file = "markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9"}, - {file = "markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581"}, - {file = "markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4"}, - {file = "markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab"}, - {file = "markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa"}, - {file = "markupsafe-3.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26"}, - {file = "markupsafe-3.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d"}, - {file = "markupsafe-3.0.3-cp39-cp39-win32.whl", hash = "sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7"}, - {file = "markupsafe-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e"}, - {file = "markupsafe-3.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8"}, - {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -description = "Python library for arbitrary-precision floating-point arithmetic" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, - {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, -] - -[package.extras] -develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] -docs = ["sphinx"] -gmpy = ["gmpy2 (>=2.1.0a4) ; platform_python_implementation != \"PyPy\""] -tests = ["pytest (>=4.6)"] - -[[package]] -name = "networkx" -version = "3.4.2" -description = "Python package for creating and manipulating graphs and networks" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, - {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, -] - -[package.extras] -default = ["matplotlib (>=3.7)", "numpy (>=1.24)", "pandas (>=2.0)", "scipy (>=1.10,!=1.11.0,!=1.11.1)"] -developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] -doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.15)", "sphinx (>=7.3)", "sphinx-gallery (>=0.16)", "texext (>=0.6.7)"] -example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy (>=0.7.2)", "osmnx (>=1.9)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] -extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] -test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "nvidia-cublas" -version = "13.1.1.3" -description = "CUBLAS native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5"}, - {file = "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436"}, - {file = "nvidia_cublas-13.1.1.3-py3-none-win_amd64.whl", hash = "sha256:b6cdce694e47ff6aadf0a69df1cab6628d696f5ff56e8d16af50309d855fa20f"}, -] - -[package.dependencies] -nvidia-cuda-nvrtc = "*" - -[[package]] -name = "nvidia-cublas" -version = "13.6.0.2" -description = "CUBLAS native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform != \"linux\" or platform_machine != \"aarch64\" and platform_machine != \"x86_64\") and platform_system == \"Linux\" and (sys_platform != \"linux\" and sys_platform != \"win32\" or platform_machine != \"x86_64\")" -files = [ - {file = "nvidia_cublas-13.6.0.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:946f6a252b1cc72d8de912c75975fd6d8ba44f67d4e5044fe764ddb909f4a688"}, - {file = "nvidia_cublas-13.6.0.2-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b82c80c886cea6da6e149a5c3bdba274f12b7e4ec4b00a050b916b0446fb4153"}, - {file = "nvidia_cublas-13.6.0.2-py3-none-win_amd64.whl", hash = "sha256:3b5bcd6bfb6f65010ebf195851bcb9b2aa34b9fe08479432002991c1fe84b67d"}, -] - -[package.dependencies] -nvidia-cuda-nvrtc = "*" - -[[package]] -name = "nvidia-cuda-cupti" -version = "13.0.85" -description = "CUDA profiling tools runtime libs." -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151"}, - {file = "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8"}, - {file = "nvidia_cuda_cupti-13.0.85-py3-none-win_amd64.whl", hash = "sha256:683f58d301548deeefcb8f6fac1b8d907691b9d8b18eccab417f51e362102f00"}, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.0.88" -description = "NVRTC native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575"}, - {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b"}, - {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-win_amd64.whl", hash = "sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872"}, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.3.33" -description = "NVRTC native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform != \"linux\" or platform_machine != \"aarch64\" and platform_machine != \"x86_64\") and platform_system == \"Linux\" and (sys_platform != \"linux\" and sys_platform != \"win32\" or platform_machine != \"x86_64\")" -files = [ - {file = "nvidia_cuda_nvrtc-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:82530788b8c6164a54d3fd9ae8bcca8893d397c4aeb998861982a03bbe41e204"}, - {file = "nvidia_cuda_nvrtc-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7b05ecda494c6dabc44231a608b060a71008a730d9dfda932cc508e6d29159e0"}, - {file = "nvidia_cuda_nvrtc-13.3.33-py3-none-win_amd64.whl", hash = "sha256:7d2af818851c0c224d5f92221e9226e51ee23c236df4b51f9194563979c888be"}, -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.0.96" -description = "CUDA Runtime native Libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55"}, - {file = "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548"}, - {file = "nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492"}, -] - -[[package]] -name = "nvidia-cudnn-cu13" -version = "9.20.0.48" -description = "cuDNN runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1"}, - {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304"}, - {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-win_amd64.whl", hash = "sha256:af8139732b99c0118be65ea5aac97f0d46018f8c552889e49d2fb0c6261a4a24"}, -] - -[package.dependencies] -nvidia-cublas = "*" - -[[package]] -name = "nvidia-cufft" -version = "12.0.0.61" -description = "CUFFT native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5"}, - {file = "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3"}, - {file = "nvidia_cufft-12.0.0.61-py3-none-win_amd64.whl", hash = "sha256:2abce5b39d2f5ae12730fb7e5db6696533e36c26e2d3e8fd1750bdd2853364eb"}, -] - -[package.dependencies] -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cufile" -version = "1.15.1.6" -description = "cuFile GPUDirect libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "sys_platform == \"linux\" and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44"}, - {file = "nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1"}, -] - -[[package]] -name = "nvidia-curand" -version = "10.4.0.35" -description = "CURAND native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a"}, - {file = "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc"}, - {file = "nvidia_curand-10.4.0.35-py3-none-win_amd64.whl", hash = "sha256:65b1710aa6961d326b411e314b374290904c5ddf41dc3f766ebc3f1d7d4ca69f"}, -] - -[[package]] -name = "nvidia-cusolver" -version = "12.0.4.66" -description = "CUDA solver native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2"}, - {file = "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112"}, - {file = "nvidia_cusolver-12.0.4.66-py3-none-win_amd64.whl", hash = "sha256:16515bd33a8e76bb54d024cfa068fa68d30e80fc34b9e1090813ea9362e0cb65"}, -] - -[package.dependencies] -nvidia-cublas = "*" -nvidia-cusparse = "*" -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cusparse" -version = "12.6.3.3" -description = "CUSPARSE native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c"}, - {file = "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b"}, - {file = "nvidia_cusparse-12.6.3.3-py3-none-win_amd64.whl", hash = "sha256:cbcf42feb737bd7ec15b4c0a63e62351886bd3f975027b8815d7f720a2b5ea79"}, -] - -[package.dependencies] -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cusparse" -version = "12.8.2.51" -description = "CUSPARSE native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "" -files = [ - {file = "nvidia_cusparse-12.8.2.51-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:00469fcf62c4d464a1225abd9b20864ecff35e3fbc9fb992572e83d358927755"}, - {file = "nvidia_cusparse-12.8.2.51-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65cbcc4e37a34fca4ee7df2fd57da103593842cda1bbb4a144664ecfe59873a5"}, - {file = "nvidia_cusparse-12.8.2.51-py3-none-win_amd64.whl", hash = "sha256:2ee59291cd362038f3d40d57c7cd09b26d689f3873ae5c94b31c3270772d41b8"}, -] - -[package.dependencies] -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cusparselt-cu13" -version = "0.8.1" -description = "NVIDIA cuSPARSELt" -optional = false -python-versions = "*" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f"}, - {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0"}, - {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-win_amd64.whl", hash = "sha256:dccbd362f91a7b9024d1f55ee9f548ac065027ff15d8c8b0db889ab3a8f31215"}, -] - -[[package]] -name = "nvidia-nccl-cu13" -version = "2.29.7" -description = "NVIDIA Collective Communication Library (NCCL) Runtime" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5"}, - {file = "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d"}, -] - -[[package]] -name = "nvidia-nvjitlink" -version = "13.3.33" -description = "Nvidia JIT LTO Library" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5"}, - {file = "nvidia_nvjitlink-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ce48b37dfeb3cb1eae4cf85adacb47d7a6539ea2272870c9a3628ce275c2037e"}, - {file = "nvidia_nvjitlink-13.3.33-py3-none-win_amd64.whl", hash = "sha256:4297ee49639b4f2e07255a1d69b3acc7ab2d011bb892b403e91ac98368962e3b"}, -] - -[[package]] -name = "nvidia-nvshmem-cu13" -version = "3.4.5" -description = "NVSHMEM creates a global address space that provides efficient and scalable communication for NVIDIA GPU clusters." -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9"}, - {file = "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80"}, -] - -[[package]] -name = "nvidia-nvtx" -version = "13.0.85" -description = "NVIDIA Tools Extension" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4"}, - {file = "nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6"}, - {file = "nvidia_nvtx-13.0.85-py3-none-win_amd64.whl", hash = "sha256:d66ea44254dd3c6eacc300047af6e1288d2269dd072b417e0adffbf479e18519"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "psutil" -version = "7.2.2" -description = "Cross-platform lib for process and system monitoring." -optional = false -python-versions = ">=3.6" -groups = ["main"] -files = [ - {file = "psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b"}, - {file = "psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea"}, - {file = "psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63"}, - {file = "psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312"}, - {file = "psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b"}, - {file = "psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9"}, - {file = "psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00"}, - {file = "psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9"}, - {file = "psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a"}, - {file = "psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf"}, - {file = "psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1"}, - {file = "psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841"}, - {file = "psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486"}, - {file = "psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979"}, - {file = "psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9"}, - {file = "psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e"}, - {file = "psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8"}, - {file = "psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc"}, - {file = "psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988"}, - {file = "psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee"}, - {file = "psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372"}, -] - -[package.extras] -dev = ["abi3audit", "black", "check-manifest", "colorama ; os_name == \"nt\"", "coverage", "packaging", "psleak", "pylint", "pyperf", "pypinfo", "pyreadline3 ; os_name == \"nt\"", "pytest", "pytest-cov", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "validate-pyproject[all]", "virtualenv", "vulture", "wheel", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] -test = ["psleak", "pytest", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "setuptools", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "safetensors" -version = "0.8.0" -description = "" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0"}, - {file = "safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846"}, - {file = "safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d"}, - {file = "safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f"}, - {file = "safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452"}, - {file = "safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d"}, -] - -[package.extras] -all = ["safetensors[convert]", "safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] -convert = ["huggingface-hub (>=1.4)", "safetensors[torch]"] -dev = ["safetensors[all]", "safetensors[pinned-tf]"] -jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "safetensors[numpy]"] -mlx = ["mlx (>=0.0.9)"] -numpy = ["numpy (>=1.24.6)"] -paddlepaddle = ["paddlepaddle (>=2.4.1)", "safetensors[numpy]"] -pinned-tf = ["safetensors[numpy]", "tensorflow (==2.18.0)"] -quality = ["ruff"] -tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] -testing = ["fsspec (>=2024.6.0)", "h5py (>=3.7.0)", "hypothesis (>=6.70.2)", "pytest (>=9.0)", "pytest-benchmark (>=5.2)", "s3fs (>=2024.6.0)", "safetensors[numpy]", "setuptools-rust (>=1.12.0)"] -tf-nightly = ["safetensors[numpy]", "tf-nightly"] -torch = ["safetensors[numpy]", "torch (>=2.4)"] - -[[package]] -name = "setuptools" -version = "83.0.0" -description = "Most extensible Python build backend with support for C/C++ extension modules" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3"}, - {file = "setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef"}, -] - -[package.extras] -check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.13.0) ; sys_platform != \"cygwin\""] -core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -enabler = ["pytest-enabler (>=3.4)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.18.*)", "pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] - -[[package]] -name = "sympy" -version = "1.14.0" -description = "Computer algebra system (CAS) in Python" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5"}, - {file = "sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517"}, -] - -[package.dependencies] -mpmath = ">=1.1.0,<1.4" - -[package.extras] -dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"] - -[[package]] -name = "tokenizers" -version = "0.22.2" -description = "" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c"}, - {file = "tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5"}, - {file = "tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92"}, - {file = "tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48"}, - {file = "tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113"}, - {file = "tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917"}, -] - -[package.dependencies] -huggingface-hub = ">=0.16.4,<2.0" - -[package.extras] -dev = ["tokenizers[testing]"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] -testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", "ty"] - -[[package]] -name = "torch" -version = "2.13.0" -description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d"}, - {file = "torch-2.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0ab4b69f3ee03a62a002cfbf77b1ca5e88aceb4ea64cb4388bb28f638ddbb045"}, - {file = "torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c78b7b4d04461855a764cf01bae9a462bb88bc93defcfa11235cbc8fdf3e12c4"}, - {file = "torch-2.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:2bd30b6b730d987fa386ce3898933762c5cb8cc82eb0535211d787cc3ce2dfeb"}, - {file = "torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8"}, - {file = "torch-2.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:092790c696a760c729fd5722835f50b9d81fd7c8f141571f3f3cf4081a8f664c"}, - {file = "torch-2.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:60fcdcb2f3876e21146cb4524ef06397d727ca9ad5f020818547e25075fe3cb7"}, - {file = "torch-2.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a0d8b11f16a48d60e2015d8213aa0390744cbebb98e58b62b3514dddc656e330"}, - {file = "torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027"}, - {file = "torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4"}, - {file = "torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b"}, - {file = "torch-2.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:024c6cc0c1b085f2f91f20a3dc27b0471d021c31ce84b81be3afdc39f791fd9d"}, - {file = "torch-2.13.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:33449899ce5496c1b84b4853179d94fd102028ae1407314d9fb956bb79e70d09"}, - {file = "torch-2.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1e09d6a722504957c694faceca843acde562786df1144ebcc5a74075ec7f6005"}, - {file = "torch-2.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a3a9a21312872af8a26950b2c15680335a386a1f56ed03e780653d78b9607e9e"}, - {file = "torch-2.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:49b58f1e2c52440abb6f17c28f0335fe6c6d01ad1a7f55b0183b81e4b34d64e6"}, - {file = "torch-2.13.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d849b390e07d8d333ce8ecaf91b273c656c598379a19c9acf1318a883f6b391c"}, - {file = "torch-2.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a3893dc2da0a972a8ca5d698c85a9f967559ac5f8ee1797b77408aa8734d073c"}, - {file = "torch-2.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:49f1ea385c754e54919408a9bb3b5a72b0b755bbe2c916c1d6f70afbec4908a2"}, - {file = "torch-2.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f8573e3ce9ebcd53fe922f01077a6085ccdfbe5f12fd215883a9d87d7a744fd"}, - {file = "torch-2.13.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c28def70706c2f9ecc752574766e8ae4da9b810ab6676b611166761a78a9f1e1"}, - {file = "torch-2.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:31061ff56ed8fbf26c749806905aeb749ebeb819810fd5d52508aa5afd90dddc"}, - {file = "torch-2.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:cc26eead4cf51d0b544e31e364dcf000846549c273bd148936fe9d24d29acb92"}, - {file = "torch-2.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a7de8a313090dc5c7d7ba4bfe5c3be222528f9a4dba1acc83bddb1157360c4b8"}, -] - -[package.dependencies] -cuda-bindings = {version = ">=13.0.3,<14", markers = "platform_system == \"Linux\" and python_version < \"3.15\""} -cuda-toolkit = {version = "13.0.3", extras = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], markers = "platform_system == \"Linux\""} -filelock = "*" -fsspec = ">=0.8.5" -jinja2 = "*" -networkx = ">=2.5.1" -nvidia-cudnn-cu13 = {version = "9.20.0.48", markers = "platform_system == \"Linux\""} -nvidia-cusparselt-cu13 = {version = "0.8.1", markers = "platform_system == \"Linux\""} -nvidia-nccl-cu13 = {version = "2.29.7", markers = "platform_system == \"Linux\""} -nvidia-nvshmem-cu13 = {version = "3.4.5", markers = "platform_system == \"Linux\""} -setuptools = ">=77.0.3" -sympy = ">=1.13.3" -triton = {version = "3.7.1", markers = "platform_system == \"Linux\" and python_version < \"3.15\""} -typing-extensions = ">=4.10.0" - -[package.extras] -opt-einsum = ["opt-einsum (>=3.3)"] -optree = ["optree (>=0.13.0)"] -pyyaml = ["pyyaml"] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "transformers" -version = "4.56.0" -description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" -optional = false -python-versions = ">=3.9.0" -groups = ["main"] -files = [ - {file = "transformers-4.56.0-py3-none-any.whl", hash = "sha256:bacf539c38dd850690856881c4974321af93a22f2ee96bcc994741a2121d8e71"}, - {file = "transformers-4.56.0.tar.gz", hash = "sha256:6ca9c3f38aa4da93ebf877db7156368c1c188c7465f09dbe70951e7622e987fa"}, -] - -[package.dependencies] -filelock = "*" -huggingface-hub = ">=0.34.0,<1.0" -numpy = ">=1.17" -packaging = ">=20.0" -pyyaml = ">=5.1" -regex = "!=2019.12.17" -requests = "*" -safetensors = ">=0.4.3" -tokenizers = ">=0.22.0,<=0.23.0" -tqdm = ">=4.27" - -[package.extras] -accelerate = ["accelerate (>=0.26.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "av", "codecarbon (>=2.8.1)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "jinja2 (>=3.1.0)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "kernels (>=0.6.1,<=0.9)", "librosa", "mistral-common[opencv] (>=1.6.3)", "num2words", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm (!=1.0.18,<=1.0.19)", "tokenizers (>=0.22.0,<=0.23.0)", "torch (>=2.2)", "torchaudio", "torchvision"] -audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] -benchmark = ["optimum-benchmark (>=0.3.0)"] -chat-template = ["jinja2 (>=3.1.0)"] -codecarbon = ["codecarbon (>=2.8.1)"] -deepspeed = ["accelerate (>=0.26.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.26.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "libcst", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "optuna", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio", "pytest-order", "pytest-rerunfailures", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.11.2)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "av", "beautifulsoup4", "codecarbon (>=2.8.1)", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "jinja2 (>=3.1.0)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "kernels (>=0.6.1,<=0.9)", "libcst", "librosa", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "num2words", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "pandas (<2.3.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio", "pytest-order", "pytest-rerunfailures", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.11.2)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm (!=1.0.18,<=1.0.19)", "tokenizers (>=0.22.0,<=0.23.0)", "torch (>=2.2)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "libcst", "librosa", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "pandas (<2.3.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio", "pytest-order", "pytest-rerunfailures", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.11.2)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.22.0,<=0.23.0)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "beautifulsoup4", "codecarbon (>=2.8.1)", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "kenlm", "kernels (>=0.6.1,<=0.9)", "libcst", "librosa", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "num2words", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "pandas (<2.3.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio", "pytest-order", "pytest-rerunfailures", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.11.2)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm (!=1.0.18,<=1.0.19)", "tokenizers (>=0.22.0,<=0.23.0)", "torch (>=2.2)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)", "scipy (<1.13.0)"] -flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] -ftfy = ["ftfy"] -hf-xet = ["hf-xet"] -hub-kernels = ["kernels (>=0.6.1,<=0.9)"] -integrations = ["kernels (>=0.6.1,<=0.9)", "optuna", "ray[tune] (>=2.7.0)", "sigopt"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] -mistral-common = ["mistral-common[opencv] (>=1.6.3)"] -modelcreation = ["cookiecutter (==1.7.3)"] -natten = ["natten (>=0.14.6,<0.15.0)"] -num2words = ["num2words"] -onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] -onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] -open-telemetry = ["opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk"] -optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "pandas (<2.3.0)", "rich", "ruff (==0.11.2)", "urllib3 (<2.0.0)"] -ray = ["ray[tune] (>=2.7.0)"] -retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] -ruff = ["ruff (==0.11.2)"] -sagemaker = ["sagemaker (>=2.31.0)"] -sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["accelerate (>=0.26.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "starlette", "torch (>=2.2)", "uvicorn"] -sigopt = ["sigopt"] -sklearn = ["scikit-learn"] -speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "libcst", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "parameterized (>=0.9)", "psutil", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio", "pytest-order", "pytest-rerunfailures", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.11.2)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] -tf-cpu = ["keras (>2.9,<2.16)", "keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow-cpu (>2.9,<2.16)", "tensorflow-probability (<0.24)", "tensorflow-text (<2.16)", "tf2onnx"] -tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] -tiktoken = ["blobfile", "tiktoken"] -timm = ["timm (!=1.0.18,<=1.0.19)"] -tokenizers = ["tokenizers (>=0.22.0,<=0.23.0)"] -torch = ["accelerate (>=0.26.0)", "torch (>=2.2)"] -torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.34.0,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.22.0,<=0.23.0)", "torch (>=2.2)", "tqdm (>=4.27)"] -video = ["av"] -vision = ["Pillow (>=10.0.1,<=15.0)"] - -[[package]] -name = "triton" -version = "3.7.1" -description = "A language and compiler for custom Deep Learning operations" -optional = false -python-versions = "<3.15,>=3.10" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "triton-3.7.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3daf64305d6cea88d3334c65ebc9bcd0c64c9564a977084366aa768d57cbcf64"}, - {file = "triton-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee89fbf782ec2ad50391dd1cf26cbea4f4467154c37f4773026da8fc31c0f58e"}, - {file = "triton-3.7.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a0e1cd4c4a76370ed74a8432a53cea28716827d19e40ffc732233e35ceb3f6"}, - {file = "triton-3.7.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6744957e9fd610a29680ec2346057d0c86948ed3812468670719f391e94b44a5"}, - {file = "triton-3.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1"}, - {file = "triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728"}, - {file = "triton-3.7.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cdbfc09d9ec58bc5e68321525653220de7515c199e7a8097a97c85e62b52cd0a"}, - {file = "triton-3.7.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58c0e131da05134a2a4788ccbcc0c1105cf0f54c8e98f19e34cd465396dc15eb"}, - {file = "triton-3.7.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe4ea396a06171f1f1f58cbd39c70b09294398f7dd7c620939bab54ad6f934fa"}, - {file = "triton-3.7.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2020153b08280415ec0da6607834e79166442147e78e144df06b508c75b186d2"}, - {file = "triton-3.7.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c58e4c61f0c73b5dba3b5d19b4a7093c32f90dc18b2a7f121a7c16ccd31107b7"}, - {file = "triton-3.7.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10ba85fa2cca4a2fbdeb36bf1cb082f2c252bda55bf9fccd74f65ec5bc647e68"}, -] - -[package.extras] -build = ["cmake (>=3.20,<4.0)", "lit"] -tests = ["autopep8", "isort", "llnl-hatchet", "numpy", "pytest", "pytest-forked", "pytest-xdist", "scipy (>=1.7.1)"] -tutorials = ["matplotlib", "pandas", "tabulate"] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "fca5d825b7c614e1b977e03e79938be6e037a8c868b500fdb2b16d3905a8a287" diff --git a/security_scanning/examples/models/core/mixtral/pyproject.toml b/security_scanning/examples/models/core/mixtral/pyproject.toml deleted file mode 100644 index 44d9a35823f7..000000000000 --- a/security_scanning/examples/models/core/mixtral/pyproject.toml +++ /dev/null @@ -1,17 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "transformers (==4.56.0)", - "accelerate (==0.25.0)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/core/nemotron/poetry.lock b/security_scanning/examples/models/core/nemotron/poetry.lock deleted file mode 100644 index 6b9f6e06bcde..000000000000 --- a/security_scanning/examples/models/core/nemotron/poetry.lock +++ /dev/null @@ -1,2323 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "datasets" -version = "3.1.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, - {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, -] - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.9" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} -huggingface-hub = ">=0.23.0" -multiprocess = "<0.70.17" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=15.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] -docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -quality = ["ruff (>=0.3.0)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2024.9.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, - {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.16" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, - {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, - {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, -] - -[package.dependencies] -dill = ">=0.3.8" - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "dea83d726b2f535994f0c63ae08ff606e50a3c3e8a27db8373645990ef2c938f" diff --git a/security_scanning/examples/models/core/nemotron/pyproject.toml b/security_scanning/examples/models/core/nemotron/pyproject.toml deleted file mode 100644 index b49de73c80f2..000000000000 --- a/security_scanning/examples/models/core/nemotron/pyproject.toml +++ /dev/null @@ -1,18 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "datasets (==3.1.0)", - "evaluate (>=0.4.6,<0.5.0)", - "rouge-score (>=0.1.2,<0.2.0)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/core/qwen2audio/poetry.lock b/security_scanning/examples/models/core/qwen2audio/poetry.lock deleted file mode 100644 index 9eb914a21db7..000000000000 --- a/security_scanning/examples/models/core/qwen2audio/poetry.lock +++ /dev/null @@ -1,2745 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "annotated-doc" -version = "0.0.4" -description = "Document parameters, class attributes, return types, and variables inline, with Annotated." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320"}, - {file = "annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4"}, -] - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "datasets" -version = "3.1.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, - {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, -] - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.9" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} -huggingface-hub = ">=0.23.0" -multiprocess = "<0.70.17" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=15.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] -docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -quality = ["ruff (>=0.3.0)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "einops" -version = "0.8.2" -description = "A new flavour of deep learning operations" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "einops-0.8.2-py3-none-any.whl", hash = "sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193"}, - {file = "einops-0.8.2.tar.gz", hash = "sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827"}, -] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2024.9.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, - {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "markdown-it-py" -version = "4.2.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a"}, - {file = "markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "markdown-it-pyrs", "mistletoe (>=1.0,<2.0)", "mistune (>=3.0,<4.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins (>=0.5.0)"] -profiling = ["gprof2dot"] -rtd = ["ipykernel", "jupyter_sphinx", "mdit-py-plugins (>=0.5.0)", "myst-parser", "pyyaml", "sphinx", "sphinx-book-theme (>=1.0,<2.0)", "sphinx-copybutton", "sphinx-design"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions", "pytest-timeout", "requests"] - -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.16" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, - {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, - {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, -] - -[package.dependencies] -dill = ">=0.3.8" - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "pygments" -version = "2.20.0" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, - {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, -] - -[package.extras] -windows-terminal = ["colorama (>=0.4.6)"] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rich" -version = "15.0.0" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false -python-versions = ">=3.9.0" -groups = ["main"] -files = [ - {file = "rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb"}, - {file = "rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36"}, -] - -[package.dependencies] -markdown-it-py = ">=2.2.0" -pygments = ">=2.13.0,<3.0.0" - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "safetensors" -version = "0.8.0" -description = "" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0"}, - {file = "safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846"}, - {file = "safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d"}, - {file = "safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f"}, - {file = "safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452"}, - {file = "safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d"}, -] - -[package.extras] -all = ["safetensors[convert]", "safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] -convert = ["huggingface-hub (>=1.4)", "safetensors[torch]"] -dev = ["safetensors[all]", "safetensors[pinned-tf]"] -jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "safetensors[numpy]"] -mlx = ["mlx (>=0.0.9)"] -numpy = ["numpy (>=1.24.6)"] -paddlepaddle = ["paddlepaddle (>=2.4.1)", "safetensors[numpy]"] -pinned-tf = ["safetensors[numpy]", "tensorflow (==2.18.0)"] -quality = ["ruff"] -tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] -testing = ["fsspec (>=2024.6.0)", "h5py (>=3.7.0)", "hypothesis (>=6.70.2)", "pytest (>=9.0)", "pytest-benchmark (>=5.2)", "s3fs (>=2024.6.0)", "safetensors[numpy]", "setuptools-rust (>=1.12.0)"] -tf-nightly = ["safetensors[numpy]", "tf-nightly"] -torch = ["safetensors[numpy]", "torch (>=2.4)"] - -[[package]] -name = "sentencepiece" -version = "0.2.2" -description = "Unsupervised text tokenizer and detokenizer." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_arm64.whl", hash = "sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d"}, - {file = "sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6"}, -] - -[package.extras] -numpy = ["numpy"] -protobuf = ["protobuf"] -test = ["numpy", "protobuf", "pytest"] - -[[package]] -name = "shellingham" -version = "1.5.4" -description = "Tool to Detect Surrounding Shell" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, - {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, -] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "tiktoken" -version = "0.13.0" -description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tiktoken-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:47b1df8d73390a24f94980c75158cdd5c56d256f16d55f30cb49c230caba9ba4"}, - {file = "tiktoken-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7d40c6c5aab171dcd6eb8455bc567bde404bb9def60cdb8c1299cc782b242bb9"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:9b842981fa91accdffd48ff6408a977b7a91c3fbda55d353c3c68114d5c9d69e"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:ed5a30027cb4d8c7ca8b273d4766f3db3cf58fad9e9f3b1a68a351ffb54873d5"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7ab10f4a21c2999846940113f6dbd72e0fa06a24119feddd74cc47e85818e06d"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a2937ad042d49d50eac6e1ba07c5661d4bd3942a5b1e0c0d08475c4df83676e1"}, - {file = "tiktoken-0.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:44733b99bfd72b590cd0936b1c01b3b4dd73122db2d544bc1ceeb18a7678c910"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7bfe1849caa65d1e1d9871817170ec497bbb7984e182012e1bdce72f66608cdb"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:91c180fe255bd5a86d8316210d2833a1d4d33d026cd86a67812f4773743c8d26"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:059c8ecf554eb5b41e6e054ba467b871b03277d267dee7244380aca4359747d4"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:36217497eaffc158607a3b26f065300db2aefd43b115263f3b9688ce38146173"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:303f7d91b4fce3baddbcde05c139091d4caa5026ac7214c1dc7ff7a71ee429ff"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5d48843bee149630eb735a99e1f4a85b47308d21868ea63163f6e87768d3cfed"}, - {file = "tiktoken-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:fc1c44cd37b43fc46bae593129164f4f281e82ea116b57a85aa81bda57eafc94"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91"}, - {file = "tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615"}, - {file = "tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f"}, - {file = "tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51"}, - {file = "tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424"}, - {file = "tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:35e1ea1e0631c04f551297284a1ab7e1f65a3c55a9a48728d5e0f66b4527c04a"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a3b536c55802fe42f4b4644d2be4f04bf788506b48de0a0a658cb58f8bce232"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b8ac2d6420ff05841a89ba5205c6d45f56c4f6843454f3c884b7eb1a2a8dddb2"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:477c9a38e20d0ed248090509acf1e839ad3967a4f00b4b0f958210049f656dee"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:da86f8c96ac1c235d7a3b3eebff1eacfdbcfb8ad792706943268d4d2938fbafe"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9b8858b29804b3a0add25ce9e62fb00f89f621dc754d75d03ca419d17e8ddf67"}, - {file = "tiktoken-0.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:b967dfb9d0adf9a631953b1b40717684f04478270fc51bbccdd2f838d67a2f00"}, - {file = "tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1"}, -] - -[package.dependencies] -regex = "*" -requests = "*" - -[package.extras] -blobfile = ["blobfile (>=3)"] - -[[package]] -name = "tokenizers" -version = "0.22.2" -description = "" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c"}, - {file = "tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5"}, - {file = "tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92"}, - {file = "tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48"}, - {file = "tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113"}, - {file = "tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917"}, -] - -[package.dependencies] -huggingface-hub = ">=0.16.4,<2.0" - -[package.extras] -dev = ["tokenizers[testing]"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] -testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", "ty"] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "transformers" -version = "5.14.0" -description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "transformers-5.14.0-py3-none-any.whl", hash = "sha256:288287da4fdcdae0b611be32b5be20d4bcbf5caaa5f269292927fb1ff888d19d"}, - {file = "transformers-5.14.0.tar.gz", hash = "sha256:9e1a2b345c195d216afd10082394b915a82e339fa09e9013c32094ff835ca9db"}, -] - -[package.dependencies] -huggingface-hub = ">=1.5.0,<2.0" -numpy = ">=1.17" -packaging = ">=20.0" -pyyaml = ">=5.1" -regex = ">=2025.10.22" -safetensors = ">=0.8.0" -tokenizers = ">=0.22.0,<=0.23.0" -tqdm = ">=4.60" -typer = "*" - -[package.extras] -accelerate = ["accelerate (>=1.1.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.11.5)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] -audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -benchmark = ["optimum-benchmark (>=0.3.0)"] -chat-template = ["jinja2 (>=3.1.0)"] -codecarbon = ["codecarbon (>=2.8.1)"] -deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.11.5)", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] -docs = ["hf-doc-builder"] -integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.15.2,<0.16)", "optuna", "ray[tune] (>=2.7.0)"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] -kernels = ["kernels (>=0.15.2,<0.16)"] -mistral-common = ["mistral-common[image] (>=1.11.5)"] -num2words = ["num2words"] -optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] -ray = ["ray[tune] (>=2.7.0)"] -retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] -sagemaker = ["sagemaker (>=2.31.0)"] -sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] -sklearn = ["scikit-learn"] -testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -tiktoken = ["blobfile", "tiktoken"] -timm = ["timm (>=1.0.23)"] -torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] -video = ["av"] -vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] - -[[package]] -name = "transformers-stream-generator" -version = "0.0.5" -description = "This is a text generation method which returns a generator, streaming out each token in real-time during inference, based on Huggingface/Transformers." -optional = false -python-versions = ">=3.5" -groups = ["main"] -files = [ - {file = "transformers-stream-generator-0.0.5.tar.gz", hash = "sha256:271deace0abf9c0f83b36db472c8ba61fdc7b04d1bf89d845644acac2795ed57"}, -] - -[package.dependencies] -transformers = ">=4.26.1" - -[[package]] -name = "typer" -version = "0.27.0" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, -] - -[package.dependencies] -annotated-doc = ">=0.0.2" -colorama = {version = "*", markers = "platform_system == \"Windows\""} -rich = ">=13.8.0" -shellingham = ">=1.3.0" - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "843b185416adc1328889eb6f9f5d0ce3307b2ebb0b593cf62b70d909e2d63e22" diff --git a/security_scanning/examples/models/core/qwen2audio/pyproject.toml b/security_scanning/examples/models/core/qwen2audio/pyproject.toml deleted file mode 100644 index 3bdb6830ca6c..000000000000 --- a/security_scanning/examples/models/core/qwen2audio/pyproject.toml +++ /dev/null @@ -1,23 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "datasets (==3.1.0)", - "evaluate (>=0.4.6,<0.5.0)", - "rouge-score (>=0.1.2,<0.2.0)", - "transformers (>=4.45.0)", - "transformers-stream-generator (>=0.0.5,<0.0.6)", - "sentencepiece (>=0.1.99)", - "tiktoken (>=0.13.0,<0.14.0)", - "einops (>=0.8.2,<0.9.0)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/models/core/qwenvl/poetry.lock b/security_scanning/examples/models/core/qwenvl/poetry.lock deleted file mode 100644 index baca11806a6c..000000000000 --- a/security_scanning/examples/models/core/qwenvl/poetry.lock +++ /dev/null @@ -1,3567 +0,0 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -description = "Happy Eyeballs for asyncio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, - {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -description = "Async http client/server framework (asyncio)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, -] - -[package.dependencies] -aiohappyeyeballs = ">=2.5.0" -aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} -attrs = ">=17.3.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} -yarl = ">=1.17.0,<2.0" - -[package.extras] -speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\" and sys_platform != \"android\" and sys_platform != \"ios\"", "aiodns (>=3.3.0) ; sys_platform != \"android\" and sys_platform != \"ios\"", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\" and sys_platform != \"android\" and sys_platform != \"ios\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] - -[[package]] -name = "aiosignal" -version = "1.4.0" -description = "aiosignal: a list of registered asynchronous callbacks" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, - {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, -] - -[package.dependencies] -frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} - -[[package]] -name = "annotated-doc" -version = "0.0.4" -description = "Document parameters, class attributes, return types, and variables inline, with Annotated." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320"}, - {file = "annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4"}, -] - -[[package]] -name = "anyio" -version = "4.14.2" -description = "High-level concurrency and networking framework on top of asyncio or Trio" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, - {file = "anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} - -[package.extras] -trio = ["trio (>=0.32.0)"] - -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - -[[package]] -name = "attrs" -version = "26.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, - {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee"}, - {file = "charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1"}, - {file = "charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0"}, - {file = "charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115"}, - {file = "charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b"}, - {file = "charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177"}, - {file = "charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win32.whl", hash = "sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_amd64.whl", hash = "sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b"}, - {file = "charset_normalizer-3.4.9-cp39-cp39-win_arm64.whl", hash = "sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe"}, - {file = "charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5"}, - {file = "charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b"}, -] - -[[package]] -name = "click" -version = "8.4.2" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, - {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] -markers = "platform_system == \"Windows\"" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "cuda-bindings" -version = "13.3.1" -description = "Python bindings for CUDA" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86"}, - {file = "cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0"}, - {file = "cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051"}, - {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474"}, - {file = "cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708"}, - {file = "cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1"}, - {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49"}, - {file = "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a"}, - {file = "cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff"}, - {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf"}, - {file = "cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7"}, - {file = "cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202"}, - {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8"}, - {file = "cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80"}, - {file = "cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb"}, - {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76"}, - {file = "cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9"}, - {file = "cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d"}, -] - -[package.dependencies] -cuda-pathfinder = ">=1.4.2" - -[package.extras] -all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)", "nvidia-cudla (==13.*) ; platform_system == \"Linux\" and platform_machine == \"aarch64\""] - -[[package]] -name = "cuda-pathfinder" -version = "1.5.6" -description = "Pathfinder for CUDA components" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, -] - -[[package]] -name = "cuda-toolkit" -version = "13.0.3" -description = "CUDA Toolkit meta-package" -optional = false -python-versions = "*" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f"}, -] - -[package.dependencies] -nvidia-cublas = {version = "==13.1.1.3.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cublas\" or extra == \"cusolver\")"} -nvidia-cuda-cupti = {version = "==13.0.85.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cupti\""} -nvidia-cuda-nvrtc = {version = "==13.0.88.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cublas\" or extra == \"nvrtc\")"} -nvidia-cuda-runtime = {version = "==13.0.96.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cudart\""} -nvidia-cufft = {version = "==12.0.0.61.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cufft\""} -nvidia-cufile = {version = "==1.15.1.6.*", optional = true, markers = "sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cufile\""} -nvidia-curand = {version = "==10.4.0.35.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"curand\""} -nvidia-cusolver = {version = "==12.0.4.66.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"cusolver\""} -nvidia-cusparse = {version = "==12.6.3.3.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cusolver\" or extra == \"cusparse\")"} -nvidia-nvjitlink = {version = ">=13.0.88,<14", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (extra == \"cufft\" or extra == \"cusolver\" or extra == \"cusparse\" or extra == \"nvjitlink\")"} -nvidia-nvtx = {version = "==13.0.85.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and extra == \"nvtx\""} - -[package.extras] -all = ["nvidia-cublas (==13.1.1.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-cccl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-culibos (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")", "nvidia-cuda-cupti (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-cuxxfilt (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvcc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-opencl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-profiler-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cufft (==12.0.0.61.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cufile (==1.15.1.6.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")", "nvidia-curand (==10.4.0.35.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusolver (==12.0.4.66.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-npp (==13.0.1.2.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvfatbin (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjpeg (==13.0.1.86.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvml-dev (==13.0.87.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvptxcompiler (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvtx (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cccl = ["nvidia-cuda-cccl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -crt = ["nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cublas = ["nvidia-cublas (==13.1.1.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cudart = ["nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cufft = ["nvidia-cufft (==12.0.0.61.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cufile = ["nvidia-cufile (==1.15.1.6.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")"] -culibos = ["nvidia-cuda-culibos (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")"] -cupti = ["nvidia-cuda-cupti (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -curand = ["nvidia-curand (==10.4.0.35.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cusolver = ["nvidia-cublas (==13.1.1.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusolver (==12.0.4.66.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cusparse = ["nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -cuxxfilt = ["nvidia-cuda-cuxxfilt (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -npp = ["nvidia-npp (==13.0.1.2.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvcc = ["nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-nvcc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\"", "nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvfatbin = ["nvidia-nvfatbin (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvjitlink = ["nvidia-nvjitlink (>=13.0.88,<14) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvjpeg = ["nvidia-nvjpeg (==13.0.1.86.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvml = ["nvidia-nvml-dev (==13.0.87.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvptxcompiler = ["nvidia-nvptxcompiler (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvrtc = ["nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvtx = ["nvidia-nvtx (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -nvvm = ["nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -opencl = ["nvidia-cuda-opencl (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -profiler = ["nvidia-cuda-profiler-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] -sanitizer = ["nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\") or (sys_platform == \"linux\" or sys_platform == \"win32\") and platform_machine == \"x86_64\""] - -[[package]] -name = "datasets" -version = "3.1.0" -description = "HuggingFace community-driven open-source library of datasets" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, - {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, -] - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.9" -filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} -huggingface-hub = ">=0.23.0" -multiprocess = "<0.70.17" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=15.0.0" -pyyaml = ">=5.1" -requests = ">=2.32.2" -tqdm = ">=4.66.3" -xxhash = "*" - -[package.extras] -audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\""] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] -docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] -jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -quality = ["ruff (>=0.3.0)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.6.0)"] -tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tensorflow (>=2.16.0) ; python_version >= \"3.10\"", "tensorflow (>=2.6.0) ; python_version < \"3.10\"", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (<8.0.0)", "jax (>=0.3.14) ; sys_platform != \"win32\"", "jaxlib (>=0.3.14) ; sys_platform != \"win32\"", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0) ; python_version >= \"3.9\"", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=9.4.0)"] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - -[[package]] -name = "einops" -version = "0.8.2" -description = "A new flavour of deep learning operations" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "einops-0.8.2-py3-none-any.whl", hash = "sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193"}, - {file = "einops-0.8.2.tar.gz", hash = "sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827"}, -] - -[[package]] -name = "evaluate" -version = "0.4.6" -description = "HuggingFace community-driven open-source library of evaluation" -optional = false -python-versions = ">=3.8.0" -groups = ["main"] -files = [ - {file = "evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29"}, - {file = "evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21"}, -] - -[package.dependencies] -datasets = ">=2.0.0" -dill = "*" -fsspec = {version = ">=2021.5.0", extras = ["http"]} -huggingface-hub = ">=0.7.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -docs = ["s3fs"] -evaluator = ["scipy (>=1.7.1)", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] -template = ["cookiecutter", "gradio (>=3.0.0)"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] -tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "numpy (<2.0.0)", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy (>=1.10.0)", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] -torch = ["torch"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.29.7" -description = "A platform independent file lock." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, - {file = "frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450"}, - {file = "frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f"}, - {file = "frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7"}, - {file = "frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6"}, - {file = "frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9"}, - {file = "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581"}, - {file = "frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd"}, - {file = "frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967"}, - {file = "frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b"}, - {file = "frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, - {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, - {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, - {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, - {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, - {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7"}, - {file = "frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5"}, - {file = "frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8"}, - {file = "frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed"}, - {file = "frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231"}, - {file = "frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c"}, - {file = "frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714"}, - {file = "frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0"}, - {file = "frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888"}, - {file = "frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f"}, - {file = "frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e"}, - {file = "frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30"}, - {file = "frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7"}, - {file = "frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0"}, - {file = "frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed"}, - {file = "frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a"}, - {file = "frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd"}, - {file = "frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca"}, - {file = "frozenlist-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61"}, - {file = "frozenlist-1.8.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178"}, - {file = "frozenlist-1.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda"}, - {file = "frozenlist-1.8.0-cp39-cp39-win32.whl", hash = "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a"}, - {file = "frozenlist-1.8.0-cp39-cp39-win_arm64.whl", hash = "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103"}, - {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, - {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, -] - -[[package]] -name = "fsspec" -version = "2024.9.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, - {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, -] - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dev = ["pre-commit", "ruff"] -doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] -test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] -tqdm = ["tqdm"] - -[[package]] -name = "h11" -version = "0.16.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, - {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, -] - -[[package]] -name = "hf-xet" -version = "1.5.1" -description = "Fast transfer of large files with the Hugging Face Hub." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\")" -files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "httpcore" -version = "1.0.9" -description = "A minimal low-level HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, -] - -[package.dependencies] -certifi = "*" -h11 = ">=0.16" - -[package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] - -[[package]] -name = "httpx" -version = "0.28.1" -description = "The next generation HTTP client." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, -] - -[package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" - -[package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] -http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "huggingface-hub" -version = "1.23.0" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, -] - -[package.dependencies] -click = ">=8.4.2,<9.0.0" -filelock = ">=3.10.0" -fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.5.1,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} -httpx = ">=0.23.0,<1" -packaging = ">=20.9" -pyyaml = ">=5.1" -tqdm = ">=4.42.1" -typing-extensions = ">=4.1.0" - -[package.extras] -all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -gradio = ["gradio (>=5.0.0)", "requests"] -hf-xet = ["hf-xet (>=1.5.1,<2.0.0)"] -mcp = ["mcp (>=1.8.0)"] -oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] -testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] - -[[package]] -name = "idna" -version = "3.18" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, - {file = "idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848"}, -] - -[package.extras] -all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "jinja2" -version = "3.1.6" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, - {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "joblib" -version = "1.5.3" -description = "Lightweight pipelining with Python functions" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, - {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, -] - -[[package]] -name = "markdown-it-py" -version = "4.2.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a"}, - {file = "markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "markdown-it-pyrs", "mistletoe (>=1.0,<2.0)", "mistune (>=3.0,<4.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins (>=0.5.0)"] -profiling = ["gprof2dot"] -rtd = ["ipykernel", "jupyter_sphinx", "mdit-py-plugins (>=0.5.0)", "myst-parser", "pyyaml", "sphinx", "sphinx-book-theme (>=1.0,<2.0)", "sphinx-copybutton", "sphinx-design"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions", "pytest-timeout", "requests"] - -[[package]] -name = "markupsafe" -version = "3.0.3" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, - {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1"}, - {file = "markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa"}, - {file = "markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8"}, - {file = "markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1"}, - {file = "markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad"}, - {file = "markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a"}, - {file = "markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19"}, - {file = "markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01"}, - {file = "markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c"}, - {file = "markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e"}, - {file = "markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b"}, - {file = "markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d"}, - {file = "markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c"}, - {file = "markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f"}, - {file = "markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795"}, - {file = "markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12"}, - {file = "markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed"}, - {file = "markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5"}, - {file = "markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485"}, - {file = "markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73"}, - {file = "markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287"}, - {file = "markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe"}, - {file = "markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe"}, - {file = "markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9"}, - {file = "markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581"}, - {file = "markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4"}, - {file = "markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab"}, - {file = "markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa"}, - {file = "markupsafe-3.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26"}, - {file = "markupsafe-3.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d"}, - {file = "markupsafe-3.0.3-cp39-cp39-win32.whl", hash = "sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7"}, - {file = "markupsafe-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e"}, - {file = "markupsafe-3.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8"}, - {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, -] - -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -description = "Python library for arbitrary-precision floating-point arithmetic" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, - {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, -] - -[package.extras] -develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] -docs = ["sphinx"] -gmpy = ["gmpy2 (>=2.1.0a4) ; platform_python_implementation != \"PyPy\""] -tests = ["pytest (>=4.6)"] - -[[package]] -name = "multidict" -version = "6.7.1" -description = "multidict implementation" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, - {file = "multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190"}, - {file = "multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962"}, - {file = "multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505"}, - {file = "multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122"}, - {file = "multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df"}, - {file = "multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e"}, - {file = "multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0"}, - {file = "multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0"}, - {file = "multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa"}, - {file = "multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a"}, - {file = "multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b"}, - {file = "multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd"}, - {file = "multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a"}, - {file = "multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a"}, - {file = "multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba"}, - {file = "multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511"}, - {file = "multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19"}, - {file = "multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2"}, - {file = "multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed"}, - {file = "multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d"}, - {file = "multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33"}, - {file = "multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3"}, - {file = "multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5"}, - {file = "multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963"}, - {file = "multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd"}, - {file = "multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52"}, - {file = "multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108"}, - {file = "multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32"}, - {file = "multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8"}, - {file = "multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2"}, - {file = "multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37"}, - {file = "multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1"}, - {file = "multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b"}, - {file = "multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d"}, - {file = "multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f"}, - {file = "multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a"}, - {file = "multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d"}, - {file = "multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9"}, - {file = "multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2"}, - {file = "multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7"}, - {file = "multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5"}, - {file = "multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358"}, - {file = "multidict-6.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f"}, - {file = "multidict-6.7.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de"}, - {file = "multidict-6.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5"}, - {file = "multidict-6.7.1-cp39-cp39-win32.whl", hash = "sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0"}, - {file = "multidict-6.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4"}, - {file = "multidict-6.7.1-cp39-cp39-win_arm64.whl", hash = "sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9"}, - {file = "multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56"}, - {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "multiprocess" -version = "0.70.16" -description = "better multiprocessing and multithreading in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"}, - {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"}, - {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"}, - {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"}, - {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, - {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, -] - -[package.dependencies] -dill = ">=0.3.8" - -[[package]] -name = "networkx" -version = "3.4.2" -description = "Python package for creating and manipulating graphs and networks" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, - {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, -] - -[package.extras] -default = ["matplotlib (>=3.7)", "numpy (>=1.24)", "pandas (>=2.0)", "scipy (>=1.10,!=1.11.0,!=1.11.1)"] -developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] -doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.15)", "sphinx (>=7.3)", "sphinx-gallery (>=0.16)", "texext (>=0.6.7)"] -example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy (>=0.7.2)", "osmnx (>=1.9)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] -extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] -test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] - -[[package]] -name = "networkx" -version = "3.6.1" -description = "Python package for creating and manipulating graphs and networks" -optional = false -python-versions = "!=3.14.1,>=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762"}, - {file = "networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509"}, -] - -[package.extras] -benchmarking = ["asv", "virtualenv"] -default = ["matplotlib (>=3.8)", "numpy (>=1.25)", "pandas (>=2.0)", "scipy (>=1.11.2)"] -developer = ["mypy (>=1.15)", "pre-commit (>=4.1)"] -doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=10)", "pydata-sphinx-theme (>=0.16)", "sphinx (>=8.0)", "sphinx-gallery (>=0.18)", "texext (>=0.6.7)"] -example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "iplotx (>=0.9.0)", "momepy (>=0.7.2)", "osmnx (>=2.0.0)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] -extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] -release = ["build (>=0.10)", "changelist (==0.5)", "twine (>=4.0)", "wheel (>=0.40)"] -test = ["pytest (>=7.2)", "pytest-cov (>=4.0)", "pytest-xdist (>=3.0)"] -test-extras = ["pytest-mpl", "pytest-randomly"] - -[[package]] -name = "nltk" -version = "3.10.0" -description = "Natural Language Toolkit" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nltk-3.10.0-py3-none-any.whl", hash = "sha256:54ff84d4916d3ef127e8953bee0023f6a6b320b75d634a19e06ef056d3d244bf"}, - {file = "nltk-3.10.0.tar.gz", hash = "sha256:4fbac1d98203cbcd1b5d94a2877fb822300072d80604a5e7fae49d2c5f84e8c1"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "numpy" -version = "2.2.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163"}, - {file = "numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83"}, - {file = "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680"}, - {file = "numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289"}, - {file = "numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d"}, - {file = "numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42"}, - {file = "numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a"}, - {file = "numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1"}, - {file = "numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab"}, - {file = "numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47"}, - {file = "numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3"}, - {file = "numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87"}, - {file = "numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49"}, - {file = "numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de"}, - {file = "numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4"}, - {file = "numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d"}, - {file = "numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f"}, - {file = "numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868"}, - {file = "numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d"}, - {file = "numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd"}, - {file = "numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40"}, - {file = "numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f"}, - {file = "numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571"}, - {file = "numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1"}, - {file = "numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff"}, - {file = "numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543"}, - {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, - {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, -] - -[[package]] -name = "numpy" -version = "2.4.6" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version == \"3.11\"" -files = [ - {file = "numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8"}, - {file = "numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47"}, - {file = "numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8"}, - {file = "numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6"}, - {file = "numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8"}, - {file = "numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, - {file = "numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41"}, - {file = "numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f"}, - {file = "numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a"}, - {file = "numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2"}, - {file = "numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45"}, - {file = "numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751"}, - {file = "numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f"}, - {file = "numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b"}, - {file = "numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a"}, - {file = "numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605"}, - {file = "numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91"}, - {file = "numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359"}, - {file = "numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe"}, - {file = "numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20"}, - {file = "numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67"}, - {file = "numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd"}, - {file = "numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab"}, - {file = "numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75"}, - {file = "numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5"}, - {file = "numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b"}, - {file = "numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402"}, - {file = "numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb"}, - {file = "numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1"}, - {file = "numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261"}, - {file = "numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e"}, - {file = "numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43"}, - {file = "numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895"}, - {file = "numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4"}, - {file = "numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063"}, - {file = "numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627"}, - {file = "numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02"}, - {file = "numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73"}, - {file = "numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"}, -] - -[[package]] -name = "numpy" -version = "2.5.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.12" -groups = ["main"] -markers = "python_version == \"3.12\"" -files = [ - {file = "numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0"}, - {file = "numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75"}, - {file = "numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3"}, - {file = "numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9"}, - {file = "numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2"}, - {file = "numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2"}, - {file = "numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d"}, - {file = "numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd"}, - {file = "numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7"}, - {file = "numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6"}, - {file = "numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9"}, - {file = "numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74"}, - {file = "numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2"}, - {file = "numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95"}, - {file = "numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373"}, - {file = "numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438"}, - {file = "numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace"}, - {file = "numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a"}, - {file = "numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7"}, - {file = "numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09"}, - {file = "numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1"}, - {file = "numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077"}, - {file = "numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf"}, - {file = "numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af"}, - {file = "numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb"}, - {file = "numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3"}, -] - -[[package]] -name = "nvidia-cublas" -version = "13.1.1.3" -description = "CUBLAS native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5"}, - {file = "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436"}, - {file = "nvidia_cublas-13.1.1.3-py3-none-win_amd64.whl", hash = "sha256:b6cdce694e47ff6aadf0a69df1cab6628d696f5ff56e8d16af50309d855fa20f"}, -] - -[package.dependencies] -nvidia-cuda-nvrtc = "*" - -[[package]] -name = "nvidia-cublas" -version = "13.6.0.2" -description = "CUBLAS native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform != \"linux\" or platform_machine != \"aarch64\" and platform_machine != \"x86_64\") and platform_system == \"Linux\" and (sys_platform != \"linux\" and sys_platform != \"win32\" or platform_machine != \"x86_64\")" -files = [ - {file = "nvidia_cublas-13.6.0.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:946f6a252b1cc72d8de912c75975fd6d8ba44f67d4e5044fe764ddb909f4a688"}, - {file = "nvidia_cublas-13.6.0.2-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b82c80c886cea6da6e149a5c3bdba274f12b7e4ec4b00a050b916b0446fb4153"}, - {file = "nvidia_cublas-13.6.0.2-py3-none-win_amd64.whl", hash = "sha256:3b5bcd6bfb6f65010ebf195851bcb9b2aa34b9fe08479432002991c1fe84b67d"}, -] - -[package.dependencies] -nvidia-cuda-nvrtc = "*" - -[[package]] -name = "nvidia-cuda-cupti" -version = "13.0.85" -description = "CUDA profiling tools runtime libs." -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151"}, - {file = "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8"}, - {file = "nvidia_cuda_cupti-13.0.85-py3-none-win_amd64.whl", hash = "sha256:683f58d301548deeefcb8f6fac1b8d907691b9d8b18eccab417f51e362102f00"}, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.0.88" -description = "NVRTC native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575"}, - {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b"}, - {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-win_amd64.whl", hash = "sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872"}, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.3.33" -description = "NVRTC native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform != \"linux\" or platform_machine != \"aarch64\" and platform_machine != \"x86_64\") and platform_system == \"Linux\" and (sys_platform != \"linux\" and sys_platform != \"win32\" or platform_machine != \"x86_64\")" -files = [ - {file = "nvidia_cuda_nvrtc-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:82530788b8c6164a54d3fd9ae8bcca8893d397c4aeb998861982a03bbe41e204"}, - {file = "nvidia_cuda_nvrtc-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7b05ecda494c6dabc44231a608b060a71008a730d9dfda932cc508e6d29159e0"}, - {file = "nvidia_cuda_nvrtc-13.3.33-py3-none-win_amd64.whl", hash = "sha256:7d2af818851c0c224d5f92221e9226e51ee23c236df4b51f9194563979c888be"}, -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.0.96" -description = "CUDA Runtime native Libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55"}, - {file = "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548"}, - {file = "nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492"}, -] - -[[package]] -name = "nvidia-cudnn-cu13" -version = "9.20.0.48" -description = "cuDNN runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1"}, - {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304"}, - {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-win_amd64.whl", hash = "sha256:af8139732b99c0118be65ea5aac97f0d46018f8c552889e49d2fb0c6261a4a24"}, -] - -[package.dependencies] -nvidia-cublas = "*" - -[[package]] -name = "nvidia-cufft" -version = "12.0.0.61" -description = "CUFFT native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5"}, - {file = "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3"}, - {file = "nvidia_cufft-12.0.0.61-py3-none-win_amd64.whl", hash = "sha256:2abce5b39d2f5ae12730fb7e5db6696533e36c26e2d3e8fd1750bdd2853364eb"}, -] - -[package.dependencies] -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cufile" -version = "1.15.1.6" -description = "cuFile GPUDirect libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "sys_platform == \"linux\" and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44"}, - {file = "nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1"}, -] - -[[package]] -name = "nvidia-curand" -version = "10.4.0.35" -description = "CURAND native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a"}, - {file = "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc"}, - {file = "nvidia_curand-10.4.0.35-py3-none-win_amd64.whl", hash = "sha256:65b1710aa6961d326b411e314b374290904c5ddf41dc3f766ebc3f1d7d4ca69f"}, -] - -[[package]] -name = "nvidia-cusolver" -version = "12.0.4.66" -description = "CUDA solver native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2"}, - {file = "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112"}, - {file = "nvidia_cusolver-12.0.4.66-py3-none-win_amd64.whl", hash = "sha256:16515bd33a8e76bb54d024cfa068fa68d30e80fc34b9e1090813ea9362e0cb65"}, -] - -[package.dependencies] -nvidia-cublas = "*" -nvidia-cusparse = "*" -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cusparse" -version = "12.6.3.3" -description = "CUSPARSE native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c"}, - {file = "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b"}, - {file = "nvidia_cusparse-12.6.3.3-py3-none-win_amd64.whl", hash = "sha256:cbcf42feb737bd7ec15b4c0a63e62351886bd3f975027b8815d7f720a2b5ea79"}, -] - -[package.dependencies] -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cusparse" -version = "12.8.2.51" -description = "CUSPARSE native runtime libraries" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "" -files = [ - {file = "nvidia_cusparse-12.8.2.51-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:00469fcf62c4d464a1225abd9b20864ecff35e3fbc9fb992572e83d358927755"}, - {file = "nvidia_cusparse-12.8.2.51-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65cbcc4e37a34fca4ee7df2fd57da103593842cda1bbb4a144664ecfe59873a5"}, - {file = "nvidia_cusparse-12.8.2.51-py3-none-win_amd64.whl", hash = "sha256:2ee59291cd362038f3d40d57c7cd09b26d689f3873ae5c94b31c3270772d41b8"}, -] - -[package.dependencies] -nvidia-nvjitlink = "*" - -[[package]] -name = "nvidia-cusparselt-cu13" -version = "0.8.1" -description = "NVIDIA cuSPARSELt" -optional = false -python-versions = "*" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f"}, - {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0"}, - {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-win_amd64.whl", hash = "sha256:dccbd362f91a7b9024d1f55ee9f548ac065027ff15d8c8b0db889ab3a8f31215"}, -] - -[[package]] -name = "nvidia-nccl-cu13" -version = "2.29.7" -description = "NVIDIA Collective Communication Library (NCCL) Runtime" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5"}, - {file = "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d"}, -] - -[[package]] -name = "nvidia-nvjitlink" -version = "13.3.33" -description = "Nvidia JIT LTO Library" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5"}, - {file = "nvidia_nvjitlink-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ce48b37dfeb3cb1eae4cf85adacb47d7a6539ea2272870c9a3628ce275c2037e"}, - {file = "nvidia_nvjitlink-13.3.33-py3-none-win_amd64.whl", hash = "sha256:4297ee49639b4f2e07255a1d69b3acc7ab2d011bb892b403e91ac98368962e3b"}, -] - -[[package]] -name = "nvidia-nvshmem-cu13" -version = "3.4.5" -description = "NVSHMEM creates a global address space that provides efficient and scalable communication for NVIDIA GPU clusters." -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9"}, - {file = "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80"}, -] - -[[package]] -name = "nvidia-nvtx" -version = "13.0.85" -description = "NVIDIA Tools Extension" -optional = false -python-versions = ">=3" -groups = ["main"] -markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and (sys_platform == \"linux\" or platform_machine == \"x86_64\") and platform_system == \"Linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\")" -files = [ - {file = "nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4"}, - {file = "nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6"}, - {file = "nvidia_nvtx-13.0.85-py3-none-win_amd64.whl", hash = "sha256:d66ea44254dd3c6eacc300047af6e1288d2269dd072b417e0adffbf479e18519"}, -] - -[[package]] -name = "packaging" -version = "26.2" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, - {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, -] - -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pandas" -version = "3.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} -python-dateutil = ">=2.8.2" -tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] -aws = ["s3fs (>=2024.10.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] -compression = ["zstandard (>=0.23.0)"] -computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] -feather = ["pyarrow (>=13.0.0)"] -fss = ["fsspec (>=2024.10.0)"] -gcp = ["gcsfs (>=2024.10.0)"] -hdf5 = ["tables (>=3.10.1)"] -html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] -iceberg = ["pyiceberg (>=0.8.1)"] -mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] -output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=13.0.0)"] -performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] -plot = ["matplotlib (>=3.9.3)"] -postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] -pyarrow = ["pyarrow (>=13.0.0)"] -spss = ["pyreadstat (>=1.2.8)"] -sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] -timezone = ["pytz (>=2020.1)"] -xml = ["lxml (>=5.3.0)"] - -[[package]] -name = "pillow" -version = "12.3.0" -description = "Python Imaging Library (fork)" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pillow-12.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a"}, - {file = "pillow-12.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7"}, - {file = "pillow-12.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f"}, - {file = "pillow-12.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec"}, - {file = "pillow-12.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468"}, - {file = "pillow-12.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed"}, - {file = "pillow-12.3.0-cp310-cp310-win32.whl", hash = "sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1"}, - {file = "pillow-12.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb"}, - {file = "pillow-12.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f"}, - {file = "pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756"}, - {file = "pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6"}, - {file = "pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd"}, - {file = "pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd"}, - {file = "pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c"}, - {file = "pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5"}, - {file = "pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b"}, - {file = "pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a"}, - {file = "pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26"}, - {file = "pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965"}, - {file = "pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7"}, - {file = "pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9"}, - {file = "pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91"}, - {file = "pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c"}, - {file = "pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df"}, - {file = "pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f"}, - {file = "pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09"}, - {file = "pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510"}, - {file = "pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89"}, - {file = "pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace"}, - {file = "pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec"}, - {file = "pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66"}, - {file = "pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35"}, - {file = "pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65"}, - {file = "pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3"}, - {file = "pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a"}, - {file = "pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e"}, - {file = "pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f"}, - {file = "pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8"}, - {file = "pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b"}, - {file = "pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330"}, - {file = "pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217"}, - {file = "pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930"}, - {file = "pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8"}, - {file = "pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0"}, - {file = "pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321"}, - {file = "pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b"}, - {file = "pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198"}, - {file = "pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130"}, - {file = "pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a"}, - {file = "pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d"}, - {file = "pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838"}, - {file = "pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e"}, - {file = "pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17"}, - {file = "pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385"}, - {file = "pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c"}, - {file = "pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d"}, - {file = "pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931"}, - {file = "pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7"}, - {file = "pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c"}, - {file = "pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45"}, - {file = "pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139"}, - {file = "pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402"}, - {file = "pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c"}, - {file = "pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f"}, - {file = "pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701"}, - {file = "pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace"}, - {file = "pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4"}, - {file = "pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39"}, - {file = "pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71"}, - {file = "pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827"}, - {file = "pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5"}, - {file = "pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658"}, - {file = "pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf"}, - {file = "pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64"}, - {file = "pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e"}, - {file = "pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777"}, - {file = "pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1"}, - {file = "pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9"}, - {file = "pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8"}, - {file = "pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418"}, - {file = "pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59"}, - {file = "pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468"}, - {file = "pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94"}, - {file = "pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e"}, - {file = "pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3"}, - {file = "pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a"}, - {file = "pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce"}, -] - -[package.extras] -docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-autobuild", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] -fpx = ["olefile"] -mic = ["olefile"] -test-arrow = ["arro3-compute", "arro3-core", "nanoarrow", "pyarrow"] -tests = ["coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "setuptools", "trove-classifiers (>=2024.10.12)"] -xmp = ["defusedxml"] - -[[package]] -name = "propcache" -version = "0.5.2" -description = "Accelerated property cache" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c"}, - {file = "propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d"}, - {file = "propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e"}, - {file = "propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274"}, - {file = "propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe"}, - {file = "propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d"}, - {file = "propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959"}, - {file = "propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b"}, - {file = "propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27"}, - {file = "propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f"}, - {file = "propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0"}, - {file = "propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82"}, - {file = "propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a"}, - {file = "propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476"}, - {file = "propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33"}, - {file = "propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a"}, - {file = "propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031"}, - {file = "propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42"}, - {file = "propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117"}, - {file = "propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d"}, - {file = "propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704"}, - {file = "propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4"}, - {file = "propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d"}, - {file = "propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757"}, - {file = "propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa"}, - {file = "propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc"}, - {file = "propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55"}, - {file = "propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568"}, - {file = "propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191"}, - {file = "propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7"}, - {file = "propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e"}, - {file = "propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825"}, - {file = "propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5"}, - {file = "propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4"}, - {file = "propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0"}, - {file = "propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c"}, - {file = "propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078"}, - {file = "propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335"}, - {file = "propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d"}, - {file = "propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2"}, - {file = "propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821"}, - {file = "propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370"}, - {file = "propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6"}, - {file = "propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe"}, - {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3"}, - {file = "pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af"}, - {file = "pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194"}, - {file = "pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0"}, - {file = "pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517"}, - {file = "pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062"}, - {file = "pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f"}, - {file = "pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e"}, - {file = "pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e"}, - {file = "pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f"}, - {file = "pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537"}, - {file = "pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b"}, - {file = "pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc"}, - {file = "pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be"}, - {file = "pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d"}, - {file = "pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1"}, - {file = "pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b"}, - {file = "pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec"}, - {file = "pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62"}, - {file = "pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f"}, - {file = "pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e"}, - {file = "pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50"}, - {file = "pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc"}, - {file = "pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0"}, - {file = "pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849"}, - {file = "pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4"}, -] - -[[package]] -name = "pygments" -version = "2.20.0" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, - {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, -] - -[package.extras] -windows-terminal = ["colorama (>=0.4.6)"] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2026.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "regex" -version = "2026.7.10" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, -] - -[[package]] -name = "requests" -version = "2.34.2" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, - {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, -] - -[package.dependencies] -certifi = ">=2023.5.7" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.26,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] - -[[package]] -name = "rich" -version = "15.0.0" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false -python-versions = ">=3.9.0" -groups = ["main"] -files = [ - {file = "rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb"}, - {file = "rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36"}, -] - -[package.dependencies] -markdown-it-py = ">=2.2.0" -pygments = ">=2.13.0,<3.0.0" - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - -[[package]] -name = "safetensors" -version = "0.8.0" -description = "" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0"}, - {file = "safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78"}, - {file = "safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d"}, - {file = "safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846"}, - {file = "safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d"}, - {file = "safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f"}, - {file = "safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452"}, - {file = "safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d"}, -] - -[package.extras] -all = ["safetensors[convert]", "safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] -convert = ["huggingface-hub (>=1.4)", "safetensors[torch]"] -dev = ["safetensors[all]", "safetensors[pinned-tf]"] -jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "safetensors[numpy]"] -mlx = ["mlx (>=0.0.9)"] -numpy = ["numpy (>=1.24.6)"] -paddlepaddle = ["paddlepaddle (>=2.4.1)", "safetensors[numpy]"] -pinned-tf = ["safetensors[numpy]", "tensorflow (==2.18.0)"] -quality = ["ruff"] -tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] -testing = ["fsspec (>=2024.6.0)", "h5py (>=3.7.0)", "hypothesis (>=6.70.2)", "pytest (>=9.0)", "pytest-benchmark (>=5.2)", "s3fs (>=2024.6.0)", "safetensors[numpy]", "setuptools-rust (>=1.12.0)"] -tf-nightly = ["safetensors[numpy]", "tf-nightly"] -torch = ["safetensors[numpy]", "torch (>=2.4)"] - -[[package]] -name = "sentencepiece" -version = "0.2.2" -description = "Unsupervised text tokenizer and detokenizer." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b"}, - {file = "sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711"}, - {file = "sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0"}, - {file = "sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790"}, - {file = "sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e"}, - {file = "sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820"}, - {file = "sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a"}, - {file = "sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383"}, - {file = "sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c"}, - {file = "sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0"}, - {file = "sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9"}, - {file = "sentencepiece-0.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53"}, - {file = "sentencepiece-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b"}, - {file = "sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719"}, - {file = "sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708"}, - {file = "sentencepiece-0.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497"}, - {file = "sentencepiece-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032"}, - {file = "sentencepiece-0.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99"}, - {file = "sentencepiece-0.2.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c"}, - {file = "sentencepiece-0.2.2-cp39-cp39-win_arm64.whl", hash = "sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d"}, - {file = "sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6"}, -] - -[package.extras] -numpy = ["numpy"] -protobuf = ["protobuf"] -test = ["numpy", "protobuf", "pytest"] - -[[package]] -name = "setuptools" -version = "83.0.0" -description = "Most extensible Python build backend with support for C/C++ extension modules" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3"}, - {file = "setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef"}, -] - -[package.extras] -check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.13.0) ; sys_platform != \"cygwin\""] -core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -enabler = ["pytest-enabler (>=3.4)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.18.*)", "pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] - -[[package]] -name = "shellingham" -version = "1.5.4" -description = "Tool to Detect Surrounding Shell" -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, - {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, -] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "sympy" -version = "1.14.0" -description = "Computer algebra system (CAS) in Python" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5"}, - {file = "sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517"}, -] - -[package.dependencies] -mpmath = ">=1.1.0,<1.4" - -[package.extras] -dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"] - -[[package]] -name = "tiktoken" -version = "0.13.0" -description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tiktoken-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:47b1df8d73390a24f94980c75158cdd5c56d256f16d55f30cb49c230caba9ba4"}, - {file = "tiktoken-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7d40c6c5aab171dcd6eb8455bc567bde404bb9def60cdb8c1299cc782b242bb9"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:9b842981fa91accdffd48ff6408a977b7a91c3fbda55d353c3c68114d5c9d69e"}, - {file = "tiktoken-0.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:ed5a30027cb4d8c7ca8b273d4766f3db3cf58fad9e9f3b1a68a351ffb54873d5"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7ab10f4a21c2999846940113f6dbd72e0fa06a24119feddd74cc47e85818e06d"}, - {file = "tiktoken-0.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a2937ad042d49d50eac6e1ba07c5661d4bd3942a5b1e0c0d08475c4df83676e1"}, - {file = "tiktoken-0.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:44733b99bfd72b590cd0936b1c01b3b4dd73122db2d544bc1ceeb18a7678c910"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7bfe1849caa65d1e1d9871817170ec497bbb7984e182012e1bdce72f66608cdb"}, - {file = "tiktoken-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:91c180fe255bd5a86d8316210d2833a1d4d33d026cd86a67812f4773743c8d26"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:059c8ecf554eb5b41e6e054ba467b871b03277d267dee7244380aca4359747d4"}, - {file = "tiktoken-0.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:36217497eaffc158607a3b26f065300db2aefd43b115263f3b9688ce38146173"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:303f7d91b4fce3baddbcde05c139091d4caa5026ac7214c1dc7ff7a71ee429ff"}, - {file = "tiktoken-0.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5d48843bee149630eb735a99e1f4a85b47308d21868ea63163f6e87768d3cfed"}, - {file = "tiktoken-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:fc1c44cd37b43fc46bae593129164f4f281e82ea116b57a85aa81bda57eafc94"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791"}, - {file = "tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7"}, - {file = "tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b"}, - {file = "tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91"}, - {file = "tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154"}, - {file = "tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2"}, - {file = "tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486"}, - {file = "tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615"}, - {file = "tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67"}, - {file = "tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d"}, - {file = "tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2"}, - {file = "tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f"}, - {file = "tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471"}, - {file = "tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881"}, - {file = "tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273"}, - {file = "tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51"}, - {file = "tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b"}, - {file = "tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a"}, - {file = "tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e"}, - {file = "tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424"}, - {file = "tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:35e1ea1e0631c04f551297284a1ab7e1f65a3c55a9a48728d5e0f66b4527c04a"}, - {file = "tiktoken-0.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a3b536c55802fe42f4b4644d2be4f04bf788506b48de0a0a658cb58f8bce232"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b8ac2d6420ff05841a89ba5205c6d45f56c4f6843454f3c884b7eb1a2a8dddb2"}, - {file = "tiktoken-0.13.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:477c9a38e20d0ed248090509acf1e839ad3967a4f00b4b0f958210049f656dee"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:da86f8c96ac1c235d7a3b3eebff1eacfdbcfb8ad792706943268d4d2938fbafe"}, - {file = "tiktoken-0.13.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9b8858b29804b3a0add25ce9e62fb00f89f621dc754d75d03ca419d17e8ddf67"}, - {file = "tiktoken-0.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:b967dfb9d0adf9a631953b1b40717684f04478270fc51bbccdd2f838d67a2f00"}, - {file = "tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1"}, -] - -[package.dependencies] -regex = "*" -requests = "*" - -[package.extras] -blobfile = ["blobfile (>=3)"] - -[[package]] -name = "tokenizers" -version = "0.22.2" -description = "" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c"}, - {file = "tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b"}, - {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a"}, - {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5"}, - {file = "tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92"}, - {file = "tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48"}, - {file = "tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195"}, - {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37"}, - {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113"}, - {file = "tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917"}, -] - -[package.dependencies] -huggingface-hub = ">=0.16.4,<2.0" - -[package.extras] -dev = ["tokenizers[testing]"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] -testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", "ty"] - -[[package]] -name = "torch" -version = "2.13.0" -description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d"}, - {file = "torch-2.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0ab4b69f3ee03a62a002cfbf77b1ca5e88aceb4ea64cb4388bb28f638ddbb045"}, - {file = "torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c78b7b4d04461855a764cf01bae9a462bb88bc93defcfa11235cbc8fdf3e12c4"}, - {file = "torch-2.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:2bd30b6b730d987fa386ce3898933762c5cb8cc82eb0535211d787cc3ce2dfeb"}, - {file = "torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8"}, - {file = "torch-2.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:092790c696a760c729fd5722835f50b9d81fd7c8f141571f3f3cf4081a8f664c"}, - {file = "torch-2.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:60fcdcb2f3876e21146cb4524ef06397d727ca9ad5f020818547e25075fe3cb7"}, - {file = "torch-2.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a0d8b11f16a48d60e2015d8213aa0390744cbebb98e58b62b3514dddc656e330"}, - {file = "torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027"}, - {file = "torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4"}, - {file = "torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b"}, - {file = "torch-2.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:024c6cc0c1b085f2f91f20a3dc27b0471d021c31ce84b81be3afdc39f791fd9d"}, - {file = "torch-2.13.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:33449899ce5496c1b84b4853179d94fd102028ae1407314d9fb956bb79e70d09"}, - {file = "torch-2.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1e09d6a722504957c694faceca843acde562786df1144ebcc5a74075ec7f6005"}, - {file = "torch-2.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a3a9a21312872af8a26950b2c15680335a386a1f56ed03e780653d78b9607e9e"}, - {file = "torch-2.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:49b58f1e2c52440abb6f17c28f0335fe6c6d01ad1a7f55b0183b81e4b34d64e6"}, - {file = "torch-2.13.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d849b390e07d8d333ce8ecaf91b273c656c598379a19c9acf1318a883f6b391c"}, - {file = "torch-2.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a3893dc2da0a972a8ca5d698c85a9f967559ac5f8ee1797b77408aa8734d073c"}, - {file = "torch-2.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:49f1ea385c754e54919408a9bb3b5a72b0b755bbe2c916c1d6f70afbec4908a2"}, - {file = "torch-2.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f8573e3ce9ebcd53fe922f01077a6085ccdfbe5f12fd215883a9d87d7a744fd"}, - {file = "torch-2.13.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c28def70706c2f9ecc752574766e8ae4da9b810ab6676b611166761a78a9f1e1"}, - {file = "torch-2.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:31061ff56ed8fbf26c749806905aeb749ebeb819810fd5d52508aa5afd90dddc"}, - {file = "torch-2.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:cc26eead4cf51d0b544e31e364dcf000846549c273bd148936fe9d24d29acb92"}, - {file = "torch-2.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a7de8a313090dc5c7d7ba4bfe5c3be222528f9a4dba1acc83bddb1157360c4b8"}, -] - -[package.dependencies] -cuda-bindings = {version = ">=13.0.3,<14", markers = "platform_system == \"Linux\" and python_version < \"3.15\""} -cuda-toolkit = {version = "13.0.3", extras = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], markers = "platform_system == \"Linux\""} -filelock = "*" -fsspec = ">=0.8.5" -jinja2 = "*" -networkx = ">=2.5.1" -nvidia-cudnn-cu13 = {version = "9.20.0.48", markers = "platform_system == \"Linux\""} -nvidia-cusparselt-cu13 = {version = "0.8.1", markers = "platform_system == \"Linux\""} -nvidia-nccl-cu13 = {version = "2.29.7", markers = "platform_system == \"Linux\""} -nvidia-nvshmem-cu13 = {version = "3.4.5", markers = "platform_system == \"Linux\""} -setuptools = ">=77.0.3" -sympy = ">=1.13.3" -triton = {version = "3.7.1", markers = "platform_system == \"Linux\" and python_version < \"3.15\""} -typing-extensions = ">=4.10.0" - -[package.extras] -opt-einsum = ["opt-einsum (>=3.3)"] -optree = ["optree (>=0.13.0)"] -pyyaml = ["pyyaml"] - -[[package]] -name = "torchvision" -version = "0.28.0" -description = "image and video datasets and models for torch deep learning" -optional = false -python-versions = "!=3.14.1,>=3.10" -groups = ["main"] -files = [ - {file = "torchvision-0.28.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:2a1ef4b6f4bf5828b48cfad97372c8982db906830884b2868ba5c3df937a7d81"}, - {file = "torchvision-0.28.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:546fd85345cf8652f6cd099d4f9884b0ca5c2f3fae78689a21dd2f35ea6b622f"}, - {file = "torchvision-0.28.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:6dfb0f45e2b4ceb4e76f158c3fbb5f44387099f3c466e3423a09ab665a194aba"}, - {file = "torchvision-0.28.0-cp310-cp310-win_amd64.whl", hash = "sha256:7fad44dc9582570c7d92c4487d36ac46998f40cc39b438e8b8f5111a935ce4e8"}, - {file = "torchvision-0.28.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:83fe6c020866a85acd7d97deccc45ff11d66daf42916d04396a4309c66c0ccb8"}, - {file = "torchvision-0.28.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5a38bc6da3d72621be003400b66f66a2b4c6d644fde05f680c2cb7ca8cf8dd6c"}, - {file = "torchvision-0.28.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7e80f543b22503d9415e126db5f0ff3917036925e38560ee6b9ae38c571a4002"}, - {file = "torchvision-0.28.0-cp311-cp311-win_amd64.whl", hash = "sha256:9a45ea67235d965ef52187130d20002a4de20c54ea3d927a24286961d268dc37"}, - {file = "torchvision-0.28.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:e9f54c30cd52e3ef7fd034cc69b7bb7e0964e1c8f8743e018ab92e95b40f9eee"}, - {file = "torchvision-0.28.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5cf78ebc401ce64ae19b8c55de866bb836797d559a4de9c25ccbe74cfa642d3a"}, - {file = "torchvision-0.28.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:028a3d481b37d785605620d7cdad897064c5a55bae2aa1f2658766333e291940"}, - {file = "torchvision-0.28.0-cp312-cp312-win_amd64.whl", hash = "sha256:87dc16b2df427c1318ad335f1e2be2b3b15b2cf20f7934c83b0505a48425ee5d"}, - {file = "torchvision-0.28.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:d483b4aa3f5237569053f749cd1a2b5bb548ca456e40461a5dd087f21149d123"}, - {file = "torchvision-0.28.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:bb6dd6918460ed89cc7644adcc2402991474d6933cf1ce92b390641cb233fddf"}, - {file = "torchvision-0.28.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ad7b3a439265cc3739a4ab5b4c998c0e38ea99c0ee7ca4dea35c5d0b099ec237"}, - {file = "torchvision-0.28.0-cp313-cp313-win_amd64.whl", hash = "sha256:7e9dd6f60d6e15f8dc27d4f877fdb6002fc70d70272412135f1c2ff9cfa08d3b"}, - {file = "torchvision-0.28.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3bd9dba55224a9db4a2d77f6feaa5651770d8c8e86d3d0ddb0fa6bec54c8712b"}, - {file = "torchvision-0.28.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:89f90e29b0966352811b12589f3a3c61943bf2bb9487b9d7bbec10efb1096bb5"}, - {file = "torchvision-0.28.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:36beb0782976906069ca03d4c9aacaf4b6b838b06ed6c20960ea9c51cce7acdd"}, - {file = "torchvision-0.28.0-cp314-cp314-win_amd64.whl", hash = "sha256:3557cc7b539f46dabcda2b6f2b14017ccbeef024de466d4fc5835fc3f287f769"}, - {file = "torchvision-0.28.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:09ce8f56e81f19b9c378ae7bb109f83f6659fd8bc3cd14241a48e4af46e9ed49"}, - {file = "torchvision-0.28.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:62c7d110f86a039245b587e4fae60278c649f3bd42ff79cfbc1178eca4e72542"}, - {file = "torchvision-0.28.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:904cf89af220f8c6b2ed0296bb5065b474ce43b77558e48b2bf9de8b0ba17204"}, - {file = "torchvision-0.28.0-cp314-cp314t-win_amd64.whl", hash = "sha256:46f581979c010ad6da6bd85ee602aa707e1ff44312670223b7a0ee517ad06d47"}, -] - -[package.dependencies] -numpy = "*" -pillow = ">=5.3.0,<8.3.dev0 || >=8.4.dev0" -torch = "2.13.0" - -[package.extras] -gdown = ["gdown (>=4.7.3)"] -scipy = ["scipy"] - -[[package]] -name = "tqdm" -version = "4.68.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -discord = ["envwrap", "requests"] -notebook = ["ipywidgets (>=6)"] -slack = ["envwrap", "slack-sdk"] -telegram = ["envwrap", "requests"] - -[[package]] -name = "transformers" -version = "5.14.0" -description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." -optional = false -python-versions = ">=3.10.0" -groups = ["main"] -files = [ - {file = "transformers-5.14.0-py3-none-any.whl", hash = "sha256:288287da4fdcdae0b611be32b5be20d4bcbf5caaa5f269292927fb1ff888d19d"}, - {file = "transformers-5.14.0.tar.gz", hash = "sha256:9e1a2b345c195d216afd10082394b915a82e339fa09e9013c32094ff835ca9db"}, -] - -[package.dependencies] -huggingface-hub = ">=1.5.0,<2.0" -numpy = ">=1.17" -packaging = ">=20.0" -pyyaml = ">=5.1" -regex = ">=2025.10.22" -safetensors = ">=0.8.0" -tokenizers = ">=0.22.0,<=0.23.0" -tqdm = ">=4.60" -typer = "*" - -[package.extras] -accelerate = ["accelerate (>=1.1.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.11.5)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] -audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -benchmark = ["optimum-benchmark (>=0.3.0)"] -chat-template = ["jinja2 (>=3.1.0)"] -codecarbon = ["codecarbon (>=2.8.1)"] -deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.11.5)", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] -docs = ["hf-doc-builder"] -integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.15.2,<0.16)", "optuna", "ray[tune] (>=2.7.0)"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] -kernels = ["kernels (>=0.15.2,<0.16)"] -mistral-common = ["mistral-common[image] (>=1.11.5)"] -num2words = ["num2words"] -optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] -ray = ["ray[tune] (>=2.7.0)"] -retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] -sagemaker = ["sagemaker (>=2.31.0)"] -sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] -sklearn = ["scikit-learn"] -testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -tiktoken = ["blobfile", "tiktoken"] -timm = ["timm (>=1.0.23)"] -torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] -video = ["av"] -vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] - -[[package]] -name = "transformers-stream-generator" -version = "0.0.5" -description = "This is a text generation method which returns a generator, streaming out each token in real-time during inference, based on Huggingface/Transformers." -optional = false -python-versions = ">=3.5" -groups = ["main"] -files = [ - {file = "transformers-stream-generator-0.0.5.tar.gz", hash = "sha256:271deace0abf9c0f83b36db472c8ba61fdc7b04d1bf89d845644acac2795ed57"}, -] - -[package.dependencies] -transformers = ">=4.26.1" - -[[package]] -name = "triton" -version = "3.7.1" -description = "A language and compiler for custom Deep Learning operations" -optional = false -python-versions = "<3.15,>=3.10" -groups = ["main"] -markers = "platform_system == \"Linux\"" -files = [ - {file = "triton-3.7.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3daf64305d6cea88d3334c65ebc9bcd0c64c9564a977084366aa768d57cbcf64"}, - {file = "triton-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee89fbf782ec2ad50391dd1cf26cbea4f4467154c37f4773026da8fc31c0f58e"}, - {file = "triton-3.7.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a0e1cd4c4a76370ed74a8432a53cea28716827d19e40ffc732233e35ceb3f6"}, - {file = "triton-3.7.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6744957e9fd610a29680ec2346057d0c86948ed3812468670719f391e94b44a5"}, - {file = "triton-3.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1"}, - {file = "triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728"}, - {file = "triton-3.7.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cdbfc09d9ec58bc5e68321525653220de7515c199e7a8097a97c85e62b52cd0a"}, - {file = "triton-3.7.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58c0e131da05134a2a4788ccbcc0c1105cf0f54c8e98f19e34cd465396dc15eb"}, - {file = "triton-3.7.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe4ea396a06171f1f1f58cbd39c70b09294398f7dd7c620939bab54ad6f934fa"}, - {file = "triton-3.7.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2020153b08280415ec0da6607834e79166442147e78e144df06b508c75b186d2"}, - {file = "triton-3.7.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c58e4c61f0c73b5dba3b5d19b4a7093c32f90dc18b2a7f121a7c16ccd31107b7"}, - {file = "triton-3.7.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10ba85fa2cca4a2fbdeb36bf1cb082f2c252bda55bf9fccd74f65ec5bc647e68"}, -] - -[package.extras] -build = ["cmake (>=3.20,<4.0)", "lit"] -tests = ["autopep8", "isort", "llnl-hatchet", "numpy", "pytest", "pytest-forked", "pytest-xdist", "scipy (>=1.7.1)"] -tutorials = ["matplotlib", "pandas", "tabulate"] - -[[package]] -name = "typer" -version = "0.27.0" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, -] - -[package.dependencies] -annotated-doc = ">=0.0.2" -colorama = {version = "*", markers = "platform_system == \"Windows\""} -rich = ">=13.8.0" -shellingham = ">=1.3.0" - -[[package]] -name = "typing-extensions" -version = "4.16.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, - {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, -] - -[[package]] -name = "tzdata" -version = "2026.3" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -groups = ["main"] -markers = "python_version == \"3.10\" or sys_platform == \"win32\" or sys_platform == \"emscripten\"" -files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] - -[[package]] -name = "xxhash" -version = "3.8.1" -description = "Python binding for xxHash" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937"}, - {file = "xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb"}, - {file = "xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4"}, - {file = "xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb"}, - {file = "xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626"}, - {file = "xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf"}, - {file = "xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147"}, - {file = "xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60"}, - {file = "xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872"}, - {file = "xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef"}, - {file = "xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792"}, - {file = "xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f"}, - {file = "xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc"}, - {file = "xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc"}, - {file = "xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975"}, - {file = "xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a"}, - {file = "xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e"}, - {file = "xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724"}, - {file = "xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913"}, - {file = "xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed"}, - {file = "xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0"}, - {file = "xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437"}, - {file = "xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893"}, - {file = "xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3"}, - {file = "xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a"}, - {file = "xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55"}, - {file = "xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc"}, - {file = "xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647"}, - {file = "xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398"}, - {file = "xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838"}, - {file = "xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629"}, - {file = "xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba"}, - {file = "xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5"}, - {file = "xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9"}, - {file = "xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b"}, - {file = "xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231"}, - {file = "xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe"}, - {file = "xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab"}, - {file = "xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f"}, - {file = "xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c"}, - {file = "xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542"}, - {file = "xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07"}, - {file = "xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799"}, - {file = "xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684"}, - {file = "xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f93e408255ddce525189bf11feaa1be7ee35e55f486c299c97d9caa68d724a5b"}, - {file = "xxhash-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dfdf19b0d5433a75d61f19dc85737af0f0b95e445c1ad69c855115d05efed45"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:947a585bcaa235702b7c59433b485489397f9a163b3f56058b9463a46fd9b74c"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:848182a391fffdc25605443e832f5b443f25498edeccf9a64343fd84421ca04b"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498017fbf2d13a768b3110d084bde39f2bd8664c1de0b8084f8ccc84425b7c88"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3e1107fe5ca030f946dfa59fdbb66b5df121c8432f14b0bdd282d17b297f4eb"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1ffcc98d8878e449e86dec008cea6f44cfd3a954d2ef24ae7d1cc9f725beec7d"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed8bcdab6692fd4ad0dd6241807a24a640a376764460023b8d462d745e6b7b27"}, - {file = "xxhash-3.8.1-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83d879362ddd0fedd3f2ab8ce7cce3da2049a6d51d16da8af73011c6edf4752f"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:afe6380a0e9653a87aa1e6e88fb47718113e5563c7a1cb2bcc23c1d8e17e3961"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:15790b686f8723b845fec6f612a343beb815a25c83117a7fa408d7c8ee5aa8fd"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c919f38cd3f0b5e8d30b81fd6cac688cf9221560340f0c35cbbb8b2bd77ad6ac"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:12a3cf79dadbab9631230ebc4c51c7c60f1e9cdfb890c15fb733eaafe2e7713c"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:1731407102b9332cd3c9dadee07db498bc3d437b95d752b5b1a5f7eb730a3738"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89df64c10adfe340fb00330042537cdd6bf0d8d78bad73f29cfe5427eed7b084"}, - {file = "xxhash-3.8.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3c0d84c5f2e086b120bae4e7f551cbda804c1deb10d958478bed4f89ba286dfe"}, - {file = "xxhash-3.8.1-cp38-cp38-win32.whl", hash = "sha256:4d6e88ddb3c741fbf29e1e7faf429880f8cd1d7aff4303247435a549726b4fb1"}, - {file = "xxhash-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:bbcdf9c92d21c65bc75426eecea724c8fa0d35a6e201fdf1630011d4cc3aa685"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:314d05fbc55719ae2438eaaba77bf2508ca4f030b26fa4c9c8c380e81c48fa33"}, - {file = "xxhash-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e605e0b8abca9457abd5bee737e086ab145a20c25083ef1113013612268872ff"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f8ed8940435834141061da26d27c4dd0d18fb69777bf431f5c6cc46b43349113"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c7574528bc922f8757f34dd78ed60ab52b1c7973b630f5eae7ba33ec133ce71"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d48acabb1e5cb0071009f80d71d7f01b6ba2c1d4b869b1352bb5df3f11bf7dfd"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:614bca2c7cfa87ec95b703e691c3c5eb6c448b6dabbe9776ac53883152951729"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1153265daa10750a9bf8e9b01753d7618024a300925591efaf16b1b7fa536699"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d45eee3a95a8b61e5b568580caac91f1502ddb731aaf8f4aa448a98660b2fb4"}, - {file = "xxhash-3.8.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:632a34590c090d1285ed5efa5a02be919f3f9a56a64bd25f693fe1e2d27a27fb"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cf633fe83b1d4e6519d7259b33afe40fbba5d3f438730156971dd0cf7730610"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:b6fa3116e40e14e7782fb1a9f872f94b5997de21127c95545ce40196ac1351c5"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:454d78e786602278a2a4383d08048482052f4f0c61fa677ca590af08914d9bca"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23e710118a5778a45db740b431943a3f2a82a571a052c2768cce6544d9c8c62e"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:5da703225374e3a4c8d4fd90e26fe7213a52004ec77f88b42b42e9e86d8c6d57"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f8044cf4c77f37968b8c4cbcbf7a0f355d8a437877ae18eba23e3aad953a6cc7"}, - {file = "xxhash-3.8.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4bec8b2c909bcfae9a0dc702346007e02a8c9ba5bbde83ffb224aa194f4f9efc"}, - {file = "xxhash-3.8.1-cp39-cp39-win32.whl", hash = "sha256:57f80a898544db78ec6b0be6183bd1bc008933193d4199f5cde36b0e6bd5e062"}, - {file = "xxhash-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb70573d2995d23932e2871120f78d798ebc3572e54c09e694a18ced95c5f8d9"}, - {file = "xxhash-3.8.1-cp39-cp39-win_arm64.whl", hash = "sha256:402db908ea70eaf9800d9182a66596fc86f36655df8f63fdecf7c11da741d86f"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20"}, - {file = "xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42"}, - {file = "xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46"}, -] - -[[package]] -name = "yarl" -version = "1.24.2" -description = "Yet another URL library" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, -] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" -propcache = ">=0.2.1" - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10,<3.13" -content-hash = "3909021069b52c41dd11be51bcda1e9262f0de2e076e6d8c9aec488a1b4e4a58" diff --git a/security_scanning/examples/models/core/qwenvl/pyproject.toml b/security_scanning/examples/models/core/qwenvl/pyproject.toml deleted file mode 100644 index bd0a33740cd1..000000000000 --- a/security_scanning/examples/models/core/qwenvl/pyproject.toml +++ /dev/null @@ -1,23 +0,0 @@ -[project] -name = "unknown-package" -version = "0.1.0" -description = "" -authors = [ - {name = "TensorRT LLM [90828364+tensorrt-cicd@users.noreply.github.com]"} -] -requires-python = ">=3.10,<3.13" -dependencies = [ - "datasets (==3.1.0)", - "evaluate (>=0.4.6,<0.5.0)", - "rouge-score (>=0.1.2,<0.2.0)", - "transformers-stream-generator (>=0.0.5,<0.0.6)", - "sentencepiece (>=0.1.99)", - "tiktoken (>=0.13.0,<0.14.0)", - "einops (>=0.8.2,<0.9.0)", - "torchvision (>=0.28.0,<0.29.0)" -] - - -[build-system] -requires = ["poetry-core>=2.0.0,<3.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/security_scanning/examples/ngram/poetry.lock b/security_scanning/examples/ngram/poetry.lock index 53e70e9e8011..ce7276117ea5 100644 --- a/security_scanning/examples/ngram/poetry.lock +++ b/security_scanning/examples/ngram/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.1" +version = "3.14.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32"}, + {file = "aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7"}, + {file = "aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19"}, + {file = "aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71"}, + {file = "aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf"}, + {file = "aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7"}, + {file = "aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc"}, ] [package.dependencies] @@ -230,14 +230,14 @@ files = [ [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -499,14 +499,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.29.7" +version = "3.32.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, + {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, + {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, ] [[package]] @@ -706,38 +706,30 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.1" +version = "1.5.2" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, + {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, ] [package.extras] @@ -792,14 +784,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.23.0" +version = "1.25.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, + {file = "huggingface_hub-1.25.1-py3-none-any.whl", hash = "sha256:004d4e70350517e24c68a7dbb7dc5e40b2b6aefef8f94bf7a85f6f9835102ea5"}, + {file = "huggingface_hub-1.25.1.tar.gz", hash = "sha256:21129595ca7a753be479b319913e22cc8808361ac118bd76cc413db831b28a99"}, ] [package.dependencies] @@ -1394,61 +1386,55 @@ xml = ["lxml (>=4.9.2)"] [[package]] name = "pandas" -version = "3.0.3" +version = "3.0.5" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.11" groups = ["main"] markers = "python_version >= \"3.11\"" files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, + {file = "pandas-3.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2946e77e4a53cd248cbde631a12f0e51c8324ce354c3eba4d20147c1ad6f4282"}, + {file = "pandas-3.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71ecc8fb7ed1a7aa4392316b5309a6347e8e7f832f38fd897846b3a1457a9298"}, + {file = "pandas-3.0.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b173f5951ff6b8b0ec7675e20dff3c97b7e7a57dfcce387c2d7c5afe87cb7899"}, + {file = "pandas-3.0.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c0cf1dd9b55a22d105fc46c1b489af3bd42264fcba7c66297bf47a9a1d9c78a"}, + {file = "pandas-3.0.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0fac0010c75e4efb6b99e249c183a8993ce0dc95c240f9b120a5e67c727b7928"}, + {file = "pandas-3.0.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:08d24fe11a17dc33bd6e937dc9c665f9cba08fbdc9f657f405713515febe300d"}, + {file = "pandas-3.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b1261758dfb6cf12c3cff8300e21cefad30e7ec709abb4c24ac7318e6a52462a"}, + {file = "pandas-3.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:679f4e85b30ddb1515458ab1e788d3e260eae369b1f78da7a3aa4cac8ebf4a2a"}, + {file = "pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d"}, + {file = "pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc"}, + {file = "pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc"}, + {file = "pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34"}, + {file = "pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6"}, + {file = "pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7"}, + {file = "pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce"}, + {file = "pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41"}, + {file = "pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49"}, + {file = "pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b"}, + {file = "pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3"}, + {file = "pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b"}, + {file = "pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be"}, + {file = "pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58"}, + {file = "pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee"}, + {file = "pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6"}, + {file = "pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e"}, + {file = "pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36"}, + {file = "pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c"}, + {file = "pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0"}, + {file = "pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b"}, + {file = "pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94"}, + {file = "pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da"}, + {file = "pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92"}, + {file = "pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85"}, + {file = "pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca"}, + {file = "pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da"}, + {file = "pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a"}, + {file = "pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040"}, + {file = "pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd"}, + {file = "pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c"}, + {file = "pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea"}, + {file = "pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c"}, + {file = "pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712"}, ] [package.dependencies] @@ -1478,7 +1464,7 @@ postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psyc pyarrow = ["pyarrow (>=13.0.0)"] spss = ["pyreadstat (>=1.2.8)"] sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] +test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4,<9.1)", "pytest-xdist (>=3.6.1)"] timezone = ["pytz (>=2020.1)"] xml = ["lxml (>=5.3.0)"] @@ -1683,15 +1669,15 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2026.2" +version = "2026.3.post1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" groups = ["main"] markers = "python_version == \"3.10\"" files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, + {file = "pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815"}, + {file = "pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d"}, ] [[package]] @@ -1779,126 +1765,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.10" +version = "2026.7.19" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, + {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, + {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, + {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, + {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, + {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, + {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, + {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, + {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, + {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, + {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, + {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, + {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, + {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, + {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, + {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, + {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, + {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, + {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, + {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, + {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, + {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, + {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, ] [[package]] @@ -1954,14 +1940,14 @@ files = [ [[package]] name = "tqdm" -version = "4.68.4" +version = "4.70.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, + {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, + {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, ] [package.dependencies] @@ -2215,116 +2201,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.2" +version = "1.24.5" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, + {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, + {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, + {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, + {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, + {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, + {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, + {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, + {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, + {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, + {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, + {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, + {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, + {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, + {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, ] [package.dependencies] diff --git a/security_scanning/examples/quantization/poetry.lock b/security_scanning/examples/quantization/poetry.lock index 44b860ba0232..86ba1072adec 100644 --- a/security_scanning/examples/quantization/poetry.lock +++ b/security_scanning/examples/quantization/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.1" +version = "3.14.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32"}, + {file = "aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7"}, + {file = "aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19"}, + {file = "aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71"}, + {file = "aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf"}, + {file = "aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7"}, + {file = "aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc"}, ] [package.dependencies] @@ -242,14 +242,14 @@ files = [ [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -475,14 +475,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.29.7" +version = "3.32.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, + {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, + {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, ] [[package]] @@ -682,38 +682,30 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.1" +version = "1.5.2" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, + {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, ] [package.extras] @@ -768,14 +760,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.23.0" +version = "1.25.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, + {file = "huggingface_hub-1.25.1-py3-none-any.whl", hash = "sha256:004d4e70350517e24c68a7dbb7dc5e40b2b6aefef8f94bf7a85f6f9835102ea5"}, + {file = "huggingface_hub-1.25.1.tar.gz", hash = "sha256:21129595ca7a753be479b319913e22cc8808361ac118bd76cc413db831b28a99"}, ] [package.dependencies] @@ -1409,61 +1401,55 @@ xml = ["lxml (>=4.9.2)"] [[package]] name = "pandas" -version = "3.0.3" +version = "3.0.5" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.11" groups = ["main"] markers = "python_version >= \"3.11\"" files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, + {file = "pandas-3.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2946e77e4a53cd248cbde631a12f0e51c8324ce354c3eba4d20147c1ad6f4282"}, + {file = "pandas-3.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71ecc8fb7ed1a7aa4392316b5309a6347e8e7f832f38fd897846b3a1457a9298"}, + {file = "pandas-3.0.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b173f5951ff6b8b0ec7675e20dff3c97b7e7a57dfcce387c2d7c5afe87cb7899"}, + {file = "pandas-3.0.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c0cf1dd9b55a22d105fc46c1b489af3bd42264fcba7c66297bf47a9a1d9c78a"}, + {file = "pandas-3.0.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0fac0010c75e4efb6b99e249c183a8993ce0dc95c240f9b120a5e67c727b7928"}, + {file = "pandas-3.0.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:08d24fe11a17dc33bd6e937dc9c665f9cba08fbdc9f657f405713515febe300d"}, + {file = "pandas-3.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b1261758dfb6cf12c3cff8300e21cefad30e7ec709abb4c24ac7318e6a52462a"}, + {file = "pandas-3.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:679f4e85b30ddb1515458ab1e788d3e260eae369b1f78da7a3aa4cac8ebf4a2a"}, + {file = "pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d"}, + {file = "pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc"}, + {file = "pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc"}, + {file = "pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34"}, + {file = "pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6"}, + {file = "pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7"}, + {file = "pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce"}, + {file = "pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41"}, + {file = "pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49"}, + {file = "pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b"}, + {file = "pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3"}, + {file = "pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b"}, + {file = "pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be"}, + {file = "pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58"}, + {file = "pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee"}, + {file = "pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6"}, + {file = "pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e"}, + {file = "pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36"}, + {file = "pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c"}, + {file = "pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0"}, + {file = "pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b"}, + {file = "pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94"}, + {file = "pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da"}, + {file = "pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92"}, + {file = "pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85"}, + {file = "pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca"}, + {file = "pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da"}, + {file = "pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a"}, + {file = "pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040"}, + {file = "pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd"}, + {file = "pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c"}, + {file = "pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea"}, + {file = "pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c"}, + {file = "pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712"}, ] [package.dependencies] @@ -1493,7 +1479,7 @@ postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psyc pyarrow = ["pyarrow (>=13.0.0)"] spss = ["pyreadstat (>=1.2.8)"] sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] +test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4,<9.1)", "pytest-xdist (>=3.6.1)"] timezone = ["pytz (>=2020.1)"] xml = ["lxml (>=5.3.0)"] @@ -1713,15 +1699,15 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2026.2" +version = "2026.3.post1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" groups = ["main"] markers = "python_version == \"3.10\"" files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, + {file = "pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815"}, + {file = "pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d"}, ] [[package]] @@ -1809,126 +1795,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.10" +version = "2026.7.19" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, + {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, + {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, + {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, + {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, + {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, + {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, + {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, + {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, + {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, + {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, + {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, + {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, + {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, + {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, + {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, + {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, + {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, + {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, + {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, + {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, + {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, + {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, ] [[package]] @@ -2173,14 +2159,14 @@ testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", [[package]] name = "tqdm" -version = "4.68.4" +version = "4.70.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, + {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, + {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, ] [package.dependencies] @@ -2194,14 +2180,14 @@ telegram = ["envwrap", "requests"] [[package]] name = "transformers" -version = "5.14.0" +version = "5.14.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.0-py3-none-any.whl", hash = "sha256:288287da4fdcdae0b611be32b5be20d4bcbf5caaa5f269292927fb1ff888d19d"}, - {file = "transformers-5.14.0.tar.gz", hash = "sha256:9e1a2b345c195d216afd10082394b915a82e339fa09e9013c32094ff835ca9db"}, + {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, + {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, ] [package.dependencies] @@ -2520,116 +2506,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.2" +version = "1.24.5" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, + {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, + {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, + {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, + {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, + {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, + {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, + {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, + {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, + {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, + {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, + {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, + {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, + {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, + {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, ] [package.dependencies] diff --git a/security_scanning/examples/ray_orchestrator/poetry.lock b/security_scanning/examples/ray_orchestrator/poetry.lock index 378e3fef149f..e3ed070d7b32 100644 --- a/security_scanning/examples/ray_orchestrator/poetry.lock +++ b/security_scanning/examples/ray_orchestrator/poetry.lock @@ -14,131 +14,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.1" +version = "3.14.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32"}, + {file = "aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7"}, + {file = "aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19"}, + {file = "aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71"}, + {file = "aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf"}, + {file = "aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7"}, + {file = "aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc"}, ] [package.dependencies] @@ -188,14 +188,14 @@ typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} [[package]] name = "annotated-types" -version = "0.7.0" +version = "0.8.0" description = "Reusable constraint types to use with typing.Annotated" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, - {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, + {file = "annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0"}, + {file = "annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7"}, ] [[package]] @@ -225,14 +225,14 @@ files = [ [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -572,14 +572,14 @@ files = [ [[package]] name = "filelock" -version = "3.29.7" +version = "3.32.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, + {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, + {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, ] [[package]] @@ -724,14 +724,14 @@ files = [ [[package]] name = "google-api-core" -version = "2.31.0" +version = "2.33.0" description = "Google API client core library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "google_api_core-2.31.0-py3-none-any.whl", hash = "sha256:ef79fb3784c71cbac89cbd03301ba0c8fb8ad2aa95d7f9204dd9628f7adf59ab"}, - {file = "google_api_core-2.31.0.tar.gz", hash = "sha256:2be84ee0f584c48e6bde1b36766e23348b361fb7e55e56135fc76ce1c397f9c2"}, + {file = "google_api_core-2.33.0-py3-none-any.whl", hash = "sha256:a2e22a0c1d0f03eafff1858b38cf46f832d5902b0c052235bf0ab8402929fbdc"}, + {file = "google_api_core-2.33.0.tar.gz", hash = "sha256:3a36bcc3e319783f4c97da41f6f45ea6ffcaa55848e341de16e09cb70243c2bb"}, ] [package.dependencies] @@ -747,14 +747,14 @@ grpc = ["grpcio (>=1.41.0,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version > [[package]] name = "google-auth" -version = "2.56.0" +version = "2.56.2" description = "Google Authentication Library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "google_auth-2.56.0-py3-none-any.whl", hash = "sha256:6e88c10217e07a92bfd01cac8ee99e32ccfb08414c3102e6c5b8d58f37a0d1e0"}, - {file = "google_auth-2.56.0.tar.gz", hash = "sha256:f90fa030b569a92654b9d690665a073841df33d57487be53db583a9a0867a553"}, + {file = "google_auth-2.56.2-py3-none-any.whl", hash = "sha256:c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6"}, + {file = "google_auth-2.56.2.tar.gz", hash = "sha256:e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051"}, ] [package.dependencies] @@ -794,70 +794,70 @@ grpc = ["grpcio (>=1.44.0,<2.0.0)"] [[package]] name = "grpcio" -version = "1.82.1" +version = "1.83.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "grpcio-1.82.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:91859d1cac5f47caec5fc40e9f827500cdb54ce5b36450dc9a65616b5af49c17"}, - {file = "grpcio-1.82.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c80c9741dcef192f669876a81957cf7713b441c2f0c43631350d75fa49321d31"}, - {file = "grpcio-1.82.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b89cff456796d2f0581783726ad017a2c70aff2d27b0f05504c34e2e417f7560"}, - {file = "grpcio-1.82.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:d6e8a08f7038ba7a77f71e250804e4aba84fe91d22cfc54ff43c07b7529c4728"}, - {file = "grpcio-1.82.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:50fd2fe83426b1b1c6cdc4d72d555223b7dddf8ce07c5bac218b13fc6d684c6f"}, - {file = "grpcio-1.82.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b758540a24d5394a9c578bf9f6126389f474b106ac3d9df1d53de56cb14c9fd9"}, - {file = "grpcio-1.82.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c4ba4aac238f685743575d9d700003ac16537cce26e7c774993134f530652464"}, - {file = "grpcio-1.82.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed6fc621d6f366c88a60f0b971d5afd21d441d9aa561ee688de5b7acdb2cf901"}, - {file = "grpcio-1.82.1-cp310-cp310-win32.whl", hash = "sha256:bd2f45e46fff5b91c10997d0743a987517a7dde67c64c592835c2dcaac66f587"}, - {file = "grpcio-1.82.1-cp310-cp310-win_amd64.whl", hash = "sha256:5e171d5f0d6a0af78ea7512783f170a44f80c165259d8773e3a354a7f991f2b5"}, - {file = "grpcio-1.82.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:0ddb18a9a9e1f46692b3567ae4abb3f8d117ce6afea48650f8eca06d8ab5d06f"}, - {file = "grpcio-1.82.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cf855b1af246720f567b0ce5d0724d45dfa4188eecc3296a2a69257b11b9e94b"}, - {file = "grpcio-1.82.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb30cb13e25bc13cea70ffc69d6d90c49d36ea6c1d4549e6912f70177834cac"}, - {file = "grpcio-1.82.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1e822b2774f719c017cbe700b6e47173b6ae290fb84906f52a5a3c2c60b62e1e"}, - {file = "grpcio-1.82.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5dafb1ece8ed45dee7c738f166ec82e19673221ed5ab8967f72858a4685345b2"}, - {file = "grpcio-1.82.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e06503106e7271e0a49fd5a1ac04747f1e47e87d900476db6fe45bc87ee411f4"}, - {file = "grpcio-1.82.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ff99bc8cafb6a952201c37b995f425e641c93ffa6e072258525feab57290141d"}, - {file = "grpcio-1.82.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:644ae1b94266ac785330f4590a69e52b6a7eb73029043a02209db81c81397d69"}, - {file = "grpcio-1.82.1-cp311-cp311-win32.whl", hash = "sha256:e203d2e19d471630084a16c815616f8211dff21c268ab3c5f5bf38417832e074"}, - {file = "grpcio-1.82.1-cp311-cp311-win_amd64.whl", hash = "sha256:0d8299c285fe6cc6a1f56badf8d3bc5078c8d20273ee64bafa3783b4bc29a769"}, - {file = "grpcio-1.82.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:c09bd5fa0d5b1fbd773ec349fe61441c3e4ebf168c229aa7538a820bdfad6a58"}, - {file = "grpcio-1.82.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1eae24810720734598e3e6a1a528d5de0f265fe3fc86575e9ecce424b9ec7379"}, - {file = "grpcio-1.82.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a6bd5daf5bde7b24d7ad2cbaf8bf9eac620d96222016bb5e7ddde930dec0673f"}, - {file = "grpcio-1.82.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1ecfde669cb687ac020d31ff76debe5dc7a62213335f02262eb6625628da1c03"}, - {file = "grpcio-1.82.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:011c8badee95734dee8bf05ce3464756a0ac3ebb8d443afd20c0e2b5e4640ad9"}, - {file = "grpcio-1.82.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b85f4564926fb23114d239392bdcae200db1e6179629edd7d7ab0ab89c96a197"}, - {file = "grpcio-1.82.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2c0c8270833395644c3fe6b6a806397955a2bc0538000a19a78b90c05a6c16e0"}, - {file = "grpcio-1.82.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2ba199205ff46c7778290fe1673c91ac8e7e45678dd5c86e9e56fa33ec8788f6"}, - {file = "grpcio-1.82.1-cp312-cp312-win32.whl", hash = "sha256:06127691866e295c14e84a1fb86356dd962254f6abd0da4ca4b001eea9e89438"}, - {file = "grpcio-1.82.1-cp312-cp312-win_amd64.whl", hash = "sha256:1fa3223a3a2e1db74f4c2b255189eb7ea875dfba56e221d252ee3fc7b204778e"}, - {file = "grpcio-1.82.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:b454a2d97bfab7565683a02345f86bd182ab69fd7c2bdb7414171e7538f266b1"}, - {file = "grpcio-1.82.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:3dde70abfc80b3be11de53ba0d601c439e7fb2afd3583ad1788d1146bec92fdc"}, - {file = "grpcio-1.82.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5523099c98c292ea1ae08e617249db760c56a78f8deae879027fe7d1ffbcbf6"}, - {file = "grpcio-1.82.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5e5c4dc0a59b0f8490a6bdfd6fc8395b9d8ad8a8407c7d67ca7b5bba15c0877f"}, - {file = "grpcio-1.82.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c40d94ba820329cc191981bc22fa6f6eed0799c6d921f3c6709521d59d4a2fd7"}, - {file = "grpcio-1.82.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c816180e31e273caaec6f8bd86a8392499d5bbb26f41da44e3dce48bde69095"}, - {file = "grpcio-1.82.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e31fd780b261830720cb70b0fd8f0aa51d49e75a66d7464ad2e31d4b765f2580"}, - {file = "grpcio-1.82.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d76152d7c31d7210d4a106e5d8b64da5bba5d6abf11be30e2f7b0a0c59bbcbf"}, - {file = "grpcio-1.82.1-cp313-cp313-win32.whl", hash = "sha256:38e9dcb5258226fb3282630b31b16a968df52c8c6ad514af540646e0a4578f8a"}, - {file = "grpcio-1.82.1-cp313-cp313-win_amd64.whl", hash = "sha256:3dbfb52c36d9511ac2b8e6c94fdde837b393ae520cc321f52a333a2deedf5a90"}, - {file = "grpcio-1.82.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:35f990f7784c8fd2872644f07f96ebb4d9e48e145a190ab80d0280af91a1bfb2"}, - {file = "grpcio-1.82.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:46536a4a1f4434df3c851b9254ff6fc7df5705b273681a15ca277d5921c178a0"}, - {file = "grpcio-1.82.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d6650a7c1ebb7921c70e12a385439a8118efb99e669fa9ed31cf25db1843937c"}, - {file = "grpcio-1.82.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b8e110c66df5204c0506d6c8787b35d48b8b699ef5aa366d6c4d67325c67fe9a"}, - {file = "grpcio-1.82.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f853eae07235a51a27bb5d6a9a175a59ca55dc9b99edc6ce2f76f07332d333ae"}, - {file = "grpcio-1.82.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:60b0f2c95337694fc094b77d9f60f50566c84b5677393e342eb98daeee242d98"}, - {file = "grpcio-1.82.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:b064fc444812bdaa9825d33c26f8d732d63ee6a5d78557c1faf92c98687fed27"}, - {file = "grpcio-1.82.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7d7ede11d747b4e1bd05e3bc0260e155b65a88735a895a10f6521f19b889511e"}, - {file = "grpcio-1.82.1-cp314-cp314-win32.whl", hash = "sha256:3d21f19838dc255ecbb79321b15ae9b98fbddff4c3d4aedb0a81bdd7f4ab572a"}, - {file = "grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5"}, - {file = "grpcio-1.82.1.tar.gz", hash = "sha256:707b24abd90fcb1e45bcc080577da1dbf9971d107490589b9539af8e1e77b4b5"}, + {file = "grpcio-1.83.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:fba099b716e73512d61b97f71ea3c31a72abb36904036e316bf4dd148ca8dcc8"}, + {file = "grpcio-1.83.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6755ed67cc3e454d51ae9f6e1915b80d3942fa4de956ef48dacd45ab7f40b727"}, + {file = "grpcio-1.83.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5882c1a721b50ce0123ee5e839e1ab059ad72a7ade76cdf2d5bd833b56791acf"}, + {file = "grpcio-1.83.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4e3eedfc92b6b9f2960115e7e620cf0cbf80bb7849a51ce3820dc54dfd88b6b9"}, + {file = "grpcio-1.83.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4fcaa7c45c45b4a89e2867d1f1785d9481a788399d915e341ed2eb49aeef9dd4"}, + {file = "grpcio-1.83.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6b6c666a1d5613ff360c9e90f44665e3a88b25a815209ddbc0917eec281931cb"}, + {file = "grpcio-1.83.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6be5c807b717be3dd649446f021301fd7907e376318675d2147823071034112a"}, + {file = "grpcio-1.83.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c834e86d8fd2f03d7e4db49a027f7c5b89c5b88eed305543a5295bd6fee61e40"}, + {file = "grpcio-1.83.0-cp310-cp310-win32.whl", hash = "sha256:35a5b1c192496b6c25956eebfa963468935612206fd2543ac3ce981e6a5e0f03"}, + {file = "grpcio-1.83.0-cp310-cp310-win_amd64.whl", hash = "sha256:8f6c395e493d20c39b29392ca200e9aaeb78d0bc2f04db0c0a7da7ddc939aa57"}, + {file = "grpcio-1.83.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:8ff0b8767ddd62704e0d9571c1890af08d84a3a689ebba1807e62519d0b3277f"}, + {file = "grpcio-1.83.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:4772402f43517b4824980be4b3b2274a81eec0004a70009473c31b340d43e223"}, + {file = "grpcio-1.83.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f4cee5fc86e84a0cf7ad1574b454c3320e087c07f55b7df5dc0ac6a873fb90c0"}, + {file = "grpcio-1.83.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f5e822a7e7d03282f6ad225e710493c48b9057a353358344a5f7c42b2b37618d"}, + {file = "grpcio-1.83.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f5f410d7c2903eabb34789dfd6342eef04af1ad459943936b7e09a9f5bd417b9"}, + {file = "grpcio-1.83.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee94a4016fdf8699fb1fd8a38652475ff677f1c72074cee44deeeb9a7e95e745"}, + {file = "grpcio-1.83.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c6444666317338e903093c7c756e6cc88eee59f798cb8dd41e87725bf54e1617"}, + {file = "grpcio-1.83.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa074041231f03959cb097dd5517b0677b8ea49215bae01d5710a7b69dd59969"}, + {file = "grpcio-1.83.0-cp311-cp311-win32.whl", hash = "sha256:cb056f6e171c42639a50460b2929c82241fda51f71cf3dcdd68090fe45095a45"}, + {file = "grpcio-1.83.0-cp311-cp311-win_amd64.whl", hash = "sha256:7416952ca770477990257206276999056f8316d79196f2f25942393e58a20b49"}, + {file = "grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930"}, + {file = "grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da"}, + {file = "grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16"}, + {file = "grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf"}, + {file = "grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd"}, + {file = "grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c"}, + {file = "grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5"}, + {file = "grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5"}, + {file = "grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc"}, + {file = "grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df"}, + {file = "grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0"}, + {file = "grpcio-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:74fe6f9e8a35c7dbf32255ee154d15e3e5338a81ed39173d079d594d2e544cd1"}, + {file = "grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867"}, + {file = "grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881"}, + {file = "grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f"}, + {file = "grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61"}, + {file = "grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45"}, + {file = "grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c"}, + {file = "grpcio-1.83.0-cp313-cp313-win32.whl", hash = "sha256:8fe04f1050a59f875601eb55d42b4f66946fe89817f967e34db1462ccd07dadf"}, + {file = "grpcio-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:6e01ecd9d8ef280abe1365138a4dc318f9a5287f4cb1b41d07816f796653f735"}, + {file = "grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa"}, + {file = "grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c"}, + {file = "grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b"}, + {file = "grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c"}, + {file = "grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df"}, + {file = "grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b"}, + {file = "grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404"}, + {file = "grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af"}, + {file = "grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33"}, + {file = "grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9"}, + {file = "grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24"}, ] [package.dependencies] typing-extensions = ">=4.12,<5.0" [package.extras] -protobuf = ["grpcio-tools (>=1.82.1)"] +protobuf = ["grpcio-tools (>=1.83.0)"] [[package]] name = "idna" @@ -1177,14 +1177,14 @@ files = [ [[package]] name = "opentelemetry-api" -version = "1.43.0" +version = "1.44.0" description = "OpenTelemetry Python API" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_api-1.43.0-py3-none-any.whl", hash = "sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd"}, - {file = "opentelemetry_api-1.43.0.tar.gz", hash = "sha256:107d0d03857ea8fc7c5fcbbbd83f800c281f0d560553d61c1d675fccfd1761c1"}, + {file = "opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef"}, + {file = "opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a"}, ] [package.dependencies] @@ -1192,31 +1192,31 @@ typing-extensions = ">=4.5.0" [[package]] name = "opentelemetry-exporter-prometheus" -version = "0.64b0" +version = "0.65b0" description = "Prometheus Metric Exporter for OpenTelemetry" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_exporter_prometheus-0.64b0-py3-none-any.whl", hash = "sha256:9979a15f8d007d442bc7a6e16f4cbde5e8e0c5e99689887ceb5f33da251f0655"}, - {file = "opentelemetry_exporter_prometheus-0.64b0.tar.gz", hash = "sha256:96fec79be9527cb9dc994d7e663051df35161eb936fe2d41954725e4595abbc1"}, + {file = "opentelemetry_exporter_prometheus-0.65b0-py3-none-any.whl", hash = "sha256:3b3d24b586d0ad9712c7b52b7d19c8a9dfbb318b9b284121b5f95e90ed019367"}, + {file = "opentelemetry_exporter_prometheus-0.65b0.tar.gz", hash = "sha256:2777cbf41c403c119e10f418fce5d645c956b47f673a2ee120285d1d0c6df2d5"}, ] [package.dependencies] opentelemetry-api = ">=1.12,<2.0" -opentelemetry-sdk = ">=1.43.0,<1.44.0" +opentelemetry-sdk = ">=1.44.0,<1.45.0" prometheus-client = ">=0.5.0,<1.0.0" [[package]] name = "opentelemetry-proto" -version = "1.43.0" +version = "1.44.0" description = "OpenTelemetry Python Proto" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_proto-1.43.0-py3-none-any.whl", hash = "sha256:c58f1f7ef84bc7dc2834016c0c37fe0081dde7ca9f6339be1970fbf9cdaaa90d"}, - {file = "opentelemetry_proto-1.43.0.tar.gz", hash = "sha256:224778df17e1f3fafeaaa21d874236ca5f6ffc2f86e0899298ec7351aac27924"}, + {file = "opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56"}, + {file = "opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3"}, ] [package.dependencies] @@ -1224,38 +1224,38 @@ protobuf = ">=5.0,<8.0" [[package]] name = "opentelemetry-sdk" -version = "1.43.0" +version = "1.44.0" description = "OpenTelemetry Python SDK" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_sdk-1.43.0-py3-none-any.whl", hash = "sha256:d1323a547c1ce69d6a069a17a44b7da82bb8b332051ecb074041f87642c86823"}, - {file = "opentelemetry_sdk-1.43.0.tar.gz", hash = "sha256:d8187c81c162df9913e4003dd6485f7390d9a24fc17026ec7387b8b8218b08e9"}, + {file = "opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad"}, + {file = "opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b"}, ] [package.dependencies] -opentelemetry-api = "1.43.0" -opentelemetry-semantic-conventions = "0.64b0" +opentelemetry-api = "1.44.0" +opentelemetry-semantic-conventions = "0.65b0" typing-extensions = ">=4.5.0" [package.extras] -file-configuration = ["jsonschema (>=4.0)", "pyyaml (>=6.0)"] +file-configuration = ["opentelemetry-configuration (==0.65b0)"] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.64b0" +version = "0.65b0" description = "OpenTelemetry Semantic Conventions" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_semantic_conventions-0.64b0-py3-none-any.whl", hash = "sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6"}, - {file = "opentelemetry_semantic_conventions-0.64b0.tar.gz", hash = "sha256:72f76fb2d1582d9d033dd1fcd84532e961e6ff3d90d24ba6fabc72975a83864c"}, + {file = "opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb"}, + {file = "opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60"}, ] [package.dependencies] -opentelemetry-api = "1.43.0" +opentelemetry-api = "1.44.0" typing-extensions = ">=4.5.0" [[package]] @@ -1272,26 +1272,26 @@ files = [ [[package]] name = "platformdirs" -version = "4.10.0" +version = "4.11.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, - {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, + {file = "platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74"}, + {file = "platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0"}, ] [[package]] name = "prometheus-client" -version = "0.25.0" +version = "0.26.0" description = "Python client for the Prometheus monitoring system." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1"}, - {file = "prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28"}, + {file = "prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6"}, + {file = "prometheus_client-0.26.0.tar.gz", hash = "sha256:04a91bcf94e2cf74a44a1a874d651a2e853ed354b6e822f3b7487751465d5c2b"}, ] [package.extras] @@ -1432,14 +1432,14 @@ files = [ [[package]] name = "proto-plus" -version = "1.28.1" +version = "1.28.2" description = "Beautiful, Pythonic protocol buffers" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "proto_plus-1.28.1-py3-none-any.whl", hash = "sha256:6660f5f1970874bdcfc3088b435188a36a37bd3596668f7d726417c4ae8cfbed"}, - {file = "proto_plus-1.28.1.tar.gz", hash = "sha256:832e68e7fe064cf90ab153b6e5eb935b27891bb89aaeb68b115e9b702f6cb168"}, + {file = "proto_plus-1.28.2-py3-none-any.whl", hash = "sha256:b874236fcac2358f601e4330bcb76cb8b89c851303ccf4078408b3d4774d1c52"}, + {file = "proto_plus-1.28.2.tar.gz", hash = "sha256:26d843eb99c1e32fdf1d20ff0faae56607f7748fe774acf9ecd5cfe6c6472501"}, ] [package.dependencies] @@ -1684,24 +1684,20 @@ typing-extensions = ">=4.14.1" [[package]] name = "python-discovery" -version = "1.4.4" +version = "1.5.0" description = "Python interpreter discovery" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "python_discovery-1.4.4-py3-none-any.whl", hash = "sha256:abebe9120b43453b68c908acfb1e72a19d1a959ed2cb620ad38fc57d08056dbe"}, - {file = "python_discovery-1.4.4.tar.gz", hash = "sha256:5cad33982d412c1f3ffb8f9ca4ea292c9680bca3942451d30b69c37fce53a4a3"}, + {file = "python_discovery-1.5.0-py3-none-any.whl", hash = "sha256:70c4fc61b4e7404e44f01d6fc44a715c4d685ca6cea83d295922f05891877c98"}, + {file = "python_discovery-1.5.0.tar.gz", hash = "sha256:3e014c6327154d3dda27939a9a0dc9c5c000439f1906d3f303b48f984bd2ecef"}, ] [package.dependencies] filelock = ">=3.15.4" platformdirs = ">=4.3.6,<5" -[package.extras] -docs = ["furo (>=2025.12.19)", "sphinx (>=9.1)", "sphinx-autodoc-typehints (>=3.6.3)", "sphinxcontrib-mermaid (>=2)", "sphinxcontrib-towncrier (>=0.4)", "towncrier (>=25.8)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.5.4)", "pytest (>=8.3.5)", "pytest-mock (>=3.14)", "setuptools (>=75.1)"] - [[package]] name = "pyyaml" version = "6.0.3" @@ -1787,30 +1783,30 @@ files = [ [[package]] name = "ray" -version = "2.56.0" +version = "2.56.1" description = "Ray provides a simple, universal API for building distributed applications." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "ray-2.56.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:f34b2345a47ad144292c1b34eeba2ed8d556078f7bd118d1adf2090d5199c843"}, - {file = "ray-2.56.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:15ea7ac36bfa3961c1eb2b2a099ed7dcf892f001f462920b6ec379ccafb038b0"}, - {file = "ray-2.56.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:58be75df2d4a6a85b5e514e4d3261760fe21cc09ba974ce22cc08a8e4e07c449"}, - {file = "ray-2.56.0-cp310-cp310-win_amd64.whl", hash = "sha256:b837c5a905647c9b6a4f55061c2782437da24009e4eb4781db1c9c4badc84d0b"}, - {file = "ray-2.56.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:a9ad4e26941eb2f8dbd494ad07f9f2227143164c6114132b26b23ad4f20b1c6f"}, - {file = "ray-2.56.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:aea655831d25084cb343002a8e67a77b6aa552ddb776a65461d49f62884f096a"}, - {file = "ray-2.56.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:c5bf1a4384c0e2aa4420c95474b734d064cac354b0f00760be02d886afe96ca4"}, - {file = "ray-2.56.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e57781685bb4332edf8a7cfb1135ff53d50002b6a0504006e68365bcd26aa7c"}, - {file = "ray-2.56.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:684a427c50745989e92a332343f0812c93b8506f71c768b95b1eefc113492699"}, - {file = "ray-2.56.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:e1fd03c6ecc5fe4c31466569e41ce0a4faf26fb930798c9d1f1eb1f405a687c8"}, - {file = "ray-2.56.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:78ef34a71383c1fcf335e531e0e590867857fce9069f06ed351be6ce7a58fc50"}, - {file = "ray-2.56.0-cp312-cp312-win_amd64.whl", hash = "sha256:a8fc809dab6fc07cf05d45ea93a776c852c990512eb1fac0e3d15819fe6df10a"}, - {file = "ray-2.56.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:992047f50473b5bfea74c8f528f999968e0b4bc735af23ad476a0f4e04741aea"}, - {file = "ray-2.56.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:a0e9cfe92c88ab74abca23923c15a592f49bc7617ffdda4190daca7785a7c4f6"}, - {file = "ray-2.56.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:54d2725f8b65d9615c933fec5ec62e54a67b8f2e14286026fb014606253670ef"}, - {file = "ray-2.56.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:f38e03b77c53e3d94091aedb84b14efe7ee5b581d85b7d13925066bcd48c44a2"}, - {file = "ray-2.56.0-cp314-cp314-manylinux2014_aarch64.whl", hash = "sha256:c3a16d43d75283a3d64fa1d904a3adaf3f526f3f508f447505b8bb8dc70bad6c"}, - {file = "ray-2.56.0-cp314-cp314-manylinux2014_x86_64.whl", hash = "sha256:73edb6fb5fd05481b1f358ac2e8a4c7f6a031d89f9b823d25a587ddb7529070f"}, + {file = "ray-2.56.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:58b8c037a9f2b7b7866439cb6fe19d452ba3961f2b62d0a247dc3adf2ca45265"}, + {file = "ray-2.56.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:7005a13785c7478c3d183b6f1e7a344c069c4c4fb187707c4fc31ca8c84d8d9f"}, + {file = "ray-2.56.1-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:49aa1f8aa1799a6b63adc8e4edd600b949b2bb648461abedcbcccd56c8ef0f82"}, + {file = "ray-2.56.1-cp310-cp310-win_amd64.whl", hash = "sha256:ad87fa57c5c69c77edab14820226868842648de7181894dba4fb1e874a0ab27e"}, + {file = "ray-2.56.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5eebd776cd461edebc5874d2dfba3005147b6b45645d8a16a6549311c5efffea"}, + {file = "ray-2.56.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:4df45f33cc176b11c69191efdcfb4aceaeea22fecfa784f6485b6504ef8c8b26"}, + {file = "ray-2.56.1-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:e7003a47a42ef2ad33ec0b34dc5b6afb03f63fe59465e6f4c8f6d05492d9e4a6"}, + {file = "ray-2.56.1-cp311-cp311-win_amd64.whl", hash = "sha256:c102fb09e82c2e10828d5c21cfb744e27bca690b287c7d0a77d0d883c0c86907"}, + {file = "ray-2.56.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:44bc0000c5bfad85b2ff6e0ef91e95f901d1a2d2fdd72f94f08a046eb494cd61"}, + {file = "ray-2.56.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:8fdd6b096215906cf1f9acdc7898c9d6140606f2d27245778b8385a9f19e6cb0"}, + {file = "ray-2.56.1-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:e5d3173696831134c76bd09451dfe95c32d72c271253b0d6b09d2df9994aa660"}, + {file = "ray-2.56.1-cp312-cp312-win_amd64.whl", hash = "sha256:8052573ee5ef8c4fdd7aeb6a257c80542e69c48f3f6d117101f95c970ffdc7e2"}, + {file = "ray-2.56.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:93dedab658334af81877b6ed840c4e5f85e2e0b1b3641b20eaaee37cad835cc6"}, + {file = "ray-2.56.1-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:7fdc47de4e230f0db7c6c668a9e161f864dac548bd32230986e0b0c36e386eb5"}, + {file = "ray-2.56.1-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:81f2db202cc31bc3f5c4acf9ef154d10f1f39ece602d9d2d3108875c49bf01c3"}, + {file = "ray-2.56.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:76f6268a669c1d9910f1d7b73903de3ede9850ed94d3e28318d495559bd37d0e"}, + {file = "ray-2.56.1-cp314-cp314-manylinux2014_aarch64.whl", hash = "sha256:ea372c7f95b14f1f76f0bdd2abc9602c56e88bc30699d2228f78133e7749b2f1"}, + {file = "ray-2.56.1-cp314-cp314-manylinux2014_x86_64.whl", hash = "sha256:ae91fe578fadea38c13a208a0fec27e1ced238ccdf5fff95ef3e30ac3071dec9"}, ] [package.dependencies] @@ -1843,10 +1839,10 @@ virtualenv = {version = ">=20.0.24,<20.21.1 || >20.21.1", optional = true, marke adag = ["cupy-cuda12x ; sys_platform != \"darwin\""] air = ["aiohttp (>=3.13.3)", "aiohttp_cors", "colorful", "fastapi (>=0.133.0)", "fsspec", "grpcio (>=1.42.0)", "mmh3", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "pandas", "pandas (>=2.2.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyarrow (>=17.0.0)", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "requests", "smart_open", "starlette (>=1.0.1)", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] all = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm_tree", "fastapi (>=0.133.0)", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.42.0)", "gymnasium (==1.2.2)", "lz4", "memray ; sys_platform != \"win32\"", "mmh3", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (>=1.7.0)", "pandas", "pandas (>=2.2.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=17.0.0)", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "pyyaml", "requests", "scipy", "smart_open", "starlette (>=1.0.1)", "taskiq", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] -all-cpp = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm_tree", "fastapi (>=0.133.0)", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.42.0)", "gymnasium (==1.2.2)", "lz4", "memray ; sys_platform != \"win32\"", "mmh3", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (>=1.7.0)", "pandas", "pandas (>=2.2.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=17.0.0)", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "pyyaml", "ray-cpp (==2.56.0)", "requests", "scipy", "smart_open", "starlette (>=1.0.1)", "taskiq", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] +all-cpp = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm_tree", "fastapi (>=0.133.0)", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.42.0)", "gymnasium (==1.2.2)", "lz4", "memray ; sys_platform != \"win32\"", "mmh3", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (>=1.7.0)", "pandas", "pandas (>=2.2.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=17.0.0)", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "pyyaml", "ray-cpp (==2.56.1)", "requests", "scipy", "smart_open", "starlette (>=1.0.1)", "taskiq", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] cgraph = ["cupy-cuda12x ; sys_platform != \"darwin\""] client = ["grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\""] -cpp = ["ray-cpp (==2.56.0)"] +cpp = ["ray-cpp (==2.56.1)"] data = ["fsspec", "numpy (>=1.20)", "pandas (>=2.2.3)", "pyarrow (>=17.0.0)"] default = ["aiohttp (>=3.13.3)", "aiohttp_cors", "colorful", "grpcio (>=1.42.0)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "requests", "smart_open", "virtualenv (>=20.0.24,!=20.21.1)"] llm = ["aiohttp (>=3.13.3)", "aiohttp_cors", "async-timeout ; python_version < \"3.11\"", "colorful", "fastapi (>=0.133.0)", "fsspec", "grpcio (>=1.42.0)", "hf_transfer", "jsonref (>=1.1.0)", "jsonschema", "meson", "mmh3", "ninja", "nixl (==1.1.0)", "nixl-cu13 (==1.1.0)", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "pandas (>=2.2.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyarrow (>=17.0.0)", "pybind11", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "requests", "smart_open", "starlette (>=1.0.1)", "typer", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "vllm[audio] (==0.22.0)", "watchfiles"] @@ -2236,14 +2232,14 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "virtualenv" -version = "21.6.1" +version = "21.7.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "virtualenv-21.6.1-py3-none-any.whl", hash = "sha256:afe991df855715a2b2f60edfcc0107ef95a79fdfd8cb4cdaa71603d1c12e463b"}, - {file = "virtualenv-21.6.1.tar.gz", hash = "sha256:15f978b7cd329f24855ff4a0c4b4899cc7678589f49adbdcbbb4d3232e641128"}, + {file = "virtualenv-21.7.0-py3-none-any.whl", hash = "sha256:a8370c1c5530fbabf955e40b8fbbc68a431648b10f9433faa587db30a06e51dd"}, + {file = "virtualenv-21.7.0.tar.gz", hash = "sha256:7f9519b9432ff11b6e1a3e94061664efc2ff99ea21780e3cf4f6bd0a5da8b37c"}, ] [package.dependencies] @@ -2358,116 +2354,116 @@ dev = ["pytest", "setuptools"] [[package]] name = "yarl" -version = "1.24.2" +version = "1.24.5" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, + {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, + {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, + {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, + {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, + {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, + {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, + {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, + {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, + {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, + {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, + {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, + {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, + {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, + {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, ] [package.dependencies] @@ -2478,4 +2474,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "0d340bb8bd7c6f0d2b6e1b59b8633a4a890c075680cfaa3e97d1615acff94789" +content-hash = "e8827da46bd551d6f12e3b6f66cd75e4fdb3716ef47dd8160170552c64b142d6" diff --git a/security_scanning/examples/ray_orchestrator/pyproject.toml b/security_scanning/examples/ray_orchestrator/pyproject.toml index 6724fffb5f98..86dd96fb56ab 100644 --- a/security_scanning/examples/ray_orchestrator/pyproject.toml +++ b/security_scanning/examples/ray_orchestrator/pyproject.toml @@ -7,7 +7,7 @@ authors = [ ] requires-python = ">=3.10,<3.13" dependencies = [ - "ray[default] (>=2.56.0,<3.0.0)" + "ray[default] (>=2.56.1,<3.0.0)" ] diff --git a/security_scanning/examples/serve/poetry.lock b/security_scanning/examples/serve/poetry.lock index c8a4e19b5005..46853ec19f8b 100644 --- a/security_scanning/examples/serve/poetry.lock +++ b/security_scanning/examples/serve/poetry.lock @@ -286,14 +286,14 @@ files = [ [[package]] name = "annotated-types" -version = "0.7.0" +version = "0.8.0" description = "Reusable constraint types to use with typing.Annotated" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, - {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, + {file = "annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0"}, + {file = "annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7"}, ] [[package]] @@ -495,14 +495,14 @@ cffi = {version = ">=1.0.0", markers = "python_version < \"3.13\""} [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -768,14 +768,14 @@ files = [ [[package]] name = "colorlog" -version = "6.10.1" +version = "6.12.0" description = "Add colours to the output of Python's logging module." optional = false python-versions = ">=3.6" groups = ["main"] files = [ - {file = "colorlog-6.10.1-py3-none-any.whl", hash = "sha256:2d7e8348291948af66122cff006c9f8da6255d224e7cf8e37d8de2df3bad8c9c"}, - {file = "colorlog-6.10.1.tar.gz", hash = "sha256:eb4ae5cb65fe7fec7773c2306061a8e63e02efc2c72eba9d27b0fa23c94f1321"}, + {file = "colorlog-6.12.0-py3-none-any.whl", hash = "sha256:30d392604e9110045a2c2aeefc27d7a017abbab63f3a8aee594eac0801df784e"}, + {file = "colorlog-6.12.0.tar.gz", hash = "sha256:2a7924c1dadf18b22a0eb8b06d1c7b01d5341707ec1641eb6fcc4fde0c3e8e5f"}, ] [package.dependencies] @@ -1012,14 +1012,14 @@ tests = ["pytest", "pytest-cov", "pytest-xdist"] [[package]] name = "cyclopts" -version = "4.21.0" +version = "4.22.2" description = "Intuitive, easy CLIs based on type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cyclopts-4.21.0-py3-none-any.whl", hash = "sha256:ded3ddb15b0c815f44d245011fc4cdd5a4809a3bb8202869e9e02195a87c0e18"}, - {file = "cyclopts-4.21.0.tar.gz", hash = "sha256:477c18c791c924cca4836f79fce000a7bae45f551e340d9e1654e102c6d9ab9d"}, + {file = "cyclopts-4.22.2-py3-none-any.whl", hash = "sha256:9c2cdf6a621886cd0af631a67437eb7d0084f33f9e8fba2d2562a1aecf75f2ff"}, + {file = "cyclopts-4.22.2.tar.gz", hash = "sha256:0721e90e7209885e78f7637cfba255c12e89206b633e35d185305e349ba20ecd"}, ] [package.dependencies] @@ -1189,14 +1189,14 @@ test = ["pytest (>=6)"] [[package]] name = "fastapi" -version = "0.139.0" +version = "0.140.7" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "fastapi-0.139.0-py3-none-any.whl", hash = "sha256:cf15e1e9e667ddb0ad63811e60bd11390d1aac838ca4a7a23f421807b2308189"}, - {file = "fastapi-0.139.0.tar.gz", hash = "sha256:99ab7b2d92223c76d6cf10757ab3f89d45b38267fc20b2a136cf02f6beac3145"}, + {file = "fastapi-0.140.7-py3-none-any.whl", hash = "sha256:960bb9696d8fd19dff488aa4f67f276364542cfcce9f7e68a82fe49dce126626"}, + {file = "fastapi-0.140.7.tar.gz", hash = "sha256:09a640af2d29006345e1f28e4f031fa60f89b1a75d29f26070f3afa677d66cce"}, ] [package.dependencies] @@ -1231,14 +1231,14 @@ dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4 [[package]] name = "filelock" -version = "3.29.7" +version = "3.32.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, + {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, + {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, ] [[package]] @@ -1535,92 +1535,92 @@ files = [ [[package]] name = "greenlet" -version = "3.5.3" +version = "3.5.4" description = "Lightweight in-process concurrent programming" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\"" files = [ - {file = "greenlet-3.5.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c180d22d325fb613956b443c3c6f4406eb70e6defc70d3974da2a7b59e06f48c"}, - {file = "greenlet-3.5.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:483d08c11181c83a6ce1a7a61df0f624a208ec40817a3bb2302714592eee4f04"}, - {file = "greenlet-3.5.3-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1dae6e0091eae084317e411f047f0b7cb241c6db570f7c45fd6b900a274914ce"}, - {file = "greenlet-3.5.3-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0f6ff50ff8dbd51fae9b37f4101648b04ea0df19b3f50ab2beb5061e7716a5c8"}, - {file = "greenlet-3.5.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bcd2d72ccd70a1ec68ba6ef93e7fbb4420ef9997dabc7010d893bd4015e0bec"}, - {file = "greenlet-3.5.3-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:37bf9c538f5ae6e63d643f88dec37c0c83bdf0e2ebc62961dedcf458822f7b71"}, - {file = "greenlet-3.5.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:73f152c895e09907e0dbe24f6c2db37beb085cd63db91c3825a0fcd0064124a8"}, - {file = "greenlet-3.5.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8bdb43e1a1d1873721acab2be99c5befd4d2044ddfd52e4d610801019880a702"}, - {file = "greenlet-3.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:0909f9355a9f24845d3299f3112e266a06afb68302041989fd26bd68894933db"}, - {file = "greenlet-3.5.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:aca9b4ce85b152b5524ef7d88170efdff80dc0032aa8b75f9aaf7f3479ea95b4"}, - {file = "greenlet-3.5.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f71be4920368fe1fabeeaa53d1e3548337e2b223d9565f8ad5e392a75ba23fc"}, - {file = "greenlet-3.5.3-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d77e67f65f98449e3fb83f795b5d0a8437aead2f874ca89c96576caf4be3af6"}, - {file = "greenlet-3.5.3-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e18619ba655ac05d78d80fc83cac4ba892bd6927b99e3b8237aee861aaacc8bb"}, - {file = "greenlet-3.5.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8540f1e6205bd13ca0ce685581037219ca54a1b41a0a15d228c6c9b8ad5903d7"}, - {file = "greenlet-3.5.3-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:d27c0c653a60d9535f690226474a5cc1036a8b0d7b57504d1c4f89c44a07a80c"}, - {file = "greenlet-3.5.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7ef56fe650f50575bf843acde967b9c567687f3c22340941a899b7bc56e956a8"}, - {file = "greenlet-3.5.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5121af01cf911e70056c00d4b46d5e9b5d1415550038573d744138bacb59e6b8"}, - {file = "greenlet-3.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:0f41e4a05a3c0cb31b17023eff28dd111e1d16bf7d7d00406cd7df23f31398a7"}, - {file = "greenlet-3.5.3-cp311-cp311-win_arm64.whl", hash = "sha256:ec6f1af59f6b5f3fc9678e2ea062d8377d22ac644f7844cb7a292910cf12ff44"}, - {file = "greenlet-3.5.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:719757059f5a53fd0dde23f78cffeafcdd97b21c850ddb7ca684a3c1a1f122e2"}, - {file = "greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b"}, - {file = "greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab"}, - {file = "greenlet-3.5.3-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5795cd1101371140551c645f2d408b8d3c01a5a29cf8a9bce6e759c983682d23"}, - {file = "greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861"}, - {file = "greenlet-3.5.3-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:af4923b3096e26a36d7e9cf24ab88083a20f97d191e3b97f253731ce9b41b28c"}, - {file = "greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149"}, - {file = "greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea"}, - {file = "greenlet-3.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:cde8adafa2365676f74a979744629589999093bc86e2484214f58e61df08902c"}, - {file = "greenlet-3.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:c4e7b79d83805475f0102008843f6eb45fd3bb0b2e88c774adab5fbaab27117d"}, - {file = "greenlet-3.5.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c8d87c2134d871df96ecdea9cec7cbaab286dadab0f56476e57aaf9e8ac11550"}, - {file = "greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5"}, - {file = "greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3"}, - {file = "greenlet-3.5.3-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9ad04dd75458c6300b047c61b8639092433d205a25a14e310d6582a480efcca1"}, - {file = "greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a"}, - {file = "greenlet-3.5.3-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:afaabdd554cd7ae9bbb3ca070b0d7fdfd207dbf1d16865f7233837709d354bda"}, - {file = "greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb"}, - {file = "greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b"}, - {file = "greenlet-3.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:c82304750f057167ff60d188df1d0cc1764ce9567eadf03e6a7443bcedd0b30b"}, - {file = "greenlet-3.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:dc133a1569ee667b2a6ef56ce551084aeefd87a5acbc4736d336d1e2edc6cfc4"}, - {file = "greenlet-3.5.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:fd2e02fa07485778536a036222d616ab957b1d533f36b3ed98ce725d9c9d3117"}, - {file = "greenlet-3.5.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0a0628d1597eb0897b62f55d1343f772405fd25f3b2a796c76874b0c2e22e8"}, - {file = "greenlet-3.5.3-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ebd933a6adabc298bab47731a130fe6bfb888bd934eee37810f151159544540d"}, - {file = "greenlet-3.5.3-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8d19fe6c39ebff9259f07bcc685d3290f8fa4ea2278e51dd0008e4d6b0f2d814"}, - {file = "greenlet-3.5.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b9d501b40e80b70e32323c799dd9b420a5577a9601469d362ae1ffb690f3a7c"}, - {file = "greenlet-3.5.3-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:962c5df2db8cb446da51edf1ca5296c389d93b99c9d8aa2ee4c7d0d8f1218260"}, - {file = "greenlet-3.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a1fad1d11e7d6aab184107baa8e4ece11ccba3ec9599cd7efa5ff4d70d43256a"}, - {file = "greenlet-3.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fad5aec764399f1b5cc347ad250a59660f20c8f8888ea6bae1f93b769cce1154"}, - {file = "greenlet-3.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:7669aa24cf2a1041d6f7899575b494a3ab4cf68bfcc8609b1dc0be7272db835e"}, - {file = "greenlet-3.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:5b4807c4082c9d1b6d9eed56fcd041863e37f2228106eef24c30ca096e238605"}, - {file = "greenlet-3.5.3-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:271a8ea7c1024e8a0d7dd2be66dd66dda8a07193f41a17b9e924f7600f5b62be"}, - {file = "greenlet-3.5.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19131729ae0ddc3c2e1ef85e650169b5e37ee32e400f215f78b94d7b0d567310"}, - {file = "greenlet-3.5.3-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1540dd8e5fc2a5aec40fbb98ef8e149fa47c89a4b4a1cf2575a14d3d1869d7a8"}, - {file = "greenlet-3.5.3-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b897d97759425953f69a9c0fac67f8fe333ec0ce7377ef186fb2b0c3ad5e354d"}, - {file = "greenlet-3.5.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e81fa194a1d20967877bdf9c7794db2bc99063e5be36aee710c08f04c5bb087f"}, - {file = "greenlet-3.5.3-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:3236754d423955ea08e9bb5f6c04a7895f9e22c290b66aa7653fcb922d839eb0"}, - {file = "greenlet-3.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55cf4d777485d43110e47133cbba6d74a8885a87ec1227ef0267f9ee80c5aa21"}, - {file = "greenlet-3.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:12a248ba75f6a9a236375f52296c498c89ff1d8badf32deb9eca7abd5853f7da"}, - {file = "greenlet-3.5.3-cp314-cp314t-win_amd64.whl", hash = "sha256:efc6bd60ea02e085862c74a3ef64b147ffc6f1a5ea7d9f26e7a939943f68c1e3"}, - {file = "greenlet-3.5.3-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:ea03f2f04367845d6b58eeed276e1e56e51f0b97d8ad5a88a7d20a91dc9056cc"}, - {file = "greenlet-3.5.3-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78dbef602fda6d97d957eb7937f70c9ce9e9527330347f8f6b6f9e554a9e7a47"}, - {file = "greenlet-3.5.3-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f73857adb8fee13fa56c172bd11262f888c0c648f9fea113e777bb2c7904a81"}, - {file = "greenlet-3.5.3-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cefa9cef4b371f9844c6053db71f1138bc6807bab1578b0dae5149c1f1141357"}, - {file = "greenlet-3.5.3-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:232fec92e823addaf02d9472cf7381e24a1d046a6ced1103c5caa4c21b9dfc1d"}, - {file = "greenlet-3.5.3-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:6219b6d04dbf6ba6084d77dc609e8473060dc55f759cbf626d512122781fa128"}, - {file = "greenlet-3.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2421c3564da9429d5586d46ca31ebb26516b5498a802cf65c041a8e8a8980d34"}, - {file = "greenlet-3.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e0f0d160f0b2e558e6c75f7930967183255dc9735e5f5b8cae58ee09c9576d8b"}, - {file = "greenlet-3.5.3-cp315-cp315-win_amd64.whl", hash = "sha256:dd99329bbc15ca78dcc583dba05d0b1b0bae01ab6c2174989f5aaee3e41ac930"}, - {file = "greenlet-3.5.3-cp315-cp315-win_arm64.whl", hash = "sha256:499fef2acede88c1864a57bb586b4bf533c81e1b82df7ab93451cdb47dfec227"}, - {file = "greenlet-3.5.3-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:176bc16a721fa5fc294d70b87b4dfa5fbdd251b3da5d5372735ecef9bd7d6d0c"}, - {file = "greenlet-3.5.3-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:629b614d2b786e89c50440e246f33eea78f58a962d0bdbbcc809e6d13605903f"}, - {file = "greenlet-3.5.3-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b2e857ae16f5f72142edf75f9f176fe7526ba19a2841df1420516f83831c9f2"}, - {file = "greenlet-3.5.3-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d192579ed281051396dddd7f7754dac6259e6b1fb26378c87b66622f8e3f91"}, - {file = "greenlet-3.5.3-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e515757e2e36bcbf1fad09a46e1557e8b1ae1797d4b44d09da7deed88ad28608"}, - {file = "greenlet-3.5.3-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:4399eb8d041f20b68d943918bc55502a93d6fdc0a37c14da7881c04139acee9d"}, - {file = "greenlet-3.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b363d46ed1ea431825fdb01471bb024fc08399bad1572a616e853c7684415adb"}, - {file = "greenlet-3.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e44da2f5bbdaabaf7d80b73dbb430c7035771e9f244e3c8b769715c9d8fa0a16"}, - {file = "greenlet-3.5.3-cp315-cp315t-win_amd64.whl", hash = "sha256:8ff8bed3e3baa20a3ea261ce00526f1898ad4801d4886fd2220580ee0ad8fadf"}, - {file = "greenlet-3.5.3-cp315-cp315t-win_arm64.whl", hash = "sha256:b7068bd09f761f3f5b4d214c2bed063186b2a86148c740b3873e3f56d79bac31"}, - {file = "greenlet-3.5.3.tar.gz", hash = "sha256:a61efc018fd3eb317eeca31aba90ee9e7f26f22884a79b6c6ec715bf71bb62f1"}, + {file = "greenlet-3.5.4-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:ac5bf81d79d2c8eeb2ef6359b2e1687a1e9ebf46c2b1f970da9a9255df51d190"}, + {file = "greenlet-3.5.4-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89f3738167bab8c1084b94e23023d41d247117ac149fa0fbcb5bd4cf6262b353"}, + {file = "greenlet-3.5.4-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e9a5e3406e3ed8125ae1a3b37c12f3434e2b1f0fa053197c5557895b4fb09606"}, + {file = "greenlet-3.5.4-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a2d614cb2372c7101a12ea8b96dd56f81c986d247c5a73db67063f3ed1ca4a52"}, + {file = "greenlet-3.5.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ab9f0704bccf6d3b38e0d2130b7b33271cff11453690da074fa280c3aa8e8e7"}, + {file = "greenlet-3.5.4-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:188e4d142f243051d92a1f5c244a741da02dddc070a0620c842804d7b56d008c"}, + {file = "greenlet-3.5.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2cdaadc3d31445a8f782bde3cd37e49a2c2a9c6da6daf76a3e34c683b271a3c7"}, + {file = "greenlet-3.5.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:70bdfacdc183dac838b2a0aaff2dd6134a457c52fe68a9c6bbab435483d2b9df"}, + {file = "greenlet-3.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:69173331fbc5d64bfac0065d7e22c39cfcd089e9b18d125bdcd5079363b09616"}, + {file = "greenlet-3.5.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e883de250e299654b1f1680f72a1a9f9ba62c9bd1bce84099c90657349a8dfbb"}, + {file = "greenlet-3.5.4-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32802705c2c1ff25e8237b3bdacf2594fa02be80af8a66703eb7853ea7e68686"}, + {file = "greenlet-3.5.4-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:57aa201b351f7c7c75627c60d29e4d5b97a07d37efeb62b903466fca42c097d7"}, + {file = "greenlet-3.5.4-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9667862a2e38ad379f11b845daeda22c8989186def44f06962c9c4c05e556da7"}, + {file = "greenlet-3.5.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3fe76c2cac86b4f7a1e92865ac0a54384deb05c92986287c1a7110d9bd53071"}, + {file = "greenlet-3.5.4-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:ae53534b5dec0f4c2ec26f898f538dc8ea1ca3ef2927d597a9439e40a09da937"}, + {file = "greenlet-3.5.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1e1a4a684b16c45ba324e60b32a4386a87722bcb815d2a149d2182f9b401ca72"}, + {file = "greenlet-3.5.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e849e6e139b9671adeac505f72fc05f4af7fd1921faef40295e214fc3b361b59"}, + {file = "greenlet-3.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc418cf4c873357964d6624445ed09472e50def990c65dd4e76fc3ba8cd9cef6"}, + {file = "greenlet-3.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:c38c902a0986eba1f6e7ba1ab39ad5195926abde90f3fe080e08212db62176da"}, + {file = "greenlet-3.5.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3d66250e8b09f182ede05490998c818b5961f7a3640332d44c4927caec7bbfe4"}, + {file = "greenlet-3.5.4-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c90e930c9c192e5b3ee9fb8bcd920ea3926155e2e3ded39fc697323addecee17"}, + {file = "greenlet-3.5.4-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:791fdfeeb9c6e0c7b10fa151bf110d2a6974866f13dcb5b1c7efae698245893a"}, + {file = "greenlet-3.5.4-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b7c895310363f310361e0fe2072af85269d2a2a285cd04c0c59e79a5e3670dcf"}, + {file = "greenlet-3.5.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f88193799d43dbf8c8a806d6405c9c52fe2af40bf75072a606357b33cc336c7f"}, + {file = "greenlet-3.5.4-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:13b980043cb1b3134e81ea469da1250ddcc6bfe6d245bbaa59168d9cdc8f228f"}, + {file = "greenlet-3.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7a5f095767c4493afcd06067f2bb3b8716e3f3f9e92b99c88e7e99f885b3d4d"}, + {file = "greenlet-3.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:42afdc1ab5f66da8c586c32af9224a74a706b4f0ea0dc3a4188a0860a09c65c9"}, + {file = "greenlet-3.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:60149df8f462d1b230038e6590c23c3b4768bb5d6c022b3b6e82532b34b0b8a3"}, + {file = "greenlet-3.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:77d6ce04fed0d9aeed42e0f37923cc43eba9b027bdd9c34546bb4ccd143d0fe0"}, + {file = "greenlet-3.5.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02"}, + {file = "greenlet-3.5.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356"}, + {file = "greenlet-3.5.4-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef"}, + {file = "greenlet-3.5.4-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c"}, + {file = "greenlet-3.5.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0"}, + {file = "greenlet-3.5.4-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861"}, + {file = "greenlet-3.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd"}, + {file = "greenlet-3.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f"}, + {file = "greenlet-3.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c"}, + {file = "greenlet-3.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f"}, + {file = "greenlet-3.5.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22"}, + {file = "greenlet-3.5.4-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf"}, + {file = "greenlet-3.5.4-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9"}, + {file = "greenlet-3.5.4-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c"}, + {file = "greenlet-3.5.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8"}, + {file = "greenlet-3.5.4-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c"}, + {file = "greenlet-3.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3"}, + {file = "greenlet-3.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec"}, + {file = "greenlet-3.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c"}, + {file = "greenlet-3.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f"}, + {file = "greenlet-3.5.4-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3"}, + {file = "greenlet-3.5.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867"}, + {file = "greenlet-3.5.4-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c"}, + {file = "greenlet-3.5.4-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8"}, + {file = "greenlet-3.5.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66"}, + {file = "greenlet-3.5.4-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd"}, + {file = "greenlet-3.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7"}, + {file = "greenlet-3.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e"}, + {file = "greenlet-3.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132"}, + {file = "greenlet-3.5.4-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809"}, + {file = "greenlet-3.5.4-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927"}, + {file = "greenlet-3.5.4-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5"}, + {file = "greenlet-3.5.4-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3"}, + {file = "greenlet-3.5.4-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0"}, + {file = "greenlet-3.5.4-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb"}, + {file = "greenlet-3.5.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88"}, + {file = "greenlet-3.5.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c"}, + {file = "greenlet-3.5.4-cp315-cp315-win_amd64.whl", hash = "sha256:ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da"}, + {file = "greenlet-3.5.4-cp315-cp315-win_arm64.whl", hash = "sha256:f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667"}, + {file = "greenlet-3.5.4-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c"}, + {file = "greenlet-3.5.4-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9"}, + {file = "greenlet-3.5.4-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25"}, + {file = "greenlet-3.5.4-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d"}, + {file = "greenlet-3.5.4-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde"}, + {file = "greenlet-3.5.4-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05"}, + {file = "greenlet-3.5.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8"}, + {file = "greenlet-3.5.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2"}, + {file = "greenlet-3.5.4-cp315-cp315t-win_amd64.whl", hash = "sha256:f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2"}, + {file = "greenlet-3.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994"}, + {file = "greenlet-3.5.4.tar.gz", hash = "sha256:0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20"}, ] [package.extras] @@ -1641,38 +1641,30 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.1" +version = "1.5.2" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, + {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, ] [package.extras] @@ -1787,14 +1779,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.23.0" +version = "1.25.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, + {file = "huggingface_hub-1.25.1-py3-none-any.whl", hash = "sha256:004d4e70350517e24c68a7dbb7dc5e40b2b6aefef8f94bf7a85f6f9835102ea5"}, + {file = "huggingface_hub-1.25.1.tar.gz", hash = "sha256:21129595ca7a753be479b319913e22cc8808361ac118bd76cc413db831b28a99"}, ] [package.dependencies] @@ -2320,65 +2312,65 @@ dev = ["meson-python (>=0.13.1,<0.17.0)", "pybind11 (>=2.13.2,!=2.13.3)", "setup [[package]] name = "matplotlib" -version = "3.11.0" +version = "3.11.1" description = "Python plotting package" optional = false python-versions = ">=3.11" groups = ["main"] markers = "python_version >= \"3.11\"" files = [ - {file = "matplotlib-3.11.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f857524b442f0f36e641868ce2171aafa88cb0bc0644f4e1d8a5df9b32649fef"}, - {file = "matplotlib-3.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:57baa92fdc82948ed716eae6d2579d4d6f40965cd8d2f416755b4a72580a3233"}, - {file = "matplotlib-3.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630eee0e67d35cce2019a0e670719f4816e3b86aff0fa72729f6c69786fceb45"}, - {file = "matplotlib-3.11.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5106c444d0bf966eee2853548c03772af4ab7199118e086c62fbac8ccb07c055"}, - {file = "matplotlib-3.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4d7aea652b58e686444079be3376ef546bffa1eee9b9bb9c472b9fcf6cf410d3"}, - {file = "matplotlib-3.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:70a5b3e9a5dab708c0f039709ae7c68d5b4d254e291ef76492cdba230c8bb5e4"}, - {file = "matplotlib-3.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:3d68266213e73823ac3be90615bab0cf31f88851e114cdb1dd25dacf3b01e1a7"}, - {file = "matplotlib-3.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:06b5872e9cf11adc8f589ded3ce11bc3e1061ad498259664fabc1f6615beb918"}, - {file = "matplotlib-3.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0515d495124be3124340e59f164d901ed4484e2246a5b74cfa483cac3b80bd97"}, - {file = "matplotlib-3.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be5f93a1d21981bfb802ded0d77a0caa92d4342a47d45754fac77e314a506344"}, - {file = "matplotlib-3.11.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41635d7909d19e52e924a521dde6d8f670b0f53ab1d0e8c331fa831554f681d1"}, - {file = "matplotlib-3.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:94f5000f67ca9faa300863ea17f8bce9175cb67b88bec4bc7780502d53dd7c9e"}, - {file = "matplotlib-3.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:ac6f1ef39f3d0f9e2463303013094992cdbe0f85f43bc54155bc472b2042768e"}, - {file = "matplotlib-3.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:9dd11fb612ce7bc60b1de5b4fc87ff959d22317b5de42aabf392f66f97af22eb"}, - {file = "matplotlib-3.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6ce3b839b34ae1f430b4616893a2945a2999debaa7e94e7e29a2a8bbf286f7b5"}, - {file = "matplotlib-3.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:373db8f91214e8ccaf35ac833cc1dd59dd961e148bbd55dd027141591dde1313"}, - {file = "matplotlib-3.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be152b7570324dc8d01574cc9474dd2d803237acf528bcbb5b211fa347461a09"}, - {file = "matplotlib-3.11.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:126f256df600652d7e4b394cf3164ff75210a00038f287c95a012a6f58d0e83f"}, - {file = "matplotlib-3.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:03acfeddf87b0dddb11b081ef7740ad445a3ca8bcb6b8e3011b08f2cf802b75c"}, - {file = "matplotlib-3.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:ab3722f04f3ff34c23b5012c5873d2894174e06c3822fcdac3610965a5ac7d06"}, - {file = "matplotlib-3.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:c945824670fb8915b4ac879e5e61f3c58e0913022f70a0de4c082b17372f8771"}, - {file = "matplotlib-3.11.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3489c3dc487669b4a980bc3068f87856de7a1564248d3f6c629efb2a58b03f24"}, - {file = "matplotlib-3.11.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6a98f5476ce784a50ce09998f4ae1e6a9f25043cef8a480c98949902eda74620"}, - {file = "matplotlib-3.11.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:565af866fd63e4bd3f987d580afe27c44c2552a3b3305f4ecbb85133601ea6f3"}, - {file = "matplotlib-3.11.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b3e64dea5062c570f04358e2711859f3531b459f29516274fbad889079e4f3"}, - {file = "matplotlib-3.11.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:942b37c5db1899610bd1543ce8e13e4ecff9a4633e7f63bb6aa9205d2644ebd1"}, - {file = "matplotlib-3.11.0-cp313-cp313t-win_amd64.whl", hash = "sha256:c08e649a6313e1291e713623b97a38e5bb4aa580b2a100a94a3309bc6b9c8eb3"}, - {file = "matplotlib-3.11.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2746cd2c113742ff6ce37a864c5ac5fd7aa644568f445e66166e457ac78e40e0"}, - {file = "matplotlib-3.11.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3338e3e3de128cf50d0d2fb92a122815daf9c755bd882a474343c05f8fd7ec79"}, - {file = "matplotlib-3.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:25c2e5455efd8d99f41fb79871a31feb7d301569642e332ec58d72cfe9282bc3"}, - {file = "matplotlib-3.11.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9695457a467ff86d23f35037a43deb6f1134dd6d3e2ac8ce1e2087cff09ffb9"}, - {file = "matplotlib-3.11.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19c16c61dea63b3582918503e6b294193961261d9daa806d4ae2151f1ad05430"}, - {file = "matplotlib-3.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2d72ea8b7924f3cb955e61518d21e43b3df1e6c8a793b480a0c1214f185d30ba"}, - {file = "matplotlib-3.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:1c02da0a629dfa9debf52725ea06866b74c1fb70a895bae05e4493d34074f9f2"}, - {file = "matplotlib-3.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:aa55d73b3117d4b07f959cd9eb6f69b375d8df3414139c479388e551aa5d999d"}, - {file = "matplotlib-3.11.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a9d8c6e7cd2f0ddf11d8d92e520dd1d9d2abb0cf6ac8831e338666c81e905847"}, - {file = "matplotlib-3.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:be050fcf32f729eda99f7f75a80bf67612ce16ab9ac1c23a387dcaede95cb70e"}, - {file = "matplotlib-3.11.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfabef0230d0697aa0d717385194dd41162e00207a68bf4abf94c2bf4c27dca0"}, - {file = "matplotlib-3.11.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1644db30e759199443493ac5e5caec24fdb775a8f6123021f85ba47c4133c3cb"}, - {file = "matplotlib-3.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:15b0d160079cb10699a0e98b5989c70677b2df7cacdc62af67c30f2facec46d9"}, - {file = "matplotlib-3.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:446307e6b04b57b1f1239e228a1ec2af0d589a1008cebc3dfa3f5441d095cfb6"}, - {file = "matplotlib-3.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:652fb5696271d4c50f196d22a5ff4f8e4444c74f847423570d7dc0aa2bbd0159"}, - {file = "matplotlib-3.11.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:81ae77077a1e16d37a5b61096ccb07c8d90a99b518fa8256b8f21578932f2f62"}, - {file = "matplotlib-3.11.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ddef37840695f5eef65f9f070fe2d2f510f584c2156203f9f622a5b0584efffd"}, - {file = "matplotlib-3.11.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cf662e5ac5707658cb931e19972c4bd99f7b4f8b7bf79d3c821d239fa6b71e64"}, - {file = "matplotlib-3.11.0.tar.gz", hash = "sha256:68c0c7be01b30dcca3638934f7f591df73401235cbdbf0d1ab1c71e7db7f8b57"}, + {file = "matplotlib-3.11.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b7cf158e7add54a8d51ac9b5a84abd6d4e13ed4951b4f25f1c5139f41c2addb2"}, + {file = "matplotlib-3.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d2ace7273b9a5061a3b420918a16fae1f2dc5dfee1abcc13aba71b5d94b1820c"}, + {file = "matplotlib-3.11.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aee55e9041211bf84302ab55ec3965df18dd90ae19f8b58332a7feaf208bfe83"}, + {file = "matplotlib-3.11.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f4bdeea33a8d15a071dbfe6d119451b1d719c733ac666d65357082901a9099"}, + {file = "matplotlib-3.11.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b4c78ceb2f11bcac7389d305cda17aeb1f4586a857854ab5780bd3dd8dbfc407"}, + {file = "matplotlib-3.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:7f33a781e12b1e53b278deb2f5373c2e55ec4f10727be3440c0cfb5cda9f944f"}, + {file = "matplotlib-3.11.1-cp311-cp311-win_arm64.whl", hash = "sha256:67e4c3cd578c65ebd81bdc09a1b6592ceafee6dfafe116dc85dfcb647b5bbb18"}, + {file = "matplotlib-3.11.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e15ef41507f3d525f46154ac9e3ae785dacde9f20e593a25de8986267892ef74"}, + {file = "matplotlib-3.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21a67b961a6d597bca54fae826cd20695ba4a6e4d05424a08da6e13e3176fd6b"}, + {file = "matplotlib-3.11.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba8f811b8ddfac493734d6af0b2dff96919d0c28ca0d641858dab4262777c6ea"}, + {file = "matplotlib-3.11.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c52f7ad20ef476806ed212380b1d54d20310c8b86bdc2c9a68b51f0024a44472"}, + {file = "matplotlib-3.11.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b14eb22961fe865efb0e4ff167e333e428908b00115a8d800ccb65ee108e481"}, + {file = "matplotlib-3.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:88a2a27dd9691ae448dfae4b26f59036be90c3c28757edd3553a29559d00859f"}, + {file = "matplotlib-3.11.1-cp312-cp312-win_arm64.whl", hash = "sha256:480194afceca4df2f137c2721227d3cba67121fbf4397b69cee7f83714b0a58a"}, + {file = "matplotlib-3.11.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6771b0cd7838c6a857a7209814158c0ad09bfef878db3033dd82d70ad101f191"}, + {file = "matplotlib-3.11.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2abdee5ffa2fe11b2d19f7a5c63b785fb7c28cc46c7bc1814156341d9d1a33e1"}, + {file = "matplotlib-3.11.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0a19dcf73406d3746d25a5ed42d713604c9a3e024d129b102852b0d941cb9f3"}, + {file = "matplotlib-3.11.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7389b77ed2ab0552f46d9a90b81b7b8e6dfcdc42adc36c37a0865799843e0e3e"}, + {file = "matplotlib-3.11.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c90be0b73568da4f662afac580956a76e308437e641b4a45aa08925eeb67d95f"}, + {file = "matplotlib-3.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:68408341f2312836fbbdf6b3c78047f65b2d8752f5fd221c3e72d348f5b34f8b"}, + {file = "matplotlib-3.11.1-cp313-cp313-win_arm64.whl", hash = "sha256:0c1f44890d435c1b4ef52f701ad5828cb450ea97bcc83918fda6be74965d6cd2"}, + {file = "matplotlib-3.11.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:5e510088c27a89d53580a752f959146893563e63c330e161d159b0fee652af6f"}, + {file = "matplotlib-3.11.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:1524e2bdd48a93557aa47ddcfe9c225dfdd57d5a01a5c49128c20f0632980ee1"}, + {file = "matplotlib-3.11.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:11664c551345553db92e61cae6cf1376f138f8c47cafdf13b64b18f3e3e9e464"}, + {file = "matplotlib-3.11.1-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e1f8922ba31959cf6a9dfb51be64b7f7bc582801a3957dc0c2f3afcd3537adf"}, + {file = "matplotlib-3.11.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83235693abde86e5e0129998f80ee39fc7f58e6d56a88fafb28a9278833e9d5f"}, + {file = "matplotlib-3.11.1-cp313-cp313t-win_amd64.whl", hash = "sha256:9a076f4fc5cdc43fdf510f5981418d25c2db4973418d9f22d8bb3dc8045ada78"}, + {file = "matplotlib-3.11.1-cp313-cp313t-win_arm64.whl", hash = "sha256:216fbb93a74add02ddb4cb38ef5348f59ac00b3e84567eaf16598772d40e150a"}, + {file = "matplotlib-3.11.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:30c492d4ba9448595b6fd8708c6725963f8148e25c0d8842948da5b05f0ee8d3"}, + {file = "matplotlib-3.11.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ac104be2768ffdd8655db9e71b768cbb45f2b9aa7b450cf1595e8f65d3822319"}, + {file = "matplotlib-3.11.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6be943cb68bc6660ead58c55b3aa6366cba2ef7feb06460fbcce32360376f19f"}, + {file = "matplotlib-3.11.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5af0dcda57d471440a7b5b623e70e0a61003518443d9098f211a96ecfbbc25be"}, + {file = "matplotlib-3.11.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3d3fd84082b1afbd9398466c81309e20045be20d48fe0fb18c43504d164cbbb2"}, + {file = "matplotlib-3.11.1-cp314-cp314-win_amd64.whl", hash = "sha256:9601a1e90be21e4884c53b4f3dc3ee0544654946f9975258d691f1c2e2f119c6"}, + {file = "matplotlib-3.11.1-cp314-cp314-win_arm64.whl", hash = "sha256:ae30c6109848ac0f9fa36c5d6270938487614c47ba31860bd5361266dabc5685"}, + {file = "matplotlib-3.11.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dadfe80797174e2984aae3be0b77594a3c72d2c0a40fbd4a0de48d2728caf3ae"}, + {file = "matplotlib-3.11.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:89b193b255f4f6f7948dbcee3691f4f341ab05d9a8874a67b45ddb4182922eda"}, + {file = "matplotlib-3.11.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:191163532cdefcb1571ca38a6d7e6474baccde64495783e6ba47aa07ec4b9bbb"}, + {file = "matplotlib-3.11.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fdf1c818ab05d0e74002091ddaf414478a3a449ec9d51c8976d45be7e3a01e2"}, + {file = "matplotlib-3.11.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b937b9dba5f5f6c1e31c47abe2186c865c0914fd18f2ce0dfc39c9adcef5951d"}, + {file = "matplotlib-3.11.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f2912f647f3fbe1ccf085f91e213936f9101bead81a5e670565b1f1b3712f4fb"}, + {file = "matplotlib-3.11.1-cp314-cp314t-win_arm64.whl", hash = "sha256:54d47b8ae8b579633a3902ca5b4ad6c1e132a5626d64447b2e22a66394e79987"}, + {file = "matplotlib-3.11.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:427258425f9a3fc4ed79a91f9e9b9aaf5a82cb6571e85dc14063cc6fbb993741"}, + {file = "matplotlib-3.11.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:1ac697e591c11b6ad04679a73c2d2f9980fe9d9f0311fb414a2e329706343dfb"}, + {file = "matplotlib-3.11.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4b9ac2f1f607ecda2af90a5232beee2af7582fce1cc30c4b6a1b012dc21ee99"}, + {file = "matplotlib-3.11.1.tar.gz", hash = "sha256:69647db5746941c793d6e445a4cd349323ffb87d9cc958c2ad84a659b4832d30"}, ] [package.dependencies] contourpy = ">=1.0.1" cycler = ">=0.10" -fonttools = ">=4.22.0" +fonttools = ">=4.28.2" kiwisolver = ">=1.3.1" numpy = ">=1.25" packaging = ">=20.0" @@ -3255,14 +3247,14 @@ xmp = ["defusedxml"] [[package]] name = "platformdirs" -version = "4.10.0" +version = "4.11.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, - {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, + {file = "platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74"}, + {file = "platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0"}, ] [[package]] @@ -3839,14 +3831,14 @@ cli = ["click (>=5.0)"] [[package]] name = "pytz" -version = "2026.2" +version = "2026.3.post1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" groups = ["main"] files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, + {file = "pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815"}, + {file = "pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d"}, ] [[package]] @@ -4040,126 +4032,126 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} [[package]] name = "regex" -version = "2026.7.10" +version = "2026.7.19" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, + {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, + {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, + {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, + {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, + {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, + {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, + {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, + {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, + {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, + {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, + {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, + {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, + {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, + {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, + {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, + {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, + {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, + {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, + {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, + {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, + {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, + {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, ] [[package]] @@ -5069,14 +5061,14 @@ full = ["httpx (>=0.27.0,<0.29.0)", "httpx2 (>=2.0.0)", "itsdangerous", "jinja2" [[package]] name = "starlette-compress" -version = "1.7.1" +version = "1.8.0" description = "Compression middleware for Starlette - supporting ZStd, Brotli, and GZip" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "starlette_compress-1.7.1-py3-none-any.whl", hash = "sha256:cd229d64f93789f90137bc08391ca946639812f514c9f5db72ef232687753cea"}, - {file = "starlette_compress-1.7.1.tar.gz", hash = "sha256:f4df7aa6b0029ec5c4ae960040cd5d375563a4d3f7fc134bd108ebc0ed61536c"}, + {file = "starlette_compress-1.8.0-py3-none-any.whl", hash = "sha256:3c991e21d31c7f845a004055db9ae3c7e79ed59cac34c8444e9e7e689456b623"}, + {file = "starlette_compress-1.8.0.tar.gz", hash = "sha256:4dc0077468a2276b6dfd1b4f946fb61b5465a41c996c010486248e65af15b4b6"}, ] [package.dependencies] @@ -5283,14 +5275,14 @@ files = [ [[package]] name = "tqdm" -version = "4.68.4" +version = "4.70.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, + {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, + {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, ] [package.dependencies] @@ -5304,14 +5296,14 @@ telegram = ["envwrap", "requests"] [[package]] name = "transformers" -version = "5.14.0" +version = "5.14.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.0-py3-none-any.whl", hash = "sha256:288287da4fdcdae0b611be32b5be20d4bcbf5caaa5f269292927fb1ff888d19d"}, - {file = "transformers-5.14.0.tar.gz", hash = "sha256:9e1a2b345c195d216afd10082394b915a82e339fa09e9013c32094ff835ca9db"}, + {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, + {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, ] [package.dependencies] @@ -5659,121 +5651,121 @@ anyio = ">=3.0.0" [[package]] name = "websockets" -version = "16.1" +version = "16.1.1" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "websockets-16.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de72a9c611178b15557d98eabd3101c9663c4d68938510478a6d162f99afd213"}, - {file = "websockets-16.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:37b0e4d726ffea3776670092d3d13e1cb605076f036a695fd1259de0d9b9fe02"}, - {file = "websockets-16.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:00d50c0a27098fcb7ab47b3d99a1b1159b534dbcd959fbf05113ebc37e5f927b"}, - {file = "websockets-16.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1acb698bff1da1782b31aebd8d7a24d7d05453964abcd7d03dbf6e25893908e8"}, - {file = "websockets-16.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc2c453f3b5f99c56b16e233aad5299860558487d26adb2ed27a00c14ca24b8c"}, - {file = "websockets-16.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1a9f08a0728b0835f1c6abe1d9b746ab3de49b7336a0e1919cf96be1e76273eb"}, - {file = "websockets-16.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a089979d6173b27af18026c8d8b0077f83669a9169174482c4651e9f5739a5b6"}, - {file = "websockets-16.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a3c18dba232ec2b92a68579c9fed8ff5a18f853d1e09fc0b6ca3159e94f689fe"}, - {file = "websockets-16.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c1eb7df4170d5068892a8834fb5c07b9552353deb0dbeb0bff3820481ae4792"}, - {file = "websockets-16.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c522bd48e625b6d557aa228967258d6d3da031c4cc21d3352fb302479aa9ba0a"}, - {file = "websockets-16.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:d106396927a7f00b0f3a69215c3357f87bf0bca6844247121f7e8291e826a3b1"}, - {file = "websockets-16.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d71bed12909b8039955536e192867d02d76cd3797cedfd0facf822e7668636c3"}, - {file = "websockets-16.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:9c1cf6f9a936b030b5bed0e800c5ee32069338129084546baf5ff5014dc62fa9"}, - {file = "websockets-16.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:3fd3e6a7af2c8fcdcf4ffbeaf7f54a567b91a83267204187797f31faaa2a4efa"}, - {file = "websockets-16.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dddd27175bf640acae5561fa79b77e8ec71fc445816200523e5c19b6a556fb72"}, - {file = "websockets-16.1-cp310-cp310-win32.whl", hash = "sha256:cce36c80b3f2fede7942f1756d3d885fa6fa086766c8c1bcf00695ab80f0d51a"}, - {file = "websockets-16.1-cp310-cp310-win_amd64.whl", hash = "sha256:115fc4695b94bb855995b23fb1abcb66099a5995575d3d5bc5605a616c58d0eb"}, - {file = "websockets-16.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a9b1d7a63cba8e6b9b77e499a81eab29d31100298d090ad4507d1048c0b9cae0"}, - {file = "websockets-16.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bedbc5efeb96621aa2921d2d92608246691399418cac22acba427eb11877ea1f"}, - {file = "websockets-16.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fd847ab82133015afe65d778e7966ab42dba16bd7ad2e5b8a7918db6539f3f94"}, - {file = "websockets-16.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e2fb33ccb16ee40a95cc676d7b0ff451a9a2632f11a0dbc2e666326892b2e1de"}, - {file = "websockets-16.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f15b6d9ea9c2eaf6ccab964a082b09bfa6634a495bb0c2e9e7ee6943f58976"}, - {file = "websockets-16.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:638cf57c48b4ad8ac1ff1e453f4f97db2426b690ddc111e6da96b27b4a340bc3"}, - {file = "websockets-16.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c1c85f61bc9d5eac57ce705d848dc2d2ce3680638300bf4e1da7d749e2cf4ce"}, - {file = "websockets-16.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:eeab6d27f51c7e579023c971f5e6dff200deadf01faf6831beaecd32052dfaef"}, - {file = "websockets-16.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2ed64e5a97b0b97a0b66e18bfe281317a75fbbd5afe692f939ea8d14a4292f2c"}, - {file = "websockets-16.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9b3b021d0ed4bc16eea9775f62c9fa71acdacba0fc790b38581754dedf29ca60"}, - {file = "websockets-16.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:6eb604a4167f0a0d53c2243dfc667a29f0b43c3436057184e070bb82a1000fa2"}, - {file = "websockets-16.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9a3f125e44c3e34d61d111652e608e0f5b85ce08c225c8d56ad0eb822fa40030"}, - {file = "websockets-16.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8fdf0b00d0d1f30d1f06a92cab46fe542eec3eb302a7aee7163f142d0780f216"}, - {file = "websockets-16.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:67b56828712f5fa7852de4c0265c28827311a657a4d275b7312ed0d1a918bee4"}, - {file = "websockets-16.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:39c7e7730be33b8f0cd6f0aa8e8c82f9cdd1813f159765e073b2ece65f4824b5"}, - {file = "websockets-16.1-cp311-cp311-win32.whl", hash = "sha256:c54fe94fb2f11e11b48920c5f971e298cec73ac35db56efe57a49db63dfc95d4"}, - {file = "websockets-16.1-cp311-cp311-win_amd64.whl", hash = "sha256:f9f4fb9ae8b802e55609685db98382d48fd3feb1397804e1e774968dea0f28c7"}, - {file = "websockets-16.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b6aa3f7ad345cf3862c21f4fbf2ef5e14d911348476c2845e137c091fe3a3f0b"}, - {file = "websockets-16.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b43fcfb521ac2f34ba80b7b8ea16303e4ad82dd8af667bf40839ad3a5d37b164"}, - {file = "websockets-16.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2bd3e12cd9afbe2baedae0b1eeade8ba64329b60fe2f9abdc966bd10fd2c2ef5"}, - {file = "websockets-16.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:35f41979c8623df9bd30d949d82010a8fda5c56ff12cd8508a5b7272b6d4b53a"}, - {file = "websockets-16.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a24d1f35aef07d794a16c853c688e74956c50239bec37b4f2de080056046419b"}, - {file = "websockets-16.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0c64c024ddf7a35331b21fcddb562a039c275d2c82e8c2d12939e7da23997270"}, - {file = "websockets-16.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c3e99757f5baafe20fc598e202ea6f5b0b265186ad38d0a17bd8beca16296955"}, - {file = "websockets-16.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:353f3bc6e058ac1ccab4b3588e8598837a8c04cfc8351233e6d523be675d844c"}, - {file = "websockets-16.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0352f5b38b40e857b6428d468fa21dbb4dd4a567d933c26d9831b4efe1b92f43"}, - {file = "websockets-16.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:70bd789afab579602968c39f21cb925466505f3edff22f0ae852bca54978a4f9"}, - {file = "websockets-16.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d0fb4b46f121eccd539353baebd1083a8767a9a351109453d1d1caecd1ba40c2"}, - {file = "websockets-16.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c14b6634af01541e4efe2954fd8f263386f7aa6d37c01e55dd8109fd17661452"}, - {file = "websockets-16.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a58532c49a851bcb481e58c1be23b315c17fe2fbbed509d75aeea12f543d2c15"}, - {file = "websockets-16.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4e969170c3b08e1d8dabd990fef1fa702c4233aeaabec33f871806e444f6a0e4"}, - {file = "websockets-16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ff9b000064b88787ba9f7a3cb2af2b68a658ca5aad76458a46469e7124b678a0"}, - {file = "websockets-16.1-cp312-cp312-win32.whl", hash = "sha256:b9f5d83f80f4d7c4bba6d97f3755ac05850c784dce0fd2ab371c4e41172f53ff"}, - {file = "websockets-16.1-cp312-cp312-win_amd64.whl", hash = "sha256:6852c9f653966c16109d3b6f31181fd734f7914927e3f0fa1117af7a18c9aa21"}, - {file = "websockets-16.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:b0232ed141cec3df2af5a3959a071c51f40036336b0d37e17faf9ef52fc73e47"}, - {file = "websockets-16.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a71b73d143991714144e159f767b698f03c4a70b8a65ae1733b650cff488045b"}, - {file = "websockets-16.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:187323204c3b2fc465e8fc2609e60437c521790cb9c1acb49c4c452a33e57f37"}, - {file = "websockets-16.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9dba74233c8c3ce368850818c98354dad2570f57231b3fd3bd00d7aa57628881"}, - {file = "websockets-16.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63339bc8c63c86a463177775cb7c677691f5bcfac7b3b2f01b286d42acd41600"}, - {file = "websockets-16.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:23e545ea8ae4263e37cdfd4e22a217f519e48e432728bc461185bbf585f38a83"}, - {file = "websockets-16.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2237081454846fb40403a80ba86d82e2038b9c45865ab96af0abe7d002a91045"}, - {file = "websockets-16.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5f5218de1ed047385ca53744caba9435d65f75d008364970a3fae95a05812cf9"}, - {file = "websockets-16.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:75c98e3920039d0edff03b74478ada504b7ce3a1bc406db2cabfca84320f7baf"}, - {file = "websockets-16.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1facd189d8190af30487a55b4c3688484dd50801628a3b5b2ccd26db08e67057"}, - {file = "websockets-16.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:cc0c6a6eef613c7da32d4fb068f82ef834b58134f6a16b54e6c1e5bf9529ab3d"}, - {file = "websockets-16.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:ad9411eded8988b879be6038206698bf7106c85a78f642c004485bcb95be17eb"}, - {file = "websockets-16.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:cd68f0914f3b64694895bc5e9b14e8b447e41d7bf5ffaf989bb8dcb5e2dfdce7"}, - {file = "websockets-16.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fef2debfe7f7ebdda12176f26166f95b7af17af05ba06150fcf889032e0213e9"}, - {file = "websockets-16.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a3cd6c9b798218798f4bb7b2e71c38f0e744bb94ca537b13376f88019d46384d"}, - {file = "websockets-16.1-cp313-cp313-win32.whl", hash = "sha256:84c170c6869633536921e4474b1cce7254c0c9b0053ef5725f966cee47e718e4"}, - {file = "websockets-16.1-cp313-cp313-win_amd64.whl", hash = "sha256:bef52d327d70fa75dad93ee61ea2cb1d1489aca9f35c188833563f5a3b4df0a5"}, - {file = "websockets-16.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f881fca0a45dd6789939bd6637cd98169b92f1c3fdc78262f2cb9ec2cb1f324e"}, - {file = "websockets-16.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:30c379d5b207d3a7f0ba4c2e4602a895b0bcc63fb5f5371a4ae7fbddb03b672b"}, - {file = "websockets-16.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:98ab58a4faa72b46da0127ccc1931dcbfc0985b0778892300a092185910c4cbe"}, - {file = "websockets-16.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e9c4e369fc181b2d41a99e01477215cecdc8546a39f7d41a59cc0a7065a0b09"}, - {file = "websockets-16.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0704df094b2d5fa7f6f410925a594c2a5c9a09167731a76292e5410934208209"}, - {file = "websockets-16.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b22b1f4950f6ab7126623329c3b47b3b90a14c05db517f2db2a026ad6c928352"}, - {file = "websockets-16.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1ae4a686a662964a6671069f84f7f908cc3475e782227726b0c622c715962105"}, - {file = "websockets-16.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:856bdd638f8277f86465057bfdd4da097c73058fb0f9d2bd5baea29e2bf2d367"}, - {file = "websockets-16.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9003a1fde1c21a322a3ca3fa0c4bda8c639da81dbc925162766086643b05ba87"}, - {file = "websockets-16.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39e947b1f5fdab045174306e3916785bf3ed537648acc1549827c08c33b10953"}, - {file = "websockets-16.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5dd0e666b5931c0509cf65714686a1c5126771e663a79ac5d40da4f58b1f9502"}, - {file = "websockets-16.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a0285df7925657ad65a65fb8dc330808bce082827538fd50ef45fa12d1fc5bca"}, - {file = "websockets-16.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:82d1c2cab3c133e9d059b3a5420bed9376bd30e21c185c63dda4ddadf6ddda47"}, - {file = "websockets-16.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c39907f1eaf11f6277def65aa02d68f30576b693d0c1ca332aafa3caa723ac6d"}, - {file = "websockets-16.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:45c5ea55446171949eb99fd34b771ceddd511ca21958d40d0197ced33159e5ee"}, - {file = "websockets-16.1-cp314-cp314-win32.whl", hash = "sha256:b8ef8b1c8d6bd029a475ac432e730fba2dfd456715d26c473e2a82291024b99c"}, - {file = "websockets-16.1-cp314-cp314-win_amd64.whl", hash = "sha256:7358ff21632b5d062707f73e859c824f1c3807e73d8ca25e71caca7c4cdcf145"}, - {file = "websockets-16.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d0f38f4c3e9b359e257c339c2cc1967ccaeedb102e57c1c986bdce4bf4f32268"}, - {file = "websockets-16.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3c3d2cbd1602593bad49bd86fa3fbb25407d87a3b4bf8857c0ac5ac4914e1901"}, - {file = "websockets-16.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:36069b74671e7e667f48a7484249f84c45a825a134c8b1bdc01875d0daa10d79"}, - {file = "websockets-16.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:587f83c2ce8a5d628e166384d77fa7f0ac69b9007d515ab442123e6615aa8da3"}, - {file = "websockets-16.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6db7972d52bc1b66cefe2246902e256cbaebc9ba8a45eac09343d7eb6671b2"}, - {file = "websockets-16.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e7d6014888a0632e1ed7a4095248bb3095232999447f2d83bfb1900987dd9ed9"}, - {file = "websockets-16.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9cb074d150e4ad2a77aa8a332c2be85f3f64f2681519d2570c1225c12c9821ff"}, - {file = "websockets-16.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d19c9067e1fe9490f974bffbc0e443b80a7674c5efb4980c429cc00771f07c5a"}, - {file = "websockets-16.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d440ff0c6c7469ad59c0a412c383c235935b43635e89425e3f6a0c36de90c31b"}, - {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8613129a2533f08de24505e69a3e403cedaadae49abdb043c4d170ca71b7e4bd"}, - {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:a5bf9c23f197b4ec88290fd5463f33db67362a1bb10f85fc2e8e7627f0ddab97"}, - {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:520b0fd0395f075febb283c76755af724ab9fd19dffa4f3bfd18cb4e622790a3"}, - {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:7143aa09a67e1c013be44e81a88dfe90fc6244198ab86c7edd064152cf619805"}, - {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:7acb811fad08e611755800d1560e395c67e11a6bd563598ea6abb319afb86938"}, - {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c5cf88e3faa2f7931bc6baeee7599c97656a3f6ac7f831f4fccba233e141783a"}, - {file = "websockets-16.1-cp314-cp314t-win32.whl", hash = "sha256:589f8842521c8307684ce0b40ce4ad70c5e0aa46484c6f1225a94ef4b8970341"}, - {file = "websockets-16.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c0e0857c30bbbc2bb5c30687508f0b7ec19aa026cd9f2ff8424d0fee42dcc07"}, - {file = "websockets-16.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7289d899c79e763e6221c8dcb8959361cb43274418538d7c7ad16a43b01d12f9"}, - {file = "websockets-16.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:e22e9e3719f5131bd62da4db63c8da63eb8c91cc99e16c1cbd122f130e1ae07a"}, - {file = "websockets-16.1-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:83bdabafef431247e6b11a9aab8a0893fd8e82e1ed95b32e0373625b03ffce4a"}, - {file = "websockets-16.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b8d13ceabc5c60995f201b5211d76876e17e68706ebf5d3bc666b32eefff1a6"}, - {file = "websockets-16.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:81495f9c0085361c582efbc3207fb877174cfe03370f17d9cd70624404aa526f"}, - {file = "websockets-16.1-py3-none-any.whl", hash = "sha256:c5149dfe490ec7e5ee5dbf624c642fb725f93a5575c7f00ab594ca9eddb8dd81"}, - {file = "websockets-16.1.tar.gz", hash = "sha256:299468cbe42e2b9981134c7c51d99387d8a7bf562b00183b3eec53f882846dad"}, + {file = "websockets-16.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:49ae99bdfcae803a885c926bf14f886196e84925395bb3f568fef5c0f0979d7d"}, + {file = "websockets-16.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5bfd1ac19b1b9986a9c95a82d5e23a391ebb09e12c34d7be6094b86efcc35731"}, + {file = "websockets-16.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9246a0d063cfcbcc85f2359dd6876d681213f4790832272aa16641b4ed5d64d4"}, + {file = "websockets-16.1.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1214e673c404684b9bf7154f5cf43b45025b1a6160fac3a9e438e9c1a97e22cb"}, + {file = "websockets-16.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90001d893bc368e302ef168d82130b4e4fdd27b85fa094682df9b667c2d48838"}, + {file = "websockets-16.1.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:130937b167a52af203c8d58e78d67705874e82759862e3b9671a452fec4abc87"}, + {file = "websockets-16.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c9f23004a3d40e89c01a7955d186a6cc83418d93b749701944ce2de3e95a1f3"}, + {file = "websockets-16.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f55f0b01956a094c8587146d9558c91937e78789c333860ffaf35931a6e5dbc4"}, + {file = "websockets-16.1.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6aaface73b9c71974c6497366d8b9628357f6c9749e09c4ea3610176c63f2ae3"}, + {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dc0fad4933f427acd5b1cec210f3ea6dce7089e1724e4b9ec6ef47c6c04d1b3b"}, + {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f2769a0344a09e9ccf5b3cce538bc75a51b53eff3275d3896310c8552049195d"}, + {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f70541f3104339f59f830522d94ebadb1bf47426287381623443d8bb1cdbf33d"}, + {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:dc385593a42e31cd6fb60c19f0ecb015b386603818fc2c6c274fb42bd2bb4165"}, + {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:387e8e4aa5df2f90b198fa3cad3478822a89cf905b6a6d6c97dc3664689640cc"}, + {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fd46fff7eb62c24804d234f0051c7a8ea81285ad63e0337d3dcf33ca82aee58a"}, + {file = "websockets-16.1.1-cp310-cp310-win32.whl", hash = "sha256:7883388947767080f094950b342b30d35a2a06b849cd967c422fa0db72b40ea9"}, + {file = "websockets-16.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:d57685547e0060cc6fd90ee6a28405d6bd395e525545f13c8d7cd99c78afd79f"}, + {file = "websockets-16.1.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d0fcf657e9f13ff4b177960ab2200237b12994232dfb6df16f1cfe1d4339f93c"}, + {file = "websockets-16.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b852788aa51764e2d8e4cf5493d559326bcae5e38d16ba25ffa322b034df272a"}, + {file = "websockets-16.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1427fb4cf0d72f66333e2cacc3ff5f575bf2d7008166ce991a4a470b21d51a22"}, + {file = "websockets-16.1.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:da4ca1a9d72f9030b3146b8d7022719a9f3d478f61efe6f7dd51d243f61c51b2"}, + {file = "websockets-16.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86d7f0f8bdb25d2c632b72527325e4776430fd5bc61b9118de4e2b8ddb5f5b01"}, + {file = "websockets-16.1.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dfcad78ea1492ee3a9ec765cb7f51bbc17d477107aaf6b22abf7b2558d1c5a0"}, + {file = "websockets-16.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fb9a0a6dc3d1b3986cb88091b6899f0396651e0f74e2c9766ab8d6ffc3842e29"}, + {file = "websockets-16.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29dfa8114c4a620c69591c5973860f768eac29d3fd6904f37f34266cb219c512"}, + {file = "websockets-16.1.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ff9417c0ada4d0f7d212f928303e5579bdf3ace4c802fa4afabb30995da58c3"}, + {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fe0b50da2d84535fb4f7b4bfa951280f97ce3d558a0443b541166d609e67b57"}, + {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:34420aaa64440ebd51ac72ca8a45ef4626429438c9b02e633ae412ed43f925d3"}, + {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a6a61aff018180c9c50b7b0da33bfd29d378af3497429c95006c589a23a11648"}, + {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:04fd29a0e2fe9414a95b00e92c67ae51bf900c50c0f8a4b2dafdad621f49ea1d"}, + {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5c31aa7e39ee3e8a358573257f1c0bb5c52430d1b637030dd9c8cc2c282926be"}, + {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d14bfb217eb4701e850f1525c9d29d79c44794cdf1c299ead25f39f8c78dea81"}, + {file = "websockets-16.1.1-cp311-cp311-win32.whl", hash = "sha256:2e28e602bb13da44fbe518c1781a88e3b9d4c3d48d02c9bad83e546164336f57"}, + {file = "websockets-16.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:7421fad442de870a8cbf2287d1cad7e706ece0dbfeba5e911df132cbdc1cb56a"}, + {file = "websockets-16.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cc97814dfb786a83b6e2dc2e79351e1b83e6d715647d6887fcabd83026417a00"}, + {file = "websockets-16.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e047dc87ef7ca50f4d309bf775ad4a71711c58556d75d7bd0604b2317f43e94b"}, + {file = "websockets-16.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:01fbdcbac298efe19360b94bc0039c8f746f0220ba570f327577bfee81059175"}, + {file = "websockets-16.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0f62863e8a00a6d33c3d6566ec0b89f23787b747ffe0c3bc71ec0e76b82c94b1"}, + {file = "websockets-16.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8087e82f842609734c9b5a1330464f8e94e346ba0e18c832c08bafa4b0d63c15"}, + {file = "websockets-16.1.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2bb5d041a8307d2e18782e7ce777f6fdb1e8c2f5d09291484b18c294b789d9aa"}, + {file = "websockets-16.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1db4de4a0e95673f7545d393c49eeb0c2f18ac1ef93073218c79d5cdb2ee75ab"}, + {file = "websockets-16.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f17dbe07eb3ea7f99e4df9b7e0efefe80fbf30d37a8cc4d561a0aed310bc8847"}, + {file = "websockets-16.1.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4b57693728576d84ede0a77987ab16881b783d2cd9f1dc180a8fbbc3f79c4428"}, + {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a636ff1e7a5c4edf71ef0e79adae7f25dba93b4fcbe3dc958733477ffeb0eaf"}, + {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d6bec75c290fe484a8ba4cacdf838501e17c06ecfbbf31eede81a9e431bd7751"}, + {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:54509b8e92fee4453e152b7558ddef37ce9705a044922f2095a6105e3f80c96f"}, + {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:f0aa4aad3b1b69ad3fd85a0fd0952ec64331c762bd77ec51cc814170873890b2"}, + {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:42290eb6db4ccaca7012656738214f8514082fb6fa40cdeb61bb9a471b52e383"}, + {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:53260c8930da5771cec89439bff99c20c8cb03ddb9588b980697355a83cd4bd3"}, + {file = "websockets-16.1.1-cp312-cp312-win32.whl", hash = "sha256:1d27fa8462ad6a1cb36206a3d0640b2333340def181fae11ed7f9adeaa5c0747"}, + {file = "websockets-16.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:b436f6ec4fc3a6b4237c84d3f83170ed2b40bb584222f0ac47a0c8a5921980c7"}, + {file = "websockets-16.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ab59169ace05dcb49a1d4118f0bde139557adf45091bd85747e36bf5de984dd1"}, + {file = "websockets-16.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5e3b7d601f6f84156b08cc4a5e541c2b50ad7b36cfc302b657a12477c904a5df"}, + {file = "websockets-16.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cd2ca96a082a36964aca83e992f72abeb61b7306c1a6cba4c7d06a7b93750cac"}, + {file = "websockets-16.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f5d497865f05bb222cab7016c6034542e84e5f29f49c6fd3f4939cda7197b5b8"}, + {file = "websockets-16.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bae954c382e013d5ea5b190d2830526bfa45ad121c326da0049b8c769f185db6"}, + {file = "websockets-16.1.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e09f753a169951eb4f28c2c774f71069304f66e7277e0f5a2892423599cfa854"}, + {file = "websockets-16.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:024193f8551a2b0eafbdd160911012c4e6c228c28430c84433253299a9e42d6a"}, + {file = "websockets-16.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:aabe464bfd13bd25f4821faf111da6fefdc389f870265a53105580e45b0a2e49"}, + {file = "websockets-16.1.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a28fcbc9b6baf54a2e23f8655f308e4ccc6afdd7266f8fe7954f320dcda0f785"}, + {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:79eace538c6a97e96d0d03d4f9d314f9677f5ed85a8a984992ffd90b13cb8a56"}, + {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:496af849a472b531f758dbd4d61338f5000538cb1a7b3d20d9d32a264517f509"}, + {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5283810d2646741a0d8da2aa733d6aefa0545809afccb2a5d105a26bc45125f1"}, + {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:4e3b680b1e0a27457e727a0d572fd81dffa87b6dbf8b228ab57da64f7d85aead"}, + {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:69159730a823dde3ea8d08783e8d47ef135a6d7e8d44eb127e32b321c9db8e3e"}, + {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ed5bb271084b46530ee2ddc0410537a9961152c5ccba2fc98c5276d992ccba87"}, + {file = "websockets-16.1.1-cp313-cp313-win32.whl", hash = "sha256:cfb70b4eb56cac4da0a83588f3ad50d46beb0690391082f3d4e2d488c70b68ea"}, + {file = "websockets-16.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:d9531d9cbeac99af6f038fb1bc351403531f7d634a2c2e10e2f7c854c6ed5b68"}, + {file = "websockets-16.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:443aefe96b7fdb132e2a70806cca1f2af49bb3f28e47abcd7c2e9dcf4d8fa1b8"}, + {file = "websockets-16.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6456ff333092d509127d75a638cb411afae8ff17f092635015d1902efec8a293"}, + {file = "websockets-16.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fce6c48559c86d1ac3632ccb1bebc7d5442fbe79bd9bb0e40379ee54be2a4051"}, + {file = "websockets-16.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:92b820d345f7a3fc7b8163949ee92df910f290c3fc517b3d5301c78065adafe1"}, + {file = "websockets-16.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a606d9c24035242a3e256e9d5b77ed9cd6bccfcb7cf993e5ca3c0f6f68fb6a7"}, + {file = "websockets-16.1.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:414e596c75f74e0994084694189d7dc9229fb278e33064d6784b73ffbba3ca31"}, + {file = "websockets-16.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:536676848fc5961aca9d20389951f59169508f765637a172403dc5434d722fa0"}, + {file = "websockets-16.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:97fd3a0e8b53efa41970ac1dff3d8cf0d2884cadeb4caaf95db7ad1526926ee3"}, + {file = "websockets-16.1.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7b1b19636af86a3c7995d4d028dbe376f39b4bf31541146f9c123582a6c94562"}, + {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:41c8e77f17294c0ac18008a7309b99b34ee72247ef10b6dff4c3f8b5ac29896b"}, + {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9f63bcef7f4b02b06b35fc01c93b96c43b5e88e1e8868676caacf493d5a31f3a"}, + {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:dab9eb87869da2d6ed3af3f3adf28414baae6ec9d4df355ffc18889132f3436c"}, + {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:43e3a9fdd7cbf7ba6040c31fae0faf84ca1474fef777c4e37912f1540f854499"}, + {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:056ae37939ed7e9974f364f5864e76e49182622d8f9751ac1903c0d09b013985"}, + {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a0eadbbf2c30f01efa58e1f110eb6fa293261f6b0b1aa38f7f48707107690af9"}, + {file = "websockets-16.1.1-cp314-cp314-win32.whl", hash = "sha256:195c978b065fa40910582464f99d6b15c8b314c68e0546549a55ed83f4735328"}, + {file = "websockets-16.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:4e8d01cc3bcae7bbf8167f944aeafefed590fae5693552bba9794a9df68371cc"}, + {file = "websockets-16.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0ffd3031ea8bda8d61762e84220186105ba3b748b3c8da2ae4f7816fac03e573"}, + {file = "websockets-16.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:84a2cef8deffbd9ab8ee0ea546a2a6a7030c28f44e6cdd4547dbfeb489eb8999"}, + {file = "websockets-16.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3df13f73af9b3b38ab1195eb299ecb67a4330c911c97ae04043ff74085728abe"}, + {file = "websockets-16.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:23253dd5bcae3f9aaee0a1d30967a8dbd52e5d3cff93a2e5b84df57b77d4750d"}, + {file = "websockets-16.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c1c5705e314449e3308872fe084b8571ce078ee4fc55a98a769bdefe5917392"}, + {file = "websockets-16.1.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:69e52d175a0a7d1e13b4b67ad41c560b7d98e8c6f6126eb0bda496c784faf8c7"}, + {file = "websockets-16.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f79c89b5eb034d1722938a891916582f8f7f503f58ca22518a63c3f2cd18499"}, + {file = "websockets-16.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39f2a024af5c345ffe8fcf1ee18c049c024c94df393bb09b044a6917c77bde43"}, + {file = "websockets-16.1.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:952303a7318d4cbe1011400839bb2051c9f84fa0a35923267f5daba34b15d458"}, + {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:249116b4a76063d930a46391ad56e135c286e4562a18309029fc2c73f4ed4c62"}, + {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:61922544a0587a13fd3f53e4c0e5e606510c7b0d9d22c8444e5fae22a06b38cb"}, + {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:46dcaa042cd1de6c59e7d9269fa63ff7572b6df40510600b678f0826b3c7af51"}, + {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:38565aca3e01ea8734e578fb2118dade0ecb0250533f29e22b8d1a7a196cf4d0"}, + {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:42f599f4d48c7e1a3338fdaac3acd075be3b3cf02d4b274f3bf2767aedd3d217"}, + {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dcc04fedf83effaeb9cce98abc9469bb1b42ef85f03e01c8c1f4438ef7555737"}, + {file = "websockets-16.1.1-cp314-cp314t-win32.whl", hash = "sha256:8483c2096363120eea8b07c06ae7304d520f686665fffd4811fad423930a65d7"}, + {file = "websockets-16.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:bcce07e23e5769375158f5efdcdafa8d5cd014b93c6683865b840ed65b96f231"}, + {file = "websockets-16.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:820fb8450edddae3812fd58cbc08e2bf22812cb248ecb5f06dbb82119a56e869"}, + {file = "websockets-16.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:125f22dbefaf1554fea66fc83851490edb284ce4f501d37ffed2752f418332d9"}, + {file = "websockets-16.1.1-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:30bbe120437b5648a77d3519b7024ea09530e0b5b18d3698c5a0ae536fe0cc2e"}, + {file = "websockets-16.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b6b9dadbef0cccd9f4c4ee96b08898afa73e26803bbe0f6aeb5bb12b0074206d"}, + {file = "websockets-16.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56cd5fc4f10a9ea8aa0804bddb7b42506cf9e136046f3b4c27de8fec9e2ecba5"}, + {file = "websockets-16.1.1-py3-none-any.whl", hash = "sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3"}, + {file = "websockets-16.1.1.tar.gz", hash = "sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57"}, ] [[package]] @@ -5993,116 +5985,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.2" +version = "1.24.5" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, + {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, + {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, + {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, + {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, + {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, + {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, + {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, + {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, + {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, + {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, + {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, + {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, + {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, + {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, ] [package.dependencies] diff --git a/security_scanning/examples/trtllm-eval/poetry.lock b/security_scanning/examples/trtllm-eval/poetry.lock index 1a2cf927a354..4ffa036e4d38 100644 --- a/security_scanning/examples/trtllm-eval/poetry.lock +++ b/security_scanning/examples/trtllm-eval/poetry.lock @@ -59,131 +59,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.1" +version = "3.14.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32"}, + {file = "aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7"}, + {file = "aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19"}, + {file = "aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71"}, + {file = "aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf"}, + {file = "aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7"}, + {file = "aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc"}, ] [package.dependencies] @@ -275,14 +275,14 @@ files = [ [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -464,14 +464,14 @@ all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == [[package]] name = "cuda-pathfinder" -version = "1.5.6" +version = "1.6.0" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, + {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, ] [[package]] @@ -678,14 +678,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.29.7" +version = "3.32.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, + {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, + {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, ] [[package]] @@ -885,38 +885,30 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.1" +version = "1.5.2" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "(sys_platform == \"linux\" or sys_platform == \"win32\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\")" files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, + {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, ] [package.extras] @@ -971,14 +963,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.23.0" +version = "1.25.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, + {file = "huggingface_hub-1.25.1-py3-none-any.whl", hash = "sha256:004d4e70350517e24c68a7dbb7dc5e40b2b6aefef8f94bf7a85f6f9835102ea5"}, + {file = "huggingface_hub-1.25.1.tar.gz", hash = "sha256:21129595ca7a753be479b319913e22cc8808361ac118bd76cc413db831b28a99"}, ] [package.dependencies] @@ -1756,6 +1748,7 @@ description = "Fast numerical expression evaluator for NumPy" optional = false python-versions = ">=3.10" groups = ["main"] +markers = "python_version == \"3.10\"" files = [ {file = "numexpr-2.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d0fab3fd06a04f6b86102552b26aa5d85e20ac7d8296c15764c726eeabae6cc8"}, {file = "numexpr-2.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:64ae5dfd62d74a3ef82fe0b37f80527247f3626171ad82025900f46ffca4b39a"}, @@ -1819,6 +1812,66 @@ files = [ [package.dependencies] numpy = ">=1.23.0" +[[package]] +name = "numexpr" +version = "2.14.2" +description = "Fast numerical expression evaluator for NumPy" +optional = false +python-versions = ">=3.11" +groups = ["main"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "numexpr-2.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2aa65ddc2243f19c6915f34ee0978b4a2df20f297230a793c4ee6d55f3472599"}, + {file = "numexpr-2.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf959e6df6cb603611c034b6cba7b03a361be0ad0b80b73f163fab95f5ccbb7f"}, + {file = "numexpr-2.14.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d534ecb456a4ae3995f99c8a5deb469bfff05d4ec610a7885c175c881d12f710"}, + {file = "numexpr-2.14.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f41170e9d0dbba76851e35d80cfa9f4ca5fe78628c5bf24d941cf3364940ab7a"}, + {file = "numexpr-2.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6acafb2fdbeaaa6681a8f1a1d8b3f7dcd33704baace7057b950754b258be7c43"}, + {file = "numexpr-2.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7ca9e71195b36cc7aeafe97347549e1e1c1e889ff700238782ef6447651ec26d"}, + {file = "numexpr-2.14.2-cp311-cp311-win32.whl", hash = "sha256:779129d50974e7d6d6581d322f75b8f8375e96215b6861a2d5460347997ef649"}, + {file = "numexpr-2.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:2f132777d7d425471c458af5617e023402f13f5006301eacf8a1a6e7118ea70c"}, + {file = "numexpr-2.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:f1de5c88515ed9fbcad42699a0e2b5821b4d0f0adb0da6fb7e009e5cb19d8493"}, + {file = "numexpr-2.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:606ceaf5722e295ef965ca591736fc26d9e5f13ad950a479e64cead1947f8a3d"}, + {file = "numexpr-2.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:790da022539fe7c37dc893acf530a91c2ca6964d7ba11f464131383729d058f3"}, + {file = "numexpr-2.14.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:327be9ee62251c173236dc620147ff2d0e732a32f5bad918d78a10082f502f63"}, + {file = "numexpr-2.14.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6a5d8fc7016bf6f6e1808b011510aa7c3bd75ec1407f7650874ec591db59f5e"}, + {file = "numexpr-2.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4b1ff261c3e69c4c59578d3a9ca6132603619d38ae1abe73325563bed3b9bbaf"}, + {file = "numexpr-2.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b8384592c49cb15a91caa54e2cd84d1ce18edb7af030bb76cd29b52e5dc155d"}, + {file = "numexpr-2.14.2-cp312-cp312-win32.whl", hash = "sha256:41cdeacf1b4e51c1143983ea61fcee68139ca47222b55a9265b4fa73826c4260"}, + {file = "numexpr-2.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:8fc55d14bcf17b3fe69213bea14f999451892b4690717008c66f2edfd6a085ce"}, + {file = "numexpr-2.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:806a4471310fe20aa7cb1b2816a6f5e508073a1ad1c2e18041b83e57066fad6a"}, + {file = "numexpr-2.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0741efbd75c284e709b0fd430c85c31982b44c9962922ba8a9cbbea1bf413321"}, + {file = "numexpr-2.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92b00c78664070e3af155c6be713a0a5d75d598647ce32a5609adb79a8f961d3"}, + {file = "numexpr-2.14.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:149ab5744a5222f07b1d60455c4021c754d395e44938944ac7c7c2495f7feb54"}, + {file = "numexpr-2.14.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fd2f5882a66a7792aa6614c68831aa20085b499d41422aedd001080624ebb14c"}, + {file = "numexpr-2.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:375d8bee15be42dab22100a0a3de05fe6689a2de853eca012858768a9a7e02ab"}, + {file = "numexpr-2.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c1ffaf805d8636c3f95d0996517ecf9684c9ac62d768030ca78d1d00af2b3504"}, + {file = "numexpr-2.14.2-cp313-cp313-win32.whl", hash = "sha256:449a57fb9d38de136e742b1fc429572b42f29778f1d695c3fe50ffec9d3c9a71"}, + {file = "numexpr-2.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:dd905922d7dce457947d54b84c7ac345cef37332b724445e159a5a1a2080ce2b"}, + {file = "numexpr-2.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:b02738853b9b5b8a995f6c680f8f6ef33e8f419395b8fa380e38690495fdb911"}, + {file = "numexpr-2.14.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:76e87c7bd70d721ce4d418e81f4fb7ecf9e7e67d7cea8102527b07fd3d3facf9"}, + {file = "numexpr-2.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:939c89f613b814e64bb568859397dc9f99b219c3ef681a72fb99a86e435262f9"}, + {file = "numexpr-2.14.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b20c1c55aba7812ff2f2c6a50006425d02282fabb1eaf8d75fe638ffcf6deb02"}, + {file = "numexpr-2.14.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bac00898930f962f360c3d763a8e2273fc931f65a1759ff1bf64b3cf13d65aee"}, + {file = "numexpr-2.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:022e61a3d5dbf5807746264b62126d1c2c24057ad90052478a4d4482ab2555c2"}, + {file = "numexpr-2.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1d4593e2c6fa060cd7441e8b6ef25c16321a6be2144b3c82d1e00885f1fb6e94"}, + {file = "numexpr-2.14.2-cp314-cp314-win32.whl", hash = "sha256:66f3b125b1104241322811de87918724d6709bf082dc0703722d0cecb7b29e82"}, + {file = "numexpr-2.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:ef576a1cded27ba2f3129bc3c42df452a1c498072680d560793f98b0024cd7e6"}, + {file = "numexpr-2.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8274c51ae1842948f3ae7fe6951a23dcf4ddcbeeaff3737e978e7740b754662d"}, + {file = "numexpr-2.14.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:f3526699350f94c6277fb16863773a1af9defd95a6f78bbd69b1f0338fd94756"}, + {file = "numexpr-2.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:91e7928435f14fcb351c0157000bce65122b897cc8b0df6bcc48251f25850a6d"}, + {file = "numexpr-2.14.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c66925deb968f0b5280f723e2bb5918c11e6be2ca60e9e1530006286ab44031d"}, + {file = "numexpr-2.14.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a404c9a55902572eec810068d06b79a7c99e96f0400f5a7d73f39dff5ec5e371"}, + {file = "numexpr-2.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:44dc6b1dfa9abcbfc9917297f0d2af7c87c16b6ecd45747a8e70f54399a3a2f9"}, + {file = "numexpr-2.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:93233040f4bed3bce5abb0c2d20aeb1074511f29cbaa9c14828f86bcfa44d321"}, + {file = "numexpr-2.14.2-cp314-cp314t-win32.whl", hash = "sha256:2aceefa08f8f86317fa6e8fe9f6dc20d24ab8365d715be4a26306acf406d2dbe"}, + {file = "numexpr-2.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:cd684ac9daa539fcdac3437678834797b29d7780cfaad71111745132d466d51f"}, + {file = "numexpr-2.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:2ef72de3d3dd466cb0c435cae7141c99b0f8091b1eae9d03dcb38690f56c3f79"}, + {file = "numexpr-2.14.2.tar.gz", hash = "sha256:e7144e83ea9e581f2273e0304f15836736c4e470e2bd2e378ce617662a1ca278"}, +] + +[package.dependencies] +numpy = ">=1.26.0" + [[package]] name = "numpy" version = "2.2.6" @@ -2406,61 +2459,55 @@ xml = ["lxml (>=4.9.2)"] [[package]] name = "pandas" -version = "3.0.3" +version = "3.0.5" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.11" groups = ["main"] markers = "python_version >= \"3.11\"" files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, + {file = "pandas-3.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2946e77e4a53cd248cbde631a12f0e51c8324ce354c3eba4d20147c1ad6f4282"}, + {file = "pandas-3.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71ecc8fb7ed1a7aa4392316b5309a6347e8e7f832f38fd897846b3a1457a9298"}, + {file = "pandas-3.0.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b173f5951ff6b8b0ec7675e20dff3c97b7e7a57dfcce387c2d7c5afe87cb7899"}, + {file = "pandas-3.0.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c0cf1dd9b55a22d105fc46c1b489af3bd42264fcba7c66297bf47a9a1d9c78a"}, + {file = "pandas-3.0.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0fac0010c75e4efb6b99e249c183a8993ce0dc95c240f9b120a5e67c727b7928"}, + {file = "pandas-3.0.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:08d24fe11a17dc33bd6e937dc9c665f9cba08fbdc9f657f405713515febe300d"}, + {file = "pandas-3.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b1261758dfb6cf12c3cff8300e21cefad30e7ec709abb4c24ac7318e6a52462a"}, + {file = "pandas-3.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:679f4e85b30ddb1515458ab1e788d3e260eae369b1f78da7a3aa4cac8ebf4a2a"}, + {file = "pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d"}, + {file = "pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc"}, + {file = "pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc"}, + {file = "pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34"}, + {file = "pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6"}, + {file = "pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7"}, + {file = "pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce"}, + {file = "pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41"}, + {file = "pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49"}, + {file = "pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b"}, + {file = "pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3"}, + {file = "pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b"}, + {file = "pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be"}, + {file = "pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58"}, + {file = "pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee"}, + {file = "pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6"}, + {file = "pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e"}, + {file = "pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36"}, + {file = "pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c"}, + {file = "pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0"}, + {file = "pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b"}, + {file = "pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94"}, + {file = "pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da"}, + {file = "pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92"}, + {file = "pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85"}, + {file = "pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca"}, + {file = "pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da"}, + {file = "pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a"}, + {file = "pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040"}, + {file = "pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd"}, + {file = "pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c"}, + {file = "pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea"}, + {file = "pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c"}, + {file = "pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712"}, ] [package.dependencies] @@ -2490,7 +2537,7 @@ postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psyc pyarrow = ["pyarrow (>=13.0.0)"] spss = ["pyreadstat (>=1.2.8)"] sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] +test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4,<9.1)", "pytest-xdist (>=3.6.1)"] timezone = ["pytz (>=2020.1)"] xml = ["lxml (>=5.3.0)"] @@ -2864,14 +2911,14 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2026.2" +version = "2026.3.post1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" groups = ["main"] files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, + {file = "pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815"}, + {file = "pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d"}, ] [[package]] @@ -2991,126 +3038,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.10" +version = "2026.7.19" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, + {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, + {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, + {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, + {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, + {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, + {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, + {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, + {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, + {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, + {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, + {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, + {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, + {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, + {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, + {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, + {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, + {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, + {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, + {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, + {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, + {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, + {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, ] [[package]] @@ -3883,14 +3930,14 @@ pyyaml = ["pyyaml"] [[package]] name = "tqdm" -version = "4.68.4" +version = "4.70.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, + {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, + {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, ] [package.dependencies] @@ -3923,14 +3970,14 @@ dev = ["twine"] [[package]] name = "transformers" -version = "5.14.0" +version = "5.14.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.0-py3-none-any.whl", hash = "sha256:288287da4fdcdae0b611be32b5be20d4bcbf5caaa5f269292927fb1ff888d19d"}, - {file = "transformers-5.14.0.tar.gz", hash = "sha256:9e1a2b345c195d216afd10082394b915a82e339fa09e9013c32094ff835ca9db"}, + {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, + {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, ] [package.dependencies] @@ -4296,116 +4343,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.2" +version = "1.24.5" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, + {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, + {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, + {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, + {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, + {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, + {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, + {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, + {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, + {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, + {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, + {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, + {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, + {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, + {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, ] [package.dependencies] diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index 934b83107380..bf824fbca828 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "8b5d9ea5fcee582faafcd5a7efb6babab0ff13fe", - "timestamp": "2026-07-16T02:49:15Z" + "commit_hash": "c4f33538bfb1ff686acd7868e1e97caa73e0c1b3", + "timestamp": "2026-07-28T02:40:43Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index 0627702ffd3a..d6f2e0b07974 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -60,131 +60,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.1" +version = "3.14.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32"}, + {file = "aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7"}, + {file = "aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19"}, + {file = "aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71"}, + {file = "aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf"}, + {file = "aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7"}, + {file = "aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc"}, ] [package.dependencies] @@ -231,14 +231,14 @@ files = [ [[package]] name = "annotated-types" -version = "0.7.0" +version = "0.8.0" description = "Reusable constraint types to use with typing.Annotated" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, - {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, + {file = "annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0"}, + {file = "annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7"}, ] [[package]] @@ -581,14 +581,14 @@ ray = ["ray (>=2.0)", "safetensors (>=0.5.3)"] [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -1004,13 +1004,13 @@ cu13 = ["cuda-bindings[all] (==13.*)", "cuda-toolkit (==13.*)"] [[package]] name = "cuda-pathfinder" -version = "1.5.6" +version = "1.6.0" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, + {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, ] [[package]] @@ -1327,14 +1327,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.29.7" +version = "3.32.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, + {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, + {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, ] [[package]] @@ -1364,21 +1364,23 @@ dev = ["pytest", "pytest-xdist", "ruff"] [[package]] name = "flashinfer-python" -version = "0.6.14" +version = "0.6.15" description = "FlashInfer: Kernel Library for LLM Serving" optional = false python-versions = "<4.0,>=3.10" groups = ["main"] files = [ - {file = "flashinfer_python-0.6.14-py3-none-any.whl", hash = "sha256:d124369346a3d48eac67e31c42f7a3c813bcc0abc10e2e36db413b7b3dfd97df"}, - {file = "flashinfer_python-0.6.14.tar.gz", hash = "sha256:f4da8b5e005601784e85e0dcaa3389f908ee2d32c2560142d67124ab10e4a070"}, + {file = "flashinfer_python-0.6.15-py3-none-any.whl", hash = "sha256:da6c339e14db4831ade0d593324f02907d44bd4b86b640faad4727d9e089477b"}, + {file = "flashinfer_python-0.6.15.tar.gz", hash = "sha256:2a3f1ed47129f9ac9505a26a8f12cadefc0f27d3104fb623ae281032f49eae5f"}, ] [package.dependencies] apache-tvm-ffi = ">=0.1.6,<0.1.8 || >0.1.8,<0.1.8.post0 || >0.1.8.post0,<0.2" click = "*" +cuda-python = ">=12.0" cuda-tile = ">=1.4.0" einops = "*" +nccl4py = ">=0.3.1" ninja = "*" numpy = "*" nvidia-cudnn-frontend = ">=1.13.0" @@ -1393,7 +1395,6 @@ tqdm = "*" [package.extras] cu12 = ["nvidia-cutlass-dsl (>=4.5.0)"] cu13 = ["nvidia-cutlass-dsl[cu13] (>=4.5.0)"] -nvep = ["cuda-python (>=13.0)"] [[package]] name = "frozenlist" @@ -1612,70 +1613,70 @@ test = ["coverage", "pytest (>=7,<8.1)", "pytest-cov", "pytest-mock (>=3)"] [[package]] name = "grpcio" -version = "1.82.1" +version = "1.83.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "grpcio-1.82.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:91859d1cac5f47caec5fc40e9f827500cdb54ce5b36450dc9a65616b5af49c17"}, - {file = "grpcio-1.82.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c80c9741dcef192f669876a81957cf7713b441c2f0c43631350d75fa49321d31"}, - {file = "grpcio-1.82.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b89cff456796d2f0581783726ad017a2c70aff2d27b0f05504c34e2e417f7560"}, - {file = "grpcio-1.82.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:d6e8a08f7038ba7a77f71e250804e4aba84fe91d22cfc54ff43c07b7529c4728"}, - {file = "grpcio-1.82.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:50fd2fe83426b1b1c6cdc4d72d555223b7dddf8ce07c5bac218b13fc6d684c6f"}, - {file = "grpcio-1.82.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b758540a24d5394a9c578bf9f6126389f474b106ac3d9df1d53de56cb14c9fd9"}, - {file = "grpcio-1.82.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c4ba4aac238f685743575d9d700003ac16537cce26e7c774993134f530652464"}, - {file = "grpcio-1.82.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed6fc621d6f366c88a60f0b971d5afd21d441d9aa561ee688de5b7acdb2cf901"}, - {file = "grpcio-1.82.1-cp310-cp310-win32.whl", hash = "sha256:bd2f45e46fff5b91c10997d0743a987517a7dde67c64c592835c2dcaac66f587"}, - {file = "grpcio-1.82.1-cp310-cp310-win_amd64.whl", hash = "sha256:5e171d5f0d6a0af78ea7512783f170a44f80c165259d8773e3a354a7f991f2b5"}, - {file = "grpcio-1.82.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:0ddb18a9a9e1f46692b3567ae4abb3f8d117ce6afea48650f8eca06d8ab5d06f"}, - {file = "grpcio-1.82.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cf855b1af246720f567b0ce5d0724d45dfa4188eecc3296a2a69257b11b9e94b"}, - {file = "grpcio-1.82.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb30cb13e25bc13cea70ffc69d6d90c49d36ea6c1d4549e6912f70177834cac"}, - {file = "grpcio-1.82.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1e822b2774f719c017cbe700b6e47173b6ae290fb84906f52a5a3c2c60b62e1e"}, - {file = "grpcio-1.82.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5dafb1ece8ed45dee7c738f166ec82e19673221ed5ab8967f72858a4685345b2"}, - {file = "grpcio-1.82.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e06503106e7271e0a49fd5a1ac04747f1e47e87d900476db6fe45bc87ee411f4"}, - {file = "grpcio-1.82.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ff99bc8cafb6a952201c37b995f425e641c93ffa6e072258525feab57290141d"}, - {file = "grpcio-1.82.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:644ae1b94266ac785330f4590a69e52b6a7eb73029043a02209db81c81397d69"}, - {file = "grpcio-1.82.1-cp311-cp311-win32.whl", hash = "sha256:e203d2e19d471630084a16c815616f8211dff21c268ab3c5f5bf38417832e074"}, - {file = "grpcio-1.82.1-cp311-cp311-win_amd64.whl", hash = "sha256:0d8299c285fe6cc6a1f56badf8d3bc5078c8d20273ee64bafa3783b4bc29a769"}, - {file = "grpcio-1.82.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:c09bd5fa0d5b1fbd773ec349fe61441c3e4ebf168c229aa7538a820bdfad6a58"}, - {file = "grpcio-1.82.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1eae24810720734598e3e6a1a528d5de0f265fe3fc86575e9ecce424b9ec7379"}, - {file = "grpcio-1.82.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a6bd5daf5bde7b24d7ad2cbaf8bf9eac620d96222016bb5e7ddde930dec0673f"}, - {file = "grpcio-1.82.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1ecfde669cb687ac020d31ff76debe5dc7a62213335f02262eb6625628da1c03"}, - {file = "grpcio-1.82.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:011c8badee95734dee8bf05ce3464756a0ac3ebb8d443afd20c0e2b5e4640ad9"}, - {file = "grpcio-1.82.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b85f4564926fb23114d239392bdcae200db1e6179629edd7d7ab0ab89c96a197"}, - {file = "grpcio-1.82.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2c0c8270833395644c3fe6b6a806397955a2bc0538000a19a78b90c05a6c16e0"}, - {file = "grpcio-1.82.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2ba199205ff46c7778290fe1673c91ac8e7e45678dd5c86e9e56fa33ec8788f6"}, - {file = "grpcio-1.82.1-cp312-cp312-win32.whl", hash = "sha256:06127691866e295c14e84a1fb86356dd962254f6abd0da4ca4b001eea9e89438"}, - {file = "grpcio-1.82.1-cp312-cp312-win_amd64.whl", hash = "sha256:1fa3223a3a2e1db74f4c2b255189eb7ea875dfba56e221d252ee3fc7b204778e"}, - {file = "grpcio-1.82.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:b454a2d97bfab7565683a02345f86bd182ab69fd7c2bdb7414171e7538f266b1"}, - {file = "grpcio-1.82.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:3dde70abfc80b3be11de53ba0d601c439e7fb2afd3583ad1788d1146bec92fdc"}, - {file = "grpcio-1.82.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5523099c98c292ea1ae08e617249db760c56a78f8deae879027fe7d1ffbcbf6"}, - {file = "grpcio-1.82.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5e5c4dc0a59b0f8490a6bdfd6fc8395b9d8ad8a8407c7d67ca7b5bba15c0877f"}, - {file = "grpcio-1.82.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c40d94ba820329cc191981bc22fa6f6eed0799c6d921f3c6709521d59d4a2fd7"}, - {file = "grpcio-1.82.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c816180e31e273caaec6f8bd86a8392499d5bbb26f41da44e3dce48bde69095"}, - {file = "grpcio-1.82.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e31fd780b261830720cb70b0fd8f0aa51d49e75a66d7464ad2e31d4b765f2580"}, - {file = "grpcio-1.82.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d76152d7c31d7210d4a106e5d8b64da5bba5d6abf11be30e2f7b0a0c59bbcbf"}, - {file = "grpcio-1.82.1-cp313-cp313-win32.whl", hash = "sha256:38e9dcb5258226fb3282630b31b16a968df52c8c6ad514af540646e0a4578f8a"}, - {file = "grpcio-1.82.1-cp313-cp313-win_amd64.whl", hash = "sha256:3dbfb52c36d9511ac2b8e6c94fdde837b393ae520cc321f52a333a2deedf5a90"}, - {file = "grpcio-1.82.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:35f990f7784c8fd2872644f07f96ebb4d9e48e145a190ab80d0280af91a1bfb2"}, - {file = "grpcio-1.82.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:46536a4a1f4434df3c851b9254ff6fc7df5705b273681a15ca277d5921c178a0"}, - {file = "grpcio-1.82.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d6650a7c1ebb7921c70e12a385439a8118efb99e669fa9ed31cf25db1843937c"}, - {file = "grpcio-1.82.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b8e110c66df5204c0506d6c8787b35d48b8b699ef5aa366d6c4d67325c67fe9a"}, - {file = "grpcio-1.82.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f853eae07235a51a27bb5d6a9a175a59ca55dc9b99edc6ce2f76f07332d333ae"}, - {file = "grpcio-1.82.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:60b0f2c95337694fc094b77d9f60f50566c84b5677393e342eb98daeee242d98"}, - {file = "grpcio-1.82.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:b064fc444812bdaa9825d33c26f8d732d63ee6a5d78557c1faf92c98687fed27"}, - {file = "grpcio-1.82.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7d7ede11d747b4e1bd05e3bc0260e155b65a88735a895a10f6521f19b889511e"}, - {file = "grpcio-1.82.1-cp314-cp314-win32.whl", hash = "sha256:3d21f19838dc255ecbb79321b15ae9b98fbddff4c3d4aedb0a81bdd7f4ab572a"}, - {file = "grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5"}, - {file = "grpcio-1.82.1.tar.gz", hash = "sha256:707b24abd90fcb1e45bcc080577da1dbf9971d107490589b9539af8e1e77b4b5"}, + {file = "grpcio-1.83.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:fba099b716e73512d61b97f71ea3c31a72abb36904036e316bf4dd148ca8dcc8"}, + {file = "grpcio-1.83.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6755ed67cc3e454d51ae9f6e1915b80d3942fa4de956ef48dacd45ab7f40b727"}, + {file = "grpcio-1.83.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5882c1a721b50ce0123ee5e839e1ab059ad72a7ade76cdf2d5bd833b56791acf"}, + {file = "grpcio-1.83.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4e3eedfc92b6b9f2960115e7e620cf0cbf80bb7849a51ce3820dc54dfd88b6b9"}, + {file = "grpcio-1.83.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4fcaa7c45c45b4a89e2867d1f1785d9481a788399d915e341ed2eb49aeef9dd4"}, + {file = "grpcio-1.83.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6b6c666a1d5613ff360c9e90f44665e3a88b25a815209ddbc0917eec281931cb"}, + {file = "grpcio-1.83.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6be5c807b717be3dd649446f021301fd7907e376318675d2147823071034112a"}, + {file = "grpcio-1.83.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c834e86d8fd2f03d7e4db49a027f7c5b89c5b88eed305543a5295bd6fee61e40"}, + {file = "grpcio-1.83.0-cp310-cp310-win32.whl", hash = "sha256:35a5b1c192496b6c25956eebfa963468935612206fd2543ac3ce981e6a5e0f03"}, + {file = "grpcio-1.83.0-cp310-cp310-win_amd64.whl", hash = "sha256:8f6c395e493d20c39b29392ca200e9aaeb78d0bc2f04db0c0a7da7ddc939aa57"}, + {file = "grpcio-1.83.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:8ff0b8767ddd62704e0d9571c1890af08d84a3a689ebba1807e62519d0b3277f"}, + {file = "grpcio-1.83.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:4772402f43517b4824980be4b3b2274a81eec0004a70009473c31b340d43e223"}, + {file = "grpcio-1.83.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f4cee5fc86e84a0cf7ad1574b454c3320e087c07f55b7df5dc0ac6a873fb90c0"}, + {file = "grpcio-1.83.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f5e822a7e7d03282f6ad225e710493c48b9057a353358344a5f7c42b2b37618d"}, + {file = "grpcio-1.83.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f5f410d7c2903eabb34789dfd6342eef04af1ad459943936b7e09a9f5bd417b9"}, + {file = "grpcio-1.83.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee94a4016fdf8699fb1fd8a38652475ff677f1c72074cee44deeeb9a7e95e745"}, + {file = "grpcio-1.83.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c6444666317338e903093c7c756e6cc88eee59f798cb8dd41e87725bf54e1617"}, + {file = "grpcio-1.83.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa074041231f03959cb097dd5517b0677b8ea49215bae01d5710a7b69dd59969"}, + {file = "grpcio-1.83.0-cp311-cp311-win32.whl", hash = "sha256:cb056f6e171c42639a50460b2929c82241fda51f71cf3dcdd68090fe45095a45"}, + {file = "grpcio-1.83.0-cp311-cp311-win_amd64.whl", hash = "sha256:7416952ca770477990257206276999056f8316d79196f2f25942393e58a20b49"}, + {file = "grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930"}, + {file = "grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da"}, + {file = "grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16"}, + {file = "grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf"}, + {file = "grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd"}, + {file = "grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c"}, + {file = "grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5"}, + {file = "grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5"}, + {file = "grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc"}, + {file = "grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df"}, + {file = "grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0"}, + {file = "grpcio-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:74fe6f9e8a35c7dbf32255ee154d15e3e5338a81ed39173d079d594d2e544cd1"}, + {file = "grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867"}, + {file = "grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881"}, + {file = "grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f"}, + {file = "grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61"}, + {file = "grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45"}, + {file = "grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c"}, + {file = "grpcio-1.83.0-cp313-cp313-win32.whl", hash = "sha256:8fe04f1050a59f875601eb55d42b4f66946fe89817f967e34db1462ccd07dadf"}, + {file = "grpcio-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:6e01ecd9d8ef280abe1365138a4dc318f9a5287f4cb1b41d07816f796653f735"}, + {file = "grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa"}, + {file = "grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c"}, + {file = "grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b"}, + {file = "grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c"}, + {file = "grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df"}, + {file = "grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b"}, + {file = "grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404"}, + {file = "grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af"}, + {file = "grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33"}, + {file = "grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9"}, + {file = "grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24"}, ] [package.dependencies] typing-extensions = ">=4.12,<5.0" [package.extras] -protobuf = ["grpcio-tools (>=1.82.1)"] +protobuf = ["grpcio-tools (>=1.83.0)"] [[package]] name = "h11" @@ -1730,38 +1731,30 @@ numpy = ">=1.19.3" [[package]] name = "hf-xet" -version = "1.5.1" +version = "1.5.2" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, + {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, ] [package.extras] @@ -2555,14 +2548,14 @@ typing = ["mypy", "typing_extensions ; python_version < \"3.8\""] [[package]] name = "mistral-common" -version = "1.11.5" +version = "1.11.7" description = "Mistral-common is a library of common utilities for Mistral AI." optional = false python-versions = "<3.15,>=3.10.0" groups = ["main"] files = [ - {file = "mistral_common-1.11.5-py3-none-any.whl", hash = "sha256:7c1b09f43a589027315840bfd6f3528d5abf520eed701f8d8b7a922d6e5b855e"}, - {file = "mistral_common-1.11.5.tar.gz", hash = "sha256:ef8c03ad8359fa1386d66ee08d534d8f4a65a6955c9b24bd5caa2e005066cdec"}, + {file = "mistral_common-1.11.7-py3-none-any.whl", hash = "sha256:a9511b88eacacbe7dacddd9d3498c1739f56847b7fdddbd5a22e7844fd9def95"}, + {file = "mistral_common-1.11.7.tar.gz", hash = "sha256:d3b79583595cf6d96a2ab33e42cb8449768383147b8c56cac5a4f193be19d20d"}, ] [package.dependencies] @@ -2587,65 +2580,6 @@ server = ["click (>=8.1.0)", "fastapi[standard] (>=0.118.3)", "pydantic-settings soundfile = ["soundfile (>=0.12.1)"] soxr = ["soxr (>=0.5.0)"] -[[package]] -name = "ml-dtypes" -version = "0.5.4" -description = "ml_dtypes is a stand-alone implementation of several NumPy dtype extensions used in machine learning." -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "ml_dtypes-0.5.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b95e97e470fe60ed493fd9ae3911d8da4ebac16bd21f87ffa2b7c588bf22ea2c"}, - {file = "ml_dtypes-0.5.4-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4b801ebe0b477be666696bda493a9be8356f1f0057a57f1e35cd26928823e5a"}, - {file = "ml_dtypes-0.5.4-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:388d399a2152dd79a3f0456a952284a99ee5c93d3e2f8dfe25977511e0515270"}, - {file = "ml_dtypes-0.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:4ff7f3e7ca2972e7de850e7b8fcbb355304271e2933dd90814c1cb847414d6e2"}, - {file = "ml_dtypes-0.5.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6c7ecb74c4bd71db68a6bea1edf8da8c34f3d9fe218f038814fd1d310ac76c90"}, - {file = "ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc11d7e8c44a65115d05e2ab9989d1e045125d7be8e05a071a48bc76eb6d6040"}, - {file = "ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b9a53598f21e453ea2fbda8aa783c20faff8e1eeb0d7ab899309a0053f1483"}, - {file = "ml_dtypes-0.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:7c23c54a00ae43edf48d44066a7ec31e05fdc2eee0be2b8b50dd1903a1db94bb"}, - {file = "ml_dtypes-0.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:557a31a390b7e9439056644cb80ed0735a6e3e3bb09d67fd5687e4b04238d1de"}, - {file = "ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac"}, - {file = "ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900"}, - {file = "ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff"}, - {file = "ml_dtypes-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7"}, - {file = "ml_dtypes-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460"}, - {file = "ml_dtypes-0.5.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48"}, - {file = "ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b"}, - {file = "ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d"}, - {file = "ml_dtypes-0.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328"}, - {file = "ml_dtypes-0.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175"}, - {file = "ml_dtypes-0.5.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6"}, - {file = "ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d"}, - {file = "ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298"}, - {file = "ml_dtypes-0.5.4-cp313-cp313t-win_amd64.whl", hash = "sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6"}, - {file = "ml_dtypes-0.5.4-cp313-cp313t-win_arm64.whl", hash = "sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1"}, - {file = "ml_dtypes-0.5.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2b857d3af6ac0d39db1de7c706e69c7f9791627209c3d6dedbfca8c7e5faec22"}, - {file = "ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465"}, - {file = "ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f"}, - {file = "ml_dtypes-0.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:8c6a2dcebd6f3903e05d51960a8058d6e131fe69f952a5397e5dbabc841b6d56"}, - {file = "ml_dtypes-0.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:5a0f68ca8fd8d16583dfa7793973feb86f2fbb56ce3966daf9c9f748f52a2049"}, - {file = "ml_dtypes-0.5.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:bfc534409c5d4b0bf945af29e5d0ab075eae9eecbb549ff8a29280db822f34f9"}, - {file = "ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7"}, - {file = "ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf"}, - {file = "ml_dtypes-0.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4381fe2f2452a2d7589689693d3162e876b3ddb0a832cde7a414f8e1adf7eab1"}, - {file = "ml_dtypes-0.5.4-cp314-cp314t-win_arm64.whl", hash = "sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d"}, - {file = "ml_dtypes-0.5.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d81fdb088defa30eb37bf390bb7dde35d3a83ec112ac8e33d75ab28cc29dd8b0"}, - {file = "ml_dtypes-0.5.4-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88c982aac7cb1cbe8cbb4e7f253072b1df872701fcaf48d84ffbb433b6568f24"}, - {file = "ml_dtypes-0.5.4-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9b61c19040397970d18d7737375cffd83b1f36a11dd4ad19f83a016f736c3ef"}, - {file = "ml_dtypes-0.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:3d277bf3637f2a62176f4575512e9ff9ef51d00e39626d9fe4a161992f355af2"}, - {file = "ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453"}, -] - -[package.dependencies] -numpy = [ - {version = ">=1.21.2", markers = "python_version >= \"3.10\""}, - {version = ">=1.23.3", markers = "python_version >= \"3.11\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, -] - -[package.extras] -dev = ["absl-py", "pyink", "pylint (>=2.6.0)", "pytest", "pytest-xdist"] - [[package]] name = "mpi4py" version = "4.1.2" @@ -3074,6 +3008,39 @@ pyspark-connect = ["pyspark[connect] (>=3.5.0)"] sql = ["narwhals[duckdb]", "sqlparse (>=0.5.5)"] sqlframe = ["sqlframe (>=3.22.0,!=3.39.3)"] +[[package]] +name = "nccl4py" +version = "0.3.1" +description = "NCCL4Py: Python bindings for NCCL" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "nccl4py-0.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:15517f1255a84cf3f48d36e406ff925afa40425925a1d68a7672621286488e0a"}, + {file = "nccl4py-0.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3115d8471cf4c207720c624174a0c023d8ef9893452310e6447dbc78a9e04ee"}, + {file = "nccl4py-0.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8e247d3a2b6e0253567322d518ec94320caa92649475416abceefaf0bc2db71"}, + {file = "nccl4py-0.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7945acf8c0224f727f87db40eb3a6060ad7809256b0f37dde6b94d9d8fa6b65c"}, + {file = "nccl4py-0.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0b1bab08b374ba21bb36612710866173e703c06dc197ab13b1e093436ac27ce"}, + {file = "nccl4py-0.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f96117a0aed13744d2636760962f1cb45be9138846023b69c5a8053e531cc76"}, + {file = "nccl4py-0.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50068bb4e6f60dd831b2d394a9789f08a6f4346f0e51a9d717536868a8fdd398"}, + {file = "nccl4py-0.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fd7862777777162f4735b85472951d91b8847cd7010d8f60bfefe58c7285583"}, + {file = "nccl4py-0.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13db9f786c7919ed1df7079c03acbe49eb569d5625fd5cec2344075f37b5e140"}, + {file = "nccl4py-0.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b684a3ac083fd76bf1e57e69e76575502bcd2aface23e2dad7a88461ce8916d"}, + {file = "nccl4py-0.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa09f12a93e0eb7b2dbbffbcb0a3daeeb3c211f2433cc6cc896faeec85ea4b2e"}, + {file = "nccl4py-0.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c58bf4db2eb27636587b3ce899fe47d72c16bf1eeb76c5e1c9bb1feaf57048f5"}, +] + +[package.dependencies] +"cuda.core" = ">=1.0,<2.0" +cuda-pathfinder = ">=1.5.4,<2.0.0" +numpy = "*" +packaging = "*" +typing_extensions = {version = "*", markers = "python_version < \"3.13\""} + +[package.extras] +cu12 = ["cuda-bindings (>=12.0,<13.0)", "nvidia-cutlass-dsl (>=4.5.2,<5.0)", "nvidia-nccl-cu12"] +cu13 = ["cuda-bindings (>=13.0,<14.0)", "nvidia-cutlass-dsl[cu13] (>=4.5.2,<5.0)", "nvidia-nccl-cu13"] + [[package]] name = "networkx" version = "3.4.2" @@ -3162,76 +3129,6 @@ files = [ llvmlite = "==0.48.*" numpy = ">=1.22,<2.5" -[[package]] -name = "numexpr" -version = "2.14.1" -description = "Fast numerical expression evaluator for NumPy" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "numexpr-2.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d0fab3fd06a04f6b86102552b26aa5d85e20ac7d8296c15764c726eeabae6cc8"}, - {file = "numexpr-2.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:64ae5dfd62d74a3ef82fe0b37f80527247f3626171ad82025900f46ffca4b39a"}, - {file = "numexpr-2.14.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:955c92b064f9074d2970cf3138f5e3b965be673b82024962ed526f39bc25a920"}, - {file = "numexpr-2.14.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:75440c54fc01e130396650fdf307aa9d41a67dc06ddbfb288971b591c13a395b"}, - {file = "numexpr-2.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dde9fa47ed319e1e1728940a539df3cb78326b7754bc7c6ab3152afc91808f9b"}, - {file = "numexpr-2.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76db0bc6267e591ab9c4df405ffb533598e4c88239db7338d11ae9e4b368a85a"}, - {file = "numexpr-2.14.1-cp310-cp310-win32.whl", hash = "sha256:0d1dcbdc4d0374c0d523cee2f94f06b001623cbc1fd163612841017a3495427c"}, - {file = "numexpr-2.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:823cd82c8e7937981339f634e7a9c6a92cb2d0b9d0a5cf627a5e394fffc05377"}, - {file = "numexpr-2.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2d03fcb4644a12f70a14d74006f72662824da5b6128bf1bcd10cc3ed80e64c34"}, - {file = "numexpr-2.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2773ee1133f77009a1fc2f34fe236f3d9823779f5f75450e183137d49f00499f"}, - {file = "numexpr-2.14.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebe4980f9494b9f94d10d2e526edc29e72516698d3bf95670ba79415492212a4"}, - {file = "numexpr-2.14.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a381e5e919a745c9503bcefffc1c7f98c972c04ec58fc8e999ed1a929e01ba6"}, - {file = "numexpr-2.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d08856cfc1b440eb1caaa60515235369654321995dd68eb9377577392020f6cb"}, - {file = "numexpr-2.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03130afa04edf83a7b590d207444f05a00363c9b9ea5d81c0f53b1ea13fad55a"}, - {file = "numexpr-2.14.1-cp311-cp311-win32.whl", hash = "sha256:db78fa0c9fcbaded3ae7453faf060bd7a18b0dc10299d7fcd02d9362be1213ed"}, - {file = "numexpr-2.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:e9b2f957798c67a2428be96b04bce85439bed05efe78eb78e4c2ca43737578e7"}, - {file = "numexpr-2.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:91ebae0ab18c799b0e6b8c5a8d11e1fa3848eb4011271d99848b297468a39430"}, - {file = "numexpr-2.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:47041f2f7b9e69498fb311af672ba914a60e6e6d804011caacb17d66f639e659"}, - {file = "numexpr-2.14.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d686dfb2c1382d9e6e0ee0b7647f943c1886dba3adbf606c625479f35f1956c1"}, - {file = "numexpr-2.14.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee6d4fbbbc368e6cdd0772734d6249128d957b3b8ad47a100789009f4de7083"}, - {file = "numexpr-2.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3a2839efa25f3c8d4133252ea7342d8f81226c7c4dda81f97a57e090b9d87a48"}, - {file = "numexpr-2.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9f9137f1351b310436662b5dc6f4082a245efa8950c3b0d9008028df92fefb9b"}, - {file = "numexpr-2.14.1-cp312-cp312-win32.whl", hash = "sha256:36f8d5c1bd1355df93b43d766790f9046cccfc1e32b7c6163f75bcde682cda07"}, - {file = "numexpr-2.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:fdd886f4b7dbaf167633ee396478f0d0aa58ea2f9e7ccc3c6431019623e8d68f"}, - {file = "numexpr-2.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:09078ba73cffe94745abfbcc2d81ab8b4b4e9d7bfbbde6cac2ee5dbf38eee222"}, - {file = "numexpr-2.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dce0b5a0447baa7b44bc218ec2d7dcd175b8eee6083605293349c0c1d9b82fb6"}, - {file = "numexpr-2.14.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:06855053de7a3a8425429bd996e8ae3c50b57637ad3e757e0fa0602a7874be30"}, - {file = "numexpr-2.14.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f9366d23a2e991fd5a8b5e61a17558f028ba86158a4552f8f239b005cdf83c"}, - {file = "numexpr-2.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c5f1b1605695778896534dfc6e130d54a65cd52be7ed2cd0cfee3981fd676bf5"}, - {file = "numexpr-2.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a4ba71db47ea99c659d88ee6233fa77b6dc83392f1d324e0c90ddf617ae3f421"}, - {file = "numexpr-2.14.1-cp313-cp313-win32.whl", hash = "sha256:638dce8320f4a1483d5ca4fda69f60a70ed7e66be6e68bc23fb9f1a6b78a9e3b"}, - {file = "numexpr-2.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:9fdcd4735121658a313f878fd31136d1bfc6a5b913219e7274e9fca9f8dac3bb"}, - {file = "numexpr-2.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:557887ad7f5d3c2a40fd7310e50597045a68e66b20a77b3f44d7bc7608523b4b"}, - {file = "numexpr-2.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:af111c8fe6fc55d15e4c7cab11920fc50740d913636d486545b080192cd0ad73"}, - {file = "numexpr-2.14.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33265294376e7e2ae4d264d75b798a915d2acf37b9dd2b9405e8b04f84d05cfc"}, - {file = "numexpr-2.14.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83647d846d3eeeb9a9255311236135286728b398d0d41d35dedb532dca807fe9"}, - {file = "numexpr-2.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6e575fd3ad41ddf3355d0c7ef6bd0168619dc1779a98fe46693cad5e95d25e6e"}, - {file = "numexpr-2.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:67ea4771029ce818573b1998f5ca416bd255156feea017841b86176a938f7d19"}, - {file = "numexpr-2.14.1-cp313-cp313t-win32.whl", hash = "sha256:15015d47d3d1487072d58c0e7682ef2eb608321e14099c39d52e2dd689483611"}, - {file = "numexpr-2.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:94c711f6d8f17dfb4606842b403699603aa591ab9f6bf23038b488ea9cfb0f09"}, - {file = "numexpr-2.14.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ede79f7ff06629f599081de644546ce7324f1581c09b0ac174da88a470d39c21"}, - {file = "numexpr-2.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2eac7a5a2f70b3768c67056445d1ceb4ecd9b853c8eda9563823b551aeaa5082"}, - {file = "numexpr-2.14.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5aedf38d4c0c19d3cecfe0334c3f4099fb496f54c146223d30fa930084bc8574"}, - {file = "numexpr-2.14.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439ec4d57b853792ebe5456e3160312281c3a7071ecac5532ded3278ede614de"}, - {file = "numexpr-2.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e23b87f744e04e302d82ac5e2189ae20a533566aec76a46885376e20b0645bf8"}, - {file = "numexpr-2.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:44f84e0e5af219dbb62a081606156420815890e041b87252fbcea5df55214c4c"}, - {file = "numexpr-2.14.1-cp314-cp314-win32.whl", hash = "sha256:1f1a5e817c534539351aa75d26088e9e1e0ef1b3a6ab484047618a652ccc4fc3"}, - {file = "numexpr-2.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:587c41509bc373dfb1fe6086ba55a73147297247bedb6d588cda69169fc412f2"}, - {file = "numexpr-2.14.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:ec368819502b64f190c3f71be14a304780b5935c42aae5bf22c27cc2cbba70b5"}, - {file = "numexpr-2.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7e87f6d203ac57239de32261c941e9748f9309cbc0da6295eabd0c438b920d3a"}, - {file = "numexpr-2.14.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd72d8c2a165fe45ea7650b16eb8cc1792a94a722022006bb97c86fe51fd2091"}, - {file = "numexpr-2.14.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70d80fcb418a54ca208e9a38e58ddc425c07f66485176b261d9a67c7f2864f73"}, - {file = "numexpr-2.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:edea2f20c2040df8b54ee8ca8ebda63de9545b2112872466118e9df4d0ae99f3"}, - {file = "numexpr-2.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:790447be6879a6c51b9545f79612d24c9ea0a41d537a84e15e6a8ddef0b6268e"}, - {file = "numexpr-2.14.1-cp314-cp314t-win32.whl", hash = "sha256:538961096c2300ea44240209181e31fae82759d26b51713b589332b9f2a4117e"}, - {file = "numexpr-2.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:a40b350cd45b4446076fa11843fa32bbe07024747aeddf6d467290bf9011b392"}, - {file = "numexpr-2.14.1.tar.gz", hash = "sha256:4be00b1086c7b7a5c32e31558122b7b80243fe098579b170967da83f3152b48b"}, -] - -[package.dependencies] -numpy = ">=1.23.0" - [[package]] name = "numpy" version = "2.2.6" @@ -3604,42 +3501,6 @@ files = [ {file = "nvidia_ml_py-13.610.43.tar.gz", hash = "sha256:65437eb73d68d0c62c931ca4d45038472faff03bd0b8729abba4b899f70d60f2"}, ] -[[package]] -name = "nvidia-modelopt" -version = "0.37.0" -description = "Nvidia TensorRT Model Optimizer: a unified model optimization and deployment toolkit." -optional = false -python-versions = "<3.13,>=3.10" -groups = ["main"] -files = [ - {file = "nvidia_modelopt-0.37.0-py3-none-any.whl", hash = "sha256:3490b6d6aea3541aa5d475d81230fee627e2c16ff47bbab1cba4b80a1eb119a2"}, -] - -[package.dependencies] -ninja = "*" -numpy = "*" -nvidia-ml-py = ">=12" -packaging = "*" -pulp = "*" -pydantic = ">=2.0" -regex = "*" -rich = "*" -safetensors = "*" -scipy = "*" -torch = ">=2.6" -torchprofile = ">=0.0.4" -tqdm = "*" - -[package.extras] -all = ["accelerate (>=1.0.0)", "cppimport", "cupy-cuda12x ; platform_machine != \"aarch64\" and platform_system != \"Darwin\"", "datasets (>=3.0.0)", "deepspeed (>=0.9.6) ; platform_system != \"Darwin\" and platform_system != \"Windows\"", "diffusers (>=0.32.2)", "huggingface_hub (>=0.24.0)", "ml_dtypes", "onnx (>=1.19.0,<1.20.0)", "onnx-graphsurgeon", "onnxconverter-common (>=1.16.0,<1.17.0)", "onnxruntime (>=1.22.0,<1.23.0) ; platform_machine == \"aarch64\" or platform_system == \"Darwin\"", "onnxruntime-directml (==1.20.0) ; platform_system == \"Windows\"", "onnxruntime-gpu (>=1.22.0,<1.23.0) ; platform_machine != \"aarch64\" and platform_system != \"Darwin\" and platform_system != \"Windows\"", "onnxscript", "onnxsim ; python_version < \"3.12\" and platform_machine != \"aarch64\"", "peft (>=0.17.0)", "polygraphy (>=0.49.22)", "transformers (>=4.48,<4.57)"] -dev = ["accelerate (>=1.0.0)", "accelerate (>=1.0.0)", "autodoc_pydantic (>=2.1.0)", "bandit[toml] (==1.7.9)", "coverage", "cppimport", "cppimport", "cupy-cuda12x ; platform_machine != \"aarch64\" and platform_system != \"Darwin\"", "cupy-cuda12x ; platform_machine != \"aarch64\" and platform_system != \"Darwin\"", "cython", "datasets (>=3.0.0)", "datasets (>=3.0.0)", "deepspeed (>=0.9.6) ; platform_system != \"Darwin\" and platform_system != \"Windows\"", "deepspeed (>=0.9.6) ; platform_system != \"Darwin\" and platform_system != \"Windows\"", "diffusers (>=0.32.2)", "diffusers (>=0.32.2)", "huggingface_hub (>=0.24.0)", "huggingface_hub (>=0.24.0)", "ml_dtypes", "ml_dtypes", "mypy (==1.17.1)", "onnx (>=1.19.0,<1.20.0)", "onnx (>=1.19.0,<1.20.0)", "onnx-graphsurgeon", "onnx-graphsurgeon", "onnxconverter-common (>=1.16.0,<1.17.0)", "onnxconverter-common (>=1.16.0,<1.17.0)", "onnxruntime (>=1.22.0,<1.23.0) ; platform_machine == \"aarch64\" or platform_system == \"Darwin\"", "onnxruntime (>=1.22.0,<1.23.0) ; platform_machine == \"aarch64\" or platform_system == \"Darwin\"", "onnxruntime-directml (==1.20.0) ; platform_system == \"Windows\"", "onnxruntime-directml (==1.20.0) ; platform_system == \"Windows\"", "onnxruntime-gpu (>=1.22.0,<1.23.0) ; platform_machine != \"aarch64\" and platform_system != \"Darwin\" and platform_system != \"Windows\"", "onnxruntime-gpu (>=1.22.0,<1.23.0) ; platform_machine != \"aarch64\" and platform_system != \"Darwin\" and platform_system != \"Windows\"", "onnxscript", "onnxscript", "onnxsim ; python_version < \"3.12\" and platform_machine != \"aarch64\"", "onnxsim ; python_version < \"3.12\" and platform_machine != \"aarch64\"", "peft (>=0.17.0)", "peft (>=0.17.0)", "polygraphy (>=0.49.22)", "polygraphy (>=0.49.22)", "pre-commit (==4.3.0)", "pytest", "pytest-cov", "pytest-timeout", "ruff (==0.12.11)", "setuptools (>=80)", "setuptools-scm (>=8)", "sphinx (>=8.1.0,<8.2.0)", "sphinx-argparse (>=0.5.2)", "sphinx-autobuild (>=2024.10.3)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinx-rtd-theme (>=3.0.0,<3.1.0)", "sphinx-togglebutton (>=0.3.2)", "timm", "torchvision", "tox (>4.18)", "tox-current-env (>=0.0.12)", "transformers (>=4.48,<4.57)", "transformers (>=4.48,<4.57)"] -dev-build = ["cython", "setuptools (>=80)", "setuptools-scm (>=8)"] -dev-docs = ["autodoc_pydantic (>=2.1.0)", "sphinx (>=8.1.0,<8.2.0)", "sphinx-argparse (>=0.5.2)", "sphinx-autobuild (>=2024.10.3)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinx-rtd-theme (>=3.0.0,<3.1.0)", "sphinx-togglebutton (>=0.3.2)"] -dev-lint = ["bandit[toml] (==1.7.9)", "mypy (==1.17.1)", "pre-commit (==4.3.0)", "ruff (==0.12.11)"] -dev-test = ["coverage", "pytest", "pytest-cov", "pytest-timeout", "timm", "torchvision", "tox (>4.18)", "tox-current-env (>=0.0.12)"] -hf = ["accelerate (>=1.0.0)", "datasets (>=3.0.0)", "deepspeed (>=0.9.6) ; platform_system != \"Darwin\" and platform_system != \"Windows\"", "diffusers (>=0.32.2)", "huggingface_hub (>=0.24.0)", "peft (>=0.17.0)", "transformers (>=4.48,<4.57)"] -onnx = ["cppimport", "cupy-cuda12x ; platform_machine != \"aarch64\" and platform_system != \"Darwin\"", "ml_dtypes", "onnx (>=1.19.0,<1.20.0)", "onnx-graphsurgeon", "onnxconverter-common (>=1.16.0,<1.17.0)", "onnxruntime (>=1.22.0,<1.23.0) ; platform_machine == \"aarch64\" or platform_system == \"Darwin\"", "onnxruntime-directml (==1.20.0) ; platform_system == \"Windows\"", "onnxruntime-gpu (>=1.22.0,<1.23.0) ; platform_machine != \"aarch64\" and platform_system != \"Darwin\" and platform_system != \"Windows\"", "onnxscript", "onnxsim ; python_version < \"3.12\" and platform_machine != \"aarch64\"", "polygraphy (>=0.49.22)"] - [[package]] name = "nvidia-nccl-cu13" version = "2.28.9" @@ -3748,75 +3609,16 @@ files = [ antlr4-python3-runtime = "==4.9.*" PyYAML = ">=5.1.0" -[[package]] -name = "onnx" -version = "1.22.0" -description = "Open Neural Network Exchange" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "onnx-1.22.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:6d0ffffd63a4ecc21ddaeddd5bf02099cb701aa4243f2de00122726869065ca4"}, - {file = "onnx-1.22.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33ce94119bbb7f05d9caea4ea7549f5185a54369f6bbc9f70171bd5ee6935bbc"}, - {file = "onnx-1.22.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87a3077958f66f9a26dec10077ac28326d9cec2cbe1f0b040947243449754573"}, - {file = "onnx-1.22.0-cp310-cp310-win32.whl", hash = "sha256:8a5eccce2d5fc6c5046928a9aa7cdd9750ea4a586f8de341d3d40d820c35fdec"}, - {file = "onnx-1.22.0-cp310-cp310-win_amd64.whl", hash = "sha256:5c1c0408a9d4b4df33851672e5fc7590b96301ee123396d608f9ab6f045ab06b"}, - {file = "onnx-1.22.0-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:2d8f229a553fa440fe623ed7b36fca5e7762da3af871c3f8f8ce451df73e2914"}, - {file = "onnx-1.22.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1a89a7cb9ba13d78f009bdec448ec82a98972589734f157022a2bff7a5973a6"}, - {file = "onnx-1.22.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d0a2bdb15eb2b3cb65c438f3423d9620d14fdce32f92380e6bb1b2e09568ef5"}, - {file = "onnx-1.22.0-cp311-cp311-win32.whl", hash = "sha256:239958534464612fbcb6ed23d5228aaa925b39b8773f58726809ffdccb4edd1c"}, - {file = "onnx-1.22.0-cp311-cp311-win_amd64.whl", hash = "sha256:8561a2c00041c07e08db0c228593b5b4694100398685f348532af7dbb84189da"}, - {file = "onnx-1.22.0-cp311-cp311-win_arm64.whl", hash = "sha256:8907b9b9389893bc0dc6314cc00ee1e3a69844e48d689eacc6a0340411a7da58"}, - {file = "onnx-1.22.0-cp312-abi3-macosx_12_0_universal2.whl", hash = "sha256:596fbf0490947533c1c1045ba860851dc9fb77471023dac9a71ba5b42ceab103"}, - {file = "onnx-1.22.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae5a563f281cd9d2845622cecf6c092a57e4ee1b138f66fdbbdd4200567a5e16"}, - {file = "onnx-1.22.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:955e02e1f6d385b53d52f9cd7b9cdf5caf417c300bcfe3c64c6d542be763845b"}, - {file = "onnx-1.22.0-cp312-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:82e9f27fc1223cb06d68a56bed6f9d3caf3d0dad1b61bce45006d529b15bd94c"}, - {file = "onnx-1.22.0-cp312-abi3-win32.whl", hash = "sha256:cc8b66b312f8f03a53e268afb67180a2d97dd12cc79e2b61361c6c0073448016"}, - {file = "onnx-1.22.0-cp312-abi3-win_amd64.whl", hash = "sha256:72ccebab3bac07215c204ce8848d42e78eaaa666badbf72d25cd359b9f269e3a"}, - {file = "onnx-1.22.0-cp312-abi3-win_arm64.whl", hash = "sha256:f3c120dcdb70ad738f3c061b32798f408ea299eb69f84dd69ab4a6bf3c2ec01f"}, - {file = "onnx-1.22.0-cp314-cp314t-macosx_12_0_universal2.whl", hash = "sha256:19e45e4af88e3fe3261458d4b8cc461957ae2782a358a3560503569bf3b23b72"}, - {file = "onnx-1.22.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c21a0e59fd967a95b358e4a6e756d1f1eec2d304a83480f329f66e30d2bf0223"}, - {file = "onnx-1.22.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2632406b8f523ef2e2873c363f90b20a3d88c0fbcfac757d3addffccf8f452c2"}, - {file = "onnx-1.22.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a3a39fc4643867aecb33417fdddb11e308ee79d2d4a584b9d50cc7aec2091b13"}, - {file = "onnx-1.22.0-cp314-cp314t-win_arm64.whl", hash = "sha256:8e268cdc0547e3949799ffd4a44451dc2b9080b57d0824a2db680b6ec65506f0"}, - {file = "onnx-1.22.0.tar.gz", hash = "sha256:ef40c0aaf0b643857ea9306fc7eddce17eaf9fb0407e4801f1fc5758443a38e0"}, -] - -[package.dependencies] -ml_dtypes = ">=0.5.4" -numpy = ">=1.23.2" -protobuf = ">=4.25.1" -typing_extensions = ">=4.15.0" - -[package.extras] -reference = ["Pillow"] - -[[package]] -name = "onnx-graphsurgeon" -version = "0.6.1" -description = "ONNX GraphSurgeon" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "onnx_graphsurgeon-0.6.1-py2.py3-none-any.whl", hash = "sha256:fabc53fc60909dd032cfd889016dd5d4139ab566af4a9039002818f552c73547"}, -] - -[package.dependencies] -ml-dtypes = "*" -numpy = "*" -onnx = ">=1.14.0" - [[package]] name = "openai" -version = "2.45.0" +version = "2.49.0" description = "The official Python library for the openai API" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "openai-2.45.0-py3-none-any.whl", hash = "sha256:5df105f5f8c9b711fcb9d06d2d3888cebc82506db216484c14a4e53cdf651777"}, - {file = "openai-2.45.0.tar.gz", hash = "sha256:10d34ca9c5643bce775852fddbfc172505cb1d4de1ccd101696c3ecff358765d"}, + {file = "openai-2.49.0-py3-none-any.whl", hash = "sha256:b694201eaa42a1ccf2aa125fe29458150108fb22df1abfb55d7188599da81d8c"}, + {file = "openai-2.49.0.tar.gz", hash = "sha256:80f934333b5b83cef2fde9af7151dacaa72e150f43f92b7675f7647ca6157f48"}, ] [package.dependencies] @@ -3830,9 +3632,10 @@ tqdm = ">4" typing-extensions = ">=4.14,<5" [package.extras] -aiohttp = ["aiohttp", "httpx-aiohttp (>=0.1.9)"] -bedrock = ["botocore (>=1.40.0,<1.43) ; python_version < \"3.10\"", "botocore (>=1.40.0,<2) ; python_version >= \"3.10\""] +aiohttp = ["aiohttp (>=3.14.1)", "httpx-aiohttp (>=0.1.9)"] +bedrock = ["botocore (>=1.40.0,<2)"] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +httpx2 = ["anyio (>=4.10.0,<5)", "httpx (>=0.25.1,<1)", "httpx2 (>=2.7.0,<3)"] realtime = ["websockets (>=13,<16)"] voice-helpers = ["numpy (>=2.0.2)", "sounddevice (>=0.5.1)"] @@ -4031,61 +3834,55 @@ xml = ["lxml (>=4.9.2)"] [[package]] name = "pandas" -version = "3.0.3" +version = "3.0.5" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.11" groups = ["main"] markers = "python_version >= \"3.11\"" files = [ - {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, - {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, - {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, - {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, - {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, - {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, - {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, - {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, - {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, - {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, - {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, - {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, - {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, - {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, - {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, - {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, - {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, - {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, - {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, - {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, - {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, - {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, - {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, - {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, - {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, - {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, - {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, - {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, - {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, - {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, - {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, + {file = "pandas-3.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2946e77e4a53cd248cbde631a12f0e51c8324ce354c3eba4d20147c1ad6f4282"}, + {file = "pandas-3.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71ecc8fb7ed1a7aa4392316b5309a6347e8e7f832f38fd897846b3a1457a9298"}, + {file = "pandas-3.0.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b173f5951ff6b8b0ec7675e20dff3c97b7e7a57dfcce387c2d7c5afe87cb7899"}, + {file = "pandas-3.0.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c0cf1dd9b55a22d105fc46c1b489af3bd42264fcba7c66297bf47a9a1d9c78a"}, + {file = "pandas-3.0.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0fac0010c75e4efb6b99e249c183a8993ce0dc95c240f9b120a5e67c727b7928"}, + {file = "pandas-3.0.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:08d24fe11a17dc33bd6e937dc9c665f9cba08fbdc9f657f405713515febe300d"}, + {file = "pandas-3.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b1261758dfb6cf12c3cff8300e21cefad30e7ec709abb4c24ac7318e6a52462a"}, + {file = "pandas-3.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:679f4e85b30ddb1515458ab1e788d3e260eae369b1f78da7a3aa4cac8ebf4a2a"}, + {file = "pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d"}, + {file = "pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc"}, + {file = "pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc"}, + {file = "pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34"}, + {file = "pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6"}, + {file = "pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7"}, + {file = "pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce"}, + {file = "pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41"}, + {file = "pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49"}, + {file = "pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b"}, + {file = "pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3"}, + {file = "pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b"}, + {file = "pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be"}, + {file = "pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58"}, + {file = "pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee"}, + {file = "pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6"}, + {file = "pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e"}, + {file = "pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36"}, + {file = "pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c"}, + {file = "pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0"}, + {file = "pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b"}, + {file = "pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94"}, + {file = "pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da"}, + {file = "pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92"}, + {file = "pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85"}, + {file = "pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca"}, + {file = "pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da"}, + {file = "pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a"}, + {file = "pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040"}, + {file = "pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd"}, + {file = "pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c"}, + {file = "pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea"}, + {file = "pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c"}, + {file = "pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712"}, ] [package.dependencies] @@ -4115,7 +3912,7 @@ postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psyc pyarrow = ["pyarrow (>=13.0.0)"] spss = ["pyreadstat (>=1.2.8)"] sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] -test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] +test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4,<9.1)", "pytest-xdist (>=3.6.1)"] timezone = ["pytz (>=2020.1)"] xml = ["lxml (>=5.3.0)"] @@ -4293,14 +4090,14 @@ xmp = ["defusedxml"] [[package]] name = "platformdirs" -version = "4.10.0" +version = "4.11.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, - {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, + {file = "platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74"}, + {file = "platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0"}, ] [[package]] @@ -4366,14 +4163,14 @@ xxhash = ["xxhash (>=1.4.3)"] [[package]] name = "prometheus-client" -version = "0.25.0" +version = "0.26.0" description = "Python client for the Prometheus monitoring system." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1"}, - {file = "prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28"}, + {file = "prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6"}, + {file = "prometheus_client-0.26.0.tar.gz", hash = "sha256:04a91bcf94e2cf74a44a1a874d651a2e853ed354b6e822f3b7487751465d5c2b"}, ] [package.extras] @@ -4383,14 +4180,14 @@ twisted = ["twisted"] [[package]] name = "prometheus-fastapi-instrumentator" -version = "8.0.2" +version = "8.1.0" description = "Instrument your FastAPI app with Prometheus metrics" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "prometheus_fastapi_instrumentator-8.0.2-py3-none-any.whl", hash = "sha256:746002ec1e2c58b93f61444e1d104de959a9463a6a3f1c8909ac3757e16c3866"}, - {file = "prometheus_fastapi_instrumentator-8.0.2.tar.gz", hash = "sha256:3c252e748151768a7aefd66824a04a870144f71de48a67aed211749a9ca2a548"}, + {file = "prometheus_fastapi_instrumentator-8.1.0-py3-none-any.whl", hash = "sha256:b9f40b2cff3f7891ca0610b3ae4fc6ec723fd326b04bb659819aaeb821a0fc7d"}, + {file = "prometheus_fastapi_instrumentator-8.1.0.tar.gz", hash = "sha256:b77f3043665e8d28e2bbd21017506195a43d9adf1d402d01bf95b494b7e560e1"}, ] [package.dependencies] @@ -5031,15 +4828,15 @@ files = [ [[package]] name = "pytz" -version = "2026.2" +version = "2026.3.post1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" groups = ["main"] markers = "python_version == \"3.10\"" files = [ - {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, - {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, + {file = "pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815"}, + {file = "pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d"}, ] [[package]] @@ -5305,126 +5102,126 @@ typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} [[package]] name = "regex" -version = "2026.7.10" +version = "2026.7.19" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, + {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, + {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, + {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, + {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, + {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, + {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, + {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, + {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, + {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, + {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, + {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, + {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, + {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, + {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, + {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, + {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, + {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, + {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, + {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, + {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, + {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, + {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, ] [[package]] @@ -6297,14 +6094,14 @@ test = ["pytest"] [[package]] name = "sse-starlette" -version = "3.4.5" +version = "3.4.6" description = "SSE plugin for Starlette" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "sse_starlette-3.4.5-py3-none-any.whl", hash = "sha256:e71bad53323f65573c3864a6c3bd0c1eb6e5f092b2e48082b0c35927d19ca296"}, - {file = "sse_starlette-3.4.5.tar.gz", hash = "sha256:83072538bc211a2f68b7b0422226c4af3e9b62e106e07034664b832ca019842a"}, + {file = "sse_starlette-3.4.6-py3-none-any.whl", hash = "sha256:56217ab4c9a9f9c5db7b21e08732d3e7c2b807f45231ad23de0551a24c4a41f6"}, + {file = "sse_starlette-3.4.6.tar.gz", hash = "sha256:725f8a1bd6d26ae1b2c9610c0ef5065dfdd496f3988d28adcf8c4b49dc25c627"}, ] [package.dependencies] @@ -6387,80 +6184,6 @@ files = [ [package.extras] widechars = ["wcwidth"] -[[package]] -name = "tensorrt" -version = "10.16.1.11" -description = "TensorRT Metapackage" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tensorrt-10.16.1.11.tar.gz", hash = "sha256:5c31ef98e1a1b53197acc600327d6b1e4abb503024119a2e66f21a5654ea3996"}, -] - -[package.dependencies] -tensorrt_cu13 = "10.16.1.11" - -[[package]] -name = "tensorrt-cu13" -version = "10.16.1.11" -description = "A high performance deep learning inference library" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tensorrt_cu13-10.16.1.11.tar.gz", hash = "sha256:5d34203a92f38851b150c4eddd32b9b55c01fd40fc4d19bff83e8dfef1d8f69e"}, -] - -[package.dependencies] -tensorrt_cu13_bindings = "10.16.1.11" -tensorrt_cu13_libs = "10.16.1.11" - -[package.extras] -numpy = ["numpy"] - -[[package]] -name = "tensorrt-cu13-bindings" -version = "10.16.1.11" -description = "A high performance deep learning inference library" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "tensorrt_cu13_bindings-10.16.1.11-cp310-none-manylinux_2_28_x86_64.whl", hash = "sha256:c8e171511a01a2678cb3e00ca4792bc1d7ea6c48da9781ab127970f6ad1edc4a"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp310-none-manylinux_2_35_aarch64.whl", hash = "sha256:9a7330fecbc0fff40ff227d0f2ee2d2d5f6fd55aafdd9fe0364e4e06fa4a9612"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp310-none-win_amd64.whl", hash = "sha256:631c7f0c979d740696a6daf1c17f1fd128863939cc3720282cc53d21cdd05e33"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp311-none-manylinux_2_28_x86_64.whl", hash = "sha256:f986d47980d042cc126506a9b84586e55ad13737a7a46826fe45457c04d66294"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp311-none-manylinux_2_35_aarch64.whl", hash = "sha256:6fa40558659d027ef6d71b475f012e0531f3d69c7b27b4a2b92779cc6de24a9a"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp311-none-win_amd64.whl", hash = "sha256:483a58b8cf3a7e97d66d7245084d08f9ee9cd19d09210305ed43b34fbb721f7c"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp312-none-manylinux_2_28_x86_64.whl", hash = "sha256:2fb87ac24a5e5f42e43f283c16d1b661dc72ef18568fd847ce2080b7db9ca232"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp312-none-manylinux_2_35_aarch64.whl", hash = "sha256:80d0cea41cb695e73a442ab5a0c1fb023730c488caa93ee3ca9c49d57314821d"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp312-none-win_amd64.whl", hash = "sha256:70ca2d73301ff427104d7986c26d5fa56f6a91578242d20e82b37c139be14ec0"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp313-none-manylinux_2_28_x86_64.whl", hash = "sha256:da17c222ba99f46e77e1dab568cd17c4652b0432d4fae074e4c79285356aa0ac"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp313-none-manylinux_2_35_aarch64.whl", hash = "sha256:2cba1722377323516a46ed15859deaf03f20ab2a47e63a89d69b3898d110fcac"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp313-none-win_amd64.whl", hash = "sha256:44b44154027e5e9e39d74e14b453704af50ba774b3775704f6d40408c8457526"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp38-none-manylinux_2_28_x86_64.whl", hash = "sha256:74ad6eb30fc6db3cc72eeacf7e989f887905d89eba92dcefaa0bf7310efba707"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp38-none-manylinux_2_35_aarch64.whl", hash = "sha256:50ea9129ac563d9bcfafb495b8135710107da795c664f499d1072d39ab09e5fb"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp38-none-win_amd64.whl", hash = "sha256:3fd27a1441173257eaa20b41562981a0bbcd1914bce4fab22a5feae1d352facb"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp39-none-manylinux_2_28_x86_64.whl", hash = "sha256:fbbd3ccfcb794c7aafec6900d6eefe289d783d89c065d8af57f9d11724307e51"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp39-none-manylinux_2_35_aarch64.whl", hash = "sha256:cc085106269123f6fa75998095671e43717df55289971cee68a1984a3c14f2bc"}, - {file = "tensorrt_cu13_bindings-10.16.1.11-cp39-none-win_amd64.whl", hash = "sha256:2ab1dd2297046ae3d4a88f0c5a5b0601bb03928b558a72f158d605fbfd0bb285"}, -] - -[package.extras] -numpy = ["numpy"] - -[[package]] -name = "tensorrt-cu13-libs" -version = "10.16.1.11" -description = "TensorRT Libraries" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "tensorrt_cu13_libs-10.16.1.11.tar.gz", hash = "sha256:0e86cd02321b7258c2521495a7d8f0591852e6966ed7e43cfe33640c737495a3"}, -] - [[package]] name = "threadpoolctl" version = "3.6.0" @@ -6759,34 +6482,16 @@ files = [ [package.extras] dev = ["bitsandbytes", "blobfile", "cmake (>=3.19.0,<4.0.0)", "diskcache", "expecttest", "fire", "hypothesis", "importlib_metadata", "lm_eval", "matplotlib", "ninja", "packaging", "pandas", "parameterized", "pre-commit", "pycocotools", "pytest (==8.4.2)", "ruff (==0.11.6)", "sentencepiece", "tabulate", "tiktoken", "tqdm", "transformers", "unittest-xml-reporting"] -[[package]] -name = "torchprofile" -version = "0.1.0" -description = "Count the MACs / FLOPs of PyTorch models" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "torchprofile-0.1.0-py3-none-any.whl", hash = "sha256:39aefcd194927befb0cfe7ed886be565dee384e3889d0b6890bace5ae1614578"}, - {file = "torchprofile-0.1.0.tar.gz", hash = "sha256:aeb725e9446b3a6d8f82d0f0cf944a6ffb9cced4e5db1354ad3b9692ce820dac"}, -] - -[package.dependencies] -torch = ">=2.0" - -[package.extras] -test = ["pytest", "torchvision (>=0.15)", "transformers"] - [[package]] name = "tqdm" -version = "4.68.4" +version = "4.70.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, + {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, + {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, ] [package.dependencies] @@ -7056,14 +6761,14 @@ files = [ [[package]] name = "xdsl" -version = "0.68.0" +version = "0.69.0" description = "xDSL" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "xdsl-0.68.0-py3-none-any.whl", hash = "sha256:9f28d722a1828df77b71128a5a725adbaddafcf6e20deb4de85a55cb53d8e90c"}, - {file = "xdsl-0.68.0.tar.gz", hash = "sha256:e0331528e02bbd6baab529e32a1c942518d37294e14710ab141086e9d77af25f"}, + {file = "xdsl-0.69.0-py3-none-any.whl", hash = "sha256:40445d6a63d8539ba9527ffea30e5c47138fd9fa19b5f2c4eae978b5450ce76b"}, + {file = "xdsl-0.69.0.tar.gz", hash = "sha256:cc365cc1c746bd5cbf07f348e6c99e2da85a814f75ee44a91cb9caa20ea3c166"}, ] [package.dependencies] @@ -7072,9 +6777,9 @@ ordered-set = "4.1" typing-extensions = ">=4.7,<5" [package.extras] -dev = ["coverage (<8)", "filecheck (==1.0.3)", "ipykernel", "lit (<19)", "marimo (>=0.23,<0.24)", "nbconvert (>=7.7.2,<8)", "nbval (<0.12)", "prek (>=0.4.0,<0.5.0)", "pyright (==1.1.410)", "pytest (<9.2)", "pytest-asyncio", "pytest-cov", "ruff (==0.15.20)", "sympy (==1.14)", "textual-dev (==1.8)", "toml (<0.11)"] +dev = ["coverage (<8)", "filecheck (==1.0.3)", "ipykernel", "lit (<19)", "marimo (>=0.23,<0.24)", "nbconvert (>=7.7.2,<8)", "nbval (<0.12)", "prek (>=0.4.0,<0.5.0)", "pyright (==1.1.411)", "pytest (<9.2)", "pytest-asyncio", "pytest-cov", "ruff (==0.15.20)", "sympy (==1.14)", "textual-dev (==1.8)", "toml (<0.11)"] gui = ["pyclip (==0.7)", "textual (>=8,<9)"] -heir = ["heir-py (==2026.6.5.dev0)"] +heir = ["heir-py (==2026.7.1)"] llvm = ["llvmlite (>=0.47.0,<0.48.0)"] [[package]] @@ -7333,116 +7038,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.2" +version = "1.24.5" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, + {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, + {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, + {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, + {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, + {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, + {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, + {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, + {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, + {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, + {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, + {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, + {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, + {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, + {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, ] [package.dependencies] @@ -7473,4 +7178,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "b06afe749afa10cd9c5a937761c1646b266fcba96b4c99894da37a82c87726db" +content-hash = "8dd2a71905f3e3891c6fe7b0832518bd5d9a93f3c4b5c556e4f783aa8f087d72" diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index 906ab2a31586..d6b4228d3659 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -18,10 +18,8 @@ dependencies = [ "lazy-loader (>=0.5,<1.0)", "mpi4py (>=4.1.2,<5.0.0)", "numpy (>=2.0.0,<2.4)", - "onnx (>=1.21.0)", - "onnx-graphsurgeon (>=0.5.2)", "graphviz (>=0.21,<0.22)", - "openai (>=2.45.0,<3.0.0)", + "openai (>=2.49.0,<3.0.0)", "polygraphy (>=0.50.3,<0.51.0)", "psutil (>=7.2.2,<8.0.0)", "nvidia-ml-py (>=13)", @@ -29,13 +27,12 @@ dependencies = [ "h5py (==3.12.1)", "strenum (>=0.4.15,<0.5.0)", "sentencepiece (>=0.1.99)", - "tensorrt (>=10.16.1,<10.17.0)", "torch (>=2.11.0,<=2.13.0a0)", - "nvidia-modelopt[torch] (>=0.37.0,<0.38.0)", "nvidia-nccl-cu13 (>=2.28.9,<=2.30.4)", + "nccl4py (>=0.3.1,<0.4)", "transformers (==5.5.4)", - "prometheus-client (>=0.25.0,<0.26.0)", - "prometheus-fastapi-instrumentator (>=8.0.2,<9.0.0)", + "prometheus-client (>=0.26.0,<0.27.0)", + "prometheus-fastapi-instrumentator (>=8.1.0,<9.0.0)", "pydantic (>=2.9.1)", "pydantic-settings[yaml] (>=2.14.2,<3.0.0)", "msgspec (>=0.21.1,<0.22.0)", @@ -56,7 +53,7 @@ dependencies = [ "peft (>=0.18.1,<0.19.0)", "patchelf (>=0.17.2.4,<0.18.0.0)", "einops (>=0.8.2,<0.9.0)", - "flashinfer-python (==0.6.14)", + "flashinfer-python (==0.6.15)", "xgrammar (==0.1.32)", "llguidance (==0.7.29)", "jsonschema (>=4.26.0,<5.0.0)", @@ -73,8 +70,9 @@ dependencies = [ "openai-harmony (==0.0.4)", "nvidia-cutlass-dsl[cu13] (==4.5.0)", "nvidia-matmul-heuristics (==0.1.0.27)", + "quack-kernels (>=0.2.10)", + "jinja2 (>=3.1.6,<4.0.0)", "plotly (>=6.9.0,<7.0.0)", - "numexpr (>=2.14.1,<3.0.0)", "partial-json-parser (>=0.2.1.1.post7,<0.3.0.0)", "mcp (>=1.28.1,<2.0.0)", "torch-c-dlpack-ext (==0.1.3)", diff --git a/security_scanning/triton_backend/poetry.lock b/security_scanning/triton_backend/poetry.lock index 9f3c0fa0086d..160db2e9524f 100644 --- a/security_scanning/triton_backend/poetry.lock +++ b/security_scanning/triton_backend/poetry.lock @@ -14,131 +14,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.1" +version = "3.14.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, - {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, - {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, - {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, - {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, - {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, - {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, - {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, - {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, - {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, - {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, - {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, - {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, - {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, - {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, - {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, - {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, - {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, - {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, - {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, - {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, - {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, - {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, - {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, - {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, - {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, - {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, - {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, - {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, - {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, - {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, - {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, - {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, - {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, - {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a"}, + {file = "aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479"}, + {file = "aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f"}, + {file = "aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32"}, + {file = "aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c"}, + {file = "aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a"}, + {file = "aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db"}, + {file = "aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910"}, + {file = "aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7"}, + {file = "aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d"}, + {file = "aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228"}, + {file = "aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42"}, + {file = "aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d"}, + {file = "aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19"}, + {file = "aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a"}, + {file = "aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86"}, + {file = "aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c"}, + {file = "aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0"}, + {file = "aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5"}, + {file = "aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e"}, + {file = "aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71"}, + {file = "aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883"}, + {file = "aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062"}, + {file = "aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7"}, + {file = "aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646"}, + {file = "aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147"}, + {file = "aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41"}, + {file = "aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf"}, + {file = "aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100"}, + {file = "aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85"}, + {file = "aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9"}, + {file = "aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b"}, + {file = "aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7"}, + {file = "aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc"}, ] [package.dependencies] @@ -353,14 +353,14 @@ files = [ [[package]] name = "certifi" -version = "2026.6.17" +version = "2026.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"}, - {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"}, + {file = "certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] [[package]] @@ -578,13 +578,13 @@ cu13 = ["cuda-bindings[all] (==13.*)"] [[package]] name = "cuda-pathfinder" -version = "1.5.6" +version = "1.6.0" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, + {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, ] [[package]] @@ -627,14 +627,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.29.7" +version = "3.32.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, + {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, + {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, ] [[package]] @@ -837,59 +837,58 @@ tqdm = ["tqdm"] [[package]] name = "gevent" -version = "26.5.0" +version = "26.7.0" description = "Coroutine-based network library" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "gevent-26.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:2ba673dcbf7747513b58fa64ca7e9d6a828bc5c604d1552d23db89006d7911df"}, - {file = "gevent-26.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:271b1474d81bb33036631adb16a35e5a1ee9dc414b05c999d6b01dc839a89975"}, - {file = "gevent-26.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:cd3dc60581687e2618286108f8e2f820d8446be4b34131065011c066e911d39c"}, - {file = "gevent-26.5.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:dc7fa28b2d627f8e87595f39043b6dec71e8e7fb97e685e5506c47cf3ff8cb2e"}, - {file = "gevent-26.5.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:68c5fc21cef80268cdff88a4ae6c025fabb019b071f6f8ee4d20a7bccbddb873"}, - {file = "gevent-26.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d325502eb0695708ef8c899f605573ed6847f3961f8159627dba267fbf3ce457"}, - {file = "gevent-26.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a11daf3a588b932c8bf965fb18444c69aff48badec88435e988cf8d67137075a"}, - {file = "gevent-26.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1101b5ef82a3fb178550cfd80f32293dc8dd2f3d0828292223ebba29d6f76e33"}, - {file = "gevent-26.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:5233109ad4f3af16393ba9888f238919a05ce15ce68d6831ac8a0da8dfb750ae"}, - {file = "gevent-26.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:3be804565168ffacebeb21af9f1cd689831a89f0f12fc0c3f423c730c3c9eb31"}, - {file = "gevent-26.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:e80ad2a8a1e8bdaa5605e3bf4929e0cebf9ea7b8237c83362f7257698bb14280"}, - {file = "gevent-26.5.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fe42c037253580a3386fce275f8a2a845e540f5a729916934a732f13d42e72cc"}, - {file = "gevent-26.5.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:9f463c7d6f69d13b6fe8e3b832a6175a6e95328a940f38495d25496d1ae8ad88"}, - {file = "gevent-26.5.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:96d5e96b1b14a4c1023dcfcc114533217f13febc3b6169254f23fc18d19fee29"}, - {file = "gevent-26.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:bccff69c462e3650a0fd1d4e9cfc8b6effe15f3e9b1cad20a7bb5ce14b057efd"}, - {file = "gevent-26.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f519139354d5ca7625df9ddb1b2ffada885c14abc5b4dbae3682e967ddf79669"}, - {file = "gevent-26.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0bf57df54f1c66273bf3601c2a1e41b12138fe848933718369663bc54f177ca2"}, - {file = "gevent-26.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:e49ce0de007dfd7412edbc2b5d41cce33b049bb1b7086f50be5a09e601bde603"}, - {file = "gevent-26.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:5c5ff29495a2eed2a244de8150f21893d6c1b15d8b4b5719ab4bbfa06db1e28f"}, - {file = "gevent-26.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:9b4d3f34c913d1a6bec6d030365a517f3b527a9773b12e58cf56c3339bbe96e6"}, - {file = "gevent-26.5.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1d8da4e799431feeb4c9e441ac7431f0baabb9106976790d884289d08ac08359"}, - {file = "gevent-26.5.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:51becdb4c30a8f45c1c028ad7a97bf5a1ed141f74b159a31aa9cc6aa1e6263a6"}, - {file = "gevent-26.5.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:c42bbcd3d453b08ad8915fd3feaf3d44a3562cdf1c7b208f9837149711e16d9d"}, - {file = "gevent-26.5.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:bd3445e4fbeeb46690ed8efe94b8d1d46b14aa04af8866ae7a8da5997828d1c6"}, - {file = "gevent-26.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b573d5b2826edc705f31f07da6889ad483a6a0d64944ebd8d32205f7c5bf46fb"}, - {file = "gevent-26.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4d53b1b28f2082a151bded2850b53f6baed02f742d2a1584029e8bd42d457fb4"}, - {file = "gevent-26.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:23569ce0c254eb821fc3dcfe250843dde8b3180b09bae9e222e41aa3fa4885b7"}, - {file = "gevent-26.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:40cdcdb2e404b6c82b82a4576bdb33958f23fc2deb0d933e9e022b362001e647"}, - {file = "gevent-26.5.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:75a0050e4b87f08ddee7e56f59e6014cd7fcdc3153046c09a847940515d12c85"}, - {file = "gevent-26.5.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:fd1a0b83a04e19378d9466ae0ee2b5937cf1d7fbfdcb916b2aea82179a208574"}, - {file = "gevent-26.5.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:4c964c15076e76391d523ec24202f579a2535f7e301a40efb1656ae046d3eb69"}, - {file = "gevent-26.5.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:45d5438d1c84da5df7e832434627624709543630977332bb4e2d05ecca362cc9"}, - {file = "gevent-26.5.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:354f35924113abc954819216c2a6ee16751958c615681e0490946e31b437bd2f"}, - {file = "gevent-26.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a47cd2d32f6404212d374ad8014a3491d7477dcf0cc09c5a2308ad6d325fd663"}, - {file = "gevent-26.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:032157cebdedb84f2f52cdd980f2f5f2623eed6a8f083aadf44b44c47f628642"}, - {file = "gevent-26.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:9c414935ba5fc88359110968851d3616f119082c937390d00a1c0f4f59be814f"}, - {file = "gevent-26.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:2a0f5993a04b95a35b3a118b1a58ba272833f9b547b774001dea29f90620882f"}, - {file = "gevent-26.5.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2e117df896a2660c9ebd4e2b5afc02dfd6e2ddf9b495e787e67c72d105432b09"}, - {file = "gevent-26.5.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:af5ffe9c11ffb8a39b6bef2e8b722aa2043ae4980977915c6aa8c68b4bc26e46"}, - {file = "gevent-26.5.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:7da34aef7e87c43dd3662e5785e79ed505c01399a7cb42876d2d8925969fd75f"}, - {file = "gevent-26.5.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:1c6293a7046bcc6f3d8972a74b19cd7a4cfd02d3881edf0fcf827aa514bd247b"}, - {file = "gevent-26.5.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:d3bde0f140a275b2fa88e4b6516bda85551930e10bc2fd95e18c1b7d11cb780c"}, - {file = "gevent-26.5.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:e29fb4b17d9958ec8cb7f6339a111b29bc23f2c2efbef86189d1248bb4862d17"}, - {file = "gevent-26.5.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b2239df2f7570efa03736678f3f053bb1bdd22a8a16cd28a2feb7d32ea5f533f"}, - {file = "gevent-26.5.0-cp315-cp315-win_amd64.whl", hash = "sha256:aae214952fd38d27a42dc416bb70193962ec932384b63445d29bbb5817a1c042"}, - {file = "gevent-26.5.0-cp315-cp315-win_arm64.whl", hash = "sha256:f7067564f139e33bf26a31ee3b13d168d76eb99a44b85ced626652b158baa80c"}, - {file = "gevent-26.5.0.tar.gz", hash = "sha256:1655eb04c1e20d71b2aa4a3c7528162dd58ff6cc46a037af1f01f534c80fefba"}, + {file = "gevent-26.7.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:80e98fc808bd9cc5c911d78a443d214bf0c8f96c9fdd296893df7e40364d5f37"}, + {file = "gevent-26.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bf4b946b47cc6fdbdf9221f891db9a44df92166435c027760ee7dbdfb4039adc"}, + {file = "gevent-26.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:55ce0b7f87f9befcc788d77eb039b1de89a35f37afc31942e12c7ae090a563b8"}, + {file = "gevent-26.7.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7f7143823ef99bc657534a2b6e8cbadedc910750cc0b4f4b4438a58d9fe43ab2"}, + {file = "gevent-26.7.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:ca4019899830471910129968251c795c8aee59e225fd16326ae01c1f93f3cfa6"}, + {file = "gevent-26.7.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:e4042da317a96d12110831cc404855f0c501a5a5aa476a7a18c3b480a5a59233"}, + {file = "gevent-26.7.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5c97ca98e1aae427a267eae0fbfe8d0884327e6b1cd51fc2ef6642b8b0b82701"}, + {file = "gevent-26.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5d5d1864bc3db92d1f82d1790395eda99f98b47fd9f7ec02c4e182d7828a8251"}, + {file = "gevent-26.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:15fd2d88ed5370f8084079758758df91f26d2f68575e1ee76fce604ddba83e5e"}, + {file = "gevent-26.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:514bda3fff741d7e5ab108ee1d31550a7f4b2fd3dc6e3b6f38dfb8685efdafaa"}, + {file = "gevent-26.7.0-cp311-cp311-win_arm64.whl", hash = "sha256:0f26f9a8c32ac0a73f6084c59b63deeacb350e7f1fee5301d95c5e0683a390d4"}, + {file = "gevent-26.7.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:92f256285fb43a57f152bd2e51a59cde1cd0b20869ae1e6da583b6beab88ed8a"}, + {file = "gevent-26.7.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0e4fea187c5df7168b9538b4f543fcb0fcbaeb93be3d6cd499c324652c740704"}, + {file = "gevent-26.7.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:ce732fe08d0ea65de07eff6e46bade8ac6a6fdb65cc748c713f3d31ae122529e"}, + {file = "gevent-26.7.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:c25b3522072137aecf3389031039230190038f888e257f490b3897d0e0620f74"}, + {file = "gevent-26.7.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:eaaa75c9014df3f8c310c64f53f1152af8c6be32e82734396bed91e1d0e6f35c"}, + {file = "gevent-26.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f1956032a9926ac9b4152b2a50bc5a2cc020722ec16928ccaf32e227ee0aae47"}, + {file = "gevent-26.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d989a1ad6cc54f5c69bb7304360f98b4fda80da2b773f1047db9fba61ae7379a"}, + {file = "gevent-26.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:e0c9ce2d80fc0f8894d748a1045ff26ad188e294bad656b29839271800827c85"}, + {file = "gevent-26.7.0-cp312-cp312-win_arm64.whl", hash = "sha256:959effe0c56cdee0bf761e5c4e78ab62880be147a2f2aa31112ca2f7e5754e53"}, + {file = "gevent-26.7.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b1b89eb5566f75aa8b2bbdb0308e1ac8d9113ca7cff85b45366aea9faad639a1"}, + {file = "gevent-26.7.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:449857ce058183442e2d71d83ff0c587a3ddff631e93c6d19a6dffb4814eccad"}, + {file = "gevent-26.7.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:8260a3f38b05fcf3c283417b18617562dbec74f5784f748e4ba3866789d7f3a4"}, + {file = "gevent-26.7.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:30894398d06747b433c8923a6a77ede61259ce6822a99f6c6e7fa0216ccb73c3"}, + {file = "gevent-26.7.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0b753522498118c9489753de7c612d4baed0edf384d9df2bf9492233ba1c20ff"}, + {file = "gevent-26.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:055a643026dc28daff2be228555a2097937448cc9b58307edebcf81b9d78ff4b"}, + {file = "gevent-26.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e1dc6a2712de67fd210e1f1a408601f6908b042f6420e188106f2f37f94ec71"}, + {file = "gevent-26.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:44e5280296129c0915addaefdb37d6e9bc124a77a433b1b1c8ddf1853c53f4e7"}, + {file = "gevent-26.7.0-cp313-cp313-win_arm64.whl", hash = "sha256:9f08b1aa6729f794409ca137e25f671e0d9bbda4451200c5e28a769375365388"}, + {file = "gevent-26.7.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:0e0e3bf7ae0f82dbc5c6be26b4781e86c97f1e28d516b7a9746ac8b04bcc6948"}, + {file = "gevent-26.7.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:740050b53048207b080a1e183a377c47809ad0b7b7b0cd7eab0dea1045f7e480"}, + {file = "gevent-26.7.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:67983607eb6c7bafa362c5c43b69a27145b936c34a3d6441ed42413d62fae0a6"}, + {file = "gevent-26.7.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:475848518d708e07d1987c3d94cb8ff53e2b3a69df32e39feda2779cafe400b0"}, + {file = "gevent-26.7.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f8ed457dd616bfe6682569f92730f9ab45aafb1aeca5e80eb2f6b9a2ce26d11"}, + {file = "gevent-26.7.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:15373c68cf1fa14114bec2f09b16e2c65374bd5309e897e0a28740b09ce329e0"}, + {file = "gevent-26.7.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:73f3d53f2f390369e290c933b75bd87f1f2261f2f2f2175aa667c43ee3049bad"}, + {file = "gevent-26.7.0-cp314-cp314-win_amd64.whl", hash = "sha256:f11b558d544ad2249029ba023cd6519ec3a0eee54a3d027e6515c1eaa322422a"}, + {file = "gevent-26.7.0-cp314-cp314-win_arm64.whl", hash = "sha256:3871f4ca59ec2328c3ef638a0fe01a28a825443a133368dc78eb5ceadcad7609"}, + {file = "gevent-26.7.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3e3d6e20a94239ad353b776e72b8ce18c35dbe4e98c279aef3932651553d8404"}, + {file = "gevent-26.7.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:ddbd3cc76b9bc69df651a216c2a62fc6415ad463b3ac9c6cbbbb8b7b8224af17"}, + {file = "gevent-26.7.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:01ceab7e608dc1b9859d9511a0a29d7ce2e7d909ab19fddc860e70a2ed5b10ce"}, + {file = "gevent-26.7.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:2e6c917b2b8baeb6080797a6b25e35e1fd784319a05bb92b87c53546e5578eb2"}, + {file = "gevent-26.7.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:df75a1748b26030f2f7f10042cc45640b22954d9d0dc6b4b6f0dbe0b6751a2d4"}, + {file = "gevent-26.7.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ee1b389587e5d5c1eb19d0455b5b4d7a0fb5c5287af4e226ec66d9dfd2548107"}, + {file = "gevent-26.7.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c2918641ba756f46aa01ab9dd82d6dfceec403c77c2787298746b411dcf0288e"}, + {file = "gevent-26.7.0.tar.gz", hash = "sha256:5b333a556e38a302b1b8c80525bef16d437e16f1e7767947789406841856a102"}, ] [package.dependencies] @@ -1153,38 +1152,30 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.1" +version = "1.5.2" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, - {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, - {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, - {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, - {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, - {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, - {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, - {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, - {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, - {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, - {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, - {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, - {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, + {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, + {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, + {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, + {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, + {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, + {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, + {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, + {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, ] [package.extras] @@ -1239,14 +1230,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.23.0" +version = "1.25.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, - {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, + {file = "huggingface_hub-1.25.1-py3-none-any.whl", hash = "sha256:004d4e70350517e24c68a7dbb7dc5e40b2b6aefef8f94bf7a85f6f9835102ea5"}, + {file = "huggingface_hub-1.25.1.tar.gz", hash = "sha256:21129595ca7a753be479b319913e22cc8808361ac118bd76cc413db831b28a99"}, ] [package.dependencies] @@ -2115,126 +2106,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.10" +version = "2026.7.19" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, - {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, - {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, - {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, - {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, - {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, - {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, - {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, - {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, - {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, - {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, - {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, - {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, - {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, - {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, - {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, - {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, - {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, - {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, - {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, - {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, - {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, - {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, - {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, - {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, - {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, - {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, - {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, - {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, - {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, - {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, - {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, - {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, - {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, - {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, - {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, - {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, - {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, - {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, - {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, - {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, - {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, - {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, - {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, + {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, + {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, + {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, + {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, + {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, + {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, + {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, + {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, + {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, + {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, + {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, + {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, + {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, + {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, + {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, + {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, + {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, + {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, + {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, + {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, + {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, + {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, ] [[package]] @@ -2399,14 +2390,14 @@ dev = ["bitsandbytes", "blobfile", "cmake (>=3.19.0,<4.0.0)", "diskcache", "expe [[package]] name = "tqdm" -version = "4.68.4" +version = "4.70.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, - {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, + {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, + {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, ] [package.dependencies] @@ -2555,116 +2546,116 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "yarl" -version = "1.24.2" +version = "1.24.5" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, - {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, - {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, - {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, - {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, - {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, - {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, - {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, - {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, - {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, - {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, - {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, - {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, - {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, - {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, - {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, - {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, - {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, - {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, - {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, - {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, - {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, - {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, - {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, - {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, - {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, - {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, - {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, - {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, - {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, - {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, - {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, - {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, + {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, + {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, + {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, + {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, + {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, + {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, + {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, + {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, + {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, + {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, + {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, + {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, + {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, + {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, + {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, + {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, + {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, + {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, + {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, + {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, + {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, + {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, + {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, + {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, + {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, + {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, + {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, + {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, + {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, + {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, + {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, + {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, ] [package.dependencies] @@ -2747,4 +2738,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "5caec91a6dcd3e66c173ee2cb35f0c5563e0531aae1f701d98e15e50e839ff07" +content-hash = "5bd084cbcf8afceb839ea1f0a8f0500a95c66d59c772bf028b514d68f275790e" diff --git a/security_scanning/triton_backend/pyproject.toml b/security_scanning/triton_backend/pyproject.toml index 43476e749d04..f23e1fef69fa 100644 --- a/security_scanning/triton_backend/pyproject.toml +++ b/security_scanning/triton_backend/pyproject.toml @@ -7,7 +7,7 @@ authors = [ ] requires-python = ">=3.10,<3.13" dependencies = [ - "regex (>=2026.7.10,<2027.0.0)", + "regex (>=2026.7.19,<2027.0.0)", "fire (>=0.7.1,<0.8.0)", "tritonclient[all] (>=2.70.0,<3.0.0)", "transformers (==5.5.4)", diff --git a/setup.py b/setup.py index 7a3b7d2af8f7..e71819c66fb9 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,9 @@ # limitations under the License. import os import platform +import re +import subprocess +import sys from pathlib import Path from setuptools import find_packages, setup @@ -70,6 +73,23 @@ def get_version(): if version is None: raise RuntimeError(f"Could not set version from {version_file}") + # For develop / editable installs (`pip install -e .` or + # `python setup.py develop`), append the git commit hash as a PEP 440 + # local version segment so the installed package is identifiable, + # e.g. "1.3.0rc21+58d8964d13". + is_develop = any(arg in sys.argv for arg in ("develop", "editable_wheel")) + if is_develop: + try: + commit = subprocess.check_output( + ["git", "rev-parse", "--short=10", "HEAD"], + cwd=Path(__file__).resolve().parent, + stderr=subprocess.DEVNULL).decode().strip() + except (subprocess.CalledProcessError, FileNotFoundError, OSError): + commit = "" + commit = re.sub(r"[^A-Za-z0-9.]", "", commit) + if commit: + version = f"{version}+{commit}" + return version @@ -109,6 +129,7 @@ def has_ext_modules(self): devel_deps, _ = parse_requirements( Path("requirements-dev-windows.txt" if on_windows else "requirements-dev.txt")) +mx_deps = ["modelexpress==0.4.1"] constraints_file = Path("constraints.txt") if constraints_file.exists(): constraints, _ = parse_requirements(constraints_file) @@ -116,15 +137,13 @@ def has_ext_modules(self): if on_windows: package_data = [ - 'libs/th_common.dll', 'libs/tensorrt_llm.dll', - 'libs/nvinfer_plugin_tensorrt_llm.dll', 'bindings.*.pyd', "include/**/*" + 'libs/th_common.dll', 'libs/tensorrt_llm.dll', 'bindings.*.pyd', + "include/**/*" ] else: package_data = [ - 'bin/executorWorker', 'libs/libtensorrt_llm.so', 'libs/libth_common.so', - 'libs/libnvinfer_plugin_tensorrt_llm.so', 'libs/libtensorrt_llm_ucx_wrapper.so', 'libs/libdecoder_attention_0.so', 'libs/libtensorrt_llm_nixl_wrapper.so', @@ -147,7 +166,6 @@ def has_ext_modules(self): 'deep_gemm/include/**/*', 'deep_gemm/*.py', 'deep_gemm_cpp_tllm.*.so', - 'scripts/install_tensorrt.sh', 'flash_mla/LICENSE', 'flash_mla/*.py', 'flash_mla_cpp_tllm.*.so', @@ -163,14 +181,11 @@ def has_ext_modules(self): package_data += [ 'bindings/*.pyi', 'bindings/**/*.pyi', - 'tools/plugin_gen/templates/*', - 'bench/build/benchmark_config.yml', 'evaluate/lm_eval_tasks/**/*', "_torch/auto_deploy/config/*.yaml", # Include CUDA source for fused MoE align extension so runtime JIT can find it in wheels '_torch/auto_deploy/custom_ops/fused_moe/moe_align_kernel.cu', '_torch/auto_deploy/custom_ops/fused_moe/triton_fused_moe_configs/*', - '_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/**/*.so', 'usage/schemas/*.json', ] @@ -406,6 +421,9 @@ def extract_from_precompiled(precompiled_location: str, package_data: list[str], # internal absolute imports (e.g., "from triton_kernels.foo import bar") work. packages += find_packages(include=["triton_kernels", "triton_kernels.*"]) +msa_package_dir = {"fmha_sm100": "3rdparty/MSA/python/fmha_sm100"} +packages += ["fmha_sm100"] + # https://setuptools.pypa.io/en/latest/references/keywords.html setup( name='tensorrt_llm', @@ -420,6 +438,7 @@ def extract_from_precompiled(precompiled_location: str, package_data: list[str], url="https://github.com/NVIDIA/TensorRT-LLM", download_url="https://github.com/NVIDIA/TensorRT-LLM/tags", packages=packages, + package_dir=msa_package_dir, exclude_package_data=exclude_package_data, # TODO Add windows support for python bindings. classifiers=[ @@ -432,8 +451,17 @@ def extract_from_precompiled(precompiled_location: str, package_data: list[str], license="Apache License 2.0", keywords="nvidia tensorrt deeplearning inference", package_data={ - 'tensorrt_llm': package_data, + 'tensorrt_llm': + package_data, 'triton_kernels': ['LICENSE', 'VERSION', 'README.md'], + 'fmha_sm100': [ + '*.py', + 'csrc/**/*', + 'cute/**/*', + 'cutlass/include/**/*', + 'cutlass/tools/util/include/**/*', + 'cutlass/LICENSE.txt', + ], }, license_files=get_license(), entry_points={ @@ -446,10 +474,7 @@ def extract_from_precompiled(precompiled_location: str, package_data: list[str], scripts=['tensorrt_llm/llmapi/trtllm-llmapi-launch'], extras_require={ "devel": devel_deps, - # MX remains prototype-only and is intentionally not declared as an - # optional package extra until its external dependency completes OSS - # allowlist onboarding. Keep install instructions in docs/PR text - # rather than packaging metadata. + "mx": mx_deps, }, zip_safe=True, install_requires=required_deps, diff --git a/tensorrt_llm/__init__.py b/tensorrt_llm/__init__.py index 91e5f98a500d..d707ca0e0694 100644 --- a/tensorrt_llm/__init__.py +++ b/tensorrt_llm/__init__.py @@ -104,24 +104,6 @@ def _setup_vendored_triton_kernels(): # ImportError: libc10.so: cannot open shared object file: No such file or directory import torch # noqa - -def _preload_tensorrt_libs(): - """Preload the TensorRT libraries needed by the bindings extension. - - The C++ runtime still links against the TensorRT libraries until it is - decoupled from TensorRT. Importing the tensorrt package loads libnvinfer - from the tensorrt_libs wheel for environments where it is not on the - system loader path; without it, importing tensorrt_llm.bindings raises - ImportError: libnvinfer.so.10: cannot open shared object file. - """ - try: - import tensorrt # noqa: F401 - except ImportError: - pass - - -_preload_tensorrt_libs() - import tensorrt_llm._torch.models as torch_models import tensorrt_llm.math_utils as math_utils import tensorrt_llm.models as models diff --git a/tensorrt_llm/_common.py b/tensorrt_llm/_common.py index 00bac35cc11b..8dc41df25fcb 100644 --- a/tensorrt_llm/_common.py +++ b/tensorrt_llm/_common.py @@ -13,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ctypes import os import platform import threading import time -from functools import wraps from pathlib import Path import torch @@ -46,16 +44,6 @@ def _init(log_level: object = None) -> None: project_dir = str(Path(__file__).parent.absolute()) - # Promote libtensorrt_llm.so symbols to the process's global scope. The KV - # cache transfer agent wrappers (libtensorrt_llm_nixl_wrapper.so, - # libtensorrt_llm_ucx_wrapper.so) are dlopen'ed at runtime without linking - # against libtensorrt_llm.so and resolve its symbols from the global symbol - # table, while Python extension modules and their dependencies load with - # RTLD_LOCAL. This promotion was previously a side effect of loading the - # TensorRT plugin library with RTLD_GLOBAL. - if platform.system() != "Windows": - ctypes.CDLL(project_dir + "/libs/libtensorrt_llm.so", mode=ctypes.RTLD_GLOBAL) - # Load FT decoder layer and torch custom ops. if platform.system() == "Windows": ft_decoder_lib = project_dir + "/libs/th_common.dll" @@ -89,27 +77,3 @@ def _print_stacks(): print_stacks_thread.start() logger.info("TensorRT LLM inited.") - - -# TODO: dead on the Python side (no remaining @_is_building users); the IS_BUILDING -# env var is only read by the C++ isBuilding() in the TensorRT plugins. Remove this -# together with that C++ half in the C++ decouple step. -class _BuildingFlag: - def __enter__(self): - os.environ["IS_BUILDING"] = "1" - - def __exit__(self, type, value, tb): - del os.environ["IS_BUILDING"] - - -def _is_building(f): - """Use this to decorate functions which are called during engine building/refitting process, - otherwise, the plugin registration will fail. - """ - - @wraps(f) - def decorated(*args, **kwargs): - with _BuildingFlag(): - return f(*args, **kwargs) - - return decorated diff --git a/tensorrt_llm/_deprecation.py b/tensorrt_llm/_deprecation.py deleted file mode 100644 index 1d436040ef9a..000000000000 --- a/tensorrt_llm/_deprecation.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Legacy-workflow warnings for the TensorRT engine-build path. - -The TensorRT engine-build workflow (convert_checkpoint.py -> trtllm-build -> -run.py) is a legacy path. The PyTorch backend (trtllm-serve / LLM API) is the -recommended approach for new projects. - -This module provides a shared warning function that can be called from legacy -scripts and internal chokepoints to inform users about the recommended -migration path. -""" - -import warnings - -_DEPRECATION_DOCS_URL = "https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html" - -_warned: set = set() - - -def emit_engine_arch_deprecation(caller_name: str) -> None: - """Emit a one-time FutureWarning for legacy engine-architecture usage. - - Each unique *caller_name* triggers the warning at most once per process, - so hot paths like ``builder.build()`` don't spam the console. - - Args: - caller_name: Human-readable identifier for the caller - (e.g., ``"convert_checkpoint.py"``, ``"trtllm-build"``, - ``"builder.build()"``). - """ - if caller_name in _warned: - return - _warned.add(caller_name) - - warnings.warn( - f"\n{'=' * 70}\n" - f"LEGACY WARNING: {caller_name}\n" - f"{'=' * 70}\n" - f"This is part of the legacy TensorRT engine-build workflow.\n" - f"New projects should use the PyTorch backend instead.\n\n" - f" # Serve a model (recommended):\n" - f" trtllm-serve \n\n" - f" # Python API:\n" - f" from tensorrt_llm import LLM\n" - f" llm = LLM(model='')\n" - f" output = llm.generate(['Hello, how are you?'])\n\n" - f"Documentation: {_DEPRECATION_DOCS_URL}\n" - f"{'=' * 70}\n", - FutureWarning, - stacklevel=2, - ) diff --git a/tensorrt_llm/_torch/attention_backend/flashinfer.py b/tensorrt_llm/_torch/attention_backend/flashinfer.py index 48714e841c4c..733d477009cb 100644 --- a/tensorrt_llm/_torch/attention_backend/flashinfer.py +++ b/tensorrt_llm/_torch/attention_backend/flashinfer.py @@ -1,3 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import functools import math import os @@ -207,6 +221,10 @@ class FlashInferWrappers: repr=False) host_decode_block_tables: Optional[torch.Tensor] = field(default=None, repr=False) + # Remember the previously populated rectangle so a smaller subsequent batch can clear stale rows + # and columns before narrowing future updates. + decode_block_table_active_rows: int = field(default=0, repr=False) + decode_block_table_active_width: int = field(default=0, repr=False) @dataclass(kw_only=True) @@ -630,7 +648,7 @@ def _post_init_with_buffers(self, buffers) -> None: self._host_pool_indices: Dict[int, torch.Tensor] = {} self._host_paged_kv_indices: Optional[torch.Tensor] = None self._host_paged_kv_indptr_decode: Optional[torch.Tensor] = None - self._max_num_blocks = 0 + self._max_num_blocks_per_seq = 0 # VSWA (Variable Sliding Window Attention): models with per-layer # max_attention_window create separate V2 pool groups with independent @@ -650,18 +668,16 @@ def _post_init_with_buffers(self, buffers) -> None: ) # Maximum block count across ALL pools: sizes the VSWA pool - # buffers below and bounds the per-request width of the - # persistent trtllm-gen decode block tables (a request can - # never reference more blocks than its pool holds). Computed - # for every model, not just VSWA — non-VSWA managers have a - # single pool, so this stays blocks_in_primary_pool for them. + # buffers below. Computed for every model, not just VSWA — + # non-VSWA managers have a single pool, so this stays + # blocks_in_primary_pool for them. max_num_blocks = blocks_in_primary_pool if hasattr(self.kv_cache_manager, 'layer_offsets'): for lid in self.kv_cache_manager.layer_offsets: lbuf = self.kv_cache_manager.get_buffers(lid) if lbuf is not None: max_num_blocks = max(max_num_blocks, lbuf.shape[0]) - self._max_num_blocks = max_num_blocks + self._max_num_blocks_per_seq = self.kv_cache_manager.max_blocks_per_seq # Layers may share one page-index list only when they are in the # same pool AND have the same page-index scale: VSWA splits pools, @@ -989,9 +1005,9 @@ def _build_decode_block_tables( gen_num_blocks = np.asarray(self.num_blocks[self.num_contexts:], dtype=np.int64) max_n = int(gen_num_blocks.max()) - if max_n > self._max_num_blocks: - # A request can never reference more blocks than any pool - # holds; defensive guard for inconsistent metadata. + if max_n > self._max_num_blocks_per_seq: + # A request can never reference more than max_blocks_per_seq; + # defensive guard for inconsistent metadata. return None block_tables = wrappers.decode_block_tables if (self.is_cuda_graph and block_tables is not None @@ -1003,12 +1019,12 @@ def _build_decode_block_tables( if self.is_cuda_graph: # Allocated once at capture warmup; full capacity width so # replays never need a wider table. - width = self._max_num_blocks + width = self._max_num_blocks_per_seq else: # Eager path replans (and re-reads the table) every step, # so the buffer may grow geometrically as sequences do. width = min(max(64, 1 << (max_n - 1).bit_length()), - self._max_num_blocks) + self._max_num_blocks_per_seq) try: block_tables = torch.zeros((self.max_num_requests, width), dtype=torch.int32, @@ -1039,6 +1055,10 @@ def _build_decode_block_tables( # completed. block_tables[:num_gens, :max_n].copy_( host_block_tables[:num_gens, :max_n], non_blocking=True) + # Seed the extent used by `prepare()` to clear entries if the first graph replay has fewer + # requests or a narrower block table. + wrappers.decode_block_table_active_rows = num_gens + wrappers.decode_block_table_active_width = max_n return block_tables[:num_gens] def _clean_cached_plans(self, *, defer_plan: bool): @@ -1310,7 +1330,7 @@ def _to_int32_tensor(arr: np.ndarray) -> torch.Tensor: if (self.is_cuda_graph and self._vswa_layer_to_pool is not None and self._vswa_pool_indices_cache is not None and self.num_generations > 0): - decode_blocks = self.num_blocks[self.num_contexts:] + decode_blocks = num_blocks[self.num_contexts:] head_dim_to_pool = getattr(self, '_vswa_head_dim_to_pool', None) for plan_params, wrappers in self._plan_params_to_wrappers.items(): if plan_params.attention_mask_data is not None: @@ -1323,34 +1343,62 @@ def _to_int32_tensor(arr: np.ndarray) -> torch.Tensor: if head_dim_to_pool else None) if pool_id is None: continue - pool_buf = self._vswa_pool_indices_cache[pool_id] batch_size, table_width = block_tables.shape rows = min(batch_size, self.num_generations) - # Vectorized equivalent of a per-request copy loop: row i - # gets pool_buf[offset + row_starts[i] :] for its first - # min(num_blocks_per_row[i], table_width) columns, zero- - # padded — one gather + where instead of ~batch slice - # copies per pool per step. - num_blocks_per_row = torch.tensor( - decode_blocks[:rows], - dtype=torch.int64).to(device=block_tables.device, - non_blocking=True) - row_starts = torch.cumsum(num_blocks_per_row, - dim=0) - num_blocks_per_row - columns = torch.arange(table_width, - dtype=torch.int64, - device=block_tables.device) - mask = columns.unsqueeze(0) < num_blocks_per_row.clamp( - max=table_width).unsqueeze(1) - source_indices = (self.num_context_blocks + - row_starts.unsqueeze(1) + - columns.unsqueeze(0)).clamp( - max=pool_buf.numel() - 1) - new_block_tables = torch.zeros_like(block_tables) - new_block_tables[:rows] = torch.where( - mask, pool_buf[source_indices.reshape(-1)].view( - rows, table_width), new_block_tables[:rows]) - block_tables.copy_(new_block_tables) + num_blocks_per_row = decode_blocks[:rows] + active_width = min(int(num_blocks_per_row.max()), table_width) + # Include the previous live extent once when the batch or its longest sequence + # shrinks. The zero-filled copy below clears entries that are no longer live; the + # next update can shrink. + update_rows = min( + batch_size, + max(rows, wrappers.decode_block_table_active_rows)) + update_width = min( + table_width, + max(active_width, wrappers.decode_block_table_active_width)) + host_block_tables = wrappers.host_decode_block_tables + if (host_block_tables is None + or host_block_tables.size(0) < update_rows + or host_block_tables.size(1) < update_width): + # Grow geometrically to avoid reallocating whenever the longest sequence + # acquires one more KV-cache block. + host_width = min( + max(64, 1 << (update_width - 1).bit_length()), + table_width) + host_block_tables = torch.zeros( + (self.max_num_requests, host_width), + dtype=torch.int32, + pin_memory=prefer_pinned()) + wrappers.host_decode_block_tables = host_block_tables + + # Build only the live (or previously live) rectangle on the host. This preserves + # padded-row zeroing when batch size shrinks without launching full-capacity + # `arange`, `gather`, `where`, `zero`, and copy kernels on every decode step. + host_table = host_block_tables[:update_rows, :update_width] + host_table.zero_() + host_pool_indices = self._host_pool_indices[pool_id] + # Pool indices are flattened as context blocks followed by each generation request's + # blocks. + source_offset = self.num_context_blocks + # This loop scales with the number of generation requests. Vectorizing ragged + # rows would require padded mask or index tensor, so keep contiguous row copies + # until profiling identifies this as a CPU bottleneck. + for row, num_blocks_for_row in enumerate(num_blocks_per_row): + num_blocks_for_row = int(num_blocks_for_row) + copy_width = min(num_blocks_for_row, table_width) + host_table[row, :copy_width].copy_( + host_pool_indices[source_offset:source_offset + + copy_width]) + # Advance by the uncropped count so the next request starts at the correct + # offset even if this row hit `table_width`. + source_offset += num_blocks_for_row + # Keep the graph-captured device address stable and transfer only the compact + # rectangle prepared in pinned host memory. + block_tables[:update_rows, :update_width].copy_( + host_block_tables[:update_rows, :update_width], + non_blocking=True) + wrappers.decode_block_table_active_rows = rows + wrappers.decode_block_table_active_width = active_width kv_lens_buf = getattr(decode_wrapper, '_kv_lens_buffer', None) if kv_lens_buf is not None: decode_kv_lens = _to_int32_tensor( diff --git a/tensorrt_llm/_torch/attention_backend/fmha/__init__.py b/tensorrt_llm/_torch/attention_backend/fmha/__init__.py index 1c3981abcf91..b2cd1e75ec7e 100644 --- a/tensorrt_llm/_torch/attention_backend/fmha/__init__.py +++ b/tensorrt_llm/_torch/attention_backend/fmha/__init__.py @@ -16,6 +16,7 @@ from .fallback import FallbackFmha from .flashinfer_trtllm_gen import FlashInferTrtllmGenFmha from .interface import Fmha +from .msa_sparse_gqa import MsaSparseGqaFmha from .phased import FmhaParams, PhasedFmha from .registry import DEFAULT_FMHA_LIBS, FMHA_LIBS, FmhaCls, get_enabled_fmha_lib_classes @@ -27,6 +28,7 @@ "Fmha", "FmhaCls", "FmhaParams", + "MsaSparseGqaFmha", "PhasedFmha", "get_enabled_fmha_lib_classes", ] diff --git a/tensorrt_llm/_torch/attention_backend/fmha/fallback.py b/tensorrt_llm/_torch/attention_backend/fmha/fallback.py index 8f1da5dc36f9..9c081bf19dfc 100644 --- a/tensorrt_llm/_torch/attention_backend/fmha/fallback.py +++ b/tensorrt_llm/_torch/attention_backend/fmha/fallback.py @@ -98,6 +98,7 @@ def forward( spec_decoding_bl_tree_mask_offset=metadata.spec_decoding_bl_tree_mask_offset, spec_decoding_bl_tree_mask=metadata.spec_decoding_bl_tree_mask, spec_decoding_target_max_draft_tokens=metadata.max_total_draft_tokens, + force_prepare_spec_dec_tree_mask=metadata.force_prepare_spec_dec_tree_mask, spec_bl_tree_first_sparse_mask_offset_kv=metadata.spec_bl_tree_first_sparse_mask_offset_kv, num_sparse_topk=metadata.num_sparse_topk, flash_mla_tile_scheduler_metadata=metadata.flash_mla_tile_scheduler_metadata, diff --git a/tensorrt_llm/_torch/attention_backend/fmha/flashinfer_trtllm_gen.py b/tensorrt_llm/_torch/attention_backend/fmha/flashinfer_trtllm_gen.py index 1098a5e7297e..0e9b0f8022af 100644 --- a/tensorrt_llm/_torch/attention_backend/fmha/flashinfer_trtllm_gen.py +++ b/tensorrt_llm/_torch/attention_backend/fmha/flashinfer_trtllm_gen.py @@ -78,26 +78,21 @@ def _get_mla_backend(backend: str) -> str: return backend -def _clear_multi_ctas_kv_counter_workspace( - fmha_workspace: torch.Tensor, - num_heads: int, - max_num_requests: int, - multi_processor_count: Optional[int], -) -> None: - counter_size = _get_multi_ctas_kv_counter_size( - num_heads, - max_num_requests, - multi_processor_count, - ) - fmha_workspace.flatten().narrow(0, 0, counter_size).zero_() +_MULTI_CTAS_KV_COUNTER_ALIGNMENT = 8 def _get_multi_ctas_kv_counter_size( num_heads: int, max_num_requests: int, - multi_processor_count: Optional[int], + multi_processor_count: int, ) -> int: - return max(num_heads * max_num_requests, multi_processor_count or 0) * torch.int32.itemsize + num_counters = max(num_heads * max_num_requests, multi_processor_count) + aligned_num_counters = ( + (num_counters + _MULTI_CTAS_KV_COUNTER_ALIGNMENT - 1) + // _MULTI_CTAS_KV_COUNTER_ALIGNMENT + * _MULTI_CTAS_KV_COUNTER_ALIGNMENT + ) + return aligned_num_counters * torch.int32.itemsize def _get_bmm1_scale_log2(bmm1_scale: torch.Tensor) -> torch.Tensor: @@ -188,6 +183,7 @@ def _trtllm_gen_batch_decode_with_kv_cache( query: torch.Tensor, kv_pool: torch.Tensor, workspace_buffer: torch.Tensor, + multi_ctas_kv_counter_buffer: torch.Tensor, block_tables: torch.Tensor, seq_lens: torch.Tensor, max_seq_len: int, @@ -227,6 +223,7 @@ def _trtllm_gen_batch_decode_with_kv_cache( kv_pool, kv_pool, workspace_buffer, + multi_ctas_kv_counter_buffer, block_tables, seq_lens, decode_max_q_len, @@ -258,6 +255,7 @@ def _trtllm_gen_batch_context_with_kv_cache( query: torch.Tensor, kv_pool: torch.Tensor, workspace_buffer: torch.Tensor, + multi_ctas_kv_counter_buffer: torch.Tensor, block_tables: torch.Tensor, seq_lens: torch.Tensor, max_q_len: int, @@ -288,6 +286,7 @@ def _trtllm_gen_batch_context_with_kv_cache( kv_pool, kv_pool, workspace_buffer, + multi_ctas_kv_counter_buffer, block_tables, seq_lens, max_q_len, @@ -516,6 +515,7 @@ def __init__(self, attn: "TrtllmAttention"): # Lazily set on the first forward() call from the query device. self._multi_processor_count: Optional[int] = None + self._multi_ctas_kv_counter_buffer: Optional[torch.Tensor] = None def _get_total_num_blocks(self, meta: "TrtllmAttentionMetadata") -> int: kv_cache_manager = meta.kv_cache_manager @@ -903,6 +903,28 @@ def prepare_workspace( if self._multi_processor_count is None: self._multi_processor_count = self._get_multi_processor_count(q.device) + required_counter_size = _get_multi_ctas_kv_counter_size( + attn.num_heads, + metadata.max_num_requests, + self._multi_processor_count, + ) + counter_buffer = self._multi_ctas_kv_counter_buffer + if ( + counter_buffer is None + or counter_buffer.device != q.device + or counter_buffer.numel() * counter_buffer.element_size() < required_counter_size + ): + if metadata.is_cuda_graph and torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "The trtllm-gen multi-CTA KV counter buffer must be allocated " + "before CUDA graph capture." + ) + self._multi_ctas_kv_counter_buffer = torch.zeros( + required_counter_size, + dtype=torch.uint8, + device=q.device, + ) + num_tokens = q.size(0) attention_input_type = forward_args.attention_input_type is_gen_only = attention_input_type == AttentionInputType.generation_only @@ -962,6 +984,12 @@ def prepare_workspace( required_workspace_numel = math.ceil(required_workspace_size / workspace.element_size()) workspace.resize_((required_workspace_numel,)) + def _get_multi_ctas_kv_counter_buffer(self) -> torch.Tensor: + counter_buffer = self._multi_ctas_kv_counter_buffer + if counter_buffer is None: + raise RuntimeError("The trtllm-gen multi-CTA KV counter buffer is not initialized.") + return counter_buffer + @staticmethod def _compute_window_left( cyclic_attention_window_size: int, @@ -1088,6 +1116,7 @@ def run_context( q_processed, # query kv_pool, # kv_pool fmha_workspace, # workspace_buffer + self._get_multi_ctas_kv_counter_buffer(), # multi_ctas_kv_counter_buffer block_tables, # block_tables params.sequence_lengths, # seq_lens max_q_len, # max_q_len @@ -1219,22 +1248,6 @@ def run_generation( params.is_cross, # is_cross ) - # FIXME: Flashinfer trtllm-gen API doesn't support a separate - # multi CTAs counter buffer. We have to clear a small buffer - # before trtllm_gen_batch_decode_with_kv_cache. - # - # We must also avoid clearing the workspace only when it is - # resized. The warmup phase may have already cached the workspace - # pointer; if the capture phase skips the zeroing step, the - # CUDA graph will not include the counter initialization. We - # have already verified—specifically in the context of the GPTOSS-20B - # test graph replay scenario—that this skipping logic is unsafe. - # - # https://github.com/flashinfer-ai/flashinfer/issues/3433 - _clear_multi_ctas_kv_counter_workspace( - fmha_workspace, attn.num_heads, meta.max_num_requests, self._multi_processor_count - ) - q_len_per_req = None if is_multi_token_gen else params.input_seq_length decode_max_q_len = max_q_len if is_multi_token_gen else None decode_cu_seqlens = cu_seqlens if is_multi_token_gen else None @@ -1258,6 +1271,7 @@ def run_generation( q_processed, # query kv_pool, # kv_pool fmha_workspace, # workspace_buffer + self._get_multi_ctas_kv_counter_buffer(), # multi_ctas_kv_counter_buffer block_tables, # block_tables params.sequence_lengths, # seq_lens max_kv_len, # max_seq_len @@ -1407,12 +1421,6 @@ def run_mla_generation( else: sequence_lengths = params.sequence_lengths workspace_buffer = params.workspace.view(-1, 4) - _clear_multi_ctas_kv_counter_workspace( - workspace_buffer, - attn.num_heads, - meta.max_num_requests, - self._multi_processor_count, - ) uses_shared_paged_kv_idx = self.USE_SHARED_PAGED_KV_IDX flashinfer.mla.trtllm_batch_decode_with_kv_cache_mla( @@ -1436,4 +1444,11 @@ def run_mla_generation( is_var_seq=True, uses_shared_paged_kv_idx=uses_shared_paged_kv_idx, cute_dsl_impl="monolithic", + # flashinfer rejects the counter buffer unless the trtllm-gen + # runner is selected; the cute-dsl MLA path must pass None. + multi_ctas_kv_counter_buffer=( + self._get_multi_ctas_kv_counter_buffer() + if self._mla_backend != "cute-dsl" + else None + ), ) diff --git a/tensorrt_llm/_torch/attention_backend/fmha/msa_sparse_gqa.py b/tensorrt_llm/_torch/attention_backend/fmha/msa_sparse_gqa.py new file mode 100644 index 000000000000..0bbb1ad1509a --- /dev/null +++ b/tensorrt_llm/_torch/attention_backend/fmha/msa_sparse_gqa.py @@ -0,0 +1,237 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Block-sparse GQA FMHA backed by MSA's fmha_sm100 kernel. + +MsaSparseGqaFmha wraps the fmha_sm100 paged sparse GQA kernel and +participates in the standard TrtllmAttention.forward dispatch loop. The +owning MiniMax-M3 MSA attention layer runs an MsaIndexer to select the +per-query KV blocks and publishes them on forward_args.sparse_prediction; +this class attends over them. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +import torch + +from tensorrt_llm._utils import is_sm_100f + +from .interface import Fmha + +if TYPE_CHECKING: + from tensorrt_llm._torch.attention_backend.interface import AttentionForwardArgs + from tensorrt_llm._torch.attention_backend.trtllm import ( + TrtllmAttention, + TrtllmAttentionMetadata, + ) + + +def run_msa_sparse_gqa( + q: torch.Tensor, + k_paged: torch.Tensor, + v_paged: torch.Tensor, + kv_block_indexes: Optional[torch.Tensor] = None, + *, + kv_indices: torch.Tensor, + sm_scale: float, + qo_lens_cpu: Optional[torch.Tensor] = None, + kv_lens_cpu: Optional[torch.Tensor] = None, + qo_offset_cpu: Optional[torch.Tensor] = None, + causal: bool = True, + head_dim: int = 128, + plan: Optional[tuple] = None, + out: Optional[torch.Tensor] = None, + use_fp8: bool = False, +) -> None: + """Run fmha_sm100 paged GQA (plan/run split). + + `kv_block_indexes`: if set, sparse top-k mode (fixed `kv_block_num=topk`); + if None, dense mode attending all pages in `kv_indices`. + `plan`: prebuilt execution plan; if None, built inline from the CPU length + tensors (eager prefill/tests vs. CUDA-graph decode). + `out`: destination buffer the kernel writes in place. + `use_fp8`: FP8 KV cache. The caller must pass FP8 `q` to match the FP8 paged + K/V, since the kernel variant shares one dtype across q/k/v. Also selects the + FP8 AOT kernels for an inline sparse-prefill plan; no-op for the decode planner. + """ + from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_utils import require_msa_module + + fmha_sm100 = require_msa_module() + + if q.dim() != 3: + raise ValueError( + f"MsaSparseGqaFmha expects q [total_q, num_qo_heads, head_dim]; got {tuple(q.shape)}." + ) + if q.shape[-1] != head_dim: + raise NotImplementedError( + f"MsaSparseGqaFmha supports head_dim={head_dim}; got {q.shape[-1]}." + ) + if k_paged.dim() != 4 or v_paged.dim() != 4: + raise ValueError( + "MsaSparseGqaFmha expects paged KV [num_pages, num_kv_heads, page_size, head_dim]; " + f"got k={tuple(k_paged.shape)}, v={tuple(v_paged.shape)}." + ) + if k_paged.shape != v_paged.shape: + raise ValueError( + f"MsaSparseGqaFmha requires k and v to share shape; " + f"got k={tuple(k_paged.shape)}, v={tuple(v_paged.shape)}." + ) + + if plan is None: + # kv_block_num is planned only for the sparse (block-indexed) path; + # dense paged GQA leaves it unset and attends the full page table. + kv_block_num = int(kv_block_indexes.shape[-1]) if kv_block_indexes is not None else -1 + plan = fmha_sm100.fmha_sm100_plan( + qo_lens_cpu, + kv_lens_cpu, + int(q.shape[1]), # num query heads. + num_kv_heads=int(k_paged.shape[1]), + qo_offset=qo_offset_cpu, + page_size=int(k_paged.shape[2]), + kv_block_num=kv_block_num, + causal=causal, + num_kv_splits=1, + use_fp8_kvcache=use_fp8, + ) + fmha_sm100.fmha_sm100( + q, + k_paged, + v_paged, + plan, + kv_indices=kv_indices, + kv_block_indexes=kv_block_indexes, + out=out, + sm_scale=sm_scale, + output_maxscore=False, + ) + + +def run_msa_paged_gqa( + attn: "TrtllmAttention", + q: torch.Tensor, + k: Optional[torch.Tensor], + v: Optional[torch.Tensor], + metadata: "TrtllmAttentionMetadata", + output: torch.Tensor, + *, + kv_block_indexes: Optional[torch.Tensor], + plan: Optional[tuple], +) -> None: + """Write the new-token main K/V, then run paged GQA into output in place. + + Shared by the sparse layers (kv_block_indexes is the per-query top-k table, + with the sparse plan) and the dense layers (kv_block_indexes None, with the + dense plan, attending the full page table). fmha_sm100 reads the paged cache + directly, so the new-token K/V must be resident before the run. + """ + from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_utils import ( + msa_paged_kv, + write_msa_main_kv, + ) + + layer_idx = attn.layer_idx + head_dim = attn.head_dim + kv_cache_manager = metadata.kv_cache_manager + num_tokens = int(q.shape[0]) + if k is not None and v is not None: + write_msa_main_kv( + kv_cache_manager, layer_idx, metadata.msa_out_cache_loc[:num_tokens], k, v + ) + + q_view = q.view(num_tokens, attn.num_heads, head_dim) + out_view = output.view(num_tokens, attn.num_heads, head_dim) + k_paged, v_paged = msa_paged_kv(kv_cache_manager, layer_idx) + sm_scale = (head_dim**-0.5) / float(attn.q_scaling) + + # The fmha_sm100 variant is chosen from q.dtype and shares one dtype across + # q/k/v, so q must be FP8 to match an FP8 paged K/V. MiniMax-M3 has no + # KV-cache scales, so the scale is 1.0 and this is a plain E4M3 cast. + use_fp8 = k_paged.dtype == torch.float8_e4m3fn + if use_fp8: + q_view = q_view.to(torch.float8_e4m3fn) + + run_msa_sparse_gqa( + q_view, + k_paged, + v_paged, + kv_block_indexes, + kv_indices=metadata.msa_kv_indices, + sm_scale=sm_scale, + qo_lens_cpu=metadata.msa_qo_lens_cpu, + kv_lens_cpu=metadata.msa_kv_lens_cpu, + qo_offset_cpu=metadata.msa_qo_offset_cpu, + causal=True, + head_dim=head_dim, + plan=plan, + out=out_view, + use_fp8=use_fp8, + ) + + +class MsaSparseGqaFmha(Fmha): + """SM100 paged GQA FMHA powered by MSA's fmha_sm100 kernel. + + Handles every MiniMax-M3 MSA layer. Sparse layers pass the indexer's + selected KV block indices on forward_args.sparse_prediction.sparse_attn_indices + and attend those blocks; dense layers leave the indices None and attend the + full page table. + + Inherits Fmha rather than PhasedFmha: fmha_sm100 takes a single plan and + the selected block indices span the whole batch, so it handles a mixed + context and generation batch in one call and there is no + context/generation split from PhasedFmha to reuse. Requires head_dim 128 + and 4-D HND paged K/V. + """ + + @classmethod + def is_available(cls, attn: Optional["TrtllmAttention"] = None) -> bool: + # fmha_sm100 runs only on the SM100 family and ships in the MSA git + # submodule, so it is unavailable off SM100 or without the package. + # Imported lazily because the minimax_m3 package init imports the trtllm + # attention classes, which a module-scope import here would cycle with. + from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_utils import ( + msa_package_available, + ) + + if not is_sm_100f() or not msa_package_available(): + return False + # Only the MiniMax-M3 MSA layer uses this library. Matching the lowered + # sparse algorithm lets the base create_fmha_libs add it to that layer + # alone, so no create_fmha_libs override is needed. + return attn.sparse_params is not None and attn.sparse_params.algorithm == "minimax_m3" + + def forward( + self, + q: torch.Tensor, + k: Optional[torch.Tensor], + v: Optional[torch.Tensor], + metadata: "TrtllmAttentionMetadata", + forward_args: "AttentionForwardArgs", + ) -> None: + output = forward_args.output + if output is None: + raise RuntimeError(f"{type(self).__name__} requires an output buffer.") + + # Sparse layers attend the per-query top-k blocks with the sparse plan; + # dense layers leave the indices None and attend the full page table + # with the dense plan. + kv_block_indexes = forward_args.sparse_prediction.sparse_attn_indices + plan = ( + metadata.msa_decode_gqa_plan + if kv_block_indexes is not None + else metadata.msa_decode_dense_plan + ) + run_msa_paged_gqa( + self.attn, + q, + k, + v, + metadata, + output, + kv_block_indexes=kv_block_indexes, + plan=plan, + ) + + +__all__ = ["MsaSparseGqaFmha"] diff --git a/tensorrt_llm/_torch/attention_backend/fmha/registry.py b/tensorrt_llm/_torch/attention_backend/fmha/registry.py index 97467657e9b5..2b3ef3fc7ff3 100644 --- a/tensorrt_llm/_torch/attention_backend/fmha/registry.py +++ b/tensorrt_llm/_torch/attention_backend/fmha/registry.py @@ -22,10 +22,24 @@ FmhaCls: TypeAlias = type[Fmha] -FMHA_LIBS: dict[str, FmhaCls] = { - "flashinfer_trtllm_gen": FlashInferTrtllmGenFmha, - "fallback": FallbackFmha, -} + +def init_fmha_libs() -> dict[str, "FmhaCls"]: + """Build the ordered FMHA library registry. + + Backend classes are imported inside this factory rather than at module + scope, so backends can import trtllm attention classes at module scope + without an import cycle. + """ + from .msa_sparse_gqa import MsaSparseGqaFmha + + return { + "msa_sparse_gqa": MsaSparseGqaFmha, + "flashinfer_trtllm_gen": FlashInferTrtllmGenFmha, + "fallback": FallbackFmha, + } + + +FMHA_LIBS: dict[str, FmhaCls] = init_fmha_libs() DEFAULT_FMHA_LIBS: tuple[str, ...] = tuple(FMHA_LIBS) @@ -78,4 +92,5 @@ def get_enabled_fmha_lib_classes() -> list[FmhaCls]: "FMHA_LIBS", "FmhaCls", "get_enabled_fmha_lib_classes", + "init_fmha_libs", ] diff --git a/tensorrt_llm/_torch/attention_backend/interface.py b/tensorrt_llm/_torch/attention_backend/interface.py index fc6cfdd50222..ea6cfbf2f85e 100644 --- a/tensorrt_llm/_torch/attention_backend/interface.py +++ b/tensorrt_llm/_torch/attention_backend/interface.py @@ -411,10 +411,14 @@ def create_cuda_graph_metadata(self, return cuda_graph_metadata def prepare_for_spec_dec(self, *fields) -> None: - assert len(self._saved_tensors) == 0 + assert len(self._saved_tensors) == 0, ( + "prepare_for_spec_dec called while fields " + f"{list(self._saved_tensors)} are still saved; a previous " + "forward likely raised between prepare_for_spec_dec and " + "restore_from_spec_dec") for f in fields: v = getattr(self, f) - assert isinstance(v, torch.Tensor) + assert isinstance(v, torch.Tensor), f"{f} is not a torch.Tensor" self._saved_tensors[f] = v setattr(self, f, v.clone()) @@ -423,6 +427,11 @@ def restore_from_spec_dec(self) -> None: setattr(self, f, v) self._saved_tensors.clear() + @property + def has_spec_dec_saved_state(self) -> bool: + """True when prepare_for_spec_dec state has not been restored yet.""" + return bool(self._saved_tensors) + def update_spec_dec_param( self, batch_size, @@ -665,6 +674,7 @@ def from_config(config) -> "RopeParams": rope_params = RopeParams() hf_rope_parameters = getattr(config, 'rope_parameters', None) + normalized_rope_parameters = hf_rope_parameters if hf_rope_parameters is not None: if set(hf_rope_parameters.keys()).issubset( ALLOWED_ATTENTION_LAYER_TYPES): @@ -672,17 +682,17 @@ def from_config(config) -> "RopeParams": # Pick "full_attention" as the default; callers override theta # for sliding-window layers independently. if "full_attention" in hf_rope_parameters: - flat = hf_rope_parameters["full_attention"] + normalized_rope_parameters = hf_rope_parameters[ + "full_attention"] else: fallback_key = next(iter(hf_rope_parameters)) logger.warning( f"Per-layer-type rope_parameters has no 'full_attention' entry; " f"falling back to '{fallback_key}'. Available layer types: " f"{list(hf_rope_parameters.keys())}.") - flat = hf_rope_parameters[fallback_key] - config.update(flat) - else: - config.update(hf_rope_parameters) + normalized_rope_parameters = hf_rope_parameters[ + fallback_key] + config.update(normalized_rope_parameters) # get rotary parameters. hidden_size = config.hidden_size @@ -691,6 +701,8 @@ def from_config(config) -> "RopeParams": if not isinstance(head_dim, int): head_dim = hidden_size // num_attention_heads rope_scaling = getattr(config, 'rope_scaling', None) + if rope_scaling is None and normalized_rope_parameters is not None: + rope_scaling = normalized_rope_parameters rope_params.max_positions = config.max_position_embeddings rope_params.theta = get_hf_rope_theta(config, 10000.0) rope_percentage = (getattr(config, 'rotary_pct', None) diff --git a/tensorrt_llm/_torch/attention_backend/sparse/__init__.py b/tensorrt_llm/_torch/attention_backend/sparse/__init__.py index 0b2994441bf0..f293f9547506 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/__init__.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/__init__.py @@ -1,13 +1,3 @@ -# yapf: disable -from .minimax_m3 import (MiniMaxM3SparseAttention, - MiniMaxM3SparseAttentionMetadata, - MiniMaxM3SparseConfig, MiniMaxM3SparseIndexCache, - allocate_minimax_m3_static_buffers, - build_runtime_metadata_from_kv_manager, - get_minimax_m3_attention_backend_cls, - get_minimax_m3_kv_cache_manager_cls, - minimax_m3_sparse_decode, minimax_m3_sparse_prefill) -# yapf: enable from .utils import (get_flashinfer_sparse_attn_attention_backend, get_sparse_attn_kv_cache_manager, get_trtllm_sparse_attn_attention_backend, @@ -18,14 +8,4 @@ "get_vanilla_sparse_attn_attention_backend", "get_trtllm_sparse_attn_attention_backend", "get_flashinfer_sparse_attn_attention_backend", - "MiniMaxM3SparseAttention", - "MiniMaxM3SparseAttentionMetadata", - "MiniMaxM3SparseConfig", - "MiniMaxM3SparseIndexCache", - "allocate_minimax_m3_static_buffers", - "build_runtime_metadata_from_kv_manager", - "get_minimax_m3_attention_backend_cls", - "get_minimax_m3_kv_cache_manager_cls", - "minimax_m3_sparse_decode", - "minimax_m3_sparse_prefill", ] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/cache_manager.py b/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/cache_manager.py index c7ca1891b642..933c1dc443b9 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/cache_manager.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/cache_manager.py @@ -14,16 +14,13 @@ # limitations under the License. from collections import defaultdict +from dataclasses import replace from typing import Dict, List, Optional, Tuple import torch from tensorrt_llm._torch.pyexecutor import llm_request -from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import ( - GPU_LEVEL, - BlockReusePolicy, - KVCacheManagerV2, -) +from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import GPU_LEVEL, KVCacheManagerV2 from tensorrt_llm._utils import ( TensorWrapper, convert_to_torch_tensor, @@ -39,16 +36,11 @@ from tensorrt_llm.runtime import ModelConfig from tensorrt_llm.runtime.kv_cache_manager_v2 import ( AttentionLayerConfig, - BatchDesc, BufferConfig, DataRole, - GpuCacheTierConfig, - HostCacheTierConfig, - KVCacheDesc, LayerId, PageIndexMode, ScratchDesc, - SwaScratchReuseConfig, ) from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheManagerConfig as KVCacheManagerConfigPy from tensorrt_llm.runtime.kv_cache_manager_v2._common import BAD_PAGE_INDEX @@ -183,6 +175,14 @@ class DeepseekV4CacheManager(KVCacheManagerV2): # For other attention types, block size is tokens_per_block. compressed_block_sizes: List[int] + def _get_typical_seq_len(self, kv_cache_config: KvCacheConfig) -> int: + """Retain DeepSeek-V4's max-length pool-sizing model by default.""" + return ( + kv_cache_config.avg_seq_len + if kv_cache_config.avg_seq_len is not None + else self.max_seq_len + ) + def __init__( self, kv_cache_config: KvCacheConfig, @@ -254,7 +254,6 @@ def __init__( self._init_indexer_dtype(sparse_attn_config) - # _build_cache_config() needs them to build constraints self._max_input_len = max_input_len self._max_num_tokens = max_num_tokens @@ -271,6 +270,7 @@ def __init__( vocab_size=vocab_size, mapping=mapping, dtype=dtype, + max_num_tokens=max_num_tokens, **kwargs, ) self.is_vswa = True # DeepSeek-V4 must has VSWA @@ -768,16 +768,9 @@ def _get_max_tokens_from_quota(self, quota: int) -> float: return float("inf") return self._max_num_tokens + (quota - context_limit_quota) / generation_size_per_token - def _build_cache_config( - self, - kv_cache_config: KvCacheConfig, - *, - tokens_per_block: int, - vocab_size: int | None, - cache_tiers: List[GpuCacheTierConfig | HostCacheTierConfig], - ) -> KVCacheManagerConfigPy: + def _build_cache_config(self, config: KVCacheManagerConfigPy) -> KVCacheManagerConfigPy: """ - Create the cache manager config for DeepSeek-V4. + Add DeepSeek-V4 layers to the cache config. """ layers: List[AttentionLayerConfig] = [] layer_attn_to_layer_id: Dict[Tuple[int, DeepseekV4AttentionType], LayerId] = {} @@ -864,89 +857,9 @@ def _add_layer( # number of layers in the KVCacheManagerPy self._num_manager_layers = len(layers) - # Build constraints and typical_step for better pool ratio. - max_batch_size = self.max_batch_size - max_seq_len = self.max_seq_len - max_num_tokens = self._max_num_tokens - max_draft_len = self._max_draft_len - typical_step = None - constraints = [] - if kv_cache_config.pool_ratio is None: - typical_seq_len = ( - kv_cache_config.avg_seq_len - if kv_cache_config.avg_seq_len is not None - else max_seq_len - ) - if typical_seq_len > max_seq_len: - raise ValueError( - f"kv_cache_config.avg_seq_len ({typical_seq_len}) must be less than or " - f"equal to max_seq_len ({max_seq_len})" - ) - - # For aggregated serving in large batch size: - # Use 1 context request + (max_batch_size - 1) generation requests as - # the typical step. An all-generation typical_step over-provisions the - # compressed-cache pool at the expense of the SWA pool, starving the - # SWA pool and artificially capping the achievable batch size. - ctx_capacity = max_num_tokens if max_num_tokens is not None else typical_seq_len - generation_history_length = max(0, typical_seq_len - max_draft_len - 1) - typical_step = BatchDesc( - kv_caches=[ - KVCacheDesc(capacity=ctx_capacity, history_length=0), - ] - + [ - KVCacheDesc( - capacity=typical_seq_len, - history_length=generation_history_length, - ) - ] - * (max_batch_size - 1), - ) - - # Constraint 1: cuda graph generation warmup — one decode request that has - # accumulated to the tail of max_seq_len. Using history_length=max_seq_len-1 - # (instead of 0) lets SWA / SSM pools collapse to their windowed working set, - # while full-cache pools still need max_seq_len/tokens_per_block blocks - # because they don't age. - constraints.append( - BatchDesc([KVCacheDesc(capacity=max_seq_len, history_length=max_seq_len - 1)]) - ) - - # Constraint 2: general / chunked-prefill warmup — one fresh context request - # at max_num_tokens (the per-iteration token budget). - if max_num_tokens is not None: - constraints.append( - BatchDesc( - [ - KVCacheDesc( - capacity=max_num_tokens + self.num_extra_kv_tokens, history_length=0 - ) - ] - ) - ) - - scratch_reuse_config = None - if self.enable_swa_scratch_reuse: - # Context requests will allocate num_extra_kv_tokens tokens for spec decoding. - # Cache manager should not take them into account when calculating scratch range. - # Therefore set max_rewind_len to num_extra_kv_tokens. - scratch_reuse_config = SwaScratchReuseConfig(max_rewind_len=self.num_extra_kv_tokens) - - return KVCacheManagerConfigPy( - tokens_per_block=tokens_per_block, - cache_tiers=cache_tiers, - max_util_for_resume=kv_cache_config.max_util_for_resume, - enable_partial_reuse=kv_cache_config.enable_partial_reuse, - swa_scratch_reuse=scratch_reuse_config, - commit_min_snapshot=( - kv_cache_config.enable_block_reuse - and self.block_reuse_policy != BlockReusePolicy.ALL_REUSABLE - ), + return replace( + config, layers=layers, - typical_step=typical_step, - constraints=constraints, - enable_stats=self.enable_stats, - initial_pool_ratio=kv_cache_config.pool_ratio, ) def _init_indexer_dtype(self, sparse_attn_config: DeepSeekV4SparseAttentionConfig) -> None: @@ -1161,9 +1074,9 @@ def get_layer_bytes_per_token( local_layer_idx: int, data_role: DataRole, ) -> int: - raise NotImplementedError( - "DeepSeek-V4 doesn't support get_layer_bytes_per_token, use _get_attn_bytes_per_block" - ) + # The generic layers in the base config are replaced by + # _build_cache_config, so their buffer sizes are only placeholders. + return 1 def get_indexer_k_cache_buffers(self, layer_idx: int) -> torch.Tensor: """ @@ -1301,8 +1214,13 @@ def copy_batch_block_offsets( beam_width: int, num_contexts: int, num_seqs: int, + max_blocks: Optional[int] = None, ) -> None: - """For compatibility with AttentionOp, copy only the SWA block offsets.""" + """For compatibility with AttentionOp, copy only the SWA block offsets. + + max_blocks is accepted for signature parity with KVCacheManager; the + copy below is already bounded by the precomputed SWA table width. + """ assert beam_width == 1, "DSV4 only supports beam width 1 now" assert dst_tensor.is_cuda, "copy_batch_block_offsets expects a CUDA destination" dst_tensor.fill_(BAD_PAGE_INDEX) diff --git a/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/deepseek_v4.py b/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/deepseek_v4.py index 52f16c84ffe9..1f76e5a73e51 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/deepseek_v4.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/deepseek_v4.py @@ -32,7 +32,7 @@ from tensorrt_llm._torch.modules.multi_stream_utils import do_multi_stream from tensorrt_llm._torch.modules.rotary_embedding import RotaryEmbedding from tensorrt_llm._torch.utils import maybe_compile -from tensorrt_llm._utils import prefer_pinned +from tensorrt_llm._utils import is_sm_100f, prefer_pinned from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization.utils import fp8_utils from tensorrt_llm.runtime.kv_cache_manager_v2 import DataRole @@ -250,6 +250,7 @@ def make_deepseek_v4_sparse_metadata_params( ), enable_indexer_skip=sparse_attention_config.skip_indexer_for_short_seqs, enable_heuristic_topk=sparse_attention_config.enable_heuristic_topk, + use_cute_dsl_topk=sparse_attention_config.use_cute_dsl_topk, use_cute_dsl_paged_mqa_logits=(sparse_attention_config.use_cute_dsl_paged_mqa_logits), q_split_threshold=sparse_attention_config.q_split_threshold, compress_ratios=sparse_attention_config.compress_ratios, @@ -274,6 +275,7 @@ def __post_init__(self): super().__post_init__() self.num_total_compressed_tokens = {} self.max_ctx_compressed_tokens = {} + self._ctx_output_sizes: Optional[Dict[int, int]] = None sparse_metadata_params = self.sparse_metadata_params if not isinstance(sparse_metadata_params, DeepSeekV4MetadataParams): raise ValueError("DeepSeek-V4 sparse attention metadata params are not set") @@ -529,6 +531,10 @@ def prepare_for_indexer_k_cache(self): self.host_indexer_k_cache_block_offsets[: self.num_seqs], non_blocking=True, ) + # Columns beyond each sequence's allocated indexer blocks contain BAD_PAGE_INDEX (-1). + # CUDA-graph padded token slots may still compute scatter addresses from those columns + # before being ignored, so map them to block 0, matching the base DSA metadata path. + self.indexer_k_cache_block_offsets.clamp_(min=0) def prepare_for_block_tables(self): """Prepare block tables for sliding-window and compressed attention.""" @@ -732,12 +738,18 @@ def prepare(self): kv_lens_slice = kv_lens[:num_requests] cached_slice = cached_token_lens[:num_requests] + # Host-side per-ratio ctx compressed-token counts (Python ints), so + # _compute_ctx_compressed_position_ids never reads a device scalar + # (implicit D2H + stream sync) for its arange size / slice bound. + ctx_output_sizes: Optional[Dict[int, int]] = None if num_contexts > 0: # Prefill path: need per-request tensor ops for ctx scalar metadata. + ctx_output_sizes = {} for compress_ratio in self.compress_ratio_set: new_comp_kv_lens = kv_lens_slice // compress_ratio - cached_slice // compress_ratio cu_new = new_comp_kv_lens.cumsum(0) num_ctx_compressed_tokens = cu_new[num_contexts - 1].item() + ctx_output_sizes[compress_ratio] = num_ctx_compressed_tokens num_gen_compressed_tokens = num_generations * ( (num_gen_tokens_per_seq + compress_ratio - 1) // compress_ratio ) @@ -756,12 +768,15 @@ def prepare(self): ) self.max_ctx_compressed_tokens[compress_ratio] = 0 + # Cached for on_update_kv_lens(); see the reuse gate there. + self._ctx_output_sizes = ctx_output_sizes + # 2) CUDA-side: fill *_cuda buffers on device. kv_lens_cuda = ( self.cached_token_lens_cuda[:num_requests] + self._seq_lens_cuda[:num_requests] ) cached_tokens_cuda = self.cached_token_lens_cuda[:num_requests] - self.prepare_compressed_kv_metadata(kv_lens_cuda, cached_tokens_cuda) + self.prepare_compressed_kv_metadata(kv_lens_cuda, cached_tokens_cuda, ctx_output_sizes) self._compute_compressed_mask( self.new_comp_kv_lens_cuda, @@ -776,6 +791,7 @@ def prepare_compressed_kv_metadata( self, kv_lens: torch.Tensor, cached_tokens: torch.Tensor, + ctx_output_sizes: Optional[Dict[int, int]] = None, ): """Compute per-ratio compressed KV lens and position IDs on device. @@ -784,6 +800,12 @@ def prepare_compressed_kv_metadata( Args: kv_lens: Total KV lengths per request (device tensor, [batch_size]). cached_tokens: Cached token counts per request (device tensor, [batch_size]). + ctx_output_sizes: Optional per-ratio host-computed ctx + compressed-token counts (Python ints); avoids implicit + device-scalar reads (D2H + stream sync) in the ctx position-id + computation. prepare() always passes it; on_update_kv_lens() + reuses the cached copy unless the extend_ctx path may have + mutated ctx-row kv_lens on device. """ batch_size = kv_lens.shape[0] num_contexts = self.num_contexts @@ -807,6 +829,7 @@ def prepare_compressed_kv_metadata( self.compressed_position_ids_cuda, num_contexts, self._compress_ratios_sorted, + ctx_output_sizes, ) if self.num_gen_tokens_per_seq > 0 and num_generations > 0: @@ -843,7 +866,13 @@ def on_update_kv_lens(self): num_gen_tokens // self.num_generations if self.num_generations > 0 else 0 ) - self.prepare_compressed_kv_metadata(kv_lens, cached_tokens) + # Reuse prepare()'s host-computed ctx sizes unless the extend_ctx path + # (num_chunked_ctx_requests > 0) may have mutated ctx-row kv_lens on + # device; every other path only changes gen rows. + ctx_output_sizes = ( + self._ctx_output_sizes if getattr(self, "num_chunked_ctx_requests", 0) == 0 else None + ) + self.prepare_compressed_kv_metadata(kv_lens, cached_tokens, ctx_output_sizes) self._compute_compressed_mask( self.new_comp_kv_lens_cuda, @@ -999,14 +1028,24 @@ def _compute_ctx_compressed_position_ids( compressed_position_ids_bufs: Dict[int, torch.Tensor], num_contexts: int, compress_ratios: list, + ctx_output_sizes: Optional[Dict[int, int]] = None, ): - """Context-only compressed position IDs (eager, data-dependent shapes).""" + """Context-only compressed position IDs (eager, data-dependent shapes). + + ctx_output_sizes (host ints) keeps the arange size and slice bound off + the device; the 0-dim-CUDA fallback costs two implicit D2H syncs per + ratio. + """ device = past_kv_lens_bufs[compress_ratios[0]].device for compress_ratio in compress_ratios: past_kv = past_kv_lens_bufs[compress_ratio] cu_new_comp = cu_new_comp_kv_bufs[compress_ratio] - total_ctx_comp = cu_new_comp[num_contexts] + total_ctx_comp = ( + ctx_output_sizes[compress_ratio] + if ctx_output_sizes is not None + else cu_new_comp[num_contexts] + ) ctx_idx = torch.arange(total_ctx_comp, dtype=torch.int32, device=device) ctx_cu = cu_new_comp[: num_contexts + 1].to(torch.int32) ctx_req = torch.searchsorted(ctx_cu[1:], ctx_idx, right=True) @@ -1040,6 +1079,11 @@ def __init__( layer_idx, aux_stream, ) + # Preserve the checkpoint's 128x128 FP8 quantization while deriving a + # native-MXF8 (sf_vec=32) scale view for the TRT-LLM CuTe DSL fused + # indexer-Q projection. FP8BlockScalesLinearMethod materializes the + # derived, swizzled scale once after weight loading. + self.wq_b.use_indexer_q_cutedsl_fusion = True # Override base Indexer.weights_proj to bf16 (matches V4 checkpoint). self.weights_proj = Linear( self.hidden_size, @@ -1099,6 +1143,10 @@ def _qk_projection_and_rope(self, qr: torch.Tensor, position_ids: torch.Tensor): """ q = self.wq_b(qr) q = q.view(-1, self.n_heads, self.head_dim) + return self._apply_q_rope(q, position_ids) + + def _apply_q_rope(self, q: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor: + """Apply RoPE in-place to a projected indexer Q tensor.""" # Fused in-place RoPE on the rope portion of each head nope_dim = self.head_dim - self.rope_dim torch.ops.trtllm.mla_rope_inplace( @@ -1113,6 +1161,49 @@ def _qk_projection_and_rope(self, qr: torch.Tensor, position_ids: torch.Tensor): ) return q + def _project_and_quantize_q( + self, qr: torch.Tensor, position_ids: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Project Q and produce the cache precision consumed by the indexer. + + The DSv4 MXFP4 configuration can fuse projection, interleaved RoPE, + and FP4 quantization. If the optional Hadamard transform is active, + retain the legacy path because that transform changes all 128 values + in a head and is not part of the CuTe DSL kernel. + """ + use_fused_project_mxfp4 = ( + self.indexer_cache_dtype == KVCacheDtype.MXFP4_BLOCKWISE + and not HAS_FAST_HADAMARD + and not self.rotary_emb.is_neox + and self.head_dim == 128 + and self.rope_dim == 64 + and self.wq_b.has_fp8_block_scales + and hasattr(self.wq_b, "indexer_q_weight_scale_cutedsl") + and hasattr( + torch.ops.trtllm, + "cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell", + ) + and qr.dtype == torch.bfloat16 + and is_sm_100f() + ) + if use_fused_project_mxfp4: + q_fp4, q_scale = torch.ops.trtllm.cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell( + qr, + self.wq_b.weight, + self.wq_b.indexer_q_weight_scale_cutedsl, + position_ids.view(-1), + self.rotary_emb.rotary_cos_sin.view(-1, self.rope_dim), + self.wq_b.indexer_q_alpha_cutedsl, + use_tvm_ffi=True, + ) + return q_fp4.view(-1, self.n_heads, self.head_dim // 2), q_scale.view( + -1, self.n_heads, 1 + ) + + q = self.wq_b(qr).view(-1, self.n_heads, self.head_dim) + q = self._apply_q_rope(q, position_ids) + return self._quantize_q(q) + def _quantize_q(self, q: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: # Rotate + quantize (layout matches compressor K: [nope|pe]). After # rotate_activation (Hadamard) the nope/rope split becomes a linear @@ -1242,7 +1333,7 @@ def _run_overlapped_indexer_prepare( if pre_aux is None: self.indexer_start_event.record() - q = self._qk_projection_and_rope(qr, position_ids) + q_fp8, q_scale = self._project_and_quantize_q(qr, position_ids) with torch.cuda.stream(self.aux_stream): self.indexer_start_event.wait() @@ -1263,9 +1354,7 @@ def _run_overlapped_indexer_prepare( k_fp8.record_stream(cur_stream) if k_scale is not None: k_scale.record_stream(cur_stream) - q = self._qk_projection_and_rope(qr, position_ids) - - q_fp8, q_scale = self._quantize_q(q) + q_fp8, q_scale = self._project_and_quantize_q(qr, position_ids) self.weights_proj_event.wait() weights = self._apply_weight_scale(weights, q_scale) @@ -1282,9 +1371,7 @@ def _run_serial_indexer_prepare( ) -> Tuple[ torch.Tensor, torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], torch.Tensor ]: - q = self._qk_projection_and_rope(qr, position_ids) - - q_fp8, q_scale = self._quantize_q(q) + q_fp8, q_scale = self._project_and_quantize_q(qr, position_ids) weights = self.weights_proj(hidden_states) diff --git a/tensorrt_llm/_torch/attention_backend/sparse/dsa.py b/tensorrt_llm/_torch/attention_backend/sparse/dsa.py index af85390cf72e..a57fdadf0776 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/dsa.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/dsa.py @@ -90,6 +90,7 @@ class DSAMetadataParams(SparseMetadataParams): index_head_dim: int enable_indexer_skip: bool enable_heuristic_topk: bool + use_cute_dsl_topk: bool use_cute_dsl_paged_mqa_logits: bool q_split_threshold: int @@ -196,6 +197,16 @@ def warmup_heuristic_topk_decode(top_k: int = 2048, # SM100-aware num_math_warpgroups in the metadata JIT impl). _DG_SCHEDULE_BLOCK_KV = 64 +# dtype of the indexer MQA-logits that feed the top-k. All paged_mqa_logits +# paths produce fp32 today (DSL fp8/fp4 default output_dtype=fp32; DeepGEMM +# fp8 hardcodes kFloat; DeepGEMM fp4 defaults logits_dtype=kFloat32 and is not +# overridden here), and the decode forward feeds logits to the top-k without a +# cast. dtype is a top-k compile-key dimension, so the warmup pre-compiles for +# exactly this value. If a paged_mqa_logits caller ever emits a different dtype +# (e.g. overriding the DeepGEMM fp4 logits_dtype to bf16), update this constant +# or the warmup silently compiles the wrong variant. +_INDEXER_LOGITS_DTYPE = torch.float32 + def _pick_dsl_expand( next_n: int, @@ -673,6 +684,9 @@ def __post_init__(self): self.indexer_head_dim = sparse_metadata_params.index_head_dim self.indexer_quant_block_size = 128 self.enable_indexer_skip = (sparse_metadata_params.enable_indexer_skip) + self.use_cute_dsl_topk = (sparse_metadata_params.use_cute_dsl_topk + and IS_CUTLASS_DSL_AVAILABLE) + self.kv_lens_row_reorder = None capture_graph = self.is_cuda_graph # Plain DSA has no compression and uses the default [1]. DeepSeek-V4's # metadata params carry the model-specific compression ratios. @@ -748,6 +762,54 @@ def get_indexer_max_seq_len(self) -> int: 1, self.kv_cache_manager.max_seq_len // self._indexer_compress_ratio) + def warmup_cute_dsl_radix_topk(self, next_n: int) -> None: + """Pre-compile the radix-filter CuTe DSL decode top-k during warmup. + + Eager decode iters (mixed prefill+decode batch, or ``cuda_graph`` + disabled) whose ``num_rows`` lands in a ``cluster_size`` band that + graph capture did not exercise otherwise pay a first-touch JIT stall + on a live request. ``num_cols`` is fixed at ``indexer_max_seq_len``, + so only the ``cluster_size`` dimension needs sweeping; delegate to the + custom-op warmup helper, which owns the band enumeration. + + ``next_n`` (a compile-key dimension) is supplied by the caller from + the engine's static spec-decode config. + + No-op unless decode actually routes to + ``cute_dsl_indexer_topk_decode``: heuristic top-k uses the GVR kernel + and plain (no cute_dsl_topk) decode uses the C++ op. Called once from + ``ModelEngine.warmup``. + """ + if not self.use_cute_dsl_topk or self.enable_heuristic_topk: + return + if self.kv_cache_manager is None: + return + top_k = getattr(self.sparse_metadata_params, "index_topk", None) + if not top_k: + return + # The radix-filter DSL kernel does not support a compressed indexer + # combined with multi-row MTP: decode dispatches to it only when + # compress_ratio == 1 or next_n == 1. The compress_ratio > 1 && + # next_n > 1 case routes to the C++ op (or GVR when heuristic top-k is + # on), so there is nothing to pre-compile here. + # TODO: extending the radix-filter path to compress_ratio > 1 && + # next_n > 1 is straightforward; once the dispatch above is relaxed to + # use it there, drop this guard so the case is pre-compiled too. + if self._indexer_compress_ratio > 1 and next_n > 1: + return + try: + from ...custom_ops.cute_dsl_custom_ops import \ + warmup_cute_dsl_radix_topk_decode + except ImportError: + return + warmup_cute_dsl_radix_topk_decode( + top_k=int(top_k), + num_cols=int(self.get_indexer_max_seq_len()), + next_n=next_n, + dtype=_INDEXER_LOGITS_DTYPE, + num_sms=self.num_sms, + ) + def on_update_kv_lens(self): # After changing the kv_lens/kv_lens_cuda, we may need to update other metadatas. # Especially for the changes in the _preprocess_inputs() of model_engine.py. @@ -856,8 +918,37 @@ def on_update_kv_lens(self): _DG_SCHEDULE_BLOCK_KV, self.num_sms) self.scheduler_metadata_buffer_expanded.copy_( scheduler_metadata_buffer_expanded, non_blocking=True) + self._compute_kv_lens_row_reorder() self.prepare_dense_topk_indices(self.kv_lens_cuda, device=True) + def _compute_kv_lens_row_reorder(self): + """LJF (longest-job-first) row-reorder for the GVR DSL top-k path. + + Writes ``argsort(gen_kv_lens, descending)`` into the stable buffer when + the multi-wave threshold is met, otherwise leaves ``order_row`` None. + Called from ``on_update_kv_lens()`` (both base and DeepSeek-V4 via + super()) unconditionally every forward step so the GVR op sees a fresh + valid permutation and never a stale one from a prior step. Copies into + the stable buffer (not a fresh tensor) so the CUDA-Graph-captured op + reads a valid permutation on every replay. + """ + # Gate on row count (num_generations * next_n) rather than request count + # so the threshold aligns with the kernel-side tuning note that records + # the win region starting at num_rows >= 2 * num_sms. Using + # num_generations alone is only correct for next_n == 2; for next_n == 1 + # it engages inside the measured regression band, and for next_n == 4 it + # misses the win region between 2*num_sms and 4*num_sms rows. + next_n = 1 + self.max_draft_tokens + if (self.enable_heuristic_topk and self.use_cute_dsl_topk + and self.num_generations * next_n >= 2 * self.num_sms): + gen_kv_lens = self.kv_lens_cuda[self.num_contexts:self.num_seqs] + order = torch.argsort(gen_kv_lens, descending=True).to(torch.int32) + self.kv_lens_row_reorder_buffer[:self.num_generations].copy_(order) + self.kv_lens_row_reorder = \ + self.kv_lens_row_reorder_buffer[:self.num_generations] + else: + self.kv_lens_row_reorder = None + def update_for_spec_dec(self): super().update_for_spec_dec() # host @@ -1105,15 +1196,30 @@ def create_buffers_for_indexer(self, capture_graph=False): # Pre-allocated with stable address for CUDA Graph compatibility # (replaces cudaMallocAsync/cudaFreeAsync inside the kernel launcher). # Shape: [max_gen_tokens, topK] where max_gen_tokens = max_batch * (1 + max_draft). - max_gen_tokens = self.max_num_sequences * (1 + - self.max_draft_tokens) - self.heuristic_scratch_values = self.get_empty( - self.cuda_graph_buffers, - (max_gen_tokens, self.num_sparse_topk), - cache_name="heuristic_scratch_values", - dtype=torch.float32, - capture_graph=capture_graph, - ) + # Only the C++ indexer_topk_decode path consumes it; the GVR DSL + # path does not, so skip the allocation when use_cute_dsl_topk. + if not self.use_cute_dsl_topk: + max_gen_tokens = self.max_num_sequences * ( + 1 + self.max_draft_tokens) + self.heuristic_scratch_values = self.get_empty( + self.cuda_graph_buffers, + (max_gen_tokens, self.num_sparse_topk), + cache_name="heuristic_scratch_values", + dtype=torch.float32, + capture_graph=capture_graph, + ) + # Stable-address buffer for the GVR DSL LJF row-reorder + # (order_row = argsort(gen_kv_lens, descending)). Must not be + # fresh-allocated per step: under CUDA Graph the captured op reads + # a frozen address, so prepare() copies into this buffer instead. + if self.use_cute_dsl_topk: + self.kv_lens_row_reorder_buffer = self.get_empty( + self.cuda_graph_buffers, + (self.max_num_sequences, ), + cache_name="kv_lens_row_reorder_buffer", + dtype=torch.int32, + capture_graph=capture_graph, + ) # Persistent scratch for the Radix-split-work indexer path. Re-created # in update_spec_dec_param when max_draft_tokens changes so it stays @@ -1210,7 +1316,9 @@ def update_spec_dec_param( if self.max_num_sequences * (1 + self.max_draft_tokens) != init_shape: self.create_expanded_buffers(capture_graph=capture_graph) # Resize heuristic scratch buffer for new max_draft_tokens. - if self.enable_heuristic_topk: + # Skip when use_cute_dsl_topk (GVR path never consumes it), matching + # the allocation guard in create_buffers_for_indexer. + if self.enable_heuristic_topk and not self.use_cute_dsl_topk: max_gen_tokens = self.max_num_sequences * ( 1 + self.max_draft_tokens) self.heuristic_scratch_values = self.get_empty( @@ -1480,11 +1588,17 @@ def prepare_for_indexer_k_cache(self): 1) // tokens_per_block max_blocks_used = num_blocks_per_seq.max().item( ) if self.num_seqs > 0 else 1 - # pool_indices already has correct values; set padding to -1 - host_block_table = pool_indices[:, :max_blocks_used].clone() - for i in range(self.num_seqs): - if num_blocks_per_seq[i] < max_blocks_used: - host_block_table[i, num_blocks_per_seq[i]:] = -1 + # pool_indices already has correct values; set padding to -1. + # Stage through a fresh pinned buffer: an async H2D from pageable + # memory would block the host behind the busy execution stream. + host_block_table = torch.empty((pool_indices.shape[0], max_blocks_used), + dtype=pool_indices.dtype, + pin_memory=prefer_pinned()) + host_block_table.copy_(pool_indices[:, :max_blocks_used]) + pad_cols = torch.arange(max_blocks_used, dtype=num_blocks_per_seq.dtype) + host_block_table.masked_fill_( + pad_cols.unsqueeze(0) + >= num_blocks_per_seq[:self.num_seqs].unsqueeze(1), -1) # Copy to GPU self.block_table[:self.num_seqs, :max_blocks_used].copy_( host_block_table, non_blocking=True) @@ -1760,16 +1874,6 @@ def __init__(self, self._enable_heuristic_topk = (sparse_params.enable_heuristic_topk and get_sm_version() >= 100) - if (self.use_cute_dsl_topk - or self.use_cute_dsl_paged_mqa_logits) and layer_idx == 0: - from tensorrt_llm._torch.custom_ops import cute_dsl_custom_ops - - if self.use_cute_dsl_topk: - # the dtype of topk input tensor, which is float32 now. - # Note, need to update it if the dtype of topk input tensor is changed. - cute_dsl_custom_ops.warmup_cute_dsl_indexer_topk( - dtype=torch.float32, top_k=self.index_topk) - if self._enable_heuristic_topk and layer_idx == 0: # Populate static caches (sm_count, L2 cache size) inside the C++ # Scheme X dispatcher before any CUDA Graph capture so the host @@ -2308,16 +2412,27 @@ def _gather_k_cache_for_chunk( return k_fp8, k_scale - def _call_mqa_logits(self, q_fp8: torch.Tensor, k_fp8: torch.Tensor, - k_scale: torch.Tensor, weights: torch.Tensor, - cu_seqlen_ks: torch.Tensor, cu_seqlen_ke: torch.Tensor, - q_scale: Optional[torch.Tensor]) -> torch.Tensor: + def _call_mqa_logits(self, + q_fp8: torch.Tensor, + k_fp8: torch.Tensor, + k_scale: torch.Tensor, + weights: torch.Tensor, + cu_seqlen_ks: torch.Tensor, + cu_seqlen_ke: torch.Tensor, + q_scale: Optional[torch.Tensor], + clean_logits: bool = True) -> torch.Tensor: """Dispatch fp8_mqa_logits vs fp8_fp4_mqa_logits based on use_fp4. For FP4 the gather output keeps the legacy float8_e4m3fn dtype for API compatibility; reinterpret the bytes as the int8 / int32 layout the DeepGEMM kernel expects. The scale tensor is collapsed to 1D for the kv side and 2D for the q side per the kernel's asserts. + + clean_logits=False skips DeepGEMM's smxx_clean_logits pass that fills + everything outside each row's [ks, ke) window with -inf. Safe only + when the consumer never reads outside that window (the custom + indexer_topk_prefill kernel); the torch topk fallback scans the full + padded row and needs the fill. """ if self.use_fp4: k_fp4_bytes = k_fp8.view(torch.int8) @@ -2331,9 +2446,13 @@ def _call_mqa_logits(self, q_fp8: torch.Tensor, k_fp8: torch.Tensor, weights, cu_seqlen_ks, cu_seqlen_ke, + clean_logits=clean_logits, ) - return fp8_mqa_logits(q_fp8, (k_fp8, k_scale.reshape(-1)), weights, - cu_seqlen_ks, cu_seqlen_ke) + return fp8_mqa_logits(q_fp8, (k_fp8, k_scale.reshape(-1)), + weights, + cu_seqlen_ks, + cu_seqlen_ke, + clean_logits=clean_logits) def _call_paged_mqa_logits(self, q_decode: torch.Tensor, k_cache: torch.Tensor, @@ -2478,6 +2597,7 @@ def sparse_attn_indexer( chunk.cu_seqlen_ks[c0:c1], chunk.cu_seqlen_ke[c0:c1], tile_q_scale, + clean_logits=not use_custom_topk, ) if use_custom_topk: torch.ops.trtllm.indexer_topk_prefill( @@ -2533,6 +2653,7 @@ def sparse_attn_indexer( cu_seqlen_ks, cu_seqlen_ke, ctx_q_scale, + clean_logits=not use_custom_topk, ) if use_custom_topk: torch.ops.trtllm.indexer_topk_prefill( @@ -2750,17 +2871,31 @@ def sparse_attn_indexer( # handled inside the C++ kernel (preIdxOffset += 1). pre_idx = metadata.heuristic_prev_topk[ local_layer, :num_generations] - heuristic_scratch = \ - metadata.heuristic_scratch_values[ - :num_gen_tokens] - - # CuTE DSL top-k allocates O(num_gen_tokens * kv_len) global - # memory. Beyond 256 tokens the extra memory becomes significant, - # so we cap it at 256 for now and fall back to the CUDA C++ - # indexer_topk_decode. This limit can be removed if GPU memory - # is not a bottleneck. - if (self.use_cute_dsl_topk and num_gen_tokens <= 256 - and (self.compress_ratio == 1 or next_n == 1)): + # heuristic_scratch is only consumed by the C++ + # indexer_topk_decode path; the GVR DSL op does not take it. + # Guard on the metadata flag so this stays consistent with + # the buffer allocation (also gated on the same flag). + if not metadata.use_cute_dsl_topk: + heuristic_scratch = \ + metadata.heuristic_scratch_values[ + :num_gen_tokens] + + if self.use_cute_dsl_topk and self._enable_heuristic_topk: + # GVR DSL: supports all compress_ratio and next_n values. + torch.ops.trtllm.cute_dsl_gvr_topk_decode( + logits_decode, + pre_idx, + gen_kv_lens_cuda, + topk_indices_buffer[num_ctx_tokens:num_ctx_tokens + + num_gen_tokens, :], + self.index_topk, + next_n=next_n, + compress_ratio=self.compress_ratio, + max_seq_len=indexer_max_seq_len, + order_row=metadata.kv_lens_row_reorder, + ) + elif (self.use_cute_dsl_topk + and (self.compress_ratio == 1 or next_n == 1)): torch.ops.trtllm.cute_dsl_indexer_topk_decode( logits_decode, context_lens if self.compress_ratio > 1 else gen_kv_lens_cuda, diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/__init__.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/__init__.py index c248ffd91119..522bf2d5b9ee 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/__init__.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/__init__.py @@ -4,24 +4,37 @@ Layered as: - * :mod:`.kernels` -- OpenAI Triton kernels (per-block max - score, masked softmax for sparse GQA). - * :mod:`.metadata` -- ``MiniMaxM3SparseConfig`` / - ``MiniMaxM3SparseAttentionMetadata`` - dataclasses, CUDA-graph-stable buffer - allocator + builder, and the - :class:`AttentionMetadata` subclass - factory. - * :mod:`.cache_manager` -- standalone side index cache used by tests - and the :class:`KVCacheManagerV2` - subclass factory. - * :mod:`.backend` -- the algorithm itself (vectorized - paged-cache helpers, prefill / decode - entry points, the thin - :class:`MiniMaxM3SparseAttention` - orchestrator) and the - :class:`AttentionBackend` subclass - factory. + * :mod:`.triton_metadata` -- ``MiniMaxM3TritonSparseAttentionMetadata`` + dataclass, CUDA-graph-stable buffer + allocator + builder, and the + :class:`AttentionMetadata` subclass + factory for the Triton reference path. + * :mod:`.cache_manager` -- standalone side index cache used by tests + and the :class:`KVCacheManagerV2` + subclass factory. Shared by both backends. + * :mod:`.common` -- backend-neutral config bundles, the paged + KV-slot writer, block-priority sentinels, and + the paged-cache slot mapping builder shared by + both backends. + * :mod:`.msa_utils` -- MSA-only (fmha_sm100) helpers: import guard, + kernel precondition constants, HND paged-cache + adapters, main-KV writer, page-table builder, + valid-block counting, and top-k selection. + * :mod:`.triton_kernels` -- OpenAI Triton kernels (per-block max + score, masked softmax for sparse GQA). + * :mod:`.triton_backend` -- the Triton reference algorithm (vectorized + paged-cache helpers, prefill / decode + entry points, the thin + :class:`MiniMaxM3TritonSparseAttention` + orchestrator) and its + :class:`AttentionBackend` subclass + factory. + * :mod:`.msa_backend` -- the MSA (fmha_sm100) backend, its flat + metadata, and the backend factory. + * :mod:`.msa_indexer` -- the MSA proxy scoring + top-k block + selection submodule. + * :mod:`.msa_availability`-- SM100 and fmha_sm100 gating for the MSA + path. This package's public surface re-exports the names callers historically imported from ``...sparse.minimax_m3`` so external @@ -29,48 +42,20 @@ working unchanged. """ -# Re-export the algorithm-internal helpers focused unit tests reach -# into so the package preserves the surface the monolithic module -# exposed. These are not part of ``__all__`` (still package-private) -# but stay importable as ``from ...minimax_m3 import _write_main_kv_slots``. -from .backend import ( # noqa: F401 - MiniMaxM3SparseAttention, - _compute_index_attn_chunk_q, - _compute_sparse_gqa_chunk_q, +# The dense Triton oracle in the model imports these paged-cache helpers, so +# they stay importable from the package. They are package-private and are not +# part of __all__. Every other backend/metadata/config symbol is imported +# directly from its defining submodule by the code that needs it. +from .cache_manager import MiniMaxM3KVCacheManagerV2 +from .msa_backend import MiniMaxM3MsaSparseAttention +from .triton_backend import ( # noqa: F401 + MiniMaxM3SparseRuntimeBackend, _gather_paged_batched, - _index_attention_and_select, - _write_main_kv_slots, _write_main_kv_slots_to_pool, - get_minimax_m3_attention_backend_cls, - minimax_m3_sparse_decode, - minimax_m3_sparse_prefill, -) -from .cache_manager import ( - MiniMaxM3KVCacheManagerV2, - MiniMaxM3SparseIndexCache, - get_minimax_m3_kv_cache_manager_cls, -) -from .metadata import ( - MiniMaxM3SparseAttentionMetadata, - MiniMaxM3SparseConfig, - allocate_minimax_m3_static_buffers, - build_runtime_metadata_from_kv_manager, - get_minimax_m3_attention_metadata_cls, - replace_metadata, ) __all__ = [ "MiniMaxM3KVCacheManagerV2", - "MiniMaxM3SparseAttention", - "MiniMaxM3SparseAttentionMetadata", - "MiniMaxM3SparseConfig", - "MiniMaxM3SparseIndexCache", - "allocate_minimax_m3_static_buffers", - "build_runtime_metadata_from_kv_manager", - "get_minimax_m3_attention_backend_cls", - "get_minimax_m3_attention_metadata_cls", - "get_minimax_m3_kv_cache_manager_cls", - "minimax_m3_sparse_decode", - "minimax_m3_sparse_prefill", - "replace_metadata", + "MiniMaxM3MsaSparseAttention", + "MiniMaxM3SparseRuntimeBackend", ] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/cache_manager.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/cache_manager.py index b9badca6ea35..4609414a8731 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/cache_manager.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/cache_manager.py @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """KV cache management for MiniMax-M3 sparse attention. Provides: @@ -13,21 +25,17 @@ from __future__ import annotations -from typing import List, Optional +from typing import List, Optional, Sequence import torch -from tensorrt_llm._utils import ( - TensorWrapper, - binding_to_torch_dtype, - convert_to_torch_tensor, - prefer_pinned, -) +from tensorrt_llm._torch.disaggregation.resource.page import MapperKind +from tensorrt_llm._utils import TensorWrapper, binding_to_torch_dtype, convert_to_torch_tensor from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.internal.batch_manager import CacheType as CacheTypeCpp -from tensorrt_llm.runtime.kv_cache_manager_v2 import BufferConfig, LayerId +from tensorrt_llm.runtime.kv_cache_manager_v2 import BufferConfig, PageIndexMode from tensorrt_llm.runtime.kv_cache_manager_v2._common import BAD_PAGE_INDEX -from tensorrt_llm.runtime.kv_cache_manager_v2._utils import typed_range +from tensorrt_llm.runtime.kv_cache_manager_v2._config import DataRole from ....pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2, Role @@ -168,7 +176,7 @@ def __init__( disable_index_value_layer_ids = list(sparse_layer_ids) # Must be set BEFORE super().__init__ — the base - # ``_build_cache_config`` invokes ``_extra_buffers_per_layer`` + # ``_build_base_config`` invokes ``_extra_buffers_per_layer`` # which reads these attributes. self.sparse_layer_ids = sorted(int(i) for i in sparse_layer_ids) self.disable_index_value_layer_ids = set(int(i) for i in disable_index_value_layer_ids) @@ -176,6 +184,15 @@ def __init__( super().__init__(*args, **kwargs) + index_v_layer_ids = set(self.sparse_layer_ids) - self.disable_index_value_layer_ids + if self.is_disagg and index_v_layer_ids: + raise ValueError( + "MiniMax M3 disaggregated serving requires disable_index_value=True " + "for every sparse layer because the optional test-only index-V cache " + "is not managed or transferred by KVCacheManagerV2; enabled layers=" + f"{sorted(index_v_layer_ids)}" + ) + # Optional plain-tensor index-V cache for non-disabled sparse # layers (test-only; production has disable_index_value=True # on every sparse layer). @@ -198,7 +215,7 @@ def _extra_buffers_per_layer(self, *, tokens_per_block): ``size`` is bytes per **block**: ``1 * sparse_index_dim * elem_bytes * tokens_per_block``. Keyed by **local** layer id — - the base ``_build_cache_config`` iterates local ids, so keying + the base ``_build_base_config`` iterates local ids, so keying by global ids would silently skip registration on non-trivial PP ranks. """ @@ -212,6 +229,13 @@ def _extra_buffers_per_layer(self, *, tokens_per_block): if layer_id in self.layer_offsets } + def get_disagg_role_mapper_kinds(self) -> dict[DataRole, MapperKind]: + """Declare MiniMax M3's token-major K/V and replicated index-K.""" + return { + Role.ALL: MapperKind.NHD, + Role.INDEX_KEY: MapperKind.REPLICATED, + } + def _compute_num_total_slots(self) -> int: """Total token slots across all blocks in the main K pool. @@ -234,18 +258,18 @@ def _torch_dtype_for_index_cache(self) -> torch.dtype: return torch.float32 return torch.bfloat16 - def get_index_k_buffer(self, layer_idx: int) -> Optional[torch.Tensor]: + def get_index_k_buffer(self, layer_idx: int, kv_layout: str = "NHD") -> Optional[torch.Tensor]: """Return the V2-managed paged index-K view for ``layer_idx``. - Shape: ``[num_pages, tokens_per_block, 1, sparse_index_dim]``. - Reads/writes decompose ``slot = (page, within)`` and use - multi-dim fancy indexing; writes propagate to pool storage. + NHD shape is ``[num_pages, tokens_per_block, 1, sparse_index_dim]``; + HND shape is ``[num_pages, 1, tokens_per_block, sparse_index_dim]``. """ return super().get_index_k_buffer( layer_idx, num_heads=1, head_dim=self.sparse_index_dim, dtype=self._torch_dtype_for_index_cache(), + kv_layout=kv_layout, ) def get_index_v_buffer(self, layer_idx: int) -> Optional[torch.Tensor]: @@ -260,12 +284,12 @@ def get_buffers(self, layer_idx: int, kv_layout: str = "NHD") -> Optional[torch. The base :meth:`KVCacheManagerV2.get_buffers` produces a ``[num_pages, kv_factor, ...]`` view with contiguous strides - that assume each slot contains exactly K+V. With INDEX_KEY - registered, sparse layers may have ``scale > 2`` per-slot - buffers (e.g. M3 TP=8 coalesces K, V, INDEX_K into one pool - where ``scale == 3 * num_sparse + 2 * num_dense``), and the - base view's dim-0 stride no longer reaches the next slot's K - for this layer. + that assume the slot holds exactly one layer's K+V. In M3's + pool the slot packs K+V for *all* layers of the group + (``scale >= 2 * num_layers_in_group``), so the base view's + dim-0 stride does not reach the next slot's K for this layer. + (When INDEX_KEY's per-block size coincides with K/V's, it is + coalesced into the same pool and contributes to ``scale`` too.) The override builds a ``[num_slots, scale, ...]`` view rooted at K's base, then slices ``[:, :2]`` to extract K+V. The slice @@ -341,75 +365,30 @@ def get_buffers(self, layer_idx: int, kv_layout: str = "NHD") -> Optional[torch. full_view = convert_to_torch_tensor(TensorWrapper(addr_key, torch_dtype, full_slot_shape)) return full_view[:, :2] - def _build_pool_mapping_tensors(self): - """Compute pool-mapping offsets from layer position in the pool group. - - The base method does ``exact_div(addr_offset, key_bytes * - kv_factor * tokens_per_block)``, which assumes each layer - contributes exactly K+V. When INDEX_KEY coincidentally shares - the same per-block size as K/V (M3 production at TP=8: all - three are 256 B/token), V2 coalesces all three into one pool - and the per-layer stride becomes ``3 * single_buffer_size`` — - the base ``exact_div`` then asserts. - - Compute ``offset`` directly from - ``self.impl.layer_grouping[group_id]`` so the formula stays - correct regardless of how many extra buffers coalesce with - K/V. The M3 forward path uses :meth:`get_buffers` / - :meth:`get_index_k_buffer` rather than this mapping, so the - offset just needs to be consistent (layer position in group). + def _kv_pool_mapping_offset(self, layer_id, layer_group_id, key_base_addr) -> int: + """Pool-mapping offset from the layer's physical position in its pool. + + The base formula ``exact_div(addr_offset, key_bytes * kv_factor * + tokens_per_block)`` assumes each layer contributes exactly K+V to + its pool slot. When index-K coalesces into the K/V pool the layer + stride is non-uniform (sparse layers add an INDEX_KEY sub-page), + so no uniform-stride offset exists. The M3 forward path uses + :meth:`get_buffers` / :meth:`get_index_k_buffer` rather than this + mapping, so the offset just needs to be a consistent per-layer + position. Rank the group's layers by their K base address instead + of by ``layer_grouping`` iteration order: the ordering of + ``layer_grouping`` is not a V2 API contract, while the address + rank always reflects the physical slot layout (and keeps the + NVFP4 ``block_scale_offset == offset`` cross-check in the base + pool-mapping loop meaningful). """ - kv_cache_pool_pointers = torch.tensor( - [ - [ - self.impl.get_mem_pool_base_address( - self.impl.layer_grouping[pool_id][0], Role.KEY - ), - 0, - ] - for pool_id in range(self.num_pools) - ], - dtype=torch.int64, - device="cpu", - pin_memory=prefer_pinned(), + layers_by_addr = sorted( + self.impl.layer_grouping[int(layer_group_id)], + key=lambda lid: self.impl.get_mem_pool_base_address( + lid, Role.KEY, PageIndexMode.SHARED + ), ) - - if self.dtype == DataType.NVFP4: - kv_cache_pool_pointers = torch.stack( - [ - kv_cache_pool_pointers, - torch.tensor( - [ - [ - self.impl.get_mem_pool_base_address( - self.impl.layer_grouping[pool_id][0], Role.KEY_BLOCK_SCALE - ), - 0, - ] - for pool_id in range(self.num_pools) - ], - dtype=torch.int64, - device="cpu", - pin_memory=prefer_pinned(), - ), - ], - dim=-1, - ) - - kv_cache_pool_mapping_list = [] - for layer_id in typed_range(LayerId(self.num_local_layers)): - layer_group_id = self.impl.get_layer_group_id(layer_id) - layers_in_group = list(self.impl.layer_grouping[int(layer_group_id)]) - offset = layers_in_group.index(int(layer_id)) - kv_cache_pool_mapping_list.append([int(layer_group_id), offset]) - - kv_cache_pool_mapping = torch.tensor( - kv_cache_pool_mapping_list, - dtype=torch.int32, - device="cpu", - pin_memory=prefer_pinned(), - ) - return kv_cache_pool_pointers, kv_cache_pool_mapping + return layers_by_addr.index(int(layer_id)) def _get_batch_cache_indices_by_pool_id( self, @@ -417,34 +396,45 @@ def _get_batch_cache_indices_by_pool_id( *, pool_id: int = 0, is_kv_aggregate: bool = True, + num_blocks_per_seq: Optional[Sequence[int]] = None, + index_scale: Optional[int] = None, ): - """Return per-request slot ids in ``[0, num_slots)`` directly. + """Return page indices; padded entries remain ``BAD_PAGE_INDEX`` (-1). The base method converts slot ids to V1-style block ids via ``base_idx * index_scales[pool_id] // kv_factor``, which is - only correct when each layer contributes exactly K+V. With - INDEX_KEY-coalesced sparse pools (M3 production), the scale - breaks the V1 conversion and produces out-of-bounds block ids - during V2 warmup. + only correct when each layer contributes exactly K+V. M3's slot + packs K+V for all layers of the group, so the scale breaks the + V1 conversion and produces out-of-bounds block ids during V2 + warmup. Bypass the conversion: the M3 forward path indexes paged views (built by :meth:`get_buffers` / :meth:`get_index_k_buffer`) directly by slot id. - ``BAD_PAGE_INDEX`` slots stay as 0 to match the legacy - padding contract. + ``BAD_PAGE_INDEX`` slots remain ``-1`` here because disaggregation's + :class:`KVRegionExtractorV1` filters ``region_ids >= 0``. + :meth:`get_block_ids_per_seq` maps them to zero for the attention + metadata's padded tensor. + + Args: + request_ids: Request IDs whose page-index rows are returned. + pool_id: V2 pool whose page indices are requested. + is_kv_aggregate: Kept for compatibility with the base virtual method. + num_blocks_per_seq: Optional per-request truncation limits. When + omitted, preserve the full padded width required by MiniMax + CUDA-graph metadata initialization. + index_scale: Kept for compatibility with the base virtual method; + M3 bypasses the V1 block-id conversion entirely, so any + caller-supplied scale is ignored alongside ``index_scales``. """ res = [] - for req_id in request_ids: - idx_tensor = torch.as_tensor(self.kv_cache_map[req_id].get_base_page_indices(pool_id)) - res.append( - ( - torch.where( - idx_tensor != BAD_PAGE_INDEX, - idx_tensor, - torch.full_like(idx_tensor, BAD_PAGE_INDEX), - ) - ).tolist() - ) + for req_idx, req_id in enumerate(request_ids): + kv_cache = self.kv_cache_map[req_id] + base_page_indices = kv_cache.get_base_page_indices(pool_id) + if num_blocks_per_seq is not None: + num_blocks = min(kv_cache.num_blocks, num_blocks_per_seq[req_idx]) + base_page_indices = base_page_indices[:num_blocks] + res.append(list(base_page_indices)) return res def get_block_ids_per_seq(self, request_ids): diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/common.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/common.py new file mode 100644 index 000000000000..9f2863846df2 --- /dev/null +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/common.py @@ -0,0 +1,252 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared building blocks for the MiniMax-M3 sparse attention backends. + +Both the Triton reference and the MSA (fmha_sm100) path share these +backend-neutral pieces: the lowered parameter and per-rank kernel config +bundles, block-priority sentinels, KV-slot writers, and the paged-cache +slot mapping builder. MSA-only helpers live in :mod:`.msa_utils`. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, List, Literal, Optional, Tuple + +import torch + +from ..params import SparseMetadataParams, SparseParams + +if TYPE_CHECKING: + from tensorrt_llm.mapping import Mapping + +# Sentinel scores that force init and local blocks into the top-k regardless +# of their computed score. Init outranks local. +_INIT_SCORE = 1e30 +_LOCAL_SCORE = 1e29 + + +@dataclass(frozen=True) +class MiniMaxM3SparseParams(SparseParams): + """Lowered runtime parameters for the MiniMax-M3 sparse backend.""" + + algorithm: Literal["minimax_m3"] = field(init=False, default="minimax_m3") + num_index_heads: int = 4 + sparse_index_dim: int = 128 + block_size: int = 128 + topk: int = 16 + init_blocks: int = 0 + local_blocks: int = 1 + score_type: str = "max" + disable_index_value: bool = True + implementation: Literal["triton", "msa"] = "triton" + + @property + def indices_block_size(self) -> int: + """Block granularity of the selected sparse indices. + + Read by the shared TrtllmAttention forward when publishing the + sparse prediction. It equals the per-block scoring size. + """ + return self.block_size + + +@dataclass(frozen=True) +class MiniMaxM3SparseMetadataParams(SparseMetadataParams): + """Metadata-facing MiniMax-M3 sparse geometry.""" + + global_num_q_heads: int = 0 + global_num_kv_heads: int = 0 + num_index_heads: int = 4 + topk: int = 16 + + def sharded_head_counts(self, mapping: Optional["Mapping"] = None) -> Tuple[int, int]: + """Return per-rank (num_q_heads, num_kv_heads) for mapping. + + Matches the model's attention sharding: no split under attention data + parallelism, otherwise split by tp_size. + """ + if mapping is not None and not getattr(mapping, "enable_attention_dp", False): + tp_size = int(getattr(mapping, "tp_size", 1) or 1) + else: + tp_size = 1 + + def _shard(num_heads: int) -> int: + return (int(num_heads) + tp_size - 1) // tp_size + + return _shard(self.global_num_q_heads), _shard(self.global_num_kv_heads) + + +@dataclass(frozen=True) +class MiniMaxM3SparseConfig: + """Per-rank kernel parameter bundle for MiniMax-M3 sparse attention. + + This is **not** a user-facing config (use + :class:`tensorrt_llm.llmapi.llm_args.MiniMaxM3SparseAttentionConfig` + for that). It is the layer-invariant, post-TP-shard parameter bundle + that backend kernels and reference helpers consume. The user knobs + come from :class:`MiniMaxM3SparseParams`; ``num_q_heads`` / + ``num_kv_heads`` / ``head_dim`` come from the per-rank model + geometry and must be supplied by the caller (typically via + :meth:`from_sparse_params`). + """ + + num_q_heads: int + num_kv_heads: int + head_dim: int + num_index_heads: int + sparse_index_dim: int + block_size: int + topk: int + init_blocks: int = 0 + local_blocks: int = 1 + score_type: str = "max" + + def __post_init__(self) -> None: + if self.num_q_heads % self.num_kv_heads != 0: + raise ValueError( + f"num_q_heads ({self.num_q_heads}) must be divisible by " + f"num_kv_heads ({self.num_kv_heads})" + ) + if self.num_index_heads % self.num_kv_heads != 0: + raise ValueError( + f"num_index_heads ({self.num_index_heads}) must be divisible " + f"by num_kv_heads ({self.num_kv_heads})" + ) + if self.block_size <= 0: + raise ValueError(f"block_size must be > 0, got {self.block_size}") + if self.topk <= 0: + raise ValueError(f"topk must be > 0, got {self.topk}") + if self.init_blocks < 0: + raise ValueError(f"init_blocks must be >= 0, got {self.init_blocks}") + if self.local_blocks < 0: + raise ValueError(f"local_blocks must be >= 0, got {self.local_blocks}") + if self.score_type != "max": + # SGLang exposes only "max" today and that is what the MiniMax-M3 + # checkpoint config specifies. Reject anything else explicitly so + # a config drift surfaces immediately. + raise ValueError( + f"score_type={self.score_type!r} is not supported " + "(only 'max' matches the SGLang reference)" + ) + + @classmethod + def from_sparse_params( + cls, + sparse_params: "MiniMaxM3SparseParams", + *, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + ) -> "MiniMaxM3SparseConfig": + """Build a kernel param bundle from lowered ``MiniMaxM3SparseParams`` + and the per-rank model geometry. + """ + return cls( + num_q_heads=int(num_q_heads), + num_kv_heads=int(num_kv_heads), + head_dim=int(head_dim), + num_index_heads=int(sparse_params.num_index_heads), + sparse_index_dim=int(sparse_params.sparse_index_dim), + block_size=int(sparse_params.block_size), + topk=int(sparse_params.topk), + init_blocks=int(sparse_params.init_blocks), + local_blocks=int(sparse_params.local_blocks), + score_type=str(sparse_params.score_type), + ) + + +def write_kv_slots( + cache: torch.Tensor, + out_cache_loc: torch.Tensor, + values: torch.Tensor, + *, + layout: Literal["NHD", "HND"] = "NHD", +) -> None: + """Write per-token values into a K, V, or index-K cache at given slots. + + Handles a 3-D flat-slot cache and a 4-D paged view. `layout` sets the paged + axis order: "NHD" is [num_pages, tokens_per_block, num_heads, channel], + "HND" is [num_pages, num_heads, tokens_per_block, channel]. The paged view + is non-contiguous, so the slot id is split into (page, within) and written + by multi-dim assignment. `values` is always [num_tokens, num_heads, channel]. + """ + with torch.no_grad(): + if cache.ndim >= 4: + token_axis = 2 if layout == "HND" else 1 + tokens_per_block = int(cache.shape[token_axis]) + out_long = out_cache_loc.to(torch.long) + page = out_long // tokens_per_block + within = out_long % tokens_per_block + if layout == "HND": + # Advanced indices on dims 0 and 2 broadcast to [num_tokens] and + # move front, giving a [num_tokens, num_heads, channel] target. + cache[page, :, within, :] = values.to(cache.dtype) + else: + cache[page, within] = values.to(cache.dtype) + else: + cache.index_copy_(0, out_cache_loc.to(torch.long), values.to(cache.dtype)) + + +def build_paged_kv_slot_mapping( + *, + kv_cache_manager, + request_ids, + qo_lens_cpu: torch.Tensor, + qo_offset_cpu: torch.Tensor, + device: torch.device, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Build the backend-neutral paged-cache slot mapping. + + Returns (req_to_token, slot_ids, out_cache_loc), derived only from the paged + KV cache manager and the per-request query geometry, with no dependency on + any backend-specific metadata. + + req_to_token is the [batch, max_kv_len] int32 map from (request, position) + to a global slot id, expanded from get_block_ids_per_seq with + tokens_per_block as block_id * tokens_per_block + offset_within_block. + slot_ids is the [batch] identity row index into req_to_token. out_cache_loc + lists the per-new-token slot ids in flattened query order: request b + contributes positions qo_offset[b] through qo_offset[b] + qo_lens[b] - 1. + That one formula covers prefill (qo_offset is the prefix length) and decode + (qo_offset is kv_len - 1 with qo_len 1). + + The req_to_token reads that build out_cache_loc sync the host, so call this + only from prepare(), never from the forward path. + """ + tokens_per_block = int(kv_cache_manager.tokens_per_block) + # block_ids_per_seq is a [batch, max_blocks_per_seq] tensor; row b holds the + # block ids assigned to request_ids[b] in order. + block_ids = kv_cache_manager.get_block_ids_per_seq(list(request_ids)) + batch = int(qo_lens_cpu.shape[0]) + max_blocks = int(block_ids.shape[1]) + max_kv_len = max_blocks * tokens_per_block + + # Expand block ids -> per-token slot ids. + block_ids_dev = block_ids.to(device).to(torch.int64) + within_block = torch.arange(tokens_per_block, device=device, dtype=torch.int64) + # Outer product per batch entry: [batch, max_blocks, tokens_per_block] + slot_grid = block_ids_dev.unsqueeze(-1) * tokens_per_block + within_block + req_to_token = slot_grid.reshape(batch, max_kv_len).to(torch.int32) + slot_ids = torch.arange(batch, device=device, dtype=torch.int32) + + # out_cache_loc: per-new-token slot ids, in flattened query-token order. + req_to_token_cpu = req_to_token.to("cpu") + qo_lens_list = qo_lens_cpu.to(torch.long).tolist() + qo_offset_list = qo_offset_cpu.to(torch.long).tolist() + out_cache_loc_list: List[int] = [] + for b in range(batch): + start = int(qo_offset_list[b]) + for offset in range(int(qo_lens_list[b])): + out_cache_loc_list.append(int(req_to_token_cpu[b, start + offset].item())) + out_cache_loc = torch.tensor(out_cache_loc_list, dtype=torch.int32, device=device) + return req_to_token, slot_ids, out_cache_loc + + +__all__ = [ + "MiniMaxM3SparseConfig", + "MiniMaxM3SparseMetadataParams", + "MiniMaxM3SparseParams", + "build_paged_kv_slot_mapping", + "write_kv_slots", +] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_availability.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_availability.py new file mode 100644 index 000000000000..c4751cdec2e4 --- /dev/null +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_availability.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Availability checks for the MiniMax-M3 MSA sparse attention kernels. + +The MSA kernels are provided by the fmha_sm100 package from the MSA git +submodule at 3rdparty/MSA and run only on the SM100 architecture family +(SM100 and SM103). These helpers gate backend selection so a request for the +MSA path fails early with a clear message on unsupported systems. +""" + +from __future__ import annotations + +from tensorrt_llm._utils import get_sm_version, is_sm_100f + +from .msa_utils import msa_package_available + +# fmha_sm100 runs on the SM100 architecture family (SM100 and SM103). Other +# architectures, including SM120, are not supported. +MSA_PACKAGE = "fmha_sm100" + + +def ensure_msa_available() -> None: + """Raise RuntimeError if the MSA sparse attention path cannot run here.""" + if not msa_package_available(): + raise RuntimeError( + f"MiniMax-M3 MSA sparse attention requires the {MSA_PACKAGE} kernels " + "from the MSA git submodule at 3rdparty/MSA. Initialize it with " + "'git submodule update --init --recursive'." + ) + if not is_sm_100f(): + sm_version = get_sm_version() + raise RuntimeError( + "MiniMax-M3 MSA sparse attention requires an SM100 or SM103 device, " + f"but the current device reports SM version {sm_version}." + ) + + +__all__ = [ + "MSA_PACKAGE", + "ensure_msa_available", +] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_backend.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_backend.py new file mode 100644 index 000000000000..3b0bbb0d2b77 --- /dev/null +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_backend.py @@ -0,0 +1,790 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""MSA-backed MiniMax-M3 sparse attention on the TrtllmAttention stack. + + * MiniMaxM3MsaSparseAttention subclasses TrtllmAttention and reuses its + inherited forward, overriding only the sparse hooks and owning an + MsaIndexer. + * The main sparse GQA runs through the registered MsaSparseGqaFmha. + * The indexer calls fmha_sm100 directly to produce the per-query selected + block indices, which the model layer threads through + forward_args.topk_indices. + * MiniMaxM3MsaSparseAttentionMetadata subclasses TrtllmAttentionMetadata and + stores its per-forward MSA tensors in CUDA-graph-stable buffers. + The buffers are allocated once in __post_init__ via + get_empty(capture_graph=...), and prepare() copies the per-step values + into them. The standard CUDAGraphRunner clones one metadata per graph + batch size (create_cuda_graph_metadata), so no per-batch-size cache is + needed here. + +The classes subclass TrtllmAttention and TrtllmAttentionMetadata, imported at +module scope. This is cycle-free because the fmha registry defers its +MsaSparseGqaFmha import (see fmha/registry.py), so trtllm's import chain does +not reach this module. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional, Tuple + +import torch + +from tensorrt_llm._torch.attention_backend.interface import AttentionForwardArgs +from tensorrt_llm._torch.attention_backend.trtllm import TrtllmAttention, TrtllmAttentionMetadata + +from .common import ( + MiniMaxM3SparseConfig, + MiniMaxM3SparseMetadataParams, + build_paged_kv_slot_mapping, + write_kv_slots, +) +from .msa_indexer import MsaIndexer +from .msa_utils import ( + MSA_REQUIRED_HEAD_DIM, + MSA_REQUIRED_TOPK, + build_kv_page_indices, + per_token_valid_blocks, + require_msa_module, +) + + +def _cache_device(meta) -> torch.device: + """Device hosting the paged KV buffers, else the current CUDA device.""" + kv_cache_manager = meta.kv_cache_manager + if kv_cache_manager is not None: + try: + return kv_cache_manager.get_buffers(0).device + except Exception: + pass + return torch.device(f"cuda:{torch.cuda.current_device()}") + + +def _worst_case_proxy_max_k_tiles( + fmha_sm100, + *, + num_index_heads: int, + kv_cache_manager, + max_batch: int, +) -> int: + """Return max_k_tiles for a proxy plan at the manager's max KV length.""" + page_size = int(kv_cache_manager.tokens_per_block) + max_kv_len = int(kv_cache_manager.max_blocks_per_seq) * page_size + qo_lens = torch.ones(max_batch, dtype=torch.int32) + kv_lens = torch.full((max_batch,), max_kv_len, dtype=torch.int32) + qo_offset = kv_lens - qo_lens + proxy_plan = fmha_sm100.fmha_sm100_plan( + qo_lens, + kv_lens, + num_index_heads, + num_kv_heads=1, + qo_offset=qo_offset, + page_size=page_size, + output_maxscore=True, + num_kv_splits=1, + causal=True, + ) + return int(proxy_plan[3]["max_k_tiles"]) + + +# Per-step fmha_sm100 plan tensors that must live in CUDA-graph-stable buffers. +# At num_kv_splits=1 the plan carries no split-KV workspaces, and +# cute_workspace_buffer is the vendor's cached scratch (kept by reference, not +# copied). +_MSA_PLAN_STABLE_KEYS = ( + "packed_work_range", + "packed_work_info", + "qo_segment_offsets", + "kv_segment_offsets", + "kv_page_indptr", + "qo_segment_lens", + "kv_segment_lens", + "qo_offset", +) +_MSA_PLAN_INT64_KEYS = ("packed_work_range", "packed_work_info") +# fmha_sm100 sizes packed_work_info at 131072 * max(num_kv_splits, 1); forcing +# num_kv_splits=1 pins this worklist width. +_MSA_PACKED_WORK_INFO_LEN = 131072 +_MSA_SPLIT_KV_KEYS = ( + "kv_tile_begin_indices", + "kv_tile_end_indices", + "kv_split_indices", + "num_kv_splits_per_row", + "workspace_o", + "workspace_lse", +) + + +class _MsaGraphSafePlan: + """CUDA-graph-stable mirror of one fmha_sm100 decode plan. + + Owns fixed device buffers for the per-step plan worklists. refresh() copies + a freshly built plan into them and returns a plan tuple pointing at the + stable buffers, so the captured fmha_sm100 run reads addresses that do not + change across replays. Mirrors FlashInfer's fixed indptr/indices buffers. + + Only valid at num_kv_splits=1: the plan then has no split-KV workspaces + (refresh() asserts this), and cute_workspace_buffer and the scalar fields + pass through unchanged. + """ + + def __init__(self, metadata, name: str, *, max_batch: int, num_ctas: int, capture_graph: bool): + buffers = metadata.cuda_graph_buffers + self._buf = {} + # Set by refresh(), read through the plan property. + self._plan: Optional[tuple] = None + # cute_workspace_buffer must keep a fixed address across steps for the + # captured graph to replay correctly. Pin it on first use and fail if + # it moves. + self._ws_ptr: Optional[int] = None + for key in _MSA_PLAN_STABLE_KEYS: + if key == "packed_work_range": + shape = (num_ctas,) + elif key == "packed_work_info": + shape = (_MSA_PACKED_WORK_INFO_LEN,) + elif key in ("qo_segment_offsets", "kv_segment_offsets", "kv_page_indptr"): + shape = (max_batch + 1,) + else: + shape = (max_batch,) + dtype = torch.int64 if key in _MSA_PLAN_INT64_KEYS else torch.int32 + self._buf[key] = metadata.get_empty( + buffers, + shape, + cache_name=f"{name}_{key}", + dtype=dtype, + capture_graph=capture_graph, + ) + + @property + def plan(self) -> Optional[tuple]: + """The current graph-safe plan tuple, or None if no decode plan is live.""" + return self._plan + + def reset(self) -> None: + """Drop the live plan tuple (e.g. for a prefill/mixed or captured step).""" + self._plan = None + + def refresh(self, plan_tuple) -> tuple: + has_mixed, split, batch, decode, prefill = plan_tuple + if has_mixed: + raise RuntimeError( + "MSA decode expects a single (non-mixed) fmha_sm100 plan; a decode " + "batch must be pure decode." + ) + for key in _MSA_SPLIT_KV_KEYS: + if decode.get(key) is not None: + raise RuntimeError( + f"MSA decode plan used split-KV workspace {key!r}; num_kv_splits=1 " + "is required for graph-safe decode." + ) + ws = decode.get("cute_workspace_buffer") + if ws is not None: + if self._ws_ptr is None: + self._ws_ptr = ws.data_ptr() + elif ws.data_ptr() != self._ws_ptr: + raise RuntimeError( + "cute_workspace_buffer moved across steps; the fmha_sm100 plan " + "is not CUDA-graph safe." + ) + rebuilt = dict(decode) + for key in _MSA_PLAN_STABLE_KEYS: + src = decode.get(key) + if src is None: + continue + n = int(src.shape[0]) + dst = self._buf[key] + if n > dst.shape[0]: + raise ValueError( + f"MSA plan buffer {key} ({dst.shape[0]}) is smaller than the plan tensor ({n})." + ) + dst[:n].copy_(src, non_blocking=True) + rebuilt[key] = dst[:n] + self._plan = (has_mixed, split, batch, rebuilt, prefill) + return self._plan + + +@dataclass(init=False) +class MiniMaxM3MsaSparseAttentionMetadata(TrtllmAttentionMetadata): + """TrtllmAttentionMetadata for MiniMax-M3 MSA sparse layers. + + Tensors read inside the captured forward are CUDA-graph-stable: the + cache slots (msa_out_cache_loc), page table (msa_kv_indices), and proxy + scratch (msa_max_score, msa_n_valid_blocks) are allocated once from the + manager's worst-case geometry. msa_out_cache_loc, msa_kv_indices, and + msa_n_valid_blocks are refreshed via copy_, while the fmha_sm100 proxy pass + writes msa_max_score directly (see msa_proxy_max_score_view). Decode-plan + worklists live on _MsaGraphSafePlan owners, surfaced via msa_decode_*_plan. + + Length inputs to fmha_sm100_plan (msa_qo_lens_cpu, msa_kv_lens_cpu, + msa_qo_offset_cpu) are host properties of the base seq_lens/kv_lens, + read only while building plans in prepare() (outside capture), so they + need no graph-stable storage. Plans are built in _build_decode_plans and + are absent for prefill/mixed batches, which run eagerly. + """ + + # Graph-stable buffers; consumers slice to the live count at the call + # site. Filled once the current step's cache write is prepared. + msa_out_cache_loc: Optional[torch.Tensor] = None + msa_kv_indices: Optional[torch.Tensor] = None + msa_max_score: Optional[torch.Tensor] = None + msa_n_valid_blocks: Optional[torch.Tensor] = None + + # _msa_buffers_ready gates the once-only device buffers; + # _msa_fields_ready marks that the current step's buffers are populated. + _msa_buffers_ready: bool = False + _msa_fields_ready: bool = False + # Sparse geometry the decode plans need. + _msa_params: Optional[MiniMaxM3SparseMetadataParams] = None + # Plan owners, created lazily when the decode plans are first built and + # reused across steps. Each owns its graph-safe plan buffers and the + # current refreshed plan tuple. + _msa_proxy_plan: Optional["_MsaGraphSafePlan"] = None + _msa_gqa_plan: Optional["_MsaGraphSafePlan"] = None + _msa_dense_plan: Optional["_MsaGraphSafePlan"] = None + + def __post_init__(self) -> None: + super().__post_init__() + params = self.sparse_metadata_params + self._msa_params = params if isinstance(params, MiniMaxM3SparseMetadataParams) else None + self._create_msa_buffers() + + @property + def msa_qo_lens_cpu(self) -> Optional[torch.Tensor]: + """Per-request query length (host int32), from the base seq_lens.""" + seq_lens = self.seq_lens + if seq_lens is None: + return None + out = seq_lens[: self.num_seqs] + return out if out.dtype == torch.int32 else out.to(torch.int32) + + @property + def msa_kv_lens_cpu(self) -> Optional[torch.Tensor]: + """Per-request KV length, cached plus new tokens (host int32).""" + kv_lens = getattr(self, "kv_lens", None) + if self.seq_lens is None or kv_lens is None: + return None + out = kv_lens[: self.num_seqs] + return out if out.dtype == torch.int32 else out.to(torch.int32) + + @property + def msa_qo_offset_cpu(self) -> Optional[torch.Tensor]: + """Per-request causal offset (kv_len - qo_len), the cached prefix length.""" + qo = self.msa_qo_lens_cpu + kv = self.msa_kv_lens_cpu + if qo is None or kv is None: + return None + return kv - qo + + @property + def msa_decode_proxy_plan(self) -> Optional[tuple]: + """Proxy (max-score) plan tuple, or None outside decode.""" + plan = self._msa_proxy_plan + return plan.plan if plan is not None else None + + @property + def msa_decode_gqa_plan(self) -> Optional[tuple]: + """Sparse GQA plan tuple, or None outside decode.""" + plan = self._msa_gqa_plan + return plan.plan if plan is not None else None + + @property + def msa_decode_dense_plan(self) -> Optional[tuple]: + """Dense GQA plan tuple, shared by dense layers 0 to 2.""" + plan = self._msa_dense_plan + return plan.plan if plan is not None else None + + def _create_msa_buffers(self) -> None: + """Allocate the CUDA-graph-stable MSA device buffers. + + Buffers come from the shared graph buffer pool so they are reserved + under capture. Sizing follows the worst-case graph geometry: + max_num_tokens for cache slots, max_num_sequences * max_blocks_per_seq + for the page table, and worst-case max_k_tiles for proxy scratch. + """ + kv_cache_manager = self.kv_cache_manager + self._msa_buffers_ready = False + if kv_cache_manager is None or not hasattr(kv_cache_manager, "get_index_k_buffer"): + return + capture_graph = self.is_cuda_graph + buffers = self.cuda_graph_buffers + max_num_sequences = int(self.max_num_sequences) + max_blocks_per_seq = int(kv_cache_manager.max_blocks_per_seq) + max_total_pages = max_num_sequences * max_blocks_per_seq + max_num_tokens = int(self.max_num_tokens) + + self.msa_out_cache_loc = self.get_empty( + buffers, + (max_num_tokens,), + cache_name="msa_out_cache_loc", + dtype=torch.int32, + capture_graph=capture_graph, + ) + self.msa_kv_indices = self.get_empty( + buffers, + (max_total_pages,), + cache_name="msa_kv_indices", + dtype=torch.int32, + capture_graph=capture_graph, + ) + # The proxy scratch needs the fmha_sm100 plan geometry. This metadata + # exists only for the MSA backend, whose selection already required the + # kernels, so a failed import here is a hard error rather than a reason + # to skip allocation. + params = self._msa_params + if params is not None: + fmha_sm100 = require_msa_module() + max_k_tiles = _worst_case_proxy_max_k_tiles( + fmha_sm100, + num_index_heads=params.num_index_heads, + kv_cache_manager=kv_cache_manager, + max_batch=max_num_sequences, + ) + self._alloc_msa_proxy_scratch( + num_index_heads=params.num_index_heads, + max_batch=max_num_sequences, + max_k_tiles=max_k_tiles, + capture_graph=capture_graph, + ) + self._msa_buffers_ready = True + + def _alloc_msa_proxy_scratch( + self, + *, + num_index_heads: int, + max_batch: int, + max_k_tiles: int, + capture_graph: bool, + ) -> None: + """Allocate the flat proxy max-score store and the valid-block scratch. + + The store is sized for the worst-case max_k_tiles so one allocation + serves every decode step. msa_proxy_max_score_view slices the per-step + shape out of it. + """ + buffers = self.cuda_graph_buffers + self.msa_max_score = self.get_empty( + buffers, + (num_index_heads * max_k_tiles * max_batch,), + cache_name="msa_max_score", + dtype=torch.float32, + capture_graph=capture_graph, + ) + self.msa_n_valid_blocks = self.get_empty( + buffers, + (max_batch,), + cache_name="msa_n_valid_blocks", + dtype=torch.int32, + capture_graph=capture_graph, + ) + + def _ensure_msa_decode_scratch_buffers( + self, + *, + num_index_heads: int, + max_batch: int, + capture_graph: bool, + required_max_k_tiles: int, + ) -> None: + """Ensure proxy scratch buffers exist and cover the current plan.""" + required_numel = num_index_heads * required_max_k_tiles * max_batch + if self.msa_max_score is not None: + if self.msa_max_score.numel() < required_numel: + raise ValueError( + f"msa_max_score backing store ({self.msa_max_score.numel()} " + f"elements) is smaller than the decode plan needs " + f"({required_numel} = {num_index_heads} heads * " + f"{required_max_k_tiles} k-tiles * {max_batch} batch)." + ) + return + + kv_cache_manager = self.kv_cache_manager + if kv_cache_manager is None: + return + + fmha_sm100 = require_msa_module() + max_k_tiles = _worst_case_proxy_max_k_tiles( + fmha_sm100, + num_index_heads=num_index_heads, + kv_cache_manager=kv_cache_manager, + max_batch=max_batch, + ) + if max_k_tiles < required_max_k_tiles: + raise ValueError( + f"Worst-case max_k_tiles ({max_k_tiles}) is less than the " + f"decode plan ({required_max_k_tiles})." + ) + self._alloc_msa_proxy_scratch( + num_index_heads=num_index_heads, + max_batch=max_batch, + max_k_tiles=max_k_tiles, + capture_graph=capture_graph, + ) + + def prepare(self) -> None: + super().prepare() + self._build_msa_fields() + self._build_decode_plans() + + def _build_decode_plans(self) -> None: + """Build the graph-safe decode plans and buffers for this step. + + Runs in prepare(), outside CUDA graph capture. The plans are + layer-invariant for MiniMax-M3, so they are built once per step from + the shared sparse geometry, mirrored into CUDA-graph-stable buffers, + and reused by every layer. Mirrors FlashInfer's plan() split. + Prefill/mixed batches leave the plans cleared and run eagerly. + """ + # Drop any plan tuples from the previous step; the msa_decode_*_plan + # properties then report None until they are rebuilt below. + for plan in (self._msa_proxy_plan, self._msa_gqa_plan, self._msa_dense_plan): + if plan is not None: + plan.reset() + if not self._msa_fields_ready: + return + # A decode batch is pure generation (no context requests). + if int(self.num_contexts or 0) > 0: + return + # Geometry is captured in __post_init__; skip when it is unavailable. + params = self._msa_params + if params is None: + return + num_index_heads = params.num_index_heads + num_q_heads, num_kv_heads = params.sharded_head_counts(self.mapping) + topk = params.topk + + fmha_sm100 = require_msa_module() + qo_lens_cpu = self.msa_qo_lens_cpu + kv_lens_cpu = self.msa_kv_lens_cpu + qo_offset_cpu = self.msa_qo_offset_cpu + if qo_lens_cpu is None or kv_lens_cpu is None or qo_offset_cpu is None: + return + batch = int(qo_lens_cpu.shape[0]) + device = _cache_device(self) + page_size = int(self.kv_cache_manager.tokens_per_block) + capture_graph = self.is_cuda_graph + max_batch = int(self.max_num_sequences) + + # Proxy plan: MQA (num_kv_heads=1) max-score pass over the index + # branch; output_maxscore feeds the indexer's top-k block selection. + proxy_plan = fmha_sm100.fmha_sm100_plan( + qo_lens_cpu, + kv_lens_cpu, + num_index_heads, + num_kv_heads=1, + qo_offset=qo_offset_cpu, + page_size=page_size, + output_maxscore=True, + num_kv_splits=1, + causal=True, + ) + # Sparse-layer plan: kv_block_num=topk limits attention to top-k blocks. + gqa_plan = fmha_sm100.fmha_sm100_plan( + qo_lens_cpu, + kv_lens_cpu, + num_q_heads, + num_kv_heads=num_kv_heads, + qo_offset=qo_offset_cpu, + page_size=page_size, + kv_block_num=topk, + num_kv_splits=1, + causal=True, + ) + # Dense-layer plan: no kv_block_num, so it attends the full page table. + dense_plan = fmha_sm100.fmha_sm100_plan( + qo_lens_cpu, + kv_lens_cpu, + num_q_heads, + num_kv_heads=num_kv_heads, + qo_offset=qo_offset_cpu, + page_size=page_size, + num_kv_splits=1, + causal=True, + ) + + required_max_k_tiles = int(proxy_plan[3]["max_k_tiles"]) + self._ensure_msa_decode_scratch_buffers( + num_index_heads=num_index_heads, + max_batch=max_batch, + capture_graph=capture_graph, + required_max_k_tiles=required_max_k_tiles, + ) + + # Allocate the graph-safe plan owners once per metadata; later steps + # only refresh their contents below. + if self._msa_proxy_plan is None: + num_ctas = torch.cuda.get_device_properties(device).multi_processor_count + self._msa_proxy_plan = _MsaGraphSafePlan( + self, + "msa_proxy_plan", + max_batch=max_batch, + num_ctas=num_ctas, + capture_graph=capture_graph, + ) + self._msa_gqa_plan = _MsaGraphSafePlan( + self, + "msa_gqa_plan", + max_batch=max_batch, + num_ctas=num_ctas, + capture_graph=capture_graph, + ) + self._msa_dense_plan = _MsaGraphSafePlan( + self, + "msa_dense_plan", + max_batch=max_batch, + num_ctas=num_ctas, + capture_graph=capture_graph, + ) + + # refresh() stores each plan tuple on its owner, surfaced by the + # msa_decode_*_plan properties. + self._msa_proxy_plan.refresh(proxy_plan) + self._msa_gqa_plan.refresh(gqa_plan) + self._msa_dense_plan.refresh(dense_plan) + + n_valid = per_token_valid_blocks( + qo_lens_cpu, kv_lens_cpu, qo_offset_cpu, causal=True, block_size=page_size + ) + self.msa_n_valid_blocks[:batch].copy_(n_valid.to(torch.int32), non_blocking=True) + + def _build_msa_fields(self) -> None: + """Populate the MSA cache-write buffers for this step. + + The page table and per-new-token cache slots are derived via the + build_paged_kv_slot_mapping helper, then copied into the persistent + buffers. The transient builder tensors are discarded. + """ + self._msa_fields_ready = False + if not self._msa_buffers_ready: + return + request_ids = self.request_ids + qo_lens_cpu = self.msa_qo_lens_cpu + kv_lens_cpu = self.msa_kv_lens_cpu + qo_offset_cpu = self.msa_qo_offset_cpu + if request_ids is None or qo_lens_cpu is None: + return + batch_size = int(qo_lens_cpu.shape[0]) + if batch_size == 0: + return + + kv_cache_manager = self.kv_cache_manager + cache_device = _cache_device(self) + page_size = int(kv_cache_manager.tokens_per_block) + + is_prefill = int(self.num_contexts or 0) > 0 + if not is_prefill and int(qo_lens_cpu.max().item()) > 1: + raise NotImplementedError( + "MiniMax-M3 MSA attention does not support speculative decoding " + "(multiple query tokens per decode step). Disable speculative " + "decoding or use the non-MSA MiniMax-M3 backend." + ) + + # Built in prepare() (outside capture), so these transients are + # fine: forwards read only the persistent buffers filled below. + # qo_offset is the prefix length, so one build covers prefill + # (num_cached) and decode (kv_len - 1 with qo_len 1). + req_to_token, slot_ids, out_cache_loc = build_paged_kv_slot_mapping( + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + qo_lens_cpu=qo_lens_cpu, + qo_offset_cpu=qo_offset_cpu, + device=cache_device, + ) + kv_indices = build_kv_page_indices(req_to_token, slot_ids, kv_lens_cpu, page_size) + + total_new_tokens = int(out_cache_loc.shape[0]) + total_pages = int(kv_indices.shape[0]) + if total_new_tokens > self.msa_out_cache_loc.shape[0]: + raise ValueError( + f"MSA out_cache_loc buffer ({self.msa_out_cache_loc.shape[0]}) is " + f"smaller than the step's new-token count ({total_new_tokens})." + ) + if total_pages > self.msa_kv_indices.shape[0]: + raise ValueError( + f"MSA kv_indices buffer ({self.msa_kv_indices.shape[0]}) is " + f"smaller than the step's page count ({total_pages})." + ) + + self.msa_out_cache_loc[:total_new_tokens].copy_(out_cache_loc, non_blocking=True) + self.msa_kv_indices[:total_pages].copy_(kv_indices, non_blocking=True) + self._msa_fields_ready = True + + def msa_idx_k_cache(self, layer_idx: int) -> torch.Tensor: + """Return the paged index-K cache in the HND layout MSA consumes.""" + return self.kv_cache_manager.get_index_k_buffer(layer_idx, kv_layout="HND") + + def msa_write_idx_k(self, layer_idx: int, idx_k: torch.Tensor) -> None: + """Write the new-token index-K into the side cache at out_cache_loc.""" + cache = self.msa_idx_k_cache(layer_idx) + sparse_index_dim = int(cache.shape[-1]) + num_tokens = int(idx_k.shape[0]) + write_kv_slots( + cache, + self.msa_out_cache_loc[:num_tokens], + idx_k.reshape(num_tokens, 1, sparse_index_dim), + layout="HND", + ) + + def msa_proxy_max_score_view( + self, num_index_heads: int, plan_max_k_tiles: int, num_tokens: int + ) -> torch.Tensor: + """Return a contiguous [num_index_heads, plan_max_k_tiles, num_tokens] view. + + fmha_sm100 ignores the passed tensor's strides and writes a contiguous + [num_index_heads, plan_max_k_tiles, total_q] block sized by the current + decode plan, so it must receive a tensor contiguous in exactly that + shape. The view is taken from the flat store's prefix starting at offset + 0, so its data_ptr is stable for CUDA graph replay. Capture builds the + decode plan at the worst-case max_k_tiles, so replays only shrink it. + """ + store = self.msa_max_score + numel = num_index_heads * plan_max_k_tiles * num_tokens + if numel > store.numel(): + raise ValueError( + f"msa_max_score backing store ({store.numel()} elements) is " + f"smaller than the proxy view needs ({numel} = {num_index_heads} " + f"heads * {plan_max_k_tiles} k-tiles * {num_tokens} tokens)." + ) + return store[:numel].view(num_index_heads, plan_max_k_tiles, num_tokens) + + +class MiniMaxM3MsaSparseAttention(TrtllmAttention): + """MSA-backed MiniMax-M3 sparse attention.""" + + Metadata = MiniMaxM3MsaSparseAttentionMetadata + + def __init__( + self, + layer_idx: int, + num_heads: int, + head_dim: int, + num_kv_heads: Optional[int] = None, + quant_config=None, + *, + sparse_params, + **kwargs, + ): + TrtllmAttention.__init__( + self, + layer_idx, + num_heads, + head_dim, + num_kv_heads=num_kv_heads, + quant_config=quant_config, + sparse_params=sparse_params, + **kwargs, + ) + self.m3_config = MiniMaxM3SparseConfig.from_sparse_params( + sparse_params, + num_q_heads=num_heads, + num_kv_heads=num_kv_heads or num_heads, + head_dim=head_dim, + ) + self.disable_index_value = bool(sparse_params.disable_index_value) + self._validate_msa_preconditions() + self.indexer = MsaIndexer(self.m3_config) + + def _validate_msa_preconditions(self) -> None: + config = self.m3_config + if not self.disable_index_value: + raise NotImplementedError( + "MSA backend requires disable_index_value=True; the proxy pass " + "consumes only the max score and has no index-V path." + ) + if config.head_dim != MSA_REQUIRED_HEAD_DIM: + raise NotImplementedError( + f"MSA backend requires head_dim={MSA_REQUIRED_HEAD_DIM}, got {config.head_dim}." + ) + if config.sparse_index_dim != MSA_REQUIRED_HEAD_DIM: + raise NotImplementedError( + f"MSA backend requires sparse_index_dim={MSA_REQUIRED_HEAD_DIM}, " + f"got {config.sparse_index_dim}." + ) + if config.topk != MSA_REQUIRED_TOPK: + raise NotImplementedError( + f"MSA backend requires topk={MSA_REQUIRED_TOPK}, got {config.topk}." + ) + + @classmethod + def support_fused_rope(cls) -> bool: + # The MiniMax-M3 model layer applies partial RoPE to the main and + # index branches explicitly. + return False + + def run_indexer( + self, + idx_q: torch.Tensor, + idx_k: torch.Tensor, + metadata, + *, + idx_sm_scale: Optional[float] = None, + ) -> torch.Tensor: + """Write the index-K cache and return the selected block indices. + + The model layer runs this before forward and threads the result through + forward_args.topk_indices. Returns [total_q, num_kv_heads, topk]. + Decode uses the prebuilt graph-safe proxy plan; prefill plans + eagerly. + """ + config = self.m3_config + idx_sm_scale = idx_sm_scale if idx_sm_scale is not None else config.sparse_index_dim**-0.5 + num_tokens = int(idx_q.shape[0]) + idx_q_view = idx_q.view(num_tokens, config.num_index_heads, config.sparse_index_dim) + idx_k_view = idx_k.view(num_tokens, 1, config.sparse_index_dim) + + metadata.msa_write_idx_k(self.layer_idx, idx_k_view) + idx_k_cache = metadata.msa_idx_k_cache(self.layer_idx) + + # One selection path: decode passes the prebuilt graph-safe proxy + # plan plus the proxy scratch shaped to the live query count; prefill + # leaves them None and the proxy plan is built inline. + proxy_plan = metadata.msa_decode_proxy_plan + if proxy_plan is not None: + # proxy_plan is (has_mixed, split, batch, decode_dict, prefill); + # decode_dict carries max_k_tiles for the contiguous score view. + plan_max_k_tiles = int(proxy_plan[3]["max_k_tiles"]) + max_score = metadata.msa_proxy_max_score_view( + config.num_index_heads, plan_max_k_tiles, num_tokens + ) + n_valid_blocks = metadata.msa_n_valid_blocks[:num_tokens] + else: + max_score = None + n_valid_blocks = None + return self.indexer.select_blocks( + idx_q_view, + idx_k_cache, + idx_sm_scale=idx_sm_scale, + kv_indices=metadata.msa_kv_indices, + qo_lens_cpu=metadata.msa_qo_lens_cpu, + kv_lens_cpu=metadata.msa_kv_lens_cpu, + qo_offset_cpu=metadata.msa_qo_offset_cpu, + proxy_plan=proxy_plan, + max_score=max_score, + n_valid_blocks=n_valid_blocks, + ) + + def sparse_attn_predict( + self, + q: torch.Tensor, + k: Optional[torch.Tensor], + metadata, + forward_args: "AttentionForwardArgs", + ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: + # The model layer runs run_indexer and passes the selected block + # indices through forward_args.topk_indices. Publish them as the + # sparse attention indices MsaSparseGqaFmha reads. + return forward_args.topk_indices, None + + def sparse_kv_predict( + self, + q: torch.Tensor, + k: Optional[torch.Tensor], + metadata, + forward_args: "AttentionForwardArgs", + ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: + return None, None + + +__all__ = [ + "MiniMaxM3MsaSparseAttention", + "MiniMaxM3MsaSparseAttentionMetadata", +] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_indexer.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_indexer.py new file mode 100644 index 000000000000..b4a836d03ffa --- /dev/null +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_indexer.py @@ -0,0 +1,199 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""MiniMax-M3 MSA sparse-attention indexer. + +Mirrors the DSA indexer pattern: a submodule owned by the sparse backend +that runs the predictor pass and returns the per-query selected KV block +indices the main attention consumes. It calls fmha_sm100 directly in +output_maxscore mode, reduces the per-index-head max score to KV-head +granularity, and selects the top-k blocks per query. + +Results are [total_q, num_kv_heads, topk] int32, ascending with -1 padding. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +import torch + +from .msa_utils import ( + MSA_REQUIRED_TOPK, + per_token_valid_blocks, + require_msa_module, + select_blocks_from_maxscore, +) + +if TYPE_CHECKING: + from .common import MiniMaxM3SparseConfig + + +def _proxy_max_score( + idx_q: torch.Tensor, + idx_k_paged: torch.Tensor, + *, + qo_lens_cpu: torch.Tensor, + kv_lens_cpu: torch.Tensor, + qo_offset_cpu: Optional[torch.Tensor], + kv_indices: torch.Tensor, + sm_scale: float, + causal: bool, +) -> torch.Tensor: + """Run the fmha_sm100 MQA proxy pass and return the per-block max score. + + Follows MSA's two-call pattern: fmha_sm100_plan builds the plan with + output_maxscore and num_kv_heads 1, then fmha_sm100 runs with output_o + disabled so only the per-block max score is produced. Returns + [num_index_heads, max_k_tiles, total_q] float32. + """ + fmha_sm100 = require_msa_module() + + if idx_q.dim() != 3: + raise ValueError( + "MsaIndexer expects idx_q [total_q, num_index_heads, head_dim]; " + f"got {tuple(idx_q.shape)}." + ) + if idx_k_paged.dim() != 4 or idx_k_paged.shape[1] != 1: + raise ValueError( + "MsaIndexer expects MQA paged index-K [num_pages, 1, page_size, head_dim]; " + f"got {tuple(idx_k_paged.shape)}." + ) + + page_size = int(idx_k_paged.shape[2]) + proxy_plan = fmha_sm100.fmha_sm100_plan( + qo_lens_cpu, + kv_lens_cpu, + idx_q.shape[1], + num_kv_heads=1, + qo_offset=qo_offset_cpu, + page_size=page_size, + output_maxscore=True, + causal=causal, + num_kv_splits=1, + ) + _, max_score = fmha_sm100.fmha_sm100( + idx_q, + idx_k_paged, + idx_k_paged, + proxy_plan, + kv_indices=kv_indices, + output_o=False, + output_maxscore=True, + sm_scale=sm_scale, + ) + return max_score + + +def _group_max_reduce( + max_score: torch.Tensor, + config: "MiniMaxM3SparseConfig", +) -> torch.Tensor: + """Reduce per-index-head max score to per-KV-head granularity by amax. + + Index heads are assumed to be grouped contiguously per KV head, so head h + maps to KV group h // group. + """ + group, rem = divmod(config.num_index_heads, config.num_kv_heads) + if rem != 0: + raise ValueError( + "num_index_heads must be divisible by num_kv_heads for group max " + f"reduce; got num_index_heads={config.num_index_heads}, " + f"num_kv_heads={config.num_kv_heads}." + ) + if group > 1: + return max_score.view( + config.num_kv_heads, group, max_score.shape[1], max_score.shape[2] + ).amax(dim=1) + return max_score + + +class MsaIndexer: + """Predictor submodule: proxy MQA scoring and top-k block selection. + + Owned by the MSA attention layer. Stateless in eager mode: it reads the + per-forward page table and lengths from the attention metadata and calls + the kernel directly. + """ + + def __init__(self, config: "MiniMaxM3SparseConfig"): + self.config = config + + def select_blocks( + self, + idx_q: torch.Tensor, + idx_k_paged: torch.Tensor, + *, + idx_sm_scale: float, + kv_indices: torch.Tensor, + qo_lens_cpu: Optional[torch.Tensor] = None, + kv_lens_cpu: Optional[torch.Tensor] = None, + qo_offset_cpu: Optional[torch.Tensor] = None, + proxy_plan: Optional[tuple] = None, + max_score: Optional[torch.Tensor] = None, + n_valid_blocks: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Return [total_q, num_kv_heads, topk] selected block indices. + + Plan/run split, mirroring the sparse GQA. When `proxy_plan` is None + (prefill and focused tests) the proxy plan is built inline and the + per-query valid-block count is derived here; when provided (CUDA-graph + decode) the proxy runs from the prebuilt plan into the preallocated + `max_score` buffer with a precomputed `n_valid_blocks`, so there is + no host sync inside the captured region. The same top-k selection serves + both, and generation is the one-query-token-per-request special case. + """ + config = self.config + + if proxy_plan is None: + max_score = _proxy_max_score( + idx_q, + idx_k_paged, + qo_lens_cpu=qo_lens_cpu, + kv_lens_cpu=kv_lens_cpu, + qo_offset_cpu=qo_offset_cpu, + kv_indices=kv_indices, + sm_scale=idx_sm_scale, + causal=True, + ) + else: + fmha_sm100 = require_msa_module() + _, max_score = fmha_sm100.fmha_sm100( + idx_q, + idx_k_paged, + idx_k_paged, + proxy_plan, + kv_indices=kv_indices, + output_o=False, + output_maxscore=True, + max_score=max_score, + sm_scale=idx_sm_scale, + ) + max_score_kv = _group_max_reduce(max_score, config) + + if n_valid_blocks is None: + n_valid_blocks = per_token_valid_blocks( + qo_lens_cpu, + kv_lens_cpu, + qo_offset_cpu, + causal=True, + block_size=int(idx_k_paged.shape[2]), + ) + # The empty-selection guard uses a host sync, so it only runs on the + # eager path; a decode batch always has valid blocks. + if n_valid_blocks.numel() == 0 or int(n_valid_blocks.max().item()) <= 0: + return torch.full( + (idx_q.shape[0], config.num_kv_heads, MSA_REQUIRED_TOPK), + -1, + dtype=torch.int32, + device=idx_q.device, + ) + return select_blocks_from_maxscore( + max_score_kv, + topk=MSA_REQUIRED_TOPK, + n_valid_blocks=n_valid_blocks, + init_blocks=config.init_blocks, + local_blocks=config.local_blocks, + ) + + +__all__ = ["MsaIndexer"] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_utils.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_utils.py new file mode 100644 index 000000000000..262f5e10ec0d --- /dev/null +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_utils.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""MSA (fmha_sm100) specific helpers for the MiniMax-M3 sparse backend.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from typing import Optional, Tuple + +import torch + +from .common import _INIT_SCORE, _LOCAL_SCORE, write_kv_slots + +# fmha_sm100 ships only head_dim 128 variants and the MiniMax-M3 checkpoint +# selects topk 16. Callers enforce these early so a misconfiguration fails +# with a clear message rather than a cryptic shape error inside the kernel. +MSA_REQUIRED_TOPK = 16 +MSA_REQUIRED_HEAD_DIM = 128 + +# Path of the fmha_sm100 package inside the MSA git submodule relative to the +# repository root (see 3rdparty/MSA/LICENSE and 3rdparty/MSA/NOTICE). +_MSA_PYTHON_RELPATH = Path("3rdparty") / "MSA" / "python" + + +def _find_msa_python_dir() -> Optional[Path]: + """Locate the fmha_sm100 package dir by walking up from this file. + + Returns None in installed layouts where the 3rdparty submodule is not + shipped. Walking up avoids hardcoding this module's depth below the + repository root. + """ + for parent in Path(__file__).resolve().parents: + candidate = parent / _MSA_PYTHON_RELPATH + if candidate.is_dir(): + return candidate + return None + + +def _ensure_msa_on_path() -> None: + """Prepend the MSA python package directory to sys.path if present.""" + msa_python = _find_msa_python_dir() + if msa_python is not None and str(msa_python) not in sys.path: + sys.path.insert(0, str(msa_python)) + + +def msa_package_available() -> bool: + """True if fmha_sm100 can be imported (submodule checkout or installed).""" + _ensure_msa_on_path() + return importlib.util.find_spec("fmha_sm100") is not None + + +def require_msa_module(): + """Import fmha_sm100 from the MSA submodule or raise a clear error. + + The import is deferred to first kernel use so the MSA backend can be + advertised in the config schema on systems where the kernels cannot load. + The 3rdparty/MSA/python directory is added to sys.path first, so a source + checkout with the submodule initialized resolves without a separate install. + A missing package is a hard error, never a silent fallback to another backend. + """ + _ensure_msa_on_path() + try: + import fmha_sm100 + except ImportError as exc: + raise RuntimeError( + "MiniMax-M3 MSA attention requires the fmha_sm100 kernels from the " + "MSA git submodule at 3rdparty/MSA. Initialize it with " + "'git submodule update --init --recursive', or install fmha_sm100." + ) from exc + return fmha_sm100 + + +def msa_paged_kv(kv_cache_manager, layer_idx: int) -> Tuple[torch.Tensor, torch.Tensor]: + """Return per-layer paged K and V in fmha_sm100 HND layout, zero-copy. + + The cache is stored head-major (see `write_msa_main_kv`), so the "HND" + buffer view is already the [num_slots, num_kv_heads, page_size, head_dim] + layout fmha_sm100 expects. The kernel reads the page and head strides at + runtime and needs only each page's [page_size, head_dim] block to be + contiguous, which this view satisfies, so no copy is required. + """ + buffers = kv_cache_manager.get_buffers(layer_idx, kv_layout="HND") + return buffers[:, 0], buffers[:, 1] + + +def write_msa_main_kv( + kv_cache_manager, + layer_idx: int, + out_cache_loc: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, +) -> None: + """Write new-token K and V into the paged main cache at out_cache_loc. + + fmha_sm100 reads the paged cache directly, so the new-token K and V must be + resident before the sparse GQA runs. The write uses the head-major HND view + so `msa_paged_kv` can return a zero-copy view. + """ + buffers = kv_cache_manager.get_buffers(layer_idx, kv_layout="HND") + k_view, v_view = buffers[:, 0], buffers[:, 1] + num_kv_heads = int(k_view.shape[1]) + head_dim = int(k_view.shape[3]) + num_tokens = int(k.shape[0]) + write_kv_slots( + k_view, out_cache_loc, k.reshape(num_tokens, num_kv_heads, head_dim), layout="HND" + ) + write_kv_slots( + v_view, out_cache_loc, v.reshape(num_tokens, num_kv_heads, head_dim), layout="HND" + ) + + +def build_kv_page_indices( + req_to_token: torch.Tensor, + slot_ids: torch.Tensor, + kv_lens_cpu: torch.Tensor, + page_size: int, +) -> torch.Tensor: + """Build the flattened per-request page table fmha_sm100 consumes. + + Returns int32 global page ids concatenated per request. A request's + pages come from the first slot of each page in its req_to_token row. + Page ids are global and non-contiguous in production, so they are not + clamped to a per-request bound. + """ + device = req_to_token.device + req_rows = req_to_token.index_select(0, slot_ids.to(torch.long)).to(torch.long) + batch = int(req_rows.shape[0]) + kv_lens_list = kv_lens_cpu.to(torch.long).tolist() + + page_lists = [] + for b in range(batch): + kv_len = int(kv_lens_list[b]) + if kv_len <= 0: + continue + num_pages = (kv_len + page_size - 1) // page_size + page_starts = torch.arange(num_pages, device=device, dtype=torch.long) * page_size + page_ids = req_rows[b].gather(0, page_starts) // page_size + page_lists.append(page_ids.to(torch.int32)) + + if page_lists: + return torch.cat(page_lists, dim=0) + return torch.empty(0, dtype=torch.int32, device=device) + + +def per_token_valid_blocks( + qo_lens_cpu: torch.Tensor, + kv_lens_cpu: torch.Tensor, + qo_offset_cpu: Optional[torch.Tensor], + *, + causal: bool, + block_size: int, +) -> torch.Tensor: + """Return the per-query number of valid KV blocks, on CPU. + + Expands per-request lengths and offsets to a per-token vector so block + selection can honour each query token's own causal extent. + """ + qo = qo_lens_cpu.to(torch.long) + kv = kv_lens_cpu.to(torch.long) + batch = int(qo.shape[0]) + total = int(qo.sum().item()) + if total == 0: + return torch.zeros(0, dtype=torch.long) + batch_row = torch.repeat_interleave(torch.arange(batch, dtype=torch.long), qo) + starts = torch.zeros(batch, dtype=torch.long) + if batch > 1: + starts[1:] = torch.cumsum(qo, 0)[:-1] + intra = torch.arange(total, dtype=torch.long) - starts[batch_row] + kv_per = kv[batch_row] + if causal: + if qo_offset_cpu is not None: + off = qo_offset_cpu.to(torch.long)[batch_row] + else: + off = (kv - qo)[batch_row] + eff = torch.minimum(off + intra + 1, kv_per) + else: + eff = kv_per + return (eff + block_size - 1) // block_size + + +def select_blocks_from_maxscore( + max_score_kv: torch.Tensor, + *, + topk: int, + n_valid_blocks: torch.Tensor, + init_blocks: int, + local_blocks: int, +) -> torch.Tensor: + """Select per-query top-k blocks from per-KV-head block scores. + + Applies init and local forced blocks and per-query valid-block masking + on the amax-reduced scores [num_kv_heads, n_blocks, total_q]. Returns + [total_q, num_kv_heads, topk] int32 ascending block ids with -1 tail + padding. + """ + num_kv_heads, n_blocks, total_q = max_score_kv.shape + device = max_score_kv.device + scores = max_score_kv.permute(2, 0, 1).to(torch.float32).clone() + block_ids = torch.arange(n_blocks, device=device, dtype=torch.long) + nvb = n_valid_blocks.to(device=device, dtype=torch.long) + + if init_blocks > 0: + init_mask = block_ids.view(1, 1, -1) < init_blocks + scores = torch.where(init_mask, torch.full_like(scores, _INIT_SCORE), scores) + if local_blocks > 0: + local_start = (nvb - local_blocks).clamp_min(0) + local_mask = (block_ids.view(1, -1) >= local_start.view(-1, 1)) & ( + block_ids.view(1, -1) < nvb.view(-1, 1) + ) + scores = torch.where(local_mask.unsqueeze(1), torch.full_like(scores, _LOCAL_SCORE), scores) + block_valid = block_ids.view(1, -1) < nvb.view(-1, 1) + scores = scores.masked_fill(~block_valid.unsqueeze(1), float("-inf")) + + k = min(topk, n_blocks) + vals, idx = scores.topk(k=k, dim=-1) + idx = torch.where(vals != float("-inf"), idx, torch.full_like(idx, -1)) + sort_key = torch.where(idx < 0, torch.full_like(idx, n_blocks), idx) + sort_key, _ = torch.sort(sort_key, dim=-1) + idx = torch.where(sort_key >= n_blocks, torch.full_like(sort_key, -1), sort_key) + if k < topk: + pad = torch.full((total_q, num_kv_heads, topk - k), -1, dtype=idx.dtype, device=device) + idx = torch.cat([idx, pad], dim=-1) + return idx.to(torch.int32) + + +__all__ = [ + "MSA_REQUIRED_HEAD_DIM", + "MSA_REQUIRED_TOPK", + "build_kv_page_indices", + "msa_package_available", + "msa_paged_kv", + "per_token_valid_blocks", + "require_msa_module", + "select_blocks_from_maxscore", + "write_msa_main_kv", +] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/backend.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_backend.py similarity index 63% rename from tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/backend.py rename to tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_backend.py index 090c6d7956fd..844fd979469c 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/backend.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_backend.py @@ -46,7 +46,7 @@ ----------------- All scalar max lengths (``max_seqlen_q``, ``max_seqlen_k``) are -pre-computed CPU-side in :meth:`MiniMaxM3SparseAttentionMetadata.prepare` +pre-computed CPU-side in :meth:`MiniMaxM3TritonSparseAttentionMetadata.prepare` and stored as plain Python ints. The hot path uses only batched tensor ops with static shapes derived from those CPU-side scalars; no ``.item()`` or other GPU-CPU sync runs inside the forward @@ -56,29 +56,29 @@ from __future__ import annotations -import functools import os from dataclasses import dataclass from typing import TYPE_CHECKING, Optional, Tuple import torch -from .kernels import triton_block_max_score, triton_sparse_softmax -from .metadata import ( - MiniMaxM3SparseAttentionMetadata, - MiniMaxM3SparseConfig, +from ...interface import ( + AttentionBackend, + AttentionForwardArgs, + AttentionMetadata, + merge_attention_forward_args, +) +from .common import _INIT_SCORE, _LOCAL_SCORE, MiniMaxM3SparseConfig, write_kv_slots +from .triton_kernels import triton_block_max_score, triton_sparse_softmax +from .triton_metadata import ( + MiniMaxM3AttentionMetadata, + MiniMaxM3TritonSparseAttentionMetadata, ensure_metadata_on_device, - get_minimax_m3_attention_metadata_cls, ) if TYPE_CHECKING: from .cache_manager import MiniMaxM3SparseIndexCache - from .metadata import MiniMaxM3SparseParams - -# Sentinel block score for blocks that init / local priority forces into -# the top-k regardless of their numerical score. -_INIT_SCORE = 1e30 -_LOCAL_SCORE = 1e29 + from .common import MiniMaxM3SparseParams # --------------------------------------------------------------------------- @@ -143,53 +143,6 @@ def _gather_paged_batched( return flat.view(batch, max_k, *cache.shape[1:]) -def _assert_paged_write_in_bounds( - name: str, - cache: torch.Tensor, - page: torch.Tensor, - within: torch.Tensor, -) -> None: - """Optional CPU-side bounds check for paged-cache writes. - - The runtime computes per-token slot ids from - ``KVCacheManagerV2``'s block ids; if the runtime ever produces a - block id that does not fit the per-layer view's dim-0 the write - falls into another layer's coalesced memory and corrupts the - cache, or fires the CUDA ``IndexKernel.cu`` device-side assert - during fancy indexing. Both are far enough away from the root - cause to be hard to triage. - - When ``TRTLLM_MINIMAX_M3_DEBUG_BOUNDS`` is set this check runs a - CPU-side max/min comparison against the cache's dim-0 and dim-2 - bounds, surfacing the misindex with the exact tensor names and - values instead of a device-side assert spam. It is opt-in - because the comparison forces a CPU sync. - """ - if not os.environ.get("TRTLLM_MINIMAX_M3_DEBUG_BOUNDS"): - return - num_pages = int(cache.shape[0]) - tokens_per_block = int(cache.shape[1]) if cache.ndim == 4 else int(cache.shape[2]) - page_max = int(page.max().item()) if page.numel() else -1 - page_min = int(page.min().item()) if page.numel() else 0 - within_max = int(within.max().item()) if within.numel() else -1 - within_min = int(within.min().item()) if within.numel() else 0 - assert 0 <= page_min and page_max < num_pages, ( - f"{name}: page index out of bounds — page.min={page_min} " - f"page.max={page_max} but cache.shape[0]={num_pages} " - f"(shape={tuple(cache.shape)}). This usually means the " - f"runtime's get_block_ids_per_seq produced a block id wider " - f"than the per-layer paged view's dim-0; check that the M3 " - f"override path returns slot ids in [0, num_slots)." - ) - assert 0 <= within_min and within_max < tokens_per_block, ( - f"{name}: within-page offset out of bounds — within.min=" - f"{within_min} within.max={within_max} but tokens_per_block=" - f"{tokens_per_block} (shape={tuple(cache.shape)}). This " - f"usually means out_cache_loc was computed with a different " - f"tokens_per_block than the cache was allocated with." - ) - - def _write_main_kv_slots_to_pool( pool: torch.Tensor, kv_index: int, @@ -201,40 +154,12 @@ def _write_main_kv_slots_to_pool( ``pool`` is the 5-D main K/V pool returned by :meth:`KVCacheManagerV2.get_buffers` with the NHD layout ``[num_pages, kv_factor, tokens_per_block, num_kv_heads, head_dim]``. - ``values`` has shape ``[num_new_tokens, num_kv_heads, head_dim]`` - and ``out_cache_loc`` is the 1-D ``[num_new_tokens]`` int tensor of - flat slot ids the caller wants to update. - - The write decomposes each flat slot id into - ``(page = s // tokens_per_block, within = s % tokens_per_block)`` - and uses multi-dim fancy-index assignment so the writes propagate - to the underlying pool storage. The previously used pattern - ``pool[:, kv_index].reshape(-1, num_kv_heads, head_dim) - .index_copy_(0, ...)`` instead wrote into a silent copy (see - :func:`_gather_paged_batched`), so the next forward call read - zeros for the prefilled positions. - - The optional CPU-side bounds assertion (enabled when the - ``TRTLLM_MINIMAX_M3_DEBUG_BOUNDS`` env var is set) catches - block_ids overflowing the pool's dim-0 before the device-side - ``IndexKernel.cu`` assert fires deep inside the kernel. The - assertion is a CPU sync, so the env var keeps it opt-in for - production runs that need a clean fast path. + ``values`` has shape ``[num_new_tokens, num_kv_heads, head_dim]`` and + ``out_cache_loc`` is the 1-D ``[num_new_tokens]`` int tensor of flat slot + ids to update. ``pool[:, kv_index]`` is a storage-sharing view, so the + shared :func:`common.write_kv_slots` propagates the write to the pool. """ - tokens_per_block = int(pool.shape[2]) - out_long = out_cache_loc.to(torch.long) - page = out_long // tokens_per_block - within = out_long % tokens_per_block - _assert_paged_write_in_bounds("pool", pool, page, within) - # KV-cache writes never need to participate in autograd. Wrap the - # fancy-index assignment in ``torch.no_grad()`` so callers that - # enter this path with an active grad context (e.g. unit tests - # exercising :class:`MiniMaxM3Attention` without ``inference_mode``) - # do not trip the "leaf Variable that requires grad is being used - # in an in-place operation" autograd guard on the view chain. - with torch.no_grad(): - # Multi-dim fancy assignment writes into the underlying pool buffer. - pool[page, kv_index, within] = values.to(pool.dtype) + write_kv_slots(pool[:, kv_index], out_cache_loc, values) def _write_main_kv_slots( @@ -242,39 +167,13 @@ def _write_main_kv_slots( out_cache_loc: torch.Tensor, values: torch.Tensor, ) -> None: - """Layout-aware writer for K (or V) caches used by the M3 backend. - - Supports two layouts, mirroring :func:`_gather_paged_batched`: - - * **3-D flat-slot** ``[num_slots, num_kv_heads, head_dim]``: used - by focused unit tests that allocate the cache as a contiguous - flat-slot tensor. ``index_copy_(0, ...)`` writes propagate - because the tensor IS the storage. - * **4-D multi-dim paged** ``[num_pages, tokens_per_block, - num_kv_heads, head_dim]``: used when the cache is a view of - ``kv_pool[:, 0]`` / ``kv_pool[:, 1]``. The view is - non-contiguous (its dim-0 stride is 2× the contiguous stride - because dim 1 separates K from V in the pool), so - ``index_copy_(0, ...)`` would silently fork a copy and the - write would be lost. Decompose the flat slot id into - ``(page, within)`` and use multi-dim fancy assignment so the - write propagates through the view to the underlying pool. + """Write per-new-token K or V into a cache view via the shared writer. + + Delegates to :func:`common.write_kv_slots`, which handles both the 3-D + flat-slot layout used by focused unit tests and the 4-D paged view of + ``kv_pool[:, 0]`` / ``kv_pool[:, 1]``. """ - # KV-cache writes never need to participate in autograd. Wrap both - # branches in ``torch.no_grad()`` so callers that enter this path - # with an active grad context (e.g. unit tests exercising - # :class:`MiniMaxM3Attention` without ``inference_mode``) do not - # trip the autograd in-place guard on the cache view chain. - with torch.no_grad(): - if cache.ndim >= 4: - tokens_per_block = int(cache.shape[1]) - out_long = out_cache_loc.to(torch.long) - page = out_long // tokens_per_block - within = out_long % tokens_per_block - _assert_paged_write_in_bounds("cache", cache, page, within) - cache[page, within] = values.to(cache.dtype) - else: - cache.index_copy_(0, out_cache_loc.to(torch.long), values.to(cache.dtype)) + write_kv_slots(cache, out_cache_loc, values) def _scatter_topk_to_block_mask( @@ -699,7 +598,7 @@ def minimax_m3_sparse_decode( v_cache: torch.Tensor, idx_k_cache: torch.Tensor, idx_v_cache: Optional[torch.Tensor], - metadata: MiniMaxM3SparseAttentionMetadata, + metadata: MiniMaxM3TritonSparseAttentionMetadata, config: MiniMaxM3SparseConfig, *, disable_index_value: bool, @@ -794,7 +693,7 @@ def minimax_m3_sparse_prefill( idx_q: torch.Tensor, idx_k_cache: torch.Tensor, idx_v_cache: Optional[torch.Tensor], - metadata: MiniMaxM3SparseAttentionMetadata, + metadata: MiniMaxM3TritonSparseAttentionMetadata, config: MiniMaxM3SparseConfig, *, disable_index_value: bool, @@ -890,18 +789,8 @@ def minimax_m3_sparse_prefill( # --------------------------------------------------------------------------- -# Lazy import alias to avoid a circular import at module load — the side -# cache class lives in ``cache_manager`` which imports from this module -# is fine (no cycle), but keeping the indirection explicit makes the -# dependency direction in this file clearer. -def _import_index_cache_cls(): - from .cache_manager import MiniMaxM3SparseIndexCache - - return MiniMaxM3SparseIndexCache - - @dataclass -class MiniMaxM3SparseAttention: +class MiniMaxM3TritonSparseAttention: """Thin orchestrator for :func:`minimax_m3_sparse_prefill` and :func:`minimax_m3_sparse_decode`. @@ -909,7 +798,7 @@ class MiniMaxM3SparseAttention: :class:`MiniMaxM3SparseIndexCache`. The caller is responsible for routing the projected Q, K, V, ``idx_q``, ``idx_k`` (and optional ``idx_v``) tensors plus the populated - :class:`MiniMaxM3SparseAttentionMetadata`. + :class:`MiniMaxM3TritonSparseAttentionMetadata`. """ config: MiniMaxM3SparseConfig @@ -946,7 +835,7 @@ def forward( idx_q: torch.Tensor, k_cache: torch.Tensor, v_cache: torch.Tensor, - metadata: MiniMaxM3SparseAttentionMetadata, + metadata: MiniMaxM3TritonSparseAttentionMetadata, *, disable_index_value: bool, sm_scale: Optional[float] = None, @@ -1001,322 +890,304 @@ def forward( # --------------------------------------------------------------------------- -@functools.lru_cache(maxsize=1) -def get_minimax_m3_attention_backend_cls(): - """Return :class:`MiniMaxM3SparseRuntimeBackend` (lazy import). +class MiniMaxM3SparseRuntimeBackend(AttentionBackend[AttentionMetadata]): + """:class:`AttentionBackend` for MiniMax-M3 sparse layers. - Deferring the :class:`AttentionBackend` import keeps the algorithm - module usable from test paths that do not need the runtime backend. - """ - from ...interface import ( - AttentionBackend, - AttentionForwardArgs, - AttentionMetadata, - merge_attention_forward_args, - ) + Constructed under the standard ``create_attention(...)`` dispatch + when ``SparseAttentionConfig(algorithm='minimax_m3', ...)`` is + configured. Drives the MiniMax-M3 sparse algorithm directly via + :func:`minimax_m3_sparse_prefill` and + :func:`minimax_m3_sparse_decode`. - metadata_cls = get_minimax_m3_attention_metadata_cls() - - class MiniMaxM3SparseRuntimeBackend(AttentionBackend[AttentionMetadata]): - """:class:`AttentionBackend` for MiniMax-M3 sparse layers. - - Constructed under the standard ``create_attention(...)`` dispatch - when ``SparseAttentionConfig(algorithm='minimax_m3', ...)`` is - configured. Drives the MiniMax-M3 sparse algorithm directly via - :func:`minimax_m3_sparse_prefill` and - :func:`minimax_m3_sparse_decode`. - - The standard :class:`AttentionForwardArgs` surface does not - carry ``idx_q`` / ``idx_k`` slots, so the model layer threads - the index branch through ``**kwargs`` of :meth:`forward`. When - ``forward`` is called without ``idx_q`` it raises - :class:`NotImplementedError` with a pointer at the model layer - — the backend's ``forward`` is **executable**, but it is not a - substitute for the MiniMax-specific projection / norm / RoPE - steps the model layer is responsible for. - - The backend exposes - :meth:`forward_sparse` for callers that want a name-explicit - entry point (the model layer calls it directly); :meth:`forward` - is the standard contract entry point and routes to - :meth:`forward_sparse` when ``idx_q`` is supplied. - """ + The standard :class:`AttentionForwardArgs` surface does not + carry ``idx_q`` / ``idx_k`` slots, so the model layer threads + the index branch through ``**kwargs`` of :meth:`forward`. When + ``forward`` is called without ``idx_q`` it raises + :class:`NotImplementedError` with a pointer at the model layer + — the backend's ``forward`` is **executable**, but it is not a + substitute for the MiniMax-specific projection / norm / RoPE + steps the model layer is responsible for. + + The backend exposes + :meth:`forward_sparse` for callers that want a name-explicit + entry point (the model layer calls it directly); :meth:`forward` + is the standard contract entry point and routes to + :meth:`forward_sparse` when ``idx_q`` is supplied. + """ - Metadata = metadata_cls + Metadata = MiniMaxM3AttentionMetadata - def __init__( - self, - layer_idx: int, - num_heads: int, - head_dim: int, - num_kv_heads: Optional[int] = None, - quant_config=None, - sparse_params: Optional["MiniMaxM3SparseParams"] = None, + def __init__( + self, + layer_idx: int, + num_heads: int, + head_dim: int, + num_kv_heads: Optional[int] = None, + quant_config=None, + sparse_params: Optional["MiniMaxM3SparseParams"] = None, + **kwargs, + ): + if sparse_params is None: + raise ValueError("sparse_params is required for MiniMaxM3SparseRuntimeBackend") + super().__init__( + layer_idx, + num_heads, + head_dim, + num_kv_heads=num_kv_heads, + quant_config=quant_config, + sparse_params=sparse_params, **kwargs, - ): - if sparse_params is None: - raise ValueError("sparse_params is required for MiniMaxM3SparseRuntimeBackend") - super().__init__( - layer_idx, - num_heads, - head_dim, - num_kv_heads=num_kv_heads, - quant_config=quant_config, - sparse_params=sparse_params, - **kwargs, + ) + self.m3_config = MiniMaxM3SparseConfig.from_sparse_params( + sparse_params, + num_q_heads=num_heads, + num_kv_heads=num_kv_heads or num_heads, + head_dim=head_dim, + ) + self.disable_index_value = bool(sparse_params.disable_index_value) + + @staticmethod + def support_fused_rope() -> bool: + # The MiniMax-M3 model layer applies RoPE explicitly because + # both the main and the index branches need partial RoPE, + # and the standard fused-RoPE attention op does not have a + # hook for the index branch. + return False + + def forward_sparse( + self, + *, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + idx_q: torch.Tensor, + idx_k: torch.Tensor, + idx_v: Optional[torch.Tensor], + k_cache: torch.Tensor, + v_cache: torch.Tensor, + idx_k_cache: torch.Tensor, + idx_v_cache: Optional[torch.Tensor], + out_cache_loc: torch.Tensor, + m3_metadata: "MiniMaxM3TritonSparseAttentionMetadata", + sm_scale: Optional[float] = None, + idx_sm_scale: Optional[float] = None, + output: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Execute the MiniMax-M3 sparse path end-to-end. + + Inputs: + ``q``, ``k``, ``v`` : new-token projections, already + per-head norm + RoPE applied. + ``idx_q``, ``idx_k`` : index-branch projections, already + per-head norm + RoPE applied. + ``idx_v`` : index-V projection (only when + ``disable_index_value=False``). + ``k_cache``, ``v_cache``: flat-slot view of the paged + main K/V cache, + ``[num_slots, num_kv_heads, head_dim]``. + ``idx_k_cache`` : side index-K cache, + ``[num_slots, 1, sparse_index_dim]``. + ``idx_v_cache`` : side index-V cache (or ``None``). + ``out_cache_loc`` : ``[num_new_tokens]`` int slot + indices to write the new + token's K/V/idx_K to. + ``m3_metadata`` : populated + :class:`MiniMaxM3TritonSparseAttentionMetadata`. + ``output`` : optional preallocated final output, + ``[num_tokens, num_q_heads * head_dim]``. + + Returns ``[num_tokens, num_q_heads * head_dim]``. + """ + num_kv_heads = self.m3_config.num_kv_heads + head_dim = self.m3_config.head_dim + sparse_index_dim = self.m3_config.sparse_index_dim + num_idx_heads = self.m3_config.num_index_heads + + num_tokens = int(q.shape[0]) + q_view = q.view(num_tokens, self.num_heads, head_dim) + k_view = k.view(num_tokens, num_kv_heads, head_dim) + v_view = v.view(num_tokens, num_kv_heads, head_dim) + idx_q_view = idx_q.view(num_tokens, num_idx_heads, sparse_index_dim) + idx_k_view = idx_k.view(num_tokens, 1, sparse_index_dim) + + # Production paths build the M3 metadata on the cache + # device in ``MiniMaxM3AttentionMetadata.prepare`` (called + # outside any CUDA-graph capture window), and test paths + # construct it directly on the desired device. So all + # metadata tensors should already live on ``k_cache.device`` + # by this point. We keep a same-device pass for resilience + # against legacy test callers that produce metadata on a + # different device, but it must not introduce CPU->GPU + # copies inside the capture window. ``ensure_metadata_on_device`` + # is a no-op when each tensor is already on the target + # device; under capture that no-op path is the contract. + cache_device = k_cache.device + if any( + t is not None and t.device != cache_device + for t in ( + m3_metadata.req_to_token, + m3_metadata.slot_ids, + m3_metadata.seq_lens, + m3_metadata.prefix_lens, + m3_metadata.cu_seqlens_q, + m3_metadata.q_batch_row, + m3_metadata.q_positions, ) - self.m3_config = MiniMaxM3SparseConfig.from_sparse_params( - sparse_params, - num_q_heads=num_heads, - num_kv_heads=num_kv_heads or num_heads, - head_dim=head_dim, + ): + m3_metadata = ensure_metadata_on_device(m3_metadata, cache_device) + + # Write new K/V/idx_K to the configured slots. + # ``out_cache_loc`` comes from the pre-built attachment so + # it already lives on the cache device. The write goes + # through :func:`_write_main_kv_slots`, which is layout- + # aware: + # + # * 4-D multi-dim paged caches (the production V2 path: + # main K/V is ``kv_pool[:, 0]`` / ``kv_pool[:, 1]``, and + # ``idx_k_cache`` is the V2 4-D paged view ``[num_pages, + # tokens_per_block, 1, sparse_index_dim]``): decomposes + # each slot id into + # ``(page, within)`` and uses multi-dim fancy + # assignment so the write propagates to the underlying + # pool. A plain ``index_copy_(0, ...)`` would either + # silently fork a copy (non-contiguous main K/V view) + # or raise a shape mismatch (4-D index-K view), but + # the layout-aware helper sidesteps both failure + # modes. + # * 3-D flat-slot caches (focused unit tests that + # allocate plain ``torch.zeros((num_slots, num_heads, + # channel))`` tensors): falls back to + # ``index_copy_(0, ...)`` because the tensor IS the + # storage. + _write_main_kv_slots(k_cache, out_cache_loc, k_view) + _write_main_kv_slots(v_cache, out_cache_loc, v_view) + _write_main_kv_slots(idx_k_cache, out_cache_loc, idx_k_view) + if idx_v is not None and idx_v_cache is not None: + idx_v_view = idx_v.view(num_tokens, 1, sparse_index_dim) + _write_main_kv_slots(idx_v_cache, out_cache_loc, idx_v_view) + + if m3_metadata.is_prefill: + _, o = minimax_m3_sparse_prefill( + q_view, + k_cache, + v_cache, + idx_q_view, + idx_k_cache, + None if self.disable_index_value else idx_v_cache, + m3_metadata, + self.m3_config, + disable_index_value=self.disable_index_value, + sm_scale=sm_scale, + idx_sm_scale=idx_sm_scale, + output=output, ) - self.disable_index_value = bool(sparse_params.disable_index_value) - - @staticmethod - def support_fused_rope() -> bool: - # The MiniMax-M3 model layer applies RoPE explicitly because - # both the main and the index branches need partial RoPE, - # and the standard fused-RoPE attention op does not have a - # hook for the index branch. - return False - - def forward_sparse( - self, - *, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - idx_q: torch.Tensor, - idx_k: torch.Tensor, - idx_v: Optional[torch.Tensor], - k_cache: torch.Tensor, - v_cache: torch.Tensor, - idx_k_cache: torch.Tensor, - idx_v_cache: Optional[torch.Tensor], - out_cache_loc: torch.Tensor, - m3_metadata: "MiniMaxM3SparseAttentionMetadata", - sm_scale: Optional[float] = None, - idx_sm_scale: Optional[float] = None, - output: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """Execute the MiniMax-M3 sparse path end-to-end. - - Inputs: - ``q``, ``k``, ``v`` : new-token projections, already - per-head norm + RoPE applied. - ``idx_q``, ``idx_k`` : index-branch projections, already - per-head norm + RoPE applied. - ``idx_v`` : index-V projection (only when - ``disable_index_value=False``). - ``k_cache``, ``v_cache``: flat-slot view of the paged - main K/V cache, - ``[num_slots, num_kv_heads, head_dim]``. - ``idx_k_cache`` : side index-K cache, - ``[num_slots, 1, sparse_index_dim]``. - ``idx_v_cache`` : side index-V cache (or ``None``). - ``out_cache_loc`` : ``[num_new_tokens]`` int slot - indices to write the new - token's K/V/idx_K to. - ``m3_metadata`` : populated - :class:`MiniMaxM3SparseAttentionMetadata`. - ``output`` : optional preallocated final output, - ``[num_tokens, num_q_heads * head_dim]``. - - Returns ``[num_tokens, num_q_heads * head_dim]``. - """ - num_kv_heads = self.m3_config.num_kv_heads - head_dim = self.m3_config.head_dim - sparse_index_dim = self.m3_config.sparse_index_dim - num_idx_heads = self.m3_config.num_index_heads - - num_tokens = int(q.shape[0]) - q_view = q.view(num_tokens, self.num_heads, head_dim) - k_view = k.view(num_tokens, num_kv_heads, head_dim) - v_view = v.view(num_tokens, num_kv_heads, head_dim) - idx_q_view = idx_q.view(num_tokens, num_idx_heads, sparse_index_dim) - idx_k_view = idx_k.view(num_tokens, 1, sparse_index_dim) - - # Production paths build the M3 metadata on the cache - # device in ``MiniMaxM3AttentionMetadata.prepare`` (called - # outside any CUDA-graph capture window), and test paths - # construct it directly on the desired device. So all - # metadata tensors should already live on ``k_cache.device`` - # by this point. We keep a same-device pass for resilience - # against legacy test callers that produce metadata on a - # different device, but it must not introduce CPU->GPU - # copies inside the capture window. ``ensure_metadata_on_device`` - # is a no-op when each tensor is already on the target - # device; under capture that no-op path is the contract. - cache_device = k_cache.device - if any( - t is not None and t.device != cache_device - for t in ( - m3_metadata.req_to_token, - m3_metadata.slot_ids, - m3_metadata.seq_lens, - m3_metadata.prefix_lens, - m3_metadata.cu_seqlens_q, - m3_metadata.q_batch_row, - m3_metadata.q_positions, - ) - ): - m3_metadata = ensure_metadata_on_device(m3_metadata, cache_device) - - # Write new K/V/idx_K to the configured slots. - # ``out_cache_loc`` comes from the pre-built attachment so - # it already lives on the cache device. The write goes - # through :func:`_write_main_kv_slots`, which is layout- - # aware: - # - # * 4-D multi-dim paged caches (the production V2 path: - # main K/V is ``kv_pool[:, 0]`` / ``kv_pool[:, 1]``, and - # ``idx_k_cache`` is the V2 4-D paged view ``[num_pages, - # tokens_per_block, 1, sparse_index_dim]``): decomposes - # each slot id into - # ``(page, within)`` and uses multi-dim fancy - # assignment so the write propagates to the underlying - # pool. A plain ``index_copy_(0, ...)`` would either - # silently fork a copy (non-contiguous main K/V view) - # or raise a shape mismatch (4-D index-K view), but - # the layout-aware helper sidesteps both failure - # modes. - # * 3-D flat-slot caches (focused unit tests that - # allocate plain ``torch.zeros((num_slots, num_heads, - # channel))`` tensors): falls back to - # ``index_copy_(0, ...)`` because the tensor IS the - # storage. - _write_main_kv_slots(k_cache, out_cache_loc, k_view) - _write_main_kv_slots(v_cache, out_cache_loc, v_view) - _write_main_kv_slots(idx_k_cache, out_cache_loc, idx_k_view) - if idx_v is not None and idx_v_cache is not None: - idx_v_view = idx_v.view(num_tokens, 1, sparse_index_dim) - _write_main_kv_slots(idx_v_cache, out_cache_loc, idx_v_view) - - if m3_metadata.is_prefill: - _, o = minimax_m3_sparse_prefill( - q_view, - k_cache, - v_cache, - idx_q_view, - idx_k_cache, - None if self.disable_index_value else idx_v_cache, - m3_metadata, - self.m3_config, - disable_index_value=self.disable_index_value, - sm_scale=sm_scale, - idx_sm_scale=idx_sm_scale, - output=output, - ) - else: - _, o = minimax_m3_sparse_decode( - q_view, - idx_q_view, - k_cache, - v_cache, - idx_k_cache, - None if self.disable_index_value else idx_v_cache, - m3_metadata, - self.m3_config, - disable_index_value=self.disable_index_value, - sm_scale=sm_scale, - idx_sm_scale=idx_sm_scale, - output=output, - ) - return o - - def forward( - self, - q: torch.Tensor, - k: Optional[torch.Tensor], - v: Optional[torch.Tensor], - metadata=None, - forward_args: Optional[AttentionForwardArgs] = None, - *, - output: Optional[torch.Tensor] = None, - idx_q: Optional[torch.Tensor] = None, - idx_k: Optional[torch.Tensor] = None, - idx_v: Optional[torch.Tensor] = None, - k_cache: Optional[torch.Tensor] = None, - v_cache: Optional[torch.Tensor] = None, - idx_k_cache: Optional[torch.Tensor] = None, - idx_v_cache: Optional[torch.Tensor] = None, - out_cache_loc: Optional[torch.Tensor] = None, - m3_metadata: Optional["MiniMaxM3SparseAttentionMetadata"] = None, - sm_scale: Optional[float] = None, - idx_sm_scale: Optional[float] = None, - **kwargs, - ) -> torch.Tensor: - """Standard ``AttentionBackend.forward`` entry point. - - The MiniMax-M3 sparse path needs the index branch projection - and the M3-shaped metadata; both arrive through keyword - arguments because the standard - :class:`AttentionForwardArgs` surface does not carry them. - - When ``idx_q`` is omitted, this method raises - :class:`NotImplementedError` to make the misuse loud — a - generic AttentionBackend dispatch site cannot drive this - backend without supplying the index branch. - """ - forward_args = merge_attention_forward_args(forward_args, kwargs) - if ( - output is not None - and forward_args.output is not None - and output is not forward_args.output - ): - raise ValueError("output was supplied both directly and through forward_args") - if output is None: - output = forward_args.output - if idx_q is None or idx_k is None or m3_metadata is None: - raise NotImplementedError( - f"MiniMaxM3SparseRuntimeBackend.forward (layer " - f"{self.layer_idx}) requires the M3 index branch and " - "metadata to be passed as keyword arguments " - "(`idx_q`, `idx_k`, `m3_metadata`, " - "`out_cache_loc`, `k_cache`, `v_cache`, " - "`idx_k_cache`). The standard AttentionForwardArgs " - "surface does not carry them; the model layer " - "(`MiniMaxM3Attention.forward`) supplies them when " - "calling this backend." - ) - if ( - k is None - or v is None - or k_cache is None - or v_cache is None - or idx_k_cache is None - or out_cache_loc is None - ): - raise ValueError( - "MiniMaxM3SparseRuntimeBackend.forward requires k, v, " - "k_cache, v_cache, idx_k_cache, and out_cache_loc to " - "be supplied alongside idx_q / idx_k / m3_metadata." - ) - return self.forward_sparse( - q=q, - k=k, - v=v, - idx_q=idx_q, - idx_k=idx_k, - idx_v=idx_v, - k_cache=k_cache, - v_cache=v_cache, - idx_k_cache=idx_k_cache, - idx_v_cache=idx_v_cache, - out_cache_loc=out_cache_loc, - m3_metadata=m3_metadata, + else: + _, o = minimax_m3_sparse_decode( + q_view, + idx_q_view, + k_cache, + v_cache, + idx_k_cache, + None if self.disable_index_value else idx_v_cache, + m3_metadata, + self.m3_config, + disable_index_value=self.disable_index_value, sm_scale=sm_scale, idx_sm_scale=idx_sm_scale, output=output, ) + return o - return MiniMaxM3SparseRuntimeBackend + def forward( + self, + q: torch.Tensor, + k: Optional[torch.Tensor], + v: Optional[torch.Tensor], + metadata=None, + forward_args: Optional[AttentionForwardArgs] = None, + *, + output: Optional[torch.Tensor] = None, + idx_q: Optional[torch.Tensor] = None, + idx_k: Optional[torch.Tensor] = None, + idx_v: Optional[torch.Tensor] = None, + k_cache: Optional[torch.Tensor] = None, + v_cache: Optional[torch.Tensor] = None, + idx_k_cache: Optional[torch.Tensor] = None, + idx_v_cache: Optional[torch.Tensor] = None, + out_cache_loc: Optional[torch.Tensor] = None, + m3_metadata: Optional["MiniMaxM3TritonSparseAttentionMetadata"] = None, + sm_scale: Optional[float] = None, + idx_sm_scale: Optional[float] = None, + **kwargs, + ) -> torch.Tensor: + """Standard ``AttentionBackend.forward`` entry point. + + The MiniMax-M3 sparse path needs the index branch projection + and the M3-shaped metadata; both arrive through keyword + arguments because the standard + :class:`AttentionForwardArgs` surface does not carry them. + + When ``idx_q`` is omitted, this method raises + :class:`NotImplementedError` to make the misuse loud — a + generic AttentionBackend dispatch site cannot drive this + backend without supplying the index branch. + """ + forward_args = merge_attention_forward_args(forward_args, kwargs) + if ( + output is not None + and forward_args.output is not None + and output is not forward_args.output + ): + raise ValueError("output was supplied both directly and through forward_args") + if output is None: + output = forward_args.output + if idx_q is None or idx_k is None or m3_metadata is None: + raise NotImplementedError( + f"MiniMaxM3SparseRuntimeBackend.forward (layer " + f"{self.layer_idx}) requires the M3 index branch and " + "metadata to be passed as keyword arguments " + "(`idx_q`, `idx_k`, `m3_metadata`, " + "`out_cache_loc`, `k_cache`, `v_cache`, " + "`idx_k_cache`). The standard AttentionForwardArgs " + "surface does not carry them; the model layer " + "(`MiniMaxM3Attention.forward`) supplies them when " + "calling this backend." + ) + if ( + k is None + or v is None + or k_cache is None + or v_cache is None + or idx_k_cache is None + or out_cache_loc is None + ): + raise ValueError( + "MiniMaxM3SparseRuntimeBackend.forward requires k, v, " + "k_cache, v_cache, idx_k_cache, and out_cache_loc to " + "be supplied alongside idx_q / idx_k / m3_metadata." + ) + return self.forward_sparse( + q=q, + k=k, + v=v, + idx_q=idx_q, + idx_k=idx_k, + idx_v=idx_v, + k_cache=k_cache, + v_cache=v_cache, + idx_k_cache=idx_k_cache, + idx_v_cache=idx_v_cache, + out_cache_loc=out_cache_loc, + m3_metadata=m3_metadata, + sm_scale=sm_scale, + idx_sm_scale=idx_sm_scale, + output=output, + ) __all__ = [ - "MiniMaxM3SparseAttention", - "get_minimax_m3_attention_backend_cls", + "MiniMaxM3SparseRuntimeBackend", + "MiniMaxM3TritonSparseAttention", "minimax_m3_sparse_decode", "minimax_m3_sparse_prefill", ] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/kernels.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_kernels.py similarity index 100% rename from tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/kernels.py rename to tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_kernels.py diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/metadata.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_metadata.py similarity index 61% rename from tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/metadata.py rename to tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_metadata.py index 925ca6c0189b..be4b8ae24143 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/metadata.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_metadata.py @@ -1,131 +1,31 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""MiniMax-M3 sparse attention configuration + per-forward metadata. +"""MiniMax-M3 Triton reference per-forward metadata. Contains: - * :class:`MiniMaxM3SparseConfig` -- post-TP-shard kernel - parameter bundle. - * :class:`MiniMaxM3SparseAttentionMetadata` -- per-forward metadata - dataclass with a - CUDA-graph-safe - :meth:`prepare`. + * :class:`MiniMaxM3TritonSparseAttentionMetadata` -- per-forward metadata + dataclass with a CUDA-graph-safe :meth:`prepare`. * Helpers to migrate metadata across devices, build it from a real - :class:`KVCacheManagerV2`, and pre-allocate CUDA-graph-stable - buffers. - * :func:`get_minimax_m3_attention_metadata_cls` -- lazy factory for - the :class:`AttentionMetadata` subclass the pyexecutor wires into - the M3 sparse layer's forward path. + :class:`KVCacheManagerV2`, and pre-allocate CUDA-graph-stable buffers. + * :class:`MiniMaxM3AttentionMetadata` -- the :class:`AttentionMetadata` + subclass the pyexecutor wires into the M3 sparse layer's forward path. """ from __future__ import annotations import dataclasses -import functools from dataclasses import dataclass, field -from typing import List, Literal, Optional, Tuple +from typing import List, Optional, Tuple import torch -from ..params import SparseParams - - -@dataclass(frozen=True) -class MiniMaxM3SparseParams(SparseParams): - """Lowered runtime parameters for the MiniMax-M3 sparse backend.""" - - algorithm: Literal["minimax_m3"] = field(init=False, default="minimax_m3") - num_index_heads: int = 4 - sparse_index_dim: int = 128 - block_size: int = 128 - topk: int = 16 - init_blocks: int = 0 - local_blocks: int = 1 - score_type: str = "max" - disable_index_value: bool = True - - -@dataclass(frozen=True) -class MiniMaxM3SparseConfig: - """Per-rank kernel parameter bundle for MiniMax-M3 sparse attention. - - This is **not** a user-facing config (use - :class:`tensorrt_llm.llmapi.llm_args.MiniMaxM3SparseAttentionConfig` - for that). It is the layer-invariant, post-TP-shard parameter bundle - that backend kernels and reference helpers consume. The user knobs - come from :class:`MiniMaxM3SparseParams`; ``num_q_heads`` / - ``num_kv_heads`` / ``head_dim`` come from the per-rank model - geometry and must be supplied by the caller (typically via - :meth:`from_sparse_params`). - """ - - num_q_heads: int - num_kv_heads: int - head_dim: int - num_index_heads: int - sparse_index_dim: int - block_size: int - topk: int - init_blocks: int = 0 - local_blocks: int = 1 - score_type: str = "max" - - def __post_init__(self) -> None: - if self.num_q_heads % self.num_kv_heads != 0: - raise ValueError( - f"num_q_heads ({self.num_q_heads}) must be divisible by " - f"num_kv_heads ({self.num_kv_heads})" - ) - if self.num_index_heads % self.num_kv_heads != 0: - raise ValueError( - f"num_index_heads ({self.num_index_heads}) must be divisible " - f"by num_kv_heads ({self.num_kv_heads})" - ) - if self.block_size <= 0: - raise ValueError(f"block_size must be > 0, got {self.block_size}") - if self.topk <= 0: - raise ValueError(f"topk must be > 0, got {self.topk}") - if self.init_blocks < 0: - raise ValueError(f"init_blocks must be >= 0, got {self.init_blocks}") - if self.local_blocks < 0: - raise ValueError(f"local_blocks must be >= 0, got {self.local_blocks}") - if self.score_type != "max": - # SGLang exposes only "max" today and that is what the MiniMax-M3 - # checkpoint config specifies. Reject anything else explicitly so - # a config drift surfaces immediately. - raise ValueError( - f"score_type={self.score_type!r} is not supported " - "(only 'max' matches the SGLang reference)" - ) - - @classmethod - def from_sparse_params( - cls, - sparse_params: "MiniMaxM3SparseParams", - *, - num_q_heads: int, - num_kv_heads: int, - head_dim: int, - ) -> "MiniMaxM3SparseConfig": - """Build a kernel param bundle from lowered ``MiniMaxM3SparseParams`` - and the per-rank model geometry. - """ - return cls( - num_q_heads=int(num_q_heads), - num_kv_heads=int(num_kv_heads), - head_dim=int(head_dim), - num_index_heads=int(sparse_params.num_index_heads), - sparse_index_dim=int(sparse_params.sparse_index_dim), - block_size=int(sparse_params.block_size), - topk=int(sparse_params.topk), - init_blocks=int(sparse_params.init_blocks), - local_blocks=int(sparse_params.local_blocks), - score_type=str(sparse_params.score_type), - ) +from ...interface import AttentionMetadata +from .common import build_paged_kv_slot_mapping @dataclass -class MiniMaxM3SparseAttentionMetadata: +class MiniMaxM3TritonSparseAttentionMetadata: """Per-forward metadata for MiniMax-M3 sparse attention. Mirrors the shape of SGLang's @@ -269,12 +169,12 @@ def prepare(self) -> None: def ensure_metadata_on_device( - metadata: "MiniMaxM3SparseAttentionMetadata", + metadata: "MiniMaxM3TritonSparseAttentionMetadata", device: torch.device, -) -> "MiniMaxM3SparseAttentionMetadata": +) -> "MiniMaxM3TritonSparseAttentionMetadata": """Return ``metadata`` with every GPU-consumed tensor on ``device``. - Constructs a new :class:`MiniMaxM3SparseAttentionMetadata` whose + Constructs a new :class:`MiniMaxM3TritonSparseAttentionMetadata` whose tensor fields are migrated to ``device`` when they live elsewhere. The CPU-side mirror ``seq_lens_cpu`` is preserved because the algorithm reads it only when explicitly noted (e.g. for @@ -306,18 +206,6 @@ def _move(t: Optional[torch.Tensor]) -> Optional[torch.Tensor]: ) -def replace_metadata( - metadata: MiniMaxM3SparseAttentionMetadata, - **changes, -) -> MiniMaxM3SparseAttentionMetadata: - """Helper around :func:`dataclasses.replace` for ``metadata``. - - Provided so callers can build a decode metadata from a prefill - metadata without manually re-typing every field. - """ - return dataclasses.replace(metadata, **changes) - - def allocate_minimax_m3_static_buffers( *, max_num_sequences: int, @@ -405,6 +293,82 @@ def allocate_minimax_m3_static_buffers( } +def _build_runtime_metadata_fresh( + *, + kv_cache_manager, + request_ids, + seq_lens: torch.Tensor, + seq_lens_cpu: torch.Tensor, + is_prefill: bool, + prefix_lens: Optional[torch.Tensor], + extend_seq_lens_cpu: Optional[List[int]], + device: torch.device, +) -> Tuple[MiniMaxM3TritonSparseAttentionMetadata, torch.Tensor]: + """Fresh-allocation build of the Triton reference metadata. + + Delegates the backend-neutral req_to_token, slot_ids and out_cache_loc + derivation to common.build_paged_kv_slot_mapping and adds the Triton-only + fields: cu_seqlens_q, prefix_lens on device, and the scalar max_seqlen + values and per-query-token tensors that prepare() computes. Used when no + graph-stable static_buffers are supplied; the static_buffers path in + build_runtime_metadata_from_kv_manager keeps its own in-place buffer writes. + """ + batch = int(seq_lens.shape[0]) + seq_lens_dev = seq_lens.to(device) if seq_lens.device != device else seq_lens + + if is_prefill: + if extend_seq_lens_cpu is None: + raise ValueError("prefill metadata requires extend_seq_lens_cpu") + if prefix_lens is None: + raise ValueError("prefill metadata requires prefix_lens") + prefix_lens_dev = prefix_lens.to(device) if prefix_lens.device != device else prefix_lens + qo_lens_cpu = torch.tensor([int(x) for x in extend_seq_lens_cpu], dtype=torch.int32) + qo_offset_cpu = prefix_lens.detach().to(device="cpu", dtype=torch.int32) + req_to_token, slot_ids, out_cache_loc = build_paged_kv_slot_mapping( + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + qo_lens_cpu=qo_lens_cpu, + qo_offset_cpu=qo_offset_cpu, + device=device, + ) + cu_q: List[int] = [0] + for ext in extend_seq_lens_cpu: + cu_q.append(cu_q[-1] + int(ext)) + cu_seqlens_q = torch.tensor(cu_q, dtype=torch.int32, device=device) + meta = MiniMaxM3TritonSparseAttentionMetadata( + is_prefill=True, + req_to_token=req_to_token, + slot_ids=slot_ids, + seq_lens=seq_lens_dev, + seq_lens_cpu=seq_lens_cpu, + prefix_lens=prefix_lens_dev, + cu_seqlens_q=cu_seqlens_q, + extend_seq_lens_cpu=list(extend_seq_lens_cpu), + q_batch_row=None, + q_positions=None, + ) + else: + # Decode: the new token sits at position seq_lens[b] - 1. + qo_lens_cpu = torch.ones(batch, dtype=torch.int32) + qo_offset_cpu = seq_lens_cpu.detach().to(device="cpu", dtype=torch.int32) - 1 + req_to_token, slot_ids, out_cache_loc = build_paged_kv_slot_mapping( + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + qo_lens_cpu=qo_lens_cpu, + qo_offset_cpu=qo_offset_cpu, + device=device, + ) + meta = MiniMaxM3TritonSparseAttentionMetadata( + is_prefill=False, + req_to_token=req_to_token, + slot_ids=slot_ids, + seq_lens=seq_lens_dev, + seq_lens_cpu=seq_lens_cpu, + ) + meta.prepare() + return meta, out_cache_loc + + def build_runtime_metadata_from_kv_manager( *, kv_cache_manager, @@ -416,8 +380,8 @@ def build_runtime_metadata_from_kv_manager( extend_seq_lens_cpu: Optional[List[int]] = None, device: Optional[torch.device] = None, static_buffers: Optional[dict] = None, -) -> Tuple[MiniMaxM3SparseAttentionMetadata, torch.Tensor]: - """Build a :class:`MiniMaxM3SparseAttentionMetadata` from a real +) -> Tuple[MiniMaxM3TritonSparseAttentionMetadata, torch.Tensor]: + """Build a :class:`MiniMaxM3TritonSparseAttentionMetadata` from a real :class:`MiniMaxM3KVCacheManagerV2`. Returns the populated metadata plus an ``out_cache_loc`` tensor @@ -452,6 +416,21 @@ def build_runtime_metadata_from_kv_manager( the end-to-end runtime path without going through the full LLM forward. """ + if device is None: + device = seq_lens.device + + if static_buffers is None: + return _build_runtime_metadata_fresh( + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens_cpu, + is_prefill=is_prefill, + prefix_lens=prefix_lens, + extend_seq_lens_cpu=extend_seq_lens_cpu, + device=device, + ) + tokens_per_block = int(kv_cache_manager.tokens_per_block) # block_ids_per_seq is a [batch_size, max_blocks_per_seq] tensor; row b # holds the block ids assigned to request_ids[b] in order. @@ -459,8 +438,6 @@ def build_runtime_metadata_from_kv_manager( batch = int(seq_lens.shape[0]) max_blocks = int(block_ids.shape[1]) max_kv_len = max_blocks * tokens_per_block - if device is None: - device = seq_lens.device if static_buffers is not None: if static_buffers.get("device") != device: @@ -631,7 +608,7 @@ def build_runtime_metadata_from_kv_manager( cu_seqlens_q = torch.tensor(cu_q, dtype=torch.int32, device=device) q_batch_row = None q_positions = None - meta = MiniMaxM3SparseAttentionMetadata( + meta = MiniMaxM3TritonSparseAttentionMetadata( is_prefill=True, req_to_token=req_to_token, slot_ids=slot_ids, @@ -663,7 +640,7 @@ def build_runtime_metadata_from_kv_manager( out_cache_loc = out_cache_loc_buf[:batch] else: out_cache_loc = torch.tensor(out_cache_loc_list, dtype=torch.int32, device=device) - meta = MiniMaxM3SparseAttentionMetadata( + meta = MiniMaxM3TritonSparseAttentionMetadata( is_prefill=False, req_to_token=req_to_token, slot_ids=slot_ids, @@ -674,242 +651,222 @@ def build_runtime_metadata_from_kv_manager( return meta, out_cache_loc -@functools.lru_cache(maxsize=1) -def get_minimax_m3_attention_metadata_cls(): - """Return :class:`MiniMaxM3AttentionMetadata` (lazy import). +class MiniMaxM3AttentionMetadata(AttentionMetadata): + """:class:`AttentionMetadata` that pre-builds MiniMax-M3 metadata. + + Overrides :meth:`prepare` so the M3-sparse + :class:`MiniMaxM3TritonSparseAttentionMetadata` and the per-new-token + ``out_cache_loc`` are built **once per scheduler step**, on the + cache device, before the model forward runs. The result is + stored as ``self.minimax_m3 = {"metadata": m3_meta, + "out_cache_loc": out_cache_loc}`` so the model layer's + ``_dense_forward`` and ``_sparse_forward`` can read it without + any device migration. - The class extends :class:`AttentionMetadata` so the pyexecutor's - metadata-creation/prepare hooks (model_engine.py) drive M3 metadata - construction outside the CUDA-graph capture window. Building the - M3-sparse ``req_to_token`` / ``slot_ids`` / ``out_cache_loc`` - tensors during ``prepare()`` lands them on the GPU **before** the - forward call; the forward path then reads from the pre-built - attachment and performs no CPU->GPU copies, which is required for - CUDA-graph capture safety (``cudaErrorStreamCaptureUnsupported`` - fires for CPU->GPU ``memcpyAsync`` calls inside a captured stream). + Test paths that build their own metadata can short-circuit by + attaching ``attn_metadata.minimax_m3`` directly before calling + the forward; those paths do not go through :meth:`prepare`. """ - from ...interface import AttentionMetadata - - class MiniMaxM3AttentionMetadata(AttentionMetadata): - """:class:`AttentionMetadata` that pre-builds MiniMax-M3 metadata. - - Overrides :meth:`prepare` so the M3-sparse - :class:`MiniMaxM3SparseAttentionMetadata` and the per-new-token - ``out_cache_loc`` are built **once per scheduler step**, on the - cache device, before the model forward runs. The result is - stored as ``self.minimax_m3 = {"metadata": m3_meta, - "out_cache_loc": out_cache_loc}`` so the model layer's - ``_dense_forward`` and ``_sparse_forward`` can read it without - any device migration. - - Test paths that build their own metadata can short-circuit by - attaching ``attn_metadata.minimax_m3`` directly before calling - the forward; those paths do not go through :meth:`prepare`. + + minimax_m3: Optional[dict] = None + # Lazily allocated dict of persistent device buffers used to keep + # ``MiniMaxM3TritonSparseAttentionMetadata`` tensor addresses stable + # across CUDA-graph capture/replay. None until the first + # ``prepare()`` call decides to use them (``is_cuda_graph`` / + # graph-stable mode). + _m3_static_buffers: Optional[dict] = None + + def _maybe_get_m3_static_buffers( + self, cache_device: torch.device, kv_cache_manager + ) -> Optional[dict]: + """Return persistent M3 buffers when graph stability is + required. + + Allocates the persistent buffer dict the first time it is + needed and caches it on ``self._m3_static_buffers``. We + allocate the buffers under two conditions: + + * ``self.is_cuda_graph`` is True -- the captured graph + requires stable ``data_ptr()`` across replays; OR + * the previous prepare() already allocated buffers -- + we keep using them so the algorithm sees the same + addresses even between non-graph and graph-mode calls + (which can happen when the model engine alternates + between eager warmup and graph replay). + + Returns ``None`` when no static buffers should be used (e.g. + eager-only test paths that rely on per-call allocations). """ + need_static = ( + bool(getattr(self, "is_cuda_graph", False)) or self._m3_static_buffers is not None + ) + if not need_static: + return None + if self._m3_static_buffers is not None: + bufs = self._m3_static_buffers + if bufs.get("device") == cache_device: + return bufs + + # First-time use: return an empty placeholder dict. + # ``build_runtime_metadata_from_kv_manager`` performs the + # actual allocation lazily on the first call where the + # current scheduler step's geometry (max_kv_len from the + # manager's block-id table, total_q from extend_seq_lens, + # actual batch size after CUDA-graph padding) is known. + # That removes the need to predict the warmup geometry up + # front. The first allocation pins the buffer addresses + # for the rest of this metadata instance's lifetime, so all + # subsequent prepare() calls reuse the same ``data_ptr()``s + # and CUDA graph capture/replay stays valid. + placeholder: dict = { + "device": cache_device, + # Caller-provided hints used by the lazy allocator below + # when it sizes the persistent buffers on the first real + # prepare() call. + "max_num_sequences_hint": int( + getattr(self, "max_num_sequences", None) or self.max_num_requests + ), + "max_num_tokens_hint": int( + getattr(self, "max_num_tokens", None) + or (int(getattr(self, "max_num_sequences", None) or self.max_num_requests)) + ), + } + self._m3_static_buffers = placeholder + return placeholder - minimax_m3: Optional[dict] = None - # Lazily allocated dict of persistent device buffers used to keep - # ``MiniMaxM3SparseAttentionMetadata`` tensor addresses stable - # across CUDA-graph capture/replay. None until the first - # ``prepare()`` call decides to use them (``is_cuda_graph`` / - # graph-stable mode). - _m3_static_buffers: Optional[dict] = None - - def _maybe_get_m3_static_buffers( - self, cache_device: torch.device, kv_cache_manager - ) -> Optional[dict]: - """Return persistent M3 buffers when graph stability is - required. - - Allocates the persistent buffer dict the first time it is - needed and caches it on ``self._m3_static_buffers``. We - allocate the buffers under two conditions: - - * ``self.is_cuda_graph`` is True -- the captured graph - requires stable ``data_ptr()`` across replays; OR - * the previous prepare() already allocated buffers -- - we keep using them so the algorithm sees the same - addresses even between non-graph and graph-mode calls - (which can happen when the model engine alternates - between eager warmup and graph replay). - - Returns ``None`` when no static buffers should be used (e.g. - eager-only test paths that rely on per-call allocations). - """ - need_static = ( - bool(getattr(self, "is_cuda_graph", False)) or self._m3_static_buffers is not None + def prepare(self) -> None: + super().prepare() + + # Always rebuild the M3 metadata block on each prepare() + # call so it reflects the current scheduler step's seq_lens + # / request_ids / num_cached_tokens. Production + # ``model_engine`` invokes ``prepare()`` outside any CUDA + # graph capture window, so the (potentially expensive) build + # is safe to perform here. + # + # When CUDA graph is enabled the inner ``build_runtime_metadata_from_kv_manager`` + # call writes into the persistent ``_m3_static_buffers`` so + # the captured graph keeps reading from stable ``data_ptr()``s + # across replays. Without this the captured ``index_select`` + # over ``req_to_token``/``slot_ids`` reads from freed warmup + # memory and either produces wrong tokens or fires + # ``Indexing.cu:1515`` ``srcIndex < srcSelectDimSize``. + self.minimax_m3 = None + + # Production path: build the M3 metadata from the standard + # AttentionMetadata fields. Requires kv_cache_manager + the + # M3 sparse-cache contract. + kv_cache_manager = getattr(self, "kv_cache_manager", None) + if kv_cache_manager is None or not hasattr(kv_cache_manager, "get_index_k_buffer"): + # Not an M3 KV cache manager: nothing to build. The + # forward path will raise a clear error if the M3 + # backend ends up dispatched without the M3 cache. + return + request_ids = getattr(self, "request_ids", None) + seq_lens = self.seq_lens + if request_ids is None or seq_lens is None: + return + num_contexts = int(getattr(self, "num_contexts", 0) or 0) + batch_size = int(seq_lens.shape[0]) + if batch_size == 0: + return + + # The cache device hosts every paged buffer; this is the + # device the forward path consumes. + try: + layer_buf = kv_cache_manager.get_buffers(0) + cache_device = layer_buf.device + except Exception: + cache_device = torch.device(f"cuda:{torch.cuda.current_device()}") + + seq_lens_cpu = ( + getattr(self, "seq_lens_cpu", None) + if hasattr(self, "seq_lens_cpu") + else seq_lens.detach().to("cpu") + ) + if seq_lens_cpu is None: + seq_lens_cpu = seq_lens.detach().to("cpu") + + kv_cache_params = getattr(self, "kv_cache_params", None) + num_cached_per_seq = ( + kv_cache_params.num_cached_tokens_per_seq + if kv_cache_params is not None + else [0] * batch_size + ) + + # ``attn_metadata.seq_lens`` from the PyExecutor is the + # per-step new-token count. The M3 sparse-attention algorithm + # consumes a *cumulative* kv length: ``minimax_m3_sparse_*`` + # masks reads against ``metadata.seq_lens`` as the per-request + # K-side extent. Compute that cumulative kv length per request + # and feed it into the algorithm metadata builder. + kv_lens_cpu_list = [ + int(num_cached_per_seq[b]) + int(seq_lens_cpu[b].item()) for b in range(batch_size) + ] + kv_lens_cpu = torch.tensor(kv_lens_cpu_list, dtype=torch.int32) + kv_lens_dev = kv_lens_cpu.to(device=cache_device, non_blocking=True) + + static_buffers = self._maybe_get_m3_static_buffers(cache_device, kv_cache_manager) + + # Any batch containing a context (prefill or chunked extend) + # request takes the extend path. For prefill rows + # ``num_cached_per_seq`` is ``prefix_lens`` and the full new + # chunk is ``extend_seq_len``; for decode rows + # ``num_cached`` is ``kv_len - 1`` and ``extend_seq_len`` is + # 1, so the same builder produces the correct one-slot + # entry. Pure-decode batches (``num_contexts == 0``) still + # take the decode optimization for CUDA-graph warmup + # geometry. + # + # Mixed prefill+decode batches always take the extend path: + # the prefill kernel handles decode rows as 1-slot extends. + # The decode branch below is a pure-decode-only perf + # specialization. (iter-131 regression: previously a wrong + # predicate routed mixed batches into the decode branch and + # crashed in index_copy_.) + is_extend = num_contexts > 0 + if is_extend: + prefix_lens_list = [int(num_cached_per_seq[b]) for b in range(batch_size)] + extend_seq_lens_cpu = [ + kv_lens_cpu_list[b] - prefix_lens_list[b] for b in range(batch_size) + ] + prefix_lens = torch.tensor( + prefix_lens_list, + dtype=torch.int32, + device=cache_device, ) - if not need_static: - return None - if self._m3_static_buffers is not None: - bufs = self._m3_static_buffers - if bufs.get("device") == cache_device: - return bufs - - # First-time use: return an empty placeholder dict. - # ``build_runtime_metadata_from_kv_manager`` performs the - # actual allocation lazily on the first call where the - # current scheduler step's geometry (max_kv_len from the - # manager's block-id table, total_q from extend_seq_lens, - # actual batch size after CUDA-graph padding) is known. - # That removes the need to predict the warmup geometry up - # front. The first allocation pins the buffer addresses - # for the rest of this metadata instance's lifetime, so all - # subsequent prepare() calls reuse the same ``data_ptr()``s - # and CUDA graph capture/replay stays valid. - placeholder: dict = { - "device": cache_device, - # Caller-provided hints used by the lazy allocator below - # when it sizes the persistent buffers on the first real - # prepare() call. - "max_num_sequences_hint": int( - getattr(self, "max_num_sequences", None) or self.max_num_requests - ), - "max_num_tokens_hint": int( - getattr(self, "max_num_tokens", None) - or (int(getattr(self, "max_num_sequences", None) or self.max_num_requests)) - ), - } - self._m3_static_buffers = placeholder - return placeholder - - def prepare(self) -> None: - super().prepare() - - # Always rebuild the M3 metadata block on each prepare() - # call so it reflects the current scheduler step's seq_lens - # / request_ids / num_cached_tokens. Production - # ``model_engine`` invokes ``prepare()`` outside any CUDA - # graph capture window, so the (potentially expensive) build - # is safe to perform here. - # - # When CUDA graph is enabled the inner ``build_runtime_metadata_from_kv_manager`` - # call writes into the persistent ``_m3_static_buffers`` so - # the captured graph keeps reading from stable ``data_ptr()``s - # across replays. Without this the captured ``index_select`` - # over ``req_to_token``/``slot_ids`` reads from freed warmup - # memory and either produces wrong tokens or fires - # ``Indexing.cu:1515`` ``srcIndex < srcSelectDimSize``. - self.minimax_m3 = None - - # Production path: build the M3 metadata from the standard - # AttentionMetadata fields. Requires kv_cache_manager + the - # M3 sparse-cache contract. - kv_cache_manager = getattr(self, "kv_cache_manager", None) - if kv_cache_manager is None or not hasattr(kv_cache_manager, "get_index_k_buffer"): - # Not an M3 KV cache manager: nothing to build. The - # forward path will raise a clear error if the M3 - # backend ends up dispatched without the M3 cache. - return - request_ids = getattr(self, "request_ids", None) - seq_lens = self.seq_lens - if request_ids is None or seq_lens is None: - return - num_contexts = int(getattr(self, "num_contexts", 0) or 0) - batch_size = int(seq_lens.shape[0]) - if batch_size == 0: - return - - # The cache device hosts every paged buffer; this is the - # device the forward path consumes. - try: - layer_buf = kv_cache_manager.get_buffers(0) - cache_device = layer_buf.device - except Exception: - cache_device = torch.device(f"cuda:{torch.cuda.current_device()}") - - seq_lens_cpu = ( - getattr(self, "seq_lens_cpu", None) - if hasattr(self, "seq_lens_cpu") - else seq_lens.detach().to("cpu") + m3_meta, out_cache_loc = build_runtime_metadata_from_kv_manager( + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + seq_lens=kv_lens_dev, + seq_lens_cpu=kv_lens_cpu, + is_prefill=True, + prefix_lens=prefix_lens, + extend_seq_lens_cpu=extend_seq_lens_cpu, + device=cache_device, + static_buffers=static_buffers, ) - if seq_lens_cpu is None: - seq_lens_cpu = seq_lens.detach().to("cpu") - - kv_cache_params = getattr(self, "kv_cache_params", None) - num_cached_per_seq = ( - kv_cache_params.num_cached_tokens_per_seq - if kv_cache_params is not None - else [0] * batch_size + else: + m3_meta, out_cache_loc = build_runtime_metadata_from_kv_manager( + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + seq_lens=kv_lens_dev, + seq_lens_cpu=kv_lens_cpu, + is_prefill=False, + device=cache_device, + static_buffers=static_buffers, ) - # ``attn_metadata.seq_lens`` from the PyExecutor is the - # per-step new-token count. The M3 sparse-attention algorithm - # consumes a *cumulative* kv length: ``minimax_m3_sparse_*`` - # masks reads against ``metadata.seq_lens`` as the per-request - # K-side extent. Compute that cumulative kv length per request - # and feed it into the algorithm metadata builder. - kv_lens_cpu_list = [ - int(num_cached_per_seq[b]) + int(seq_lens_cpu[b].item()) for b in range(batch_size) - ] - kv_lens_cpu = torch.tensor(kv_lens_cpu_list, dtype=torch.int32) - kv_lens_dev = kv_lens_cpu.to(device=cache_device, non_blocking=True) - - static_buffers = self._maybe_get_m3_static_buffers(cache_device, kv_cache_manager) - - # Any batch containing a context (prefill or chunked extend) - # request takes the extend path. For prefill rows - # ``num_cached_per_seq`` is ``prefix_lens`` and the full new - # chunk is ``extend_seq_len``; for decode rows - # ``num_cached`` is ``kv_len - 1`` and ``extend_seq_len`` is - # 1, so the same builder produces the correct one-slot - # entry. Pure-decode batches (``num_contexts == 0``) still - # take the decode optimization for CUDA-graph warmup - # geometry. - # - # Mixed prefill+decode batches always take the extend path: - # the prefill kernel handles decode rows as 1-slot extends. - # The decode branch below is a pure-decode-only perf - # specialization. (iter-131 regression: previously a wrong - # predicate routed mixed batches into the decode branch and - # crashed in index_copy_.) - is_extend = num_contexts > 0 - if is_extend: - prefix_lens_list = [int(num_cached_per_seq[b]) for b in range(batch_size)] - extend_seq_lens_cpu = [ - kv_lens_cpu_list[b] - prefix_lens_list[b] for b in range(batch_size) - ] - prefix_lens = torch.tensor( - prefix_lens_list, - dtype=torch.int32, - device=cache_device, - ) - m3_meta, out_cache_loc = build_runtime_metadata_from_kv_manager( - kv_cache_manager=kv_cache_manager, - request_ids=request_ids, - seq_lens=kv_lens_dev, - seq_lens_cpu=kv_lens_cpu, - is_prefill=True, - prefix_lens=prefix_lens, - extend_seq_lens_cpu=extend_seq_lens_cpu, - device=cache_device, - static_buffers=static_buffers, - ) - else: - m3_meta, out_cache_loc = build_runtime_metadata_from_kv_manager( - kv_cache_manager=kv_cache_manager, - request_ids=request_ids, - seq_lens=kv_lens_dev, - seq_lens_cpu=kv_lens_cpu, - is_prefill=False, - device=cache_device, - static_buffers=static_buffers, - ) - - self.minimax_m3 = { - "metadata": m3_meta, - "out_cache_loc": out_cache_loc, - } - - return MiniMaxM3AttentionMetadata + self.minimax_m3 = { + "metadata": m3_meta, + "out_cache_loc": out_cache_loc, + } __all__ = [ - "MiniMaxM3SparseConfig", - "MiniMaxM3SparseAttentionMetadata", + "MiniMaxM3AttentionMetadata", + "MiniMaxM3TritonSparseAttentionMetadata", "allocate_minimax_m3_static_buffers", "build_runtime_metadata_from_kv_manager", "ensure_metadata_on_device", - "get_minimax_m3_attention_metadata_cls", - "replace_metadata", ] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/utils.py b/tensorrt_llm/_torch/attention_backend/sparse/utils.py index f438d998939f..1762268bf242 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/utils.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/utils.py @@ -42,14 +42,30 @@ def get_sparse_attn_kv_cache_manager( ) +def _resolve_minimax_m3_backend_cls( + sparse_params: "SparseParams") -> Type["AttentionBackend"]: + """Select the MiniMax-M3 sparse backend from the lowered params. + + The Triton reference is the default. When implementation is 'msa' the MSA + (fmha_sm100) backend is used instead, gated on SM100 availability so an + unsupported system fails early rather than at kernel launch. + """ + from .minimax_m3 import MiniMaxM3SparseRuntimeBackend + if getattr(sparse_params, "implementation", "triton") == "msa": + from .minimax_m3 import MiniMaxM3MsaSparseAttention + from .minimax_m3.msa_availability import ensure_msa_available + ensure_msa_available() + return MiniMaxM3MsaSparseAttention + return MiniMaxM3SparseRuntimeBackend + + def get_vanilla_sparse_attn_attention_backend( sparse_params: "SparseParams") -> Type["AttentionBackend"]: - from .minimax_m3 import get_minimax_m3_attention_backend_cls from .rocket import RocketVanillaAttention if sparse_params.algorithm == "rocket": return RocketVanillaAttention elif sparse_params.algorithm == "minimax_m3": - return get_minimax_m3_attention_backend_cls() + return _resolve_minimax_m3_backend_cls(sparse_params) else: raise ValueError( f"Unsupported sparse attention algorithm in vanilla attention backend: {sparse_params.algorithm}" @@ -62,7 +78,6 @@ def get_trtllm_sparse_attn_attention_backend( from .deepseek_v4 import DeepseekV4TrtllmAttention from .dsa import DSATrtllmAttention - from .minimax_m3 import get_minimax_m3_attention_backend_cls from .rocket import RocketTrtllmAttention if sparse_params.algorithm == "rocket": return RocketTrtllmAttention @@ -78,7 +93,7 @@ def get_trtllm_sparse_attn_attention_backend( # `create_attention(...)` dispatch in `Attention.__init__` # returns an instantiable AttentionBackend under the trtllm # attention backend slot. - return get_minimax_m3_attention_backend_cls() + return _resolve_minimax_m3_backend_cls(sparse_params) else: raise ValueError( f"Unsupported sparse attention algorithm in trtllm attention backend: {sparse_params.algorithm}" @@ -87,9 +102,8 @@ def get_trtllm_sparse_attn_attention_backend( def get_flashinfer_sparse_attn_attention_backend( sparse_params: "SparseParams") -> Type["AttentionBackend"]: - from .minimax_m3 import get_minimax_m3_attention_backend_cls if sparse_params.algorithm == "minimax_m3": - return get_minimax_m3_attention_backend_cls() + return _resolve_minimax_m3_backend_cls(sparse_params) raise ValueError( f"Unsupported sparse attention algorithm in flashinfer attention backend: {sparse_params.algorithm}" ) diff --git a/tensorrt_llm/_torch/attention_backend/trtllm.py b/tensorrt_llm/_torch/attention_backend/trtllm.py index 06ed33e63799..da0e005f1b4f 100644 --- a/tensorrt_llm/_torch/attention_backend/trtllm.py +++ b/tensorrt_llm/_torch/attention_backend/trtllm.py @@ -31,6 +31,7 @@ from tensorrt_llm._utils import get_sm_version, maybe_pin_memory, prefer_pinned from tensorrt_llm.bindings.internal import thop from tensorrt_llm.functional import AttentionMaskType +from tensorrt_llm.math_utils import ceil_div from tensorrt_llm.models.modeling_utils import QuantConfig from ..utils import (compute_swizzled_sf_shape, get_global_attrs, @@ -116,6 +117,7 @@ def effective_beam_width(self) -> int: is_spec_dec_tree: bool = False # if spec-dec tree wouldn't be changed at all, the mask won't be computed every step. is_spec_dec_dynamic_tree: bool = False + force_prepare_spec_dec_tree_mask: bool = False # parameters required for spec-dec mode max_total_draft_tokens: Optional[int] = None @@ -601,24 +603,47 @@ def prepare(self) -> None: # kv block offsets assert self.request_ids is not None if self.kv_cache_manager is not None: - self.kv_cache_manager.copy_batch_block_offsets( - self.kv_cache_block_offsets, self.request_ids, self.beam_width, - self.num_contexts, self.num_seqs) - - error_message = ( - f"The max KV cache length of input sequences ({self.kv_lens[:self.num_seqs].max()}) " + max_kv_len = int(self.kv_lens[:self.num_seqs].max()) + assert max_kv_len <= self.kv_cache_manager.max_seq_len, ( + f"The max KV cache length of input sequences ({max_kv_len}) " f"exceeds the KV cache manager's maximum supported length " f"({self.kv_cache_manager.max_seq_len}).") - assert self.kv_lens[:self.num_seqs].max( - ) <= self.kv_cache_manager.max_seq_len, error_message + # On the non-speculative path the host kv_lens snapshot bounds + # every block-table access, so the staged/H2D width can be capped + # at the batch's maximum instead of max_seq_len's worth of + # columns. Speculative decoding must stage the full width: + # draft/tree sub-steps and the overlap scheduler advance + # kv_lens_cuda on device past the host snapshot, and their + # kernels dereference block columns a host-derived cap would + # leave unstaged (uninitialized in this buffer). + spec_active = (self.draft_kv_cache_manager is not None + or self.is_spec_decoding_enabled + or bool(self.kv_cache_params.num_extra_kv_tokens) or + (self.runtime_features is not None and + self.runtime_features.has_speculative_draft_tokens)) + max_blocks = None + if not spec_active and self.kv_cache_manager.tokens_per_block: + max_blocks = ceil_div(max_kv_len, + self.kv_cache_manager.tokens_per_block) + self.kv_cache_manager.copy_batch_block_offsets( + self.kv_cache_block_offsets, + self.request_ids, + self.beam_width, + self.num_contexts, + self.num_seqs, + max_blocks=max_blocks) # Also prepare draft KV cache block offsets if draft_kv_cache_manager exists if self.draft_kv_cache_manager is not None: # Use the wrapper method which works for both V1 and V2 self.draft_kv_cache_manager.copy_batch_block_offsets( - self.draft_kv_cache_block_offsets, self.request_ids, - self.beam_width, self.num_contexts, self.num_seqs) + self.draft_kv_cache_block_offsets, + self.request_ids, + self.beam_width, + self.num_contexts, + self.num_seqs, + max_blocks=max_blocks) # Don't pass self.kv_lens as kv_lens here because it includes extra # tokens. Use the actual KV length (without extra tokens) for @@ -994,6 +1019,11 @@ def update_spec_dec_param( self.use_spec_decoding = self.is_spec_decoding_enabled self.is_spec_dec_tree = is_spec_dec_tree self.is_spec_dec_dynamic_tree = is_spec_dec_dynamic_tree + # A hybrid model's first executed attention layer can have a nonzero + # cache-local index because recurrent layers precede it. Do not rely + # on the C++ ``layer_idx == 0`` fallback to rebuild the target mask: + # the dynamic draft loop clears that mask before the next target step. + self.force_prepare_spec_dec_tree_mask = is_spec_dec_dynamic_tree # Forward static tree length to FMHA kernel selection. self.max_total_draft_tokens = max_total_draft_tokens @@ -1101,8 +1131,9 @@ def update_spec_dec_param( # Case 2/3: static tree elif self.is_spec_dec_tree and not self.is_spec_dec_dynamic_tree and spec_metadata is not None: - assert spec_metadata.spec_dec_mode.is_eagle3( - ), "Tree decoding is only supported for Eagle3 now" + assert (spec_metadata.spec_dec_mode.is_eagle3() + or spec_metadata.spec_dec_mode.is_eagle3_one_model() + ), "Tree decoding is only supported for Eagle3 now" is_target_model = not getattr(spec_metadata, 'is_draft_model', False) diff --git a/tensorrt_llm/_torch/auto_deploy/__init__.py b/tensorrt_llm/_torch/auto_deploy/__init__.py index a8a22a945c05..bbd1d6398854 100644 --- a/tensorrt_llm/_torch/auto_deploy/__init__.py +++ b/tensorrt_llm/_torch/auto_deploy/__init__.py @@ -37,9 +37,3 @@ # import AutoDeploy LLM and LlmArgs (require TRT-LLM base classes) from .llm import * from .llm_args import * - - try: - # This will overwrite the AutoModelForCausalLM.from_config to support modelopt quantization - import modelopt - except ImportError: - pass diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/quantization/quant.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/quantization/quant.py index f2ebb177f2a2..197741281124 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/quantization/quant.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/quantization/quant.py @@ -109,7 +109,8 @@ def _trtllm_fp8_prequant_linear_core( enable_cuda_core = False if torch.cuda.is_available(): capability = torch.cuda.get_device_capability(0) - enable_cuda_core = capability == (8, 9) or capability == (12, 0) + # enable cuda core for sm89, sm120, and sm121 + enable_cuda_core = capability in ((8, 9), (12, 0), (12, 1)) if x.shape[0] <= 8 and enable_cuda_core: output = torch.ops.trtllm.cuda_scaled_mm( diff --git a/tensorrt_llm/_torch/auto_deploy/export/export.py b/tensorrt_llm/_torch/auto_deploy/export/export.py index 09a54b3c46e8..c7f0a5de205a 100644 --- a/tensorrt_llm/_torch/auto_deploy/export/export.py +++ b/tensorrt_llm/_torch/auto_deploy/export/export.py @@ -32,10 +32,8 @@ from ..utils.pipeline_cache_hooks import mark_pipeline_cache_hook from .interface import apply_export_patches -try: - from modelopt.torch.quantization.utils import export_torch_mode as torch_export_context -except ImportError: - torch_export_context = nullcontext +# modelopt quantization support has been removed; use a null export context. +torch_export_context = nullcontext # ===================================================================== diff --git a/tensorrt_llm/_torch/auto_deploy/export/library/modelopt_context.py b/tensorrt_llm/_torch/auto_deploy/export/library/modelopt_context.py deleted file mode 100644 index 1a0df9c41119..000000000000 --- a/tensorrt_llm/_torch/auto_deploy/export/library/modelopt_context.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Patch for modelopt's torch_export_context.""" - -from contextlib import nullcontext - -from ..interface import ContextManagerPatch, ExportPatchRegistry - - -@ExportPatchRegistry.register("modelopt_context") -class ModeloptContextPatch(ContextManagerPatch): - """Patch to apply modelopt's torch_export_context during export. - - This patch applies the modelopt quantization context manager around - the export process when available, otherwise uses a null context. - """ - - def init_context_manager(self): - """Initialize and return the modelopt context manager or nullcontext if not available.""" - try: - from modelopt.torch.quantization.utils import export_torch_mode as torch_export_context - - return torch_export_context() - except ImportError: - return nullcontext() diff --git a/tensorrt_llm/_torch/auto_deploy/shim/demollm.py b/tensorrt_llm/_torch/auto_deploy/shim/demollm.py index 104ca0c7b266..0b3dbc1ca392 100644 --- a/tensorrt_llm/_torch/auto_deploy/shim/demollm.py +++ b/tensorrt_llm/_torch/auto_deploy/shim/demollm.py @@ -22,7 +22,7 @@ import torch import torch.multiprocessing as mp -from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import ( +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( greedy_search_sampling_batch, top_k_top_p_sampling_batch, ) diff --git a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py index cace2f37e603..198cdab433f3 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py @@ -28,19 +28,9 @@ from .logger import ad_logger -try: - # import modelopt to get quantize_op - from modelopt.torch.quantization import tensor_quant # noqa: F401 - - if hasattr(torch.ops, "tensorrt"): - modelopt_quantize_op = torch.ops.tensorrt.quantize_op - modelopt_dynamic_block_quantize_op = torch.ops.tensorrt.dynamic_block_quantize_op - else: - modelopt_quantize_op = None - modelopt_dynamic_block_quantize_op = None -except ImportError: - modelopt_quantize_op = None - modelopt_dynamic_block_quantize_op = None +# modelopt quantization support has been removed; these ops are no longer available. +modelopt_quantize_op = None +modelopt_dynamic_block_quantize_op = None OpOrOverload = Union[OpOverloadPacket, OpOverload] OperatorLike = Union[OpOrOverload, Callable] diff --git a/tensorrt_llm/_torch/autotuner.py b/tensorrt_llm/_torch/autotuner.py index 9ef81873b510..934487fce8dd 100644 --- a/tensorrt_llm/_torch/autotuner.py +++ b/tensorrt_llm/_torch/autotuner.py @@ -261,7 +261,8 @@ def unique_id(self): @contextlib.contextmanager def autotune(tune_mode: bool = True, cache_path: str = None, - skip_dynamic_tuning_buckets: bool = False): + skip_dynamic_tuning_buckets: bool = False, + post_tune_merge_dist: Optional[Distributed] = None): """Context manager for autotuning with distributed support. Args: @@ -271,8 +272,21 @@ def autotune(tune_mode: bool = True, _optimization_profiles() so only actual input shapes from warmup are profiled. Useful for workloads (e.g. diffusion) where the LLM-oriented M-bucket sweep is unnecessary. + post_tune_merge_dist: Optional ``Distributed``. When set, tactics are + tuned per-rank independently and merged across ranks in one + collective at context exit (before the cache is saved) — for + pipelines whose warmup is not SPMD and so cannot use the per-op + distributed strategies. """ autotuner = AutoTuner.get() + if post_tune_merge_dist is not None: + # Save the singleton's prior state: the full-world dist attached for the + # merge is temporary and must not leak into later sessions. Attach the + # real (world-sized) mapping now for a correct rank, but with no dist + # yet: _is_distributed() stays False so per-op sync is skipped while a + # non-SPMD pipeline tunes; the dist is attached at exit to merge. + prev_mapping, prev_dist = autotuner.mapping, autotuner._dist + autotuner.setup_distributed_state(post_tune_merge_dist.mapping, None) rank = autotuner.mapping.rank # if cache_path is provided, use the rank-specific file @@ -303,10 +317,24 @@ def autotune(tune_mode: bool = True, if autotune_enabled: logger.info("[Autotuner] Autotuning process ends") - # save cache - if cache_path is not None: - logger.info(f"[Autotuner] Saving cache to {cache_path}") - autotuner.profiling_cache.save_cache(cache_path, rank) + try: + # Merge tactics across ranks in one collective before the cache is + # saved, so non-SPMD pipelines can tune independently and still agree. + if post_tune_merge_dist is not None: + autotuner.setup_distributed_state(post_tune_merge_dist.mapping, + post_tune_merge_dist) + autotuner.post_tune_merge_tactics() + + # save cache + if cache_path is not None: + logger.info(f"[Autotuner] Saving cache to {cache_path}") + autotuner.profiling_cache.save_cache(cache_path, rank) + finally: + # Restore the singleton's prior distributed state (see enter) so the + # temporary full-world state does not persist past this context. + if post_tune_merge_dist is not None: + autotuner.mapping = prev_mapping + autotuner._dist = prev_dist @dataclass @@ -760,9 +788,17 @@ def _serialize_cache_data(self, # Convert any simple object to string for JSON compatibility key_str = str(key) runner_id, tactic, min_time = value - tactic_str = repr(tactic) + # Enum tactics (e.g. Fp4QuantTactic) repr as "", + # which ast.literal_eval can't parse back -> load_cache crash. Serialize + # the underlying value (reload yields that value; an IntEnum compares + # equal to it and kernels take the int). + is_enum = isinstance(tactic, enum.Enum) + tactic_check = tactic.value if is_enum else tactic + tactic_str = repr(tactic_check) try: - assert tactic == ast.literal_eval( + # Verify the serialized value round-trips (compare against the + # value we store, not the enum member — else non-IntEnum fails). + assert tactic_check == ast.literal_eval( tactic_str ), f"Tactic is not compatible with json.dumps/json.loads" except Exception as e: @@ -804,10 +840,16 @@ def _deserialize_cache_data( continue try: tactic = ast.literal_eval(value["tactic"]) - except (ValueError, TypeError): + except (ValueError, TypeError, SyntaxError): + # Skip any tactic whose repr is not a Python literal so one bad + # entry can't crash the whole load; the op just re-profiles. + # (#16782 removes the current enum trigger at the serialize side.) + # SyntaxError is caught too; `continue` is required, else the + # entry reuses the previous tactic. logger.warning_once( - f"[AutoTuner] Could not deserialize tactic: {value['tactic']} for cache key {key_str}", + f"[AutoTuner] Could not deserialize tactic: {value['tactic']} for cache key {key_str}; skipping entry.", key=value["tactic"]) + continue runner_id = value["runner_id"] min_time = value["min_time"] @@ -1954,6 +1996,19 @@ def _merge_cache_data(self, custom_op: str): self.profiling_cache.merge_cache_data(merged_cache_data) + def post_tune_merge_tactics(self) -> None: + """Merge tactics across ranks after tuning: all-gather every rank's whole + profiling cache and keep the fastest tactic per key. One-shot + (session-level, not per-op). Collective — all ranks must call it.""" + if not self._is_distributed(): + return + merged = dict() + for data in self._dist.tp_cp_allgather(obj=self.profiling_cache.cache): + for key, value in data.items(): + if value[-1] < merged.get(key, [float('inf')])[-1]: + merged[key] = value + self.profiling_cache.merge_cache_data(merged) + def _broadcast_cache_data( self, custom_op: str, diff --git a/tensorrt_llm/_torch/compilation/remove_copy_pass.py b/tensorrt_llm/_torch/compilation/remove_copy_pass.py index 8e5eb7a81148..f9bb925a1e1c 100644 --- a/tensorrt_llm/_torch/compilation/remove_copy_pass.py +++ b/tensorrt_llm/_torch/compilation/remove_copy_pass.py @@ -1,3 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from operator import getitem import torch @@ -23,13 +37,49 @@ def remove_functionalize_inner(node: Node, mutates_args: dict, is_v2=False): ] kwargs = {k: v for k, v in node.kwargs.items() if not k.startswith("_")} + tensor_list_replacements = {} if is_v2: - for k, v in mutates_args.items(): - kwargs[v] = node.kwargs["_all_bases"][k - 1] + all_bases = node.kwargs["_all_bases"] + # Mutated bases are flattened into consecutive outputs after the + # operator's regular outputs. + first_mutated_output = min(mutates_args) + for arg in inplace_func._schema.arguments: + if arg.alias_info is None or not arg.alias_info.is_write: + continue + length_key = f"_{arg.name}_length" + if length_key in node.kwargs: + length = node.kwargs[length_key] + if length is None: + kwargs[arg.name] = None + continue + + kwargs[arg.name] = [] + for index in range(length): + base_index = node.kwargs[ + f"_{arg.name}_{index}_base_index"] + base = (None if base_index is None else + all_bases[base_index]) + kwargs[arg.name].append(base) + if base_index is not None: + tensor_list_replacements[first_mutated_output + + base_index] = (arg.name, + base) + else: + base_index = node.kwargs[f"_{arg.name}_base_index"] + kwargs[arg.name] = (None if base_index is None else + all_bases[base_index]) for getitem_node in getitem_nodes: idx = getitem_node.args[1] - getitem_node.replace_all_uses_with(kwargs[mutates_args[idx]]) + if idx in tensor_list_replacements: + mutated_arg, replacement = tensor_list_replacements[idx] + else: + mutated_arg = mutates_args[idx] + replacement = kwargs[mutated_arg] + assert replacement is not None, ( + f"getitem user for optional output '{mutated_arg}' " + "has no base tensor -- graph is malformed") + getitem_node.replace_all_uses_with(replacement) nodes_to_remove.append(getitem_node) with graph.inserting_before(node): @@ -51,7 +101,11 @@ def remove_functionalize_inner(node: Node, mutates_args: dict, is_v2=False): # We do not know the inplace op continue - remove_functionalize_inner(node, inplace_map[inplace_func]) + remove_functionalize_inner( + node, + inplace_map[inplace_func], + is_v2=node.target == auto_functionalized_v2, + ) for node in nodes_to_remove: graph.erase_node(node) diff --git a/tensorrt_llm/_torch/configs/__init__.py b/tensorrt_llm/_torch/configs/__init__.py index c7ad6b2cb856..82bcf5209e2e 100644 --- a/tensorrt_llm/_torch/configs/__init__.py +++ b/tensorrt_llm/_torch/configs/__init__.py @@ -9,6 +9,7 @@ ) from tensorrt_llm._torch.configs.kimi_linear import KimiLinearConfig from tensorrt_llm._torch.configs.laguna import LagunaConfig +from tensorrt_llm._torch.configs.minicpmv4_6 import MiniCPMV4_6Config, MiniCPMV4_6VisionConfig def _register_custom_configs_with_transformers() -> None: @@ -44,6 +45,10 @@ def _register_custom_configs_with_transformers() -> None: # "kimi_linear" without trust_remote_code. "kimi_linear": KimiLinearConfig, "laguna": LagunaConfig, + # minicpmv4_6 is only registered in transformers>=5.7.0; register our + # own composite config so AutoTokenizer.from_pretrained works on older + # releases (the model itself is built via load_pretrained_config). + "minicpmv4_6": MiniCPMV4_6Config, "gemma4_unified": Gemma4UnifiedConfig, "gemma4_unified_text": Gemma4UnifiedTextConfig, "gemma4_unified_vision": Gemma4UnifiedVisionConfig, @@ -72,4 +77,6 @@ def _register_custom_configs_with_transformers() -> None: "Gemma4UnifiedVisionConfig", "KimiLinearConfig", "LagunaConfig", + "MiniCPMV4_6Config", + "MiniCPMV4_6VisionConfig", ] diff --git a/tensorrt_llm/_torch/configs/minicpmv4_6.py b/tensorrt_llm/_torch/configs/minicpmv4_6.py new file mode 100644 index 000000000000..1913dd8d624f --- /dev/null +++ b/tensorrt_llm/_torch/configs/minicpmv4_6.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Self-contained config classes for MiniCPM-V 4.6. + +The composite ``minicpmv4_6`` HF config is only registered in +``transformers>=5.7.0``. TRT-LLM must load the model on older transformers +releases too, so we ship lightweight ``PretrainedConfig`` subclasses that mirror +the HF fields we consume, and build them from ``config.json`` inside +``pyexecutor.config_utils.load_pretrained_config`` (bypassing ``AutoConfig``). + +The inner text config is normalized separately into a ``Qwen3NextConfig`` (the +runtime model that backs the Qwen3.5 dense text tower) via the shared Qwen3.5 +compatibility shim; it is passed in already constructed. + +.. note:: + This is a transitional import/registration-safety shim, **not** a way to + avoid the ``transformers>=5.7.0`` runtime requirement: actually running the + model still needs the native ``MiniCPMV4_6Processor`` for image/video + preprocessing (see ``_ensure_transformers_supports_minicpmv4_6`` in + ``modeling_minicpmv4_6``). Its only job is to keep ``import tensorrt_llm``, + ``AutoConfig``/``AutoTokenizer`` and CI unit-test collection working on the + repo's currently-pinned transformers 5.5.4. + + TODO: remove this module once the repo pins ``transformers>=5.7.0`` and + switch ``_build_minicpmv4_6_config`` to the native + ``transformers.MiniCPMV4_6Config``. +""" + +from transformers import PretrainedConfig + +__all__ = ["MiniCPMV4_6Config", "MiniCPMV4_6VisionConfig"] + + +class MiniCPMV4_6VisionConfig(PretrainedConfig): + """SigLIP2-style variable-resolution ViT config for MiniCPM-V 4.6.""" + + model_type = "minicpmv4_6_vision" + base_config_key = "vision_config" + + def __init__( + self, + hidden_size: int = 1152, + intermediate_size: int = 4304, + num_hidden_layers: int = 27, + num_attention_heads: int = 16, + num_channels: int = 3, + image_size: int = 980, + patch_size: int = 14, + hidden_act: str = "gelu_pytorch_tanh", + layer_norm_eps: float = 1e-6, + attention_dropout: float = 0.0, + insert_layer_id: int = 6, + window_kernel_size=(2, 2), + **kwargs, + ): + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.image_size = image_size + self.patch_size = patch_size + self.hidden_act = hidden_act + self.layer_norm_eps = layer_norm_eps + self.attention_dropout = attention_dropout + self.insert_layer_id = insert_layer_id + self.window_kernel_size = tuple(window_kernel_size) + super().__init__(**kwargs) + + @property + def window_hidden_size(self) -> int: + return self.hidden_size * self.window_kernel_size[0] * self.window_kernel_size[1] + + @property + def window_intermediate_size(self) -> int: + return self.intermediate_size * self.window_kernel_size[0] * self.window_kernel_size[1] + + +class MiniCPMV4_6Config(PretrainedConfig): + """Composite config: SigLIP2 vision tower + Qwen3.5 dense text tower.""" + + model_type = "minicpmv4_6" + sub_configs = {"vision_config": MiniCPMV4_6VisionConfig} + + def __init__( + self, + text_config=None, + vision_config=None, + insert_layer_id: int = 6, + image_size: int = 448, + drop_vision_last_layer: bool = False, + image_token_id=None, + video_token_id=None, + downsample_mode: str = "16x", + merge_kernel_size=(2, 2), + merger_times: int = 1, + tie_word_embeddings: bool = False, + **kwargs, + ): + if isinstance(vision_config, dict): + vision_config = dict(vision_config) + vision_config.pop("model_type", None) + vision_config = MiniCPMV4_6VisionConfig(**vision_config) + elif vision_config is None: + vision_config = MiniCPMV4_6VisionConfig() + self.vision_config = vision_config + self.vision_config.insert_layer_id = insert_layer_id + + # The loader passes an already-constructed Qwen3NextConfig. When + # AutoConfig builds this class from a raw dict (e.g. AutoTokenizer), + # keep the dict as-is; only the loader path needs a real text config. + self.text_config = text_config + + self.insert_layer_id = insert_layer_id + self.image_size = image_size + self.drop_vision_last_layer = drop_vision_last_layer + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.downsample_mode = downsample_mode + self.merge_kernel_size = tuple(merge_kernel_size) + self.merger_times = merger_times + self.patch_size = self.vision_config.patch_size + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) diff --git a/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py b/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py index d7d240f337f8..9ca0b3fca484 100644 --- a/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py @@ -761,6 +761,15 @@ def _(input: torch.Tensor): dtype=torch.float8_e4m3fn), input.new_empty( (m, num_packed_sf_k), dtype=torch.int32) + @torch.library.register_fake("trtllm::fp8_quantize_1x128_cutedsl_ue8m0") + def _(input: torch.Tensor): + m, k = input.shape + padded_m = fp4_utils.pad_up(m, 128) + sf_cols = fp4_utils.pad_up(k // 32, 4) + return torch.empty_like(input, + dtype=torch.float8_e4m3fn), input.new_empty( + (padded_m * sf_cols, ), dtype=torch.uint8) + @torch.library.register_fake("trtllm::causal_conv1d_fwd") def _( x: torch.Tensor, @@ -1398,6 +1407,40 @@ def _( sf_out = x.new_empty((sf_size, ), dtype=torch.uint8) return y_fp4, sf_out + @torch.library.register_fake("trtllm::fused_adaptive_layernorm") + def _( + x: torch.Tensor, + ln_weight: Optional[torch.Tensor], + ln_bias: Optional[torch.Tensor], + scale_msa: Optional[torch.Tensor], + shift_msa: Optional[torch.Tensor], + seq_len_per_batch: int, + eps: float, + ) -> torch.Tensor: + return x.new_empty(x.shape, dtype=torch.bfloat16) + + @torch.library.register_fake("trtllm::fused_adaptive_layernorm_quant") + def _( + x: torch.Tensor, + ln_weight: Optional[torch.Tensor], + ln_bias: Optional[torch.Tensor], + scale_msa: Optional[torch.Tensor], + shift_msa: Optional[torch.Tensor], + sf_scale: torch.Tensor, + seq_len_per_batch: int, + eps: float, + ) -> Tuple[torch.Tensor, torch.Tensor]: + D = x.shape[-1] + M = 1 + for d in x.shape[:-1]: + M *= d + y_fp4 = x.new_empty((M, D // 2), dtype=torch.uint8) + _, scale_shape = fp4_utils.get_fp4_shape((M, D), + 16, + is_swizzled_layout=True) + sf_out = x.new_empty((scale_shape, ), dtype=torch.uint8) + return y_fp4, sf_out + @torch.library.register_fake("trtllm::fused_relu2_quantize") def _( input: torch.Tensor, diff --git a/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py b/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py index 0f674cb20e67..e53d58693a4b 100644 --- a/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py @@ -344,6 +344,8 @@ def get_dense_gemm_approximate_cta_nums( Sm100BlockScaledPersistentDenseGemmKernel as DenseGemmSwigluKernel from ..cute_dsl_kernels.blackwell.top_k.filtered_top_k_decode_varlen import \ FilteredTopKKernelVarlenDecode + from ..cute_dsl_kernels.blackwell.top_k.filtered_top_k_prefill_varlen import \ + FilteredTopKKernelVarlenPrefill from ..cute_dsl_kernels.blackwell.top_k.single_pass_multi_cta_radix_topk import \ STATE_SIZE as DISTRIBUTED_TOPK_STATE_SIZE from ..cute_dsl_kernels.blackwell.top_k.single_pass_multi_cta_radix_topk import \ @@ -3636,6 +3638,331 @@ def _fake_single_b( device=input_scale.device) return output, output_scale + _INDEXER_Q_CUTEDSL_TUNING_BUCKETS = ( + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1024, + 2048, + 4096, + 8192, + 16384, + ) + _INDEXER_Q_POSITION_IDS_INPUT_INDEX = 3 + + def _map_cutedsl_indexer_q_tuning_bucket(num_tokens: int) -> int: + if num_tokens <= 4: + return 4 + if num_tokens <= 8: + return 8 + if num_tokens <= 16: + return 16 + return max(32, last_positive_power_of_2(num_tokens)) + + def _prepare_cutedsl_indexer_q_tuning_inputs( + inputs: List[torch.Tensor]) -> List[torch.Tensor]: + # The autotuner resizes position_ids to the token bucket, leaving newly + # allocated values uninitialized. Use position zero for every tuning + # row so the fused RoPE lookup always stays inside cos_sin_cache. + position_ids = inputs[_INDEXER_Q_POSITION_IDS_INPUT_INDEX] + inputs[_INDEXER_Q_POSITION_IDS_INPUT_INDEX] = torch.zeros_like( + position_ids) + return inputs + + class CuteDSLIndexerQBlackwellRunner(TunableRunner): + """Native MXF8 GEMM with fused DSv4 indexer-Q RoPE/MXFP4 output.""" + + kernel_class = Sm100BlockScaledPersistentDenseGemmActFusionKernel + small_m_kernel_class = Sm100BlockScaledPersistentDenseGemmKernel + kernel_cache = dict() + tuning_config = TuningConfig( + dynamic_tensor_specs=(DynamicTensorSpec( + 0, + 0, + _INDEXER_Q_CUTEDSL_TUNING_BUCKETS, + _map_cutedsl_indexer_q_tuning_bucket, + ), ), + constraint_specs=(ConstraintSpec(3, 0, + lambda shapes: shapes[0][0]), ), + inputs_pre_hook=_prepare_cutedsl_indexer_q_tuning_inputs, + use_cold_l2_cache=True, + # CuTe kernels are JIT compiled into a process-local cache while + # the autotuner profiles tactics. Never persist only the selected + # tactic: a new process would otherwise skip that compilation and + # pay for cute.compile in its first inference forward. + exclude_from_cache=True, + # Every rank owns a process-local CuTe module cache. Profiling in + # parallel across ranks would leave the winning tactic uncompiled + # on ranks that benchmarked a different subset. + distributed_tuning_strategy=DistributedTuningStrategy.INDEPENDENT, + ) + + _small_m_tactics = ( + ("swap_ab", (128, 16), (1, 1), False, 4), + ("swap_ab", (128, 16), (1, 1), False, 8), + ) + _native_tactics = ( + ("native", (128, 128), (1, 1), False, 0), + ("native", (128, 128), (1, 2), False, 0), + ("native", (128, 128), (2, 1), False, 0), + ("native", (128, 128), (2, 1), True, 0), + ("native", (256, 128), (2, 1), False, 0), + ("native", (256, 128), (2, 1), True, 0), + ("native", (256, 128), (2, 2), True, 0), + ) + + def __init__(self, use_tvm_ffi: bool = True): + super().__init__() + self.use_tvm_ffi = use_tvm_ffi + + def unique_id(self): + return (self.use_tvm_ffi, ) + + def get_valid_tactics( + self, + inputs: List[torch.Tensor], + profile: OptimizationProfile, + **kwargs, + ) -> List[Tuple]: + if not is_sm_100f(): + return [] + m, k = inputs[0].shape + n = inputs[1].shape[0] + tactics = [] + if self._small_m_kernel_is_supported(m, n, k): + tactics.extend(self.__class__._small_m_tactics) + + tactics.extend([ + tactic for tactic in self.__class__._native_tactics + if self.__class__.kernel_class.can_implement( + cutlass.Float8E4M3FN, + cutlass.Float8E8M0FNU, + 32, + cutlass.Float4E2M1FN, + tactic[1], + tactic[2], + m, + n, + k, + 1, + "k", + "k", + "n", + ) + ]) + return tactics + + @staticmethod + def _small_m_kernel_is_supported(m: int, n: int, k: int) -> bool: + return 0 < m <= 16 and n % 128 == 0 and k % 128 == 0 + + @classmethod + def _fallback_tactic(cls, m: int, n: int, k: int) -> Tuple: + """Safe eager-mode fallback when TRT-LLM autotuning is disabled.""" + if cls._small_m_kernel_is_supported(m, n, k): + if m <= 4: + return ("swap_ab", (128, 16), (1, 1), False, 4) + if m <= 8: + return ("swap_ab", (128, 16), (1, 1), False, 8) + return ("native", (256, 128), (2, 1), False, 0) + + @staticmethod + def _ptr(tensor: torch.Tensor, dtype, align: int = 16): + return make_ptr( + dtype, + tensor.data_ptr(), + cute.AddressSpace.gmem, + assumed_align=align, + ) + + def forward( + self, + inputs: List[torch.Tensor], + tactic, + ) -> Tuple[torch.Tensor, torch.Tensor]: + input, weight, weight_scale, position_ids, cos_sin_cache, alpha = inputs + m, k = input.shape + n = weight.shape[0] + if tactic == -1: + tactic = self._fallback_tactic(m, n, k) + (kernel_kind, mma_tiler_mn, cluster_shape_mn, use_prefetch, + transform_warps) = tactic + if kernel_kind == "swap_ab": + if not 0 < m <= mma_tiler_mn[1]: + raise ValueError( + "The small-M indexer-Q kernel requires one non-empty " + f"token tile: M={m}, tile N={mma_tiler_mn[1]}") + if n % 128 != 0 or k % 128 != 0: + raise ValueError( + "The small-M indexer-Q kernel requires N and K to be " + f"divisible by 128, but got N={n}, K={k}") + + a, a_sf = torch.ops.trtllm.fp8_quantize_1x128_cutedsl_ue8m0(input) + packed = torch.empty((m, n // 2), + dtype=torch.uint8, + device=input.device) + output_scale = torch.empty((m, n // 32), + dtype=torch.uint8, + device=input.device) + + a_ptr = self._ptr(a, cutlass.Float8E4M3FN) + b_ptr = self._ptr(weight, cutlass.Float8E4M3FN) + a_sf_ptr = self._ptr(a_sf, cutlass.Float8E8M0FNU) + b_sf_ptr = self._ptr(weight_scale, cutlass.Float8E8M0FNU) + packed_ptr = self._ptr(packed, cutlass.Uint8) + output_scale_ptr = self._ptr(output_scale, cutlass.Float8E8M0FNU) + position_ids_ptr = self._ptr(position_ids, cutlass.Int32, 4) + cos_sin_ptr = self._ptr(cos_sin_cache, cutlass.Float32, 32) + alpha_cute = cute.runtime.from_dlpack(alpha) + + if self.use_tvm_ffi: + stream = cute.runtime.make_fake_stream( + use_tvm_ffi_env_stream=True) + else: + stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + + cache_key = (kernel_kind, mma_tiler_mn, cluster_shape_mn, + use_prefetch, transform_warps, self.use_tvm_ffi) + if cache_key not in self.__class__.kernel_cache: + if kernel_kind == "swap_ab": + gemm = self.__class__.small_m_kernel_class( + 32, + mma_tiler_mn, + cluster_shape_mn, + use_prefetch=use_prefetch, + indexer_q_fusion=True, + indexer_transform_warps=transform_warps, + ) + compile_entry = gemm.wrapper_indexer_q_swap_ab + else: + gemm = self.__class__.kernel_class( + 32, + mma_tiler_mn, + cluster_shape_mn, + True, + use_prefetch, + activation_type=ActivationType.Identity, + indexer_q_fusion=True, + ) + compile_entry = gemm.wrapper_indexer_q + hardware_info = cutlass.utils.HardwareInfo() + max_active_clusters = hardware_info.get_max_active_clusters( + cluster_shape_mn[0] * cluster_shape_mn[1]) + compiled = cute.compile( + compile_entry, + m, + n, + k, + pad_up(m, 128) // 128, + pad_up(n, 128) // 128, + pad_up(k // 32, 4) // 4, + cos_sin_cache.shape[0], + 1, + a_ptr, + b_ptr, + a_sf_ptr, + b_sf_ptr, + packed_ptr, + output_scale_ptr, + position_ids_ptr, + cos_sin_ptr, + alpha_cute, + max_active_clusters, + stream, + options="--opt-level 2 --enable-tvm-ffi" + if self.use_tvm_ffi else "--opt-level 2", + ) + self.__class__.kernel_cache[cache_key] = compiled + else: + compiled = self.__class__.kernel_cache[cache_key] + + dynamic_args = [ + m, + n, + k, + pad_up(m, 128) // 128, + pad_up(n, 128) // 128, + pad_up(k // 32, 4) // 4, + cos_sin_cache.shape[0], + ] + if self.use_tvm_ffi: + compiled( + *dynamic_args, + a.data_ptr(), + weight.data_ptr(), + a_sf.data_ptr(), + weight_scale.data_ptr(), + packed.data_ptr(), + output_scale.data_ptr(), + position_ids.data_ptr(), + cos_sin_cache.data_ptr(), + alpha, + ) + else: + compiled( + *dynamic_args, + a_ptr, + b_ptr, + a_sf_ptr, + b_sf_ptr, + packed_ptr, + output_scale_ptr, + position_ids_ptr, + cos_sin_ptr, + alpha_cute, + stream, + ) + return packed.view(torch.int8), output_scale.view(torch.int32) + + @torch.library.custom_op( + "trtllm::cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell", + mutates_args=(), + device_types="cuda", + ) + def cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + position_ids: torch.Tensor, + cos_sin_cache: torch.Tensor, + alpha: torch.Tensor, + use_tvm_ffi: bool = True, + ) -> Tuple[torch.Tensor, torch.Tensor]: + runner = CuteDSLIndexerQBlackwellRunner(use_tvm_ffi) + inputs = [ + input, weight, weight_scale, position_ids, cos_sin_cache, alpha + ] + tuner = AutoTuner.get() + _, tactic = tuner.choose_one( + "trtllm::cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell", + [runner], + runner.__class__.tuning_config, + inputs, + ) + return runner(inputs, tactic=tactic) + + @torch.library.register_fake( + "trtllm::cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell") + def _( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + position_ids: torch.Tensor, + cos_sin_cache: torch.Tensor, + alpha: torch.Tensor, + use_tvm_ffi: bool = True, + ): + m, n = input.shape[0], weight.shape[0] + return ( + input.new_empty((m, n // 2), dtype=torch.int8), + input.new_empty((m, n // 128), dtype=torch.int32), + ) + class CuteDSLFp8BlackwellRunner(TunableRunner): kernel_class = Sm100BlockwiseGemmKernel kernel_cache = dict() @@ -5052,7 +5379,7 @@ class CuteDSLTopKDecodeSingleCTARunner: Attributes: kernel_cache: Class-level dict mapping configuration tuples to compiled kernels. Keys are (dtype, num_cols, top_k, next_n, return_val, num_copy_bits, - load_balance, large_occupancy). + large_occupancy, overflow_policy). Note: - Requires Blackwell architecture (SM100+) @@ -5064,8 +5391,16 @@ class CuteDSLTopKDecodeSingleCTARunner: buffers = get_memory_buffers() @classmethod - def _compile(cls, dtype, bucketed_num_cols, top_k, next_n, return_val, - num_copy_bits, load_balance, large_occupancy): + def _compile(cls, + dtype, + bucketed_num_cols, + top_k, + next_n, + return_val, + num_copy_bits, + large_occupancy, + overflow_policy, + cache_smem_values=False): """Compile and cache a single-CTA top-k kernel for the given config.""" key = ( dtype, @@ -5074,8 +5409,9 @@ def _compile(cls, dtype, bucketed_num_cols, top_k, next_n, return_val, next_n, return_val, num_copy_bits, - load_balance, large_occupancy, + overflow_policy, + cache_smem_values, ) if key in cls.kernel_cache: return @@ -5087,12 +5423,15 @@ def _compile(cls, dtype, bucketed_num_cols, top_k, next_n, return_val, stride_order=(1, 0), assumed_align=32) - buffer_fake = cute.runtime.make_fake_compact_tensor( - cutlass.Int32, - (n_rows, cute.sym_int(), n_cols), - stride_order=(2, 1, 0), - assumed_align=32, - ) + if overflow_policy == "GMEM_SPILL": + buffer_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_rows, cute.sym_int(), n_cols), + stride_order=(2, 1, 0), + assumed_align=32, + ) + else: + buffer_fake = None seqlen_fake = cute.runtime.make_fake_compact_tensor( cutlass.Int32, (n_batch, ), @@ -5122,24 +5461,18 @@ def _compile(cls, dtype, bucketed_num_cols, top_k, next_n, return_val, num_copy_bits=num_copy_bits, return_val=return_val, large_occupancy=large_occupancy, - num_sms=_get_num_sms(), + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, ) - if load_balance: - g_global_counter_fake = cute.runtime.make_fake_compact_tensor( - cutlass.Int32, (1, ), stride_order=(0, )) - else: - g_global_counter_fake = None compiled_kernel = cute.compile( filtered_topk_func, input_fake, None, # indices_fake buffer_fake, - g_global_counter_fake, seqlen_fake, output_indices_fake, output_values_fake, stream=fake_stream, - enable_persistent_dynamic_scheduling=load_balance, min_blocks_per_mp=4 if large_occupancy else 1, options="--enable-tvm-ffi", ) @@ -5154,8 +5487,9 @@ def forward( next_n: int, return_val: bool = False, num_copy_bits: int = 256, - load_balance: bool = False, + overflow_policy: str = "REREAD", output_indices: Optional[torch.Tensor] = None, + cache_smem_values: bool = False, ): """Execute filtered top-k selection on input logits.""" torch_dtype = input_values.dtype @@ -5173,8 +5507,9 @@ def forward( next_n, return_val, num_copy_bits, - load_balance, large_occupancy, + overflow_policy, + cache_smem_values, ) cls._compile(*key) compiled_kernel = cls.kernel_cache[key] @@ -5198,37 +5533,272 @@ def forward( else: output_values_torch = None - # Prepare buffer - # extra buffer: num_rows * buffer_numbers * num_cols * 4 bytes - # fp32: up to 256 MB (256 * 2 * 262144 * 4) - # fp16/bf16: up to 128 MB (256 * 1 * 262144 * 4) - if dtype == cutlass.Float32: - buffer_numbers = 2 - else: - buffer_numbers = 1 - buffer_torch = cls.buffers.get_buffer( - [num_rows, buffer_numbers, bucketed_num_cols], - torch.int32, - buffer_name="single_cta_buffer", - reserve_buffer=reserve) - buffer_torch = buffer_torch[:, :, :num_cols] - # Prepare global counter for persistent dynamic scheduling - if load_balance: - g_global_counter_torch = cls.buffers.get_buffer( - [1], + # Prepare buffer (GMEM_SPILL only; other policies use None) + if overflow_policy == "GMEM_SPILL": + # extra buffer: num_rows * buffer_numbers * num_cols * 4 bytes + # fp32: up to 256 MB (256 * 2 * 262144 * 4) + # fp16/bf16: up to 128 MB (256 * 1 * 262144 * 4) + buffer_numbers = 2 if dtype == cutlass.Float32 else 1 + buffer_torch = cls.buffers.get_buffer( + [num_rows, buffer_numbers, bucketed_num_cols], torch.int32, - buffer_name="single_cta_g_global_counter", + buffer_name="single_cta_buffer", reserve_buffer=reserve) - g_global_counter_torch.zero_() + buffer_torch = buffer_torch[:, :, :num_cols] else: - g_global_counter_torch = None + buffer_torch = None # Execute kernel (TVM FFI uses env stream automatically) compiled_kernel( input_values, None, # indices buffer_torch, - g_global_counter_torch, + seq_lens, + output_indices_torch, + output_values_torch, + ) + + return output_indices_torch, output_values_torch + + # TODO: rename, CuteDSLTopKDecodeRadixFilterSPMultiCTARunner -> CuteDSLRadixFilterTopKSPMultiCTARunner + class CuteDSLTopKDecodeRadixFilterSPMultiCTARunner: + """Runner for the radix-FILTER single-pass multi-CTA decode top-k kernel. + + Distinct from the existing radix-SELECT SP multi-CTA runners + (``CuteDSLTopKDecodeSinglePassMultiCTA[Cluster]Runner``): this one drives + ``FilteredTopKKernelVarlenDecode`` with ``single_pass_multi_cta=True`` — + a cluster of ``cluster_size`` CTAs cooperates on one row via DSMEM + histogram merge + DSMEM prefix-scan collection (no GMEM state). + + ``cluster_size`` is a REQUIRED argument (no auto-config yet — radix-select's + ``_get_chunk_config`` tuning does not transfer; auto-config is a documented + TODO). ``chunk_size_per_cta = ceil(bucketed_num_cols / cluster_size)``. + """ + kernel_cache = dict() + buffers = get_memory_buffers() + + @classmethod + def _compile( + cls, + dtype, + bucketed_num_cols, + top_k, + next_n, + return_val, + num_copy_bits, + cluster_size, + chunk_size_per_cta, + overflow_policy, + cache_smem_values=False, + ): + key = ( + dtype, + bucketed_num_cols, + top_k, + next_n, + return_val, + num_copy_bits, + cluster_size, + chunk_size_per_cta, + overflow_policy, + cache_smem_values, + ) + if key in cls.kernel_cache: + return + n_rows = cute.sym_int() + n_cols = cute.sym_int() + n_batch = cute.sym_int() + input_fake = cute.runtime.make_fake_compact_tensor(dtype, + (n_rows, n_cols), + stride_order=(1, + 0), + assumed_align=32) + if overflow_policy == "GMEM_SPILL": + # Per-CTA spill buffer: (num_rows * cluster_size, num_buffers, + # chunk_size_per_cta) — independent dims from the input tensor. + buffer_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (cute.sym_int(), cute.sym_int(), cute.sym_int()), + stride_order=(2, 1, 0), + assumed_align=32, + ) + else: + buffer_fake = None + seqlen_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_batch, ), stride_order=(0, )) + output_indices_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_rows, top_k), stride_order=(1, 0)) + if return_val: + output_values_fake = cute.runtime.make_fake_compact_tensor( + dtype, (n_rows, top_k), stride_order=(1, 0)) + else: + output_values_fake = None + fake_stream = cute.runtime.make_fake_stream( + use_tvm_ffi_env_stream=True) + + filtered_topk_func = FilteredTopKKernelVarlenDecode( + dtype, + bucketed_num_cols, + top_k, + next_n, + num_copy_bits=num_copy_bits, + return_val=return_val, + chunk_size_per_cta=chunk_size_per_cta, + num_ctas_per_row=cluster_size, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, + single_pass_multi_cta=True, + ) + compiled_kernel = cute.compile( + filtered_topk_func, + input_fake, + None, # indices_fake + buffer_fake, + seqlen_fake, + output_indices_fake, + output_values_fake, + stream=fake_stream, + min_blocks_per_mp=1, + options="--enable-tvm-ffi", + ) + cls.kernel_cache[key] = compiled_kernel + + @staticmethod + def auto_cluster_size(num_tokens: int, + num_rows: int, + is_fp32: bool, + num_sms: Optional[int] = None) -> int: + """cluster_size for the REREAD overflow policy (has the large_occupancy + re-scan cliff). Shares peak_cs/occ_cap with auto_cluster_size_truncate + but floors nr > num_sms at cs=2 for huge N to dodge the single-CTA + large_occupancy REREAD blowup. Re-tuned on a B200 REREAD sweep + (~1.2% mean overhead vs oracle). Caveat: tuned on randn, fixed-length + seqlen inputs; real (concentrated) logits or varlen may shift it. + """ + num_sms = num_sms or _get_num_sms() + n = num_tokens + # peak_cs by N; fp32 stays single up to 16K (4 refine rounds cost more). + peak = (1 if n <= 8192 or (is_fp32 and n <= 16384) else + 4 if n <= 32768 else 8 if n <= 131072 else 16) + # occ_cap by num_rows; grid budget tightens as cs grows. nr > num_sms + # -> cs=2 for huge N (split the row so single-CTA large_occupancy REREAD + # blowup is avoided), else single. + occ = (16 if num_rows <= 4 else 8 if num_rows <= 8 else 4 if + num_rows <= 32 else 2 if num_rows <= 64 else 1 if num_rows <= + num_sms else 2 if n >= 262144 else 1) + cs = min(peak, occ, _query_max_cluster_size()) + return cs if cs >= 2 else 1 + + @staticmethod + def auto_cluster_size_truncate(num_tokens: int, + num_rows: int, + is_fp32: bool, + num_sms: Optional[int] = None) -> int: + """cluster_size for cliff-free overflow policies (TRUNCATE/GMEM_SPILL). + NOT interchangeable with auto_cluster_size (REREAD-tuned): here + nr > num_sms uses single-CTA (no REREAD large_occupancy blowup). + Re-tuned on a B200 TRUNCATE sweep (~0.6% mean overhead vs oracle). + Caveat: tuned on randn, fixed-length seqlen inputs; real (concentrated) + logit distributions or varlen seqlens may shift the optimum. + """ + num_sms = num_sms or _get_num_sms() + n = num_tokens + # peak_cs by N; fp32 stays single up to 16K (4 refine rounds cost more). + peak = (1 if n <= 8192 or (is_fp32 and n <= 16384) else + 4 if n <= 32768 else 8 if n <= 131072 else 16) + # occ_cap by num_rows; grid budget tightens as cs grows. nr > num_sms + # -> single, except a narrow just-over-one-wave SP band at large N. + occ = (16 if num_rows <= 4 else + 8 if num_rows <= 8 else 4 if num_rows <= 32 else + 2 if num_rows <= 64 else 1 if num_rows <= num_sms else 2 if + (num_rows <= 200 and n >= 131072) else 1) + cs = min(peak, occ, _query_max_cluster_size()) + return cs if cs >= 2 else 1 + + @classmethod + def forward( + cls, + input_values: torch.Tensor, + seq_lens: torch.Tensor, + top_k: int, + next_n: int, + cluster_size: int, + return_val: bool = False, + num_copy_bits: int = 256, + overflow_policy: str = "REREAD", + output_indices: Optional[torch.Tensor] = None, + cache_smem_values: bool = False, + ): + """Execute radix-filter SP multi-CTA cluster top-k. + + ``cluster_size`` (= ctas_per_group) must be provided by the caller. + """ + assert cluster_size >= 1, f"cluster_size must be >= 1, got {cluster_size}" + hw_max_cluster = _query_max_cluster_size() + assert cluster_size <= hw_max_cluster, ( + f"cluster_size={cluster_size} exceeds hardware max cluster " + f"size {hw_max_cluster}") + torch_dtype = input_values.dtype + dtype = _TORCH_TO_CUTLASS_DTYPE[torch_dtype] + num_rows, num_cols = input_values.shape + bucketed_num_cols = next_positive_power_of_2(num_cols) + chunk_size_per_cta = math.ceil(bucketed_num_cols / cluster_size) + + key = ( + dtype, + bucketed_num_cols, + top_k, + next_n, + return_val, + num_copy_bits, + cluster_size, + chunk_size_per_cta, + overflow_policy, + cache_smem_values, + ) + cls._compile(*key) + compiled_kernel = cls.kernel_cache[key] + reserve = torch.cuda.is_current_stream_capturing() + + if output_indices is not None: + output_indices_torch = output_indices + else: + output_indices_torch = cls.buffers.get_buffer( + [num_rows, top_k], + torch.int32, + buffer_name="rf_sp_multi_cta_output_indices", + reserve_buffer=reserve, + ) + if return_val: + output_values_torch = cls.buffers.get_buffer( + [num_rows, top_k], + torch_dtype, + buffer_name="rf_sp_multi_cta_output_values", + reserve_buffer=reserve, + ) + else: + output_values_torch = None + + if overflow_policy == "GMEM_SPILL": + # Per-CTA extra buffer: (num_rows * cluster_size, buffer_numbers, + # chunk_size_per_cta). + buffer_numbers = 2 if dtype == cutlass.Float32 else 1 + buffer_torch = cls.buffers.get_buffer( + [ + num_rows * cluster_size, buffer_numbers, + chunk_size_per_cta + ], + torch.int32, + buffer_name="rf_sp_multi_cta_buffer", + reserve_buffer=reserve, + ) + else: + buffer_torch = None + + compiled_kernel( + input_values, + None, # indices + buffer_torch, seq_lens, output_indices_torch, output_values_torch, @@ -5245,24 +5815,22 @@ def cute_dsl_topk_decode_blackwell( top_k: int, next_n: int = 1, num_copy_bits: int = 256, - load_balance: bool = False, ) -> torch.Tensor: """CuteDSL-based Top-K selection optimized for Blackwell decode phase. Args: input_values: Input logits tensor [batch_size * next_n, vocab_size] seq_lens: Sequence lengths for each batch [batch_size] - top_k: Number of top elements to select (max 2048) + top_k: Number of top elements to select (max 16384) next_n: Number of candidates per sequence (for speculative decoding) num_copy_bits: Number of bits for vectorized memory copy (128 or 256) - load_balance: Enable persistent dynamic scheduling for load balancing Returns: indices: Top-k indices [batch_size * next_n, top_k] Note: This function requires Blackwell architecture (SM100+) and CuTE DSL support. - Maximum supported top_k is 2048. + Maximum supported top_k is 16384. """ # Validate SM version sm_version = get_sm_version() @@ -5272,10 +5840,10 @@ def cute_dsl_topk_decode_blackwell( "Use standard top-k implementation for older architectures.") # Validate inputs - if top_k <= 0 or top_k > 2048: + if top_k <= 0 or top_k > 16384: raise ValueError( - f"top_k must be in range [1, 2048], got {top_k}. " - "Maximum supported top_k is 2048 for Blackwell architecture.") + f"top_k must be in range [1, 16384], got {top_k}. " + "Maximum supported top_k is 16384 for Blackwell architecture.") if next_n <= 0: raise ValueError(f"next_n must be positive, got {next_n}") @@ -5305,7 +5873,6 @@ def cute_dsl_topk_decode_blackwell( next_n=next_n, return_val=False, # Only return indices num_copy_bits=num_copy_bits, - load_balance=load_balance, ) return indices @@ -5316,7 +5883,6 @@ def _( top_k: int, next_n: int = 1, num_copy_bits: int = 256, - load_balance: bool = False, ): num_rows = input_values.shape[0] input_values.dtype @@ -5325,6 +5891,229 @@ def _( indices = input_values.new_empty((num_rows, top_k), dtype=torch.int32) return indices + class CuteDSLTopKPrefillSingleCTARunner: + """Runner for CuTE DSL Top-K prefill kernel (single CTA per row). + + Uses FilteredTopKKernelVarlenPrefill with large_occupancy=True (512 + threads/CTA, reduced SMEM). Row extents are supplied as row_starts / + row_ends tensors; output indices are LOCAL (0-indexed within each row's + valid range), matching the CUDA indexer_topk_prefill convention. + + All methods are class-level — no instantiation needed. + """ + + kernel_cache: dict = {} + buffers = get_memory_buffers() + + @classmethod + def _compile(cls, + dtype, + bucketed_num_cols, + top_k, + return_val, + num_copy_bits, + overflow_policy, + cache_smem_values=False): + """Compile and cache a single-CTA prefill top-k kernel.""" + key = (dtype, bucketed_num_cols, top_k, return_val, num_copy_bits, + overflow_policy, cache_smem_values) + if key in cls.kernel_cache: + return + n_rows = cute.sym_int() + n_cols = cute.sym_int() + input_fake = cute.runtime.make_fake_compact_tensor( + dtype, + (n_rows, n_cols), + stride_order=(1, 0), + assumed_align=32, + ) + row_starts_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_rows, ), + stride_order=(0, ), + ) + row_ends_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_rows, ), + stride_order=(0, ), + ) + if overflow_policy == "GMEM_SPILL": + buffer_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_rows, cute.sym_int(), n_cols), + stride_order=(2, 1, 0), + assumed_align=32, + ) + else: + buffer_fake = None + output_indices_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_rows, top_k), + stride_order=(1, 0), + ) + if return_val: + output_values_fake = cute.runtime.make_fake_compact_tensor( + dtype, + (n_rows, top_k), + stride_order=(1, 0), + ) + else: + output_values_fake = None + fake_stream = cute.runtime.make_fake_stream( + use_tvm_ffi_env_stream=True) + + filtered_topk_func = FilteredTopKKernelVarlenPrefill( + dtype, + bucketed_num_cols, + top_k, + num_copy_bits=num_copy_bits, + return_val=return_val, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, + ) + compiled_kernel = cute.compile( + filtered_topk_func, + input_fake, + row_starts_fake, + row_ends_fake, + buffer_fake, + output_indices_fake, + output_values_fake, + stream=fake_stream, + min_blocks_per_mp=4, + options="--enable-tvm-ffi", + ) + cls.kernel_cache[key] = compiled_kernel + + @classmethod + def forward( + cls, + input_values: torch.Tensor, + row_starts: torch.Tensor, + row_ends: torch.Tensor, + top_k: int, + return_val: bool = False, + num_copy_bits: int = 256, + overflow_policy: str = "REREAD", + output_indices: Optional[torch.Tensor] = None, + cache_smem_values: bool = False, + ): + """Execute filtered top-k selection for prefill rows.""" + torch_dtype = input_values.dtype + dtype = _TORCH_TO_CUTLASS_DTYPE[torch_dtype] + num_rows, num_cols = input_values.shape + bucketed_num_cols = next_positive_power_of_2(num_cols) + + key = (dtype, bucketed_num_cols, top_k, return_val, num_copy_bits, + overflow_policy, cache_smem_values) + cls._compile(*key) + compiled_kernel = cls.kernel_cache[key] + reserve = torch.cuda.is_current_stream_capturing() + + if output_indices is not None: + output_indices_torch = output_indices + else: + output_indices_torch = cls.buffers.get_buffer( + [num_rows, top_k], + torch.int32, + buffer_name="prefill_single_cta_output_indices", + reserve_buffer=reserve, + ) + if return_val: + output_values_torch = cls.buffers.get_buffer( + [num_rows, top_k], + torch_dtype, + buffer_name="prefill_single_cta_output_values", + reserve_buffer=reserve, + ) + else: + output_values_torch = None + + if overflow_policy == "GMEM_SPILL": + buffer_numbers = 2 if dtype == cutlass.Float32 else 1 + buffer_torch = cls.buffers.get_buffer( + [num_rows, buffer_numbers, bucketed_num_cols], + torch.int32, + buffer_name="prefill_single_cta_buffer", + reserve_buffer=reserve, + ) + buffer_torch = buffer_torch[:, :, :num_cols] + else: + buffer_torch = None + + compiled_kernel( + input_values, + row_starts, + row_ends, + buffer_torch, + output_indices_torch, + output_values_torch, + ) + return output_indices_torch, output_values_torch + + @torch.library.custom_op( + "trtllm::cute_dsl_indexer_topk_prefill_blackwell", + mutates_args=("output_indices", ), + device_types="cuda", + ) + def cute_dsl_indexer_topk_prefill_blackwell( + input_values: torch.Tensor, + row_starts: torch.Tensor, + row_ends: torch.Tensor, + output_indices: torch.Tensor, + top_k: int, + num_copy_bits: int = 256, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, + ) -> None: + """CuTE DSL radix-based top-k for prefill. + + Args: + input_values: Logits tensor of shape (num_rows, num_cols). + row_starts: Per-row start column (inclusive), shape (num_rows,), int32. + row_ends: Per-row end column (exclusive), shape (num_rows,), int32. + top_k: Number of top-k indices to select per row. + num_copy_bits: Vector copy width in bits (default 256). + overflow_policy: How to handle threshold-bucket SMEM overflow. + "GMEM_SPILL" (default, exact) or "TRUNCATE" (non-exact, + no extra buffer). + cache_smem_values: Cache ordered values in SMEM to avoid re-reading from + GMEM in refinement rounds (reduces S by 2x). + + output_indices: Pre-allocated Int32 tensor of shape (num_rows, top_k), + written in place with LOCAL indices (0-indexed within + [row_start, row_end) for each row). Padding positions + are -1. + """ + # Write into the caller-provided output_indices (mutates_args) rather than + # returning the runner's reusable pool buffer, matching the decode op and + # the CUDA indexer_topk_prefill contract (write into a caller buffer slice). + CuteDSLTopKPrefillSingleCTARunner.forward( + input_values, + row_starts, + row_ends, + top_k, + return_val=False, + num_copy_bits=num_copy_bits, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, + output_indices=output_indices, + ) + + @torch.library.register_fake( + "trtllm::cute_dsl_indexer_topk_prefill_blackwell") + def _( + input_values: torch.Tensor, + row_starts: torch.Tensor, + row_ends: torch.Tensor, + output_indices: torch.Tensor, + top_k: int, + num_copy_bits: int = 256, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, + ): + return None + class CuteDSLTopKDecodeMultiCTARunner: """Runner for CuTE DSL Top-K decode kernel (multi CTA version). @@ -5366,11 +6155,12 @@ def _compile(cls, next_n, return_val, num_copy_bits, - load_balance, large_occupancy, chunk_size_per_cta, num_ctas_per_row, - dynamic=False): + overflow_policy="REREAD", + dynamic=False, + cache_smem_values=False): """Compile and cache multi-CTA top-k kernels for the given config.""" key = ( dtype, @@ -5378,11 +6168,12 @@ def _compile(cls, next_n, return_val, num_copy_bits, - load_balance, large_occupancy, chunk_size_per_cta, num_ctas_per_row, + overflow_policy, dynamic, + cache_smem_values, ) if key in cls.kernel_cache: return @@ -5394,12 +6185,17 @@ def _compile(cls, stride_order=(1, 0), assumed_align=32) - buffer_fake = cute.runtime.make_fake_compact_tensor( - cutlass.Int32, - (cute.sym_int(), cute.sym_int(), cute.sym_int()), - stride_order=(2, 1, 0), - assumed_align=32, - ) + # extra_buffer for GMEM_SPILL: spills threshold-bin candidates that + # overflow SMEM. Not needed for TRUNCATE/REREAD/REREAD_ALWAYS. + if overflow_policy == "GMEM_SPILL": + buffer_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (cute.sym_int(), cute.sym_int(), cute.sym_int()), + stride_order=(2, 1, 0), + assumed_align=32, + ) + else: + buffer_fake = None seqlen_fake = cute.runtime.make_fake_compact_tensor( cutlass.Int32, (n_batch, ), @@ -5434,18 +6230,18 @@ def _compile(cls, num_ctas_per_row=num_ctas_per_row, merge_blocks=False, enable_dynamic_multi_cta=dynamic, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, ) compiled_kernel_first = cute.compile( filtered_topk_func_first, input_fake, None, # indices_fake buffer_fake, - None, # g_global_counter_fake seqlen_fake, first_kernel_output_indices_fake, first_kernel_output_values_fake, stream=fake_stream, - enable_persistent_dynamic_scheduling=load_balance, min_blocks_per_mp=1, options="--enable-tvm-ffi", ) @@ -5482,18 +6278,18 @@ def _compile(cls, enable_multi_cta=False, merge_blocks=True, varlen_merge_input=dynamic, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, ) compiled_kernel_second = cute.compile( filtered_topk_func_second, input_fake, indices_fake, buffer_fake, - None, # g_global_counter_fake seqlen_fake, output_indices_fake, output_values_fake, stream=fake_stream, - enable_persistent_dynamic_scheduling=load_balance, min_blocks_per_mp=1, options="--enable-tvm-ffi", ) @@ -5510,8 +6306,10 @@ def forward( return_val: bool = False, num_copy_bits: int = 256, chunk_size_per_cta: int = 16384, + overflow_policy: str = "REREAD", dynamic: bool = True, output_indices: Optional[torch.Tensor] = None, + cache_smem_values: bool = False, ): """Execute multi-CTA filtered top-k selection on input logits.""" torch_dtype = input_values.dtype @@ -5520,7 +6318,6 @@ def forward( num_sms = _get_num_sms() large_occupancy = num_rows > num_sms - load_balance = False num_ctas_per_row = math.ceil(num_cols / chunk_size_per_cta) merge_cols = num_ctas_per_row * top_k @@ -5531,22 +6328,18 @@ def forward( next_n, return_val, num_copy_bits, - load_balance, large_occupancy, chunk_size_per_cta, num_ctas_per_row, + overflow_policy, dynamic, + cache_smem_values, ) cls._compile(*key) compiled_kernel_first, compiled_kernel_second = \ cls.kernel_cache[key] reserve = torch.cuda.is_current_stream_capturing() - if dtype == cutlass.Float32: - buffer_numbers = 2 - else: - buffer_numbers = 1 - # Intermediate buffers for first kernel output first_output_indices = cls.buffers.get_buffer( [num_rows, merge_cols], @@ -5559,13 +6352,18 @@ def forward( buffer_name="multi_cta_first_output_values", reserve_buffer=reserve) - # Shared buffer for both kernels (they run sequentially) - buffer_dim2 = max(chunk_size_per_cta, merge_cols) - buffer_torch = cls.buffers.get_buffer( - [num_rows * num_ctas_per_row, buffer_numbers, buffer_dim2], - torch.int32, - buffer_name="multi_cta_buffer", - reserve_buffer=reserve) + # extra_buffer for GMEM_SPILL: spills threshold-bin candidates that + # overflow SMEM. Not needed for TRUNCATE/REREAD/REREAD_ALWAYS. + if overflow_policy == "GMEM_SPILL": + buffer_numbers = 2 if dtype == cutlass.Float32 else 1 + buffer_dim2 = max(chunk_size_per_cta, merge_cols) + buffer_torch = cls.buffers.get_buffer( + [num_rows * num_ctas_per_row, buffer_numbers, buffer_dim2], + torch.int32, + buffer_name="multi_cta_buffer", + reserve_buffer=reserve) + else: + buffer_torch = None # Final output tensors if output_indices is not None: @@ -5590,7 +6388,6 @@ def forward( input_values, None, # indices buffer_torch, - None, # g_global_counter_torch seq_lens, first_output_indices, first_output_values, @@ -5601,7 +6398,6 @@ def forward( first_output_values, first_output_indices, buffer_torch, - None, # g_global_counter_torch seq_lens, output_indices_torch, output_values_torch, @@ -5987,7 +6783,7 @@ def cute_dsl_topk_decode_multi_cta_blackwell( Args: input_values: Input logits tensor [batch_size * next_n, vocab_size] seq_lens: Sequence lengths for each batch [batch_size] - top_k: Number of top elements to select (max 2048) + top_k: Number of top elements to select (max 16384) next_n: Number of candidates per sequence (for speculative decoding) num_copy_bits: Number of bits for vectorized memory copy (128 or 256) chunk_size_per_cta: Number of columns each CTA processes @@ -6007,10 +6803,10 @@ def cute_dsl_topk_decode_multi_cta_blackwell( "Use standard top-k implementation for older architectures.") # Validate inputs - if top_k <= 0 or top_k > 2048: + if top_k <= 0 or top_k > 16384: raise ValueError( - f"top_k must be in range [1, 2048], got {top_k}. " - "Maximum supported top_k is 2048 for Blackwell architecture.") + f"top_k must be in range [1, 16384], got {top_k}. " + "Maximum supported top_k is 16384 for Blackwell architecture.") if next_n <= 0: raise ValueError(f"next_n must be positive, got {next_n}") @@ -6066,6 +6862,31 @@ def _( indices = input_values.new_empty((num_rows, top_k), dtype=torch.int32) return indices + def _radix_select_preferred(dtype, num_tokens: int, num_rows: int) -> bool: + """radix-SELECT SP beats radix-FILTER at small N: select loads the whole + chunk into SMEM and radix-selects in place, which is leaner than filter's + histogram + refine when the chunk fits SMEM and the fixed overhead is not + amortized over few elements. SMEM-capacity driven, so distribution-robust. + fp32 (4B) fills SMEM at half the N and its 4 refine rounds favor filter -> + always filter. Tuned on a B200 randn sweep (filter-vs-select, best-vs-best + + tuned): bf16 wins up to ~32%, fp16 up to ~12%. + + bf16 N=32768 is batch-split: select wins only at large batch (grid + pressure forces both to single-CTA, where select single beats filter + single); at small batch filter's cluster is better, so keep filter. + """ + # Large batch (num_rows > num_sms) forces the filter path to single-CTA + # (auto_cluster_size -> cs=1), which beats the radix-SELECT cluster runner + # here; only prefer select while the batch still fits within one SM wave. + num_sms = _get_num_sms() + if dtype == torch.bfloat16: + return num_rows <= num_sms and (num_tokens <= 16384 or + (num_tokens == 32768 + and num_rows >= 74)) + if dtype == torch.float16: + return num_rows <= num_sms and num_tokens <= 16384 + return False + @torch.library.custom_op("trtllm::cute_dsl_indexer_topk_decode", mutates_args=("output_indices", ), device_types="cuda") @@ -6079,13 +6900,32 @@ def cute_dsl_indexer_topk_decode( dynamic: bool = True, single_pass_multi_cta: bool = False, single_pass_multi_cta_cluster: bool = False, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, + radix_filter_single_pass_multi_cta: bool = True, ) -> None: - """Unified CuTE DSL Top-K that auto-selects single-CTA or multi-CTA (2-pass multi-CTA) or - single-pass multi-CTA. When single_pass_multi_cta=True, it selects between single-CTA - and multi-CTA (1-pass multi-CTA). When single_pass_multi_cta=False, it selects between - single-CTA and multi-CTA (2-pass multi-CTA). - - Writes results directly into the pre-allocated ``output_indices`` buffer. + """Unified CuTE DSL decode Top-K. Writes results directly into the + pre-allocated ``output_indices`` buffer. + + Three mutually exclusive dispatch modes, selected by boolean with + precedence ``radix_filter_single_pass_multi_cta`` > + ``single_pass_multi_cta`` > 2-pass (evaluated as an if/elif chain, so + the first True wins and the others are ignored): + + 1. ``radix_filter_single_pass_multi_cta=True`` (default) -- ADAPTIVE, + best-performance path. Internally auto-selects among radix-SELECT + (small N), radix-FILTER single-pass multi-CTA cluster, and + single-CTA by (dtype, N, num_rows, overflow_policy); no caller + tuning needed. Prefer this. When it is enabled the other two mode + booleans MUST be left False (asserted below) -- they would be + silently ignored otherwise. + 2. ``single_pass_multi_cta=True`` (only when mode 1 is False) -- legacy + single-pass multi-CTA path (kept for A/B and fallback). Auto-selects + single-CTA vs single-pass multi-CTA by the SM-wave heuristic below; + ``single_pass_multi_cta_cluster=True`` forces the cluster variant + within this mode (no effect in the other modes). + 3. neither set -- legacy 2-pass multi-CTA path (A/B and fallback); + vocab-threshold + SM-utilization heuristic. Dispatch logic (``single_pass_multi_cta=True`` path): @@ -6118,18 +6958,96 @@ def cute_dsl_indexer_topk_decode( input_values: Input logits tensor [batch_size * next_n, vocab_size] seq_lens: Sequence lengths for each batch [batch_size] output_indices: Pre-allocated output buffer [batch_size * next_n, top_k] - top_k: Number of top elements to select (max 2048) + top_k: Number of top elements to select (max 16384) next_n: Number of candidates per sequence (for speculative decoding) num_copy_bits: Number of bits for vectorized memory copy (128 or 256) dynamic: Use dynamic multi-CTA scheduling (for 2-pass multi-CTA) - single_pass_multi_cta: Use single-pass multi-CTA radix top-k - single_pass_multi_cta_cluster: Force cluster-accelerated variant - (only effective when single_pass_multi_cta=True) + single_pass_multi_cta: Mode-2 override -- use the legacy single-pass + multi-CTA path. Only takes effect when + radix_filter_single_pass_multi_cta=False. + single_pass_multi_cta_cluster: Force the cluster-accelerated variant + within mode 2 (only effective when single_pass_multi_cta=True). + overflow_policy: Threshold-bucket SMEM overflow handling + ("REREAD" default, exact). See FilteredTopKKernelVarlen. + cache_smem_values: Cache ordered values in SMEM to skip a reload. + radix_filter_single_pass_multi_cta: Mode-1 (default True) -- the + adaptive best-performance path; see the mode list above. Set + False to select mode 2 or 3. """ + # Validate inputs + if top_k <= 0 or top_k > 16384: + raise ValueError( + f"top_k must be in range [1, 16384], got {top_k}. " + "Maximum supported top_k is 16384 for Blackwell architecture.") + num_rows = input_values.shape[0] num_tokens = input_values.shape[1] - if single_pass_multi_cta: + if radix_filter_single_pass_multi_cta: + # Mode 1 is the adaptive default and dominates the if/elif chain + # below. Reject a conflicting mode-2/3 override rather than silently + # ignoring it (set radix_filter_single_pass_multi_cta=False to opt + # into the legacy single_pass_multi_cta / 2-pass paths). + assert not single_pass_multi_cta and not single_pass_multi_cta_cluster, ( + "radix_filter_single_pass_multi_cta (adaptive default) takes " + "precedence over single_pass_multi_cta / " + "single_pass_multi_cta_cluster; set it False to use those " + "legacy overrides.") + _R = CuteDSLTopKDecodeRadixFilterSPMultiCTARunner + _is_fp32 = input_values.dtype == torch.float32 + # At small N (bf16 <= 32K, fp16 <= 16K) radix-SELECT SP beats + # radix-FILTER; route there. The select cluster runner auto-picks + # ctas (=1 => select single-CTA for small batch, else cluster), so + # this covers both. Falls through to filter only if select can't + # fit the problem (capacity -> None), which small N never hits. + if _radix_select_preferred(input_values.dtype, num_tokens, + num_rows): + _sel = CuteDSLTopKDecodeSinglePassMultiCTAClusterRunner.forward( + input_values=input_values, + seq_lens=seq_lens, + top_k=top_k, + next_n=next_n, + return_val=False, + num_copy_bits=num_copy_bits, + output_indices=output_indices, + ) + if _sel[0] is not None: + return + # radix-FILTER single-CTA vs SP multi-CTA (cluster DSMEM). Heuristic + # is overflow-policy-coupled: REREAD has a large_occupancy re-scan + # cliff, cliff-free policies don't -> pick the matching tune. + if overflow_policy == "REREAD": + cluster_size = _R.auto_cluster_size(num_tokens, num_rows, + _is_fp32) + else: + cluster_size = _R.auto_cluster_size_truncate( + num_tokens, num_rows, _is_fp32) + if cluster_size >= 2: + CuteDSLTopKDecodeRadixFilterSPMultiCTARunner.forward( + input_values=input_values, + seq_lens=seq_lens, + top_k=top_k, + next_n=next_n, + cluster_size=cluster_size, + return_val=False, + num_copy_bits=num_copy_bits, + overflow_policy=overflow_policy, + output_indices=output_indices, + cache_smem_values=cache_smem_values, + ) + else: + CuteDSLTopKDecodeSingleCTARunner.forward( + input_values=input_values, + seq_lens=seq_lens, + top_k=top_k, + next_n=next_n, + return_val=False, + num_copy_bits=num_copy_bits, + overflow_policy=overflow_policy, + output_indices=output_indices, + cache_smem_values=cache_smem_values, + ) + elif single_pass_multi_cta: # --- heuristic for single-CTA vs single-pass multi-CTA --- # Determines whether the single-pass multi-CTA kernel # is faster than single-CTA based on SM wave occupancy analysis. @@ -6209,7 +7127,9 @@ def cute_dsl_indexer_topk_decode( next_n=next_n, return_val=False, num_copy_bits=num_copy_bits, + overflow_policy=overflow_policy, output_indices=output_indices, + cache_smem_values=cache_smem_values, ) else: # --- 2-pass multi-CTA dispatch --- @@ -6241,8 +7161,10 @@ def cute_dsl_indexer_topk_decode( return_val=False, num_copy_bits=num_copy_bits, chunk_size_per_cta=chunk_size_per_cta, + overflow_policy=overflow_policy, dynamic=dynamic, output_indices=output_indices, + cache_smem_values=cache_smem_values, ) else: CuteDSLTopKDecodeSingleCTARunner.forward( @@ -6252,7 +7174,9 @@ def cute_dsl_indexer_topk_decode( next_n=next_n, return_val=False, num_copy_bits=num_copy_bits, + overflow_policy=overflow_policy, output_indices=output_indices, + cache_smem_values=cache_smem_values, ) @torch.library.register_fake("trtllm::cute_dsl_indexer_topk_decode") @@ -6266,9 +7190,82 @@ def _( dynamic: bool = True, single_pass_multi_cta: bool = False, single_pass_multi_cta_cluster: bool = False, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, + radix_filter_single_pass_multi_cta: bool = True, ) -> None: return None + def warmup_cute_dsl_radix_topk_decode( + top_k: int, + num_cols: int, + next_n: int = 1, + dtype: torch.dtype = torch.float32, + num_copy_bits: int = 256, + num_sms: Optional[int] = None, + ) -> None: + """Pre-compile the radix-filter DSL decode top-k for every + ``cluster_size`` variant the runtime dispatch can pick for this + deployment. + + ``cute_dsl_indexer_topk_decode`` JIT-compiles a fresh CuTe DSL kernel + per compile-key ``(dtype, bucketed_num_cols, top_k, next_n, ..., + cluster_size)`` on first touch (~seconds). Every key dimension is + fixed for a deployment (``num_cols = indexer_max_seq_len``) except + ``cluster_size = auto_cluster_size(num_cols, num_rows, ...)``, which + steps across the coarse ``num_rows`` occupancy bands + (<=4 / <=8 / <=32 / <=64 / <=num_sms / >num_sms). CUDA-graph warmup + only exercises ``cuda_graph_batch_sizes``; eager iters (mixed + prefill+decode batch, or ``cuda_graph`` disabled) whose ``num_rows`` + lands in an uncovered band otherwise pay the JIT stall on a live + request. Issuing one decode per representative ``num_rows`` funnels + every ``cluster_size`` compile into warmup — the op's own dispatch + (radix-SELECT / radix-FILTER cluster / single-CTA) picks and compiles + exactly what the runtime would. + + Meant to run during warmup, before serving. Captured geometries are + already compiled by the warmup-step forwards; this fills in the bands + the eager (non-captured) path can still hit. Best-effort: per-band + failures are logged and skipped so one broken bucket does not abort + startup. + """ + if top_k <= 0 or num_cols <= 0 or next_n <= 0: + return + num_sms = num_sms or _get_num_sms() + device = torch.device("cuda") + # One representative num_rows per auto_cluster_size occupancy band. + # Rounded up to a multiple of next_n (kernel shape contract: + # num_rows % next_n == 0); identical num_rows are de-duplicated. + band_targets = (4, 8, 32, 64, num_sms, num_sms + 1) + seen = set() + for target in band_targets: + num_gen = max(1, -(-target // next_n)) # ceil(target / next_n) + num_rows = num_gen * next_n + if num_rows in seen: + continue + seen.add(num_rows) + logits = torch.zeros((num_rows, num_cols), + dtype=dtype, + device=device) + seq_lens = torch.full((num_gen, ), + num_cols, + dtype=torch.int32, + device=device) + output_indices = torch.empty((num_rows, top_k), + dtype=torch.int32, + device=device) + try: + torch.ops.trtllm.cute_dsl_indexer_topk_decode( + logits, seq_lens, output_indices, top_k, next_n, + num_copy_bits) + except RuntimeError as e: + logger.warning( + f"[DSL topk warmup] radix-filter prewarm failed for " + f"num_rows={num_rows} (num_cols={num_cols}, top_k={top_k}, " + f"next_n={next_n}); skipping band. " + f"{type(e).__name__}: {e}") + torch.cuda.synchronize() + # ------------------------------------------------------------------ # # CuTe DSL GVR Top-K Decode # # ------------------------------------------------------------------ # @@ -6937,148 +7934,6 @@ def _( # ``_compile_lb`` and the ``counters is not None`` branch of # ``forward`` (shares ``_pick_tuning`` with the single-CTA path). - def warmup_cute_dsl_indexer_topk( - dtype: torch.dtype, - top_k: int, - next_n: int = 1, - num_copy_bits: int = 256, - min_seq_len_log2: int = 10, - max_seq_len_log2: int = 18, - single_pass_multi_cta: bool = False, - single_pass_multi_cta_cluster: bool = False, - ) -> None: - """Pre-compile all CuTE DSL top-k kernel variants for every - power-of-2 bucketed_num_cols in [2^min_seq_len_log2, 2^max_seq_len_log2]. - - Because the kernel compilation key uses - ``bucketed_num_cols = next_positive_power_of_2(num_cols)``, only - a small number of distinct kernels are needed regardless of the - actual ``max_seq_len``. This allows warmup to be called at model - init time without knowing the runtime ``max_seq_len``. - - Must be called before CUDA Graph capture so that JIT compilation - does not occur during capture/replay. - - Args: - dtype: Data type of the logits (e.g. torch.bfloat16). - top_k: Number of top elements to select. - next_n: Number of candidates per sequence (speculative decoding). - num_copy_bits: Vectorized memory copy width (128 or 256). - min_seq_len_log2: Log2 of minimum bucketed_num_cols (default 10 → 1024). - max_seq_len_log2: Log2 of maximum bucketed_num_cols (default 18 → 262144). - single_pass_multi_cta: Use single-pass multi-CTA radix top-k - dispatch path instead of the legacy two-pass kernels. - single_pass_multi_cta_cluster: Force cluster-accelerated variant - (only effective when single_pass_multi_cta=True). - """ - cutlass_dtype = _TORCH_TO_CUTLASS_DTYPE[dtype] - return_val = False - chunk_size_per_cta = 16384 - - # Multi-CTA vocab thresholds by dtype - if dtype == torch.float32: - multi_cta_threshold = 65536 - else: - multi_cta_threshold = 131072 - - # SingleCTA: enumerate all power-of-2 bucketed_num_cols - for log2_n in range(min_seq_len_log2, max_seq_len_log2 + 1): - bucketed_num_cols = 1 << log2_n - for large_occupancy in (False, True): - CuteDSLTopKDecodeSingleCTARunner._compile( - cutlass_dtype, - bucketed_num_cols, - top_k, - next_n, - return_val, - num_copy_bits, - load_balance=False, - large_occupancy=large_occupancy, - ) - - if single_pass_multi_cta: - # Single-pass multi-CTA: enumerate all (chunk_size, ctas_per_group) - # pairs. chunk_size is snapped to power-of-2 (+ max_chunk clamp), - # so the set of possible values is small and deterministic. - num_sms = _get_num_sms() - possible_chunks = CuteDSLTopKDecodeSinglePassMultiCTARunner._get_possible_chunk_sizes( - cutlass_dtype, num_copy_bits) - max_chunk, vec_size = CuteDSLTopKDecodeSinglePassMultiCTARunner._compute_max_chunk( - cutlass_dtype, num_copy_bits) - single_pass_multi_cta_configs = set() - for cs in possible_chunks: - for log2_n in range(min_seq_len_log2, max_seq_len_log2 + 1): - num_cols = 1 << log2_n - ctas = math.ceil(num_cols / cs) - if ctas >= 1: - single_pass_multi_cta_configs.add((cs, ctas)) - # Also cover FlashInfer-style fallback path (large batch): - # ctas_per_group = ceil(num_cols / max_chunk), chunk_size aligned - for log2_n in range(min_seq_len_log2, max_seq_len_log2 + 1): - num_cols = 1 << log2_n - ctas = math.ceil(num_cols / max_chunk) - if ctas >= 1: - cs = math.ceil(num_cols / ctas) - cs = ((cs + vec_size - 1) // vec_size) * vec_size - if cs > max_chunk: - cs = max_chunk - single_pass_multi_cta_configs.add((cs, ctas)) - for cs, ctas in sorted(single_pass_multi_cta_configs): - CuteDSLTopKDecodeSinglePassMultiCTARunner._compile( - cutlass_dtype, cs, top_k, next_n, num_copy_bits, ctas, - num_sms, return_val) - - # Cluster variant: enumerate configs using the cluster runner's - # _get_chunk_config (which clamps to hw max cluster size). - cluster_configs = set() - if single_pass_multi_cta_cluster: - for log2_n in range(min_seq_len_log2, max_seq_len_log2 + 1): - num_cols = 1 << log2_n - for nr in [1, 4, 16, 64, 256]: - cfg = CuteDSLTopKDecodeSinglePassMultiCTAClusterRunner._get_chunk_config( - cutlass_dtype, - num_cols, - num_copy_bits=num_copy_bits, - num_rows=nr) - if cfg[0] is not None: - cluster_configs.add((cfg[0], cfg[1])) - for cs, ctas in sorted(cluster_configs): - CuteDSLTopKDecodeSinglePassMultiCTAClusterRunner._compile( - cutlass_dtype, cs, top_k, next_n, num_copy_bits, ctas, - num_sms, return_val) - - multi_cta_info = ( - f"SinglePassMultiCTA ({len(single_pass_multi_cta_configs)} configs" - f", cluster {len(cluster_configs)} configs)") - else: - # 2-pass MultiCTA: enumerate all possible num_ctas_per_row values - # num_ctas_per_row = ceil(num_cols / chunk_size_per_cta) - # fp32: num_cols in [65536, 262144] → num_ctas_per_row in [4, 16] - # fp16/bf16: num_cols in [131072, 262144] → num_ctas_per_row in [8, 16] - min_ctas = math.ceil(multi_cta_threshold / chunk_size_per_cta) - max_ctas = math.ceil((1 << max_seq_len_log2) / chunk_size_per_cta) - for num_ctas_per_row in range(min_ctas, max_ctas + 1): - for large_occupancy in (False, True): - CuteDSLTopKDecodeMultiCTARunner._compile( - cutlass_dtype, - top_k, - next_n, - return_val, - num_copy_bits, - load_balance=False, - large_occupancy=large_occupancy, - chunk_size_per_cta=chunk_size_per_cta, - num_ctas_per_row=num_ctas_per_row, - dynamic=True, - ) - multi_cta_info = ( - f"MultiCTA num_ctas_per_row=[{min_ctas}..{max_ctas}]") - - logger.info( - f"Warmed up CuTE DSL indexer top-k kernels: dtype={dtype}, " - f"SingleCTA bucketed_num_cols=[2^{min_seq_len_log2}..2^{max_seq_len_log2}], " - f"{multi_cta_info}, top_k={top_k}, next_n={next_n}") - # ------------------------------------------------------------------ # # CuTE DSL FP8 Paged MQA Logits (Blackwell SM100) # # ------------------------------------------------------------------ # @@ -7241,10 +8096,14 @@ def forward( SPLIT_KV = compute_block_kv * 2 # NUM_MATH_WG = 2 aligned_max_ctx = ( (max_context_len + SPLIT_KV - 1) // SPLIT_KV) * SPLIT_KV - logits = torch.empty( - (B * next_n, aligned_max_ctx), - device=q.device, - dtype=output_dtype, + # Use a persistent arena buffer instead of a per-forward torch.empty + # so the output address stays stable across CUDA-graph replays. + _reserve = torch.cuda.is_current_stream_capturing() + logits = get_memory_buffers().get_buffer( + [B * next_n, aligned_max_ctx], + output_dtype, + buffer_name="cute_dsl_mqa_logits", + reserve_buffer=_reserve, ) logits = logits[:, :max_context_len] @@ -8089,10 +8948,14 @@ def forward( SPLIT_KV = compute_block_kv * 2 # NUM_MATH_WG = 2 aligned_max_ctx = ( (max_context_len + SPLIT_KV - 1) // SPLIT_KV) * SPLIT_KV - logits = torch.empty( - (B * next_n, aligned_max_ctx), - device=q.device, - dtype=output_dtype, + # Use a persistent arena buffer instead of a per-forward torch.empty + # so the output address stays stable across CUDA-graph replays. + _reserve = torch.cuda.is_current_stream_capturing() + logits = get_memory_buffers().get_buffer( + [B * next_n, aligned_max_ctx], + output_dtype, + buffer_name="cute_dsl_mqa_logits", + reserve_buffer=_reserve, ) logits = logits[:, :max_context_len] diff --git a/tensorrt_llm/_torch/custom_ops/cute_dsl_megamoe_custom_op.py b/tensorrt_llm/_torch/custom_ops/cute_dsl_megamoe_custom_op.py index 5b103acd3802..60a2b18e0397 100644 --- a/tensorrt_llm/_torch/custom_ops/cute_dsl_megamoe_custom_op.py +++ b/tensorrt_llm/_torch/custom_ops/cute_dsl_megamoe_custom_op.py @@ -15,8 +15,10 @@ round-trips (required by the autotuner cache). * ``trtllm::cute_dsl_megamoe_nvfp4_blackwell`` is the registered torch custom op that the ``MegaMoECuteDsl`` backend calls from - ``run_moe``. It runs ``AutoTuner.choose_one`` once per call to pick - the best tactic and forwards to the runner. + ``run_moe``. ``tactic_autotune=True`` (bench-only, enabled via the + ``MEGAMOE_TACTIC_AUTOTUNE=1`` env var read by the backend) runs + ``AutoTuner.choose_one`` per call; the default bypasses the AutoTuner + and uses the deterministic token-bucket heuristic tactic. The backend never instantiates :class:`Sm100MegaMoENvfp4Runner` directly; this mirrors how ``CuteDslFusedMoE`` only consumes @@ -29,7 +31,8 @@ from __future__ import annotations import dataclasses -import time +import functools +import weakref from typing import Any, List, Optional, Tuple import torch @@ -37,6 +40,7 @@ from tensorrt_llm.logger import logger from ..._utils import get_sm_version +from ...math_utils import ceil_div, pad_up from ..autotuner import ( AutoTuner, ConstraintSpec, @@ -49,13 +53,21 @@ from ..cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE from ..utils import get_last_power_of_2_num_tokens_buckets, last_positive_power_of_2 + +def _import_megamoe_kernel(): + """Lazy import so non-SM100 / no-cutlass-dsl envs can still import this module.""" + from ..cute_dsl_kernels.mega_moe_nvfp4 import import_kernel + from ..cute_dsl_kernels.mega_moe_nvfp4.token_comm import CombineFormat + + return import_kernel(), CombineFormat + + __all__ = [ - "DEFAULT_MEGAMOE_TACTIC", "IS_MEGAMOE_OP_AVAILABLE", "MEGAMOE_OP_UNAVAILABLE_REASON", + "default_megamoe_tactic", "enumerate_megamoe_candidate_tactics", "megamoe_activation_sf_bytes_per_row", - "resolve_megamoe_group_hint", "validate_megamoe_tactic", ] @@ -69,58 +81,198 @@ IS_MEGAMOE_OP_AVAILABLE: bool = False MEGAMOE_OP_UNAVAILABLE_REASON: Optional[str] = None +# (local_ws_ptr, shared_ws_ptr, world_size) keys whose in-kernel barrier +# (counter, signal) was fenced-reset. Module-global because the runner is +# rebuilt every op call; the fence must fire ONCE per key (see forward()). +_MEGAMOE_FENCED_KEYS: set = set() + +# fkey -> (weakref(local UntypedStorage), weakref(shared UntypedStorage)) +# recorded when the fence ran. Fence keys are RAW pointers, so a new +# allocation on a recycled VA (ABA) is only detectable via these storage refs +# (identity-stable, unlike the provider's throwaway per-call views). +_MEGAMOE_FENCED_KEY_WS_REFS: dict = {} + + +def _megamoe_prune_fenced_keys(dead_keys) -> None: + """Drop ``dead_keys`` from ``_MEGAMOE_FENCED_KEYS`` and their recorded + workspace storage weakrefs (single helper so the two cannot drift).""" + _MEGAMOE_FENCED_KEYS.difference_update(dead_keys) + for _key in dead_keys: + _MEGAMOE_FENCED_KEY_WS_REFS.pop(_key, None) + + +def _megamoe_fence_key_live(fkey: Tuple[int, int, int]) -> bool: + """True iff ``fkey`` was fenced AND both recorded workspace storages are + still alive. + + A dead weakref means this pointer is a NEW allocation on a recycled VA: + skipping its fence would pair a virgin barrier side with the other side's + persisted phase (dispatch barrier spins forever). A new workspace + re-fences on EVERY rank (dead-ref prune or plain set miss), so the + collective fence cannot desync. Host-only; capture-safe. + """ + if fkey not in _MEGAMOE_FENCED_KEYS: + return False + refs = _MEGAMOE_FENCED_KEY_WS_REFS.get(fkey) + if refs is not None and refs[0]() is not None and refs[1]() is not None: + return True + _megamoe_prune_fenced_keys({fkey}) + return False + + +# Shared-workspace pointers whose pairings were replayed under CUDA-graph +# capture. A LATER eager fence must never zero such a shared side: captured +# pairings cannot re-fence -> phase-decoupled barrier -> silent all-rank hang. +_MEGAMOE_CAPTURED_SHARED_PTRS: set = set() + +# True once ANY MegaMoE launch was captured into a CUDA graph. Before that +# the whole local-workspace cache is safely evictable; afterwards only +# tuning-latched entries may go. See ``release_megamoe_profiling_scratch``. +_MEGAMOE_GRAPH_CAPTURE_SEEN: bool = False + # --------------------------------------------------------------------------- -# Tactic representation +# Tactic representation (v3: 8-tuple perf knobs) # --------------------------------------------------------------------------- # # A tactic is a tuple of JSON-friendly primitives (lists / ints / bools / -# strings) so it round-trips through ``json.dumps``/``json.loads`` *and* -# ``eval(repr(tactic))`` — both are required by ``TunableRunner`` cache -# serialization. Order matches the kernel constructor kwargs. +# strings / nested tuples) so it round-trips through ``json.dumps`` *and* +# ``eval(repr(tactic))`` (required by ``TunableRunner`` cache serialization). +# Order matches the kernel constructor kwargs. +# +# (mma_tiler_mnk, # list[int] of length 3 (M decides use_2cta) +# cluster_shape_mnk, # list[int] of length 3 +# group_hint, # int, TUNED (>=512); NOT max_active_clusters +# load_balance_mode, # str: "static" | "atomic_counter" +# token_back_mode, # "epi_warps" | "standalone_warps" | "reuse_dispatch_warps" +# use_bulk_fc2_store, # bool (bulk store valid only with epi_warps) +# flag_batch, # int >= 1 (standalone_warps requires == 1) +# epi_flag_batch) # (int, int) fc1/fc2 done-counter publish batch # -# (mma_tiler_mnk, # list[int] of length 3 -# cluster_shape_mnk, # list[int] of length 3 -# use_2cta_instrs, # bool -# resolved_group_hint, # int (always resolved before cache lookup) -# load_balance_mode, # str: "static" | "atomic_counter" -# use_bf16_redg) # bool: form A (False) vs form B (True) +# Derived / out-of-tuple: +# use_2cta_instrs = (mma_tiler_mnk[0] == 256), derived in _build_kernel. +# in_kernel_fc2_reduce -- functional config in runner unique_id, NOT a perf knob. # # Tuple wrapping makes the tactic hashable, which AutoTuner needs for the -# tactics cache. Lists nested inside the tuple are reconstructed from -# JSON intact. +# tactics cache. + +_TACTIC_LEN = 8 + +# Kernel-side ceiling: ``flag_batch`` is hard-checked ``[1, 32]`` and +# ``epi_flag_batch`` entries are SILENTLY clamped to ``[1, 32]``; reject > 32 +# here so a hand-supplied tactic fails fast instead of silently running at 32. +_FLAG_BATCH_MAX = 32 + + +def _unpack_tactic(tactic: Tuple) -> Tuple: + """Return the tactic's 8 fields in canonical order -- the single source of + truth for the field layout; every consumer unpacks through here. Plain + positional unpack, no validation (see :func:`validate_megamoe_tactic`).""" + ( + mma_tiler, + cluster_shape, + group_hint, + load_balance_mode, + token_back_mode, + use_bulk_fc2_store, + flag_batch, + epi_flag_batch, + ) = tactic + return ( + mma_tiler, + cluster_shape, + group_hint, + load_balance_mode, + token_back_mode, + use_bulk_fc2_store, + flag_batch, + epi_flag_batch, + ) -DEFAULT_MEGAMOE_TACTIC: Tuple[List[int], List[int], bool, int, str, bool] = ( - [128, 128, 256], - [1, 1, 1], - False, - 1, # placeholder; the launcher always resolves group_hint first - "static", - False, -) +def default_megamoe_tactic(num_tokens: int) -> Tuple: + """Deterministic token-bucket fallback tactic (autotune disabled / + cache miss / tactic=-1); never profiled by the autotuner.""" + if num_tokens <= 1024: + # decode winner: N128 only helps epi_warps + bulk. + return ([256, 128, 256], [2, 1, 1], 512, "static", "epi_warps", True, 1, (1, 1)) + if num_tokens <= 8192: + return ([256, 256, 256], [2, 1, 1], 512, "static", "epi_warps", True, 4, (1, 1)) + # prefill: atomic_counter only helps the very large tail (>=16384). + return ( + [256, 256, 256], + [2, 1, 1], + 512, + "atomic_counter" if num_tokens >= 16384 else "static", + "reuse_dispatch_warps", + False, + 8, + (2, 4), + ) + + +# --------------------------------------------------------------------------- +# Curated tuning space (~36 candidates per token bucket), filtered through +# ``validate_megamoe_tactic``. MUST be identical on every EP rank: the MERGE +# tuning sweep runs the candidate list in lockstep across ranks, so there is +# deliberately NO runtime knob selecting a different space. +# --------------------------------------------------------------------------- +# token_back_mode -> the only legal use_bulk_fc2_store (bulk binds to epi_warps). +_TOKEN_BACK_STORE_BINDING: dict = { + "epi_warps": True, + "reuse_dispatch_warps": False, + "standalone_warps": False, +} -# Candidate tactic geometries derived from the upstream functional test -# matrix ``moe_nvfp4_swapab/run_mega_tests.sh`` (M01..M20). Each entry is -# ``(mma_tiler_mnk, cluster_shape_mnk, use_2cta_instrs)``. Other tactic -# fields (load_balance_mode, use_bf16_redg) are intentionally constrained -# here. Expanding either axis needs the backend buffer contract and tests to -# move with it. -_RUN_MEGA_TESTS_CANDIDATE_GEOMETRIES: Tuple[ - Tuple[Tuple[int, int, int], Tuple[int, int, int], bool], ... -] = ( - ((128, 128, 256), (1, 1, 1), False), - ((256, 256, 256), (2, 1, 1), True), - ((256, 256, 256), (4, 1, 1), True), +# (mma_tiler_mnk, cluster_shape_mnk) geometries. use_2cta is derived (M==256). +# N64, M128/1-CTA and cluster_m4 never won a bucket upstream. +_GEOMETRIES: Tuple[Tuple[Tuple[int, int, int], Tuple[int, int, int]], ...] = ( + ((256, 256, 256), (2, 1, 1)), + ((256, 128, 256), (2, 1, 1)), ) +# token_back modes scanned (standalone_warps dropped: never a sole winner). +_TOKEN_BACK_MODES: Tuple[str, ...] = ("epi_warps", "reuse_dispatch_warps") + # Load-balance modes supported by the integrated fused FC12 path (see # ImplDesc.__post_init__ in fc1_fc2_fuse_sched.py). # ``clc`` is intentionally excluded -- it routes through a separate # scheduler class not wired through the fused FC12 kernel here. _LOAD_BALANCE_MODE_CANDIDATES: Tuple[str, ...] = ("static", "atomic_counter") +# Quantized combine (fp8/fp4) and form-B reject the bulk fc2 store at kernel +# construction; this is their deterministic default and sizing-probe tactic. +_MEGAMOE_NONBULK_STANDALONE_TACTIC: Tuple = ( + [256, 256, 256], + [2, 1, 1], + 512, + "static", + "standalone_warps", + False, + 1, + (2, 4), +) + +# group_hint saturates ~512; omission (-> ~74) is worst. +_GROUP_HINTS: Tuple[int, ...] = (512, 1024) + +_FLAG_BATCHES: Tuple[int, ...] = (1, 4, 8) + +# epi_flag_batch is fixed per token bucket, NOT a free scan axis; see +# ``_epi_flag_batch_for_tokens``. +_EPI_FLAG_BATCH_SMALL: Tuple[int, int] = (1, 1) +_EPI_FLAG_BATCH_LARGE: Tuple[int, int] = (2, 4) +_EPI_FLAG_BATCH_TOKEN_THRESHOLD = 8192 + + +def _epi_flag_batch_for_tokens(num_tokens: int) -> Tuple[int, int]: + """Token-bucket epi_flag_batch: ``<=8192 -> (1,1)``, ``>8192 -> (2,4)``.""" + if num_tokens > _EPI_FLAG_BATCH_TOKEN_THRESHOLD: + return _EPI_FLAG_BATCH_LARGE + return _EPI_FLAG_BATCH_SMALL + + # Kernel-construction knobs locked until the backend owns the corresponding # runtime buffer / scheduler contracts. _LOCKED_KERNEL_KWARGS = { @@ -151,11 +303,10 @@ def megamoe_activation_sf_bytes_per_row(hidden_size: int) -> int: if hidden_size <= 0 or hidden_size % 32 != 0: raise ValueError(f"hidden_size must be a positive multiple of 32, got {hidden_size}") # ceil(hidden / 16) rounded up to multiples of 4 FP8 columns - # (= `round_up(ceil(hidden_size / scaling_vector_size), 4)`), matching + # (= `pad_up(ceil_div(hidden_size, scaling_vector_size=16), 4)`), matching # the kernel's TMA load width and the ``can_implement`` hidden_size # alignment rule. - sf_cols = (hidden_size + 15) // 16 - return ((sf_cols + 3) // 4) * 4 + return pad_up(ceil_div(hidden_size, 16), 4) def validate_megamoe_tactic(tactic: Tuple) -> None: @@ -170,13 +321,21 @@ def validate_megamoe_tactic(tactic: Tuple) -> None: SupportedMmaTileN, ) - if (not isinstance(tactic, tuple)) or len(tactic) != 6: + if (not isinstance(tactic, tuple)) or len(tactic) != _TACTIC_LEN: raise ValueError( - f"MegaMoE tactic must be a 6-tuple, got {type(tactic).__name__}={tactic!r}" + f"MegaMoE tactic must be an {_TACTIC_LEN}-tuple, got " + f"{type(tactic).__name__} len={len(tactic) if isinstance(tactic, tuple) else 'NA'}={tactic!r}" ) - (mma_tiler, cluster_shape, use_2cta, resolved_group_hint, load_balance_mode, use_bf16_redg) = ( - tactic - ) + ( + mma_tiler, + cluster_shape, + group_hint, + load_balance_mode, + token_back_mode, + use_bulk_fc2_store, + flag_batch, + epi_flag_batch, + ) = _unpack_tactic(tactic) if (not isinstance(mma_tiler, (list, tuple))) or len(mma_tiler) != 3: raise ValueError(f"mma_tiler_mnk must be a 3-tuple/list, got {mma_tiler!r}") @@ -195,6 +354,8 @@ def validate_megamoe_tactic(tactic: Tuple) -> None: f"_validate_mma_*." ) + use_2cta = mma_tiler[0] == 256 + if (not isinstance(cluster_shape, (list, tuple))) or len(cluster_shape) != 3: raise ValueError(f"cluster_shape_mnk must be a 3-tuple/list, got {cluster_shape!r}") if cluster_shape[2] != 1: @@ -215,87 +376,96 @@ def validate_megamoe_tactic(tactic: Tuple) -> None: f"{cluster_shape[0] * cluster_shape[1]}." ) - if not isinstance(use_2cta, bool): - raise ValueError(f"use_2cta_instrs must be bool, got {use_2cta!r}.") - expected_2cta = mma_tiler[0] == 256 - if use_2cta != expected_2cta: - raise ValueError( - f"use_2cta_instrs must be {expected_2cta} for mma_tiler_mnk[0]={mma_tiler[0]}, got {use_2cta}." - ) - if cluster_shape[0] % (2 if use_2cta else 1) != 0: raise ValueError( f"cluster_shape_mnk[0] ({cluster_shape[0]}) must be divisible by " - f"{(2 if use_2cta else 1)} when use_2cta_instrs={use_2cta}." + f"{(2 if use_2cta else 1)} when use_2cta_instrs={use_2cta} " + f"(derived from mma_tiler_mnk[0]={mma_tiler[0]})." ) - if (not isinstance(resolved_group_hint, int)) or resolved_group_hint <= 0: - raise ValueError( - f"resolved_group_hint must be a positive int (resolved before " - f"cache lookup), got {resolved_group_hint!r}." - ) + if (not isinstance(group_hint, int)) or isinstance(group_hint, bool) or group_hint < 512: + raise ValueError(f"group_hint must be an int >= 512, got {group_hint!r}.") if load_balance_mode not in {"static", "atomic_counter"}: raise ValueError( f"load_balance_mode must be 'static' or 'atomic_counter', got {load_balance_mode!r}." ) - if not isinstance(use_bf16_redg, bool): - raise ValueError(f"use_bf16_redg must be bool, got {use_bf16_redg!r}.") - if use_bf16_redg: + if token_back_mode not in {"epi_warps", "standalone_warps", "reuse_dispatch_warps"}: raise ValueError( - "use_bf16_redg=True (form-B in-kernel top-k reduction) is not " - "wired in MegaMoECuteDsl yet. The backend allocates form-A " - "combine_output with shape (T, top_k, hidden) and performs the " - "top-k reduction on the host, so cached or manually supplied " - "form-B tactics are rejected." + f"token_back_mode must be one of 'epi_warps' / 'standalone_warps' / " + f"'reuse_dispatch_warps', got {token_back_mode!r}." ) + if not isinstance(use_bulk_fc2_store, bool): + raise ValueError(f"use_bulk_fc2_store must be bool, got {use_bulk_fc2_store!r}.") + if use_bulk_fc2_store and token_back_mode != "epi_warps": # nosec B105 + raise ValueError( + f"use_bulk_fc2_store=True requires token_back_mode='epi_warps', got " + f"{token_back_mode!r} (non-epi token-back forces non-bulk fc2 store)." + ) + if mma_tiler[1] == 128 and token_back_mode != "epi_warps": # nosec B105 + raise ValueError( + f"mma_tiler_mnk[1]=128 (N128) is only recommended with " + f"token_back_mode='epi_warps'; got {token_back_mode!r}." + ) -def resolve_megamoe_group_hint(cluster_shape_mnk: Tuple[int, int, int]) -> int: - """Resolve ``group_hint=None`` to ``HardwareInfo().get_max_active_clusters``. - - The kernel uses ``group_hint`` as a construction-time constant - (``Sm100MegaMoEKernel.__init__``); caching under ``None`` would - produce a wrong cache key. Falls back to 1 on hosts without - CUDA / Cutlass DSL so the tactic remains JSON-serializable. - """ - cluster_size = cluster_shape_mnk[0] * cluster_shape_mnk[1] * cluster_shape_mnk[2] - if cluster_size <= 0: - cluster_size = 1 - try: - from cutlass.utils import HardwareInfo + if ( + (not isinstance(flag_batch, int)) + or isinstance(flag_batch, bool) + or not (1 <= flag_batch <= _FLAG_BATCH_MAX) + ): + raise ValueError( + f"flag_batch must be an int in [1, {_FLAG_BATCH_MAX}] (kernel " + f"TokenInPullTokenBackPush hard limit), got {flag_batch!r}." + ) + if token_back_mode == "standalone_warps" and flag_batch != 1: # nosec B105 + raise ValueError( + f"token_back_mode='standalone_warps' requires flag_batch == 1, got {flag_batch}." + ) - return max(1, int(HardwareInfo().get_max_active_clusters(cluster_size))) - except Exception: # pragma: no cover - host without CUDA / Cutlass DSL - return 1 + if (not isinstance(epi_flag_batch, (tuple, list))) or len(epi_flag_batch) != 2: + raise ValueError(f"epi_flag_batch must be a 2-tuple (fc1, fc2), got {epi_flag_batch!r}.") + for v in epi_flag_batch: + if (not isinstance(v, int)) or isinstance(v, bool) or not (1 <= v <= _FLAG_BATCH_MAX): + raise ValueError( + f"epi_flag_batch entries must be ints in [1, {_FLAG_BATCH_MAX}] " + f"(epilogue clamp range), got {epi_flag_batch!r}." + ) -def enumerate_megamoe_candidate_tactics() -> List[Tuple]: - """Return the integrated candidate tactic list, fully resolved. +def enumerate_megamoe_candidate_tactics(num_tokens: int) -> List[Tuple]: + """Return the curated candidate tactic list for the current token bucket. - Each candidate has its ``resolved_group_hint`` stamped to the value - returned by ``HardwareInfo.get_max_active_clusters`` for that - cluster shape. Form A is the only supported reduction mode until the - backend wires the form-B output buffer and reduction path. + Cartesian product over the curated dimension ranges, filtered through + ``validate_megamoe_tactic``. ``epi_flag_batch`` is fixed per token + bucket (see ``_epi_flag_batch_for_tokens``). """ + epi_flag_batch = _epi_flag_batch_for_tokens(num_tokens) + candidates: List[Tuple] = [] - for mma_tiler, cluster_shape, use_2cta in _RUN_MEGA_TESTS_CANDIDATE_GEOMETRIES: - for load_balance_mode in _LOAD_BALANCE_MODE_CANDIDATES: - tactic = ( - list(mma_tiler), - list(cluster_shape), - use_2cta, - resolve_megamoe_group_hint(cluster_shape), - load_balance_mode, - False, - ) - try: - validate_megamoe_tactic(tactic) - except ValueError as e: - logger.debug(f"[MegaMoE] dropping candidate tactic {tactic!r}: {e}") - continue - candidates.append(tactic) + for mma_tiler, cluster_shape in _GEOMETRIES: + for token_back_mode in _TOKEN_BACK_MODES: + use_bulk = _TOKEN_BACK_STORE_BINDING[token_back_mode] + for load_balance_mode in _LOAD_BALANCE_MODE_CANDIDATES: + for group_hint in _GROUP_HINTS: + for flag_batch in _FLAG_BATCHES: + tactic = ( + list(mma_tiler), + list(cluster_shape), + int(group_hint), + load_balance_mode, + token_back_mode, + use_bulk, + int(flag_batch), + tuple(epi_flag_batch), + ) + try: + validate_megamoe_tactic(tactic) + except ValueError as e: + logger.debug(f"[MegaMoE] dropping candidate tactic {tactic!r}: {e}") + continue + candidates.append(tactic) return candidates @@ -310,8 +480,6 @@ def enumerate_megamoe_candidate_tactics() -> List[Tuple]: # op is unregistered. try: import cutlass - import cutlass.cute as cute - import cutlass.torch as cutlass_torch import torch.distributed as dist import torch.distributed._symmetric_memory as torch_symm_mem from cutlass.cute.nvgpu import cpasync, tcgen05 # noqa: F401 @@ -328,7 +496,6 @@ def enumerate_megamoe_candidate_tactics() -> List[Tuple]: Nvfp4BlockSize, # noqa: F401 SfPaddingBlock, ) - from ..cute_dsl_kernels.mega_moe_nvfp4.sym_buffer import SymBufferHost IS_MEGAMOE_OP_AVAILABLE = True except Exception as _megamoe_import_err: # pragma: no cover - env-specific @@ -357,51 +524,89 @@ def enumerate_megamoe_candidate_tactics() -> List[Tuple]: # multi-rank and is supplied by the caller (the MegaMoECuteDsl backend's # MegaMoeSymmMemProvider carves it out of the rendezvous'd buffer). _MEGAMOE_LOCAL_WORKSPACE_CACHE: dict = {} + # Cache keys allocated during the autotune sweep. Each candidate has a + # DISTINCT key (workspace SIZE varies with the tactic) and a multi-GiB + # workspace; persisting all would OOM, so each candidate's workspace is + # freed when the next one allocates (only the winner is reused). + _MEGAMOE_TUNING_WORKSPACE_KEYS: set = set() + + @functools.lru_cache(maxsize=1) + def _cute_launch_helpers(): + """Import the cutlass-dsl launch surface once per process and build the + tensor/pointer converters (hoisted out of forward(); lazy so the + module imports without cutlass-dsl / a GPU).""" + import cutlass + import cutlass.cute as cute + import cutlass.torch as cutlass_torch + import cutlass.utils as cutlass_utils + from cutlass.cute.typing import AddressSpace + + from ..cute_dsl_kernels.mega_moe_nvfp4.sym_buffer import SymBufferHost + + def to_cute(t, assumed_align=16): + ct = cutlass_torch.from_dlpack(t, assumed_align=assumed_align) + return ct.mark_layout_dynamic(leading_dim=cutlass_torch.get_leading_dim(t)) + + def to_cute_ptr(t, assumed_align=16): + return cute.runtime.make_ptr( + cutlass.Uint8, t.data_ptr(), AddressSpace.gmem, assumed_align=assumed_align + ) + + return to_cute, to_cute_ptr, SymBufferHost, cute, cutlass_utils + + def _evict_latched_tuning_workspaces() -> None: + """Pop every tuning-latched local workspace and prune its fence keys. + + Fence keys hold the raw local ptr; if the allocator hands the freed + block to the next candidate, a stale key would ABA-skip the + barrier-reset fence and hang the dispatch barrier mid-sweep. + """ + for stale_key in _MEGAMOE_TUNING_WORKSPACE_KEYS: + stale_ws = _MEGAMOE_LOCAL_WORKSPACE_CACHE.pop(stale_key, None) + if stale_ws is not None: + _stale_ptr = int(stale_ws.data_ptr()) + _megamoe_prune_fenced_keys({k for k in _MEGAMOE_FENCED_KEYS if k[0] == _stale_ptr}) + _MEGAMOE_TUNING_WORKSPACE_KEYS.clear() def _get_or_alloc_local_workspace( - kernel, cache_key: Tuple, device: torch.device + kernel, cache_key: Tuple, device: torch.device, latch_in_tuning_mode: bool = True ) -> torch.Tensor: cached = _MEGAMOE_LOCAL_WORKSPACE_CACHE.get(cache_key) if cached is not None: return cached + # Cache MISS = new (tactic, shape): release any workspace latched for + # a previously-profiled candidate (never relaunched; profiling is + # eager, so no captured graph holds the buffer). Within one + # candidate's multi-launch profiling the key HITs, so nothing is + # freed mid-candidate; outside tuning the latch set is empty. + if _MEGAMOE_TUNING_WORKSPACE_KEYS: + _evict_latched_tuning_workspaces() local_bytes, _ = kernel.get_workspace_sizes() - # MUST be zero-initialised: the local workspace embeds Int32 - # atomic counters (l1_arrival_count, fc1_done_counter, - # grid_sync_counter) whose spin_wait expects v >= positive - # threshold; a stray negative byte from ``torch.empty`` makes - # the wait unsatisfiable and hangs the kernel at 100% SM. - local_workspace = torch.zeros(local_bytes, dtype=torch.uint8, device=device) + # Only the atomic counters need zeroing (a stray negative byte hangs + # spin_wait at 100% SM); the multi-GiB bulk is overwritten every launch + # (full ``torch.zeros`` = huge wasted memset). Zero the leading counter + # prefix plus the persisted ``nvlink_barrier_counter``, deliberately + # OUTSIDE that prefix so the device tail-reset never touches it. + local_workspace = torch.empty(local_bytes, dtype=torch.uint8, device=device) + _lead = int(kernel.require_zero_workspace_leading_bytes[0]) + local_workspace[:_lead].zero_() + _bc_off = int(kernel._local_offsets["nvlink_barrier_counter"]) + _bc_n = int(kernel._local_region_by_name["nvlink_barrier_counter"].nbytes) + local_workspace[_bc_off : _bc_off + _bc_n].zero_() _MEGAMOE_LOCAL_WORKSPACE_CACHE[cache_key] = local_workspace + # ``latch_in_tuning_mode=False`` (tactic_autotune opted OUT): launches + # inside a global autotune() are real fallback-tactic runs, not sweep + # candidates, so their workspaces must persist. + if latch_in_tuning_mode and AutoTuner.get().is_tuning_mode: + _MEGAMOE_TUNING_WORKSPACE_KEYS.add(cache_key) return local_workspace - def _zero_local_workspace_preserving_phase(local_workspace, kernel) -> None: - """Per-launch zero of the local workspace that PRESERVES the - self-priming ``nvlink_barrier_counter`` region (multi-rank EP path). - - The kernel's reusable phase-flip NVLink barrier keeps its cross-rank - ``nvlink_barrier_signal`` (in the symmetric shared workspace, which is - NOT re-zeroed per launch) in lockstep with this per-rank - ``nvlink_barrier_counter``. Re-zeroing the counter while the signal is - not reset would decouple the phase and deadlock the barrier. Every - other local counter (l1_arrival_count, fc1_done_counter, - fc2_done_counter, expert_send_count, ...) still needs a per-launch - reset, so we zero the whole buffer except the counter's byte range. - """ - off = int(kernel._local_offsets["nvlink_barrier_counter"]) - nbytes = int(kernel._local_region_by_name["nvlink_barrier_counter"].nbytes) - total = local_workspace.numel() - if off > 0: - local_workspace[:off].zero_() - end = off + nbytes - if end < total: - local_workspace[end:].zero_() - # ----- Symmetric-memory provider (NVSHMEM-equivalent) ------------------- # # PyTorch's ``torch.distributed._symmetric_memory`` is an NVSHMEM-equivalent # symmetric-heap provider built on cuMem APIs. It exposes per-rank buffer # pointers (``handle.buffer_ptrs``) which we use to populate the - # ``SymBufferHost(base_addr, offsets, rank_idx, num_max_ranks)`` payload + # ``SymBufferHost(offsets, rank_idx, num_max_ranks)`` payload # the MegaMoE kernel expects. # # We allocate ONE large symmetric buffer per (group, layout_key) and @@ -416,9 +621,6 @@ def _zero_local_workspace_preserving_phase(local_workspace, kernel) -> None: # ``_MEGA_MOE_SYMM_BUFFER_CACHE`` in ``mega_moe_deepgemm.py``). _MEGAMOE_SYMM_PROVIDER_CACHE: dict = {} - def _round_up_to(value: int, alignment: int) -> int: - return ((value + alignment - 1) // alignment) * alignment - @dataclasses.dataclass class MegaMoeSymmRegions: """User-domain symmetric tensors carved out of a single rendezvous'd @@ -439,7 +641,8 @@ class MegaMoeSymmRegions: # kernel sf_addr formula in dispatch_kernel.py. activation_sf: torch.Tensor # (max_T, sf_bytes_per_row) uint8 (FP8 SF) topk_weights: torch.Tensor # (max_T, num_topk) float32 - combine_output: torch.Tensor # (max_T, num_topk, hidden) output_dtype + # (max_T, 1, hidden): both reduction forms collapse the top-k axis in-op. + combine_output: torch.Tensor shared_workspace: torch.Tensor # (shared_ws_bytes,) uint8 peer_offsets: List[int] # symmetric peer-pointer deltas rank: int @@ -504,6 +707,10 @@ def __init__( self.max_tokens_per_rank = int(max_tokens_per_rank) self.num_topk = int(num_topk) self.output_dtype = output_dtype + # Kernel output is unified (T, hidden), so the symmetric combine + # region is (max_T, 1, hidden). combine_k MUST match the kernel + # ``combine_output`` shape or the region is silently corrupted. + self.combine_k = 1 # Region byte sizes (worst case across launches; staging # writes only the live ``T`` rows). NVFP4 packs 2 elems / byte @@ -515,17 +722,13 @@ def __init__( act_bytes_per_row = hidden_size // 2 sf_bytes_per_row = megamoe_activation_sf_bytes_per_row(hidden_size) topkw_bytes_per_row = num_topk * 4 # float32 - combine_bytes_per_row = num_topk * hidden_size * output_dtype.itemsize + combine_bytes_per_row = self.combine_k * hidden_size * output_dtype.itemsize - act_region = _round_up_to(max_tokens_per_rank * act_bytes_per_row, self._REGION_ALIGN) - sf_region = _round_up_to(max_tokens_per_rank * sf_bytes_per_row, self._REGION_ALIGN) - topkw_region = _round_up_to( - max_tokens_per_rank * topkw_bytes_per_row, self._REGION_ALIGN - ) - combine_region = _round_up_to( - max_tokens_per_rank * combine_bytes_per_row, self._REGION_ALIGN - ) - shared_region = _round_up_to(shared_workspace_bytes, self._REGION_ALIGN) + act_region = pad_up(max_tokens_per_rank * act_bytes_per_row, self._REGION_ALIGN) + sf_region = pad_up(max_tokens_per_rank * sf_bytes_per_row, self._REGION_ALIGN) + topkw_region = pad_up(max_tokens_per_rank * topkw_bytes_per_row, self._REGION_ALIGN) + combine_region = pad_up(max_tokens_per_rank * combine_bytes_per_row, self._REGION_ALIGN) + shared_region = pad_up(shared_workspace_bytes, self._REGION_ALIGN) self._region_offsets: dict = {} self._region_sizes: dict = {} @@ -562,7 +765,7 @@ def __init__( peer_ptr = int(self._handle.buffer_ptrs[r]) self.peer_offsets.append(peer_ptr - local_base) - logger.info( + logger.debug( "[MegaMoeSymmMemProvider] group=%s rank=%d/%d total_bytes=%d " "(activation=%d sf=%d topk_weights=%d combine=%d shared=%d)", self.group_name, @@ -591,6 +794,7 @@ def get_regions(self) -> MegaMoeSymmRegions: hidden = self.hidden_size max_t = self.max_tokens_per_rank top_k = self.num_topk + combine_k = self.combine_k # ``sf_bytes_per_row`` MUST match the byte width used at # allocation time (``__init__`` above) and at the backend's # ``quantize_input`` output: kernel reads @@ -606,7 +810,7 @@ def get_regions(self) -> MegaMoeSymmRegions: ), topk_weights=self._region_view("topk_weights", (max_t, top_k), torch.float32), combine_output=self._region_view( - "combine_output", (max_t, top_k, hidden), self.output_dtype + "combine_output", (max_t, combine_k, hidden), self.output_dtype ), shared_workspace=self._region_view( "shared_workspace", (self._region_sizes["shared_workspace"],), torch.uint8 @@ -625,6 +829,7 @@ def get_megamoe_symm_provider( max_tokens_per_rank: int, num_topk: int, output_dtype: torch.dtype, + combine_format: str, shared_workspace_bytes: int, ) -> MegaMoeSymmMemProvider: """Return a cached provider for (group, layout). The cache is @@ -648,6 +853,7 @@ def get_megamoe_symm_provider( int(max_tokens_per_rank), int(num_topk), str(output_dtype), + str(combine_format), int(shared_workspace_bytes), ) cached = _MEGAMOE_SYMM_PROVIDER_CACHE.get(cache_key) @@ -666,10 +872,137 @@ def get_megamoe_symm_provider( _MEGAMOE_SYMM_PROVIDER_CACHE[cache_key] = provider return provider + # ---- AutoTuner profiling scratch (symmetric, transient) --------------- + # The kernel's SINGLE ``peer_rank_ptr_mapper`` requires ALL cross-rank + # tensors to live in one symmetric allocation at consistent offsets; the + # AutoTuner's regenerated NON-symmetric inputs would peer-map outside any + # symmetric region (multi-rank IMA). So profiling runs on a SEPARATE + # symmetric scratch (never the real staging buffer), shared across + # same-layout layers and freed by ``release_megamoe_profiling_scratch()``. + _MEGAMOE_PROFILING_SCRATCH_CACHE: dict = {} + _ACTIVE_MEGAMOE_PROFILING_SCRATCH = None + # Deferred variant: zero-arg callable returning MegaMoeSymmRegions, set + # when the scratch is not allocated. The profiling pre-hook calls it only + # when REAL profiling launches, so a tuning-mode cache HIT never pays the + # multi-GiB allocation. + _ACTIVE_MEGAMOE_PROFILING_SCRATCH_FACTORY = None + + def get_megamoe_profiling_scratch( + *, + process_group, + world_size: int, + rank: int, + hidden_size: int, + max_tokens_per_rank: int, + num_topk: int, + output_dtype: torch.dtype, + combine_format: str, + shared_workspace_bytes: int, + ): + """Return a cached, transient symmetric scratch provider used ONLY for + AutoTuner profiling; ``None`` for single-rank.""" + if int(world_size) <= 1: + return None + if not hasattr(process_group, "group_name"): + raise RuntimeError( + "get_megamoe_profiling_scratch requires a ProcessGroup with " + ".group_name (mapping.moe_ep_group_pg)." + ) + cache_key = ( + str(process_group.group_name), + int(hidden_size), + int(max_tokens_per_rank), + int(num_topk), + str(output_dtype), + str(combine_format), + int(shared_workspace_bytes), + ) + cached = _MEGAMOE_PROFILING_SCRATCH_CACHE.get(cache_key) + if cached is not None: + return cached + provider = MegaMoeSymmMemProvider( + process_group=process_group, + world_size=world_size, + rank=rank, + hidden_size=hidden_size, + max_tokens_per_rank=max_tokens_per_rank, + num_topk=num_topk, + output_dtype=output_dtype, + shared_workspace_bytes=shared_workspace_bytes, + ) + _MEGAMOE_PROFILING_SCRATCH_CACHE[cache_key] = provider + return provider + + def set_active_megamoe_profiling_scratch(regions) -> None: + """Set (``None`` clears) the :class:`MegaMoeSymmRegions` used for + AutoTuner profiling on the current call (set by the backend around + the op invocation; consumed inside ``choose_one``).""" + global _ACTIVE_MEGAMOE_PROFILING_SCRATCH + _ACTIVE_MEGAMOE_PROFILING_SCRATCH = regions + + def set_active_megamoe_profiling_scratch_factory(factory) -> None: + """Set (``None`` clears) a zero-arg factory returning the profiling + scratch regions; used instead of the eager setter when the scratch is + not allocated. See ``_ACTIVE_MEGAMOE_PROFILING_SCRATCH_FACTORY``.""" + global _ACTIVE_MEGAMOE_PROFILING_SCRATCH_FACTORY + _ACTIVE_MEGAMOE_PROFILING_SCRATCH_FACTORY = factory + + def release_megamoe_profiling_scratch() -> None: + """Free all profiling-scratch symmetric buffers (call after warmup); + profiling re-allocates lazily if it runs again.""" + global _ACTIVE_MEGAMOE_PROFILING_SCRATCH + global _ACTIVE_MEGAMOE_PROFILING_SCRATCH_FACTORY + _ACTIVE_MEGAMOE_PROFILING_SCRATCH = None + _ACTIVE_MEGAMOE_PROFILING_SCRATCH_FACTORY = None + # Prune fence keys pairing with the dying scratch shared buffers: a + # future allocation on the recycled VA must re-fence (ABA). + _dead_shared = { + int(_p.get_regions().shared_workspace.data_ptr()) + for _p in _MEGAMOE_PROFILING_SCRATCH_CACHE.values() + } + if _dead_shared: + _megamoe_prune_fenced_keys({k for k in _MEGAMOE_FENCED_KEYS if k[1] in _dead_shared}) + _MEGAMOE_PROFILING_SCRATCH_CACHE.clear() + # EVICT stale local workspaces -- do not merely clear the latch: the + # latch may hold the LAST LOSER, and a losing default's multi-GiB + # workspace would shrink the KV-cache budget for the process lifetime. + # Before the FIRST capture nothing holds raw pointers, so the whole + # cache is evictable; after a capture only latched entries go. + if not _MEGAMOE_GRAPH_CAPTURE_SEEN: + for _key, _ws in list(_MEGAMOE_LOCAL_WORKSPACE_CACHE.items()): + _ptr = int(_ws.data_ptr()) + _megamoe_prune_fenced_keys({k for k in _MEGAMOE_FENCED_KEYS if k[0] == _ptr}) + del _MEGAMOE_LOCAL_WORKSPACE_CACHE[_key] + _MEGAMOE_TUNING_WORKSPACE_KEYS.clear() + else: + _evict_latched_tuning_workspaces() + + def reset_megamoe_workspace_state() -> None: + """TEST/BENCH ONLY: drop ALL process-global MegaMoE workspace state. + + PRECONDITION: every CUDA graph that replayed a MegaMoE launch is + destroyed and nothing will launch on the dropped workspaces. Fence / + captured-ptr sets are cleared TOGETHER with the buffer caches (stale + entries on recycled addresses would ABA-skip the fence or false-positive + the capture guard). Must run in lockstep on ALL EP ranks. + """ + global _ACTIVE_MEGAMOE_PROFILING_SCRATCH + global _ACTIVE_MEGAMOE_PROFILING_SCRATCH_FACTORY + global _MEGAMOE_GRAPH_CAPTURE_SEEN + _ACTIVE_MEGAMOE_PROFILING_SCRATCH = None + _ACTIVE_MEGAMOE_PROFILING_SCRATCH_FACTORY = None + _MEGAMOE_PROFILING_SCRATCH_CACHE.clear() + _MEGAMOE_SYMM_PROVIDER_CACHE.clear() + _MEGAMOE_LOCAL_WORKSPACE_CACHE.clear() + _MEGAMOE_TUNING_WORKSPACE_KEYS.clear() + _MEGAMOE_FENCED_KEYS.clear() + _MEGAMOE_FENCED_KEY_WS_REFS.clear() + _MEGAMOE_CAPTURED_SHARED_PTRS.clear() + _MEGAMOE_GRAPH_CAPTURE_SEEN = False + def query_megamoe_shared_workspace_bytes( *, world_size: int, - local_rank: int, num_topk: int, num_experts_per_rank: int, hidden_size: int, @@ -679,43 +1012,43 @@ def query_megamoe_shared_workspace_bytes( tactic: Optional[Tuple] = None, apply_topk_in_fc1: bool = True, gate_up_clamp: Optional[float] = None, + in_kernel_fc2_reduce: bool = False, + combine_format: str = "bf16", ) -> int: """Probe ``Sm100MegaMoEKernel.get_workspace_sizes()`` for the - shared workspace byte count. The shared workspace size is + shared workspace byte count. The SHARED workspace size is invariant across all candidate tactics and across the codegen-time graph/clamp modes (its regions depend only on world_size / num_experts_per_rank / num_topk / max_tokens_per_rank -- see _build_shared_region_specs in megamoe_kernel.py), so we use the - default tactic for the probe. ``apply_topk_in_fc1`` / ``gate_up_clamp`` - are still threaded so the probe kernel ctor signature is satisfied - and matches the real build. + default 8-tuple tactic for the probe; the remaining kwargs are + threaded only to satisfy the kernel ctor and match the real build. """ - from ..cute_dsl_kernels.mega_moe_nvfp4 import import_kernel if tactic is None: - cluster = tuple(DEFAULT_MEGAMOE_TACTIC[1]) - tactic = ( - list(DEFAULT_MEGAMOE_TACTIC[0]), - list(cluster), - DEFAULT_MEGAMOE_TACTIC[2], - resolve_megamoe_group_hint(cluster), - DEFAULT_MEGAMOE_TACTIC[4], - DEFAULT_MEGAMOE_TACTIC[5], - ) + # Sizing is tactic-invariant, but quantized combine (fp8/fp4) + # rejects the bulk fc2 store at kernel construction, so those + # modes must probe with the non-bulk standalone tactic. + if combine_format != "bf16": + tactic = _MEGAMOE_NONBULK_STANDALONE_TACTIC + else: + tactic = default_megamoe_tactic(0) ( mma_tiler, cluster_shape, - use_2cta, - resolved_group_hint, + group_hint, load_balance_mode, - use_bf16_redg, - ) = tactic - kernel_cls = import_kernel() - probe = kernel_cls( - mma_tiler_mnk=tuple(mma_tiler), + token_back_mode, + use_bulk_fc2_store, + flag_batch, + epi_flag_batch, + ) = _unpack_tactic(tactic) + mma_tiler = tuple(mma_tiler) + common = dict( + mma_tiler_mnk=mma_tiler, cluster_shape_mnk=tuple(cluster_shape), - use_2cta_instrs=bool(use_2cta), - group_hint=int(resolved_group_hint), + use_2cta_instrs=bool(mma_tiler[0] == 256), + group_hint=int(group_hint), token_padding_block=64, sf_padding_block=SfPaddingBlock, load_balance_mode=str(load_balance_mode), @@ -725,34 +1058,35 @@ def query_megamoe_shared_workspace_bytes( hidden_size, ), world_size=int(world_size), - local_rank=int(local_rank), num_topk=int(num_topk), max_tokens_per_rank=int(max_tokens_per_rank), hidden=int(hidden_size), fc2_output_dtype=cutlass.BFloat16, - in_kernel_fc2_reduce=bool(use_bf16_redg), + in_kernel_fc2_reduce=bool(in_kernel_fc2_reduce), + token_back_mode=str(token_back_mode), + non_ubulk_fc2_store=(not bool(use_bulk_fc2_store)), + flag_batch=int(flag_batch), + epi_flag_batch=tuple(epi_flag_batch), apply_topk_in_fc1=bool(apply_topk_in_fc1), gate_up_clamp=(None if gate_up_clamp is None else float(gate_up_clamp)), **_LOCKED_KERNEL_KWARGS, ) + # The probe MUST build the SAME kernel that runs (same combine_format): + # otherwise the provider carves an undersized shared region and the + # combine staging writes OOB (single-rank IMA; EP>1 looks like a hang). + kernel_cls, CombineFormat = _import_megamoe_kernel() + probe = kernel_cls(combine_format=CombineFormat.parse(combine_format), **common) _, shared_bytes = probe.get_workspace_sizes() return int(shared_bytes) - def _to_cute( - tensor: torch.Tensor, - assumed_align: int = 16, - force_static_layout: bool = False, - ) -> "cute.Tensor": - cute_tensor = cutlass_torch.from_dlpack(tensor, assumed_align=assumed_align) - # The local workspace's internal region offsets/strides are codegen-time - # static constants (see megamoe_kernel _layout_regions); marking it - # layout-dynamic invalidates those static accesses and corrupts the - # FC1-output / pool / counter regions. The upstream runner passes the - # local workspace with force_static_layout=True for exactly this reason. - if force_static_layout: - return cute_tensor - leading_dim = cutlass_torch.get_leading_dim(tensor) - return cute_tensor.mark_layout_dynamic(leading_dim=leading_dim) + def _megamoe_autotune_num_tokens(shapes: List[torch.Size]) -> int: + """Shape-derivation rule: the activation token count (input[0] dim0) + drives every other tensor's leading axis. + + Module-scope on purpose: ConstraintSpec hashes include the callable + and the AutoTuner lru_caches on it; a per-call closure would defeat + memoization and leak cache entries.""" + return shapes[0][0] class Sm100MegaMoENvfp4Runner(TunableRunner): """TunableRunner for the ported MegaMoE CuteDSL NVFP4 kernel. @@ -767,13 +1101,6 @@ class Sm100MegaMoENvfp4Runner(TunableRunner): # Module-scope compile cache shared by every runner instance. kernel_cache: dict = {} - # Module-scope tuning-config cache keyed on ``unique_id()``. The op - # rebuilds a runner per call, so an instance-level cache would never - # hit; keeping it at class scope amortizes the config build across - # calls (mirrors the ``tuning_config_cache`` of the CuteDSL - # grouped-gemm runners in ``cute_dsl_custom_ops.py``). - tuning_config_cache: dict = {} - def __init__( self, *, @@ -788,8 +1115,9 @@ def __init__( output_dtype: torch.dtype, apply_topk_in_fc1: bool = True, gate_up_clamp: Optional[float] = None, - token_back_by_dispatch: bool = False, - non_ubulk_fc2_store: bool = True, + in_kernel_fc2_reduce: bool = False, + combine_format: str = "bf16", + tactic_autotune: bool = False, ) -> None: super().__init__() if (sm_version := get_sm_version()) not in (100, 103): @@ -818,18 +1146,34 @@ def __init__( ) self.max_tokens_per_rank = int(max_tokens_per_rank) self.output_dtype = output_dtype - # Codegen-time graph/clamp modes. They change the generated - # kernel, so they are part of ``unique_id`` (and therefore the - # compile-cache key) -- never per-call runtime kwargs. + # Codegen-time modes: they change the generated kernel, so they are + # part of ``unique_id`` (compile + workspace cache key), never + # per-call runtime kwargs. self.apply_topk_in_fc1 = bool(apply_topk_in_fc1) self.gate_up_clamp = None if gate_up_clamp is None else float(gate_up_clamp) - self.token_back_by_dispatch = bool(token_back_by_dispatch) - self.non_ubulk_fc2_store = bool(non_ubulk_fc2_store) + # Symmetric profiling scratch, set by the op around ``choose_one`` + # so the pre-hook routes cross-rank inputs through symmetric + # memory; None outside tuning and for single-rank. + self._profiling_scratch = None + # Deferred scratch factory; the pre-hook materializes it on the + # first REAL profiling launch (tuning-mode cache HITs never allocate). + self._profiling_scratch_factory = None + self.in_kernel_fc2_reduce = bool(in_kernel_fc2_reduce) + # combine wire format: bf16 / 32e4m3xe8m0 (fp8) / 16e2m1xbf16 (fp4); + # quantized changes the shared_workspace size, so part of unique_id. + self.combine_format = str(combine_format) + # Tactic-autotune opt-in (default OFF). Does NOT change the + # generated kernel, so deliberately EXCLUDED from unique_id / + # _tactic_cache_key: opted-in and opted-out runs share caches. + self.tactic_autotune = bool(tactic_autotune) def unique_id(self): + # local_rank is intentionally excluded: every EP rank must run the + # SAME tactic and MERGE merges timings by cache key across ranks, + # so the key MUST be rank-identical. world_size stays so + # single-rank and multi-rank never share entries. return ( self.world_size, - self.local_rank, self.num_topk, self.num_experts_per_rank, self.hidden_size, @@ -839,8 +1183,8 @@ def unique_id(self): str(self.output_dtype), self.apply_topk_in_fc1, self.gate_up_clamp, - self.token_back_by_dispatch, - self.non_ubulk_fc2_store, + self.in_kernel_fc2_reduce, + self.combine_format, ) def get_valid_tactics( @@ -849,44 +1193,48 @@ def get_valid_tactics( profile: OptimizationProfile, **kwargs, ) -> List[Tuple]: - del inputs, profile, kwargs - return enumerate_megamoe_candidate_tactics() + del profile, kwargs + num_tokens = int(inputs[0].shape[0]) + candidates = enumerate_megamoe_candidate_tactics(num_tokens) + # Non-bulk is HARD for form-B (a bulk store collapses the K routes + # UNSUMMED -> silent wrong output) and fp4 combine (UBLK cannot + # scalar-deref sub-byte data -> kernel raises); fp8 bulk is legal + # and stays tunable. ``_unpack_tactic(t)[5]`` is use_bulk_fc2_store. + if self.in_kernel_fc2_reduce or self.combine_format.startswith("16e2m1"): + candidates = [t for t in candidates if not _unpack_tactic(t)[5]] + return candidates def _autotuner_inputs_pre_hook(self, inputs: List[torch.Tensor]) -> List[torch.Tensor]: """Sanitize ONLY the autotuner-regenerated fake inputs. - ``AutoTuner._prepare_input_tensors`` rebuilds fresh fake tensors - for the dynamic / constraint inputs -- activation (0), - activation_sf (1), topk_idx (2), topk_weights (3), - combine_output (11) -- and passes every STATIC input through BY - REFERENCE (``tensor = inputs[i]`` for non-dynamic dims). The static - inputs here are the caller's REAL weight-side tensors: fc1_weight - (4), fc1_weight_sf (5), fc2_weight (6), fc2_weight_sf (7), - fc1_alpha (8), fc2_alpha (9), fc1_norm_const (10). - - Therefore this hook must mirror ``CuteDslFusedMoE.inputs_pre_hook``: - only fix up the regenerated tensors and pass the real weights / - scales through untouched. The fresh ``topk_idx`` is filled with - random ints in ``[-5, 4]`` whose out-of-range values index a - per-CTA SMEM histogram + the peer-rank pointer table and trigger - illegal memory access, so we rewrite it to a valid round-robin; - the fresh ``activation_sf`` / ``topk_weights`` are zeroed to keep - the FP8/FP32 epilogue NaN-free (autotuning measures runtime, not - numerics). - - We intentionally do NOT touch indices 4-10. An in-place - ``zero_()`` / ``fill_()`` on those would permanently clobber the - caller's REAL per-expert weight scale factors / alphas (they are - not regenerated), zeroing the weight SF and forcing every - post-tuning forward to emit an all-zero ``combine_output``. The - real weights are already valid (no NaN, non-zero norm_const), so - they need no sanitization. This keeps the hook copy-free. + AutoTuner rebuilds inputs 0-3 and 11; the static inputs (4-10) are + the caller's REAL weights/scales, passed by reference -- filling + those would clobber them. topk_idx becomes a valid round-robin + (random ints index OOB); SF / weights are filled with 1.0, NOT 0 + (SF==0 degenerates the GEMMs and skews tactic timing). """ + # Runs ONLY on a real profiling MISS (a tuning-mode cache HIT + # early-returns in choose_one), so materializing the DEFERRED + # scratch here keeps cache-hitting forwards allocation-free. The + # factory's collective rendezvous is safe: MERGE lockstep tuning + # gets every EP rank to this pre-hook before any coupled launch. + if self._profiling_scratch is None and self._profiling_scratch_factory is not None: + self._profiling_scratch = self._profiling_scratch_factory() + if self._profiling_scratch is None and self.world_size > 1: + # Multi-rank profiling without symmetric scratch would peer-map + # non-symmetric tensors: cross-rank IMA. Fail loud. + raise RuntimeError( + "MegaMoE-CuteDSL autotune profiling requires the " + "symmetric profiling scratch on multi-rank, but none is " + "active (and no scratch factory produced one)." + ) inputs = list(inputs) total_experts = self.num_experts_per_rank * self.world_size if total_experts <= 0: return inputs + # topk_idx is consumed LOCALLY (not peer-mapped), so the + # regenerated tensor stays; just make the fake ids in-range. topk_idx = inputs[2] if isinstance(topk_idx, torch.Tensor) and topk_idx.dim() == 2: T, K = topk_idx.shape @@ -900,17 +1248,41 @@ def _autotuner_inputs_pre_hook(self, inputs: List[torch.Tensor]) -> List[torch.T ).view(T, K) topk_idx.copy_(valid) - # activation_sf (1) and topk_weights (3) are autotuner-regenerated - # fresh tensors; zero them to keep the FC1/FC2 epilogue paths - # NaN-free against random ``uint8`` -> FP8 reinterpretation. The - # weight SF (5, 7) and per-expert alphas (8, 9, 10) are the real, - # already-valid backend tensors and are deliberately left alone. + scratch = self._profiling_scratch + if scratch is not None: + # Multi-rank: route the CROSS-RANK inputs through the symmetric + # scratch (sliced to the regenerated token count) so the peer + # mapper resolves inside a symmetric region; staging untouched. + m = int(inputs[0].shape[0]) + act = scratch.activation[:m] + act.copy_(inputs[0].view(torch.uint8)) + inputs[0] = act.view(inputs[0].dtype) + + sf = scratch.activation_sf[:m] + sf.fill_(0x38) # raw byte == FP8 1.0 + inputs[1] = sf if inputs[1].dtype == torch.uint8 else sf.view(inputs[1].dtype) + + w = scratch.topk_weights[:m] + w.fill_(1.0) + inputs[3] = w + + comb = scratch.combine_output[:m] + comb.zero_() # form-B accumulates onto live rows; form-A overwrites + inputs[11] = comb + return inputs + + # Single-rank / scratch-disabled: sanitize in place. fill_(1.0) on + # the FP8 view writes byte 0x38; on a uint8 view write the raw + # byte. Do NOT fill_(0x38) on the FP8 view -- that is 56.0. activation_sf = inputs[1] if isinstance(activation_sf, torch.Tensor): - activation_sf.zero_() + if activation_sf.dtype == torch.float8_e4m3fn: + activation_sf.fill_(1.0) # FP8 1.0 (exact, byte 0x38) + else: + activation_sf.view(torch.uint8).fill_(0x38) # raw byte = FP8 1.0 topk_weights = inputs[3] if isinstance(topk_weights, torch.Tensor): - topk_weights.zero_() + topk_weights.fill_(1.0) return inputs @@ -922,20 +1294,11 @@ def get_tuning_config(self) -> TuningConfig: autotuner does not double-enumerate tile sizes for independent token axes. - The config is cached at class scope keyed on ``unique_id()``. - Every field below is a constant except ``inputs_pre_hook``. + Rebuilt on every call, NEVER cached across runners: + ``inputs_pre_hook`` is bound to THIS runner's ``_profiling_scratch``; + a cached hook bound to a dead runner would mix non-symmetric + profiling tensors with the live runner's peer offsets (IMA + hang). """ - key = self.unique_id() - cached = self.__class__.tuning_config_cache.get(key) - if cached is not None: - return cached - - # Constraints reuse the runner's own shape-derivation rules - # (the activation token count drives every other tensor's - # leading axis). We pass shape-derivation lambdas that pull - # the runtime ``num_tokens`` from input[0]. - def _num_tokens(shapes: List[torch.Size]) -> int: - return shapes[0][0] config = TuningConfig( dynamic_tensor_specs=( @@ -944,59 +1307,54 @@ def _num_tokens(shapes: List[torch.Size]) -> int: ), ), constraint_specs=( - ConstraintSpec(1, 0, _num_tokens), # activation_sf - ConstraintSpec(2, 0, _num_tokens), # topk_idx - ConstraintSpec(3, 0, _num_tokens), # topk_weights + ConstraintSpec(1, 0, _megamoe_autotune_num_tokens), # activation_sf + ConstraintSpec(2, 0, _megamoe_autotune_num_tokens), # topk_idx + ConstraintSpec(3, 0, _megamoe_autotune_num_tokens), # topk_weights # combine_output moved from idx 8 -> 11 after inserting # fc1_alpha(8) / fc2_alpha(9) / fc1_norm_const(10). - ConstraintSpec(11, 0, _num_tokens), # combine_output + ConstraintSpec(11, 0, _megamoe_autotune_num_tokens), # combine_output ), - # ``inputs_pre_hook`` is a bound method of THIS runner - # instance, yet caching the whole config across instances is - # safe: the hook only reads ``num_experts_per_rank`` and - # ``world_size`` (see ``_autotuner_inputs_pre_hook``), and both - # are part of ``unique_id()`` -- so every runner that maps to - # the same cache key has a functionally identical hook. The - # first instance for a given key is retained alive by this - # bound method (one runner object per distinct layer config, - # negligible). Mirrors how the CuteDSL grouped-gemm runners - # cache a ``helper.inputs_pre_hook`` keyed on ``unique_id()``. inputs_pre_hook=self._autotuner_inputs_pre_hook, - use_cold_l2_cache=True, + # MUST stay False for multi-rank: cold-L2 rotation clone()s the + # profiling inputs into NON-symmetric buffers, so the kernel's + # peer mapping (clone ptr + scratch peer_offsets) resolves + # outside any symmetric region -> cross-rank IMA while tuning. + use_cold_l2_cache=False, + # Pin the bucket ladder to the per-rank token ceiling instead + # of whatever (KV-clamped, possibly non-pow2) num_tokens the + # warmup forward fed. Mirrors trtllm-gen's tune_max_num_tokens. + tune_max_num_tokens=self.max_tokens_per_rank, # CUDA Graph capture cannot reproduce MegaMoE's runtime # peer-pointer table / dispatch-counter view and would # spin inside the captured barrier when the autotuner's # L2-cache buffers rotate. Plain repeat-loop profiling # is correct and only marginally slower. use_cuda_graph=False, - # FUSED_COMM hard requirement: every EP rank must run - # the same compiled tactic per chunk so the NVLink - # dispatch barrier and peer pointer mapping line up. - # PARALLEL strategy keeps tactic selection lockstep - # across ranks (same as every multi-rank CuteDSL op in - # ``cute_dsl_custom_ops.py``). - distributed_tuning_strategy=DistributedTuningStrategy.PARALLEL, + # Ranks couple inside the fused kernel, so every EP rank must + # profile the SAME tactic in lockstep with identical launch + # counts; MERGE does that and all-gathers timings so all ranks + # converge on one global-best tactic. PARALLEL would split + # tactics across ranks -> contaminated timing + barrier desync. + distributed_tuning_strategy=DistributedTuningStrategy.MERGE, ) - self.__class__.tuning_config_cache[key] = config return config def _build_kernel(self, tactic: Tuple): ( mma_tiler, cluster_shape, - use_2cta, - resolved_group_hint, + group_hint, load_balance_mode, - use_bf16_redg, - ) = tactic - from ..cute_dsl_kernels.mega_moe_nvfp4 import import_kernel - - kernel_cls = import_kernel() - return kernel_cls( - mma_tiler_mnk=tuple[Any, ...](mma_tiler), + token_back_mode, + use_bulk_fc2_store, + flag_batch, + epi_flag_batch, + ) = _unpack_tactic(tactic) + common = dict( + mma_tiler_mnk=tuple(mma_tiler), cluster_shape_mnk=tuple(cluster_shape), - use_2cta_instrs=bool(use_2cta), - group_hint=int(resolved_group_hint), + use_2cta_instrs=bool(mma_tiler[0] == 256), + group_hint=int(group_hint), token_padding_block=64, sf_padding_block=SfPaddingBlock, load_balance_mode=str(load_balance_mode), @@ -1006,81 +1364,52 @@ def _build_kernel(self, tactic: Tuple): self.hidden_size, ), world_size=self.world_size, - local_rank=self.local_rank, num_topk=self.num_topk, max_tokens_per_rank=self.max_tokens_per_rank, hidden=self.hidden_size, fc2_output_dtype=cutlass.BFloat16, - in_kernel_fc2_reduce=bool(use_bf16_redg), + in_kernel_fc2_reduce=self.in_kernel_fc2_reduce, + token_back_mode=str(token_back_mode), + non_ubulk_fc2_store=(not bool(use_bulk_fc2_store)), + flag_batch=int(flag_batch), + epi_flag_batch=tuple(epi_flag_batch), apply_topk_in_fc1=self.apply_topk_in_fc1, gate_up_clamp=self.gate_up_clamp, - token_back_by_dispatch=self.token_back_by_dispatch, - non_ubulk_fc2_store=self.non_ubulk_fc2_store, **_LOCKED_KERNEL_KWARGS, ) + kernel_cls, CombineFormat = _import_megamoe_kernel() + return kernel_cls(combine_format=CombineFormat.parse(self.combine_format), **common) def _tactic_cache_key(self, tactic: Tuple) -> Tuple: - # Hashable cache key shared by the compile cache and the - # local-workspace cache. ``unique_id()`` already carries - # apply_topk_in_fc1 / gate_up_clamp, so the codegen-time - # graph/clamp modes are part of the cache key without listing - # them again here. + """Hashable cache key over ``unique_id()`` + the FULL 8-tuple. + + Both the compile cache and the local-workspace cache MUST key on + the full tactic: the local workspace SIZE varies with it (non-epi + token_back / atomic_counter add large regions), so an under-keyed + cache would reuse a wrong-sized buffer -> OOB / 100% SM hang. + """ ( mma_tiler, cluster_shape, - use_2cta, - resolved_group_hint, + group_hint, load_balance_mode, - use_bf16_redg, - ) = tactic + token_back_mode, + use_bulk_fc2_store, + flag_batch, + epi_flag_batch, + ) = _unpack_tactic(tactic) return ( self.unique_id(), tuple(mma_tiler), tuple(cluster_shape), - bool(use_2cta), - int(resolved_group_hint), + int(group_hint), str(load_balance_mode), - bool(use_bf16_redg), + str(token_back_mode), + bool(use_bulk_fc2_store), + int(flag_batch), + tuple(epi_flag_batch), ) - def _compile_or_get(self, tactic: Tuple, kernel, runtime_kwargs): - ( - mma_tiler, - cluster_shape, - use_2cta, - resolved_group_hint, - load_balance_mode, - use_bf16_redg, - ) = tactic - cache_key = self._tactic_cache_key(tactic) - compiled = self.__class__.kernel_cache.get(cache_key) - if compiled is not None: - return compiled - compile_kwargs = dict(runtime_kwargs) - hardware_info = cutlass.utils.HardwareInfo() - cluster_size = cluster_shape[0] * cluster_shape[1] * cluster_shape[2] - compile_kwargs["max_active_clusters"] = hardware_info.get_max_active_clusters( - max(cluster_size, 1) - ) - # CuTe DSL compile is the dominant first-launch cost; log - # start/end at INFO so the long compile gap is visible through - # the standard TRT-LLM logger (honors TLLM_LOG_LEVEL). - logger.info( - f"[MegaMoECuteDsl] cute.compile START tactic=" - f"(mma_tiler={mma_tiler}, cluster={cluster_shape}, " - f"use_2cta={use_2cta}, group_hint={resolved_group_hint}, " - f"load_balance={load_balance_mode!r}, use_bf16_redg={use_bf16_redg})" - ) - t_compile_start = time.perf_counter() - compiled = cute.compile(kernel, **compile_kwargs) - t_compile_ms = (time.perf_counter() - t_compile_start) * 1000 - logger.info( - f"[MegaMoECuteDsl] cute.compile DONE in {t_compile_ms:.0f} ms " - f"(cache_keys_now={len(self.__class__.kernel_cache) + 1})" - ) - self.__class__.kernel_cache[cache_key] = compiled - return compiled - def forward( self, inputs: List[torch.Tensor], @@ -1091,17 +1420,15 @@ def forward( **kwargs, ) -> None: del kwargs - t_forward_start = time.perf_counter() - # Resolve fallback tactic. if tactic == -1 or tactic is None: - tactic_t = ( - list(DEFAULT_MEGAMOE_TACTIC[0]), - list(DEFAULT_MEGAMOE_TACTIC[1]), - DEFAULT_MEGAMOE_TACTIC[2], - resolve_megamoe_group_hint(tuple(DEFAULT_MEGAMOE_TACTIC[1])), - DEFAULT_MEGAMOE_TACTIC[4], - DEFAULT_MEGAMOE_TACTIC[5], - ) + num_tokens = int(inputs[0].shape[0]) + # Form-B / quantized combine require non-bulk (see + # get_valid_tactics); this deterministic fallback keeps ALL + # quantized combine non-bulk (the tuner may recover fp8 bulk). + if self.in_kernel_fc2_reduce or self.combine_format != "bf16": + tactic_t = _MEGAMOE_NONBULK_STANDALONE_TACTIC + else: + tactic_t = default_megamoe_tactic(num_tokens) elif isinstance(tactic, list): tactic_t = tuple(tactic) else: @@ -1122,6 +1449,13 @@ def forward( fc1_norm_const, combine_output, ) = inputs[:12] + if self._profiling_scratch is not None: + # Profiling launch: with the DEFERRED factory, choose_one's + # kwargs were bound to the STAGING buffers before the pre-hook + # materialized the scratch -- rebind both cross-rank args here. + # No-op for an eager scratch; never fires on the real run. + peer_offsets = list(self._profiling_scratch.peer_offsets) + shared_workspace = self._profiling_scratch.shared_workspace assert peer_offsets is not None, ( "Sm100MegaMoENvfp4Runner.forward requires peer_offsets kwarg " "(length = world_size); single-rank degenerate mode passes " @@ -1133,11 +1467,18 @@ def forward( kernel = self._build_kernel(tactic_t) - # ``local_workspace`` is per-rank private; cached across calls. + # form-B accumulates the top-k reduction into ``combine_output``, + # so its live rows MUST start at 0; the CALLER zeros [:num_tokens] + # (the op has no live token count -- activation is padded to max_T + # -- and a full zero wastes decode time). form-A overwrites. + + # Cached per FULL tactic: local workspace SIZE is tactic-dependent + # (see _tactic_cache_key). local_workspace = _get_or_alloc_local_workspace( kernel, cache_key=self._tactic_cache_key(tactic_t), device=activation.device, + latch_in_tuning_mode=self.tactic_autotune, ) # ``shared_workspace`` is peer-mapped (symmetric heap) for # multi-rank or local CUDA for the single-rank degenerate @@ -1168,92 +1509,149 @@ def forward( # peer rank's in-kernel dispatch barrier write into this rank's # ``nvlink_barrier_signal``: a fast peer ``red_add(+1)``s our slot, # then our late ``zero_()`` wipes it, so the barrier never reaches - # ``world_size`` and the whole grid deadlocks (the EPLB multi-rank - # dispatch-barrier hang). The symmetric workspace's peer-written - # count regions (expert_recv_count[_sum]) are instead reset - # device-side by the kernel's ``tail_reset_shared_counters``, - # ``nvlink_barrier_signal`` self-primes (phase-flip), and - # ``src_token_topk_idx`` is overwritten by dispatch each launch -- - # so the shared workspace needs no per-launch host zero at all. - if self.world_size > 1: - _zero_local_workspace_preserving_phase(local_workspace, kernel) - else: - shared_workspace.zero_() - local_workspace.zero_() - - activation_cute = _to_cute(activation) - activation_sf_cute = _to_cute(activation_sf) - topk_idx_cute = _to_cute(topk_idx) - topk_weights_cute = _to_cute(topk_weights) - # The weights are stored ``(slots, N, K_bytes)`` (K = hidden//2 for - # fc1 / intermediate//2 for fc2, innermost / stride-1). The kernel - # reads them K-major with K innermost; present a ``transpose(1, 2)`` - # VIEW ``(slots, K_bytes, N)`` so K stays stride-1. Do NOT - # ``.contiguous()`` -- materializing would move K off the innermost - # axis (N would become stride-1) and corrupt the GEMM (cosine ~0). - fc1_weight_cute = _to_cute(fc1_weight.transpose(1, 2)) - fc1_weight_sf_cute = _to_cute(fc1_weight_sf) - fc2_weight_cute = _to_cute(fc2_weight.transpose(1, 2)) - fc2_weight_sf_cute = _to_cute(fc2_weight_sf) - # Per-expert fp32 scale tensors are 1-D ``(num_local_slots,)``; - # 4-byte alignment matches the fp32 element size (the kernel - # reads them as a plain fp32 vector, no 16-byte TMA tile). - fc1_alpha_cute = _to_cute(fc1_alpha, assumed_align=4) - fc2_alpha_cute = _to_cute(fc2_alpha, assumed_align=4) - fc1_norm_const_cute = _to_cute(fc1_norm_const, assumed_align=4) - combine_output_cute = _to_cute(combine_output) - local_workspace_cute = _to_cute(local_workspace, force_static_layout=True) - shared_workspace_cute = _to_cute(shared_workspace) - - torch_stream = torch.cuda.current_stream() - stream = cuda.CUstream(torch_stream.cuda_stream) - - # SymBufferHost contract: ``base_addr`` is any local pointer - # inside the symmetric heap; ``offsets[r] = peer_base - - # local_base``. All five regions share the same delta - # because ``MegaMoeSymmMemProvider`` carves them out of one - # symmetric allocation, so peer_rank_ptr_mapper.map(local, - # r, off) maps any region's local pointer to its peer. - sym_buf = SymBufferHost( - base_addr=int(activation.data_ptr()), - offsets=tuple(int(off) for off in peer_offsets), - rank_idx=int(self.local_rank), - num_max_ranks=int(self.world_size), + # ``world_size`` and the whole grid deadlocks. Peer-written count + # regions are reset device-side, the signal self-primes + # (phase-flip) -- no per-launch host zero needed. + # + # Phase coherence: the sense-reversing dispatch barrier pairs a + # PERSISTED local nvlink_barrier_counter with a PERSISTED shared + # nvlink_barrier_signal; on a TACTIC CHANGE (autotune boundary, + # profiling->real) the pair can be phase-DECOUPLED -> the next + # barrier spins forever. So reset both to phase 0 under a + # cross-rank fence ONCE per tactic. The fence (host sync + + # collective barrier) is ILLEGAL under capture (eager warmup + # fences first) and valid only for pure DEP (guard below). + _capturing = torch.cuda.is_current_stream_capturing() + # Fence ONCE per (local_ws, shared_ws, world) key, tracked + # module-globally. Keys are raw pointers, so "already fenced" also + # requires the STORAGES to be alive (ABA; _megamoe_fence_key_live). + _fkey = ( + int(local_workspace.data_ptr()), + int(shared_workspace.data_ptr()), + int(self.world_size), ) - + _fkey_live = _megamoe_fence_key_live(_fkey) + if _capturing: + if not _fkey_live: + # Eager pre-passes fence every pairing a captured forward + # uses; reaching capture unfenced means two runners share a + # workspace side. Fail loud (replay would hang silently). + raise RuntimeError( + "MegaMoE-CuteDSL: CUDA-graph capture reached an " + "unfenced (local, shared) workspace pairing; the " + "barrier-reset fence cannot run during capture." + ) + # A later EAGER fence must never zero this captured shared side. + _MEGAMOE_CAPTURED_SHARED_PTRS.add(_fkey[1]) + global _MEGAMOE_GRAPH_CAPTURE_SEEN + _MEGAMOE_GRAPH_CAPTURE_SEEN = True + if (not _fkey_live) and not _capturing: + import torch.distributed as _dist + + _sig_off = int(kernel._shared_offsets["nvlink_barrier_signal"]) + _sig_n = int(kernel._shared_region_by_name["nvlink_barrier_signal"].nbytes) + _bc_off = int(kernel._local_offsets["nvlink_barrier_counter"]) + _bc_n = int(kernel._local_region_by_name["nvlink_barrier_counter"].nbytes) + _have_dist = _dist.is_available() and _dist.is_initialized() + # The fence barriers the default (WORLD) group -- correct ONLY + # for pure DEP. Under TP x EP / PP the WORLD barrier spans ranks + # that never reach this fence -> deadlock; fail loud instead. + # (Threading the EP subgroup is the general fix.) + if _have_dist and _dist.get_world_size() != int(self.world_size): + # Not an assert: the guard must survive ``python -O``. + raise RuntimeError( + "MegaMoE-CuteDSL barrier-reset fence uses the WORLD process " + "group, valid only for pure DEP (world == EP). Detected WORLD=" + f"{_dist.get_world_size()} != EP={self.world_size} (TP x EP / " + "PP) -- fence must run on the EP subgroup." + ) + # Never re-zero a shared side baked into a captured graph + # (captured pairings cannot re-fence -> silent hang). + if _fkey[1] in _MEGAMOE_CAPTURED_SHARED_PTRS: + raise RuntimeError( + "MegaMoE-CuteDSL: eager barrier-reset fence would " + "zero a shared workspace already used by a captured " + "CUDA graph; the captured pairings cannot re-fence." + ) + torch.cuda.current_stream().synchronize() + if _have_dist: + _dist.barrier() + # Reset BOTH sides to phase 0. The local counter can be + # ALREADY-ADVANCED here (at the profiling->real transition the + # winning tactic's cached local ran under the scratch's shared + # ptr; the real launch's new fkey re-fences it). Zero only the + # tiny counter slice; the multi-GiB bulk stays uninitialized. + local_workspace[_bc_off : _bc_off + _bc_n].zero_() + shared_workspace[_sig_off : _sig_off + _sig_n].zero_() + # The zeros are stream-async and dist.barrier() is NOT + # device-ordered after them: a released peer's in-kernel signal + # WRITE could be CLOBBERED by our still-pending zero (barrier + # spins forever). Sync so the zeros LAND before peers launch. + torch.cuda.current_stream().synchronize() + if _have_dist: + _dist.barrier() + # Both sides were just re-zeroed, so every OTHER fenced pairing + # sharing either side is now phase-stale (the autotune + # real->scratch->real ABA); drop them so they re-fence. + # Invariant: fkey in set => neither side touched under any + # other pairing since its fence. + _megamoe_prune_fenced_keys( + {k for k in _MEGAMOE_FENCED_KEYS if k[0] == _fkey[0] or k[1] == _fkey[1]} + ) + _MEGAMOE_FENCED_KEYS.add(_fkey) + # Record the fenced ALLOCATIONS for recycled-VA (ABA) detection. + _MEGAMOE_FENCED_KEY_WS_REFS[_fkey] = ( + weakref.ref(local_workspace.untyped_storage()), + weakref.ref(shared_workspace.untyped_storage()), + ) + # else: SAME tactic -- the kernel's device-side tail self-reset suffices. + + # cute.compile (JIT) launch. The + # uint8 workspaces MUST be cute.Pointer, NOT cute.Tensor: the + # 32-bit memref shape field overflows once shared_workspace passes + # 2 GiB (the kernel addresses by raw base + Int64 byte offset). + _to_cute, _to_cute_ptr, SymBufferHost, cute, cutlass_utils = _cute_launch_helpers() + + # combine_output (max_T, 1, hidden) reshapes freely to 2D. Weights + # present K stride-1 via a transpose VIEW (DLPack carries the + # strides); do NOT ``.contiguous()``. + output_activation = combine_output.reshape(combine_output.shape[0], self.hidden_size) runtime_kwargs = dict( - activation=activation_cute, - activation_sf=activation_sf_cute, - topk_idx=topk_idx_cute, - topk_weights=topk_weights_cute, - fc1_weight=fc1_weight_cute, - fc1_weight_sf=fc1_weight_sf_cute, - fc2_weight=fc2_weight_cute, - fc2_weight_sf=fc2_weight_sf_cute, - fc1_alpha=fc1_alpha_cute, - fc2_alpha=fc2_alpha_cute, - fc1_norm_const=fc1_norm_const_cute, - combine_output=combine_output_cute, - local_workspace=local_workspace_cute, - shared_workspace=shared_workspace_cute, - peer_rank_ptr_mapper_host=sym_buf, - stream=stream, + activation=_to_cute(activation), + activation_sf=_to_cute(activation_sf), + topk_idx=_to_cute(topk_idx), + topk_weights=_to_cute(topk_weights), + fc1_weight=_to_cute(fc1_weight.transpose(1, 2)), + fc1_weight_sf=_to_cute(fc1_weight_sf), + fc2_weight=_to_cute(fc2_weight.transpose(1, 2)), + fc2_weight_sf=_to_cute(fc2_weight_sf), + fc1_alpha=_to_cute(fc1_alpha, assumed_align=4), + fc2_alpha=_to_cute(fc2_alpha, assumed_align=4), + fc1_norm_const=_to_cute(fc1_norm_const, assumed_align=4), + output_activation=_to_cute(output_activation), + local_workspace=_to_cute_ptr(local_workspace), + shared_workspace=_to_cute_ptr(shared_workspace), + peer_rank_ptr_mapper_host=SymBufferHost( + offsets=tuple(int(off) for off in peer_offsets), + rank_idx=int(self.local_rank), + num_max_ranks=int(self.world_size), + ), + stream=cuda.CUstream(torch.cuda.current_stream().cuda_stream), ) - compiled = self._compile_or_get(tactic_t, kernel, runtime_kwargs) - t_launch_start = time.perf_counter() + cache_key = self._tactic_cache_key(tactic_t) + compiled = self.__class__.kernel_cache.get(cache_key) + if compiled is None: + # ``max_active_clusters`` is a compile-time Constexpr (baked in, + # omitted at launch); cluster_size = cluster_shape M*N. + _cluster = _unpack_tactic(tactic_t)[1] + _max_active_clusters = cutlass_utils.HardwareInfo().get_max_active_clusters( + int(_cluster[0]) * int(_cluster[1]) + ) + compiled = cute.compile( + kernel, max_active_clusters=_max_active_clusters, **runtime_kwargs + ) + self.__class__.kernel_cache[cache_key] = compiled compiled(**runtime_kwargs) - t_launch_ms = (time.perf_counter() - t_launch_start) * 1000 - t_forward_ms = (time.perf_counter() - t_forward_start) * 1000 - logger.debug( - "[MegaMoECuteDsl] forward DONE tactic=" - "(mma_tiler=%s, cluster=%s, load_balance=%r) " - "launch+sync=%.0fms total=%.0fms", - tactic_t[0], - tactic_t[1], - tactic_t[4], - t_launch_ms, - t_forward_ms, - ) return combine_output # ----- torch op --------------------------------------------------------- @@ -1288,24 +1686,32 @@ def cute_dsl_megamoe_nvfp4_blackwell( peer_offsets: List[int], apply_topk_in_fc1: bool = True, gate_up_clamp: Optional[float] = None, - token_back_by_dispatch: bool = False, - non_ubulk_fc2_store: bool = True, + in_kernel_fc2_reduce: bool = False, + combine_format: str = "bf16", + tactic_autotune: bool = False, + num_tokens: int = -1, ) -> None: """Run the fused MegaMoE CuteDSL NVFP4 kernel. Inputs are pre-staged by the caller (the ``MegaMoECuteDsl`` - backend in ``mega_moe_cute_dsl.py``). The op runs AutoTuner once - per call to pick the best tactic for the current shape and - invokes the runner. + backend in ``mega_moe_cute_dsl.py``). ``tactic_autotune=True`` + (``MEGAMOE_TACTIC_AUTOTUNE=1`` bench opt-in) picks the tactic via + AutoTuner per call; the default runs the deterministic token-bucket + heuristic (``default_megamoe_tactic``). ``shared_workspace`` MUST be a symmetric-heap tensor for ``world_size > 1`` (use :class:`MegaMoeSymmMemProvider`); a local CUDA tensor is acceptable for the single-rank degenerate path. ``combine_output`` is mutated in place; the op does not return it because torch custom_op forbids the return value from - aliasing any mutated input. Form A semantics: ``combine_output`` - keeps its ``(T, num_topk, hidden)`` layout, and the caller is - responsible for the host-side ``.sum(dim=1)``. + aliasing any mutated input. + + ``in_kernel_fc2_reduce`` selects the reduction form: ``False`` + (form-A) runs the deterministic standalone TopkReduce; ``True`` + (form-B) folds top-k into the kernel and is NON-deterministic (float + accumulation order). Both write ``combine_output`` with shape + ``(T, 1, hidden)``. Perf knobs come from the tactic, not op + arguments. """ sm_version = get_sm_version() if sm_version not in (100, 103): @@ -1314,6 +1720,14 @@ def cute_dsl_megamoe_nvfp4_blackwell( f"SM 103 (B300); got SM {sm_version}." ) + # Live-token trim: TopkReduce sizes its grid from THIS tensor's dim0, + # so slicing to the live count skips dead reduce rows (~5-8us/layer at + # decode). num_tokens < 0 or oversized keeps the full bucket; so does + # num_tokens == 0 (a zero-token rank still launches so peers can cross + # the NVLink barrier -- a 0-row slice would build a zero grid). + if num_tokens > 0: + combine_output = combine_output[: min(num_tokens, combine_output.shape[0])] + runner = Sm100MegaMoENvfp4Runner( world_size=world_size, local_rank=local_rank, @@ -1326,8 +1740,9 @@ def cute_dsl_megamoe_nvfp4_blackwell( output_dtype=combine_output.dtype, apply_topk_in_fc1=apply_topk_in_fc1, gate_up_clamp=gate_up_clamp, - token_back_by_dispatch=token_back_by_dispatch, - non_ubulk_fc2_store=non_ubulk_fc2_store, + in_kernel_fc2_reduce=in_kernel_fc2_reduce, + combine_format=combine_format, + tactic_autotune=tactic_autotune, ) inputs = [ activation, @@ -1343,15 +1758,67 @@ def cute_dsl_megamoe_nvfp4_blackwell( fc1_norm_const, combine_output, ] + if not tactic_autotune: + # Opt-OUT (default; serving never opts in): skip the AutoTuner. + # ``choose_one`` is NOT a safe no-op -- in tuning mode a cache miss + # materializes the multi-GiB scratch and MERGE-sweeps ~36 + # candidates. tactic=-1 guarantees the deterministic heuristic + # with no tuning collectives, even inside a global autotune(). + runner( + inputs, + tactic=-1, + peer_offsets=peer_offsets, + shared_workspace=shared_workspace, + ) + return tuner = AutoTuner.get() - _, best_tactic = tuner.choose_one( - "trtllm::cute_dsl_megamoe_nvfp4_blackwell", - [runner], - runner.get_tuning_config(), - inputs, - peer_offsets=peer_offsets, - shared_workspace=shared_workspace, - ) + # Opt-IN: in tuning mode the MERGE lockstep sweep runs (made safe by + # the tactic-change fence in forward); outside it choose_one is a pure + # cache lookup (tuned tactic, or -1 -> deterministic heuristic). + # Profiling must use SYMMETRIC cross-rank buffers (the transient + # scratch); only single-rank may fall back to the staging buffer. + prof_scratch = _ACTIVE_MEGAMOE_PROFILING_SCRATCH if world_size > 1 else None + prof_factory = _ACTIVE_MEGAMOE_PROFILING_SCRATCH_FACTORY if world_size > 1 else None + if ( + prof_scratch is None + and prof_factory is None + and world_size > 1 + and tuner.is_tuning_mode + ): + # Not a fallback: profiling would peer-map non-symmetric tensors + # (cross-rank IMA). The backend hands either a live scratch or a + # DEFERRED factory; reaching here means both were bypassed. + raise RuntimeError( + "MegaMoE-CuteDSL autotune profiling requires the " + "symmetric profiling scratch (or a scratch factory) on " + "multi-rank, but neither is active." + ) + if prof_scratch is not None: + runner._profiling_scratch = prof_scratch + prof_peer_offsets = list(prof_scratch.peer_offsets) + prof_shared_workspace = prof_scratch.shared_workspace + else: + # DEFERRED path: the pre-hook materializes the scratch only if + # real profiling launches; ``forward`` then rebinds these staging + # values from it, and on a cache hit choose_one never launches -- + # so they are never consumed for profiling. + runner._profiling_scratch = None + runner._profiling_scratch_factory = prof_factory + prof_peer_offsets = peer_offsets + prof_shared_workspace = shared_workspace + try: + _, best_tactic = tuner.choose_one( + "trtllm::cute_dsl_megamoe_nvfp4_blackwell", + [runner], + runner.get_tuning_config(), + inputs, + peer_offsets=prof_peer_offsets, + shared_workspace=prof_shared_workspace, + ) + finally: + runner._profiling_scratch = None + runner._profiling_scratch_factory = None + # Real run always uses the caller's staging buffer + peer_offsets. runner( inputs, tactic=best_tactic, @@ -1385,7 +1852,9 @@ def _( peer_offsets: List[int], apply_topk_in_fc1: bool = True, gate_up_clamp: Optional[float] = None, - token_back_by_dispatch: bool = False, - non_ubulk_fc2_store: bool = True, + in_kernel_fc2_reduce: bool = False, + combine_format: str = "bf16", + tactic_autotune: bool = False, + num_tokens: int = -1, ) -> None: return None diff --git a/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py b/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py index 7e4129672083..f9cdf0873790 100644 --- a/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py @@ -2056,7 +2056,9 @@ def _(a, b, a_scale, b_scale, tune_max_num_tokens=4096): def silu_and_mul(x: torch.Tensor, scale: Optional[torch.Tensor] = None, dtype: Optional[torch.dtype] = None, - swiglu_limit: Optional[float] = None) -> torch.Tensor: + swiglu_limit: Optional[float] = None, + swiglu_alpha: Optional[float] = None, + swiglu_beta: Optional[float] = None) -> torch.Tensor: b, n = x.shape assert n % 2 == 0 @@ -2076,6 +2078,8 @@ def grid(meta: Mapping[str, int]) -> tuple[int, int]: x_stride=x.stride(0), d=d, swiglu_limit=swiglu_limit or 0.0, + swiglu_alpha=swiglu_alpha if swiglu_alpha is not None else 1.0, + swiglu_beta=swiglu_beta if swiglu_beta is not None else 0.0, BLOCK_SIZE=1024, HAS_O_SCALE=scale is not None, HAS_SWIGLU_LIMIT=swiglu_limit is not None and swiglu_limit > 0.0, @@ -2090,6 +2094,8 @@ def _( scale: Optional[torch.Tensor] = None, dtype: Optional[torch.dtype] = None, swiglu_limit: Optional[float] = None, + swiglu_alpha: Optional[float] = None, + swiglu_beta: Optional[float] = None, ) -> torch.Tensor: b, n = x.shape diff --git a/tensorrt_llm/_torch/custom_ops/trtllm_gen_custom_ops.py b/tensorrt_llm/_torch/custom_ops/trtllm_gen_custom_ops.py index 74d9946ee05a..ed84c2ca8acb 100644 --- a/tensorrt_llm/_torch/custom_ops/trtllm_gen_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/trtllm_gen_custom_ops.py @@ -128,6 +128,19 @@ def prepare_dummy_topk_and_hook( # experts and only ~1/ep_size of slots hit local. is_local = use_dp + def pad_with_remote_dummy_experts(topk_ids_local: torch.Tensor, + device: torch.device) -> torch.Tensor: + """Fill remaining slots with deterministic out-of-shard ids, mirroring + production rows when local_num_experts < top_k: a row's top_k picks + are distinct global experts, at most local_num_experts of them local. + """ + num_tokens, k_local = topk_ids_local.shape + remote = (local_expert_offset + local_num_experts + torch.arange( + top_k - k_local, device=device, dtype=torch.int32)) % num_experts + return torch.cat( + [topk_ids_local, + remote.unsqueeze(0).expand(num_tokens, -1)], dim=1) + def make_balanced_dummy_topk( num_tokens: int, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]: @@ -154,24 +167,26 @@ def make_balanced_dummy_topk( n_target = local_num_experts if is_local else num_experts topk_ids[t, k] = (t + k * stride) % n_target (+ local_expert_offset if is_local) - Stride is picked so each row's `top_k` entries stay distinct; - the assertion guards against degenerate (n_target < top_k) configs - that would produce in-row duplicates. + Stride is picked so each row's filled entries stay distinct; when + n_target < top_k (local regime only) the row is padded with + out-of-shard ids (see pad_with_remote_dummy_experts). """ n_target = local_num_experts if is_local else num_experts - assert n_target >= top_k, ( - f"make_balanced_dummy_topk requires n_target>={top_k}; " - f"got n_target={n_target}, is_local={is_local}, " - f"num_experts={num_experts}, local_num_experts={local_num_experts}") - stride = max(1, min(local_num_experts, n_target // top_k)) - if stride * top_k > n_target: - stride = max(1, n_target // top_k) - base = torch.arange(top_k, device=device, dtype=torch.int32) * stride + assert is_local or n_target >= top_k, ( + f"make_balanced_dummy_topk requires num_experts>={top_k} in " + f"the global regime; got num_experts={num_experts}") + k_fill = min(top_k, n_target) + stride = max(1, min(local_num_experts, n_target // k_fill)) + if stride * k_fill > n_target: + stride = max(1, n_target // k_fill) + base = torch.arange(k_fill, device=device, dtype=torch.int32) * stride token_idx = torch.arange(num_tokens, device=device, dtype=torch.int32).unsqueeze(1) topk_ids = (base + token_idx) % n_target if is_local: topk_ids = topk_ids + local_expert_offset + if k_fill < top_k: + topk_ids = pad_with_remote_dummy_experts(topk_ids, device) topk_weights = torch.ones(num_tokens, top_k, dtype=torch.bfloat16, @@ -261,9 +276,7 @@ def make_routing_dummy_topk( would observe. """ if is_local: - assert local_num_experts >= top_k, ( - f"random_local requires local_num_experts >= top_k; " - f"got local_num_experts={local_num_experts}, top_k={top_k}") + k_fill = min(top_k, local_num_experts) if (logits is None or logits.shape[0] != num_tokens or logits.shape[-1] != local_num_experts): logits = torch.randn(num_tokens, @@ -272,9 +285,12 @@ def make_routing_dummy_topk( device=device) # Plain topk over local logits — bypasses the model's # routing_method on purpose (see docstring caveat re: - # grouped routings like DeepSeekV3 / Llama4). - topk_ids = torch.topk(logits.float(), top_k, dim=-1).indices.to( + # grouped routings like DeepSeekV3 / Llama4). If the shard has + # fewer than top_k experts, take all and pad (see helper). + topk_ids = torch.topk(logits.float(), k_fill, dim=-1).indices.to( torch.int32) + local_expert_offset + if k_fill < top_k: + topk_ids = pad_with_remote_dummy_experts(topk_ids, device) topk_weights = torch.ones(num_tokens, top_k, dtype=torch.bfloat16, diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_act_fusion.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_act_fusion.py index b96186049f67..88215a759fc2 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_act_fusion.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_act_fusion.py @@ -125,6 +125,7 @@ def __init__( vectorized_f32: bool, use_prefetch: bool = False, activation_type: ActivationType = ActivationType.Swiglu, + indexer_q_fusion: bool = False, ): """Initializes the configuration for a Blackwell dense GEMM kernel with fused activation. @@ -182,19 +183,28 @@ def __init__( self.vectorized_f32 = vectorized_f32 self.activation_type = activation_type + self.indexer_q_fusion = indexer_q_fusion self.is_gated = is_gated_activation(activation_type) # Precompute per-activation flags as plain Python bools so the epilogue # dispatch can use cutlass.const_expr(...) on them (like is_gated). An # inline enum comparison inside @cute.jit is not folded as a constant. self._act_is_swiglu = activation_type == ActivationType.Swiglu self._act_is_gelu = activation_type == ActivationType.Gelu + self._act_is_identity = activation_type == ActivationType.Identity # Host-side guard (raise is not allowed inside the @cute.kernel epilogue): # only SwiGLU and GELU(tanh) are wired in the fused epilogue. - if not (self._act_is_swiglu or self._act_is_gelu): + if not (self._act_is_swiglu or self._act_is_gelu or self._act_is_identity): raise NotImplementedError( f"Fused epilogue activation {activation_type} not implemented " - f"(only Swiglu and Gelu are wired)." + f"(only Swiglu, Gelu, and the indexer-Q identity epilogue are wired)." ) + if self.indexer_q_fusion: + if activation_type != ActivationType.Identity: + raise ValueError("indexer_q_fusion requires ActivationType.Identity") + if sf_vec_size != 32 or mma_tiler_mn[1] != 128: + raise ValueError( + "indexer_q_fusion requires MXF8 sf_vec_size=32 and a 128-column MMA tile" + ) def _setup_attributes(self): """Set up configurations that are dependent on GEMM inputs @@ -298,7 +308,7 @@ def _setup_attributes(self): self.is_b_mcast = self.num_mcast_ctas_b > 1 self.is_sfb_mcast = self.num_mcast_ctas_sfb > 1 - # SwiGLU: hardcoded epilogue tile matching grouped swiglu variant + # SwiGLU: hardcoded epilogue tile matching grouped swiglu variant. self.epi_tile = (128, 64) self.epi_tile_cnt = ( self.cta_tile_shape_mnk_c[0] // self.epi_tile[0], @@ -389,6 +399,9 @@ def __call__( sfc_tensor: Optional[cute.Tensor] = None, norm_const_tensor: Optional[cute.Tensor] = None, bias_tensor: Optional[cute.Tensor] = None, + indexer_scale_tensor: Optional[cute.Tensor] = None, + position_ids_tensor: Optional[cute.Tensor] = None, + cos_sin_cache_tensor: Optional[cute.Tensor] = None, ): """Execute the GEMM operation with SwiGLU fusion in steps: - Setup static attributes before smem/grid/tma computation @@ -444,6 +457,20 @@ def __call__( # Setup sfc tensor by filling C tensor to scale factor atom layout self.generate_sfc = sfc_tensor is not None and norm_const_tensor is not None + if cutlass.const_expr( + self.indexer_q_fusion + and ( + not self.generate_sfc + or self.c_dtype != cutlass.Float4E2M1FN + or indexer_scale_tensor is None + or position_ids_tensor is None + or cos_sin_cache_tensor is None + ) + ): + raise ValueError( + "indexer_q_fusion requires FP4 C, SFC generation, contiguous " + "output scales, position IDs, and a cos/sin cache" + ) if cutlass.const_expr(self.generate_sfc): sfc_layout = blockscaled_utils.tile_atom_to_shape_SF(c_tensor.shape, self.sf_vec_size) sfc_tensor = cute.make_tensor(sfc_tensor.iterator, sfc_layout) @@ -621,6 +648,9 @@ class SharedStorage: sfc_tensor, norm_const_tensor, bias_tensor, + indexer_scale_tensor, + position_ids_tensor, + cos_sin_cache_tensor, self.cluster_layout_vmnk, self.cluster_layout_sfb_vmnk, self.a_smem_layout_staged, @@ -662,6 +692,9 @@ def kernel( mSFC_mnl: Optional[cute.Tensor], norm_const_tensor: Optional[cute.Tensor], mBias_mnl: Optional[cute.Tensor], + mIndexerScale: Optional[cute.Tensor], + mPositionIds: Optional[cute.Tensor], + mCosSinCache: Optional[cute.Tensor], cluster_layout_vmnk: cute.Layout, cluster_layout_sfb_vmnk: cute.Layout, a_smem_layout_staged: cute.ComposedLayout, @@ -1365,6 +1398,7 @@ def kernel( tTR_rAcc_up, tTR_rAcc_gate, tTR_gBias_base, + tTR_cC, ) = self.epilog_tmem_copy_and_partition( epi_tidx, tCtAcc_base, tCgC, epi_tile, use_2cta_instrs, tCgBias ) @@ -1503,6 +1537,7 @@ def kernel( # up * silu(gate) # subtile_cnt = cute.size(tTR_tAcc.shape, mode=[3]) + packed_indexer_scale = cutlass.Uint32(0) for subtile_idx in cutlass.range(0, subtile_cnt, 2 if self.is_gated else 1): if cutlass.const_expr(self.is_gated): @@ -1568,6 +1603,60 @@ def kernel( self._apply_gelu_epilogue( acc_vec_up, alpha_val, tCompute, bias_vec=bias_vec ) + elif cutlass.const_expr(self._act_is_identity): + # Preserve the existing Q-projection contract: the GEMM + # accumulator is rounded to BF16 before RoPE. The + # indexer-Q block below performs that boundary rounding; + # this copy keeps the accumulator in FP32 until then. + tCompute.store(acc_vec_up) + + if cutlass.const_expr(self.indexer_q_fusion): + # The legacy path rounds GEMM output to BF16 before + # RoPE, then rounds the rotated values to BF16 again. + tCompute.store(tCompute.load().to(cutlass.BFloat16).to(cutlass.Float32)) + subtile_n_for_rope = real_subtile_idx % self.epi_tile_cnt[1] + subtiles_per_head = 128 // self.epi_tile[1] + rotary_subtile_begin = 64 // self.epi_tile[1] + subtile_in_head = subtile_n_for_rope % subtiles_per_head + if subtile_in_head >= rotary_subtile_begin: + subtile_m_for_rope = real_subtile_idx // self.epi_tile_cnt[1] + rope_base_m = ( + cur_tile_coord[0] * self.cta_tile_shape_mnk_c[0] + + subtile_m_for_rope * self.epi_tile[0] + ) + rope_m_idx = rope_base_m + tTR_cC[0][0] + if rope_m_idx < mPositionIds.shape[0]: + position = mPositionIds[rope_m_idx] + rope_row = mCosSinCache[position, None] + copy_atom_f32x8 = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), + cutlass.Float32, + num_bits_per_copy=256, + ) + cos_values = cute.make_rmem_tensor((8,), cutlass.Float32) + sin_values = cute.make_rmem_tensor((8,), cutlass.Float32) + for chunk_idx in cutlass.range_constexpr(4): + cos_tile = cute.local_tile(rope_row, (8,), (chunk_idx,)) + sin_tile = cute.local_tile(rope_row, (8,), (chunk_idx + 4,)) + cute.copy( + copy_atom_f32x8, + cute.coalesce(cos_tile), + cute.coalesce(cos_values), + ) + cute.copy( + copy_atom_f32x8, + cute.coalesce(sin_tile), + cute.coalesce(sin_values), + ) + for pair_in_chunk in cutlass.range_constexpr(8): + pair_idx = chunk_idx * 8 + pair_in_chunk + cosine = cos_values[pair_in_chunk] + sine = sin_values[pair_in_chunk] + x = tCompute[pair_idx * 2] + y = tCompute[pair_idx * 2 + 1] + tCompute[pair_idx * 2] = cosine * x - sine * y + tCompute[pair_idx * 2 + 1] = cosine * y + sine * x + tCompute.store(tCompute.load().to(cutlass.BFloat16).to(cutlass.Float32)) if cutlass.const_expr(self.generate_sfc): # @@ -1639,19 +1728,52 @@ def kernel( * norm_const ) - # TODO: need to add f32x2 -> f8x2 conversion - tCrSFC.store(tCrSFC_pvscale.load().to(self.sf_dtype)) + # CuTe's packed f32x2 -> UE8M0 conversion currently + # lowers a two-element result through vector<0xi32>. + # MXF8 with sf_vec_size=32 has exactly two output scale + # values per epilogue thread, so keep these conversions + # scalar until that lowering is fixed upstream. + if cutlass.const_expr(self.sf_vec_size == 32): + for vi in cutlass.range_constexpr(cute.size(tCrSFC)): + tCrSFC[vi] = tCrSFC_pvscale[vi].to(self.sf_dtype) + else: + tCrSFC.store(tCrSFC_pvscale.load().to(self.sf_dtype)) # # Store SFC to global memory # - cute.autovec_copy(tCrSFC, tCgSFC) + if cutlass.const_expr(self.indexer_q_fusion): + subtile_m = real_subtile_idx // self.epi_tile_cnt[1] + subtile_n = real_subtile_idx % self.epi_tile_cnt[1] + scale_m = ( + cur_tile_coord[0] * self.cta_tile_shape_mnk_c[0] + + subtile_m * self.epi_tile[0] + + tTR_cC[0][0] + ) + if scale_m < mIndexerScale.shape[0]: + scale_bytes = cute.recast_tensor(tCrSFC, cutlass.Uint8) + for vi in cutlass.range_constexpr(cute.size(tCrSFC)): + scale_byte_idx = subtile_n * 2 + vi + packed_indexer_scale = packed_indexer_scale | ( + cutlass.Uint32(scale_bytes[vi]) + << cutlass.Uint32(scale_byte_idx * 8) + ) + else: + cute.autovec_copy(tCrSFC, tCgSFC) # # Compute quantized output values and convert to C type # - # TODO: need to add f8x2 -> f32x2 conversion - tCrSFC_qpvscale_up = tCrSFC.load().to(cutlass.Float32) + # Same two-element UE8M0 lowering issue applies in the + # reverse direction. + if cutlass.const_expr(self.sf_vec_size == 32): + tCrSFC_qpvscale_up = cute.make_rmem_tensor( + tCrSFC.shape, cutlass.Float32 + ) + for vi in cutlass.range_constexpr(cute.size(tCrSFC)): + tCrSFC_qpvscale_up[vi] = cutlass.Float32(tCrSFC[vi]) + else: + tCrSFC_qpvscale_up = tCrSFC.load().to(cutlass.Float32) fp32_max = cutlass.Float32(3.40282346638528859812e38) if cutlass.const_expr(self.vectorized_f32): for vi in cutlass.range_constexpr(0, cute.size(tCrSFC), 2): @@ -1686,6 +1808,30 @@ def kernel( acc_vec = tiled_copy_r2s.retile(tCompute).load() tRS_rC.store(acc_vec.to(self.c_dtype)) + if cutlass.const_expr(self.indexer_q_fusion): + packed_words = cute.recast_tensor(tTR_rC, cutlass.Uint32) + for word_idx in cutlass.range_constexpr(cute.size(packed_words)): + packed = packed_words[word_idx] + midpoint_adjust = cutlass.Uint32(0) + for nibble_idx in cutlass.range_constexpr(8): + scaled_value = tCompute[word_idx * 8 + nibble_idx] + abs_value = cute.arch.fmax(scaled_value, -scaled_value) + nibble_adjust = cutlass.Uint32(1) << (nibble_idx * 4) + if abs_value == cutlass.Float32(0.75): + midpoint_adjust = midpoint_adjust | nibble_adjust + if abs_value == cutlass.Float32(1.75): + midpoint_adjust = midpoint_adjust | nibble_adjust + if abs_value == cutlass.Float32(3.5): + midpoint_adjust = midpoint_adjust | nibble_adjust + packed = packed - midpoint_adjust + magnitude = packed & cutlass.Uint32(0x77777777) + nonzero_sign = ( + (magnitude | (magnitude << 1) | (magnitude << 2)) + & cutlass.Uint32(0x44444444) + ) << 1 + packed_words[word_idx] = packed & ( + cutlass.Uint32(0x77777777) | nonzero_sign + ) else: # # Convert to C type (non-SFC path) @@ -1732,6 +1878,10 @@ def kernel( barrier_id=self.epilog_sync_bar_id, number_of_threads=epilog_threads, ) + if cutlass.const_expr(self.indexer_q_fusion and self.generate_sfc): + scale_m = cur_tile_coord[0] * self.cta_tile_shape_mnk_c[0] + tTR_cC[0][0] + if scale_m < mIndexerScale.shape[0]: + mIndexerScale[scale_m, cur_tile_coord[1], 0] = packed_indexer_scale # # Async arrive accumulator buffer empty @@ -1944,7 +2094,14 @@ def epilog_tmem_copy_and_partition( epi_tile: cute.Tile, use_2cta_instrs: Union[cutlass.Boolean, bool], tCgBias: Optional[cute.Tensor] = None, - ) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor, cute.Tensor, Optional[cute.Tensor]]: + ) -> Tuple[ + cute.TiledCopy, + cute.Tensor, + cute.Tensor, + cute.Tensor, + Optional[cute.Tensor], + cute.Tensor, + ]: """ Make tiledCopy for tensor memory load, then use it to partition tensor memory (source) and register array (destination). @@ -2007,6 +2164,8 @@ def epilog_tmem_copy_and_partition( tTR_rAcc_gate = cute.make_rmem_tensor( tTR_gC[(None, None, None, 0, 0, 0, 0, 0)].shape, self.acc_dtype ) + cC = cute.make_identity_tensor(epi_tile) + tTR_cC = thr_copy_t2r.partition_D(cC) # Partition the per-N bias EXACTLY like C (broadcast over M via stride 0). # Same flat_divide + partition_D path -> tTR_gBias has the same layout as @@ -2018,7 +2177,14 @@ def epilog_tmem_copy_and_partition( ) tTR_gBias = thr_copy_t2r.partition_D(gBias_mnl_epi) - return tiled_copy_t2r, tTR_tAcc, tTR_rAcc_up, tTR_rAcc_gate, tTR_gBias + return ( + tiled_copy_t2r, + tTR_tAcc, + tTR_rAcc_up, + tTR_rAcc_gate, + tTR_gBias, + tTR_cC, + ) def epilog_smem_copy_and_partition( self, @@ -2606,6 +2772,98 @@ def wrapper( bias_tensor=bias_tensor, ) + @cute.jit + def wrapper_indexer_q( + self, + m: cutlass.Int64, + n: cutlass.Int64, + k: cutlass.Int64, + sf_m: cutlass.Int64, + sf_n: cutlass.Int64, + sf_k: cutlass.Int64, + cos_sin_rows: cutlass.Int64, + l: cutlass.Constexpr, # noqa: E741 + a_ptr: cute.Pointer, + b_ptr: cute.Pointer, + a_sf_ptr: cute.Pointer, + b_sf_ptr: cute.Pointer, + packed_ptr: cute.Pointer, + scale_ptr: cute.Pointer, + position_ids_ptr: cute.Pointer, + cos_sin_ptr: cute.Pointer, + alpha_tensor: cute.Tensor, + max_active_clusters: cutlass.Constexpr, + current_stream: cuda.CUstream, + ): + """MXF8 GEMM with the DeepSeek-V4 indexer-Q RoPE/MXFP4 epilogue. + + ``packed_ptr`` and ``scale_ptr`` use the indexer's existing contiguous + layouts: uint8 [M, N/2] and UE8M0 [M, N/32]. The output tensor is a + logical FP4 view so the repository's native FP4 shared-memory layout + and TMA store path can be reused. + """ + a_tensor = cute.make_tensor( + a_ptr, + layout=cute.make_ordered_layout((m, k, l), order=(1, 0, 2)), + ) + b_tensor = cute.make_tensor( + b_ptr, + layout=cute.make_ordered_layout((n, k, l), order=(1, 0, 2)), + ) + c_tensor = cute.make_tensor( + cute.recast_ptr(packed_ptr, dtype=cutlass.Float4E2M1FN), + layout=cute.make_ordered_layout((m, n, l), order=(1, 0, 2)), + ) + scale_tensor = cute.make_tensor( + cute.recast_ptr(scale_ptr, dtype=cutlass.Uint32), + layout=cute.make_ordered_layout((m, n // 128, l), order=(1, 0, 2)), + ) + # generate_sfc needs a scale-factor-shaped tensor to derive its register + # partition. Indexer-Q stores the values through scale_tensor above, + # preserving the consumer's contiguous per-head four-byte layout. + sfc_tensor = cute.make_tensor( + scale_ptr, + layout=cute.make_ordered_layout((32, 4, sf_m, 4, sf_n, l), order=(2, 1, 4, 0, 3, 5)), + ) + position_ids_tensor = cute.make_tensor( + position_ids_ptr, + layout=cute.make_layout((m,)), + ) + cos_sin_cache_tensor = cute.make_tensor( + cos_sin_ptr, + layout=cute.make_ordered_layout((cos_sin_rows, 64), order=(1, 0)), + ) + sfa_tensor = cute.make_tensor( + a_sf_ptr, + layout=cute.make_ordered_layout( + (32, 4, sf_m, 4, sf_k, l), + order=(2, 1, 4, 0, 3, 5), + ), + ) + sfb_tensor = cute.make_tensor( + b_sf_ptr, + layout=cute.make_ordered_layout( + (32, 4, sf_n, 4, sf_k, l), + order=(2, 1, 4, 0, 3, 5), + ), + ) + + self( + a_tensor, + b_tensor, + sfa_tensor, + sfb_tensor, + c_tensor, + alpha_tensor, + max_active_clusters, + current_stream, + sfc_tensor=sfc_tensor, + norm_const_tensor=alpha_tensor, + indexer_scale_tensor=scale_tensor, + position_ids_tensor=position_ids_tensor, + cos_sin_cache_tensor=cos_sin_cache_tensor, + ) + @cute.jit def wrapper_fp4out( self, diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_persistent.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_persistent.py index 78c8532414b7..0194cbc8359e 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_persistent.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_persistent.py @@ -52,13 +52,48 @@ import cutlass.utils as utils import cutlass.utils.blackwell_helpers as sm100_utils import cutlass.utils.blockscaled_layout as blockscaled_utils +from cutlass._mlir.dialects import llvm from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cutlass_dsl import dsl_user_op from .custom_pipeline import PipelineTmaUmma, PipelineUmmaAsync from .utils import (TRTLLM_ENABLE_PDL, griddepcontrol_launch_dependents, griddepcontrol_wait, is_power_of_2) +@dsl_user_op +def _indexer_q_pack_fp4x4(value0: cutlass.Float32, + value1: cutlass.Float32, + value2: cutlass.Float32, + value3: cutlass.Float32, + *, + loc=None, + ip=None) -> cutlass.Uint16: + """Pack four FP32 values with two native E2M1x2 conversions.""" + return cutlass.Uint16( + llvm.inline_asm( + cutlass.Uint16.mlir_type, + [ + value0.ir_value(loc=loc, ip=ip), + value1.ir_value(loc=loc, ip=ip), + value2.ir_value(loc=loc, ip=ip), + value3.ir_value(loc=loc, ip=ip), + ], + """{ + .reg .b8 byte0, byte1; + cvt.rn.satfinite.e2m1x2.f32 byte0, $2, $1; + cvt.rn.satfinite.e2m1x2.f32 byte1, $4, $3; + mov.b16 $0, {byte0, byte1}; + }""", + "=h,f,f,f,f", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + )) + + class Sm100BlockScaledPersistentDenseGemmKernel: """Implements batched matrix multiplication (C = A x SFA x B x SFB) with support for various data types and Blackwell GPU architectural features, including persistent tile scheduling and warp specialization. @@ -100,6 +135,8 @@ def __init__( use_prefetch: bool = False, swizzle_size: int = 1, raster_along_m: bool = True, + indexer_q_fusion: bool = False, + indexer_transform_warps: int = 4, ): """Initializes the configuration for a Blackwell dense GEMM kernel. @@ -134,6 +171,16 @@ def __init__( self.use_prefetch = use_prefetch self.swizzle_size = swizzle_size self.raster_along_m = raster_along_m + self.indexer_q_fusion = indexer_q_fusion + self.indexer_transform_warps = indexer_transform_warps + if self.indexer_q_fusion and (sf_vec_size != 32 or mma_tiler_mn[0] + != 128 or mma_tiler_mn[1] != 16 + or cluster_shape_mn != (1, 1) + or indexer_transform_warps not in (4, 8)): + raise ValueError( + "The swapped indexer-Q epilogue currently requires " + "MXF8, a 128x16 MMA tile, a 1x1 cluster, and 4 or 8 " + "row-transform warps") self.cta_group = (tcgen05.CtaGroup.TWO if self.use_2cta_instrs else tcgen05.CtaGroup.ONE) @@ -147,8 +194,12 @@ def __init__( ) self.mma_warp_id = 4 self.tma_warp_id = 5 + self.indexer_extra_warp_id = tuple( + range(self.tma_warp_id + 1, self.tma_warp_id + 1 + + indexer_transform_warps - 4)) if self.indexer_q_fusion else () self.threads_per_cta = 32 * len( - (self.mma_warp_id, self.tma_warp_id, *self.epilog_warp_id)) + (self.mma_warp_id, self.tma_warp_id, *self.epilog_warp_id, + *self.indexer_extra_warp_id)) # Set barrier id for cta sync, epilogue sync and tmem ptr sync self.cta_sync_bar_id = 0 self.epilog_sync_bar_id = 1 @@ -257,6 +308,14 @@ def _setup_attributes(self): ) self.epi_tile_n = cute.size(self.epi_tile[1]) + expected_indexer_epi_n = min(self.mma_tiler[1], 32) + if (self.indexer_q_fusion + and (cute.size(self.epi_tile[0]) != 128 + or self.epi_tile_n != expected_indexer_epi_n)): + raise ValueError( + f"Unexpected swapped indexer-Q epilogue tile {self.epi_tile}; " + f"expected a 128-feature by {expected_indexer_epi_n}-token " + "BF16 stage") # Setup A/B/C stage count in shared memory and ACC stage count in tensor memory self.num_acc_stage, self.num_ab_stage, self.num_c_stage = self._compute_stages( @@ -308,6 +367,10 @@ def _setup_attributes(self): ) self.overlapping_accum = self.num_acc_stage == 1 + if self.indexer_q_fusion and self.overlapping_accum: + raise ValueError( + "The swapped indexer-Q transform-warp schedule requires at " + "least two accumulator stages") sf_atom_mn = 32 self.num_sfa_tmem_cols = (self.cta_tile_shape_mnk[0] // sf_atom_mn) * mma_inst_tile_k @@ -336,6 +399,10 @@ def __call__( max_active_clusters: cutlass.Constexpr, stream: cuda.CUstream, epilogue_op: cutlass.Constexpr = lambda x: x, + packed_tensor: Optional[cute.Tensor] = None, + indexer_scale_tensor: Optional[cute.Tensor] = None, + position_ids_tensor: Optional[cute.Tensor] = None, + cos_sin_cache_tensor: Optional[cute.Tensor] = None, ): """Execute the GEMM operation in steps: - Setup static attributes before smem/grid/tma computation @@ -368,6 +435,17 @@ def __call__( b_tensor).mma_major_mode() self.c_layout = utils.LayoutEnum.from_tensor(c_tensor) + if cutlass.const_expr( + self.indexer_q_fusion and + (self.c_dtype != cutlass.BFloat16 + or self.c_layout != utils.LayoutEnum.COL_MAJOR + or packed_tensor is None or indexer_scale_tensor is None + or position_ids_tensor is None or cos_sin_cache_tensor is None)): + raise ValueError( + "The swapped indexer-Q epilogue requires a column-major BF16 " + "GEMM boundary plus packed output, scale, position, and RoPE tensors" + ) + # Check if input data types are compatible with MMA instruction if cutlass.const_expr(self.a_dtype != self.b_dtype): raise TypeError( @@ -573,6 +651,10 @@ class SharedStorage: tma_tensor_sfb, tma_atom_c, tma_tensor_c, + packed_tensor, + indexer_scale_tensor, + position_ids_tensor, + cos_sin_cache_tensor, self.cluster_layout_vmnk, self.cluster_layout_sfb_vmnk, self.a_smem_layout_staged, @@ -611,6 +693,10 @@ def kernel( mSFB_nkl: cute.Tensor, tma_atom_c: Optional[cute.CopyAtom], mC_mnl: cute.Tensor, + mPacked_nml: Optional[cute.Tensor], + mIndexerScale_nml: Optional[cute.Tensor], + mPositionIds: Optional[cute.Tensor], + mCosSinCache: Optional[cute.Tensor], cluster_layout_vmnk: cute.Layout, cluster_layout_sfb_vmnk: cute.Layout, a_smem_layout_staged: cute.ComposedLayout, @@ -1430,7 +1516,9 @@ def kernel( "async.shared", space="cta", ) - epilog_threads = 32 * len(self.epilog_warp_id) + epilog_threads = 32 * (self.indexer_transform_warps + if self.indexer_q_fusion else len( + self.epilog_warp_id)) cute.arch.barrier( barrier_id=self.epilog_sync_bar_id, number_of_threads=epilog_threads, @@ -1439,15 +1527,31 @@ def kernel( # # TMA store C to global memory # - if warp_idx == self.epilog_warp_id[0]: - cute.copy( - tma_atom_c, - bSG_sC[(None, c_buffer)], - bSG_gC[(None, real_subtile_idx)], + if cutlass.const_expr(self.indexer_q_fusion): + self._indexer_q_transform_rows( + sC, + c_buffer, + warp_idx, + self.indexer_transform_warps, + cur_tile_coord[0], + cur_tile_coord[1], + cur_tile_coord[2], + real_subtile_idx, + mPacked_nml, + mIndexerScale_nml, + mPositionIds, + mCosSinCache, ) - # Fence and barrier to make sure shared memory store is visible to TMA store - c_pipeline.producer_commit() - c_pipeline.producer_acquire() + else: + if warp_idx == self.epilog_warp_id[0]: + cute.copy( + tma_atom_c, + bSG_sC[(None, c_buffer)], + bSG_gC[(None, real_subtile_idx)], + ) + # Fence and barrier to make sure shared memory store is visible to TMA store + c_pipeline.producer_commit() + c_pipeline.producer_acquire() cute.arch.barrier( barrier_id=self.epilog_sync_bar_id, number_of_threads=epilog_threads, @@ -1487,10 +1591,184 @@ def kernel( # # Wait for C store complete # - c_pipeline.producer_tail() + if cutlass.const_expr(not self.indexer_q_fusion): + c_pipeline.producer_tail() + + # Optional row-transform-only warps. The four canonical epilogue + # warps remain the sole TMEM drain owners; these warps join only the + # two shared-stage barriers and process disjoint complete token rows. + if cutlass.const_expr(self.indexer_q_fusion + and self.indexer_transform_warps > 4): + if (warp_idx >= self.indexer_extra_warp_id[0] + and warp_idx <= self.indexer_extra_warp_id[-1]): + tile_sched = utils.StaticPersistentTileScheduler.create( + tile_sched_params, cute.arch.block_idx(), + cute.arch.grid_dim()) + work_tile = tile_sched.initial_work_tile_info() + subtile_cnt = (self.cta_tile_shape_mnk[1] // self.epi_tile_n) + transform_threads = 32 * self.indexer_transform_warps + transform_warp_idx = (len(self.epilog_warp_id) + warp_idx - + self.indexer_extra_warp_id[0]) + + while work_tile.is_valid_tile: + cur_tile_coord = work_tile.tile_idx + num_prev_subtiles = (tile_sched.num_tiles_executed * + subtile_cnt) + for subtile_idx in cutlass.range(subtile_cnt): + c_buffer = ((num_prev_subtiles + subtile_idx) % + self.num_c_stage) + cute.arch.barrier( + barrier_id=self.epilog_sync_bar_id, + number_of_threads=transform_threads, + ) + self._indexer_q_transform_rows( + sC, + c_buffer, + transform_warp_idx, + self.indexer_transform_warps, + cur_tile_coord[0], + cur_tile_coord[1], + cur_tile_coord[2], + subtile_idx, + mPacked_nml, + mIndexerScale_nml, + mPositionIds, + mCosSinCache, + ) + cute.arch.barrier( + barrier_id=self.epilog_sync_bar_id, + number_of_threads=transform_threads, + ) + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() griddepcontrol_launch_dependents() + @cute.jit + def _indexer_q_transform_rows( + self, + sC: cute.Tensor, + c_buffer: cutlass.Int32, + row_start: cutlass.Int32, + row_stride: cutlass.Constexpr, + head_idx: cutlass.Int32, + token_tile_idx: cutlass.Int32, + batch_idx: cutlass.Int32, + real_subtile_idx: cutlass.Int32, + mPacked_nml: cute.Tensor, + mIndexerScale_nml: cute.Tensor, + mPositionIds: cute.Tensor, + mCosSinCache: cute.Tensor, + ): + """Transform complete BF16 token rows from the shared epilogue tile.""" + lane_idx = cute.arch.lane_idx() + for row in cutlass.range(row_start, + self.epi_tile_n, + row_stride, + unroll_full=True): + token_idx = (token_tile_idx * self.cta_tile_shape_mnk[1] + + real_subtile_idx * self.epi_tile_n + row) + if token_idx < mPositionIds.shape[0]: + values = cute.make_rmem_tensor((4, ), cutlass.Float32) + for value_idx in cutlass.range_constexpr(4): + feature_idx = lane_idx * 4 + value_idx + values[value_idx] = cutlass.Float32(sC[(feature_idx, row, + c_buffer)]) + + if lane_idx >= 16: + position = mPositionIds[token_idx] + pair_base = (lane_idx * 4 - 64) // 2 + for value_idx in cutlass.range_constexpr(0, 4, 2): + cosine = mCosSinCache[position, + pair_base + value_idx // 2] + sine = mCosSinCache[position, + pair_base + value_idx // 2 + 32] + x = values[value_idx] + y = values[value_idx + 1] + values[value_idx] = (cosine * x - sine * y).to( + cutlass.BFloat16).to(cutlass.Float32) + values[value_idx + 1] = (cosine * y + sine * x).to( + cutlass.BFloat16).to(cutlass.Float32) + + amax = cute.arch.fmax( + cute.arch.fmax(values[0], -values[0]), + cute.arch.fmax(values[1], -values[1]), + ) + amax = cute.arch.fmax( + amax, + cute.arch.fmax( + cute.arch.fmax(values[2], -values[2]), + cute.arch.fmax(values[3], -values[3]), + ), + ) + amax = cute.arch.fmax( + amax, cute.arch.shuffle_sync_bfly(amax, offset=1)) + amax = cute.arch.fmax( + amax, cute.arch.shuffle_sync_bfly(amax, offset=2)) + amax = cute.arch.fmax( + amax, cute.arch.shuffle_sync_bfly(amax, offset=4)) + if amax < cutlass.Float32(1.0e-12): + amax = cutlass.Float32(1.0e-12) + + scale_reg = cute.make_rmem_tensor((1, ), cutlass.Float8E8M0FNU) + scale_reg[0] = (amax * cutlass.Float32(1.0 / 6.0)).to( + cutlass.Float8E8M0FNU) + scale_byte = cute.recast_tensor(scale_reg, cutlass.Uint8)[0] + exponent = cutlass.Uint32(scale_byte) + inverse_bits = cutlass.Uint32(0) + if exponent == cutlass.Uint32(254): + inverse_bits = cutlass.Uint32(0x00400000) + else: + inverse_bits = (cutlass.Uint32(254) - exponent) << 23 + inverse_scale = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, + inverse_bits.ir_value())) + for value_idx in cutlass.range_constexpr(4): + values[value_idx] = values[value_idx] * inverse_scale + + packed = _indexer_q_pack_fp4x4( + values[0], + values[1], + values[2], + values[3], + ) + midpoint_adjust = cutlass.Uint16(0) + for value_idx in cutlass.range_constexpr(4): + abs_value = cute.arch.fmax(values[value_idx], + -values[value_idx]) + nibble_adjust = cutlass.Uint16(1 << (value_idx * 4)) + if abs_value == cutlass.Float32(0.75): + midpoint_adjust = midpoint_adjust | nibble_adjust + if abs_value == cutlass.Float32(1.75): + midpoint_adjust = midpoint_adjust | nibble_adjust + if abs_value == cutlass.Float32(3.5): + midpoint_adjust = midpoint_adjust | nibble_adjust + packed = packed - midpoint_adjust + magnitude = packed & cutlass.Uint16(0x7777) + nonzero_sign = ((magnitude | (magnitude << 1) + | (magnitude << 2)) + & cutlass.Uint16(0x4444)) << 1 + packed = cutlass.Uint16(packed & (cutlass.Uint16(0x7777) + | nonzero_sign)) + mPacked_nml[head_idx * 32 + lane_idx, token_idx, + batch_idx] = packed + + exponent0 = cutlass.Uint32( + cute.arch.shuffle_sync(exponent, cutlass.Int32(0))) + exponent1 = cutlass.Uint32( + cute.arch.shuffle_sync(exponent, cutlass.Int32(8))) + exponent2 = cutlass.Uint32( + cute.arch.shuffle_sync(exponent, cutlass.Int32(16))) + exponent3 = cutlass.Uint32( + cute.arch.shuffle_sync(exponent, cutlass.Int32(24))) + if lane_idx == 0: + mIndexerScale_nml[ + head_idx, + token_idx, + batch_idx, + ] = (exponent0 | (exponent1 << 8) | (exponent2 << 16) + | (exponent3 << 24)) + def mainloop_s2t_copy_and_partition( self, sSF: cute.Tensor, @@ -1940,7 +2218,7 @@ def is_valid_tensor_alignment( m: cutlass.Int64, n: cutlass.Int64, k: cutlass.Int64, - l: cutlass.Int64, + l: cutlass.Int64, # noqa: E741 - CUTLASS names the batch mode L. ab_dtype: Type[cutlass.Numeric], c_dtype: Type[cutlass.Numeric], a_major: str, @@ -1992,7 +2270,7 @@ def can_implement( m: cutlass.Int64, n: cutlass.Int64, k: cutlass.Int64, - l: cutlass.Int64, + l: cutlass.Int64, # noqa: E741 - CUTLASS names the batch mode L. a_major: str, b_major: str, c_major: str, @@ -2050,7 +2328,7 @@ def wrapper( sf_m: cutlass.Int64, sf_n: cutlass.Int64, sf_k: cutlass.Int64, - l: cutlass.Constexpr, + l: cutlass.Constexpr, # noqa: E741 - Preserve the generic wrapper API. a_ptr: cute.Pointer, b_ptr: cute.Pointer, a_sf_ptr: cute.Pointer, @@ -2126,6 +2404,102 @@ def wrapper( self(a_tensor, b_tensor, sfa_tensor, sfb_tensor, c_tensor, alpha_tensor, max_active_clusters, current_stream, epilogue_op) + @cute.jit + def wrapper_indexer_q_swap_ab( + self, + m: cutlass.Int64, + n: cutlass.Int64, + k: cutlass.Int64, + sf_m: cutlass.Int64, + sf_n: cutlass.Int64, + sf_k: cutlass.Int64, + cos_sin_rows: cutlass.Int64, + batch_count: cutlass.Constexpr, + a_ptr: cute.Pointer, + b_ptr: cute.Pointer, + a_sf_ptr: cute.Pointer, + b_sf_ptr: cute.Pointer, + packed_ptr: cute.Pointer, + scale_ptr: cute.Pointer, + position_ids_ptr: cute.Pointer, + cos_sin_ptr: cute.Pointer, + alpha_tensor: cute.Tensor, + max_active_clusters: cutlass.Constexpr, + current_stream: cuda.CUstream, + ): + """Run indexer Q with features on MMA-M and tokens on MMA-N. + + The GEMM boundary is a logical column-major BF16 ``[N, M]`` view. + It is never written to global memory; the custom epilogue drains it to + the existing BF16 shared-memory stage, then writes the production + packed-FP4 and four-UE8M0-per-head outputs directly. + """ + # Swap A/B so the fixed 8192 output features occupy hardware MMA-M and + # the small dynamic token count occupies hardware MMA-N. + weight_tensor = cute.make_tensor( + b_ptr, + layout=cute.make_ordered_layout((n, k, batch_count), + order=(1, 0, 2)), + ) + input_tensor = cute.make_tensor( + a_ptr, + layout=cute.make_ordered_layout((m, k, batch_count), + order=(1, 0, 2)), + ) + weight_sf_tensor = cute.make_tensor( + b_sf_ptr, + layout=cute.make_ordered_layout( + (32, 4, sf_n, 4, sf_k, batch_count), + order=(2, 1, 4, 0, 3, 5), + ), + ) + input_sf_tensor = cute.make_tensor( + a_sf_ptr, + layout=cute.make_ordered_layout( + (32, 4, sf_m, 4, sf_k, batch_count), + order=(2, 1, 4, 0, 3, 5), + ), + ) + + # Only the shape/layout participate in scheduling and TMEM partitioning. + # The custom epilogue does not issue a TMA store through this tensor. + c_boundary_tensor = cute.make_tensor( + cute.recast_ptr(packed_ptr, dtype=cutlass.BFloat16), + layout=cute.make_ordered_layout((n, m, batch_count), + order=(0, 1, 2)), + ) + packed_tensor = cute.make_tensor( + cute.recast_ptr(packed_ptr, dtype=cutlass.Uint16), + layout=cute.make_ordered_layout((n // 4, m, batch_count), + order=(0, 1, 2)), + ) + scale_tensor = cute.make_tensor( + cute.recast_ptr(scale_ptr, dtype=cutlass.Uint32), + layout=cute.make_ordered_layout((n // 128, m, batch_count), + order=(0, 1, 2)), + ) + position_ids_tensor = cute.make_tensor(position_ids_ptr, + cute.make_layout((m, ))) + cos_sin_cache_tensor = cute.make_tensor( + cos_sin_ptr, + layout=cute.make_ordered_layout((cos_sin_rows, 64), order=(1, 0)), + ) + + self( + weight_tensor, + input_tensor, + weight_sf_tensor, + input_sf_tensor, + c_boundary_tensor, + alpha_tensor, + max_active_clusters, + current_stream, + packed_tensor=packed_tensor, + indexer_scale_tensor=scale_tensor, + position_ids_tensor=position_ids_tensor, + cos_sin_cache_tensor=cos_sin_cache_tensor, + ) + @cute.jit def cvt_sf_MKL_to_M32x4xrm_K4xrk_L( diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/block_scan.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/block_scan.py index 8e8b8cb52219..0f29f41ca98d 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/block_scan.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/block_scan.py @@ -71,11 +71,10 @@ def block_prefix_sum_kernel( warp_id = tidx // 32 lane_id = tidx % 32 - # Currently, we only support num_warps > 1, will support num_warps <= 1 logic later. assert num_threads % 32 == 0, "num_threads must be divisible by 32, but got {}".format( num_threads ) - assert num_warps > 1, "num_warps must be > 1, but got {}".format(num_warps) + assert num_warps >= 1, "num_warps must be >= 1, but got {}".format(num_warps) assert num_warps == 2 ** int(math.log2(num_warps)), "num_warps must be a power of 2" # Step 1: Warp-level prefix sum using shuffle diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_decode_varlen.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_decode_varlen.py index 66b79c6986f8..5669146c7ca1 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_decode_varlen.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_decode_varlen.py @@ -22,7 +22,6 @@ import cutlass.utils as utils import torch from cutlass.torch import dtype as torch_dtype -from cutlass.utils.distributed import atomicAdd from ..utils import TRTLLM_ENABLE_PDL, griddepcontrol_launch_dependents, griddepcontrol_wait from .block_scan import block_prefix_sum_kernel @@ -33,6 +32,14 @@ run_reference_top_k, ) + +def _get_num_sms() -> int: + """Return the number of SMs on the current device (cached).""" + if not hasattr(_get_num_sms, "_value"): + _get_num_sms._value = torch.cuda.get_device_properties().multi_processor_count + return _get_num_sms._value + + """ A high-performance topk kernel example based on radix-based filter algorithm for the NVIDIA Blackwell SM100 architecture based on CuTe DSL. @@ -56,7 +63,6 @@ --top_k 2048 --do_ref_check --return_val --do_benchmark Constraints for this example: -* The problem size of top_k <= 2048. * The input tensor has data contiguous on the n dimension (row-major). * The supported input data types are Float32, Float16, or BFloat16. """ @@ -166,9 +172,11 @@ def __init__( merge_blocks: bool = False, enable_dynamic_multi_cta: bool = False, varlen_merge_input: bool = False, - num_sms: int = 148, - debug: bool = False, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, + single_pass_multi_cta: bool = False, ): + self._large_occupancy = large_occupancy super().__init__( dtype, max_num_cols, @@ -179,6 +187,12 @@ def __init__( chunk_size_per_cta, num_ctas_per_row, merge_blocks, + overflow_policy=overflow_policy, + # large_occupancy always uses 512 threads; pass it early so that + # _compute_smem_input_size_for_occupancy() sees the correct num_warps. + num_threads_override=512 if large_occupancy else 0, + cache_smem_values=cache_smem_values, + single_pass_multi_cta=single_pass_multi_cta, ) self.next_n = next_n self.enable_multi_cta = enable_multi_cta @@ -187,112 +201,20 @@ def __init__( self.num_ctas_per_row = num_ctas_per_row self.enable_dynamic_multi_cta = enable_dynamic_multi_cta self.varlen_merge_input = varlen_merge_input - self.num_sms = num_sms - - if cutlass.const_expr(large_occupancy): - # tuned value, could be tuned further. - # reduce the smem usage and improve occupancy. - if self.max_num_cols >= 262144: - self.filtered_topk_smem_input_size = 4096 - elif self.max_num_cols >= 131072: - self.filtered_topk_smem_input_size = 3072 - elif self.max_num_cols >= 65536: - self.filtered_topk_smem_input_size = 2048 - elif self.max_num_cols >= 32768: - self.filtered_topk_smem_input_size = 1024 - elif self.max_num_cols >= 16384: - self.filtered_topk_smem_input_size = 1024 - elif self.max_num_cols >= 8192: - self.filtered_topk_smem_input_size = 512 - else: - self.filtered_topk_smem_input_size = 256 - if cutlass.const_expr(self.max_num_cols > self.filtered_topk_smem_input_size): - self.enable_gmem_store = True - else: - self.enable_gmem_store = False - - # set the number of threads per cta to 512. - if cutlass.const_expr(not self.merge_blocks): - self.num_threads_per_cta = 512 - else: - # For merge_blocks, cap num_threads_per_cta so that the tile - # width (num_threads_per_cta * vec_size) does not exceed - # max_num_cols. Otherwise, out-of-bounds padding elements - # created by _fill_oob are counted in the radix histogram - # and may be selected as top-k candidates with invalid - # indices, causing incorrect results. - self.num_threads_per_cta = min(self.max_num_cols // self.vec_size, 512) - - # only used for debug info - if cutlass.const_expr(debug): - print(f"dtype: {self.dtype}, vec_size: {self.vec_size}") - print( - f"max_num_cols: {self.max_num_cols}, num_threads_per_cta: {self.num_threads_per_cta}" - ) - print(f"filtered_topk_smem_input_size: {self.filtered_topk_smem_input_size}") - print(f"enable_gmem_store: {self.enable_gmem_store}") - print(f"return_val: {self.return_val}") - print(f"large_occupancy: {large_occupancy}") - print(f"filtered_topk_smem_input_size: {self.filtered_topk_smem_input_size}") - print( - f"first_refine_shift: {self.first_refine_shift}, num_refine_rounds: {self.num_refine_rounds}" + if cutlass.const_expr(self.merge_blocks): + # Cap vec_size so tile_width (num_threads_per_cta * vec_size) <= max_num_cols, + # preventing OOB s_indices from _fill_oob padding. + _vec_cap = max( + 1, 2 ** int(math.log2(max(self.max_num_cols // self.num_threads_per_cta, 1))) ) + self.num_copy_bits = min(self.num_copy_bits, _vec_cap * self.dtype.width) + self.vec_size = self.num_copy_bits // self.dtype.width - @cute.jit - def run_kernel( - self, - input, - indices, - extra_buffer, - output_indices, - output_values, - tiler_mn, - copy_atom, - tiled_copy, - seqlen, - task_id, - s_histogram, - s_counter, - s_threshold_bin_id, - s_num_input, - g_num_input, - s_indices, - s_input_idx, - s_last_remain, - num_warps, - s_warp_sums, - ): - # TODO: update row_start to align with multi-cta version. - row_start = 0 - seq_len = seqlen[task_id // self.next_n] - row_end = seq_len - self.next_n + (task_id % self.next_n) + 1 - - length = row_end - row_start - - self.filtered_topk_kernel_per_row( - input, - indices, - extra_buffer, - output_indices, - output_values, - tiler_mn, - copy_atom, - tiled_copy, - row_start, - length, - task_id, - s_histogram, - s_counter, - s_threshold_bin_id, - s_num_input, - g_num_input, - s_indices, - s_input_idx, - s_last_remain, - num_warps, - s_warp_sums, - ) + def _compute_smem_input_size(self) -> int: + if cutlass.const_expr(self._large_occupancy): + return self._compute_smem_input_size_for_occupancy(target_blocks_per_sm=4) + return self._compute_smem_input_size_for_occupancy(target_blocks_per_sm=1) @cute.kernel def filtered_topk_kernel( @@ -300,15 +222,9 @@ def filtered_topk_kernel( input: cute.Tensor, indices: cute.Tensor, extra_buffer: cute.Tensor, - g_global_counter: cute.Tensor, seqlen: cute.Tensor, output_indices: cute.Tensor, output_values: cute.Tensor, - tiler_mn: cute.Shape, - copy_atom: cute.CopyAtom, - tiled_copy: cute.TiledCopy, - enable_persistent_dynamic_scheduling: cutlass.Constexpr[bool] = False, - min_blocks_per_mp: cutlass.Constexpr[int] = 1, ): """CuTe DSL implementation of TopK kernel based on radix-based filter algorithm.""" griddepcontrol_wait() @@ -346,23 +262,48 @@ def filtered_topk_kernel( g_num_input = None s_indices = smem.allocate_tensor( element_type=self.index_type, - layout=cute.make_ordered_layout((self.filtered_topk_max_k,), order=(0)), + layout=cute.make_ordered_layout((self.top_k,), order=(0)), byte_alignment=128, ) - s_input_idx = smem.allocate_tensor( - element_type=self.index_type, - layout=cute.make_ordered_layout( - ( - self.num_buffer_smem_input_idx, - self.filtered_topk_smem_input_size, + if cutlass.const_expr(not self.enable_reread_always): + s_input_idx = smem.allocate_tensor( + element_type=self.index_type, + layout=cute.make_ordered_layout( + ( + self.num_buffer_smem_input_idx, + self.filtered_topk_smem_input_size, + ), + order=(1, 0), ), - order=(1, 0), - ), - byte_alignment=128, - ) + byte_alignment=128, + ) + else: + s_input_idx = None + if cutlass.const_expr(self.cache_smem_values and not self.enable_reread_always): + s_input_val = smem.allocate_tensor( + element_type=self.ordered_type, + layout=cute.make_ordered_layout( + ( + self.num_buffer_smem_input_idx, + self.filtered_topk_smem_input_size, + ), + order=(1, 0), + ), + byte_alignment=128, + ) + else: + s_input_val = None + if cutlass.const_expr(self.enable_reread): + s_overflow_flag = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((1,), order=(0,)), + byte_alignment=128, + ) + else: + s_overflow_flag = None s_last_remain = smem.allocate_tensor( element_type=cutlass.Int32, - layout=cute.make_ordered_layout((1), order=(0)), + layout=cute.make_ordered_layout((1,), order=(0,)), byte_alignment=128, ) num_warps = cutlass.const_expr( @@ -373,11 +314,54 @@ def filtered_topk_kernel( layout=cute.make_ordered_layout((num_warps,), order=(0,)), byte_alignment=128, ) + # SP multi-CTA (radix-filter cluster): separate DSMEM merge target so the + # local s_histogram is never written in-place while peers read it. + # (The collection prefix-scan scratch reuses s_histogram, not this buffer.) + if cutlass.const_expr(self.single_pass_multi_cta): + s_hist_merged = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((self.radix + 1), order=(0)), + byte_alignment=128, + ) + else: + s_hist_merged = None + + # Thread and block indexing + bidx, bidy, _ = cute.arch.block_idx() + + # ---- SP multi-CTA (radix-filter cluster) block indexing + dispatch ---- + # 1D grid = (num_rows * ctas_per_group,), cluster = (ctas_per_group,1,1). + # row_id / cta_in_group derived from the global block index; needed_ctas + # from seqlen decides solo (needed_ctas==1) vs cluster (>=2) at runtime. + need_cluster_sync = False + cta_in_group = 0 + # TODO: move this if to line 411-412? make the logic more clear. + if cutlass.const_expr(self.single_pass_multi_cta): + row_id = bidx // self.num_ctas_per_row + cta_in_group = bidx % self.num_ctas_per_row + _batch = row_id // self.next_n + _off = row_id % self.next_n + _eff = seqlen[_batch] - self.next_n + _off + 1 + chunk_start = self.chunk_size_per_cta * cta_in_group + row_start = chunk_start + row_end = min(_eff, chunk_start + self.chunk_size_per_cta) + length = row_end - row_start + _needed = (_eff + self.chunk_size_per_cta - 1) // self.chunk_size_per_cta + if _needed < 1: + _needed = 1 + need_cluster_sync = _needed >= 2 + # _eff <= top_k: whole row is selected, and the cluster radix path + # can't find a threshold (histogram total == _eff never exceeds + # top_k). Collapse to solo trivial -- only cta 0 runs (guard below, + # cluster-uniform so no deadlock) and takes the full row. + if _eff <= self.top_k: + need_cluster_sync = False + row_start = 0 + length = _eff + # score/dst index the row (not the global block). + bidx = row_id - if cutlass.const_expr(not enable_persistent_dynamic_scheduling): - # Thread and block indexing - bidx, bidy, _ = cute.arch.block_idx() - + if cutlass.const_expr(not self.single_pass_multi_cta): if cutlass.const_expr(self.enable_dynamic_multi_cta): # 2D grid with early exit: bidx = row_id, bidy = chunk_id. # Each CTA computes how many chunks its row actually needs @@ -395,171 +379,79 @@ def filtered_topk_kernel( row_end = seq_len - self.next_n + (bidx % self.next_n) + 1 length = row_end - row_start - if cutlass.const_expr(self.enable_multi_cta): - # update row_start and row_end. - row_start = self.chunk_size_per_cta * bidy - row_end = min(row_end, row_start + self.chunk_size_per_cta) - length = row_end - row_start - output_indices = cute.flat_divide(output_indices, (1, self.top_k))[ - 0, None, bidx, bidy - ] - output_values = cute.flat_divide(output_values, (1, self.top_k))[ - 0, None, bidx, bidy - ] - - if cutlass.const_expr(self.merge_blocks): - if cutlass.const_expr(self.varlen_merge_input): - # Varlen merge: compute per-row valid length from seqlen. - _batch = bidx // self.next_n - _off = bidx % self.next_n - _eff = seqlen[_batch] - self.next_n + _off + 1 - _num_ctas = (_eff + self.chunk_size_per_cta - 1) // self.chunk_size_per_cta - if _num_ctas < 1: - _num_ctas = 1 - merge_width = _num_ctas * self.top_k - row_end = merge_width - length = merge_width - else: - # Existing fixed-length path - # Note, after 1st kernel, the output is fix-lenght. - # Note, for merge_block kernels, need to ensure max_num_cols is the same as bucketed_num_cols. - row_end = self.max_num_cols - length = self.max_num_cols - - # Skip CTAs that exceed this row's actual chunk count. - _should_run = True - if cutlass.const_expr(self.enable_dynamic_multi_cta): - _batch_check = bidx // self.next_n - _off_check = bidx % self.next_n - _eff_check = seqlen[_batch_check] - self.next_n + _off_check + 1 - _needed_ctas = (_eff_check + self.chunk_size_per_cta - 1) // self.chunk_size_per_cta - if _needed_ctas < 1: - _needed_ctas = 1 - _should_run = (bidx < num_rows_val) and (bidy < _needed_ctas) - - if _should_run: - self.filtered_topk_kernel_per_row( - input, - indices, - extra_buffer, - output_indices, - output_values, - tiler_mn, - copy_atom, - tiled_copy, - row_start, - length, - bidx, - s_histogram, - s_counter, - s_threshold_bin_id, - s_num_input, - g_num_input, - s_indices, - s_input_idx, - s_last_remain, - num_warps, - s_warp_sums, - ) - else: - num_rows = input.shape[0] - tidx, _, _ = cute.arch.thread_idx() - bidx, _, _ = cute.arch.block_idx() - - row_start = cutlass.Int32(0) - row_end = cutlass.Int32(0) - length = cutlass.Int32(0) - seq_len = cutlass.Int32(0) - - # Persistent dynamic scheduler. - # First task: use bidx directly (no atomic needed). - # Subsequent tasks: use atomicAdd (counter pre-initialized - # to grid_size on host, so values start from grid_size). - s_row_id = smem.allocate_tensor( - element_type=cute.Int32, - layout=cute.make_ordered_layout((1,), order=(0,)), - byte_alignment=128, + if cutlass.const_expr(self.enable_multi_cta): + # update row_start and row_end. + row_start = self.chunk_size_per_cta * bidy + row_end = min(row_end, row_start + self.chunk_size_per_cta) + length = row_end - row_start + output_indices = cute.flat_divide(output_indices, (1, self.top_k))[0, None, bidx, bidy] + output_values = cute.flat_divide(output_values, (1, self.top_k))[0, None, bidx, bidy] + + if cutlass.const_expr(self.merge_blocks): + if cutlass.const_expr(self.varlen_merge_input): + # Varlen merge: compute per-row valid length from seqlen. + _batch = bidx // self.next_n + _off = bidx % self.next_n + _eff = seqlen[_batch] - self.next_n + _off + 1 + _num_ctas = (_eff + self.chunk_size_per_cta - 1) // self.chunk_size_per_cta + if _num_ctas < 1: + _num_ctas = 1 + merge_width = _num_ctas * self.top_k + row_end = merge_width + length = merge_width + else: + # Existing fixed-length path + # Note, after 1st kernel, the output is fix-lenght. + # Note, for merge_block kernels, need to ensure max_num_cols is the same as bucketed_num_cols. + row_end = self.max_num_cols + length = self.max_num_cols + + # Skip CTAs that exceed this row's actual chunk count. + _should_run = True + if cutlass.const_expr(self.enable_dynamic_multi_cta): + _batch_check = bidx // self.next_n + _off_check = bidx % self.next_n + _eff_check = seqlen[_batch_check] - self.next_n + _off_check + 1 + _needed_ctas = (_eff_check + self.chunk_size_per_cta - 1) // self.chunk_size_per_cta + if _needed_ctas < 1: + _needed_ctas = 1 + _should_run = (bidx < num_rows_val) and (bidy < _needed_ctas) + if cutlass.const_expr(self.single_pass_multi_cta): + # Solo fast path (needed_ctas == 1): only cta_in_group 0 has data; + # the rest exit silently. Because the branch is cluster-uniform + # (all CTAs of a cluster compute the same need_cluster_sync) no CTA + # waits on a cluster barrier, so this cannot deadlock. In cluster + # mode (need_cluster_sync) every CTA must run (no early exit). + if (not need_cluster_sync) and cta_in_group != 0: + _should_run = False + + if _should_run: + self.filtered_topk_kernel_per_row( + input, + indices, + extra_buffer, + output_indices, + output_values, + row_start, + length, + bidx, + s_histogram, + s_counter, + s_threshold_bin_id, + s_num_input, + g_num_input, + s_indices, + s_input_idx, + s_input_val, + s_last_remain, + num_warps, + s_warp_sums, + s_overflow_flag, + need_cluster_sync, + s_hist_merged, + cta_in_group, ) - # First task: deterministic assignment by block index. - task_id = bidx - if task_id < num_rows: - row_start = 0 - seq_len = seqlen[task_id // self.next_n] - row_end = seq_len - self.next_n + (task_id % self.next_n) + 1 - length = row_end - row_start - - self.filtered_topk_kernel_per_row( - input, - indices, - extra_buffer, - output_indices, - output_values, - tiler_mn, - copy_atom, - tiled_copy, - row_start, - length, - task_id, - s_histogram, - s_counter, - s_threshold_bin_id, - s_num_input, - g_num_input, - s_indices, - s_input_idx, - s_last_remain, - num_warps, - s_warp_sums, - ) - - # Subsequent tasks: dynamic work stealing via atomic counter. - # Counter starts at 0, so offset by grid_size to skip - # the first-round tasks already handled by bidx. - grid_size_x, _, _ = cute.arch.grid_dim() - work_remaining = task_id < num_rows - while work_remaining: - if tidx == 0: - s_row_id[0] = ( - atomicAdd(g_global_counter.iterator, cutlass.Int32(1)) + grid_size_x - ) - cute.arch.barrier() - - row_id = s_row_id[0] - has_work = row_id < num_rows - - if has_work: - task_id = row_id - row_start = 0 - seq_len = seqlen[task_id // self.next_n] - row_end = seq_len - self.next_n + (task_id % self.next_n) + 1 - length = row_end - row_start - - self.filtered_topk_kernel_per_row( - input, - indices, - extra_buffer, - output_indices, - output_values, - tiler_mn, - copy_atom, - tiled_copy, - row_start, - length, - task_id, - s_histogram, - s_counter, - s_threshold_bin_id, - s_num_input, - g_num_input, - s_indices, - s_input_idx, - s_last_remain, - num_warps, - s_warp_sums, - ) - work_remaining = has_work - griddepcontrol_launch_dependents() @cute.jit @@ -568,52 +460,36 @@ def __call__( input_values, indices, extra_buffer, - g_global_counter, seqlen, output_indices, output_values, stream: cuda.CUstream, - enable_persistent_dynamic_scheduling: cutlass.Constexpr[bool] = False, min_blocks_per_mp: cutlass.Constexpr[int] = 1, ): """Host function for the filtered topk kernel""" - # now we don't support it. - assert not (self.enable_multi_cta and enable_persistent_dynamic_scheduling), ( - "enable_multi_cta and enable_persistent_dynamic_scheduling cannot both be True" - ) - num_rows = input_values.shape[0] - # each cta processes one row of input. - if cutlass.const_expr(self.enable_dynamic_multi_cta): - blocks = (num_rows, self.num_ctas_per_row, 1) - elif cutlass.const_expr(not enable_persistent_dynamic_scheduling): - blocks = (num_rows, self.num_ctas_per_row, 1) + if cutlass.const_expr(self.single_pass_multi_cta): + # 1D grid = num_rows * ctas_per_group; each cluster owns one row. + blocks = (num_rows * self.num_ctas_per_row, 1, 1) + cluster = (self.num_ctas_per_row, 1, 1) else: - blocks = (min(self.num_sms * min_blocks_per_mp, num_rows), self.num_ctas_per_row, 1) + blocks = (num_rows, self.num_ctas_per_row, 1) + cluster = None - ( - copy_atom, - tiled_copy, - tiler_mn, - ) = self._get_tiled_copy() self.filtered_topk_kernel( input_values, indices, extra_buffer, - g_global_counter, seqlen, output_indices, output_values, - tiler_mn, - copy_atom, - tiled_copy, - enable_persistent_dynamic_scheduling, - min_blocks_per_mp, ).launch( grid=blocks, - block=(tiled_copy.size, 1, 1), + block=(self.num_threads_per_cta, 1, 1), + cluster=cluster, stream=stream, use_pdl=TRTLLM_ENABLE_PDL, + min_blocks_per_mp=min_blocks_per_mp, ) return @@ -647,22 +523,22 @@ def _bucket_num_cols(num_cols: int) -> int: compiled_filter_topk_dict = {} -def cute_dsl_topk_wrapper( +def cute_dsl_radix_filter_topk_wrapper( input_values, seq_lens, top_k, next_n, return_val=True, - load_balance=False, num_copy_bits=256, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, ): torch_dtype = input_values.dtype dtype = _TORCH_TO_CUTLASS_DTYPE[torch_dtype] num_rows, num_cols = input_values.shape bucketed_num_cols = _bucket_num_cols(num_cols) - large_occupancy = num_rows > 148 - assert not load_balance + large_occupancy = num_rows > _get_num_sms() # Note: don't forget num_cols, which means the maximum columns. key = ( @@ -672,8 +548,9 @@ def cute_dsl_topk_wrapper( next_n, return_val, num_copy_bits, - load_balance, large_occupancy, + overflow_policy, + cache_smem_values, ) if key not in compiled_filter_topk_dict: # Create fake tensors for compilation @@ -683,13 +560,16 @@ def cute_dsl_topk_wrapper( input_fake = cute.runtime.make_fake_compact_tensor( dtype, (n_rows, n_cols), stride_order=(1, 0), assumed_align=32 ) - # used for large num_cols - buffer_fake = cute.runtime.make_fake_compact_tensor( - cutlass.Int32, - (cute.sym_int(), cute.sym_int(), cute.sym_int()), - stride_order=(2, 1, 0), - assumed_align=32, - ) + # used for large num_cols (GMEM_SPILL only) + if overflow_policy == "GMEM_SPILL": + buffer_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (cute.sym_int(), cute.sym_int(), cute.sym_int()), + stride_order=(2, 1, 0), + assumed_align=32, + ) + else: + buffer_fake = None seqlen_fake = cute.runtime.make_fake_compact_tensor( cute.Int32, (n_batch,), @@ -718,21 +598,22 @@ def cute_dsl_topk_wrapper( num_copy_bits=num_copy_bits, return_val=return_val, large_occupancy=large_occupancy, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, ) # Compile the kernel compiled_kernel = cute.compile( filtered_topk_func, input_fake, - None, # indices_fake, + None, # indices_fake: unused in this path; pass None to match runtime buffer_fake, - None, # g_global_counter_fake, seqlen_fake, output_indices_fake, output_values_fake, stream=fake_stream, - enable_persistent_dynamic_scheduling=load_balance, - min_blocks_per_mp=1, # TODO: do we need this one? + # TODO: check the perf. + min_blocks_per_mp=4 if large_occupancy else 1, options="--enable-tvm-ffi", ) compiled_filter_topk_dict[key] = compiled_kernel @@ -745,20 +626,19 @@ def cute_dsl_topk_wrapper( else: output_values_torch = None - if dtype == cutlass.Float32: - buffer_numbers = 2 + if overflow_policy == "GMEM_SPILL": + buffer_numbers = 2 if dtype == cutlass.Float32 else 1 + buffer_torch = torch.empty( + num_rows, buffer_numbers, num_cols, dtype=torch.int32, device="cuda" + ) else: - buffer_numbers = 1 - # Note: zeros will trigger an elementwise_add kernel. - buffer_torch = torch.empty(num_rows, buffer_numbers, num_cols, dtype=torch.int32, device="cuda") - g_global_counter_torch = None + buffer_torch = None # TVM FFI uses env stream automatically compiled_kernel( input_values, None, # indices, used for merge blocks kernel of the multi-cta. buffer_torch, - g_global_counter_torch, seq_lens, output_indices_torch, output_values_torch, @@ -766,23 +646,150 @@ def cute_dsl_topk_wrapper( return output_indices_torch, output_values_torch -def cute_dsl_topk_multi_cta_wrapper( +def cute_dsl_radix_filter_topk_single_pass_multi_cta_wrapper( + input_values, + seq_lens, + top_k, + next_n, + cluster_size, + return_val=True, + num_copy_bits=256, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, +): + """Standalone compile+launch for the radix-filter single-pass multi-CTA + (cluster DSMEM) decode kernel. + """ + torch_dtype = input_values.dtype + dtype = _TORCH_TO_CUTLASS_DTYPE[torch_dtype] + num_rows, num_cols = input_values.shape + bucketed_num_cols = _bucket_num_cols(num_cols) + chunk_size_per_cta = math.ceil(bucketed_num_cols / cluster_size) + + # single_pass marker + cluster_size / chunk_size in the key to avoid + # colliding with the single-CTA / 2-pass compiled kernels. + key = ( + "sp_multi_cta", + dtype, + bucketed_num_cols, + top_k, + next_n, + return_val, + num_copy_bits, + cluster_size, + chunk_size_per_cta, + overflow_policy, + cache_smem_values, + ) + if key not in compiled_filter_topk_dict: + n_rows = cute.sym_int() + n_cols = cute.sym_int() + n_batch = cute.sym_int() + input_fake = cute.runtime.make_fake_compact_tensor( + dtype, (n_rows, n_cols), stride_order=(1, 0), assumed_align=32 + ) + if overflow_policy == "GMEM_SPILL": + # Per-CTA spill buffer: (num_rows * cluster_size, num_buffers, chunk). + buffer_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (cute.sym_int(), cute.sym_int(), cute.sym_int()), + stride_order=(2, 1, 0), + assumed_align=32, + ) + else: + buffer_fake = None + seqlen_fake = cute.runtime.make_fake_compact_tensor( + cute.Int32, (n_batch,), stride_order=(0,) + ) + output_indices_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_rows, top_k), stride_order=(1, 0) + ) + if return_val: + output_values_fake = cute.runtime.make_fake_compact_tensor( + dtype, (n_rows, top_k), stride_order=(1, 0) + ) + else: + output_values_fake = None + fake_stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + + filtered_topk_func = FilteredTopKKernelVarlenDecode( + dtype, + bucketed_num_cols, + top_k, + next_n, + num_copy_bits=num_copy_bits, + return_val=return_val, + chunk_size_per_cta=chunk_size_per_cta, + num_ctas_per_row=cluster_size, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, + single_pass_multi_cta=True, + ) + + compiled_kernel = cute.compile( + filtered_topk_func, + input_fake, + None, # indices_fake: unused in this path + buffer_fake, + seqlen_fake, + output_indices_fake, + output_values_fake, + stream=fake_stream, + min_blocks_per_mp=1, + options="--enable-tvm-ffi", + ) + compiled_filter_topk_dict[key] = compiled_kernel + else: + compiled_kernel = compiled_filter_topk_dict[key] + + output_indices_torch = torch.empty(num_rows, top_k, dtype=torch.int32, device="cuda") + if return_val: + output_values_torch = torch.empty(num_rows, top_k, dtype=torch_dtype, device="cuda") + else: + output_values_torch = None + + if overflow_policy == "GMEM_SPILL": + buffer_numbers = 2 if dtype == cutlass.Float32 else 1 + buffer_torch = torch.empty( + num_rows * cluster_size, + buffer_numbers, + chunk_size_per_cta, + dtype=torch.int32, + device="cuda", + ) + else: + buffer_torch = None + + # TVM FFI uses env stream automatically; the kernel __call__ issues the + # cluster launch (cluster_dim = cluster_size) internally. + compiled_kernel( + input_values, + None, # indices + buffer_torch, + seq_lens, + output_indices_torch, + output_values_torch, + ) + return output_indices_torch, output_values_torch + + +def cute_dsl_radix_filter_topk_multi_cta_wrapper( input_values, seq_lens, top_k, next_n, return_val=True, - load_balance=False, num_copy_bits=256, chunk_size_per_cta=16384, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, ): torch_dtype = input_values.dtype dtype = _TORCH_TO_CUTLASS_DTYPE[torch_dtype] num_rows, num_cols = input_values.shape bucketed_num_cols = _bucket_num_cols(num_cols) - large_occupancy = num_rows > 148 - assert not load_balance + large_occupancy = num_rows > _get_num_sms() # Note: don't forget num_cols, which means the maximum columns. enable_multi_cta = True @@ -794,11 +801,12 @@ def cute_dsl_topk_multi_cta_wrapper( next_n, return_val, num_copy_bits, - load_balance, large_occupancy, enable_multi_cta, chunk_size_per_cta, num_ctas_per_row, + overflow_policy, + cache_smem_values, ) if key not in compiled_filter_topk_dict: # Create fake tensors for compilation @@ -808,13 +816,16 @@ def cute_dsl_topk_multi_cta_wrapper( input_fake = cute.runtime.make_fake_compact_tensor( dtype, (n_rows, n_cols), stride_order=(1, 0), assumed_align=32 ) - # used for large num_cols - buffer_fake = cute.runtime.make_fake_compact_tensor( - cutlass.Int32, - (cute.sym_int(), cute.sym_int(), cute.sym_int()), - stride_order=(2, 1, 0), - assumed_align=32, - ) + # Shared buffer for both kernels: only needed when policy spills to GMEM + if overflow_policy == "GMEM_SPILL": + buffer_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (cute.sym_int(), cute.sym_int(), cute.sym_int()), + stride_order=(2, 1, 0), + assumed_align=32, + ) + else: + buffer_fake = None seqlen_fake = cute.runtime.make_fake_compact_tensor( cute.Int32, (n_batch,), @@ -849,21 +860,21 @@ def cute_dsl_topk_multi_cta_wrapper( chunk_size_per_cta=chunk_size_per_cta, num_ctas_per_row=num_ctas_per_row, merge_blocks=False, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, ) # Compile the kernel compiled_kernel_first = cute.compile( filtered_topk_func_first, input_fake, - None, # indices_fake, + None, # indices_fake: unused in this path; pass None to match runtime buffer_fake, - None, # g_global_counter_fake, seqlen_fake, # output_indices_fake, # output_values_fake, first_kernel_output_indices_fake, first_kernel_output_values_fake, stream=fake_stream, - enable_persistent_dynamic_scheduling=load_balance, min_blocks_per_mp=1, options="--enable-tvm-ffi", ) @@ -896,6 +907,8 @@ def cute_dsl_topk_multi_cta_wrapper( # chunk_size_per_cta=chunk_size_per_cta, # no use # num_ctas_per_row=1, # no use merge_blocks=True, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, ) # Compile the kernel compiled_kernel_second = cute.compile( @@ -903,12 +916,10 @@ def cute_dsl_topk_multi_cta_wrapper( input_fake, indices_fake, buffer_fake, - None, # g_global_counter_fake, seqlen_fake, output_indices_fake, output_values_fake, stream=fake_stream, - enable_persistent_dynamic_scheduling=load_balance, min_blocks_per_mp=1, options="--enable-tvm-ffi", ) @@ -929,25 +940,22 @@ def cute_dsl_topk_multi_cta_wrapper( else: output_values_torch = None - if dtype == cutlass.Float32: - buffer_numbers = 2 + if overflow_policy == "GMEM_SPILL": + buffer_numbers = 2 if dtype == cutlass.Float32 else 1 + buffer_torch = torch.empty( + num_rows * num_ctas_per_row, + buffer_numbers, + max(chunk_size_per_cta, num_ctas_per_row * top_k), + dtype=torch.int32, + device="cuda", + ) else: - buffer_numbers = 1 - buffer_torch = torch.empty( - num_rows * num_ctas_per_row, - buffer_numbers, - max(chunk_size_per_cta, num_ctas_per_row * top_k), - dtype=torch.int32, - device="cuda", - ) - g_global_counter_torch = None - + buffer_torch = None # TVM FFI uses env stream automatically compiled_kernel_first( input_values, None, # indices, used for merge blocks kernel of the multi-cta. buffer_torch, - g_global_counter_torch, seq_lens, first_kernel_output_indices_torch, first_kernel_output_values_torch, @@ -957,7 +965,6 @@ def cute_dsl_topk_multi_cta_wrapper( first_kernel_output_values_torch, first_kernel_output_indices_torch, buffer_torch, - g_global_counter_torch, seq_lens, output_indices_torch, output_values_torch, @@ -988,7 +995,6 @@ def run_filtered_topk_decode( max_num_cols, top_k, next_n, - load_balance: bool = False, num_copy_bits=256, return_val=True, large_occupancy=False, @@ -998,6 +1004,8 @@ def run_filtered_topk_decode( iterations=100, use_cold_l2=True, print_verbose=True, + overflow_policy: str = "GMEM_SPILL", + cache_smem_values: bool = False, ): """ Prepare input tensors, launch GPU kernel, and reference checking. @@ -1017,7 +1025,6 @@ def run_filtered_topk_decode( print(f" next_n: {next_n}") print(f" max_num_cols: {max_num_cols}") print(f" top_k: {top_k}") - print(f" load_balance: {load_balance}") print(f" num_copy_bits: {num_copy_bits}") print(f" return_val: {return_val}") print(f" large_occupancy: {large_occupancy}") @@ -1044,20 +1051,16 @@ def run_filtered_topk_decode( input_fake = cute.runtime.make_fake_compact_tensor( dtype, (n_rows, n_cols), stride_order=(1, 0), assumed_align=32 ) - # TODO - if dtype == cutlass.Float32: - buffer_numbers = 2 + if overflow_policy == "GMEM_SPILL": + buffer_numbers = 2 if dtype == cutlass.Float32 else 1 + buffer_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_rows, cute.sym_int(), n_cols), + stride_order=(2, 1, 0), + assumed_align=32, + ) else: - buffer_numbers = 1 - buffer_fake = cute.runtime.make_fake_compact_tensor( - cutlass.Int32, - (n_rows, cute.sym_int(), n_cols), - stride_order=(2, 1, 0), - assumed_align=32, - ) - g_global_counter_fake = cute.runtime.make_fake_compact_tensor( - cutlass.Int32, (1,), stride_order=(0,) - ) + buffer_fake = None seqlen_fake = cute.runtime.make_fake_compact_tensor( cute.Int32, (n_batch,), @@ -1086,6 +1089,8 @@ def run_filtered_topk_decode( num_copy_bits=num_copy_bits, return_val=return_val, large_occupancy=large_occupancy, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, ) # Compile the kernel @@ -1094,12 +1099,10 @@ def run_filtered_topk_decode( input_fake, None, # indices, used for merge blocks kernel of the multi-cta. buffer_fake, - g_global_counter_fake, seqlen_fake, output_indices_fake, output_values_fake, stream=fake_stream, - enable_persistent_dynamic_scheduling=load_balance, # TODO: confirm this parameter. min_blocks_per_mp=4 if large_occupancy else 1, options="--enable-tvm-ffi", @@ -1107,7 +1110,6 @@ def run_filtered_topk_decode( # Set input data # num_gen_tokens is the number of rows in the input tensor - g_global_counter_torch = torch.zeros(1, dtype=torch.int32, device="cuda") torch.cuda.synchronize() num_gen_tokens = batch_size * next_n # Use the same variable name as dsa.py row_starts = torch.zeros(num_gen_tokens, dtype=torch.int32, device="cuda") @@ -1131,20 +1133,22 @@ def run_filtered_topk_decode( ) else: output_values_torch = None - buffer_torch = torch.zeros( - num_gen_tokens, - buffer_numbers, - input_torch.shape[1], - dtype=torch.int32, - device="cuda", - ) + if overflow_policy == "GMEM_SPILL": + buffer_torch = torch.zeros( + num_gen_tokens, + buffer_numbers, + input_torch.shape[1], + dtype=torch.int32, + device="cuda", + ) + else: + buffer_torch = None # TVM FFI uses env stream automatically compiled_kernel( input_torch, None, # indices, used for merge blocks kernel of the multi-cta. buffer_torch, - g_global_counter_torch, seq_lens, output_indices_torch, output_values_torch, @@ -1166,73 +1170,69 @@ def run_filtered_topk_decode( if print_verbose: print("PASSED") - if not load_balance: - wrapper_output_indices, wrapper_output_values = cute_dsl_topk_wrapper( + wrapper_output_indices, wrapper_output_values = cute_dsl_radix_filter_topk_wrapper( + input_torch, + seq_lens, + top_k, + next_n, + return_val, + num_copy_bits, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, + ) + wrapper_output_val_sorted = torch.sort( + wrapper_output_values.cpu(), dim=1, descending=True + ).values + output_val_ref_sorted = torch.sort(output_values_torch.cpu(), dim=1, descending=True).values + + assert torch.allclose(wrapper_output_val_sorted, output_val_ref_sorted, atol=1e-5), ( + "CUDA top_k_per_row results don't match wrapper" + ) + if print_verbose: + print("Wrapper: PASSED") + + # test multi-cta version. + wrapper_output_indices_multi_cta, wrapper_output_values_multi_cta = ( + cute_dsl_radix_filter_topk_multi_cta_wrapper( input_torch, seq_lens, top_k, next_n, return_val, - load_balance, num_copy_bits, + chunk_size_per_cta=8192, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, ) - wrapper_output_val_sorted = torch.sort( - wrapper_output_values.cpu(), dim=1, descending=True - ).values - output_val_ref_sorted = torch.sort( - output_values_torch.cpu(), dim=1, descending=True - ).values - - assert torch.allclose(wrapper_output_val_sorted, output_val_ref_sorted, atol=1e-5), ( - "CUDA top_k_per_row results don't match wrapper" - ) - if print_verbose: - print("Wrapper: PASSED") - - # test multi-cta version. - wrapper_output_indices_multi_cta, wrapper_output_values_multi_cta = ( - cute_dsl_topk_multi_cta_wrapper( - input_torch, - seq_lens, - top_k, - next_n, - return_val, - load_balance, - num_copy_bits, - chunk_size_per_cta=8192, - ) - ) - wrapper_output_val_sorted_multi_cta = torch.sort( - wrapper_output_values_multi_cta.cpu(), dim=1, descending=True - ).values - output_val_ref_sorted = torch.sort( - output_values_torch.cpu(), dim=1, descending=True - ).values - - for i in range(num_gen_tokens): - if not torch.allclose( - wrapper_output_val_sorted_multi_cta[i, :], - output_val_ref_sorted[i, :], - atol=1e-5, - ): - print(f"FAILED for row_id: {i}") - print( - f"wrapper_output_val_sorted_multi_cta: {wrapper_output_val_sorted_multi_cta[i]}" - ) - print(f"output_values_torch: {output_val_ref_sorted[i]}") - break - assert torch.allclose( - wrapper_output_val_sorted_multi_cta, - output_val_ref_sorted.cpu(), + ) + wrapper_output_val_sorted_multi_cta = torch.sort( + wrapper_output_values_multi_cta.cpu(), dim=1, descending=True + ).values + output_val_ref_sorted = torch.sort(output_values_torch.cpu(), dim=1, descending=True).values + + for i in range(num_gen_tokens): + if not torch.allclose( + wrapper_output_val_sorted_multi_cta[i, :], + output_val_ref_sorted[i, :], atol=1e-5, - ), "CUDA top_k_per_row results don't match wrapper multi-cta" - if print_verbose: - print("Wrapper multi-cta: PASSED") + ): + print(f"FAILED for row_id: {i}") + print( + f"wrapper_output_val_sorted_multi_cta: {wrapper_output_val_sorted_multi_cta[i]}" + ) + print(f"output_values_torch: {output_val_ref_sorted[i]}") + break + assert torch.allclose( + wrapper_output_val_sorted_multi_cta, + output_val_ref_sorted.cpu(), + atol=1e-5, + ), "CUDA top_k_per_row results don't match wrapper multi-cta" + if print_verbose: + print("Wrapper multi-cta: PASSED") if do_benchmark: def generate_inputs(): - g_global_counter_torch = torch.zeros(1, dtype=torch.int32, device="cuda") torch.cuda.synchronize() input_tensor = create_random_logits( row_starts, @@ -1254,45 +1254,25 @@ def generate_inputs(): input_tensor, None, # indices, used for merge blocks kernel of the multi-cta. buffer_torch, - g_global_counter_torch, seq_lens, output_indices_tensor, output_values_tensor, ) - workspace_count = 1 - if use_cold_l2: - one_workspace_bytes = ( - input_torch.numel() * input_torch.element_size() - + row_starts.numel() * row_starts.element_size() - + row_ends.numel() * row_ends.element_size() - + seq_lens.numel() * seq_lens.element_size() - + output_indices_torch.numel() * output_indices_torch.element_size() - + ( - output_values_torch.numel() * output_values_torch.element_size() - if return_val - else 0 - ) - ) - workspace_count = cute.testing.get_workspace_count( - one_workspace_bytes, warmup_iterations, iterations - ) - # Note: when load-balance is enabled, we need to memset g_global_counter_torch to 0 for each iteration. - # without this, the kernel will accumulate the global counter from previous iterations. - # Here, we war the memset by setting the workspace_count to the sum of warmup_iterations and iterations. - workspace_count = iterations + warmup_iterations - print("workspace_count: ", workspace_count) + workspace_count = iterations + warmup_iterations if use_cold_l2 else 1 + print("workspace_count: ", workspace_count) torch_stream = torch.cuda.Stream() benchmark_stream = cuda.CUstream(torch_stream.cuda_stream) - time = cute.testing.benchmark( - compiled_kernel, - workspace_generator=generate_inputs, - workspace_count=workspace_count, - warmup_iterations=warmup_iterations, - iterations=iterations, - use_cuda_graphs=True, - stream=benchmark_stream, - ) + with torch.cuda.stream(torch_stream): + time = cute.testing.benchmark( + compiled_kernel, + workspace_generator=generate_inputs, + workspace_count=workspace_count, + warmup_iterations=warmup_iterations, + iterations=iterations, + use_cuda_graphs=True, + stream=benchmark_stream, + ) if print_verbose: print(f"Time: {time} us") print(f"{dtype}-{batch_size}-{max_num_cols}-{top_k} {time}") @@ -1305,7 +1285,6 @@ def run_topk_decode( max_num_cols: int, top_k: int, next_n: int, - load_balance: bool = False, num_copy_bits: int = 256, return_val: bool = True, large_occupancy: bool = False, @@ -1314,6 +1293,8 @@ def run_topk_decode( warmup_iterations: int = 10, iterations: int = 10, use_cold_l2: bool = True, + overflow_policy: str = "GMEM_SPILL", + cache_smem_values: bool = False, ): run_filtered_topk_decode( dtype, @@ -1321,7 +1302,6 @@ def run_topk_decode( max_num_cols, top_k, next_n, - load_balance, num_copy_bits, return_val, large_occupancy, @@ -1330,6 +1310,8 @@ def run_topk_decode( warmup_iterations, iterations, use_cold_l2, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, ) @@ -1355,12 +1337,6 @@ def run_topk_decode( parser.add_argument("--max_num_cols", type=int, default=4096, help="max_num_cols") parser.add_argument("--next_n", type=int, default=3, help="next_n") parser.add_argument("--top_k", type=int, default=2048, help="top_k") - parser.add_argument( - "--load_balance", - action="store_true", - default=False, - help="Use load balance for varlen optimization", - ) parser.add_argument( "--num_copy_bits", type=int, @@ -1391,18 +1367,27 @@ def run_topk_decode( parser.add_argument("--warmup_iterations", type=int, default=10, help="Warmup iterations") parser.add_argument("--iterations", type=int, default=100, help="Iterations") parser.add_argument("--use_cold_l2", action="store_true", default=True, help="Use cold L2") + parser.add_argument( + "--overflow_policy", + type=str, + default="GMEM_SPILL", + choices=["GMEM_SPILL", "TRUNCATE", "REREAD", "REREAD_ALWAYS"], + help="Overflow policy when candidates exceed SMEM capacity", + ) + parser.add_argument( + "--cache_smem_values", + action="store_true", + default=False, + help="Cache ordered values alongside indices in SMEM to avoid re-reading from GMEM in refinement rounds", + ) args = parser.parse_args() - if args.top_k % 2 != 0: - parser.error("top_k must be a multiple of 2 (got top_k={})".format(args.top_k)) - run_topk_decode( dtype=args.dtype, batch_size=args.batch_size, max_num_cols=args.max_num_cols, top_k=args.top_k, next_n=args.next_n, - load_balance=args.load_balance, num_copy_bits=args.num_copy_bits, return_val=args.return_val, large_occupancy=args.large_occupancy, @@ -1411,4 +1396,6 @@ def run_topk_decode( warmup_iterations=args.warmup_iterations, iterations=args.iterations, use_cold_l2=args.use_cold_l2, + overflow_policy=args.overflow_policy, + cache_smem_values=args.cache_smem_values, ) diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_prefill_varlen.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_prefill_varlen.py new file mode 100644 index 000000000000..be5e90db351f --- /dev/null +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_prefill_varlen.py @@ -0,0 +1,647 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Type + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import cutlass.utils as utils +import torch +from cutlass.torch import dtype as torch_dtype + +from ..utils import TRTLLM_ENABLE_PDL, griddepcontrol_launch_dependents, griddepcontrol_wait +from .filtered_top_k_varlen_util import ( + FilteredTopKKernelVarlen, + compare_top_k_results, + create_random_logits, + run_reference_top_k, +) + +""" +Prefill top-k kernel using the radix-based filter algorithm. + +Differences from the decode variant: +- Row extents come from row_starts / row_ends tensors rather than seq_lens, + so row_start may be non-zero. +- Always single-CTA per row (no multi-CTA / merge path). +- Always large_occupancy=True (num_rows >> num_sms in prefill). +- Outputs LOCAL indices relative to row_start, matching the CUDA kernel + contract in IndexerTopKOp.cpp / indexerTopK.cu. +""" + + +class FilteredTopKKernelVarlenPrefill(FilteredTopKKernelVarlen): + """Single-CTA large-occupancy top-k kernel for the prefill phase. + + Key differences vs FilteredTopKKernelVarlenDecode: + - Takes row_starts / row_ends per-row tensors; row_start may be non-zero. + - Always single-CTA (no multi-CTA / merge blocks). + - Always 512 threads (large_occupancy path) with reduced SMEM for high + occupancy. + - Output indices are LOCAL (0-indexed within [row_start, row_end)), matching + the CUDA indexer_topk_prefill convention. + """ + + def __init__( + self, + dtype: cutlass.Numeric, + max_num_cols: int, + top_k: int, + num_copy_bits: int = 256, + return_val: bool = False, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, + ): + super().__init__( + dtype, + max_num_cols, + top_k, + num_copy_bits, + return_val, + enable_multi_cta=False, + chunk_size_per_cta=16384, + num_ctas_per_row=1, + merge_blocks=False, + overflow_policy=overflow_policy, + num_threads_override=512, # always 512 threads for large-occupancy path + cache_smem_values=cache_smem_values, + ) + + # Output local indices: subtract row_start before writing to output. + self.subtract_row_start_on_output = True + + def _compute_smem_input_size(self) -> int: + return self._compute_smem_input_size_for_occupancy(target_blocks_per_sm=4) + + @cute.kernel + def filtered_topk_kernel( + self, + input: cute.Tensor, + row_starts: cute.Tensor, + row_ends: cute.Tensor, + extra_buffer: cute.Tensor, + output_indices: cute.Tensor, + output_values: cute.Tensor, + ): + """CuTe DSL top-k kernel for the prefill phase.""" + griddepcontrol_wait() + + smem = utils.SmemAllocator() + s_histogram = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((self.radix + 1), order=(0)), + byte_alignment=128, + ) + s_counter = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((1), order=(0)), + byte_alignment=128, + ) + s_threshold_bin_id = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((1), order=(0)), + byte_alignment=128, + ) + s_num_input = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((2,), order=(0)), + byte_alignment=128, + ) + if cutlass.const_expr(self.enable_gmem_store): + g_num_input = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((2), order=(0)), + byte_alignment=128, + ) + else: + g_num_input = None + s_indices = smem.allocate_tensor( + element_type=self.index_type, + layout=cute.make_ordered_layout((self.filtered_topk_max_k,), order=(0)), + byte_alignment=128, + ) + if cutlass.const_expr(not self.enable_reread_always): + s_input_idx = smem.allocate_tensor( + element_type=self.index_type, + layout=cute.make_ordered_layout( + (self.num_buffer_smem_input_idx, self.filtered_topk_smem_input_size), + order=(1, 0), + ), + byte_alignment=128, + ) + else: + s_input_idx = None + if cutlass.const_expr(self.cache_smem_values and not self.enable_reread_always): + s_input_val = smem.allocate_tensor( + element_type=self.ordered_type, + layout=cute.make_ordered_layout( + ( + self.num_buffer_smem_input_idx, + self.filtered_topk_smem_input_size, + ), + order=(1, 0), + ), + byte_alignment=128, + ) + else: + s_input_val = None + if cutlass.const_expr(self.enable_reread): + s_overflow_flag = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((1,), order=(0,)), + byte_alignment=128, + ) + else: + s_overflow_flag = None + s_last_remain = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((1,), order=(0,)), + byte_alignment=128, + ) + num_warps = cutlass.const_expr( + min(self.radix, self.num_threads_per_cta) // cutlass.Int32(32) + ) + s_warp_sums = smem.allocate_tensor( + element_type=cute.Int32, + layout=cute.make_ordered_layout((num_warps,), order=(0,)), + byte_alignment=128, + ) + + bidx, _, _ = cute.arch.block_idx() + row_start = cutlass.Int32(row_starts[bidx]) + row_end = cutlass.Int32(row_ends[bidx]) + length = row_end - row_start + + self.filtered_topk_kernel_per_row( + input, + output_indices, # dummy: input_indices unused when merge_blocks=False + extra_buffer, + output_indices, + output_values, + row_start, + length, + bidx, + s_histogram, + s_counter, + s_threshold_bin_id, + s_num_input, + g_num_input, + s_indices, + s_input_idx, + s_input_val, + s_last_remain, + num_warps, + s_warp_sums, + s_overflow_flag, + ) + + griddepcontrol_launch_dependents() + + @cute.jit + def __call__( + self, + input_values, + row_starts, + row_ends, + extra_buffer, + output_indices, + output_values, + stream: cuda.CUstream, + min_blocks_per_mp: cutlass.Constexpr[int] = 4, + ): + """Host function: launch one CTA per row.""" + num_rows = input_values.shape[0] + self.filtered_topk_kernel( + input_values, + row_starts, + row_ends, + extra_buffer, + output_indices, + output_values, + ).launch( + grid=(num_rows, 1, 1), + block=(self.num_threads_per_cta, 1, 1), + stream=stream, + use_pdl=TRTLLM_ENABLE_PDL, + min_blocks_per_mp=min_blocks_per_mp, + ) + + +def _next_positive_power_of_2(x: int) -> int: + if x <= 0: + return 1 + return 1 << (x - 1).bit_length() + + +_TORCH_TO_CUTLASS_DTYPE = { + torch.float16: cutlass.Float16, + torch.bfloat16: cutlass.BFloat16, + torch.float32: cutlass.Float32, +} + + +def _bucket_num_cols(num_cols: int) -> int: + return _next_positive_power_of_2(num_cols) + + +compiled_filter_topk_prefill_dict = {} + + +def cute_dsl_topk_prefill_wrapper( + input_values, + row_starts, + row_ends, + top_k, + return_val=False, + num_copy_bits=256, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, +): + torch_dtype_ = input_values.dtype + dtype = _TORCH_TO_CUTLASS_DTYPE[torch_dtype_] + num_rows, num_cols = input_values.shape + bucketed_num_cols = _bucket_num_cols(num_cols) + + key = ( + dtype, + bucketed_num_cols, + top_k, + return_val, + num_copy_bits, + overflow_policy, + cache_smem_values, + ) + if key not in compiled_filter_topk_prefill_dict: + n_rows = cute.sym_int() + n_cols = cute.sym_int() + input_fake = cute.runtime.make_fake_compact_tensor( + dtype, (n_rows, n_cols), stride_order=(1, 0), assumed_align=32 + ) + row_starts_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_rows,), stride_order=(0,) + ) + row_ends_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_rows,), stride_order=(0,) + ) + if overflow_policy == "GMEM_SPILL": + buffer_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_rows, cute.sym_int(), n_cols), + stride_order=(2, 1, 0), + assumed_align=32, + ) + else: + buffer_fake = None + output_indices_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_rows, top_k), stride_order=(1, 0) + ) + if return_val: + output_values_fake = cute.runtime.make_fake_compact_tensor( + dtype, (n_rows, top_k), stride_order=(1, 0) + ) + else: + output_values_fake = None + fake_stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + + filtered_topk_func = FilteredTopKKernelVarlenPrefill( + dtype, + bucketed_num_cols, + top_k, + num_copy_bits=num_copy_bits, + return_val=return_val, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, + ) + compiled_kernel = cute.compile( + filtered_topk_func, + input_fake, + row_starts_fake, + row_ends_fake, + buffer_fake, + output_indices_fake, + output_values_fake, + stream=fake_stream, + min_blocks_per_mp=4, + options="--enable-tvm-ffi", + ) + compiled_filter_topk_prefill_dict[key] = compiled_kernel + else: + compiled_kernel = compiled_filter_topk_prefill_dict[key] + + output_indices_torch = torch.empty(num_rows, top_k, dtype=torch.int32, device="cuda") + if return_val: + output_values_torch = torch.empty(num_rows, top_k, dtype=torch_dtype_, device="cuda") + else: + output_values_torch = None + + if overflow_policy == "GMEM_SPILL": + buffer_numbers = 2 if dtype == cutlass.Float32 else 1 + buffer_torch = torch.empty( + num_rows, buffer_numbers, num_cols, dtype=torch.int32, device="cuda" + ) + else: + buffer_torch = None + + compiled_kernel( + input_values, + row_starts, + row_ends, + buffer_torch, + output_indices_torch, + output_values_torch, + ) + return output_indices_torch, output_values_torch + + +def run_filtered_topk_prefill( + dtype: Type[cutlass.Numeric], + num_rows: int, + max_num_cols: int, + top_k: int, + num_copy_bits: int = 256, + return_val: bool = False, + do_ref_check: bool = True, + do_benchmark: bool = False, + warmup_iterations: int = 10, + iterations: int = 100, + use_cold_l2: bool = True, + print_verbose: bool = True, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, +): + """Prepare input tensors, launch prefill top-k kernel, and check reference.""" + if print_verbose: + print("=" * 60) + print("Launching Blackwell Filtered TopK Prefill Test") + print("-" * 60) + print(f"dtype: {dtype}") + print(f"num_rows: {num_rows}") + print(f"max_num_cols: {max_num_cols}") + print(f"top_k: {top_k}") + print(f"num_copy_bits: {num_copy_bits}") + print(f"return_val: {return_val}") + print(f"overflow_policy: {overflow_policy}") + print(f"Do reference checking: {do_ref_check}") + print(f"Do benchmark: {do_benchmark}") + print("=" * 60) + + if not torch.cuda.is_available(): + raise RuntimeError("GPU is required to run this example!") + + seed = 1111 + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + + # Each row spans [0, row_end); row lengths drawn uniformly from [top_k, max_num_cols]. + row_starts = torch.zeros(num_rows, dtype=torch.int32, device="cuda") + row_ends = torch.randint(top_k, max_num_cols + 1, (num_rows,), dtype=torch.int32, device="cuda") + + input_torch = create_random_logits(row_starts, row_ends, torch_dtype(dtype), seed) + + n_rows, n_cols = input_torch.shape + + n_rows_fake = cute.sym_int() + n_cols_fake = cute.sym_int() + input_fake = cute.runtime.make_fake_compact_tensor( + dtype, (n_rows_fake, n_cols_fake), stride_order=(1, 0), assumed_align=32 + ) + row_starts_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_rows_fake,), stride_order=(0,) + ) + row_ends_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_rows_fake,), stride_order=(0,) + ) + if overflow_policy == "GMEM_SPILL": + buffer_numbers = 2 if dtype == cutlass.Float32 else 1 + buffer_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (n_rows_fake, cute.sym_int(), n_cols_fake), + stride_order=(2, 1, 0), + assumed_align=32, + ) + else: + buffer_fake = None + output_indices_fake = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, (n_rows_fake, top_k), stride_order=(1, 0) + ) + if return_val: + output_values_fake = cute.runtime.make_fake_compact_tensor( + dtype, (n_rows_fake, top_k), stride_order=(1, 0) + ) + else: + output_values_fake = None + fake_stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + + filtered_topk_func = FilteredTopKKernelVarlenPrefill( + dtype, + max_num_cols, + top_k, + num_copy_bits=num_copy_bits, + return_val=return_val, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, + ) + compiled_kernel = cute.compile( + filtered_topk_func, + input_fake, + row_starts_fake, + row_ends_fake, + buffer_fake, + output_indices_fake, + output_values_fake, + stream=fake_stream, + min_blocks_per_mp=4, + options="--enable-tvm-ffi", + ) + + output_indices_torch = torch.empty(n_rows, top_k, dtype=torch.int32, device="cuda") + if return_val: + output_values_torch = torch.empty(n_rows, top_k, dtype=torch_dtype(dtype), device="cuda") + else: + output_values_torch = None + if overflow_policy == "GMEM_SPILL": + buffer_torch = torch.empty(n_rows, buffer_numbers, n_cols, dtype=torch.int32, device="cuda") + else: + buffer_torch = None + + compiled_kernel( + input_torch, + row_starts, + row_ends, + buffer_torch, + output_indices_torch, + output_values_torch, + ) + + if do_ref_check and top_k <= max_num_cols: + torch.cuda.synchronize() + torch_indices = run_reference_top_k(input_torch, row_starts, row_ends, top_k) + assert compare_top_k_results( + input_torch, output_indices_torch, torch_indices, row_starts, row_ends, top_k + ), "prefill top-k results don't match torch.topk" + if print_verbose: + print("PASSED") + + wrapper_indices, wrapper_values = cute_dsl_topk_prefill_wrapper( + input_torch, + row_starts, + row_ends, + top_k, + return_val, + num_copy_bits, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, + ) + assert compare_top_k_results( + input_torch, wrapper_indices, torch_indices, row_starts, row_ends, top_k + ), "prefill wrapper results don't match torch.topk" + if print_verbose: + print("Wrapper: PASSED") + + if do_benchmark: + + def generate_inputs(): + torch.cuda.synchronize() + input_tensor = create_random_logits(row_starts, row_ends, torch_dtype(dtype), seed) + output_indices_tensor = torch.empty(n_rows, top_k, dtype=torch.int32, device="cuda") + if return_val: + output_values_tensor = torch.empty( + n_rows, top_k, dtype=torch_dtype(dtype), device="cuda" + ) + else: + output_values_tensor = None + return cute.testing.JitArguments( + input_tensor, + row_starts, + row_ends, + buffer_torch, + output_indices_tensor, + output_values_tensor, + ) + + workspace_count = iterations + warmup_iterations if use_cold_l2 else 1 + print("workspace_count: ", workspace_count) + torch_stream = torch.cuda.Stream() + benchmark_stream = cuda.CUstream(torch_stream.cuda_stream) + with torch.cuda.stream(torch_stream): + time = cute.testing.benchmark( + compiled_kernel, + workspace_generator=generate_inputs, + workspace_count=workspace_count, + warmup_iterations=warmup_iterations, + iterations=iterations, + use_cuda_graphs=True, + stream=benchmark_stream, + ) + if print_verbose: + print(f"Time: {time} us") + print(f"{dtype}-{num_rows}-{max_num_cols}-{top_k} {time}") + torch.cuda.synchronize() + + +def run_topk_prefill( + dtype: Type[cutlass.Numeric], + num_rows: int, + max_num_cols: int, + top_k: int, + num_copy_bits: int = 256, + return_val: bool = False, + do_ref_check: bool = True, + do_benchmark: bool = False, + warmup_iterations: int = 10, + iterations: int = 10, + use_cold_l2: bool = True, + overflow_policy: str = "REREAD", + cache_smem_values: bool = False, +): + run_filtered_topk_prefill( + dtype, + num_rows, + max_num_cols, + top_k, + num_copy_bits, + return_val, + do_ref_check, + do_benchmark, + warmup_iterations, + iterations, + use_cold_l2, + overflow_policy=overflow_policy, + cache_smem_values=cache_smem_values, + ) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Blackwell CuTE DSL filtered top-k prefill benchmark." + ) + parser.add_argument( + "--dtype", + type=cutlass.dtype, + default=cutlass.Float32, + choices=[cutlass.Float32, cutlass.Float16, cutlass.BFloat16], + help="Data type of the input matrix", + ) + parser.add_argument("--num_rows", type=int, default=256, help="Number of rows (sequences)") + parser.add_argument("--max_num_cols", type=int, default=8192, help="Maximum sequence length") + parser.add_argument("--top_k", type=int, default=1024, help="top_k") + parser.add_argument( + "--num_copy_bits", type=int, default=256, help="num_copy_bits, used for vectorization" + ) + parser.add_argument("--return_val", action="store_true", default=False, help="Return values") + parser.add_argument( + "--do_ref_check", action="store_true", default=False, help="Do reference checking" + ) + parser.add_argument( + "--do_benchmark", action="store_true", default=False, help="Do benchmark test" + ) + parser.add_argument("--warmup_iterations", type=int, default=10, help="Warmup iterations") + parser.add_argument("--iterations", type=int, default=100, help="Iterations") + parser.add_argument("--use_cold_l2", action="store_true", default=True, help="Use cold L2") + parser.add_argument( + "--overflow_policy", + type=str, + default="REREAD", + choices=["GMEM_SPILL", "TRUNCATE", "REREAD", "REREAD_ALWAYS"], + help="Overflow policy when candidates exceed SMEM capacity", + ) + parser.add_argument( + "--cache_smem_values", + action="store_true", + default=False, + help="Cache ordered values alongside indices in SMEM to avoid re-reading from GMEM in refinement rounds", + ) + + args = parser.parse_args() + if args.top_k % 2 != 0: + parser.error("top_k must be a multiple of 2 (got top_k={})".format(args.top_k)) + + run_topk_prefill( + dtype=args.dtype, + num_rows=args.num_rows, + max_num_cols=args.max_num_cols, + top_k=args.top_k, + num_copy_bits=args.num_copy_bits, + return_val=args.return_val, + do_ref_check=args.do_ref_check, + do_benchmark=args.do_benchmark, + warmup_iterations=args.warmup_iterations, + iterations=args.iterations, + use_cold_l2=args.use_cold_l2, + overflow_policy=args.overflow_policy, + cache_smem_values=args.cache_smem_values, + ) diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_varlen_util.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_varlen_util.py index be796763e8e2..159d44cb3445 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_varlen_util.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_varlen_util.py @@ -13,14 +13,82 @@ # limitations under the License. +import math + import cutlass import cutlass.cute as cute import torch from cutlass._mlir.dialects import llvm +from cutlass.cute.typing import Int32 as CuteInt32 +from cutlass.cute.typing import Pointer as CutePointer +from cutlass.cutlass_dsl import T, dsl_user_op from cutlass.utils.distributed import atomicAdd from .block_scan import block_prefix_sum_kernel, fence_acq_rel_cta + +# --------------------------------------------------------------------------- +# Cluster DSMEM primitives (inline PTX) — used only by the single-pass +# multi-CTA (radix-filter) path. Defined locally to avoid a circular import +# with single_pass_multi_cta_radix_topk_cluster (which imports this module). +# --------------------------------------------------------------------------- +@dsl_user_op +def _mapa_shared_cluster( + smem_ptr: CutePointer, peer_rank: CuteInt32, *, loc=None, ip=None +) -> CuteInt32: + """Map a local SMEM address to a peer CTA's SMEM in cluster address space. + + PTX: mapa.shared::cluster.u32 $0, $1, $2; + """ + smem_ptr_i32 = smem_ptr.toint(loc=loc, ip=ip).ir_value() + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [smem_ptr_i32, peer_rank.ir_value(loc=loc, ip=ip)], + "mapa.shared::cluster.u32 $0, $1, $2;", + "=r,r,r", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +@cute.jit +def mapa_shared_cluster(smem_ptr, peer_rank): + """Map a local SMEM address to a peer CTA's SMEM in cluster address space.""" + return _mapa_shared_cluster(smem_ptr, peer_rank) + + +@dsl_user_op +def _ld_shared_cluster_i32(mapped_addr: CuteInt32, *, loc=None, ip=None) -> CuteInt32: + """Load an int32 from a cluster SMEM address. + + PTX: ld.shared::cluster.u32 $0, [$1]; + """ + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [mapped_addr.ir_value(loc=loc, ip=ip)], + "ld.shared::cluster.u32 $0, [$1];", + "=r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + ) + + +@cute.jit +def ld_shared_cluster_i32(mapped_addr): + """Load an int32 from a cluster SMEM address.""" + return _ld_shared_cluster_i32(mapped_addr) + + """ top-k varlen utils. could be used by prefill and decode phase. """ @@ -48,7 +116,42 @@ def __init__( chunk_size_per_cta: int = 16384, num_ctas_per_row: int = 1, merge_blocks: bool = False, + overflow_policy: str = "REREAD", + num_threads_override: int = 0, + cache_smem_values: bool = False, + single_pass_multi_cta: bool = False, ): + """ + Args: + overflow_policy: Controls behavior when threshold-bin candidates exceed the + SMEM input buffer (filtered_topk_smem_input_size). Only takes effect + when max_num_cols > filtered_topk_smem_input_size; otherwise all policies + are equivalent and no extra cost is incurred. + + "GMEM_SPILL" -- Spill excess candidates to a pre-allocated GMEM + extra_buffer. Exact result. Requires caller to + allocate extra_buffer proportional to batch size; + may OOM at large batch. + "TRUNCATE" -- Discard candidates that overflow SMEM. Only + retained candidates contribute to the refinement + histogram, so refinement operates consistently on + the stored set. Non-exact (may output fewer than + top_k indices when the threshold bin is dense). No + extra_buffer needed. Requires + top_k <= filtered_topk_smem_input_size; larger + top_k is rejected at construction because too few + candidates could be retained to fill the output. + "REREAD_ALWAYS" -- Skip SMEM collection entirely in the coarse pass; + always perform a second GMEM scan to collect + threshold-bin candidates. Exact result. No + extra_buffer needed; costs one extra GMEM read per + row unconditionally. + "REREAD" -- Optimistic: attempt SMEM collection first. If + overflow is detected at runtime (s_overflow_flag), + fall back to a REREAD_ALWAYS-style second GMEM scan. + Exact result. No extra_buffer needed; pays the + extra GMEM read only when overflow actually occurs. + """ self.dtype = dtype self.max_num_cols = max_num_cols self.top_k = top_k @@ -57,9 +160,30 @@ def __init__( self.chunk_size_per_cta = chunk_size_per_cta self.num_ctas_per_row = num_ctas_per_row self.merge_blocks = merge_blocks + self.overflow_policy = overflow_policy + # Single-pass multi-CTA (radix-filter cluster) mode. Compile-time flag; + # when False all cluster branches are const-folded away (single-CTA path + # keeps identical SASS). Reuses chunk_size_per_cta / num_ctas_per_row for + # chunk partitioning (num_ctas_per_row == ctas_per_group / cluster size). + self.single_pass_multi_cta = single_pass_multi_cta + assert overflow_policy in ("GMEM_SPILL", "TRUNCATE", "REREAD_ALWAYS", "REREAD"), ( + f"Unknown overflow_policy: {overflow_policy}" + ) + + # top_k sizes the shared-memory index staging (s_indices), so bound it + # here. Direct callers and the run_topk_decode CLI bypass the decode + # wrappers, and an oversized top_k would otherwise surface as an opaque + # smem launch failure. 16384 matches the wrapper guards in + # cute_dsl_custom_ops.py. + if top_k <= 0 or top_k > 16384: + raise ValueError( + f"top_k must be in range [1, 16384], got {top_k}. " + "Maximum supported top_k is 16384 for Blackwell architecture." + ) - # Note: now we only support top_k <= 2048, we could change the code here to support larger top_k. - self.filtered_topk_max_k = 2048 + # s_indices only needs top_k slots; size to top_k to save SMEM. Still + # referenced by the prefill kernel (filtered_top_k_prefill_varlen.py). + self.filtered_topk_max_k = top_k # 8 bits for radix-based filter. self.radix = 256 @@ -69,38 +193,24 @@ def __init__( self.num_buffer_smem_input_idx = 1 # 65536 is the max index value for uint16. - if cutlass.const_expr(enable_multi_cta): + # SP multi-CTA reuses the same chunk partitioning as 2-pass multi-CTA. + if cutlass.const_expr(enable_multi_cta or single_pass_multi_cta): self.per_row_max_num_cols = chunk_size_per_cta * num_ctas_per_row else: self.per_row_max_num_cols = self.max_num_cols if cutlass.const_expr(self.per_row_max_num_cols <= 65536): self.index_type = cutlass.Uint16 - if cutlass.const_expr(self.num_buffer_smem_input_idx == 2): - self.max_smem_input_size = 32 * 1024 - else: - self.max_smem_input_size = 64 * 1024 else: self.index_type = cutlass.Uint32 - if cutlass.const_expr(self.num_buffer_smem_input_idx == 2): - self.max_smem_input_size = 16 * 1024 - else: - self.max_smem_input_size = 32 * 1024 - - self.filtered_topk_smem_input_size = min(self.max_smem_input_size, self.max_num_cols) - - if cutlass.const_expr(self.max_num_cols > self.filtered_topk_smem_input_size): - self.enable_gmem_store = True - else: - self.enable_gmem_store = False - - self.return_val = return_val self.vec_size = num_copy_bits // dtype.width if cutlass.const_expr(dtype not in [cutlass.Float32, cute.BFloat16, cutlass.Float16]): raise ValueError(f"Unsupported dtype: {dtype}") - if cutlass.const_expr(dtype == cutlass.Float32): + if num_threads_override > 0: + self.num_threads_per_cta = num_threads_override + elif cutlass.const_expr(dtype == cutlass.Float32): if self.max_num_cols >= self.vec_size * 1024: self.num_threads_per_cta = 1024 else: @@ -117,7 +227,8 @@ def __init__( else: self.num_threads_per_cta = 256 - # radix-based filter parameters. + # radix-based filter parameters — set before _compute_smem_input_size() so + # ordered_type.width is available in the SMEM budget formula. if cutlass.const_expr(dtype == cutlass.Float32): self.ordered_type = cute.Uint32 self.first_refine_shift = 24 @@ -127,6 +238,84 @@ def __init__( self.first_refine_shift = 0 self.num_refine_rounds = 1 + self.cache_smem_values = cache_smem_values + + # num_threads_per_cta must be set before _compute_smem_input_size() since + # _compute_smem_input_size_for_occupancy() uses it to derive num_warps. + self.filtered_topk_smem_input_size = self._compute_smem_input_size() + + _needs_extra = self.max_num_cols > self.filtered_topk_smem_input_size + # TRUNCATE retains at most filtered_topk_smem_input_size threshold-bin + # candidates. When top_k exceeds that, a dense threshold bin can leave + # fewer than top_k valid candidates, and the unfilled s_indices tail + # would be written back as undefined/duplicate indices (violating the + # no-duplicate contract). This configuration is under-determined, so + # reject it up front; use REREAD or GMEM_SPILL for larger top_k. + if overflow_policy == "TRUNCATE" and self.top_k > self.filtered_topk_smem_input_size: + raise ValueError( + f"TRUNCATE overflow_policy requires top_k ({self.top_k}) <= " + f"filtered_topk_smem_input_size ({self.filtered_topk_smem_input_size}); " + "use REREAD or GMEM_SPILL for larger top_k." + ) + self.enable_gmem_store = (overflow_policy == "GMEM_SPILL") and _needs_extra + self.enable_truncate = (overflow_policy == "TRUNCATE") and _needs_extra + self.enable_reread_always = overflow_policy == "REREAD_ALWAYS" + self.enable_reread = (overflow_policy == "REREAD") and _needs_extra + + self.return_val = return_val + # Subclasses set to True to subtract row_start from absolute indices before + # writing output (used in prefill where row_start may be non-zero). + self.subtract_row_start_on_output = False + + def _compute_smem_input_size(self) -> int: + return self._compute_smem_input_size_for_occupancy(target_blocks_per_sm=1) + + def _compute_smem_input_size_for_occupancy(self, target_blocks_per_sm: int) -> int: + """Compute max candidate-buffer size (S) for a given occupancy target. + + input_idx_budget = 128 KB // target_blocks_per_sm + S = input_idx_budget // (num_buffer * idx_sz) + + This keeps total SMEM ≈ 38 KB/block at 4 blocks/SM, preserving ~104 KB + of unified L1 per SM for LDG caching. Using the full per-block budget + (256 KB / 4 = 64 KB) shrinks L1 to ~32 KB and causes a ~5-10% regression. + + Resulting S values (matches old hardcoded values): + 4 blocks/SM: Uint16/nb=2→8192 Uint16/nb=1→16384 + Uint32/nb=2→4096 Uint32/nb=1→8192 + 1 block/SM: Uint16/nb=2→32768 Uint16/nb=1→65536 + Uint32/nb=2→16384 Uint32/nb=1→32768 + + NOTE (large top_k): this budget only accounts for the s_input_idx + candidate buffer. The separate s_indices staging is sized to top_k + (top_k * idx_sz bytes) and stacks on top of the 128 KB reserved here, + without being subtracted from it. That extra term is negligible at + top_k<=2048 (4-8 KB) but grows to 32 KB (Uint16) / 64 KB (Uint32) at + the top_k=16384 limit, eroding the ~104 KB L1 assumption above (L1 can + drop to ~90 KB / ~60 KB respectively) and, together with s_input_val + when cache_smem_values=True, risks exceeding the 228 KB SMEM cap. If + large top_k is exercised in practice, subtract top_k * idx_sz from + input_idx_budget here so S leaves room for s_indices. + """ + idx_sz = 2 if self.index_type == cutlass.Uint16 else 4 + if not self.cache_smem_values: + # cache_smem_values=False: reserve ~104 KB L1 for LDG caching. + INPUT_IDX_BUDGET_BASE = 128 * 1024 # 128 KB at 1 block/SM + input_idx_budget = INPUT_IDX_BUDGET_BASE // target_blocks_per_sm + max_S = input_idx_budget // (self.num_buffer_smem_input_idx * idx_sz) + else: + # cache_smem_values=True: same 128 KB budget as csv=False, with slot_sz + # = idx_sz + val_sz so SMEM per block stays ~38 KB at target=4 → L1 + # unchanged. A device-budget formula that maximises S (→4864 for fp32) + # was tried but caused 5-9% regressions on large-num_tokens single-CTA + # configs; root cause not yet confirmed, kept in git history. + INPUT_IDX_BUDGET_BASE = 128 * 1024 # 128 KB at 1 block/SM + input_idx_budget = INPUT_IDX_BUDGET_BASE // target_blocks_per_sm + val_sz = self.ordered_type.width // 8 # fp32→Uint32=4B, fp16/bf16→Uint16=2B + slot_sz = idx_sz + val_sz + max_S = input_idx_budget // (self.num_buffer_smem_input_idx * slot_sz) + return min(max_S, self.max_num_cols) + @cute.jit def to_coarse_key(self, x): """Convert to coarse 8-bit key for histogram""" @@ -149,10 +338,7 @@ def to_coarse_key(self, x): return cute.Uint8((key >> 8) & 0xFF) else: # For half/bfloat16, extract high 8 bits directly - if cutlass.const_expr(self.dtype == cutlass.Float16): - bits = half_as_ushort(x) - else: # BFloat16 - bits = half_as_ushort(x) + bits = half_as_ushort(x) key = cute.Uint16(0) if bits & 0x8000: @@ -175,10 +361,7 @@ def to_ordered(self, x): key = (bits ^ cutlass.Uint32(0xFFFFFFFF)) & cutlass.Uint32(0x7FFFFFFF) return cute.Uint32(key) else: - if cutlass.const_expr(self.dtype == cutlass.Float16): - bits = half_as_ushort(x) - else: # BFloat16 - bits = half_as_ushort(x) + bits = half_as_ushort(x) key = cute.Uint16(0) if bits & 0x8000: @@ -187,6 +370,870 @@ def to_ordered(self, x): key = (bits ^ cute.Uint16(0xFFFF)) & cute.Uint16(0x7FFF) return cute.Uint16(key) + @cute.jit + def to_ordered_and_coarse(self, x): + """Return (ordered, coarse_key) for x. + For bf16/fp16, shares the half_as_ushort + sign-flip computation. + For fp32, the two transforms differ (fp32->fp16 truncation vs full 32-bit + sign-flip), so both are computed independently. + """ + if cutlass.const_expr(self.dtype == cutlass.Float32): + return self.to_ordered(x), self.to_coarse_key(x) + else: + ordered = self.to_ordered(x) + coarse_shift = cutlass.const_expr(self.ordered_type.width - int(math.log2(self.radix))) + coarse = cute.Uint8( + (ordered >> self.ordered_type(coarse_shift)) & self.ordered_type(0xFF) + ) + return ordered, coarse + + @cute.jit + def _collect_below_threshold_coarse( + self, + tidx, + threshold_bin, + s_counter, + s_indices, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + score, + row_start, + prologue_elems, + left_start, + left_size, + ): + """Collect all indices with coarse bin < threshold_bin from GMEM, then barrier.""" + val_one = cutlass.Int32(1) + _elem_bytes = self.dtype.width // 8 + _align_bytes = self.num_copy_bits // 8 + _step_vec = self.num_threads_per_cta * self.vec_size + vec_size = self.vec_size + ic = tidx * cutlass.Int32(vec_size) + while ic + cutlass.Int32(vec_size - 1) < aligned_size: + cute.copy( + _copy_atom, + cute.make_tensor( + cute.make_ptr( + self.dtype, + _aligned_base + cutlass.Int64(ic) * cutlass.Int64(_elem_bytes), + cute.AddressSpace.gmem, + assumed_align=_align_bytes, + ), + cute.make_layout((vec_size,)), + ), + scan_frag, + ) + for j in cutlass.range_constexpr(vec_size): + bin_val = self.to_coarse_key(scan_frag[j]) + if bin_val < threshold_bin: + pos = atomicAdd(s_counter.iterator, val_one) + s_indices[pos] = self.index_type(vec_start + ic + cutlass.Int32(j)) + ic = ic + cutlass.Int32(_step_vec) + + for j in range(tidx, prologue_elems, self.num_threads_per_cta): + col_idx = cutlass.Int32(row_start + j) + raw = score[col_idx] + bin_val = self.to_coarse_key(raw) + if bin_val < threshold_bin: + pos = atomicAdd(s_counter.iterator, val_one) + s_indices[pos] = self.index_type(col_idx) + + for j in range(tidx, left_size, self.num_threads_per_cta): + col_idx = cutlass.Int32(left_start + j) + raw = score[col_idx] + bin_val = self.to_coarse_key(raw) + if bin_val < threshold_bin: + pos = atomicAdd(s_counter.iterator, val_one) + s_indices[pos] = self.index_type(col_idx) + + cute.arch.barrier() + + @cute.jit + def _collect_below_threshold_refine( + self, + tidx, + threshold, + offset, + num_input, + r_idx, + s_input_idx, + s_input_val, + score, + s_counter, + s_indices, + cur_g_num_input, + buffer, + ): + """Collect all indices with refined bin < threshold from SMEM (and GMEM buffer), then barrier.""" + val_one = cutlass.Int32(1) + for i in range(tidx, num_input, self.num_threads_per_cta): + idx = s_input_idx[r_idx, i] + idx = cutlass.Int32(cutlass.Uint32(idx)) + if cutlass.const_expr(self.cache_smem_values): + bin_val = (self.ordered_type(s_input_val[r_idx, i]) >> offset) & 0xFF + else: + bin_val = (self.to_ordered(score[idx]) >> offset) & 0xFF + if bin_val < threshold: + pos = atomicAdd(s_counter.iterator, val_one) + s_indices[pos] = self.index_type(idx) + if cutlass.const_expr(self.enable_gmem_store): + for i in range(tidx, cur_g_num_input, self.num_threads_per_cta): + idx = buffer[r_idx, i] + bin_val = (self.to_ordered(score[idx]) >> offset) & 0xFF + if bin_val < threshold: + pos = atomicAdd(s_counter.iterator, val_one) + s_indices[pos] = self.index_type(idx) + cute.arch.barrier() + + @cute.jit + def _filter_and_histogram_per_elem_coarse( + self, + bin_val, + threshold_bin, + idx, + raw_input, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + g_num_input, + buffer, + s_overflow_flag, + ): + """Per-element if/elif handler for the coarse filter pass. + + bin_val < threshold_bin → write to s_indices. + bin_val == threshold_bin → store to s_input_idx (+ optional buffer) and + update s_histogram for the next refinement round. + """ + val_one = cutlass.Int32(1) + if bin_val < threshold_bin: + pos = atomicAdd(s_counter.iterator, val_one) + s_indices[pos] = idx + elif bin_val == threshold_bin: + if cutlass.const_expr(self.enable_gmem_store): + # Hoist ordered before the pos < S check so s_input_val can be written inside it. + ordered = self.to_ordered(raw_input) + pos = atomicAdd(s_num_input.iterator, val_one) + if pos < self.filtered_topk_smem_input_size: + s_input_idx[0, pos] = idx + if cutlass.const_expr(self.cache_smem_values): + s_input_val[0, pos] = ordered + else: + buffer_pos = atomicAdd(g_num_input.iterator, val_one) + buffer[0, buffer_pos] = cutlass.Int32(cutlass.Uint32(idx)) + sub_bin = (ordered >> self.first_refine_shift) & 0xFF + atomicAdd(s_histogram.iterator + cutlass.Int32(sub_bin), val_one) + elif cutlass.const_expr(self.enable_truncate): + if cutlass.const_expr(self.dtype == cutlass.Float32): + ordered = cutlass.Uint32(0) + sub_bin = cutlass.Uint32(0) + else: + ordered = cutlass.Uint16(0) + sub_bin = cutlass.Int32(0) + pos = atomicAdd(s_num_input.iterator, val_one) + if pos < self.filtered_topk_smem_input_size: + s_input_idx[0, pos] = idx + ordered = self.to_ordered(raw_input) + if cutlass.const_expr(self.cache_smem_values): + s_input_val[0, pos] = ordered + sub_bin = (ordered >> self.first_refine_shift) & 0xFF + atomicAdd(s_histogram.iterator + cutlass.Int32(sub_bin), val_one) + elif cutlass.const_expr(self.enable_reread): + # Hoist ordered before the pos < S check so s_input_val can be written inside it. + ordered = self.to_ordered(raw_input) + pos = atomicAdd(s_num_input.iterator, val_one) + if pos < self.filtered_topk_smem_input_size: + s_input_idx[0, pos] = idx + if cutlass.const_expr(self.cache_smem_values): + s_input_val[0, pos] = ordered + else: + # Use atomicAdd (not plain store) to avoid concurrent non-atomic writes + # from multiple threads to the same SMEM address. Any non-zero value + # means overflow; the did_overflow check uses != 0. + atomicAdd(s_overflow_flag.iterator, val_one) + sub_bin = (ordered >> self.first_refine_shift) & 0xFF + atomicAdd(s_histogram.iterator + cutlass.Int32(sub_bin), val_one) + else: + # Hoist ordered before the pos < S check so s_input_val can be written inside it. + ordered = self.to_ordered(raw_input) + if cutlass.const_expr(not self.enable_reread_always): + pos = atomicAdd(s_num_input.iterator, val_one) + if pos < self.filtered_topk_smem_input_size: + s_input_idx[0, pos] = idx + if cutlass.const_expr(self.cache_smem_values): + s_input_val[0, pos] = ordered + sub_bin = (ordered >> self.first_refine_shift) & 0xFF + atomicAdd(s_histogram.iterator + cutlass.Int32(sub_bin), val_one) + + @cute.jit + def _filter_and_histogram_per_elem_refine( + self, + bin_val, + threshold, + idx_int32, + ordered_val, + offset, + r_idx, + is_last_round, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + s_last_remain, + g_num_input, + buffer, + ): + """Per-element if/elif handler for refinement rounds. + + idx_int32 – Int32 column index, used for score lookup and buffer writes. + ordered_val – pre-computed self.to_ordered(raw_input); avoids recomputing + it for sub_bin extraction when bin_val == threshold. + + bin_val < threshold → write to s_indices. + bin_val == threshold → last round: s_last_remain countdown; + otherwise: store to s_input_idx[r_idx^1] (+ optional + buffer) and update s_histogram for the next round. + """ + val_one = cutlass.Int32(1) + val_one_negative = cutlass.Int32(-1) + idx = self.index_type(idx_int32) + if bin_val < threshold: + pos = atomicAdd(s_counter.iterator, val_one) + s_indices[pos] = idx + elif bin_val == threshold: + if is_last_round: + cur_pos = atomicAdd(s_last_remain.iterator, val_one_negative) + if cur_pos > 0: + s_indices[self.top_k - cur_pos] = idx + else: + cur_pos = atomicAdd(s_num_input.iterator + (r_idx ^ 1), val_one) + if cutlass.const_expr(self.enable_gmem_store): + if cur_pos < self.filtered_topk_smem_input_size: + s_input_idx[r_idx ^ 1, cur_pos] = idx + if cutlass.const_expr(self.cache_smem_values): + s_input_val[r_idx ^ 1, cur_pos] = ordered_val + else: + buffer_pos = atomicAdd(g_num_input.iterator + (r_idx ^ 1), val_one) + buffer[r_idx ^ 1, buffer_pos] = idx_int32 + sub_bin = (ordered_val >> (offset - 8)) & 0xFF + atomicAdd(s_histogram.iterator + cutlass.Int32(sub_bin), val_one) + else: + if cutlass.const_expr(self.dtype == cutlass.Float32): + sub_bin = cutlass.Uint32(0) + else: + sub_bin = cutlass.Int32(0) + if cur_pos < self.filtered_topk_smem_input_size: + s_input_idx[r_idx ^ 1, cur_pos] = idx + if cutlass.const_expr(self.cache_smem_values): + s_input_val[r_idx ^ 1, cur_pos] = ordered_val + sub_bin = (ordered_val >> (offset - 8)) & 0xFF + atomicAdd(s_histogram.iterator + cutlass.Int32(sub_bin), val_one) + + @cute.jit + def _filter_and_histogram_coarse( + self, + tidx, + threshold_bin, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + g_num_input, + buffer, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + score, + row_start, + prologue_elems, + left_start, + left_size, + s_overflow_flag, + ): + """Reset histogram, filter all input elements through three loops, then barrier. + + Covers vec-aligned GMEM, prologue scalar, and left scalar segments. + """ + _elem_bytes = self.dtype.width // 8 + _align_bytes = self.num_copy_bits // 8 + _step_vec = self.num_threads_per_cta * self.vec_size + cute.arch.barrier() + for _hi in range(tidx, self.radix + 1, self.num_threads_per_cta): + s_histogram[_hi] = 0 + cute.arch.barrier() + + vec_size = self.vec_size + ic = tidx * cutlass.Int32(vec_size) + while ic + cutlass.Int32(vec_size - 1) < aligned_size: + cute.copy( + _copy_atom, + cute.make_tensor( + cute.make_ptr( + self.dtype, + _aligned_base + cutlass.Int64(ic) * cutlass.Int64(_elem_bytes), + cute.AddressSpace.gmem, + assumed_align=_align_bytes, + ), + cute.make_layout((vec_size,)), + ), + scan_frag, + ) + for j in cutlass.range_constexpr(vec_size): + raw_input = scan_frag[j] + bin_val = self.to_coarse_key(raw_input) + idx = self.index_type(vec_start + ic + cutlass.Int32(j)) + self._filter_and_histogram_per_elem_coarse( + bin_val, + threshold_bin, + idx, + raw_input, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + g_num_input, + buffer, + s_overflow_flag, + ) + ic = ic + cutlass.Int32(_step_vec) + + for j in range(tidx, prologue_elems, self.num_threads_per_cta): + col_idx = cutlass.Int32(row_start + j) + raw = score[col_idx] + bin_val = self.to_coarse_key(raw) + idx = self.index_type(col_idx) + self._filter_and_histogram_per_elem_coarse( + bin_val, + threshold_bin, + idx, + raw, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + g_num_input, + buffer, + s_overflow_flag, + ) + + for j in range(tidx, left_size, self.num_threads_per_cta): + col_idx = cutlass.Int32(left_start + j) + raw = score[col_idx] + bin_val = self.to_coarse_key(raw) + idx = self.index_type(col_idx) + self._filter_and_histogram_per_elem_coarse( + bin_val, + threshold_bin, + idx, + raw, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + g_num_input, + buffer, + s_overflow_flag, + ) + fence_acq_rel_cta() + cute.arch.barrier() + + @cute.jit + def _reread_always_per_elem_output( + self, + include_threshold, + raw, + col_idx, + threshold_bin, + T2, + offset, + chain_mask, + chain_prefix, + s_counter, + s_indices, + s_last_remain, + ): + """Per-element handler for REREAD_ALWAYS output scan. + include_threshold is a compile-time bool. + chain_mask is a DSL Int32 runtime value; chain_prefix is a runtime DSL + ordered_type value. Both carry accumulated prior-round constraints. + When chain_mask == 0 (round 0), ordered & 0 == 0 is always True. + """ + ordered, coarse = self.to_ordered_and_coarse(raw) + if coarse == threshold_bin: + passes_chain = (ordered & self.ordered_type(chain_mask)) == chain_prefix + if passes_chain: + bin_val = (ordered >> offset) & 0xFF + idx = self.index_type(col_idx) + val_one = cutlass.Int32(1) + if bin_val < T2: + pos = atomicAdd(s_counter.iterator, val_one) + s_indices[pos] = idx + elif cutlass.const_expr(include_threshold): + if bin_val == T2: + cur_pos = atomicAdd(s_last_remain.iterator, cutlass.Int32(-1)) + if cur_pos > 0: + s_indices[self.top_k - cur_pos] = idx + + @cute.jit + def _reread_always_per_elem_combined( + self, + raw, + col_idx, + threshold_bin, + T2, + offset, + chain_mask, + chain_prefix, + s_counter, + s_indices, + s_histogram, + ): + """Per-element handler for REREAD_ALWAYS non-last-round combined scan. + chain_mask is a DSL Int32 runtime value; chain_prefix is a runtime DSL + ordered_type value. Both carry accumulated prior-round constraints. + For elements passing coarse + chain filters: + bin_val < T2 → write col_idx to s_indices (definitely top-K). + bin_val == T2 → histogram at (ordered >> (offset - 8)) & 0xFF. + """ + ordered, coarse = self.to_ordered_and_coarse(raw) + if coarse == threshold_bin: + passes_chain = (ordered & self.ordered_type(chain_mask)) == chain_prefix + if passes_chain: + bin_val = (ordered >> offset) & 0xFF + val_one = cutlass.Int32(1) + if bin_val < T2: + pos = atomicAdd(s_counter.iterator, val_one) + s_indices[pos] = self.index_type(col_idx) + elif bin_val == T2: + next_sub_bin = (ordered >> (offset - 8)) & 0xFF + atomicAdd(s_histogram.iterator + cutlass.Int32(next_sub_bin), val_one) + + @cute.jit + def _reread_always_gmem_output_scan( + self, + include_threshold, + tidx, + threshold_bin, + T2, + offset, + chain_mask, + chain_prefix, + score, + s_counter, + s_indices, + s_last_remain, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + row_start, + prologue_elems, + left_start, + left_size, + ): + """GMEM scan for REREAD_ALWAYS output phase. + include_threshold is a compile-time bool. + chain_mask is a DSL Int32 runtime value; chain_prefix is a runtime DSL + ordered_type value — both carry prior-round constraints. + Scans all three GMEM segments and writes qualifying indices to s_indices. + Ends with cute.arch.barrier() to sync all writes before Phase 3. + """ + _elem_bytes = self.dtype.width // 8 + _align_bytes = self.num_copy_bits // 8 + _step_vec = self.num_threads_per_cta * self.vec_size + vec_size = self.vec_size + + ic = tidx * cutlass.Int32(vec_size) + while ic + cutlass.Int32(vec_size - 1) < aligned_size: + cute.copy( + _copy_atom, + cute.make_tensor( + cute.make_ptr( + self.dtype, + _aligned_base + cutlass.Int64(ic) * cutlass.Int64(_elem_bytes), + cute.AddressSpace.gmem, + assumed_align=_align_bytes, + ), + cute.make_layout((vec_size,)), + ), + scan_frag, + ) + for j in cutlass.range_constexpr(vec_size): + col_idx = cutlass.Int32(vec_start + ic + cutlass.Int32(j)) + self._reread_always_per_elem_output( + include_threshold, + scan_frag[j], + col_idx, + threshold_bin, + T2, + offset, + chain_mask, + chain_prefix, + s_counter, + s_indices, + s_last_remain, + ) + ic = ic + cutlass.Int32(_step_vec) + + for j in range(tidx, prologue_elems, self.num_threads_per_cta): + col_idx = cutlass.Int32(row_start + j) + raw = score[col_idx] + self._reread_always_per_elem_output( + include_threshold, + raw, + col_idx, + threshold_bin, + T2, + offset, + chain_mask, + chain_prefix, + s_counter, + s_indices, + s_last_remain, + ) + + for j in range(tidx, left_size, self.num_threads_per_cta): + col_idx = cutlass.Int32(left_start + j) + raw = score[col_idx] + self._reread_always_per_elem_output( + include_threshold, + raw, + col_idx, + threshold_bin, + T2, + offset, + chain_mask, + chain_prefix, + s_counter, + s_indices, + s_last_remain, + ) + + cute.arch.barrier() + + @cute.jit + def _reread_always_gmem_combined_scan( + self, + tidx, + threshold_bin, + T2, + offset, + chain_mask, + chain_prefix, + score, + s_counter, + s_indices, + s_histogram, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + row_start, + prologue_elems, + left_start, + left_size, + ): + """GMEM scan for REREAD_ALWAYS non-last rounds: reset histogram, output < T2 + elements, and build histogram for the next round. + chain_mask is a DSL Int32 runtime value; chain_prefix is a runtime DSL + ordered_type value — both carry prior-round constraints. + Ends with fence_acq_rel_cta() + cute.arch.barrier(). + Returns updated chain_prefix (runtime DSL value); caller updates chain_mask + via | (cutlass.Int32(0xFF) << offset). + """ + _elem_bytes = self.dtype.width // 8 + _align_bytes = self.num_copy_bits // 8 + _step_vec = self.num_threads_per_cta * self.vec_size + vec_size = self.vec_size + + # Barrier before clearing s_histogram: ensures all threads have already + # read s_histogram[threshold-1] to update topk_remaining in the caller + # before any thread starts zeroing it here. + cute.arch.barrier() + for _hi in range(tidx, self.radix + 1, self.num_threads_per_cta): + s_histogram[_hi] = 0 + cute.arch.barrier() + + ic = tidx * cutlass.Int32(vec_size) + while ic + cutlass.Int32(vec_size - 1) < aligned_size: + cute.copy( + _copy_atom, + cute.make_tensor( + cute.make_ptr( + self.dtype, + _aligned_base + cutlass.Int64(ic) * cutlass.Int64(_elem_bytes), + cute.AddressSpace.gmem, + assumed_align=_align_bytes, + ), + cute.make_layout((vec_size,)), + ), + scan_frag, + ) + for j in cutlass.range_constexpr(vec_size): + col_idx = cutlass.Int32(vec_start + ic + cutlass.Int32(j)) + self._reread_always_per_elem_combined( + scan_frag[j], + col_idx, + threshold_bin, + T2, + offset, + chain_mask, + chain_prefix, + s_counter, + s_indices, + s_histogram, + ) + ic = ic + cutlass.Int32(_step_vec) + + for j in range(tidx, prologue_elems, self.num_threads_per_cta): + col_idx = cutlass.Int32(row_start + j) + raw = score[col_idx] + self._reread_always_per_elem_combined( + raw, + col_idx, + threshold_bin, + T2, + offset, + chain_mask, + chain_prefix, + s_counter, + s_indices, + s_histogram, + ) + + for j in range(tidx, left_size, self.num_threads_per_cta): + col_idx = cutlass.Int32(left_start + j) + raw = score[col_idx] + self._reread_always_per_elem_combined( + raw, + col_idx, + threshold_bin, + T2, + offset, + chain_mask, + chain_prefix, + s_counter, + s_indices, + s_histogram, + ) + + fence_acq_rel_cta() + cute.arch.barrier() + + # Return updated chain_prefix (runtime DSL value). + # Caller updates chain_mask via | (cutlass.Int32(0xFF) << offset). + return chain_prefix | self.ordered_type(self.ordered_type(T2) << self.ordered_type(offset)) + + @cute.jit + def _reread_gmem_rescan( + self, + topk_remaining, + is_last_round, + tidx, + threshold_bin, + threshold, + offset, + chain_mask, + chain_prefix, + score, + s_counter, + s_indices, + s_last_remain, + s_histogram, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + row_start, + prologue_elems, + left_start, + left_size, + ): + """GMEM re-scan phase shared by REREAD_ALWAYS and REREAD-overflow paths. + + Returns (run_next_round, chain_mask, chain_prefix). + """ + run_next_round = True + if topk_remaining == 0: + self._reread_always_gmem_output_scan( + False, + tidx, + threshold_bin, + threshold, + offset, + chain_mask, + chain_prefix, + score, + s_counter, + s_indices, + s_last_remain, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + row_start, + prologue_elems, + left_start, + left_size, + ) + run_next_round = False + else: + if is_last_round: + self._reread_always_gmem_output_scan( + True, + tidx, + threshold_bin, + threshold, + offset, + chain_mask, + chain_prefix, + score, + s_counter, + s_indices, + s_last_remain, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + row_start, + prologue_elems, + left_start, + left_size, + ) + else: + chain_prefix = self._reread_always_gmem_combined_scan( + tidx, + threshold_bin, + threshold, + offset, + chain_mask, + chain_prefix, + score, + s_counter, + s_indices, + s_histogram, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + row_start, + prologue_elems, + left_start, + left_size, + ) + chain_mask = chain_mask | (cutlass.Int32(0xFF) << cutlass.Int32(offset)) + return run_next_round, chain_mask, chain_prefix + + @cute.jit + def _filter_and_histogram_refine( + self, + tidx, + threshold, + offset, + r_idx, + is_last_round, + num_input, + cur_g_num_input, + score, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + s_last_remain, + g_num_input, + buffer, + ): + """Reset histogram, filter all threshold-bucket elements, then barrier. + + Covers SMEM s_input_idx loop and optional GMEM buffer loop. + """ + cute.arch.barrier() + for _hi in range(tidx, self.radix + 1, self.num_threads_per_cta): + s_histogram[_hi] = 0 + cute.arch.barrier() + + for i in range(tidx, num_input, self.num_threads_per_cta): + idx_tmp = s_input_idx[r_idx, i] + idx_int32 = cutlass.Int32(cutlass.Uint32(idx_tmp)) + if cutlass.const_expr(self.cache_smem_values): + ordered_val = self.ordered_type(s_input_val[r_idx, i]) + else: + raw_input = score[idx_int32] + ordered_val = self.to_ordered(raw_input) + bin_val = (ordered_val >> offset) & 0xFF + self._filter_and_histogram_per_elem_refine( + bin_val, + threshold, + idx_int32, + ordered_val, + offset, + r_idx, + is_last_round, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + s_last_remain, + g_num_input, + buffer, + ) + + if cutlass.const_expr(self.enable_gmem_store): + cute.arch.barrier() + for i in range(tidx, cur_g_num_input, self.num_threads_per_cta): + idx_int32 = buffer[r_idx, i] + raw_input = score[idx_int32] + ordered_val = self.to_ordered(raw_input) + bin_val = (ordered_val >> offset) & 0xFF + self._filter_and_histogram_per_elem_refine( + bin_val, + threshold, + idx_int32, + ordered_val, + offset, + r_idx, + is_last_round, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + s_last_remain, + g_num_input, + buffer, + ) + fence_acq_rel_cta() + cute.arch.barrier() + @cute.jit def prefix_sum_and_find_threshold_coarse( self, @@ -367,6 +1414,173 @@ def prefix_sum_and_find_threshold_fine_grained( # sync among all threads in a cta. cute.arch.barrier() + @cute.jit + def _cluster_reduce_histogram(self, tidx, s_histogram, s_hist_merged): + """DSMEM histogram reduction for the single-pass multi-CTA path. + + Each thread sums bin ``my_bin`` across all peer CTAs' LOCAL + ``s_histogram`` (including self) via cluster DSMEM and writes the total + to the SEPARATE ``s_hist_merged`` buffer. Never writes in-place: a peer + may still be reading our ``s_histogram`` via DSMEM. + + The caller owns the surrounding barriers:: + + cluster_arrive(); cluster_wait() # publish local histograms + _cluster_reduce_histogram(...) + cute.arch.barrier() # s_hist_merged ready (intra-CTA) + + cluster_arrive_relaxed(); cluster_wait() # peers done reading before rebuild + + The two arrives differ on purpose: the publish one is non-relaxed + ``cluster_arrive`` (release fence, so the s_histogram stores are visible + to a peer's ld.shared::cluster; relaxed would risk stale reads — cf. GVR + fix bc6d0e83a3), while the post-read one is relaxed (the peer reads + drained into s_hist_merged before the intra-CTA barrier, so it is a + liveness-only WAR barrier). + + Only bins ``0 .. radix-1`` are merged; the ``radix`` guard slot is not + read by the prefix-sum helpers. + """ + for my_bin in range(tidx, self.radix, self.num_threads_per_cta): + acc = cutlass.Int32(0) + local_ptr = s_histogram.iterator + cutlass.Int32(my_bin) + for peer in cutlass.range_constexpr(self.num_ctas_per_row): + remote = mapa_shared_cluster(local_ptr, cutlass.Int32(peer)) + acc = acc + ld_shared_cluster_i32(remote) + s_hist_merged[my_bin] = acc + + @cute.jit + def _cluster_collect( + self, + tidx, + s_indices, + s_counter, + s_last_remain, + s_prefix, + cta_in_group, + topk_remaining, + output_indices_row, + score, + output_values_row, + ): + """Unified DSMEM prefix-scan output collection (Path A + Path B). + + ``s_prefix`` is a per-CTA scratch (reuses ``s_histogram``; needs >= 4 + int32 slots): [0]=group-1 count, [1]=group-2 count, [2]/[3]=computed + exclusive offsets. ``s_indices`` already holds this CTA's local + group-1 at [0, s_counter) and group-2 at [top_k-topk_remaining, ...), + filled by the reused Path A/B collection. + + Decode-only: indices are absolute column indices, written directly. + """ + num_threads = self.num_threads_per_cta + # 1. Publish this CTA's group-1 / group-2 counts. + # group-2 count = topk_remaining - max(0, s_last_remain[0]); s_last_remain + # starts at the final topk_remaining and is decremented per group-2 write. + if tidx == 0: + s_prefix[0] = s_counter[0] + slr = s_last_remain[0] + if slr < cutlass.Int32(0): + slr = cutlass.Int32(0) + s_prefix[1] = topk_remaining - slr + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + + # 2. Exclusive prefix over peers p < cta_in_group (thread 0 computes). + if tidx == 0: + eo1 = cutlass.Int32(0) + eo2 = cutlass.Int32(0) + p0 = s_prefix.iterator + cutlass.Int32(0) + p1 = s_prefix.iterator + cutlass.Int32(1) + for peer in cutlass.range_constexpr(self.num_ctas_per_row): + if cutlass.Int32(peer) < cta_in_group: + eo1 = eo1 + ld_shared_cluster_i32(mapa_shared_cluster(p0, cutlass.Int32(peer))) + eo2 = eo2 + ld_shared_cluster_i32(mapa_shared_cluster(p1, cutlass.Int32(peer))) + s_prefix[2] = eo1 + s_prefix[3] = eo2 + cute.arch.barrier() + # Liveness barrier before exit: relaxed (peer s_prefix reads already drained). + cute.arch.cluster_arrive_relaxed() + cute.arch.cluster_wait() + + exclusive_offset_1 = s_prefix[2] + exclusive_offset_2 = s_prefix[3] + group1_total = self.top_k - topk_remaining + group2_count = s_prefix[1] + local_g1 = s_counter[0] + + # 3. group-1: s_indices[0 .. s_counter-1] -> output[exclusive_offset_1 + i] + for i in range(tidx, local_g1, num_threads): + idx = cutlass.Int32(cutlass.Uint32(s_indices[i])) + pos = exclusive_offset_1 + i + output_indices_row[pos] = idx + if cutlass.const_expr(self.return_val): + output_values_row[pos] = score[idx] + + # 4. group-2: s_indices[top_k-topk_remaining + i] -> output[group1_total + eo2 + i] + # (Path A: group2_count == 0, loop is a no-op) + for i in range(tidx, group2_count, num_threads): + pos = group1_total + exclusive_offset_2 + i + if pos < self.top_k: + src = self.top_k - topk_remaining + i + idx = cutlass.Int32(cutlass.Uint32(s_indices[src])) + output_indices_row[pos] = idx + if cutlass.const_expr(self.return_val): + output_values_row[pos] = score[idx] + + @cute.jit + def _phase3_writeback(self, tidx, row_start, s_indices, score, indices, dst, dst_values): + """Write the selected top-k from s_indices (+ values) back to GMEM output. + + Extracted verbatim from filtered_topk_kernel_per_row (no logic change) so the + single-pass multi-CTA path can dispatch between this and a cluster collector. + """ + # Phase 3: Output phase + output_vector_width = 2 if self.top_k % 2 == 0 else 1 + vecsize_out = cutlass.const_expr( + min( + self.top_k, + cute.ceil_div(self.top_k, self.num_threads_per_cta), + self.num_copy_bits // self.dtype.width, + # TODO: only tested for float32. need to check for other dtypes. + output_vector_width, + ) + ) + assert self.top_k % vecsize_out == 0 + + nvec_per_thread = cutlass.const_expr( + cute.ceil_div(self.top_k, vecsize_out * self.num_threads_per_cta) + ) + topk_vals = cute.make_fragment((vecsize_out, nvec_per_thread), self.dtype) + topk_indices = cute.make_fragment((vecsize_out, nvec_per_thread), cutlass.Int32) + + stride = self.num_threads_per_cta * vecsize_out + for i in cutlass.range(nvec_per_thread, unroll_full=True): + idx = i * stride + tidx % self.num_threads_per_cta * vecsize_out + if idx < self.top_k: + for v in cutlass.range(vecsize_out, unroll_full=True): + index_raw = s_indices[idx + v] + index = cutlass.Int32(cutlass.Uint32(index_raw)) + if cutlass.const_expr(self.return_val): + topk_vals[v, i] = score[index] + if cutlass.const_expr(self.merge_blocks): + topk_indices[v, i] = indices[index] + elif cutlass.const_expr(self.subtract_row_start_on_output): + topk_indices[v, i] = index - cutlass.Int32(row_start) + else: + topk_indices[v, i] = index + # [atom, rest_vec] + mIndices_store = cute.tiled_divide(dst, (vecsize_out,)) + if cutlass.const_expr(self.return_val): + mValues_store = cute.tiled_divide(dst_values, (vecsize_out,)) + # i represents the index of the vector in the output. + for i in cutlass.range(cute.size(topk_vals.shape, [1]), unroll_full=True): + col = i * self.num_threads_per_cta + tidx % self.num_threads_per_cta + if col < self.top_k // vecsize_out: + cute.autovec_copy(topk_indices[None, i], mIndices_store[None, col]) + if cutlass.const_expr(self.return_val): + cute.autovec_copy(topk_vals[None, i], mValues_store[None, col]) + @cute.jit def filtered_topk_kernel_per_row( self, @@ -376,9 +1590,6 @@ def filtered_topk_kernel_per_row( extra_buffer: cute.Tensor, output_indices: cute.Tensor, output_values: cute.Tensor, - tiler_mn: cute.Shape, - copy_atom: cute.CopyAtom, - tiled_copy: cute.TiledCopy, row_start: int, length: int, bidx: int, @@ -389,28 +1600,52 @@ def filtered_topk_kernel_per_row( g_num_input, s_indices, s_input_idx, + s_input_val, s_last_remain, num_warps, s_warp_sums, + s_overflow_flag, + need_cluster_sync=False, + s_hist_merged=None, + cta_in_group=0, ): - """CuTe DSL implementation of TopK kernel based on radix-based filter algorithm.""" + """CuTe DSL implementation of TopK kernel based on radix-based filter algorithm. + + Single-pass multi-CTA (radix-filter cluster) extras — only live when + ``self.single_pass_multi_cta`` is True (const-folded away otherwise): + - ``need_cluster_sync`` (runtime): True for cluster cooperation + (needed_ctas >= 2), False for the solo fast path. + - ``s_hist_merged``: separate DSMEM merge target (radix+1 int32). + - ``cta_in_group``: this CTA's rank within its cluster. + """ # # Thread and block indexing tidx, _, _ = cute.arch.thread_idx() score = input[bidx, None] if cutlass.const_expr(self.merge_blocks): indices = input_indices[bidx, None] + else: + indices = None if cutlass.const_expr(self.enable_multi_cta): dst = output_indices if cutlass.const_expr(self.return_val): dst_values = output_values + else: + dst_values = None else: dst = output_indices[bidx, None] if cutlass.const_expr(self.return_val): dst_values = output_values[bidx, None] + else: + dst_values = None # Note, for multi-cta version, each ctas must have its own extra_buffer. + buffer = None if cutlass.const_expr(self.enable_gmem_store): - if cutlass.const_expr(self.enable_multi_cta): + if cutlass.const_expr(self.single_pass_multi_cta): + # Per-CTA spill buffer: (num_rows * ctas_per_group, ...). bidx has + # already been set to row_id by the decode kernel. + buffer = extra_buffer[bidx * self.num_ctas_per_row + cta_in_group, None, None] + elif cutlass.const_expr(self.enable_multi_cta): grid_dim_x, grid_dim_y, _ = cute.arch.grid_dim() bidx_val, bidy_val, _ = cute.arch.block_idx() buffer_row_id = bidx_val * grid_dim_y + bidy_val @@ -434,43 +1669,54 @@ def filtered_topk_kernel_per_row( prologue_elems = cutlass.Int32(fix_bytes // elem_bytes) - remaining = length - prologue_elems + # SP multi-CTA cluster mode: an empty chunk (chunk_start >= eff_len -> + # length <= 0) must scan NOTHING. Otherwise the prologue/left loops + # (bounded by alignment, not length) would read -inf padding past the + # row end and corrupt the DSMEM-merged histogram. Clamp so the total + # scanned == max(length, 0). Guarded under const_expr so single-CTA / + # 2-pass codegen is unchanged (there length > top_k in this branch). + if cutlass.const_expr(self.single_pass_multi_cta): + _len_nonneg = length + if _len_nonneg < 0: + _len_nonneg = cutlass.Int32(0) + if prologue_elems > _len_nonneg: + prologue_elems = _len_nonneg + remaining = _len_nonneg - prologue_elems + else: + remaining = length - prologue_elems aligned_size = (remaining // self.vec_size) * self.vec_size left_size = remaining - aligned_size vec_start = row_start + prologue_elems left_start = vec_start + aligned_size - shape = input.shape - - idX = cute.make_identity_tensor((shape[0], aligned_size)) - input_ptr = input.iterator + vec_start - input_addr_u64 = input_ptr.toint() - input_ptr_aligned = cute.make_ptr(self.dtype, input_addr_u64, assumed_align=align_bytes) - - input_tensor = cute.make_tensor( - input_ptr_aligned, - cute.make_layout((shape[0], aligned_size), stride=input.stride), + # GVR-style direct GMEM load constants (all Python ints, compile-time). + # Loop bounds computed from runtime aligned_size so threads past the + # actual row end execute zero iterations — no OOB waste for short rows. + vec_size = self.vec_size + _elem_bytes = self.dtype.width // 8 + _align_bytes = self.num_copy_bits // 8 + _step_vec = self.num_threads_per_cta * self.vec_size + # Byte address of the aligned portion start for this row (score[vec_start]). + _aligned_base = (score.iterator + vec_start).toint() + # TODO: add invariant=True for .CONSTANT cache hint once validated + _copy_atom = cute.make_copy_atom( + cute.nvgpu.CopyG2ROp(), + self.dtype, + num_bits_per_copy=self.num_copy_bits, ) - # slice for CTAs - gX, cX = [cute.local_tile(mT, tiler_mn, (bidx, None)) for mT in (input_tensor, idX)] - # Note, we use gX_aligned here to avoid the alignment issue when the input is not aligned. - gX_aligned_ptr = cute.make_ptr(self.dtype, gX.iterator.toint(), assumed_align=align_bytes) - gX_aligned = cute.make_tensor(gX_aligned_ptr, cute.make_layout(gX.shape, stride=gX.stride)) - - self.num_sub_tiles = gX.shape[2] - - thr_copy = tiled_copy.get_slice(tidx) - - tXgX = thr_copy.partition_S(gX_aligned) - tXcX = thr_copy.partition_S(cX)[(0, None), None, None, None] - tXrX = cute.make_fragment_like(tXgX[None, None, None, 0]) - - tXcX_tile = thr_copy.partition_S(cX) - - # Trivial case: length <= top_k - if length <= self.top_k: + scan_frag = cute.make_fragment((vec_size,), self.dtype) + + # Trivial case: length <= top_k. In SP multi-CTA cluster mode this + # per-chunk shortcut is unsafe (a CTA taking it would skip the cluster + # barriers -> deadlock, and emit its whole chunk as the row's top-k); + # force the full radix path so every CTA cooperates. + take_trivial = length <= self.top_k + if cutlass.const_expr(self.single_pass_multi_cta): + if need_cluster_sync: + take_trivial = False + if take_trivial: for i in range(tidx, self.top_k, self.num_threads_per_cta): # TODO: add multi-cta version support here. if i < length: @@ -481,10 +1727,11 @@ def filtered_topk_kernel_per_row( else: dst[i] = i if cutlass.const_expr(self.return_val): - if cutlass.const_expr(self.enable_multi_cta): - dst_values[i] = score[i + row_start] - else: - dst_values[i] = score[i] + # dst[i] is a local index i; its value lives at the + # absolute column row_start + i (row_start may be + # non-zero for prefill). enable_multi_cta writes the + # absolute index but reads the same absolute column. + dst_values[i] = score[i + row_start] else: dst[i] = -1 if cutlass.const_expr(self.return_val): @@ -495,39 +1742,37 @@ def filtered_topk_kernel_per_row( topk_remaining = self.top_k val_one = cutlass.Int32(1) - val_one_negative = cutlass.Int32(-1) # Stage 1: Coarse histogram. - if tidx < self.radix + 1: - s_histogram[tidx] = 0 + # Use a strided loop so every bin is cleared even when + # num_threads_per_cta < radix (e.g. 128 < 256). + for _hi in range(tidx, self.radix + 1, self.num_threads_per_cta): + s_histogram[_hi] = 0 + if cutlass.const_expr(self.enable_reread): + if tidx == 0: + s_overflow_flag[0] = 0 cute.arch.barrier() - # 1.1 Build histogram with vectorized loads - vec_size = self.vec_size - - for tile_idx in range(self.num_sub_tiles): - tXpX_tile = self.predicate_tile( - tXcX_tile[None, None, None, tile_idx], - cutlass.Int32(aligned_size), - ) + # 1.1 Build histogram + ic = tidx * cutlass.Int32(vec_size) + while ic + cutlass.Int32(vec_size - 1) < aligned_size: cute.copy( - copy_atom, - tXgX[None, None, None, tile_idx], - tXrX, - pred=tXpX_tile[None, None, None], - ) - self._fill_oob( - tXrX, - tXpX_tile[None, None, None], - -tXrX.element_type.inf, + _copy_atom, + cute.make_tensor( + cute.make_ptr( + self.dtype, + _aligned_base + cutlass.Int64(ic) * cutlass.Int64(_elem_bytes), + cute.AddressSpace.gmem, + assumed_align=_align_bytes, + ), + cute.make_layout((vec_size,)), + ), + scan_frag, ) - - for i in cutlass.range(cute.size(tXrX), unroll_full=True): - bin_val = self.to_coarse_key(tXrX[i]) - atomicAdd( - s_histogram.iterator + cutlass.Int32(bin_val), - val_one, - ) + for j in cutlass.range_constexpr(vec_size): + bin_val = self.to_coarse_key(scan_frag[j]) + atomicAdd(s_histogram.iterator + cutlass.Int32(bin_val), val_one) + ic = ic + cutlass.Int32(_step_vec) # for initial scalar load part. for j in range(tidx, prologue_elems, self.num_threads_per_cta): @@ -551,520 +1796,362 @@ def filtered_topk_kernel_per_row( cute.arch.barrier() - # 1.2 and 1.3 Suffix sum to find threshold and find threshold bin - self.prefix_sum_and_find_threshold_coarse( - tidx, - s_histogram, - s_warp_sums, - num_warps, - s_threshold_bin_id, - s_num_input, - s_counter, - s_last_remain, - topk_remaining, - g_num_input, - s_num_input_idx=0, - ) + # 1.2 and 1.3 Suffix sum to find threshold and find threshold bin. + # SP multi-CTA cluster: DSMEM-merge peer histograms into s_hist_merged + # first, then prefix-sum the merged buffer. The prefix-sum / threshold + # subtraction are duplicated per branch (rather than selecting the + # buffer into a variable) because the DSL cannot phi-merge two distinct + # tensors across a runtime `if`. threshold_bin (a shared SMEM scalar) + # is read straight-line; only the buffer read in the -= differs. + if cutlass.const_expr(self.single_pass_multi_cta): + if need_cluster_sync: + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + self._cluster_reduce_histogram(tidx, s_histogram, s_hist_merged) + cute.arch.barrier() + self.prefix_sum_and_find_threshold_coarse( + tidx, + s_hist_merged, + s_warp_sums, + num_warps, + s_threshold_bin_id, + s_num_input, + s_counter, + s_last_remain, + topk_remaining, + g_num_input, + s_num_input_idx=0, + ) + # WAR barrier: relaxed (peer reads already drained, see docstring). + cute.arch.cluster_arrive_relaxed() + cute.arch.cluster_wait() + else: + self.prefix_sum_and_find_threshold_coarse( + tidx, + s_histogram, + s_warp_sums, + num_warps, + s_threshold_bin_id, + s_num_input, + s_counter, + s_last_remain, + topk_remaining, + g_num_input, + s_num_input_idx=0, + ) + else: + self.prefix_sum_and_find_threshold_coarse( + tidx, + s_histogram, + s_warp_sums, + num_warps, + s_threshold_bin_id, + s_num_input, + s_counter, + s_last_remain, + topk_remaining, + g_num_input, + s_num_input_idx=0, + ) threshold_bin = s_threshold_bin_id[0] if threshold_bin > 0: - topk_remaining -= s_histogram[threshold_bin - 1] + if cutlass.const_expr(self.single_pass_multi_cta): + if need_cluster_sync: + topk_remaining -= s_hist_merged[threshold_bin - 1] + else: + topk_remaining -= s_histogram[threshold_bin - 1] + else: + topk_remaining -= s_histogram[threshold_bin - 1] # 1.4 Collect indices if topk_remaining == 0: - # Collect indices where bin > threshold - for tile_idx in range(self.num_sub_tiles): - tXpX_tile = self.predicate_tile( - tXcX_tile[None, None, None, tile_idx], - cutlass.Int32(aligned_size), - ) - cute.copy( - copy_atom, - tXgX[None, None, None, tile_idx], - tXrX, - pred=tXpX_tile[None, None, None], - ) - self._fill_oob( - tXrX, - tXpX_tile[None, None, None], - -tXrX.element_type.inf, - ) - for i in cutlass.range(cute.size(tXrX), unroll_full=True): - cur_tXcX = tXcX[None, None, None, tile_idx] - bin_val = self.to_coarse_key(tXrX[i]) - if bin_val < threshold_bin: - pos = atomicAdd(s_counter.iterator, val_one) - idx = self.index_type( - cur_tXcX[i // vec_size][1] + i % vec_size + vec_start - ) - s_indices[pos] = idx - - # for initial scalar load part. - for j in range(tidx, prologue_elems, self.num_threads_per_cta): - col_idx = cutlass.Int32(row_start + j) - raw = score[col_idx] - bin_val = self.to_coarse_key(raw) - if bin_val < threshold_bin: - pos = atomicAdd(s_counter.iterator, val_one) - idx = self.index_type(col_idx) - s_indices[pos] = idx - - # for left part (left_size) - for j in range(tidx, left_size, self.num_threads_per_cta): - col_idx = cutlass.Int32(left_start + j) - raw = score[col_idx] - bin_val = self.to_coarse_key(raw) - if bin_val < threshold_bin: - pos = atomicAdd(s_counter.iterator, val_one) - idx = self.index_type(col_idx) - s_indices[pos] = idx - - cute.arch.barrier() - + self._collect_below_threshold_coarse( + tidx, + threshold_bin, + s_counter, + s_indices, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + score, + row_start, + prologue_elems, + left_start, + left_size, + ) else: - # Reset histogram for refinement - cute.arch.barrier() - if tidx < self.radix + 1: - s_histogram[tidx] = 0 - cute.arch.barrier() - - # Filter and build refinement histogram - for tile_idx in range(self.num_sub_tiles): - tXpX_tile = self.predicate_tile( - tXcX_tile[None, None, None, tile_idx], - cutlass.Int32(aligned_size), - ) - cute.copy( - copy_atom, - tXgX[None, None, None, tile_idx], - tXrX, - pred=tXpX_tile[None, None, None], - ) - self._fill_oob( - tXrX, - tXpX_tile[None, None, None], - -tXrX.element_type.inf, - ) - - for i in cutlass.range(cute.size(tXrX), unroll_full=True): - raw_input = tXrX[i] - bin_val = self.to_coarse_key(raw_input) - cur_tXcX = tXcX[None, None, None, tile_idx] - idx = self.index_type(cur_tXcX[i // vec_size][1] + i % vec_size + vec_start) - if bin_val < threshold_bin: - pos = atomicAdd(s_counter.iterator, val_one) - s_indices[pos] = idx - elif bin_val == threshold_bin: - # pos = atomicAdd(s_num_input[0], 1) - pos = atomicAdd(s_num_input.iterator, val_one) - if cutlass.const_expr(self.enable_gmem_store): - if pos < self.filtered_topk_smem_input_size: - s_input_idx[0, pos] = idx - else: - buffer_pos = atomicAdd( - g_num_input.iterator, - val_one, - ) - buffer[0, buffer_pos] = cutlass.Int32(cutlass.Uint32(idx)) - ordered = self.to_ordered(raw_input) - sub_bin = (ordered >> self.first_refine_shift) & 0xFF - # atomicAdd(s_histogram[sub_bin], 1) - atomicAdd( - s_histogram.iterator + cutlass.Int32(sub_bin), - val_one, - ) - else: - if pos < self.filtered_topk_smem_input_size: - s_input_idx[0, pos] = idx - ordered = self.to_ordered(raw_input) - sub_bin = (ordered >> self.first_refine_shift) & 0xFF - # atomicAdd(s_histogram[sub_bin], 1) - atomicAdd( - s_histogram.iterator + cutlass.Int32(sub_bin), - val_one, - ) - - # for initial scalar load part. - for j in range(tidx, prologue_elems, self.num_threads_per_cta): - col_idx = cutlass.Int32(row_start + j) - raw = score[col_idx] - bin_val = self.to_coarse_key(raw) - if bin_val < threshold_bin: - pos = atomicAdd(s_counter.iterator, val_one) - idx = self.index_type(col_idx) - s_indices[pos] = idx - elif bin_val == threshold_bin: - pos = atomicAdd( - s_num_input.iterator, - val_one, - ) - # TODO: add gmem buffer here. - if cutlass.const_expr(self.enable_gmem_store): - if pos < self.filtered_topk_smem_input_size: - s_input_idx[0, pos] = self.index_type(col_idx) - else: - buffer_pos = atomicAdd( - g_num_input.iterator, - val_one, - ) - buffer[0, buffer_pos] = cutlass.Int32(col_idx) - ordered = self.to_ordered(raw) - sub_bin = (ordered >> self.first_refine_shift) & 0xFF - atomicAdd( - s_histogram.iterator + cutlass.Int32(sub_bin), - val_one, - ) - else: - # TODO: how to handle the type of sub_bin and ordered? - if cutlass.const_expr(self.dtype == cutlass.Float32): - ordered = cutlass.Uint32(0) - sub_bin = cutlass.Uint32(0) - else: - ordered = cutlass.Uint16(0) - sub_bin = cutlass.Int32(0) - if pos < self.filtered_topk_smem_input_size: - s_input_idx[0, pos] = self.index_type(col_idx) - ordered = self.to_ordered(raw) - sub_bin = (ordered >> self.first_refine_shift) & 0xFF - atomicAdd( - s_histogram.iterator + cutlass.Int32(sub_bin), - val_one, - ) - - # for left part - for j in range(tidx, left_size, self.num_threads_per_cta): - col_idx = cutlass.Int32(left_start + j) - raw = score[col_idx] - bin_val = self.to_coarse_key(raw) - if bin_val < threshold_bin: - pos = atomicAdd(s_counter.iterator, val_one) - idx = self.index_type(col_idx) - s_indices[pos] = idx - elif bin_val == threshold_bin: - pos = atomicAdd( - s_num_input.iterator, - val_one, - ) - # TODO: add gmem buffer here. - if cutlass.const_expr(self.enable_gmem_store): - if pos < self.filtered_topk_smem_input_size: - s_input_idx[0, pos] = self.index_type(col_idx) - else: - buffer_pos = atomicAdd( - g_num_input.iterator, - val_one, - ) - buffer[0, buffer_pos] = cutlass.Int32(col_idx) - ordered = self.to_ordered(raw) - sub_bin = (ordered >> self.first_refine_shift) & 0xFF - atomicAdd( - s_histogram.iterator + cutlass.Int32(sub_bin), - val_one, - ) - else: - # TODO: how to handle the type of sub_bin and ordered? - if cutlass.const_expr(self.dtype == cutlass.Float32): - ordered = cutlass.Uint32(0) - sub_bin = cutlass.Uint32(0) - else: - ordered = cutlass.Uint16(0) - sub_bin = cutlass.Int32(0) - if pos < self.filtered_topk_smem_input_size: - s_input_idx[0, pos] = self.index_type(col_idx) - ordered = self.to_ordered(raw) - sub_bin = (ordered >> self.first_refine_shift) & 0xFF - atomicAdd( - s_histogram.iterator + cutlass.Int32(sub_bin), - val_one, - ) - fence_acq_rel_cta() - cute.arch.barrier() + self._filter_and_histogram_coarse( + tidx, + threshold_bin, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + g_num_input, + buffer, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + score, + row_start, + prologue_elems, + left_start, + left_size, + s_overflow_flag, + ) # Phase 2: Refinement rounds + # chain_mask (DSL Int32) and chain_prefix (runtime DSL ordered_type) + # accumulate prior-round constraints for REREAD_ALWAYS / REREAD overflow + # fallback. chain_mask is Int32 so it survives DSL phi-merge across the + # dynamic loop. + chain_mask = cutlass.Int32(0) + chain_prefix = self.ordered_type(0) + # REREAD: read overflow flag once before the loop; runtime bool that + # selects SMEM refinement (no overflow) vs GMEM re-scan (overflow). + # Visibility of s_overflow_flag[0] is guaranteed by the fence_acq_rel_cta() + # + barrier() at the end of _filter_and_histogram_coarse above; no additional + # barrier is needed here. If that function's terminal barrier is ever moved + # to the call site, a barrier must be inserted before this read. + if cutlass.const_expr(self.enable_reread): + did_overflow = s_overflow_flag[0] != 0 run_next_round = True for round in range(self.num_refine_rounds): if run_next_round: r_idx = round % 2 - self.prefix_sum_and_find_threshold_fine_grained( - tidx, - s_histogram, - s_warp_sums, - num_warps, - s_threshold_bin_id, - s_num_input, - s_counter, - s_last_remain, - topk_remaining, - g_num_input, - s_num_input_idx=r_idx ^ 1, - ) - num_input = min(s_num_input[r_idx], self.filtered_topk_smem_input_size) - if cutlass.const_expr(self.enable_gmem_store): - cur_g_num_input = g_num_input[r_idx] - + # SP multi-CTA cluster: DSMEM-merge peer histograms before + # the per-round prefix sum (same shape as the coarse site; + # duplicated per branch to avoid a tensor phi-merge). + if cutlass.const_expr(self.single_pass_multi_cta): + if need_cluster_sync: + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + self._cluster_reduce_histogram(tidx, s_histogram, s_hist_merged) + cute.arch.barrier() + self.prefix_sum_and_find_threshold_fine_grained( + tidx, + s_hist_merged, + s_warp_sums, + num_warps, + s_threshold_bin_id, + s_num_input, + s_counter, + s_last_remain, + topk_remaining, + g_num_input, + s_num_input_idx=r_idx ^ 1, + ) + # WAR barrier: relaxed (peer reads already drained). + cute.arch.cluster_arrive_relaxed() + cute.arch.cluster_wait() + else: + self.prefix_sum_and_find_threshold_fine_grained( + tidx, + s_histogram, + s_warp_sums, + num_warps, + s_threshold_bin_id, + s_num_input, + s_counter, + s_last_remain, + topk_remaining, + g_num_input, + s_num_input_idx=r_idx ^ 1, + ) + else: + self.prefix_sum_and_find_threshold_fine_grained( + tidx, + s_histogram, + s_warp_sums, + num_warps, + s_threshold_bin_id, + s_num_input, + s_counter, + s_last_remain, + topk_remaining, + g_num_input, + s_num_input_idx=r_idx ^ 1, + ) threshold = s_threshold_bin_id[0] if threshold > 0: - topk_remaining -= s_histogram[threshold - 1] + if cutlass.const_expr(self.single_pass_multi_cta): + if need_cluster_sync: + topk_remaining -= s_hist_merged[threshold - 1] + else: + topk_remaining -= s_histogram[threshold - 1] + else: + topk_remaining -= s_histogram[threshold - 1] offset = self.first_refine_shift - round * 8 is_last_round = round == self.num_refine_rounds - 1 - if topk_remaining == 0: - for i in range(tidx, num_input, self.num_threads_per_cta): - idx = s_input_idx[r_idx, i] - idx = cutlass.Int32(cutlass.Uint32(idx)) - bin_val = (self.to_ordered(score[idx]) >> offset) & 0xFF - if bin_val < threshold: - pos = atomicAdd(s_counter.iterator, val_one) - s_indices[pos] = self.index_type(idx) - if cutlass.const_expr(self.enable_gmem_store): - for i in range( + if cutlass.const_expr(self.enable_reread_always): + run_next_round, chain_mask, chain_prefix = self._reread_gmem_rescan( + topk_remaining, + is_last_round, + tidx, + threshold_bin, + threshold, + offset, + chain_mask, + chain_prefix, + score, + s_counter, + s_indices, + s_last_remain, + s_histogram, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + row_start, + prologue_elems, + left_start, + left_size, + ) + elif cutlass.const_expr(self.enable_reread): + if did_overflow: + # Overflow fallback: REREAD_ALWAYS-style GMEM re-scan. + run_next_round, chain_mask, chain_prefix = self._reread_gmem_rescan( + topk_remaining, + is_last_round, tidx, - cur_g_num_input, - self.num_threads_per_cta, - ): - idx = buffer[r_idx, i] - bin_val = (self.to_ordered(score[idx]) >> offset) & 0xFF - if bin_val < threshold: - pos = atomicAdd(s_counter.iterator, val_one) - s_indices[pos] = self.index_type(idx) - cute.arch.barrier() - # break - run_next_round = False + threshold_bin, + threshold, + offset, + chain_mask, + chain_prefix, + score, + s_counter, + s_indices, + s_last_remain, + s_histogram, + _copy_atom, + scan_frag, + _aligned_base, + vec_start, + aligned_size, + row_start, + prologue_elems, + left_start, + left_size, + ) + else: + # No overflow: SMEM-based refinement (same as GMEM_SPILL). + num_input = min( + s_num_input[r_idx], self.filtered_topk_smem_input_size + ) + cur_g_num_input = cutlass.Int32(0) + if topk_remaining == 0: + self._collect_below_threshold_refine( + tidx, + threshold, + offset, + num_input, + r_idx, + s_input_idx, + s_input_val, + score, + s_counter, + s_indices, + cur_g_num_input, + None, + ) + run_next_round = False + else: + self._filter_and_histogram_refine( + tidx, + threshold, + offset, + r_idx, + is_last_round, + num_input, + cur_g_num_input, + score, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + s_last_remain, + None, + None, + ) else: - # Reset histogram - cute.arch.barrier() - if tidx < self.radix + 1: - s_histogram[tidx] = 0 - cute.arch.barrier() - - for i in range(tidx, num_input, self.num_threads_per_cta): - idx = s_input_idx[r_idx, i] - idx_int32 = cutlass.Int32(cutlass.Uint32(idx)) - raw_input = score[idx_int32] - idx = self.index_type(idx_int32) - bin_val = (self.to_ordered(raw_input) >> offset) & 0xFF - if bin_val < threshold: - pos = atomicAdd(s_counter.iterator, val_one) - s_indices[pos] = idx - elif bin_val == threshold: - if is_last_round: - cur_pos = atomicAdd( - s_last_remain.iterator, - val_one_negative, - ) - if cur_pos > 0: - s_indices[self.top_k - cur_pos] = idx - else: - # pos = atomicAdd(s_num_input[r_idx ^ 1], 1) - cur_pos = atomicAdd( - s_num_input.iterator + (r_idx ^ 1), - val_one, - ) - # TODO: remove this if logic for gmem store? - # num_input < filter_topk_smem_input_size - if cutlass.const_expr(self.enable_gmem_store): - if cur_pos < self.filtered_topk_smem_input_size: - s_input_idx[r_idx ^ 1, cur_pos] = idx - else: - buffer_pos = atomicAdd( - g_num_input.iterator + (r_idx ^ 1), - val_one, - ) - buffer[r_idx ^ 1, buffer_pos] = idx_int32 - bin32 = self.to_ordered(raw_input) - sub_bin = (bin32 >> (offset - 8)) & 0xFF - # atomicAdd(s_histogram[sub_bin], 1) - atomicAdd( - s_histogram.iterator + cutlass.Int32(sub_bin), - val_one, - ) - else: - # TODO: how to handle the type of sub_bin and bin32? - if cutlass.const_expr(self.dtype == cutlass.Float32): - bin32 = cutlass.Uint32(0) - sub_bin = cutlass.Uint32(0) - else: - bin32 = cutlass.Uint16(0) - sub_bin = cutlass.Int32(0) - if cur_pos < self.filtered_topk_smem_input_size: - s_input_idx[r_idx ^ 1, cur_pos] = idx - bin32 = self.to_ordered(raw_input) - sub_bin = (bin32 >> (offset - 8)) & 0xFF - # atomicAdd(s_histogram[sub_bin], 1) - atomicAdd( - s_histogram.iterator + cutlass.Int32(sub_bin), - val_one, - ) - - cute.arch.barrier() + num_input = min(s_num_input[r_idx], self.filtered_topk_smem_input_size) + cur_g_num_input = cutlass.Int32(0) if cutlass.const_expr(self.enable_gmem_store): - for i in range( + cur_g_num_input = g_num_input[r_idx] + + if topk_remaining == 0: + self._collect_below_threshold_refine( tidx, + threshold, + offset, + num_input, + r_idx, + s_input_idx, + s_input_val, + score, + s_counter, + s_indices, cur_g_num_input, - self.num_threads_per_cta, - ): - # int32 - idx = buffer[r_idx, i] - raw_input = score[idx] - bin_val = (self.to_ordered(raw_input) >> offset) & 0xFF - if bin_val < threshold: - pos = atomicAdd( - s_counter.iterator, - val_one, - ) - s_indices[pos] = self.index_type(idx) - elif bin_val == threshold: - if is_last_round: - cur_pos = atomicAdd( - s_last_remain.iterator, - val_one_negative, - ) - if cur_pos > 0: - s_indices[self.top_k - cur_pos] = self.index_type( - idx - ) - else: - # pos = atomicAdd(s_num_input[r_idx ^ 1], 1) - cur_pos = atomicAdd( - s_num_input.iterator + (r_idx ^ 1), - val_one, - ) - if cutlass.const_expr(self.enable_gmem_store): - if cur_pos < self.filtered_topk_smem_input_size: - s_input_idx[r_idx ^ 1, cur_pos] = ( - self.index_type(idx) - ) - else: - buffer_pos = atomicAdd( - g_num_input.iterator + (r_idx ^ 1), - val_one, - ) - buffer[r_idx ^ 1, buffer_pos] = idx - bin32 = self.to_ordered(raw_input) - sub_bin = (bin32 >> (offset - 8)) & 0xFF - # atomicAdd(s_histogram[sub_bin], 1) - atomicAdd( - s_histogram.iterator + cutlass.Int32(sub_bin), - val_one, - ) - else: - if cur_pos < self.filtered_topk_smem_input_size: - s_input_idx[r_idx ^ 1, cur_pos] = idx - bin32 = self.to_ordered(raw_input) - sub_bin = (bin32 >> (offset - 8)) & 0xFF - # atomicAdd(s_histogram[sub_bin], 1) - atomicAdd( - s_histogram.iterator - + cutlass.Int32(sub_bin), - val_one, - ) - fence_acq_rel_cta() - cute.arch.barrier() - - # Phase 3: Output phase - vecsize_out = cutlass.const_expr( - min( - self.top_k, - cute.ceil_div(self.top_k, self.num_threads_per_cta), - self.num_copy_bits // self.dtype.width, - # TODO: only tested for float32. need to check for other dtypes. - 2, - ) - ) - assert self.top_k % vecsize_out == 0 - - nvec_per_thread = cutlass.const_expr( - cute.ceil_div(self.top_k, vecsize_out * self.num_threads_per_cta) - ) - topk_vals = cute.make_fragment((vecsize_out, nvec_per_thread), self.dtype) - topk_indices = cute.make_fragment((vecsize_out, nvec_per_thread), cutlass.Int32) - - stride = self.num_threads_per_cta * vecsize_out - for i in cutlass.range(nvec_per_thread, unroll_full=True): - idx = i * stride + tidx % self.num_threads_per_cta * vecsize_out - if idx < self.top_k: - for v in cutlass.range(vecsize_out, unroll_full=True): - index_raw = s_indices[idx + v] - index = cutlass.Int32(cutlass.Uint32(index_raw)) - if cutlass.const_expr(self.return_val): - topk_vals[v, i] = score[index] - if cutlass.const_expr(self.merge_blocks): - topk_indices[v, i] = indices[index] - else: - topk_indices[v, i] = index - # [atom, rest_vec] - mIndices_store = cute.tiled_divide(dst, (vecsize_out,)) - if cutlass.const_expr(self.return_val): - mValues_store = cute.tiled_divide(dst_values, (vecsize_out,)) - # i represents the index of the vector in the output. - for i in cutlass.range(cute.size(topk_vals.shape, [1]), unroll_full=True): - col = i * self.num_threads_per_cta + tidx % self.num_threads_per_cta - if col < self.top_k // vecsize_out: - cute.autovec_copy(topk_indices[None, i], mIndices_store[None, col]) - if cutlass.const_expr(self.return_val): - cute.autovec_copy(topk_vals[None, i], mValues_store[None, col]) - - def _get_tiled_copy(self): - threads_per_row = self.num_threads_per_cta - tiler_mn = ( - 1, - self.vec_size * threads_per_row, - ) - - copy_atom = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - self.dtype, - num_bits_per_copy=self.num_copy_bits, - ) - - thr_layout = cute.make_ordered_layout( - (1, threads_per_row), - order=(1, 0), - ) - val_layout = cute.make_layout((1, self.vec_size)) - tiled_copy = cute.make_tiled_copy_tv(copy_atom, thr_layout, val_layout) - - return ( - copy_atom, - tiled_copy, - tiler_mn, - ) - - @cute.jit - def predicate_tile(self, tAcA: cute.Tensor, limit: cutlass.Int32) -> cute.Tensor: - tApA = cute.make_fragment( - cute.make_layout( - ( - cute.size(tAcA, mode=[0, 1]), - cute.size(tAcA, mode=[1]), - cute.size(tAcA, mode=[2]), - ), - stride=(cute.size(tAcA, mode=[2]), 0, 1), - ), - cutlass.Boolean, - ) - for rest_v in range(tApA.shape[0]): - for rest_k in range(tApA.shape[2]): - tApA[rest_v, 0, rest_k] = cute.elem_less(tAcA[(0, rest_v), 0, rest_k][1], limit) - return tApA - - @cute.jit - def _fill_oob(self, tXrX: cute.Tensor, tXpX: cute.Tensor, fill_value: cute.Numeric) -> None: - """Fill out-of-bounds values in register tensor. + buffer, + ) + run_next_round = False + else: + self._filter_and_histogram_refine( + tidx, + threshold, + offset, + r_idx, + is_last_round, + num_input, + cur_g_num_input, + score, + s_counter, + s_indices, + s_input_idx, + s_input_val, + s_num_input, + s_histogram, + s_last_remain, + g_num_input, + buffer, + ) - Args: - tXrX: Register tensor to fill - tXpX: Predicate tensor indicating valid elements - fill_value: Value to fill OOB locations with - """ - tXrX_fill = cute.make_fragment_like(tXrX[(None, 0), None, 0]) - tXrX_fill.fill(fill_value) - for rest_v in range(tXrX.shape[0][1]): - for rest_k in range(tXrX.shape[2]): - if cutlass.const_expr(tXpX is not None): - if not tXpX[0, rest_v, rest_k]: - cute.autovec_copy(tXrX_fill, tXrX[(None, rest_v), None, rest_k]) + # Phase 3: Output phase. + # SP multi-CTA cluster: collect via DSMEM prefix scan (each CTA + # writes only its slice). Solo / single-CTA: full-row writeback. + if cutlass.const_expr(self.single_pass_multi_cta): + if need_cluster_sync: + self._cluster_collect( + tidx, + s_indices, + s_counter, + s_last_remain, + s_histogram, + cta_in_group, + topk_remaining, + dst, + score, + dst_values, + ) + else: + self._phase3_writeback( + tidx, row_start, s_indices, score, indices, dst, dst_values + ) + else: + self._phase3_writeback(tidx, row_start, s_indices, score, indices, dst, dst_values) def create_random_logits( diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py index 72849ee9adc6..1fe0dfa5eddc 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py @@ -26,11 +26,13 @@ cluster_size: 1 (default), 2, 4 (B200 GPC limit caps at ~16). """ +import math from dataclasses import dataclass from typing import Optional import cutlass import cutlass.cute as cute +import cutlass.cute.math as cmath from cutlass._mlir.dialects import llvm from cutlass.cutlass_dsl import T, dsl_user_op from cutlass.utils.distributed import atomicAdd @@ -127,6 +129,25 @@ def float_as_uint32(float_val): return llvm.bitcast(cutlass.Uint32.mlir_type, float_val.ir_value()) +def float_as_int32(float_val): + """Interpret FP32 value as int32 bit pattern (cuTe DSL bit-cast).""" + return cutlass.Int32(llvm.bitcast(cutlass.Int32.mlir_type, float_val.ir_value())) + + +def f32_order_key(float_val): + """Order-preserving fp32 -> int32 key (unsigned-monotonic bit pattern). + + ``s ^ ((s >> 31) | 0x80000000)``: positive floats map to + ``bits | 0x80000000``, negative floats to ``~bits`` — the standard radix + transform whose UNSIGNED order equals fp32 order (NaN-free inputs). The + returned Int32 must only be consumed digit-wise (``(k >> s) & 0xFF``) or + via equality / prefix-equality; for a full ordered compare, flip the top + bit first (``k ^ 0x80000000`` is signed-monotonic). + """ + s = float_as_int32(float_val) + return s ^ ((s >> cutlass.Int32(31)) | cutlass.Int32(-2147483648)) + + def _fmin_f32_inline(a, b): """Single PTX ``min.f32`` → one SASS FMNMX. @@ -204,9 +225,13 @@ class GvrTopKKernel: Algorithm phases: P1: preIdx Min/Max/Mean → initial threshold - P2: Secant threshold search loop (count-only) + P1b: 256-bin histogram over prev-topK gathered values → M rung + thresholds (enable_r0 only) + P2: threshold admission — default (enable_r0=True) is a single-pass + multi-threshold rung-ladder; enable_r0=False keeps the classic + secant threshold search loop (count-only), also the R0-miss fallback P3: Ballot-free candidate collect into smem keys[]/vals[] - P4: Histogram snap (cand → exact top-K) + writeback + P4: rank-and-scatter (enable_r0) / histogram snap → exact top-K + writeback For different compress_ratio: cr = 1: preIdxOffset = (row_idx % next_n) + 1. V3.2 decode +1 temporal shift. @@ -231,7 +256,30 @@ def __init__( enable_smem_cache: bool = False, smem_cache_elems: int = 32768, seqlen_sorted: bool = False, + kc_diet: Optional[bool] = None, + enable_r0: bool = True, + r0_qfracs: Optional[tuple] = None, + mt_unroll: int = 4, + p1b_cache: Optional[bool] = None, + fb_fix: bool = True, + fb_alpha: float = 0.2, + r0_vseed: Optional[bool] = None, + enable_p4_rank_scatter: Optional[bool] = None, + enable_p4_rank_scatter_exact: Optional[bool] = None, + p4_exact_tail: Optional[bool] = None, + p4_tail_fast: Optional[bool] = None, # [p4tt] + p4_warp_redundant: bool = True, + p2_warp_redundant: bool = True, ): + # Redundant-warp sync reduction: every warp replays the block + # reduce + decision from the same staged SMEM partials in the + # same fp32 order, so results are bit-identical across warps and + # the publish barrier + leader serialization disappear. + # p4_warp_redundant: P4 k-th bin search + snap loop (1 barrier/iter). + # p2_warp_redundant: P2 secant cadence (cluster_size == 1 only). + # Both default ON; OFF restores the leader-based paths (A/B). + self.p4_warp_redundant = p4_warp_redundant + self.p2_warp_redundant = p2_warp_redundant # cluster_size: number of CTAs cooperating per row. 1 = single-CTA # path; 2/4 = thread-block cluster with DSMEM aggregation. Capped at # 16 by B200's per-GPC SM count. @@ -342,6 +390,188 @@ def __init__( self.FLT_MAX = 3.4028235e38 self.NEG_FLT_MAX = -self.FLT_MAX + # --- op#26 R0 histogram-ladder admission (default ON) --- + # enable_r0: replace the Phase-2 secant search with a single-pass + # multi-threshold "rung ladder" admission seeded by a 256-bin + # histogram over the prev-topK gathered values (P1b). + # DEFAULT True: validated on real DSv4/V3.2 decode-capture + # workloads (25-cell seq-len scan) where R0 wins 24/25 vs the + # secant baseline, geomean 1.33x (pro 128k 2.10x). Correctness is + # value-set-exact vs torch.topk (186/186 across dtype/K/N/BS/cluster + # + tie plateaus). The secant path is retained verbatim and remains + # reachable via enable_r0=False; it is the exact fallback for the + # large-N / cold-hint (low preIdx hit-rate) regime where R0 can + # regress on the synthetic worst axis — a follow-up PR adds a + # data-driven dispatch guard to route between the two. All R0 fields + # are const-foldable, so an enable_r0=False kernel is byte-identical + # to the pre-R0 upstream base. + # r0_qfracs: descending h-space quantile fractions defining the M + # candidate rungs (ascending threshold values); None => no rungs. + # r0_vseed: park P1's pmean (the secant init probe) as one extra + # "virtual seed" rung column in the M-ary count pass (no extra + # memory traffic or sync; the column reuses the secant per-thread + # count buffer, so SMEM does not grow). Adapts the admission + # ladder to the row's value distribution: fixes the fat-admission + # regime (a coarse quantile rung admitting ~kC candidates where + # pmean admits ~K) and donates a measured interior bracket point + # to the fallback refine on a full miss. None => enable_r0. + # mt_unroll: 4-way unroll factor for block_count_ge_multi. + # p1b_cache: stash the K gathered preIdx values in SMEM so P1b skips + # a second GMEM random gather (dtype-gated in a later commit). + # fb_fix: R0-miss fallback re-measures the rung bracket ends before + # refining (excludes the R2-class unmeasured-seed failure mode). + self.enable_r0 = bool(enable_r0) + self.mt_unroll = int(mt_unroll) + self.fb_fix = bool(fb_fix) + # C7 dispatch (op#26 host policy folded into the ctor; all gated on + # enable_r0 so an OFF kernel is byte-identical to the base): + # - qfracs default = M2D (0.85, 0.35): dispatch_r0_op26 ships M2D for + # every (dtype, K, N); the M=2 pass is ~free and the R1 falsi shot + # covers the 3-7% bracket misses. uh4 (M=4) was silicon-falsified + # (mc geomean 0.956 — admission != latency). + # - p1b_cache default is cs-aware: + # * cs>1 (cluster): ON for ALL dtypes. The SMEM gather-cache win + # holds and the fp32 occupancy regression that hurts the + # single-CTA path does NOT reproduce in the cluster kernel + # (latency-bound, different SMEM budget). nsys cs=4: K1024 + # ~1.01x / K2048 ~1.02x / K512 wash, 0 losses, exact. Matches + # op26 dispatch_p1bc_mc (unconditional ON). + # * cs=1 (single-CTA): (dtype != fp32). The gather-cache wins + # +0.8-2.8% on 16-bit (random half-prec gather is the cost) but + # is flat/negative on fp32 (occupancy at kC=6144), so OFF there. + # - kC-diet: K512 single-CTA -> kC=3072 (saves 16KB SMEM; 16-bit win, + # fp32 neutral). kC>=2560 is the K512 16-bit tie-safety contract so + # 3072 is safe; the cluster port and K1024/K2048 stay stock. + if r0_vseed is None: + r0_vseed = enable_r0 + if enable_r0 and r0_qfracs is None: + # Per-K default (2026-07-16 vseed full-envelope audit, 2772 + # cells): with the virtual seed rung on, pmean covers q.35's + # admission region for K512/K1024 (2 count columns = zero + # column tax); K2048 keeps q.35 (kC/K = 2.5 makes a fat admit + # costlier than a slim 2-pass miss). Without vseed, q.35 must + # stay for all K (it is the only slim rung). + # K2048 low rung 0.85 -> 0.6 (2026-07-19 real-content rung + # recalibration + paired nsys cold-L2 A/B, B200): the shipped + # 0.85 rung's admission straddles [K, kC] on real V3.2 decode + # captures (bracket on 86% of steps -> one extra falsi pass); + # 0.6 lands the first pass. Measured: real V3.2 geomean + # +2.2-2.8% across fp32/bf16/fp16 and the full BS grid (8K + # rung +10-13% at every BS, no loser cell), favorable + # synthetic +9-11%, adverse synthetic wash, exact everywhere. + # K512/K1024 unchanged: moving or widening their ladder + # measured wash-to-loss (the extra count column costs 3-7%). + if top_k == 2048: + r0_qfracs = (0.6, 0.35) if r0_vseed else (0.85, 0.35) + else: + r0_qfracs = (0.85,) if r0_vseed else (0.85, 0.35) + if enable_r0 and p1b_cache is None: + if cluster_size > 1: + p1b_cache = True + else: + p1b_cache = dtype != cutlass.Float32 + self.p1b_cache = bool(p1b_cache) + # kc_diet: None → diet iff single-CTA (tuned default). The LB hybrid + # kernel passes False for BOTH member instances so their SMEM layouts + # stay byte-identical (the DSL sizes the launch from the last-traced + # SmemAllocator only; see GvrTopKLBKernel). + if kc_diet is None: + kc_diet = cluster_size == 1 + if enable_r0 and top_k == 512 and kc_diet and self.kC > 3072: + self.kC = 3072 + # K2048 R0 Phase-4 histogram diet: 2048 -> 512 bins (2026-07-19 + # paired nsys cold-L2 A/B on B200, all cells exact). The P4 zero / + # atomic build / serial scan all shrink 4x; the deeper boundary-bin + # recursion costs less than the saved passes at kC=6144 candidates. + # Measured vs this head: real V3.2 decode captures geomean +6.1% + # (fp32) / +10.9% (bf16) / +6.3% (fp16); favorable synthetic + # +5.2-11.0%, adverse synthetic +5.1-10.6%; no losing cell + # (fp32 min 0.994, bf16 min 1.035, fp16 min 0.999). Gated on + # enable_r0 so the retained secant path (which shares GvrParams + # and its own P4 histogram) stays byte-identical. P1b reuses this + # buffer and needs >= 256 bins, so 512 is safe. K512/K1024 + # measured as a wash under the same protocol and stay stock. + if enable_r0 and top_k == 2048 and self.kNumBins > 512: + self.kNumBins = 512 + self.r0_qfracs = tuple(float(q) for q in r0_qfracs) if r0_qfracs else () + if self.r0_qfracs: + assert all(0.0 < q < 1.0 for q in self.r0_qfracs), self.r0_qfracs + assert list(self.r0_qfracs) == sorted(self.r0_qfracs, reverse=True), ( + "r0_qfracs must be descending h (ascending threshold value)" + ) + self.M_thr = len(self.r0_qfracs) + # --- vseed (2026-07-16): fold P1's pmean (the secant init + # probe) into the M-ary R0 count pass as one extra "virtual rung". + # Fixes the flash-1M fat-admission regression (the coarse q.85 rung + # admits ~4400 candidates where pmean admits ~630 -> 7x P3/P4 cand + # cost) and, on a true miss, donates a measured interior bracket + # point to the fallback refine. Const-folded: r0_vseed=False kernels + # are byte-identical to before. M_qf = rungs P1b places from qneeds; + # M_thr = total columns counted/admitted (M_qf + 1 when vseed). + self.r0_vseed = bool(r0_vseed) and bool(enable_r0) and self.M_thr > 0 + self.M_qf = self.M_thr + if self.r0_vseed: + self.M_thr = self.M_qf + 1 + # need[m] = ceil(q_m * K) prev-topK values >= rung m. + self.qneeds = tuple(max(1, int(math.ceil(q * self.top_k))) for q in self.r0_qfracs) + # R1 inline shot aim in log2-count space: geometric center of the + # [K, kC] acceptance window. + self.log2_r1aim = math.log2(math.sqrt(self.top_k * self.kC)) if self.r0_qfracs else 0.0 + # fb_fix interior aim (HLS grid optimum): log2(K * (kC/K)**fb_alpha). + self.log2_mstar = ( + math.log2(self.top_k * (self.kC / self.top_k) ** float(fb_alpha)) + if self.r0_qfracs + else 0.0 + ) + + # --- op#7 P4 fused rank-and-scatter (inert until enable_p4_rank_scatter) --- + # Replaces phase4_histogram_snap's k-th-bin search + 2-pass writeback + # with a single rank-and-scatter pass (op#7 PR#15709), cutting Phase-4 + # barriers ~14 -> ~7. On a latency-bound kernel that is a whole-kernel + # win (~1.078x, HW-invariant). enable_p4_rank_scatter_exact adds ONE + # fine-histogram recursion on the straddling coarse bin so the result is + # bit-exact vs torch.topk (adds a few barriers back but still < snap). + # Default ON with R0: nsys over the op22 4k-1M BS=1 best/worst envelope + # gives geomean ~1.09x (K1024 1.12 / K2048 1.12 / K512 1.05) with NO + # cell regressing >2%. Resolves to OFF when enable_r0 is False, so the + # base kernel stays byte-identical to upstream. + if enable_p4_rank_scatter is None: + enable_p4_rank_scatter = bool(enable_r0) + if enable_p4_rank_scatter_exact is None: + enable_p4_rank_scatter_exact = bool(enable_p4_rank_scatter) + self.enable_p4_rank_scatter = bool(enable_p4_rank_scatter) + self.enable_p4_rank_scatter_exact = bool(enable_p4_rank_scatter_exact) + # p4_exact_tail: ambiguity-gated exact tie-resolution for the fine + # straddling bin (fp32 inputs only; see phase4_rank_scatter). The + # fine recursion resolves values to range/(kNumBins*256); two fp32 + # values closer than that straddling the kK boundary inside one fine + # bin were previously picked in arrival order (observed as |miss|=1 + # with |dv| ~ 3e-6 on real Pro 512k-ISL captures). Default ON for + # fp32 rank-scatter-exact kernels; 16-bit inputs keep the arrival + # fill (their upconverted keys are already fully resolved by the + # two-level histogram, and 16-bit tie plateaus are bitwise-equal, + # where arrival order is value-exact). + if p4_exact_tail is None: + p4_exact_tail = self.enable_p4_rank_scatter_exact and dtype == cutlass.Float32 + self.p4_exact_tail = bool(p4_exact_tail) and self.enable_p4_rank_scatter_exact + # [p4tt] p4_tail_fast: tiny-tie COLLECT+SELECT fast path inside the + # exact-tail fire branch. When the (b*, sb*) tie class holds <= 128 + # entries (the real firing cells have 2), ONE candidate pass collects + # (value_bits, cand_idx) pairs into SMEM and thread0 selects the + # top-need exactly, replacing the 4 unconditional radix passes + # (~5.3us -> ~1 pass on pro/512k). Larger tie classes fall through to + # the existing radix select. Pure optimization (the radix backstop + # keeps exactness identical either way); False compiles the original + # text (byte-identical PTX modulo kernel name) for A/B. + # Default gate = p4_exact_tail AND top_k >= 1024: the non-firing + # codegen tax concentrates at K512 cs=1 mid-N (flash 64k/128k + # -6.6/-9.1%, cross-GPU reproducible, 2026-07-20 b200-035) while the + # fire census (pro/512k bench + 9 per-layer fixture cells) contains + # NO K512 cell — so K512 keeps the original byte-identical kernel. + if p4_tail_fast is None: # [p4tt] + p4_tail_fast = self.p4_exact_tail and top_k >= 1024 + self.p4_tail_fast = bool(p4_tail_fast) and self.p4_exact_tail # [p4tt] + # ------------------------------------------------------------------ # SMEM slice cache loader. Streams this CTA's slice GMEM → SMEM via # LDG → STS so Phase 2/3 can read LDS instead of re-streaming GMEM. @@ -492,6 +722,38 @@ def warp_reduce_max_f32(self, val): # PTX redux.sync.fmax.f32 (sm_100). return cute.arch.warp_redux_sync(val, "fmax") + # ------------------------------------------------------------------ + # Raw-address SMEM scalar access through a pre-hoisted window base. + # + # Tensor-indexed SMEM access (smem_keys[i]) makes the compiler + # re-derive the cluster SMEM window per access (S2R SR_CgaCtaId + + # LEA<<24) — ncu shows this as the top single-instruction stall in + # the P3 stream-write and P4 snap loops. Hoisting the base once via + # iterator.toint() (one S2R per call site) turns every subsequent + # access into plain integer addressing — the same pattern the P2 + # scan loops already use for smem_input, whose SASS regions show no + # S2R at all. + # ------------------------------------------------------------------ + @cute.jit + def _smem_ref(self, dtype: cutlass.Constexpr, base_addr, idx): + elem_bytes = cutlass.const_expr(dtype.width // 8) + p = cute.make_ptr( + dtype, + base_addr + cutlass.Int64(idx) * cutlass.Int64(elem_bytes), + cute.AddressSpace.smem, + assumed_align=4, + ) + return cute.make_tensor(p, cute.make_layout((1,))) + + @cute.jit + def _smem_ld(self, dtype: cutlass.Constexpr, base_addr, idx): + return self._smem_ref(dtype, base_addr, idx)[0] + + @cute.jit + def _smem_st(self, dtype: cutlass.Constexpr, base_addr, idx, val): + t = self._smem_ref(dtype, base_addr, idx) + t[0] = val + # ------------------------------------------------------------------ # Phase 1: preIdx Min/Max/Mean -> initial threshold # ------------------------------------------------------------------ @@ -512,6 +774,10 @@ def phase1_preidx_stats( tidx, warp_id, lane, + smem_gath=None, # cute.Tensor [top_k] f32 or None (p1b_cache): stash + # the gathered value per preIdx slot so P1b skips a 2nd GMEM gather. + s_mt_thr=None, # r0_vseed: P1 also parks pmean in the last rung + # column (visibility via P1's own trailing barrier -> zero extra sync). ): """preIdx scan + warp reduce + block aggregate + initial threshold. @@ -536,8 +802,12 @@ def phase1_preidx_stats( i = tidx + cutlass.Int32(u * self.num_threads) raw = pre_idx_row[i] idx = raw + pre_idx_offset + if cutlass.const_expr(smem_gath is not None): + smem_gath[i] = cutlass.Float32(self.NEG_FLT_MAX) if idx >= 0 and idx < N: v = self._load_fp32(input_row, idx) + if cutlass.const_expr(smem_gath is not None): + smem_gath[i] = v local_max = cute.arch.fmax(local_max, v) local_min = _fmin_f32_inline(local_min, v) local_sum = local_sum + v @@ -551,8 +821,12 @@ def phase1_preidx_stats( idx = cutlass.Int32(-1) if tidx < cutlass.Int32(pre_idx_count): idx = pre_idx_row[tidx] + pre_idx_offset + if cutlass.const_expr(smem_gath is not None): + smem_gath[tidx] = cutlass.Float32(self.NEG_FLT_MAX) if idx >= 0 and idx < N: v = self._load_fp32(input_row, idx) + if cutlass.const_expr(smem_gath is not None): + smem_gath[tidx] = v local_max = cute.arch.fmax(local_max, v) local_min = _fmin_f32_inline(local_min, v) local_sum = local_sum + v @@ -617,6 +891,8 @@ def phase1_preidx_stats( pmean = (pmin + pmax) * cutlass.Float32(0.5) cnt_lo_seed = pre_idx_count + (pre_idx_count >> 2) s_thr[0] = pmean + if cutlass.const_expr(self.r0_vseed): + s_mt_thr[self.M_thr - 1] = pmean s_thr[1] = pmin s_thr[2] = pmax s_iscalars[0] = cutlass.Int32(0) # cand_count @@ -652,6 +928,8 @@ def phase1_preidx_stats( cnt_lo_seed = pre_idx_count + (pre_idx_count >> 2) s_thr[0] = pmean + if cutlass.const_expr(self.r0_vseed): + s_mt_thr[self.M_thr - 1] = pmean s_thr[1] = pmin s_thr[2] = pmax s_iscalars[0] = cutlass.Int32(0) @@ -661,6 +939,173 @@ def phase1_preidx_stats( s_iscalars[4] = cutlass.Int32(0) cute.arch.barrier() + # ------------------------------------------------------------------ + # P1b — 256-bin SMEM histogram over the prev-topK gathered values + # (band [v_lo, v_hi] = P1's pmin/pmax = s_thr[1]/s_thr[2]), then M + # h-space quantile rungs into s_mt_thr (ascending value order). Reuses + # the Phase-4 smem_hist buffer (kNumBins >= 512 >= 256 in every spec; + # Phase 4 re-zeroes it later). Provides the R0 admission placement; it + # is only invoked from the enable_r0 path (added in a follow-up commit), + # so the base kernel is unaffected. + # ------------------------------------------------------------------ + @cute.jit + def phase1b_hspace_rungs( + self, + input_row, + N, + pre_idx_row, + pre_idx_count, + pre_idx_offset, + smem_hist, + s_thr, + s_mt_thr, + tidx, + warp_id, + lane, + ): + M = cutlass.const_expr(self.M_qf) + NB = cutlass.const_expr(256) + SEG = cutlass.const_expr(8) # NB / WARP_SIZE bins per lane + num_threads = cutlass.const_expr(self.num_threads) + + jz = tidx + while jz < cutlass.Int32(NB): + smem_hist[jz] = cutlass.Int32(0) + jz = jz + cutlass.Int32(num_threads) + cute.arch.barrier() + + v_lo = s_thr[1] + v_hi = s_thr[2] + width = (v_hi - v_lo) / cutlass.Float32(NB) # caller guards v_hi > v_lo + inv_w = cutlass.Float32(1.0) / width + + ig = tidx + while ig < cutlass.Int32(pre_idx_count): + idx = pre_idx_row[ig] + pre_idx_offset + if idx >= cutlass.Int32(0) and idx < N: + v = cutlass.Float32(input_row[idx]) + bf = (v - v_lo) * inv_w + b = cutlass.Int32(bf) + if b < cutlass.Int32(0): + b = cutlass.Int32(0) + if b > cutlass.Int32(NB - 1): + b = cutlass.Int32(NB - 1) + atomicAdd(smem_hist.iterator + b, cutlass.Int32(1)) + ig = ig + cutlass.Int32(num_threads) + cute.arch.barrier() + + # Warp-0-parallel rung extraction (a tid0 256-bin serial walk is a + # ~10-15us per-CTA dependency chain). Lane l owns the SEG consecutive + # bins descending from bin NB-1-l*SEG; segment sums -> 5-step shfl_up + # inclusive scan gives each lane the cumulative count of all + # higher-value bins; each lane then walks its SEG bins once and fires + # rung m at the unique crossing bin (cum_before < qneeds[m] <= + # cum_at). qfracs descending in h => thresholds ascending in m. + if warp_id == cutlass.Int32(0): + top = cutlass.Int32(NB - 1) - lane * cutlass.Int32(SEG) + seg_frag = cute.make_fragment((SEG,), cutlass.Int32) + part = cutlass.Int32(0) + for j in cutlass.range_constexpr(SEG): + v8 = smem_hist[top - cutlass.Int32(j)] + seg_frag[j] = v8 + part = part + v8 + tp = part + for off_i in cutlass.range_constexpr(5): + off_v = cutlass.const_expr(1 << off_i) + other = cute.arch.shuffle_sync_up(tp, off_v, mask_and_clamp=0) + if lane >= cutlass.Int32(off_v): + tp = tp + other + excl = tp - part # cum of all bins above my segment + total = cute.arch.shuffle_sync(tp, cutlass.Int32(self.WARP_SIZE - 1)) + run = cutlass.Int32(0) + for j in cutlass.range_constexpr(SEG): + run = run + seg_frag[j] + cum_at = excl + run + cum_before = cum_at - seg_frag[j] + for m in cutlass.range_constexpr(M): + if cum_at >= cutlass.Int32(self.qneeds[m]) and cum_before < cutlass.Int32( + self.qneeds[m] + ): + s_mt_thr[m] = v_lo + cutlass.Float32(top - cutlass.Int32(j)) * width + # unfired rungs (heavy invalid-preIdx rows: total < need): v_lo + if lane == 0: + for m in cutlass.range_constexpr(M): + if total < cutlass.Int32(self.qneeds[m]): + s_mt_thr[m] = v_lo + cute.arch.barrier() + + # ------------------------------------------------------------------ + # P1b (p1b_cache variant) — build the rung histogram from the SMEM + # gathered values that P1 stashed (smem_gath), skipping P1b's second + # GMEM random gather. Sentinel NEG_FLT_MAX marks invalid/out-of-range + # preIdx slots. Rung extraction is identical to phase1b_hspace_rungs. + # ------------------------------------------------------------------ + @cute.jit + def phase1b_hspace_rungs_cached( + self, pre_idx_count, smem_gath, smem_hist, s_thr, s_mt_thr, tidx, warp_id, lane + ): + M = cutlass.const_expr(self.M_qf) + NB = cutlass.const_expr(256) + SEG = cutlass.const_expr(8) + num_threads = cutlass.const_expr(self.num_threads) + + jz = tidx + while jz < cutlass.Int32(NB): + smem_hist[jz] = cutlass.Int32(0) + jz = jz + cutlass.Int32(num_threads) + cute.arch.barrier() + + v_lo = s_thr[1] + v_hi = s_thr[2] + width = (v_hi - v_lo) / cutlass.Float32(NB) + inv_w = cutlass.Float32(1.0) / width + + ig = tidx + while ig < cutlass.Int32(pre_idx_count): + v = smem_gath[ig] + if v > cutlass.Float32(self.NEG_FLT_MAX): + bf = (v - v_lo) * inv_w + b = cutlass.Int32(bf) + if b < cutlass.Int32(0): + b = cutlass.Int32(0) + if b > cutlass.Int32(NB - 1): + b = cutlass.Int32(NB - 1) + atomicAdd(smem_hist.iterator + b, cutlass.Int32(1)) + ig = ig + cutlass.Int32(num_threads) + cute.arch.barrier() + + if warp_id == cutlass.Int32(0): + top = cutlass.Int32(NB - 1) - lane * cutlass.Int32(SEG) + seg_frag = cute.make_fragment((SEG,), cutlass.Int32) + part = cutlass.Int32(0) + for j in cutlass.range_constexpr(SEG): + v8 = smem_hist[top - cutlass.Int32(j)] + seg_frag[j] = v8 + part = part + v8 + tp = part + for off_i in cutlass.range_constexpr(5): + off_v = cutlass.const_expr(1 << off_i) + other = cute.arch.shuffle_sync_up(tp, off_v, mask_and_clamp=0) + if lane >= cutlass.Int32(off_v): + tp = tp + other + excl = tp - part + total = cute.arch.shuffle_sync(tp, cutlass.Int32(self.WARP_SIZE - 1)) + run = cutlass.Int32(0) + for j in cutlass.range_constexpr(SEG): + run = run + seg_frag[j] + cum_at = excl + run + cum_before = cum_at - seg_frag[j] + for m in cutlass.range_constexpr(M): + if cum_at >= cutlass.Int32(self.qneeds[m]) and cum_before < cutlass.Int32( + self.qneeds[m] + ): + s_mt_thr[m] = v_lo + cutlass.Float32(top - cutlass.Int32(j)) * width + if lane == 0: + for m in cutlass.range_constexpr(M): + if total < cutlass.Int32(self.qneeds[m]): + s_mt_thr[m] = v_lo + cute.arch.barrier() + # ------------------------------------------------------------------ # block_count_ge — GE-count of input vs threshold (shared by P2/P3). # Per-thread strided accumulate → smem_ptcnt[tid] (for P3 prefix sum) @@ -683,17 +1128,31 @@ def block_count_ge( lane, do_cluster_sync, # bool: False = skip DSMEM aggregation (cs=1 / short-row degrade) smem_input=None, # optional SMEM-cached slice (smem_input[i] == input_row[slice_start+i]) + redundant=False, # trace-time: every-warp reduce, return the total + wcnt_off=None, # int32 staging bank offset into smem_wcnt (parity) ): """Count input[i] >= threshold across this CTA's row slice, then DSMEM-aggregate across the cluster. + ``redundant=True`` (p2_warp_redundant, cluster_size == 1 only): + after the staging barrier EVERY warp reduces the warp counts + lane-parallel and the block total RETURNS in a register — + bit-identical across warps — instead of a leader writing + s_iscalars[0] for a barrier-published broadcast. ``wcnt_off`` + parity-banks the smem_wcnt staging so a warp that has moved on + to the next Phase-2 round cannot clobber a slot a slower warp is + still reading (the per-round staging barrier bounds the drift to + one round). + Vectorized scan: each thread loads vec_w elements per iter (128 or 256 bits) over ``input_row[slice_start : slice_end)``; scalar tail handles the remainder. Cluster aggregation (cluster_size > 1): every CTA stages its - slice-local count into ``s_cluster_partial[0]``, syncs the cluster, - then DSMEM-reads every peer's slot and sums into ``s_iscalars[0]``. + slice-local count into ``s_cluster_partial[call & 1]`` (parity + double-buffer; slot 2 is the tid0-private call counter), syncs the + cluster, then DSMEM-reads every peer's slot and sums into + ``s_iscalars[0]``. After this every CTA's ``s_iscalars[0]`` holds the same cluster-wide cand_count, so Phase 2's secant update stays a leader-only scalar op on a value all CTAs agree on. @@ -818,10 +1277,22 @@ def block_count_ge( # Warp reduce + lane-0 write wc = self.warp_reduce_sum_i32(c) + stage_base = cutlass.Int32(0) + if cutlass.const_expr(wcnt_off is not None): + stage_base = wcnt_off if lane == 0: - smem_wcnt[warp_id] = wc + smem_wcnt[stage_base + warp_id] = wc cute.arch.barrier() + if cutlass.const_expr(redundant): + # Every warp reduces the staged counts itself; no leader, no + # publish barrier, no s_iscalars[0] round-trip. + v_r = cutlass.Int32(0) + if lane < cutlass.Int32(self.num_warps): + v_r = smem_wcnt[stage_base + lane] + total_r = self.warp_reduce_sum_i32(v_r) + return total_r + # Block aggregate (sum reduce over num_warps slots). No trailing # barrier: caller is expected to insert its own __syncthreads after # its post-processing of cand_count. @@ -861,8 +1332,18 @@ def block_count_ge( if cutlass.const_expr(cluster_size > 1): if do_cluster_sync: cute.arch.barrier() # publish s_iscalars[0] to all threads of this CTA + # Parity double-buffer: with a single slot, a straggler's + # post-wait DSMEM read races the peer's next-call overwrite + # (PTX-model data race). Writing call k into slot k&1 orders + # the call-(k+2) overwrite after my call-k reads via the + # call-(k+1) rendezvous. Slot 2 = tid0-private call counter + # (zeroed per row); do_cluster_sync is row-uniform, so CTAs + # step the counter in lockstep and parity stays aligned. + par = cutlass.Int32(0) if tidx == cutlass.Int32(0): - s_cluster_partial[0] = s_iscalars[0] + par = s_cluster_partial[2] + s_cluster_partial[par & cutlass.Int32(1)] = s_iscalars[0] + s_cluster_partial[2] = par + cutlass.Int32(1) # Non-relaxed arrive: pairs with the peer cluster_wait acquire # to release s_cluster_partial writes so the DSMEM ld below # observes them. cluster_arrive_relaxed would skip the release @@ -872,13 +1353,172 @@ def block_count_ge( cute.arch.cluster_wait() if tidx == cutlass.Int32(0): total = cutlass.Int32(0) - local_ptr = s_cluster_partial.iterator + cutlass.Int32(0) + local_ptr = s_cluster_partial.iterator + (par & cutlass.Int32(1)) for peer in cutlass.range_constexpr(cluster_size): peer_addr = mapa_shared_cluster(local_ptr, cutlass.Int32(peer)) total = total + ld_shared_cluster_i32(peer_addr) s_iscalars[0] = total cute.arch.barrier() # broadcast cluster total within this CTA + return cutlass.Int32(0) + + # ------------------------------------------------------------------ + # block_count_ge_multi — GE-count of the input row against M + # thresholds in ONE vectorized scan, reusing block_count_ge's memory + # path (same vec_w / 4-way-unroll / tail loops) with M static register + # counters. Caches all M per-thread count columns in smem_ptcnt_multi so + # the accepted rung's column seeds Phase 3 with zero rescan. This is the + # R0 admission primitive (op#18 multithresh lineage); it is only invoked + # from the enable_r0 path added in a later commit, so the base kernel is + # unaffected. Slice + cluster form: each CTA scans [slice_start, + # slice_end) and the M per-CTA totals are DSMEM all-reduced across the + # cluster (cluster_size>1, do_cluster_sync) with a release cluster_arrive + # mirroring block_count_ge; at cs==1 (or short-row degrade) the local + # totals are the answer. smem_ptcnt_multi holds slice-local per-thread + # columns (the accepted rung's column seeds Phase 3 per CTA). + # ------------------------------------------------------------------ + @cute.jit + def block_count_ge_multi( + self, + input_row, + slice_start, + slice_end, + s_mt_thr, + smem_ptcnt_multi, + smem_wcnt_multi, + s_mt_cnt, + s_cluster_partial_m, + do_cluster_sync, + tidx, + warp_id, + lane, + smem_ptcnt=None, # vseed: last column's per-thread counts land here + ): + M = cutlass.const_expr(self.M_thr) + num_threads = cutlass.const_expr(self.num_threads) + num_warps = cutlass.const_expr(self.num_warps) + cluster_size = cutlass.const_expr(self.cluster_size) + vec_w = cutlass.const_expr(self.vec_bits // self.dtype.width) + elem_bytes = cutlass.const_expr(self.dtype.width // 8) + vec_align = cutlass.const_expr(self.vec_align_bytes) + copy_atom = self._make_load_copy_atom() + step_elem = cutlass.const_expr(num_threads * vec_w) + + thr_frag = cute.make_fragment((M,), cutlass.Float32) + cnt_frag = cute.make_fragment((M,), cutlass.Int32) + for m in cutlass.range_constexpr(M): + thr_frag[m] = s_mt_thr[m] + cnt_frag[m] = cutlass.Int32(0) + + row_addr = input_row.iterator.toint() + slice_len = slice_end - slice_start + n_aligned = slice_start + (slice_len // cutlass.Int32(vec_w)) * cutlass.Int32(vec_w) + i = slice_start + tidx * cutlass.Int32(vec_w) + step = cutlass.Int32(step_elem) + + if self.enable_unroll_4: + rng_frag = cute.make_fragment((vec_w,), self.dtype) + big_iters = cutlass.Int32(0) + if slice_end > i + cutlass.Int32(vec_w - 1): + big_iters = (slice_end - i - cutlass.Int32(vec_w)) // cutlass.Int32( + step_elem + ) + cutlass.Int32(1) + for k in cutlass.range(big_iters, unroll=self.mt_unroll): + i_local = i + k * cutlass.Int32(step_elem) + src_ptr_k = cute.make_ptr( + self.dtype, + row_addr + cutlass.Int64(i_local) * cutlass.Int64(elem_bytes), + cute.AddressSpace.gmem, + assumed_align=vec_align, + ) + src_k = cute.make_tensor(src_ptr_k, cute.make_layout((vec_w,))) + cute.copy(copy_atom, src_k, rng_frag) + for j in cutlass.range_constexpr(vec_w): + if cutlass.const_expr(self.dtype == cutlass.Float32): + vj = rng_frag[j] + else: + vj = cutlass.Float32(rng_frag[j]) + for m in cutlass.range_constexpr(M): + cnt_frag[m] = cnt_frag[m] + cutlass.Int32(vj >= thr_frag[m]) + i = i + big_iters * cutlass.Int32(step_elem) + + tail_frag = cute.make_fragment((vec_w,), self.dtype) + while i + cutlass.Int32(vec_w - 1) < slice_end: + src_ptr = cute.make_ptr( + self.dtype, + row_addr + cutlass.Int64(i) * cutlass.Int64(elem_bytes), + cute.AddressSpace.gmem, + assumed_align=vec_align, + ) + src = cute.make_tensor(src_ptr, cute.make_layout((vec_w,))) + cute.copy(copy_atom, src, tail_frag) + for j in cutlass.range_constexpr(vec_w): + if cutlass.const_expr(self.dtype == cutlass.Float32): + vj = tail_frag[j] + else: + vj = cutlass.Float32(tail_frag[j]) + for m in cutlass.range_constexpr(M): + cnt_frag[m] = cnt_frag[m] + cutlass.Int32(vj >= thr_frag[m]) + i = i + step + + it = n_aligned + tidx + while it < slice_end: + v = self._load_fp32(input_row, it) + for m in cutlass.range_constexpr(M): + cnt_frag[m] = cnt_frag[m] + cutlass.Int32(v >= thr_frag[m]) + it = it + cutlass.Int32(num_threads) + + for m in cutlass.range_constexpr(M): + if cutlass.const_expr(self.r0_vseed and m == self.M_qf): + smem_ptcnt[tidx] = cnt_frag[m] + else: + smem_ptcnt_multi[m * num_threads + tidx] = cnt_frag[m] + + for m in cutlass.range_constexpr(M): + wc = self.warp_reduce_sum_i32(cnt_frag[m]) + if lane == 0: + smem_wcnt_multi[m * num_warps + warp_id] = wc + cute.arch.barrier() + # Block-reduce the M warp counts to this CTA's slice totals. Stage + # into DSMEM scratch at cs>1 (for the cluster merge below), else + # write straight to s_mt_cnt. + if warp_id == cutlass.Int32(0): + for m in cutlass.range_constexpr(M): + v = cutlass.Int32(0) + if lane < cutlass.Int32(num_warps): + v = smem_wcnt_multi[m * num_warps + lane] + total = self.warp_reduce_sum_i32(v) + if lane == cutlass.Int32(0): + if cutlass.const_expr(cluster_size > 1): + s_cluster_partial_m[m] = total + else: + s_mt_cnt[m] = total + cute.arch.barrier() + if cutlass.const_expr(cluster_size > 1): + if do_cluster_sync: + # Release arrive (NOT relaxed): pairs with the peer + # cluster_wait acquire so the staged M totals are visible + # before any CTA reads them over DSMEM. + cute.arch.cluster_arrive() + cute.arch.cluster_wait() + if tidx == cutlass.Int32(0): + local_ptr = s_cluster_partial_m.iterator + for m in cutlass.range_constexpr(M): + total = cutlass.Int32(0) + for peer in cutlass.range_constexpr(cluster_size): + peer_addr = mapa_shared_cluster( + local_ptr + cutlass.Int32(m), cutlass.Int32(peer) + ) + total = total + ld_shared_cluster_i32(peer_addr) + s_mt_cnt[m] = total + cute.arch.barrier() + else: + # short-row degrade: this CTA's local totals are the answer. + if tidx == cutlass.Int32(0): + for m in cutlass.range_constexpr(M): + s_mt_cnt[m] = s_cluster_partial_m[m] + cute.arch.barrier() + # ------------------------------------------------------------------ # Phase 2: Secant-interpolation threshold search # Refines threshold to bring cand_count into [kK, kCC] using secant @@ -896,7 +1536,7 @@ def phase2_secant_search( smem_wcnt, s_thr, # [threshold, val_lo, val_hi] s_iscalars, # [cand_count, done, cnt_lo, cnt_hi, out_count] - s_cluster_partial, # [1] int32 cluster scratch + s_cluster_partial, # [3] int32 cluster scratch (parity slots + counter) tidx, warp_id, lane, @@ -913,6 +1553,120 @@ def phase2_secant_search( kCC = cutlass.const_expr(self.kC) kFTarget = cutlass.const_expr(self.kFTarget) + if cutlass.const_expr(self.p2_warp_redundant and self.cluster_size == 1): + # ---- Redundant-warp cadence: ONE barrier per round ---- + # The whole secant state (threshold, bracket, counts, done) + # lives in registers; every warp reduces the staged warp + # counts itself (block_count_ge redundant mode) and replays + # the identical classify + secant update, so the per-round + # publish barriers and every s_thr/s_iscalars SMEM round-trip + # (with its per-access cluster-window S2R recompute) + # disappear. Canonical exit state is written once for P3. + nwp2 = cutlass.const_expr(self.num_warps) + thr_r = s_thr[0] + vlo_r = s_thr[1] + vhi_r = s_thr[2] + clo_r = s_iscalars[2] + chi_r = s_iscalars[3] + done_r = cutlass.Int32(0) + par_r = cutlass.Int32(0) + cnt_r = self.block_count_ge( + input_row, + slice_start, + slice_end, + thr_r, + smem_ptcnt, + smem_wcnt, + s_iscalars, + s_cluster_partial, + tidx, + warp_id, + lane, + cutlass.Boolean(False), # do_cluster_sync (cs==1 gate) + smem_input=smem_input, + redundant=True, + wcnt_off=par_r * cutlass.Int32(nwp2), + ) + if cnt_r >= cutlass.Int32(kK) and cnt_r <= cutlass.Int32(kCC): + done_r = cutlass.Int32(1) + elif cnt_r > cutlass.Int32(kCC): + vlo_r = thr_r + clo_r = cnt_r + else: + vhi_r = thr_r + chi_r = cnt_r + it = cutlass.Int32(0) + while it < cutlass.Int32(self.MAX_REFINE_ITERS) and done_r == cutlass.Int32(0): + rng = vhi_r - vlo_r + nv = cutlass.Float32(0.0) + if clo_r > chi_r and rng > cutlass.Float32(1e-10): + f = cutlass.Float32(clo_r - cutlass.Int32(kFTarget)) / cutlass.Float32( + clo_r - chi_r + ) + f = cute.arch.fmax(cutlass.Float32(0.05), f) + f = _fmin_f32_inline(f, cutlass.Float32(0.95)) + if it == cutlass.Int32(0): + f = _fmin_f32_inline(f, cutlass.Float32(0.5)) + nv = vlo_r + rng * f + else: + nv = (vlo_r + vhi_r) * cutlass.Float32(0.5) + if nv <= vlo_r: + nv = vlo_r + rng * cutlass.Float32(0.05) + if nv >= vhi_r: + nv = vhi_r - rng * cutlass.Float32(0.05) + if nv == vlo_r or nv == vhi_r: + nv = (vlo_r + vhi_r) * cutlass.Float32(0.5) + if nv == vlo_r or nv == vhi_r: + thr_r = vlo_r + done_r = cutlass.Int32(2) + if done_r == cutlass.Int32(0): + thr_r = nv + par_r = par_r ^ cutlass.Int32(1) + cnt_r = self.block_count_ge( + input_row, + slice_start, + slice_end, + thr_r, + smem_ptcnt, + smem_wcnt, + s_iscalars, + s_cluster_partial, + tidx, + warp_id, + lane, + cutlass.Boolean(False), # do_cluster_sync (cs==1 gate) + smem_input=smem_input, + redundant=True, + wcnt_off=par_r * cutlass.Int32(nwp2), + ) + if cnt_r >= cutlass.Int32(kK) and cnt_r <= cutlass.Int32(kCC): + done_r = cutlass.Int32(1) + elif cnt_r > cutlass.Int32(kCC): + vlo_r = thr_r + clo_r = cnt_r + else: + vhi_r = thr_r + chi_r = cnt_r + it = it + cutlass.Int32(1) + if done_r == cutlass.Int32(0): + if clo_r <= cutlass.Int32(kCC * 2): + thr_r = vlo_r + else: + thr_r = vhi_r + done_r = cutlass.Int32(2) + # Canonical exit state for Phase 3/4 (byte-compatible with the + # leader path), published once. + if tidx == 0: + s_thr[0] = thr_r + s_thr[1] = vlo_r + s_thr[2] = vhi_r + s_iscalars[0] = cnt_r + s_iscalars[1] = done_r + s_iscalars[2] = clo_r + s_iscalars[3] = chi_r + cute.arch.barrier() + return + # ---- Initial count with the Phase-1 mean as threshold ---- # TODO: smem_ptcnt is not always needed? only for the last block_count_ge. # Do we have methods to reduce its write? @@ -1201,6 +1955,11 @@ def phase3_collect_candidates( copy_atom = self._make_load_copy_atom() row_addr = input_row.iterator.toint() step_elem = cutlass.const_expr(num_threads * vec_w) + # Hoisted SMEM window bases (one S2R here vs one per emitted + # candidate below — this loop is the kernel's biggest instruction + # region at production shapes). + keys_base = smem_keys.iterator.toint() + vals_base = smem_vals.iterator.toint() slice_len = slice_end - slice_start # When reading from the cached slice, scan indices are slice-LOCAL; @@ -1258,8 +2017,10 @@ def phase3_collect_candidates( else: vj = cutlass.Float32(rng_frag[j]) if vj >= thr_final and wc < cutlass.Int32(kCC): - smem_keys[wc] = vj - smem_vals[wc] = global_base + cutlass.Int32(j) + self._smem_st(cutlass.Float32, keys_base, wc, vj) + self._smem_st( + cutlass.Int32, vals_base, wc, global_base + cutlass.Int32(j) + ) wc = wc + cutlass.Int32(1) # Advance ic past all consumed vec_w-aligned positions. ic = ic + big_iters * cutlass.Int32(step_elem) @@ -1291,8 +2052,8 @@ def phase3_collect_candidates( else: vj = cutlass.Float32(tail_frag[j]) if vj >= thr_final and wc < cutlass.Int32(kCC): - smem_keys[wc] = vj - smem_vals[wc] = global_base_t + cutlass.Int32(j) + self._smem_st(cutlass.Float32, keys_base, wc, vj) + self._smem_st(cutlass.Int32, vals_base, wc, global_base_t + cutlass.Int32(j)) wc = wc + cutlass.Int32(1) ic = ic + step @@ -1308,8 +2069,8 @@ def phase3_collect_candidates( v = self._load_fp32(input_row, it) pos_global = it if v >= thr_final and wc < cutlass.Int32(kCC): - smem_keys[wc] = v - smem_vals[wc] = pos_global + self._smem_st(cutlass.Float32, keys_base, wc, v) + self._smem_st(cutlass.Int32, vals_base, wc, pos_global) wc = wc + cutlass.Int32(1) it = it + cutlass.Int32(num_threads) cute.arch.barrier() @@ -1320,7 +2081,7 @@ def phase3_collect_candidates( @cute.jit def block_fused_snap_iter( self, - smem_keys, + keys_base, # hoisted SMEM window base of smem_keys (iterator.toint()) smem_wcnt, smem_hist, # reused as scratch for s_up/s_down warp aggregates s_thr, @@ -1345,7 +2106,7 @@ def block_fused_snap_iter( isi = tidx while isi < count: - v = smem_keys[isi] + v = self._smem_ld(cutlass.Float32, keys_base, isi) if v >= thr: lge = lge + cutlass.Int32(1) if v > thr: @@ -1435,10 +2196,262 @@ def block_fused_snap_iter( cute.arch.barrier() # ------------------------------------------------------------------ - # Phase 4: Histogram-based k-th selection + two-pass writeback + # P4 helpers: histogram build + parallel k-th bin search. Factored + # out so the level-2 refinement can rerun both over a narrowed window. # ------------------------------------------------------------------ @cute.jit - def phase4_histogram_snap( + def _hist_build(self, keys_base, smem_hist, cand_count, lo, inv, tidx): + """Zero smem_hist[0:kBins], then histogram keys[0:cand_count] with + bin = clamp(int((v - lo) * inv), 0, kBins-1). Out-of-window values + clamp into the edge bins, which keeps cumulative counts from the + top exact for the k-th search (everything above the window lands + in the top bin). Barrier after the zero pass and after the build.""" + kBins = cutlass.const_expr(self.kNumBins) + num_threads = cutlass.const_expr(self.num_threads) + i6 = tidx + while i6 < cutlass.Int32(kBins): + smem_hist[i6] = cutlass.Int32(0) + i6 = i6 + cutlass.Int32(num_threads) + cute.arch.barrier() + i7 = tidx + while i7 < cand_count: + vk = self._smem_ld(cutlass.Float32, keys_base, i7) + bin_f = (vk - lo) * inv + # Clamp in the FLOAT domain before the int cast: fptosi is + # undefined for out-of-range/NaN inputs at the IR level (PTX + # cvt.rzi saturates, but LLVM may optimize on the poison). + # fmax first canonicalizes NaN to 0; the pair keeps the + # edge-bin clamping semantics bit-identical for in-range + # values. + bin_f = cute.arch.fmax(bin_f, cutlass.Float32(0.0)) + bin_f = _fmin_f32_inline(bin_f, cutlass.Float32(kBins - 1)) + bin_i = cutlass.Int32(bin_f) + atomicAdd(smem_hist.iterator + bin_i, cutlass.Int32(1)) + i7 = i7 + cutlass.Int32(num_threads) + cute.arch.barrier() + + @cute.jit + def _kth_bin_search( + self, smem_hist, smem_wcnt, s_thr, s_iscalars, lo, binw, tidx, warp_id, lane + ): + """Parallel k-th bin search (3-step, high→low). Writes + s_thr[0] = lower edge of the selected bin (lo + bidx*binw) and + s_iscalars[4] = selected bin's count (gates the level-2 histogram + refinement). Clobbers s_iscalars[2]/[3] as staging (both are + rewritten by the snap loop before anyone else reads them). + Trailing barrier.""" + kK = cutlass.const_expr(self.top_k) + kBins = cutlass.const_expr(self.kNumBins) + bins_per_warp = cutlass.const_expr(kBins // self.num_warps) + + # Step 1: each warp sums BINS_PER_WARP bins (high→low slice). + # Lane-parallel when the slice divides evenly across the warp: + # each lane sums bins_per_warp/32 bins + one warp reduce, instead + # of every lane redundantly walking a bins_per_warp-deep serial + # LDS+IADD dependency chain (~7% of stall samples at N=8K). + warp_bin_sum = cutlass.Int32(0) + if cutlass.const_expr(bins_per_warp % self.WARP_SIZE == 0): + for jm in cutlass.range_constexpr(bins_per_warp // self.WARP_SIZE): + bidx_s = ( + cutlass.Int32(kBins - 1) + - warp_id * cutlass.Int32(bins_per_warp) + - (lane + cutlass.Int32(jm * self.WARP_SIZE)) + ) + warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] + warp_bin_sum = self.warp_reduce_sum_i32(warp_bin_sum) + else: + for jb in cutlass.range_constexpr(bins_per_warp): + bidx_s = ( + cutlass.Int32(kBins - 1) + - warp_id * cutlass.Int32(bins_per_warp) + - cutlass.Int32(jb) + ) + warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] + if lane == 0: + smem_wcnt[warp_id] = warp_bin_sum + cute.arch.barrier() + + # Step 2: tid==0 finds target warp; stores prefix-count + warp index + # into s_iscalars[2] (=cnt_lo: prefix before target warp) + # and s_iscalars[3] (=cnt_hi: target warp index) + if tidx == 0: + cum = cutlass.Int32(0) + tw = cutlass.Int32(self.num_warps - 1) + found = cutlass.Int32(0) + for w2 in cutlass.range_constexpr(self.num_warps): + cum = cum + smem_wcnt[w2] + if cum >= cutlass.Int32(kK) and found == cutlass.Int32(0): + tw = cutlass.Int32(w2) + found = cutlass.Int32(1) + # Recompute prefix BEFORE target warp + cum2 = cutlass.Int32(0) + for w3 in cutlass.range_constexpr(self.num_warps): + if cutlass.Int32(w3) < tw: + cum2 = cum2 + smem_wcnt[w3] + s_iscalars[2] = cum2 # prefix + s_iscalars[3] = tw # target warp index + cute.arch.barrier() + + # Step 3: target warp's lane 0 scans BINS_PER_WARP bins → + # threshold. Single-thread serial; the unrolled + # range_constexpr beats a runtime `for+break` (tried it: -544 + # SASS insts but -7pp fp32 / -14pp bf16, since the + # branch/counter overhead in a single thread dominates the + # static math). + target_warp = s_iscalars[3] + if warp_id == target_warp and lane == cutlass.Int32(0): + base_cum = s_iscalars[2] + thr_local = lo + sel_cnt = cutlass.Int32(0) + set_done = cutlass.Int32(0) + for jb2 in cutlass.range_constexpr(bins_per_warp): + bidx2 = ( + cutlass.Int32(kBins - 1) + - target_warp * cutlass.Int32(bins_per_warp) + - cutlass.Int32(jb2) + ) + cnt_here = smem_hist[bidx2] + base_cum = base_cum + cnt_here + if base_cum >= cutlass.Int32(kK) and set_done == cutlass.Int32(0): + thr_local = lo + cutlass.Float32(bidx2) * binw + sel_cnt = cnt_here + set_done = cutlass.Int32(1) + s_thr[0] = thr_local + s_iscalars[4] = sel_cnt + cute.arch.barrier() + + # ------------------------------------------------------------------ + # _kth_bin_search_rw — redundant-warp variant (p4_warp_redundant). + # Step 1 stages per-warp bin-slice sums exactly like _kth_bin_search + # (the ONE barrier). Then EVERY warp redundantly (a) walks the + # num_warps slot sums with broadcast SMEM reads + predicated adds to + # locate the target warp, and (b) lane-parallel walks the target + # slice — each lane owns a contiguous descending sub-range, a + # shuffle-up prefix + the unique sub-range crossing test find the + # k-th bin in O(bins_per_warp/32) LDS instead of a 64-deep serial + # LDS+IADD chain in one thread. Same inputs in the same order on + # every warp -> bit-identical results, so there is no leader, no + # publish barrier, and no s_thr/s_iscalars staging; the selected + # (threshold, bin count) return in registers. + # ------------------------------------------------------------------ + @cute.jit + def _kth_bin_search_rw(self, smem_hist, smem_wcnt, lo, binw, tidx, warp_id, lane): + kK = cutlass.const_expr(self.top_k) + kBins = cutlass.const_expr(self.kNumBins) + bins_per_warp = cutlass.const_expr(kBins // self.num_warps) + + # Step 1: identical staging to _kth_bin_search. + warp_bin_sum = cutlass.Int32(0) + if cutlass.const_expr(bins_per_warp % self.WARP_SIZE == 0): + for jm in cutlass.range_constexpr(bins_per_warp // self.WARP_SIZE): + bidx_s = ( + cutlass.Int32(kBins - 1) + - warp_id * cutlass.Int32(bins_per_warp) + - (lane + cutlass.Int32(jm * self.WARP_SIZE)) + ) + warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] + warp_bin_sum = self.warp_reduce_sum_i32(warp_bin_sum) + else: + for jb in cutlass.range_constexpr(bins_per_warp): + bidx_s = ( + cutlass.Int32(kBins - 1) + - warp_id * cutlass.Int32(bins_per_warp) + - cutlass.Int32(jb) + ) + warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] + if lane == 0: + smem_wcnt[warp_id] = warp_bin_sum + cute.arch.barrier() + + # Step 2 (every warp, lane-parallel): lane w holds slot w; an + # inclusive idx-shuffle scan + ballot locate the target warp. + # (shuffle_sync with a computed source lane is the working shfl + # idiom; shuffle_sync_up ignores its offset — probed.) + v_s = cutlass.Int32(0) + if lane < cutlass.Int32(self.num_warps): + v_s = smem_wcnt[lane] + run2 = v_s + for d2 in cutlass.range_constexpr(5): + off2 = cutlass.const_expr(1 << d2) + src2 = lane - cutlass.Int32(off2) + if src2 < cutlass.Int32(0): + src2 = cutlass.Int32(0) + up2 = cute.arch.shuffle_sync(run2, src2) + if lane >= cutlass.Int32(off2): + run2 = run2 + up2 + m2 = cute.arch.vote_ballot_sync(run2 >= cutlass.Int32(kK)) + tw = cutlass.Int32(self.num_warps - 1) + if m2 != cutlass.Uint32(0): + low2 = m2 & (cutlass.Uint32(0) - m2) + tw = cutlass.Int32(cute.arch.popc(low2 - cutlass.Uint32(1))) + incl_tw = cute.arch.shuffle_sync(run2, tw) + slot_tw = cute.arch.shuffle_sync(v_s, tw) + prefix = incl_tw - slot_tw + + # Step 3 (every warp, lane-parallel): lane l owns the contiguous + # descending positions [l*ppl, (l+1)*ppl) of the target slice. + ppl = cutlass.const_expr((bins_per_warp + self.WARP_SIZE - 1) // self.WARP_SIZE) + cnt_frag = cute.make_fragment((ppl,), cutlass.Int32) + my_sum = cutlass.Int32(0) + for j3 in cutlass.range_constexpr(ppl): + pos = lane * cutlass.Int32(ppl) + cutlass.Int32(j3) + cnt_j = cutlass.Int32(0) + if pos < cutlass.Int32(bins_per_warp): + bidx3 = cutlass.Int32(kBins - 1) - tw * cutlass.Int32(bins_per_warp) - pos + cnt_j = smem_hist[bidx3] + cnt_frag[j3] = cnt_j + my_sum = my_sum + cnt_j + # Exclusive cross-lane prefix of the lane partial sums via the + # idx-shuffle scan (5 log-steps; shuffle_sync_up ignores its + # offset — probed — so the scan uses computed source lanes). + run3 = my_sum + for d3 in cutlass.range_constexpr(5): + off3 = cutlass.const_expr(1 << d3) + src3 = lane - cutlass.Int32(off3) + if src3 < cutlass.Int32(0): + src3 = cutlass.Int32(0) + up3 = cute.arch.shuffle_sync(run3, src3) + if lane >= cutlass.Int32(off3): + run3 = run3 + up3 + base3 = prefix + (run3 - my_sum) + + # Unique crossing: the lane where the running count passes kK. + thr_loc = lo + sel_loc = cutlass.Int32(0) + hit = cutlass.Int32(0) + r3 = base3 + for j4 in cutlass.range_constexpr(ppl): + pos4 = lane * cutlass.Int32(ppl) + cutlass.Int32(j4) + cnt4 = cnt_frag[j4] + if ( + pos4 < cutlass.Int32(bins_per_warp) + and r3 < cutlass.Int32(kK) + and r3 + cnt4 >= cutlass.Int32(kK) + and hit == cutlass.Int32(0) + ): + bidx4 = cutlass.Int32(kBins - 1) - tw * cutlass.Int32(bins_per_warp) - pos4 + thr_loc = lo + cutlass.Float32(bidx4) * binw + sel_loc = cnt4 + hit = cutlass.Int32(1) + r3 = r3 + cnt4 + # Broadcast from the (at most one) hitting lane; no hit keeps + # (lo, 0) — same fallback as _kth_bin_search's set_done guard. + mask3 = cute.arch.vote_ballot_sync(hit != cutlass.Int32(0)) + thr_out = lo + sel_out = cutlass.Int32(0) + if mask3 != cutlass.Uint32(0): + low = mask3 & (cutlass.Uint32(0) - mask3) + src = cutlass.Int32(cute.arch.popc(low - cutlass.Uint32(1))) + thr_out = cute.arch.shuffle_sync(thr_loc, src) + sel_out = cute.arch.shuffle_sync(sel_loc, src) + return thr_out, sel_out + + # ------------------------------------------------------------------ + # Phase 4 (alt): op#7 fused rank-and-scatter (enable_p4_rank_scatter). + # Ported verbatim from p4_recursive_digit/gvr_topk_decode_p4.py. + # ------------------------------------------------------------------ + @cute.jit + def phase4_rank_scatter( self, smem_keys, smem_vals, @@ -1453,18 +2466,12 @@ def phase4_histogram_snap( warp_id, lane, ): - """Three branches by cand_count vs kK: - == kK: direct emit (fast path) - > kK: histogram k-th bin search → snap → 2-pass writeback - < kK: emit cand_count + pad with -FLT_MAX - """ kK = cutlass.const_expr(self.top_k) kBins = cutlass.const_expr(self.kNumBins) num_threads = cutlass.const_expr(self.num_threads) num_warps = cutlass.const_expr(self.num_warps) bins_per_warp = cutlass.const_expr(kBins // self.num_warps) - # ----- Branch A: cand_count == kK (fast path) ----- if cand_count == cutlass.Int32(kK): i4 = tidx while i4 < cutlass.Int32(kK): @@ -1473,9 +2480,7 @@ def phase4_histogram_snap( output_indices_row[i4] = smem_vals[i4] i4 = i4 + cutlass.Int32(num_threads) elif cand_count > cutlass.Int32(kK): - # ----- Branch B: cand_count > kK → histogram snap ----- - - # Block min/max over keys[0:cand_count] + # ---- block min/max over candidates ---- local_cmin = cutlass.Float32(self.FLT_MAX) local_cmax = cutlass.Float32(self.NEG_FLT_MAX) i5 = tidx @@ -1486,57 +2491,36 @@ def phase4_histogram_snap( i5 = i5 + cutlass.Int32(num_threads) cmin = self.warp_reduce_min_f32(local_cmin) cmax = self.warp_reduce_max_f32(local_cmax) - # Stage warp results into smem_wcnt[w] (cmin) and smem_hist[w] (cmax) - # as bit-cast int32. cmax stored at smem_hist[0..NW-1]. - if lane == 0: + if lane == cutlass.Int32(0): smem_wcnt[warp_id] = float_as_uint32(cmin) smem_hist[warp_id] = float_as_uint32(cmax) cute.arch.barrier() - - # Every thread independently recomputes block_min/block_max - # from the warp-staged smem slots (CUDA heuristic_topk.cuh:891-898 - # pattern). No tid==0 → s_thr broadcast → saves a block barrier. bmin_r = cutlass.Float32(self.FLT_MAX) bmax_r = cutlass.Float32(self.NEG_FLT_MAX) - # Unrolled num_warps times (16 or 32 — fixed at compile time). for w in cutlass.range_constexpr(self.num_warps): - vmin_bits = smem_wcnt[w] - vmax_bits = smem_hist[w] vmin = cutlass.Float32( - llvm.bitcast(cutlass.Float32.mlir_type, vmin_bits.ir_value()) + llvm.bitcast(cutlass.Float32.mlir_type, smem_wcnt[w].ir_value()) ) vmax = cutlass.Float32( - llvm.bitcast(cutlass.Float32.mlir_type, vmax_bits.ir_value()) + llvm.bitcast(cutlass.Float32.mlir_type, smem_hist[w].ir_value()) ) bmin_r = _fmin_f32_inline(bmin_r, vmin) bmax_r = cute.arch.fmax(bmax_r, vmax) if bmax_r <= bmin_r: bmax_r = bmin_r + cutlass.Float32(1e-6) - # Barrier required: smem_hist[0..NW-1] above doubles as cmax - # scratch and below as the histogram. Without this sync the - # zeroing pass below can clobber a cmax slot a later warp is - # still reading → wrong bmax_r → all candidates squashed into - # bin 0 (hit-rate-dependent race). cute.arch.barrier() - - # Zero histogram (must zero ALL slots since smem_hist[0..NW-1] was - # used as cmax scratch above). + # ---- zero + build histogram ---- i6 = tidx while i6 < cutlass.Int32(kBins): smem_hist[i6] = cutlass.Int32(0) i6 = i6 + cutlass.Int32(num_threads) cute.arch.barrier() - range1 = bmax_r - bmin_r - # inv1 = (kBins - 1 + 0.99) / range1 (range1 > 0 guaranteed by 1e-6 patch) inv1 = (cutlass.Float32(kBins - 1) + cutlass.Float32(0.99)) / range1 - - # Build histogram by atomicAdd. i7 = tidx while i7 < cand_count: vk = smem_keys[i7] - bin_f = (vk - bmin_r) * inv1 - bin_i = cutlass.Int32(bin_f) + bin_i = cutlass.Int32((vk - bmin_r) * inv1) if bin_i < cutlass.Int32(0): bin_i = cutlass.Int32(0) if bin_i > cutlass.Int32(kBins - 1): @@ -1544,9 +2528,7 @@ def phase4_histogram_snap( atomicAdd(smem_hist.iterator + bin_i, cutlass.Int32(1)) i7 = i7 + cutlass.Int32(num_threads) cute.arch.barrier() - - # ---- Parallel k-th bin search (3-step) ---- - # Step 1: each warp sums BINS_PER_WARP bins (high→low slice) + # ---- 3-step high→low bin search → straddling bin b* + rank_above ---- warp_bin_sum = cutlass.Int32(0) for jb in cutlass.range_constexpr(bins_per_warp): bidx_s = ( @@ -1555,14 +2537,10 @@ def phase4_histogram_snap( - cutlass.Int32(jb) ) warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] - if lane == 0: + if lane == cutlass.Int32(0): smem_wcnt[warp_id] = warp_bin_sum cute.arch.barrier() - - # Step 2: tid==0 finds target warp; stores prefix-count + warp index - # into s_iscalars[2] (=cnt_lo: prefix before target warp) - # and s_iscalars[3] (=cnt_hi: target warp index) - if tidx == 0: + if tidx == cutlass.Int32(0): cum = cutlass.Int32(0) tw = cutlass.Int32(num_warps - 1) found = cutlass.Int32(0) @@ -1571,41 +2549,862 @@ def phase4_histogram_snap( if cum >= cutlass.Int32(kK) and found == cutlass.Int32(0): tw = cutlass.Int32(w2) found = cutlass.Int32(1) - # Recompute prefix BEFORE target warp cum2 = cutlass.Int32(0) for w3 in cutlass.range_constexpr(self.num_warps): if cutlass.Int32(w3) < tw: cum2 = cum2 + smem_wcnt[w3] - s_iscalars[2] = cum2 # prefix - s_iscalars[3] = tw # target warp index + s_iscalars[2] = cum2 # prefix-count before target warp + s_iscalars[3] = tw cute.arch.barrier() - - # Step 3: target warp's lane 0 scans BINS_PER_WARP bins → - # threshold. Single-thread serial; the unrolled - # range_constexpr beats a runtime `for+break` (tried it: -544 - # SASS insts but -7pp fp32 / -14pp bf16, since the - # branch/counter overhead in a single thread dominates the - # static math). target_warp = s_iscalars[3] if warp_id == target_warp and lane == cutlass.Int32(0): base_cum = s_iscalars[2] - thr_local = bmin_r - bmin_local = bmin_r - set_done = cutlass.Int32(0) + b_star = cutlass.Int32(kBins - 1) + rank_above = base_cum + set_d = cutlass.Int32(0) for jb2 in cutlass.range_constexpr(bins_per_warp): bidx2 = ( cutlass.Int32(kBins - 1) - target_warp * cutlass.Int32(bins_per_warp) - cutlass.Int32(jb2) ) + ra_before = base_cum base_cum = base_cum + smem_hist[bidx2] - if base_cum >= cutlass.Int32(kK) and set_done == cutlass.Int32(0): - thr_local = bmin_local + cutlass.Float32(bidx2) * range1 / cutlass.Float32( - kBins - ) - set_done = cutlass.Int32(1) - s_thr[0] = thr_local + if base_cum >= cutlass.Int32(kK) and set_d == cutlass.Int32(0): + b_star = bidx2 + rank_above = ra_before # count in bins strictly above b* + set_d = cutlass.Int32(1) + s_iscalars[2] = rank_above + s_iscalars[3] = b_star + s_iscalars[4] = cutlass.Int32(0) # cnt_above + s_iscalars[1] = cutlass.Int32(0) # cnt_straddle cute.arch.barrier() + b_star = s_iscalars[3] + rank_above = s_iscalars[2] + + # ---- EXACT: one fine-histogram recursion on the straddling bin b* ---- + if cutlass.const_expr(self.enable_p4_rank_scatter_exact): + # FIXED small fine-bin count (independent of kNumBins) — cuts the + # re-zero + 3-step cost (esp. K=2048 where kNumBins=2048); 256 + # sub-bins over bin b* gives kNumBins×256 effective resolution, + # enough to resolve the straddling bin to ≤1 distinct value. + fbins = cutlass.const_expr(256) + fbpw = cutlass.const_expr(256 // self.num_warps) + # bin b* value range under the inv1 binning: [f_lo, f_lo + 1/inv1) + f_lo = bmin_r + cutlass.Float32(b_star) / inv1 + finv = (cutlass.Float32(fbins - 1) + cutlass.Float32(0.99)) * inv1 + # re-zero (only fbins slots) + build fine sub-hist of bin-b* cands + iz = tidx + while iz < cutlass.Int32(fbins): + smem_hist[iz] = cutlass.Int32(0) + iz = iz + cutlass.Int32(num_threads) + cute.arch.barrier() + ifb = tidx + while ifb < cand_count: + vf = smem_keys[ifb] + cb = cutlass.Int32((vf - bmin_r) * inv1) + if cb < cutlass.Int32(0): + cb = cutlass.Int32(0) + if cb > cutlass.Int32(kBins - 1): + cb = cutlass.Int32(kBins - 1) + if cb == b_star: + sb = cutlass.Int32((vf - f_lo) * finv) + if sb < cutlass.Int32(0): + sb = cutlass.Int32(0) + if sb > cutlass.Int32(fbins - 1): + sb = cutlass.Int32(fbins - 1) + atomicAdd(smem_hist.iterator + sb, cutlass.Int32(1)) + ifb = ifb + cutlass.Int32(num_threads) + cute.arch.barrier() + # fine 3-step search seeded at rank_above (over fbins bins) + fws = cutlass.Int32(0) + for jbf in cutlass.range_constexpr(fbpw): + bif = ( + cutlass.Int32(fbins - 1) + - warp_id * cutlass.Int32(fbpw) + - cutlass.Int32(jbf) + ) + fws = fws + smem_hist[bif] + if lane == cutlass.Int32(0): + smem_wcnt[warp_id] = fws + cute.arch.barrier() + if tidx == cutlass.Int32(0): + cumf = rank_above + twf = cutlass.Int32(num_warps - 1) + fnd = cutlass.Int32(0) + for w2 in cutlass.range_constexpr(self.num_warps): + cumf = cumf + smem_wcnt[w2] + if cumf >= cutlass.Int32(kK) and fnd == cutlass.Int32(0): + twf = cutlass.Int32(w2) + fnd = cutlass.Int32(1) + pre = rank_above + for w3 in cutlass.range_constexpr(self.num_warps): + if cutlass.Int32(w3) < twf: + pre = pre + smem_wcnt[w3] + # Stage prefix/target-warp metadata in spare s_iscalars + # slots, NOT smem_hist[0]/[1]: the last fine warp's reverse + # scan below walks fine bins down to 0/1, so reusing those + # histogram bins as scratch would corrupt sb_star/ra_fine + # when twf2 == num_warps-1. Slots [4]/[1] are dead here + # (re-zeroed at the cnt_above/cnt_strad reset below). + s_iscalars[4] = pre # prefix into target fine warp + s_iscalars[1] = twf # target fine warp + cute.arch.barrier() + pre_f = s_iscalars[4] + twf2 = s_iscalars[1] + if warp_id == twf2 and lane == cutlass.Int32(0): + base_f = pre_f + sb_star = cutlass.Int32(fbins - 1) + ra_fine = base_f + sd = cutlass.Int32(0) + for jb3 in cutlass.range_constexpr(fbpw): + sbi = ( + cutlass.Int32(fbins - 1) + - twf2 * cutlass.Int32(fbpw) + - cutlass.Int32(jb3) + ) + ra_b = base_f + base_f = base_f + smem_hist[sbi] + if base_f >= cutlass.Int32(kK) and sd == cutlass.Int32(0): + sb_star = sbi + ra_fine = ra_b + sd = cutlass.Int32(1) + smem_hist[2] = sb_star + smem_hist[3] = ra_fine + cute.arch.barrier() + if tidx == cutlass.Int32(0): + s_iscalars[4] = cutlass.Int32(0) # cnt_above + s_iscalars[0] = cutlass.Int32(0) # cnt_mid (b*, sub>sb*) + s_iscalars[1] = cutlass.Int32(0) # cnt_strad (b*, sub==sb*) + cute.arch.barrier() + sb_star = smem_hist[2] + rank_above_fine = smem_hist[3] + isc = tidx + while isc < cand_count: + v = smem_keys[isc] + bin_i = cutlass.Int32((v - bmin_r) * inv1) + if bin_i < cutlass.Int32(0): + bin_i = cutlass.Int32(0) + if bin_i > cutlass.Int32(kBins - 1): + bin_i = cutlass.Int32(kBins - 1) + if bin_i > b_star: + pos = atomicAdd(s_iscalars.iterator + cutlass.Int32(4), cutlass.Int32(1)) + if pos < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[pos] = self.dtype(v) + output_indices_row[pos] = smem_vals[isc] + elif bin_i == b_star: + sb = cutlass.Int32((v - f_lo) * finv) + if sb < cutlass.Int32(0): + sb = cutlass.Int32(0) + if sb > cutlass.Int32(fbins - 1): + sb = cutlass.Int32(fbins - 1) + if sb > sb_star: + o = atomicAdd(s_iscalars.iterator + cutlass.Int32(0), cutlass.Int32(1)) + pos = rank_above + o + if pos < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[pos] = self.dtype(v) + output_indices_row[pos] = smem_vals[isc] + elif sb == sb_star: + o = atomicAdd(s_iscalars.iterator + cutlass.Int32(1), cutlass.Int32(1)) + pos = rank_above_fine + o + if pos < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[pos] = self.dtype(v) + output_indices_row[pos] = smem_vals[isc] + isc = isc + cutlass.Int32(num_threads) + cute.arch.barrier() + cnt_strad = s_iscalars[1] + filled = rank_above_fine + cnt_strad + if filled > cutlass.Int32(kK): + filled = cutlass.Int32(kK) + ipad = filled + tidx + while ipad < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[ipad] = self.dtype(self.NEG_FLT_MAX) + output_indices_row[ipad] = cutlass.Int32(-1) + ipad = ipad + cutlass.Int32(num_threads) + + # ---- EXACT-TAIL repair (p4_exact_tail, fp32): the fine bin + # resolves values to range/(kBins*fbins); two candidates + # closer than that can straddle the kK boundary inside ONE + # fine bin, and the arrival-order fill above then keeps an + # arbitrary subset. Gated on the ONLY case where that is + # ambiguous — the tie set overfills the remaining slots — this + # re-ranks the (b*, sb*) tie set exactly via an MSB-first + # 8-bit-digit radix select over the order-preserving integer + # keys (4 levels = bit-exact for fp32) and rewrites the tie + # slot range [rank_above_fine, kK). Unambiguous rows (the + # overwhelming majority) pay two scalar compares; the counters + # and the fine histogram are reused, so SMEM does not grow. + # [p4tt] tiny-tie fast path: when the exact-tail gate fires + # with a small (b*, sb*) tie class (cnt_strad <= 128 — the + # real firing cells hold 2), ONE candidate pass collects the + # class and thread0 selects the top-need exactly, replacing + # the 4 unconditional radix passes. Larger classes take the + # UNMODIFIED radix select below (verbatim copy). + if cutlass.const_expr(self.p4_exact_tail and self.p4_tail_fast): # [p4tt] + need0 = cutlass.Int32(kK) - rank_above_fine + if cnt_strad > need0 and need0 > cutlass.Int32(0): + if cnt_strad <= cutlass.Int32(128): + # [p4tt] SMEM: (value_bits, cand_idx) pairs at + # smem_hist[2*o]/[2*o+1], o < 128 (slots 0..255). + # The 256 digit bins are dead here (the fast path + # replaces the radix levels that used them); the + # sb_star/ra staging in slots 2/3 was read by + # every thread before the pre-scatter barrier. + # Persistent radix scalars [256..258] untouched. + # Collect counter = s_iscalars[0] (dead after the + # scatter; same reuse as the radix rewrite pass). + if tidx == cutlass.Int32(0): + s_iscalars[0] = cutlass.Int32(0) + cute.arch.barrier() + itc = tidx + while itc < cand_count: + tv = smem_keys[itc] + tb = cutlass.Int32((tv - bmin_r) * inv1) + if tb < cutlass.Int32(0): + tb = cutlass.Int32(0) + if tb > cutlass.Int32(kBins - 1): + tb = cutlass.Int32(kBins - 1) + if tb == b_star: + ts = cutlass.Int32((tv - f_lo) * finv) + if ts < cutlass.Int32(0): + ts = cutlass.Int32(0) + if ts > cutlass.Int32(fbins - 1): + ts = cutlass.Int32(fbins - 1) + if ts == sb_star: + to = atomicAdd( + s_iscalars.iterator + cutlass.Int32(0), cutlass.Int32(1) + ) + if to < cutlass.Int32(128): + smem_hist[to + to] = float_as_int32(tv) + smem_hist[to + to + cutlass.Int32(1)] = smem_vals[itc] + itc = itc + cutlass.Int32(num_threads) + cute.arch.barrier() + # [p4tt] thread0 exact top-need0 select rewriting + # positions [rank_above_fine, kK). Consumed flag = + # the cand_idx slot set to -1 (indices are always + # >= 0), so a genuine -FLT_MAX value in the class + # remains selectable (no value sentinel). Ties + # (bit-equal values) pick arbitrarily: value-set + # exact. + if tidx == cutlass.Int32(0): + tj = cutlass.Int32(0) + while tj < need0: + tbv = cutlass.Float32(self.NEG_FLT_MAX) + tbi = cutlass.Int32(-1) + ti = cutlass.Int32(0) + while ti < cnt_strad: + tvi = smem_hist[ti + ti + cutlass.Int32(1)] + if tvi >= cutlass.Int32(0): + tvb = smem_hist[ti + ti] + tvv = cutlass.Float32( + llvm.bitcast( + cutlass.Float32.mlir_type, + tvb.ir_value(), + ) + ) + take = cutlass.Int32(0) + if tbi < cutlass.Int32(0): + take = cutlass.Int32(1) + elif tvv > tbv: + take = cutlass.Int32(1) + if take == cutlass.Int32(1): + tbv = tvv + tbi = ti + ti = ti + cutlass.Int32(1) + pos = rank_above_fine + tj + if cutlass.const_expr(self.return_output_values): + output_values_row[pos] = self.dtype(tbv) + output_indices_row[pos] = smem_hist[ + tbi + tbi + cutlass.Int32(1) + ] + smem_hist[tbi + tbi + cutlass.Int32(1)] = cutlass.Int32(-1) + tj = tj + cutlass.Int32(1) + cute.arch.barrier() + else: + # Persistent scalars live above the 256 digit bins + # (kNumBins >= 512 always): [256] key prefix (chosen + # digits, remaining bits 0), [257] slots still to fill + # inside the current equal-prefix set, [258] ties + # strictly above the prefix (their slots precede it). + if tidx == cutlass.Int32(0): + smem_hist[256] = cutlass.Int32(0) + smem_hist[257] = need0 + smem_hist[258] = cutlass.Int32(0) + cute.arch.barrier() + for lvl in cutlass.range_constexpr(4): + shift = cutlass.const_expr(24 - 8 * lvl) + iz2 = tidx + while iz2 < cutlass.Int32(256): + smem_hist[iz2] = cutlass.Int32(0) + iz2 = iz2 + cutlass.Int32(num_threads) + cute.arch.barrier() + uthr_cur = smem_hist[256] + it2 = tidx + while it2 < cand_count: + vt = smem_keys[it2] + bt = cutlass.Int32((vt - bmin_r) * inv1) + if bt < cutlass.Int32(0): + bt = cutlass.Int32(0) + if bt > cutlass.Int32(kBins - 1): + bt = cutlass.Int32(kBins - 1) + if bt == b_star: + st2 = cutlass.Int32((vt - f_lo) * finv) + if st2 < cutlass.Int32(0): + st2 = cutlass.Int32(0) + if st2 > cutlass.Int32(fbins - 1): + st2 = cutlass.Int32(fbins - 1) + if st2 == sb_star: + uk = f32_order_key(vt) + pmatch = cutlass.Int32(1) + if cutlass.const_expr(lvl > 0): + if (uk >> cutlass.Int32(shift + 8)) != ( + uthr_cur >> cutlass.Int32(shift + 8) + ): + pmatch = cutlass.Int32(0) + if pmatch == cutlass.Int32(1): + dg = (uk >> cutlass.Int32(shift)) & cutlass.Int32( + 0xFF + ) + atomicAdd(smem_hist.iterator + dg, cutlass.Int32(1)) + it2 = it2 + cutlass.Int32(num_threads) + cute.arch.barrier() + # Two-stage descending digit scan (mirrors the + # fine 3-step search): per-warp partial sums, + # thread0 picks the target warp, its lane0 walks + # the warp's digit range — 2*num_warps serial + # steps instead of 256. + fdw = cutlass.const_expr(256 // self.num_warps) + wsum2 = cutlass.Int32(0) + for jd in cutlass.range_constexpr(fdw): + dix = ( + cutlass.Int32(255) + - warp_id * cutlass.Int32(fdw) + - cutlass.Int32(jd) + ) + wsum2 = wsum2 + smem_hist[dix] + if lane == cutlass.Int32(0): + smem_wcnt[warp_id] = wsum2 + cute.arch.barrier() + if tidx == cutlass.Int32(0): + needl = smem_hist[257] + cw = cutlass.Int32(0) + tw3 = cutlass.Int32(num_warps - 1) + f3 = cutlass.Int32(0) + for w4 in cutlass.range_constexpr(self.num_warps): + cw = cw + smem_wcnt[w4] + if cw >= needl and f3 == cutlass.Int32(0): + tw3 = cutlass.Int32(w4) + f3 = cutlass.Int32(1) + pre3 = cutlass.Int32(0) + for w5 in cutlass.range_constexpr(self.num_warps): + if cutlass.Int32(w5) < tw3: + pre3 = pre3 + smem_wcnt[w5] + s_iscalars[4] = pre3 # prefix above target warp + s_iscalars[0] = tw3 # target warp + cute.arch.barrier() + pre4 = s_iscalars[4] + tw4 = s_iscalars[0] + if warp_id == tw4 and lane == cutlass.Int32(0): + needl2 = smem_hist[257] + base4 = pre4 + dstar = cutlass.Int32(0) + above_d = pre4 + sd4 = cutlass.Int32(0) + for jd2 in cutlass.range_constexpr(fdw): + dix2 = ( + cutlass.Int32(255) + - tw4 * cutlass.Int32(fdw) + - cutlass.Int32(jd2) + ) + ra4 = base4 + base4 = base4 + smem_hist[dix2] + if base4 >= needl2 and sd4 == cutlass.Int32(0): + dstar = dix2 + above_d = ra4 + sd4 = cutlass.Int32(1) + smem_hist[256] = uthr_cur | (dstar << cutlass.Int32(shift)) + smem_hist[257] = needl2 - above_d + smem_hist[258] = smem_hist[258] + above_d + cute.arch.barrier() + # Rewrite the tie slot range: ties with key > u_thr + # first (there are exactly cnt_ab of them), then the + # first need_eq bitwise-equal-to-u_thr ties in arrival + # order (value-exact by construction). Signed compare + # needs the top bit flipped (unsigned-monotonic key). + u_thr = smem_hist[256] + cnt_ab = smem_hist[258] + need_eq = smem_hist[257] + ks_thr = u_thr ^ cutlass.Int32(-2147483648) + if tidx == cutlass.Int32(0): + s_iscalars[4] = cutlass.Int32(0) # above-writer ctr + s_iscalars[0] = cutlass.Int32(0) # equal-writer ctr + cute.arch.barrier() + ir2 = tidx + while ir2 < cand_count: + vr = smem_keys[ir2] + br = cutlass.Int32((vr - bmin_r) * inv1) + if br < cutlass.Int32(0): + br = cutlass.Int32(0) + if br > cutlass.Int32(kBins - 1): + br = cutlass.Int32(kBins - 1) + if br == b_star: + sr = cutlass.Int32((vr - f_lo) * finv) + if sr < cutlass.Int32(0): + sr = cutlass.Int32(0) + if sr > cutlass.Int32(fbins - 1): + sr = cutlass.Int32(fbins - 1) + if sr == sb_star: + uk2 = f32_order_key(vr) + ks2 = uk2 ^ cutlass.Int32(-2147483648) + if ks2 > ks_thr: + o2 = atomicAdd( + s_iscalars.iterator + cutlass.Int32(4), + cutlass.Int32(1), + ) + pos = rank_above_fine + o2 + if pos < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[pos] = self.dtype(vr) + output_indices_row[pos] = smem_vals[ir2] + elif ks2 == ks_thr: + q2 = atomicAdd( + s_iscalars.iterator + cutlass.Int32(0), + cutlass.Int32(1), + ) + if q2 < need_eq: + pos = rank_above_fine + cnt_ab + q2 + if pos < cutlass.Int32(kK): + if cutlass.const_expr( + self.return_output_values + ): + output_values_row[pos] = self.dtype(vr) + output_indices_row[pos] = smem_vals[ir2] + ir2 = ir2 + cutlass.Int32(num_threads) + cute.arch.barrier() + elif cutlass.const_expr(self.p4_exact_tail): # [p4tt] if->elif only + need0 = cutlass.Int32(kK) - rank_above_fine + if cnt_strad > need0 and need0 > cutlass.Int32(0): + # Persistent scalars live above the 256 digit bins + # (kNumBins >= 512 always): [256] key prefix (chosen + # digits, remaining bits 0), [257] slots still to fill + # inside the current equal-prefix set, [258] ties + # strictly above the prefix (their slots precede it). + if tidx == cutlass.Int32(0): + smem_hist[256] = cutlass.Int32(0) + smem_hist[257] = need0 + smem_hist[258] = cutlass.Int32(0) + cute.arch.barrier() + for lvl in cutlass.range_constexpr(4): + shift = cutlass.const_expr(24 - 8 * lvl) + iz2 = tidx + while iz2 < cutlass.Int32(256): + smem_hist[iz2] = cutlass.Int32(0) + iz2 = iz2 + cutlass.Int32(num_threads) + cute.arch.barrier() + uthr_cur = smem_hist[256] + it2 = tidx + while it2 < cand_count: + vt = smem_keys[it2] + bt = cutlass.Int32((vt - bmin_r) * inv1) + if bt < cutlass.Int32(0): + bt = cutlass.Int32(0) + if bt > cutlass.Int32(kBins - 1): + bt = cutlass.Int32(kBins - 1) + if bt == b_star: + st2 = cutlass.Int32((vt - f_lo) * finv) + if st2 < cutlass.Int32(0): + st2 = cutlass.Int32(0) + if st2 > cutlass.Int32(fbins - 1): + st2 = cutlass.Int32(fbins - 1) + if st2 == sb_star: + uk = f32_order_key(vt) + pmatch = cutlass.Int32(1) + if cutlass.const_expr(lvl > 0): + if (uk >> cutlass.Int32(shift + 8)) != ( + uthr_cur >> cutlass.Int32(shift + 8) + ): + pmatch = cutlass.Int32(0) + if pmatch == cutlass.Int32(1): + dg = (uk >> cutlass.Int32(shift)) & cutlass.Int32(0xFF) + atomicAdd(smem_hist.iterator + dg, cutlass.Int32(1)) + it2 = it2 + cutlass.Int32(num_threads) + cute.arch.barrier() + # Two-stage descending digit scan (mirrors the + # fine 3-step search): per-warp partial sums, + # thread0 picks the target warp, its lane0 walks + # the warp's digit range — 2*num_warps serial + # steps instead of 256. + fdw = cutlass.const_expr(256 // self.num_warps) + wsum2 = cutlass.Int32(0) + for jd in cutlass.range_constexpr(fdw): + dix = ( + cutlass.Int32(255) + - warp_id * cutlass.Int32(fdw) + - cutlass.Int32(jd) + ) + wsum2 = wsum2 + smem_hist[dix] + if lane == cutlass.Int32(0): + smem_wcnt[warp_id] = wsum2 + cute.arch.barrier() + if tidx == cutlass.Int32(0): + needl = smem_hist[257] + cw = cutlass.Int32(0) + tw3 = cutlass.Int32(num_warps - 1) + f3 = cutlass.Int32(0) + for w4 in cutlass.range_constexpr(self.num_warps): + cw = cw + smem_wcnt[w4] + if cw >= needl and f3 == cutlass.Int32(0): + tw3 = cutlass.Int32(w4) + f3 = cutlass.Int32(1) + pre3 = cutlass.Int32(0) + for w5 in cutlass.range_constexpr(self.num_warps): + if cutlass.Int32(w5) < tw3: + pre3 = pre3 + smem_wcnt[w5] + s_iscalars[4] = pre3 # prefix above target warp + s_iscalars[0] = tw3 # target warp + cute.arch.barrier() + pre4 = s_iscalars[4] + tw4 = s_iscalars[0] + if warp_id == tw4 and lane == cutlass.Int32(0): + needl2 = smem_hist[257] + base4 = pre4 + dstar = cutlass.Int32(0) + above_d = pre4 + sd4 = cutlass.Int32(0) + for jd2 in cutlass.range_constexpr(fdw): + dix2 = ( + cutlass.Int32(255) + - tw4 * cutlass.Int32(fdw) + - cutlass.Int32(jd2) + ) + ra4 = base4 + base4 = base4 + smem_hist[dix2] + if base4 >= needl2 and sd4 == cutlass.Int32(0): + dstar = dix2 + above_d = ra4 + sd4 = cutlass.Int32(1) + smem_hist[256] = uthr_cur | (dstar << cutlass.Int32(shift)) + smem_hist[257] = needl2 - above_d + smem_hist[258] = smem_hist[258] + above_d + cute.arch.barrier() + # Rewrite the tie slot range: ties with key > u_thr + # first (there are exactly cnt_ab of them), then the + # first need_eq bitwise-equal-to-u_thr ties in arrival + # order (value-exact by construction). Signed compare + # needs the top bit flipped (unsigned-monotonic key). + u_thr = smem_hist[256] + cnt_ab = smem_hist[258] + need_eq = smem_hist[257] + ks_thr = u_thr ^ cutlass.Int32(-2147483648) + if tidx == cutlass.Int32(0): + s_iscalars[4] = cutlass.Int32(0) # above-writer ctr + s_iscalars[0] = cutlass.Int32(0) # equal-writer ctr + cute.arch.barrier() + ir2 = tidx + while ir2 < cand_count: + vr = smem_keys[ir2] + br = cutlass.Int32((vr - bmin_r) * inv1) + if br < cutlass.Int32(0): + br = cutlass.Int32(0) + if br > cutlass.Int32(kBins - 1): + br = cutlass.Int32(kBins - 1) + if br == b_star: + sr = cutlass.Int32((vr - f_lo) * finv) + if sr < cutlass.Int32(0): + sr = cutlass.Int32(0) + if sr > cutlass.Int32(fbins - 1): + sr = cutlass.Int32(fbins - 1) + if sr == sb_star: + uk2 = f32_order_key(vr) + ks2 = uk2 ^ cutlass.Int32(-2147483648) + if ks2 > ks_thr: + o2 = atomicAdd( + s_iscalars.iterator + cutlass.Int32(4), + cutlass.Int32(1), + ) + pos = rank_above_fine + o2 + if pos < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[pos] = self.dtype(vr) + output_indices_row[pos] = smem_vals[ir2] + elif ks2 == ks_thr: + q2 = atomicAdd( + s_iscalars.iterator + cutlass.Int32(0), + cutlass.Int32(1), + ) + if q2 < need_eq: + pos = rank_above_fine + cnt_ab + q2 + if pos < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[pos] = self.dtype(vr) + output_indices_row[pos] = smem_vals[ir2] + ir2 = ir2 + cutlass.Int32(num_threads) + cute.arch.barrier() + else: + # ---- APPROX rank-and-scatter (single pass), arbitrary straddling order ---- + isc = tidx + while isc < cand_count: + v = smem_keys[isc] + bin_i = cutlass.Int32((v - bmin_r) * inv1) + if bin_i < cutlass.Int32(0): + bin_i = cutlass.Int32(0) + if bin_i > cutlass.Int32(kBins - 1): + bin_i = cutlass.Int32(kBins - 1) + if bin_i > b_star: + pos = atomicAdd(s_iscalars.iterator + cutlass.Int32(4), cutlass.Int32(1)) + if pos < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[pos] = self.dtype(v) + output_indices_row[pos] = smem_vals[isc] + elif bin_i == b_star: + off = atomicAdd(s_iscalars.iterator + cutlass.Int32(1), cutlass.Int32(1)) + pos = rank_above + off + if pos < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[pos] = self.dtype(v) + output_indices_row[pos] = smem_vals[isc] + isc = isc + cutlass.Int32(num_threads) + cute.arch.barrier() + cnt_strad = s_iscalars[1] + filled = rank_above + cnt_strad + if filled > cutlass.Int32(kK): + filled = cutlass.Int32(kK) + ipad = filled + tidx + while ipad < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[ipad] = self.dtype(self.NEG_FLT_MAX) + output_indices_row[ipad] = cutlass.Int32(-1) + ipad = ipad + cutlass.Int32(num_threads) + else: + i10 = tidx + while i10 < cand_count: + if cutlass.const_expr(self.return_output_values): + output_values_row[i10] = self.dtype(smem_keys[i10]) + output_indices_row[i10] = smem_vals[i10] + i10 = i10 + cutlass.Int32(num_threads) + i11 = cand_count + tidx + while i11 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[i11] = self.dtype(self.NEG_FLT_MAX) + output_indices_row[i11] = cutlass.Int32(-1) + i11 = i11 + cutlass.Int32(num_threads) + + # ------------------------------------------------------------------ + # Phase 4: Histogram-based k-th selection + two-pass writeback + # ------------------------------------------------------------------ + @cute.jit + def phase4_histogram_snap( + self, + smem_keys, + smem_vals, + smem_hist, + smem_wcnt, + s_thr, + s_iscalars, + output_values_row, + output_indices_row, + cand_count, + tidx, + warp_id, + lane, + ): + """Three branches by cand_count vs kK: + == kK: direct emit (fast path) + > kK: histogram k-th bin search → snap → 2-pass writeback + < kK: emit cand_count + pad with -FLT_MAX + """ + kK = cutlass.const_expr(self.top_k) + kBins = cutlass.const_expr(self.kNumBins) + num_threads = cutlass.const_expr(self.num_threads) + # Hoisted SMEM window bases: every keys/vals element access below + # goes through raw integer addressing (see _smem_ref rationale). + keys_base = smem_keys.iterator.toint() + vals_base = smem_vals.iterator.toint() + # Scalars base for the snap-loop convergence check (read by ALL + # threads once per snap iteration — a measured per-iteration + # LDS hotspot). + isc_base = s_iscalars.iterator.toint() + + # ----- Branch A: cand_count == kK (fast path) ----- + if cand_count == cutlass.Int32(kK): + i4 = tidx + while i4 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[i4] = self.dtype( + self._smem_ld(cutlass.Float32, keys_base, i4) + ) + output_indices_row[i4] = self._smem_ld(cutlass.Int32, vals_base, i4) + i4 = i4 + cutlass.Int32(num_threads) + elif cand_count > cutlass.Int32(kK): + # ----- Branch B: cand_count > kK → histogram snap ----- + + # ---- Histogram window ---- + # Fast path: reuse the P2 exit bracket [vlo, vhi) instead of + # scanning candidates for min/max. P3 collected v >= s_thr[0] + # and P2's exit sets s_thr[0] = vlo (= s_thr[1]), so vlo + # lower-bounds every candidate; the bracket invariant + # cnt(>= vhi) < kK puts the k-th value inside [vlo, vhi). + # Out-of-window candidates (row max etc.) clamp into the edge + # bins — cumulative counts from the top stay exact — and the + # bracket is P2's acceptance band, far narrower than + # [cand_min, cand_max], so level-1 bin resolution IMPROVES. + # Any path that leaves the bracket stale (degenerate-bracket + # fallback, probe variants) fails the guard and takes the + # original min/max scan; a plausible-but-wrong bracket can + # only cost extra snap/refinement steps, never exactness. + # Uniform branch: SMEM scalars read after the P3-exit barrier. + w_lo = s_thr[1] + w_hi = s_thr[2] + bmin_r = cutlass.Float32(0.0) + bmax_r = cutlass.Float32(1e-6) + if s_thr[0] == w_lo and w_hi > w_lo and w_hi < cutlass.Float32(self.FLT_MAX): + bmin_r = w_lo + bmax_r = w_hi + else: + # Block min/max over keys[0:cand_count] + local_cmin = cutlass.Float32(self.FLT_MAX) + local_cmax = cutlass.Float32(self.NEG_FLT_MAX) + i5 = tidx + while i5 < cand_count: + v = self._smem_ld(cutlass.Float32, keys_base, i5) + local_cmin = _fmin_f32_inline(local_cmin, v) + local_cmax = cute.arch.fmax(local_cmax, v) + i5 = i5 + cutlass.Int32(num_threads) + cmin = self.warp_reduce_min_f32(local_cmin) + cmax = self.warp_reduce_max_f32(local_cmax) + # Stage warp results into smem_wcnt[w] (cmin) and smem_hist[w] (cmax) + # as bit-cast int32. cmax stored at smem_hist[0..NW-1]. + if lane == 0: + smem_wcnt[warp_id] = float_as_uint32(cmin) + smem_hist[warp_id] = float_as_uint32(cmax) + cute.arch.barrier() + + # Every thread independently recomputes block_min/block_max + # from the warp-staged smem slots (CUDA heuristic_topk.cuh:891-898 + # pattern). No tid==0 → s_thr broadcast → saves a block barrier. + bmin_r = cutlass.Float32(self.FLT_MAX) + bmax_r = cutlass.Float32(self.NEG_FLT_MAX) + # Unrolled num_warps times (16 or 32 — fixed at compile time). + for w in cutlass.range_constexpr(self.num_warps): + vmin_bits = smem_wcnt[w] + vmax_bits = smem_hist[w] + vmin = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, vmin_bits.ir_value()) + ) + vmax = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, vmax_bits.ir_value()) + ) + bmin_r = _fmin_f32_inline(bmin_r, vmin) + bmax_r = cute.arch.fmax(bmax_r, vmax) + if bmax_r <= bmin_r: + bmax_r = bmin_r + cutlass.Float32(1e-6) + # Barrier required: smem_hist[0..NW-1] above doubles as cmax + # scratch and below as the histogram. Without this sync the + # zeroing pass below can clobber a cmax slot a later warp is + # still reading → wrong bmax_r → all candidates squashed into + # bin 0 (hit-rate-dependent race). + cute.arch.barrier() + + range1 = bmax_r - bmin_r + # Overflow hardening (pre-existing): + # a candidate span > FLT_MAX (needs |v| ~ 1.7e38; fuzz-only for + # real logits) overflows range1 to +inf → inv1 = +0 → every + # candidate lands in bin 0 → thr = lo + 0*inf = NaN → all snap + # comparisons false, the walk never moves, and the whole row + # writes as padding. Clamp to FLT_MAX: the start threshold + # stays ORDERED (±inf is fine — snap's monotone walk rescues + # any ordered start; only NaN breaks it). + if range1 > cutlass.Float32(self.FLT_MAX): + range1 = cutlass.Float32(self.FLT_MAX) + # inv1 = (kBins - 1 + 0.99) / range1 (range1 > 0 guaranteed by 1e-6 patch) + inv1 = (cutlass.Float32(kBins - 1) + cutlass.Float32(0.99)) / range1 + binw1 = range1 / cutlass.Float32(kBins) + + # Predeclared register state for the redundant-warp path + # (threshold / counts / staging parity live in registers; the + # leader path below keeps them in s_thr/s_iscalars instead). + thr_reg = bmin_r + selc_reg = cutlass.Int32(0) + thr_s = bmin_r + cge_r = cutlass.Int32(0) + cgt_r = cutlass.Int32(0) + win_par = cutlass.Int32(0) + + # Level-1: histogram over [bmin, bmax] + k-th bin search. + self._hist_build(keys_base, smem_hist, cand_count, bmin_r, inv1, tidx) + if cutlass.const_expr(self.p4_warp_redundant): + thr_reg, selc_reg = self._kth_bin_search_rw( + smem_hist, smem_wcnt, bmin_r, binw1, tidx, warp_id, lane + ) + else: + self._kth_bin_search( + smem_hist, smem_wcnt, s_thr, s_iscalars, bmin_r, binw1, tidx, warp_id, lane + ) + + # ---- Level-2 histogram refinement ---- + # The snap loop below steps ONE distinct value per iteration + # (~0.45us each: full candidate re-scan + 2 barriers), and real + # logits concentrate count mass right at the k-th boundary, so + # the selected level-1 bin often holds tens of values → snap + # stragglers of 10+ us set the wall clock at N<=32K. When the + # selected bin is dense, re-histogram just that bin (bin width + # shrinks kBins x) for ~1us of extra scan, leaving the snap + # loop 0-2 steps. The snap loop converges monotonically from + # any starting threshold, so this only moves the start point — + # exactness is untouched (a level-2 edge-rounding error at + # worst costs one extra snap step). Uniform branch: everyone + # reads the same post-barrier SMEM scalar. + # Level 2 fires when a snap walk would cost more than one + # rebuild (~2 snap steps break even); level 3 only when level 2 + # failed to split the bin (>8: heavy ties or a sub-ulp-wide + # window — both rare on real logits, where ties at the k-th + # are ~1 and the acceptance band spans >>1 ulp). + binw_cur = binw1 + for _lvl in cutlass.range_constexpr(2): + if cutlass.const_expr(self.p4_warp_redundant): + sel_cnt_l = selc_reg + else: + sel_cnt_l = s_iscalars[4] + gate_l = cutlass.const_expr(2 if _lvl == 0 else 8) + if sel_cnt_l > cutlass.Int32(gate_l): + if cutlass.const_expr(self.p4_warp_redundant): + thr_el = thr_reg + # _kth_bin_search_rw has no trailing barrier; the + # zero pass of the rebuild below must not clobber + # smem_hist under a warp still in its step 3. + cute.arch.barrier() + else: + thr_el = s_thr[0] + # 2% slop each side absorbs the inv-vs-binw rounding + # difference in the previous level's edge estimate. + lo_l = thr_el - cutlass.Float32(0.02) * binw_cur + range_l = cutlass.Float32(1.04) * binw_cur + inv_l = (cutlass.Float32(kBins - 1) + cutlass.Float32(0.99)) / range_l + binw_next = range_l / cutlass.Float32(kBins) + self._hist_build(keys_base, smem_hist, cand_count, lo_l, inv_l, tidx) + if cutlass.const_expr(self.p4_warp_redundant): + thr_l2, selc_l2 = self._kth_bin_search_rw( + smem_hist, smem_wcnt, lo_l, binw_next, tidx, warp_id, lane + ) + thr_reg = thr_l2 + selc_reg = selc_l2 + else: + self._kth_bin_search( + smem_hist, + smem_wcnt, + s_thr, + s_iscalars, + lo_l, + binw_next, + tidx, + warp_id, + lane, + ) + binw_cur = binw_next # ---- Snap convergence loop ---- # Upper bound = cand_count (matches CUDA heuristic_topk.cuh:985). @@ -1617,100 +3416,267 @@ def phase4_histogram_snap( # Runtime break via a guard flag — no `break` in cute.range. si = cutlass.Int32(0) done_snap = cutlass.Int32(0) - while si < snap_limit and done_snap == cutlass.Int32(0): - self.block_fused_snap_iter( - smem_keys, - smem_wcnt, - smem_hist, - s_thr, - s_iscalars, - cand_count, - tidx, - warp_id, - lane, - ) - # After block_fused_snap_iter, s_iscalars[2]=cge, s_iscalars[3]=cgt. - if s_iscalars[3] < cutlass.Int32(kK) and s_iscalars[2] >= cutlass.Int32(kK): - done_snap = cutlass.Int32(1) - si = si + cutlass.Int32(1) - - # ---- Two-pass writeback (ballot + popc) ---- - # Per-iter: ballot collects emit flags into a 32-bit mask; - # popc gives the within-warp count; lane 0 atomicAdds the - # output base; shuffle broadcasts it. One barrier between - # passes, none within a pass. - sel_thr = s_thr[0] + if cutlass.const_expr(self.p4_warp_redundant): + # Redundant-warp snap: threshold + convergence state live + # in registers (every warp reduces the staged partials + # itself, bit-identically), so each iteration needs ONE + # barrier (staging visibility) instead of two. Staging is + # parity double-buffered in smem_hist[par*3NW ..] so a + # warp one iteration ahead writes the other bank while a + # slow warp still reads the old one; the staging barrier + # bounds the drift to a single iteration. + cute.arch.barrier() # rw-search step-3 readers vs staging + nwc = cutlass.const_expr(self.num_warps) + thr_s = thr_reg + par4 = cutlass.Int32(0) + while si < snap_limit and done_snap == cutlass.Int32(0): + lge4 = cutlass.Int32(0) + lgt4 = cutlass.Int32(0) + up4 = cutlass.Float32(self.FLT_MAX) + dn4 = cutlass.Float32(self.NEG_FLT_MAX) + isi4 = tidx + while isi4 < cand_count: + v4 = self._smem_ld(cutlass.Float32, keys_base, isi4) + if v4 >= thr_s: + lge4 = lge4 + cutlass.Int32(1) + if v4 > thr_s: + lgt4 = lgt4 + cutlass.Int32(1) + up4 = _fmin_f32_inline(up4, v4) + if v4 < thr_s: + dn4 = cute.arch.fmax(dn4, v4) + isi4 = isi4 + cutlass.Int32(num_threads) + packed4 = (lge4 << cutlass.Int32(16)) | lgt4 + packed4 = self.warp_reduce_sum_i32(packed4) + up4 = self.warp_reduce_min_f32(up4) + dn4 = self.warp_reduce_max_f32(dn4) + off4 = par4 * cutlass.Int32(3 * nwc) + if lane == 0: + smem_hist[off4 + warp_id] = packed4 + smem_hist[off4 + cutlass.Int32(nwc) + warp_id] = float_as_uint32(up4) + smem_hist[off4 + cutlass.Int32(2 * nwc) + warp_id] = float_as_uint32(dn4) + cute.arch.barrier() + v_tp = cutlass.Int32(0) + v_up = cutlass.Float32(self.FLT_MAX) + v_dn = cutlass.Float32(self.NEG_FLT_MAX) + if lane < cutlass.Int32(nwc): + v_tp = smem_hist[off4 + lane] + vu_b = smem_hist[off4 + cutlass.Int32(nwc) + lane] + vd_b = smem_hist[off4 + cutlass.Int32(2 * nwc) + lane] + v_up = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, vu_b.ir_value()) + ) + v_dn = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, vd_b.ir_value()) + ) + tp4 = self.warp_reduce_sum_i32(v_tp) + tup4 = self.warp_reduce_min_f32(v_up) + tdn4 = self.warp_reduce_max_f32(v_dn) + cge_r = tp4 >> cutlass.Int32(16) + cgt_r = tp4 & cutlass.Int32(0xFFFF) + win_par = par4 + if cgt_r >= cutlass.Int32(kK): + if tup4 < cutlass.Float32(self.FLT_MAX): + thr_s = tup4 + elif cge_r < cutlass.Int32(kK): + if tdn4 > cutlass.Float32(self.NEG_FLT_MAX): + thr_s = tdn4 + if cgt_r < cutlass.Int32(kK) and cge_r >= cutlass.Int32(kK): + done_snap = cutlass.Int32(1) + par4 = par4 ^ cutlass.Int32(1) + si = si + cutlass.Int32(1) + else: + while si < snap_limit and done_snap == cutlass.Int32(0): + self.block_fused_snap_iter( + keys_base, + smem_wcnt, + smem_hist, + s_thr, + s_iscalars, + cand_count, + tidx, + warp_id, + lane, + ) + # After block_fused_snap_iter, s_iscalars[2]=cge, s_iscalars[3]=cgt. + cgt_c = self._smem_ld(cutlass.Int32, isc_base, cutlass.Int32(3)) + cge_c = self._smem_ld(cutlass.Int32, isc_base, cutlass.Int32(2)) + if cgt_c < cutlass.Int32(kK) and cge_c >= cutlass.Int32(kK): + done_snap = cutlass.Int32(1) + si = si + cutlass.Int32(1) + + # ---- Writeback (ballot + popc) ---- + # Converged snap (the overwhelmingly common case): SINGLE pass. + # The converged iteration's cgt (s_iscalars[3]) is the exact + # strictly-greater count at sel_thr (block_fused_snap_iter does + # not move the threshold when cgt < kK <= cge), so gt entries + # can pack into [0, cgt) via counter s_iscalars[4] while + # tie(==) entries start at offset cgt via counter s_iscalars[5] + # — same [gt | eq | pad] output partition as the two-pass + # original, one candidate sweep and one barrier fewer. The + # non-converged fallback keeps the original two-pass (its cgt + # would be stale: the last iter may have moved the threshold + # after counting). + if cutlass.const_expr(self.p4_warp_redundant): + sel_thr = thr_s + else: + sel_thr = s_thr[0] if tidx == 0: - s_iscalars[4] = cutlass.Int32(0) # out_count + s_iscalars[4] = cutlass.Int32(0) # gt out_count + # s_iscalars[5] (cluster-local scratch, consumed before P4) + # is reused as the eq counter for the single-pass path. + s_iscalars[5] = cutlass.Int32(0) cute.arch.barrier() - # Pass 1: v > sel_thr, strided over (warp_id * WARP_SIZE, ...). - # The `if mask_gt != 0` guard skips popc + atomicAdd + shuffle - # when no lane in the warp emits — the SMEM atomicAdd alone is - # ~10-30 cycles. - base_w = warp_id * cutlass.Int32(self.WARP_SIZE) - while base_w < cand_count: - ix1 = base_w + lane - emit_gt = cutlass.Int32(0) - v_p1 = cutlass.Float32(self.NEG_FLT_MAX) - if ix1 < cand_count: - v_p1 = smem_keys[ix1] - if v_p1 > sel_thr: - emit_gt = cutlass.Int32(1) - mask_gt = cute.arch.vote_ballot_sync(emit_gt != cutlass.Int32(0)) - if mask_gt != cutlass.Uint32(0): - cnt_gt = cutlass.Int32(cute.arch.popc(mask_gt)) - lane_mask_gt = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32(1) - moff_gt = cutlass.Int32(cute.arch.popc(mask_gt & lane_mask_gt)) - bp_gt = cutlass.Int32(0) - if lane == cutlass.Int32(0): - bp_gt = atomicAdd( - s_iscalars.iterator + cutlass.Int32(4), - cnt_gt, + if done_snap == cutlass.Int32(1): + # Zero-atomic single pass. The converged snap iteration + # staged each warp's packed(ge<<16|gt) counts AT sel_thr in + # smem_wcnt[w] (nothing touches smem_wcnt between the snap + # exit and here), and the snap scan's tidx-strided + # partition covers exactly the same element set per warp + # as this warp-chunk scan. So every warp derives its + # deterministic output bases from a prefix over + # smem_wcnt — the ~2*cand/32 serialized SMEM atomics of + # the claim-based scheme (a top stall region in ncu at + # N=8K) disappear. Output order within the [gt | eq] + # segments changes (deterministic instead of claim order), + # which the contract allows. + if cutlass.const_expr(self.p4_warp_redundant): + cgt_base = cgt_r + else: + cgt_base = s_iscalars[3] + gt_run = cutlass.Int32(0) + eq_run = cutlass.Int32(0) + for wpre in cutlass.range_constexpr(self.num_warps): + if cutlass.const_expr(self.p4_warp_redundant): + # Converged iteration's packed counts live in the + # winning parity bank of smem_hist, not smem_wcnt. + pk_w = smem_hist[win_par * cutlass.Int32(3 * self.num_warps) + wpre] + else: + pk_w = smem_wcnt[wpre] + if cutlass.Int32(wpre) < warp_id: + wge_w = pk_w >> cutlass.Int32(16) + wgt_w = pk_w & cutlass.Int32(0xFFFF) + gt_run = gt_run + wgt_w + eq_run = eq_run + (wge_w - wgt_w) + eq_run = cgt_base + eq_run + base_w = warp_id * cutlass.Int32(self.WARP_SIZE) + while base_w < cand_count: + ix1 = base_w + lane + emit_gt = cutlass.Int32(0) + emit_eq = cutlass.Int32(0) + v_p1 = cutlass.Float32(self.NEG_FLT_MAX) + if ix1 < cand_count: + v_p1 = self._smem_ld(cutlass.Float32, keys_base, ix1) + if v_p1 > sel_thr: + emit_gt = cutlass.Int32(1) + if v_p1 == sel_thr: + emit_eq = cutlass.Int32(1) + mask_gt = cute.arch.vote_ballot_sync(emit_gt != cutlass.Int32(0)) + lane_mask = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32(1) + if mask_gt != cutlass.Uint32(0): + moff_gt = cutlass.Int32(cute.arch.popc(mask_gt & lane_mask)) + wpos_p1 = gt_run + moff_gt + if emit_gt != cutlass.Int32(0) and wpos_p1 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[wpos_p1] = self.dtype(v_p1) + output_indices_row[wpos_p1] = self._smem_ld( + cutlass.Int32, vals_base, ix1 + ) + gt_run = gt_run + cutlass.Int32(cute.arch.popc(mask_gt)) + mask_eq = cute.arch.vote_ballot_sync(emit_eq != cutlass.Int32(0)) + if mask_eq != cutlass.Uint32(0): + moff_eq = cutlass.Int32(cute.arch.popc(mask_eq & lane_mask)) + wpos_p2 = eq_run + moff_eq + if emit_eq != cutlass.Int32(0) and wpos_p2 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[wpos_p2] = self.dtype(v_p1) + output_indices_row[wpos_p2] = self._smem_ld( + cutlass.Int32, vals_base, ix1 + ) + eq_run = eq_run + cutlass.Int32(cute.arch.popc(mask_eq)) + base_w = base_w + cutlass.Int32(num_threads) + cute.arch.barrier() + else: + # Pass 1: v > sel_thr, strided over (warp_id * WARP_SIZE, ...). + base_w = warp_id * cutlass.Int32(self.WARP_SIZE) + while base_w < cand_count: + ix1 = base_w + lane + emit_gt = cutlass.Int32(0) + v_p1 = cutlass.Float32(self.NEG_FLT_MAX) + if ix1 < cand_count: + v_p1 = self._smem_ld(cutlass.Float32, keys_base, ix1) + if v_p1 > sel_thr: + emit_gt = cutlass.Int32(1) + mask_gt = cute.arch.vote_ballot_sync(emit_gt != cutlass.Int32(0)) + if mask_gt != cutlass.Uint32(0): + cnt_gt = cutlass.Int32(cute.arch.popc(mask_gt)) + lane_mask_gt = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32( + 1 ) - bp_gt = cute.arch.shuffle_sync(bp_gt, cutlass.Int32(0)) - wpos_p1 = bp_gt + moff_gt - if emit_gt != cutlass.Int32(0) and wpos_p1 < cutlass.Int32(kK): - if cutlass.const_expr(self.return_output_values): - output_values_row[wpos_p1] = self.dtype(v_p1) - output_indices_row[wpos_p1] = smem_vals[ix1] - base_w = base_w + cutlass.Int32(num_threads) - cute.arch.barrier() + moff_gt = cutlass.Int32(cute.arch.popc(mask_gt & lane_mask_gt)) + bp_gt = cutlass.Int32(0) + if lane == cutlass.Int32(0): + bp_gt = atomicAdd( + s_iscalars.iterator + cutlass.Int32(4), + cnt_gt, + ) + bp_gt = cute.arch.shuffle_sync(bp_gt, cutlass.Int32(0)) + wpos_p1 = bp_gt + moff_gt + if emit_gt != cutlass.Int32(0) and wpos_p1 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[wpos_p1] = self.dtype(v_p1) + output_indices_row[wpos_p1] = self._smem_ld( + cutlass.Int32, vals_base, ix1 + ) + base_w = base_w + cutlass.Int32(num_threads) + cute.arch.barrier() - # Pass 2: v == sel_thr (same pattern + guard as Pass 1). Empty - # iterations are much more common here since only tie-at-K - # values qualify. - base_w2 = warp_id * cutlass.Int32(self.WARP_SIZE) - while base_w2 < cand_count: - ix2 = base_w2 + lane - emit_eq = cutlass.Int32(0) - v_p2 = cutlass.Float32(self.NEG_FLT_MAX) - if ix2 < cand_count: - v_p2 = smem_keys[ix2] - if v_p2 == sel_thr: - emit_eq = cutlass.Int32(1) - mask_eq = cute.arch.vote_ballot_sync(emit_eq != cutlass.Int32(0)) - if mask_eq != cutlass.Uint32(0): - cnt_eq = cutlass.Int32(cute.arch.popc(mask_eq)) - lane_mask_eq = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32(1) - moff_eq = cutlass.Int32(cute.arch.popc(mask_eq & lane_mask_eq)) - bp_eq = cutlass.Int32(0) - if lane == cutlass.Int32(0): - bp_eq = atomicAdd( - s_iscalars.iterator + cutlass.Int32(4), - cnt_eq, + # Pass 2: v == sel_thr (same pattern + guard as Pass 1). + base_w2 = warp_id * cutlass.Int32(self.WARP_SIZE) + while base_w2 < cand_count: + ix2 = base_w2 + lane + emit_eq = cutlass.Int32(0) + v_p2 = cutlass.Float32(self.NEG_FLT_MAX) + if ix2 < cand_count: + v_p2 = self._smem_ld(cutlass.Float32, keys_base, ix2) + if v_p2 == sel_thr: + emit_eq = cutlass.Int32(1) + mask_eq = cute.arch.vote_ballot_sync(emit_eq != cutlass.Int32(0)) + if mask_eq != cutlass.Uint32(0): + cnt_eq = cutlass.Int32(cute.arch.popc(mask_eq)) + lane_mask_eq = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32( + 1 ) - bp_eq = cute.arch.shuffle_sync(bp_eq, cutlass.Int32(0)) - wpos_p2 = bp_eq + moff_eq - if emit_eq != cutlass.Int32(0) and wpos_p2 < cutlass.Int32(kK): - if cutlass.const_expr(self.return_output_values): - output_values_row[wpos_p2] = self.dtype(v_p2) - output_indices_row[wpos_p2] = smem_vals[ix2] - base_w2 = base_w2 + cutlass.Int32(num_threads) - cute.arch.barrier() + moff_eq = cutlass.Int32(cute.arch.popc(mask_eq & lane_mask_eq)) + bp_eq = cutlass.Int32(0) + if lane == cutlass.Int32(0): + bp_eq = atomicAdd( + s_iscalars.iterator + cutlass.Int32(4), + cnt_eq, + ) + bp_eq = cute.arch.shuffle_sync(bp_eq, cutlass.Int32(0)) + wpos_p2 = bp_eq + moff_eq + if emit_eq != cutlass.Int32(0) and wpos_p2 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[wpos_p2] = self.dtype(v_p2) + output_indices_row[wpos_p2] = self._smem_ld( + cutlass.Int32, vals_base, ix2 + ) + base_w2 = base_w2 + cutlass.Int32(num_threads) + cute.arch.barrier() - # Pad remainder with -self.FLT_MAX / -1 - filled_par = s_iscalars[4] + # Pad remainder with -self.FLT_MAX / -1. Single-pass filled = + # cge (= cgt + total ties at sel_thr, from the converged snap + # iteration; the zero-atomic path leaves counters untouched); + # two-pass filled = counter [4] (gt + eq accumulated). + filled_par = cutlass.Int32(0) + if done_snap == cutlass.Int32(1): + if cutlass.const_expr(self.p4_warp_redundant): + filled_par = cge_r + else: + filled_par = s_iscalars[2] + else: + filled_par = s_iscalars[4] if filled_par > cutlass.Int32(kK): filled_par = cutlass.Int32(kK) ipad = filled_par + tidx @@ -1719,14 +3685,17 @@ def phase4_histogram_snap( output_values_row[ipad] = self.dtype(self.NEG_FLT_MAX) output_indices_row[ipad] = cutlass.Int32(-1) ipad = ipad + cutlass.Int32(num_threads) + else: # ----- Branch C: cand_count < kK ----- # Emit cand_count + pad i10 = tidx while i10 < cand_count: if cutlass.const_expr(self.return_output_values): - output_values_row[i10] = self.dtype(smem_keys[i10]) - output_indices_row[i10] = smem_vals[i10] + output_values_row[i10] = self.dtype( + self._smem_ld(cutlass.Float32, keys_base, i10) + ) + output_indices_row[i10] = self._smem_ld(cutlass.Int32, vals_base, i10) i10 = i10 + cutlass.Int32(num_threads) i11 = cand_count + tidx while i11 < cutlass.Int32(kK): @@ -1906,9 +3875,14 @@ def run_one_row( byte_alignment=128, ) # warp_counts[NUM_WARPS] int32 (P3 prefix-sum scratch) + # p2_warp_redundant parity-banks the Phase-2 staging (a warp one + # round ahead writes the other half) — costs num_warps*4 bytes. smem_wcnt = smem.allocate_tensor( element_type=cutlass.Int32, - layout=cute.make_ordered_layout((num_warps,), order=(0,)), + layout=cute.make_ordered_layout( + (2 * num_warps if cutlass.const_expr(self.p2_warp_redundant) else num_warps,), + order=(0,), + ), byte_alignment=128, ) # Phase-1 warp aggregates (fp32 + int32; ~256 bytes total) @@ -1951,19 +3925,29 @@ def run_one_row( layout=cute.make_ordered_layout((6,), order=(0,)), byte_alignment=16, ) - # Per-CTA DSMEM scratch for the cluster all-reduce of cand_count. - # mapa.shared::cluster relies on every CTA holding this slot at the - # SAME SMEM offset, so it's allocated once here. Only needed at - # cs>1; skipped at cs=1 (all uses are gated by const_expr(cs>1) and - # None propagates harmlessly through the unused parameters). + # Per-CTA DSMEM scratch for the cluster all-reduce of cand_count: + # slots 0/1 = parity double-buffered count exchange (call k writes + # slot k&1 — closes the straggler-read-vs-next-write DSMEM race), + # slot 2 = tid0-private call counter. mapa.shared::cluster relies + # on every CTA holding this block at the SAME SMEM offset, so it's + # allocated once here. Only USED at cs>1 (uses are gated by + # const_expr(cs>1)), but ALLOCATED unconditionally: the LB hybrid + # kernel inlines a cs>1 and a cs=1 instance into one launch, and the + # DSL sizes the launch SMEM from the last-traced SmemAllocator only — + # the layouts must stay byte-identical across cluster_size (16B cost + # at cs=1). + s_cluster_partial = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((3,), order=(0,)), + byte_alignment=16, + ) if cutlass.const_expr(cluster_size > 1): - s_cluster_partial = smem.allocate_tensor( - element_type=cutlass.Int32, - layout=cute.make_ordered_layout((1,), order=(0,)), - byte_alignment=16, - ) - else: - s_cluster_partial = None + # Zero the call counter before any block_count_ge call. tid0- + # private (same thread reads/increments it), so program order + # suffices — but parity must start at 0 on EVERY CTA of the + # cluster for lockstep alignment. + if tidx == cutlass.Int32(0): + s_cluster_partial[2] = cutlass.Int32(0) # SMEM slice cache (optional). Sized in ``self.dtype`` so the same # vec_w-wide LDG→STS→LDS pipeline works for fp32/bf16/fp16. @@ -1978,6 +3962,74 @@ def run_one_row( else: smem_input = None + # op#26 R0 admission scratch (single-CTA fast path). Allocated only + # when enable_r0; None otherwise so the base SMEM layout is byte-for- + # byte unchanged and these propagate harmlessly through _run_phases' + # const_expr(enable_r0)-gated branch (same idiom as s_cluster_partial + # / smem_input above). smem_ptcnt_multi caches M per-thread count + # columns; s_r0col carries the accepted rung index tid0 -> all. + if cutlass.const_expr(self.enable_r0): + M_r0 = cutlass.const_expr(self.M_thr) + # vseed (v3): the pmean column's per-thread counts reuse the + # existing single-column smem_ptcnt buffer, so the BIG multi + # buffer only holds the M_qf rung columns -> zero smem growth + # (the round-1 +2-4KB column pushed 16-bit mb3/T1024 configs over + # an occupancy cliff: K2048 fp16 BS1024 -26%). + M_r0_pt = cutlass.const_expr(self.M_qf) + s_mt_thr = smem.allocate_tensor( + element_type=cutlass.Float32, + layout=cute.make_ordered_layout((M_r0,), order=(0,)), + byte_alignment=16, + ) + smem_ptcnt_multi = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((M_r0_pt * num_threads,), order=(0,)), + byte_alignment=128, + ) + smem_wcnt_multi = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((M_r0 * num_warps,), order=(0,)), + byte_alignment=64, + ) + s_mt_cnt = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((M_r0,), order=(0,)), + byte_alignment=16, + ) + s_r0col = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((1,), order=(0,)), + byte_alignment=16, + ) + # DSMEM scratch for the M-way cluster all-reduce of the R0 rung + # counts (mapa.shared::cluster needs the same offset on every + # CTA). Only USED at cs>1; allocated unconditionally so the + # cs=1 / cs>1 SMEM layouts stay byte-identical for the LB + # hybrid kernel (see s_cluster_partial above). + s_cluster_partial_m = smem.allocate_tensor( + element_type=cutlass.Int32, + layout=cute.make_ordered_layout((M_r0,), order=(0,)), + byte_alignment=16, + ) + # p1b_cache: P1 stashes the K gathered preIdx values here so P1b + # skips a second GMEM random gather (dtype-gated: 16-bit only). + if cutlass.const_expr(self.p1b_cache): + smem_gath = smem.allocate_tensor( + element_type=cutlass.Float32, + layout=cute.make_ordered_layout((self.top_k,), order=(0,)), + byte_alignment=128, + ) + else: + smem_gath = None + else: + s_mt_thr = None + smem_ptcnt_multi = None + smem_wcnt_multi = None + s_mt_cnt = None + s_r0col = None + s_cluster_partial_m = None + smem_gath = None + # ---- Per-row dispatch ---- # Three branches: # 1. Degenerate (N <= top_k): no GVR work, leader emits identity. @@ -2052,6 +4104,13 @@ def run_one_row( s_iscalars, s_cluster_partial, smem_input, + s_mt_thr, + smem_ptcnt_multi, + smem_wcnt_multi, + s_mt_cnt, + s_r0col, + s_cluster_partial_m, + smem_gath, tidx, warp_id, lane, @@ -2086,6 +4145,13 @@ def run_one_row( s_iscalars, s_cluster_partial, smem_input, + s_mt_thr, + smem_ptcnt_multi, + smem_wcnt_multi, + s_mt_cnt, + s_r0col, + s_cluster_partial_m, + smem_gath, tidx, warp_id, lane, @@ -2117,6 +4183,13 @@ def run_one_row( s_iscalars, s_cluster_partial, smem_input, + s_mt_thr, + smem_ptcnt_multi, + smem_wcnt_multi, + s_mt_cnt, + s_r0col, + s_cluster_partial_m, + smem_gath, tidx, warp_id, lane, @@ -2151,6 +4224,13 @@ def _run_phases( s_iscalars, s_cluster_partial, smem_input, + s_mt_thr, + smem_ptcnt_multi, + smem_wcnt_multi, + s_mt_cnt, + s_r0col, + s_cluster_partial_m, + smem_gath, tidx, warp_id, lane, @@ -2184,6 +4264,8 @@ def _run_phases( tidx, warp_id, lane, + smem_gath=smem_gath, # p1b_cache: stash gathered values (None-op OFF) + s_mt_thr=s_mt_thr, # r0_vseed: park pmean in the last rung column ) # Degenerate threshold init: val_hi <= -self.FLT_MAX or val_lo >= val_hi. @@ -2228,23 +4310,274 @@ def _run_phases( tidx, ) - # ---- Phase 2: secant threshold search ---- - self.phase2_secant_search( - input_row, - N, - slice_start, - slice_end, - smem_ptcnt, - smem_wcnt, - s_thr, - s_iscalars, - s_cluster_partial, - tidx, - warp_id, - lane, - do_cluster_sync=do_cluster_sync, - smem_input=smem_input, - ) + # ---- Phase 2: R0 histogram-ladder admission (single-CTA fast + # path) or the secant threshold search ---- + # enable_r0 gates to cluster_size==1 for now: op#26's R0 scans the + # full row in one CTA. The slice-parallel + cluster count-merge + # variant that lets R0 cover the cs>1 long-row branch lands in a + # later commit; until then cs>1 keeps the secant path. + if cutlass.const_expr(self.enable_r0): + # P1b rung placement -> ONE M-ary R0 count pass -> accept the + # tightest rung with count in [K, kC]. On a miss, fall back to + # the inline log-falsi R1 shot / fb_fix refine. At cs>1 each + # CTA scans its slice and block_count_ge_multi cluster-merges + # the rung counts (phase1b rungs are per-CTA identical since + # preIdx stats are full-row). + if cutlass.const_expr(self.p1b_cache): + # rungs from the SMEM gather-cache P1 stashed (no 2nd + # GMEM gather); 16-bit only. + self.phase1b_hspace_rungs_cached( + pre_idx_count, smem_gath, smem_hist, s_thr, s_mt_thr, tidx, warp_id, lane + ) + else: + self.phase1b_hspace_rungs( + input_row, + N, + pre_idx_row, + pre_idx_count, + pre_idx_offset, + smem_hist, + s_thr, + s_mt_thr, + tidx, + warp_id, + lane, + ) + self.block_count_ge_multi( + input_row, + slice_start, + slice_end, + s_mt_thr, + smem_ptcnt_multi, + smem_wcnt_multi, + s_mt_cnt, + s_cluster_partial_m, + do_cluster_sync, + tidx, + warp_id, + lane, + smem_ptcnt=smem_ptcnt, + ) + cute.arch.barrier() + if tidx == 0: + # tightest admissible rung = SMALLEST count in [K, kC]. + # (Explicit argmin: with r0_vseed the pmean column is not + # sorted into the rung order; for sorted rungs this is + # equivalent to the old "last m in window" rule.) + best_m = cutlass.Int32(-1) + best_c = cutlass.Int32(2147483647) + for m in cutlass.range_constexpr(cutlass.const_expr(self.M_thr)): + cm = s_mt_cnt[m] + if ( + cm >= cutlass.Int32(self.top_k) + and cm <= cutlass.Int32(self.kC) + and cm < best_c + ): + best_m = cutlass.Int32(m) + best_c = cm + s_r0col[0] = best_m + if best_m >= cutlass.Int32(0): + s_thr[0] = s_mt_thr[best_m] + s_iscalars[0] = s_mt_cnt[best_m] + # done=1: the threshold is admitted, so Phase 3 must + # SKIP its retry-shrink and honor s_thr[0]. (block_count + # _ge / secant leave done via their own path; the R0 + # admission must set it explicitly or Phase 3 re-searches + # and the cluster collect diverges -> wrong output.) + s_iscalars[1] = cutlass.Int32(1) + # Snapshot this CTA's LOCAL slice count for the chosen + # rung into s_iscalars[5] — the per-CTA cand_count that + # Phase 3/4's cluster gather consumes (block_count_ge + # sets it too; the R0 admission must match). Without it + # the cluster collect under-counts -> wrong output. + if cutlass.const_expr(cluster_size > 1): + s_iscalars[5] = s_cluster_partial_m[best_m] + cute.arch.barrier() + bc = s_r0col[0] + if bc >= cutlass.Int32(0) and bc < cutlass.Int32(self.M_qf): + # accepted rung column: copy its cached per-thread counts + # into the secant hand-off buffer (zero rescan). The vseed + # column (bc == M_qf) is ALREADY in smem_ptcnt (v3 reuse). + smem_ptcnt[tidx] = smem_ptcnt_multi[bc * cutlass.Int32(num_threads) + tidx] + cute.arch.barrier() + # ---- R0 miss: SEEDED bounded log-falsi refine ---- + # At large N the M2D rungs straddle [K, kC]; the refine must + # find a threshold with count in [K, kC] between the measured + # rungs. SEED the loop with the rung bracket AND its known + # counts (clo/chi) so it does log-count regula-falsi from + # iter 0 with no re-measure and no separate R1 shot -> ~2-3 + # count passes (op#26 efficiency) instead of ~6. done=1 on + # accept so Phase 3 skips its retry-shrink. + if bc < cutlass.Int32(0): + if cutlass.const_expr(self.fb_fix): + if tidx == cutlass.Int32(0): + M = cutlass.const_expr(self.M_thr) + blo = v_lo + bhi = v_hi + clo = cutlass.Int32(-1) + chi = cutlass.Int32(-1) + for m in cutlass.range_constexpr(M): + cm = s_mt_cnt[m] + tm = s_mt_thr[m] + if cm > cutlass.Int32(self.kC) and ( + clo < cutlass.Int32(0) or tm > blo + ): + blo = tm + clo = cm + if cm < cutlass.Int32(self.top_k) and ( + chi < cutlass.Int32(0) or tm < bhi + ): + bhi = tm + chi = cm + s_thr[1] = blo + s_thr[2] = bhi + s_iscalars[2] = clo # SEED known rung counts + s_iscalars[3] = chi + s_iscalars[1] = cutlass.Int32(0) # done=0 + cand = (blo + bhi) * cutlass.Float32(0.5) + if clo > cutlass.Int32(0) and chi >= cutlass.Int32(0): + chic = chi + if chic < cutlass.Int32(1): + chic = cutlass.Int32(1) + l_lo = cmath.log2(cutlass.Float32(clo), fastmath=True) + l_hi = cmath.log2(cutlass.Float32(chic), fastmath=True) + den = l_lo - l_hi + if den > cutlass.Float32(0.0): + t3 = (cutlass.Float32(self.log2_mstar) - l_hi) / den + cnd3 = bhi + t3 * (blo - bhi) + if cnd3 > blo and cnd3 < bhi: + cand = cnd3 + elif chi < cutlass.Int32(0): + cand = bhi + elif clo < cutlass.Int32(0): + cand = blo + s_thr[0] = cand + cute.arch.barrier() + rs = cutlass.Int32(0) + while rs < cutlass.Int32(8) and s_iscalars[1] == cutlass.Int32(0): + if rs > cutlass.Int32(0): + if tidx == cutlass.Int32(0): + lo3 = s_thr[1] + hi3 = s_thr[2] + clo3 = s_iscalars[2] + chi3 = s_iscalars[3] + cand = (lo3 + hi3) * cutlass.Float32(0.5) + if chi3 < cutlass.Int32(0): + cand = hi3 + elif clo3 < cutlass.Int32(0): + cand = lo3 + else: + chic = chi3 + if chic < cutlass.Int32(1): + chic = cutlass.Int32(1) + l_lo = cmath.log2(cutlass.Float32(clo3), fastmath=True) + l_hi = cmath.log2(cutlass.Float32(chic), fastmath=True) + den3 = l_lo - l_hi + if den3 > cutlass.Float32(0.0): + t3 = (cutlass.Float32(self.log2_mstar) - l_hi) / den3 + cnd3 = hi3 + t3 * (lo3 - hi3) + if cnd3 > lo3 and cnd3 < hi3: + cand = cnd3 + s_thr[0] = cand + cute.arch.barrier() + self.block_count_ge( + input_row, + slice_start, + slice_end, + s_thr[0], + smem_ptcnt, + smem_wcnt, + s_iscalars, + s_cluster_partial, + tidx, + warp_id, + lane, + do_cluster_sync=do_cluster_sync, + smem_input=smem_input, + ) + cute.arch.barrier() + if tidx == cutlass.Int32(0): + c3 = s_iscalars[0] + t3v = s_thr[0] + if c3 >= cutlass.Int32(self.top_k) and c3 <= cutlass.Int32(self.kC): + s_iscalars[1] = cutlass.Int32(1) # accept + elif c3 > cutlass.Int32(self.kC): + s_thr[1] = t3v + s_iscalars[2] = c3 + if t3v >= s_thr[2]: + rng3 = s_thr[2] - s_thr[1] + if rng3 < cutlass.Float32(1.0): + rng3 = cutlass.Float32(1.0) + s_thr[2] = s_thr[2] + rng3 * cutlass.Float32(8.0) + s_iscalars[3] = cutlass.Int32(-1) + else: + s_thr[2] = t3v + s_iscalars[3] = c3 + if t3v <= s_thr[1]: + rng3 = s_thr[2] - s_thr[1] + if rng3 < cutlass.Float32(1.0): + rng3 = cutlass.Float32(1.0) + s_thr[1] = s_thr[1] - rng3 * cutlass.Float32(8.0) + s_iscalars[2] = cutlass.Int32(-1) + cute.arch.barrier() + rs = rs + cutlass.Int32(1) + if s_iscalars[1] != cutlass.Int32(1): + # tie-plateau fail-soft: land on the measured + # undershoot side (count <= kC => no overflow). + self.block_count_ge( + input_row, + slice_start, + slice_end, + s_thr[2], + smem_ptcnt, + smem_wcnt, + s_iscalars, + s_cluster_partial, + tidx, + warp_id, + lane, + do_cluster_sync=do_cluster_sync, + smem_input=smem_input, + ) + cute.arch.barrier() + if tidx == cutlass.Int32(0): + s_thr[0] = s_thr[2] + s_iscalars[1] = cutlass.Int32(1) + cute.arch.barrier() + else: + self.phase2_secant_search( + input_row, + N, + slice_start, + slice_end, + smem_ptcnt, + smem_wcnt, + s_thr, + s_iscalars, + s_cluster_partial, + tidx, + warp_id, + lane, + do_cluster_sync=do_cluster_sync, + smem_input=smem_input, + ) + else: + self.phase2_secant_search( + input_row, + N, + slice_start, + slice_end, + smem_ptcnt, + smem_wcnt, + s_thr, + s_iscalars, + s_cluster_partial, + tidx, + warp_id, + lane, + do_cluster_sync=do_cluster_sync, + smem_input=smem_input, + ) # Cluster handoff #1 (end of Phase 2). Skipped when # do_cluster_sync is False (cs=1 or short-row degrade). @@ -2290,20 +4623,36 @@ def _run_phases( if cutlass.const_expr(cluster_size == 1): # cs=1: the single CTA per row IS the leader. cand_count_p4 = min(s_iscalars[0], cutlass.Int32(self.kC)) - self.phase4_histogram_snap( - smem_keys, - smem_vals, - smem_hist, - smem_wcnt, - s_thr, - s_iscalars, - output_values_row, - output_indices_row, - cand_count_p4, - tidx, - warp_id, - lane, - ) + if cutlass.const_expr(self.enable_p4_rank_scatter): + self.phase4_rank_scatter( + smem_keys, + smem_vals, + smem_hist, + smem_wcnt, + s_thr, + s_iscalars, + output_values_row, + output_indices_row, + cand_count_p4, + tidx, + warp_id, + lane, + ) + else: + self.phase4_histogram_snap( + smem_keys, + smem_vals, + smem_hist, + smem_wcnt, + s_thr, + s_iscalars, + output_values_row, + output_indices_row, + cand_count_p4, + tidx, + warp_id, + lane, + ) else: # cs>1: only the leader (CTA 0 in cluster) runs Phase 4. if is_leader: @@ -2351,20 +4700,36 @@ def _run_phases( # ---- Phase 4: histogram snap + writeback ---- cand_count_p4 = min(s_iscalars[0], cutlass.Int32(self.kC)) - self.phase4_histogram_snap( - smem_keys, - smem_vals, - smem_hist, - smem_wcnt, - s_thr, - s_iscalars, - output_values_row, - output_indices_row, - cand_count_p4, - tidx, - warp_id, - lane, - ) + if cutlass.const_expr(self.enable_p4_rank_scatter): + self.phase4_rank_scatter( + smem_keys, + smem_vals, + smem_hist, + smem_wcnt, + s_thr, + s_iscalars, + output_values_row, + output_indices_row, + cand_count_p4, + tidx, + warp_id, + lane, + ) + else: + self.phase4_histogram_snap( + smem_keys, + smem_vals, + smem_hist, + smem_wcnt, + s_thr, + s_iscalars, + output_values_row, + output_indices_row, + cand_count_p4, + tidx, + warp_id, + lane, + ) # Final cluster barrier: keep peer CTAs (and their SMEM) alive # until the leader's gather + Phase 4 finish. Skipped at @@ -2418,5 +4783,213 @@ def __call__( min_blocks_per_mp=self.min_blocks_per_mp, ) + # ------------------------------------------------------------------ # + # Host-side launch-shape policy + self-contained launcher # + # ------------------------------------------------------------------ # + # cluster_size / num_threads / min_blocks_per_mp / use_256bit_load are + # compile-time ctor knobs: a compiled kernel cannot change its own grid + # or cluster shape, so batch-size adaptation MUST happen at launch time + # by picking a different compiled variant. ``pick_config`` is that + # policy as a pure function colocated with the kernel (single source of + # truth), and ``launch`` is a thin variant-cache wrapper so direct-drive + # users (tests, benchmarks) get the same shapes production would pick. + # The production custom op keeps its own equivalent inline policy for + # now; unifying it onto ``pick_config`` is a call-site change deferred + # to the dispatch-guard follow-up PR. + + _NUM_SMS: Optional[int] = None + _LAUNCH_CACHE: dict = {} + + @staticmethod + def _device_num_sms() -> int: + if GvrTopKKernel._NUM_SMS is None: + import torch # local: keep the module importable without torch + + GvrTopKKernel._NUM_SMS = torch.cuda.get_device_properties( + torch.cuda.current_device() + ).multi_processor_count + return GvrTopKKernel._NUM_SMS + + @staticmethod + def pick_config( + torch_dtype, + num_rows: int, + num_candidates: int, + max_seq_len: Optional[int] = None, + num_sms: Optional[int] = None, + ) -> dict: + """Pick the launch-shape ctor kwargs for ``(dtype, BS, N)``. + + Mirrors the production runner policy (cluster_size auto-pick + + ``_pick_tuning``) so any caller instantiating the kernel directly + gets the same shapes the custom op would use. Rationale (B200, + nsys cold-L2, 2026-07-15 big-BS triage): a config frozen at the + BS=1 optimum (cs = N>=65536 ? 4 : 1, T=1024, mbpm=1) is geomean + 2.27x slower (max 6.0x) than the op-bench anchor at BS in + {64, 256, 1024}, while this policy is 0.95x (parity/better). + Multi-CTA splitting only pays while the grid is a single wave + (num_rows * cluster_size <= num_sms); past that, row parallelism + already saturates the SMs and per-row splitting is pure overhead. + + ``max_seq_len``: pass the peak runtime N under CUDA-graph capture + so the variant is picked for the replay shape, not the capture + shape (same contract as the custom op's ``_pick_tuning``). + + Returns kwargs for ``GvrTopKKernel(...)``: ``cluster_size``, + ``num_threads``, ``use_256bit_load``, ``min_blocks_per_mp``, + ``enable_warp_parallel_reduce``. + """ + import torch # local: keep the module importable without torch + + if num_sms is None: + num_sms = GvrTopKKernel._device_num_sms() + n_row = max_seq_len if max_seq_len is not None else num_candidates + is_fp32 = torch_dtype == torch.float32 + + # cluster_size: B200 SXM5 synth-data tuning (matches the custom + # op's auto-pick): N < 64K -> 1 (sync unrecouped); tiny grid at + # large N -> 8; single-wave -> 4/2; multi-wave -> 1. + if n_row < 65536: + cluster_size = 1 + elif num_rows <= 4 and n_row >= 131072: + cluster_size = 8 + elif num_rows * 4 <= num_sms: + cluster_size = 4 + elif num_rows * 2 <= num_sms: + cluster_size = 2 + else: + cluster_size = 1 + + # Cluster CTAs split the row, so tuning targets per-CTA work. + n_per_cta = n_row // cluster_size + # T=1024 needs 1 CTA/SM grid AND enough per-CTA vec work. Under + # graph capture, raise the half-prec bar so a small capture-N + # doesn't force T=1024 on small-N replays. + n_thresh_t = 131072 if (max_seq_len is not None and not is_fp32) else 65536 + num_threads = 1024 if (num_rows <= num_sms and n_per_cta >= n_thresh_t) else 512 + # V=256-bit only helps fp32 at large N; half-prec cvt doubles reg + # pressure. Caller must hand a 32B-aligned contiguous tensor + # (``launch`` downgrades on misalignment). + use_256bit_load = is_fp32 and n_per_cta >= 16384 + enable_warp_parallel_reduce = num_threads == 1024 + + # min_blocks_per_mp: reg-vs-occupancy 3-tier (fp32 wants ~70 regs + # for 4-LDG ILP -> mb<=2; half-prec fits 40 regs -> mb=3 packs + # 3 CTA/SM when rows oversubscribe the device). + vec_bits = 256 if use_256bit_load else 128 + vec_w = vec_bits // (32 if is_fp32 else 16) + n_vec_iters = max(1, n_per_cta // (num_threads * vec_w)) + if is_fp32: + if n_vec_iters < 4: + min_blocks_per_mp = 0 + elif num_rows <= num_sms: + min_blocks_per_mp = 1 + elif num_sms * 2 < num_rows <= num_sms * 3 and n_per_cta <= 32768: + min_blocks_per_mp = 3 + else: + min_blocks_per_mp = 2 + else: + if num_rows > num_sms: + min_blocks_per_mp = 3 + elif n_vec_iters < 4: + min_blocks_per_mp = 0 + else: + min_blocks_per_mp = 1 + + return dict( + cluster_size=cluster_size, + num_threads=num_threads, + use_256bit_load=use_256bit_load, + min_blocks_per_mp=min_blocks_per_mp, + enable_warp_parallel_reduce=enable_warp_parallel_reduce, + ) + + @classmethod + def launch( + cls, + logits, + pre_idx, + seq_lens, + output_indices, + top_k: int, + next_n: int = 1, + compress_ratio: int = 1, + max_seq_len: Optional[int] = None, + num_sms: Optional[int] = None, + **kernel_overrides, + ) -> None: + """Compile-and-launch with ``pick_config`` shapes (indices-only path). + + Owns a class-level compiled-variant cache keyed by every ctor knob, + so repeated calls at any (BS, N, dtype) reuse the right variant. + ``kernel_overrides`` (e.g. ``enable_r0=False``, ``cluster_size=8``) + override the picked config and participate in the cache key. + Mirrors the custom op's compile contract: sym_int shapes, tvm-ffi + env stream (launches on the ambient torch stream), fixed + ``return_output_values=False`` / ``seqlen_sorted=False``. + """ + import torch # local: keep the module importable without torch + from cutlass.cute import runtime as _crt + + _cute_dt = { + torch.float32: cutlass.Float32, + torch.float16: cutlass.Float16, + torch.bfloat16: cutlass.BFloat16, + } + num_rows, num_candidates = logits.shape + cfg = cls.pick_config( + logits.dtype, num_rows, num_candidates, max_seq_len=max_seq_len, num_sms=num_sms + ) + cfg.update(kernel_overrides) + if cfg["cluster_size"] > 1: + try: + from .single_pass_multi_cta_radix_topk_cluster import _query_max_cluster_size + + cfg["cluster_size"] = min(cfg["cluster_size"], _query_max_cluster_size()) + except ImportError: + pass # standalone snapshot: trust the [1, 16] ctor bound + if cfg.get("use_256bit_load") and logits.data_ptr() % 32 != 0: + cfg["use_256bit_load"] = False # 256-bit vec loads need 32B alignment + + key = (logits.dtype, top_k, next_n, compress_ratio) + tuple(sorted(cfg.items())) + compiled = cls._LAUNCH_CACHE.get(key) + if compiled is None: + kernel = cls( + dtype=_cute_dt[logits.dtype], + top_k=top_k, + next_n=next_n, + compress_ratio=compress_ratio, + return_output_values=False, + **cfg, + ) + n_rows, n_cols, n_batch = cute.sym_int(), cute.sym_int(), cute.sym_int() + in_align = 32 if cfg["use_256bit_load"] else 16 + input_fake = _crt.make_fake_compact_tensor( + kernel.dtype, (n_rows, n_cols), stride_order=(1, 0), assumed_align=in_align + ) + pre_idx_fake = _crt.make_fake_compact_tensor( + cutlass.Int32, (n_batch, top_k), stride_order=(1, 0), assumed_align=16 + ) + seq_lens_fake = _crt.make_fake_compact_tensor( + cutlass.Int32, (n_batch,), stride_order=(0,) + ) + out_indices_fake = _crt.make_fake_compact_tensor( + cutlass.Int32, (n_rows, top_k), stride_order=(1, 0), assumed_align=16 + ) + fake_stream = _crt.make_fake_stream(use_tvm_ffi_env_stream=True) + compiled = cute.compile( + kernel, + input_fake, + pre_idx_fake, + seq_lens_fake, + None, + out_indices_fake, + None, + stream=fake_stream, + options="--enable-tvm-ffi", + ) + cls._LAUNCH_CACHE[key] = compiled + compiled(logits, pre_idx, seq_lens, None, output_indices, None) + __all__ = ["GvrTopKKernel", "GvrParams"] diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode_load_balance.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode_load_balance.py index 166ff80394e0..fd9339620b08 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode_load_balance.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode_load_balance.py @@ -247,9 +247,45 @@ def __init__( enable_warp_parallel_reduce=enable_warp_parallel_reduce, compress_ratio=compress_ratio, return_output_values=return_output_values, + # The two instances are inlined into ONE launch, and the CuTe DSL + # sizes the launch's dynamic SMEM from the last-traced + # SmemAllocator only (CuTeDSL.track_smem_allocator holds a single + # slot). Every smem-affecting knob must therefore resolve + # identically for both instances, or the larger branch reads/ + # writes past the reserved SMEM (CI IMA, 2026-07-21: fp32 K2048 + # long branch overflowed by the p1b_cache smem_gath the single + # instance's default had skipped). Pin the two knobs whose + # defaults diverge on cluster_size: + # - p1b_cache: cs>1 default True vs cs=1 fp32 default False. + # - kc_diet: cs=1 K512 default shrinks kC 4096→3072. + p1b_cache=True, + kc_diet=False, ) self._cluster_kernel = GvrTopKKernel(cluster_size=cluster_size, **common_kwargs) self._single_kernel = GvrTopKKernel(cluster_size=1, **common_kwargs) + # Drift guard: fail fast at construction if the derived smem-layout + # attributes ever diverge again (new knobs must be added here AND to + # the pinned kwargs above). + for attr in ( + "kC", + "kNumBins", + "M_thr", + "M_qf", + "p1b_cache", + "enable_smem_cache", + "smem_cache_elems", + "num_threads", + "p2_warp_redundant", + "p4_warp_redundant", + ): + a = getattr(self._cluster_kernel, attr) + b = getattr(self._single_kernel, attr) + assert a == b, ( + f"GvrTopKLBKernel: smem-layout attribute {attr!r} diverges " + f"between the cluster ({a}) and single ({b}) member kernels; " + f"the DSL sizes the launch from one SmemAllocator, so the " + f"layouts must be byte-identical." + ) # Prepare is decoupled from main: callers run it once per # decode step (seq_lens is layer-invariant). Use # GvrTopKLBPrepareKernel directly. diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/single_pass_multi_cta_radix_topk_cluster.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/single_pass_multi_cta_radix_topk_cluster.py index 2cccee734210..96999cb879b5 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/single_pass_multi_cta_radix_topk_cluster.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/single_pass_multi_cta_radix_topk_cluster.py @@ -232,8 +232,11 @@ def _radix_round_cluster( tidx, ) - # 2. Cluster barrier: publish all local histograms - cute.arch.cluster_arrive_relaxed() + # 2. Cluster barrier: publish all local histograms. Non-relaxed arrive + # (release fence) so the local_histogram stores are visible to the peer + # ld.shared::cluster acquire in merge_histogram_dsmem below; relaxed + # would risk stale peer reads (cf. filtered_top_k_varlen_util fix). + cute.arch.cluster_arrive() cute.arch.cluster_wait() # 3. Merge histograms via DSMEM (local_histogram -> prefix_buf) diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/__init__.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/__init__.py index a0eef9b7b9ac..30c56a896cf5 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/__init__.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/__init__.py @@ -2,9 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """CuteDSL MegaMoE NVFP4 kernel package. -Hosts the ported MegaMoE fused dispatch + FC1 + activation + FC2 + combine -CuteDSL kernel (flattened from the upstream ``moe_nvfp4_swapab/`` + ``src/`` -split). The package is loaded +Hosts the MegaMoE fused dispatch + FC1 + activation + FC2 + combine CuteDSL +kernel. The package is loaded lazily by :mod:`tensorrt_llm._torch.modules.fused_moe.mega_moe.mega_moe_cute_dsl` through :func:`import_kernel` so environments without a CUDA 13 Cutlass DSL runtime can still import the backend file for capability probing. @@ -39,7 +38,6 @@ "from_blocked", "import_kernel", "import_sym_buffer_host", - "import_topk_reduce", "stack_byte_reinterpretable_tensors", "to_blocked", ] @@ -72,20 +70,3 @@ def import_sym_buffer_host(): # SymBufferHost lives at module scope as a factory; the upstream API # constructs the per-world-size variant inside sym_buffer.py. return sym_buffer - - -def import_topk_reduce(): - """Lazily import the standalone CuteDSL top-k reduce kernel API. - - Returns ``(compile_topk_reduce, launch_compiled_topk_reduce)`` from - :mod:`.topk_reduce` (mirrors :func:`import_kernel`). The reduce kernel - is only needed by the opt-in transformers graph - (``apply_topk_in_fc1=False``); the deepgemm-default route reduces on - the host via ``combine_output.sum(dim=1)`` and never imports it. Like - ``import_kernel`` this stays lazy so non-SM100 / no-cutlass-dsl - environments can import the backend for capability probing without - pulling the heavyweight CuteDSL symbols. - """ - from .topk_reduce import compile_topk_reduce, launch_compiled_topk_reduce - - return compile_topk_reduce, launch_compiled_topk_reduce diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/contract.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/contract.py index 579231b1a383..c3965a5cc963 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/contract.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/contract.py @@ -1,5 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause """Codegen-time finite mapping contracts for RMEM tensor handoff.""" from __future__ import annotations @@ -78,9 +80,8 @@ def linearize(self, coord: Sequence[int]) -> int: """Convert a coordinate tuple into a CuTe-style linear index.""" coord_tuple = tuple(coord) if len(coord_tuple) != self.rank: - raise ContractError( - f"Coordinate rank mismatch for {self.names!r}: {len(coord_tuple)} != {self.rank}" - ) + raise ContractError(f"Coordinate rank mismatch for {self.names!r}: " + f"{len(coord_tuple)} != {self.rank}") linear = 0 stride = 1 @@ -162,8 +163,8 @@ def normalize(self, *, domain: Space, codomain: Space) -> tuple[int, ...]: """Validate and return the canonical table for the given spaces.""" if len(self.table) != domain.size: raise ContractError( - f"TableMapping length must equal domain size {domain.size}, got {len(self.table)}" - ) + f"TableMapping length must equal domain size {domain.size}, " + f"got {len(self.table)}") for idx, value in enumerate(self.table): if value < 0 or value >= codomain.size: raise ContractError( @@ -395,3 +396,27 @@ class TensorWithContract: tensor: Any contract: Contract + + +def eval_function_mapping(contract: Contract, **domain_coord) -> dict: + """Evaluate a FunctionMapping contract at runtime.""" + if not isinstance(contract.mapping, FunctionMapping): + raise TypeError("runtime contract eval requires a FunctionMapping") + + result = contract.mapping.function(**domain_coord) + if isinstance(result, dict): + return result + if isinstance(result, (tuple, list)): + if len(result) != contract.codomain.rank: + raise ValueError( + "FunctionMapping result rank does not match codomain rank: " + f"{len(result)} vs {contract.codomain.rank}") + return { + name: result[i] + for i, name in enumerate(contract.codomain.names) + } + if contract.codomain.rank == 1: + return {contract.codomain.names[0]: result} + raise TypeError( + "FunctionMapping runtime eval must return dict/tuple/list, or scalar " + "for rank-1 codomain") diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/custom_ext.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/custom_ext.py index 0a7acf16ee06..cdf832245946 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/custom_ext.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/custom_ext.py @@ -1,5 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause """Sched extension for fused fc1+fc2 work-tile enrichment and GMEM slicing.""" from typing import List, Optional, Tuple, Union @@ -37,10 +39,9 @@ def __init__( cumulative_data_physical_row: Int32, cumulative_sf_physical_row: Int32, cumulative_token_block_count: Int32, - valid_tokens_in_tile: Int32, + valid_tokens_in_cta_tile: Int32, phase_and_peek: Int32, ): - # Slot 3 reuses base k_tile_cnt storage. super().__init__( expert_idx, tile_m_idx, @@ -50,7 +51,7 @@ def __init__( self.cumulative_data_physical_row = self.k_tile_cnt self.cumulative_sf_physical_row = cumulative_sf_physical_row self.cumulative_token_block_count = cumulative_token_block_count - self.valid_tokens_in_tile = valid_tokens_in_tile + self.valid_tokens_in_cta_tile = valid_tokens_in_cta_tile # Slot 7 is the packed (BlockPhase | (peek_ready << 16)) field. # The ``.phase`` and ``.peek_ready`` properties below unpack it; # consumers call them directly so the codebase reads as if the @@ -81,14 +82,14 @@ def __extract_mlir_values__(self) -> List[ir.Value]: values = super().__extract_mlir_values__() values.extend(extract_mlir_values(self.cumulative_sf_physical_row)) values.extend(extract_mlir_values(self.cumulative_token_block_count)) - values.extend(extract_mlir_values(self.valid_tokens_in_tile)) + values.extend(extract_mlir_values(self.valid_tokens_in_cta_tile)) values.extend(extract_mlir_values(self.phase_and_peek)) return values def __new_from_mlir_values__( self, values: List[ir.Value]) -> "SwapABSwigluFp4Fc12WorkTileInfo": assert len(values) == 8 - return SwapABSwigluFp4Fc12WorkTileInfo( + return type(self)( expert_idx=new_from_mlir_values(self.expert_idx, [values[0]]), tile_m_idx=new_from_mlir_values(self.tile_m_idx, [values[1]]), tile_n_idx=new_from_mlir_values(self.tile_n_idx, [values[2]]), @@ -98,8 +99,8 @@ def __new_from_mlir_values__( self.cumulative_sf_physical_row, [values[4]]), cumulative_token_block_count=new_from_mlir_values( self.cumulative_token_block_count, [values[5]]), - valid_tokens_in_tile=new_from_mlir_values(self.valid_tokens_in_tile, - [values[6]]), + valid_tokens_in_cta_tile=new_from_mlir_values( + self.valid_tokens_in_cta_tile, [values[6]]), phase_and_peek=new_from_mlir_values(self.phase_and_peek, [values[7]]), ) @@ -112,7 +113,7 @@ def to_rmem(self) -> cute.Tensor: rmem[3] = self.k_tile_cnt # = cumulative_data_physical_row rmem[4] = self.cumulative_sf_physical_row rmem[5] = self.cumulative_token_block_count - rmem[6] = self.valid_tokens_in_tile + rmem[6] = self.valid_tokens_in_cta_tile rmem[7] = self.phase_and_peek return rmem @@ -125,7 +126,7 @@ def from_rmem(cls, rmem: cute.Tensor) -> "SwapABSwigluFp4Fc12WorkTileInfo": cumulative_data_physical_row=rmem[3], # type: ignore[arg-type] cumulative_sf_physical_row=rmem[4], # type: ignore[arg-type] cumulative_token_block_count=rmem[5], # type: ignore[arg-type] - valid_tokens_in_tile=rmem[6], # type: ignore[arg-type] + valid_tokens_in_cta_tile=rmem[6], # type: ignore[arg-type] phase_and_peek=rmem[7], # type: ignore[arg-type] ) @@ -160,7 +161,7 @@ def __init__( # shows enough arrivals. Mirrors ``fc1_done_counter_ptr`` for the # fc1->fc2 link: this side is "fc1 input ready", that side is # "fc1 output done". The threshold per-tile is the tile's - # ``valid_tokens_in_tile`` (dispatch does not pull padding + # ``valid_tokens_in_cta_tile`` (dispatch does not pull padding # tokens), read straight off the base work tile -- no separate # threshold field needed. ``None`` in the lean fc1+fc2 path keeps # ``enrich_work_tile_info`` to its existing fc2-only peek shape and @@ -236,7 +237,7 @@ def enrich_work_tile_info( ``cumulative_token_block_count + tile_n_idx`` against ``self.fc2_spin_threshold`` (work-tile-invariant const). - fc1 tiles peek the dispatch->fc1 ``fc1_ready_counter`` at the - same slot index but with ``valid_tokens_in_tile`` as threshold + same slot index (``tile_n_idx``) but with ``valid_tokens_in_cta_tile`` as threshold (per-tile dynamic). This branch only emits when ``self.fc1_ready_counter_ptr is not None`` (MegaMoE mode). """ @@ -247,8 +248,7 @@ def enrich_work_tile_info( if is_valid: # Same slot index for both phases -- fc1 release-add (dispatch # pull) and fc2 release-add (fc1 epi) target the per-task-tile - # counter slot indexed by ``cumulative_token_block_count + - # tile_n_idx``. + # counter slot indexed by ``cumulative_token_block_count + tile_n_idx``. counter_slot = (base_work.cumulative_token_block_count + base_work.tile_n_idx) is_fc1 = base_work.phase == Int32(int(BlockPhase.Linear1)) @@ -257,14 +257,14 @@ def enrich_work_tile_info( # MegaMoE-only: fc1 phase peek on fc1_ready_counter. Threshold # is dynamic (per-tile valid count) because dispatch does not # pull padding tokens, so the counter's terminal value matches - # the tile's valid_tokens_in_tile (cluster_tile_m for full + # the tile's valid_tokens_in_cta_tile (cluster_tile_m for full # tiles, less for an expert's last partial tile). if cutlass.const_expr(self.fc1_ready_counter_ptr is not None): if is_fc1: counter_ptr = self.fc1_ready_counter_ptr + counter_slot peek_ready = spin_wait( counter_ptr, - lambda v: v >= base_work.valid_tokens_in_tile, + lambda v: v >= base_work.valid_tokens_in_cta_tile, peek_only=True, ) peek_bit = Int32(0) @@ -299,7 +299,7 @@ def enrich_work_tile_info( cumulative_data_physical_row=base_work.cumulative_data_physical_row, cumulative_sf_physical_row=base_work.cumulative_sf_physical_row, cumulative_token_block_count=base_work.cumulative_token_block_count, - valid_tokens_in_tile=base_work.valid_tokens_in_tile, + valid_tokens_in_cta_tile=base_work.valid_tokens_in_cta_tile, phase_and_peek=new_phase_and_peek, ) diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/epilogue_refactor.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/epilogue_refactor.py index 21d41dd488ca..290d7b861f43 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/epilogue_refactor.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/epilogue_refactor.py @@ -1,14 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause """Autonomous epilogue for the fused fc1+fc2 swap-AB MegaMoE kernel. -Component boundaries use ``TensorWithContract`` to keep per-thread RMEM layout -semantics explicit at the handoff between transpose, SwiGLU, quantize, and fc2 -store components. +Per-thread RMEM tensors flow between the transpose / SwiGLU / quantize / fc2 +store steps as bare ``cute.Tensor`` fragments; their thread distribution is a +fixed physical property of the surrounding atom sequence and is documented in +local comments. A ``Contract`` is only kept where it earns its keep: the fc2 +store-out mapping (``Fc2ProcessPipeline.store_out_mapping``) is evaluated at +runtime to drive which token/hidden cell each issue targets and therefore how +the per-token metadata is fetched. """ import dataclasses -from typing import Callable, List, Optional, Tuple, Type, Union +from typing import Any, Callable, List, Literal, Optional, Tuple, Type, Union import cutlass import cutlass.cute as cute @@ -16,162 +22,508 @@ import cutlass.utils as utils import cutlass.utils.blackwell_helpers as sm100_utils from cutlass._mlir import ir -from cutlass._mlir.dialects import llvm +from cutlass._mlir.dialects import arith, llvm, vector from cutlass.cute.nvgpu import cpasync, tcgen05 from cutlass.cute.typing import AddressSpace -from cutlass.cutlass_dsl import dsl_user_op +from cutlass.cutlass_dsl import Float32, Int64, T -from .contract import (Contract, FunctionMapping, Space, TensorWithContract, - assert_contract_equivalent) +from .contract import Contract, FunctionMapping, Space, eval_function_mapping from .fc1_fc2_fuse_sched import BlockPhase +from .flag_batch import GpuReleaseFlagBatchTracker from .iket_compat import iket -from .megamoe_constants import Nvfp4BlockSize +from .megamoe_constants import (Fp8E4M3RcpLimit, Fp32Max, Nvfp4BlockSize, + Nvfp4E2M1RcpLimit) from .moe_persistent_scheduler import (MoESchedConsumer, MoESchedExtension, MoEWorkTileInfo) +from .ptx_helpers import cp_async_bulk_s2g as _cp_async_bulk_s2g +from .ptx_helpers import \ + cp_reduce_async_bulk_add_noftz_bf16_s2g as \ + _cp_reduce_async_bulk_add_noftz_bf16_s2g +from .ptx_helpers import \ + red_add_relaxed_sys_v2_bf16x2 as _red_add_relaxed_sys_v2_bf16x2 from .sym_buffer import SymBufferDeviceBase -from .token_comm import TokenCommArgs +from .token_comm import CombineFormat, TokenCommArgs, TokenSrcMetadata -# ============================================================================= -# Module-local helpers -# ============================================================================= - -@dsl_user_op -def _red_add_relaxed_sys_v2_bf16x2( - addr, - val0_packed_bf16x2, - val1_packed_bf16x2, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, -) -> None: - """Issue ``red.relaxed.sys.global.add.v2.bf16x2 [addr], {v0, v1};``. - - Used by the fc2 REDG path to atomic-add 4 bf16 cells. Inline asm is - used because cuTeDSL has no vector-form ``red.v2.bf16x2`` surface; the - operands are packed bf16x2 bit patterns carried in 32-bit registers. - """ - llvm.inline_asm( - None, - [ - addr.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), - val0_packed_bf16x2.ir_value(loc=loc, ip=ip), - val1_packed_bf16x2.ir_value(loc=loc, ip=ip), - ], - "red.relaxed.sys.global.add.noftz.v2.bf16x2 [$0], {$1, $2};", - "l,r,r", +@cute.jit +def cvt_f32_to_ue8m0_to_f32(fp32x1, loc=None, ip=None): + """Round-trip fp32 -> ue8m0 -> fp32: the E8M0 decode-scale quantize step + (rp/satfinite downcast, then upcast through bf16).""" + src_fp32 = Float32(fp32x1).ir_value(loc=loc, ip=ip) + + asm_tmpl = ("{\n" + " .reg .b16 bf_lo;\n" + " cvt.rp.satfinite.ue8m0x2.f32 bf_lo, 0f00000000, $1;\n" + " cvt.rn.bf16x2.ue8m0x2 $0, bf_lo;\n" + "}") + packed_i32 = llvm.inline_asm( + T.i32(), + [src_fp32], + asm_tmpl, + "=r,f", has_side_effects=True, is_align_stack=False, asm_dialect=llvm.AsmDialect.AD_ATT, - loc=loc, - ip=ip, ) + vec_bf16_ty = ir.Type.parse("vector<2xbf16>") + bf2_lo = llvm.bitcast(vec_bf16_ty, packed_i32, loc=loc, ip=ip) + h0 = vector.extract(bf2_lo, [], [0], loc=loc, ip=ip) + dst_f32 = arith.extf(Float32.mlir_type, h0, loc=loc, ip=ip) + + return dst_f32 -@dsl_user_op -def _red_add_release_gpu_s32( - counter_ptr, - value, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, -) -> None: - """Issue ``red.release.gpu.add.s32`` to a GMEM int32 location. - Publishes fc1 task-tile completion after the caller has flushed the fc1 - output stores. Single-thread helper; caller guards the thread predicate. +@dataclasses.dataclass(frozen=True) +class QuantImpl: + """Register-level block quantizer shared by the fc1 / fc2 epilogues. + + Returns ``(data_regs, sf_regs)`` ONLY: the caller pre-multiplies the topk + weight / global scale into ``prequant_reg`` beforehand and owns the data / + sf plane stores afterwards. ``sf_vec_direction`` selects the per-block amax + reduction: + + * ``regs_in_thread`` -- a block is ``sf_vec`` contiguous regs of + one thread; amax is thread-local (packed bf16x2 abs-max for combine, + fp32 ``fmax`` for orthodox). + * ``threads_with_the_same_reg`` -- a block is one reg across ``sf_vec`` warp + lanes; amax is a warp CREDUX (full warp for vec=32, lane-predicated + halves for vec=16) and is fp32-only, so bf16 is upconverted first. + + ``prequant_reg`` must be 1D with size divisible by ``sf_vec``. Combine inputs + are bf16 (fc2's bf16 reorder regs); orthodox nvfp4 input is fp32 (swiglu). """ - llvm.inline_asm( - None, - [ - counter_ptr.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), - value.ir_value(loc=loc, ip=ip), - ], - "red.release.gpu.global.add.s32 [$0], $1;", - "l,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - loc=loc, - ip=ip, - ) + quant_kind: Union[str, CombineFormat] + sf_vec_direction: Literal["regs_in_thread", "threads_with_the_same_reg"] + lane_idx: Optional[Any] = None # Int32; only the across-lane path needs it -@dsl_user_op -def _cp_async_bulk_s2g( - dst_gmem, - src_smem, - size_bytes, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, -) -> None: - """Issue non-tensor descriptor-free ``cp.async.bulk`` SMEM->GMEM. - - cuTeDSL does expose ``cpasync.CopyBulkS2GOp`` / ``cute.copy`` for this - instruction family, but that abstraction bakes the transfer size into - the copy atom / static tensor layout: CuteNvGPU lowers it as an - ``arch.copy.SM90.bulk_copy_s2g`` op whose ``size`` is an ``I32Attr``. - The fc2 UBLK epilogue needs a runtime byte count for the hidden-tail - row (still 16B-aligned, but not necessarily the full 128-hidden row). - Using the cute copy atom would silently encode the wrong semantic - contract, so keep the raw PTX here until the dialect grows a dynamic-size - descriptor-free bulk-copy op. - - This helper only issues the instruction. The caller owns - ``cp_async_bulk_commit_group`` so copy and reduce bulk paths share the - same group boundary. - """ - # with cute.arch.elect_one(): - llvm.inline_asm( - None, - [ - dst_gmem.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), - src_smem.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), - size_bytes.ir_value(loc=loc, ip=ip), - ], - "cp.async.bulk.global.shared::cta.bulk_group [$0], [$1], $2;", - "l,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - loc=loc, - ip=ip, - ) + # -- config / validation -------------------------------------------------- + + def __post_init__(self): + if isinstance(self.quant_kind, CombineFormat): + if not self.quant_kind.is_quantized: + raise ValueError( + f"QuantImpl combine path needs a quantized CombineFormat, " + f"got {self.quant_kind}.") + elif self.quant_kind != "nvfp4": + raise ValueError(f"quant_kind must be a CombineFormat or 'nvfp4', " + f"got {self.quant_kind!r}.") + if self.sf_vec_direction not in ("regs_in_thread", + "threads_with_the_same_reg"): + raise ValueError( + f"sf_vec_direction must be 'regs_in_thread' or " + f"'threads_with_the_same_reg', got {self.sf_vec_direction!r}.") + # Orthodox nvfp4 only sees the transposed (regs-in-thread) layout fc1 + # produces via its TMEM transpose. + if not isinstance(self.quant_kind, CombineFormat) and ( + self.sf_vec_direction != "regs_in_thread"): + raise NotImplementedError( + "orthodox nvfp4 quant is regs_in_thread only.") + if self.sf_vec_direction == "threads_with_the_same_reg" and ( + self.lane_idx is None): + raise ValueError( + "across-lane quant needs lane_idx for the CREDUX half-warp predicate." + ) + @property + def data_dtype(self): + if isinstance(self.quant_kind, CombineFormat): + return self.quant_kind.act_dtype + return cutlass.Float4E2M1FN # orthodox nvfp4 + + @property + def scale_dtype(self): + if isinstance(self.quant_kind, CombineFormat): + return self.quant_kind.scale_dtype + return cutlass.Float8E4M3FN # orthodox nvfp4 e4m3 sfc + + @property + def sf_vec_size(self) -> int: + if isinstance(self.quant_kind, CombineFormat): + return self.quant_kind.scale_block + return Nvfp4BlockSize # 16 + + @property + def _is_combine(self) -> bool: + return isinstance(self.quant_kind, CombineFormat) + + @property + def _data_rcp_limit(self) -> float: + # 1 / max representable magnitude of the data element type. Only e2m1 + # and e4m3 data planes exist (orthodox nvfp4 + CombineFormat acts). + dt = self.data_dtype + if dt is cutlass.Float4E2M1FN: + return Nvfp4E2M1RcpLimit # 1/6 + assert dt is cutlass.Float8E4M3FN, f"unexpected data dtype {dt}" + return Fp8E4M3RcpLimit # 1/448 + + # -- dispatch ------------------------------------------------------------- -@dsl_user_op -def _cp_reduce_async_bulk_add_noftz_bf16_s2g( - dst_gmem, - src_smem, - size_bytes, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, -) -> None: - """Issue non-tensor ``cp.reduce.async.bulk`` for BF16 add. - - cuTeDSL currently exposes descriptor-free ``CopyBulkS2GOp`` but not the - matching descriptor-free reduce atom. Keep the fallback local to this - epilogue path so the rest of the bulk pipeline can still share the same - tensor/layout front-end. - """ - # with cute.arch.elect_one(): - llvm.inline_asm( - None, - [ - dst_gmem.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), - src_smem.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), - size_bytes.ir_value(loc=loc, ip=ip), - ], - "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.noftz.bf16 [$0], [$1], $2;", - "l,r,r", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - loc=loc, - ip=ip, - ) + @cute.jit + def __call__(self, prequant_reg: cute.Tensor, *, norm_const=None): + if cutlass.const_expr(cute.size(prequant_reg) % self.sf_vec_size != 0): + raise ValueError("prequant_reg size must be divisible by sf_vec.") + # Combine quantizes fc2's bf16 reorder regs; orthodox nvfp4 the fp32 swiglu. + expected_in = cutlass.BFloat16 if self._is_combine else cutlass.Float32 + if cutlass.const_expr(prequant_reg.element_type is not expected_in): + raise TypeError( + f"QuantImpl({self.quant_kind}) expects {expected_in} prequant " + f"input, got {prequant_reg.element_type}.") + if cutlass.const_expr(not self._is_combine): + return self.nvfp4_quant_impl(prequant_reg, norm_const=norm_const) + if cutlass.const_expr(self.data_dtype is cutlass.Float4E2M1FN): + if cutlass.const_expr(self.sf_vec_direction == "regs_in_thread"): + return self.nvfp4_combine_quant_regs_in_thread_impl( + prequant_reg) + return self.nvfp4_combine_quant_threads_with_the_same_reg_impl( + prequant_reg) + if cutlass.const_expr(self.sf_vec_direction == "regs_in_thread"): + return self.mxfp8_combine_quant_regs_in_thread_impl(prequant_reg) + return self.mxfp8_combine_quant_threads_with_the_same_reg_impl( + prequant_reg) + + # -- impls ---------------------------------------------------------------- + + # regs_in_thread only; fc1 promises the vec direction via its TMEM transpose. + @cute.jit + def nvfp4_quant_impl( + self, + prequant_reg: cute.Tensor, + *, + norm_const: Optional[cutlass.Float32] = None, + ) -> Tuple[cute.Tensor, cute.Tensor]: + # fp32 in -> e2m1 data + e4m3 sfc. Mirrors the prior nvfp4_quant scale + # math (sfc -> capped/masked acc_scale); topk pre-mult + sf store are the + # caller's job now. + vec = self.sf_vec_size + n_blocks = cute.size(prequant_reg) // vec + data = cute.make_rmem_tensor((cute.size(prequant_reg), ), + cutlass.Float4E2M1FN) + sf = cute.make_rmem_tensor((n_blocks, ), cutlass.Float8E4M3FN) + in_blocks = cute.zipped_divide(prequant_reg, + (vec, )) # ((vec,), (n_blocks,)) + data_blocks = [] + sf_values = [] + rcp_limit = cutlass.Float32(self._data_rcp_limit) + for vec_block_idx in cutlass.range_constexpr(n_blocks): + block = in_blocks[None, vec_block_idx] + amax = self._amax_thread_fp32(block) + if cutlass.const_expr(norm_const is not None): + sfc_fp32 = amax * rcp_limit * norm_const + else: + sfc_fp32 = amax * rcp_limit + sfc_e4m3 = sfc_fp32.to(cutlass.Float8E4M3FN) + sfc_rt = cutlass.Float32(sfc_e4m3) + if cutlass.const_expr(norm_const is not None): + acc_scale = norm_const * cute.arch.rcp_approx(sfc_rt) + else: + acc_scale = cute.arch.rcp_approx(sfc_rt) + acc_scale = cute.arch.fmin(acc_scale, Fp32Max) + mask = cute.arch.fmin(sfc_rt * cutlass.Float32(1e30), + cutlass.Float32(1.0)) + acc_scale = acc_scale * mask + sf_values.append(sfc_e4m3) + data_blocks.append(self._scale_to_e2m1_ssa(block, acc_scale)) + self._store_packed_blocks(data, data_blocks) + sf.store(self._values_to_ssa(sf_values, cutlass.Float8E4M3FN)) + return data, sf + + @cute.jit + def nvfp4_combine_quant_regs_in_thread_impl( + self, + prequant_reg: cute.Tensor, + ) -> Tuple[cute.Tensor, cute.Tensor]: + # bf16 in -> e2m1 data + per-16 bf16 amax. amax found on bf16 (packed). + vec = self.sf_vec_size + n_blocks = cute.size(prequant_reg) // vec + data = cute.make_rmem_tensor((cute.size(prequant_reg), ), + cutlass.Float4E2M1FN) + sf = cute.make_rmem_tensor((n_blocks, ), cutlass.BFloat16) + in_blocks = cute.zipped_divide(prequant_reg, + (vec, )) # ((vec,), (n_blocks,)) + data_blocks = [] + sf_values = [] + for vec_block_idx in cutlass.range_constexpr(n_blocks): + block = in_blocks[None, vec_block_idx] + amax = self._amax_thread_bf16(block) + sf_values.append(amax) + decode_scale = cutlass.Float32(amax) * cutlass.Float32( + self._data_rcp_limit) + data_blocks.append( + self._scale_to_e2m1_ssa(block, self._enc_nvfp4(decode_scale))) + self._store_packed_blocks(data, data_blocks) + sf.store(self._values_to_ssa(sf_values, cutlass.BFloat16)) + return data, sf + + # Mapping: (lane_idx, selected_sf_idx) -> (token_64, hidden_32) + # token_idx = lane_idx % 16 + selected_sf_idx * 16 + # hidden_idx = lane_idx // 16 * 16 + @cute.jit + def nvfp4_combine_quant_threads_with_the_same_reg_impl( + self, + prequant_reg: cute.Tensor, + ) -> Tuple[cute.Tensor, cute.Tensor]: + # UBLK has lane == hidden, so the warp's 32 lanes are 32 consecutive + # hidden. A scale block = sf_vec hidden, so the lanes split along hidden + # into 32 // sf_vec blocks of sf_vec lanes each (warp = blocks_per_warp * + # lanes_per_block, the EP x TP split). Only the sf_vec lanes inside a + # block share its CREDUX scale, so they pool the subtile tokens: sf_vec + # == 32 pools the whole warp, sf_vec < 32 pools fewer (more per lane). + lanes_per_block = self.sf_vec_size + n_tokens = cute.size(prequant_reg) + lane_in_block = self.lane_idx % cutlass.Int32(lanes_per_block) + data = cute.make_rmem_tensor((n_tokens, ), cutlass.Float4E2M1FN) + selected_sf = cute.make_rmem_tensor((n_tokens // lanes_per_block, ), + cutlass.BFloat16) + scaled_vec = cute.full((n_tokens, ), cutlass.Float32(0.0), + cutlass.Float32) + for token_idx in cutlass.range_constexpr(n_tokens): + value = cutlass.Float32(prequant_reg[token_idx]) + amax_bf16 = self._amax_lane(value).to(cutlass.BFloat16) + slot = token_idx // lanes_per_block + if (token_idx % lanes_per_block) == lane_in_block: + selected_sf[slot] = amax_bf16 + else: + selected_sf[slot] = selected_sf[slot] + decode_scale = cutlass.Float32(amax_bf16) * cutlass.Float32( + self._data_rcp_limit) + scaled_value = value * self._enc_nvfp4(decode_scale) + scaled_vec = cute.TensorSSA( + vector.insert( + scaled_value.ir_value(), + scaled_vec.ir_value(), + [], + [token_idx], + ), + (n_tokens, ), + cutlass.Float32, + ) + self._store_packed_data(data, scaled_vec.to(cutlass.Float4E2M1FN)) + return data, selected_sf + + @cute.jit + def mxfp8_combine_quant_regs_in_thread_impl( + self, + prequant_reg: cute.Tensor, + ) -> Tuple[cute.Tensor, cute.Tensor]: + # bf16 in -> e4m3 data + per-32 e8m0. amax found on bf16 (packed). + vec = self.sf_vec_size + n_blocks = cute.size(prequant_reg) // vec + data = cute.make_rmem_tensor((cute.size(prequant_reg), ), + cutlass.Float8E4M3FN) + sf = cute.make_rmem_tensor((n_blocks, ), cutlass.Float8E8M0FNU) + in_blocks = cute.zipped_divide(prequant_reg, + (vec, )) # ((vec,), (n_blocks,)) + data_blocks = [] + sf_values = [] + for vec_block_idx in cutlass.range_constexpr(n_blocks): + block = in_blocks[None, vec_block_idx] + # widen the native-bf16 amax to fp32 for the e8m0 round-up math. + scale_e8m0, scale_f32 = self._e8m0( + cutlass.Float32(self._amax_thread_bf16(block))) + sf_values.append(scale_e8m0) + data_blocks.append( + self._scale_to_e4m3_ssa(block, self._enc_mxfp8(scale_f32))) + self._store_packed_blocks(data, data_blocks) + sf.store(self._values_to_ssa(sf_values, cutlass.Float8E8M0FNU)) + return data, sf + + # Mapping: (lane_idx, selected_sf_idx) -> (token_64, hidden_32) + # token_idx = lane_idx + selected_sf_idx * 32 + # hidden_idx = 0 + @cute.jit + def mxfp8_combine_quant_threads_with_the_same_reg_impl( + self, + prequant_reg: cute.Tensor, + ) -> Tuple[cute.Tensor, cute.Tensor]: + # UBLK has lane == hidden, so the warp's 32 lanes are 32 consecutive + # hidden. A scale block = sf_vec hidden, so the lanes split along hidden + # into 32 // sf_vec blocks of sf_vec lanes each (warp = blocks_per_warp * + # lanes_per_block, the EP x TP split). Only the sf_vec lanes inside a + # block share its CREDUX scale, so they pool the subtile tokens. mxfp8 + # sf_vec == 32 -> the whole warp is one block, all 32 lanes pool. + lanes_per_block = self.sf_vec_size + n_tokens = cute.size(prequant_reg) + lane_in_block = self.lane_idx % cutlass.Int32(lanes_per_block) + data = cute.make_rmem_tensor((n_tokens, ), cutlass.Float8E4M3FN) + selected_sf = cute.make_rmem_tensor((n_tokens // lanes_per_block, ), + cutlass.Float8E8M0FNU) + scaled_vec = cute.full((n_tokens, ), cutlass.Float32(0.0), + cutlass.Float32) + for token_idx in cutlass.range_constexpr(n_tokens): + value = cutlass.Float32(prequant_reg[token_idx]) + scale_e8m0, scale_f32 = self._e8m0(self._amax_lane(value)) + slot = token_idx // lanes_per_block + if (token_idx % lanes_per_block) == lane_in_block: + selected_sf[slot] = scale_e8m0 + else: + selected_sf[slot] = selected_sf[slot] + scaled_value = value * self._enc_mxfp8(scale_f32) + scaled_vec = cute.TensorSSA( + vector.insert( + scaled_value.ir_value(), + scaled_vec.ir_value(), + [], + [token_idx], + ), + (n_tokens, ), + cutlass.Float32, + ) + self._store_packed_data(data, scaled_vec.to(cutlass.Float8E4M3FN)) + return data, selected_sf + + # -- shared sub-steps ----------------------------------------------------- + + @cute.jit + def _scale_to_e2m1_ssa( + self, + block: cute.Tensor, + enc: cutlass.Float32, + ) -> cute.TensorSSA: + block_f32 = block.load().to(cutlass.Float32) + enc_vec = cute.full_like(block_f32, enc, cutlass.Float32) + return (block_f32 * enc_vec).to(cutlass.Float4E2M1FN) + + @cute.jit + def _scale_to_e4m3_ssa( + self, + block: cute.Tensor, + enc: cutlass.Float32, + ) -> cute.TensorSSA: + block_f32 = block.load().to(cutlass.Float32) + enc_vec = cute.full_like(block_f32, enc, cutlass.Float32) + return (block_f32 * enc_vec).to(cutlass.Float8E4M3FN) + + @cute.jit + def _values_to_ssa(self, values, + dtype: Type[cutlass.Numeric]) -> cute.TensorSSA: + vec = vector.from_elements( + T.vector(len(values), dtype.mlir_type), + [values[i].ir_value() for i in range(len(values))], + ) + return cute.TensorSSA(vec, (len(values), ), dtype) + + @cute.jit + def _concat_i32_blocks_ssa(self, blocks) -> cute.TensorSSA: + values = [] + for block_idx in cutlass.range_constexpr(len(blocks)): + packed_block = blocks[block_idx].bitcast(cutlass.Int32) + for elem_idx in cutlass.range_constexpr( + cute.size(packed_block.shape)): + values.append(packed_block[elem_idx].ir_value()) + vec = vector.from_elements( + T.vector(len(values), cutlass.Int32.mlir_type), + values, + ) + return cute.TensorSSA(vec, (len(values), ), cutlass.Int32) + + @cute.jit + def _store_packed_blocks(self, data: cute.Tensor, blocks) -> None: + packed_data = cute.recast_tensor(data, cutlass.Int32) + packed_data.store(self._concat_i32_blocks_ssa(blocks)) + + @cute.jit + def _store_packed_data(self, data: cute.Tensor, + data_ssa: cute.TensorSSA) -> None: + packed_data = cute.recast_tensor(data, cutlass.Int32) + packed_data.store(data_ssa.bitcast(cutlass.Int32)) + + @cute.jit + def _amax_thread_fp32(self, block: cute.Tensor) -> cutlass.Float32: + # max.xorsign.abs reduces |.| in one op per element; the result sign is + # the xor of the inputs (junk for an amax), so clear it at the end. + def max_abs(lhs: cutlass.Float32, + rhs: cutlass.Float32) -> cutlass.Float32: + return cutlass.Float32( + llvm.inline_asm( + T.f32(), + [ + cutlass.Float32(lhs).ir_value(), + cutlass.Float32(rhs).ir_value() + ], + "max.xorsign.abs.f32 $0, $1, $2;", + "=f,f,f", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + )) + + acc = block[0] + for elem_idx in cutlass.range_constexpr(1, cute.size(block)): + acc = max_abs(acc, block[elem_idx]) + mag_bits = cutlass.Int32( + llvm.bitcast( + T.i32(), + cutlass.Float32(acc).ir_value())) & cutlass.Int32(0x7FFFFFFF) + return cutlass.Float32(llvm.bitcast(T.f32(), mag_bits.ir_value())) + + @cute.jit + def _amax_thread_bf16(self, block: cute.Tensor) -> cutlass.BFloat16: + # Packed bf16x2 abs-max: tree-reduce the pairs, then fold the survivor's + # two halves (high shifted into low). max.xorsign.abs leaves a junk sign, + # so the low bf16 is masked before being read back. The amax is natively + # bf16 -- exactly what the wire format stores. + def max_abs(lhs: cutlass.Int32, rhs: cutlass.Int32) -> cutlass.Int32: + return cutlass.Int32( + llvm.inline_asm( + T.i32(), + [ + cutlass.Int32(lhs).ir_value(), + cutlass.Int32(rhs).ir_value() + ], + "max.xorsign.abs.bf16x2 $0, $1, $2;", + "=r,r,r", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + )) + + pairs = cute.recast_tensor(block, cutlass.Int32) # (vec/2,) bf16x2 + acc = cutlass.Int32(pairs[0]) + for pair_idx in cutlass.range_constexpr(1, cute.size(pairs)): + acc = max_abs(acc, pairs[pair_idx]) + acc = max_abs(acc, acc >> cutlass.Int32(16)) + amax_bits = cute.make_rmem_tensor((1, ), cutlass.Int32) + amax_bits[0] = acc & cutlass.Int32(0x7FFF) + return cute.recast_tensor(amax_bits, cutlass.BFloat16)[0] + + @cute.jit + def _amax_lane(self, v: cutlass.Float32) -> cutlass.Float32: + if cutlass.const_expr(self.sf_vec_size == 32): + return cute.arch.warp_redux_sync(v, "fmax", abs=True) + first_half = (self.lane_idx % cutlass.Int32(32)) < cutlass.Int32(16) + vsel = cutlass.Float32(0.0) + if first_half: + vsel = v + amax = cute.arch.warp_redux_sync(vsel, "fmax", abs=True) + if not first_half: + amax = cute.arch.warp_redux_sync(v, "fmax", abs=True) + return amax + + @cute.jit + def _e8m0( + self, amax: cutlass.Float32 + ) -> Tuple[cutlass.Float8E8M0FNU, cutlass.Float32]: + candidate = amax * cutlass.Float32(self._data_rcp_limit) + scale_f32 = cutlass.Float32(cvt_f32_to_ue8m0_to_f32(candidate)) + return scale_f32.to(cutlass.Float8E8M0FNU), scale_f32 + + @cute.jit + def _enc_nvfp4(self, decode_scale: cutlass.Float32) -> cutlass.Float32: + # rcp.approx.ftz with the fc1 cap+mask idiom (amax==0 -> 0, no inf*0 NaN). + enc = cute.arch.fmin(cute.arch.rcp_approx(decode_scale), Fp32Max) + mask = cute.arch.fmin(decode_scale * cutlass.Float32(1e30), + cutlass.Float32(1.0)) + return enc * mask + + @cute.jit + def _enc_mxfp8(self, scale_f32: cutlass.Float32) -> cutlass.Float32: + # Skip nan + enc = cute.arch.fmin(cute.arch.rcp_approx(scale_f32), Fp32Max) + mask = cute.arch.fmin(scale_f32 * cutlass.Float32(1e30), + cutlass.Float32(1.0)) + return enc * mask # ============================================================================= @@ -192,43 +544,21 @@ class Region: class _TmemTranspose16x32Core: - """Contract-naive physical implementation of the 16x32 -> 32x16 TMEM - in-place transpose. Shared by: - - - ``TmemTranspose16x32`` : fc1 epi codomain naming - (``intermediate_output_idx``); - elements are fp32 (swiglu fold output). - - ``TmemTranspose16x32Packed`` : fc2 epi codomain naming - (``hidden_pair_idx``); elements are - 32-bit packed ``(bf16, bf16)`` pairs. + """Physical implementation of the 16x32 -> 32x16 TMEM in-place transpose. - The (lane_idx, elem_idx) physical distribution is identical for both - subclasses -- the underlying tcgen05 atoms are 32-bit element atoms, - agnostic to whether each 32-bit slot holds an fp32 or a packed bf16x2. - Only the codomain semantic names differ, expressed via the subclass's - ``InputContract`` / ``OutputContract`` class attributes. + The transpose is a fixed sequence of tcgen05 32-bit element atoms; each + 32-bit slot is opaque to it (an fp32 swiglu-fold value for fc1, or a packed + ``(bf16, bf16)`` pair for fc2 -- the physical (lane_idx, elem_idx) + distribution is identical either way). The (thread, reg) -> (tmem_dp, + tmem_col) input / output mapping is documented on the ``TmemTranspose16x32`` + subclass, which is the public entry point. - Per-thread RMEM coordinate convention (used by both subclasses' contracts): + Per-thread RMEM coordinate convention: - ``lane_idx`` -- warp lane id (= thread index within warp), in [0, 32). - ``elem_idx`` -- per-thread reg index, in [0, 16). - - Subclasses MUST override these two class attributes: - ``InputContract`` -- (lane_idx, elem_idx) -> codomain mapping after - R1.Load (or after ``reg_tensor`` is fed in for - skip-R1.Load mode). - ``OutputContract`` -- (lane_idx, elem_idx) -> codomain mapping after - ``r4_perm`` has run all four rounds. - - The Core's ``__init__`` reads ``self.InputContract`` / ``self.OutputContract`` - via Python's normal MRO attribute lookup; the subclass's overrides take - precedence at construction time. """ - # Subclasses MUST override these. - InputContract: Contract - OutputContract: Contract - _PermR1 = (0, 8, 2, 10, 4, 12, 6, 14, 1, 9, 3, 11, 5, 13, 7, 15) _PermR3 = (0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15) _PermR4 = (0, 8, 2, 10, 4, 12, 6, 14, 1, 9, 3, 11, 5, 13, 7, 15) @@ -270,24 +600,19 @@ def load_subtile_raw_acc( ``warp_lane_offset + acc_stage_col_offset + subtile_col_offset``; see ``SwapABSwigluFp4Epilogue._subtile_local_tmem_tensor``). - Returns a 4-tuple of (16,) fp32 RMEM tensors, each carrying - the (lane_idx, elem_idx) -> codomain distribution described by - ``TmemTranspose16x32.InputContract`` / - ``TmemTranspose16x32Packed.InputContract`` (physically identical - for fc1 and fc2, only codomain semantic names differ): + Returns a 4-tuple of (16,) fp32 RMEM tensors, each carrying the + (lane_idx, elem_idx) input distribution documented on + ``TmemTranspose16x32`` (physically identical for fc1 and fc2): [0] gate_lo / first-half top -- subtile cols 0..31, lanes 0..15 [1] up_lo / first-half bot -- subtile cols 0..31, lanes 16..31 [2] raw_top / second-half top -- subtile cols 32..63, lanes 0..15 [3] raw_bot / second-half bot -- subtile cols 32..63, lanes 16..31 - 4 atom calls of ``Ld16x64bOp(Repetition.x16) Float32`` -- the - same atom currently used by the per-subtile entry LDTM in - ``_run_fc1_subtile`` and by ``second_t.r1_load`` / - ``Fc2AccLoadAndPack`` per-half LDTMs. Caller is expected to - wrap each output in ``TensorWithContract`` with - ``TmemTranspose16x32{,Packed}.InputContract`` before handing - them downstream. + 4 atom calls of ``Ld16x64bOp(Repetition.x16) Float32`` -- the same + atom used by the per-subtile entry LDTM. Each output is in the + ``TmemTranspose16x32`` input distribution and can be fed straight + into a transpose as ``reg_tensor`` (skip-R1.Load mode). """ atom_ld16x64 = cute.make_copy_atom( tcgen05.Ld16x64bOp(tcgen05.Repetition.x16), @@ -358,8 +683,18 @@ def __init__( self, tmem_ptr, region: int, - reg_tensor: Optional[TensorWithContract] = None, + reg_tensor: Optional[cute.Tensor] = None, ) -> None: + # The whole transpose is built from 32-bit element atoms; _io_dtype + # drives _src_regs / output / every LDTM/STTM atom below, so guard the + # invariant once here (tautological today, defensive against future + # dtype edits). + if cutlass.const_expr(self._io_dtype.width != 32): + raise TypeError( + f"{type(self).__name__} requires a 32-bit _io_dtype (the " + f"transpose uses 32-bit element atoms), got {self._io_dtype} " + f"(width {self._io_dtype.width}).") + half_lane_off = 16 * self._TmemRowStride if region == Region.Top: src_ptr = tmem_ptr @@ -403,21 +738,31 @@ def __init__( ) self._src_regs = cute.make_rmem_tensor((16, ), self._io_dtype) - output_tensor = cute.make_rmem_tensor((16, ), self._io_dtype) - self.output = TensorWithContract( - tensor=output_tensor, - contract=self.OutputContract, - ) - + # ``output`` is a bare (16,) RMEM fragment; its (lane_idx, elem_idx) + # distribution after all four rounds is the transpose output mapping + # documented on ``TmemTranspose16x32``. + self.output = cute.make_rmem_tensor((16, ), self._io_dtype) + + # skip-R1.Load mode: ``reg_tensor`` must already be in the transpose + # input distribution (see ``TmemTranspose16x32`` / produced by + # ``load_subtile_raw_acc``); we copy it in lieu of the R1 LDTM. + # Weak entry guard (replaces the removed input contract): the transpose + # atoms are 32-bit element atoms over exactly 16 regs/lane, so the fed + # tensor must be a 32-bit element type (fp32 or packed bf16x2) of size 16. self._reg_tensor = reg_tensor if reg_tensor is not None: - assert_contract_equivalent( - reg_tensor.contract, - self.InputContract, - context=f"{type(self).__name__} skip-R1.Load reg_tensor", - ) + if cutlass.const_expr(reg_tensor.element_type.width != 32): + raise TypeError( + f"{type(self).__name__} reg_tensor must be a 32-bit element " + f"type (fp32 or packed bf16x2), got element type " + f"{reg_tensor.element_type} (width {reg_tensor.element_type.width})." + ) + if cutlass.const_expr(cute.size(reg_tensor) != 16): + raise ValueError( + f"{type(self).__name__} reg_tensor must hold exactly 16 " + f"elements, got {cute.size(reg_tensor)}.") for r in range(16): - self._src_regs[r] = reg_tensor.tensor[r] + self._src_regs[r] = reg_tensor[r] # -- R1 ------------------------------------------------------------------ @@ -433,12 +778,12 @@ def r1_load(self) -> None: def r1_perm(self) -> None: for r in range(16): - self.output.tensor[r] = self._src_regs[self._PermR1[r]] + self.output[r] = self._src_regs[self._PermR1[r]] def r1_store(self) -> None: cute.copy( self._atom_st16x128, - self._rmem_copy_view(self.output.tensor, 16), + self._rmem_copy_view(self.output, 16), self._tmem_src_full, ) @@ -476,12 +821,12 @@ def r3_load_bot(self) -> None: def r3_perm(self) -> None: for r in range(16): - self.output.tensor[r] = self._src_regs[self._PermR3[r]] + self.output[r] = self._src_regs[self._PermR3[r]] def r3_store(self) -> None: cute.copy( self._atom_st32x32, - self._rmem_copy_view(self.output.tensor, 16), + self._rmem_copy_view(self.output, 16), self._tmem_dst_full, ) @@ -503,12 +848,12 @@ def r4_load_bot(self) -> None: def r4_perm(self) -> None: for r in range(16): - self.output.tensor[r] = self._src_regs[self._PermR4[r]] + self.output[r] = self._src_regs[self._PermR4[r]] def r4_store(self) -> None: cute.copy( self._atom_st32x32, - self._rmem_copy_view(self.output.tensor, 16), + self._rmem_copy_view(self.output, 16), self._tmem_dst_full, ) @@ -528,65 +873,29 @@ def from_r1_perm_until_last_store(self) -> cute.Tensor: class TmemTranspose16x32(_TmemTranspose16x32Core): - """fc1 epi 16x32 -> 32x16 TMEM in-place transpose. - - Contract summary: - - input : ``token_idx = elem_idx * 2 + ((lane_idx // 2) % 2)`` - - output: ``token_idx = lane_idx`` - The second codomain axis is ``intermediate_output_idx``. - """ + """Public 16x32 -> 32x16 TMEM in-place transpose. - _domain = Space(("lane_idx", "elem_idx"), (32, 16)) - _codomain = Space(("token_idx", "intermediate_output_idx"), (32, 16)) + The per-thread RMEM ``(lane_idx, elem_idx) -> (tmem_dp, tmem_col)`` mapping + is fixed by the underlying atom sequence and is identical for fc1 (each slot + is an fp32 swiglu-fold value, ``tmem_col`` = intermediate-output index) and + fc2 (each slot is a packed bf16x2, ``tmem_col`` = hidden-pair index). Only + the ``tmem_col`` semantic name differs between the two uses; the physical + distribution below is the single source of truth. - InputContract = Contract( - domain=_domain, - codomain=_codomain, - mapping=FunctionMapping( - lambda lane_idx, elem_idx: { - "token_idx": elem_idx * 2 + ((lane_idx // 2) % 2), - "intermediate_output_idx": (lane_idx % 2) * 8 + lane_idx // 4, - }), - ) - OutputContract = Contract( - domain=_domain, - codomain=_codomain, - mapping=FunctionMapping(lambda lane_idx, elem_idx: { - "token_idx": lane_idx, - "intermediate_output_idx": elem_idx, - }), - ) + Input distribution -- what each (lane_idx, elem_idx) reg holds on entry + (i.e. straight after the 16-dp x 32-col source LDTM, or as fed in via + ``reg_tensor`` / ``load_subtile_raw_acc`` for skip-R1.Load mode): + tmem_dp = elem_idx * 2 + (lane_idx // 2) % 2 # in [0, 32) + tmem_col = (lane_idx % 2) * 8 + lane_idx // 4 # in [0, 16) -class TmemTranspose16x32Packed(_TmemTranspose16x32Core): - """fc2 epi 16x32 -> 32x16 TMEM in-place transpose, 32-bit packed - bf16x2 elements. + Output distribution -- after all four rounds, the 32-dp x 16-col result has + each lane owning one full dp-row of 16 cols: - Same physical atom sequence as ``TmemTranspose16x32``; codomain is - ``(token_idx, hidden_pair_idx)`` and each slot holds one packed bf16x2. + tmem_dp = lane_idx # in [0, 32) + tmem_col = elem_idx # in [0, 16) """ - _domain = Space(("lane_idx", "elem_idx"), (32, 16)) - _codomain = Space(("token_idx", "hidden_pair_idx"), (32, 16)) - - InputContract = Contract( - domain=_domain, - codomain=_codomain, - mapping=FunctionMapping( - lambda lane_idx, elem_idx: { - "token_idx": elem_idx * 2 + ((lane_idx // 2) % 2), - "hidden_pair_idx": (lane_idx % 2) * 8 + lane_idx // 4, - }), - ) - OutputContract = Contract( - domain=_domain, - codomain=_codomain, - mapping=FunctionMapping(lambda lane_idx, elem_idx: { - "token_idx": lane_idx, - "hidden_pair_idx": elem_idx, - }), - ) - # ============================================================================= # TmemTranspose32x32Inplace @@ -604,8 +913,8 @@ class TmemTranspose32x32Inplace: def __init__( self, tmem_ptr, - reg_tensor_top: Optional[TensorWithContract] = None, - reg_tensor_bot: Optional[TensorWithContract] = None, + reg_tensor_top: Optional[cute.Tensor] = None, + reg_tensor_bot: Optional[cute.Tensor] = None, ) -> None: if (reg_tensor_top is None) != (reg_tensor_bot is None): raise ValueError( @@ -693,7 +1002,8 @@ def __init__( use_2cta_instrs: bool, sf_vec_size: int, fc1_output_dtype: Type[cutlass.Numeric], - fc2_output_dtype: Type[cutlass.Numeric], + combine_format: + CombineFormat, # fc2 combine wire: act_dtype (data) + scale_dtype (sf) non_ubulk_fc2_store: bool, # Whether epilogue warps use STG or UBLK in fc2 in_kernel_fc2_reduce: @@ -702,13 +1012,12 @@ def __init__( bool = False, # Whether epilogue warps store fc2 to local or peer acc_dtype: Type[cutlass.Numeric] = cutlass.Float32, fc1_output_sf_dtype: Type[cutlass.Numeric] = cutlass.Float8E4M3FN, - fc2_output_sf_dtype: Optional[Type[ - cutlass. - Numeric]] = None, # Reserve for later low precision combine allow_overlap_acc: bool = True, static_expert_shape: Optional[Tuple[ int, int, int]] = None, # [expert, intermediate, hidden] gate_up_clamp: Optional[float] = None, # Swiglu style only + epi_flag_batch: Optional[Tuple[int, int]] = ( + 1, 1), # (fc1, fc2) done-counter publish batch ) -> None: if fc1_output_dtype is not cutlass.Float4E2M1FN: raise NotImplementedError( @@ -726,30 +1035,34 @@ def __init__( self.fc2_use_bulk = not non_ubulk_fc2_store self.reduce_topk_in_kernel = in_kernel_fc2_reduce self.token_back_by_dispatch = token_back_by_dispatch - self.fc2_output_dtype = fc2_output_dtype + self.combine_format = combine_format self.fc1_output_dtype = fc1_output_dtype self.acc_dtype = acc_dtype self.fc1_output_sf_dtype = fc1_output_sf_dtype self.sf_vec_size = sf_vec_size # Swiglu gate/up clamp limit; None disables clamping. self.gate_up_clamp = gate_up_clamp - self.cluster_tile_intermediate_downproj = ( - self._EpilogueFc1IntermediateDownTileSize * cluster_shape_mn[0]) + # Done-counter publish batch granularity + _fc1_eb, _fc2_eb = (1, 1) if epi_flag_batch is None else epi_flag_batch + self.fc1_epi_flag_batch = max(1, min(32, int(_fc1_eb))) + self.fc2_epi_flag_batch = max(1, min(32, int(_fc2_eb))) + self.cluster_tile_intermediate_downproj = self._EpilogueFc1IntermediateDownTileSize * cluster_shape_mn[ + 0] atom_thr_size = 2 if use_2cta_instrs else 1 self.cta_tile_m = self._EpilogueFc2HiddenTileSize self.cta_tile_n = mma_tiler_mnk[1] self.cta_tile_k = mma_tiler_mnk[2] - assert mma_tiler_mnk[0] // atom_thr_size == self.cta_tile_m - assert self.cta_tile_n % self._EpilogueTokenTileSize == 0 + assert (mma_tiler_mnk[0] // atom_thr_size == self.cta_tile_m) + assert (self.cta_tile_n % self._EpilogueTokenTileSize == 0) self.static_expert_shape = static_expert_shape self.acc_tmem_cols = self.cta_tile_n self.acc_sf_cols = (max(self.cta_tile_n // 128, 1) * self.cta_tile_k + max(self.cta_tile_m // 128, 1) * self.cta_tile_k) // self.sf_vec_size - if (static_expert_shape is not None and static_expert_shape[2] % - (self.cta_tile_m * cluster_shape_mn[0]) == 0): + if static_expert_shape is not None and static_expert_shape[2] % ( + self.cta_tile_m * cluster_shape_mn[0]) == 0: self.fc2_hidden_needs_predicate: bool = False else: self.fc2_hidden_needs_predicate: bool = True @@ -766,19 +1079,17 @@ def __init__( self.num_acc_stage = 2 self.num_acc_pipeline_stages = 1 if self.overlapping_accum else self.num_acc_stage self.overlapped_tmem_cols = self._EpilogueTokenTileSize if self.overlapping_accum else 0 - assert not self.overlapping_accum or self.overlapped_tmem_cols >= self.acc_sf_cols + assert (not self.overlapping_accum + or self.overlapped_tmem_cols >= self.acc_sf_cols) self.epi_smem_bytes = 8 * 1024 if self.fc1_output_dtype.width > 4: raise NotImplementedError( "Remember to adjust the smem size when switch to mxfp8 support") - self.tmem_acc_layout_py_obj = ( - (self.cta_tile_m, self.cta_tile_n, self.num_acc_stage), - ( - _TmemTranspose16x32Core._TmemRowStride, - 1, - self.cta_tile_n - self.overlapped_tmem_cols, - ), - ) + self.tmem_acc_layout_py_obj = ((self.cta_tile_m, self.cta_tile_n, + self.num_acc_stage), + (_TmemTranspose16x32Core._TmemRowStride, + 1, self.cta_tile_n - + self.overlapped_tmem_cols)) def get_epi_storage_type(self) -> Type: # This could be extended to take atoms space for the larger sf_vec_size quant. @@ -841,17 +1152,10 @@ def run( ), ) - fc1_epi = SwapABFc1Epilogue( - self, - tidx, - epi_smem_storage, - sched_ext, - tma_atom_fc1_output, - fc1_output, - fc1_output_sf, - fc1_done_counter, - optional_epi_args, - ) + fc1_epi = SwapABFc1Epilogue(self, tidx, epi_smem_storage, sched_ext, + tma_atom_fc1_output, fc1_output, + fc1_output_sf, fc1_done_counter, + optional_epi_args) fc2_epi = SwapABFc2Epilogue(self, tidx, epi_smem_storage, fc2_output, token_comm_args, optional_epi_args) @@ -862,8 +1166,15 @@ def run( num_threads=32 * self._EpilogueWarpCnt, ) is_odd_turn = cutlass.Int32(1) - work_tile_info = sched_consumer.consume_work() + + flag_tracker = GpuReleaseFlagBatchTracker( + flag_addr=Int64(0), + cumulated_flags=cutlass.Int32(0), + phase=cutlass.Int32(work_tile_info.phase), + tid=tidx % (self._EpilogueWarpCnt * 32), + ) + while work_tile_info.is_valid_tile: if cutlass.const_expr(self.overlapping_accum): tmem_stage_idx = acc_consumer_state.phase @@ -904,20 +1215,37 @@ def run( if cur_was_linear1: cute.arch.cp_async_bulk_commit_group() cute.arch.cp_async_bulk_wait_group(0, read=True) - cute.arch.fence_acq_rel_gpu() - elif cutlass.const_expr(self.token_back_by_dispatch): - cute.arch.fence_acq_rel_gpu() wait_only_named_barrier.arrive_and_wait() # Publish completion for the work tile snapshotted above. if cur_was_linear1: - fc1_epi.signal_fc1_done(prev_work_tile_info) + flag_tracker = fc1_epi.signal_fc1_done(prev_work_tile_info, + work_tile_info, + flag_tracker) else: - fc2_epi.signal_fc2_done(prev_work_tile_info) + flag_tracker = fc2_epi.signal_fc2_done(prev_work_tile_info, + work_tile_info, + flag_tracker) + # Tail flush + flag_tracker.fire() + + +class _ImmutableAfterInit: + """Froze at the point calling `_freeze()`""" + + def __setattr__(self, name, value): + if self.__dict__.get("_frozen_", False): + raise AttributeError( + f"{type(self).__name__} is immutable after __init__ " + f"(cannot set {name!r}).") + object.__setattr__(self, name, value) + + def _freeze(self) -> None: + object.__setattr__(self, "_frozen_", True) # Device only object -class SwapABFc1Epilogue: +class SwapABFc1Epilogue(_ImmutableAfterInit): def __init__( self, @@ -953,6 +1281,7 @@ def __init__( self.fc1_output_sf = fc1_output_sf self.fc1_done_counter = fc1_done_counter self.optional_epi_args = optional_epi_args + self._freeze() def __getattr__(self, name): return getattr(object.__getattribute__(self, "base"), name) @@ -971,8 +1300,9 @@ def __new_from_mlir_values__(self, return self @cute.jit - def signal_fc1_done(self, work_tile_info): - # Only in-bound intermediate_downproj tiles signal + def signal_fc1_done(self, work_tile_info, next_work_tile_info, + flag_tracker): + # Only in-bound intermediate_downproj tiles signal; OOB -> null slot. if cutlass.const_expr(self.static_expert_shape is None or self.intermediate_downproj % self.cluster_tile_intermediate_downproj != 0): @@ -981,23 +1311,25 @@ def signal_fc1_done(self, work_tile_info): < self.fc1_output.shape[1]) else: in_bound = True + slot = (work_tile_info.cumulative_token_block_count + + work_tile_info.tile_n_idx) + flag_addr = Int64(0) if in_bound: - if self.tidx == 0: - slot = work_tile_info.cumulative_token_block_count + work_tile_info.tile_n_idx - _red_add_release_gpu_s32( - self.fc1_done_counter.iterator + slot, - cutlass.Int32(1), - ) + flag_addr = (self.fc1_done_counter.iterator + slot).toint() + return flag_tracker.accumulate( + next_work_tile_info.phase, + self.fc1_epi_flag_batch, + flag_addr, + ) @cute.jit def __call__( - self, - work_tile_info: MoEWorkTileInfo, - tmem_acc_tensor: cute.Tensor, # (cta_tile_m, cta_tile_n) - acc_pipeline, - acc_consumer_state, - is_odd_turn: cutlass.Int32, - ): + self, + work_tile_info: MoEWorkTileInfo, + tmem_acc_tensor: cute.Tensor, # (cta_tile_m, cta_tile_n) + acc_pipeline, + acc_consumer_state, + is_odd_turn: cutlass.Int32): # (tokens_this_expert, intermediate_down, 1) real_fc1_output, _ = self.sched_ext.get_gmem_tensor( "c", @@ -1024,14 +1356,12 @@ def __call__( norm_const = None # (cta_tile_m, cta_tile_n) -> (epi_tile_m, epi_tile_n, iters) tmem_acc_tensor_tiled_by_epi_tile = cute.flat_divide( - tmem_acc_tensor, - (self._EpilogueFc1IntermediateGateUpTileSize, - self._EpilogueTokenTileSize), - )[None, None, 0, None] + tmem_acc_tensor, (self._EpilogueFc1IntermediateGateUpTileSize, + self._EpilogueTokenTileSize))[None, None, 0, None] acc_pipeline.consumer_wait(acc_consumer_state) iket.range_push("fc1_epi") - valid_tokens = work_tile_info.valid_tokens_in_tile + valid_tokens = work_tile_info.valid_tokens_in_cta_tile # Overlap path preloads two subtiles before releasing acc TMEM. unroll_tile_cnt = 2 if cutlass.const_expr(self.overlapping_accum) else 0 @@ -1055,9 +1385,9 @@ def __call__( # the tmem transpose consumes them. preload_subtile_first: Tuple[ cute.Tensor, cute.Tensor, cute.Tensor, - cute.Tensor] = (_TmemTranspose16x32Core.load_subtile_raw_acc( + cute.Tensor] = _TmemTranspose16x32Core.load_subtile_raw_acc( tmem_acc_tensor_tiled_by_epi_tile[None, None, - subtile_idx_first])) + subtile_idx_first]) # Release acc to next MMA unconditionally. cute.arch.fence_view_async_tmem_load() @@ -1068,9 +1398,9 @@ def __call__( # quadrant/offset invariants and opaque per-lane layout as above. preload_subtile_second: Tuple[ cute.Tensor, cute.Tensor, cute.Tensor, - cute.Tensor] = (_TmemTranspose16x32Core.load_subtile_raw_acc( + cute.Tensor] = _TmemTranspose16x32Core.load_subtile_raw_acc( tmem_acc_tensor_tiled_by_epi_tile[None, None, - subtile_idx_second])) + subtile_idx_second]) # Both unrolled subtiles borrow tmem_subtile_second as workspace. preload_pair = (preload_subtile_first, preload_subtile_second) @@ -1153,13 +1483,10 @@ def run_subtile( work_tile_info.tile_n_idx * self.cta_tile_n + subtile_idx * self._EpilogueTokenTileSize + self.lane_idx, work_tile_info.tile_n_idx * self.cta_tile_n + - subtile_idx * self._EpilogueTokenTileSize + self.lane_idx + 32, - ) + subtile_idx * self._EpilogueTokenTileSize + self.lane_idx + 32) if cutlass.const_expr(topk_score_tensor is not None): - topk_scores = ( - topk_score_tensor[current_two_token_idices[0]], - topk_score_tensor[current_two_token_idices[1]], - ) + topk_scores = (topk_score_tensor[current_two_token_idices[0]], + topk_score_tensor[current_two_token_idices[1]]) else: topk_scores = None @@ -1213,52 +1540,44 @@ def run_subtile( token_0_32_pre_quant_pre_trans = self.alpha_swiglu_clamp( gate_token_0_32, up_token_0_32, alpha_val) + # gate_token_32_64 / up_token_32_64 are already in the transpose input + # distribution (see TmemTranspose16x32 / load_subtile_raw_acc). token_32_64_tmem_trans = TmemTranspose32x32Inplace( tmem_subtile_tensor.iterator, - reg_tensor_top=TensorWithContract( - tensor=gate_token_32_64, - contract=TmemTranspose16x32.InputContract, - ), - reg_tensor_bot=TensorWithContract( - tensor=up_token_32_64, - contract=TmemTranspose16x32.InputContract, - ), + reg_tensor_top=gate_token_32_64, + reg_tensor_bot=up_token_32_64, ) - # (epi_tid, vid) -> (token_idx, intermediate_output_idx), each lane hold (token_1, intermediate_16) in this rmem tensor. - gate_token_32_64_trans_pre_act, up_token_32_64_trans_pre_act = ( - token_32_64_tmem_trans.from_r1_perm_until_last_store()) + # Transpose output: each lane holds (token_1, intermediate_16); tmem_dp + # = lane_idx (token), tmem_col = elem_idx (intermediate output idx). + gate_token_32_64_trans_pre_act, up_token_32_64_trans_pre_act = token_32_64_tmem_trans.from_r1_perm_until_last_store( + ) token_32_64_pre_quant = self.alpha_swiglu_clamp( - gate_token_32_64_trans_pre_act.tensor, - up_token_32_64_trans_pre_act.tensor, + gate_token_32_64_trans_pre_act, + up_token_32_64_trans_pre_act, alpha_val, ) token_0_32_tmem_trans = TmemTranspose16x32( tmem_subtile_tensor.iterator, Region.Top, - reg_tensor=TensorWithContract( - tensor=token_0_32_pre_quant_pre_trans, - contract=TmemTranspose16x32.InputContract, - ), + reg_tensor=token_0_32_pre_quant_pre_trans, ) token_0_32_pre_quant = token_0_32_tmem_trans.from_r1_perm_until_last_store( - ).tensor + ) # Step 2: Quant - self.nvfp4_quant( - work_tile_info=work_tile_info, - two_token=(token_0_32_pre_quant, token_32_64_pre_quant), - topk_scores=topk_scores, - norm_const=norm_const, - intermediate_output_size=cute.size(fc1_output, 1), - fc1_output_sf=fc1_output_sf, - subtile_idx=subtile_idx, - ) + self.nvfp4_quant(work_tile_info=work_tile_info, + two_token=(token_0_32_pre_quant, + token_32_64_pre_quant), + topk_scores=topk_scores, + norm_const=norm_const, + intermediate_output_size=cute.size(fc1_output, 1), + fc1_output_sf=fc1_output_sf, + subtile_idx=subtile_idx) # Step 3: TMASTG - cute.arch.fence_proxy("async.shared", space="cta") # (token_64, intermeidate_64) fc1_smem = self.smem_tensor[None, None, subtile_idx] # (token, intermediate_down, l=1) -> (cta_token, cta_intermediate_down) @@ -1270,8 +1589,8 @@ def run_subtile( fc1_gmem_subtile_view = cute.flat_divide( fc1_gmem_cta_view, (self._EpilogueTokenTileSize, - self._EpilogueFc1IntermediateDownTileSize), - )[None, None, subtile_idx, 0] + self._EpilogueFc1IntermediateDownTileSize))[None, None, + subtile_idx, 0] tma_smem_src, tma_gmem_dst = cpasync.tma_partition( self.fc1_tma_atom, 0, @@ -1286,9 +1605,11 @@ def run_subtile( barrier_id=subtile_bar_id, num_threads=self._EpilogueWarpCnt * 32, ) + cute.arch.fence_proxy("async.shared", space="cta") if self.warp_idx == subtile_idx: tma_ready_to_read_smem_named_barrier.arrive_and_wait() with cute.arch.elect_one(): + # if work_tile_info.tile_m_idx * (self.cta_tile_m // 2) < cute.size(fc1_output, 1): cute.copy(self.fc1_tma_atom, tma_smem_src, tma_gmem_dst) else: tma_ready_to_read_smem_named_barrier.arrive() @@ -1300,7 +1621,7 @@ def alpha_swiglu_clamp( Tensor, # Raw fc1 acc (pre-dequant); even-size 1D fp32 rmem up_rmem: cute. Tensor, # Raw fc1 acc (pre-dequant); even-size 1D fp32 rmem - alpha_val: Optional[cutlass.Float32], + alpha_val: Optional[cutlass.Float32] ) -> cute.Tensor: # ── Input contract checks (compile-time): fp32, 1D, even-count, rmem ── # Wrapped in const_expr so the DSL evaluates them at trace time and the @@ -1443,15 +1764,12 @@ def nvfp4_quant( intermediate_output_size: cutlass.Int32, fc1_output_sf: cute. Tensor, # MoE domain (token_this_rank, intermediate_down, 1) - subtile_idx: cutlass.Int32, - ): - _Nvfp4RcpLimit = 1.0 / 6.0 # 1 / max abs of Float4E2M1FN (= 6.0) - _Fp32Max = 3.40282346638528859812e38 + subtile_idx: cutlass.Int32): # ``two_token`` are the two post-swiglu, transposed token rmem tensors; # each lane holds one token's ``sf_vec_size`` (=16, one NVFP4 SF block) # intermediate-output values. half 0 -> token (lane), half 1 -> (lane+32). # - # Per token (ported from PostSwigluHalf._gen_sfc_quantize + stg_sfc + r2s): + # Per token: # 1. (Path A) pre-multiply topk weight into the values, if present. # 2. absmax over the (weighted) block. # 3. sfc = absmax * (1/6) * norm_const -> E4M3 scale factor. @@ -1464,8 +1782,10 @@ def nvfp4_quant( # norm_const is treated like alpha_val: None => behaves as 1.0 (factors # const-elided, not multiplied by 1.0). n = cute.size(two_token[0]) - rcp_limit = cutlass.Float32(_Nvfp4RcpLimit) - fp32_max = cutlass.Float32(_Fp32Max) + # The core block quant (amax + e4m3 sfc + capped/masked acc_scale + e2m1 + # cvt) is QuantImpl's job; this method still owns the topk pre-multiply, + # the sfc store, and the STS.64 into the shared output stage. + quant = QuantImpl("nvfp4", "regs_in_thread") intermediate_idx = (work_tile_info.tile_m_idx * (self.cta_tile_m // 2) + self.warp_idx * Nvfp4BlockSize) @@ -1505,39 +1825,12 @@ def nvfp4_quant( for i in cutlass.range_constexpr(0, n): weighted[i] = tok[i] - # 2) absmax over the block. - absmax = cutlass.Float32(0.0) - for i in cutlass.range_constexpr(0, n): - v = weighted[i] - absmax = cute.arch.fmax(absmax, cute.arch.fmax(v, -v)) - - # 3) scale factor. - if cutlass.const_expr(norm_const is not None): - sfc_fp32 = absmax * rcp_limit * norm_const - else: - sfc_fp32 = absmax * rcp_limit - sfc_e4m3 = sfc_fp32.to(self.fc1_output_sf_dtype) - sfc_rt = cutlass.Float32(sfc_e4m3) - - # 4) acc_scale = norm_const * rcp(sfc), capped, with sfc==0 guard. - if cutlass.const_expr(norm_const is not None): - acc_scale = norm_const * cute.arch.rcp_approx(sfc_rt) - else: - acc_scale = cute.arch.rcp_approx(sfc_rt) - acc_scale = cute.arch.fmin(acc_scale, fp32_max) - mask = cute.arch.fmin(sfc_rt * cutlass.Float32(1e30), - cutlass.Float32(1.0)) - acc_scale = acc_scale * mask - - scaled = cute.make_rmem_tensor((n, ), cutlass.Float32) - acc_scale_pair = (acc_scale, acc_scale) - for i in cutlass.range_constexpr(0, n, 2): - s0, s1 = cute.arch.mul_packed_f32x2( - (weighted[i], weighted[i + 1]), acc_scale_pair) - scaled[i] = s0 - scaled[i + 1] = s1 + # 2) Core block quant: amax + e4m3 sfc + capped/masked acc_scale + + # e2m1 cvt. One 16-wide block -> one e2m1 reg tensor + one sfc. + fp4_regs, sfc_regs = quant(weighted, norm_const=norm_const) + sfc_e4m3 = sfc_regs[0] - # 5) scale-factor store (predicate const-elided when statically + # 3) scale-factor store (predicate const-elided when statically # in-bound, mirroring signal_fc1_done's intermediate predicate). if cutlass.const_expr(self.static_expert_shape is None or self.intermediate_downproj % @@ -1549,9 +1842,7 @@ def nvfp4_quant( fc1_output_sf[token_idx_pair[half], intermediate_idx, 0] = sfc_e4m3 - # 6) NVFP4 cvt + STS.64 into this subtile's shared output stage. - fp4_regs = cute.make_rmem_tensor((n, ), cutlass.Float4E2M1FN) - fp4_regs.store(scaled.load().to(cutlass.Float4E2M1FN)) + # 4) STS.64 the e2m1 into this subtile's shared output stage. # ((1, 16), (token_tile_size, warp_cnt)) -> (16) smem_thread_row = smem_tiled[(0, None), (self.lane_idx + 32 * half, self.warp_idx)] @@ -1562,79 +1853,396 @@ def nvfp4_quant( ) -""" -Acc to pre-store process: - some kind of ldtm -> f2fp -> some kind of reorder - -Pre-store status: - (epi_tid, vid) -> rmem x (token, hidden) +@dataclasses.dataclass(frozen=True) +class Fc2ProcessPipeline(): + tmem_acc_load: Callable + f2fp: Callable + post_f2fp_reorder: Callable + store_function: Callable + # Kept as a finer-grained, elem-level reading aid for the store-out layout + # (never evaluated); ``store_out_mapping`` is the per-issue form that the + # router actually evaluates at runtime to drive metadata / pointer math. + fc2_cta_tile_contract: Contract + store_out_mapping: Contract # data plane, per-issue + require_tmem_trans: bool + # SF plane per-issue mapping; None for the bf16 (unquantized) paths. + sf_store_out_mapping: Optional[Contract] = None -Store process: - Mapping: (epi_tid, iter_idx) -> (token, topk, hidden). - This defines in each sending, which thread(s) send which part to which dst. However, this is impl-irrevalent. - Always starts at rmem x (token, hidden)? -""" +# Device only object +class SwapABFc2Epilogue(_ImmutableAfterInit): -def eval_function_mapping(contract: Contract, **domain_coord): - """Evaluate a FunctionMapping contract at runtime. + def __init__( + self, + base: SwapABSwigluFp4Epilogue, + tidx: cutlass.Int32, + epi_smem_storage, + fc2_output: cute.Tensor, # MoE domain (token, topk, hidden) + token_comm_args: TokenCommArgs, + optional_epi_args: NvFp4OptinalEpiArgs, + ): + self.base = base + self.tidx = tidx % (base._EpilogueWarpCnt * 32) + self.warp_idx = self.tidx // 32 + self.lane_idx = self.tidx % 32 + self.fc2_output = fc2_output + self.token_comm_args = token_comm_args + self.optional_epi_args = optional_epi_args + if cutlass.const_expr(base.fc2_use_bulk): + wire_dtype = base.combine_format.act_dtype + fc2_smem_rows = 32 + if cutlass.const_expr(fc2_smem_rows * + base._EpilogueFc2HiddenTileSize * + wire_dtype.width // 8 > base.epi_smem_bytes): + raise ValueError("fc2 UBLK data smem exceeds epi_smem budget.") + self.smem_tensor = cute.make_tensor( + cute.recast_ptr( + epi_smem_storage.epi_smem.data_ptr(), + dtype=wire_dtype, + ), + cute.make_layout( + (fc2_smem_rows, base._EpilogueFc2HiddenTileSize), + stride=(base._EpilogueFc2HiddenTileSize, 1), + ), + ) + self.process_pipeline = make_fc2_ublk_process_pipeline( + combine_format=base.combine_format, + cta_token_tile_size=base.cta_tile_n, + cta_hidden_tile_size=base.cta_tile_m, + ) + else: + self.smem_tensor = None + if cutlass.const_expr(base.reduce_topk_in_kernel): + self.process_pipeline = make_fc2_redg_process_pipeline( + combine_format=base.combine_format, + cta_token_tile_size=base.cta_tile_n, + cta_hidden_tile_size=base.cta_tile_m, + ) + else: + self.process_pipeline = make_fc2_stg_process_pipeline( + combine_format=base.combine_format, + cta_token_tile_size=base.cta_tile_n, + cta_hidden_tile_size=base.cta_tile_m, + ) + self._freeze() - This is intentionally local to the fc2 epilogue refactor for now. It only - supports FunctionMapping-backed contracts whose Python function can run in - CuTe tracing context; table-backed runtime eval can be designed later in - contract.py. - """ - if not isinstance(contract.mapping, FunctionMapping): - raise TypeError("runtime contract eval requires a FunctionMapping") - - result = contract.mapping.function(**domain_coord) - if isinstance(result, dict): - return result - if isinstance(result, (tuple, list)): - if len(result) != contract.codomain.rank: - raise ValueError( - "FunctionMapping result rank does not match codomain rank: " - f"{len(result)} vs {contract.codomain.rank}") - return { - name: result[i] - for i, name in enumerate(contract.codomain.names) - } - if contract.codomain.rank == 1: - return {contract.codomain.names[0]: result} - raise TypeError( - "FunctionMapping runtime eval must return dict/tuple/list, or scalar for rank-1 codomain" - ) + def __getattr__(self, name): + return getattr(object.__getattribute__(self, "base"), name) + def __extract_mlir_values__(self) -> List[ir.Value]: + # See SwapABFc1Epilogue.__extract_mlir_values__: this helper carries + # only loop-invariant Python context. It intentionally serializes no + # MLIR values, so changing it to store loop-carried state would be a + # correctness bug. + return [] -@dataclasses.dataclass(frozen=True) -class Fc2OutputRouter: - # (token, 3), 3 -> rank_idx, token_idx, top_k - # Later this will be changed to (token, 2), where top_k and rank_idx will be fused into 32bit. - # If metadata is None then this is a local write. - metadata: Optional[cute.Tensor] - direct_token_base_this_cta_tile: Optional[cutlass.Int32] - base_output: cute.Tensor # (token, topk, hidden) - hidden_base_this_cta_tile: Union[cutlass.Int32, int] - peer_rank_ptr_mapper: Optional[SymBufferDeviceBase] - valid_tokens_this_cta_tile: cutlass.Int32 - valid_hidden_this_cta_tile: Union[cutlass.Int32, int] - reduce_topk_in_kernel: bool - output_mapping: Contract # (epi_tid, iter_idx) -> (token_cta_tile, hidden_cta_tile). - epi_tid: cutlass.Int32 + def __new_from_mlir_values__(self, + values: List[ir.Value]) -> "SwapABFc2Epilogue": + assert len(values) == 0 + return self - # After metadata prefetch - dst_ptrs: Optional[cute.Tensor] = ( - None # i64 x (copy_iters_this_thread_cta_tile), fundamentally the pointers. - ) - valid: Optional[cute.Tensor] = None # (copy_iters_this_thread_cta_tile) + @cute.jit + def signal_fc2_done(self, work_tile_info, next_work_tile_info, + flag_tracker): + publish: cutlass.Constexpr = (self.token_back_by_dispatch + or self.combine_format.is_quantized) + if cutlass.const_expr(publish): + flag_addr = (self.token_comm_args.fc2_done_counter.iterator + + work_tile_info.expert_idx).toint() + else: + flag_addr = Int64(0) + no_fire: cutlass.Constexpr = not publish + return flag_tracker.accumulate(next_work_tile_info.phase, + self.fc2_epi_flag_batch, flag_addr, + no_fire) - def __post_init__(self) -> None: - if (self.metadata is None) == (self.direct_token_base_this_cta_tile - is None): - raise ValueError( - "Fc2OutputRouter requires exactly one of metadata or " - "direct_token_base_this_cta_tile.") - if (self.metadata is None) != (self.peer_rank_ptr_mapper is None): + @cute.jit + def _make_output_router( + self, + work_tile_info: MoEWorkTileInfo, + ) -> "Fc2OutputRouter": + task_tile_data_row_start = ( + work_tile_info.cumulative_data_physical_row + + work_tile_info.tile_n_idx * cutlass.Int32(self.cta_tile_n)) + hidden_base_this_cta_tile = (work_tile_info.tile_m_idx * + cutlass.Int32(self.cta_tile_m)) + valid_hidden_this_cta_tile = (cutlass.Int32(self.fc2_output.shape[2]) - + hidden_base_this_cta_tile) + if valid_hidden_this_cta_tile < 0: + valid_hidden_this_cta_tile = 0 + if valid_hidden_this_cta_tile > self._EpilogueFc2HiddenTileSize: + valid_hidden_this_cta_tile = self._EpilogueFc2HiddenTileSize + + metadata_u32 = None + peer_rank_ptr_mapper = None + data_token_base = task_tile_data_row_start + if cutlass.const_expr(self.token_comm_args is not None + and not self.token_back_by_dispatch): + metadata_u32 = cute.domain_offset( + (task_tile_data_row_start, 0), + cute.recast_tensor( + self.token_comm_args.token_src_metadata, + cutlass.Uint32, + ), + ) + peer_rank_ptr_mapper = self.token_comm_args.peer_rank_ptr_mapper + data_token_base = None + + if cutlass.const_expr(self.combine_format.is_quantized): + base_outputs = (self.fc2_output, self.token_comm_args.fc2_output_sf) + token_bases = (data_token_base, task_tile_data_row_start) + output_mappings = ( + self.process_pipeline.store_out_mapping, + self.process_pipeline.sf_store_out_mapping, + ) + else: + base_outputs = self.fc2_output + token_bases = data_token_base + output_mappings = self.process_pipeline.store_out_mapping + + return Fc2OutputRouter( + metadata=metadata_u32, + token_bases=token_bases, + base_outputs=base_outputs, + hidden_base_this_cta_tile=hidden_base_this_cta_tile, + peer_rank_ptr_mapper=peer_rank_ptr_mapper, + valid_tokens_this_cta_tile=work_tile_info.valid_tokens_in_cta_tile, + valid_hidden_this_cta_tile=valid_hidden_this_cta_tile, + reduce_topk_in_kernel=self.reduce_topk_in_kernel, + output_mappings=output_mappings, + epi_tid=self.tidx, + combine_format=self.combine_format, + ).prefetch() + + @cute.jit + def __call__( + self, + work_tile_info: MoEWorkTileInfo, + tmem_acc_tensor: cute.Tensor, + acc_pipeline, + acc_consumer_state, + is_odd_turn: cutlass.Int32, + ): + # subtile-irrelevant hoist: fc2 alpha scales raw fc2 accumulators before f2fp. + if cutlass.const_expr(self.optional_epi_args.fc2_alpha is not None): + alpha_val = self.optional_epi_args.fc2_alpha[ + work_tile_info.expert_idx] + else: + alpha_val = None + acc_ready = False + if not work_tile_info.peek_ready: + acc_ready = True + acc_pipeline.consumer_wait(acc_consumer_state) + fc2_output_router = self._make_output_router(work_tile_info) + # (cta_tile_m, cta_tile_n) -> (epi_tile_m, epi_tile_n, iters) + tmem_acc_tensor_tiled_by_epi_tile = cute.flat_divide( + tmem_acc_tensor, (self._EpilogueFc2HiddenTileSize, + self._EpilogueTokenTileSize))[None, None, 0, None] + + acc_pipeline.consumer_wait(acc_consumer_state, acc_ready) + iket.range_push("fc2_epi") + valid_tokens = work_tile_info.valid_tokens_in_cta_tile + + # Overlap path preloads two subtiles before releasing acc TMEM. + unroll_tile_cnt = 2 if cutlass.const_expr( + self.overlapping_accum + and self.process_pipeline.require_tmem_trans) else 0 + remain_subtile_cnt = self.subtile_cnt - unroll_tile_cnt + + if cutlass.const_expr(unroll_tile_cnt > 0): + subtile_idx_first = (cutlass.Int32(self.subtile_cnt) - + is_odd_turn) % cutlass.Int32(self.subtile_cnt) + subtile_idx_second = (cutlass.Int32(self.subtile_cnt + 1) - + is_odd_turn) % cutlass.Int32(self.subtile_cnt) + + # preload_subtile_first: subtile_idx_first's raw PRE-transpose acc, LDTM'd by + # all 128 epi threads into 4 reg tensors == the 4 quadrants of the subtile's + # (128 tmem_dp x 64 tmem_col) footprint. Only these raw-TMEM offsets are + # guaranteed: + # reg[0]/reg[1], reg[2]/reg[3] : top vs bot -> 16 apart in tmem_dp + # reg[0]/reg[2], reg[1]/reg[3] : 1st vs 2nd half -> 32 apart in tmem_col + # (so reg[0..1] = the first 128x32, reg[2..3] = the second 128x32 of the 128x64.) + # The per-lane (lane_idx, elem_idx) -> (tmem_dp, tmem_col) layout INSIDE each + # reg tensor is opaque -- do not assume it; it only becomes well-defined once + # the tmem transpose consumes them. + preload_subtile_first: Tuple[ + cute.Tensor, cute.Tensor, cute.Tensor, + cute.Tensor] = _TmemTranspose16x32Core.load_subtile_raw_acc( + tmem_acc_tensor_tiled_by_epi_tile[None, None, + subtile_idx_first]) + + # Release acc to next MMA unconditionally. + cute.arch.fence_view_async_tmem_load() + acc_pipeline.consumer_release(acc_consumer_state) + + # preload_subtile_second: same 128 tmem_dp x 64 tmem_col footprint, but for + # subtile_idx_second (the other token subtile, not the 2nd col-half). Same + # quadrant/offset invariants and opaque per-lane layout as above. + preload_subtile_second: Tuple[ + cute.Tensor, cute.Tensor, cute.Tensor, + cute.Tensor] = _TmemTranspose16x32Core.load_subtile_raw_acc( + tmem_acc_tensor_tiled_by_epi_tile[None, None, + subtile_idx_second]) + + # Both unrolled subtiles borrow tmem_subtile_second as workspace. + preload_pair = (preload_subtile_first, preload_subtile_second) + subtile_idx_pair = (subtile_idx_first, subtile_idx_second) + for i in cutlass.range_constexpr(unroll_tile_cnt): + if subtile_idx_pair[i] * cutlass.Int32( + self._EpilogueTokenTileSize) < valid_tokens: + self.run_subtile( + subtile_idx=subtile_idx_pair[i], + tmem_subtile_tensor=tmem_acc_tensor_tiled_by_epi_tile[ + None, None, subtile_idx_second], + preload_acc=preload_pair[i], + fc2_output_router=fc2_output_router, + alpha_val=alpha_val, + release_after_ldtm=False, + acc_pipeline=acc_pipeline, + acc_consumer_state=acc_consumer_state, + ) + + if cutlass.const_expr(self.overlapping_accum and unroll_tile_cnt == 0): + release_after_ldtm = True + else: + release_after_ldtm = False + for i in cutlass.range(remain_subtile_cnt, unroll=1): + # for i in cutlass.range_constexpr(remain_subtile_cnt): + real_i = i + unroll_tile_cnt + if cutlass.const_expr(self.overlapping_accum): + subtile_idx = (cutlass.Int32(real_i + self.subtile_cnt) - + is_odd_turn) % cutlass.Int32(self.subtile_cnt) + else: + subtile_idx = cutlass.Int32(real_i) + + if subtile_idx * cutlass.Int32( + self._EpilogueTokenTileSize) < valid_tokens: + self.run_subtile( + subtile_idx=subtile_idx, + tmem_subtile_tensor=tmem_acc_tensor_tiled_by_epi_tile[ + None, None, subtile_idx], + preload_acc=None, + fc2_output_router=fc2_output_router, + alpha_val=alpha_val, + release_after_ldtm=release_after_ldtm, + acc_pipeline=acc_pipeline, + acc_consumer_state=acc_consumer_state, + ) + release_after_ldtm = False + + # Non-overlap-path release: at the natural task-tile boundary. + if cutlass.const_expr(not self.overlapping_accum): + cute.arch.fence_view_async_tmem_load() + acc_pipeline.consumer_release(acc_consumer_state) + + @cute.jit + def run_subtile( + self, + subtile_idx: cutlass.Int32, + # (hidden_tile, token_subtile), fundamentally (epi_tile_m, epi_tile_n) + tmem_subtile_tensor: cute.Tensor, + preload_acc: Optional[Tuple[cute.Tensor, cute.Tensor, cute.Tensor, + cute.Tensor]], + fc2_output_router: "Fc2OutputRouter", + alpha_val: Optional[cutlass.Float32], + release_after_ldtm: Union[cutlass.Boolean, bool], + acc_pipeline, + acc_consumer_state, + ): + process_pipeline = self.process_pipeline + if cutlass.const_expr(preload_acc is None): + loaded = process_pipeline.tmem_acc_load( + tmem_subtile_tensor=tmem_subtile_tensor, + epi=self, + ) + if release_after_ldtm: + cute.arch.fence_view_async_tmem_load() + acc_pipeline.consumer_release(acc_consumer_state) + else: + loaded = preload_acc + + casted = process_pipeline.f2fp( + *loaded, + alpha_val=alpha_val, + ) + # reorder returns a bare RMEM fragment in the store's expected pre-store + # distribution; reorder + store are paired 1:1 inside the pipeline. + pre_store = process_pipeline.post_f2fp_reorder( + casted=casted, + tmem_subtile_view=tmem_subtile_tensor, + ) + process_pipeline.store_function( + epi=self, + subtile=pre_store, + subtile_idx=subtile_idx, + fc2_output_router=fc2_output_router, + ) + + +@dataclasses.dataclass(frozen=True) +class Fc2OutputRouter: + # (token, 3), 3 -> rank_idx, token_idx, top_k + # Later this will be changed to (token, 2), where top_k and rank_idx will be fused into 32bit. + # If metadata is None then this is a local write. + metadata: Optional[cute.Tensor] + # token + possible sf + token_bases: Union[Tuple[Optional[cutlass.Int32], cutlass.Int32], + Optional[cutlass.Int32]] + base_outputs: Union[Tuple[cute.Tensor, cute.Tensor], + cute.Tensor] # (token, topk, hidden) + hidden_base_this_cta_tile: Union[cutlass.Int32, int] + peer_rank_ptr_mapper: Optional[SymBufferDeviceBase] + valid_tokens_this_cta_tile: cutlass.Int32 + valid_hidden_this_cta_tile: Union[cutlass.Int32, int] + reduce_topk_in_kernel: bool + # Per-issue (epi_tid, iter_idx) -> (token_cta_tile, hidden_cta_tile). Data + # mapping, or (data mapping, sf mapping) when quantized. + output_mappings: Union[Tuple[Contract, Contract], Contract] + epi_tid: cutlass.Int32 + combine_format: CombineFormat + # After metadata prefetch + dst_ptrs: Optional[ + cute. + Tensor] = None # i64 x (copy_iters_this_thread_cta_tile), fundamentally the pointers. + valid: Optional[cute.Tensor] = None # (copy_iters_this_thread_cta_tile) + + @property + def data_output(self) -> cute.Tensor: + return self.base_outputs[0] if isinstance(self.base_outputs, + tuple) else self.base_outputs + + @property + def sf_output(self) -> Optional[cute.Tensor]: + # Present iff quantized; (pool_token, 1, hidden // sf_vec) rank-local. + return self.base_outputs[1] if isinstance(self.base_outputs, + tuple) else None + + @property + def data_token_base(self) -> Optional[cutlass.Int32]: + return self.token_bases[0] if isinstance(self.token_bases, + tuple) else self.token_bases + + @property + def sf_token_base(self) -> Optional[cutlass.Int32]: + return self.token_bases[1] if isinstance(self.token_bases, + tuple) else None + + @property + def data_mapping(self) -> Contract: + return self.output_mappings[0] if isinstance( + self.output_mappings, tuple) else self.output_mappings + + @property + def sf_mapping(self) -> Optional[Contract]: + return self.output_mappings[1] if isinstance(self.output_mappings, + tuple) else None + + def __post_init__(self) -> None: + if (self.metadata is None) == (self.data_token_base is None): + raise ValueError( + "Fc2OutputRouter requires exactly one of metadata or " + "a (data) token base.") + if (self.metadata is None) != (self.peer_rank_ptr_mapper is None): raise ValueError( "Fc2OutputRouter requires peer_rank_ptr_mapper iff metadata is set." ) @@ -1645,8 +2253,14 @@ def __post_init__(self) -> None: @cute.jit def prefetch(self) -> "Fc2OutputRouter": - iter_axis = self.output_mapping.domain.names.index("iter_idx") - copy_iters: cutlass.Constexpr[int] = self.output_mapping.domain.sizes[ + # Only the metadata (comm) path prefetches a pointer array: its + # metadata-derived address has long-latency LDGs worth issuing early. + # The local (no-comm) path computes its affine address on demand in + # get_dst() -- no array, hence no runtime-indexed local-memory spill. + if cutlass.const_expr(self.metadata is None): + return self + iter_axis = self.data_mapping.domain.names.index("iter_idx") + copy_iters: cutlass.Constexpr[int] = self.data_mapping.domain.sizes[ iter_axis] valid = cute.make_rmem_tensor((copy_iters, ), cutlass.Int32) @@ -1656,7 +2270,7 @@ def prefetch(self) -> "Fc2OutputRouter": # We should check the SASS to ensure this happens. for iter_idx in cutlass.range_constexpr(copy_iters): coord = eval_function_mapping( - self.output_mapping, + self.data_mapping, epi_tid=self.epi_tid, iter_idx=iter_idx, ) @@ -1672,27 +2286,41 @@ def prefetch(self) -> "Fc2OutputRouter": if token_valid and hidden_valid: valid[iter_idx] = cutlass.Int32(1) if cutlass.const_expr(self.metadata is None): - dst_tokens = self.direct_token_base_this_cta_tile + token_in_tile + dst_tokens = self.data_token_base + token_in_tile dst_hidden = hidden_in_tile + self.hidden_base_this_cta_tile - dst_ptrs[iter_idx] = self.base_output[ - dst_tokens, None, dst_hidden].iterator.toint() + # Int64 token coord: dst_tokens*K*H overflows int32 once + # T*K*H exceeds 2^31 (data_output is (token, topk, hidden)). + dst_ptrs[iter_idx] = self.data_output[ + Int64(dst_tokens), None, dst_hidden].iterator.toint() else: - dst_rank = cutlass.Int32(self.metadata[token_in_tile, 0]) - dst_token = cutlass.Int32(self.metadata[token_in_tile, 1]) + md = TokenSrcMetadata.load(self.metadata.iterator.toint() + + Int64(token_in_tile) * + Int64(TokenSrcMetadata.nbytes)) + dst_rank = md.src_rank + dst_token = md.src_token dst_hidden = hidden_in_tile + self.hidden_base_this_cta_tile if cutlass.const_expr(not self.reduce_topk_in_kernel): - dst_topk = cutlass.Int32(self.metadata[token_in_tile, - 2]) + dst_topk = md.src_topk else: dst_topk = 0 + # Int64 token coord: domain_offset on (token, topk, hidden) + # computes dst_token*K*H, which overflows int32 once T*K*H > 2^31. + # byte_align mirrors get_data_dst: the STG vector is 16 B + # for e2m1 wires (odd warps at row base +16 B), 32 B for + # fp8/bf16. dst_ptrs[ iter_idx] = self.peer_rank_ptr_mapper.ptr_map_to_rank( cute.domain_offset( - (dst_token, dst_topk, dst_hidden), - self.base_output).iterator, + (Int64(dst_token), dst_topk, dst_hidden), + self.data_output).iterator, dst_rank, - ).toint() + byte_align=min( + 32, + (min(32, 256 // + self.data_output.element_type.width) * + self.data_output.element_type.width) // 8, + )).toint() return dataclasses.replace( self, @@ -1700,157 +2328,113 @@ def prefetch(self) -> "Fc2OutputRouter": valid=valid, ) - # Return a tuple of (src, dst) tensors, each represents copy_atom's one call. @cute.jit - def resolve( + def get_data_dst( self, - copy_src: cute.Tensor, # (v, rest...) - iters_per_subtile: int, - subtile_idx: Union[cutlass.Int32, int], - ) -> Tuple[Tuple[cute.Tensor, cute.Tensor], ...]: - if cutlass.const_expr(self.dst_ptrs is None or self.valid is None): - raise ValueError( - "Fc2OutputRouter.resolve requires prefetch() first.") - # Normalize any strategy-specific source view into the canonical copy - # iterator form: - # - # ((atom_v, rest_v), rests...) - # - # The trailing ``rest`` modes enumerate one copy-atom issue inside the - # current subtile; their product must equal ``iters_per_subtile``. - # Everything before those trailing modes is the copy atom payload - # (``atom_v`` elements). We intentionally flatten first because callers - # may hand us nested CuTe layouts whose hierarchy is meaningful to their - # local algorithm but irrelevant to the final copy issue schedule. After - # finding the trailing rest modes, two ``group_modes`` calls make rank 0 - # the payload and rank 1 the full rest/iter space; coalescing with - # ``target_profile=(1, 1)`` preserves that two-rank profile while - # simplifying each side's internal layout. - flat_src = cute.flatten(copy_src) - flat_rank: cutlass.Constexpr[int] = cute.rank(flat_src) - - rest_start = flat_rank - rest_size = 1 - for mode in cutlass.range_constexpr(flat_rank - 1, -1, -1): - rest_start = mode - rest_size *= cute.size(flat_src, mode=[mode]) - if cutlass.const_expr(rest_size == iters_per_subtile): - break - if cutlass.const_expr(rest_size != iters_per_subtile): - raise ValueError( - "Fc2OutputRouter.resolve: trailing rest modes must multiply " - f"to iters_per_subtile={iters_per_subtile}, got {rest_size}.") - if cutlass.const_expr(rest_start == 0): - raise ValueError( - "Fc2OutputRouter.resolve requires at least one atom payload mode " - "before the trailing rest modes.") - - atom_v: cutlass.Constexpr[int] = cute.size( - flat_src) // iters_per_subtile - atom_rest = cute.group_modes(flat_src, 0, rest_start) - atom_rest = cute.group_modes(atom_rest, 1, cute.rank(atom_rest)) - atom_rest = cute.coalesce(atom_rest, target_profile=(1, 1)) - - single_copy_layout = cute.make_layout( - ((atom_v, 1), ), - stride=((1, 0), ), - ) - subtile_iter_base = cutlass.Int32(subtile_idx) * cutlass.Int32( - iters_per_subtile) - - copy_pairs = () - for local_iter in cutlass.range_constexpr(iters_per_subtile): - global_iter = subtile_iter_base + cutlass.Int32(local_iter) - src_atom = atom_rest[None, local_iter] - copy_src_i = cute.make_tensor(src_atom.iterator, single_copy_layout) - dst_ptr = cute.make_ptr( - copy_src.element_type, - self.dst_ptrs[global_iter], - AddressSpace.gmem, - assumed_align=32, - ) - copy_dst_i = cute.make_tensor(dst_ptr, single_copy_layout) - copy_pairs = copy_pairs + ((copy_src_i, copy_dst_i), ) - - return copy_pairs + iter_idx: Union[int, cutlass.Int32], + ) -> Tuple[cute.Pointer, cutlass.Int32]: + """Per-issue DATA destination: gmem pointer + validity predicate. + The router owns ``data_output`` so the caller never re-assembles a + pointer from a raw int; it just builds its own copy tensor (STG) or + feeds the pointer to inline asm (REDG/UBLK). -@dataclasses.dataclass(frozen=True) -class Fc2ProcessPipeline: - tmem_acc_load: Callable - f2fp: Callable - post_f2fp_reorder: Callable - store_function: Callable - pre_store_contract: Contract - fc2_cta_tile_contract: Contract - store_out_mapping: Contract - require_tmem_trans: bool + Alignment is unified at 32 B: only STG feeds this pointer to a real + ``cute.copy`` (256 b vector store, genuinely 32 B aligned); REDG/UBLK + only ``ptrtoint`` it for inline-asm issue, where the hint is inert. + """ + if cutlass.const_expr(self.metadata is None): + # no-comm: on-demand affine address (no prefetched array). The + # invariant base hoists out of the caller's loop via CSE; a + # constexpr iter folds the per-issue offset into the store. + coord = eval_function_mapping( + self.data_mapping, + epi_tid=self.epi_tid, + iter_idx=iter_idx, + ) + token_in_tile = cutlass.Int32(coord["token_in_cta_tile"]) + hidden_in_tile = cutlass.Int32(coord["hidden_in_cta_tile"]) + pred = cutlass.Int32(0) + addr = cutlass.Int64(0) + if (token_in_tile < self.valid_tokens_this_cta_tile + and hidden_in_tile < cutlass.Int32( + self.valid_hidden_this_cta_tile)): + pred = cutlass.Int32(1) + dst_tokens = self.data_token_base + token_in_tile + dst_hidden = hidden_in_tile + self.hidden_base_this_cta_tile + # Int64 token coord: dst_tokens*K*H overflows int32 once T*K*H > 2^31. + addr = self.data_output[Int64(dst_tokens), None, + dst_hidden].iterator.toint() + else: + # comm: read the pointer / validity prefetched by prefetch(). + addr = self.dst_ptrs[iter_idx] + pred = self.valid[iter_idx] + # Alignment must match the real STG granularity: the store vector is + # min(32, 256 // width) elements, i.e. 32 B for fp8/bf16 wires but + # only 16 B for e2m1 (4-bit) -- odd warps then land on row base +16 B, + # so an align-32 promise would be UB for the fp4 combine format. + _stg_bytes = (min(32, 256 // self.data_output.element_type.width) * + self.data_output.element_type.width) // 8 + ptr = cute.make_ptr( + self.data_output.element_type, + addr, + AddressSpace.gmem, + assumed_align=min(32, _stg_bytes), + ) + return ptr, pred + @cute.jit + def get_sf_dst( + self, + iter_idx: Union[int, cutlass.Int32], + ) -> Tuple[cute.Pointer, cutlass.Int32]: + """Per-issue SF destination: rank-local gmem pointer + validity predicate. + + SF never goes to a peer (it is staged locally and pushed token-contiguously + by the dispatch / standalone warps), so this is always the affine local + address -- no metadata routing, no prefetch. ``sf_output`` is the broadcast + plane ``(pool_token, 1, (sf_vec, hidden//sf_vec)):(., ., (0, 1))``, so the + logical hidden coordinate folds to its scale block on indexing. + """ + coord = eval_function_mapping( + self.sf_mapping, + epi_tid=self.epi_tid, + iter_idx=iter_idx, + ) + token_in_tile = cutlass.Int32(coord["token_in_cta_tile"]) + hidden_in_tile = cutlass.Int32(coord["hidden_in_cta_tile"]) + pred = cutlass.Int32(0) + addr = cutlass.Int64(0) + if (token_in_tile < self.valid_tokens_this_cta_tile and hidden_in_tile + < cutlass.Int32(self.valid_hidden_this_cta_tile)): + pred = cutlass.Int32(1) + sf_row = self.sf_token_base + token_in_tile + sf_hidden = hidden_in_tile + self.hidden_base_this_cta_tile + addr = self.sf_output[Int64(sf_row), None, + sf_hidden].iterator.toint() + # Per-block scale offsets are element-granular; claim the scale dtype's + # natural element alignment (e8m0 1 B / bf16 2 B). An align-4 promise + # here is UB: adjacent warps land on row base +1/+2/+3 for 1-byte + # scales. + sf_ptr = cute.make_ptr( + self.sf_output.element_type, + addr, + AddressSpace.gmem, + assumed_align=self.sf_output.element_type.width // 8, + ) + return sf_ptr, pred -# ============================================================================= -# fc2 STG strategy callables (subtile granularity) -# -# Faithful port of the original transpose+STG path, re-cut into the four -# Fc2ProcessPipeline steps. Originals in epilogue.py: -# - load : _TmemTranspose16x32Core.load_subtile_raw_acc -# - pack : Fc2AccLoadAndPack.__init__ (L986-997) -# - transpose : TmemTranspose16x32Packed (+ from_r1_perm_until_last_store) -# - unpack : Fc2UnpackPermuteStg._init_direct (L1510-1520) -# - store : Fc2UnpackPermuteStg._stg_direct (L1522-1605) -# -# All callables take the unified kwargs + ``**_`` (extras ignored; missing -# required -> TypeError). ``epi`` is the SwapABFc2Epilogue device object. -# ============================================================================= -# Subtile pre-store contract C (the pivot): each lane holds 64 bf16 values; -# vid in [0,32) -> token=lane (half 0), vid in [32,64) -> token=lane+32 (half 1); -# hidden = vid % 32 (this warp's 32-hidden span, natural order). -_Fc2StgSubtilePreStoreContract = Contract( - domain=Space(("lane_idx", "vid"), (32, 64)), - codomain=Space(("token_idx", "hidden_idx"), (64, 32)), - mapping=FunctionMapping(lambda lane_idx, vid: { - "token_idx": lane_idx + 32 * (vid // 32), - "hidden_idx": vid % 32, - }), -) - -# UBLK pre-store contract: after f2fp (and before R2S), each lane owns one -# hidden element across the 64 token positions of the subtile. This is -# warp-local + subtile-local; the store-out contract below is CTA-level and -# describes the later bulk issue rows, not this RMEM distribution. -_Fc2UblkSubtilePreStoreContract = Contract( - domain=Space(("lane_idx", "vid"), (32, 64)), - codomain=Space(("token_idx", "hidden_idx"), (64, 32)), - mapping=FunctionMapping(lambda lane_idx, vid: { - "token_idx": vid, - "hidden_idx": lane_idx, - }), -) - -# REDG pre-store contract: after the extra STTM + LDTM(16x256b.x2) -# reshuffle, each lane owns bf16 scalar elements arranged so every 4 -# consecutive elem_idx values form one red.v2.bf16x2 issue payload. -_Fc2RedgSubtilePreStoreContract = Contract( - domain=Space(("lane_idx", "elem_idx"), (32, 64)), - codomain=Space(("token_idx", "hidden_idx"), (64, 32)), - mapping=FunctionMapping( - lambda lane_idx, elem_idx: { - "token_idx": - (((elem_idx // 2) // 16) * 32 + (((elem_idx // 2) // 8) % 2) * 16 + - (((elem_idx // 2) // 2) % 2) * 8 + lane_idx // 4), - "hidden_idx": ((lane_idx % 4) * 4 + (((elem_idx // 2) // 4) % 2) * - 16 + ((elem_idx // 2) % 2) * 2 + (elem_idx % 2)), - }), -) - - -# TODO: Enable for non-BF16 dtypes -def make_fc2_stg_cta_store_out_contract(fc2_output_dtype: Type[cutlass.Numeric], +def make_fc2_stg_cta_store_out_contract(combine_format: CombineFormat, cta_token_tile_size: int, cta_hidden_tile_size: int): - assert cta_hidden_tile_size == 128 - assert cta_token_tile_size % 64 == 0 - assert fc2_output_dtype.width == 16 + assert (cta_hidden_tile_size == 128) + assert (cta_token_tile_size % 64 == 0) + wire_dtype = combine_format.act_dtype + assert wire_dtype.width in (4, 8, + 16), "fc2 STG wire dtype must be fp4/fp8/bf16." + elems_per_stg = min(256 // wire_dtype.width, 32) + stgs_per_hidden32 = 32 // elems_per_stg fundamental_mapping = Contract( domain=Space(("epi_tid", "elem_idx"), (128, cta_token_tile_size)), codomain=Space(("token_in_cta_tile", "hidden_in_cta_tile"), @@ -1859,28 +2443,48 @@ def make_fc2_stg_cta_store_out_contract(fc2_output_dtype: Type[cutlass.Numeric], lambda epi_tid, elem_idx: { "token_in_cta_tile": epi_tid % 32 + elem_idx // 32 * 32, "hidden_in_cta_tile": elem_idx % 32 + epi_tid // 32 * 32, - }), - ) + })) store_out_mapping = Contract( domain=Space(("epi_tid", "iter_idx"), - (128, 32 // 16 * cta_token_tile_size // 32)), + (128, stgs_per_hidden32 * cta_token_tile_size // 32)), codomain=Space(("token_in_cta_tile", "hidden_in_cta_tile"), (cta_token_tile_size, cta_hidden_tile_size)), mapping=FunctionMapping( lambda epi_tid, iter_idx: { - "token_in_cta_tile": epi_tid % 32 + iter_idx // 2 * 32, - "hidden_in_cta_tile": (iter_idx % 2) * 16 + epi_tid // 32 * 32, - }), - ) - return store_out_mapping, fundamental_mapping + "token_in_cta_tile": + epi_tid % 32 + iter_idx // stgs_per_hidden32 * 32, + "hidden_in_cta_tile": (iter_idx % stgs_per_hidden32) * + elems_per_stg + epi_tid // 32 * 32, + })) + sf_store_out_mapping = None + if combine_format.is_quantized: + + def stg_sf_mapping(epi_tid, iter_idx): + lane = epi_tid % 32 + warp = epi_tid // 32 + return { + "token_in_cta_tile": lane + iter_idx * 32, + "hidden_in_cta_tile": warp * 32, + } + + sf_store_out_mapping = Contract( + domain=Space(("epi_tid", "iter_idx"), + (128, cta_token_tile_size // 32)), + codomain=Space(("token_in_cta_tile", "hidden_in_cta_tile"), + (cta_token_tile_size, cta_hidden_tile_size)), + mapping=FunctionMapping(stg_sf_mapping), + ) + return store_out_mapping, sf_store_out_mapping, fundamental_mapping -def make_fc2_redg_cta_store_out_contract( - fc2_output_dtype: Type[cutlass.Numeric], cta_token_tile_size: int, - cta_hidden_tile_size: int): - assert cta_hidden_tile_size == 128 - assert cta_token_tile_size % 64 == 0 - assert fc2_output_dtype.width == 16 +def make_fc2_redg_cta_store_out_contract(combine_format: CombineFormat, + cta_token_tile_size: int, + cta_hidden_tile_size: int): + assert (cta_hidden_tile_size == 128) + assert (cta_token_tile_size % 64 == 0) + # In-kernel reduce is bf16-only and never quantized, so there is no SF plane. + assert (combine_format.act_dtype.width == 16) + assert not combine_format.is_quantized fundamental_mapping = Contract( domain=Space(("epi_tid", "elem_idx"), (128, cta_token_tile_size)), codomain=Space(("token_in_cta_tile", "hidden_in_cta_tile"), @@ -1894,8 +2498,7 @@ def make_fc2_redg_cta_store_out_contract( "hidden_in_cta_tile": ( (epi_tid // 32) * 32 + (epi_tid % 4) * 4 + (( (elem_idx // 4) % 4) // 2) * 16 + elem_idx % 4), - }), - ) + })) # SIMT REDG emits one 8B red.v2.bf16x2 per 4 hidden elements. Each # 64-token subtile contributes two token rows per lane and 8 hidden # segments per token row. @@ -1911,61 +2514,102 @@ def make_fc2_redg_cta_store_out_contract( (iter_idx % 4) % 2) * 8 + (epi_tid % 32) // 4), "hidden_in_cta_tile": ((epi_tid // 32) * 32 + (epi_tid % 4) * 4 + ((iter_idx % 4) // 2) * 16), - }), - ) - return store_out_mapping, fundamental_mapping + })) + return store_out_mapping, None, fundamental_mapping -def make_fc2_ublk_store_out_contract(fc2_output_dtype: Type[cutlass.Numeric], +def make_fc2_ublk_store_out_contract(combine_format: CombineFormat, cta_token_tile_size: int, cta_hidden_tile_size: int): - assert cta_hidden_tile_size == 128 - assert cta_token_tile_size % 64 == 0 - assert fc2_output_dtype.width == 16 + assert (cta_hidden_tile_size == 128) + assert (cta_token_tile_size % 64 == 0) + # UBLK pushes whole hidden rows by byte count, so the token/hidden mapping + # is element-indexed and dtype-independent (wire dtype only sets copy bytes). + assert combine_format.act_dtype.width in ( + 4, 8, 16), "fc2 UBLK wire dtype must be fp4/fp8/bf16." + assert (cta_token_tile_size <= 256) + max_token_cta_tile = 256 fundamental_mapping = Contract( domain=Space(("epi_tid", "elem_idx"), (128, cta_token_tile_size)), codomain=Space(("token_in_cta_tile", "hidden_in_cta_tile"), - (cta_token_tile_size, cta_hidden_tile_size)), + (max_token_cta_tile, cta_hidden_tile_size)), mapping=FunctionMapping( lambda epi_tid, elem_idx: { "token_in_cta_tile": - epi_tid % 8 + epi_tid // 32 * 8 + ((epi_tid % 32) // 8) * 32 + - elem_idx // cta_hidden_tile_size * 128, + elem_idx // cta_hidden_tile_size * 32 + epi_tid % 8 + epi_tid // + 32 * 8 + ((epi_tid % 32) // 8) * 64, "hidden_in_cta_tile": elem_idx % cta_hidden_tile_size, - }), - ) + })) store_out_mapping = Contract( - domain=Space(("epi_tid", "iter_idx"), - (128, (cta_token_tile_size + 127) // 128)), + domain=Space(("epi_tid", "iter_idx"), (128, 2)), codomain=Space(("token_in_cta_tile", "hidden_in_cta_tile"), - (cta_token_tile_size, cta_hidden_tile_size)), + (max_token_cta_tile, cta_hidden_tile_size)), mapping=FunctionMapping( lambda epi_tid, iter_idx: { "token_in_cta_tile": - epi_tid % 8 + epi_tid // 32 * 8 + - ((epi_tid % 32) // 8) * 32 + iter_idx * 128, + iter_idx * 32 + epi_tid % 8 + epi_tid // 32 * 8 + + ((epi_tid % 32) // 8) * 64, "hidden_in_cta_tile": 0, - }), - ) - return store_out_mapping, fundamental_mapping - + })) + # SF mapping: each lane owns one hidden across a 64-token subtile, and the + # sf_vec lanes of a block share the CREDUX scale -- so they split the block's + # tokens. sf_iter flattens (subtile, slot) to keep the whole cta-tile domain. + sf_store_out_mapping = None + if combine_format.is_quantized: + sf_vec = combine_format.scale_block + lanes_per_block = sf_vec + subtile_tokens = SwapABSwigluFp4Epilogue._EpilogueTokenTileSize + iters_per_subtile = subtile_tokens // lanes_per_block + n_subtiles = cta_token_tile_size // subtile_tokens + + def ublk_sf_mapping(epi_tid, iter_idx): + subtile_idx = iter_idx // iters_per_subtile + slot = iter_idx % iters_per_subtile + lane = epi_tid % 32 + warp = epi_tid // 32 + lane_in_block = lane % lanes_per_block + block_in_warp = lane // lanes_per_block + return { + "token_in_cta_tile": + subtile_idx * subtile_tokens + lane_in_block + + slot * lanes_per_block, + "hidden_in_cta_tile": + warp * 32 + block_in_warp * sf_vec, + } + + sf_store_out_mapping = Contract( + domain=Space(("epi_tid", "iter_idx"), + (128, n_subtiles * iters_per_subtile)), + codomain=Space(("token_in_cta_tile", "hidden_in_cta_tile"), + (max_token_cta_tile, cta_hidden_tile_size)), + mapping=FunctionMapping(ublk_sf_mapping), + ) + return store_out_mapping, sf_store_out_mapping, fundamental_mapping + + +# (...) -> ((atom_v, 1)) +@cute.jit +def wrap_into_copy_standard_layout(tensor: cute.Tensor): + tensor = cute.coalesce(cute.flatten(tensor)) + tensor = cute.append_ones(tensor, cute.rank(tensor) + 1) + tensor = cute.group_modes(tensor, 0, cute.rank(tensor) - 1) + tensor = cute.group_modes(tensor, 0, cute.rank(tensor)) + return tensor + @cute.jit def fc2_f2fp( *tensors, - fc2_output_dtype: Type[cutlass.Numeric], alpha_val: Optional[cutlass.Float32] = None, **_, ) -> cute.Tensor: - # cvt every input fp32 rmem -> fc2_output_dtype and concatenate, in order, - # into one flat rmem tensor. Each block is stored contiguously (no scalar - # element copy) at its running offset. + reorder_dtype = cutlass.BFloat16 total_size = 0 for t in tensors: total_size += cute.size(t) - converted_acc = cute.make_rmem_tensor((total_size, ), fc2_output_dtype) + converted_acc = cute.make_rmem_tensor((total_size, ), reorder_dtype) elems_processed = 0 for t in tensors: current_tensor_size = cute.size(t) @@ -1974,7 +2618,7 @@ def fc2_f2fp( cute.make_layout((current_tensor_size, )), ) if cutlass.const_expr(alpha_val is None): - dst.store(t.load().to(fc2_output_dtype)) + dst.store(t.load().to(reorder_dtype)) else: if cutlass.const_expr(current_tensor_size % 2 != 0): raise ValueError( @@ -1987,14 +2631,16 @@ def fc2_f2fp( (alpha_val, alpha_val)) scaled[i] = s0 scaled[i + 1] = s1 - dst.store(scaled.load().to(fc2_output_dtype)) + dst.store(scaled.load().to(reorder_dtype)) elems_processed += current_tensor_size return converted_acc @cute.jit -def post_f2fp_reorder_identity(*, casted: cute.Tensor, contract: Contract, **_): - return TensorWithContract(tensor=casted, contract=contract) +def post_f2fp_reorder_identity(*, casted: cute.Tensor, **_): + # UBLK: the f2fp output is already in the pre-store distribution (each lane + # owns one hidden element across the 64 subtile tokens); no reorder needed. + return casted @cute.jit @@ -2026,12 +2672,11 @@ def fc2_ublk_tmem_acc_load(*, tmem_subtile_tensor: cute.Tensor, epi, **_): @cute.jit def fc2_stg_post_f2fp_reorder( - *, - casted: cute.Tensor, # (subtile_cnt,) - fc2_output_dtype: Type[cutlass.Numeric], - tmem_subtile_view: cute.Tensor, # (epi_tile_m, epi_tile_n) - **_, -): + *, + casted: cute.Tensor, # (subtile_cnt,) + tmem_subtile_view: cute.Tensor, # (epi_tile_m, epi_tile_n) + **_): + if cutlass.const_expr(cute.size(casted) != 64): raise NotImplementedError( "fc2 stg pass expects 64 fp32 regs in total before store reorder.") @@ -2042,19 +2687,17 @@ def fc2_stg_post_f2fp_reorder( # read casted through (t, hidden, half) -> casted[t*16 + hidden + half*32] # in (t fastest) order -> [top0,bot0,top1,bot1,...] per half = packed bf16x2. # scatter: de-interleave the transposed natural-hidden regs back to the - # (token, hidden) pre-store order declared by _Fc2StgSubtilePreStoreContract. + # STG pre-store order (token = lane + 32*(vid//32), hidden = vid % 32). gather_top_bot_map = ((2, 16, 2), (16, 1, 32)) scatter_top_bot_map = ((16, 2, 2), (2, 1, 32)) - dtype = fc2_output_dtype + dtype = cutlass.BFloat16 packed = cute.make_rmem_tensor((64, ), dtype) cute.autovec_copy( cute.composition( casted, cute.make_layout((gather_top_bot_map[0], ), - stride=(gather_top_bot_map[1], ))), - packed, - ) + stride=(gather_top_bot_map[1], ))), packed) # Although this works... # packed.store( # cute.make_tensor( @@ -2065,62 +2708,42 @@ def fc2_stg_post_f2fp_reorder( packed_i32 = cute.recast_tensor(packed, cutlass.Float32) # (32,): 16 i32 per half - token_0_32_pre_scatter_back = TmemTranspose16x32Packed( + # Reuse the 32-bit transpose: each i32 slot carries one packed bf16x2 pair. + token_0_32_pre_scatter_back = TmemTranspose16x32( tmem_subtile_view.iterator, Region.Top, - reg_tensor=TensorWithContract( - tensor=cute.composition(packed_i32, (16, )), - contract=TmemTranspose16x32Packed.InputContract, - ), + reg_tensor=cute.composition(packed_i32, (16, )), ).from_r1_perm_until_last_store() - token_32_64_pre_scatter_back = TmemTranspose16x32Packed( + token_32_64_pre_scatter_back = TmemTranspose16x32( tmem_subtile_view.iterator + 32, Region.Top, - reg_tensor=TensorWithContract( - tensor=cute.composition(cute.domain_offset(16, packed_i32), (16, )), - contract=TmemTranspose16x32Packed.InputContract, - ), + reg_tensor=cute.composition(cute.domain_offset(16, packed_i32), (16, )), ).from_r1_perm_until_last_store() - cute.autovec_copy(token_0_32_pre_scatter_back.tensor, + cute.autovec_copy(token_0_32_pre_scatter_back, cute.zipped_divide(packed_i32, (16, ))[None, 0]) - cute.autovec_copy(token_32_64_pre_scatter_back.tensor, + cute.autovec_copy(token_32_64_pre_scatter_back, cute.zipped_divide(packed_i32, (16, ))[None, 1]) out = cute.make_rmem_tensor((64, ), dtype) cute.autovec_copy( cute.composition( packed, cute.make_layout((scatter_top_bot_map[0], ), - stride=(scatter_top_bot_map[1], ))), - out, - ) - return TensorWithContract(tensor=out, - contract=_Fc2StgSubtilePreStoreContract) - - -# (...) -> ((atom_v, 1)) -@cute.jit -def wrap_into_copy_standard_layout(tensor: cute.Tensor): - tensor = cute.coalesce(cute.flatten(tensor)) - tensor = cute.append_ones(tensor, cute.rank(tensor) + 1) - tensor = cute.group_modes(tensor, 0, cute.rank(tensor) - 1) - tensor = cute.group_modes(tensor, 0, cute.rank(tensor)) - return tensor + stride=(scatter_top_bot_map[1], ))), out) + return out @cute.jit def fc2_redg_post_f2fp_reorder( *, casted: cute.Tensor, - fc2_output_dtype: Type[cutlass.Numeric], tmem_subtile_view: cute.Tensor, **_, ): # (epi_tid, elem_idx) -> (token_64, hidden_128), each thread hold token_2 x hidden_32 natural = fc2_stg_post_f2fp_reorder( casted=casted, - fc2_output_dtype=fc2_output_dtype, tmem_subtile_view=tmem_subtile_view, - ).tensor + ) core_matrix_reorder_sttm_atom = cute.make_copy_atom( tcgen05.St32x32bOp(tcgen05.Repetition.x16), cutlass.Float32, @@ -2151,90 +2774,108 @@ def fc2_redg_post_f2fp_reorder( wrap_into_copy_standard_layout(current_sttm_src), wrap_into_copy_standard_layout( tmem_subtile_divided_by_token_group_divided_by_16dp[None, None, - None, i]), - ) + None, i])) cute.copy( core_matrix_reorder_ldtm_atom, wrap_into_copy_standard_layout( tmem_subtile_divided_by_token_group_divided_by_16dp[None, None, 0, i]), - wrap_into_copy_standard_layout(out_as_i32[(None, 0), i]), - ) + wrap_into_copy_standard_layout(out_as_i32[(None, 0), i])) cute.copy( core_matrix_reorder_ldtm_atom, wrap_into_copy_standard_layout( tmem_subtile_divided_by_token_group_divided_by_16dp[None, None, 1, i]), - wrap_into_copy_standard_layout(out_as_i32[(None, 1), i]), - ) + wrap_into_copy_standard_layout(out_as_i32[(None, 1), i])) - return TensorWithContract(tensor=cute.coalesce(out), - contract=_Fc2RedgSubtilePreStoreContract) + return cute.coalesce(out) @cute.jit def fc2_stg_store_function( - *, - epi, - subtile: TensorWithContract, - subtile_idx: cutlass.Int32, - fc2_output_router: Fc2OutputRouter, - **_, -): - assert_contract_equivalent( - subtile.contract, - _Fc2StgSubtilePreStoreContract, - context="fc2 STG store input", - ) - copy_atom_256b = cute.make_copy_atom( + *, + epi, + subtile: cute.Tensor, # Always bf16 pre quant tesnor + subtile_idx: cutlass.Int32, + fc2_output_router: Fc2OutputRouter, + **_): + if cutlass.const_expr(epi.combine_format.is_quantized): + data_subtile, sf_regs = QuantImpl(epi.combine_format, + "regs_in_thread")(subtile) + else: + data_subtile = subtile + sf_regs = None + stg_width_elems: cutlass.Constexpr[int] = min( + 32, 256 // data_subtile.element_type.width) + stg_bits: cutlass.Constexpr[ + int] = stg_width_elems * data_subtile.element_type.width + copy_atom_vec = cute.make_copy_atom( cute.nvgpu.CopyUniversalOp(), - subtile.tensor.element_type, - num_bits_per_copy=256, + cutlass.Int32, + num_bits_per_copy=stg_bits, ) - stg_width_elems: cutlass.Constexpr[ - int] = 256 // subtile.tensor.element_type.width - elem_axis: cutlass.Constexpr[int] = subtile.contract.domain.names.index( - "vid") - elems_per_thread: cutlass.Constexpr[int] = subtile.contract.domain.sizes[ - elem_axis] + elems_per_thread: cutlass.Constexpr[int] = cute.size(data_subtile) if cutlass.const_expr(elems_per_thread % stg_width_elems != 0): raise ValueError( "fc2 STG store requires pre-store elems per thread to be divisible " f"by STG issue width, got {elems_per_thread} and {stg_width_elems}." ) + + if cutlass.const_expr(sf_regs is not None): + sf_scales_per_stg: cutlass.Constexpr[ + int] = 32 // epi.combine_format.scale_block + token_groups_per_subtile: cutlass.Constexpr[ + int] = epi._EpilogueTokenTileSize // 32 + for token_group in cutlass.range_constexpr(token_groups_per_subtile): + sf_iter = cutlass.Int32(subtile_idx) * cutlass.Int32( + token_groups_per_subtile) + token_group + sf_ptr, sf_pred = fc2_output_router.get_sf_dst(sf_iter) + if sf_pred != cutlass.Int32(0): + sf_dst = cute.make_tensor( + sf_ptr, cute.make_layout((sf_scales_per_stg, ))) + for k in cutlass.range_constexpr(sf_scales_per_stg): + sf_dst[k] = sf_regs[token_group * sf_scales_per_stg + k] + iters_per_subtile: cutlass.Constexpr[ int] = elems_per_thread // stg_width_elems - copy_src = cute.zipped_divide(subtile.tensor, (stg_width_elems, )) - copy_pairs = fc2_output_router.resolve( - copy_src, - iters_per_subtile, - subtile_idx, - ) + copy_src = cute.zipped_divide(data_subtile, (stg_width_elems, )) + single_copy_layout = cute.make_layout(((stg_width_elems, 1), ), + stride=((1, 0), )) subtile_iter_base = cutlass.Int32(subtile_idx) * cutlass.Int32( iters_per_subtile) for local_iter in cutlass.range_constexpr(iters_per_subtile): global_iter = subtile_iter_base + cutlass.Int32(local_iter) - if fc2_output_router.valid[global_iter] != cutlass.Int32(0): - copy_src_i, copy_dst_i = copy_pairs[local_iter] - cute.copy(copy_atom_256b, copy_src_i, copy_dst_i) + dst_ptr, pred = fc2_output_router.get_data_dst(global_iter) + if pred != cutlass.Int32(0): + src_i = cute.make_tensor(copy_src[None, local_iter].iterator, + single_copy_layout) + dst_i = cute.make_tensor(dst_ptr, single_copy_layout) + cute.copy(copy_atom_vec, cute.recast_tensor(src_i, cutlass.Int32), + cute.recast_tensor(dst_i, cutlass.Int32)) @cute.jit def fc2_ublk_store_function_impl( *, epi, - subtile: TensorWithContract, + subtile: cute.Tensor, # Always bf16 pre-quant tensor subtile_idx: cutlass.Int32, fc2_output_router: Fc2OutputRouter, + **_, ): - assert_contract_equivalent( - subtile.contract, - epi.process_pipeline.pre_store_contract, - context="fc2 UBLK store input", - ) smem_tensor = epi.smem_tensor if cutlass.const_expr(smem_tensor is None): raise ValueError("fc2 UBLK store requires epi.smem_tensor.") + quantized: cutlass.Constexpr[bool] = epi.combine_format.is_quantized + if cutlass.const_expr(quantized): + data_subtile, selected_sf = QuantImpl( + epi.combine_format, + "threads_with_the_same_reg", + lane_idx=epi.lane_idx, + )(subtile) + else: + data_subtile = subtile + selected_sf = None smem_read_write_bar = pipeline.NamedBarrier( barrier_id=SwapABSwigluFp4Epilogue._EpilogueSyncWaitBarId, @@ -2244,10 +2885,7 @@ def fc2_ublk_store_function_impl( lane_idx = epi.lane_idx warp_hidden_base = cutlass.Int32(warp_idx * 32) - vid_axis: cutlass.Constexpr[int] = subtile.contract.domain.names.index( - "vid") - regs_per_thread: cutlass.Constexpr[int] = subtile.contract.domain.sizes[ - vid_axis] + regs_per_thread: cutlass.Constexpr[int] = cute.size(data_subtile) tokens_per_smem_slice: cutlass.Constexpr[int] = cute.size(smem_tensor, mode=[0]) if cutlass.const_expr(regs_per_thread % tokens_per_smem_slice != 0): @@ -2257,55 +2895,59 @@ def fc2_ublk_store_function_impl( ) loop_cnt: cutlass.Constexpr[int] = regs_per_thread // tokens_per_smem_slice - for loop_idx in cutlass.range_constexpr(loop_cnt): - if cutlass.const_expr(loop_idx > 0): + # SF straight out (no smem): the sf_vec lanes of a block share the CREDUX + # result, so each emits its slice -- one scale per slot. The sf mapping owns + # the (lane, slot) -> (token, hidden) layout; selected_sf[slot] is this + # lane's slot-th scale, aligned to sf_iter = subtile_idx*iters_per_subtile+slot. + if cutlass.const_expr(quantized): + # A scale block = sf_vec hidden = sf_vec lanes (UBLK), and those lanes + # split the subtile's tokens, so each emits subtile_tokens // lanes_per_block. + lanes_per_block: cutlass.Constexpr[int] = epi.combine_format.scale_block + iters_per_subtile: cutlass.Constexpr[ + int] = epi._EpilogueTokenTileSize // lanes_per_block + for slot in cutlass.range_constexpr(iters_per_subtile): + sf_iter = cutlass.Int32(subtile_idx) * cutlass.Int32( + iters_per_subtile) + slot + sf_ptr, sf_pred = fc2_output_router.get_sf_dst(sf_iter) + if sf_pred != cutlass.Int32(0): + sf_dst = cute.make_tensor(sf_ptr, cute.make_layout((1, ))) + sf_dst[0] = selected_sf[slot] + + for token32_group_idx in cutlass.range_constexpr(loop_cnt): + if cutlass.const_expr(token32_group_idx > 0): cute.arch.cp_async_bulk_wait_group(0, read=True) - cute.arch.sync_warp() smem_read_write_bar.arrive_and_wait() - # R2S: materialize this loop's token slice into the fixed - # (token_rows, hidden_128) scratch tile. The pre-store contract says - # each lane owns one hidden column over all subtile tokens, so loop_idx - # simply selects the next contiguous token_rows chunk from RMEM. + # R2S transpose: each lane writes its hidden column's 32 token rows for + # this group. (SF already went straight out above; only DATA transposes + # through smem here.) for token_i in cutlass.range_constexpr(tokens_per_smem_slice): - src_reg = token_i + tokens_per_smem_slice * loop_idx + src_reg = token_i + tokens_per_smem_slice * token32_group_idx smem_tensor[token_i, - warp_hidden_base + lane_idx] = subtile.tensor[src_reg] + warp_hidden_base + lane_idx] = data_subtile[src_reg] cute.arch.fence_proxy("async.shared", space="cta") - cute.arch.sync_warp() smem_read_write_bar.arrive_and_wait() - iter_idx = subtile_idx // cutlass.Int32(2) - store_coord = eval_function_mapping( - fc2_output_router.output_mapping, - epi_tid=epi.tidx, - iter_idx=iter_idx, - ) - token_in_cta_tile = cutlass.Int32(store_coord["token_in_cta_tile"]) - slice_token_start = subtile_idx * cutlass.Int32( - epi._EpilogueTokenTileSize) + cutlass.Int32( - loop_idx * tokens_per_smem_slice) - slice_token_end = slice_token_start + cutlass.Int32( - tokens_per_smem_slice) - - if (fc2_output_router.valid[iter_idx] != cutlass.Int32(0) - and token_in_cta_tile >= slice_token_start - and token_in_cta_tile < slice_token_end): - scratch_row = token_in_cta_tile - slice_token_start + # ublk_iter_idx is constexpr so get_dst indexes a constexpr slot (no spill). + # Gate / scratch_row specialize the store-out mapping: lane_idx//8 picks the + # subtile, warp_idx*8 + lane_idx%8 is the token's row within the 32-group. + ublk_iter_idx = token32_group_idx + # SF already went straight out above; here only the DATA row is pushed. + dst_ptr, pred = fc2_output_router.get_data_dst(ublk_iter_idx) + if pred != cutlass.Int32(0) and (lane_idx // + cutlass.Int32(8)) == subtile_idx: + scratch_row = warp_idx * cutlass.Int32( + 8) + lane_idx % cutlass.Int32(8) copy_elems = cutlass.Int32(128) if cutlass.const_expr(epi.fc2_hidden_needs_predicate): copy_elems = cutlass.Int32( fc2_output_router.valid_hidden_this_cta_tile) - copy_bytes = copy_elems * epi.fc2_output_dtype.width // 8 + # smem_tensor holds the combine wire dtype, so the bulk byte count + # scales with the wire width, not the bf16 compute dtype. + copy_bytes = copy_elems * epi.combine_format.act_dtype.width // 8 src_row = cute.slice_(smem_tensor, (scratch_row, None)) - dst_ptr = cute.make_ptr( - src_row.element_type, - fc2_output_router.dst_ptrs[iter_idx], - AddressSpace.gmem, - assumed_align=16, - ) if cutlass.const_expr(epi.reduce_topk_in_kernel): _cp_reduce_async_bulk_add_noftz_bf16_s2g( dst_ptr, @@ -2331,21 +2973,13 @@ def fc2_ublk_store_function_impl( def fc2_redg_store_function( *, epi, - subtile: TensorWithContract, + subtile: cute.Tensor, # Always bf16; in-kernel reduce never quantizes subtile_idx: cutlass.Int32, fc2_output_router: Fc2OutputRouter, **_, ): - assert_contract_equivalent( - subtile.contract, - _Fc2RedgSubtilePreStoreContract, - context="fc2 REDG store input", - ) redg_width_elems: cutlass.Constexpr[int] = 4 - elem_axis: cutlass.Constexpr[int] = subtile.contract.domain.names.index( - "elem_idx") - elems_per_thread: cutlass.Constexpr[int] = subtile.contract.domain.sizes[ - elem_axis] + elems_per_thread: cutlass.Constexpr[int] = cute.size(subtile) if cutlass.const_expr(elems_per_thread % redg_width_elems != 0): raise ValueError( "fc2 REDG store requires pre-store elems per thread to be divisible " @@ -2355,20 +2989,14 @@ def fc2_redg_store_function( int] = elems_per_thread // redg_width_elems subtile_iter_base = cutlass.Int32(subtile_idx) * cutlass.Int32( iters_per_subtile) - subtile_by_redg_issue = cute.zipped_divide(subtile.tensor, - (redg_width_elems, )) + subtile_by_redg_issue = cute.zipped_divide(subtile, (redg_width_elems, )) for local_iter in cutlass.range_constexpr(iters_per_subtile): global_iter = subtile_iter_base + cutlass.Int32(local_iter) - if fc2_output_router.valid[global_iter] != cutlass.Int32(0): + dst_ptr, pred = fc2_output_router.get_data_dst(global_iter) + if pred != cutlass.Int32(0): bf16x4 = subtile_by_redg_issue[None, local_iter] packed_bf16x2 = cute.recast_tensor(bf16x4, cutlass.Float32) - dst_ptr = cute.make_ptr( - fc2_output_router.base_output.element_type, - fc2_output_router.dst_ptrs[global_iter], - AddressSpace.gmem, - assumed_align=8, - ) _red_add_relaxed_sys_v2_bf16x2( dst_ptr, cutlass.Float32(packed_bf16x2[0]), @@ -2378,12 +3006,12 @@ def fc2_redg_store_function( def make_fc2_stg_process_pipeline( *, - fc2_output_dtype: Type[cutlass.Numeric], + combine_format: CombineFormat, cta_token_tile_size: int, cta_hidden_tile_size: int, ) -> Fc2ProcessPipeline: - store_out_mapping, fundamental_mapping = make_fc2_stg_cta_store_out_contract( - fc2_output_dtype, + store_out_mapping, sf_store_out_mapping, fundamental_mapping = make_fc2_stg_cta_store_out_contract( + combine_format, cta_token_tile_size, cta_hidden_tile_size, ) @@ -2392,21 +3020,21 @@ def make_fc2_stg_process_pipeline( f2fp=fc2_f2fp, post_f2fp_reorder=fc2_stg_post_f2fp_reorder, store_function=fc2_stg_store_function, - pre_store_contract=_Fc2StgSubtilePreStoreContract, fc2_cta_tile_contract=fundamental_mapping, store_out_mapping=store_out_mapping, + sf_store_out_mapping=sf_store_out_mapping, require_tmem_trans=True, ) def make_fc2_redg_process_pipeline( *, - fc2_output_dtype: Type[cutlass.Numeric], + combine_format: CombineFormat, cta_token_tile_size: int, cta_hidden_tile_size: int, ) -> Fc2ProcessPipeline: - store_out_mapping, fundamental_mapping = make_fc2_redg_cta_store_out_contract( - fc2_output_dtype, + store_out_mapping, sf_store_out_mapping, fundamental_mapping = make_fc2_redg_cta_store_out_contract( + combine_format, cta_token_tile_size, cta_hidden_tile_size, ) @@ -2415,21 +3043,21 @@ def make_fc2_redg_process_pipeline( f2fp=fc2_f2fp, post_f2fp_reorder=fc2_redg_post_f2fp_reorder, store_function=fc2_redg_store_function, - pre_store_contract=_Fc2RedgSubtilePreStoreContract, fc2_cta_tile_contract=fundamental_mapping, store_out_mapping=store_out_mapping, + sf_store_out_mapping=sf_store_out_mapping, require_tmem_trans=True, ) def make_fc2_ublk_process_pipeline( *, - fc2_output_dtype: Type[cutlass.Numeric], + combine_format: CombineFormat, cta_token_tile_size: int, cta_hidden_tile_size: int, ) -> Fc2ProcessPipeline: - store_out_mapping, fundamental_mapping = make_fc2_ublk_store_out_contract( - fc2_output_dtype, + store_out_mapping, sf_store_out_mapping, fundamental_mapping = make_fc2_ublk_store_out_contract( + combine_format, cta_token_tile_size, cta_hidden_tile_size, ) @@ -2438,306 +3066,8 @@ def make_fc2_ublk_process_pipeline( f2fp=fc2_f2fp, post_f2fp_reorder=post_f2fp_reorder_identity, store_function=fc2_ublk_store_function_impl, - pre_store_contract=_Fc2UblkSubtilePreStoreContract, fc2_cta_tile_contract=fundamental_mapping, store_out_mapping=store_out_mapping, + sf_store_out_mapping=sf_store_out_mapping, require_tmem_trans=False, ) - - -# Device only object -class SwapABFc2Epilogue: - - def __init__( - self, - base: SwapABSwigluFp4Epilogue, - tidx: cutlass.Int32, - epi_smem_storage, - fc2_output: cute.Tensor, # MoE domain (token, topk, hidden) - token_comm_args: TokenCommArgs, - optional_epi_args: NvFp4OptinalEpiArgs, - ): - self.base = base - self.tidx = tidx % (base._EpilogueWarpCnt * 32) - self.warp_idx = self.tidx // 32 - self.lane_idx = self.tidx % 32 - self.fc2_output = fc2_output - self.token_comm_args = token_comm_args - self.optional_epi_args = optional_epi_args - if cutlass.const_expr(base.fc2_use_bulk): - fc2_smem_rows = (base.epi_smem_bytes * 8 // - base._EpilogueFc2HiddenTileSize // - base.fc2_output_dtype.width) - if cutlass.const_expr(fc2_smem_rows != 32): - raise NotImplementedError( - "Remember to adjust fc2 smem structure if switch to non-bf16 combine." - ) - self.smem_tensor = cute.make_tensor( - cute.recast_ptr( - epi_smem_storage.epi_smem.data_ptr(), - dtype=base.fc2_output_dtype, - ), - cute.make_layout( - (fc2_smem_rows, base._EpilogueFc2HiddenTileSize), - stride=(base._EpilogueFc2HiddenTileSize, 1), - ), - ) - self.process_pipeline = make_fc2_ublk_process_pipeline( - fc2_output_dtype=base.fc2_output_dtype, - cta_token_tile_size=base.cta_tile_n, - cta_hidden_tile_size=base.cta_tile_m, - ) - else: - self.smem_tensor = None - if cutlass.const_expr(base.reduce_topk_in_kernel): - self.process_pipeline = make_fc2_redg_process_pipeline( - fc2_output_dtype=base.fc2_output_dtype, - cta_token_tile_size=base.cta_tile_n, - cta_hidden_tile_size=base.cta_tile_m, - ) - else: - self.process_pipeline = make_fc2_stg_process_pipeline( - fc2_output_dtype=base.fc2_output_dtype, - cta_token_tile_size=base.cta_tile_n, - cta_hidden_tile_size=base.cta_tile_m, - ) - - def __getattr__(self, name): - return getattr(object.__getattribute__(self, "base"), name) - - def __extract_mlir_values__(self) -> List[ir.Value]: - # See SwapABFc1Epilogue.__extract_mlir_values__: this helper carries - # only loop-invariant Python context. It intentionally serializes no - # MLIR values, so changing it to store loop-carried state would be a - # correctness bug. - return [] - - def __new_from_mlir_values__(self, - values: List[ir.Value]) -> "SwapABFc2Epilogue": - assert len(values) == 0 - return self - - @cute.jit - def signal_fc2_done(self, work_tile_info): - if cutlass.const_expr(self.token_back_by_dispatch): - if self.tidx == 0: - _red_add_release_gpu_s32( - self.token_comm_args.fc2_done_counter.iterator + - work_tile_info.expert_idx, - cutlass.Int32(1), - ) - - @cute.jit - def _make_output_router( - self, - work_tile_info: MoEWorkTileInfo, - ) -> Fc2OutputRouter: - task_tile_data_row_start = ( - work_tile_info.cumulative_data_physical_row + - work_tile_info.tile_n_idx * cutlass.Int32(self.cta_tile_n)) - hidden_base_this_cta_tile = work_tile_info.tile_m_idx * cutlass.Int32( - self.cta_tile_m) - valid_hidden_this_cta_tile = (cutlass.Int32(self.fc2_output.shape[2]) - - hidden_base_this_cta_tile) - if valid_hidden_this_cta_tile < 0: - valid_hidden_this_cta_tile = 0 - if valid_hidden_this_cta_tile > self._EpilogueFc2HiddenTileSize: - valid_hidden_this_cta_tile = self._EpilogueFc2HiddenTileSize - - metadata_u32 = None - peer_rank_ptr_mapper = None - direct_token_base_this_cta_tile = task_tile_data_row_start - if cutlass.const_expr(self.token_comm_args is not None - and not self.token_back_by_dispatch): - metadata_u32 = cute.domain_offset( - (task_tile_data_row_start, 0), - cute.recast_tensor( - self.token_comm_args.token_src_metadata, - cutlass.Uint32, - ), - ) - peer_rank_ptr_mapper = self.token_comm_args.peer_rank_ptr_mapper - direct_token_base_this_cta_tile = None - - return Fc2OutputRouter( - metadata=metadata_u32, - direct_token_base_this_cta_tile=direct_token_base_this_cta_tile, - base_output=self.fc2_output, - hidden_base_this_cta_tile=hidden_base_this_cta_tile, - peer_rank_ptr_mapper=peer_rank_ptr_mapper, - valid_tokens_this_cta_tile=work_tile_info.valid_tokens_in_tile, - valid_hidden_this_cta_tile=valid_hidden_this_cta_tile, - reduce_topk_in_kernel=self.reduce_topk_in_kernel, - output_mapping=self.process_pipeline.store_out_mapping, - epi_tid=self.tidx, - ).prefetch() - - @cute.jit - def __call__( - self, - work_tile_info: MoEWorkTileInfo, - tmem_acc_tensor: cute.Tensor, - acc_pipeline, - acc_consumer_state, - is_odd_turn: cutlass.Int32, - ): - # subtile-irrelevant hoist: fc2 alpha scales raw fc2 accumulators before f2fp. - if cutlass.const_expr(self.optional_epi_args.fc2_alpha is not None): - alpha_val = self.optional_epi_args.fc2_alpha[ - work_tile_info.expert_idx] - else: - alpha_val = None - acc_ready = False - if not work_tile_info.peek_ready: - acc_ready = True - acc_pipeline.consumer_wait(acc_consumer_state) - fc2_output_router = self._make_output_router(work_tile_info) - # (cta_tile_m, cta_tile_n) -> (epi_tile_m, epi_tile_n, iters) - tmem_acc_tensor_tiled_by_epi_tile = cute.flat_divide( - tmem_acc_tensor, (self._EpilogueFc2HiddenTileSize, - self._EpilogueTokenTileSize))[None, None, 0, None] - - acc_pipeline.consumer_wait(acc_consumer_state, acc_ready) - iket.range_push("fc2_epi") - valid_tokens = work_tile_info.valid_tokens_in_tile - - # Overlap path preloads two subtiles before releasing acc TMEM. - unroll_tile_cnt = (2 if cutlass.const_expr( - self.overlapping_accum and self.process_pipeline.require_tmem_trans) - else 0) - remain_subtile_cnt = self.subtile_cnt - unroll_tile_cnt - - if cutlass.const_expr(unroll_tile_cnt > 0): - subtile_idx_first = (cutlass.Int32(self.subtile_cnt) - - is_odd_turn) % cutlass.Int32(self.subtile_cnt) - subtile_idx_second = (cutlass.Int32(self.subtile_cnt + 1) - - is_odd_turn) % cutlass.Int32(self.subtile_cnt) - - # preload_subtile_first: subtile_idx_first's raw PRE-transpose acc, LDTM'd by - # all 128 epi threads into 4 reg tensors == the 4 quadrants of the subtile's - # (128 tmem_dp x 64 tmem_col) footprint. Only these raw-TMEM offsets are - # guaranteed: - # reg[0]/reg[1], reg[2]/reg[3] : top vs bot -> 16 apart in tmem_dp - # reg[0]/reg[2], reg[1]/reg[3] : 1st vs 2nd half -> 32 apart in tmem_col - # (so reg[0..1] = the first 128x32, reg[2..3] = the second 128x32 of the 128x64.) - # The per-lane (lane_idx, elem_idx) -> (tmem_dp, tmem_col) layout INSIDE each - # reg tensor is opaque -- do not assume it; it only becomes well-defined once - # the tmem transpose consumes them. - preload_subtile_first: Tuple[ - cute.Tensor, cute.Tensor, cute.Tensor, - cute.Tensor] = (_TmemTranspose16x32Core.load_subtile_raw_acc( - tmem_acc_tensor_tiled_by_epi_tile[None, None, - subtile_idx_first])) - - # Release acc to next MMA unconditionally. - cute.arch.fence_view_async_tmem_load() - acc_pipeline.consumer_release(acc_consumer_state) - - # preload_subtile_second: same 128 tmem_dp x 64 tmem_col footprint, but for - # subtile_idx_second (the other token subtile, not the 2nd col-half). Same - # quadrant/offset invariants and opaque per-lane layout as above. - preload_subtile_second: Tuple[ - cute.Tensor, cute.Tensor, cute.Tensor, - cute.Tensor] = (_TmemTranspose16x32Core.load_subtile_raw_acc( - tmem_acc_tensor_tiled_by_epi_tile[None, None, - subtile_idx_second])) - - # Both unrolled subtiles borrow tmem_subtile_second as workspace. - preload_pair = (preload_subtile_first, preload_subtile_second) - subtile_idx_pair = (subtile_idx_first, subtile_idx_second) - for i in cutlass.range_constexpr(unroll_tile_cnt): - if subtile_idx_pair[i] * cutlass.Int32( - self._EpilogueTokenTileSize) < valid_tokens: - self.run_subtile( - subtile_idx=subtile_idx_pair[i], - tmem_subtile_tensor=tmem_acc_tensor_tiled_by_epi_tile[ - None, None, subtile_idx_second], - preload_acc=preload_pair[i], - fc2_output_router=fc2_output_router, - alpha_val=alpha_val, - release_after_ldtm=False, - acc_pipeline=acc_pipeline, - acc_consumer_state=acc_consumer_state, - ) - - if cutlass.const_expr(self.overlapping_accum and unroll_tile_cnt == 0): - release_after_ldtm = True - else: - release_after_ldtm = False - for i in cutlass.range(remain_subtile_cnt, unroll=1): - # for i in cutlass.range_constexpr(remain_subtile_cnt): - real_i = i + unroll_tile_cnt - if cutlass.const_expr(self.overlapping_accum): - subtile_idx = (cutlass.Int32(real_i + self.subtile_cnt) - - is_odd_turn) % cutlass.Int32(self.subtile_cnt) - else: - subtile_idx = cutlass.Int32(real_i) - - if subtile_idx * cutlass.Int32( - self._EpilogueTokenTileSize) < valid_tokens: - self.run_subtile( - subtile_idx=subtile_idx, - tmem_subtile_tensor=tmem_acc_tensor_tiled_by_epi_tile[ - None, None, subtile_idx], - preload_acc=None, - fc2_output_router=fc2_output_router, - alpha_val=alpha_val, - release_after_ldtm=release_after_ldtm, - acc_pipeline=acc_pipeline, - acc_consumer_state=acc_consumer_state, - ) - release_after_ldtm = False - - # Non-overlap-path release: at the natural task-tile boundary. - if cutlass.const_expr(not self.overlapping_accum): - cute.arch.fence_view_async_tmem_load() - acc_pipeline.consumer_release(acc_consumer_state) - - @cute.jit - def run_subtile( - self, - subtile_idx: cutlass.Int32, - # (hidden_tile, token_subtile), fundamentally (epi_tile_m, epi_tile_n) - tmem_subtile_tensor: cute.Tensor, - preload_acc: Optional[Tuple[cute.Tensor, cute.Tensor, cute.Tensor, - cute.Tensor]], - fc2_output_router: Fc2OutputRouter, - alpha_val: Optional[cutlass.Float32], - release_after_ldtm: Union[cutlass.Boolean, bool], - acc_pipeline, - acc_consumer_state, - ): - process_pipeline = self.process_pipeline - if cutlass.const_expr(preload_acc is None): - loaded = process_pipeline.tmem_acc_load( - tmem_subtile_tensor=tmem_subtile_tensor, - epi=self, - ) - if release_after_ldtm: - cute.arch.fence_view_async_tmem_load() - acc_pipeline.consumer_release(acc_consumer_state) - else: - loaded = preload_acc - - casted = process_pipeline.f2fp( - *loaded, - fc2_output_dtype=self.fc2_output_dtype, - alpha_val=alpha_val, - ) - pre_store = process_pipeline.post_f2fp_reorder( - casted=casted, - contract=process_pipeline.pre_store_contract, - fc2_output_dtype=self.fc2_output_dtype, - tmem_subtile_view=tmem_subtile_tensor, - ) - assert_contract_equivalent( - pre_store.contract, - process_pipeline.pre_store_contract, - context="fc2 process pipeline pre-store", - ) - process_pipeline.store_function( - epi=self, - subtile=pre_store, - subtile_idx=subtile_idx, - fc2_output_router=fc2_output_router, - ) diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/fc1_fc2_fuse_sched.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/fc1_fc2_fuse_sched.py index 4602434b292f..09412f1690a3 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/fc1_fc2_fuse_sched.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/fc1_fc2_fuse_sched.py @@ -1,5 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause """Fused fc1 + fc2 MegaMoE scheduler.""" from enum import IntEnum @@ -657,19 +659,31 @@ def create( num_fc2_hidden_blocks = (hidden + params.cluster_tile_n - 1) // params.cluster_tile_n - # current_work init must use ext.WorkTileInfo (8 fields) to match the - # shape that gen_next_work writes; otherwise MLIR serialization slot - # Scheduler emits the final 8-field work tile directly. - current_work = ext.WorkTileInfo( - expert_idx=Int32(WorkTileState.DONE), - tile_m_idx=Int32(0), - tile_n_idx=Int32(0), - cumulative_data_physical_row=Int32(0), - cumulative_sf_physical_row=Int32(0), - cumulative_token_block_count=Int32(0), - valid_tokens_in_tile=Int32(0), - phase_and_peek=Int32(BlockPhase.None_), - ) + # current_work init must use ext.WorkTileInfo to match the shape that + # gen_next_work writes; otherwise MLIR serialization slots would differ. + if const_expr(params.is_swap_ab): + current_work = ext.WorkTileInfo( + expert_idx=Int32(WorkTileState.DONE), + tile_m_idx=Int32(0), + tile_n_idx=Int32(0), + cumulative_data_physical_row=Int32(0), + cumulative_sf_physical_row=Int32(0), + cumulative_token_block_count=Int32(0), + valid_tokens_in_cta_tile=Int32(0), + phase_and_peek=Int32(BlockPhase.None_), + ) + else: + current_work = ext.WorkTileInfo( + expert_idx=Int32(WorkTileState.DONE), + tile_m_idx=Int32(0), + tile_n_idx=Int32(0), + cumulative_data_physical_row=Int32(0), + cumulative_sf_physical_row=Int32(0), + cumulative_token_block_count=Int32(0), + valid_tokens_in_cta_cluster_tile=Int32(0), + phase_and_peek=Int32(BlockPhase.None_), + fc1_counter_index=Int32(0), + ) sched_producer_group = pipeline.CooperativeGroup( pipeline.Agent.Thread, 32) @@ -855,9 +869,7 @@ def _advance_work_linear_tile_idx_dynamic( result across the sched warp. Cluster-internal protocol (DSMEM broadcast + cluster_pipeline mbar - wait) is identical between the two paths. Mirrors - ``cute_dsl_kernel_library/dsl_kernels/moe/moe_persistent_scheduler.py`` - ``_fetch_next_cluster_idx`` (lines 838-894). + wait) is identical between the two paths. """ ds = self._dynamic_state cluster_pipeline = self._cluster_pipeline @@ -1176,14 +1188,14 @@ def _decode_inside_expert( cluster_intermediate_or_hidden_block_idx * params.cluster_shape_mn[1] + self.cta_id_in_cluster[1]) - # valid_tokens_in_tile: clip cta_tile_m tokens at the current expert + # valid_tokens_in_cta_tile: clip cta_tile_m tokens at the current expert # right boundary. token_idx_start_in_expert = cta_token_block_idx * Int32(cta_tile_m) remaining_in_expert = (state.current_this_expert_token_cnt - token_idx_start_in_expert) remaining_in_expert = cutlass.max(remaining_in_expert, Int32(0)) - valid_tokens_in_tile = cutlass.min(remaining_in_expert, - Int32(cta_tile_m)) + valid_tokens_in_cta_tile = cutlass.min(remaining_in_expert, + Int32(cta_tile_m)) # Swap scheduler-internal M/N back to GEMM-domain M/N on output. if const_expr(params.is_swap_ab): @@ -1194,16 +1206,41 @@ def _decode_inside_expert( tile_n_idx = cta_intermediate_or_hidden_block_idx # ext.enrich_work_tile_info may OR the peek bit into phase_and_peek. - return self._ext.WorkTileInfo( - expert_idx=state.current_expert_idx, - tile_m_idx=tile_m_idx, - tile_n_idx=tile_n_idx, - cumulative_data_physical_row=state.current_data_cumul, - cumulative_sf_physical_row=state.current_sf_cumul, - cumulative_token_block_count=state.current_token_block_cumul, - valid_tokens_in_tile=valid_tokens_in_tile, - phase_and_peek=state.current_phase, - ) + if const_expr(params.is_swap_ab): + return self._ext.WorkTileInfo( + expert_idx=state.current_expert_idx, + tile_m_idx=tile_m_idx, + tile_n_idx=tile_n_idx, + cumulative_data_physical_row=state.current_data_cumul, + cumulative_sf_physical_row=state.current_sf_cumul, + cumulative_token_block_count=state.current_token_block_cumul, + valid_tokens_in_cta_tile=valid_tokens_in_cta_tile, + phase_and_peek=state.current_phase, + ) + else: + fc1_counter_index = cluster_token_block_idx + cluster_tile_m = params.cluster_shape_mn[0] * cta_tile_m + cluster_start = cluster_token_block_idx * Int32(cluster_tile_m) + remaining_cluster = cutlass.max( + state.current_this_expert_token_cnt - cluster_start, Int32(0)) + valid_tokens_in_cluster_tile = cutlass.min(remaining_cluster, + Int32(cluster_tile_m)) + # Pack: high 16b = per-CTA tile count, low 16b = cluster-level count. + valid_tokens_in_cta_cluster_tile = ( + (valid_tokens_in_cta_tile << Int32(16)) + | valid_tokens_in_cluster_tile) + return self._ext.WorkTileInfo( + expert_idx=state.current_expert_idx, + tile_m_idx=tile_m_idx, + tile_n_idx=tile_n_idx, + cumulative_data_physical_row=state.current_data_cumul, + cumulative_sf_physical_row=state.current_sf_cumul, + cumulative_token_block_count=state.current_token_block_cumul, + valid_tokens_in_cta_cluster_tile= + valid_tokens_in_cta_cluster_tile, + phase_and_peek=state.current_phase, + fc1_counter_index=fc1_counter_index, + ) @dsl_user_op @cute.jit @@ -1218,16 +1255,29 @@ def _gen_work_from_cluster_idx( state = self._fused_state # Sentinel-by-default work tile; conditionally overwritten by decode. - base_work = self._ext.WorkTileInfo( - expert_idx=Int32(WorkTileState.DONE), - tile_m_idx=Int32(0), - tile_n_idx=Int32(0), - cumulative_data_physical_row=Int32(0), - cumulative_sf_physical_row=Int32(0), - cumulative_token_block_count=Int32(0), - valid_tokens_in_tile=Int32(0), - phase_and_peek=Int32(BlockPhase.None_), - ) + if const_expr(self.params.is_swap_ab): + base_work = self._ext.WorkTileInfo( + expert_idx=Int32(WorkTileState.DONE), + tile_m_idx=Int32(0), + tile_n_idx=Int32(0), + cumulative_data_physical_row=Int32(0), + cumulative_sf_physical_row=Int32(0), + cumulative_token_block_count=Int32(0), + valid_tokens_in_cta_tile=Int32(0), + phase_and_peek=Int32(BlockPhase.None_), + ) + else: + base_work = self._ext.WorkTileInfo( + expert_idx=Int32(WorkTileState.DONE), + tile_m_idx=Int32(0), + tile_n_idx=Int32(0), + cumulative_data_physical_row=Int32(0), + cumulative_sf_physical_row=Int32(0), + cumulative_token_block_count=Int32(0), + valid_tokens_in_cta_cluster_tile=Int32(0), + phase_and_peek=Int32(BlockPhase.None_), + fc1_counter_index=Int32(0), + ) # DSL carry for mutated self and while-condition fields. outer_group_end = state.current_group_end diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/flag_batch.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/flag_batch.py new file mode 100644 index 000000000000..ffc1a3c8b169 --- /dev/null +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/flag_batch.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Rotating-lane delayed release helper for done-counter publishing.""" + +import dataclasses +from typing import Any + +import cutlass +import cutlass.cute as cute +from cutlass.cute.typing import AddressSpace +from cutlass.cutlass_dsl import Int64 + +from .ptx_helpers import red_add_release_gpu_s32 + + +@dataclasses.dataclass(frozen=True) +class GpuReleaseFlagBatchTracker: + """Batched done-counter publisher using GPU-scope release reductions. + + Carries the loop-carried per-thread accumulation state. + """ + + flag_addr: Int64 # per-lane counter-slot address (0 == null) + cumulated_flags: cutlass.Int32 # current batch fill count (uniform) + phase: cutlass.Int32 # current accumulated phase (uniform) + tid: cutlass.Int32 # lane/thread id within the rotating group + + @cute.jit + def _make( + self, + flag_addr: Int64, + cumulated_flags: cutlass.Int32, + phase: cutlass.Int32, + ) -> "GpuReleaseFlagBatchTracker": + return GpuReleaseFlagBatchTracker( + flag_addr=flag_addr, + cumulated_flags=cumulated_flags, + phase=phase, + tid=self.tid, + ) + + @cute.jit + def fire(self) -> None: + """Publish this lane's pending slot.""" + if self.flag_addr != Int64(0): + ptr = cute.make_ptr( + cutlass.Int32, + self.flag_addr, + AddressSpace.gmem, + assumed_align=4, + ) + red_add_release_gpu_s32(ptr, cutlass.Int32(1)) + + @cute.jit + def accumulate( + self, + next_phase: Any, + flush_threshold: int, + flag_addr: Int64, + no_fire: bool = False, + ) -> "GpuReleaseFlagBatchTracker": + if cutlass.const_expr(flush_threshold == 1): + if cutlass.const_expr(not no_fire): + per_lane_addr = Int64(0) + if self.tid == 0: + per_lane_addr = flag_addr + self._make( + flag_addr=per_lane_addr, + cumulated_flags=cutlass.Int32(1), + phase=self.phase, + ).fire() + return self._make( + flag_addr=Int64(0), + cumulated_flags=cutlass.Int32(0), + phase=cutlass.Int32(next_phase), + ) + + cur_addr = self.flag_addr + cumulated = self.cumulated_flags + if self.tid == cumulated: + cur_addr = flag_addr + cumulated = cumulated + 1 + + if cumulated == flush_threshold or next_phase != self.phase: + if not no_fire: + self._make( + flag_addr=cur_addr, + cumulated_flags=cumulated, + phase=self.phase, + ).fire() + cumulated = cutlass.Int32(0) + cur_addr = Int64(0) + + return self._make( + flag_addr=cur_addr, + cumulated_flags=cumulated, + phase=cutlass.Int32(next_phase), + ) diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/grid_sync.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/grid_sync.py index 96f9f27e5a02..e5c21708e1c6 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/grid_sync.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/grid_sync.py @@ -119,11 +119,12 @@ def software_grid_sync(counter_ptr, "selp.b32 %delta, $2, $3, %is_sm0;\n\t" "atom.release.gpu.global.add.u32 %old, [$0], %delta;\n\t" "SPIN:\n\t" - "ld.acquire.gpu.global.b32 %cur, [$0];\n\t" + "ld.relaxed.gpu.global.b32 %cur, [$0];\n\t" "xor.b32 %cur, %cur, %old;\n\t" "and.b32 %cur, %cur, 0x80000000;\n\t" "setp.eq.u32 %waiting, %cur, 0;\n\t" "@%waiting bra SPIN;\n\t" + "fence.acq_rel.gpu;\n\t" "DONE:\n\t" "}"), "l,r,r,r,r", diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/kernel_fc12.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/kernel_fc12.py index 8682e566b020..ef06de0d1f03 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/kernel_fc12.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/kernel_fc12.py @@ -1,5 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause """Fused fc1+fc2 swap-AB SwiGLU NVFP4 kernel for SM100.""" from typing import Literal, Optional, Tuple, Type @@ -33,6 +35,7 @@ from .megamoe_constants import (Nvfp4BlockSize, SupportedMmaTileM, SupportedMmaTileN) from .moe_utils import spin_wait +from .token_comm import CombineFormat # token_comm_args is an opaque subclass-owned bundle. The base only forwards it # to hook methods; ``None`` keeps the lean fc1+fc2 path free of token-comm IR. @@ -55,31 +58,32 @@ class Sm100SwapABSwigluFp4Fc12Kernel: _SmemMiscBudget = 1024 def __init__( - self, - # Geometry. - mma_tiler_mnk: Tuple[int, int, int], - cluster_shape_mnk: Tuple[int, int, int], - use_2cta_instrs: bool, - # Fused fc1+fc2 scheduler knobs. - group_hint: int, - token_padding_block: int, - sf_padding_block: int, - load_balance_mode: Literal["static", "atomic_counter"] = "static", - # Optional scheduler/codegen knobs. - static_expert_shape: Optional[Tuple[int, int, int]] = None, - force_static_sched: bool = True, - clc_bundle_size: Optional[int] = None, - num_sched_stages: Optional[int] = None, - acc_dtype: Type[cutlass.Numeric] = cutlass.Float32, - sf_vec_size: int = 16, - scenario: Literal["2Dx3D"] = "2Dx3D", - *, - fc2_output_dtype: Type[cutlass.Numeric], - non_ubulk_fc2_store: bool = True, - in_kernel_fc2_reduce: bool = False, - token_back_by_dispatch: bool = False, - apply_topk_in_fc1: bool = True, - gate_up_clamp: Optional[float] = None, + self, + # Geometry. + mma_tiler_mnk: Tuple[int, int, int], + cluster_shape_mnk: Tuple[int, int, int], + use_2cta_instrs: bool, + # Fused fc1+fc2 scheduler knobs. + group_hint: int, + token_padding_block: int, + sf_padding_block: int, + load_balance_mode: Literal["static", "atomic_counter"] = "static", + # Optional scheduler/codegen knobs. + static_expert_shape: Optional[Tuple[int, int, int]] = None, + force_static_sched: bool = True, + clc_bundle_size: Optional[int] = None, + num_sched_stages: Optional[int] = None, + acc_dtype: Type[cutlass.Numeric] = cutlass.Float32, + sf_vec_size: int = 16, + scenario: Literal["2Dx3D"] = "2Dx3D", + *, + fc2_output_dtype: Type[cutlass.Numeric], + non_ubulk_fc2_store: bool = True, + in_kernel_fc2_reduce: bool = False, + token_back_by_dispatch: bool = False, + apply_topk_in_fc1: bool = True, + gate_up_clamp: Optional[float] = None, + epi_flag_batch: Optional[Tuple[int, int]] = (1, 1), ) -> None: if not force_static_sched: raise NotImplementedError( @@ -123,6 +127,7 @@ def __init__( self.token_back_by_dispatch = token_back_by_dispatch self.apply_topk_in_fc1 = apply_topk_in_fc1 self.gate_up_clamp = gate_up_clamp + self.epi_flag_batch = epi_flag_batch self._validate_mma_tiler_and_cluster_shape() self.mma_tiler = mma_tiler_mnk @@ -142,6 +147,8 @@ def __init__( self.sched_warp_id = 7 # Installed by token-comm subclasses. self.dispatch_warp_id: Optional[Tuple[int, int, int, int]] = None + self.token_back_warp_id: Optional[Tuple[int, int, int, int]] = None + self.token_back_standalone: bool = False self.threads_per_cta = 32 * len(( self.mma_warp_id, self.tma_a_warp_id, @@ -161,11 +168,39 @@ def __init__( # register allocation because setmaxnreg emission is gated by # ``self.enable_token_comm`` inside the device kernel. self.epi_reg_cnt = 256 - self.task_reg_cnt = 96 + self.task_reg_cnt = 72 self.smem_capacity = utils.get_smem_capacity_in_bytes(self.arch) self.num_tmem_alloc_cols = cute.arch.get_max_tmem_alloc_cols(self.arch) + def name(self) -> str: + """Canonical encoding of the codegen-affecting constexpr -- the compiled- + kernel cache key. ``local_rank`` excluded (deployment detail); only + ``force_static_sched`` / ``clc_bundle_size`` / ``num_sched_stages`` / + ``scenario`` are dropped -- every other constexpr is kept.""" + m, n, k = self.mma_tiler_mnk + cm, cn = self.cluster_shape_mn + exp = "x".join(map( + str, + self.static_expert_shape)) if self.static_expert_shape else "dyn" + epiflag = "x".join(map( + str, self.epi_flag_batch)) if self.epi_flag_batch else "none" + cta = "2_cta" if self.use_2cta_instrs else "1_cta" + fc2store = "fc2store_stg" if self.non_ubulk_fc2_store else "fc2store_ublk" + inkred = "inkernel_redg" if self.in_kernel_fc2_reduce else "no_inkernel_redg" + apply_topk = "apply_topk_fc1_pre_quant" if self.apply_topk_in_fc1 else "apply_topk_after_fc2" + # token-back is a token-communication concept -- it lives in the MegaMoE + # subclass name(), not the lean base. + return ( + "moe_fc12_fuse_nvfp4" + f"_mmatiler_{m}x{n}x{k}_cluster_{cm}x{cn}_{cta}_sched_{self.load_balance_mode}" + f"_expert_shape_{exp}_grouphint_{self.group_hint}" + f"_padding_{self.token_padding_block}x{self.sf_padding_block}" + f"_{fc2store}_{inkred}_{apply_topk}" + f"_fc2out{self.fc2_output_dtype.__name__}_sfvec{self.sf_vec_size}" + f"_acc{self.acc_dtype.__name__}_clamp{self.gate_up_clamp}_epiflag{epiflag}" + ) + def _validate_mma_tiler_and_cluster_shape(self) -> None: """Validate user-provided geometry against v1 fused-fc12 constraints. @@ -201,7 +236,7 @@ def _validate_mma_tiler_and_cluster_shape(self) -> None: f"cluster_shape M ({cm}) must be even when use_2cta_instrs=True" ) - def is_pow2(x): + def is_pow2(x: int) -> bool: return x > 0 and (x & (x - 1)) == 0 if cm * cn > 16 or not is_pow2(cm) or not is_pow2( @@ -312,16 +347,23 @@ def _setup_attributes(self) -> None: # dtype that lives in sC). fc2 output dtype is hard-coded as # ``BFloat16`` inside the epilogue's ``Fc2UnpackPermuteStg`` and # does not flow through this knob. + # combine_format is comm-side state: the MegaMoE subclass injects + # ``self.combine_format`` before super().__init__, so it is already on + # ``self`` here; the standalone base defaults to the bf16 (no-quant) + # combine. Only hooks / token_comm_args otherwise cross into this base. + if not hasattr(self, "combine_format"): + self.combine_format = CombineFormat.parse("bf16") self.epilogue = SwapABSwigluFp4Epilogue( mma_tiler_mnk=self.mma_tiler, cluster_shape_mn=self.cluster_shape_mn, use_2cta_instrs=self.use_2cta_instrs, sf_vec_size=self.sf_vec_size, fc1_output_dtype=self.fc1_output_dtype, - fc2_output_dtype=self.fc2_output_dtype, + combine_format=self.combine_format, non_ubulk_fc2_store=self.non_ubulk_fc2_store, in_kernel_fc2_reduce=self.in_kernel_fc2_reduce, token_back_by_dispatch=self.token_back_by_dispatch, + epi_flag_batch=self.epi_flag_batch, acc_dtype=self.acc_dtype, allow_overlap_acc=True, static_expert_shape=self.static_expert_shape, @@ -351,6 +393,13 @@ def _setup_attributes(self) -> None: self.num_sched_stages, self._smem_misc_budget_bytes(), ) + # print( + # f"[fc12 stages] num_ab_stage={self.num_ab_stage} " + # f"num_acc_stage={self.num_acc_stage} " + # f"misc_budget={self._smem_misc_budget_bytes()} " + # f"c_bytes_total={c_bytes_total} smem_cap={self.smem_capacity} " + # f"token_back_standalone={self.token_back_standalone}" + # ) self.a_smem_layout_staged = sm100_utils.make_smem_layout_a( tiled_mma, @@ -581,7 +630,7 @@ def token_comm_hook_fc1_tma_b_predispatch_spin( """Emitted on the TMA-B warp at the head of each fc1-phase task tile, before its K-loop. Default: no-op. MegaMoE: blocking spin on the dispatch->fc1 release counter at ``cumulative_token_block_count + - tile_n_idx`` until it reaches ``work_tile_info.valid_tokens_in_tile``, + tile_n_idx`` until it reaches ``work_tile_info.valid_tokens_in_cta_tile``, unless ``work_tile_info.peek_ready`` already saturated it. Skipping this in the lean path is correct because in the lean path the per-tile input is already resident in GMEM at launch time.""" @@ -598,6 +647,18 @@ def token_comm_hook_dispatch_warp_body( ): """Subclass dispatch warp body; no-op in the lean kernel.""" + @cute.jit + def token_comm_hook_token_back_warp_body( + self, + token_comm_args, + token_comm_storage, + *, + warp_idx, + lane_idx, + tidx, + ): + """Subclass standalone token-back warp body; no-op in the lean kernel.""" + @cute.jit def token_comm_hook_kernel_tail( self, @@ -735,13 +796,20 @@ def __call__( c1 = cutlass.Int32(1) cutlass.Int32(0) + def round_up(value, multiple): + return ((value + multiple - 1) // multiple) * multiple + + def is_constexpr_int(*values) -> bool: + return all(isinstance(value, int) for value in values) + # A_gemm (fc1 weights): (experts, hidden, intermediate_gateup) # -> (M=intermediate_gateup, K=hidden, L=experts). experts, hidden_b, intermediate_gateup = fc1_weight.shape fc1_weight_gemm = cute.make_tensor( fc1_weight.iterator, cute.make_layout( - (intermediate_gateup, hidden_b, experts), + (cutlass.Int32(intermediate_gateup), cutlass.Int32(hidden_b), + cutlass.Int32(experts)), stride=(fc1_weight.stride[2], fc1_weight.stride[1], fc1_weight.stride[0]), ), @@ -752,7 +820,7 @@ def __call__( activation_gemm = cute.make_tensor( activation.iterator, cute.make_layout( - (tokens_sum, hidden, 1), + (tokens_sum, cutlass.Int32(hidden), c1), stride=(activation.stride[0], activation.stride[1], 0), ), ) @@ -762,7 +830,7 @@ def __call__( fc1_output_gemm = cute.make_tensor( fc1_output.iterator, cute.make_layout( - (tokens_sum, intermediate_downproj, 1), + (tokens_sum, cutlass.Int32(intermediate_downproj), c1), stride=(fc1_output.stride[0], fc1_output.stride[1], 0), ), ) @@ -775,16 +843,30 @@ def __call__( activation_sf_gemm = cute.make_tensor( activation_sf.iterator, blockscaled_utils.tile_atom_to_shape_SF( - (tokens_sum_padded, hidden_padded, 1), self.sf_vec_size), + (tokens_sum_padded, cutlass.Int32(hidden_padded), c1), + self.sf_vec_size), ) - intermediate_gateup_padded_mul_hidden_padded = fc1_weight_sf.shape[1] - intermediate_gateup_padded = ( - intermediate_gateup_padded_mul_hidden_padded * - self.sf_vec_size) // hidden_padded + # M-side SF atom is 128 rows (same as the fc2 path below); the + # K-side ``sf_vec_size * 4`` granularity would understate the padded + # row count for non-128-aligned intermediate sizes. + intermediate_gateup_padded = round_up(intermediate_gateup, 128) + expected_fc1_weight_sf_cols = (intermediate_gateup_padded * + hidden_padded // self.sf_vec_size) + if cutlass.const_expr( + is_constexpr_int(fc1_weight_sf.shape[1], + expected_fc1_weight_sf_cols)): + if cutlass.const_expr( + fc1_weight_sf.shape[1] != expected_fc1_weight_sf_cols): + raise ValueError( + f"fc1_weight_sf.shape[1] ({fc1_weight_sf.shape[1]}) does not " + f"match intermediate_padded({intermediate_gateup_padded}) * " + f"hidden_padded({hidden_padded}) / sf_vec_size({self.sf_vec_size}) " + f"= {expected_fc1_weight_sf_cols}.") fc1_weight_sf_gemm = cute.make_tensor( fc1_weight_sf.iterator, blockscaled_utils.tile_atom_to_shape_SF( - (intermediate_gateup_padded, hidden_padded, experts), + (cutlass.Int32(intermediate_gateup_padded), + cutlass.Int32(hidden_padded), cutlass.Int32(experts)), self.sf_vec_size, ), ) @@ -799,7 +881,9 @@ def __call__( fc2_weight_gemm = cute.make_tensor( fc2_weight.iterator, cute.make_layout( - (hidden_b2, intermediate_downproj_b2, experts2), + (cutlass.Int32(hidden_b2), + cutlass.Int32(intermediate_downproj_b2), + cutlass.Int32(experts2)), stride=(fc2_weight.stride[2], fc2_weight.stride[1], fc2_weight.stride[0]), ), @@ -825,18 +909,36 @@ def __call__( fc1_output_sf_gemm_for_fc2_load = cute.make_tensor( fc1_output_sf.iterator, blockscaled_utils.tile_atom_to_shape_SF( - (tokens_sum_padded_sf, intermediate_downproj_padded, 1), + (tokens_sum_padded_sf, + cutlass.Int32(intermediate_downproj_padded), c1), self.sf_vec_size, ), ) - hidden_padded_fc2_mul_intermediate_downproj_padded = fc2_weight_sf.shape[ - 1] - hidden_padded_fc2 = (hidden_padded_fc2_mul_intermediate_downproj_padded - * self.sf_vec_size) // intermediate_downproj_padded + hidden_padded_fc2 = round_up(hidden_b2, 128) + intermediate_downproj_padded = round_up( + intermediate_downproj_b2, + self.sf_vec_size * 4, + ) + expected_fc2_weight_sf_cols = (hidden_padded_fc2 * + intermediate_downproj_padded // + self.sf_vec_size) + if cutlass.const_expr( + is_constexpr_int(fc2_weight_sf.shape[1], + expected_fc2_weight_sf_cols)): + if cutlass.const_expr( + fc2_weight_sf.shape[1] != expected_fc2_weight_sf_cols): + raise ValueError( + f"fc2_weight_sf.shape[1] ({fc2_weight_sf.shape[1]}) does not " + f"match hidden_padded({hidden_padded_fc2}) * " + f"intermediate_padded({intermediate_downproj_padded}) / " + f"sf_vec_size({self.sf_vec_size}) = {expected_fc2_weight_sf_cols}." + ) fc2_weight_sf_gemm = cute.make_tensor( fc2_weight_sf.iterator, blockscaled_utils.tile_atom_to_shape_SF( - (hidden_padded_fc2, intermediate_downproj_padded, experts2), + (cutlass.Int32(hidden_padded_fc2), + cutlass.Int32(intermediate_downproj_padded), + cutlass.Int32(experts2)), self.sf_vec_size, ), ) @@ -907,7 +1009,7 @@ def __call__( self.mma_tiler, tiled_mma, self.cluster_layout_vmnk.shape, - internal_type=cutlass.Uint64, + internal_type=cutlass.Uint16, ) # TMA load SFB1 (= activation_sf, fc1 activation SFs) @@ -922,7 +1024,7 @@ def __call__( self.mma_tiler_sfb, tiled_mma_sfb, self.cluster_layout_sfb_vmnk.shape, - internal_type=cutlass.Uint64, + internal_type=cutlass.Uint16, ) # TMA store for fc1 NVFP4 output (via SMEM-staged bulk store). @@ -980,7 +1082,7 @@ def __call__( self.mma_tiler, tiled_mma, self.cluster_layout_vmnk.shape, - internal_type=cutlass.Uint64, + internal_type=cutlass.Uint16, ) tma_atom_fc1_output_sf_as_fc2_input, tma_tensor_fc1_output_sf_as_fc2_input = cute.nvgpu.make_tiled_tma_atom_B( sfb_op, @@ -989,7 +1091,7 @@ def __call__( self.mma_tiler_sfb, tiled_mma_sfb, self.cluster_layout_sfb_vmnk.shape, - internal_type=cutlass.Uint64, + internal_type=cutlass.Uint16, ) # ── Scheduler params + grid + launch ── @@ -1731,7 +1833,7 @@ class SharedStorage: if not is_leader_cta: load_shift = dynamic_mainloop.compute_non_leader_cta_load_shift( valid_tokens_in_tile=work_tile_info. - valid_tokens_in_tile, + valid_tokens_in_cta_tile, mma_tiler_n=self.mma_tiler[1], ) real_b = cute.domain_offset((load_shift, 0, 0), @@ -1834,7 +1936,7 @@ class SharedStorage: spin_wait( counter_ptr, lambda v: v >= fc2_spin_threshold, - fail_sleep_cycles=20, + fail_sleep_cycles=500, ) iket.range_pop() @@ -1857,7 +1959,7 @@ class SharedStorage: if not is_leader_cta: load_shift = dynamic_mainloop.compute_non_leader_cta_load_shift( valid_tokens_in_tile=work_tile_info. - valid_tokens_in_tile, + valid_tokens_in_cta_tile, mma_tiler_n=self.mma_tiler[1], ) real_b = cute.domain_offset((load_shift, 0, 0), @@ -2023,8 +2125,10 @@ class SharedStorage: ab_consumer.reset() peek_ab_full_status = cutlass.Boolean(1) if k_tile_cnt > 0: + iket.range_push("mma_acquire") peek_ab_full_status = ab_consumer.try_wait() acc_pipeline.producer_acquire(acc_producer_state) + iket.range_pop() # Apply TMEM pointer offset hack when mma_tiler_n == 64. tCtSFB_mma = tCtSFB @@ -2071,7 +2175,7 @@ class SharedStorage: sfb_tensor=tCtSFB_mma, k_tile_idx=k_tile, valid_tokens_in_tile=work_tile_info. - valid_tokens_in_tile, + valid_tokens_in_cta_tile, mma_tiler_mnk=self.mma_tiler_mnk, ) handle.release() @@ -2151,13 +2255,31 @@ class SharedStorage: cute.arch.warpgroup_reg_dealloc(self.task_reg_cnt) lane_idx_for_dispatch = cute.arch.lane_idx() - self.token_comm_hook_dispatch_warp_body( - token_comm_args, - token_comm_storage, - warp_idx=warp_idx, - lane_idx=lane_idx_for_dispatch, - tidx=tidx, - ) + if cutlass.const_expr(self.token_back_standalone): + if warp_idx < self.token_back_warp_id[0]: + self.token_comm_hook_dispatch_warp_body( + token_comm_args, + token_comm_storage, + warp_idx=warp_idx, + lane_idx=lane_idx_for_dispatch, + tidx=tidx, + ) + else: + self.token_comm_hook_token_back_warp_body( + token_comm_args, + token_comm_storage, + warp_idx=warp_idx, + lane_idx=lane_idx_for_dispatch, + tidx=tidx, + ) + else: + self.token_comm_hook_dispatch_warp_body( + token_comm_args, + token_comm_storage, + warp_idx=warp_idx, + lane_idx=lane_idx_for_dispatch, + tidx=tidx, + ) # ════════════════════════════════════════════════════════════════════ # Kernel tail hook (MegaMoE-only path; lean base = no-op) diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/megamoe_constants.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/megamoe_constants.py index 2cdaa13a57bc..1a43b5682a9d 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/megamoe_constants.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/megamoe_constants.py @@ -1,6 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Shared constants for the fused fc1+fc2 MegaMoE path.""" +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +"""Shared constants for the fused fc1+fc2 MegaMoE path. + +``Fp32Max`` is the only CuteDSL-typed constant here; it is resolved +lazily via module ``__getattr__`` (PEP 562) so importing this module -- +and therefore the package ``__init__`` and the host-side tactic +enumeration in ``cute_dsl_megamoe_custom_op.py``, which only need the +plain-Python constants -- does not require a cutlass-dsl install. The +kernel modules that consume ``Fp32Max`` (``epilogue_refactor.py``) +import cutlass themselves, so the lazy resolution always succeeds +wherever the constant is actually used. +""" Nvfp4BlockSize = 16 SfPaddingBlock = 128 @@ -9,5 +21,20 @@ Nvfp4E2M1Max = 6.0 Fp8E4M3FNMax = 448.0 +Nvfp4E2M1RcpLimit = 1.0 / Nvfp4E2M1Max +Fp8E4M3RcpLimit = 1.0 / Fp8E4M3FNMax + SupportedMmaTileM = (128, 256) SupportedMmaTileN = (64, 128, 256) + + +def __getattr__(name: str): + if name == "Fp32Max": + from cutlass.cutlass_dsl import Float32 + + value = Float32(3.40282346638528859812e38) + # Cache so later lookups (and ``from ... import Fp32Max``) bind the + # SAME object instead of re-wrapping per access. + globals()[name] = value + return value + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/megamoe_kernel.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/megamoe_kernel.py index 112c441c11ed..0ea3f48ae9ac 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/megamoe_kernel.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/megamoe_kernel.py @@ -1,5 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause """MegaMoE fused dispatch + fc1 + fc2 + combine kernel. The base class owns the local fc1/fc2 GEMM pipeline. This subclass owns the @@ -38,7 +40,7 @@ # quoted explicitly. import dataclasses -from typing import Any, Dict, List, Optional, Tuple, Type +from typing import Any, Dict, List, Literal, Optional, Tuple, Type import cutlass import cutlass.cute as cute @@ -46,8 +48,10 @@ from cutlass.cutlass_dsl import Int64 from .kernel_fc12 import Sm100SwapABSwigluFp4Fc12Kernel +from .token_comm import CombineFormat from .token_comm import TokenCommArgs as ExtractedTokenCommArgs -from .token_comm import TokenInPullTokenBackPush +from .token_comm import TokenInPullTokenBackPush, TokenSrcMetadata +from .topk_reduce import TopkReduce # ============================================================================= # Module-level constants. @@ -60,9 +64,9 @@ # Dispatch warp count. _DispatchWarpCount = 4 -# Per-pool-slot provenance record consumed by combine STG redirect (S3). -# Three packed Uint32 fields = 12 bytes: ``{src_rank, src_token, src_topk}``. -_TokenMetadataBytes = 12 +# Per-pool-slot provenance record consumed by combine STG redirect (S3) and +# token-back; one i64 = {src_rank, src_token, src_topk} (see TokenSrcMetadata). +_TokenMetadataBytes = TokenSrcMetadata.nbytes # NVLink signal slots used by the DeepGEMM-style phase/sign barrier. # A separate local counter selects phase/sign; the signal slots are not reset @@ -168,17 +172,42 @@ def __init__( # MegaMoE-specific independent constants. *, world_size: int, - local_rank: int, num_topk: int, max_tokens_per_rank: int, hidden: int, fc2_output_dtype: Type[cutlass.Numeric], + combine_format: CombineFormat = CombineFormat.parse("bf16"), non_ubulk_fc2_store: bool = True, in_kernel_fc2_reduce: bool = False, - token_back_by_dispatch: bool = False, + token_back_mode: Literal["epi_warps", "standalone_warps", + "reuse_dispatch_warps"] = "epi_warps", apply_topk_in_fc1: bool = True, gate_up_clamp: Optional[float] = None, + epi_flag_batch: Optional[Tuple[int, int]] = (1, 1), + flag_batch: int = 1, ) -> None: + # The combine wire format drives the fc2 epilogue encoder, token_comm + # push, and the combine_quant/combine_sf workspace sizing. The dataflow + # (workspace carve, views, arg threading, epilogue encode, topk_reduce + # receiver) is wired for every format, quantized included -- see the + # closed-loop note above the fused launch in __call__. The guards below + # only reject combinations the kernel cannot express + # (in_kernel_fc2_reduce is bf16-only; FP4 needs non_ubulk_fc2_store). + self.combine_format = combine_format + if in_kernel_fc2_reduce and combine_format.is_quantized: + raise ValueError( + f"in_kernel_fc2_reduce requires a non-quantized (bf16) combine " + f"format; got {combine_format}.") + if in_kernel_fc2_reduce and not apply_topk_in_fc1: + raise ValueError( + "in_kernel_fc2_reduce requires apply_topk_in_fc1=True; " + "the REDG path can only atomic-add terms whose topk score " + "was already absorbed before fc2.") + if (combine_format.act_dtype is cutlass.Float4E2M1FN + and not non_ubulk_fc2_store): + raise ValueError( + f"{combine_format} combine requires non_ubulk_fc2_store=True " + "(the UBLK fc2 store path cannot scalar-dereference FP4).") if static_expert_shape is None: raise NotImplementedError( "Sm100MegaMoEKernel currently requires " @@ -191,6 +220,20 @@ def __init__( f"hidden ({hidden}) must equal " f"static_expert_shape[2] ({static_expert_shape[2]}).") + # token_back_mode selects where the cross-rank fc2 push-back runs: + # epi_warps -> epilogue warps STG directly to the peer + # standalone_warps -> dedicated warp group 12-15, concurrent + # with dispatch_pull + # reuse_dispatch_warps -> dispatch warps 8-11 push after dispatch_pull + # The two non-epi modes both stage fc2 to a local workspace first, i.e. + # token_back_by_dispatch=True; epi_warps keeps the epilogue STG redirect. + if token_back_mode not in ("epi_warps", "standalone_warps", + "reuse_dispatch_warps"): + raise ValueError( + f"token_back_mode must be 'epi_warps', 'standalone_warps', " + f"or 'reuse_dispatch_warps'; got {token_back_mode!r}.") + token_back_by_dispatch = token_back_mode != "epi_warps" # nosec B105 + super().__init__( mma_tiler_mnk=mma_tiler_mnk, cluster_shape_mnk=cluster_shape_mnk, @@ -212,23 +255,33 @@ def __init__( token_back_by_dispatch=token_back_by_dispatch, apply_topk_in_fc1=apply_topk_in_fc1, gate_up_clamp=gate_up_clamp, + epi_flag_batch=epi_flag_batch, ) self.enable_token_comm = True self.dispatch_warp_id = (8, 9, 10, 11) + # Standalone token-back: a dedicated 4-warp group (12-15) doing + # token_back_by_push concurrently with dispatch_pull, selected by the + # user-facing token_back_mode knob ("standalone_warps"). + self.token_back_mode = token_back_mode + self.token_back_standalone = token_back_mode == "standalone_warps" # nosec B105 + self.token_back_warp_id = (12, 13, 14, + 15) if self.token_back_standalone else None + num_token_back_warps = (len(self.token_back_warp_id) + if self.token_back_standalone else 0) self.threads_per_cta = 32 * ( len(self.epilogue_warp_id) + 1 # mma + 1 # tma_a + 1 # tma_b + 1 # sched - + len(self.dispatch_warp_id)) + + len(self.dispatch_warp_id) + num_token_back_warps) # Independent MegaMoE-specific constants. self.world_size = world_size - self.local_rank = local_rank self.num_topk = num_topk self.max_tokens_per_rank = max_tokens_per_rank self.hidden = hidden + self.flag_batch = flag_batch # stored so name() can encode it # static_expert_shape = (num_experts_per_rank, intermediate_gateup, hidden). self.num_experts_per_rank = static_expert_shape[0] @@ -245,6 +298,15 @@ def __init__( # Cross-rank totals: per-rank count * world_size. self.num_total_experts = world_size * self.num_experts_per_rank + # Per-(token, topk) SF block padded to a 16 B multiple so the token-back + # cp.async.bulk push moves one aligned block per shot; only the first + # hidden//scale_block entries of each block are valid (the rest is the + # alignment gap). ``None`` for the bf16 baseline (no SF plane). + self.sf_block_pad = (_round_up( + self.hidden // self.combine_format.scale_block, + 16 // (int(self.combine_format.scale_dtype.width) // 8), + ) if self.combine_format.is_quantized else None) + # Per-task-tile release-counter granularity used by dispatch_pull. self.cluster_tile_tokens = mma_tiler_mnk[1] * cluster_shape_mnk[1] @@ -275,18 +337,32 @@ def __init__( (self.hidden + cluster_fc2_tile_hidden - 1) // cluster_fc2_tile_hidden) * self.cluster_shape_mn[0] + # Token-back warps run when they push the DATA plane (dispatch modes) OR + # the SF plane (any quantized combine, including the epi_warps data path + # where the epilogue STGs the data straight to the peer but SF still + # needs the staged token-contiguous push). + self.token_back_enabled = (self.token_back_by_dispatch + or self.combine_format.is_quantized) + # Homomorphic to the fc1+fc2 scheduler: atomic_counter token-back only + # nets a win with enough tokens, the same condition that selects the + # atomic_counter fc1+fc2 scheduler. Static when token-back is off. + self.token_back_schedule_mode = (self.load_balance_mode if + self.token_back_enabled else "static") + self.token_comm = TokenInPullTokenBackPush( world_size=self.world_size, - local_rank=self.local_rank, num_topk=self.num_topk, num_experts_per_rank=self.num_experts_per_rank, num_total_experts=self.num_total_experts, hidden=self.hidden, fc1_token_dtype=cutlass.Float4E2M1FN, - fc2_output_dtype=(self.fc2_output_dtype - if self.token_back_by_dispatch else None), + combine_format=self.combine_format, + token_back_by_dispatch=self.token_back_by_dispatch, fc2_publishes_per_token_cluster_tile= fc2_publishes_per_token_cluster_tile, + token_back_reduce_topk=(self.token_back_by_dispatch + and self.in_kernel_fc2_reduce), + token_back_standalone=self.token_back_standalone, sf_uint32_per_token=self.sf_uint32_per_token, token_padding_block=self.token_padding_block, sf_padding_block=self.sf_padding_block, @@ -294,6 +370,9 @@ def __init__( cluster_shape_mn=self.cluster_shape_mn, dispatch_warp_start=self.dispatch_warp_id[0], num_other_warps=num_other_warps, + flag_batch=flag_batch, + is_swap_ab=True, + token_back_schedule_mode=self.token_back_schedule_mode, ) # Region layout (same call drives both get_workspace_sizes() and @@ -313,6 +392,22 @@ def __init__( for r in self._shared_region_specs } + # Counter-prefix byte extents: the accumulating counters are front-placed + # (see _build_*_region_specs), so the leading bytes up to the first data + # region cover exactly the per-launch zero set. ``tail_reset_counters`` + # bulk-zeros these each launch; only the first launch needs a caller-zeroed + # workspace. 128B-aligned offsets => multiples of 4 (Int32 zeroing exact). + local_leading = self._local_offsets[ + "l1_token_buffer"] # first data region + shared_leading = self._shared_offsets[ + "src_token_topk_idx"] # first data region + self.require_zero_workspace_leading_bytes: Tuple[int, int] = ( + local_leading, + shared_leading, + ) + self.local_zero_i32_count = local_leading // 4 + self.shared_zero_i32_count = shared_leading // 4 + # ========================================================================= # SMEM budget hook (base override) # ========================================================================= @@ -322,9 +417,13 @@ def _dispatch_smem_bytes(self) -> int: pull_mbar_bytes = _DispatchWarpCount * 8 expert_count_bytes = self.num_total_experts * 4 pull_buffer_bytes = _DispatchWarpCount * self.hidden_bytes - return (_round_up(pull_mbar_bytes, 16) + - _round_up(expert_count_bytes, 16) + - _round_up(pull_buffer_bytes, 128)) + total = (_round_up(pull_mbar_bytes, 16) + + _round_up(expert_count_bytes, 16) + + _round_up(pull_buffer_bytes, 128)) + if self.token_back_standalone: + total += (_round_up(_DispatchWarpCount * 8, 16) + _round_up( + _DispatchWarpCount * self.token_comm.tb_chunk_bytes, 128)) + return total def _smem_misc_budget_bytes(self) -> int: """Base misc reservation plus dispatch-warp SMEM.""" @@ -409,65 +508,108 @@ def _build_local_region_specs(self) -> List[_RegionSpec]: (pool_token_capacity + mma_tiler_n - 1) // mma_tiler_n + num_experts_per_rank) + # === Accumulating-counter prefix =========================================== + # Front-placed so the per-launch reset is a single bulk zero of + # ``[0:local_leading]`` (the bytes up to the first data region). These hold + # spin thresholds / write cursors / phase-flip counters that the kernel + # accumulates across the launch and that MUST start at 0 each launch; the + # kernel tail (``tail_reset_counters``) bulk-zeros this prefix, so only the + # FIRST launch relies on a caller-zeroed workspace. specs: List[_RegionSpec] = [ - # L1 input pool (dispatch_pull writes -> fc1 reads). Stored - # as Uint8 bytes; the NVFP4 view at the same offset is - # built inside ``__call__``. _RegionSpec( - "l1_token_buffer", - cutlass.Uint8, - (pool_token_capacity, hidden_bytes), - 128, + "l1_arrival_count", + cutlass.Int32, + (pool_task_tile_capacity, ), + 16, ), - # Stored as Int32 (dispatch_pull's 32 b read/write); the FP8 - # view for activation_sf is built at the same offset. - # 1D Int32 atom-flat buffer. Total Int32 count = pool_sf_capacity - # (M-axis token positions) * sf_uint32_per_token (K-atom count), - # laid out atom-by-atom per cute SFA layout. dispatch writes - # individual Int32 slots via the linear offset returned by - # ``src/sf_swizzle.py:sf_atom_int32_offset``; the mma side - # re-views this same byte buffer through ``tile_atom_to_shape_SF`` - # which reads back the atom-swizzled bytes. _RegionSpec( - "l1_sf_buffer", - cutlass.Int32, - (pool_sf_capacity * sf_uint32_per_token, ), + "expert_send_count", + cutlass.Int64, + (num_total_experts, ), 16, ), _RegionSpec( - "l1_topk_weights_buffer", - cutlass.Float32, - (pool_token_capacity, ), + "grid_sync_counter", + cutlass.Int32, + (_GridSyncSlotCount, ), 16, ), _RegionSpec( - "l1_arrival_count", + "fc1_done_counter", cutlass.Int32, - (pool_task_tile_capacity, ), + (fc1_done_slots, ), 16, ), + ] + if self.token_back_enabled: + # Per-expert fc2 completion gate consumed by the token-back push + # (DATA and/or SF). Published by the fc2 epilogue for every enabled + # token-back path, including the epi_warps SF-only push. + specs.append( + _RegionSpec( + "fc2_done_counter", + cutlass.Int32, + (num_experts_per_rank, ), + 16, + )) + if self.token_back_schedule_mode == "atomic_counter": # nosec B105 + specs.append( + _RegionSpec( + "token_back_schedule_counter", + cutlass.Int32, + (1, ), + 16, + )) + if self.load_balance_mode == "atomic_counter": + specs.append( + _RegionSpec( + "load_balance_counter", + cutlass.Int32, + (1, ), + 16, + )) + + # === Data buffers (overwritten each launch; NOT zeroed) ==================== + # ``l1_token_buffer`` MUST be the first data region: ``__init__`` derives + # ``local_leading`` from its offset (= end of the counter prefix). + specs += [ + # L1 input pool (dispatch_pull writes -> fc1 reads), Uint8 bytes; the + # NVFP4 view at the same offset is built inside ``__call__``. _RegionSpec( - "token_src_metadata", + "l1_token_buffer", cutlass.Uint8, - (pool_token_capacity, _TokenMetadataBytes), - 16, + (pool_token_capacity, hidden_bytes), + 128, ), + # Persisted across launches (deliberately OUT of the zero prefix): the + # sense-reversing nvlink barrier rides this phase counter across launch + # boundaries (non-ncu back-to-back), and ncu kernel replay restores it + # via its local-memory snapshot. Only the FIRST launch relies on the + # caller-zeroed workspace. _RegionSpec( - "expert_send_count", - cutlass.Int64, - (num_total_experts, ), + "nvlink_barrier_counter", + cutlass.Int32, + (1, ), 16, ), + # Int32 atom-flat SF buffer (dispatch_pull 32b read/write; FP8 view at + # the same offset). Count = pool_sf_capacity * sf_uint32_per_token. _RegionSpec( - "grid_sync_counter", + "l1_sf_buffer", cutlass.Int32, - (_GridSyncSlotCount, ), + (pool_sf_capacity * sf_uint32_per_token, ), 16, ), _RegionSpec( - "nvlink_barrier_counter", - cutlass.Int32, - (1, ), + "l1_topk_weights_buffer", + cutlass.Float32, + (pool_token_capacity, ), + 16, + ), + _RegionSpec( + "token_src_metadata", + cutlass.Uint8, + (pool_token_capacity, _TokenMetadataBytes), 16, ), _RegionSpec( @@ -482,36 +624,33 @@ def _build_local_region_specs(self) -> List[_RegionSpec]: (sf_total_rows_upper, sf_block_cols), 128, ), - _RegionSpec( - "fc1_done_counter", - cutlass.Int32, - (fc1_done_slots, ), - 16, - ), ] if self.token_back_by_dispatch: + # Local fc2 DATA staging (token_back_by_dispatch modes only); the + # wire-format dtype sizes the plane. specs.append( _RegionSpec( "fc2_output_workspace", - self.fc2_output_dtype, + self.combine_format.act_dtype, (pool_token_capacity, 1, self.hidden), 128, )) + # The per-block SF plane is ALWAYS staged locally (then pushed + # token-contiguously by the dispatch / standalone warps), independent of + # whether the DATA path goes local or straight to a peer: writing SF + # per-token to a peer would scatter one warp's 32 lanes across up to 32 + # ranks and explode the NVLink request count. So it is allocated for + # every quantized format, not only the dispatch data path. + if self.combine_format.is_quantized: + # Flat padded capacity (pool_token * sf_block_pad scale entries); the + # (pool_token, 1, hidden//scale_block) logical shape + 16 B-aligned + # per-block stride is assembled where the view is built in __call__. specs.append( _RegionSpec( - "fc2_done_counter", - cutlass.Int32, - (num_experts_per_rank, ), - 16, - )) - - if self.load_balance_mode == "atomic_counter": - specs.append( - _RegionSpec( - "load_balance_counter", - cutlass.Int32, - (1, ), - 16, + "fc2_output_sf", + self.combine_format.scale_dtype, + (pool_token_capacity * self.sf_block_pad, ), + 128, )) return specs @@ -536,13 +675,13 @@ def _build_shared_region_specs(self) -> List[_RegionSpec]: # edge any peer might publish for this rank's local experts. max_slot = max_tokens_per_rank * num_topk - return [ - _RegionSpec( - "src_token_topk_idx", - cutlass.Int32, - (num_experts_per_rank, world_size, max_slot), - 16, - ), + # Accumulating counters first (zero-prefix -> tail bulk-zeros + # ``[0:shared_leading]``); then the data/signal regions that must persist + # across launches (``src_token_topk_idx`` overwritten each launch; + # ``nvlink_barrier_signal`` is phase-flip and must NOT be zeroed). + # ``src_token_topk_idx`` MUST be the first non-counter region: ``__init__`` + # derives ``shared_leading`` from its offset (= end of the counter prefix). + specs: List[_RegionSpec] = [ _RegionSpec( "expert_recv_count", cutlass.Int64, @@ -555,6 +694,12 @@ def _build_shared_region_specs(self) -> List[_RegionSpec]: (num_experts_per_rank, ), 16, ), + _RegionSpec( + "src_token_topk_idx", + cutlass.Int32, + (num_experts_per_rank, world_size, max_slot), + 16, + ), _RegionSpec( "nvlink_barrier_signal", cutlass.Int32, @@ -562,6 +707,37 @@ def _build_shared_region_specs(self) -> List[_RegionSpec]: 16, ), ] + # separate-kernel-reduce only: the per-topk fc2 combine staging buffer is + # internalized here instead of being a caller tensor, so the public output + # is the 2D (T, hidden) reduce result. It is the cross-rank combine STG + # target, hence must live on the symmetric heap (= this shared workspace); + # appended after the data regions so it stays out of the per-launch zero + # prefix (only the first launch relies on the caller-zeroed workspace). + if not self.in_kernel_fc2_reduce: + # combine_quant: the cross-rank combine data plane, one cell per + # (token, topk). dtype follows the wire format -- the bf16 baseline is + # byte-identical to the old ``combine_partial``; fp4/e4m3 shrink it. + specs.append( + _RegionSpec( + "combine_quant", + self.combine_format.act_dtype, + (max_tokens_per_rank, num_topk, self.hidden), + 128, + )) + # combine_sf: per-block scale plane; only quantized formats carry one. + if self.combine_format.is_quantized: + # Flat padded capacity (max_tokens * num_topk * sf_block_pad scale + # entries); the (max_tokens, num_topk, hidden//scale_block) logical + # shape + 16 B-aligned per-block stride is assembled where the view + # is built in __call__. + specs.append( + _RegionSpec( + "combine_sf", + self.combine_format.scale_dtype, + (max_tokens_per_rank * num_topk * self.sf_block_pad, ), + 128, + )) + return specs # ========================================================================= # Public: workspace size query @@ -581,19 +757,24 @@ def get_workspace_sizes(self) -> Tuple[int, int]: @staticmethod def _make_typed_view( - byte_workspace: cute.Tensor, + byte_base: cute.Pointer, byte_offset: int, cute_dtype: Any, shape: Tuple[int, ...], stride: Optional[Tuple[int, ...]], assumed_align: int, ) -> cute.Tensor: - """Build a typed cute view at ``byte_offset`` of the opaque workspace.""" - # Large MegaMoE problems can place later workspace regions above the - # 2 GiB / 4 GiB boundary. Keep the base adjustment in 64-bit pointer - # arithmetic so region starts such as fc1_output_sf / counters do not - # wrap before the typed view is built. - byte_ptr = byte_workspace.iterator + Int64(byte_offset) + """Build a typed cute view at ``byte_offset`` of the opaque workspace. + + The workspace is a raw ``cute.Pointer`` (uint8 gmem base), not a tensor: + the kernel only ever needs the base address + its own byte-offset table, so + a tensor's shape would be both ignored AND, for >2 GiB workspaces (the + internalized combine staging), overflow cute's 32-bit memref shape field. + """ + # Large MegaMoE problems place later regions above the 2 GiB / 4 GiB + # boundary; keep the base adjustment in 64-bit pointer arithmetic so region + # starts (fc1_output_sf / counters) do not wrap before the typed view. + byte_ptr = byte_base + Int64(byte_offset) typed_iter = cute.make_ptr( cute_dtype, byte_ptr.toint(), @@ -605,7 +786,7 @@ def _make_typed_view( def _view_local( self, - local_workspace: cute.Tensor, + local_workspace: cute.Pointer, name: str, *, cute_dtype: Optional[Any] = None, @@ -628,7 +809,7 @@ def _view_local( def _view_shared( self, - shared_workspace: cute.Tensor, + shared_workspace: cute.Pointer, name: str, *, cute_dtype: Optional[Any] = None, @@ -646,7 +827,7 @@ def _view_shared( def _partition_region( self, - byte_workspace: cute.Tensor, + byte_workspace: cute.Pointer, offsets: Dict[str, int], spec: _RegionSpec, *, @@ -680,6 +861,39 @@ def _partition_region( # __call__ # ========================================================================= + def name(self) -> str: + """Full compiled-kernel cache key. Self-contained on purpose (no shared + helper): mirrors the base fc12 fields and appends the MegaMoE-specific + ones -- keep the shared part in sync with the base ``name()``. + ``local_rank`` excluded (deployment detail); same dropped set as base.""" + m, n, k = self.mma_tiler_mnk + cm, cn = self.cluster_shape_mn + exp = "x".join(map( + str, + self.static_expert_shape)) if self.static_expert_shape else "dyn" + epiflag = "x".join(map( + str, self.epi_flag_batch)) if self.epi_flag_batch else "none" + cta = "2_cta" if self.use_2cta_instrs else "1_cta" + fc2store = "fc2store_stg" if self.non_ubulk_fc2_store else "fc2store_ublk" + inkred = "inkernel_redg" if self.in_kernel_fc2_reduce else "no_inkernel_redg" + apply_topk = "apply_topk_fc1_pre_quant" if self.apply_topk_in_fc1 else "apply_topk_after_fc2" + token_back = { + "epi_warps": "epiwarps", + "standalone_warps": "standalone", + "reuse_dispatch_warps": "reuse_dispatch", + }.get(self.token_back_mode, self.token_back_mode) + return ( + "megamoe_nvfp4" + f"_mmatiler_{m}x{n}x{k}_cluster_{cm}x{cn}_{cta}_sched_{self.load_balance_mode}" + f"_expert_shape_{exp}_grouphint_{self.group_hint}" + f"_padding_{self.token_padding_block}x{self.sf_padding_block}" + f"_{fc2store}_{inkred}_token_back_by_{token_back}_{apply_topk}" + f"_fc2out{self.fc2_output_dtype.__name__}_combine{self.combine_format}_sfvec{self.sf_vec_size}" + f"_acc{self.acc_dtype.__name__}_clamp{self.gate_up_clamp}_epiflag{epiflag}" + # MegaMoE-specific constexpr: + f"_ep_{self.world_size}_topk_{self.num_topk}_maxtoken_{self.max_tokens_per_rank}" + f"_flagbatch_{self.flag_batch}") + @cute.jit def __call__( self, @@ -697,12 +911,14 @@ def __call__( fc1_alpha: cute.Tensor, fc2_alpha: cute.Tensor, fc1_norm_const: cute.Tensor, - # Combine destination (peer write target under S3; local fc2 - # output region under S2 -- same memory, same caller). - combine_output: cute.Tensor, # (T, num_topk, hidden) BF16 + # Final combined output the caller consumes: 2D (T, hidden). Under + # in_kernel_reduce it is the cross-rank REDG target (symmetric heap); + # under separate_kernel_reduce it is the local tail-reduce destination + # while the per-topk staging lives in the internal ``combine_quant``. + output_activation: cute.Tensor, # (T, hidden) BF16 # Opaque workspaces. - local_workspace: cute.Tensor, # (local_ws_bytes,) Uint8 - shared_workspace: cute.Tensor, # (shared_ws_bytes,) Uint8 + local_workspace: cute.Pointer, # uint8 gmem base of (local_ws_bytes,) + shared_workspace: cute.Pointer, # uint8 gmem base of (shared_ws_bytes,) # Runtime host payload; packed into ``SymBuffer{world_size}`` # before entering the device kernel. peer_rank_ptr_mapper_host, @@ -722,10 +938,13 @@ def __call__( unconstrained (cuda local or sym heap). * ``fc1_weight`` / ``fc1_weight_sf`` / ``fc2_weight`` / ``fc2_weight_sf`` are local-only. - * ``combine_output`` is the per-rank S3 combine STG target; - under S2 it acts as the rank's local BF16 fc2 output. - Placement: sym heap (peer write target) or local in the - single-rank degenerate case. + * ``output_activation`` is the 2D (T, hidden) result. Under + ``in_kernel_reduce`` it is the per-rank cross-rank combine STG + target and MUST be reachable via the peer mapper (sym heap, or + local in the single-rank degenerate case). Under + ``separate_kernel_reduce`` the cross-rank target is the internal + ``combine_quant`` staging region and ``output_activation`` only + receives the local tail reduce, so it may be plain local memory. Workspace zero-init contract: caller is currently expected to zero ``shared_workspace`` before launch (the dispatch @@ -853,34 +1072,129 @@ def __call__( stride=(2, ), ) + # MoE-domain ``(token_max, topk, hidden)`` cross-rank combine STG target. + # * in_kernel_reduce: REDG collapses topk on the fly, so this is a + # ``(T, 1, hidden)`` view of the caller's 2D ``output_activation`` + # (the epilogue's topk index is a constexpr 0 in this mode). + # * separate_kernel_reduce: peers write one cell per (token, topk) into + # the internal ``combine_quant`` staging; the tail reduce below + # collapses topk into ``output_activation``. + if cutlass.const_expr(self.in_kernel_fc2_reduce): + combine_target = cute.make_tensor( + output_activation.iterator, + cute.make_layout( + (self.max_tokens_per_rank, 1, self.hidden), + stride=(self.hidden, self.hidden, 1), + ), + ) + else: + combine_target = self._view_shared(shared_workspace, + "combine_quant") + + # Per-block scale plane parallel to combine_quant; quantized formats only + # (bf16 carries no SF, and in_kernel_reduce is bf16-by-construction). + sf_blocks = (self.hidden // self.combine_format.scale_block + if self.combine_format.is_quantized else 0) + if cutlass.const_expr(self.combine_format.is_quantized + and not self.in_kernel_fc2_reduce): + # Assemble the (token, topk, valid_blocks) logical view + 16 B-aligned + # per-block stride over the flat combine_sf capacity. + combine_sf = self._view_shared( + shared_workspace, + "combine_sf", + shape=(self.max_tokens_per_rank, self.num_topk, sf_blocks), + stride=(self.num_topk * self.sf_block_pad, self.sf_block_pad, + 1), + ) + else: + combine_sf = None + + if cutlass.const_expr(self.combine_format.is_quantized): + # Same: logical (pool_token, 1, valid_blocks) + padded per-block stride + # over the flat fc2_output_sf capacity. + fc2_output_sf_phys = self._view_local( + local_workspace, + "fc2_output_sf", + shape=(self.pool_token_capacity, 1, sf_blocks), + stride=(self.sf_block_pad, self.sf_block_pad, 1), + ) + sf_vec = self.combine_format.scale_block + sf_layout = fc2_output_sf_phys.layout + fc2_output_sf = cute.make_tensor( + fc2_output_sf_phys.iterator, + cute.make_layout( + (sf_layout.shape[0], sf_layout.shape[1], + (sf_vec, sf_layout.shape[2])), + stride=(sf_layout.stride[0], sf_layout.stride[1], + (0, sf_layout.stride[2])), + ), + ) + else: + fc2_output_sf = None + if cutlass.const_expr(self.token_back_by_dispatch): fc2_output_workspace_native = self._view_local( local_workspace, "fc2_output_workspace", ) + # Byte count = elements * dtype.width // 8 (multiply before divide so + # the 4-bit fp4 data plane is not truncated to zero bytes/element). fc2_output_workspace_u8 = self._make_typed_view( local_workspace, self._local_offsets["fc2_output_workspace"], cutlass.Uint8, - (pool_token_capacity * self.hidden * - (int(self.fc2_output_dtype.width) // 8), ), + ((pool_token_capacity * self.hidden * + int(self.combine_format.act_dtype.width)) // 8, ), None, self._local_region_by_name["fc2_output_workspace"].align, ) - fc2_done_counter = self._view_local( - local_workspace, - "fc2_done_counter", - ) combine_output_u8 = cute.recast_tensor( - combine_output, + combine_target, cutlass.Uint8, ) else: fc2_output_workspace_native = None fc2_output_workspace_u8 = None + combine_output_u8 = combine_target + + # fc2 completion gate: present whenever token-back runs (DATA and/or SF + # push), so the epi_warps SF-only push gates per expert just like the + # dispatch DATA path. + if cutlass.const_expr(self.token_back_enabled): + fc2_done_counter = self._view_local( + local_workspace, + "fc2_done_counter", + ) + else: fc2_done_counter = None - combine_output_u8 = combine_output + if cutlass.const_expr(self.token_back_schedule_mode == + "atomic_counter"): # nosec B105 + token_back_schedule_counter = self._view_local( + local_workspace, + "token_back_schedule_counter", + ).iterator + else: + token_back_schedule_counter = None + + # Int32 views over each workspace's front counter prefix; the kernel tail + # bulk-zeros them every launch (tail_reset_counters). + local_zero_prefix = self._make_typed_view( + local_workspace, + 0, + cutlass.Int32, + (self.local_zero_i32_count, ), + (1, ), + 16, + ) + shared_zero_prefix = self._make_typed_view( + shared_workspace, + 0, + cutlass.Int32, + (self.shared_zero_i32_count, ), + (1, ), + 16, + ) token_comm_args = ExtractedTokenCommArgs( input_token_buffer=activation, input_sf_buffer=activation_sf, @@ -896,14 +1210,19 @@ def __call__( fc1_ready_counter=l1_arrival_count, token_src_metadata=token_src_metadata, combine_output=combine_output_u8, + combine_sf=combine_sf, fc2_output_workspace=fc2_output_workspace_u8, + fc2_output_sf=fc2_output_sf, fc2_done_counter=fc2_done_counter, + token_back_schedule_counter=token_back_schedule_counter, nvlink_barrier_signal=nvlink_barrier_signal, nvlink_barrier_counter=nvlink_barrier_counter, grid_sync_counter=grid_sync_counter, + local_zero_prefix=local_zero_prefix, + shared_zero_prefix=shared_zero_prefix, peer_rank_ptr_mapper=peer_rank_ptr_mapper, world_size=self.world_size, - local_rank=self.local_rank, + local_rank=peer_rank_ptr_mapper_host.rank_idx, num_total_experts=self.num_total_experts, num_experts_per_rank=self.num_experts_per_rank, num_topk=self.num_topk, @@ -918,16 +1237,22 @@ def __call__( # sf_padding_block == "sf_block_m") so the pool layout and the # sched cumulative-row offsets align by construction. # - # ``combine_output`` is MoE-domain storage. Non-reduce modes use - # ``(max_tokens_per_rank, num_topk, hidden)`` and host-reduce topk; - # REDG modes use ``(max_tokens_per_rank, 1, hidden)`` and reduce in - # kernel. The epilogue return tile maps local pool rows back to the - # source rank's token row through ``token_comm_args``. + # ``combine_target`` is MoE-domain storage. separate_kernel_reduce uses + # ``(max_tokens_per_rank, num_topk, hidden)`` (internal staging) and a + # tail reduce; in_kernel_reduce uses ``(max_tokens_per_rank, 1, hidden)`` + # and reduces in kernel. The epilogue return tile maps local pool rows + # back to the source rank's token row through ``token_comm_args``. if cutlass.const_expr(self.token_back_by_dispatch): fc2_output_target = fc2_output_workspace_native else: - fc2_output_target = combine_output - + fc2_output_target = combine_target + + # Quantized combine loop is closed: the fc2 epilogue encodes the data + # plane (STG-to-peer in epi_warps, or local staging in the dispatch + # modes) and writes per-block scales to the rank-local SF plane + # (fc2_output_sf); the token-back warps push that SF plane + # token-contiguously to the peers' combine_sf (and the data plane too in + # the dispatch modes); TopkReduce below dequantizes and reduces. super().__call__( activation=l1_token_buffer_nvfp4, fc1_weight=fc1_weight, @@ -951,6 +1276,26 @@ def __call__( token_comm_args=token_comm_args, ) + # separate_kernel_reduce: collapse the per-topk ``combine_quant`` staging + # into the public 2D output via the shared TopkReduce launcher (sized from + # codegen-time hidden / num_topk / combine_format -- it dequantizes per + # format and reduces over topk). Same stream, so it is ordered strictly + # after the cross-rank combine writes landed (the mega kernel's nvlink + # barrier guarantees all peer combine STGs to this rank completed before it + # exits). Weighting follows the compute graph: deepgemm (apply_topk_in_fc1) + # folded the routing weight into fc1 -> plain K-sum; transformers applies + # topk_weights here. + if cutlass.const_expr(not self.in_kernel_fc2_reduce): + score = (topk_weights if + cutlass.const_expr(not self.apply_topk_in_fc1) else None) + TopkReduce(self.hidden, self.num_topk, self.combine_format)( + combine_target, + combine_sf, + output_activation, + score, + stream, + ) + # ========================================================================= # TokenComm delegation surface consumed by the fc1/fc2 base kernel # ========================================================================= @@ -994,6 +1339,24 @@ def token_comm_hook_dispatch_warp_body( tidx=tidx, ) + @cute.jit + def token_comm_hook_token_back_warp_body( + self, + token_comm_args, + token_comm_storage, + *, + warp_idx, + lane_idx, + tidx, + ): + self.token_comm.token_back_warp_body( + token_comm_args, + token_comm_storage, + warp_idx=warp_idx, + lane_idx=lane_idx, + tidx=tidx, + ) + @cute.jit def token_comm_hook_tail_reset_shared_counters( self, @@ -1003,8 +1366,9 @@ def token_comm_hook_tail_reset_shared_counters( local_warp_idx, lane_idx, ): - self.token_comm.tail_reset_shared_counters( + self.token_comm.tail_reset_counters( token_comm_args, + token_comm_args.shared_zero_prefix, cta_linear_id=cta_linear_id, local_warp_idx=local_warp_idx, lane_idx=lane_idx, diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/moe_utils.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/moe_utils.py index 037bb71b3101..a4e223eb0fcf 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/moe_utils.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/moe_utils.py @@ -62,7 +62,8 @@ def spin_wait( ip: Optional[ir.InsertionPoint] = None, ) -> Boolean: """Spin until condition is true, or do one condition check with peek_only.""" - current = cute.arch.load(ptr, ptr.dtype, cop="cg", loc=loc, ip=ip) + # The first load must be acquire to build the mem order. + current = cute.arch.load(ptr, ptr.dtype, sem="acquire", scope="gpu") if cutlass.const_expr(peek_only): # One-shot peek: forward the condition Boolean to the caller. return Boolean(condition(current)) @@ -70,7 +71,7 @@ def spin_wait( # Load with L1 cache bypass (ld.global.cg) if cutlass.const_expr(fail_sleep_cycles > 0): _nanosleep(fail_sleep_cycles, loc=loc, ip=ip) - current = cute.arch.load(ptr, ptr.dtype, cop="cg", loc=loc, ip=ip) + current = cute.arch.load(ptr, ptr.dtype, sem="acquire", scope="gpu") # Spin-path: condition was satisfied; uniformize return type with the # peek path so callers always see a Boolean. return Boolean(True) @@ -80,8 +81,7 @@ def spin_wait( # Cluster-DSMEM helpers (for atomic_counter dynamic scheduler) # ============================================================================= # -# Ported from cute_dsl_kernel_library/dsl_kernels/moe/moe_persistent_scheduler.py -# (lines 79-145). Used by the fused fc1+fc2 mega scheduler when +# Used by the fused fc1+fc2 mega scheduler when # load_balance_mode == 'atomic_counter' to # implement the leader-CTA atom.add + DSMEM broadcast cluster-tile-idx # fetch protocol. ``atom.add`` itself uses cute.arch.atomic_add (the diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/ptx_helpers.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/ptx_helpers.py index 0a6aafd61097..adcb04c8c1d7 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/ptx_helpers.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/ptx_helpers.py @@ -1,7 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Inline-PTX wrappers (TMA 1D load/store, fns.b32, raw-int64 peer ops) for the cuTeDSL dispatch kernel.""" +"""Inline-PTX wrappers (TMA 1D load/store, fns.b32, raw-int64 peer ops) for the cuTeDSL kernels.""" +from typing import Optional + +from cutlass._mlir import ir from cutlass._mlir.dialects import llvm from cutlass.cutlass_dsl import Float32, Int32, Int64, T, dsl_user_op @@ -92,6 +95,63 @@ def tma_store_1d(dst_gmem, src_smem, num_bytes, *, loc=None, ip=None): ) +@dsl_user_op +def cp_async_bulk_s2g(dst_gmem, + src_smem, + size_bytes, + *, + loc=None, + ip=None) -> None: + """Issue descriptor-free ``cp.async.bulk`` SMEM->GMEM. + + The caller owns ``cp_async_bulk_commit_group`` so copy and reduce bulk + paths can share the same group boundary. + """ + llvm.inline_asm( + None, + [ + dst_gmem.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), + src_smem.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), + size_bytes.ir_value(loc=loc, ip=ip), + ], + "cp.async.bulk.global.shared::cta.bulk_group [$0], [$1], $2;", + "l,r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def cp_reduce_async_bulk_add_noftz_bf16_s2g( + dst_gmem, + src_smem, + size_bytes, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, +) -> None: + """Issue descriptor-free ``cp.reduce.async.bulk`` for BF16 add.""" + llvm.inline_asm( + None, + [ + dst_gmem.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), + src_smem.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), + size_bytes.ir_value(loc=loc, ip=ip), + ], + "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.noftz.bf16 " + "[$0], [$1], $2;", + "l,r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + @dsl_user_op def fns_b32(mask: Int32, base: Int32, n: Int32, *, loc=None, ip=None) -> Int32: return Int32( @@ -204,6 +264,31 @@ def red_add_release_sys_u64_raw(addr: Int64, ) +@dsl_user_op +def red_add_relaxed_sys_u64_raw(addr: Int64, + val: Int64, + *, + loc=None, + ip=None) -> None: + """``red.relaxed.sys.global.add.u64`` via raw int64 byte address. + + Relaxed (no release fence) sibling of ``red_add_release_sys_u64_raw``. Use + when an enclosing ``bar.sync`` + a later release fence (e.g. the trailing + ``nvlink_barrier`` publish) already provides cross-rank ordering, so the + per-op release fence would just emit a redundant ``membar.sys`` drain. + """ + llvm.inline_asm( + None, + [addr.ir_value(), val.ir_value()], + "red.relaxed.sys.global.add.u64 [$0], $1;", + "l,l", + has_side_effects=True, + asm_dialect=0, + loc=loc, + ip=ip, + ) + + @dsl_user_op def red_add_release_sys_s32_raw(addr: Int64, val: Int32, @@ -227,6 +312,58 @@ def red_add_release_sys_s32_raw(addr: Int64, ) +@dsl_user_op +def red_add_relaxed_sys_v2_bf16x2( + addr, + val0_packed_bf16x2, + val1_packed_bf16x2, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, +) -> None: + """Issue ``red.relaxed.sys.global.add.v2.bf16x2 [addr], {v0, v1};``.""" + llvm.inline_asm( + None, + [ + addr.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), + val0_packed_bf16x2.ir_value(loc=loc, ip=ip), + val1_packed_bf16x2.ir_value(loc=loc, ip=ip), + ], + "red.relaxed.sys.global.add.noftz.v2.bf16x2 [$0], {$1, $2};", + "l,r,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def red_add_release_gpu_s32( + counter_ptr, + value, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, +) -> None: + """Issue ``red.release.gpu.global.add.s32`` to a GMEM int32 location.""" + llvm.inline_asm( + None, + [ + counter_ptr.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip), + value.ir_value(loc=loc, ip=ip), + ], + "red.release.gpu.global.add.s32 [$0], $1;", + "l,r", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + loc=loc, + ip=ip, + ) + + @dsl_user_op def red_async_add_release_sys_u32_raw(addr: Int64, val: Int32, diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/sym_buffer.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/sym_buffer.py index 33af2cfc5380..c63ac97d5fa0 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/sym_buffer.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/sym_buffer.py @@ -2,21 +2,28 @@ # SPDX-License-Identifier: Apache-2.0 """Symmetric-heap peer pointer mapper. -``SymBufferHost`` is the runtime payload that crosses the Python -> -generated-host-code boundary. Inside the generated host wrapper, it packs -the runtime base address and per-rank offsets into a device-side -``SymBuffer{N}`` native struct: - - { i64 base, vector offsets, i32 rank_idx } - -Device code only sees that struct and calls ``.map`` / -``.ptr_map_to_rank``. The vector field is deliberate: LLVM supports -runtime-indexed ``extractelement`` on vectors, which NVPTX lowers to an -indexed param-bank load (``LDC.U64``). +The device-side SymBuffer carries ONLY the per-rank offset table. Layout is +chosen at trace time by ``num_max_ranks`` (a constexpr): + +* ``<= 16`` ranks: a ``cute_nvgpu.grid_constant`` / ``llvm.byval`` pointer to a + ``struct<(array<16 x i64>)>`` (exactly 128B). ``map`` does a runtime-indexed + ``getelementptr`` + ``load`` that lowers to a const-bank ``LDC c[bnk][Ra]``. + The 128B byval size matches the host-side ``createKernelArgs`` copy (hardcoded + to 128B today), so it is corruption-safe up to EP16. + +* ``> 16`` ranks: a by-value ``vector`` read with ``extractelement``. + Correct but spills the runtime index -- the fallback until the byval-size fix + reaches public cutedsl. + +``base`` is intentionally dropped (``get_base_ptr`` had no callers) and +``rank_idx`` lives outside the offset table: ``_SymBufferHostAdapter`` +marshals it as a separate i32 scalar and the kernel consumes it as the device +``local_rank`` (see ``megamoe_kernel.py``). Freeing those struct slots lets +all 16 byval lanes hold offsets (EP16, not EP14). """ from dataclasses import dataclass -from typing import Any, Tuple +from typing import Any, Optional import cutlass import cutlass.cute as cute @@ -24,16 +31,64 @@ from cutlass._mlir.dialects import arith, llvm from cutlass.base_dsl.dsl import (extract_mlir_values, get_mlir_types, new_from_mlir_values) -from cutlass.base_dsl.native_struct import native_struct from cutlass.base_dsl.runtime.jit_arg_adapters import JitArgAdapterRegistry from cutlass.base_dsl.typing import get_c_pointers from cutlass.cute.typing import AddressSpace from cutlass.cutlass_dsl import Int32, Int64, dsl_user_op +try: + # GEP encodes a runtime index by storing this sentinel in rawConstantIndices; + # it is MLIR's LLVM::GEPOp::kDynamicIndex (== INT32_MIN), frozen by the IR + # encoding ABI. Track the canonical constant rather than re-hardcoding it. + from cutlass.base_dsl.typing import MLIR_DYNAMIC_INDEX +except ImportError: # older wheels: value is fixed by the GEP encoding ABI + MLIR_DYNAMIC_INDEX = -(2**31) + +_BYVAL_RANK_LIMIT = 16 # struct<(array<16 x i64>)> == exactly 128B + + +# TODO: Remove once the compiler is fixed. Workaround for a cuda-to-llvm bug: +# any kernel arg marked `grid_constant + byval` is treated as a tma_desc. +def _byval_struct_ty() -> Any: + """128B byval pointee shared by alloca / GEP / the ``llvm.byval`` attr.""" + return ir.Type.parse(f"!llvm.struct<(array<{_BYVAL_RANK_LIMIT} x i64>)>") + @dataclass(frozen=True) class SymBufferDeviceBase: - """Device-side methods shared by all generated ``SymBuffer{N}`` types.""" + """Device-side SymBuffer: the per-rank offset table only. + + ``val`` is a ``!llvm.ptr`` to a byval/grid_constant ``struct<(array<16 x i64>)>`` + when ``num_max_ranks <= 16`` (``map`` -> GEP + load -> ``LDC``), else a by-value + ``vector`` (``map`` -> ``extractelement``, spills). + """ + + val: Any + num_max_ranks: cutlass.Constexpr[int] + + def __extract_mlir_values__(self) -> list: + return [self.val] + + def __new_from_mlir_values__(self, values: list) -> "SymBufferDeviceBase": + return SymBufferDeviceBase(val=values[0], + num_max_ranks=self.num_max_ranks) + + def __get_mlir_types__(self) -> list: + if self.num_max_ranks <= _BYVAL_RANK_LIMIT: + return [ir.Type.parse("!llvm.ptr")] + return [ir.Type.parse(f"vector<{self.num_max_ranks}xi64>")] + + def __extract_mlir_attributes__(self) -> list: + if self.num_max_ranks <= _BYVAL_RANK_LIMIT: + return [ + ir.DictAttr.get({ + "cute_nvgpu.grid_constant": + ir.UnitAttr.get(), + "llvm.byval": + ir.TypeAttr.get(_byval_struct_ty()), + }) + ] + return [ir.DictAttr.get({})] @cute.jit def map( @@ -42,84 +97,103 @@ def map( dst_rank_idx: Int32, byte_off: Int64 = Int64(0), ) -> Int64: - off = Int64(llvm.extractelement(self.offsets, dst_rank_idx.ir_value())) + if cutlass.const_expr(self.num_max_ranks <= _BYVAL_RANK_LIMIT): + # Opaque ptr -> the offsets array sits at byte 0 of the byval struct, + # so a flat ``gep i64, ptr, dst_rank`` reaches offsets[dst_rank] + # directly; the byval struct type only governs the 128B const-bank copy. + i64_ty = ir.Type.parse("i64") + off_ptr = llvm.getelementptr( + ir.Type.parse("!llvm.ptr"), + self.val, + [dst_rank_idx.ir_value()], + [MLIR_DYNAMIC_INDEX], + i64_ty, + no_wrap_flags="None", + ) + off = Int64(llvm.load(i64_ty, off_ptr)) + else: + off = Int64(llvm.extractelement(self.val, dst_rank_idx.ir_value())) return local_ptr + off + byte_off @cute.jit - def get_base_ptr(self) -> Int64: - return self.base - - @cute.jit - def ptr_map_to_rank(self, ptr, dst_rank_idx: Int32): + def ptr_map_to_rank(self, + ptr, + dst_rank_idx: Int32, + byte_align: Optional[int] = None): if cutlass.const_expr(ptr.memspace != AddressSpace.gmem): raise ValueError( f"ptr_map_to_rank: source pointer must live in GMEM " f"(NVSHMEM symmetric heap), got memspace={ptr.memspace}.") + if cutlass.const_expr(byte_align is None): + byte_align = ptr.max_alignment peer_addr = self.map(ptr.toint(), dst_rank_idx, Int64(0)) return cute.make_ptr( ptr.dtype, peer_addr, ptr.memspace, - assumed_align=ptr.max_alignment, + assumed_align=byte_align, ) @dataclass(frozen=True) class SymBufferHost: - """Runtime launch payload for a device-side ``SymBuffer{N}``.""" + """Runtime launch payload for a device-side ``SymBuffer{N}``. + + Marshalled across the JIT boundary by ``_SymBufferHostAdapter`` (registered + below), which re-wraps the scalar fields with ``Int64(...)``.""" - base_addr: int - offsets: Tuple[int, ...] - rank_idx: int + offsets: tuple + rank_idx: Int32 num_max_ranks: cutlass.Constexpr[int] @staticmethod def _as_int64(value) -> Int64: return value if isinstance(value, Int64) else Int64(int(value)) - @staticmethod - def _as_int32(value) -> Int32: - return value if isinstance(value, Int32) else Int32(int(value)) - - @staticmethod - def _make_device_type(num_max_ranks: int) -> type: - if num_max_ranks <= 0: - raise ValueError( - f"num_max_ranks must be positive, got {num_max_ranks}") - - vec_ty_str = f"vector<{num_max_ranks}xi64>" - - class _OffsetsT: - - @staticmethod - def mlir_type() -> ir.Type: - return ir.Type.parse(vec_ty_str) - - @native_struct - class _SymBufferDevice(SymBufferDeviceBase): - base: Int64 - offsets: _OffsetsT - rank_idx: Int32 - - cls = _SymBufferDevice - cls.__name__ = f"SymBuffer{num_max_ranks}" - cls.__qualname__ = cls.__name__ - cls.NUM_MAX_RANKS = num_max_ranks - return cls - @dsl_user_op def make_device_obj(self, *, loc=None, ip=None) -> Any: + """Build the offsets-only device obj (see module docstring for layout).""" offsets = tuple(self.offsets) num_max_ranks = self.num_max_ranks if len(offsets) != num_max_ranks: - raise ValueError( - f"len(offsets)={len(offsets)} must equal " - f"num_max_ranks={num_max_ranks}; SymBuffer requires its " - f"runtime payload length to match the compiled vector type.") + raise ValueError(f"len(offsets)={len(offsets)} must equal " + f"num_max_ranks={num_max_ranks}.") + + if num_max_ranks <= _BYVAL_RANK_LIMIT: + ptr_ty = ir.Type.parse("!llvm.ptr") + st_ty = _byval_struct_ty() + i64_ty = ir.Type.parse("i64") + one = arith.constant( + value=ir.IntegerAttr.get(i64_ty, 1), + result=i64_ty, + loc=loc, + ip=ip, + ) + buf = llvm.alloca( + res=ptr_ty, + elem_type=st_ty, + array_size=one, + alignment=64, + loc=loc, + ip=ip, + ) + for i, off in enumerate(offsets): + slot = llvm.getelementptr( + ptr_ty, + buf, + [], + [i], + i64_ty, + no_wrap_flags="None", + loc=loc, + ip=ip, + ) + llvm.store(self._as_int64(off).ir_value(), slot, loc=loc, ip=ip) + return SymBufferDeviceBase(val=buf, num_max_ranks=num_max_ranks) + i32_ty = ir.Type.parse("i32") vec_ty = ir.Type.parse(f"vector<{num_max_ranks}xi64>") vec = llvm.mlir_zero(vec_ty, loc=loc, ip=ip) - i32_ty = ir.Type.parse("i32") for i, off in enumerate(offsets): idx = arith.constant( value=ir.IntegerAttr.get(i32_ty, i), @@ -134,14 +208,7 @@ def make_device_obj(self, *, loc=None, ip=None) -> Any: loc=loc, ip=ip, ) - - return self._make_device_type(num_max_ranks)( - base=self._as_int64(self.base_addr), - offsets=vec, - rank_idx=self._as_int32(self.rank_idx), - loc=loc, - ip=ip, - ) + return SymBufferDeviceBase(val=vec, num_max_ranks=num_max_ranks) @JitArgAdapterRegistry.register_jit_arg_adapter(SymBufferHost) @@ -150,7 +217,7 @@ class _SymBufferHostAdapter: Python-side ``SymBufferHost`` stays pure host data (ints + tuple). The adapter is the only place that maps it to DSL scalar arguments: - base/offsets are i64, rank_idx is i32, and num_max_ranks remains a + offsets are i64, rank_idx is i32, and num_max_ranks remains a constexpr carried through reconstruction. """ @@ -161,7 +228,6 @@ def __init__(self, arg: SymBufferHost) -> None: f"len(offsets)={len(tuple(arg.offsets))} must equal " f"num_max_ranks={int(arg.num_max_ranks)}.") self._fields = ( - Int64(arg.base_addr), *(Int64(x) for x in arg.offsets), Int32(arg.rank_idx), ) @@ -187,13 +253,8 @@ def __extract_mlir_values__(self) -> list[ir.Value]: def __new_from_mlir_values__(self, values: list[ir.Value]) -> SymBufferHost: idx = 0 - base_n = len(get_mlir_types(self._fields[0])) - base_addr = new_from_mlir_values(self._fields[0], - values[idx:idx + base_n]) - idx += base_n - offsets = [] - for field in self._fields[1:-1]: + for field in self._fields[:-1]: n = len(get_mlir_types(field)) offsets.append(new_from_mlir_values(field, values[idx:idx + n])) idx += n @@ -208,7 +269,6 @@ def __new_from_mlir_values__(self, values: list[ir.Value]) -> SymBufferHost: ) obj = object.__new__(SymBufferHost) - object.__setattr__(obj, "base_addr", base_addr) object.__setattr__(obj, "offsets", tuple(offsets)) object.__setattr__(obj, "rank_idx", rank_idx) object.__setattr__(obj, "num_max_ranks", self._arg.num_max_ranks) diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/token_comm.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/token_comm.py index d0c70b730ddf..fd67dd4828bf 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/token_comm.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/token_comm.py @@ -6,11 +6,14 @@ ``dispatch_kernel`` uses the same object methods as the fused MegaMoE kernel. """ -from typing import Any, Dict, List +import dataclasses +from typing import Any, ClassVar, Dict, List, Literal, Optional, Union import cutlass import cutlass.cute as cute import cutlass.pipeline as pipeline +from cutlass.base_dsl.dsl import extract_mlir_attributes +from cutlass.cute.typing import AddressSpace from cutlass.cutlass_dsl import (Float32, Int32, Int64, Uint8, Uint32, extract_mlir_values, new_from_mlir_values) @@ -27,27 +30,160 @@ from cutlass._mlir import ir +from .flag_batch import GpuReleaseFlagBatchTracker from .grid_sync import software_grid_sync -from .moe_utils import spin_wait -from .ptx_helpers import (fns_b32, ldg_b32_raw, ldg_f32_raw, - red_add_release_sys_s32_raw, - red_add_release_sys_u64_raw, stg_b32_raw, stg_b64_raw, +from .moe_utils import _nanosleep, spin_wait +from .ptx_helpers import (cp_reduce_async_bulk_add_noftz_bf16_s2g, fns_b32, + ldg_b32_raw, ldg_f32_raw, read_clock64, + red_add_relaxed_sys_u64_raw, + red_add_release_sys_s32_raw, stg_b32_raw, stg_b64_raw, tma_load_1d_raw, tma_store_1d) from .sf_swizzle import sf_atom_int32_offset +# --------------------------------------------------------------------------- +# Low-precision combine wire format (the central driver for the token-back +# quantized combine path: fc2 epilogue encoder, token_comm push, and the +# topk_reduce receiver all describe themselves through one CombineFormat). +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class CombineFormat: + """Wire format of the cross-rank combine (token-back) payload. + + The fc2 epilogue quantizes each token's hidden vector into a packed data + plane (``combine_quant``) plus a per-block scale plane (``combine_sf``); the + receiver (topk_reduce) dequantizes and reduces over topk. The consumer is + ALWAYS a software dequant -- never a tensor-core MMA -- so the format is + fully self-defined here, with no hardware scale-layout constraint. + + Canonical string ``"{scale_block}{act}x{scale}"`` (leading number = scale + block size in hidden elements), e.g. ``"16e2m1xbf16"`` (per-16 bf16 amax + + fp4 data) or ``"32e4m3xe8m0"`` (standard MXFP8). ``"bf16"`` is the + no-staging baseline: bf16 fc2 terms reduced directly, no scale plane. + """ + + # Element/scale tag <-> cuTe dtype. Only the dtypes a real format uses + # today (the bf16 baseline is ``scale_dtype is None``); extend when a new + # format is actually added rather than ahead of need. + _act_by_tag: ClassVar[Dict[str, type]] = { + "e2m1": cutlass.Float4E2M1FN, + "e4m3": cutlass.Float8E4M3FN, + } + _scale_by_tag: ClassVar[Dict[str, type]] = { + "bf16": cutlass.BFloat16, + "e8m0": cutlass.Float8E8M0FNU, + } + + act_dtype: type # cuTe dtype of the packed data-plane element + scale_dtype: Optional[ + type] # cuTe dtype of a scale entry; None == bf16 baseline + scale_block: Optional[ + int] # hidden elements per scale entry; None == baseline + + def __post_init__(self): + allowed_act = {cutlass.BFloat16, *self._act_by_tag.values()} + if self.act_dtype not in allowed_act: + raise ValueError( + f"combine act_dtype {self.act_dtype} not in {allowed_act}.") + allowed_scale = {None, *self._scale_by_tag.values()} + if self.scale_dtype not in allowed_scale: + raise ValueError( + f"combine scale_dtype {self.scale_dtype} not in {allowed_scale}." + ) + if self.scale_dtype is None: # bf16 no-staging baseline + if self.act_dtype is not cutlass.BFloat16 or self.scale_block is not None: + raise ValueError( + "baseline must be bf16 act with scale_block=None.") + return + if self.act_dtype is cutlass.BFloat16: + raise ValueError("a quantized combine cannot use a bf16 act dtype.") + # The scale dtype pins the block: per-16 bf16 amax / per-32 e8m0 power-of-two. + if self.scale_dtype is cutlass.BFloat16 and self.scale_block != 16: + raise ValueError("bf16 amax scale requires scale_block == 16.") + if self.scale_dtype is cutlass.Float8E8M0FNU and self.scale_block != 32: + raise ValueError("e8m0 scale requires scale_block == 32.") -def _store_token_src_metadata_u32x3( - token_src_metadata, - pool_token_idx, - src_rank: Uint32, - src_token: Uint32, - src_topk: Uint32, -) -> None: - """Store `{src_rank, src_token, src_topk}` as three 32-bit fields.""" - base_ptr = token_src_metadata.iterator + (pool_token_idx * Int32(12)) - cute.arch.store(base_ptr, src_rank, scope="gpu") - cute.arch.store(base_ptr + Int32(4), src_token, scope="gpu") - cute.arch.store(base_ptr + Int32(8), src_topk, scope="gpu") + @property + def is_quantized(self) -> bool: + """``False`` for the bf16 (no-staging) baseline.""" + return self.scale_dtype is not None + + @property + def name(self) -> str: + if not self.is_quantized: + return "bf16" + act_tag = next(t for t, d in self._act_by_tag.items() + if d is self.act_dtype) + scale_tag = next(t for t, d in self._scale_by_tag.items() + if d is self.scale_dtype) + return f"{self.scale_block}{act_tag}x{scale_tag}" + + def __str__(self) -> str: + return self.name + + @classmethod + def parse(cls, text: str) -> "CombineFormat": + """Build a CombineFormat from its canonical string (the argparser entry). + + Only the handful of supported wire formats are accepted; each key is the + exact string ``name`` produces (so ``parse(str(fmt)) == fmt``). + """ + # (act_dtype, scale_dtype, scale_block); None scale == bf16 baseline. + specs = { + "bf16": (cutlass.BFloat16, None, None), + "16e2m1xbf16": (cutlass.Float4E2M1FN, cutlass.BFloat16, 16), + "32e4m3xe8m0": (cutlass.Float8E4M3FN, cutlass.Float8E8M0FNU, 32), + } + token = text.strip().lower() + if token not in specs: + raise ValueError( + f"invalid combine_format {text!r}: expected one of {tuple(specs)}." + ) + act_dtype, scale_dtype, scale_block = specs[token] + return cls(act_dtype=act_dtype, + scale_dtype=scale_dtype, + scale_block=scale_block) + + +@dataclasses.dataclass(frozen=True) +class TokenSrcMetadata: + """Per pool-token routing record: written by token-in, read by token-back + and the fc2 combine-redirect epilogue. + + Wire format is one i64: low 32b = ``src_token`` (needs full width); high 32b + = ``(src_rank << 16) | src_topk`` (``src_rank < world_size`` and + ``src_topk < num_topk`` both fit in 16b). ``load`` / ``store`` accept either + a ``cute.Pointer`` or a raw ``Int64`` byte address. + """ + + src_rank: Int32 + src_token: Int32 + src_topk: Int32 + + nbytes: ClassVar[int] = 8 + + def _pack(self) -> Int64: + hi = (Int64(self.src_rank) << Int64(16)) | Int64(self.src_topk) + return (hi << Int64(32)) | (Int64(self.src_token) & Int64(0xFFFFFFFF)) + + @staticmethod + def _i64_ptr(addr: Union[cute.Pointer, Int64]) -> cute.Pointer: + addr_i = addr if isinstance(addr, Int64) else addr.toint() + return cute.make_ptr(Int64, addr_i, AddressSpace.gmem, assumed_align=8) + + def store(self, addr: Union[cute.Pointer, Int64]) -> None: + cute.arch.store(self._i64_ptr(addr), self._pack(), scope="gpu") + + @classmethod + def load(cls, addr: Union[cute.Pointer, Int64]) -> "TokenSrcMetadata": + v = Int64(cute.arch.load(cls._i64_ptr(addr), Int64, scope="gpu")) + hi = v >> Int64(32) + return cls( + src_rank=Int32((hi >> Int64(16)) & Int64(0xFFFF)), + src_token=Int32(v & Int64(0xFFFFFFFF)), + src_topk=Int32(hi & Int64(0xFFFF)), + ) _MLIR_VALUE_FIELDS = ( @@ -65,17 +201,22 @@ def _store_token_src_metadata_u32x3( "fc1_ready_counter", "token_src_metadata", "combine_output", + "combine_sf", "fc2_output_workspace", + "fc2_output_sf", "fc2_done_counter", + "token_back_schedule_counter", "nvlink_barrier_signal", "nvlink_barrier_counter", "grid_sync_counter", + "local_zero_prefix", + "shared_zero_prefix", "peer_rank_ptr_mapper", + "local_rank", ) _CONST_FIELDS = ( "world_size", - "local_rank", "num_total_experts", "num_experts_per_rank", "num_topk", @@ -110,6 +251,8 @@ def __init__( nvlink_barrier_signal: cute.Tensor, nvlink_barrier_counter: cute.Tensor, grid_sync_counter: cute.Tensor, + local_zero_prefix: cute.Tensor, + shared_zero_prefix: cute.Tensor, peer_rank_ptr_mapper: Any, world_size: int, local_rank: int, @@ -123,6 +266,9 @@ def __init__( sm_count: int, fc2_output_workspace: cute.Tensor = None, fc2_done_counter: cute.Tensor = None, + token_back_schedule_counter: cute.Pointer = None, + combine_sf: cute.Tensor = None, + fc2_output_sf: cute.Tensor = None, ): self.input_token_buffer = input_token_buffer self.input_sf_buffer = input_sf_buffer @@ -138,11 +284,16 @@ def __init__( self.fc1_ready_counter = fc1_ready_counter self.token_src_metadata = token_src_metadata self.combine_output = combine_output + self.combine_sf = combine_sf self.fc2_output_workspace = fc2_output_workspace + self.fc2_output_sf = fc2_output_sf self.fc2_done_counter = fc2_done_counter + self.token_back_schedule_counter = token_back_schedule_counter self.nvlink_barrier_signal = nvlink_barrier_signal self.nvlink_barrier_counter = nvlink_barrier_counter self.grid_sync_counter = grid_sync_counter + self.local_zero_prefix = local_zero_prefix + self.shared_zero_prefix = shared_zero_prefix self.peer_rank_ptr_mapper = peer_rank_ptr_mapper self.world_size = world_size self.local_rank = local_rank @@ -164,6 +315,17 @@ def __extract_mlir_values__(self) -> List[ir.Value]: values.extend(extract_mlir_values(attr)) return values + def __extract_mlir_attributes__(self) -> List[Any]: + # Mirror __extract_mlir_values__ 1:1 so per-arg attrs stay aligned; the + # only non-empty entry is peer_rank_ptr_mapper's byval/grid_constant. + attrs: List[Any] = [] + for name in _MLIR_VALUE_FIELDS: + attr = getattr(self, name) + if attr is None: + continue + attrs.extend(extract_mlir_attributes(attr)) + return attrs + def __new_from_mlir_values__(self, values: List[ir.Value]) -> "TokenCommArgs": idx = 0 @@ -176,9 +338,8 @@ def __new_from_mlir_values__(self, n = len(extract_mlir_values(proto)) rebuilt[name] = new_from_mlir_values(proto, values[idx:idx + n]) idx += n - assert idx == len(values), ( - f"TokenCommArgs serialization mismatch: consumed={idx} provided={len(values)}" - ) + assert idx == len(values), (f"TokenCommArgs serialization mismatch: " + f"consumed={idx} provided={len(values)}") const_kwargs = {name: getattr(self, name) for name in _CONST_FIELDS} return TokenCommArgs(**rebuilt, **const_kwargs) @@ -192,14 +353,13 @@ class TokenInPullTokenBackPush: dispatch_intra_cta_bar_id: int = 10 kernel_tail_named_barrier_id: int = 8 dispatch_to_sched_named_barrier_id: int = 9 - dispatch_to_sched_threads: int = (num_dispatch_warps + 1) * warp_threads + # dispatch_to_sched / kernel_tail thread counts are per-instance (see __init__). experts_per_dispatch_pass: int = num_dispatch_threads def __init__( self, *, world_size: int, - local_rank: int, num_topk: int, num_experts_per_rank: int, num_total_experts: int, @@ -212,11 +372,17 @@ def __init__( cluster_shape_mn, dispatch_warp_start: int, num_other_warps: int, - fc2_output_dtype=None, + combine_format: "CombineFormat" = None, + token_back_by_dispatch: bool = False, fc2_publishes_per_token_cluster_tile: int = 0, + token_back_reduce_topk: bool = False, + token_back_standalone: bool = False, + flag_batch: int = 1, + is_swap_ab: bool = False, + token_back_schedule_mode: Literal["static", + "atomic_counter"] = "static", ) -> None: self.world_size = world_size - self.local_rank = local_rank self.num_topk = num_topk self.num_experts_per_rank = num_experts_per_rank self.num_total_experts = num_total_experts @@ -228,6 +394,19 @@ def __init__( self.sf_padding_block = sf_padding_block self.cluster_tile_tokens = cluster_tile_tokens self.cluster_shape_mn = cluster_shape_mn + if flag_batch < 1 or flag_batch > 32: + raise ValueError( + f"flag_batch must be in [1, 32], got {flag_batch}.") + # Release-flag batch size consumed by dispatch_pull as a Python int. + # One warp lane carries one delayed release target. + self._flag_batch = flag_batch + self.is_swap_ab = is_swap_ab + + if token_back_schedule_mode not in ("static", "atomic_counter"): + raise ValueError( + "token_back_schedule_mode must be 'static' or " + f"'atomic_counter'; got {token_back_schedule_mode!r}.") + self.token_back_schedule_mode = token_back_schedule_mode self.dispatch_warp_start = dispatch_warp_start # Warps that share this CTA with the dispatch group but are not part # of it. They participate in kernel-tail / dispatch-with-other @@ -236,12 +415,53 @@ def __init__( # collapse to dispatch-only). self.num_other_warps = num_other_warps self.num_other_threads = num_other_warps * self.warp_threads - self.num_total_threads = self.num_dispatch_threads + self.num_other_threads + + if combine_format is None: + combine_format = CombineFormat( + act_dtype=cutlass.BFloat16, + scale_dtype=None, + scale_block=None, + ) + self.combine_format = combine_format + self.token_back_by_dispatch = token_back_by_dispatch + self.push_data = token_back_by_dispatch + self.push_sf = combine_format.is_quantized + + # Standalone token-back: a dedicated warpgroup (size == dispatch group) + self.token_back_standalone = token_back_standalone + self.num_token_back_warps = self.num_dispatch_warps if self.token_back_standalone else 0 + self.num_token_back_threads = self.num_token_back_warps * self.warp_threads + self.token_back_warp_start = dispatch_warp_start + self.num_dispatch_warps + # Standalone token-back per-warp pull buffer; token is moved in + # tb_chunk_bytes pieces (last piece carries the remainder), so this is + # independent of hidden. + self.tb_chunk_bytes = 2048 + + self.num_total_threads = (self.num_dispatch_threads + + self.num_other_threads + + self.num_token_back_threads) + self.dispatch_to_sched_threads = ( + self.num_dispatch_warps + 1 + + self.num_token_back_warps) * self.warp_threads self.kernel_tail_threads = self.num_total_threads - self.fc2_output_dtype = fc2_output_dtype - if fc2_output_dtype is not None: - self.fc2_token_bytes = hidden * int(fc2_output_dtype.width) // 8 + # The DATA wire dtype is the combine act dtype (bf16 baseline -> bf16; + # fp4/e4m3 quantized), NOT the kernel's fc2 output dtype: the cross-rank + # payload is what the receiver dequantizes. + self.fc2_output_dtype = combine_format.act_dtype + if token_back_reduce_topk: + if not token_back_by_dispatch: + raise ValueError( + "token_back_reduce_topk=True requires the dispatch " + "token-back DATA path (token_back_by_dispatch).") + if combine_format.act_dtype is not cutlass.BFloat16: + raise NotImplementedError( + "token_back_reduce_topk currently supports a bf16 combine " + f"only, got {combine_format}.") + self.token_back_reduce_topk = token_back_reduce_topk + if self.enable_token_back: + self.fc2_token_bytes = hidden * int( + combine_format.act_dtype.width) // 8 if self.fc2_token_bytes % self.hidden_bytes != 0: raise ValueError( f"fc2_token_bytes={self.fc2_token_bytes} must be a " @@ -251,8 +471,9 @@ def __init__( if fc2_publishes_per_token_cluster_tile <= 0: raise ValueError( "fc2_publishes_per_token_cluster_tile must be > 0 when " - "fc2_output_dtype is set (token_back_by_push enabled).") - self.fc2_publishes_per_token_cluster_tile = fc2_publishes_per_token_cluster_tile + "token-back is enabled (it gates the per-expert push).") + self.fc2_publishes_per_token_cluster_tile = ( + fc2_publishes_per_token_cluster_tile) else: self.fc2_token_bytes = 0 self.fc2_num_chunks = 0 @@ -260,12 +481,29 @@ def __init__( @property def enable_token_back(self) -> bool: - return self.fc2_output_dtype is not None + # token-back warps run if they push the DATA plane, the SF plane, or both. + return self.push_data or self.push_sf def extra_smem_storage_class(self) -> type: hidden_bytes = self.hidden_bytes num_total_experts = self.num_total_experts + if self.token_back_standalone: + + @cute.struct + class TokenCommStorage: + pull_mbar: cute.struct.MemRange[Int64, self.num_dispatch_warps] + smem_expert_count: cute.struct.MemRange[Int32, + num_total_experts] + pull_buffer: cute.struct.Align[cute.struct.MemRange[ + Uint8, self.num_dispatch_warps * hidden_bytes], 16] + tb_pull_mbar: cute.struct.MemRange[Int64, + self.num_token_back_warps] + tb_pull_buffer: cute.struct.Align[cute.struct.MemRange[ + Uint8, self.num_token_back_warps * self.tb_chunk_bytes], 16] + + return TokenCommStorage + @cute.struct class TokenCommStorage: pull_mbar: cute.struct.MemRange[Int64, self.num_dispatch_warps] @@ -288,14 +526,22 @@ def sched_warp_pre_init_wait(self, token_comm_args): @cute.jit def fc1_tma_b_predispatch_spin(self, token_comm_args, work_tile_info): - counter_slot = work_tile_info.cumulative_token_block_count + work_tile_info.tile_n_idx + if cutlass.const_expr(self.is_swap_ab): + counter_slot = work_tile_info.cumulative_token_block_count + work_tile_info.tile_n_idx + peek_threshold = work_tile_info.valid_tokens_in_cta_tile + else: + counter_slot = (work_tile_info.cumulative_token_block_count + + work_tile_info.tile_m_idx // + cutlass.Int32(self.cluster_shape_mn[0])) + peek_threshold = work_tile_info.valid_tokens_in_cluster_tile + counter_ptr = token_comm_args.fc1_ready_counter.iterator + counter_slot if not work_tile_info.peek_ready: _iket.range_push("tma_token_fc1_wait") spin_wait( counter_ptr, - lambda v: v >= work_tile_info.valid_tokens_in_tile, - fail_sleep_cycles=20, + lambda v: v >= peek_threshold, + fail_sleep_cycles=1000, ) _iket.range_pop() @@ -311,6 +557,7 @@ def dispatch_prep( warp_idx, lane_idx, *, + local_rank, num_tokens, num_sms, ): @@ -400,8 +647,9 @@ def dispatch_prep( topk_slot) MAX_SLOT_C: cutlass.Constexpr[ int] = num_tokens * self.num_topk - elem_off = ((local_expert * Int32(self.world_size) + Int32( - self.local_rank)) * Int32(MAX_SLOT_C) + slot) * Int32(4) + elem_off = ((local_expert * Int32(self.world_size) + + Int32(local_rank)) * Int32(MAX_SLOT_C) + + slot) * Int32(4) peer_addr = peer_rank_ptr_mapper.map( src_token_topk_idx.iterator.toint(), dst_rank, @@ -424,8 +672,9 @@ def dispatch_barrier( warp_idx, lane_idx, *, + local_rank, num_sms, - nvlink_barrier_counter=None, + nvlink_barrier_counter, ): # software_grid_sync expects a dispatch-group-relative thread id. tid_in_group = warp_idx * Int32(self.warp_threads) + lane_idx @@ -456,9 +705,9 @@ def dispatch_barrier( ) token_count_u32 = Int32(status_u64 & Int64(0xFFFFFFFF)) erc_local_base = expert_recv_count.iterator.toint() - erc_elem_off = (Int32(self.local_rank) * - Int32(self.num_experts_per_rank) + - dst_local_expert) * Int32(8) + erc_elem_off = ( + Int32(local_rank) * Int32(self.num_experts_per_rank) + + dst_local_expert) * Int32(8) erc_peer_addr = peer_rank_ptr_mapper.map( erc_local_base, dst_rank, @@ -471,7 +720,8 @@ def dispatch_barrier( dst_rank, Int64(dst_local_expert * Int32(8)), ) - red_add_release_sys_u64_raw(ercs_peer_addr, status_u64) + red_add_relaxed_sys_u64_raw(ercs_peer_addr, status_u64) + cute.arch.fence_acq_rel_sys() cute.arch.barrier( barrier_id=self.dispatch_intra_cta_bar_id, number_of_threads=self.num_dispatch_threads, @@ -485,7 +735,6 @@ def dispatch_barrier( sm_idx, warp_idx, lane_idx, - slot=0, num_sms=num_sms, prologue_grid_sync=False, epilogue_grid_sync=True, @@ -530,6 +779,19 @@ def dispatch_pull( # SF rows use their own padding; token and SF pool offsets can diverge. expert_sf_pool_block_offset = Int32(0) + # ── Release-flag batching ──────────────────────────────────────── + # Delay fc1-ready counter publication with the same rotating-lane + # tracker used by the epilogue. Each token's TMA store to the FC1 pool + # is drained CTA-locally by ``cp_async_bulk_wait_group(0)`` before its + # release target is accumulated; the eventual red.release.gpu add + # publishes the corresponding pool data to GPU scope. + flag_tracker = GpuReleaseFlagBatchTracker( + flag_addr=Int64(0), + cumulated_flags=Int32(0), + phase=Int32(0), + tid=lane_idx, + ) + stored_rank_count_lane = Int32(0) NUM_EXPERTS_PER_LANE: cutlass.Constexpr[int] = ( @@ -549,8 +811,8 @@ def dispatch_pull( int] = num_sms * self.num_dispatch_warps token_idx = sm_idx * Int32(self.num_dispatch_warps) + warp_idx - _iket_pull_emit = (sm_idx == Int32(0)) and (warp_idx == Int32(0)) and ( - lane_idx == Int32(0)) + _iket_pull_emit = ((sm_idx == Int32(0)) and (warp_idx == Int32(0)) + and (lane_idx == Int32(0))) while current_expert_idx < Int32(self.num_experts_per_rank): if _iket_pull_emit: @@ -562,17 +824,20 @@ def dispatch_pull( prev_block_count = (prev_valid_count + Int32(self.token_padding_block) - Int32(1)) // Int32(self.token_padding_block) - expert_pool_block_offset = expert_pool_block_offset + prev_block_count + expert_pool_block_offset = (expert_pool_block_offset + + prev_block_count) # Mirror cumul for the release-counter granularity (self.cluster_tile_tokens). prev_task_tile_count = ( prev_valid_count + Int32(self.cluster_tile_tokens) - Int32(1)) // Int32(self.cluster_tile_tokens) - expert_task_tile_offset = expert_task_tile_offset + prev_task_tile_count + expert_task_tile_offset = (expert_task_tile_offset + + prev_task_tile_count) # Mirror cumul for the SF axis granularity (self.sf_padding_block). prev_sf_block_count = (prev_valid_count + Int32(self.sf_padding_block) - Int32(1)) // Int32(self.sf_padding_block) - expert_sf_pool_block_offset = expert_sf_pool_block_offset + prev_sf_block_count + expert_sf_pool_block_offset = (expert_sf_pool_block_offset + + prev_sf_block_count) current_expert_idx = current_expert_idx + Int32(1) if current_expert_idx < Int32(self.num_experts_per_rank): expert_start_idx = expert_end_idx @@ -744,13 +1009,13 @@ def dispatch_pull( ) with cute.arch.elect_one(): - _store_token_src_metadata_u32x3( - token_src_metadata, - pool_token_idx, - Uint32(current_rank_in_expert_idx), - Uint32(src_token), - Uint32(src_topk), - ) + TokenSrcMetadata( + src_rank=current_rank_in_expert_idx, + src_token=src_token, + src_topk=src_topk, + ).store(token_src_metadata.iterator + + Int64(pool_token_idx) * + Int64(TokenSrcMetadata.nbytes)) with cute.arch.elect_one(): cute.arch.cp_async_bulk_commit_group() @@ -760,15 +1025,21 @@ def dispatch_pull( _iket.range_pop() # Pull.TMA_Store _iket.range_push("Pull.Arrival_Atomic") - with cute.arch.elect_one(): - task_tile_idx = expert_task_tile_offset + ( - token_idx_in_expert // Int32(self.cluster_tile_tokens)) - cute.arch.atomic_add( - fc1_ready_counter.iterator + task_tile_idx, - Int32(1), - sem="release", - scope="gpu", - ) + # Accumulate this token's release target into the rotating-lane + # batch tracker. task_tile_idx is warp-uniform (token_idx / + # expert offsets are warp-wide), so every lane runs the same + # state-machine transition while only one lane records the + # current address. + task_tile_idx = expert_task_tile_offset + ( + token_idx_in_expert // Int32(self.cluster_tile_tokens)) + + task_tile_addr = (fc1_ready_counter.iterator + + task_tile_idx).toint() + flag_tracker = flag_tracker.accumulate( + Int32(0), + self._flag_batch, + task_tile_addr, + ) cute.arch.sync_warp() if _iket_pull_emit: @@ -778,16 +1049,54 @@ def dispatch_pull( token_idx = token_idx + Int32(num_global_warps) + # Tail flush: publish any leftover (< self._flag_batch) accumulated release. + flag_tracker.fire() + cute.arch.sync_warp() + return phase_bit, stored_num_tokens_per_expert + @cute.jit + def _adaptive_pace( + self, + avg, + current_window, + *, + lo: cutlass.Constexpr[int], + hi: cutlass.Constexpr[int], + ): + # NVLink pacing: EMA the measured round-trip and nanosleep the deviation + # so outstanding NVLink requests stay bounded and don't head-of-line + # block this SM's non-NVLink (local) load/store traffic. + if current_window > avg: + avg = avg + ((current_window - avg + Int32(3)) // Int32(4)) + sleep_cycle = current_window - avg + if sleep_cycle > Int32(hi): + sleep_cycle = Int32(hi) + if sleep_cycle > Int32(50): + _nanosleep(sleep_cycle) + else: + avg = avg - ((avg - current_window + Int32(3)) // Int32(4)) + sleep_cycle = avg - current_window + if sleep_cycle > Int32(50): + _nanosleep(sleep_cycle) + if avg > Int32(hi): + avg = Int32(hi) + if avg < Int32(lo): + avg = Int32(lo) + return avg + @cute.jit def token_back_by_push( self, - token_comm_storage, + pull_buffer_ptr, + pull_mbar_ptr, fc2_output_workspace, fc2_done_counter, token_src_metadata, combine_output, + combine_sf, + fc2_output_sf, + token_back_schedule_counter, peer_rank_ptr_mapper, phase_bit, stored_num_tokens_per_expert, @@ -795,25 +1104,78 @@ def token_back_by_push( warp_idx, lane_idx, *, + local_rank, num_sms, + chunk_bytes: cutlass.Constexpr[int], ): _iket_emit = (sm_idx == Int32(0)) and (warp_idx == Int32(0)) + avg_token_back_window = Int32(2500) - chunk_bytes: cutlass.Constexpr[int] = self.hidden_bytes - num_chunks: cutlass.Constexpr[int] = self.fc2_num_chunks + # Chunk the fc2 token in ``chunk_bytes`` pieces; the last piece carries + # the remainder so any chunk_bytes works for any fc2_token_bytes. fc2_token_bytes: cutlass.Constexpr[int] = self.fc2_token_bytes - - pull_buffer_ptr = token_comm_storage.pull_buffer.data_ptr() - pull_mbar_ptr = token_comm_storage.pull_mbar.data_ptr() + num_chunks: cutlass.Constexpr[int] = (fc2_token_bytes + chunk_bytes - + 1) // chunk_bytes + last_chunk_bytes: cutlass.Constexpr[int] = ( + fc2_token_bytes - (num_chunks - 1) * chunk_bytes) + + if cutlass.const_expr(self.push_sf): + # (token, topk, hidden):(d_topkxhidden, d_hidden, 1) + combine_sf_u8 = cute.recast_tensor(combine_sf, Uint8) + sf_token_bytes: cutlass.Constexpr[int] = cute.size( + combine_sf_u8[0, None, 0].stride) + num_sf_chunks: cutlass.Constexpr[int] = ( + sf_token_bytes + chunk_bytes - 1) // chunk_bytes + last_sf_chunk_bytes: cutlass.Constexpr[int] = ( + sf_token_bytes - (num_sf_chunks - 1) * chunk_bytes) num_experts_per_lane: cutlass.Constexpr[int] = ( self.num_experts_per_rank + 31) // 32 - num_global_warps: cutlass.Constexpr[ - int] = num_sms * self.num_dispatch_warps + num_global_warps: cutlass.Constexpr[int] = (num_sms * + self.num_dispatch_warps) + schedule_mode = self.token_back_schedule_mode + + # static: stride by the global warp count. atomic_counter: claim the + # next token via one grid-scoped atomicAdd(1) so fast warps keep + # stealing work. cuTeDSL forbids closures over enclosing locals -> + # pass all in. + def update_token_idx( + token_idx, + lane_idx, + schedule_counter, + schedule_mode, + num_global_warps, + ): + if cutlass.const_expr(schedule_mode == "atomic_counter"): + base = Int32(0) + if lane_idx == Int32(0): + base = cute.arch.atomic_add( + schedule_counter, + Int32(1), + sem="relaxed", + scope="gpu", + ) + token_idx = cute.arch.shuffle_sync(base, Int32(0)) + else: + token_idx = token_idx + Int32(num_global_warps) + return token_idx - token_idx = sm_idx * Int32(self.num_dispatch_warps) + warp_idx + if cutlass.const_expr(schedule_mode == "atomic_counter"): + # Claim the initial token. + token_idx = Int32(0) + token_idx = update_token_idx( + token_idx, + lane_idx, + token_back_schedule_counter, + schedule_mode, + num_global_warps, + ) + else: + token_idx = sm_idx * Int32(self.num_dispatch_warps) + warp_idx current_expert_idx = Int32(-1) + confirmed_expert_idx = Int32(-1) + cur_expert_expected = Int32(0) expert_start_idx = Int32(0) expert_end_idx = Int32(0) expert_pool_block_offset = Int32(0) @@ -825,7 +1187,8 @@ def token_back_by_push( prev_block_count = (prev_valid_count + Int32(self.token_padding_block) - Int32(1)) // Int32(self.token_padding_block) - expert_pool_block_offset = expert_pool_block_offset + prev_block_count + expert_pool_block_offset = (expert_pool_block_offset + + prev_block_count) current_expert_idx = current_expert_idx + Int32(1) if current_expert_idx < Int32(self.num_experts_per_rank): @@ -845,81 +1208,164 @@ def token_back_by_push( cluster_tile_cnt = ( total_for_expert + Int32(self.cluster_tile_tokens) - Int32(1)) // Int32(self.cluster_tile_tokens) - expected = cluster_tile_cnt * Int32( + # Stash the threshold; the wait is deferred to the expert we + # actually land on, so stepped-over experts are never waited. + cur_expert_expected = cluster_tile_cnt * Int32( self.fc2_publishes_per_token_cluster_tile) + + if current_expert_idx < Int32(self.num_experts_per_rank): + # Wait once per processed expert (both indices monotonic; fc2 + # completes in expert order so confirming k implies all < k). + if current_expert_idx > confirmed_expert_idx: spin_wait( fc2_done_counter.iterator + current_expert_idx, - lambda v: v >= expected, + lambda v: v >= cur_expert_expected, fail_sleep_cycles=500, ) + confirmed_expert_idx = current_expert_idx - if current_expert_idx < Int32(self.num_experts_per_rank): + remain_experts = Int32( + self.num_experts_per_rank) - current_expert_idx token_idx_in_expert = token_idx - expert_start_idx pool_token_idx = ( expert_pool_block_offset * Int32(self.token_padding_block) + token_idx_in_expert) - md_base = token_src_metadata.iterator + (pool_token_idx * - Int32(12)) - src_rank = Int32( - cute.arch.load(md_base + Int32(0), Int32, scope="gpu")) - src_token = Int32( - cute.arch.load(md_base + Int32(4), Int32, scope="gpu")) - src_topk = Int32( - cute.arch.load(md_base + Int32(8), Int32, scope="gpu")) - - local_token_addr = fc2_output_workspace.iterator.toint( - ) + Int64(pool_token_idx) * Int64(fc2_token_bytes) - peer_combine_ptr = peer_rank_ptr_mapper.ptr_map_to_rank( - combine_output.iterator, - src_rank, - ) - peer_token_ptr = peer_combine_ptr + ( - Int64(src_token * Int32(self.num_topk) + src_topk) * - Int64(fc2_token_bytes)) + md = TokenSrcMetadata.load(token_src_metadata.iterator + + Int64(pool_token_idx) * + Int64(TokenSrcMetadata.nbytes)) + src_rank = md.src_rank + src_token = md.src_token + src_topk = md.src_topk + is_remote_token_back = src_rank != Int32(local_rank) smem_ptr_warp = pull_buffer_ptr + warp_idx * Int32(chunk_bytes) mbar_ptr_warp = pull_mbar_ptr + warp_idx if _iket_emit: _iket.range_push("token_back") + cute.arch.sync_warp() - for chunk in cutlass.range_constexpr(0, num_chunks, 1): - chunk_off = Int64(chunk * chunk_bytes) - # chunk_t0 = read_clock64() - - with cute.arch.elect_one(): - tma_load_1d_raw( - smem_ptr_warp, - local_token_addr + chunk_off, - mbar_ptr_warp, - Int32(chunk_bytes), - ) - cute.arch.mbarrier_arrive_and_expect_tx( - mbar_ptr_warp, - Int32(chunk_bytes), - ) - cute.arch.mbarrier_wait(mbar_ptr_warp, phase_bit) - cute.arch.sync_warp() - - with cute.arch.elect_one(): - tma_store_1d( - peer_token_ptr + chunk_off, - smem_ptr_warp, - Int32(chunk_bytes), - ) + # DATA plane: only the dispatch DATA path pushes here; epi_warps + # has the epilogue STG/UBLK the data straight to the peer. + if cutlass.const_expr(self.push_data): + local_token_addr = ( + fc2_output_workspace.iterator.toint() + + Int64(pool_token_idx) * Int64(fc2_token_bytes)) + peer_combine_ptr = peer_rank_ptr_mapper.ptr_map_to_rank( + combine_output.iterator, + src_rank, + ) + if cutlass.const_expr(self.token_back_reduce_topk): + peer_token_offset = Int64(src_token) * Int64( + fc2_token_bytes) + else: + peer_token_offset = ( + Int64(src_token * Int32(self.num_topk) + src_topk) * + Int64(fc2_token_bytes)) + peer_token_ptr = peer_combine_ptr + peer_token_offset + + for chunk in cutlass.range(num_chunks, unroll=1): + t0 = read_clock64() + chunk_off = Int64(chunk * chunk_bytes) + peer_chunk_ptr = peer_token_ptr + chunk_off + + this_bytes = Int32(chunk_bytes) + if cutlass.const_expr(last_chunk_bytes != chunk_bytes): + if chunk == Int32(num_chunks - 1): + this_bytes = Int32(last_chunk_bytes) + + with cute.arch.elect_one(): + tma_load_1d_raw( + smem_ptr_warp, + local_token_addr + chunk_off, + mbar_ptr_warp, + this_bytes, + ) + cute.arch.mbarrier_arrive_and_expect_tx( + mbar_ptr_warp, + this_bytes, + ) + cute.arch.mbarrier_wait(mbar_ptr_warp, phase_bit) + if cutlass.const_expr(self.token_back_reduce_topk): + cp_reduce_async_bulk_add_noftz_bf16_s2g( + peer_chunk_ptr, + smem_ptr_warp, + this_bytes, + ) + else: + tma_store_1d( + peer_chunk_ptr, + smem_ptr_warp, + this_bytes, + ) + phase_bit = phase_bit ^ Int32(1) cute.arch.cp_async_bulk_commit_group() cute.arch.cp_async_bulk_wait_group(0) - cute.arch.sync_warp() - - # if read_clock64() - chunk_t0 < Int64(600): - # _nanosleep(100) + t1 = read_clock64() + current_window = Int32(t1 - t0) + if is_remote_token_back and remain_experts > Int32(4): + avg_token_back_window = self._adaptive_pace( + avg_token_back_window, + current_window, + lo=1000, + hi=5000, + ) + + if cutlass.const_expr(self.push_sf): + # Int64 like the DATA/metadata paths above: the Int32 + # row index would overflow the multi-GiB SF pool offset + # at large max_tokens_per_rank x EP. + sf_local_addr = fc2_output_sf[Int64(pool_token_idx), 0, + None].iterator.toint() + sf_peer_ptr = peer_rank_ptr_mapper.ptr_map_to_rank( + combine_sf_u8[Int64(src_token), src_topk, + None].iterator, + src_rank, + ) + for chunk in cutlass.range(num_sf_chunks, unroll=1): + t0 = read_clock64() + chunk_off = Int64(chunk * chunk_bytes) + this_bytes = Int32(chunk_bytes) + if cutlass.const_expr( + last_sf_chunk_bytes != chunk_bytes): + if chunk == Int32(num_sf_chunks - 1): + this_bytes = Int32(last_sf_chunk_bytes) + with cute.arch.elect_one(): + tma_load_1d_raw( + smem_ptr_warp, + sf_local_addr + chunk_off, + mbar_ptr_warp, + this_bytes, + ) + cute.arch.mbarrier_arrive_and_expect_tx( + mbar_ptr_warp, + this_bytes, + ) + cute.arch.mbarrier_wait(mbar_ptr_warp, phase_bit) + tma_store_1d( + sf_peer_ptr + chunk_off, + smem_ptr_warp, + this_bytes, + ) + phase_bit = phase_bit ^ Int32(1) + cute.arch.cp_async_bulk_commit_group() + cute.arch.cp_async_bulk_wait_group(0) + if is_remote_token_back: + round_trip = Int32(read_clock64() - t0) + if round_trip <= this_bytes * Int32(5) // Int32(4): + _nanosleep(this_bytes // Int32(4)) - phase_bit = phase_bit ^ Int32(1) if _iket_emit: _iket.range_pop() - token_idx = token_idx + Int32(num_global_warps) + token_idx = update_token_idx( + token_idx, + lane_idx, + token_back_schedule_counter, + schedule_mode, + num_global_warps, + ) cute.arch.fence_acq_rel_sys() @@ -934,7 +1380,6 @@ def nvlink_barrier( warp_idx, lane_idx, *, - slot: cutlass.Constexpr[int], num_sms, prologue_grid_sync: cutlass.Constexpr[bool], epilogue_grid_sync: cutlass.Constexpr[bool], @@ -943,26 +1388,27 @@ def nvlink_barrier( tid_in_group = warp_idx * Int32(self.warp_threads) + lane_idx if prologue_grid_sync: - software_grid_sync( - grid_sync_counter, - sm_idx, - num_sms, - tid_in_group, - num_threads=self.num_dispatch_threads, - ) + software_grid_sync(grid_sync_counter, + sm_idx, + num_sms, + tid_in_group, + num_threads=self.num_dispatch_threads) if sm_idx == 0: if warp_idx == 0: - signal_phase = Int32(slot) + # Sense-reversing ping-pong barrier. The low 2 bits of the counter + # pick the signal slot (phase 0/1) and the direction (+1 up to + # world_size, then -1 back to 0), so the two slots self-cancel over + # a 4-call cycle and never need an explicit reset of the + # symmetric peer-memory signal. + status = nvlink_barrier_counter[0] & Int32(3) + signal_phase = status & Int32(1) + signal_sign = status >> Int32(1) signal_delta = Int32(1) target = Int32(self.world_size) - if cutlass.const_expr(nvlink_barrier_counter is not None): - status = nvlink_barrier_counter[0] & Int32(3) - signal_phase = status & Int32(1) - signal_sign = status >> Int32(1) - if signal_sign != Int32(0): - signal_delta = Int32(-1) - target = Int32(0) + if signal_sign != Int32(0): + signal_delta = Int32(-1) + target = Int32(0) nbs_local_base = nvlink_barrier_signal.iterator.toint() if lane_idx < Int32(self.world_size): @@ -975,35 +1421,25 @@ def nvlink_barrier( cute.arch.sync_warp() if lane_idx == 0: - if cutlass.const_expr(nvlink_barrier_counter is not None): - cute.arch.atomic_add( - nvlink_barrier_counter.iterator, - Int32(1), - sem="relaxed", - scope="gpu", - ) + cute.arch.atomic_add( + nvlink_barrier_counter.iterator, + Int32(1), + sem="relaxed", + scope="gpu", + ) local_signal_ptr = nvlink_barrier_signal.iterator + signal_phase - if cutlass.const_expr(nvlink_barrier_counter is None): - while (cute.arch.load(local_signal_ptr, - Int32, - sem="acquire", - scope="sys") < target): - pass - else: - while (cute.arch.load(local_signal_ptr, - Int32, - sem="acquire", - scope="sys") != target): - pass + while cute.arch.load(local_signal_ptr, + Int32, + sem="acquire", + scope="sys") != target: + pass if epilogue_grid_sync: - software_grid_sync( - grid_sync_counter, - sm_idx, - num_sms, - tid_in_group, - num_threads=self.num_dispatch_threads, - ) + software_grid_sync(grid_sync_counter, + sm_idx, + num_sms, + tid_in_group, + num_threads=self.num_dispatch_threads) @cute.jit def dispatch_warp_body( @@ -1036,6 +1472,7 @@ def dispatch_warp_body( cta_linear_id, local_warp_idx, lane_idx, + local_rank=token_comm_args.local_rank, num_tokens=token_comm_args.input_token_buffer.shape[0], num_sms=token_comm_args.sm_count, ) @@ -1054,6 +1491,7 @@ def dispatch_warp_body( cta_linear_id, local_warp_idx, lane_idx, + local_rank=token_comm_args.local_rank, num_sms=token_comm_args.sm_count, nvlink_barrier_counter=token_comm_args.nvlink_barrier_counter, ) @@ -1091,61 +1529,146 @@ def dispatch_warp_body( if iket_active: _iket.range_pop() - if cutlass.const_expr(self.enable_token_back): + if cutlass.const_expr(self.enable_token_back + and not self.token_back_standalone): if iket_active: _iket.range_push("Token_Back_By_Push") self.token_back_by_push( - token_comm_storage, + token_comm_storage.pull_buffer.data_ptr(), + token_comm_storage.pull_mbar.data_ptr(), token_comm_args.fc2_output_workspace, token_comm_args.fc2_done_counter, token_comm_args.token_src_metadata, token_comm_args.combine_output, + token_comm_args.combine_sf, + token_comm_args.fc2_output_sf, + token_comm_args.token_back_schedule_counter, token_comm_args.peer_rank_ptr_mapper, phase_bit, stored_num_tokens_per_expert, cta_linear_id, local_warp_idx, lane_idx, + local_rank=token_comm_args.local_rank, num_sms=token_comm_args.sm_count, + chunk_bytes=self.hidden_bytes, ) if iket_active: _iket.range_pop() @cute.jit - def tail_reset_shared_counters( + def token_back_warp_body( + self, + token_comm_args, + token_comm_storage, + *, + warp_idx, + lane_idx, + tidx, + ): + bidx, bidy, bidz = cute.arch.block_idx() + cta_linear_id = ( + Int32(bidx) + Int32(self.cluster_shape_mn[1]) * Int32(bidy) + + Int32(self.cluster_shape_mn[1] * self.cluster_shape_mn[0]) * + Int32(bidz)) + local_warp_idx = Int32(warp_idx) - Int32(self.token_back_warp_start) + + # Handshake: dispatch_barrier done => expert_recv_count_sum populated. + nb_dispatch_to_sched = pipeline.NamedBarrier( + barrier_id=self.dispatch_to_sched_named_barrier_id, + num_threads=self.dispatch_to_sched_threads, + ) + nb_dispatch_to_sched.arrive_and_wait() + + tb_pull_mbar_ptr = token_comm_storage.tb_pull_mbar.data_ptr() + tb_pull_buffer_ptr = token_comm_storage.tb_pull_buffer.data_ptr() + if lane_idx == Int32(0): + cute.arch.mbarrier_init(tb_pull_mbar_ptr + local_warp_idx, 1) + cute.arch.sync_warp() + + NUM_EXPERTS_PER_LANE: cutlass.Constexpr[int] = ( + self.num_experts_per_rank + 31) // 32 + stored_num_tokens_per_expert = [] + for _ in cutlass.range_constexpr(0, NUM_EXPERTS_PER_LANE, 1): + stored_num_tokens_per_expert.append(Int32(0)) + for i in cutlass.range_constexpr(0, NUM_EXPERTS_PER_LANE, 1): + e_idx_for_lane = Int32(i * self.warp_threads) + lane_idx + if e_idx_for_lane < Int32(self.num_experts_per_rank): + sum_packed_init = token_comm_args.expert_recv_count_sum[ + e_idx_for_lane] + stored_num_tokens_per_expert[i] = Int32( + Int64(sum_packed_init) & Int64(0xFFFFFFFF)) + cute.arch.sync_warp() + + iket_active = (cta_linear_id == Int32(0)) and (local_warp_idx + == Int32(0)) + if iket_active: + _iket.range_push("Token_Back_By_Push_Standalone") + + self.token_back_by_push( + tb_pull_buffer_ptr, + tb_pull_mbar_ptr, + token_comm_args.fc2_output_workspace, + token_comm_args.fc2_done_counter, + token_comm_args.token_src_metadata, + token_comm_args.combine_output, + token_comm_args.combine_sf, + token_comm_args.fc2_output_sf, + token_comm_args.token_back_schedule_counter, + token_comm_args.peer_rank_ptr_mapper, + Int32(0), + stored_num_tokens_per_expert, + cta_linear_id, + local_warp_idx, + lane_idx, + local_rank=token_comm_args.local_rank, + num_sms=token_comm_args.sm_count, + chunk_bytes=self.tb_chunk_bytes, + ) + + if iket_active: + _iket.range_pop() + + @cute.jit + def tail_reset_counters( self, token_comm_args, + target_zero_tensor, *, cta_linear_id, local_warp_idx, lane_idx, ): - thread_linear = (cta_linear_id * Int32(self.num_dispatch_warps) + - local_warp_idx) * Int32(self.warp_threads) + lane_idx + """Per-lane 4B (Int32) bulk-zero of one accumulating-counter prefix. + + ``target_zero_tensor`` is an Int32 view over a workspace's front counter + region (megamoe_kernel front-places every counter that must restart at 0 + each launch; data buffers and the phase-flip ``nvlink_barrier_signal`` sit + after the prefix and are untouched). The zeroing is spread across all + dispatch threads grid-wide. kernel_tail calls this twice: + * SHARED prefix (expert_recv_count / _sum) BETWEEN the two nvlink + barriers, so the final barrier publishes the zeros cross-rank -- needed + when the next launch is another MegaMoE reusing the shared workspace + with no intervening rank sync; + * LOCAL prefix (l1_arrival / expert_send / grid_sync / nvlink_barrier / + fc1_done [+ fc2_done / token_back_schedule / load_balance]) AFTER the + last barrier -- rank-local (next kernel sees it via stream order) and + grid_sync/nvlink barrier counters stay live until that last barrier. + Only the FIRST launch relies on a caller-zeroed workspace. + """ + thread_linear = ( + (cta_linear_id * Int32(self.num_dispatch_warps) + local_warp_idx) * + Int32(self.warp_threads) + lane_idx) stride = Int32(token_comm_args.sm_count * self.num_dispatch_threads) - recv_total: cutlass.Constexpr[ - int] = self.world_size * self.num_experts_per_rank - i = thread_linear - while i < Int32(recv_total): - rank_idx = i // Int32(self.num_experts_per_rank) - expert_idx = i % Int32(self.num_experts_per_rank) - token_comm_args.expert_recv_count[rank_idx, expert_idx] = Int64(0) - i = i + stride - + count = cute.size(target_zero_tensor) i = thread_linear - while i < Int32(self.num_experts_per_rank): - token_comm_args.expert_recv_count_sum[i] = Int64(0) + while i < Int32(count): + target_zero_tensor[i] = Int32(0) i = i + stride - if cutlass.const_expr(self.enable_token_back): - i = thread_linear - while i < Int32(self.num_experts_per_rank): - token_comm_args.fc2_done_counter[i] = Int32(0) - i = i + stride - @cute.jit def kernel_tail( self, @@ -1161,13 +1684,21 @@ def kernel_tail( ) nb_kernel_tail.arrive_and_wait() - if warp_idx >= self.dispatch_warp_start: + # Only the dispatch warps run NVLink cleanup; standalone token-back + # warps (>= token_back_warp_start) just join the rendezvous above. + if (warp_idx >= self.dispatch_warp_start) and ( + warp_idx < self.dispatch_warp_start + self.num_dispatch_warps): bidx, bidy, bidz = cute.arch.block_idx() cta_linear_id = ( Int32(bidx) + Int32(self.cluster_shape_mn[1]) * Int32(bidy) + Int32(self.cluster_shape_mn[1] * self.cluster_shape_mn[0]) * Int32(bidz)) local_warp_idx = Int32(warp_idx) - Int32(self.dispatch_warp_start) + # Per-launch nvlink barrier count is 3: 1 (dispatch_barrier) + 2 + # below (drain + publish, around the shared reset). The + # sense-reversing signal rides the phase counter across launch + # boundaries, so the count does not need to self-cancel within a + # single launch. self.nvlink_barrier( token_comm_args.nvlink_barrier_signal, token_comm_args.nvlink_barrier_counter, @@ -1176,26 +1707,15 @@ def kernel_tail( cta_linear_id, local_warp_idx, lane_idx, - slot=1, - num_sms=token_comm_args.sm_count, - prologue_grid_sync=True, - epilogue_grid_sync=True, - ) - self.nvlink_barrier( - token_comm_args.nvlink_barrier_signal, - token_comm_args.nvlink_barrier_counter, - token_comm_args.grid_sync_counter, - token_comm_args.peer_rank_ptr_mapper, - cta_linear_id, - local_warp_idx, - lane_idx, - slot=1, num_sms=token_comm_args.sm_count, prologue_grid_sync=True, epilogue_grid_sync=True, ) - self.tail_reset_shared_counters( + # Shared counters between the barriers: the slot=0 barrier below + # publishes these zeros cross-rank for a back-to-back MegaMoE relaunch. + self.tail_reset_counters( token_comm_args, + token_comm_args.shared_zero_prefix, cta_linear_id=cta_linear_id, local_warp_idx=local_warp_idx, lane_idx=lane_idx, @@ -1208,8 +1728,16 @@ def kernel_tail( cta_linear_id, local_warp_idx, lane_idx, - slot=0, num_sms=token_comm_args.sm_count, prologue_grid_sync=True, epilogue_grid_sync=True, ) + # Local counters last: rank-local, and grid_sync/nvlink_barrier + # counters above stay live until this final barrier completes. + self.tail_reset_counters( + token_comm_args, + token_comm_args.local_zero_prefix, + cta_linear_id=cta_linear_id, + local_warp_idx=local_warp_idx, + lane_idx=lane_idx, + ) diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/topk_reduce.py b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/topk_reduce.py index b99f61b04de0..9976de3d2dab 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/topk_reduce.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/mega_moe_nvfp4/topk_reduce.py @@ -1,1651 +1,509 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Standalone CuTeDSL topk reduce kernel. - -Form A writes one BF16 fc2 output row per ``(token, topk)`` cell into -``combine_output`` with logical shape ``(T, K, H)``. This module provides the -device-side final reduce used by the default form-A path: - - BF16 (T, K, H) -> FP32 accumulate over K -> FP32/BF16 (T, H) - -It also supports an explicit MXFP8 input mode: - - FP8_E4M3 (T, K, H) + UE8M0 scale -> FP32 dequant/reduce -> BF16 (T, H) - -and an explicit NVFP4 input mode: - - FP4_E2M1 (T, K, H) + per-16 FP8 scale + per-128 FP32 scale - -> FP32 dequant/reduce -> FP32/BF16 (T, H) - -It intentionally does not touch dispatch metadata, peer pointer mapping, or -the fc2 epilogue STG path. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +"""Device combine reduce: collapse per-(token, topk) fc2 cells into one row. + +The combine step writes one fc2 output per ``(token, topk)`` cell; this reduces +over the topk axis into the token-centric ``(token, hidden)`` output. The wire +format is a :class:`~src.token_comm.CombineFormat`: + + bf16 -- no staging: bf16 terms reduced directly. + 32e4m3xe8m0 -- MXFP8: fp8 e4m3 data + per-32 e8m0 (power-of-2) scale. + 16e2m1xbf16 -- fp4 e2m1 data + per-16 bf16 amax (one level, no global); + dequant per element x = fp4 * (amax * (1 / 6)). + +Task partition: each worker owns one ``(token, hidden_tile)`` and loops topk; the +flat worker index decodes into ``(token_idx, hidden_tile_idx)`` via a constant +divide by ``hidden_tiles``. The per-block scale is broadcast to a logical +per-hidden view (stride 0) so it tiles by the same worker index as the data. The +activation load stays in the topk loop (too large to hoist); the small scale and +score loads are hoisted ahead of the loop when topk is small. """ from __future__ import annotations -import argparse -from typing import Optional, Tuple +from typing import ClassVar, Dict, Optional import cuda.bindings.driver as cuda import cutlass import cutlass.cute as cute -import cutlass.torch as cutlass_torch -import torch -from cutlass.cute.typing import AddressSpace -from cutlass.cutlass_dsl import Float32, Int32 - -DEFAULT_THREADS = 256 - -BF16_VECTOR_THREADS = 512 -BF16_HIDDEN_PER_THREAD = 8 -BF16_STORE_ELEMENTS_PER_256B = 16 - -MXFP8_VECTOR_THREADS = 128 -MXFP8_HIDDEN_PER_THREAD = 16 -MXFP8_SCALE_BLOCK_SIZE = 32 - -NVFP4_VECTOR_THREADS = 128 -NVFP4_HIDDEN_PER_THREAD = 32 -NVFP4_SFC_SCALE_BLOCK_SIZE = 16 -NVFP4_SFC_PACKED_BYTES = NVFP4_SFC_SCALE_BLOCK_SIZE // 2 -NVFP4_SFC_INPUT_BITS_PER_COPY = NVFP4_SFC_PACKED_BYTES * 8 -NVFP4_GLOBAL_SCALE_BLOCK_SIZE = 128 - -NVFP4_E2M1_MAX = 6.0 -FP8_E4M3FN_MAX = 448.0 - -_Fp4DecodeTable: torch.Tensor = torch.tensor( - [ - 0.0, - 0.5, - 1.0, - 1.5, - 2.0, - 3.0, - 4.0, - 6.0, - -0.0, - -0.5, - -1.0, - -1.5, - -2.0, - -3.0, - -4.0, - -6.0, - ], - dtype=torch.float32, -) - -_Fp4ValuesEvenFirst: torch.Tensor = torch.tensor( - [ - 0.0, - 1.0, - 2.0, - 4.0, - -0.0, - -1.0, - -2.0, - -4.0, - 0.5, - 1.5, - 3.0, - 6.0, - -0.5, - -1.5, - -3.0, - -6.0, - ], - dtype=torch.float32, -) +from cutlass._mlir.dialects import llvm +from cutlass.cutlass_dsl import Float32, Int32, T -_ReorderToNibble: torch.Tensor = torch.tensor( - [ - 0x0, - 0x2, - 0x4, - 0x6, - 0x8, - 0xA, - 0xC, - 0xE, - 0x1, - 0x3, - 0x5, - 0x7, - 0x9, - 0xB, - 0xD, - 0xF, - ], - dtype=torch.uint8, -) +from .megamoe_constants import Nvfp4E2M1RcpLimit +from .token_comm import CombineFormat +# --------------------------------------------------------------------------- +# fp4 (e2m1) -> fp32 register decode. +# +# Blackwell has no e2m1->f32 upconvert: the framework's ``term.load().to(f32)`` +# lowers to an ALU subnormal-normalization path (~60% DRAM SOL). The helper +# below forces a table-driven decode instead; ``e2m1_reg`` (N e2m1 codes, N % 8 +# == 0) is read as packed b32 words and N fp32 values are written into +# ``fp32_reg`` in code order. The 16 e2m1 values are exact in fp32. +# --------------------------------------------------------------------------- -def logical_io_bytes( - combine_output: torch.Tensor, - reduced_output: torch.Tensor, - topk_score: Optional[torch.Tensor] = None, - mxfp8_scale: Optional[torch.Tensor] = None, - nvfp4_sfc_scale: Optional[torch.Tensor] = None, - nvfp4_global_scale: Optional[torch.Tensor] = None, -) -> Tuple[int, int, int]: - """Return logical read, write and total bytes for one topk reduce pass.""" - read_bytes = combine_output.numel() * combine_output.element_size() - if topk_score is not None: - read_bytes += topk_score.numel() * topk_score.element_size() - if mxfp8_scale is not None: - read_bytes += mxfp8_scale.numel() * mxfp8_scale.element_size() - if nvfp4_sfc_scale is not None: - read_bytes += nvfp4_sfc_scale.numel() * nvfp4_sfc_scale.element_size() - if nvfp4_global_scale is not None: - read_bytes += nvfp4_global_scale.numel( - ) * nvfp4_global_scale.element_size() - write_bytes = reduced_output.numel() * reduced_output.element_size() - return int(read_bytes), int(write_bytes), int(read_bytes + write_bytes) +@cute.jit +def cvt_e2m1_to_fp32_cvt_ptx(e2m1_reg: cute.Tensor, + fp32_reg: cute.Tensor) -> None: + """Decode via the e2m1->f16 HW cvt (``cvt.rn.f16x2.e2m1x2``) then widen f16->f32. -def bandwidth_gbps(num_bytes: int, elapsed_ms: float) -> float: - if elapsed_ms <= 0.0: - return float("inf") - return float(num_bytes) / (elapsed_ms * 1.0e6) - - -def make_mxfp8_input( - src: torch.Tensor, - *, - scale_rank: int = 3, -) -> tuple[torch.Tensor, torch.Tensor]: - """Quantize FP32 ``src`` to MXFP8 data plus UE8M0 dequant scale.""" - if src.dim() != 3: - raise ValueError( - f"src must have shape (T, K, H), got {tuple(src.shape)}.") - if src.dtype != torch.float32: - raise TypeError(f"src must be torch.float32, got {src.dtype}.") - if not src.is_cuda: - raise ValueError("src must be a CUDA tensor.") - - T, K, H = src.shape - block = MXFP8_SCALE_BLOCK_SIZE - scale_cols = (H + block - 1) // block - padded_abs = torch.zeros( - (T, K, scale_cols * block), - device=src.device, - dtype=torch.float32, - ) - padded_abs[:, :, :H] = src.abs() - amax = padded_abs.reshape(T, K, scale_cols, block).amax(dim=-1) - if scale_rank == 2: - scale_f32 = amax.amax(dim=1) / 448.0 - scale_for_q = scale_f32[:, None, :] - elif scale_rank == 3: - scale_f32 = amax / 448.0 - scale_for_q = scale_f32 - else: - raise ValueError(f"scale_rank must be 2 or 3, got {scale_rank}.") - - def _round_up_to_power_of_two(scale: torch.Tensor) -> torch.Tensor: - return torch.pow( - torch.full_like(scale, 2.0), - torch.ceil(torch.log2(torch.clamp(scale, min=2.0**-30))), - ) - - scale_f32 = _round_up_to_power_of_two(scale_f32) - scale_for_q = _round_up_to_power_of_two(scale_for_q) - expanded_scale = scale_for_q.repeat_interleave(block, dim=-1)[:, :, :H] - q = (src / expanded_scale).to(torch.float8_e4m3fn) - return q, scale_f32.to(torch.float8_e8m0fnu) - - -def _pack_f32_to_fp4(fp32: torch.Tensor) -> torch.Tensor: - """Round FP32 to FP4 E2M1 and pack pairs along the last dimension.""" - if fp32.dim() == 0 or fp32.shape[-1] % 2 != 0: - raise ValueError( - f"FP4 packing requires an even non-empty last dim, got {tuple(fp32.shape)}." - ) - device = fp32.device - boundaries = torch.tensor( - [ - -5.0, - -3.5, - -2.5, - -1.75, - -1.25, - -0.75, - -0.25, - 0.25, - 0.75, - 1.25, - 1.75, - 2.5, - 3.5, - 5.0, - ], - device=device, - dtype=fp32.dtype, - ) - bucket_to_nibble = torch.tensor( - [ - 0xF, - 0xE, - 0xD, - 0xC, - 0xB, - 0xA, - 0x9, - 0x0, - 0x1, - 0x2, - 0x3, - 0x4, - 0x5, - 0x6, - 0x7, - ], - device=device, - dtype=torch.uint8, - ) - bucket = torch.bucketize(fp32.contiguous(), boundaries) - indices = bucket_to_nibble[bucket] - lo = indices[..., 0::2] - hi = indices[..., 1::2] - return ((hi << 4) | lo).contiguous() - - -def unpack_fp4_to_f32(packed: torch.Tensor) -> torch.Tensor: - """Unpack a last-dim-packed FP4 tensor or uint8 byte tensor to FP32.""" - if packed.dtype == torch.uint8: - raw = packed - elif hasattr(torch, - "float4_e2m1fn_x2") and packed.dtype == torch.float4_e2m1fn_x2: - raw = packed.view(torch.uint8) - else: - raise TypeError( - f"packed must be torch.uint8 or torch.float4_e2m1fn_x2, got {packed.dtype}." - ) - lo = (raw & 0x0F).to(torch.int64) - hi = (raw >> 4).to(torch.int64) - lut = _Fp4DecodeTable.to(raw.device) - unpacked_shape = list(raw.shape) - unpacked_shape[-1] *= 2 - unpacked = torch.empty(unpacked_shape, - dtype=torch.float32, - device=raw.device) - unpacked[..., 0::2] = lut[lo] - unpacked[..., 1::2] = lut[hi] - return unpacked - - -def make_nvfp4_input( - src: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Quantize FP32 ``src`` to NVFP4 plus per-16 FP8 and per-128 FP32 scales. - - The returned scales are dequant scales along hidden: - ``x_hat = fp4 * sfc_fp8 * global_fp32``. + Safe baseline: the per-pair ``cvt`` instruction is itself a HW PRMT+F2FP, so + the e2m1->f16 step already avoids ALU normalization; f16->f32 is one cheap + ``cvt.f32.f16`` per element. """ - if not hasattr(torch, "float8_e4m3fn"): - raise TypeError("NVFP4 mode requires torch float8_e4m3fn.") - if src.dim() != 3: - raise ValueError( - f"src must have shape (T, K, H), got {tuple(src.shape)}.") - if src.dtype != torch.float32: - raise TypeError(f"src must be torch.float32, got {src.dtype}.") - if not src.is_cuda: - raise ValueError("src must be a CUDA tensor.") - if src.shape[-1] % 2 != 0: - raise ValueError( - f"NVFP4 input hidden must be even for fp4x2 packing, got {src.shape[-1]}." + src_words = cute.recast_tensor(e2m1_reg, Int32) # (N,) e2m1 -> (N/8,) b32 + for w in cutlass.range_constexpr(cute.size(src_words)): + res = llvm.inline_asm( + llvm.StructType.get_literal([T.f32()] * 8), + [src_words[w].ir_value()], + "{\n" + " .reg .b8 b0, b1, b2, b3;\n" + " .reg .b32 p0, p1, p2, p3;\n" + " .reg .b16 c0, d0, c1, d1, c2, d2, c3, d3;\n" + " mov.b32 {b0, b1, b2, b3}, $8;\n" + " cvt.rn.f16x2.e2m1x2 p0, b0;\n" + " cvt.rn.f16x2.e2m1x2 p1, b1;\n" + " cvt.rn.f16x2.e2m1x2 p2, b2;\n" + " cvt.rn.f16x2.e2m1x2 p3, b3;\n" + " mov.b32 {c0, d0}, p0;\n" + " mov.b32 {c1, d1}, p1;\n" + " mov.b32 {c2, d2}, p2;\n" + " mov.b32 {c3, d3}, p3;\n" + " cvt.f32.f16 $0, c0;\n" + " cvt.f32.f16 $1, d0;\n" + " cvt.f32.f16 $2, c1;\n" + " cvt.f32.f16 $3, d1;\n" + " cvt.f32.f16 $4, c2;\n" + " cvt.f32.f16 $5, d2;\n" + " cvt.f32.f16 $6, c3;\n" + " cvt.f32.f16 $7, d3;\n" + "}", + "=f,=f,=f,=f,=f,=f,=f,=f,r", + has_side_effects=False, ) + for i in cutlass.range_constexpr(8): + fp32_reg[w * 8 + i] = Float32(llvm.extractvalue(T.f32(), res, [i])) - T, K, H = src.shape - sfc_block = NVFP4_SFC_SCALE_BLOCK_SIZE - global_block = NVFP4_GLOBAL_SCALE_BLOCK_SIZE - sfc_cols = (H + sfc_block - 1) // sfc_block - global_cols = (H + global_block - 1) // global_block - - padded_abs_sfc = torch.zeros( - (T, K, sfc_cols * sfc_block), - device=src.device, - dtype=torch.float32, - ) - padded_abs_sfc[:, :, :H] = src.abs() - amax16 = padded_abs_sfc.reshape(T, K, sfc_cols, sfc_block).amax(dim=-1) - padded_abs_global = torch.zeros( - (T, K, global_cols * global_block), - device=src.device, - dtype=torch.float32, - ) - padded_abs_global[:, :, :H] = src.abs() - amax128 = padded_abs_global.reshape(T, K, global_cols, - global_block).amax(dim=-1) +class TopkReduce: + """Combine reduce for a fixed ``(hidden, num_topk, combine_format)``. - global_scale = torch.clamp( - amax128 / (NVFP4_E2M1_MAX * FP8_E4M3FN_MAX), - min=2.0**-16, - ) - global_for_sfc = global_scale.repeat_interleave( - global_block // sfc_block, - dim=-1, - )[:, :, :sfc_cols] - sfc_fp32 = amax16 / (NVFP4_E2M1_MAX * global_for_sfc) - sfc_fp32 = torch.clamp(sfc_fp32, min=2.0**-16, max=FP8_E4M3FN_MAX) - sfc_fp8 = sfc_fp32.to(torch.float8_e4m3fn) - sfc_rt = sfc_fp8.to(torch.float32) + ``__init__`` pins the static shape and format (and the derived launch + geometry); ``__call__`` (a ``@cute.jit`` launcher) sizes a 1D grid from the + runtime token count and dispatches the format's kernel. The caller owns the + torch->cute conversion and the ``cute.compile``. + """ - expanded_sfc = sfc_rt.repeat_interleave(sfc_block, dim=-1)[:, :, :H] - expanded_global = global_scale.repeat_interleave(global_block, - dim=-1)[:, :, :H] - q = _pack_f32_to_fp4(src / (expanded_sfc * expanded_global)) - return q, sfc_fp8, global_scale + _threads: ClassVar[int] = 128 + # combine_format.name -> hidden elements per worker (one LDG of data: + # bf16 8*2B=16B, e4m3 16*1B=16B, e2m1 16*0.5B=8B). For quantized formats this + # stays <= the scale block, so each worker reads exactly one scale entry. + _hidden_per_thread: ClassVar[Dict[str, int]] = { + "bf16": 8, + "32e4m3xe8m0": 16, + "16e2m1xbf16": 16, + } + # topk count at/below which the scale + score loads are hoisted ahead of the + # topk loop (small enough to not bloat registers; a CTA-broadcast read). + _prefetch_limit: ClassVar[int] = 16 + + def __init__(self, hidden: int, num_topk: int, + combine_format: CombineFormat) -> None: + self.hidden = int(hidden) + self.num_topk = int(num_topk) + self.combine_format = combine_format + self.hidden_per_thread = self._hidden_per_thread[combine_format.name] + # hidden must tile cleanly both into worker slices and into scale blocks. + align = max(combine_format.scale_block or self.hidden_per_thread, + self.hidden_per_thread) + if self.hidden % align != 0: + raise ValueError( + f"hidden ({self.hidden}) must be divisible by max(scale_block, " + f"hidden_per_thread) = {align} for combine_format {combine_format}." + ) + self.hidden_tiles = self.hidden // self.hidden_per_thread + # tail guard only needed when the worker count per token is not a whole + # number of CTAs; prefetch only when topk is small enough to hoist. + self.require_predicate = self.hidden_tiles % self._threads != 0 + self.prefetch = self.num_topk <= self._prefetch_limit + # -- launcher ------------------------------------------------------------- -def mxfp8_reference_sum( - q: torch.Tensor, - scale: torch.Tensor, - topk_score: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """Return K-ordered FP32 reduce of MXFP8 input after dequantization.""" - T, K, H = q.shape - block = MXFP8_SCALE_BLOCK_SIZE - if scale.dim() == 2: - scale_for_q = scale.to(torch.float32)[:, None, :] - else: - scale_for_q = scale.to(torch.float32) - expanded_scale = scale_for_q.repeat_interleave(block, dim=-1)[:, :, :H] - dequant = q.to(torch.float32) * expanded_scale - acc = torch.zeros((T, H), device=q.device, dtype=torch.float32) - for k in range(K): - contrib = dequant[:, k, :] - if topk_score is not None: - acc = torch.addcmul(acc, contrib, topk_score[:, k, None]) + @cute.jit + def __call__( + self, + combine_quant: cute.Tensor, # (token, topk, hidden) + combine_sf: Optional[cute.Tensor], # (token, topk, hidden) + reduced_output: cute.Tensor, # (token, hidden) + topk_score: Optional[cute.Tensor], # (token, topk) + stream: cuda.CUstream, + ): + threads = self._threads + total_workers = reduced_output.shape[0] * self.hidden_tiles + grid = [(total_workers + threads - 1) // threads, 1, 1] + block = [threads, 1, 1] + + combine_quant = cute.make_tensor( + combine_quant.iterator, + cute.make_layout( + (combine_quant.shape[0], self.num_topk, self.hidden), + stride=combine_quant.stride)) + reduced_output = cute.make_tensor( + reduced_output.iterator, + cute.make_layout((reduced_output.shape[0], self.hidden), + stride=reduced_output.stride)) + if cutlass.const_expr(topk_score is not None): + topk_score = cute.make_tensor( + topk_score.iterator, + cute.make_layout((topk_score.shape[0], self.num_topk), + stride=topk_score.stride)) + + if cutlass.const_expr(not self.combine_format.is_quantized): + self._reduce_bf16(combine_quant, topk_score, reduced_output).launch( + grid=grid, + block=block, + stream=stream, + ) + return + + # The mega kernel hands sf in already as the depth-2 broadcast layout; a + # plain (torch) sf is depth-1 and gets its hidden mode split into + # (sf_vec, hidden/sf_vec):(0, s_h) so logical hidden h reads block h//sf_vec. + sf_vec = self.combine_format.scale_block + if cutlass.const_expr(cute.depth(combine_sf.layout) >= 2): + sf = cute.make_tensor( + combine_sf.iterator, + cute.make_layout((combine_sf.shape[0], self.num_topk, + (sf_vec, self.hidden // sf_vec)), + stride=combine_sf.stride)) else: - acc = acc + contrib - return acc - - -def nvfp4_reference_sum( - q: torch.Tensor, - sfc_scale: torch.Tensor, - global_scale: torch.Tensor, - topk_score: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """Return K-ordered FP32 reduce of hierarchical NVFP4 input.""" - unpacked = unpack_fp4_to_f32(q) - T, K, H = unpacked.shape - expanded_sfc = sfc_scale.to(torch.float32).repeat_interleave( - NVFP4_SFC_SCALE_BLOCK_SIZE, - dim=-1, - )[:, :, :H] - expanded_global = global_scale.to(torch.float32).repeat_interleave( - NVFP4_GLOBAL_SCALE_BLOCK_SIZE, - dim=-1, - )[:, :, :H] - acc = torch.zeros((T, H), device=q.device, dtype=torch.float32) - for k in range(K): - contrib = unpacked[:, k, :] * expanded_sfc[:, k, :] - if topk_score is not None: - contrib = contrib * expanded_global[:, k, :] - acc = torch.addcmul(acc, contrib, topk_score[:, k, None]) + sf = cute.make_tensor( + combine_sf.iterator, + cute.make_layout( + (combine_sf.shape[0], self.num_topk, + (sf_vec, self.hidden // sf_vec)), + stride=(combine_sf.stride[0], combine_sf.stride[1], + (0, combine_sf.stride[2])), + ), + ) + + if cutlass.const_expr( + self.combine_format.act_dtype is cutlass.Float8E4M3FN): + self._reduce_mxfp8(combine_quant, sf, topk_score, + reduced_output).launch( + grid=grid, + block=block, + stream=stream, + ) else: - acc = torch.addcmul(acc, contrib, expanded_global[:, k, :]) - return acc - - -def weighted_reference_sum( - src: torch.Tensor, - topk_score: torch.Tensor, -) -> torch.Tensor: - """Return K-ordered FP32 weighted reduce using FMA/addcmul semantics.""" - src_f32 = src.to(torch.float32) - acc = torch.zeros( - (src.shape[0], src.shape[2]), - device=src.device, - dtype=torch.float32, - ) - for k in range(src.shape[1]): - acc = torch.addcmul(acc, src_f32[:, k, :], topk_score[:, k, None]) - return acc - - -def ordered_reference_sum(src: torch.Tensor) -> torch.Tensor: - """Return K-ordered FP32 reduce of BF16 input.""" - src_f32 = src.to(torch.float32) - acc = torch.zeros( - (src.shape[0], src.shape[2]), - device=src.device, - dtype=torch.float32, - ) - for k in range(src.shape[1]): - acc = acc + src_f32[:, k, :] - return acc + self._reduce_fp4(combine_quant, sf, topk_score, + reduced_output).launch( + grid=grid, + block=block, + stream=stream, + ) + @cute.jit + def _mark_alignment(self, tensor: cute.Tensor, + align_bytes: int) -> cute.Tensor: + p = tensor.iterator + return cute.make_tensor( + cute.make_ptr(p.dtype, + p.toint(), + p.memspace, + assumed_align=align_bytes), + tensor.layout, + ) -@cute.jit -def _fp4_e2m1_nibble_to_f32(nibble: Int32) -> Float32: - value = Float32(0.0) - if nibble == Int32(1): - value = Float32(0.5) - elif nibble == Int32(2): - value = Float32(1.0) - elif nibble == Int32(3): - value = Float32(1.5) - elif nibble == Int32(4): - value = Float32(2.0) - elif nibble == Int32(5): - value = Float32(3.0) - elif nibble == Int32(6): - value = Float32(4.0) - elif nibble == Int32(7): - value = Float32(6.0) - elif nibble == Int32(9): - value = Float32(-0.5) - elif nibble == Int32(10): - value = Float32(-1.0) - elif nibble == Int32(11): - value = Float32(-1.5) - elif nibble == Int32(12): - value = Float32(-2.0) - elif nibble == Int32(13): - value = Float32(-3.0) - elif nibble == Int32(14): - value = Float32(-4.0) - elif nibble == Int32(15): - value = Float32(-6.0) - return value - - -@cute.kernel -def topk_reduce_bf16_vec_kernel( - combine_output: cute.Tensor, - topk_score: Optional[cute.Tensor], - reduced_output: cute.Tensor, - num_topk: cutlass.Constexpr[int], - hidden: cutlass.Constexpr[int], - store_dtype: cutlass.Constexpr[str], -): - """BF16 reduce with one thread handling one 8-hidden vector.""" - - hidden_vec_block_idx, token_idx, _ = cute.arch.block_idx() - tid = cute.arch.thread_idx()[0] - block_dim = cute.arch.block_dim()[0] - vec_idx = hidden_vec_block_idx * block_dim + tid - base_h = vec_idx * Int32(BF16_HIDDEN_PER_THREAD) - - if base_h < Int32(hidden): - acc = cute.make_rmem_tensor((BF16_HIDDEN_PER_THREAD, ), cutlass.Float32) - for i in cutlass.range_constexpr(0, BF16_HIDDEN_PER_THREAD, 1): - acc[i] = Float32(0.0) + # -- kernels -------------------------------------------------------------- - copy_atom_bf16_vec = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - cutlass.BFloat16, - num_bits_per_copy=128, - ) + @cute.kernel + def _reduce_bf16( + self, + combine_output: cute.Tensor, + topk_score: Optional[cute.Tensor], + reduced_output: cute.Tensor, + ): + threads = self._threads + hidden_per_thread = self.hidden_per_thread + hidden_tiles = self.hidden_tiles + num_topk: cutlass.Constexpr[int] = self.num_topk + needs_guard = self.require_predicate + prefetch = self.prefetch + out_dtype = reduced_output.element_type + + worker_idx = cute.arch.block_idx()[0] * Int32( + threads) + cute.arch.thread_idx()[0] + token_idx = worker_idx // hidden_tiles + hidden_tile_idx = worker_idx % hidden_tiles + + score_dtype = topk_score.dtype if cutlass.const_expr( + topk_score is not None) else cutlass.Float32 + score_reg = cute.make_rmem_tensor((num_topk, ), score_dtype) + + if (not needs_guard) or token_idx < reduced_output.shape[0]: + # (token, topk, hidden) -> (topk, hidden_per_thread) + terms = cute.zipped_divide( + combine_output[token_idx, None, None], + (num_topk, hidden_per_thread), + )[(None, None), (0, hidden_tile_idx)] + # (token, hidden) -> (hidden_per_thread) + dst = cute.zipped_divide( + reduced_output[token_idx, None], + (hidden_per_thread, ), + )[(None, ), (hidden_tile_idx, )] - for k in cutlass.range_constexpr(0, num_topk, 1): - score_value = Float32(1.0) if cutlass.const_expr(topk_score is not None): - score_value = Float32(topk_score[token_idx, Int32(k)]) - score_pair = (score_value, score_value) + if cutlass.const_expr(prefetch): + cute.autovec_copy(topk_score[token_idx, None], score_reg) + else: + for k in cutlass.range_constexpr(num_topk): + score_reg[k] = score_dtype(1) - in_regs = cute.make_rmem_tensor( - (BF16_HIDDEN_PER_THREAD, ), + load_atom = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), cutlass.BFloat16, + num_bits_per_copy=128, ) - in_row = combine_output[token_idx, Int32(k), None] - in_tile = cute.local_tile( - in_row, - (BF16_HIDDEN_PER_THREAD, ), - (base_h // Int32(BF16_HIDDEN_PER_THREAD), ), - ) - in_aligned_iter = cute.make_ptr( - in_tile.element_type, - in_tile.iterator.toint(), - AddressSpace.gmem, - assumed_align=16, - ) - in_tile = cute.make_tensor(in_aligned_iter, in_tile.layout) - cute.copy( - copy_atom_bf16_vec, - cute.coalesce(in_tile), - cute.coalesce(in_regs), - ) - - for pair_i in cutlass.range_constexpr( - 0, - BF16_HIDDEN_PER_THREAD // 2, - 1, - ): - val_pair = ( - Float32(in_regs[2 * pair_i]), - Float32(in_regs[2 * pair_i + 1]), - ) - old_acc_pair = (acc[2 * pair_i], acc[2 * pair_i + 1]) - if cutlass.const_expr(topk_score is not None): - acc_pair = cute.arch.fma_packed_f32x2( - val_pair, - score_pair, - old_acc_pair, - ) - else: - acc_pair = cute.arch.add_packed_f32x2( - old_acc_pair, - val_pair, - ) - acc[2 * pair_i] = acc_pair[0] - acc[2 * pair_i + 1] = acc_pair[1] + acc = cute.make_rmem_tensor((hidden_per_thread, ), cutlass.Float32) - out_row = reduced_output[token_idx, None] - out_tile = cute.local_tile( - out_row, - (BF16_HIDDEN_PER_THREAD, ), - (base_h // Int32(BF16_HIDDEN_PER_THREAD), ), - ) - if cutlass.const_expr(store_dtype == "bf16"): - out_regs = cute.make_rmem_tensor( - (BF16_HIDDEN_PER_THREAD, ), - cutlass.BFloat16, - ) - out_regs.store(acc.load().to(cutlass.BFloat16)) - out_aligned_iter = cute.make_ptr( - out_tile.element_type, - out_tile.iterator.toint(), - AddressSpace.gmem, - assumed_align=16, - ) - out_tile = cute.make_tensor(out_aligned_iter, out_tile.layout) + for k in cutlass.range_constexpr(0, num_topk, 1): + term = cute.make_rmem_tensor((hidden_per_thread, ), + cutlass.BFloat16) + cute.copy(load_atom, terms[k, None], term) + if cutlass.const_expr(topk_score is not None and not prefetch): + score_reg[k] = topk_score[token_idx, Int32(k)] + score_pair = (Float32(score_reg[k]), Float32(score_reg[k])) + + for i in cutlass.range_constexpr(0, hidden_per_thread, 2): + value_pair = (Float32(term[i]), Float32(term[i + 1])) + if cutlass.const_expr(k != 0): + acc[i], acc[i + 1] = cute.arch.fma_packed_f32x2( + value_pair, score_pair, (acc[i], acc[i + 1])) + else: + if cutlass.const_expr(topk_score is not None): + acc[i], acc[i + 1] = cute.arch.mul_packed_f32x2( + value_pair, score_pair) + else: + acc[i] = value_pair[0] + acc[i + 1] = value_pair[1] + + out = cute.make_rmem_tensor((hidden_per_thread, ), out_dtype) + out.store(acc.load().to(out_dtype)) cute.copy( - copy_atom_bf16_vec, - cute.coalesce(out_regs), - cute.coalesce(out_tile), - ) - else: - for i in cutlass.range_constexpr(0, BF16_HIDDEN_PER_THREAD, 1): - out_tile[i] = acc[i] - - -@cute.kernel -def topk_reduce_mxfp8_vec_kernel( - combine_output: cute.Tensor, - topk_score: Optional[cute.Tensor], - mxfp8_scale: cute.Tensor, - reduced_output: cute.Tensor, - num_topk: cutlass.Constexpr[int], - hidden: cutlass.Constexpr[int], - mxfp8_scale_rank: cutlass.Constexpr[int], -): - """MXFP8 reduce with one thread handling one 16-hidden vector.""" - - hidden_vec_block_idx, token_idx, _ = cute.arch.block_idx() - tid = cute.arch.thread_idx()[0] - block_dim = cute.arch.block_dim()[0] - vec_idx = hidden_vec_block_idx * block_dim + tid - base_h = vec_idx * Int32(MXFP8_HIDDEN_PER_THREAD) - - if base_h < Int32(hidden): - acc = cute.make_rmem_tensor((MXFP8_HIDDEN_PER_THREAD, ), - cutlass.Float32) - for i in cutlass.range_constexpr(0, MXFP8_HIDDEN_PER_THREAD, 1): - acc[i] = Float32(0.0) - - copy_atom_ldg_128b = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - cutlass.Float8E4M3FN, - num_bits_per_copy=128, - ) - copy_atom_stg_256b = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - cutlass.BFloat16, - num_bits_per_copy=256, - ) - scale_col = base_h // Int32(MXFP8_SCALE_BLOCK_SIZE) + cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), + out_dtype, + num_bits_per_copy=128), + out, + self._mark_alignment(dst, + hidden_per_thread * out_dtype.width // 8), + ) + + @cute.kernel + def _reduce_mxfp8( + self, + combine_quant: cute.Tensor, + combine_sf: cute. + Tensor, # depth-2 broadcast view: logical (token, topk, hidden) e8m0 + topk_score: Optional[cute.Tensor], + reduced_output: cute.Tensor, + ): + threads = self._threads + hidden_per_thread = self.hidden_per_thread + hidden_tiles = self.hidden_tiles + num_topk: cutlass.Constexpr[int] = self.num_topk + needs_guard = self.require_predicate + prefetch = self.prefetch + out_dtype = reduced_output.element_type + + worker_idx = cute.arch.block_idx()[0] * Int32( + threads) + cute.arch.thread_idx()[0] + token_idx = worker_idx // hidden_tiles + hidden_tile_idx = worker_idx % hidden_tiles + + score_dtype = topk_score.dtype if cutlass.const_expr( + topk_score is not None) else cutlass.Float32 + score_reg = cute.make_rmem_tensor((num_topk, ), score_dtype) + scale_reg = cute.make_rmem_tensor((num_topk, ), cutlass.Float8E8M0FNU) + + if (not needs_guard) or token_idx < reduced_output.shape[0]: + # (token, topk, hidden) -> (topk, hidden_per_thread) + codes = cute.zipped_divide( + combine_quant[token_idx, None, None], + (num_topk, hidden_per_thread), + )[(None, None), (0, hidden_tile_idx)] + # (token, topk, hidden) -> (topk, hidden_per_thread) + sf = cute.zipped_divide( + combine_sf[token_idx, None, None], + (num_topk, hidden_per_thread), + )[(None, None), (0, hidden_tile_idx)] + # (token, hidden) -> (hidden_per_thread) + dst = cute.zipped_divide( + reduced_output[token_idx, None], + (hidden_per_thread, ), + )[(None, ), (hidden_tile_idx, )] - for k in cutlass.range_constexpr(0, num_topk, 1): - if cutlass.const_expr(mxfp8_scale_rank == 3): - scale = Float32(mxfp8_scale[token_idx, Int32(k), scale_col]) - else: - scale = Float32(mxfp8_scale[token_idx, scale_col]) - scale_pair = (scale, scale) - score_value = Float32(1.0) if cutlass.const_expr(topk_score is not None): - score_value = Float32(topk_score[token_idx, Int32(k)]) - score_pair = (score_value, score_value) - - in_regs = cute.make_rmem_tensor( - (MXFP8_HIDDEN_PER_THREAD, ), + if cutlass.const_expr(prefetch): + cute.autovec_copy(topk_score[token_idx, None], score_reg) + else: + for k in cutlass.range_constexpr(num_topk): + score_reg[k] = score_dtype(1) + if cutlass.const_expr(prefetch): + cute.autovec_copy( + sf[None, 0], + scale_reg) # one scale per topk slot (stride-0 broadcast) + + load_atom = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), cutlass.Float8E4M3FN, + num_bits_per_copy=128, ) - in_row = combine_output[token_idx, Int32(k), None] - in_tile = cute.local_tile( - in_row, - (MXFP8_HIDDEN_PER_THREAD, ), - (base_h // Int32(MXFP8_HIDDEN_PER_THREAD), ), - ) - in_aligned_iter = cute.make_ptr( - in_tile.element_type, - in_tile.iterator.toint(), - AddressSpace.gmem, - assumed_align=16, - ) - in_tile = cute.make_tensor(in_aligned_iter, in_tile.layout) - cute.copy( - copy_atom_ldg_128b, - cute.coalesce(in_tile), - cute.coalesce(in_regs), - ) - in_vals = cute.make_rmem_tensor( - (MXFP8_HIDDEN_PER_THREAD, ), - cutlass.Float32, - ) - in_vals.store(in_regs.load().to(cutlass.Float32)) - - for pair_i in cutlass.range_constexpr( - 0, - MXFP8_HIDDEN_PER_THREAD // 2, - 1, - ): - val_pair = ( - in_vals[2 * pair_i], - in_vals[2 * pair_i + 1], - ) - old_acc_pair = (acc[2 * pair_i], acc[2 * pair_i + 1]) - if cutlass.const_expr(topk_score is not None): - contrib_pair = cute.arch.mul_packed_f32x2( - val_pair, - scale_pair, - ) - acc_pair = cute.arch.fma_packed_f32x2( - contrib_pair, - score_pair, - old_acc_pair, - ) - else: - acc_pair = cute.arch.fma_packed_f32x2( - val_pair, - scale_pair, - old_acc_pair, - ) - acc[2 * pair_i] = acc_pair[0] - acc[2 * pair_i + 1] = acc_pair[1] + acc = cute.make_rmem_tensor((hidden_per_thread, ), cutlass.Float32) - out_row = reduced_output[token_idx, None] - for chunk in cutlass.range_constexpr( - 0, - MXFP8_HIDDEN_PER_THREAD // BF16_STORE_ELEMENTS_PER_256B, - 1, - ): - out_regs = cute.make_rmem_tensor( - (BF16_STORE_ELEMENTS_PER_256B, ), - cutlass.BFloat16, - ) - for i in cutlass.range_constexpr(0, BF16_STORE_ELEMENTS_PER_256B, - 1): - out_regs[i] = acc[chunk * BF16_STORE_ELEMENTS_PER_256B + i].to( - cutlass.BFloat16) - out_h = base_h + Int32(chunk * BF16_STORE_ELEMENTS_PER_256B) - out_tile = cute.local_tile( - out_row, - (BF16_STORE_ELEMENTS_PER_256B, ), - (out_h // Int32(BF16_STORE_ELEMENTS_PER_256B), ), - ) - out_aligned_iter = cute.make_ptr( - out_tile.element_type, - out_tile.iterator.toint(), - AddressSpace.gmem, - assumed_align=32, - ) - out_tile = cute.make_tensor(out_aligned_iter, out_tile.layout) + for k in cutlass.range_constexpr(0, num_topk, 1): + term = cute.make_rmem_tensor((hidden_per_thread, ), + cutlass.Float8E4M3FN) + cute.copy(load_atom, codes[k, None], term) + value = cute.make_rmem_tensor((hidden_per_thread, ), + cutlass.Float32) + value.store(term.load().to(cutlass.Float32)) + + if cutlass.const_expr(not prefetch): + scale_reg[k] = sf[k, 0] + if cutlass.const_expr(topk_score is not None): + score_reg[k] = topk_score[token_idx, Int32(k)] + + scale = Float32(scale_reg[k]) # e8m0 -> f32 + scale_pair = (scale, scale) + score_pair = (Float32(score_reg[k]), Float32(score_reg[k])) + + for i in cutlass.range_constexpr(0, hidden_per_thread, 2): + dequant_pair = cute.arch.mul_packed_f32x2( + (value[i], value[i + 1]), scale_pair) + if cutlass.const_expr(k != 0): + acc[i], acc[i + 1] = cute.arch.fma_packed_f32x2( + dequant_pair, score_pair, (acc[i], acc[i + 1])) + else: + if cutlass.const_expr(topk_score is not None): + acc[i], acc[i + 1] = cute.arch.mul_packed_f32x2( + dequant_pair, score_pair) + else: + acc[i] = dequant_pair[0] + acc[i + 1] = dequant_pair[1] + + out = cute.make_rmem_tensor((hidden_per_thread, ), out_dtype) + out.store(acc.load().to(out_dtype)) cute.copy( - copy_atom_stg_256b, - cute.coalesce(out_regs), - cute.coalesce(out_tile), - ) - - -@cute.kernel -def topk_reduce_kernel( - combine_output: cute.Tensor, - topk_score: Optional[cute.Tensor], - mxfp8_scale: Optional[cute.Tensor], - nvfp4_sfc_scale: Optional[cute.Tensor], - nvfp4_global_scale: Optional[cute.Tensor], - reduced_output: cute.Tensor, - num_topk: cutlass.Constexpr[int], - hidden: cutlass.Constexpr[int], - store_dtype: cutlass.Constexpr[str], - mxfp8_scale_rank: cutlass.Constexpr[int], -): - """Reduce ``combine_output[t, :, h]`` into ``reduced_output[t, h]``. - - In the default path, ``combine_output`` is BF16. In MXFP8 mode, - ``combine_output`` is FP8 E4M3 and ``mxfp8_scale`` is UE8M0 with either - shape ``(T, ceil_div(H, 32))`` or ``(T, K, ceil_div(H, 32))``. Optional - ``topk_score`` is FP32 with shape ``(T, K)`` and scales each K - contribution before accumulation. Shapes and store dtype are supplied as - constexprs by the launcher so the K loop is fully unrolled and matches the - host reference order exactly. - """ - - hidden_block_idx, token_idx, _ = cute.arch.block_idx() - - h = hidden_block_idx * cute.arch.block_dim()[0] + cute.arch.thread_idx()[0] + cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), + out_dtype, + num_bits_per_copy=256), + out, + self._mark_alignment(dst, + hidden_per_thread * out_dtype.width // 8), + ) + + @cute.kernel + def _reduce_fp4( + self, + combine_quant: cute.Tensor, # (token, topk, hidden) e2m1 (logical) + combine_sf: cute. + Tensor, # depth-2 broadcast view: logical (token, topk, hidden) bf16 amax + topk_score: Optional[cute.Tensor], + reduced_output: cute.Tensor, + ): + threads = self._threads + hidden_per_thread = self.hidden_per_thread + hidden_tiles = self.hidden_tiles + num_topk: cutlass.Constexpr[int] = self.num_topk + needs_guard = self.require_predicate + prefetch = self.prefetch + out_dtype = reduced_output.element_type + + worker_idx = cute.arch.block_idx()[0] * Int32( + threads) + cute.arch.thread_idx()[0] + token_idx = worker_idx // hidden_tiles + hidden_tile_idx = worker_idx % hidden_tiles + + score_dtype = topk_score.dtype if cutlass.const_expr( + topk_score is not None) else cutlass.Float32 + score_reg = cute.make_rmem_tensor((num_topk, ), score_dtype) + scale_reg = cute.make_rmem_tensor((num_topk, ), cutlass.BFloat16) + + if (not needs_guard) or token_idx < reduced_output.shape[0]: + # (token, topk, hidden) -> (topk, hidden_per_thread) + codes = cute.zipped_divide( + combine_quant[token_idx, None, None], + (num_topk, hidden_per_thread), + )[(None, None), (0, hidden_tile_idx)] + # (token, topk, hidden) -> (topk, hidden_per_thread) + sf = cute.zipped_divide( + combine_sf[token_idx, None, None], + (num_topk, hidden_per_thread), + )[(None, None), (0, hidden_tile_idx)] + # (token, hidden) -> (hidden_per_thread) + dst = cute.zipped_divide( + reduced_output[token_idx, None], + (hidden_per_thread, ), + )[(None, ), (hidden_tile_idx, )] - if h < Int32(hidden): - acc = Float32(0.0) - for k in cutlass.range_constexpr(0, num_topk, 1): - if cutlass.const_expr(nvfp4_sfc_scale is not None): - byte_col = h // Int32(2) - shift = (h - byte_col * Int32(2)) * Int32(4) - packed = Int32(combine_output[token_idx, Int32(k), byte_col]) - nibble = (packed >> shift) & Int32(0x0F) - contrib = _fp4_e2m1_nibble_to_f32(nibble) - sfc_col = h // Int32(NVFP4_SFC_SCALE_BLOCK_SIZE) - global_col = h // Int32(NVFP4_GLOBAL_SCALE_BLOCK_SIZE) - sfc = Float32(nvfp4_sfc_scale[token_idx, Int32(k), sfc_col]) - global_sf = Float32(nvfp4_global_scale[token_idx, - Int32(k), global_col]) - contrib = contrib * sfc * global_sf - else: - contrib = Float32(combine_output[token_idx, Int32(k), h]) - if cutlass.const_expr(mxfp8_scale is not None): - scale_col = h // Int32(MXFP8_SCALE_BLOCK_SIZE) - if cutlass.const_expr(mxfp8_scale_rank == 3): - scale = Float32(mxfp8_scale[token_idx, - Int32(k), scale_col]) - else: - scale = Float32(mxfp8_scale[token_idx, scale_col]) - contrib = contrib * scale if cutlass.const_expr(topk_score is not None): - contrib = contrib * Float32(topk_score[token_idx, Int32(k)]) - acc = acc + contrib + if cutlass.const_expr(prefetch): + cute.autovec_copy(topk_score[token_idx, None], score_reg) else: - acc = acc + contrib - if cutlass.const_expr(store_dtype == "bf16"): - reduced_output[token_idx, h] = acc.to(cutlass.BFloat16) - else: - reduced_output[token_idx, h] = acc - + for k in cutlass.range_constexpr(num_topk): + score_reg[k] = score_dtype(1) + if cutlass.const_expr(prefetch): + cute.autovec_copy(sf[None, 0], scale_reg) -@cute.kernel -def topk_reduce_nvfp4_vec_kernel( - combine_output: cute.Tensor, - topk_score: Optional[cute.Tensor], - nvfp4_sfc_scale: cute.Tensor, - nvfp4_global_scale: cute.Tensor, - reduced_output: cute.Tensor, - num_topk: cutlass.Constexpr[int], - hidden: cutlass.Constexpr[int], - store_dtype: cutlass.Constexpr[str], -): - """NVFP4 reduce with one thread handling two per-16 hidden blocks.""" - - hidden_vec_block_idx, token_idx, _ = cute.arch.block_idx() - tid = cute.arch.thread_idx()[0] - block_dim = cute.arch.block_dim()[0] - vec_idx = hidden_vec_block_idx * block_dim + tid - base_h = vec_idx * Int32(NVFP4_HIDDEN_PER_THREAD) - - if base_h < Int32(hidden): - sfc_col_base = base_h // Int32(NVFP4_SFC_SCALE_BLOCK_SIZE) - global_col = base_h // Int32(NVFP4_GLOBAL_SCALE_BLOCK_SIZE) - - copy_atom_ldg_sfc = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - cutlass.Uint8, - num_bits_per_copy=NVFP4_SFC_INPUT_BITS_PER_COPY, - ) - copy_atom_stg_256b = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - cutlass.BFloat16, - num_bits_per_copy=256, - ) - - global_regs = cute.make_rmem_tensor((num_topk, ), cutlass.Float32) - for k in cutlass.range_constexpr(0, num_topk, 1): - global_regs[k] = Float32(nvfp4_global_scale[token_idx, - Int32(k), global_col]) - if cutlass.const_expr(topk_score is not None): - score_regs = cute.make_rmem_tensor((num_topk, ), cutlass.Float32) - for k in cutlass.range_constexpr(0, num_topk, 1): - score_regs[k] = Float32(topk_score[token_idx, Int32(k)]) - - out_row = reduced_output[token_idx, None] - for sfc_block_i in cutlass.range_constexpr( - 0, - NVFP4_HIDDEN_PER_THREAD // NVFP4_SFC_SCALE_BLOCK_SIZE, - 1, - ): - acc = cute.make_rmem_tensor( - (NVFP4_SFC_SCALE_BLOCK_SIZE, ), - cutlass.Float32, + load_atom = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), + cutlass.Float4E2M1FN, + num_bits_per_copy=64, ) - for i in cutlass.range_constexpr(0, NVFP4_SFC_SCALE_BLOCK_SIZE, 1): - acc[i] = Float32(0.0) + acc = cute.make_rmem_tensor((hidden_per_thread, ), cutlass.Float32) - sfc_base_h = base_h + Int32( - sfc_block_i * NVFP4_SFC_SCALE_BLOCK_SIZE) for k in cutlass.range_constexpr(0, num_topk, 1): - global_sf = global_regs[k] - global_pair = (global_sf, global_sf) - score_value = Float32(1.0) - if cutlass.const_expr(topk_score is not None): - score_value = score_regs[k] - score_pair = (score_value, score_value) - - q_bytes = cute.make_rmem_tensor( - (NVFP4_SFC_PACKED_BYTES, ), - cutlass.Uint8, - ) - q_row = combine_output[token_idx, Int32(k), None] - q_tile = cute.local_tile( - q_row, - (NVFP4_SFC_PACKED_BYTES, ), - (sfc_base_h // Int32(NVFP4_SFC_SCALE_BLOCK_SIZE), ), - ) - q_aligned_iter = cute.make_ptr( - q_tile.element_type, - q_tile.iterator.toint(), - AddressSpace.gmem, - assumed_align=NVFP4_SFC_PACKED_BYTES, - ) - q_tile = cute.make_tensor(q_aligned_iter, q_tile.layout) - cute.copy( - copy_atom_ldg_sfc, - cute.coalesce(q_tile), - cute.coalesce(q_bytes), - ) - q_fp4 = cute.recast_tensor(q_bytes, cutlass.Float4E2M1FN) - q_vals = q_fp4.load().to(cutlass.Float32) - sfc = Float32(nvfp4_sfc_scale[ - token_idx, - Int32(k), - sfc_col_base + Int32(sfc_block_i), - ]) - sfc_pair = (sfc, sfc) - for byte_offset in cutlass.range_constexpr( - 0, - NVFP4_SFC_SCALE_BLOCK_SIZE // 2, - 1, - ): - val_pair = ( - q_vals[2 * byte_offset], - q_vals[2 * byte_offset + 1], - ) - contrib_pair = cute.arch.mul_packed_f32x2( - val_pair, sfc_pair) - old_acc_pair = ( - acc[2 * byte_offset], - acc[2 * byte_offset + 1], - ) + term = cute.make_rmem_tensor((hidden_per_thread, ), + cutlass.Float4E2M1FN) + cute.copy(load_atom, codes[k, None], term) + value = cute.make_rmem_tensor((hidden_per_thread, ), + cutlass.Float32) + cvt_e2m1_to_fp32_cvt_ptx(term, value) + + if cutlass.const_expr(not prefetch): + scale_reg[k] = sf[k, 0] if cutlass.const_expr(topk_score is not None): - contrib_pair = cute.arch.mul_packed_f32x2( - contrib_pair, - global_pair, - ) - acc_pair = cute.arch.fma_packed_f32x2( - contrib_pair, - score_pair, - old_acc_pair, - ) + score_reg[k] = topk_score[token_idx, Int32(k)] + + # amax (bf16) -> per-element scale; (1/6) folds the fp4 grid max. + scale = Float32(scale_reg[k]) * Float32(Nvfp4E2M1RcpLimit) + scale_pair = (scale, scale) + score_pair = (Float32(score_reg[k]), Float32(score_reg[k])) + + for i in cutlass.range_constexpr(0, hidden_per_thread, 2): + dequant_pair = cute.arch.mul_packed_f32x2( + (value[i], value[i + 1]), scale_pair) + if cutlass.const_expr(k != 0): + acc[i], acc[i + 1] = cute.arch.fma_packed_f32x2( + dequant_pair, score_pair, (acc[i], acc[i + 1])) + elif cutlass.const_expr(topk_score is not None): + acc[i], acc[i + 1] = cute.arch.mul_packed_f32x2( + dequant_pair, score_pair) else: - acc_pair = cute.arch.fma_packed_f32x2( - contrib_pair, - global_pair, - old_acc_pair, - ) - acc[2 * byte_offset] = acc_pair[0] - acc[2 * byte_offset + 1] = acc_pair[1] - - if cutlass.const_expr(store_dtype == "bf16"): - out_regs = cute.make_rmem_tensor( - (BF16_STORE_ELEMENTS_PER_256B, ), - cutlass.BFloat16, - ) - for i in cutlass.range_constexpr(0, - BF16_STORE_ELEMENTS_PER_256B, - 1): - out_regs[i] = acc[i].to(cutlass.BFloat16) - out_tile = cute.local_tile( - out_row, - (BF16_STORE_ELEMENTS_PER_256B, ), - (sfc_base_h // Int32(BF16_STORE_ELEMENTS_PER_256B), ), - ) - out_aligned_iter = cute.make_ptr( - out_tile.element_type, - out_tile.iterator.toint(), - AddressSpace.gmem, - assumed_align=32, - ) - out_tile = cute.make_tensor(out_aligned_iter, out_tile.layout) - cute.copy( - copy_atom_stg_256b, - cute.coalesce(out_regs), - cute.coalesce(out_tile), - ) - else: - out_tile = cute.local_tile( - out_row, - (NVFP4_SFC_SCALE_BLOCK_SIZE, ), - (sfc_base_h // Int32(NVFP4_SFC_SCALE_BLOCK_SIZE), ), - ) - for i in cutlass.range_constexpr(0, NVFP4_SFC_SCALE_BLOCK_SIZE, - 1): - out_tile[i] = acc[i] - - -def _validate_tensors( - combine_output: torch.Tensor, - reduced_output: torch.Tensor, - topk_score: Optional[torch.Tensor] = None, - mxfp8_scale: Optional[torch.Tensor] = None, - nvfp4_sfc_scale: Optional[torch.Tensor] = None, - nvfp4_global_scale: Optional[torch.Tensor] = None, -) -> Tuple[int, int, int, int]: - if combine_output.dim() != 3: - raise ValueError( - f"combine_output must have shape (T, K, H), got {tuple(combine_output.shape)}." - ) - if reduced_output.dim() != 2: - raise ValueError( - f"reduced_output must have shape (T, H), got {tuple(reduced_output.shape)}." - ) - if reduced_output.dtype not in (torch.float32, torch.bfloat16): - raise TypeError( - f"reduced_output must be torch.float32 or torch.bfloat16, got {reduced_output.dtype}." - ) - if not combine_output.is_cuda or not reduced_output.is_cuda: - raise ValueError( - "combine_output and reduced_output must both be CUDA tensors.") - if combine_output.device != reduced_output.device: - raise ValueError( - f"combine_output and reduced_output must be on the same device, got " - f"{combine_output.device} and {reduced_output.device}.") - - if mxfp8_scale is not None and (nvfp4_sfc_scale is not None - or nvfp4_global_scale is not None): - raise ValueError("MXFP8 and NVFP4 modes are mutually exclusive.") - if (nvfp4_sfc_scale is None) != (nvfp4_global_scale is None): - raise ValueError( - "nvfp4_sfc_scale and nvfp4_global_scale must be provided together.") - - T, K, H_storage = combine_output.shape - if T <= 0 or K <= 0 or H_storage <= 0: - raise ValueError( - f"combine_output shape must have positive dimensions, got " - f"{tuple(combine_output.shape)}.") - - nvfp4_mode = nvfp4_sfc_scale is not None - H = int(H_storage) * 2 if nvfp4_mode else int(H_storage) - if reduced_output.shape != (T, H): - raise ValueError( - f"reduced_output shape must be {(T, H)}, got {tuple(reduced_output.shape)}." - ) - - mxfp8_scale_rank = 0 - if mxfp8_scale is None and not nvfp4_mode: - if combine_output.dtype != torch.bfloat16: - raise TypeError( - f"combine_output must be torch.bfloat16 unless mxfp8_scale is " - f"or NVFP4 scales are provided, got {combine_output.dtype}.") - elif mxfp8_scale is not None: - if not hasattr(torch, "float8_e4m3fn") or not hasattr( - torch, "float8_e8m0fnu"): - raise TypeError( - "MXFP8 mode requires torch float8_e4m3fn and float8_e8m0fnu.") - if combine_output.dtype != torch.float8_e4m3fn: - raise TypeError( - f"MXFP8 combine_output must be torch.float8_e4m3fn, got {combine_output.dtype}." - ) - if mxfp8_scale.dtype != torch.float8_e8m0fnu: - raise TypeError( - f"mxfp8_scale must be torch.float8_e8m0fnu, got {mxfp8_scale.dtype}." - ) - if reduced_output.dtype != torch.bfloat16: - raise TypeError( - f"MXFP8 reduced_output must be torch.bfloat16, got {reduced_output.dtype}." - ) - if not mxfp8_scale.is_cuda: - raise ValueError("mxfp8_scale must be a CUDA tensor.") - if mxfp8_scale.device != combine_output.device: - raise ValueError( - f"mxfp8_scale must be on {combine_output.device}, got {mxfp8_scale.device}." - ) - scale_cols = (H + MXFP8_SCALE_BLOCK_SIZE - 1) // MXFP8_SCALE_BLOCK_SIZE - if mxfp8_scale.dim() == 2: - expected_scale_shape = (T, scale_cols) - elif mxfp8_scale.dim() == 3: - expected_scale_shape = (T, K, scale_cols) - else: - raise ValueError( - "mxfp8_scale must have shape (T, ceil_div(H, 32)) or " - f"(T, K, ceil_div(H, 32)), got {tuple(mxfp8_scale.shape)}.") - if mxfp8_scale.shape != expected_scale_shape: - raise ValueError( - f"mxfp8_scale shape must be {expected_scale_shape}, got {tuple(mxfp8_scale.shape)}." - ) - mxfp8_scale_rank = mxfp8_scale.dim() - else: - if not hasattr(torch, "float8_e4m3fn"): - raise TypeError("NVFP4 mode requires torch float8_e4m3fn.") - if combine_output.dtype != torch.uint8: - raise TypeError( - f"NVFP4 combine_output must be packed torch.uint8, got {combine_output.dtype}." - ) - if nvfp4_sfc_scale.dtype != torch.float8_e4m3fn: - raise TypeError( - f"nvfp4_sfc_scale must be torch.float8_e4m3fn, got {nvfp4_sfc_scale.dtype}." - ) - if nvfp4_global_scale.dtype != torch.float32: - raise TypeError( - f"nvfp4_global_scale must be torch.float32, got {nvfp4_global_scale.dtype}." - ) - if not nvfp4_sfc_scale.is_cuda or not nvfp4_global_scale.is_cuda: - raise ValueError("NVFP4 scales must be CUDA tensors.") - if nvfp4_sfc_scale.device != combine_output.device: - raise ValueError( - f"nvfp4_sfc_scale must be on {combine_output.device}, got {nvfp4_sfc_scale.device}." - ) - if nvfp4_global_scale.device != combine_output.device: - raise ValueError( - f"nvfp4_global_scale must be on {combine_output.device}, got " - f"{nvfp4_global_scale.device}.") - sfc_cols = (H + NVFP4_SFC_SCALE_BLOCK_SIZE - - 1) // NVFP4_SFC_SCALE_BLOCK_SIZE - global_cols = (H + NVFP4_GLOBAL_SCALE_BLOCK_SIZE - - 1) // NVFP4_GLOBAL_SCALE_BLOCK_SIZE - expected_sfc_shape = (T, K, sfc_cols) - expected_global_shape = (T, K, global_cols) - if nvfp4_sfc_scale.dim( - ) != 3 or nvfp4_sfc_scale.shape != expected_sfc_shape: - raise ValueError( - f"nvfp4_sfc_scale shape must be {expected_sfc_shape}, got " - f"{tuple(nvfp4_sfc_scale.shape)}.") - if nvfp4_global_scale.dim( - ) != 3 or nvfp4_global_scale.shape != expected_global_shape: - raise ValueError( - f"nvfp4_global_scale shape must be {expected_global_shape}, got " - f"{tuple(nvfp4_global_scale.shape)}.") - if topk_score is not None: - if topk_score.dim() != 2: - raise ValueError( - f"topk_score must have shape (T, K), got {tuple(topk_score.shape)}." - ) - if topk_score.dtype != torch.float32: - raise TypeError( - f"topk_score must be torch.float32, got {topk_score.dtype}.") - if not topk_score.is_cuda: - raise ValueError("topk_score must be a CUDA tensor.") - if topk_score.device != combine_output.device: - raise ValueError( - f"topk_score must be on {combine_output.device}, got {topk_score.device}." - ) - if topk_score.shape != (T, K): - raise ValueError( - f"topk_score shape must be {(T, K)}, got {tuple(topk_score.shape)}." - ) - return int(T), int(K), int(H), int(mxfp8_scale_rank) - - -def _infer_assumed_align(tensor: torch.Tensor, max_align: int = 16) -> int: - ptr = int(tensor.data_ptr()) - for align in (16, 8, 4, 2, 1): - if align <= max_align and ptr % align == 0: - return align - return 1 - - -def _to_cute_tensor(tensor: torch.Tensor) -> cute.Tensor: - assumed_align = _infer_assumed_align(tensor) - cute_tensor = cutlass_torch.from_dlpack(tensor, assumed_align=assumed_align) - leading_dim = cutlass_torch.get_leading_dim(tensor) - return cute_tensor.mark_layout_dynamic(leading_dim=leading_dim) - - -def compile_topk_reduce( - combine_output: torch.Tensor, - reduced_output: torch.Tensor, - topk_score: Optional[torch.Tensor] = None, - *, - mxfp8_scale: Optional[torch.Tensor] = None, - nvfp4_sfc_scale: Optional[torch.Tensor] = None, - nvfp4_global_scale: Optional[torch.Tensor] = None, - threads: Optional[int] = None, - stream: Optional[cuda.CUstream] = None, -): - """Compile a shape-specialized topk reduce launcher. - - The returned tuple is always ``(compiled, combine_cute, reduced_cute, - topk_score_cute, mxfp8_scale_cute, nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, stream)``. Missing optional inputs are represented - by ``None``. Callers that only need a one-shot reduce should use - :func:`run_topk_reduce`. - """ - T, K, H, mxfp8_scale_rank = _validate_tensors( - combine_output, - reduced_output, - topk_score, - mxfp8_scale, - nvfp4_sfc_scale, - nvfp4_global_scale, - ) - if threads is not None and threads <= 0: - raise ValueError(f"threads must be positive, got {threads}.") - if stream is None: - stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) - store_dtype = "bf16" if reduced_output.dtype == torch.bfloat16 else "fp32" + acc[i] = dequant_pair[0] + acc[i + 1] = dequant_pair[1] - combine_cute = _to_cute_tensor(combine_output) - reduced_cute = _to_cute_tensor(reduced_output) - topk_score_cute = _to_cute_tensor( - topk_score) if topk_score is not None else None - mxfp8_scale_cute = _to_cute_tensor( - mxfp8_scale) if mxfp8_scale is not None else None - nvfp4_sfc_scale_cute = _to_cute_tensor( - nvfp4_sfc_scale) if nvfp4_sfc_scale is not None else None - nvfp4_global_scale_cute = (_to_cute_tensor(nvfp4_global_scale) - if nvfp4_global_scale is not None else None) - nvfp4_mode = nvfp4_sfc_scale is not None - bf16_vectorized = ( - not nvfp4_mode and mxfp8_scale is None - and combine_output.dtype == torch.bfloat16 - and H % BF16_HIDDEN_PER_THREAD == 0 and combine_output.stride(-1) == 1 - and combine_output.stride(-2) % BF16_HIDDEN_PER_THREAD == 0 - and reduced_output.stride(-1) == 1 - and (reduced_output.dtype != torch.bfloat16 - or reduced_output.stride(0) % BF16_HIDDEN_PER_THREAD == 0)) - mxfp8_vectorized = ( - mxfp8_scale is not None and not nvfp4_mode - and H % MXFP8_HIDDEN_PER_THREAD == 0 and combine_output.stride(-1) == 1 - and combine_output.stride(-2) % MXFP8_HIDDEN_PER_THREAD == 0 - and reduced_output.dtype == torch.bfloat16 - and reduced_output.stride(-1) == 1 - and reduced_output.stride(0) % MXFP8_HIDDEN_PER_THREAD == 0) - nvfp4_vectorized = ( - nvfp4_mode and H % NVFP4_HIDDEN_PER_THREAD == 0 - and combine_output.stride(-1) == 1 - and combine_output.stride(-2) % (NVFP4_HIDDEN_PER_THREAD // 2) == 0 - and reduced_output.stride(-1) == 1 - and (reduced_output.dtype != torch.bfloat16 - or reduced_output.stride(0) % NVFP4_HIDDEN_PER_THREAD == 0)) - if bf16_vectorized: - hidden_per_thread = BF16_HIDDEN_PER_THREAD - elif mxfp8_vectorized: - hidden_per_thread = MXFP8_HIDDEN_PER_THREAD - elif nvfp4_vectorized: - hidden_per_thread = NVFP4_HIDDEN_PER_THREAD - else: - hidden_per_thread = 1 - if threads is None: - if bf16_vectorized: - launch_threads = BF16_VECTOR_THREADS - elif mxfp8_vectorized: - launch_threads = MXFP8_VECTOR_THREADS - elif nvfp4_vectorized: - launch_threads = NVFP4_VECTOR_THREADS - else: - launch_threads = DEFAULT_THREADS - else: - launch_threads = threads - hidden_blocks = (H + launch_threads * hidden_per_thread - - 1) // (launch_threads * hidden_per_thread) - launch_grid = [hidden_blocks, T, 1] - - @cute.jit - def _launcher( - combine_cute: cute.Tensor, - reduced_cute: cute.Tensor, - topk_score_cute: Optional[cute.Tensor], - mxfp8_scale_cute: Optional[cute.Tensor], - nvfp4_sfc_scale_cute: Optional[cute.Tensor], - nvfp4_global_scale_cute: Optional[cute.Tensor], - stream: cuda.CUstream, - ): - if cutlass.const_expr(bf16_vectorized): - topk_reduce_bf16_vec_kernel( - combine_cute, - topk_score_cute, - reduced_cute, - num_topk=K, - hidden=H, - store_dtype=store_dtype, - ).launch( - grid=launch_grid, - block=[launch_threads, 1, 1], - stream=stream, - ) - elif cutlass.const_expr(mxfp8_vectorized): - topk_reduce_mxfp8_vec_kernel( - combine_cute, - topk_score_cute, - mxfp8_scale_cute, - reduced_cute, - num_topk=K, - hidden=H, - mxfp8_scale_rank=mxfp8_scale_rank, - ).launch( - grid=launch_grid, - block=[launch_threads, 1, 1], - stream=stream, - ) - elif cutlass.const_expr(nvfp4_vectorized): - topk_reduce_nvfp4_vec_kernel( - combine_cute, - topk_score_cute, - nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, - reduced_cute, - num_topk=K, - hidden=H, - store_dtype=store_dtype, - ).launch( - grid=launch_grid, - block=[launch_threads, 1, 1], - stream=stream, - ) - else: - topk_reduce_kernel( - combine_cute, - topk_score_cute, - mxfp8_scale_cute, - nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, - reduced_cute, - num_topk=K, - hidden=H, - store_dtype=store_dtype, - mxfp8_scale_rank=mxfp8_scale_rank, - ).launch( - grid=launch_grid, - block=[launch_threads, 1, 1], - stream=stream, + out = cute.make_rmem_tensor((hidden_per_thread, ), out_dtype) + out.store(acc.load().to(out_dtype)) + cute.copy( + cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), + out_dtype, + num_bits_per_copy=256), + out, + self._mark_alignment(dst, + hidden_per_thread * out_dtype.width // 8), ) - - compiled = cute.compile( - _launcher, - combine_cute, - reduced_cute, - topk_score_cute, - mxfp8_scale_cute, - nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, - stream, - ) - return ( - compiled, - combine_cute, - reduced_cute, - topk_score_cute, - mxfp8_scale_cute, - nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, - stream, - ) - - -def launch_compiled_topk_reduce( - compiled, - combine_cute: cute.Tensor, - reduced_cute: cute.Tensor, - topk_score_cute: Optional[cute.Tensor], - mxfp8_scale_cute: Optional[cute.Tensor], - nvfp4_sfc_scale_cute: Optional[cute.Tensor], - nvfp4_global_scale_cute: Optional[cute.Tensor], - stream: cuda.CUstream, - *, - synchronize: bool = False, - return_elapsed_ms: bool = False, -) -> Optional[float]: - """Launch a topk reduce plan returned by :func:`compile_topk_reduce`.""" - - def _launch() -> None: - compiled( - combine_cute, - reduced_cute, - topk_score_cute, - mxfp8_scale_cute, - nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, - stream, - ) - - if return_elapsed_ms: - start = torch.cuda.Event(enable_timing=True) - stop = torch.cuda.Event(enable_timing=True) - start.record() - _launch() - stop.record() - stop.synchronize() - elapsed_ms = float(start.elapsed_time(stop)) - else: - _launch() - elapsed_ms = None - - if synchronize: - torch.cuda.synchronize() - return elapsed_ms - - -def run_topk_reduce( - combine_output: torch.Tensor, - reduced_output: torch.Tensor, - topk_score: Optional[torch.Tensor] = None, - *, - mxfp8_scale: Optional[torch.Tensor] = None, - nvfp4_sfc_scale: Optional[torch.Tensor] = None, - nvfp4_global_scale: Optional[torch.Tensor] = None, - threads: Optional[int] = None, - stream: Optional[cuda.CUstream] = None, - synchronize: bool = False, - return_elapsed_ms: bool = False, -) -> Optional[float]: - """Compile and launch the topk reduce kernel. - - Returns the measured kernel elapsed time in milliseconds when - ``return_elapsed_ms`` is True, otherwise returns ``None``. - """ - plan = compile_topk_reduce( - combine_output, - reduced_output, - topk_score, - mxfp8_scale=mxfp8_scale, - nvfp4_sfc_scale=nvfp4_sfc_scale, - nvfp4_global_scale=nvfp4_global_scale, - threads=threads, - stream=stream, - ) - ( - compiled, - combine_cute, - reduced_cute, - topk_score_cute, - mxfp8_scale_cute, - nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, - stream, - ) = plan - return launch_compiled_topk_reduce( - compiled, - combine_cute, - reduced_cute, - topk_score_cute, - mxfp8_scale_cute, - nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, - stream, - synchronize=synchronize, - return_elapsed_ms=return_elapsed_ms, - ) - - -def benchmark_topk_reduce_vs_torch_sum( - *, - tokens: int, - topk: int, - hidden: int, - warmup: int = 5, - iters: int = 50, - output_dtype: torch.dtype = torch.float32, - seed: int = 20260531, - use_topk_score: bool = False, - use_mxfp8: bool = False, - use_nvfp4: bool = False, - mxfp8_scale_rank: int = 3, - threads: Optional[int] = None, - print_result: bool = True, -) -> dict[str, float]: - """Compare CuTeDSL topk_reduce against torch K-axis sum. - - The torch baseline intentionally uses the runner/reference expression: - ``combine_output_ref.to(torch.float32).sum(dim=1)`` when ``topk_score`` - is absent, or the weighted equivalent when present. MXFP8 and NVFP4 - inputs are converted from FP32 into their quantized data plus scale tensors - before both benchmark paths. CuTeDSL compile time is excluded from the - measured kernel time. - """ - if not torch.cuda.is_available(): - raise RuntimeError("CUDA GPU is required for topk_reduce benchmark.") - if output_dtype not in (torch.float32, torch.bfloat16): - raise ValueError( - f"output_dtype must be FP32 or BF16, got {output_dtype}.") - if use_mxfp8 and use_nvfp4: - raise ValueError( - "MXFP8 and NVFP4 benchmark modes are mutually exclusive.") - if use_mxfp8 and output_dtype != torch.bfloat16: - raise ValueError("MXFP8 benchmark requires BF16 output.") - if threads is None: - if use_nvfp4: - threads = NVFP4_VECTOR_THREADS - elif use_mxfp8: - threads = MXFP8_VECTOR_THREADS - else: - threads = BF16_VECTOR_THREADS - - torch.manual_seed(seed) - combine_output_fp32 = torch.randn( - (tokens, topk, hidden), - device="cuda", - dtype=torch.float32, - ) - if use_mxfp8: - combine_output_ref, mxfp8_scale = make_mxfp8_input( - combine_output_fp32, - scale_rank=mxfp8_scale_rank, - ) - nvfp4_sfc_scale = None - nvfp4_global_scale = None - input_dtype_name = "mxfp8" - elif use_nvfp4: - combine_output_ref, nvfp4_sfc_scale, nvfp4_global_scale = make_nvfp4_input( - combine_output_fp32, ) - mxfp8_scale = None - input_dtype_name = "nvfp4" - else: - combine_output_ref = combine_output_fp32.to(torch.bfloat16) - mxfp8_scale = None - nvfp4_sfc_scale = None - nvfp4_global_scale = None - input_dtype_name = "bf16" - topk_output = torch.empty( - (tokens, hidden), - device="cuda", - dtype=output_dtype, - ) - topk_score = None - if use_topk_score: - topk_score = torch.rand((tokens, topk), - device="cuda", - dtype=torch.float32) - - if print_result: - print( - "compiling topk_reduce " - f"shape={(tokens, topk, hidden)} output_dtype={output_dtype} " - f"input_dtype={input_dtype_name} " - f"mxfp8_scale_rank={mxfp8_scale_rank if use_mxfp8 else 'none'} " - f"topk_score={'on' if topk_score is not None else 'off'} " - f"threads={threads}", - flush=True, - ) - ( - compiled, - combine_cute, - reduced_cute, - topk_score_cute, - mxfp8_scale_cute, - nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, - stream, - ) = compile_topk_reduce( - combine_output_ref, - topk_output, - topk_score, - mxfp8_scale=mxfp8_scale, - nvfp4_sfc_scale=nvfp4_sfc_scale, - nvfp4_global_scale=nvfp4_global_scale, - threads=threads, - ) - - compiled( - combine_cute, - reduced_cute, - topk_score_cute, - mxfp8_scale_cute, - nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, - stream, - ) - torch.cuda.synchronize() - - def reference_result(*, timed_baseline: bool) -> torch.Tensor: - if use_mxfp8: - return mxfp8_reference_sum( - combine_output_ref, - mxfp8_scale, - topk_score, - ).to(output_dtype) - if use_nvfp4: - return nvfp4_reference_sum( - combine_output_ref, - nvfp4_sfc_scale, - nvfp4_global_scale, - topk_score, - ).to(output_dtype) - if topk_score is None: - if output_dtype == torch.bfloat16 and not timed_baseline: - return ordered_reference_sum(combine_output_ref).to( - output_dtype) - return combine_output_ref.to( - torch.float32).sum(dim=1).to(output_dtype) - return weighted_reference_sum(combine_output_ref, - topk_score).to(output_dtype) - - expected_result = reference_result(timed_baseline=False) - torch.testing.assert_close(topk_output, - expected_result, - atol=1e-5, - rtol=1e-5) - - def measure_cuda_ms(fn) -> float: - for _ in range(warmup): - fn() - torch.cuda.synchronize() - - start = torch.cuda.Event(enable_timing=True) - stop = torch.cuda.Event(enable_timing=True) - start.record() - for _ in range(iters): - fn() - stop.record() - stop.synchronize() - return float(start.elapsed_time(stop)) / float(iters) - - def run_compiled_topk_reduce() -> None: - compiled( - combine_cute, - reduced_cute, - topk_score_cute, - mxfp8_scale_cute, - nvfp4_sfc_scale_cute, - nvfp4_global_scale_cute, - stream, - ) - - torch_result = None - - def run_torch_sum() -> None: - nonlocal torch_result - torch_result = reference_result(timed_baseline=True) - - topk_ms = measure_cuda_ms(run_compiled_topk_reduce) - torch_ms = measure_cuda_ms(run_torch_sum) - assert torch_result is not None - speedup = torch_ms / topk_ms - read_bytes, write_bytes, total_bytes = logical_io_bytes( - combine_output_ref, - topk_output, - topk_score, - mxfp8_scale, - nvfp4_sfc_scale, - nvfp4_global_scale, - ) - topk_bw = bandwidth_gbps(total_bytes, topk_ms) - torch_bw = bandwidth_gbps(total_bytes, torch_ms) - - if print_result: - print("topk_reduce_vs_torch_sum " - f"shape={(tokens, topk, hidden)} output_dtype={output_dtype} " - f"input_dtype={input_dtype_name} " - f"mxfp8_scale_rank={mxfp8_scale_rank if use_mxfp8 else 'none'} " - f"topk_score={'on' if topk_score is not None else 'off'} " - f"threads={threads} " - f"warmup={warmup} iters={iters} " - f"topk_reduce_ms={topk_ms:.6f} " - f"torch_sum_ms={torch_ms:.6f} " - f"speedup_vs_torch={speedup:.3f}x " - f"read_gb={read_bytes / 1.0e9:.6f} " - f"write_gb={write_bytes / 1.0e9:.6f} " - f"topk_reduce_bw_gbps={topk_bw:.3f} " - f"torch_sum_bw_gbps={torch_bw:.3f}") - - return { - "topk_reduce_ms": topk_ms, - "torch_sum_ms": torch_ms, - "speedup_vs_torch": speedup, - "read_bytes": float(read_bytes), - "write_bytes": float(write_bytes), - "total_bytes": float(total_bytes), - "topk_reduce_bw_gbps": topk_bw, - "torch_sum_bw_gbps": torch_bw, - "use_mxfp8": float(use_mxfp8), - "use_nvfp4": float(use_nvfp4), - "threads": float(threads), - } - - -def _parse_bench_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=( - "Benchmark CuTeDSL topk_reduce against combine_output_ref.to(torch.float32).sum(dim=1)." - )) - parser.add_argument("--tokens", type=int, default=192) - parser.add_argument("--topk", type=int, default=8) - parser.add_argument("--hidden", type=int, default=7168) - parser.add_argument("--warmup", type=int, default=5) - parser.add_argument("--iters", type=int, default=50) - parser.add_argument("--output_dtype", - choices=["fp32", "bf16"], - default="bf16") - parser.add_argument("--use_topk_score", action="store_true") - parser.add_argument("--use_mxfp8", action="store_true") - parser.add_argument("--use_nvfp4", action="store_true") - parser.add_argument("--mxfp8_scale_rank", - type=int, - choices=[2, 3], - default=3) - parser.add_argument("--threads", type=int, default=None) - parser.add_argument("--seed", type=int, default=20260531) - return parser.parse_args() - - -def main() -> int: - args = _parse_bench_args() - output_dtype = torch.bfloat16 if args.output_dtype == "bf16" else torch.float32 - benchmark_topk_reduce_vs_torch_sum( - tokens=args.tokens, - topk=args.topk, - hidden=args.hidden, - warmup=args.warmup, - iters=args.iters, - output_dtype=output_dtype, - use_topk_score=args.use_topk_score, - use_mxfp8=args.use_mxfp8, - use_nvfp4=args.use_nvfp4, - mxfp8_scale_rank=args.mxfp8_scale_rank, - threads=args.threads, - seed=args.seed, - ) - print("DONE") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tensorrt_llm/_torch/disaggregation/native/mixers/attention/peer.py b/tensorrt_llm/_torch/disaggregation/native/mixers/attention/peer.py index 12d7cadfee4f..36aeb34ffdcd 100644 --- a/tensorrt_llm/_torch/disaggregation/native/mixers/attention/peer.py +++ b/tensorrt_llm/_torch/disaggregation/native/mixers/attention/peer.py @@ -1,3 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Sequence + import numpy as np from tensorrt_llm import logger @@ -12,107 +29,124 @@ from tensorrt_llm._utils import nvtx_range -class IdentityMapper(RegionMapperBase): +class IntactMapper(RegionMapperBase): """ - ---- mapper_identity ---- + ---- mapper_intact ---- + + Copy the selected layers' class regions between slots. Consumes slot-base + pointers from the extractor and expands them with slot-relative per-layer + byte offsets taken from the logical view's buffer entries, so it handles + non-uniform layer strides (a slot may interleave other role classes + between this class's layers). - Pass-through mapping. Do not change pointers or sizes. + src slot: [ base ]--+off(L2)--> [L2 region] --+off(L3)--> [L3 region] ... + dst slot: [ base ]--+off'(L2)-> [L2 region] --+off'(L3)-> [L3 region] ... - src_ptrs: [ S0 ] [ S1 ] [ S2 ] ... - | | | - v v v - dst_ptrs: [ D0 ] [ D1 ] [ D2 ] ... + Layers whose regions are contiguous on BOTH sides are merged into one + fragment, so a fully contiguous class (e.g. K/V in a dedicated pool) + degrades to a single whole-region copy per block. """ - @nvtx_range("IdentityMapper.map") - def map(self, src_regions: SpecRegion, dst_regions: SpecRegion) -> SpecRegionPair: + def __init__( + self, + src_layer_offsets: Sequence[int] | np.ndarray, + dst_layer_offsets: Sequence[int] | np.ndarray, + self_bytes_per_layer: int, + peer_bytes_per_layer: int, + *, + mapper_name: str = "Intact", + ) -> None: + if self_bytes_per_layer != peer_bytes_per_layer: + raise ValueError( + f"{mapper_name} cache region size mismatch: " + f"local={self_bytes_per_layer}, peer={peer_bytes_per_layer}" + ) + src = np.asarray(src_layer_offsets, dtype=np.int64) + dst = np.asarray(dst_layer_offsets, dtype=np.int64) + if src.size == 0 or src.size != dst.size: + raise ValueError( + f"{mapper_name} layer offsets must be non-empty and equal-length: " + f"src={src.size}, dst={dst.size}" + ) + self._runs = self._merge_contiguous(src, dst, self_bytes_per_layer) + + @staticmethod + def _merge_contiguous( + src: np.ndarray, dst: np.ndarray, bytes_per_layer: int + ) -> list[tuple[int, int, int]]: + runs: list[tuple[int, int, int]] = [] + run_start = 0 + for i in range(1, src.size + 1): + if ( + i == src.size + or src[i] != src[i - 1] + bytes_per_layer + or dst[i] != dst[i - 1] + bytes_per_layer + ): + run_layers = i - run_start + runs.append( + (int(src[run_start]), int(dst[run_start]), run_layers * bytes_per_layer) + ) + run_start = i + return runs + + @nvtx_range("IntactMapper.map") + def map(self, src_regions: SpecRegion, dst_regions: SpecRegion): src_group = src_regions.memory dst_group = dst_regions.memory - assert src_group.ptrs.size == dst_group.ptrs.size, ( - f"Number of regions of src({src_group.ptrs.size}) and dst({dst_group.ptrs.size}) must match" - ) - return SpecRegionPair( - src=SpecRegion(memory=src_group, spec=src_regions.spec), - dst=SpecRegion(memory=dst_group, spec=dst_regions.spec), - ) + if src_group.ptrs.size != dst_group.ptrs.size: + raise ValueError( + f"Number of regions of src({src_group.ptrs.size}) and " + f"dst({dst_group.ptrs.size}) must match" + ) + pairs = [ + SpecRegionPair( + src=SpecRegion( + memory=MemRegionGroup( + ptrs=src_group.ptrs + src_off, bytes_per_region=run_bytes + ), + spec=src_regions.spec, + ), + dst=SpecRegion( + memory=MemRegionGroup( + ptrs=dst_group.ptrs + dst_off, bytes_per_region=run_bytes + ), + spec=dst_regions.spec, + ), + ) + for src_off, dst_off, run_bytes in self._runs + ] + return pairs[0] if len(pairs) == 1 else pairs -class HeadMatchMapper(RegionMapperBase): - """ - ---- mapper_head_match ---- - - Move/copy entire contiguous block(s) (multi-layer fragment) as a single chunk. - Align by whole fragment size (frag_size) and apply a constant source/destination block offset. - - src_ptrs: [ S0 ] [ S1 ] ... - | | - + src_off + src_off - | | - [ S0 + src_off ] [ S1 + src_off ] -> (each points to a frag of size frag_size) - copy whole frag - | | - v v - [ D0 + dst_off ] [ D1 + dst_off ] -> (destination frags) - - Contiguous-layer assumption: - This mapper assumes that ``transfer_layers`` consecutive layers - starting at ``src_layer_off`` (and ``dst_layer_off``) are laid out - contiguously within each slot. This holds because - ``buffer_attributes()`` in the storage config assigns buffer - offsets sequentially from 0 for each layer_group (life cycle), - and each PoolDescriptor only contains layers belonging to a single - layer_group. Even when multiple layer_groups share the same - physical storage pool_group, each layer_group independently - occupies the full slot (offsets start from 0), so the contiguous - layout is preserved. +class ReplicatedMapper(IntactMapper): + """Copy TP-replicated per-layer regions without KV-head remapping. + + Every TP rank holds identical bytes per layer (MiniMax M3 index-key, DSA + indexer K), so no head slicing applies. Layer selection under partial PP + overlap happens through the per-layer offsets, and fan-in routing (one + owning sender per destination) is decided upstream by + ``PeerRegistrar.should_send_pool``. """ def __init__( self, - transfer_layers: int, - src_layer_off: int, - dst_layer_off: int, - self_ri: RankInfo, - peer_ri: RankInfo, - slot_size_per_layer: int, - ): - if not isinstance(slot_size_per_layer, int): - raise TypeError( - f"slot_size_per_layer must be int, got {type(slot_size_per_layer).__name__} " - f"(value={slot_size_per_layer}). Use // instead of / for integer division." - ) - self._kv_factor = self_ri.attention.kv_factor - self._frag_size = self._block_size(transfer_layers, slot_size_per_layer=slot_size_per_layer) - self._src_block_off = self._block_size( - src_layer_off, slot_size_per_layer=slot_size_per_layer - ) - self._dst_block_off = self._block_size( - dst_layer_off, slot_size_per_layer=slot_size_per_layer + src_layer_offsets: Sequence[int] | np.ndarray, + dst_layer_offsets: Sequence[int] | np.ndarray, + self_bytes_per_layer: int, + peer_bytes_per_layer: int, + ) -> None: + super().__init__( + src_layer_offsets, + dst_layer_offsets, + self_bytes_per_layer, + peer_bytes_per_layer, + mapper_name="Replicated", ) - @nvtx_range("HeadMatchMapper.map") - def map(self, src_regions: SpecRegion, dst_regions: SpecRegion) -> SpecRegionPair: - src_group = src_regions.memory - dst_group = dst_regions.memory - assert src_group.ptrs.size == dst_group.ptrs.size, ( - f"Number of regions of src({src_group.ptrs.size}) and dst({dst_group.ptrs.size}) must match" - ) - new_src_ptrs = src_group.ptrs + self._src_block_off - new_dst_ptrs = dst_group.ptrs + self._dst_block_off - new_src = MemRegionGroup(ptrs=new_src_ptrs, bytes_per_region=self._frag_size) - new_dst = MemRegionGroup(ptrs=new_dst_ptrs, bytes_per_region=self._frag_size) - return SpecRegionPair( - src=SpecRegion(memory=new_src, spec=src_regions.spec), - dst=SpecRegion(memory=new_dst, spec=dst_regions.spec), - ) - - def _block_size(self, layer_num: int, slot_size_per_layer: int) -> int: - return layer_num * slot_size_per_layer - -class HeadMismatchMapper(RegionMapperBase): +class HNDHeadMismatchMapper(RegionMapperBase): """ - ---- mapper_head_mismatch ---- + ---- mapper_hnd_head_mismatch ---- Fine-grained mapping when head counts or TP/DP partitioning differ. Split layers into per-head (or contiguous-heads) fragments and map them individually. @@ -134,26 +168,60 @@ class HeadMismatchMapper(RegionMapperBase): def __init__( self, - transfer_layers: int, - src_layer_off: int, - peer_layer_off: int, + *, + src_layer_offsets: "Sequence[int] | np.ndarray", + dst_layer_offsets: "Sequence[int] | np.ndarray", self_ri: RankInfo, peer_ri: RankInfo, + self_bytes_per_layer: int, + peer_bytes_per_layer: int, + self_buffers_per_layer: int, + peer_buffers_per_layer: int, ): self._ri = self_ri self._peer_ri = peer_ri - kv_factor = self_ri.attention.kv_factor self_tp_per_dp = self_ri.tp_size_per_dp_group peer_tp_per_dp = peer_ri.tp_size_per_dp_group self_tp_rank = self_ri.tp_rank peer_tp_rank = peer_ri.tp_rank - bytes_per_head = ( - self._ri.attention.tokens_per_block - * self._ri.attention.dims_per_head - * self._ri.attention.element_bytes + if self_buffers_per_layer != peer_buffers_per_layer: + raise ValueError( + "HND buffer count per layer mismatch: " + f"local={self_buffers_per_layer}, peer={peer_buffers_per_layer}" + ) + src_offsets = np.asarray(src_layer_offsets, dtype=np.int64) + dst_offsets = np.asarray(dst_layer_offsets, dtype=np.int64) + if src_offsets.size == 0 or src_offsets.size != dst_offsets.size: + raise ValueError( + "HND layer offsets must be non-empty and equal-length: " + f"src={src_offsets.size}, dst={dst_offsets.size}" + ) + + # Byte geometry is derived from the per-layer region size registered + # by storage (always whole bytes) rather than element_bytes x dims + # arithmetic, so sub-byte dtypes (e.g. NVFP4) stay exact-integer. + src_buffer_bytes = self._bytes_per_buffer( + bytes_per_layer=self_bytes_per_layer, + buffers_per_layer=self_buffers_per_layer, + side="local", + ) + dst_buffer_bytes = self._bytes_per_buffer( + bytes_per_layer=peer_bytes_per_layer, + buffers_per_layer=peer_buffers_per_layer, + side="peer", + ) + bytes_per_head = self._bytes_per_head( + src_buffer_bytes, self._ri.attention.kv_heads_per_rank, side="local" ) + peer_bytes_per_head = self._bytes_per_head( + dst_buffer_bytes, peer_ri.attention.kv_heads_per_rank, side="peer" + ) + if bytes_per_head != peer_bytes_per_head: + raise ValueError( + f"HND bytes per head mismatch: local={bytes_per_head}, peer={peer_bytes_per_head}" + ) self._bytes_cont_heads = ( min(self._ri.attention.kv_heads_per_rank, peer_ri.attention.kv_heads_per_rank) * bytes_per_head @@ -168,55 +236,41 @@ def __init__( peer_kv_heads=peer_ri.attention.kv_heads_per_rank, bytes_per_head=bytes_per_head, ) - self._peer_layer_off = peer_layer_off - - # --- Pre-compute flat 1D offset arrays --- - # - # Each KV cache block (slot) is laid out as: - # - # block_base ──► [layer_0 kv_0] [layer_0 kv_1] [layer_1 kv_0] [layer_1 kv_1] ... - # ◄─ layer_kv ─► ◄─ layer_kv ─► - # ◄────── layer_num (= layer_kv * kv_factor) ──────► - # - # To address fragment (layer=j, kv=k) within a block at base_ptr: - # - # frag_ptr = base_ptr - # + layer_num * (layer_off + j) # skip to the right layer - # + layer_kv * k # skip to key or value - # + head_off # head offset for TP mismatch - # - # The original code computed this as a 3D broadcast in map(): - # bases[:, None, None] + layer_offsets[None, :, None] - # + kv_offsets[None, None, :] + head_off - # producing shape (n_blocks, transfer_layers, kv_factor) then .ravel(). - # - # Optimization: since the (layer, kv) offsets are independent of the - # per-call block base pointers, we pre-compute them here as a flat 1D - # array of length (transfer_layers * kv_factor). At map() time we only - # need np.add.outer(bases, flat_offsets).ravel(), which produces the - # same result in the same C-order traversal (blocks outer, offsets inner) - # but with fewer intermediate allocations. - layer_indices = np.arange(transfer_layers, dtype=np.int64) - kv_indices = np.arange(kv_factor, dtype=np.int64) - - src_layer_kv_num = self._get_layer_kv_num(self._ri) - src_layer_num = src_layer_kv_num * kv_factor - # Shape (transfer_layers, kv_factor) → ravel to 1D - self._src_flat_offsets = ( - src_layer_num * (src_layer_off + layer_indices)[:, None] - + src_layer_kv_num * kv_indices[None, :] - + self._src_head_off - ).ravel() - dst_layer_kv_num = self._get_layer_kv_num(self._peer_ri) - dst_layer_num = dst_layer_kv_num * kv_factor - self._dst_flat_offsets = ( - dst_layer_num * (peer_layer_off + layer_indices)[:, None] - + dst_layer_kv_num * kv_indices[None, :] - + self._dst_head_off + # Pre-compute flat 1D offset arrays: one fragment per (layer, buffer) + # where buffers are the layer's K/V (or scale) buffers laid out + # back-to-back within the layer's region. Layer starts come from the + # view's buffer entries, so interleaved role classes (non-uniform + # layer strides) are handled the same way as everywhere else. At + # map() time a single np.add.outer(bases, flat_offsets) expands the + # per-block base pointers. + self._src_flat_offsets = self._build_flat_offsets( + layer_offsets=src_offsets, + buffers_per_layer=self_buffers_per_layer, + buffer_bytes=src_buffer_bytes, + head_offset=self._src_head_off, + ) + self._dst_flat_offsets = self._build_flat_offsets( + layer_offsets=dst_offsets, + buffers_per_layer=peer_buffers_per_layer, + buffer_bytes=dst_buffer_bytes, + head_offset=self._dst_head_off, + ) + + @staticmethod + def _build_flat_offsets( + *, + layer_offsets: np.ndarray, + buffers_per_layer: int, + buffer_bytes: int, + head_offset: int, + ) -> np.ndarray: + buffer_indices = np.arange(buffers_per_layer, dtype=np.int64) + return ( + layer_offsets[:, None] + buffer_bytes * buffer_indices[None, :] + head_offset ).ravel() - @nvtx_range("HeadMismatchMapper.map") + @nvtx_range("HNDHeadMismatchMapper.map") def map(self, src_regions: SpecRegion, dst_regions: SpecRegion) -> SpecRegionPair: src_group = src_regions.memory dst_group = dst_regions.memory @@ -261,57 +315,167 @@ def _compute_head_offsets( return 0, dst_head_idx * bytes_per_head @staticmethod - def _get_layer_kv_num(ri: RankInfo) -> int: - return ( - ri.attention.kv_heads_per_rank - * ri.attention.tokens_per_block - * ri.attention.dims_per_head - * ri.attention.element_bytes - ) + def _bytes_per_buffer(*, bytes_per_layer: int, buffers_per_layer: int, side: str) -> int: + """Bytes of one buffer (K or V) within a layer's region.""" + if buffers_per_layer <= 0 or bytes_per_layer % buffers_per_layer != 0: + raise ValueError( + f"HND layer geometry is not evenly divisible ({side}): " + f"bytes_per_layer={bytes_per_layer}, buffers_per_layer={buffers_per_layer}" + ) + return bytes_per_layer // buffers_per_layer + @staticmethod + def _bytes_per_head(layer_kv_bytes: int, heads: int, *, side: str) -> int: + """Bytes of one head's rows within a K/V buffer. + + Byte-granular head slicing is only valid when a head lands on a byte + boundary; sub-byte dtypes (e.g. NVFP4) satisfy this whenever the + per-head element count covers whole bytes, which this divisibility + check enforces without any fractional arithmetic. + """ + if heads <= 0 or layer_kv_bytes % heads != 0: + raise ValueError( + f"HND head slicing is not byte-aligned ({side}): " + f"layer_kv_bytes={layer_kv_bytes}, kv_heads={heads}" + ) + return layer_kv_bytes // heads -class IndexerKCacheHeadMatchMapper(RegionMapperBase): - """ - Mapper for indexer K cache when head counts match. - Moves contiguous block(s) as a single chunk, aligned by block_size_per_layer, - with constant source/destination block offsets. +class NHDHeadMismatchMapper(HNDHeadMismatchMapper): + """Map heterogeneous KV heads stored token-major as ``[N, H, D]``. + + ``HNDHeadMismatchMapper`` selects one contiguous head range per K/V buffer, + which is correct for HND storage. In NHD storage, the selected head range + is contiguous only within one token, so this mapper emits one fragment per + ``(layer, K/V, token)``. Only offset precomputation differs from the + parent; the inherited :meth:`map` consumes ``_src_flat_offsets``, + ``_dst_flat_offsets``, and ``_bytes_cont_heads``. """ def __init__( self, - transfer_layers: int, - src_layer_off: int, - dst_layer_off: int, + *, + src_layer_offsets: Sequence[int] | np.ndarray, + dst_layer_offsets: Sequence[int] | np.ndarray, self_ri: RankInfo, peer_ri: RankInfo, - block_size_per_layer: int, - ): - if not isinstance(block_size_per_layer, int): - raise TypeError( - f"block_size_per_layer must be int, got {type(block_size_per_layer).__name__} " - f"(value={block_size_per_layer}). Use // instead of / for integer division." + self_bytes_per_layer: int, + peer_bytes_per_layer: int, + self_buffers_per_layer: int, + peer_buffers_per_layer: int, + ) -> None: + # Deliberately do not call HNDHeadMismatchMapper.__init__: its offsets + # assume HND-contiguous heads. Initialize the three attributes consumed + # by the inherited map() with NHD token-granular geometry instead. + self_tpb = self_ri.attention.tokens_per_block + peer_tpb = peer_ri.attention.tokens_per_block + if self_tpb != peer_tpb: + raise ValueError( + "NHDHeadMismatchMapper requires equal tokens_per_block; " + f"local={self_tpb}, peer={peer_tpb}" ) - self._frag_size = block_size_per_layer * transfer_layers - self._src_block_off = block_size_per_layer * src_layer_off - self._dst_block_off = block_size_per_layer * dst_layer_off - @nvtx_range("IndexerKCacheHeadMatchMapper.map") - def map(self, src_regions: SpecRegion, dst_regions: SpecRegion) -> SpecRegionPair: - src_group = src_regions.memory - dst_group = dst_regions.memory - assert src_group.ptrs.size == dst_group.ptrs.size, ( - f"Number of regions of src({src_group.ptrs.size}) and dst({dst_group.ptrs.size}) must match" + self_heads = self_ri.attention.kv_heads_per_rank + peer_heads = peer_ri.attention.kv_heads_per_rank + if self_buffers_per_layer != peer_buffers_per_layer: + raise ValueError( + "NHD buffer count per layer mismatch: " + f"local={self_buffers_per_layer}, peer={peer_buffers_per_layer}" + ) + src_offsets = np.asarray(src_layer_offsets, dtype=np.int64) + dst_offsets = np.asarray(dst_layer_offsets, dtype=np.int64) + if src_offsets.size == 0 or src_offsets.size != dst_offsets.size: + raise ValueError( + "NHD layer offsets must be non-empty and equal-length: " + f"src={src_offsets.size}, dst={dst_offsets.size}" + ) + + self_bytes_per_token_head = self._bytes_per_token_head( + bytes_per_layer=self_bytes_per_layer, + buffers_per_layer=self_buffers_per_layer, + tokens_per_block=self_tpb, + heads=self_heads, ) - new_src_ptrs = src_group.ptrs + self._src_block_off - new_dst_ptrs = dst_group.ptrs + self._dst_block_off - new_src = MemRegionGroup(ptrs=new_src_ptrs, bytes_per_region=self._frag_size) - new_dst = MemRegionGroup(ptrs=new_dst_ptrs, bytes_per_region=self._frag_size) - return SpecRegionPair( - src=SpecRegion(memory=new_src, spec=src_regions.spec), - dst=SpecRegion(memory=new_dst, spec=dst_regions.spec), + peer_bytes_per_token_head = self._bytes_per_token_head( + bytes_per_layer=peer_bytes_per_layer, + buffers_per_layer=peer_buffers_per_layer, + tokens_per_block=peer_tpb, + heads=peer_heads, + ) + if self_bytes_per_token_head != peer_bytes_per_token_head: + raise ValueError( + "NHD bytes per token/head mismatch: " + f"local={self_bytes_per_token_head}, peer={peer_bytes_per_token_head}" + ) + self._bytes_cont_heads = min(self_heads, peer_heads) * self_bytes_per_token_head + + src_head_off, dst_head_off = HNDHeadMismatchMapper._compute_head_offsets( + self_ri.tp_size_per_dp_group, + peer_ri.tp_size_per_dp_group, + self_ri.tp_rank, + peer_ri.tp_rank, + self_kv_heads=self_heads, + peer_kv_heads=peer_heads, + bytes_per_head=self_bytes_per_token_head, ) + self._src_flat_offsets = self._build_flat_offsets( + layer_offsets=src_offsets, + buffers_per_layer=self_buffers_per_layer, + tokens_per_block=self_tpb, + heads=self_heads, + bytes_per_token_head=self_bytes_per_token_head, + head_offset=src_head_off, + ) + self._dst_flat_offsets = self._build_flat_offsets( + layer_offsets=dst_offsets, + buffers_per_layer=peer_buffers_per_layer, + tokens_per_block=peer_tpb, + heads=peer_heads, + bytes_per_token_head=peer_bytes_per_token_head, + head_offset=dst_head_off, + ) + + @staticmethod + def _bytes_per_token_head( + *, + bytes_per_layer: int, + buffers_per_layer: int, + tokens_per_block: int, + heads: int, + ) -> int: + denominator = buffers_per_layer * tokens_per_block * heads + if denominator <= 0 or bytes_per_layer % denominator != 0: + raise ValueError( + "NHD region geometry is not evenly divisible: " + f"bytes_per_layer={bytes_per_layer}, " + f"buffers_per_layer={buffers_per_layer}, " + f"tokens_per_block={tokens_per_block}, kv_heads={heads}, " + f"denominator={denominator}" + ) + return bytes_per_layer // denominator + + @staticmethod + def _build_flat_offsets( + *, + layer_offsets: np.ndarray, + buffers_per_layer: int, + tokens_per_block: int, + heads: int, + bytes_per_token_head: int, + head_offset: int, + ) -> np.ndarray: + buffer_indices = np.arange(buffers_per_layer, dtype=np.int64) + token_indices = np.arange(tokens_per_block, dtype=np.int64) + token_bytes = heads * bytes_per_token_head + buffer_bytes = tokens_per_block * token_bytes + return ( + layer_offsets[:, None, None] + + buffer_bytes * buffer_indices[None, :, None] + + token_bytes * token_indices[None, None, :] + + head_offset + ).ravel() + class AttentionPolicy: def __init__(self, self_rank_info: RankInfo): @@ -335,9 +499,29 @@ def _mismatch(self, field: str, local, peer) -> bool: local != peer, f"{field} mismatch", field=field, local=local, peer=peer ) - def _tpb_check(self, local: int, peer: int) -> bool: + @staticmethod + def _uses_exact_tpb_mapper(ri: RankInfo) -> bool: + """NHD / replicated pools address bytes inside a block, so their + geometry only lines up when both sides use the same tokens_per_block.""" + if ri.page_table is None: + return False + return any( + pool_view.mapper_kind in (MapperKind.NHD, MapperKind.REPLICATED) + for layer_group in ri.page_table.layer_groups + for pool_view in getattr(layer_group, "pool_views", ()) + ) + + def _tpb_check(self, local: int, peer: int, peer_ri: RankInfo) -> bool: if local == peer: return False + if self._uses_exact_tpb_mapper(self._ri) or self._uses_exact_tpb_mapper(peer_ri): + logger.warning( + "AttentionPolicy: incompatible: tokens_per_block mismatch for " + "NHD/replicated pools; local=%d peer=%d", + local, + peer, + ) + return True larger, smaller = max(local, peer), min(local, peer) if larger % smaller != 0: logger.warning( @@ -367,7 +551,15 @@ def check_peer_compatible(self, peer_ri: RankInfo) -> bool: peer=peer_ri.cp_size, ) or self._mismatch("element_bytes", a.element_bytes, b.element_bytes) - or self._tpb_check(a.tokens_per_block, b.tokens_per_block) + or self._fail_if( + not self.head_match(peer_ri)[0] + and not float(a.tokens_per_block * a.dims_per_head * a.element_bytes).is_integer(), + "sub-byte head slicing is not byte-aligned", + tokens_per_block=a.tokens_per_block, + dims_per_head=a.dims_per_head, + element_bytes=a.element_bytes, + ) + or self._tpb_check(a.tokens_per_block, b.tokens_per_block, peer_ri) or self._mismatch("dims_per_head", a.dims_per_head, b.dims_per_head) or self._fail_if( a.is_mla and (a.kv_heads_per_rank != 1 or b.kv_heads_per_rank != 1), @@ -395,6 +587,11 @@ def head_match(self, peer_ri: RankInfo) -> tuple[bool, bool]: return head_match, is_dup_head def duplicate_head_factors(self, peer_ri: RankInfo) -> tuple[int, int]: + # Head duplication only applies to attention KV. In particular, do + # not divide 0 / 0 for an attention-free hybrid PP stage; MambaPolicy + # computes its TP mapping separately. + if self._ri.attention.kv_heads_per_rank == 0 or peer_ri.attention.kv_heads_per_rank == 0: + return 1, 1 factor_self, factor_peer = self._head_factors(peer_ri) dup_head = max(1, factor_self // factor_peer) peer_dup_head = max(1, factor_peer // factor_self) @@ -405,53 +602,66 @@ def build_kv_mapper( *, peer_ri: RankInfo, mapper_kind: MapperKind, - transfer_layers: int, - self_layer_offset: int, - peer_layer_offset: int, - self_pool_num_layers: int, - peer_pool_num_layers: int, - self_pool_slot_bytes: int, - peer_pool_slot_bytes: int, + self_layer_offsets: "Sequence[int] | np.ndarray", + peer_layer_offsets: "Sequence[int] | np.ndarray", + self_bytes_per_layer: int, + peer_bytes_per_layer: int, + self_buffers_per_layer: int = 1, + peer_buffers_per_layer: int = 1, ) -> RegionMapperBase: - head_match, _ = self.head_match(peer_ri) - - if head_match and transfer_layers == self_pool_num_layers == peer_pool_num_layers: - return IdentityMapper() + """Pick the mapper for one view pair. + + Every view is entries-driven: layer selection always uses explicit + slot-relative per-layer byte offsets, never positional arithmetic + (other role classes may interleave between this class's layers). + The kind only decides the two irreducible semantic differences: + + - REPLICATED skips head matching entirely (bytes are identical on + every TP rank; fan-in ownership is decided upstream). + - Under head mismatch, HND (INDEXED) slices one contiguous head + range per K/V buffer, while NHD must slice inside every token. + + Head-matched views of any kind collapse into IntactMapper, + whose run merging degrades to a single whole-region copy per block + when the selected layers are contiguous on both sides. + """ + if mapper_kind == MapperKind.REPLICATED: + return ReplicatedMapper( + self_layer_offsets, + peer_layer_offsets, + self_bytes_per_layer, + peer_bytes_per_layer, + ) + head_match, _ = self.head_match(peer_ri) if head_match: - if mapper_kind == MapperKind.FLAT: - block_size_per_layer = self_pool_slot_bytes // self_pool_num_layers - return IndexerKCacheHeadMatchMapper( - transfer_layers=transfer_layers, - src_layer_off=self_layer_offset, - dst_layer_off=peer_layer_offset, - self_ri=self._ri, - peer_ri=peer_ri, - block_size_per_layer=block_size_per_layer, - ) - - slot_size_per_layer = self_pool_slot_bytes // self_pool_num_layers - peer_size_per_layer = peer_pool_slot_bytes // peer_pool_num_layers - assert slot_size_per_layer == peer_size_per_layer, ( - f"slot_size_per_layer mismatch between self ({slot_size_per_layer}) " - f"and peer ({peer_size_per_layer}) for HeadMatchMapper" + return IntactMapper( + self_layer_offsets, + peer_layer_offsets, + self_bytes_per_layer, + peer_bytes_per_layer, + mapper_name=mapper_kind.name, ) - return HeadMatchMapper( - transfer_layers=transfer_layers, - src_layer_off=self_layer_offset, - dst_layer_off=peer_layer_offset, + + if mapper_kind == MapperKind.NHD: + return NHDHeadMismatchMapper( + src_layer_offsets=self_layer_offsets, + dst_layer_offsets=peer_layer_offsets, self_ri=self._ri, peer_ri=peer_ri, - slot_size_per_layer=slot_size_per_layer, + self_bytes_per_layer=self_bytes_per_layer, + peer_bytes_per_layer=peer_bytes_per_layer, + self_buffers_per_layer=self_buffers_per_layer, + peer_buffers_per_layer=peer_buffers_per_layer, ) - if mapper_kind == MapperKind.FLAT: - raise ValueError("IndexerKCacheHeadMatchMapper is not supported for head mismatch case") - - return HeadMismatchMapper( - transfer_layers=transfer_layers, - src_layer_off=self_layer_offset, - peer_layer_off=peer_layer_offset, + return HNDHeadMismatchMapper( + src_layer_offsets=self_layer_offsets, + dst_layer_offsets=peer_layer_offsets, self_ri=self._ri, peer_ri=peer_ri, + self_bytes_per_layer=self_bytes_per_layer, + peer_bytes_per_layer=peer_bytes_per_layer, + self_buffers_per_layer=self_buffers_per_layer, + peer_buffers_per_layer=peer_buffers_per_layer, ) diff --git a/tensorrt_llm/_torch/disaggregation/native/mixers/attention/spec.py b/tensorrt_llm/_torch/disaggregation/native/mixers/attention/spec.py index 4c3db7cc8638..931eef38dc16 100644 --- a/tensorrt_llm/_torch/disaggregation/native/mixers/attention/spec.py +++ b/tensorrt_llm/_torch/disaggregation/native/mixers/attention/spec.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from dataclasses import asdict, dataclass @@ -7,7 +22,7 @@ class AttentionInfo: kv_heads_per_rank: int tokens_per_block: int dims_per_head: int - element_bytes: int + element_bytes: int | float enable_attention_dp: bool is_mla: bool diff --git a/tensorrt_llm/_torch/disaggregation/native/mixers/ssm/peer.py b/tensorrt_llm/_torch/disaggregation/native/mixers/ssm/peer.py index a1be935fc9ec..1e55deae36e9 100644 --- a/tensorrt_llm/_torch/disaggregation/native/mixers/ssm/peer.py +++ b/tensorrt_llm/_torch/disaggregation/native/mixers/ssm/peer.py @@ -1,3 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from typing import Dict, List, Optional, Tuple import numpy as np @@ -403,7 +417,9 @@ def _check_global(field: str, self_bytes: int, peer_bytes: int) -> None: "enabled on both sides." ) - _check_global("ssm slot_bytes", self_mlg.ssm_states.slot_bytes, peer_mlg.ssm_states.slot_bytes) + _check_global( + "ssm slot_bytes", self_mlg.ssm_states.slot_bytes, peer_mlg.ssm_states.slot_bytes + ) _check_global( "conv slot_bytes", self_mlg.conv_states.slot_bytes, peer_mlg.conv_states.slot_bytes ) @@ -437,13 +453,17 @@ def _build_layer_ptrs( overlapping_layers: List[int], slot: int, ) -> np.ndarray: - """Build per-layer pointers for a given pool (conv or ssm) and slot.""" - ptrs = [] - for glid in overlapping_layers: - lid = layer_offsets[glid] - ptrs.append( - pool.base_address + lid * pool.num_slots * pool.slot_bytes + slot * pool.slot_bytes - ) + """Build per-layer pointers from a pool's affine layer/slot layout.""" + slot_stride_bytes = pool.slot_stride_bytes + layer_stride_bytes = pool.layer_stride_bytes + assert slot_stride_bytes is not None + assert layer_stride_bytes is not None + ptrs = [ + pool.base_address + + layer_offsets[global_layer_id] * layer_stride_bytes + + slot * slot_stride_bytes + for global_layer_id in overlapping_layers + ] return np.array(ptrs, dtype=np.int64) @staticmethod @@ -530,10 +550,16 @@ def build_mamba_frags( (self_mlg.ssm_states, peer_mlg.ssm_states, False), ]: src_ptrs = MambaPolicy._build_layer_ptrs( - self_pool, self_mlg.mamba_layer_offsets, overlapping_layers, src_slot + self_pool, + self_mlg.mamba_layer_offsets, + overlapping_layers, + src_slot, ) dst_ptrs = MambaPolicy._build_layer_ptrs( - peer_pool, peer_mlg.mamba_layer_offsets, overlapping_layers, dst_slot + peer_pool, + peer_mlg.mamba_layer_offsets, + overlapping_layers, + dst_slot, ) src_region = SpecRegion( diff --git a/tensorrt_llm/_torch/disaggregation/native/peer.py b/tensorrt_llm/_torch/disaggregation/native/peer.py index 44d2a54ac5c4..569fc2125e90 100644 --- a/tensorrt_llm/_torch/disaggregation/native/peer.py +++ b/tensorrt_llm/_torch/disaggregation/native/peer.py @@ -1,20 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import Counter from dataclasses import dataclass, field from typing import Dict, List, Tuple +import numpy as np + from tensorrt_llm import logger from tensorrt_llm._torch.disaggregation.base.region import RegionMapperBase from tensorrt_llm._torch.disaggregation.native.mixers.attention.peer import AttentionPolicy from tensorrt_llm._torch.disaggregation.native.mixers.ssm.peer import MambaPolicy from tensorrt_llm._torch.disaggregation.native.rank_info import RankInfo from tensorrt_llm._torch.disaggregation.resource.kv_extractor import KVRegionExtractorV1 -from tensorrt_llm._torch.disaggregation.resource.page import AttentionLayerGroup, MapperKind +from tensorrt_llm._torch.disaggregation.resource.page import ( + AttentionLayerGroup, + MapperKind, + PoolView, +) from tensorrt_llm._torch.disaggregation.resource.utils import ( - get_global_layer_ids, - get_layer_group_num_layers, + get_layer_byte_ranges, get_layer_to_layer_group, - get_physical_pool, get_pool_view_global_layer_ids, - get_pool_view_num_layers, ) # Type alias for (lg_idx, pool_idx) pair @@ -58,6 +77,33 @@ def register(self, peer_name: str, peer_rank: int, peer_ri: RankInfo): extractor = KVRegionExtractorV1(peer_ri.page_table) self._peer_ext_cache[key] = extractor + head_match, _ = self._attention_policy.head_match(peer_ri) + if not head_match: + self_page_table = self._self_ext_cache.page_table + nhd_fragments_per_token = sum( + len( + self_page_table.layer_groups[layer_group_id].pool_views[pool_idx].buffer_entries + ) + for layer_group_id, pool_idx in self.get_pool_mapping(peer_ri) + if self_page_table.layer_groups[layer_group_id].pool_views[pool_idx].mapper_kind + == MapperKind.NHD + ) + if nhd_fragments_per_token: + local_heads = self._ri.attention.kv_heads_per_rank + peer_heads = peer_ri.attention.kv_heads_per_rank + logger.warning_once( + "NHD head-mismatched disaggregated KV transfer has no " + "contiguous staging path and will emit approximately " + f"{nhd_fragments_per_token} NIXL descriptors per transferred " + "token per peer, excluding block-level replicated pools " + f"(local_kv_heads={local_heads}, peer_kv_heads={peer_heads}). " + "Long-context TEP/DEP transfers may have high latency.", + key=( + "native-nhd-head-mismatch-" + f"{local_heads}-{peer_heads}-{nhd_fragments_per_token}" + ), + ) + def peer_extractor(self, peer_name: str, peer_rank: int) -> KVRegionExtractorV1: return self._peer_ext_cache[self._unique_key(peer_name, peer_rank)] @@ -129,6 +175,13 @@ def get_pool_mapping(self, peer_ri: RankInfo) -> Dict[LGPoolKey, LGPoolKey]: Layer-overlap is required: a peer pool with the same pool_role but zero layer overlap with self is *not* a match — the two pools cover disjoint layers and have nothing to transfer. + + A self layer group never matches multiple peer layer groups, so the + result is one peer pool per self pool. Layer groups partition each + rank's layers by attention/life-cycle class, which both sides derive + from the same model config; PP only changes which layers overlap (the + fan-out across peer PP ranks is handled by calling this method once + per peer rank). Step 1 raises if this invariant is ever violated. """ key = self._unique_key(peer_ri.instance_name, peer_ri.instance_rank) if key in self._lg_pool_mapping_cache: @@ -151,27 +204,40 @@ def get_pool_mapping(self, peer_ri: RankInfo) -> Dict[LGPoolKey, LGPoolKey]: if not isinstance(self_lg, AttentionLayerGroup): continue for self_pi, self_pv in enumerate(self_lg.pool_views): - # The only place mapper_kind affects pool matching: - # INDEXED → pool may cover a subset of the LG; read - # buffer_entries to find the exact layer set. - # FLAT → pool covers the entire LG by convention; - # use the LG's layer ids directly. - self_is_flat = self_pv.mapper_kind == MapperKind.FLAT - pv_global_ids = ( - get_global_layer_ids(self_lg) - if self_is_flat - else get_pool_view_global_layer_ids(self_pv, self_lg) - ) + # Every view carries buffer_entries, so a view's exact layer + # set always comes from its entries (a view may cover a + # subset of the LG when V2 splits an LG into multiple pools + # by buffer-size class, or when a role class exists only on + # some layers, e.g. sparse-layer index-K). + pv_global_ids = get_pool_view_global_layer_ids(self_pv, self_lg) if not pv_global_ids: continue - # Step 1: find peer layer_group via any overlapping global_layer_id. - peer_lg_idx = next( - (peer_layer_to_group[g] for g in pv_global_ids if g in peer_layer_to_group), - None, - ) - if peer_lg_idx is None: + # Step 1: find the peer layer_group via overlapping global_layer_ids. + # A self layer group (hence each of its pool views) never matches + # multiple peer layer groups: layer groups partition a rank's + # layers by attention/life-cycle class, both sides derive that + # class from the same model config, and global ids are + # PP-invariant. So PP only changes WHICH layers overlap — layers + # the peer doesn't hold are a legal skip (PP slices, one-sided + # MTP layers) — never how many peer LGs they land in; the PP + # fan-out is handled by per-peer-rank calls of this method. A + # multi-LG hit therefore means the two peers group layers + # differently (unsupported topology), and we fail loudly instead + # of silently transferring only the first LG's overlap. + peer_lg_indices = { + peer_layer_to_group[g] for g in pv_global_ids if g in peer_layer_to_group + } + if not peer_lg_indices: continue + if len(peer_lg_indices) > 1: + raise ValueError( + "PeerRegistrar.get_pool_mapping: pool view " + f"(lg={self_lg_idx}, pool={self_pi}) spans multiple peer " + f"layer groups {sorted(peer_lg_indices)}; mismatched layer " + "grouping between peers is not supported" + ) + peer_lg_idx = next(iter(peer_lg_indices)) peer_lg = peer_pt.layer_groups[peer_lg_idx] # Step 2: pick the first peer pool with the same pool_role @@ -192,11 +258,7 @@ def get_pool_mapping(self, peer_ri: RankInfo) -> Dict[LGPoolKey, LGPoolKey]: for peer_pi, peer_pv in enumerate(peer_lg.pool_views): if peer_pv.pool_role != self_pv.pool_role: continue - peer_global_ids = ( - get_global_layer_ids(peer_lg) - if peer_pv.mapper_kind == MapperKind.FLAT - else get_pool_view_global_layer_ids(peer_pv, peer_lg) - ) + peer_global_ids = get_pool_view_global_layer_ids(peer_pv, peer_lg) if not set(peer_global_ids) & self_layer_set: continue if peer_pv.mapper_kind != self_pv.mapper_kind: @@ -251,53 +313,89 @@ def get_kv_map( f"(local={self_pv.mapper_kind.name}, peer={peer_pv.mapper_kind.name})" ) - # FLAT pools carry no per-buffer layer info, so layer ids and - # layer count come from the layer_group itself. - # - # Sort by global_layer_id so that ``.index(first_overlap_layer)`` - # below returns the layer's slot position. This relies on the - # convention that managers (V1 / V2 / DSv4) assign global_layer_id - # monotonically with the layer's byte offset in the slot. - if self_pv.mapper_kind == MapperKind.FLAT: - self_global_ids = sorted(get_global_layer_ids(self_lg)) - peer_global_ids = sorted(get_global_layer_ids(peer_lg)) - self_num_layers = get_layer_group_num_layers(self_lg) - peer_num_layers = get_layer_group_num_layers(peer_lg) - else: - self_global_ids = sorted(get_pool_view_global_layer_ids(self_pv, self_lg)) - peer_global_ids = sorted(get_pool_view_global_layer_ids(peer_pv, peer_lg)) - self_num_layers = get_pool_view_num_layers(self_pv) - peer_num_layers = get_pool_view_num_layers(peer_pv) - - overlapping_layers = sorted(set(self_global_ids) & set(peer_global_ids)) - transfer_layers = len(overlapping_layers) - - if transfer_layers > 0: - first_overlap_layer = overlapping_layers[0] - self_layer_offset = self_global_ids.index(first_overlap_layer) - peer_layer_offset = peer_global_ids.index(first_overlap_layer) - else: - self_layer_offset = 0 - peer_layer_offset = 0 - - self_phys = get_physical_pool(self_pt, self_lg_idx, self_pv.pool_idx) - peer_phys = get_physical_pool(peer_pt, peer_lg_idx, peer_pv.pool_idx) + # Every view is entries-driven: resolve the overlap layers to + # slot-relative byte offsets on each side from the views' buffer + # entries. Layer selection is explicit, so mappers never assume a + # uniform layer stride (other role classes may interleave), and no + # convention about global-id/byte-offset ordering is needed. + self_global_ids = get_pool_view_global_layer_ids(self_pv, self_lg) + peer_global_ids = get_pool_view_global_layer_ids(peer_pv, peer_lg) + # Iterate the overlap in self's physical slot order (not sorted by + # global id) so that layers whose regions are contiguous on both + # sides stay adjacent in the offset arrays and the mappers can merge + # them into one fragment even when global-id order diverges from the + # physical layout. Order only affects run merging (a perf property): + # each layer's byte offset is looked up explicitly below, so any + # iteration order transfers correct bytes. + overlap = set(self_global_ids) & set(peer_global_ids) + overlapping_layers = [gid for gid in self_global_ids if gid in overlap] + + self_starts, self_bytes_per_layer = get_layer_byte_ranges(self_pv) + peer_starts, peer_bytes_per_layer = get_layer_byte_ranges(peer_pv) + self_g2l = {ll.global_layer_id: ll.local_layer_id for ll in self_lg.local_layers} + peer_g2l = {ll.global_layer_id: ll.local_layer_id for ll in peer_lg.local_layers} + self_layer_offsets = np.array( + [self_starts[self_g2l[gid]] for gid in overlapping_layers], dtype=np.int64 + ) + peer_layer_offsets = np.array( + [peer_starts[peer_g2l[gid]] for gid in overlapping_layers], dtype=np.int64 + ) + # Per-layer buffer count (K and V are separate buffers within a + # layer's region); head-mismatch mappers slice heads inside each. + self_buffers_per_layer = self._get_buffers_per_layer( + self_pv, + layer_group_id=self_lg_idx, + pool_idx=self_pi, + ) + peer_buffers_per_layer = self._get_buffers_per_layer( + peer_pv, + layer_group_id=peer_lg_idx, + pool_idx=peer_pi, + ) mapper = self._attention_policy.build_kv_mapper( peer_ri=peer_ri, mapper_kind=self_pv.mapper_kind, - transfer_layers=transfer_layers, - self_layer_offset=self_layer_offset, - peer_layer_offset=peer_layer_offset, - self_pool_num_layers=self_num_layers, - peer_pool_num_layers=peer_num_layers, - self_pool_slot_bytes=self_phys.slot_bytes, - peer_pool_slot_bytes=peer_phys.slot_bytes, + self_layer_offsets=self_layer_offsets, + peer_layer_offsets=peer_layer_offsets, + self_bytes_per_layer=self_bytes_per_layer, + peer_bytes_per_layer=peer_bytes_per_layer, + self_buffers_per_layer=self_buffers_per_layer, + peer_buffers_per_layer=peer_buffers_per_layer, ) self._kv_map_cache[cache_key] = mapper return mapper + @staticmethod + def _get_buffers_per_layer( + pool_view: PoolView, + *, + layer_group_id: int, + pool_idx: int, + ) -> int: + """Per-layer buffer count of a view (e.g. K+V -> 2, key-only -> 1). + + Views are bucketed per (layer group, pool, mapper kind) at page-table + build time, so every layer in a view carries the same role set and + hence the same entry count — a skewed distribution should never occur. + Still verify it per layer rather than via total-count divisibility: + e.g. 1 + 3 entries over two layers passes ``total % layers == 0`` yet + would make head-slicing mappers split every layer at wrong offsets. + """ + entries = pool_view.buffer_entries + if len(entries) == 0: + return 1 + counts = Counter(int(e["local_layer_id"]) for e in entries) + distinct = set(counts.values()) + if len(distinct) != 1: + raise ValueError( + "PoolView buffer entries are not evenly distributed across layers: " + f"layer_group={layer_group_id}, pool={pool_idx}, " + f"per-layer entry counts={sorted(counts.items())}" + ) + return distinct.pop() + @staticmethod def _find_overlap(self_val, peer_val, self_rank, peer_rank=None): if self_val <= peer_val: @@ -379,6 +477,44 @@ def should_send_kv(self, peer_overlap: PeerOverlap, peer_rank_info: RankInfo) -> self_tp_rank_in_dp_group % dup_head_factor ) + def _owns_tp_fan_in(self, peer_rank_info: RankInfo) -> bool: + """Elect one owner when replicated bytes fan in across TP ranks. + + A peer with fewer TP shards receives identical replicated data from + several local ranks. Rotate the elected owner by the destination's + DP rank (mirroring ``should_send_kv``'s head-duplication pairing) so + that with a multi-DP-group generation side the extra replicated + traffic spreads across local ranks instead of always landing on the + first rank of each fan-in group. + """ + ratio = max( + 1, + self._ri.tp_size_per_dp_group // peer_rank_info.tp_size_per_dp_group, + ) + self_tp_rank = self._ri.tp_rank % self._ri.tp_size_per_dp_group + return self_tp_rank % ratio == peer_rank_info.dp_rank % ratio + + def should_send_pool( + self, + peer_overlap: PeerOverlap, + peer_rank_info: RankInfo, + layer_group_id: int, + pool_idx: int, + ) -> bool: + """Return whether this rank owns the transfer of one view pair. + + ``pool_idx`` indexes the layer group's ``pool_views`` list (one view + per role class; several views may share a physical pool). Each view + is kind-homogeneous, so ownership is a single per-view decision: + replicated views use one sender per fan-in group, sharded views + retain head-duplication routing. + """ + layer_group = self._self_ext_cache.page_table.layer_groups[layer_group_id] + pool_view = layer_group.pool_views[pool_idx] + if pool_view.mapper_kind == MapperKind.REPLICATED: + return self._owns_tp_fan_in(peer_rank_info) + return self.should_send_kv(peer_overlap, peer_rank_info) + def should_send_aux(self, peer_rank_info: RankInfo) -> bool: # to ensure the transfer aux is not duplicated diff --git a/tensorrt_llm/_torch/disaggregation/native/perf_logger.py b/tensorrt_llm/_torch/disaggregation/native/perf_logger.py index 3c491df3495b..5528b8f1eb08 100644 --- a/tensorrt_llm/_torch/disaggregation/native/perf_logger.py +++ b/tensorrt_llm/_torch/disaggregation/native/perf_logger.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -130,11 +130,13 @@ def get_task_latency(self, peer_rank: int) -> float: class PerfLogManager: """Singleton manager for KV transfer performance logging. - Logic: - - TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO not set: no output - - TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO set, TLLM_KV_TRANSFER_PERF_LOG_FILE not set: - logger.info to stdout - - Both set: CSV output to {TLLM_KV_TRANSFER_PERF_LOG_FILE}_{instance_name}_{instance_rank}.csv + Logic (checked in priority order): + 1. TRTLLM_KVCACHE_TIME_OUTPUT_PATH set (C++ standard): enabled, CSV to + ``{path}/{instance_name}_{instance_rank}.csv`` + 2. TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO=1 with TLLM_KV_TRANSFER_PERF_LOG_FILE: + enabled, CSV to ``{base}_{instance_name}_{instance_rank}.csv`` + 3. TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO=1 alone: enabled, logger.info to stdout + 4. None of the above: disabled """ _instance = None @@ -154,8 +156,18 @@ def __init__(self): self._initialized = True self._file_loggers = {} # (instance_name, instance_rank) -> logger self._file_lock = threading.Lock() - self._perf_enabled = os.getenv("TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO", "0") == "1" - self._log_file_base = os.getenv("TLLM_KV_TRANSFER_PERF_LOG_FILE") + + # Primary: C++ standard env var (directory path) + cpp_output_path = os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH") + if cpp_output_path: + self._perf_enabled = True + self._log_file_base = cpp_output_path + self._use_cpp_naming = True + else: + # Fallback: existing Python env vars + self._perf_enabled = os.getenv("TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO", "0") == "1" + self._log_file_base = os.getenv("TLLM_KV_TRANSFER_PERF_LOG_FILE") + self._use_cpp_naming = False @property def enabled(self) -> bool: @@ -175,8 +187,12 @@ def _get_or_create_file_logger(self, instance_name: str, instance_rank: int): if key in self._file_loggers: return self._file_loggers[key] - # Create file path: {base}_{instance_name}_{instance_rank}.csv - log_file = f"{self._log_file_base}_{instance_name}_{instance_rank}.csv" + if self._use_cpp_naming: + # C++ pattern: {dir}/{instance_name}_{instance_rank}.csv + log_file = os.path.join(self._log_file_base, f"{instance_name}_{instance_rank}.csv") + else: + # Legacy Python pattern: {base}_{instance_name}_{instance_rank}.csv + log_file = f"{self._log_file_base}_{instance_name}_{instance_rank}.csv" try: # Create directory if needed @@ -315,6 +331,73 @@ def log_recv_task_perf( ) self.log(instance_name, instance_rank, csv_line, info_msg) + def log_gen_transfer_summary( + self, + unique_rid: int, + instance_name: str, + instance_rank: int, + gen_side_transfer_time_ms: float, + kv_cache_size: int, + ) -> None: + """Log a gen-side transfer summary row to a separate CSV. + + Written after timing sync across ranks so values are globally + consistent. Only active when ``TRTLLM_KVCACHE_TIME_OUTPUT_PATH`` + is set. + + Args: + unique_rid: Unique request id. + instance_name: Instance name for file naming. + instance_rank: Instance rank for file naming. + gen_side_transfer_time_ms: Synced gen-side transfer time in ms. + kv_cache_size: Total KV cache size across ranks (bytes). + """ + cpp_output_path = os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH") + if not cpp_output_path: + return + + _GEN_SUMMARY_HEADER = "timestamp,RequestID,gen_side_transfer_time(ms),kv_cache_size" + key = ("gen_summary", instance_name, instance_rank) + + if key not in self._file_loggers: + with self._file_lock: + if key not in self._file_loggers: + log_file = os.path.join( + cpp_output_path, + f"{instance_name}_{instance_rank}_gen_transfer_summary.csv", + ) + try: + log_dir = os.path.dirname(log_file) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + + write_header = not os.path.exists(log_file) + file_logger = logging.getLogger( + f"kv_gen_summary_{instance_name}_{instance_rank}" + ) + file_logger.setLevel(logging.INFO) + file_logger.propagate = False + file_handler = logging.FileHandler(log_file, mode="a", encoding="utf-8") + formatter = logging.Formatter( + fmt="%(asctime)s.%(msecs)03d,%(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + file_handler.setFormatter(formatter) + file_logger.addHandler(file_handler) + if write_header: + file_handler.stream.write(_GEN_SUMMARY_HEADER + "\n") + file_handler.stream.flush() + self._file_loggers[key] = file_logger + except OSError as e: + sys.stderr.write( + f"[KV Transfer] Warning: Failed to create gen summary log file {log_file}: {e}\n" + ) + return + + file_logger = self._file_loggers.get(key) + if file_logger: + file_logger.info(f"{unique_rid},{gen_side_transfer_time_ms:.3f},{kv_cache_size}") + # Singleton instance perf_log_manager = PerfLogManager() diff --git a/tensorrt_llm/_torch/disaggregation/native/rank_info.py b/tensorrt_llm/_torch/disaggregation/native/rank_info.py index 12a614ca2dad..e2803042c7fc 100644 --- a/tensorrt_llm/_torch/disaggregation/native/rank_info.py +++ b/tensorrt_llm/_torch/disaggregation/native/rank_info.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from dataclasses import asdict, dataclass from typing import List, Optional @@ -59,6 +74,19 @@ def from_kv_cache_manager( m = kv_cache_manager.mapping kvm = kv_cache_manager enable_attention_dp = m.enable_attention_dp + # Keep AttentionInfo on attention-free PP stages so it can still carry + # the attention-DP topology used by Mamba transfers. A zero head count + # means that this rank has no local attention cache; AttentionPolicy + # must not perform head-ratio arithmetic for such ranks. + kv_heads_per_rank = next((h for h in kvm.num_kv_heads_per_layer if h > 0), 0) + # Eight is the smallest element count guaranteed to occupy whole bytes + # for every supported sub-byte cache dtype (including NVFP4). + bytes_for_eight_elements = get_size_in_bytes(8, kvm.dtype) + element_bytes = ( + bytes_for_eight_elements // 8 + if bytes_for_eight_elements % 8 == 0 + else bytes_for_eight_elements / 8 + ) return cls( instance_name=instance_name, instance_rank=m.rank, @@ -77,10 +105,10 @@ def from_kv_cache_manager( self_endpoint="", transfer_engine_info=bytes(), attention=AttentionInfo( - kv_heads_per_rank=kvm.num_kv_heads_per_layer[0], + kv_heads_per_rank=kv_heads_per_rank, tokens_per_block=kvm.tokens_per_block, dims_per_head=kvm.head_dim, - element_bytes=get_size_in_bytes(1, kvm.dtype), + element_bytes=element_bytes, enable_attention_dp=enable_attention_dp, is_mla=kvm.kv_factor == 1, ), diff --git a/tensorrt_llm/_torch/disaggregation/native/transfer.py b/tensorrt_llm/_torch/disaggregation/native/transfer.py index 1f2d7243eb65..0a7664e6bcb9 100644 --- a/tensorrt_llm/_torch/disaggregation/native/transfer.py +++ b/tensorrt_llm/_torch/disaggregation/native/transfer.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import annotations import os @@ -51,6 +66,7 @@ from tensorrt_llm._torch.disaggregation.native.utils import get_local_ip from tensorrt_llm._torch.disaggregation.nixl.agent import NixlTransferAgent from tensorrt_llm._torch.disaggregation.resource.kv_extractor import KVRegionExtractorV1 +from tensorrt_llm._torch.disaggregation.resource.page import MapperKind from tensorrt_llm._torch.disaggregation.resource.utils import get_unique_pool_memory_descs from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager @@ -163,16 +179,16 @@ class AgentResult(Enum): FAILED = "FAILED" -# KV_AGENT_RESULT prefix in one struct frame (was 5 ascii frames serialized/parsed under the -# GIL per slice per writer): instance_rank, unique_rid, slice_id, is_last, status. The optional -# bounce tail follows at message[2:]. -_KV_RESULT_PREFIX = struct.Struct(" Write peer_extractor = self._registrar.peer_extractor( peer_ri.instance_name, peer_ri.instance_rank ) - if self._registrar.should_send_kv(targets, peer_ri): - pool_mapping = self._registrar.get_pool_mapping(peer_ri) - dst_block_ids_per_groups = req_info.block_ids_per_layer_groups - src_block_ids_per_groups = task._slice.block_ids_per_layer_groups - - # Aggregate fragments from all matching pools using numpy concatenation - for (self_lg, self_pi), (peer_lg, peer_pi) in pool_mapping.items(): - src_block_ids = src_block_ids_per_groups[self_lg] - dst_block_ids = dst_block_ids_per_groups[peer_lg] - - # Both sides trim block lists to ceil(prompt_len / tpb) in - # _create_kv_slice, so dst must never exceed src. A smaller dst - # (generation prefix-cache reuse) is handled via dst_start below. - block_diff = dst_block_ids.size - src_block_ids.size - if block_diff > 0: - raise ValueError( - f"src/dst block count mismatch: {src_block_ids.size} vs " - f"{dst_block_ids.size} (dst must not exceed src)" - ) - tpb = extractor.page_table.tokens_per_block - token_range = task._slice.token_range - lg_info = extractor.page_table.layer_groups[self_lg] - window_size = getattr(lg_info, "sliding_window_size", None) - - # Block lists are the suffix of [..., slice_end); cached prefix - # is implicit in their size. token_start = (total_blocks - n) * tpb. - slice_end = token_range.end if token_range is not None else 0 - total_blocks = (slice_end + tpb - 1) // tpb - src_beam0_blocks = Sender._beam0_block_count( - src_block_ids, total_blocks, task._beam_width - ) - dst_beam0_blocks = Sender._beam0_block_count( - dst_block_ids, total_blocks, task._beam_width - ) - assert src_beam0_blocks <= total_blocks, ( - f"src beam-0 block list ({src_beam0_blocks}) exceeds total slice " - f"blocks ({total_blocks}); slice_end={slice_end}, tpb={tpb}" - ) - assert dst_beam0_blocks <= total_blocks, ( - f"dst beam-0 block list ({dst_beam0_blocks}) exceeds total slice " - f"blocks ({total_blocks}); slice_end={slice_end}, tpb={tpb}" + pool_mapping = self._registrar.get_pool_mapping(peer_ri) + dst_block_ids_per_groups = req_info.block_ids_per_layer_groups + src_block_ids_per_groups = task._slice.block_ids_per_layer_groups + + # Aggregate fragments from all matching pools using numpy concatenation. + # Send ownership is per pool: replicated pools elect one fan-in + # owner, sharded pools keep head-duplication routing. + for (self_lg, self_pi), (peer_lg, peer_pi) in pool_mapping.items(): + if not self._registrar.should_send_pool(targets, peer_ri, self_lg, self_pi): + continue + src_block_ids = src_block_ids_per_groups[self_lg] + dst_block_ids = dst_block_ids_per_groups[peer_lg] + + # Both sides trim block lists to ceil(prompt_len / tpb) in + # _create_kv_slice, so dst must never exceed src. A smaller dst + # (generation prefix-cache reuse) is handled via dst_start below. + block_diff = dst_block_ids.size - src_block_ids.size + if block_diff > 0: + raise ValueError( + f"src/dst block count mismatch: {src_block_ids.size} vs " + f"{dst_block_ids.size} (dst must not exceed src)" ) - src_start = (total_blocks - src_beam0_blocks) * tpb - dst_start = (total_blocks - dst_beam0_blocks) * tpb - if req_info.dst_start_token is not None: - dst_start = max(dst_start, req_info.dst_start_token) - if window_size is not None: - # SWA stale_end uses the request prompt_len (not slice_end — - # they differ for non-final slices). prompt_len must be plumbed - # via the session; falling back to slice_end is wrong on - # non-final slices. - assert task._prompt_len is not None, ( - "SWA layer requires session.prompt_len; " - "set TxSession(prompt_len=request.prompt_len)." - ) - stale_end = max(0, (task._prompt_len + 1 - window_size) // tpb) - src_start = max(stale_end * tpb, src_start) - dst_start = max(stale_end * tpb, dst_start) - src_block_ids, dst_block_ids = Sender._align_kv_blocks( - src_block_ids, - dst_block_ids, - src_token_start=src_start, - dst_token_start=dst_start, - tokens_per_block=tpb, + tpb = extractor.page_table.tokens_per_block + token_range = task._slice.token_range + lg_info = extractor.page_table.layer_groups[self_lg] + window_size = getattr(lg_info, "sliding_window_size", None) + + # Block lists are the suffix of [..., slice_end); cached prefix + # is implicit in their size. token_start = (total_blocks - n) * tpb. + slice_end = token_range.end if token_range is not None else 0 + total_blocks = (slice_end + tpb - 1) // tpb + src_beam0_blocks = Sender._beam0_block_count( + src_block_ids, total_blocks, task._beam_width + ) + dst_beam0_blocks = Sender._beam0_block_count( + dst_block_ids, total_blocks, task._beam_width + ) + assert src_beam0_blocks <= total_blocks, ( + f"src beam-0 block list ({src_beam0_blocks}) exceeds total slice " + f"blocks ({total_blocks}); slice_end={slice_end}, tpb={tpb}" + ) + assert dst_beam0_blocks <= total_blocks, ( + f"dst beam-0 block list ({dst_beam0_blocks}) exceeds total slice " + f"blocks ({total_blocks}); slice_end={slice_end}, tpb={tpb}" + ) + src_start = (total_blocks - src_beam0_blocks) * tpb + dst_start = (total_blocks - dst_beam0_blocks) * tpb + if req_info.dst_start_token is not None: + dst_start = max(dst_start, req_info.dst_start_token) + if window_size is not None: + # SWA stale_end uses the request prompt_len (not slice_end — + # they differ for non-final slices). prompt_len must be plumbed + # via the session; falling back to slice_end is wrong on + # non-final slices. + assert task._prompt_len is not None, ( + "SWA layer requires session.prompt_len; " + "set TxSession(prompt_len=request.prompt_len)." ) + stale_end = max(0, (task._prompt_len + 1 - window_size) // tpb) + src_start = max(stale_end * tpb, src_start) + dst_start = max(stale_end * tpb, dst_start) + src_block_ids, dst_block_ids = Sender._align_kv_blocks( + src_block_ids, + dst_block_ids, + src_token_start=src_start, + dst_token_start=dst_start, + tokens_per_block=tpb, + ) - src_region = extractor.extract( - src_block_ids, layer_group_id=self_lg, pool_idx=self_pi - ) - dst_region = peer_extractor.extract( - dst_block_ids, layer_group_id=peer_lg, pool_idx=peer_pi - ) - mapper = self._registrar.get_kv_map(peer_ri, (self_lg, self_pi), (peer_lg, peer_pi)) - region_pair = mapper.map(src_region, dst_region) - region_pairs = region_pair if isinstance(region_pair, list) else [region_pair] - for rp in region_pairs: - src_frag_parts.append(rp.src.memory.ptrs) - dst_frag_parts.append(rp.dst.memory.ptrs) - size_specs.append((rp.src.memory.ptrs.size, rp.src.memory.bytes_per_region)) + src_region = extractor.extract(src_block_ids, layer_group_id=self_lg, pool_idx=self_pi) + dst_region = peer_extractor.extract( + dst_block_ids, layer_group_id=peer_lg, pool_idx=peer_pi + ) + mapper = self._registrar.get_kv_map(peer_ri, (self_lg, self_pi), (peer_lg, peer_pi)) + region_pair = mapper.map(src_region, dst_region) + region_pairs = region_pair if isinstance(region_pair, list) else [region_pair] + for rp in region_pairs: + src_frag_parts.append(rp.src.memory.ptrs) + dst_frag_parts.append(rp.dst.memory.ptrs) + size_specs.append((rp.src.memory.ptrs.size, rp.src.memory.bytes_per_region)) if src_frag_parts: src_frags = np.concatenate(src_frag_parts) @@ -1197,6 +1219,8 @@ def __init__( self._exception: Optional[Exception] = None self._closed = False self._terminal_status: Optional[SessionStatus] = None + self.transfer_start_time = None + self.transfer_end_time = None # Must be last: makes session visible to listener thread, # so all attributes above must be initialized first. self._sender.setup_session(self) @@ -1229,6 +1253,8 @@ def status(self) -> SessionStatus: return SessionStatus.READY if self.receiver_ready else SessionStatus.INIT def send(self, slice: KVSlice) -> None: + if self.transfer_start_time is None: + self.transfer_start_time = tensorrt_llm.bindings.global_steady_clock_now() with self.lock: params = self._base_args.params slice_id = len(self.kv_tasks) @@ -1502,13 +1528,23 @@ def _get_session(self, unique_rid: Optional[int]) -> Optional["RxSession"]: def _build_recv_req_info(self, task: KVRecvTask) -> RecvReqInfo: self_ri = self._registrar.self_rank_info - assert task._params.ctx_request_id is not None, ( - f"ctx_request_id is None for task unique_rid={task._unique_rid}" - ) assert task._unique_rid is not None, "KVRecvTask unique_rid is None" + # Some requests arrive with ctx_request_id None while disagg_request_id + # is set; disagg_request_id is the receive-session key, so fall back to + # it instead of failing here (nvbugs/6482576). + sender_req_id = task._params.ctx_request_id + if sender_req_id is None: + sender_req_id = task._params.disagg_request_id + if sender_req_id is None: + # Not an assert: must survive python -O so a None id never reaches + # RecvReqInfo.sender_req_id / the wire. + raise ValueError( + "both ctx_request_id and disagg_request_id are None for task " + f"unique_rid={task._unique_rid}" + ) # Receiver's cached prefix is implicit in block_ids size; sender derives dst_start. return RecvReqInfo( - sender_req_id=task._params.ctx_request_id, + sender_req_id=sender_req_id, instance_name=self_ri.instance_name, instance_rank=self_ri.instance_rank, block_ids_per_layer_groups=task._kv_slice.block_ids_per_layer_groups, @@ -1538,6 +1574,14 @@ def _fanin_bounce_safe(overlap, peer_ri) -> bool: lpp = getattr(peer_ri, "layer_num_per_pp", None) if not lpp or len(lpp) < overlap.overlap_pp_size or len(set(lpp)) != 1: return False + # Replicated pools (e.g. MiniMax M3 index-key) are sent by one elected + # fan-in owner only, so with multiple writers their contributions + # differ in size and the equal split is invalid. + if len(overlap.ranks) > 1 and peer_ri.page_table is not None: + for layer_group in peer_ri.page_table.layer_groups: + for pool_view in getattr(layer_group, "pool_views", ()): + if pool_view.mapper_kind == MapperKind.REPLICATED: + return False return True def dispatch_task(self, task: KVRecvTask): @@ -1738,7 +1782,7 @@ def _process_kv_agent_result(self, _send_id: bytes, message: list[bytes]): f"_process_kv_agent_result: unexpected msg_type={message[0]!r}, expected KV_AGENT_RESULT" ) return - peer_rank, unique_rid, sender_slice_id, is_last_slice, status_code = ( + peer_rank, unique_rid, sender_slice_id, is_last_slice, status_code, transfer_size = ( _KV_RESULT_PREFIX.unpack(message[1]) ) from .bounce import decode_result_tail @@ -1758,6 +1802,7 @@ def _process_kv_agent_result(self, _send_id: bytes, message: list[bytes]): dst_ptrs=dst_ptrs, sizes=sizes, src_base=src_base, + transfer_size=transfer_size, ) def _process_aux_agent_result(self, _send_id: bytes, message: list[bytes]): @@ -1815,6 +1860,9 @@ def __init__( self._exception: Optional[Exception] = None self._closed = False self._terminal_status: Optional[SessionStatus] = None + self.transfer_start_time = None + self.transfer_end_time = None + self.kv_cache_size_bytes: int = 0 self._kv_tasks: list[KVRecvTask] = [] self._aux_count = 0 self._aux_status: TaskStatus = TaskStatus.INIT @@ -1855,6 +1903,8 @@ def mark_transferring(self, slice_id: int): self._kv_tasks[slice_id].status = TaskStatus.TRANSFERRING def receive(self, slice: KVSlice) -> None: + if self.transfer_start_time is None: + self.transfer_start_time = tensorrt_llm.bindings.global_steady_clock_now() params = self._base_args.params slice_id = len(self._kv_tasks) task = KVRecvTask( @@ -1876,8 +1926,10 @@ def process_kv_agent_result( dst_ptrs=None, sizes=None, src_base=None, + transfer_size: int = 0, ): with self.lock: + self.kv_cache_size_bytes += transfer_size assert sender_slice_id < len(self._kv_tasks), ( f"Receiver got slice_id={sender_slice_id} from sender but only has " f"{len(self._kv_tasks)} receive task(s) for request {self.request_id}. " @@ -1932,6 +1984,13 @@ def on_done( f"slice={sender_slice_id}: {e}" ) task.complete() + # Transfer end for perf/time-sync: only meaningful once every slice has + # landed. Plain attribute write (atomic under the GIL); on_done must stay + # lock-free, and consumers only read it after wait_complete succeeds. + if all(t.status == TaskStatus.TRANSFERRED for t in self._kv_tasks): + self.transfer_end_time = ( + tensorrt_llm.bindings.global_steady_clock_now() + ) logger.debug( f"KV transfer complete for request {request_id} " f"slice={sender_slice_id}" diff --git a/tensorrt_llm/_torch/disaggregation/resource/kv_extractor.py b/tensorrt_llm/_torch/disaggregation/resource/kv_extractor.py index 92f0a9c2f4ad..9248bf506bce 100644 --- a/tensorrt_llm/_torch/disaggregation/resource/kv_extractor.py +++ b/tensorrt_llm/_torch/disaggregation/resource/kv_extractor.py @@ -1,6 +1,23 @@ -from typing import Dict, List +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict +from typing import Dict, List, Sequence import numpy as np +import torch from tensorrt_llm._torch.disaggregation.base.region import ( DataLayout, @@ -20,12 +37,31 @@ PhysicalPoolGroup, PoolView, ) -from tensorrt_llm._torch.disaggregation.resource.utils import get_physical_pool -from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import MambaHybridCacheManager +from tensorrt_llm._torch.disaggregation.resource.utils import ( + compute_layer_byte_ranges, + get_physical_pool, +) +from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import Role +from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import ( + MambaHybridCacheManager, + MambaHybridCacheManagerV2, +) from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager from tensorrt_llm._utils import get_size_in_bytes, nvtx_range from tensorrt_llm.bindings import DataType +# Mapper kinds a V2 manager may declare via get_disagg_role_mapper_kinds(). +# A physical pool may mix kinds (V2 storage coalesces buffers purely by +# size within a life cycle); the page-table builder emits one PoolView per +# (pool, kind) so each view stays kind-homogeneous. +_V2_ROLE_MAPPER_KINDS = frozenset( + { + MapperKind.INDEXED, + MapperKind.REPLICATED, + MapperKind.NHD, + } +) + class KVRegionExtractorV1(RegionExtractorBase): """ @@ -60,8 +96,9 @@ def extract( described by region_ids. For KV cache: each ptr = base_address + slot_id * slot_bytes, pointing - to the start of a full slot. The slot contains buffer entries for all - layers in this layer_group laid out contiguously from offset 0. + to the start of a full slot. Sub-slot selection (layers, role classes, + heads) is the mappers' responsibility; logical views carry that + geometry in their buffer entries. Args: layer_group_id: The layer group index (= life cycle index). @@ -73,10 +110,12 @@ def extract( base_ptr = pool.base_address block_size = pool.slot_bytes + block_stride = pool.slot_stride_bytes + assert block_stride is not None # KV cache: filter out invalid block_ids (BAD_PAGE_INDEX = -1) valid = region_ids >= 0 - ptrs = base_ptr + block_size * region_ids[valid] + ptrs = base_ptr + block_stride * region_ids[valid] memory = MemRegionGroup(ptrs=ptrs, bytes_per_region=block_size) return SpecRegion(memory=memory) @@ -101,16 +140,19 @@ def _build_layer_group_for_mamba( base_address=conv_state.data_ptr(), slot_bytes=conv_state.stride(1) * conv_state.element_size(), num_slots=conv_state.shape[1], + layer_stride_bytes=conv_state.stride(0) * conv_state.element_size(), ) ssm_pool = PhysicalPool( base_address=ssm_state.data_ptr(), slot_bytes=ssm_state.stride(1) * ssm_state.element_size(), num_slots=ssm_state.shape[1], + layer_stride_bytes=ssm_state.stride(0) * ssm_state.element_size(), ) # Per-section bytes for conv_state and per-head bytes for ssm_state. - # conv_state layout: [x: d_inner/tp | B: ng*ds/tp | C: ng*ds/tp] x (d_conv-1) + # The section ordering is supplied by the cache manager because Mamba2 + # uses [x | B | C], while GDN uses [Q | K | V]. # ssm_state layout: (nheads/tp, head_dim, d_state) d_conv_m1 = conv_state.shape[3] conv_elem_size = conv_state.element_size() @@ -132,6 +174,91 @@ def _build_layer_group_for_mamba( ) +def _slot_stride_bytes(tensor: torch.Tensor) -> int: + return int(tensor.stride(0) * tensor.element_size()) + + +def _build_v2_mamba_state_pool(states: Sequence[torch.Tensor]) -> PhysicalPool: + """Describe affine layer/slot addressing for one V2 Mamba state role.""" + if not states: + raise ValueError("V2 Mamba state pool requires at least one layer") + + first_state = states[0] + base_address = int(first_state.data_ptr()) + num_slots = int(first_state.shape[0]) + slot_bytes = int(first_state[0].numel() * first_state.element_size()) + slot_stride_bytes = _slot_stride_bytes(first_state) + + num_layers = len(states) + if slot_stride_bytes % num_layers != 0: + raise ValueError("V2 Mamba physical slot must divide evenly across layers") + # Each role appears once per layer in its size-class pool. Equal-size SSM + # and convolution states share that pool and are interleaved, so their + # layer stride includes both role payloads. + layer_stride_bytes = slot_stride_bytes // num_layers + + for layer_offset, state in enumerate(states): + state_slot_bytes = int(state[0].numel() * state.element_size()) + if ( + int(state.shape[0]) != num_slots + or state_slot_bytes != slot_bytes + or _slot_stride_bytes(state) != slot_stride_bytes + ): + raise ValueError("V2 Mamba state tensors must share one slot layout per role") + expected_address = base_address + layer_offset * layer_stride_bytes + if int(state.data_ptr()) != expected_address: + raise ValueError("V2 Mamba state tensors must have a uniform layer stride per role") + + return PhysicalPool( + base_address=base_address, + slot_bytes=slot_bytes, + num_slots=num_slots, + slot_stride_bytes=slot_stride_bytes, + layer_stride_bytes=layer_stride_bytes, + ) + + +def _build_layer_group_for_v2_mamba( + manager: MambaHybridCacheManagerV2, pool_group_idx: int +) -> MambaLayerGroup: + mamba_layer_offsets = { + int(global_layer_id): int(local_layer_id) + for global_layer_id, local_layer_id in manager.mamba_layer_offsets.items() + } + + expected_offsets = list(range(len(mamba_layer_offsets))) + if sorted(mamba_layer_offsets.values()) != expected_offsets: + raise ValueError("V2 Mamba layer offsets must be dense") + if len(manager.all_conv_states) != len(expected_offsets) or len(manager.all_ssm_states) != len( + expected_offsets + ): + raise ValueError("V2 Mamba state tensors must match the layer-offset table") + + first_conv_state = manager.all_conv_states[0] + first_ssm_state = manager.all_ssm_states[0] + conv_pool = _build_v2_mamba_state_pool(manager.all_conv_states) + ssm_pool = _build_v2_mamba_state_pool(manager.all_ssm_states) + if conv_pool.num_slots != ssm_pool.num_slots: + raise ValueError("V2 Mamba convolution and SSM states must have the same number of slots") + + d_conv_m1 = manager.conv_state_shape[1] + conv_elem_size = first_conv_state.element_size() + _, head_dim, d_state = manager.ssm_state_shape + conv_section_bytes = [dim * d_conv_m1 * conv_elem_size for dim in manager.conv_section_dims] + + ssm_elem_size = first_ssm_state.element_size() + ssm_bytes_per_head = head_dim * d_state * ssm_elem_size + + return MambaLayerGroup( + pool_group_idx=pool_group_idx, + mamba_layer_offsets=mamba_layer_offsets, + conv_states=conv_pool, + ssm_states=ssm_pool, + conv_section_bytes=conv_section_bytes, + ssm_bytes_per_head=ssm_bytes_per_head, + ) + + def build_page_table(kv_cache_manager: KVCacheManager) -> KVCachePageTable: """Build a KVCachePageTable from a KVCacheManager (V1).""" if kv_cache_manager.dtype == DataType.NVFP4: @@ -193,11 +320,15 @@ def build_page_table(kv_cache_manager: KVCacheManager) -> KVCachePageTable: buffer_entries=np.array(entries, dtype=BUFFER_ENTRY_DTYPE), pool_role=frozenset(kv_role_names), mapper_kind=MapperKind.INDEXED, + bytes_per_layer=stride, ) physical_pools = [kv_physical] pool_views = [kv_view] - # Indexer K cache support + # Indexer K cache support. The DSA indexer K cache is identical on + # every TP rank (single index head), so its view is REPLICATED with + # one synthesized buffer entry per local layer: the slot packs the + # layers equal-sized in local-layer order. if getattr(kv_cache_manager, "enable_indexer_k_cache", False): indexer_pool = kv_cache_manager.impl.get_indexer_k_cache_pool() # indexer_pool shape: (numBlocks, numLayers, kvFactor, blockSize), dtype=UINT8 @@ -211,11 +342,19 @@ def build_page_table(kv_cache_manager: KVCacheManager) -> KVCachePageTable: slot_bytes=indexer_slot_bytes, num_slots=num_blocks, ) + indexer_bytes_per_layer = indexer_slot_bytes // len(local_layer_ids) indexer_view = PoolView( pool_idx=1, - buffer_entries=np.array([], dtype=BUFFER_ENTRY_DTYPE), + buffer_entries=np.array( + [ + (lid, i * indexer_bytes_per_layer, indexer_bytes_per_layer) + for i, lid in enumerate(local_layer_ids) + ], + dtype=BUFFER_ENTRY_DTYPE, + ), pool_role=frozenset({"indexer_k"}), - mapper_kind=MapperKind.FLAT, + mapper_kind=MapperKind.REPLICATED, + bytes_per_layer=indexer_bytes_per_layer, ) physical_pools.append(indexer_physical) pool_views.append(indexer_view) @@ -288,18 +427,48 @@ def _compute_global_layer_ids(manager, lg_idx: int) -> List[int]: def _build_page_table_v2(manager) -> KVCachePageTable: """Build a KVCachePageTable from a KVCacheManagerV2. - Uses KVCacheManagerV2's public pool_group_descs layout API. A physical - pool group may be shared by several layer groups; layer_groups remains - indexed by layer_group_id while pool_group_idx points at the shared - physical pool group entry. - - Each PoolView is stamped with the manager's native role-name strings + Uses KVCacheManagerV2's public ``pool_group_descs`` layout API and + stamps each PoolView with the manager's native role-name strings (``pool_role``) plus the closed-set ``mapper_kind`` discriminator used by ``build_kv_mapper``. + + A physical pool group may be shared by several layer groups (life + cycles whose coalesced-buffer sizes are identical); each layer group + is exactly one ``SlotDescVariant`` of one pool group, so iterating + variants visits every layer group once. ``layer_groups`` stays indexed + by layer_group_id while ``pool_group_idx`` points at the shared + physical pool group entry, so per-window transfer logic keeps working. """ config = manager.impl.init_config pool_group_descs = manager.impl.pool_group_descs + # Every V2 manager declares how native roles map to the closed set of + # disaggregation mapper kinds; Role.ALL is the required fallback. + role_mapper_kinds = manager.get_disagg_role_mapper_kinds() + if Role.ALL not in role_mapper_kinds: + raise ValueError("Disaggregation role mapping must define Role.ALL") + for role, mapper_kind in role_mapper_kinds.items(): + if not isinstance(mapper_kind, MapperKind): + raise ValueError( + f"Invalid disaggregation mapper kind {mapper_kind!r} for role {role!s}" + ) + if mapper_kind not in _V2_ROLE_MAPPER_KINDS: + supported = ", ".join(kind.name for kind in sorted(_V2_ROLE_MAPPER_KINDS)) + raise ValueError( + f"Unsupported V2 disaggregation mapper kind {mapper_kind.name} " + f"for role {role!s}; supported kinds: {supported}" + ) + # INDEXED is the whole-manager legacy default, not a per-role + # choice: it may only appear as the Role.ALL fallback. Side-cache + # roles (e.g. INDEX_KEY) may declare their own non-INDEXED kind + # alongside it. + if mapper_kind is MapperKind.INDEXED and role != Role.ALL: + raise ValueError( + f"MapperKind.INDEXED is only valid as the Role.ALL mapping; " + f"got it for role {role!s}" + ) + default_mapper_kind = role_mapper_kinds[Role.ALL] + def _window_size_for_layer(internal_layer_id: int): if internal_layer_id < len(config.layers): return getattr(config.layers[internal_layer_id], "window_size", None) @@ -336,9 +505,22 @@ def _window_size_for_layer(internal_layer_id: int): ) ) + # Each variant is one layer group (life cycle) drawing slots from + # this pool group. Multiple layer groups share a pool group when + # their coalesced-buffer sizes are identical; within a slot, each + # layer group's buffer offsets start from 0 independently — the + # memory is reused, not concatenated. for variant in pg_desc.slot_desc.variants: layer_group_id = int(variant.layer_group_id) all_internal_layer_ids = list(manager.impl.layer_grouping[layer_group_id]) + if isinstance(manager, MambaHybridCacheManagerV2) and any( + manager._is_local_mamba_layer(int(layer_id)) for layer_id in all_internal_layer_ids + ): + layer_groups_by_id[layer_group_id] = _build_layer_group_for_v2_mamba( + manager, storage_pg_to_list_idx[storage_pg_idx] + ) + continue + all_global_layer_ids = _compute_global_layer_ids(manager, layer_group_id) local_layers = [ @@ -346,31 +528,78 @@ def _window_size_for_layer(internal_layer_id: int): for iid, gid in zip(all_internal_layer_ids, all_global_layer_ids) ] - pool_views = [] + # Bucket buffer entries by (pool, mapper kind). One PoolView is + # emitted per bucket and spans every layer of that role class, + # so the view count per layer group is bounded by the number of + # role classes — never by the layer count. A physical pool may + # hold several classes (V2 storage coalesces buffers purely by + # size within a layer group, so e.g. MiniMax M3's index-K shares + # the K/V pool when their per-block sizes coincide); each class + # still gets its own view, which keeps peer matching independent + # of that physical coalescing decision. ``pool_role`` stays the + # manager-supplied equivalence label used for peer matching + # without enumerating role names. Buffer offsets within a slot + # follow ``buffer_ids`` order: the i-th buffer of a coalesced + # buffer lives at ``i * single_buffer_size``. + bucket_entries: Dict[tuple, list] = defaultdict(list) + bucket_roles: Dict[tuple, set] = defaultdict(set) for pool_idx, coalesced_buffer in enumerate(variant.coalesced_buffers): - entries = [] - # Native role-name strings for this pool — used as - # ``PoolView.pool_role``, the manager-supplied equivalence - # label that disagg uses to match pools across peers without - # enumerating roles. - native_roles: set = set() - offset = 0 single_buffer_size = int(coalesced_buffer.single_buffer_size) + offset = 0 for buffer_id in coalesced_buffer.buffer_ids: - entries.append((int(buffer_id.layer_id), offset, single_buffer_size)) - native_roles.add(str(buffer_id.role)) + kind = role_mapper_kinds.get(buffer_id.role, default_mapper_kind) + bucket_key = (pool_idx, kind) + bucket_entries[bucket_key].append( + (int(buffer_id.layer_id), offset, single_buffer_size) + ) + bucket_roles[bucket_key].add(str(buffer_id.role)) offset += single_buffer_size - if entries: - pool_views.append( - PoolView( - pool_idx=pool_idx, - buffer_entries=np.array(entries, dtype=BUFFER_ENTRY_DTYPE), - pool_role=frozenset(native_roles), - mapper_kind=MapperKind.INDEXED, - ) + # Emit this layer group's views: one per (pool, mapper-kind + # class of roles). Roles sharing a kind share a view + # (KEY+VALUE); roles with different kinds in the same physical + # pool get separate views (M3 coalesced index-K). + # All ordering below is canonicalization — the page table is + # serialized and matched against peers, so view order (pool, + # then lowest slot offset), entry order (slot offset), and role + # text must not depend on dict/set iteration order. + pool_views = [] + lg_bucket_keys = sorted( + bucket_entries, + key=lambda key: (key[0], min(entry[1] for entry in bucket_entries[key])), + ) + for bucket_key in lg_bucket_keys: + pool_idx, mapper_kind = bucket_key + roles = frozenset(bucket_roles[bucket_key]) + entries = np.array( + sorted(bucket_entries[bucket_key], key=lambda entry: entry[1]), + dtype=BUFFER_ENTRY_DTYPE, + ) + # Fail fast on invalid geometry and record the uniform + # per-layer region size on the wire. Every kind is + # entries-driven, so the contiguous-layer-region / + # uniform-size invariants apply to all views uniformly. + _, bytes_per_layer = compute_layer_byte_ranges( + entries, + context=( + f"View(layer_group={layer_group_id}, pool={pool_idx}, " + f"kind={mapper_kind.name}, role={sorted(roles)})" + ), + ) + pool_views.append( + PoolView( + pool_idx=pool_idx, + buffer_entries=entries, + pool_role=roles, + mapper_kind=mapper_kind, + bytes_per_layer=bytes_per_layer, ) + ) + # Determine layer group metadata. + # For managers with virtual layers, internal layer_ids + # may exceed the length of num_kv_heads_per_layer. Use index 0 as + # all layers within a pool group share the same kv_heads count. first_local_layer = all_internal_layer_ids[0] if first_local_layer < len(manager.num_kv_heads_per_layer): num_kv_heads = manager.num_kv_heads_per_layer[first_local_layer] @@ -392,7 +621,9 @@ def _window_size_for_layer(internal_layer_id: int): raise ValueError(f"Missing V2 layer group descriptor for layer group {layer_group_id}") layer_groups.append(layer_group) - if isinstance(manager, MambaHybridCacheManager): + if isinstance(manager, MambaHybridCacheManager) and not isinstance( + manager, MambaHybridCacheManagerV2 + ): mamba_layer_group_idx = len(pool_groups) mamba_layer_group = _build_layer_group_for_mamba(manager, mamba_layer_group_idx) layer_groups.append(mamba_layer_group) diff --git a/tensorrt_llm/_torch/disaggregation/resource/page.py b/tensorrt_llm/_torch/disaggregation/resource/page.py index 81514c06d6a0..b15f21764e61 100644 --- a/tensorrt_llm/_torch/disaggregation/resource/page.py +++ b/tensorrt_llm/_torch/disaggregation/resource/page.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import annotations from dataclasses import dataclass, field @@ -16,23 +31,38 @@ class MapperKind(IntEnum): - """Slot metadata shape — selects how disagg derives the pool's layer set. - - INDEXED: PoolView.buffer_entries lists ``(local_layer_id, offset, size)`` - per buffer. Disagg reads ``local_layer_id`` to know *which* layers - from the LG live in this pool (a pool may cover a subset when V2 - splits an LG into multiple pools by buffer-size class). The - ``offset`` / ``size`` columns are carried for future use but are not - currently consumed at byte-transfer time. - FLAT: PoolView.buffer_entries is empty. Disagg assumes the pool - covers *all* layers of the LG, packed equal-sized in - ``local_layers`` order. Used today by the DSA (DeepSeek Sparse - Attention, v3.2) indexer K cache pool, whose slot layout is a dense - ``(numLayers, kvFactor, blockSize)`` array. - - Byte arithmetic is the same for both kinds: per-layer stride is - ``slot_bytes // num_layers``. The kind only affects how disagg discovers - the pool's layer set during pool matching. + """Transfer semantics of one physical pool's bytes. + + Every PoolView carries ``buffer_entries`` listing + ``(local_layer_id, offset, size)`` per buffer; the view's exact layer + set always comes from those entries (a view may cover a subset of the + LG when V2 splits an LG into multiple pools by buffer-size class, or + when a role class exists only on some layers). The kind selects how + bytes move between heterogeneous topologies: + + INDEXED: Head-major (HND) K/V — the layout written by the TRTLLM + attention kernels and the default for V1 and standard V2 managers. + Heterogeneous-head transfer selects one contiguous head-major range + per K/V buffer. + REPLICATED: The pool holds bytes that are identical on every TP rank + (MiniMax M3 index-key, DSA indexer K). Copied without KV-head + remapping using per-layer strides; fan-in routing elects one owning + sender per destination so each peer receives exactly one copy. + NHD: Ordinary K/V whose per-buffer storage is token-major + ``[token, head, dim]``. Heterogeneous-head transfer must select the + corresponding head slice inside every token rather than a single + contiguous head-major range. + + A physical pool may hold roles of different kinds: V2 storage coalesces + buffers purely by ``(life_cycle, buffer size)``, so e.g. MiniMax M3's + replicated index-K shares the K/V pool at TP degrees where their + per-block sizes coincide. The page-table builder therefore emits one + PoolView per ``(physical pool, mapper kind)`` — a view covers exactly + the bytes of one role class, and its per-layer byte ranges come from + ``buffer_entries`` (offsets are per layer because another class may + interleave between layers; only the per-layer size is uniform, recorded + in ``bytes_per_layer``). View count per layer group is bounded by the + number of role classes, never by layer count. Mamba state pools do not use this enum: Mamba's transfer is dispatched through :class:`MambaPolicy` which hard-codes the ``is_conv`` switch and @@ -40,20 +70,47 @@ class MapperKind(IntEnum): """ INDEXED = 0 - FLAT = 1 + REPLICATED = 1 + NHD = 2 @dataclass class PhysicalPool: + """Affine view of a physical pool over logical layers and slots. + + ``slot_bytes`` is the transferable payload for one ``(layer, slot)`` and + ``num_slots`` is the number of logical slots. The payload address is + ``base_address + layer * layer_stride_bytes + slot * slot_stride_bytes``. + The strides describe the physical layout independently of payload size and + slot count. Their defaults describe dense layer-major storage, where + ``slot_stride_bytes == slot_bytes`` and + ``layer_stride_bytes == num_slots * slot_stride_bytes``. V2 Mamba supplies + both explicitly for its slot-major, role-interleaved pools. + """ + base_address: int # uint64 slot_bytes: int num_slots: int + slot_stride_bytes: Optional[int] = None + layer_stride_bytes: Optional[int] = None + + def __post_init__(self) -> None: + if self.slot_stride_bytes is None: + self.slot_stride_bytes = self.slot_bytes + if self.layer_stride_bytes is None: + self.layer_stride_bytes = self.num_slots * self.slot_stride_bytes + if self.slot_stride_bytes < self.slot_bytes: + raise ValueError("slot_stride_bytes must be greater than or equal to slot_bytes") + if self.layer_stride_bytes < self.slot_bytes: + raise ValueError("layer_stride_bytes must be greater than or equal to slot_bytes") def to_dict(self) -> dict: return { "base_address": int(self.base_address), "slot_bytes": int(self.slot_bytes), "num_slots": int(self.num_slots), + "slot_stride_bytes": int(self.slot_stride_bytes), + "layer_stride_bytes": int(self.layer_stride_bytes), } @staticmethod @@ -62,6 +119,16 @@ def from_dict(data: dict) -> "PhysicalPool": base_address=int(data["base_address"]), slot_bytes=int(data["slot_bytes"]), num_slots=int(data["num_slots"]), + slot_stride_bytes=( + int(data["slot_stride_bytes"]) + if data.get("slot_stride_bytes") is not None + else None + ), + layer_stride_bytes=( + int(data["layer_stride_bytes"]) + if data.get("layer_stride_bytes") is not None + else None + ), ) @@ -107,7 +174,7 @@ class PoolView: pool_idx: Index of the physical pool within its pool group. buffer_entries: Structured array using ``BUFFER_ENTRY_DTYPE``. Each entry records a buffer's ``local_layer_id`` and its byte ``offset`` - and ``size`` within the pool slot. FLAT pools have no entries. + and ``size`` within the pool slot. pool_role: Set of native role-name strings (whatever the cache manager uses, e.g. ``"key"`` / ``"value"`` / ``"deepseek_v4_swa"``) that live in this pool. Used as the *equivalence label* for peer-to-peer @@ -115,12 +182,19 @@ class PoolView: are equal. Disagg never enumerates the role-name vocabulary — adding a new role on the manager side requires no disagg change. mapper_kind: Closed-set discriminator for picking the Mapper family. + bytes_per_layer: Uniform byte size of one layer's region within the + slot. The per-layer *offsets* live in ``buffer_entries``; only + the size is uniform, because a slot may interleave other role + classes between layers, making the layer stride non-uniform. + Set for every kind; ``None`` only in tables serialized by older + builders, where consumers re-derive it from the entries. """ pool_idx: int buffer_entries: np.ndarray # dtype=BUFFER_ENTRY_DTYPE pool_role: FrozenSet[str] = field(default_factory=frozenset) mapper_kind: MapperKind = MapperKind.INDEXED + bytes_per_layer: Optional[int] = None def to_dict(self) -> dict: return { @@ -128,6 +202,9 @@ def to_dict(self) -> dict: "buffer_entries": self.buffer_entries.tolist(), "pool_role": sorted(self.pool_role), "mapper_kind": int(self.mapper_kind), + "bytes_per_layer": ( + int(self.bytes_per_layer) if self.bytes_per_layer is not None else None + ), } @staticmethod @@ -143,6 +220,9 @@ def from_dict(data: dict) -> "PoolView": ), pool_role=frozenset(data["pool_role"]), mapper_kind=MapperKind(int(data["mapper_kind"])), + bytes_per_layer=( + int(data["bytes_per_layer"]) if data.get("bytes_per_layer") is not None else None + ), ) diff --git a/tensorrt_llm/_torch/disaggregation/resource/utils.py b/tensorrt_llm/_torch/disaggregation/resource/utils.py index 21c4d98bd2aa..2e39ed9af17e 100644 --- a/tensorrt_llm/_torch/disaggregation/resource/utils.py +++ b/tensorrt_llm/_torch/disaggregation/resource/utils.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import annotations from typing import Dict, List, Set @@ -10,7 +25,7 @@ def get_pool_bytes(pool: PhysicalPool) -> int: - """Total bytes across all slots in this pool.""" + """Total transferable payload bytes across all slots in this pool.""" return pool.slot_bytes * pool.num_slots @@ -18,7 +33,8 @@ def get_slot_address(pool: PhysicalPool, slot_id: int) -> int: """Base address of *slot_id*.""" if slot_id >= pool.num_slots: raise ValueError(f"slot_id {slot_id} >= num_slots {pool.num_slots}") - return pool.base_address + slot_id * pool.slot_bytes + assert pool.slot_stride_bytes is not None + return pool.base_address + slot_id * pool.slot_stride_bytes # ------------------------------------------------------------------------- @@ -26,6 +42,64 @@ def get_slot_address(pool: PhysicalPool, slot_id: int) -> int: # ------------------------------------------------------------------------- +def compute_layer_byte_ranges( + buffer_entries, + *, + declared_bytes_per_layer: "int | None" = None, + context: str = "PoolView", +) -> tuple[Dict[int, int], int]: + """Per-layer slot-relative byte offsets from raw buffer entries. + + Returns ``({local_layer_id: start_offset}, bytes_per_layer)``. A layer's + region is the concatenation of its buffer entries, which must be + contiguous within the slot; the region size must be uniform across + layers (the slot may interleave other role classes between layers, so + only the *size* is uniform — offsets are per layer). ``context`` labels + error messages; ``declared_bytes_per_layer`` cross-checks a size that + was recorded elsewhere. + """ + starts: Dict[int, int] = {} + totals: Dict[int, int] = {} + entries_by_layer: Dict[int, list] = {} + for entry in buffer_entries: + entries_by_layer.setdefault(int(entry["local_layer_id"]), []).append( + (int(entry["offset"]), int(entry["size"])) + ) + if not entries_by_layer: + raise ValueError(f"{context} has no buffer entries; per-layer byte ranges are undefined") + for layer_id, spans in entries_by_layer.items(): + spans.sort() + for (off, size), (next_off, _) in zip(spans, spans[1:]): + if off + size != next_off: + raise ValueError( + f"{context} layer {layer_id} buffers are " + f"not contiguous: [{off}, {off + size}) is followed by offset {next_off}" + ) + starts[layer_id] = spans[0][0] + totals[layer_id] = sum(size for _, size in spans) + distinct_totals = set(totals.values()) + if len(distinct_totals) != 1: + raise ValueError( + f"{context} per-layer region sizes are not uniform: {sorted(totals.items())}" + ) + bytes_per_layer = distinct_totals.pop() + if declared_bytes_per_layer is not None and declared_bytes_per_layer != bytes_per_layer: + raise ValueError( + f"{context} declares bytes_per_layer={declared_bytes_per_layer} but buffer " + f"entries sum to {bytes_per_layer} per layer" + ) + return starts, bytes_per_layer + + +def get_layer_byte_ranges(pool_view: PoolView) -> tuple[Dict[int, int], int]: + """Per-layer byte ranges of a view; see :func:`compute_layer_byte_ranges`.""" + return compute_layer_byte_ranges( + pool_view.buffer_entries, + declared_bytes_per_layer=pool_view.bytes_per_layer, + context=f"PoolView(pool_idx={pool_view.pool_idx}, role={sorted(pool_view.pool_role)})", + ) + + def get_unique_layers(pool_view: PoolView) -> Set[int]: """Unique local layer IDs in *pool_view*.""" return {int(e["local_layer_id"]) for e in pool_view.buffer_entries} @@ -117,9 +191,22 @@ def get_unique_pool_memory_descs( pool_counter = 0 for lg_idx, lg in enumerate(page_table.layer_groups): if isinstance(lg, MambaLayerGroup): - num_mamba_layers = len(lg.mamba_layer_offsets) - for pool in [lg.conv_states, lg.ssm_states]: - pool_size = num_mamba_layers * pool.num_slots * pool.slot_bytes + # V2 Mamba layer groups reference manager-owned physical pools. + # V1 Mamba state views are standalone and use the first invalid + # pool-group index after the attention groups. + has_physical_pool_group = 0 <= int(lg.pool_group_idx) < len(page_table.pool_groups) + if has_physical_pool_group: + pools_and_sizes = [ + (pool, get_pool_bytes(pool)) + for pool in page_table.pool_groups[int(lg.pool_group_idx)].pools + ] + else: + num_mamba_layers = len(lg.mamba_layer_offsets) + pools_and_sizes = [ + (pool, num_mamba_layers * pool.num_slots * pool.slot_bytes) + for pool in [lg.conv_states, lg.ssm_states] + ] + for pool, pool_size in pools_and_sizes: pool_key = (pool.base_address, pool_size) if pool_key not in unique_pools: unique_pools[pool_key] = pool_counter @@ -144,13 +231,24 @@ def get_unique_pool_memory_descs( def get_layer_to_layer_group(page_table: KVCachePageTable) -> Dict[int, int]: """ - Build ``{global_layer_id: lg_idx}`` mapping + Build ``{global_layer_id: lg_idx}`` mapping. + + Layer groups must partition a rank's attention layers: every + global_layer_id belongs to exactly one group. Peer matching relies on + this, so a duplicate raises instead of silently keeping the last group. """ out: Dict[int, int] = {} for lg_idx, lg in enumerate(page_table.layer_groups): if isinstance(lg, AttentionLayerGroup): for ll in lg.local_layers: - out[int(ll.global_layer_id)] = int(lg_idx) + gid = int(ll.global_layer_id) + if gid in out: + raise ValueError( + f"global_layer_id {gid} appears in layer groups " + f"{out[gid]} and {lg_idx}; layer groups must partition " + "a rank's attention layers" + ) + out[gid] = int(lg_idx) return out diff --git a/tensorrt_llm/_torch/disaggregation/transceiver.py b/tensorrt_llm/_torch/disaggregation/transceiver.py index f95022bbccf8..d065534396a0 100644 --- a/tensorrt_llm/_torch/disaggregation/transceiver.py +++ b/tensorrt_llm/_torch/disaggregation/transceiver.py @@ -12,15 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import time import uuid -from collections import defaultdict +from collections import Counter, defaultdict from itertools import chain from typing import Any, Callable, Dict, List, Optional, cast import numpy as np import torch +import tensorrt_llm.bindings from tensorrt_llm import logger from tensorrt_llm._torch.disaggregation.base.transfer import ( KVSlice, @@ -34,6 +36,7 @@ from tensorrt_llm._torch.disaggregation.native.bounce import ( config_from_size as bounce_config_from_size, ) +from tensorrt_llm._torch.disaggregation.native.perf_logger import perf_log_manager from tensorrt_llm._torch.disaggregation.native.transfer import TransferWorker, TransferWorkerConfig from tensorrt_llm._torch.disaggregation.resource.cache_reuse import ( CacheReuseAdapter, @@ -44,7 +47,10 @@ from tensorrt_llm._torch.distributed.communicator import Distributed from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import KvCacheTransceiver from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest -from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import MambaHybridCacheManager +from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import ( + MambaHybridCacheManager, + MambaHybridCacheManagerV2, +) from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager from tensorrt_llm._utils import nvtx_range from tensorrt_llm.bindings import LlmRequestState @@ -169,7 +175,9 @@ def _init_sync_policy(self): def _exchange_rank_info(self): endpoints = cast(list, self._dist.allgather(self._transfer_worker.sender_endpoint)) layer_num = len(self._kv_cache_manager.pp_layers) - if isinstance(self._kv_cache_manager, MambaHybridCacheManager): + if isinstance(self._kv_cache_manager, MambaHybridCacheManager) and not isinstance( + self._kv_cache_manager, MambaHybridCacheManagerV2 + ): layer_num += len(self._kv_cache_manager._impl.mamba_layer_offsets) layer_num_per_pp = cast(list, getattr(self._dist, "pp_allgather")(layer_num)) self._transfer_worker.populate_instance_and_rank_info( @@ -179,6 +187,48 @@ def _exchange_rank_info(self): logger.info(f"layer_num_per_pp: {layer_num_per_pp}") logger.info(f"self._context_info_endpoint: {self._context_info_endpoint}") + def get_status_dump(self) -> str: + """Return a one-line summary of transceiver state for debugging hangs.""" + + def summarize( + sessions: Dict[int, Any], + include_receiver_ready: bool, + ) -> str: + sessions_snapshot = list(sessions.values()) + status_counts = Counter() + receiver_ready = 0 + for session in sessions_snapshot: + status = session.status + if isinstance(status, SessionStatus): + status_counts[status] += 1 + else: + status_counts["unknown"] += 1 + + if include_receiver_ready: + receiver_ready += int(bool(session.receiver_ready)) + + fields = [ + f"sessions={len(sessions_snapshot)}", + f"init={status_counts[SessionStatus.INIT]}", + f"ready_to_transfer={status_counts[SessionStatus.READY]}", + f"transferring={status_counts[SessionStatus.TRANSFERRING]}", + f"kv_transferred={status_counts[SessionStatus.KV_TRANSFERRED]}", + f"fully_transferred={status_counts[SessionStatus.FULLY_TRANSFERRED]}", + f"error={status_counts[SessionStatus.ERROR]}", + f"cancelled={status_counts[SessionStatus.CANCELLED]}", + f"unknown={status_counts['unknown']}", + ] + if include_receiver_ready: + fields.append(f"peer_ready={receiver_ready}/{len(sessions_snapshot)}") + return ", ".join(fields) + + tx_status = summarize(self._send_sessions, include_receiver_ready=True) + rx_status = summarize(self._recv_sessions, include_receiver_ready=False) + return ( + f"KV cache transceiver | backend=NIXL | TX({tx_status}) | RX({rx_status}) | " + f"waiting_for_peer_info={len(self._wait_reqs)}" + ) + def shutdown(self): if getattr(self, "_shutdown", False): return @@ -262,7 +312,12 @@ def _create_kv_slice(self, req: LlmRequest) -> KVSlice: groups.append(block_ids) mamba_state_index = None - if isinstance(self._kv_cache_manager, MambaHybridCacheManager): + if isinstance(self._kv_cache_manager, MambaHybridCacheManagerV2): + if self._kv_cache_manager.local_num_mamba_layers > 0: + mamba_state_index = self._kv_cache_manager._request_id_to_state_index[ + req.py_request_id + ] + elif isinstance(self._kv_cache_manager, MambaHybridCacheManager): mamba_state_index = self._kv_cache_manager.mamba_cache_index[req.py_request_id] return KVSlice( @@ -449,6 +504,57 @@ def _ctx_consensus_outcome(self, to_process, cancelled, failed, completed, timed c, f, d = self._consensus_outcome(to_process, c, f, d, pp_allgather, True) return c, f, d, timed_out + def _sync_transfer_timing(self, reqs: list): + """Allgather timing for a batch of completed requests in one collective. + + Matches C++ ``batchUpdateKVCacheTransferBW()`` in ``cacheTransceiver.cpp``. + Only runs when ``TRTLLM_KVCACHE_TIME_OUTPUT_PATH`` is set (same gate + as C++) and multi-rank sync is needed. All ranks that participate in + the allgather update their local request objects. + """ + if not reqs: + return + if not os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH"): + return + if not self._gen_need_sync: + return + + # Pack local timing for all completed requests into one dict. + local_data = { + get_unique_rid(req): ( + req.get_kv_cache_transfer_start(), + req.get_kv_cache_transfer_end(), + req.kv_cache_size, + ) + for req in reqs + } + + # Single allgather for the whole batch. + all_data = self._gen_allgather(local_data) + + # Merge: per-rid min(start), max(end), sum(size) across ranks. + merged: dict = {} + for rank_data in all_data: + for rid, (start, end, size) in rank_data.items(): + if rid in merged: + prev = merged[rid] + merged[rid] = ( + min(prev[0], start), + max(prev[1], end), + prev[2] + size, + ) + else: + merged[rid] = (start, end, size) + + # Every rank updates its own local requests. + rid_to_req = {get_unique_rid(r): r for r in reqs} + for rid, (min_start, max_end, total_size) in merged.items(): + req = rid_to_req.get(rid) + if req is not None: + req.set_kv_cache_transfer_start(min_start) + req.set_kv_cache_transfer_end(max_end) + req.set_kv_cache_size(total_size) + def _collect_done(self, sessions: dict, reqs: dict): """Scan sessions and return (completed_rids, failed_rids).""" completed, failed = [], [] @@ -525,6 +631,7 @@ def _finalize_send(self, req: LlmRequest, session: TxSessionBase): @nvtx_range("KvCacheTransceiverV2.respond_and_send_async") def respond_and_send_async(self, req: LlmRequest): self._ever_had_send_session = True + req.set_kv_cache_transfer_start(tensorrt_llm.bindings.global_steady_clock_now()) session = self._get_or_create_send_session(req) req.state = LlmRequestState.DISAGG_CONTEXT_TRANS_IN_PROGRESS session.send(self._create_kv_slice(req)) @@ -569,6 +676,7 @@ def request_and_receive_sync(self, req: LlmRequest): @nvtx_range("KvCacheTransceiverV2.request_and_receive_async") def request_and_receive_async(self, req: LlmRequest): self._ever_had_recv_session = True + req.set_kv_cache_transfer_start(tensorrt_llm.bindings.global_steady_clock_now()) rid = get_unique_rid(req) if rid in self._recv_sessions: logger.warning( @@ -596,12 +704,20 @@ def check_context_transfer_status( return [], [] block_all = at_least_request_num is None wait_num = at_least_request_num if not block_all else 0 + need_progress = wait_num > 0 + if need_progress: + self._poll_sessions_for_interval( + self._send_sessions, + self._send_reqs, + wait_num, + self._sender_future_timeout_ms, + ) local_completed, local_failed = self._collect_done(self._send_sessions, self._send_reqs) to_process = self._build_to_process( self._send_sessions, self._ctx_consensus(local_completed + local_failed), - wait_num, + 0 if need_progress else wait_num, block_all, ) @@ -676,6 +792,11 @@ def check_gen_transfer_status(self, at_least_request_num: Optional[int]): # distinguish the two cases and set the appropriate state. cancelled.append(rid) elif result == WaitResult.COMPLETED: + req = self._recv_reqs[rid] + if session.transfer_end_time is not None: + req.set_kv_cache_transfer_end(session.transfer_end_time) + if session.kv_cache_size_bytes > 0: + req.set_kv_cache_size(session.kv_cache_size_bytes) completed.append(rid) elif result == WaitResult.FAILED: failed.append(rid) @@ -693,6 +814,20 @@ def check_gen_transfer_status(self, at_least_request_num: Optional[int]): del self._recv_reqs[rid] del self._recv_sessions[rid] + # Log gen-side transfer summary after consensus. + if completed and os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH"): + # Batch-sync timing for all completed requests in one allgather. + self._sync_transfer_timing([self._recv_reqs[rid] for rid in completed]) + for rid in completed: + req = self._recv_reqs[rid] + perf_log_manager.log_gen_transfer_summary( + unique_rid=rid, + instance_name=self._instance_name, + instance_rank=self._mapping.rank, + gen_side_transfer_time_ms=req.kv_cache_transfer_time_ms, + kv_cache_size=req.kv_cache_size, + ) + for rid in completed: session = self._recv_sessions[rid] req = self._recv_reqs[rid] @@ -715,16 +850,30 @@ def check_gen_transfer_status(self, at_least_request_num: Optional[int]): return completed, failed, cancelled_reqs def _poll_gen_sessions_for_poll_interval(self, wait_num: int) -> None: - poll_interval_s = (self.kv_transfer_poll_interval_ms or 0) / 1000.0 + self._poll_sessions_for_interval( + self._recv_sessions, + self._recv_reqs, + wait_num, + self.kv_transfer_poll_interval_ms, + ) + + def _poll_sessions_for_interval( + self, + sessions: dict, + reqs: dict, + wait_num: int, + poll_interval_ms: Optional[int], + ) -> None: + poll_interval_s = (poll_interval_ms or 0) / 1000.0 deadline = time.monotonic() + poll_interval_s while True: - completed, failed = self._collect_done(self._recv_sessions, self._recv_reqs) + completed, failed = self._collect_done(sessions, reqs) if len(completed) + len(failed) >= wait_num: return remaining_s = deadline - time.monotonic() if remaining_s <= 0: return - for session in self._recv_sessions.values(): + for session in sessions.values(): session.wait_complete(blocking=False) time.sleep(min(0.001, remaining_s)) diff --git a/tensorrt_llm/_torch/distributed/communicator.py b/tensorrt_llm/_torch/distributed/communicator.py index 34acfc741947..2e0fc1e0e4c3 100644 --- a/tensorrt_llm/_torch/distributed/communicator.py +++ b/tensorrt_llm/_torch/distributed/communicator.py @@ -17,10 +17,10 @@ MPI = None # deferred; functions will error if used when ENABLE_MULTI_DEVICE is True from tensorrt_llm._mnnvl_utils import init_helix_cp_comm -from tensorrt_llm._utils import (mpi_allgather, mpi_barrier, mpi_comm, - mpi_disabled, mpi_isend, mpi_isend_object, - mpi_recv, mpi_recv_object, mpi_send, - mpi_send_object, mpi_world_size, +from tensorrt_llm._utils import (local_mpi_size, mpi_allgather, mpi_barrier, + mpi_comm, mpi_disabled, mpi_isend, + mpi_isend_object, mpi_recv, mpi_recv_object, + mpi_send, mpi_send_object, mpi_world_size, torch_pybind11_abi) from tensorrt_llm.bindings.BuildInfo import ENABLE_MULTI_DEVICE from tensorrt_llm.bindings.internal.process_group import init_pg @@ -158,6 +158,11 @@ def has_cp_helix(self): def cp_config(self): return self.mapping.cp_config + @property + @abstractmethod + def local_world_size(self): + """Number of ranks co-located on this physical node.""" + @abstractmethod def barrier(self): pass @@ -668,6 +673,10 @@ def broadcast(self, obj, root=0, chunk_size: int = 4 * 1024 * 1024): def allgather(self, obj): return mpi_allgather(obj) + @property + def local_world_size(self): + return local_mpi_size() + def barrier(self): mpi_barrier() @@ -792,6 +801,10 @@ class TorchDist(Distributed): def rank(self): return torch.distributed.get_rank() + @property + def local_world_size(self): + return dist.get_world_size(group=self.local_comm) + def __init__(self, mapping: Mapping): super().__init__(mapping) assert dist.is_initialized( diff --git a/benchmarks/cpp/__init__.py b/tensorrt_llm/_torch/kv_cache_compression/__init__.py similarity index 100% rename from benchmarks/cpp/__init__.py rename to tensorrt_llm/_torch/kv_cache_compression/__init__.py diff --git a/tensorrt_llm/_torch/kv_cache_compression/compaction.py b/tensorrt_llm/_torch/kv_cache_compression/compaction.py new file mode 100644 index 000000000000..876d80d010ec --- /dev/null +++ b/tensorrt_llm/_torch/kv_cache_compression/compaction.py @@ -0,0 +1,278 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Batched physical KV-cache compaction: an algorithm-neutral mover. + +``build_compaction_params`` pre-binds one cache's launch parameters; +each round the caller writes its keep decision into the agreed rows and +``compact`` packs every cache's move sources and fires its native launches. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _pack_move_sources_kernel( + kept_ordinal_rows, + valid_seq_lens, + dense_move_offsets, + dense_move_indices, + swa_move_offsets, + swa_move_indices, + KEEP_COUNT: tl.constexpr, + DECISION_ROWS: tl.constexpr, + MOVE_CAPACITY: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + PER_LAYER: tl.constexpr, + DENSE_TOTAL: tl.constexpr, + SWA_TOTAL: tl.constexpr, + SWA_WINDOW: tl.constexpr, + BLOCK: tl.constexpr = 256, +): + """Pack one decision row into one family's move sources: dense rows emit the + kept tokens then the protected tail; SWA rows emit the latest window + (ascending order: the native copy moves in place).""" + BROADCAST: tl.constexpr = DECISION_ROWS == 1 + HAS_SWA: tl.constexpr = SWA_TOTAL > 0 + request = tl.program_id(0) + decision_row = tl.program_id(1) + row = request * DECISION_ROWS + decision_row + kept_row = kept_ordinal_rows + row * KEEP_COUNT + dense_begin = tl.load(dense_move_offsets + request) + dense_end = tl.load(dense_move_offsets + request + 1) + dense_count = dense_end - dense_begin + valid_len = tl.load(valid_seq_lens + request) + if HAS_SWA: + swa_begin = tl.load(swa_move_offsets + request) + swa_end = tl.load(swa_move_offsets + request + 1) + swa_count = swa_end - swa_begin + for move_start in tl.static_range(0, MOVE_CAPACITY, BLOCK): + move = move_start + tl.arange(0, BLOCK) + kept = tl.load( + kept_row + move, + mask=move < KEEP_COUNT, + other=0, + ) + dense_source = tl.where(move < KEEP_COUNT, kept, valid_len + move - KEEP_COUNT) + if BROADCAST: + # The one decision row per request feeds every KV head's packed row. + for head in tl.static_range(0, NUM_KV_HEADS): + tl.store( + dense_move_indices + head * DENSE_TOTAL + dense_begin.to(tl.int64) + move, + dense_source, + mask=move < dense_count, + ) + else: + dense_output = decision_row.to(tl.int64) * DENSE_TOTAL + dense_begin.to(tl.int64) + move + tl.store(dense_move_indices + dense_output, dense_source, mask=move < dense_count) + if HAS_SWA: + swa_source = valid_len - SWA_WINDOW + move + if BROADCAST: + for head in tl.static_range(0, NUM_KV_HEADS): + tl.store( + swa_move_indices + head * SWA_TOTAL + swa_begin.to(tl.int64) + move, + swa_source, + mask=move < swa_count, + ) + else: + swa_mask = move < swa_count + if PER_LAYER: + # SWA has one shared row per head; the first layer's decision rows write it. + swa_mask = swa_mask & (decision_row < NUM_KV_HEADS) + head = decision_row % NUM_KV_HEADS + swa_output = head.to(tl.int64) * SWA_TOTAL + swa_begin.to(tl.int64) + move + tl.store( + swa_move_indices + swa_output, + swa_source, + mask=swa_mask, + ) + + +@dataclass +class CompactionParams: + decision_rows: int = 0 + pack_args: Tuple[Optional[torch.Tensor], ...] = () + pack_constexprs: Dict[str, object] = field(default_factory=dict) + compact_args: List[Tuple[object, ...]] = field(default_factory=list) + + +def build_compaction_params( + layout: Dict[str, object], + *, + block_offsets: torch.Tensor, + kept_ordinals: torch.Tensor, + source_lengths: torch.Tensor, + dense_destination_bases: torch.Tensor, + dense_move_offsets: torch.Tensor, + protected_tail_capacity: int, + swa_move_offsets: Optional[torch.Tensor] = None, + swa_destination_bases: Optional[torch.Tensor] = None, +) -> CompactionParams: + """Pre-bind one compacted cache's launch parameters; only :func:`compact` reads them.""" + layer_pools = layout["layer_pools"] + dense_layers = tuple(int(layer) for layer in layout["dense_layers"]) + swa_layers = tuple(int(layer) for layer in layout["swa_layers"]) + layer_pool_ids = tuple(int(pool_id) for pool_id in layout["layer_pool_ids"]) + kv_block_offsets = block_offsets + kept_ordinal_rows = kept_ordinals + valid_seq_lens = source_lengths + token_starts = dense_destination_bases + protected_tail_capacity = int(protected_tail_capacity) + + params = CompactionParams() + first_pool = layer_pools[dense_layers[0]] + device = first_pool.device + max_requests = int(valid_seq_lens.shape[0]) + keep_count = int(kept_ordinal_rows.shape[1]) + params.decision_rows = int(kept_ordinal_rows.shape[0]) // max_requests + # Pool shape [pages, K/V, heads, tokens, dim]. + num_kv_heads = int(first_pool.shape[2]) + if params.decision_rows * max_requests != int(kept_ordinal_rows.shape[0]): + raise ValueError("kept_ordinal_rows rows must be a multiple of max_requests") + if params.decision_rows not in (1, num_kv_heads, len(dense_layers) * num_kv_heads): + raise ValueError( + f"unsupported decision layout: {params.decision_rows} rows for " + f"{num_kv_heads} heads x {len(dense_layers)} dense layers" + ) + per_layer_sources = ( + len(dense_layers) > 1 and params.decision_rows == len(dense_layers) * num_kv_heads + ) + dense_index_prefix = (len(dense_layers), num_kv_heads) if per_layer_sources else (num_kv_heads,) + dense_move_indices = torch.empty( + (*dense_index_prefix, (keep_count + protected_tail_capacity) * max_requests), + dtype=torch.int32, + device=device, + ) + dense_entries = [ + ( + layer, + layer_pools[layer], + kv_block_offsets[layer_pool_ids[layer], :max_requests, 0], + ) + for layer in dense_layers + ] + dense_slots = ( + {layer: slot for slot, layer in enumerate(dense_layers)} if per_layer_sources else None + ) + + swa_move_indices = None + swa_window = 0 + # One move group per family axis (dense / SWA): the layers + the tensors driving their moves. + move_groups = [ + (dense_entries, dense_move_indices, dense_move_offsets, token_starts, dense_slots), + ] + if swa_layers: + swa_window = int(layout["swa_window"]) + swa_move_indices = torch.empty( + (num_kv_heads, (swa_window + protected_tail_capacity) * max_requests), + dtype=torch.int32, + device=device, + ) + # SWA layers stage against their own page-table slots. + swa_entries = [ + ( + layer, + layer_pools[layer], + kv_block_offsets[layer_pool_ids[layer], :max_requests, 0], + ) + for layer in swa_layers + ] + move_groups.append( + (swa_entries, swa_move_indices, swa_move_offsets, swa_destination_bases, None) + ) + + # Widest per-request move count any staged offsets may express. + move_capacity = keep_count + protected_tail_capacity + if swa_layers: + move_capacity = max(move_capacity, swa_window + protected_tail_capacity) + + params.pack_args = ( + kept_ordinal_rows, + valid_seq_lens, + dense_move_offsets, + dense_move_indices, + swa_move_offsets, + swa_move_indices, + ) + params.pack_constexprs = dict( + KEEP_COUNT=keep_count, + DECISION_ROWS=params.decision_rows, + MOVE_CAPACITY=move_capacity, + NUM_KV_HEADS=num_kv_heads, + PER_LAYER=per_layer_sources, + DENSE_TOTAL=int(dense_move_indices.shape[-1]), + SWA_TOTAL=int(swa_move_indices.shape[-1]) if swa_move_indices is not None else 0, + SWA_WINDOW=swa_window, + ) + for entries, move_indices, move_offsets, destination_bases, slots in move_groups: + grouped = {} + for layer, pool, page_table in entries: + key = ( + layer_pool_ids[layer], + str(pool.dtype), + str(pool.device), + tuple(int(value) for value in pool.shape[1:]), + tuple(int(value) for value in page_table.shape), + ) + grouped.setdefault(key, []).append((layer, pool, page_table)) + for group_entries in grouped.values(): + layers = tuple(entry[0] for entry in group_entries) + pools = list(entry[1] for entry in group_entries) + source_layer_indices = None + if slots is not None: + source_layer_indices = torch.tensor( + [slots[layer] for layer in layers], + dtype=torch.int32, + device=device, + ) + params.compact_args.append( + ( + pools, + torch.tensor( + [pool.data_ptr() for pool in pools], + dtype=torch.int64, + device=device, + ), + group_entries[0][2], + move_indices, + move_offsets, + destination_bases, + source_layer_indices, + ) + ) + + return params + + +def compact( + params: Tuple[CompactionParams, ...], + request_count: int, +) -> None: + """Pack each cache's move sources and fire its native compacts, in order + (pure mover: the caller owns the decision rows and the round's completion ordering). + ``request_count`` only sizes the Triton pack grid; the native op takes the batch + from the page-table view's row count and validates the companion tensors against it.""" + # One launch per (cache, pool group); each launch covers every layer in the group. + for cache_params in params: + _pack_move_sources_kernel[(request_count, cache_params.decision_rows)]( + *cache_params.pack_args, **cache_params.pack_constexprs + ) + for args in cache_params.compact_args: + torch.ops.trtllm.sparse_kv_cache_compact_layers(*args) diff --git a/tensorrt_llm/_torch/kv_cache_compression/interface.py b/tensorrt_llm/_torch/kv_cache_compression/interface.py new file mode 100644 index 000000000000..cd4e7bf32068 --- /dev/null +++ b/tensorrt_llm/_torch/kv_cache_compression/interface.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from enum import IntEnum, auto +from typing import Optional + + +class KvCacheCompressionMode(IntEnum): + """Algorithm-level traits of a KV-cache compression method. + + Configs map their ``algorithm`` string to a member here; callers read the + ``is_*`` predicates instead of comparing strings. + """ + + NONE = auto() + + def is_eviction_method(self): + """Whether this method physically evicts cached tokens. Evicting + algorithms add their member and extend this predicate.""" + return False + + @staticmethod + def from_string(name: Optional[str]) -> "KvCacheCompressionMode": + if name is None: + return KvCacheCompressionMode.NONE + try: + return KvCacheCompressionMode[name.upper()] + except KeyError: + return KvCacheCompressionMode.NONE diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 2c7c301cbf01..2d2bc9c314f2 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -51,8 +51,9 @@ if TYPE_CHECKING: from tensorrt_llm.bindings import ModelConfig as ModelConfigCpp - from tensorrt_llm.llmapi.llm_args import (DecodingBaseConfig, LoraConfig, - SparseAttentionConfig, + from tensorrt_llm.llmapi.llm_args import (DecodingBaseConfig, + KvCacheCompressionConfig, + LoraConfig, SparseAttentionConfig, SpeculativeConfig) TConfig = TypeVar("TConfig", bound=transformers.PretrainedConfig) @@ -168,6 +169,7 @@ class ModelConfig(Generic[TConfig]): lm_head_gather_output: bool = True lora_config: Optional["LoraConfig"] = None sparse_attention_config: Optional["SparseAttentionConfig"] = None + kv_cache_compression_config: Optional["KvCacheCompressionConfig"] = None is_generation: bool = True is_encoder_decoder: bool = False @@ -512,12 +514,15 @@ def load_hf_quant_config(hf_quant_config, moe_backend, checkpoint_dir=None): # Read exclude_modules from HF config if present (HF format module names) hf_exclude_modules = hf_quant_config.get('modules_to_not_convert', None) - # DeepSeek V3 FP8 ckpt - if hf_quant_config.get("quant_method") == "fp8" and hf_quant_config.get( - "weight_block_size", []): + # FP8 ckpt: DeepSeek V3 style (weight_block_size) or + # per-tensor static activation scale style (activation_scheme="static", + # e.g. Ministral / Pixtral). + if hf_quant_config.get("quant_method") == "fp8" and ( + hf_quant_config.get("weight_block_size") + or hf_quant_config.get("activation_scheme") == "static"): quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES - block_size = hf_quant_config.get("weight_block_size", []) + block_size = hf_quant_config.get("weight_block_size", [128, 128]) assert tuple(block_size) == ( 128, 128), "FP8_BLOCK_SCALES only supports block_size=(128,128)" quant_config.group_size = block_size[0] @@ -976,9 +981,14 @@ def update_sparse_attention_indexer_config(pretrained_config, kwargs): pretrained_config, 'compress_ratios', None) num_base_layers = pretrained_config.num_hidden_layers spec_config = kwargs.get('spec_config', None) - if (spec_config is not None - and getattr(spec_config, 'num_nextn_predict_layers', - None) is None): + # ``num_nextn_predict_layers`` is MTP-specific (only read on + # the is_mtp_one_model path). Only set it on configs that + # actually declare the field; other DeepSeek-V4 spec modes + # (e.g. DSpark, which carries its own draft stage count) do + # not, and a blind setattr would fail pydantic validation. + if (spec_config is not None and 'num_nextn_predict_layers' + in type(spec_config).model_fields + and spec_config.num_nextn_predict_layers is None): spec_config.num_nextn_predict_layers = getattr( pretrained_config, 'num_nextn_predict_layers', 1) mtp_enabled = (spec_config is not None and @@ -1010,6 +1020,22 @@ def update_sparse_attention_indexer_config(pretrained_config, kwargs): if window_size is None: window_size = pretrained_config.sliding_window + # DeepSeek-V4 needs explicit per-layer compress ratios. They + # must come from the checkpoint config or a user override; we + # intentionally do not synthesize a default list (it would + # silently change sparse-attention semantics). Fail fast with + # an actionable message instead of letting the normalization + # below raise an opaque TypeError on None. + if compress_ratios is None: + raise ValueError( + "DeepSeek-V4 requires per-layer `compress_ratios`, " + "but none were found in the checkpoint config and " + "none were provided via `sparse_attention_config`. " + "Set `compress_ratios` in the model's config.json, or " + "pass `sparse_attention_config=" + "DeepSeekV4SparseAttentionConfig(compress_ratios=[...])`" + " in --extra_llm_api_options.") + # Normalize checkpoint-facing ratio 0 (SWA-only/uncompressed) # to 1 internally so cache allocation math works. The # external config keeps the original semantics. diff --git a/tensorrt_llm/_torch/models/__init__.py b/tensorrt_llm/_torch/models/__init__.py index 8fa9c1abc205..d48fc280c157 100644 --- a/tensorrt_llm/_torch/models/__init__.py +++ b/tensorrt_llm/_torch/models/__init__.py @@ -34,6 +34,7 @@ from .modeling_laguna import LagunaForCausalLM from .modeling_llama import LlamaForCausalLM from .modeling_llava_next import LlavaNextModel +from .modeling_minicpmv4_6 import MiniCPMV4_6Model from .modeling_minimaxm2 import MiniMaxM2ForCausalLM from .modeling_minimaxm3 import (MiniMaxM3ForCausalLM, MiniMaxM3VLForConditionalGeneration) @@ -65,6 +66,7 @@ from .modeling_t5 import T5ForConditionalGeneration from .modeling_utils import get_model_architecture from .modeling_vila import VilaModel +from .modeling_whisper import WhisperForConditionalGeneration # Note: for better readiblity, this should have same order as imports above __all__ = [ @@ -91,6 +93,7 @@ "KimiLinearForCausalLM", "LlamaForCausalLM", "LlavaNextModel", + "MiniCPMV4_6Model", "Mistral3VLM", "MistralForCausalLM", "MixtralForCausalLM", @@ -132,6 +135,7 @@ "Cohere2ForCausalLM", "Step3p7ForCausalLM", "Step3p7VLForConditionalGeneration", + "WhisperForConditionalGeneration", ] if transformers.__version__ >= "4.45.1": diff --git a/tensorrt_llm/_torch/models/checkpoints/base_weight_loader.py b/tensorrt_llm/_torch/models/checkpoints/base_weight_loader.py index 10d72a3a86c7..9ee2a8a55e17 100644 --- a/tensorrt_llm/_torch/models/checkpoints/base_weight_loader.py +++ b/tensorrt_llm/_torch/models/checkpoints/base_weight_loader.py @@ -70,6 +70,30 @@ def update(self, other: Dict[str, Any]) -> None: with self._lock: self._weights.update(other) + def clear(self) -> None: + """Drop every remaining reference. + + Use once a downstream dict owns the tensors: a derived dict aliases the + source tensors it did not rewrite, so consuming it frees nothing while + this dict still holds them. + """ + with self._lock: + self._weights.clear() + + def mark_consumed_keys(self, keys) -> int: + """Delete an exact set of keys to free memory. + + Use instead of :meth:`mark_consumed` when a module consumed specific + tensors rather than a whole ``name.*`` subtree. + """ + deleted = 0 + with self._lock: + for key in keys: + if key in self._weights: + del self._weights[key] + deleted += 1 + return deleted + def mark_consumed(self, prefix: str) -> int: """ Delete all keys starting with the given prefix to free memory. diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/gemma4_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/gemma4_weight_mapper.py index e336ef4b894b..22cde4599c43 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/gemma4_weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/gemma4_weight_mapper.py @@ -23,6 +23,10 @@ _LANG_PREFIX = "model.language_model." _MODEL_PREFIX = "model." _LAYER_IDX_RE = re.compile(r"layers\.(\d+)$") +_LAYER_SCALAR_KEY_RE = re.compile(r"^(?:language_model\.)?model\.layers\.(\d+)\.layer_scalar$") +_K_PROJ_KEY_RE = re.compile( + r"^((?:language_model\.)?model\.layers\.(\d+)\.self_attn\.)k_proj(\..+)$" +) @register_mapper("HF", "Gemma4ForCausalLM") @@ -273,9 +277,6 @@ def _remap_moe_keys(self, weights: dict) -> dict: def _handle_buffers_and_kvdup(self, weights: dict) -> dict: """Load layer_scalar buffers and duplicate k_proj for k_eq_v layers.""" - # Determine the layer scalar key pattern and accessor based on - # whether any key starts with "language_model." (VLM sub-model - # weights after filter_weights) or "model." (text-only). # Navigate to decoder layers regardless of model structure # (multimodal wrapper has .llm.model.layers, text-only has .model.layers) _root = self.model @@ -289,17 +290,9 @@ def _handle_buffers_and_kvdup(self, weights: dict) -> dict: def get_layer(idx): return _layers[idx] if _layers else None - sample = next(iter(weights), "") - if sample.startswith("language_model.model."): - scalar_pattern = r"language_model\.model\.layers\.(\d+)\.layer_scalar" - key_tmpl = "language_model.model.layers.{}.self_attn.{}_proj.weight" - else: - scalar_pattern = r"model\.layers\.(\d+)\.layer_scalar" - key_tmpl = "model.layers.{}.self_attn.{}_proj.weight" - layer_scalar_keys = [k for k in weights if k.endswith(".layer_scalar")] for key in layer_scalar_keys: - m = re.match(scalar_pattern, key) + m = _LAYER_SCALAR_KEY_RE.match(key) if m: layer_idx = int(m.group(1)) try: @@ -312,12 +305,17 @@ def get_layer(idx): config = self.model.config if getattr(config, "attention_k_eq_v", False): layer_types = getattr(config, "layer_types", []) - for layer_idx, lt in enumerate(layer_types): - if lt == "full_attention": - k_key = key_tmpl.format(layer_idx, "k") - v_key = key_tmpl.format(layer_idx, "v") - if k_key in weights and v_key not in weights: - weights[v_key] = weights[k_key] + for k_key, value in list(weights.items()): + match = _K_PROJ_KEY_RE.match(k_key) + if match is None: + continue + layer_idx = int(match.group(2)) + if layer_idx >= len(layer_types) or layer_types[layer_idx] != "full_attention": + continue + suffix = match.group(3) + suffix = {".k_scale": ".v_scale", ".k_bias": ".v_bias"}.get(suffix, suffix) + v_key = f"{match.group(1)}v_proj{suffix}" + weights.setdefault(v_key, value) # KV shared layers: HF omits k_proj/v_proj for shared layers. # The model uses Q-only projection for these layers, so no dummy diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/minimaxm3_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/minimaxm3_weight_mapper.py new file mode 100644 index 000000000000..bb6d58957fd3 --- /dev/null +++ b/tensorrt_llm/_torch/models/checkpoints/hf/minimaxm3_weight_mapper.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from torch import Tensor, nn + +from tensorrt_llm._torch.models.checkpoints.hf.weight_mapper import HfWeightMapper +from tensorrt_llm._torch.models.modeling_utils import register_mapper + +MINIMAX_M3_PARAMS_MAP = { + r"^(.*\.block_sparse_moe)\.e_score_correction_bias$": r"\1.gate.e_score_correction_bias", +} + + +@register_mapper("HF", "MiniMaxM3SparseForCausalLM") +@register_mapper("HF", "MiniMaxM3SparseForConditionalGeneration") +class MiniMaxM3HfWeightMapper(HfWeightMapper): + """Handle M3 gate naming and MXFP8 GQA duplication for loader v2.""" + + def __init__(self) -> None: + super().__init__() + self.params_map = MINIMAX_M3_PARAMS_MAP + + def _duplicate_kv_weights( + self, module: nn.Module, new_name: str, weights: dict[str, Tensor] + ) -> dict[str, Tensor]: + if new_name not in ["k_proj", "v_proj"]: + return weights + + duplicated_keys = ["weight", "bias"] + quant_config = getattr(module, "quant_config", None) + if quant_config is not None: + quant_mode = quant_config.quant_mode + if quant_mode.has_nvfp4(): + duplicated_keys.append("weight_scale") + if quant_mode.has_mxfp8(): + duplicated_keys.extend(["weight_scale", "weight_scale_inv"]) + + return { + key: self._duplicate_kv( + weight=value[:], num_kv_heads=self._num_kv_heads, tensor_parallel_size=self._tp_size + ) + if key in duplicated_keys + else value + for key, value in weights.items() + } diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/nemotron_h_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/nemotron_h_weight_mapper.py index 23462d725259..a1a38e6382f1 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/nemotron_h_weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/nemotron_h_weight_mapper.py @@ -1,6 +1,8 @@ import re +from typing import Optional import torch +from torch import nn import tensorrt_llm.logger as logger from tensorrt_llm._torch.models.checkpoints.hf.weight_mapper import \ @@ -42,10 +44,14 @@ def _split_mamba2_mixer_in_proj(w: torch.Tensor) -> torch.Tensor: w = torch.concat(w).contiguous() return w - is_nvfp4 = self.config.quant_config.quant_algo == "NVFP4" n_groups = config.n_groups d_state = config.ssm_state_size nheads = config.mamba_num_heads + # Full in_proj out_features = concat([z, x, B, C, dt]). Only its + # per-output-row block scale spans this dim 0 and takes the same + # structured split as the weight; per-tensor scalars (weight_scale_2, + # input_scale) do not and are left alone. + d_in_proj = 2 * d_inner + 2 * n_groups * d_state + nheads new_weights = {} for name, _ in weights.items(): @@ -71,10 +77,8 @@ def _split_mamba2_mixer_in_proj(w: torch.Tensor) -> torch.Tensor: if "A_log" in key: key = key.replace("A_log", "A") - if ("mixer.in_proj" in key - or "mixer.out_proj" in key) and "_scale" in key: - # Special handing for nvfp4 Mamba2 mixer in_proj.weight_scale. - if is_nvfp4 and "in_proj.weight_scale_2" not in key and "in_proj.weight_scale" in key: + if "mixer.in_proj" in key and "_scale" in key: + if self._num_rows(weights[name]) == d_in_proj: new_weights[key] = _split_mamba2_mixer_in_proj( weights[name]) else: @@ -188,3 +192,41 @@ def _split_mamba2_mixer_in_proj(w: torch.Tensor) -> torch.Tensor: new_weights[key] = weights[name] return new_weights + + @staticmethod + def _num_rows(tensor) -> Optional[int]: + """Size of dim 0 (the output-channel axis), or None for a scalar. + + Works for materialized tensors and lazy safetensors slices. A weight + scale that shares this axis with the weight (NVFP4 / MXFP8 / FP8 block + scale, FP8 rowwise) is per-output-channel and must undergo the same TP + transform as the weight; a per-tensor scalar scale has no such axis. + """ + shape = tensor.get_shape() if hasattr(tensor, + "get_shape") else tensor.shape + return shape[0] if shape else None + + def _duplicate_kv_weights(self, module: nn.Module, new_name: str, + weights: dict): + # Override of the base NVFP4-only rule: NemotronH attention may be + # FP8/MXFP8 (MIXED_PRECISION checkpoints), so duplicate ANY + # per-output-channel weight_scale (one that shares dim 0 with the + # weight) alongside the replicated kv weight, not just the NVFP4 case. + if new_name not in ('k_proj', 'v_proj'): + return weights + + num_kv_heads = self._num_kv_heads + duplicated_keys = ["weight", "bias"] + weight, scale = weights.get("weight"), weights.get("weight_scale") + if (weight is not None and scale is not None + and self._num_rows(scale) == self._num_rows(weight)): + duplicated_keys.append("weight_scale") + + return { + k: + self._duplicate_kv(weight=v[:], + num_kv_heads=num_kv_heads, + tensor_parallel_size=self._tp_size) + if k in duplicated_keys else v + for k, v in weights.items() + } diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/qwen3_5_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/qwen3_5_weight_mapper.py index 27982fb75424..076d95a2ad2b 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/qwen3_5_weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/qwen3_5_weight_mapper.py @@ -5,6 +5,7 @@ import torch from torch import nn +from tensorrt_llm._torch.models.checkpoints.base_weight_loader import ConsumableWeightsDict from tensorrt_llm._torch.models.checkpoints.hf.qwen3_next_weight_mapper import ( Qwen3NextHfWeightMapper, ) @@ -556,6 +557,7 @@ def _remap_dense_mlp_weights(self, weights: dict) -> dict: return remapped_weights def preprocess_weights(self, weights: dict) -> dict: + is_consumable = isinstance(weights, ConsumableWeightsDict) quant_algo = self.config.quant_config.quant_algo normalized_weights = self._normalize_weight_names(weights) @@ -584,4 +586,7 @@ def preprocess_weights(self, weights: dict) -> dict: if not getattr(self.config.pretrained_config, "num_experts", 0): packed_weights = self._remap_dense_mlp_weights(packed_weights) - return super().preprocess_weights(packed_weights) + processed_weights = super().preprocess_weights(packed_weights) + if is_consumable and not isinstance(processed_weights, ConsumableWeightsDict): + return ConsumableWeightsDict(processed_weights) + return processed_weights diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py index cd73f1b8ead0..d77e4fb76ac4 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py +++ b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py @@ -328,7 +328,7 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig: quant_config = QuantConfig() layer_quant_config = None - hf_quant_config = pretrained_config.quantization_config + hf_quant_config = getattr(pretrained_config, "quantization_config", {}) or {} if hf_quant_config.get("quant_method") == "compressed-tensors": if "NVFP4" in hf_quant_config.get("config_groups"): quant_config.quant_algo = QuantAlgo.NVFP4 @@ -390,7 +390,14 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig: from tensorrt_llm._torch.models.modeling_mistral_large3 import Mistral3Gate model_config.pretrained_config.gate_cls = Mistral3Gate - model_config.pretrained_config.input_processor_type = "mistral_large_3" - model_config.pretrained_config.model_type = "mistral_large_3" + # Native (mistral-format) checkpoints are served through the + # mistral-common tokenizer/processor, which applies its own chat + # template. Tag the config with a dedicated serving model_type so the + # serving layer resolves the PASSTHROUGH placeholder/chat-template + # metadata (registered under "mistral_common" in modeling_mistral.py) + # via the normal config path - i.e. resolve_top_level_model_type() - + # instead of having to inspect the live input processor. + model_config.pretrained_config.input_processor_type = "mistral_common" + model_config.pretrained_config.model_type = "mistral_common" model_config._frozen = True return model_config diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py b/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py index eb7ac6b573d3..426e85592ba4 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py +++ b/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py @@ -301,4 +301,4 @@ def hf_decode_incrementally( def apply_chat_template( self, conversation: Union[list[dict[str, str]], list[list[dict[str, str]]]], *args, **kwargs ) -> Union[str, list[int], list[str], list[list[int]]]: - raise NotImplementedError + return self.transformers_tokenizer.apply_chat_template(conversation, *args, **kwargs) diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py index dd6e0332b849..e4d50578fe5a 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py @@ -35,6 +35,9 @@ def __init__(self): "tok_embeddings": "model.embed_tokens", "output": "lm_head", "norm": "model.norm", + # For text-only models: preprocess_weights adds "language_model." prefix + "language_model.tok_embeddings": "model.embed_tokens", + "language_model.output": "lm_head", # For Eagle3 "language_model.eagle_linear": "model.fc", "language_model.layers": "layers", diff --git a/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py b/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py index e61bb73df269..91dbe496976b 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py +++ b/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py @@ -35,9 +35,7 @@ from contextlib import contextmanager from enum import Enum from pathlib import Path -from typing import Any, Callable, Optional, Type, Union - -import grpc +from typing import Any, Callable, Iterator, MutableMapping, Optional, Protocol, Type, Union from tensorrt_llm._torch.models.checkpoints.base_config_loader import BaseConfigLoader from tensorrt_llm._torch.models.checkpoints.base_weight_loader import BaseWeightLoader @@ -58,10 +56,13 @@ # for a source. On a cold cluster (no donor up yet), this means the very # first replica blocks for an hour before falling back to disk. We cap # the default at 30 s so first-replica startup degrades gracefully; users -# can still override via the env var or a future per-loader knob. +# can still override via the env var or the per-loader `query_timeout_s` setting. # Tracked as MX-4 in §15 (non-blocking source-query API upstream). _MX_SOURCE_QUERY_TIMEOUT_DEFAULT_S = "30" -_MX_PUBLISH_ENV_LOCK = threading.Lock() +# ModelExpress 0.4.1 reads transfer configuration from process-wide +# environment variables and exposes a module-level identity builder. Keep all +# temporary mutation of that shared state in one critical section. +_MX_TRANSFER_STATE_LOCK = threading.Lock() _MX_SOURCE_IDENTITY_METADATA_KEY = "trtllm_source_identity" _MX_WEIGHT_LAYOUT_METADATA_KEY = "trtllm_weight_layout" _MX_TRANSFORM_PROTOCOL_VERSION_METADATA_KEY = "trtllm_transform_protocol_version" @@ -75,9 +76,15 @@ class _MxWeightLayoutStatus(Enum): UNSUPPORTED = "unsupported" +class _MxSourceIdentity(Protocol): + """Subset of ModelExpress's protobuf SourceIdentity used by this adapter.""" + + extra_parameters: MutableMapping[str, str] + + @contextmanager -def _temporary_env(key: str, value: Optional[str]): - """Temporarily set or clear one environment variable.""" +def _temporary_env(key: str, value: Optional[str]) -> Iterator[None]: + """Temporarily set one environment variable when a value is provided.""" if value is None: yield return @@ -92,6 +99,95 @@ def _temporary_env(key: str, value: Optional[str]): os.environ[key] = prior +def _serialize_source_identity(identity: SourceIdentity) -> str: + """Serialize TRT-LLM's layout identity for MX's identity map.""" + payload = identity.to_dict() + # `model_name` is a cleartext discovery descriptor and is deliberately + # excluded from SourceIdentity compatibility checks. The outer MX identity + # already carries the normalized model name; embedding a local checkpoint + # path here would make otherwise-compatible no-shards receivers hash to a + # different MX source. + payload.pop("model_name", None) + return json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ) + + +def _attach_trtllm_metadata_to_mx_identity( + mx_identity: _MxSourceIdentity, source_identity: Optional[SourceIdentity] +) -> _MxSourceIdentity: + """Attach TRT-LLM compatibility metadata to an MX SourceIdentity.""" + if source_identity is None: + return mx_identity + + extra_parameters = getattr(mx_identity, "extra_parameters", None) + if extra_parameters is None: + raise RuntimeError( + "MX SourceIdentity has no extra_parameters field; cannot attach " + "TRT-LLM SourceIdentity for compatibility filtering." + ) + + try: + for key, value in _build_mx_source_metadata(source_identity).items(): + extra_parameters[key] = value + except (AttributeError, TypeError, ValueError) as e: + raise RuntimeError( + "Failed to attach TRT-LLM compatibility metadata to MX " + "SourceIdentity; MX P2P compatibility filtering will reject " + "this source." + ) from e + return mx_identity + + +@contextmanager +def _patched_trtllm_identity_builder( + mx_transfer: Any, source_identity: Optional[SourceIdentity] +) -> Iterator[None]: + """Temporarily wrap upstream TRT-LLM identity construction.""" + original = getattr(mx_transfer, "_build_trtllm_identity", None) + if source_identity is None or not callable(original): + yield + return + + def _wrapped_build_identity(*args: Any, **kwargs: Any) -> _MxSourceIdentity: + return _attach_trtllm_metadata_to_mx_identity( + original(*args, **kwargs), + source_identity, + ) + + mx_transfer._build_trtllm_identity = _wrapped_build_identity + try: + yield + finally: + mx_transfer._build_trtllm_identity = original + + +def _close_mx_client(client: Any) -> None: + """Close a best-effort MX discovery client without masking its result.""" + if client is None: + return + close = getattr(client, "close", None) + if not callable(close): + return + try: + close() + except Exception: + logger.warning( + f"Failed to close MX discovery client; continuing with the " + f"completed probe result.\n{traceback.format_exc()}" + ) + + +def _synchronize_cuda_for_mx_publish() -> None: + """Finish pending CUDA writes before exposing source buffers through MX.""" + import torch + + if torch.cuda.is_initialized(): + torch.cuda.synchronize() + + @register_checkpoint_loader("MX") class MXCheckpointLoader(HfCheckpointLoader): """Checkpoint loader for MX (ModelExpress) P2P weight transfer. @@ -102,9 +198,10 @@ class MXCheckpointLoader(HfCheckpointLoader): publishes its weights after `post_load_weights()` runs, together with metadata that lets compatible targets skip one-shot post-load transforms. - When the MX server or library is unavailable, this loader - transparently falls back to standard HuggingFace checkpoint - loading via the parent `HfCheckpointLoader`. + When the MX server is unavailable, this loader transparently falls back + to standard HuggingFace checkpoint loading via the parent + `HfCheckpointLoader`. A missing MX client is treated as a configuration + error and reported with an actionable installation command. All transport-level mechanics (NIXL, dtype casts, source matching, fallback) are delegated to `modelexpress.trtllm_live_transfer` @@ -135,7 +232,7 @@ def __init__( # `model_name` is the human-readable identity to publish/look up # under on the MX server. Typically the user-supplied # `llm_args.model` (a Hub ID like `"Qwen/Qwen2.5-72B-Instruct"` - # or a local path). `publish_as_source()` resolves it via + # or a local path). Transfer and publish paths resolve it via # :func:`_resolve_mx_model_name` (with HF-snapshot path fallback). self._model_name = str(model_name) if model_name is not None else None self._query_timeout_s = query_timeout_s @@ -160,9 +257,9 @@ def model_name(self) -> Optional[str]: """Explicit model identity passed to the constructor (if any). Note this is the *as-configured* value (e.g. `llm_args.model`), - not the final resolved identity that ends up in the published + not the final resolved identity passed to ModelExpress as `MODEL_NAME`. The full resolution (with env var and basename - fallbacks) happens inside :meth:`publish_as_source`. + fallbacks) happens inside the transfer and publish paths. """ return self._model_name @@ -253,23 +350,68 @@ def load_weights(self, checkpoint_dir: str, mapping: Mapping, **kwargs) -> dict[ ) try: - from modelexpress.trtllm_live_transfer import ( # type: ignore[import-not-found] - MxClient, - MxLiveWeightLoader, - _build_trtllm_identity, + from modelexpress import ( + trtllm_live_transfer as mx_transfer, # type: ignore[import-not-found] ) - except ImportError: + except ImportError as exc: + raise ImportError( + "ModelExpress checkpoint loading was explicitly requested, " + "but the ModelExpress client could not be imported. Install " + 'the MX dependencies with `pip install "tensorrt-llm[mx]"`, ' + "or select a different " + "`checkpoint_format` to continue without MX." + ) from exc + + try: + with _MX_TRANSFER_STATE_LOCK: + MxClient = mx_transfer.MxClient + MxLiveWeightLoader = mx_transfer.MxLiveWeightLoader + build_trtllm_identity = mx_transfer._build_trtllm_identity + # Resolve once so discovery and the released ModelExpress + # loader query the same source identity. The lock prevents a + # concurrent MX publish from temporarily changing MODEL_NAME or + # the identity builder while this state is captured. + resolved_name = self._resolve_publish_name(checkpoint_dir) + except AttributeError: logger.warning( - "modelexpress library not installed; cannot use MX P2P " - "weight transfer. Install from " - "https://github.com/ai-dynamo/modelexpress (Python client at " - "modelexpress_client/python). Falling back to disk loading." + "modelexpress TRT-LLM live-transfer symbols are missing; " + "cannot use MX P2P weight transfer. Falling back to disk " + "loading." ) return self._fallback_to_disk(checkpoint_dir, mapping, **kwargs) - source_metadata = self._fetch_source_metadata( - checkpoint_dir, MxClient, _build_trtllm_identity - ) + try: + source_metadata = self._fetch_source_metadata( + checkpoint_dir, + MxClient, + build_trtllm_identity, + model_name=resolved_name, + ) + except Exception: + # Deliberately broad: source discovery is part of the optional MX + # fast path, so an upstream client failure must preserve disk + # loading as the correctness path. + logger.warning( + "MX source metadata fetch failed; falling back to disk " + f"loading.\n{traceback.format_exc()}" + ) + return self._fallback_to_disk( + checkpoint_dir, + mapping, + reason="MX source metadata probe failed", + **kwargs, + ) + + source_registered = source_metadata is not None + if not source_registered and self._local_source_identity is not None: + # ModelExpress 0.4.1 hashes every SourceIdentity field, including + # extra_parameters. Proceed to MxLiveWeightLoader.load_weights() + # even though this immediate probe found no source: that method + # retries list_sources every five seconds until a source appears or + # query_timeout_s expires. It uses this same patched identity, so + # any source discovered later necessarily carries the expected + # TRT-LLM identity and layout metadata. + source_metadata = _build_mx_source_metadata(self._local_source_identity) # Pre-transfer compatibility gate: on mismatch, skip the transfer # before any RDMA work starts and fall back to disk. self._source_identity_compatible_for_last_load = self._source_metadata_identity_compatible( @@ -327,27 +469,30 @@ def load_weights(self, checkpoint_dir: str, mapping: Mapping, **kwargs) -> dict[ prepare_post_transform_receiver(model) timeout_override = self._resolve_query_timeout_override( - checkpoint_dir, - MxClient, - _build_trtllm_identity, + source_registered=source_registered, + model_name=resolved_name, ) - with _temporary_env("MX_SOURCE_QUERY_TIMEOUT", timeout_override): - try: + try: + with ( + _MX_TRANSFER_STATE_LOCK, + _temporary_env("MX_SOURCE_QUERY_TIMEOUT", timeout_override), + _temporary_env("MODEL_NAME", resolved_name), + _patched_trtllm_identity_builder(mx_transfer, self._local_source_identity), + ): mx_loader = MxLiveWeightLoader(mx_server=self._mx_server_url) fallback_weights = mx_loader.load_weights( checkpoint_dir, mapping=mapping, model=model, ) - except Exception: - # Deliberately broad: MX is an opportunistic fast path and HF - # disk loading remains the correctness path. Preserve the full - # traceback so unexpected upstream failures are diagnosable. - logger.warning( - "MX P2P transfer failed; falling back to disk loading.\n" - f"{traceback.format_exc()}" - ) - return self._fallback_to_disk(checkpoint_dir, mapping, **kwargs) + except Exception: + # Deliberately broad: MX is an opportunistic fast path and HF + # disk loading remains the correctness path. Preserve the full + # traceback so unexpected upstream failures are diagnosable. + logger.warning( + f"MX P2P transfer failed; falling back to disk loading.\n{traceback.format_exc()}" + ) + return self._fallback_to_disk(checkpoint_dir, mapping, **kwargs) if fallback_weights: fallback_bytes = sum( @@ -396,7 +541,10 @@ def load_weights(self, checkpoint_dir: str, mapping: Mapping, **kwargs) -> dict[ return {} def _resolve_query_timeout_override( - self, checkpoint_dir: str, MxClient: Type[Any], build_identity: Callable[..., Any] + self, + *, + source_registered: bool, + model_name: str, ) -> Optional[str]: """Return temporary `MX_SOURCE_QUERY_TIMEOUT` override, if any.""" if self._query_timeout_s is not None: @@ -405,64 +553,18 @@ def _resolve_query_timeout_override( if os.environ.get("MX_SOURCE_QUERY_TIMEOUT"): return None - if self._has_any_source_instance(checkpoint_dir, MxClient, build_identity): + if source_registered: return None logger.warning( "No MX source is currently registered for " - f"{self._resolve_publish_name(checkpoint_dir)}; " + f"{model_name}; " f"using MX_SOURCE_QUERY_TIMEOUT={_MX_SOURCE_QUERY_TIMEOUT_DEFAULT_S} " "for fast disk fallback. Set mx_config.server_query_timeout_s or " "MX_SOURCE_QUERY_TIMEOUT for long-running donor-load deployments." ) return _MX_SOURCE_QUERY_TIMEOUT_DEFAULT_S - def _has_any_source_instance( - self, checkpoint_dir: str, MxClient: Type[Any], build_identity: Callable[..., Any] - ) -> bool: - """Best-effort fast probe for registered MX source instances.""" - client = None - try: - identity = build_identity(model_name=self._resolve_publish_name(checkpoint_dir)) - client = MxClient(server_url=self._mx_server_url) - list_resp = client.list_sources(identity=identity) - return bool(getattr(list_resp, "instances", [])) - except (AttributeError, RuntimeError, TimeoutError, grpc.RpcError): - # If the probe cannot complete, prefer fast fallback over the - # upstream 1-hour default. The actual MxLiveWeightLoader call below - # remains the source of truth and may still succeed. - logger.warning( - f"MX source probe failed; using fast fallback timeout.\n{traceback.format_exc()}" - ) - return False - finally: - if client is not None and hasattr(client, "close"): - client.close() - - def _source_identity_compatible( - self, checkpoint_dir: str, MxClient: Type[Any], build_identity: Callable[..., Any] - ) -> bool: - """Whether the MX source's identity is compatible with this receiver. - - Compares the receiver's local :class:`SourceIdentity` against the - publisher's via `check_weight_sharing_compatibility` with the `WARN_FALLBACK` - policy. - - Args: - checkpoint_dir: The checkpoint directory identifying the source. - MxClient: The MX discovery client type (forwarded to the fetch - seam). - build_identity: Builder used to derive the publisher identity - (forwarded to the fetch seam). - - Returns: - `True` to proceed with P2P only when both identities are present - and compatible. `False` when either identity is missing or the - identities mismatch, so the caller falls back to disk loading. - """ - source_identity = self._fetch_source_identity(checkpoint_dir, MxClient, build_identity) - return self._source_identity_compatible_with_source(source_identity) - def _source_metadata_identity_compatible(self, metadata: Optional[dict[str, Any]]) -> bool: source_identity = _source_identity_from_metadata(metadata) return self._source_identity_compatible_with_source(source_identity) @@ -478,30 +580,23 @@ def _source_identity_compatible_with_source( ) return decision.should_share - def _fetch_source_identity( - self, checkpoint_dir: str, MxClient: Type[Any], build_identity: Callable[..., Any] - ) -> Optional[SourceIdentity]: - """Fetch the publisher's serialized :class:`SourceIdentity`. - - Args: - checkpoint_dir: The checkpoint directory identifying the source. - MxClient: The MX discovery client type. - build_identity: Builder used to derive the publisher identity. - - Returns: - The publisher's identity, or `None` when it cannot be fetched - yet (the compatibility gate then rejects P2P and falls back). - """ - metadata = self._fetch_source_metadata(checkpoint_dir, MxClient, build_identity) - return _source_identity_from_metadata(metadata) - def _fetch_source_metadata( - self, checkpoint_dir: str, MxClient: Type[Any], build_identity: Callable[..., Any] + self, + checkpoint_dir: str, + MxClient: Type[Any], + build_identity: Callable[..., Any], + *, + model_name: Optional[str] = None, ) -> Optional[dict[str, Any]]: """Fetch TRT-LLM metadata for the selected MX source, if available.""" client = None try: - identity = build_identity(model_name=self._resolve_publish_name(checkpoint_dir)) + identity = self._build_mx_identity( + checkpoint_dir, + build_identity, + self._local_source_identity, + model_name=model_name, + ) client = MxClient(server_url=self._mx_server_url) for method_name in ("get_source_metadata", "get_metadata", "get_worker_metadata"): method = getattr(client, method_name, None) @@ -510,7 +605,13 @@ def _fetch_source_metadata( try: metadata = method(identity=identity) except TypeError: - metadata = method(identity) + try: + metadata = method(identity) + except TypeError: + # modelexpress 0.4.1 get_metadata() takes + # mx_source_id/worker_id rather than an identity. Fall + # through to the exact-identity list_sources query. + continue metadata_dict = _metadata_to_dict(metadata) if _metadata_has_trtllm_key(metadata_dict): return metadata_dict @@ -522,16 +623,36 @@ def _fetch_source_metadata( metadata_dict = _source_instance_metadata(instance) if metadata_dict: metadata_candidates.append(metadata_dict) - return self._select_source_metadata(metadata_candidates) - except (AttributeError, RuntimeError, TimeoutError, TypeError, ValueError, grpc.RpcError): - logger.warning( - f"MX source metadata fetch failed; falling back to disk loading.\n" - f"{traceback.format_exc()}" - ) + selected_metadata = self._select_source_metadata(metadata_candidates) + if selected_metadata is not None: + return selected_metadata + + # modelexpress 0.4.1 SourceInstanceRef intentionally omits the + # queried SourceIdentity. A non-empty response still proves an + # exact match because list_sources hashes every identity field, + # including extra_parameters. Reconstruct the metadata that was + # embedded in the exact query so the compatibility/layout checks + # remain fail-closed without a second metadata channel. + if instances and self._local_source_identity is not None: + return _build_mx_source_metadata(self._local_source_identity) return None finally: - if client is not None and hasattr(client, "close"): - client.close() + _close_mx_client(client) + + def _build_mx_identity( + self, + checkpoint_dir: str, + build_identity: Callable[..., _MxSourceIdentity], + source_identity: Optional[SourceIdentity], + *, + model_name: Optional[str] = None, + ) -> _MxSourceIdentity: + """Build the MX identity used for discovery and attach TRT-LLM identity.""" + resolved_name = model_name or self._resolve_publish_name(checkpoint_dir) + return _attach_trtllm_metadata_to_mx_identity( + build_identity(model_name=resolved_name), + source_identity, + ) def _select_source_metadata( self, metadata_candidates: list[dict[str, Any]] @@ -602,70 +723,71 @@ def publish_as_source( return try: - from modelexpress.trtllm_live_transfer import ( - publish_model_params, # type: ignore[import-not-found] + from modelexpress import ( + trtllm_live_transfer as mx_transfer, # type: ignore[import-not-found] ) except ImportError: logger.debug("modelexpress library not installed; skipping MX publish.") return + try: + publish_model_params = mx_transfer.publish_model_params + except AttributeError: + logger.debug("modelexpress publish_model_params is missing; skipping MX publish.") + return # THREADSAFETY: upstream publish_model_params reads MODEL_EXPRESS_URL and # MODEL_NAME from the environment. Set both from our resolved # configuration so per-instance values (URL passed via # llm_args.mx_config.server_url, identity from llm_args.model) are - # respected, then restore prior state. This is safe for the current - # sequential TRT-LLM worker path, but co-resident ranks in one Python - # interpreter would race on process-wide env. Tracked as MX-2 in §15 - # (the env-var dance goes away when upstream exports a public identity - # builder / publish API). - resolved_name = self._resolve_publish_name(checkpoint_dir) + # respected, then restore prior state. MX transfer and publish calls in + # this interpreter are serialized while upstream requires process-wide + # state. Tracked as MX-2 in §15 (the env-var dance goes away when + # upstream exports a public identity builder / publish API). metadata = _build_mx_source_metadata(source_identity) - metadata_kwargs = _publish_metadata_kwargs(publish_model_params, metadata) - if metadata_kwargs is None: + metadata_kwargs = _publish_metadata_kwargs(publish_model_params, metadata) or {} + identity_builder = getattr(mx_transfer, "_build_trtllm_identity", None) + if not metadata_kwargs and not callable(identity_builder): logger.warning( "Skipping MX post-transform publish because " - "publish_model_params does not accept metadata; receivers " - "cannot safely verify transformed weights." + "publish_model_params does not accept metadata and MX does " + "not expose its TRT-LLM identity builder; receivers cannot " + "safely verify transformed weights." ) return - env_overrides = { - "MODEL_EXPRESS_URL": self._mx_server_url, - "MODEL_NAME": resolved_name, - } if threading.active_count() > 1: logger.warning_once( "MX publish uses process-wide MODEL_EXPRESS_URL/MODEL_NAME " - "environment variables; concurrent publish calls in one Python " - "process are serialized, but unrelated env readers can still " - "observe transient values. Tracked by MX-2.", + "environment variables; concurrent MX transfer and publish calls " + "in one Python process are serialized, but unrelated env readers " + "can still observe transient values. Tracked by MX-2.", key="mx_publish_env_threaded_warning", ) - with _MX_PUBLISH_ENV_LOCK: - prior = {key: os.environ.get(key) for key in env_overrides} - for key, value in env_overrides.items(): - os.environ[key] = value - - try: - publish_model_params(model, **metadata_kwargs) + try: + with _MX_TRANSFER_STATE_LOCK: + resolved_name = self._resolve_publish_name(checkpoint_dir) + # Post-load transforms may enqueue asynchronous writes. Make + # the source buffers globally ready before MX publishes their + # addresses and allows a receiver to issue RDMA reads. + _synchronize_cuda_for_mx_publish() + with ( + _temporary_env("MODEL_EXPRESS_URL", self._mx_server_url), + _temporary_env("MODEL_NAME", resolved_name), + _patched_trtllm_identity_builder(mx_transfer, source_identity), + ): + publish_model_params(model, **metadata_kwargs) logger.info( "Published post-transform weights to MX server at %s as model=%r", self._mx_server_url, resolved_name, ) - except Exception: - # Deliberately broad: publish is best-effort. A publish failure - # should not fail the local worker that already loaded weights. - logger.warning( - f"Failed to publish weights to MX server at {self._mx_server_url}.\n" - f"{traceback.format_exc()}" - ) - finally: - for key, prior_value in prior.items(): - if prior_value is None: - os.environ.pop(key, None) - else: - os.environ[key] = prior_value + except Exception: + # Deliberately broad: publish is best-effort. A publish failure + # should not fail the local worker that already loaded weights. + logger.warning( + f"Failed to publish weights to MX server at {self._mx_server_url}.\n" + f"{traceback.format_exc()}" + ) def post_load_publish( self, @@ -761,9 +883,7 @@ def _build_mx_source_metadata(source_identity: Optional[SourceIdentity]) -> dict _MX_TRANSFORM_PROTOCOL_VERSION_METADATA_KEY: str(_MX_STAGED_TRANSFORM_PROTOCOL_VERSION), } if source_identity is not None: - metadata[_MX_SOURCE_IDENTITY_METADATA_KEY] = json.dumps( - source_identity.to_dict(), sort_keys=True - ) + metadata[_MX_SOURCE_IDENTITY_METADATA_KEY] = _serialize_source_identity(source_identity) return metadata diff --git a/tensorrt_llm/_torch/models/dspark/__init__.py b/tensorrt_llm/_torch/models/dspark/__init__.py new file mode 100644 index 000000000000..b33d7553d877 --- /dev/null +++ b/tensorrt_llm/_torch/models/dspark/__init__.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DSpark draft-model components.""" diff --git a/tensorrt_llm/_torch/models/dspark/attention.py b/tensorrt_llm/_torch/models/dspark/attention.py new file mode 100644 index 000000000000..7f63a40edfbe --- /dev/null +++ b/tensorrt_llm/_torch/models/dspark/attention.py @@ -0,0 +1,462 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The DSpark captured-context attention primitives are ported from DeepSeek's +# DeepSpec reference ``inference/kernel.py`` (``sparse_attn``) and +# ``inference/model.py`` (``get_dspark_topk_idxs``). The reference computes these +# with a TileLang kernel; this is a functional-first pure-PyTorch port with the +# same math (index-gather + online softmax + a learnable attention sink that +# contributes only to the softmax denominator). +"""DSpark draft captured-context attention primitives (hardware-agnostic). + +The DSpark draft uses *dense* sliding-window MLA (``compress_ratio == 0``): the +query comes from the block's draft tokens, while the keys/values are gathered +from a small per-request set of positions (a sliding window of the projected +captured context plus the current block's own positions). Two primitives capture +the parts that differ from the standard MLA path: + +* :func:`get_dspark_topk_idxs` — the (window-context + block) position list. +* :func:`dspark_sparse_attn` — index-gathered attention with an attention sink. +""" + +from functools import lru_cache + +import torch +import torch.nn.functional as F + +__all__ = [ + "get_dspark_topk_idxs", + "get_dspark_topk_idxs_batched", + "dspark_sparse_attn", + "precompute_dspark_freqs_cis", + "apply_dspark_rotary", + "apply_dspark_rotary_batched", + "dspark_attention_forward", + "dspark_attention_forward_batched", +] + + +def precompute_dspark_freqs_cis( + rope_head_dim: int, + seqlen: int, + rope_theta: float = 10000.0, + device: torch.device | str = "cpu", +) -> torch.Tensor: + """Plain (non-YaRN) RoPE complex exponentials for the DSpark draft. + + The dense draft attention (``compress_ratio == 0``) disables YaRN and uses the + base ``rope_theta`` (DeepSpec ``precompute_freqs_cis`` with + ``original_seq_len == 0``). + + Returns: + complex64 tensor ``[seqlen, rope_head_dim // 2]``. + """ + freqs = 1.0 / ( + rope_theta + ** (torch.arange(0, rope_head_dim, 2, dtype=torch.float32, device=device) / rope_head_dim) + ) + t = torch.arange(seqlen, dtype=torch.float32, device=device) + freqs = torch.outer(t, freqs) + return torch.polar(torch.ones_like(freqs), freqs) + + +def apply_dspark_rotary( + x: torch.Tensor, freqs_cis: torch.Tensor, inverse: bool = False +) -> torch.Tensor: + """Apply (or, with ``inverse``, de-apply) rotary embeddings, DeepSpec-style. + + Functional (non-in-place) port of DeepSpec ``apply_rotary_emb``: treats the + last dim as adjacent (re, im) pairs, rotates by ``freqs_cis`` indexed along the + sequence axis, and conjugates for the inverse (de-rotation applied to the + attention output). ``x`` is the rope-dim slice only: ``[b, s, rd]`` (3D) or + ``[b, s, h, rd]`` (4D), with ``freqs_cis`` of shape ``[s, rd // 2]``. + """ + orig_dtype = x.dtype + xc = torch.view_as_complex(x.float().unflatten(-1, (-1, 2))) + if inverse: + freqs_cis = freqs_cis.conj() + if xc.ndim == 3: + fc = freqs_cis.view(1, xc.size(1), xc.size(-1)) + else: + fc = freqs_cis.view(1, xc.size(1), 1, xc.size(-1)) + out = torch.view_as_real(xc * fc).flatten(-2) + return out.to(orig_dtype) + + +def apply_dspark_rotary_batched( + x: torch.Tensor, freqs_cis: torch.Tensor, inverse: bool = False +) -> torch.Tensor: + """Per-row (batched) variant of :func:`apply_dspark_rotary`. + + Identical math, but ``freqs_cis`` carries a leading batch axis so each row of + ``x`` is rotated by its own per-request phases (the generation draft runs each + request at a different absolute ``start_pos``). ``x`` is the rope-dim slice + only: ``[G, s, rd]`` (3D) or ``[G, s, h, rd]`` (4D), with ``freqs_cis`` of shape + ``[G, s, rd // 2]``. + """ + orig_dtype = x.dtype + xc = torch.view_as_complex(x.float().unflatten(-1, (-1, 2))) + if inverse: + freqs_cis = freqs_cis.conj() + g, s, half = freqs_cis.shape + if xc.ndim == 3: + fc = freqs_cis.view(g, s, half) + else: + fc = freqs_cis.view(g, s, 1, half) + out = torch.view_as_real(xc * fc).flatten(-2) + return out.to(orig_dtype) + + +@lru_cache(maxsize=64) +def _topk_matrix(window_size: int, block_size: int, start_pos: int) -> torch.Tensor: + # [min(window, start_pos+1)] context positions in the rolling KV window, + # followed by [block_size] positions for the current block's own K/V (which + # the caller appends to the window at offset ``window_size``). + ctx = torch.arange(min(window_size, start_pos + 1)) + blk = window_size + torch.arange(block_size) + return torch.cat([ctx, blk]).int() + + +def get_dspark_topk_idxs( + window_size: int, + bsz: int, + block_size: int, + start_pos: int, + device: torch.device | str = "cpu", +) -> torch.Tensor: + """Per-query attended-position indices for the DSpark draft block. + + Mirrors DeepSpec ``get_dspark_topk_idxs``: every one of the ``block_size`` + query positions attends to the same set — the ``min(window_size, start_pos+1)`` + most-recent context positions in the rolling KV window, then the + ``block_size`` positions of the current block (stored at offset + ``window_size`` in the concatenated KV). Note this is *non-causal* within the + block (every position sees every block position), matching the reference. + + Args: + window_size: sliding-window length of the captured-context KV cache. + bsz: batch size. + block_size: number of draft positions per request. + start_pos: absolute decode position (must be > 0); bounds the context. + device: device for the returned index tensor. + + Returns: + int32 tensor ``[bsz, block_size, topk]`` with + ``topk = min(window_size, start_pos+1) + block_size``. + """ + assert start_pos > 0, "DSpark draft attention runs at generation (start_pos > 0)" + matrix = _topk_matrix(int(window_size), int(block_size), int(start_pos)).to(device) + return matrix.view(1, 1, -1).expand(bsz, block_size, -1).contiguous() + + +def get_dspark_topk_idxs_batched( + window_size: int, + block_size: int, + start_pos: torch.Tensor, +) -> torch.Tensor: + """Sync-free, fixed-size (CUDA-graph-safe) batched ``get_dspark_topk_idxs``. + + Unlike the scalar :func:`get_dspark_topk_idxs` (whose ``topk`` width + ``min(window_size, start_pos+1) + block_size`` depends on the host int + ``start_pos``), this always returns the **fixed** width ``window_size + + block_size`` and masks the unfilled context slots with ``-1``. The masked + slots are excluded by :func:`dspark_sparse_attn` exactly as if they were + absent, so the result is numerically identical to gathering only the + ``min(window_size, start_pos+1)`` valid context positions — but the shape no + longer depends on the data, which is what CUDA-graph capture requires. + + Every query position attends to the same set: context window slots + ``0..window_size-1`` (slot ``c`` valid iff ``c <= start_pos[g]``, i.e. it has + been written) followed by the ``block_size`` block positions at offset + ``window_size`` (always valid). + + Args: + window_size: sliding-window length of the captured-context KV cache. + block_size: number of draft positions per request. + start_pos: ``[G]`` int tensor of per-request absolute decode positions. + + Returns: + int32 tensor ``[G, block_size, window_size + block_size]``. + """ + device = start_pos.device + g = start_pos.shape[0] + ctx_cols = torch.arange(window_size, device=device) # [win] + # Context slot c holds a written key iff c <= start_pos (slots 0..start_pos + # filled; for start_pos >= window_size-1 the whole rolling window is filled). + valid = ctx_cols.unsqueeze(0) <= start_pos.unsqueeze(1) # [G, win] + ctx_idx = torch.where( + valid, ctx_cols.unsqueeze(0).expand(g, -1), torch.full_like(valid, -1, dtype=torch.long) + ) + blk_idx = window_size + torch.arange(block_size, device=device) # [block] + blk_idx = blk_idx.unsqueeze(0).expand(g, -1) # [G, block] + row = torch.cat([ctx_idx, blk_idx], dim=1).to(torch.int32) # [G, win+block] + return row.unsqueeze(1).expand(g, block_size, -1).contiguous() + + +def dspark_sparse_attn( + q: torch.Tensor, + kv: torch.Tensor, + attn_sink: torch.Tensor, + topk_idxs: torch.Tensor, + softmax_scale: float, +) -> torch.Tensor: + """Index-gathered multi-query attention with an attention sink. + + Functional-first port of the DeepSpec ``sparse_attn`` TileLang kernel. For + each ``(batch, query, head)`` it gathers the ``topk`` KV rows named by + ``topk_idxs`` (an index of ``-1`` masks that slot), computes a scaled + dot-product softmax over them, and adds a per-head learnable *sink* logit that + participates only in the softmax denominator (i.e. an "attend-to-nothing" + option with a zero value vector). KV is shared across query heads (MQA). + + Args: + q: ``[b, m, h, d]`` query (``m`` = block_size, ``h`` = heads). + kv: ``[b, n, d]`` keys/values (shared across heads). + attn_sink: ``[h]`` per-head sink logits (fp32). + topk_idxs: ``[b, m, topk]`` int gather indices into ``kv`` (``-1`` masks). + softmax_scale: scalar applied to the q·k scores (``head_dim ** -0.5``). + + Returns: + ``[b, m, h, d]`` attention output, in ``q.dtype``. + """ + b, m, h, d = q.shape + idx = topk_idxs.long() # [b, m, topk] + valid = idx >= 0 + safe = idx.clamp(min=0) + + # Invalid slots read kv[0, :] (via safe.clamp), but masked_fill below + # zeros their softmax probs, so the einsum nullifies them. + kv_exp = kv.unsqueeze(1).expand(b, m, kv.shape[1], d) + gathered = torch.gather(kv_exp, 2, safe.unsqueeze(-1).expand(b, m, safe.shape[-1], d)).float() + + # Scores [b, m, h, topk]; mask invalid slots to -inf before the softmax. + scores = torch.einsum("bmhd,bmkd->bmhk", q.float(), gathered) * softmax_scale + scores = scores.masked_fill(~valid.unsqueeze(2), float("-inf")) + + # Online-softmax max is taken over gathered positions only (the sink is added + # to the denominator afterwards), matching the kernel's reduce order. + smax = scores.max(dim=-1, keepdim=True).values # [b, m, h, 1] + smax = torch.where(torch.isinf(smax), torch.zeros_like(smax), smax) + probs = torch.exp(scores - smax) # masked slots -> exp(-inf) = 0 + sink = torch.exp(attn_sink.to(torch.float32).view(1, 1, h) - smax.squeeze(-1)) + denom = probs.sum(dim=-1) + sink # [b, m, h] + out = torch.einsum("bmhk,bmkd->bmhd", probs, gathered) / denom.unsqueeze(-1) + return out.to(q.dtype) + + +def _rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: + """RMSNorm matching the DeepSpec reference (fp32 reduce, then * weight).""" + dtype = x.dtype + xf = x.float() + xf = xf * torch.rsqrt(xf.square().mean(-1, keepdim=True) + eps) + return (weight.float() * xf).to(dtype) + + +def _rope_last_dims( + t: torch.Tensor, rope_head_dim: int, freqs_cis: torch.Tensor, inverse: bool = False +) -> torch.Tensor: + """Apply RoPE to the last ``rope_head_dim`` dims; pass the rest through.""" + nope = t[..., :-rope_head_dim] + rope = apply_dspark_rotary(t[..., -rope_head_dim:], freqs_cis, inverse=inverse) + return torch.cat([nope, rope], dim=-1) + + +def _rope_last_dims_batched( + t: torch.Tensor, rope_head_dim: int, freqs_cis: torch.Tensor, inverse: bool = False +) -> torch.Tensor: + """Per-row variant of :func:`_rope_last_dims` (``freqs_cis`` has a batch axis).""" + nope = t[..., :-rope_head_dim] + rope = apply_dspark_rotary_batched(t[..., -rope_head_dim:], freqs_cis, inverse=inverse) + return torch.cat([nope, rope], dim=-1) + + +def dspark_attention_forward( + x: torch.Tensor, + main_x: torch.Tensor, + start_pos: int, + kv_cache: torch.Tensor, + *, + wq_a: torch.Tensor, + q_norm_w: torch.Tensor, + wq_b: torch.Tensor, + wkv: torch.Tensor, + kv_norm_w: torch.Tensor, + wo_a: torch.Tensor, + wo_b: torch.Tensor, + attn_sink: torch.Tensor, + n_heads: int, + head_dim: int, + rope_head_dim: int, + n_groups: int, + o_lora_rank: int, + window_size: int, + eps: float, + softmax_scale: float, + freqs_cis: torch.Tensor, + persist: bool = False, +) -> torch.Tensor: + """Captured-context DSpark draft attention (generation path, ``start_pos > 0``). + + Functional port of DeepSpec ``DSparkAttention.forward`` for the dense + (``compress_ratio == 0``) draft: low-rank Q (``wq_a`` -> ``q_norm`` -> ``wq_b``) + with a per-head RMS + RoPE, MQA K/V from ``wkv`` (shared across heads), keys + gathered from a rolling captured-context window (``kv_cache``, into which the + projected ``main_x`` context is written at ``start_pos % window_size``) plus the + block's own positions, attention-sink softmax, inverse-RoPE on the output, and a + grouped low-rank O projection (``wo_a`` einsum + ``wo_b``). + + Weights are plain tensors for ``F.linear`` (the caller supplies the loaded / + dequantized projection weights); ``wo_a`` is the raw grouped weight matrix + ``[n_groups * o_lora_rank, n_heads * head_dim // n_groups]``. ``kv_cache`` is + ``[b, window_size, head_dim]`` and is updated functionally (cloned). + + Returns: + ``[b, block_size, dim]`` attention output (residual stream contribution). + """ + assert start_pos > 0, "DSpark draft attention runs at generation (start_pos > 0)" + b, block, _ = x.shape + rd = rope_head_dim + main_freqs = freqs_cis[start_pos : start_pos + 1] + blk_freqs = freqs_cis[start_pos + 1 : start_pos + 1 + block] + + # Captured-context K/V from main_x (MQA, shared across heads). + main_kv = _rmsnorm(F.linear(main_x, wkv), kv_norm_w, eps) # [b, 1, head_dim] + main_kv = _rope_last_dims(main_kv, rd, main_freqs) + + # Query: low-rank + per-head RMS + RoPE. + q = _rmsnorm(F.linear(x, wq_a), q_norm_w, eps) + q = F.linear(q, wq_b).unflatten(-1, (n_heads, head_dim)) # [b, block, h, head_dim] + # Per-head RMS in the query dtype (matches the reference inline normalization, + # which is NOT the fp32 RMSNorm path). + q = q * torch.rsqrt(q.square().mean(-1, keepdim=True) + eps) + q = _rope_last_dims(q, rd, blk_freqs) + + # Block K/V. + kv = _rmsnorm(F.linear(x, wkv), kv_norm_w, eps) # [b, block, head_dim] + kv = _rope_last_dims(kv, rd, blk_freqs) + + # Write the context K/V into the rolling window, then attend over + # [window context | block] with the sink. ``persist=True`` writes through + # to the caller's buffer (cross-step decode, worker-owned window); the + # default clones so single-shot callers (golden / unit tests) stay pure. + cache = kv_cache if persist else kv_cache.clone() + cache[:, start_pos % window_size] = main_kv.squeeze(1) + kv_full = torch.cat([cache, kv], dim=1) # [b, window + block, head_dim] + topk = get_dspark_topk_idxs(window_size, b, block, start_pos, device=x.device) + o = dspark_sparse_attn(q, kv_full, attn_sink, topk, softmax_scale) # [b, block, h, head_dim] + o = _rope_last_dims(o, rd, blk_freqs, inverse=True) + + # Grouped low-rank O projection. + o = o.reshape(b, block, n_groups, -1) + wo_a_v = wo_a.view(n_groups, o_lora_rank, -1) + o = torch.einsum("bsgd,grd->bsgr", o, wo_a_v) + return F.linear(o.flatten(2), wo_b) + + +def dspark_attention_forward_batched( + x: torch.Tensor, + main_x: torch.Tensor, + start_pos: torch.Tensor, + kv_cache: torch.Tensor, + slots: torch.Tensor, + *, + wq_a: torch.Tensor, + q_norm_w: torch.Tensor, + wq_b: torch.Tensor, + wkv: torch.Tensor, + kv_norm_w: torch.Tensor, + wo_a: torch.Tensor, + wo_b: torch.Tensor, + attn_sink: torch.Tensor, + n_heads: int, + head_dim: int, + rope_head_dim: int, + n_groups: int, + o_lora_rank: int, + window_size: int, + eps: float, + softmax_scale: float, + freqs_cis: torch.Tensor, + persist: bool = False, +) -> torch.Tensor: + """Batched, CUDA-graph-safe captured-context DSpark draft attention. + + Numerically identical, per request, to :func:`dspark_attention_forward`, but + free of host syncs and data-dependent shapes so it can be captured into a CUDA + graph (the one-engine drafter runs inside the target's graph). The differences + from the scalar path are purely mechanical: + + * ``start_pos`` is a ``[G]`` int tensor (one absolute decode position per gen + request) instead of a python int; RoPE phases are *gathered* per request from + the fixed ``freqs_cis`` table rather than sliced. + * the rolling-window context K/V is written/read through the ``slots`` index + into a shared ``kv_cache`` (``persist=True`` writes through to the caller's + worker-owned buffer; otherwise a clone is used), instead of mutating a + per-request cache in place. + * the attended-position list has the fixed width ``window_size + block_size`` + with ``-1`` masking (see :func:`get_dspark_topk_idxs_batched`). + + Args: + x: ``[G, block, dim]`` block layer input (per gen request). + main_x: ``[G, 1, hidden]`` projected captured context. + start_pos: ``[G]`` int tensor of absolute decode positions (> 0). + kv_cache: ``[N, window_size, head_dim]`` rolling captured-context windows + (``N`` rows indexed by ``slots``; ``N == G`` for single-shot callers). + slots: ``[G]`` int tensor mapping each request to its ``kv_cache`` row. + freqs_cis: ``[maxlen, rope_head_dim // 2]`` precomputed plain-RoPE table; + must satisfy ``maxlen > start_pos.max() + block_size``. + + Returns: + ``[G, block, dim]`` attention output (residual stream contribution). + """ + g, block, _ = x.shape + rd = rope_head_dim + # Per-request RoPE phases gathered from the fixed table (no host-int slicing). + main_freqs = freqs_cis[start_pos].unsqueeze(1) # [G, 1, rd//2] + blk_pos = start_pos.unsqueeze(1) + 1 + torch.arange(block, device=x.device) # [G, block] + blk_freqs = freqs_cis[blk_pos] # [G, block, rd//2] + + # Captured-context K/V from main_x (MQA, shared across heads). + main_kv = _rmsnorm(F.linear(main_x, wkv), kv_norm_w, eps) # [G, 1, head_dim] + main_kv = _rope_last_dims_batched(main_kv, rd, main_freqs) + + # Query: low-rank + per-head RMS + RoPE. + q = _rmsnorm(F.linear(x, wq_a), q_norm_w, eps) + q = F.linear(q, wq_b).unflatten(-1, (n_heads, head_dim)) # [G, block, h, head_dim] + q = q * torch.rsqrt(q.square().mean(-1, keepdim=True) + eps) + q = _rope_last_dims_batched(q, rd, blk_freqs) + + # Block K/V. + kv = _rmsnorm(F.linear(x, wkv), kv_norm_w, eps) # [G, block, head_dim] + kv = _rope_last_dims_batched(kv, rd, blk_freqs) + + # Write the context K/V into the rolling window at slot start_pos%window_size, + # then attend over [window context | block]. ``persist=True`` writes through to + # the worker-owned buffer (cross-step decode); otherwise clone so single-shot + # callers stay pure. Indexed scatter/gather by (slots, slot_pos) is graph-safe. + write_target = kv_cache if persist else kv_cache.clone() + slot_pos = start_pos % window_size # [G] + write_target[slots, slot_pos] = main_kv.squeeze(1).to(write_target.dtype) + cache_rows = write_target[slots] # [G, window, head_dim] + kv_full = torch.cat([cache_rows, kv], dim=1) # [G, window + block, head_dim] + topk = get_dspark_topk_idxs_batched(window_size, block, start_pos) + o = dspark_sparse_attn(q, kv_full, attn_sink, topk, softmax_scale) # [G, block, h, head_dim] + o = _rope_last_dims_batched(o, rd, blk_freqs, inverse=True) + + # Grouped low-rank O projection. + o = o.reshape(g, block, n_groups, -1) + wo_a_v = wo_a.view(n_groups, o_lora_rank, -1) + o = torch.einsum("bsgd,grd->bsgr", o, wo_a_v) + return F.linear(o.flatten(2), wo_b) diff --git a/tensorrt_llm/_torch/models/dspark/draft.py b/tensorrt_llm/_torch/models/dspark/draft.py new file mode 100644 index 000000000000..1b47f4922965 --- /dev/null +++ b/tensorrt_llm/_torch/models/dspark/draft.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DSpark draft I/O logic is ported from DeepSeek's DeepSeek-V4-Pro-DSpark +# reference (`inference/model.py`, DSparkBlock.forward_embed / forward_head). +"""DSpark draft I/O: block input and proposal stages. + +This module holds the *framework-agnostic* (pure-torch) input/output stages of +the DSpark draft block, separated from the heavy V4 backbone (MLA + MoE + mHC) so +they can be unit-tested in isolation: + + - ``build_draft_input_ids``: ``[bonus_token, noise, noise, ...]`` block input. + - ``dspark_propose``: given the per-position backbone ``base_logits`` and the + Markov / confidence heads, run the autoregressive Markov refinement to sample + the block tokens and apply the static confidence-threshold truncation. + +The backbone (3 V4 blocks producing ``block_hidden``) lives in the model module; +this file is the part fully specified by the reference and validated against it. +""" + +from typing import Optional + +import torch +from torch import nn + +from .heads import confident_prefix_length + + +def build_draft_input_ids( + bonus_token_ids: torch.Tensor, *, block_size: int, noise_token_id: int +) -> torch.Tensor: + """``[batch] -> [batch, block_size]`` = ``[bonus, noise, noise, ...]``. + + The first position is the verified bonus token (the target's last accepted + token); the rest are the DSpark noise/mask token (id 128799 for V4-Pro). + """ + batch = bonus_token_ids.shape[0] + out = bonus_token_ids.new_full((batch, block_size), int(noise_token_id)) + out[:, 0] = bonus_token_ids + return out + + +def dspark_propose( + base_logits: torch.Tensor, + *, + bonus_token_ids: torch.Tensor, + block_hidden: torch.Tensor, + markov_head: Optional[nn.Module], + confidence_head: Optional[nn.Module], + block_size: int, + temperature: float = 0.0, + confidence_threshold: float = 0.0, + return_logits: bool = False, +) -> tuple: + """Produce DSpark draft tokens for one block (functional-first, static length). + + Args: + base_logits: ``[batch, block_size, vocab]`` from the backbone + lm_head. + bonus_token_ids: ``[batch]`` the token preceding the first draft position. + block_hidden: ``[batch, block_size, hidden]`` backbone hidden (feeds the + confidence head, and the RNN-head variant). + markov_head / confidence_head: the validated DSpark heads (may be None). + Returns: + draft_tokens: ``[batch, block_size]`` sampled tokens (full block; callers + keep the tensor fixed-width for CUDA-graph safety). + num_proposed: ``[batch]`` int32 — how many leading tokens survive the + static confidence-threshold truncation (== block_size when no head / + threshold<=0). + """ + batch = base_logits.shape[0] + # ``draft_logits`` are the per-position distributions the draft token is drawn + # from (markov-corrected when a head is present, else the raw base logits). + # Surfaced under ``return_logits`` for the §7.9 probabilistic-acceptance + # (1-TV) measurement; the normal path ignores them. + draft_logits = base_logits + if markov_head is not None: + draft_tokens, corrected = markov_head.sample_block_tokens( + base_logits, + first_prev_token_ids=bonus_token_ids, + hidden_states=block_hidden, + temperature=temperature, + ) + draft_logits = corrected + else: + from .heads import greedy_or_sample + + draft_tokens = greedy_or_sample(base_logits, temperature) + + # Scaffolding: confidence-based dynamic drafting is NOT enabled in this PR. + # The worker always calls with confidence_threshold=0.0, so the block below is + # inert and num_proposed stays == block_size (the full block is proposed). The + # returned num_proposed is intentionally not yet consumed by the speculative + # scheduler/verifier; wiring it through is a follow-up (see PR description). + num_proposed = torch.full( + (batch,), int(block_size), dtype=torch.int32, device=base_logits.device + ) + if confidence_head is not None and confidence_threshold > 0.0: + # prev token at position k is [bonus, draft_0, ..., draft_{k-1}] + prev_ids = torch.cat([bonus_token_ids.unsqueeze(1), draft_tokens[:, :-1]], dim=1) + prev_emb = ( + markov_head.get_prev_embeddings(prev_ids) + if (markov_head is not None and getattr(confidence_head, "with_markov", False)) + else None + ) + conf_logits = ( + confidence_head(block_hidden, prev_embeddings=prev_emb) + if prev_emb is not None + else confidence_head(block_hidden) + ) + # Per-request prefix truncation (batch handled row-wise to stay simple; + # functional-first scope typically runs batch=1 for the draft). + for b in range(batch): + num_proposed[b] = confident_prefix_length( + conf_logits[b : b + 1], block_size=block_size, threshold=confidence_threshold + ) + if return_logits: + return draft_tokens, num_proposed, draft_logits + return draft_tokens, num_proposed diff --git a/tensorrt_llm/_torch/models/dspark/heads.py b/tensorrt_llm/_torch/models/dspark/heads.py new file mode 100644 index 000000000000..c49e35fbafa0 --- /dev/null +++ b/tensorrt_llm/_torch/models/dspark/heads.py @@ -0,0 +1,254 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The DSpark Markov/RNN/confidence-head math is ported from DeepSeek's DeepSpec +# reference implementation (https://github.com/deepseek-ai/DeepSpec, MIT License). +"""DSpark draft-network heads (pure-torch, framework-agnostic). + +These modules implement the *sequential refinement* and *acceptance-confidence* +parts of DeepSeek's DSpark speculative-decoding draft network: + + - Markov head: a low-rank token-bigram logit bias ``logits_k += W2(W1[t_{k-1}])`` + applied autoregressively across the ``block_size`` draft positions (the cheap + "sequential" half of DSpark's "semi-parallel" drafting). RNN variant carries + a GRU-style recurrent state across positions. + - Confidence head: predicts a per-position acceptance probability; the cumulative + product over positions estimates prefix-acceptance and is used only to + *truncate* the proposed draft length (NOT to decide acceptance). + +This file deliberately depends on ``torch`` only so it can be unit-tested in +isolation (token-for-token) against the DeepSpec reference. +""" + +from typing import Optional + +import torch +from torch import nn + + +def greedy_or_sample(logits: torch.Tensor, temperature: float) -> torch.Tensor: + """Argmax for temperature<=0, else temperature-scaled multinomial. + + Args: + logits: ``[..., vocab]``. + Returns: + token ids with the trailing vocab dim reduced. + """ + if temperature <= 0.0: + return logits.argmax(dim=-1) + probs = torch.softmax(logits.float() / temperature, dim=-1) + flat = probs.reshape(-1, probs.shape[-1]) + sampled = torch.multinomial(flat, num_samples=1).squeeze(-1) + return sampled.view(probs.shape[:-1]) + + +class VanillaMarkov(nn.Module): + """Low-rank token-bigram logit bias: ``bias = W2(W1[token])``.""" + + markov_head_type = "vanilla" + + def __init__(self, *, vocab_size: int, markov_rank: int): + super().__init__() + self.vocab_size = int(vocab_size) + self.markov_rank = int(markov_rank) + assert self.markov_rank > 0, ( + f"VanillaMarkov requires markov_rank > 0, got {self.markov_rank}." + ) + self.markov_w1 = nn.Embedding(self.vocab_size, self.markov_rank) + self.markov_w2 = nn.Linear(self.markov_rank, self.vocab_size, bias=False) + + def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor: + return self.markov_w1(token_ids.long()) + + def project_bias(self, latent_states: torch.Tensor) -> torch.Tensor: + return self.markov_w2(latent_states) + + def compute_step_bias( + self, token_ids: torch.Tensor, hidden_states: Optional[torch.Tensor] + ) -> torch.Tensor: + del hidden_states + return self.project_bias(self.get_prev_embeddings(token_ids)) + + def apply_step_logits( + self, + logits: torch.Tensor, + *, + token_ids: torch.Tensor, + hidden_states: Optional[torch.Tensor], + ) -> torch.Tensor: + return logits + self.compute_step_bias(token_ids, hidden_states) + + def sample_block_tokens( + self, + base_logits: torch.Tensor, + *, + first_prev_token_ids: torch.Tensor, + hidden_states: Optional[torch.Tensor], + temperature: float = 0.0, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Autoregressive block sampling with the (memoryless) Markov bias. + + Args: + base_logits: ``[batch, block_size, vocab]`` from the backbone+lm_head. + first_prev_token_ids: ``[batch]`` token preceding the first position. + hidden_states: ``[batch, block_size, d]`` (unused by vanilla/gated). + Returns: + sampled_tokens ``[batch, block_size]``, corrected_logits ``[batch, block_size, vocab]``. + """ + batch_size, block_size = base_logits.shape[:2] + if block_size == 0: + empty = torch.empty(batch_size, 0, dtype=torch.long, device=base_logits.device) + return empty, base_logits + sampled, corrected = [], [] + prev = first_prev_token_ids.long() + for k in range(block_size): + step_hidden = None if hidden_states is None else hidden_states[:, k] + step_logits = self.apply_step_logits( + base_logits[:, k], token_ids=prev, hidden_states=step_hidden + ) + corrected.append(step_logits.unsqueeze(1)) + prev = greedy_or_sample(step_logits, temperature) + sampled.append(prev) + return torch.stack(sampled, dim=1), torch.cat(corrected, dim=1) + + +class GatedMarkovHead(VanillaMarkov): + """Markov bias gated by a sigmoid of [hidden, prev_embedding].""" + + markov_head_type = "gated" + + def __init__(self, *, vocab_size: int, markov_rank: int, hidden_size: int): + super().__init__(vocab_size=vocab_size, markov_rank=markov_rank) + self.gate_proj = nn.Linear(hidden_size + markov_rank, markov_rank) + + def compute_step_bias( + self, token_ids: torch.Tensor, hidden_states: Optional[torch.Tensor] + ) -> torch.Tensor: + assert hidden_states is not None + prev_emb = self.get_prev_embeddings(token_ids) + gate = torch.sigmoid(self.gate_proj(torch.cat([hidden_states, prev_emb], dim=-1))).to( + dtype=prev_emb.dtype + ) + return self.project_bias(gate * prev_emb) + + +class RNNHead(VanillaMarkov): + """GRU-style head carrying recurrent state across block positions.""" + + markov_head_type = "rnn" + + def __init__(self, *, vocab_size: int, markov_rank: int, hidden_size: int): + super().__init__(vocab_size=vocab_size, markov_rank=markov_rank) + self.hidden_size = int(hidden_size) + # [s_{k-1}; W1[x_{k-1}]; h_k] -> [gate; candidate; output] + self.joint_proj = nn.Linear(2 * markov_rank + hidden_size, 3 * markov_rank) + + def _rnn_step(self, state, prev_embeddings, hidden_states): + z = torch.cat([state, prev_embeddings, hidden_states], dim=-1) + gate_raw, cand_raw, out_raw = self.joint_proj(z).chunk(3, dim=-1) + gate = torch.sigmoid(gate_raw) + candidate = torch.tanh(cand_raw) + new_state = gate * state + (1.0 - gate) * candidate + bias = self.project_bias(torch.tanh(out_raw)) + return new_state, bias + + def sample_block_tokens( + self, + base_logits: torch.Tensor, + *, + first_prev_token_ids: torch.Tensor, + hidden_states: Optional[torch.Tensor], + temperature: float = 0.0, + ) -> tuple[torch.Tensor, torch.Tensor]: + assert hidden_states is not None + batch_size, block_size = base_logits.shape[:2] + if block_size == 0: + empty = torch.empty(batch_size, 0, dtype=torch.long, device=base_logits.device) + return empty, base_logits + state = torch.zeros( + batch_size, self.markov_rank, device=base_logits.device, dtype=hidden_states.dtype + ) + sampled, corrected = [], [] + prev = first_prev_token_ids.long() + for k in range(block_size): + prev_emb = self.get_prev_embeddings(prev) + state, bias = self._rnn_step(state, prev_emb, hidden_states[:, k]) + step_logits = base_logits[:, k] + bias + corrected.append(step_logits.unsqueeze(1)) + prev = greedy_or_sample(step_logits, temperature) + sampled.append(prev) + return torch.stack(sampled, dim=1), torch.cat(corrected, dim=1) + + +def build_markov_head( + *, markov_head_type: str, vocab_size: int, markov_rank: int, hidden_size: int +) -> Optional[nn.Module]: + """Factory mirroring DeepSpec ``build_markov_head``; returns None if rank==0.""" + if int(markov_rank) <= 0: + return None + kind = str(markov_head_type).lower() + if kind == "vanilla": + return VanillaMarkov(vocab_size=vocab_size, markov_rank=markov_rank) + if kind == "gated": + return GatedMarkovHead( + vocab_size=vocab_size, markov_rank=markov_rank, hidden_size=hidden_size + ) + if kind == "rnn": + return RNNHead(vocab_size=vocab_size, markov_rank=markov_rank, hidden_size=hidden_size) + raise ValueError(f"Unsupported markov_head_type: {markov_head_type!r}") + + +class DSparkConfidenceHead(nn.Module): + """Per-position acceptance-confidence predictor (DeepSpec AcceptRatePredictor). + + Input features are the backbone hidden state, optionally concatenated with the + Markov head's previous-token embedding. Output is a single logit per position. + """ + + def __init__(self, *, hidden_size: int, markov_rank: int = 0, with_markov: bool = False): + super().__init__() + self.with_markov = bool(with_markov) + input_dim = int(hidden_size) + (int(markov_rank) if with_markov else 0) + # The checkpoint stores ``proj`` as a bias-free bf16 weight, but the + # confidence score is computed in fp32 (mirrors the DeepSpec reference + # ``Linear(input_dim, 1, dtype=torch.float32)`` with the fp32 matmul). + self.proj = nn.Linear(input_dim, 1, bias=False, dtype=torch.float32) + + def forward( + self, hidden_states: torch.Tensor, prev_embeddings: Optional[torch.Tensor] = None + ) -> torch.Tensor: + if self.with_markov: + assert prev_embeddings is not None + features = torch.cat([hidden_states, prev_embeddings.to(hidden_states.dtype)], dim=-1) + else: + features = hidden_states + # fp32 matmul for a stable confidence score (mirrors the reference). + return self.proj(features.float()).squeeze(-1) + + +def confident_prefix_length( + confidence_logits: torch.Tensor, *, block_size: int, threshold: float +) -> int: + """First position k where ``sigmoid(confidence_k) < threshold``. + + Returns ``block_size`` when threshold<=0 (no truncation) or all positions + are confident. Assumes batch size 1 (functional-first scope). + """ + if threshold <= 0.0: + return int(block_size) + below = confidence_logits.sigmoid() < threshold + if not bool(below[0].any().item()): + return int(block_size) + return int(torch.nonzero(below[0], as_tuple=False)[0].item()) diff --git a/tensorrt_llm/_torch/models/modeling_bart.py b/tensorrt_llm/_torch/models/modeling_bart.py index 893f88f22dde..314537b3ccbc 100644 --- a/tensorrt_llm/_torch/models/modeling_bart.py +++ b/tensorrt_llm/_torch/models/modeling_bart.py @@ -18,20 +18,21 @@ Key differences from T5: - LayerNorm instead of RMSNorm. - - Post-norm (residual → add → LayerNorm) instead of pre-norm. + - BART uses post-norm; mBART uses pre-norm and final stack norms. - Learned absolute positional embeddings (not relative bias). - - GELU activation (not ReLU / gated). + - The checkpoint selects the MLP activation (typically GELU for BART and + ReLU for mBART). - Bias in attention and MLP projections. - - Embedding scale = sqrt(d_model). + - mBART scales token embeddings by sqrt(d_model). """ import math from typing import Dict, Optional import torch -import torch.nn.functional as F from torch import nn from transformers import BartConfig +from transformers.activations import ACT2FN from ..attention_backend import AttentionMetadata from ..attention_backend.interface import PredefinedAttentionMask @@ -86,6 +87,18 @@ def _bart_head_dim(config: BartConfig) -> int: return config.d_model // config.encoder_attention_heads +def _bart_normalize_before(config: BartConfig) -> bool: + return getattr(config, "model_type", None) == "mbart" or bool( + getattr(config, "normalize_before", False) + ) + + +def _bart_add_final_layer_norm(config: BartConfig) -> bool: + return getattr(config, "model_type", None) == "mbart" or bool( + getattr(config, "add_final_layer_norm", False) + ) + + def _packed_position_ids( position_ids: Optional[torch.IntTensor], hidden_states: torch.Tensor, @@ -167,7 +180,7 @@ def __init__( class BartEncoderLayer(nn.Module): - """BART encoder layer: self-attention → add+LN → MLP → add+LN (post-norm).""" + """BART/mBART encoder layer with configurable pre- or post-norm.""" def __init__( self, @@ -179,6 +192,7 @@ def __init__( hidden_size = config.d_model ffn_dim = _bart_encoder_ffn_dim(config) num_heads = _bart_encoder_num_heads(config) + self.normalize_before = _bart_normalize_before(config) self.self_attn = BartSelfAttention(model_config, num_heads=num_heads, layer_idx=layer_idx) @@ -187,13 +201,14 @@ def __init__( eps=1e-5, dtype=config.torch_dtype, has_bias=True, + residual_in_fp32=False, ) self.mlp = MLP( hidden_size=hidden_size, intermediate_size=ffn_dim, bias=True, - activation=F.gelu, + activation=ACT2FN[config.activation_function], dtype=config.torch_dtype, config=model_config, layer_idx=layer_idx, @@ -204,6 +219,7 @@ def __init__( eps=1e-5, dtype=config.torch_dtype, has_bias=True, + residual_in_fp32=False, ) def forward( @@ -214,19 +230,27 @@ def forward( **kwargs, ) -> torch.Tensor: residual = hidden_states + if self.normalize_before: + hidden_states = self.self_attn_layer_norm(hidden_states) hidden_states = self.self_attn( position_ids=position_ids, hidden_states=hidden_states, attn_metadata=attn_metadata, attention_mask=PredefinedAttentionMask.FULL, ) - hidden_states = residual + hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) + if self.normalize_before: + hidden_states = residual + hidden_states + else: + hidden_states, _ = self.self_attn_layer_norm(hidden_states, residual) residual = hidden_states + if self.normalize_before: + hidden_states = self.final_layer_norm(hidden_states) hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - hidden_states = self.final_layer_norm(hidden_states) + if self.normalize_before: + hidden_states = residual + hidden_states + else: + hidden_states, _ = self.final_layer_norm(hidden_states, residual) return hidden_states @@ -237,7 +261,7 @@ def forward( class BartDecoderLayer(nn.Module): - """BART decoder layer: self-attn → add+LN → cross-attn → add+LN → MLP → add+LN.""" + """BART/mBART decoder layer with configurable pre- or post-norm.""" def __init__( self, @@ -249,6 +273,7 @@ def __init__( hidden_size = config.d_model ffn_dim = _bart_decoder_ffn_dim(config) num_heads = _bart_decoder_num_heads(config) + self.normalize_before = _bart_normalize_before(config) self.self_attn = BartSelfAttention(model_config, num_heads=num_heads, layer_idx=layer_idx) @@ -257,6 +282,7 @@ def __init__( eps=1e-5, dtype=config.torch_dtype, has_bias=True, + residual_in_fp32=False, ) self.cross_attn = BartCrossAttention(model_config, layer_idx=layer_idx) @@ -266,13 +292,14 @@ def __init__( eps=1e-5, dtype=config.torch_dtype, has_bias=True, + residual_in_fp32=False, ) self.mlp = MLP( hidden_size=hidden_size, intermediate_size=ffn_dim, bias=True, - activation=F.gelu, + activation=ACT2FN[config.activation_function], dtype=config.torch_dtype, config=model_config, layer_idx=layer_idx, @@ -283,6 +310,7 @@ def __init__( eps=1e-5, dtype=config.torch_dtype, has_bias=True, + residual_in_fp32=False, ) def forward( @@ -295,19 +323,25 @@ def forward( skip_cross_kv_projection: bool = False, **kwargs, ) -> torch.Tensor: - # Self-attention (post-norm) + # Self-attention residual = hidden_states + if self.normalize_before: + hidden_states = self.self_attn_layer_norm(hidden_states) hidden_states = self.self_attn( position_ids=position_ids, hidden_states=hidden_states, attn_metadata=attn_metadata, attention_mask=PredefinedAttentionMask.CAUSAL, ) - hidden_states = residual + hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) + if self.normalize_before: + hidden_states = residual + hidden_states + else: + hidden_states, _ = self.self_attn_layer_norm(hidden_states, residual) - # Cross-attention (post-norm) + # Cross-attention residual = hidden_states + if self.normalize_before: + hidden_states = self.cross_attn_layer_norm(hidden_states) hidden_states = self.cross_attn( hidden_states=hidden_states, encoder_hidden_states=encoder_hidden_states, @@ -315,14 +349,20 @@ def forward( cross_attn_metadata=cross_attn_metadata, skip_cross_kv_projection=skip_cross_kv_projection, ) - hidden_states = residual + hidden_states - hidden_states = self.cross_attn_layer_norm(hidden_states) + if self.normalize_before: + hidden_states = residual + hidden_states + else: + hidden_states, _ = self.cross_attn_layer_norm(hidden_states, residual) - # MLP (post-norm) + # MLP residual = hidden_states + if self.normalize_before: + hidden_states = self.final_layer_norm(hidden_states) hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - hidden_states = self.final_layer_norm(hidden_states) + if self.normalize_before: + hidden_states = residual + hidden_states + else: + hidden_states, _ = self.final_layer_norm(hidden_states, residual) return hidden_states @@ -333,15 +373,15 @@ def forward( class BartEncoder(nn.Module): - """BART encoder: positional embedding + encoder layers.""" + """BART/mBART encoder: positional embedding + encoder layers.""" def __init__(self, model_config: ModelConfig[BartConfig]): super().__init__() config = model_config.pretrained_config num_layers = _bart_encoder_num_layers(config) - # HF BART uses offset=2 for the padding token, so the actual embedding - # table has max_position_embeddings + 2 entries. + # HF BART/mBART uses offset=2 for the padding token, so the actual + # embedding table has max_position_embeddings + 2 entries. self.embed_positions = Embedding( config.max_position_embeddings + 2, config.d_model, @@ -353,6 +393,16 @@ def __init__(self, model_config: ModelConfig[BartConfig]): dtype=config.torch_dtype, has_bias=True, ) + self.layer_norm = ( + LayerNorm( + hidden_size=config.d_model, + eps=1e-5, + dtype=config.torch_dtype, + has_bias=True, + ) + if _bart_add_final_layer_norm(config) + else None + ) self.layers = nn.ModuleList( [BartEncoderLayer(model_config, layer_idx=i) for i in range(num_layers)] ) @@ -374,11 +424,13 @@ def forward( attn_metadata=attn_metadata, position_ids=position_ids, ) + if self.layer_norm is not None: + hidden_states = self.layer_norm(hidden_states) return hidden_states class BartDecoder(nn.Module): - """BART decoder: positional embedding + decoder layers.""" + """BART/mBART decoder: positional embedding + decoder layers.""" def __init__(self, model_config: ModelConfig[BartConfig]): super().__init__() @@ -396,6 +448,16 @@ def __init__(self, model_config: ModelConfig[BartConfig]): dtype=config.torch_dtype, has_bias=True, ) + self.layer_norm = ( + LayerNorm( + hidden_size=config.d_model, + eps=1e-5, + dtype=config.torch_dtype, + has_bias=True, + ) + if _bart_add_final_layer_norm(config) + else None + ) self.layers = nn.ModuleList( [BartDecoderLayer(model_config, layer_idx=i) for i in range(num_layers)] ) @@ -423,6 +485,8 @@ def forward( cross_attn_metadata=cross_attn_metadata, skip_cross_kv_projection=skip_cross_kv_projection, ) + if self.layer_norm is not None: + hidden_states = self.layer_norm(hidden_states) return hidden_states @@ -623,6 +687,7 @@ def _convert_hf_bart_weights( model.shared.weight model.encoder.embed_positions.weight model.encoder.layernorm_embedding.{weight,bias} + model.encoder.layer_norm.{weight,bias} # mBART model.encoder.layers.{i}.self_attn.{q_proj,k_proj,v_proj,out_proj}.{weight,bias} model.encoder.layers.{i}.self_attn_layer_norm.{weight,bias} model.encoder.layers.{i}.fc1.{weight,bias} @@ -630,6 +695,7 @@ def _convert_hf_bart_weights( model.encoder.layers.{i}.final_layer_norm.{weight,bias} model.decoder.embed_positions.weight model.decoder.layernorm_embedding.{weight,bias} + model.decoder.layer_norm.{weight,bias} # mBART model.decoder.layers.{i}.self_attn.{q_proj,k_proj,v_proj,out_proj}.{weight,bias} model.decoder.layers.{i}.self_attn_layer_norm.{weight,bias} model.decoder.layers.{i}.encoder_attn.{q_proj,k_proj,v_proj,out_proj}.{weight,bias} @@ -675,6 +741,8 @@ def _wb(prefix: str) -> dict: # Encoder positional embedding out["model.encoder.embed_positions"] = [{"weight": _get(f"{p}encoder.embed_positions.weight")}] out["model.encoder.layernorm_embedding"] = [_wb(f"{p}encoder.layernorm_embedding")] + if _maybe(f"{p}encoder.layer_norm.weight") is not None: + out["model.encoder.layer_norm"] = [_wb(f"{p}encoder.layer_norm")] # Encoder layers for i in range(enc_layers): @@ -700,6 +768,8 @@ def _wb(prefix: str) -> dict: # Decoder positional embedding out["model.decoder.embed_positions"] = [{"weight": _get(f"{p}decoder.embed_positions.weight")}] out["model.decoder.layernorm_embedding"] = [_wb(f"{p}decoder.layernorm_embedding")] + if _maybe(f"{p}decoder.layer_norm.weight") is not None: + out["model.decoder.layer_norm"] = [_wb(f"{p}decoder.layer_norm")] # Decoder layers for i in range(dec_layers): diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv3.py b/tensorrt_llm/_torch/models/modeling_deepseekv3.py index 2ccc34a3cda4..c95aac6c8147 100755 --- a/tensorrt_llm/_torch/models/modeling_deepseekv3.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv3.py @@ -28,7 +28,7 @@ import copy import math import os -from typing import Dict, List, Optional, Tuple +from typing import Any, Dict, List, Literal, Optional, Tuple import torch import triton @@ -1893,6 +1893,30 @@ def forward( class DeepseekV3ForCausalLM(SpecDecOneEngineForCausalLM[DeepseekV3Model, PretrainedConfig]): + @classmethod + def get_preferred_transceiver_runtime(cls, + pretrained_config: Any = None + ) -> Optional[Literal["PYTHON"]]: + """GLM-5 family checkpoints default to the Python (v2) KV-cache transceiver. + + This implementation class is shared by DeepSeek-V3/V3.2 and the GLM-5 family — both + GLM-5 and GLM-5.2 declare ``GlmMoeDsaForCausalLM`` / ``glm_moe_dsa`` — so the preference + is differentiated per checkpoint: only GLM checkpoints opt into the Python transceiver. + The MLA backbone transfers a large latent KV, which the Python transceiver handles better + in disaggregated serving. This is only adopted when the user leaves + ``cache_transceiver_config.transceiver_runtime`` at 'auto' and the effective backend is + NIXL; otherwise the C++ transceiver is used. + """ + if pretrained_config is None: + return None + architectures = getattr(pretrained_config, 'architectures', None) or [] + # model_type is checked as a fallback: it is 'glm_moe_dsa' on GLM + # checkpoints until __init__ rewrites it to 'deepseek_v32'. + if ("GlmMoeDsaForCausalLM" in architectures or getattr( + pretrained_config, 'model_type', None) == 'glm_moe_dsa'): + return "PYTHON" + return None + def __init__(self, model_config: ModelConfig[PretrainedConfig]): self.mapping_with_cp = None # Note: Currently the usage of mapping is all over the place making its usage brittle diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv4.py b/tensorrt_llm/_torch/models/modeling_deepseekv4.py index 2fb611ecb999..99c12ba07168 100644 --- a/tensorrt_llm/_torch/models/modeling_deepseekv4.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv4.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + # -------------------------------------------------- # Portions of this code were derived from DeepSeek‑V3: # https://github.com/deepseek-ai/DeepSeek-V3 @@ -228,6 +231,66 @@ def moe_reduce_add_shared_output(routed_output, shared_output): } +def _get_deepseek_v4_routed_moe_scale_name(weights: Dict, key_prefix: str) -> str: + """Return the model scale suffix for routed-expert checkpoint tensors.""" + for key, value in weights.items(): + if ( + key.startswith(key_prefix) + and ".ffn.experts." in key + and key.endswith(".weight") + and getattr(value, "ndim", 0) == 2 + and getattr(value, "dtype", None) in (torch.int8, torch.uint8) + ): + return "weight_scale" + return "weight_scale_inv" + + +def _rename_deepseek_v4_attn_subkey(rest: str) -> str: + """Rename a DeepSeek-V4 attention checkpoint subkey.""" + if rest == "attn_sink": + return "attn_sink" + if rest == "wo_a.weight": + return "o_a_proj" + if rest == "wo_a.scale": + return "o_a_proj.weight_scale_inv" + if rest.startswith("compressor.") or rest.startswith("indexer."): + return rest.replace(".scale", ".weight_scale_inv") + head, sep, tail = rest.partition(".") + new_head = _ATTN_PARAM_RENAME.get(head, head) + if tail == "scale": + tail = "weight_scale_inv" + return f"{new_head}.{tail}" if sep else new_head + + +def _rename_deepseek_v4_ffn_subkey(rest: str, routed_moe_scale_name: str) -> str: + """Rename a DeepSeek-V4 FFN checkpoint subkey.""" + if rest == "gate.bias": + return "gate.e_score_correction_bias" + if rest.startswith("experts.") and rest.endswith(".scale"): + return f"{rest[: -len('.scale')]}.{routed_moe_scale_name}" + rest = rest.replace(".scale", ".weight_scale_inv") + if rest.startswith("shared_experts."): + parts = rest.split(".") + if len(parts) >= 2 and parts[1] in _SHARED_EXPERT_RENAME: + parts[1] = _SHARED_EXPERT_RENAME[parts[1]] + rest = ".".join(parts) + return rest + + +def _maybe_view_deepseek_v4_routed_moe_tensor( + model_key: str, tensor: torch.Tensor, routed_moe_scale_name: str +) -> torch.Tensor: + """Expose packed MXFP4 routed-expert tensors through their uint8 view.""" + if ( + routed_moe_scale_name == "weight_scale" + and ".mlp.experts." in model_key + and (model_key.endswith(".weight") or model_key.endswith(".weight_scale")) + and tensor.dtype != torch.uint8 + ): + return tensor.view(torch.uint8) + return tensor + + def _resolve_enable_fused_hc(config: PretrainedConfig) -> bool: """Resolve the DeepSeek-V4 fused HC boundary-fusion knob.""" env = os.environ.get("TRTLLM_MHC_ENABLE_FUSED_HC") @@ -236,6 +299,43 @@ def _resolve_enable_fused_hc(config: PretrainedConfig) -> bool: return bool(getattr(config, "enable_fused_hc", True)) +def _normalize_deepseek_v4_nvfp4_mixed_precision_config( + model_config: ModelConfig[PretrainedConfig], +) -> ModelConfig[PretrainedConfig]: + """Resolve FP8 base layers in DeepSeek-V4 NVFP4 checkpoints.""" + quant_config = model_config.quant_config + hf_quant_config = getattr(model_config.pretrained_config, "quantization_config", None) + layer_quant_configs = model_config.quant_config_dict or {} + has_nvfp4_experts = any( + name.endswith(".mlp.experts") and config.quant_algo == QuantAlgo.NVFP4 + for name, config in layer_quant_configs.items() + ) + if ( + quant_config.quant_algo != QuantAlgo.MIXED_PRECISION + or not has_nvfp4_experts + or not isinstance(hf_quant_config, dict) + or hf_quant_config.get("quant_method") != "fp8" + or tuple(hf_quant_config.get("weight_block_size", ())) != (128, 128) + ): + return model_config + + default_exclude = ["*kv_b_proj*", "*k_b_proj*", "*eh_proj*"] + hf_exclude_modules = hf_quant_config.get("modules_to_not_convert") or [] + exclude_modules = list(dict.fromkeys(list(hf_exclude_modules) + default_exclude)) + fp8_quant_config = quant_config.model_copy( + deep=True, + update={ + "quant_algo": QuantAlgo.FP8_BLOCK_SCALES, + "group_size": 128, + "exclude_modules": exclude_modules, + }, + ) + fp8_quant_config.__dict__.pop("quant_mode", None) + fp8_quant_config.__dict__.pop("layer_quant_mode", None) + model_config.quant_config = fp8_quant_config + return model_config + + def _copy_deepseek_v4_fused_a_weight_scale( module: Linear, fused_a: torch.Tensor, fused_a_scale: torch.Tensor ) -> None: @@ -315,66 +415,7 @@ def _remap_deepseek_v4_checkpoint_keys( carries it but matches the main head, so we let the main head win. """ mtp_layer_prefix = f"model.layers.{num_hidden_layers}" - routed_moe_scale_name = "weight_scale_inv" - for key, value in weights.items(): - if ( - key.startswith("layers.") - and ".ffn.experts." in key - and key.endswith(".weight") - and getattr(value, "ndim", 0) == 2 - and value.dtype in (torch.int8, torch.uint8) - ): - routed_moe_scale_name = "weight_scale" - break - - def _rename_attn_subkey(rest: str) -> Optional[str]: - # rest examples: "wq_a.weight", "wq_a.scale", "wo_a.weight", - # "attn_sink", "compressor.wkv.weight", "indexer.wq_b.scale", - # "kv_norm.weight" - # ``attn_sink`` is loaded by the ``mqa`` branch in the per-module - # loader, which reads it under the parent ``self_attn.attn_sink`` - # key. Pass through unchanged. - if rest == "attn_sink": - return "attn_sink" - # `wo_a` is an nn.Parameter on the model side (not a Linear), so - # `wo_a.weight` carries the value directly into `o_a_proj` without - # a trailing ``.weight``. Retain `.scale` so the loader can dequantize - # FP8 block-scaled checkpoints before assigning the bf16 parameter. - if rest == "wo_a.weight": - return "o_a_proj" - if rest == "wo_a.scale": - return "o_a_proj.weight_scale_inv" - # Compressor / indexer paths — pass through with .scale rename, plus - # wkv+wgate fusion handled separately below. - if rest.startswith("compressor.") or rest.startswith("indexer."): - return rest.replace(".scale", ".weight_scale_inv") - head, sep, tail = rest.partition(".") - new_head = _ATTN_PARAM_RENAME.get(head, head) - if tail == "scale": - tail = "weight_scale_inv" - return f"{new_head}.{tail}" if sep else new_head - - def _rename_ffn_subkey(rest: str) -> str: - # Examples: - # gate.weight / gate.tid2eid → gate.weight / gate.tid2eid - # gate.bias → gate.e_score_correction_bias - # experts... → experts... - # shared_experts.. → shared_experts._proj. - if rest == "gate.bias": - return "gate.e_score_correction_bias" - if rest.startswith("experts.") and rest.endswith(".scale"): - return f"{rest[: -len('.scale')]}.{routed_moe_scale_name}" - rest = rest.replace(".scale", ".weight_scale_inv") - # Non-hashed layers carry the routing logit bias as `gate.bias`; the - # model wires it through `DeepseekV4Gate.e_score_correction_bias`. - if rest == "gate.bias": - return "gate.e_score_correction_bias" - if rest.startswith("shared_experts."): - parts = rest.split(".") - if len(parts) >= 2 and parts[1] in _SHARED_EXPERT_RENAME: - parts[1] = _SHARED_EXPERT_RENAME[parts[1]] - rest = ".".join(parts) - return rest + routed_moe_scale_name = _get_deepseek_v4_routed_moe_scale_name(weights, "layers.") def _rename_layer_subkey(rest: str) -> Optional[str]: # rest examples: "attn_norm.weight", "ffn_norm.weight", @@ -390,10 +431,10 @@ def _rename_layer_subkey(rest: str) -> Optional[str]: if rest.startswith("hc_attn_") or rest.startswith("hc_ffn_"): return rest if rest.startswith("attn."): - new_sub = _rename_attn_subkey(rest[len("attn.") :]) - return None if new_sub is None else f"self_attn.{new_sub}" + return f"self_attn.{_rename_deepseek_v4_attn_subkey(rest[len('attn.') :])}" if rest.startswith("ffn."): - return f"mlp.{_rename_ffn_subkey(rest[len('ffn.') :])}" + new_sub = _rename_deepseek_v4_ffn_subkey(rest[len("ffn.") :], routed_moe_scale_name) + return f"mlp.{new_sub}" return rest out: Dict[str, torch.Tensor] = {} @@ -417,13 +458,7 @@ def _emit_or_collect(model_key: str, tensor: torch.Tensor): part = "wkv" if model_key.endswith(".wkv.weight") else "wgate" _record_compressor_part(model_key, part, tensor) return - if ( - routed_moe_scale_name == "weight_scale" - and ".mlp.experts." in model_key - and (model_key.endswith(".weight") or model_key.endswith(".weight_scale")) - and tensor.dtype != torch.uint8 - ): - tensor = tensor.view(torch.uint8) + tensor = _maybe_view_deepseek_v4_routed_moe_tensor(model_key, tensor, routed_moe_scale_name) out[model_key] = tensor for k, v in weights.items(): @@ -1482,7 +1517,17 @@ def __init__( moe_cls in (TRTLLMGenFusedMoE, WideEPMoE) and experts_quant_config.quant_mode.has_nvfp4() ) - if supports_swiglu_limit and not kernel_requires_bias_for_swiglu_limit: + # DeepSeek-V4 supplies a uniform scalar limit. The TRTLLM-Gen FP8 + # path consumes it directly and rejects the redundant tensor. + requires_scalar_only_swiglu_limit = ( + moe_cls is TRTLLMGenFusedMoE + and experts_quant_config.quant_mode.has_fp8_block_scales() + ) + if ( + supports_swiglu_limit + and not kernel_requires_bias_for_swiglu_limit + and not requires_scalar_only_swiglu_limit + ): moe_load_balancer_config = getattr(model_config, "moe_load_balancer", None) num_slots = ( moe_load_balancer_config.num_slots @@ -1703,7 +1748,7 @@ def __init__( model_config: ModelConfig[PretrainedConfig], layer_idx: int, aux_stream_dict: Dict[AuxStreamType, torch.cuda.Stream], - is_separate_draft_engine: bool = False, + attention_layer_idx: Optional[int] = None, mapping_with_cp: Optional[Mapping] = None, disable_post_moe_fusion: bool = False, ): @@ -1732,14 +1777,12 @@ def __init__( post_mult_value=2.0, ) - layer_idx_for_attention = layer_idx - if is_separate_draft_engine: - # KVCacheManager only support 1 layer for separate draft engine - layer_idx_for_attention = layer_idx - model_config.pretrained_config.num_hidden_layers + if attention_layer_idx is None: + attention_layer_idx = layer_idx self.self_attn = DeepseekV4Attention( model_config, - layer_idx=layer_idx_for_attention, + layer_idx=attention_layer_idx, aux_stream=aux_stream_dict[AuxStreamType.Attention], reduce_output=not self.enable_attention_dp and self.mapping.tp_size > 1, ) @@ -1960,8 +2003,18 @@ def forward( # No engram concern here because engram only fires at layer entry. # When enable_fused_hc=False, fall back to the unfused chain. # ------------------------------------------------------------------- - if spec_metadata is not None and spec_metadata.is_layer_capture(self.layer_idx): + capture_this_layer = spec_metadata is not None and spec_metadata.is_layer_capture( + self.layer_idx + ) + is_dspark_capture = capture_this_layer and spec_metadata.spec_dec_mode.is_dspark() + if capture_this_layer: self.fusion_config.POST_MOE_FUSION = False + if is_dspark_capture: + # DSpark captures the FULL post-mapped mHC residual stream (reference + # `h.mean(dim=2)`), which is only materialized after + # hc_ffn.post_mapping -- so post_mapping must resolve in-layer rather + # than being deferred into the next layer's fused_hc. + self.defer_post_mapping = False if self.enable_fused_hc: residual, post_mix, comb_mix, layer_input = self.hc_ffn.fused_hc( x_prev=x_attn, @@ -2003,6 +2056,17 @@ def forward( post_layer_mix=post_mix, comb_res_mix=comb_mix, ) + if is_dspark_capture: + # Capture the full mHC residual stream [N, hc_mult*hidden]; the DSpark + # metadata means over the hc streams (reference `h.mean(dim=2)`) to + # form the draft's captured context (``main_x``). This is the correct + # representation -- NOT the pre-post_mapping MoE delta that the generic + # capture in forward_MoE records for other spec modes. + spec_metadata.maybe_capture_hidden_states( + self.layer_idx, + resolved_residual.reshape(resolved_residual.shape[0], -1), + None, + ) return HCState.resolved(resolved_residual) def _entry_boundary(self, hc_state, engram_embeddings, has_engram): @@ -2131,7 +2195,14 @@ def _run_MoE(hidden_states, hidden_states_fp4, do_finalize, input_ids): fc2_output, all_reduce_params=moe_all_reduce_params ) else: - if spec_metadata is not None and spec_metadata.is_layer_capture(self.layer_idx): + # DSpark captures the post-mapped mHC residual stream after this layer + # (done in the decoder-layer forward), not the pre-post_mapping MoE + # output recorded here for other spec modes. + if ( + spec_metadata is not None + and spec_metadata.is_layer_capture(self.layer_idx) + and not spec_metadata.spec_dec_mode.is_dspark() + ): spec_metadata.maybe_capture_hidden_states(self.layer_idx, hidden_states, None) return hidden_states @@ -2143,13 +2214,13 @@ def __init__( model_config: ModelConfig[PretrainedConfig], layer_idx: int, aux_stream_dict: Dict[AuxStreamType, torch.cuda.Stream], - is_separate_draft_engine: bool = False, + attention_layer_idx: Optional[int] = None, ): super().__init__( model_config, layer_idx, aux_stream_dict, - is_separate_draft_engine, + attention_layer_idx=attention_layer_idx, disable_post_moe_fusion=True, ) config = model_config.pretrained_config @@ -2462,6 +2533,7 @@ def get_model_defaults(cls, llm_args: "TorchLlmArgs") -> dict: } def __init__(self, model_config: ModelConfig[PretrainedConfig]): + model_config = _normalize_deepseek_v4_nvfp4_mixed_precision_config(model_config) self.mapping_with_cp = None # Note: Currently the usage of mapping is all over the place making its usage brittle # in this file. As a temporary WAR, we hold on to an original copy of mapping when CP diff --git a/tensorrt_llm/_torch/models/modeling_dspark.py b/tensorrt_llm/_torch/models/modeling_dspark.py new file mode 100644 index 000000000000..290732a73037 --- /dev/null +++ b/tensorrt_llm/_torch/models/modeling_dspark.py @@ -0,0 +1,1245 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DSpark backbone ported from the DeepSeek-V4-Pro-DSpark reference +# (`inference/model.py`: DSparkBlock / Transformer.forward_spec). +"""DeepSeek-V4-Pro DSpark speculative-decoding draft backbone. + +The DSpark draft is ``n_mtp_layers`` (3 for V4-Pro) **full DeepSeek-V4 blocks** +stored under the ``mtp.*`` checkpoint namespace — it reuses the V4 decoder block +(MLA attention + MoE + manifold Hyper-Connections) and adds: + + - **stage 0**: ``main_proj`` (Linear, fp8) + ``main_norm`` (RMSNorm) — projects + the concatenation of captured target-layer hidden states ([58,59,60]) into the + draft's cross-attention context (``main_x``); replaces vanilla-MTP's + enorm/hnorm + e_proj/h_proj single-hidden mixing. + - **last stage**: ``norm`` + ``markov_head`` + ``confidence_head`` + + flat ``hc_head`` — the block-draft output head (see dspark_heads/dspark_draft). + +The per-stage *backbone* forward (block attention whose K/V derive from ``main_x``, ++ MoE + mHC) is brought up and numerically validated against the real fp8 weights +separately; ``forward_embed`` (capture) and ``forward_head`` (block draft) below +are the reference-faithful, unit-validated I/O stages. +""" + +import copy +import json +import os +import re +from typing import Dict, List, Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from tensorrt_llm.logger import logger +from tensorrt_llm.quantization.mode import QuantAlgo + +from ..distributed import AllReduceParams +from ..modules.linear import Linear +from ..modules.mhc.hyper_connection import HCHead +from ..modules.rms_norm import RMSNorm +from ..utils import AuxStreamType +from .dspark.attention import ( + _rmsnorm, + _rope_last_dims, + _rope_last_dims_batched, + dspark_attention_forward, + dspark_attention_forward_batched, + precompute_dspark_freqs_cis, +) +from .dspark.draft import build_draft_input_ids, dspark_propose +from .dspark.heads import DSparkConfidenceHead, build_markov_head +from .modeling_deepseekv4 import ( + DeepseekV4DecoderLayer, + DeepseekV4WeightLoader, + _get_deepseek_v4_routed_moe_scale_name, + _maybe_view_deepseek_v4_routed_moe_tensor, + _normalize_deepseek_v4_nvfp4_mixed_precision_config, + _rename_deepseek_v4_attn_subkey, + _rename_deepseek_v4_ffn_subkey, +) + +# Matches the draft namespace ``mtp..`` in the V4-Pro-DSpark +# checkpoint. Each draft stage is a full DeepSeek-V4 block stored under this +# prefix; the main model's keys (``layers.*``, ``embed.weight``, ``head.weight``, +# top-level ``norm.weight`` / ``hc_head_*``) are loaded by the target model. +_DSPARK_MTP_RE = re.compile(r"^mtp\.(\d+)\.(.+)$") + + +def _active_moe_load_balancer(): + """The engine-wide ``MoeLoadBalancer``, or None when EPLB is not active. + + Non-None exactly inside ``maybe_create_moe_load_balancer(...)`` when EPLB is + really enabled for this engine (supported arch, ``moe_ep_size > 1``, no smart + router, ``moe_load_balancer`` configured) -- i.e. exactly the condition under + which ``MoE._init_load_balancer`` consumes ``model_config.moe_load_balancer``. + Gating every DSpark EPLB check on it keeps the non-EPLB path untouched. + """ + from ..modules.fused_moe.moe_load_balancer import get_moe_load_balancer + + return get_moe_load_balancer() + + +def validate_dspark_eplb_layer_base(model_config, draft_config) -> None: + """Require the draft's layer namespace to match the target's, under EPLB. + + DSpark stages take ``layer_idx = draft_config.num_hidden_layers + stage_id`` + and register as extra EPLB layers in the *target* engine's balancer, whose + ``initial_global_assignments`` are keyed by target layer index. If the draft + checkpoint's config reports a different depth the stages silently land on the + wrong keys, so fail fast instead. Only enforced when EPLB is active; a + draft-only checkpoint config remains valid without EPLB. + """ + if _active_moe_load_balancer() is None: + return + target_layers = model_config.pretrained_config.num_hidden_layers + draft_layers = draft_config.pretrained_config.num_hidden_layers + if target_layers != draft_layers: + raise ValueError( + "DSpark + EPLB requires the draft checkpoint config to report the " + f"same num_hidden_layers as the target (target={target_layers}, " + f"draft={draft_layers}). DSpark stage layer indices are derived as " + "draft num_hidden_layers + stage_id and must line up with the " + "target layer namespace that initial_global_assignments is keyed by." + ) + + +def validate_dspark_eplb_stage_layers(model_config, base: int, num_stages: int) -> None: + """Validate the EPLB config actually covers the DSpark draft stages. + + DSpark registers each stage as an independent EPLB layer at index + ``base + stage_id`` (``base = num_hidden_layers``). Two failure modes are + caught here, before any DSpark MoE layer is built, so the user gets one + actionable error instead of a bare ``KeyError`` from deep inside MoE init: + + 1. online EPLB, which DSpark does not support (see below); + 2. an ``initial_global_assignments`` map generated without DSpark enabled, + which therefore lacks the draft stage indices. + """ + if _active_moe_load_balancer() is None: + return + lb_config = getattr(model_config, "moe_load_balancer", None) + if lb_config is None: + return + + draft_layers = list(range(base, base + num_stages)) + + # DSpark supports STATIC EPLB only. Online EPLB requires every registered MoE + # layer to run exactly once per iteration, but the DSpark draft MoE is skipped + # on iterations with no generation requests anywhere (context-only batches, + # warmup), and the balancer's CPU worker then spins forever in its untimed + # waitCpuStage() waiting for a GPU signal that is only emitted from an MoE + # forward -- a silent deadlock. + if getattr(lb_config, "layer_updates_per_iter", 0) > 0: + raise ValueError( + "DSpark speculative decoding supports static EPLB only, but " + f"layer_updates_per_iter={lb_config.layer_updates_per_iter} requests " + "online EPLB. The DSpark draft MoE does not run on iterations without " + "generation requests (context-only batches, warmup), which deadlocks " + "the MoE load balancer worker. Set layer_updates_per_iter=0 in the " + "load balancer config, or disable DSpark." + ) + + assignments = getattr(lb_config, "initial_global_assignments", None) + if not assignments: + # No custom placement: the auto-generated assignment covers every layer. + return + missing = [layer_idx for layer_idx in draft_layers if layer_idx not in assignments] + if missing: + raise ValueError( + f"initial_global_assignments is missing DSpark layer(s) {missing}. " + f"The {num_stages} DSpark draft stages register as additional EPLB " + f"layers with indices [{base}, {base + num_stages}). Regenerate the " + "EPLB config from statistics collected with DSpark enabled (see " + "examples/wide_ep/ep_load_balancer/README.md), or omit " + "initial_global_assignments to use the auto-generated placement." + ) + + +def count_dspark_stages(ckpt_dir: str) -> Optional[int]: + """Count the DSpark draft stages (``mtp.{s}.*``) in a checkpoint index. + + The HF ``config.json`` does not expose ``n_mtp_layers`` (only the reference + ``inference/config.json`` does), so the authoritative draft stage count is + the number of distinct ``mtp.`` prefixes in the weight index. Returns + ``None`` if the index is missing or has no ``mtp.*`` keys (caller falls back + to the config-derived default). + """ + index = os.path.join(ckpt_dir, "model.safetensors.index.json") + if not os.path.isfile(index): + return None + with open(index, encoding="utf-8") as f: + weight_map = json.load(f).get("weight_map", {}) + stages = {int(m.group(1)) for k in weight_map if (m := _DSPARK_MTP_RE.match(k))} + return (max(stages) + 1) if stages else None + + +def _rename_dspark_stage_subkey(rest: str, routed_scale: str) -> str: + """Map a per-stage checkpoint subkey to the ``DSparkBlock`` param subkey.""" + if rest == "attn_norm.weight": + return "input_layernorm.weight" + if rest == "ffn_norm.weight": + return "post_attention_layernorm.weight" + # Flat manifold-Hyper-Connections / draft-head weights are loaded via + # ``load_flat_hc_weights`` (keyed by the parent module stem), so pass the + # flat-underscore form through unchanged: + # hc_attn_* / hc_ffn_* -> mHC on every block + # hc_head_* -> HCHead on the last stage + if rest.startswith(("hc_attn_", "hc_ffn_", "hc_head_")): + return rest + # DSpark capture projection (stage 0): fp8 Linear .scale -> .weight_scale_inv. + if rest == "main_proj.scale": + return "main_proj.weight_scale_inv" + if rest.startswith("attn."): + return f"self_attn.{_rename_deepseek_v4_attn_subkey(rest[len('attn.') :])}" + if rest.startswith("ffn."): + return f"mlp.{_rename_deepseek_v4_ffn_subkey(rest[len('ffn.') :], routed_scale)}" + # main_proj.weight, main_norm.weight, norm.weight, markov_head.*, + # confidence_head.* map 1:1 onto the DSparkBlock submodules. + return rest + + +def remap_dspark_draft_keys(weights: Dict, num_stages: int) -> Dict: + """Convert checkpoint ``mtp.{s}.*`` keys to ``mtp_layers.{s}.*`` model keys. + + Only the draft namespace is consumed (stages ``[0, num_stages)``); shared + ``embed_tokens`` / ``lm_head`` and other top-level keys belong to the target + model and are skipped here. The routed-expert scale suffix mirrors the V4 + loader: ``weight_scale`` for the packed MXFP4 layout, else ``weight_scale_inv``. + """ + routed_scale = _get_deepseek_v4_routed_moe_scale_name(weights, "mtp.") + out: Dict[str, torch.Tensor] = {} + for k, v in weights.items(): + m = _DSPARK_MTP_RE.match(k) + if not m: + continue + stage = int(m.group(1)) + if stage >= num_stages: + continue + sub = _rename_dspark_stage_subkey(m.group(2), routed_scale) + model_key = f"mtp_layers.{stage}.{sub}" + v = _maybe_view_deepseek_v4_routed_moe_tensor(model_key, v, routed_scale) + out[model_key] = v + return out + + +# The checkpoint stores +# ``mtp.{s}.attn.wo_a`` as fp8_e4m3 + a UE8M0 128x128 block scale (verified), and +# the reference (`inference/model.py`, ``self.wo_a`` is a bf16 ColumnParallelLinear +# loaded from the fp8 ckpt) uses the DEQUANTIZED bf16 ``wo_a`` (== ``wo_a_fp8 * +# scale`` ~ absmean 0.065). The bf16 captured-context path historically skipped +# this dequant (raw fp8-cast-to-bf16, ~993x too large); the correct behavior is to +# dequantize ``wo_a`` (cos 1.0 vs ``wo_a_fp8 * scale``). Always dequantize now. + + +class DSparkBlock(DeepseekV4DecoderLayer): + """One DSpark draft stage = a DeepSeek-V4 decoder block + DSpark extras. + + ``stage_id`` in ``[0, num_stages)``; only stage 0 owns the capture projection + and only the last stage owns the draft heads, matching the ``mtp.*`` schema. + """ + + def __init__( + self, + model_config, + layer_idx: int, + aux_stream_dict: Dict[AuxStreamType, torch.cuda.Stream], + *, + stage_id: int, + num_stages: int, + num_capture_layers: int, + ): + # The inherited attention uses a draft-local layer index, while the + # decoder layer keeps its model-level index for weights and captures. + super().__init__( + model_config, + layer_idx, + aux_stream_dict, + attention_layer_idx=stage_id, + disable_post_moe_fusion=True, + ) + config = model_config.pretrained_config + spec_cfg = getattr(model_config, "spec_config", None) + self.stage_id = int(stage_id) + self.num_stages = int(num_stages) + # mask_token_id is a user override on the speculative_config; None means + # fall back to the draft checkpoint's dspark_noise_token_id. + mask_token_id = getattr(spec_cfg, "mask_token_id", None) + self.noise_token_id = int( + mask_token_id + if mask_token_id is not None + else getattr(config, "dspark_noise_token_id", config.vocab_size) + ) + self.markov_rank = int(getattr(config, "dspark_markov_rank", 0)) + self.hc_mult = config.hc_mult + # markov_head_type is a user override on the speculative_config; None + # means fall back to the draft checkpoint's dspark_markov_head_type. + markov_head_type = getattr(spec_cfg, "markov_head_type", None) + if markov_head_type is None: + markov_head_type = getattr(config, "dspark_markov_head_type", "vanilla") + self.markov_head_type = markov_head_type + + # Stage 0: capture projection of the concatenated target-layer hiddens. + if self.has_capture: + self.main_proj = Linear( + config.hidden_size * num_capture_layers, + config.hidden_size, + bias=False, + dtype=config.torch_dtype, + quant_config=model_config.get_quant_config(), + skip_create_weights_in_init=model_config.skip_create_weights_in_init, + ) + self.main_norm = RMSNorm( + hidden_size=config.hidden_size, eps=config.rms_norm_eps, dtype=config.torch_dtype + ) + + # Last stage: the block-draft output heads + mHC head + final norm. + if self.has_heads: + self.norm = RMSNorm( + hidden_size=config.hidden_size, eps=config.rms_norm_eps, dtype=config.torch_dtype + ) + self.hc_head = HCHead(config.hc_mult, config.hidden_size) + self.markov_head = build_markov_head( + markov_head_type=self.markov_head_type, + vocab_size=config.vocab_size, + markov_rank=self.markov_rank, + hidden_size=config.hidden_size, + ) + self.confidence_head = DSparkConfidenceHead( + hidden_size=config.hidden_size, + markov_rank=self.markov_rank, + # Only concat the Markov prev-token embedding when a Markov head + # actually exists (build_markov_head returns None for + # markov_rank <= 0); otherwise dspark_propose passes no + # prev_embeddings and DSparkConfidenceHead.forward would assert. + with_markov=self.markov_rank > 0, + ) + + @property + def has_capture(self) -> bool: + return self.stage_id == 0 + + @property + def has_heads(self) -> bool: + return self.stage_id == self.num_stages - 1 + + +class DSparkDraftModel(nn.Module): + """The ``n_mtp_layers``-stage DSpark draft stacked on a DeepSeek-V4 target. + + Shares ``embed_tokens`` / ``lm_head`` with the target model. ``forward_embed`` + builds the block input from the captured context; the per-stage backbone runs + the 3 blocks; ``forward_head`` produces the block draft tokens + confidence. + """ + + def __init__( + self, + model_config, + aux_stream_dict: Dict[AuxStreamType, torch.cuda.Stream], + num_stages: Optional[int] = None, + block_size: Optional[int] = None, + ): + super().__init__() + config = model_config.pretrained_config + self.model_config = model_config + self.config = config + # The DSpark stage count is NOT the HF ``num_nextn_predict_layers`` (=1). + # It is ``n_mtp_layers`` (3 for V4-Pro), which lives in the draft + # sub-checkpoint config (inference/config.json) and is reflected by the + # ``mtp.{0..n-1}.*`` weight namespace. Resolve it from (in priority): + # an explicit override, the spec config's ``num_draft_layers``, a + # pretrained-config ``n_mtp_layers``, else fall back to nextn. + spec_cfg = getattr(model_config, "spec_config", None) + self.num_stages = int( + num_stages + if num_stages is not None + else getattr(spec_cfg, "num_draft_layers", None) + or getattr(config, "n_mtp_layers", None) + or config.num_nextn_predict_layers + ) + # Production passes the validated speculative-config value explicitly; + # direct construction falls back to the checkpoint's trained block size. + self.block_size = int( + block_size if block_size is not None else getattr(config, "dspark_block_size", 5) + ) + # mask_token_id is a user override on the speculative_config; None means + # fall back to the draft checkpoint's dspark_noise_token_id. + mask_token_id = getattr(spec_cfg, "mask_token_id", None) + self.noise_token_id = int( + mask_token_id + if mask_token_id is not None + else getattr(config, "dspark_noise_token_id", config.vocab_size) + ) + self.hc_mult = config.hc_mult + target_layer_ids = getattr(config, "dspark_target_layer_ids", []) + self.num_capture_layers = len(target_layer_ids) + base = config.num_hidden_layers + # Each DSpark stage becomes an independent EPLB layer at index base + s. + # Validate the load-balancer config covers them (and rejects online EPLB) + # before building any MoE, so a stale config fails with one actionable + # error instead of a bare KeyError from inside MoE._init_load_balancer. + validate_dspark_eplb_stage_layers(model_config, base, self.num_stages) + # Derive a draft-only model_config (a shallow copy so the shared config + # and the target model are untouched) carrying two draft-specific fixes: + # + # 1. compress_ratios SLICE — the draft runs as a separate engine, so the + # inherited DeepSeek-V4 block remaps each block's layer_idx to a + # draft-local index in [0, num_stages) (the 1-layer-style draft KV + # cache). Sparse-attention compress_ratios / RoPE are indexed by that + # draft-local id, so they must be the draft slice + # (compress_ratios[base : base + num_stages]); otherwise indices + # 0..n-1 resolve to the first *main* layers' sparse ratios — building + # a compressor the DSpark draft lacks and selecting YaRN over the + # dense path. For V4-Pro the draft slice is [1, 1, 1] (dense). + # + # 2. quant_config_dict EXTENSION — the checkpoint's per-module quant map + # only enumerates the base layers, so the draft layers' routed + # experts fall back to the global fp8 config and build fp8-shaped + # buffers. The draft experts are physically MXFP4 (same as the main + # MoE layers), so copy a main MoE layer's experts quant onto the + # draft layer keys. + draft_model_config = self._derive_draft_model_config(model_config, base, self.num_stages) + self.mtp_layers = nn.ModuleList( + [ + DSparkBlock( + draft_model_config, + base + s, + aux_stream_dict, + stage_id=s, + num_stages=self.num_stages, + num_capture_layers=self.num_capture_layers, + ) + for s in range(self.num_stages) + ] + ) + # Shared with target; wired by the spec wrapper after construction. + self.embed_tokens: Optional[nn.Module] = None + self.lm_head: Optional[nn.Module] = None + + # Scalar attention params for the captured-context draft attention. These + # are the dense (compress_ratio == 0) DSparkAttention constants — see the + # reference ``inference/model.py`` ``Attention.__init__``. ``head_dim`` is + # the MLA latent (MQA) dim; ``softmax_scale = head_dim ** -0.5``; the dense + # draft disables YaRN and uses the base ``rope_theta``. + self._attn_params = dict( + n_heads=int(config.num_attention_heads), + head_dim=int( + getattr(config, "head_dim", config.kv_lora_rank + config.qk_rope_head_dim) + ), + rope_head_dim=int(config.qk_rope_head_dim), + n_groups=int(config.o_groups), + o_lora_rank=int(config.o_lora_rank), + window_size=int(getattr(config, "window_size", 128)), + eps=float(config.rms_norm_eps), + ) + self._attn_params["softmax_scale"] = self._attn_params["head_dim"] ** -0.5 + self._rope_theta = float(getattr(config, "rope_theta", 10000.0)) + # Fixed-cap plain-RoPE table shared by the eager and CUDA-graph-safe + # batched paths. It is built once per device and gathered/sliced by the + # runtime decode positions, so the cache does not grow with sequence + # length and the batched consuming op's shape remains static. + self._freqs_cap = ( + int(getattr(config, "max_position_embeddings", 163840)) + self.block_size + 2 + ) + self._freqs_table_cache: Dict = {} + + def post_load_weights(self) -> None: + """Run the one-shot post-load transforms for the draft's quant linears. + + The fp8 UE8M0 linears we invoke as modules (``main_proj``, shared experts, + the heads) need ``resmooth_to_fp8_e8m0`` + ``transform_sf_into_required_layout`` + before the first forward, or the kernel reads raw scales and emits NaNs. + ``Linear.transform_weights`` is idempotent; the routed-expert MoE packs + itself in its own ``load_weights``. + + The bf16 captured-context attention does NOT use the MLA module's forward — + it runs ``dspark_attention_forward`` on dequantized bf16 weights cached via + :meth:`cache_attn_weights_from_checkpoint` — so the MLA projection linears are + skipped here (they would otherwise be transformed into the deep_gemm layout we + don't consume). + """ + attn_linear_ids = set() + for stage in self.mtp_layers: + for m in stage.self_attn.modules(): + if isinstance(m, Linear): + attn_linear_ids.add(id(m)) + + for module in self.modules(): + if isinstance(module, Linear) and id(module) not in attn_linear_ids: + module.transform_weights() + + @staticmethod + def _block_dequant(w_fp8: torch.Tensor, scale: torch.Tensor, block: int = 128) -> torch.Tensor: + """DeepSeek ``block``×``block`` block-scale dequant → bf16: ``real = fp8 * scale``. + + ``scale`` (possibly UE8M0) is broadcast over each ``block``×``block`` tile. + Pure-torch (matches the golden-validated reference dequant), robust to the + e8m0 scale dtype. + """ + wf = w_fp8.float() + out, inn = wf.shape + s = scale.float() + s_full = s.repeat_interleave(block, 0)[:out].repeat_interleave(block, 1)[:, :inn] + return (wf * s_full).to(torch.bfloat16) + + def _cache_attn_weights(self, src: Dict) -> None: + """Populate each stage's ``_dspark_attn`` from a dict of raw ``mtp.{s}.attn.*`` + tensors (source-agnostic core shared by the two public entry points). + + The captured-context attention runs the validated ``dspark_attention_forward`` + free function on reference-layout bf16 weights dequantized here. Sourcing the + separate ``wq_a``/``wkv`` (plain 128×128 block scale) sidesteps the TRT-LLM + ``MLA`` module's fused + interleaved fp8 storage (``kv_a_proj_with_mqa`` fuses + ``q_a``+``kv`` and stores the scale interleaved). This mirrors the + golden-validated dequant exactly. + """ + for s, stage in enumerate(self.mtp_layers): + pref = f"mtp.{s}.attn." + dev = stage.input_layernorm.weight.device + + def deq(name: str, fp8: bool) -> torch.Tensor: + w = src[f"{pref}{name}.weight"].to(dev) + if fp8: + return self._block_dequant(w, src[f"{pref}{name}.scale"].to(dev)) + return w.to(torch.bfloat16) + + stage._dspark_attn = dict( + wq_a=deq("wq_a", True), + q_norm_w=src[f"{pref}q_norm.weight"].to(dev).to(torch.bfloat16), + wq_b=deq("wq_b", True), + wkv=deq("wkv", True), + kv_norm_w=src[f"{pref}kv_norm.weight"].to(dev).to(torch.bfloat16), + # wo_a IS fp8+scale in the checkpoint (verified); always dequant. + wo_a=deq("wo_a", True), + wo_b=deq("wo_b", True), + attn_sink=src[f"{pref}attn_sink"].to(dev).float(), + ) + + def cache_attn_weights_from_checkpoint(self, ckpt_dir: str, weight_map: Dict[str, str]) -> None: + """Populate ``_dspark_attn`` by reading the ``mtp.{s}.attn.*`` tensors from the + checkpoint shards on disk, then dequantizing via :meth:`_cache_attn_weights`. + + TODO(step 3): source these from the loaded ``MLA`` modules instead, once the + fused/interleaved fp8 scale layout is decoded, to drop the checkpoint I/O. + """ + from safetensors import safe_open + + prefixes = tuple(f"mtp.{s}.attn." for s in range(len(self.mtp_layers))) + shards: Dict[str, list] = {} + for k in weight_map: + if k.startswith(prefixes): + shards.setdefault(weight_map[k], []).append(k) + raw: Dict[str, torch.Tensor] = {} + for shard, ks in shards.items(): + with safe_open(os.path.join(ckpt_dir, shard), framework="pt", device="cpu") as f: + for k in ks: + raw[k] = f.get_tensor(k) + self._cache_attn_weights(raw) + + def cache_attn_weights_from_state_dict(self, weights: Dict) -> None: + """Populate ``_dspark_attn`` from an already-loaded in-memory ``weights`` dict + (no extra disk I/O); used on the one-engine load path + (``DSparkForCausalLM.load_weights``). Delegates to :meth:`_cache_attn_weights`. + """ + self._cache_attn_weights(weights) + + def _dspark_freqs_table(self, device: torch.device) -> torch.Tensor: + """Return the fixed-size plain-RoPE table cached for ``device``.""" + key = str(device) + cached = self._freqs_table_cache.get(key) + if cached is None: + cached = precompute_dspark_freqs_cis( + self._attn_params["rope_head_dim"], + self._freqs_cap, + rope_theta=self._rope_theta, + device=device, + ) + self._freqs_table_cache[key] = cached + return cached + + @classmethod + def _derive_draft_model_config(cls, model_config, base: int, num_stages: int): + """Return a draft-only ``model_config`` copy with draft-specific fixes. + + Applies (1) the ``compress_ratios`` draft slice and (2) the + ``quant_config_dict`` MXFP4 extension for the draft layers' routed + experts. A single shallow copy is made (and only when something needs to + change) so the shared ``model_config`` and the target model are untouched. + + The draft MoE backend is **inherited** from the target's + ``model_config.moe_backend`` (carried by the shallow copy) — not pinned — + matching every other drafter (the MTP module reuses the V4 decoder layer, + whose MoE is built with ``moe_backend=model_config.moe_backend``; separate + Eagle3/DFlash drafts resolve it from their own config the same way). The + draft ``mtp.*`` stages are full V4 blocks, so they share the target's + MXFP4 ``n_routed_experts=384`` / ``n_group=8`` (= 48 experts/group) layout + and therefore the same backend constraints: pick a backend that supports + it (CUTLASS today, DeepGEMM megaMoE once available) on the target and the + draft follows. Note the TRTLLM-Gen ``blockScaleMoe`` routing kernel asserts + ``experts/group <= 32`` (warp size), so it is incompatible with this layout + for both the target and the draft. + """ + new_sa = cls._draft_sparse_config(model_config, base, num_stages) + new_qcd = cls._draft_quant_config_dict(model_config, base, num_stages) + new_qc = cls._draft_normalized_quant_config(model_config) + if new_sa is None and new_qcd is None and new_qc is None: + return model_config + draft_cfg = copy.copy(model_config) + # ModelConfig is a frozen dataclass; bypass the guard for these fields. + if new_sa is not None: + object.__setattr__(draft_cfg, "sparse_attention_config", new_sa) + if new_qcd is not None: + object.__setattr__(draft_cfg, "quant_config_dict", new_qcd) + if new_qc is not None: + object.__setattr__(draft_cfg, "quant_config", new_qc) + return draft_cfg + + @staticmethod + def _draft_normalized_quant_config(model_config): + """Resolved global ``quant_config`` for NVFP4 DSpark checkpoints, or None. + + NVFP4 DSpark checkpoints declare a global ``MIXED_PRECISION`` quant algo + (per-layer NVFP4 routed experts over an FP8 base). The target resolves it + in :func:`_normalize_deepseek_v4_nvfp4_mixed_precision_config` + (base -> ``FP8_BLOCK_SCALES``); the separately-built draft config needs + the same, otherwise the inherited ``DeepseekV4DecoderLayer`` asserts + ``"MIXED_PRECISION is ambiguous"`` when it builds the draft stages. + Returns the resolved ``quant_config`` to set on the draft copy, or None + when nothing changes (e.g. the MXFP4 checkpoint, whose global algo is not + ``MIXED_PRECISION``) so the draft config is left byte-identical. + """ + qc = getattr(model_config, "quant_config", None) + if qc is None or getattr(qc, "quant_algo", None) != QuantAlgo.MIXED_PRECISION: + return None + # Reuse the target normalizer on a throwaway shallow copy so we only + # extract the resolved global quant_config; the shared ``model_config`` + # is left untouched (the normalizer rebinds ``.quant_config`` on the copy). + probe = copy.copy(model_config) + object.__setattr__(probe, "_frozen", False) + normalized = _normalize_deepseek_v4_nvfp4_mixed_precision_config(probe) + resolved = getattr(normalized, "quant_config", qc) + return resolved if resolved is not qc else None + + @staticmethod + def _draft_sparse_config(model_config, base: int, num_stages: int): + """Sparse-attention config sliced to the draft layers, or None if N/A. + + The inherited block remaps ``layer_idx`` to a draft-local index, so the + sparse config must expose the draft layers' per-layer ratios at indices + ``[0, num_stages)``. + """ + sa = getattr(model_config, "sparse_attention_config", None) + compress_ratios = getattr(sa, "compress_ratios", None) if sa is not None else None + if not compress_ratios or len(compress_ratios) < base + num_stages: + return None + draft_ratios = list(compress_ratios)[base : base + num_stages] + # Already draft-local (e.g. a draft-only checkpoint config); no slice. + if ( + draft_ratios == list(compress_ratios)[:num_stages] + and len(compress_ratios) == num_stages + ): + return None + return sa.model_copy(update={"compress_ratios": draft_ratios}) + + @staticmethod + def _draft_quant_config_dict(model_config, base: int, num_stages: int): + """quant_config_dict extended to cover draft-layer experts, or None. + + The checkpoint's per-module quant map only enumerates the base layers, so + ``model.layers.{base+s}.mlp.experts`` would fall back to the global fp8 + config and build fp8-shaped expert buffers. The draft routed experts are + physically MXFP4 (identical to the main MoE layers), so copy a + representative main MoE layer's experts quant onto the draft layer keys. + """ + qcd = getattr(model_config, "quant_config_dict", None) + if not qcd: + return None + src = next( + ( + qcd[f"model.layers.{li}.mlp.experts"] + for li in range(base) + if f"model.layers.{li}.mlp.experts" in qcd + ), + None, + ) + if src is None: + return None + new_qcd = dict(qcd) + changed = False + for s in range(num_stages): + key = f"model.layers.{base + s}.mlp.experts" + if new_qcd.get(key) is not src: + new_qcd[key] = src + changed = True + return new_qcd if changed else None + + @torch.inference_mode() + def write_context_windows( + self, + main_hidden: torch.Tensor, + positions: torch.Tensor, + stage_windows: torch.Tensor, + ) -> None: + """Write captured-context ``main_kv`` into the rolling per-stage KV windows. + + Replicates exactly the per-position context write that + :func:`dspark_attention_forward` performs each generation step + (``main_kv = RoPE_pos(rmsnorm(wkv @ main_x))`` written at + ``window[pos % window_size]``), but for an arbitrary set of + ``(captured-hidden, position)`` pairs. Used to (a) seed a request's + window from its prompt at prefill and (b) back-fill the intermediate + accepted tokens of a multi-accept step — both of which the per-step + generation path would otherwise leave as holes, starving the draft + attention of context (acceptance-rate only; verified decoding keeps + output correctness regardless). + + Args: + main_hidden: ``[M, num_capture * hidden]`` captured target hiddens. + positions: ``[M]`` absolute window positions (used for BOTH the RoPE + phase and the slot ``pos % window_size``). By the generation-path + convention this is ``committed_position + 1``. Must hold at most + ``window_size`` entries with distinct slots (the caller passes a + contiguous, deduplicated range) so the scatter is well defined. + stage_windows: ``[num_stages, window_size, head_dim]`` window for one + request's slot; updated in place. + """ + if getattr(self.mtp_layers[0], "_dspark_attn", None) is None: + return + M = int(main_hidden.shape[0]) + if M == 0: + return + win = int(self._attn_params["window_size"]) + rd = int(self._attn_params["rope_head_dim"]) + eps = float(self._attn_params["eps"]) + positions = positions.to(main_hidden.device).long() + # main_x is stage-invariant (stage 0's projection), matching forward_embed. + stage0 = self.mtp_layers[0] + main_x = stage0.main_norm(stage0.main_proj(main_hidden)) # [M, hidden] + freqs = self._dspark_freqs_table(main_x.device)[positions] + slots = positions % win # [M] + mx = main_x.unsqueeze(0) # [1, M, hidden] for the per-position RoPE layout + for s, stage in enumerate(self.mtp_layers): + a = stage._dspark_attn + kv = _rmsnorm(F.linear(mx, a["wkv"]), a["kv_norm_w"], eps) # [1, M, head_dim] + kv = _rope_last_dims(kv, rd, freqs) # [1, M, head_dim] + stage_windows[s, slots] = kv[0].to(stage_windows.dtype) + + def write_context_windows_batched( + self, + main_hidden: torch.Tensor, + positions: torch.Tensor, + slots: torch.Tensor, + mask: torch.Tensor, + kv_windows: torch.Tensor, + ) -> None: + """CUDA-graph-safe batched + masked variant of :meth:`write_context_windows`. + + Back-fills the intermediate accepted tokens of a multi-accept step into the + rolling per-stage KV windows for ALL gen requests at once, with a fixed + ``[G, M]`` shape (``M = max interim per request``) and a validity mask + (invalid entries are no-ops via a read-modify-write), so nothing depends on + the per-request accept count. Same per-position math as the scalar version + (``RoPE_pos(rmsnorm(wkv @ main_x))`` written at ``window[pos % window]``), + but indexed/scattered through ``slots`` into the shared persistent buffer. + + Args: + main_hidden: ``[G, M, num_capture * hidden]`` captured target hiddens + (rows beyond a request's interim count are masked). + positions: ``[G, M]`` absolute window positions (RoPE phase + slot). + slots: ``[G]`` row index of each request into ``kv_windows``. + mask: ``[G, M]`` bool — which ``(g, m)`` entries are real interim writes. + kv_windows: ``[N, num_stages, window_size, head_dim]`` persistent buffer; + updated in place. + """ + if getattr(self.mtp_layers[0], "_dspark_attn", None) is None: + return + G, M = positions.shape + if G == 0 or M == 0: + return + win = int(self._attn_params["window_size"]) + rd = int(self._attn_params["rope_head_dim"]) + eps = float(self._attn_params["eps"]) + positions = positions.long() + slots = slots.long() + freqs = self._dspark_freqs_table(main_hidden.device)[positions] # [G, M, rd//2] + cols = positions % win # [G, M] + rows = slots[:, None].expand(-1, M) # [G, M] + mask3 = mask.unsqueeze(-1) # [G, M, 1] + stage0 = self.mtp_layers[0] + main_x = stage0.main_norm(stage0.main_proj(main_hidden)) # [G, M, hidden] + for s, stage in enumerate(self.mtp_layers): + a = stage._dspark_attn + kv = _rmsnorm(F.linear(main_x, a["wkv"]), a["kv_norm_w"], eps) # [G, M, head_dim] + kv = _rope_last_dims_batched(kv, rd, freqs) # [G, M, head_dim] + win_s = kv_windows[:, s] # [N, win, head_dim] view onto the base buffer + # Read-modify-write so masked-out (g, m) entries keep their current + # value — a graph-safe masked scatter (no dynamic-shape compaction). + cur = win_s[rows, cols] # [G, M, head_dim] + win_s[rows, cols] = torch.where(mask3, kv.to(win_s.dtype), cur) + + def forward_embed( + self, main_hidden: torch.Tensor, bonus_token_ids: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Build the draft block input and the cross-attention context. + + Args: + main_hidden: ``[num_tokens, num_capture * hidden]`` captured context. + bonus_token_ids: ``[num_tokens]`` last accepted token per request. + Returns: + x: hc-expanded block embeddings ``[num_tokens, block_size, hc_mult, hidden]`` + main_x: projected context ``[num_tokens, hidden]`` + draft_ids: block input token ids ``[num_tokens, block_size]`` (the callers + reuse these as the per-position MoE routing ids, so we return them + rather than rebuild). + """ + stage0 = self.mtp_layers[0] + main_x = stage0.main_norm(stage0.main_proj(main_hidden)) + draft_ids = build_draft_input_ids( + bonus_token_ids, block_size=self.block_size, noise_token_id=self.noise_token_id + ) + x = self.embed_tokens(draft_ids) + x = x.unsqueeze(-2).repeat(1, 1, self.hc_mult, 1) + return x, main_x, draft_ids + + def _forward_stage( + self, + stage: "DSparkBlock", + h: torch.Tensor, + main_x: torch.Tensor, + start_pos, + freqs_cis: torch.Tensor, + moe_input_ids: torch.Tensor, + stage_window: Optional[torch.Tensor] = None, + slots: Optional[torch.Tensor] = None, + all_rank_num_tokens: Optional[List[int]] = None, + ) -> torch.Tensor: + """One DSpark stage = reference ``Block.forward`` with captured-context attn. + + ``h`` is the mHC residual stream ``[T, block, hc_mult, hidden]``. The mHC + ``pre_mapping``/``post_mapping`` preserve the leading ``[T, block]`` dims; + the captured-context attention and MoE run on the collapsed token axis. + Mirrors the reference (unfused) mHC boundaries exactly: + ``hc_pre → attn_norm → DSparkAttention → hc_post`` then + ``hc_pre → ffn_norm → MoE → hc_post``. + + ``stage_window`` is this stage's persistent rolling captured-context KV + window ``[T, window_size, head_dim]`` owned by the worker across decode + steps; the attention writes the current ``main_kv`` into it in place. When + ``None`` (golden / single-shot) a zero window is allocated per call. + + When ``slots`` (a ``[G]`` int tensor) is given, the CUDA-graph-safe batched + attention (:func:`dspark_attention_forward_batched`) is used instead: it + takes ``start_pos`` as a ``[G]`` tensor and writes/reads ``stage_window`` + (then shaped ``[N, window_size, head_dim]``) through the ``slots`` index. + """ + T, block, _, hidden = h.shape + + # --- attention sub-block (captured-context, not paged-KV MLA) --- + residual = h + post_mix, comb_mix, layer_input = stage.hc_attn.pre_mapping(residual) + layer_input = stage.input_layernorm(layer_input) # [T, block, hidden] + # Rolling-window cache: persist through the worker-owned ``stage_window`` + # for cross-step decode, else a fresh zero window for a single block. + persist = stage_window is not None + kv_cache = ( + stage_window + if persist + else torch.zeros( + T, + self._attn_params["window_size"], + self._attn_params["head_dim"], + dtype=torch.bfloat16, + device=h.device, + ) + ) + if slots is not None: + # Batched, CUDA-graph-safe path (start_pos is a [G] tensor; window is + # written/read through ``slots``). + attn = dspark_attention_forward_batched( + layer_input, + main_x, + start_pos, + kv_cache, + slots, + freqs_cis=freqs_cis, + persist=True, + **stage._dspark_attn, + **self._attn_params, + ) + else: + attn = dspark_attention_forward( + layer_input, + main_x, + start_pos, + kv_cache, + freqs_cis=freqs_cis, + persist=persist, + **stage._dspark_attn, + **self._attn_params, + ) + if stage.enable_fused_hc: + residual, post_mix, comb_mix, layer_input = stage.hc_ffn.fused_hc( + x_prev=attn, + residual_prev=residual, + post_mix_prev=post_mix, + comb_mix_prev=comb_mix, + norm_weight=stage.post_attention_layernorm.weight, + norm_eps=stage.post_attention_layernorm.variance_epsilon, + ) + else: + residual = stage.hc_attn.post_mapping( + x=attn, + residual=residual, + post_layer_mix=post_mix, + comb_res_mix=comb_mix, + ) + post_mix, comb_mix, layer_input = stage.hc_ffn.pre_mapping(residual) + layer_input = stage.post_attention_layernorm(layer_input) + num_tokens = T * block + # FUSED_COMM MoE backends (DeepGEMM MegaMoE) size their in-kernel + # NVLink-barrier chunk loop from ``max(all_rank_num_tokens)`` and index + # the local slice by ``moe_ep_rank``, so every EP rank must pass the same + # globally-gathered per-rank list (here: gen tokens = num_gens * block per + # rank). Passing only the local ``[num_tokens]`` desyncs the phase-flip + # barrier across ranks (hang / "unspecified launch failure"). Fall back to + # the local count for single-rank / non-ADP runs where no list is threaded. + moe_all_rank_num_tokens = ( + all_rank_num_tokens if all_rank_num_tokens is not None else [num_tokens] + ) + # The draft captured-context MoE must mirror the target DeepseekV4MoE's TP + # reduction policy. Under attention DP each rank is data-parallel (owns a + # distinct set of requests) and needs no cross-rank reduction; but under + # plain tensor parallelism (attention_dp off, tp_size > 1) the expert-sharded + # MoE output must be all-reduced across ranks -- exactly what the target MoE + # does (enable_allreduce = not (POST_MOE_FUSION or tp_size == 1)). Previously + # this was hard-coded to False, which dropped the reduction on the non-ADP + # path, corrupting the draft block proposals and roughly halving DSpark + # acceptance length whenever attention_dp was disabled. + moe_enable_allreduce = ( + not self.model_config.mapping.enable_attention_dp + and self.model_config.mapping.tp_size > 1 + ) + moe_out = stage.mlp( + layer_input.reshape(num_tokens, hidden), + input_ids=moe_input_ids, + all_rank_num_tokens=moe_all_rank_num_tokens, + final_all_reduce_params=AllReduceParams(enable_allreduce=moe_enable_allreduce), + do_finalize=True, + ).reshape(T, block, hidden) + h = stage.hc_ffn.post_mapping( + x=moe_out, residual=residual, post_layer_mix=post_mix, comb_res_mix=comb_mix + ) + return h + + def forward( + self, + main_hidden: torch.Tensor, + bonus_token_ids: torch.Tensor, + start_pos: int, + *, + kv_windows: Optional[torch.Tensor] = None, + temperature: float = 0.0, + confidence_threshold: float = 0.0, + return_logits: bool = False, + all_rank_num_tokens: Optional[List[int]] = None, + ) -> tuple: + """Full block-draft forward: chain the ``num_stages`` DSpark stages. + + Mirrors the reference ``Transformer.forward_spec`` (generation path, + ``start_pos > 0``): ``forward_embed`` builds the block input + captured + context, each stage runs the captured-context backbone, and + ``forward_head`` emits the block draft tokens + per-position confidence. + + Args: + main_hidden: ``[T, num_capture * hidden]`` captured target context. + bonus_token_ids: ``[T]`` last accepted token per request. + start_pos: absolute decode position (must be > 0). + kv_windows: optional persistent per-stage rolling captured-context KV + windows ``[T, num_stages, window_size, head_dim]`` owned by the + worker; updated in place each call. ``None`` allocates fresh zero + windows (single-shot golden / test path). + Returns: + ``(draft_tokens [T, block], num_proposed [T])`` from ``forward_head``. + """ + assert start_pos > 0, "DSpark draft runs at generation (start_pos > 0)" + if getattr(self.mtp_layers[0], "_dspark_attn", None) is None: + raise RuntimeError( + "DSpark attention weights not cached; call " + "cache_attn_weights_from_checkpoint(ckpt_dir, weight_map) after loading." + ) + x, main_x, draft_ids = self.forward_embed(main_hidden, bonus_token_ids) + main_x = main_x.unsqueeze(1) # [T, 1, hidden] for the MQA K/V projection + freqs_cis = self._dspark_freqs_table(x.device) + moe_input_ids = draft_ids.reshape(-1) + + h = x + for s, stage in enumerate(self.mtp_layers): + stage_window = kv_windows[:, s] if kv_windows is not None else None + h = self._forward_stage( + stage, + h, + main_x, + start_pos, + freqs_cis, + moe_input_ids, + stage_window, + all_rank_num_tokens=all_rank_num_tokens, + ) + + return self.forward_head( + h, + bonus_token_ids, + temperature=temperature, + confidence_threshold=confidence_threshold, + return_logits=return_logits, + ) + + def forward_batched( + self, + main_hidden: torch.Tensor, + bonus_token_ids: torch.Tensor, + start_pos: torch.Tensor, + *, + kv_windows: torch.Tensor, + slots: torch.Tensor, + temperature: float = 0.0, + confidence_threshold: float = 0.0, + return_logits: bool = False, + all_rank_num_tokens: Optional[List[int]] = None, + ) -> tuple: + """CUDA-graph-safe batched block-draft forward (all gen requests at once). + + Same computation as :meth:`forward`, but every host-int / data-dependent + operation is tensorized so the whole path can be captured into the target's + CUDA graph (DSpark is a one-engine drafter — its worker runs inside the + graph). ``start_pos`` is a ``[G]`` tensor (one absolute decode position per + gen request); the rolling captured-context windows are written/read through + ``slots`` into the worker-owned ``kv_windows`` buffer; RoPE phases are + gathered from a fixed table. ``forward_head`` is run with + ``confidence_threshold == 0`` (the worker proposes the full block), which is + the graph-safe branch of :func:`dspark_propose`. + + Args: + main_hidden: ``[G, num_capture * hidden]`` captured target context. + bonus_token_ids: ``[G]`` last accepted token per gen request. + start_pos: ``[G]`` int tensor of absolute decode positions (> 0). + kv_windows: ``[N, num_stages, window_size, head_dim]`` persistent rolling + windows; written in place through ``slots``. + slots: ``[G]`` int tensor mapping each request to its ``kv_windows`` row. + Returns: + ``(draft_tokens [G, block], num_proposed [G])`` from ``forward_head``. + """ + if getattr(self.mtp_layers[0], "_dspark_attn", None) is None: + raise RuntimeError( + "DSpark attention weights not cached; call " + "cache_attn_weights_from_checkpoint(ckpt_dir, weight_map) after loading." + ) + x, main_x, draft_ids = self.forward_embed(main_hidden, bonus_token_ids) + main_x = main_x.unsqueeze(1) # [G, 1, hidden] for the MQA K/V projection + freqs_cis = self._dspark_freqs_table(x.device) + moe_input_ids = draft_ids.reshape(-1) + + h = x + for s, stage in enumerate(self.mtp_layers): + stage_window = kv_windows[:, s] # [N, window_size, head_dim] + h = self._forward_stage( + stage, + h, + main_x, + start_pos, + freqs_cis, + moe_input_ids, + stage_window, + slots, + all_rank_num_tokens=all_rank_num_tokens, + ) + + return self.forward_head( + h, + bonus_token_ids, + temperature=temperature, + confidence_threshold=confidence_threshold, + return_logits=return_logits, + ) + + def run_moe_lockstep_noop( + self, all_rank_num_tokens: Optional[List[int]], device: torch.device + ) -> None: + """Cross the FUSED_COMM MoE NVLink barrier the same number of times as + gen-bearing ranks, for an EP rank whose local draft batch is empty. + + DeepGEMM MegaMoE (``scheduler_kind == FUSED_COMM``) synchronizes EP ranks + with an in-kernel phase-flip NVLink barrier that flips on every kernel + call, so every rank must invoke the MoE the same number of times or the + barrier desyncs (hang / "unspecified launch failure"). In the DSpark + draft only the MoE carries a cross-rank barrier (the captured-context + attention and the markov/confidence heads are per-rank), so a rank with + zero local generation requests replays just the per-stage MoE call with a + single 1-row dummy (its entry in ``all_rank_num_tokens`` is ``1``). The + scheduler runs its ``max``-derived chunk count, slicing this rank to the + 1 dummy row and zero-padding the remaining chunks, keeping the barrier + lockstep. No-op when there is no cross-rank work (single-rank / non-ADP, + or every rank is empty). + """ + if all_rank_num_tokens is None or max(all_rank_num_tokens) == 0: + return + hidden = self.config.hidden_size + # Use a 1-row dummy, NOT a 0-row tensor: DeepseekV4MoE's router / + # shared-expert dense GEMMs reject a 0-row input (cuBLAS + # CUBLAS_STATUS_INVALID_VALUE). The paired ``all_rank_num_tokens`` encodes + # 1 for this rank, so the FUSED_COMM scheduler slices to this 1 dummy row + # and still launches ``num_chunks`` cross-rank barrier crossings in + # lockstep with the gen-bearing ranks. + dummy_x = torch.zeros((1, hidden), dtype=torch.bfloat16, device=device) + dummy_ids = torch.zeros((1,), dtype=torch.long, device=device) + for stage in self.mtp_layers: + stage.mlp( + dummy_x, + input_ids=dummy_ids, + all_rank_num_tokens=all_rank_num_tokens, + final_all_reduce_params=AllReduceParams(enable_allreduce=False), + do_finalize=True, + ) + + def forward_head( + self, + block_hidden: torch.Tensor, + bonus_token_ids: torch.Tensor, + *, + temperature: float = 0.0, + confidence_threshold: float = 0.0, + return_logits: bool = False, + ) -> tuple: + """Block-draft head: hc_head + norm + lm_head -> markov refine + confidence. + + ``block_hidden`` is the last stage's mHC residual ``[*, block, hc_mult, hidden]``. + Returns (draft_tokens [*, block], num_proposed [*]); with ``return_logits`` + also returns the per-position draft logits [*, block, vocab] (§7.9 1-TV). + """ + last = self.mtp_layers[-1] + h = last.hc_head(block_hidden) + h = last.norm(h) + base_logits = self.lm_head(h) + return dspark_propose( + base_logits, + bonus_token_ids=bonus_token_ids, + block_hidden=h, + markov_head=last.markov_head, + confidence_head=last.confidence_head, + block_size=self.block_size, + temperature=temperature, + confidence_threshold=confidence_threshold, + return_logits=return_logits, + ) + + +class DSparkForCausalLM(nn.Module): + """One-engine draft wrapper for DSpark (mirrors ``DFlashForCausalLM``). + + Wraps :class:`DSparkDraftModel` (the ``n_mtp_layers``-stage ``mtp.*`` backbone) + for the single-engine external-drafter flow: created by ``get_draft_model``, + appended to the target's epilogue, and driven by ``DSparkWorker``. + + ``embed_tokens`` / ``lm_head`` are shared with the target model + (:meth:`load_weights_from_target_model`). The draft weights live in the SAME + checkpoint under ``mtp.*``; :meth:`load_weights` remaps them + (``remap_dspark_draft_keys``), loads via ``DeepseekV4WeightLoader``, runs the + fp8 ``post_load_weights`` transforms, and caches the bf16 captured-context + attention weights from the in-memory state dict. + """ + + def __init__(self, draft_config, aux_stream_dict=None, num_stages=None, block_size=None): + super().__init__() + self.dspark_model = DSparkDraftModel( + draft_config, + aux_stream_dict, + num_stages=num_stages, + block_size=block_size, + ) + # Generic handles expected by the loader / weight mappers. + self.model = self.dspark_model + self.model_config = draft_config + self.config = draft_config.pretrained_config + # Worker-facing interface (the worker receives this wrapper as + # ``draft_model`` and calls forward()/reads these properties and scalars). + self.num_stages = self.dspark_model.num_stages + self._attn_params = self.dspark_model._attn_params + self.lm_head = None # shared from the target (load_weights_from_target_model) + self.logits_processor = None # set by the caller after construction + + @property + def block_size(self): + return self.dspark_model.block_size + + @property + def embed_tokens(self): + return self.dspark_model.embed_tokens + + def forward(self, main_hidden, bonus_token_ids, start_pos, **kwargs): + return self.dspark_model.forward(main_hidden, bonus_token_ids, start_pos, **kwargs) + + def forward_batched(self, main_hidden, bonus_token_ids, start_pos, **kwargs): + """CUDA-graph-safe batched draft forward (delegates to the draft model).""" + return self.dspark_model.forward_batched(main_hidden, bonus_token_ids, start_pos, **kwargs) + + def run_moe_lockstep_noop(self, all_rank_num_tokens, device): + """Empty-batch MoE barrier lockstep (delegates to the draft model).""" + return self.dspark_model.run_moe_lockstep_noop(all_rank_num_tokens, device) + + def write_context_windows(self, main_hidden, positions, stage_windows): + """Seed / back-fill the rolling KV windows (delegates to the draft model).""" + return self.dspark_model.write_context_windows(main_hidden, positions, stage_windows) + + def write_context_windows_batched(self, main_hidden, positions, slots, mask, kv_windows): + """Batched + masked window back-fill (delegates to the draft model).""" + return self.dspark_model.write_context_windows_batched( + main_hidden, positions, slots, mask, kv_windows + ) + + def load_weights(self, weights: Dict, weight_mapper=None, **kwargs): + """Load the ``mtp.*`` draft weights from the (full) checkpoint dict. + + ``weight_mapper`` is accepted for interface parity with the draft-weight + loader but unused: DSpark does its own ``mtp.{s}.* -> mtp_layers.{s}.*`` + remap (``remap_dspark_draft_keys``) onto the shared V4 weight loader. + """ + remapped = remap_dspark_draft_keys(weights, num_stages=self.num_stages) + logger.info( + f"[DSpark] loading {len(remapped)} draft params across {self.num_stages} stages" + ) + DeepseekV4WeightLoader(self.dspark_model).load_weights(remapped) + self.dspark_model.post_load_weights() + # bf16 captured-context attention path: dequantize the raw mtp.{s}.attn.* + # tensors for ``dspark_attention_forward``. + self.dspark_model.cache_attn_weights_from_state_dict(weights) + logger.info("[DSpark] draft weight load complete") + + def load_weights_from_target_model(self, target_model): + """Share the target's embed_tokens / lm_head (DSpark has neither).""" + if self.dspark_model.embed_tokens is None: + self.dspark_model.embed_tokens = target_model.model.embed_tokens + if self.lm_head is None: + self.lm_head = target_model.lm_head + self.dspark_model.lm_head = target_model.lm_head + + +__all__ = [ + "DSparkBlock", + "DSparkDraftModel", + "DSparkForCausalLM", + "validate_dspark_eplb_layer_base", + "validate_dspark_eplb_stage_layers", +] diff --git a/tensorrt_llm/_torch/models/modeling_gemma4_unified.py b/tensorrt_llm/_torch/models/modeling_gemma4_unified.py index f2def0290eec..77a111d05874 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma4_unified.py +++ b/tensorrt_llm/_torch/models/modeling_gemma4_unified.py @@ -33,15 +33,14 @@ `layer_scalar`, final-logit softcap, tied embeddings). 12B has PLE and KV-sharing **off** (`hidden_size_per_layer_input=0`, `num_kv_shared_layers=0`). -This module reuses the existing Gemma 4 multimodal wrapper -(:class:`Gemma4ForConditionalGeneration`) for all engine plumbing +This module inherits the shared Gemma 4 multimodal wrapper base +(:class:`Gemma4MultimodalModelBase`) for all engine plumbing (`post_config` / `get_sub_model_config` / `infer_max_seq_len` / `vocab_size_padded` / `get_model_defaults`), and reuses :class:`Gemma4MultimodalEmbedder` (audio) and :class:`Gemma4InputProcessor` (HF `AutoProcessor` resolves to `Gemma4UnifiedProcessor`; the output dict -keys match). It overrides `__init__` / `forward` / `_get_image_features` / -`_get_audio_features` / `load_weights` to drop the encoder towers and use the -encoder-free projections instead. +keys match). It implements `__init__`, `_get_image_features`, +`_get_audio_features`, and `load_weights` for the encoder-free projections. TRT-LLM provides its own `gemma4_unified` config classes (`_torch/configs/gemma4_unified.py`) and multimodal preprocessing (the vendored @@ -55,13 +54,12 @@ import math import os import re -from typing import Dict, List, Optional +from typing import Dict import numpy as np import torch from torch import nn from torchvision.transforms.v2 import functional as tvF -from transformers import PreTrainedModel from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor from transformers.image_processing_backends import TorchvisionBackend from transformers.image_processing_utils import BatchFeature @@ -77,20 +75,13 @@ MultimodalPlaceholderPlacement, register_input_processor, ) -from ...inputs.multimodal import MultimodalParams -from ..attention_backend import AttentionMetadata from ..modules.layer_norm import LayerNorm from ..modules.linear import Linear from .modeling_gemma4 import Gemma4ForCausalLM from .modeling_gemma4mm import ( - Gemma4ForConditionalGeneration, Gemma4InputProcessor, Gemma4MultimodalEmbedder, -) -from .modeling_multimodal_utils import ( - find_input_mm_embeds, - fuse_input_embeds, - get_multimodal_embeddings, + Gemma4MultimodalModelBase, ) from .modeling_utils import ModelConfig, filter_weights, register_auto_model @@ -230,27 +221,25 @@ def __init__(self, *args, **kwargs): interleave_placeholders=True, ), ) -class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration): - """Gemma 4 12B Unified (encoder-free). Reuses the Gemma 4 MM wrapper for - engine plumbing + the text core (:class:`Gemma4ForCausalLM`); replaces the - vision/audio towers with encoder-free linear embedders.""" +class Gemma4UnifiedForConditionalGeneration(Gemma4MultimodalModelBase): + """Gemma 4 12B Unified model with encoder-free multimodal projections.""" def __init__(self, model_config: ModelConfig): config = model_config.pretrained_config - # Skip Gemma4ForConditionalGeneration.__init__ (it builds vision/audio - # *towers* the unified architecture does not have) and init the HF base. - PreTrainedModel.__init__(self, config) + if config.image_token_id is None: + raise ValueError( + "Gemma4UnifiedForConditionalGeneration requires config.image_token_id." + ) + super().__init__(config) # ModelConfig always has `mapping`, and Mapping always has `local_rank`. local_rank = model_config.mapping.local_rank self._device = f"cuda:{local_rank}" self.model_dtype = getattr(config, "torch_dtype", torch.bfloat16) self._top_config = config - # HF always defines these token ids (each may be None if that modality is absent). - self.image_token_ids = ( - torch.tensor([config.image_token_id], dtype=torch.int32, device=self._device) - if config.image_token_id is not None - else None + # Image tokens are required; audio and video may be absent. + self.image_token_ids = torch.tensor( + [config.image_token_id], dtype=torch.int32, device=self._device ) self.audio_token_ids = ( torch.tensor([config.audio_token_id], dtype=torch.int32, device=self._device) @@ -354,169 +343,6 @@ def _get_audio_features(self, audio_features, audio_features_mask=None): features = features.reshape(-1, features.shape[-1]) return features.contiguous() - @property - def multimodal_data_device_paths(self) -> List[str]: - return [ - "image.pixel_values", - "image.image_position_ids", - "video.pixel_values", - "video.image_position_ids", - "audio.audio_features", - "audio.audio_features_mask", - ] - - @staticmethod - def _has_active_multimodal_tokens(multimodal_param: MultimodalParams) -> bool: - """Whether a context parameter needs embeddings in this forward. - - Mirrors `Gemma4ForConditionalGeneration._has_active_multimodal_tokens` - from #15848 (this class overrides `forward` entirely, so the parent's - version would not apply here; deduplicate once #15848 lands on main). - """ - runtime = multimodal_param.multimodal_runtime - if runtime is not None and runtime.num_mm_tokens_in_chunk == 0: - return False - - multimodal_data = multimodal_param.multimodal_data - if multimodal_data.get("multimodal_embedding") is not None: - return True - - payload_fields = ( - ("image", "pixel_values"), - ("video", "pixel_values"), - ("audio", "audio_features"), - ) - return any( - multimodal_data.get(modality, {}).get(field) is not None - for modality, field in payload_fields - ) - - def _forward_multimodal_encoder( - self, multimodal_params: List[MultimodalParams] - ) -> torch.Tensor: - """Run the encoder-free projectors for all uncached multimodal payloads. - - Called by `get_multimodal_embeddings`, which caches the result in - `multimodal_data["multimodal_embedding"]` so later prefill chunks reuse - the embedding without re-running the projectors (mirrors the pattern in - `Gemma4ForConditionalGeneration._forward_multimodal_encoder` from #15848). - """ - pixel_values_list, image_position_ids_list = [], [] - audio_features_list, audio_mask_list = [], [] - video_pixel_values_list, video_position_ids_list = [], [] - for multimodal_param in multimodal_params: - image_data = multimodal_param.multimodal_data.get("image", {}) - if image_data.get("pixel_values") is not None: - pixel_values_list.append(image_data["pixel_values"]) - if image_data.get("image_position_ids") is not None: - image_position_ids_list.append(image_data["image_position_ids"]) - audio_data = multimodal_param.multimodal_data.get("audio", {}) - if audio_data.get("audio_features") is not None: - audio_features_list.append(audio_data["audio_features"]) - audio_mask_list.append(audio_data.get("audio_features_mask")) - video_data = multimodal_param.multimodal_data.get("video", {}) - if video_data.get("pixel_values") is not None: - video_pixel_values_list.append(video_data["pixel_values"]) - if video_data.get("image_position_ids") is not None: - video_position_ids_list.append(video_data["image_position_ids"]) - - embeddings = [] - if pixel_values_list and self.embed_vision is not None: - pixel_values = torch.cat(pixel_values_list) - image_position_ids = ( - torch.cat(image_position_ids_list) - if len(image_position_ids_list) == len(pixel_values_list) - else None - ) - embeddings.append(self._get_image_features(pixel_values, image_position_ids)) - - if video_pixel_values_list and self.embed_vision is not None: - video_pixel_values = torch.cat(video_pixel_values_list) - video_position_ids = ( - torch.cat(video_position_ids_list) - if len(video_position_ids_list) == len(video_pixel_values_list) - else None - ) - embeddings.append(self._get_image_features(video_pixel_values, video_position_ids)) - - if audio_features_list and self.embed_audio is not None: - per_clip = [] - for clip_index, audio_feature in enumerate(audio_features_list): - per_clip.append( - self._get_audio_features(audio_feature, audio_mask_list[clip_index]) - ) - embeddings.append(torch.cat(per_clip, dim=0)) - - return torch.cat(embeddings, dim=0) if embeddings else torch.empty(0) - - def forward( - self, - attn_metadata: AttentionMetadata, - input_ids: Optional[torch.LongTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - return_context_logits: bool = False, - **kwargs, - ) -> torch.Tensor: - multimodal_params = kwargs.get("multimodal_params", []) - - # Filter to params that have active multimodal tokens in this chunk. - # get_multimodal_embeddings caches the result after the first run, so - # later prefill chunks reuse the embedding without re-running the projectors. - active_multimodal_params = [ - mp for mp in multimodal_params if self._has_active_multimodal_tokens(mp) - ] - - mm_embeds: List[torch.Tensor] = [] - all_mm_token_ids: List[torch.Tensor] = [] - mm_token_type_ids = None - - if active_multimodal_params: - mm_embeds = get_multimodal_embeddings( - encoder_forward_fn=self._forward_multimodal_encoder, - multimodal_params=active_multimodal_params, - ) - mm_embeds = find_input_mm_embeds(mm_embeds, active_multimodal_params) - - # Collect every defined multimodal token id. On cache-hit chunks the - # raw payloads (pixel_values / audio_features) may be absent while the - # cached embedding is used, so the ids cannot be derived from payload - # presence; extra ids are harmless (they simply match no position). - for token_ids in (self.image_token_ids, self.video_token_ids, self.audio_token_ids): - if token_ids is not None: - all_mm_token_ids.append(token_ids) - - # Integer mm_token_type_ids (0=text,1=image,2=video,3=audio) drive the - # inherited bidirectional-vision attention mask in Gemma4ForCausalLM. - if mm_embeds and input_ids is not None: - mm_token_type_ids = torch.zeros_like(input_ids, dtype=torch.long) - if self.image_token_ids is not None: - mm_token_type_ids[torch.isin(input_ids, self.image_token_ids)] = 1 - if self.video_token_ids is not None: - mm_token_type_ids[torch.isin(input_ids, self.video_token_ids)] = 2 - if self.audio_token_ids is not None: - mm_token_type_ids[torch.isin(input_ids, self.audio_token_ids)] = 3 - - fuse_token_ids = torch.cat(all_mm_token_ids) if all_mm_token_ids else self.image_token_ids - - input_ids, inputs_embeds = fuse_input_embeds( - embedding_layer=self.llm.model.embed_tokens, - input_ids=input_ids, - mm_embeds=mm_embeds, - mm_token_ids=fuse_token_ids, - **kwargs, - ) - # 12B has PLE off (hidden_size_per_layer_input=0) -> no ple_input_ids. - return self.llm.forward( - attn_metadata=attn_metadata, - input_ids=input_ids, - position_ids=position_ids, - inputs_embeds=inputs_embeds, - return_context_logits=return_context_logits, - mm_token_type_ids=mm_token_type_ids, - lora_params=kwargs.get("lora_params", None), - ) - def load_weights(self, weights: Dict, weight_mapper): # Text backbone: "model.language_model.X" -> "model.X" (same as the # parent), then load via the reused Gemma4 text core. diff --git a/tensorrt_llm/_torch/models/modeling_gemma4_vision.py b/tensorrt_llm/_torch/models/modeling_gemma4_vision.py index b921a442b6e9..40c595dfe5de 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma4_vision.py +++ b/tensorrt_llm/_torch/models/modeling_gemma4_vision.py @@ -629,11 +629,11 @@ def _position_embeddings( self, pixel_position_ids: torch.Tensor, padding_positions: torch.Tensor ) -> torch.Tensor: clamped = pixel_position_ids.clamp(min=0) - one_hot = F.one_hot(clamped, num_classes=self.position_embedding_size) - one_hot = one_hot.permute(0, 2, 1, 3).to(self.position_embedding_table) - position_embeddings = one_hot @ self.position_embedding_table - position_embeddings = position_embeddings.sum(dim=1) - return torch.where(padding_positions.unsqueeze(-1), 0.0, position_embeddings) + position_embeddings = ( + self.position_embedding_table[0, clamped[..., 0]] + + self.position_embedding_table[1, clamped[..., 1]] + ) + return position_embeddings.masked_fill(padding_positions.unsqueeze(-1), 0.0) def forward( self, diff --git a/tensorrt_llm/_torch/models/modeling_gemma4mm.py b/tensorrt_llm/_torch/models/modeling_gemma4mm.py index 0f5a340c0a9a..67622709c417 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma4mm.py +++ b/tensorrt_llm/_torch/models/modeling_gemma4mm.py @@ -47,18 +47,13 @@ from ...inputs.multimodal import MultimodalParams from ...logger import logger from ...sampling_params import SamplingParams -from ..attention_backend import AttentionMetadata +from ..modules.embedding import Embedding from ..modules.linear import Linear from .modeling_gemma4 import Gemma4ForCausalLM from .modeling_gemma4_audio import Gemma4AudioModel from .modeling_gemma4_vision import Gemma4VisionModel -from .modeling_multimodal_utils import ( - _MULTIMODAL_ENV_NAME, - _is_mm_disagg, - find_input_mm_embeds, - fuse_input_embeds, - get_multimodal_embeddings, -) +from .modeling_multimodal_mixin import MultimodalModelMixin, PreparedLlmInputs +from .modeling_multimodal_utils import _MULTIMODAL_ENV_NAME, _is_mm_disagg from .modeling_utils import ModelConfig, filter_weights, register_auto_model _MIN_TRANSFORMERS_FOR_GEMMA4 = "5.5.0" @@ -550,6 +545,250 @@ def call_with_text_prompt( # --------------------------------------------------------------------------- +class Gemma4MultimodalModelBase(MultimodalModelMixin, PreTrainedModel): + """Shared multimodal encoder flow for Gemma4 conditional generation models.""" + + supports_encoder_cache = True + + @classmethod + def get_model_defaults(cls, llm_args) -> dict: + """Gemma4-specific defaults — see Gemma4ForCausalLM.get_model_defaults.""" + return { + "attn_backend": "FLASHINFER", + } + + def _check_and_adjust_experts_implementation(self, *args, **kwargs): + # transformers 5.x ``PreTrainedModel.__init__`` calls this with an + # ``experts_implementation`` argument and fails for VL wrapper models + # that do not directly contain MoE layers. TRT-LLM manages expert + # implementations independently, so skip the check. + return None + + @property + def multimodal_data_device_paths(self) -> List[str]: + """Dotted multimodal-data paths that the engine transfers to the GPU.""" + return [ + "image.pixel_values", + "image.image_position_ids", + "video.pixel_values", + "video.image_position_ids", + "audio.audio_features", + "audio.audio_features_mask", + ] + + def encode_multimodal_inputs(self, multimodal_params: List[MultimodalParams]) -> torch.Tensor: + """Encode uncached Gemma4 image, video, and audio payloads.""" + modality_inputs = ( + ("image", "pixel_values"), + ("video", "pixel_values"), + ("audio", "audio_features"), + ) + param_modalities = [ + [ + modality + for modality, input_field in modality_inputs + if multimodal_param.multimodal_data.get(modality, {}).get(input_field) is not None + ] + for multimodal_param in multimodal_params + ] + + if any(len(modalities) > 1 for modalities in param_modalities): + raise ValueError( + "Gemma4 requests containing multiple multimodal input types are not supported." + ) + + # The cache helper splits this tensor by request. Preserve that order while still batching + # consecutive requests with the same modality. + if ( + all(len(modalities) == 1 for modalities in param_modalities) + and len({modalities[0] for modalities in param_modalities}) > 1 + ): + embeddings = [] + for _, param_group in groupby( + zip(param_modalities, multimodal_params), key=lambda item: item[0][0] + ): + embeddings.append( + self.encode_multimodal_inputs( + [multimodal_param for _, multimodal_param in param_group] + ) + ) + return torch.cat(embeddings, dim=0) + + pixel_values_list = [] + image_position_ids_list = [] + image_seq_lens_extended: List[int] = [] + audio_features_list = [] + audio_features_mask_list: List[Optional[torch.Tensor]] = [] + video_pixel_values_list = [] + video_position_ids_list = [] + video_seq_lens_extended: List[int] = [] + + for multimodal_param in multimodal_params: + multimodal_data = multimodal_param.multimodal_data + image_data = multimodal_data.get("image", {}) + pixel_values = image_data.get("pixel_values") + if pixel_values is not None: + pixel_values_list.append(pixel_values) + image_position_ids = image_data.get("image_position_ids") + if image_position_ids is not None: + image_position_ids_list.append(image_position_ids) + image_seq_lens = image_data.get("image_seq_lens") + if image_seq_lens is not None: + image_seq_lens_extended.extend(image_seq_lens) + + audio_data = multimodal_data.get("audio", {}) + audio_features = audio_data.get("audio_features") + if audio_features is not None: + audio_features_list.append(audio_features) + audio_features_mask_list.append(audio_data.get("audio_features_mask")) + + video_data = multimodal_data.get("video", {}) + video_pixel_values = video_data.get("pixel_values") + if video_pixel_values is not None: + video_pixel_values_list.append(video_pixel_values) + video_position_ids = video_data.get("image_position_ids") + if video_position_ids is not None: + video_position_ids_list.append(video_position_ids) + video_seq_lens = video_data.get("image_seq_lens") + if video_seq_lens is not None: + video_seq_lens_extended.extend(video_seq_lens) + + multimodal_embeddings = [] + if pixel_values_list: + pixel_values = torch.cat(pixel_values_list) + image_position_ids = ( + torch.cat(image_position_ids_list) + if len(image_position_ids_list) == len(pixel_values_list) + else None + ) + multimodal_embeddings.append( + self._get_image_features( + pixel_values=pixel_values, + image_position_ids=image_position_ids, + image_seq_lens=( + image_seq_lens_extended + if len(image_seq_lens_extended) == pixel_values.shape[0] + else None + ), + ) + ) + + if video_pixel_values_list: + video_pixel_values = torch.cat(video_pixel_values_list) + video_position_ids = ( + torch.cat(video_position_ids_list) + if len(video_position_ids_list) == len(video_pixel_values_list) + else None + ) + multimodal_embeddings.append( + self._get_image_features( + pixel_values=video_pixel_values, + image_position_ids=video_position_ids, + image_seq_lens=( + video_seq_lens_extended + if len(video_seq_lens_extended) == video_pixel_values.shape[0] + else None + ), + ) + ) + + if audio_features_list and self.embed_audio is not None: + per_audio_embeddings = [ + self._get_audio_features(audio_features, audio_features_mask_list[index]) + for index, audio_features in enumerate(audio_features_list) + ] + multimodal_embeddings.append(torch.cat(per_audio_embeddings, dim=0)) + + if not multimodal_embeddings: + raise ValueError("Gemma4 received active multimodal parameters without encoder inputs") + return torch.cat(multimodal_embeddings, dim=0) + + @staticmethod + def get_sub_model_config( + model_config: ModelConfig[Gemma4Config], + name: str, + ) -> ModelConfig: + assert name in ["text_config", "vision_config", "audio_config"], ( + f"Expected subconfig name to be 'text_config', 'vision_config', " + f"or 'audio_config'. Got {name} instead." + ) + pretrained_config = getattr(model_config.pretrained_config, name) + quant_config = model_config.quant_config if name == "text_config" else None + preferred_backend = "FLASHINFER" if name == "text_config" else "TRTLLM" + sub_config: ModelConfig = dataclasses.replace( + model_config, + pretrained_config=pretrained_config, + attn_backend=preferred_backend, + quant_config=quant_config, + ) + if ( + hasattr(sub_config.pretrained_config, "torch_dtype") + and sub_config.pretrained_config.torch_dtype is None + ): + sub_config.pretrained_config.torch_dtype = model_config.pretrained_config.torch_dtype + return sub_config + + def post_config(self): + self.config = self.llm.config + self.model_config.pretrained_config = self.llm.config + + @property + def language_model(self) -> torch.nn.Module: + return self.llm + + def get_language_model_extra_forward_kwargs( + self, + *, + raw_input_ids: Optional[torch.Tensor], + position_ids: Optional[torch.Tensor], + mm_inputs: PreparedLlmInputs, + lora_params=None, + **forward_kwargs, + ) -> Dict: + """Build Gemma4-specific language-model forward arguments.""" + del position_ids, forward_kwargs + mm_token_type_ids = None + if raw_input_ids is not None and mm_inputs.input_ids is None: + mm_token_type_ids = torch.zeros_like(raw_input_ids, dtype=torch.long) + mm_token_type_ids[raw_input_ids == self.image_token_ids[0]] = 1 + if self.video_token_ids is not None: + mm_token_type_ids[raw_input_ids == self.video_token_ids[0]] = 2 + if self.audio_token_ids is not None: + mm_token_type_ids[raw_input_ids == self.audio_token_ids[0]] = 3 + + ple_input_ids = None + if mm_token_type_ids is not None and self.llm.model.hidden_size_per_layer_input: + text_config = getattr(self.config, "text_config", self.config) + pad_id = getattr(text_config, "pad_token_id", None) + if pad_id is not None: + ple_input_ids = torch.where( + mm_token_type_ids > 0, + torch.full_like(raw_input_ids, pad_id), + raw_input_ids, + ) + return { + "mm_token_type_ids": mm_token_type_ids, + "ple_input_ids": ple_input_ids, + "lora_params": lora_params, + } + + @property + def multimodal_token_ids(self) -> torch.Tensor: + return self._mm_token_ids + + @property + def text_embedding_layer(self) -> Embedding: + return self.llm.model.embed_tokens + + @property + def embedding_dim(self) -> int: + return self.text_embedding_layer.embedding_dim + + @property + def embedding_dtype(self) -> torch.dtype: + return self.text_embedding_layer.weight.dtype + + @register_auto_model("Gemma4ForConditionalGeneration") @register_input_processor( Gemma4InputProcessor, @@ -578,7 +817,7 @@ def call_with_text_prompt( interleave_placeholders=True, ), ) -class Gemma4ForConditionalGeneration(PreTrainedModel): +class Gemma4ForConditionalGeneration(Gemma4MultimodalModelBase): """Gemma4 multimodal model: LLM + vision tower + multimodal embedder. Follows the Gemma3VLM pattern but adapted for Gemma4's architecture: @@ -588,20 +827,6 @@ class Gemma4ForConditionalGeneration(PreTrainedModel): - mm_token_type_ids-based bidirectional masking """ - @classmethod - def get_model_defaults(cls, llm_args) -> dict: - """Gemma4-specific defaults — see Gemma4ForCausalLM.get_model_defaults.""" - return { - "attn_backend": "FLASHINFER", - } - - def _check_and_adjust_experts_implementation(self, *args, **kwargs): - # transformers 5.x ``PreTrainedModel.__init__`` calls this with an - # ``experts_implementation`` argument and fails for VL wrapper models - # that do not directly contain MoE layers. TRT-LLM manages expert - # implementations independently, so skip the check. - return None - def __init__(self, model_config: ModelConfig[Gemma4Config]): if _is_mm_disagg(): raise NotImplementedError( @@ -715,31 +940,6 @@ def __init__(self, model_config: ModelConfig[Gemma4Config]): self.post_config() self.is_loaded = True - @staticmethod - def get_sub_model_config( - model_config: ModelConfig[Gemma4Config], - name: str, - ) -> ModelConfig: - assert name in ["text_config", "vision_config", "audio_config"], ( - f"Expected subconfig name to be 'text_config', 'vision_config', " - f"or 'audio_config'. Got {name} instead." - ) - pretrained_config = getattr(model_config.pretrained_config, name) - quant_config = model_config.quant_config if name == "text_config" else None - preferred_backend = "FLASHINFER" if name == "text_config" else "TRTLLM" - sub_config: ModelConfig = dataclasses.replace( - model_config, - pretrained_config=pretrained_config, - attn_backend=preferred_backend, - quant_config=quant_config, - ) - if ( - hasattr(sub_config.pretrained_config, "torch_dtype") - and sub_config.pretrained_config.torch_dtype is None - ): - sub_config.pretrained_config.torch_dtype = model_config.pretrained_config.torch_dtype - return sub_config - def load_weights(self, weights: Dict, weight_mapper: BaseWeightMapper): # Gemma4 checkpoint keys: model.language_model.X -> need model.X for LLM # Remap: "model.language_model.layers.0..." -> "model.layers.0..." @@ -771,35 +971,6 @@ def load_weights(self, weights: Dict, weight_mapper: BaseWeightMapper): embed_a_weights = filter_weights("embed_audio", stripped) self.embed_audio.load_weights(embed_a_weights) - def post_config(self): - self.config = self.llm.config - self.model_config.pretrained_config = self.llm.config - - def infer_max_seq_len(self) -> int: - return self.llm.infer_max_seq_len() - - @property - def vocab_size_padded(self) -> int: - return self.llm.vocab_size_padded - - @property - def multimodal_data_device_paths(self) -> List[str]: - """Dotted paths in ``multimodal_data`` that the engine should ship to - GPU. Anything not listed stays CPU-resident — notably - ``image.image_seq_lens`` / ``video.image_seq_lens`` (Python - ``List[int]`` carrying per-image valid-patch counts, consumed - host-side by ``Gemma4VisionModel.forward`` to populate - ``attn_metadata.prompt_lens`` without a GPU→CPU sync). - """ - return [ - "image.pixel_values", - "image.image_position_ids", - "video.pixel_values", - "video.image_position_ids", - "audio.audio_features", - "audio.audio_features_mask", - ] - @nvtx_range("[Vision] process") def _get_image_features( self, @@ -878,229 +1049,3 @@ def _get_audio_features( else: projected = projected.reshape(-1, projected.shape[-1]) return projected.contiguous() - - @staticmethod - def _has_active_multimodal_tokens(multimodal_param: MultimodalParams) -> bool: - """Whether a context parameter needs embeddings in this forward.""" - runtime = multimodal_param.multimodal_runtime - if runtime is not None and runtime.num_mm_tokens_in_chunk == 0: - return False - - multimodal_data = multimodal_param.multimodal_data - if multimodal_data.get("multimodal_embedding") is not None: - return True - - payload_fields = ( - ("image", "pixel_values"), - ("video", "pixel_values"), - ("audio", "audio_features"), - ) - return any( - multimodal_data.get(modality, {}).get(field) is not None - for modality, field in payload_fields - ) - - def _forward_multimodal_encoder( - self, multimodal_params: List[MultimodalParams] - ) -> torch.Tensor: - """Encode uncached Gemma4 image, video, and audio payloads.""" - modality_inputs = ( - ("image", "pixel_values"), - ("video", "pixel_values"), - ("audio", "audio_features"), - ) - param_modalities = [ - [ - modality - for modality, input_field in modality_inputs - if multimodal_param.multimodal_data.get(modality, {}).get(input_field) is not None - ] - for multimodal_param in multimodal_params - ] - - # The cache helper splits this tensor by request. Preserve that order while still batching - # consecutive requests with the same modality. - if ( - all(len(modalities) == 1 for modalities in param_modalities) - and len({modalities[0] for modalities in param_modalities}) > 1 - ): - embeddings = [] - for _, param_group in groupby( - zip(param_modalities, multimodal_params), key=lambda item: item[0][0] - ): - embeddings.append( - self._forward_multimodal_encoder( - [multimodal_param for _, multimodal_param in param_group] - ) - ) - return torch.cat(embeddings, dim=0) - - pixel_values_list = [] - image_position_ids_list = [] - image_seq_lens_extended: List[int] = [] - audio_features_list = [] - audio_features_mask_list: List[Optional[torch.Tensor]] = [] - video_pixel_values_list = [] - video_position_ids_list = [] - video_seq_lens_extended: List[int] = [] - - for multimodal_param in multimodal_params: - multimodal_data = multimodal_param.multimodal_data - image_data = multimodal_data.get("image", {}) - pixel_values = image_data.get("pixel_values") - if pixel_values is not None: - pixel_values_list.append(pixel_values) - image_position_ids = image_data.get("image_position_ids") - if image_position_ids is not None: - image_position_ids_list.append(image_position_ids) - image_seq_lens = image_data.get("image_seq_lens") - if image_seq_lens is not None: - image_seq_lens_extended.extend(image_seq_lens) - - audio_data = multimodal_data.get("audio", {}) - audio_features = audio_data.get("audio_features") - if audio_features is not None: - audio_features_list.append(audio_features) - audio_features_mask_list.append(audio_data.get("audio_features_mask")) - - video_data = multimodal_data.get("video", {}) - video_pixel_values = video_data.get("pixel_values") - if video_pixel_values is not None: - video_pixel_values_list.append(video_pixel_values) - video_position_ids = video_data.get("image_position_ids") - if video_position_ids is not None: - video_position_ids_list.append(video_position_ids) - video_seq_lens = video_data.get("image_seq_lens") - if video_seq_lens is not None: - video_seq_lens_extended.extend(video_seq_lens) - - multimodal_embeddings = [] - if pixel_values_list: - pixel_values = torch.cat(pixel_values_list) - image_position_ids = ( - torch.cat(image_position_ids_list) - if len(image_position_ids_list) == len(pixel_values_list) - else None - ) - multimodal_embeddings.append( - self._get_image_features( - pixel_values=pixel_values, - image_position_ids=image_position_ids, - image_seq_lens=( - image_seq_lens_extended - if len(image_seq_lens_extended) == pixel_values.shape[0] - else None - ), - ) - ) - - # Video frames use the same vision tower as images. - if video_pixel_values_list: - video_pixel_values = torch.cat(video_pixel_values_list) - video_position_ids = ( - torch.cat(video_position_ids_list) - if len(video_position_ids_list) == len(video_pixel_values_list) - else None - ) - multimodal_embeddings.append( - self._get_image_features( - pixel_values=video_pixel_values, - image_position_ids=video_position_ids, - image_seq_lens=( - video_seq_lens_extended - if len(video_seq_lens_extended) == video_pixel_values.shape[0] - else None - ), - ) - ) - - if audio_features_list and self.audio_tower is not None: - per_audio_embeddings = [ - self._get_audio_features(audio_features, audio_features_mask_list[i]) - for i, audio_features in enumerate(audio_features_list) - ] - multimodal_embeddings.append(torch.cat(per_audio_embeddings, dim=0)) - - if not multimodal_embeddings: - raise ValueError("Gemma4 received active multimodal parameters without encoder inputs") - return torch.cat(multimodal_embeddings, dim=0) - - @torch.inference_mode() - def forward( - self, - attn_metadata: AttentionMetadata, - input_ids: Optional[torch.LongTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - return_context_logits: Optional[bool] = False, - **kwargs, - ) -> torch.Tensor: - multimodal_params = kwargs.get("multimodal_params", [])[: attn_metadata.num_contexts] - active_multimodal_params = [ - multimodal_param - for multimodal_param in multimodal_params - if self._has_active_multimodal_tokens(multimodal_param) - ] - - mm_embeds = [] - if active_multimodal_params: - mm_embeds = get_multimodal_embeddings( - encoder_forward_fn=self._forward_multimodal_encoder, - multimodal_params=active_multimodal_params, - ) - mm_embeds = find_input_mm_embeds(mm_embeds, active_multimodal_params) - - mm_token_type_ids = None - if mm_embeds: - # Build integer token types only when this chunk contains active - # multimodal embeddings. Singleton comparisons avoid repeated - # torch.isin scans over the packed input. - mm_token_type_ids = torch.zeros_like(input_ids, dtype=torch.long) - mm_token_type_ids[input_ids == self.image_token_ids[0]] = 1 - if self.video_token_ids is not None: - mm_token_type_ids[input_ids == self.video_token_ids[0]] = 2 - if self.audio_token_ids is not None: - mm_token_type_ids[input_ids == self.audio_token_ids[0]] = 3 - - # Build a PLE-safe view of the original input_ids where every - # multimodal token is replaced by the text pad_token_id. The - # Gemma4 Per-Layer Embedding lookup uses this view so the PLE table - # is not consulted at audio/image/video positions (matches HF's - # Gemma4Model behaviour). Without this, multimodal requests on - # E2B/E4B (which use PLE) produce garbage output because - # ``fuse_input_embeds`` returns ``input_ids=None`` and PLE is then - # silently skipped inside ``Gemma4TextModel.forward``. - ple_input_ids = None - if mm_token_type_ids is not None and self.llm.model.hidden_size_per_layer_input: - text_config = getattr(self.config, "text_config", self.config) - pad_id = getattr(text_config, "pad_token_id", None) - if pad_id is not None: - ple_input_ids = torch.where( - mm_token_type_ids > 0, - torch.full_like(input_ids, pad_id), - input_ids, - ) - - input_ids, inputs_embeds = fuse_input_embeds( - embedding_layer=self.llm.model.embed_tokens, - input_ids=input_ids, - mm_embeds=mm_embeds, - mm_token_ids=self.mm_token_ids, - mm_token_indices=kwargs.get("mm_token_indices"), - text_token_indices=kwargs.get("text_token_indices"), - ) - logits = self.llm.forward( - attn_metadata=attn_metadata, - input_ids=input_ids, - position_ids=position_ids, - inputs_embeds=inputs_embeds, - return_context_logits=return_context_logits, - mm_token_type_ids=mm_token_type_ids, - ple_input_ids=ple_input_ids, - lora_params=kwargs.get("lora_params", None), - ) - return logits - - @property - def mm_token_ids(self) -> torch.Tensor: - return self._mm_token_ids diff --git a/tensorrt_llm/_torch/models/modeling_gpt_oss.py b/tensorrt_llm/_torch/models/modeling_gpt_oss.py index 00b5c77c8951..d48d454715e1 100644 --- a/tensorrt_llm/_torch/models/modeling_gpt_oss.py +++ b/tensorrt_llm/_torch/models/modeling_gpt_oss.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Any, Dict, Literal, Optional import torch from torch import nn @@ -60,7 +60,10 @@ def __init__( beta_fast=pretrained_config.rope_scaling['beta_fast'], beta_slow=pretrained_config.rope_scaling['beta_slow'], duplicate_data=False), - is_neox=False, + # GPT-OSS applies NeoX-style (rotate-half) RoPE, matching the HF + # reference. The fused kernel ignores this flag for yarn (which + # masked the wrong value); the unfused path honors it. + is_neox=True, ) super().__init__( @@ -549,6 +552,13 @@ def forward( @register_auto_model("GptOssForCausalLM") class GptOssForCausalLM(SpecDecOneEngineForCausalLM[Transformer, GptOssConfig]): + @classmethod + def get_preferred_transceiver_runtime( + cls, + pretrained_config: Any = None, + ) -> Optional[Literal["CPP", "PYTHON"]]: + return "PYTHON" + params_map = { # TRTLLM module name : GptOss module name "qkv_proj": "qkv", diff --git a/tensorrt_llm/_torch/models/modeling_kimi_k25.py b/tensorrt_llm/_torch/models/modeling_kimi_k25.py index e4706cb79502..a216e916df0f 100644 --- a/tensorrt_llm/_torch/models/modeling_kimi_k25.py +++ b/tensorrt_llm/_torch/models/modeling_kimi_k25.py @@ -34,7 +34,7 @@ import os import tempfile from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Literal, Optional, Tuple, Union import numpy as np import torch @@ -1517,6 +1517,21 @@ class KimiK25ForConditionalGeneration(PreTrainedModel): _LANG_PREFIX = "language_model." + @classmethod + def get_preferred_transceiver_runtime( + cls, + pretrained_config: Any = None, + ) -> Literal["PYTHON"]: + """Kimi-K2.5 defaults to the Python (v2) KV-cache transceiver. + + The DeepSeek-V3 MLA backbone transfers a large latent KV, which the + Python transceiver handles better in disaggregated serving. This is + only adopted when the user leaves + ``cache_transceiver_config.transceiver_runtime`` at 'auto' and the + effective backend is NIXL; otherwise the C++ transceiver is used. + """ + return "PYTHON" + def __init__( self, model_config: ModelConfig[PretrainedConfig], diff --git a/tensorrt_llm/_torch/models/modeling_minicpmv4_6.py b/tensorrt_llm/_torch/models/modeling_minicpmv4_6.py new file mode 100644 index 000000000000..c223f61b4561 --- /dev/null +++ b/tensorrt_llm/_torch/models/modeling_minicpmv4_6.py @@ -0,0 +1,967 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""MiniCPM-V 4.6 (openbmb/MiniCPM-V-4.6) for the TRT-LLM PyTorch backend. + +Composition: + * A SigLIP2-style variable-resolution ViT (NaViT-packed) with an intermediate + window-attention merger and a final 2x2 downsample-MLP merger, ported to + TRT-LLM modules (``Attention`` / ``Linear`` / ``LayerNorm`` / ``MLP``). + * A Qwen3.5 dense hybrid (linear + full attention) text tower, resolved from + ``text_config`` through TRT-LLM's ``AutoModelForCausalLM`` (Qwen3_5ForCausalLM). + +Both image and video modalities are wired up. Video reuses the exact same +NaViT-packed vision path: the HF processor packs every frame/slice into +``pixel_values_videos`` / ``target_sizes_videos`` with the same layout as image +inputs, and HF's ``get_video_features`` is (for batch/beam == 1) an identity +repack of ``get_image_features`` -- so the ported ``_get_image_features`` is +correct for video too. + +The vision tower uses learned (interpolated) absolute position embeddings and no +rotary embedding, so the text tower's standard 1-D RoPE path is untouched (this +model does *not* use mRoPE). +""" + +import dataclasses +from typing import Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +import transformers +from packaging.version import Version +from transformers import AutoProcessor, AutoTokenizer, PretrainedConfig, PreTrainedModel + +from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.mapping import Mapping + +from ..._utils import nvtx_range +from ...inputs import ( + BaseMultimodalDummyInputsBuilder, + BaseMultimodalInputProcessor, + ContentFormat, + ExtraProcessedInputs, + MultimodalPlaceholderMetadata, + MultimodalPlaceholderPlacement, + TextPrompt, + register_input_processor, +) +from ...sampling_params import SamplingParams +from ..attention_backend import AttentionMetadata +from ..attention_backend.interface import PredefinedAttentionMask +from ..attention_backend.utils import get_attention_backend +from ..model_config import ModelConfig +from ..modules.attention import Attention +from ..modules.layer_norm import LayerNorm +from ..modules.linear import Linear +from ..modules.mlp import MLP +from .checkpoints.base_weight_mapper import BaseWeightMapper +from .checkpoints.hf.qwen3_5_weight_mapper import Qwen3_5MoeHfWeightMapper +from .modeling_auto import AutoModelForCausalLM +from .modeling_multimodal_utils import ( + _is_mm_disagg, + find_input_mm_embeds, + fuse_input_embeds, + get_multimodal_embeddings, +) +from .modeling_qwen2vl import _prepare_qwen_vl_vision_attn_metadata +from .modeling_utils import QuantConfig, _load_weights_impl, register_auto_model + + +def _gelu_tanh(x: torch.Tensor) -> torch.Tensor: + """``gelu_pytorch_tanh`` activation used by the ViT encoder / window merger.""" + return F.gelu(x, approximate="tanh") + + +# MiniCPM-V 4.6 was upstreamed into transformers as a native model +# (``minicpmv4_6``); its processor/config classes only ship in >=5.7.0, and the +# checkpoint carries no remote code (``auto_map``) to fall back on. The TRT-LLM +# model/config code is import-safe on older releases, so only the input +# processor (which loads the native ``MiniCPMV4_6Processor``) hard-requires it. +_MINICPMV4_6_MIN_TRANSFORMERS = "5.7.0" + + +def _ensure_transformers_supports_minicpmv4_6() -> None: + """Raise a clear error if transformers is too old for the HF processor.""" + installed = transformers.__version__ + if Version(installed) < Version(_MINICPMV4_6_MIN_TRANSFORMERS): + raise RuntimeError( + f"MiniCPM-V 4.6 requires transformers>=" + f"{_MINICPMV4_6_MIN_TRANSFORMERS} for its native processor/config " + f"(installed: {installed}). This model was upstreamed into " + f"transformers and ships no remote code to fall back on. Please " + f"upgrade, e.g. `pip install 'transformers>=" + f"{_MINICPMV4_6_MIN_TRANSFORMERS}'`." + ) + + +# --------------------------------------------------------------------------- +# Vision tower +# --------------------------------------------------------------------------- +class MiniCPMV4_6VisionEmbeddings(nn.Module): + """Conv2d patch embedding + interpolated absolute position embeddings. + + Adapted from SigLIP's NaViT variant: images keep their native aspect ratio, + so position ids are bucketized per (h, w) grid rather than looked up from a + fixed square grid. + """ + + def __init__(self, config: PretrainedConfig, dtype: torch.dtype): + super().__init__() + self.embed_dim = config.hidden_size + self.patch_size = config.patch_size + self.image_size = config.image_size + + self.patch_embedding = nn.Conv2d( + in_channels=config.num_channels, + out_channels=self.embed_dim, + kernel_size=self.patch_size, + stride=self.patch_size, + padding="valid", + dtype=dtype, + ) + self.num_patches_per_side = self.image_size // self.patch_size + self.num_positions = self.num_patches_per_side**2 + self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim, dtype=dtype) + + def forward(self, pixel_values: torch.Tensor, target_sizes: torch.Tensor) -> torch.Tensor: + # pixel_values: NaViT-packed [1, C, patch_size, total_patch_axis] + patch_embeds = self.patch_embedding(pixel_values) + embeddings = patch_embeds.flatten(2).transpose(1, 2) # [1, T, D] + + nps = self.num_patches_per_side + boundaries = torch.arange(1 / nps, 1.0, 1 / nps) + position_embeddings = [] + # target_sizes lives on CPU (used for integer arithmetic here). + for target_size in target_sizes: + h, w = int(target_size[0]), int(target_size[1]) + fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / h) + fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / w) + bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True) + bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True) + pos_ids = ( + (bucket_coords_h[:, None] * nps + bucket_coords_w) + .flatten() + .to(self.position_embedding.weight.device) + ) + position_embeddings.append(self.position_embedding(pos_ids)) + + position_embeddings = torch.concat(position_embeddings, dim=0).unsqueeze(0) + return embeddings + position_embeddings + + +class MiniCPMV4_6VisionAttention(Attention): + """Variable-length full self-attention (no RoPE) for the ViT tower. + + Mirrors ``Qwen2_5_VLVisionAttention``: fused ``qkv_proj`` / ``o_proj``, a + per-segment ``attn_metadata`` (cu_seqlens) built by the caller, and a + custom forward that skips the generic RoPE path. Runs replicated (tp=1). + """ + + def __init__(self, model_config: ModelConfig[PretrainedConfig], layer_idx: int): + config = model_config.pretrained_config.vision_config + text_config = model_config.pretrained_config.text_config + super().__init__( + hidden_size=config.hidden_size, + num_attention_heads=config.num_attention_heads, + num_key_value_heads=config.num_attention_heads, + max_position_embeddings=text_config.max_position_embeddings, + bias=True, + pos_embd_params=None, + rope_fusion=False, + layer_idx=layer_idx, + dtype=config.torch_dtype, + config=model_config, + reduce_output=False, + head_dim=config.hidden_size // config.num_attention_heads, + ) + # Vision attention runs eagerly from the VL wrapper (outside the + # compiled LM region); unregister so `forward_impl` uses the eager path + # with the vision-local `attn_metadata` instead of the LM's. + if self.register_to_config: + model_config.extra_attrs.get("attn_layers", {}).pop(self.layer_idx_str, None) + self.register_to_config = False + + def forward( + self, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + qkv = self.qkv_proj(hidden_states) + q, k, v = self.split_qkv(qkv, None, None) + q, k, v = self.convert_qkv(q, k, v) + output = self.forward_impl( + q=q, + k=k, + v=v, + attn_metadata=attn_metadata, + attention_mask=PredefinedAttentionMask.FULL, + attention_window_size=None, + attention_mask_data=None, + mrope_config=None, + attention_sinks=None, + ) + return self.o_proj(output, layer_idx=self.layer_idx) + + def forward_window(self, hidden_states: torch.Tensor, window_len: int) -> torch.Tensor: + """Fixed-size window self-attention via batched SDPA. + + The ViT window merger reorders tokens into consecutive, equal-length + windows (``window_h * window_w`` tokens each) that never attend across + window boundaries. Running this as one variable-length fused-attention + call produces one segment per window; with many frames/slices the + segment count is large enough to overflow the fused-attention kernel's + launch limits. Because every window is the *same* small length, it maps + exactly onto a regular batched attention instead -- which is both + cheaper and free of that segment-count ceiling. + """ + num_tokens = hidden_states.shape[0] + num_windows = num_tokens // window_len + qkv = self.qkv_proj(hidden_states) + q, k, v = self.split_qkv(qkv) + + def to_windows(x: torch.Tensor) -> torch.Tensor: + # [num_windows * window_len, H*D] -> [num_windows, H, window_len, D] + return x.view(num_windows, window_len, self.num_heads, self.head_dim).transpose(1, 2) + + q, k, v = to_windows(q), to_windows(k), to_windows(v) + output = F.scaled_dot_product_attention(q, k, v) + output = output.transpose(1, 2).reshape(num_tokens, self.num_heads * self.head_dim) + return self.o_proj(output, layer_idx=self.layer_idx) + + +class MiniCPMV4_6VisionMLP(MLP): + """ViT feed-forward (``fc1`` -> gelu_tanh -> ``fc2``) on TRT-LLM ``MLP``.""" + + def __init__(self, model_config: ModelConfig[PretrainedConfig], layer_idx: int): + config = model_config.pretrained_config.vision_config + super().__init__( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + bias=True, + activation=_gelu_tanh, + dtype=config.torch_dtype, + config=model_config, + layer_idx=layer_idx, + ) + + +class MiniCPMV4_6VisionEncoderLayer(nn.Module): + def __init__(self, model_config: ModelConfig[PretrainedConfig], layer_idx: int): + super().__init__() + config = model_config.pretrained_config.vision_config + dtype = config.torch_dtype + self.layer_norm1 = LayerNorm( + hidden_size=config.hidden_size, eps=config.layer_norm_eps, dtype=dtype + ) + self.self_attn = MiniCPMV4_6VisionAttention(model_config, layer_idx) + self.layer_norm2 = LayerNorm( + hidden_size=config.hidden_size, eps=config.layer_norm_eps, dtype=dtype + ) + self.mlp = MiniCPMV4_6VisionMLP(model_config, layer_idx) + + def forward( + self, hidden_states: torch.Tensor, attn_metadata: AttentionMetadata + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) + hidden_states = self.self_attn(hidden_states, attn_metadata) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +class MiniCPMV4_6VisionEncoder(nn.Module): + def __init__(self, model_config: ModelConfig[PretrainedConfig]): + super().__init__() + config = model_config.pretrained_config.vision_config + self.layers = nn.ModuleList( + [ + MiniCPMV4_6VisionEncoderLayer(model_config, layer_idx) + for layer_idx in range(config.num_hidden_layers) + ] + ) + + +class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module): + """Intermediate window self-attention + 2x2 window merge (inserted mid-ViT). + + Reorders tokens into 2x2 windows, runs window-local self-attention, then + merges every 2x2 window into one token (with a mean-pool residual) through a + block-structured MLP (``linear_1`` -> gelu_tanh -> ``linear_2``). Halves the + per-axis patch grid. + """ + + def __init__(self, model_config: ModelConfig[PretrainedConfig]): + super().__init__() + config = model_config.pretrained_config.vision_config + dtype = config.torch_dtype + self.window_kernel_size = tuple(config.window_kernel_size) + self.embed_dim = config.hidden_size + mapping = model_config.mapping + + self.self_attn = MiniCPMV4_6VisionAttention( + model_config, layer_idx=config.num_hidden_layers + ) + self.layer_norm1 = LayerNorm( + hidden_size=self.embed_dim, eps=config.layer_norm_eps, dtype=dtype + ) + self.pre_norm = LayerNorm( + hidden_size=config.window_hidden_size, eps=config.layer_norm_eps, dtype=dtype + ) + self.linear_1 = Linear( + config.window_hidden_size, + config.window_intermediate_size, + bias=True, + dtype=dtype, + mapping=mapping, + tensor_parallel_mode=None, + ) + self.linear_2 = Linear( + config.window_intermediate_size, + self.embed_dim, + bias=True, + dtype=dtype, + mapping=mapping, + tensor_parallel_mode=None, + ) + + def get_window_index( + self, target_sizes: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, int]: + window_h, window_w = self.window_kernel_size + max_seqlens = window_h * window_w + + window_index_list = [] + cu_seqlens = [0] + token_offset = 0 + for height, width in target_sizes: + height, width = int(height), int(width) + if height % window_h != 0 or width % window_w != 0: + raise ValueError( + f"height={height}, width={width} must be divisible by " + f"window size {self.window_kernel_size}" + ) + index = torch.arange(height * width).reshape(height, width) + num_windows_h = height // window_h + num_windows_w = width // window_w + num_windows = num_windows_h * num_windows_w + index = index.reshape(num_windows_h, window_h, num_windows_w, window_w) + index = index.permute(0, 2, 1, 3).reshape(num_windows, window_h * window_w) + window_index_list.append(index.reshape(-1) + token_offset) + cu_this = torch.arange(1, num_windows + 1) * (window_h * window_w) + cu_seqlens[-1] + cu_seqlens.extend(cu_this.tolist()) + token_offset += height * width + + window_index = torch.cat(window_index_list) + cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32) + return window_index, cu_seqlens, max_seqlens + + def forward( + self, hidden_states: torch.Tensor, target_sizes: torch.Tensor, cu_seqlens: List[int] + ) -> torch.Tensor: + # hidden_states: [T, D] + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) + device = hidden_states.device + + window_index, _, window_len = self.get_window_index(target_sizes) + window_index = window_index.to(device) + + # Every window is exactly `window_len` (= window_h*window_w) tokens, so + # the window self-attention is a regular batched attention rather than a + # variable-length one (see MiniCPMV4_6VisionAttention.forward_window). + hidden_states = hidden_states[window_index, :] + hidden_states = self.self_attn.forward_window(hidden_states, window_len) + hidden_states = hidden_states[torch.argsort(window_index), :] + hidden_states = residual + hidden_states + + window_h, window_w = self.window_kernel_size + embed_dim = hidden_states.shape[-1] + merged = [] + for i in range(len(target_sizes)): + height, width = int(target_sizes[i][0]), int(target_sizes[i][1]) + patch = hidden_states[cu_seqlens[i] : cu_seqlens[i + 1], :] + merged_h, merged_w = height // window_h, width // window_w + patch_5d = patch.view(merged_h, window_h, merged_w, window_w, embed_dim).permute( + 0, 2, 1, 3, 4 + ) + hidden_state = patch_5d.reshape(merged_h * merged_w, window_h * window_w * embed_dim) + mean_residual = patch_5d.reshape( + merged_h * merged_w, window_h * window_w, embed_dim + ).mean(dim=1) + hidden_state = self.pre_norm(hidden_state) + hidden_state = self.linear_1(hidden_state) + hidden_state = _gelu_tanh(hidden_state) + hidden_state = self.linear_2(hidden_state) + merged.append(hidden_state + mean_residual) + + return torch.concat(merged, dim=0) + + +class MiniCPMV4_6DownsampleMLP(nn.Module): + """2x2 spatial-merge projection (``pre_norm`` -> ``linear_1`` -> GELU -> ``linear_2``).""" + + def __init__(self, model_config: ModelConfig[PretrainedConfig], hidden_size: int, out_dim: int): + super().__init__() + dtype = model_config.pretrained_config.vision_config.torch_dtype + mapping = model_config.mapping + merge_kernel_size = tuple(model_config.pretrained_config.merge_kernel_size) + merge_factor = merge_kernel_size[0] * merge_kernel_size[1] + merged_hidden_size = hidden_size * merge_factor + self.pre_norm = LayerNorm(hidden_size=merged_hidden_size, eps=1e-6, dtype=dtype) + self.linear_1 = Linear( + merged_hidden_size, + merged_hidden_size, + bias=True, + dtype=dtype, + mapping=mapping, + tensor_parallel_mode=None, + ) + self.linear_2 = Linear( + merged_hidden_size, + out_dim, + bias=True, + dtype=dtype, + mapping=mapping, + tensor_parallel_mode=None, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.pre_norm(hidden_states).view(-1, self.linear_1.in_features) + hidden_states = self.linear_1(hidden_states) + hidden_states = F.gelu(hidden_states) + hidden_states = self.linear_2(hidden_states) + return hidden_states + + +class MiniCPMV4_6Merger(nn.Module): + """Final per-image/frame 2x2 merge into the LLM embedding space.""" + + def __init__(self, model_config: ModelConfig[PretrainedConfig]): + super().__init__() + config = model_config.pretrained_config + self.merge_kernel_size = tuple(config.merge_kernel_size) + self.merger_times = config.merger_times + hidden_size = config.vision_config.hidden_size + llm_embed_dim = config.text_config.hidden_size + mlps = [ + MiniCPMV4_6DownsampleMLP(model_config, hidden_size, hidden_size) + for _ in range(self.merger_times - 1) + ] + mlps.append(MiniCPMV4_6DownsampleMLP(model_config, hidden_size, llm_embed_dim)) + self.mlp = nn.ModuleList(mlps) + + def forward( + self, hidden_states: torch.Tensor, target_sizes: torch.Tensor + ) -> List[torch.Tensor]: + merge_h, merge_w = self.merge_kernel_size + start = 0 + processed_features = [] + for i in range(len(target_sizes)): + height, width = int(target_sizes[i][0]), int(target_sizes[i][1]) + num_patches = height * width + embed_dim = hidden_states.shape[-1] + merged_h, merged_w = height // merge_h, width // merge_w + hidden_state = ( + hidden_states[start : start + num_patches, :] + .view(merged_h, merge_h, merged_w, merge_w, embed_dim) + .permute(0, 2, 1, 3, 4) + .reshape(merged_h * merged_w, merge_h * merge_w * embed_dim) + ) + hidden_state = self.mlp[0](hidden_state) + for j in range(1, self.merger_times): + height = height // merge_h + width = width // merge_w + inner_dim = hidden_state.shape[-1] + merged_h, merged_w = height // merge_h, width // merge_w + hidden_state = ( + hidden_state.view(merged_h, merge_h, merged_w, merge_w, inner_dim) + .permute(0, 2, 1, 3, 4) + .reshape(merged_h * merged_w, merge_h * merge_w * inner_dim) + ) + hidden_state = self.mlp[j](hidden_state) + start += num_patches + processed_features.append(hidden_state) + return processed_features + + +class MiniCPMV4_6VisionModel(nn.Module): + """Full vision tower: embeddings -> encoder (+window merger) -> merger. + + Consumes a batch of :class:`MultimodalParams`, concatenates every request's + NaViT-packed ``pixel_values`` / ``target_sizes`` into one packed sequence + (Contract 4), and returns a single ``[total_mm_tokens, llm_hidden]`` tensor. + Runs replicated on every rank (tp=1); the vision encoder is excluded from + quantization. + """ + + def __init__(self, model_config: ModelConfig[PretrainedConfig]): + super().__init__() + config = model_config.pretrained_config + self.vision_config = config.vision_config + # Authoritative model dtype (set by ModelConfig.from_pretrained on the + # composite; the raw config.json omits torch_dtype, so fall back to + # bf16 which matches the released checkpoint). Mirror it onto the + # vision sub-config so ported modules pick up a concrete dtype. + self.dtype = config.torch_dtype or torch.bfloat16 + self.vision_config.torch_dtype = self.dtype + self.downsample_mode = config.downsample_mode + self.insert_layer_id = self.vision_config.insert_layer_id + + # Replicated, unquantized ModelConfig for the vision sub-modules. + # dataclasses.replace gives a fresh extra_attrs (init=False), so the + # vision attention's transient layer registration can't collide with + # the LLM's. + # NOTE: mapping=Mapping() (tp=1) is load-bearing for correctness, not + # just perf. It makes the whole vision tower run REPLICATED on every + # rank (all vision Linears use tensor_parallel_mode=None and vision + # attention uses reduce_output=False). Under TP that means every rank + # computes identical vision embeds from the broadcast multimodal data; + # do NOT switch this to the LLM's real mapping -- a row-parallel + # projection would then all-reduce data that was never sharded across + # ranks and produce wrong output. + vision_model_config = dataclasses.replace( + model_config, quant_config=QuantConfig(), mapping=Mapping() + ) + self.model_config = vision_model_config + self.config = self.vision_config + + self.embeddings = MiniCPMV4_6VisionEmbeddings(self.vision_config, self.dtype) + self.encoder = MiniCPMV4_6VisionEncoder(vision_model_config) + self.post_layernorm = LayerNorm( + hidden_size=self.vision_config.hidden_size, + eps=self.vision_config.layer_norm_eps, + dtype=self.dtype, + ) + self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger(vision_model_config) + self.merger = MiniCPMV4_6Merger(vision_model_config) + + self.metadata_cls = get_attention_backend(model_config.attn_backend).Metadata + + def _make_attn_metadata(self, seq_lens: List[int]) -> AttentionMetadata: + num_segments = len(seq_lens) + attn_metadata = self.metadata_cls( + max_num_requests=num_segments + 1, + max_num_tokens=sum(seq_lens) + 1, + kv_cache_manager=None, + ) + return _prepare_qwen_vl_vision_attn_metadata(seq_lens, attn_metadata) + + @staticmethod + def _grid_seq_lens(target_sizes: torch.Tensor) -> List[int]: + return (target_sizes[:, 0] * target_sizes[:, 1]).tolist() + + @staticmethod + def _grid_cu_seqlens(target_sizes: torch.Tensor) -> List[int]: + cu = [0] + for i in range(len(target_sizes)): + cu.append(cu[-1] + int(target_sizes[i][0]) * int(target_sizes[i][1])) + return cu + + def _get_image_features( + self, pixel_values: torch.Tensor, target_sizes: torch.Tensor + ) -> List[torch.Tensor]: + pixel_values = pixel_values.to(self.dtype) + hidden_states = self.embeddings(pixel_values, target_sizes).squeeze(0) + + use_vit_merger = self.downsample_mode != "4x" + if use_vit_merger and self.insert_layer_id >= 0: + attn_metadata = self._make_attn_metadata(self._grid_seq_lens(target_sizes)) + for layer_index, encoder_layer in enumerate(self.encoder.layers): + hidden_states = encoder_layer(hidden_states, attn_metadata) + if layer_index == self.insert_layer_id: + cu_seqlens = self._grid_cu_seqlens(target_sizes) + hidden_states = self.vit_merger(hidden_states, target_sizes, cu_seqlens) + window_h, window_w = self.vit_merger.window_kernel_size + target_sizes = target_sizes // torch.tensor( + [window_h, window_w], dtype=target_sizes.dtype + ) + attn_metadata = self._make_attn_metadata(self._grid_seq_lens(target_sizes)) + else: + attn_metadata = self._make_attn_metadata(self._grid_seq_lens(target_sizes)) + for encoder_layer in self.encoder.layers: + hidden_states = encoder_layer(hidden_states, attn_metadata) + + hidden_states = self.post_layernorm(hidden_states) + return self.merger(hidden_states, target_sizes) + + @torch.inference_mode() + @nvtx_range("MiniCPMV4_6 vision encoder") + def forward(self, multimodal_params: List[MultimodalParams]) -> List[torch.Tensor]: + pixel_values_list = [] + target_sizes_list = [] + for param in multimodal_params: + # Image and video share the NaViT-packed vision path. A request + # carries at most one modality (the input processor rejects mixed + # image+video requests), so the fixed image-then-video iteration + # order here is unambiguous and matches the placeholder order in + # `input_ids`. + for modality in ("image", "video"): + modality_data = param.multimodal_data.get(modality) + if modality_data is None: + continue + pixel_values_list.append(modality_data["pixel_values"]) + target_sizes_list.append(modality_data["target_sizes"]) + + if not pixel_values_list: + return [] + + # NaViT packing: concat along the packed patch axis; target grids stack. + pixel_values = ( + torch.cat(pixel_values_list, dim=-1) + if len(pixel_values_list) > 1 + else pixel_values_list[0] + ) + target_sizes = ( + torch.cat(target_sizes_list, dim=0) + if len(target_sizes_list) > 1 + else target_sizes_list[0] + ) + target_sizes = target_sizes.to("cpu") + + features = self._get_image_features(pixel_values, target_sizes) + fused = torch.cat(features, dim=0) + + # Contract-4 invariants (this forward returns ONE pre-concatenated + # tensor that find_input_mm_embeds / _cache_multimodal_embeddings slice + # back per request): + # I. Order -- `features` are concatenated in `multimodal_params` + # order; the loop above and the per-grid merger both preserve it, + # which is exactly the order the downstream slicers assume. (Not + # assertable cheaply -- covered by the multi-request E2E test.) + # II. Count -- the produced row count must equal the sum of each + # request's placeholder count (`total_embeds_in_request`). A + # mismatch means the vision output and the fused `input_ids` will + # misalign, so fail loudly here instead of silently scattering + # wrong rows downstream. Only checked when every request carries + # runtime counts (absent on some warmup / JIT paths). + runtimes = [p.multimodal_runtime for p in multimodal_params] + if all(r is not None and r.total_embeds_in_request is not None for r in runtimes): + expected = sum(r.total_embeds_in_request for r in runtimes) + assert fused.shape[0] == expected, ( + f"MiniCPMV4_6 vision produced {fused.shape[0]} embedding rows " + f"but the batch's per-request placeholder counts sum to " + f"{expected}; vision output would misalign with input_ids." + ) + return [fused] + + def load_weights(self, weights: Dict[str, torch.Tensor]): + converted_weights = {} + for key, value in weights.items(): + if key.startswith("model.vision_tower."): + converted_weights[key[len("model.vision_tower.") :]] = value + elif key.startswith("model.merger."): + converted_weights["merger." + key[len("model.merger.") :]] = value + + # q/k/v -> qkv_proj fusion is handled by _load_weights_impl's params_map; + # only the projection / MLP renames need explicit regex substitutions. + pattern_mapping = { + r"(.*?)self_attn\.out_proj\.(.*)": r"\1self_attn.o_proj.\2", + r"(.*?)mlp\.fc1\.(.*)": r"\1mlp.up_proj.\2", + r"(.*?)mlp\.fc2\.(.*)": r"\1mlp.down_proj.\2", + } + _load_weights_impl(self, converted_weights, params_map=pattern_mapping) + + +# --------------------------------------------------------------------------- +# Input processor +# --------------------------------------------------------------------------- +class MiniCPMV4_6InputProcessor(BaseMultimodalInputProcessor, BaseMultimodalDummyInputsBuilder): + """Runs the HF MiniCPM-V processor and rewrites image tokens to the OOV + sentinel expected by ``fuse_input_embeds``.""" + + def __init__( + self, + model_path: str, + config: PretrainedConfig, + tokenizer: AutoTokenizer, + trust_remote_code: bool = True, + **kwargs, + ): + _ensure_transformers_supports_minicpmv4_6() + super().__init__( + model_path=model_path, + config=config, + tokenizer=tokenizer, + trust_remote_code=trust_remote_code, + **kwargs, + ) + self._config = config + self._model_path = model_path + self._tokenizer = ( + tokenizer + if tokenizer is not None + else AutoTokenizer.from_pretrained(model_path, trust_remote_code=trust_remote_code) + ) + self._processor = AutoProcessor.from_pretrained( + model_path, use_fast=self.use_fast, trust_remote_code=trust_remote_code + ) + self._dtype = config.torch_dtype or config.text_config.torch_dtype or torch.bfloat16 + self.tllm_multimodal_token_id = self.get_vocab_size() + 1 + self.image_token_str = getattr(self._processor, "image_token", "<|image_pad|>") + self.video_token_str = getattr(self._processor, "video_token", "<|video_pad|>") + + def get_vocab_size(self) -> int: + return self.config.text_config.vocab_size + + @property + def config(self) -> PretrainedConfig: + return self._config + + @property + def tokenizer(self) -> AutoTokenizer: + return self._tokenizer + + @property + def model_path(self) -> str: + return self._model_path + + @property + def processor(self) -> AutoProcessor: + return self._processor + + @property + def dtype(self) -> torch.dtype: + return self._dtype + + def _mm_token_ids(self) -> List[int]: + return [ + tid + for attr in ("image_token_id", "video_token_id") + if (tid := getattr(self.config, attr, None)) is not None + ] + + def _postprocess(self, input_ids: torch.IntTensor) -> torch.IntTensor: + token_ids = self._mm_token_ids() + if token_ids: + ids_tensor = torch.tensor(token_ids, device=input_ids.device, dtype=input_ids.dtype) + input_ids[torch.isin(input_ids, ids_tensor)] = self.tllm_multimodal_token_id + return input_ids + + def _preprocess( + self, text_prompt: str, images, videos, video_metadata, mm_processor_kwargs: Dict + ): + do_rescale = True + if images and isinstance(images[0], torch.Tensor): + do_rescale = False + # load_video hands us CHW float frames already scaled to [0, 1], so + # the HF rescale (x / 255) must be skipped for video. + if videos and isinstance(videos[0][0], torch.Tensor): + do_rescale = False + call_kwargs = dict( + text=[text_prompt], + images=images, + do_rescale=do_rescale, + return_tensors="pt", + **mm_processor_kwargs, + ) + if videos is not None: + # Frames are already sampled by TRT-LLM's load_video, so disable the + # HF processor's own frame sampling (which would otherwise require + # full metadata and re-index against the original frame count). + call_kwargs["videos"] = videos + call_kwargs["video_metadata"] = video_metadata + call_kwargs["do_sample_frames"] = False + return self.processor(**call_kwargs) + + def get_num_tokens_per_image(self, *, image, **kwargs) -> int: + do_rescale = not isinstance(image, torch.Tensor) + processed = self.processor( + text=[self.image_token_str], images=[image], do_rescale=do_rescale, return_tensors="pt" + ) + input_ids = processed["input_ids"][0] + return int((input_ids == self.config.image_token_id).sum().item()) + + def get_num_tokens_per_video(self, *, video, video_metadata=None, **kwargs) -> int: + # `video` is the list of pre-sampled frames (see find_mm_token_lengths). + frames = video + do_rescale = not (frames and isinstance(frames[0], torch.Tensor)) + processed = self.processor( + text=[self.video_token_str], + videos=[frames], + video_metadata=[video_metadata] if video_metadata is not None else None, + do_sample_frames=False, + do_rescale=do_rescale, + return_tensors="pt", + ) + input_ids = processed["input_ids"][0] + return int((input_ids == self.config.video_token_id).sum().item()) + + @nvtx_range("MiniCPMV4_6InputProcessor call_with_text_prompt") + @torch.inference_mode() + def call_with_text_prompt( + self, + inputs: TextPrompt, + sampling_params: SamplingParams, + ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: + text_prompt = inputs.get("prompt") + mm_data = inputs.get("multi_modal_data", {}) + mm_processor_kwargs = inputs.get("mm_processor_kwargs", {}) or {} + + if not mm_data: + input_ids = self.tokenizer(text_prompt, return_tensors="pt").input_ids + return input_ids[0].to(torch.int32).tolist(), None + + images = mm_data.get("image") + video_datas = mm_data.get("video") + # Single-modality-per-request guard. `_postprocess` rewrites both the + # image and video placeholders to the same sentinel + # (`tllm_multimodal_token_id`), so once fused into `input_ids` the two + # modalities are indistinguishable and their relative order is lost. + # The vision tower, meanwhile, always concatenates embeddings in a + # fixed image-then-video order. A request that interleaves image and + # video placeholders would therefore misalign embeddings with their + # placeholder positions. Until order-preserving fusion is implemented, + # reject mixed image+video requests instead of producing silently + # wrong output (single-modality requests are unaffected). + if images and video_datas: + raise ValueError( + "MiniCPM-V 4.6 supports only a single modality (image OR " + "video) per request, but this request carries both. Please " + "split the image and video content into separate requests." + ) + videos = video_metadata = None + if video_datas is not None: + # Each item is a VideoData (frames + decode metadata) from + # load_video, mirroring the Qwen-VL input processor. + videos = [vd.frames for vd in video_datas] + video_metadata = [getattr(vd, "metadata", None) for vd in video_datas] + processed_inputs = self._preprocess( + text_prompt, images, videos, video_metadata, mm_processor_kwargs + ) + + multimodal_data = {} + pixel_values = processed_inputs.get("pixel_values") + if pixel_values is not None: + multimodal_data["image"] = { + # target_sizes stays on CPU: the vision encoder uses it for + # integer window/grid arithmetic (not listed in + # multimodal_data_device_paths). + "pixel_values": pixel_values.to(self.dtype), + "target_sizes": processed_inputs.get("target_sizes"), + } + # Video is packed identically to image (pixel_values_videos is a NaViT + # [1, C, patch, seq] tensor); store it under the "video" key so the + # vision tower runs the shared encoder path on it. + pixel_values_videos = processed_inputs.get("pixel_values_videos") + if pixel_values_videos is not None: + multimodal_data["video"] = { + "pixel_values": pixel_values_videos.to(self.dtype), + "target_sizes": processed_inputs.get("target_sizes_videos"), + } + + fused_input_ids = self._postprocess(processed_inputs["input_ids"][0]) + return fused_input_ids.to(torch.int32).tolist(), { + "multimodal_data": multimodal_data, + } + + +# --------------------------------------------------------------------------- +# Top-level VLM wrapper +# --------------------------------------------------------------------------- +@register_auto_model("MiniCPMV4_6ForConditionalGeneration") +@register_input_processor( + MiniCPMV4_6InputProcessor, + model_type="minicpmv4_6", + placeholder_metadata=MultimodalPlaceholderMetadata( + placeholder_map={ + "image": "<|image_pad|>", + "video": "<|video_pad|>", + }, + placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT, + placeholders_separator="\n", + content_format=ContentFormat.STRING, + ), +) +class MiniCPMV4_6Model(PreTrainedModel): + def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): + config = model_config.pretrained_config + super().__init__(config) + if hasattr(self, "llm"): + return + + if _is_mm_disagg(): + raise NotImplementedError( + "MiniCPMV4_6 does not support disaggregated multimodal serving " + "yet. Unset TLLM_MULTIMODAL_DISAGGREGATED or set it to '0'." + ) + + self.model_config = model_config + self.mm_encoder = MiniCPMV4_6VisionModel(model_config).eval() + + # Inner LLM resolved from text_config (Qwen3_5ForCausalLM, hybrid). + llm_model_config = dataclasses.replace(model_config, pretrained_config=config.text_config) + # Share the wrapper's extra_attrs so the LM's attention layers register + # into the same dict the engine binds via with_model_extra_attrs (needed + # for the compiled / piecewise-CUDA-graph LM path). dataclasses.replace + # would otherwise give a fresh (init=False) extra_attrs. + llm_model_config.extra_attrs = model_config.extra_attrs + self.llm = AutoModelForCausalLM.from_config(llm_model_config) + + self.image_token_id = config.image_token_id + self.post_config() + + def post_config(self): + # Downstream (KV cache manager, engine) expects the LLM-shaped config. + self.config = self.llm.config + self.model_config.pretrained_config = self.llm.config + + @property + def vocab_size_padded(self) -> int: + return self.llm.vocab_size_padded + + def infer_max_seq_len(self) -> int: + return self.llm.infer_max_seq_len() + + @torch.inference_mode() + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: Optional[torch.IntTensor] = None, + position_ids: Optional[torch.IntTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + return_context_logits: bool = False, + **kwargs, + ) -> torch.Tensor: + num_context_requests = attn_metadata.num_contexts + multimodal_params = kwargs.get("multimodal_params", []) + + mm_embeds = [] + if len(multimodal_params) > 0 and not _is_mm_disagg(): + mm_embeds = get_multimodal_embeddings( + encoder_forward_fn=self.mm_encoder.forward, + multimodal_params=multimodal_params[:num_context_requests], + ) + mm_embeds = find_input_mm_embeds(mm_embeds, multimodal_params[:num_context_requests]) + + input_ids, inputs_embeds = fuse_input_embeds( + self.llm.model.embed_tokens, input_ids, mm_embeds, **kwargs + ) + return self.llm.forward( + attn_metadata=attn_metadata, + input_ids=input_ids, + position_ids=position_ids, + inputs_embeds=inputs_embeds, + return_context_logits=return_context_logits, + ) + + @property + def multimodal_data_device_paths(self) -> List[str]: + return ["image.pixel_values", "video.pixel_values", "multimodal_embedding"] + + def load_weights(self, weights: Dict[str, torch.Tensor], weight_mapper: BaseWeightMapper): + self.mm_encoder.load_weights(weights) + if hasattr(weights, "mark_consumed"): + weights.mark_consumed("model.vision_tower") + weights.mark_consumed("model.merger") + + llm_weight_mapper = Qwen3_5MoeHfWeightMapper() + llm_weight_mapper.init_model_and_config(self.llm, self.model_config) + llm_weights = {k: v for k, v in weights.items() if k.startswith("model.language_model.")} + self.llm.load_weights(llm_weights, llm_weight_mapper) + if hasattr(weights, "mark_consumed"): + weights.mark_consumed("model.language_model") diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3.py b/tensorrt_llm/_torch/models/modeling_minimaxm3.py index e38b9f8be115..5566a1f3051d 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3.py @@ -21,7 +21,7 @@ import copy import dataclasses -from functools import partial +import os from typing import Any, Dict, List, Optional, Tuple from typing import Mapping as TMapping @@ -35,14 +35,31 @@ from tensorrt_llm.models.modeling_utils import QuantConfig from ..attention_backend import AttentionMetadata -from ..attention_backend.interface import PositionalEmbeddingParams, RopeParams -from ..distributed import AllReduce, AllReduceParams, MiniMaxAllReduceRMS +from ..attention_backend.interface import ( + AttentionForwardArgs, + PositionalEmbeddingParams, + RopeParams, +) +from ..attention_backend.sparse.minimax_m3 import ( + MiniMaxM3MsaSparseAttention, + MiniMaxM3SparseRuntimeBackend, + _gather_paged_batched, + _write_main_kv_slots_to_pool, +) +from ..distributed import AllReduce, AllReduceFusionOp, AllReduceParams, MiniMaxAllReduceRMS from ..modules.attention import Attention from ..modules.decoder_layer import DecoderLayer from ..modules.embedding import Embedding from ..modules.fused_moe import MiniMaxM3MoeRoutingMethod, create_moe from ..modules.gated_mlp import GatedMLP -from ..modules.linear import Linear, TensorParallelMode, copy_weight, load_weight_shard +from ..modules.linear import ( + Linear, + TensorParallelMode, + WeightMode, + WeightsLoadingConfig, + copy_weight, + load_weight_shard, +) from ..modules.multi_stream_utils import maybe_execute_in_parallel from ..modules.rms_norm import RMSNorm from ..utils import ( @@ -52,7 +69,15 @@ get_model_extra_attrs, is_torch_compiling, ) -from .modeling_utils import DecoderModel, DecoderModelForCausalLM, ModelConfig, register_auto_model +from .checkpoints.base_weight_mapper import BaseWeightMapper +from .checkpoints.hf.minimaxm3_weight_mapper import MINIMAX_M3_PARAMS_MAP, MiniMaxM3HfWeightMapper +from .modeling_utils import ( + DecoderModel, + DecoderModelForCausalLM, + ModelConfig, + filter_weights, + register_auto_model, +) # Dense layers use SDPA with non-contiguous Q/K/V and a bool attn_mask. # Limit backends to memory-efficient and math; cuDNN SDPA fails for this layout, @@ -234,11 +259,18 @@ def _build_swiglu_oai_dense_mlp( # MoE composition) carries its own reduction unless ADP collapses # TP to 1. reduce_output = False if is_shared_expert else (not enable_adp) + # SwiGLU-OAI is plain SwiGLU with an alpha gain and an (up + 1) offset, so + # the fused silu_and_mul kernel runs it in one launch with an optional fp8 + # epilogue. Mirrors the routed-expert SwigluBias path; the math lives in + # _minimax_m3_swiglu_oai. return GatedMLP( hidden_size=config.hidden_size, intermediate_size=intermediate_size, bias=False, - activation=partial(_minimax_m3_swiglu_oai, alpha=swiglu_alpha, limit=swiglu_limit), + activation=torch.nn.functional.silu, + swiglu_alpha=swiglu_alpha, + swiglu_beta=1.0, + swiglu_limit=swiglu_limit, dtype=config.torch_dtype, config=model_config, overridden_tp_size=1 if enable_adp else None, @@ -617,7 +649,7 @@ def minimax_m3_attn_custom_op_inplace( """Run MiniMax-M3 cache and attention work behind a compile boundary.""" attn_metadata, attn_layer = _extract_minimax_m3_attention_extra_attrs(layer_idx) num_tokens = attn_metadata.num_tokens - attn_layer._attention_core( + attn_layer._dispatch_attention_backend( q[:num_tokens], k[:num_tokens], v[:num_tokens], @@ -633,12 +665,10 @@ class MiniMaxM3Attention(Attention): Both branches share the same dense GQA scaffolding (``qkv_proj`` + ``o_proj`` + per-head Gemma Q/K norm + partial RoPE). Sparse layers - additionally carry the MiniMax index branch (``index_q_proj``, - ``index_k_proj`` and their per-head norms). The index value/output - branch is omitted because the M3 checkpoint sets - ``sparse_disable_index_value=True`` on every sparse layer; if a - future config variant flips that flag the gate will catch the - unmapped keys. + additionally carry the MiniMax index branch: a fused index_qk_proj that + outputs [idx_q | idx_k], plus per-head index norms. The index value and + output branch is omitted because the M3 checkpoint sets + sparse_disable_index_value=True on every sparse layer. """ def __init__( @@ -705,33 +735,23 @@ def __init__( self.sparse_local_block = int(sparse_cfg.get("sparse_local_block", 1)) self.sparse_score_type = str(sparse_cfg.get("sparse_score_type", "max")) - # index_q_proj is **replicated** across TP ranks. The sparse - # forward reshapes idx_q to - # ``[num_tokens, sparse_num_index_heads, sparse_index_dim]``, - # which requires the rank-local idx_q to carry all heads. - index_q_total = self.sparse_num_index_heads * self.sparse_index_dim - self.index_q_proj = Linear( + # Index Q and K are both replicated and project the same + # hidden_states, so fuse them into one GEMM with output + # [idx_q | idx_k]. idx_q holds all index heads; idx_k is a single K + # per token, broadcast across heads when scoring. + self.index_q_size = self.sparse_num_index_heads * self.sparse_index_dim + self.index_k_size = self.sparse_index_dim + self.index_qk_proj = Linear( config.hidden_size, - index_q_total, - bias=False, - dtype=config.torch_dtype, - mapping=model_config.mapping, - tensor_parallel_mode=None, - quant_config=None, - skip_create_weights_in_init=model_config.skip_create_weights_in_init, - ) - # index_k_proj is also replicated across TP ranks and - # outputs ``sparse_index_dim`` channels — a single K per - # token (not per-head), broadcast across index heads when - # scoring blocks. - self.index_k_proj = Linear( - config.hidden_size, - self.sparse_index_dim, + self.index_q_size + self.index_k_size, bias=False, dtype=config.torch_dtype, mapping=model_config.mapping, tensor_parallel_mode=None, quant_config=None, + weights_loading_config=WeightsLoadingConfig( + weight_mode=WeightMode.FUSED_GATE_UP_LINEAR + ), skip_create_weights_in_init=model_config.skip_create_weights_in_init, ) # Per-head Gemma RMSNorm of width ``sparse_index_dim``; @@ -816,6 +836,7 @@ def forward( position_ids: Optional[torch.IntTensor] = None, hidden_states: Optional[torch.Tensor] = None, attn_metadata: Optional[AttentionMetadata] = None, + all_reduce_params: Optional[AllReduceParams] = None, **kwargs, ): """Dispatch sparse layers to the MiniMax-M3 sparse algorithm. @@ -835,14 +856,27 @@ def forward( side index-K buffer. """ if not self.is_sparse_attention_layer: - return self._dense_forward(position_ids, hidden_states, attn_metadata, **kwargs) - return self._sparse_forward(position_ids, hidden_states, attn_metadata, **kwargs) + return self._dense_forward( + position_ids, + hidden_states, + attn_metadata, + all_reduce_params=all_reduce_params, + **kwargs, + ) + return self._sparse_forward( + position_ids, + hidden_states, + attn_metadata, + all_reduce_params=all_reduce_params, + **kwargs, + ) def _dense_forward( self, position_ids: torch.IntTensor, hidden_states: torch.Tensor, attn_metadata: AttentionMetadata, + all_reduce_params: Optional[AllReduceParams] = None, **kwargs, ) -> torch.Tensor: """Dense MiniMax-M3 attention for layers 0-2. @@ -900,9 +934,13 @@ def _dense_forward( # torch.compile. Only the metadata/cache-dependent attention core is # hidden behind the inplace custom op. o = self._forward_attention_core(q, k, v, None, None, attn_metadata) - return self.o_proj(o) + # all_reduce_params lets the decoder defer the o_proj output AllReduce so + # it can be fused with post_attention_layernorm (RESIDUAL_RMS_NORM). + # Passing None preserves the standalone o_proj reduction used by the + # single-GPU, attention-DP, and fusion-disabled paths. + return self.o_proj(o, all_reduce_params=all_reduce_params) - def _dense_attention_core( + def _sdpa_dense_attention_core( self, q: torch.Tensor, k: torch.Tensor, @@ -910,12 +948,7 @@ def _dense_attention_core( attn_metadata: AttentionMetadata, output: torch.Tensor, ) -> torch.Tensor: - """Run dense cache updates and attention into ``output``.""" - from ..attention_backend.sparse.minimax_m3 import ( - _gather_paged_batched, - _write_main_kv_slots_to_pool, - ) - + """Run dense cache updates and attention into ``output`` (SDPA path).""" kv_cache_manager = getattr(attn_metadata, "kv_cache_manager", None) if kv_cache_manager is None: raise RuntimeError( @@ -1104,10 +1137,10 @@ def _forward_attention_core( output, ) else: - self._attention_core(q, k, v, idx_q, idx_k, attn_metadata, output) + self._dispatch_attention_backend(q, k, v, idx_q, idx_k, attn_metadata, output) return output - def _attention_core( + def _dispatch_attention_backend( self, q: torch.Tensor, k: torch.Tensor, @@ -1117,17 +1150,58 @@ def _attention_core( attn_metadata: AttentionMetadata, output: torch.Tensor, ) -> torch.Tensor: + """Route the attention core to the configured backend. + + self.attn is either a :class:`MiniMaxM3MsaSparseAttention`, which + handles both dense and sparse layers, or a + :class:`MiniMaxM3SparseRuntimeBackend` (Triton). + + * MSA → :meth:`_msa_attention_core` + * Triton sparse → :meth:`_triton_sparse_attention_core` + * SDPA dense → :meth:`_sdpa_dense_attention_core` + """ + if isinstance(self.attn, MiniMaxM3MsaSparseAttention): + return self._msa_attention_core(q, k, v, idx_q, idx_k, attn_metadata, output) if self.is_sparse_attention_layer: assert idx_q is not None and idx_k is not None - return self._sparse_attention_core(q, k, v, idx_q, idx_k, attn_metadata, output) + return self._triton_sparse_attention_core(q, k, v, idx_q, idx_k, attn_metadata, output) assert idx_q is None and idx_k is None - return self._dense_attention_core(q, k, v, attn_metadata, output) + return self._sdpa_dense_attention_core(q, k, v, attn_metadata, output) + + def _msa_attention_core( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + idx_q: Optional[torch.Tensor], + idx_k: Optional[torch.Tensor], + attn_metadata: AttentionMetadata, + output: torch.Tensor, + ) -> torch.Tensor: + """Run the MSA backend (:class:`MiniMaxM3MsaSparseAttention`). + + The backend runs the sparse GQA or dense paged GQA through its inherited + FMHA forward; this layer selects the top-k blocks (sparse only) and + builds the forward_args the FMHA reads. + """ + if self.is_sparse_attention_layer: + assert idx_q is not None and idx_k is not None + # Publish the selected blocks so the FMHA runs the sparse path. + kv_block_indexes = self.attn.run_indexer(idx_q, idx_k, attn_metadata) + forward_args = AttentionForwardArgs(output=output, topk_indices=kv_block_indexes) + else: + assert idx_q is None and idx_k is None + # No top-k selection means the FMHA attends the full page table. + forward_args = AttentionForwardArgs(output=output) + self.attn.forward(q, k, v, attn_metadata, forward_args=forward_args) + return output def _sparse_forward( self, position_ids: torch.IntTensor, hidden_states: torch.Tensor, attn_metadata: AttentionMetadata, + all_reduce_params: Optional[AllReduceParams] = None, **kwargs, ) -> torch.Tensor: """Run a MiniMax-M3 sparse attention forward end-to-end. @@ -1141,7 +1215,7 @@ def _sparse_forward( 4. Pull paged main K/V cache (reshaped to flat-slot view) and paged side index-K cache from the :class:`MiniMaxM3KVCacheManagerV2`. - 5. Build a :class:`MiniMaxM3SparseAttentionMetadata` from the + 5. Build a :class:`MiniMaxM3TritonSparseAttentionMetadata` from the standard :class:`AttentionMetadata` (using ``request_ids`` + ``seq_lens`` + ``num_cached_tokens_per_seq``). 6. Write the new token's K/V/idx_K to the slots named by the @@ -1154,7 +1228,7 @@ def _sparse_forward( Production callers (the LLM API path) drive :meth:`MiniMaxM3AttentionMetadata.prepare` outside any CUDA-graph capture window; that method attaches a pre-built - :class:`MiniMaxM3SparseAttentionMetadata` and an + :class:`MiniMaxM3TritonSparseAttentionMetadata` and an ``out_cache_loc`` tensor as ``attn_metadata.minimax_m3``. Test callers attach the same dict manually. This forward path always reads the pre-built attachment and never builds metadata @@ -1171,8 +1245,8 @@ def _sparse_forward( # 1. Projections. qkv = self.qkv_proj(hidden_states) q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) - idx_q = self.index_q_proj(hidden_states) - idx_k = self.index_k_proj(hidden_states) + idx_qk = self.index_qk_proj(hidden_states) + idx_q, idx_k = idx_qk.split([self.index_q_size, self.index_k_size], dim=-1) # 2. Per-head Gemma RMSNorm on both branches. q, k = self.apply_qk_norm(q, k) @@ -1186,9 +1260,13 @@ def _sparse_forward( idx_q, idx_k = self.rotary_emb(position_ids, [idx_q, idx_k]) o = self._forward_attention_core(q, k, v, idx_q, idx_k, attn_metadata) - return self.o_proj(o) + # all_reduce_params lets the decoder defer the o_proj output AllReduce so + # it can be fused with post_attention_layernorm (RESIDUAL_RMS_NORM). + # Passing None preserves the standalone o_proj reduction used by the + # single-GPU, attention-DP, and fusion-disabled paths. + return self.o_proj(o, all_reduce_params=all_reduce_params) - def _sparse_attention_core( + def _triton_sparse_attention_core( self, q: torch.Tensor, k: torch.Tensor, @@ -1198,7 +1276,7 @@ def _sparse_attention_core( attn_metadata: AttentionMetadata, output: torch.Tensor, ) -> torch.Tensor: - """Run sparse cache updates and attention into ``output``.""" + """Run sparse cache updates and attention into ``output`` (Triton path).""" kv_cache_manager = getattr(attn_metadata, "kv_cache_manager", None) if kv_cache_manager is None: raise RuntimeError( @@ -1268,10 +1346,7 @@ def _sparse_attention_core( # registers :class:`MiniMaxM3SparseRuntimeBackend` as # ``self.attn``; any other backend on a sparse layer is a # configuration error. - from ..attention_backend.sparse.minimax_m3 import get_minimax_m3_attention_backend_cls - - m3_backend_cls = get_minimax_m3_attention_backend_cls() - if not isinstance(self.attn, m3_backend_cls): + if not isinstance(self.attn, MiniMaxM3SparseRuntimeBackend): raise RuntimeError( f"MiniMax-M3 sparse forward (layer {self.layer_idx}) requires " f"self.attn to be a MiniMaxM3SparseRuntimeBackend, got " @@ -1312,6 +1387,7 @@ def __init__( self.hidden_size = config.hidden_size self.layer_idx = layer_idx self.mapping = model_config.mapping + self.enable_attention_dp = self.mapping.enable_attention_dp _, sparse_layer_ids = get_sparse_layer_ids(config) disable_index_value_ids = set(get_sparse_disable_index_value_layer_ids(config)) @@ -1342,19 +1418,55 @@ def __init__( ) self.block_sparse_moe = None + # Layer-boundary RMSNorms are plain (non-Gemma) norms so they can drive + # the fused AllReduce+residual+RMSNorm epilogue + # (AllReduceFusionOp.RESIDUAL_RMS_NORM), whose kernel applies a plain + # weight * x scaling with no Gemma (1 + weight) offset. When the + # checkpoint stores Gemma norms (use_gemma_norm=True), the loader folds + # (1 + weight) into the stored weight at load time (see + # _fold_gemma_boundary_norm_weights), so the runtime norm is numerically + # identical to the original Gemma norm on every path. The per-head + # q/k/index norms keep use_gemma because they are consumed by the + # separate fused_qk_norm_rope kernel, which handles Gemma directly. self.input_layernorm = RMSNorm( hidden_size=config.hidden_size, eps=config.rms_norm_eps, dtype=config.torch_dtype, - use_gemma=bool(getattr(config, "use_gemma_norm", False)), + use_gemma=False, ) self.post_attention_layernorm = RMSNorm( hidden_size=config.hidden_size, eps=config.rms_norm_eps, dtype=config.torch_dtype, - use_gemma=bool(getattr(config, "use_gemma_norm", False)), + use_gemma=False, ) + # DeepSeek-V3-style layer-boundary AllReduce fusion. Each layer folds + # the attention o_proj output AllReduce into post_attention_layernorm + # (PRE fusion) and the MoE/MLP output AllReduce into the next layer's + # input_layernorm (POST fusion, wired via next_layer_layernorm in + # setup_aliases). Fusion is only meaningful when there is a real + # cross-rank reduction to fold, i.e. TP>1 and not attention-DP (each DP + # rank owns independent tokens, so no attention/MoE AllReduce happens + # there). An env override matches the DeepSeek-V3 escape hatch. + self.enable_fusion = os.environ.get("TRTLLM_MINIMAX_M3_EAGER_FUSION_DISABLED", "0") == "0" + self.enable_fusion &= (not self.enable_attention_dp) and self.mapping.tp_size > 1 + self.pre_feed_forward_fusion = self.enable_fusion + self.post_feed_forward_fusion = self.enable_fusion + + self.allreduce = None + if not self.enable_attention_dp and self.mapping.tp_size > 1: + self.allreduce = AllReduce( + mapping=model_config.mapping, + strategy=model_config.allreduce_strategy, + dtype=config.torch_dtype, + ) + + # Wired by MiniMaxM3ForCausalLM.setup_aliases after weight load to the + # next layer's input_layernorm (or the final model norm for the last + # layer). None disables POST fusion and boundary-norm folding. + self.next_layer_layernorm: Optional[RMSNorm] = None + def forward( self, position_ids: torch.IntTensor, @@ -1363,24 +1475,131 @@ def forward( residual: Optional[torch.Tensor], **kwargs, ) -> torch.Tensor: + # Layer-0 prologue only. For every subsequent layer the input_layernorm + # (an add+RMSNorm at the layer boundary) was already applied by the + # previous layer as its next_layer_layernorm, so residual is not None + # here and this block is skipped (matches DeepSeek-V3). if residual is None: residual = hidden_states hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) + # When PRE fusion is active the attention defers its o_proj AllReduce so + # it can be fused into post_attention_layernorm below; otherwise the + # o_proj reduces as usual (all_reduce_params=None preserves the + # single-GPU, attention-DP, and fusion-disabled behavior exactly). + attn_all_reduce_params = ( + AllReduceParams(enable_allreduce=False) if self.pre_feed_forward_fusion else None + ) hidden_states = self.self_attn( position_ids=position_ids, hidden_states=hidden_states, attn_metadata=attn_metadata, + all_reduce_params=attn_all_reduce_params, **kwargs, ) - hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) if self.block_sparse_moe is not None: - hidden_states = self.block_sparse_moe(hidden_states, attn_metadata) + hidden_states, residual = self.forward_MoE(hidden_states, attn_metadata, residual) else: - hidden_states = self.mlp(hidden_states) + hidden_states, residual = self.forward_mlp(hidden_states, residual) + + return hidden_states, residual + + def _apply_pre_feed_forward_norm( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """AllReduce(+residual+RMSNorm) between attention and the feed-forward. + + On the PRE-fusion path the deferred attention o_proj AllReduce is fused + with post_attention_layernorm into one kernel; otherwise it is the plain + add+RMSNorm and the attention already reduced its own output. + """ + if self.pre_feed_forward_fusion: + return self.allreduce( + hidden_states, + all_reduce_params=AllReduceParams( + fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, + residual=residual, + norm_weight=self.post_attention_layernorm.weight, + eps=self.post_attention_layernorm.variance_epsilon, + trigger_completion_at_end=False, + ), + ) + return self.post_attention_layernorm(hidden_states, residual) + + def _apply_next_layer_layernorm( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Apply the next layer's input_layernorm at the layer boundary. + + On the POST-fusion path the deferred feed-forward output AllReduce is + fused with next_layer_layernorm (the next layer's input_layernorm, or + the final model norm for the last layer). Off the fusion path it is the + plain add+RMSNorm. When next_layer_layernorm has not been wired (e.g. a + standalone unit test that never ran setup_aliases) the + (hidden_states, residual) pair is returned unchanged so the model can + apply the final norm itself. + """ + if self.next_layer_layernorm is None: + return hidden_states, residual + if self.post_feed_forward_fusion: + return self.allreduce( + hidden_states, + all_reduce_params=AllReduceParams( + fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, + residual=residual, + norm_weight=self.next_layer_layernorm.weight, + eps=self.next_layer_layernorm.variance_epsilon, + trigger_completion_at_end=False, + ), + ) + return self.next_layer_layernorm(hidden_states, residual) + + def _feed_forward_all_reduce_params(self) -> Optional[AllReduceParams]: + """AllReduce params handed to the MoE or dense-MLP output projection. + + Disables the module's internal output AllReduce when POST fusion will + fold it into next_layer_layernorm; otherwise None preserves the module's + own reduction (single-GPU, attention-DP, fusion-disabled). + """ + if self.post_feed_forward_fusion: + return AllReduceParams(enable_allreduce=False) + return None + + def forward_MoE( + self, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + residual: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + hidden_states, residual = self._apply_pre_feed_forward_norm(hidden_states, residual) + + hidden_states = self.block_sparse_moe( + hidden_states, + attn_metadata, + final_all_reduce_params=self._feed_forward_all_reduce_params(), + ) + + hidden_states, residual = self._apply_next_layer_layernorm(hidden_states, residual) + return hidden_states, residual + + def forward_mlp( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + hidden_states, residual = self._apply_pre_feed_forward_norm(hidden_states, residual) + + hidden_states = self.mlp( + hidden_states, + final_all_reduce_params=self._feed_forward_all_reduce_params(), + ) + + hidden_states, residual = self._apply_next_layer_layernorm(hidden_states, residual) return hidden_states, residual @@ -1418,11 +1637,17 @@ def __init__(self, model_config: "ModelConfig[PretrainedConfig]"): for layer_idx in range(config.num_hidden_layers) ] ) + # Final norm is a plain (non-Gemma) RMSNorm for the same reason as the + # layer-boundary norms (see MiniMaxM3DecoderLayer.__init__): it doubles + # as the last layer's next_layer_layernorm, so the last MoE/MLP output + # AllReduce folds into it via RESIDUAL_RMS_NORM. The Gemma (1 + weight) + # offset is folded into the stored weight at load time (see + # _fold_gemma_boundary_norm_weights). self.norm = RMSNorm( hidden_size=config.hidden_size, eps=config.rms_norm_eps, dtype=config.torch_dtype, - use_gemma=bool(getattr(config, "use_gemma_norm", False)), + use_gemma=False, ) def forward( @@ -1449,22 +1674,80 @@ def forward( residual=residual, ) - hidden_states, _ = self.norm(hidden_states, residual) + # When setup_aliases has chained the final norm into the last decoder + # layer (next_layer_layernorm = self.norm), the last layer's boundary + # step already applied it (fused or plain), so hidden_states is normed + # and this is skipped. The fallback covers paths that never ran + # setup_aliases (e.g. standalone unit tests): there the last layer + # returns the unnormed (hidden_states, residual) pair and the final + # add+RMSNorm is applied here. + if self.layers[-1].next_layer_layernorm is None: + hidden_states, _ = self.norm(hidden_states, residual) return hidden_states -# HF MiniMax-M3 stores the routed score-correction bias one level above -# the router weight (``block_sparse_moe.e_score_correction_bias``, -# sibling of ``block_sparse_moe.gate.weight``). The TRT-LLM module tree -# binds it to :class:`MiniMaxM3Gate`, so the generic loader expects to -# see it at ``block_sparse_moe.gate.e_score_correction_bias``. The -# regex below moves the key into the gate's prefix before the loader -# dispatches; this lets ``mark_consumed("...gate")`` cleanly remove -# both the weight and the bias together without disturbing the sibling -# ``block_sparse_moe.experts.*`` backend subtree. -_M3_GATE_BIAS_RENAME_MAP = { - r"^(.*\.block_sparse_moe)\.e_score_correction_bias$": (r"\1.gate.e_score_correction_bias"), -} +def _load_index_qk_proj_weights(model: nn.Module, weights) -> None: + """Fuse checkpoint index_q_proj and index_k_proj into index_qk_proj. + + The shared weight loader fuses only qkv_proj and gate_up_proj by name, so + load the sibling checkpoint tensors into each fused index module through the + FUSED_GATE_UP_LINEAR row-concatenation path and mark the sources consumed. + """ + for name, module in model.named_modules(): + if name.split(".")[-1] != "index_qk_proj": + continue + parent = name.rsplit(".", 1)[0] + q_weights = filter_weights(f"{parent}.index_q_proj", weights) + k_weights = filter_weights(f"{parent}.index_k_proj", weights) + # Missing sources make Linear.load_weights assert rather than leave + # the fused module silently uninitialized. + module.load_weights(weights=[q_weights, k_weights]) + if hasattr(weights, "mark_consumed"): + weights.mark_consumed(f"{parent}.index_q_proj") + weights.mark_consumed(f"{parent}.index_k_proj") + else: + for key in list(weights.keys()): + if key.startswith(f"{parent}.index_q_proj.") or key.startswith( + f"{parent}.index_k_proj." + ): + del weights[key] + + +# Layer-boundary RMSNorms whose Gemma (1 + weight) scaling is folded into the +# stored weight at load time so the runtime norm is a plain RMSNorm (see +# MiniMaxM3DecoderLayer.__init__ / MiniMaxM3Model.__init__). These are exactly +# the norms that drive the DeepSeek-V3-style fused AllReduce+residual+RMSNorm +# epilogue, whose kernel has no Gemma offset. The per-head q/k/index norms are +# intentionally excluded: they feed the separate fused_qk_norm_rope kernel, +# which handles Gemma directly and stays use_gemma=True. +_M3_BOUNDARY_NORM_SUFFIXES = ( + ".input_layernorm.weight", + ".post_attention_layernorm.weight", +) +_M3_FINAL_NORM_KEY = "model.norm.weight" + + +def _fold_gemma_boundary_norm_weights(weights): + """Fold Gemma (1 + weight) into the layer-boundary RMSNorm weights. + + MiniMax-M3 stores every RMSNorm as a Gemma norm (use_gemma_norm=True), which + computes (1 + weight) * x. The layer-boundary norms are constructed as plain + norms (use_gemma=False, weight * x) so they can drive the fused + AllReduce+RMSNorm kernels, so their stored weights must be pre-incremented by + 1.0. This is a numerically exact, load-time-only rewrite; the resulting norm + is identical to the original Gemma norm on every path. + + Only the decoder input_layernorm / post_attention_layernorm and the final + model.norm are touched. A no-op for keys that are absent (partial load) so it + is safe to call unconditionally, but it must only run when the checkpoint + actually uses Gemma norms (guarded by the caller). + """ + for key in list(weights.keys()): + if key.endswith(_M3_BOUNDARY_NORM_SUFFIXES) or key == _M3_FINAL_NORM_KEY: + w = weights[key] + w = w[:] if hasattr(w, "__getitem__") else w + weights[key] = w + 1.0 + return weights @register_auto_model("MiniMaxM3SparseForCausalLM") @@ -1482,13 +1765,48 @@ def __init__(self, model_config: "ModelConfig[PretrainedConfig]"): vocab_size=model_config.pretrained_config.vocab_size, ) - def load_weights(self, weights, *args, **kwargs): - # Merge the M3-specific gate-bias rename into any caller- - # supplied ``params_map`` so the VL wrapper and any downstream - # tooling that already passes one keep working. - params_map = kwargs.pop("params_map", None) or {} - merged = {**_M3_GATE_BIAS_RENAME_MAP, **params_map} - return super().load_weights(weights, *args, params_map=merged, **kwargs) + def load_weights( + self, + weights: Dict, + weight_mapper: Optional[BaseWeightMapper] = None, + params_map: Optional[Dict[str, str]] = None, + allow_partial_loading: bool = False, + ) -> None: + # The generic loader has no rule for this fusion. The VL subclass routes + # its text weights through here, so both paths are covered. + _load_index_qk_proj_weights(self, weights) + # Fold Gemma (1 + weight) into the layer-boundary RMSNorm weights so the + # runtime norms can be plain (non-Gemma) and drive the fused + # AllReduce+residual+RMSNorm epilogue. Only when the checkpoint actually + # stores Gemma norms; otherwise the boundary norms are already plain. + if bool(getattr(self.config, "use_gemma_norm", False)): + weights = _fold_gemma_boundary_norm_weights(weights) + if weight_mapper is None: + weight_mapper = MiniMaxM3HfWeightMapper() + weight_mapper.init_model_and_config(self, self.model_config) + merged_params_map = {**MINIMAX_M3_PARAMS_MAP, **(params_map or {})} + super().load_weights( + weights=weights, + weight_mapper=weight_mapper, + params_map=merged_params_map, + allow_partial_loading=allow_partial_loading, + ) + + def setup_aliases(self) -> None: + """Chain each decoder layer's next_layer_layernorm for POST fusion. + + Wired after weight load (the generic loader skips next_layer_layernorm + aliases). Each layer's MoE/MLP output AllReduce is fused into the next + layer's input_layernorm; the last layer chains the final model norm so + its output AllReduce folds the final normalization too. + """ + layers = self.model.layers + num_layers = len(layers) + for idx, layer in enumerate(layers): + if idx == num_layers - 1: + layer.next_layer_layernorm = self.model.norm + else: + layer.next_layer_layernorm = layers[idx + 1].input_layernorm def _strip_language_model_prefix( @@ -1624,7 +1942,13 @@ def __init__(self, model_config: "ModelConfig[PretrainedConfig]"): self.last_loaded_vision_keys = [] self.last_missing_vision_keys = [] - def load_weights(self, weights, *args, **kwargs): + def load_weights( + self, + weights: Dict, + weight_mapper: Optional[BaseWeightMapper] = None, + params_map: Optional[Dict[str, str]] = None, + allow_partial_loading: bool = False, + ) -> None: text_cfg = self.config if is_minimax_m3_vl_config(text_cfg): text_cfg = get_text_config(text_cfg) @@ -1647,7 +1971,12 @@ def load_weights(self, weights, *args, **kwargs): self.last_loaded_vision_keys = loaded self.last_missing_vision_keys = missing - return super().load_weights(text_weights, *args, **kwargs) + super().load_weights( + weights=text_weights, + weight_mapper=weight_mapper, + params_map=params_map, + allow_partial_loading=allow_partial_loading, + ) def forward( self, diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py index fce64e1a561c..55ce331d176c 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py @@ -2071,9 +2071,14 @@ def get_minimax_m3_vl_input_processor_cls(): path. Re-type by dynamic subclassing here so the registered class inherits from the base. """ - from tensorrt_llm.inputs.registry import BaseMultimodalInputProcessor + from tensorrt_llm.inputs.registry import ( + BaseMultimodalDummyInputsBuilder, + BaseMultimodalInputProcessor, + ) - class _Registered(MiniMaxM3VLInputProcessor, BaseMultimodalInputProcessor): + class _Registered( + MiniMaxM3VLInputProcessor, BaseMultimodalInputProcessor, BaseMultimodalDummyInputsBuilder + ): pass _Registered.__name__ = "MiniMaxM3VLInputProcessor" diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py index 2246a73f8fdd..55c4633cc4bf 100644 --- a/tensorrt_llm/_torch/models/modeling_mistral.py +++ b/tensorrt_llm/_torch/models/modeling_mistral.py @@ -39,7 +39,7 @@ from tensorrt_llm._torch.modules.rms_norm import RMSNorm from tensorrt_llm._torch.speculative import SpecMetadata from tensorrt_llm._utils import nvtx_range -from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.functional import PositionEmbeddingType, RotaryScalingType from tensorrt_llm.inputs import (BaseMultimodalDummyInputsBuilder, BaseMultimodalInputProcessor, ContentFormat, ExtraProcessedInputs, @@ -47,6 +47,7 @@ MultimodalPlaceholderPlacement, TextPrompt, register_input_processor) from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY from tensorrt_llm.inputs.utils import encode_base64_image from tensorrt_llm.llmapi import SamplingParams from tensorrt_llm.logger import logger @@ -61,10 +62,8 @@ def __init__( ): config = model_config.pretrained_config rope_params = RopeParams.from_config(config) - rope_params_section = getattr(config, "rope_scaling", None) or getattr( - config, "rope_parameters", None) - rope_type = getattr(rope_params_section, "rope_type", None) - if rope_type == "yarn": + + if rope_params.scale_type == RotaryScalingType.yarn: pos_embd_params = PositionalEmbeddingParams( type=PositionEmbeddingType.yarn, rope=rope_params, @@ -256,6 +255,18 @@ def __init__( vocab_size=model_config.pretrained_config.vocab_size, ) + def load_weights(self, weights: Dict, weight_mapper=None, *args, **kwargs): + if weight_mapper and type(weight_mapper) is MistralWeightMapper: + weight_mapper.permute_qk(weights=weights, config=self.config) + super().load_weights(weights, + weight_mapper=weight_mapper, + params_map=weight_mapper.mistral_llm_mapping) + else: + super().load_weights(weights, + weight_mapper=weight_mapper, + *args, + **kwargs) + class MistralCommonImageProcessor: @@ -311,18 +322,10 @@ def get_num_tokens_per_image(self, image_size): return ncols * nrows + nrows def __call__(self, text, images=None, **kwargs): - if not images: - # Plain-text inputs (e.g. text-only evaluation like MMLU/GSM8K): tokenize - # directly without wrapping in a multi-modal chat conversation, which would - # otherwise inject chat-template tokens and corrupt continuation prompts. - encoded = self.tokenizer.transformers_tokenizer(text, - return_tensors='pt') - return {"input_ids": encoded["input_ids"]} - mm_items = [{ "type": "image", "base64": encode_base64_image(image) - } for image in images] + } for image in (images or [])] conversation = [{ "role": "user", @@ -352,18 +355,16 @@ def __call__(self, text, images=None, **kwargs): return processed -class Mistral3InputProcessor(BaseMultimodalInputProcessor, - BaseMultimodalDummyInputsBuilder): +class MistralHFInputProcessor(BaseMultimodalInputProcessor, + BaseMultimodalDummyInputsBuilder): + """Input processor for Mistral VLM checkpoints in HuggingFace format.""" - def __init__( - self, - model_path: str, - config: PretrainedConfig, - tokenizer: AutoTokenizer | None, - trust_remote_code: bool = False, - model_type: str = "mistral3", - **kwargs, - ): + def __init__(self, + model_path: str, + config: PretrainedConfig, + tokenizer: AutoTokenizer, + trust_remote_code: bool = True, + **kwargs): super().__init__(model_path=model_path, config=config, tokenizer=tokenizer, @@ -371,27 +372,19 @@ def __init__( **kwargs) self._config = config self._dtype = self._config.torch_dtype - self._tokenizer = tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained( - model_path, - config=config, - use_fast=self.use_fast, - trust_remote_code=trust_remote_code) self._model_path = model_path - auto_processor = AutoProcessor.from_pretrained( + self._tokenizer = (tokenizer if tokenizer is not None else + AutoTokenizer.from_pretrained( + model_path, + config=config, + use_fast=True, + trust_remote_code=True)) + self._processor = AutoProcessor.from_pretrained( model_path, use_fast=self.use_fast, trust_remote_code=trust_remote_code) - if model_type == "mistral_large_3": - # For mistral large 3, we add chat template in the model forward, and the - # MistralCommonImageProcessor is used to process the input when both text and images are provided. - # When the input only contains text, we use the text processor to process the input. - self._processor = MistralCommonImageProcessor( - tokenizer=self._tokenizer, dtype=self.dtype) - self.text_processor = auto_processor - else: - # For other mistral models, we use the AutoProcessor to process the input. - self._processor = auto_processor - self.text_processor = self._processor + logger.info(f"[mistral] HF processor={type(self._processor).__name__} " + f"tokenizer={type(self._tokenizer).__name__}") @property def config(self) -> PretrainedConfig: @@ -425,17 +418,10 @@ def call_with_text_prompt( # format is "pt" (pytorch tensors), but not for "pil" (PIL images). do_rescale = False - if images is not None: - processed = self.processor( - text=inputs["prompt"], - images=images, - do_rescale=do_rescale, - ) - else: - processed = self.text_processor( - text=inputs["prompt"], - do_rescale=do_rescale, - ) + prompt = inputs["prompt"] + processed = self.processor(text=prompt, + images=images, + do_rescale=do_rescale) input_ids = processed.pop("input_ids").tolist()[0] # Remaining in `processed`: # * "attention_mask": [B, num_input_tokens] @@ -576,7 +562,6 @@ def get_vocab_size(self) -> int: return self.config.text_config.vocab_size def get_mm_token_ids(self) -> torch.Tensor: - """Get the IDs of all multimodal tokens (placeholders and special tokens alike).""" return torch.tensor([ # This is the `[IMG]` token id inserted into the prompt that should be replaced with image # embeddings. @@ -588,73 +573,120 @@ def get_mm_token_ids(self) -> torch.Tensor: ]) def get_mm_special_token_ids(self) -> torch.Tensor: - """Get the IDs of special multimodal tokens (placeholders not included).""" return torch.tensor([ self.processor.image_break_token_id, self.processor.image_end_token_id, ]) -class MistralCommonInputProcessor(Mistral3InputProcessor): +class MistralNativeInputProcessor(BaseMultimodalInputProcessor, + BaseMultimodalDummyInputsBuilder): + """Input processor for Mistral VLM checkpoints in mistral-native format.""" def __init__( self, model_path: str, config: PretrainedConfig, - tokenizer: AutoTokenizer, + tokenizer: AutoTokenizer | None, trust_remote_code: bool = False, **kwargs, ): - tokenizer = self.load_tokenizer(model_path, - config=config, - tokenizer=tokenizer) super().__init__(model_path=model_path, config=config, tokenizer=tokenizer, trust_remote_code=trust_remote_code, - model_type=getattr(config, "input_processor_type", - "mistral3"), **kwargs) + self._config = config + self._dtype = self._config.torch_dtype + self._model_path = model_path + self._tokenizer = MistralTokenizer.from_pretrained(model_path) + self._processor = MistralCommonImageProcessor(tokenizer=self._tokenizer, + dtype=self.dtype) + logger.info( + f"[mistral] native processor={type(self._processor).__name__} " + f"tokenizer={type(self._tokenizer).__name__}") - @staticmethod - def load_tokenizer(model_path: str, - config: PretrainedConfig, - tokenizer: AutoTokenizer | None = None): - if getattr(config, "input_processor_type", None) == "mistral_large_3": - try: - return MistralTokenizer.from_pretrained(model_path) + @property + def config(self) -> PretrainedConfig: + return self._config - except ValueError: - logger.info( - f"Could not load mistral-common tokenizer from {model_path}, falling back to HuggingFace" - ) + @property + def tokenizer(self) -> AutoTokenizer: + return self._tokenizer - tokenizer = tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained( - model_path, config=config, use_fast=True, trust_remote_code=True) - return tokenizer + @property + def model_path(self) -> str: + return self._model_path + @property + def processor(self) -> AutoProcessor: + return self._processor -@register_auto_model("Mistral3ForConditionalGeneration") -@register_auto_model("PixtralForConditionalGeneration") -@register_input_processor( - MistralCommonInputProcessor, - model_type="mistral_large_3", - placeholder_metadata=MultimodalPlaceholderMetadata( - placeholder_map={ - # NOTE: mistral-common uses the tokenizer to set placeholders, this will be ignored - "image": "[IMG]", - }, + @property + def dtype(self) -> torch.dtype: + return self._dtype + + def get_vocab_size(self) -> int: + return self.config.text_config.vocab_size + + def get_mm_token_ids(self) -> torch.Tensor: + return torch.tensor([ + self.processor.image_token_id, + self.processor.image_break_token_id, + self.processor.image_end_token_id, + ]) + + def get_mm_special_token_ids(self) -> torch.Tensor: + return torch.tensor([ + self.processor.image_break_token_id, + self.processor.image_end_token_id, + ]) + + @torch.inference_mode() + def call_with_text_prompt( + self, inputs: TextPrompt, sampling_params: SamplingParams + ) -> Tuple[List[int], ExtraProcessedInputs | None]: + images = inputs.get("multi_modal_data", {}).get("image") + if not images: + # Text-only: tokenize directly without wrapping in a chat template. + # The chat template is either already applied by the caller (serve + # path) or intentionally absent (e.g. raw few-shot eval like MMLU). + input_ids = self.tokenizer.transformers_tokenizer.encode( + inputs["prompt"]) + return input_ids, None + # Multimodal: MistralCommonImageProcessor builds the full conversation + # and applies the mistral-common chat template with image tokens. + processed = self.processor(text=inputs["prompt"], images=images) + input_ids = processed.pop("input_ids").tolist()[0] + processed.pop("attention_mask", None) + processed["image_sizes"] = processed["image_sizes"].tolist() + return input_ids, {"multimodal_data": {"image": {**processed}}} + + +# Register the native processor's content-format metadata. We do this +# directly rather than via @register_input_processor because that decorator +# also writes to INPUT_PROCESSOR_REGISTRY (keyed by model class), which would +# overwrite the MistralHFInputProcessor entry for Mistral3VLM. +# Mistral is the only supported case where HF preprocessor can be used with +# non-HF checkpoints, so this hack is preferred to changing the registry itself. +MULTIMODAL_PLACEHOLDER_REGISTRY.set_placeholder_metadata( + "mistral_common", + MultimodalPlaceholderMetadata( + placeholder_map={"image": "[IMG]"}, placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT, content_format=ContentFormat.PASSTHROUGH, )) +MistralNativeInputProcessor._registered_model_type = "mistral_common" + + +@register_auto_model("Mistral3ForConditionalGeneration") +@register_auto_model("PixtralForConditionalGeneration") @register_input_processor( - MistralCommonInputProcessor, + MistralHFInputProcessor, model_type="mistral3", placeholder_metadata=MultimodalPlaceholderMetadata( - placeholder_map={ - "image": "[IMG]", - }, - # NOTE: for mistral3 multimodal models, it does not strictly have to be before the text. + placeholder_map={"image": "[IMG]"}, + # NOTE: for mistral3 multimodal models, placeholder_placement does not strictly have to be before the text. # Ref: https://github.com/mistralai/mistral-common/blob/039465db2bdc0486df36365c9bdb428188482a18/ # src/mistral_common/tokens/tokenizers/base.py#L326 # However, accuracy tests show that the model generates higher quality output when the image @@ -662,6 +694,15 @@ def load_tokenizer(model_path: str, placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT, content_format=ContentFormat.STRING, )) +@register_input_processor( + MistralHFInputProcessor, + model_type="mistral_large_3", + placeholder_metadata=MultimodalPlaceholderMetadata( + # NOTE: mistral-common uses the tokenizer to set placeholders, this will be ignored + placeholder_map={"image": "[IMG]"}, + placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT, + content_format=ContentFormat.STRING, + )) class Mistral3VLM(MultimodalModelMixin, PreTrainedModel): """Mistral3VLM implementation for TRTLLM. @@ -743,11 +784,7 @@ def load_weights(self, weights: Dict, weight_mapper=None, *args, **kwargs): llm_weights = filter_weights(weights=weights, prefix="language_model") logger.debug(f"Loading weights for {type(self.llm)}") if weight_mapper and type(weight_mapper) is MistralWeightMapper: - weight_mapper.permute_qk(weights=llm_weights, - config=self.llm.config) - self.llm.load_weights(llm_weights, - weight_mapper=weight_mapper, - params_map=weight_mapper.mistral_llm_mapping) + self.llm.load_weights(llm_weights, weight_mapper=weight_mapper) else: self.llm.load_weights(llm_weights) logger.debug(f"Successfully loaded weights for {type(self.llm)}") @@ -807,17 +844,6 @@ def draft_config(self): def draft_model(self): return self.llm.draft_model - @property - def load_draft_weights(self): - return self.llm.load_draft_weights - - @property - def vocab_size_padded(self) -> int: - return self.llm.vocab_size_padded - - def infer_max_seq_len(self) -> int: - return self.llm.infer_max_seq_len() - def encode_multimodal_inputs( self, multimodal_params: Sequence[MultimodalParams], @@ -825,81 +851,24 @@ def encode_multimodal_inputs( mm_embeds = self._vision_forward(list(multimodal_params)) return mm_embeds[0] - def get_language_model_forward_kwargs( + def get_language_model_extra_forward_kwargs( self, *, - attn_metadata: AttentionMetadata, - input_ids: torch.Tensor | None, + raw_input_ids: torch.Tensor | None, position_ids: torch.Tensor | None, - inputs_embeds: torch.Tensor | None, mm_inputs: PreparedLlmInputs, - return_context_logits: bool, spec_metadata: SpecMetadata | None, - resource_manager: Any | None, + resource_manager: Any | None = None, + lora_params: Any | None = None, + **forward_kwargs: Any, ) -> dict[str, Any]: + del raw_input_ids, position_ids, mm_inputs, forward_kwargs return { - "attn_metadata": attn_metadata, - "input_ids": input_ids, - "position_ids": position_ids, - "inputs_embeds": inputs_embeds, - "return_context_logits": return_context_logits, "spec_metadata": spec_metadata, "resource_manager": resource_manager, + "lora_params": lora_params, } - @torch.inference_mode() - def forward( - self, - attn_metadata: AttentionMetadata, - input_ids: torch.LongTensor | None = None, - position_ids: torch.LongTensor | None = None, - inputs_embeds: torch.FloatTensor | None = None, - return_context_logits: bool = False, - spec_metadata: SpecMetadata | None = None, - **kwargs, - ) -> torch.Tensor: - """Forward method.""" - num_context_requests = attn_metadata.num_contexts - # multimodal_params is consumed by prepare_multimodal_inputs; remove it - # from passthrough kwargs to avoid rebinding it via **kwargs. - multimodal_params = kwargs.pop("multimodal_params", []) - - mm_inputs = self.prepare_multimodal_inputs( - input_ids=input_ids, - positions=position_ids, - multimodal_params=multimodal_params, - num_context_requests=num_context_requests, - attn_metadata=attn_metadata, - **kwargs, - ) - if inputs_embeds is not None: - if mm_inputs.inputs_embeds is not None: - # The caller supplied pre-computed inputs_embeds while the - # multimodal pipeline also produced fused embeds. Refuse to - # silently drop one or the other; let the caller resolve it. - raise ValueError( - "Mistral3VLM.forward received both caller-supplied inputs_embeds " - "and multimodal-derived inputs_embeds. These paths are mutually " - "exclusive; pass at most one.") - mm_inputs = PreparedLlmInputs( - input_ids=None, - inputs_embeds=inputs_embeds, - extra_embeds=mm_inputs.extra_embeds, - ) - - llm_kwargs = self.get_language_model_forward_kwargs( - attn_metadata=attn_metadata, - input_ids=mm_inputs.input_ids, - position_ids=position_ids, - inputs_embeds=mm_inputs.inputs_embeds, - mm_inputs=mm_inputs, - return_context_logits=return_context_logits, - spec_metadata=spec_metadata, - resource_manager=kwargs.get("resource_manager"), - ) - - return self.language_model.forward(**llm_kwargs) - @staticmethod def _get_sub_model_config( model_config: ModelConfig[MistralConfig], diff --git a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py index b1a2c5df6845..bec4e802c16e 100644 --- a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py +++ b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py @@ -15,17 +15,23 @@ # SPDX-License-Identifier: Apache-2.0 import contextlib +import copy +import itertools from dataclasses import dataclass from typing import ( TYPE_CHECKING, Any, + Callable, ClassVar, Dict, Hashable, Iterable, Iterator, + List, Optional, Sequence, + Tuple, + Union, ) import torch @@ -33,16 +39,183 @@ from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.tensor_lru_cache import TensorLRUCache from tensorrt_llm._utils import prefer_pinned -from tensorrt_llm.inputs.multimodal import MultimodalParams, MultimodalRuntimeData +from tensorrt_llm.inputs.multimodal import MultimodalInput, MultimodalParams, MultimodalRuntimeData from tensorrt_llm.logger import logger from .modeling_multimodal_utils import ( - _cache_multimodal_embeddings, + _store_chunked_prefill_embeddings, find_input_mm_embeds, fuse_input_embeds, get_multimodal_embeddings, ) + +@dataclass(frozen=True) +class EncoderGroup: + """Modalities that share a single encoder call. + + Batching all items in a group into one encoder invocation amortizes + fixed costs (kernel launches, dispatch) across items. The framework + splits the output back per-modality and reorders it into prompt order + via each request's `mm_item_order` manifest. + + Contract between `build_batched_input` and `encoder_fn`: + + * `build_batched_input` must concatenate items across requests in + `modalities` order (all items for the first modality across requests, + then all items for the second modality, etc.). + * Within a modality, items must appear in the same per-request iteration + order that `_lengths_by_modality` uses (i.e. the order of + `multimodal_params` passed in). + * `encoder_fn` must return one tensor whose rows correspond 1:1 to the + input layout produced by `build_batched_input`, so the framework can + split the output by `_lengths_by_modality` and reorder into prompt + order via each request's `mm_item_order` manifest. + """ + + modalities: Tuple[str, ...] + """Ordered modality names that share this encoder. Defines the row + layout of the encoder output tensor: first all items of + `modalities[0]`, then all items of `modalities[1]`, etc.""" + + encoder_fn: Callable[..., torch.Tensor] + """Encoder call invoked as `encoder_fn(**build_batched_input(params))`. + Returns a single tensor with one row per embedding, laid out per the + contract above.""" + + build_batched_input: Callable[[List[MultimodalParams]], Dict[str, Any]] + """Builds the kwargs dict passed to `encoder_fn`. Responsible for + concatenating raw per-item tensors from `multimodal_data` across + requests in the order described in the class docstring.""" + + +def _lengths_by_modality( + multimodal_params: List[MultimodalParams], + modalities: Tuple[str, ...], +) -> Dict[str, List[int]]: + """Invert prompt-ordered `multimodal_embedding_lengths` (number of + embedding rows per item) into per-modality per-item lengths, matching + the per-modality item order used by `EncoderGroup.build_batched_input`. + """ + by_modality: Dict[str, List[int]] = {m: [] for m in modalities} + for mp in multimodal_params: + flat = mp.multimodal_data.get("multimodal_embedding_lengths") or [] + if mp.mm_item_order: + for entry, length in zip(mp.mm_item_order, flat, strict=True): + if entry["modality"] in by_modality: + by_modality[entry["modality"]].append(length) + continue + # Raw-prompt entrypoints (non chat-parsing) do not attach a manifest, + # so this is the single enforcement point that a >1-modality request + # must carry `mm_item_order` to make prompt-order reordering possible. + present = [m for m in modalities if mp.multimodal_data.get(m) is not None] + if len(present) > 1: + raise ValueError( + "Request with multiple modalities present " + f"({present}) must carry mm_item_order on MultimodalParams." + ) + if present: + by_modality[present[0]].extend(flat) + return by_modality + + +def _reorder_embeds_by_manifest( + multimodal_params: List[MultimodalParams], + per_modality_embeds: Dict[str, torch.Tensor], + per_modality_lengths: Dict[str, List[int]], +) -> torch.Tensor: + """Slice per-modality tensors item-by-item and concat in prompt order.""" + per_modality_row_starts: Dict[str, List[int]] = { + m: list(itertools.accumulate(lens, initial=0)) for m, lens in per_modality_lengths.items() + } + + slices: List[torch.Tensor] = [] + # `entry["index"]` is per-request per-modality; advance a cursor to + # translate it into a global item index within `per_modality_embeds`. + per_modality_cursor: Dict[str, int] = {m: 0 for m in per_modality_embeds} + for mp in multimodal_params: + manifest = mp.mm_item_order or _synthesize_single_modality_manifest( + mp, per_modality_embeds.keys() + ) + req_counts: Dict[str, int] = {} + for entry in manifest: + m = entry["modality"] + if m not in per_modality_embeds: + continue + i = per_modality_cursor[m] + entry["index"] + starts = per_modality_row_starts[m] + slices.append(per_modality_embeds[m][starts[i] : starts[i + 1]]) + req_counts[m] = req_counts.get(m, 0) + 1 + for m, c in req_counts.items(): + per_modality_cursor[m] += c + if not slices: + # No items resolved for any request. This happens on the executor's + # KV-cache profiling pass: `_encode_dummy_inputs` runs the encoder on a + # worst-case dummy batch that carries the encoder tensors but no + # `multimodal_embedding_lengths`, so the per-modality lengths (and thus + # the sliced `per_modality_embeds`) come back empty. The encoder forward + # still ran (its activation is what peak-memory profiling captures), so + # return a correctly-typed empty embedding tensor instead of crashing on + # `torch.cat([])`. `per_modality_embeds` values are already zero-row + # slices of the encoder output, so their concat preserves dtype/device + # and the hidden dim. + if per_modality_embeds: + return torch.cat(list(per_modality_embeds.values()), dim=0) + return torch.empty(0) + return torch.cat(slices, dim=0) + + +def _synthesize_single_modality_manifest( + mp: MultimodalParams, + modalities: Iterable[str], +) -> List[Dict[str, Union[str, int]]]: + """Trivial manifest for requests with only one modality present.""" + flat = mp.multimodal_data.get("multimodal_embedding_lengths") or [] + for m in modalities: + if mp.multimodal_data.get(m) is not None: + return [{"modality": m, "index": i} for i in range(len(flat))] + return [] + + +def encode_multimodal_by_groups( + mm_encoder_groups: Sequence["EncoderGroup"], + multimodal_params: List[MultimodalParams], +) -> torch.Tensor: + """Run each group's encoder over its batched items and reorder into + per-request prompt order. + + For each group present in the batch, one encoder call is issued over all + items across all requests belonging to that group's modalities + (arithmetic-intensity win). The output is split back per-modality using + the prompt-ordered `multimodal_embedding_lengths` already stashed on + `multimodal_data`, then reordered into each request's `mm_item_order` + prompt sequence. + + Shared entry point for both the aggregated (`MultimodalModelMixin`) and + mm-encoder-only (`Qwen3VisionModelBase.forward`) paths so the ordering + contract lives in one place. + """ + per_modality_embeds: Dict[str, torch.Tensor] = {} + per_modality_lengths: Dict[str, List[int]] = {} + for group in mm_encoder_groups: + group_params = [ + mp + for mp in multimodal_params + if any(mp.multimodal_data.get(m) is not None for m in group.modalities) + ] + if not group_params: + continue + out = group.encoder_fn(**group.build_batched_input(group_params)) + lengths = _lengths_by_modality(group_params, group.modalities) + cursor = 0 + for m in group.modalities: + total = sum(lengths[m]) + per_modality_embeds[m] = out[cursor : cursor + total] + cursor += total + per_modality_lengths.update(lengths) + return _reorder_embeds_by_manifest(multimodal_params, per_modality_embeds, per_modality_lengths) + + if TYPE_CHECKING: from ..pyexecutor.llm_request import LlmRequest @@ -52,6 +225,26 @@ _MM_ENCODER_CACHE_LOG_NAME = "mm_encoder_cache" +def _build_request_multimodal_input( + request: "LlmRequest", cache_enabled: bool +) -> Optional[MultimodalInput]: + """Build the encoder-cache key metadata carried by one request.""" + # Skip construction (and `from_components` validation) when no persistent cache consumes it. + if not cache_enabled or request.multimodal_hashes is None: + return None + # `MultimodalModelMixin._encoder_cache_keys` uses UUID-aware multimodal hashes internally. + # Although the UUIDs are not exposed as an attribute, they remain in the backing C++ request + # for KV-cache block keys and cache events. + return MultimodalInput.from_components( + request.multimodal_hashes, + request.multimodal_positions, + request.multimodal_lengths, + mm_item_run_cu_offsets=request.multimodal_item_run_cu_offsets, + mm_run_positions=request.multimodal_run_positions, + mm_run_lengths=request.multimodal_run_lengths, + ) + + def _get_mm_aux_stream(max_prefetch_ahead: int = 0) -> Optional[torch.cuda.Stream]: """Return the side CUDA stream used for multimodal encoder prefetch. @@ -115,6 +308,28 @@ class PreparedLlmInputs: extra_embeds: Sequence[torch.Tensor] = () +@dataclass(frozen=True) +class EncoderCachePartition: + """Per-item cache partition for a single `MultimodalParams`. + + `hits` maps item index to its cached embedding row-block; `miss_indices` lists item + indices that still require encoder work; `keys` is aligned to item order so miss + embeddings can be written back after they are computed. + """ + + hits: Dict[int, torch.Tensor] + miss_indices: list[int] + keys: list[Hashable] + + @property + def is_full_hit(self) -> bool: + return bool(self.keys) and not self.miss_indices + + @property + def is_full_miss(self) -> bool: + return bool(self.keys) and not self.hits + + class MultimodalModelMixin: """Template-method mixin for PyTorch multimodal causal LM models. @@ -124,10 +339,11 @@ class MultimodalModelMixin: Current limitations: * For the time being, the persistent multimodal encoder cache stores per-item embeddings for - single-modality `MultimodalParams` objects. - * Cache reuse is all-or-nothing for each `MultimodalParams` object: every item in that object - hit the cache before cached embeddings are reused. Mixed-modality `MultimodalParams` objects - bypass the persistent cache. + single-modality `MultimodalParams` objects. Mixed-modality objects bypass the cache. + * A partially cached `MultimodalParams` is handled by encoding only its miss items + and interleaving cached items back in original per-item order. The default + `build_multimodal_encoder_input` handles stacked-on-dim-0 and packed-with-grid-thw + layouts; models with other layouts override that method. """ supports_encoder_cache: ClassVar[bool] = False @@ -153,6 +369,14 @@ def convert(tensor: torch.Tensor) -> torch.Tensor: return module._apply(convert) + # Per-model registration of encoder-batching groups. Each `EncoderGroup` + # bundles a set of modalities that share one encoder call. Set as a class + # attribute or on `self` in `__init__` (when `encoder_fn` binds to instance + # methods). Consumers call the module-level `encode_multimodal_by_groups` + # with these groups; both the aggregated and mm-encoder-only paths share + # that helper so the ordering contract lives in one place. + mm_encoder_groups: Sequence[EncoderGroup] = () + def encode_multimodal_inputs( self, multimodal_params: Sequence[MultimodalParams], @@ -173,7 +397,7 @@ def multimodal_token_ids(self) -> Optional[Sequence[int] | torch.Tensor]: by multimodal embeddings. Return `None` to use the out-of-vocabulary sentinel behavior in `fuse_input_embeds`. """ - raise NotImplementedError + return None @property def text_embedding_layer(self): @@ -190,6 +414,25 @@ def embedding_dtype(self) -> torch.dtype: """Return the dtype of each cached multimodal embedding row.""" raise NotImplementedError + @property + def encoder_cache_active(self) -> bool: + """Whether the persistent encoder cache is active for this model. + + Single source of truth shared by: + + * the in-iter consume path + * the side-stream prefetch dispatch + * the engine's cache-related gate + * the KV-cache memory reservation. + + The cache is only active when the model opts in via `supports_encoder_cache` and configures + a positive capacity. + """ + if not self.supports_encoder_cache: + return False + multimodal_config = self.model_config.multimodal_config + return multimodal_config is not None and multimodal_config.encoder_cache_max_bytes > 0 + def select_multimodal_params( self, multimodal_params: Sequence[MultimodalParams], @@ -197,16 +440,126 @@ def select_multimodal_params( ) -> Sequence[MultimodalParams]: """Select the params that participate in multimodal encoder work. - Returns the context-slice params with multimodal content. Helpers below + Returns the context-slice params with active multimodal content. Helpers below this method (`get_multimodal_embeddings`, `find_input_mm_embeds`, `fuse_input_embeds`) operate on the returned list and therefore see only `has_content()` params. Models overriding this hook must preserve that invariant. """ return [ - param for param in list(multimodal_params)[:num_context_requests] if param.has_content() + param + for param in list(multimodal_params)[:num_context_requests] + if param.has_content() + and ( + param.multimodal_runtime is None + or param.multimodal_runtime.num_mm_tokens_in_chunk != 0 + ) ] + @property + def language_model(self) -> torch.nn.Module: + """Return the inner language model that receives prepared inputs.""" + raise NotImplementedError + + @property + def vocab_size_padded(self) -> int: + """Return the inner language model's padded vocabulary size.""" + return self.language_model.vocab_size_padded + + def infer_max_seq_len(self) -> int: + """Return the inner language model's maximum sequence length.""" + return self.language_model.infer_max_seq_len() + + def get_language_model_extra_forward_kwargs( + self, + *, + raw_input_ids: Optional[torch.Tensor], + position_ids: Optional[torch.Tensor], + mm_inputs: PreparedLlmInputs, + **forward_kwargs: Any, + ) -> dict[str, Any]: + """Return model-specific arguments for the inner language-model forward.""" + return {} + + def get_language_model_forward_kwargs( + self, + *, + attn_metadata: Any, + input_ids: Optional[torch.Tensor], + raw_input_ids: Optional[torch.Tensor], + position_ids: Optional[torch.Tensor], + inputs_embeds: Optional[torch.Tensor], + mm_inputs: PreparedLlmInputs, + return_context_logits: bool, + **forward_kwargs: Any, + ) -> dict[str, Any]: + """Build common and model-specific inner language-model forward arguments.""" + llm_kwargs = { + "attn_metadata": attn_metadata, + "input_ids": input_ids, + "position_ids": position_ids, + "inputs_embeds": inputs_embeds, + "return_context_logits": return_context_logits, + } + llm_kwargs.update( + self.get_language_model_extra_forward_kwargs( + raw_input_ids=raw_input_ids, + position_ids=position_ids, + mm_inputs=mm_inputs, + **forward_kwargs, + ) + ) + return llm_kwargs + + @torch.inference_mode() + def forward( + self, + attn_metadata: Any, + input_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + return_context_logits: bool = False, + spec_metadata: Any = None, + **kwargs: Any, + ) -> torch.Tensor: + """Prepare multimodal inputs and dispatch them to the language model.""" + multimodal_params = kwargs.pop("multimodal_params", []) + mm_inputs = self.prepare_multimodal_inputs( + input_ids=input_ids, + positions=position_ids, + multimodal_params=multimodal_params, + num_context_requests=attn_metadata.num_contexts, + attn_metadata=attn_metadata, + **kwargs, + ) + if inputs_embeds is not None: + if mm_inputs.inputs_embeds is not None: + raise ValueError( + "MultimodalModelMixin.forward received both caller-supplied inputs_embeds " + "and multimodal-derived inputs_embeds. These paths are mutually exclusive; " + "pass at most one." + ) + mm_inputs = PreparedLlmInputs( + input_ids=None, + inputs_embeds=inputs_embeds, + extra_embeds=mm_inputs.extra_embeds, + ) + + llm_kwargs = self.get_language_model_forward_kwargs( + attn_metadata=attn_metadata, + input_ids=mm_inputs.input_ids, + raw_input_ids=input_ids, + position_ids=position_ids, + inputs_embeds=mm_inputs.inputs_embeds, + mm_inputs=mm_inputs, + return_context_logits=return_context_logits, + multimodal_params=multimodal_params, + num_generation_requests=attn_metadata.num_generations, + spec_metadata=spec_metadata, + **kwargs, + ) + return self.language_model.forward(**llm_kwargs) + def after_full_multimodal_embeddings( self, *, @@ -240,6 +593,147 @@ def after_active_multimodal_embeddings( # them as extra embeds without changing the base flow. return active_embeddings, () + def build_multimodal_encoder_input( + self, + param: MultimodalParams, + item_indices: Sequence[int], + ) -> MultimodalParams: + """Return a `MultimodalParams` whose raw modality inputs contain only + `item_indices` from `param`, in that order. + + Default handles three common single-modality layouts: + + - Image, stacked on dim 0 (Mistral 3 / Pixtral / LLaVA-family): `pixel_values` + `[B, C, H, W]` with a parallel `image_sizes` list; both sliced by item. + - Image / video, packed with `*_grid_thw` offsets (Qwen2-VL family): + `pixel_values` `[total_patches, feat]` + `image_grid_thw` `[B, 3]`; + prefix-summed patch counts locate each item's slice, and `image_grid_thw` + is sliced in parallel. + - Audio, stacked on dim 0 (Whisper / Qwen2-Audio / Gemma4 audio): + `input_features` `[B, mel_bins, T]` sliced by item. + + Any additional sibling field in the modality dict whose first-axis length equals + the item count is also sliced -- covers per-item metadata such as + `second_per_grid_ts` (Qwen2.5-VL video) or `input_features_mask` / + `feature_attention_mask` (audio) without model-specific code. + + Models with a different layout (e.g. mixed-modality per param, custom packed + formats) should override this method. The parallel per-item metadata + (`multimodal_embedding_lengths`, `multimodal_hashes`) is model-agnostic and is + re-sliced by the mixin after this returns, so overrides need only handle the + modality-specific raw data. + """ + modality = self._encoder_cache_modality(param) + if modality is None: + raise NotImplementedError( + "Default `build_multimodal_encoder_input` only supports single-modality " + "params. Override for other layouts." + ) + modality_data = param.multimodal_data[modality] + if not isinstance(modality_data, dict): + raise TypeError( + f"multimodal_data[{modality!r}] must be a dict, got {type(modality_data).__name__}" + ) + + indices = list(item_indices) + grid_key = {"image": "image_grid_thw", "video": "video_grid_thw"}.get(modality) + pixel_key = {"image": "pixel_values", "video": "pixel_values_videos"}.get(modality) + + if ( + (grid_key and pixel_key) + and (grid_key in modality_data) + and (pixel_key in modality_data) + ): + # Packed layout: prefix-sum patch counts to locate each item's slab, then + # concat the requested subset in item-index order. + grids = modality_data[grid_key] + n_items = grids.shape[0] + patch_counts = [int(c) for c in torch.prod(grids, dim=1).tolist()] + per_item = torch.split(modality_data[pixel_key], patch_counts, dim=0) + sliced = { + pixel_key: torch.cat([per_item[i] for i in indices], dim=0), + grid_key: grids[indices], + } + elif ( + modality == "image" + and "pixel_values" in modality_data + and "image_sizes" in modality_data + ): + # Stacked layout: dim-0 select from `pixel_values` and list-index `image_sizes`. + n_items = modality_data["pixel_values"].shape[0] + miss_sizes = [modality_data["image_sizes"][i] for i in indices] + miss_pixel = modality_data["pixel_values"][indices] + # `pixel_values` was padded to the request-wide max H/W by the input + # processor. After keeping only the miss subset, crop the trailing H/W back + # down to that subset's own max true size -- otherwise a downstream re-batch + # step (e.g. Mistral 3's `batch_pixel_values`) that pads to + # `max(residual.image_sizes)` would compute a negative pad amount whenever + # the omitted items were the largest in the original request. + if miss_sizes and miss_pixel.dim() >= 4: + max_h = max(int(s[0]) for s in miss_sizes) + max_w = max(int(s[1]) for s in miss_sizes) + miss_pixel = miss_pixel[..., :max_h, :max_w] + sliced = { + "pixel_values": miss_pixel, + "image_sizes": miss_sizes, + } + elif modality == "audio" and "input_features" in modality_data: + # Stacked layout: `input_features [B, mel_bins, T]` sliced on dim 0. + # Per-item masks (`input_features_mask`, `feature_attention_mask`, ...) + # are handled by the sibling-slice pass below. + n_items = modality_data["input_features"].shape[0] + sliced = { + "input_features": modality_data["input_features"][indices], + } + else: + raise NotImplementedError( + f"Default `build_multimodal_encoder_input` cannot slice {modality} layout " + f"with fields {sorted(modality_data)}; override this method." + ) + + # Sibling per-item fields (e.g. `second_per_grid_ts` on Qwen2.5-VL video) + # must be sliced alongside the load-bearing keys above, or the residual + # carries a shape-mismatched encoder input. + sliced = { + **modality_data, + **sliced, + **self._slice_per_item_sibling_fields(modality_data, n_items, indices, sliced.keys()), + } + + # Shallow-copy `multimodal_input` so `_apply_metadata_slice` can rewrite + # `multimodal_hashes` on the residual without mutating the source. + residual_input = ( + copy.copy(param.multimodal_input) if param.multimodal_input is not None else None + ) + return MultimodalParams( + multimodal_data={**param.multimodal_data, modality: sliced}, + multimodal_input=residual_input, + ) + + @staticmethod + def _slice_per_item_sibling_fields( + modality_data: Dict[str, Any], + n_items: int, + item_indices: Sequence[int], + already_sliced: Iterable[str], + ) -> Dict[str, Any]: + """Slice modality-dict siblings whose first axis is parallel to items. + + Anything with `shape[0] == n_items` (tensor) or `len == n_items` (list) is + assumed to be per-item metadata and sliced by `item_indices`. Fields already + handled by the caller (`already_sliced`) and everything else pass through. + """ + skip = set(already_sliced) + sliced: Dict[str, Any] = {} + for key, value in modality_data.items(): + if key in skip: + continue + if isinstance(value, torch.Tensor) and value.dim() > 0 and value.shape[0] == n_items: + sliced[key] = value[item_indices] + elif isinstance(value, list) and len(value) == n_items: + sliced[key] = [value[i] for i in item_indices] + return sliced + # A future optional mixin-owned forward can build on the same template method. def prepare_multimodal_inputs( self, @@ -310,17 +804,39 @@ def _get_or_encode_multimodal_embeddings( Delegates cache lookup and gather behavior to `get_multimodal_embeddings`, then validates the single tensor contract for both encoded and cached-only paths. + + During side-stream prefetch, this runs with the auxiliary stream current, so the H2D copies, + the encoder, and every persistent-cache `put()` are issued on that stream. `TensorLRUCache` + records each entry's producer event on the issuing (aux) stream; the next iteration's + main-stream consumer waits on the request-level `encoder_event` for ordering. """ encoder_cache = self._get_multimodal_encoder_cache() - cache_misses = [] + cache_misses: list[MultimodalParams] = [] + partial_hits: list[tuple[MultimodalParams, EncoderCachePartition]] = [] if encoder_cache is not None: for param in multimodal_params: if param.multimodal_data.get("multimodal_embedding") is not None: - # The forward that attached this request-local embedding already populated the - # persistent cache. + # A present embedding means either an earlier forward already wrote the + # persistent cache, or a prefetch hit attached a cache-owned tensor. Either + # way, skip re-lookup and re-write. `get_multimodal_embeddings` waits on the + # request event and records the attached tensor on the consuming stream before + # gathering it. continue - if not self._attach_encoder_cache_hit(param, encoder_cache): + partition = self.partition_encoder_cache(param, encoder_cache) + if partition is None or partition.is_full_miss: cache_misses.append(param) + continue + if partition.is_full_hit: + param.multimodal_data["multimodal_embedding"] = self.assemble_full_embedding( + partition.hits, len(partition.keys) + ) + continue + partial_hits.append((param, partition)) + + if partial_hits: + # `encoder_cache` is non-None here because partitions are only produced when the cache + # exists. + self._encode_with_partial_cache(partial_hits, encoder_cache) embeddings = get_multimodal_embeddings( encoder_forward_fn=self.encode_multimodal_inputs, @@ -341,19 +857,16 @@ def _get_multimodal_encoder_cache(self) -> Optional[TensorLRUCache]: The cache stores per-item embeddings for params that can be represented by one modality. See `_encoder_cache_keys` for the mixed-modality skip path and its technical limitation. """ - multimodal_config = self.model_config.multimodal_config - if multimodal_config is None: - return None - - max_bytes = multimodal_config.encoder_cache_max_bytes - if max_bytes <= 0: + if not self.encoder_cache_active: logger.debug_once( - f"{_MM_ENCODER_CACHE_LOG_NAME}: disabled because " - "multimodal_config.encoder_cache_max_bytes=0.", + f"{_MM_ENCODER_CACHE_LOG_NAME}: disabled because the model does not opt in via " + "supports_encoder_cache or multimodal_config.encoder_cache_max_bytes=0.", key="mm_encoder_cache_disabled", ) return None + multimodal_config = self.model_config.multimodal_config + max_bytes = multimodal_config.encoder_cache_max_bytes if self._multimodal_encoder_cache is None: # Per-item embeddings are views produced by splitting a request-level encoder output. # Clone them so a cached item neither aliases mutable caller output nor retains the @@ -363,6 +876,7 @@ def _get_multimodal_encoder_cache(self) -> Optional[TensorLRUCache]: self._multimodal_encoder_cache = TensorLRUCache( max_bytes, name=_MM_ENCODER_CACHE_LOG_NAME, + cuda_stream_aware=multimodal_config.encoder_side_stream_max_ahead > 0, ) try: embedding_dim = self.embedding_dim @@ -476,47 +990,163 @@ def _encoder_cache_keys( ] @classmethod - def _attach_encoder_cache_hit( + def partition_encoder_cache( cls, param: MultimodalParams, encoder_cache: TensorLRUCache, - ) -> bool: - """Attach a full persistent-cache hit and report whether one was found.""" + ) -> Optional[EncoderCachePartition]: + """Look up every item of `param` and return the per-item partition. + + Returns `None` when the param is not cacheable (mixed modality, missing metadata, + request-local embedding already attached); the caller should treat that as a + full miss. + """ if param.multimodal_data.get("multimodal_embedding") is not None: logger.debug( f"{_MM_ENCODER_CACHE_LOG_NAME}: request-local multimodal embedding present; " "skipping persistent cache lookup" ) - return False + return None keys = cls._encoder_cache_keys(param) if not keys: - return False + return None - cached_embeddings = [] - for key in keys: - cached_embedding = encoder_cache.get(key) - if cached_embedding is None: - # TODO(TRTLLM-13996): allow re-computing only the uncached items. - # `get_multimodal_embeddings` treats a param as either fully cached or uncached. - # Attaching partial hits would make the later concatenated tensor ambiguous because - # there is no placeholder for missing item rows inside `multimodal_embedding`. - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: cache miss; hit_items={len(cached_embeddings)}," - f" total_items={len(keys)}." - ) - return False - cached_embeddings.append(cached_embedding) + hits: Dict[int, torch.Tensor] = {} + miss_indices: list[int] = [] + for i, key in enumerate(keys): + cached = encoder_cache.get(key) + if cached is None: + miss_indices.append(i) + else: + hits[i] = cached - if len(cached_embeddings) == 1: - param.multimodal_data["multimodal_embedding"] = cached_embeddings[0] - else: - param.multimodal_data["multimodal_embedding"] = torch.cat(cached_embeddings, dim=0) logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: full cache hit for {len(keys)} item entries, " - f"rows={param.multimodal_data['multimodal_embedding'].shape[0]}." + f"{_MM_ENCODER_CACHE_LOG_NAME}: partition hit_items={len(hits)}, " + f"miss_items={len(miss_indices)}, total_items={len(keys)}." ) - return True + return EncoderCachePartition(hits=hits, miss_indices=miss_indices, keys=keys) + + @staticmethod + def assemble_full_embedding( + item_tensors: Dict[int, torch.Tensor], + total_items: int, + ) -> torch.Tensor: + """Copy per-item embedding tensors into a single contiguous buffer in + item-index order. + + `item_tensors` must contain every index in `[0, total_items)`. Sizes the + buffer from the item row counts and `copy_`s each item into its row range: + one predictable allocation of the exact final size, with no `torch.cat` + temporary competing with the sources for peak memory. + """ + if total_items == 1: + return item_tensors[0] + first = item_tensors[0] + total_rows = sum(item_tensors[i].shape[0] for i in range(total_items)) + buffer = torch.empty( + (total_rows, *first.shape[1:]), + dtype=first.dtype, + device=first.device, + ) + offset = 0 + for i in range(total_items): + item = item_tensors[i] + rows = item.shape[0] + buffer[offset : offset + rows].copy_(item) + offset += rows + return buffer + + @staticmethod + def _apply_metadata_slice( + residual: MultimodalParams, + source: MultimodalParams, + item_indices: Sequence[int], + ) -> None: + """Overwrite `residual`'s per-item metadata to match the sliced items. + + Models slice raw modality tensors in `build_multimodal_encoder_input`; the mixin owns + the parallel per-item metadata slice so every model gets it identically. + """ + source_lengths = source.multimodal_data["multimodal_embedding_lengths"] + residual.multimodal_data["multimodal_embedding_lengths"] = [ + source_lengths[i] for i in item_indices + ] + if residual.multimodal_input is not None and source.multimodal_input is not None: + source_hashes = source.multimodal_input.multimodal_hashes + residual.multimodal_input.multimodal_hashes = [source_hashes[i] for i in item_indices] + + def _encode_with_partial_cache( + self, + partials: Sequence[tuple[MultimodalParams, EncoderCachePartition]], + encoder_cache: TensorLRUCache, + ) -> None: + """Encode only the miss items of each partial-hit param and stitch results. + + Miss residuals from all partial-hit params in the batch are encoded in a + single call and the concatenated output is split back per param, mirroring + how `get_multimodal_embeddings` batches full-miss params. After this returns, + each param's `multimodal_embedding` has the same shape as a full encoder run + so downstream `get_multimodal_embeddings` treats it as fully cached. + """ + if not partials: + return + + # Cross-iter prefetch may have staged some params' raw MM tensors on the aux + # stream. If a prefetch encoder call then raised, the request reaches this + # iteration with an `encoder_event` but no `multimodal_embedding`; slicing + # those tensors on the main stream before the event would race the aux-stream + # H2D copy. Wait per param up front, before any raw-tensor read. + for param, _ in partials: + if param.encoder_event is not None: + torch.cuda.current_stream().wait_event(param.encoder_event) + + # Build every residual, then run one batched encoder call over the whole set. + residuals: list[MultimodalParams] = [] + per_param_miss_lengths: list[list[int]] = [] + for param, partition in partials: + residual = self.build_multimodal_encoder_input(param, partition.miss_indices) + self._apply_metadata_slice(residual, param, partition.miss_indices) + residuals.append(residual) + per_param_miss_lengths.append( + [ + param.multimodal_data["multimodal_embedding_lengths"][i] + for i in partition.miss_indices + ] + ) + + batched_output = self.encode_multimodal_inputs(residuals) + per_param_slabs = torch.split( + batched_output, [sum(lengths) for lengths in per_param_miss_lengths], dim=0 + ) + + for (param, partition), slab, miss_lengths in zip( + partials, per_param_slabs, per_param_miss_lengths, strict=True + ): + miss_tensors = torch.split(slab, miss_lengths, dim=0) + + by_item: Dict[int, torch.Tensor] = dict(partition.hits) + for miss_idx, tensor in zip(partition.miss_indices, miss_tensors, strict=True): + by_item[miss_idx] = tensor + param.multimodal_data["multimodal_embedding"] = self.assemble_full_embedding( + by_item, len(partition.keys) + ) + + inserted = 0 + rejected = 0 + for miss_idx, tensor in zip(partition.miss_indices, miss_tensors, strict=True): + if encoder_cache.put(partition.keys[miss_idx], tensor): + inserted += 1 + else: + rejected += 1 + logger.debug( + f"{_MM_ENCODER_CACHE_LOG_NAME}: partial-hit encode " + f"total_items={len(partition.keys)} " + f"hit_items={len(partition.hits)} " + f"encoded_items={len(partition.miss_indices)} " + f"cache_writes_inserted={inserted} " + f"cache_writes_rejected={rejected}" + ) @classmethod def _write_encoder_cache_entries( @@ -711,16 +1341,19 @@ def _dispatch_cross_iter_prefetch( The event covers all work queued in the aux-stream block, so the same event object is shared across all candidates. """ + encoder_cache_enabled = model.encoder_cache_active params_list = [ MultimodalParams( + multimodal_input=_build_request_multimodal_input(req, encoder_cache_enabled), multimodal_data=mm_data, multimodal_runtime=MultimodalRuntimeData( past_seen_token_num=0, chunk_end_pos=cumsum.numel(), embed_mask_cumsum=cumsum, ), + mm_item_order=req.py_mm_item_order, ) - for _, mm_data, cumsum in candidates + for req, mm_data, cumsum in candidates ] # Prefetch targets requests outside the current iteration, so their @@ -728,14 +1361,33 @@ def _dispatch_cross_iter_prefetch( # this after the iteration's LLM kernels so aux-stream H2D copies and # encoder work can overlap them. # - # Ordering is handled by `encoder_event`, and tensor lifetime is anchored - # by `req.py_multimodal_data` until the request is consumed or terminated. - # Keeping `record_stream` out also keeps this path modality-neutral. + # Request-local ordering is handled by `encoder_event`; the consume path also `record_stream`s + # attached embeddings before gathering them so post-prefill request cleanup cannot release + # storage while main-stream work is pending. Persistent-cache clones use their own producer + # events and consumer `record_stream` calls inside `TensorLRUCache`. encoder_event = None try: with _run_on_aux_stream(aux_stream) as encoder_event: - for p in params_list: - p.to_device( + encoder_cache = model._get_multimodal_encoder_cache() if encoder_cache_enabled else None + cache_misses: list[MultimodalParams] = [] + partial_hits: list[tuple[MultimodalParams, EncoderCachePartition]] = [] + if encoder_cache is None: + cache_misses = params_list + else: + for param in params_list: + partition = model.partition_encoder_cache(param, encoder_cache) + if partition is None or partition.is_full_miss: + cache_misses.append(param) + elif partition.is_full_hit: + param.multimodal_data["multimodal_embedding"] = ( + model.assemble_full_embedding(partition.hits, len(partition.keys)) + ) + else: + partial_hits.append((param, partition)) + + params_to_transfer = cache_misses + [param for param, _ in partial_hits] + for param in params_to_transfer: + param.to_device( "multimodal_data", "cuda", pin_memory=prefer_pinned(), @@ -748,8 +1400,23 @@ def _dispatch_cross_iter_prefetch( # model_engine._prepare_inputs. for (req, _, _), p in zip(candidates, params_list): req.py_multimodal_data = p.multimodal_data - encoder_output = model.encode_multimodal_inputs(params_list) - _cache_multimodal_embeddings(params_list, [encoder_output]) + + if partial_hits and encoder_cache is not None: + model._encode_with_partial_cache(partial_hits, encoder_cache) + if cache_misses: + encoder_output = model.encode_multimodal_inputs(cache_misses) + _store_chunked_prefill_embeddings(cache_misses, [encoder_output]) + if encoder_cache is not None: + for param in cache_misses: + model._write_encoder_cache_entries(param, encoder_cache) + + # Prefetch only needs to attach each request's embedding. Validate each request + # independently instead of gathering the unused batch output with `torch.cat`. + for param in params_list: + embedding = param.multimodal_data.get("multimodal_embedding") + if not isinstance(embedding, torch.Tensor): + raise ValueError("Multimodal encoder prefetch did not produce an embedding.") + model._validate_embeddings([embedding], [param]) finally: # Stash the event on every candidate's durable LlmRequest (not the # per-iter `MultimodalParams`), since `_prepare_inputs` rebuilds the diff --git a/tensorrt_llm/_torch/models/modeling_multimodal_utils.py b/tensorrt_llm/_torch/models/modeling_multimodal_utils.py index 3dd866015dd3..63dedd274294 100644 --- a/tensorrt_llm/_torch/models/modeling_multimodal_utils.py +++ b/tensorrt_llm/_torch/models/modeling_multimodal_utils.py @@ -178,13 +178,14 @@ def _get_uncached_multimodal_params( return params_to_run -def _cache_multimodal_embeddings( +def _store_chunked_prefill_embeddings( multimodal_params: List[MultimodalParams], embeddings: List[torch.Tensor], ) -> None: """ - Cache computed multimodal embeddings back to multimodal_data to avoid recomputation. - Note this function only caches multimodal embeddings within the current request context, + Store computed multimodal embeddings back to multimodal_data to avoid recomputation. + + NOTE: this function only stores multimodal embeddings within the current request context, mostly for chunked prefill. It does not persist embeddings across different requests or sessions. """ # TODO: support multiple multimodal modalities per request @@ -267,12 +268,23 @@ def get_multimodal_embeddings( if not multimodal_params: return [] - # Wait before touching tensors produced on the MM side stream. Do not - # clear the event here; repeated stream-side waits are cheap, and leaving - # the event field untouched avoids races if a caller accidentally reuses it. + # Wait before touching tensors produced on the MM side stream. Do not clear the event here; + # repeated stream-side waits are cheap, and leaving the event field untouched avoids races if a + # caller accidentally reuses it. + # Register attached embeddings with the consumer stream as well: the request drops its Python + # references after prefill, potentially before the asynchronous gather below has finished + # reading them. for param in multimodal_params: if param.encoder_event is not None: - torch.cuda.current_stream().wait_event(param.encoder_event) + consumer_stream = torch.cuda.current_stream() + consumer_stream.wait_event(param.encoder_event) + embeds = param.multimodal_data.get("multimodal_embedding") + if isinstance(embeds, torch.Tensor): + embeds = [embeds] + if isinstance(embeds, list): + for embed in embeds: + if isinstance(embed, torch.Tensor) and embed.is_cuda: + embed.record_stream(consumer_stream) # Step 1: Find uncached multimodal params that need encoder processing uncached_multimodal_params = _get_uncached_multimodal_params( @@ -307,8 +319,8 @@ def get_multimodal_embeddings( return encoder_embeddings # Step 3: Cache the computed embeddings to multimodal_data["multimodal_embedding"] - _cache_multimodal_embeddings(uncached_multimodal_params, - encoder_embeddings) + _store_chunked_prefill_embeddings(uncached_multimodal_params, + encoder_embeddings) # Step 4: Gather all embeddings for the batch for param in multimodal_params: @@ -333,9 +345,11 @@ def get_attached_multimodal_embeddings( multimodal_params: List[MultimodalParams]) -> List[torch.Tensor]: """Gather embeddings already stored on MultimodalParams. - Use this on E/P prefill workers and cached-only paths. The encoder already - ran somewhere else. This only makes the tensor list that - find_input_mm_embeds slices. + Use this on E/P prefill workers and cached-only paths. The encoder already ran somewhere else. + This only makes the tensor list that `find_input_mm_embeds` slices. + + Side-stream-prefetched requests must use `get_multimodal_embeddings`, which waits on + `encoder_event` and registers attached tensors with the consuming stream before gathering them. """ attached_embeddings = [] for param in multimodal_params: diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h.py b/tensorrt_llm/_torch/models/modeling_nemotron_h.py index 5c2e60c4706a..8516a3dbdb1e 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_h.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_h.py @@ -17,7 +17,7 @@ import re from contextlib import contextmanager from dataclasses import replace -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal import torch @@ -985,11 +985,22 @@ def load_weights(self, def get_model_defaults(cls, llm_args: "TorchLlmArgs") -> dict: """Model-specific defaults for NemotronH. - Disables block reuse due to SSM/hybrid architecture constraints. + Uses KV cache manager V2 for the hybrid state layout. Block reuse + remains opt-in because it also requires a Mamba snapshot policy. """ - # TODO: Remove enable_block_reuse=False once KV cache block reuse - # is supported for Mamba/SSM-based models - return {"kv_cache_config": {"enable_block_reuse": False}} + return { + "kv_cache_config": { + "enable_block_reuse": False, + "use_kv_cache_manager_v2": True, + } + } + + @classmethod + def get_preferred_transceiver_runtime(cls, + pretrained_config: object + | None = None) -> Literal["PYTHON"]: + """Use the Python transceiver for hybrid-state transfers.""" + return "PYTHON" @staticmethod def lora_config(model_dir: str): @@ -1275,6 +1286,8 @@ def forward( residual=residual, attn_metadata=attn_metadata, all_rank_num_tokens=all_rank_num_tokens, + spec_metadata=spec_metadata, + mamba_metadata=attn_metadata.mamba_metadata, lora_params=lora_params, ) return hidden_states diff --git a/tensorrt_llm/_torch/models/modeling_qwen2vl.py b/tensorrt_llm/_torch/models/modeling_qwen2vl.py index 0a4107bd3d9b..dcf09c4567da 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen2vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen2vl.py @@ -154,7 +154,14 @@ def _prepare_qwen_vl_mrope_config( if len(delta_tensors) != num_seq_slots: raise RuntimeError( "Missing MRoPE position deltas for seq-slot cache update") - deltas = torch.cat(delta_tensors, dim=0) + # `delta_tensors` originate from per-request `multimodal_data` and may + # be CPU-resident when the owning model's `multimodal_data_device_paths` + # does not cover `mrope_config.*` (or a path skips the engine's H2D + # move), while the seq-slot cache and `seq_slots` live on the model + # device. `index_copy_` requires all tensors on the same device. + deltas = torch.cat(delta_tensors, + dim=0).to(device=mrope_position_deltas_cache.device, + non_blocking=True) mrope_position_deltas_cache.index_copy_(0, seq_slots, deltas) if position_ids is not None \ @@ -298,6 +305,9 @@ def get_num_tokens_per_image(self, *, image, **kwargs) -> int: divided by ``spatial_merge_unit`` (the post-merger placeholder count).""" if isinstance(image, torch.Tensor): image_h, image_w = int(image.shape[-2]), int(image.shape[-1]) + elif isinstance(image, np.ndarray): + # HWC uint8 from ImageMediaIO's "np" format. + image_h, image_w = int(image.shape[0]), int(image.shape[1]) else: image_h, image_w = image.height, image.width encoder_tokens = self._num_vision_tokens(width=image_w, @@ -313,6 +323,10 @@ def get_num_tokens_per_video(self, *, video, **kwargs) -> int: if isinstance(first_frame, torch.Tensor): frame_h = int(first_frame.shape[-2]) frame_w = int(first_frame.shape[-1]) + elif isinstance(first_frame, np.ndarray): + # HWC uint8 from VideoMediaIO's "np" format. + frame_h = int(first_frame.shape[0]) + frame_w = int(first_frame.shape[1]) else: frame_h, frame_w = first_frame.height, first_frame.width encoder_tokens = self._num_vision_tokens(width=frame_w, @@ -859,18 +873,18 @@ def call_with_text_prompt( # Text-only fast path: skip the multi-modal HF processor (tokenizer # output matches it bit-exactly when `images` / `videos` are `None`) - # while still populating mrope_config since the LM is M-RoPE. + # and emit no multimodal data at all. Without vision spans the M-RoPE + # coordinates degenerate to the scalar token positions on all three + # axes and the position delta is zero, which is exactly what the model + # engine falls back to for a request carrying no `mrope_config`. + # Synthesizing them would cost an O(seq_len) (3, 1, N) tensor per + # request that the engine then moves to device, and in disaggregated + # serving the prefill worker re-registers that tensor as a CUDA IPC + # handle no one ever consumes. if not mm_data: input_ids = self.tokenizer(text_prompt, return_tensors="pt").input_ids - attention_mask = torch.ones_like(input_ids) - mrope_config = self.get_mrope_config(input_ids, None, None, - attention_mask, None) - return input_ids[0].to(torch.int32).tolist(), { - "multimodal_data": { - "mrope_config": mrope_config - }, - } + return input_ids[0].to(torch.int32).tolist(), None processed_inputs = self._preprocess(text_prompt, mm_data, mm_processor_kwargs) @@ -890,7 +904,8 @@ def call_with_text_prompt( "video_grid_thw": processed_inputs.get('video_grid_thw') } - # NOTE: Even on the text-only prompts, we still need 'mrope_position_ids'. + # Computed from the fused ids so the vision spans get their per-axis + # coordinates; the text-only path above returns before reaching here. mrope_config = self.get_mrope_config( processed_inputs['input_ids'], processed_inputs.get('image_grid_thw', None), @@ -1160,7 +1175,10 @@ def apply_rope(self, # uses head_dim=80 (e.g. 1280 hidden / 16 heads), so use PyTorch RoPE. if IS_FLASHINFER_AVAILABLE and self.head_dim % 64 == 0 and position_ids is not None: try: - cos_sin_cache = torch.cat([cos, sin], dim=-1).contiguous() + # flashinfer requires cos_sin_cache in float32; upstream may cache + # cos/sin in the vision tower dtype (e.g. bf16) as a perf hint. + cos_sin_cache = torch.cat([cos, sin], dim=-1).to( + torch.float32).contiguous() flashinfer_apply_rope_with_cos_sin_cache_inplace( position_ids, q, @@ -1170,7 +1188,7 @@ def apply_rope(self, is_neox=True, ) return q, k, v - except RuntimeError as err: + except (RuntimeError, ValueError) as err: logger.warning( "Qwen2.5-VL vision RoPE: FlashInfer failed (%s); " "falling back to PyTorch RotaryEmbedding.apply_rotary_pos_emb.", @@ -1857,8 +1875,12 @@ def forward( multimodal_params = kwargs.get("multimodal_params", []) mm_embeds = [] mrope_config = {} - # NOTE: Qwen*-VL series has mrope_config even on the text-only prompts, so we need to separate - # the entries that do have multimodal data from those that correspond to text-only prompts. + # `multimodal_params` holds one entry per request that carried any + # multimodal data, context entries first, followed by the generation + # entries that only seed the MRoPE delta cache. The slice bounds the + # scan to the context prefix; `_get_requests_with_mm_data` is what + # actually selects the entries with encoder input, so this does not + # rely on the entries lining up with the context requests. if num_context_requests > 0: mm_multimodal_params = self._get_requests_with_mm_data( multimodal_params[:num_context_requests]) diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_5.py b/tensorrt_llm/_torch/models/modeling_qwen3_5.py index 6fc43129a353..07b6a43d6d1b 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_5.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_5.py @@ -15,7 +15,7 @@ import re from types import SimpleNamespace -from typing import Dict, List +from typing import Dict, List, Literal import torch from transformers import PretrainedConfig @@ -663,17 +663,25 @@ class _Qwen3_5VLModel(Qwen3VLModelBase): decorators (outer arch string + input-processor `model_type`). """ + supports_encoder_cache = True + @classmethod def get_model_defaults(cls, llm_args): # `ModelLoader` applies `get_model_defaults()` on the resolved outer # model class (this VLM wrapper), not on the inner decoder. Both # inner LMs (`Qwen3_5MoeForCausalLM` / `Qwen3_5ForCausalLM`) inherit # `Qwen3NextForCausalLM`'s defaults unchanged, so delegate to it to - # propagate `enable_block_reuse=False` — the hybrid Mamba/SSM path - # doesn't support KV-cache block reuse. Without this the VLM path - # would silently fall back to the global default (block reuse on). + # propagate the V2 manager selection and keep block reuse disabled + # until a recurrent-state snapshot policy is configured. return Qwen3NextForCausalLM.get_model_defaults(llm_args) + @classmethod + def get_preferred_transceiver_runtime( + cls, pretrained_config: object | None = None + ) -> Literal["PYTHON"]: + """Match the hybrid text decoder's Python disaggregated route.""" + return "PYTHON" + def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): kwargs["vision_model_class"] = Qwen3VisionModel kwargs["disable_fuse_rope"] = kwargs.get("disable_fuse_rope", False) diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_next.py b/tensorrt_llm/_torch/models/modeling_qwen3_next.py index 0d91b4ebad74..5e4a893a517c 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_next.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_next.py @@ -1,5 +1,7 @@ # Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py # Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/configs/qwen3_next.py +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 # coding=utf-8 # Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. # @@ -18,7 +20,7 @@ import copy import os from types import SimpleNamespace -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Dict, List, Literal, Optional import torch @@ -36,6 +38,7 @@ from tensorrt_llm._torch.pyexecutor.config_utils import \ get_qwen3_hybrid_layer_types from tensorrt_llm._utils import get_sm_version +from tensorrt_llm.models.modeling_utils import QuantConfig from ...logger import logger from ..attention_backend import AttentionMetadata @@ -60,6 +63,84 @@ from .modeling_utils import DecoderModel, EagerFusionConfig, register_auto_model +def _fused_norm_weight(norm: RMSNorm) -> torch.Tensor: + """Weight to feed the fused AllReduce+RMSNorm op for ``norm``. + + Gemma RMSNorm scales by ``(1 + weight)`` (see RMSNorm.forward), but the + fused AR+RMSNorm kernels (and the NCCL / NCCL_SYMMETRIC fallbacks the + AUTO strategy may pick) only apply ``weight``. Baking the ``+1`` into the + weight makes EVERY allreduce backend produce the correct gemma result + without any backend-specific flag. + + For gemma norms the ``(1 + weight)`` tensor is precomputed once in + ``Qwen3NextForCausalLM.cache_derived_state`` and cached on the module as + ``_fused_norm_weight``. Computing it inline here would re-run a cast+add + elementwise kernel every forward inside the CUDA graph. The inline path + below is only a correctness fallback if the cache is absent. + """ + cached = getattr(norm, "_fused_norm_weight", None) + if cached is not None: + return cached + w = norm.weight + if getattr(norm, "use_gemma", False): + return (w.float() + 1.0).to(w.dtype) + return w + + +def _precompute_fused_norm_weights(module: nn.Module) -> None: + """Bake ``(1 + weight)`` once for every gemma RMSNorm under ``module``. + + Caches the result on each norm as ``_fused_norm_weight`` so the fused + AllReduce+RMSNorm path reads a ready tensor instead of recomputing the + cast+add every forward. Non-gemma norms are left untouched (the fused op + uses their ``weight`` directly). Must run after weights are loaded onto the + device; the cached tensor is a plain attribute, not a registered buffer, so + it stays out of the state dict. + + Norms whose ``weight`` was stripped are skipped: the layer-wise benchmark + runs ``remove_weights`` on unused layers (``skip_forward``), leaving a + ``use_gemma`` norm without a ``weight`` parameter; those layers never run + the fused path, so there is nothing to precompute. + """ + for norm in module.modules(): + if isinstance(norm, RMSNorm) and getattr(norm, "use_gemma", False): + w = getattr(norm, "weight", None) + if w is None: + continue + norm._fused_norm_weight = (w.float() + 1.0).to(w.dtype) + + +def _eager_fusion_enabled(enable_attention_dp: bool) -> bool: + return (os.environ.get("TRTLLM_QWEN3_EAGER_FUSION_DISABLED", "0") == "0" + and not enable_attention_dp) + + +def _experts_excluded_from_quant(model_config: ModelConfig[Qwen3NextConfig], + layer_idx: Optional[int]) -> bool: + """Is this layer's routed-experts module listed in ``exclude_modules``? + + ``exclude_modules`` is applied only after every module is built, but + ``create_moe`` has to pick the MoE backend class before that. A backend + picked for FP8/NVFP4 weights rejects the experts if they turn out to be + bf16, so look the answer up now instead of waiting for that pass. + """ + quant_config = model_config.quant_config + if layer_idx is None or not quant_config.exclude_modules: + return False + candidates = [f"model.layers.{layer_idx}.mlp.experts"] + n_hidden_layers = getattr(model_config.pretrained_config, + "num_hidden_layers", None) + # One MTP layer has two names: mtp.layers. in the checkpoint and + # model.layers. at runtime. Only the Qwen3.5 entry + # points rewrite the first into the second, so try both names here. + if n_hidden_layers is not None and layer_idx >= n_hidden_layers: + candidates.append( + f"mtp.layers.{layer_idx - n_hidden_layers}.mlp.experts") + return any( + quant_config.is_module_excluded_from_quantization(candidate) + for candidate in candidates) + + class Qwen3NextGate(nn.Module): def __init__( @@ -159,6 +240,11 @@ def __init__( if quant_config_dict and layer_idx is not None: expert_quant_config = quant_config_dict.get( f"model.layers.{layer_idx}.mlp.experts") + # Excluded experts end up bf16 whatever the per-layer entry says, so + # hand create_moe a bf16 config and let it pick a backend serving bf16. + if _experts_excluded_from_quant(model_config, layer_idx): + expert_quant_config = QuantConfig(kv_cache_quant_algo=model_config. + quant_config.kv_cache_quant_algo) self.experts = create_moe( num_experts=self.num_experts, routing_method=self.gate.routing_method, @@ -366,17 +452,15 @@ def __init__( self.next_layer_layernorm: RMSNorm = None self.fusion_config = EagerFusionConfig() - ### TODO: enable eager_fusion by default - self.enable_fusion = os.environ.get( - "TRTLLM_QWEN3_EAGER_FUSION_DISABLED", "1") == "0" - self.enable_fusion &= not self.enable_attention_dp + self.enable_fusion = _eager_fusion_enabled(self.enable_attention_dp) has_tp = self.mapping.has_tp() has_pp = self.mapping.has_pp() self.fusion_config.PRE_MOE_FUSION = self.enable_fusion and has_tp - self.fusion_config.POST_MOE_FUSION = self.fusion_config.PRE_MOE_FUSION and not has_pp and self.enable_attention_dp - self.disable_attn_allreduce = (self.mapping.tp_size == 1 + self.fusion_config.POST_MOE_FUSION = self.fusion_config.PRE_MOE_FUSION and not has_pp + self.disable_attn_allreduce = (self.fusion_config.PRE_MOE_FUSION + or self.mapping.tp_size == 1 or self.enable_attention_dp) self.moe_allreduce = MoEAllReduce(mapping=model_config.mapping) @@ -413,21 +497,18 @@ def forward( all_reduce_params=AllReduceParams( fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, residual=residual, - norm_weight=self.post_attention_layernorm.weight, + norm_weight=_fused_norm_weight( + self.post_attention_layernorm), eps=self.post_attention_layernorm.variance_epsilon, - enable_allreduce=not self.disable_attn_allreduce, )) else: # No fusion hidden_states, residual = self.post_attention_layernorm( hidden_states, residual) - # Note: this fusion pattern is only supported for TRTLLM-nvfp4 backend now - do_finalize = not (self.fusion_config.POST_MOE_FUSION - and hidden_states.shape[0] - <= self.moe_allreduce.max_token - and self.model_config.moe_backend == 'TRTLLM' - and self.mlp.experts.has_nvfp4) + # Qwen3NextSparseMoeBlock does not implement do_finalize=False. Defer + # only its final all-reduce so the decoder can fuse it with RMSNorm. + do_finalize = True hidden_states = self.mlp( hidden_states, @@ -446,7 +527,8 @@ def forward( all_reduce_params=AllReduceParams( fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, residual=residual, - norm_weight=self.next_layer_layernorm.weight, + norm_weight=_fused_norm_weight( + self.next_layer_layernorm), eps=self.next_layer_layernorm.variance_epsilon, )) else: @@ -489,6 +571,7 @@ def __init__(self, model_config: ModelConfig[Qwen3NextConfig], fuse_qk_norm_rope=fuse_qk_norm_rope, attn_output_gate=True, use_gemma_rms_norm=True) + self._fuse_qk_norm_rope_gate = True class Qwen3NextFullAttentionDecoderLayer(DecoderLayer): @@ -531,17 +614,24 @@ def __init__(self, model_config: ModelConfig[Qwen3NextConfig], self.next_layer_layernorm: RMSNorm = None self.fusion_config = EagerFusionConfig() - self.enable_fusion = os.environ.get( - "TRTLLM_QWEN3_EAGER_FUSION_DISABLED", "0") == "0" - self.enable_fusion &= not self.enable_attention_dp + self.enable_fusion = _eager_fusion_enabled(self.enable_attention_dp) has_tp = self.mapping.has_tp() has_pp = self.mapping.has_pp() self.fusion_config.PRE_MOE_FUSION = self.enable_fusion and has_tp - self.fusion_config.POST_MOE_FUSION = self.fusion_config.PRE_MOE_FUSION and not has_pp and self.enable_attention_dp - self.disable_attn_allreduce = (self.mapping.tp_size == 1 + # POST_MOE_FUSION fuses the MoE-output all-reduce with the next layer's + # RMSNorm. It is a tensor-parallel (TEP) optimization: it is only valid + # when ranks share the same tokens (not attention_dp, where each rank holds + # different tokens and the MoE block does no cross-rank all-reduce). This + # mirrors the DeepSeek-V3 pattern (POST == PRE in the non-attention_dp path). + self.fusion_config.POST_MOE_FUSION = self.fusion_config.PRE_MOE_FUSION and not has_pp + # When PRE_MOE_FUSION is on, the attention all-reduce is deferred to the + # fused PRE all-reduce+RMSNorm, so disable the in-attention all-reduce to + # avoid reducing twice. + self.disable_attn_allreduce = (self.fusion_config.PRE_MOE_FUSION + or self.mapping.tp_size == 1 or self.enable_attention_dp) self.moe_allreduce = MoEAllReduce(mapping=model_config.mapping) @@ -574,13 +664,14 @@ def forward( **kwargs, ) - if self.fusion_config.PRE_MOE_FUSION and self.enable_attention_dp: + if self.fusion_config.PRE_MOE_FUSION: hidden_states, residual = self.allreduce( hidden_states, all_reduce_params=AllReduceParams( fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, residual=residual, - norm_weight=self.post_attention_layernorm.weight, + norm_weight=_fused_norm_weight( + self.post_attention_layernorm), eps=self.post_attention_layernorm.variance_epsilon, )) else: @@ -588,12 +679,12 @@ def forward( hidden_states, residual = self.post_attention_layernorm( hidden_states, residual) - # Note: this fusion pattern is only supported for TRTLLM-nvfp4 backend now - do_finalize = not (hidden_states.shape[0] - <= self.moe_allreduce.max_token - and self.fusion_config.POST_MOE_FUSION - and self.model_config.moe_backend == 'TRTLLM' - and self.mlp.experts.has_nvfp4) + # The fully-fused do_finalize=False MoE path (MoEAllReduce on the + # unfinalized expert output) is not implemented by Qwen3NextSparseMoeBlock + # (it raises NotImplementedError). Keep do_finalize=True so POST_MOE_FUSION + # still fuses the *finalized* MoE all-reduce with the next layer's RMSNorm + # via the do_finalize branch below, without hitting the unimplemented path. + do_finalize = True hidden_states = self.mlp( hidden_states, attn_metadata, @@ -611,7 +702,8 @@ def forward( all_reduce_params=AllReduceParams( fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, residual=residual, - norm_weight=self.next_layer_layernorm.weight, + norm_weight=_fused_norm_weight( + self.next_layer_layernorm), eps=self.next_layer_layernorm.variance_epsilon, )) else: @@ -706,20 +798,12 @@ class Qwen3NextMTP(Qwen3NextFullAttentionDecoderLayer): def __init__(self, model_config: ModelConfig[Qwen3NextConfig], layer_idx: int, aux_stream_dict: Dict[AuxStreamType, torch.cuda.Stream]): - # Some HF checkpoints (e.g. Qwen3.5 NVFP4) keep the MTP layer entirely - # unquantized -- including its MoE experts (no weight_scale tensors). - # Most non-CUTLASS MoE backends (TRTLLMGen, CuteDsl, ...) reject - # unquantized / kv-cache-only quant_modes at create_weights or - # forward time (e.g. "TRTLLMGenFusedMoE doesn't support - # fp16/bf16/fp32 MoE", "CuteDslFusedMoE doesn't support quantization - # mode [128]" where bit 128 is FP8_KV_CACHE only). CUTLASS MoE - # supports both BF16 and quantized MoE, so when the checkpoint marks - # MTP as excluded from quantization, fall back to CUTLASS *only* for - # the MTP layer. Regular layers (0..num_hidden_layers-1) keep the - # user-selected backend. + # Some HF checkpoints (e.g. Qwen3.5 NVFP4) keep the whole MTP layer in + # bf16. Given a bf16 config most backends switch to CUTLASS on their + # own; DEEPGEMM and WIDEEP do not, so switch for them here. mtp_model_config = model_config if (model_config.moe_backend != "CUTLASS" - and Qwen3NextMTP._is_mtp_excluded_from_quant(model_config)): + and _experts_excluded_from_quant(model_config, layer_idx)): original_backend = model_config.moe_backend mtp_model_config = copy.copy(model_config) mtp_model_config._frozen = False @@ -777,33 +861,9 @@ def __init__(self, model_config: ModelConfig[Qwen3NextConfig], use_cute_dsl_blockscaling_mm=False, ) self.shared_head = Qwen3NextMTPHead(mtp_model_config) - - @staticmethod - def _is_mtp_excluded_from_quant( - model_config: ModelConfig[Qwen3NextConfig]) -> bool: - """Heuristic: did the checkpoint mark MTP as excluded from quantization? - - ``apply_quant_config_exclude_modules`` runs *after* this constructor. - For Qwen3.5 paths the exclude_modules list has already been - translated to ``model.layers.*`` by - ``_normalize_qwen35_exclude_modules`` -- detect that. Raw HF - ``mtp.*`` patterns are also accepted (defensive: if some future - weight mapper does not translate them, this still triggers the - backend fallback so the MoE path doesn't crash). - """ - qc = getattr(model_config, "quant_config", None) - if qc is None or not getattr(qc, "exclude_modules", None): - return False - n_layers = getattr(model_config.pretrained_config, "num_hidden_layers", - None) - target_prefix = (f"model.layers.{n_layers}" - if n_layers is not None else None) - for pat in qc.exclude_modules: - if pat.startswith("mtp."): - return True - if target_prefix is not None and pat.startswith(target_prefix): - return True - return False + # MTP applies shared_head.norm after the base decoder forward, so its + # MoE-output all-reduce cannot consume next_layer_layernorm. + self.fusion_config.POST_MOE_FUSION = False def forward( self, @@ -988,9 +1048,24 @@ def __init__( @classmethod def get_model_defaults(cls, llm_args: 'TorchLlmArgs') -> dict: - # TODO: Remove enable_block_reuse=False once KV cache block reuse - # is supported for Mamba/SSM-based models - return {"kv_cache_config": {"enable_block_reuse": False}} + """Use V2 for the hybrid state layout. + + Block reuse remains opt-in because it also requires a recurrent-state + snapshot policy. + """ + return { + "kv_cache_config": { + "enable_block_reuse": False, + "use_kv_cache_manager_v2": True, + } + } + + @classmethod + def get_preferred_transceiver_runtime(cls, + pretrained_config: object + | None = None) -> Literal["PYTHON"]: + """Use the Python transceiver for hybrid-state transfers.""" + return "PYTHON" def load_weights(self, weights: dict, @@ -998,6 +1073,12 @@ def load_weights(self, params_map: Optional[Dict[str, str]] = None, allow_partial_loading: bool = False): new_weights = weight_mapper.preprocess_weights(weights) + # `new_weights` aliases the source tensors for every key + # `preprocess_weights` did not rewrite -- the routed experts, i.e. most + # of a MoE checkpoint. Holding `weights` too would pin them, so + # consuming `new_weights` during the load would free nothing. + if hasattr(weights, "clear"): + weights.clear() super().load_weights( new_weights, weight_mapper=weight_mapper, @@ -1013,3 +1094,7 @@ def setup_aliases(self) -> None: else: layer.next_layer_layernorm = self.model.layers[ idx + 1].input_layernorm + + def cache_derived_state(self) -> None: + super().cache_derived_state() + _precompute_fused_norm_weights(self) diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl.py b/tensorrt_llm/_torch/models/modeling_qwen3vl.py index 6d4c0a6249c1..4dec5d64b165 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl.py @@ -5,7 +5,7 @@ import math import re from functools import lru_cache -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import numpy as np import torch @@ -19,7 +19,10 @@ Qwen3VLVisionPatchEmbed as HFQwen3VLVisionPatchEmbed, ) -from tensorrt_llm._torch.models.modeling_multimodal_utils import _is_mm_disagg +from tensorrt_llm._torch.models.modeling_multimodal_utils import ( + _is_mm_disagg, + filter_mm_token_from_input_ids, +) from tensorrt_llm.functional import PositionEmbeddingType from tensorrt_llm.mapping import Mapping @@ -37,6 +40,7 @@ from ..attention_backend import AttentionMetadata from ..attention_backend.interface import PositionalEmbeddingParams, RopeParams from ..attention_backend.utils import get_attention_backend +from ..modules.embedding import Embedding from ..modules.layer_norm import LayerNorm from ..modules.linear import Linear, TensorParallelMode from ..modules.mlp import MLP @@ -45,13 +49,11 @@ from .checkpoints.hf.qwen3vl_weight_mapper import Qwen3VLHfWeightMapper from .modeling_auto import AutoModelForCausalLM from .modeling_multimodal_encoder import MultimodalEncoderMixin -from .modeling_multimodal_mixin import MultimodalModelMixin -from .modeling_multimodal_utils import ( - filter_mm_token_from_input_ids, - find_input_mm_embeds, - fuse_input_embeds, - get_attached_multimodal_embeddings, - get_multimodal_embeddings, +from .modeling_multimodal_mixin import ( + EncoderGroup, + MultimodalModelMixin, + PreparedLlmInputs, + encode_multimodal_by_groups, ) from .modeling_qwen2vl import ( Qwen2_5_VLVisionAttention, @@ -303,11 +305,23 @@ def _preprocess( videos = [video_data.frames for video_data in video_datas] else: videos = None - do_rescale = True - if images and isinstance(images[0], torch.Tensor): - do_rescale = False - if videos and isinstance(videos[0][0], torch.Tensor): - do_rescale = False + # HF's Qwen3-VL processor takes a single `do_rescale` kwarg that is + # applied to both images and videos. Mixed pre-rescale states leave + # the raw side at 0-255 while the pre-rescaled side gets skipped — + # the ViT then produces garbage for the raw modality. Fail fast; the + # default `get_preferred_media_io_kwargs` ships `np` for both, so a + # divergence here means a caller override drifted. + image_is_pre = bool(images) and isinstance(images[0], torch.Tensor) + video_is_pre = bool(videos) and isinstance(videos[0][0], torch.Tensor) + if images and videos and image_is_pre != video_is_pre: + raise ValueError( + "Qwen3-VL requires image and video items to arrive in the same " + "pre-rescaled state (both torch.Tensor, or both uint8 numpy). " + f"Got image_is_pre_rescaled={image_is_pre}, " + f"video_is_pre_rescaled={video_is_pre}. Configure consistent " + "formats via `media_io_kwargs`." + ) + do_rescale = not (image_is_pre if images else video_is_pre) do_sample_frames = _decide_do_sample_frames(video_datas, mm_processor_kwargs) @@ -398,9 +412,12 @@ def get_num_tokens_per_video( return self.get_num_tokens_per_video(video=video, video_grid_thw=vgt) def get_preferred_media_io_kwargs(self) -> Dict[str, Dict[str, Any]]: - # uint8 HWC frames let the HF processor rescale/permute once, skipping - # the per-frame CHW-float conversion in the IO loader. - return {"video": {"format": "np"}} + # uint8 HWC arrays for both modalities so the HF processor rescales + # and normalizes uniformly in one pass. A single `do_rescale` kwarg + # controls both modalities; if they arrive in mismatched + # pre-rescale states (e.g. image=Tensor pre-rescaled, video=uint8 + # np), the raw side is left at 0-255 and the ViT sees garbage. + return {"image": {"format": "np"}, "video": {"format": "np"}} def build_disagg_prefill_multimodal_inputs( self, inputs: TextPrompt, mm_handles: List[Dict[str, Any]] @@ -1008,6 +1025,28 @@ def forward( return hidden_states, deepstack_feature_lists +def _qwen3vl_build_batched_input( + multimodal_params: list[MultimodalParams], +) -> dict[str, Any]: + """Cat image items then video items across requests (matching the + ``EncoderGroup.modalities`` order) into one ViT input.""" + pixels: list[torch.Tensor] = [] + grids: list[torch.Tensor] = [] + for m, pv_key, thw_key in ( + ("image", "pixel_values", "image_grid_thw"), + ("video", "pixel_values_videos", "video_grid_thw"), + ): + for mp in multimodal_params: + bucket = mp.multimodal_data.get(m) + if bucket is not None: + pixels.append(bucket[pv_key]) + grids.append(bucket[thw_key]) + return { + "pixel_values": torch.cat(pixels, dim=0), + "grid_thw": torch.cat(grids, dim=0), + } + + class Qwen3VisionModelBase(nn.Module): def __init__( self, @@ -1059,94 +1098,52 @@ def load_weights(self, weights: Dict[str, torch.Tensor]): self.visual.config.num_attention_heads = self.visual.config.num_heads _load_weights_impl(self.visual, converted_weights, params_map=pattern_mapping) - def _parse_and_batch_multimodal_data( - self, multimodal_params: List[MultimodalParams] - ) -> Tuple[Dict[str, Any], Dict[str, List[Any]]]: - pixel_values_list = [] - pixel_values_videos_list = [] - image_grid_thw_list = [] - video_grid_thw_list = [] - - for multimodal_param in multimodal_params: - multimodal_data = multimodal_param.multimodal_data - # Process images if present - if multimodal_data.get("image") is not None: - pixel_values_list.append(multimodal_data["image"]["pixel_values"]) - image_grid_thw_list.append(multimodal_data["image"]["image_grid_thw"]) - - # Process videos if present - if multimodal_data.get("video") is not None: - pixel_values_videos_list.append(multimodal_data["video"]["pixel_values_videos"]) - video_grid_thw_list.append(multimodal_data["video"]["video_grid_thw"]) - - # Concatenate tensors - mm_content_dict = {} - if pixel_values_list: - mm_content_dict["pixel_values"] = ( - torch.cat(pixel_values_list, dim=0) - if len(pixel_values_list) > 1 - else pixel_values_list[0] - ) - if pixel_values_videos_list: - mm_content_dict["pixel_values_videos"] = ( - torch.cat(pixel_values_videos_list, dim=0) - if len(pixel_values_videos_list) > 1 - else pixel_values_videos_list[0] - ) - - # Prepare extra data - mm_extra_data = {} - if image_grid_thw_list: - mm_extra_data["image_grid_thw"] = ( - torch.cat(image_grid_thw_list, dim=0) - if len(image_grid_thw_list) > 1 - else image_grid_thw_list[0] - ) - if video_grid_thw_list: - mm_extra_data["video_grid_thw"] = ( - torch.cat(video_grid_thw_list, dim=0) - if len(video_grid_thw_list) > 1 - else video_grid_thw_list[0] - ) - - return mm_content_dict, mm_extra_data - @torch.inference_mode() - def forward(self, multimodal_params: List[MultimodalParams]) -> List[torch.Tensor]: - mm_content_data, mm_extra_data = self._parse_and_batch_multimodal_data(multimodal_params) - pixel_values = mm_content_data.get("pixel_values", None) - pixel_values_videos = mm_content_data.get("pixel_values_videos", None) + def encode_batched( + self, + pixel_values: torch.Tensor, + grid_thw: torch.Tensor, + ) -> torch.Tensor: + """Run the ViT on one concat'd batch and fold deepstack streams into + the hidden dim. Modality-agnostic — image and video items are + distinguished only by ``grid_thw`` rows (image ``t=1``, video ``t>1``). + """ + pixel_values = pixel_values.to(self.model_dtype) + embeds, deepstack = self.visual(pixel_values, grid_thw=grid_thw) + # Shape: [seq_len, hidden_dim * (num_deepstack_layers + 1)] + return torch.cat([embeds] + deepstack, dim=1) - if pixel_values is not None and pixel_values_videos is not None: - raise ValueError("Currently only support single modality per request") + @property + def mm_encoder_groups(self) -> Tuple[EncoderGroup, ...]: + """Single source of truth for Qwen3-VL's encoder-batching group. - image_grid_thw = mm_extra_data.get("image_grid_thw", None) - video_grid_thw = mm_extra_data.get("video_grid_thw", None) + One modality-blind ViT handles both image and video (image has + `grid_thw.t==1`, video `t>1`), so both modalities share one call. + Both the aggregated path (via `Qwen3VLModelBase.mm_encoder_groups`, + which delegates here) and the mm-encoder-only `forward` consume this. + """ + return ( + EncoderGroup( + modalities=("image", "video"), + encoder_fn=self.encode_batched, + build_batched_input=_qwen3vl_build_batched_input, + ), + ) - embeds = [] - if pixel_values is not None: - pixel_values = pixel_values.to(self.model_dtype) - image_embeds, deepstack_image_embeds = self.visual( - pixel_values, grid_thw=image_grid_thw - ) - # NOTE: We concatenate deepstack_embeds to mm_embeds - # The shape will be [seq_len, hidden_dim * (num_deepstack_layers + 1)] - mixed_image_embeds = torch.cat([image_embeds] + deepstack_image_embeds, dim=1) - embeds.append(mixed_image_embeds) - - if pixel_values_videos is not None: - pixel_values_videos = pixel_values_videos.to(self.model_dtype) - video_embeds, deepstack_video_embeds = self.visual( - pixel_values_videos, grid_thw=video_grid_thw - ) - # NOTE: We concatenate deepstack_embeds to mm_embeds - # The shape will be [seq_len, hidden_dim * (num_deepstack_layers + 1)] - mixed_video_embeds = torch.cat([video_embeds] + deepstack_video_embeds, dim=1) - embeds.append(mixed_video_embeds) - return embeds + def forward(self, multimodal_params: List[MultimodalParams]) -> List[torch.Tensor]: + """Standalone mm-encoder-only executor entry. + + `_forward_step_mm_encoder_only` invokes this and then splits the + returned tensor request-by-request using request-ordered + `split_lengths`, so the rows must be in request-then-prompt order. + Delegating to `encode_multimodal_by_groups` runs the same + modality-batched ViT the aggregated path uses and applies the + per-request `mm_item_order` reorder before returning. + """ + return [encode_multimodal_by_groups(self.mm_encoder_groups, multimodal_params)] -class Qwen3VLModelBase(PreTrainedModel, MultimodalModelMixin): +class Qwen3VLModelBase(MultimodalModelMixin, PreTrainedModel): def encode_multimodal_inputs( self, multimodal_params: List[MultimodalParams], **encoder_kwargs: Any ) -> torch.Tensor: @@ -1154,13 +1151,17 @@ def encode_multimodal_inputs( Runs the vision encoder over ``multimodal_params`` and returns the embeddings as a single tensor (Qwen3-VL folds deepstack streams into - the hidden dim, so the single-tensor contract holds). Used by the - startup memory profiler to invoke the encoder directly; the model's - own ``forward`` keeps its custom deepstack fusion path. + the hidden dim, so the single-tensor contract holds). """ - mm_embeds = get_multimodal_embeddings( - encoder_forward_fn=self.mm_encoder.forward, multimodal_params=list(multimodal_params) - ) + if self.mm_encoder is None: + raise ValueError("Raw multimodal inputs require a local multimodal encoder.") + + mm_embeds = self.mm_encoder.forward(list(multimodal_params), **encoder_kwargs) + if len(mm_embeds) != 1: + raise ValueError( + "Qwen3-VL multimodal encoder must return one packed embedding tensor, " + f"but returned {len(mm_embeds)} tensors." + ) return mm_embeds[0] def _check_and_adjust_experts_implementation(self, *args, **kwargs): @@ -1260,6 +1261,9 @@ def __init__( self.mm_encoder = Qwen3VisionModelBase( copy.deepcopy(model_config), kwargs.get("vision_model_class", None) ).eval() + # Reuse the encoder's own group definition so the aggregated and + # mm-encoder-only paths share a single source of truth. + self.mm_encoder_groups = self.mm_encoder.mm_encoder_groups elif model_config.disable_mm_encoder: logger.info( f"{type(self).__name__}: multimodal encoder disabled " @@ -1270,11 +1274,10 @@ def __init__( self.deepstack_num_level = ( len(config.vision_config.deepstack_visual_indexes) if self.use_deepstack else 0 ) - if self.use_deepstack: - # Pre-allocated `(L, max_num_tokens, hidden)` scratch buffer for - # per-layer deepstack embeddings; replaces `L` fresh - # `torch.zeros` + `L` scatters per prefill. - # `persistent=False` keeps it out of `state_dict`. + if self.deepstack_num_level > 0: + # Reuse one `(L, max_num_tokens, hidden)` scratch allocation for + # per-layer deepstack embeddings. The generic extra-embedding path + # allocates and scatters one full-sequence tensor per level. self.register_buffer( "deepstack_input_embeds", torch.zeros( @@ -1302,20 +1305,12 @@ def __init__( self.post_config() @property - def mm_token_ids(self) -> torch.Tensor: + def multimodal_token_ids(self) -> torch.Tensor: return self._mm_token_ids - def post_config(self): - # use llm.config as config for pytorch model engine - self.model_config.pretrained_config = self.llm.config - self.config = self.model_config.pretrained_config - @property - def vocab_size_padded(self) -> int: - return self.llm.vocab_size_padded - - def infer_max_seq_len(self) -> int: - return self.llm.infer_max_seq_len() + def language_model(self) -> torch.nn.Module: + return self.llm # Draft-model (two-model speculative decoding, e.g. DFlash / Eagle3) # delegation: `ModelLoader.load` reads `draft_config` / `draft_model` and @@ -1337,6 +1332,23 @@ def draft_model(self): def load_draft_weights(self, weights: Dict, weight_mapper: Optional[BaseWeightMapper] = None): return self.llm.load_draft_weights(weights, weight_mapper=weight_mapper) + @property + def text_embedding_layer(self) -> Embedding: + return self.llm.model.embed_tokens + + @property + def embedding_dim(self) -> int: + return self.text_embedding_layer.embedding_dim * (self.deepstack_num_level + 1) + + @property + def embedding_dtype(self) -> torch.dtype: + return self.text_embedding_layer.weight.dtype + + def post_config(self): + # use llm.config as config for pytorch model engine + self.model_config.pretrained_config = self.llm.config + self.config = self.model_config.pretrained_config + def apply_llm_torch_compile(self, *, backend: Any, fullgraph: bool) -> None: # TODO: Move this hook to MultimodalModelMixin once multimodal models # consistently expose an LLM compile contract. @@ -1386,138 +1398,137 @@ def split_mm_embeds(self, mm_embed, deepstack_num_level): mm_embed_chunks = torch.split(mm_embed, [num_elements] * (deepstack_num_level + 1), dim=1) return mm_embed_chunks[0], list(mm_embed_chunks[1:]) - @torch.inference_mode() - def forward( + def select_multimodal_params( self, - attn_metadata: AttentionMetadata, - input_ids: Optional[torch.IntTensor] = None, - position_ids: Optional[torch.IntTensor] = None, - input_embeds: Optional[torch.Tensor] = None, - return_context_logits: bool = False, - **kwargs, - ) -> torch.Tensor: - """ - VLM forward logic with inflight batching support. - """ - num_context_requests, num_generation_requests = ( - attn_metadata.num_contexts, - attn_metadata.num_generations, + multimodal_params: List[MultimodalParams], + num_context_requests: int, + ) -> List[MultimodalParams]: + """Select requests with image/video embeddings for the current context batch.""" + context_params = super().select_multimodal_params(multimodal_params, num_context_requests) + multimodal_params, has_raw_image_or_video_data = self._get_requests_with_mm_data( + context_params ) - - multimodal_params = kwargs.get("multimodal_params", []) - mm_embeds = [] - mrope_config = {} - deepstack_embeds = [] - - # NOTE: Qwen*-VL series has mrope_config even on the text-only prompts, - # so we need to separate the mm_multimodal_params from the text-only prompts. - if num_context_requests > 0: - mm_multimodal_params, has_raw_image_or_video_data = self._get_requests_with_mm_data( - multimodal_params[:num_context_requests] + if not multimodal_params: + return [] + if has_raw_image_or_video_data and self.mm_encoder is None: + raise ValueError( + "Raw multimodal inputs require a local multimodal encoder on this " + "worker, or multimodal_embedding handles from an encoder handoff." ) - else: - mm_multimodal_params = [] - has_raw_image_or_video_data = False - if len(mm_multimodal_params) > 0: - # Raw image/video tensors: run local encoder. - if has_raw_image_or_video_data and self.mm_encoder is not None: - mm_embeds = get_multimodal_embeddings( - encoder_forward_fn=self.mm_encoder.forward, - multimodal_params=mm_multimodal_params, - ) - # Raw image/video tensors on a worker with no encoder: bad route. - elif has_raw_image_or_video_data: - raise ValueError( - "Raw multimodal inputs require a local multimodal encoder on this " - "worker, or multimodal_embedding handles from an encoder handoff." - ) - # support_mm_disagg is only set in subclasses of Qwen3VLModelBase that support EPD - elif not getattr(self, "support_mm_disagg", False): - raise NotImplementedError( - f"{type(self)} does not support disaggregated inference yet. Please unset " - "the TLLM_MULTIMODAL_DISAGGREGATED environment variable, or set it to '0'." - ) - # E/P prefill: encoder already ran; use attached embeddings. - else: - mm_embeds = get_attached_multimodal_embeddings(mm_multimodal_params) - mm_embeds = find_input_mm_embeds(mm_embeds, mm_multimodal_params) + if not has_raw_image_or_video_data and not getattr(self, "support_mm_disagg", False): + raise NotImplementedError( + f"{type(self)} does not support disaggregated inference yet. Please unset " + "the TLLM_MULTIMODAL_DISAGGREGATED environment variable, or set it to '0'." + ) + return multimodal_params - if self.use_deepstack: - for i, mm_embed in enumerate(mm_embeds): - mm_embed, deepstack_embed = self.split_mm_embeds( - mm_embed, self.deepstack_num_level - ) - mm_embeds[i] = mm_embed - deepstack_embeds.extend(deepstack_embed) + def after_active_multimodal_embeddings( + self, + *, + active_embeddings: List[torch.Tensor], + multimodal_params: List[MultimodalParams], + **forward_kwargs: Any, + ) -> tuple[List[torch.Tensor], List[torch.Tensor]]: + """Separate Qwen3-VL's packed deepstack streams from primary embeddings.""" + if not self.use_deepstack: + return active_embeddings, [] - if not self.model_config.pretrained_config.disable_fuse_rope: - mrope_config = self.prepare_mrope_config( - multimodal_params, - num_generation_requests, - position_ids, - mrope_delta_write_seq_slots=kwargs.get("mrope_delta_write_seq_slots"), - mrope_delta_read_seq_slots=kwargs.get("mrope_delta_read_seq_slots"), + deepstack_embeds = [] + for index, mm_embed in enumerate(active_embeddings): + active_embeddings[index], deepstack_embed = self.split_mm_embeds( + mm_embed, self.deepstack_num_level ) + deepstack_embeds.extend(deepstack_embed) + return active_embeddings, deepstack_embeds - # Prefer the indices the executor already computed (CPU-side - # `filter_mm_token_from_input_ids` + async H2D) and forwarded via - # kwargs; fall back to filtering only on engine-bypass paths - # (e.g., direct `forward` calls in unit tests). - text_token_indices = kwargs.get("text_token_indices") - mm_token_indices = kwargs.get("mm_token_indices") - if len(mm_embeds) > 0 and (text_token_indices is None or mm_token_indices is None): + def _fuse_multimodal_embeddings( + self, + *, + input_ids: torch.Tensor, + multimodal_embeddings: List[torch.Tensor], + mm_token_ids: Optional[Sequence[int] | torch.Tensor], + embedding_layer, + extra_embeds: Sequence[torch.Tensor], + text_token_indices: Optional[torch.Tensor] = None, + mm_token_indices: Optional[torch.Tensor] = None, + ) -> tuple[Optional[torch.Tensor], Optional[torch.Tensor], Sequence[torch.Tensor]]: + """Fuse primary embeddings and expand deepstack features in reusable scratch.""" + # Qwen only needs the explicit MM indices below when deepstack features + # must be scattered into its reusable buffer. Without extra embeds, + # `fuse_input_embeds` performs its normal index fallback inside `super()`. + if extra_embeds and (text_token_indices is None or mm_token_indices is None): text_token_indices, mm_token_indices = filter_mm_token_from_input_ids( input_ids, - vocab_size=self.llm.model.embed_tokens.num_embeddings, - mm_token_ids=self.mm_token_ids, - ) - - # Expand the per-level deepstack mm embeddings into the pre-allocated - # `(L, max_num_tokens, H)` buffer with a single packed scatter, - # avoiding `L` fresh `torch.zeros` + `L` scatters inside - # `fuse_input_embeds`. - if self.use_deepstack and len(deepstack_embeds) > 0: - num_tokens = input_ids.shape[0] - deepstack_buffer = self.deepstack_input_embeds[:, :num_tokens, :] - deepstack_buffer.zero_() - packed_deepstack = torch.stack(deepstack_embeds, dim=0) - deepstack_buffer[:, mm_token_indices, :] = packed_deepstack.to( - dtype=deepstack_buffer.dtype, device=deepstack_buffer.device + vocab_size=embedding_layer.num_embeddings, + mm_token_ids=mm_token_ids, ) - deepstack_embeds = list(deepstack_buffer.unbind(0)) - # Preserve the pre-fusion token IDs. `fuse_input_embeds` collapses - # input_ids -> None when MM embeddings are fused in, but spec - # decoding (MTP / Eagle) still needs the original prompt token - # IDs for drafter context preparation; pass them through as a - # dedicated kwarg consumed by `SpecDecOneEngineForCausalLM.forward`. - orig_input_ids = input_ids - - input_ids, input_embeds = fuse_input_embeds( - self.llm.model.embed_tokens, - input_ids, - mm_embeds, + fused_input_ids, inputs_embeds, _ = super()._fuse_multimodal_embeddings( + input_ids=input_ids, + multimodal_embeddings=multimodal_embeddings, + mm_token_ids=mm_token_ids, + embedding_layer=embedding_layer, + # Keep auxiliary fusion out of the generic path: passing non-empty + # `extra_embeds` would allocate and scatter one full-sequence tensor + # per deepstack level before Qwen replaces them with its buffer views. + extra_embeds=(), text_token_indices=text_token_indices, mm_token_indices=mm_token_indices, ) + if not extra_embeds: + return fused_input_ids, inputs_embeds, () - output_prob = self.llm.forward( - attn_metadata=attn_metadata, - input_ids=input_ids, - position_ids=position_ids, - inputs_embeds=input_embeds, - return_context_logits=return_context_logits, - deepstack_embeds=deepstack_embeds, - mrope_config=mrope_config, - spec_metadata=kwargs.get("spec_metadata"), - resource_manager=kwargs.get("resource_manager"), - orig_input_ids=orig_input_ids, + # Expand the per-level deepstack mm embeddings into the pre-allocated + # `(L, max_num_tokens, H)` buffer with a single packed scatter, avoiding `L` fresh + # `torch.zeros` + `L` scatters inside `fuse_input_embeds`. + deepstack_buffer = self.deepstack_input_embeds[:, : input_ids.shape[0], :] + deepstack_buffer.zero_() + packed_deepstack = torch.stack(tuple(extra_embeds), dim=0) + deepstack_buffer[:, mm_token_indices, :] = packed_deepstack.to( + dtype=deepstack_buffer.dtype, + device=deepstack_buffer.device, ) - # Spec-decoding (MTP / Eagle) returns a dict (accepted tokens, - # draft tokens, logits); plain forward returns a tensor. - if hasattr(output_prob, "shape"): - logger.debug(f"output shape: {output_prob.shape}") - return output_prob + return fused_input_ids, inputs_embeds, tuple(deepstack_buffer.unbind(0)) + + def get_language_model_extra_forward_kwargs( + self, + *, + raw_input_ids: Optional[torch.Tensor], + position_ids: Optional[torch.Tensor], + mm_inputs: PreparedLlmInputs, + multimodal_params: List[MultimodalParams], + num_generation_requests: int, + spec_metadata: Any, + resource_manager: Any = None, + mrope_delta_write_seq_slots: Optional[torch.Tensor] = None, + mrope_delta_read_seq_slots: Optional[torch.Tensor] = None, + mm_token_indices: Optional[torch.Tensor] = None, + **forward_kwargs: Any, + ) -> Dict[str, Any]: + """Build Qwen3-VL-specific language-model forward arguments.""" + mrope_config = {} + if not self.model_config.pretrained_config.disable_fuse_rope: + mrope_config = self.prepare_mrope_config( + multimodal_params, + num_generation_requests, + position_ids, + mrope_delta_write_seq_slots=mrope_delta_write_seq_slots, + mrope_delta_read_seq_slots=mrope_delta_read_seq_slots, + ) + + deepstack_embeds = list(mm_inputs.extra_embeds) + # `prepare_multimodal_inputs` passes these through `fuse_input_embeds`, which has already + # expanded each packed deepstack feature to the full input sequence. Do not scatter them + # a second time here: their leading dimension is now `num_tokens`, not the number of + # multimodal placeholders. + + return { + "deepstack_embeds": deepstack_embeds, + "mrope_config": mrope_config, + "spec_metadata": spec_metadata, + "resource_manager": resource_manager, + "orig_input_ids": raw_input_ids, + } def _get_requests_with_mm_data(self, multimodal_params): mm_multimodal_params = [] @@ -1556,6 +1567,8 @@ def _get_requests_with_mm_data(self, multimodal_params): ), ) class Qwen3VLModel(Qwen3VLModelBase): + supports_encoder_cache = True + def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): # NOTE: HF implementation. kwargs["vision_model_class"] = Qwen3VisionModel diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl_moe.py b/tensorrt_llm/_torch/models/modeling_qwen3vl_moe.py index fd7fd20f323f..41912d28a0f9 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl_moe.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl_moe.py @@ -45,6 +45,8 @@ ), ) class Qwen3MoeVLModel(Qwen3VLModelBase): + supports_encoder_cache = True + def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): # NOTE: HF implementation. kwargs["vision_model_class"] = Qwen3VisionModel diff --git a/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py b/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py index 92d3cccb5550..aacb121f0d66 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py +++ b/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py @@ -1,12 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from dataclasses import replace from typing import Dict, List, Optional import torch from transformers import PretrainedConfig from tensorrt_llm._torch.models.modeling_multimodal_utils import _is_mm_disagg +from tensorrt_llm.llmapi.llm_args import MultimodalConfig from ...inputs import ( ContentFormat, @@ -40,10 +42,16 @@ class _QwenImageBenchModelMixin: @property def multimodal_data_device_paths(self) -> List[str]: + # Keep the mrope_config entries in sync with `_Qwen3_5VLModel` + # (modeling_qwen3_5.py): the shared Qwen-VL mRoPE seq-slot cache path + # consumes `mrope_position_deltas` on the model device, so the engine + # must move them H2D along with the rest of the multimodal payload. return [ "image.pixel_values", "video.pixel_values_videos", "multimodal_embedding", + "mrope_config.mrope_position_ids", + "mrope_config.mrope_position_deltas", ] @property @@ -89,6 +97,19 @@ def load_weights( ) class QwenImageBenchModel(_QwenImageBenchModelMixin, Qwen3VLModelBase): def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): + if model_config.multimodal_config is not None: + # dataclasses.replace() resets init=False fields, so preserve runtime metadata. + extra_attrs = model_config.extra_attrs + model_config = replace( + model_config, + multimodal_config=MultimodalConfig( + **model_config.multimodal_config.model_dump( + exclude={"encoder_cache_max_bytes"} + ), + encoder_cache_max_bytes=0, + ), + ) + model_config.extra_attrs = extra_attrs kwargs["vision_model_class"] = Qwen3VisionModel kwargs["disable_fuse_rope"] = kwargs.get("disable_fuse_rope", False) super().__init__(model_config, *args, **kwargs) diff --git a/tensorrt_llm/_torch/models/modeling_speculative.py b/tensorrt_llm/_torch/models/modeling_speculative.py index a3b8bb0c3f24..9821186dcb21 100755 --- a/tensorrt_llm/_torch/models/modeling_speculative.py +++ b/tensorrt_llm/_torch/models/modeling_speculative.py @@ -664,6 +664,7 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, spec_metadata: SpecMetadata | None = None, hidden_states: torch.Tensor | None = None, + all_rank_num_tokens: Optional[List[int]] = None, ) -> torch.Tensor: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError( @@ -676,18 +677,27 @@ def forward( assert hidden_states is not None - # NOTE: If hidden states from the target model have to be concatenated, - # we expect that to happen outside the model definition. This helps us - # avoid data-dependent control flow and gives us better CUDA graph - # coverage. - residual = None - hidden_states = torch.cat([inputs_embeds, hidden_states], dim=-1) - hidden_states = self.fc(hidden_states) - hidden_states, residual = self.layers[0](position_ids=position_ids, - hidden_states=hidden_states, - attn_metadata=attn_metadata, - residual=None, - spec_metadata=spec_metadata) + previous_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + if all_rank_num_tokens is not None: + attn_metadata.all_rank_num_tokens = all_rank_num_tokens + + try: + # NOTE: If hidden states from the target model have to be concatenated, + # we expect that to happen outside the model definition. This helps us + # avoid data-dependent control flow and gives us better CUDA graph + # coverage. + residual = None + hidden_states = torch.cat([inputs_embeds, hidden_states], dim=-1) + hidden_states = self.fc(hidden_states) + hidden_states, residual = self.layers[0]( + position_ids=position_ids, + hidden_states=hidden_states, + attn_metadata=attn_metadata, + residual=None, + spec_metadata=spec_metadata) + finally: + if all_rank_num_tokens is not None: + attn_metadata.all_rank_num_tokens = previous_all_rank_num_tokens return hidden_states, hidden_states @@ -854,8 +864,7 @@ def dspark_layer_window_size(use_swa: bool, swa_window: int, layer_types, return (swa_window - 1, swa_window - 1) -def dspark_markov_step_bias(prev_tokens: torch.Tensor, - markov_w1: torch.Tensor, +def dspark_markov_step_bias(prev_tokens: torch.Tensor, markov_w1: torch.Tensor, markov_w2: torch.Tensor) -> torch.Tensor: """Vanilla Markov head logit bias for one intra-block draft step. @@ -991,13 +1000,11 @@ def __init__(self, draft_config): # Confidence-scheduled verification is NOT implemented yet: the # confidence_proj weights are loaded (for the follow-up MR) but never # used, and drafting always proposes the full K tokens. - self._dspark_shift_label = bool(dflash_config.get('shift_label', - False)) + self._dspark_shift_label = bool(dflash_config.get('shift_label', False)) self._dspark_use_swa = bool(dflash_config.get('use_swa', False)) self._dspark_swa_window = int( dflash_config.get('swa_window_size', 0) or 0) - self._dspark_markov_rank = int(dflash_config.get('markov_rank', 0) - or 0) + self._dspark_markov_rank = int(dflash_config.get('markov_rank', 0) or 0) self._dspark_markov_head_type = str( dflash_config.get('markov_head_type', 'vanilla') or 'vanilla').lower() @@ -1145,8 +1152,10 @@ def apply_markov_chain_logits(self, return base_logits markov_w2 = self.markov_w2 if vocab_slice is None else \ self.markov_w2[vocab_slice] - return dspark_markov_chain_logits(base_logits, first_prev_tokens, - self.markov_w1, markov_w2, + return dspark_markov_chain_logits(base_logits, + first_prev_tokens, + self.markov_w1, + markov_w2, argmax_fn=argmax_fn) def _post_attention_gate(self, attn_output, gate_input, attn_mod, num_heads, @@ -1202,10 +1211,7 @@ def load_weights(self, weights: Dict, weight_mapper=None, **kwargs): # follow-up MR but are not used yet. dspark_keys = ('markov_w1.weight', 'markov_w2.weight', 'confidence_proj.weight', 'confidence_proj.bias') - dspark_weights = { - k: weights[k] - for k in dspark_keys if k in weights - } + dspark_weights = {k: weights[k] for k in dspark_keys if k in weights} if dspark_weights: weights = { k: v @@ -1697,9 +1703,8 @@ def dflash_forward( # layers ((-1, -1) == flash-attn default == no window otherwise). # KV index == token position in the pool, so this restricts draft # queries to the last swa_window context tokens + the block. - window_size = (self._dspark_layer_windows[layer_idx] - if layer_idx < len(self._dspark_layer_windows) else - (-1, -1)) + window_size = (self._dspark_layer_windows[layer_idx] if layer_idx + < len(self._dspark_layer_windows) else (-1, -1)) out = flash_attn_with_kvcache( q=Q_bshd, k_cache=layer_k_cache, @@ -2048,6 +2053,36 @@ def forward(self, ) +def external_drafter_config_kwargs(model_config, spec_config) -> dict: + """`ModelConfig.from_pretrained` kwargs for a one-model external drafter. + + The drafter is a separate checkpoint, so it gets its own `ModelConfig`; the + kwargs below are the execution-layout properties it must inherit from the + target engine it runs inside. + + `moe_load_balancer` is propagated for DSpark ONLY. DSpark's draft stages are + full DeepSeek-V4 blocks sharing the target's expert topology and layer-index + namespace (`layer_idx = num_hidden_layers + stage_id`), so they can register + into the target's EPLB manager. Other external drafters (PARD, DFlash, + draft-target) are independent checkpoints whose expert topology and layer + numbering need not match the target's, and whose EPLB configs would therefore + be keyed against a different namespace -- do not generalize this without + designing a per-drafter EPLB config domain and layer identity first. + """ + kwargs = dict( + trust_remote_code=True, + attn_backend=model_config.attn_backend, + moe_backend=model_config.moe_backend, + mapping=model_config.mapping, + spec_config=None, # Avoid recursive spec-dec + max_num_tokens=model_config.max_num_tokens, + moe_max_num_tokens=model_config.moe_max_num_tokens, + ) + if spec_config.spec_dec_mode.is_dspark(): + kwargs["moe_load_balancer"] = model_config.moe_load_balancer + return kwargs + + def get_draft_model(model_config, draft_config, lm_head, model): """Construct the draft model for the configured speculative-decoding mode (EAGLE3 / MTP / PARD / DFlash). The DFlash branch selects the Laguna drafter @@ -2082,6 +2117,22 @@ def get_draft_model(model_config, draft_config, lm_head, model): if any("Laguna" in arch for arch in draft_arches): return DFlashLagunaForCausalLM(draft_config) return DFlashForCausalLM(draft_config) + elif spec_dec_mode.is_dspark(): + # Lazy import to avoid a cycle (modeling_dspark -> modeling_deepseekv4 -> + # modeling_speculative). The DSpark draft reuses the target's aux streams. + # The draft stage count (n_mtp_layers) is not in the HF config, so derive + # it from the checkpoint's mtp.* namespace. + from .modeling_dspark import (DSparkForCausalLM, count_dspark_stages, + validate_dspark_eplb_layer_base) + num_stages = count_dspark_stages( + model_config.spec_config.speculative_model) + validate_dspark_eplb_layer_base(model_config, draft_config) + return DSparkForCausalLM( + draft_config, + getattr(model, "aux_stream_dict", None), + num_stages=num_stages, + block_size=model_config.spec_config.block_size, + ) elif spec_dec_mode.is_draft_target_one_model(): # Keep the draft LM head vocab-sharded so greedy draft sampling uses the # lighter TP gather (see SpecWorkerBase.greedy_sample_draft_with_tp_gather). @@ -2102,8 +2153,8 @@ class SpecDecOneEngineForCausalLM(DecoderModelForCausalLM[TModel, TConfig], def __init__(self, model: TModel, model_config: ModelConfig[TConfig], - hidden_size: Optional[int] = None, - vocab_size: Optional[int] = None): + hidden_size: int | None = None, + vocab_size: int | None = None) -> None: # Composite configs (e.g. vision-language wrappers) may not expose # hidden_size/vocab_size at the top level; callers can pass the # text-config values explicitly. @@ -2158,13 +2209,8 @@ def __init__(self, elif spec_config.spec_dec_mode.is_external_drafter(): self.draft_config = ModelConfig.from_pretrained( model_config.spec_config.speculative_model, - trust_remote_code=True, - attn_backend=model_config.attn_backend, - moe_backend=model_config.moe_backend, - mapping=model_config.mapping, - spec_config=None, # Avoid recursive spec-dec - max_num_tokens=model_config.max_num_tokens, - moe_max_num_tokens=model_config.moe_max_num_tokens) + **external_drafter_config_kwargs( + model_config, spec_config)) self.draft_config.quant_config.kv_cache_quant_algo = \ model_config.quant_config.kv_cache_quant_algo self.draft_config.extra_attrs = model_config.extra_attrs @@ -2287,7 +2333,8 @@ def load_draft_weights(self, if self.spec_config and ( not self.spec_config.spec_dec_mode.is_external_drafter() - or self.spec_config.spec_dec_mode.is_dflash()): + or self.spec_config.spec_dec_mode.is_dflash() + or self.spec_config.spec_dec_mode.is_dspark()): self.draft_model.load_weights_from_target_model(self) def set_guided_decoder(self, diff --git a/tensorrt_llm/_torch/models/modeling_step3p7.py b/tensorrt_llm/_torch/models/modeling_step3p7.py index 0a17e7fd0227..e9e8378c14fa 100644 --- a/tensorrt_llm/_torch/models/modeling_step3p7.py +++ b/tensorrt_llm/_torch/models/modeling_step3p7.py @@ -377,8 +377,8 @@ class Step3p7MoeRoutingMethod(MiniMaxM2MoeRoutingMethod): Inherits from ``MiniMaxM2MoeRoutingMethod`` so the TRTLLMGen ``_extract_routing_params`` helper recognises us via ``isinstance`` and - feeds the bias pointer to the kernel. The MiniMax2 C++ routing path - hard-codes ``routeScale = 1.0f`` (see ``runner.cu``), so + feeds the bias pointer to the kernel. The generic MiniMax2 metadata does + not supply a route scale and therefore defaults to ``1.0f``, so ``routed_scaling_factor`` is applied to the MoE output in ``Step3p7MoE.forward`` instead of inside the kernel. """ @@ -903,7 +903,7 @@ def forward( all_rank_num_tokens=attn_metadata.all_rank_num_tokens, use_dp_padding=False, ) - # TRTLLMGen MiniMax2 kernel hard-codes routeScale=1.0, so apply + # Step3p7 uses the generic MiniMax2 metadata with routeScale=1.0, so apply # ``routed_scaling_factor`` to the MoE output here (mathematically # equivalent to scaling each topk weight). if self.routed_scaling_factor != 1.0: diff --git a/tensorrt_llm/_torch/models/modeling_t5.py b/tensorrt_llm/_torch/models/modeling_t5.py index 5a8a58083346..07891f605af4 100644 --- a/tensorrt_llm/_torch/models/modeling_t5.py +++ b/tensorrt_llm/_torch/models/modeling_t5.py @@ -31,7 +31,7 @@ """ import math -from typing import Dict, Optional +from typing import Dict, Optional, Tuple, Union import torch import torch.nn.functional as F @@ -43,6 +43,7 @@ from ..attention_backend import AttentionMetadata from ..attention_backend.interface import PositionalEmbeddingParams, PredefinedAttentionMask +from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE from ..model_config import ModelConfig from ..modules.attention import Attention from ..modules.cross_attention import CrossAttention @@ -112,8 +113,12 @@ def _gelu_new(x: torch.Tensor) -> torch.Tensor: def _t5_gated_act_fn(config: T5Config): act_fn = _t5_dense_act_fn(config) + act_name = getattr(config, "dense_act_fn", None) or "relu" + use_fused_gelu = act_name == "gelu_new" and IS_FLASHINFER_AVAILABLE def gated_act_fn(hidden_states: torch.Tensor) -> torch.Tensor: + if use_fused_gelu and hidden_states.dtype in (torch.float16, torch.bfloat16): + return torch.ops.trtllm.flashinfer_gelu_tanh_and_mul(hidden_states) gate, up = hidden_states.chunk(2, dim=-1) return act_fn(gate) * up @@ -152,19 +157,36 @@ def __init__( super().__init__(hidden_size=hidden_size, eps=eps, dtype=dtype) self._use_hopper_rms_norm: Optional[bool] = None - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + def forward( + self, + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: if self._use_hopper_rms_norm is None and hidden_states.is_cuda: sm_version = get_sm_version() self._use_hopper_rms_norm = 90 <= sm_version < 100 + if residual is not None and hidden_states.dtype == torch.float16: + hidden_states = _clamp_fp16_infs(hidden_states + residual) + return self.forward(hidden_states), hidden_states + if self._use_hopper_rms_norm and hidden_states.dtype in (torch.float16, torch.bfloat16): - return super().forward(hidden_states) + if residual is None: + return super().forward(hidden_states) + return super().forward(hidden_states, residual) + + if residual is not None: + hidden_states = hidden_states + residual + residual = hidden_states variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) if self.weight.dtype in (torch.float16, torch.bfloat16): hidden_states = hidden_states.to(self.weight.dtype) - return self.weight * hidden_states + hidden_states = self.weight * hidden_states + if residual is None: + return hidden_states + return hidden_states, residual def _t5_encoder_num_layers(config: T5Config) -> int: @@ -472,10 +494,14 @@ def forward( attn_metadata: AttentionMetadata, position_ids: Optional[torch.IntTensor] = None, position_bias: Optional[torch.Tensor] = None, + residual: Optional[torch.Tensor] = None, **kwargs, - ) -> torch.Tensor: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) + ) -> Tuple[torch.Tensor, torch.Tensor]: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) hidden_states = self.self_attn( position_ids=position_ids, @@ -484,16 +510,10 @@ def forward( attention_mask=PredefinedAttentionMask.FULL, position_bias=position_bias, ) - hidden_states = residual + hidden_states - hidden_states = _clamp_fp16_infs(hidden_states) - - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - hidden_states = _clamp_fp16_infs(hidden_states) - return hidden_states + return hidden_states, residual # --------------------------------------------------------------------------- @@ -570,11 +590,15 @@ def forward( position_bias: Optional[torch.Tensor] = None, relative_attention_bias: Optional[torch.Tensor] = None, relative_attention_max_distance: int = 0, + residual: Optional[torch.Tensor] = None, **kwargs, - ) -> torch.Tensor: + ) -> Tuple[torch.Tensor, torch.Tensor]: # Self-attention (pre-norm) - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) hidden_states = self.self_attn( position_ids=position_ids, hidden_states=hidden_states, @@ -584,12 +608,9 @@ def forward( relative_attention_bias=relative_attention_bias, relative_attention_max_distance=relative_attention_max_distance, ) - hidden_states = residual + hidden_states - hidden_states = _clamp_fp16_infs(hidden_states) # Cross-attention (pre-norm) - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) hidden_states = self.cross_attn( hidden_states=hidden_states, encoder_hidden_states=encoder_hidden_states, @@ -597,17 +618,12 @@ def forward( cross_attn_metadata=cross_attn_metadata, skip_cross_kv_projection=skip_cross_kv_projection, ) - hidden_states = residual + hidden_states - hidden_states = _clamp_fp16_infs(hidden_states) # MLP (pre-norm) - residual = hidden_states - hidden_states = self.cross_attn_layernorm(hidden_states) + hidden_states, residual = self.cross_attn_layernorm(hidden_states, residual) hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - hidden_states = _clamp_fp16_infs(hidden_states) - return hidden_states + return hidden_states, residual # --------------------------------------------------------------------------- @@ -654,14 +670,16 @@ def forward( seq_len = hidden_states.shape[0] if seq_lens is None else int(seq_lens.max().item()) position_bias = self.relative_position_bias(seq_len, seq_len, hidden_states.device) + residual = None for layer in self.layers: - hidden_states = layer( + hidden_states, residual = layer( hidden_states=hidden_states, attn_metadata=attn_metadata, position_ids=position_ids, position_bias=position_bias, + residual=residual, ) - hidden_states = self.final_layernorm(hidden_states) + hidden_states, _ = self.final_layernorm(hidden_states, residual) return hidden_states @@ -716,8 +734,9 @@ def forward( ) relative_attention_max_distance = self.relative_position_bias.max_distance + residual = None for layer in self.layers: - hidden_states = layer( + hidden_states, residual = layer( position_ids=position_ids, hidden_states=hidden_states, attn_metadata=attn_metadata, @@ -727,8 +746,9 @@ def forward( position_bias=position_bias, relative_attention_bias=relative_attention_bias, relative_attention_max_distance=relative_attention_max_distance, + residual=residual, ) - hidden_states = self.final_layernorm(hidden_states) + hidden_states, _ = self.final_layernorm(hidden_states, residual) return hidden_states diff --git a/tensorrt_llm/_torch/models/modeling_utils.py b/tensorrt_llm/_torch/models/modeling_utils.py index 237ed00af403..eb91a6456209 100755 --- a/tensorrt_llm/_torch/models/modeling_utils.py +++ b/tensorrt_llm/_torch/models/modeling_utils.py @@ -378,6 +378,72 @@ class DecoderModelForCausalLM(nn.Module, Generic[TModel, TConfig], metaclass=PostInitCaller): + @staticmethod + def _checkpoint_has_lm_head_scale(config: ModelConfig[TConfig]) -> bool: + """Whether the checkpoint stores a quantized lm_head (a weight scale). + + Used to decide lm_head quantization for homogeneous checkpoints, which + carry no explicit per-layer quant entry. Reads only the safetensors + header for ``lm_head.weight_scale`` (no weight load). + """ + checkpoint_dir = getattr(config.pretrained_config, "_name_or_path", + None) + if not checkpoint_dir or not os.path.isdir(checkpoint_dir): + return False + return ModelConfig._get_safetensors_header_for_tensor( + checkpoint_dir, "lm_head.weight_scale") is not None + + @staticmethod + def _resolve_lm_head_quant_config( + config: ModelConfig[TConfig]) -> Optional[QuantConfig]: + """Resolve the quant config for lm_head, or None to keep it unquantized. + + lm_head is quantized only when the checkpoint actually stores a + quantized lm_head: MIXED_PRECISION checkpoints carry an explicit + per-layer entry in ``quant_config_dict``; homogeneous checkpoints have + no per-layer entry, so we additionally require the checkpoint to carry + an ``lm_head`` weight scale. (A bf16 lm_head is commonly left OUT of + ``exclude_modules``, so "not excluded" alone must NOT imply quantized — + otherwise a homogeneous checkpoint with a bf16 lm_head would be built + quantized and fail / change its logits.) Several further cases force it + back to unquantized. + """ + if config.quant_config_dict is not None: + quant_config = config.quant_config_dict.get("lm_head") + elif (config.quant_config is not None + and config.quant_config.quant_algo is not None and + DecoderModelForCausalLM._checkpoint_has_lm_head_scale(config)): + quant_config = config.quant_config + else: + quant_config = None + if quant_config is None: + return None + + # exclude_modules always wins (an FP16 lm_head is listed there). + if (config.quant_config is not None + and config.quant_config.is_module_excluded_from_quantization( + "lm_head")): + return None + + # Tied embeddings replace lm_head.weight with the dense bf16 embedding + # weight, which would silently clash with a quantized (packed) weight. + if getattr(config.pretrained_config, 'tie_word_embeddings', False): + logger.info("Ignoring lm_head quant entry: tie_word_embeddings " + "shares the dense embedding weight, so lm_head stays " + "unquantized") + return None + + # lm_head TP in ADP slices the dense weight at forward time for the + # spec-decoding head (see LMHead.forward), which is incompatible with + # quantized (packed) weights — LMHead rejects that at construction. + if (config.mapping.enable_attention_dp + and config.mapping.enable_lm_head_tp_in_adp): + logger.info("Ignoring lm_head quant entry: lm_head TP in ADP " + "slices the dense weight, so lm_head stays unquantized") + return None + + return quant_config + def __init__(self, model: TModel, *, config: ModelConfig[TConfig], hidden_size: int, vocab_size: int): super().__init__() @@ -387,11 +453,16 @@ def __init__(self, model: TModel, *, config: ModelConfig[TConfig], self.pp_size = config.mapping.pp_size self.has_custom_lm_head = False + # Quant config for lm_head (applies to both the attention-DP replicated + # and TP lm_head below); None keeps it unquantized. + lm_head_quant_config = self._resolve_lm_head_quant_config(config) + if config.mapping.enable_attention_dp and not config.mapping.enable_lm_head_tp_in_adp: self.lm_head = LMHead( vocab_size, hidden_size, dtype=config.pretrained_config.torch_dtype, + quant_config=lm_head_quant_config, ) else: if (hasattr(config, 'lora_config') @@ -405,29 +476,10 @@ def __init__(self, model: TModel, *, config: ModelConfig[TConfig], self.has_custom_lm_head = True vocab_size = lora_loader.vocab_size - # Per-layer quant entry for lm_head (e.g. ModelOpt MIXED_PRECISION - # checkpoints that quantize lm_head to NVFP4). Model-specific - # config normalizers opt in by keeping/synthesizing this entry; - # exclude_modules still wins. - lm_head_quant_config = None - if not self.has_custom_lm_head and config.quant_config_dict is not None: - lm_head_quant_config = config.quant_config_dict.get("lm_head") - if (lm_head_quant_config is not None - and config.quant_config is not None - and config.quant_config. - is_module_excluded_from_quantization("lm_head")): - lm_head_quant_config = None - if (lm_head_quant_config is not None - and getattr(config.pretrained_config, - 'tie_word_embeddings', False)): - # Tied embeddings replace lm_head.weight with the dense - # bf16 embedding weight below, which would silently clash - # with a quantized (packed) weight and its quant method. - logger.info( - "Ignoring lm_head quant entry: tie_word_embeddings " - "shares the dense embedding weight, so lm_head stays " - "unquantized") - lm_head_quant_config = None + # A custom LoRA lm_head replaces the checkpoint weight with a + # dense bf16 tensor, so the quant entry must not apply. + if self.has_custom_lm_head: + lm_head_quant_config = None self.lm_head = LMHead( vocab_size, @@ -1132,16 +1184,28 @@ def load_single_module(name, module): module.load_weights( weights=[module_weights], allow_partial_loading=allow_partial_loading) + loaded_own_params = None else: + loaded_own_params = [] for n, p in module.named_parameters(recurse=False): if not allow_partial_loading: assert n in module_weights if n in module_weights: p.data.copy_(module_weights[n][:]) - - # Mark consumed weights + loaded_own_params.append(n) + + # Only a module handed the full `name.*` subtree may + # consume it wholesale. Otherwise just its own + # `recurse=False` params were loaded, and since + # `named_modules()` is pre-order, consuming the subtree + # would drop weights its descendants have not loaded yet -- + # they would silently keep uninitialized weights. if hasattr(weights, 'mark_consumed'): - weights.mark_consumed(name) + if loaded_own_params is None: + weights.mark_consumed(name) + elif loaded_own_params: + weights.mark_consumed_keys( + f'{name}.{n}' for n in loaded_own_params) if os.environ.get("TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL", "False") in ["True", "true", "1", "yes", "y"]: @@ -1233,6 +1297,8 @@ def load_single_module(name, module): module_name, module_weights, allow_partial_loading=allow_partial_loading) + # Handed the full subtree, like the `load_weights` case. + loaded_own_params = None elif hasattr(module, 'load_weights'): if "linear_attn.conv1d" in name: module_weights['weight'] = module_weights[ @@ -1245,7 +1311,9 @@ def load_single_module(name, module): module.load_weights( weights=[module_weights], allow_partial_loading=allow_partial_loading) + loaded_own_params = None else: + loaded_own_params = [] for n, p in module.named_parameters(recurse=False): weight_mapper.handle_manual_copy( module_name, @@ -1253,10 +1321,16 @@ def load_single_module(name, module): n, p, allow_partial_loading=allow_partial_loading) + loaded_own_params.append(n) - # Mark consumed weights + # Consume precisely what was loaded; see the matching + # comment in `_load_weights_impl`. if hasattr(weights, 'mark_consumed'): - weights.mark_consumed(name) + if loaded_own_params is None: + weights.mark_consumed(name) + elif loaded_own_params: + weights.mark_consumed_keys( + f'{name}.{n}' for n in loaded_own_params) if os.environ.get("TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL", "False") in ["True", "true", "1", "yes", "y"]: diff --git a/tensorrt_llm/_torch/models/modeling_whisper.py b/tensorrt_llm/_torch/models/modeling_whisper.py new file mode 100644 index 000000000000..420b7ea6f9de --- /dev/null +++ b/tensorrt_llm/_torch/models/modeling_whisper.py @@ -0,0 +1,1088 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch-flow Whisper encoder-decoder (ASR) model for TensorRT-LLM. + +Covers `WhisperForConditionalGeneration` (OpenAI Whisper). + +Whisper is an audio encoder-decoder: a log-mel spectrogram is consumed by a +convolutional + Transformer audio encoder, and a text decoder attends to the +encoder output via cross-attention while generating a transcript +autoregressively. + +The request carries the raw 30 s-padded waveform (not the mel): the input +processor only pads/validates audio on the host, and the encoder computes the +log-mel spectrogram on GPU inside the engine process (batched across the +encoder step; see `WhisperLogMelFrontend`). + +Key differences from BART: + - Pre-norm (LayerNorm → sub-layer → residual add) instead of post-norm. + - The encoder ingests a mel-feature tensor through a 2x `Conv1d` stem + (the second conv has stride 2, halving the time axis), not token ids. + - Encoder and decoder embeddings are NOT tied (the encoder has no vocab + embedding); `lm_head` (`proj_out`) is tied to the decoder token + embedding. + - Learned absolute positional embeddings with NO index offset (BART uses 2). + - No embedding scale. + - `k_proj` has no bias (`q_proj`/`v_proj`/`out_proj` do); the missing + key bias is materialized as zeros at weight-load time. +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn +from transformers import AutoProcessor, WhisperConfig + +from ...inputs import ( + ExtraProcessedInputs, + InputProcessor, + MultimodalPlaceholderMetadata, + TextPrompt, + register_input_processor, +) +from ...logger import logger +from ...sampling_params import SamplingParams +from ..attention_backend import AttentionMetadata +from ..attention_backend.interface import PredefinedAttentionMask +from ..model_config import ModelConfig +from ..modules.attention import Attention +from ..modules.cross_attention import CrossAttention +from ..modules.embedding import Embedding, LMHead +from ..modules.layer_norm import LayerNorm +from ..modules.linear import TensorParallelMode +from ..modules.logits_processor import LogitsProcessor +from ..modules.mlp import MLP +from .modeling_utils import PostInitCaller, register_auto_model + +if TYPE_CHECKING: + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + +# WhisperConfig carries no layer-norm epsilon; HF builds every Whisper LayerNorm +# as `nn.LayerNorm(embed_dim)`, i.e. torch's default. +_LAYER_NORM_EPS = 1e-5 + +# --------------------------------------------------------------------------- +# Config helpers +# --------------------------------------------------------------------------- + + +def _packed_position_ids( + position_ids: Optional[torch.IntTensor], + hidden_states: torch.Tensor, +) -> Optional[torch.IntTensor]: + if position_ids is None: + return None + + position_ids = position_ids.reshape(-1) + if position_ids.numel() != hidden_states.shape[0]: + raise ValueError( + "Whisper packed position_ids must match hidden_states tokens: " + f"got {position_ids.numel()} positions for {hidden_states.shape[0]} tokens." + ) + return position_ids + + +# --------------------------------------------------------------------------- +# Whisper Attention +# --------------------------------------------------------------------------- + + +class WhisperSelfAttention(Attention): + """Whisper-style MHA with bias and no in-kernel positional encoding. + + Whisper adds learned positional embeddings to the input before the + attention layer, so no RoPE or other in-kernel positional encoding is + used. Query scaling by `head_dim**-0.5` is applied inside the standard + attention path (`q_scaling=1.0`), which is numerically identical to + Whisper's convention of pre-scaling the query and using `scaling=1.0`. + """ + + def __init__( + self, + model_config: ModelConfig[WhisperConfig], + num_heads: int, + max_positions: int, + layer_idx: Optional[int] = None, + ): + config = model_config.pretrained_config + super().__init__( + hidden_size=config.d_model, + num_attention_heads=num_heads, + num_key_value_heads=num_heads, + max_position_embeddings=max_positions, + bias=True, + pos_embd_params=None, + layer_idx=layer_idx, + dtype=config.torch_dtype, + config=model_config, + ) + + def apply_rope(self, q, k, v, position_ids): + """Whisper uses learned pos embeddings, not RoPE — pass through.""" + return q, k, v + + +class WhisperCrossAttention(CrossAttention): + """Whisper-style cross-attention with bias (decoder attends to encoder).""" + + def __init__( + self, + model_config: ModelConfig[WhisperConfig], + layer_idx: Optional[int] = None, + ): + config = model_config.pretrained_config + num_heads = config.decoder_attention_heads + super().__init__( + hidden_size=config.d_model, + num_attention_heads=num_heads, + num_key_value_heads=num_heads, + encoder_hidden_size=config.d_model, + bias=True, + layer_idx=layer_idx, + dtype=config.torch_dtype, + config=model_config, + ) + + +# --------------------------------------------------------------------------- +# Encoder layer (pre-norm) +# --------------------------------------------------------------------------- + + +class WhisperEncoderLayer(nn.Module): + """Whisper encoder layer (pre-norm): LN → self-attn → add → LN → MLP → add.""" + + def __init__( + self, + model_config: ModelConfig[WhisperConfig], + layer_idx: int, + ): + super().__init__() + config = model_config.pretrained_config + hidden_size = config.d_model + + self.self_attn = WhisperSelfAttention( + model_config, + num_heads=config.encoder_attention_heads, + max_positions=config.max_source_positions, + layer_idx=layer_idx, + ) + self.self_attn_layer_norm = LayerNorm( + hidden_size=hidden_size, + eps=_LAYER_NORM_EPS, + dtype=config.torch_dtype, + has_bias=True, + ) + self.mlp = MLP( + hidden_size=hidden_size, + intermediate_size=config.encoder_ffn_dim, + bias=True, + activation=F.gelu, + dtype=config.torch_dtype, + config=model_config, + layer_idx=layer_idx, + ) + self.final_layer_norm = LayerNorm( + hidden_size=hidden_size, + eps=_LAYER_NORM_EPS, + dtype=config.torch_dtype, + has_bias=True, + ) + + def forward( + self, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + position_ids: Optional[torch.IntTensor] = None, + **kwargs, + ) -> torch.Tensor: + # Self-attention (pre-norm, bidirectional/FULL mask) + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + hidden_states = self.self_attn( + position_ids=position_ids, + hidden_states=hidden_states, + attn_metadata=attn_metadata, + attention_mask=PredefinedAttentionMask.FULL, + ) + + # MLP (pre-norm); self-attn residual add fused into the LayerNorm + hidden_states, residual = self.final_layer_norm(hidden_states, residual) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + # HF parity: clamp fp16 activations so next LayerNorm doesn't see inf (which becomes NaN) + if hidden_states.dtype == torch.float16: + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + return hidden_states + + +# --------------------------------------------------------------------------- +# Decoder layer (pre-norm) +# --------------------------------------------------------------------------- + + +class WhisperDecoderLayer(nn.Module): + """Whisper decoder layer (pre-norm): self-attn → cross-attn → MLP.""" + + def __init__( + self, + model_config: ModelConfig[WhisperConfig], + layer_idx: int, + ): + super().__init__() + config = model_config.pretrained_config + hidden_size = config.d_model + + self.self_attn = WhisperSelfAttention( + model_config, + num_heads=config.decoder_attention_heads, + max_positions=config.max_target_positions, + layer_idx=layer_idx, + ) + self.self_attn_layer_norm = LayerNorm( + hidden_size=hidden_size, + eps=_LAYER_NORM_EPS, + dtype=config.torch_dtype, + has_bias=True, + ) + + self.cross_attn = WhisperCrossAttention(model_config, layer_idx=layer_idx) + self.cross_attn_layer_norm = LayerNorm( + hidden_size=hidden_size, + eps=_LAYER_NORM_EPS, + dtype=config.torch_dtype, + has_bias=True, + ) + + self.mlp = MLP( + hidden_size=hidden_size, + intermediate_size=config.decoder_ffn_dim, + bias=True, + activation=F.gelu, + dtype=config.torch_dtype, + config=model_config, + layer_idx=layer_idx, + ) + self.final_layer_norm = LayerNorm( + hidden_size=hidden_size, + eps=_LAYER_NORM_EPS, + dtype=config.torch_dtype, + has_bias=True, + ) + + def forward( + self, + position_ids: torch.IntTensor, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + encoder_hidden_states: Optional[torch.Tensor] = None, + cross_attn_metadata: Optional[AttentionMetadata] = None, + skip_cross_kv_projection: bool = False, + **kwargs, + ) -> torch.Tensor: + # Self-attention (pre-norm, causal) + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + hidden_states = self.self_attn( + position_ids=position_ids, + hidden_states=hidden_states, + attn_metadata=attn_metadata, + attention_mask=PredefinedAttentionMask.CAUSAL, + ) + + # Cross-attention (pre-norm); self-attn residual add fused into the LayerNorm + hidden_states, residual = self.cross_attn_layer_norm(hidden_states, residual) + hidden_states = self.cross_attn( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + attn_metadata=attn_metadata, + cross_attn_metadata=cross_attn_metadata, + skip_cross_kv_projection=skip_cross_kv_projection, + ) + + # MLP (pre-norm); cross-attn residual add fused into the LayerNorm + hidden_states, residual = self.final_layer_norm(hidden_states, residual) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +# --------------------------------------------------------------------------- +# Encoder / Decoder stacks +# --------------------------------------------------------------------------- + + +def _load_hf_feature_extractor(config: WhisperConfig): + """The checkpoint's `WhisperFeatureExtractor` (`preprocessor_config.json`). + + STFT/mel parameters live there, not in the model config, so they are + re-loaded from `config.name_or_path`. Falls back to Whisper defaults + with `config.num_mel_bins` filters when the checkpoint ships no + preprocessor config or its `feature_size` contradicts the model config. + """ + from transformers import WhisperFeatureExtractor + + extractor = None + name_or_path = config.name_or_path + if name_or_path: + try: + extractor = WhisperFeatureExtractor.from_pretrained(name_or_path) + except (OSError, ValueError) as e: + logger.warning( + f"Could not load a Whisper feature-extractor config from " + f"{name_or_path!r} ({e}); using Whisper default STFT/mel parameters." + ) + if extractor is not None and int(extractor.feature_size) != int(config.num_mel_bins): + logger.warning( + f"preprocessor_config.json feature_size ({extractor.feature_size}) " + f"contradicts config.num_mel_bins ({config.num_mel_bins}); using " + f"Whisper default STFT/mel parameters with {config.num_mel_bins} mel bins." + ) + extractor = None + if extractor is None: + extractor = WhisperFeatureExtractor(feature_size=config.num_mel_bins) + return extractor + + +class WhisperLogMelFrontend(nn.Module): + """GPU log-mel spectrogram front-end, numerics-identical to the HF + `WhisperFeatureExtractor` torch path (`_torch_extract_fbank_features`). + + Consumes the raw zero-padded waveform batch shipped by + `WhisperInputProcessor` and produces `[batch, num_mel_bins, frames]` + in fp32 (STFT precision; the caller casts to the model dtype). Kept as a + separate module so a future encoder CUDA-graph capture can choose to keep + the STFT outside the graphed region. + + STFT/mel parameters and the filterbank come from the checkpoint's feature + extractor. + """ + + def __init__(self, config: WhisperConfig): + super().__init__() + extractor = _load_hf_feature_extractor(config) + self.n_fft = int(extractor.n_fft) + self.hop_length = int(extractor.hop_length) + # Pre-STFT Gaussian noise, applied where HF applies it; 0.0 (all + # official checkpoints) disables it. + self.dither = float(getattr(extractor, "dither", 0.0)) + self._mel_filters_np = extractor.mel_filters + # Materialized lazily on the input device (NOT register_buffer: these + # are derived constants that must stay fp32 and out of the state dict, + # and lazy creation sidesteps meta-device module initialization). + self._mel_filters: Optional[torch.Tensor] = None + self._window: Optional[torch.Tensor] = None + + def forward(self, waveforms: torch.Tensor) -> torch.Tensor: + # waveforms: [batch, n_samples] fp32, zero-padded to the fixed window. + if self._mel_filters is None or self._mel_filters.device != waveforms.device: + self._mel_filters = torch.from_numpy(self._mel_filters_np).to( + waveforms.device, torch.float32 + ) + self._window = torch.hann_window(self.n_fft, device=waveforms.device) + + # Follow HF's fp32 reference path: the STFT and mel matmul below lose + # accuracy in half precision — fp16 flushes the 1e-10 log floor to zero, + # bf16 lacks the mantissa bits. + waveforms = waveforms.to(torch.float32) + if self.dither != 0.0: + # Out-of-place: `.to(torch.float32)` above is a no-op for an already-fp32 + # request, so an in-place add would corrupt the caller's audio buffer. + waveforms = waveforms + self.dither * torch.randn_like(waveforms) + stft = torch.stft( + waveforms, + self.n_fft, + self.hop_length, + window=self._window, + return_complex=True, + ) + magnitudes = stft[..., :-1].abs() ** 2 + mel_spec = self._mel_filters.T @ magnitudes + log_spec = torch.clamp(mel_spec, min=1e-10).log10_() + # Per-sample dynamic-range floor (batched samples must not share a max). + max_val = log_spec.max(dim=2, keepdim=True)[0].max(dim=1, keepdim=True)[0] + log_spec = torch.maximum(log_spec, max_val - 8.0) + return (log_spec + 4.0) / 4.0 + + +class WhisperEncoder(nn.Module): + """Whisper audio encoder: log-mel front-end + 2x Conv1d stem + positions + + self-attn layers.""" + + def __init__(self, model_config: ModelConfig[WhisperConfig]): + super().__init__() + config = model_config.pretrained_config + embed_dim = config.d_model + + self.log_mel = WhisperLogMelFrontend(config) + + # 2x Conv1d stem. conv2 has stride 2, halving the time axis + # (3000 mel frames -> 1500 encoder positions). + self.conv1 = nn.Conv1d( + config.num_mel_bins, + embed_dim, + kernel_size=3, + padding=1, + dtype=config.torch_dtype, + ) + self.conv2 = nn.Conv1d( + embed_dim, + embed_dim, + kernel_size=3, + stride=2, + padding=1, + dtype=config.torch_dtype, + ) + + # Learned absolute positional embedding for all source positions. + self.embed_positions = Embedding( + config.max_source_positions, + embed_dim, + dtype=config.torch_dtype, + ) + self.layers = nn.ModuleList( + [WhisperEncoderLayer(model_config, layer_idx=i) for i in range(config.encoder_layers)] + ) + self.layer_norm = LayerNorm( + hidden_size=embed_dim, + eps=_LAYER_NORM_EPS, + dtype=config.torch_dtype, + has_bias=True, + ) + + def forward( + self, + input_features: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + # input_features: [batch, n_samples] raw 30 s-padded waveform (the + # request contract), or a precomputed [batch, num_mel_bins, 3000] mel + # (kept for direct-callers/validation harnesses). + if input_features.dim() == 2: + input_features = self.log_mel(input_features) + input_features = input_features.to(self.conv1.weight.dtype) + inputs_embeds = F.gelu(self.conv1(input_features)) + inputs_embeds = F.gelu(self.conv2(inputs_embeds)) + # [batch, embed_dim, seq_len] -> [batch, seq_len, embed_dim] + inputs_embeds = inputs_embeds.permute(0, 2, 1) + + hidden_states = inputs_embeds + self.embed_positions.weight + # Pack to [num_tokens, embed_dim] for the attention backend. + hidden_states = hidden_states.reshape(-1, hidden_states.shape[-1]) + + for layer in self.layers: + hidden_states = layer(hidden_states=hidden_states, attn_metadata=attn_metadata) + + hidden_states = self.layer_norm(hidden_states) + return hidden_states + + +class WhisperDecoder(nn.Module): + """Whisper text decoder: token + positional embedding + decoder layers.""" + + def __init__(self, model_config: ModelConfig[WhisperConfig]): + super().__init__() + config = model_config.pretrained_config + + self.embed_tokens = Embedding( + config.vocab_size, + config.d_model, + dtype=config.torch_dtype, + mapping=model_config.mapping, + tensor_parallel_mode=TensorParallelMode.COLUMN, + gather_output=True, + ) + # Whisper decoder positions have NO index offset (unlike BART's +2). + self.embed_positions = Embedding( + config.max_target_positions, + config.d_model, + dtype=config.torch_dtype, + ) + self.layers = nn.ModuleList( + [WhisperDecoderLayer(model_config, layer_idx=i) for i in range(config.decoder_layers)] + ) + self.layer_norm = LayerNorm( + hidden_size=config.d_model, + eps=_LAYER_NORM_EPS, + dtype=config.torch_dtype, + has_bias=True, + ) + + def forward( + self, + input_ids: Optional[torch.IntTensor], + attn_metadata: AttentionMetadata, + position_ids: Optional[torch.IntTensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + cross_attn_metadata: Optional[AttentionMetadata] = None, + skip_cross_kv_projection: bool = False, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if inputs_embeds is None: + assert input_ids is not None + inputs_embeds = self.embed_tokens(input_ids) + + hidden_states = inputs_embeds + position_ids = _packed_position_ids(position_ids, hidden_states) + if position_ids is not None: + hidden_states = hidden_states + self.embed_positions(position_ids) + + for layer in self.layers: + hidden_states = layer( + position_ids=position_ids, + hidden_states=hidden_states, + attn_metadata=attn_metadata, + encoder_hidden_states=encoder_hidden_states, + cross_attn_metadata=cross_attn_metadata, + skip_cross_kv_projection=skip_cross_kv_projection, + ) + + hidden_states = self.layer_norm(hidden_states) + return hidden_states + + +# --------------------------------------------------------------------------- +# Input processor (audio -> padded waveform + forced decoder prompt) +# --------------------------------------------------------------------------- + + +class WhisperInputProcessor(InputProcessor): + """Host-side Whisper preprocessing for the LLM API. + + Validates and zero-pads `multi_modal_data["audio"]` to the fixed 30 s + window (the log-mel spectrogram itself is computed on GPU inside the + engine, see `WhisperLogMelFrontend`), and returns the forced + decoder prompt as the request's token ids. An empty text prompt selects + the checkpoint default + (`<|startoftranscript|>[<|en|>][<|transcribe|>]<|notimestamps|>`); a + non-empty text prompt is tokenized verbatim as the decoder prompt, which + is how language/task are overridden, e.g. + `"<|startoftranscript|><|de|><|transcribe|><|notimestamps|>"`. + Pre-tokenized `prompt_token_ids` are not consumed. + + The padded waveform rides `multimodal_data["audio"]` under + `encoder_input_features` + `encoder_output_len`, which + `executor_request_to_llm_request` forwards into the request's native + encoder fields. + """ + + # Marks this model as feature-driven for the encoder side: prompts + # without audio cannot be served and are rejected at submission. + requires_encoder_features = True + + def __init__(self, model_path, config, tokenizer, trust_remote_code: bool = True, **kwargs): + self.model_path = model_path + self.config = config + self.tokenizer = tokenizer + # No decoder-side placeholder tokens exist to hash against; skip the + # multimodal-hashing probe (the registry then always uses this + # processor directly). + self.multimodal_hashing_supported = False + # WhisperProcessor = feature extractor (the reference log-mel + # implementation) + tokenizer. Also consumed by accuracy evaluators + # via `llm.input_processor.processor`. + self.processor = AutoProcessor.from_pretrained( + model_path, trust_remote_code=trust_remote_code + ) + # Audio window/sampling rate from the checkpoint's feature extractor + # (WhisperLogMelFrontend reads the same config engine-side). Longer + # audio is rejected instead of silently truncated (long-form chunking + # is a separate feature). + extractor = self.processor.feature_extractor + self.sampling_rate = int(extractor.sampling_rate) + self.n_samples = int(extractor.n_samples) + self.max_audio_seconds = self.n_samples / float(self.sampling_rate) + # The mel frames halved by the conv stem must fill the encoder + # position table exactly, or every downstream cross-KV size is wrong. + hop_length = int(extractor.hop_length) + encoder_positions = self.n_samples // hop_length // 2 + if encoder_positions != int(self.config.max_source_positions): + raise ValueError( + f"Inconsistent Whisper checkpoint: the feature extractor's " + f"{self.max_audio_seconds:.1f}s window at {self.sampling_rate} Hz " + f"(hop {hop_length}) yields {encoder_positions} encoder " + f"positions, but config.max_source_positions is " + f"{self.config.max_source_positions}." + ) + self._decoder_prompt = self._build_decoder_prompt() + + # The decoder prompt contains no multimodal placeholder tokens (the audio + # feeds the encoder), so the embed-mask/cumsum machinery has nothing to + # find — returning None from all three makes it skip cleanly. + def get_vocab_size(self) -> Optional[int]: + return None + + def get_mm_token_ids(self) -> Optional[torch.Tensor]: + return None + + def get_mm_special_token_ids(self) -> Optional[torch.Tensor]: + return None + + def _build_decoder_prompt(self) -> List[int]: + """`[decoder_start] + forced task tokens` from the checkpoint.""" + start_id = getattr(self.config, "decoder_start_token_id", None) + if start_id is None: + raise ValueError( + "Whisper requires config.decoder_start_token_id to build the decoder prompt." + ) + try: + # Multilingual checkpoints force <|lang|><|task|>. Defaults are + # English transcription; user-facing overrides are a follow-up. + forced = self.processor.get_decoder_prompt_ids( + language="en", task="transcribe", no_timestamps=True + ) + except ValueError: + # English-only checkpoints (*.en) have no language/task tokens. + forced = self.processor.get_decoder_prompt_ids(no_timestamps=True) + return [int(start_id)] + [int(tok) for _, tok in sorted(forced)] + + def _resolve_decoder_prompt(self, prompt_text: Optional[str]) -> List[int]: + """Checkpoint-default forced prompt, or the user's decoder prompt. + + A non-empty text prompt is tokenized verbatim (special tokens + resolve to their ids) and must start with `<|startoftranscript|>`; + this is the language/task override mechanism. + """ + prompt_text = (prompt_text or "").strip() + if not prompt_text: + return list(self._decoder_prompt) + + decoder_prompt = self.processor.tokenizer.encode(prompt_text, add_special_tokens=False) + start_id = int(self.config.decoder_start_token_id) + if not decoder_prompt or decoder_prompt[0] != start_id: + raise ValueError( + "A Whisper text prompt overrides the decoder prompt and must " + f"start with <|startoftranscript|> (token {start_id}), e.g. " + "'<|startoftranscript|><|de|><|transcribe|><|notimestamps|>'; " + f"got {prompt_text[:80]!r}." + ) + max_prompt = int(self.config.max_target_positions) - 1 + if len(decoder_prompt) > max_prompt: + raise ValueError( + f"Whisper decoder prompt has {len(decoder_prompt)} tokens; at " + f"most {max_prompt} fit the decoder position table." + ) + return [int(token) for token in decoder_prompt] + + def _load_waveform(self, item: Any) -> np.ndarray: + if isinstance(item, str): + from ...inputs.utils import load_audio + + waveform, sample_rate = load_audio(item) + elif isinstance(item, dict) and "array" in item: + # HF datasets audio format: {"array": ..., "sampling_rate": ...} + waveform = item["array"] + sample_rate = item.get("sampling_rate", self.sampling_rate) + elif isinstance(item, (tuple, list)) and len(item) == 2: + waveform, sample_rate = item + else: + raise TypeError( + "Unsupported audio item for Whisper: expected a file " + "path/URL, an (array, sample_rate) tuple, or a dict with " + f"'array'/'sampling_rate'; got {type(item).__name__}." + ) + + if int(sample_rate) != self.sampling_rate: + raise ValueError( + f"Whisper expects {self.sampling_rate} Hz audio; got " + f"{sample_rate} Hz. Resample on the client side." + ) + + waveform = np.asarray(waveform, dtype=np.float32) + if waveform.ndim == 2: + # soundfile returns [frames, channels]; downmix to mono. + waveform = waveform.mean(axis=1) + if waveform.shape[0] > self.n_samples: + duration = waveform.shape[0] / float(self.sampling_rate) + raise ValueError( + f"Audio is {duration:.2f}s long, but Whisper supports at " + f"most {self.max_audio_seconds:.1f}s per request; chunk the " + "input on the client side." + ) + return waveform + + def __call__( + self, inputs: TextPrompt, sampling_params: SamplingParams + ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: + mm_data = inputs.get("multi_modal_data") or {} + audio_items = mm_data.get("audio") + if audio_items is None: + raise ValueError( + "Whisper requires multi_modal_data['audio'] (the text prompt " + "carries no encoder input)." + ) + if not isinstance(audio_items, list): + audio_items = [audio_items] + if len(audio_items) != 1: + raise ValueError( + f"Whisper supports exactly one audio clip per request; got {len(audio_items)}." + ) + + decoder_prompt = self._resolve_decoder_prompt(inputs.get("prompt")) + + # The engine's max_seq_len covers the 1500-position encoder pass, so it + # can't protect the smaller decoder position table (max_target_positions + # rows). Cap generation to the table here instead; re-capping a shared + # SamplingParams is idempotent. + decoder_budget = int(self.config.max_target_positions) - len(decoder_prompt) + if sampling_params.max_tokens is None or sampling_params.max_tokens > decoder_budget: + if sampling_params.max_tokens is not None: + logger.warning( + f"Capping max_tokens from {sampling_params.max_tokens} to " + f"{decoder_budget}: Whisper's decoder position table has " + f"{self.config.max_target_positions} rows and the prompt " + f"uses {len(decoder_prompt)}." + ) + sampling_params.max_tokens = decoder_budget + + waveform = self._load_waveform(audio_items[0]) + # Zero-pad to the fixed window, as the HF extractor does. Shipped as + # fp32 [1, n_samples]: the STFT needs fp32, and the leading dim of 1 + # keeps the C++ request's encoder-input-length bookkeeping unchanged. + n_samples = self.n_samples + padded = np.zeros((1, n_samples), dtype=np.float32) + padded[0, : waveform.shape[0]] = waveform + input_features = torch.from_numpy(padded) + + extra = { + "multimodal_data": { + "audio": { + # Purpose-specific key (not the generic HF "input_features"): + # its presence routes the request through the enc-dec + # encoder step. + "encoder_input_features": input_features.contiguous(), + # Post-conv position count (mel frames // 2): the cross-KV + # capacity every downstream consumer sizes against. + "encoder_output_len": int(self.config.max_source_positions), + } + } + } + return decoder_prompt, extra + + +# --------------------------------------------------------------------------- +# Top-level model +# --------------------------------------------------------------------------- + + +class WhisperModel(nn.Module): + """Whisper encoder-decoder body (no lm_head).""" + + def __init__(self, model_config: ModelConfig[WhisperConfig]): + super().__init__() + self.model_config = model_config + + self.encoder = WhisperEncoder(model_config) + self.decoder = WhisperDecoder(model_config) + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: Optional[torch.IntTensor] = None, + input_features: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + position_ids: Optional[torch.IntTensor] = None, + encoder_attn_metadata: Optional[AttentionMetadata] = None, + cross_attn_metadata: Optional[AttentionMetadata] = None, + skip_cross_kv_projection: bool = False, + inputs_embeds: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + # Run the audio encoder unless its output is already supplied. + if encoder_hidden_states is None and input_features is not None: + assert encoder_attn_metadata is not None + encoder_hidden_states = self.encoder( + input_features=input_features, + attn_metadata=encoder_attn_metadata, + ) + + decoder_output = self.decoder( + input_ids=input_ids, + attn_metadata=attn_metadata, + position_ids=position_ids, + encoder_hidden_states=encoder_hidden_states, + cross_attn_metadata=cross_attn_metadata, + skip_cross_kv_projection=skip_cross_kv_projection, + inputs_embeds=inputs_embeds, + ) + return decoder_output + + +@register_input_processor( + WhisperInputProcessor, + model_type="whisper", + # Whisper has no decoder-side audio placeholder tokens (the audio goes to + # the encoder); the metadata only affects chat-template serving. + placeholder_metadata=MultimodalPlaceholderMetadata(placeholder_map={"audio": ""}), +) +@register_auto_model("WhisperForConditionalGeneration") +class WhisperForConditionalGeneration(nn.Module, metaclass=PostInitCaller): + """Whisper encoder-decoder model with LM head.""" + + def __init__(self, model_config: ModelConfig[WhisperConfig]): + super().__init__() + self.model_config = model_config + config = model_config.pretrained_config + + self.model = WhisperModel(model_config) + + self.lm_head = LMHead( + config.vocab_size, + config.d_model, + dtype=config.torch_dtype, + mapping=model_config.mapping, + tensor_parallel_mode=TensorParallelMode.COLUMN, + gather_output=True, + reduce_output=False, + ) + + # Whisper ties the LM head (`proj_out`) to the decoder token embedding. + if config.tie_word_embeddings: + self.lm_head.weight = self.model.decoder.embed_tokens.weight + + self.logits_processor = LogitsProcessor() + + @classmethod + def get_model_defaults(cls, llm_args: "TorchLlmArgs") -> dict: + # NCCL_SYMMETRIC has observed issues for TP>1 fp32 on NVLS-disabled Hopper + # (https://nvbugs/6522483). Temporary workaround to pin NCCL strategy + if llm_args.tensor_parallel_size > 1: + return {"allreduce_strategy": "NCCL"} + return {} + + def __post_init__(self): + for _, module in self.named_modules(): + if callable(getattr(module, "create_weights", None)): + module.create_weights() + + def __pp_init__(self): + pass + + @property + def config(self): + return self.model_config.pretrained_config + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: Optional[torch.IntTensor] = None, + position_ids: Optional[torch.IntTensor] = None, + input_features: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attn_metadata: Optional[AttentionMetadata] = None, + cross_attn_metadata: Optional[AttentionMetadata] = None, + skip_cross_kv_projection: bool = False, + inputs_embeds: Optional[torch.Tensor] = None, + return_context_logits: bool = False, + **kwargs, + ) -> torch.Tensor: + hidden_states = self.model( + attn_metadata=attn_metadata, + input_ids=input_ids, + input_features=input_features, + encoder_hidden_states=encoder_hidden_states, + position_ids=position_ids, + encoder_attn_metadata=encoder_attn_metadata, + cross_attn_metadata=cross_attn_metadata, + skip_cross_kv_projection=skip_cross_kv_projection, + inputs_embeds=inputs_embeds, + ) + + return self.logits_processor.forward( + hidden_states, + self.lm_head, + attn_metadata, + return_context_logits, + ) + + def infer_max_seq_len(self) -> int: + # One engine-level max_seq_len must cover the encoder pass, which packs + # max_source_positions (1500) — more than the decoder's + # max_target_positions (448). Decoder generation length is capped to the + # position table separately, in WhisperInputProcessor. + config = self.model_config.pretrained_config + return max(config.max_target_positions, config.max_source_positions) + + def load_weights(self, weights: Dict, **kwargs): + config = self.model_config.pretrained_config + tllm_weights = _convert_hf_whisper_weights( + weights, config, dtype=self.model_config.torch_dtype + ) + + consumed = set() + for name, module in self.named_modules(): + if len(list(module.parameters(recurse=False))) == 0: + continue + if name not in tllm_weights: + continue + w = tllm_weights[name] + if hasattr(module, "load_weights"): + module.load_weights(weights=w) + else: + for n, p in module.named_parameters(recurse=False): + if n in w[0]: + p.data.copy_(w[0][n][:]) + consumed.add(name) + + # A converter key that matches no module would silently leave that + # module at its random init (usually non-NaN) — fail loudly instead. + unconsumed = sorted(set(tllm_weights) - consumed) + if unconsumed: + raise ValueError( + "Converted Whisper weights match no module in the model tree " + f"(converter/module-name mismatch): {unconsumed}" + ) + + +def _convert_hf_whisper_weights( + hf_weights: Dict[str, torch.Tensor], + config: WhisperConfig, + dtype: Optional[torch.dtype] = None, +) -> Dict: + """Map HuggingFace Whisper state_dict keys to TRT-LLM module-tree keys. + + HF Whisper weight layout (prefix `model.`; `proj_out` is the LM head, + tied to `model.decoder.embed_tokens`): + model.encoder.conv1.{weight,bias} + model.encoder.conv2.{weight,bias} + model.encoder.embed_positions.weight + model.encoder.layers.{i}.self_attn.{q_proj,k_proj,v_proj,out_proj}.{weight,bias?} + model.encoder.layers.{i}.self_attn_layer_norm.{weight,bias} + model.encoder.layers.{i}.fc1.{weight,bias}, fc2.{weight,bias} + model.encoder.layers.{i}.final_layer_norm.{weight,bias} + model.encoder.layer_norm.{weight,bias} + model.decoder.embed_tokens.weight + model.decoder.embed_positions.weight + model.decoder.layers.{i}.self_attn.{q_proj,k_proj,v_proj,out_proj}.{weight,bias?} + model.decoder.layers.{i}.self_attn_layer_norm.{weight,bias} + model.decoder.layers.{i}.encoder_attn.{q_proj,k_proj,v_proj,out_proj}.{weight,bias?} + model.decoder.layers.{i}.encoder_attn_layer_norm.{weight,bias} + model.decoder.layers.{i}.fc1.{weight,bias}, fc2.{weight,bias} + model.decoder.layers.{i}.final_layer_norm.{weight,bias} + model.decoder.layer_norm.{weight,bias} + proj_out.weight + + Whisper's `k_proj` has no bias while `q`/`v`/`out` do; because the + TRT-LLM fused QKV / cross-attn projections carry a bias when `bias=True`, + a zero bias is materialized for the key projection (numerically identical). + """ + out: Dict[str, list] = {} + enc_layers = config.encoder_layers + dec_layers = config.decoder_layers + + def _get(key: str) -> torch.Tensor: + # Cast lazily per-tensor: an eager whole-dict rewrite would hold a + # second full-checkpoint copy in host memory at peak. + if key in hf_weights: + w = hf_weights[key] + return w.to(dtype) if dtype is not None else w + raise KeyError(f"Missing expected HF weight: {key}") + + def _maybe(key: str): + w = hf_weights.get(key, None) + if w is not None and dtype is not None: + w = w.to(dtype) + return w + + def _wb(prefix: str) -> dict: + d = {"weight": _get(f"{prefix}.weight")} + b = _maybe(f"{prefix}.bias") + if b is not None: + d["bias"] = b + return d + + def _attn_qkv(hpfx: str) -> list: + """Fused QKV weight-dicts; synthesize a zero bias for the (bias-less) + key. MHA with num_heads * head_dim == d_model, so the bias length is + d_model for encoder and decoder alike.""" + q = _wb(f"{hpfx}.q_proj") + k = _wb(f"{hpfx}.k_proj") + v = _wb(f"{hpfx}.v_proj") + if "bias" not in k and ("bias" in q or "bias" in v): + k = dict(k) + k["bias"] = torch.zeros( + config.d_model, + dtype=q["weight"].dtype, + ) + return [q, k, v] + + def _cross_kv(hpfx: str, proj: str) -> list: + """Separate cross-attn projection; synthesize a zero bias for the key.""" + d = _wb(f"{hpfx}.{proj}") + if proj == "k_proj" and "bias" not in d: + d = dict(d) + d["bias"] = torch.zeros( + config.d_model, + dtype=d["weight"].dtype, + ) + return [d] + + # LM head (proj_out); tied to decoder embed_tokens but load explicitly too. + if "proj_out.weight" in hf_weights: + out["lm_head"] = [{"weight": _get("proj_out.weight")}] + + # ------------------------------------------------------------------ Encoder + out["model.encoder.conv1"] = [_wb("model.encoder.conv1")] + out["model.encoder.conv2"] = [_wb("model.encoder.conv2")] + out["model.encoder.embed_positions"] = [ + {"weight": _get("model.encoder.embed_positions.weight")} + ] + out["model.encoder.layer_norm"] = [_wb("model.encoder.layer_norm")] + + for i in range(enc_layers): + hpfx = f"model.encoder.layers.{i}" + tgt = f"model.encoder.layers.{i}" + + out[f"{tgt}.self_attn.qkv_proj"] = _attn_qkv(f"{hpfx}.self_attn") + out[f"{tgt}.self_attn.o_proj"] = [_wb(f"{hpfx}.self_attn.out_proj")] + out[f"{tgt}.self_attn_layer_norm"] = [_wb(f"{hpfx}.self_attn_layer_norm")] + + out[f"{tgt}.mlp.up_proj"] = [_wb(f"{hpfx}.fc1")] + out[f"{tgt}.mlp.down_proj"] = [_wb(f"{hpfx}.fc2")] + out[f"{tgt}.final_layer_norm"] = [_wb(f"{hpfx}.final_layer_norm")] + + # ------------------------------------------------------------------ Decoder + out["model.decoder.embed_tokens"] = [{"weight": _get("model.decoder.embed_tokens.weight")}] + out["model.decoder.embed_positions"] = [ + {"weight": _get("model.decoder.embed_positions.weight")} + ] + out["model.decoder.layer_norm"] = [_wb("model.decoder.layer_norm")] + + for i in range(dec_layers): + hpfx = f"model.decoder.layers.{i}" + tgt = f"model.decoder.layers.{i}" + + out[f"{tgt}.self_attn.qkv_proj"] = _attn_qkv(f"{hpfx}.self_attn") + out[f"{tgt}.self_attn.o_proj"] = [_wb(f"{hpfx}.self_attn.out_proj")] + out[f"{tgt}.self_attn_layer_norm"] = [_wb(f"{hpfx}.self_attn_layer_norm")] + + # Cross-attention (separate q/k/v/o projections). + out[f"{tgt}.cross_attn.q_proj"] = [_wb(f"{hpfx}.encoder_attn.q_proj")] + out[f"{tgt}.cross_attn.k_proj"] = _cross_kv(f"{hpfx}.encoder_attn", "k_proj") + out[f"{tgt}.cross_attn.v_proj"] = [_wb(f"{hpfx}.encoder_attn.v_proj")] + out[f"{tgt}.cross_attn.o_proj"] = [_wb(f"{hpfx}.encoder_attn.out_proj")] + out[f"{tgt}.cross_attn_layer_norm"] = [_wb(f"{hpfx}.encoder_attn_layer_norm")] + + out[f"{tgt}.mlp.up_proj"] = [_wb(f"{hpfx}.fc1")] + out[f"{tgt}.mlp.down_proj"] = [_wb(f"{hpfx}.fc2")] + out[f"{tgt}.final_layer_norm"] = [_wb(f"{hpfx}.final_layer_norm")] + + return out diff --git a/tensorrt_llm/_torch/modules/attention.py b/tensorrt_llm/_torch/modules/attention.py index 80e9c67d74d3..e902a892af3e 100644 --- a/tensorrt_llm/_torch/modules/attention.py +++ b/tensorrt_llm/_torch/modules/attention.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import math import weakref from typing import List, Optional, Tuple, Union @@ -696,6 +699,42 @@ def split_qkv(self, q, k=None, v=None): q, k, v = q.split([self.q_size, self.kv_size, self.kv_size], dim=-1) return q, k, v + def preprocess_qkv( + self, qkv: torch.Tensor, position_ids: Optional[torch.Tensor] + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], + Optional[torch.Tensor]]: + """Transform the fused QKV projection into attention inputs. + + Splits out the optional attention output gate and applies RoPE (plus + any subclass-specific processing such as QK norm, via apply_rope). + Subclasses may override this to fuse these steps into a single kernel. + + Returns: + tuple: (q, k, v, gate). k and v are None when q holds the fused + QKV tensor; gate is None when attn_output_gate is disabled. + """ + gate = None + if self.attn_output_gate: + q_gate, k, v = qkv.split( + [self.q_size * 2, self.kv_size, self.kv_size], dim=-1) + orig_shape = q_gate.shape[:-1] + # Single line: view -> chunk -> reshape both q and gate + q, gate = [ + t.reshape(*orig_shape, -1) for t in torch.chunk( + q_gate.view(*orig_shape, self.num_heads, -1), 2, dim=-1) + ] + else: + q, k, v = qkv, None, None + q, k, v = self.apply_rope(q, k, v, position_ids) + return q, k, v, gate + + def apply_output_gate(self, attention_output: torch.Tensor, + gate: torch.Tensor) -> torch.Tensor: + """Apply the attention output gate.""" + if gate.shape != attention_output.shape: + gate = gate.reshape(attention_output.shape) + return attention_output * torch.sigmoid(gate) + def convert_qkv(self, q, k, v): if k is None and v is None and not self.support_fused_qkv: q, k, v = self.split_qkv(q) @@ -983,18 +1022,6 @@ def forward( if qkv_lora is not None: qkv = qkv + qkv_lora - if self.attn_output_gate: - q_gate, k, v = qkv.split( - [self.q_size * 2, self.kv_size, self.kv_size], dim=-1) - orig_shape = q_gate.shape[:-1] - # Single line: view -> chunk -> reshape both q and gate - q, gate = [ - t.reshape(*orig_shape, -1) for t in torch.chunk( - q_gate.view(*orig_shape, self.num_heads, -1), 2, dim=-1) - ] - else: - q, k, v = qkv, None, None - # For dynamic tree spec decoding with Python RoPE, adjust position_ids # to use tree offsets (same as C++ kernel: past_seq_len + offset). if (not self.rope_fusion @@ -1008,7 +1035,7 @@ def forward( position_ids = self._adjust_position_ids_for_spec_dec( position_ids, attn_metadata) - q, k, v = self.apply_rope(q, k, v, position_ids) + q, k, v, gate = self.preprocess_qkv(qkv, position_ids) q, k, v = self.convert_qkv(q, k, v) if attention_sinks is not None: @@ -1034,8 +1061,7 @@ def forward( ) if self.attn_output_gate: - gate = torch.sigmoid(gate) - attn_output = attn_output * gate + attn_output = self.apply_output_gate(attn_output, gate) attn_output = _helix_cp_output_projection(self.o_proj, attn_output, attn_metadata, diff --git a/tensorrt_llm/_torch/modules/fla/cached_replay.py b/tensorrt_llm/_torch/modules/fla/cached_replay.py new file mode 100644 index 000000000000..1eba282eaba4 --- /dev/null +++ b/tensorrt_llm/_torch/modules/fla/cached_replay.py @@ -0,0 +1,815 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional + +import torch +import triton +import triton.language as tl +import triton.language.extra.libdevice as tldevice + +from tensorrt_llm._torch.modules.fla.op import exp +from tensorrt_llm._torch.modules.fla.utils import input_guard +from tensorrt_llm._utils import get_sm_version + +_SMALL_GRID_HEAD_TILES = 512 +_EIGHT_WARP_COMMIT_HEAD_TILES = 1024 +_PIPELINED_COMMIT_HEAD_TILES = 2048 +_TWO_STAGE_REPLAY_HEAD_TILES = 4096 +_L2_STREAMING_HEAD_TILES = 8192 + +CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE = 16 + + +@triton.jit +def _gdc_wait_with_memory_clobber(): + tl.inline_asm_elementwise( + "griddepcontrol.wait; // dummy $0", + "=r,~{memory}", + [], + dtype=tl.int32, + is_pure=False, + pack=1, + ) + + +@triton.jit +def _cached_replay_kernel( + q, + k, + v, + packed_qkv, + g, + beta, + o, + h0_source, + h0_indices, + old_u, + old_k, + old_G, + cache_buf_idx, + pnat, + scale, + pool_stride_slot, + A_log, + dt_bias, + T: tl.constexpr, + HIST: tl.constexpr, + BT: tl.constexpr, + BH: tl.constexpr, + H: tl.constexpr, + HV: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + USE_QK_L2NORM_IN_KERNEL: tl.constexpr, + STATE_FP32: tl.constexpr, + FUSED_GATING: tl.constexpr, + USE_L2_STATE_CACHE: tl.constexpr, + USE_L2_STREAMING_INPUTS: tl.constexpr, + USE_L2_SHARED_INPUTS: tl.constexpr, + USE_PACKED_QKV: tl.constexpr, + LAUNCH_WITH_PDL: tl.constexpr, + ENABLE_STATE_COMMIT: tl.constexpr, +): + """Replay new tokens from cached causal cached updates. + + old_u/old_k/old_G contain the prefix-invariant cached update vectors, + normalized keys, and cumulative log-decay from the current checkpoint. + Only the T new cached updates are solved on each verify step. + """ + i_v, i_nh = tl.program_id(0), tl.program_id(1) + i_n, i_hv = i_nh // HV, i_nh % HV + i_h = i_hv // (HV // H) + + o_k = tl.arange(0, BK) + o_v = i_v * BV + tl.arange(0, BV) + o_t = tl.arange(0, BT) + o_hist = tl.arange(0, BH) + mask_k = o_k < K + mask_v = o_v < V + mask_t = o_t < T + mask_hist_capacity = o_hist < HIST + + slot = tl.load(h0_indices + i_n) + if slot >= 0: + b_pnat = tl.load(pnat + slot) + b_buf = tl.load(cache_buf_idx + slot) + is_write = (b_pnat + T) > HIST + w_buf = tl.where(is_write, 1 - b_buf, b_buf) + w_off = tl.where(is_write, 0, b_pnat) + is_hist = (o_hist < b_pnat) & mask_hist_capacity + + hk_base = old_k + (slot.to(tl.int64) * 2 + b_buf) * HIST * H * K + i_h * K + hu_base = old_u + (slot.to(tl.int64) * 2 + b_buf) * HIST * HV * V + i_hv * V + hG_base = old_G + ((slot.to(tl.int64) * 2 + b_buf) * HV + i_hv) * HIST + wk_base = old_k + (slot.to(tl.int64) * 2 + w_buf) * HIST * H * K + i_h * K + wu_base = old_u + (slot.to(tl.int64) * 2 + w_buf) * HIST * HV * V + i_hv * V + wG_base = old_G + ((slot.to(tl.int64) * 2 + w_buf) * HV + i_hv) * HIST + + if USE_L2_SHARED_INPUTS: + b_kh = tl.load( + hk_base + o_hist[:, None] * H * K + o_k[None, :], + mask=is_hist[:, None] & mask_k[None, :], + other=0, + cache_modifier=".cg", + ) + else: + b_kh = tl.load( + hk_base + o_hist[:, None] * H * K + o_k[None, :], + mask=is_hist[:, None] & mask_k[None, :], + other=0, + ) + if USE_L2_STREAMING_INPUTS: + b_uh = tl.load( + hu_base + o_hist[:, None] * HV * V + o_v[None, :], + mask=is_hist[:, None] & mask_v[None, :], + other=0, + cache_modifier=".cg", + ) + else: + b_uh = tl.load( + hu_base + o_hist[:, None] * HV * V + o_v[None, :], + mask=is_hist[:, None] & mask_v[None, :], + other=0, + ) + b_Gh = tl.load(hG_base + o_hist, mask=is_hist, other=0.0).to(tl.float32) + g_start = tl.load( + hG_base + b_pnat - 1, + mask=b_pnat > 0, + other=0.0, + ).to(tl.float32) + + if LAUNCH_WITH_PDL: + _gdc_wait_with_memory_clobber() + + if USE_PACKED_QKV: + qkv_row = (i_n * T + o_t[:, None]) * (2 * H * K + HV * V) + p_k = packed_qkv + qkv_row + H * K + i_h * K + o_k[None, :] + p_q = packed_qkv + qkv_row + i_h * K + o_k[None, :] + p_v = packed_qkv + qkv_row + 2 * H * K + i_hv * V + o_v[None, :] + else: + p_k = k + ((i_n * T + o_t[:, None]) * H + i_h) * K + o_k[None, :] + p_q = q + ((i_n * T + o_t[:, None]) * H + i_h) * K + o_k[None, :] + p_v = v + ((i_n * T + o_t[:, None]) * HV + i_hv) * V + o_v[None, :] + + if USE_L2_SHARED_INPUTS: + b_k = tl.load( + p_k, + mask=mask_t[:, None] & mask_k[None, :], + other=0, + cache_modifier=".cg", + ) + b_q = tl.load( + p_q, + mask=mask_t[:, None] & mask_k[None, :], + other=0, + cache_modifier=".cg", + ) + else: + b_k = tl.load( + p_k, + mask=mask_t[:, None] & mask_k[None, :], + other=0, + ) + b_q = tl.load( + p_q, + mask=mask_t[:, None] & mask_k[None, :], + other=0, + ) + if USE_L2_STREAMING_INPUTS: + b_v = tl.load( + p_v, + mask=mask_t[:, None] & mask_v[None, :], + other=0, + cache_modifier=".cg", + ) + else: + b_v = tl.load( + p_v, + mask=mask_t[:, None] & mask_v[None, :], + other=0, + ) + b_g = tl.load( + g + (i_n * T + o_t) * HV + i_hv, + mask=mask_t, + other=0.0, + ).to(tl.float32) + b_beta = tl.load( + beta + (i_n * T + o_t) * HV + i_hv, + mask=mask_t, + other=0.0, + ).to(tl.float32) + if FUSED_GATING: + g_A_exp = tl.exp(tl.load(A_log + i_hv).to(tl.float32)) + g_dt_bias = tl.load(dt_bias + i_hv).to(tl.float32) + x = b_g + g_dt_bias + softplus = tl.where( + x <= 20.0, + 0.6931471805599453 * tldevice.fast_log2f(1.0 + tldevice.fast_expf(x)), + x, + ) + b_g = tl.where(mask_t, -g_A_exp * softplus, 0.0) + b_beta = tl.where( + mask_t, + tldevice.fast_dividef(1.0, 1.0 + tldevice.fast_expf(-b_beta)), + 0.0, + ) + + if USE_QK_L2NORM_IN_KERNEL: + b_kf = b_k.to(tl.float32) + b_qf = b_q.to(tl.float32) + inv_k = 1.0 / (tl.sqrt(tl.sum(b_kf * b_kf, 1)) + 1e-6) + inv_q = scale / (tl.sqrt(tl.sum(b_qf * b_qf, 1)) + 1e-6) + b_kn = (b_kf * inv_k[:, None]).to(b_k.dtype) + b_qn = (b_qf * inv_q[:, None]).to(b_q.dtype) + else: + b_kn = b_k + b_qn = (b_q.to(tl.float32) * scale).to(b_q.dtype) + + b_G_local = tl.cumsum(b_g, 0) + b_G = g_start + b_G_local + + p_h0 = ( + h0_source + + slot.to(tl.int64) * pool_stride_slot + + i_hv * V * K + + o_k[:, None] + + o_v[None, :] * K + ) + if STATE_FP32: + if USE_L2_STATE_CACHE: + b_h = tl.load( + p_h0, + mask=mask_k[:, None] & mask_v[None, :], + other=0, + cache_modifier=".cg", + ) + else: + b_h = tl.load( + p_h0, + mask=mask_k[:, None] & mask_v[None, :], + other=0, + ) + b_h_hi = b_h.to(b_kn.dtype) + b_h_lo = (b_h - b_h_hi.to(tl.float32)).to(b_kn.dtype) + b_kh0 = tl.dot(b_kn, b_h_hi) + tl.dot(b_kn, b_h_lo) + b_qh0 = tl.dot(b_qn, b_h_hi) + tl.dot(b_qn, b_h_lo) + else: + if USE_L2_STATE_CACHE: + b_h = tl.load( + p_h0, + mask=mask_k[:, None] & mask_v[None, :], + other=0, + cache_modifier=".cg", + ).to(b_kn.dtype) + else: + b_h = tl.load( + p_h0, + mask=mask_k[:, None] & mask_v[None, :], + other=0, + ).to(b_kn.dtype) + b_kh0 = tl.dot(b_kn, b_h) + b_qh0 = tl.dot(b_qn, b_h) + + hist_decay = exp(b_G[:, None] - b_Gh[None, :]) + b_kk_hist = tl.dot(b_kn, tl.trans(b_kh)) + b_qk_hist = tl.dot(b_qn, tl.trans(b_kh)) + b_k_hist_coeff = tl.where(is_hist[None, :], b_kk_hist * hist_decay, 0.0) + b_q_hist_coeff = tl.where(is_hist[None, :], b_qk_hist * hist_decay, 0.0) + b_k_hist = tl.dot(b_k_hist_coeff.to(b_uh.dtype), b_uh) + b_q_hist = tl.dot(b_q_hist_coeff.to(b_uh.dtype), b_uh) + + b_rhs = b_beta[:, None] * (b_v.to(tl.float32) - exp(b_G)[:, None] * b_kh0 - b_k_hist) + lower = o_t[:, None] > o_t[None, :] + b_kk_new = tl.dot(b_kn, tl.trans(b_kn)) + new_decay = exp(b_G[:, None] - b_G[None, :]) + b_A = tl.where( + lower, + b_beta[:, None] * b_kk_new * new_decay, + 0.0, + ) + + # T is at most eight for the cached replay path. Forward substitution + # avoids constructing/inverting the full HIST+T triangular system. + b_U = tl.zeros([BT, BV], dtype=tl.float32) + for row in range(T): + row_mask = o_t == row + rhs_row = tl.sum(tl.where(row_mask[:, None], b_rhs, 0.0), axis=0) + a_row = tl.sum(tl.where(row_mask[:, None], b_A, 0.0), axis=0) + correction = tl.sum(a_row[:, None] * b_U, axis=0) + u_row = rhs_row - correction + b_U += tl.where(row_mask[:, None], u_row[None, :], 0.0) + + incl = o_t[:, None] >= o_t[None, :] + b_qk_new = tl.dot(b_qn, tl.trans(b_kn)) + b_q_new_coeff = tl.where(incl, b_qk_new * new_decay, 0.0) + # BT is only four for MTP draft-3 verification, while tl.dot requires + # a reduction dimension of at least 16 on this architecture. Keep + # this genuinely small operation scalar instead of padding it into a + # mostly-empty tensor-core GEMM. + b_q_new = tl.zeros([BT, BV], dtype=tl.float32) + for row in range(T): + row_mask = o_t == row + coeff_row = tl.sum(tl.where(row_mask[:, None], b_q_new_coeff, 0.0), axis=0) + q_new_row = tl.sum(coeff_row[:, None] * b_U, axis=0) + b_q_new += tl.where(row_mask[:, None], q_new_row[None, :], 0.0) + b_o = exp(b_G)[:, None] * b_qh0 + b_q_hist + b_q_new + p_o = o + ((i_n * T + o_t[:, None]) * HV + i_hv) * V + o_v[None, :] + tl.store( + p_o, + b_o.to(p_o.dtype.element_ty), + mask=mask_t[:, None] & mask_v[None, :], + ) + + write_pos = w_off + o_t + tl.store( + wu_base + write_pos[:, None] * HV * V + o_v[None, :], + b_U.to(wu_base.dtype.element_ty), + mask=mask_t[:, None] & mask_v[None, :], + ) + if i_v == 0: + stored_G = tl.where(is_write, b_G_local, b_G) + tl.store(wG_base + write_pos, stored_G, mask=mask_t) + if i_hv % (HV // H) == 0: + tl.store( + wk_base + write_pos[:, None] * H * K + o_k[None, :], + b_kn.to(wk_base.dtype.element_ty), + mask=mask_t[:, None] & mask_k[None, :], + ) + + if ENABLE_STATE_COMMIT: + if is_write: + commit_decay = exp(g_start - b_Gh) + b_Uc = b_uh.to(tl.float32) * tl.where(is_hist, commit_decay, 0.0)[:, None] + b_Uc_hi = b_Uc.to(b_kh.dtype) + b_Uc_lo = (b_Uc - b_Uc_hi.to(tl.float32)).to(b_kh.dtype) + b_hc = ( + b_h.to(tl.float32) * exp(g_start) + + tl.dot(tl.trans(b_kh), b_Uc_hi) + + tl.dot(tl.trans(b_kh), b_Uc_lo) + ) + tl.store( + p_h0, + b_hc.to(p_h0.dtype.element_ty), + mask=mask_k[:, None] & mask_v[None, :], + ) + + +@triton.jit +def _cached_replay_layered_commit_kernel( + h0_source, + old_u, + old_k, + old_G, + replay_work_items, + n_writes, + pool_stride_layer, + pool_stride_slot, + old_u_stride_layer, + old_k_stride_layer, + old_G_stride_layer, + HIST: tl.constexpr, + BH: tl.constexpr, + NUM_LAYERS: tl.constexpr, + H: tl.constexpr, + HV: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + NV: tl.constexpr, + NUM_PERSISTENT: tl.constexpr, + USE_L2_STATE_CACHE: tl.constexpr, + USE_L2_STREAMING_INPUTS: tl.constexpr, + USE_L2_SHARED_INPUTS: tl.constexpr, + PIPE_STAGES: tl.constexpr, +): + """Advance every local GDN layer from one cached-history snapshot.""" + pid = tl.program_id(0) + total_work = tl.load(n_writes) * NUM_LAYERS * HV * NV + for tile_id in tl.range( + pid, + total_work, + NUM_PERSISTENT, + num_stages=PIPE_STAGES, + flatten=True, + ): + i_v = tile_id % NV + tile_id = tile_id // NV + i_hv = tile_id % HV + tile_id = tile_id // HV + layer = tile_id % NUM_LAYERS + work_idx = tile_id // NUM_LAYERS + i_h = i_hv // (HV // H) + + work_base = replay_work_items + work_idx * 4 + slot = tl.load(work_base + 1).to(tl.int64) + b_pnat = tl.load(work_base + 2) + b_buf = tl.load(work_base + 3) + layer_i64 = layer.to(tl.int64) + + o_k = tl.arange(0, BK) + o_v = i_v * BV + tl.arange(0, BV) + o_hist = tl.arange(0, BH) + mask_k = o_k < K + mask_v = o_v < V + is_hist = (o_hist < b_pnat) & (o_hist < HIST) + + hk_base = ( + old_k + layer_i64 * old_k_stride_layer + (slot * 2 + b_buf) * HIST * H * K + i_h * K + ) + hu_base = ( + old_u + layer_i64 * old_u_stride_layer + (slot * 2 + b_buf) * HIST * HV * V + i_hv * V + ) + hG_base = old_G + layer_i64 * old_G_stride_layer + ((slot * 2 + b_buf) * HV + i_hv) * HIST + + if USE_L2_SHARED_INPUTS: + b_kh = tl.load( + hk_base + o_hist[:, None] * H * K + o_k[None, :], + mask=is_hist[:, None] & mask_k[None, :], + other=0, + cache_modifier=".cg", + ) + else: + b_kh = tl.load( + hk_base + o_hist[:, None] * H * K + o_k[None, :], + mask=is_hist[:, None] & mask_k[None, :], + other=0, + ) + if USE_L2_STREAMING_INPUTS: + b_uh = tl.load( + hu_base + o_hist[:, None] * HV * V + o_v[None, :], + mask=is_hist[:, None] & mask_v[None, :], + other=0, + cache_modifier=".cg", + ) + else: + b_uh = tl.load( + hu_base + o_hist[:, None] * HV * V + o_v[None, :], + mask=is_hist[:, None] & mask_v[None, :], + other=0, + ) + b_Gh = tl.load(hG_base + o_hist, mask=is_hist, other=0.0).to(tl.float32) + g_start = tl.load(hG_base + b_pnat - 1).to(tl.float32) + + p_h0 = ( + h0_source + + layer_i64 * pool_stride_layer + + slot * pool_stride_slot + + i_hv * V * K + + o_k[:, None] + + o_v[None, :] * K + ) + if USE_L2_STATE_CACHE: + b_h = tl.load( + p_h0, + mask=mask_k[:, None] & mask_v[None, :], + other=0, + cache_modifier=".cg", + ) + else: + b_h = tl.load( + p_h0, + mask=mask_k[:, None] & mask_v[None, :], + other=0, + ) + commit_decay = exp(g_start - b_Gh) + b_Uc = b_uh.to(tl.float32) * tl.where(is_hist, commit_decay, 0.0)[:, None] + b_Uc_hi = b_Uc.to(b_kh.dtype) + b_Uc_lo = (b_Uc - b_Uc_hi.to(tl.float32)).to(b_kh.dtype) + b_hc = ( + b_h.to(tl.float32) * exp(g_start) + + tl.dot(tl.trans(b_kh), b_Uc_hi) + + tl.dot(tl.trans(b_kh), b_Uc_lo) + ) + tl.store( + p_h0, + b_hc.to(p_h0.dtype.element_ty), + mask=mask_k[:, None] & mask_v[None, :], + ) + + +def commit_gdn_cached_replay_history_layers( + *, + ssm_states: torch.Tensor, + old_u: torch.Tensor, + old_k: torch.Tensor, + old_G: torch.Tensor, + replay_work_items: torch.Tensor, + n_writes: torch.Tensor, + history_size: int, + persistent_waves: int = 2, + commit_block_v: Optional[int] = None, + commit_num_warps: Optional[int] = None, + commit_pipeline_stages: Optional[int] = None, +) -> None: + """Advance all local layer checkpoints from cached replay histories.""" + num_layers, _, HV, V, K = ssm_states.shape + assert old_u.ndim == 6 and old_k.ndim == 6 and old_G.ndim == 5 + H = old_k.shape[-2] + assert old_u.shape[0] == num_layers and old_u.shape[-2:] == (HV, V) + assert old_k.shape[0] == num_layers and old_k.shape[-1] == K + assert old_G.shape[0] == num_layers and old_G.shape[-2:] == (HV, history_size) + assert old_u.is_contiguous() and old_k.is_contiguous() and old_G.is_contiguous() + assert replay_work_items.ndim == 2 and replay_work_items.shape[1] == 4 + assert replay_work_items.dtype == torch.int32 + assert n_writes.dtype == torch.int32 and n_writes.numel() == 1 + assert persistent_waves > 0 + + N = replay_work_items.shape[0] + if N == 0 or num_layers == 0: + return + BK = triton.next_power_of_2(K) + BH = triton.next_power_of_2(history_size) + assert BK == K and BH <= 16 + per_layer_head_tiles = N * HV + use_tuned_bf16_mapping = ( + history_size <= 16 + and HV == 4 * H + and K == 128 + and V == 128 + and ssm_states.dtype == torch.bfloat16 + ) + use_small_grid_mapping = ( + use_tuned_bf16_mapping and per_layer_head_tiles <= _SMALL_GRID_HEAD_TILES + ) + if commit_block_v is None: + commit_block_v = 64 if use_small_grid_mapping else triton.next_power_of_2(V) + if commit_num_warps is None: + commit_num_warps = ( + 8 + if use_tuned_bf16_mapping + and per_layer_head_tiles >= _EIGHT_WARP_COMMIT_HEAD_TILES + and commit_block_v == 128 + else 2 + if use_tuned_bf16_mapping + else 4 + ) + use_large_workload_mapping = ( + per_layer_head_tiles >= _L2_STREAMING_HEAD_TILES + if use_tuned_bf16_mapping + else N >= 128 and ssm_states.dtype == torch.bfloat16 + ) + if commit_pipeline_stages is None: + commit_pipeline_stages = ( + 5 + if use_tuned_bf16_mapping and per_layer_head_tiles >= _PIPELINED_COMMIT_HEAD_TILES + else 5 + if not use_tuned_bf16_mapping and use_large_workload_mapping + else 1 + ) + assert triton.next_power_of_2(commit_block_v) == commit_block_v + assert commit_block_v <= V and commit_pipeline_stages > 0 + + commit_nv = triton.cdiv(V, commit_block_v) + num_sms = torch.cuda.get_device_properties(ssm_states.device).multi_processor_count + total_tiles = N * num_layers * HV * commit_nv + num_persistent = min(num_sms * persistent_waves, total_tiles) + _cached_replay_layered_commit_kernel[(num_persistent,)]( + h0_source=ssm_states, + old_u=old_u, + old_k=old_k, + old_G=old_G, + replay_work_items=replay_work_items, + n_writes=n_writes, + pool_stride_layer=ssm_states.stride(0), + pool_stride_slot=ssm_states.stride(1), + old_u_stride_layer=old_u.stride(0), + old_k_stride_layer=old_k.stride(0), + old_G_stride_layer=old_G.stride(0), + HIST=history_size, + BH=BH, + NUM_LAYERS=num_layers, + H=H, + HV=HV, + K=K, + V=V, + BK=BK, + BV=commit_block_v, + NV=commit_nv, + NUM_PERSISTENT=num_persistent, + USE_L2_STATE_CACHE=use_large_workload_mapping, + USE_L2_STREAMING_INPUTS=use_large_workload_mapping, + USE_L2_SHARED_INPUTS=use_large_workload_mapping, + PIPE_STAGES=commit_pipeline_stages, + num_warps=commit_num_warps, + num_stages=commit_pipeline_stages, + ) + + +@input_guard( + exclude_args=[ + "q", + "k", + "v", + "packed_qkv", + "ssm_states", + "old_u", + "old_k", + "old_G", + "old_beta", + "cache_buf_idx", + "prev_num_accepted_tokens", + "replay_work_items", + "n_writes", + "output", + ] +) +def fused_recurrent_gated_delta_rule_cached_replay_update( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + ssm_states: torch.Tensor, + state_indices: torch.Tensor, + old_u: torch.Tensor, + old_k: torch.Tensor, + old_G: torch.Tensor, + old_beta: torch.Tensor, + cache_buf_idx: torch.Tensor, + prev_num_accepted_tokens: torch.Tensor, + history_size: int, + scale: Optional[float] = None, + use_qk_l2norm_in_kernel: bool = False, + A_log: Optional[torch.Tensor] = None, + dt_bias: Optional[torch.Tensor] = None, + launch_with_pdl: bool = False, + replay_work_items: Optional[torch.Tensor] = None, + n_writes: Optional[torch.Tensor] = None, + block_v: Optional[int] = None, + num_warps: Optional[int] = None, + use_l2_state_cache: Optional[bool] = None, + use_l2_streaming_inputs: Optional[bool] = None, + use_l2_shared_inputs: Optional[bool] = None, + main_num_stages: Optional[int] = None, + packed_qkv: Optional[torch.Tensor] = None, + use_all_layer_commit: bool = False, + output: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """GDN replay using cached causal updates rather than raw history.""" + del old_beta # Signature-compatible placeholder; cached updates embed beta. + N, T, H, K = k.shape + HV, V = v.shape[2], v.shape[3] + assert q.shape == k.shape + assert v.shape[:2] == (N, T) + use_packed_qkv = packed_qkv is not None + if use_packed_qkv: + qkv_width = 2 * H * K + HV * V + assert packed_qkv is not None + assert packed_qkv.shape == (N * T, qkv_width) + assert packed_qkv.dtype == q.dtype + assert packed_qkv.device == q.device + assert packed_qkv.is_contiguous() + else: + q = q.contiguous() + k = k.contiguous() + v = v.contiguous() + packed_qkv = q + BK = triton.next_power_of_2(K) + # GB200 dispatch for the production Qwen3.5 MTP per-CTA shape. Balanced + # DEP and TEP runs at the same global batch have the same N * HV head-tile + # count, so use that workload measure instead of topology-specific H/HV + # values or the per-rank batch alone. + use_tuned_bf16_mapping = ( + T == 4 + and history_size <= 16 + and HV == 4 * H + and K == 128 + and V == 128 + and ssm_states.dtype == torch.bfloat16 + ) + head_tiles = N * HV + use_small_grid_mapping = use_tuned_bf16_mapping and head_tiles <= _SMALL_GRID_HEAD_TILES + if block_v is None: + block_v = 64 if use_small_grid_mapping else triton.next_power_of_2(V) + if num_warps is None: + num_warps = ( + 2 if use_tuned_bf16_mapping and (use_small_grid_mapping or launch_with_pdl) else 4 + ) + BV = block_v + use_large_workload_mapping = ( + head_tiles >= _L2_STREAMING_HEAD_TILES + if use_tuned_bf16_mapping + else N >= 128 and ssm_states.dtype == torch.bfloat16 + ) + if use_l2_state_cache is None: + use_l2_state_cache = use_large_workload_mapping + if use_l2_streaming_inputs is None: + use_l2_streaming_inputs = use_large_workload_mapping + if use_l2_shared_inputs is None: + use_l2_shared_inputs = use_large_workload_mapping + if main_num_stages is None: + if use_tuned_bf16_mapping: + main_num_stages = 2 if head_tiles >= _TWO_STAGE_REPLAY_HEAD_TILES else 1 + else: + main_num_stages = 3 if use_large_workload_mapping else 1 + BT = triton.next_power_of_2(T) + BH = triton.next_power_of_2(history_size) + assert BK == K and triton.next_power_of_2(BV) == BV and BV <= V + assert main_num_stages > 0 + assert T <= 8 + assert history_size >= T + assert BH <= 16 + if scale is None: + scale = K**-0.5 + + fused_gating = A_log is not None + if fused_gating: + assert dt_bias is not None + assert A_log.numel() == HV and dt_bias.numel() == HV + else: + A_log = q + dt_bias = q + + if launch_with_pdl and get_sm_version() < 90: + launch_with_pdl = False + + s_h0_0, s_h0_1, s_h0_2, s_h0_3 = ssm_states.stride() + assert s_h0_3 == 1 and s_h0_2 == K and s_h0_1 == V * K + for name, tensor in (("old_u", old_u), ("old_k", old_k), ("old_G", old_G)): + assert tensor.is_contiguous(), f"{name} must be contiguous" + if (replay_work_items is None) != (n_writes is None): + raise ValueError("replay_work_items and n_writes must either both be set or both be None") + if replay_work_items is not None: + assert replay_work_items.shape == (N, 4) + assert replay_work_items.dtype == torch.int32 + assert replay_work_items.is_contiguous() + assert n_writes is not None and n_writes.dtype == torch.int32 + assert n_writes.numel() == 1 + if not use_all_layer_commit: + raise ValueError("Partitioned replay requires the all-layer commit") + elif use_all_layer_commit: + raise ValueError("use_all_layer_commit requires replay work items") + + if output is None: + output = q.new_empty(N, T, HV, V) + else: + assert output.is_contiguous(), "output must be contiguous" + NV = triton.cdiv(V, BV) + grid = (NV, N * HV) + + def launch( + *, + enable_state_commit: bool, + use_pdl: bool, + ): + _cached_replay_kernel[grid]( + q=q, + k=k, + v=v, + packed_qkv=packed_qkv, + g=g, + beta=beta, + o=output, + h0_source=ssm_states, + h0_indices=state_indices, + old_u=old_u, + old_k=old_k, + old_G=old_G, + cache_buf_idx=cache_buf_idx, + pnat=prev_num_accepted_tokens, + scale=scale, + pool_stride_slot=s_h0_0, + A_log=A_log, + dt_bias=dt_bias, + T=T, + HIST=history_size, + BT=BT, + BH=BH, + H=H, + HV=HV, + K=K, + V=V, + BK=BK, + BV=BV, + USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, + STATE_FP32=ssm_states.dtype == torch.float32, + FUSED_GATING=fused_gating, + USE_L2_STATE_CACHE=use_l2_state_cache, + USE_L2_STREAMING_INPUTS=use_l2_streaming_inputs, + USE_L2_SHARED_INPUTS=use_l2_shared_inputs, + USE_PACKED_QKV=use_packed_qkv, + LAUNCH_WITH_PDL=use_pdl, + ENABLE_STATE_COMMIT=enable_state_commit, + num_warps=num_warps, + num_stages=main_num_stages, + launch_pdl=use_pdl, + ) + + if replay_work_items is None: + launch( + enable_state_commit=True, + use_pdl=launch_with_pdl, + ) + else: + # Large batches keep replay free of the checkpoint-state expression. + # The cache manager advances every local GDN layer in one launch after + # all layers have populated their history caches. + launch( + enable_state_commit=False, + use_pdl=launch_with_pdl, + ) + return output diff --git a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py index f9f21a911a95..e819f1690728 100644 --- a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py +++ b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py @@ -16,10 +16,13 @@ (the FlashInfer prefill kernel does NOT apply L2 norm internally; the ``use_qk_l2norm_in_kernel`` parameter on ``flashinfer.chunk_gated_delta_rule`` is currently a dead arg, see ``flashinfer/gdn_prefill.py:317-356``). - * Pre-gather and post-scatter of indexed SSM state (FlashInfer requires - packed ``[num_seqs, H, V, K]`` fp32 initial/output state). TRT-LLM's GDN - state pool uses the same ``[N, H, V, K]`` logical layout, so the adapter - casts/gathers/scatters without transposing the last two dims. + * Pre-gather and post-scatter of indexed SSM state into FlashInfer's packed + ``[num_seqs, H, V, K]`` layout. TRT-LLM's GDN state pool uses the same + ``[N, H, V, K]`` logical layout, so the adapter gathers/scatters without + transposing the last two dims. The SM100/SM103 kernel carries the recurrent + state in fp32 in TMEM regardless of the initial/output-state I/O dtype, so + the round-trip stays in the native pool dtype (bf16/fp16) there with no + precision change; only SM90/SM120 need an fp32 up-cast/down-cast. This module is only imported when ``TLLM_USE_FLASHINFER_GDN_PREFILL=1`` is set at process start; do not import it lazily inside hot paths. @@ -34,6 +37,7 @@ gather_cast_vk_to_fp32_vk, ) from tensorrt_llm._torch.modules.fla.l2norm import l2norm_fwd +from tensorrt_llm._utils import is_sm_100f # Mirror the @torch.compiler.disable on the legacy Triton wrapper @@ -103,10 +107,18 @@ def chunk_gated_delta_rule( q3 = l2norm_fwd(q3) k3 = l2norm_fwd(k3) - # --- Step 4: gather initial state and cast dtype --------------------- + # --- Step 4: gather initial state (+ cast dtype only when required) --- # TRT-LLM's GDN kernels and FlashInfer both use [N, H, V, K] state layout. - # Fuse gather + cast-to-fp32 + contiguous into a single Triton kernel. - gathered_init = gather_cast_vk_to_fp32_vk(initial_state, initial_state_indices) + # The SM100/SM103 kernel carries the recurrent state in fp32 in TMEM + # regardless of the initial/output-state I/O dtype (the state tensors are + # only the gmem load/store format), so passing bf16/fp16 state there is + # numerically identical to the fp32 round-trip while moving half the bytes. + # SM90/SM120 still require fp32 state. Fuse gather (+ optional cast) and + # contiguous into a single Triton kernel. + state_dtype = initial_state.dtype if is_sm_100f() else torch.float32 + gathered_init = gather_cast_vk_to_fp32_vk( + initial_state, initial_state_indices, out_dtype=state_dtype + ) # --- Step 5+6: call FlashInfer with pre-allocated output/state buffers # FI 0.6.10 accepts `output=` / `output_state=`; pre-allocating skips its @@ -126,7 +138,10 @@ def chunk_gated_delta_rule( ) if need_state: num_seqs = cu_seqlens.shape[0] - 1 - state_buf = q3.new_empty(num_seqs, num_o_heads, head_size, head_size, dtype=torch.float32) + # Match the initial-state dtype (native bf16/fp16 on SM100/SM103, else + # fp32); FlashInfer writes the final state in this dtype and the scatter + # below adapts to the destination pool dtype without an extra cast. + state_buf = q3.new_empty(num_seqs, num_o_heads, head_size, head_size, dtype=state_dtype) out_packed, out_state = flashinfer.chunk_gated_delta_rule( q=q3, k=k3, @@ -158,8 +173,9 @@ def chunk_gated_delta_rule( ) out_state = None - # --- Step 7: cast state back, scatter / return --------------------- - # Fuse cast (fp32 -> initial_state.dtype) + optional indexed scatter into a + # --- Step 7: cast state back (if needed), scatter / return --------- + # Fuse cast (out_state.dtype -> destination dtype; a no-op on SM100/SM103 + # where both are the native pool dtype) + optional indexed scatter into a # single Triton pass, mirroring Step 4. The inplace branch writes only the # slots named by ``initial_state_indices`` and leaves the rest untouched. if inplace_indexed_state_update: diff --git a/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py b/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py index 961189d97346..ee6fea9dc15e 100644 --- a/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py +++ b/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py @@ -265,6 +265,24 @@ def _flashinfer_gdn_decode( HV = v.shape[2] V = v.shape[3] + # The FlashInfer CuTe-DSL kernel requires every input tensor's data pointer + # to be 32-byte aligned (enforced in build_memref_desc). ``a`` and ``b`` are + # per-head-scalar slices of the fused ``in_proj_ba`` output: ``b`` starts at + # offset 0 (aligned) but ``a`` starts ``num_v_heads_per_tp`` bf16 elements in, + # so when ``num_v_heads_per_tp`` is not a multiple of 16 (e.g. Qwen3.6-35B-A3B + # TEP4: 32 v-heads / 4 = 8 -> 16-byte offset) the slice base is not 32-byte + # aligned and the kernel aborts. ``.contiguous()`` is NOT enough: at decode + # the token dim is 1, so the strided/offset slice already reports as + # contiguous (size-1 dims are ignored by is_contiguous) and ``.contiguous()`` + # is a no-op that keeps the misaligned pointer. Clone into fresh (allocator- + # aligned) storage instead, and only when misaligned so the common aligned + # case (e.g. Qwen3.5-397B TEP4: 64 / 4 = 16 -> 32-byte offset) stays zero-copy. + # q/k/v are sliced on 128-element head boundaries (>=256 B), always aligned. + if a.data_ptr() % 32 != 0: + a = a.clone(memory_format=torch.contiguous_format) + if b.data_ptr() % 32 != 0: + b = b.clone(memory_format=torch.contiguous_format) + # Reshape from packed varlen [1, N*T, ...] to batched [N, T, ...]. q_bat = q.view(N, T_per_seq, q.shape[2], q.shape[3]) k_bat = k.view(N, T_per_seq, k.shape[2], k.shape[3]) @@ -368,9 +386,17 @@ def _flashinfer_gdn_verify( output = (output.view(N, T, HV, V) if output is not None else q.new_empty( N, T, HV, V)) # The FI CuTe-DSL kernel asserts 32-byte data alignment on every tensor - # argument. The int32 index tensor may be a slice of a larger buffer + # argument. ``a`` starts ``num_v_heads_per_tp`` bf16 elements into the fused + # ``in_proj_ba`` output, so it is misaligned when that count is not a + # multiple of 16 (e.g. Qwen3.5 TEP16: 128 / 16 = 8) -- see the note in + # _flashinfer_gdn_decode for why this clones instead of .contiguous(). + if a.data_ptr() % 32 != 0: + a = a.clone(memory_format=torch.contiguous_format) + if b.data_ptr() % 32 != 0: + b = b.clone(memory_format=torch.contiguous_format) + # The int32 index tensor may likewise be a slice of a larger buffer # (e.g. state_indices_d = cache_indices[num_prefills:]) whose 4*offset - # storage offset breaks that; .int() is a no-op for int32, so realign + # storage offset breaks alignment; .int() is a no-op for int32, so realign # with an explicit copy when needed. initial_state_indices = initial_state_indices.int() if initial_state_indices.data_ptr() % 32 != 0: diff --git a/tensorrt_llm/_torch/modules/fla/fused_state_io.py b/tensorrt_llm/_torch/modules/fla/fused_state_io.py index c5885f5b6011..ee420993fec3 100644 --- a/tensorrt_llm/_torch/modules/fla/fused_state_io.py +++ b/tensorrt_llm/_torch/modules/fla/fused_state_io.py @@ -79,8 +79,16 @@ def _gather_cast_vk_to_fp32_vk_kernel( def gather_cast_vk_to_fp32_vk( initial_state: torch.Tensor, initial_state_indices: Optional[torch.Tensor], + out_dtype: Optional[torch.dtype] = None, ) -> torch.Tensor: - """Fused ``initial_state[indices].to(fp32).contiguous()`` for ``[N, H, V, K]`` state.""" + """Fused ``initial_state[indices].to(out_dtype).contiguous()`` for ``[N, H, V, K]`` state. + + ``out_dtype`` defaults to ``torch.float32``, the dtype the SM90/SM120 + FlashInfer GDN prefill kernels require. On the SM100/SM103 kernel, which + reads native bf16/fp16 state and casts to fp32 internally, pass + ``initial_state.dtype`` to gather without an up-cast (paired with a matching + scatter that skips the down-cast). + """ assert initial_state.dim() == 4, f"initial_state must be 4D, got {initial_state.shape}" n_pool, h, v, k = initial_state.shape if initial_state_indices is not None: @@ -93,7 +101,9 @@ def gather_cast_vk_to_fp32_vk( # K and V are typically 128 in GDN; one (BLOCK_K, BLOCK_V) tile covers the full K and V dimensions. # entire (K, V) plane per (seq, head). Larger tiles save grid overhead; # smaller tiles improve occupancy at small num_seqs * H. - output = torch.empty(num_seqs, h, v, k, dtype=torch.float32, device=initial_state.device) + if out_dtype is None: + out_dtype = torch.float32 + output = torch.empty(num_seqs, h, v, k, dtype=out_dtype, device=initial_state.device) block_v = min(v, 128) block_k = min(k, 128) num_v_blocks = triton.cdiv(v, block_v) diff --git a/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md b/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md index b84f0e5c4577..331b1ca34fa9 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md +++ b/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md @@ -150,7 +150,7 @@ Still on old path (standalone, with embedded communication): | `fused_moe_cute_dsl_b12x.py` | `CuteDslB12xFusedMoE` | SM120/SM121 | NVFP4 hybrid CUTLASS-prefill / FlashInfer NVFP4 MoE decode — best perf on RTX PRO 6000 (SM120) and DGX Spark (SM121); select via the `CUTEDSL` backend path (auto-promoted when flashinfer is importable) | `EXTERNAL_COMM` | | `mega_moe/mega_moe_deepgemm.py` | `MegaMoEDeepGemm` | SM100/SM103 | W4A8_MXFP4_MXFP8 via DeepGEMM `fp8_fp4_mega_moe` fused dispatch+GEMM+act+GEMM+combine kernel; requires `hidden_size % 512 == 0` | `FUSED_COMM` | | `mega_moe/mega_moe_cute_dsl.py` | `MegaMoECuteDsl` | SM100/SM103 | NVFP4 via ported CuteDSL `Sm100MegaMoEKernel` fused dispatch+FC1+act+FC2+combine kernel; requires CUDA 13 Cutlass DSL runtime (PR #14354) and NVSHMEM provider (hard gate); threads per-expert `fc31_alpha`/`fc2_alpha`/`fc1_norm_const` through the kernel ABI and supports SwiGLU clamp via `swiglu_limit`; default deepgemm graph (topk score folded before fc1-out quant, host `combine_output.sum(dim=1)`) | `FUSED_COMM` | -| `fused_moe_marlin.py` | `MarlinFusedMoE` | SM90 only | W4A16 NVFP4 on Hopper (BF16 activations + FP4 weights, fused single-launch `marlin_nvfp4_moe_gemm` kernel); no dynamic EPLB | `EXTERNAL_COMM` | +| `fused_moe_marlin.py` | `MarlinFusedMoE` | SM90 only | W4A16 NVFP4 on Hopper (BF16 activations + FP4 weights, fused single-launch `marlin_nvfp4_moe_gemm` kernel); supports attention-DP + EP via external comm (scheduler precomputes routing; dispatch payload is plain BF16, no activation scales); non-NVFP4 layers (e.g. unquantized MTP draft layers) fall back to Cutlass in `get_moe_cls`; no dynamic EPLB | `EXTERNAL_COMM` | | `fused_moe_triton.py` | `TritonFusedMoE` | SM90 only | GPT-OSS on Hopper (requires `swiglu_gptoss_style=True`) | (legacy path) | | `fused_moe_wide_ep.py` | `WideEPMoE` | All GPUs | Deprecating — use ConfigurableMoE instead | (legacy path) | | `fused_moe_vanilla.py` | `VanillaMoE` | All devices | Reference / debugging only | (legacy path) | @@ -251,5 +251,5 @@ When adding new components, use these reference implementations: - **Schedulers MUST NOT write `moe.repeat_idx`** — `repeat_idx` is wrapper state advanced once per `forward_impl` regardless of chunk count - **Do NOT allocate symmetric memory from `run_moe` in `FUSED_COMM` backends** — Symmetric-memory rendezvous is a build-time collective and is unsafe under PP / layer-skip or CUDA graph capture; allocate from `create_weights()` after `ConfigurableMoE` has synchronized EPLB-derived attributes. See `mega_moe/mega_moe_deepgemm.py` for the DG pattern and `mega_moe/mega_moe_cute_dsl.py:_alloc_symm_provider` for the NVSHMEM-equivalent provider. - **Do NOT add a new `FUSED_COMM` backend without a zero-token `quantize_input` regression test** — `FusedCommMoEScheduler` calls `quantize_input` for every chunk (including zero-token chunks) so each backend must return its own empty-tensor layout. See `tests/unittest/_torch/modules/moe/test_moe_backend.py::test_megamoe_deepgemm_quantize_input_zero_tokens` and `test_megamoe_cutedsl_quantize_input_zero_tokens` for the pattern. -- **Do NOT use a dataclass for an autotuner tactic without a tested `__repr__` round-trip** — `AutoTuner` serializes tactic values through `json.dumps`/`json.loads` and `eval(repr(tactic))`; a plain dataclass fails the `eval(repr(...))` check. Prefer a JSON-friendly **tuple of primitives or lists of primitives** (lists are JSON-friendly; tuples round-trip via `eval(repr(...))`). See `Sm100MegaMoENvfp4Runner` in `tensorrt_llm/_torch/custom_ops/cute_dsl_megamoe_custom_op.py` for the 6-tuple tactic pattern (mma_tiler/cluster_shape as `list[int]`, the rest as `bool`/`int`/`str`). The fallback tactic is built inline in `Sm100MegaMoENvfp4Runner.forward(tactic=-1)` from `DEFAULT_MEGAMOE_TACTIC`, not via a separate `fallback_tactic()` method. -- **Do NOT forget `distributed_tuning_strategy=DistributedTuningStrategy.PARALLEL` on a multi-rank `FUSED_COMM` backend's `TuningConfig`** — Every EP rank must converge on the same compiled tactic for every chunk, otherwise the in-kernel NVLink dispatch barrier deadlocks. Reference: `Sm100MegaMoENvfp4Runner.get_tuning_config` and every multi-rank op in `cute_dsl_custom_ops.py`. +- **Do NOT use a dataclass for an autotuner tactic without a tested `__repr__` round-trip** — `AutoTuner` serializes tactic values through `json.dumps`/`json.loads` and `eval(repr(tactic))`; a plain dataclass fails the `eval(repr(...))` check. Prefer a JSON-friendly **tuple of primitives or lists of primitives** (lists are JSON-friendly; tuples round-trip via `eval(repr(...))`). See the tactic-representation comment block in `tensorrt_llm/_torch/custom_ops/cute_dsl_megamoe_custom_op.py` for the 8-tuple tactic pattern (mma_tiler/cluster_shape as `list[int]`, `epi_flag_batch` as a nested `(int, int)` tuple, the rest as `bool`/`int`/`str`; `_unpack_tactic` is the single source of truth for the field order). The fallback tactic is the token-aware `default_megamoe_tactic(num_tokens)` helper, selected by `Sm100MegaMoENvfp4Runner.forward(tactic=-1)`, not a separate `fallback_tactic()` method. +- **Use `distributed_tuning_strategy=DistributedTuningStrategy.MERGE` on a multi-rank `FUSED_COMM` backend's `TuningConfig`** — Every EP rank must converge on the same compiled tactic for every chunk, otherwise the in-kernel NVLink dispatch barrier deadlocks. `PARALLEL` can profile different tactics on different ranks and is unsafe for fused collectives. Reference: `Sm100MegaMoENvfp4Runner.get_tuning_config`. diff --git a/tensorrt_llm/_torch/modules/fused_moe/communication/__init__.py b/tensorrt_llm/_torch/modules/fused_moe/communication/__init__.py index 0d44ecd2df1e..9858693c5f37 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/communication/__init__.py +++ b/tensorrt_llm/_torch/modules/fused_moe/communication/__init__.py @@ -34,6 +34,7 @@ from .communication_factory import CommunicationFactory from .deep_ep import DeepEP from .deep_ep_low_latency import DeepEPLowLatency +from .nccl_ep import NcclEP from .nvlink_one_sided import NVLinkOneSided from .nvlink_two_sided import NVLinkTwoSided @@ -46,6 +47,7 @@ "NVLinkOneSided", "DeepEP", "DeepEPLowLatency", + "NcclEP", # Factory "CommunicationFactory", ] diff --git a/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py b/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py index af03d15f2188..f3237880d178 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py +++ b/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py @@ -33,6 +33,7 @@ from .base import Communication from .deep_ep import DeepEP from .deep_ep_low_latency import DeepEPLowLatency +from .nccl_ep import NcclEP from .nvlink_one_sided import NVLinkOneSided from .nvlink_two_sided import NVLinkTwoSided from .nvlink_two_sided_flashinfer import NVLinkTwoSidedFlashinfer @@ -69,6 +70,7 @@ def create_strategy( 2. Auto-selection (tries in order): - NVLinkOneSided (highest priority for throughput) - NVLinkTwoSided (high priority for latency) + - NcclEP (if nccl-ep is available) - DeepEP (if enabled via TRTLLM_CAN_USE_DEEP_EP) - DeepEPLowLatency (if enabled via TRTLLM_CAN_USE_DEEP_EP) - AllGather + ReduceScatter (fallback, always works) @@ -133,7 +135,7 @@ def create_strategy( ) # Auto-selection: Try strategies in priority order using try-catch - # Priority: NVLinkOneSided > NVLinkTwoSided > DeepEP > DeepEPLowLatency > AllGather + # Priority: NVLinkOneSided > NVLinkTwoSided > NcclEP > DeepEP > DeepEPLowLatency > AllGather try: enable_eplb = model_config.moe_load_balancer is not None @@ -191,6 +193,34 @@ def create_strategy( except Exception as e: logger.info(f"NVLinkTwoSided not available: {e}") + # Try NCCL EP (rank-major LL). Falls through to DeepEP/AllGather if + # prerequisites are not met or libnccl_ep.so is not available. + nccl_ep_unavailable_reason = CommunicationFactory._get_nccl_ep_unavailable_reason( + act_dtype, + quant_config, + num_slots, + hidden_size, + max_num_tokens, + moe_max_num_tokens, + top_k, + ) + if nccl_ep_unavailable_reason is None: + try: + strategy = NcclEP( + mapping, + num_slots, + hidden_size, + max_num_tokens, + moe_max_num_tokens, + top_k=top_k, + ) + logger.info("Selected communication strategy: NcclEP") + return strategy + except RuntimeError as e: + logger.debug(f"NcclEP not available: {e}") + else: + logger.debug(f"NcclEP not available: {nccl_ep_unavailable_reason}") + # Try DeepEP (if enabled and weight dtype is bfloat16) if os.environ.get("TRTLLM_CAN_USE_DEEP_EP", "1") == "1" and act_dtype == torch.bfloat16: try: @@ -345,7 +375,56 @@ def _create_forced_method( use_low_precision_combine, moe_max_num_tokens, ) + elif method == "NCCL_EP": + nccl_ep_unavailable_reason = CommunicationFactory._get_nccl_ep_unavailable_reason( + act_dtype, + quant_config, + num_slots, + hidden_size, + max_num_tokens, + moe_max_num_tokens, + top_k, + ) + if nccl_ep_unavailable_reason is not None: + raise ValueError(nccl_ep_unavailable_reason) + return NcclEP( + mapping, + num_slots, + hidden_size, + max_num_tokens, + moe_max_num_tokens, + top_k=top_k, + ) elif method == "ALLGATHER": return AllGatherReduceScatter(mapping) else: raise ValueError(f"Unknown communication method: {method}") + + @staticmethod + def _get_nccl_ep_unavailable_reason( + act_dtype: torch.dtype, + quant_config, + num_slots: int, + hidden_size: int, + max_num_tokens: int, + moe_max_num_tokens: Optional[int], + top_k: int, + ) -> Optional[str]: + if act_dtype != torch.bfloat16: + return f"NcclEP requires act_dtype=torch.bfloat16, got {act_dtype}." + if quant_config is not None: + quant_mode = getattr(quant_config, "layer_quant_mode", None) + if quant_mode is not None and quant_mode.has_any_quant(exclude_kv_cache=True): + return "NcclEP v0.1 does not support quantized MoE communication." + if num_slots <= 0 or hidden_size <= 0 or max_num_tokens <= 0: + return ( + "NcclEP requires positive num_slots, hidden_size, and max_num_tokens, got " + f"{num_slots=}, {hidden_size=}, {max_num_tokens=}." + ) + if moe_max_num_tokens is not None and moe_max_num_tokens <= 0: + return ( + f"NcclEP requires moe_max_num_tokens > 0 when provided, got {moe_max_num_tokens}." + ) + if top_k <= 0 or top_k > num_slots: + return f"NcclEP requires 0 < top_k <= num_slots, got {top_k=}, {num_slots=}." + return None diff --git a/tensorrt_llm/_torch/modules/fused_moe/communication/nccl_ep.py b/tensorrt_llm/_torch/modules/fused_moe/communication/nccl_ep.py new file mode 100644 index 000000000000..53cf15b2469c --- /dev/null +++ b/tensorrt_llm/_torch/modules/fused_moe/communication/nccl_ep.py @@ -0,0 +1,397 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NCCL EP (Expert Parallelism) Communication Strategy for MoE -- LL rank-major. + +Targets the ``nccl.ep`` Python package shipped in the nccl4py wheel (built +against an NCCL master tree containing ``contrib/nccl_ep``). The dispatch +returns rank-major LL outputs: + + * ``recv_x`` : 3D ``[ep_size, max_tokens_per_rank, hidden]`` bf16, + reshaped to 2D for the downstream MoE pipeline. + * ``recv_topk_idx`` : 2D ``[..., top_k]`` int32 with real expert IDs (-1 for invalid rows) + * ``recv_topk_weights`` : 2D ``[..., top_k]`` float32 (the original router weights) + +This matches NVLinkOneSided's contract directly, so NO +``_modify_output_to_adapt_fused_moe`` adapter is needed. The MoE backend's +``fused_moe`` runs top_k experts per row, applies the weights, and produces one +reduced output per row. ``handle.combine`` then sums per-source-rank +contributions back to the home rank. + +Persistent handle: ``Group.create_handle`` is called ONCE (first dispatch); +subsequent dispatches call ``handle.update(topk_idx, ...)`` to rebind routing. +CUDA-graph capture is supported once the handle exists. +""" + +from typing import List, Optional, Tuple + +import torch + +from tensorrt_llm.logger import logger +from tensorrt_llm.mapping import Mapping + +from .base import Communication + +_NCCL_RUNTIME_ERRORS = (RuntimeError, OSError) + + +class NcclEP(Communication): + """NCCL EP Low-Latency rank-major communication strategy for MoE expert parallelism.""" + + def __init__( + self, + mapping: Mapping, + num_slots: int, + hidden_size: int, + max_num_tokens: int = 1024, + moe_max_num_tokens: Optional[int] = None, + top_k: int = 8, + ): + super().__init__(mapping) + + from tensorrt_llm._torch.modules.fused_moe.nccl_ep_utils import is_nccl_ep_installed + + if not is_nccl_ep_installed(): + raise RuntimeError("nccl-ep is not installed.") + + if self.ep_size <= 0: + raise ValueError(f"NcclEP requires moe_ep_size > 0, got {self.ep_size}") + if not 0 <= self.ep_rank < self.ep_size: + raise ValueError( + f"NcclEP requires 0 <= moe_ep_rank < moe_ep_size, " + f"got {self.ep_rank=}, {self.ep_size=}" + ) + if num_slots <= 0: + raise ValueError(f"NcclEP requires num_slots > 0, got {num_slots}") + if num_slots % self.ep_size != 0: + raise ValueError( + f"NcclEP requires num_slots divisible by moe_ep_size, " + f"got {num_slots=}, {self.ep_size=}" + ) + if hidden_size <= 0: + raise ValueError(f"NcclEP requires hidden_size > 0, got {hidden_size}") + if top_k <= 0 or top_k > num_slots: + raise ValueError(f"NcclEP requires 0 < top_k <= num_slots, got {top_k=}, {num_slots=}") + if max_num_tokens <= 0: + raise ValueError(f"NcclEP requires max_num_tokens > 0, got {max_num_tokens}") + if moe_max_num_tokens is not None and moe_max_num_tokens <= 0: + raise ValueError( + f"NcclEP requires moe_max_num_tokens > 0 when provided, got {moe_max_num_tokens}" + ) + + self.num_slots = num_slots + self.num_experts = num_slots + self.hidden_size = hidden_size + self.num_local_experts = num_slots // self.ep_size + self.max_top_k = top_k + + self.max_tokens_per_rank = ( + max_num_tokens + if moe_max_num_tokens is None + else min(max_num_tokens, moe_max_num_tokens) + ) + self.max_recv_tokens = self.ep_size * self.max_tokens_per_rank + + # Singleton NCCL EP context: owns the EP group, RDMA buffers, and + # persistent OUTPUT Tensor descriptors. Allocate it lazily on first + # dispatch because full-model construction runs under MetaInitMode, + # which redirects torch.empty to the meta device even when a CUDA + # device is passed explicitly. + self._ctx = None + + # Persistent dispatch handle. Created on first dispatch via + # group.create_handle; reused thereafter via handle.update so + # subsequent dispatches are CUDA-graph-safe. + self._handle = None # nccl.ep.Handle | None + self._dispatch_state: dict = {} + + @staticmethod + def is_platform_supported() -> bool: + from tensorrt_llm._torch.modules.fused_moe.nccl_ep_utils import is_nccl_ep_installed + + return is_nccl_ep_installed() + + def is_workload_feasible(self, all_rank_num_tokens: List[int], num_chunks: int) -> bool: + if num_chunks > 1: + return False + if max(all_rank_num_tokens) > self.max_tokens_per_rank: + return False + return True + + def _get_context(self): + if self._ctx is None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "NcclEP context must be initialized before CUDA graph capture. " + "Run an eager warmup forward before enabling or capturing CUDA graphs." + ) + from nccl.ep import Layout + + from tensorrt_llm._torch.modules.fused_moe.nccl_ep_utils import get_nccl_ep_context + + self._ctx = get_nccl_ep_context( + self.mapping, + self.num_experts, + self.max_tokens_per_rank, + self.hidden_size, + self.max_top_k, + Layout.RANK_MAJOR, + ) + return self._ctx + + def _setup_handle(self, ctx, topk_nd, stream): + """Ensure self._handle exists; rebind topk via handle.update on subsequent calls.""" + if self._handle is None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "NcclEP dispatch handle must be initialized before CUDA graph capture. " + "Run an eager warmup forward before enabling or capturing CUDA graphs." + ) + self._handle = ctx.ep_group.create_handle( + ctx.layout, + topk_nd, + stream=stream, + ) + else: + self._handle.update(topk_nd, stream=stream) + return self._handle + + # ------------------------------------------------------------------ + # Dispatch -- rank-major LL + # ------------------------------------------------------------------ + + def dispatch( + self, + hidden_states: torch.Tensor, + hidden_states_sf: Optional[torch.Tensor], + token_selected_slots: torch.Tensor, + token_final_scales: Optional[torch.Tensor], + all_rank_num_tokens: List[int], + use_dp_padding: Optional[bool] = None, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], torch.Tensor, Optional[torch.Tensor]]: + """Dispatch tokens via NCCL EP LL rank-major. + + Returns rank-major-shaped tensors directly: + (recv_hs [N, H], recv_sf [N, H/128] or None, recv_slots [N, top_k] int32, + recv_scales [N, top_k] float32) + + where N = ep_size * max_tokens_per_rank. Rows beyond + ``recv_rank_counter[r]`` for source rank r have recv_slots = -1 + (sentinel), naturally skipped by the MoE backend. + """ + from nccl.ep import DispatchConfig, DispatchInputs, DispatchOutputs, LayoutInfo, Tensor + + ctx = self._get_context() + + all_rank_max_num_tokens = max(all_rank_num_tokens) + if all_rank_max_num_tokens > self.max_tokens_per_rank: + raise ValueError( + f"all_rank_max_num_tokens={all_rank_max_num_tokens} > " + f"max_tokens_per_rank={self.max_tokens_per_rank}" + ) + + num_tokens = hidden_states.shape[0] + top_k = token_selected_slots.shape[1] + if top_k > self.max_top_k: + raise ValueError(f"top_k={top_k} exceeds configured max_top_k={self.max_top_k}") + if token_final_scales is None: + raise RuntimeError( + "NcclEP rank-major dispatch requires token_final_scales " + "(router weights) -- it is an INPUT to handle.dispatch." + ) + + stream = ctx.get_stream() + + # TODO(NCCL): topk_weights still requires float32; once bf16/native + # weights are accepted upstream, drop this conversion too. + weights_f32 = ( + token_final_scales + if token_final_scales.dtype == torch.float32 + else token_final_scales.to(torch.float32) + ) + hidden_states_c = hidden_states.contiguous() + weights_f32_c = weights_f32.contiguous() + + input_tokens_nd = Tensor(hidden_states_c) + input_topk_weights_nd = Tensor(weights_f32_c) + + # Mark padding rows with the -1 sentinel so fused_moe skips them. + # The dispatch kernel only writes recv_topk_idx for slots that + # received tokens; rows beyond `recv_rank_counter[r]` keep stale + # data from prior dispatches. recv_rank_counter is written fresh + # by the dispatch kernel (low_latency.cu:877) so it does not need + # pre-zeroing, and recv_topk_weights on -1 rows is don't-care + # (fused_moe ignores the weight when the expert id is -1). + ctx.recv_topk_idx_buf.fill_(-1) + + outputs = DispatchOutputs( + tokens=ctx.output_tokens_nd, + topk_weights=ctx.recv_topk_weights_nd, + topk_idx=ctx.recv_topk_idx_nd, + scales=None, + ) + layout_info = LayoutInfo(src_rank_counters=ctx.recv_rank_counter_nd) + # The v0.2-gated capability path asks the kernel to emit global + # expert ids directly; v0.1 retains the default local-id contract + # and uses the translation below. + if ctx._expert_id_kind_global is not None: + layout_info._lowpp.recv_topk_idx_kind = ctx._expert_id_kind_global + + topk_idx_dev = token_selected_slots.to(ctx.topk_idx_dtype).contiguous() + topk_nd = Tensor(topk_idx_dev) + handle = self._setup_handle(ctx, topk_nd, stream) + inputs = DispatchInputs( + tokens=input_tokens_nd, + topk_weights=input_topk_weights_nd, + ) + handle.dispatch( + inputs, + outputs, + layout_info=layout_info, + config=DispatchConfig(round_scales=0), + stream=stream, + ) + + # The handle internally references topk_nd; keep both the Tensor + # descriptor and its backing torch tensor alive until combine completes. + self._dispatch_state = { + "num_tokens": num_tokens, + "topk_nd": topk_nd, + "topk_idx_dev": topk_idx_dev, + } + + # Match NVLinkOneSided's contract: token_selected_slots in + # [0, num_experts) for valid rows, -1 for invalid. When the kernel + # writes GLOBAL ids directly (opt-in detected at ctx init), the + # buffer is already in the right space and we pass it through. + # Otherwise the kernel writes LOCAL ids in [0, num_local_experts) + # and we add ep_rank * num_local_experts to restore the global + # numbering downstream consumers expect. + # The dispatch buffer is 3D [ep_size, max_tokens_per_rank, max_top_k] + # per the LL rank-major contract; flatten to 2D for downstream. + recv_topk_idx_flat = ctx.recv_topk_idx_buf.view(self.max_recv_tokens, self.max_top_k) + if ctx.kernel_writes_global_ids: + recv_slots_global = recv_topk_idx_flat + else: + recv_slots_global = torch.where( + recv_topk_idx_flat >= 0, + recv_topk_idx_flat + self.ep_rank * self.num_local_experts, + recv_topk_idx_flat, + ) + + # Output buffers are 3D [ep_size, max_tokens_per_rank, ...] per the + # LL rank-major contract; downstream MoE pipeline expects 2D -- + # flatten via view. + return ( + ctx.output_tokens_buf.view(self.max_recv_tokens, self.hidden_size), + None, + recv_slots_global, + ctx.recv_topk_weights_buf.view(self.max_recv_tokens, self.max_top_k), + ) + + # ------------------------------------------------------------------ + # Combine -- rank-major LL + # ------------------------------------------------------------------ + + def combine( + self, + final_hidden_states: torch.Tensor, + **kwargs, + ) -> torch.Tensor: + """Combine MoE-reduced rank-major output back to the home rank. + + Input: [max_recv_tokens, hidden] -- already weighted per-row by fused_moe. + Output: [num_tokens, hidden] -- combined to original token order. + """ + from nccl.ep import CombineInputs, CombineOutputs, Tensor + + ctx = self._ctx + if ctx is None: + raise RuntimeError("NcclEP.combine called before dispatch.") + state = self._dispatch_state + stream = ctx.get_stream() + + num_tokens = state["num_tokens"] + + # NCCL-EP LL combine consumes rank-major tokens with shape + # [ep_size, max_tokens_per_rank, hidden]. The scheduler normally + # provides the equivalent 2D [max_recv_tokens, hidden] view. + if final_hidden_states.dim() == 2: + expected_shape = (self.max_recv_tokens, self.hidden_size) + if tuple(final_hidden_states.shape) != expected_shape: + raise ValueError( + f"combine input shape={tuple(final_hidden_states.shape)} " + f"expected={expected_shape}" + ) + final_hidden_states = final_hidden_states.view( + self.ep_size, + self.max_tokens_per_rank, + self.hidden_size, + ) + elif final_hidden_states.dim() == 3: + expected_shape = (self.ep_size, self.max_tokens_per_rank, self.hidden_size) + if tuple(final_hidden_states.shape) != expected_shape: + raise ValueError( + f"combine input shape={tuple(final_hidden_states.shape)} " + f"expected={expected_shape}" + ) + else: + raise ValueError( + "NcclEP combine input must be 2D [max_recv_tokens, hidden] or " + "3D [ep_size, max_tokens_per_rank, hidden], got " + f"shape={tuple(final_hidden_states.shape)}" + ) + + combine_input_c = final_hidden_states.contiguous() + combine_output = torch.empty( + num_tokens, + self.hidden_size, + dtype=torch.bfloat16, + device=combine_input_c.device, + ) + + combine_input_nd = Tensor(combine_input_c) + combine_output_nd = Tensor(combine_output) + + # Rank-major combine: no layout_info, no config required (send_only=0 + # is the default; defaults round-trip fine). + self._handle.combine( + CombineInputs(tokens=combine_input_nd), + CombineOutputs(tokens=combine_output_nd), + stream=stream, + ) + + self._dispatch_state = {} + return combine_output + + def destroy(self): + """Release per-instance NCCL EP resources (handle). + + NcclEpContext is shared across instances and released through a + refcounted cache. + """ + if self._handle is not None: + try: + self._handle.destroy() + except _NCCL_RUNTIME_ERRORS as e: + logger.warning(f"Handle.destroy error during destroy: {e}") + self._handle = None + + from tensorrt_llm._torch.modules.fused_moe.nccl_ep_utils import release_nccl_ep_context + + if self._ctx is not None: + release_nccl_ep_context(self._ctx) + self._ctx = None + self._dispatch_state = {} diff --git a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py index c9f1300deaaf..16ff3b20010c 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py @@ -292,9 +292,8 @@ def _create_and_sync_backend( """Build the MoE backend, mirror EPLB attrs, then create weights. Why this dance: - - ``init_load_balancer=False`` / ``without_comm=True``: the backend - would otherwise re-register itself with the load balancer and - initialize its own communication; ConfigurableMoE owns both. + - ``init_load_balancer=False``: the backend would otherwise + re-register itself with the load balancer; ConfigurableMoE owns it. - ``layer_idx=None``: the wrapper passes the real ``layer_idx`` to ``MoE.__init__`` to drive load-balancer setup. The backend receives ``None`` so its own EPLB hooks no-op until we sync the @@ -341,7 +340,6 @@ def _create_and_sync_backend( swiglu_limit=kwargs.get("swiglu_limit"), swiglu_limit_scalar=kwargs.get("swiglu_limit_scalar"), init_load_balancer=False, - without_comm=True, activation_type=self.activation_type, trtllm_gen_activation_type=trtllm_gen_activation_type, trtllm_gen_activation_alpha=trtllm_gen_activation_alpha, diff --git a/tensorrt_llm/_torch/modules/fused_moe/create_moe.py b/tensorrt_llm/_torch/modules/fused_moe/create_moe.py index adfd8ed48722..c428f57b3e8f 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/create_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/create_moe.py @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import os from typing import Dict, Optional, Type import torch @@ -64,10 +63,15 @@ def get_moe_cls( quant_config = override_quant_config layer_prefix = f"[layer_idx={layer_idx}] " if layer_idx is not None else "" if moe_backend.upper() == "MARLIN": - # Marlin MoE is a Hopper-specific NVFP4 W4A16 backend. Require nvfp4 - # quantization explicitly so a misconfigured model fails fast. + # Marlin MoE is a Hopper-specific NVFP4 W4A16 backend. Layers without + # NVFP4 quantization (e.g. deliberately-unquantized MTP draft layers in + # MIXED_PRECISION checkpoints) fall back to CutlassFusedMoE, matching + # the CUTEDSL / DENSEGEMM / MEGAMOE_* fallback behavior below. if quant_config is None or not quant_config.quant_mode.has_nvfp4(): - raise ValueError("MarlinFusedMoE only supports NVFP4 quantization.") + logger.warning(f"{layer_prefix}MarlinFusedMoE only supports NVFP4 " + "quantization. Check out details in quant_config: " + f"{quant_config}. Using CutlassFusedMoE instead.") + return CutlassFusedMoE return MarlinFusedMoE if moe_backend.upper() == "CUTLASS": return CutlassFusedMoE @@ -256,7 +260,6 @@ def create_moe_backend( swiglu_limit: Optional[torch.Tensor] = None, swiglu_limit_scalar: Optional[float] = None, init_load_balancer: bool = True, - without_comm: bool = False, activation_type: ActivationType = ActivationType.Swiglu, trtllm_gen_activation_type: Optional[ActType_TrtllmGen] = None, trtllm_gen_activation_alpha: Optional[float] = None, @@ -344,9 +347,11 @@ def create_moe_backend( ], f"swiglu_limit is not supported in {moe_cls.__name__}." if swiglu_limit_scalar is not None: + # MegaMoECuteDsl uses the scalar only as a fallback when no per-expert + # tensor limit is given (see the MegaMoE branch below). assert moe_cls in [ CutlassFusedMoE, TRTLLMGenFusedMoE, WideEPMoE, DeepGemmFusedMoE, - MegaMoEDeepGemm, CuteDslFusedMoE + MegaMoEDeepGemm, CuteDslFusedMoE, MegaMoECuteDsl ], f"swiglu_limit_scalar is not supported in {moe_cls.__name__}." if moe_cls == TRTLLMGenFusedMoE: @@ -369,17 +374,15 @@ def create_moe_backend( swiglu_limit=swiglu_limit, swiglu_limit_scalar=swiglu_limit_scalar, init_load_balancer=init_load_balancer, - without_comm=without_comm, activation_type=activation_type, trtllm_gen_activation_type=trtllm_gen_activation_type, trtllm_gen_activation_alpha=trtllm_gen_activation_alpha, trtllm_gen_activation_beta=trtllm_gen_activation_beta, ) - if any(value is not None - for value in (trtllm_gen_activation_type, - trtllm_gen_activation_alpha, - trtllm_gen_activation_beta)): + if any(value is not None for value in (trtllm_gen_activation_type, + trtllm_gen_activation_alpha, + trtllm_gen_activation_beta)): raise ValueError( "TRTLLM-Gen backend-local activation options are only supported " f"by TRTLLMGenFusedMoE, got {moe_cls.__name__}") @@ -405,7 +408,6 @@ def create_moe_backend( swiglu_limit=swiglu_limit, swiglu_limit_scalar=swiglu_limit_scalar, init_load_balancer=init_load_balancer, - without_comm=without_comm, activation_type=activation_type, ) elif moe_cls == WideEPMoE: @@ -457,7 +459,6 @@ def create_moe_backend( layer_idx=layer_idx, swiglu_limit_scalar=swiglu_limit_scalar, init_load_balancer=init_load_balancer, - without_comm=without_comm, activation_type=activation_type, ) elif moe_cls == DeepGemmFusedMoE: @@ -476,7 +477,6 @@ def create_moe_backend( init_load_balancer=init_load_balancer, swiglu_limit=swiglu_limit, swiglu_limit_scalar=swiglu_limit_scalar, - without_comm=without_comm, ) elif moe_cls == TritonFusedMoE: assert not apply_router_weight_on_input, "apply_router_weight_on_input is not supported in TritonFusedMoE." @@ -510,7 +510,6 @@ def create_moe_backend( apply_router_weight_on_input=apply_router_weight_on_input, layer_idx=layer_idx, init_load_balancer=init_load_balancer, - without_comm=without_comm, activation_type=activation_type, ) elif moe_cls in (MegaMoEDeepGemm, MegaMoECuteDsl): @@ -532,11 +531,14 @@ def create_moe_backend( apply_router_weight_on_input=apply_router_weight_on_input, layer_idx=layer_idx, init_load_balancer=init_load_balancer, - without_comm=without_comm, activation_type=activation_type, ) if moe_cls is MegaMoECuteDsl: - megamoe_kwargs["swiglu_limit"] = swiglu_limit + # ``_resolve_gate_up_clamp`` accepts tensor or scalar; fall back + # to the scalar form when only that was wired. + megamoe_kwargs["swiglu_limit"] = (swiglu_limit + if swiglu_limit is not None else + swiglu_limit_scalar) else: megamoe_kwargs["swiglu_limit_scalar"] = swiglu_limit_scalar return moe_cls(**megamoe_kwargs) @@ -619,67 +621,45 @@ def create_moe( moe_cls = resolve_moe_cls(model_config, routing_method, dtype, override_quant_config, layer_idx) - if (any(value is not None - for value in (trtllm_gen_activation_type, - trtllm_gen_activation_alpha, - trtllm_gen_activation_beta)) + if (any(value is not None for value in (trtllm_gen_activation_type, + trtllm_gen_activation_alpha, + trtllm_gen_activation_beta)) and moe_cls is not TRTLLMGenFusedMoE): raise ValueError( "A TRTLLM-Gen backend-local activation requires " "TRTLLMGenFusedMoE without backend fallback, but resolved " f"{moe_cls.__name__}.") - enable_configurable_moe = os.environ.get("ENABLE_CONFIGURABLE_MOE", - "1") == "1" - if enable_configurable_moe or moe_cls in (CuteDslFusedMoE, - CuteDslB12xFusedMoE): - if moe_cls in (DeepGemmFusedMoE, TRTLLMGenFusedMoE, CuteDslFusedMoE, - CuteDslB12xFusedMoE, CutlassFusedMoE, DenseGEMMFusedMoE, - MegaMoEDeepGemm, MegaMoECuteDsl, MarlinFusedMoE): - return ConfigurableMoE( - routing_method=routing_method, - num_experts=num_experts, - hidden_size=hidden_size, - intermediate_size=intermediate_size, - dtype=dtype, - reduce_results=reduce_results, - model_config=model_config, - aux_stream_dict=aux_stream_dict, - weight_loading_mode=weight_loading_mode, - apply_router_weight_on_input=apply_router_weight_on_input, - layer_idx=layer_idx, - override_quant_config=override_quant_config, - bias=bias, - swiglu_alpha=swiglu_alpha, - swiglu_beta=swiglu_beta, - swiglu_limit=swiglu_limit, - swiglu_limit_scalar=swiglu_limit_scalar, - activation_type=activation_type, - trtllm_gen_activation_type=trtllm_gen_activation_type, - trtllm_gen_activation_alpha=trtllm_gen_activation_alpha, - trtllm_gen_activation_beta=trtllm_gen_activation_beta, - communication_method=communication_method, - ) - else: - # Check if this is a TRTLLM or CUTEDSL backend request that fell back to CutlassFusedMoE - requested_backend = model_config.moe_backend.upper() - if requested_backend in ("TRTLLM", - "CUTEDSL") and moe_cls == CutlassFusedMoE: - # Workaround for test cases where TRTLLM backend falls back to CutlassFusedMoE due to quant_config incompatibility - # Log warning and continue with the fallback backend - logger.warning( - f"ENABLE_CONFIGURABLE_MOE is set but {requested_backend} backend fell back to {moe_cls.__name__} due to quant_config. " - f"ConfigurableMoE only supports TRTLLMGenFusedMoE and CuteDslFusedMoE backends. " - f"Continuing with legacy MoE backend {moe_cls.__name__}.") - else: - # Other backends are not supported by ConfigurableMoE, fallback to legacy backend - # This is a WAR to make sure all the CI test cases pass. - # TODO: Remove this workaround when ConfigurableMoE is supported by all backends. - logger.warning( - f"ENABLE_CONFIGURABLE_MOE is set but {moe_cls.__name__} is not supported by ConfigurableMoE. " - f"Continuing with legacy MoE backend {moe_cls.__name__}.") - - # Use legacy create_moe_backend for other backends or when ConfigurableMoE is disabled + if moe_cls in (DeepGemmFusedMoE, TRTLLMGenFusedMoE, CuteDslFusedMoE, + CuteDslB12xFusedMoE, CutlassFusedMoE, DenseGEMMFusedMoE, + MegaMoEDeepGemm, MegaMoECuteDsl, MarlinFusedMoE): + return ConfigurableMoE( + routing_method=routing_method, + num_experts=num_experts, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + dtype=dtype, + reduce_results=reduce_results, + model_config=model_config, + aux_stream_dict=aux_stream_dict, + weight_loading_mode=weight_loading_mode, + apply_router_weight_on_input=apply_router_weight_on_input, + layer_idx=layer_idx, + override_quant_config=override_quant_config, + bias=bias, + swiglu_alpha=swiglu_alpha, + swiglu_beta=swiglu_beta, + swiglu_limit=swiglu_limit, + swiglu_limit_scalar=swiglu_limit_scalar, + activation_type=activation_type, + trtllm_gen_activation_type=trtllm_gen_activation_type, + trtllm_gen_activation_alpha=trtllm_gen_activation_alpha, + trtllm_gen_activation_beta=trtllm_gen_activation_beta, + communication_method=communication_method, + ) + + # WideEPMoE, TritonFusedMoE and VanillaMoE are not wrapped by ConfigurableMoE + # and own their communication and forward paths. if communication_method is not None: raise ValueError("communication_method requires ConfigurableMoE.") return create_moe_backend( diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py index 2db1e7aa7cba..35ac71b9f1b1 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py @@ -31,14 +31,12 @@ Sm100BlockScaledContiguousGroupedGemmFinalizeFusionRunner, Sm100BlockScaledContiguousGroupedGemmRunner, Sm100BlockScaledContiguousGroupedGemmSwigluFusionRunner) -from ...distributed import allgather from ...model_config import ModelConfig from ...utils import (ActivationType, AuxStreamType, EventType, Fp4QuantizedTensor, get_last_power_of_2_num_tokens_buckets, last_positive_power_of_2) from .fused_moe_cutlass import CutlassFusedMoE -from .interface import AlltoallMethodType from .quantization import MoEWeightLoadingMode, NVFP4CuteDslFusedMoEMethod from .routing import BaseMoeRoutingMethod @@ -436,7 +434,6 @@ def __init__( layer_idx: Optional[int] = None, swiglu_limit_scalar: Optional[float] = None, init_load_balancer: bool = True, - without_comm: bool = False, activation_type: ActivationType = ActivationType.Swiglu, ): super().__init__( @@ -453,7 +450,6 @@ def __init__( layer_idx=layer_idx, swiglu_limit_scalar=swiglu_limit_scalar, init_load_balancer=init_load_balancer, - without_comm=without_comm, activation_type=activation_type, ) self.swiglu_limit_scalar = swiglu_limit_scalar or float("inf") @@ -482,9 +478,6 @@ def _build_local_weight_view(self) -> NvFp4WeightView: slot_start=self.slot_start, ) - def select_alltoall_method_type(self) -> AlltoallMethodType: - return AlltoallMethodType.NotEnabled - def _get_quant_method(self): if self.quant_config is not None and self.quant_config.layer_quant_mode.has_any_quant( exclude_kv_cache=True): @@ -855,44 +848,6 @@ def run_moe( ) return result - def forward_chunk( - self, - x: Union[torch.Tensor, Fp4QuantizedTensor], - router_logits: torch.Tensor, - input_ids: Optional[torch.IntTensor] = None, - output_dtype: Optional[torch.dtype] = None, - all_rank_num_tokens: Optional[List[int]] = None, - use_dp_padding: Optional[bool] = None, - repeating_info: tuple = (True, True), - ) -> torch.Tensor: - # Currently, the default path is that ConfigurableMoE calls CuteDslFusedMoE.run_moe. - # This forward_chunk method is a reference implementation of the legacy path. - # Apply routing - token_selected_experts, token_final_scales = self.routing_method.apply( - router_logits, input_ids) - assert token_selected_experts.shape[ - 1] == self.routing_method.experts_per_token - assert token_selected_experts.shape == token_final_scales.shape - assert token_selected_experts.shape[0] == router_logits.shape[0] - assert token_final_scales.dtype == torch.float32 - assert token_selected_experts.dtype == torch.int32 - - x, x_sf = self.quantize_input(x) - - if self.use_dp and self.parallel_size > 1: - x, x_sf, token_selected_experts, token_final_scales = allgather( - [x, x_sf, token_selected_experts, token_final_scales], - self.mapping, - dim=0, - sizes=None if use_dp_padding else all_rank_num_tokens) - - x = self.run_moe(x=x, - token_selected_experts=token_selected_experts, - token_final_scales=token_final_scales, - x_sf=x_sf, - enable_alltoall=False) - return x - def load_weights(self, weights: List[Dict], allow_partial_loading: bool = False): diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cutlass.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cutlass.py index af612a4ff0d3..cad3629f0d44 100755 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cutlass.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_cutlass.py @@ -14,21 +14,13 @@ # limitations under the License. import inspect -import os -from functools import cached_property from typing import Dict, List, Optional, Tuple, Union import torch -from tensorrt_llm._mnnvl_utils import MnnvlMemory, MnnvlMoe -from tensorrt_llm._torch.distributed.moe_alltoall import MoeAlltoAll from tensorrt_llm._utils import get_sm_version -from tensorrt_llm.logger import logger from tensorrt_llm.models.modeling_utils import QuantAlgo -from tensorrt_llm.tools.layer_wise_benchmarks import get_calibrator -from ...distributed import allgather -from ...expert_statistic import ExpertStatistic from ...model_config import ModelConfig from ...peft.lora.layer import (MOE_LORA_MODULE_NAMES, MOE_LORA_MODULE_TO_KERNEL_SLOT, LoraModuleType, @@ -36,9 +28,8 @@ from ...peft.lora.validation import has_moe_lora_targets from ...utils import (ActivationType, AuxStreamType, EventType, Fp4QuantizedTensor) -from .interface import AlltoallMethodType, MoE +from .interface import MoE from .quantization import UnquantizedFusedMoEMethod -from .wide_ep_ft import get_wide_ep_ft_options # isort: off from .quantization import ( @@ -267,7 +258,6 @@ def __init__( swiglu_limit: Optional[torch.Tensor] = None, swiglu_limit_scalar: Optional[float] = None, init_load_balancer: bool = True, - without_comm: bool = False, activation_type: ActivationType = ActivationType.Swiglu, ): @@ -331,72 +321,6 @@ def __init__( self.has_been_profiled = False self.has_been_profiled_min_latency = False - # When without_comm=True, skip communication initialization (ConfigurableMoE will handle it) - if not without_comm: - self.alltoall_method_type = self.select_alltoall_method_type() - logger.info_once( - f"{self.__class__.__name__} selects alltoall_method_type {self.alltoall_method_type!r}", - key="alltoall_method_type") - self.alltoall_workspace = None - self.alltoall_prepare_workspace = None - self.use_low_precision_combine = False - if self.enable_alltoall: - self.use_low_precision_combine = model_config.use_low_precision_moe_combine - - if self.alltoall_method_type == AlltoallMethodType.NVLinkTwoSided: - MnnvlMemory.initialize() - self.alltoall_workspace = MnnvlMoe.get_moe_workspaces( - model_config.mapping) - self.alltoall_prepare_workspace = MnnvlMoe.get_moe_prepare_workspace( - model_config.mapping) - elif self.alltoall_method_type == AlltoallMethodType.NVLinkOneSided: - # Calculate required workspace size - ep_size = self.mapping.moe_ep_size - max_num_tokens = model_config.max_num_tokens - hidden_size = self.hidden_size - dtype = self.dtype or torch.float16 - - workspace_size = MoeAlltoAll.calculate_required_workspace_size( - ep_size, - self.routing_method.experts_per_token, - max_num_tokens, - hidden_size, - dtype, - self.num_experts if self.layer_load_balancer else None, - ) - ep_group_health, watchdog_timeout_s, watchdog_poll_interval_s = ( - get_wide_ep_ft_options(model_config)) - - self.moe_a2a = MoeAlltoAll( - mapping=self.mapping, - max_num_tokens=model_config.max_num_tokens, - top_k=self.routing_method.experts_per_token, - num_slots=self.num_slots, - workspace_size_per_rank=workspace_size, - num_experts=self.num_experts - if self.layer_load_balancer else None, - ep_group_health=ep_group_health, - alltoall_watchdog_timeout_s=watchdog_timeout_s, - alltoall_watchdog_poll_interval_s= - watchdog_poll_interval_s, - ) - elif self.alltoall_method_type == AlltoallMethodType.DeepEP or self.alltoall_method_type == AlltoallMethodType.DeepEPLowLatency: - raise NotImplementedError( - "DeepEP and DeepEPLowLatency are not supported for CutlassFusedMoE yet" - ) - else: - raise NotImplementedError( - f"Unsupported alltoall method type: {self.alltoall_method_type!r}" - ) - else: - # When without_comm=True, set minimal attributes - # Communication will be handled by parent wrapper (e.g., ConfigurableMoE) - self.alltoall_method_type = AlltoallMethodType.NotEnabled - self.alltoall_workspace = None - self.alltoall_prepare_workspace = None - self.use_low_precision_combine = False - self.moe_a2a = None - # If True, the router weight will be multiplied on the input rather than at the end of FC2 self.apply_router_weight_on_input = apply_router_weight_on_input @@ -483,10 +407,12 @@ def reserve_moe_lora_cuda_graph_workspace(self, max_num_tokens: int, """ if not self._moe_lora_enabled or max_num_tokens <= 0: return - # MoE LoRA only runs on the unquantized fp16/bf16 path (the C++ op - # rejects quantized weights), so a quantized layer can never reach the - # LoRA scratch; skip and let the (impossible) runtime path error loudly. - if getattr(self, "has_any_quant", False): + # MoE LoRA runs on the unquantized fp16/bf16 path or the per-tensor FP8 + # (qdq) path (see moeOp.cpp). Any other quant mode (FP8 block-scale, + # NVFP4, MXFP8, integer WoQ) is rejected by the C++ op, so a layer in + # those modes can never reach the LoRA scratch; skip and let the runtime + # path error loudly. + if getattr(self, "has_any_quant", False) and not self.has_fp8_qdq: return # Weights must exist to read the runner's weight dtype. If they have not # been created yet, skip; the lazy sizing + in-capture guard still @@ -502,27 +428,35 @@ def reserve_moe_lora_cuda_graph_workspace(self, max_num_tokens: int, assert max_lora_size > 0, ( "reserve_moe_lora_cuda_graph_workspace requires max_lora_size > 0 " f"(got {max_lora_size}).") - # MoE LoRA only runs on the unquantized fp16/bf16 path, so the MoERunner - # instance key below (all-False quant flags, x/weight/out == self.dtype) - # must match the key the runtime fused_moe op uses on the same layer; - # otherwise the reservation lands on a different cached C++ runner. - assert self.dtype in (torch.float16, torch.bfloat16), ( - "MoE LoRA requires fp16/bf16 activations to reserve a deterministic " - f"FusedMoeRunner key; got {self.dtype}.") + + # The reservation must land on the *same* cached C++ FusedMoeRunner that + # the runtime torch.ops.trtllm.fused_moe op uses on this layer, so the + # MoERunner instance key must match the runtime key exactly. The runtime + # key uses the activation dtype the op sees: + # - per-tensor FP8 (qdq): quantize_input casts activations to e4m3, so + # x/weight are fp8 and the output (LoRA compute) dtype is self.dtype; + # - unquantized fp16/bf16: x/weight/output all equal self.dtype. + # Every quant flag in the key is False for both (per-tensor FP8 is not + # block-scaled / MXFP8 / W4). If a runtime call ever uses a different + # key, the C++ capture guard surfaces a clear error rather than + # corrupting replay. + weight_dtype = self.w3_w1_weight.dtype + if self.has_fp8_qdq: + act_dtype = torch.float8_e4m3fn + output_dtype = self.dtype + else: + assert self.dtype in (torch.float16, torch.bfloat16), ( + "MoE LoRA requires fp16/bf16 activations to reserve a " + f"deterministic FusedMoeRunner key; got {self.dtype}.") + act_dtype = self.dtype + output_dtype = self.dtype from ...custom_ops.torch_custom_ops import MoERunner - # Build the MoERunner with the same instance key the functional - # torch.ops.trtllm.fused_moe op uses, so we reserve on the *same* cached - # C++ FusedMoeRunner that capture will use. For the unquantized LoRA - # path x/weight/output dtypes all equal self.dtype and every quant flag - # is False. If a runtime call ever uses a different key, the C++ - # capture guard surfaces a clear error rather than corrupting replay. - weight_dtype = self.w3_w1_weight.dtype runner = MoERunner( - x_dtype=self.dtype, + x_dtype=act_dtype, weight_dtype=weight_dtype, - output_dtype=self.dtype, + output_dtype=output_dtype, top_k=self.routing_method.experts_per_token, tp_size=self.tp_size, tp_rank=self.tp_rank, @@ -766,41 +700,6 @@ def has_int8_woq_per_channel(self): return self.quant_config and self.quant_config.layer_quant_mode.is_int8_weight_only( ) and not self.quant_config.layer_quant_mode.has_per_group_scaling() - def select_alltoall_method_type(self) -> AlltoallMethodType: - # If no attention DP, no need to use AlltoAll. - if self.mapping.dp_size == 1: - return AlltoallMethodType.NotEnabled - - # AlltoAll cannot support MoE TP. - if self.mapping.moe_tp_size != 1: - return AlltoallMethodType.NotEnabled - - if not MnnvlMemory.supports_mnnvl(): - return AlltoallMethodType.NotEnabled - - all2all_method_type = os.environ.get("TRTLLM_FORCE_ALLTOALL_METHOD") - if all2all_method_type is not None: - if AlltoallMethodType[all2all_method_type] in [ - AlltoallMethodType.DeepEP, - AlltoallMethodType.DeepEPLowLatency - ]: - raise NotImplementedError( - "DeepEP and DeepEPLowLatency are not supported for CutlassFusedMoE yet" - ) - return AlltoallMethodType[all2all_method_type] - - # TODO: We found that NVLinkOneSided performs better than NCCL AllGather/ReduceScatter, - # regardless of the relationship between EP size and topK. We favor NVLinkOneSided for now. - # if not self.mapping.moe_ep_size > self.routing_method.experts_per_token: - # return AlltoallMethodType.NotEnabled - return AlltoallMethodType.NVLinkOneSided - - @cached_property - def enable_alltoall(self): - """ enable_alltoall (bool): whether to enable alltoall instead of allgather/reducescatter - """ - return self.alltoall_method_type != AlltoallMethodType.NotEnabled - def quantize_input( self, x: Union[torch.Tensor, Fp4QuantizedTensor], @@ -1202,413 +1101,6 @@ def _run_moe_w4a16_nvfp4( return moe_output return result[0] - def forward_chunk( - self, - x: Union[torch.Tensor, Fp4QuantizedTensor], - router_logits: torch.Tensor, - input_ids: Optional[torch.IntTensor], - output_dtype: Optional[torch.dtype] = None, - all_rank_num_tokens: Optional[List[int]] = None, - use_dp_padding: Optional[bool] = None, - repeating_info: tuple = (True, True), - lora_params: Optional[Dict] = None, - ) -> torch.Tensor: - if isinstance(x, Fp4QuantizedTensor): - assert output_dtype is not None - else: - output_dtype = x.dtype - - is_first_call, is_last_call = repeating_info - - self._load_balancer_start_wait_gpu_stage(is_first_call) - - # apply routing - token_selected_experts, token_final_scales = self.routing_method.apply( - router_logits, input_ids) - assert token_selected_experts.shape[ - 1] == self.routing_method.experts_per_token - assert token_selected_experts.shape == token_final_scales.shape - assert token_selected_experts.shape[0] == router_logits.shape[0] - assert token_final_scales.dtype == torch.float32 - assert token_selected_experts.dtype == torch.int32 - - if self.layer_load_balancer: - self._load_balancer_done_wait_gpu_stage(is_first_call) - ignore_allreduce = self.enable_alltoall and self.alltoall_method_type in ( - AlltoallMethodType.NVLinkTwoSided, - AlltoallMethodType.NVLinkOneSided, - ) - self._load_balancer_update_statistic( - token_selected_experts, - is_first_call, - is_last_call, - ignore_allreduce=ignore_allreduce) - token_selected_slots = self._load_balancer_route( - token_selected_experts, self.use_dp) - else: - token_selected_slots = token_selected_experts - - # If load balancer is disabled, the statistics are collected from expert IDs. - # If load balancer is enabled, the statistics are collected from expert slot IDs. - ExpertStatistic.set_layer(self.layer_idx) - ExpertStatistic.maybe_add_info(self.num_slots, token_selected_slots) - token_selected_slots = get_calibrator().maybe_collect_or_replay_slots( - self.num_slots, token_selected_slots) - - if self.apply_router_weight_on_input: - assert x.dtype != torch.float8_e4m3fn, "Current workaround for apply_router_weight_on_input does not support fp8 input" - x = x * token_final_scales.to(x.dtype) - # TODO: remove this once we have correct fusedmoe kernel ready - token_final_scales = None - - run_post_quant_allgather = self.use_dp and self.parallel_size > 1 - - # Quantize inputs using extracted method - # For post_quant_comm scenarios, x_sf will be reshaped to 2D inside quantize_input - post_quant_comm = run_post_quant_allgather or self.enable_alltoall - x, x_sf = self.quantize_input(x, post_quant_comm=post_quant_comm) - - # Prepare additional information for profiling in case padding is applied when using alltoall. - # Only the non-alltoall case is considered for profiling in the warmup phase. - # Therefore, to get the correct tactics during the actual inference, the inputs to the tuner should be the same as when not using alltoall. - if self.enable_alltoall: - if all_rank_num_tokens is not None: - tuner_num_tokens = sum(all_rank_num_tokens) - else: - tuner_num_tokens = x.shape[0] * self.mapping.tp_size - tuner_top_k = token_selected_slots.shape[1] - else: - tuner_num_tokens = None - tuner_top_k = None - - # Alltoall or allgather for attention DP - token_count = x.shape[0] - alltoall_info = None # Store for later combine - is_sf_swizzled = True # In case of post-quant communication, scaling factors will not be swizzled before communication, and swizzling after communication is merged into MoE. - if self.enable_alltoall: - assert all_rank_num_tokens is not None, "all_rank_num_tokens required for alltoall" - # Prepare alltoall indices - top_k = self.routing_method.experts_per_token - runtime_max_tokens_per_rank = max( - all_rank_num_tokens) if all_rank_num_tokens else token_count - - # Handle case where token_final_scales might be None (when apply_router_weight_on_input=True) - if token_final_scales is None: - token_final_scales = torch.ones_like(token_selected_slots, - dtype=torch.float32) - - if self.alltoall_method_type == AlltoallMethodType.NVLinkTwoSided: - assert self.alltoall_prepare_workspace is not None, "alltoall_prepare_workspace should be initialized" - if is_last_call: - loadbalancer_local_statistic_info = self._load_balancer_get_local_statistic_tensor( - ) - else: - loadbalancer_local_statistic_info = None - alltoall_info, gathered_loadbalancer_local_statistic_info = MnnvlMoe.mnnvl_moe_alltoallv_prepare_without_allgather( - token_selected_slots, loadbalancer_local_statistic_info, - self.alltoall_prepare_workspace, - runtime_max_tokens_per_rank, self.ep_rank, self.ep_size, - self.num_experts, self.num_slots, top_k) - if gathered_loadbalancer_local_statistic_info is not None: - gathered_loadbalancer_local_statistic_info = gathered_loadbalancer_local_statistic_info.view( - (self.mapping.moe_ep_size, self.num_experts)) - self._load_balancer_update_statistic_with_gathered_statistic( - gathered_loadbalancer_local_statistic_info) - - # Dispatch x, x_sf, token_selected_slots, token_final_scales in one alltoall kernel - x, x_sf, token_selected_slots, token_final_scales = MnnvlMoe.mnnvl_moe_alltoallv( - [x, x_sf, token_selected_slots, token_final_scales], - alltoall_info, self.alltoall_workspace, self.ep_rank, - self.ep_size) - - torch.ops.trtllm.memset_expert_ids( - token_selected_slots, alltoall_info.recv_rank_count_cumsum, - runtime_max_tokens_per_rank, top_k, self.num_slots, - self.ep_size) - elif self.alltoall_method_type == AlltoallMethodType.NVLinkOneSided: - # Python MoeAlltoAll path - - payloads = [] - payloads.append(x) - if x_sf is not None: - payloads.append(x_sf) - expert_id_payload_index = 2 - else: - expert_id_payload_index = 1 - payloads.append(token_selected_slots) - payloads.append(token_final_scales) - - loadbalancer_local_statistic_info = None - if self.layer_load_balancer and is_last_call: - loadbalancer_local_statistic_info = self._load_balancer_get_local_statistic_tensor( - ) - if loadbalancer_local_statistic_info is not None: - recv_tensors = self.moe_a2a.dispatch( - token_selected_slots, - payloads, - runtime_max_tokens_per_rank, - invalid_token_expert_id=self. - num_slots, # Caution: Cutlass MoE uses num_slots as invalid token expert id - expert_id_payload_index=expert_id_payload_index, - eplb_local_stats=loadbalancer_local_statistic_info, - ) - gathered_stats = self.moe_a2a._state.eplb_gathered_stats - self._load_balancer_update_statistic_with_gathered_statistic( - gathered_stats) - else: - recv_tensors = self.moe_a2a.dispatch( - token_selected_slots, - payloads, - runtime_max_tokens_per_rank, - invalid_token_expert_id=self. - num_slots, # Caution: Cutlass MoE uses num_slots as invalid token expert id - expert_id_payload_index=expert_id_payload_index, - ) - - if x_sf is not None: - x_recv, x_sf_recv, token_selected_slots_recv, token_final_scales_recv = recv_tensors - x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1]) - else: - x_recv, token_selected_slots_recv, token_final_scales_recv = recv_tensors - x = x_recv.view(-1, x_recv.shape[-1]) - token_selected_slots = token_selected_slots_recv.view( - -1, token_selected_slots_recv.shape[-1]) - token_final_scales = token_final_scales_recv.view( - -1, token_final_scales_recv.shape[-1]) - else: - raise ValueError( - f"Unsupported moe alltoall method type: {self.alltoall_method_type}" - ) - - elif run_post_quant_allgather: - # Original allgather logic - # x_sf is already 2D after quantize_input with post_quant_comm=True - - x, x_sf, token_selected_slots, token_final_scales = allgather( - [x, x_sf, token_selected_slots, token_final_scales], - self.mapping, - dim=0, - sizes=None if use_dp_padding else all_rank_num_tokens) - - # Optionally provide an output tensor to fused_moe so it writes directly to our buffer - moe_output: Optional[torch.Tensor] = None - if self.alltoall_method_type == AlltoallMethodType.NVLinkOneSided: - # Retrieve a workspace-backed output tensor sized by runtime tokens - runtime_max_tokens_per_rank = max( - all_rank_num_tokens) if all_rank_num_tokens else x.shape[0] - moe_output = self.moe_a2a.get_combine_payload_tensor_in_workspace( - runtime_max_tokens_per_rank, self.unpadded_hidden_size, - output_dtype) - - # Call extracted run_moe method - final_hidden_states = self.run_moe( - x=x, - token_selected_experts=token_selected_slots, - token_final_scales=token_final_scales, - x_sf=x_sf, - is_sf_swizzled=not post_quant_comm, - output_dtype=output_dtype, - tuner_num_tokens=tuner_num_tokens, - tuner_top_k=tuner_top_k, - moe_output=moe_output, - lora_params=lora_params, - ) - - self._load_balancer_start_set_cpu_stage(is_last_call) - - # Combine results if using alltoall - if self.enable_alltoall: - if self.alltoall_method_type == AlltoallMethodType.NVLinkTwoSided: - if alltoall_info is not None: - top_k = self.routing_method.experts_per_token - final_hidden_states = MnnvlMoe.mnnvl_moe_alltoallv_combine( - final_hidden_states, - alltoall_info, - self.alltoall_workspace, - ep_rank=self.ep_rank, - ep_size=self.ep_size, - top_k=top_k, - use_low_precision_combine=self. - use_low_precision_combine, - token_count=token_count) - elif self.alltoall_method_type == AlltoallMethodType.NVLinkOneSided: - output_hidden_size = final_hidden_states.shape[-1] - runtime_max_tokens_per_rank = max( - all_rank_num_tokens) if all_rank_num_tokens else token_count - final_hidden_states = self.moe_a2a.combine( - final_hidden_states.view(self.ep_size, - runtime_max_tokens_per_rank, - output_hidden_size), - runtime_max_tokens_per_rank, - payload_in_workspace=True) - else: - raise ValueError( - f"Unsupported moe alltoall method type: {self.alltoall_method_type}" - ) - - self._load_balancer_done_set_cpu_stage(is_last_call) - - return final_hidden_states - - def split_chunk(self, split_token_num: int, split_num_chunks: int): - val_div = split_token_num // split_num_chunks - val_mod = split_token_num % split_num_chunks - split_chunk_size_list = [val_div + 1] * val_mod + [val_div] * ( - split_num_chunks - val_mod) - return split_chunk_size_list - - def forward_impl( - self, - x: Union[torch.Tensor, Fp4QuantizedTensor], - router_logits: torch.Tensor, - *, - input_ids: Optional[torch.IntTensor] = None, - do_finalize: bool = True, # used by other MoE backends - output_dtype: Optional[torch.dtype] = None, - all_rank_num_tokens: Optional[List[int]] = None, - use_dp_padding: Optional[bool] = None, - lora_params: Optional[Dict] = None, - **kwargs, - ) -> torch.Tensor: - assert do_finalize, "CutlassFusedMoE does not support do_finalize=False" - if not self._moe_lora_enabled and self._moe_lora_active(lora_params): - # Caller passed MoE LoRA tensors but this layer was not configured - # for it. Surface a clear error rather than silently ignoring. - raise RuntimeError( - "Received MoE LoRA params for a CutlassFusedMoE layer that was " - "not configured with LoRA target modules. Ensure " - "`lora_config.lora_target_modules` includes the desired MoE modules." - ) - if self.use_dp and self.parallel_size > 1: - assert all_rank_num_tokens is not None - assert use_dp_padding is not None - num_rows = sum(all_rank_num_tokens) - else: - num_rows = x.shape[0] - - if use_dp_padding: - all_rank_num_tokens_padded = [max(all_rank_num_tokens) - ] * len(all_rank_num_tokens) - num_rows = sum(all_rank_num_tokens_padded) - else: - all_rank_num_tokens_padded = all_rank_num_tokens - - # in case of num_rows is larger than max_chunk_size, we need to split the input into multiple chunks - num_chunks = (num_rows + self.moe_max_num_tokens - - 1) // self.moe_max_num_tokens - - if num_chunks > 1 and self._moe_lora_active(lora_params): - raise_moe_lora_multichunk_unsupported(num_chunks) - - if num_chunks == 1: - is_first_call = self.repeat_idx == 0 - is_last_call = self.repeat_idx == self.repeat_count - 1 - outputs = self.forward_chunk( - x, - router_logits, - input_ids=input_ids, - output_dtype=output_dtype, - all_rank_num_tokens=all_rank_num_tokens_padded, - use_dp_padding=use_dp_padding, - repeating_info=(is_first_call, is_last_call), - lora_params=lora_params) - outputs = self.reducescatter_or_allreduce( - outputs, - all_rank_num_tokens=all_rank_num_tokens_padded, - use_dp_padding=use_dp_padding) - else: - if self.use_dp: - all_rank_chunk_size_list = [ - self.split_chunk(val, num_chunks) - for val in all_rank_num_tokens_padded - ] - all_rank_num_tokens_list = [[ - val[idx_chunk] for val in all_rank_chunk_size_list - ] for idx_chunk in range(num_chunks)] - chunk_size_list = all_rank_chunk_size_list[self.parallel_rank] - else: - all_rank_num_tokens_list = [None] * num_chunks - chunk_size_list = self.split_chunk(x.shape[0], num_chunks) - - x_list = x.split(chunk_size_list) - router_logits_list = router_logits.split(chunk_size_list) - input_ids_list = input_ids.split( - chunk_size_list) if input_ids is not None else [None - ] * num_chunks - - self.event_dict[EventType.Main].record() - with torch.cuda.stream(self.aux_stream): - self.event_dict[EventType.Main].wait() - - def _forward_chunk(x_, router_logits_, input_ids_, idx): - is_first_call = idx == 0 and self.repeat_idx == 0 - is_last_call = idx == num_chunks - 1 and self.repeat_idx == self.repeat_count - 1 - return self.forward_chunk( - x_, - router_logits_, - input_ids=input_ids_, - all_rank_num_tokens=all_rank_num_tokens_list[idx] - if self.use_dp else None, - use_dp_padding=use_dp_padding, - repeating_info=(is_first_call, is_last_call), - lora_params=lora_params) - - def _reducescatter_or_allreduce(x_, idx): - return self.reducescatter_or_allreduce( - x_, - all_rank_num_tokens=all_rank_num_tokens_list[idx], - use_dp_padding=use_dp_padding) - - outputs_list = [] - # Postpone reduce-scatter/all-reduce to the next iteration to achieve better overlap - for idx_chunk, (x, router_logits, input_ids) in enumerate( - zip(x_list, router_logits_list, input_ids_list)): - if not (self.alltoall_method_type - == AlltoallMethodType.NVLinkOneSided - or self.alltoall_method_type - == AlltoallMethodType.NVLinkTwoSided): - if idx_chunk % 2 == 0: - with torch.cuda.stream(self.aux_stream): - outputs = _forward_chunk(x, router_logits, - input_ids, idx_chunk) - if idx_chunk > 0: - outputs_list[-1] = _reducescatter_or_allreduce( - outputs_list[-1], idx_chunk - 1) - else: - outputs = _forward_chunk(x, router_logits, input_ids, - idx_chunk) - with torch.cuda.stream(self.aux_stream): - outputs_list[-1] = _reducescatter_or_allreduce( - outputs_list[-1], idx_chunk - 1) - else: - outputs = _forward_chunk(x, router_logits, input_ids, - idx_chunk) - - outputs_list.append(outputs) - - if not (self.alltoall_method_type - == AlltoallMethodType.NVLinkOneSided - or self.alltoall_method_type - == AlltoallMethodType.NVLinkTwoSided): - if num_chunks % 2 == 0: - outputs_list[-1] = _reducescatter_or_allreduce( - outputs_list[-1], -1) - else: - with torch.cuda.stream(self.aux_stream): - outputs_list[-1] = _reducescatter_or_allreduce( - outputs_list[-1], -1) - with torch.cuda.stream(self.aux_stream): - self.event_dict[EventType.MoeChunkingOverlap].record() - self.event_dict[EventType.MoeChunkingOverlap].wait() - - outputs = torch.cat(outputs_list) - - if self.use_dp and self.parallel_size > 1: - rank = self.parallel_rank - outputs = outputs[:all_rank_num_tokens[rank]] - self.repeat_idx = 0 if self.repeat_idx == self.repeat_count - 1 else self.repeat_idx + 1 - return outputs - def forward_fake( self, x: Union[torch.Tensor, Fp4QuantizedTensor], diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_deepgemm.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_deepgemm.py index 601bc0abed74..f030559b358e 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_deepgemm.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_deepgemm.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union import torch import triton @@ -24,12 +24,10 @@ from tensorrt_llm._utils import get_sm_version, nvtx_range from tensorrt_llm.models.modeling_utils import QuantAlgo -from ...distributed import allgather from ...memory_buffer_utils import get_memory_buffers from ...model_config import ModelConfig -from ...utils import AuxStreamType, EventType, Fp4QuantizedTensor +from ...utils import AuxStreamType, Fp4QuantizedTensor from .fused_moe_cutlass import CutlassFusedMoE -from .interface import AlltoallMethodType from .quantization import (DeepSeekFP8BlockScalesFusedMoEMethodDeepGemm, MoEWeightLoadingMode, UnquantizedFusedMoEMethod) from .routing import BaseMoeRoutingMethod @@ -800,7 +798,6 @@ def __init__( swiglu_limit: Optional[torch.Tensor] = None, swiglu_limit_scalar: Optional[float] = None, init_load_balancer: bool = True, - without_comm: bool = False, ): # moe_max_num_tokens is set in ModelConfig.__post_init__ if not specified # The default value is max_num_tokens * dp_size @@ -831,7 +828,6 @@ def __init__( swiglu_limit=swiglu_limit, swiglu_limit_scalar=swiglu_limit_scalar, init_load_balancer=init_load_balancer, - without_comm=without_comm, ) def get_workspace(self, m_max: int, group_size: int): @@ -900,10 +896,6 @@ def _get_quant_method(self): else: return UnquantizedFusedMoEMethod() - def select_alltoall_method_type(self) -> AlltoallMethodType: - """DeepGEMM backend currently doesn't support alltoall; honor overrides but default to disabled.""" - return AlltoallMethodType.NotEnabled - def quantize_input( self, x: Union[torch.Tensor, Fp4QuantizedTensor], @@ -1117,205 +1109,3 @@ def run_moe( ) return final_hidden_states - - @nvtx_range("[DG] forward") - def forward_chunk( - self, - x: Union[torch.Tensor, Fp4QuantizedTensor], - router_logits: torch.Tensor, - input_ids: Optional[torch.IntTensor] = None, - output_dtype: Optional[torch.dtype] = None, - all_rank_num_tokens: Optional[List[int]] = None, - use_dp_padding: Optional[bool] = None, - workspace: Optional[dict] = None, - ) -> torch.Tensor: - if isinstance(x, Fp4QuantizedTensor): - assert output_dtype is not None - else: - output_dtype = x.dtype - - # apply routing - token_selected_experts, token_final_scales = self.routing_method.apply( - router_logits, input_ids) - assert token_selected_experts.shape[ - 1] == self.routing_method.experts_per_token - assert token_selected_experts.shape == token_final_scales.shape - assert token_selected_experts.shape[0] == router_logits.shape[0] - assert token_final_scales.dtype == torch.float32 - assert token_selected_experts.dtype == torch.int32 - - if self.apply_router_weight_on_input: - assert self.routing_method.top_k == 1, "Current workaround only supports top-1 routing" - assert x.dtype != torch.float8_e4m3fn, "Current workaround for apply_router_weight_on_input does not support fp8 input" - x = x * token_final_scales.to(x.dtype) - # TODO: remove this once we have correct fusedmoe kernel ready - token_final_scales = None - - # quantize inputs - x_sf = None - if self.has_any_quant: - if self.has_deepseek_fp8_block_scales: - pass - else: - raise ValueError( - f"unsupported quantization mode for CUTEDSL backend: {self.quant_config.quant_mode}" - ) - - use_allgather = self.use_dp and self.parallel_size > 1 - if use_allgather: - x, x_sf, token_selected_experts, token_final_scales = allgather( - [x, x_sf, token_selected_experts, token_final_scales], - self.mapping, - dim=0, - sizes=None if use_dp_padding else all_rank_num_tokens) - - # Call run_moe to handle the core MoE computation - final_hidden_states = self.run_moe( - x=x, - token_selected_experts=token_selected_experts, - token_final_scales=token_final_scales, - x_sf=x_sf, - workspace=workspace, - ) - - return final_hidden_states - - def forward_impl( - self, - x: Union[torch.Tensor, Fp4QuantizedTensor], - router_logits: torch.Tensor, - *, - input_ids: Optional[torch.IntTensor] = None, - do_finalize: bool = True, # used by other MoE backends - output_dtype: Optional[torch.dtype] = None, - all_rank_num_tokens: Optional[List[int]] = None, - use_dp_padding: Optional[bool] = None, - **kwargs, - ) -> torch.Tensor: - assert do_finalize, "CutlassFusedMoE does not support do_finalize=False" - if self.use_dp and self.parallel_size > 1: - assert all_rank_num_tokens is not None - assert use_dp_padding is not None - num_rows = sum(all_rank_num_tokens) - else: - num_rows = x.shape[0] - - # In case of num_rows is larger than max_chunk_size * 2, we need to split the input into multiple chunks. - # Because we will use two streams in chunked moe and preallocate two workspaces. - num_chunks = 1 - if num_rows > self.moe_max_num_tokens * 2: - num_chunks = (num_rows + self.moe_max_num_tokens - - 1) // self.moe_max_num_tokens - - if use_dp_padding: - all_rank_num_tokens_padded = [max(all_rank_num_tokens) - ] * len(all_rank_num_tokens) - else: - all_rank_num_tokens_padded = all_rank_num_tokens - - if num_chunks == 1: - # create workspace - num_rows = x.shape[0] - if self.use_dp: - num_rows = sum(all_rank_num_tokens_padded) - workspaces = self.get_workspaces([num_rows]) - outputs = self.forward_chunk( - x, - router_logits, - input_ids=input_ids, - output_dtype=output_dtype, - all_rank_num_tokens=all_rank_num_tokens_padded, - use_dp_padding=use_dp_padding, - workspace=workspaces[0]) - outputs = self.reducescatter_or_allreduce( - outputs, - all_rank_num_tokens=all_rank_num_tokens_padded, - use_dp_padding=use_dp_padding) - else: - if self.use_dp: - all_rank_chunk_size_list = [ - self.split_chunk(val, num_chunks) - for val in all_rank_num_tokens_padded - ] - all_rank_num_tokens_list = [[ - val[idx_chunk] for val in all_rank_chunk_size_list - ] for idx_chunk in range(num_chunks)] - chunk_size_list = all_rank_chunk_size_list[self.parallel_rank] - else: - all_rank_num_tokens_list = [None] * num_chunks - chunk_size_list = self.split_chunk(x.shape[0], num_chunks) - - # create workspace - chunk_size_0 = sum(all_rank_num_tokens_list[0] - ) if self.use_dp else chunk_size_list[0] - chunk_size_1 = sum(all_rank_num_tokens_list[1] - ) if self.use_dp else chunk_size_list[1] - workspaces = self.get_workspaces([chunk_size_0, chunk_size_1]) - workspace_0 = workspaces[0] - workspace_1 = workspaces[1] - - x_list = x.split(chunk_size_list) - router_logits_list = router_logits.split(chunk_size_list) - input_ids_list = input_ids.split( - chunk_size_list) if input_ids is not None else [None - ] * num_chunks - - self.event_dict[EventType.Main].record() - with torch.cuda.stream(self.aux_stream): - self.event_dict[EventType.Main].wait() - - def _forward_chunk(x_, router_logits_, input_ids_, idx, workspace): - return self.forward_chunk( - x_, - router_logits_, - input_ids=input_ids_, - all_rank_num_tokens=all_rank_num_tokens_list[idx] - if self.use_dp else None, - use_dp_padding=use_dp_padding, - workspace=workspace) - - def _reducescatter_or_allreduce(x_, idx): - return self.reducescatter_or_allreduce( - x_, - all_rank_num_tokens=all_rank_num_tokens_list[idx], - use_dp_padding=use_dp_padding) - - outputs_list = [] - # Postpone reduce-scatter/all-reduce to the next iteration to achieve better overlap - for idx_chunk, (x, router_logits, input_ids_chunk) in enumerate( - zip(x_list, router_logits_list, input_ids_list)): - - if idx_chunk % 2 == 0: - with torch.cuda.stream(self.aux_stream): - outputs = _forward_chunk(x, router_logits, - input_ids_chunk, idx_chunk, - workspace_0) - if idx_chunk > 0: - outputs_list[-1] = _reducescatter_or_allreduce( - outputs_list[-1], idx_chunk - 1) - else: - outputs = _forward_chunk(x, router_logits, input_ids_chunk, - idx_chunk, workspace_1) - with torch.cuda.stream(self.aux_stream): - outputs_list[-1] = _reducescatter_or_allreduce( - outputs_list[-1], idx_chunk - 1) - - outputs_list.append(outputs) - - if num_chunks % 2 == 0: - outputs_list[-1] = _reducescatter_or_allreduce( - outputs_list[-1], -1) - else: - with torch.cuda.stream(self.aux_stream): - outputs_list[-1] = _reducescatter_or_allreduce( - outputs_list[-1], -1) - with torch.cuda.stream(self.aux_stream): - self.event_dict[EventType.MoeChunkingOverlap].record() - self.event_dict[EventType.MoeChunkingOverlap].wait() - - outputs = torch.cat(outputs_list) - - if self.use_dp and self.parallel_size > 1: - rank = self.parallel_rank - outputs = outputs[:all_rank_num_tokens[rank]] - return outputs diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_densegemm.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_densegemm.py index df5c71bb179e..6aac29f34ce6 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_densegemm.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_densegemm.py @@ -10,7 +10,6 @@ from tensorrt_llm.models.modeling_utils import QuantAlgo from tensorrt_llm.quantization.utils import fp4_utils -from ...distributed import allgather from ...memory_buffer_utils import get_memory_buffers from ...model_config import ModelConfig from ...utils import AuxStreamType, EventType, Fp4QuantizedTensor, swizzle_sf, unswizzle_sf @@ -157,7 +156,6 @@ def __init__( apply_router_weight_on_input: bool = False, layer_idx: Optional[int] = None, init_load_balancer: bool = True, - without_comm: bool = False, activation_type=None, ): # DenseGEMM CuTe DSL kernels only support SM100 and SM103. @@ -201,9 +199,9 @@ def __init__( ) # Call MoE base class directly (not CutlassFusedMoE). - # Note: `without_comm` and `apply_router_weight_on_input` are accepted - # for API compatibility with create_moe_backend() but are not passed to - # MoE.__init__() since DenseGEMM does not use alltoall communication. + # Note: `apply_router_weight_on_input` is accepted for API + # compatibility with create_moe_backend() but is not passed to + # MoE.__init__(). super().__init__( routing_method=routing_method, num_experts=num_experts, @@ -535,77 +533,3 @@ def run_moe( x_sf=x_sf, enable_alltoall=enable_alltoall, ) - - def forward_chunk( - self, - x: Union[torch.Tensor, Fp4QuantizedTensor], - router_logits: torch.Tensor, - output_dtype: Optional[torch.dtype] = None, - all_rank_num_tokens: Optional[List[int]] = None, - use_dp_padding: Optional[bool] = None, - repeating_info: tuple = (True, True), - ) -> torch.Tensor: - # Currently, the default path is that ConfigurableMoE calls DenseGEMMFusedMoE.run_moe. - # This forward_chunk method is a reference implementation of the legacy path. - # Apply routing - token_selected_experts, token_final_scales = self.routing_method.apply(router_logits) - assert token_selected_experts.shape[1] == self.routing_method.experts_per_token - assert token_selected_experts.shape == token_final_scales.shape - assert token_selected_experts.shape[0] == router_logits.shape[0] - assert token_final_scales.dtype == torch.float32 - assert token_selected_experts.dtype == torch.int32 - - x, x_sf = self.quantize_input(x) - - if self.use_dp and self.parallel_size > 1: - x, x_sf, token_selected_experts, token_final_scales = allgather( - [x, x_sf, token_selected_experts, token_final_scales], - self.mapping, - dim=0, - sizes=None if use_dp_padding else all_rank_num_tokens, - ) - - x = self.run_moe( - x=x, - token_selected_experts=token_selected_experts, - token_final_scales=token_final_scales, - x_sf=x_sf, - enable_alltoall=False, - ) - return x - - def forward_impl( - self, - x: Union[torch.Tensor, Fp4QuantizedTensor], - router_logits: torch.Tensor, - *, - do_finalize: bool = True, - output_dtype: Optional[torch.dtype] = None, - all_rank_num_tokens: Optional[List[int]] = None, - use_dp_padding: Optional[bool] = None, - **kwargs, - ) -> torch.Tensor: - assert do_finalize, "DenseGEMMFusedMoE does not support do_finalize=False" - - is_first_call = self.repeat_idx == 0 - is_last_call = self.repeat_idx == self.repeat_count - 1 - - outputs = self.forward_chunk( - x, - router_logits, - output_dtype, - all_rank_num_tokens=all_rank_num_tokens, - use_dp_padding=use_dp_padding, - repeating_info=(is_first_call, is_last_call), - ) - outputs = self.reducescatter_or_allreduce( - outputs, - all_rank_num_tokens=all_rank_num_tokens, - use_dp_padding=use_dp_padding, - ) - - if self.use_dp and self.parallel_size > 1: - rank = self.parallel_rank - outputs = outputs[: all_rank_num_tokens[rank]] - self.repeat_idx = 0 if self.repeat_idx == self.repeat_count - 1 else self.repeat_idx + 1 - return outputs diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_marlin.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_marlin.py index e6e9f3c7ea7c..c4728be2b140 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_marlin.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_marlin.py @@ -104,31 +104,6 @@ def _get_quant_method(self): def _supports_load_balancer(self) -> bool: return False - def validate_configurable_moe(self, moe) -> None: - """Reject configs that require external-communication MoE dispatch. - - Marlin is W4A16 (``quantize_input`` produces no activation scale) and - routes internally inside ``run_moe``. The host-side all-to-all - dispatch/combine path that ConfigurableMoE uses for attention-DP - expert parallelism needs the routing decided *before* dispatch and a - per-token scale payload, so it is incompatible with Marlin and fails - inside ``moe_a2a_dispatch``. - - ConfigurableMoE only creates an external communication strategy when - ``enable_attention_dp and dp_size > 1`` (see CommunicationFactory); - ``moe.comm`` is not assigned yet when this hook runs, so check that - same condition via the mapping. Single-GPU, and TP/EP without - attention DP, are supported. - """ - if moe.use_dp and moe.mapping.dp_size > 1: - raise ValueError( - "MarlinFusedMoE does not support external-communication MoE " - "(attention data parallelism combined with expert " - "parallelism): its W4A16 layout and internal routing are " - "incompatible with the all-to-all dispatch path. Use Marlin " - "single-node, or with TP/EP without attention DP." - ) - def _apply_activation(self, gemm1_out: torch.Tensor) -> torch.Tensor: """Apply the activation function to the gemm1 output. @@ -202,6 +177,15 @@ def run_moe( local_n = self.expert_size_per_partition if local_n != self.num_experts: + # EP: non-local token-expert pairs are clamped to local expert 0 + # with a zero final scale, so they still run through both GEMMs + # and are discarded at the combine. + # TODO(perf): skip them instead — e.g. mark non-local pairs with a + # sentinel expert id that moe_align_block_size drops. Requires + # bounds guards in moeAlignKernels.cu (the histogram and + # count_and_sort kernels index shared/cumsum buffers with the raw + # id) and zero-initializing unscheduled GEMM output rows before + # the index_add_ combine. slot_start = self.slot_start is_local = (token_selected_experts >= slot_start) & ( token_selected_experts < slot_start + local_n diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py index ab7b4631ba95..472439042bee 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py @@ -16,28 +16,21 @@ import inspect import os from dataclasses import dataclass -from functools import cached_property from typing import Dict, List, Optional, Tuple, Union import torch from torch import nn -from tensorrt_llm._mnnvl_utils import MnnvlMemory, MnnvlMoe -from tensorrt_llm._torch.distributed.moe_alltoall import MoeAlltoAll from tensorrt_llm._utils import get_sm_version -from tensorrt_llm.logger import logger from tensorrt_llm.models.modeling_utils import QuantAlgo from ...custom_ops.trtllm_gen_custom_ops import \ fp4_block_scale_fake_output_without_finalize -from ...distributed import allgather -from ...expert_statistic import ExpertStatistic from ...model_config import ModelConfig from ...utils import (ActivationType, ActType_TrtllmGen, AuxStreamType, Fp4QuantizedTensor, MxFp8QuantizedTensor) -from .interface import AlltoallMethodType, MoE, MoEWeightLoadingMode +from .interface import MoE, MoEWeightLoadingMode from .moe_op_backend import MoEOpBackend, TRTLLMOpBackend, get_op_backend -from .wide_ep_ft import get_wide_ep_ft_options # isort: off from .quantization import ( @@ -48,7 +41,7 @@ # isort: on from .routing import (BaseMoeRoutingMethod, DeepSeekV3MoeRoutingMethod, DeepSeekV4MoeRoutingMethod, DefaultMoeRoutingMethod, - MiniMaxM2MoeRoutingMethod) + MiniMaxM2MoeRoutingMethod, MiniMaxM3MoeRoutingMethod) @dataclass @@ -211,7 +204,6 @@ def __init__( swiglu_limit: Optional[torch.Tensor] = None, swiglu_limit_scalar: Optional[float] = None, init_load_balancer: bool = True, - without_comm: bool = False, activation_type: ActivationType = ActivationType.Swiglu, trtllm_gen_activation_type: Optional[ActType_TrtllmGen] = None, trtllm_gen_activation_alpha: Optional[float] = None, @@ -272,69 +264,6 @@ def __init__( # - self.expert_size_per_partition = self.num_experts // self.ep_size # - self.initial_global_assignments, self.slot_start, self.slot_end, etc. - # When without_comm=True, skip communication initialization (ConfigurableMoE will handle it) - if not without_comm: - self.alltoall_method_type = self.select_alltoall_method_type() - logger.info_once( - f"{self.__class__.__name__} selects alltoall_method_type {self.alltoall_method_type!r}", - key="alltoall_method_type") - self.alltoall_workspace = None - self.alltoall_prepare_workspace = None - self.use_low_precision_combine = False - if self.enable_alltoall: - self.use_low_precision_combine = model_config.use_low_precision_moe_combine - - if self.alltoall_method_type == AlltoallMethodType.NVLinkTwoSided: - # Initialize appropriate MnnvlMemory implementation - MnnvlMemory.initialize() - self.alltoall_workspace = MnnvlMoe.get_moe_workspaces( - model_config.mapping) - self.alltoall_prepare_workspace = MnnvlMoe.get_moe_prepare_workspace( - model_config.mapping) - - elif self.alltoall_method_type == AlltoallMethodType.NVLinkOneSided: - # Calculate required workspace size - ep_size = self.mapping.moe_ep_size - max_num_tokens = model_config.max_num_tokens - hidden_size = self.hidden_size - dtype = self.dtype or torch.bfloat16 - - workspace_size = MoeAlltoAll.calculate_required_workspace_size( - ep_size, self.routing_method.experts_per_token, - max_num_tokens, hidden_size, dtype, - self.num_experts if self.layer_load_balancer else None) - ep_group_health, watchdog_timeout_s, watchdog_poll_interval_s = ( - get_wide_ep_ft_options(model_config)) - - self.moe_a2a = MoeAlltoAll( - mapping=self.mapping, - max_num_tokens=model_config.max_num_tokens, - top_k=self.routing_method.experts_per_token, - num_slots=self.num_slots, - workspace_size_per_rank=workspace_size, - num_experts=self.num_experts - if self.layer_load_balancer else None, - ep_group_health=ep_group_health, - alltoall_watchdog_timeout_s=watchdog_timeout_s, - alltoall_watchdog_poll_interval_s= - watchdog_poll_interval_s) - elif self.alltoall_method_type == AlltoallMethodType.DeepEP or self.alltoall_method_type == AlltoallMethodType.DeepEPLowLatency: - raise NotImplementedError( - "DeepEP and DeepEPLowLatency are not supported for TRTLLMGenFusedMoE yet" - ) - else: - raise NotImplementedError( - f"Unsupported alltoall method type: {self.alltoall_method_type!r}" - ) - else: - # When without_comm=True, set minimal attributes - # Communication will be handled by parent wrapper (e.g., ConfigurableMoE) - self.alltoall_method_type = AlltoallMethodType.NotEnabled - self.alltoall_workspace = None - self.alltoall_prepare_workspace = None - self.use_low_precision_combine = False - self.moe_a2a = None - self._weights_created = False if not model_config.skip_create_weights_in_init: self.create_weights() @@ -501,35 +430,6 @@ def _get_data_or_none(self, attr_name: str) -> Optional[torch.Tensor]: attr = getattr(self, attr_name, None) return attr.data if attr is not None else None - def select_alltoall_method_type(self) -> AlltoallMethodType: - # If no attention DP, no need to use AlltoAll. - if self.mapping.dp_size == 1: - return AlltoallMethodType.NotEnabled - - # AlltoAll cannot support MoE TP. - if self.mapping.moe_tp_size != 1: - return AlltoallMethodType.NotEnabled - - if not MnnvlMemory.supports_mnnvl(): - return AlltoallMethodType.NotEnabled - - all2all_method_type = os.environ.get("TRTLLM_FORCE_ALLTOALL_METHOD") - if all2all_method_type is not None: - if AlltoallMethodType[all2all_method_type] in [ - AlltoallMethodType.DeepEP, - AlltoallMethodType.DeepEPLowLatency - ]: - raise NotImplementedError( - "DeepEP and DeepEPLowLatency are not supported for CutlassFusedMoE yet" - ) - return AlltoallMethodType[all2all_method_type] - - # We found that NVLinkOneSided performs better than NCCL AllGather/ReduceScatter, - # regardless of the relationship between EP size and topK. We favor NVLinkOneSided for now. - # if not self.mapping.moe_ep_size > self.routing_method.experts_per_token: - # return AlltoallMethodType.NotEnabled - return AlltoallMethodType.NVLinkOneSided - def _supports_load_balancer(self) -> bool: """Whether separated routing (top-k outside the kernel) is used. @@ -541,12 +441,6 @@ def _supports_load_balancer(self) -> bool: return True return self.use_dp and self.parallel_size > 1 - @cached_property - def enable_alltoall(self): - """ enable_alltoall (bool): whether to enable alltoall instead of allgather/reducescatter - """ - return self.alltoall_method_type != AlltoallMethodType.NotEnabled - def _check_configs(self): assert not self.has_any_quant \ or self.has_deepseek_fp8_block_scales \ @@ -787,6 +681,14 @@ def _extract_routing_params(self) -> RoutingParams: routed_scaling_factor=self.routing_method.routing_impl. routed_scaling_factor, ) + elif isinstance(self.routing_method, MiniMaxM3MoeRoutingMethod): + return RoutingParams( + top_k=self.routing_method.top_k, + routing_bias=self.routing_method.e_score_correction_bias, + n_group=None, + topk_group=None, + routed_scaling_factor=self.routing_method.routed_scaling_factor, + ) elif isinstance(self.routing_method, MiniMaxM2MoeRoutingMethod): return RoutingParams( top_k=self.routing_method.top_k, @@ -1087,286 +989,6 @@ def run_moe( return final_hidden_states - def forward_impl( - self, - x: Union[torch.Tensor, Fp4QuantizedTensor], - router_logits: torch.Tensor, - *, - input_ids: Optional[torch.IntTensor] = None, - do_finalize: bool = True, - all_rank_num_tokens: Optional[List[int]] = None, - use_dp_padding: Optional[bool] = None, - **kwargs, - ) -> torch.Tensor: - assert x.dtype == torch.bfloat16 - - top_k = self._extract_routing_params().top_k - - run_post_quant_allgather = (self.use_dp and self.parallel_size > 1 - and not self.enable_alltoall) - post_quant_comm = run_post_quant_allgather or self.enable_alltoall - requires_separated_routing = ( - self.routing_method.requires_separated_routing or os.environ.get( - "TLLM_TRTLLMGEN_FORCE_SEPARATED_ROUTING", "0") == "1") - - x_sf = None - token_selected_experts = None - token_final_scales = None - token_count = x.shape[0] - alltoall_info = None - # Determine if this is first/last call (TRTLLMGenFusedMoE doesn't use chunking) - is_first_call = self.repeat_idx == 0 - is_last_call = self.repeat_idx == self.repeat_count - 1 - - if post_quant_comm or requires_separated_routing: - self._load_balancer_start_wait_gpu_stage(is_first_call) - - token_selected_experts, token_final_scales = self.routing_method.apply( - router_logits, input_ids) - token_selected_experts = token_selected_experts.to(torch.int32) - if token_final_scales is not None: - token_final_scales = token_final_scales.to(torch.bfloat16) - - self._load_balancer_done_wait_gpu_stage(is_first_call) - - ignore_allreduce = self.enable_alltoall and self.alltoall_method_type in ( - AlltoallMethodType.NVLinkTwoSided, - AlltoallMethodType.NVLinkOneSided, - ) - self._load_balancer_update_statistic( - token_selected_experts, - is_first_call, - is_last_call, - ignore_allreduce=ignore_allreduce) - - # Route tokens to slots - token_selected_slots = self._load_balancer_route( - token_selected_experts, self.use_dp) - - # Update expert statistics - ExpertStatistic.set_layer(self.layer_idx) - ExpertStatistic.maybe_add_info(self.num_slots, token_selected_slots) - - # Use routed slots for subsequent processing - token_selected_experts = token_selected_slots - - if post_quant_comm: - x, x_sf = self.quantize_input(x) - - if self.enable_alltoall: - assert all_rank_num_tokens is not None, "all_rank_num_tokens required for alltoall" - - runtime_max_tokens_per_rank = max( - all_rank_num_tokens) if all_rank_num_tokens else token_count - - if token_final_scales is None: - token_final_scales = torch.ones_like(token_selected_experts, - dtype=torch.float32) - else: - token_final_scales = token_final_scales.to(torch.float32) - - if self.alltoall_method_type == AlltoallMethodType.NVLinkTwoSided: - assert self.alltoall_prepare_workspace is not None, "alltoall_prepare_workspace should be initialized" - if is_last_call: - loadbalancer_local_statistic_info = self._load_balancer_get_local_statistic_tensor( - ) - else: - loadbalancer_local_statistic_info = None - - alltoall_info, gathered_loadbalancer_local_statistic_info = MnnvlMoe.mnnvl_moe_alltoallv_prepare_without_allgather( - token_selected_experts, - loadbalancer_local_statistic_info, - self.alltoall_prepare_workspace, - runtime_max_tokens_per_rank, - self.ep_rank, - self.ep_size, - self.num_experts, - self.num_slots, - top_k, - ) - if gathered_loadbalancer_local_statistic_info is not None: - gathered_loadbalancer_local_statistic_info = gathered_loadbalancer_local_statistic_info.view( - (self.mapping.moe_ep_size, self.num_experts)) - self._load_balancer_update_statistic_with_gathered_statistic( - gathered_loadbalancer_local_statistic_info) - - if self.enable_alltoall: - if self.alltoall_method_type == AlltoallMethodType.NVLinkTwoSided: - x, x_sf, token_selected_experts, token_final_scales = MnnvlMoe.mnnvl_moe_alltoallv( - [x, x_sf, token_selected_experts, token_final_scales], - alltoall_info, - self.alltoall_workspace, - self.ep_rank, - self.ep_size, - ) - - torch.ops.trtllm.memset_expert_ids( - token_selected_experts, - alltoall_info.recv_rank_count_cumsum, - runtime_max_tokens_per_rank, - top_k, - -1, # Caution: TRTLLM-Gen uses -1 as invalid token expert id - self.ep_size, - ) - - if token_final_scales is not None: - token_final_scales = token_final_scales.to(torch.bfloat16) - elif self.alltoall_method_type == AlltoallMethodType.NVLinkOneSided: - payloads = [] - payloads.append(x) - if x_sf is not None: - payloads.append(x_sf) - expert_id_payload_index = 2 - else: - expert_id_payload_index = 1 - payloads.append(token_selected_experts) - payloads.append(token_final_scales) - - loadbalancer_local_statistic_info = None - if self.layer_load_balancer and is_last_call: - loadbalancer_local_statistic_info = self._load_balancer_get_local_statistic_tensor( - ) - if loadbalancer_local_statistic_info is not None: - recv_tensors = self.moe_a2a.dispatch( - token_selected_experts, - payloads, - runtime_max_tokens_per_rank, - invalid_token_expert_id= - -1, # Caution: TRTLLM-Gen uses -1 as invalid token expert id - expert_id_payload_index=expert_id_payload_index, - eplb_local_stats=loadbalancer_local_statistic_info, - ) - gathered_stats = self.moe_a2a._state.eplb_gathered_stats - self._load_balancer_update_statistic_with_gathered_statistic( - gathered_stats) - else: - recv_tensors = self.moe_a2a.dispatch( - token_selected_experts, - payloads, - runtime_max_tokens_per_rank, - invalid_token_expert_id= - -1, # Caution: TRTLLM-Gen uses -1 as invalid token expert id - expert_id_payload_index=expert_id_payload_index, - ) - - if x_sf is not None: - x_recv, x_sf_recv, token_selected_experts_recv, token_final_scales_recv = recv_tensors - x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1]) - else: - x_recv, token_selected_experts_recv, token_final_scales_recv = recv_tensors - x = x_recv.view(-1, x_recv.shape[-1]) - token_selected_experts = token_selected_experts_recv.view( - -1, token_selected_experts_recv.shape[-1]) - token_final_scales = token_final_scales_recv.view( - -1, token_final_scales_recv.shape[-1]) - - if token_final_scales is not None: - token_final_scales = token_final_scales.to(torch.bfloat16) - else: - raise ValueError( - f"Unsupported moe alltoall method type: {self.alltoall_method_type}" - ) - - elif run_post_quant_allgather: - if x_sf is not None: - assert len( - x_sf.shape - ) == 2, "The hidden states scaling factor should be 2D tensor before allgather" - x, x_sf, token_selected_experts, token_final_scales = allgather( - [x, x_sf, token_selected_experts, token_final_scales], - self.mapping, - dim=0, - sizes=None if use_dp_padding else all_rank_num_tokens) - else: - # No communication path: use non-post-quant-comm quantization - x, x_sf = self.quantize_input(x, post_quant_comm=False) - - moe_output: Optional[torch.Tensor] = None - use_workspace_output = False - if do_finalize and self.alltoall_method_type == AlltoallMethodType.NVLinkOneSided and self.supports_moe_output_in_alltoall_workspace( - ): - moe_output = self.moe_a2a.get_combine_payload_tensor_in_workspace( - runtime_max_tokens_per_rank, self.hidden_size, torch.bfloat16) - use_workspace_output = True - - # Call the extracted run_moe interface - # Determine router_logits based on post_quant_comm - router_logits_arg = None if ( - post_quant_comm or requires_separated_routing) else router_logits - - final_hidden_states = self.run_moe( - x=x, - token_selected_experts=token_selected_experts, - token_final_scales=token_final_scales, - x_sf=x_sf, - # TRTLLMGenFusedMoE extra parameters - router_logits=router_logits_arg, - do_finalize=do_finalize, - moe_output=moe_output, - ) - - self._load_balancer_start_set_cpu_stage(is_last_call) - - # Combine results if using alltoall - if self.enable_alltoall: - if self.alltoall_method_type == AlltoallMethodType.NVLinkTwoSided: - if alltoall_info is not None: - final_hidden_states = MnnvlMoe.mnnvl_moe_alltoallv_combine( - final_hidden_states, - alltoall_info, - self.alltoall_workspace, - ep_rank=self.ep_rank, - ep_size=self.ep_size, - top_k=top_k, - use_low_precision_combine=self. - use_low_precision_combine, - token_count=token_count, - ) - elif self.alltoall_method_type == AlltoallMethodType.NVLinkOneSided: - # If use_workspace_output=True, the MoE result is already in workspace - # Otherwise, we need to reshape and pass it - if use_workspace_output: - # Workspace payload is returned as 2D [ep_size * max_tokens, hidden]; reshape to 3D. - hidden = final_hidden_states.shape[-1] - payload = moe_output.view(self.ep_size, - runtime_max_tokens_per_rank, - hidden) - final_hidden_states = self.moe_a2a.combine( - payload, - runtime_max_tokens_per_rank, - payload_in_workspace=True) - else: - hidden = final_hidden_states.shape[-1] - payload = final_hidden_states.view( - self.ep_size, runtime_max_tokens_per_rank, hidden) - final_hidden_states = self.moe_a2a.combine( - payload, - runtime_max_tokens_per_rank, - payload_in_workspace=False) - else: - raise ValueError( - f"Unsupported moe alltoall method type: {self.alltoall_method_type}" - ) - - final_hidden_states = self.reducescatter_or_allreduce( - final_hidden_states, - all_rank_num_tokens=all_rank_num_tokens, - use_dp_padding=use_dp_padding, - ) - - self._load_balancer_done_set_cpu_stage(is_last_call) - - if use_dp_padding: - rank = self.parallel_rank - final_hidden_states = final_hidden_states[: - all_rank_num_tokens[rank]] - - # Update repeat index for load balancer - if self.layer_load_balancer: - self.repeat_idx = 0 if self.repeat_idx == self.repeat_count - 1 else self.repeat_idx + 1 - - return final_hidden_states - def forward_fake( self, x: Union[torch.Tensor, Fp4QuantizedTensor, MxFp8QuantizedTensor], diff --git a/tensorrt_llm/_torch/modules/fused_moe/interface.py b/tensorrt_llm/_torch/modules/fused_moe/interface.py index 2de479eda79b..53e34d374c86 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/interface.py +++ b/tensorrt_llm/_torch/modules/fused_moe/interface.py @@ -103,6 +103,8 @@ class AlltoallMethodType(IntEnum): DeepEP = 3 # DeepEP low latency: CUDA Graphs are supported, IBGDA is required DeepEPLowLatency = 4 + # NCCL EP: Low-latency expert parallelism via NCCL EP library + NcclEP = 5 class MoESchedulerKind(Enum): diff --git a/tensorrt_llm/_torch/modules/fused_moe/mega_moe/mega_moe_cute_dsl.py b/tensorrt_llm/_torch/modules/fused_moe/mega_moe/mega_moe_cute_dsl.py index 2d57eb24354a..00c67ecc4ea6 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/mega_moe/mega_moe_cute_dsl.py +++ b/tensorrt_llm/_torch/modules/fused_moe/mega_moe/mega_moe_cute_dsl.py @@ -31,8 +31,8 @@ * BF16 -> NVFP4 activation quantization (``quantize_input``) * ``run_moe`` boundary: stage activation + topk into the kernel ABI, build the ``MegaMoECuteDslWeightView`` from the quant method, call - ``torch.ops.trtllm.cute_dsl_megamoe_nvfp4_blackwell``, sum the - per-topk axis (form A), return ``(T, hidden)`` output. + ``torch.ops.trtllm.cute_dsl_megamoe_nvfp4_blackwell``, return the + ``(T, hidden)`` output (the kernel collapses the top-k axis). ``run_moe`` is a single unified path for both topologies. Only the SOURCE of the kernel's input/output buffers branches on ``ep_size``: @@ -49,7 +49,7 @@ ``peer_rank_ptr_mapper.map``. ``_acquire_buffers`` is the only branch point; staging, kernel launch, -and the host-side top-k reduction are identical across topologies. +and output finalization are identical across topologies. Remaining hard gate: @@ -68,6 +68,9 @@ from __future__ import annotations +import os +import socket +import weakref from dataclasses import dataclass from typing import Dict, List, Optional, Tuple, Union @@ -76,8 +79,11 @@ from tensorrt_llm._utils import get_sm_version, is_sm_100f from tensorrt_llm.logger import logger +from tensorrt_llm.math_utils import ceil_div from tensorrt_llm.models.modeling_utils import QuantAlgo +from ....autotuner import AutoTuner + # ``megamoe_activation_sf_bytes_per_row`` lives at module top of the # custom-op file (NOT inside its ``IS_MEGAMOE_OP_AVAILABLE`` gate), so # it is always importable. The provider / shared-workspace helpers used @@ -329,6 +335,10 @@ class MegaMoECuteDsl(MoE): _SUPPORTED_ACTIVATION_DTYPES = frozenset({torch.bfloat16}) + # Legal combine wire formats; must stay in sync with + # ``CombineFormat.parse`` in the kernel package's token_comm.py. + _SUPPORTED_COMBINE_FORMATS = frozenset({"bf16", "32e4m3xe8m0", "16e2m1xbf16"}) + # Kernel owns dispatch + GEMM1 + SwiGLU + GEMM2 + combine via the # CuteDSL three-stage dispatch primitives + NVLink barrier; the # scheduler must skip host-side comm and lockstep every chunk. @@ -423,7 +433,6 @@ def __init__( apply_router_weight_on_input: bool = False, layer_idx: Optional[int] = None, init_load_balancer: bool = True, - without_comm: bool = False, activation_type: ActivationType = ActivationType.Swiglu, swiglu_limit: Optional[torch.Tensor] = None, **kwargs, @@ -486,29 +495,28 @@ def __init__( # topk-score application point. v2 default is the deepgemm graph # (apply_topk_in_fc1=True): the fused kernel folds the topk score into - # the SwiGLU output before the fc1-out NVFP4 quant and the host reduces - # combine_output.sum(dim=1). Kept as an internal backend constant until - # the transformers route is GPU-validated and promoted to MoeConfig. + # the SwiGLU output before the fc1-out NVFP4 quant. Kept as an internal + # backend constant until the transformers route is GPU-validated and + # promoted to MoeConfig. self.apply_topk_in_fc1 = True - # Cross-rank combine path. ``False`` (default): the FC2 epilogue writes - # ``combine_output`` directly (scattered symmetric writes back to the - # source rank). ``True``: the kernel stages FC2 output in a local - # ``fc2_output_workspace`` and a fused in-kernel NVLink ``token_back_by_push`` - # bulk-returns it to the source rank's ``combine_output`` -- faster for - # multi-rank EP at the cost of the extra (local) fc2_output_workspace + - # fc2_done_counter budget (auto-sized by ``get_workspace_sizes``). The - # ``combine_output`` shape / host ``.sum(dim=1)`` reduce are unchanged - # (those depend on ``in_kernel_fc2_reduce``, not this knob). Internal - # backend constant for now; flip to opt into the fused-combine path. - self.token_back_by_dispatch = False - - # FC2 output store path (codegen-time). ``True`` (default): non-bulk - # TMA store (upstream default). ``False``: bulk store path. Kept as an - # internal backend attribute so different shapes/cases can pick the - # cheaper store; it changes the generated kernel, so it is part of the - # runner ``unique_id`` / compile-cache key (never a per-call runtime kwarg). - self.non_ubulk_fc2_store = True + # Cross-rank combine wire format is selected with + # MEGAMOE_COMBINE_FORMAT (default bf16). It MUST be rank-identical: + # it sizes the symmetric workspace and picks the compiled kernel, so + # divergence desyncs the rendezvous / NVLink barrier. + combine_format = os.environ.get("MEGAMOE_COMBINE_FORMAT", "bf16") + if combine_format not in self._SUPPORTED_COMBINE_FORMATS: + raise ValueError( + f"MEGAMOE_COMBINE_FORMAT must be one of " + f"{sorted(self._SUPPORTED_COMBINE_FORMATS)}; got {combine_format!r}." + ) + self.combine_format = combine_format + + # AutoTuner tactic-sweep opt-in via MEGAMOE_TACTIC_AUTOTUNE=1 + # (bench-only knob; default OFF so serving warmup never pays the + # MERGE lockstep sweep). Must be set identically on ALL ranks (it + # gates collective tuning behavior). + self.tactic_autotune = os.environ.get("MEGAMOE_TACTIC_AUTOTUNE", "0") == "1" # SwiGLU clamp: map the model-provided per-layer ``swiglu_limit`` tensor # to the kernel's codegen-time scalar ``gate_up_clamp``. The MegaMoE @@ -526,6 +534,24 @@ def __init__( or 4096 ) + # Adaptive ``max_tokens_per_rank`` bucketing: one symmetric provider + + # kernel per ladder bucket; per launch pick the smallest bucket that + # fits the lockstep chunk max (see ``_select_launch_max_tokens``). The + # ladder cap is min(moe_max_num_tokens, engine max_num_tokens) -- both + # rank-identical, keeping the ladder rank-identical (the + # NVLink-barrier invariant). + per_rank_cap = self.max_num_tokens + _engine_max = int(getattr(model_config, "max_num_tokens", 0) or 0) + if _engine_max > 0: + per_rank_cap = min(per_rank_cap, _engine_max) + self._maxt_buckets = self._resolve_maxt_buckets(per_rank_cap) + # Per-bucket symmetric providers (bucket -> MegaMoeSymmMemProvider); + # filled in ``create_weights`` on the multi-rank EP path. + self._symm_providers = {} + # Lockstep per-chunk cross-rank max for the NEXT run_moe (set via + # ``set_adaptive_launch_tokens``); ``None`` -> full bucket. + self._active_launch_max_tokens: Optional[int] = None + # Resolve EP ProcessGroup at construction. Resolving at forward # time would be collective on a non-synchronous call stack and # deadlock under PP / layer-skip. Construction is globally @@ -540,24 +566,37 @@ def __init__( f"to single-rank degenerate mode at run_moe time." ) self._ep_pg = None + # Pure-DEP invariant checked at (rank-synchronous) construction: the + # op's barrier-reset fence runs on the WORLD group, so WORLD != EP + # would build fine and then fail or hang mid-serving. ValueError on + # purpose -- the ``except RuntimeError`` above is the single-rank + # fallback and must not swallow this. + if dist.is_available() and dist.is_initialized() and dist.get_world_size() != self.ep_size: + raise ValueError( + f"MegaMoECuteDsl requires pure DEP (torch.distributed WORLD " + f"== EP): the forward-time barrier-reset fence runs on the " + f"WORLD process group. Got WORLD={dist.get_world_size()}, " + f"EP={self.ep_size}." + ) - # Weight tensors are owned by the quant method. ``_symm_provider`` - # is the symmetric-memory provider for multi-rank EP execution; - # allocated build-time in ``create_weights`` (collective - # rendezvous), shared across MoE layers via the module-scope - # cache in ``cute_dsl_megamoe_custom_op.py``. ``None`` for the - # single-rank degenerate path. - self._symm_provider = None + # WEAK ref to the AutoTuner profiling scratch provider: the custom + # op's process-global cache is the SOLE strong owner, so + # ``release_megamoe_profiling_scratch()`` reclaims it and this ref + # goes dead (no per-module teardown). Materialized lazily by the + # deferred factory in ``_launch_megamoe_kernel``. + self._profiling_scratch_provider_ref = None + self._profiling_scratch_kwargs = None self._weights_loaded = False self._weights_created = False - self._post_load_done = False self.quant_method = None # Per-instance staging cache (key -> {tensor name: tensor}) and # last-staged-T tracker; together they implement the always-pad- # to-max_T launch contract by refreshing only the rows that - # changed between calls. + # changed between calls. ``_last_staged_T`` is keyed by bucket max_T: + # each bucket's staging buffer has its own row-reset history (a shared + # scalar would reset the wrong buffer when buckets alternate). self._local_staging_cache: Dict[Tuple, Dict[str, torch.Tensor]] = {} - self._last_staged_T: Optional[int] = None + self._last_staged_T: Dict[int, int] = {} if not model_config.skip_create_weights_in_init: self.create_weights() @@ -595,6 +634,62 @@ def _resolve_gate_up_clamp( ) return float(first.item()) + @staticmethod + def _resolve_maxt_buckets(max_num_tokens: int) -> List[int]: + """Resolve the adaptive ``max_tokens_per_rank`` bucket ladder. + + MUST be a pure function of ``max_num_tokens`` so every EP rank derives + the identical ladder (divergence breaks the cross-rank NVLink barrier): + {256, 1024, 4096} rungs below the per-rank cap plus the cap itself. + Kept small -- each rung multiplies the compile/capture/memory surface. + """ + full = int(max_num_tokens) + if full <= 0: + full = 4096 + cand = {b for b in (256, 1024, 4096) if b < full} + cand.add(full) + return sorted(cand) + + def _select_launch_max_tokens(self, chunk_max_tokens: Optional[int]) -> int: + """Pick the smallest ladder bucket that fits ``chunk_max_tokens``. + + ``chunk_max_tokens`` is the lockstep ``max(all_rank_num_tokens)`` for + the chunk (rank-identical) or ``None`` (-> full bucket); input and + ladder rank-identical => bucket rank-identical, so all ranks launch + the same compiled kernel against same-sized symmetric buffers. + + Tuning mode with the ``tactic_autotune`` opt-in ALWAYS takes the top + bucket (profiling scratch and tuned-cache keys are sized/keyed for + it); both gates are rank-identical so the bucket stays lockstep. + """ + buckets = self._maxt_buckets + if self.tactic_autotune and AutoTuner.get().is_tuning_mode: + return buckets[-1] + if chunk_max_tokens is None or chunk_max_tokens <= 0: + return buckets[-1] + for b in buckets: + if chunk_max_tokens <= b: + return b + # RAISE (rank-identical input -> lockstep failure). Clamping instead + # would raise only on ranks whose local count exceeds the clamp, + # desyncing the NVLink barrier and hanging the survivors. + raise RuntimeError( + f"MegaMoE-CuteDSL: chunk max_tokens_per_rank={chunk_max_tokens} exceeds " + f"the top adaptive bucket {buckets[-1]} (per-rank cap = " + f"min(moe_max_num_tokens, max_num_tokens)). Raise moe_max_num_tokens or " + f"reduce the per-rank chunk." + ) + + def set_adaptive_launch_tokens(self, chunk_max_tokens: Optional[int]) -> None: + """Scheduler hook: record the lockstep per-chunk cross-rank max token + count for the next ``run_moe``. + + MUST be ``max(all_rank_num_tokens)`` for the chunk (rank-identical) so + all ranks select the same bucket; ``None`` -> full bucket. Consumed + and reset by the next ``_run_moe``. + """ + self._active_launch_max_tokens = chunk_max_tokens + def _supports_load_balancer(self) -> bool: # Both static and dynamic EPLB are supported: the four MegaMoE- # format derived parameters (``mega_fc{1,2}_weight{,_sf}``) are @@ -663,12 +758,80 @@ def validate_configurable_moe(self, moe) -> None: # ------------------------------------------------------------------ # EP process-group resolution (no collective at forward time) # ------------------------------------------------------------------ + def _maybe_init_torch_dist_under_mpi(self): + """Bootstrap a torch.distributed NCCL WORLD group under the MPI orchestrator. + + The MPI orchestrator (trtllm-bench / mpirun) never calls + ``init_process_group``, but the symmetric-memory rendezvous needs a + ProcessGroup; for pure DEP a node-local NCCL WORLD suffices. No-op + under Ray or single-rank. + """ + if not dist.is_available() or dist.is_initialized(): + return + from tensorrt_llm._utils import mpi_comm, mpi_rank, mpi_world_size + + try: + world = mpi_world_size() + rank = mpi_rank() + except Exception as e: # not under MPI either -> leave uninitialized + logger.debug( + f"[MegaMoECuteDsl] MPI rank query failed ({e!r}); " + "skipping torch.distributed bootstrap." + ) + return + if world <= 1: + return + # Check pure DEP BEFORE any collective: under PP/CP hybrids some ranks + # never construct a MegaMoE module, so the bcast/init below would hang + # instead of failing loud. Rank-identical inputs -> lockstep raise. + if world != self.ep_size: + raise ValueError( + f"MegaMoECuteDsl requires pure DEP (MPI WORLD == EP) to " + f"bootstrap torch.distributed: got MPI world={world}, " + f"EP={self.ep_size}." + ) + + # Rank 0 draws a free port (a fixed one collides across same-node + # jobs). The bcast is UNCONDITIONAL and rank 0 alone decides + # (honoring ITS pre-set MASTER_* env): gating the collective on + # per-rank env presence would desync the communicator when the vars + # are exported on only a subset of ranks. No bind-race retry: a sound + # one needs cross-rank failure agreement, and the close()->bind() + # window is negligible next to the fixed-port collision this replaces. + def _pick_rendezvous(): + addr = os.environ.get("MASTER_ADDR") + port = os.environ.get("MASTER_PORT") + if addr and port: + return (addr, port) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("", 0)) + port = sock.getsockname()[1] + try: + addr = socket.gethostbyname(socket.gethostname()) + except OSError: + # Unresolvable hostname (minimal containers / broken DNS): + # this bootstrap is documented single-node, loopback works. + addr = "127.0.0.1" + return (addr, str(port)) + + host, port = mpi_comm().bcast(_pick_rendezvous() if rank == 0 else None, root=0) + os.environ["MASTER_ADDR"] = str(host) + os.environ["MASTER_PORT"] = str(port) + logger.info( + f"[MegaMoECuteDsl] torch.distributed not initialized under MPI; " + f"bootstrapping NCCL WORLD group (rank={rank}/{world}, " + f"{os.environ['MASTER_ADDR']}:{os.environ['MASTER_PORT']}) for the " + f"EP rendezvous." + ) + dist.init_process_group(backend="cuda:nccl,cpu:gloo", rank=rank, world_size=world) + def _resolve_ep_pg(self): """Return the torch.distributed ProcessGroup for the EP sub-world. Mirrors :meth:`MegaMoEDeepGemm._resolve_ep_pg` so the two MegaMoE backends share the same fallback chain. """ + self._maybe_init_torch_dist_under_mpi() if not dist.is_available() or not dist.is_initialized(): raise RuntimeError( "MegaMoECuteDsl requires torch.distributed to be initialized " @@ -730,22 +893,30 @@ def create_weights(self): return # Step 1: build-time symmetric memory allocation (multi-rank only). # Single-rank degenerate uses local CUDA tensors and skips here. - self._symm_provider = None if self.ep_size > 1: - self._symm_provider = self._alloc_symm_provider() + self._alloc_symm_provider() # Step 2-3: quant method registers all NVFP4 + MegaMoE-format params. self.quant_method = self._get_quant_method() self.quant_method.create_weights(self) # Step 4. self._weights_created = True + @property + def _symm_provider(self): + """Full-bucket symmetric provider, or ``None`` (pre-``create_weights`` + or single-rank) -- callers use this as the provider-available guard. + """ + return self._symm_providers.get(self._maxt_buckets[-1]) + def _alloc_symm_provider(self): """Build-time symmetric provider allocation. See ``create_weights``. - Returns a :class:`MegaMoeSymmMemProvider` from the module-scope - cache. Raises :class:`MegaMoeCuteDslUnavailable` with an - actionable message when no ProcessGroup is available -- that - would block the rendezvous and is a hard error for multi-rank. + One provider per adaptive bucket. Each ``get_megamoe_symm_provider`` + call is a COLLECTIVE rendezvous, so the loop must iterate identical + buckets in identical order on every EP rank (guaranteed: the ladder is + a pure function of rank-identical inputs); the module-scope cache + makes only the first layer pay. Raises MegaMoeCuteDslUnavailable when + no ProcessGroup is available (hard error for multi-rank). """ from ....custom_ops.cute_dsl_megamoe_custom_op import ( get_megamoe_symm_provider, @@ -759,26 +930,58 @@ def _alloc_symm_provider(self): "or initialize torch.distributed before model build." ) top_k = self.routing_method.experts_per_token - shared_workspace_bytes = query_megamoe_shared_workspace_bytes( - world_size=self.ep_size, - local_rank=self.ep_rank, - num_topk=top_k, - num_experts_per_rank=int(self.expert_size_per_partition), - hidden_size=self.hidden_size, - intermediate_size_per_partition=int(self.intermediate_size_per_partition), - expand_intermediate_size_per_partition=int(self.expand_intermediate_size_per_partition), - max_tokens_per_rank=int(self.max_num_tokens), - ) - return get_megamoe_symm_provider( - process_group=self._ep_pg, - world_size=self.ep_size, - rank=self.ep_rank, - hidden_size=self.hidden_size, - max_tokens_per_rank=int(self.max_num_tokens), - num_topk=top_k, - output_dtype=self.dtype or torch.bfloat16, - shared_workspace_bytes=shared_workspace_bytes, - ) + full_T = self._maxt_buckets[-1] + self._profiling_scratch_provider_ref = None + for max_T in self._maxt_buckets: + shared_workspace_bytes = query_megamoe_shared_workspace_bytes( + world_size=self.ep_size, + num_topk=top_k, + num_experts_per_rank=int(self.expert_size_per_partition), + hidden_size=self.hidden_size, + intermediate_size_per_partition=int(self.intermediate_size_per_partition), + expand_intermediate_size_per_partition=int( + self.expand_intermediate_size_per_partition + ), + max_tokens_per_rank=max_T, + in_kernel_fc2_reduce=False, + combine_format=self.combine_format, + ) + provider = get_megamoe_symm_provider( + process_group=self._ep_pg, + world_size=self.ep_size, + rank=self.ep_rank, + hidden_size=self.hidden_size, + max_tokens_per_rank=max_T, + num_topk=top_k, + output_dtype=self.dtype or torch.bfloat16, + combine_format=self.combine_format, + shared_workspace_bytes=shared_workspace_bytes, + ) + self._symm_providers[max_T] = provider + # AutoTuner profiling scratch: record only the request kwargs + # (full bucket only, matching tuning mode's forced top bucket); + # the multi-GiB allocation itself is deferred to the op's + # profiling pre-hook via the factory in ``_launch_megamoe_kernel``, + # so a process that never tunes never pays it. + if max_T == full_T and self.tactic_autotune: + self._profiling_scratch_kwargs = dict( + process_group=self._ep_pg, + world_size=self.ep_size, + rank=self.ep_rank, + hidden_size=self.hidden_size, + max_tokens_per_rank=max_T, + num_topk=top_k, + output_dtype=self.dtype or torch.bfloat16, + combine_format=self.combine_format, + shared_workspace_bytes=shared_workspace_bytes, + ) + if len(self._maxt_buckets) > 1 and self.ep_rank == 0: + logger.info( + "[MegaMoECuteDsl] adaptive max_tokens_per_rank buckets=%s " + "(one symmetric provider + kernel per bucket; per-launch bucket " + "= smallest >= max(all_rank_num_tokens))", + self._maxt_buckets, + ) def load_weights(self, weights: List[Dict], allow_partial_loading: bool = False) -> None: if self.quant_method is None: @@ -804,14 +1007,6 @@ def load_weights(self, weights: List[Dict], allow_partial_loading: bool = False) self.quant_method.load_weights( self, weights, self.weight_loading_mode, allow_partial_loading=allow_partial_loading ) - # Eager loading path: ``FusedMoEMethodBase.load_weights`` already - # ran ``quant_method.process_weights_after_loading(self)`` at its - # tail. Mark the sentinel so a subsequent - # ``backend.process_weights_after_loading()`` becomes a no-op - # instead of re-stacking ``mega_fc*_weight*`` from - # already-finalised parent buffers. - if not allow_partial_loading: - self._post_load_done = True def post_load_weights(self) -> None: if self.quant_method is None: @@ -819,45 +1014,24 @@ def post_load_weights(self) -> None: self.transform_weights() self.cache_derived_state() - def process_weights_after_loading(self) -> None: - """Run quant-method weight transforms; idempotent across calls. - - The real MegaMoE-format build (``[w3|w1]`` cat, 16-atom gate/up - interleave, ``to_blocked`` swizzle, and ``fc1_norm_const`` setup) lives in - :meth:`NVFP4MegaMoECuteDslMethod.process_weights_after_loading`. - This hook must dispatch to that method directly so two paths - both reach it: - - * Eager loading (``allow_partial_loading=False``) -- fired by - ``FusedMoEMethodBase.load_weights`` itself. - * Partial loading (RLHF reload, etc.) -- ``load_weights`` - skips its tail call, so the caller invokes this hook on - ``ConfigurableMoE`` -> backend to finalise. - - ``_post_load_done`` keeps the call idempotent: a second - invocation after eager finalisation must not re-run the - transforms (``_build_mega_format_weights`` would re-stack - ``mega_fc*_weight*`` from already-finalised parent buffers). - """ - if getattr(self, "_post_load_done", False): - return + # Staged-hook guards: ConfigurableMoE delegates these straight to the + # backend, and the base MoE implementations dereference + # ``self.quant_method`` unguarded. + def transform_weights(self) -> None: if self.quant_method is None: self.create_weights() - self.quant_method.process_weights_after_loading(self) - self._post_load_done = True + super().transform_weights() - def pre_reload_weights(self) -> None: - """Reset cached state before a hot weight reload. + def cache_derived_state(self) -> None: + if self.quant_method is None: + self.create_weights() + super().cache_derived_state() - ``_post_load_done`` is cleared so the next ``process_weights_after_loading`` - re-runs the MegaMoE-format weight transforms over the new - checkpoint bytes. The symmetric-memory provider is forward-time - scratch that does not need to be re-rendezvoused on weight - reload; we keep it as-is to avoid an unnecessary collective. - """ - self._post_load_done = False - if self.quant_method is not None and hasattr(self.quant_method, "pre_reload_weights"): - self.quant_method.pre_reload_weights(self) + def pre_reload_weights(self) -> None: + raise NotImplementedError( + "MegaMoE-CuteDSL does not support hot weight reloading; its " + "source weights are released after initial packing." + ) def _build_weight_view(self) -> MegaMoECuteDslWeightView: """Bundle the MegaMoE-format weight tensors registered by the @@ -923,10 +1097,11 @@ def quantize_input( ) # ``fp4_quantize(is_sf_swizzled=False)`` returns LINEAR layout # ``(rows, ceil(hidden/16))`` with no column pad. The kernel TMA - # load needs ``round_up(ceil(hidden/16), 4)`` bytes per row, so + # load needs ``pad_up(ceil_div(hidden, 16), 4)`` bytes per row + # (== ``megamoe_activation_sf_bytes_per_row``), so # 32-aligned-but-not-64-aligned hidden sizes (1568, 1632, 2080) # come back 2 bytes short; pad the tail before returning. - raw_cols = (hidden + 15) // 16 + raw_cols = ceil_div(hidden, 16) x_sf_raw = x_sf.view(x_bf16.shape[0], raw_cols) if sf_cols == raw_cols: return x_fp4, x_sf_raw @@ -949,8 +1124,8 @@ def run_moe( Casts ``token_selected_experts`` to ``int64`` (the scheduler keeps ``int32`` for the EPLB stats kernel; the MegaMoE kernel reads ``topk_idx`` as Int64) and delegates the staging + kernel launch - to :meth:`_run_moe`. The host then sums the form-A - ``(T, top_k, hidden)`` combine output along the top-k axis. + to :meth:`_run_moe`, which returns the reduced ``(T, hidden)`` + output. """ del unused_kwargs if output_dtype is None: @@ -1000,7 +1175,7 @@ def run_moe( output_dtype=output_dtype, ) - def _ensure_local_staging(self, *, top_k: int, hidden: int, device, output_dtype): + def _ensure_local_staging(self, *, top_k: int, hidden: int, device, output_dtype, max_T: int): """Allocate (and cache) the per-instance local staging tensors. Always allocates ``topk_idx`` (the kernel reads it as a local-only @@ -1011,16 +1186,17 @@ def _ensure_local_staging(self, *, top_k: int, hidden: int, device, output_dtype for ``ep_size == 1``; multi-rank pulls them from the symmetric provider's regions instead. - All staging tensors are sized to ``max_num_tokens`` along dim 0 - so the kernel's constexpr ``num_tokens`` matches the buffer-time - ``max_tokens_per_rank``. Diverging the two would make + All staging tensors are sized to ``max_T`` (the per-launch adaptive + bucket) along dim 0 so the kernel's constexpr ``num_tokens`` matches the + buffer-time ``max_tokens_per_rank``. Diverging the two would make ``_dispatch_prep`` round 3 (``MAX_SLOT_C = num_tokens * num_topk`` in dispatch_kernel.py) write per-(expert, rank) advertise cards at the wrong stride relative to the symm allocation (``max_tokens_per_rank * num_topk`` in megamoe_kernel.py), - silently corrupting multi-rank metadata. + silently corrupting multi-rank metadata. The cache is keyed by + ``max_T`` so each bucket owns a distinct staging set. """ - max_T = int(self.max_num_tokens) + max_T = int(max_T) cache_key = (max_T, top_k, hidden, str(device), output_dtype) cached = self._local_staging_cache if cache_key in cached: @@ -1045,8 +1221,11 @@ def _ensure_local_staging(self, *, top_k: int, hidden: int, device, output_dtype staging["activation_sf"] = torch.empty( (max_T, sf_bytes_per_row), dtype=torch.uint8, device=device ) + # Always 1: the kernel collapses the top-k axis in-op (TopkReduce + # form-A / REDG form-B); the op aliases this as the 2D output. + combine_k = 1 staging["combine_output"] = torch.empty( - (max_T, top_k, hidden), + (max_T, combine_k, hidden), dtype=torch.bfloat16, device=device, ) @@ -1058,7 +1237,6 @@ def _ensure_local_staging(self, *, top_k: int, hidden: int, device, output_dtype shared_bytes = query_megamoe_shared_workspace_bytes( world_size=1, - local_rank=0, num_topk=top_k, num_experts_per_rank=int(self.expert_size_per_partition), hidden_size=hidden, @@ -1067,24 +1245,34 @@ def _ensure_local_staging(self, *, top_k: int, hidden: int, device, output_dtype self.expand_intermediate_size_per_partition ), max_tokens_per_rank=max_T, + in_kernel_fc2_reduce=False, + combine_format=self.combine_format, ) - staging["shared_workspace"] = torch.empty( + # zeros (not empty): the leading counter prefix is an atomic-add + # target the kernel only tail-resets for the NEXT launch, so the + # FIRST launch needs it pre-zeroed (one-time; cached per bucket). + staging["shared_workspace"] = torch.zeros( shared_bytes, dtype=torch.uint8, device=device ) cached[cache_key] = staging return staging - def _acquire_buffers(self, *, top_k: int, hidden: int, device, output_dtype) -> _MegaMoeBuffers: - """Resolve the kernel's input/output buffers. + def _acquire_buffers( + self, *, top_k: int, hidden: int, device, output_dtype, launch_max_T: int + ) -> _MegaMoeBuffers: + """Resolve the kernel's input/output buffers for the ``launch_max_T`` + adaptive bucket. This is the ONLY structural branch between single-rank and multi- rank execution; the source of activation / activation_sf / topk_weights / combine_output / shared_workspace differs per the :class:`_MegaMoeBuffers` contract. ``topk_idx_local`` always lives - in plain CUDA memory. + in plain CUDA memory. ``launch_max_T`` selects both the local staging + set and (multi-rank) the symmetric provider, which MUST share the same + ``max_T`` as the kernel's compile-time ``max_tokens_per_rank``. """ staging = self._ensure_local_staging( - top_k=top_k, hidden=hidden, device=device, output_dtype=output_dtype + top_k=top_k, hidden=hidden, device=device, output_dtype=output_dtype, max_T=launch_max_T ) if self.ep_size == 1: return _MegaMoeBuffers( @@ -1111,13 +1299,16 @@ def _acquire_buffers(self, *, top_k: int, hidden: int, device, output_dtype) -> f"model_config.skip_create_weights_in_init was not set " f"without a follow-up create_weights() call." ) - if self._symm_provider.num_topk != top_k: + # ``launch_max_T`` comes from ``_select_launch_max_tokens``, so it is + # always a ladder member. + provider = self._symm_providers[launch_max_T] + if provider.num_topk != top_k: raise MegaMoeCuteDslUnavailable( f"MegaMoECuteDsl symm provider was built for top_k=" - f"{self._symm_provider.num_topk} but run_moe called with " + f"{provider.num_topk} but run_moe called with " f"top_k={top_k}; recreate the backend." ) - regions = self._symm_provider.get_regions() + regions = provider.get_regions() return _MegaMoeBuffers( activation=regions.activation, activation_sf=regions.activation_sf, @@ -1163,7 +1354,7 @@ def _stage_inputs( it. """ max_T = bufs.topk_idx_local.shape[0] - last_T = getattr(self, "_last_staged_T", None) + last_T = self._last_staged_T.get(max_T) if last_T is not None and last_T > num_tokens: bufs.topk_idx_local[num_tokens:last_T].fill_(-1) if num_tokens > 0: @@ -1173,7 +1364,7 @@ def _stage_inputs( bufs.topk_weights[:num_tokens, :top_k].copy_(topk_weights, non_blocking=True) if num_tokens < max_T: bufs.topk_weights[num_tokens:max_T, :top_k].zero_() - self._last_staged_T = num_tokens + self._last_staged_T[max_T] = num_tokens def _launch_megamoe_kernel( self, @@ -1192,53 +1383,104 @@ def _launch_megamoe_kernel( peer_offsets: List[int], num_tokens: int, output_dtype: torch.dtype, + launch_max_T: int, ) -> torch.Tensor: - """Launch the fused MegaMoE CuteDSL kernel and reduce form-A output. + """Launch the fused MegaMoE CuteDSL kernel and return its reduced output. Single-rank and multi-rank reach this point with identical kernel inputs; only the source of the staged buffers differs (decided - upstream by :meth:`_acquire_buffers`). The host-side top-k reduction - is the same across topologies. NVFP4 / FP8-SF dtype views happen - through the module-level :func:`_as_nvfp4` / :func:`_as_fp8_sf` - helpers (the kernel rejects raw uint8 byte tensors). + upstream by :meth:`_acquire_buffers`). NVFP4 / FP8-SF dtype views + happen through the module-level :func:`_as_nvfp4` / + :func:`_as_fp8_sf` helpers (the kernel rejects raw uint8 byte + tensors). """ - torch.ops.trtllm.cute_dsl_megamoe_nvfp4_blackwell( - activation=_as_nvfp4(activation), - activation_sf=_as_fp8_sf(activation_sf), - topk_idx=topk_idx, - topk_weights=topk_weights, - fc1_weight=_as_nvfp4(weight_view.fc1_weight), - fc1_weight_sf=_as_fp8_sf(weight_view.fc1_weight_sf), - fc2_weight=_as_nvfp4(weight_view.fc2_weight), - fc2_weight_sf=_as_fp8_sf(weight_view.fc2_weight_sf), - fc1_alpha=weight_view.fc31_alpha, - fc2_alpha=weight_view.fc2_alpha, - fc1_norm_const=weight_view.fc1_norm_const, - combine_output=combine_output, - shared_workspace=shared_workspace, - world_size=world_size, - local_rank=local_rank, - num_topk=top_k, - num_experts_per_rank=int(self.expert_size_per_partition), - hidden_size=hidden, - intermediate_size_per_partition=int(self.intermediate_size_per_partition), - expand_intermediate_size_per_partition=int(self.expand_intermediate_size_per_partition), - max_tokens_per_rank=int(self.max_num_tokens), - peer_offsets=peer_offsets, - apply_topk_in_fc1=bool(self.apply_topk_in_fc1), - gate_up_clamp=self.gate_up_clamp, - token_back_by_dispatch=bool(self.token_back_by_dispatch), - non_ubulk_fc2_store=bool(self.non_ubulk_fc2_store), + # Hand the symmetric profiling scratch (or a deferred factory) to the + # op for tuning-mode profiling launches; cleared in the ``finally`` so + # a later same-process call for a different shape never sees a stale + # scratch (ownership: see ``_profiling_scratch_provider_ref``). + from ....custom_ops.cute_dsl_megamoe_custom_op import ( + set_active_megamoe_profiling_scratch, + set_active_megamoe_profiling_scratch_factory, + ) + + scratch_provider = ( + self._profiling_scratch_provider_ref() + if self._profiling_scratch_provider_ref is not None + else None + ) + scratch_factory = None + if ( + scratch_provider is None + and self._profiling_scratch_kwargs is not None + and world_size > 1 + ): + if AutoTuner.get().is_tuning_mode: + # Tuning but no live provider (lazy-only, or an earlier + # engine's release freed it): hand a DEFERRED factory -- a + # cache-hitting tuning forward (e.g. the spec-decode draft) + # must never allocate the multi-GiB scratch. It runs in the + # op's profiling pre-hook (where the collective rendezvous is + # lockstep-safe) and refreshes this module's weakref. + from ....custom_ops.cute_dsl_megamoe_custom_op import get_megamoe_profiling_scratch + + def _deferred_scratch_factory(_self=self, _get=get_megamoe_profiling_scratch): + provider = _get(**_self._profiling_scratch_kwargs) + _self._profiling_scratch_provider_ref = weakref.ref(provider) + return provider.get_regions() + + scratch_factory = _deferred_scratch_factory + set_active_megamoe_profiling_scratch( + scratch_provider.get_regions() + if (scratch_provider is not None and world_size > 1) + else None ) + set_active_megamoe_profiling_scratch_factory(scratch_factory) + try: + torch.ops.trtllm.cute_dsl_megamoe_nvfp4_blackwell( + activation=_as_nvfp4(activation), + activation_sf=_as_fp8_sf(activation_sf), + topk_idx=topk_idx, + topk_weights=topk_weights, + fc1_weight=_as_nvfp4(weight_view.fc1_weight), + fc1_weight_sf=_as_fp8_sf(weight_view.fc1_weight_sf), + fc2_weight=_as_nvfp4(weight_view.fc2_weight), + fc2_weight_sf=_as_fp8_sf(weight_view.fc2_weight_sf), + fc1_alpha=weight_view.fc31_alpha, + fc2_alpha=weight_view.fc2_alpha, + fc1_norm_const=weight_view.fc1_norm_const, + combine_output=combine_output, + shared_workspace=shared_workspace, + world_size=world_size, + local_rank=local_rank, + num_topk=top_k, + num_experts_per_rank=int(self.expert_size_per_partition), + hidden_size=hidden, + intermediate_size_per_partition=int(self.intermediate_size_per_partition), + expand_intermediate_size_per_partition=int( + self.expand_intermediate_size_per_partition + ), + # Bucket max_T == the staging/symm buffer leading dim above. + max_tokens_per_rank=int(launch_max_T), + peer_offsets=peer_offsets, + apply_topk_in_fc1=bool(self.apply_topk_in_fc1), + gate_up_clamp=self.gate_up_clamp, + # Keep the deterministic standalone TopkReduce until form-B + # has dedicated GPU correctness and performance coverage. + in_kernel_fc2_reduce=False, + combine_format=self.combine_format, + tactic_autotune=bool(self.tactic_autotune), + num_tokens=num_tokens, + ) + finally: + # Always clear: a raising op call must not leave the factory + # global alive (its closure strongly references this module). + set_active_megamoe_profiling_scratch(None) + set_active_megamoe_profiling_scratch_factory(None) if num_tokens == 0: return torch.empty((0, hidden), dtype=output_dtype, device=combine_output.device) - # Deepgemm graph (apply_topk_in_fc1=True): the kernel already folded - # the topk score into the per-route BF16 terms, so the host reduce is - # a plain sum over the top-k axis. Accumulate in fp32 explicitly to - # match the design reference ``bf16(sum_fp32(term))`` and to be robust - # against any future change to the bf16 reduction accumulator type. - out = combine_output[:num_tokens].to(torch.float32).sum(dim=1).to(output_dtype) - return out + # The kernel already collapsed the top-k axis into the (T, 1, hidden) + # output (TopkReduce form-A / REDG form-B); just drop the singleton. + return combine_output[:num_tokens].squeeze(1).to(output_dtype) def _run_moe( self, @@ -1256,29 +1498,41 @@ def _run_moe( ) -> torch.Tensor: """Unified MegaMoE CuteDSL forward: acquire -> stage -> launch. - The kernel is always launched with ``T = max_num_tokens`` (its - compile-time constexpr); live tokens fill the first - ``num_tokens`` rows and the tail is masked via ``topk_idx == -1`` - (skipped by dispatch_kernel) and zero ``topk_weights`` (combine - stale-data guard). + The kernel is launched with ``T = launch_max_T`` -- the smallest + adaptive bucket >= the lockstep cross-rank chunk token count (set by + ``set_adaptive_launch_tokens``), capped at ``max_num_tokens``. Live + tokens fill the first ``num_tokens`` rows and the tail is masked via + ``topk_idx == -1`` (skipped by dispatch_kernel) and zero + ``topk_weights`` (combine stale-data guard). ``FusedCommMoEScheduler`` invariant 7 forces every EP rank to cross the NVLink barrier even with zero local tokens; only single-rank short-circuits ``num_tokens == 0`` because no peer is waiting. """ - if num_tokens > self.max_num_tokens: + # Pick and consume the adaptive bucket; un-hinted callers default to + # the full bucket. + launch_max_T = self._select_launch_max_tokens(self._active_launch_max_tokens) + self._active_launch_max_tokens = None + if num_tokens > launch_max_T: + # By construction num_tokens <= max(all_rank_num_tokens) <= + # launch_max_T; fail loudly rather than desync the barrier. raise RuntimeError( f"MegaMoECuteDsl run_moe got {num_tokens} tokens but the " - f"staging buffer is sized for {self.max_num_tokens}. Raise " - f"model_config.moe_max_num_tokens so peers do not read " - f"invalid rows." + f"selected adaptive bucket is sized for {launch_max_T} " + f"(buckets={self._maxt_buckets}, max_num_tokens=" + f"{self.max_num_tokens}). This indicates the scheduler's " + f"lockstep chunk-max hint diverged from the staged tokens." ) if num_tokens == 0 and self.ep_size == 1: return torch.empty((0, hidden), dtype=output_dtype, device=device) bufs = self._acquire_buffers( - top_k=top_k, hidden=hidden, device=device, output_dtype=output_dtype + top_k=top_k, + hidden=hidden, + device=device, + output_dtype=output_dtype, + launch_max_T=launch_max_T, ) self._stage_inputs( bufs=bufs, @@ -1304,4 +1558,5 @@ def _run_moe( peer_offsets=bufs.peer_offsets, num_tokens=num_tokens, output_dtype=output_dtype, + launch_max_T=launch_max_T, ) diff --git a/tensorrt_llm/_torch/modules/fused_moe/mega_moe/mega_moe_deepgemm.py b/tensorrt_llm/_torch/modules/fused_moe/mega_moe/mega_moe_deepgemm.py index 72fddb08d074..23b82228d196 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/mega_moe/mega_moe_deepgemm.py +++ b/tensorrt_llm/_torch/modules/fused_moe/mega_moe/mega_moe_deepgemm.py @@ -203,7 +203,6 @@ def __init__( layer_idx: Optional[int] = None, activation_type: ActivationType = ActivationType.Swiglu, init_load_balancer: bool = True, - without_comm: bool = False, # DG tunables. ``swiglu_limit_scalar`` mirrors the upstream MoE # kwarg; bridged to DG's ``activation_clamp`` at the call site. activation: str = "swiglu", @@ -564,8 +563,9 @@ def _alloc_symm_buffer(self) -> None: self.routing_method.experts_per_token, self.hidden_size, self.intermediate_size, - True, - self.activation, + num_shared_experts=0, + mma_type="fp8xfp4", + activation=self.activation, ) _MEGA_MOE_SYMM_BUFFER_CACHE[key] = cached # Log only on the first layer; deeper layers reuse the cache diff --git a/tensorrt_llm/_torch/modules/fused_moe/moe_op_backend.py b/tensorrt_llm/_torch/modules/fused_moe/moe_op_backend.py index e908d74794f9..15434d924fe0 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/moe_op_backend.py +++ b/tensorrt_llm/_torch/modules/fused_moe/moe_op_backend.py @@ -214,11 +214,9 @@ class TRTLLMOpBackend(MoEOpBackend): def __init__(self): from tensorrt_llm._mnnvl_utils import MnnvlMemory, MnnvlMoe - from tensorrt_llm._torch.distributed.moe_alltoall import MoeAlltoAll self._MnnvlMemory = MnnvlMemory self._MnnvlMoe = MnnvlMoe - self._MoeAlltoAll = MoeAlltoAll # Quantization def fp4_quantize( diff --git a/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py b/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py index 966c9fb6e046..02380f2106b5 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py +++ b/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py @@ -52,7 +52,7 @@ from tensorrt_llm._torch.utils import EventType, Fp4QuantizedTensor, MxFp8QuantizedTensor from tensorrt_llm.tools.layer_wise_benchmarks import get_calibrator -from .communication import DeepEP, DeepEPLowLatency, NVLinkOneSided, NVLinkTwoSided +from .communication import DeepEP, DeepEPLowLatency, NcclEP, NVLinkOneSided, NVLinkTwoSided from .communication.nvlink_two_sided_flashinfer import NVLinkTwoSidedFlashinfer from .fused_moe_cute_dsl import CuteDslFusedMoE from .fused_moe_cutlass import CutlassFusedMoE, raise_moe_lora_multichunk_unsupported @@ -387,9 +387,14 @@ def _forward_chunk_impl( moe._load_balancer_start_wait_gpu_stage(is_first_call) # ========== Step 2: Apply routing ========== + # External dispatch (Step 5) sends per-token expert/scale payloads, so + # routing must be precomputed whenever a comm strategy is active — even + # for backends whose run_moe can otherwise route internally from + # router_logits (e.g. MarlinFusedMoE under attention-DP + EP). requires_separated_routing = ( moe.backend._supports_load_balancer() or moe.routing_method.requires_separated_routing + or moe.comm is not None or FORCE_SEPARATED_ROUTING ) if requires_separated_routing: @@ -420,10 +425,9 @@ def _forward_chunk_impl( "Current workaround for apply_router_weight_on_input does not support fp8 input" ) x = x * token_final_scales.to(x.dtype) - # DeepEP variants need a non-None token_final_scales tensor - # (they don't tolerate None), so feed all-ones; other strategies - # accept None and skip the multiply. - if isinstance(moe.comm, (DeepEP, DeepEPLowLatency)): + # These strategies need non-None token_final_scales, so feed + # all-ones after folding the real weights into x. + if isinstance(moe.comm, (DeepEP, DeepEPLowLatency, NcclEP)): token_final_scales = torch.ones_like(token_final_scales) else: token_final_scales = None @@ -1217,6 +1221,13 @@ def _forward_chunk( # quantize_input contracts. moe_input, x_sf = moe.backend.quantize_input(x_chunk_real) + # CuteDSL needs the scheduler's rank-identical chunk maximum to select + # one adaptive bucket on every EP rank; using a local token count could + # diverge and deadlock its in-kernel NVLink barrier. + set_adaptive = getattr(moe.backend, "set_adaptive_launch_tokens", None) + if set_adaptive is not None: + set_adaptive(max(all_rank_num_tokens) if all_rank_num_tokens else None) + # ----- MoE compute ----- # ``token_selected_slots`` is in [0, num_slots), matching the kernel's # ``num_experts`` template parameter (SymmBuffer / weights sized to diff --git a/tensorrt_llm/_torch/modules/fused_moe/nccl_ep_utils.py b/tensorrt_llm/_torch/modules/fused_moe/nccl_ep_utils.py new file mode 100644 index 000000000000..9822a0f90a2a --- /dev/null +++ b/tensorrt_llm/_torch/modules/fused_moe/nccl_ep_utils.py @@ -0,0 +1,421 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NCCL EP utilities backed by the nccl4py wheel's ``nccl.ep`` package. + +Owns the long-lived NCCL EP resources (communicator, group, persistent receive +NDTensors) for the MoE NcclEP communication strategy. Per-step dispatch handles +are created in ``communication/nccl_ep.py``. +""" + +from typing import Optional + +import torch + +from tensorrt_llm._utils import TensorWrapper, convert_to_torch_tensor +from tensorrt_llm.logger import logger +from tensorrt_llm.mapping import Mapping + +_MIN_NCCL_RUNTIME_VERSION = "2.30.4" +_MIN_NCCL_EP_INT32_TOPK_VERSION = "0.2" +_NCCL_RUNTIME_ERRORS = (RuntimeError, OSError) +_NCCL_AVAILABILITY_ERRORS = (ImportError,) + _NCCL_RUNTIME_ERRORS + +_nccl_ep_installed: Optional[bool] = None + + +def is_nccl_ep_installed() -> bool: + """Return True iff ``nccl.ep`` is usable. + + Requires that ``nccl.ep`` imports cleanly AND the loaded ``libnccl.so`` + runtime version is >= 2.30.4. + """ + global _nccl_ep_installed + if _nccl_ep_installed is not None: + return _nccl_ep_installed + try: + import nccl + from packaging.version import Version + + runtime = nccl.get_version().nccl.version + if runtime < Version(_MIN_NCCL_RUNTIME_VERSION): + logger.info( + f"NCCL EP disabled: libnccl runtime {runtime} " + f"< required {_MIN_NCCL_RUNTIME_VERSION}" + ) + _nccl_ep_installed = False + return False + import nccl.ep # noqa: F401 + + _nccl_ep_installed = True + except _NCCL_AVAILABILITY_ERRORS as e: + logger.info(f"NCCL EP disabled: nccl.ep is not usable ({e!r})") + _nccl_ep_installed = False + return _nccl_ep_installed + + +def _nccl_ep_supports_int32_topk_idx() -> bool: + """Return True when the loaded libnccl_ep supports int32 input topk_idx.""" + try: + import nccl + from packaging.version import Version + + nccl_ep_info = nccl.get_version().nccl_ep + nccl_ep_version = nccl_ep_info.version if nccl_ep_info is not None else None + except (ImportError, AttributeError, RuntimeError, OSError) as e: + logger.info( + f"NCCL EP int32 topk_idx disabled: could not determine libnccl_ep version ({e!r})" + ) + return False + + if nccl_ep_version is None: + logger.info("NCCL EP int32 topk_idx disabled: libnccl_ep version is not available") + return False + + if nccl_ep_version < Version(_MIN_NCCL_EP_INT32_TOPK_VERSION): + logger.info( + f"NCCL EP int32 topk_idx disabled: libnccl_ep {nccl_ep_version} " + f"< required {_MIN_NCCL_EP_INT32_TOPK_VERSION}" + ) + return False + + return True + + +# Singleton EP context keyed by (ep_size, ep_rank, max_tokens, num_experts, +# hidden, max_top_k, layout). +_ep_group_cache: dict = {} +_ep_group_refcounts: dict = {} + + +class NcclEpContext: + """Long-lived NCCL EP group + receive buffers, shared across NcclEP instances. + + Owns the :class:`nccl.ep.Group`, the source :class:`nccl.core.Communicator`, + and the rank-major LL persistent receive buffers (tokens, top-k idx / weights, + per-source-rank counter) wrapped as + :class:`nccl.ep.Tensor` descriptors. + + Per-step routing handles (``Handle``) are created in ``NcclEP``, not here. + """ + + def __init__( + self, + mapping: Mapping, + num_experts: int, + max_tokens_per_rank: int, + hidden_size: int, + max_top_k: int, + layout: Optional[int] = None, + ): + import nccl.core as nccl_core + from nccl.ep import Algorithm, Group, GroupConfig, Layout, Tensor + + from tensorrt_llm._utils import mpi_comm + + self.mapping = mapping + self.ep_size = mapping.moe_ep_size + self.ep_rank = mapping.moe_ep_rank + self.num_experts = num_experts + self.num_local_experts = num_experts // self.ep_size + self.max_tokens_per_rank = max_tokens_per_rank + self.max_top_k = max_top_k + self.hidden_size = hidden_size + self.layout = Layout.RANK_MAJOR if layout is None else Layout(layout) + self.max_recv_tokens = self.ep_size * max_tokens_per_rank + + # topk_idx dtype passed to the EP runtime. NCCL-EP < 0.2 asserts + # int64 in ncclEpUpdateHandle; 0.2+ supports TRT-LLM's native int32 + # routing ids and avoids the per-iter widening conversion. + self.topk_idx_dtype = torch.int32 if _nccl_ep_supports_int32_topk_idx() else torch.int64 + self._v0_2_features_enabled = self.topk_idx_dtype == torch.int32 + + # NCCL-EP v0.2+ may expose a configurable receive expert-id kind. + # Within that version-gated path, detect whether the linked binding supports a + # configurable recv_topk_idx kind on LayoutInfo. When the field + # is present we set it to GLOBAL and skip the post-dispatch + # local->global rewrite; otherwise the kernel writes LOCAL ids + # unconditionally (older nccl-ep builds) and the dispatch + # wrapper applies torch.where to restore the global contract + # NVLinkOneSided also advertises. + self.kernel_writes_global_ids = False + self._expert_id_kind_global = None + if self._v0_2_features_enabled: + try: + from nccl.bindings.nccl_ep import ExpertIdKind as _ExpertIdKind + from nccl.bindings.nccl_ep import LayoutInfo as _LowLayoutInfo + + self.kernel_writes_global_ids = hasattr(_LowLayoutInfo(), "recv_topk_idx_kind") + self._expert_id_kind_global = ( + int(_ExpertIdKind.GLOBAL) if self.kernel_writes_global_ids else None + ) + except (ImportError, AttributeError): + pass + + # NCCL-EP v0.2+ may support opportunistic zero-copy dispatch. + # When the Pythonic GroupConfig facade exposes `zero_copy` (i.e., + # the wheel was built against a libnccl_ep.so that has the field + # in ncclEpGroupConfig_t), we allocate a VMM-backed, + # window-registered dispatch output buffer; the LL dispatch + # opportunistically picks zero-copy when recv_x->win_hdl is set + # (nvlink-only + rank-major). The config flag itself stays + # AUTO/OFF -- strict zero_copy=ON requires combine inputs to be + # windowed too, which would force a caller-side interface change + # (the MLP output is caller-owned). The C-side strict-ON check + # remains in the library for future use. + self.zerocopy_enabled = self._v0_2_features_enabled and "zero_copy" in getattr( + GroupConfig, "__dataclass_fields__", {} + ) + + # MPI sub-communicator scoped to the EP group. Mirrors the + # DeepEPLowLatency pattern (see deep_ep_utils.py:104): split + # MPI_COMM_WORLD by pp_rank so each pipeline stage gets its own EP + # comm, keyed by moe_ep_rank. Avoids the wheel's + # nccl.ep.get_nccl_comm_from_group() helper which requires + # torch.distributed.init_process_group() -- the test infrastructure + # (mpi_pool_executor) and microbenchmarks use MPI4PY only. + self._ep_mpi_comm = mpi_comm().Split(mapping.pp_rank, mapping.moe_ep_rank) + ep_world_rank = self._ep_mpi_comm.Get_rank() + ep_world_size = self._ep_mpi_comm.Get_size() + unique_id = nccl_core.get_unique_id() if ep_world_rank == 0 else None + unique_id = self._ep_mpi_comm.bcast(unique_id, root=0) + self.comm = nccl_core.Communicator.init( + nranks=ep_world_size, + rank=ep_world_rank, + unique_id=unique_id, + ) + + cfg = GroupConfig( + algorithm=Algorithm.LOW_LATENCY, + num_experts=num_experts, + max_dispatch_tokens_per_rank=max_tokens_per_rank, + max_recv_tokens_per_rank=self.max_recv_tokens, + max_token_bytes=hidden_size * 2, # bfloat16 + ) + self.ep_group = Group.create(self.comm, cfg) + + logger.info( + f"NCCL EP group created: ep_size={self.ep_size}, " + f"num_experts={num_experts}, max_tokens_per_rank={max_tokens_per_rank}, " + f"hidden_size={hidden_size}, max_top_k={max_top_k}, " + f"layout={self.layout.name}" + ) + + device_id = torch.cuda.current_device() + device = torch.device("cuda", device_id) + + # Dispatch output tokens: 3D [ep_size, max_tokens_per_rank, hidden] + # for LL rank-major. When zerocopy is enabled the buffer must be + # VMM-backed (cuMemMap) so ncclCommWindowRegister's internal + # cuMemGetAddressRange call succeeds -- torch's caching + # allocator returns plain cudaMalloc memory which fails that + # check with CUDA_ERROR_INVALID_VALUE. Allocate via + # nccl.core.mem_alloc (VMM-backed) then build a zero-copy torch + # view over the raw pointer via the TRT-LLM CAI wrapper. + token_shape = (self.ep_size, max_tokens_per_rank, hidden_size) + token_nbytes = self.ep_size * max_tokens_per_rank * hidden_size * 2 + self._output_tokens_alloc = None + self._recv_x_window = None + if self.zerocopy_enabled: + self._output_tokens_alloc = nccl_core.mem_alloc( + token_nbytes, + device=device_id, + ) + self.output_tokens_buf = convert_to_torch_tensor( + TensorWrapper( + int(self._output_tokens_alloc.handle), + dtype=torch.bfloat16, + shape=token_shape, + ) + ) + self._recv_x_window = self.comm.register_window( + self._output_tokens_alloc, + ) + else: + self.output_tokens_buf = torch.empty( + *token_shape, + dtype=torch.bfloat16, + device=device, + ) + # Received topk indices: int32 [ep_size, max_tokens_per_rank, max_top_k] + # for the LL rank-major dispatch contract. -1 marks invalid rows. + # Downstream consumers want 2D [max_recv, max_top_k]; flatten via view. + self.recv_topk_idx_buf = torch.empty( + self.ep_size, + max_tokens_per_rank, + max_top_k, + dtype=torch.int32, + device=device, + ) + # Received topk weights: float32 [ep_size, max_tokens_per_rank, max_top_k] + self.recv_topk_weights_buf = torch.empty( + self.ep_size, + max_tokens_per_rank, + max_top_k, + dtype=torch.float32, + device=device, + ) + # Per-source-rank received-token counter (passed via + # LayoutInfo.src_rank_counters at dispatch time). + self.recv_rank_counter_buf = torch.empty( + self.ep_size, + dtype=torch.int32, + device=device, + ) + # Wrap each persistent buffer as a Tensor descriptor. Torch owns the + # storage; the descriptor only carries shape + a pointer (+ window + # handle on dispatch output when zerocopy is on, so libnccl_ep's + # opportunistic LL zero-copy path can fire). + if self.zerocopy_enabled and self._recv_x_window is not None: + self.output_tokens_nd = Tensor( + self.output_tokens_buf, + window=self._recv_x_window, + window_offset=0, + ) + else: + self.output_tokens_nd = Tensor(self.output_tokens_buf) + self.recv_topk_idx_nd = Tensor(self.recv_topk_idx_buf) + self.recv_topk_weights_nd = Tensor(self.recv_topk_weights_buf) + self.recv_rank_counter_nd = Tensor(self.recv_rank_counter_buf) + + def get_stream(self) -> int: + """Current CUDA stream as a raw int handle (accepted by ``nccl.ep`` APIs).""" + return torch.cuda.current_stream().cuda_stream + + def destroy(self): + """Release EP group, NCCL comm, and MPI sub-comm in LIFO order. + + Avoids relying on Python GC ordering between the group, the comm it + was built from, and the MPI sub-comm seeding it: the group must go + first (uses the comm), then ``finalize`` + ``destroy`` on the comm + (the recommended nccl4py pattern), then ``Free`` on the MPI comm. + """ + if self.ep_group is not None: + try: + self.ep_group.destroy() + except _NCCL_RUNTIME_ERRORS as e: + logger.warning(f"NCCL EP group destroy error: {e}") + self.ep_group = None + + # Deregister windows before the comm goes away. close() is + # idempotent and local; the comm would auto-close any leftover + # windows on destroy, but explicit LIFO release matches the rest + # of this teardown path. Both dispatch-output and combine-input + # windows are registered only when zerocopy is on. + for attr in ("_combine_input_window", "_recv_x_window"): + w = getattr(self, attr, None) + if w is not None: + try: + w.close() + except _NCCL_RUNTIME_ERRORS as e: + logger.warning(f"NCCL EP window close error ({attr}): {e}") + setattr(self, attr, None) + + # Drop torch view + EP descriptor before freeing the underlying + # NCCL-allocated Buffer (CAI view doesn't refcount the source). + # close() the cuda.core.Buffer to call nccl.core.mem_free; the + # alloc is only populated when zerocopy is enabled. + self.output_tokens_nd = None + self.output_tokens_buf = None + if getattr(self, "_output_tokens_alloc", None) is not None: + try: + self._output_tokens_alloc.close() + except _NCCL_RUNTIME_ERRORS as e: + logger.warning(f"NCCL EP recv_x buffer free error: {e}") + self._output_tokens_alloc = None + + if self.comm is not None: + try: + self.comm.finalize() + self.comm.destroy() + except _NCCL_RUNTIME_ERRORS as e: + logger.warning(f"NCCL EP comm destroy error: {e}") + self.comm = None + + if self._ep_mpi_comm is not None: + from mpi4py import MPI + + try: + self._ep_mpi_comm.Free() + except MPI.Exception as e: + logger.warning(f"EP MPI sub-comm free error: {e}") + self._ep_mpi_comm = None + + +def get_nccl_ep_context( + mapping: Mapping, + num_experts: int, + max_tokens_per_rank: int, + hidden_size: int, + max_top_k: int, + layout: Optional[int] = None, +) -> NcclEpContext: + """Get or create a singleton :class:`NcclEpContext` for the given configuration.""" + from nccl.ep import Layout + + if layout is None: + layout = Layout.RANK_MAJOR + key = ( + mapping.moe_ep_size, + mapping.moe_ep_rank, + max_tokens_per_rank, + num_experts, + hidden_size, + max_top_k, + int(layout), + ) + if key not in _ep_group_cache: + _ep_group_cache[key] = NcclEpContext( + mapping, + num_experts, + max_tokens_per_rank, + hidden_size, + max_top_k, + layout, + ) + _ep_group_refcounts[key] = _ep_group_refcounts.get(key, 0) + 1 + return _ep_group_cache[key] + + +def release_nccl_ep_context(ctx: Optional[NcclEpContext]) -> None: + """Release one reference to a cached :class:`NcclEpContext`.""" + if ctx is None: + return + + key = next((key for key, cached_ctx in _ep_group_cache.items() if cached_ctx is ctx), None) + if key is None: + return + + refcount = _ep_group_refcounts.get(key, 0) - 1 + if refcount > 0: + _ep_group_refcounts[key] = refcount + return + + _ep_group_refcounts.pop(key, None) + cached_ctx = _ep_group_cache.pop(key) + try: + cached_ctx.destroy() + except _NCCL_RUNTIME_ERRORS as e: + logger.warning(f"Error destroying NCCL EP context: {e}") + + +def destroy_all_nccl_ep_contexts(): + """Destroy all cached NCCL EP contexts (call at process teardown).""" + for ctx in list(_ep_group_cache.values()): + try: + ctx.destroy() + except _NCCL_RUNTIME_ERRORS as e: + logger.warning(f"Error destroying NCCL EP context: {e}") + _ep_group_cache.clear() + _ep_group_refcounts.clear() diff --git a/tensorrt_llm/_torch/modules/fused_moe/quantization.py b/tensorrt_llm/_torch/modules/fused_moe/quantization.py index 3f3754f7eec0..e134fd26b546 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/quantization.py +++ b/tensorrt_llm/_torch/modules/fused_moe/quantization.py @@ -15,6 +15,7 @@ import inspect import math +import threading from abc import ABC, abstractmethod from enum import Enum, auto from typing import Dict, List, NamedTuple, Optional, Tuple, Union @@ -2094,6 +2095,11 @@ class NVFP4FusedMoEMethod(FusedMoEMethodBase): """ eplb_support_status = EplbSupportStatus.SUPPORTED + # Whether raw per-expert block-scale staging is an EPLB migration + # target. Children that migrate derived formats and free the raw + # sources (MegaMoE-CuteDSL) set this False. + _eplb_migrate_raw_block_scales = True + def get_weights_shapes(self, module: torch.nn.Module, weight_vec_size: int, block_scales_vec_size: int): # Divide by 16 because we use int64 to pack 16 fp4 values @@ -2647,11 +2653,14 @@ def process_weights_after_loading(self, module: torch.nn.Module): # before register_all_parameter_slot_and_to_fix_weight_fns below. self._prepare_shared_weight_scales_for_finalization(module) weight_fns = { - 'w3_w1_weight_scale': module.local_shared_w3_w1_scale_tensors, - 'w2_weight_scale': module.local_shared_w2_scale_tensors, 'fc31_alpha': shared_fc31_alpha, 'fc2_alpha': shared_fc2_alpha, } + if self._eplb_migrate_raw_block_scales: + weight_fns['w3_w1_weight_scale'] = ( + module.local_shared_w3_w1_scale_tensors) + weight_fns['w2_weight_scale'] = ( + module.local_shared_w2_scale_tensors) if shared_fc31_weight_scale_2 is not None: weight_fns['fc31_weight_scale_2'] = shared_fc31_weight_scale_2 if shared_fc2_weight_scale_2 is not None: @@ -3446,21 +3455,22 @@ class NVFP4MegaMoECuteDslMethod(NVFP4FusedMoEMethod): MegaMoE-format derived tensors, and fills the per-slot ``fc1_norm_const`` tensor from each expert's raw ``w2.input_scale``. - EPLB support is ``SUPPORTED``: dynamic EPLB migrates the four - ``mega_fc*_weight*`` derived parameters and per-expert - ``fc1_norm_const`` via CPU shared-staging buffers built in - :meth:`_build_mega_shared_staging` / :meth:`_build_fc1_norm_const` and - registered through :meth:`register_all_parameter_slot_and_to_fix_weight_fns`, - in addition to the standard NVFP4 family (``w3_w1_weight`` / - ``w2_weight`` / ``w*_weight_scale`` / ``fc*_alpha``) handled by the - base / grandparent classes. Slot migration replaces all raw + - MegaMoE-derived parameters atomically with byte-consistent values from - the source rank (the source built mega = transform(raw) once at - load time, so the migrated raw and mega bytes stay paired). + EPLB support is ``SUPPORTED`` with DERIVED-ONLY migration (the + MegaMoE-DeepGemm pattern): dynamic EPLB migrates the ``mega_fc*`` + derived parameters and ``fc1_norm_const`` via CPU shared staging, plus + the per-expert ``fc*_alpha`` / ``fc*_weight_scale_2`` handled by the + NVFP4 parent. The RAW source params are NOT migration targets (see + :meth:`_finalize_shared_weights`, ``_eplb_migrate_raw_block_scales``) + and are freed unconditionally after packing (``run_moe`` reads only + the mega buffers). Dynamic EPLB + hot weight RELOAD is unsupported. """ eplb_support_status = EplbSupportStatus.SUPPORTED + # Raw block scales are freed after packing; migrating them would slice + # 0-element placeholders (see class docstring). + _eplb_migrate_raw_block_scales = False + # On-device NVFP4 byte formats. Same constants the Cutlass child # uses; they describe the NVFP4 weight / FP8 block-scale packing, # not anything Cutlass-kernel-specific. @@ -3508,7 +3518,9 @@ def _round_up_int(a: int, b: int) -> int: @classmethod def fc1_sf_flat_size(cls, intermediate: int, hidden: int) -> int: """``round_up(expand_intermediate, SfPaddingBlock=128) * - round_up(ceil(hidden / 16), 4)`` -- matches kernel_fc12.py:880-890. + round_up(ceil(hidden / 16), 4)`` -- matches the FC1 weight-SF view + in ``kernel_fc12.py`` (``intermediate_gateup_padded`` / + ``expected_fc1_weight_sf_cols``). ``expand_intermediate = 2 * intermediate``. """ expand_intermediate = intermediate * 2 @@ -3518,11 +3530,25 @@ def fc1_sf_flat_size(cls, intermediate: int, hidden: int) -> int: @classmethod def fc2_sf_flat_size(cls, hidden: int, intermediate: int) -> int: """``round_up(hidden, SfPaddingBlock=128) * - round_up(ceil(intermediate / 16), 4)`` -- matches runner_fc12.py:1305. + round_up(ceil(intermediate / 16), 4)`` -- matches the FC2 weight-SF + view in ``kernel_fc12.py`` (``hidden_padded_fc2`` / + ``expected_fc2_weight_sf_cols``). """ return (cls._round_up_int(hidden, 128) * cls._round_up_int(cls._ceil_div_int(intermediate, 16), 4)) + # Source-checkpoint tensors kept as 0-element placeholders outside the + # load window so the full source set and the mega buffers never coexist + # (streaming load: peak = steady set + ONE layer of sources). + _STREAMED_SOURCE_PARAMS = ("w3_w1_weight", "w3_w1_weight_scale", + "w2_weight", "w2_weight_scale") + + # Serializes the transient source-set window (materialize -> load -> + # eager finalize) across MODULES: the generic loader runs module loads + # on a ThreadPoolExecutor, and concurrent windows would break the + # "peak = steady set + ONE layer" bound. Class-level on purpose. + _streamed_transient_lock = threading.Lock() + # ----------------------------------------------------------------- # create_weights: register MegaMoE-format parameters in addition to # the grandparent's standard NVFP4 parameters. @@ -3545,6 +3571,12 @@ def create_weights(self, module: torch.nn.Module): f"got expand_intermediate=" f"{module.expand_intermediate_size_per_partition}, " f"intermediate={module.intermediate_size_per_partition}.") + # Fail fast: the derived-only EPLB migration path registers no bias + # staging. + if module.bias and self.need_load_shared_weights(module): + raise NotImplementedError( + "NVFP4MegaMoECuteDslMethod does not support expert bias " + "together with dynamic-EPLB shared weight loading.") weight_vec_size = torch.iinfo(self.weight_dtype).bits // 4 self.block_scales_vec_size = torch.iinfo( @@ -3560,6 +3592,21 @@ def create_weights(self, module: torch.nn.Module): self.block_scales_dtype, self.block_scales_vec_size) + # Streaming load: shrink the source params to 0-element placeholders + # (full shapes preserved in rebuild_tensor_metadata); load_weights + # rematerializes one module at a time. Otherwise init materializes + # the full source set AND the mega buffers (~229 GB/rank). + for _name in self._STREAMED_SOURCE_PARAMS: + _p = getattr(module, _name) + replace_parameter_and_save_metadata( + module, _name, + nn.Parameter(torch.empty(0, dtype=_p.dtype), + requires_grad=False), + module.rebuild_tensor_metadata) + # Rebind quant_scales to the placeholders so the full-size CPU + # init tensors are actually released. + self.setup_quant_scales(module) + num_local_slots = module.expert_size_per_partition hidden = module.hidden_size intermediate = module.intermediate_size_per_partition @@ -3624,6 +3671,55 @@ def create_weights(self, module: torch.nn.Module): ) module.register_parameter("fc1_norm_const", fc1_norm_const) + def _materialize_source_params(self, module: torch.nn.Module): + """Rematerialize this module's streamed source params (full shape) + so the loader can fill them; no-op when already materialized. + """ + for name in self._STREAMED_SOURCE_PARAMS: + p = getattr(module, name, None) + if (p is not None and p.data.numel() == 0 + and name in module.rebuild_tensor_metadata): + meta = module.rebuild_tensor_metadata[name]['meta'] + module.register_parameter( + name, + nn.Parameter(torch.empty_like(meta, device="cuda"), + requires_grad=False)) + + def _finalize_shared_weights(self, module: torch.nn.Module): + # Derived-only EPLB migration: skip the base raw-weight + # registration -- the balancer would IndexError slicing the freed + # 0-element raw placeholders (see class docstring). + if not self.need_load_shared_weights(module): + return + if not hasattr(module, 'local_shared_w3_w1_tensors'): + # Already finalized (idempotent, mirroring the base contract). + return + if module.bias: + raise ValueError( + "MegaMoE-CuteDSL EPLB shared loading does not support " + "expert bias.") + delattr(module, 'local_shared_w3_w1_tensors') + delattr(module, 'local_shared_w2_tensors') + module.layer_load_balancer.host_tensor_sharer.finalize_layer_weights() + + def load_weights(self, + module: torch.nn.Module, + weights: List[Dict], + weight_loading_mode: MoEWeightLoadingMode, + allow_partial_loading: bool = False): + if allow_partial_loading: + raise NotImplementedError( + "MegaMoE-CuteDSL only supports full initial weight loading.") + + # The transient source-set window is serialized across modules + # (_streamed_transient_lock). + with NVFP4MegaMoECuteDslMethod._streamed_transient_lock: + self._materialize_source_params(module) + super().load_weights(module, + weights, + weight_loading_mode, + allow_partial_loading=allow_partial_loading) + # ----------------------------------------------------------------- # Loader overrides (4x @abstractmethod hooks on the grandparent). # Each one stashes the raw checkpoint shard in a tmp dict keyed by @@ -3638,23 +3734,18 @@ def load_expert_w3_w1_weight(self, dst_w3_w1_weight: torch.Tensor, allow_partial_loading: bool = False, expert_idx: int = -1): - if not allow_partial_loading: - assert w1_weight is not None and w3_weight is not None - if w1_weight is None and w3_weight is None: - return + assert w1_weight is not None and w3_weight is not None device = dst_w3_w1_weight.device - w1_weight_shard = load_weight_shard( - w1_weight, - module.tp_size, - module.tp_rank, - TensorParallelMode.COLUMN, - device=device) if w1_weight is not None else None - w3_weight_shard = load_weight_shard( - w3_weight, - module.tp_size, - module.tp_rank, - TensorParallelMode.COLUMN, - device=device) if w3_weight is not None else None + w1_weight_shard = load_weight_shard(w1_weight, + module.tp_size, + module.tp_rank, + TensorParallelMode.COLUMN, + device=device) + w3_weight_shard = load_weight_shard(w3_weight, + module.tp_size, + module.tp_rank, + TensorParallelMode.COLUMN, + device=device) if not hasattr(module, 'tmp_cutlass_w3_w1_weights'): module.tmp_cutlass_w3_w1_weights = {} @@ -3663,22 +3754,17 @@ def load_expert_w3_w1_weight(self, dict_key = (dst_base, expert_idx) expert_entry = module.tmp_cutlass_w3_w1_weights.setdefault(dict_key, {}) expert_entry['dst'] = dst_w3_w1_weight - if w1_weight_shard is not None: - expert_entry['w1'] = w1_weight_shard.contiguous().view( - dst_w3_w1_weight.dtype) - if w3_weight_shard is not None: - expert_entry['w3'] = w3_weight_shard.contiguous().view( - dst_w3_w1_weight.dtype) + expert_entry['w1'] = w1_weight_shard.contiguous().view( + dst_w3_w1_weight.dtype) + expert_entry['w3'] = w3_weight_shard.contiguous().view( + dst_w3_w1_weight.dtype) def load_expert_w2_weight(self, module: torch.nn.Module, w2_weight: torch.Tensor, dst_w2_weight: torch.Tensor, allow_partial_loading: bool = False): - if not allow_partial_loading: - assert w2_weight is not None - if w2_weight is None: - return + assert w2_weight is not None device = dst_w2_weight.device w2_weight_shard = load_weight_shard(w2_weight, module.tp_size, @@ -3690,6 +3776,9 @@ def load_expert_w2_weight(self, cast_w2_weight_shard = self._maybe_padding_shape( cast_w2_weight_shard, dst_w2_weight) dst_w2_weight.copy_(cast_w2_weight_shard, non_blocking=True) + if not hasattr(module, '_streamed_w2_covered'): + module._streamed_w2_covered = set() + module._streamed_w2_covered.add(dst_w2_weight.data_ptr()) def load_expert_w3_w1_weight_scale_nvfp4( self, @@ -3699,18 +3788,16 @@ def load_expert_w3_w1_weight_scale_nvfp4( dst_w3_w1_weight_scale: torch.Tensor, expert_idx: int = -1): device = dst_w3_w1_weight_scale.device - w1_weight_scale = load_weight_shard( - w1_weight_scale, - module.tp_size, - module.tp_rank, - TensorParallelMode.COLUMN, - device=device) if w1_weight_scale is not None else None - w3_weight_scale = load_weight_shard( - w3_weight_scale, - module.tp_size, - module.tp_rank, - TensorParallelMode.COLUMN, - device=device) if w3_weight_scale is not None else None + w1_weight_scale = load_weight_shard(w1_weight_scale, + module.tp_size, + module.tp_rank, + TensorParallelMode.COLUMN, + device=device) + w3_weight_scale = load_weight_shard(w3_weight_scale, + module.tp_size, + module.tp_rank, + TensorParallelMode.COLUMN, + device=device) if not hasattr(module, 'tmp_cutlass_w3_w1_weight_scales'): module.tmp_cutlass_w3_w1_weight_scales = {} @@ -3720,12 +3807,10 @@ def load_expert_w3_w1_weight_scale_nvfp4( expert_entry = module.tmp_cutlass_w3_w1_weight_scales.setdefault( dict_key, {}) expert_entry['dst'] = dst_w3_w1_weight_scale - if w3_weight_scale is not None: - expert_entry['w3'] = w3_weight_scale.contiguous().view( - dst_w3_w1_weight_scale.dtype) - if w1_weight_scale is not None: - expert_entry['w1'] = w1_weight_scale.contiguous().view( - dst_w3_w1_weight_scale.dtype) + expert_entry['w3'] = w3_weight_scale.contiguous().view( + dst_w3_w1_weight_scale.dtype) + expert_entry['w1'] = w1_weight_scale.contiguous().view( + dst_w3_w1_weight_scale.dtype) def load_expert_w2_weight_scale_nvfp4(self, module: torch.nn.Module, w2_weight_scale: torch.Tensor, @@ -3750,6 +3835,9 @@ def load_expert_w2_weight_scale_nvfp4(self, module: torch.nn.Module, cast_w2_weight_scale = self._maybe_padding_shape( cast_w2_weight_scale, dst_w2_weight_scale) dst_w2_weight_scale.copy_(cast_w2_weight_scale) + if not hasattr(module, '_streamed_w2_scale_covered'): + module._streamed_w2_scale_covered = set() + module._streamed_w2_scale_covered.add(dst_w2_weight_scale.data_ptr()) @staticmethod def _maybe_padding_shape(source_tensor: torch.Tensor, @@ -3781,7 +3869,111 @@ def _maybe_padding_shape(source_tensor: torch.Tensor, # mega-format CPU staging with the load balancer. ``fc1_norm_const`` is # built before the parent deletes raw input-scale staging. # ----------------------------------------------------------------- + def _streamed_coverage(self, module: torch.nn.Module) -> Dict[str, int]: + """Per-component count of routed local experts that received data. + A w3_w1 stash entry counts only when BOTH halves arrived; the + direct-copy w2 paths are tracked via row-pointer sets. Routed + slots are told apart from EPLB shared staging by storage base. + """ + + def _stash_covered(stash_name: str, param) -> int: + base = param.data.storage().data_ptr() + stash = getattr(module, stash_name, {}) + return sum(1 for (b, _idx), e in stash.items() + if b == base and 'w1' in e and 'w3' in e) + + def _rows_covered(set_name: str, param) -> int: + # Tensor (not storage) base: the recorded row addresses came + # from ``dst.data_ptr()`` of slices of this tensor. + base = param.data.data_ptr() + end = base + param.data.numel() * param.data.element_size() + ptrs = getattr(module, set_name, ()) + return sum(1 for p in ptrs if base <= p < end) + + return { + 'w3_w1_weight': + _stash_covered('tmp_cutlass_w3_w1_weights', module.w3_w1_weight), + 'w3_w1_weight_scale': + _stash_covered('tmp_cutlass_w3_w1_weight_scales', + module.w3_w1_weight_scale), + 'w2_weight': + _rows_covered('_streamed_w2_covered', module.w2_weight), + 'w2_weight_scale': + _rows_covered('_streamed_w2_scale_covered', module.w2_weight_scale), + } + + def _check_initial_aux_scale_coverage(self, + module: torch.nn.Module) -> None: + """Reject partially populated NVFP4 auxiliary-scale families.""" + n_slots = module.expert_size_per_partition + n_experts = module.num_experts + weight_scale_2 = getattr(module, 'tmp_weight_scale_2', None) or {} + raw_input_scales = getattr(module, 'tmp_raw_input_scales', None) or {} + + families = { + 'weight_scale_2': ( + sum(1 for entry in weight_scale_2.values() if entry), + sum(1 for entry in weight_scale_2.values() + if {'w1', 'w3', 'w2'} <= set(entry)), + n_slots, + ), + 'w1/w3 input_scale': ( + sum(1 for entry in raw_input_scales.values() + if 'w1' in entry or 'w3' in entry), + sum(1 for entry in raw_input_scales.values() + if 'w1' in entry and 'w3' in entry), + n_experts, + ), + 'w2 input_scale': ( + sum(1 for entry in raw_input_scales.values() if 'w2' in entry), + sum(1 for entry in raw_input_scales.values() if 'w2' in entry), + n_experts, + ), + } + incomplete = { + name: (complete, required) + for name, (present, complete, required) in families.items() + if present and complete < required + } + if incomplete: + raise RuntimeError( + "MegaMoE-CuteDSL initial load delivered partial auxiliary " + "scale families: " + + ", ".join(f"{name}={complete}/{required}" + for name, (complete, required) in incomplete.items())) + + has_weight_scale_2 = families['weight_scale_2'][0] > 0 + has_input_scale = (families['w1/w3 input_scale'][0] > 0 + or families['w2 input_scale'][0] > 0) + if has_input_scale and not has_weight_scale_2: + raise RuntimeError( + "MegaMoE-CuteDSL initial load provided input_scale without " + "weight_scale_2; both are required to derive expert alphas.") + if has_weight_scale_2 and not families['w2 input_scale'][0]: + raise RuntimeError( + "MegaMoE-CuteDSL initial load provided weight_scale_2 without " + "w2 input_scale; both are required to derive fc2 alpha.") + def process_weights_after_loading(self, module: torch.nn.Module): + if module.w3_w1_weight.data.numel() == 0: + return + + n_slots = module.expert_size_per_partition + coverage = self._streamed_coverage(module) + incomplete = {k: v for k, v in coverage.items() if v < n_slots} + if incomplete: + raise RuntimeError( + "MegaMoE-CuteDSL initial load left source components " + f"partially covered ({n_slots} local experts required per " + "component): " + ", ".join(f"{k}={v}/{n_slots}" + for k, v in incomplete.items()) + + ". The uncovered rows are uninitialized; full initial " + "loading must provide w1+w3+w2 weights and block scales for " + "every local expert.") + self._check_initial_aux_scale_coverage(module) + for attr in ('_streamed_w2_covered', '_streamed_w2_scale_covered'): + if hasattr(module, attr): + delattr(module, attr) # ---- Cat raw w3+w1 weights ---- # Iterates BOTH routed (module.w3_w1_weight.data) and shared # (module.local_shared_w3_w1_tensors) entries: the loader keys @@ -3850,6 +4042,31 @@ def process_weights_after_loading(self, module: torch.nn.Module): if self.need_load_shared_weights(module): self._register_mega_shared_staging(module) + # ---- Release the now-dead source NVFP4 routed weights ---- + # run_moe reads only the mega buffers, and EPLB migration targets + # are mega-format too (see class docstring), so the raw set is + # freed unconditionally. Rebinding quant_scales inside the shrink + # matters: otherwise every layer's freed source scales (~11 GB/rank) + # stay reachable until the model-wide sweep rebinds them. + self._shrink_streamed_source_params(module) + + def _shrink_streamed_source_params(self, module: torch.nn.Module): + """Re-shrink materialized streamed sources to 0-element placeholders + and rebind quant_scales to them. Idempotent. + """ + for _name in self._STREAMED_SOURCE_PARAMS: + _p = getattr(module, _name, None) + if _p is not None and _p.data.numel() > 0: + # On repeat calls replace_parameter_and_save_metadata + # re-registers the ORIGINAL saved placeholder; the device + # of the tensor passed here is effectively ignored. + replace_parameter_and_save_metadata( + module, _name, + nn.Parameter(torch.empty(0, dtype=_p.data.dtype), + requires_grad=False), + module.rebuild_tensor_metadata) + self.setup_quant_scales(module) + @staticmethod def _build_fc1_norm_const_tensor(raw_input_scales: Dict, expert_ids: List[int], @@ -3887,6 +4104,10 @@ def _build_fc1_norm_const(self, module: torch.nn.Module) -> None: module.fc1_norm_const.data.copy_( scalar.expand(num_local_slots).contiguous()) return + if not any('w2' in e for e in raw_input_scales.values()): + # Weights-only reload: the stash exists but carries no w2 input + # scales; keep the previously built values. + return routed_norm_const = self._build_fc1_norm_const_tensor( raw_input_scales, @@ -4061,12 +4282,18 @@ def _build_mega_format_weights(self, module: torch.nn.Module): mega_fc2_weight_sf}`` via :meth:`_build_mega_format_buffers` (the transform pipeline itself). """ + + # CPU-staged reload sources: upload ONE layer transiently so the + # pack pipeline runs on GPU (see _materialize_source_params). + def _on_cuda(t: torch.Tensor) -> torch.Tensor: + return t if t.is_cuda else t.cuda() + mega_fc1, mega_fc1_sf, mega_fc2, mega_fc2_sf = ( self._build_mega_format_buffers( - raw_w3_w1=module.w3_w1_weight.data, - raw_w3_w1_sf=module.w3_w1_weight_scale.data, - raw_w2=module.w2_weight.data, - raw_w2_sf=module.w2_weight_scale.data, + raw_w3_w1=_on_cuda(module.w3_w1_weight.data), + raw_w3_w1_sf=_on_cuda(module.w3_w1_weight_scale.data), + raw_w2=_on_cuda(module.w2_weight.data), + raw_w2_sf=_on_cuda(module.w2_weight_scale.data), num_slots=module.expert_size_per_partition, intermediate=module.intermediate_size_per_partition, hidden=module.hidden_size, diff --git a/tensorrt_llm/_torch/modules/fused_moe/routing.py b/tensorrt_llm/_torch/modules/fused_moe/routing.py index 3273269c4ed1..28dd1c2bcc0a 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/routing.py +++ b/tensorrt_llm/_torch/modules/fused_moe/routing.py @@ -676,12 +676,33 @@ def __init__( output_dtype=output_dtype, ) self.routed_scaling_factor = float(routed_scaling_factor) + self.routing_impl = Deepseekv3RoutingImpl( + top_k=top_k, + n_group=1, + topk_group=1, + routed_scaling_factor=self.routed_scaling_factor, + ) def apply( self, router_logits: torch.Tensor, input_ids: Optional[torch.Tensor] = None, ) -> tuple[torch.Tensor, torch.Tensor]: + # M3 produces contiguous FP32 logits and bias. Keep the eager path for + # alternate dtype, device, shape, or layout contracts because the native + # op consumes raw contiguous CUDA buffers and emits FP32 weights. + if (router_logits.is_cuda and router_logits.dtype == torch.float32 + and router_logits.ndim == 2 and router_logits.is_contiguous() + and self.output_dtype == torch.float32): + routing_bias = self.e_score_correction_bias + if (routing_bias.is_cuda + and routing_bias.device == router_logits.device + and routing_bias.dtype == torch.float32 + and routing_bias.ndim == 1 + and routing_bias.numel() == router_logits.shape[1] + and routing_bias.is_contiguous()): + return self.routing_impl.apply(router_logits, routing_bias) + # ``moe_scheduler.MoEScheduler`` calls ``routing_method.apply(router_logits, # input_ids)`` positionally; mirror the parent's signature so the # MiniMax-M3 routing accepts (but ignores) the optional ``input_ids`` diff --git a/tensorrt_llm/_torch/modules/fused_ops/fused_qk_norm_rope_gate.py b/tensorrt_llm/_torch/modules/fused_ops/fused_qk_norm_rope_gate.py new file mode 100644 index 000000000000..d1f51326a3eb --- /dev/null +++ b/tensorrt_llm/_torch/modules/fused_ops/fused_qk_norm_rope_gate.py @@ -0,0 +1,348 @@ +# Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/kernels/ops/attention/fused_qk_rmsnorm_rope_gate.py +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fused Qwen3.5 full-attention preprocessing and output gating. + +The projection produces interleaved ``[q0, gate0, q1, gate1, ..., K, V]``. +The preprocessing kernel reads that layout directly and writes packed +``[Q, K, V]`` while applying per-head Gemma RMSNorm and NeoX RoPE to Q/K. +Gate deinterleave is fused into the same pass. Keeping the checkpoint layout +unchanged avoids coupling the optimization to weight loading, quantization, +LoRA, or attention-backend fallback behavior. +""" + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _fused_qkv_gemma_rmsnorm_rope_gate_kernel( + qkv_ptr, + qkv_out_ptr, + gate_out_ptr, + q_weight_ptr, + k_weight_ptr, + cos_sin_ptr, + positions_ptr, + num_tokens, + max_positions, + qkv_stride_token, + qkv_out_stride_token, + gate_out_stride_token, + cos_sin_stride_position, + num_q_heads: tl.constexpr, + num_kv_heads: tl.constexpr, + head_dim: tl.constexpr, + rotary_dim: tl.constexpr, + half_rotary: tl.constexpr, + head_block: tl.constexpr, + rotary_block: tl.constexpr, + eps: tl.constexpr, + output_fp16: tl.constexpr, + has_pass_through: tl.constexpr, + use_mrope: tl.constexpr, + mrope_section1: tl.constexpr, + mrope_section2: tl.constexpr, +): + token = tl.program_id(0).to(tl.int64) + head = tl.program_id(1) + q_size = num_q_heads * head_dim + kv_size = num_kv_heads * head_dim + output_dtype = tl.float16 if output_fp16 else tl.bfloat16 + + if head < num_q_heads + num_kv_heads: + is_k = head >= num_q_heads + local_head = tl.where(is_k, head - num_q_heads, head) + if is_k: + input_base = qkv_ptr + token * qkv_stride_token + 2 * q_size + local_head * head_dim + output_base = ( + qkv_out_ptr + token * qkv_out_stride_token + q_size + local_head * head_dim + ) + weight_ptr = k_weight_ptr + else: + input_base = qkv_ptr + token * qkv_stride_token + local_head * 2 * head_dim + output_base = qkv_out_ptr + token * qkv_out_stride_token + local_head * head_dim + weight_ptr = q_weight_ptr + + head_offsets = tl.arange(0, head_block) + head_mask = head_offsets < head_dim + x = tl.load(input_base + head_offsets, mask=head_mask, other=0.0).to(tl.float32) + weight = tl.load(weight_ptr + head_offsets, mask=head_mask, other=0.0).to(tl.float32) + inverse_rms = tl.rsqrt(tl.sum(x * x, axis=0) / head_dim + eps) + normalized = (x * inverse_rms * (weight + 1.0)).to(output_dtype).to(tl.float32) + + if has_pass_through: + pass_mask = head_mask & (head_offsets >= rotary_dim) + tl.store(output_base + head_offsets, normalized, mask=pass_mask) + + rotary_offsets = tl.arange(0, rotary_block) + rotary_mask = rotary_offsets < half_rotary + x_first = tl.load(input_base + rotary_offsets, mask=rotary_mask, other=0.0).to(tl.float32) + x_second = tl.load( + input_base + half_rotary + rotary_offsets, mask=rotary_mask, other=0.0 + ).to(tl.float32) + weight_first = tl.load(weight_ptr + rotary_offsets, mask=rotary_mask, other=0.0).to( + tl.float32 + ) + weight_second = tl.load( + weight_ptr + half_rotary + rotary_offsets, mask=rotary_mask, other=0.0 + ).to(tl.float32) + x_first = (x_first * inverse_rms * (weight_first + 1.0)).to(output_dtype).to(tl.float32) + x_second = (x_second * inverse_rms * (weight_second + 1.0)).to(output_dtype).to(tl.float32) + + if use_mrope: + section = tl.where( + (rotary_offsets % 3 == 1) & (rotary_offsets < mrope_section1 * 3), + 1, + tl.where( + (rotary_offsets % 3 == 2) & (rotary_offsets < mrope_section2 * 3), + 2, + 0, + ), + ) + position = tl.load( + positions_ptr + section * num_tokens + token, + mask=rotary_mask, + other=0, + ).to(tl.int64) + else: + position = tl.load(positions_ptr + token).to(tl.int64) + position_is_valid = (position >= 0) & (position < max_positions) + tl.device_assert(position_is_valid, "position is outside the RoPE table") + safe_position = tl.where(position_is_valid, position, 0) + cos_sin_base = cos_sin_ptr + safe_position * cos_sin_stride_position + cos = tl.load(cos_sin_base + rotary_offsets, mask=rotary_mask, other=0.0).to(tl.float32) + sin = tl.load(cos_sin_base + half_rotary + rotary_offsets, mask=rotary_mask, other=0.0).to( + tl.float32 + ) + cos = tl.where(position_is_valid, cos, float("nan")) + sin = tl.where(position_is_valid, sin, float("nan")) + tl.store( + output_base + rotary_offsets, + x_first * cos - x_second * sin, + mask=rotary_mask, + ) + tl.store( + output_base + half_rotary + rotary_offsets, + x_second * cos + x_first * sin, + mask=rotary_mask, + ) + + if not is_k: + gate_input = input_base + head_dim + gate_output = gate_out_ptr + token * gate_out_stride_token + local_head * head_dim + gate = tl.load(gate_input + head_offsets, mask=head_mask, other=0.0) + tl.store(gate_output + head_offsets, gate, mask=head_mask) + else: + local_head = head - num_q_heads - num_kv_heads + head_offsets = tl.arange(0, head_block) + head_mask = head_offsets < head_dim + input_base = ( + qkv_ptr + token * qkv_stride_token + 2 * q_size + kv_size + local_head * head_dim + ) + output_base = ( + qkv_out_ptr + token * qkv_out_stride_token + q_size + kv_size + local_head * head_dim + ) + value = tl.load(input_base + head_offsets, mask=head_mask, other=0.0) + tl.store(output_base + head_offsets, value, mask=head_mask) + + +def fused_qkv_gemma_rmsnorm_rope_gate( + qkv: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin: torch.Tensor, + positions: torch.Tensor, + eps: float, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + rotary_dim: int, + mrope_section: Optional[Tuple[int, int, int]] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Prepare packed QKV and gate with one Triton kernel. + + Args: + qkv: ``[num_tokens, 2 * q_size + 2 * kv_size]`` BF16/FP16 projection + output in per-head interleaved Q/G layout. + q_weight: ``[head_dim]`` raw Gemma RMSNorm Q weights. + k_weight: ``[head_dim]`` raw Gemma RMSNorm K weights. + cos_sin: ``[max_positions, 2, rotary_dim // 2]`` FP32 NeoX table. + positions: Flattenable ``[num_tokens]`` plain-RoPE positions, or + ``[3, ..., num_tokens]`` interleaved-MRoPE positions. + eps: RMSNorm epsilon. + num_q_heads: Local query-head count. + num_kv_heads: Local key/value-head count. + head_dim: Per-head Q/K/V dimension. + rotary_dim: Prefix dimension receiving NeoX RoPE. + mrope_section: Temporal/height/width rotary-half dimensions. ``None`` + selects plain RoPE; a tuple selects Qwen-style interleaved MRoPE. + + Returns: + Packed QKV ``[num_tokens, q_size + 2 * kv_size]`` and gate + ``[num_tokens, num_q_heads, head_dim]``. + """ + assert qkv.dim() == 2 and qkv.stride(-1) == 1 + assert qkv.dtype in (torch.bfloat16, torch.float16) + assert q_weight.shape == (head_dim,) and k_weight.shape == (head_dim,) + assert q_weight.device == qkv.device and k_weight.device == qkv.device + assert cos_sin.dtype == torch.float32 and cos_sin.is_contiguous() + assert cos_sin.device == qkv.device and positions.device == qkv.device + assert rotary_dim > 0 and rotary_dim <= head_dim and rotary_dim % 2 == 0 + assert cos_sin.shape == (cos_sin.shape[0], 2, rotary_dim // 2) + assert cos_sin.shape[0] > 0 + + num_tokens = qkv.shape[0] + use_mrope = mrope_section is not None + if use_mrope: + assert len(mrope_section) == 3 + assert sum(mrope_section) == rotary_dim // 2 + assert positions.numel() == 3 * num_tokens + positions = positions.reshape(3, num_tokens) + else: + assert positions.numel() == num_tokens + positions = positions.reshape(-1) + assert positions.is_contiguous() + assert positions.dtype in (torch.int32, torch.int64) + + q_size = num_q_heads * head_dim + kv_size = num_kv_heads * head_dim + assert qkv.shape[1] == 2 * q_size + 2 * kv_size + + qkv_out = torch.empty((num_tokens, q_size + 2 * kv_size), dtype=qkv.dtype, device=qkv.device) + gate_out = torch.empty((num_tokens, num_q_heads, head_dim), dtype=qkv.dtype, device=qkv.device) + if num_tokens == 0: + return qkv_out, gate_out + + half_rotary = rotary_dim // 2 + head_block = triton.next_power_of_2(head_dim) + rotary_block = triton.next_power_of_2(half_rotary) + grid = (num_tokens, num_q_heads + 2 * num_kv_heads) + _fused_qkv_gemma_rmsnorm_rope_gate_kernel[grid]( + qkv, + qkv_out, + gate_out, + q_weight, + k_weight, + cos_sin, + positions, + num_tokens, + cos_sin.shape[0], + qkv.stride(0), + qkv_out.stride(0), + gate_out.stride(0), + cos_sin.stride(0), + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + rotary_dim=rotary_dim, + half_rotary=half_rotary, + head_block=head_block, + rotary_block=rotary_block, + eps=eps, + output_fp16=qkv.dtype == torch.float16, + has_pass_through=rotary_dim < head_dim, + use_mrope=use_mrope, + mrope_section1=mrope_section[1] if use_mrope else 0, + mrope_section2=mrope_section[2] if use_mrope else 0, + num_warps=4, + ) + return qkv_out, gate_out + + +@triton.jit +def _fused_sigmoid_mul_kernel( + output_ptr, + attention_ptr, + gate_ptr, + output_stride_token, + attention_stride_token, + gate_stride_token, + gate_stride_head, + hidden_size: tl.constexpr, + head_dim: tl.constexpr, + block_size: tl.constexpr, +): + token = tl.program_id(0).to(tl.int64) + block = tl.program_id(1) + offsets = block * block_size + tl.arange(0, block_size) + mask = offsets < hidden_size + head = offsets // head_dim + dim = offsets - head * head_dim + + attention = tl.load( + attention_ptr + token * attention_stride_token + offsets, + mask=mask, + other=0.0, + ).to(tl.float32) + gate = tl.load( + gate_ptr + token * gate_stride_token + head * gate_stride_head + dim, + mask=mask, + other=0.0, + ).to(tl.float32) + tl.store( + output_ptr + token * output_stride_token + offsets, + attention * tl.sigmoid(gate), + mask=mask, + ) + + +def fused_sigmoid_mul( + attention_output: torch.Tensor, + gate: torch.Tensor, + *, + inplace: bool = False, +) -> torch.Tensor: + """Compute ``attention_output * sigmoid(gate)`` in one kernel.""" + assert attention_output.dim() == 2 and attention_output.stride(-1) == 1 + num_tokens, hidden_size = attention_output.shape + if gate.dim() == 3: + assert gate.shape[0] == num_tokens + assert gate.shape[1] * gate.shape[2] == hidden_size + assert gate.stride(-1) == 1 + head_dim = gate.shape[2] + gate_stride_token = gate.stride(0) + gate_stride_head = gate.stride(1) + else: + assert gate.dim() == 2 and gate.shape == attention_output.shape + assert gate.stride(-1) == 1 + head_dim = hidden_size + gate_stride_token = gate.stride(0) + gate_stride_head = hidden_size + + output = attention_output if inplace else torch.empty_like(attention_output) + if num_tokens == 0: + return output + + max_block_size = 1024 if num_tokens < 1024 else 2048 + block_size = min(triton.next_power_of_2(hidden_size), max_block_size) + grid = (num_tokens, triton.cdiv(hidden_size, block_size)) + _fused_sigmoid_mul_kernel[grid]( + output, + attention_output, + gate, + output.stride(0), + attention_output.stride(0), + gate_stride_token, + gate_stride_head, + hidden_size=hidden_size, + head_dim=head_dim, + block_size=block_size, + num_warps=4, + ) + return output diff --git a/tensorrt_llm/_torch/modules/gated_mlp.py b/tensorrt_llm/_torch/modules/gated_mlp.py index fb7f43f689b1..d5787ccd0044 100644 --- a/tensorrt_llm/_torch/modules/gated_mlp.py +++ b/tensorrt_llm/_torch/modules/gated_mlp.py @@ -35,6 +35,8 @@ def __init__( use_custom_cublas_mm: bool = False, is_shared_expert: bool = False, swiglu_limit: Optional[float] = None, + swiglu_alpha: Optional[float] = None, + swiglu_beta: Optional[float] = None, ): super().__init__() @@ -45,6 +47,12 @@ def __init__( self.use_cute_dsl_blockscaling_mm = use_cute_dsl_blockscaling_mm self.swiglu_limit = float( swiglu_limit) if swiglu_limit is not None else None + # SwiGLU-OAI shape parameters, left None for plain SwiGLU, where the + # kernel defaults to alpha=1.0 and beta=0.0. + self.swiglu_alpha = float( + swiglu_alpha) if swiglu_alpha is not None else None + self.swiglu_beta = float( + swiglu_beta) if swiglu_beta is not None else None config = config or ModelConfig() use_cute_dsl_bf16_gemm = getattr(config, "use_cute_dsl_bf16_gemm", @@ -67,13 +75,33 @@ def __init__( # Calculate local intermediate size after tensor parallel sharding tp_size = mapping.tp_size - local_intermediate_size = self.intermediate_size // tp_size + local_intermediate_start = Linear._calc_shard(self.intermediate_size, + mapping.tp_size, + mapping.tp_rank) + local_intermediate_end = Linear._calc_shard(self.intermediate_size, + mapping.tp_size, + mapping.tp_rank + 1) + local_intermediate_size = local_intermediate_end - local_intermediate_start + + self._uneven_tp_blocks_lora = (mapping.tp_size > 1 + and self.intermediate_size % + mapping.tp_size != 0) + + # gateup_shard_indices_mapping is the local offset and size for each sub-weight + # in this rank's concatenated (gate || up) buffer. + # override_tp_sharding is the absolute range of the global weight from which + # this rank pulls each sub-weight. gateup_shard_indices_mapping = { 'gate': (0, local_intermediate_size), 'up': (local_intermediate_size, local_intermediate_size), } + override_tp_sharding = { + 'gate': (local_intermediate_start, local_intermediate_end), + 'up': (local_intermediate_start, local_intermediate_end), + } + self.gate_up_proj = Linear( self.hidden_size, self.intermediate_size * 2, @@ -93,6 +121,7 @@ def __init__( disable_deep_gemm=disable_deep_gemm, fused_weight_shard_indices_mapping=gateup_shard_indices_mapping, use_custom_cublas_mm=use_custom_cublas_mm, + override_tp_sharding=override_tp_sharding, ) if is_shared_expert: @@ -146,14 +175,22 @@ def _apply_activation(self, x, *, has_lora: bool = False): logger.warning( f"GatedMLP._apply_activation: LoRA path active; forcing non-FP8 activation dtype bf16/fp16, layer_idx={self.layer_idx}" ) - return swiglu(x, swiglu_limit=self.swiglu_limit) + return swiglu(x, + swiglu_limit=self.swiglu_limit, + swiglu_alpha=self.swiglu_alpha, + swiglu_beta=self.swiglu_beta) else: return swiglu(x, quant_scale=self.down_proj.input_scale, quant_type=torch.float8_e4m3fn, - swiglu_limit=self.swiglu_limit) + swiglu_limit=self.swiglu_limit, + swiglu_alpha=self.swiglu_alpha, + swiglu_beta=self.swiglu_beta) else: - return swiglu(x, swiglu_limit=self.swiglu_limit) + return swiglu(x, + swiglu_limit=self.swiglu_limit, + swiglu_alpha=self.swiglu_alpha, + swiglu_beta=self.swiglu_beta) elif callable(self.activation): return self.activation(x) elif self.activation is None: @@ -163,17 +200,26 @@ def _apply_activation(self, x, *, has_lora: bool = False): f"Activation {self.activation} not yet implemented for fused GatedMLP" ) + def _is_plain_swiglu(self): + """True when the SwiGLU has no alpha gain or (up + beta) offset. + + The fused CuteDSL GEMM+SwiGLU epilogue implements plain silu(gate) * up + only; a swigluoai-style alpha/beta must use the Triton swiglu kernel. + """ + return ((self.swiglu_alpha is None or self.swiglu_alpha == 1.0) + and (self.swiglu_beta is None or self.swiglu_beta == 0.0)) + def _can_fuse_gate_up_swiglu(self): """Check if fused GEMM + SwiGLU path is available. Returns True when all conditions are met: - CuteDSL blockscaling mode is enabled (implies Blackwell + CuteDSL) - - Activation is SwiGLU (F.silu) + - Activation is plain SwiGLU (F.silu), see _is_plain_swiglu - gate_up_proj uses NVFP4 quantization - gate_up_proj has no bias (bias not supported in fused kernel) """ return (self.use_cute_dsl_blockscaling_mm and self.activation == F.silu - and self.gate_up_proj.has_nvfp4 + and self._is_plain_swiglu() and self.gate_up_proj.has_nvfp4 and not self.gate_up_proj.has_bias) def _can_fuse_gate_up_swiglu_fp4out(self): @@ -306,6 +352,10 @@ def forward_lora( ) -> torch.Tensor: assert lora_params is not None assert self.layer_idx is not None, "layer_idx is required for lora" + if self._uneven_tp_blocks_lora: + raise NotImplementedError( + "LoRA is not supported with uneven TP for GatedMLP " + "(intermediate_size not divisible by tp_size).") h1 = self.gate_up_proj(x) diff --git a/tensorrt_llm/_torch/modules/layer_norm.py b/tensorrt_llm/_torch/modules/layer_norm.py index 811067952c52..8b52c52a0354 100644 --- a/tensorrt_llm/_torch/modules/layer_norm.py +++ b/tensorrt_llm/_torch/modules/layer_norm.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,6 +34,9 @@ class LayerNorm(nn.Module): device: Optional device for parameters. has_weights: Whether to include learnable weight parameters. has_bias: Whether to include learnable bias parameters. + residual_in_fp32: Whether to accumulate the residual in FP32 before + normalization. If false, preserve the input dtype for the residual + addition and convert the result to FP32 for normalization. """ def __init__( @@ -45,6 +48,7 @@ def __init__( device: Optional[torch.device] = None, has_weights: bool = True, has_bias: bool = True, + residual_in_fp32: bool = True, ): super().__init__() if has_weights: @@ -66,6 +70,7 @@ def __init__( device=device), persistent=False) self.variance_epsilon = eps + self.residual_in_fp32 = residual_in_fp32 @maybe_compile(dynamic=True) def forward( @@ -84,16 +89,23 @@ def forward( """ input_dtype = hidden_states.dtype - hidden_states = hidden_states.to(torch.float32) if isinstance(residual, torch.Tensor): - hidden_states = hidden_states + residual.to(torch.float32) + if self.residual_in_fp32: + hidden_states = hidden_states.to(torch.float32) + residual.to( + torch.float32) + else: + hidden_states = (hidden_states + residual).to(torch.float32) residual = hidden_states.to(input_dtype) + else: + hidden_states = hidden_states.to(torch.float32) + # Eager torch.layer_norm needs weight/bias in the fp32 compute dtype; + # torch.compile does this promotion implicitly (bitwise-identical). hidden_states = nn.functional.layer_norm( hidden_states, (hidden_states.shape[-1], ), - weight=self.weight, - bias=self.bias, + weight=self.weight.to(torch.float32), + bias=self.bias.to(torch.float32), eps=self.variance_epsilon, ).to(input_dtype) diff --git a/tensorrt_llm/_torch/modules/linear.py b/tensorrt_llm/_torch/modules/linear.py index 0d77f4c57fe4..744661d415e4 100644 --- a/tensorrt_llm/_torch/modules/linear.py +++ b/tensorrt_llm/_torch/modules/linear.py @@ -107,6 +107,12 @@ def load_weight_shard( device: torch.device = torch.device('cpu'), return_slice_indices: bool = False, ) -> torch.Tensor: + """Legacy weight shard helper using ceil-divide sharding. + + `Linear.load_shard` is preferred — it respects uneven-TP overrides, fused + QKV/gate-up sharding dicts, and quant-specific scale/packing semantics that + this function does not. + """ # Skip device transfers on integrated GPUs to conserve shared memory if weight.device.type != device.type and is_device_integrated(): # For integrated GPU systems (e.g., DGX Spark), CPU and GPU share limited physical memory. @@ -178,7 +184,8 @@ def load_weights_vanilla_helper(module: Linear, weights: List[Dict], weight_transform=lambda x: x, bias_transform=lambda x: x, - allow_partial_loading: bool = False): + allow_partial_loading: bool = False, + elm_packing: int = 1): assert len(weights) == 1 if not allow_partial_loading: assert "weight" in weights[0] @@ -186,9 +193,10 @@ def load_weights_vanilla_helper(module: Linear, assert "bias" in weights[0] device = torch.device('cuda') - weight = load_weight_shard(weights[0]['weight'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "weight" in weights[0] else None + weight = module.load_shard(weights[0], + 'weight', + device=device, + elm_packing=elm_packing) if weight is not None: if module.has_weight_only_quant: @@ -203,9 +211,7 @@ def load_weights_vanilla_helper(module: Linear, copy_weight(module.weight, weight_transform(weight)) if module.bias is not None: - bias = load_weight_shard(weights[0]['bias'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "bias" in weights[0] else None + bias = module.load_shard(weights[0], 'bias', device=device) if bias is not None: copy_weight(module.bias, bias_transform(bias)) @@ -215,7 +221,8 @@ def load_weights_fused_qkv_helper( weights: List[Dict], weight_transform=lambda x: x, bias_transform=lambda x: x, - allow_partial_loading: bool = False + allow_partial_loading: bool = False, + elm_packing: int = 1, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: if not allow_partial_loading: assert all('weight' in weights[i] for i in range(3)) @@ -227,26 +234,27 @@ def load_weights_fused_qkv_helper( ) is not None, "Fused weight shard indices mapping is required in partial loading" device = torch.device('cuda') - q_weight = load_weight_shard(weights[0]['weight'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "weight" in weights[0] else None - k_weight = load_weight_shard(weights[1]['weight'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "weight" in weights[1] else None - v_weight = load_weight_shard(weights[2]['weight'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "weight" in weights[2] else None + q_weight = module.load_shard(weights[0], + 'weight', + device=device, + name='q', + elm_packing=elm_packing) + k_weight = module.load_shard(weights[1], + 'weight', + device=device, + name='k', + elm_packing=elm_packing) + v_weight = module.load_shard(weights[2], + 'weight', + device=device, + name='v', + elm_packing=elm_packing) if module.bias is not None: - q_bias = load_weight_shard(weights[0]['bias'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "bias" in weights[0] else None - k_bias = load_weight_shard(weights[1]['bias'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "bias" in weights[1] else None - v_bias = load_weight_shard(weights[2]['bias'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "bias" in weights[2] else None + q_bias = module.load_shard(weights[0], 'bias', device=device, name='q') + k_bias = module.load_shard(weights[1], 'bias', device=device, name='k') + v_bias = module.load_shard(weights[2], 'bias', device=device, name='v') + if not allow_partial_loading: copy_weight(module.bias, bias_transform(torch.cat((q_bias, k_bias, v_bias)))) @@ -264,11 +272,12 @@ def load_weights_fused_qkv_helper( def load_weights_fused_gate_up_helper( - module: Linear, - weights: List[Dict], - weight_transform=lambda x: x, - bias_transform=lambda x: x, - allow_partial_loading: bool = False + module: Linear, + weights: List[Dict], + weight_transform=lambda x: x, + bias_transform=lambda x: x, + allow_partial_loading: bool = False, + elm_packing: int = 1, ) -> tuple[torch.Tensor, torch.Tensor]: if not allow_partial_loading: assert all('weight' in weights[i] for i in range(2)) @@ -280,19 +289,26 @@ def load_weights_fused_gate_up_helper( ) is not None, "Fused weight shard indices mapping is required in partial loading" device = torch.device('cuda') - gate_weight = load_weight_shard(weights[0]['weight'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "weight" in weights[0] else None - up_weight = load_weight_shard(weights[1]['weight'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "weight" in weights[1] else None + gate_weight = module.load_shard(weights[0], + 'weight', + device=device, + name='gate', + elm_packing=elm_packing) + up_weight = module.load_shard(weights[1], + 'weight', + device=device, + name='up', + elm_packing=elm_packing) + if module.bias is not None: - gate_bias = load_weight_shard(weights[0]['bias'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "bias" in weights[0] else None - up_bias = load_weight_shard(weights[1]['bias'], module.tp_size, - module.tp_rank, module.tp_mode, - device) if "bias" in weights[1] else None + gate_bias = module.load_shard(weights[0], + 'bias', + device=device, + name='gate') + up_bias = module.load_shard(weights[1], + 'bias', + device=device, + name='up') if not allow_partial_loading: copy_weight(module.bias, bias_transform(torch.cat((gate_bias, up_bias)))) @@ -470,6 +486,12 @@ def pre_reload_weights(self, module: Linear) -> None: requires_grad=False) module.register_parameter(param_name, param) + def get_tp_alignment(self, + tp_mode: Optional[TensorParallelMode], + quant_config: Optional[QuantConfig] = None) -> int: + """ Alignment required for TP shard boundaries. """ + return 1 + class UnquantizedLinearMethod(LinearMethodBase): @@ -721,18 +743,12 @@ def load_weight_scales(self, shard_keys: list[str] = None): input_scales, weight_scales = {}, {} if shard_keys is None: - for w in weights: - if "input_scale" in w: - input_scales[None] = w["input_scale"][...].reshape([]) - if "weight_scale" in w: - weight_scales[None] = w["weight_scale"][...].reshape([]) - else: - for shard_key, w in zip(shard_keys, weights): - if "input_scale" in w: - input_scales[shard_key] = w["input_scale"][...].reshape([]) - if "weight_scale" in w: - weight_scales[shard_key] = w["weight_scale"][...].reshape( - []) + shard_keys = [None] + for shard_key, w in zip(shard_keys, weights): + if "input_scale" in w: + input_scales[shard_key] = w["input_scale"][...].reshape([]) + if "weight_scale" in w: + weight_scales[shard_key] = w["weight_scale"][...].reshape([]) return input_scales, weight_scales def load_weights_vanilla(self, @@ -1018,9 +1034,7 @@ def load_weights_vanilla(self, module, weights, allow_partial_loading=allow_partial_loading) scale_name = self._get_scale_name(weights) if scale_name in weights[0]: - weight_scale = load_weight_shard(weights[0][scale_name], - module.tp_size, module.tp_rank, - module.tp_mode) + weight_scale = module.load_shard(weights[0], scale_name) # compressed-tensors stores per-channel weight scales as [out, 1]; # the weight_scale buffer is 1-D [out] (ModelOpt/DS recipes store # it 1-D). Flatten so copy_ does not broadcast to [out, out]. @@ -1037,16 +1051,12 @@ def load_weights_fused_qkv_linear(self, allow_partial_loading: bool = False): super().load_weights_fused_qkv_linear( module, weights, allow_partial_loading=allow_partial_loading) + scale_name = self._get_scale_name(weights) - q_scale = load_weight_shard( - weights[0][scale_name], module.tp_size, module.tp_rank, - module.tp_mode) if scale_name in weights[0] else None - k_scale = load_weight_shard( - weights[1][scale_name], module.tp_size, module.tp_rank, - module.tp_mode) if scale_name in weights[1] else None - v_scale = load_weight_shard( - weights[2][scale_name], module.tp_size, module.tp_rank, - module.tp_mode) if scale_name in weights[2] else None + q_scale = module.load_shard(weights[0], scale_name, name='q') + k_scale = module.load_shard(weights[1], scale_name, name='k') + v_scale = module.load_shard(weights[2], scale_name, name='v') + for shard_key, scale in zip( module.fused_weight_shard_indices_mapping.keys(), [q_scale, k_scale, v_scale]): @@ -1066,13 +1076,11 @@ def load_weights_fused_gate_up_linear( allow_partial_loading: bool = False) -> None: super().load_weights_fused_gate_up_linear( module, weights, allow_partial_loading=allow_partial_loading) + scale_name = self._get_scale_name(weights) - gate_scale = load_weight_shard( - weights[0][scale_name], module.tp_size, module.tp_rank, - module.tp_mode) if scale_name in weights[0] else None - up_scale = load_weight_shard( - weights[1][scale_name], module.tp_size, module.tp_rank, - module.tp_mode) if scale_name in weights[1] else None + gate_scale = module.load_shard(weights[0], scale_name, name='gate') + up_scale = module.load_shard(weights[1], scale_name, name='up') + for shard_key, scale in zip( module.fused_weight_shard_indices_mapping.keys(), [gate_scale, up_scale]): @@ -1091,6 +1099,9 @@ class FP8BlockScalesLinearMethod(UnquantizedLinearMethod): # fp8_block_scaling_gemm does not support writing into an NCCL window buffer. supports_nccl_symmetric_memory_window_output: ClassVar[bool] = False + def get_tp_alignment(self, tp_mode, quant_config=None): + return 128 + def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): weight_shape = (out_features, in_features) @@ -1183,8 +1194,8 @@ def load_weights_vanilla(self, # modelopt fp8_pb_wo can have 2 extra singleton dimensions if full_weight_scale.dim() == 4: full_weight_scale = full_weight_scale.squeeze(1).squeeze(-1) - weight_scale = load_weight_shard(full_weight_scale, module.tp_size, - module.tp_rank, module.tp_mode) + + weight_scale = module.load_shard(full_weight_scale, scale_span=128) copy_weight(module.weight_scale, weight_scale) if "input_scale" in weights[0]: copy_weight(module.input_scale, weights[0]["input_scale"]) @@ -1228,8 +1239,9 @@ def load_weights_fused_qkv_linear( ] scales = [ - load_weight_shard(s, module.tp_size, module.tp_rank, module.tp_mode) - if s is not None else None for s in full_scales_squeezed + module.load_shard(s, scale_span=128, name=name) + if s is not None else None + for s, name in zip(full_scales_squeezed, ('q', 'k', 'v')) ] processed_mapping = self.remap_fused_shard_indices_by_divisible_factor( module.fused_weight_shard_indices_mapping, 128) @@ -1256,9 +1268,11 @@ def load_weights_fused_gate_up_linear( for s in full_scales ] scales = [ - load_weight_shard(s, module.tp_size, module.tp_rank, module.tp_mode) - if s is not None else None for s in full_scales_squeezed + module.load_shard(s, scale_span=128, name=name) + if s is not None else None + for s, name in zip(full_scales_squeezed, ('gate', 'up')) ] + processed_mapping = self.remap_fused_shard_indices_by_divisible_factor( module.fused_weight_shard_indices_mapping, 128) for shard_key, scale in zip(processed_mapping.keys(), scales): @@ -1269,11 +1283,40 @@ def load_weights_fused_gate_up_linear( def transform_weights(self, module: Linear) -> None: super().transform_weights(module) - if (is_sm_100f() and not (module.use_cute_dsl_blockscaling_mm - or module.disable_deep_gemm)) or \ - get_sm_version() == 120: + use_deep_gemm_layout = ( + is_sm_100f() + and not (module.use_cute_dsl_blockscaling_mm + or module.disable_deep_gemm)) or get_sm_version() == 120 + use_indexer_q_cutedsl_layout = (use_deep_gemm_layout and getattr( + module, "use_indexer_q_cutedsl_fusion", False)) + if use_deep_gemm_layout or use_indexer_q_cutedsl_layout: weight, weight_scale = resmooth_to_fp8_e8m0(module.weight, module.weight_scale) + + if use_indexer_q_cutedsl_layout: + # Native SM100 MXF8 MMA consumes one scale per 32 K values. + # The checkpoint/production quantization contract remains + # 128x128: expand each row block and repeat each K scale four + # times, then materialize CUTLASS/CuTe's 128x4 swizzle once at + # weight-load time. + n = weight.shape[0] + scale_cutedsl = weight_scale.repeat_interleave(128, dim=0)[:n] + scale_cutedsl = scale_cutedsl.repeat_interleave(4, dim=1) + scale_cutedsl = torch.ops.trtllm.block_scale_interleave( + scale_cutedsl.to(torch.float8_e8m0fnu).view(torch.uint8)) + module.register_buffer( + "indexer_q_weight_scale_cutedsl", + scale_cutedsl, + persistent=False, + ) + module.register_buffer( + "indexer_q_alpha_cutedsl", + torch.ones((1, ), dtype=torch.float32, + device=weight.device), + persistent=False, + ) + + if use_deep_gemm_layout: transformed_scale = transform_sf_into_required_layout( weight_scale, mn=weight.shape[0], @@ -1299,6 +1342,13 @@ class NVFP4LinearMethod(LinearMethodBase): # construction; LLM paths leave it False to avoid host overhead. use_tunable_quantize: bool = False + def get_tp_alignment(self, tp_mode, quant_config=None): + # 32-element alignment for both modes. ROW shards in_features which + # is packed 2:1, so 32 → 16 packed, meeting GEMM col_alignment=16. + # COLUMN must also be 32 because column output feeds row input, and + # row's packed weight K dimension needs 16-alignment. + return 32 + def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): module.scaling_vector_size = 16 @@ -1408,10 +1458,18 @@ def _input_prepare(self, module: Linear, input: torch.Tensor): def apply(self, module: Linear, input: torch.Tensor, bias: Optional[torch.Tensor]): # Handle multi-dimensional inputs (e.g., 3D: batch, seq, hidden). - # NVFP4 GEMM requires a 2D mat1; flatten here and unflatten the output below. + # GEMM requires 2D. Fp4QuantizedTensor from fused LayerNorm paths may + # arrive as 3D [B, S, D/8] — flatten fp4_tensor and restore after. original_shape = None - if not isinstance(input, - (tuple, Fp4QuantizedTensor)) and input.dim() > 2: + if isinstance(input, Fp4QuantizedTensor) and input.fp4_tensor.dim() > 2: + original_shape = input.fp4_tensor.shape + input = Fp4QuantizedTensor( + input.fp4_tensor.reshape(-1, input.fp4_tensor.shape[-1]), + input.scaling_factor, + input.is_sf_swizzled, + ) + elif not isinstance(input, + (tuple, Fp4QuantizedTensor)) and input.dim() > 2: original_shape = input.shape input = input.reshape(-1, input.shape[-1]) elif isinstance(input, @@ -1508,17 +1566,17 @@ def load_weight_scales(self, """ device = torch.device("cuda") + scale_span = 16 if module.tp_mode == TensorParallelMode.ROW else 1 # Per-shard weight_scale: load, TP-shard, store in tmp dict keyed by shard if shard_keys is not None: if not hasattr(module, "tmp_nvfp4_weight_scales"): module.tmp_nvfp4_weight_scales = {} for shard_key, w in zip(shard_keys, weights): if "weight_scale" in w: - ws = load_weight_shard(w["weight_scale"], - module.tp_size, - module.tp_rank, - module.tp_mode, - device=device).contiguous() + ws = module.load_shard(w["weight_scale"], + device=device, + scale_span=scale_span, + name=shard_key).contiguous() assert ws.dtype == torch.float8_e4m3fn module.tmp_nvfp4_weight_scales[shard_key] = ws.view( fp4_utils.float4_sf_dtype) @@ -1526,11 +1584,9 @@ def load_weight_scales(self, # Vanilla: single weight_scale, load + interleave directly w = weights[0] if "weight_scale" in w: - ws = load_weight_shard(w["weight_scale"], - module.tp_size, - module.tp_rank, - module.tp_mode, - device=device).contiguous() + ws = module.load_shard(w["weight_scale"], + device=device, + scale_span=scale_span).contiguous() ws = ws.view(fp4_utils.float4_sf_dtype) ws = torch.ops.trtllm.block_scale_interleave(ws) copy_weight(module.weight_scale, ws) @@ -1626,9 +1682,12 @@ def load_weights_vanilla(self, module: Linear, weights: List[Dict], allow_partial_loading: bool = False) -> None: + + elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 load_weights_vanilla_helper(module, weights, - allow_partial_loading=allow_partial_loading) + allow_partial_loading=allow_partial_loading, + elm_packing=elm_packing) # Load scales (vanilla = no shard_keys) self.load_weight_scales(module, weights, shard_keys=None) @@ -1636,13 +1695,14 @@ def load_weights_vanilla(self, # Load pre_quant_scale if it exists (for NVFP4_AWQ) if "pre_quant_scale" in weights[0]: device = module.weight.device - pre_quant_scale = load_weight_shard( + # scale_span is flipped because flip_tp=True + act_scale_span = 1 if module.tp_mode == TensorParallelMode.ROW else 16 + pre_quant_scale = module.load_shard( weights[0]["pre_quant_scale"], - module.tp_size, - module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - TensorParallelMode.flip(module.tp_mode), - device, + flip_tp=True, + device=device, + scale_span=act_scale_span, ) module.pre_quant_scale = Parameter( @@ -1656,8 +1716,12 @@ def load_weights_fused_qkv_linear( module: Linear, weights: List[Dict], allow_partial_loading: bool = False) -> None: + elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights, allow_partial_loading=allow_partial_loading) + module, + weights, + allow_partial_loading=allow_partial_loading, + elm_packing=elm_packing) weight_mode = module.weights_loading_config.weight_mode @@ -1729,8 +1793,12 @@ def load_weights_fused_gate_up_linear( module: Linear, weights: List[Dict], allow_partial_loading: bool = False) -> None: + elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights, allow_partial_loading=allow_partial_loading) + module, + weights, + allow_partial_loading=allow_partial_loading, + elm_packing=elm_packing) weight_mode = module.weights_loading_config.weight_mode device = torch.device("cuda") @@ -1749,14 +1817,13 @@ def load_weights_fused_gate_up_linear( # Load pre_quant_scale if it exists (for NVFP4_AWQ) # NOTE: pre_quant_scale is the same for gate and up since modelopt checks which layer shared the same input if "pre_quant_scale" in weights[0]: - device = module.weight.device - pre_quant_scale = load_weight_shard( + act_scale_span = 1 if module.tp_mode == TensorParallelMode.ROW else 16 + pre_quant_scale = module.load_shard( weights[0]["pre_quant_scale"], - module.tp_size, - module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - TensorParallelMode.flip(module.tp_mode), - device, + flip_tp=True, + device=module.weight.device, + scale_span=act_scale_span, ) module.pre_quant_scale = Parameter( @@ -1984,6 +2051,10 @@ def apply(self, module: Linear, input: torch.Tensor, class W4A8NVFP4FP8LinearMethod(LinearMethodBase): + def get_tp_alignment(self, tp_mode, quant_config=None): + # Same as NVFP4: 32-element alignment for both modes. + return 32 + def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): module.epilogue_tile_m = 128 @@ -2052,10 +2123,9 @@ def apply(self, module: Linear, input: torch.Tensor, def load_weight_scales( self, + module: Linear, weights: List[Dict], - tp_size: int = 1, - tp_rank: int = 0, - tp_mode: Optional[TensorParallelMode] = None, + shard_keys: Optional[List[str]] = None, ): # For concatenated weights (qkv_proj / up_gate_proj), the global scaling factors and input scaling factors should be shared. input_scale = None @@ -2063,6 +2133,27 @@ def load_weight_scales( weight_scale = [] device = torch.device("cuda") + scale_span = 32 if module.tp_mode == TensorParallelMode.ROW else 1 + + if shard_keys is not None: + for shard_key, w in zip(shard_keys, weights): + if "weight_scale" in w: + ws = module.load_shard(w["weight_scale"], + device=device, + scale_span=scale_span, + name=shard_key).contiguous() + assert ws.dtype == torch.float8_e4m3fn + weight_scale.append( + ws.view(dtype=fp4_utils.float4_sf_dtype)) + else: + for w in weights: + if "weight_scale" in w: + ws = module.load_shard(w["weight_scale"], + device=device, + scale_span=scale_span).contiguous() + assert ws.dtype == torch.float8_e4m3fn + weight_scale.append( + ws.view(dtype=fp4_utils.float4_sf_dtype)) for w in weights: if "input_scale" in w: @@ -2071,14 +2162,6 @@ def load_weight_scales( else: assert input_scale == w["input_scale"][ ...], "The input_scale should be same for all the weights" - if "weight_scale" in w: - ws = load_weight_shard(w["weight_scale"], - tp_size, - tp_rank, - tp_mode, - device=device).contiguous() - assert ws.dtype == torch.float8_e4m3fn - weight_scale.append(ws.view(dtype=fp4_utils.float4_sf_dtype)) if "weight_scale_2" in w: if weight_scale_2 is None: weight_scale_2 = w["weight_scale_2"][...] @@ -2094,16 +2177,16 @@ def load_weight_scales( return input_scale, weight_scale, weight_scale_2, alpha def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: + elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 # FIXME: this depends on the kernel internals load_weights_vanilla_helper( - module, weights, - lambda w: fp4_utils.shuffle_matrix_a(w, module.epilogue_tile_m)) + module, + weights, + lambda w: fp4_utils.shuffle_matrix_a(w, module.epilogue_tile_m), + elm_packing=elm_packing) input_scale, weight_scale, weight_scale_2, alpha = self.load_weight_scales( - weights, - tp_size=module.tp_size, - tp_rank=module.tp_rank, - tp_mode=module.tp_mode) + module, weights) assert len(weights) == 1 weight_scale = weight_scale[0] @@ -2119,14 +2202,13 @@ def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: def load_weights_fused_qkv_linear(self, module: Linear, weights: List[Dict]) -> None: + elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights) + module, weights, elm_packing=elm_packing) + weight_mode = module.weights_loading_config.weight_mode input_scale, weight_scales, weight_scale_2, alpha = self.load_weight_scales( - weights, - tp_size=module.tp_size, - tp_rank=module.tp_rank, - tp_mode=module.tp_mode) + module, weights, shard_keys=weight_mode.shard_keys) # Swizzle weight scales after concatenation weight_scale = torch.cat(weight_scales, 0) # Shuffle and Swizzle weight scale @@ -2146,18 +2228,17 @@ def load_weights_fused_qkv_linear(self, module: Linear, def load_weights_fused_gate_up_linear(self, module: Linear, weights: List[Dict]) -> None: + elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights) + module, weights, elm_packing=elm_packing) fused_weight = torch.cat((gate_weight, up_weight)) fused_weight = fp4_utils.shuffle_matrix_a(fused_weight, module.epilogue_tile_m) copy_weight(module.weight, fused_weight) + weight_mode = module.weights_loading_config.weight_mode input_scale, weight_scales, weight_scale_2, alpha = self.load_weight_scales( - weights, - tp_size=module.tp_size, - tp_rank=module.tp_rank, - tp_mode=module.tp_mode) + module, weights, shard_keys=weight_mode.shard_keys) # Swizzle weight scales after concatenation weight_scale = torch.cat(weight_scales, 0) # Shuffle and Swizzle weight scale @@ -2173,6 +2254,10 @@ def load_weights_fused_gate_up_linear(self, module: Linear, class W4A8MXFP4FP8LinearMethod(LinearMethodBase): + def get_tp_alignment(self, tp_mode, quant_config=None): + # Same as NVFP4: 32-element alignment for both modes. + return 32 + def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): module.scaling_vector_size = 32 @@ -2219,32 +2304,30 @@ def apply(self, module: Linear, input: torch.Tensor, return output def load_weight_scales(self, + module: Linear, weights: List[Dict], - tp_size: int = 1, - tp_rank: int = 0, - tp_mode: Optional[TensorParallelMode] = None): - # For concatenated weights (qkv_proj / up_gate_proj), the global scaling factors and input scaling factors should be shared. + shard_keys: Optional[List[str]] = None): weight_scale = [] device = torch.device("cuda") - for w in weights: + scale_span = 32 if module.tp_mode == TensorParallelMode.ROW else 1 + + if shard_keys is None: + shard_keys = [None] + for shard_key, w in zip(shard_keys, weights): if "weight_scale" in w: - ws = load_weight_shard(w["weight_scale"], - tp_size, - tp_rank, - tp_mode, - device=device).contiguous() - # Should be E8M0 for MXFP4 + ws = module.load_shard(w["weight_scale"], + device=device, + scale_span=scale_span, + name=shard_key).contiguous() assert ws.dtype == torch.uint8 weight_scale.append(ws.view(fp4_utils.float4_sf_dtype)) return weight_scale def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: - load_weights_vanilla_helper(module, weights) + elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 + load_weights_vanilla_helper(module, weights, elm_packing=elm_packing) - weight_scale = self.load_weight_scales(weights, - tp_size=module.tp_size, - tp_rank=module.tp_rank, - tp_mode=module.tp_mode) + weight_scale = self.load_weight_scales(module, weights) assert len(weights) == 1 weight_scale = weight_scale[0] # Swizzle weight scale @@ -2253,31 +2336,30 @@ def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: def load_weights_fused_qkv_linear(self, module: Linear, weights: List[Dict]) -> None: + elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights) + module, weights, elm_packing=elm_packing) fused_weight = torch.cat((q_weight, k_weight, v_weight)) copy_weight(module.weight, fused_weight) - weight_scale = self.load_weight_scales(weights, - tp_size=module.tp_size, - tp_rank=module.tp_rank, - tp_mode=module.tp_mode) + weight_mode = module.weights_loading_config.weight_mode + weight_scale = self.load_weight_scales( + module, weights, shard_keys=weight_mode.shard_keys) weight_scale = torch.cat(weight_scale, 0) weight_scale = torch.ops.trtllm.block_scale_interleave(weight_scale) copy_weight(module.weight_scale, weight_scale) def load_weights_fused_gate_up_linear(self, module: Linear, weights: List[Dict]) -> None: + elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights) + module, weights, elm_packing=elm_packing) fused_weight = torch.cat((gate_weight, up_weight)) copy_weight(module.weight, fused_weight) - weight_scale = self.load_weight_scales(weights, - tp_size=module.tp_size, - tp_rank=module.tp_rank, - tp_mode=module.tp_mode) - # Swizzle weight scales after concatenation + weight_mode = module.weights_loading_config.weight_mode + weight_scale = self.load_weight_scales( + module, weights, shard_keys=weight_mode.shard_keys) weight_scale = torch.cat(weight_scale, 0) weight_scale = torch.ops.trtllm.block_scale_interleave(weight_scale) copy_weight(module.weight_scale, weight_scale) @@ -2285,6 +2367,17 @@ def load_weights_fused_gate_up_linear(self, module: Linear, class WeightOnlyQuantLinearMethod(LinearMethodBase): + def get_tp_alignment(self, tp_mode, quant_config=None): + # preprocess_weights_for_mixed_gemm requires: + # - ROW (in_features): % B_ROWS_PER_MMA (32 for INT4, 16 for INT8) + # - COLUMN (out_features): % MMA_SHAPE_N (8) + # COLUMN must also satisfy ROW of next layer in a column->row pipeline. + if quant_config is not None and quant_config.layer_quant_mode.is_int4_weight_only( + ): + return 32 + else: + return 16 + def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype) -> None: @@ -2323,69 +2416,61 @@ def apply(self, module: Linear, input: torch.Tensor, def load_weight_scales( self, + module: Linear, weights: List[Dict], - tp_size: int = 1, - tp_rank: int = 0, - tp_mode: Optional[TensorParallelMode] = None) -> List[torch.Tensor]: + shard_keys: Optional[List[Optional[str]]] = None + ) -> List[torch.Tensor]: device = torch.device("cuda") - q_weight_scale = load_weight_shard(weights[0]['weight_scale'], - tp_size, - tp_rank, - tp_mode, - device=device) - k_weight_scale = load_weight_shard(weights[1]['weight_scale'], - tp_size, - tp_rank, - tp_mode, - device=device) - v_weight_scale = load_weight_shard(weights[2]['weight_scale'], - tp_size, - tp_rank, - tp_mode, - device=device) - weight_scales = [q_weight_scale, k_weight_scale, v_weight_scale] - + if shard_keys is None: + shard_keys = [None] + weight_scales = [] + for shard_key, w in zip(shard_keys, weights): + ws = module.load_shard(w, + 'weight_scale', + device=device, + name=shard_key) + weight_scales.append(ws) return weight_scales def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: - load_weights_vanilla_helper(module, weights) - - device = torch.device('cuda') - weight_scale = load_weight_shard(weights[0]['weight_scale'], - module.tp_size, module.tp_rank, - module.tp_mode, device) + weight_dtype, weight_id = get_weight_dtype_and_id(module) + # INT4 checkpoint tensors are packed 2:1 along the output dimension + # before preprocessing, so COLUMN logical shard boundaries must be + # converted to packed coordinates. ROW shards the unpacked K dimension. + elm_packing = weight_id if module.tp_mode == TensorParallelMode.COLUMN else 1 + load_weights_vanilla_helper(module, weights, elm_packing=elm_packing) - copy_weight(module.weight_scale, weight_scale) + weight_scales = self.load_weight_scales(module, weights) + assert len(weight_scales) == 1 + copy_weight(module.weight_scale, weight_scales[0]) def load_weights_fused_qkv_linear(self, module: Linear, weights: List[Dict]) -> None: + weight_dtype, weight_id = get_weight_dtype_and_id(module) + elm_packing = weight_id if module.tp_mode == TensorParallelMode.COLUMN else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights) + module, weights, elm_packing=elm_packing) fused_weight = torch.cat((q_weight, k_weight, v_weight)) - weight_dtype, _ = get_weight_dtype_and_id(module) fused_weight = preprocess_weights_for_mixed_gemm( fused_weight.to(torch.int8).T.contiguous().cpu(), weight_dtype, torch.float16).cuda().contiguous() copy_weight(module.weight, fused_weight) - weight_scales = self.load_weight_scales(weights, - tp_size=module.tp_size, - tp_rank=module.tp_rank, - tp_mode=module.tp_mode) - - # Create concatenated weight scale tensor + weight_mode = module.weights_loading_config.weight_mode + weight_scales = self.load_weight_scales( + module, weights, shard_keys=weight_mode.shard_keys) cat_weight_scale = torch.cat(weight_scales, dim=0) copy_weight(module.weight_scale, cat_weight_scale) def load_weights_fused_gate_up_linear(self, module: Linear, weights: List[Dict]) -> None: - device = torch.device('cuda') - weight_dtype, _ = get_weight_dtype_and_id(module) + weight_dtype, weight_id = get_weight_dtype_and_id(module) + elm_packing = weight_id if module.tp_mode == TensorParallelMode.COLUMN else 1 gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights) + module, weights, elm_packing=elm_packing) fused_weight = torch.cat((gate_weight, up_weight)) @@ -2395,18 +2480,22 @@ def load_weights_fused_gate_up_linear(self, module: Linear, copy_weight(module.weight, fused_weight) - left_scale = load_weight_shard(weights[0]['weight_scale'], - module.tp_size, module.tp_rank, - module.tp_mode, device).contiguous() - right_scale = load_weight_shard(weights[1]['weight_scale'], - module.tp_size, module.tp_rank, - module.tp_mode, device).contiguous() - fused_scale = torch.cat([left_scale, right_scale], dim=0) + weight_mode = module.weights_loading_config.weight_mode + weight_scales = self.load_weight_scales( + module, weights, shard_keys=weight_mode.shard_keys) + fused_scale = torch.cat(weight_scales, dim=0) copy_weight(module.weight_scale, fused_scale) class W4A16_AWQ_LinearMethod(LinearMethodBase): + def get_tp_alignment(self, tp_mode, quant_config=None): + if quant_config is None: + return 1 + # ROW shards input groups directly. COLUMN shards output features, which + # feed the next row-parallel layer's grouped input dimension in MLPs. + return quant_config.group_size + def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype) -> None: @@ -2457,60 +2546,51 @@ def apply(self, module: Linear, input: torch.Tensor, def load_weight_scales( self, + module: Linear, weights: List[Dict], - tp_size: int = 1, - tp_rank: int = 0, - tp_mode: Optional[TensorParallelMode] = None) -> List[torch.Tensor]: + shard_keys: Optional[List[str]] = None) -> List[torch.Tensor]: device = torch.device("cuda") - q_weight_scale = load_weight_shard(weights[0]['weight_scale'], - tp_size, - tp_rank, - tp_mode, - device=device) - k_weight_scale = load_weight_shard(weights[1]['weight_scale'], - tp_size, - tp_rank, - tp_mode, - device=device) - v_weight_scale = load_weight_shard(weights[2]['weight_scale'], - tp_size, - tp_rank, - tp_mode, - device=device) - weight_scales = [q_weight_scale, k_weight_scale, v_weight_scale] - + scale_span = module.quant_config.group_size if module.tp_mode == TensorParallelMode.ROW else 1 + weight_scales = [] + if shard_keys is None: + shard_keys = [None] + for shard_key, w in zip(shard_keys, weights): + weight_scales.append( + module.load_shard(w, + 'weight_scale', + device=device, + name=shard_key, + scale_span=scale_span)) return weight_scales def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: - load_weights_vanilla_helper(module, weights) + elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 + load_weights_vanilla_helper(module, weights, elm_packing=elm_packing) # Use the same device as the weight tensor # as we register pre_quant_scale after sharded model weights are moved to respective gpus device = module.weight.device - pre_quant_scale = load_weight_shard( + pre_quant_scale = module.load_shard( weights[0]["pre_quant_scale"], - module.tp_size, - module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - TensorParallelMode.flip(module.tp_mode), - device, + flip_tp=True, + device=device, ) module.pre_quant_scale = Parameter( torch.ones((module.in_features, ), dtype=pre_quant_scale.dtype), requires_grad=False).to(device=device) - weight_scale = load_weight_shard(weights[0]['weight_scale'], - module.tp_size, module.tp_rank, - module.tp_mode, device) + weight_scale = self.load_weight_scales(module, weights)[0] copy_weight(module.pre_quant_scale, pre_quant_scale) copy_weight(module.weight_scale, weight_scale.T.contiguous()) def load_weights_fused_qkv_linear(self, module: Linear, weights: List[Dict]) -> None: + elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights) + module, weights, elm_packing=elm_packing) fused_weight = torch.cat((q_weight, k_weight, v_weight)) fused_weight = preprocess_weights_for_mixed_gemm( @@ -2519,8 +2599,9 @@ def load_weights_fused_qkv_linear(self, module: Linear, copy_weight(module.weight, fused_weight) - weight_scales = self.load_weight_scales(weights, module.tp_size, - module.tp_rank, module.tp_mode) + weight_mode = module.weights_loading_config.weight_mode + weight_scales = self.load_weight_scales( + module, weights, shard_keys=weight_mode.shard_keys) # Create concatenated weight scale tensor cat_weight_scale = torch.cat(weight_scales, dim=0).T.contiguous() @@ -2528,9 +2609,9 @@ def load_weights_fused_qkv_linear(self, module: Linear, def load_weights_fused_gate_up_linear(self, module: Linear, weights: List[Dict]) -> None: - device = torch.device('cuda') + elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights) + module, weights, elm_packing=elm_packing) fused_weight = torch.cat((gate_weight, up_weight)) fused_weight = preprocess_weights_for_mixed_gemm( @@ -2539,18 +2620,22 @@ def load_weights_fused_gate_up_linear(self, module: Linear, copy_weight(module.weight, fused_weight) - left_scale = load_weight_shard(weights[0]['weight_scale'], - module.tp_size, module.tp_rank, - module.tp_mode, device).contiguous() - right_scale = load_weight_shard(weights[1]['weight_scale'], - module.tp_size, module.tp_rank, - module.tp_mode, device).contiguous() - fused_scale = torch.cat([left_scale, right_scale], dim=0).T.contiguous() + weight_mode = module.weights_loading_config.weight_mode + weight_scales = self.load_weight_scales( + module, weights, shard_keys=weight_mode.shard_keys) + fused_scale = torch.cat(weight_scales, dim=0).T.contiguous() copy_weight(module.weight_scale, fused_scale) class W4A8_AWQ_LinearMethod(LinearMethodBase): + def get_tp_alignment(self, tp_mode, quant_config=None): + if quant_config is None: + return 1 + # Same grouped INT4 weight layout as W4A16_AWQ. ROW must keep input + # groups intact, and COLUMN output shards feed row-parallel grouped K. + return quant_config.group_size + def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): # Quantized weights @@ -2630,18 +2715,20 @@ def apply(self, module: Linear, input: torch.Tensor, return output def load_weight_scales_w4a8(self, + module: Linear, weights: List[Dict], - tp_size: int = 1, - tp_rank: int = 0, - tp_mode: Optional[TensorParallelMode] = None): + shard_keys: Optional[List[str]] = None): # For concatenated weights (qkv_proj / up_gate_proj), the global scaling factors and input scaling factors should be shared. input_scale = None weight_scale_2 = None weight_scale = [] device = torch.device("cuda") + scale_span = module.quant_config.group_size if module.tp_mode == TensorParallelMode.ROW else 1 + if shard_keys is None: + shard_keys = [None] - for w in weights: + for shard_key, w in zip(shard_keys, weights): if "input_scale" in w: if input_scale is None: input_scale = w["input_scale"][...] @@ -2649,11 +2736,11 @@ def load_weight_scales_w4a8(self, assert input_scale == w["input_scale"][ ...], "The input_scale should be same for all the weights" if "weight_scale" in w: - ws = load_weight_shard(w["weight_scale"], - tp_size, - tp_rank, - tp_mode, - device=device) + ws = module.load_shard(w, + 'weight_scale', + device=device, + name=shard_key, + scale_span=scale_span) weight_scale.append(ws.to(torch.float16)) if "weight_scale_2" in w: @@ -2669,18 +2756,17 @@ def load_weight_scales_w4a8(self, return input_scale, weight_scale, alpha, weight_scale_2 def load_weights_vanilla(self, module: Linear, weights: List[Dict]): - load_weights_vanilla_helper(module, weights) + elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 + load_weights_vanilla_helper(module, weights, elm_packing=elm_packing) # Use the same device as the weight tensor # as we register pre_quant_scale after sharded model weights are moved to respective gpus device = module.weight.device - pre_quant_scale = load_weight_shard( + pre_quant_scale = module.load_shard( weights[0]["pre_quant_scale"], - module.tp_size, - module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - TensorParallelMode.flip(module.tp_mode), - device, + flip_tp=True, + device=device, ) assert pre_quant_scale.dtype == module.dtype @@ -2692,10 +2778,7 @@ def load_weights_vanilla(self, module: Linear, weights: List[Dict]): copy_weight(module.pre_quant_scale, pre_quant_scale) input_scale, weight_scale, alpha, weight_scale_2 = self.load_weight_scales_w4a8( - weights=weights, - tp_size=module.tp_size, - tp_rank=module.tp_rank, - tp_mode=module.tp_mode) + module, weights) assert len(weight_scale) == 1, "there should be only one weight scale" @@ -2712,8 +2795,9 @@ def load_weights_vanilla(self, module: Linear, weights: List[Dict]): def load_weights_fused_qkv_linear(self, module: Linear, weights: List[Dict]): + elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights) + module, weights, elm_packing=elm_packing) fused_weight = torch.cat((q_weight, k_weight, v_weight)) fused_weight = preprocess_weights_for_mixed_gemm( @@ -2722,11 +2806,9 @@ def load_weights_fused_qkv_linear(self, module: Linear, copy_weight(module.weight, fused_weight) + weight_mode = module.weights_loading_config.weight_mode input_scale, weight_scales, alpha, weight_scale_2 = self.load_weight_scales_w4a8( - weights=weights, - tp_size=module.tp_size, - tp_rank=module.tp_rank, - tp_mode=module.tp_mode) + module, weights, shard_keys=weight_mode.shard_keys) # Create concatenated weight scale tensor cat_weight_scale = (torch.cat(weight_scales, dim=0).T / @@ -2742,13 +2824,11 @@ def load_weights_fused_qkv_linear(self, module: Linear, # Use the same device as the weight tensor # as we register pre_quant_scale after sharded model weights are moved to respective gpus device = module.weight.device - pre_quant_scale = load_weight_shard( + pre_quant_scale = module.load_shard( weights[0]["pre_quant_scale"], - module.tp_size, - module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - TensorParallelMode.flip(module.tp_mode), - device, + flip_tp=True, + device=device, ) module.pre_quant_scale = Parameter( @@ -2760,8 +2840,9 @@ def load_weights_fused_qkv_linear(self, module: Linear, def load_weights_fused_gate_up_linear(self, module: Linear, weights: List[Dict]): + elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights) + module, weights, elm_packing=elm_packing) fused_weight = torch.cat((gate_weight, up_weight)) fused_weight = preprocess_weights_for_mixed_gemm( @@ -2770,11 +2851,9 @@ def load_weights_fused_gate_up_linear(self, module: Linear, copy_weight(module.weight, fused_weight) + weight_mode = module.weights_loading_config.weight_mode input_scale, weight_scale, alpha, weight_scale_2 = self.load_weight_scales_w4a8( - weights=weights, - tp_size=module.tp_size, - tp_rank=module.tp_rank, - tp_mode=module.tp_mode) + module, weights, shard_keys=weight_mode.shard_keys) fused_scale = (torch.cat(weight_scale, dim=0).T / weight_scale_2).contiguous() @@ -2788,13 +2867,11 @@ def load_weights_fused_gate_up_linear(self, module: Linear, # Use the same device as the weight tensor # as we register pre_quant_scale after sharded model weights are moved to respective gpus device = module.weight.device - pre_quant_scale = load_weight_shard( + pre_quant_scale = module.load_shard( weights[0]["pre_quant_scale"], - module.tp_size, - module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - TensorParallelMode.flip(module.tp_mode), - device, + flip_tp=True, + device=device, ) # NOTE:Create this tensor in load_weights, since not all layer have this tensor and memory is not allocated for it (same as W4A16) @@ -3189,6 +3266,9 @@ def __init__( fused_weight_shard_indices_mapping: Optional[dict] = None, nvfp4_allowed_backends: Optional[List[str]] = None, enable_gemm_allreduce_fusion: bool = True, + override_tp_sharding: Optional[Union[tuple[int, int], + Dict[str, tuple[int, + int]]]] = None, ): """ Args: @@ -3228,25 +3308,70 @@ def __init__( 'cutlass', 'cublaslt', 'cuda_core' ] - local_in_features = in_features - local_out_features = out_features - - if self.tp_mode == TensorParallelMode.ROW: - assert in_features % self.tp_size == 0, ( - f'in_features {in_features} must be divisible by tp_size {self.tp_size}' - ) - local_in_features = in_features // self.tp_size - elif self.tp_mode == TensorParallelMode.COLUMN: - assert out_features % self.tp_size == 0, ( - f'out_features {out_features} must be divisible by tp_size {self.tp_size}' + if self.tp_mode not in (TensorParallelMode.ROW, + TensorParallelMode.COLUMN, None): + raise ValueError( + f"Invalid tp_mode {self.tp_mode!r}; expected ROW, COLUMN, or None." ) - local_out_features = out_features // self.tp_size - reduce_output = False if self.mapping.enable_attention_dp else reduce_output + + # Init TP sharding either from override or auto generated + _uneven_tp_unsupported = {QuantAlgo.NVFP4_ARC} + _quant_algo = quant_config.quant_algo if quant_config else None + if override_tp_sharding is not None: + if _quant_algo in _uneven_tp_unsupported: + raise ValueError( + f"Uneven TP is not supported with QuantAlgo {_quant_algo}") + self.tp_sharding = override_tp_sharding + self._tp_sharding_is_auto = False + elif self.tp_size > 1 and self.tp_mode is not None \ + and self.weights_loading_config.weight_mode == WeightMode.VANILLA \ + and _quant_algo not in _uneven_tp_unsupported \ + and not skip_create_weights_in_init: + features = in_features if self.tp_mode == TensorParallelMode.ROW else out_features + self.tp_sharding = self._auto_tp_sharding(features, quant_config) + self._tp_sharding_is_auto = True else: - assert self.tp_mode is None, f'unsupported tensor parallel mode: {self.tp_mode}' + self.tp_sharding = None + self._tp_sharding_is_auto = False + if self.tp_size > 1 and self.tp_mode is not None: + features = in_features if self.tp_mode == TensorParallelMode.ROW else out_features + assert features % self.tp_size == 0, ( + f"Uneven TP not supported for this configuration " + f"(weight_mode={self.weights_loading_config.weight_mode}, " + f"quant_algo={_quant_algo}). " + f"features={features} must be divisible by tp_size={self.tp_size}." + ) - self.in_features = local_in_features - self.out_features = local_out_features + self.in_features = self.calculate_local_in_features(in_features) + self.out_features = self.calculate_local_out_features(out_features) + + # allgather with sizes=None requires every rank to hold the same shape, so an + # unevenly sharded COLUMN output has to declare the per-rank widths explicitly. + self.gather_output_sizes = None + if self.gather_output and self.tp_size > 1 \ + and self.tp_mode == TensorParallelMode.COLUMN: + if self._tp_sharding_is_auto: + sizes = [ + end - start for start, end in ( + self._auto_tp_sharding(out_features, quant_config, rank) + for rank in range(self.tp_size)) + ] + if len(set(sizes)) > 1: + self.gather_output_sizes = sizes + elif self.tp_sharding is not None: + # allgather concatenates in rank order, so gather_output assumes rank i + # holds the i-th ascending, contiguous slice. _auto_tp_sharding + # guarantees that; an arbitrary override guarantees neither the widths + # nor the ordering, and `sizes` carries widths only — it cannot express + # a permuted or non-contiguous layout even when the widths are equal. + raise ValueError( + f"gather_output=True is not supported together with " + f"override_tp_sharding ({self.tp_sharding}); gather_output " + f"requires the rank-ordered contiguous layout that only " + f"automatic TP sharding provides.") + + if self.tp_mode == TensorParallelMode.COLUMN: + reduce_output = False if self.mapping.enable_attention_dp else reduce_output self.all_reduce = AllReduce(mapping=self.mapping, strategy=allreduce_strategy, @@ -3284,9 +3409,8 @@ def __init__( if torch.cuda.is_available(): capability = torch.cuda.get_device_capability( torch.device('cuda:0')) - # enable cuda core for sm89 and sm120 - self.enable_cuda_core = (capability[0] == 8 and capability[1] == 9) \ - or (capability[0] == 12 and capability[1] == 0) + # enable cuda core for sm89, sm120, and sm121 + self.enable_cuda_core = capability in ((8, 9), (12, 0), (12, 1)) if not skip_create_weights_in_init: self.create_weights() @@ -3294,6 +3418,166 @@ def __init__( def get_quant_method(self, quant_config: Optional[QuantConfig] = None): return get_quant_method(quant_config) + @staticmethod + def _calc_shard(total, tp_size, rank): + return (total // tp_size) * rank + min(total % tp_size, rank) + + def _auto_tp_sharding(self, features, quant_config, rank=None): + """Auto-generate tp_sharding tuple based on quant alignment requirements. + + VANILLA mode only. Fused modes (FUSED_QKV, FUSED_GATE_UP) require explicit + override_tp_sharding because individual sub-weight sizes (Q vs K vs V; gate + vs up) are not knowable here — they aren't always equal (e.g. GQA), and + cross-rank consistency must be decided by the caller. + + `rank` defaults to this module's own TP rank; pass an explicit rank to query + another rank's range (e.g. to build the per-rank sizes an allgather needs). + """ + assert self.weights_loading_config.weight_mode == WeightMode.VANILLA, ( + f"_auto_tp_sharding only supports VANILLA mode, got " + f"{self.weights_loading_config.weight_mode}. Fused modes require " + f"explicit override_tp_sharding.") + rank = self.tp_rank if rank is None else rank + alignment = get_quant_method(quant_config).get_tp_alignment( + self.tp_mode, quant_config) + if alignment <= 1: + # No alignment constraint — use standard element-level distribution + start = self._calc_shard(features, self.tp_size, rank) + end = self._calc_shard(features, self.tp_size, rank + 1) + else: + # Distribute whole alignment-sized blocks across ranks + assert features % alignment == 0, ( + f"Feature dim ({features}) must be divisible by quant alignment " + f"({alignment}) for TP sharding") + num_blocks = features // alignment + block_start = self._calc_shard(num_blocks, self.tp_size, rank) + block_end = self._calc_shard(num_blocks, self.tp_size, rank + 1) + start = block_start * alignment + end = block_end * alignment + return (start, end) + + def _calculate_local_features_helper(self, features): + if isinstance(self.tp_sharding, tuple): + assert self.weights_loading_config.weight_mode == WeightMode.VANILLA + start, end = self.tp_sharding + return end - start + elif isinstance(self.tp_sharding, dict): + assert self.weights_loading_config.weight_mode in ( + WeightMode.FUSED_GATE_UP_LINEAR, WeightMode.FUSED_QKV_LINEAR) + return sum(end - start for start, end in self.tp_sharding.values()) + else: + assert features % self.tp_size == 0 + return features // self.tp_size + + def calculate_local_in_features(self, in_features): + """Local input feature count after TP sharding (full size if not row-parallel).""" + if self.tp_mode != TensorParallelMode.ROW: + return in_features + + return self._calculate_local_features_helper(in_features) + + def calculate_local_out_features(self, out_features): + """Local output feature count after TP sharding (full size if not column-parallel).""" + if self.tp_mode != TensorParallelMode.COLUMN: + return out_features + + return self._calculate_local_features_helper(out_features) + + def load_shard( + self, + weights: Union[Dict, torch.Tensor], + label: Optional[str] = None, + device: torch.device = torch.device('cpu'), + name: Optional[str] = None, + flip_tp: bool = False, # for input activation scales + scale_span: Optional[int] = None, + # number of elms in a given "slot", used for fp4 since + # 2 are packed in each 8 bit element of the tensor + elm_packing: int = 1, + ) -> torch.Tensor: + """Slice a weight tensor for this rank's TP shard. + + Unified entry point for module-aware weight loading: respects + `self.tp_sharding` (uneven TP overrides, fused QKV/gate-up dicts) and + quant-specific knobs (`scale_span`, `elm_packing`). Pass `weights` as a + dict with `label` to pick the entry, or as a bare tensor when there's + only one. Supersedes the free function `load_weight_shard`. + """ + if label: + if label not in weights: + return None + weight = weights[label] + else: + weight = weights + + # Skip device transfers on integrated GPUs to conserve shared memory + if weight.device.type != device.type and is_device_integrated(): + # For integrated GPU systems (e.g., DGX Spark), CPU and GPU share limited physical memory. + # Avoiding device transfers reduces memory consumption and unnecessary data copies, + # enabling support for larger models on memory-constrained systems. + logger.warning_once( + f"[Linear.load_shard] Skipping device transfer from {weight.device} to {device} on integrated GPU to conserve shared memory.", + key="load_weight_shard_skip_device_transfer_with_integrated_gpu" + ) + device = weight.device + if isinstance(weight, torch.Tensor): + tensor_shape = weight.shape + + def maybe_convert_to_torch_tensor(tensor: torch.Tensor, + indices: list[slice] + | None = None): + if indices is None: + # Avoid unnecessary copy + return tensor.to(device) + else: + return tensor[indices].to(device) + + # WAR to check whether it is a safetensor slice since safetensor didn't register the type to the module + # safetensors slice, supports lazy loading, type(weight) is `builtin.PySafeSlice` + elif hasattr(weight, "get_shape"): + tensor_shape = weight.get_shape() + + def maybe_convert_to_torch_tensor( + tensor, indices: Union[slice, tuple[slice]] = slice(None)): + return tensor[indices].to(device) + else: + raise ValueError(f'unsupported weight type: {type(weight)}') + if self.tp_mode is None or self.tp_size <= 1: + return maybe_convert_to_torch_tensor(weight) + + tp_mode = TensorParallelMode.flip( + self.tp_mode) if flip_tp else self.tp_mode + split_dim = TensorParallelMode.split_dim(tp_mode) + + if len(tensor_shape) == 1 and split_dim == 1: + return maybe_convert_to_torch_tensor(weight) + + width = tensor_shape[split_dim] + if width == 1: + return maybe_convert_to_torch_tensor(weight) + + if self.tp_sharding is None: + slice_start = self._calc_shard(width, self.tp_size, self.tp_rank) + slice_end = self._calc_shard(width, self.tp_size, self.tp_rank + 1) + else: + if isinstance(self.tp_sharding, tuple): + slice_start, slice_end = self.tp_sharding + elif isinstance(self.tp_sharding, dict): + slice_start, slice_end = self.tp_sharding[name] + + if scale_span: + assert slice_end % scale_span == 0 and slice_start % scale_span == 0 + slice_start //= scale_span + slice_end //= scale_span + + assert slice_start % elm_packing == 0 and slice_end % elm_packing == 0 + slice_start //= elm_packing + slice_end //= elm_packing + + slice_obj = [slice(d) for d in tensor_shape] + slice_obj[split_dim] = slice(slice_start, slice_end) + return maybe_convert_to_torch_tensor(weight, tuple(slice_obj)) + def create_weights(self): if self._weights_created: return @@ -3456,7 +3740,9 @@ def forward( output = self.apply_linear(input, self.bias, lora_params, layer_idx) if self.gather_output: from ..distributed import allgather - output = allgather(output, self.mapping) + output = allgather(output, + self.mapping, + sizes=self.gather_output_sizes) else: output = self.apply_linear(input, self.bias, lora_params, layer_idx) diff --git a/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py b/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py index a01eec1113d4..f786417a4e46 100644 --- a/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py +++ b/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py @@ -14,6 +14,10 @@ from torch import nn from transformers import Qwen3NextConfig +from tensorrt_llm._torch.modules.fla.cached_replay import ( + CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, + fused_recurrent_gated_delta_rule_cached_replay_update, +) from tensorrt_llm._torch.modules.fla.fused_recurrent import fused_recurrent_gated_delta_rule_update from tensorrt_llm._torch.modules.fla.fused_sigmoid_gating_recurrent import ( _can_use_flashinfer_gdn_verify, @@ -21,13 +25,14 @@ fused_sigmoid_gating_delta_rule_update, ) from tensorrt_llm._utils import is_flashinfer_gdn_supported_arch +from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from ...attention_backend import AttentionMetadata from ...distributed import AllReduceParams from ...model_config import ModelConfig from ...speculative import SpecMetadata -from ...utils import EventType, get_model_extra_attrs, is_torch_compiling +from ...utils import EventType, get_model_extra_attrs, is_gdn_replay_enabled, is_torch_compiling from ..linear import FP8QDQLinearMethod, Linear, TensorParallelMode from ..multi_stream_utils import maybe_execute_in_parallel from .causal_conv1d import causal_conv1d_fn, causal_conv1d_update @@ -84,6 +89,70 @@ def _extract_gdn_extra_attrs(layer_idx: str): return metadata, gdn_layer, extra_attrs.get("spec_metadata", None) +@triton.jit +def _reset_gdn_states_kernel( + ssm_states, + conv_states, + state_indices, + has_initial_states, + ssm_state_stride, + conv_state_stride, + NUM_CACHE_LINES: tl.constexpr, + SSM_STATE_SIZE: tl.constexpr, + CONV_STATE_SIZE: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + request_idx = tl.program_id(0) + offsets = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + state_idx = tl.load(state_indices + request_idx).to(tl.int64) + needs_reset = ~tl.load(has_initial_states + request_idx).to(tl.int1) + valid_state = (state_idx >= 0) & (state_idx < NUM_CACHE_LINES) + ssm_row_offset = state_idx * ssm_state_stride.to(tl.int64) + conv_row_offset = state_idx * conv_state_stride.to(tl.int64) + + tl.store( + ssm_states + ssm_row_offset + offsets, + 0.0, + mask=needs_reset & valid_state & (offsets < SSM_STATE_SIZE), + ) + tl.store( + conv_states + conv_row_offset + offsets, + 0.0, + mask=needs_reset & valid_state & (offsets < CONV_STATE_SIZE), + ) + + +def _reset_gdn_states( + ssm_states: torch.Tensor, + conv_states: torch.Tensor, + state_indices: torch.Tensor, + has_initial_states: torch.Tensor, +) -> None: + num_requests = state_indices.shape[0] + if num_requests == 0: + return + + ssm_state_size = ssm_states.numel() // ssm_states.shape[0] + conv_state_size = conv_states.numel() // conv_states.shape[0] + block_size = 256 + grid = ( + num_requests, + triton.cdiv(max(ssm_state_size, conv_state_size), block_size), + ) + _reset_gdn_states_kernel[grid]( + ssm_states, + conv_states, + state_indices, + has_initial_states, + ssm_states.stride(0), + conv_states.stride(0), + ssm_states.shape[0], + ssm_state_size, + conv_state_size, + block_size, + ) + + @torch.library.custom_op("trtllm::gdn_custom_op_inplace", mutates_args=("output",)) def gdn_custom_op_inplace( mixed_qkv: torch.Tensor, @@ -173,6 +242,17 @@ def __init__( config = model_config.pretrained_config self.model_config = model_config self.pretrained_config = config + replay_enabled = is_gdn_replay_enabled() + if replay_enabled: + logger.info_once( + "Configured GDN MTP replay implementation: cached", + key="gdn_mtp_replay_cached", + ) + else: + logger.info_once( + "GDN MTP replay disabled; set TRTLLM_USE_GDN_REPLAY=1 to enable it", + key="gdn_mtp_replay_disabled", + ) # tensor parallel tp_size = model_config.mapping.tp_size @@ -381,6 +461,69 @@ def _postprocess_gdn_output( attn_out = attn_out.view(-1, self.value_dim_per_tp) return self.out_proj(attn_out, all_reduce_params=all_reduce_params) + def _replay_verify_recurrent( + self, + query, + key, + value, + a, + b, + ssm_states, + state_indices_d, + num_decodes, + draft_token_num, + replay_metadata, + layer_cache, + replay_work_items, + replay_n_writes, + output_d=None, + packed_qkv=None, + use_all_layer_commit=False, + ): + """Run MTP target verification via the replay kernel. + + This avoids intermediate-state writes and accepted-state copies; commits + are deferred via the compact history cache. Cached replay reuses the + Mamba2 fields as old_x<->U, old_B<->normalized k, and + old_dt<->cumulative G. + """ + assert replay_metadata is not None, ( + "GDN replay enabled but replay metadata was not allocated." + ) + assert draft_token_num == replay_metadata.replay_step_width, ( + "GDN replay does not support dynamic draft length yet: " + f"{draft_token_num} != {replay_metadata.replay_step_width}" + ) + if draft_token_num > 8 or replay_metadata.replay_history_size > 16: + raise RuntimeError( + "GDN cached replay requires draft_token_num <= 8 and replay_history_size <= 16." + ) + return fused_recurrent_gated_delta_rule_cached_replay_update( + q=query, + k=key, + v=value, + g=a, + beta=b, + A_log=self.A_log, + dt_bias=self.dt_bias, + launch_with_pdl=True, + ssm_states=ssm_states, + state_indices=state_indices_d[:num_decodes], + old_u=layer_cache.old_x, + old_k=layer_cache.old_B, + old_G=layer_cache.old_dt, + old_beta=layer_cache.old_dA_cumsum, + cache_buf_idx=layer_cache.cache_buf_idx, + prev_num_accepted_tokens=layer_cache.prev_num_accepted_tokens, + history_size=replay_metadata.replay_history_size, + replay_work_items=replay_work_items, + n_writes=replay_n_writes, + use_qk_l2norm_in_kernel=True, + packed_qkv=packed_qkv, + use_all_layer_commit=use_all_layer_commit, + output=output_d, + ) + def forward_decode( self, conv_states, @@ -406,7 +549,7 @@ def forward_decode( assert a.shape[0] == num_decodes * draft_token_num assert b.shape[0] == num_decodes * draft_token_num assert intermediate_conv_states is not None - assert intermediate_ssm_states is not None + assert kwargs.get("use_replay", False) or intermediate_ssm_states is not None # Speculative verification path: # 1. run conv update with per-step intermediate cache writes @@ -426,6 +569,8 @@ def forward_decode( conv_state_indices=cache_indices[:num_decodes], intermediate_conv_window=intermediate_conv_states, intermediate_state_indices=intermediate_state_indices, + # PDL chain: conv1d -> replay verify kernel (replay only) + launch_dependent_kernels=kwargs.get("use_replay", False), ) mixed_qkv = mixed_qkv_processed.transpose(1, 2).reshape( num_decodes * draft_token_num, -1 @@ -454,6 +599,39 @@ def forward_decode( # intermediate states written to the batch-scoped [:num_decodes] # prefix consumed by update_mamba_states()); fall back to the # Triton recurrent kernel when unavailable. + if kwargs.get("use_replay", False): + output_d = None + if output is not None: + output_d = output.view( + num_decodes, + draft_token_num, + self.num_v_heads // self.attn_tp_size, + self.head_v_dim, + ) + return self._replay_verify_recurrent( + query, + key, + value, + a, + b, + ssm_states, + cache_indices, + num_decodes, + draft_token_num, + kwargs.get("replay_metadata"), + kwargs.get("layer_cache"), + kwargs.get("replay_work_items"), + kwargs.get("replay_n_writes"), + output_d, + packed_qkv=mixed_qkv, + use_all_layer_commit=kwargs.get("use_cached_replay_all_layer_commit", False), + ).view( + 1, + num_decodes * draft_token_num, + self.num_v_heads // self.attn_tp_size, + self.head_v_dim, + ) + if _can_use_flashinfer_gdn_verify( ssm_states, self.head_k_dim, self.head_v_dim, draft_token_num ): @@ -632,7 +810,7 @@ def forward_extend( assert a_d.shape[0] == num_decodes * draft_token_num assert b_d.shape[0] == num_decodes * draft_token_num assert intermediate_conv_states is not None - assert intermediate_ssm_states is not None + assert kwargs.get("use_replay", False) or intermediate_ssm_states is not None intermediate_state_indices = torch.arange( num_decodes, dtype=torch.int32, device=state_indices_d.device @@ -647,6 +825,8 @@ def forward_extend( conv_state_indices=state_indices_d, intermediate_conv_window=intermediate_conv_states, intermediate_state_indices=intermediate_state_indices, + # PDL chain: conv1d -> replay verify kernel (replay only) + launch_dependent_kernels=kwargs.get("use_replay", False), ) mixed_qkv_d = mixed_qkv_d.transpose(1, 2).reshape(num_decode_tokens, -1) else: @@ -768,7 +948,25 @@ def forward_extend( self.head_v_dim, ) - if _can_use_flashinfer_gdn_verify( + if kwargs.get("use_replay", False): + attn_out_decode = self._replay_verify_recurrent( + query_d, + key_d, + value_d, + a_d, + b_d, + ssm_states, + state_indices_d, + num_decodes, + draft_token_num, + kwargs.get("replay_metadata"), + kwargs.get("layer_cache"), + kwargs.get("replay_work_items"), + kwargs.get("replay_n_writes"), + output_d, + use_all_layer_commit=kwargs.get("use_cached_replay_all_layer_commit", False), + ).reshape(1, num_decode_tokens, out_v_heads, self.head_v_dim) + elif _can_use_flashinfer_gdn_verify( ssm_states, self.head_k_dim, self.head_v_dim, draft_token_num ): # FI gathers the initial state from the pool via state_indices_d @@ -904,11 +1102,11 @@ def forward_core( if num_prefills > 0: # PyExecutor guarantees prefill requests are placed before decode requests has_initial_states_p = has_initial_states[:num_prefills] - ssm_states[state_indices_p[~has_initial_states_p]] = torch.zeros( - (), dtype=ssm_states.dtype, device=ssm_states.device - ) - conv_states[state_indices_p[~has_initial_states_p]] = torch.zeros( - (), dtype=conv_states.dtype, device=conv_states.device + _reset_gdn_states( + ssm_states, + conv_states, + state_indices_p, + has_initial_states_p, ) is_target_verify = ( @@ -921,7 +1119,23 @@ def forward_core( layer_cache.intermediate_conv_window if is_target_verify else None ) intermediate_ssm_states = layer_cache.intermediate_ssm if is_target_verify else None + use_replay = is_target_verify and getattr( + attn_metadata.kv_cache_manager, "use_replay_state_update", False + ) + replay_metadata = ( + attn_metadata.kv_cache_manager.get_replay_state_update_metadata() + if use_replay + else None + ) + use_cached_replay_all_layer_commit = ( + num_decodes >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE + and getattr( + attn_metadata.kv_cache_manager, + "use_gdn_cached_replay_all_layer_commit", + False, + ) + ) kwargs = { "mixed_qkv": mixed_qkv, "a": a, @@ -937,6 +1151,16 @@ def forward_core( "state_indices_d": state_indices_d, "num_prefill": num_prefills, "num_decodes": num_decodes, + "use_replay": use_replay, + "replay_metadata": replay_metadata, + "layer_cache": layer_cache, + "replay_work_items": mamba_metadata.replay_work_items[:num_decodes] + if use_replay and num_decodes >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE + else None, + "replay_n_writes": mamba_metadata.replay_n_writes + if use_replay and num_decodes >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE + else None, + "use_cached_replay_all_layer_commit": use_cached_replay_all_layer_commit, } if num_prefills > 0: attn_out = self.forward_extend( diff --git a/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py b/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py index 52ca525c1791..26b57e1e486a 100644 --- a/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py +++ b/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import math from typing import Tuple @@ -214,6 +215,23 @@ def cu_seqlens_to_chunk_indices_offsets( class Mamba2Metadata: + # Warmup-only knob: when set via ``force_initial_states_for_warmup``, + # ``prepare()`` forces ``has_initial_states_cpu[:num_contexts]`` to True so + # the ``HAS_INITSTATES=True`` variants of the SSD Triton kernels compile + # during warmup. Class-scoped (not env-var) so it cannot leak into real + # inference from a stray shell export or a forked worker. + _warmup_force_initial_states: bool = False + + @classmethod + @contextlib.contextmanager + def force_initial_states_for_warmup(cls): + prev = cls._warmup_force_initial_states + cls._warmup_force_initial_states = True + try: + yield + finally: + cls._warmup_force_initial_states = prev + def __init__(self, max_batch_size: int, chunk_size: int): self.max_batch_size = max_batch_size self.chunk_size = chunk_size @@ -377,9 +395,14 @@ def prepare(self, attn_metadata: AttentionMetadata): self.state_indices_cpu[:batch_size], non_blocking=True) else: # indices is a Python sequence (e.g. List[int]); data - # already lives on host, CPU staging is fine. - for i, idx in enumerate(indices): - self.state_indices_cpu[i] = idx + # already lives on host, CPU staging is fine. One bulk + # conversion instead of a per-element tensor write. + assert len(indices) == batch_size, ( + f"get_state_indices() returned {len(indices)} entries for " + f"a batch of {batch_size} requests.") + self.state_indices_cpu[:batch_size].copy_( + torch.as_tensor(indices, + dtype=self.state_indices_cpu.dtype)) self.state_indices[:batch_size].copy_( self.state_indices_cpu[:batch_size], non_blocking=True) @@ -428,6 +451,15 @@ def prepare(self, attn_metadata: AttentionMetadata): device='cpu') self.has_initial_states_cpu[:num_contexts].copy_(initial_states_cpu) + # Warmup-only override: force HAS_INITSTATES=True path so the + # HAS_INITSTATES=True variants of _state_passing_fwd_kernel, + # _chunk_scan_fwd_kernel, and _chunk_state_varlen_kernel compile + # during warmup instead of the first real-request iter that hits + # chunked prefill with cached tokens. Gate is a class-scoped + # context manager (see ``force_initial_states_for_warmup``) so it + # cannot silently affect real inference. + if Mamba2Metadata._warmup_force_initial_states: + self.has_initial_states_cpu[:num_contexts].fill_(True) # Mirror CPU staging flags to the CUDA-side buffer asynchronously. self.has_initial_states[:num_contexts].copy_( self.has_initial_states_cpu[:num_contexts], non_blocking=True) diff --git a/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py b/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py index e807a904978f..688c48b147de 100644 --- a/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py +++ b/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py @@ -430,16 +430,47 @@ def forward( f"{draft_token_num} must match fixed replay step " f"width {replay_step_width}.") - intermediate_state_indices = _cached_arange( - attn_metadata.kv_cache_manager.get_max_resource_count(), - state_indices_d.device)[:num_decodes] + # Dynamic-tree verify uses per-request links; linear MTP skips it. + is_dyn_tree = getattr(spec_metadata, 'is_spec_dec_dynamic_tree', + False) + retrieve_next_token = retrieve_next_sibling = None + retrieve_parent_token = None + if is_dyn_tree: + if use_replay: + raise NotImplementedError( + "Dynamic-tree Mamba verify is not supported with " + "the replay SSM-cache path (TRTLLM_USE_MAMBA_REPLAY)." + ) + retrieve_next_token = spec_metadata.retrieve_next_token + retrieve_next_sibling = spec_metadata.retrieve_next_sibling + assert (retrieve_next_token is not None + and retrieve_next_sibling is not None), ( + "Dynamic-tree verify requires retrieve link " + "tensors on spec_metadata.") + retrieve_next_token = retrieve_next_token[:num_decodes] + retrieve_next_sibling = retrieve_next_sibling[:num_decodes] + # conv1d fills parent links used by tree-aware SSM restore. + retrieve_parent_token = torch.empty( + (num_decodes, draft_token_num), + dtype=torch.int32, + device=state_indices_d.device) + + # Prefer the cache_manager-owned arange; cached fallback storage + # can be recycled by CUDA graph warmup. + _km_isi = getattr(attn_metadata.kv_cache_manager, + 'intermediate_state_indices', None) + if _km_isi is not None: + intermediate_state_indices = _km_isi[:num_decodes] + else: + intermediate_state_indices = _cached_arange( + attn_metadata.kv_cache_manager.get_max_resource_count(), + state_indices_d.device)[:num_decodes] - # Reshape for batch processing - xbc_d_reshaped = xbc_d.view(num_decodes, draft_token_num, - -1).transpose(1, 2) + # Use reshape because dynamic-tree tokens may be non-contiguous. + xbc_d_reshaped = xbc_d.reshape(num_decodes, draft_token_num, + -1).transpose(1, 2) def conv1d(): - # TODO:support tree structure [TRTLLM-10320] xbc_d_processed = causal_conv1d_update_triton( xbc_d_reshaped, conv_states, @@ -449,11 +480,15 @@ def conv1d(): conv_state_indices=state_indices_d[:num_decodes], intermediate_conv_window=intermediate_conv_states, intermediate_state_indices=intermediate_state_indices, + # None on linear MTP. + retrieve_next_token=retrieve_next_token, + retrieve_next_sibling=retrieve_next_sibling, + retrieve_parent_token=retrieve_parent_token, # PDL chain: conv1d → precompute → main (replay only) launch_dependent_kernels=use_replay, ) - return xbc_d_processed.transpose(1, 2).view( + return xbc_d_processed.transpose(1, 2).reshape( num_decode_tokens, -1) else: @@ -588,13 +623,23 @@ def convert_dt(): state_batch_indices=state_batch_indices, disable_state_update=True, intermediate_state_indices=intermediate_state_indices, + # None for linear MTP; tree parent map for dynamic tree. + retrieve_parent_token=retrieve_parent_token, ) else: # Triton kernel + flashinfer need contiguous for alignment. x_d_4d = x_d_4d.contiguous() B_d_4d = B_d_4d.contiguous() C_d_4d = C_d_4d.contiguous() - self.selective_state_update_func( + if is_dyn_tree: + # flashinfer SSU cannot restore tree-parent states. + ssu_func = selective_state_update_native + ssu_extra = dict( + retrieve_parent_token=retrieve_parent_token) + else: + ssu_func = self.selective_state_update_func + ssu_extra = {} + ssu_func( ssm_states, x_d_4d, dt_d_4d, @@ -611,6 +656,7 @@ def convert_dt(): intermediate_states_buffer=intermediate_ssm_states, cache_steps=draft_token_num, intermediate_state_indices=intermediate_state_indices, + **ssu_extra, **philox_kwargs, ) else: diff --git a/tensorrt_llm/_torch/modules/mhc/hyper_connection.py b/tensorrt_llm/_torch/modules/mhc/hyper_connection.py index 73f2b59c6efe..477ea75ddd66 100644 --- a/tensorrt_llm/_torch/modules/mhc/hyper_connection.py +++ b/tensorrt_llm/_torch/modules/mhc/hyper_connection.py @@ -276,7 +276,13 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: if not _cuda_available: raise RuntimeError("CUDA MHC kernels not available") dtype = x.dtype - x_bf16 = x.to(torch.bfloat16).contiguous() + # The CUDA head consumes a flat ``[M, mult, hidden]`` tensor. Preserve any + # leading dims (e.g. ``[batch, block, mult, hidden]`` from the DSpark draft + # block) by collapsing to a single token axis and restoring afterwards; + # the RMS-norm + weighted sum is independent per token, so this matches the + # reference ``hc_head`` which keeps the leading dims. + lead = x.shape[:-2] + x_bf16 = x.reshape(-1, self.mult, self.hidden_size).to(torch.bfloat16).contiguous() y = mhc_hc_head_cuda( x_bf16, self.fn, @@ -287,7 +293,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: norm_eps=self.norm_eps, eps=self.eps, ) - return y.to(dtype) + return y.reshape(*lead, self.hidden_size).to(dtype) def skip_forward(self, x: torch.Tensor) -> torch.Tensor: """Skip HCHead computation for pipeline parallelism on non-last ranks.""" diff --git a/tensorrt_llm/_torch/modules/mla.py b/tensorrt_llm/_torch/modules/mla.py index e2f708525af2..63f9a1bd65d3 100644 --- a/tensorrt_llm/_torch/modules/mla.py +++ b/tensorrt_llm/_torch/modules/mla.py @@ -495,6 +495,15 @@ def __init__( if config is not None: if "mla_layers" not in config.extra_attrs: config.extra_attrs["mla_layers"] = {} + suffix = 0 + # ``layer_idx`` is local to an attention stack, while this registry is shared + # by target and draft modules in one-model speculative decoding. Keep the first + # registration under ``""`` and suffix later collisions as + # ``"_"`` so custom ops resolve the originating module without + # overwriting another stack's weak reference. + while self.layer_idx_str in config.extra_attrs["mla_layers"]: + self.layer_idx_str = str(layer_idx) + f"_{suffix}" + suffix += 1 config.extra_attrs["mla_layers"][self.layer_idx_str] = weakref.ref(self) self.register_to_config = True diff --git a/tensorrt_llm/_torch/modules/mlp.py b/tensorrt_llm/_torch/modules/mlp.py index dcd2cc6ebfe5..b676d28ab69d 100644 --- a/tensorrt_llm/_torch/modules/mlp.py +++ b/tensorrt_llm/_torch/modules/mlp.py @@ -102,7 +102,10 @@ def create_weights(self): has_nvfp4 = hasattr(self.down_proj, 'has_nvfp4') and self.down_proj.has_nvfp4 has_kernel = hasattr(torch.ops.trtllm, 'fused_relu2_quantize') - has_scale = hasattr(self.down_proj, 'input_scale') + # NVFP4LinearMethod.create_weights always allocates input_scale as a + # Parameter, but excluded layers reset it to None at load time. Check + # for a real tensor. + has_scale = getattr(self.down_proj, 'input_scale', None) is not None is_relu2 = self.activation is relu2 # The fused relu2+fp4_quantize kernel body is guarded by # ``__CUDA_ARCH__ >= 1000`` (see fusedActivationQuant.cu). On pre-SM100 diff --git a/tensorrt_llm/_torch/modules/qk_norm_attention.py b/tensorrt_llm/_torch/modules/qk_norm_attention.py index 0ee43381dace..bebdfc81980e 100644 --- a/tensorrt_llm/_torch/modules/qk_norm_attention.py +++ b/tensorrt_llm/_torch/modules/qk_norm_attention.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,13 +19,17 @@ import torch from transformers import PretrainedConfig +from tensorrt_llm.functional import RotaryScalingType from tensorrt_llm.mapping import Mapping from ..attention_backend.interface import PositionalEmbeddingParams from ..model_config import ModelConfig from ..modules.attention import Attention +from ..modules.fused_ops.fused_qk_norm_rope_gate import ( + fused_qkv_gemma_rmsnorm_rope_gate, fused_sigmoid_mul) from ..modules.multi_stream_utils import maybe_execute_in_parallel from ..modules.rms_norm import RMSNorm +from ..utils import is_torch_compiling # Move out from this class @@ -47,8 +51,7 @@ def compute_yarn_parameters( # If config does not contain rope_scaling or rope_type is not yarn, it means the model is not using yarn rope_scaling = getattr(config, "rope_scaling", None) - if rope_scaling is None or getattr(rope_scaling, "rope_type", - None) != "yarn": + if rope_scaling is None or rope_scaling.get("rope_type", None) != "yarn": return 1.0, 0, 0, 1.0 base = config.rope_theta @@ -57,7 +60,7 @@ def compute_yarn_parameters( head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) dim = int(head_dim * partial_rotary_factor) - factor = getattr(rope_scaling, "factor", 1.0) + factor = rope_scaling.get("factor", 1.0) attention_factor = rope_scaling.get("attention_factor") mscale = rope_scaling.get("mscale") mscale_all_dim = rope_scaling.get("mscale_all_dim") @@ -172,6 +175,7 @@ def __init__( # Gemma-style RMSNorm (scale by (1 + weight)) is supported by the fused # qk_norm_rope kernel via the use_gemma flag threaded through below. self.use_gemma_rms_norm = use_gemma_rms_norm + self._fuse_qk_norm_rope_gate = False # If fuse_qk_norm_rope is true, do not apply fused RoPE in attention OP, and self.rotary_emb # will be skipped in the overridden apply_rope. @@ -213,6 +217,87 @@ def __init__( self.aux_stream = torch.cuda.Stream() self.ln_events = [torch.cuda.Event(), torch.cuda.Event()] + def _get_qk_norm_rotary_dim(self) -> int: + partial_rotary_factor = getattr(self.pretrained_config, + "partial_rotary_factor", 1.0) + return int(self.head_dim * partial_rotary_factor) + + def _can_use_fused_qk_norm_rope_gate( + self, qkv: torch.Tensor, + position_ids: Optional[torch.Tensor]) -> bool: + if not self._fuse_qk_norm_rope_gate or is_torch_compiling(): + return False + if torch.version.hip is not None or qkv.device.type != "cuda": + return False + if qkv.dim() != 2 or qkv.stride(-1) != 1: + return False + if qkv.dtype not in (torch.bfloat16, torch.float16): + return False + if position_ids is None or position_ids.dim() > 3: + return False + if position_ids.dtype not in (torch.int32, torch.int64): + return False + use_mrope = position_ids.dim() == 3 + if use_mrope and not (getattr( + self.pos_embd_params, "mrope_interleaved", False) and getattr( + self.pos_embd_params, "mrope_section", None) is not None + and position_ids.shape[0] == 3): + return False + expected_positions = qkv.shape[0] * (3 if use_mrope else 1) + if position_ids.numel() != expected_positions: + return False + if not (self.attn_output_gate and self.fuse_qk_norm_rope + and self.is_qk_norm and self.use_gemma_rms_norm + and not self.skip_rope): + return False + if self.pos_embd_params is None or self.pos_embd_params.rope is None: + return False + if not self.pos_embd_params.is_neox or self.rotary_emb is None: + return False + rope = self.pos_embd_params.rope + if rope.scale_type not in (RotaryScalingType.none, + RotaryScalingType.mrope): + return False + if rope.scale != 1.0: + return False + rotary_dim = self._get_qk_norm_rotary_dim() + return (rotary_dim == rope.dim and rotary_dim > 0 + and rotary_dim <= self.head_dim and rotary_dim % 2 == 0) + + def preprocess_qkv(self, qkv, position_ids): + """Fuse gate split + gemma QK norm + RoPE into a single Triton kernel + when supported; otherwise fall back to the generic unfused path.""" + if not self._can_use_fused_qk_norm_rope_gate(qkv, position_ids): + return super().preprocess_qkv(qkv, position_ids) + use_mrope = position_ids.dim() == 3 + positions = position_ids.reshape( + 3, -1) if use_mrope else position_ids.reshape(-1) + qkv, gate = fused_qkv_gemma_rmsnorm_rope_gate( + qkv, + self.q_norm.weight, + self.k_norm.weight, + self.rotary_emb.rotary_cos_sin, + positions.contiguous(), + self.q_norm.variance_epsilon, + self.num_heads, + self.num_key_value_heads, + self.head_dim, + self._get_qk_norm_rotary_dim(), + tuple(self.pos_embd_params.mrope_section) if use_mrope else None, + ) + return qkv, None, None, gate + + def apply_output_gate(self, attention_output, gate): + if (self._fuse_qk_norm_rope_gate and not is_torch_compiling() + and torch.version.hip is None + and attention_output.device.type == "cuda" + and attention_output.dim() == 2 + and attention_output.stride(-1) == 1 and gate.dim() in (2, 3) + and gate.stride(-1) == 1 + and gate.numel() == attention_output.numel()): + return fused_sigmoid_mul(attention_output, gate, inplace=True) + return super().apply_output_gate(attention_output, gate) + def apply_qk_norm(self, q, k): def q_l2norm(): @@ -238,9 +323,7 @@ def apply_qk_norm_rope(self, qkv, position_ids): factor, low, high, attention_factor = compute_yarn_parameters( self.pretrained_config) - partial_rotary_factor = self.pretrained_config.partial_rotary_factor if hasattr( - self.pretrained_config, "partial_rotary_factor") else 1.0 - rotary_dim = int(self.head_dim * partial_rotary_factor) + rotary_dim = self._get_qk_norm_rotary_dim() # Interleaved mRoPE: position_ids is 3D [3, ...] (temporal/height/width) # and each rotary half-dim picks a section per diff --git a/tensorrt_llm/_torch/modules/swiglu.py b/tensorrt_llm/_torch/modules/swiglu.py index 328bb4e9fdcb..6462705ec961 100644 --- a/tensorrt_llm/_torch/modules/swiglu.py +++ b/tensorrt_llm/_torch/modules/swiglu.py @@ -27,7 +27,8 @@ def scale_and_clamp(x, scale, dtype): @triton.jit def silu_and_mul_kernel(o_ptr, o_stride, o_scale_ptr, x_ptr, x_stride, d, - swiglu_limit: tl.constexpr, BLOCK_SIZE: tl.constexpr, + swiglu_limit: tl.constexpr, swiglu_alpha: tl.constexpr, + swiglu_beta: tl.constexpr, BLOCK_SIZE: tl.constexpr, HAS_O_SCALE: tl.constexpr, HAS_SWIGLU_LIMIT: tl.constexpr) -> None: i = tl.program_id(axis=0).to(tl.int64) @@ -43,10 +44,15 @@ def silu_and_mul_kernel(o_ptr, o_stride, o_scale_ptr, x_ptr, x_stride, d, b = tl.load(x_row_ptr + offsets + d, mask=mask).to(tl.float32) if HAS_SWIGLU_LIMIT: + # Gate clamp is upper-side only, up clamp is symmetric, as both plain + # SwiGLU-with-limit and the SGLang swigluoai activation require. a = tl.minimum(a, swiglu_limit) b = tl.clamp(b, -swiglu_limit, swiglu_limit) - result = tl.sigmoid(a) * a * b + # swiglu_alpha=1.0, swiglu_beta=0.0 recovers plain silu_and_mul; + # swiglu_alpha=1.702, swiglu_beta=1.0 is the swigluoai shape (alpha gain + # inside the sigmoid, (up + 1) offset). + result = a * tl.sigmoid(swiglu_alpha * a) * (b + swiglu_beta) if HAS_O_SCALE: o_scale = tl.load(o_scale_ptr) @@ -58,7 +64,9 @@ def silu_and_mul_kernel(o_ptr, o_stride, o_scale_ptr, x_ptr, x_stride, d, def swiglu(x, quant_scale: Optional[torch.Tensor] = None, quant_type=None, - swiglu_limit: Optional[float] = None): + swiglu_limit: Optional[float] = None, + swiglu_alpha: Optional[float] = None, + swiglu_beta: Optional[float] = None): if quant_scale is not None: assert quant_type is not None return torch.ops.trtllm.silu_and_mul( @@ -66,6 +74,11 @@ def swiglu(x, scale=quant_scale, dtype=quant_type, swiglu_limit=swiglu_limit, + swiglu_alpha=swiglu_alpha, + swiglu_beta=swiglu_beta, ) - return torch.ops.trtllm.silu_and_mul(x, swiglu_limit=swiglu_limit) + return torch.ops.trtllm.silu_and_mul(x, + swiglu_limit=swiglu_limit, + swiglu_alpha=swiglu_alpha, + swiglu_beta=swiglu_beta) diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index 56c6bfd29037..21ea2fa453b3 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -21,13 +21,9 @@ import tensorrt_llm import tensorrt_llm.bindings.executor as trtllm -from tensorrt_llm._utils import ( - confidential_compute_enabled, - get_sm_version, - prefer_pinned, - str_dtype_to_binding, - torch_dtype_to_str, -) +from tensorrt_llm._utils import (confidential_compute_enabled, get_sm_version, + prefer_pinned, str_dtype_to_binding, + torch_dtype_to_str) from tensorrt_llm.bindings.executor import DecodingMode from tensorrt_llm.inputs.multimodal import MultimodalParams @@ -39,7 +35,8 @@ TorchLlmArgs, WaitingQueuePolicy) # isort: on from tensorrt_llm.logger import logger -from tensorrt_llm.lora_helper import LoraConfig, get_default_trtllm_modules_to_hf_modules +from tensorrt_llm.lora_helper import (LoraConfig, + get_default_trtllm_modules_to_hf_modules) from tensorrt_llm.lora_manager import load_torch_lora from tensorrt_llm.mapping import CpType, Mapping @@ -47,50 +44,33 @@ from ..hostfunc import set_low_latency_dispatch from ..model_config import ModelConfig from ..models.modeling_multimodal_mixin import MultimodalModelMixin -from ..speculative import ( - get_num_extra_kv_tokens, - get_num_spec_layers, - get_spec_decoder, - should_use_separate_draft_kv_cache, -) -from .config_utils import ( - extract_mamba_kv_cache_params, - is_gemma4_hybrid, - is_hybrid_linear, - is_kimi_linear, - is_mla, - is_nemotron_hybrid, - is_qwen3_hybrid, -) +from ..speculative import (get_num_extra_kv_tokens, get_num_spec_layers, + get_spec_decoder, should_use_separate_draft_kv_cache) +from ..utils import is_gdn_replay_enabled +from .config_utils import (MambaKVCacheParams, extract_mamba_kv_cache_params, + is_gemma4_hybrid, is_hybrid_linear, is_kimi_linear, + is_mla, is_nemotron_hybrid, is_qwen3_hybrid) from .connectors.kv_cache_connector import KvCacheConnectorManager from .dwdp import DwdpManager from .guided_decoder import GuidedDecoder from .kv_cache_manager_v2 import KVCacheManagerV2 from .kv_cache_transceiver import AttentionTypeCpp, create_kv_cache_transceiver from .llm_request import ExecutorResponse, LlmRequestState -from .mamba_cache_manager import ( - BaseMambaCacheManager, - CppMambaHybridCacheManager, - MixedMambaHybridCacheManager, - use_py_mamba_cache_manager, -) +from .mamba_cache_manager import (BaseMambaCacheManager, + CppMambaHybridCacheManager, + MambaHybridCacheManagerV2, + MixedMambaHybridCacheManager, + use_py_mamba_cache_manager) from .model_engine import PyTorchModelEngine from .py_executor import PyExecutor -from .resource_manager import ( - BaseKVCacheCompressionManager, - KVCacheManager, - PeftCacheManager, - ResourceManager, - ResourceManagerType, -) -from .sampler import EarlyStopSampler, EarlyStopWithMMResult, TorchSampler, TRTLLMSampler -from .scheduler import ( - BindCapacityScheduler, - BindMicroBatchScheduler, - KVCacheV2Scheduler, - SimpleScheduler, - SimpleUnifiedScheduler, -) +from .resource_manager import (KVCacheCompressionManager, KVCacheManager, + PeftCacheManager, ResourceManager, + ResourceManagerType) +from .sampler import (EarlyStopSampler, EarlyStopWithMMResult, TorchSampler, + TRTLLMSampler) +from .scheduler import (BindCapacityScheduler, BindMicroBatchScheduler, + KVCacheV2Scheduler, SimpleScheduler, + SimpleUnifiedScheduler) from .seq_slot_manager import SeqSlotManager GB = 1 << 30 @@ -108,6 +88,22 @@ def _non_hybrid_kv_cache_manager_cls(config, kv_cache_config: KvCacheConfig): return KVCacheManagerV2 if needs_v2 else KVCacheManager +def _resolve_disagg_transceiver_route( + cache_transceiver_config: Optional[CacheTransceiverConfig], +) -> tuple[Optional[str], Optional[str]]: + """Return the effective backend and runtime used for manager routing.""" + if cache_transceiver_config is None: + return None, None + + backend, _ = cache_transceiver_config._resolve_default_backend() + runtime = cache_transceiver_config.transceiver_runtime + if runtime == "auto": + # Model loading normally resolves ``auto``. Paths that skip model + # defaults use the global C++ fallback, matching transceiver creation. + runtime = None + return backend, runtime + + def get_kv_cache_manager_cls( model_config: ModelConfig, kv_cache_config: KvCacheConfig, @@ -115,14 +111,20 @@ def get_kv_cache_manager_cls( cache_transceiver_config: Optional[CacheTransceiverConfig] = None): """Resolve the concrete KV cache manager class for ``model_config``. - For hybrid mamba models the choice between ``Mixed`` (TRTLLM_USE_PY_MAMBA) - and ``Cpp`` (unified pool with block reuse) is made here. Callers that - don't care about disagg can omit ``is_disagg`` and get the unified-pool - default. + For hybrid mamba models the choice between + ``MambaHybridCacheManagerV2`` and compatibility managers is made here. + Callers that don't care about disagg can omit ``is_disagg`` and get the + unified-pool default. - Env-var overrides (agg mode only — disagg picks its inner impl via - ``cache_transceiver_config.transceiver_runtime``): - * ``TRTLLM_USE_PY_MAMBA=1`` — Mixed manager with PythonMambaCacheManager. + Model loading resolves ``use_kv_cache_manager_v2="auto"`` to V2 for + supported hybrid Mamba models. An explicit ``False`` selects a + compatibility manager. In disaggregated serving, V2 additionally requires + the Python transceiver with the NIXL backend. Unsupported V2 routes fail + rather than falling back to a different manager. + + Env-var overrides: + * ``TRTLLM_USE_PY_MAMBA=1`` — Mixed manager in aggregated serving. + * ``TLLM_MAMBA_MANAGER_PREFERENCE`` — explicit manager preference. """ config = model_config.pretrained_config sparse_attn_config = model_config.sparse_attention_config @@ -143,6 +145,17 @@ def get_kv_cache_manager_cls( f"Sparse attention algorithm {sparse_attn_algorithm!r} is not " "supported with hybrid Mamba / linear-attention models.") + state_config = kv_cache_config.mamba_state_config + has_additional_snapshots = bool( + state_config.additional_snapshot_offsets_from_start + or state_config.additional_snapshot_offsets_from_end) + use_v2 = kv_cache_config.use_kv_cache_manager_v2 is True + + if has_additional_snapshots and not use_v2: + raise ValueError("Mamba additional snapshot offsets require " + "use_kv_cache_manager_v2=True; V1 supports only " + "periodic_snapshot_interval.") + # Kimi K3 (KDA + MLA hybrid): block reuse uses the unified C++ pool # (CppMambaHybridCacheManager) like the other hybrid linear models — # per-block KDA state snapshots every mamba_state_cache_interval @@ -152,6 +165,11 @@ def get_kv_cache_manager_cls( # manager's SpeculativeState scratch path only; reuse + SA is # unvalidated. if is_kimi_linear(config): + if use_v2: + raise ValueError( + "Kimi K3 (KDA) is not supported on " + "MambaHybridCacheManagerV2 yet (TRTLLM-14769); unset " + "use_kv_cache_manager_v2.") if kv_cache_config.enable_block_reuse: logger.info( "Using CppMambaHybridCacheManager for Kimi K3 hybrid " @@ -163,42 +181,88 @@ def get_kv_cache_manager_cls( # Skip Softmax only changes attention kernels. Hybrid models still # need a Mamba-capable cache manager for recurrent state. - if use_py_mamba_cache_manager(): + if is_disagg: + backend, runtime = _resolve_disagg_transceiver_route( + cache_transceiver_config) + if use_v2: + if runtime != "PYTHON" or backend != "NIXL": + raise ValueError( + "KV cache manager V2 for hybrid Mamba disaggregated " + "serving requires transceiver_runtime='PYTHON' with " + "backend='NIXL'.") + else: + if (kv_cache_config.enable_block_reuse and runtime == "PYTHON"): + raise ValueError( + "Hybrid Mamba disaggregated serving with block reuse " + "and transceiver_runtime='PYTHON' requires " + "use_kv_cache_manager_v2=True.") + if kv_cache_config.enable_block_reuse: + return CppMambaHybridCacheManager + if runtime == "PYTHON" and backend == "NIXL": + logger.info("Python transceiver detected; using " + "MixedMambaHybridCacheManager for hybrid model") + return MixedMambaHybridCacheManager + return CppMambaHybridCacheManager + + if use_py_mamba_cache_manager() and not is_disagg: + if use_v2: + raise ValueError( + "TRTLLM_USE_PY_MAMBA=1 conflicts with explicit " + "use_kv_cache_manager_v2=True.") if kv_cache_config.enable_block_reuse: raise ValueError( "TRTLLM_USE_PY_MAMBA=1 forces " "MixedMambaHybridCacheManager, which does not support " "block reuse. Disable block reuse or unset " - "TRTLLM_USE_PY_MAMBA to use CppMambaHybridCacheManager.") + "TRTLLM_USE_PY_MAMBA to use the configured cache manager.") logger.info( "Using MixedMambaHybridCacheManager for hybrid mamba model") return MixedMambaHybridCacheManager - if kv_cache_config.enable_block_reuse: - return CppMambaHybridCacheManager - if (cache_transceiver_config is not None - and cache_transceiver_config.transceiver_runtime == "PYTHON"): - logger.info("Python transceiver detected; using " - "MixedMambaHybridCacheManager for hybrid mamba model") - return MixedMambaHybridCacheManager - default_cls = CppMambaHybridCacheManager env_override = os.environ.get('TLLM_MAMBA_MANAGER_PREFERENCE', None) if env_override is not None: - if env_override.upper() == 'MIXED': + env_override = env_override.upper() + if env_override == 'MIXED': + if use_v2: + raise ValueError( + "TLLM_MAMBA_MANAGER_PREFERENCE=MIXED conflicts with " + "explicit use_kv_cache_manager_v2=True.") + if kv_cache_config.enable_block_reuse: + raise ValueError( + "TLLM_MAMBA_MANAGER_PREFERENCE=MIXED forces " + "MixedMambaHybridCacheManager, which does not support " + "block reuse. Disable block reuse, use the CPP " + "preference, or explicitly enable KV cache manager " + "V2.") logger.warning( - "Environment variable TLLM_MAMBA_MANAGER_PREFERENCE=MIXED overrides the default Mamba cache manager to MixedMambaHybridCacheManager. This may lead to increased memory usage due to lack of block reuse, but can be necessary for disaggregated setups or to avoid potential issues with the C++ manager. Set TLLM_MAMBA_MANAGER_PREFERENCE=CPP to use the CppMambaHybridCacheManager instead, which is the default for non-disaggregated setups without block reuse explicitly disabled." - ) + "Environment variable TLLM_MAMBA_MANAGER_PREFERENCE=MIXED " + "overrides the default Mamba cache manager to " + "MixedMambaHybridCacheManager.") return MixedMambaHybridCacheManager - elif env_override.upper() == 'CPP': + if env_override == 'CPP': + if use_v2: + raise ValueError( + "TLLM_MAMBA_MANAGER_PREFERENCE=CPP conflicts with " + "explicit use_kv_cache_manager_v2=True.") logger.warning( - "Environment variable TLLM_MAMBA_MANAGER_PREFERENCE=CPP overrides the default Mamba cache manager to CppMambaHybridCacheManager. This enables block reuse and can reduce memory usage, but may not be compatible with disaggregated setups. Set TLLM_MAMBA_MANAGER_PREFERENCE=MIXED to use the MixedMambaHybridCacheManager instead if you encounter issues with the C++ manager or are running in a disaggregated environment." - ) + "Environment variable TLLM_MAMBA_MANAGER_PREFERENCE=CPP " + "overrides the default Mamba cache manager to " + "CppMambaHybridCacheManager.") return CppMambaHybridCacheManager - else: - logger.warning( - f"Unrecognized value for TLLM_MAMBA_MANAGER_PREFERENCE: {env_override}. " - f"Expected 'CPP' or 'MIXED'. Using default {default_cls.__name__}." - ) - return default_cls + logger.warning( + f"Unrecognized value for TLLM_MAMBA_MANAGER_PREFERENCE: {env_override}. " + "Expected 'CPP' or 'MIXED'. Using the configured " + "KV cache manager default.") + + if not use_v2: + return CppMambaHybridCacheManager + + if (kv_cache_config.enable_block_reuse + and kv_cache_config.enable_kv_pool_rebalance): + raise ValueError( + "V2 Mamba block reuse is not compatible with " + "enable_kv_pool_rebalance because the rebalancer does not " + "yet model retained recurrent-state snapshots.") + return MambaHybridCacheManagerV2 elif sparse_attn_config is not None: return get_sparse_attn_kv_cache_manager(sparse_attn_config) else: @@ -381,6 +445,31 @@ def __init__( KVCacheManagerV2) self._draft_config = draft_config self._skip_est = skip_est + self._maybe_enable_fabric_memory_for_python_transceiver() + + def _maybe_enable_fabric_memory_for_python_transceiver(self) -> None: + """Default TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=1 for the Python + transceiver on the C++ V1 KV cache manager. + + The Python transceiver (KvCacheTransceiverV2) transfers KV blocks + directly out of the C++ pool, so the pool should be allocated with + fabric memory to enable MNNVL transfers. This must run before any + pool allocation because the C++ env getter caches the value on first + read. Explicit user settings are respected, and platforms without + fabric memory support fall back to standard allocation in C++. + """ + if (self._cache_transceiver_config is None + or self._cache_transceiver_config.backend is None or + self._cache_transceiver_config.transceiver_runtime != "PYTHON"): + return + if not issubclass(self._kv_cache_manager_cls, KVCacheManager): + return + if os.environ.get("TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY") is None: + os.environ["TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY"] = "1" + logger.info( + "Python cache transceiver with C++ KV cache manager detected; " + "defaulting TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=1 (set it " + "to 0 explicitly to disable)") def _get_model_kv_cache_manager_cls( self, @@ -397,22 +486,16 @@ def _get_model_kv_cache_manager_cls( cache_transceiver_config=self._cache_transceiver_config) cls = self._fallback_if_unsupported_kv_cache_manager_v2( cls, model_config, kv_cache_config) - # The V1-route hybrid mamba managers (disagg, TRTLLM_USE_CPP_MAMBA, - # TRTLLM_USE_PY_MAMBA, or one-model speculative decoding) keep mamba - # state in a separate cache that doesn't honor block reuse. Warn at - # the routing site so users see the warning where the decision is - # actually made. + # Compatibility managers do not support MTP block reuse. Warn at the + # routing site so users see the concrete manager selected for the + # incompatible combination. if is_hybrid_linear(model_engine.model.model_config.pretrained_config) \ - and kv_cache_config.enable_block_reuse: - uses_v1_mamba_route = self._is_disagg \ - or os.environ.get('TRTLLM_USE_CPP_MAMBA', '0') == '1' \ - or os.environ.get('TRTLLM_USE_PY_MAMBA', '0') == '1' \ - or self._speculative_config is not None - if uses_v1_mamba_route: + and kv_cache_config.enable_block_reuse \ + and self._speculative_config is not None: + if not issubclass(cls, MambaHybridCacheManagerV2): logger.warning( "Block reuse does not work with MTP for hybrid linear models " - "when using the legacy MambaCacheManager (TRTLLM_USE_CPP_MAMBA=1)" - ) + f"when using non-V2 Mamba cache manager {cls.__name__}") return cls def _fallback_if_unsupported_kv_cache_manager_v2( @@ -429,7 +512,7 @@ def _fallback_if_unsupported_kv_cache_manager_v2( if self._kv_connector_manager is not None: incompat.append("kv_connector_manager") if self._max_beam_width is not None and self._max_beam_width > 1: - incompat.append("beam_width > 1") + incompat.append("max_beam_width > 1") if incompat: incompat_str = ", ".join(incompat) # Some models are structurally bound to V2 and cannot fall @@ -454,6 +537,12 @@ def _fallback_if_unsupported_kv_cache_manager_v2( f"Gemma4 hybrid attention requires KVCacheManagerV2, " f"which is not yet supported with {incompat_str}. " f"Disable these features to run Gemma4 hybrid models.") + if is_hybrid_linear(config): + raise NotImplementedError( + "Hybrid Mamba cache managers do not support " + f"{incompat_str}; CppMambaHybridCacheManager does not " + "provide a compatible fallback. Use max_beam_width=1 " + "and disable the KV connector.") # Plain V2 (explicitly enabled or selected by a model default): # V2 was a preference, not a structural requirement, so we can # safely fall back to V1. @@ -485,6 +574,15 @@ def _per_manager_cache_cost(self, spec_config=self._speculative_config, **extra_kwargs)) + def _get_one_model_draft_layer_mask(self) -> List[bool]: + """Return the same draft-only mask used by runtime construction.""" + num_draft_layers = self._get_num_draft_layers() + if self._speculative_config.spec_dec_mode.is_external_drafter(): + return [True] * num_draft_layers + target_num_layers = (self._model_engine.model.model_config. + pretrained_config.num_hidden_layers) + return [False] * target_num_layers + [True] * num_draft_layers + def _get_kv_size_per_token(self, kv_cache_config: Optional[KvCacheConfig] = None ) -> CacheCost: @@ -496,8 +594,13 @@ def _get_kv_size_per_token(self, kv_cache_config = (kv_cache_config if kv_cache_config is not None else self._kv_cache_config) model_config = self._model_engine.model.model_config - total = self._per_manager_cache_cost(self._kv_cache_manager_cls, - model_config, kv_cache_config) + use_separate_draft_kv_cache = ( + self._should_create_separate_draft_kv_cache()) + total = self._per_manager_cache_cost( + self._kv_cache_manager_cls, + model_config, + kv_cache_config, + use_separate_draft_kv_cache=use_separate_draft_kv_cache) if self._is_encoder_decoder(): total += CacheCost.from_raw(self._get_cross_kv_size_per_token()) if self._draft_model_engine is not None: @@ -507,7 +610,7 @@ def _get_kv_size_per_token(self, total += self._per_manager_cache_cost(draft_kv_cache_manager_cls, draft_model_config, kv_cache_config) - elif self._should_create_separate_draft_kv_cache(): + elif use_separate_draft_kv_cache: # One-model draft with separate KV cache layout. # Pass num_layers explicitly since the HF config may report a # different layer count than what is actually used at runtime @@ -532,7 +635,8 @@ def _get_kv_size_per_token(self, self._kv_cache_manager_cls, effective_draft_config, kv_cache_config, - num_layers=self._get_num_draft_layers()) + num_layers=self._get_num_draft_layers(), + is_draft=True) return total def _cal_max_memory(self, peak_memory, total_gpu_memory, fraction, @@ -679,13 +783,10 @@ def _reserve_multimodal_encoder_cache_memory( """Reserve encoder-cache capacity when the model implements that cache.""" model = self._model_engine.model if (not isinstance(model, MultimodalModelMixin) - or not model.supports_encoder_cache): + or not model.encoder_cache_active): return peak_memory - multimodal_config = model.model_config.multimodal_config - if multimodal_config is None: - return peak_memory - return peak_memory + multimodal_config.encoder_cache_max_bytes + return peak_memory + model.model_config.multimodal_config.encoder_cache_max_bytes def _get_token_num_for_estimation(self) -> int: """Compute KV cache capacity required for estimate_max_kv_cache_tokens to succeed.""" @@ -805,6 +906,17 @@ def try_prepare_estimation(self) -> bool: logger.info( "KV cache size estimation is not supported for Vanilla attention backend, disable it." ) + if getattr(model_config, "is_encoder_decoder", False): + # The estimation dummies are text-only, and the cross-KV block + # accounting needs an encoder length (getEncoderOutputLen throws). + # _skip_est (not just the local flag) so build_managers runs + # configure_kv_cache_capacity(), which KVCacheManagerV2 needs for + # its memory quota — the TRTLLM_SKIP_KV_CACHE_ESTIMATION=1 path. + self._skip_est = True + estimating_kv_cache = False + logger.info( + "KV cache size estimation is not supported for encoder-decoder " + "models, disable it.") if estimating_kv_cache: estimate_max_tokens = self._get_token_num_for_estimation() @@ -1130,20 +1242,8 @@ def _create_one_model_draft_kv_cache_manager( Create a KV cache manager for draft model layers in one-model mode when target and draft have different KV cache layouts. """ - # Get target model's num_hidden_layers to compute correct layer indices. - # Draft model layers in one-model mode start at target_num_layers. - target_pretrained_config = self._model_engine.model.model_config.pretrained_config - target_num_layers = target_pretrained_config.num_hidden_layers - - # PARD, External Drafter: draft is a separate model, layers start from 0. - # Other methods (EAGLE3, MTP): draft layers are appended after target layers. num_draft_layers = self._get_num_draft_layers() - if self._speculative_config.spec_dec_mode.is_external_drafter(): - spec_dec_layer_mask = [True] * num_draft_layers - else: - spec_dec_layer_mask = [False] * target_num_layers + [ - True - ] * num_draft_layers + spec_dec_layer_mask = self._get_one_model_draft_layer_mask() # Get the effective draft config (explicit draft_config if available, # otherwise fall back to target model config for MTP). @@ -1230,9 +1330,13 @@ def _get_target_and_draft_cache_costs( target_kv_cache_config = (kv_cache_config if kv_cache_config is not None else self._kv_cache_config) total_kv = self._get_kv_size_per_token(target_kv_cache_config) + use_separate_draft_kv_cache = ( + self._should_create_separate_draft_kv_cache()) target_kv = self._per_manager_cache_cost( - self._kv_cache_manager_cls, self._model_engine.model.model_config, - target_kv_cache_config) + self._kv_cache_manager_cls, + self._model_engine.model.model_config, + target_kv_cache_config, + use_separate_draft_kv_cache=use_separate_draft_kv_cache) # The draft contribution is whatever the aggregate has on top of the # target. Both pieces are CacheCost; subtraction is component-wise. draft_kv = CacheCost(slope=total_kv.slope - target_kv.slope, @@ -1735,6 +1839,21 @@ def _build_per_layer_num_kv_heads( ] * num_spec_layers +def _get_mamba_cache_layer_masks( + mamba_params: MambaKVCacheParams, + mapping: Mapping, + spec_config: Optional[SpeculativeConfig], + is_draft: bool, +) -> tuple[List[bool], List[bool]]: + use_separate_draft_kv_cache = ( + not mapping.enable_attention_dp + and should_use_separate_draft_kv_cache(spec_config)) + return mamba_params.get_layer_masks( + is_draft=is_draft, + use_separate_draft_kv_cache=use_separate_draft_kv_cache, + ) + + def _create_kv_cache_manager( model_engine: Optional[PyTorchModelEngine], kv_cache_manager_cls, @@ -1765,6 +1884,19 @@ def _create_kv_cache_manager( Returns: A KVCacheManager instance for the given model engine or model config """ + if (estimating_kv_cache + and issubclass(kv_cache_manager_cls, KVCacheManagerV2) + and kv_cache_config.pool_ratio is None + and kv_cache_config.avg_seq_len is not None + and kv_cache_config.avg_seq_len > max_seq_len): + # Estimation can build multiple managers from the same temporary + # config. The first manager may reduce max_seq_len to fit max_tokens, + # so later draft/cross managers need a per-manager workload length. + # Keep the shared config untouched because it is restored after + # estimation. + kv_cache_config = kv_cache_config.model_copy( + update={"avg_seq_len": max_seq_len}) + # Extract config from model_engine or use provided model_config if model_config is not None: config = model_config.pretrained_config @@ -1878,6 +2010,8 @@ def _create_kv_cache_manager( manager_extra_kwargs = {} if issubclass(kv_cache_manager_cls, KVCacheManagerV2): manager_extra_kwargs["enable_stats"] = enable_kv_cache_stats + if issubclass(kv_cache_manager_cls, MambaHybridCacheManagerV2): + manager_extra_kwargs["is_disagg"] = is_disagg if is_kimi_linear(config): # Kimi K3 hybrid: KDA (Kimi Delta Attention) recurrent/conv states on @@ -1895,10 +2029,18 @@ def _create_kv_cache_manager( ) mamba_params = extract_mamba_kv_cache_params( config, - layer_mask=layer_mask, spec_config=spec_config, quant_config=quant_config, ) + mamba_layer_mask, full_attention_layer_mask = ( + _get_mamba_cache_layer_masks( + mamba_params, + mapping, + spec_config, + is_draft, + )) + num_mamba_layers = (0 if is_draft and mamba_params.num_draft_layers > 0 + else mamba_params.num_mamba_layers) # Kimi K3 KDA state sharding follows the attention-family TP # semantics (Qwen3-Next pattern): replicated under attention-DP, # head-sharded across tp_size otherwise. That is exactly the cache @@ -1912,9 +2054,10 @@ def _create_kv_cache_manager( # states in place, replacing the intermediate-buffer + promotion # flow for KDA layers. kimi_extra_kwargs = {} - if spec_config is not None and issubclass( - kv_cache_manager_cls, MixedMambaHybridCacheManager): - from ..modules.kimi_kda._kda_kernels import is_kda_mtp_verify_available + if spec_config is not None and issubclass(kv_cache_manager_cls, + MixedMambaHybridCacheManager): + from ..modules.kimi_kda._kda_kernels import \ + is_kda_mtp_verify_available if is_kda_mtp_verify_available(): kimi_extra_kwargs["kda_replay_num_spec"] = ( spec_config.tokens_per_gen_step - 1) @@ -1925,15 +2068,15 @@ def _create_kv_cache_manager( mamba_params.num_heads, mamba_params.n_groups, mamba_params.head_dim, - mamba_params.num_mamba_layers, - mamba_params.mamba_layer_mask, + num_mamba_layers, + mamba_layer_mask, mamba_params.dtype, mamba_params.mamba_ssm_cache_dtype, # kv cache parameters (MLA latent cache) kv_cache_config, tensorrt_llm.bindings.internal.batch_manager.CacheType.SELFKONLY, - num_layers=mamba_params.num_full_attention_layers, - layer_mask=mamba_params.full_attention_layer_mask, + num_layers=sum(full_attention_layer_mask), + layer_mask=full_attention_layer_mask, num_kv_heads=1, head_dim=config.kv_lora_rank + config.qk_rope_head_dim, tokens_per_block=tokens_per_block, @@ -1990,10 +2133,18 @@ def _create_kv_cache_manager( mamba_params = extract_mamba_kv_cache_params( config, - layer_mask=layer_mask, spec_config=spec_config, quant_config=quant_config, ) + mamba_layer_mask, full_attention_layer_mask = ( + _get_mamba_cache_layer_masks( + mamba_params, + mapping, + spec_config, + is_draft, + )) + num_mamba_layers = (0 if is_draft and mamba_params.num_draft_layers > 0 + else mamba_params.num_mamba_layers) # Replay state update kernel for MTP: default on for sm >= 80; gates # below disable it for incompatible feature combinations. Cpp cache @@ -2052,6 +2203,11 @@ def _create_kv_cache_manager( mamba_ssm_stochastic_rounding = (stochastic_rounding and mamba_params.mamba_ssm_cache_dtype == torch.float16) + mamba_manager_extra_kwargs = dict(manager_extra_kwargs) + if issubclass(kv_cache_manager_cls, MambaHybridCacheManagerV2): + mamba_manager_extra_kwargs["conv_state_layout"] = "x_b_c" + else: + mamba_manager_extra_kwargs["model_type"] = "nemotron_hybrid" kv_cache_manager = kv_cache_manager_cls( # mamba cache parameters mamba_params.state_size, @@ -2059,15 +2215,15 @@ def _create_kv_cache_manager( mamba_params.num_heads, mamba_params.n_groups, mamba_params.head_dim, - mamba_params.num_mamba_layers, - mamba_params.mamba_layer_mask, + num_mamba_layers, + mamba_layer_mask, mamba_params.dtype, mamba_params.mamba_ssm_cache_dtype, # kv cache parameters kv_cache_config, tensorrt_llm.bindings.internal.batch_manager.CacheType.SELF, - num_layers=mamba_params.num_full_attention_layers, - layer_mask=mamba_params.full_attention_layer_mask, + num_layers=sum(full_attention_layer_mask), + layer_mask=full_attention_layer_mask, num_kv_heads=per_layer_num_kv_heads, head_dim=head_dim, tokens_per_block=tokens_per_block, @@ -2079,10 +2235,9 @@ def _create_kv_cache_manager( spec_config=spec_config, is_estimating_kv_cache=estimating_kv_cache, execution_stream=execution_stream, - model_type="nemotron_hybrid", use_replay_state_update=use_replay, mamba_ssm_stochastic_rounding=mamba_ssm_stochastic_rounding, - **manager_extra_kwargs, + **mamba_manager_extra_kwargs, ) elif is_qwen3_hybrid(config): if max_beam_width > 1: @@ -2095,10 +2250,73 @@ def _create_kv_cache_manager( ) mamba_params = extract_mamba_kv_cache_params( config, - layer_mask=layer_mask, spec_config=spec_config, quant_config=quant_config, ) + mamba_layer_mask, full_attention_layer_mask = ( + _get_mamba_cache_layer_masks( + mamba_params, + mapping, + spec_config, + is_draft, + )) + num_mamba_layers = (0 if is_draft and mamba_params.num_draft_layers > 0 + else mamba_params.num_mamba_layers) + # Replay state update for GDN MTP: mirrors the nemotron_hybrid gating + # above, minus the Mamba2-specific stochastic-rounding/Philox gate. + # The GDN replay kernel does a plain cast on checkpoint commit, so + # quantized SSM cache dtypes stay on the legacy path. + sm = get_sm_version() + use_replay = spec_config is not None and sm >= 80 + if spec_config is None: + logger.info( + "GDN replay kernel requires speculative decoding; using " + "non-replay path") + elif spec_config.tokens_per_gen_step > 8: + logger.info("GDN cached replay supports at most 8 tokens per " + "generation step; using non-replay path") + use_replay = False + + # Tree attention: replay assumes a linear token sequence. + if (spec_config is not None + and (getattr(spec_config, 'eagle_choices', None) is not None + or getattr(spec_config, 'use_dynamic_tree', False))): + logger.info("GDN replay kernel incompatible with tree attention; " + "using legacy MTP path") + use_replay = False + + if mamba_params.mamba_ssm_cache_dtype not in (torch.float32, + torch.bfloat16, + torch.float16): + logger.info( + "GDN replay kernel does not support quantized SSM cache " + f"dtype {mamba_params.mamba_ssm_cache_dtype}; using legacy " + "MTP path") + use_replay = False + + # Replay is opt-in because its end-to-end benefit is workload-dependent. + if not is_gdn_replay_enabled(): + logger.info("GDN replay kernel is disabled; set " + "TRTLLM_USE_GDN_REPLAY=1 to enable it") + use_replay = False + + # Upstream GDN replay commits all local layer checkpoints through the + # contiguous C++ manager state view. V2 exposes per-layer state views, + # so enabling the same path there would fail for partitioned batches. + if (use_replay and issubclass(kv_cache_manager_cls, + MambaHybridCacheManagerV2)): + logger.info( + "GDN replay is not supported by MambaHybridCacheManagerV2; " + "using the legacy MTP path") + use_replay = False + logger.info("GDN replay state update: " + + ("ENABLED" if use_replay else "DISABLED")) + + mamba_manager_extra_kwargs = dict(manager_extra_kwargs) + if issubclass(kv_cache_manager_cls, MambaHybridCacheManagerV2): + mamba_manager_extra_kwargs["conv_state_layout"] = "q_k_v" + else: + mamba_manager_extra_kwargs["model_type"] = "qwen3_next" kv_cache_manager = kv_cache_manager_cls( # mamba cache parameters mamba_params.state_size, @@ -2106,15 +2324,15 @@ def _create_kv_cache_manager( mamba_params.num_heads, mamba_params.n_groups, mamba_params.head_dim, - mamba_params.num_mamba_layers, - mamba_params.mamba_layer_mask, + num_mamba_layers, + mamba_layer_mask, mamba_params.dtype, mamba_params.mamba_ssm_cache_dtype, # kv cache parameters kv_cache_config, tensorrt_llm.bindings.internal.batch_manager.CacheType.SELF, - num_layers=mamba_params.num_full_attention_layers, - layer_mask=mamba_params.full_attention_layer_mask, + num_layers=sum(full_attention_layer_mask), + layer_mask=full_attention_layer_mask, num_kv_heads=per_layer_num_kv_heads, head_dim=head_dim, tokens_per_block=tokens_per_block, @@ -2126,8 +2344,8 @@ def _create_kv_cache_manager( spec_config=spec_config, is_estimating_kv_cache=estimating_kv_cache, execution_stream=execution_stream, - model_type="qwen3_next", - **manager_extra_kwargs, + use_replay_state_update=use_replay, + **mamba_manager_extra_kwargs, ) else: # NOTE: this is a workaround for VSWA to switch to calculate_max_num_blocks_for_vswa in KVCahceManager @@ -2179,16 +2397,38 @@ def _create_kv_cache_manager( return kv_cache_manager +def validate_kv_cache_compression_with_spec( + config: KvCacheCompressionConfig, + spec_config: Optional[SpeculativeConfig], + draft_kv_cache_manager: Optional[KVCacheManagerV2], +) -> None: + """Reject speculative setups the compression method cannot run with.""" + if (spec_config is None + or not config.kv_cache_compression_mode.is_eviction_method()): + return + # Evicting methods co-compact the draft KV, so the draft must be a + # standard paged cache in the same forward (one-model speculation). + mode = spec_config.spec_dec_mode + if not (mode.is_mtp_one_model() or mode.is_eagle3_one_model()): + raise ValueError( + f"KV-cache compression algorithm {config.algorithm!r} does not " + f"support speculative decoding mode {mode.name}: the draft KV " + "must be a standard paged cache compacted together with the " + "target (one-model MTP/EAGLE3).") + + def create_kv_cache_compression_manager( config: KvCacheCompressionConfig, kv_cache_manager: KVCacheManagerV2, -) -> Optional[BaseKVCacheCompressionManager]: + draft_kv_cache_manager: Optional[KVCacheManagerV2] = None, +) -> Optional[KVCacheCompressionManager]: """Build the KV-cache compression manager for ``config.algorithm``, or return None if no algorithm matches. Called from ``create_py_executor`` and registered as a resource manager, like the KV cache manager itself. Concrete algorithms add a dispatch branch - here; the framework ships none. + here; the framework ships none. Speculative-decoding compatibility is + checked by the caller via ``validate_kv_cache_compression_with_spec``. """ logger.warning( "KV-cache compression algorithm '%s' is not registered; running without " @@ -2442,8 +2682,16 @@ def create_py_executor_instance( kv_cache_compression_config = getattr(llm_args, "kv_cache_compression_config", None) if kv_cache_compression_config is not None: + draft_kv_cache_manager = resources.get( + ResourceManagerType.DRAFT_KV_CACHE_MANAGER) + validate_kv_cache_compression_with_spec(kv_cache_compression_config, + spec_config, + draft_kv_cache_manager) compression_manager = create_kv_cache_compression_manager( - kv_cache_compression_config, kv_cache_manager) + kv_cache_compression_config, + kv_cache_manager, + draft_kv_cache_manager=draft_kv_cache_manager, + ) if compression_manager is not None: resources[ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER] = ( compression_manager) @@ -2455,17 +2703,16 @@ def create_py_executor_instance( if kv_cache_manager is not None: resource_manager.resource_managers.move_to_end( ResourceManagerType.KV_CACHE_MANAGER, last=True) - # Compression manager runs after the cache manager: reconciles history once it's resized. - if (ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER - in resource_manager.resource_managers): - resource_manager.resource_managers.move_to_end( - ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER, last=True) - cross_kv_cache_manager = resources.get( ResourceManagerType.CROSS_KV_CACHE_MANAGER) if cross_kv_cache_manager is not None: resource_manager.resource_managers.move_to_end( ResourceManagerType.CROSS_KV_CACHE_MANAGER, last=True) + # Compression is the final reconciler after every native KV manager. + if (ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER + in resource_manager.resource_managers): + resource_manager.resource_managers.move_to_end( + ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER, last=True) # When scheduler_capacity == 1, attention dp dummy request will prevent the scheduling of DISAGG_GENERATION_INIT. # Enlarge scheduler capacity to avoid DISAGG_GENERATION_INIT stuck in the scheduler. @@ -2583,6 +2830,17 @@ def create_py_executor_instance( if scheduler_config is not None else WaitingQueuePolicy.FCFS) + # For enc-dec models max_seq_len covers the (longer) encoder sequence, so + # cap the executor's per-request max_tokens at the decoder position table + # (max_target_positions). + executor_max_seq_len = max_seq_len + if model_engine.model.model_config.is_encoder_decoder: + decoder_position_limit = getattr(config, "max_target_positions", None) + if (decoder_position_limit is not None + and executor_max_seq_len is not None): + executor_max_seq_len = min(executor_max_seq_len, + int(decoder_position_limit)) + return PyExecutor( resource_manager, scheduler, @@ -2606,7 +2864,7 @@ def create_py_executor_instance( garbage_collection_gen0_threshold=garbage_collection_gen0_threshold, kv_connector_manager=kv_connector_manager, resource_governor_queue=resource_governor_queue, - max_seq_len=max_seq_len, + max_seq_len=executor_max_seq_len, peft_cache_config=peft_cache_config, virtual_memory_pools=virtual_memory_pools, execution_stream=execution_stream, diff --git a/tensorrt_llm/_torch/pyexecutor/config_utils.py b/tensorrt_llm/_torch/pyexecutor/config_utils.py index 303b377bac41..c99cc6c36607 100644 --- a/tensorrt_llm/_torch/pyexecutor/config_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/config_utils.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import dataclasses from typing import List, Optional @@ -141,6 +144,11 @@ def is_mla(config): return False +def is_minimax_m3(sparse_attention_config): + """True when the sparse attention config selects the MiniMax-M3 algorithm.""" + return sparse_attention_config is not None and sparse_attention_config.algorithm == "minimax_m3" + + def is_qwen3_next(config): return hasattr( config, 'architectures' @@ -230,18 +238,49 @@ class MambaKVCacheParams: n_groups: int # n_groups | linear_num_key_heads head_dim: int # mamba_head_dim | linear_value_head_dim - # Per-layer masks and counts (trailing entries cover MTP/draft layers, - # which are attention-only and carry no Mamba state). + # Target-layer masks and counts. Appended MTP/draft layers need only a + # count because every draft layer is full attention. mamba_layer_mask: List[bool] - full_attention_layer_mask: List[bool] + target_full_attention_layer_mask: List[bool] num_mamba_layers: int - num_full_attention_layers: int + num_draft_layers: int # Dtypes dtype: torch.dtype # config.torch_dtype mamba_ssm_cache_dtype: Optional[ torch.dtype] # quant_config.mamba_ssm_cache_dtype + def get_layer_masks( + self, + *, + is_draft: bool = False, + use_separate_draft_kv_cache: bool = False, + ) -> tuple[List[bool], List[bool]]: + """Return Mamba and attention masks for one cache manager. + + Target masks use target-model layer indices. Appended one-model draft + layers use indices immediately following the target layers, so a + draft-only manager receives target-sized false prefixes. A combined + manager receives the concatenated target and draft layouts. + """ + target_mamba_mask = list(self.mamba_layer_mask) + target_attention_mask = list(self.target_full_attention_layer_mask) + num_target_layers = len(target_mamba_mask) + num_draft_layers = self.num_draft_layers + draft_attention_mask = [True] * num_draft_layers + + if is_draft and num_draft_layers > 0: + return ( + [False] * (num_target_layers + num_draft_layers), + [False] * num_target_layers + draft_attention_mask, + ) + if use_separate_draft_kv_cache: + return target_mamba_mask, target_attention_mask + return ( + target_mamba_mask + [False] * num_draft_layers, + target_attention_mask + draft_attention_mask, + ) + def get_states_bytes_per_layer(self, mapping) -> int: """Return the total bytes of Mamba state per layer, used for budgeting.""" tp_size = mapping.tp_size if not mapping.enable_attention_dp else 1 @@ -259,68 +298,8 @@ def get_states_bytes_per_layer(self, mapping) -> int: return state_bytes_per_layer -def _nemotron_hybrid_layer_masks(config, layer_mask): - pattern = config.hybrid_override_pattern - if layer_mask is None: - return ([c == "*" for c in pattern], [c == "M" for c in pattern]) - - # One-model speculative decoding: layer_mask may extend past the hybrid - # pattern; treat trailing positions as attention-only draft layers. - full_attn, mamba = [], [] - for i, include in enumerate(layer_mask): - if i < len(pattern): - is_attn = pattern[i] == "*" - is_mamba = pattern[i] == "M" - else: - is_attn, is_mamba = True, False - full_attn.append(is_attn and include) - mamba.append(is_mamba and include) - return full_attn, mamba - - -def _qwen3_hybrid_layer_masks(config, layer_mask): - full_attn, mamba = get_qwen3_hybrid_layer_masks(config) - if layer_mask is None: - return full_attn, mamba - - if len(layer_mask) < len(full_attn): - raise ValueError( - "layer_mask is shorter than the Qwen3 hybrid layer pattern") - base_len = len(full_attn) - new_full_attn, new_mamba = [], [] - for i, include in enumerate(layer_mask): - if i < base_len: - new_full_attn.append(full_attn[i] and include) - new_mamba.append(mamba[i] and include) - else: - new_full_attn.append(include) - new_mamba.append(False) - return new_full_attn, new_mamba - - -def _kimi_linear_layer_masks(config, layer_mask): - full_attn, kda = get_kimi_linear_layer_masks(config) - if layer_mask is None: - return full_attn, kda - - if len(layer_mask) < len(full_attn): - raise ValueError( - "layer_mask is shorter than the Kimi K3 hybrid layer pattern") - base_len = len(full_attn) - new_full_attn, new_kda = [], [] - for i, include in enumerate(layer_mask): - if i < base_len: - new_full_attn.append(full_attn[i] and include) - new_kda.append(kda[i] and include) - else: - new_full_attn.append(include) - new_kda.append(False) - return new_full_attn, new_kda - - def extract_mamba_kv_cache_params( config, - layer_mask: Optional[List[bool]] = None, spec_config=None, quant_config=None, ) -> MambaKVCacheParams: @@ -330,13 +309,9 @@ def extract_mamba_kv_cache_params( Args: config: HuggingFace model config of a hybrid Mamba model. - layer_mask: Optional per-layer keep mask used by one-model speculative - decoding. Entries past the underlying hybrid pattern length are - treated as attention-only draft layers. When provided, the caller - is responsible for already including spec layers in the mask. - spec_config: When `layer_mask` is None, used to extend the masks with - MTP/draft attention layers (no Mamba state) so they receive KV - cache entries. + spec_config: Optional speculative-decoding config used to describe + appended attention-only MTP/draft layers separately from target + layers. quant_config: Optional, used only to surface `mamba_ssm_cache_dtype`. Returns: @@ -348,16 +323,16 @@ def extract_mamba_kv_cache_params( num_heads = config.mamba_num_heads n_groups = config.n_groups head_dim = config.mamba_head_dim - full_attn_mask, mamba_mask = _nemotron_hybrid_layer_masks( - config, layer_mask) + pattern = config.hybrid_override_pattern + target_full_attn_mask = [layer_type == "*" for layer_type in pattern] + mamba_mask = [layer_type == "M" for layer_type in pattern] elif is_qwen3_hybrid(config): state_size = config.linear_key_head_dim conv_kernel = config.linear_conv_kernel_dim num_heads = config.linear_num_value_heads n_groups = config.linear_num_key_heads head_dim = config.linear_value_head_dim - full_attn_mask, mamba_mask = _qwen3_hybrid_layer_masks( - config, layer_mask) + target_full_attn_mask, mamba_mask = get_qwen3_hybrid_layer_masks(config) elif is_kimi_linear(config): # Kimi K3 KDA (Kimi Delta Attention) state, mapped onto the Mamba # cache-manager parametrization (see PythonMambaCacheManager): @@ -374,23 +349,17 @@ def extract_mamba_kv_cache_params( num_heads = lin["num_heads"] n_groups = lin["num_heads"] head_dim = lin["head_dim"] - full_attn_mask, mamba_mask = _kimi_linear_layer_masks( - config, layer_mask) + target_full_attn_mask, mamba_mask = get_kimi_linear_layer_masks(config) else: raise ValueError( f"{type(config).__name__} is not a supported hybrid Mamba config") - # When no explicit layer_mask is given, extend the masks here so MTP/draft - # layers (attention-only, no Mamba state) get KV cache entries. With an - # explicit layer_mask, the caller already encoded those entries. - if layer_mask is None and spec_config is not None: + num_draft_layers = 0 + if spec_config is not None: # Imported lazily to avoid a circular dependency between # config_utils and tensorrt_llm._torch.speculative. from ..speculative.utils import get_num_spec_layers - num_spec_layers = get_num_spec_layers(spec_config) - if num_spec_layers > 0: - full_attn_mask.extend([True] * num_spec_layers) - mamba_mask.extend([False] * num_spec_layers) + num_draft_layers = get_num_spec_layers(spec_config) or 0 mamba_ssm_cache_dtype = None if quant_config is not None: @@ -417,9 +386,9 @@ def extract_mamba_kv_cache_params( n_groups=n_groups, head_dim=head_dim, mamba_layer_mask=mamba_mask, - full_attention_layer_mask=full_attn_mask, + target_full_attention_layer_mask=target_full_attn_mask, num_mamba_layers=sum(mamba_mask), - num_full_attention_layers=sum(full_attn_mask), + num_draft_layers=num_draft_layers, dtype=resolve_hf_torch_dtype(config) or torch.bfloat16, mamba_ssm_cache_dtype=mamba_ssm_cache_dtype, ) @@ -456,6 +425,77 @@ def is_qwen_image_bench_config(config_dict: dict) -> bool: and required_multimodal_token_ids.issubset(config_dict)) +def _resolve_composite_torch_dtype(*config_dicts: dict) -> torch.dtype: + """Resolve a concrete torch dtype from one or more raw config dicts. + + Respects an explicit ``torch_dtype``/``dtype`` declaration (string or + ``torch.dtype``) from the first dict that provides one, and otherwise falls + back to ``bfloat16`` (TensorRT-LLM's default, matching the checkpoint). + """ + for config_dict in config_dicts: + for key in ("torch_dtype", "dtype"): + coerced = _coerce_torch_dtype(config_dict.get(key)) + if coerced is not None: + return coerced + return torch.bfloat16 + + +def _build_minicpmv4_6_config( + config_dict: dict) -> transformers.PretrainedConfig: + """Build the composite MiniCPM-V 4.6 config from a raw config.json dict. + + The top-level ``minicpmv4_6`` model_type is only known to + ``transformers>=5.7.0``; rebuild it locally so the PyTorch backend loads on + older releases. The inner text tower is normalized into a + ``Qwen3NextConfig`` through the shared ``Qwen35ConfigCompat`` shim (the same + path standalone Qwen3.5 dense uses), and the top-level ``model_type`` is kept + as ``minicpmv4_6`` so ``MULTIMODAL_PLACEHOLDER_REGISTRY`` lookup succeeds. + + TODO: this local builder is a transition path for the repo's pinned + transformers 5.5.4. Once the pin is bumped to ``>=5.7.0``, drop the + ``MiniCPMV4_6Config`` shim and build the config from the native + ``transformers.MiniCPMV4_6Config`` instead. + """ + from tensorrt_llm._torch.configs.minicpmv4_6 import MiniCPMV4_6Config + from tensorrt_llm._torch.models.modeling_qwen3_5 import Qwen35ConfigCompat + + # Extract + normalize the Qwen3.5 dense text tower (rope flatten, + # quantization inheritance, architectures -> Qwen3_5ForCausalLM). MiniCPM-V + # 4.6 always nests its language model under ``text_config``, so force + # extraction of that nested config (same path Qwen-Image-Bench uses). + text_dict = Qwen35ConfigCompat.normalize(config_dict, + require_text_config=True) + text_config = transformers.Qwen3NextConfig.from_dict(text_dict) + + # MiniCPM-V 4.6's config.json declares no torch_dtype/dtype (neither at the + # top level nor inside text_config). Several engine-build paths read + # ``pretrained_config.torch_dtype`` directly with no fallback -- the hybrid + # (mamba) KV-cache byte sizing and ``validate_and_set_mamba_ssm_cache_dtype`` + # -- so a ``None`` dtype crashes with ``None.itemsize``. Pin a concrete + # dtype on both the text tower and the composite config so every downstream + # consumer (config validation before model build, and the post_config swap + # that replaces pretrained_config with the inner text_config) agrees. + resolved_dtype = _resolve_composite_torch_dtype(config_dict, text_dict) + text_config.torch_dtype = resolved_dtype + + composite_config = MiniCPMV4_6Config( + text_config=text_config, + vision_config=config_dict.get("vision_config"), + insert_layer_id=config_dict.get("insert_layer_id", 6), + image_size=config_dict.get("image_size", 448), + drop_vision_last_layer=config_dict.get("drop_vision_last_layer", False), + image_token_id=config_dict.get("image_token_id"), + video_token_id=config_dict.get("video_token_id"), + downsample_mode=config_dict.get("downsample_mode", "16x"), + merge_kernel_size=config_dict.get("merge_kernel_size", (2, 2)), + merger_times=config_dict.get("merger_times", 1), + tie_word_embeddings=config_dict.get("tie_word_embeddings", False), + architectures=config_dict.get("architectures"), + ) + composite_config.torch_dtype = resolved_dtype + return composite_config + + # TODO: remove this once the transformers can support all of those models in _CONFIG_REGISTRY class LazyConfigDict(dict): @@ -479,6 +519,11 @@ def load_pretrained_config(model_name_or_path: str, trust_remote_code: bool = False, checkpoint_format: Optional[str] = None, **kwargs) -> transformers.PretrainedConfig: + if checkpoint_format in ("mistral", "mistral_large_3"): + from tensorrt_llm._torch.models.checkpoints.mistral.config_loader import \ + MistralConfigLoader + return MistralConfigLoader().load(model_name_or_path).pretrained_config + config_dict, _ = transformers.PretrainedConfig.get_config_dict( model_name_or_path, **kwargs) model_type = config_dict.get("model_type") @@ -548,6 +593,10 @@ def load_pretrained_config(model_name_or_path: str, config_class = _CONFIG_REGISTRY[model_type] model_config = config_class.from_pretrained(model_name_or_path, **kwargs) + elif model_type == "minicpmv4_6" or ( + architectures + and architectures[0] == "MiniCPMV4_6ForConditionalGeneration"): + model_config = _build_minicpmv4_6_config(config_dict) elif model_type in ("qwen3_5", "qwen3_5_text", "qwen3_5_moe", "qwen3_5_moe_text") or ( architectures and architectures[0] in ( diff --git a/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py b/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py index e0bd91ac52a8..cd6ee6adfd4d 100644 --- a/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py +++ b/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py @@ -39,6 +39,27 @@ KeyType: TypeAlias = Tuple[int, int, bool, bool, bool] +def _save_spec_decode_capture_state( + attn_metadata: Any, enable_spec_decode: bool) -> Optional[torch.Tensor]: + if not enable_spec_decode or not hasattr(attn_metadata, 'kv_lens_cuda'): + return None + return attn_metadata.kv_lens_cuda[:attn_metadata.num_seqs].clone() + + +def _restore_spec_decode_capture_state( + attn_metadata: Any, saved_kv_lens_cuda: Optional[torch.Tensor]) -> None: + if saved_kv_lens_cuda is None: + return + # Speculative decoding updates kv_lens_cuda in-place during every forward. + # CUDA graph warmup reuses one dummy request for multiple eager forwards, so + # letting those updates accumulate would make later warmups/capture advertise + # more KV tokens than the dummy request actually allocated. Restore the + # single-step input state outside the graph after each forward instead. + batch_size = saved_kv_lens_cuda.shape[0] + attn_metadata.kv_lens_cuda[:batch_size].copy_(saved_kv_lens_cuda) + attn_metadata.on_update_kv_lens() + + @dataclass class CUDAGraphRunnerConfig: """Configuration for the CUDAGraphRunner, passed from the ModelEngine.""" @@ -100,7 +121,7 @@ class CUDAGraphRunner: and low-level execution (capturing, resource management, replaying) for multiple graphs, keyed by (batch size, draft_len, is_first_draft). """ - WARMUP_STEPS = 2 + WARMUP_STEPS = 1 def __init__(self, config: CUDAGraphRunnerConfig): self.config = config @@ -132,6 +153,7 @@ def __init__(self, config: CUDAGraphRunnerConfig): # tensor reallocation from invalidating addresses baked into existing # CUDA graphs. Use allow_capture() context manager during warmup. self._capture_allowed = False + self.is_warmup_only = False def _create_shared_static_tensors(self): """Allocates static tensors sized for the largest possible batch.""" @@ -182,7 +204,9 @@ def _get_seq_len_mode( num_draft_tokens = self.spec_config.max_draft_len if is_spec_request else 0 # First draft if request.py_is_first_draft: - total_seq_len = len(request.get_tokens(0)) + # get_num_tokens is O(1); len(get_tokens(0)) marshals the + # whole O(seq_len) VecTokens into a Python list just for len. + total_seq_len = request.get_num_tokens(0) # With overlap scheduler disabled or dummy request or not assigned to a batch, elif not overlap_scheduler_enabled or request.is_dummy or request.py_batch_idx is None: total_seq_len = request.max_beam_num_tokens + num_draft_tokens @@ -321,7 +345,7 @@ def maybe_get_cuda_graph( key = self.get_graph_key(batch, new_tensors_device, spec_resource_manager, spec_metadata) - if key in self.graphs: + if key in self.graph_metadata: return self.graph_metadata[key][ "attn_metadata"], self.graph_metadata[key]["spec_metadata"], key @@ -346,6 +370,37 @@ def maybe_get_cuda_graph( graph_spec_metadata = None return graph_attn_metadata, graph_spec_metadata, key + def clear_capture_only_spec_state(self) -> int: + """Clear capture-scoped state from every cached graph SpecMetadata. + + ``create_cuda_graph_metadata`` shallow-copies the live SpecMetadata, so a + copy made while ``_run_capture_pass(force_non_greedy=True)`` is active + inherits ``_force_non_greedy_for_capture=True``. That copy is cached here + and reseated as the live spec_metadata on every later replay of its graph, + while the capture pass clears the flag on the base object only. Without + this cleanup the copies keep the flag forever and + ``_scan_one_model_sampling`` rewrites EVERY serving request's sampling + params to the synthetic capture values (temperature 0.7 / top_k 50 / + top_p 0.9), silently ignoring what the client asked for. + + The flag must NOT be cleared at copy time instead: it is load-bearing + *during* capture. It is what makes the pass-2 populate scan non-greedy on + parameter-less warmup requests, so that the advanced-sampling branch (not + the argmax fast path, and with the top-k/top-p kernels present) is the one + recorded into the graph. Clearing it here -- after the pass has captured + every graph -- keeps capture correct and serving clean. + + Returns the number of cached metadata objects cleared. + """ + cleared = 0 + for stored in self.graph_metadata.values(): + spec_metadata = stored.get("spec_metadata") + if spec_metadata is not None and getattr( + spec_metadata, "_force_non_greedy_for_capture", False): + spec_metadata._force_non_greedy_for_capture = False + cleared += 1 + return cleared + def needs_capture(self, key: KeyType): return self._capture_allowed and key not in self.graph_outputs @@ -378,8 +433,8 @@ def capture(self, forward_fn: Callable, initial_inputs: Dict[str, Any], enable_spec_decode: bool = False, - postprocess_fn: Optional[Callable] = None): - """Captures the forward pass for a given batch size.""" + postprocess_fn: Optional[Callable] = None) -> Any: + """Warm up and/or capture the forward pass for a graph key.""" batch_size = key[0] # [CUDA graph spec decode padding] # We pad input IDs/position IDs to the maximum draft length (token per request). @@ -407,9 +462,12 @@ def capture(self, capture_inputs = initial_inputs.copy() capture_inputs.update(sliced_static_tensors) + attn_metadata = capture_inputs["attn_metadata"] + saved_kv_lens_cuda = _save_spec_decode_capture_state( + attn_metadata, enable_spec_decode) self.graph_metadata[key] = { - "attn_metadata": initial_inputs["attn_metadata"], + "attn_metadata": attn_metadata, "spec_metadata": initial_inputs.get("spec_metadata", None), } @@ -422,27 +480,38 @@ def _setup_spec_decoding_and_forward(key: KeyType, forward_fn: Callable, capture_inputs['attn_metadata'].use_spec_decoding = True return forward_fn(capture_inputs) - # We have to do warm up runs to initialize PyTorch's - # internal states according to the docs: - # https://pytorch.org/docs/stable/notes/cuda.html#cuda-graph-semantics - # This also lets us initialize states in the attn_metadata. - graph = torch.cuda.CUDAGraph() + output = None with with_multi_stream(True), piecewise_cuda_graph(False): + # We have to do a warmup run to initialize PyTorch's internal + # states according to the docs: + # https://pytorch.org/docs/stable/notes/cuda.html#cuda-graph-semantics + # This also lets us initialize states in the attn_metadata and + # resize the shared attention workspace before any graph is captured. for _ in range(self.WARMUP_STEPS): - _setup_spec_decoding_and_forward(key, forward_fn, - capture_inputs) + output = _setup_spec_decoding_and_forward( + key, forward_fn, capture_inputs) if postprocess_fn is not None: postprocess_fn(capture_inputs) + _restore_spec_decode_capture_state(attn_metadata, + saved_kv_lens_cuda) + if self.is_warmup_only: + return output + + graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph, pool=self.memory_pool): output = _setup_spec_decoding_and_forward( key, forward_fn, capture_inputs) if postprocess_fn is not None: postprocess_fn(capture_inputs) + _restore_spec_decode_capture_state(attn_metadata, + saved_kv_lens_cuda) self.graphs[key] = graph - self.graph_outputs[key] = make_weak_ref(output) + graph_output = make_weak_ref(output) + self.graph_outputs[key] = graph_output self.memory_pool = graph.pool() + return graph_output def replay(self, key: KeyType, current_inputs: Dict[str, Any]) -> Optional[torch.Tensor]: @@ -704,7 +773,7 @@ class EncoderCUDAGraphRunner: Restricted to `TrtllmAttentionMetadata` — FlashInfer's per-batch planner state is not compatible with CUDA graph capture/replay. """ - WARMUP_STEPS = 2 + WARMUP_STEPS = 1 def __init__(self, config: EncoderCUDAGraphRunnerConfig): self.config = config @@ -730,6 +799,7 @@ def __init__(self, config: EncoderCUDAGraphRunnerConfig): self.cuda_graph_meta_buffers = get_memory_buffers() self._capture_allowed = False + self.is_warmup_only = False # CUDA graph H2D memcpy nodes require pinned host sources. In CC mode # prefer_pinned() is false: pageable host buffers are preferred, so the @@ -916,7 +986,7 @@ def maybe_get_cuda_graph( or not is_padding_successful: return None, None - if key in self.graphs: + if key in self.graph_metadata: return self.graph_metadata[key]["attn_metadata"], key # New key not yet captured. Only create metadata if capture is @@ -980,8 +1050,8 @@ def capture( key: EncoderKeyType, forward_fn: Callable[[Dict[str, Any]], Any], inputs: Dict[str, Any], - ) -> None: - """Capture a CUDA graph for the given key.""" + ) -> Any: + """Warm up and/or capture the forward pass for a graph key.""" _, padded_num_tokens, _ = key sliced_static_tensors = { @@ -1003,17 +1073,21 @@ def capture( attn_md = capture_inputs["attn_metadata"] - self.graph_metadata[key] = { - "attn_metadata": attn_md, - } + self.graph_metadata[key] = {"attn_metadata": attn_md} - graph = torch.cuda.CUDAGraph() - # Warmup runs required by CUDA graph semantics. See - # https://pytorch.org/docs/stable/notes/cuda.html#cuda-graph-semantics + output = None with with_multi_stream(True), piecewise_cuda_graph(False): + # Warmup runs required by CUDA graph semantics. See + # https://pytorch.org/docs/stable/notes/cuda.html#cuda-graph-semantics + # Warmups initialize PyTorch and attention metadata state, and + # resize the shared attention workspace before any graph is captured. for _ in range(self.WARMUP_STEPS): - forward_fn(capture_inputs) + output = forward_fn(capture_inputs) + + if self.is_warmup_only: + return output + graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph, pool=self.memory_pool): if self._capture_h2d_copy: # H2D copies for captured inside the graph: at replay @@ -1034,8 +1108,10 @@ def capture( "Encoder CUDA graph does not support nested tensor outputs. " "Disable encoder CUDA graphs for models with ragged outputs.") self.graphs[key] = graph - self.graph_outputs[key] = make_weak_ref(output) + graph_output = make_weak_ref(output) + self.graph_outputs[key] = graph_output self.memory_pool = graph.pool() + return graph_output def replay( self, diff --git a/tensorrt_llm/_torch/pyexecutor/hang_detector.py b/tensorrt_llm/_torch/pyexecutor/hang_detector.py index 2ae692ed5902..f6dde7c58959 100644 --- a/tensorrt_llm/_torch/pyexecutor/hang_detector.py +++ b/tensorrt_llm/_torch/pyexecutor/hang_detector.py @@ -100,6 +100,7 @@ def __init__( self.lock = threading.Lock() self.active = False self._detected = False + self._status_providers: list[Callable[[], str]] = [] def start(self): """Enable hang detection.""" @@ -113,13 +114,38 @@ def run_loop(): self.loop_thread = threading.Thread(target=run_loop, daemon=True, name="hang_detector_loop") self.loop_thread.start() - async def _detect_hang(self): + def register_status_provider(self, provider: Callable[[], str]) -> None: + """Register a nonblocking callable that returns status to dump on hang detection.""" + with self.lock: + self._status_providers.append(provider) + + async def _detect_hang(self) -> None: await asyncio.sleep(self.timeout) with self.lock: - self._detected = True - logger.error(f"Hang detected after {self.timeout} seconds.") + status_providers = tuple(self._status_providers) + + # All diagnostics are best-effort: nothing may prevent on_detected() + # (hard-kill propagation) from firing. + _best_effort_log_error(f"Hang detected after {self.timeout} seconds.") + for provider in status_providers: + try: + status = provider() + if status: + _best_effort_log_error(status) + except Exception as error: # noqa: BLE001 - isolate diagnostic providers + _best_effort_log_error( + f"HangDetector: status provider failed with {type(error).__name__}: {error}" + ) + try: print_all_stacks() - self.on_detected() + except Exception: # noqa: BLE001 - stack dump must not block hard kill + pass + + # Set _detected last so observers (and tests) see it only once + # diagnostics are done and on_detected is about to fire. + with self.lock: + self._detected = True + self.on_detected() def detected(self): """Return True if hang is detected.""" diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py b/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py index 46d3c55a0771..4b7bde587dbf 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py @@ -17,12 +17,14 @@ import os import sys from collections import OrderedDict, defaultdict +from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Dict, Iterable, List, NamedTuple, Optional, Sequence, Tuple, Union import numpy as np import torch from strenum import StrEnum +from tensorrt_llm._torch.disaggregation.resource.page import MapperKind from tensorrt_llm._torch.distributed.communicator import Distributed, ReduceOp from tensorrt_llm._utils import ( TensorWrapper, @@ -45,6 +47,7 @@ GPU_LEVEL, AttentionLayerConfig, AttnLifeCycle, + BatchDesc, BufferConfig, CacheLevel, CacheTierConfig, @@ -53,15 +56,17 @@ DiskCacheTierConfig, GpuCacheTierConfig, HostCacheTierConfig, + KVCacheDesc, KVCacheEventManager, KVCacheIterationStatsDelta, LayerId, - LifeCycleId, PageIndexMode, + PlannedDropHandle, PoolGroupPeakBlockStats, ReuseScope, SwaScratchReuseConfig, TokenIdExt, + _introspection, _KVCache, exact_div, gen_multimodal_cache_key_tokens, @@ -81,6 +86,8 @@ KVCacheV2IterationStatsReport, KVCacheV2LifeCycleIterationStats, KVCacheV2PoolGroupIterationStats, + KVCacheV2SsmLifeCycleIterationStats, + KVCacheV2SsmSnapshotIterationStats, ) from .llm_request import LlmRequest, LlmRequestState, SamplingConfig, get_draft_token_length from .resource_manager import ( @@ -121,8 +128,8 @@ class Role: # Sparse-attention per-layer index-K cache (MiniMax-M3 and similar # sparse-block-selection backends). Registered as a native V2 # BufferConfig on sparse layers via the extra_buffers_per_layer hook on - # _build_cache_config, so allocation, free, slot reuse, and prefix - # reuse share the lifecycle of the main K/V buffers for the same layer. + # _build_base_config, so allocation, free, slot reuse, and prefix reuse + # share the lifecycle of the main K/V buffers for the same layer. INDEX_KEY = DataRole("index_key") ALL = DataRole("all") @@ -130,6 +137,90 @@ class Role: class BlockReusePolicy(StrEnum): ALL_REUSABLE = "all_reusable" PER_REQUEST = "per_request" + PER_CONVERSATION = "per_conversation" + + +def _request_conversation_id(request: LlmRequest) -> Optional[str]: + if request.is_dummy_request: + return None + conversation_params = request.py_conversation_params + if conversation_params is None: + return None + conversation_id = conversation_params.conversation_id.strip() + return conversation_id or None + + +@dataclass(slots=True) +class _ConversationState: + current_request_id: Optional[int] = None + planned_drop_handle: Optional[PlannedDropHandle] = None + + +class ConversationManager: + """Track the current request and drop plan for each conversation.""" + + def __init__(self) -> None: + self._conversation_states: Dict[str, _ConversationState] = {} + + def save_drop_plan(self, request: LlmRequest, kv_cache: _KVCache) -> None: + """Save a completed context's drop plan and apply the preceding plan on success.""" + request_id = request.py_request_id + conversation_id = _request_conversation_id(request) + if conversation_id is None: + return + + state = self._conversation_states[conversation_id] + if state.current_request_id != request_id: + return + + drop_handle = kv_cache.plan_committed_block_drop() + if drop_handle is None: + logger.warning( + f"Committed blocks for request {request_id} in conversation " + f"{conversation_id} have been dropped." + ) + else: + previous_handle = state.planned_drop_handle + state.planned_drop_handle = drop_handle + if previous_handle is not None: + previous_handle.drop() + + self.finish_request(request) + + def prepare_request(self, request: LlmRequest) -> None: + """Register a context request unless its conversation has another active one.""" + conversation_id = _request_conversation_id(request) + if conversation_id is None: + return + request_id = request.py_request_id + state = self._conversation_states.setdefault(conversation_id, _ConversationState()) + current_request_id = state.current_request_id + if current_request_id is not None and current_request_id != request_id: + logger.warning( + f"Conversation {conversation_id} already has current request " + f"{current_request_id}. Request {request_id} will ignore " + "conversation params." + ) + return + + state.current_request_id = request_id + + def finish_request(self, request: LlmRequest) -> None: + """Clear a request as active while preserving any saved drop plan.""" + conversation_id = _request_conversation_id(request) + if conversation_id is None: + return + state = self._conversation_states.get(conversation_id) + if state is None or state.current_request_id != request.py_request_id: + return + + state.current_request_id = None + if state.planned_drop_handle is None: + self._conversation_states.pop(conversation_id) + + def clear(self) -> None: + """Clear state after reusable KV-cache blocks have been cleared.""" + self._conversation_states.clear() def _estimate_full_attn_size_per_token( @@ -142,6 +233,46 @@ def _estimate_full_attn_size_per_token( ) +def _compute_auto_host_tier_quota( + quota: int, + local_ranks: int, + mem_available: float, + memlock_limit: float, +) -> int: + """Compute the auto-provisioned host cache tier quota for a single rank. + + The host tier backs the MAX_UTILIZATION scheduler's suspend/resume path, + so it must be positive. It defaults to the device quota, capped by the + per-node available-memory budget shared with co-located ranks and by the + pinnable-memory (RLIMIT_MEMLOCK) limit. + + Args: + quota: Device (GPU) KV cache quota in bytes; must be positive. + local_ranks: Number of ranks co-located on this physical node. + mem_available: Available host memory in bytes, or ``float("inf")`` + if unknown. + memlock_limit: RLIMIT_MEMLOCK soft limit in bytes, or + ``float("inf")`` if unlimited or unknown. + + Returns: + Host cache tier quota in bytes (always positive). + """ + candidates = [quota] + if mem_available != float("inf"): + candidates.append(int(mem_available / local_ranks * 0.5)) + if memlock_limit != float("inf"): + candidates.append(int(memlock_limit * 0.8)) + host_quota = min(candidates) + if host_quota <= 0: + logger.warning( + f"KV cache manager v2 auto host tier sizing computed a " + f"non-positive quota ({host_quota}); falling back to the " + f"device quota {quota / (1 << 30):.2f}GiB" + ) + host_quota = quota + return host_quota + + def _estimate_swa_cache_size( layer_sizes: Sequence[int], attention_windows: Sequence[Optional[int]], @@ -635,6 +766,7 @@ def __init__( execution_stream: Optional[torch.cuda.Stream] = None, is_disagg: bool = False, enable_stats: bool = False, + num_reserved_index_slots: int = 1, **kwargs, ) -> None: self.mapping = mapping @@ -657,6 +789,8 @@ def __init__( layer_mask=layer_mask, ) self.is_draft = is_draft + # Set True by a compression manager; generation-step resize then leaves history untouched. + self.kv_compression_manages_history: bool = False self.enable_swa_scratch_reuse = ( kv_cache_config.enable_swa_scratch_reuse and not self.is_draft ) @@ -797,6 +931,7 @@ def append_to_kv_heads_per_layer( self.is_vswa = len(set(self.max_attention_window_vec)) > 1 + max_util_for_resume = kv_cache_config.max_util_for_resume quota = sys.maxsize if ( kv_cache_config.max_gpu_total_bytes is not None @@ -808,7 +943,7 @@ def append_to_kv_heads_per_layer( quota_from_max_tokens = int( math.ceil( self._get_quota_from_max_tokens(kv_cache_config.max_tokens) - / kv_cache_config.max_util_for_resume + / max_util_for_resume ) ) quota = min(quota, quota_from_max_tokens) @@ -822,13 +957,13 @@ def append_to_kv_heads_per_layer( "Quota not set. Check kv_cache_config.max_tokens or kv_cache_config.max_gpu_total_bytes" ) - # Sync KV cache token capacity across ranks so all ranks allocate - # the same number of tokens and the scheduler produces identical - # batches. Normalize to token count before the allreduce because - # bytes_per_token varies across PP ranks (different local layers). + # Sync resumable token capacity across ranks so the scheduler produces + # identical batches. Normalize to tokens because cache costs vary + # across PP ranks, including fixed per-rank costs. if mapping.world_size > 1: dist = Distributed.get(mapping) - max_tokens = self._get_max_tokens_from_quota(quota) + resumable_quota = int(quota * max_util_for_resume) + max_tokens = self._get_max_tokens_from_quota(resumable_quota) max_tokens = dist.allreduce(max_tokens, op=ReduceOp.MIN) # inf max_tokens means all layers are SWA and every rank quota can # fit all SWA fixed cache. @@ -837,7 +972,10 @@ def append_to_kv_heads_per_layer( # token↔quota round-trip is not identity when SWA layers # dominate (full_attn_size_per_token==0), so clamp to guard # against a bogus inflation (nvbugs/6418103). - quota = min(quota, self._get_quota_from_max_tokens(max_tokens)) + synced_quota = int( + math.ceil(self._get_quota_from_max_tokens(max_tokens) / max_util_for_resume) + ) + quota = min(quota, synced_quota) logger.info(f"KV cache manager v2 device quota set to {quota / (1 << 30)}GiB") @@ -856,6 +994,14 @@ def append_to_kv_heads_per_layer( # memory and pinnable memory limit to avoid allocation failures. import resource + # Rank-aware host tier sizing: divide the per-node memory budget + # by the number of ranks sharing this physical node. Each rank + # independently provisions a host tier; without this, N + # co-located ranks each reserving a device-quota-sized block can + # OOM the host (observed on GB300 NVL72 with 4 ranks/node and + # ~170GiB device quota each on a 975GiB node). + local_ranks = max(1, Distributed.get(mapping).local_world_size) + try: mem_available = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_AVPHYS_PAGES") except (ValueError, OSError): @@ -865,14 +1011,14 @@ def append_to_kv_heads_per_layer( memlock_limit = _soft if _soft != resource.RLIM_INFINITY else float("inf") except (ValueError, OSError): memlock_limit = float("inf") - candidates = [quota] - if mem_available != float("inf"): - candidates.append(int(mem_available * 0.5)) - if memlock_limit != float("inf"): - candidates.append(int(memlock_limit * 0.8)) - host_quota = min(candidates) - if host_quota <= 0: - host_quota = quota + host_quota = _compute_auto_host_tier_quota( + quota, local_ranks, mem_available, memlock_limit + ) + logger.info( + f"KV cache manager v2 auto host tier sizing: " + f"{local_ranks} co-located rank(s) on this node, " + f"available host memory {mem_available / (1 << 30):.2f}GiB" + ) if host_quota > 0: cache_tiers.append(HostCacheTierConfig(quota=int(host_quota))) logger.info( @@ -891,12 +1037,12 @@ def append_to_kv_heads_per_layer( self.vocab_size = vocab_size - config = self._build_cache_config( + config = self._build_base_config( kv_cache_config, tokens_per_block=tokens_per_block, - vocab_size=vocab_size, cache_tiers=cache_tiers, ) + config = self._build_cache_config(config) self.kv_cache_manager_py_config = config @@ -910,12 +1056,7 @@ def append_to_kv_heads_per_layer( "Retrying without host cache tier." ) cache_tiers_gpu_only = [t for t in cache_tiers if isinstance(t, GpuCacheTierConfig)] - config = self._build_cache_config( - kv_cache_config, - tokens_per_block=tokens_per_block, - vocab_size=vocab_size, - cache_tiers=cache_tiers_gpu_only, - ) + config = replace(config, cache_tiers=cache_tiers_gpu_only) cache_tiers = cache_tiers_gpu_only self.kv_cache_manager_py_config = config self.impl = KVCacheManagerPy(config, event_manager=self.event_manager) @@ -990,10 +1131,17 @@ def append_to_kv_heads_per_layer( self.enable_block_reuse = kv_cache_config.enable_block_reuse self.enable_partial_reuse = kv_cache_config.enable_partial_reuse self.disk_prefetch_num_reqs = kv_cache_config.disk_prefetch_num_reqs + enable_conversation_manager = ( + self.enable_block_reuse + and self.block_reuse_policy == BlockReusePolicy.PER_CONVERSATION + and not self.is_draft + ) + self.conversation_manager = ConversationManager() if enable_conversation_manager else None # With pipeline parallelism, multiple microbatches can be in-flight # simultaneously, so we need slots for all concurrent sequences. - # Plus 1 for cuda graph dummy request. + # Reserve stable slots for persistent request IDs such as CUDA-graph + # padding requests. The default preserves the main-branch allocation. # In disaggregated mode, use a coefficient of 2: at any moment up to # `max_num_sequences` requests can be actively generating while another # up to `max_num_sequences` requests are still in KV transfer @@ -1001,10 +1149,15 @@ def append_to_kv_heads_per_layer( # capacity lets the next batch of active requests acquire slots without # waiting for the previous batch's transfers to finish. max_num_sequences = max_batch_size * mapping.pp_size - index_mapper_capacity = max_num_sequences * (2 if is_disagg else 1) + 1 + assert num_reserved_index_slots >= 0, "num_reserved_index_slots must be non-negative" + index_mapper_capacity = ( + max_num_sequences * (2 if is_disagg else 1) + num_reserved_index_slots + ) logger.info( f"KVCacheManagerV2: IndexMapper capacity={index_mapper_capacity} " - f"(max_num_sequences={max_num_sequences}, is_disagg={is_disagg}, max_beam_width={max_beam_width})" + f"(max_num_sequences={max_num_sequences}, is_disagg={is_disagg}, " + f"num_reserved_index_slots={num_reserved_index_slots}, " + f"max_beam_width={max_beam_width})" ) self.index_mapper = IndexMapper(index_mapper_capacity, max_beam_width) self._early_freed_index_requests: set[int] = set() @@ -1012,26 +1165,50 @@ def append_to_kv_heads_per_layer( self._log_kv_cache_pool_lifecycle_mapping() - def _prepare_page_table_tensor(self, index_mapper_capacity: int) -> None: + def _get_pool_roles(self, pool_id: int) -> Tuple[DataRole, Optional[DataRole]]: + """Return the roles represented by the two page-table index lanes. + + When present, role B must be addressable from role A using a constant + page-index offset. + """ + role_b = None if self.kv_cache_type == CacheTypeCpp.SELFKONLY else Role.VALUE + return Role.KEY, role_b + + def _get_block_scale_role(self, role_a: DataRole) -> Optional[DataRole]: + if self.dtype != DataType.NVFP4 or role_a != Role.KEY: + return None + return Role.KEY_BLOCK_SCALE + + def _build_pool_mapping_tensors(self): + """Build the (kv_cache_pool_pointers, kv_cache_pool_mapping) tensors. + + An overridable hook for subclasses whose pools coalesce extra + per-layer buffers alongside K/V. + """ kv_cache_pool_pointers_list = [] kv_cache_pool_mapping_list = [] block_scale_pool_pointers_list = [] if self.enable_swa_scratch_reuse: for layer_id in typed_range(LayerId(self.num_local_layers)): + pool_id = self.impl.get_layer_group_id(layer_id) + role_a, _ = self._get_pool_roles(pool_id) kv_cache_pool_pointers_list.append( [ self.impl.get_mem_pool_base_address( - layer_id, Role.KEY, PageIndexMode.PER_LAYER + layer_id, role_a, PageIndexMode.PER_LAYER ), 0, ] ) if self.dtype == DataType.NVFP4: + block_scale_role = self._get_block_scale_role(role_a) block_scale_pool_pointers_list.append( [ self.impl.get_mem_pool_base_address( - layer_id, Role.KEY_BLOCK_SCALE, PageIndexMode.PER_LAYER - ), + layer_id, block_scale_role, PageIndexMode.PER_LAYER + ) + if block_scale_role is not None + else 0, 0, ] ) @@ -1039,63 +1216,82 @@ def _prepare_page_table_tensor(self, index_mapper_capacity: int) -> None: else: for pool_id in range(self.num_pools): layer_id = self.impl.layer_grouping[pool_id][0] - kv_cache_pool_pointers_list.append( - [ - self.impl.get_mem_pool_base_address( - layer_id, Role.KEY, PageIndexMode.SHARED - ), - 0, - ] + role_a, _ = self._get_pool_roles(pool_id) + key_base_addr = self.impl.get_mem_pool_base_address( + layer_id, role_a, PageIndexMode.SHARED ) + kv_cache_pool_pointers_list.append([key_base_addr, 0]) if self.dtype == DataType.NVFP4: - block_scale_pool_pointers_list.append( - [ + # The KEY/scale pointers are a (rep-layer, 0-offset) origin + # against which each layer's kv_cache_pool_mapping offset is + # resolved. The block-scale origin must reproduce the SAME + # per-layer offset() as KEY, so mirror the KEY base for the + # same representative layer and shift it back by that layer's + # offset. For the base manager offset(rep) == 0, so this is + # just the rep layer's scale base; for address-ranked + # subclasses (MiniMax-M3) offset(rep) may be non-zero, and the + # shift lands the origin on the pool's slot-0 scale address. + # This keeps block_scale_offset == offset without depending on + # the non-contractual layer_grouping order. + block_scale_role = self._get_block_scale_role(role_a) + if block_scale_role is not None: + rep_offset = self._kv_pool_mapping_offset(layer_id, pool_id, key_base_addr) + scale_stride = ( + self.get_layer_bytes_per_token(layer_id, block_scale_role) + * self.kv_factor + * self.tokens_per_block + ) + scale_base_addr = ( self.impl.get_mem_pool_base_address( - layer_id, Role.KEY_BLOCK_SCALE, PageIndexMode.SHARED - ), - 0, - ] - ) + layer_id, block_scale_role, PageIndexMode.SHARED + ) + - rep_offset * scale_stride + ) + else: + scale_base_addr = 0 + block_scale_pool_pointers_list.append([scale_base_addr, 0]) for layer_id in typed_range(LayerId(self.num_local_layers)): layer_group_id = self.impl.get_layer_group_id(layer_id) - if self.dtype != DataType.NVFP4: - key_base_addr = kv_cache_pool_pointers_list[layer_group_id][0] + role_a, role_b = self._get_pool_roles(layer_group_id) + index_base_addr = kv_cache_pool_pointers_list[layer_group_id][0] + if role_a == Role.KEY: + offset = self._kv_pool_mapping_offset(layer_id, layer_group_id, index_base_addr) + else: addr_offset = ( - self.impl.get_mem_pool_base_address( - layer_id, Role.KEY, PageIndexMode.SHARED - ) - - key_base_addr + self.impl.get_mem_pool_base_address(layer_id, role_a, PageIndexMode.SHARED) + - index_base_addr + ) + offset_divisor = self.impl.get_page_stride(layer_id, role_a) + if role_b is not None: + offset_divisor *= self.kv_factor + offset = exact_div( + addr_offset, + offset_divisor, ) + + if self.dtype != DataType.NVFP4 or role_a != Role.KEY: + block_scale_offset = None else: - key_base_addr = kv_cache_pool_pointers_list[layer_group_id][0] - block_scale_base_addr = block_scale_pool_pointers_list[layer_group_id][0] - addr_offset = ( - self.impl.get_mem_pool_base_address( - layer_id, Role.KEY, PageIndexMode.SHARED + block_scale_role = self._get_block_scale_role(role_a) + if block_scale_role is None: + block_scale_offset = None + else: + block_scale_base_addr = block_scale_pool_pointers_list[layer_group_id][0] + block_scale_addr_offset = ( + self.impl.get_mem_pool_base_address( + layer_id, block_scale_role, PageIndexMode.SHARED + ) + - block_scale_base_addr ) - - key_base_addr - ) - block_scale_addr_offset = ( - self.impl.get_mem_pool_base_address( - layer_id, Role.KEY_BLOCK_SCALE, PageIndexMode.SHARED + block_scale_offset = exact_div( + block_scale_addr_offset, + self.get_layer_bytes_per_token(layer_id, block_scale_role) + * self.kv_factor + * self.tokens_per_block, ) - - block_scale_base_addr - ) - block_scale_offset = exact_div( - block_scale_addr_offset, - self.get_layer_bytes_per_token(layer_id, Role.KEY_BLOCK_SCALE) - * self.kv_factor - * self.tokens_per_block, - ) - offset = exact_div( - addr_offset, - self.get_layer_bytes_per_token(layer_id, Role.KEY) - * self.kv_factor - * self.tokens_per_block, - ) - if self.dtype == DataType.NVFP4: + if block_scale_offset is not None: assert block_scale_offset == offset, ( "Block scale offset and offset should be the same" ) @@ -1110,18 +1306,22 @@ def _prepare_page_table_tensor(self, index_mapper_capacity: int) -> None: [pool_pointers[1], block_scale_pool_pointers[1]], ] - self.kv_cache_pool_pointers = torch.tensor( + kv_cache_pool_pointers = torch.tensor( kv_cache_pool_pointers_list, dtype=torch.int64, device="cpu", pin_memory=prefer_pinned(), ) - self.kv_cache_pool_mapping = torch.tensor( + kv_cache_pool_mapping = torch.tensor( kv_cache_pool_mapping_list, dtype=torch.int32, device="cpu", pin_memory=prefer_pinned(), ) + return kv_cache_pool_pointers, kv_cache_pool_mapping + + def _prepare_page_table_tensor(self, index_mapper_capacity: int) -> None: + self.kv_cache_pool_pointers, self.kv_cache_pool_mapping = self._build_pool_mapping_tensors() self.index_scales = torch.empty( self.num_pools, dtype=torch.int32, pin_memory=prefer_pinned(), device="cpu" ) @@ -1130,12 +1330,13 @@ def _prepare_page_table_tensor(self, index_mapper_capacity: int) -> None: ) for pool_id in range(self.num_pools): layer_id = self.impl.layer_grouping[pool_id][0] - self.index_scales[pool_id] = self.impl.get_page_index_scale(layer_id, Role.KEY) - if self.kv_cache_type != CacheTypeCpp.SELFKONLY: + role_a, role_b = self._get_pool_roles(pool_id) + self.index_scales[pool_id] = self.impl.get_page_index_scale(layer_id, role_a) + if role_b is not None: self.kv_offset[pool_id] = exact_div( - self.impl.get_mem_pool_base_address(layer_id, Role.VALUE, PageIndexMode.SHARED) - - self.impl.get_mem_pool_base_address(layer_id, Role.KEY, PageIndexMode.SHARED), - self.impl.get_page_stride(layer_id, Role.KEY), + self.impl.get_mem_pool_base_address(layer_id, role_b, PageIndexMode.SHARED) + - self.impl.get_mem_pool_base_address(layer_id, role_a, PageIndexMode.SHARED), + self.impl.get_page_stride(layer_id, role_a), ) else: self.kv_offset[pool_id] = 0 @@ -1156,6 +1357,29 @@ def _prepare_page_table_tensor(self, index_mapper_capacity: int) -> None: if self.enable_swa_scratch_reuse: self._prepare_swa_scratch_copy_tensors(index_mapper_capacity) + def _kv_pool_mapping_offset( + self, layer_id: LayerId, layer_group_id: int, key_base_addr: int + ) -> int: + """Per-layer offset recorded in ``kv_cache_pool_mapping``. + + The default derives the layer's position within its pool from the K + base address, assuming every layer contributes exactly K(+V) to the + pool slot so the layer stride is uniform. Managers whose pool slots + may interleave extra per-layer buffers between layers (non-uniform + layer strides — e.g. MiniMax M3 when the index-K buffer coalesces + into the K/V pool) must override this with a positional formula. + """ + addr_offset = ( + self.impl.get_mem_pool_base_address(layer_id, Role.KEY, PageIndexMode.SHARED) + - key_base_addr + ) + return exact_div( + addr_offset, + self.get_layer_bytes_per_token(layer_id, Role.KEY) + * self.kv_factor + * self.tokens_per_block, + ) + def _get_runtime_cache_size_layer_components(self) -> tuple[List[int], List[Optional[int]]]: layer_sizes = [] attention_windows = [] @@ -1253,7 +1477,8 @@ def _get_event_window_sizes_by_layer_group(self) -> Dict[int, int]: # mixed windows in one group, this needs to fan out per-layer. def get_event_window_size(layer_id: int) -> int: - window_size = self.kv_cache_manager_py_config.layers[layer_id].sliding_window_size + layer_config = self.kv_cache_manager_py_config.layers[layer_id] + window_size = getattr(layer_config, "sliding_window_size", None) return self.max_seq_len if window_size is None else int(window_size) return { @@ -1262,14 +1487,19 @@ def get_event_window_size(layer_id: int) -> int: } def _format_kv_cache_pool_lifecycle_entry(self, layer_id: LayerId, role: DataRole) -> str: - attr = self.impl._storage.get_buffer_attr(layer_id, role) - pool_group_id = self.impl._storage.get_pool_group_index(attr.life_cycle_id) - lifecycle = self.impl._life_cycles.get_life_cycle(attr.life_cycle_id) - return ( - f"role={str(role)}, pool_group_id={int(pool_group_id)}, " - f"lifecycle_id={int(attr.life_cycle_id)}, " - f"lifecycle={lifecycle}" - ) + for pool_group in self.impl.pool_group_descs: + for variant in pool_group.slot_desc.variants: + for coalesced in variant.coalesced_buffers: + for buffer_id in coalesced.buffer_ids: + if int(buffer_id.layer_id) == int(layer_id) and str(buffer_id.role) == str( + role + ): + return ( + f"role={role!s}, " + f"pool_group_id={int(pool_group.pool_group_index)}, " + f"layer_group_id={int(variant.layer_group_id)}" + ) + return f"role={role!s}, pool_group_id=?, layer_group_id=?" def _log_kv_cache_pool_lifecycle_mapping(self) -> None: entries = OrderedDict() @@ -1304,9 +1534,11 @@ def _prepare_swa_scratch_copy_tensors(self, index_mapper_capacity: int) -> None: for local_layer_idx in range(self.num_local_layers): layer_id = LayerId(local_layer_idx) pool_id = self.layer_to_pool_mapping_dict[layer_id] - roles = [Role.KEY, Role.VALUE] - if self.kv_cache_type == CacheTypeCpp.SELFKONLY: - roles[1] = Role.KEY + role_a, role_b = self._get_pool_roles(pool_id) + roles = [ + role_a, + role_a if role_b is None else role_b, + ] for role_idx, role in enumerate(roles): converter = self.impl.get_page_index_converter(layer_id, role) if converter.expansion != 1: @@ -1489,17 +1721,83 @@ def _copy_batch_block_offsets_per_layer( non_blocking=True, ) - def _build_cache_config( + def _build_base_config( self, kv_cache_config: KvCacheConfig, *, tokens_per_block: int, - vocab_size: int | None, cache_tiers: List[CacheTierConfig], ) -> KVCacheManagerConfigPy: - # Kept in the virtual method contract for cache-manager subclasses. - # The generic C++ config no longer stores the vocabulary size. - del vocab_size + """Build the general cache configuration used by most models. + + Models that need a custom configuration should subclass + :class:`KVCacheManagerV2` and override :meth:`_build_cache_config` to + update the necessary fields. + """ + scratch_reuse_config = None + if self.enable_swa_scratch_reuse: + # Context requests allocate num_extra_kv_tokens for spec decoding. + # They should not count toward the scratch range. + scratch_reuse_config = SwaScratchReuseConfig(max_rewind_len=self.num_extra_kv_tokens) + + typical_step = None + constraints = [] + if kv_cache_config.pool_ratio is None: + typical_seq_len = self._get_typical_seq_len(kv_cache_config) + if typical_seq_len is not None and typical_seq_len > self.max_seq_len: + raise ValueError( + f"kv_cache_config.avg_seq_len ({typical_seq_len}) must be less than or " + f"equal to max_seq_len ({self.max_seq_len})" + ) + + if typical_seq_len is not None: + # Model one context request and enough generation requests to fill + # max_batch_size without over-provisioning windowed cache pools. + context_capacity = ( + self.max_num_tokens if self.max_num_tokens is not None else typical_seq_len + ) + self.num_extra_kv_tokens + generation_history_length = max(0, typical_seq_len - self.max_draft_len - 1) + typical_step = BatchDesc( + [KVCacheDesc(capacity=context_capacity, history_length=0)] + + [ + KVCacheDesc( + capacity=typical_seq_len, + history_length=generation_history_length, + ) + ] + * (self.max_batch_size - 1) + ) + + # CUDA graph generation warmup uses one request at max_seq_len and + # enough minimal decode requests to fill max_batch_size. + min_decode_capacity = 1 + self.max_draft_len + self.num_extra_kv_tokens + constraints.append( + BatchDesc( + [ + KVCacheDesc( + capacity=self.max_seq_len, + history_length=self.max_seq_len - 1, + ) + ] + + [KVCacheDesc(capacity=min_decode_capacity, history_length=0)] + * (self.max_batch_size - 1) + ) + ) + + # General and chunked-prefill warmup uses one fresh context request + # at the per-iteration token budget. + if self.max_num_tokens is not None: + constraints.append( + BatchDesc( + [ + KVCacheDesc( + capacity=self.max_num_tokens + self.num_extra_kv_tokens, + history_length=0, + ) + ] + ) + ) + buffer_type = [Role.KEY] if self.kv_cache_type != CacheTypeCpp.SELFKONLY: buffer_type.append(Role.VALUE) @@ -1512,12 +1810,6 @@ def _build_cache_config( if self.kv_cache_type != CacheTypeCpp.SELFKONLY: buffer_type.append(Role.VALUE_BLOCK_SCALE) - scratch_reuse_config = None - if self.enable_swa_scratch_reuse: - # Context requests allocate num_extra_kv_tokens for spec decoding. - # They should not count toward the scratch range. - scratch_reuse_config = SwaScratchReuseConfig(max_rewind_len=self.num_extra_kv_tokens) - # Subclasses (e.g. MiniMax-M3 sparse cache) can register additional # per-layer BufferConfig entries — for example a sparse index-K # buffer — without overriding the K/V/NVFP4 scale wiring above. @@ -1559,6 +1851,9 @@ def _build_cache_config( return KVCacheManagerConfigPy( tokens_per_block=tokens_per_block, cache_tiers=cache_tiers, + layers=layer_configs, + typical_step=typical_step, + constraints=constraints, max_util_for_resume=kv_cache_config.max_util_for_resume, enable_partial_reuse=kv_cache_config.enable_partial_reuse, enable_stats=self.enable_stats, @@ -1568,9 +1863,16 @@ def _build_cache_config( and self.block_reuse_policy != BlockReusePolicy.ALL_REUSABLE ), initial_pool_ratio=kv_cache_config.pool_ratio, - layers=layer_configs, ) + def _build_cache_config(self, config: KVCacheManagerConfigPy) -> KVCacheManagerConfigPy: + """Customize the general cache config for a specialized cache manager.""" + return config + + def _get_typical_seq_len(self, kv_cache_config: KvCacheConfig) -> int | None: + """Return the configured typical sequence length, if any.""" + return kv_cache_config.avg_seq_len + def _extra_buffers_per_layer( self, *, tokens_per_block: int ) -> Optional[dict[int, List[BufferConfig]]]: @@ -1582,12 +1884,44 @@ def _extra_buffers_per_layer( a sparse index-K buffer for each sparse local layer. Each ``BufferConfig.size`` is interpreted as bytes per block (i.e., ``bytes_per_token * tokens_per_block``), matching the standard - buffers built in :meth:`_build_cache_config`. The block storage + buffers built in :meth:`_build_base_config`. The block storage groups buffers by lifecycle and size with an opaque role key, so new roles do not require C++ changes. """ return None + def get_disagg_role_mapper_kinds(self) -> dict[DataRole, MapperKind]: + """Map native cache roles to disaggregation mapper kinds. + + ``Role.ALL`` is the required fallback for roles without an explicit + entry. The default is the head-major (HND) ``INDEXED`` layout written + by the TRTLLM attention kernels — correct for V1 and standard V2 + managers. ``Role.INDEX_KEY`` defaults to ``REPLICATED``: every + index-key side cache shipped so far (DSA indexer-K on V1, MiniMax M3 + on V2) computes its projection replicated across TP ranks, so the + cache bytes are identical per rank; the entry is inert unless a + subclass actually registers ``INDEX_KEY`` buffers via + ``_extra_buffers_per_layer``. Model-specific managers may declare + logical layouts without requiring the shared extractor to inspect + private attributes or role names. MiniMax M3, for example, maps + ordinary K/V to ``NHD`` and keeps index-key ``REPLICATED``. + + This declaration does not influence storage pooling: V2 storage + coalesces buffers purely by ``(life_cycle, buffer size)``, so roles + with different transfer semantics may share a pool slot when their + per-block sizes coincide (e.g. MiniMax M3 at TP degrees where + K == V == INDEX_KEY bytes per block). The disagg page-table builder + splits each physical pool into one logical view per mapper kind, so + transfer correctness never depends on the coalescing outcome. + + Pool memory is layout-agnostic; this declaration describes what the + manager's paired attention backend actually writes. A static mapping + is valid only for a fixed manager/backend pair. A manager whose backend + selects the layout at runtime must derive the mapping from that + backend's configuration. + """ + return {Role.ALL: MapperKind.INDEXED, Role.INDEX_KEY: MapperKind.REPLICATED} + @property def blocks_in_primary_pool(self) -> int: """ @@ -1648,18 +1982,19 @@ def get_index_k_buffer( num_heads: int = 1, head_dim: int, dtype: Union[torch.dtype, "DataType", str] = torch.bfloat16, + kv_layout: str = "NHD", ) -> Optional[torch.Tensor]: """Return a torch view over the V2-managed paged ``Role.INDEX_KEY`` buffer for ``layer_idx``, or ``None`` when the layer has no INDEX_KEY buffer registered (e.g. dense layers in a sparse model, or non-local layers on the current PP rank). - The view has shape ``[num_pages, tokens_per_block, num_heads, - head_dim]`` where ``num_pages == impl.get_page_index_upper_bound( - layer_idx, Role.INDEX_KEY)``. Sparse modeling code addresses - entries by ``(page, within_page, head, dim)`` after decomposing - the per-token slot id used by the main paged K/V cache into - ``(page, within_page)``. + For ``kv_layout="NHD"``, the view has shape ``[num_pages, + tokens_per_block, num_heads, head_dim]``. For ``kv_layout="HND"``, + it has shape ``[num_pages, num_heads, tokens_per_block, head_dim]``. + Sparse modeling code decomposes the per-token slot id used by the + main paged K/V cache into ``(page, within_page)`` and indexes the + token axis selected by ``kv_layout``. Because :class:`BufferConfig` only carries an opaque byte ``size`` per block, the dtype and head shape are caller-side contracts. @@ -1678,20 +2013,30 @@ def get_index_k_buffer( propagate to the pool, and successive calls return views over the same backing storage. """ + if kv_layout not in ("NHD", "HND"): + raise ValueError(f"Unsupported kv_layout: {kv_layout}") if layer_idx not in self.layer_offsets: return None layer_offset = self.layer_offsets[layer_idx] try: addr = self.impl.get_mem_pool_base_address(layer_offset, Role.INDEX_KEY) - page_stride = self.impl.get_page_stride(layer_offset, Role.INDEX_KEY) - page_upper = self.impl.get_page_index_upper_bound(layer_offset, Role.INDEX_KEY) - converter = self.impl.get_page_index_converter(layer_offset, Role.INDEX_KEY) - except KeyError: + except (KeyError, IndexError): # INDEX_KEY not registered for this layer (default V2 manager # registers only K/V/scale; sparse subclasses register # INDEX_KEY only on sparse layers via # ``_extra_buffers_per_layer``). + # + # The python backend raises ``KeyError`` from the missing dict + # lookup; the C++ backend's ``getBufferAttr`` throws + # ``std::out_of_range`` for an unknown buffer id, which nanobind + # maps to ``IndexError``. Catch both so the "role not registered + # -> None" contract holds on either backend. return None + # INDEX_KEY is registered; the remaining lookups must succeed. Keep them + # outside the try so genuine failures surface instead of returning None. + page_stride = self.impl.get_page_stride(layer_offset, Role.INDEX_KEY) + page_upper = self.impl.get_page_index_upper_bound(layer_offset, Role.INDEX_KEY) + converter = self.impl.get_page_index_converter(layer_offset, Role.INDEX_KEY) if isinstance(dtype, DataType): torch_dtype = binding_to_torch_dtype(dtype) @@ -1732,21 +2077,21 @@ def get_index_k_buffer( f"{num_slots_total} is not divisible by scale = {scale}." ) num_slots = num_slots_total // scale + if kv_layout == "NHD": + page_shape = [self.tokens_per_block, num_heads, head_dim] + else: + page_shape = [num_heads, self.tokens_per_block, head_dim] if scale == 1: # Non-coalesced INDEX_KEY pool: the per-buffer stride is - # the entire page, so ``[page_upper, tokens_per_block, - # num_heads, head_dim]`` is the correct contiguous view. - shape = [page_upper, self.tokens_per_block, num_heads, head_dim] + # the entire page, so either layout is a contiguous view. + shape = [page_upper, *page_shape] return convert_to_torch_tensor(TensorWrapper(addr, torch_dtype, shape)) - # Coalesced pool: build a ``[num_slots, scale, tokens_per_block, - # num_heads, head_dim]`` view at INDEX_KEY's base, then slice - # ``[:, 0]`` to extract this layer's INDEX_KEY data. The slice - # preserves dim-0 stride = ``scale * page_stride`` bytes, so - # ``view[s, w, h, d]`` lands on the correct byte for any - # ``s`` in [0, num_slots). - full_slot_shape = [num_slots, scale, self.tokens_per_block, num_heads, head_dim] + # Coalesced pool: include the buffers-per-slot dimension, then + # slice ``[:, 0]`` to extract this layer's INDEX_KEY data while + # preserving dim-0 stride = ``scale * page_stride`` bytes. + full_slot_shape = [num_slots, scale, *page_shape] full_view = convert_to_torch_tensor(TensorWrapper(addr, torch_dtype, full_slot_shape)) return full_view[:, 0] @@ -1954,6 +2299,8 @@ def prepare_context(self, req: LlmRequest) -> bool: def _prepare_context_impl(self, req: LlmRequest) -> bool: if req.is_first_context_chunk: + if self.conversation_manager is not None: + self.conversation_manager.prepare_request(req) kv_cache = self.kv_cache_map.get(req.py_request_id) if kv_cache is None: all_tokens = req.get_tokens(DEFAULT_BEAM_INDEX) @@ -2239,13 +2586,16 @@ def _stats_life_cycle_window_size(self, life_cycle) -> Optional[int]: return self._stats_window_size(life_cycle.window_size) def _get_storage_statistics(self, cache_level: CacheLevel): - return self.impl._storage.get_statistics(cache_level) + return _introspection.storage_statistics(self.impl, cache_level) def _stats_life_cycle_metadata(self) -> dict[int, tuple[int, Optional[int], str]]: - pool_groups_by_life_cycle = [ - self.impl._storage.get_pool_group_index(LifeCycleId(life_cycle_id)) - for life_cycle_id in range(len(self.impl.layer_grouping)) - ] + # life cycle (== layer group) -> pool group is static structure exposed by + # the public pool_group_descs API; no introspection needed. + pool_groups_by_life_cycle = { + int(variant.layer_group_id): int(pool_group.pool_group_index) + for pool_group in self.impl.pool_group_descs + for variant in pool_group.slot_desc.variants + } metadata: dict[int, tuple[int, Optional[int], str]] = {} for life_cycle_id, layer_ids in enumerate(self.impl.layer_grouping): @@ -2491,7 +2841,7 @@ def _build_pool_group_iteration_stats( ), ) - def _build_life_cycle_iteration_stats( + def _build_attention_life_cycle_iteration_stats( self, life_cycle_id: int, life_cycle_metadata, @@ -2502,6 +2852,7 @@ def _build_life_cycle_iteration_stats( reuse_delta, ) -> KVCacheV2LifeCycleIterationStats: pool_group_id, window_size, kind = life_cycle_metadata[life_cycle_id] + assert kind == "attention" return KVCacheV2LifeCycleIterationStats( life_cycle_id=life_cycle_id, pool_group_id=pool_group_id, @@ -2518,6 +2869,29 @@ def _build_life_cycle_iteration_stats( ), ) + @staticmethod + def _build_ssm_life_cycle_iteration_stats( + life_cycle_id: int, + life_cycle_metadata, + snapshot_delta, + ) -> KVCacheV2SsmLifeCycleIterationStats: + pool_group_id, window_size, kind = life_cycle_metadata[life_cycle_id] + assert kind == "ssm" + assert window_size is None + return KVCacheV2SsmLifeCycleIterationStats( + life_cycle_id=life_cycle_id, + pool_group_id=pool_group_id, + snapshot_stats=KVCacheV2SsmSnapshotIterationStats( + iter_snapshot_lookups=snapshot_delta.iter_snapshot_lookups, + iter_snapshot_hits=snapshot_delta.iter_snapshot_hits, + iter_snapshot_misses=snapshot_delta.iter_snapshot_misses, + iter_reused_tokens=snapshot_delta.iter_reused_tokens, + iter_unreused_tokens=snapshot_delta.iter_unreused_tokens, + iter_aligned_snapshot_hits=snapshot_delta.iter_aligned_snapshot_hits, + iter_unaligned_snapshot_hits=snapshot_delta.iter_unaligned_snapshot_hits, + ), + ) + def get_kv_cache_stats(self): kv_cache_stats = KvCacheStats() pool_group_stats = self._get_storage_statistics(GPU_LEVEL) @@ -2566,6 +2940,7 @@ def get_iteration_stats(self): pool_groups_by_window = self._storage_pool_groups_by_window() windows_by_pool_group = self._windows_by_pool_group(pool_groups_by_window) raw_iteration_stats = self.impl.get_and_reset_iteration_stats() + raw_ssm_snapshot_iteration_stats = self.impl.get_and_reset_ssm_snapshot_iteration_stats() primary_peak_stats = self._get_and_reset_iteration_peak_block_stats(GPU_LEVEL) num_cache_levels = len(self.impl.cache_tier_list) secondary_peak_stats_by_level = [ @@ -2602,7 +2977,10 @@ def get_iteration_stats(self): for window_size in sorted(windows) } - pool_group_ids = sorted(set(windows_by_pool_group) | set(pool_group_deltas)) + all_pool_group_ids = set(range(len(primary_stats))) + pool_group_ids = sorted( + all_pool_group_ids | set(windows_by_pool_group) | set(pool_group_deltas) + ) stats_by_pool_group = { pool_group_id: self._build_pool_group_iteration_stats( pool_group_id, @@ -2617,7 +2995,7 @@ def get_iteration_stats(self): } stats_by_life_cycle = { - life_cycle_id: self._build_life_cycle_iteration_stats( + life_cycle_id: self._build_attention_life_cycle_iteration_stats( life_cycle_id, life_cycle_metadata, primary_stats, @@ -2628,6 +3006,14 @@ def get_iteration_stats(self): ) for life_cycle_id, reuse_delta in sorted(reuse_deltas_by_life_cycle.items()) } + for life_cycle_id, snapshot_delta in sorted(raw_ssm_snapshot_iteration_stats.items()): + assert int(life_cycle_id) not in stats_by_life_cycle + stats_by_life_cycle[int(life_cycle_id)] = self._build_ssm_life_cycle_iteration_stats( + int(life_cycle_id), + life_cycle_metadata, + snapshot_delta, + ) + stats_by_life_cycle = dict(sorted(stats_by_life_cycle.items())) return KVCacheV2IterationStatsReport( stats_by_window, stats_by_pool_group, stats_by_life_cycle @@ -2825,6 +3211,8 @@ def release_index_slot(self, request_id: int) -> None: self._early_freed_index_requests.add(request_id) def free_resources(self, request: LlmRequest, pin_on_release: bool = False): + if self.conversation_manager is not None: + self.conversation_manager.finish_request(request) self._allocated_draft_lens.pop(request.py_request_id, None) kv_cache = self.kv_cache_map.pop(request.py_request_id, None) if kv_cache is None: @@ -3027,19 +3415,25 @@ def calculate_scaling_factor_size_bytes( ) return get_size_in_bytes(cache_size // quant_vector_size, scaling_factor_dtype) - def check_invalid_values_in_kv_cache(self, fill_with_zero: bool = False) -> bool: - some_checks_unavailable = False - has_invalid_values = torch.tensor( - [False], dtype=torch.bool, device=torch.cuda.current_device() - ) + def _iter_cache_buffers_for_invalid_check(self) -> Iterable[torch.Tensor]: pool_handled = set() - - # Handle each layer from start to end to traverse the whole KV cache. for layer_id, layer_offset in self.layer_offsets.items(): pool_id = self.layer_to_pool_mapping_dict[layer_offset] if pool_id in pool_handled: continue buffer = self.get_buffers(layer_id) + if buffer is None: + continue + yield buffer + pool_handled.add(pool_id) + + def check_invalid_values_in_kv_cache(self, fill_with_zero: bool = False) -> bool: + some_checks_unavailable = False + has_invalid_values = torch.tensor( + [False], dtype=torch.bool, device=torch.cuda.current_device() + ) + + for buffer in self._iter_cache_buffers_for_invalid_check(): # process in chunks of 256 pages to avoid OoM for i in range(0, buffer.shape[0], 256): buffer_slice = buffer[i : i + 256] @@ -3050,7 +3444,6 @@ def check_invalid_values_in_kv_cache(self, fill_with_zero: bool = False) -> bool some_checks_unavailable = True if fill_with_zero: buffer.zero_() - pool_handled.add(pool_id) torch.cuda.synchronize() if some_checks_unavailable: @@ -3065,6 +3458,8 @@ def shutdown(self): kv_cache.close() self.kv_cache_map.clear() self.impl.shutdown() + if self.conversation_manager is not None: + self.conversation_manager.clear() def get_max_resource_count(self) -> int: # TODO: implement this @@ -3146,6 +3541,8 @@ def update_context_resources(self, scheduled_batch: ScheduledRequests): if should_commit: self.try_commit_blocks(req) if req.context_remaining_length == 0: + if self.conversation_manager is not None: + self.conversation_manager.save_drop_plan(req, kv_cache) # Scratch blocks are only for prefill chunks. Disable them at # the context/generation boundary so generation uses normal KV # pages before the first generation allocation. @@ -3174,17 +3571,29 @@ def update_resources( # will be resumed by the scheduler on the next iteration. if not kv_cache.is_active: continue + rewind_len = req.py_rewind_len + if self.is_draft: + runtime_draft_len = req.py_rewind_len + req.py_num_accepted_draft_tokens + # Dynamic-tree draft managers reserve K * max_draft_len slots, + # which can exceed the tree's runtime draft width. Reclaim that + # reserve slack together with rejected draft tokens; otherwise + # it accumulates in the draft KV cache after every generation + # step. Target managers do not allocate this reserve slack. + rewind_len += max(self._kv_reserve_draft_tokens - runtime_draft_len, 0) new_capacity = ( None if req.state in (LlmRequestState.GENERATION_COMPLETE, LlmRequestState.CONTEXT_INIT) - else kv_cache.capacity - req.py_rewind_len + else kv_cache.capacity - rewind_len + ) + history_length = ( + None if self.kv_compression_manages_history else req.max_beam_num_tokens - 1 ) - success = kv_cache.resize(new_capacity, req.max_beam_num_tokens - 1) + success = kv_cache.resize(new_capacity, history_length) if not success: raise ValueError( f"Failed to resize KV cache for request {req.py_request_id} " f"to capacity {new_capacity} and history length " - f"{req.max_beam_num_tokens - 1} tokens at generation update" + f"{history_length} tokens at generation update" ) def copy_batch_block_offsets( @@ -3194,7 +3603,10 @@ def copy_batch_block_offsets( beam_width: int, num_contexts: int, num_seqs: int, + max_blocks: Optional[int] = None, ): + # max_blocks is accepted for signature parity with KVCacheManager; the + # device-side copy op here already scales with allocated blocks only. assert beam_width == 1, "beam_width must be 1 for KVCacheManagerV2" copy_idx = self.index_mapper.get_copy_index(request_ids, num_contexts, beam_width) @@ -3325,3 +3737,5 @@ def prefetch_for_context_tokens(self, requests: list) -> bool: def reset_reuse_state(self): self.impl.clear_reusable_blocks() + if self.conversation_manager is not None: + self.conversation_manager.clear() diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache_stats.py b/tensorrt_llm/_torch/pyexecutor/kv_cache_stats.py index d9841ccbfe86..5ffa3742f3ef 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache_stats.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache_stats.py @@ -14,7 +14,7 @@ # limitations under the License. from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal KV_CACHE_ITERATION_STATS_REUSE_KEYS = ( "iterReusedBlocks", @@ -70,11 +70,39 @@ class KVCacheV2LifeCycleIterationStats: stats: Any +@dataclass(slots=True) +class KVCacheV2SsmSnapshotIterationStats: + iter_snapshot_lookups: int + iter_snapshot_hits: int + iter_snapshot_misses: int + iter_reused_tokens: int + iter_unreused_tokens: int + iter_aligned_snapshot_hits: int + iter_unaligned_snapshot_hits: int + + @property + def iter_snapshot_hit_rate(self) -> float: + if self.iter_snapshot_hits == 0 or self.iter_snapshot_lookups == 0: + return 0.0 + return self.iter_snapshot_hits / self.iter_snapshot_lookups + + +@dataclass(slots=True) +class KVCacheV2SsmLifeCycleIterationStats: + life_cycle_id: int + pool_group_id: int + snapshot_stats: KVCacheV2SsmSnapshotIterationStats + window_size: None = field(default=None, init=False) + kind: Literal["ssm"] = field(default="ssm", init=False) + + @dataclass(slots=True) class KVCacheV2IterationStatsReport: by_window_size: dict[int, Any] by_pool_group: dict[int, KVCacheV2PoolGroupIterationStats] - by_life_cycle: dict[int, KVCacheV2LifeCycleIterationStats] = field(default_factory=dict) + by_life_cycle: dict[ + int, KVCacheV2LifeCycleIterationStats | KVCacheV2SsmLifeCycleIterationStats + ] = field(default_factory=dict) def serialize_kv_cache_iteration_stats(stats, keys: tuple[str, ...] | None = None) -> dict: @@ -115,6 +143,21 @@ def serialize_kv_cache_iteration_stats(stats, keys: tuple[str, ...] | None = Non return {key: fields[key] for key in keys} +def serialize_ssm_snapshot_iteration_stats( + stats: KVCacheV2SsmSnapshotIterationStats, +) -> dict: + return { + "iterSnapshotLookups": stats.iter_snapshot_lookups, + "iterSnapshotHits": stats.iter_snapshot_hits, + "iterSnapshotMisses": stats.iter_snapshot_misses, + "iterSnapshotHitRate": stats.iter_snapshot_hit_rate, + "iterReusedTokens": stats.iter_reused_tokens, + "iterUnreusedTokens": stats.iter_unreused_tokens, + "iterAlignedSnapshotHits": stats.iter_aligned_snapshot_hits, + "iterUnalignedSnapshotHits": stats.iter_unaligned_snapshot_hits, + } + + def append_kv_cache_iteration_stats(stats_dict: dict, kv_iter_stats) -> None: if kv_iter_stats is None: return @@ -147,13 +190,21 @@ def append_kv_cache_iteration_stats(stats_dict: dict, kv_iter_stats) -> None: if not kv_iter_stats.by_life_cycle: return - stats_dict["kvCacheIterationStatsByLifecycle"] = { - str(life_cycle_id): { + stats_by_life_cycle = {} + for life_cycle_id, stats in kv_iter_stats.by_life_cycle.items(): + serialized = { "lifeCycleId": stats.life_cycle_id, "poolGroupId": stats.pool_group_id, "windowSize": stats.window_size, "kind": stats.kind, - **serialize_kv_cache_iteration_stats(stats.stats, KV_CACHE_ITERATION_STATS_REUSE_KEYS), } - for life_cycle_id, stats in kv_iter_stats.by_life_cycle.items() - } + if isinstance(stats, KVCacheV2SsmLifeCycleIterationStats): + serialized["snapshotStats"] = serialize_ssm_snapshot_iteration_stats( + stats.snapshot_stats + ) + else: + serialized.update( + serialize_kv_cache_iteration_stats(stats.stats, KV_CACHE_ITERATION_STATS_REUSE_KEYS) + ) + stats_by_life_cycle[str(life_cycle_id)] = serialized + stats_dict["kvCacheIterationStatsByLifecycle"] = stats_by_life_cycle diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py b/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py index c64f780a0229..43a7a0d407d9 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py @@ -15,7 +15,9 @@ from .llm_request import LlmRequest from .mamba_cache_manager import (BaseMambaCacheManager, - CppMambaHybridCacheManager) + CppMambaHybridCacheManager, + MambaHybridCacheManagerV2, + MixedMambaHybridCacheManager) from .resource_manager import KVCacheManager CacheTransceiverCpp = tensorrt_llm.bindings.internal.batch_manager.CacheTransceiver @@ -129,6 +131,12 @@ def create_kv_cache_transceiver( if cache_transceiver_config.transceiver_runtime == "auto": cache_transceiver_config.transceiver_runtime = None + if (cache_transceiver_config.transceiver_runtime != "PYTHON" + and isinstance(mamba_cache_manager, MixedMambaHybridCacheManager)): + raise ValueError( + "MixedMambaHybridCacheManager requires the Python transceiver " + "runtime in disaggregated serving.") + _validate_disagg_inflight_cancel_config(cache_transceiver_config) if cache_transceiver_config.backend == "DEFAULT": @@ -150,14 +158,33 @@ def create_kv_cache_transceiver( "UCX_CUDA_IPC_ENABLE_MNNVL=n, UCX_RNDV_SCHEME=put_zcopy and/or unset UCX_NET_DEVICES upon server " "hangs or lower-than-expected performance.") - # Select transceiver implementation based on transceiver_runtime + # Select transceiver implementation based on transceiver_runtime. # transceiver_runtime == None or "CPP" -> use C++ transceiver (default) - # transceiver_runtime == "PYTHON" -> use Python transceiver - if cache_transceiver_config.transceiver_runtime == "PYTHON": - # Python transceiver currently only supports NIXL and DEFAULT backend - if cache_transceiver_config.backend not in ("DEFAULT", "NIXL"): + # transceiver_runtime == "PYTHON" -> use Python transceiver. + # + # MambaHybridCacheManagerV2 is backed by the Python KVCacheManagerV2 core, + # not the C++ BaseKVCacheManager binding required by CacheTransceiverCpp. + is_v2_mamba_hybrid = isinstance(mamba_cache_manager, + MambaHybridCacheManagerV2) + use_python_transceiver = ( + cache_transceiver_config.transceiver_runtime == "PYTHON") + + if is_v2_mamba_hybrid and not use_python_transceiver: + raise ValueError( + "MambaHybridCacheManagerV2 requires transceiver_runtime='PYTHON' " + "with backend='NIXL'; it cannot use the C++ transceiver.") + + if use_python_transceiver: + if isinstance(mamba_cache_manager, CppMambaHybridCacheManager): + raise ValueError( + "transceiver_runtime='PYTHON' cannot drive " + "CppMambaHybridCacheManager (C++ pool backed). Use " + "transceiver_runtime='CPP', or select the V2 manager " + "with use_kv_cache_manager_v2=True.") + # DEFAULT has already been resolved above, so Python must see NIXL. + if cache_transceiver_config.backend != "NIXL": raise ValueError( - f"Python transceiver currently only supports NIXL or DEFAULT backend, " + f"Python transceiver currently only supports the NIXL backend, " f"got {cache_transceiver_config.backend}. " f"Please use transceiver_runtime='CPP' for MPI, UCX, or MOONCAKE backends." ) @@ -230,6 +257,14 @@ def get_disaggregated_params(self) -> Dict[str, Any]: def commit_blocks_for_reuse(self, req: LlmRequest) -> None: """Commit received KV blocks to the radix tree for prefix reuse. No-op by default.""" + def get_data_transceiver_state(self) -> bytes: + """Get the serialized DataTransceiverState (CacheState + CommState).""" + return b"" + + def get_status_dump(self) -> str: + """Return a human-readable dump of transceiver state for debugging hangs.""" + return "" + def shutdown(self): """Shut down the transceiver and release registered resources.""" @@ -324,10 +359,16 @@ def has_poisoned_transfer_buffer(self) -> bool: return False return self.impl.has_poisoned_transfer_buffer() + def get_status_dump(self) -> str: + return self.impl.get_status_dump() + def prepare_context_requests(self, requests: List[LlmRequest]): # not implemented, an empty placeholder to allow being invoked unconditionally ... + def get_data_transceiver_state(self) -> bytes: + return self.impl.get_serialized_data_transceiver_state() + def get_disaggregated_params(self): # Cpp kv cache transceiver will set the disaggregated params to context response # Only new py cache transceiver will support gen-first disagg diff --git a/tensorrt_llm/_torch/pyexecutor/llm_request.py b/tensorrt_llm/_torch/pyexecutor/llm_request.py index 6d7522ec79a8..8389304f96d6 100644 --- a/tensorrt_llm/_torch/pyexecutor/llm_request.py +++ b/tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -47,7 +47,7 @@ ATTENTION_DP_DUMMY_REQUEST_ID = 0 if TYPE_CHECKING: - from .sampler.sampling_utils import Strategy + from .sampler.sampler_strategy import Strategy @dataclass(slots=True) @@ -677,8 +677,14 @@ def __init__( self.py_lora_path: str | None = kwargs.pop("py_lora_path", None) # Multimodal data self.py_multimodal_data = kwargs.pop("py_multimodal_data", None) + self.py_mm_item_order = kwargs.pop("py_mm_item_order", None) encoder_input_tokens = kwargs.get("encoder_input_tokens") encoder_output_len = kwargs.get("encoder_output_len") + # Python-side handle to the encoder feature tensor (audio enc-dec + # models): the C++ binding takes the kwarg but exposes no getter, so + # the encoder step reads it here. Kept for the request's lifetime — + # pause() re-enters ENCODER_INIT and re-runs the encoder from it. + self.py_encoder_input_features = kwargs.get("encoder_input_features") return_encoder_output = bool(kwargs.get("return_encoder_output", False)) if return_encoder_output: kwargs["return_encoder_output"] = False @@ -730,6 +736,8 @@ def __init__( self.py_batch_idx = None self.py_draft_pages_allocated = 0 self.py_rewind_len = 0 + # Tokens physically evicted by KV-cache compression; deducted in the engine. + self.py_num_compressed_tokens = 0 self.py_draft_tokens = [] if self.draft_tokens is None else self.draft_tokens self.py_last_context_chunk = (None, None) self.py_draft_logits = None @@ -1112,6 +1120,29 @@ def executor_request_to_llm_request( if getattr(executor_request, "py_scheduling_params", None) is not None: agent_hierarchy = executor_request.py_scheduling_params.agent_hierarchy + # Audio encoder-decoder models (e.g. Whisper) carry the encoder input as a + # feature tensor, not encoder token ids. Route it into the request's native + # encoder_input_features / encoder_output_len fields so the C++ state machine + # admits it to the encoder step and cross-KV sizing sees the post-encoder + # length rather than a token count. + encoder_input_features = None + encoder_output_len = None + py_mm_data = getattr(executor_request, "py_multimodal_data", None) or {} + audio_mm_data = py_mm_data.get("audio") or {} + if isinstance(audio_mm_data, dict): + # Only enc-dec input processors emit encoder_input_features (decoder-only + # audio models use the generic HF input_features), so its presence is a + # safe routing signal. + encoder_input_features = audio_mm_data.get("encoder_input_features") + if encoder_input_features is not None: + if "encoder_output_len" not in audio_mm_data: + raise ValueError( + "multimodal_data['audio'] carries encoder_input_features " + "without encoder_output_len; encoder-decoder input " + "processors must emit both (the post-encoder length " + "sizes the cross-KV cache).") + encoder_output_len = int(audio_mm_data["encoder_output_len"]) + llm_request = LlmRequest( request_id=req_id, max_new_tokens=executor_request.max_tokens, @@ -1171,6 +1202,8 @@ def executor_request_to_llm_request( py_logits_post_processors=getattr(executor_request, "py_logits_post_processors", None), encoder_input_tokens=executor_request.encoder_input_token_ids, + encoder_input_features=encoder_input_features, + encoder_output_len=encoder_output_len, return_encoder_output=executor_request.output_config. return_encoder_output, client_id=executor_request.client_id @@ -1182,6 +1215,7 @@ def executor_request_to_llm_request( arrival_time=getattr(executor_request, "py_arrival_time", None), py_multimodal_data=getattr(executor_request, "py_multimodal_data", None), + py_mm_item_order=getattr(executor_request, "py_mm_item_order", None), kv_cache_retention_config=executor_request.kv_cache_retention_config, agent_hierarchy=agent_hierarchy, logprobs_mode=getattr(executor_request, "py_logprobs_mode", @@ -1190,6 +1224,20 @@ def executor_request_to_llm_request( "py_logprobs_simple_format", False), ) + # Bad-words list for the TorchSampler path, kept in its native + # list[list[int]] form (single- and multi-token words). This is the + # TorchSampler's own input and is independent of any other sampler. + llm_request.py_bad_words = [ + list(word) for word in executor_request.bad_words + ] if executor_request.bad_words else None + + # No-repeat-ngram size for the TorchSampler path, normalized once here so + # the per-step sampler gate is a plain attribute read. The C++ + # SamplingConfig rejects negative values up front and 0 means disabled + # (same convention as the C++ banRepeatNgram kernel), so falsy == off. + ngram_size = executor_request.sampling_config.no_repeat_ngram_size + llm_request.py_no_repeat_ngram_size = ngram_size if ngram_size else None + llm_request.py_original_end_id = getattr(executor_request, "py_original_end_id", llm_request.py_end_id) diff --git a/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py b/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py index 46badaa0c922..a0785487536a 100644 --- a/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py @@ -16,8 +16,9 @@ import math import os from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Union +from dataclasses import dataclass, replace +from typing import (TYPE_CHECKING, Dict, Iterable, List, Literal, NamedTuple, + Optional, Tuple, Union) import torch import triton @@ -27,21 +28,29 @@ from tensorrt_llm._torch.attention_backend.interface import AttentionMetadata from tensorrt_llm.llmapi.llm_args import DecodingBaseConfig -from tensorrt_llm._torch.pyexecutor.llm_request import ATTENTION_DP_DUMMY_REQUEST_ID, LlmRequest +from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import ( + BlockReusePolicy, KVCacheManagerV2, Role) +from tensorrt_llm._torch.pyexecutor.llm_request import ( + ATTENTION_DP_DUMMY_REQUEST_ID, LlmRequest) from tensorrt_llm._torch.pyexecutor.resource_manager import ( - BaseResourceManager, - CacheTypeCpp, - DataType, - KVCacheManager, - PoolConfiguration, - get_pp_layers, -) + BaseResourceManager, CacheTypeCpp, DataType, KVCacheManager, + PoolConfiguration, get_pp_layers) from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests -from tensorrt_llm._utils import nvtx_range, prefer_pinned, torch_dtype_to_binding -from tensorrt_llm.bindings.internal.batch_manager import LinearAttentionMetadata, LinearCacheType +from tensorrt_llm._utils import (TensorWrapper, convert_to_torch_tensor, + nvtx_range, prefer_pinned, + torch_dtype_to_binding) +from tensorrt_llm.bindings.internal.batch_manager import ( + LinearAttentionMetadata, LinearCacheType) from tensorrt_llm.llmapi.llm_args import KvCacheConfig from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping +from tensorrt_llm.runtime.kv_cache_manager_v2 import (DEFAULT_BEAM_INDEX, + BatchDesc, BufferConfig, + DataRole, KVCacheDesc) +from tensorrt_llm.runtime.kv_cache_manager_v2 import \ + KVCacheManagerConfig as KVCacheManagerConfigPy +from tensorrt_llm.runtime.kv_cache_manager_v2 import (LayerId, PageIndexMode, + SsmLayerConfig) GB = 1 << 30 @@ -60,6 +69,63 @@ MIN_REPLAY_HISTORY_SIZE = 16 +def _get_num_cuda_graph_padding_dummy_slots( + spec_config: Optional["DecodingBaseConfig"], + max_batch_size: int, +) -> int: + """Return the number of persistent CUDA-graph padding dummy IDs. + + This is computed before ``ModelEngine`` exists and covers draft lengths + reachable at every batch size, including the zero-length acceptance-rate + fallback. ``ModelEngine._compute_dynamic_draft_len_mapping`` is created + later and covers only configured CUDA-graph batch sizes, so it cannot size + this persistent ID set. + """ + if spec_config is None: + return 1 + + draft_len_schedule = getattr(spec_config, "draft_len_schedule", None) + spec_dec_mode = getattr(spec_config, "spec_dec_mode", None) + supports_dynamic_draft_len = (spec_dec_mode is not None and hasattr( + spec_dec_mode, "support_dynamic_draft_len") + and spec_dec_mode.support_dynamic_draft_len()) + if draft_len_schedule and supports_dynamic_draft_len: + runtime_draft_lengths = set() + first_uncovered_batch_size = 1 + for batch_size_threshold, draft_len in draft_len_schedule.items(): + if first_uncovered_batch_size > max_batch_size: + break + if batch_size_threshold >= first_uncovered_batch_size: + runtime_draft_lengths.add(draft_len) + first_uncovered_batch_size = batch_size_threshold + 1 + if first_uncovered_batch_size <= max_batch_size: + runtime_draft_lengths.add(0) + else: + max_draft_len = getattr(spec_config, "max_draft_len", 0) or 0 + max_total_draft_tokens = (getattr(spec_config, "max_total_draft_tokens", + 0) or 0) + is_linear_tree = getattr( + spec_config, + "is_linear_tree", + max_draft_len == max_total_draft_tokens, + ) + static_draft_len = (max_draft_len + if is_linear_tree else max_total_draft_tokens) + runtime_draft_lengths = {static_draft_len or 0} + + if ((getattr(spec_config, "acceptance_rate_window_size", 0) or 0) > 0 and + (getattr(spec_config, "acceptance_rate_threshold", 0) or 0) > 0): + runtime_draft_lengths.add(0) + return len(runtime_draft_lengths) + + +class MambaRole: + """V2 buffer roles owned only by the hybrid Mamba manager.""" + + SSM_STATE = DataRole("ssm_state") + CONV_STATE = DataRole("conv_state") + + def get_tensor_size_bytes(tensor): """Calculate tensor size in bytes.""" if isinstance(tensor, torch.Tensor): @@ -148,9 +214,9 @@ def use_py_mamba_cache_manager() -> bool: Returns True if TRTLLM_USE_PY_MAMBA='1' is set, False otherwise. Agg-mode-only override: forces the V1-route MixedMambaHybridCacheManager - with PythonMambaCacheManager inside instead of the default unified-pool - CppMambaHybridCacheManager. Disagg mode is unaffected — it already picks - PythonMambaCacheManager when transceiver_runtime='PYTHON'. + with PythonMambaCacheManager inside instead of the configured manager. + Disagg mode is unaffected — its compatibility routing selects Mixed or + Cpp based on the transceiver configuration. """ return os.environ.get('TRTLLM_USE_PY_MAMBA', '0') == '1' @@ -163,6 +229,40 @@ class ReplayStateUpdateMetadata(NamedTuple): replay_history_size: int +def _advance_replay_state( + replay_metadata: ReplayStateUpdateMetadata, + state_indices: torch.Tensor, + accepted_tokens: torch.Tensor, + is_dummy_request: Optional[torch.Tensor] = None, +) -> None: + """Advance replay bookkeeping after a speculative generation step.""" + slots = state_indices.long() + accepted_tokens = accepted_tokens.to( + replay_metadata.prev_num_accepted_tokens.dtype) + prev_num_accepted_tokens = replay_metadata.prev_num_accepted_tokens[slots] + wrote_checkpoint = (prev_num_accepted_tokens + + replay_metadata.replay_step_width + > replay_metadata.replay_history_size) + next_num_accepted_tokens = torch.where( + wrote_checkpoint, + accepted_tokens, + prev_num_accepted_tokens + accepted_tokens, + ) + cache_buf_idx = replay_metadata.cache_buf_idx[slots] + next_cache_buf_idx = torch.where(wrote_checkpoint, 1 - cache_buf_idx, + cache_buf_idx) + if is_dummy_request is not None: + next_num_accepted_tokens = torch.where( + is_dummy_request, + prev_num_accepted_tokens, + next_num_accepted_tokens, + ) + next_cache_buf_idx = torch.where(is_dummy_request, cache_buf_idx, + next_cache_buf_idx) + replay_metadata.prev_num_accepted_tokens[slots] = next_num_accepted_tokens + replay_metadata.cache_buf_idx[slots] = next_cache_buf_idx + + class BaseMambaCacheManager(ABC): """Abstract interface for accessing mamba/recurrent state caches.""" @@ -472,27 +572,25 @@ def _dim_contiguous_conv_cache(): spec_kwargs['kda_conv_q'] = _dim_contiguous_conv_cache() spec_kwargs['kda_conv_k'] = _dim_contiguous_conv_cache() spec_kwargs['kda_conv_v'] = _dim_contiguous_conv_cache() - spec_kwargs['kda_qkg_cache'] = torch.zeros( - num_local_layers, - max_batch_size, - M, - 3, - section_dim, - dtype=torch.float32, - device=device) + spec_kwargs['kda_qkg_cache'] = torch.zeros(num_local_layers, + max_batch_size, + M, + 3, + section_dim, + dtype=torch.float32, + device=device) spec_kwargs['kda_v_cache'] = torch.zeros(num_local_layers, max_batch_size, M, section_dim, dtype=torch.float32, device=device) - spec_kwargs['kda_beta_cache'] = torch.zeros( - num_local_layers, - max_batch_size, - M, - nheads, - dtype=torch.float32, - device=device) + spec_kwargs['kda_beta_cache'] = torch.zeros(num_local_layers, + max_batch_size, + M, + nheads, + dtype=torch.float32, + device=device) ssm_spec_cache = [ spec_kwargs['kda_conv_q'], spec_kwargs['kda_conv_k'], spec_kwargs['kda_conv_v'], spec_kwargs['kda_qkg_cache'], @@ -677,8 +775,8 @@ def _prepare_mamba_cache_blocks(self, request_ids: List[int]): self._seed_rank_offset)) @torch.inference_mode() - def seed_kda_replay_caches_for_disagg_gen( - self, request_ids: List[int]) -> None: + def seed_kda_replay_caches_for_disagg_gen(self, + request_ids: List[int]) -> None: """Seed the fused-verify KDA replay conv caches from the conv pool. On a disaggregated generation server the ctx->gen transfer populates @@ -718,13 +816,14 @@ def seed_kda_replay_caches_for_disagg_gen( ): # cache: [L, slots, D, committed + num_spec]; zero the draft # tail columns and seed the committed window in one copy. - seeded = torch.zeros((cache.shape[0], idx.numel()) + - cache.shape[2:], - dtype=cache.dtype, - device=cache.device) + seeded = torch.zeros( + (cache.shape[0], idx.numel()) + cache.shape[2:], + dtype=cache.dtype, + device=cache.device) seeded[:, :, :, :committed] = section[:, :, :, 1:].to(cache.dtype) cache.index_copy_(1, idx, seeded) - for buf in (self.mamba_cache.kda_qkg_cache, self.mamba_cache.kda_v_cache, + for buf in (self.mamba_cache.kda_qkg_cache, + self.mamba_cache.kda_v_cache, self.mamba_cache.kda_beta_cache): buf.index_fill_(1, idx, 0) self.mamba_cache.prev_num_accepted_tokens[idx] = 0 @@ -746,7 +845,8 @@ def _is_padding_sentinel(self, request_id: int) -> bool: # cuda_graph_runner caches one dummy per runtime_draft_len value # (see _get_padded_batch), so any id in the range of dummy request IDs # may be live concurrently. - from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import CUDA_GRAPH_DUMMY_REQUEST_ID + from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import \ + CUDA_GRAPH_DUMMY_REQUEST_ID max_dl = self.speculative_num_draft_tokens or 0 return (CUDA_GRAPH_DUMMY_REQUEST_ID - max_dl <= request_id <= CUDA_GRAPH_DUMMY_REQUEST_ID) @@ -957,14 +1057,20 @@ def _drop(tensor): torch.cuda.empty_cache() @torch.compile(options={"max-autotune": True}) - def update_mamba_states(self, attn_metadata: "AttentionMetadata", - num_accepted_tokens: torch.Tensor, - state_indices: torch.Tensor): + def update_mamba_states( + self, + attn_metadata: "AttentionMetadata", + num_accepted_tokens: torch.Tensor, + state_indices: torch.Tensor, + accepted_leaf_positions: Optional[torch.Tensor] = None): batch_size = attn_metadata.num_seqs num_contexts = attn_metadata.num_contexts num_gens = batch_size - num_contexts num_accepted_draft_tokens = num_accepted_tokens[ num_contexts:num_contexts + num_gens] - 1 + # Dynamic tree passes tree-node leaf positions; linear MTP uses depth. + accepted_positions = (accepted_leaf_positions if accepted_leaf_positions + is not None else num_accepted_draft_tokens) state_indices_d = state_indices[num_contexts:num_contexts + num_gens] src_state_indices = self.intermediate_state_indices[:num_gens] @@ -984,39 +1090,22 @@ def update_mamba_states(self, attn_metadata: "AttentionMetadata", return if self._use_replay_state_update: - # SSM state is handled incrementally by the kernel. Mirror the - # kernel's per-slot checkpoint predicate from the previous PNAT and - # fixed replay step width: checkpoint steps write a fresh history - # buffer and flip, while no-checkpoint steps append to the active - # buffer and keep reading from it next step. - accepted_tokens = num_accepted_tokens[num_contexts:num_contexts + - num_gens] - prev_num_accepted_tokens = \ - self.mamba_cache.prev_num_accepted_tokens[state_indices_d] - wrote_checkpoint = (prev_num_accepted_tokens + - self.replay_step_width - > self.replay_history_size) - next_num_accepted_tokens = torch.where( - wrote_checkpoint, accepted_tokens, - prev_num_accepted_tokens + accepted_tokens) - cache_buf_idx = self.mamba_cache.cache_buf_idx[state_indices_d] is_dummy_request = self._dummy_request_mask[ num_contexts:num_contexts + num_gens] - next_num_accepted_tokens = torch.where(is_dummy_request, - prev_num_accepted_tokens, - next_num_accepted_tokens) - self.mamba_cache.prev_num_accepted_tokens[state_indices_d] = \ - next_num_accepted_tokens - self.mamba_cache.cache_buf_idx[state_indices_d] = \ - torch.where(is_dummy_request, cache_buf_idx, - torch.where(wrote_checkpoint, 1 - cache_buf_idx, - cache_buf_idx)) + replay_metadata = self.get_replay_state_update_metadata() + assert replay_metadata is not None + _advance_replay_state( + replay_metadata, + state_indices_d, + num_accepted_tokens[num_contexts:num_contexts + num_gens], + is_dummy_request, + ) else: # Legacy: copy accepted SSM state from intermediate cache. ssm_states = self.mamba_cache.temporal intermediate_ssm_cache = self.mamba_cache.intermediate_ssm accepted_ssm_state = intermediate_ssm_cache[:, src_state_indices, - num_accepted_draft_tokens] + accepted_positions] ssm_states[:, state_indices_d, :] = accepted_ssm_state # Conv: both paths save all intermediate conv windows, carry over the accepted one. @@ -1024,7 +1113,7 @@ def update_mamba_states(self, attn_metadata: "AttentionMetadata", intermediate_conv_window_cache = self.mamba_cache.intermediate_conv_window accepted_conv_state = intermediate_conv_window_cache[:, src_state_indices, - num_accepted_draft_tokens] + accepted_positions] conv_states[:, state_indices_d, :] = accepted_conv_state @@ -1113,11 +1202,10 @@ def mamba_cache_index(self) -> Dict[int, int]: def get_conv_states(self, layer_idx: int) -> torch.Tensor: return self._impl.get_conv_states(layer_idx) - def seed_kda_replay_caches_for_disagg_gen( - self, request_ids: List[int]) -> None: + def seed_kda_replay_caches_for_disagg_gen(self, + request_ids: List[int]) -> None: self._impl.seed_kda_replay_caches_for_disagg_gen(request_ids) - def get_ssm_states(self, layer_idx: int) -> torch.Tensor: return self._impl.get_ssm_states(layer_idx) @@ -1175,25 +1263,272 @@ def mamba_layer_cache( def shutdown(self): self._impl.shutdown() - def update_mamba_states(self, attn_metadata: "AttentionMetadata", - num_accepted_tokens: torch.Tensor, - state_indices: torch.Tensor): + def update_mamba_states( + self, + attn_metadata: "AttentionMetadata", + num_accepted_tokens: torch.Tensor, + state_indices: torch.Tensor, + accepted_leaf_positions: Optional[torch.Tensor] = None): # Non-speculative configs don't allocate intermediate state; the # promotion is a clean no-op. if not self._impl.is_speculative(): return self._impl.update_mamba_states(attn_metadata, num_accepted_tokens, - state_indices) + state_indices, accepted_leaf_positions) class MambaHybridCacheManager(BaseResourceManager, BaseMambaCacheManager): - """Marker base class for hybrid mamba cache manager implementations. + """Shared interface and state plumbing for hybrid Mamba managers. - Used purely for ``isinstance`` / type-hint purposes so callers can refer - to the family without caring about the concrete implementation. Concrete - selection (Mixed vs Cpp) lives in ``_util.py:_get_model_kv_cache_manager_cls``. + Concrete storage, state-index, and resource lifecycles remain owned by the + Cpp and V2 implementations. """ + _supports_additional_snapshot_offsets = False + + def _setup_mtp_intermediate_states(self, spec_config, + max_batch_size: int) -> None: + self.spec_config = spec_config + self.intermediate_ssm_states = None + self.intermediate_conv_states = None + self.intermediate_state_indices = None + if spec_config is None or self.local_num_mamba_layers == 0: + return + + tokens_per_gen_step = spec_config.tokens_per_gen_step + if not self._use_replay_state_update: + self.intermediate_ssm_states = torch.zeros( + size=[ + self.local_num_mamba_layers, max_batch_size, + tokens_per_gen_step + ] + self.ssm_state_shape, + dtype=self.ssm_state_dtype, + device="cuda", + ) + self.intermediate_conv_states = torch.zeros( + size=[ + self.local_num_mamba_layers, max_batch_size, tokens_per_gen_step + ] + self.conv_state_shape, + dtype=self.conv_state_dtype, + device="cuda", + ) + self.intermediate_state_indices = torch.arange(max_batch_size, + dtype=torch.int32, + device="cuda") + + def _allocate_pool_replay_buffers( + self, + spec_config, + cache_size: int, + device: Optional[torch.device], + ) -> bool: + """Allocate replay tensors shared by the Cpp and V2 state pools.""" + self.prev_num_accepted_tokens = None + self.cache_buf_idx = None + self.mamba_ssm_rand_seed = None + self._dummy_request_mask = None + self._dummy_request_mask_host = None + self.old_x = None + self.old_B = None + self.old_dt = None + self.old_dA_cumsum = None + + if (self.local_num_mamba_layers == 0 + or (not self._use_replay_state_update + and not self._mamba_ssm_stochastic_rounding)): + return False + + assert device is not None + self.mamba_ssm_rand_seed = _allocate_mamba_seed_buffer( + cache_size, self._seed_rank_offset, device) + if spec_config is None or not self._use_replay_state_update: + return False + + history_size = self.replay_history_size + assert history_size is not None + nheads, head_dim, d_state = self.ssm_state_shape + common_shape = [self.local_num_mamba_layers, cache_size, 2] + self.prev_num_accepted_tokens = torch.zeros(cache_size, + dtype=torch.int32, + device=device) + self.cache_buf_idx = torch.zeros(cache_size, + dtype=torch.int32, + device=device) + self.old_x = torch.zeros( + common_shape + [history_size, nheads, head_dim], + dtype=self.conv_state_dtype, + device=device, + ) + self.old_B = torch.zeros( + common_shape + [history_size, self._n_groups_per_rank, d_state], + dtype=self.conv_state_dtype, + device=device, + ) + self.old_dt = torch.zeros( + common_shape + [nheads, history_size], + dtype=torch.float32, + device=device, + ) + self.old_dA_cumsum = torch.zeros( + common_shape + [nheads, history_size], + dtype=torch.float32, + device=device, + ) + return True + + @torch.inference_mode() + def _refresh_dummy_request_mask(self, is_dummy: List[bool]) -> None: + if self._dummy_request_mask is None: + return + + n = len(is_dummy) + assert n <= self._dummy_request_mask_host.shape[0] + self._dummy_request_mask_host.zero_() + if n > 0: + self._dummy_request_mask_host[:n].copy_( + torch.tensor(is_dummy, dtype=torch.bool)) + self._dummy_request_mask.copy_(self._dummy_request_mask_host, + non_blocking=True) + + def _reset_context_mamba_slots(self, num_contexts: int) -> None: + if num_contexts == 0: + return + + context_slots = self.cuda_state_indices[:num_contexts].long() + if (self._use_replay_state_update + and self.prev_num_accepted_tokens is not None + and self.cache_buf_idx is not None): + self.prev_num_accepted_tokens[context_slots] = 0 + self.cache_buf_idx[context_slots] = 0 + if self.old_x is not None: + self.old_x[:, context_slots] = 0 + if self.old_B is not None: + self.old_B[:, context_slots] = 0 + if self.old_dt is not None: + self.old_dt[:, context_slots] = 0 + if self.old_dA_cumsum is not None: + self.old_dA_cumsum[:, context_slots] = 0 + + if self.mamba_ssm_rand_seed is None: + return + self._seed_request_counter += 1 + counter = self._seed_request_counter + rank_offset = self._seed_rank_offset + host_slots = self._host_state_indices[:num_contexts].tolist() + new_seeds = [ + _compute_deterministic_mamba_seed(counter, slot, rank_offset) + for slot in host_slots + ] + self.mamba_ssm_rand_seed[context_slots] = torch.tensor( + new_seeds, + dtype=torch.int64, + device=self.mamba_ssm_rand_seed.device, + ) + + def prepare_expect_snapshot_points(self, + requests: List[LlmRequest]) -> None: + """Set reusable Mamba snapshot boundaries before scheduling.""" + if not self.enable_block_reuse: + for request in requests: + request.expect_snapshot_points = [] + return + + state_config = self.kv_cache_config.mamba_state_config + interval = state_config.periodic_snapshot_interval + for request in requests: + snapshot_points = set() + if interval is not None and interval > 0: + snapshot_points.update( + range(interval, request.prompt_len + 1, interval)) + if self._supports_additional_snapshot_offsets: + for offset in state_config.additional_snapshot_offsets_from_start: + if offset <= request.prompt_len: + snapshot_points.add(offset) + for offset in state_config.additional_snapshot_offsets_from_end: + point = request.prompt_len - offset + if point > 0: + snapshot_points.add(point) + request.expect_snapshot_points = sorted(snapshot_points) + + def is_speculative(self) -> bool: + return self.spec_config is not None + + def get_ssm_states(self, layer_idx: int) -> torch.Tensor: + return self.all_ssm_states[self.mamba_layer_offsets[layer_idx]] + + def get_conv_states(self, layer_idx: int) -> torch.Tensor: + return self.all_conv_states[self.mamba_layer_offsets[layer_idx]] + + def get_intermediate_ssm_states(self, + layer_idx: int) -> Optional[torch.Tensor]: + if self.intermediate_ssm_states is None: + return None + return self.intermediate_ssm_states[self.mamba_layer_offsets[layer_idx]] + + def get_intermediate_conv_states(self, + layer_idx: int) -> Optional[torch.Tensor]: + if self.intermediate_conv_states is None: + return None + return self.intermediate_conv_states[ + self.mamba_layer_offsets[layer_idx]] + + def mamba_layer_cache( + self, layer_idx: int + ) -> Union[PythonMambaCacheManager.State, + PythonMambaCacheManager.SpeculativeState, None]: + conv = self.get_conv_states(layer_idx) + ssm = self.get_ssm_states(layer_idx) + if self.spec_config is not None: + layer_offset = self.mamba_layer_offsets[layer_idx] + spec_kwargs = {} + if self.mamba_ssm_rand_seed is not None: + spec_kwargs['mamba_ssm_rand_seed'] = self.mamba_ssm_rand_seed + if self._use_replay_state_update: + spec_kwargs['old_x'] = self.old_x[layer_offset] + spec_kwargs['old_B'] = self.old_B[layer_offset] + spec_kwargs['old_dt'] = self.old_dt[layer_offset] + spec_kwargs['old_dA_cumsum'] = self.old_dA_cumsum[layer_offset] + spec_kwargs['cache_buf_idx'] = self.cache_buf_idx + spec_kwargs['prev_num_accepted_tokens'] = ( + self.prev_num_accepted_tokens) + else: + spec_kwargs['intermediate_ssm'] = self.intermediate_ssm_states[ + layer_offset] + return PythonMambaCacheManager.SpeculativeState( + conv=conv, + temporal=ssm, + intermediate_conv_window=self. + intermediate_conv_states[layer_offset], + **spec_kwargs, + ) + return PythonMambaCacheManager.State(conv=conv, temporal=ssm) + + @property + def use_replay_state_update(self) -> bool: + return self.get_replay_state_update_metadata() is not None + + def get_replay_state_update_metadata( + self) -> Optional[ReplayStateUpdateMetadata]: + prev_num_accepted_tokens = getattr(self, 'prev_num_accepted_tokens', + None) + cache_buf_idx = getattr(self, 'cache_buf_idx', None) + if (not self._use_replay_state_update + or prev_num_accepted_tokens is None or cache_buf_idx is None + or self.replay_step_width is None + or self.replay_history_size is None): + return None + return ReplayStateUpdateMetadata( + prev_num_accepted_tokens=prev_num_accepted_tokens, + cache_buf_idx=cache_buf_idx, + replay_step_width=self.replay_step_width, + replay_history_size=self.replay_history_size) + + def get_mamba_ssm_cache_dtype(self) -> torch.dtype: + return self.ssm_state_dtype + + def get_mamba_ssm_rand_seed(self) -> Optional[torch.Tensor]: + return getattr(self, 'mamba_ssm_rand_seed', None) + def _get_mamba_hybrid_pool_size(max_batch_size: int, mapping: Mapping) -> int: """Return the internal Mamba state pool size for MixedMambaHybridCacheManager.""" @@ -1390,36 +1725,15 @@ def _maybe_promote_drafter_states(self, scheduled_batch, attn_metadata): # the reserved padding slot (a harmless scratch write). slot_index = self.mamba_cache_index padding_slot = self._impl._padding_slot - state_indices = torch.tensor( - [ - slot_index.get(r.py_request_id, padding_slot) - for r in scheduled_batch.context_requests + gen_requests - ], - dtype=torch.int32, - device=device) + state_indices = torch.tensor([ + slot_index.get(r.py_request_id, padding_slot) + for r in scheduled_batch.context_requests + gen_requests + ], + dtype=torch.int32, + device=device) self.update_mamba_states(attn_metadata, num_accepted, state_indices) -def calc_context_stop_positions(prompt_len: int, - tokens_per_block: int, - mamba_state_cache_interval: int, - save_last_snapshot: bool = False) -> list[int]: - """Compute token positions at which mamba state snapshots should be saved. - - Returns positions spaced by ``mamba_state_cache_interval`` plus the final - prompt length (and optionally the last block-aligned position). - """ - stop_positions = list( - range(mamba_state_cache_interval, prompt_len, - mamba_state_cache_interval)) - last_ckpt = prompt_len // tokens_per_block * tokens_per_block - if save_last_snapshot and (last_ckpt not in stop_positions): - stop_positions.append(last_ckpt) - if prompt_len not in stop_positions: - stop_positions.append(prompt_len) - return stop_positions - - @triton.jit def _promote_mamba_state_kernel( src_ptr, @@ -1511,12 +1825,169 @@ def _promote_mamba_state_triton(dst: torch.Tensor, ) +def _mamba_snapshot_rule_counts( + kv_cache_config: KvCacheConfig, + max_seq_len: Optional[int], + tokens_per_block: int, +) -> Tuple[int, int]: + """Return reachable fixed rules and their partial-block upper bound.""" + if not kv_cache_config.enable_block_reuse: + return 0, 0 + + num_rules = 0 + num_unaligned_rules = 0 + state_config = kv_cache_config.mamba_state_config + for offset in set(state_config.additional_snapshot_offsets_from_start): + if max_seq_len is not None and offset > max_seq_len: + continue + num_rules += 1 + num_unaligned_rules += int(offset % tokens_per_block != 0) + for offset in set(state_config.additional_snapshot_offsets_from_end): + # A from-end offset is reachable iff some valid prompt is longer than + # the offset. Its absolute alignment depends on that prompt. + if max_seq_len is not None and offset >= max_seq_len: + continue + num_rules += 1 + num_unaligned_rules += 1 + return num_rules, num_unaligned_rules + + +def _mamba_regular_snapshot_interval( + kv_cache_config: KvCacheConfig, + max_seq_len: Optional[int], +) -> Optional[int]: + if not kv_cache_config.enable_block_reuse: + return None + interval = kv_cache_config.mamba_state_config.periodic_snapshot_interval + if interval is None or interval <= 0: + return None + if max_seq_len is not None and interval > max_seq_len: + return None + return interval + + +def _get_local_mamba_cache_layout( + model_config, + mapping: Mapping, + *, + spec_config=None, + is_draft: bool = False, + use_separate_draft_kv_cache: bool = False, +): + """Return normalized params and local Mamba/attention layer counts. + + Cache construction and affine sizing must follow the model's PP layout: + partition base layers first, then place appended speculative layers on the + last PP rank. The normalized params retain target masks and the appended + draft-layer count so estimation selects the same combined or per-manager + layout as runtime. + """ + from tensorrt_llm._torch.pyexecutor.config_utils import \ + extract_mamba_kv_cache_params + + params = extract_mamba_kv_cache_params( + model_config.pretrained_config, + spec_config=spec_config, + quant_config=model_config.quant_config, + ) + mamba_layer_mask, full_attention_layer_mask = params.get_layer_masks( + is_draft=is_draft, + use_separate_draft_kv_cache=use_separate_draft_kv_cache, + ) + combined_layer_mask = [ + is_mamba or is_attention for is_mamba, is_attention in zip( + mamba_layer_mask, full_attention_layer_mask) + ] + local_layer_indices, _ = get_pp_layers( + sum(combined_layer_mask), + mapping, + spec_config=spec_config, + layer_mask=combined_layer_mask, + ) + local_mamba_layers = sum(mamba_layer_mask[layer_idx] + for layer_idx in local_layer_indices) + local_attention_layers = sum(full_attention_layer_mask[layer_idx] + for layer_idx in local_layer_indices) + return params, local_mamba_layers, local_attention_layers + + +def _estimate_mamba_hybrid_cache_cost( + model_config, + mapping: Mapping, + *, + max_batch_size: int, + kv_cache_config: KvCacheConfig, + tokens_per_block: int, + max_seq_len: Optional[int], + num_reserved_dummy_slots: int, + include_explicit_snapshots: bool, + cap_partial_attention_snapshots: bool, + is_draft: bool = False, + use_separate_draft_kv_cache: bool = False, + **kwargs, +) -> Tuple[int, int]: + spec_config = kwargs.get("spec_config") + params, local_mamba_layers, local_attention_layers = ( + _get_local_mamba_cache_layout( + model_config, + mapping, + spec_config=spec_config, + is_draft=is_draft, + use_separate_draft_kv_cache=use_separate_draft_kv_cache, + )) + attention_slope = (KVCacheManager.get_cache_size_per_token( + model_config, + mapping, + num_layers=local_attention_layers, + **kwargs, + ) if local_attention_layers > 0 else 0) + state_bytes_per_rank = (local_mamba_layers * + params.get_states_bytes_per_layer(mapping)) + max_resident_sequences = max_batch_size * mapping.pp_size + + if include_explicit_snapshots: + fixed_rules, unaligned_fixed_rules = _mamba_snapshot_rule_counts( + kv_cache_config, max_seq_len, tokens_per_block) + else: + fixed_rules = 0 + unaligned_fixed_rules = 0 + fixed_state_slots = (max_resident_sequences + num_reserved_dummy_slots + + max_resident_sequences * fixed_rules) + attention_block_bytes = attention_slope * tokens_per_block + + interval = _mamba_regular_snapshot_interval(kv_cache_config, max_seq_len) + has_unaligned_periodic_snapshot = (interval is not None + and interval % tokens_per_block != 0) + if cap_partial_attention_snapshots: + # Snapshot alignment is unknown while estimating cache cost. Once a + # snapshot is possible, reserve one retained partial attention page + # per resident lineage. Dummy requests carry no attention capacity. + has_non_live_ssm_capacity = fixed_rules > 0 or interval is not None + partial_attention_slots = (max_resident_sequences + if has_non_live_ssm_capacity else 0) + else: + partial_attention_slots = (max_resident_sequences * + unaligned_fixed_rules) + intercept = (fixed_state_slots * state_bytes_per_rank + + partial_attention_slots * attention_block_bytes) + + if interval is None: + regular_slope = 0 + else: + regular_slope = math.ceil(state_bytes_per_rank / interval) + if (has_unaligned_periodic_snapshot + and not cap_partial_attention_snapshots): + regular_slope += math.ceil(attention_block_bytes / interval) + return attention_slope + regular_slope, intercept + + class CppMambaHybridCacheManager(KVCacheManager, MambaHybridCacheManager): """Hybrid cache manager storing mamba states inside the KVCacheManager pool. Both KV cache blocks and recurrent state blocks are managed by the unified C++ KVCacheManager, enabling block reuse / prefix caching across attention - and mamba layers. This is the default hybrid manager. + and mamba layers. This compatibility manager remains available through the + manager preference override and legacy disaggregated routing. """ @@ -1591,6 +2062,9 @@ def __init__( # accessors (get_mamba_ssm_cache_dtype, use_replay_state_update) work # on ranks with no local mamba layers. self._use_replay_state_update = use_replay_state_update + self._use_gdn_cached_replay_all_layer_commit = ( + use_replay_state_update and model_type == "qwen3_next" + and self.local_num_mamba_layers > 0) self.replay_step_width: Optional[int] = ( spec_config.tokens_per_gen_step if spec_config is not None and use_replay_state_update else None) @@ -1606,6 +2080,12 @@ def __init__( # seed values without any torch.randint. self._seed_request_counter = 0 self.ssm_state_dtype = mamba_ssm_cache_dtype + # Keep the shared Mamba interface valid on PP ranks that do not own a + # local Mamba layer. + self.spec_config = spec_config + self.intermediate_ssm_states = None + self.intermediate_conv_states = None + self.intermediate_state_indices = None if self.local_num_mamba_layers == 0: logger.info( @@ -1627,6 +2107,13 @@ def __init__( is_estimating_kv_cache=is_estimating_kv_cache, is_draft=is_draft, ) + # PP ranks replay the same scheduling decisions, so a rank without + # local Mamba layers must still publish the configured boundaries. + self.kv_cache_config = kv_cache_config + self.linear_attention_metadata = LinearAttentionMetadata() + self.linear_attention_metadata.states_snapshot_interval = ( + kv_cache_config.mamba_state_config.periodic_snapshot_interval + if kv_cache_config.enable_block_reuse else 0) return # Derive ssm_state_shape and conv_state_shape from mamba params (same as MambaCacheManager) @@ -1663,10 +2150,10 @@ def __init__( # Section dims are derived from mHiddenSize and mNGroups*mDState in C++; # we just need to tell C++ which ordering to use. self._rnn_conv_section_layout = model_type # "nemotron_hybrid" or "qwen3_next" - self.ssm_count = math.prod(self.ssm_state_shape) - self.conv_count = math.prod(self.conv_state_shape) - self.ssm_bytes = self.ssm_count * self.ssm_state_dtype.itemsize - self.conv_bytes = self.conv_count * self.conv_state_dtype.itemsize + self.ssm_bytes = (math.prod(self.ssm_state_shape) * + self.ssm_state_dtype.itemsize) + self.conv_bytes = (math.prod(self.conv_state_shape) * + self.conv_state_dtype.itemsize) total_bytes = self.ssm_bytes + self.conv_bytes if total_bytes % self.ssm_state_dtype.itemsize != 0: @@ -1685,7 +2172,7 @@ def __init__( self.linear_attention_metadata.cache_type = LinearCacheType.RECURRENT_STATES.value self.linear_attention_metadata.all_recurrent_states_bytes = self.ssm_bytes + self.conv_bytes self.linear_attention_metadata.states_snapshot_interval = ( - kv_cache_config.mamba_state_cache_interval + kv_cache_config.mamba_state_config.periodic_snapshot_interval if kv_cache_config.enable_block_reuse else 0) # RNN model params for disagg TP-mismatch split/concat. conv_section_map = {"nemotron_hybrid": 1, "qwen3_next": 2} @@ -1790,15 +2277,17 @@ def __init__( device="cpu") self._request_id_to_state_index = {} self._request_id_to_is_dummy = {} - # Batch-order mask aligned with state_indices; duplicate dummy request - # IDs mark every batch row even when they share one cache slot. - self._dummy_request_mask = None - self._dummy_request_mask_host = None self.kv_cache_config = kv_cache_config self.is_estimating_kv_cache = is_estimating_kv_cache self._setup_states() self._setup_replay_buffers(spec_config) + if use_replay_state_update and model_type == "qwen3_next": + logger.info_once( + "Configured GDN cached replay commit mode: small-batch fused, " + "large-batch all-layer", + key="gdn_cached_replay_commit_mode_fused", + ) @staticmethod def get_cache_size_per_token( @@ -1808,68 +2297,74 @@ def get_cache_size_per_token( max_batch_size: int, kv_cache_config: KvCacheConfig, num_layers: Optional[int] = None, + tokens_per_block: int = 32, + max_seq_len: Optional[int] = None, **kwargs, ): """Affine memory model for the unified hybrid KV pool. Returns ``(slope_bytes_per_token, intercept_bytes)``: - * ``slope`` = attention KV bytes per token (parent's formula) plus - the amortized regular-snapshot bytes per token from mamba layers. - * ``intercept`` = ``max_batch_size * num_mamba_layers_per_rank * - state_bytes_per_layer * STATIC_SLOTS_PER_REQUEST``. + * ``slope`` = attention KV bytes per token plus conservative periodic + Mamba-state and partial-attention-snapshot costs. + * ``intercept`` = live and CUDA-graph dummy Mamba state. Memory budget -> max tokens then becomes ``T = (budget - intercept) // slope`` instead of plain ``T = budget // bytes_per_token``. """ - # Lazy import to avoid pulling config_utils into module import order. - from tensorrt_llm._torch.pyexecutor.config_utils import extract_mamba_kv_cache_params + return _estimate_mamba_hybrid_cache_cost( + model_config, + mapping, + max_batch_size=max_batch_size, + kv_cache_config=kv_cache_config, + tokens_per_block=tokens_per_block, + max_seq_len=max_seq_len, + num_reserved_dummy_slots=1, + include_explicit_snapshots=False, + cap_partial_attention_snapshots=False, + **kwargs, + ) - # Attention slope from the parent's existing formula. - attention_slope = KVCacheManager.get_cache_size_per_token( - model_config, mapping, num_layers=num_layers, **kwargs) + @property + def use_gdn_cached_replay_all_layer_commit(self) -> bool: + return self._use_gdn_cached_replay_all_layer_commit - params = extract_mamba_kv_cache_params( - model_config.pretrained_config, - quant_config=model_config.quant_config, - ) + def _commit_gdn_cached_replay_history_layers( + self, + attn_metadata: "AttentionMetadata", + num_decodes: int, + ) -> None: + """Synchronously advance every local GDN checkpoint in one launch.""" + from tensorrt_llm._torch.modules.fla.cached_replay import ( + CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, + commit_gdn_cached_replay_history_layers) - state_bytes_per_layer = params.get_states_bytes_per_layer(mapping) - - # This not precise since pp layers are sharded by their order in model, not by their types. - # e.g. the upper half are all mamba layers while the lower half are all attention layers. - # But that's close enough for real world models where mamba and attention layers are interleaved - # and we don't have access with layer_masks at this point. - num_mamba_layers_per_rank = len( - mapping.pp_layers(params.num_mamba_layers)) - state_bytes_per_rank = num_mamba_layers_per_rank * state_bytes_per_layer - - # Per-request fixed cost. STATIC_SLOTS_PER_REQUEST = 1 today (the - # live mamba state); fixed-position snapshots are not yet - # implemented and would simply increment this constant. With - # pipeline parallelism, multiple microbatches are in-flight - # concurrently on the same rank, so each rank holds Mamba state - # for up to ``max_batch_size * pp_size`` concurrent sequences. - STATIC_SLOTS_PER_REQUEST = 1 - pp_size = mapping.pp_size if mapping is not None else 1 - intercept = (max_batch_size * pp_size * state_bytes_per_rank * - STATIC_SLOTS_PER_REQUEST) - - # Regular-snapshot bytes per token. None / non-positive intervals - # mean "no regular snapshots", so the mamba contribution is zero. - interval = kv_cache_config.mamba_state_cache_interval if kv_cache_config.enable_block_reuse else 0 - if interval is None or interval <= 0: - mamba_slope = 0 - else: - mamba_slope = state_bytes_per_rank // interval - # heuristic: When block reuse is enabled, we assume the mamba snapshots are dominant instead of active states, - # otherwise we may run out of kv cache blocks prior to mamba blocks due to the large number of max_batch_size. - # So we ignore intercept and only calculate max_tokens based on slope - # This can be improved by a more accurate max_batch_size and ISL/OSL estimation in the future. - if mamba_slope > 0: - intercept = 0 - return attention_slope + mamba_slope, intercept + if (not self._use_gdn_cached_replay_all_layer_commit + or num_decodes < CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE): + return + if (self.all_ssm_states is None or self.old_x is None + or self.old_B is None or self.old_dt is None + or self.replay_history_size is None): + raise RuntimeError( + "GDN cached replay all-layer commit requires replay state buffers." + ) + + mamba_metadata = attn_metadata.mamba_metadata + if mamba_metadata.replay_num_decodes != num_decodes: + raise RuntimeError( + "GDN replay metadata contains " + f"{mamba_metadata.replay_num_decodes} decode requests, " + f"but state update received {num_decodes}.") + commit_gdn_cached_replay_history_layers( + ssm_states=self.all_ssm_states, + old_u=self.old_x, + old_k=self.old_B, + old_G=self.old_dt, + replay_work_items=mamba_metadata.replay_work_items[:num_decodes], + n_writes=mamba_metadata.replay_n_writes, + history_size=self.replay_history_size, + ) def shutdown(self): # Release tensor views into the pool before the pool memory is freed, @@ -1956,47 +2451,7 @@ def _prepare_resources(self, scheduled_batch: ScheduledRequests): self._pending_state_transfers = self.impl.copy_linear_attention_block_batch( self.requests) self._setup_state_indices() - # Reset replay double-buffer state for fresh context blocks. A reused - # block (prefix-cache hit or block recycled across requests) may carry - # stale prev_num_accepted_tokens / cache_buf_idx values from a prior - # owner; the replay kernel reads these on the first decode step. - num_contexts = len(scheduled_batch.context_requests) - if num_contexts > 0: - ctx_slots = self.cuda_state_indices[:num_contexts].long() - if (self._use_replay_state_update - and self.prev_num_accepted_tokens is not None - and self.cache_buf_idx is not None): - self.prev_num_accepted_tokens[ctx_slots] = 0 - self.cache_buf_idx[ctx_slots] = 0 - if self.old_x is not None: - self.old_x[:, ctx_slots] = 0 - if self.old_B is not None: - self.old_B[:, ctx_slots] = 0 - if self.old_dt is not None: - self.old_dt[:, ctx_slots] = 0 - if self.old_dA_cumsum is not None: - self.old_dA_cumsum[:, ctx_slots] = 0 - # Deterministic per-context-slot seed rotation. Runs whenever - # the seed buffer exists, including the non-replay SR path. - # Bump the host counter once per batch and write one new seed - # per fresh context slot from a pure function of - # (counter, slot, rank). No torch.randint involved. - if self.mamba_ssm_rand_seed is not None: - self._seed_request_counter += 1 - counter = self._seed_request_counter - rank_offset = self._seed_rank_offset - host_slots = ctx_slots.cpu().tolist() - new_seeds = [ - _compute_deterministic_mamba_seed(counter, slot, - rank_offset) - for slot in host_slots - ] - seed_tensor = torch.tensor( - new_seeds, - dtype=torch.int64, - device=self.mamba_ssm_rand_seed.device, - ) - self.mamba_ssm_rand_seed[ctx_slots] = seed_tensor + self._reset_context_mamba_slots(len(scheduled_batch.context_requests)) def prepare_resources(self, scheduled_batch: ScheduledRequests): super().prepare_resources(scheduled_batch) @@ -2014,14 +2469,13 @@ def flush_state_transfers(self) -> None: self.impl.refresh_blocks() self._pending_state_transfers = False - def is_speculative(self) -> bool: - return self.spec_config is not None - @nvtx_range("hybrid_update_mamba_states") - def update_mamba_states(self, - attn_metadata: "AttentionMetadata", - num_accepted_tokens: torch.Tensor, - state_indices: Optional[torch.Tensor] = None): + def update_mamba_states( + self, + attn_metadata: "AttentionMetadata", + num_accepted_tokens: torch.Tensor, + state_indices: Optional[torch.Tensor] = None, + accepted_leaf_positions: Optional[torch.Tensor] = None): if self.local_num_mamba_layers == 0: return batch_size = attn_metadata.num_seqs @@ -2030,6 +2484,10 @@ def update_mamba_states(self, num_accepted_draft_tokens = ( num_accepted_tokens[num_contexts:num_contexts + num_gens] - 1).to( torch.int32) + # Dynamic tree passes tree-node leaf positions; linear MTP uses depth. + accepted_positions = (accepted_leaf_positions.to(torch.int32) + if accepted_leaf_positions is not None else + num_accepted_draft_tokens) # Match the API of MambaCacheManager.update_mamba_states: callers # may pass per-request state slot indices explicitly (e.g. MTP via # attn_metadata.mamba_metadata.state_indices). Fall back to this @@ -2047,60 +2505,35 @@ def update_mamba_states(self, # writes through the view's real strides (~85% of HBM peak, one launch # per state). if self._use_replay_state_update: - # SSM state is handled incrementally by the kernel. Mirror the - # kernel's checkpoint predicate from the previous PNAT and fixed - # replay step width: checkpoint steps flip buffers, while no-write - # steps append to the active history. - slots = state_indices_d.long() - accepted = num_accepted_tokens[num_contexts:num_contexts + - num_gens].to( - self.prev_num_accepted_tokens. - dtype) - prev_num_accepted_tokens = self.prev_num_accepted_tokens[slots] - wrote_checkpoint = (prev_num_accepted_tokens + - self.replay_step_width - > self.replay_history_size) - next_num_accepted_tokens = torch.where( - wrote_checkpoint, accepted, prev_num_accepted_tokens + accepted) - cache_buf_idx = self.cache_buf_idx[slots] + # Every GDN layer has finished reading the old checkpoint and + # writing its candidate history. Advance all local checkpoints + # in one launch before PNAT and the active history buffer change. + self._commit_gdn_cached_replay_history_layers( + attn_metadata, num_gens) assert self._dummy_request_mask is not None is_dummy_request = self._dummy_request_mask[ num_contexts:num_contexts + num_gens] - next_num_accepted_tokens = torch.where(is_dummy_request, - prev_num_accepted_tokens, - next_num_accepted_tokens) - self.prev_num_accepted_tokens[slots] = next_num_accepted_tokens - self.cache_buf_idx[slots] = torch.where( - is_dummy_request, cache_buf_idx, - torch.where(wrote_checkpoint, 1 - cache_buf_idx, cache_buf_idx)) + replay_metadata = self.get_replay_state_update_metadata() + assert replay_metadata is not None + _advance_replay_state( + replay_metadata, + state_indices_d, + num_accepted_tokens[num_contexts:num_contexts + num_gens], + is_dummy_request, + ) else: # Legacy: copy the accepted SSM state from the intermediate buffer. _promote_mamba_state_triton(self.all_ssm_states, self.intermediate_ssm_states, - src_state_indices, - num_accepted_draft_tokens, + src_state_indices, accepted_positions, state_indices_d) # Conv: both paths save all intermediate conv windows, carry over the # accepted one. _promote_mamba_state_triton(self.all_conv_states, self.intermediate_conv_states, - src_state_indices, - num_accepted_draft_tokens, state_indices_d) - - @torch.inference_mode() - def _refresh_dummy_request_mask(self, is_dummy: List[bool]) -> None: - if self._dummy_request_mask is None: - return - - n = len(is_dummy) - assert n <= self._dummy_request_mask_host.shape[0] - self._dummy_request_mask_host.zero_() - if n > 0: - self._dummy_request_mask_host[:n].copy_( - torch.tensor(is_dummy, dtype=torch.bool)) - self._dummy_request_mask.copy_(self._dummy_request_mask_host, - non_blocking=True) + src_state_indices, accepted_positions, + state_indices_d) def get_num_available_tokens(self, token_num_upper_bound: int, @@ -2117,7 +2550,10 @@ def get_num_available_tokens(self, # request, so no additional capping is required here. interval = (self.linear_attention_metadata.states_snapshot_interval if self.linear_attention_metadata is not None else 0) - if interval and interval > 0: + # Attention-only PP ranks keep the interval so every rank publishes + # identical scheduling boundaries, but they have no recurrent-state + # pool whose capacity should constrain their attention KV cache. + if self.local_num_mamba_layers > 0 and interval and interval > 0: stats = self.impl.get_kv_cache_stats() rs_free = stats.num_free_blocks_per_window_size.get( LinearCacheType.RECURRENT_STATES.value, 0) @@ -2129,64 +2565,6 @@ def get_num_available_tokens(self, result = min(result, rs_token_cap) return max(result, 0) - def get_ssm_states(self, layer_idx: int) -> torch.Tensor: - return self.all_ssm_states[self.mamba_layer_offsets[layer_idx]] - - def get_conv_states(self, layer_idx: int) -> torch.Tensor: - return self.all_conv_states[self.mamba_layer_offsets[layer_idx]] - - def get_intermediate_ssm_states(self, - layer_idx: int) -> Optional[torch.Tensor]: - if self.intermediate_ssm_states is None: - return None - layer_offset = self.mamba_layer_offsets[layer_idx] - return self.intermediate_ssm_states[layer_offset] - - def get_intermediate_conv_states(self, - layer_idx: int) -> Optional[torch.Tensor]: - if self.intermediate_conv_states is None: - return None - layer_offset = self.mamba_layer_offsets[layer_idx] - return self.intermediate_conv_states[layer_offset] - - def mamba_layer_cache( - self, layer_idx: int - ) -> Union[PythonMambaCacheManager.State, - PythonMambaCacheManager.SpeculativeState, None]: - conv = self.get_conv_states(layer_idx) - ssm = self.get_ssm_states(layer_idx) - if self.spec_config is not None: - layer_offset = self.mamba_layer_offsets[layer_idx] - spec_kwargs = {} - # Per-cache-slot Philox seed buffer is shared across replay and - # non-replay MTP paths. The mixer asserts non-None on both - # branches when SR is enabled, so pass it through whenever it - # exists — not just on the replay branch. - if self.mamba_ssm_rand_seed is not None: - spec_kwargs['mamba_ssm_rand_seed'] = self.mamba_ssm_rand_seed - if self._use_replay_state_update: - # Per-layer slices for the replay kernel; shared 1D tensors - # (cache_buf_idx, prev_num_accepted_tokens) are passed - # untouched via the SpeculativeState._SHARED_FIELDS contract. - spec_kwargs['old_x'] = self.old_x[layer_offset] - spec_kwargs['old_B'] = self.old_B[layer_offset] - spec_kwargs['old_dt'] = self.old_dt[layer_offset] - spec_kwargs['old_dA_cumsum'] = self.old_dA_cumsum[layer_offset] - spec_kwargs['cache_buf_idx'] = self.cache_buf_idx - spec_kwargs['prev_num_accepted_tokens'] = ( - self.prev_num_accepted_tokens) - else: - spec_kwargs['intermediate_ssm'] = self.intermediate_ssm_states[ - layer_offset] - return PythonMambaCacheManager.SpeculativeState( - conv=conv, - temporal=ssm, - intermediate_conv_window=self. - intermediate_conv_states[layer_offset], - **spec_kwargs, - ) - return PythonMambaCacheManager.State(conv=conv, temporal=ssm) - def free_resources(self, request: LlmRequest, pin_on_release: bool = False): if request in self.requests: self.requests.remove(request) @@ -2227,6 +2605,31 @@ def _setup_state_indices(self, requests=None) -> None: values = self.host_block_offsets[self.recurrent_states_pool_index, rows, 0, bi] invalid_mask = (values < 0) | (values >= max_blocks) + # The C++ recurrent-state manager uses null page-table entries for + # logical blocks that are not snapshot boundaries. Usually a + # context chunk ends exactly at an allocated snapshot (or at the + # final live-state block), but the scheduler may shorten a chunk + # further when KV capacity is tight. In that case the Mamba + # kernel must keep accumulating into the next allocated snapshot + # or final block, matching copyLinearAttentionBlock(), which also + # skips placeholders when it advances the live state. + for bad_i in invalid_mask.nonzero( + as_tuple=False).flatten().tolist(): + req = requests[bad_i] + if req.is_context_finished: + continue + last_prompt_block = (req.prompt_len - + 1) // self.tokens_per_block + row = self.host_block_offsets[self.recurrent_states_pool_index, + rows[bad_i], 0] + candidates = row[block_indices[bad_i]:last_prompt_block + 1] + valid_candidates = ((candidates >= 0) + & (candidates < max_blocks)).nonzero( + as_tuple=False) + if valid_candidates.numel() > 0: + values[bad_i] = candidates[valid_candidates[0, 0]] + + invalid_mask = (values < 0) | (values >= max_blocks) if invalid_mask.any(): bad_i = int(invalid_mask.nonzero(as_tuple=False)[0, 0]) req = requests[bad_i] @@ -2247,19 +2650,27 @@ def _setup_state_indices(self, requests=None) -> None: self.cuda_state_indices.copy_(self._host_state_indices, non_blocking=True) + is_dummy = [req.is_dummy for req in requests] self._refresh_dummy_request_mask( - [req.is_dummy for req in self.requests]) + is_dummy if requests is + self.requests else [req.is_dummy for req in self.requests]) # Build request_id → pool block offset mapping so that # get_state_indices can return indices in arbitrary request order. - for i, req in enumerate(requests): - self._request_id_to_state_index[ - req.py_request_id] = self._host_state_indices[i].item() - self._request_id_to_is_dummy[req.py_request_id] = req.is_dummy + # Bulk tolist avoids a per-request tensor-index + .item() round-trip. + state_values = self._host_state_indices[:n].tolist() + for req, value, dummy in zip(requests, state_values, is_dummy): + self._request_id_to_state_index[req.py_request_id] = value + self._request_id_to_is_dummy[req.py_request_id] = dummy def get_state_indices(self, request_ids: Optional[List[int]] = None, is_padding: Optional[List[bool]] = None) -> list: + if self.local_num_mamba_layers == 0: + # Mamba metadata is prepared on every PP rank even when this rank + # owns only attention layers. No local kernel consumes these + # indices, so avoid consulting state that is intentionally absent. + return [0] * len(request_ids) if request_ids is not None else [] if request_ids is not None: # Return indices in the order of the caller's request_ids, # not the internal self.requests order. This is critical when @@ -2279,39 +2690,6 @@ def get_state_indices(self, return indices return self.cuda_state_indices - def calc_next_context_chunk_size(self, request: LlmRequest) -> int: - """Compute the next prefill chunk size for a context request when block reuse is enabled. - - When kv_cache_config.enable_block_reuse is True, context prefill must stop exactly at - the positions returned by calc_context_stop_positions (mamba_state_cache_interval boundaries - and block boundaries). This returns the chunk_size to use for the next prefill step so - that the next stop position is not exceeded. - - Args: - request: Context request with prompt_len and context_current_position set. - - Returns: - Number of tokens to prefill in the next step (0 if context is already complete). - """ - prompt_len = request.prompt_len - current = request.context_current_position - if current >= prompt_len: - return 0 - if not self.kv_cache_config.enable_block_reuse: - assert current == 0, ( - "Expected context_current_position to be 0 when block reuse is " - f"disabled, but got {current}") - return prompt_len - current - step = self.linear_attention_metadata.states_snapshot_interval - stop_positions = calc_context_stop_positions(prompt_len, - self.tokens_per_block, - step) - stop_positions = sorted(set(stop_positions)) - for pos in stop_positions: - if pos > current: - return pos - current - return prompt_len - current - def _setup_states(self) -> None: # Pool layout: {numLocalLayers, numBlocks, ssm_bytes + conv_bytes} (as uint8) pool: torch.Tensor = self.impl.get_recurrent_states_pool().view( @@ -2331,164 +2709,870 @@ def _setup_states(self) -> None: self.all_ssm_states.zero_() self.all_conv_states.zero_() - def _setup_mtp_intermediate_states(self, spec_config, - max_batch_size) -> None: - self.spec_config = spec_config - self.intermediate_ssm_states = None - self.intermediate_conv_states = None - self.intermediate_state_indices = None - if self.spec_config is not None: - # DFlash/PARD use 2K query tokens per gen, so size by tokens_per_gen_step. - speculative_num_draft_tokens = self.spec_config.tokens_per_gen_step - 1 - num_local_mamba_layers = len(self.mamba_pp_layers) - - # Legacy SSM intermediate buffer is only needed when replay is - # disabled; replay reads from the per-block double-buffered cache - # set up in _setup_replay_buffers instead. - if not self._use_replay_state_update: - self.intermediate_ssm_states = torch.zeros( - size=[ - num_local_mamba_layers, max_batch_size, - speculative_num_draft_tokens + 1 - ] + self.ssm_state_shape, - dtype=self.ssm_state_dtype, - device="cuda", - ) + def _setup_replay_buffers(self, spec_config) -> None: + cache_size = self.all_ssm_states.shape[1] + device = self.all_ssm_states.device + if not self._allocate_pool_replay_buffers(spec_config, cache_size, + device): + return - self.intermediate_conv_states = torch.zeros( - size=[ - num_local_mamba_layers, max_batch_size, - speculative_num_draft_tokens + 1 - ] + self.conv_state_shape, - dtype=self.conv_state_dtype, - device="cuda", + self._dummy_request_mask = torch.zeros(self.max_batch_size, + dtype=torch.bool, + device=device) + self._dummy_request_mask_host = torch.zeros( + self.max_batch_size, + dtype=torch.bool, + pin_memory=prefer_pinned(), + ) + + +class MambaHybridCacheManagerV2(KVCacheManagerV2, MambaHybridCacheManager): + """Hybrid Mamba cache manager backed by KVCacheManagerV2. + + Attention KV pages and Mamba recurrent-state pages are both owned by the + Python V2 cache manager. Mamba layers are represented as V2 SSM layers, + while this wrapper exposes the state tensors and slot indices expected by + the PyTorch Mamba kernels. + """ + + _supports_additional_snapshot_offsets = True + + def __init__( + self, + # mamba cache parameters + mamba_d_state: int, + mamba_d_conv: int, + mamba_num_heads: int, + mamba_n_groups: int, + mamba_head_dim: int, + mamba_num_layers: int, + mamba_layer_mask: List[bool], + mamba_cache_dtype: torch.dtype, + mamba_ssm_cache_dtype: torch.dtype, + kv_cache_config: KvCacheConfig, + kv_cache_type: CacheTypeCpp, + *, + num_layers: int, + num_kv_heads: Union[int, List[Optional[int]]], + head_dim: int, + tokens_per_block: int, + max_seq_len: int, + max_batch_size: int, + mapping: Mapping, + dtype: DataType = DataType.HALF, + spec_config: Optional["DecodingBaseConfig"] = None, + layer_mask: Optional[List[bool]] = None, + is_estimating_kv_cache: bool = False, + is_draft: bool = False, + use_replay_state_update: bool = False, + mamba_ssm_stochastic_rounding: bool = False, + conv_state_layout: Literal["x_b_c", "q_k_v"] = "x_b_c", + **kwargs, + ) -> None: + if conv_state_layout not in ("x_b_c", "q_k_v"): + raise ValueError( + f"Unsupported convolution state layout: {conv_state_layout!r}") + total_layers = len(mamba_layer_mask) + if layer_mask is None: + full_attention_layer_mask = [False] * total_layers + elif len(layer_mask) != total_layers: + raise ValueError( + f"layer_mask length ({len(layer_mask)}) must match " + f"mamba_layer_mask length ({total_layers})") + else: + full_attention_layer_mask = list(layer_mask) + + combined_layer_mask = [ + mamba_layer_mask[i] or full_attention_layer_mask[i] + for i in range(total_layers) + ] + + self._mamba_layer_mask = list(mamba_layer_mask) + self._use_replay_state_update = use_replay_state_update + self.replay_step_width: Optional[int] = ( + spec_config.tokens_per_gen_step + if spec_config is not None and use_replay_state_update else None) + self.replay_history_size: Optional[int] = ( + max(MIN_REPLAY_HISTORY_SIZE, self.replay_step_width) + if self.replay_step_width is not None else None) + self._mamba_ssm_stochastic_rounding = mamba_ssm_stochastic_rounding + self._seed_rank_offset = _mamba_rank_offset(mapping) + self._seed_request_counter = 0 + num_cuda_graph_padding_dummy_slots = ( + _get_num_cuda_graph_padding_dummy_slots(spec_config, + max_batch_size)) + self._num_reserved_dummy_slots = (num_cuda_graph_padding_dummy_slots + + int(mapping.enable_attention_dp)) + self.ssm_state_dtype = (mamba_ssm_cache_dtype if mamba_ssm_cache_dtype + is not None else mamba_cache_dtype) + self.conv_state_dtype = mamba_cache_dtype + + self.pp_layers, _ = get_pp_layers( + mamba_num_layers + num_layers, + mapping, + spec_config=spec_config, + layer_mask=combined_layer_mask, + ) + self.mamba_pp_layers = [ + layer_idx for layer_idx in self.pp_layers + if mamba_layer_mask[layer_idx] + ] + self.local_num_mamba_layers = len(self.mamba_pp_layers) + + if self.local_num_mamba_layers > 0: + tp_size = mapping.tp_size if not mapping.enable_attention_dp else 1 + d_inner = mamba_head_dim * mamba_num_heads + grouped_state_dim = mamba_n_groups * mamba_d_state + conv_dim = d_inner + 2 * grouped_state_dim + nheads = mamba_num_heads + assert nheads % tp_size == 0, "mamba_num_heads must be divisible by tp_size" + assert conv_dim % tp_size == 0, "conv_dim must be divisible by tp_size" + if kwargs.get("is_disagg", + False) and grouped_state_dim % tp_size != 0: + raise ValueError( + "Disaggregated Mamba transfer requires each convolution " + "state section to be divisible by tp_size") + if use_replay_state_update: + assert mamba_n_groups % tp_size == 0, \ + "replay state update requires mamba_n_groups divisible by tp_size" + self._n_groups_per_rank = mamba_n_groups // tp_size + d_inner_local = d_inner // tp_size + grouped_state_dim_local = grouped_state_dim // tp_size + conv_dim = conv_dim // tp_size + nheads = nheads // tp_size + self.conv_state_shape = [conv_dim, mamba_d_conv - 1] + self.ssm_state_shape = [nheads, mamba_head_dim, mamba_d_state] + # TP-mismatch disaggregated transfers must split the flat + # convolution state at its true semantic boundaries. Mamba2 stores + # [x | B | C], while GDN stores [Q | K | V]. The large section is + # therefore first for Mamba2 and last for GDN. + if conv_state_layout == "x_b_c": + self.conv_section_dims = [ + d_inner_local, + grouped_state_dim_local, + grouped_state_dim_local, + ] + else: + self.conv_section_dims = [ + grouped_state_dim_local, + grouped_state_dim_local, + d_inner_local, + ] + self.ssm_bytes = (math.prod(self.ssm_state_shape) * + self.ssm_state_dtype.itemsize) + self.conv_bytes = (math.prod(self.conv_state_shape) * + self.conv_state_dtype.itemsize) + else: + logger.info( + "No local mamba layers for this rank, skipping mamba state views" ) + self._n_groups_per_rank = 0 + self.conv_state_shape = [] + self.ssm_state_shape = [] + self.conv_section_dims = [] + self.ssm_bytes = 0 + self.conv_bytes = 0 - self.intermediate_state_indices = torch.arange(max_batch_size, - dtype=torch.int32, - device="cuda") + if isinstance(num_kv_heads, int): + per_layer_kv_heads = [num_kv_heads] * total_layers + else: + if len(num_kv_heads) != total_layers: + raise ValueError( + f"num_kv_heads list length ({len(num_kv_heads)}) does not " + f"match total layers ({total_layers})") + per_layer_kv_heads = list(num_kv_heads) + for i, is_mamba in enumerate(mamba_layer_mask): + if is_mamba: + per_layer_kv_heads[i] = 0 - def _setup_replay_buffers(self, spec_config) -> None: - """Allocate per-pool-block replay buffers used by replay_selective_state_update. + self._setup_mtp_intermediate_states(spec_config, max_batch_size) - Unlike the Mixed cache manager (where slots are 0..max_batch_size-1), - the unified C++ KV pool assigns recurrent-state block indices up to - ``num_blocks_in_pool``. The replay kernel indexes ``cache_buf_idx`` and - ``prev_num_accepted_tokens`` by these block indices, so the buffers - must match the pool extent rather than ``max_batch_size``. - """ - # Replay tensors require spec_config + replay path enabled. The - # rand_seed buffer is separable from replay and must also be - # allocated for non-replay SR so the flashinfer path has a - # persistent deterministic seed source. - self.prev_num_accepted_tokens = None - self.cache_buf_idx = None - self.mamba_ssm_rand_seed = None - self.old_x = None - self.old_B = None - self.old_dt = None - self.old_dA_cumsum = None + kv_cache_config = kv_cache_config.model_copy(deep=True) + if any(mamba_layer_mask) and kv_cache_config.enable_block_reuse: + block_reuse_policy = BlockReusePolicy( + kv_cache_config.block_reuse_policy) + if block_reuse_policy == BlockReusePolicy.ALL_REUSABLE: + # SSM reuse is valid only at explicit snapshot boundaries. + kv_cache_config.block_reuse_policy = ( + BlockReusePolicy.PER_REQUEST.value) + self.kv_cache_config = kv_cache_config - if (not self._use_replay_state_update - and not self._mamba_ssm_stochastic_rounding): - return + super().__init__( + kv_cache_config, + kv_cache_type, + num_layers=mamba_num_layers + num_layers, + num_kv_heads=per_layer_kv_heads, + head_dim=head_dim, + tokens_per_block=tokens_per_block, + max_seq_len=max_seq_len, + max_batch_size=max_batch_size, + mapping=mapping, + dtype=dtype, + spec_config=spec_config, + layer_mask=combined_layer_mask, + is_draft=is_draft, + is_estimating_kv_cache=is_estimating_kv_cache, + num_reserved_index_slots=self._num_reserved_dummy_slots, + **kwargs, + ) - cache_size = self.all_ssm_states.shape[1] - device = self.all_ssm_states.device - # Always-available deterministic seed buffer when SR (or replay) - # is on. Works for non-MTP runs because we don't depend on - # spec_config to allocate it. - self.mamba_ssm_rand_seed = _allocate_mamba_seed_buffer( - cache_size, self._seed_rank_offset, device) + self.mamba_layer_offsets = { + layer_id: offset + for offset, layer_id in enumerate(self.mamba_pp_layers) + } + self._request_id_to_state_index = {} + self._request_id_to_is_dummy = {} - if spec_config is None or not self._use_replay_state_update: - # Without spec_config or replay we still keep the seed buffer - # (above) so the non-MTP flashinfer SR path has a persistent - # rand_seed source. - self.prev_num_accepted_tokens = None - self.cache_buf_idx = None - self.old_x = None - self.old_B = None - self.old_dt = None - self.old_dA_cumsum = None - self._dummy_request_mask = None - self._dummy_request_mask_host = None - return + state_index_capacity = (self.max_batch_size + + self._num_reserved_dummy_slots) + self.cuda_state_indices = torch.zeros([state_index_capacity], + dtype=torch.int32, + device="cuda") + self._host_state_indices = torch.zeros([state_index_capacity], + dtype=torch.int32, + pin_memory=prefer_pinned()) - history_size = self.replay_history_size - num_local_mamba_layers = self.local_num_mamba_layers - nheads, head_dim, d_state = self.ssm_state_shape - n_groups_per_rank = self._n_groups_per_rank + if self.local_num_mamba_layers > 0: + first_mamba_local_layer = self.layer_offsets[ + self.mamba_pp_layers[0]] + self.ssm_layer_group_id = self.impl.get_layer_group_id( + LayerId(first_mamba_local_layer)) + self._ssm_page_index_scale = self.impl.get_page_index_scale( + LayerId(first_mamba_local_layer), MambaRole.SSM_STATE) + num_ssm_pages = self.impl.get_page_index_upper_bound( + LayerId(first_mamba_local_layer), MambaRole.SSM_STATE) + num_ssm_slots = ((num_ssm_pages + self._ssm_page_index_scale - 1) // + self._ssm_page_index_scale) + required_live_slots = (self._max_resident_sequences() + + self._num_reserved_dummy_slots) + if num_ssm_slots < required_live_slots: + KVCacheManagerV2.shutdown(self) + raise ValueError( + "The V2 Mamba state pool has only " + f"{num_ssm_slots} slots but needs at least " + f"{required_live_slots} live/dummy slots. Increase the " + "KV cache budget or allocate a larger Mamba pool_ratio.") + self._setup_states() + self._setup_replay_buffers(spec_config) + else: + self.ssm_layer_group_id = None + self._ssm_page_index_scale = 1 + self.all_ssm_states = [] + self.all_conv_states = [] + self._setup_replay_buffers(spec_config) - # Shared across layers (consumed by the replay kernel via slot index). - self.prev_num_accepted_tokens = torch.zeros(cache_size, - dtype=torch.int32, - device=device) - self.cache_buf_idx = torch.zeros(cache_size, - dtype=torch.int32, - device=device) - self._dummy_request_mask = torch.zeros(self.max_batch_size, + @staticmethod + def get_cache_size_per_token(model_config, + mapping: Mapping, + *, + max_batch_size: int, + kv_cache_config: KvCacheConfig, + num_layers: Optional[int] = None, + tokens_per_block: int = 32, + max_seq_len: Optional[int] = None, + **kwargs): + spec_config = kwargs.get("spec_config") + num_reserved_dummy_slots = (_get_num_cuda_graph_padding_dummy_slots( + spec_config, max_batch_size) + int(mapping.enable_attention_dp)) + return _estimate_mamba_hybrid_cache_cost( + model_config, + mapping, + max_batch_size=max_batch_size, + kv_cache_config=kv_cache_config, + tokens_per_block=tokens_per_block, + max_seq_len=max_seq_len, + num_reserved_dummy_slots=num_reserved_dummy_slots, + include_explicit_snapshots=True, + cap_partial_attention_snapshots=True, + **kwargs, + ) + + def _is_local_mamba_layer(self, local_layer_idx: int) -> bool: + return self._mamba_layer_mask[self.pp_layers[local_layer_idx]] + + def _get_pool_roles(self, + pool_id: int) -> Tuple[DataRole, Optional[DataRole]]: + layer_id = int(self.impl.layer_grouping[pool_id][0]) + if self._is_local_mamba_layer(layer_id): + return MambaRole.SSM_STATE, None + return super()._get_pool_roles(pool_id) + + def _max_resident_sequences(self) -> int: + return self.max_batch_size * self.mapping.pp_size + + def _mamba_state_bytes_per_slot(self) -> int: + return self.local_num_mamba_layers * (self.ssm_bytes + self.conv_bytes) + + def _num_ssm_snapshots_for_capacity( + self, + capacity: int, + kv_cache_config: KvCacheConfig, + ) -> int: + if capacity <= 0 or not kv_cache_config.enable_block_reuse: + return 0 + + fixed_rules, _ = _mamba_snapshot_rule_counts(kv_cache_config, + self.max_seq_len, + self.tokens_per_block) + interval = _mamba_regular_snapshot_interval(kv_cache_config, + self.max_seq_len) + regular_snapshots = capacity // interval if interval is not None else 0 + return (self._max_resident_sequences() * fixed_rules + + regular_snapshots) + + def _num_ssm_states_per_typical_request( + self, + capacity: int, + kv_cache_config: KvCacheConfig, + ) -> int: + fixed_rules, _ = _mamba_snapshot_rule_counts( + kv_cache_config, + capacity, + self.tokens_per_block, + ) + # Additional snapshots are stable boundaries that must remain alive. + # Periodic snapshots are evictable cache entries and therefore do not + # increase the guaranteed state count represented by BatchDesc. + return 1 + fixed_rules + + def _typical_request_descs( + self, + capacity: int, + kv_cache_config: KvCacheConfig, + ) -> List[KVCacheDesc]: + """Model one request with one descriptor per live SSM state.""" + num_states = self._num_ssm_states_per_typical_request( + capacity, kv_cache_config) + capacity_per_state, capacity_remainder = divmod(capacity, num_states) + capacities = [ + capacity_per_state + int(i < capacity_remainder) + for i in range(num_states) + ] + return [ + KVCacheDesc( + capacity=state_capacity, + history_length=max(0, state_capacity - 1), + ) for state_capacity in capacities + ] + + def _get_typical_request_capacity( + self, + kv_cache_config: KvCacheConfig, + ) -> int: + if kv_cache_config.avg_seq_len is not None: + return kv_cache_config.avg_seq_len + + fallback_capacity = max(1, self.max_seq_len // 2) + logger.warning( + "'kv_cache_config.avg_seq_len' is not set for a hybrid Mamba " + "model using KV cache manager V2. Falling back to " + f"max_seq_len / 2={fallback_capacity} for cache-pool sizing. Set " + "'kv_cache_config.avg_seq_len' in the YAML configuration to the " + "workload's average total sequence length for an accurate KV/SSM " + "pool ratio.") + return fallback_capacity + + def _get_quota_from_max_tokens(self, max_tokens: int) -> int: + attention_quota = super()._get_quota_from_max_tokens(max_tokens) + num_request_lineages = self._max_resident_sequences() + snapshot_slots = self._num_ssm_snapshots_for_capacity( + max_tokens, self.kv_cache_config) + state_slots = (num_request_lineages + self._num_reserved_dummy_slots + + snapshot_slots) + state_quota = state_slots * self._mamba_state_bytes_per_slot() + # Once the plan contains any non-live SSM capacity, reserve one partial + # attention page per request lineage. This remains conservative when + # the plan contains fewer than one non-live slot per lineage. + extra_attention_quota = (num_request_lineages * + self._attention_cache_bytes_per_token() * + self.tokens_per_block + if snapshot_slots > 0 else 0) + return attention_quota + state_quota + extra_attention_quota + + def _get_max_tokens_from_quota(self, quota: int) -> float: + if self._get_quota_from_max_tokens(0) > quota: + return 0 + + low = 0 + high = 1 + while self._get_quota_from_max_tokens(high) <= quota: + low = high + high *= 2 + if high >= 1 << 62: + return float("inf") + + while low + 1 < high: + mid = (low + high) // 2 + if self._get_quota_from_max_tokens(mid) <= quota: + low = mid + else: + high = mid + return low + + def _minimum_live_gpu_quota(self) -> int: + """Return the minimum quota for live states and one attention page.""" + attention_block_quota = (self._attention_cache_bytes_per_token() * + self.tokens_per_block) + num_state_slots = (self._max_resident_sequences() + + self._num_reserved_dummy_slots) + state_quota = num_state_slots * self._mamba_state_bytes_per_slot() + return max( + self._get_quota_from_max_tokens(0), + state_quota + attention_block_quota, + ) + + def _build_cache_config( + self, config: KVCacheManagerConfigPy) -> KVCacheManagerConfigPy: + kv_cache_config = self.kv_cache_config + cache_tiers = config.cache_tiers + gpu_quota = cache_tiers[0].quota + minimum_live_quota = self._minimum_live_gpu_quota() + if minimum_live_quota > gpu_quota: + raise ValueError( + "The V2 Mamba GPU cache quota is too small for live recurrent " + f"states and attention pages: got {gpu_quota} bytes, need at " + f"least {minimum_live_quota} bytes.") + # _build_base_config already constructed every attention layer, + # including dtype-specific scale and subclass-provided side buffers. + # Preserve those configs and replace only the local Mamba layers. + layers = list(config.layers) + for local_layer_idx, global_layer_idx in enumerate(self.pp_layers): + if self._mamba_layer_mask[global_layer_idx]: + layer_id = LayerId(local_layer_idx) + layers[local_layer_idx] = SsmLayerConfig( + layer_id=layer_id, + buffers=[ + BufferConfig(role=MambaRole.SSM_STATE, + size=self.ssm_bytes), + BufferConfig(role=MambaRole.CONV_STATE, + size=self.conv_bytes), + ], + ) + + dummy_requests = [ + KVCacheDesc(capacity=0, history_length=0) + for _ in range(self._num_reserved_dummy_slots) + ] + constraints = [ + replace( + batch, + kv_caches=[*batch.kv_caches, *dummy_requests], + ) for batch in config.constraints + ] + + typical_step = config.typical_step + if config.initial_pool_ratio is None: + typical_capacity = self._get_typical_request_capacity( + kv_cache_config) + request_descs = self._typical_request_descs(typical_capacity, + kv_cache_config) + typical_step = BatchDesc(request_descs * + self._max_resident_sequences() + + dummy_requests) + # The recurrent (SSM) state pool must hold one slot per resident + # sequence plus every reserved dummy slot. Unlike attention pages, a + # Mamba state is fixed-size per sequence, so this floor is independent + # of sequence length. The base config only emits constraints when + # ``avg_seq_len`` is set, and speculative decoding inflates the reserved + # dummy slots (CUDA-graph padding), so without an explicit floor the SSM + # pool can be undersized (see the live/dummy-slot check in _setup_states + # / __init__). Add a min-slots constraint of zero-capacity requests: + # these cost no attention pages but reserve one SSM slot each. + if any(isinstance(layer, SsmLayerConfig) for layer in layers): + ssm_floor_slots = (self._max_resident_sequences() + + self._num_reserved_dummy_slots) + constraints = [ + *constraints, + BatchDesc([ + KVCacheDesc(capacity=0, history_length=0) + for _ in range(ssm_floor_slots) + ]), + ] + return replace( + config, + layers=layers, + typical_step=typical_step, + constraints=constraints, + # SSM lifecycles require minimum-snapshot commit semantics. The + # flag is harmless when reuse is disabled because no commits are + # attempted, while the runtime config still needs the invariant. + commit_min_snapshot=True, + ) + + def _get_state_buffer(self, local_layer_idx: int, role, dtype: torch.dtype, + state_shape: List[int]) -> torch.Tensor: + addr = self.impl.get_mem_pool_base_address(LayerId(local_layer_idx), + role, PageIndexMode.SHARED) + num_pages = self.impl.get_page_index_upper_bound( + LayerId(local_layer_idx), role) + raw = convert_to_torch_tensor( + TensorWrapper(addr, dtype, [num_pages] + state_shape)) + page_index_scale = self.impl.get_page_index_scale( + LayerId(local_layer_idx), role) + num_slots = (num_pages + page_index_scale - 1) // page_index_scale + # V2 coalesces same-size per-layer buffers inside each slot. Kernels + # index Mamba states by logical slot id, so expose only this layer's + # sub-page from each coalesced slot instead of the raw page-index view. + return raw.as_strided( + [num_slots] + state_shape, + [raw.stride(0) * page_index_scale] + list(raw.stride()[1:]), + ) + + def _setup_states(self) -> None: + local_layer_ids = [ + self.layer_offsets[layer_id] for layer_id in self.mamba_pp_layers + ] + self.all_ssm_states = [ + self._get_state_buffer(local_layer_idx, MambaRole.SSM_STATE, + self.ssm_state_dtype, self.ssm_state_shape) + for local_layer_idx in local_layer_ids + ] + self.all_conv_states = [ + self._get_state_buffer(local_layer_idx, MambaRole.CONV_STATE, + self.conv_state_dtype, self.conv_state_shape) + for local_layer_idx in local_layer_ids + ] + + def _setup_replay_buffers(self, spec_config) -> None: + cache_size = 0 + device = None + if self.local_num_mamba_layers > 0: + cache_size = self.all_ssm_states[0].shape[0] + assert all(t.shape[0] == cache_size for t in self.all_ssm_states) + device = self.all_ssm_states[0].device + if not self._allocate_pool_replay_buffers(spec_config, cache_size, + device): + return + + mask_capacity = self._host_state_indices.shape[0] + self._dummy_request_mask = torch.zeros(mask_capacity, dtype=torch.bool, device=device) - self._dummy_request_mask_host = torch.zeros(self.max_batch_size, - dtype=torch.bool, - pin_memory=prefer_pinned()) - self.old_x = torch.zeros(num_local_mamba_layers, - cache_size, - 2, - history_size, - nheads, - head_dim, - dtype=self.conv_state_dtype, - device=device) - # Per-layer double-buffered caches. - self.old_B = torch.zeros(num_local_mamba_layers, - cache_size, - 2, - history_size, - n_groups_per_rank, - d_state, - dtype=self.conv_state_dtype, - device=device) - self.old_dt = torch.zeros(num_local_mamba_layers, - cache_size, - 2, - nheads, - history_size, - dtype=torch.float32, - device=device) - self.old_dA_cumsum = torch.zeros(num_local_mamba_layers, - cache_size, - 2, - nheads, - history_size, - dtype=torch.float32, - device=device) + self._dummy_request_mask_host = torch.zeros( + mask_capacity, + dtype=torch.bool, + pin_memory=prefer_pinned(), + ) - @property - def use_replay_state_update(self) -> bool: - return self.get_replay_state_update_metadata() is not None + def _attention_cache_bytes_per_token(self) -> int: + # Mamba layers have zero KV heads, so the generic calculation naturally + # returns only bytes owned by local attention layers. + return super().get_cache_bytes_per_token() + + def get_cache_bytes_per_token(self) -> int: + cache_bytes = self._attention_cache_bytes_per_token() + + interval = ( + self.kv_cache_config.mamba_state_config.periodic_snapshot_interval) + if (self.kv_cache_config.enable_block_reuse and interval is not None + and interval > 0): + cache_bytes += self._mamba_state_bytes_per_slot() // interval + if cache_bytes == 0 and self.local_num_mamba_layers > 0: + cache_bytes = self._mamba_state_bytes_per_slot() + return max(1, cache_bytes) + + def get_num_free_blocks(self) -> int: + assert len(self.kv_cache_map) == 0, ( + "get_num_free_blocks is only used when the kv cache manager is empty" + ) + attention_pages = [] + ssm_pages = [] + for local_layer_idx in range(self.num_local_layers): + layer_id = LayerId(local_layer_idx) + if self._is_local_mamba_layer(local_layer_idx): + ssm_pages.append( + self.impl.get_page_index_upper_bound( + layer_id, MambaRole.SSM_STATE) // + self._ssm_page_index_scale) + else: + attention_pages.append( + self.impl.get_page_index_upper_bound(layer_id, Role.KEY) // + self.kv_factor) + if attention_pages: + return max(attention_pages) + return max(ssm_pages) if ssm_pages else 0 - def get_replay_state_update_metadata( - self) -> Optional[ReplayStateUpdateMetadata]: - prev_num_accepted_tokens = getattr(self, 'prev_num_accepted_tokens', - None) - cache_buf_idx = getattr(self, 'cache_buf_idx', None) - if (not self._use_replay_state_update - or prev_num_accepted_tokens is None or cache_buf_idx is None - or self.replay_step_width is None - or self.replay_history_size is None): + @property + def blocks_in_primary_pool(self) -> int: + for local_layer_idx in range(self.num_local_layers): + if self._is_local_mamba_layer(local_layer_idx): + continue + return self.impl.get_page_index_upper_bound( + LayerId(local_layer_idx), Role.KEY) + return 0 + + def get_buffers(self, + layer_idx: int, + kv_layout: str = "NHD") -> Optional[torch.Tensor]: + local_layer_idx = self.layer_offsets[layer_idx] + if self._is_local_mamba_layer(local_layer_idx): return None - return ReplayStateUpdateMetadata( - prev_num_accepted_tokens=prev_num_accepted_tokens, - cache_buf_idx=cache_buf_idx, - replay_step_width=self.replay_step_width, - replay_history_size=self.replay_history_size) + return super().get_buffers(layer_idx, kv_layout) - def get_mamba_ssm_cache_dtype(self) -> torch.dtype: - return self.ssm_state_dtype + def _iter_cache_buffers_for_invalid_check(self) -> Iterable[torch.Tensor]: + for global_layer_id, local_layer_id in self.layer_offsets.items(): + if self._is_local_mamba_layer(local_layer_id): + continue + # A layer group is a lifecycle, not a physical memory pool. + # Differently sized attention buffers can share one lifecycle, + # so scan every attention layer in this diagnostic path. + yield KVCacheManagerV2.get_buffers(self, global_layer_id) - def get_mamba_ssm_rand_seed(self) -> Optional[torch.Tensor]: - """Return the persistent (cache_size,) int64 Philox seed buffer or - None when stochastic rounding is not active for this manager.""" - return getattr(self, 'mamba_ssm_rand_seed', None) + yield from self.all_ssm_states + yield from self.all_conv_states + + def add_dummy_requests( + self, + request_ids: List[int], + token_nums: Optional[List[int]] = None, + is_gen: bool = False, + prepare_resource: bool = True, + max_num_draft_tokens: int = 0, + kv_reserve_draft_tokens: Optional[int] = None, + use_mrope: bool = False, + max_beam_width: int = 1, + encoder_output_lens: Optional[List[int]] = None, + num_extra_decoding_steps: int = 0, + draft_kv_cache_manager: Optional[BaseResourceManager] = None, + ) -> List[LlmRequest]: + requests = super().add_dummy_requests( + request_ids=request_ids, + token_nums=token_nums, + is_gen=is_gen, + prepare_resource=prepare_resource, + max_num_draft_tokens=max_num_draft_tokens, + kv_reserve_draft_tokens=kv_reserve_draft_tokens, + use_mrope=use_mrope, + max_beam_width=max_beam_width, + encoder_output_lens=encoder_output_lens, + num_extra_decoding_steps=num_extra_decoding_steps, + draft_kv_cache_manager=draft_kv_cache_manager, + ) + if requests and prepare_resource: + self._setup_state_indices(requests) + return requests + + def free_resources(self, request: LlmRequest, pin_on_release: bool = False): + kv_cache = self.kv_cache_map.get(request.py_request_id) + if kv_cache is not None and kv_cache.is_active: + self.try_commit_blocks(request, kv_cache) + self._request_id_to_state_index.pop(request.py_request_id, None) + self._request_id_to_is_dummy.pop(request.py_request_id, None) + super().free_resources(request, pin_on_release) + + def prepare_resources(self, scheduled_batch: ScheduledRequests): + super().prepare_resources(scheduled_batch) + if self.local_num_mamba_layers == 0: + return + requests = (scheduled_batch.context_requests + + scheduled_batch.generation_requests) + self._setup_state_indices(requests) + num_contexts = len(scheduled_batch.context_requests) + self._reset_context_mamba_slots(num_contexts) + + def _setup_state_indices(self, requests: List[LlmRequest]) -> None: + if self.local_num_mamba_layers == 0: + return + n = len(requests) + assert n <= self._host_state_indices.shape[0], ( + f"State-index batch size {n} exceeds max_batch_size " + f"{self._host_state_indices.shape[0]}") + self._host_state_indices.zero_() + if n > 0: + for i, req in enumerate(requests): + kv_cache = self.kv_cache_map.get(req.py_request_id) + if kv_cache is None: + raise RuntimeError( + f"Missing V2 KV cache for request {req.py_request_id}") + base_index = kv_cache.get_ssm_block_base_index( + self.ssm_layer_group_id) + if base_index < 0: + raise RuntimeError( + f"Invalid SSM state block index {base_index} for " + f"request {req.py_request_id}") + self._host_state_indices[i] = base_index + + self.cuda_state_indices.copy_(self._host_state_indices, + non_blocking=True) + is_dummy = [req.is_dummy for req in requests] + self._refresh_dummy_request_mask(is_dummy) + state_values = self._host_state_indices[:n].tolist() + for req, value, dummy in zip(requests, state_values, is_dummy): + self._request_id_to_state_index[req.py_request_id] = value + self._request_id_to_is_dummy[req.py_request_id] = dummy + + def get_state_indices(self, + request_ids: Optional[List[int]] = None, + is_padding: Optional[List[bool]] = None): + if self.local_num_mamba_layers == 0: + # Mamba metadata is still prepared on attention-only PP ranks, + # but no local kernel consumes these indices. Return harmless + # placeholders instead of consulting an intentionally empty map. + if request_ids is not None: + return [0] * len(request_ids) + return self.cuda_state_indices + if request_ids is not None: + indices = [ + self._request_id_to_state_index[rid] for rid in request_ids + ] + if is_padding is None: + is_padding = [False] * len(request_ids) + assert len(request_ids) == len(is_padding) + is_dummy = [ + self._request_id_to_is_dummy.get(rid, False) or padding + for rid, padding in zip(request_ids, is_padding) + ] + self._refresh_dummy_request_mask(is_dummy) + return indices + return self.cuda_state_indices + + def get_max_resource_count(self) -> int: + return self.max_batch_size + + def update_mamba_states( + self, + attn_metadata: "AttentionMetadata", + num_accepted_tokens: torch.Tensor, + state_indices: Optional[torch.Tensor] = None, + accepted_leaf_positions: Optional[torch.Tensor] = None): + if self.local_num_mamba_layers == 0: + return + batch_size = attn_metadata.num_seqs + num_contexts = attn_metadata.num_contexts + num_gens = batch_size - num_contexts + num_accepted_draft_tokens = ( + num_accepted_tokens[num_contexts:num_contexts + num_gens] - 1).to( + torch.int32) + # Dynamic tree selects a tree node rather than a linear draft depth. + accepted_positions = (accepted_leaf_positions.to(torch.int32) + if accepted_leaf_positions is not None else + num_accepted_draft_tokens) + if state_indices is None: + state_indices = self.get_state_indices() + state_indices_d = state_indices[num_contexts:num_contexts + + num_gens].to(torch.int32) + src_state_indices = self.intermediate_state_indices[:num_gens] + + if self._use_replay_state_update: + assert self._dummy_request_mask is not None + is_dummy_request = self._dummy_request_mask[ + num_contexts:num_contexts + num_gens] + replay_metadata = self.get_replay_state_update_metadata() + assert replay_metadata is not None + _advance_replay_state( + replay_metadata, + state_indices_d, + num_accepted_tokens[num_contexts:num_contexts + num_gens], + is_dummy_request, + ) + else: + for layer_offset, dst in enumerate(self.all_ssm_states): + _promote_mamba_state_triton( + dst.unsqueeze(0), + self.intermediate_ssm_states[layer_offset:layer_offset + 1], + src_state_indices, + accepted_positions, + state_indices_d, + ) + + for layer_offset, dst in enumerate(self.all_conv_states): + _promote_mamba_state_triton( + dst.unsqueeze(0), + self.intermediate_conv_states[layer_offset:layer_offset + 1], + src_state_indices, + accepted_positions, + state_indices_d, + ) + + def _mark_context_position_as_history(self, request: LlmRequest, + kv_cache) -> None: + """Advance history without making later recurrent state reusable.""" + history_length = request.context_current_position + if history_length <= kv_cache.history_length: + return + capacity = max(kv_cache.capacity, history_length) + if not kv_cache.resize(capacity, history_length=history_length): + raise ValueError( + "Failed to resize history length of V2 Mamba cache for " + f"request {request.py_request_id} to {history_length} tokens") + + def try_commit_blocks(self, request: LlmRequest, kv_cache=None) -> None: + should_block_reuse = (self.enable_block_reuse and not self.is_draft + and not request.is_dummy_request) + if not should_block_reuse: + return + + if kv_cache is None: + kv_cache = self.kv_cache_map.get(request.py_request_id) + if kv_cache is None: + return + + snapshot_points = request.expect_snapshot_points + commit_limit = (min(max(snapshot_points), request.prompt_len) + if snapshot_points else request.prompt_len) + commit_end = min(request.context_current_position, commit_limit) + if (request.context_current_position in request.expect_snapshot_points + and commit_end > kv_cache.num_committed_tokens): + tokens = self._augment_tokens_for_block_reuse( + request.get_tokens(DEFAULT_BEAM_INDEX), + request, + start=kv_cache.num_committed_tokens, + end=commit_end, + ) + kv_cache.commit(tokens) + if request.context_current_position >= commit_limit: + self._mark_context_position_as_history(request, kv_cache) + if request.context_remaining_length == 0: + kv_cache.stop_committing() + + def update_context_resources(self, + scheduled_batch: ScheduledRequests) -> None: + for request in scheduled_batch.context_requests: + kv_cache = self.kv_cache_map.get(request.py_request_id) + if kv_cache is None or not kv_cache.is_active: + continue + + should_block_reuse = (self.enable_block_reuse and not self.is_draft + and not request.is_dummy_request) + is_all_reusable = ( + self.block_reuse_policy == BlockReusePolicy.ALL_REUSABLE) + is_snapshot_boundary = (request.context_current_position + in request.expect_snapshot_points) + has_pending_snapshot = any( + point > request.context_current_position + for point in request.expect_snapshot_points) + should_resize = (not should_block_reuse or + (not is_all_reusable and not has_pending_snapshot)) + should_commit = (is_all_reusable or is_snapshot_boundary + or request.context_remaining_length == 0) + + if should_resize and not kv_cache.resize( + None, request.context_current_position): + raise ValueError( + "Failed to resize history length of V2 Mamba cache for " + f"request {request.py_request_id} to " + f"{request.context_current_position} tokens at context " + "update") + if should_commit: + self.try_commit_blocks(request, kv_cache) + if request.context_remaining_length == 0: + if self.conversation_manager is not None: + self.conversation_manager.save_drop_plan(request, kv_cache) + kv_cache.enable_swa_scratch_reuse = False + + def shutdown(self): + self.all_ssm_states = [] + self.all_conv_states = [] + self.intermediate_ssm_states = None + self.intermediate_conv_states = None + self.intermediate_state_indices = None + self.prev_num_accepted_tokens = None + self.cache_buf_idx = None + self.mamba_ssm_rand_seed = None + self._dummy_request_mask = None + self._dummy_request_mask_host = None + self.old_x = None + self.old_B = None + self.old_dt = None + self.old_dA_cumsum = None + super().shutdown() diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 0109aed475b8..05310fdecf06 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -22,7 +22,7 @@ prefer_pinned, release_gc, torch_dtype_to_str, trace_func) from tensorrt_llm.bindings.internal.runtime import TaskLayerModuleConfig -from tensorrt_llm.inputs.multimodal import (MultimodalInput, MultimodalParams, +from tensorrt_llm.inputs.multimodal import (MultimodalParams, MultimodalRuntimeData, _has_mm_payload_keys, check_mm_embed_cumsum_if_needed, @@ -54,11 +54,13 @@ from ..metadata import KVCacheParams from ..models.checkpoints.base_checkpoint_loader import BaseCheckpointLoader from ..models.modeling_multimodal_encoder import MultimodalEncoderMixin -from ..models.modeling_multimodal_mixin import MultimodalModelMixin +from ..models.modeling_multimodal_mixin import (MultimodalModelMixin, + _build_request_multimodal_input) from ..models.modeling_multimodal_utils import filter_mm_token_from_input_ids from ..models.modeling_utils import DecoderModelForCausalLM from ..modules.fused_moe.moe_load_balancer import (MoeLoadBalancer, MoeLoadBalancerIterContext) +from ..modules.mamba.mamba2_metadata import Mamba2Metadata from ..peft.lora.cuda_graph_lora_manager import CudaGraphLoraManager from ..speculative import (SpecMetadata, get_draft_kv_cache_manager, get_num_extra_kv_tokens, get_spec_metadata, @@ -125,29 +127,35 @@ def _filter_piecewise_capture_num_tokens( ) -> Tuple[list[int], list[int]]: """Cap piecewise CUDA graph capture candidates at the engine's reachable `num_tokens` ceiling `max_batch_size * (max_seq_len - 1 - num_extra_decoding_steps)` - and ensure the ceiling itself is captured. + clamping user-requested sizes above it down to the ceiling. Each in-flight request must leave room for at least one decode token, so the ceiling is the largest forward-pass `num_tokens` the warmup - builder can construct. Including it in the capture set closes the - runtime padding gap between the next-largest candidate and the ceiling - (otherwise ISLs in that gap have no graph >= them and fall back to - eager). - - Returns `(kept, unrecordable)` where `kept` is sorted ascending, - deduped, and contains the ceiling whenever it is positive. + builder can construct. Candidates above the ceiling cannot be + recorded; clamping them down to the ceiling preserves the user's + intent (a requested 128 becomes 127 when only 127 is recordable) + without inventing capture sizes the user never asked + for. Appending sizes beyond the user's list is harmful: runtime + padding rounds iterations up to the nearest captured size, so a far + appended ceiling (e.g. 65536 over a list topping at 13914) would + make every iteration in the gap execute the full ceiling shape. + + Returns `(kept, unrecordable)` where `kept` is sorted ascending and + deduped, with above-ceiling candidates clamped to the ceiling. `unrecordable` is the sorted unique set of input entries above the - ceiling but within `max_num_tokens`. + ceiling but within `max_num_tokens` (the clamped ones, reported so + the caller's warning fires). """ max_capturable_num_tokens = max( 0, max_batch_size * (max_seq_len - 1 - num_extra_decoding_steps)) piecewise_capacity_limit = min(max_num_tokens, max_capturable_num_tokens) - kept = sorted( - {i - for i in candidate_num_tokens if 0 < i <= piecewise_capacity_limit}) - if piecewise_capacity_limit > 0 and (not kept or kept[-1] - < piecewise_capacity_limit): - kept.append(piecewise_capacity_limit) + if piecewise_capacity_limit > 0: + kept = sorted({ + min(i, piecewise_capacity_limit) + for i in candidate_num_tokens if 0 < i <= max_num_tokens + }) + else: + kept = [] unrecordable = sorted({ i for i in candidate_num_tokens @@ -156,25 +164,6 @@ def _filter_piecewise_capture_num_tokens( return kept, unrecordable -def _build_request_multimodal_input( - request: LlmRequest, cache_enabled: bool) -> Optional[MultimodalInput]: - # Skip building this input (and its `from_components` validation) when the cache is disabled. - if not cache_enabled or request.multimodal_hashes is None: - return None - # `multimodal_input` is consumed only by the encoder-cache key path - # (`MultimodalModelMixin._encoder_cache_keys`), which uses UUID-aware multimodal hashes - # internally. Although the `multimodal_uuids` are not exposed as an attribute, they remain in - # the backing C++ request for KV-cache block keys and cache events. - return MultimodalInput.from_components( - request.multimodal_hashes, - request.multimodal_positions, - request.multimodal_lengths, - mm_item_run_cu_offsets=request.multimodal_item_run_cu_offsets, - mm_run_positions=request.multimodal_run_positions, - mm_run_lengths=request.multimodal_run_lengths, - ) - - def _filter_cuda_graph_batch_sizes(cuda_graph_batch_sizes: list[int], max_batch_size: int, max_num_tokens: int, max_total_draft_tokens: int, @@ -459,6 +448,21 @@ def __init__( "decoder CUDA graphs. CUDA graphs will be disabled.") self.cuda_graph_config = None + if (self.cuda_graph_config is not None and self.dtype == torch.float32 + and self._is_encoder_decoder_model()): + # fp32 enc-dec runs unfused cross-attention, whose thop workspace + # size query hardcodes cross_kv_length=0 (attentionOp.cpp, + # Runner::getWorkspaceSize) and undersizes the workspace. The + # graph-capture warmup runs cross_attn in isolation, so the carve + # overruns the allocation (surfaces as cublas EXECUTION_FAILED). + # Keep eager until the upstream size query is fixed. + logger.warning( + "CUDA graphs are not supported for float32 encoder-decoder " + "models. CUDA graphs will be disabled; use a half-precision " + "checkpoint or model_kwargs={'torch_dtype': ...} to enable " + "them.") + self.cuda_graph_config = None + cuda_graph_batch_sizes = self.cuda_graph_config.batch_sizes if self.cuda_graph_config else CudaGraphConfig.model_fields[ 'batch_sizes'].default cuda_graph_padding_enabled = self.cuda_graph_config.enable_padding if self.cuda_graph_config else CudaGraphConfig.model_fields[ @@ -524,7 +528,7 @@ def __init__( f"{unrecordable}: exceeds reachable ceiling " f"max_batch_size*(max_seq_len-1-num_extra_decoding_steps)=" f"{max(0, self.batch_size * (self.max_seq_len - 1 - num_extra_decoding_steps))}. " - f"Capturing the ceiling itself; raise max_seq_len for larger graphs." + f"Clamping them to the ceiling; raise max_seq_len for larger graphs." ) try: @@ -614,7 +618,13 @@ def __init__( ) or self.model_is_wrapped self.max_total_draft_tokens = spec_config.tokens_per_gen_step - 1 self.max_draft_len = spec_config.max_draft_len - self.runtime_draft_len = spec_config.max_draft_len + # Mutable per-iteration draft length (updated each iteration when + # dynamic draft length is enabled; otherwise stays fixed). Tree + # modes verify all tree nodes per step, which can be wider than the + # tree depth used by the drafter loop. + self.runtime_draft_len = (self.max_total_draft_tokens + if not spec_config.is_linear_tree else + self.max_draft_len) else: self.without_logits = False @@ -675,6 +685,18 @@ def __init__( self.position_ids_cuda = torch.empty((self.max_num_tokens, ), dtype=torch.int, device='cuda') + # Steady-state generation-only prepare cache (non-speculative overlap + # decode). Holds the per-request lists that are invariant while the + # scheduled generation batch keeps the same composition, plus a pinned + # cached-token counter advanced by one per step (host-side bookkeeping + # only; the device position buffer is advanced in place and this + # buffer is never the source of an async H2D). Invalidated (set to + # None) by every full _prepare_tp_inputs pass. + self._steady_gen_cache: Optional[Dict[str, Any]] = None + self._steady_gen_positions_pinned = torch.empty( + (self.max_num_tokens, ), + dtype=torch.int, + pin_memory=prefer_pinned()) if self.use_mrope: self.mrope_position_ids_cuda = torch.empty( (3, 1, self.max_num_tokens), dtype=torch.int, device='cuda') @@ -788,6 +810,12 @@ def __init__( self._prepare_inputs_event: Optional[torch.cuda.Event] = None + # Cache for enc-dec cross-attention stable generation steps. + # Populated on the first CUDA-graph generation step; cleared whenever + # the batch composition changes (new encoder request arrives). + self._cross_attn_stable_cached_tokens: Optional[List[int]] = None + self._cross_attn_stable_request_ids: Optional[List[int]] = None + def register_forward_pass_callable(self, callable: Callable): self.forward_pass_callable = callable @@ -863,9 +891,9 @@ def use_mrope(self): @functools.cached_property def _mm_encoder_cache_enabled(self) -> bool: """Whether the multimodal encoder cache is active for this model.""" - multimodal_config = self.model.model_config.multimodal_config - return (multimodal_config is not None - and multimodal_config.encoder_cache_max_bytes > 0) + model = self.model + return (isinstance(model, MultimodalModelMixin) + and model.encoder_cache_active) @property def is_warmup(self): @@ -1132,15 +1160,46 @@ def warmup(self, resource_manager: ResourceManager) -> None: if not is_enc_dec and not self.mapping.has_cp_helix(): self._run_autotuner_warmup(resource_manager) log_mem_snapshot("warmup/after_autotuner") + # Pre-JIT Mamba SSD multi-seq + HAS_INITSTATES=True Triton kernels + # for Mamba hybrid models. Runs regardless of enable_autotuner, + # since MambaHybridCacheManager skips _general_warmup and the + # default autotuner shape is single-seq / no-initstates. Safe + # no-op for non-Mamba models. + self._run_mamba_hybrid_warmup(resource_manager) + log_mem_snapshot("warmup/after_mamba_hybrid") # Release the autotuner's exploration-mode intermediates. The # exploration leftovers are pure waste that hide tens of GiB from # non-torch allocators (cuBLAS handle workspace, UCX/NIXL, # NVSHMEM). gc.collect() torch.cuda.empty_cache() + # Warm up every graph shape before capturing any graph. Attention + # kernels can switch implementations at smaller batch sizes and require + # a larger workspace, so the first pass grows the workspace to its + # maximum size. The second pass runs the final per-shape warmup and + # captures without resizing the workspace. with self.cuda_graph_runner.allow_capture(): + self.cuda_graph_runner.is_warmup_only = True + try: + self._run_cuda_graph_warmup(resource_manager) + finally: + self.cuda_graph_runner.is_warmup_only = False + self.cuda_graph_runner.padding_dummy_requests = {} self._run_cuda_graph_warmup(resource_manager) log_mem_snapshot("warmup/after_cuda_graph_capture") + # Pre-compile DeepGEMM paged_mqa_logits_metadata for every 32-aligned + # batch bucket the runtime can produce (max_batch_size scaled by the + # MTP / DSL expansion factor when applicable). CUDA-graph warmup only + # exercises the batch sizes in cuda_graph_batch_sizes, which round + # up to a subset of buckets; any inference iter whose + # context_lens.size(0) lands on an uncovered bucket triggers an + # nvcc-driven JIT compile (~3s stall inside _prepare_inputs) on + # first touch. Pre-touching every bucket funnels that cost into + # warmup. No-op on non-DSA models. + self._warmup_dg_paged_mqa_logits_metadata() + log_mem_snapshot("warmup/after_dg_paged_mqa_logits_metadata") + self._warmup_cute_dsl_radix_topk() + log_mem_snapshot("warmup/after_cute_dsl_radix_topk") if can_run_general_warmup: # Pre-populate the memory pool with max-shape allocations to reduce # fragmentation at runtime. @@ -1149,6 +1208,124 @@ def warmup(self, resource_manager: ResourceManager) -> None: self._general_warmup(resource_manager, warmup_requests_configs) log_mem_snapshot("warmup/after_memory_pool_prepop") + def _warmup_dg_paged_mqa_logits_metadata(self) -> None: + """Pre-compile DeepGEMM's `get_paged_mqa_logits_metadata` helper for + every 32-aligned batch bucket the runtime can produce. + + DSA's `Indexer.prepare_scheduler_metadata` calls + `deep_gemm.get_paged_mqa_logits_metadata(context_lens, block_kv, + num_sms)` inside `_prepare_inputs` every iteration. The underlying + kernel is templated on `` + where `kAlignedBatchSize = align(context_lens.size(0), 32)` and + `split_kv` / `num_sms` are fixed for a given (block_kv, device). + deep_gemm's Python-side JIT compiles a fresh cubin (spawning + nvcc/cicc/ptxas, ~3s on GB300) the first time each `aligned_bs` + is requested. CUDA-graph warmup exercises only the batch sizes in + `cuda_graph_batch_sizes`, which round up to a subset of the 32- + aligned buckets; every uncovered bucket that the inference + workload later touches produces a 3s stall on that iteration. + Pre-touching every bucket here funnels those compiles into the + deterministic warmup phase. + + `context_lens.size(0)` is not always `num_generations`. For MTP + with `use_expanded_buffers_for_mtp=True` the expanded call passes + `num_generations * (1 + max_draft_tokens)`. For DSL expansion the + call passes `num_generations * dsl_expand_factor`, where + `dsl_expand_factor = next_n // eff` (`eff in kernel_atoms`, see + `_pick_dsl_expand` in `dsa.py`); its worst case is + `next_n = 1 + max_draft_tokens` when `eff == 1`. Reading the + current `dsl_expand_factor` off the metadata would under-estimate + the eventual max (it defaults to 1 before any prepare() has run, + and per-iter picks can differ across iters when CUDA graph is + off), so we use the static upper bound `1 + max_draft_tokens` + for both expansion paths. Bucket range is also scaled by + `max_beam_width` as a defense-in-depth ceiling for future beam + support (no-op today — DSA does not use beam). No-op on non-DSA + models. + + Best-effort: per-bucket JIT failures are logged and skipped so a + single broken bucket does not abort PyExecutor startup. + """ + attn_meta = getattr(self, "attn_metadata", None) + if attn_meta is None: + return + try: + from tensorrt_llm._torch.attention_backend.sparse.dsa import ( + _DG_SCHEDULE_BLOCK_KV, DSAtrtllmAttentionMetadata) + except ImportError: + return + if not isinstance(attn_meta, DSAtrtllmAttentionMetadata): + return + try: + from tensorrt_llm.deep_gemm import get_paged_mqa_logits_metadata + except ImportError: + logger.info( + "[DG warmup] deep_gemm.get_paged_mqa_logits_metadata not " + "available; skipping paged_mqa_logits_metadata prewarm.") + return + + num_sms = attn_meta.num_sms + max_bs = max(1, int(self.batch_size)) + beam_width = max(1, int(getattr(self, "max_beam_width", 1) or 1)) + # Static upper bound on the row-count multiplier applied to + # `context_lens`. Both MTP-expanded and DSL-expanded call sites + # are bounded above by `(1 + max_draft_tokens)`; see the + # docstring for why we don't read the runtime `dsl_expand_factor` + # here. + max_draft_tokens = int(getattr(attn_meta, "max_draft_tokens", 0) or 0) + expands_batch = (getattr(attn_meta, "use_expanded_buffers_for_mtp", + False) + or getattr(attn_meta, "expand_for_dsl", False)) + expand_factor = 1 + max_draft_tokens if expands_batch else 1 + max_aligned = ((max_bs * beam_width * expand_factor + 31) // 32) * 32 + buckets = list(range(32, max_aligned + 32, 32)) + logger.info(f"[DG warmup] Pre-compiling paged_mqa_logits_metadata for " + f"{len(buckets)} aligned batch buckets up to {max_aligned} " + f"(block_kv={_DG_SCHEDULE_BLOCK_KV}, num_sms={num_sms}, " + f"max_bs={max_bs}, beam_width={beam_width}, " + f"expand_factor={expand_factor})") + for aligned_bs in buckets: + # Kernel scans `context_lens` and prefix-sums schedules; a + # zero-filled 2D tensor of shape (aligned_bs, 1) is enough to + # trigger dispatch and compile — the metadata output is + # discarded. + dummy = torch.zeros(aligned_bs, 1, dtype=torch.int32, device="cuda") + try: + _ = get_paged_mqa_logits_metadata(dummy, _DG_SCHEDULE_BLOCK_KV, + num_sms) + except RuntimeError as e: + # Narrow to RuntimeError so signature drifts in + # get_paged_mqa_logits_metadata (TypeError / ValueError) + # surface loudly instead of silently degrading perf. + logger.warning( + f"[DG warmup] paged_mqa_logits_metadata prewarm failed " + f"for aligned_bs={aligned_bs} " + f"(block_kv={_DG_SCHEDULE_BLOCK_KV}, num_sms={num_sms}); " + f"skipping bucket. {type(e).__name__}: {e}") + torch.cuda.synchronize() + + def _warmup_cute_dsl_radix_topk(self) -> None: + """Pre-compile the DSA radix-filter CuTe DSL decode top-k for every + cluster_size band during warmup, before serving. + + Captured geometries are already compiled by the warmup-step forwards; + this fills in the bands the eager (non-captured) decode path can still + hit (mixed prefill+decode batch, or cuda_graph disabled) so they do + not pay a first-touch JIT stall on a live request. DSA-specific params + live on the metadata, so delegate to it. No-op on non-DSA models. + """ + attn_meta = getattr(self, "attn_metadata", None) + if attn_meta is None: + return + try: + from ..attention_backend.sparse.dsa import \ + DSAtrtllmAttentionMetadata + except ImportError: + return + if isinstance(attn_meta, DSAtrtllmAttentionMetadata): + next_n = 1 + self.original_max_draft_len + attn_meta.warmup_cute_dsl_radix_topk(next_n) + def _general_warmup(self, resource_manager: ResourceManager, warmup_requests_configs: List[Tuple[int, int]]): """ @@ -1293,11 +1470,9 @@ def trtllm_gen_fmha_jit_warmup(): # not compile the optimized single-sequence K123 variant. A # non-aligned five-chunk context enters the pure K123 path. _KIMI_KDA_PREFILL_WARMUP_TOKENS = 257 - logger.info( - "Adding Kimi KDA pure-prefill warmup with " - f"{_KIMI_KDA_PREFILL_WARMUP_TOKENS} context tokens") - warmup_requests_configs.append( - (_KIMI_KDA_PREFILL_WARMUP_TOKENS, 0)) + logger.info("Adding Kimi KDA pure-prefill warmup with " + f"{_KIMI_KDA_PREFILL_WARMUP_TOKENS} context tokens") + warmup_requests_configs.append((_KIMI_KDA_PREFILL_WARMUP_TOKENS, 0)) for num_tokens, num_gen_requests in warmup_requests_configs: warmup_request = self._create_warmup_request( @@ -1318,6 +1493,18 @@ def trtllm_gen_fmha_jit_warmup(): resource_manager=resource_manager) torch.cuda.synchronize() + @staticmethod + def _release_megamoe_profiling_scratch(): + # MegaMoE tuning resources are shared across layers, so only the engine + # can release them after its full autotune warmup and before graph + # capture. Later eviction could invalidate a captured workspace pointer. + from ..custom_ops import cute_dsl_megamoe_custom_op as _megamoe_op + release_megamoe_scratch = getattr(_megamoe_op, + "release_megamoe_profiling_scratch", + None) + if release_megamoe_scratch is not None: + release_megamoe_scratch() + def _run_autotuner_warmup(self, resource_manager: ResourceManager): """Runs a forward pass to populate the autotuner cache.""" if not self.llm_args.enable_autotuner: @@ -1371,6 +1558,8 @@ def _run_autotuner_warmup(self, resource_manager: ResourceManager): ) AutoTuner.get().print_profiling_cache() + self._release_megamoe_profiling_scratch() + # Clear workspace buffers allocated during the autotuner forward pass. # The autotuner runs a context-only forward with max_num_tokens, which # causes the global Buffers pool to cache large MoE/GEMM workspaces. @@ -1379,6 +1568,145 @@ def _run_autotuner_warmup(self, resource_manager: ResourceManager): clear_memory_buffers() torch.cuda.empty_cache() + def _run_mamba_hybrid_warmup(self, resource_manager: ResourceManager): + """Pre-JIT the Mamba SSD multi-seq + HAS_INITSTATES=True Triton kernels. + + Mamba hybrid models (e.g. Nemotron 3 Super 120B, Nemotron-Nano-12B-v2) + skip ``_general_warmup`` because ``can_run_general_warmup`` is False + when the KV cache manager is a ``MambaHybridCacheManager``. The default + ``_run_autotuner_warmup`` then issues a single ``least_requests=True`` + prefill = 1 sequence with ``num_cached_tokens_per_seq = 0``, which only + compiles the ``num_seqs == 1`` / ``HAS_INITSTATES=False`` variants of + the SSD kernels. The first real serve iteration with chunked prefill + and multiple context requests then triggers autotune of the missing + variants mid-inference, producing a ~30 s stall / large P99 spike. + + This method runs two extra forward passes to compile those variants + during warmup: + + 1. ``least_requests=False`` — splits ``curr_max_num_tokens`` into many + short sequences, forcing the multi-seq path of + ``cu_seqlens_to_chunk_indices_offsets_triton`` and its + ``_cu_seqlens_triton_kernel``. + 2. ``least_requests=False`` inside + ``Mamba2Metadata.force_initial_states_for_warmup()`` — same as (1) + plus the ``HAS_INITSTATES=True`` variants of + ``_state_passing_fwd_kernel``, ``_chunk_scan_fwd_kernel``, and + ``_chunk_state_varlen_kernel``. + + Runs regardless of ``enable_autotuner``. Wraps in ``autotune()`` when + the autotuner is enabled so op-level (M,N,K) caches also get primed + for these shapes. Set ``TLLM_MAMBA_MULTISEQ_WARMUP=0`` to disable. + """ + if os.environ.get("TLLM_MAMBA_MULTISEQ_WARMUP", "1") != "1": + return + kv_cache_manager = resource_manager.get_resource_manager( + self.kv_cache_manager_key) + if kv_cache_manager is None or not isinstance(kv_cache_manager, + MambaHybridCacheManager): + return + + token_num_upper_bound = min(self.max_num_tokens, + self.batch_size * (self.max_seq_len - 1)) + curr_max_num_tokens = kv_cache_manager.get_num_available_tokens( + token_num_upper_bound=token_num_upper_bound, + max_num_draft_tokens=self.original_max_draft_len) + if curr_max_num_tokens < 4: + return + + # Cap the multi-seq warmup token count so we don't fill the KV cache + # to the brim. The autotuner warmup that ran just before this uses + # ``least_requests=True`` (few long sequences) which fits comfortably + # even when ``curr_max_num_tokens`` is close to the block ceiling. + # ``least_requests=False`` instead spreads the token budget across + # ``batch_size`` short sequences; when each sequence's length lands + # exactly on a block boundary AND the KV cache has ``num_extra_kv_tokens`` + # or ``num_extra_decoding_steps`` > 0 (e.g. spec decoding cases), + # ``add_token`` needs to allocate one extra block per sequence, which + # ``_create_warmup_request``'s ``blocks_to_use`` estimate doesn't + # account for. On a small KV pool (e.g. Qwen3.5 hybrid with DFlash spec + # decoding on a single H100: 259 blocks total, ``max_num_tokens=8192`` + # nearly saturates it), that extra per-sequence block overflows the + # pool and crashes with "Can't allocate new blocks for window size N". + # The point of this warmup is only to trigger ``num_seqs > 1`` + + # ``HAS_INITSTATES=True`` kernel variants — a modest token budget + # achieves that with plenty of block headroom. + WARMUP_TOKEN_CAP = 4096 + capped_num_tokens = min(curr_max_num_tokens, WARMUP_TOKEN_CAP) + + logger.info( + "Running Mamba hybrid warmup (multi-seq + HAS_INITSTATES=True)...") + + # (num_tokens, num_gen_requests, least_requests, force_initstates) + mamba_warmup_shapes = [ + (capped_num_tokens, 0, False, False), + (capped_num_tokens, 0, False, True), + ] + + autotuner_enabled = self.llm_args.enable_autotuner + cache_path = os.environ.get("TLLM_AUTOTUNER_CACHE_PATH", None) + autotune_ctx = (autotune(cache_path=cache_path) + if autotuner_enabled else contextlib.nullcontext()) + + with self.no_cuda_graph(), autotune_ctx: + for (num_tokens_i, num_gen_requests_i, least_req_i, + force_init_i) in mamba_warmup_shapes: + init_ctx = (Mamba2Metadata.force_initial_states_for_warmup() + if force_init_i else contextlib.nullcontext()) + try: + with init_ctx: + warmup_request = self._create_warmup_request( + resource_manager, + num_tokens_i, + num_gen_requests_i, + least_requests=least_req_i) + with self._release_batch_context( + warmup_request, resource_manager) as batch: + if batch is None and self.mapping.tp_size <= 1: + continue + self._assert_all_tp_ranks_have_warmup_batch( + batch, num_tokens_i) + if batch is None: + continue + spec_resource_manager = resource_manager.get_resource_manager( + ResourceManagerType.SPEC_RESOURCE_MANAGER) + if self.is_draft_model and isinstance( + spec_resource_manager, + Eagle3ResourceManager): + spec_resource_manager.is_first_draft = True + + self.forward(batch, + new_tensors_device=None, + resource_manager=resource_manager) + + if autotuner_enabled: + AutoTuner.get().cache_pp_recv() + AutoTuner.get().cache_pp_send() + AutoTuner.get().clean_pp_flag() + + torch.cuda.synchronize() + except (torch.OutOfMemoryError, RuntimeError) as e: + # Catch both OOM and RuntimeError. C++ KV cache block + # allocation ("Can't allocate new blocks for window size + # N") surfaces as RuntimeError, not torch.OutOfMemoryError. + # This warmup is a pure perf optimization: if a shape + # doesn't fit for any reason, log and skip; the model then + # JIT-compiles the missing kernel variants lazily on the + # first real request (i.e. the pre-fix behavior). + logger.warning(f"Mamba hybrid warmup skipped for shape " + f"num_tokens={num_tokens_i}, " + f"num_gen_requests={num_gen_requests_i}, " + f"force_initstates={force_init_i}: " + f"{type(e).__name__}: {e}") + # Mirror _general_warmup_impl: an OOM between dispatch() + # and combine() leaves MoE A2A state in ``dispatched``, + # tripping ``dispatch called twice`` on the next forward. + self._reset_moe_alltoall_state() + torch.cuda.empty_cache() + + clear_memory_buffers() + torch.cuda.empty_cache() + def _compute_dynamic_draft_len_mapping(self) -> Optional[Dict[int, int]]: """Compute graph_bs → draft_len mapping for dynamic draft length feature. @@ -1479,23 +1807,27 @@ def _get_graphs_to_capture( for draft_len in draft_lengths] def _run_cuda_graph_warmup(self, resource_manager: ResourceManager): - """Captures CUDA graphs for various batch sizes and draft lengths.""" + """Warm up or capture CUDA graphs for the configured graph shapes.""" if not (self.cuda_graph_runner.enabled or self._torch_compile_piecewise_cuda_graph): return self._capture_generation_cuda_graphs(resource_manager) - self._capture_piecewise_cuda_graphs(resource_manager) + # Piecewise graphs have separate capture machinery and do not use the + # whole-model attention workspace. Capture them only on the second pass. + if not self.cuda_graph_runner.is_warmup_only: + self._capture_piecewise_cuda_graphs(resource_manager) def _capture_generation_cuda_graphs(self, resource_manager: ResourceManager): - """Captures CUDA graphs for pure generation steps.""" + """Warm up or capture pure-generation CUDA graph shapes.""" if not self.cuda_graph_runner.enabled: return - logger.info( - f"Creating CUDA graph instances for {len(self._cuda_graph_batch_sizes)} batch sizes." - ) + operation = ("warmup" + if self.cuda_graph_runner.is_warmup_only else "capture") + logger.info(f"Running CUDA graph {operation} for " + f"{len(self._cuda_graph_batch_sizes)} batch sizes.") spec_resource_manager = resource_manager.get_resource_manager( ResourceManagerType.SPEC_RESOURCE_MANAGER) @@ -1503,7 +1835,7 @@ def _capture_generation_cuda_graphs(self, cuda_graph_batch_sizes = sorted(self._cuda_graph_batch_sizes, reverse=True) - # Determine which graphs to capture + # Determine which graph shapes to process. graphs_to_capture = self._get_graphs_to_capture(cuda_graph_batch_sizes, spec_resource_manager) graphs_to_capture = sorted(graphs_to_capture, reverse=True) @@ -1630,7 +1962,7 @@ def _run_capture_pass(force_non_greedy: bool, label: str) -> None: f"not enough KV cache space.") continue logger.info( - f"Run generation-only CUDA graph warmup ({label}) " + f"Run generation-only CUDA graph {operation} ({label}) " f"for batch size={bs}, draft_len={draft_len}, " f"max_seq_len={max_seq_len}") self.enable_spec_decode = draft_len > 0 or self.is_draft_model or ( @@ -1648,6 +1980,21 @@ def _run_capture_pass(force_non_greedy: bool, label: str) -> None: finally: if force_non_greedy and spec_metadata is not None: spec_metadata._force_non_greedy_for_capture = False + # The base object is not the only holder of the flag: every + # graph captured during this pass cached its own SHALLOW COPY + # of spec_metadata (create_cuda_graph_metadata -> copy.copy), + # which inherited the flag. Those copies are reseated as the + # live spec_metadata on every later replay, so leaving the + # flag set there makes _scan_one_model_sampling overwrite + # every serving request's sampling params with the synthetic + # capture values (0.7 / 50 / 0.9). Clear them here -- after + # the pass has finished capturing, so the flag was still in + # effect for every capture that needed it. + cleared = self.cuda_graph_runner.clear_capture_only_spec_state( + ) + logger.info( + f"Cleared capture-only sampling override from {cleared} " + "cached CUDA graph spec metadata object(s).") # Pass 1: greedy fast-path (dummy requests carry no sampling params, # so is_all_greedy_sample is naturally True). @@ -1977,6 +2324,16 @@ def free_warmup_requests() -> None: model_config = self.model.model_config.pretrained_config max_position_embeddings = getattr(model_config, 'max_position_embeddings', None) + if is_enc_dec: + # For enc-dec models the engine max_seq_len covers the encoder + # sequence, which may exceed the decoder's position table (e.g. + # Whisper: 1500 encoder positions vs max_target_positions=448). + decoder_position_limit = getattr(model_config, + 'max_target_positions', None) + if decoder_position_limit is not None: + max_position_embeddings = ( + decoder_position_limit if max_position_embeddings is None + else min(max_position_embeddings, decoder_position_limit)) if max_position_embeddings is not None: token_num = min(token_num, max_position_embeddings - _kv_draft) @@ -2586,15 +2943,55 @@ def _get_all_rank_ctx_requests(self, num_ctx_requests: int): return list(self.dist.tp_allgather(num_ctx_requests)) return None + def _sync_group_all_greedy_sample(self, spec_metadata) -> None: + """All-gather the per-rank greedy flags and store the group AND. + + Why the sampling-path choice must be group-uniform under + ADP + LM-head TP is documented on the anchor, + ``SpecMetadata.group_all_greedy_sample``. Local contract: called once + per iteration, right after ``update_is_all_greedy_sample`` and BEFORE + the CUDA graph key is built. The gate is pure config (identical on + every rank), so ranks also agree on whether the exchange happens; the + gather spans the whole TP group, a superset of any LM-head-TP + subgroup. A dedicated host all-gather rather than a piggyback on the + ``all_rank_num_tokens`` exchange, which runs in ``_prepare_inputs`` -- + after the graph key, too late for the key to see the synced value. + """ + # enable_lm_head_tp_in_adp implies enable_attention_dp (asserted in + # Mapping.__init__), so ADP needs no separate check here. + if not (self.mapping.enable_lm_head_tp_in_adp + and spec_metadata.use_rejection_sampling): + return + local_flag = bool(spec_metadata.is_all_greedy_sample) + all_flags = self.dist.tp_allgather(local_flag) + spec_metadata.group_all_greedy_sample = all(all_flags) + # Also overwrite the live flag directly: this iteration's scan already + # ran (update_is_all_greedy_sample just returned) and the CUDA graph + # key reads the flag next -- the stored override only takes effect on + # the NEXT rescan (populate), which is after key selection. + spec_metadata.is_all_greedy_sample = spec_metadata.group_all_greedy_sample + def _set_spec_metadata_all_rank_num_tokens( - self, spec_metadata: SpecMetadata, + self, + spec_metadata: SpecMetadata, spec_all_rank_num_tokens: List[int], - all_rank_num_seqs: List[int]) -> None: + all_rank_num_seqs: List[int], + all_rank_num_gens: Optional[List[int]] = None) -> None: # Eagle3 / MTP-eagle one-model use subseq_all_rank_num_tokens for # draft loop iterations i>0 (per-sequence counts, since each # sequence contributes one token per iteration). spec_metadata.all_rank_num_tokens = spec_all_rank_num_tokens spec_metadata.all_rank_num_seqs = all_rank_num_seqs + # DSpark can draft only after the target processes the current bonus token, + # because it consumes captured target-layer hidden states for that token. + # Prefill computes hidden states for prompt tokens; the first generated token + # is sampled from the last prompt logits and has not itself passed through the + # target layers. Thus context requests seed the rolling window but do not run + # the draft. On mixed steps, num_seqs therefore over-counts the draft MoE + # workload; gen-only per-rank counts keep the FUSED_COMM (DeepGEMM MegaMoE) + # chunk loop identical across EP ranks. + if all_rank_num_gens is not None: + spec_metadata.all_rank_num_gens = all_rank_num_gens if (spec_metadata.spec_dec_mode.is_mtp_eagle_one_model() or spec_metadata.spec_dec_mode.is_eagle3_one_model()): spec_metadata.subseq_all_rank_num_tokens = all_rank_num_seqs @@ -2669,8 +3066,11 @@ def get_padded_piecewise_tokens(tokens): def _prepare_multimodal_indices(self, input_ids: list[int]): input_ids = torch.tensor(input_ids, dtype=torch.int, device="cpu") vocab_size = self.model.config.vocab_size - # TODO: unify naming of mm_token_ids across models - mm_token_ids = getattr(self.model, "mm_token_ids", None) + # `multimodal_token_ids` is the common wrapper-model contract. Keep the legacy name as a + # fallback for models not yet migrated to `MultimodalModelMixin`. + mm_token_ids = getattr(self.model, "multimodal_token_ids", None) + if mm_token_ids is None: + mm_token_ids = getattr(self.model, "mm_token_ids", None) text_token_indices, mm_token_indices = filter_mm_token_from_input_ids( input_ids, vocab_size=vocab_size, mm_token_ids=mm_token_ids) @@ -2744,12 +3144,45 @@ def _prepare_enc_dec_cross_attn_inputs( skip_cross_kv_projection = True if attn_metadata.is_cuda_graph and attn_metadata.has_cross_sub_metadata: - cross_attn_metadata = attn_metadata.update_cross_metadata( - encoder_seq_lens=encoder_seq_lens, - cross_kv_cache_manager=cross_kv_cache_manager, - encoder_num_cached_tokens_per_seq= - encoder_num_cached_tokens_per_seq, + # Fast path for stable CUDA-graph generation steps: the encoder + # KV lengths (kv_lens_cuda) and the frozen prompt lengths + # (prompt_lens_cuda) are identical across all generation steps + # for a fixed batch. Skip the expensive torch.tensor() allocations + # and H2D copies inside prepare() when nothing has changed. + is_stable_gen_step = ( + new_encoder_tokens == 0 # pure generation, no new cross-KV + and self._cross_attn_stable_cached_tokens + == encoder_num_cached_tokens_per_seq + and self._cross_attn_stable_request_ids + == attn_metadata.request_ids # same batch and row order ) + if is_stable_gen_step: + cross_attn_metadata = attn_metadata.cross + # Only refresh the decoder-side Python references that the + # kernel reads; these are pointer-level updates with no alloc. + cross_attn_metadata._seq_lens = attn_metadata.seq_lens + cross_attn_metadata._seq_lens_cuda = attn_metadata.seq_lens_cuda + cross_attn_metadata.prompt_lens = attn_metadata.prompt_lens + cross_attn_metadata.request_ids = attn_metadata.request_ids + cross_attn_metadata.num_contexts = attn_metadata.num_contexts + else: + cross_attn_metadata = attn_metadata.update_cross_metadata( + encoder_seq_lens=encoder_seq_lens, + cross_kv_cache_manager=cross_kv_cache_manager, + encoder_num_cached_tokens_per_seq= + encoder_num_cached_tokens_per_seq, + ) + cross_attn_metadata.prepare() + if new_encoder_tokens == 0: + # Record this stable state for future fast-path use. + self._cross_attn_stable_cached_tokens = list( + encoder_num_cached_tokens_per_seq) + self._cross_attn_stable_request_ids = list( + attn_metadata.request_ids) + else: + # Batch changed (new encoder request); reset cache. + self._cross_attn_stable_cached_tokens = None + self._cross_attn_stable_request_ids = None else: cross_attn_metadata = attn_metadata.create_cross_metadata( cross_kv_cache_manager=cross_kv_cache_manager, @@ -2759,7 +3192,18 @@ def _prepare_enc_dec_cross_attn_inputs( ) if attn_metadata.is_cuda_graph: attn_metadata.cross = cross_attn_metadata - cross_attn_metadata.prepare() + if new_encoder_tokens == 0: + self._cross_attn_stable_cached_tokens = list( + encoder_num_cached_tokens_per_seq) + self._cross_attn_stable_request_ids = list( + attn_metadata.request_ids) + else: + self._cross_attn_stable_cached_tokens = None + self._cross_attn_stable_request_ids = None + else: + self._cross_attn_stable_cached_tokens = None + self._cross_attn_stable_request_ids = None + cross_attn_metadata.prepare() return { "encoder_hidden_states": packed_encoder_hidden_states, @@ -2923,12 +3367,14 @@ def _prepare_incremental_update_metadata( # Handle distributed spec metadata if enable_attention_dp: sequence_lengths = spec_metadata.seq_lens - all_rank_num_tokens = self.dist.tp_cp_allgather( - [spec_metadata.num_tokens, - len(sequence_lengths)]) + all_rank_num_tokens = self.dist.tp_cp_allgather([ + spec_metadata.num_tokens, + len(sequence_lengths), attn_metadata.num_generations + ]) self._set_spec_metadata_all_rank_num_tokens( spec_metadata, [item[0] for item in all_rank_num_tokens], - [item[1] for item in all_rank_num_tokens]) + [item[1] for item in all_rank_num_tokens], + [item[2] for item in all_rank_num_tokens]) # Set iteration states - batch dictionary updates self.iter_states.update({ @@ -3182,6 +3628,10 @@ def _apply_incremental_update_target( else: prompt_lengths[idx] = request.py_prompt_len + # Physical KV length for the kernels: subtract the tokens a + # KV-cache compression manager evicted (tracked on the request, + # 0 without compression). Position ids and the cached_tokens stat + # keep the logical count. if request.is_dummy: num_cached_tokens_per_seq[idx] = base_past_seen request.cached_tokens = base_past_seen @@ -3192,9 +3642,11 @@ def _apply_incremental_update_target( num_previous_batch] = request.py_batch_idx num_previous_batch += 1 - num_cached_tokens_per_seq[ - idx] = base_past_seen + num_tokens_per_extend_request - request.cached_tokens = num_cached_tokens_per_seq[idx].item() + request.cached_tokens = (base_past_seen + + num_tokens_per_extend_request) + num_cached_tokens_per_seq[idx] = ( + base_past_seen + num_tokens_per_extend_request - + request.py_num_compressed_tokens) request.py_batch_idx = request.py_seq_slot @@ -3282,6 +3734,143 @@ def _apply_incremental_update_target( return inputs, self.gather_ids_cuda[:num_generation_tokens] + def _can_use_steady_gen_fast_prepare( + self, scheduled_requests: ScheduledRequests, + new_tokens_device: Optional[torch.Tensor], + next_draft_tokens_device: Optional[torch.Tensor], + spec_metadata: Optional[SpecMetadata]) -> bool: + """Check whether the cached steady-state generation prepare applies. + + The cache is only recorded by a full _prepare_tp_inputs pass whose + batch consisted purely of non-dummy generation requests that all had + a previous overlap-scheduler tensor (see the recording site), so the + per-step check only needs to confirm the dynamic conditions: still a + generation-only batch with the exact same requests in the same order. + """ + cache = self._steady_gen_cache + if cache is None or self.is_warmup: + return False + if new_tokens_device is None or next_draft_tokens_device is not None \ + or spec_metadata is not None: + return False + if scheduled_requests.num_context_requests > 0: + return False + generation_requests = scheduled_requests.generation_requests + if len(generation_requests) != cache['num_requests']: + return False + return cache['request_ids'] == [ + request.py_request_id for request in generation_requests + ] + + @nvtx_range("_apply_steady_gen_fast_prepare") + def _apply_steady_gen_fast_prepare( + self, kv_cache_manager: Union[KVCacheManager, KVCacheManagerV2], + attn_metadata: AttentionMetadata, + new_tensors_device: SampleStateTensors, + resource_manager: Optional[ResourceManager]): + """Prepare inputs for an unchanged generation-only batch. + + Every request advanced by exactly one committed token since the last + prepare, so instead of re-walking the batch in Python this advances + the cached positions in place (device position buffer plus a pinned + host counter), reuses the seq-slot buffer already on device, and + refreshes only the per-step metadata. For mrope models (recorded only + for batches with no actual mrope work) the (3,1,N) broadcast buffer + the model reads is the one advanced. + """ + cache = self._steady_gen_cache + num_requests = cache['num_requests'] + + # Positions and cached-token counts are the same values in this + # regime; advance both by one. The device-side position buffer is + # advanced in place: it still holds the previous step's positions + # because only _prepare_tp_inputs writes it and the cache validity + # invariant guarantees the previous pass wrote these same rows. This + # avoids reusing a mutated pinned buffer as the source of an async + # H2D whose previous-step copy may still be pending under the overlap + # scheduler (the nvbug 6293536 hazard class; see + # KVCacheManager._stage_block_offsets_for_copy). The pinned buffer is + # host-side bookkeeping only. + use_mrope = cache['use_mrope'] + positions = self._steady_gen_positions_pinned[:num_requests] + positions.add_(1) + if use_mrope: + # Text-only batch on an mrope model: the recording pass broadcast + # the scalar positions onto all three axes of the (3,1,N) buffer, + # which is what the model (and any captured CUDA graph) reads, so + # advance it in place. position_ids_cuda is reseeded by the next + # full pass. + self.mrope_position_ids_cuda[:, :, :num_requests].add_(1) + else: + self.position_ids_cuda[:num_requests].add_(1) + num_cached_tokens_per_seq = positions.tolist() + + # Gather this step's input tokens from the previous iteration's device + # sample buffer; the seq-slot indices in previous_batch_indices_cuda + # are unchanged since the last full pass. + previous_slots = self.previous_batch_indices_cuda[:num_requests] + new_tokens = new_tensors_device.new_tokens[:1, previous_slots, :self. + max_beam_width] + self.input_ids_cuda[:num_requests * self.max_beam_width].copy_( + new_tokens.flatten(), non_blocking=True) + + if not attn_metadata.is_cuda_graph: + attn_metadata.seq_lens = cache['seq_lens_ones'] + attn_metadata.beam_width = 1 + attn_metadata.request_ids = cache['request_ids'] + attn_metadata.prompt_lens = cache['prompt_lens'] + attn_metadata.num_contexts = 0 + attn_metadata.num_chunked_ctx_requests = 0 + attn_metadata.kv_cache_params = KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=num_cached_tokens_per_seq, + num_extra_kv_tokens=get_num_extra_kv_tokens(None)) + attn_metadata.kv_cache_manager = kv_cache_manager + if hasattr(self.model.model_config.pretrained_config, 'chunk_size'): + attn_metadata.mamba_chunk_size = \ + self.model.model_config.pretrained_config.chunk_size + with nvtx_range("steady_gen_metadata_prepare"): + attn_metadata.prepare() + + attn_all_rank_num_tokens = self._get_all_rank_num_tokens(attn_metadata) + padded_num_tokens, can_run_piecewise_cuda_graph, attn_all_rank_num_tokens = \ + self._get_padding_params(num_requests, 0, attn_all_rank_num_tokens) + set_per_request_piecewise_cuda_graph_flag(can_run_piecewise_cuda_graph) + attn_metadata.padded_num_tokens = ( + padded_num_tokens if padded_num_tokens != num_requests else None) + virtual_num_tokens = num_requests + if attn_metadata.padded_num_tokens is not None: + self.input_ids_cuda[num_requests:padded_num_tokens].fill_(0) + # Zero-fill the padding tail of whichever position layout the + # model consumes, matching the full pass. + if use_mrope: + self.mrope_position_ids_cuda[:, :, num_requests: + padded_num_tokens].fill_(0) + else: + self.position_ids_cuda[num_requests:padded_num_tokens].fill_(0) + virtual_num_tokens = padded_num_tokens + + self.iter_states['num_ctx_requests'] = 0 + self.iter_states['num_ctx_tokens'] = 0 + self.iter_states['num_generation_tokens'] = num_requests + self.iter_states['cached_kv_tokens'] = sum(num_cached_tokens_per_seq) + + if use_mrope: + final_position_ids = \ + self.mrope_position_ids_cuda[:, :, :virtual_num_tokens] + else: + final_position_ids = \ + self.position_ids_cuda[:virtual_num_tokens].unsqueeze(0) + inputs = { + 'attn_metadata': attn_metadata, + 'input_ids': self.input_ids_cuda[:virtual_num_tokens], + 'position_ids': final_position_ids, + 'inputs_embeds': None, + 'multimodal_params': [], + 'resource_manager': resource_manager, + } + return inputs, None + def _prepare_tp_inputs( self, scheduled_requests: ScheduledRequests, @@ -3318,12 +3907,28 @@ def _prepare_tp_inputs( if self._can_use_incremental_update(scheduled_requests, new_tokens_device, next_draft_tokens_device): + # Spec engines never record the steady-gen cache, but invalidate + # defensively so the two fast paths can never interleave if the + # gates ever evolve. + self._steady_gen_cache = None return self._apply_incremental_update( scheduled_requests, kv_cache_manager, attn_metadata, spec_metadata, new_tensors_device, cache_indirection_buffer, num_accepted_tokens_device, req_id_to_old_request, resource_manager) + if self._can_use_steady_gen_fast_prepare(scheduled_requests, + new_tokens_device, + next_draft_tokens_device, + spec_metadata): + return self._apply_steady_gen_fast_prepare(kv_cache_manager, + attn_metadata, + new_tensors_device, + resource_manager) + # Any full pass invalidates the steady-state cache; it is re-recorded + # at the end of this pass when the batch qualifies. + self._steady_gen_cache = None + # Hoist self.use_mrope to a function-scope local so the per-request / # per-context-request mrope branches use LOAD_FAST instead of LOAD_ATTR. _use_mrope = self.use_mrope @@ -3348,8 +3953,14 @@ def _prepare_tp_inputs( ] # (start_idx, end_idx, (3,1,L) mrope_pos_ids) per multimodal request mrope_delta_write_seq_slots = [] mrope_delta_read_seq_slots = [] + # Whether any generation request in this batch carries real MRoPE + # metadata; see the post-loop cleanup below. + has_gen_mrope_delta = False # Extra model-side cache slot reserved for CUDA graph / warmup dummy - # requests, whose outputs are discarded. + # requests, whose outputs are discarded, and for generation requests + # that carry no MRoPE metadata at all. The cache is zero-initialized and + # the write path only ever targets real ``py_seq_slot``s, so this slot + # permanently reads back a zero delta. mrope_dummy_seq_slot = self.max_num_tokens * self.mapping.pp_size num_accepted_draft_tokens = [] # per request is_enc_dec = self._is_encoder_decoder_model() @@ -3407,11 +4018,16 @@ def append_cross_attention_state(request: LlmRequest, for request in scheduled_requests.context_requests: request_ids.append(request.py_request_id) - all_prompt_tokens = request.get_tokens(0) draft_lens.append(0) begin_compute = request.context_current_position end_compute = begin_compute + request.context_chunk_size - prompt_tokens = all_prompt_tokens[begin_compute:end_compute] + # Fetch only the current chunk. get_tokens(0) marshals the whole + # O(seq_len) VecTokens into a Python list of boxed ints; chunked + # prefill re-enters this loop for every chunk of the same prompt, so + # that is O(L) per chunk = O(L^2/chunk) over the prefill. + # get_tokens_range copies only [begin, end) -> O(chunk). + prompt_tokens = request.get_tokens_range(0, begin_compute, + end_compute) position_ids.extend( range(begin_compute, begin_compute + len(prompt_tokens))) @@ -3440,8 +4056,9 @@ def append_cross_attention_state(request: LlmRequest, py_request_id] = request.py_num_accepted_draft_tokens_indices prompt_lengths.append(len(prompt_tokens)) past_seen_token_num = begin_compute - num_cached_tokens_per_seq.append(past_seen_token_num) - request.cached_tokens = num_cached_tokens_per_seq[-1] + num_cached_tokens_per_seq.append(past_seen_token_num - + request.py_num_compressed_tokens) + request.cached_tokens = past_seen_token_num append_cross_attention_state( request, project_encoder_output=not request.py_skip_cross_kv_projection @@ -3455,7 +4072,7 @@ def append_cross_attention_state(request: LlmRequest, request.py_multimodal_data, begin_compute=past_seen_token_num, end_compute=end_compute, - prompt_len=len(all_prompt_tokens), + prompt_len=request.get_num_tokens(0), ) mm_data = request.py_multimodal_data or {} cumsum = mm_data.get('multimodal_embed_mask_cumsum') @@ -3472,6 +4089,7 @@ def append_cross_attention_state(request: LlmRequest, request, self._mm_encoder_cache_enabled), multimodal_data=request.py_multimodal_data, multimodal_runtime=py_multimodal_runtime, + mm_item_order=getattr(request, "py_mm_item_order", None), input_ids_start_offset=context_start_idx) # Transfer any cross-iter MM encoder prefetch event stamped on the request onto the # freshly-built MultimodalParams. The downstream consume site reads it from the wrapper, @@ -3485,7 +4103,9 @@ def append_cross_attention_state(request: LlmRequest, multimodal_params.encoder_event = mm_encoder_event request.py_mm_encoder_event = None if multimodal_params.has_content(): - # TODO: Visit later to decide the appropriate position of sending multimodal data & selectively sending multimodal data + # TODO(TRTLLM-14726): Check the persistent MM encoder cache before H2D and avoid + # transferring raw encoder inputs for full hits in both regular and + # side-stream-prefetched paths. multimodal_params.to_device("multimodal_data", "cuda", pin_memory=prefer_pinned(), @@ -3494,16 +4114,25 @@ def append_cross_attention_state(request: LlmRequest, "multimodal_data_device_paths", None)) if _use_mrope: - mrope_config = multimodal_params.multimodal_data[ - 'mrope_config'] - mrope_pos_ids = mrope_config['mrope_position_ids'] - ctx_mrope_position_ids = mrope_pos_ids[:, :, begin_compute: - begin_compute + - len(prompt_tokens)] - # Record as (start_idx, end_idx, (3,1,L) mrope_pos_ids) - mrope_position_ids.append( - (len(position_ids) - len(prompt_tokens), - len(position_ids), ctx_mrope_position_ids)) + # A request may carry multimodal content but no MRoPE + # metadata (a text-only prompt whose input processor skips + # ``mrope_config``, or a model that does not consume it). + # Its per-axis positions are just the scalar positions, + # which the (3,1,N) seeding further below already + # broadcasts, so leave that span alone. + mrope_config = multimodal_params.multimodal_data.get( + 'mrope_config') or {} + mrope_pos_ids = mrope_config.get('mrope_position_ids') + if mrope_pos_ids is not None: + ctx_mrope_position_ids = mrope_pos_ids[:, :, + begin_compute: + begin_compute + + len(prompt_tokens + )] + # Record as (start_idx, end_idx, (3,1,L) mrope_pos_ids) + mrope_position_ids.append( + (len(position_ids) - len(prompt_tokens), + len(position_ids), ctx_mrope_position_ids)) mrope_position_delta = mrope_config.get( 'mrope_position_deltas') if mrope_position_delta is not None: @@ -3632,8 +4261,9 @@ def append_cross_attention_state(request: LlmRequest, list( range(past_seen_token_num, past_seen_token_num + 1 + num_draft_tokens))) - num_cached_tokens_per_seq.append(past_seen_token_num) - request.cached_tokens = num_cached_tokens_per_seq[-1] + num_cached_tokens_per_seq.append( + past_seen_token_num - request.py_num_compressed_tokens) + request.cached_tokens = past_seen_token_num # update batch index request.py_batch_idx = request.py_seq_slot else: @@ -3660,9 +4290,11 @@ def append_cross_attention_state(request: LlmRequest, previous_pos_indices.extend([previous_batch_idx] * runtime_tokens_per_gen_step) - num_cached_tokens_per_seq.append(past_seen_token_num + - runtime_tokens_per_gen_step) - request.cached_tokens = num_cached_tokens_per_seq[-1] + num_cached_tokens_per_seq.append( + past_seen_token_num + runtime_tokens_per_gen_step - + request.py_num_compressed_tokens) + request.cached_tokens = (past_seen_token_num + + runtime_tokens_per_gen_step) if self.enable_spec_decode and spec_config.spec_dec_mode.extend_ctx( self.attn_backend) and spec_config.is_linear_tree: prompt_lengths.append(runtime_tokens_per_gen_step) @@ -3673,12 +4305,16 @@ def append_cross_attention_state(request: LlmRequest, for request in first_draft_requests: request_ids.append(request.py_request_id) - all_prompt_tokens = request.get_tokens(0) draft_lens.append(0) - begin_compute = len( - all_prompt_tokens) - self.original_max_draft_len - 1 + # Only the length and the last (original_max_draft_len+1) tokens are + # needed here; get_num_tokens is O(1) and get_tokens_range copies only + # the requested window, whereas get_tokens(0) marshals the whole + # O(seq_len) VecTokens into a Python list. + _num_tokens = request.get_num_tokens(0) + begin_compute = _num_tokens - self.original_max_draft_len - 1 end_compute = begin_compute + self.original_max_draft_len + 1 - prompt_tokens = all_prompt_tokens[begin_compute:end_compute] + prompt_tokens = request.get_tokens_range(0, begin_compute, + end_compute) position_ids.extend( range(begin_compute, begin_compute + len(prompt_tokens))) @@ -3719,7 +4355,8 @@ def append_cross_attention_state(request: LlmRequest, py_request_id] = request.py_num_accepted_draft_tokens_indices prompt_lengths.append(request.py_prompt_len) past_seen_token_num = begin_compute - num_cached_tokens_per_seq.append(past_seen_token_num) + num_cached_tokens_per_seq.append(past_seen_token_num - + request.py_num_compressed_tokens) append_cross_attention_state(request, project_encoder_output=False) # update batch index @@ -3796,7 +4433,8 @@ def append_cross_attention_state(request: LlmRequest, request.cached_tokens = past_seen_token_num for beam in range(beam_width): position_ids.append(position_id) - num_cached_tokens_per_seq.append(past_seen_token_num) + num_cached_tokens_per_seq.append( + past_seen_token_num - request.py_num_compressed_tokens) prompt_lengths.append(request.py_prompt_len) gather_ids.append(len(position_ids) - 1) @@ -3805,19 +4443,21 @@ def append_cross_attention_state(request: LlmRequest, "py_mrope_position_delta", None) if mrope_position_delta is None and request.py_multimodal_data: - mrope_config = request.py_multimodal_data[ - 'mrope_config'] - mrope_position_delta = mrope_config[ - 'mrope_position_deltas'] - if mrope_position_delta.device.type == "cpu": - mrope_position_delta = maybe_pin_memory( - mrope_position_delta).to(device='cuda', - dtype=torch.int32, - non_blocking=True) - mrope_config[ - 'mrope_position_deltas'] = mrope_position_delta - request.py_mrope_position_delta = mrope_position_delta + mrope_config = request.py_multimodal_data.get( + 'mrope_config') or {} + mrope_position_delta = mrope_config.get( + 'mrope_position_deltas') + if mrope_position_delta is not None: + if mrope_position_delta.device.type == "cpu": + mrope_position_delta = maybe_pin_memory( + mrope_position_delta).to(device='cuda', + dtype=torch.int32, + non_blocking=True) + mrope_config[ + 'mrope_position_deltas'] = mrope_position_delta + request.py_mrope_position_delta = mrope_position_delta if mrope_position_delta is not None: + has_gen_mrope_delta = True # NOTE: Expanding position_ids to 3D tensor who is using mrope gen_mrope_position_ids = (past_seen_token_num + mrope_position_delta).expand( @@ -3851,6 +4491,21 @@ def append_cross_attention_state(request: LlmRequest, gen_mrope_position_ids)) mrope_delta_read_seq_slots.append( delta_read_seq_slot) + else: + # No MRoPE metadata for this request (text-only prompt + # on an MRoPE model): its delta is zero by construction, + # so read the reserved zero slot instead of skipping the + # append. The kernel indexes ``mrope_position_deltas`` + # by *generation batch index* + # (decoderMaskedMultiheadAttentionTemplate.h), so a list + # that is sparse w.r.t. the generation batch would + # silently shift every later request onto another + # request's delta. No ``mrope_position_ids`` span is + # recorded: the broadcast scalar position is already + # this request's answer on all three axes. + for _ in range(beam_width): + mrope_delta_read_seq_slots.append( + mrope_dummy_seq_slot) # Equivalent to the original `is_generation_admission and # request.py_multimodal_data`. The batch-level flag is checked # first so non-multimodal models pay one LOAD_FAST per request @@ -3868,6 +4523,14 @@ def append_cross_attention_state(request: LlmRequest, if not request.is_cuda_graph_dummy: gen_request_seq_slots.append(request.py_seq_slot) + if _use_mrope and not has_gen_mrope_delta: + # Every generation request in this batch resolved to the zero slot, + # so the gathered deltas would be an all-zero vector -- identical to + # passing no deltas at all. Dropping the list keeps the steady-state + # generation fast path (which requires the mrope lists to be empty) + # reachable for text-only batches on MRoPE models. + mrope_delta_read_seq_slots.clear() + previous_batch_len = len(previous_batch_indices) def previous_seq_slots_device(): @@ -4239,6 +4902,12 @@ def previous_seq_slots_device(): if hasattr(self.model.model_config.pretrained_config, 'chunk_size'): attn_metadata.mamba_chunk_size = self.model.model_config.pretrained_config.chunk_size + # Some sparse backends (RocketKV) clamp + # kv_cache_params.num_cached_tokens_per_seq in place during prepare(), + # and KVCacheParams holds the list by reference. Snapshot the true + # pre-prepare counts so the steady-gen recording below stores values + # that the per-step prepare() can re-clamp from scratch. + num_cached_tokens_snapshot = list(num_cached_tokens_per_seq) attn_metadata.prepare() cross_attention_inputs = (self._prepare_enc_dec_cross_attn_inputs( cross_encoder_hidden_states, @@ -4327,6 +4996,16 @@ def previous_seq_slots_device(): num_accepted_draft_tokens)] if isinstance(spec_metadata, Eagle3SpecMetadata): spec_metadata.request_accepted_path = request_accepted_path + # The capture-only sampling override must never be live outside CUDA + # graph warmup: it replaces every request's sampling params with + # synthetic capture values. It leaked here once already (inherited by + # the cached graph metadata shallow copies), so assert rather than + # trust the teardown. + assert self.is_warmup or not getattr( + spec_metadata, '_force_non_greedy_for_capture', False + ), ("capture-only sampling override (_force_non_greedy_for_capture) " + "is set outside CUDA graph warmup; serving requests would be " + "silently decoded with the synthetic capture sampling params") # No-op for non 1-model spec_metadata.populate_sampling_params_for_one_model( scheduled_requests.all_requests()) @@ -4339,12 +5018,14 @@ def previous_seq_slots_device(): inputs['spec_metadata'] = spec_metadata if self.enable_attention_dp: - all_rank_num_tokens = self.dist.tp_cp_allgather( - [spec_metadata.num_tokens, - len(sequence_lengths)]) + all_rank_num_tokens = self.dist.tp_cp_allgather([ + spec_metadata.num_tokens, + len(sequence_lengths), spec_metadata.num_generations + ]) self._set_spec_metadata_all_rank_num_tokens( spec_metadata, [item[0] for item in all_rank_num_tokens], - [item[1] for item in all_rank_num_tokens]) + [item[1] for item in all_rank_num_tokens], + [item[2] for item in all_rank_num_tokens]) if mm_token_indices is not None: self._ship_multimodal_indices( @@ -4367,6 +5048,52 @@ def previous_seq_slots_device(): self.previous_request_ids = all_gen_request_ids self.has_previous_device_draft = next_draft_tokens_device is not None + # Record the steady-state generation cache when this pass handled + # purely non-dummy generation requests that all carried a previous + # overlap-scheduler tensor (previous_batch_len == _n_gen implies + # every request took that branch and none appended input_ids). + # While the batch composition holds, the next passes only need to + # advance positions by one and refresh per-step metadata. + # MRoPE models are supported only for batches with no actual mrope + # work (text-only requests, empty mrope lists below): the full + # pass routes use_mrope models through the (3,1,N) + # mrope_position_ids_cuda layout even then (to keep torch.compile + # guards stable), with all three axes equal to the scalar + # positions, so the fast path advances that buffer in place and + # returns the same layout (see _apply_steady_gen_fast_prepare). + if (self.spec_config is None and not self.is_draft_model + and spec_metadata is None and new_tokens_device is not None + and self.guided_decoder is None + and not self.enable_attention_dp and not mrope_position_ids + and not mrope_delta_write_seq_slots + and not mrope_delta_read_seq_slots + and not self.use_beam_search and self.max_beam_width == 1 + and not is_enc_dec and not _has_cp_helix + and num_ctx_requests == 0 and not extend_requests + and not first_draft_requests and _n_gen > 0 + and previous_batch_len == _n_gen and num_tokens == 0 + and not _has_any_multimodal_request + and not multimodal_params_list and not lora_params + and attn_metadata.padded_num_tokens is None + and self._get_position_id_offset() == 0 + and not getattr(kv_cache_manager, + "kv_compression_manages_history", False)): + self._steady_gen_positions_pinned[:_n_gen].copy_( + torch.as_tensor(num_cached_tokens_snapshot, + dtype=torch.int)) + self._steady_gen_cache = { + 'num_requests': + _n_gen, + 'request_ids': + all_gen_request_ids, + 'prompt_lens': + prompt_lengths, + 'seq_lens_ones': + maybe_pin_memory(torch.ones(_n_gen, dtype=torch.int)), + 'use_mrope': + _use_mrope, + } + return inputs, self.gather_ids_cuda[:len( gather_ids)] if self.enable_spec_decode else None @@ -4412,6 +5139,7 @@ def _prepare_tp_inputs_no_cache( multimodal_input=_build_request_multimodal_input( request, self._mm_encoder_cache_enabled), multimodal_data=request.py_multimodal_data, + mm_item_order=getattr(request, "py_mm_item_order", None), input_ids_start_offset=context_start_idx) multimodal_params.to_device("multimodal_data", "cuda", @@ -4539,14 +5267,15 @@ def _prepare_tp_inputs_no_cache( if spec_metadata is not None: all_rank_num_tokens = self.dist.tp_cp_allgather([ attn_metadata.num_tokens, spec_metadata.num_tokens, - len(sequence_lengths) + len(sequence_lengths), spec_metadata.num_generations ]) attn_metadata.all_rank_num_tokens = [ item[0] for item in all_rank_num_tokens ] self._set_spec_metadata_all_rank_num_tokens( spec_metadata, [item[1] for item in all_rank_num_tokens], - [item[2] for item in all_rank_num_tokens]) + [item[2] for item in all_rank_num_tokens], + [item[3] for item in all_rank_num_tokens]) else: all_rank_num_tokens = self.dist.tp_cp_allgather( attn_metadata.num_tokens) @@ -5210,7 +5939,17 @@ def warmup_encoder(self) -> None: torch.cuda.empty_cache() self._run_autotuner_warmup_encoder() + # Warm up every encoder graph shape before capturing any graph. Some + # attention kernels switch implementations at smaller shapes and need + # a larger workspace, so the first pass grows the workspace to its + # maximum size. The second pass runs the final per-shape warmup and + # captures without resizing the workspace. with self.encoder_cuda_graph_runner.allow_capture(): + self.encoder_cuda_graph_runner.is_warmup_only = True + try: + self._run_cuda_graph_warmup_encoder() + finally: + self.encoder_cuda_graph_runner.is_warmup_only = False self._run_cuda_graph_warmup_encoder() # Pre-populate the memory pool with max-shape allocations to reduce @@ -5260,14 +5999,14 @@ def _run_autotuner_warmup_encoder(self) -> None: AutoTuner.get().print_profiling_cache() def _run_cuda_graph_warmup_encoder(self) -> None: - """Captures whole-model CUDA graphs for the encode-only path.""" + """Warm up or capture whole-model encode-only CUDA graphs.""" if not self.encoder_cuda_graph_runner.enabled: return self._capture_encoder_cuda_graphs() def _capture_encoder_cuda_graphs(self) -> None: - """Capture whole-model encoder CUDA graphs for all feasible keys. + """Warm up or capture encoder CUDA graphs for all feasible keys. Feasibility filter (also used in source): nt >= prev_sl + bs (enough tokens for this sl bucket) @@ -5283,8 +6022,9 @@ def _capture_encoder_cuda_graphs(self) -> None: num_tokens_list = sorted(self._cuda_graph_num_tokens) seq_lens_list = sorted(self._cuda_graph_seq_lens) - num_captured = 0 - logger.info("Capturing encoder CUDA graphs ...") + operation = "warmup" if runner.is_warmup_only else "capture" + num_processed = 0 + logger.info(f"Running encoder CUDA graph {operation} ...") for bs in batch_sizes: if bs > self.batch_size: continue @@ -5303,13 +6043,14 @@ def _capture_encoder_cuda_graphs(self) -> None: if inputs is None: continue - logger.info(f"Encoder CUDA graph capture: " + logger.info(f"Encoder CUDA graph {operation}: " f"bs={bs}, nt={nt}, sl={sl}") self.encoder_forward(inputs) torch.cuda.synchronize() - num_captured += 1 + num_processed += 1 - logger.info(f"Captured {num_captured} encoder CUDA graph(s).") + logger.info(f"Completed encoder CUDA graph {operation} for " + f"{num_processed} graph shape(s).") @torch.inference_mode() @with_model_extra_attrs(lambda self: self.model.extra_attrs) @@ -5362,7 +6103,9 @@ def encoder_forward(self, inputs: Dict[str, Any], return self._forward_step(model_inputs, **forward_kwargs) - if self.encoder_cuda_graph_runner.needs_capture(key): + needs_capture = self.encoder_cuda_graph_runner.needs_capture( + key) + if needs_capture: def forward_fn( capture_inputs: Dict[str, Any]) -> Dict[str, Any]: @@ -5372,16 +6115,20 @@ def forward_fn( return self._forward_step(capture_inputs, **forward_kwargs) - self.encoder_cuda_graph_runner.capture( + capture_outputs = self.encoder_cuda_graph_runner.capture( key, forward_fn, { **model_inputs, "_forward_kwargs": forward_kwargs }) - with MoeLoadBalancerIterContext(moe_load_balancer): - graph_outputs = self.encoder_cuda_graph_runner.replay( - key, { - **model_inputs, "_forward_kwargs": forward_kwargs - }) + if self.encoder_cuda_graph_runner.is_warmup_only: + graph_outputs = capture_outputs + else: + with MoeLoadBalancerIterContext(moe_load_balancer): + graph_outputs = self.encoder_cuda_graph_runner.replay( + key, { + **model_inputs, "_forward_kwargs": + forward_kwargs + }) # Return a clone to avoid sharing data_ptr with the static buffers. outputs = {} @@ -5506,6 +6253,7 @@ def forward(self, if spec_metadata is not None: spec_metadata.update_is_all_greedy_sample( padded_requests.all_requests()) + self._sync_group_all_greedy_sample(spec_metadata) maybe_attn_metadata, maybe_spec_metadata, key = self.cuda_graph_runner.maybe_get_cuda_graph( padded_requests, @@ -5555,7 +6303,8 @@ def forward(self, gather_ids=gather_ids, gather_context_logits=gather_context_logits) else: - if self.cuda_graph_runner.needs_capture(key): + needs_capture = self.cuda_graph_runner.needs_capture(key) + if needs_capture: def capture_forward_fn(inputs: Dict[str, Any]): with MoeLoadBalancerIterContext(moe_load_balancer): @@ -5567,13 +6316,16 @@ def capture_forward_fn(inputs: Dict[str, Any]): def capture_postprocess_fn(inputs: Dict[str, Any]): self._postprocess_inputs(inputs) - self.cuda_graph_runner.capture( + capture_outputs = self.cuda_graph_runner.capture( key, capture_forward_fn, inputs, enable_spec_decode=self.enable_spec_decode, postprocess_fn=capture_postprocess_fn) + if self.cuda_graph_runner.is_warmup_only: + outputs = capture_outputs + elif needs_capture: # Pre-replay: set DSA slot mappings for current batch's draft cache (fixes 2nd warmup) saved_draft = prepare_attn_metadata_for_draft_replay( attn_metadata, draft_kv_cache_manager) @@ -5758,6 +6510,95 @@ def _forward_step_mm_encoder_only( return result + def _make_encoder_attn_metadata( + self, + sequence_lengths: List[int], + request_ids: List[int], + ): + """Build fresh, no-cache attention metadata for one packed encoder + batch. ``self.attn_metadata`` is not reused because that object is + bound to the decoder's KV-cache manager.""" + sparse_metadata_params = ( + self.sparse_attention_config.to_sparse_metadata_params( + pretrained_config=self.model.model_config.pretrained_config) + if self.sparse_attention_config is not None else None) + encoder_attn_metadata = self.attn_backend.Metadata( + max_num_requests=self.batch_size, + max_num_tokens=self.max_num_tokens, + max_num_sequences=self.batch_size * self.max_beam_width, + kv_cache_manager=None, + mapping=self.mapping, + runtime_features=self.attn_runtime_features, + enable_flash_mla=self.model.model_config.enable_flash_mla, + enable_context_mla_with_cached_kv=False, + cache_indirection=None, + sparse_metadata_params=sparse_metadata_params, + num_heads_per_kv=1, + ) + assert isinstance( + encoder_attn_metadata, + (VanillaAttentionMetadata, TrtllmAttentionMetadata) + ), "Only vanilla and trtllm attention metadata are supported for the encoder pass" + + encoder_attn_metadata.seq_lens = torch.tensor( + sequence_lengths, + dtype=torch.int, + pin_memory=prefer_pinned(), + ) + encoder_attn_metadata.num_contexts = len(sequence_lengths) + encoder_attn_metadata.max_seq_len = self.max_seq_len + encoder_attn_metadata.request_ids = request_ids + encoder_attn_metadata.prepare_encoder_only() + return encoder_attn_metadata + + @nvtx_range("_prepare_tp_inputs_encoder_features") + def _prepare_tp_inputs_encoder_features( + self, + encoder_requests: List[LlmRequest], + resource_manager: Optional[ResourceManager] = None, + ): + """Pack encoder inputs for feature-driven audio encoders (Whisper). + + The encoder input is a per-request feature tensor (an opaque audio + tensor, e.g. Whisper's 30 s-padded waveform) rather than token ids, and + the packed sequence lengths are the post-encoder position counts + (``encoder_output_len``), not the raw feature length. + """ + features: List[torch.Tensor] = [] + sequence_lengths: List[int] = [] + request_ids: List[int] = [] + + for request in encoder_requests: + request_features = request.py_encoder_input_features + if request_features is None: + raise ValueError( + f"Encoder request {request.py_request_id} has no " + "encoder_input_features; feature- and token-driven " + "encoder requests cannot share one batch.") + features.append(request_features) + sequence_lengths.append(int(request.encoder_output_len)) + request_ids.append(request.py_request_id) + + num_tokens = sum(sequence_lengths) + assert num_tokens <= self.max_num_tokens, ( + f"encoder packed length ({num_tokens}) exceeds max_num_tokens " + f"({self.max_num_tokens})") + + encoder_attn_metadata = self._make_encoder_attn_metadata( + sequence_lengths, request_ids) + + inputs = { + 'input_features': + torch.cat(features, dim=0).to('cuda', non_blocking=True), + 'encoder_attn_metadata': + encoder_attn_metadata, + 'encoder_seq_lens': + sequence_lengths, + 'resource_manager': + resource_manager, + } + return inputs + @nvtx_range("_prepare_tp_inputs_encoder") def _prepare_tp_inputs_encoder( self, @@ -5782,6 +6623,15 @@ def _prepare_tp_inputs_encoder( raise ValueError( "_prepare_tp_inputs_encoder called with no encoder requests") + # Feature-driven audio encoders (Whisper) carry a tensor instead of + # encoder token ids; they take a dedicated prep path (which rejects + # mixed feature/token batches). + if any( + getattr(request, "py_encoder_input_features", None) is not None + for request in encoder_requests): + return self._prepare_tp_inputs_encoder_features( + encoder_requests, resource_manager=resource_manager) + encoder_input_ids: List[int] = [] encoder_position_ids: List[int] = [] sequence_lengths: List[int] = [] @@ -5806,40 +6656,8 @@ def _prepare_tp_inputs_encoder( f"encoder packed length ({num_tokens}) exceeds max_num_tokens " f"({self.max_num_tokens})") - # Build a fresh, no-cache attention metadata for the encoder - # pass. We do not reuse ``self.attn_metadata`` because that - # object is bound to the decoder's KV-cache manager. - sparse_metadata_params = ( - self.sparse_attention_config.to_sparse_metadata_params( - pretrained_config=self.model.model_config.pretrained_config) - if self.sparse_attention_config is not None else None) - encoder_attn_metadata = self.attn_backend.Metadata( - max_num_requests=self.batch_size, - max_num_tokens=self.max_num_tokens, - max_num_sequences=self.batch_size * self.max_beam_width, - kv_cache_manager=None, - mapping=self.mapping, - runtime_features=self.attn_runtime_features, - enable_flash_mla=self.model.model_config.enable_flash_mla, - enable_context_mla_with_cached_kv=False, - cache_indirection=None, - sparse_metadata_params=sparse_metadata_params, - num_heads_per_kv=1, - ) - assert isinstance( - encoder_attn_metadata, - (VanillaAttentionMetadata, TrtllmAttentionMetadata) - ), "Only vanilla and trtllm attention metadata are supported for the encoder pass" - - encoder_attn_metadata.seq_lens = torch.tensor( - sequence_lengths, - dtype=torch.int, - pin_memory=prefer_pinned(), - ) - encoder_attn_metadata.num_contexts = len(encoder_requests) - encoder_attn_metadata.max_seq_len = self.max_seq_len - encoder_attn_metadata.request_ids = request_ids - encoder_attn_metadata.prepare() + encoder_attn_metadata = self._make_encoder_attn_metadata( + sequence_lengths, request_ids) encoder_input_ids_t = torch.tensor(encoder_input_ids, dtype=torch.int, @@ -5885,6 +6703,17 @@ def _forward_step_encoder( "models must define a top-level `encoder` (or `model.encoder`) " "stack to participate in the encoder iteration.") + # Feature-driven encoders (Whisper): the feature tensor is opaque to + # the engine — no token embedding, no position ids, no dtype cast. + # The model's forward casts internally (Whisper's raw waveforms must + # reach the log-mel STFT in fp32). + input_features = inputs.get('input_features') + if input_features is not None: + return encoder( + input_features=input_features, + attn_metadata=inputs['encoder_attn_metadata'], + ) + # Encoder operates on packed token IDs. Models like T5 own the # shared embedding on ``self.model`` rather than inside the # encoder stack, so we go through the top-level model when @@ -5993,8 +6822,10 @@ def _apply_logits_processors(request, logits_processors, logits_tensor, "defined in `tensorrtllm.sampling_params`.") lp(request.py_request_id, logits_rows, token_ids, None, None) - logits_tensor[logits_row_offset:logits_row_offset + - beam_width] = logits_rows.view(beam_width, -1) + # logits_rows is a view into logits_tensor (narrow + view never + # copy), so the processors already mutated it in place. Writing it + # back would be a self-assignment, which torch rejects for the + # non-contiguous slices a TP-padded vocab produces. def _execute_logit_post_processors(self, scheduled_requests: ScheduledRequests, diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index 57cd50b01cc3..56e2e71d265c 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -13,10 +13,10 @@ from tensorrt_llm._torch.models.checkpoints.base_checkpoint_loader import ( AutoCheckpointMapper, BaseCheckpointLoader) from tensorrt_llm._torch.weight_sharing import ( - IdentityCheckPolicy, PostTransformFeature, PostTransformProfile, - PostTransformProfileRegistry, PostTransformQualificationDecision, - PostTransformTransferScope, SourceIdentity, - check_weight_sharing_compatibility) + ArtifactIdentity, IdentityCheckPolicy, PostTransformFeature, + PostTransformProfile, PostTransformProfileRegistry, + PostTransformQualificationDecision, PostTransformTransferScope, + SourceIdentity, check_weight_sharing_compatibility) from tensorrt_llm._utils import str_dtype_to_torch from tensorrt_llm.llmapi.llm_args import (DecodingBaseConfig, ExecutorMemoryType, @@ -42,7 +42,8 @@ MoeLoadBalancer, maybe_create_moe_load_balancer) from ..virtual_memory import RestoreMode from ..virtual_memory import scope as virtual_memory_scope -from .config_utils import resolve_hf_torch_dtype, resolve_ssm_cache_dtype +from .config_utils import (is_hybrid_linear, resolve_hf_torch_dtype, + resolve_ssm_cache_dtype) _KV_CACHE_MAP = { "fp8": QuantAlgo.FP8.value, @@ -52,6 +53,36 @@ _VALID_KV_CACHE_DTYPES = ("fp8", "nvfp4", "auto") +def _validate_and_adjust_mamba_snapshot_config(config: ModelConfig, + llm_args: TorchLlmArgs) -> None: + """Validate snapshot reuse after the model and V2 setting are resolved.""" + if not is_hybrid_linear(config.pretrained_config): + return + + kv_cache_config = llm_args.kv_cache_config + state_config = kv_cache_config.mamba_state_config + has_additional_snapshots = bool( + state_config.additional_snapshot_offsets_from_start + or state_config.additional_snapshot_offsets_from_end) + if (has_additional_snapshots + and kv_cache_config.use_kv_cache_manager_v2 is not True): + raise ValueError( + "Mamba additional snapshot offsets require " + "kv_cache_config.use_kv_cache_manager_v2=True after resolving " + "the model configuration.") + + has_periodic_snapshots = state_config.periodic_snapshot_interval > 0 + if (kv_cache_config.enable_block_reuse and not has_periodic_snapshots + and not has_additional_snapshots): + logger.warning( + "Disabling KV cache block reuse for the hybrid Mamba model " + "because no Mamba state snapshot policy is configured. Set " + "kv_cache_config.mamba_state_config.periodic_snapshot_interval " + "to a positive value or provide additional snapshot offsets to " + "enable block reuse.") + kv_cache_config.enable_block_reuse = False + + def validate_and_set_mamba_ssm_cache_dtype( config: ModelConfig, mamba_ssm_cache_dtype: str, @@ -71,7 +102,7 @@ def validate_and_set_mamba_ssm_cache_dtype( def validate_and_set_kv_cache_quant(model_config: ModelConfig, - pyt_kv_cache_dtype: str) -> QuantAlgo: + pyt_kv_cache_dtype: str) -> None: logger.info( f'Validating KV Cache config against kv_cache_dtype="{pyt_kv_cache_dtype}"' ) @@ -105,6 +136,14 @@ def validate_and_set_kv_cache_quant(model_config: ModelConfig, # Apply explicit override from kv_cache_config.dtype. model_config.quant_config.kv_cache_quant_algo = mapped_pyt_quant + # MIXED_PRECISION checkpoints carry per-layer QuantConfigs in + # quant_config_dict; modules built from them (e.g. attention) must agree + # with the global config on the KV element size, otherwise the KV pool is + # allocated with the overridden dtype while attention layers read/write + # with the checkpoint dtype -> out-of-bounds access. + if model_config.quant_config_dict is not None: + for layer_quant_config in model_config.quant_config_dict.values(): + layer_quant_config.kv_cache_quant_algo = mapped_pyt_quant def validate_encoder_decoder_kv_cache_config(model_config: ModelConfig, @@ -383,6 +422,8 @@ def load_config_and_apply_defaults( config.pretrained_config) model_cls = AutoModelForCausalLM._resolve_class(config) + use_kv_cache_manager_v2 = ( + llm_args.kv_cache_config.use_kv_cache_manager_v2) # model_cls is None when the architecture is unknown/unsupported. model_defaults = {} @@ -396,15 +437,6 @@ def load_config_and_apply_defaults( f"Applied model defaults for {model_cls.__name__}: {applied_defaults}" ) - use_kv_cache_manager_v2 = llm_args.kv_cache_config.use_kv_cache_manager_v2 - _resolve_kv_cache_manager_v2_auto(llm_args, model_defaults) - if use_kv_cache_manager_v2 == "auto": - logger.info( - "Resolved use_kv_cache_manager_v2='auto' to %s for %s", - llm_args.kv_cache_config.use_kv_cache_manager_v2, - model_cls.__name__ - if model_cls is not None else "unknown model") - # The transceiver preference follows the checkpoint's original # architecture: _resolve_class may rewrite it to an execution class # (e.g. MTPDraftModelForCausalLM), which must not drop the target @@ -418,6 +450,15 @@ def load_config_and_apply_defaults( # Resolve "auto" sentinel values after model defaults are applied. _resolve_transceiver_runtime_auto(llm_args, preference_cls, config.pretrained_config) + _resolve_kv_cache_manager_v2_auto( + llm_args, model_defaults, original_setting=use_kv_cache_manager_v2) + _validate_and_adjust_mamba_snapshot_config(config, llm_args) + if use_kv_cache_manager_v2 == "auto": + logger.info( + "Resolved use_kv_cache_manager_v2='auto' to %s for %s", + llm_args.kv_cache_config.use_kv_cache_manager_v2, + model_cls.__name__ + if model_cls is not None else "unknown model") return llm_args @@ -432,6 +473,39 @@ def _needs_source_identity(checkpoint_loader: BaseCheckpointLoader, """ return load_format == LoadFormat.GMS or checkpoint_loader.checkpoint_format == "MX" + @staticmethod + def _build_source_identity( + config: ModelConfig, + model: DecoderModelForCausalLM, + *, + checkpoint_dir: str, + model_name: str, + fallback_on_artifact_error: bool, + ) -> Optional[SourceIdentity]: + """Build the local identity without weakening artifact validation. + + Artifact construction remains fail-closed. MX may convert an artifact + error into an unavailable local identity so its compatibility gate + falls back to disk; GMS propagates the error because it has no fallback. + """ + try: + artifact_identity = ArtifactIdentity.from_checkpoint(checkpoint_dir) + except (OSError, RuntimeError, ValueError) as error: + if not fallback_on_artifact_error: + raise + logger.warning( + "Unable to build checkpoint artifact identity for MX checkpoint " + f"{checkpoint_dir}; falling back to regular checkpoint loading: {error}" + ) + return None + + return SourceIdentity.from_model_config( + config, + model, + artifact_identity=artifact_identity, + model_name=model_name, + ) + def load( self, checkpoint_dir: str, @@ -475,12 +549,16 @@ def load( # ground truth; building it here (post-construction, # pre-weight-load) gives producer and consumer a common, # comparable lifecycle point. - self._source_identity = SourceIdentity.from_model_config( + self._source_identity = self._build_source_identity( config, model, + checkpoint_dir=checkpoint_dir, model_name=str( getattr(self.llm_args, "model", None) or checkpoint_dir), + fallback_on_artifact_error=( + load_format != LoadFormat.GMS + and checkpoint_loader.checkpoint_format == "MX"), ) memo: dict[torch.Tensor, torch.Tensor] = {} @@ -1272,6 +1350,8 @@ def _load_and_validate_config( force_dynamic_quantization=self.llm_args.force_dynamic_quantization, spec_config=self.spec_config, sparse_attention_config=self.sparse_attention_config, + kv_cache_compression_config=( + self.llm_args.kv_cache_compression_config), max_num_tokens=self.max_num_tokens, max_seq_len=self.max_seq_len, moe_max_num_tokens=self.llm_args.moe_config.max_num_tokens, diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 6531586192b4..744a3bcdda2a 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -42,7 +42,7 @@ from tensorrt_llm.llmapi.llm_args import PeftCacheConfig, WaitingQueuePolicy from tensorrt_llm.logger import logger from tensorrt_llm.mapping import CpType -from tensorrt_llm.runtime.kv_cache_manager_v2._exceptions import OutOfPagesError +from tensorrt_llm.runtime.kv_cache_manager_v2 import OutOfPagesError from tensorrt_llm.tools.layer_wise_benchmarks import get_calibrator from tensorrt_llm.tools.profiler.host_profile_tools.host_profiler import ( get_global_profiler, host_profiler_context) @@ -765,10 +765,10 @@ def __init__( self.inflight_req_ids = ReqIdsSet() # Encoder-decoder models execute the encoder and decoder in separate - # iterations. The encoder branch lives in ``_executor_loop`` only; - # ``_executor_loop_overlap`` has not been threaded yet. Reject - # pp_size > 1 for parity with the legacy TRT path (Encoder PP support - # is intentionally out of scope for this port). + # iterations in both executor loops. PP usage is very rare for these + # models, so encoder PP send/recv support is not implemented in the + # PyTorch path for now. Reject pp_size > 1. + # TODO: Add support for pp + encoder models is_encoder_decoder = bool( getattr(getattr(self.model_engine.model, "model_config", None), "is_encoder_decoder", False)) @@ -778,11 +778,6 @@ def __init__( "pp_size > 1 is not supported for encoder-decoder models " "in the PyTorch flow; encoder send/recv hooks are out of " "scope. Set pp_size=1 to run T5/BART/mBART.") - if not self.disable_overlap_scheduler: - raise NotImplementedError( - "Overlap scheduler is not yet wired for encoder-decoder " - "models. Set disable_overlap_scheduler=True for " - "encoder-decoder runs.") if getattr(self.model_engine, "_torch_compile_piecewise_cuda_graph", False): raise NotImplementedError( @@ -919,9 +914,13 @@ def on_detected(): # under steady state — see ping-pong comment in _profiler). self._latest_host_step_time_ms: Optional[float] = None self._latest_prev_device_step_time_ms: Optional[float] = None + self._emit_initial_stats() self.gather_all_responses = False self.kv_cache_transceiver = kv_cache_transceiver + if kv_cache_transceiver is not None: + self.hang_detector.register_status_provider( + kv_cache_transceiver.get_status_dump) cache_transceiver_config = getattr(self.llm_args, "cache_transceiver_config", None) max_tokens_in_buffer = getattr(cache_transceiver_config, @@ -1280,6 +1279,21 @@ def start_worker(self): def _set_global_steady_clock_offset(self): assert self.global_rank >= 0, "rank should be >= 0" + # First calibration wins (mirrors the C++ guard in CacheTransceiver). + # PyExecutor is constructed twice per process (memory-profiling dry run, + # then the real executor), so this method runs twice. Recalibration is + # idempotent as long as the measurement below reads the raw + # steady_clock, but skipping it avoids a redundant barrier+allgather + # and protects the offset if the measurement path ever becomes + # offset-aware (which would make a second pass observe ~zero skew and + # wipe the correct value). + if LlmRequest.global_steady_clock_offset is not None: + logger.info( + f"global_steady_clock_offset already set " + f"({LlmRequest.global_steady_clock_offset}); skipping recalibration " + f"for rank {self.global_rank}") + return + # Sync all ranks self.dist.barrier() # Immediately take the local steady clock timestamp @@ -2017,6 +2031,9 @@ def _update_iter_stats( # Aggregate stats from all generation requests for req in scheduled_batch.generation_requests: + # exclude attention dp dummy / CUDA Graph padding requests from AL calculation + if getattr(req, 'is_dummy', False): + continue draft_len = getattr(req, 'num_draft_tokens', 0) py_draft_tokens = getattr(req, 'py_draft_tokens', None) py_num_accepted = getattr(req, 'py_num_accepted_draft_tokens', @@ -2550,6 +2567,10 @@ def _executor_loop_pp(self): # Retry until current rank can run first PP's schedule result. self._pp_retry_until_can_schedule(scheduled_batch) # Run scheduler locally because scheduler may change llm requests' state. + if hasattr(self.kv_cache_manager, + "prepare_expect_snapshot_points"): + self.kv_cache_manager.prepare_expect_snapshot_points( + self.active_requests) local_scheduler_output = self.scheduler.schedule_request( self.active_requests, self.inflight_req_ids) if self.kv_cache_transceiver: @@ -2584,6 +2605,9 @@ def _executor_loop_pp(self): f'{scheduled_batch.num_generation_requests} generation requests' ) + if scheduled_batch.encoder_requests: + self._run_encoder_step(scheduled_batch.encoder_requests) + can_queue, _ = self._can_queue(scheduled_batch) if not can_queue: self._revert_gen_alloc(scheduled_batch) @@ -3256,6 +3280,7 @@ def _handle_dynamic_draft_len(self, if spec_config is not None and spec_config.is_linear_tree else self.model_engine.max_total_draft_tokens) + @nvtx_range("_can_queue") def _can_queue(self, scheduled_batch): # can_queue_this_rank is for case that the batch is not empty on this rank, but empty on other ranks @@ -3860,6 +3885,7 @@ def _check_benchmark_disagg_gate(self, scheduled_batch: ScheduledRequests, return can_forward, True return can_forward, False + @nvtx_range("_handle_disagg_cache_errors_synced") def _handle_disagg_cache_errors_synced(self): """Rank-safe disagg cache error and poison handler. @@ -3939,6 +3965,25 @@ def _handle_disagg_cache_errors_synced(self): charge_budget=False, ) + def _emit_initial_stats(self) -> None: + """Emit a startup stats snapshot so that cache_config_info is + immediately available to external metric scrapers (e.g. the + Kubernetes Inference Gateway EPP) before any inference request.""" + if not self.enable_iter_perf_stats: + return + stats = self._get_init_iter_stats(0, 0) + kv_cache_manager = self.resource_manager.resource_managers.get( + ResourceManagerType.KV_CACHE_MANAGER) + if kv_cache_manager is not None: + kv_stats = kv_cache_manager.get_kv_cache_stats() + kv_stats_to_save = KvCacheStats() + kv_stats_to_save.max_num_blocks = kv_stats.max_num_blocks + kv_stats_to_save.tokens_per_block = kv_stats.tokens_per_block + kv_stats_to_save.free_num_blocks = kv_stats.free_num_blocks + kv_stats_to_save.used_num_blocks = kv_stats.used_num_blocks + stats.kv_cache_stats = kv_stats_to_save + self._append_iter_stats(stats) + def _executor_loop(self): torch.cuda.set_device(self.device_id) # ensure the context is created, otherwise, some MPI calls will fail. @@ -4456,6 +4501,9 @@ def _executor_loop_overlap(self): if not self._is_kv_manager_v2: self._terminate_requests(scheduled_batch.paused_requests) + if scheduled_batch.encoder_requests: + self._run_encoder_step(scheduled_batch.encoder_requests) + gpu_forward_events_from_perf_pool = False can_queue, can_queue_this_rank = self._can_queue( scheduled_batch) @@ -5225,6 +5273,10 @@ def _waiting_requests(self, context_requests: list[LlmRequest], @nvtx_range("_schedule") def _schedule(self): + if hasattr(self.kv_cache_manager, "prepare_expect_snapshot_points"): + self.kv_cache_manager.prepare_expect_snapshot_points( + self.active_requests) + scheduler_output = self.scheduler.schedule_request( self.active_requests, self.inflight_req_ids) @@ -5828,8 +5880,8 @@ def _prepare_disagg_gen_transmission_complete(self, scheduled_batch): if self.model_engine.enable_spec_decode: kv_mgr = self.resource_manager.resource_managers.get( ResourceManagerType.KV_CACHE_MANAGER) - seed = getattr(kv_mgr, - 'seed_kda_replay_caches_for_disagg_gen', None) + seed = getattr(kv_mgr, 'seed_kda_replay_caches_for_disagg_gen', + None) if seed is not None: seed([ req.py_request_id @@ -6223,10 +6275,10 @@ def _maybe_prefetch_next_iter_mm_encoders( ) except Exception: # Speculative prefetch is best-effort and must never crash the - # executor loop. On failure, `py_mm_encoder_event` is not stamped, - # so the next iteration's `_prepare_inputs` falls back to the - # standard in-iter encode path (which re-runs `to_device` and the - # encoder unconditionally when no cached embedding is present). + # executor loop. The dispatch helper stamps an event when it queued + # any auxiliary-stream work, even on a partial failure, so the next + # iteration can safely inspect the request-local data and fall back + # to the standard in-iter encode path for any missing embedding. logger.warning( f"Cross-iter MM encoder prefetch failed; falling back to " f"in-iter encode.\n{traceback.format_exc()}") diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index a4c2e48ddc53..0b95852b0be8 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -40,7 +40,7 @@ from ._util import (KvCacheCreator, _adjust_torch_mem_fraction, create_py_executor_instance, instantiate_sampler, is_mla, validate_feature_combination) -from .config_utils import is_hybrid_linear +from .config_utils import is_hybrid_linear, is_minimax_m3 from .connectors.kv_cache_connector import KvCacheConnectorManager from .dwdp import DwdpManager from .guided_decoder import CapturableGuidedDecoder, GuidedDecoder @@ -48,6 +48,15 @@ from .model_loader import ModelLoader, _construct_checkpoint_loader from .py_executor import PyExecutor +_MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS = (90, 100, 103, 120, 121) +_MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS = (90, 100, 103, 120) +_MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS_STR = "/".join( + f"SM{sm_version}" + for sm_version in _MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS) +_MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS_STR = "/".join( + f"SM{sm_version}" + for sm_version in _MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS) + class _ExecutorMemoryMonitor: """Currently this focuses on tracking memory usage and related errors.""" @@ -352,10 +361,29 @@ def create_py_executor( A fully initialized PyExecutor instance. """ - skip_est = os.environ.get("TRTLLM_SKIP_KV_CACHE_ESTIMATION", '0') == '1' llm_args, checkpoint_loader = _load_config_and_create_checkpoint_loader( llm_args, checkpoint_dir) + skip_est = os.environ.get("TRTLLM_SKIP_KV_CACHE_ESTIMATION", '0') == '1' + if llm_args.speculative_config is not None and not skip_est: + # TEMPORARY workaround for TRTLLM-14903: force-skip the KV cache + # size estimation phase whenever speculative decoding is enabled. + # With spec decoding + # and a self-spawned MPI session, the estimation executor's warmup + # hangs indefinitely while exercising the q>1 generation-path + # attention kernels that only its spec-mode dummy requests reach; + # the pre-merge branch tip (ec52c6418b) passes the identical run. + # The skip path sizes the cache analytically via + # configure_kv_cache_capacity() — the same path KVCacheManagerV2 + # uses for its memory quota. Non-speculative runs keep the normal + # estimation behavior. Remove this override once the + # estimation-phase hang (TRTLLM-14903) is fixed. + logger.info( + "Skipping KV cache size estimation with speculative decoding " + "enabled (TRTLLM-14903 estimation-phase hang workaround); the " + "cache is sized analytically instead.") + skip_est = True + garbage_collection_gen0_threshold = llm_args.garbage_collection_gen0_threshold lora_config = llm_args.lora_config kv_connector_config = llm_args.kv_connector_config @@ -411,6 +439,13 @@ def create_py_executor( if llm_args.attn_backend == "VANILLA": tokens_per_block = max_num_tokens + # The MSA kernels require a page size of 128; the Triton reference uses TRT-LLM's default + # of 32. + m3_sparse_config = llm_args.sparse_attention_config + if is_minimax_m3(m3_sparse_config): + tokens_per_block = 128 if m3_sparse_config.implementation == "msa" else 32 + kv_cache_config.tokens_per_block = tokens_per_block + if llm_args.attn_backend in ["FLASHINFER", "FLASHINFER_STAR_ATTENTION"]: # Workaround for flashinfer and star attention if kv_cache_config.enable_block_reuse: @@ -668,23 +703,9 @@ def drafting_loop_wrapper(model): max_num_tokens = model_engine.max_num_tokens sparse_attention_config = model_engine.sparse_attention_config - # Set default value for cache_transceiver_config.max_tokens_in_buffer - if cache_transceiver_config and cache_transceiver_config.max_tokens_in_buffer is None: - cache_transceiver_config.max_tokens_in_buffer = net_max_seq_len - config = model_engine.model.model_config.pretrained_config max_num_seq_slots = getattr(model_engine, "max_num_seq_slots", max_batch_size * getattr(mapping, "pp_size", 1)) - if is_hybrid_linear(config) and kv_cache_config.enable_block_reuse and ( - cache_transceiver_config is not None - and cache_transceiver_config.backend is not None - and cache_transceiver_config.transceiver_runtime == "PYTHON"): - logger.warning( - "Disabling block reuse for MambaHybridCacheManager-based models when disagg + Python transceiver enabled" - ) - kv_cache_config.enable_block_reuse = False - _set_model_engines_cache_reuse([model_engine, draft_model_engine], - False) if is_mla(config): if model_engine.model.model_config.enable_flash_mla: tokens_per_block = 64 @@ -704,12 +725,11 @@ def drafting_loop_wrapper(model): ) sm_version = get_sm_version() - if kv_cache_config.enable_block_reuse and sm_version not in [ - 90, 100, 103, 120 - ]: - logger.warning( - f"KV cache reuse for MLA can only be enabled on SM90/SM100/SM103/SM120, " - f"disable enable_block_reuse for SM{sm_version}") + if (kv_cache_config.enable_block_reuse and sm_version + not in _MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS): + logger.warning("KV cache reuse for MLA can only be enabled on " + f"{_MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS_STR}, " + f"disable enable_block_reuse for SM{sm_version}") kv_cache_config.enable_block_reuse = False _set_model_engines_cache_reuse([model_engine, draft_model_engine], False) @@ -725,15 +745,27 @@ def drafting_loop_wrapper(model): kv_cache_config.enable_block_reuse = False _set_model_engines_cache_reuse([model_engine, draft_model_engine], False) - if enable_chunked_context and sm_version not in [90, 100, 103, 120]: - logger.warning( - "Chunked Prefill for MLA can only be enabled on SM90/SM100/SM103/SM120, " - f"disable enable_chunked_context for SM{sm_version}") + if (enable_chunked_context and sm_version + not in _MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS): + logger.warning("Chunked Prefill for MLA can only be enabled on " + f"{_MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS_STR}, " + f"disable enable_chunked_context for SM{sm_version}") enable_chunked_context = False model_engine.attn_runtime_features.chunked_prefill = False if draft_model_engine is not None: draft_model_engine.attn_runtime_features.chunked_prefill = False + # Set default value for cache_transceiver_config.max_tokens_in_buffer. + # Placed after the FlashMLA tokens_per_block override and rounded up to a + # tokens_per_block multiple: CacheTransBufferManager requires + # max_tokens_in_buffer % tokens_per_block == 0 (cacheTransBuffer.cpp), + # and net_max_seq_len is in general not aligned (e.g. max_seq_len plus a + # non-power-of-two seq_len offset). + if cache_transceiver_config and cache_transceiver_config.max_tokens_in_buffer is None: + cache_transceiver_config.max_tokens_in_buffer = ( + (net_max_seq_len + tokens_per_block - 1) // tokens_per_block * + tokens_per_block) + if enable_chunked_context: chunk_unit_size = tokens_per_block max_attention_window = kv_cache_config.max_attention_window @@ -754,8 +786,9 @@ def drafting_loop_wrapper(model): ctx_chunk_config = None if kv_cache_config.enable_block_reuse and is_hybrid_linear(config): - ctx_chunk_config = (ContextChunkingPolicy.FORCE_CHUNK, - kv_cache_config.mamba_state_cache_interval) + # Snapshot boundaries come from expect_snapshot_points. The unit is + # only used to align chunks shortened by the scheduling budget. + ctx_chunk_config = (ContextChunkingPolicy.FORCE_CHUNK, tokens_per_block) guided_decoder: Optional[GuidedDecoder] = None if guided_decoding_config is not None: @@ -890,17 +923,16 @@ def drafting_loop_wrapper(model): if is_disagg and is_hybrid: # NOTE: TRTLLM_USE_PY_MAMBA is an agg-mode-only override and has - # no effect in disagg. The disagg manager choice is driven solely - # by transceiver_runtime: PYTHON => PythonMambaCacheManager, - # otherwise CppMambaHybridCacheManager (unified pool, default). + # no effect in disagg. The disagg manager choice is driven by + # get_kv_cache_manager_cls and cache_transceiver_config. if os.environ.get("TRTLLM_USE_PY_MAMBA", "0") == "1": logger.warning( "TRTLLM_USE_PY_MAMBA is ignored in disaggregated serving; " - "use cache_transceiver_config.transceiver_runtime='PYTHON' " - "to select PythonMambaCacheManager.") + "configure transceiver_runtime='PYTHON' with backend='NIXL' " + "to select MixedMambaHybridCacheManager.") else: logger.info("Disaggregated serving with hybrid model detected. " - "Using CppMambaHybridCacheManager.") + "Using the configured Mamba cache manager.") # Get draft config for one-engine speculative decoding if available draft_config = getattr(model_engine.model, 'draft_config', None) @@ -986,7 +1018,7 @@ def drafting_loop_wrapper(model): kv_connector_manager=kv_connector_manager if not estimating_kv_cache else None, resource_governor_queue=resource_governor_queue, - max_seq_len=max_seq_len, + max_seq_len=net_max_seq_len, max_batch_size=max_batch_size, max_beam_width=max_beam_width, max_num_tokens=max_num_tokens, @@ -1006,6 +1038,7 @@ def drafting_loop_wrapper(model): assert kv_cache_creator is not None with allocation_scope(ExecutorMemoryType.MODEL_EXTRA): kv_cache_creator.configure_kv_cache_capacity(py_executor) + # Shut down the transceiver before tearing down KV cache managers so # that NIXL-registered (pinned) GPU memory is deregistered first; # otherwise the old KV cache memory stays pinned and the subsequent @@ -1017,13 +1050,11 @@ def drafting_loop_wrapper(model): finally: kv_cache_creator.teardown_managers(resources) - # Release Phase-1 CUDA graph pools before final KV allocation to avoid overshoot. + # configure_kv_cache_capacity shuts down the Phase-1 executor, which + # releases its CUDA graphs before its resource managers. Only the + # profiling attention metadata remains to be discarded here. for eng in [model_engine, draft_model_engine]: - if eng is None: - continue - if eng.attn_metadata is not None: - if llm_args.cuda_graph_config is not None: - eng._release_cuda_graphs() + if eng is not None: eng.attn_metadata = None del py_executor # free before constructing new @@ -1061,7 +1092,7 @@ def drafting_loop_wrapper(model): garbage_collection_gen0_threshold, kv_connector_manager=kv_connector_manager, resource_governor_queue=resource_governor_queue, - max_seq_len=max_seq_len, + max_seq_len=net_max_seq_len, max_batch_size=max_batch_size, max_beam_width=max_beam_width, max_num_tokens=max_num_tokens, diff --git a/tensorrt_llm/_torch/pyexecutor/resource_manager.py b/tensorrt_llm/_torch/pyexecutor/resource_manager.py index a9bc0319e8d2..9ad6a6cdcc9b 100644 --- a/tensorrt_llm/_torch/pyexecutor/resource_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/resource_manager.py @@ -19,8 +19,8 @@ from abc import ABC, abstractmethod from collections import OrderedDict, defaultdict, deque from dataclasses import dataclass -from typing import (TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, - Set, Tuple, Union) +from typing import (TYPE_CHECKING, ClassVar, Dict, Iterable, List, Optional, + Sequence, Set, Tuple, Union) import torch from mpi4py import MPI @@ -307,6 +307,9 @@ def __init__( self.mapping = mapping self.dtype = dtype self.kv_cache_type = kv_cache_type + # Consumed by the disaggregation page-table builder to expose the DSA + # indexer K cache pool as a REPLICATED pool view. + self.enable_indexer_k_cache = enable_indexer_k_cache self.spec_config = spec_config self.pp_layers, self.num_layers = get_pp_layers( num_layers, @@ -464,10 +467,13 @@ def append_to_kv_heads_per_layer(num_kv_heads_per_layer: List[int], pp_size = self.mapping.pp_size if self.mapping is not None else 1 live_state_slots = self.max_batch_size * pp_size max_snapshots = live_state_slots - if kv_cache_config.enable_block_reuse: - max_snapshots += ( - kv_cache_config.max_tokens // - linear_attention_metadata.states_snapshot_interval) + snapshot_interval = ( + linear_attention_metadata.states_snapshot_interval) + if (kv_cache_config.enable_block_reuse + and snapshot_interval is not None + and snapshot_interval > 0): + max_snapshots += (kv_cache_config.max_tokens // + snapshot_interval) blocks_per_window[LinearCacheType.RECURRENT_STATES.value] = ( int(max_snapshots), 0) @@ -1936,12 +1942,23 @@ def _calculate_max_num_blocks_for_linear_attention( pp_size = self.mapping.pp_size if self.mapping is not None else 1 intercept = self.max_batch_size * pp_size * state_bytes_local - max_tokens = max((primary_budget - intercept) // slope, 0) + if slope > 0: + max_tokens = max((primary_budget - intercept) // slope, 0) + elif primary_budget >= intercept: + # With snapshots disabled, a rank containing only recurrent-state + # layers has no per-token cache cost after its live slots are + # allocated. Bound the otherwise-unlimited token count by the + # configured capacity or by all resident sequences at max length. + max_tokens = (kv_cache_config.max_tokens + if kv_cache_config.max_tokens is not None else + self.max_seq_len * self.max_batch_size * pp_size) + else: + max_tokens = 0 if kv_cache_config.max_tokens is not None: max_tokens = min(kv_cache_config.max_tokens, max_tokens) if max_tokens < kv_cache_config.max_tokens: logger.warning( - f'The memory budget for Mamba + KV cache cannot fit the user-specified max_tokens of {kv_cache_config.max_tokens}. The calculated max_tokens based on the memory budget is {max_tokens}. Please consider adjusting max_batch_size/max_tokens/mamba_state_cache_interval.' + f'The memory budget for Mamba + KV cache cannot fit the user-specified max_tokens of {kv_cache_config.max_tokens}. The calculated max_tokens based on the memory budget is {max_tokens}. Please consider adjusting max_batch_size/max_tokens/mamba_state_config.periodic_snapshot_interval.' ) kv_blocks_in_primary_pool = int(max_tokens // self.tokens_per_block) @@ -2180,9 +2197,13 @@ def _validate_and_adjust_attention_windows( def pin_blocks(self, request_id: int): self.impl.pin_blocks(request_id) - def copy_batch_block_offsets(self, dst_tensor: torch.Tensor, - request_ids: List[int], beam_width: int, - num_context: int, num_seqs: int): + def copy_batch_block_offsets(self, + dst_tensor: torch.Tensor, + request_ids: List[int], + beam_width: int, + num_context: int, + num_seqs: int, + max_blocks: Optional[int] = None): # Fill the persistent host buffer in place, exactly as before. CPU-side # consumers read self.host_kv_cache_block_offsets directly and depend on # its persistent, max_batch-sized layout: DSA sparse attention, the @@ -2248,23 +2269,39 @@ def copy_batch_block_offsets(self, dst_tensor: torch.Tensor, # matching the already-safe kv_lens / block_ids_per_seq staging. The # persistent buffer above is untouched by this and stays valid for the # synchronous CPU readers. - host_block_offsets = self._stage_block_offsets_for_copy(num_seqs) + host_block_offsets = self._stage_block_offsets_for_copy( + num_seqs, max_blocks) + width = host_block_offsets.shape[-1] for pool_idx in range(self.num_pools): - dst_tensor[pool_idx, :num_seqs].copy_(host_block_offsets[pool_idx], - non_blocking=True) + dst_tensor[pool_idx, :num_seqs, :, :width].copy_( + host_block_offsets[pool_idx], non_blocking=True) - def _stage_block_offsets_for_copy(self, num_rows: int) -> torch.Tensor: + def _stage_block_offsets_for_copy( + self, + num_rows: int, + max_blocks: Optional[int] = None) -> torch.Tensor: """Snapshot the first ``num_rows`` rows of the persistent host block offset buffer into a fresh pinned buffer, to serve as the private source - of an asynchronous H2D copy (nvbug 6293536).""" + of an asynchronous H2D copy (nvbug 6293536). + + ``max_blocks`` bounds the copied block width. The buffer is laid out + for max_seq_len (max_blocks_per_seq columns) but consumers only read + each sequence's allocated block prefix, so a caller that knows the + batch's maximum KV length can skip the unused tail — with a large + max_seq_len the tail dominates the copy cost.""" + if max_blocks is None: + width = self.max_blocks_per_seq + else: + width = min(max(max_blocks, 1), self.max_blocks_per_seq) host_block_offsets = torch.empty(self.num_pools, num_rows, 2, - self.max_blocks_per_seq, + width, dtype=torch.int32, pin_memory=prefer_pinned(), device='cpu') - host_block_offsets.copy_(self.host_kv_cache_block_offsets[:, :num_rows]) + host_block_offsets.copy_( + self.host_kv_cache_block_offsets[:, :num_rows, :, :width]) return host_block_offsets def truncate_blocks(self, target_tokens: List[int], @@ -2399,7 +2436,7 @@ def _free_blocks(self, block_list: list): # --------------------------------------------------------------------- # -class BaseKVCacheCompressionManager(BaseResourceManager): +class KVCacheCompressionManager(BaseResourceManager): """Framework-level base class for all KV-cache compression managers. Inherits :class:`BaseResourceManager` so PyExecutor's main loop @@ -2408,14 +2445,34 @@ class BaseKVCacheCompressionManager(BaseResourceManager): base implementations below translate those callbacks into the lifecycle hooks. - Concrete compression methods subclass this directly. All 4 hooks default to + Concrete compression methods subclass this directly. The hooks default to no-op; subclasses override what they need. The manager never inherits from any cache manager because this layer decides *how* the physical KV is used, not *what* physical KV exists. Subclasses hold ``KVCacheManagerV2`` as a tool. + + A subclass compacts through the ``KVCacheManagerV2`` it holds and records + the evicted count on ``LlmRequest.py_num_compressed_tokens``; the model + engine subtracts that count when building ``num_cached_tokens_per_seq``. """ - def __init__(self, kv_cache_manager: "KVCacheManagerV2"): + adjusts_generation_kv_length: ClassVar[bool] = False + """Whether this manager can make target and logical KV lengths diverge.""" + + def __init__( + self, + kv_cache_manager: "KVCacheManagerV2", + draft_kv_cache_manager: Optional["KVCacheManagerV2"] = None, + ): + from .kv_cache_manager_v2 import KVCacheManagerV2 + + if not isinstance(kv_cache_manager, KVCacheManagerV2): + raise TypeError("KV-cache compression requires KVCacheManagerV2") + if draft_kv_cache_manager is not None and not isinstance( + draft_kv_cache_manager, KVCacheManagerV2): + raise TypeError( + "draft KV-cache compression requires KVCacheManagerV2") self.kv_cache_manager = kv_cache_manager + self.draft_kv_cache_manager = draft_kv_cache_manager # Compression evicts/rewrites stored keys and values, so a shared prefix # block is no longer safe to reuse (same constraint as RocketKVCacheManager). if kv_cache_manager.enable_block_reuse: @@ -2423,9 +2480,18 @@ def __init__(self, kv_cache_manager: "KVCacheManagerV2"): f"{type(self).__name__} changes stored keys and values and cannot " f"run with KV-cache block reuse. Set " f"KvCacheConfig.enable_block_reuse to False.") + kv_cache_manager.kv_compression_manages_history = self.adjusts_generation_kv_length + if draft_kv_cache_manager is not None: + # The draft cache is compacted together with the target. + draft_kv_cache_manager.kv_compression_manages_history = ( + self.adjusts_generation_kv_length) + + @property + def has_independent_draft_kv_cache(self) -> bool: + return self.draft_kv_cache_manager is not None # ================================================================== # - # KV-cache lifecycle hooks (4, in temporal order). # + # KV-cache lifecycle hooks (5, in temporal order). # # Subclasses override what they need; all default to no-op. # # ================================================================== # @@ -2436,20 +2502,23 @@ def on_request_init(self, request: "LlmRequest", **kwargs) -> None: scoring buffers). """ - def on_context_step_end( + def on_context_step_end(self, requests: List["LlmRequest"], + **kwargs) -> None: + """Fired once per iteration with the requests whose prefill finished + (their final chunk) this step. Batched like the generation hook so a + one-shot prefill-end eviction can process the cohort in one launch. + """ + + def on_generation_step_begin( self, - request: "LlmRequest", - metadata: "AttentionMetadata", + scheduled_batch: "ScheduledRequests", **kwargs, ) -> None: - """Fired once per request, when its prefill finishes (its final - chunk). Override for a one-shot prefill-end eviction. - """ + """Fired once per generation step before this step's forward.""" def on_generation_step_end( self, scheduled_batch: "ScheduledRequests", - attn_metadata: "AttentionMetadata", **kwargs, ) -> None: """Fired once per generation step, after every layer's forward @@ -2489,6 +2558,7 @@ def prepare_resources(self, scheduled_batch: "ScheduledRequests") -> None: for req in scheduled_batch.context_requests: if req.is_first_context_chunk: self.on_request_init(req) + self.on_generation_step_begin(scheduled_batch) def update_resources( self, @@ -2496,8 +2566,8 @@ def update_resources( attn_metadata: Optional["AttentionMetadata"] = None, kv_cache_dtype_byte_size: Optional[float] = None, ) -> None: - """Fire :meth:`on_context_step_end` once per request, on the iteration its - final prefill chunk runs, then :meth:`on_generation_step_end` once. + """Fire :meth:`on_context_step_end` with the requests whose final + prefill chunk ran this iteration, then :meth:`on_generation_step_end`. Uses the scheduler's ``context_requests_last_chunk`` split (computed at schedule time from ``is_last_context_chunk``) rather than tracking @@ -2508,9 +2578,10 @@ def update_resources( managers so PyExecutor passes ``attn_metadata`` / ``kv_cache_dtype_byte_size`` through transparently. """ - for req in scheduled_batch.context_requests_last_chunk: - self.on_context_step_end(req, attn_metadata) - self.on_generation_step_end(scheduled_batch, attn_metadata) + if scheduled_batch.context_requests_last_chunk: + self.on_context_step_end( + scheduled_batch.context_requests_last_chunk) + self.on_generation_step_end(scheduled_batch) def free_resources(self, request: "LlmRequest") -> None: """Fire :meth:`on_request_finish`.""" @@ -2547,9 +2618,9 @@ def update_resources( attn_metadata: Optional["AttentionMetadata"] = None, kv_cache_dtype_byte_size: Optional[float] = None, ): - for _, resource_manager in self.resource_managers.items(): + for resource_type, resource_manager in self.resource_managers.items(): if hasattr(resource_manager, "update_resources"): - if isinstance(resource_manager, KVCacheManager): + if resource_type == ResourceManagerType.KV_CACHE_MANAGER: resource_manager.update_resources(scheduled_batch, attn_metadata, kv_cache_dtype_byte_size) diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/__init__.py b/tensorrt_llm/_torch/pyexecutor/sampler/__init__.py index 64ecca48d7d9..56ed1c458729 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/__init__.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/__init__.py @@ -16,14 +16,14 @@ The upper-level orchestration (``Sampler`` / ``TorchSampler`` / ``TRTLLMSampler``) lives in ``sampler.py`` and depends on operation-level APIs in -``sampling_utils.py``. Implementation-specific kernel providers (FlashInfer, +``sampler_strategy.py``. Implementation-specific kernel providers (FlashInfer, vanilla/PyTorch, TRT-LLM ops) live under ``ops/`` and are selected internally, never exposed as interchangeable backends to callers. Public symbols from ``sampler.py`` are re-exported here so existing ``pyexecutor.sampler`` import paths keep working. The re-export is lazy (PEP 562 ``__getattr__``) so that importing lightweight submodules such as -``pyexecutor.sampler.sampling_utils`` does not eagerly pull in ``sampler.py`` +``pyexecutor.sampler.sampler_strategy`` does not eagerly pull in ``sampler.py`` and its heavy dependency chain (which would create import cycles with ``speculative.interface``). """ @@ -32,7 +32,7 @@ # Submodules of this package — never forward these to sampler.py (that would # recurse, since accessing e.g. `.sampler` before it is bound re-enters here). -_SUBMODULES = frozenset({"sampler", "sampling_utils", "ops"}) +_SUBMODULES = frozenset({"sampler", "sampler_common", "sampler_strategy", "ops"}) def __getattr__(name: str): diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/finish_reasons.py b/tensorrt_llm/_torch/pyexecutor/sampler/finish_reasons.py new file mode 100644 index 000000000000..e79281176391 --- /dev/null +++ b/tensorrt_llm/_torch/pyexecutor/sampler/finish_reasons.py @@ -0,0 +1,873 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Finish-reason handling for ``TorchSampler``. + +Owns the stop criteria evaluated on the device each step -- end-id match, +max-length, and stop words -- along with the per-slot buffers they need +(stop-word rules, the past-token window) and their resize/refresh logic. +``TorchSampler`` holds one :class:`FinishReasonsHandler` and drives it +through request admission and the per-step write. +""" + +from dataclasses import dataclass + +import torch + +from tensorrt_llm._utils import nvtx_range, prefer_pinned +from tensorrt_llm.bindings.executor import FinishReason + +from ...utils import torch_multi_arange +from ..llm_request import LlmRequest +from .sampler_common import int_tensor + +__all__ = ["FinishReasonsHandler"] + + +class FinishReasonsHandler: + _EMPTY_STOP_WORD_TOKEN_ID: int = -2 + _PAD_STOP_WORD_TOKEN_ID: int = -1 + + @dataclass(kw_only=True) + class _FinishReasonsStore: + """Auxiliary data structures used for finish reasons handling.""" + + # Per-request dynamic data + finish_reasons_cuda: torch.Tensor + """Shape: [max_tokens, batch_size, beam_width] + Usage: Stores the determined finish reasons for all sampled tokens + for each request. Some (draft) tokens and corresponding + finish reasons might still be discarded.""" + + # Per-request static data + max_lengths_cuda: torch.Tensor + """Shape: [batch_size] + Usage: Stores the maximum sequence lengths for each request""" + end_ids_cuda: torch.Tensor + """Shape: batch_size + Usage: Stores the end ids for each request""" + stop_words_cuda: torch.Tensor + """Shape: [max_num_stop_words, max_stop_word_length, batch_size] + Usage: Stores the stop words for each request as a padded tensor.""" + past_tokens_cuda: torch.Tensor + """Shape: [max_stop_word_length,batch_size, beam_width] + Usage: Stores the last max_stop_word_length tokens for each beam.""" + max_stop_word_lengths_host: torch.Tensor + """Shape: [batch_size] + Usage: Stores the size of the longest stop word for each request.""" + num_accepted_draft_tokens_host: torch.Tensor + """Shape: [batch_size] + Usage: Stores the number of accepted tokens for each request.""" + + def __init__( + self, + *, + max_stop_word_length: int, + max_num_stop_words: int, + max_num_sequences: int, + max_beam_width: int, + max_tokens: int, + max_seq_len: int, + ): + self._update_sizes( + max_stop_word_length=max_stop_word_length, + max_num_stop_words=max_num_stop_words, + max_num_sequences=max_num_sequences, + max_beam_width=max_beam_width, + max_tokens=max_tokens, + max_seq_len=max_seq_len, + ) + self._setup_store() + self._setup_helper_tensors() + self._temp_data: FinishReasonsHandler._TemporaryData = self._TemporaryData() + + @property + def _use_speculative_decoding(self) -> bool: + return self._max_tokens > 1 + + @property + def new_max_lens(self) -> list[int]: + return self._temp_data.max_lens + + @property + def new_end_ids(self) -> list[int]: + return self._temp_data.end_ids + + def _update_sizes( + self, + *, + max_stop_word_length: int, + max_num_stop_words: int, + max_num_sequences: int, + max_beam_width: int, + max_tokens: int, + max_seq_len: int, + ) -> None: + """Updates the sizes of the finish reasons handler + + Sets member variables to store the current sizes. + These sizes are used to initialize the buffer tensors. + """ + self._max_stop_word_length: int = max_stop_word_length + self._max_num_stop_words: int = max_num_stop_words + self._max_num_sequences: int = max_num_sequences + self._max_beam_width: int = max_beam_width + self._max_tokens: int = max_tokens + self._max_seq_len: int = max_seq_len + self._stop_words_shape: tuple[int, int, int] = ( + self._max_num_stop_words, + self._max_stop_word_length, + self._max_num_sequences, + ) + self._past_tokens_shape: tuple[int, int, int] = ( + self._max_stop_word_length - 1 + self._max_tokens, + self._max_num_sequences, + self._max_beam_width, + ) + + def _setup_store(self) -> None: + """Sets up the store for the finish reasons handler by initializing all buffer tensors.""" + finish_reasons_cuda = int_tensor( + (self._max_tokens, self._max_num_sequences, self._max_beam_width) + ) + max_lengths_cuda = int_tensor((self._max_num_sequences,)) + end_ids_cuda = int_tensor((self._max_num_sequences,)) + stop_words_cuda = int_tensor(self._stop_words_shape) + past_tokens_cuda = int_tensor(self._past_tokens_shape) + max_stop_word_lengths_host = torch.empty( + self._max_num_sequences, device="cpu", dtype=torch.int32 + ) + num_accepted_draft_tokens_host = torch.empty( + self._max_num_sequences, device="cpu", dtype=torch.int32 + ) + self.store: FinishReasonsHandler._FinishReasonsStore = self._FinishReasonsStore( + finish_reasons_cuda=finish_reasons_cuda, + max_lengths_cuda=max_lengths_cuda, + end_ids_cuda=end_ids_cuda, + stop_words_cuda=stop_words_cuda, + past_tokens_cuda=past_tokens_cuda, + max_stop_word_lengths_host=max_stop_word_lengths_host, + num_accepted_draft_tokens_host=num_accepted_draft_tokens_host, + ) + + def _setup_helper_tensors(self) -> None: + # Helper tensors for finish_reasons: + """Preallocate buffer needed for torch.nonzero_static(..., out=finish_reasons_nonzero_static_buffer). + See `def _write_reason`.""" + # setup local buffer for max tokens checking + self._max_tokens_offset_cuda: torch.Tensor = torch.arange( + 1, self._max_tokens + 1, device="cuda", dtype=torch.int32 + ).view(-1, 1, 1) + + self._stop_words_index_offset_cuda: torch.Tensor = torch.arange( + max(0, self._max_stop_word_length - 1), device="cuda" + ).unsqueeze(1) + + self._past_token_buffer_cuda: torch.Tensor = torch.empty( + self._past_tokens_shape, device="cuda", dtype=torch.int32 + ) + starts = torch.arange(self._max_tokens, device="cuda") + ends = starts + self._max_stop_word_length + self._multi_arange_indexing: torch.Tensor = torch_multi_arange( + ends=ends, + starts=starts, + output_length=self._max_tokens * self._max_stop_word_length, + ) + + def _resize_stop_word_buffers(self) -> None: + self._stop_words_index_offset_cuda = torch.arange( + max(0, self._max_stop_word_length - 1), device="cuda" + ).unsqueeze(1) + + self._past_tokens_shape = ( + self._max_stop_word_length - 1 + self._max_tokens, + self._max_num_sequences, + self._max_beam_width, + ) + self._stop_words_shape = ( + self._max_num_stop_words, + self._max_stop_word_length, + self._max_num_sequences, + ) + self._past_token_buffer_cuda = torch.empty( + self._past_tokens_shape, device="cuda", dtype=torch.int32 + ) + starts = torch.arange(self._max_tokens, device="cuda") + ends = starts + self._max_stop_word_length + self._multi_arange_indexing = torch_multi_arange( + ends=ends, + starts=starts, + output_length=self._max_tokens * self._max_stop_word_length, + ) + # resize the stop words buffer if necessary + # if the sizes are constant, this does nothing + store = self.store + _ = store.stop_words_cuda.resize_(self._stop_words_shape) + _ = store.past_tokens_cuda.resize_(self._past_tokens_shape) + + @dataclass(kw_only=True) + class _TemporaryData: + """Data structure to store the temporary data during setup_sampler_step for new requests""" + + def __init__(self) -> None: + # list of device tensors + self.stop_words_cuda_list: list[torch.Tensor] = [] + self.past_tokens_cuda_list: list[torch.Tensor] = [] + # list of integers + self.stop_word_seq_slots: list[int] = [] + self.max_lens: list[int] = [] + self.end_ids: list[int] = [] + self.max_stop_word_lengths: list[int] = [] + # integers + self.total_max_length: int = 0 + self.total_max_num_stop_words: int = 0 + + def clear(self) -> None: + self.stop_words_cuda_list = [] + self.past_tokens_cuda_list = [] + self.stop_word_seq_slots = [] + self.max_lens = [] + self.end_ids = [] + self.max_stop_word_lengths = [] + self.total_max_length = 0 + self.total_max_num_stop_words = 0 + + def setup_new_request_handling(self) -> None: + """Setup the new request handling for the finish reasons handler + + Clears the temporary data for the new request handling. + This should be called before processing new requests, to avoid + stale data from previous requests. + """ + self._temp_data.clear() + + def prepare_for_new_request(self, request: LlmRequest) -> None: + """Fill _temp_data with the corresponding data from new requests to be used during setup_sampler_step + + Args: + request: The request to prepare for. + """ + + self._temp_data.max_lens.append( + min(self._max_seq_len, request.orig_prompt_len + request.py_max_new_tokens) + ) + self._temp_data.end_ids.append(end_id if (end_id := request.py_end_id) is not None else -1) + + if (stop_words_list := request.py_stop_words_list) is not None: + assert (seq_slot := request.py_seq_slot) is not None + self._temp_data.stop_word_seq_slots.append(seq_slot) + extracted_stop_words_cuda, max_length, num_stop_words = self._extract_stop_words( + stop_words_list + ) + self._temp_data.stop_words_cuda_list.append(extracted_stop_words_cuda) + self._temp_data.past_tokens_cuda_list.append(self._get_past_tokens(request)) + self._temp_data.total_max_length = max(self._temp_data.total_max_length, max_length) + self._temp_data.total_max_num_stop_words = max( + self._temp_data.total_max_num_stop_words, num_stop_words + ) + self._temp_data.max_stop_word_lengths.append(max_length) + else: + # max stop word length is used to determine if a request has stop words + # explicitly set it to 0 here to avoid stale data from previous requests + self._temp_data.max_stop_word_lengths.append(0) + + def update_for_new_request( + self, + *, + seq_slots_cuda_long: torch.Tensor, + max_lengths_cuda: torch.Tensor, + end_ids_cuda: torch.Tensor, + seq_slots_host: torch.Tensor, + all_sampling_requests: list[LlmRequest], + ) -> None: + """Update tensors of this store with the new request data. + + If stop words are present, also update the stop words buffers. + If the new stop words exceed either the current max_num_stop_words or max_stop_word_length values, + a resize of the stop words buffers is triggered. If a resize is necessary, all requests in the batch + need to be re-processed. + + Args: + seq_slots_cuda_long: The sequence slots of the processed requests, as int64 + CUDA indices (required by ``index_copy_``). Shape: [len(requests)] + max_lengths_cuda: The maximum lengths for each request. + Shape: [len(requests)] + end_ids_cuda: The end ids for each request. + Shape: [len(requests)] + seq_slots_host: The sequence slots of the processed requests. Used for accessing host buffers. + Shape: [len(requests)] + all_sampling_requests: If a resize of the stop words related buffers is necessary, all sampling requests + need to be re-processed. + """ + + temp_data = self._temp_data + store = self.store + store.max_lengths_cuda.index_copy_(0, seq_slots_cuda_long, max_lengths_cuda) + store.end_ids_cuda.index_copy_(0, seq_slots_cuda_long, end_ids_cuda) + store.max_stop_word_lengths_host[seq_slots_host] = torch.tensor( + temp_data.max_stop_word_lengths, device="cpu", dtype=torch.int32 + ) + + # Handle stop words only if any new ones are added + if temp_data.stop_word_seq_slots: + self._update_stop_words_buffer( + all_sampling_requests, + temp_data.total_max_length, + temp_data.total_max_num_stop_words, + temp_data.stop_words_cuda_list, + temp_data.past_tokens_cuda_list, + temp_data.stop_word_seq_slots, + ) + + def _maybe_resize_stop_words_buffer( + self, total_max_length: int, total_max_num_stop_words: int + ) -> bool: + """Checks if the stop words buffer needs to be resized and resizes it if necessary + + If the total maximum length or number of stop words exceeds the current maximum values, + the stop words buffer is resized to the new maximum values. + + Args: + total_max_length: The maximum length of the stop words in this batch. + total_max_num_stop_words: The maximum number of stop words of a request in this batch. + Returns: + True if the stop words buffer needs to be resized, False otherwise. + """ + if ( + total_max_length > self._max_stop_word_length + or total_max_num_stop_words > self._max_num_stop_words + ): + self._max_stop_word_length = max(total_max_length, self._max_stop_word_length) + self._max_num_stop_words = max(total_max_num_stop_words, self._max_num_stop_words) + self._resize_stop_word_buffers() + return True + return False + + def _reprocess_stop_words_buffer( + self, requests: list[LlmRequest] + ) -> tuple[list[torch.Tensor], list[torch.Tensor], list[int]]: + """Reprocesses the stop words buffer with the new maximum values + + If a resize of the stop words buffer is necessary, all requests in the batch need to be re-processed. + + Args: + requests: The requests to reprocess the stop words buffer for. + Returns: + stop_words_cuda_list: A list of device tensors containing the stop words per request with stop words. + past_tokens_cuda_list: A list of device tensors containing the past tokens per request with stop words. + stop_word_seq_slots: A list of sequence slot indices (int) per request with stop words. + """ + stop_words_cuda_list: list[torch.Tensor] = [] + past_tokens_cuda_list: list[torch.Tensor] = [] + stop_word_seq_slots: list[int] = [] + # Rerun with the new size. Set the stop words and past tokens for all the requests. + for request in requests: + if (stop_words_list := request.py_stop_words_list) is not None: + extracted_stop_words_cuda, _, _ = self._extract_stop_words(stop_words_list) + assert (seq_slot := request.py_seq_slot) is not None + stop_word_seq_slots.append(seq_slot) + stop_words_cuda_list.append(extracted_stop_words_cuda) + past_tokens_cuda_list.append(self._get_past_tokens(request)) + return stop_words_cuda_list, past_tokens_cuda_list, stop_word_seq_slots + + def _update_stop_words_buffer( + self, + all_sampling_requests: list[LlmRequest], + total_max_length: int, + total_max_num_stop_words: int, + stop_words_cuda_list: list[torch.Tensor], + past_tokens_cuda_list: list[torch.Tensor], + stop_word_seq_slots: list[int], + ) -> None: + """Updates the stop words buffer with the new maximum values + + Args: + all_sampling_requests: If a resize of the stop words related buffers is necessary, all sampling requests + need to be re-processed. + total_max_length: The maximum length of the stop words in this batch. + total_max_num_stop_words: The maximum number of stop words of a request in this batch. + stop_words_cuda_list: A list of device tensors containing the stop words per request with stop words. + past_tokens_cuda_list: A list of device tensors containing the past tokens per request with stop words. + stop_word_seq_slots: Sequence slot index (int) per request with stop words; + same order as the lists above. + """ + # Potentially resize the buffers and update + # stop_words, past_tokens and stop_word_seq_slots + # In case of a resize all requests in the batch need to be re-processed + if self._maybe_resize_stop_words_buffer(total_max_length, total_max_num_stop_words): + stop_words_cuda_list, past_tokens_cuda_list, stop_word_seq_slots = ( + self._reprocess_stop_words_buffer(all_sampling_requests) + ) + + # Host Tensor for host access of self.store.num_accepted_draft_tokens + stop_word_seq_slots_tensor_host = torch.tensor( + stop_word_seq_slots, device="cpu", dtype=torch.int32, pin_memory=prefer_pinned() + ) + # Device Tensor for device access of self.store.stop_words and self.store.past_tokens + stop_word_seq_slots_tensor_cuda = stop_word_seq_slots_tensor_host.to( + device="cuda", non_blocking=True + ) + # stop_word_seq_slots x max_num_stop_words x max_stop_word_length + stop_words_cuda_tensor = torch.stack(stop_words_cuda_list) + # stop_word_seq_slots x max_stop_word_length x beam_width + past_tokens_cuda_tensor = torch.stack(past_tokens_cuda_list) + + store = self.store + # Reset the accepted tokens buffer for the stop word sequence slots + store.num_accepted_draft_tokens_host[stop_word_seq_slots_tensor_host] = 0 + + store.stop_words_cuda[..., stop_word_seq_slots_tensor_cuda] = ( + stop_words_cuda_tensor.permute(1, 2, 0) + ) + # Past tokens will be shifted by 1 to the left on their first sampling iteration + # We need to consider this here. + store.past_tokens_cuda[1 : self._max_stop_word_length, stop_word_seq_slots_tensor_cuda] = ( + past_tokens_cuda_tensor.permute(1, 0, 2) + ) + + def _extract_stop_words( + self, stop_words_list: list[list[int]] + ) -> tuple[torch.Tensor, int, int]: + """Extract the stop words and size information from the stop words list + + Processes the stop words list and stores the stop words in a padded device tensor. + Stop words shorter than FinishReasonsHandler.max_stop_word_length + are padded with _PAD_STOP_WORD_TOKEN_ID to max_stop_word_length. + Unused stop word slots are padded with _EMPTY_STOP_WORD_TOKEN_ID. + This function additionally returns the maximum stop word length and the number of stop words + in the processed stop words list. + + + Args: + stop_words_list: A list of two lists: the first contains the token ids of all stop sequences + (concatenated); the second contains the cumulative lengths (prefix sum) of the stop word lengths. + + Returns: + stop_words: A padded device tensor containing the stop words + Shape: [max_num_stop_words, max_stop_word_length] + max_stop_word_length: The maximum stop word length in the stop words list + num_stop_words: The number of stop words in the stop words list + """ + stop_words_host = torch.empty( + self._max_num_stop_words, + self._max_stop_word_length, + device="cpu", + dtype=torch.int32, + ) + _ = stop_words_host.fill_(self._EMPTY_STOP_WORD_TOKEN_ID) + words, cumulative_stop_word_lengths = stop_words_list + words_host = torch.tensor( + words, device="cpu", dtype=torch.int32, pin_memory=prefer_pinned() + ) + begin = 0 + max_stop_word_length = 0 + num_stop_words = 0 + for idx, end in enumerate(cumulative_stop_word_lengths): + if end == -1: + break + length = end - begin + max_stop_word_length = max(max_stop_word_length, length) + num_stop_words += 1 + # skip processing if either the length or the index is greater than the current max values. + # These will be updated outside this function. + if length > self._max_stop_word_length or idx >= self._max_num_stop_words: + begin = end + continue + stop_words_host[idx, -length:] = words_host[begin:end] + stop_words_host[idx, :-length] = self._PAD_STOP_WORD_TOKEN_ID + begin = end + return ( + stop_words_host.to("cuda", non_blocking=True), + max_stop_word_length, + num_stop_words, + ) + + def _get_past_tokens(self, request: LlmRequest) -> torch.Tensor: + """Get the past tokens from the request and return the past tokens device tensor + + Args: + request: The request to get the past tokens for + + Returns: + past_tokens: The past tokens device tensor + Shape: [max_stop_word_length - 1, max_beam_width] + """ + past_tokens_host = torch.zeros( + max(0, self._max_stop_word_length - 1), + self._max_beam_width, + device="cpu", + dtype=torch.int32, + pin_memory=prefer_pinned(), + ) + tokens = request.get_tokens() + for beam_idx in range(self._max_beam_width): + max_len = min(past_tokens_host.shape[0], len(tokens[beam_idx])) + past_tokens_host[past_tokens_host.shape[0] - max_len :, beam_idx] = torch.tensor( + tokens[beam_idx][len(tokens[beam_idx]) - max_len :], + device="cpu", + dtype=torch.int32, + ) + return past_tokens_host.to("cuda", non_blocking=True) + + def write_finish_reasons( + self, + seq_slots_host: torch.Tensor, + is_draft_batch: bool, + seq_slots_cuda: torch.Tensor, + seq_lens_cuda: torch.Tensor, + new_tokens_cuda: torch.Tensor, + first_finish_reasons_cuda: torch.Tensor | None = None, + ) -> torch.Tensor: + """Calculates the finish reasons for each request and returns the finish reasons tensor. + + Prepares stop word handling for each requests and processes all newly generated tokens + per request to determine if any finish reason is met. Returns the device finish reasons + tensor from the store, which is updated with the calculated finish reason for each newly + generated token. + + Args: + seq_slots_host: The sequence slots of the processed requests. Used to determine which + requests need stop word processing on the host. + Shape: [len(requests)] + is_draft_batch: Whether the batch consists of draft requests. + seq_slots_cuda: The sequence slots of the processed requests. Used for accessing device buffers. + Shape: [len(requests)] + seq_lens_cuda: The sequence lengths of the processed requests. + Shape: [len(requests)] + new_tokens_cuda: A buffer containing the newly generated tokens. + Shape: [max_tokens, max_batch_size, max_beam_width] + first_finish_reasons_cuda: The first finish reason of each beam. Used only for beam search. + Shape: [max_batch_size, max_beam_width] + Returns: + finish_reasons_cuda: The finish reasons tensor. + Shape: [max_tokens, max_batch_size, max_beam_width] + """ + num_accepted_tokens_cuda, stop_word_indices_cuda, single_token_stop_words_only = ( + self._prepare_stop_word_handling_for_finish_reasons( + seq_slots_host, + is_draft_batch, + ) + ) + self._write_finish_reasons( + seq_slots=seq_slots_cuda, + seq_lens=seq_lens_cuda, + new_tokens=new_tokens_cuda, + num_accepted_tokens=num_accepted_tokens_cuda, + stop_word_indices=stop_word_indices_cuda, + single_token_stop_words_only=single_token_stop_words_only, + first_finish_reasons=first_finish_reasons_cuda, + ) + return self.store.finish_reasons_cuda + + def _prepare_stop_word_handling_for_finish_reasons( + self, + seq_slots_host: torch.Tensor, + is_draft_batch: bool, + ) -> tuple[torch.Tensor | int | None, torch.Tensor | None, bool]: + """Prepare stop word handling for finish reasons. + + Args: + seq_slots_host: The sequence slots of the processed requests. Used for accessing host buffers. + Shape: [len(requests)] + is_draft_batch: Whether the batch consists of draft requests. + Returns: + num_accepted_tokens_cuda: The number of accepted draft tokens +1 for each request. + Shape: [len(requests)] if torch.Tensor + stop_word_indices_cuda: The indices of the requests that have stop words in the current batch. + Shape: [len(requests_with_stop_words)] + single_token_stop_words_only: Whether all stop words in this batch are of length 1. + """ + # Filter all requests, that have stop words + store = self.store + num_accepted_tokens_cuda: torch.Tensor | int | None = None + stop_word_indices_cuda: torch.Tensor | None = None + single_token_stop_words_only: bool = False + + # NB: is_draft_batch is a workaround + # as draft requests can be in the sampler + # without having setup a slot in the FinishReasonsHandler. + # These can be removed once this can be avoided. + if is_draft_batch: + # Do not process stop words for draft requests + return ( + num_accepted_tokens_cuda, + stop_word_indices_cuda, + single_token_stop_words_only, + ) + + stop_word_mask = store.max_stop_word_lengths_host[seq_slots_host] > 0 + batch_has_stop_words = stop_word_mask.any() + + if batch_has_stop_words: + num_accepted_tokens_cuda = 1 + # Only calculate num_accepted_tokens from the accepted draft tokens if speculative decoding is enabled + if self._use_speculative_decoding: + num_accepted_tokens_cuda = ( + store.num_accepted_draft_tokens_host[seq_slots_host].to( + device="cuda", non_blocking=True + ) + + 1 + ) + stop_word_indices_cuda = torch.nonzero(stop_word_mask)[:, 0].to( + device="cuda", non_blocking=True + ) + single_token_stop_words_only = ( + store.max_stop_word_lengths_host[seq_slots_host].max().item() == 1 + ) + return num_accepted_tokens_cuda, stop_word_indices_cuda, single_token_stop_words_only + + @nvtx_range("_write_finish_reasons") + def _write_finish_reasons( + self, + *, + seq_slots: torch.Tensor, + seq_lens: torch.Tensor, + new_tokens: torch.Tensor, + num_accepted_tokens: torch.Tensor | int | None = None, + stop_word_indices: torch.Tensor | None = None, + single_token_stop_words_only: bool = False, + first_finish_reasons: torch.Tensor | None = None, + ) -> None: + """Writes the finish reasons to the finish_reasons tensor. + + The finish reasons are written to the finish_reasons tensor in the following order: + - Stop words + - Max length + - End ID + Later finish reasons overwrite earlier ones, in reverse precedence order. + + Args: + seq_slots: The sequence slots of the processed requests. Used for accessing device buffers. + Shape: [len(requests)] + seq_lens: The sequence lengths of the processed requests. + Shape: [len(requests)] + new_tokens: A buffer containing the newly generated tokens. + Shape: [max_tokens, max_batch_size, max_beam_width] + num_accepted_tokens: A buffer containing the number of accepted draft tokens +1 for each request. + Shape: [max_batch_size] if torch.Tensor + stop_word_indices: The indices of the requests that have stop words in the current batch. + Shape: [len(requests_with_stop_words)] + single_token_stop_words_only: Whether all stop words in this batch are of length 1 + first_finish_reasons: The first finish reason of each beam. + Shape: [max_batch_size, max_beam_width] + """ + + # Seq Slots should be on the same device as new_tokens + assert seq_slots.device == new_tokens.device + assert seq_lens.device == new_tokens.device + tokens = new_tokens[:, seq_slots] + + store = self.store + finish_reasons = store.finish_reasons_cuda + + # we need to fill with NOT_FINISHED so we can differentiate between + # previous requests that had the same seq slot + _ = finish_reasons.index_fill_(1, seq_slots, FinishReason.NOT_FINISHED.value) + batched_finish_reasons = finish_reasons[:, seq_slots] + + if stop_word_indices is not None: + assert num_accepted_tokens is not None, "draft_lengths is required for stop words" + stop_seq_slots = seq_slots[stop_word_indices] + stop_tokens = new_tokens[:, stop_seq_slots] + stop_words_func = ( + self._are_stop_words + if not single_token_stop_words_only + else self._are_stop_words_single_token + ) + batched_finish_reasons_stop_words = batched_finish_reasons[:, stop_word_indices] + _ = batched_finish_reasons_stop_words.masked_fill_( + stop_words_func( + stop_seq_slots, + stop_tokens, + num_accepted_tokens[stop_word_indices] + if isinstance(num_accepted_tokens, torch.Tensor) + else num_accepted_tokens, + ), + FinishReason.STOP_WORDS.value, + ) + batched_finish_reasons[:, stop_word_indices] = batched_finish_reasons_stop_words + + _ = batched_finish_reasons.masked_fill_( + self._are_max_length(seq_lens, store.max_lengths_cuda[seq_slots]), + FinishReason.LENGTH.value, + ) + + _ = batched_finish_reasons.masked_fill_( + self._are_end_id(store.end_ids_cuda[seq_slots], tokens), + FinishReason.END_ID.value, + ) + + finish_reasons[:, seq_slots] = batched_finish_reasons + if first_finish_reasons is not None: + # store the first stop reason for each beam of a seq_slot. + batched_first_finish_reasons = first_finish_reasons[seq_slots] + first_finish_reasons[seq_slots, ...] = torch.where( + batched_first_finish_reasons == FinishReason.NOT_FINISHED.value, + batched_finish_reasons, + batched_first_finish_reasons, + ) + + def _are_end_id(self, end_ids_cuda: torch.Tensor, tokens_cuda: torch.Tensor) -> torch.Tensor: + """Checks if the tokens are the end id + + Args: + end_ids_cuda: The end ids of the requests to check the end id of. + Shape: [len(requests)] + tokens_cuda: A buffer containing the newly generated tokens. + Shape: [max_tokens, len(requests), max_beam_width] + Returns: + A tensor where each element is True if the corresponding token is the end id, False otherwise + Shape: [max_tokens, len(requests), max_beam_width] + """ + return tokens_cuda == end_ids_cuda.view(1, -1, 1).expand( + self._max_tokens, -1, self._max_beam_width + ) + + def _are_max_length( + self, seq_lens_cuda: torch.Tensor, max_seq_lens_cuda: torch.Tensor + ) -> torch.Tensor: + """Checks which sequences are at or beyond the max length + + Args: + seq_lens_cuda: The sequence lengths of the requests to check the max length of. + Shape: [len(requests)] + max_seq_lens_cuda: The maximum sequence lengths of the requests to check the max length of. + Shape: [len(requests)] + Returns: + A tensor where each element is True if the sequence at the corresponding token + is at or beyond the max length, False otherwise + Shape: [max_tokens, len(requests), max_beam_width] + """ + lengths_tensor_cuda = (seq_lens_cuda.view(1, -1, 1) + self._max_tokens_offset_cuda).expand( + self._max_tokens, -1, self._max_beam_width + ) + max_lengths_tensor_cuda = max_seq_lens_cuda.view(1, -1, 1).expand( + self._max_tokens, -1, self._max_beam_width + ) + return lengths_tensor_cuda >= max_lengths_tensor_cuda + + @nvtx_range("_are_stop_words") + def _are_stop_words( + self, + seq_slots: torch.Tensor, + tokens: torch.Tensor, + num_accepted_tokens: torch.Tensor | int, + ) -> torch.Tensor: + """Checks if the tokens are stop words + + Args: + seq_slots: The sequence slots of the processed requests. Used for accessing device buffers. + Shape: [len(requests)] + tokens: A buffer containing the newly generated tokens. + Shape: [max_tokens, len(requests), max_beam_width] + num_accepted_tokens: The number of accepted draft tokens +1 for each request. + Shape: [len(requests)] if torch.Tensor + Returns: + A tensor where each element is True if the sequence at the corresponding token + ends with a stop word, False otherwise + Shape: [max_tokens, len(requests), max_beam_width] + """ + store = self.store + # num_words, len_words, batch_size + # unsqueeze the beam_width dimension to match the past tokens tensor + stop_words = ( + store.stop_words_cuda[..., seq_slots] + .unsqueeze(3) + .expand(-1, -1, -1, self._max_beam_width) + ) + # Get the past tokens + # num_steps, batch_size, beam_width + past_tokens_batch = store.past_tokens_cuda[:, seq_slots] + # Shift the past tokens to the left by the number of accepted draft tokens + + full_tokens = self._past_token_buffer_cuda[:, : seq_slots.shape[0]] + + index_tensor = ( + (self._stop_words_index_offset_cuda + num_accepted_tokens) + .unsqueeze(2) + .expand(-1, seq_slots.shape[0], self._max_beam_width) + ) + _ = torch.gather( + past_tokens_batch, + dim=0, + index=index_tensor, + out=full_tokens[: index_tensor.shape[0]], + ) + # Fill in the new tokens at the end of the past tokens buffer + full_tokens[-self._max_tokens :] = tokens + # short words are padded with _PAD_STOP_WORD_TOKEN_ID, so we need to mask them + mask = stop_words == self._PAD_STOP_WORD_TOKEN_ID + matches = torch.empty( + ( + self._max_tokens, + stop_words.shape[0], + stop_words.shape[1], + stop_words.shape[2], + stop_words.shape[3], + ), + device="cuda", + dtype=torch.bool, + ) + + # Get the comparison sequence for each step + full_tokens_for_match = full_tokens[self._multi_arange_indexing].view( + self._max_tokens, + 1, # Unsqueeze on dimension 1 to match the num_stop_words dimension of stop words + self._max_stop_word_length, + seq_slots.shape[0], + self._max_beam_width, + ) + # Unsqueeze on dimension 0 to match the max_tokens dimension of full tokens + stop_words_for_match = stop_words.unsqueeze(0) + _ = torch.eq(full_tokens_for_match, stop_words_for_match, out=matches) + # Mask the padding tokens + _ = matches.masked_fill_(mask.unsqueeze(0).expand(self._max_tokens, -1, -1, -1, -1), True) + # Update the past tokens storage for the next iteration + store.past_tokens_cuda[:, seq_slots] = full_tokens + # Return the result + word_len_dim = 2 + num_words_dim = 1 + return torch.any(matches.all(dim=word_len_dim), dim=num_words_dim) + + @nvtx_range("_are_stop_words_single_token") + def _are_stop_words_single_token( + self, + seq_slots: torch.Tensor, + tokens: torch.Tensor, + _num_accepted_tokens: torch.Tensor | int, + ) -> torch.Tensor: + """Checks if the tokens are stop words (single token per stop word only) + + Args: + seq_slots: The sequence slots of the processed requests. Used for accessing device buffers. + Shape: [len(requests)] + tokens: A buffer containing the newly generated tokens. + Shape: [max_tokens, len(requests), max_beam_width] + _num_accepted_tokens: Unused + Returns: + A tensor where each element is True if the sequence at the corresponding token + ends with a stop word, False otherwise + Shape: [max_tokens, len(requests), max_beam_width] + """ + per_step = torch.zeros( + (self._max_tokens, seq_slots.shape[0], self._max_beam_width), + dtype=torch.bool, + device="cuda", + ) + # num_words, 1, batch_size + stop_words = self.store.stop_words_cuda[:, -1:, seq_slots].unsqueeze(3) + full_tokens = tokens.unsqueeze(0) + matches = full_tokens == stop_words + _ = torch.any(matches, dim=0, out=per_step) + return per_step diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/logprobs.py b/tensorrt_llm/_torch/pyexecutor/sampler/logprobs.py new file mode 100644 index 000000000000..ffbdc4b0a98e --- /dev/null +++ b/tensorrt_llm/_torch/pyexecutor/sampler/logprobs.py @@ -0,0 +1,275 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Log-probs state and conversion helpers for ``TorchSampler``. + +Holds the device-side log-probs buffers (:class:`LogProbsStore`), the staged +host copies (:class:`LogProbsState` / :class:`LogProbsStateList`), and the +pure conversions from those into the per-request result format. + +Sizing and the per-step device gather stay in ``TorchSampler``: they depend on +the sampler-wide shapes (``TOPK_LOGPROBS_SHAPE`` and friends) that the sampler +grows in place as batches demand more top-k slots. +""" + +from dataclasses import dataclass +from typing import TypeAlias, cast + +import torch + +from tensorrt_llm._utils import prefer_pinned +from tensorrt_llm.executor.result import Logprob, SimpleTokenLogprobs, TokenLogprobs + +from ..llm_request import LlmRequest + +__all__ = [ + "LogProbsState", + "LogProbsStateList", + "LogProbsStore", + "convert_logprobs_tensor_to_list", + "get_logprobs_from_request", + "store_logprobs_list_to_request", +] + + +@dataclass(kw_only=True) +class LogProbsState: + sampled_vals: torch.Tensor + sampled_indices: torch.Tensor + sampled_rank: torch.Tensor + topk_vals: torch.Tensor + topk_indices: torch.Tensor + + +_LogProbsFloatState: TypeAlias = list[list[list[float]]] +_LogProbsIntState: TypeAlias = list[list[list[int]]] + + +@dataclass(kw_only=True) +class LogProbsStateList: + sampled_vals: _LogProbsFloatState + sampled_indices: _LogProbsIntState + sampled_rank: _LogProbsIntState + topk_vals: _LogProbsFloatState + topk_indices: _LogProbsIntState + + @staticmethod + def from_logprobs_state(logprobs_state: LogProbsState) -> "LogProbsStateList": + return LogProbsStateList( + sampled_vals=logprobs_state.sampled_vals.tolist(), + sampled_indices=logprobs_state.sampled_indices.tolist(), + topk_vals=logprobs_state.topk_vals.tolist(), + topk_indices=logprobs_state.topk_indices.tolist(), + sampled_rank=logprobs_state.sampled_rank.tolist(), + ) + + +@dataclass(kw_only=True) +class LogProbsStore: + """Auxiliary data structures used for log-probs handling.""" + + sampled_log_prob_indices: torch.Tensor + """Shape: batch_size, beam_width, max_tokens + Usage: Stores the token indices of the sampled logprobs""" + sampled_log_probs: torch.Tensor + """Shape: batch_size, beam_width, max_tokens + Usage: Stores the values of the sampled logprobs""" + sampled_log_prob_ranks: torch.Tensor + """Shape: batch_size, beam_width, max_tokens + Usage: Stores the ranks of the sampled logprobs""" + topk_indices: torch.Tensor + """Shape: batch_size, max_tokens, max_topk_logprobs + Usage: Stores the token indices of the topk logprobs""" + topk_vals: torch.Tensor + """Shape: batch_size, max_tokens, max_topk_logprobs + Usage: Stores the values of the topk logprobs""" + + +def convert_logprobs_tensor_to_list( + token_tensor: torch.Tensor, + logprobs_tensor: torch.Tensor, +) -> list[list[dict[int, Logprob]]]: + """Convert the logprobs tensor to a list of lists of dictionaries of Logprob objects + + Logprobs storage expects logprobs as a list[list[dict[int, Logprob]]] object + + args: + token_tensor: torch.Tensor. Shape: beam_width, num_tokens, num_logprobs + logprobs_tensor: torch.Tensor. Shape: beam_width, num_tokens, num_logprobs + output: + list[list[dict[int, Logprob]]]. Shape: (beam_width, num_tokens) + """ + assert token_tensor.dim() == 3 and logprobs_tensor.dim() == 3, ( + f"Token and logprobs tensors must have 3 dimensions (beam_width, num_tokens, num_logprobs). \ + Got shapes (token_tensor) {token_tensor.shape} and (logprobs_tensor) {logprobs_tensor.shape} instead" + ) + + token_log_probs: list[list[dict[int, Logprob]]] = [] + token_list = token_tensor.tolist() + logprobs_list = logprobs_tensor.tolist() + for beam_idx in range(token_tensor.shape[0]): + beam_token_log_probs: list[dict[int, Logprob]] = [] + for topk_token, topk_logprob in zip(token_list[beam_idx], logprobs_list[beam_idx]): + logprobs = { + token: Logprob(logprob=logprob, rank=rank + 1) + for rank, (token, logprob) in enumerate(zip(topk_token, topk_logprob)) + } + beam_token_log_probs.append(logprobs) + token_log_probs.append(beam_token_log_probs) + + return token_log_probs + + +def store_logprobs_list_to_request( + logprobs_state_list: LogProbsStateList, + req_seq_slot: int, + beam_width: int, + count: int, + num_topk_logprobs: int, + simple_format: bool = False, +) -> list[list[dict[int, Logprob]]] | list[list[float]]: + """Convert the LogProbsStateList object to per-token logprobs. + + By default returns ``list[list[dict[int, Logprob]]]``. When + ``simple_format`` is True and ``num_topk_logprobs == 0`` the result is a + flat ``list[list[float]]`` (one logprob per generated token, per beam). + + args: + logprobs_state_list: LogProbsStateList. Contains the topk indices, topk values, + sampled indices, sampled values, and sampled ranks. + req_seq_slot: int. The sequence slot of the request. + beam_width: int. The beam width of the request. + count: int. The number of tokens to store. + num_topk_logprobs: int. The number of topk logprobs of each token. + simple_format: bool. If True (and num_topk_logprobs == 0), return + ``list[list[float]]`` instead of the dict format. Avoids per-token + dict allocation when only the sampled-token logprob is needed. + output: + list[list[dict[int, Logprob]]] (default) or list[list[float]] (simple format). + Shape: (beam_width, count) + """ + + sampled_log_probs_indices_list = logprobs_state_list.sampled_indices[req_seq_slot] + sampled_log_probs_vals_list = logprobs_state_list.sampled_vals[req_seq_slot] + sampled_log_probs_rank_list = logprobs_state_list.sampled_rank[req_seq_slot] + + if num_topk_logprobs == 0: + if simple_format: + token_log_probs_simple: list[list[float]] = [ + [sampled_log_probs_vals_list[beam_idx][step_idx] for step_idx in range(count)] + for beam_idx in range(beam_width) + ] + return token_log_probs_simple + + token_log_probs: list[list[dict[int, Logprob]]] = [ + [ + { + sampled_log_probs_indices_list[beam_idx][step_idx]: Logprob( + sampled_log_probs_vals_list[beam_idx][step_idx], + sampled_log_probs_rank_list[beam_idx][step_idx] + 1, + ) + } + for step_idx in range(count) + ] + for beam_idx in range(beam_width) + ] + else: + token_list = logprobs_state_list.topk_indices[req_seq_slot] + logprobs_list = logprobs_state_list.topk_vals[req_seq_slot] + token_log_probs = [[] for _ in range(beam_width)] + for step_idx in range(count): + topk_tokens = token_list[step_idx][:num_topk_logprobs] + topk_logprobs = logprobs_list[step_idx][:num_topk_logprobs] + min_rank = len(topk_tokens) + 1 + + topk_logprob_dict = { + token: Logprob(logprob=logprob, rank=rank + 1) + for rank, (token, logprob) in enumerate(zip(topk_tokens, topk_logprobs)) + } + + for beam_idx in range(beam_width): + # NB: Keeps sampled token in the first position (cf. https://stackoverflow.com/a/67786863) + logprobs = { + sampled_log_probs_indices_list[beam_idx][step_idx]: Logprob( + logprob=sampled_log_probs_vals_list[beam_idx][step_idx], + rank=max( + min_rank, + sampled_log_probs_rank_list[beam_idx][step_idx] + 1, + ), + ), + **topk_logprob_dict, + } + token_log_probs[beam_idx].append(logprobs) + + return token_log_probs + + +def get_logprobs_from_request( + request: LlmRequest, + pin_memory: bool = True, + preallocate_extra_steps: int = 0, +) -> tuple[torch.Tensor, torch.Tensor]: + """Extract the logprobs from the request. + + Returns: + logprobs_tensor: A tensor of shape (beam_width, num_generated_tokens, num_logprobs) + logprobs_indices_tensor: A tensor of shape (beam_width, num_generated_tokens, num_logprobs) + """ + pin_memory = pin_memory and prefer_pinned() + num_generated_tokens = request.max_beam_num_tokens - request.py_prompt_len + assert request.py_num_logprobs == 0, ( + "Beam search only supports returning the sampled logprob per token" + ) + logprobs_tensor_full = torch.empty( + ( + request.py_beam_width, + num_generated_tokens + preallocate_extra_steps, + request.py_num_logprobs + 1, + ), + pin_memory=pin_memory, + dtype=torch.float32, + ) + logprobs_indices_tensor_full = torch.empty( + ( + request.py_beam_width, + num_generated_tokens + preallocate_extra_steps, + request.py_num_logprobs + 1, + ), + pin_memory=pin_memory, + dtype=torch.int32, + ) + # NB: forward slicing, because [:, :-0, :] would yield an empty view + # instead of the full history when preallocate_extra_steps == 0. + logprobs_tensor = logprobs_tensor_full[:, :num_generated_tokens, :] + logprobs_indices_tensor = logprobs_indices_tensor_full[:, :num_generated_tokens, :] + if logprobs_tensor.numel() > 0: + logprobs_list = request.py_result.log_probs + assert logprobs_list is not None + + if request.py_logprobs_simple_format: + tokens = request.get_tokens() + for beam_idx, beam_logprobs in enumerate(logprobs_list): + beam_logprobs = cast(SimpleTokenLogprobs, beam_logprobs) + for token_idx, token_logprobs_simple in enumerate(beam_logprobs): + logprobs_tensor[beam_idx, token_idx, 0] = token_logprobs_simple + logprobs_indices_tensor[beam_idx, token_idx, 0] = tokens[beam_idx][token_idx] + else: + for beam_idx, beam_logprobs in enumerate(logprobs_list): + beam_logprobs = cast(TokenLogprobs, beam_logprobs) + for token_idx, token_logprobs in enumerate(beam_logprobs): + for key, value in token_logprobs.items(): + assert value.rank is not None + logprobs_tensor[beam_idx, token_idx, value.rank - 1] = value.logprob + logprobs_indices_tensor[beam_idx, token_idx, value.rank - 1] = key + return logprobs_tensor_full, logprobs_indices_tensor_full diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py b/tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py index 0dc72d6cb158..a0e1c7c44322 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py @@ -14,31 +14,56 @@ """FlashInfer-accelerated sampling kernels. -These ops depend on flashinfer; the import is guarded so the module stays -importable without it. +The flashinfer import is guarded so this module stays importable without it; +each op then raises an ImportError when called. ``TorchSampler`` checks +availability in its constructor. -Randomness can be supplied either way (flashinfer accepts both in one -signature; explicit ``seed``/``offset`` take precedence over ``generator``): +Randomness is supplied either as a ``generator`` (host-side ``torch.Generator``, +for eager paths) or as stateless ``seed``/``offset`` device tensors, which are +required under CUDA graph capture. Explicit ``seed``/``offset`` take precedence. -- ``generator``: stateful host-side ``torch.Generator``, for eager paths. -- ``seed``/``offset``: stateless device tensors, required under CUDA graph - capture (a ``torch.Generator`` advances host-side at launch time, so its - state would be frozen into the graph and every replay would reuse the same - random values). +Every op is ``@_compiler_disable``d to keep one clean Dynamo graph break per op +instead of a per-call break from tracing flashinfer's lazy JIT bootstrap. """ -from typing import Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, Union, cast import torch from tensorrt_llm._torch.flashinfer_utils import IS_FLASHINFER_AVAILABLE, get_env_enable_pdl +if TYPE_CHECKING: + from tensorrt_llm.llmapi.llm_args import AdvancedSamplingMode + +_OpT = TypeVar("_OpT", bound=Callable[..., Any]) + + +def _compiler_disable(fn: _OpT) -> _OpT: + """``torch.compiler.disable``, typed: the torch stub is untyped and would + fail mypy strict (untyped-decorator) if applied directly.""" + return cast(_OpT, torch.compiler.disable(fn)) + + if IS_FLASHINFER_AVAILABLE: import flashinfer.sampling +else: + + class _FlashInferUnavailable: + """Placeholder that raises on first use instead of a bare NameError.""" + + def __getattr__(self, name: str) -> Any: + raise ImportError( + "flashinfer is required for the FlashInfer sampling ops but is " + "not installed; please install the version pinned in " + "requirements.txt." + ) + + flashinfer = _FlashInferUnavailable() # type: ignore[assignment] SeedOrTensor = Union[int, torch.Tensor] +@_compiler_disable def top_k_top_p_sampling_from_logits_op( logits: torch.Tensor, top_k: torch.Tensor, @@ -68,6 +93,42 @@ def top_k_top_p_sampling_from_logits_op( return tokens +@_compiler_disable +def top_k_top_p_sampling_from_probs_op( + probs: torch.Tensor, + top_k: torch.Tensor, + top_p: torch.Tensor, + *, + generator: Optional[torch.Generator] = None, + seed: Optional[SeedOrTensor] = None, + offset: Optional[SeedOrTensor] = None, + check_nan: bool = False, +) -> torch.Tensor: + """Fused top-k + top-p filtering and sampling from a probability distribution. + + ``filter_apply_order="top_k_first"`` matches the renorm pipeline (top-k, then + top-p over the renormalized survivors). Sampling straight from probs skips + the filtered full-vocab tensor that separate renorm + sample would + materialize, so this is the cheapest way to terminate a renorm chain -- + at the cost that no filter may run after it. + Randomness: pass ``generator`` (eager) or ``seed``/``offset`` (CUDA graph); + see module docstring for the full contract. + """ + tokens: torch.Tensor = flashinfer.sampling.top_k_top_p_sampling_from_probs( + probs, + top_k=top_k, + top_p=top_p, + filter_apply_order="top_k_first", + deterministic=True, + check_nan=check_nan, + generator=generator, + seed=seed, + offset=offset, + ) + return tokens + + +@_compiler_disable def sampling_from_probs_op( probs: torch.Tensor, *, @@ -92,6 +153,7 @@ def sampling_from_probs_op( return tokens +@_compiler_disable def top_k_sampling_from_probs_op( probs: torch.Tensor, top_k: torch.Tensor, @@ -118,6 +180,7 @@ def top_k_sampling_from_probs_op( return tokens +@_compiler_disable def top_p_sampling_from_probs_op( probs: torch.Tensor, top_p: torch.Tensor, @@ -144,12 +207,13 @@ def top_p_sampling_from_probs_op( return tokens -# The three ops below wrap the mask -> softmax -> renorm pipeline stages 1:1. +# The four ops below wrap the mask -> softmax -> renorm pipeline stages 1:1. # The wrappers exist so callers stay importable without flashinfer installed # (the flashinfer import above is guarded); softmax_op additionally centralizes # the PDL env decision. +@_compiler_disable def softmax_op( logits: torch.Tensor, temperature: Optional[torch.Tensor], @@ -160,6 +224,7 @@ def softmax_op( return probs +@_compiler_disable def top_k_mask_logits_op( logits: torch.Tensor, top_k: torch.Tensor, @@ -168,6 +233,16 @@ def top_k_mask_logits_op( return masked +@_compiler_disable +def top_k_renorm_probs_op( + probs: torch.Tensor, + top_k: torch.Tensor, +) -> torch.Tensor: + renormed: torch.Tensor = flashinfer.sampling.top_k_renorm_probs(probs, top_k) + return renormed + + +@_compiler_disable def top_p_renorm_probs_op( probs: torch.Tensor, top_p: torch.Tensor, @@ -176,6 +251,7 @@ def top_p_renorm_probs_op( return renormed +@_compiler_disable def compute_probs_from_logits_op( logits: torch.Tensor, temperatures: torch.Tensor, @@ -196,3 +272,99 @@ def compute_probs_from_logits_op( if top_p is not None: probs = flashinfer.sampling.top_p_renorm_probs(probs, top_p) return probs + + +# --------------------------------------------------------------------------- +# Speculative-decoding samplers (per-request tensor params). These build on the +# flashinfer ops above, sharing only vanilla's greedy-temperature threshold; +# used by the speculative-decoding paths (one-model draft sampling, rejection +# sampling). +# --------------------------------------------------------------------------- + + +def sanitize_top_k(top_k: torch.Tensor, vocab_size: int) -> torch.Tensor: + """Map ``top_k`` into a backend-safe range before top-k filtering. + + Per ``SamplingParams``, ``top_k == 0`` means "all logits" (top-k disabled), + but the flashinfer top-k kernels (``top_k_mask_logits``) break on a literal + 0 — they mask the entire row (all-zero probs). Map any non-positive value + (and any oversized disable sentinel such as ``INT32_MAX``) to + ``vocab_size`` (== keep all tokens), leaving genuine top_k values + untouched. + """ + return top_k.clamp(max=vocab_size).masked_fill_(top_k <= 0, vocab_size) + + +@torch.compile(options={"max-autotune": True}) +def compute_probs_from_logits( + logits: torch.Tensor, + temperatures: torch.Tensor, + top_k: Optional[torch.Tensor], + top_p: Optional[torch.Tensor], +) -> torch.Tensor: + """Compute filtered+normalized probs via flashinfer (hard dependency). + + ``temperatures``, ``top_k``, ``top_p`` are per-request tensors matching the + spec-decoding call site in interface.py. A ``None`` top_k / top_p skips that + filter's kernel. + """ + if top_k is not None: + top_k = sanitize_top_k(top_k, logits.shape[-1]) + + return compute_probs_from_logits_op(logits, temperatures, top_k, top_p) + + +def resolve_advanced_sampling_filters( + advanced_sampling_mode: "AdvancedSamplingMode", + top_k: Optional[torch.Tensor], + top_p: Optional[torch.Tensor], +) -> tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: + """Resolve advanced_sampling_mode to effective (top_k, top_p) tensors. + + A filter the mode disables (via ``skips_top_k`` / ``skips_top_p``), or one already + None, becomes None so the downstream op skips that kernel; kept filters pass through + unchanged (the op sanitizes top_k internally). + """ + eff_top_k = None if advanced_sampling_mode.skips_top_k or top_k is None else top_k + eff_top_p = None if advanced_sampling_mode.skips_top_p or top_p is None else top_p + return eff_top_k, eff_top_p + + +@torch.compile(options={"max-autotune": True}) +def sample_from_logits_op( + logits: torch.Tensor, + temperatures: torch.Tensor, + top_k: Optional[torch.Tensor] = None, + top_p: Optional[torch.Tensor] = None, + seed: Optional[torch.Tensor] = None, + offset: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """CUDA-graph compatible one-model sampler; returns sampled tokens. + + ``top_k`` / ``top_p`` are None when the caller's advanced_sampling_mode disables that + filter. + """ + if top_k is not None: + top_k = sanitize_top_k(top_k, logits.shape[-1]) + logits = top_k_mask_logits_op(logits, top_k) + probs = softmax_op(logits, temperatures) + if top_p is not None: + return top_p_sampling_from_probs_op(probs, top_p, seed=seed, offset=offset) + return sampling_from_probs_op(probs, seed=seed, offset=offset) + + +@torch.compile(options={"max-autotune": True}) +def sampling_batch_spec_dec_one_model_for_rejection( + logits: torch.Tensor, + temperatures: torch.Tensor, + top_k: Optional[torch.Tensor], + top_p: Optional[torch.Tensor], + seed: Optional[torch.Tensor] = None, + offset: Optional[torch.Tensor] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Draft sampler returning tokens AND probs for the downstream rejection-sampling path.""" + # Rejection sampling relies on flashinfer's seed/offset support for + # determinism and cross-rank consistency. + probs = compute_probs_from_logits(logits, temperatures, top_k, top_p) + tokens = sampling_from_probs_op(probs, seed=seed, offset=offset) + return tokens, probs diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py b/tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py index 5307876ae61d..7f041c245aa0 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py @@ -15,9 +15,10 @@ """PyTorch-native sampling kernels. Pure tensor functions that operate on logits and probabilities with no -dependency on the sampling_utils interface or other backend implementation modules. +dependency on the sampler_strategy interface or other backend implementation modules. """ +import math from dataclasses import dataclass from typing import Optional, cast @@ -50,32 +51,60 @@ class BeamSearchMetadata(StrategyMetadata): beam_idx_arange: torch.Tensor +def min_p_renorm_probs( + probs: torch.Tensor, + min_p: torch.Tensor | float, +) -> torch.Tensor: + """Keep tokens with prob >= ``min_p`` times the per-row max, then renormalize. + + ``min_p`` is a scalar or a per-request tensor. + """ + max_probs = probs.max(dim=-1, keepdim=True).values + if isinstance(min_p, torch.Tensor): + min_p = min_p.reshape(-1, 1) + thresholds = min_p * max_probs + probs = probs.masked_fill(probs < thresholds, 0.0) + probs = probs / probs.sum(dim=-1, keepdim=True) + return probs + + def top_k_top_p_sampling_batch( logits: torch.Tensor, *, temperature: float, top_k: Optional[int] = None, top_p: float = 1.0, + min_p: float = 0.0, generator: Optional[torch.Generator] = None, ) -> tuple[torch.Tensor, torch.Tensor]: - """Temperature + optional top-k / top-p filtering + multinomial sampling. + """Temperature + optional min-p / top-k / top-p filtering + multinomial sampling. ``top_k=None`` (or ``vocab_size``) disables top-k filtering; ``top_p=1`` - disables top-p filtering. With both disabled this is plain temperature - sampling. + disables top-p filtering; ``min_p=0`` disables min-p filtering. With all + disabled this is plain temperature sampling. """ logits_dim = logits.dim() assert logits_dim == 2, "logits should be 2D: [batch_size, vocab_size]" assert temperature > 0, "non-greedy sampling requires valid temperature" logits = logits / max(temperature, 1e-5) batch_size, vocab_size = logits.size() - if top_k is None: + # 0 / non-positive means "keep all" (the min_p disabled-top_k sentinel), + # matching sanitize_top_k on the flashinfer path. + if top_k is None or top_k <= 0: top_k = vocab_size assert top_k > 1, "non-greedy sampling requires valid top_k" need_top_k = top_k < vocab_size assert top_p > 0, "non-greedy sampling requires valid top_p" need_top_p = top_p < 1 + assert 0 <= min_p < 1, "non-greedy sampling requires valid min_p" + need_min_p = min_p > 0 + + if need_min_p: + # Thresholding logits at max_logit + log(min_p) keeps the tokens with + # prob >= min_p * max_prob; the softmax below renormalizes. + min_values = logits.max(dim=-1, keepdim=True).values + math.log(min_p) + logits = torch.where(logits < min_values, torch.full_like(logits, float("-inf")), logits) if need_top_k: values, _ = torch.topk(logits, top_k, dim=-1) @@ -290,25 +319,6 @@ def sample_rejected( GREEDY_TEMPERATURE_THRESHOLD = 1e-4 -def safely_apply_temperature_inplace( - logits_inout: torch.Tensor, temp: torch.Tensor -) -> torch.Tensor: - """Divide logits by per-row temperature in place, guarding the greedy sentinel. - - Greedy requests carry a temperature of 0 / <= ``GREEDY_TEMPERATURE_THRESHOLD``. - Dividing by it would blow logits up to inf/nan and corrupt downstream sampling - (argmax / softmax / multinomial). Those rows are clamped to a temperature of 1.0 - so the division is numerically safe; callers are expected to overwrite the greedy - rows with their argmax result afterwards (e.g. via ``torch.where(is_greedy, ...)``), - so the value used for the clamped rows here does not affect the final output. - - ``logits_inout`` is modified in place (``div_``) and also returned for - convenience; ``temp`` is left untouched. - """ - safe_temp = torch.where(temp <= GREEDY_TEMPERATURE_THRESHOLD, torch.ones_like(temp), temp) - return logits_inout.div_(safe_temp.unsqueeze(dim=1)) - - class Fusions: @staticmethod @torch.compile(dynamic=None, fullgraph=True) @@ -373,3 +383,280 @@ def gather_log_softmax(inputs_cuda: torch.Tensor, indices_cuda: torch.Tensor) -> torch._dynamo.mark_dynamic(inputs_cuda, 0) torch._dynamo.mark_dynamic(indices_cuda, 0) return Fusions._gather_log_softmax_impl(inputs_cuda, indices_cuda) + + # --- Top-P Decay ops --------------------------------------------------- + # Host-launch-bound per-step ops (a few dozen elements per row), fused with + # Inductor to keep the launch count low. mode="max-autotune-no-cudagraphs": + # cudagraphs is unsafe here (the update mutates persistent per-slot state + # in place and the gather's output is consumed outside the compiled region; + # cudagraph static output buffers get overwritten by subsequent replays). + # mark_dynamic on the batch-varying dims avoids recompilation as the batch + # composition changes. Compilation is lazy: the first decay-active request + # pays it (roughly a second); non-decay workloads never trigger it. See + # top_p_decay.TopPDecayStore for the feature-level semantics. + + @staticmethod + @torch.compile(mode="max-autotune-no-cudagraphs") + def _top_p_decay_update_impl( + runtime_top_p: torch.Tensor, + initial_top_p: torch.Tensor, + top_p_decay: torch.Tensor, + top_p_min: torch.Tensor, + reset_ids: torch.Tensor, + is_decay_slot: torch.Tensor, + step_tokens: torch.Tensor, + sampled_slots: torch.Tensor, + ) -> None: + active = is_decay_slot[sampled_slots] + current = runtime_top_p[sampled_slots] + updated = torch.where( + step_tokens[sampled_slots] == reset_ids[sampled_slots], + initial_top_p[sampled_slots], + torch.maximum(current * top_p_decay[sampled_slots], top_p_min[sampled_slots]), + ) + runtime_top_p[sampled_slots] = torch.where(active, updated, current) + + @staticmethod + def top_p_decay_update( + *, + runtime_top_p: torch.Tensor, + initial_top_p: torch.Tensor, + top_p_decay: torch.Tensor, + top_p_min: torch.Tensor, + reset_ids: torch.Tensor, + is_decay_slot: torch.Tensor, + step_tokens: torch.Tensor, + sampled_slots: torch.Tensor, + ) -> None: + """Fused in-place update of ``runtime_top_p`` for the sampled decay slots. + + Applies the Top-P Decay recurrence (see ``top_p_decay.TopPDecayStore`` + for the feature-level semantics) to every sampled row whose slot is + decay-active per ``is_decay_slot``. + + All per-slot tensors are 1-D of length ``max_num_sequences``; + ``step_tokens`` is a slot-indexed 1-D (possibly strided) int32 view of + the new-tokens buffer for a fixed step/beam + (``new_tokens[step, :, beam]``); ``sampled_slots`` is 1-D of length + ``num_sampled`` (this iteration's rows). ``runtime_top_p`` is mutated + in place; nothing is returned. + """ + torch._dynamo.mark_dynamic(sampled_slots, 0) + Fusions._top_p_decay_update_impl( + runtime_top_p, + initial_top_p, + top_p_decay, + top_p_min, + reset_ids, + is_decay_slot, + step_tokens, + sampled_slots, + ) + + @staticmethod + @torch.compile(mode="max-autotune-no-cudagraphs") + def _top_p_decay_gather_impl( + runtime_top_p: torch.Tensor, + is_decay_slot: torch.Tensor, + static_top_p: torch.Tensor, + slots: torch.Tensor, + ) -> torch.Tensor: + return torch.where(is_decay_slot[slots], runtime_top_p[slots], static_top_p) + + @staticmethod + def top_p_decay_gather( + *, + runtime_top_p: torch.Tensor, + is_decay_slot: torch.Tensor, + static_top_p: torch.Tensor, + slots: torch.Tensor, + ) -> torch.Tensor: + """Fused pre-sample per-row top-p gather for decay-active rows. + + Returns a new per-row tensor:: + + row_top_p[i] = runtime_top_p[slots[i]] if is_decay_slot[slots[i]] + = static_top_p[i] otherwise + + ``runtime_top_p`` / ``is_decay_slot`` are per-slot arrays; + ``static_top_p`` and ``slots`` are per-row (length = the group's + per-step row count). + """ + torch._dynamo.mark_dynamic(slots, 0) + torch._dynamo.mark_dynamic(static_top_p, 0) + return Fusions._top_p_decay_gather_impl(runtime_top_p, is_decay_slot, static_top_p, slots) + + # --- Occurrence penalties (repetition / presence / frequency) ----------- + # torch/torch.compile counterpart of the C++ ``batchApplyPenalty`` kernel, + # driven by ``PenaltyHandler`` in penalties.py, which owns the + # workspace and documents its semantics (see ``PenaltyStore`` there). + + @staticmethod + def update_occurrence_workspace( + counts_cuda: torch.Tensor, + presence_prefix_cuda: Optional[torch.Tensor], + counted_slots: torch.Tensor, + counted_tokens: torch.Tensor, + prefix_slots: Optional[torch.Tensor] = None, + prefix_tokens: Optional[torch.Tensor] = None, + ) -> None: + """Scatter (slot, token) pairs into the persistent occurrence workspace. + + Args: + counts_cuda: ``int32[num_slots, vocab_size]``, incremented in place. + presence_prefix_cuda: ``bool[num_slots, vocab_size]`` prefix-presence + mask, or ``None`` when no active request uses + ``prompt_ignore_length``. + counted_slots / counted_tokens: 1-D int64 pairs to increment in + ``counts_cuda``. + prefix_slots / prefix_tokens: 1-D int64 pairs to mark in + ``presence_prefix_cuda``; ``None`` when there is nothing to mark. + """ + if counted_slots.numel() > 0: + ones = torch.ones( + counted_slots.shape[0], dtype=counts_cuda.dtype, device=counts_cuda.device + ) + # accumulate=True sums repeated (slot, token) pairs -> occurrence count. + counts_cuda.index_put_((counted_slots, counted_tokens), ones, accumulate=True) + if ( + presence_prefix_cuda is not None + and prefix_slots is not None + and prefix_tokens is not None + and prefix_slots.numel() > 0 + ): + # Marking a dense bool mask is idempotent, so duplicate tokens are safe. + presence_prefix_cuda[prefix_slots, prefix_tokens] = True + + # fullgraph=True is safe here: served model has fixed shapes and compiles ~2 graphs, + # well under the default limit (8) + @staticmethod + @torch.compile(fullgraph=True, mode="max-autotune-no-cudagraphs") + def _apply_occurrence_penalties_impl( + logits: torch.Tensor, + counts_cuda: torch.Tensor, + prefix_seen_cuda: Optional[torch.Tensor], + active_cuda: torch.Tensor, + has_previous_token_cuda: torch.Tensor, + new_tokens: torch.Tensor, + seq_slots: torch.Tensor, + request_offsets: torch.Tensor, + request_num_steps: torch.Tensor, + repetition_cuda: torch.Tensor, + presence_cuda: torch.Tensor, + frequency_cuda: torch.Tensor, + ) -> None: + vocab = logits.size(-1) + + # Fold the device-pending sampled token into the persistent counts, once per armed + # active slot, before the gather reads them, via one flat scatter_add. Masked entries + # add 0 at counts[slot, 0], so inactive/unarmed/out-of-range slots are no-ops. + previous_token = new_tokens[0, seq_slots, 0].to(torch.int64) + fold_ok = ( + active_cuda[seq_slots] + & has_previous_token_cuda[seq_slots] + & (request_num_steps > 0) + & (previous_token >= 0) + & (previous_token < vocab) + ) + flat_index = seq_slots * vocab + torch.where( + fold_ok, previous_token, previous_token.new_zeros(()) + ) + counts_cuda.view(-1).scatter_add_(0, flat_index, fold_ok.to(counts_cuda.dtype)) + + # Map each logits row to its owning request with a broadcasted range comparison. + # This is O(T * R), but T and R are both small (rows per step x requests) and the + # whole thing fuses into the surrounding elementwise graph, so it measures faster + # than either a searchsorted lookup or a repeat_interleave expansion. Notably + # torch.repeat_interleave must NOT be used here: its output length is + # sum(num_steps), which -- without an explicit host-provided output_size -- torch + # reads back from the device, and that per-step D2H sync destroys the overlap + # between the sampler's host work and the model forward (measured ~20x slower). + rows = torch.arange(logits.size(0), device=logits.device).unsqueeze(1) # [T, 1] + owned = (rows >= request_offsets) & (rows < request_offsets + request_num_steps) # [T, R] + row_owned = owned.any(dim=1) # [T] + row_slot = (owned * seq_slots).sum(dim=1) # [T]; slot per row, 0 for unowned + row_active = row_owned & active_cuda[row_slot] + + count = counts_cuda[row_slot] + rep = repetition_cuda[row_slot].unsqueeze(1) + pre = presence_cuda[row_slot].unsqueeze(1) + freq = frequency_cuda[row_slot].unsqueeze(1) + + seen = count > 0 + if prefix_seen_cuda is not None: + # Prompt-ignore-prefix tokens count for repetition only, not presence/frequency. + seen = seen | prefix_seen_cuda[row_slot] + + penalized = logits.float() + repeated = torch.where(penalized < 0, penalized * rep, penalized / rep) + penalized = torch.where(seen, repeated, penalized) + penalized = penalized - torch.where( + count > 0, + pre + freq * count.to(torch.float32), + penalized.new_zeros(()), + ) + limit = torch.finfo(logits.dtype).max + penalized = penalized.clamp(-limit, limit).to(logits.dtype) + # Cast before the select so inactive rows stay bit-identical, then write in place. + logits.copy_(torch.where(row_active.unsqueeze(1), penalized, logits)) + + @staticmethod + def apply_batched_occurrence_penalties( + logits: torch.Tensor, + counts_cuda: torch.Tensor, + presence_prefix_cuda: Optional[torch.Tensor], + active_cuda: torch.Tensor, + has_previous_token_cuda: torch.Tensor, + new_tokens: torch.Tensor, + seq_slots: torch.Tensor, + request_offsets: torch.Tensor, + request_num_steps: torch.Tensor, + repetition_cuda: torch.Tensor, + presence_cuda: torch.Tensor, + frequency_cuda: torch.Tensor, + ) -> None: + """Apply occurrence penalties to ``logits`` in place, before temperature handling. + + Args: + logits: ``[T, vocab_size]`` packed generated-token logits, where + ``T == sum(num_steps * num_beams)``. Request ``r`` owns the rows + ``request_offsets[r] + step`` for ``step in [0, request_num_steps[r])``; + rows no request owns are left bit-identical. Modified in place. + counts_cuda / presence_prefix_cuda: the occurrence workspace; see + ``PenaltyHandler.PenaltyStore`` for their semantics. + active_cuda / has_previous_token_cuda / repetition_cuda / presence_cuda / + frequency_cuda: per-slot buffers of length ``max_num_sequences``. + new_tokens: ``[max_tokens, max_num_sequences, max_beam_width]`` device + buffer holding the previous step's sampled token. + seq_slots: ``int64[R]`` slot per request. + request_offsets / request_num_steps: ``[R]`` device tensors, already + staged by the caller. The owned spans must not overlap, but they need + not be ordered, and rows they skip are left bit-identical. + + All heavy lifting is fused into the single compiled ``_apply_occurrence_penalties_impl`` + graph; this wrapper only marks the batch-varying dims dynamic. + """ + if seq_slots.numel() == 0 or logits.size(0) == 0: + return + + # Batch-varying dim-0 tensors; mark every one, or an unmarked peer forces the marked + # dims to specialize (ConstraintViolationError under dynamic=None). counts/active/params + # keep dim 0 == max_num_sequences (fixed) and new_tokens dim 1 == max_num_sequences. + torch._dynamo.mark_dynamic(logits, 0) + torch._dynamo.mark_dynamic(seq_slots, 0) + torch._dynamo.mark_dynamic(request_offsets, 0) + torch._dynamo.mark_dynamic(request_num_steps, 0) + Fusions._apply_occurrence_penalties_impl( + logits, + counts_cuda, + presence_prefix_cuda, + active_cuda, + has_previous_token_cuda, + new_tokens, + seq_slots, + request_offsets, + request_num_steps, + repetition_cuda, + presence_cuda, + frequency_cuda, + ) diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/penalties.py b/tensorrt_llm/_torch/pyexecutor/sampler/penalties.py new file mode 100644 index 000000000000..4f3ea18a2613 --- /dev/null +++ b/tensorrt_llm/_torch/pyexecutor/sampler/penalties.py @@ -0,0 +1,483 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Occurrence penalties (repetition / presence / frequency) for ``TorchSampler``. + +The feature's persistent device state lives in :class:`PenaltyStore` (which +documents the workspace semantics) and its whole lifecycle in +:class:`PenaltyHandler`; ``TorchSampler`` owns one instance and drives it +through request validation, admission, the per-step apply, and the +post-processing commit of finalized tokens. +""" + +from dataclasses import dataclass + +import torch + +from tensorrt_llm._utils import nvtx_range, prefer_pinned + +from ..llm_request import LlmRequest +from .ops.vanilla import Fusions +from .sampler_common import _get_max_beam_width, _unwrap_singleton + +__all__ = ["PenaltyHandler", "PenaltyStore"] + + +def _has_occurrence_penalty(request: LlmRequest) -> bool: + sampling_config = request.sampling_config + repetition = _unwrap_singleton(sampling_config.repetition_penalty) + presence = _unwrap_singleton(sampling_config.presence_penalty) + frequency = _unwrap_singleton(sampling_config.frequency_penalty) + return ( + (repetition is not None and repetition != 1.0) + or (presence is not None and presence != 0.0) + or (frequency is not None and frequency != 0.0) + ) + + +@dataclass(kw_only=True) +class PenaltyStore: + """Persistent device state: penalty-parameter buffers + occurrence workspace. + + This is the torch counterpart of the tensors ``PenaltyLayer`` allocates, and + the anchor for the workspace semantics the ops and the handler rely on: + + * The **parameter buffers** (``repetition_cuda`` / ``presence_cuda`` / + ``frequency_cuda``, plus the ``active_cuda`` gate) are the counterpart of + ``allocateBuffer`` + ``fillBuffers``: one entry per sequence slot, written + once per request and gathered every step, never rebuilt on the host. + * The **occurrence workspace** (``counts_cuda`` and ``presence_prefix_cuda``) + is the counterpart of ``allocateWorkspace`` / ``mPenaltyWorkspaceDevice``, + updated incrementally each step. A token in the ignored prompt prefix + ``[0, prompt_ignore_length)`` only sets ``presence_prefix_cuda``, so it + contributes to the repetition penalty but not to presence/frequency; every + other token (the rest of the prompt plus each generated token) increments + ``counts_cuda``, which drives presence/frequency and -- via ``counts > 0`` -- + repetition as well. + """ + + max_num_sequences: int + device: torch.device + + # --- Penalty parameters (allocateBuffer counterpart), shape [max_num_sequences] --- + repetition_cuda: torch.Tensor + """float32; per-slot repetition penalty (default 1.0).""" + presence_cuda: torch.Tensor + """float32; per-slot presence penalty (default 0.0).""" + frequency_cuda: torch.Tensor + """float32; per-slot frequency penalty (default 0.0).""" + active_cuda: torch.Tensor + """bool[slots]; whether a slot has an active occurrence penalty.""" + has_previous_token_cuda: torch.Tensor + """bool[slots]; whether ``new_tokens`` contains a token to accumulate.""" + + # --- Occurrence workspace (allocateWorkspace counterpart), allocated lazily --- + counts_cuda: torch.Tensor | None = None + """int32[slots, vocab_size] or None; occurrence counts (see class docstring).""" + presence_prefix_cuda: torch.Tensor | None = None + """bool[slots, vocab_size] or None; ignored-prompt-prefix presence mask.""" + + # Per-step request metadata, staged into persistent device buffers by + # ``stage_request_metadata`` so the hot path does not allocate per step. + request_offsets_cuda: torch.Tensor | None = None + request_num_steps_cuda: torch.Tensor | None = None + + @classmethod + def create(cls, *, max_num_sequences: int, device: torch.device) -> "PenaltyStore": + """Allocate the vocab-independent buffers with their no-op defaults. + + ``inference_mode(False)`` guards every allocation in this class: the + buffers persist across sampler steps and are mutated in place later, which + inference-mode tensors forbid. + """ + with torch.inference_mode(False): + return cls( + max_num_sequences=max_num_sequences, + device=device, + repetition_cuda=torch.ones(max_num_sequences, dtype=torch.float32, device=device), + presence_cuda=torch.zeros(max_num_sequences, dtype=torch.float32, device=device), + frequency_cuda=torch.zeros(max_num_sequences, dtype=torch.float32, device=device), + active_cuda=torch.zeros(max_num_sequences, dtype=torch.bool, device=device), + has_previous_token_cuda=torch.zeros( + max_num_sequences, dtype=torch.bool, device=device + ), + ) + + def ensure_workspace(self, *, vocab_size: int, needs_prefix: bool) -> None: + """Allocate the vocab-sized workspace on first use. + + Deferred because ``vocab_size`` is only known once logits arrive, mirroring + ``PenaltyLayer::allocateWorkspace`` being gated on penalty usage. The prefix + mask is allocated only if some request has used ``prompt_ignore_length``. + """ + with torch.inference_mode(False): + if self.counts_cuda is None: + self.counts_cuda = torch.zeros( + (self.max_num_sequences, vocab_size), + dtype=torch.int32, + device=self.device, + ) + if needs_prefix and self.presence_prefix_cuda is None: + self.presence_prefix_cuda = torch.zeros( + (self.max_num_sequences, vocab_size), + dtype=torch.bool, + device=self.device, + ) + + def stage_request_metadata( + self, request_offsets_host: torch.Tensor, request_num_steps_host: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """Copy this step's ``[R]`` request metadata into persistent device buffers. + + The host tensors are already pinned by the caller, so each step costs two + small async H2D copies into a reused allocation rather than two fresh + device tensors. Returned views are only valid until the next call. + """ + num_requests = request_offsets_host.numel() + with torch.inference_mode(False): + if ( + self.request_offsets_cuda is None + or self.request_offsets_cuda.numel() < num_requests + ): + capacity = max(num_requests, self.max_num_sequences) + self.request_offsets_cuda = torch.empty( + capacity, dtype=request_offsets_host.dtype, device=self.device + ) + self.request_num_steps_cuda = torch.empty( + capacity, dtype=request_num_steps_host.dtype, device=self.device + ) + assert self.request_num_steps_cuda is not None + offsets = self.request_offsets_cuda[:num_requests] + num_steps = self.request_num_steps_cuda[:num_requests] + offsets.copy_(request_offsets_host, non_blocking=True) + num_steps.copy_(request_num_steps_host, non_blocking=True) + return offsets, num_steps + + +class PenaltyHandler: + """Applies the occurrence penalties: repetition, presence and frequency. + + These rescale or subtract from a token's logit based on how often it has already + occurred, and run before the sampling strategy divides by temperature. Bans that + force a logit to -inf (min_length, bad words, no-repeat-ngram) are a different + kind of transform and live in ``TokenBanHandler``. + + The implementation follows the C++ ``batchApplyPenalty`` kernel + (``cpp/tensorrt_llm/kernels/penaltyKernels.cu``) as driven by ``PenaltyLayer``. + Its persistent device state lives in :class:`PenaltyStore`, which documents the + workspace semantics. Per-slot parameter buffers are filled once per request, + batched across all requests admitted in a step (``prepare_for_new_request`` + accumulates on the host, ``update_for_new_requests`` issues the device updates). + Vocab-sized workspaces are allocated lazily and skipped entirely when no matching + request uses an occurrence penalty. + """ + + @dataclass(kw_only=True) + class _SlotState: + """Per-slot host-only bookkeeping (never read by the ops).""" + + prompt_ignore_length: int + initialized: bool = False + + def __init__( + self, + *, + max_num_sequences: int, + device: torch.device | str, + ): + self._max_num_sequences = max_num_sequences + self._device = torch.device(device) + # Whether any (past or current) active request uses prompt_ignore_length > 0, + # which requires allocating the presence-prefix mask. + self._needs_prefix = False + self._num_active_slots = 0 + # Per-slot state; None marks a slot without active occurrence penalties. + self._slots: list[PenaltyHandler._SlotState | None] = [None] * max_num_sequences + # Slots admitted this step that carry an occurrence penalty, with their + # parameters; drained by ``update_for_new_requests``. + self._new_slots: list[int] = [] + self._new_repetition: list[float] = [] + self._new_presence: list[float] = [] + self._new_frequency: list[float] = [] + self.store = PenaltyStore.create(max_num_sequences=max_num_sequences, device=self._device) + + @staticmethod + def validate_request(request: LlmRequest) -> None: + """Reject unsupported combinations for a penalized request. + + Called from ``TorchSampler.validate_request`` (request admission), so a + violating request is failed individually instead of aborting the whole batch. + """ + if _get_max_beam_width(request) > 1 and _has_occurrence_penalty(request): + raise ValueError( + "TorchSampler does not support repetition, presence, or frequency " + "penalties with beam search." + ) + + def _to_device(self, values: list[int], dtype: torch.dtype) -> torch.Tensor: + return torch.tensor(values, dtype=dtype, pin_memory=prefer_pinned()).to( + self._device, non_blocking=True + ) + + def prepare_for_new_request(self, request: LlmRequest, slot: int) -> None: + """Record the slot's penalty parameters for this step's batched flush. + + Called from ``TorchSampler.setup_sampler_step`` for each new request, mirroring + ``PenaltyLayer::setup`` (``fillBuffers`` + per-``batchSlot`` ``setZero``). This + only touches host state; ``update_for_new_requests`` issues the device updates + for all requests admitted in the step at once. Inactive slots are never + gathered, so their stale parameters/counts are left untouched. + """ + was_active = self._slots[slot] is not None + if not (_get_max_beam_width(request) == 1 and _has_occurrence_penalty(request)): + self._slots[slot] = None + if was_active: + self._num_active_slots -= 1 + return + + sampling_config = request.sampling_config + repetition = _unwrap_singleton(sampling_config.repetition_penalty) + presence = _unwrap_singleton(sampling_config.presence_penalty) + frequency = _unwrap_singleton(sampling_config.frequency_penalty) + prompt_ignore_length = _unwrap_singleton(sampling_config.prompt_ignore_length) + # min(prompt_ignore_length, inputLen), matching the C++ kernel. + prompt_ignore_length = min( + prompt_ignore_length if prompt_ignore_length is not None else 0, + request.py_orig_prompt_len, + ) + if prompt_ignore_length > 0: + self._needs_prefix = True + + self._slots[slot] = self._SlotState(prompt_ignore_length=prompt_ignore_length) + if not was_active: + self._num_active_slots += 1 + + self._new_slots.append(slot) + self._new_repetition.append(repetition if repetition is not None else 1.0) + self._new_presence.append(presence if presence is not None else 0.0) + self._new_frequency.append(frequency if frequency is not None else 0.0) + + def update_for_new_requests(self, *, new_seq_slots_cuda_long: torch.Tensor) -> None: + """Flush this step's admissions to the device in a handful of batched updates. + + ``new_seq_slots_cuda_long`` holds *every* slot admitted this step. Clearing the + active gate and the pending-token flag across all of them also covers slot + reuse: a slot whose prior occupant was penalized but whose new occupant is not + must read False. + """ + store = self.store + store.active_cuda.index_fill_(0, new_seq_slots_cuda_long, False) + store.has_previous_token_cuda.index_fill_(0, new_seq_slots_cuda_long, False) + + if not self._new_slots: + return + + slots_cuda = self._to_device(self._new_slots, torch.int64) + # One [3, N] host tensor -> one H2D for all three parameter buffers. + params_cuda = torch.tensor( + [self._new_repetition, self._new_presence, self._new_frequency], + dtype=torch.float32, + pin_memory=prefer_pinned(), + ).to(self._device, non_blocking=True) + store.repetition_cuda.index_copy_(0, slots_cuda, params_cuda[0]) + store.presence_cuda.index_copy_(0, slots_cuda, params_cuda[1]) + store.frequency_cuda.index_copy_(0, slots_cuda, params_cuda[2]) + store.active_cuda.index_fill_(0, slots_cuda, True) + + # Re-zero the workspace rows so a prior occupant's counts do not leak in. + if store.counts_cuda is not None: + store.counts_cuda.index_fill_(0, slots_cuda, 0) + if store.presence_prefix_cuda is not None: + store.presence_prefix_cuda.index_fill_(0, slots_cuda, False) + + self._new_slots.clear() + self._new_repetition.clear() + self._new_presence.clear() + self._new_frequency.clear() + + def _initialize_workspace( + self, + request: LlmRequest, + state: "PenaltyHandler._SlotState", + vocab_size: int, + ) -> None: + """Initialize one regular slot from its prompt exactly once.""" + if state.initialized: + return + + slot = request.py_seq_slot + assert slot is not None + counts_cuda = self.store.counts_cuda + assert counts_cuda is not None + + prompt = request.get_tokens(0)[: request.py_orig_prompt_len] + state.initialized = True + if not prompt: + return + + # One conversion for the whole prompt; the split point is just + # prompt_ignore_length, so the two groups are plain slices. + tokens = self._to_device(prompt, torch.int64) + prefix_tokens = tokens[: state.prompt_ignore_length] + counted_tokens = tokens[state.prompt_ignore_length :] + + # Multimodal models place placeholder ids >= vocab_size in the prompt (see + # _torch/models/modeling_multimodal_utils.py), so out-of-range ids reach us + # here and must be dropped before they index the workspace. + counted_tokens = counted_tokens[(counted_tokens >= 0) & (counted_tokens < vocab_size)] + prefix_tokens = prefix_tokens[(prefix_tokens >= 0) & (prefix_tokens < vocab_size)] + + Fusions.update_occurrence_workspace( + counts_cuda, + self.store.presence_prefix_cuda, + torch.full_like(counted_tokens, slot), + counted_tokens, + torch.full_like(prefix_tokens, slot), + prefix_tokens, + ) + + def update_token_counts( + self, + updates: list[tuple[int, list[int]]], + ) -> None: + """Commit finalized sampled tokens that replaced the device pending token. + + This is used after sampler-side postprocessing has finalized a multi-token + result. The complete confirmed sequence is counted here, then the raw first + token left in ``new_tokens`` is marked consumed so the next kernel cannot count + it again. Regular one-token sampling never calls this method and keeps its + fused device-pending fast path. + """ + if not updates or self._num_active_slots == 0: + return + + counts_cuda = self.store.counts_cuda + assert counts_cuda is not None + vocab_size = counts_cuda.size(-1) + consumed_slots: list[int] = [] + counted_slots: list[int] = [] + counted_tokens: list[int] = [] + + for slot, tokens in updates: + if self._slots[slot] is None: + continue + consumed_slots.append(slot) + for token in tokens: + if 0 <= token < vocab_size: + counted_slots.append(slot) + counted_tokens.append(token) + + if consumed_slots: + self.store.has_previous_token_cuda.index_fill_( + 0, self._to_device(consumed_slots, torch.int64), False + ) + + if not counted_tokens: + return + + Fusions.update_occurrence_workspace( + counts_cuda, + self.store.presence_prefix_cuda, + self._to_device(counted_slots, torch.int64), + self._to_device(counted_tokens, torch.int64), + ) + + @nvtx_range("apply_penalties") + @torch.inference_mode() + def apply( + self, + logits: torch.Tensor, + requests: list[LlmRequest], + *, + new_tokens: torch.Tensor, + seq_slots: torch.Tensor, + request_offsets: torch.Tensor, + request_num_steps: torch.Tensor, + is_draft_batch: bool = False, + ) -> None: + """Apply the occurrence penalties to ``logits`` in place. + + ``logits`` is the packed generated-token logits ``[sum(num_steps * num_beams), + vocab_size]``; request ``r`` owns ``request_num_steps[r]`` consecutive rows + starting at ``request_offsets[r]``, in beam-major / step-minor order. + ``request_offsets`` / ``request_num_steps`` are the caller's pinned host + tensors and are staged to the device here. + + Args: + is_draft_batch: draft batches share this sampler but draw ``py_seq_slot`` + from a separate numbering space that collides with target slots, so + penalizing them would read/write an unrelated target request's + occurrence state; skip them like the pending-steps tracking. + """ + if is_draft_batch or not requests or self._num_active_slots == 0: + return + + # Cheap per-batch scan so the vocab-sized workspace is only allocated when this + # batch actually contains a penalized request. + active_requests: list[tuple[LlmRequest, "PenaltyHandler._SlotState"]] = [] + for request in requests: + slot = request.py_seq_slot + assert slot is not None + state = self._slots[slot] + if state is not None: + active_requests.append((request, state)) + if not active_requests: + return + + store = self.store + store.ensure_workspace(vocab_size=logits.size(-1), needs_prefix=self._needs_prefix) + counts_cuda = store.counts_cuda + assert counts_cuda is not None + for request, state in active_requests: + self._initialize_workspace(request, state, logits.size(-1)) + + request_offsets_cuda, request_num_steps_cuda = store.stage_request_metadata( + request_offsets, request_num_steps + ) + Fusions.apply_batched_occurrence_penalties( + logits, + counts_cuda, + store.presence_prefix_cuda, + store.active_cuda, + store.has_previous_token_cuda, + new_tokens, + seq_slots, + request_offsets_cuda, + request_num_steps_cuda, + store.repetition_cuda, + store.presence_cuda, + store.frequency_cuda, + ) + # Arm has_previous_token for the slots this call penalized (active, num_steps > 0) + # so the next apply folds their sampled new_tokens. Done here rather than in the + # compiled op because the op's fold reads the flag for every request row; flipping + # it in the same graph would make the result depend on execution order within the + # kernel. + # + # The scan is kept on the host deliberately. The same thing can be expressed on + # device as active_cuda[seq_slots] & (num_steps > 0), avoiding this loop and the + # H2D, but that costs several extra kernel launches and measured 5-7us slower for + # batches up to 32 and no better at 64-256: the loop overlaps with the model + # forward, the launches do not. + pending_token_slots: list[int] = [] + for request, num_steps in zip(requests, request_num_steps.tolist()): + slot = request.py_seq_slot + if slot is None: + continue + if self._slots[slot] is not None and num_steps > 0: + pending_token_slots.append(slot) + if pending_token_slots: + store.has_previous_token_cuda.index_fill_( + 0, self._to_device(pending_token_slots, torch.int64), True + ) diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py index f034a18b2570..2e65851b4445 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py @@ -26,6 +26,7 @@ Any, Callable, Dict, + Final, Generic, List, Optional, @@ -70,7 +71,7 @@ DecoderState, GptDecoderBatched, ) -from tensorrt_llm.executor.result import Logprob, SimpleTokenLogprobs, TokenLogprobs +from tensorrt_llm.executor.result import Logprob from tensorrt_llm.llmapi.llm_args import KvCacheConfig from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping @@ -88,7 +89,26 @@ from ..llm_request import LlmRequest, LlmRequestState, get_draft_token_length from ..resource_manager import ResourceManager, ResourceManagerType from ..scheduler import ScheduledRequests -from .sampling_utils import ( +from .finish_reasons import FinishReasonsHandler +from .logprobs import ( + LogProbsState, + LogProbsStateList, + LogProbsStore, + convert_logprobs_tensor_to_list, + get_logprobs_from_request, + store_logprobs_list_to_request, +) +from .penalties import PenaltyHandler +from .sampler_common import ( + DEFAULT_BEAM_IDX, + DEFAULT_STEP_IDX, + FinishReasonsList, + _get_max_beam_width, + _request_get_sampling_params, + add_token, + int_tensor, +) +from .sampler_strategy import ( BEAM_SEARCH_PAD_TOKEN, GREEDY, BeamSearchMetadata, @@ -97,12 +117,14 @@ GenericStrategyKeyType, Strategy, StrategyMetadata, - UtilsSamplingParams, + TopPDecayMetadata, + _request_strategy, get_rejected_indices, - resolve_sampling_strategy, sample, sample_rejected, ) +from .token_ban import OverlappedTokenBanHandler, SynchronousTokenBanHandler, TokenBanHandler +from .top_p_decay import TopPDecayHandler if sys.version_info[:2] >= (3, 12): from typing import override @@ -120,38 +142,6 @@ T = TypeVar("T") -@dataclass(kw_only=True) -class LogProbsState: - sampled_vals: torch.Tensor - sampled_indices: torch.Tensor - sampled_rank: torch.Tensor - topk_vals: torch.Tensor - topk_indices: torch.Tensor - - -_LogProbsFloatState: TypeAlias = list[list[list[float]]] -_LogProbsIntState: TypeAlias = list[list[list[int]]] - - -@dataclass(kw_only=True) -class LogProbsStateList: - sampled_vals: _LogProbsFloatState - sampled_indices: _LogProbsIntState - sampled_rank: _LogProbsIntState - topk_vals: _LogProbsFloatState - topk_indices: _LogProbsIntState - - @staticmethod - def from_logprobs_state(logprobs_state: LogProbsState) -> "LogProbsStateList": - return LogProbsStateList( - sampled_vals=logprobs_state.sampled_vals.tolist(), - sampled_indices=logprobs_state.sampled_indices.tolist(), - topk_vals=logprobs_state.topk_vals.tolist(), - topk_indices=logprobs_state.topk_indices.tolist(), - sampled_rank=logprobs_state.sampled_rank.tolist(), - ) - - @dataclass(kw_only=True) class SampleStateTensors: new_tokens: torch.Tensor @@ -438,78 +428,6 @@ def is_generation_model(self) -> bool: return False -# Due to tensorrt_llm::runtime::SamplingConfig using vectors, params -# in LlmRequest.sampling_params are either None or single-element lists. -# This helper method simplifies code using such params. -def _unwrap_singleton(p: Optional[List[T]]) -> Optional[T]: - if p is None: - return None - (t,) = p - return t - - -def _get_beam_width_in(request: LlmRequest) -> int: - return ( - 1 - if request.is_context_init_state - else request.get_beam_width_by_iter(for_next_iteration=False) - ) - - -def _get_beam_width_out(request: LlmRequest) -> int: - return request.get_beam_width_by_iter(for_next_iteration=True) - - -def _get_max_beam_width(request: LlmRequest) -> int: - sampling_config = request.sampling_config - max_beam_width = cast(int, sampling_config.beam_width) - if sampling_config.beam_width_array is not None: - max_beam_width = max( - max_beam_width, - cast( - int, torch.tensor(sampling_config.beam_width_array, dtype=torch.int32).max().item() - ), - ) - return max_beam_width - - -def _request_get_sampling_params(request: LlmRequest) -> UtilsSamplingParams: - sampling_config = request.sampling_config - temperature = _unwrap_singleton(cast(Optional[list[float]], sampling_config.temperature)) - top_p = _unwrap_singleton(cast(Optional[list[float]], sampling_config.top_p)) - top_k = _unwrap_singleton(cast(Optional[list[int]], sampling_config.top_k)) - beam_width_out = _get_beam_width_out(request) - beam_width_in = _get_beam_width_in(request) - use_beam_search = _get_max_beam_width(request) > 1 - - return UtilsSamplingParams( - temperature=temperature, - top_p=top_p, - top_k=top_k, - beam_width_in=beam_width_in, - beam_width_out=beam_width_out, - use_beam_search=use_beam_search, - ) - - -def _request_sampling_params_cachable(params: UtilsSamplingParams) -> bool: - return not params.use_beam_search - - -def _request_strategy(request: LlmRequest, *, vocab_size: int) -> Strategy: - # We try to cache the resolved strategy on the request object, as it's not cheap enough to - # resolve it on every iteration. - cached_sampling_strategy = request.py_sampling_strategy - if cached_sampling_strategy is not None: - return cached_sampling_strategy - - params = _request_get_sampling_params(request) - sampling_strategy = resolve_sampling_strategy(params, vocab_size=vocab_size) - if _request_sampling_params_cachable(params): - request.py_sampling_strategy = sampling_strategy - return sampling_strategy - - class _CachingRequestGrouper(Generic[GenericStrategyKeyType]): """Efficiently groups requests for batched sampling.""" @@ -773,21 +691,6 @@ def _provision_gid() -> int: return result -def add_token( - request: LlmRequest, new_tokens: list[list[list[int]]], *, beam_idx: int, step: int = 0 -) -> int: - # NB: Accessing nested lists faster than torch.Tensor or numpy.ndarray - seq_slot = request.py_seq_slot - assert seq_slot is not None - new_token = new_tokens[step][seq_slot][beam_idx] - request.add_new_token(new_token, beam_idx) - return new_token - - -def int_tensor(shape: tuple[int, ...], device: str = "cuda") -> torch.Tensor: - return torch.empty(shape, dtype=torch.int, device=device) - - @dataclass(kw_only=True, frozen=True) class _BatchedSamplingResult: # Original request indices for all requests (permuted due to batching by strategy): @@ -1048,14 +951,6 @@ def __init__( raise ValueError(f"Invalid dim_order: {dim_order}") -# Beam index to use when no beam search is used but a beam index is required -DEFAULT_BEAM_IDX = 0 -# Step index to use when no speculative decoding is used but a step index is required -DEFAULT_STEP_IDX = 0 - -FinishReasonsList: TypeAlias = list[list[list[int]]] - - @dataclass(kw_only=True) class BeamHistory: """ @@ -1333,8 +1228,8 @@ def _record_sampler_event( class TorchSampler(Sampler[SampleStateTorch], AsyncWorkerMixin): - DEFAULT_MAX_STOP_WORD_LENGTH = 20 - DEFAULT_MAX_STOP_WORDS = 10 + DEFAULT_MAX_STOP_WORD_LENGTH: Final[int] = 20 + DEFAULT_MAX_STOP_WORDS: Final[int] = 10 SampleState = SampleStateTorch @@ -1348,853 +1243,6 @@ def get_cache_indirection(self) -> torch.Tensor | None: def is_generation_model(self) -> bool: return True - class FinishReasonsHandler: - _EMPTY_STOP_WORD_TOKEN_ID: int = -2 - _PAD_STOP_WORD_TOKEN_ID: int = -1 - - @dataclass(kw_only=True) - class _FinishReasonsStore: - """Auxiliary data structures used for finish reasons handling.""" - - # Per-request dynamic data - finish_reasons_cuda: torch.Tensor - """Shape: [max_tokens, batch_size, beam_width] - Usage: Stores the determined finish reasons for all sampled tokens - for each request. Some (draft) tokens and corresponding - finish reasons might still be discarded.""" - - # Per-request static data - max_lengths_cuda: torch.Tensor - """Shape: [batch_size] - Usage: Stores the maximum sequence lengths for each request""" - end_ids_cuda: torch.Tensor - """Shape: batch_size - Usage: Stores the end ids for each request""" - stop_words_cuda: torch.Tensor - """Shape: [max_num_stop_words, max_stop_word_length, batch_size] - Usage: Stores the stop words for each request as a padded tensor.""" - past_tokens_cuda: torch.Tensor - """Shape: [max_stop_word_length,batch_size, beam_width] - Usage: Stores the last max_stop_word_length tokens for each beam.""" - max_stop_word_lengths_host: torch.Tensor - """Shape: [batch_size] - Usage: Stores the size of the longest stop word for each request.""" - num_accepted_draft_tokens_host: torch.Tensor - """Shape: [batch_size] - Usage: Stores the number of accepted tokens for each request.""" - - def __init__( - self, - *, - max_stop_word_length: int, - max_num_stop_words: int, - max_num_sequences: int, - max_beam_width: int, - max_tokens: int, - max_seq_len: int, - ): - self._update_sizes( - max_stop_word_length=max_stop_word_length, - max_num_stop_words=max_num_stop_words, - max_num_sequences=max_num_sequences, - max_beam_width=max_beam_width, - max_tokens=max_tokens, - max_seq_len=max_seq_len, - ) - self._setup_store() - self._setup_helper_tensors() - self._temp_data: TorchSampler.FinishReasonsHandler._TemporaryData = ( - self._TemporaryData() - ) - - @property - def _use_speculative_decoding(self) -> bool: - return self._max_tokens > 1 - - @property - def new_max_lens(self) -> list[int]: - return self._temp_data.max_lens - - @property - def new_end_ids(self) -> list[int]: - return self._temp_data.end_ids - - def _update_sizes( - self, - *, - max_stop_word_length: int, - max_num_stop_words: int, - max_num_sequences: int, - max_beam_width: int, - max_tokens: int, - max_seq_len: int, - ) -> None: - """Updates the sizes of the finish reasons handler - - Sets member variables to store the current sizes. - These sizes are used to initialize the buffer tensors. - """ - self._max_stop_word_length: int = max_stop_word_length - self._max_num_stop_words: int = max_num_stop_words - self._max_num_sequences: int = max_num_sequences - self._max_beam_width: int = max_beam_width - self._max_tokens: int = max_tokens - self._max_seq_len: int = max_seq_len - self._stop_words_shape: tuple[int, int, int] = ( - self._max_num_stop_words, - self._max_stop_word_length, - self._max_num_sequences, - ) - self._past_tokens_shape: tuple[int, int, int] = ( - self._max_stop_word_length - 1 + self._max_tokens, - self._max_num_sequences, - self._max_beam_width, - ) - - def _setup_store(self) -> None: - """Sets up the store for the finish reasons handler by initializing all buffer tensors.""" - finish_reasons_cuda = int_tensor( - (self._max_tokens, self._max_num_sequences, self._max_beam_width) - ) - max_lengths_cuda = int_tensor((self._max_num_sequences,)) - end_ids_cuda = int_tensor((self._max_num_sequences,)) - stop_words_cuda = int_tensor(self._stop_words_shape) - past_tokens_cuda = int_tensor(self._past_tokens_shape) - max_stop_word_lengths_host = torch.empty( - self._max_num_sequences, device="cpu", dtype=torch.int32 - ) - num_accepted_draft_tokens_host = torch.empty( - self._max_num_sequences, device="cpu", dtype=torch.int32 - ) - self.store: TorchSampler.FinishReasonsHandler._FinishReasonsStore = ( - self._FinishReasonsStore( - finish_reasons_cuda=finish_reasons_cuda, - max_lengths_cuda=max_lengths_cuda, - end_ids_cuda=end_ids_cuda, - stop_words_cuda=stop_words_cuda, - past_tokens_cuda=past_tokens_cuda, - max_stop_word_lengths_host=max_stop_word_lengths_host, - num_accepted_draft_tokens_host=num_accepted_draft_tokens_host, - ) - ) - - def _setup_helper_tensors(self) -> None: - # Helper tensors for finish_reasons: - """Preallocate buffer needed for torch.nonzero_static(..., out=finish_reasons_nonzero_static_buffer). - See `def _write_reason`.""" - # setup local buffer for max tokens checking - self._max_tokens_offset_cuda: torch.Tensor = torch.arange( - 1, self._max_tokens + 1, device="cuda", dtype=torch.int32 - ).view(-1, 1, 1) - - self._stop_words_index_offset_cuda: torch.Tensor = torch.arange( - max(0, self._max_stop_word_length - 1), device="cuda" - ).unsqueeze(1) - - self._past_token_buffer_cuda: torch.Tensor = torch.empty( - self._past_tokens_shape, device="cuda", dtype=torch.int32 - ) - starts = torch.arange(self._max_tokens, device="cuda") - ends = starts + self._max_stop_word_length - self._multi_arange_indexing: torch.Tensor = torch_multi_arange( - ends=ends, - starts=starts, - output_length=self._max_tokens * self._max_stop_word_length, - ) - - def _resize_stop_word_buffers(self) -> None: - self._stop_words_index_offset_cuda = torch.arange( - max(0, self._max_stop_word_length - 1), device="cuda" - ).unsqueeze(1) - - self._past_tokens_shape = ( - self._max_stop_word_length - 1 + self._max_tokens, - self._max_num_sequences, - self._max_beam_width, - ) - self._stop_words_shape = ( - self._max_num_stop_words, - self._max_stop_word_length, - self._max_num_sequences, - ) - self._past_token_buffer_cuda = torch.empty( - self._past_tokens_shape, device="cuda", dtype=torch.int32 - ) - starts = torch.arange(self._max_tokens, device="cuda") - ends = starts + self._max_stop_word_length - self._multi_arange_indexing = torch_multi_arange( - ends=ends, - starts=starts, - output_length=self._max_tokens * self._max_stop_word_length, - ) - # resize the stop words buffer if necessary - # if the sizes are constant, this does nothing - store = self.store - _ = store.stop_words_cuda.resize_(self._stop_words_shape) - _ = store.past_tokens_cuda.resize_(self._past_tokens_shape) - - @dataclass(kw_only=True) - class _TemporaryData: - """Data structure to store the temporary data during setup_sampler_step for new requests""" - - def __init__(self) -> None: - # list of device tensors - self.stop_words_cuda_list: list[torch.Tensor] = [] - self.past_tokens_cuda_list: list[torch.Tensor] = [] - # list of integers - self.stop_word_seq_slots: list[int] = [] - self.max_lens: list[int] = [] - self.end_ids: list[int] = [] - self.max_stop_word_lengths: list[int] = [] - # integers - self.total_max_length: int = 0 - self.total_max_num_stop_words: int = 0 - - def clear(self) -> None: - self.stop_words_cuda_list = [] - self.past_tokens_cuda_list = [] - self.stop_word_seq_slots = [] - self.max_lens = [] - self.end_ids = [] - self.max_stop_word_lengths = [] - self.total_max_length = 0 - self.total_max_num_stop_words = 0 - - def setup_new_request_handling(self) -> None: - """Setup the new request handling for the finish reasons handler - - Clears the temporary data for the new request handling. - This should be called before processing new requests, to avoid - stale data from previous requests. - """ - self._temp_data.clear() - - def prepare_for_new_request(self, request: LlmRequest) -> None: - """Fill _temp_data with the corresponding data from new requests to be used during setup_sampler_step - - Args: - request: The request to prepare for. - """ - - self._temp_data.max_lens.append( - min(self._max_seq_len, request.orig_prompt_len + request.py_max_new_tokens) - ) - self._temp_data.end_ids.append( - end_id if (end_id := request.py_end_id) is not None else -1 - ) - - if (stop_words_list := request.py_stop_words_list) is not None: - assert (seq_slot := request.py_seq_slot) is not None - self._temp_data.stop_word_seq_slots.append(seq_slot) - extracted_stop_words_cuda, max_length, num_stop_words = self._extract_stop_words( - stop_words_list - ) - self._temp_data.stop_words_cuda_list.append(extracted_stop_words_cuda) - self._temp_data.past_tokens_cuda_list.append(self._get_past_tokens(request)) - self._temp_data.total_max_length = max(self._temp_data.total_max_length, max_length) - self._temp_data.total_max_num_stop_words = max( - self._temp_data.total_max_num_stop_words, num_stop_words - ) - self._temp_data.max_stop_word_lengths.append(max_length) - else: - # max stop word length is used to determine if a request has stop words - # explicitly set it to 0 here to avoid stale data from previous requests - self._temp_data.max_stop_word_lengths.append(0) - - def update_for_new_request( - self, - *, - seq_slots_cuda_long: torch.Tensor, - max_lengths_cuda: torch.Tensor, - end_ids_cuda: torch.Tensor, - seq_slots_host: torch.Tensor, - all_sampling_requests: list[LlmRequest], - ) -> None: - """Update tensors of this store with the new request data. - - If stop words are present, also update the stop words buffers. - If the new stop words exceed either the current max_num_stop_words or max_stop_word_length values, - a resize of the stop words buffers is triggered. If a resize is necessary, all requests in the batch - need to be re-processed. - - Args: - seq_slots_cuda_long: The sequence slots of the processed requests, as int64 - CUDA indices (required by ``index_copy_``). Shape: [len(requests)] - max_lengths_cuda: The maximum lengths for each request. - Shape: [len(requests)] - end_ids_cuda: The end ids for each request. - Shape: [len(requests)] - seq_slots_host: The sequence slots of the processed requests. Used for accessing host buffers. - Shape: [len(requests)] - all_sampling_requests: If a resize of the stop words related buffers is necessary, all sampling requests - need to be re-processed. - """ - - temp_data = self._temp_data - store = self.store - store.max_lengths_cuda.index_copy_(0, seq_slots_cuda_long, max_lengths_cuda) - store.end_ids_cuda.index_copy_(0, seq_slots_cuda_long, end_ids_cuda) - store.max_stop_word_lengths_host[seq_slots_host] = torch.tensor( - temp_data.max_stop_word_lengths, device="cpu", dtype=torch.int32 - ) - - # Handle stop words only if any new ones are added - if temp_data.stop_word_seq_slots: - self._update_stop_words_buffer( - all_sampling_requests, - temp_data.total_max_length, - temp_data.total_max_num_stop_words, - temp_data.stop_words_cuda_list, - temp_data.past_tokens_cuda_list, - temp_data.stop_word_seq_slots, - ) - - def _maybe_resize_stop_words_buffer( - self, total_max_length: int, total_max_num_stop_words: int - ) -> bool: - """Checks if the stop words buffer needs to be resized and resizes it if necessary - - If the total maximum length or number of stop words exceeds the current maximum values, - the stop words buffer is resized to the new maximum values. - - Args: - total_max_length: The maximum length of the stop words in this batch. - total_max_num_stop_words: The maximum number of stop words of a request in this batch. - Returns: - True if the stop words buffer needs to be resized, False otherwise. - """ - if ( - total_max_length > self._max_stop_word_length - or total_max_num_stop_words > self._max_num_stop_words - ): - self._max_stop_word_length = max(total_max_length, self._max_stop_word_length) - self._max_num_stop_words = max(total_max_num_stop_words, self._max_num_stop_words) - self._resize_stop_word_buffers() - return True - return False - - def _reprocess_stop_words_buffer( - self, requests: list[LlmRequest] - ) -> tuple[list[torch.Tensor], list[torch.Tensor], list[int]]: - """Reprocesses the stop words buffer with the new maximum values - - If a resize of the stop words buffer is necessary, all requests in the batch need to be re-processed. - - Args: - requests: The requests to reprocess the stop words buffer for. - Returns: - stop_words_cuda_list: A list of device tensors containing the stop words per request with stop words. - past_tokens_cuda_list: A list of device tensors containing the past tokens per request with stop words. - stop_word_seq_slots: A list of sequence slot indices (int) per request with stop words. - """ - stop_words_cuda_list: list[torch.Tensor] = [] - past_tokens_cuda_list: list[torch.Tensor] = [] - stop_word_seq_slots: list[int] = [] - # Rerun with the new size. Set the stop words and past tokens for all the requests. - for request in requests: - if (stop_words_list := request.py_stop_words_list) is not None: - extracted_stop_words_cuda, _, _ = self._extract_stop_words(stop_words_list) - assert (seq_slot := request.py_seq_slot) is not None - stop_word_seq_slots.append(seq_slot) - stop_words_cuda_list.append(extracted_stop_words_cuda) - past_tokens_cuda_list.append(self._get_past_tokens(request)) - return stop_words_cuda_list, past_tokens_cuda_list, stop_word_seq_slots - - def _update_stop_words_buffer( - self, - all_sampling_requests: list[LlmRequest], - total_max_length: int, - total_max_num_stop_words: int, - stop_words_cuda_list: list[torch.Tensor], - past_tokens_cuda_list: list[torch.Tensor], - stop_word_seq_slots: list[int], - ) -> None: - """Updates the stop words buffer with the new maximum values - - Args: - all_sampling_requests: If a resize of the stop words related buffers is necessary, all sampling requests - need to be re-processed. - total_max_length: The maximum length of the stop words in this batch. - total_max_num_stop_words: The maximum number of stop words of a request in this batch. - stop_words_cuda_list: A list of device tensors containing the stop words per request with stop words. - past_tokens_cuda_list: A list of device tensors containing the past tokens per request with stop words. - stop_word_seq_slots: Sequence slot index (int) per request with stop words; - same order as the lists above. - """ - # Potentially resize the buffers and update - # stop_words, past_tokens and stop_word_seq_slots - # In case of a resize all requests in the batch need to be re-processed - if self._maybe_resize_stop_words_buffer(total_max_length, total_max_num_stop_words): - stop_words_cuda_list, past_tokens_cuda_list, stop_word_seq_slots = ( - self._reprocess_stop_words_buffer(all_sampling_requests) - ) - - # Host Tensor for host access of self.store.num_accepted_draft_tokens - stop_word_seq_slots_tensor_host = torch.tensor( - stop_word_seq_slots, device="cpu", dtype=torch.int32, pin_memory=prefer_pinned() - ) - # Device Tensor for device access of self.store.stop_words and self.store.past_tokens - stop_word_seq_slots_tensor_cuda = stop_word_seq_slots_tensor_host.to( - device="cuda", non_blocking=True - ) - # stop_word_seq_slots x max_num_stop_words x max_stop_word_length - stop_words_cuda_tensor = torch.stack(stop_words_cuda_list) - # stop_word_seq_slots x max_stop_word_length x beam_width - past_tokens_cuda_tensor = torch.stack(past_tokens_cuda_list) - - store = self.store - # Reset the accepted tokens buffer for the stop word sequence slots - store.num_accepted_draft_tokens_host[stop_word_seq_slots_tensor_host] = 0 - - store.stop_words_cuda[..., stop_word_seq_slots_tensor_cuda] = ( - stop_words_cuda_tensor.permute(1, 2, 0) - ) - # Past tokens will be shifted by 1 to the left on their first sampling iteration - # We need to consider this here. - store.past_tokens_cuda[ - 1 : self._max_stop_word_length, stop_word_seq_slots_tensor_cuda - ] = past_tokens_cuda_tensor.permute(1, 0, 2) - - def _extract_stop_words( - self, stop_words_list: list[list[int]] - ) -> tuple[torch.Tensor, int, int]: - """Extract the stop words and size information from the stop words list - - Processes the stop words list and stores the stop words in a padded device tensor. - Stop words shorter than FinishReasonsHandler.max_stop_word_length - are padded with _PAD_STOP_WORD_TOKEN_ID to max_stop_word_length. - Unused stop word slots are padded with _EMPTY_STOP_WORD_TOKEN_ID. - This function additionally returns the maximum stop word length and the number of stop words - in the processed stop words list. - - - Args: - stop_words_list: A list of two lists: the first contains the token ids of all stop sequences - (concatenated); the second contains the cumulative lengths (prefix sum) of the stop word lengths. - - Returns: - stop_words: A padded device tensor containing the stop words - Shape: [max_num_stop_words, max_stop_word_length] - max_stop_word_length: The maximum stop word length in the stop words list - num_stop_words: The number of stop words in the stop words list - """ - stop_words_host = torch.empty( - self._max_num_stop_words, - self._max_stop_word_length, - device="cpu", - dtype=torch.int32, - ) - _ = stop_words_host.fill_(self._EMPTY_STOP_WORD_TOKEN_ID) - words, cumulative_stop_word_lengths = stop_words_list - words_host = torch.tensor( - words, device="cpu", dtype=torch.int32, pin_memory=prefer_pinned() - ) - begin = 0 - max_stop_word_length = 0 - num_stop_words = 0 - for idx, end in enumerate(cumulative_stop_word_lengths): - if end == -1: - break - length = end - begin - max_stop_word_length = max(max_stop_word_length, length) - num_stop_words += 1 - # skip processing if either the length or the index is greater than the current max values. - # These will be updated outside this function. - if length > self._max_stop_word_length or idx >= self._max_num_stop_words: - begin = end - continue - stop_words_host[idx, -length:] = words_host[begin:end] - stop_words_host[idx, :-length] = self._PAD_STOP_WORD_TOKEN_ID - begin = end - return ( - stop_words_host.to("cuda", non_blocking=True), - max_stop_word_length, - num_stop_words, - ) - - def _get_past_tokens(self, request: LlmRequest) -> torch.Tensor: - """Get the past tokens from the request and return the past tokens device tensor - - Args: - request: The request to get the past tokens for - - Returns: - past_tokens: The past tokens device tensor - Shape: [max_stop_word_length - 1, max_beam_width] - """ - past_tokens_host = torch.zeros( - max(0, self._max_stop_word_length - 1), - self._max_beam_width, - device="cpu", - dtype=torch.int32, - pin_memory=prefer_pinned(), - ) - tokens = request.get_tokens() - for beam_idx in range(self._max_beam_width): - max_len = min(past_tokens_host.shape[0], len(tokens[beam_idx])) - past_tokens_host[past_tokens_host.shape[0] - max_len :, beam_idx] = torch.tensor( - tokens[beam_idx][len(tokens[beam_idx]) - max_len :], - device="cpu", - dtype=torch.int32, - ) - return past_tokens_host.to("cuda", non_blocking=True) - - def write_finish_reasons( - self, - seq_slots_host: torch.Tensor, - is_draft_batch: bool, - seq_slots_cuda: torch.Tensor, - seq_lens_cuda: torch.Tensor, - new_tokens_cuda: torch.Tensor, - first_finish_reasons_cuda: torch.Tensor | None = None, - ) -> torch.Tensor: - """Calculates the finish reasons for each request and returns the finish reasons tensor. - - Prepares stop word handling for each requests and processes all newly generated tokens - per request to determine if any finish reason is met. Returns the device finish reasons - tensor from the store, which is updated with the calculated finish reason for each newly - generated token. - - Args: - seq_slots_host: The sequence slots of the processed requests. Used to determine which - requests need stop word processing on the host. - Shape: [len(requests)] - is_draft_batch: Whether the batch consists of draft requests. - seq_slots_cuda: The sequence slots of the processed requests. Used for accessing device buffers. - Shape: [len(requests)] - seq_lens_cuda: The sequence lengths of the processed requests. - Shape: [len(requests)] - new_tokens_cuda: A buffer containing the newly generated tokens. - Shape: [max_tokens, max_batch_size, max_beam_width] - first_finish_reasons_cuda: The first finish reason of each beam. Used only for beam search. - Shape: [max_batch_size, max_beam_width] - Returns: - finish_reasons_cuda: The finish reasons tensor. - Shape: [max_tokens, max_batch_size, max_beam_width] - """ - num_accepted_tokens_cuda, stop_word_indices_cuda, single_token_stop_words_only = ( - self._prepare_stop_word_handling_for_finish_reasons( - seq_slots_host, - is_draft_batch, - ) - ) - self._write_finish_reasons( - seq_slots=seq_slots_cuda, - seq_lens=seq_lens_cuda, - new_tokens=new_tokens_cuda, - num_accepted_tokens=num_accepted_tokens_cuda, - stop_word_indices=stop_word_indices_cuda, - single_token_stop_words_only=single_token_stop_words_only, - first_finish_reasons=first_finish_reasons_cuda, - ) - return self.store.finish_reasons_cuda - - def _prepare_stop_word_handling_for_finish_reasons( - self, - seq_slots_host: torch.Tensor, - is_draft_batch: bool, - ) -> tuple[torch.Tensor | int | None, torch.Tensor | None, bool]: - """Prepare stop word handling for finish reasons. - - Args: - seq_slots_host: The sequence slots of the processed requests. Used for accessing host buffers. - Shape: [len(requests)] - is_draft_batch: Whether the batch consists of draft requests. - Returns: - num_accepted_tokens_cuda: The number of accepted draft tokens +1 for each request. - Shape: [len(requests)] if torch.Tensor - stop_word_indices_cuda: The indices of the requests that have stop words in the current batch. - Shape: [len(requests_with_stop_words)] - single_token_stop_words_only: Whether all stop words in this batch are of length 1. - """ - # Filter all requests, that have stop words - store = self.store - num_accepted_tokens_cuda: torch.Tensor | int | None = None - stop_word_indices_cuda: torch.Tensor | None = None - single_token_stop_words_only: bool = False - - # NB: is_draft_batch is a workaround - # as draft requests can be in the sampler - # without having setup a slot in the FinishReasonsHandler. - # These can be removed once this can be avoided. - if is_draft_batch: - # Do not process stop words for draft requests - return ( - num_accepted_tokens_cuda, - stop_word_indices_cuda, - single_token_stop_words_only, - ) - - stop_word_mask = store.max_stop_word_lengths_host[seq_slots_host] > 0 - batch_has_stop_words = stop_word_mask.any() - - if batch_has_stop_words: - num_accepted_tokens_cuda = 1 - # Only calculate num_accepted_tokens from the accepted draft tokens if speculative decoding is enabled - if self._use_speculative_decoding: - num_accepted_tokens_cuda = ( - store.num_accepted_draft_tokens_host[seq_slots_host].to( - device="cuda", non_blocking=True - ) - + 1 - ) - stop_word_indices_cuda = torch.nonzero(stop_word_mask)[:, 0].to( - device="cuda", non_blocking=True - ) - single_token_stop_words_only = ( - store.max_stop_word_lengths_host[seq_slots_host].max().item() == 1 - ) - return num_accepted_tokens_cuda, stop_word_indices_cuda, single_token_stop_words_only - - @nvtx_range("_write_finish_reasons") - def _write_finish_reasons( - self, - *, - seq_slots: torch.Tensor, - seq_lens: torch.Tensor, - new_tokens: torch.Tensor, - num_accepted_tokens: torch.Tensor | int | None = None, - stop_word_indices: torch.Tensor | None = None, - single_token_stop_words_only: bool = False, - first_finish_reasons: torch.Tensor | None = None, - ) -> None: - """Writes the finish reasons to the finish_reasons tensor. - - The finish reasons are written to the finish_reasons tensor in the following order: - - Stop words - - Max length - - End ID - Later finish reasons overwrite earlier ones, in reverse precedence order. - - Args: - seq_slots: The sequence slots of the processed requests. Used for accessing device buffers. - Shape: [len(requests)] - seq_lens: The sequence lengths of the processed requests. - Shape: [len(requests)] - new_tokens: A buffer containing the newly generated tokens. - Shape: [max_tokens, max_batch_size, max_beam_width] - num_accepted_tokens: A buffer containing the number of accepted draft tokens +1 for each request. - Shape: [max_batch_size] if torch.Tensor - stop_word_indices: The indices of the requests that have stop words in the current batch. - Shape: [len(requests_with_stop_words)] - single_token_stop_words_only: Whether all stop words in this batch are of length 1 - first_finish_reasons: The first finish reason of each beam. - Shape: [max_batch_size, max_beam_width] - """ - - # Seq Slots should be on the same device as new_tokens - assert seq_slots.device == new_tokens.device - assert seq_lens.device == new_tokens.device - tokens = new_tokens[:, seq_slots] - - store = self.store - finish_reasons = store.finish_reasons_cuda - - # we need to fill with NOT_FINISHED so we can differentiate between - # previous requests that had the same seq slot - _ = finish_reasons.index_fill_(1, seq_slots, FinishReason.NOT_FINISHED.value) - batched_finish_reasons = finish_reasons[:, seq_slots] - - if stop_word_indices is not None: - assert num_accepted_tokens is not None, "draft_lengths is required for stop words" - stop_seq_slots = seq_slots[stop_word_indices] - stop_tokens = new_tokens[:, stop_seq_slots] - stop_words_func = ( - self._are_stop_words - if not single_token_stop_words_only - else self._are_stop_words_single_token - ) - batched_finish_reasons_stop_words = batched_finish_reasons[:, stop_word_indices] - _ = batched_finish_reasons_stop_words.masked_fill_( - stop_words_func( - stop_seq_slots, - stop_tokens, - num_accepted_tokens[stop_word_indices] - if isinstance(num_accepted_tokens, torch.Tensor) - else num_accepted_tokens, - ), - FinishReason.STOP_WORDS.value, - ) - batched_finish_reasons[:, stop_word_indices] = batched_finish_reasons_stop_words - - _ = batched_finish_reasons.masked_fill_( - self._are_max_length(seq_lens, store.max_lengths_cuda[seq_slots]), - FinishReason.LENGTH.value, - ) - - _ = batched_finish_reasons.masked_fill_( - self._are_end_id(store.end_ids_cuda[seq_slots], tokens), - FinishReason.END_ID.value, - ) - - finish_reasons[:, seq_slots] = batched_finish_reasons - if first_finish_reasons is not None: - # store the first stop reason for each beam of a seq_slot. - batched_first_finish_reasons = first_finish_reasons[seq_slots] - first_finish_reasons[seq_slots, ...] = torch.where( - batched_first_finish_reasons == FinishReason.NOT_FINISHED.value, - batched_finish_reasons, - batched_first_finish_reasons, - ) - - def _are_end_id( - self, end_ids_cuda: torch.Tensor, tokens_cuda: torch.Tensor - ) -> torch.Tensor: - """Checks if the tokens are the end id - - Args: - end_ids_cuda: The end ids of the requests to check the end id of. - Shape: [len(requests)] - tokens_cuda: A buffer containing the newly generated tokens. - Shape: [max_tokens, len(requests), max_beam_width] - Returns: - A tensor where each element is True if the corresponding token is the end id, False otherwise - Shape: [max_tokens, len(requests), max_beam_width] - """ - return tokens_cuda == end_ids_cuda.view(1, -1, 1).expand( - self._max_tokens, -1, self._max_beam_width - ) - - def _are_max_length( - self, seq_lens_cuda: torch.Tensor, max_seq_lens_cuda: torch.Tensor - ) -> torch.Tensor: - """Checks which sequences are at or beyond the max length - - Args: - seq_lens_cuda: The sequence lengths of the requests to check the max length of. - Shape: [len(requests)] - max_seq_lens_cuda: The maximum sequence lengths of the requests to check the max length of. - Shape: [len(requests)] - Returns: - A tensor where each element is True if the sequence at the corresponding token - is at or beyond the max length, False otherwise - Shape: [max_tokens, len(requests), max_beam_width] - """ - lengths_tensor_cuda = ( - seq_lens_cuda.view(1, -1, 1) + self._max_tokens_offset_cuda - ).expand(self._max_tokens, -1, self._max_beam_width) - max_lengths_tensor_cuda = max_seq_lens_cuda.view(1, -1, 1).expand( - self._max_tokens, -1, self._max_beam_width - ) - return lengths_tensor_cuda >= max_lengths_tensor_cuda - - @nvtx_range("_are_stop_words") - def _are_stop_words( - self, - seq_slots: torch.Tensor, - tokens: torch.Tensor, - num_accepted_tokens: torch.Tensor | int, - ) -> torch.Tensor: - """Checks if the tokens are stop words - - Args: - seq_slots: The sequence slots of the processed requests. Used for accessing device buffers. - Shape: [len(requests)] - tokens: A buffer containing the newly generated tokens. - Shape: [max_tokens, len(requests), max_beam_width] - num_accepted_tokens: The number of accepted draft tokens +1 for each request. - Shape: [len(requests)] if torch.Tensor - Returns: - A tensor where each element is True if the sequence at the corresponding token - ends with a stop word, False otherwise - Shape: [max_tokens, len(requests), max_beam_width] - """ - store = self.store - # num_words, len_words, batch_size - # unsqueeze the beam_width dimension to match the past tokens tensor - stop_words = ( - store.stop_words_cuda[..., seq_slots] - .unsqueeze(3) - .expand(-1, -1, -1, self._max_beam_width) - ) - # Get the past tokens - # num_steps, batch_size, beam_width - past_tokens_batch = store.past_tokens_cuda[:, seq_slots] - # Shift the past tokens to the left by the number of accepted draft tokens - - full_tokens = self._past_token_buffer_cuda[:, : seq_slots.shape[0]] - - index_tensor = ( - (self._stop_words_index_offset_cuda + num_accepted_tokens) - .unsqueeze(2) - .expand(-1, seq_slots.shape[0], self._max_beam_width) - ) - _ = torch.gather( - past_tokens_batch, - dim=0, - index=index_tensor, - out=full_tokens[: index_tensor.shape[0]], - ) - # Fill in the new tokens at the end of the past tokens buffer - full_tokens[-self._max_tokens :] = tokens - # short words are padded with _PAD_STOP_WORD_TOKEN_ID, so we need to mask them - mask = stop_words == self._PAD_STOP_WORD_TOKEN_ID - matches = torch.empty( - ( - self._max_tokens, - stop_words.shape[0], - stop_words.shape[1], - stop_words.shape[2], - stop_words.shape[3], - ), - device="cuda", - dtype=torch.bool, - ) - - # Get the comparison sequence for each step - full_tokens_for_match = full_tokens[self._multi_arange_indexing].view( - self._max_tokens, - 1, # Unsqueeze on dimension 1 to match the num_stop_words dimension of stop words - self._max_stop_word_length, - seq_slots.shape[0], - self._max_beam_width, - ) - # Unsqueeze on dimension 0 to match the max_tokens dimension of full tokens - stop_words_for_match = stop_words.unsqueeze(0) - _ = torch.eq(full_tokens_for_match, stop_words_for_match, out=matches) - # Mask the padding tokens - _ = matches.masked_fill_( - mask.unsqueeze(0).expand(self._max_tokens, -1, -1, -1, -1), True - ) - # Update the past tokens storage for the next iteration - store.past_tokens_cuda[:, seq_slots] = full_tokens - # Return the result - word_len_dim = 2 - num_words_dim = 1 - return torch.any(matches.all(dim=word_len_dim), dim=num_words_dim) - - @nvtx_range("_are_stop_words_single_token") - def _are_stop_words_single_token( - self, - seq_slots: torch.Tensor, - tokens: torch.Tensor, - _num_accepted_tokens: torch.Tensor | int, - ) -> torch.Tensor: - """Checks if the tokens are stop words (single token per stop word only) - - Args: - seq_slots: The sequence slots of the processed requests. Used for accessing device buffers. - Shape: [len(requests)] - tokens: A buffer containing the newly generated tokens. - Shape: [max_tokens, len(requests), max_beam_width] - _num_accepted_tokens: Unused - Returns: - A tensor where each element is True if the sequence at the corresponding token - ends with a stop word, False otherwise - Shape: [max_tokens, len(requests), max_beam_width] - """ - per_step = torch.zeros( - (self._max_tokens, seq_slots.shape[0], self._max_beam_width), - dtype=torch.bool, - device="cuda", - ) - # num_words, 1, batch_size - stop_words = self.store.stop_words_cuda[:, -1:, seq_slots].unsqueeze(3) - full_tokens = tokens.unsqueeze(0) - matches = full_tokens == stop_words - _ = torch.any(matches, dim=0, out=per_step) - return per_step - @dataclass(kw_only=True) class BeamSearchStore: """Auxiliary data structures required for beam search.""" @@ -2227,26 +1275,6 @@ class BeamSearchStore: Usage: Cached `arange(max_beam_width)` used as the scatter source in the per-step ``cache_indirection.scatter_``.""" - @dataclass(kw_only=True) - class LogProbsStore: - """Auxiliary data structures used for log-probs handling.""" - - sampled_log_prob_indices: torch.Tensor - """Shape: batch_size, beam_width, max_tokens - Usage: Stores the token indices of the sampled logprobs""" - sampled_log_probs: torch.Tensor - """Shape: batch_size, beam_width, max_tokens - Usage: Stores the values of the sampled logprobs""" - sampled_log_prob_ranks: torch.Tensor - """Shape: batch_size, beam_width, max_tokens - Usage: Stores the ranks of the sampled logprobs""" - topk_indices: torch.Tensor - """Shape: batch_size, max_tokens, max_topk_logprobs - Usage: Stores the token indices of the topk logprobs""" - topk_vals: torch.Tensor - """Shape: batch_size, max_tokens, max_topk_logprobs - Usage: Stores the values of the topk logprobs""" - @dataclass(kw_only=True) class Store: new_tokens: torch.Tensor @@ -2256,7 +1284,7 @@ class Store: """ beam_search_store: "TorchSampler.BeamSearchStore | None" = None """Holds data related to beam search.""" - log_probs_store: "TorchSampler.LogProbsStore" + log_probs_store: LogProbsStore """Holds data related to log-probs handling.""" def _create_store(self) -> Store: @@ -2273,7 +1301,7 @@ def _create_store(self) -> Store: # These are 0 sized tensors, if topk-logprobs are not used topk_indices = torch.empty(self.TOPK_LOGPROBS_SHAPE, device="cuda", dtype=torch.int32) topk_vals = torch.empty(self.TOPK_LOGPROBS_SHAPE, device="cuda", dtype=torch.float32) - log_probs_store = self.LogProbsStore( + log_probs_store = LogProbsStore( sampled_log_prob_indices=sampled_log_prob_indices, sampled_log_probs=sampled_log_probs, sampled_log_prob_ranks=sampled_log_prob_ranks, @@ -2329,6 +1357,9 @@ def __init__(self, args: Args): self.max_seq_len = args.max_seq_len self.max_tokens = args.max_total_draft_tokens + 1 self.max_beam_width = args.max_beam_width + # Snapshot of `not self._use_beam_search` so the update_requests + # fast-path avoids a property call per iteration. + self._batch_fastpath_eligible: bool = self.max_beam_width == 1 # The current maximum number of topk logprobs which can be stored in the sampler's store self.max_topk_logprobs = MAX_TOP_LOGPROBS # The maximum number of topk logprobs for the current batch of requests @@ -2336,6 +1367,13 @@ def __init__(self, args: Args): if args.max_total_draft_tokens > 0 and args.max_beam_width > 1: raise ValueError("TorchSampler does not support beam search with speculative decoding") self.max_num_sequences = args.max_num_sequences + # With the overlap scheduler, sample_async for step i runs before + # update_requests for step i-1, so the host-side token lists lag the + # device state. Track, per seq slot, how many sampled steps have not + # been folded back into the request yet; bad-words handling uses this + # to decide whether the newest token must be read device-side. + self._track_pending_steps = not args.disable_overlap_scheduler + self._pending_steps = [0] * self.max_num_sequences self.NEW_TOKENS_SHAPE = (self.max_tokens, self.max_num_sequences, self.max_beam_width) self.CACHE_INDIRECTION_SHAPE = ( self.max_num_sequences, @@ -2354,6 +1392,18 @@ def __init__(self, args: Args): "in requirements.txt." ) self._grouped_sampler_cls = FlashInferGroupedStrategySampler + # Per-slot Top-P Decay runtime state (FlashInfer path). Allocated for all + # sampler instances; only decay-admitted slots are ever read. + self._top_p_decay = TopPDecayHandler(self.max_num_sequences) + + # Token-ban handling (bad words, no-repeat ngram). The overlap-aware + # variant is selected once here from whether the overlap scheduler is + # enabled; only it produces the conditional (stale-host) bans. + self._token_ban_handler: TokenBanHandler = ( + OverlappedTokenBanHandler() + if self._track_pending_steps + else SynchronousTokenBanHandler() + ) # AutoDeploy build creates the sampler in inference mode, # which would disallow in-place mutating of new_tokens. @@ -2363,7 +1413,7 @@ def __init__(self, args: Args): self._request_grouper: _CachingRequestGrouper[Any] = _CachingRequestGrouper( self.max_num_sequences ) - self._finish_reasons_handler = self.FinishReasonsHandler( + self._finish_reasons_handler = FinishReasonsHandler( max_stop_word_length=self.DEFAULT_MAX_STOP_WORD_LENGTH, max_num_stop_words=self.DEFAULT_MAX_STOP_WORDS, max_num_sequences=self.max_num_sequences, @@ -2375,6 +1425,10 @@ def __init__(self, args: Args): self.store.new_tokens.shape == self._finish_reasons_handler.store.finish_reasons_cuda.shape ) + self._penalty_handler = PenaltyHandler( + max_num_sequences=self.max_num_sequences, + device="cuda", + ) # Initialize seed for multi-GPU consistency self._global_seed = 42 @@ -2406,6 +1460,22 @@ def __init__(self, args: Args): None ] * self.max_num_sequences + @staticmethod + def _is_draft_batch(requests: list[LlmRequest]) -> bool: + """Whether this batch belongs to the draft model. + + Batches are homogeneous by construction: ModelDrafter builds all-draft + batches for its sample_async/update_requests calls on this shared + sampler, and PyExecutor's batches are all-target. The pending-steps + accounting relies on this to skip draft batches wholesale; assert it so + a mixed batch fails loudly instead of silently corrupting the counters. + """ + is_draft: bool = requests[0].py_is_draft + assert all(r.py_is_draft == is_draft for r in requests), ( + "sampler batch must be homogeneous (all-draft or all-target)" + ) + return is_draft + def get_generator(self, device: torch.device) -> torch.Generator: """Get a deterministic generator for the specified device. @@ -2587,125 +1657,6 @@ def _update_original_tokens( ) original_tokens[seq_slots, :, seq_lens] = new_tokens[0, seq_slots, :] - def _convert_logprobs_tensor_to_list( - self, - token_tensor: torch.Tensor, - logprobs_tensor: torch.Tensor, - ) -> list[list[dict[int, Logprob]]]: - """Convert the logprobs tensor to a list of lists of dictionaries of Logprob objects - - Logprobs storage expects logprobs as a list[list[dict[int, Logprob]]] object - - args: - token_tensor: torch.Tensor. Shape: beam_width, num_tokens, num_logprobs - logprobs_tensor: torch.Tensor. Shape: beam_width, num_tokens, num_logprobs - output: - list[list[dict[int, Logprob]]]. Shape: (beam_width, num_tokens) - """ - assert token_tensor.dim() == 3 and logprobs_tensor.dim() == 3, ( - f"Token and logprobs tensors must have 3 dimensions (beam_width, num_tokens, num_logprobs). \ - Got shapes (token_tensor) {token_tensor.shape} and (logprobs_tensor) {logprobs_tensor.shape} instead" - ) - - token_log_probs: list[list[dict[int, Logprob]]] = [] - token_list = token_tensor.tolist() - logprobs_list = logprobs_tensor.tolist() - for beam_idx in range(token_tensor.shape[0]): - beam_token_log_probs: list[dict[int, Logprob]] = [] - for topk_token, topk_logprob in zip(token_list[beam_idx], logprobs_list[beam_idx]): - logprobs = { - token: Logprob(logprob=logprob, rank=rank + 1) - for rank, (token, logprob) in enumerate(zip(topk_token, topk_logprob)) - } - beam_token_log_probs.append(logprobs) - token_log_probs.append(beam_token_log_probs) - - return token_log_probs - - def _store_logprobs_list_to_request( - self, - logprobs_state_list: LogProbsStateList, - req_seq_slot: int, - beam_width: int, - count: int, - num_topk_logprobs: int, - simple_format: bool = False, - ) -> list[list[dict[int, Logprob]]] | list[list[float]]: - """Convert the LogProbsStateList object to per-token logprobs. - - By default returns ``list[list[dict[int, Logprob]]]``. When - ``simple_format`` is True and ``num_topk_logprobs == 0`` the result is a - flat ``list[list[float]]`` (one logprob per generated token, per beam). - - args: - logprobs_state_list: LogProbsStateList. Contains the topk indices, topk values, - sampled indices, sampled values, and sampled ranks. - req_seq_slot: int. The sequence slot of the request. - beam_width: int. The beam width of the request. - count: int. The number of tokens to store. - num_topk_logprobs: int. The number of topk logprobs of each token. - simple_format: bool. If True (and num_topk_logprobs == 0), return - ``list[list[float]]`` instead of the dict format. Avoids per-token - dict allocation when only the sampled-token logprob is needed. - output: - list[list[dict[int, Logprob]]] (default) or list[list[float]] (simple format). - Shape: (beam_width, count) - """ - - sampled_log_probs_indices_list = logprobs_state_list.sampled_indices[req_seq_slot] - sampled_log_probs_vals_list = logprobs_state_list.sampled_vals[req_seq_slot] - sampled_log_probs_rank_list = logprobs_state_list.sampled_rank[req_seq_slot] - - if num_topk_logprobs == 0: - if simple_format: - token_log_probs_simple: list[list[float]] = [ - [sampled_log_probs_vals_list[beam_idx][step_idx] for step_idx in range(count)] - for beam_idx in range(beam_width) - ] - return token_log_probs_simple - - token_log_probs: list[list[dict[int, Logprob]]] = [ - [ - { - sampled_log_probs_indices_list[beam_idx][step_idx]: Logprob( - sampled_log_probs_vals_list[beam_idx][step_idx], - sampled_log_probs_rank_list[beam_idx][step_idx] + 1, - ) - } - for step_idx in range(count) - ] - for beam_idx in range(beam_width) - ] - else: - token_list = logprobs_state_list.topk_indices[req_seq_slot] - logprobs_list = logprobs_state_list.topk_vals[req_seq_slot] - token_log_probs = [[] for _ in range(beam_width)] - for step_idx in range(count): - topk_tokens = token_list[step_idx][:num_topk_logprobs] - topk_logprobs = logprobs_list[step_idx][:num_topk_logprobs] - min_rank = len(topk_tokens) + 1 - - topk_logprob_dict = { - token: Logprob(logprob=logprob, rank=rank + 1) - for rank, (token, logprob) in enumerate(zip(topk_tokens, topk_logprobs)) - } - - for beam_idx in range(beam_width): - # NB: Keeps sampled token in the first position (cf. https://stackoverflow.com/a/67786863) - logprobs = { - sampled_log_probs_indices_list[beam_idx][step_idx]: Logprob( - logprob=sampled_log_probs_vals_list[beam_idx][step_idx], - rank=max( - min_rank, - sampled_log_probs_rank_list[beam_idx][step_idx] + 1, - ), - ), - **topk_logprob_dict, - } - token_log_probs[beam_idx].append(logprobs) - - return token_log_probs - def handle_logprobs( self, request: LlmRequest, @@ -2718,7 +1669,7 @@ def handle_logprobs( assert request.py_num_logprobs is not None, "request.py_num_logprobs must be provided" assert logprobs_state_list is not None, "logprobs_state_list must be provided" assert request.py_seq_slot is not None - token_log_probs = self._store_logprobs_list_to_request( + token_log_probs = store_logprobs_list_to_request( logprobs_state_list, request.py_seq_slot, beam_width, @@ -2898,6 +1849,11 @@ def _collect_new_requests_for_setup( @override def validate_request(self, request: LlmRequest) -> None: + # Reject unsupported top-p-decay and penalty combinations at admission, so + # only the offending request fails (raising later, inside setup_sampler_step + # or sampling, would abort the whole executor step). + self._top_p_decay.validate_request(request) + self._penalty_handler.validate_request(request) if self._use_beam_search: if request.py_return_log_probs: if request.py_num_logprobs > 1: @@ -2946,6 +1902,7 @@ def setup_sampler_step(self, scheduled_requests: ScheduledRequests) -> None: self._prev_first_finish_reasons_host[slot] = None self._request_grouper.prepare_for_new_request(request, slot) + self._penalty_handler.prepare_for_new_request(request, slot) max_lens = self._finish_reasons_handler.new_max_lens end_ids = self._finish_reasons_handler.new_end_ids @@ -2972,6 +1929,14 @@ def setup_sampler_step(self, scheduled_requests: ScheduledRequests) -> None: all_sampling_requests=new_requests + scheduled_requests.generation_requests, ) + self._top_p_decay.setup_for_new_requests( + new_requests, new_seq_slots_cuda_long=seq_slots_tensor_cuda_long + ) + + self._penalty_handler.update_for_new_requests( + new_seq_slots_cuda_long=seq_slots_tensor_cuda_long + ) + if self._use_beam_search: beam_search_store = self.store.beam_search_store assert beam_search_store is not None @@ -3139,68 +2104,6 @@ def process_draft_tokens( request, new_tokens_list=new_tokens_list, new_tokens_tensor=new_tokens_tensor ) - def _get_logprobs_from_request( - self, - request: LlmRequest, - pin_memory: bool = True, - preallocate_extra_steps: int = 0, - ) -> tuple[torch.Tensor, torch.Tensor]: - """Extract the logprobs from the request. - - Returns: - logprobs_tensor: A tensor of shape (beam_width, num_generated_tokens, num_logprobs) - logprobs_indices_tensor: A tensor of shape (beam_width, num_generated_tokens, num_logprobs) - """ - pin_memory = pin_memory and prefer_pinned() - num_generated_tokens = request.max_beam_num_tokens - request.py_prompt_len - assert request.py_num_logprobs == 0, ( - "Beam search only supports returning the sampled logprob per token" - ) - logprobs_tensor_full = torch.empty( - ( - request.py_beam_width, - num_generated_tokens + preallocate_extra_steps, - request.py_num_logprobs + 1, - ), - pin_memory=pin_memory, - dtype=torch.float32, - ) - logprobs_indices_tensor_full = torch.empty( - ( - request.py_beam_width, - num_generated_tokens + preallocate_extra_steps, - request.py_num_logprobs + 1, - ), - pin_memory=pin_memory, - dtype=torch.int32, - ) - # NB: forward slicing, because [:, :-0, :] would yield an empty view - # instead of the full history when preallocate_extra_steps == 0. - logprobs_tensor = logprobs_tensor_full[:, :num_generated_tokens, :] - logprobs_indices_tensor = logprobs_indices_tensor_full[:, :num_generated_tokens, :] - if logprobs_tensor.numel() > 0: - logprobs_list = request.py_result.log_probs - assert logprobs_list is not None - - if request.py_logprobs_simple_format: - tokens = request.get_tokens() - for beam_idx, beam_logprobs in enumerate(logprobs_list): - beam_logprobs = cast(SimpleTokenLogprobs, beam_logprobs) - for token_idx, token_logprobs_simple in enumerate(beam_logprobs): - logprobs_tensor[beam_idx, token_idx, 0] = token_logprobs_simple - logprobs_indices_tensor[beam_idx, token_idx, 0] = tokens[beam_idx][ - token_idx - ] - else: - for beam_idx, beam_logprobs in enumerate(logprobs_list): - beam_logprobs = cast(TokenLogprobs, beam_logprobs) - for token_idx, token_logprobs in enumerate(beam_logprobs): - for key, value in token_logprobs.items(): - assert value.rank is not None - logprobs_tensor[beam_idx, token_idx, value.rank - 1] = value.logprob - logprobs_indices_tensor[beam_idx, token_idx, value.rank - 1] = key - return logprobs_tensor_full, logprobs_indices_tensor_full - def _prepare_beam_history( self, request: LlmRequest, @@ -3370,7 +2273,7 @@ def _postprocess_beam_logprobs( beam's history through `cache_indirection`. Returns the gathered (logprobs, logprobs_indices, cum_logprobs) triple. """ - current_logprobs, current_logprobs_indices = self._get_logprobs_from_request( + current_logprobs, current_logprobs_indices = get_logprobs_from_request( request, preallocate_extra_steps=1 ) # concatenate the newly generated logprobs and newly @@ -3441,7 +2344,7 @@ def _finalize_beam( assert beam_history.logprobs_indices is not None assert beam_history.logprobs is not None gen_log_probs_list.append( - self._convert_logprobs_tensor_to_list( + convert_logprobs_tensor_to_list( beam_history.logprobs_indices[beam_idx : beam_idx + 1, :beam_valid_tokens], beam_history.logprobs[beam_idx : beam_idx + 1, :beam_valid_tokens], )[0] @@ -3467,6 +2370,7 @@ def _add_metadata_to_grouped_requests( *, seq_slots_cuda: torch.Tensor, seq_lens_cuda: torch.Tensor, + req_num_steps: torch.Tensor, ) -> dict[RequestGroupKey[GenericStrategyKeyType], RequestGroupValueWithMetadata]: grouped_requests_with_metadata: dict[ RequestGroupKey[GenericStrategyKeyType], RequestGroupValueWithMetadata @@ -3476,6 +2380,7 @@ def _add_metadata_to_grouped_requests( num_requests = len(requests) for key, value in grouped_requests.items(): metadata_type = get_metadata_type_for_group_fn(key.strategy_key) + metadata: StrategyMetadata | None if metadata_type is BeamSearchMetadata: assert beam_search_store is not None assert seq_lens is not None, "seq_lens is required for beam search" @@ -3504,6 +2409,13 @@ def _add_metadata_to_grouped_requests( seq_offsets=beam_search_store.seq_offsets, beam_idx_arange=beam_search_store.beam_idx_arange, ) + elif metadata_type is TopPDecayMetadata: + metadata = self._top_p_decay.build_metadata( + group_req_indices=value.indices, + req_num_steps=req_num_steps, + seq_slots=seq_slots, + seq_slots_cuda=seq_slots_cuda, + ) elif metadata_type is None: metadata = None else: @@ -3624,6 +2536,12 @@ def update_requests( if not state.requests: return + if self._track_pending_steps and not self._is_draft_batch(state.requests): + for req in state.requests: + slot = req.py_seq_slot + if slot is not None and self._pending_steps[slot] > 0: + self._pending_steps[slot] -= 1 + assert state.host is not None new_tokens = state.host.new_tokens finish_reasons = state.host.finish_reasons_list() @@ -3651,8 +2569,52 @@ def _maybe_build_beam_history(req_idx: int) -> BeamHistory | None: else: return None + finalized_token_updates: list[tuple[int, list[int]]] = [] + # Fast-path (batched pybind): when the batch is greedy with no beam + # search, no logprobs, no draft tokens, no stop-words, and no + # speculative tree, collapse per-request pybind chatter into one + # batched add_new_tokens_to_requests call. Single-pass eligibility + # check with early-break; falls through when any invariant breaks. + if ( + self._batch_fastpath_eligible + and logprobs_state_list is None + and self.get_spec_tree_manager(resource_manager) is None + ): + alive_reqs: list[LlmRequest] = [] + tokens_flat: list[int] = [] + fastpath_ok = True + new_tokens_step0 = new_tokens_list[0] + for req in state.requests: + if req.state == LlmRequestState.GENERATION_COMPLETE: + continue + if get_draft_token_length(req) != 0 or req.py_stop_words_list: + fastpath_ok = False + break + assert req.py_seq_slot is not None + alive_reqs.append(req) + tokens_flat.append(new_tokens_step0[req.py_seq_slot][DEFAULT_BEAM_IDX]) + if fastpath_ok and alive_reqs: + add_new_tokens_to_requests(alive_reqs, tokens_flat, DEFAULT_BEAM_IDX) + _valid_finish_reasons = { + FinishReason.END_ID, + FinishReason.LENGTH, + FinishReason.STOP_WORDS, + } + for req in alive_reqs: + assert req.py_seq_slot is not None + reason_val = finish_reasons[req.py_seq_slot][0][DEFAULT_BEAM_IDX] + if reason_val != 0: + reason = FinishReason(reason_val) + if reason in _valid_finish_reasons: + req.finish_by(reason, DEFAULT_BEAM_IDX) + req.py_num_accepted_draft_tokens = 0 + req.py_rewind_len = 0 + req.py_decoding_iter += 1 + return + for req_idx, req in enumerate(state.requests): if req.state == LlmRequestState.GENERATION_COMPLETE: + self._top_p_decay.retire_slot(req) continue if req.py_beam_width > 1: @@ -3698,6 +2660,7 @@ def _maybe_build_beam_history(req_idx: int) -> BeamHistory | None: req.py_rewind_len = 0 else: processed = 1 + num_tokens_before = req.get_num_tokens(DEFAULT_BEAM_IDX) num_accepted = self.process_draft_tokens( req, new_tokens_tensor=new_tokens, @@ -3712,6 +2675,12 @@ def _maybe_build_beam_history(req_idx: int) -> BeamHistory | None: req.py_num_accepted_draft_tokens = 0 req.py_rewind_len = 0 processed += num_accepted + if actual_draft_len > 0: + num_new_tokens = req.get_num_tokens(DEFAULT_BEAM_IDX) - num_tokens_before + if num_new_tokens > 0: + assert req.py_seq_slot is not None + confirmed_tokens = req.get_tokens(DEFAULT_BEAM_IDX)[-num_new_tokens:] + finalized_token_updates.append((req.py_seq_slot, confirmed_tokens)) self.handle_logprobs(req, logprobs_state_list=logprobs_state_list, count=processed) req.py_decoding_iter += 1 # Check None or empty list @@ -3719,6 +2688,10 @@ def _maybe_build_beam_history(req_idx: int) -> BeamHistory | None: self._finish_reasons_handler.store.num_accepted_draft_tokens_host[ req.py_seq_slot ] = req.py_num_accepted_draft_tokens + if req.state == LlmRequestState.GENERATION_COMPLETE: + self._top_p_decay.retire_slot(req) + + self._penalty_handler.update_token_counts(finalized_token_updates) def _return_log_probs(self, requests: list[LlmRequest]) -> bool: return any(req.py_return_log_probs for req in requests) @@ -3758,6 +2731,16 @@ def sample_async( self.setup_sampler_step(scheduled_requests) new_tokens = self.store.new_tokens + if self._track_pending_steps: + # A context request claims a (possibly reused) slot: clear any + # counter leaked by a prior occupant that never got its final + # update_requests. Must happen before _process_requests, which + # reads the counters for bad-words staleness. + for r in scheduled_requests.context_requests: + if not r.py_is_draft: + assert r.py_seq_slot is not None + self._pending_steps[r.py_seq_slot] = 0 + # seq_slots_cuda / seq_lens_cuda are cast once inside # _process_requests and shared with the beam-search metadata builder. ( @@ -3774,6 +2757,11 @@ def sample_async( num_context_logits_prefix_sum, ) + if self._track_pending_steps and requests and not self._is_draft_batch(requests): + for r in requests: + assert r.py_seq_slot is not None + self._pending_steps[r.py_seq_slot] += 1 + finish_reasons_host: torch.Tensor | None = None first_finish_reasons_host: torch.Tensor | None = None beam_history_builders: list[BeamHistoryBuilder | None] | None = None @@ -4007,6 +2995,7 @@ def _sample_batched_by_strategy( get_metadata_type_for_group_fn=self._grouped_sampler_cls.get_metadata_type_for_group, seq_slots_cuda=seq_slots_cuda, seq_lens_cuda=seq_lens_cuda, + req_num_steps=req_num_steps, ) generator_cuda = self.get_generator(cuda_device) @@ -4240,6 +3229,7 @@ def _unbatch_sampling_results( new_tokens_cuda: torch.Tensor, req_num_generated_tokens: torch.Tensor, seq_slots: torch.Tensor, + seq_slots_cuda: torch.Tensor, ) -> torch.Tensor: batch_req_indices = batched_sampling_result.batch_req_indices batch_next_tokens_cuda_int = batched_sampling_result.batch_next_tokens_cuda_int @@ -4273,70 +3263,66 @@ def _dims_canonically_ordered(t: torch.Tensor) -> bool: new_tokens_cuda.view(-1, *new_tokens_cuda.shape[2:]).scatter_( 0, batch_dest_indices_1d_cuda, batch_next_tokens_cuda_int ) + # Post-sample: decay the runtime top-p for any decay-active slots that were + # sampled this iteration (must run after tokens land in new_tokens_cuda). + # batch_req_indices is a permutation of all sampled requests, so the set of + # sampled slots is exactly seq_slots (the kernel updates each slot + # independently; order is irrelevant) -- pass the resident device copy + # instead of gathering seq_slots[batch_req_indices] on host and copying it. + self._top_p_decay.update_after_sample( + step_tokens=new_tokens_cuda[DEFAULT_STEP_IDX, :, DEFAULT_BEAM_IDX], + sampled_slots_cuda=seq_slots_cuda, + ) return self._copy_to_host(new_tokens_cuda) - @staticmethod - @torch.inference_mode() - def _apply_min_length_penalty( - logits: torch.Tensor, - requests: list[LlmRequest], - num_steps_tensor: torch.Tensor, - num_beams_tensor: torch.Tensor, - ) -> None: - """Apply min_length_penalty to logits, mutating ``logits`` in place. - - Args: - logits: The logits to apply min length penalty to - requests: The requests to apply min length penalty to - num_steps_tensor: The number of steps per request (host tensor) - num_beams_tensor: The number of beams per request (host tensor) + def _compute_pending_steps(self, requests: list[LlmRequest]) -> list[int] | None: + """Per-request count of tokens sampled but not yet written back, or None. + + With the overlap scheduler ``sample_async`` for step ``i`` runs before + ``update_requests`` for step ``i - 1``, so the host token list — and + hence ``get_num_tokens()`` — lags the true sequence by this many tokens. + Length-based bans (min_length) add it back to recover the real generated + length. Unlike the suffix-rule staleness this needs no device-side + lookup, so it is not restricted to the single-step / single-beam case. + Returns None when the overlap scheduler is off, on a draft batch, or + when nothing is pending, so callers can skip the correction entirely. """ - if not any( - r.py_min_length and (r.max_beam_num_tokens - r.py_orig_prompt_len) < r.py_min_length[0] - for r in requests - ): - return - - # Deferred host conversion: only needed on the (rare) penalty path. - num_steps = num_steps_tensor.tolist() - num_beams = num_beams_tensor.tolist() - - rows: list[int] = [] - cols: list[int] = [] - current_offset = 0 - for index, r in enumerate(requests): - # Advance the offset before any guard below can skip the request: - # every request occupies its logits rows, penalized or not. - req_offset = current_offset - current_offset += num_steps[index] * num_beams[index] - - if not r.py_min_length: - continue - # Use the original end_id (before ignore_eos override) - # so we suppress the real EOS token, not token -1. - end_id = getattr(r, "py_original_end_id", r.py_end_id) - if end_id is None or end_id <= -1: - continue - - for beam_idx in range(num_beams[index]): - for step in range(num_steps[index]): - if (r.get_num_tokens(beam_idx) - r.py_orig_prompt_len) + step < r.py_min_length[ - 0 - ]: - rows.append(req_offset + num_steps[index] * beam_idx + step) - cols.append(end_id) - else: - break - - if rows: - neg_inf = torch.full((), float("-inf"), dtype=logits.dtype, device=logits.device) - row_idx = torch.tensor(rows, dtype=torch.long, pin_memory=prefer_pinned()).to( - logits.device, non_blocking=True - ) - col_idx = torch.tensor(cols, dtype=torch.long, pin_memory=prefer_pinned()).to( - logits.device, non_blocking=True - ) - logits.index_put_((row_idx, col_idx), neg_inf, accumulate=False) + if not self._track_pending_steps or self._is_draft_batch(requests): + return None + pending = [ + self._pending_steps[r.py_seq_slot] if r.py_seq_slot is not None else 0 for r in requests + ] + return pending if any(pending) else None + + def _compute_stale_by_one(self, requests: list[LlmRequest]) -> list[bool] | None: + """Per-request overlap-scheduler stale flags, or None when not applicable. + + Returns a list where entry ``i`` is True when request ``i``'s host token + history lags the device state by exactly one token (the previous step's + token was sampled but not yet written back). Only the single-step, + single-beam overlap case is reconstructible on the device side; under + speculative decoding or beam search the missing history cannot be + recovered, so bans are matched against the lagging host history and may + be enforced one step late (warned once). Returns None when the overlap + scheduler is off, on a draft batch, or when nothing is pending. + """ + if not self._track_pending_steps or self._is_draft_batch(requests): + return None + pending = [ + self._pending_steps[r.py_seq_slot] if r.py_seq_slot is not None else 0 for r in requests + ] + if not any(pending): + return None + if self.max_tokens == 1 and self.max_beam_width == 1 and max(pending) == 1: + return [p > 0 for p in pending] + logger.warning_once( + "bad_words / no_repeat_ngram_size with the overlap scheduler and " + "speculative decoding or beam search: bans are matched against a " + "host token history that lags the device state and may be enforced " + "inexactly.", + key="bad_words_stale_overlap", + ) + return None @staticmethod def _select_generated_logits( @@ -4643,13 +3629,58 @@ def _process_requests( logits_cuda, sampling_requests, sampling_requests_metadata.req_num_steps ) - self._apply_min_length_penalty( + # Apply repetition/presence/frequency penalties in place, before the greedy fast + # path, so both greedy and grouped-sampling logits are penalized. + self._penalty_handler.apply( logits_cuda, sampling_requests, - sampling_requests_metadata.req_num_steps, - sampling_requests_metadata.req_num_beams, + new_tokens=new_tokens_cuda, + seq_slots=seq_slots_cuda, + request_offsets=sampling_requests_metadata.req_offsets, + request_num_steps=sampling_requests_metadata.req_num_steps, + # _is_draft_batch reads requests[0]; an empty batch has no penalties to apply + # anyway, so short-circuit rather than index into it. + is_draft_batch=bool(sampling_requests) and self._is_draft_batch(sampling_requests), ) + has_min_length = any(getattr(r, "py_min_length", None) for r in sampling_requests) + has_bad_words = any(getattr(r, "py_bad_words", None) for r in sampling_requests) + # Normalized in executor_request_to_llm_request: a positive int, or + # None when the restriction is disabled for the request. + ngram_sizes = [getattr(r, "py_no_repeat_ngram_size", None) for r in sampling_requests] + has_no_repeat_ngram = any(size is not None for size in ngram_sizes) + if has_min_length or has_bad_words or has_no_repeat_ngram: + # Overlap-scheduler stale flags (per request): True when the host + # token history lags the device by one token. Only the overlap + # handler consumes them; computed here as it needs sampler state. + # Only the suffix-matching bans (bad words, no-repeat ngram) care: + # min_length bans EOS from a length count, never from token values, + # so a lagging history cannot mismatch it. Skipping the call for + # min-length-only batches also avoids emitting the stale-history + # warning, which names features such a batch does not use. + stale_by_one = ( + self._compute_stale_by_one(sampling_requests) + if (has_bad_words or has_no_repeat_ngram) + else None + ) + # min_length compares against get_num_tokens(), which counts only + # the host history; add back the tokens still pending write-back so + # the generated length is exact under the overlap scheduler. + pending_steps = ( + self._compute_pending_steps(sampling_requests) if has_min_length else None + ) + bans = self._token_ban_handler.generate_ban_list( + sampling_requests, + sampling_requests_metadata.req_num_steps.tolist(), + sampling_requests_metadata.req_num_beams.tolist(), + ngram_sizes, + stale_by_one=stale_by_one, + pending_steps=pending_steps, + ) + self._token_ban_handler.apply_ban_list( + logits_cuda, bans, new_tokens_cuda=new_tokens_cuda + ) + # Fast path for greedy sampling if self._can_use_fast_greedy_path(sampling_requests): # Compute destination indices on CPU (same pattern as _unbatch_sampling_results) @@ -4728,6 +3759,7 @@ def _process_requests( new_tokens_cuda=new_tokens_cuda, req_num_generated_tokens=sampling_requests_metadata.req_num_generated_tokens, seq_slots=seq_slots_host, + seq_slots_cuda=seq_slots_cuda, ) # NB: update_requests syncs w/ device computation and async D2H copies @@ -4746,6 +3778,7 @@ def should_provide_draft_probs(self, request: LlmRequest) -> bool: temperature = params.temperature top_p = params.top_p top_k = params.top_k + min_p = params.min_p # Do not request draft probs when sampling is greedy. return not SamplingParams.params_imply_greedy_decoding( @@ -4753,6 +3786,7 @@ def should_provide_draft_probs(self, request: LlmRequest) -> bool: top_p=top_p, top_k=top_k, use_beam_search=self._use_beam_search, + min_p=min_p, ) diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/sampler_common.py b/tensorrt_llm/_torch/pyexecutor/sampler/sampler_common.py new file mode 100644 index 000000000000..708406d579be --- /dev/null +++ b/tensorrt_llm/_torch/pyexecutor/sampler/sampler_common.py @@ -0,0 +1,151 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared building blocks for the sampler package. + +The package's base layer: tensor helpers, the shared step/beam index constants, +and the per-request queries that read an ``LlmRequest``'s sampling config into +:class:`UtilsSamplingParams`. Imports nothing else from the package. + +Resolving a request's ``Strategy`` lives in ``sampler_strategy``. +""" + +from dataclasses import dataclass +from typing import List, Optional, TypeAlias, TypeVar, cast + +import torch + +from ..llm_request import LlmRequest + +T = TypeVar("T") + +# Beam index to use when no beam search is used but a beam index is required +DEFAULT_BEAM_IDX = 0 +# Step index to use when no speculative decoding is used but a step index is required +DEFAULT_STEP_IDX = 0 + +FinishReasonsList: TypeAlias = list[list[list[int]]] + + +@dataclass(frozen=True, kw_only=True) +class UtilsSamplingParams: + """Subset of tensorrt_llm::runtime::SamplingConfig supported by the torch sampler. + + Args: + temperature: The temperature to use for sampling. + top_p: The top-p to use for sampling. + top_k: The top-k to use for sampling. + min_p: The min-p to use for sampling. + use_beam_search: Whether to use beam search. + beam_width_in: The beam_width of a request before the sampling step. + beam_width_out: The beam_width of a request after the sampling step. + top_p_decay: Per-step multiplicative decay applied to the runtime top-p. + top_p_min: Lower bound for the decayed runtime top-p. + top_p_reset_ids: Token id which, when sampled, resets the runtime top-p to + its initial value. A value < 0 never matches a token. + """ + + temperature: Optional[float] + top_p: Optional[float] + top_k: Optional[int] + use_beam_search: Optional[bool] + min_p: Optional[float] = None + beam_width_in: Optional[int] = None + beam_width_out: Optional[int] = None + top_p_decay: Optional[float] = None + top_p_min: Optional[float] = None + top_p_reset_ids: Optional[int] = None + + +def int_tensor(shape: tuple[int, ...], device: str = "cuda") -> torch.Tensor: + return torch.empty(shape, dtype=torch.int, device=device) + + +def add_token( + request: LlmRequest, new_tokens: list[list[list[int]]], *, beam_idx: int, step: int = 0 +) -> int: + # NB: Accessing nested lists faster than torch.Tensor or numpy.ndarray + seq_slot = request.py_seq_slot + assert seq_slot is not None + new_token = new_tokens[step][seq_slot][beam_idx] + request.add_new_token(new_token, beam_idx) + return new_token + + +def _unwrap_singleton(p: Optional[List[T]]) -> Optional[T]: + if p is None: + return None + (t,) = p + return t + + +def _get_beam_width_in(request: LlmRequest) -> int: + return ( + 1 + if request.is_context_init_state + else request.get_beam_width_by_iter(for_next_iteration=False) + ) + + +def _get_beam_width_out(request: LlmRequest) -> int: + return request.get_beam_width_by_iter(for_next_iteration=True) + + +def _get_max_beam_width(request: LlmRequest) -> int: + sampling_config = request.sampling_config + max_beam_width = cast(int, sampling_config.beam_width) + if sampling_config.beam_width_array is not None: + max_beam_width = max( + max_beam_width, + cast( + int, torch.tensor(sampling_config.beam_width_array, dtype=torch.int32).max().item() + ), + ) + return max_beam_width + + +def _request_get_sampling_params(request: LlmRequest) -> UtilsSamplingParams: + sampling_config = request.sampling_config + # These sampling fields live on the C++ SamplingConfig as optional> + # (a shape designed for the batched TRT-LLM sampler); the torch sampler consumes + # them per request, so we unwrap the singleton lists into scalars here. When the + # TRT-LLM sampler is removed, this SamplingConfig-based plumbing should be removed + # too in favor of reading the values directly from the per-request params. + temperature = _unwrap_singleton(cast(Optional[list[float]], sampling_config.temperature)) + top_p = _unwrap_singleton(cast(Optional[list[float]], sampling_config.top_p)) + top_k = _unwrap_singleton(cast(Optional[list[int]], sampling_config.top_k)) + min_p = _unwrap_singleton(cast(Optional[list[float]], sampling_config.min_p)) + top_p_decay = _unwrap_singleton(cast(Optional[list[float]], sampling_config.top_p_decay)) + top_p_min = _unwrap_singleton(cast(Optional[list[float]], sampling_config.top_p_min)) + top_p_reset_ids = _unwrap_singleton(cast(Optional[list[int]], sampling_config.top_p_reset_ids)) + beam_width_out = _get_beam_width_out(request) + beam_width_in = _get_beam_width_in(request) + use_beam_search = _get_max_beam_width(request) > 1 + + return UtilsSamplingParams( + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + beam_width_in=beam_width_in, + beam_width_out=beam_width_out, + use_beam_search=use_beam_search, + top_p_decay=top_p_decay, + top_p_min=top_p_min, + top_p_reset_ids=top_p_reset_ids, + ) + + +def _request_sampling_params_cachable(params: UtilsSamplingParams) -> bool: + return not params.use_beam_search diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/sampling_utils.py b/tensorrt_llm/_torch/pyexecutor/sampler/sampler_strategy.py similarity index 69% rename from tensorrt_llm/_torch/pyexecutor/sampler/sampling_utils.py rename to tensorrt_llm/_torch/pyexecutor/sampler/sampler_strategy.py index fc93d622b64a..1a64c079f86a 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/sampling_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/sampler_strategy.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Helper functions for sampling. +"""Sampling strategies: what to draw, and how to get there from a request. -Code in this module should operate on logits and probs, without -referring to types like LlmRequest. +Holds the :data:`Strategy` types, their implementations and the grouped +samplers, which operate on logits and probs; ``_request_strategy`` maps an +``LlmRequest`` onto a strategy, reading its sampling config via +``sampler_common``. """ import abc @@ -26,53 +28,67 @@ import torch -from tensorrt_llm._torch.pyexecutor.sampler.ops import flashinfer, vanilla +from tensorrt_llm._torch.pyexecutor.sampler.ops import vanilla # These op wrappers are safe to import without flashinfer installed; they are # only called on the flashinfer sampler / speculative-worker paths. from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( - sampling_from_probs_op as sampling_from_probs_op, -) -from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import softmax_op as softmax_op -from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( - top_k_mask_logits_op as top_k_mask_logits_op, -) -from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( - top_k_sampling_from_probs_op as top_k_sampling_from_probs_op, -) -from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( - top_k_top_p_sampling_from_logits_op as top_k_top_p_sampling_from_logits_op, -) -from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( - top_p_renorm_probs_op as top_p_renorm_probs_op, -) -from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( - top_p_sampling_from_probs_op as top_p_sampling_from_probs_op, + sampling_from_probs_op, + sanitize_top_k, + softmax_op, + top_k_mask_logits_op, + top_k_renorm_probs_op, + top_k_sampling_from_probs_op, + top_k_top_p_sampling_from_logits_op, + top_k_top_p_sampling_from_probs_op, + top_p_renorm_probs_op, + top_p_sampling_from_probs_op, ) from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( - GREEDY_TEMPERATURE_THRESHOLD as GREEDY_TEMPERATURE_THRESHOLD, -) -from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( - BeamSearchMetadata as BeamSearchMetadata, -) -from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import Fusions as Fusions -from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import StrategyMetadata as StrategyMetadata -from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( - beam_search_sampling_batch as beam_search_sampling_batch, -) -from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( - get_rejected_indices as get_rejected_indices, -) -from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( - greedy_search_sampling_batch as greedy_search_sampling_batch, -) -from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import sample_rejected as sample_rejected -from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( - top_k_top_p_sampling_batch as top_k_top_p_sampling_batch, + GREEDY_TEMPERATURE_THRESHOLD, + BeamSearchMetadata, + Fusions, + StrategyMetadata, + beam_search_sampling_batch, + get_rejected_indices, + greedy_search_sampling_batch, + min_p_renorm_probs, + sample_rejected, + top_k_top_p_sampling_batch, ) from tensorrt_llm._utils import prefer_pinned from tensorrt_llm.sampling_params import SamplingParams +from ..llm_request import LlmRequest +from .sampler_common import ( + UtilsSamplingParams, + _request_get_sampling_params, + _request_sampling_params_cachable, +) + +# Ops imported above are re-exported for dependent modules (sampler, drafting +# loops, tests). mypy runs in strict mode (no implicit re-export), so they must +# be listed here. +__all__ = [ + "GREEDY_TEMPERATURE_THRESHOLD", + "BeamSearchMetadata", + "Fusions", + "StrategyMetadata", + "beam_search_sampling_batch", + "get_rejected_indices", + "greedy_search_sampling_batch", + "sample_rejected", + "sampling_from_probs_op", + "softmax_op", + "top_k_mask_logits_op", + "top_k_sampling_from_probs_op", + "top_k_top_p_sampling_batch", + "top_k_top_p_sampling_from_logits_op", + "top_k_top_p_sampling_from_probs_op", + "top_p_renorm_probs_op", + "top_p_sampling_from_probs_op", +] + if sys.version_info[:2] >= (3, 12): from typing import override else: @@ -83,35 +99,46 @@ TopK: TypeAlias = tuple[Literal["top_k"], int, float] TopP: TypeAlias = tuple[Literal["top_p"], float, float] TopKTopP: TypeAlias = tuple[Literal["top_k_top_p"], int, float, float] +# (tag, top_k, top_p, min_p, temperature) +MinP: TypeAlias = tuple[Literal["min_p"], int, float, float, float] Greedy: TypeAlias = tuple[Literal["greedy"], None] BeamSearch: TypeAlias = tuple[Literal["beam_search"], int, int, float] GREEDY: Greedy = ("greedy", None) -Strategy: TypeAlias = TopK | TopP | Greedy | TopKTopP | TemperatureOnly | BeamSearch +Strategy: TypeAlias = TopK | TopP | Greedy | TopKTopP | TemperatureOnly | MinP | BeamSearch # Re-exported from the beam-search op implementation (single source of truth). BEAM_SEARCH_PAD_TOKEN = vanilla.BEAM_SEARCH_PAD_TOKEN -@dataclass(frozen=True, kw_only=True) -class UtilsSamplingParams: - """Subset of tensorrt_llm::runtime::SamplingConfig supported by sampling_utils. +@dataclass(kw_only=True) +class TopPDecayMetadata(StrategyMetadata): + """Per-group runtime top-p override for Top-P Decay (attached to the + top-p-carrying groups -- top_p, top_k_top_p and min_p -- via the + ``StrategyMetadata`` mechanism). - Args: - temperature: The temperature to use for sampling. - top_p: The top-p to use for sampling. - top_k: The top-k to use for sampling. - use_beam_search: Whether to use beam search. - beam_width_in: The beam_width of a request before the sampling step. - beam_width_out: The beam_width of a request after the sampling step. + ``slots`` maps each per-step group row to its sequence slot; the decayed + per-row top-p is gathered on-device from the per-slot ``runtime_top_p`` + store, gated by ``is_decay_slot`` (non-decay rows keep their static top-p). + Consumed by the TopP*/TopKTopP*/MinP* strategy impls in ``sample()``. See + ``top_p_decay.TopPDecayStore`` for the feature-level semantics. """ - temperature: Optional[float] - top_p: Optional[float] - top_k: Optional[int] - use_beam_search: Optional[bool] - beam_width_in: Optional[int] = None - beam_width_out: Optional[int] = None + slots: torch.Tensor + """Per-step group rows' sequence slots (int64, device).""" + runtime_top_p: torch.Tensor + """Per-slot runtime (decayed) top-p store (float32, device).""" + is_decay_slot: torch.Tensor + """Per-slot decay-active gate (bool, device).""" + + +def top_p_decay_active(params: UtilsSamplingParams) -> bool: + """Whether dynamic top-p decay is active for a request. + + Delegates to the single-source predicate on SamplingParams; note that + ``top_p_min`` / ``top_p_reset_ids`` alone do not activate dynamic behavior. + """ + return SamplingParams.params_imply_top_p_decay_active(params.top_p_decay) def resolve_sampling_strategy(params: UtilsSamplingParams, *, vocab_size: int) -> Strategy: @@ -121,12 +148,18 @@ def resolve_sampling_strategy(params: UtilsSamplingParams, *, vocab_size: int) - temperature = params.temperature top_p = params.top_p top_k = params.top_k + min_p = params.min_p + # The greedy verdict (including the top-p-decay override of the implicit + # all-unset greedy default, and explicit greedy controls winning over decay) + # is single-sourced in SamplingParams.params_imply_greedy_decoding. if SamplingParams.params_imply_greedy_decoding( temperature=temperature, top_p=top_p, top_k=top_k, use_beam_search=use_beam_search, + min_p=min_p, + top_p_decay=params.top_p_decay, ): return GREEDY @@ -134,7 +167,7 @@ def resolve_sampling_strategy(params: UtilsSamplingParams, *, vocab_size: int) - # NB: not greedy, hence temperature != 0 if specified temperature = temperature or 1.0 - # Beam search does not rely on top_p or top_k, so we can return the strategy here + # Beam search does not rely on top_p, top_k or min_p, so we can return the strategy here if use_beam_search: assert params.beam_width_in is not None and params.beam_width_out is not None, ( "beam_width_in and beam_width_out must be specified for beam search" @@ -150,7 +183,16 @@ def resolve_sampling_strategy(params: UtilsSamplingParams, *, vocab_size: int) - assert top_k > 1, "non-greedy sampling requires valid top_k" need_top_k = top_k < vocab_size assert top_p > 0, "non-greedy sampling requires valid top_p" - need_top_p = top_p < 1 + # A decay-active request must go through a top-p-capable path even when its + # initial top_p is 1.0, so the runtime top-p (sourced per-row at sample time) + # can shrink the nucleus on later steps. + need_top_p = top_p < 1 or top_p_decay_active(params) + + # Disabled top_k is 0 ("keep all"), not vocab_size, which can be the + # fast-greedy probe (2**31) and overflow the int32 tensor; _compute_probs + # sanitizes it. + if min_p is not None and min_p > 0.0: + return ("min_p", top_k if need_top_k else 0, top_p, min_p, temperature) if need_top_p: if need_top_k: @@ -200,6 +242,15 @@ def sample( temperature=cast(float, temperature), generator=generator, ) + case ("min_p", top_k, top_p, min_p, temperature): + tokens, softmax = top_k_top_p_sampling_batch( + logits, + top_k=cast(int, top_k), + top_p=cast(float, top_p), + min_p=cast(float, min_p), + temperature=cast(float, temperature), + generator=generator, + ) case ("greedy", None): tokens, softmax = greedy_search_sampling_batch(logits, return_probs=return_probs) temperature = None @@ -319,31 +370,86 @@ def _sample_greedy_with_probs( return tokens, probs @classmethod - def _sample_with_probs( + def _compute_probs( cls, logits: torch.Tensor, *, group_logit_indices: Optional[torch.Tensor], top_k: Optional[torch.Tensor], top_p: Optional[torch.Tensor], + min_p: Optional[torch.Tensor], temperature: torch.Tensor, - generator: Optional[torch.Generator], - ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + ) -> torch.Tensor: + """Temperature + softmax + optional min-p / top-k / top-p renorm. + + min_p runs first (vLLM semantics): its threshold is relative to the + max probability of the unfiltered row, and top_k/top_p renormalize, + which inflates that max and would make a later min_p stricter. + """ + probs = cls._prepare_probs_with_temperature(logits, group_logit_indices, temperature) + if min_p is not None: + probs = min_p_renorm_probs(probs, min_p) + if top_k is not None: - logits = cls._prepare_logits_with_temperature( - logits, group_logit_indices, temperature - ) - logits = top_k_mask_logits_op(logits, top_k) - probs = cls._prepare_probs_with_temperature(logits, None, None) - else: - probs = cls._prepare_probs_with_temperature( - logits, group_logit_indices, temperature - ) + top_k = sanitize_top_k(top_k, probs.shape[-1]) + probs = top_k_renorm_probs_op(probs, top_k) + if top_p is not None: probs = top_p_renorm_probs_op(probs, top_p) + + return probs + + @classmethod + def _sample_with_probs( + cls, + logits: torch.Tensor, + *, + group_logit_indices: Optional[torch.Tensor], + top_k: Optional[torch.Tensor], + top_p: Optional[torch.Tensor], + min_p: Optional[torch.Tensor], + temperature: torch.Tensor, + generator: Optional[torch.Generator], + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + probs = cls._compute_probs( + logits, + group_logit_indices=group_logit_indices, + top_k=top_k, + top_p=top_p, + min_p=min_p, + temperature=temperature, + ) new_tokens = cls._sample_from_probs(probs, generator=generator) return new_tokens, probs + class TopPDecayMixin: + """Mixed into the TopP*/TopKTopP*/MinP* impls (the owners of a per-row + ``_top_p`` tensor) to consume ``TopPDecayMetadata``.""" + + _top_p: torch.Tensor + + def _maybe_apply_top_p_decay(self, group_metadata: Optional[StrategyMetadata]) -> None: + """Override the per-row static top-p with the decayed runtime top-p. + + Only decay-active rows (per the on-device ``is_decay_slot`` gate) are + overridden, so a group mixing top-p-decay and plain top-p requests + keeps each row's correct value. The overridden ``self._top_p`` tensor + then feeds both sampling and ``top_p_renorm_probs_op`` (so processed + logprobs match). Fused via torch.compile (gather + gate + select). + """ + if not isinstance(group_metadata, TopPDecayMetadata): + return + assert self._top_p.shape == group_metadata.slots.shape, ( + self._top_p.shape, + group_metadata.slots.shape, + ) + self._top_p = Fusions.top_p_decay_gather( + runtime_top_p=group_metadata.runtime_top_p, + is_decay_slot=group_metadata.is_decay_slot, + static_top_p=self._top_p, + slots=group_metadata.slots, + ) + class StrategyImplWithProbs(StrategyImpl): @override @classmethod @@ -372,7 +478,7 @@ def sample( ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: return self._sample_greedy_with_probs(logits, group_logit_indices=group_logit_indices) - class TopKTopPWithProbs(StrategyImplWithProbs): + class TopKTopPWithProbs(TopPDecayMixin, StrategyImplWithProbs): def __init__(self, top_k: torch.Tensor, top_p: torch.Tensor, temperature: torch.Tensor): self._top_k = top_k self._top_p = top_p @@ -398,11 +504,13 @@ def sample( generator: Optional[torch.Generator] = None, group_metadata: Optional[StrategyMetadata] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + self._maybe_apply_top_p_decay(group_metadata) return self._sample_with_probs( logits, group_logit_indices=group_logit_indices, top_k=self._top_k, top_p=self._top_p, + min_p=None, temperature=self._temperature, generator=generator, ) @@ -436,11 +544,12 @@ def sample( group_logit_indices=group_logit_indices, top_k=self._top_k, top_p=None, + min_p=None, temperature=self._temperature, generator=generator, ) - class TopPWithProbs(StrategyImplWithProbs): + class TopPWithProbs(TopPDecayMixin, StrategyImplWithProbs): def __init__(self, top_p: torch.Tensor, temperature: torch.Tensor): self._top_p = top_p self._temperature = temperature @@ -464,11 +573,13 @@ def sample( generator: Optional[torch.Generator] = None, group_metadata: Optional[StrategyMetadata] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + self._maybe_apply_top_p_decay(group_metadata) return self._sample_with_probs( logits, group_logit_indices=group_logit_indices, top_k=None, top_p=self._top_p, + min_p=None, temperature=self._temperature, generator=generator, ) @@ -498,6 +609,52 @@ def sample( group_logit_indices=group_logit_indices, top_k=None, top_p=None, + min_p=None, + temperature=self._temperature, + generator=generator, + ) + + class MinPWithProbs(TopPDecayMixin, StrategyImplWithProbs): + def __init__( + self, + top_k: torch.Tensor, + top_p: torch.Tensor, + min_p: torch.Tensor, + temperature: torch.Tensor, + ): + self._top_k = top_k + self._top_p = top_p + self._min_p = min_p + self._temperature = temperature + + @override + @classmethod + def from_strategies( + cls, strategies: list[Any], cuda_device: torch.device + ) -> "_StrategyImpls.MinPWithProbs": + return cls( + cls._make_tensor([s[1] for s in strategies], torch.int32, cuda_device), + cls._make_tensor([s[2] for s in strategies], torch.float32, cuda_device), + cls._make_tensor([s[3] for s in strategies], torch.float32, cuda_device), + cls._make_tensor([s[4] for s in strategies], torch.float32, cuda_device), + ) + + @override + def sample( + self, + logits: torch.Tensor, + *, + group_logit_indices: Optional[torch.Tensor] = None, + generator: Optional[torch.Generator] = None, + group_metadata: Optional[StrategyMetadata] = None, + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + self._maybe_apply_top_p_decay(group_metadata) + return self._sample_with_probs( + logits, + group_logit_indices=group_logit_indices, + top_k=self._top_k, + top_p=self._top_p, + min_p=self._min_p, temperature=self._temperature, generator=generator, ) @@ -532,7 +689,7 @@ def sample( logits = logits[group_logit_indices] return torch.argmax(logits, dim=-1), None - class TopKTopPSampleOnly(StrategyImplSampleOnly): + class TopKTopPSampleOnly(TopPDecayMixin, StrategyImplSampleOnly): def __init__(self, top_k: torch.Tensor, top_p: torch.Tensor, temperature: torch.Tensor): self._top_k = top_k self._top_p = top_p @@ -558,6 +715,7 @@ def sample( generator: Optional[torch.Generator] = None, group_metadata: Optional[StrategyMetadata] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + self._maybe_apply_top_p_decay(group_metadata) logits = self._prepare_logits_with_temperature( logits, group_logit_indices, self._temperature ) @@ -603,7 +761,7 @@ def sample( check_nan=self._flashinfer_check_nans(probs), ), None - class TopPSampleOnly(StrategyImplSampleOnly): + class TopPSampleOnly(TopPDecayMixin, StrategyImplSampleOnly): def __init__(self, top_p: torch.Tensor, temperature: torch.Tensor): self._top_p = top_p self._temperature = temperature @@ -627,6 +785,7 @@ def sample( generator: Optional[torch.Generator] = None, group_metadata: Optional[StrategyMetadata] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + self._maybe_apply_top_p_decay(group_metadata) probs = self._prepare_probs_with_temperature( logits, group_logit_indices, self._temperature ) @@ -662,11 +821,66 @@ def sample( group_logit_indices=group_logit_indices, top_k=None, top_p=None, + min_p=None, temperature=self._temperature, generator=generator, ) return new_tokens, None + class MinPSampleOnly(TopPDecayMixin, StrategyImplSampleOnly): + def __init__( + self, + top_k: torch.Tensor, + top_p: torch.Tensor, + min_p: torch.Tensor, + temperature: torch.Tensor, + ): + self._top_k = top_k + self._top_p = top_p + self._min_p = min_p + self._temperature = temperature + + @override + @classmethod + def from_strategies( + cls, strategies: list[Any], cuda_device: torch.device + ) -> "_StrategyImpls.MinPSampleOnly": + return cls( + cls._make_tensor([s[1] for s in strategies], torch.int32, cuda_device), + cls._make_tensor([s[2] for s in strategies], torch.float32, cuda_device), + cls._make_tensor([s[3] for s in strategies], torch.float32, cuda_device), + cls._make_tensor([s[4] for s in strategies], torch.float32, cuda_device), + ) + + @override + def sample( + self, + logits: torch.Tensor, + *, + group_logit_indices: Optional[torch.Tensor] = None, + generator: Optional[torch.Generator] = None, + group_metadata: Optional[StrategyMetadata] = None, + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + self._maybe_apply_top_p_decay(group_metadata) + # With min_p applied first, nothing has to run after top_k/top_p, so + # the fused kernel can filter and sample in one pass instead of two + # renorms plus a separate sampling step. + probs = self._compute_probs( + logits, + group_logit_indices=group_logit_indices, + top_k=None, + top_p=None, + min_p=self._min_p, + temperature=self._temperature, + ) + return top_k_top_p_sampling_from_probs_op( + probs, + sanitize_top_k(self._top_k, probs.shape[-1]), + self._top_p, + generator=generator, + check_nan=self._flashinfer_check_nans(probs), + ), None + class BeamSearchMixin(StrategyImpl): def __init__(self, beam_width_in: int, beam_width_out: int, temperature: torch.Tensor): self._beam_width_in = beam_width_in @@ -720,6 +934,7 @@ class BeamSearchSampleOnly(BeamSearchMixin, StrategyImplSampleOnly): | Literal["top_k"] | Literal["top_p"] | Literal["top_k_top_p"] + | Literal["min_p"] | Literal["greedy"] | tuple[Literal["beam_search"], int, int] ) @@ -738,6 +953,7 @@ def strategy_grouping_key(strategy: Strategy) -> _STRATEGY_KEY_TYPE: | ("top_p", _, _) | ("top_k_top_p", _, _, _) | ("temperature", _) + | ("min_p", _, _, _, _) | ("greedy", None) ): return cast(_STRATEGY_KEY_TYPE, strategy[0]) @@ -753,6 +969,8 @@ def get_metadata_type_for_group( match strategy_key: case ("beam_search", _, _): return BeamSearchMetadata + case "top_p" | "top_k_top_p" | "min_p": + return TopPDecayMetadata case _: return None @@ -786,6 +1004,8 @@ def sample_grouped_strategies( strategy_impl_cls = _StrategyImpls.TopKTopPWithProbs case "temperature": strategy_impl_cls = _StrategyImpls.TemperatureOnlyWithProbs + case "min_p": + strategy_impl_cls = _StrategyImpls.MinPWithProbs case "greedy": strategy_impl_cls = _StrategyImpls.GreedyWithProbs case ("beam_search", beam_width_in_key, _): @@ -803,6 +1023,8 @@ def sample_grouped_strategies( strategy_impl_cls = _StrategyImpls.TopKTopPSampleOnly case "temperature": strategy_impl_cls = _StrategyImpls.TemperatureOnlySampleOnly + case "min_p": + strategy_impl_cls = _StrategyImpls.MinPSampleOnly case "greedy": strategy_impl_cls = _StrategyImpls.GreedySampleOnly case ("beam_search", beam_width_in_key, _): @@ -824,85 +1046,15 @@ def sample_grouped_strategies( return next_tokens, softmax, strategy_impl.get_temperature() -# Re-export the torch greedy op (used by drafting_loops and speculative/interface). -greedy = greedy_search_sampling_batch - -# --------------------------------------------------------------------------- -# Spec-decoding interface: compute_probs_from_logits (per-request tensor params) -# --------------------------------------------------------------------------- - - -def sanitize_top_k(top_k: torch.Tensor, vocab_size: int) -> torch.Tensor: - """Map ``top_k`` into a backend-safe range before top-k filtering. - - Per ``SamplingParams``, ``top_k == 0`` means "all logits" (top-k disabled), - but the flashinfer top-k kernels (``top_k_mask_logits``) break on a literal - 0 — they mask the entire row (all-zero probs). Map any non-positive value - (and any oversized disable sentinel such as ``INT32_MAX``) to - ``vocab_size`` (== keep all tokens), leaving genuine top_k values - untouched. - """ - return torch.where(top_k > 0, top_k, torch.full_like(top_k, vocab_size)).clamp(max=vocab_size) - - -@torch.compile(options={"max-autotune": True}) -def compute_probs_from_logits( - logits: torch.Tensor, - temperatures: torch.Tensor, - top_k: Optional[torch.Tensor], - top_p: Optional[torch.Tensor], -) -> torch.Tensor: - """Compute filtered+normalized probs via flashinfer (hard dependency). - - ``temperatures``, ``top_k``, ``top_p`` are per-request tensors matching the - spec-decoding call site in interface.py. - """ - if top_k is not None: - top_k = sanitize_top_k(top_k, logits.shape[-1]) +def _request_strategy(request: LlmRequest, *, vocab_size: int) -> Strategy: + # We try to cache the resolved strategy on the request object, as it's not cheap enough to + # resolve it on every iteration. + cached_sampling_strategy = request.py_sampling_strategy + if cached_sampling_strategy is not None: + return cached_sampling_strategy - return flashinfer.compute_probs_from_logits_op(logits, temperatures, top_k, top_p) - - -@torch.compile(options={"max-autotune": True}) -def sampling_batch_spec_dec_one_model( - logits: torch.Tensor, - temperatures: torch.Tensor, - top_k: torch.Tensor, - top_p: torch.Tensor, - seed: Optional[torch.Tensor] = None, - offset: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """CUDA-graph compatible sampling; supports mixed sampling params. Returns sampled tokens.""" - top_k = sanitize_top_k(top_k, logits.shape[-1]) - # Greedy rows (temperature <= threshold) must return the argmax token, not a - # sample from the temperature-scaled distribution. Capture the argmax from the - # *original* logits up front; safely_apply_temperature_inplace then guards the division - # against the greedy sentinel, and torch.where restores the greedy rows below. - # All ops are branch-free (no data-dependent control flow), so this stays - # CUDA-graph safe. - is_greedy = temperatures <= vanilla.GREEDY_TEMPERATURE_THRESHOLD - greedy_tokens = logits.argmax(dim=-1) - logits = vanilla.safely_apply_temperature_inplace(logits, temperatures) - sampled = flashinfer.top_k_top_p_sampling_from_logits_op( - logits, top_k, top_p, seed=seed, offset=offset - ) - # argmax yields int64; cast so torch.where preserves the sampler's dtype - # (flashinfer returns int32) instead of promoting the result to int64. - return torch.where(is_greedy, greedy_tokens.to(sampled.dtype), sampled) - - -@torch.compile(options={"max-autotune": True}) -def sampling_batch_spec_dec_one_model_for_rejection( - logits: torch.Tensor, - temperatures: torch.Tensor, - top_k: torch.Tensor, - top_p: torch.Tensor, - seed: Optional[torch.Tensor] = None, - offset: Optional[torch.Tensor] = None, -) -> tuple[torch.Tensor, torch.Tensor]: - """Draft sampler returning tokens AND probs for the downstream rejection-sampling path.""" - # Rejection sampling relies on flashinfer's seed/offset support for - # determinism and cross-rank consistency. - probs = compute_probs_from_logits(logits, temperatures, top_k, top_p) - tokens = flashinfer.sampling_from_probs_op(probs, seed=seed, offset=offset) - return tokens, probs + params = _request_get_sampling_params(request) + sampling_strategy = resolve_sampling_strategy(params, vocab_size=vocab_size) + if _request_sampling_params_cachable(params): + request.py_sampling_strategy = sampling_strategy + return sampling_strategy diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/token_ban.py b/tensorrt_llm/_torch/pyexecutor/sampler/token_ban.py new file mode 100644 index 000000000000..2f0ab8a7cd96 --- /dev/null +++ b/tensorrt_llm/_torch/pyexecutor/sampler/token_ban.py @@ -0,0 +1,708 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Token-ban handling for the TorchSampler. + +A *token ban* masks specific logits to ``-inf`` so certain tokens cannot be +sampled. Several features reduce to this: bad words, no-repeat n-gram, and the +min-length EOS suppression. + +All bans are accumulated on the host into a single :class:`TokenBans` and then +applied to the logits in one batched step per category. Two handler variants +exist, selected once at construction from whether the overlap scheduler is +enabled: + +* :class:`SynchronousTokenBanHandler` -- overlap disabled. The host token + history is always complete, so every ban is unconditional. +* :class:`OverlappedTokenBanHandler` -- overlap enabled. The host history may + lag the device state by one token, so a batch mixes "fresh" requests (history + complete) with "stale" ones (missing the previous step's token, still on the + device); the latter produce conditional bans resolved on the device at apply + time without a device-to-host sync. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections import defaultdict +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field +from weakref import WeakKeyDictionary + +import torch + +from tensorrt_llm._utils import prefer_pinned + +from ..llm_request import LlmRequest + +# A suffix-ban rule ``(prefix, col)``: ban token ``col`` when the sequence ends +# with ``prefix`` (an empty prefix bans unconditionally). +BanRule = tuple[list[int], int] +RuleGenerator = Callable[[int, LlmRequest, Callable[[], list[int]]], Iterable[BanRule]] + + +def _memoizing_tokens_getter(request: LlmRequest, beam_idx: int) -> Callable[[], list[int]]: + """Return a thunk for ``request.get_tokens(beam_idx)`` that fetches once. + + ``get_tokens`` copies the whole token history through the bindings (~40us + at 2048 tokens), so rule generators receive this thunk and only pay for + the copy if they actually need the history. + """ + tokens: list[int] | None = None + + def get() -> list[int]: + nonlocal tokens + if tokens is None: + tokens = request.get_tokens(beam_idx) + return tokens + + return get + + +@dataclass(kw_only=True) +class TokenBans: + """Accumulated token bans, applied by ``TokenBanHandler.apply_ban_list``. + + Host-side accumulator; nothing here lives on the GPU. ``apply_ban_list`` + turns the populated lists into tensors and writes ``-inf`` into the logits, + one batched transfer and ``index_put_`` per category. + """ + + # Unconditional bans: logits[row, col] = -inf (fresh requests + min_length). + rows: list[int] = field(default_factory=list) + cols: list[int] = field(default_factory=list) + # Conditional bans (overlap/stale): ban only where the device-side previous + # token new_tokens_cuda[0, slot, 0] == expected. + cond_rows: list[int] = field(default_factory=list) + cond_cols: list[int] = field(default_factory=list) + cond_slots: list[int] = field(default_factory=list) + cond_expected: list[int] = field(default_factory=list) + # Conditional special case: the banned column IS the device token itself + # (overlap + no_repeat_ngram n == 1). Only OverlappedTokenBanHandler fills + # these. + dev_rows: list[int] = field(default_factory=list) + dev_slots: list[int] = field(default_factory=list) + + +@dataclass +class _NgramIndexCache: + """Incrementally maintained n-gram index for a single request (single beam). + + A request's ngram size is fixed for its lifetime, so the index is only ever + grown, never rebuilt for a different ``n``; the size is therefore not stored. + """ + + tokens: list[int] + """Host-side mirror of the request's token history.""" + indexed_up_to: int + """How far into ``tokens`` the indices have been built, so each window is + indexed exactly once.""" + fresh_idx: defaultdict[tuple[int, ...], set[int]] + """Maps each ``(n - 1)``-gram to the tokens that followed it.""" + stale_idx: defaultdict[tuple[int, ...], set[tuple[int, int]]] + """Populated only for ``n > 2``: maps each ``(n - 2)``-gram to its + ``(next, next-but-one)`` token pairs.""" + + +class TokenBanHandler(ABC): + """Base class for token-ban handling: state + shared collect/apply helpers. + + Subclasses implement ``generate_ban_list`` (build a :class:`TokenBans`) and + ``apply_ban_list`` (write it into the logits). Shared feature logic (bad + words, no-repeat ngram, min-length EOS suppression, the suffix matcher, and + the unconditional flush) lives here. + """ + + def __init__(self) -> None: + # Per-request incrementally maintained n-gram indices, keyed weakly on + # the request so entries are reclaimed when the request is freed. + self._ngram_index_caches: WeakKeyDictionary[LlmRequest, _NgramIndexCache] = ( + WeakKeyDictionary() + ) + + # ---- entry points (subclass-specific) -------------------------------- + + @abstractmethod + def generate_ban_list( + self, + requests: list[LlmRequest], + num_steps: list[int], + num_beams: list[int], + ngram_sizes: list[int | None], + *, + stale_by_one: list[bool] | None = None, + pending_steps: list[int] | None = None, + ) -> TokenBans: + """Collect all enabled feature bans into a fresh :class:`TokenBans`. + + ``stale_by_one`` is the per-request overlap-scheduler flag; it is only + meaningful for :class:`OverlappedTokenBanHandler` and ignored otherwise. + ``pending_steps`` is the per-request count of tokens sampled but not yet + written back to the host, used to correct the min-length count under the + overlap scheduler; ``None`` means no correction is needed. + """ + + @abstractmethod + def apply_ban_list( + self, + logits: torch.Tensor, + bans: TokenBans, + *, + new_tokens_cuda: torch.Tensor | None = None, + ) -> None: + """Write the accumulated bans into ``logits`` in-place.""" + + # ---- feature rule sources (shared) ----------------------------------- + + def _add_bad_words_bans( + self, + bans: TokenBans, + requests: list[LlmRequest], + num_steps: list[int], + num_beams: list[int], + *, + stale_by_one: list[bool] | None, + ) -> None: + """Add "bad words" bans. + + A single-token word is banned unconditionally; a multi-token word + ``[t0, ..., t_{k-1}]`` bans its final token ``t_{k-1}`` only when the + ``k-1`` most recently generated tokens match the prefix + ``[t0, ..., t_{k-2}]``. + """ + + def fresh_rules( + index: int, r: LlmRequest, get_context: Callable[[], list[int]] + ) -> list[BanRule]: + # A bad word is the suffix rule "after word[:-1], ban word[-1]"; + # matching is left to the driver. + return [ + (word[:-1], word[-1]) for word in getattr(r, "py_bad_words", None) or () if word + ] + + def stale_rules( + index: int, r: LlmRequest, get_context: Callable[[], list[int]] + ) -> list[BanRule]: + # Stale contract: check the first k - 2 prefix tokens here; the + # last one is compared against the device token d at flush time. + rules: list[BanRule] = [] + context: list[int] | None = None + for word in getattr(r, "py_bad_words", None) or (): + k = len(word) + if k == 0: + continue + if k == 1: + rules.append(([], word[0])) + continue + if context is None: + context = get_context() + # True sequence length is len(context) + 1; need >= k - 1. + if len(context) < k - 2: + continue + if k > 2 and context[-(k - 2) :] != word[: k - 2]: + continue + rules.append((word[:-1], word[-1])) + return rules + + self._add_suffix_rule_bans( + bans, + requests, + num_steps, + num_beams, + active=lambda index, r: bool(getattr(r, "py_bad_words", None)), + fresh_rules=fresh_rules, + stale_rules=stale_rules, + stale_by_one=stale_by_one, + ) + + def _add_min_length_bans( + self, + bans: TokenBans, + requests: list[LlmRequest], + num_steps: list[int], + num_beams: list[int], + *, + pending_steps: list[int] | None = None, + ) -> None: + """Add bans that suppress EOS until a request reaches its min length. + + For a request with ``py_min_length``, the end-of-sequence token is + banned on every step whose generated length is still below the minimum. + The ban is unconditional: it depends on the generated *length*, never on + token values, so unlike the suffix-rule features it needs no device-side + comparison and is identical under both handlers. + + It does, however, need the length to be correct. ``get_num_tokens()`` + counts the host token list, which under the overlap scheduler still + misses the tokens sampled but not yet written back by + ``update_requests``. ``pending_steps`` supplies that per-request count so + the comparison uses the true generated length; without it the length + reads one short at the boundary and EOS stays banned for one extra step, + making the request exceed ``min_length``. + """ + current_offset = 0 + for index, r in enumerate(requests): + # Advance the offset before any guard below can skip the request: + # every request occupies its logits rows, penalized or not. + request_offset = current_offset + current_offset += num_steps[index] * num_beams[index] + + if not r.py_min_length: + continue + # Use the original end_id (before ignore_eos override) so we + # suppress the real EOS token, not token -1. + end_id = getattr(r, "py_original_end_id", r.py_end_id) + if end_id is None or end_id <= -1: + continue + + pending = pending_steps[index] if pending_steps is not None else 0 + for beam_idx in range(num_beams[index]): + for step in range(num_steps[index]): + if ( + r.get_num_tokens(beam_idx) - r.py_orig_prompt_len + ) + pending + step < r.py_min_length[0]: + bans.rows.append(request_offset + num_steps[index] * beam_idx + step) + bans.cols.append(end_id) + else: + break + + def _add_no_repeat_ngram_bans( + self, + bans: TokenBans, + requests: list[LlmRequest], + num_steps: list[int], + num_beams: list[int], + ngram_sizes: list[int | None], + *, + stale_by_one: list[bool] | None, + ) -> None: + """Add bans for tokens that would repeat an existing n-gram. + + For no-repeat-ngram size ``n``, the last ``n - 1`` tokens of the + sequence (prompt + generated) form the current prefix; the final token + of every existing n-gram whose first ``n - 1`` tokens equal that prefix + is masked to ``-inf`` (same semantics as the C++ ``banRepeatNgram`` + kernel and HF's ``NoRepeatNGramLogitsProcessor``). ``n == 1`` bans every + token already present. The restriction is per-beam. + + On the stale-host overlap path the rules cover the true sequence + ``tokens + [d]``: the window ending at ``d`` reduces to a host-known + column, and ``n == 1`` additionally bans ``d`` itself (``stale_extra``). + + ``ngram_sizes`` holds the effective per-request size (None or a + non-positive value disables the restriction). + """ + + def fresh_rules( + index: int, r: LlmRequest, get_context: Callable[[], list[int]] + ) -> list[BanRule]: + # Every existing n-gram is the suffix rule "after its first n - 1 + # tokens, ban its last token". + n = ngram_sizes[index] + if not n: # None or non-positive: restriction disabled + return [] + if num_beams[index] == 1: + # Single beam: O(1) index lookup, rules come out pre-matched. + tokens, fresh_idx, _ = self._extend_ngram_index(r, n) + if len(tokens) < n: + return [] + key = tuple(tokens[-(n - 1) :]) if n > 1 else () + return [([], col) for col in fresh_idx.get(key, ())] + # Multi-beam: histories diverge, so scan this beam's context. + context = get_context() + length = len(context) + if length < n: + return [] + if n == 1: + return [([], col) for col in set(context)] + # Only a window whose head equals the context suffix can ever fire, + # and the suffix is known right here. Match now and emit pre-matched + # rules with an empty prefix (as the single-beam branch does) instead + # of handing every window to the driver: that built one prefix list + # per window each step, only for the driver to discard all but the + # few that match. + suffix = context[-(n - 1) :] + return [ + ([], context[i + n - 1]) + for i in range(length - n + 1) + if context[i : i + n - 1] == suffix + ] + + def stale_rules( + index: int, r: LlmRequest, get_context: Callable[[], list[int]] + ) -> list[BanRule]: + # Stale contract: rules are already host-matched; only the device + # comparison of d against prefix[-1] remains. + n = ngram_sizes[index] + if not n: # None or non-positive: restriction disabled + return [] + tokens, fresh_idx, stale_idx = self._extend_ngram_index(r, n) + if n == 1: + # Host tokens; d itself is banned via stale_extra below. + return [([], col) for col in fresh_idx.get((), ())] + if n == 2: + # Empty (n - 2)-gram head: every indexed bigram is a candidate; + # reconstruct the pairs from fresh_idx. + pairs: Iterable[tuple[int, int]] = ( + (key[0], col) for key, cols in fresh_idx.items() for col in cols + ) + else: + pairs = stale_idx.get(tuple(tokens[-(n - 2) :]), ()) + rules: list[BanRule] = [([expected], col) for expected, col in pairs] + # The window ending at d bans d itself; the match condition forces + # d == tokens[-1] (host-known). Check its host-side part here. + if len(tokens) >= n - 1 and (n == 2 or tokens[-(n - 1) : -1] == tokens[-(n - 2) :]): + rules.append(([tokens[-1]], tokens[-1])) + return rules + + def stale_extra(bans: TokenBans, request_offset: int, index: int, r: LlmRequest) -> None: + if ngram_sizes[index] == 1: + # n == 1 also bans the device-only token, via a gathered column. + assert r.py_seq_slot is not None + bans.dev_rows.append(request_offset) + bans.dev_slots.append(r.py_seq_slot) + + self._add_suffix_rule_bans( + bans, + requests, + num_steps, + num_beams, + active=lambda index, r: bool(ngram_sizes[index]), + fresh_rules=fresh_rules, + stale_rules=stale_rules, + stale_extra=stale_extra, + stale_by_one=stale_by_one, + ) + + def _extend_ngram_index( + self, request: LlmRequest, n: int + ) -> tuple[ + list[int], dict[tuple[int, ...], set[int]], dict[tuple[int, ...], set[tuple[int, int]]] + ]: + """Incrementally maintained n-gram index for a single-beam request. + + Returns ``(tokens, fresh_idx, stale_idx)``: a host-side mirror of the + token history, a map from each ``(n - 1)``-gram to the tokens that + followed it, and (for ``n > 2`` only) a map from each ``(n - 2)``-gram + to its ``(next, next-but-one)`` token pairs. The mirror grows via the + scalar ``get_num_tokens`` / ``get_last_tokens`` accessors — a full + ``get_tokens()`` copy costs ~40us at 2048 tokens — and each window is + indexed once, so the steady-state per-step cost is O(new tokens). + Rebuilt when the history shrinks (speculative rollback); single-beam + only. + """ + num_tokens = request.get_num_tokens(0) + cache = self._ngram_index_caches.get(request) + # A request's ngram size is fixed for its lifetime and the cache is + # keyed on the request, so the cached index is always for this ``n``. + if cache is not None and len(cache.tokens) <= num_tokens: + tokens, indexed_up_to = cache.tokens, cache.indexed_up_to + fresh_idx, stale_idx = cache.fresh_idx, cache.stale_idx + if len(tokens) == num_tokens - 1: + # Common decode step: exactly one new token since last call. + tokens.append(request.get_last_tokens(0)) + elif len(tokens) < num_tokens: + # Several tokens landed at once (e.g. accepted draft tokens). + tokens = request.get_tokens(0) + else: + # First use or shrunken history (speculative rollback). + tokens, indexed_up_to = request.get_tokens(0), 0 + fresh_idx, stale_idx = defaultdict(set), defaultdict(set) + end = max(len(tokens) - n + 1, 0) + for i in range(indexed_up_to, end): + # defaultdict(set) creates the entry on first access, avoiding an + # empty set constructed (and discarded) on every already-seen key. + fresh_idx[tuple(tokens[i : i + n - 1])].add(tokens[i + n - 1]) + if n > 2: + stale_idx[tuple(tokens[i : i + n - 2])].add((tokens[i + n - 2], tokens[i + n - 1])) + self._ngram_index_caches[request] = _NgramIndexCache( + tokens=tokens, indexed_up_to=end, fresh_idx=fresh_idx, stale_idx=stale_idx + ) + return tokens, fresh_idx, stale_idx + + # ---- shared suffix matcher + packed-row bookkeeping ------------------ + + def _add_suffix_rule_bans( + self, + bans: TokenBans, + requests: list[LlmRequest], + num_steps: list[int], + num_beams: list[int], + *, + active: Callable[[int, LlmRequest], bool], + fresh_rules: RuleGenerator, + stale_rules: RuleGenerator, + stale_extra: Callable[[TokenBans, int, int, LlmRequest], None] | None = None, + stale_by_one: list[bool] | None = None, + ) -> None: + """Shared matcher for suffix-rule token-ban features. + + A rule ``(prefix, col)`` bans token ``col`` when the sequence (prompt + + generated) ends with ``prefix``; an empty prefix bans unconditionally. + Both bad words and no-repeat ngram reduce to sets of such rules; this + driver owns the packed-row bookkeeping and the suffix matching. + + Stale-host overlap path: with the overlap scheduler, ``sample_async`` + for step ``i`` runs before ``update_requests`` for step ``i - 1``, so + the host token list misses exactly the previous step's token ``d``, + which still lives device-side in ``new_tokens_cuda[0, seq_slot, 0]``. + For requests flagged in ``stale_by_one``, a rule's suffix match against + the true sequence ``context + [d]`` is split into a host-side check of + ``prefix[:-1]`` and a device-side comparison of ``d`` against + ``prefix[-1]``, resolved at flush time without any device-to-host + synchronization. This path only supports ``num_steps == 1`` and + ``num_beams == 1`` (no speculation, no beam search). + + Args: + bans: Accumulator the collected bans are appended to; row indices + refer to the flattened ``[total_rows, vocab]`` logits layout + (``num_steps * num_beams`` consecutive rows per request, in + beam-major / step-minor order). + requests: The requests, aligned with the packed logits rows. + num_steps: Number of steps per request. + num_beams: Number of beams per request. + active: ``(index, request) -> bool``; requests where this is False + are skipped entirely. + fresh_rules: ``(index, request, get_context) -> iterable`` of + ``(prefix, col)`` rules; the driver matches them against the + beam context. ``get_context()`` fetches (and memoizes) the + beam's token history — a full copy through the bindings — so + generators should call it only when needed. + stale_rules: Like ``fresh_rules``, but for the true sequence + ``context + [d]`` and already host-matched by the generator: + the driver only compares ``d`` against ``prefix[-1]`` on the + device (empty prefix = unconditional). Prefixes and cols must + consist of host-known tokens. + stale_extra: Optional hook ``(bans, request_offset, index, request)`` + invoked once per stale request for bans the rule form cannot + express (e.g. banning the device token itself). + stale_by_one: Per-request flag; True when the host token list lags + the device state by exactly one token (overlap scheduler). + These bans need ``new_tokens_cuda`` when flushed. + """ + current_offset = 0 + for index, r in enumerate(requests): + request_offset = current_offset + # Advance to the next request's rows before any early continue. + current_offset += num_steps[index] * num_beams[index] + + if not active(index, r): + continue + + if stale_by_one is not None and stale_by_one[index]: + assert num_steps[index] == 1 and num_beams[index] == 1, ( + "stale-host token-ban path only supports a single step and beam" + ) + assert r.py_seq_slot is not None + # Rules arrive host-matched; the device comparison of d against + # prefix[-1] is resolved at flush time. + seen_uncond: set[int] = set() + seen_cond: set[tuple[int, int]] = set() + for prefix, col in stale_rules(index, r, _memoizing_tokens_getter(r, 0)): + if not prefix: + if col not in seen_uncond: + seen_uncond.add(col) + bans.rows.append(request_offset) + bans.cols.append(col) + continue + key = (prefix[-1], col) + if key in seen_cond: + continue + seen_cond.add(key) + bans.cond_rows.append(request_offset) + bans.cond_cols.append(col) + bans.cond_slots.append(r.py_seq_slot) + bans.cond_expected.append(prefix[-1]) + if stale_extra is not None: + stale_extra(bans, request_offset, index, r) + continue + + for beam_idx in range(num_beams[index]): + # The context (prompt + generated) is fetched at most once, + # and only if a rule with a non-empty prefix needs it. + get_context = _memoizing_tokens_getter(r, beam_idx) + seen_cols: set[int] = set() + for prefix, col in fresh_rules(index, r, get_context): + if col in seen_cols: + continue + k = len(prefix) + if k > 0: + context = get_context() + if len(context) < k or context[-k:] != prefix: + continue + seen_cols.add(col) + # Apply to every step row of this beam. With speculation + # the banned set for later steps is approximated from the + # host context. + for step in range(num_steps[index]): + bans.rows.append(request_offset + num_steps[index] * beam_idx + step) + bans.cols.append(col) + + # ---- shared unconditional apply (both variants) ---------------------- + + @staticmethod + @torch.inference_mode() + def _apply_unconditional_bans(logits: torch.Tensor, bans: TokenBans) -> None: + """Write the unconditional bans (``bans.rows`` / ``bans.cols``) to -inf. + + Rows and cols are packed into one ``[2, N]`` tensor so the host-to-device + copy is a single transfer; the two index rows are split on the device. + """ + if not bans.rows: + return + neg_inf = torch.full((), float("-inf"), dtype=logits.dtype, device=logits.device) + rowcol_idx = torch.tensor( + [bans.rows, bans.cols], dtype=torch.long, pin_memory=prefer_pinned() + ).to(logits.device, non_blocking=True) + logits.index_put_((rowcol_idx[0], rowcol_idx[1]), neg_inf, accumulate=False) + + +class SynchronousTokenBanHandler(TokenBanHandler): + """Token bans without the overlap scheduler: the host history is complete, + so every request is "fresh" and all bans are unconditional.""" + + def generate_ban_list( + self, + requests: list[LlmRequest], + num_steps: list[int], + num_beams: list[int], + ngram_sizes: list[int | None], + *, + stale_by_one: list[bool] | None = None, + pending_steps: list[int] | None = None, + ) -> TokenBans: + assert stale_by_one is None, "synchronous handler has no stale requests" + # Without the overlap scheduler update_requests has already run, so the + # host history is complete and no length correction applies. + assert not pending_steps, "synchronous handler has no pending steps" + bans = TokenBans() + if any(getattr(r, "py_min_length", None) for r in requests): + self._add_min_length_bans(bans, requests, num_steps, num_beams) + if any(getattr(r, "py_bad_words", None) for r in requests): + self._add_bad_words_bans(bans, requests, num_steps, num_beams, stale_by_one=None) + if any(size is not None for size in ngram_sizes): + self._add_no_repeat_ngram_bans( + bans, requests, num_steps, num_beams, ngram_sizes, stale_by_one=None + ) + # Without overlap there are no stale requests, hence no conditional bans. + assert not bans.cond_rows and not bans.dev_rows + return bans + + def apply_ban_list( + self, + logits: torch.Tensor, + bans: TokenBans, + *, + new_tokens_cuda: torch.Tensor | None = None, + ) -> None: + self._apply_unconditional_bans(logits, bans) + + +class OverlappedTokenBanHandler(TokenBanHandler): + """Token bans with the overlap scheduler: the host history can lag the + device by one token, so a batch mixes fresh and stale requests. Stale + requests produce conditional bans resolved on the device at apply time. + + The per-request ``stale_by_one`` flags are computed by the sampler (which + owns the pending-step counter and the draft-batch / step / beam-width + context) and passed into ``generate_ban_list``. + """ + + def generate_ban_list( + self, + requests: list[LlmRequest], + num_steps: list[int], + num_beams: list[int], + ngram_sizes: list[int | None], + *, + stale_by_one: list[bool] | None = None, + pending_steps: list[int] | None = None, + ) -> TokenBans: + bans = TokenBans() + if any(getattr(r, "py_min_length", None) for r in requests): + # No stale *suffix* variant, but the length count must still account + # for tokens pending write-back; see _add_min_length_bans. + self._add_min_length_bans( + bans, requests, num_steps, num_beams, pending_steps=pending_steps + ) + if any(getattr(r, "py_bad_words", None) for r in requests): + self._add_bad_words_bans( + bans, requests, num_steps, num_beams, stale_by_one=stale_by_one + ) + if any(size is not None for size in ngram_sizes): + self._add_no_repeat_ngram_bans( + bans, requests, num_steps, num_beams, ngram_sizes, stale_by_one=stale_by_one + ) + return bans + + def apply_ban_list( + self, + logits: torch.Tensor, + bans: TokenBans, + *, + new_tokens_cuda: torch.Tensor | None = None, + ) -> None: + self._apply_unconditional_bans(logits, bans) # fresh requests (+ min_length) + self._apply_conditional_bans(logits, bans, new_tokens_cuda) # stale requests + + @staticmethod + @torch.inference_mode() + def _apply_conditional_bans( + logits: torch.Tensor, + bans: TokenBans, + new_tokens_cuda: torch.Tensor | None, + ) -> None: + """Write the overlap/stale bans, resolving the device-token comparison. + + ``dev_*``: the banned column is the device token ``d`` itself. + ``cond_*``: ban only where ``d == expected``; the additive + ``torch.where`` update keeps the op shape-static (boolean-mask indexing + would force a device-to-host sync). + """ + neg_inf = torch.full((), float("-inf"), dtype=logits.dtype, device=logits.device) + device = logits.device + + if bans.dev_rows: + assert new_tokens_cuda is not None + dev_row_idx = torch.tensor( + bans.dev_rows, dtype=torch.long, pin_memory=prefer_pinned() + ).to(device, non_blocking=True) + dev_slot_idx = torch.tensor( + bans.dev_slots, dtype=torch.long, pin_memory=prefer_pinned() + ).to(device, non_blocking=True) + prev_tokens = new_tokens_cuda[0].index_select(0, dev_slot_idx)[:, 0].to(torch.long) + logits.index_put_((dev_row_idx, prev_tokens), neg_inf, accumulate=False) + + if bans.cond_rows: + assert new_tokens_cuda is not None + cond_row_idx = torch.tensor( + bans.cond_rows, dtype=torch.long, pin_memory=prefer_pinned() + ).to(device, non_blocking=True) + cond_col_idx = torch.tensor( + bans.cond_cols, dtype=torch.long, pin_memory=prefer_pinned() + ).to(device, non_blocking=True) + slot_idx = torch.tensor( + bans.cond_slots, dtype=torch.long, pin_memory=prefer_pinned() + ).to(device, non_blocking=True) + expected = torch.tensor( + bans.cond_expected, dtype=new_tokens_cuda.dtype, pin_memory=prefer_pinned() + ).to(device, non_blocking=True) + # Previous step's token per request (single step, single beam). + prev_tokens = new_tokens_cuda[0].index_select(0, slot_idx)[:, 0] + penalty = torch.where( + prev_tokens == expected, + neg_inf, + torch.zeros((), dtype=logits.dtype, device=device), + ) + logits.index_put_((cond_row_idx, cond_col_idx), penalty, accumulate=True) diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/top_p_decay.py b/tensorrt_llm/_torch/pyexecutor/sampler/top_p_decay.py new file mode 100644 index 000000000000..853b687fe3a7 --- /dev/null +++ b/tensorrt_llm/_torch/pyexecutor/sampler/top_p_decay.py @@ -0,0 +1,377 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Top-P Decay support for ``TorchSampler``. + +The feature's per-slot runtime state and its whole lifecycle live in +:class:`TopPDecayHandler`; ``TorchSampler`` owns one instance and drives it +through the four lifecycle hooks documented on :class:`TopPDecayStore`. +""" + +from dataclasses import dataclass +from typing import Optional + +import torch + +from tensorrt_llm._utils import prefer_pinned + +from ..llm_request import LlmRequest, get_draft_token_length +from .ops.vanilla import Fusions +from .sampler_common import _request_get_sampling_params +from .sampler_strategy import TopPDecayMetadata, top_p_decay_active + +__all__ = ["TopPDecayHandler", "TopPDecayStore"] + + +@dataclass(kw_only=True) +class TopPDecayStore: + """Per-slot runtime state for Top-P Decay -- the single source of truth + for the feature's semantics on the torch path. + + Semantics (matching the legacy C++ ``computeToppDecay`` kernel): after + every sampled token of a decay-active request:: + + runtime_top_p = initial_top_p if token == reset_id + = max(runtime_top_p * top_p_decay, top_p_min) otherwise + + A negative ``reset_ids`` sentinel (-1, "reset disabled") never matches, + since sampled token ids are non-negative. Decay is active iff + ``top_p_decay`` is set and < 1.0 (``SamplingParams. + params_imply_top_p_decay_active``); an active decay forces a + top-p-capable strategy even for an otherwise implicitly-greedy request + (initial top-p defaults to 1.0), while explicit greedy controls win. + Beam search and speculative draft tokens are rejected at admission + (``TopPDecayHandler.validate_request``); parameter ranges are enforced by + ``SamplingParams._validate`` / the executor::SamplingConfig constructor. + + Lifecycle per slot (each tensor has shape ``(max_num_sequences,)``): + + 1. Admission (``TopPDecayHandler.setup_for_new_requests``): membership is + cleared then re-set for the newly-admitted slots -- both the host-side + ``TopPDecayHandler._slots`` set (an O(1) hot-path early-out) + and its device mirror ``is_top_p_decay_slot_cuda`` (the gate the + fused ops use, so the hot path needs no host-side filtering) -- + and the per-slot buffers are initialized. This clear-then-set also + covers slot reuse: stale entries from a prior occupant are never + consumed. + 2. Pre-sample (``TopPDecayHandler.build_metadata`` -> ``TopPDecayMetadata`` + -> ``TopPDecayMixin``): the per-row top-p fed to top_p / + top_k_top_p / min_p sampling is overridden with the decayed runtime value + for decay-active rows (fused gather, ``top_p_decay_gather``). + 3. Post-sample (``TopPDecayHandler.update_after_sample``): the recurrence + above is applied in place for the sampled decay-active slots (fused + update, ``top_p_decay_update``). + 4. Finish (``TopPDecayHandler.retire_slot``): the slot leaves the + membership set so the early-outs re-arm; the device buffers need no + cleanup (a freed slot is never sampled, reuse re-initializes it). + """ + + runtime_top_p_decay_cuda: torch.Tensor + """The current (decaying) top-p per slot; mutated post-sample each step.""" + initial_top_p_decay_cuda: torch.Tensor + """The initial top-p per slot; used to reset on a reset-id match.""" + top_p_decay_cuda: torch.Tensor + """Per-slot multiplicative decay factor.""" + top_p_decay_min_cuda: torch.Tensor + """Per-slot lower bound for the decayed top-p.""" + top_p_decay_reset_ids_cuda: torch.Tensor + """Per-slot reset token id (< 0 never matches a sampled token).""" + is_top_p_decay_slot_cuda: torch.Tensor + """Per-slot bool gate (device mirror of ``TopPDecayHandler._slots``). Lets the + fused post-sample update op filter decay-active slots on the GPU, + avoiding a host-side ``.tolist()`` / set intersection each step.""" + + @classmethod + def create(cls, max_num_sequences: int) -> "TopPDecayStore": + n = (max_num_sequences,) + return cls( + runtime_top_p_decay_cuda=torch.empty(n, dtype=torch.float32, device="cuda"), + initial_top_p_decay_cuda=torch.empty(n, dtype=torch.float32, device="cuda"), + top_p_decay_cuda=torch.empty(n, dtype=torch.float32, device="cuda"), + top_p_decay_min_cuda=torch.empty(n, dtype=torch.float32, device="cuda"), + top_p_decay_reset_ids_cuda=torch.empty(n, dtype=torch.int, device="cuda"), + # The gate buffer IS the gate, so (unlike the others) it must start + # False: a slot that was never admitted as decay-active must read + # False. + is_top_p_decay_slot_cuda=torch.zeros(n, dtype=torch.bool, device="cuda"), + ) + + +class TopPDecayHandler: + """Owns the Top-P Decay store and drives the feature's lifecycle. + + The host-side membership set is the sole gate for reading/updating the + per-slot :class:`TopPDecayStore` buffers, and doubles as an O(1) early-out + on the hot path: with no decay-active request the pre/post-sample hooks + return without touching the device. + """ + + def __init__(self, max_num_sequences: int): + # Allocated for all sampler instances; only slots in self._slots are + # ever read. + self.store = TopPDecayStore.create(max_num_sequences) + # Slots with an active top-p-decay request. Discarded on slot reuse so + # stale buffer entries are never consumed. + self._slots: set[int] = set() + + @property + def active(self) -> bool: + """Whether any resident request currently uses top-p decay.""" + return bool(self._slots) + + @staticmethod + def validate_request(request: LlmRequest) -> None: + """Reject unsupported combinations for a top-p-decay-active request. + + Top-p decay is supported only for single-token decode steps without beam + search. Called from validate_request (request admission), so a violating + request is failed individually instead of aborting the whole batch. + """ + params = _request_get_sampling_params(request) + # NB: value ranges need no re-check here. Every request enters through + # the executor::SamplingConfig constructor, which hard-validates + # top_p_decay in (0, 1], top_p_min in (0, 1] and top_p_reset_ids >= 0 + # (samplingConfig.cpp check* helpers) for all frontends. A reset id + # >= vocab_size is not checked anywhere but is semantically inert: it + # can never match a sampled token, i.e. it behaves as "reset disabled". + if not top_p_decay_active(params): + return + if params.use_beam_search: + raise ValueError("top_p_decay is not supported with beam search.") + # A non-zero draft length means the request carries speculative draft + # tokens and produces multiple tokens per step (req_num_steps = + # 1 + draft_token_length). One-model speculation (vanilla MTP, one-model + # Eagle3 / MTP-Eagle, SA, draft-target-one-model) uses its own + # SpecSamplerBase-derived sampler and never reaches TorchSampler; the + # drafter-based modes that DO flow draft tokens through TorchSampler + # (two-model draft-target, NGram, user-provided, two-model Eagle3 / + # MTP-Eagle) are what can make this length non-zero. top-p decay does not + # support these multi-token steps. + # NB: at admission time the draft tokens of drafter-based modes are + # usually not attached yet, so this check is best-effort. Two-model + # speculation (the only source of such requests in TorchSampler) is + # slated for removal, so in practice no speculative request reaches the + # decay path; a debug assert in build_metadata guards the + # invariant at sample time. + if get_draft_token_length(request) > 0: + raise ValueError( + "top_p_decay is not supported for requests carrying speculative " + "draft tokens (req_num_steps > 1). This covers the drafter-based " + "modes routed through TorchSampler (two-model draft-target, NGram, " + "user-provided, two-model Eagle3 / MTP-Eagle); one-model " + "speculation uses its own sampler and is unaffected." + ) + + def setup_for_new_requests( + self, + new_requests: list[LlmRequest], + *, + new_seq_slots_cuda_long: torch.Tensor, + ) -> None: + """Refresh top-p-decay membership and per-slot buffers for admitted requests + (lifecycle step 1, see :class:`TopPDecayStore`). + + Drops stale membership from prior occupants of the newly-admitted slots + (host set and device gate), then re-admits the decay-active requests and + initializes their per-slot store entries. Unsupported decay combinations + were already rejected per-request in validate_request at admission. + """ + # Clear the device decay gate for every newly-admitted slot (covers slot + # reuse: a slot previously decay-active but reused by a non-decay request + # must read False). Decay-active slots are then set True below. + decay_gate = self.store.is_top_p_decay_slot_cuda + decay_gate.index_fill_(0, new_seq_slots_cuda_long, False) + + decay_seq_slots: list[int] = [] + initial_top_p: list[float] = [] + top_p_decay: list[float] = [] + top_p_min: list[float] = [] + top_p_reset_ids: list[int] = [] + for request in new_requests: + slot = request.py_seq_slot + assert slot is not None + self._slots.discard(slot) + sampling_params = _request_get_sampling_params(request) + if not top_p_decay_active(sampling_params): + continue + self._slots.add(slot) + decay_seq_slots.append(slot) + # Initial runtime top-p defaults to 1.0 when top_p is unset. + initial_top_p.append( + sampling_params.top_p if sampling_params.top_p is not None else 1.0 + ) + # decay is guaranteed non-None and < 1.0 here (top_p_decay_active); + # min/reset fall back to the C++ runtime defaults when unset. + assert sampling_params.top_p_decay is not None + top_p_decay.append(sampling_params.top_p_decay) + top_p_min.append( + sampling_params.top_p_min if sampling_params.top_p_min is not None else 1e-6 + ) + top_p_reset_ids.append( + sampling_params.top_p_reset_ids + if sampling_params.top_p_reset_ids is not None + else -1 + ) + + if decay_seq_slots: + self._update_store_for_new_requests( + decay_seq_slots=decay_seq_slots, + initial_top_p=initial_top_p, + top_p_decay=top_p_decay, + top_p_min=top_p_min, + top_p_reset_ids=top_p_reset_ids, + ) + + def _update_store_for_new_requests( + self, + *, + decay_seq_slots: list[int], + initial_top_p: list[float], + top_p_decay: list[float], + top_p_min: list[float], + top_p_reset_ids: list[int], + ) -> None: + """Initialize per-slot Top-P Decay buffers for newly admitted decay requests. + + runtime_top_p and initial_top_p both start at the effective initial top-p; + the runtime value is decayed post-sample each step. + """ + store = self.store + device = store.runtime_top_p_decay_cuda.device + slots_cuda = torch.tensor( + decay_seq_slots, device="cpu", dtype=torch.int64, pin_memory=prefer_pinned() + ).to(device, non_blocking=True) + floats_host = torch.tensor( + [initial_top_p, top_p_decay, top_p_min], + device="cpu", + dtype=torch.float32, + pin_memory=prefer_pinned(), + ) + floats_cuda = floats_host.to(device, non_blocking=True) + initial_cuda = floats_cuda[0] + reset_ids_cuda = torch.tensor( + top_p_reset_ids, device="cpu", dtype=torch.int32, pin_memory=prefer_pinned() + ).to(device, non_blocking=True) + + store.runtime_top_p_decay_cuda.index_copy_(0, slots_cuda, initial_cuda) + store.initial_top_p_decay_cuda.index_copy_(0, slots_cuda, initial_cuda) + store.top_p_decay_cuda.index_copy_(0, slots_cuda, floats_cuda[1]) + store.top_p_decay_min_cuda.index_copy_(0, slots_cuda, floats_cuda[2]) + store.top_p_decay_reset_ids_cuda.index_copy_(0, slots_cuda, reset_ids_cuda) + # Enable the device gate for these decay-active slots (cleared for all new + # slots in setup_sampler_step just before this call). + store.is_top_p_decay_slot_cuda.index_fill_(0, slots_cuda, True) + + def build_metadata( + self, + *, + group_req_indices: torch.Tensor, + req_num_steps: torch.Tensor, + seq_slots: torch.Tensor, + seq_slots_cuda: torch.Tensor, + ) -> Optional[TopPDecayMetadata]: + """Build the Top-P Decay metadata for a top_p / top_k_top_p / min_p group. + + Lifecycle step 2, see :class:`TopPDecayStore`. Returns None when no request + currently uses decay. The metadata's ``slots`` tensor is aligned to the + group's per-STEP row order (matching group_strategies_per_step); + non-decay rows (possibly multi-step draft rows) are gated out on-device + by ``is_decay_slot``, so decay presence in the group is not checked + host-side: a group without decay rows samples every row with its static + top-p -- same result as returning None. + """ + if not self._slots: + return None + store = self.store + # Fast path (steady-state decoding): if every row in the group is + # single-token, the per-STEP row order equals the per-request order, and + # (group_req_indices being sorted ascending) a contiguous group's slots + # are just a slice of seq_slots_cuda -- no host layout build and no H2D + # copy. + first_req = int(group_req_indices[0].item()) + last_req = int(group_req_indices[-1].item()) + group_steps = req_num_steps[group_req_indices] + if last_req - first_req + 1 == group_req_indices.size(0) and ( + group_steps.max().item() == 1 + ): + per_step_slots_cuda = seq_slots_cuda[first_req : last_req + 1] + else: + # Build the per-STEP slot layout (each request contributes + # req_num_steps rows). + group_seq_slots = seq_slots[group_req_indices] + if __debug__: + # Internal invariant (stripped under python -O): a decay-active + # row is always single-token -- the only source of multi-step + # rows in TorchSampler is two-model speculation (slated for + # removal), and decay + draft tokens is rejected per-request at + # admission in validate_request. + decay_row_steps = group_steps[ + torch.isin(group_seq_slots, torch.tensor(list(self._slots))) + ] + assert decay_row_steps.numel() == 0 or decay_row_steps.max().item() == 1, ( + "top_p_decay row with req_num_steps != 1; decay + draft tokens " + "should have been rejected at admission" + ) + per_step_slots_cuda = torch.repeat_interleave(group_seq_slots.long(), group_steps).to( + seq_slots_cuda.device, non_blocking=True + ) + return TopPDecayMetadata( + slots=per_step_slots_cuda, + runtime_top_p=store.runtime_top_p_decay_cuda, + is_decay_slot=store.is_top_p_decay_slot_cuda, + ) + + def update_after_sample( + self, + *, + step_tokens: torch.Tensor, + sampled_slots_cuda: torch.Tensor, + ) -> None: + """Apply the post-sample decay recurrence for sampled decay-active slots. + + See :class:`TopPDecayStore` for the feature-level semantics (lifecycle + step 3). Restricting to the sampled slots avoids reading stale + new_tokens_cuda for slots that were not scheduled this iteration. + + ``step_tokens`` holds the token sampled this step per slot; decay is + single-token-only, so the caller passes local step 0, beam 0. + """ + # Host-side O(1) early-out: skip the kernel launch when no request uses + # decay; otherwise a single fused (torch.compile) op gates on + # is_decay_slot on-device and gathers the sampled token in place. + if not self._slots: + return + store = self.store + Fusions.top_p_decay_update( + runtime_top_p=store.runtime_top_p_decay_cuda, + initial_top_p=store.initial_top_p_decay_cuda, + top_p_decay=store.top_p_decay_cuda, + top_p_min=store.top_p_decay_min_cuda, + reset_ids=store.top_p_decay_reset_ids_cuda, + is_decay_slot=store.is_top_p_decay_slot_cuda, + step_tokens=step_tokens, + sampled_slots=sampled_slots_cuda, + ) + + def retire_slot(self, req: LlmRequest) -> None: + """Retire a finished request's slot from the top-p-decay membership set + (lifecycle step 4, see :class:`TopPDecayStore`), so the O(1) hot-path + early-outs re-arm once decay traffic drains. + + Callers ensure ``req`` has finished. Requests that finish outside the + sampler (e.g. cancellation) are covered by the slot-reuse cleanup at + admission instead. + """ + if self._slots and req.py_seq_slot is not None: + self._slots.discard(req.py_seq_slot) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py b/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py index acea0242edcb..d19051d683c4 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py @@ -201,6 +201,7 @@ def create( dist=dist, max_sessions=attention_dp_config.kv_cache_routing_max_sessions, fair_share_multiplier=attention_dp_config.kv_cache_routing_fair_share_multiplier, + new_conv_placement=attention_dp_config.kv_cache_routing_new_conv_placement, ) if ( @@ -797,10 +798,11 @@ def _sort_key(req_item): class ConversationAwareADPRouter(ADPRouter): """Pins each conversation to a single attention-DP rank: the first request - of a conversation is round-robined, and every later request with the same + of a conversation is placed by ``new_conv_placement`` (round-robin by + default, or least-queued), and every later request with the same ``conversation_id`` returns to that rank, keeping the conversation's - KV-cache prefix on one rank. Falls back to load-balanced round-robin when no - ``conversation_id`` is present. + KV-cache prefix on one rank. Requests without a ``conversation_id`` fall + back to the same ``new_conv_placement`` policy. """ # Default LRU cap on the conversation->rank map (entries are ~tens of @@ -812,11 +814,15 @@ def __init__( dist: "Distributed", max_sessions: int = DEFAULT_MAX_SESSIONS, fair_share_multiplier: float = 2.0, + new_conv_placement: str = "round_robin", ): super().__init__(dist) self._conv_to_rank: "OrderedDict[str, int]" = OrderedDict() self._max_sessions = max(1, int(max_sessions)) self._fair_share_multiplier = max(1.0, float(fair_share_multiplier)) + self._new_conv_placement = ( + "least_queued" if new_conv_placement == "least_queued" else "round_robin" + ) self._round_robin_cursor = 0 def create_rank_state( @@ -851,6 +857,42 @@ def _record_target_rank(self, conv_id: str, rank: int) -> None: while len(self._conv_to_rank) > self._max_sessions: self._conv_to_rank.popitem(last=False) + def _assign_new_conversation_explicit_dp_ranks( + self, + requests: List["RequestQueueItem"], + all_ranks_new_requests: Dict[int, List["RequestQueueItem"]], + all_ranks_num_active_requests: List[int], + max_num_active_requests: int, + ) -> List["RequestQueueItem"]: + """Place explicit first turns and establish their affinity binding. + + Once a conversation is bound, its recorded rank takes precedence over + later explicit rank hints. Requests whose explicit target is full remain + eligible for the normal affinity/load-balanced path below. + """ + remaining: List["RequestQueueItem"] = [] + for req_item in requests: + conv_id = self._conversation_id(req_item) + if conv_id is not None and conv_id in self._conv_to_rank: + remaining.append(req_item) + continue + + scheduling_params = getattr(req_item.request, "py_scheduling_params", None) + target_dp_rank = ( + scheduling_params.attention_dp_rank if scheduling_params is not None else None + ) + if ( + target_dp_rank is not None + and all_ranks_num_active_requests[target_dp_rank] < max_num_active_requests + ): + all_ranks_num_active_requests[target_dp_rank] += 1 + all_ranks_new_requests[target_dp_rank].append(req_item) + if conv_id is not None: + self._record_target_rank(conv_id, target_dp_rank) + else: + remaining.append(req_item) + return remaining + def route_requests( self, all_rank_states: list[RankState], @@ -871,8 +913,9 @@ def get_relax_value(req_item): sorted_requests = sorted(new_requests, key=get_relax_value) - # 1) Honour an explicit attention_dp_rank first (strict placement). - remaining_unscheduled = self._assign_explicit_dp_ranks( + # 1) Honour an explicit attention_dp_rank for a new conversation and + # record that placement. Existing conversations keep their binding. + remaining_unscheduled = self._assign_new_conversation_explicit_dp_ranks( sorted_requests, all_ranks_new_requests, all_ranks_num_active_requests, @@ -923,8 +966,13 @@ def _next_rr(soft_cap: int) -> int: if rank is None: # First turn of a new conversation, sticky-overflow, or no - # conversation_id -> round-robin spread under the soft cap. - rank = _next_rr(expected_num_active_requests) + # conversation_id -> spread under the soft cap: round-robin + # (count-uniform) or least-queued (steers away from ranks kept + # busy by heavy pinned conversations). + if self._new_conv_placement == "least_queued": + rank = _least_loaded(expected_num_active_requests) + else: + rank = _next_rr(expected_num_active_requests) if conv_id is not None and conv_id not in self._conv_to_rank: # Bind this new conversation to its first-turn rank. self._record_target_rank(conv_id, rank) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py index 22d7ac2a8821..caa8e3cb3de1 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py @@ -116,6 +116,22 @@ def _get_lora_task_id(req: LlmRequest): return (1, lora_id) +def _get_forced_context_chunk_size(req: LlmRequest) -> int: + next_point = min( + (point for point in req.expect_snapshot_points if point > req.context_current_position), + default=None, + ) + if next_point is None: + return req.context_remaining_length + next_position = min(next_point, req.prompt_len) + return max(0, next_position - req.context_current_position) + + +def _is_forced_context_chunk_boundary(req: LlmRequest, chunk_size: int) -> bool: + next_position = req.context_current_position + chunk_size + return next_position >= req.prompt_len or next_position in req.expect_snapshot_points + + class ScheduledRequests: """Scheduled requests separated into disjoint sets. @@ -664,6 +680,7 @@ def schedule( all_context_requests_fit = False if ctx_chunk_config and ctx_chunk_config.chunking_policy == ChunkingPolicy.FORCE_CHUNK: + # Run snapshot-boundary selection even when the full contexts fit. all_context_requests_fit = False # 3. Apply Chunking Strategy if needed @@ -886,8 +903,9 @@ def _chunk_fcfs( def _chunk_forced(self, requests: RequestList, capacity: Optional[int], unit_size: int): """Mirrors the kFORCE_CHUNK specialization of setCtxRequestsChunkSize (microBatchScheduler.cpp). - Every request is assigned exactly min(context_remaining_length, unit_size) tokens. - Requests that would exceed the capacity budget are zeroed out. + Requests advance to their next expected snapshot point. With no + remaining snapshot point, they consume the full remaining context. + Capacity-limited chunks are rounded down to a unit_size multiple. This policy is designed for linear attention / Mamba2 state caching, which doesn't support estimating reusable tokens, so we don't subtract them from the budget. @@ -899,9 +917,14 @@ def _chunk_forced(self, requests: RequestList, capacity: Optional[int], unit_siz ) total_tokens = 0 for req in requests: - req.context_chunk_size = min(req.context_remaining_length, unit_size) - if capacity is not None and total_tokens + req.context_chunk_size > capacity: - req.context_chunk_size = 0 + assert isinstance(req.expect_snapshot_points, list) + chunk_size = _get_forced_context_chunk_size(req) + if self.max_context_length is not None and chunk_size > self.max_context_length: + chunk_size = (self.max_context_length // unit_size) * unit_size + if capacity is not None and total_tokens + chunk_size > capacity: + remaining_capacity = max(0, capacity - total_tokens) + chunk_size = (min(chunk_size, remaining_capacity) // unit_size) * unit_size + req.context_chunk_size = int(chunk_size) total_tokens += req.context_chunk_size assert capacity is None or total_tokens <= capacity diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py index 2accbcc152b3..958afc59ac04 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py @@ -17,7 +17,7 @@ import os from typing import Optional -from tensorrt_llm.llmapi.llm_args import CapacitySchedulerPolicy +from tensorrt_llm.llmapi.llm_args import CapacitySchedulerPolicy, ContextChunkingPolicy from tensorrt_llm.logger import logger from ..llm_request import LlmRequest, LlmRequestState, get_draft_token_length @@ -25,7 +25,9 @@ RequestList, RequestScheduler, SchedulerOutput, + _get_forced_context_chunk_size, _get_lora_task_id, + _is_forced_context_chunk_boundary, drop_decoder_context_requests_waiting_for_encoder_output, ) @@ -175,8 +177,9 @@ def __init__( self.policy = scheduler_policy self.peft_cache_manager = peft_cache_manager - # Chunking config — only FCFS supported + # Chunking config. self.chunking_enabled = False + self.chunking_policy: Optional[ContextChunkingPolicy] = None self.chunk_unit_size = 0 self.max_context_length = max_num_tokens self.tokens_per_block = kv_cache_manager.tokens_per_block @@ -194,6 +197,7 @@ def __init__( ) if ctx_chunk_config is not None: self.chunking_enabled = True + self.chunking_policy = ctx_chunk_config[0] self.chunk_unit_size = ctx_chunk_config[1] # State value caches for fast comparison. @@ -574,10 +578,13 @@ def _try_schedule_context_chunked( """ remaining_budget = budget.remaining_tokens pre_prepare_context_remaining = req.context_remaining_length + force_chunk = self.chunking_policy == ContextChunkingPolicy.FORCE_CHUNK - # Min budget check — need at least one chunk unit - if remaining_budget is not None and remaining_budget < self.chunk_unit_size: - return ScheduleAction.SKIP, 0, False + if remaining_budget is not None: + no_budget = remaining_budget <= 0 + fcfs_under_min = not force_chunk and remaining_budget < self.chunk_unit_size + if no_budget or fcfs_under_min: + return ScheduleAction.SKIP, 0, False # Prepare context (create _KVCache, block reuse, resume — no resize) if not self.kv_cache_manager.prepare_context(req): @@ -587,22 +594,36 @@ def _try_schedule_context_chunked( # Calculate chunk size from remaining budget # (context_remaining_length is now correct after block reuse) context_remaining = req.context_remaining_length - budget_context_remaining = ( - context_remaining - if self.enable_prefix_aware_scheduling - else pre_prepare_context_remaining - ) - chunk_size = ( - min(remaining_budget, budget_context_remaining) - if remaining_budget is not None - else budget_context_remaining - ) + if force_chunk: + # Snapshot boundaries can be shorter than chunk_unit_size. + assert isinstance(req.expect_snapshot_points, list) + # With no remaining state to snapshot, avoid artificial boundaries. + chunk_size = _get_forced_context_chunk_size(req) + budget_context_remaining = context_remaining + else: + budget_context_remaining = ( + context_remaining + if self.enable_prefix_aware_scheduling + else pre_prepare_context_remaining + ) + chunk_size = ( + min(remaining_budget, budget_context_remaining) + if remaining_budget is not None + else budget_context_remaining + ) if self.max_context_length is not None: chunk_size = min(chunk_size, self.max_context_length) - # Round down to chunk_unit_size boundary (unless last chunk). - if chunk_size < budget_context_remaining: + chunk_size = min( + chunk_size, remaining_budget if remaining_budget is not None else chunk_size + ) + + # Round down to chunk_unit_size boundary only when not hitting the end + # or a checkpoint. + if chunk_size < budget_context_remaining and not ( + force_chunk and _is_forced_context_chunk_boundary(req, chunk_size) + ): chunk_size = (chunk_size // self.chunk_unit_size) * self.chunk_unit_size if chunk_size <= 0: diff --git a/tensorrt_llm/_torch/speculative/dflash.py b/tensorrt_llm/_torch/speculative/dflash.py index db564aeab4be..84db6eb9abf8 100644 --- a/tensorrt_llm/_torch/speculative/dflash.py +++ b/tensorrt_llm/_torch/speculative/dflash.py @@ -54,9 +54,7 @@ def dflash_draft_slot_ids( used, and slot 0 — the anchor token slot — predicts the first draft token. """ - request_bases = ( - torch.arange(num_gens, dtype=torch.long, device=device) * block_size - ) + request_bases = torch.arange(num_gens, dtype=torch.long, device=device) * block_size offsets = torch.arange(num_draft_tokens, dtype=torch.long, device=device) first_slot = 0 if shift_label else 1 return (request_bases.unsqueeze(1) + first_slot + offsets.unsqueeze(0)).flatten() @@ -131,10 +129,10 @@ def prepare(self): worker._ctx_len[slot] = 0 worker._free_slots.append(slot) - # Default to slot 0 for unknown request IDs (e.g. during warmup - # where synthetic requests may not have assigned slots). + # Route unknown request IDs (cuda graph padding or warmup dummies) + # to dummy slot to avoid corrupting real request's context mapping = torch.tensor( - [worker._req_to_slot.get(rid, 0) for rid in self.request_ids], + [worker._req_to_slot.get(rid, worker._dummy_slot) for rid in self.request_ids], dtype=torch.long, device="cpu", pin_memory=prefer_pinned(), @@ -201,24 +199,36 @@ def __init__( # graph compatible. self._ctx_buf_inited = False self._ctx_len = None + # Snapshot for rolling back in-place _ctx_len updates when a forward + # fails (or after warmup). See _ensure_spec_dec_state_restored. + self._saved_ctx_len = None + self._ctx_len_restore_pending = False + # Deferred kv_lens_cuda rewind state (see _prepare_kv_for_draft_forward, + # _apply_kv_rewind_after_draft, _ensure_spec_dec_state_restored). + self._kv_rewind_pending = False + self._kv_rewind_amount = None + self._kv_rewind_nc = None + self._kv_rewind_bs = None self._batch_to_slot = None self._max_ctx = 0 - self._ctx_k_buf = None # [max_batch, L, max_ctx+block, nkv, hd] + self._ctx_k_buf = None # [max_batch+1, L, max_ctx+block, nkv, hd] self._ctx_v_buf = None # Slot management (Python, updated in prepare() and eager mode) self._req_to_slot = {} # request_id -> slot index self._free_slots = deque() # available slot indices + self._dummy_slot = None # for cudagraph padding or warmup dummy requests # Opt-in acceptance-statistics recorder (None unless # TLLM_DFLASH_ACCEPT_STATS_DIR is set; see accept_stats.py). Only # consulted behind `is not None` checks — zero overhead when off. self._accept_stats = maybe_create_recorder( - spec_config.max_draft_len, getattr(mapping, "rank", 0) or 0) + spec_config.max_draft_len, getattr(mapping, "rank", 0) or 0 + ) if self._accept_stats is not None: logger.info( - f"DFlash: acceptance-statistics recording enabled -> " - f"{self._accept_stats.path}") + f"DFlash: acceptance-statistics recording enabled -> {self._accept_stats.path}" + ) logger.info( f"DFlashWorker initialized with use_separate_draft_kv_cache={use_separate_draft_kv_cache}" @@ -260,7 +270,13 @@ def _lazy_init_ctx_buffers(self, draft_model, spec_metadata, attn_metadata): dtype = draft_model.fc.weight.dtype if hasattr(draft_model, "fc") else torch.bfloat16 - self._ctx_len = torch.zeros(max_batch, dtype=torch.long, device="cuda") + # Reserve slot index max_batch as a scratch slot for padding/unknown + # dummies; real requests only draw slots 0..max_batch-1, so dummy + # writes land here and can't corrupt a real request's context. + self._dummy_slot = max_batch + num_slots = max_batch + 1 + + self._ctx_len = torch.zeros(num_slots, dtype=torch.long, device="cuda") self._batch_to_slot = torch.zeros(max_batch, dtype=torch.long, device="cuda") self._free_slots = deque(range(max_batch)) @@ -279,7 +295,7 @@ def _lazy_init_ctx_buffers(self, draft_model, spec_metadata, attn_metadata): L = draft_model._num_attn_layers nkv = draft_model._num_kv_heads hd = draft_model._head_dim - kv_shape = (max_batch, L, self._max_ctx + self._resolved_block_size, nkv, hd) + kv_shape = (num_slots, L, self._max_ctx + self._resolved_block_size, nkv, hd) self._ctx_k_buf = torch.zeros(kv_shape, dtype=dtype, device="cuda") self._ctx_v_buf = torch.zeros(kv_shape, dtype=dtype, device="cuda") self._ctx_buf_inited = True @@ -313,16 +329,20 @@ def _prepare_kv_for_draft_forward( if batch_size > num_contexts: attn_metadata.kv_lens_cuda[num_contexts:batch_size] += 1 + self._kv_rewind_pending = True attn_metadata.update_for_spec_dec() def _apply_kv_rewind_after_draft(self, attn_metadata, spec_metadata): """Apply the deferred kv_lens rewind after the draft forward.""" + self._kv_rewind_pending = False is_warmup = spec_metadata.is_cuda_graph and not torch.cuda.is_current_stream_capturing() if is_warmup: + # kv_lens_cuda was saved by prepare_for_spec_dec in this mode and + # is restored wholesale, so no rewind is needed. return - if hasattr(self, "_kv_rewind_amount") and hasattr(attn_metadata, "kv_lens_cuda"): + if self._kv_rewind_amount is not None and hasattr(attn_metadata, "kv_lens_cuda"): nc = self._kv_rewind_nc bs = self._kv_rewind_bs attn_metadata.kv_lens_cuda[nc:bs] -= self._kv_rewind_amount @@ -405,7 +425,28 @@ def _store_prefill_context( self._ctx_v_buf[slot, :, cur:end] = chunk_v.permute(1, 0, 2, 3) offset += slen - def forward( + def _ensure_spec_dec_state_restored(self, attn_metadata, spec_metadata): + # Restore first (in warmup mode kv_lens_cuda was saved and comes back + # wholesale), then apply any pending rewind for the other modes so a + # failed draft forward does not leave kv_lens_cuda incremented. + super()._ensure_spec_dec_state_restored(attn_metadata, spec_metadata) + if ( + getattr(self, "_kv_rewind_pending", False) + and attn_metadata is not None + and spec_metadata is not None + ): + self._apply_kv_rewind_after_draft(attn_metadata, spec_metadata) + if ( + getattr(self, "_ctx_len_restore_pending", False) + and self._ctx_len is not None + and self._saved_ctx_len is not None + ): + # A failed forward must not keep this iteration's in-place + # _ctx_len updates: roll back to the pre-forward snapshot. + self._ctx_len.copy_(self._saved_ctx_len) + self._ctx_len_restore_pending = False + + def _forward_impl( self, input_ids, position_ids, @@ -438,10 +479,18 @@ def forward( self._lazy_init_ctx_buffers(draft_model, spec_metadata, attn_metadata) spec_metadata._dflash_worker = self - # Save context lengths before warmup to prevent accumulation + # Save context lengths so both warmup and a failed forward can roll + # back the in-place _ctx_len updates made during drafting. is_warmup = spec_metadata.is_cuda_graph and not torch.cuda.is_current_stream_capturing() - if is_warmup: - saved_ctx_len = self._ctx_len.clone() + if not torch.cuda.is_current_stream_capturing(): + # Never allocate the snapshot while capturing a CUDA graph: the + # clone would live in the graph memory pool, and its replay-time + # writes could alias blocks reused by later captures. Rollback is + # only meaningful for eager/warmup forwards anyway; a failure + # during capture aborts the graph itself, and captured ops do not + # mutate _ctx_len until replay. + self._saved_ctx_len = self._ctx_len.clone() + self._ctx_len_restore_pending = True self._execute_guided_decoder_if_present(logits) @@ -452,13 +501,17 @@ def forward( # Opt-in acceptance recording (env-gated; eager-mode measurement # runs only). Skipped for CUDA-graph batches (capture/replay/warmup # use synthetic requests and forbid the host sync). - if (self._accept_stats is not None and num_gens > 0 - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None): + if ( + self._accept_stats is not None + and num_gens > 0 + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + ): self._accept_stats.on_accept( spec_metadata.request_ids[num_contexts:batch_size], - num_accepted_tokens[num_contexts:batch_size].tolist()) + num_accepted_tokens[num_contexts:batch_size].tolist(), + ) # Update GDN/Mamba recurrent states to the accepted token's state. if num_gens > 0 and isinstance(attn_metadata.kv_cache_manager, MambaHybridCacheManager): @@ -492,7 +545,10 @@ def forward( # Rebuild batch_to_slot after prefill assigns new slots if self._ctx_buf_inited and spec_metadata.request_ids: num_seqs = len(spec_metadata.request_ids) - mapping = [self._req_to_slot.get(rid, 0) for rid in spec_metadata.request_ids] + mapping = [ + self._req_to_slot.get(rid, self._dummy_slot) + for rid in spec_metadata.request_ids + ] self._batch_to_slot[:num_seqs].copy_( torch.tensor(mapping, dtype=torch.long, device="cuda") ) @@ -560,18 +616,21 @@ def forward( # DSpark confidence-scheduled verification MR supplies the # real provider, see accept_stats.py). Guarded off for # CUDA-graph batches and d2t vocab-mapped drafters. - if (self._accept_stats is not None - and self._accept_stats.confidence_provider is not None - and not spec_metadata.is_cuda_graph - and not torch.cuda.is_current_stream_capturing() - and spec_metadata.request_ids is not None - and self._d2t is None): + if ( + self._accept_stats is not None + and self._accept_stats.confidence_provider is not None + and not spec_metadata.is_cuda_graph + and not torch.cuda.is_current_stream_capturing() + and spec_metadata.request_ids is not None + and self._d2t is None + ): self._accept_stats.record_draft_confidence( spec_metadata.request_ids[num_contexts:batch_size], draft_model, - hidden_states_out[gen_gather_ids].reshape( - num_gens, K, -1), - inputs["first_prev_tokens"], gen_draft_tokens) + hidden_states_out[gen_gather_ids].reshape(num_gens, K, -1), + inputs["first_prev_tokens"], + gen_draft_tokens, + ) else: gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") @@ -601,9 +660,10 @@ def forward( num_accepted_tokens, ) - # Restore context lengths after warmup + # Restore context lengths after warmup; real runs keep the updates. if is_warmup: - self._ctx_len.copy_(saved_ctx_len) + self._ctx_len.copy_(self._saved_ctx_len) + self._ctx_len_restore_pending = False return { "logits": raw_logits, diff --git a/tensorrt_llm/_torch/speculative/draft_target.py b/tensorrt_llm/_torch/speculative/draft_target.py index 6df31b2bd223..6edeac334fdd 100644 --- a/tensorrt_llm/_torch/speculative/draft_target.py +++ b/tensorrt_llm/_torch/speculative/draft_target.py @@ -153,7 +153,7 @@ def _update_kv_for_chained_draft_step( attn_metadata.update_for_spec_dec() - def forward( + def _forward_impl( self, input_ids, position_ids, diff --git a/tensorrt_llm/_torch/speculative/drafting_loops.py b/tensorrt_llm/_torch/speculative/drafting_loops.py index 231f7f21c147..133814bb2d22 100644 --- a/tensorrt_llm/_torch/speculative/drafting_loops.py +++ b/tensorrt_llm/_torch/speculative/drafting_loops.py @@ -15,7 +15,8 @@ import torch from tensorrt_llm._torch.attention_backend.interface import AttentionMetadata -from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import greedy +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import \ + greedy_search_sampling_batch from tensorrt_llm._torch.speculative.eagle3 import Eagle3SpecMetadata from tensorrt_llm._torch.speculative.interface import SpecMetadata from tensorrt_llm._torch.speculative.spec_tree_manager import SpecTreeManager @@ -150,7 +151,7 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor, def sample(self, logits: torch.Tensor) -> torch.Tensor: # TODO: inject the sampler here so we can support non-greedy - tokens, _ = greedy(logits, return_probs=False) + tokens, _ = greedy_search_sampling_batch(logits, return_probs=False) if hasattr(self.draft_model.model, "d2t"): d2t = self.draft_model.model.d2t.data return tokens + d2t[tokens] diff --git a/tensorrt_llm/_torch/speculative/dspark.py b/tensorrt_llm/_torch/speculative/dspark.py new file mode 100644 index 000000000000..7f43f103ec4e --- /dev/null +++ b/tensorrt_llm/_torch/speculative/dspark.py @@ -0,0 +1,613 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DSpark worker / metadata mirror the DFlash plumbing (capture target-layer +# hidden states, accept the previous block with standard verification, draft a +# new block in one backbone forward), adapted to DSpark's draft model which +# produces the whole block (and its confidence-truncated length) inside a single +# ``DSparkDraftModel.forward`` rather than via mask-token cross-attention. + +from collections import deque +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional + +import torch + +from tensorrt_llm._utils import prefer_pinned +from tensorrt_llm.logger import logger +from tensorrt_llm.mapping import Mapping + +from ..pyexecutor.llm_request import ATTENTION_DP_DUMMY_REQUEST_ID +from .interface import SpecMetadata, SpecWorkerBase + +if TYPE_CHECKING: + from ...llmapi.llm_args import DSparkDecodingConfig + + +@dataclass +class DSparkSpecMetadata(SpecMetadata): + """Metadata for DSpark speculative decoding. + + Captures hidden states from the target model's ``layers_to_capture`` during + the target forward pass. DSpark captures the *mean over the multi-head + (mHC) residual streams* at each captured layer (handled by the target-side + capture hook), concatenated across layers, and feeds them to the draft + model's ``main_proj`` + ``main_norm`` (inside ``DSparkDraftModel.forward``) + as the captured-context attention input (``main_x``). + + Mirrors :class:`DFlashSpecMetadata`; the only DSpark-specific detail is that + the per-layer captured width is the model hidden size (post hc-mean), so the + buffer is ``[max_num_tokens, hidden_size * num_capture_layers]``. + """ + + batch_indices_cuda: Optional[torch.Tensor] = None + + # Hidden state capture fields + layers_to_capture: Optional[List[int]] = None + hidden_size: int = 0 + max_num_tokens: int = 0 + dtype: torch.dtype = torch.bfloat16 + captured_hidden_states: Optional[torch.Tensor] = None + + def __post_init__(self): + self.batch_indices_cuda = torch.empty( + [self.max_num_requests], + dtype=torch.int, + device="cuda", + ) + + self.is_spec_dec_tree = False + self.is_spec_dec_dynamic_tree = False + + # Set up hidden state capture buffer + if self.layers_to_capture is not None and len(self.layers_to_capture) > 0: + self.layers_to_capture = sorted(list(self.layers_to_capture)) + self.num_capture_layers = len(self.layers_to_capture) + # O(1) lookups for is_layer_capture() and maybe_capture_hidden_states() + self._capture_layer_set = frozenset(self.layers_to_capture) + self._layer_to_idx = {lid: i for i, lid in enumerate(self.layers_to_capture)} + self.captured_hidden_states = torch.empty( + (self.max_num_tokens, self.hidden_size * self.num_capture_layers), + dtype=self.dtype, + device="cuda", + ) + logger.info( + f"DSpark: capturing hidden states from layers {self.layers_to_capture}, " + f"buffer shape {self.captured_hidden_states.shape}" + ) + else: + self.num_capture_layers = 0 + self._capture_layer_set = frozenset() + self._layer_to_idx = {} + + def prepare(self): + assert self.request_ids is not None + num_seqs = len(self.request_ids) + batch_indices = torch.arange( + num_seqs, dtype=torch.int, device="cpu", pin_memory=prefer_pinned() + ) + self.batch_indices_cuda[:num_seqs].copy_(batch_indices, non_blocking=True) + + # CUDA-graph-safe path: maintain the request->slot mapping on the host + # (outside the captured region) and mirror it into ``_batch_to_slot`` so the + # captured gen forward can index the rolling windows by tensor. Mirrors + # ``DFlashSpecMetadata.prepare`` (dflash.py:96-113). + worker = getattr(self, "_dspark_worker", None) + if worker is not None and worker._win_inited: + current = set(self.request_ids) + for rid in list(worker._req_to_slot.keys()): + if rid not in current: + slot = worker._req_to_slot.pop(rid) + worker._ctx_len[slot] = 0 + worker._kv_windows[slot].zero_() + worker._free_slots.append(slot) + # Assign a persistent rolling-window slot to every real generation + # request that never ran a context/seed forward on this worker. In + # disaggregated serving the prompt is prefilled (and the window + # seeded) on the *context* server, so ``_seed_context_windows`` never + # runs on the generation server and ``_req_to_slot`` stays empty; + # without this, all concurrent gen requests fall through to the shared + # scratch row below and corrupt each other's draft window at batch + # size > 1 (GitHub #16767). Context-prefix entries are left to + # ``_seed_context_windows``; the ADP-idle (id 0) and CUDA-graph + # padding dummies are kept on the scratch row. + num_contexts = max(0, len(self.request_ids) - self.num_generations) + for rid in self.request_ids[num_contexts:]: + if ( + rid != ATTENTION_DP_DUMMY_REQUEST_ID + and rid < worker._graph_dummy_id_floor + and rid not in worker._req_to_slot + ): + worker._assign_slot(rid, reset=False) + # Unknown request IDs (synthetic warmup / CUDA-graph padding, ADP idle + # requests, or disagg seed forwards without a real id) map to the + # dedicated throwaway scratch row so they cannot overwrite a live + # request's rolling window (they previously aliased to slot 0). + scratch = worker._scratch_slot + mapping = torch.tensor( + [worker._req_to_slot.get(rid, scratch) for rid in self.request_ids], + dtype=torch.long, + device="cpu", + pin_memory=prefer_pinned(), + ) + worker._batch_to_slot[:num_seqs].copy_(mapping, non_blocking=True) + + def is_layer_capture(self, layer_id: int) -> bool: + return layer_id in self._capture_layer_set + + def maybe_capture_hidden_states( + self, layer_id: int, hidden_states: torch.Tensor, residual: Optional[torch.Tensor] = None + ) -> None: + """Capture hidden states from a target model layer into the buffer. + + DeepSeek-V4 keeps the multi-head (mHC) residual stream flattened as + ``[num_tokens, hc_mult * hidden]``; DSpark captures the *mean over the hc + streams* (reference ``h.mean(dim=2)`` with ``h`` shaped + ``[*, hc_mult, hidden]``). We reduce here so the V4 decoder layer's + existing capture call is unchanged. A ``[num_tokens, hidden]`` input + (already reduced / non-mHC) is stored as-is. + """ + if self.captured_hidden_states is None: + return + i = self._layer_to_idx.get(layer_id) + if i is not None: + num_tokens = hidden_states.shape[0] + to_save = hidden_states + residual if residual is not None else hidden_states + # mHC residual -> mean over the hc_mult streams. + if to_save.shape[-1] != self.hidden_size: + hc_mult = to_save.shape[-1] // self.hidden_size + to_save = to_save.reshape(num_tokens, hc_mult, self.hidden_size).mean(dim=1) + self.captured_hidden_states[ + :num_tokens, i * self.hidden_size : (i + 1) * self.hidden_size + ].copy_(to_save, non_blocking=True) + + def get_hidden_states(self, num_tokens: int) -> Optional[torch.Tensor]: + """Get captured hidden states (all layers concatenated).""" + if self.captured_hidden_states is None: + return None + return self.captured_hidden_states[ + :num_tokens, : self.hidden_size * self.num_capture_layers + ] + + +class DSparkWorker(SpecWorkerBase): + """Worker for DSpark speculative decoding. + + DSpark drafts a whole block of ``block_size`` tokens in one backbone forward + (``DSparkDraftModel.forward``): it projects the captured target-layer hidden + states (``main_proj`` + ``main_norm``) into the draft's captured-context + attention, runs the ``num_stages`` DSpark blocks over a rolling captured + window, refines the per-position logits with the Markov head, and predicts a + per-position acceptance confidence used to truncate the proposed prefix. + + Unlike DFlash, the draft does NOT use the paged KV cache or mask-token + cross-attention: its attention K/V come from the worker-owned rolling window + of projected captured context (one ``main_kv`` per decode step, per stage). + Acceptance of the previous block goes through the unified + :meth:`SpecWorkerBase.sample_and_accept_draft_tokens` (strict target-verify, + or rejection sampling for a non-greedy batch), so greedy parity with no-spec + is preserved regardless of draft quality. + + The rolling window is kept consistent across the whole decode: it is seeded + from the prompt's captured context at prefill and back-filled with the + intermediate accepted tokens of a multi-accept step (both via + ``DSparkDraftModel.write_context_windows``), in addition to the per-step bonus + write done by the generation path. These affect draft acceptance rate only, + not correctness, which the standard target verify guarantees. + + Reference: DeepSeek DeepSpec (https://github.com/deepseek-ai/DeepSpec). + """ + + def __init__( + self, + spec_config: "DSparkDecodingConfig", + mapping: Mapping, + use_separate_draft_kv_cache: bool = False, + ): + super().__init__(use_separate_draft_kv_cache) + self.spec_config = spec_config + self.mapping = mapping + + # Per-slot rolling captured-context KV windows, built lazily on the + # first forward (fixed-size for slot-indexed reads/writes). + self._win_inited = False + self._kv_windows: Optional[torch.Tensor] = None # [max_batch, num_stages, win, hd] + self._ctx_len: Optional[torch.Tensor] = None # [max_batch] abs decode position + self._win = 0 + + # Slot management. ``_req_to_slot`` (python dict) + ``_free_slots`` are the + # source of truth, updated in prepare()/forward(); ``_batch_to_slot`` is the + # CUDA mirror (request-order -> slot) read by the CUDA-graph-safe batched + # gen path (set on the host in prepare(), so the captured forward indexes + # the rolling windows through a tensor instead of a python dict lookup). + self._req_to_slot = {} # request_id -> slot index + self._free_slots = deque() # available slot indices + self._batch_to_slot: Optional[torch.Tensor] = None # [max_batch] long, cuda + # Index of the throwaway "scratch" window row that absorbs padded / + # unknown request IDs (set in ``_lazy_init`` to ``max_batch``); it is + # never handed out through ``_free_slots``. + self._scratch_slot = 0 + + # The generation draft path is the batched, host-sync-free + # ``_draft_gen_block_batched`` + ``DSparkDraftModel.forward_batched`` + + # ``dspark_attention_forward_batched``: it is correct in eager mode AND safe + # to capture into the target's CUDA graph (DSpark is a one-engine drafter — + # its worker forward runs inside that graph, so the draft path MUST be + # capture-safe whenever ``cuda_graph_config`` is set). + + logger.info( + f"DSparkWorker initialized with " + f"use_separate_draft_kv_cache={use_separate_draft_kv_cache}" + ) + + @property + def max_draft_len(self) -> int: + return self.spec_config.max_draft_len + + def _lazy_init(self, draft_model, spec_metadata) -> None: + block_size = int(draft_model.block_size) + if block_size != self.max_draft_len: + raise ValueError( + "DSpark draft model block_size must equal worker max_draft_len; " + f"got block_size={block_size} and max_draft_len={self.max_draft_len}" + ) + + if self._win_inited: + return + max_batch = spec_metadata.max_num_requests + num_stages = draft_model.num_stages + self._win = int(draft_model._attn_params["window_size"]) + head_dim = int(draft_model._attn_params["head_dim"]) + + # Real requests occupy slots ``[0, max_batch)``; one extra "scratch" row + # at index ``max_batch`` absorbs padded / unknown request IDs (CUDA-graph + # padding, ADP idle requests, or disagg seed forwards that arrive without + # a real request id) so they can never overwrite a live request's rolling + # window. Previously such IDs aliased to slot 0 and corrupted whichever + # real request occupied it. The scratch row is never handed out through + # ``_free_slots`` and its contents are throwaway. + self._scratch_slot = max_batch + num_rows = max_batch + 1 + + # CUDA-graph padding requests carry ids in + # ``[CUDA_GRAPH_DUMMY_REQUEST_ID - runtime_draft_len, CUDA_GRAPH_DUMMY_REQUEST_ID]``, + # while real request ids start at ``max_batch_size`` and grow, so a simple + # floor cleanly separates them. Together with ``ATTENTION_DP_DUMMY_REQUEST_ID`` + # (0) these dummies must route to the scratch row (see ``prepare()``) and + # never consume a real slot. Imported lazily to break the + # dspark -> cuda_graph_runner -> speculative.utils -> dspark import cycle. + from ..pyexecutor.cuda_graph_runner import CUDA_GRAPH_DUMMY_REQUEST_ID + + self._graph_dummy_id_floor = CUDA_GRAPH_DUMMY_REQUEST_ID - self.max_draft_len + + self._kv_windows = torch.zeros( + (num_rows, num_stages, self._win, head_dim), + dtype=torch.bfloat16, + device="cuda", + ) + self._ctx_len = torch.zeros(num_rows, dtype=torch.long, device="cuda") + self._batch_to_slot = torch.zeros(max_batch, dtype=torch.long, device="cuda") + self._free_slots = deque(range(max_batch)) + self._req_to_slot = {} + self._win_inited = True + logger.info( + f"DSpark: allocated rolling KV windows " + f"[{num_rows}, {num_stages}, {self._win}, {head_dim}] " + f"({max_batch} request slots + 1 scratch row)" + ) + + def _assign_slot(self, req_id: int, reset: bool) -> int: + """Get (or refresh) the slot for a request; reset clears its window.""" + if reset and req_id in self._req_to_slot: + old = self._req_to_slot.pop(req_id) + self._ctx_len[old] = 0 + self._kv_windows[old].zero_() + self._free_slots.append(old) + if req_id not in self._req_to_slot: + if not self._free_slots: + raise RuntimeError( + "DSpark has no free rolling-window slots for request " + f"{req_id}; increase max_num_requests" + ) + slot = self._free_slots.popleft() + self._req_to_slot[req_id] = slot + self._ctx_len[slot] = 0 + self._kv_windows[slot].zero_() + return self._req_to_slot[req_id] + + def _seed_context_windows( + self, + draft_model, + spec_metadata: "DSparkSpecMetadata", + attn_metadata, + position_ids: torch.Tensor, + total_target_tokens: int, + ) -> None: + """Seed context chunks using their absolute positions. + + A request can arrive in multiple prefill chunks. Only its first chunk + starts at position zero and resets the persistent rolling window; + continuation chunks append to the same request slot. + """ + captured = spec_metadata.get_hidden_states(total_target_tokens) + flat_position_ids = position_ids.reshape(-1) + context_offset = 0 + for i in range(attn_metadata.num_contexts): + chunk_len = int(attn_metadata._seq_lens[i]) + chunk_positions = flat_position_ids[context_offset : context_offset + chunk_len].long() + if chunk_len == 0: + context_offset += chunk_len + continue + + req_id = spec_metadata.request_ids[i] + first_position = int(chunk_positions[0].item()) + slot = self._assign_slot(req_id, reset=first_position == 0) + self._ctx_len[slot] = chunk_positions[-1] + 1 + + if captured is not None: + keep = min(self._win, chunk_len) + hidden = captured[context_offset + chunk_len - keep : context_offset + chunk_len] + # A prompt token at absolute position p is stored in frame p+1, + # matching the generation path's start_pos convention. + window_positions = chunk_positions[-keep:] + 1 + draft_model.write_context_windows(hidden, window_positions, self._kv_windows[slot]) + context_offset += chunk_len + + def _draft_gen_block_batched( + self, + draft_model, + spec_metadata: "DSparkSpecMetadata", + attn_metadata, + accepted_tokens: torch.Tensor, + num_accepted_tokens: torch.Tensor, + num_contexts: int, + batch_size: int, + total_target_tokens: int, + all_rank_num_tokens: Optional[List[int]] = None, + ) -> torch.Tensor: + """CUDA-graph-safe batched gen draft (all gen requests in one forward). + + Free of host syncs and data-dependent shapes: per-request quantities + (``nacc``, the bonus, ``main_hidden``, ``start_pos``, the multi-accept + back-fill) are gathered as tensors, slots come from the host-built + ``_batch_to_slot`` mirror, and the backbone runs once via + ``DSparkDraftModel.forward_batched``. Returns the per-position corrected + block logits ``[num_gens, K, vocab]`` (or ``None`` when there is nothing to + draft); the worker feeds them to ``SpecWorkerBase.sample_draft_tokens``. + Confidence truncation stays disabled — the full block is proposed. + """ + num_gens = batch_size - num_contexts + K = self.max_draft_len + Kp1 = K + 1 + device = accepted_tokens.device + + if num_gens == 0: + return None + captured = spec_metadata.get_hidden_states(total_target_tokens) + if captured is None: + return None + + # gen-only graph batches have num_ctx_tokens == 0; mixed eager batches put + # the gen tokens after the context tokens. + gen_start = attn_metadata.num_ctx_tokens + slots = self._batch_to_slot[num_contexts:batch_size] # [G] + nacc = num_accepted_tokens[num_contexts:batch_size].long() # [G] + gidx = nacc - 1 # [G] index of the bonus within each verified prefix + + # Bonus token = last accepted token of the verified prefix. + bonus = ( + accepted_tokens[num_contexts:batch_size].gather(1, gidx.unsqueeze(1)).squeeze(1).long() + ) # [G] + + # Captured target hidden at the bonus position within each request's Kp1 + # processed tokens. + arange_g = torch.arange(num_gens, device=device) + base = gen_start + arange_g * Kp1 # [G] + main_hidden = captured[base + gidx] # [G, ncap*hidden] + + # Fixed-size ([G, K]) masked back-fill of the intermediate accepted tokens + # (everything but the bonus) into the rolling window — same frames as the + # eager path (old+1 .. old+nacc-1), with j >= nacc-1 masked out. + old = self._ctx_len[slots] # [G] pre-increment decode position + j = torch.arange(K, device=device) # [K] + interim_valid = j.unsqueeze(0) < (nacc.unsqueeze(1) - 1) # [G, K] + interim_pos = old.unsqueeze(1) + 1 + j.unsqueeze(0) # [G, K] + interim_base = (base.unsqueeze(1) + j.unsqueeze(0)).clamp( + min=0, max=captured.shape[0] - 1 + ) # [G, K] (clamped; invalid entries are masked out anyway) + interim_hidden = captured[interim_base] # [G, K, ncap*hidden] + draft_model.write_context_windows_batched( + interim_hidden, interim_pos, slots, interim_valid, self._kv_windows + ) + + # Advance the decode position by the accepted count; start_pos (= post- + # increment ctx_len) matches the eager path's frame value. + start_pos = old + nacc # [G] + self._ctx_len[slots] = start_pos + + # Surface the per-position corrected block logits ([num_gens, K, vocab]) + # and let SpecWorkerBase.sample_draft_tokens do the (greedy or rejection) + # sampling + TP gather + draft_probs scatter, rather than argmaxing here. + _toks, _num_proposed, block_logits = draft_model.forward_batched( + main_hidden, + bonus, + start_pos, + kv_windows=self._kv_windows, + slots=slots, + temperature=0.0, + confidence_threshold=0.0, + return_logits=True, + all_rank_num_tokens=all_rank_num_tokens, + ) + return block_logits + + def _forward_impl( + self, + input_ids, + position_ids, + hidden_states, + logits, + attn_metadata, + spec_metadata, + draft_model, + resource_manager=None, + ): + batch_size = attn_metadata.num_seqs + num_contexts = attn_metadata.num_contexts + num_gens = batch_size - num_contexts + raw_logits = logits + K = self.max_draft_len + + self._lazy_init(draft_model, spec_metadata) + # Backref so DSparkSpecMetadata.prepare() can maintain the host slot map + # and mirror it into _batch_to_slot for the CUDA-graph-safe gen path. + spec_metadata._dspark_worker = self + self._execute_guided_decoder_if_present(logits) + + # Target-verify acceptance via the unified SpecWorkerBase entry: it + # reshapes the stored draft tokens (default (num_gens, runtime_draft_len) + # hook), then routes to strict or rejection sampling. Greedy parity with + # the previous hand-rolled path is preserved (rejection only engages for a + # non-greedy batch with valid draft_probs). + accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( + logits, attn_metadata, spec_metadata + ) + + total_target_tokens = input_ids.shape[0] + + # CUDA-graph warmup guard: the warmup forwards (is_cuda_graph set, stream + # NOT yet capturing) run synthetic gen batches that would otherwise advance + # the persistent rolling-window state. Snapshot and restore it so warmup is + # side-effect-free. (During the capture pass itself the stream IS capturing, + # so we skip the save/restore and let the ops be recorded; real requests + # reset their slot's window+ctx_len at prefill, wiping any capture-time + # mutation.) + is_warmup = ( + getattr(spec_metadata, "is_cuda_graph", False) + and not torch.cuda.is_current_stream_capturing() + ) + if is_warmup: + saved_ctx_len = self._ctx_len.clone() + saved_windows = self._kv_windows.clone() + + # Assign / reset window slots for context (prefill) requests and seed each + # request's rolling KV window from its prompt's captured context, so the + # first generation step drafts against real context instead of an all-zero + # window (acceptance-rate only; verified decoding keeps output correct). + if num_contexts > 0: + self._seed_context_windows( + draft_model, + spec_metadata, + attn_metadata, + position_ids, + total_target_tokens, + ) + + # FUSED_COMM MoE backends (DeepGEMM MegaMoE) synchronize EP ranks with an + # in-kernel phase-flip NVLink barrier that flips on every kernel call, so + # every rank must invoke the draft MoE the same number of times and with + # the same globally-gathered per-rank token list, or the barrier desyncs + # (hang / "unspecified launch failure"). The draft runs over generation + # requests only, each expanded to ``block`` positions, so the per-rank + # draft-MoE token count is ``num_gens * block``. ``all_rank_num_gens`` is + # gathered at metadata-prep time (model_engine, outside any CUDA-graph + # capture region); it is None for non-ADP / single-rank runs, where the + # local ``[num_tokens]`` fallback in ``_forward_stage`` is correct. + block = int(draft_model.block_size) + all_rank_num_gens = getattr(spec_metadata, "all_rank_num_gens", None) + # A rank with zero local gen requests still has to cross the draft MoE's + # cross-rank barrier, but DeepseekV4MoE's router / shared-expert dense + # GEMMs reject a 0-row input (cuBLAS CUBLAS_STATUS_INVALID_VALUE), so such + # a rank runs a single 1-row dummy through the MoE (like ADP padding). + # Encode that as ``1`` in the globally-shared per-rank token list so every + # rank agrees on the FUSED_COMM chunk count and per-rank slice. + all_rank_draft_tokens = ( + [max(1, int(g) * block) for g in all_rank_num_gens] + if all_rank_num_gens is not None + else None + ) + global_has_gen = ( + max(all_rank_num_gens) > 0 if all_rank_num_gens is not None else num_gens > 0 + ) + + if num_gens > 0: + # The batched gen-block draft returns the per-position corrected block + # logits [num_gens, K, vocab] and is CUDA-graph-safe. + gen_logits = self._draft_gen_block_batched( + draft_model, + spec_metadata, + attn_metadata, + accepted_tokens, + num_accepted_tokens, + num_contexts, + batch_size, + total_target_tokens, + all_rank_num_tokens=all_rank_draft_tokens, + ) + if gen_logits is not None: + # SpecWorkerBase samples the draft tokens (greedy argmax, or + # rejection sampling for a non-greedy batch), performs the TP + # gather, and scatters the proposal distribution into draft_probs. + gen_draft_tokens = self.sample_draft_tokens( + gen_logits, spec_metadata, batch_size, num_contexts=num_contexts + ) + # The context one-hot must match the width the gen scatter just + # published to draft_probs, NOT gen_logits.shape[-1]: under TP the + # draft logits are vocab-sharded and sample_draft_tokens gathers + # them to full vocab before scattering, so the pre-gather shard + # width would leave stale columns and corrupt rejection. + gen_vocab = spec_metadata.draft_probs_last_dim + else: + gen_draft_tokens = torch.zeros((num_gens, K), dtype=torch.int32, device="cuda") + gen_vocab = None + else: + # No local generation requests: if any peer EP rank has some, we must + # still cross the draft MoE's cross-rank barrier the same number of + # times (zero-token) so a FUSED_COMM phase-flip barrier stays lockstep. + if global_has_gen: + draft_model.run_moe_lockstep_noop(all_rank_draft_tokens, accepted_tokens.device) + gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") + gen_vocab = None + + # Context requests are not drafted by the block worker (zero placeholder + # token); fill their draft-prob slot rows with a legal one-hot so they are + # a valid distribution when they become gen requests next iteration. + self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) + + if num_contexts > 0: + ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") + next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) + else: + next_draft_tokens = gen_draft_tokens + + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) + + if is_warmup: + self._ctx_len.copy_(saved_ctx_len) + self._kv_windows.copy_(saved_windows) + + return { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + } diff --git a/tensorrt_llm/_torch/speculative/dynamic_tree_ops.py b/tensorrt_llm/_torch/speculative/dynamic_tree_ops.py index 3b440b7de7f6..f208342e8f54 100644 --- a/tensorrt_llm/_torch/speculative/dynamic_tree_ops.py +++ b/tensorrt_llm/_torch/speculative/dynamic_tree_ops.py @@ -27,7 +27,7 @@ import torch -from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import compute_probs_from_logits +from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import compute_probs_from_logits class DynamicTreeOpsConverter: diff --git a/tensorrt_llm/_torch/speculative/eagle3.py b/tensorrt_llm/_torch/speculative/eagle3.py index d9b5be1ecc1e..a1f4f82869bd 100644 --- a/tensorrt_llm/_torch/speculative/eagle3.py +++ b/tensorrt_llm/_torch/speculative/eagle3.py @@ -398,6 +398,10 @@ class Eagle3OneModelSpecMetadata(SpecMetadata): # prepare() before self.num_tokens is decremented to the attention-DP subseq # shape; maybe_capture_hidden_states must bound by this, not self.num_tokens. num_capture_tokens: int = 0 + # Per-generation tree links for Mamba verify in dynamic-tree one-model paths. + retrieve_next_token: Optional[torch.Tensor] = None + retrieve_next_sibling: Optional[torch.Tensor] = None + retrieve_parent_token: Optional[torch.Tensor] = None def __post_init__(self): if self.layers_to_capture is None: @@ -443,9 +447,10 @@ def __post_init__(self): self.hidden_size * len(self.layers_to_capture)), dtype=self.dtype, device='cuda') - if (self.spec_resource_manager is not None - and self.spec_resource_manager.batch_indices_cuda is not None): - self.batch_indices_cuda = self.spec_resource_manager.batch_indices_cuda + batch_indices_cuda = getattr(self.spec_resource_manager, + "batch_indices_cuda", None) + if batch_indices_cuda is not None: + self.batch_indices_cuda = batch_indices_cuda assert self.batch_indices_cuda.shape[0] >= self.max_num_requests, ( f"batch_indices_cuda shape mismatch: " f"{type(self.spec_resource_manager).__name__} has " @@ -530,6 +535,23 @@ def prepare(self): if gen_request_ids: sa_manager.prepare(gen_request_ids, self.runtime_draft_len) + self.retrieve_next_token = None + self.retrieve_next_sibling = None + self.retrieve_parent_token = None + spec_tree_manager = getattr(self.spec_resource_manager, + 'spec_tree_manager', None) + if self.use_dynamic_tree and spec_tree_manager is not None: + num_gens = self.num_generations + if num_gens > 0: + num_contexts = num_seqs - num_gens + slot_storage = spec_tree_manager.slot_storage + gen_slot_ids = slot_storage.all_ids_buf[ + num_contexts:num_contexts + num_gens] + next_token, next_sibling = slot_storage.next_links_from_slots( + gen_slot_ids, num_gens) + self.retrieve_next_token = next_token + self.retrieve_next_sibling = next_sibling + def maybe_capture_hidden_states( self, layer_id: int, @@ -624,6 +646,14 @@ def __init__(self, # MTP Eagle: lazily-resolved flag for Mamba hybrid cache support self._is_mamba_hybrid_cache = None + # Worker-side saved spec-dec params; initialized so that a failure + # inside _prepare_attn_metadata_for_spec_dec (e.g. an OOM in the + # clone calls) cannot turn the cleanup restore into AttributeError. + self._saved_packed_mask = None + self._saved_position_offsets = None + self._saved_position_offsets_cpp = None + self._saved_generation_lengths = None + @property def max_draft_len(self) -> int: return self.spec_config.max_draft_len @@ -679,15 +709,15 @@ def _restore_attn_metadata_from_spec_dec(self, attn_metadata): # Skip torch.compile for now since current Torch is not compatible with Triton 3.4 # @torch.compile(options={"max-autotune": True}) - def forward(self, - input_ids, - position_ids, - hidden_states, - logits, - attn_metadata, - spec_metadata, - draft_model, - resource_manager=None): + def _forward_impl(self, + input_ids, + position_ids, + hidden_states, + logits, + attn_metadata, + spec_metadata, + draft_model, + resource_manager=None): runtime_draft_len = spec_metadata.runtime_draft_len # skip the draft forward if the runtime draft length is 0 @@ -846,11 +876,32 @@ def _forward_linear_draft_loop(self, inputs, attn_metadata, spec_metadata, # ADP+LM-head-TP padding to ``max_num_requests`` so every TP # rank produces logits of the same shape. # Eagle3: logits_processor of the EAGLE draft model. - use_lm_head_tp_in_adp = ( + # + # The LM-head-TP fast path (group-stacked rows, vocab-sharded + # weight) is only usable when the consumer is an argmax: the + # distributed greedy sampler recovers the global argmax from + # vocab-shard maxima without materializing full distributions. + # Advanced (rejection) sampling needs each request's full-vocab + # distribution, so a batch headed for the advanced path + # bypasses LM-head-TP and computes full-vocab logits locally -- + # under ADP the lm_head weight is replicated (the sliced-shard + # trick is a runtime optimization), exactly like the target + # head. With rejection off, non-greedy batches keep the + # LM-head-TP argmax path unconditionally, where the per-rank + # greedy flag never enters control flow. This branch is safe + # to take group-uniformly because is_all_greedy_sample is + # group-synchronized whenever rejection+ADP+LM-head-TP are + # combined -- see SpecMetadata.group_all_greedy_sample (anchor + # for the group-sync semantics). + advanced_draft_sampling = ( + spec_metadata.wants_advanced_draft_sampling) + # enable_lm_head_tp_in_adp implies enable_attention_dp + # (asserted in Mapping.__init__); no separate ADP check. + lm_head_tp_in_adp_configured = ( self.is_mtp_eagle and self.model_config is not None - and self.model_config.mapping.enable_attention_dp - and getattr(self.model_config.mapping, - 'enable_lm_head_tp_in_adp', False)) + and self.model_config.mapping.enable_lm_head_tp_in_adp) + use_lm_head_tp_in_adp = (lm_head_tp_in_adp_configured + and not advanced_draft_sampling) if self.is_mtp_eagle: if use_lm_head_tp_in_adp: hidden_states_gathered = hidden_states[gather_ids] @@ -876,6 +927,15 @@ def _forward_linear_draft_loop(self, inputs, attn_metadata, spec_metadata, logits = draft_model.mtp_layers[0].shared_head( padded_hidden_states, draft_model.lm_head, attn_metadata, True) + elif lm_head_tp_in_adp_configured: + # Advanced-sampling bypass: the model's shared_head + # would re-apply the LM-head-TP stacked/sharded path + # from config on its own, so call lm_head directly. + # Under ADP the LMHead weight is replicated and + # is_spec_decoding_head defaults to False, so this is + # a plain local full-vocab GEMM over this rank's own + # rows -- the same computation the target head runs. + logits = draft_model.lm_head(hidden_states[gather_ids]) else: logits = draft_model.mtp_layers[0].shared_head( hidden_states[gather_ids], draft_model.lm_head, @@ -894,18 +954,17 @@ def _forward_linear_draft_loop(self, inputs, attn_metadata, spec_metadata, self._d2t, draft_step=i) - # When ADP+LM-head-TP pads logits to max_num_requests, the - # padded rows are zero-filled placeholders only required so - # every TP rank produces logits of identical shape for the - # LM-head-TP all-gather. Drop them *before* sampling: the - # per-request sampling params (temperatures/top_k/top_p) are - # sized to token_count (== batch_size), so the padded logits - # would otherwise fail to broadcast in apply_temperature. This - # also keeps next_draft_tokens and the draft_probs buffer - # token_count-sized without a post-hoc trim. + # ADP+LM-head-TP logits are the LM-head-TP group's row-stacked + # batch (each rank's rows padded to max_num_requests, then + # all-gathered along dim 0) with the vocab sharded across the + # group. Rows [:token_count] would be group rank 0's requests, + # not this rank's, and a per-rank argmax would return a + # shard-local index -- so keep the full stacked logits and let + # greedy_sample_draft_with_tp_gather combine the group's vocab + # shards and slice this rank's own row segment; only then trim + # the max_num_requests padding down to token_count. mapping_lm_head_tp = None if use_lm_head_tp_in_adp: - logits = logits[:token_count] # The MTP head built this per-forward mapping when producing # the vocab-sharded logits; the sampler needs it to gather. mapping_lm_head_tp = getattr( @@ -917,6 +976,8 @@ def _forward_linear_draft_loop(self, inputs, attn_metadata, spec_metadata, batch_size, draft_step=i, mapping_lm_head_tp=mapping_lm_head_tp) + if use_lm_head_tp_in_adp: + new_draft_token = new_draft_token[:token_count] next_draft_tokens.append(new_draft_token) # Update hidden states for the next iteration. diff --git a/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py b/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py index 7edbcc5e6756..f8b68fc67a3c 100644 --- a/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py +++ b/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py @@ -24,7 +24,7 @@ from tensorrt_llm._utils import get_sm_version, nvtx_range from ..attention_backend import AttentionMetadata -from ..pyexecutor.sampler.sampling_utils import sampling_batch_spec_dec_one_model +from ..pyexecutor.sampler.ops.flashinfer import sample_from_logits_op from .eagle3 import Eagle3OneModelWorker if TYPE_CHECKING: @@ -375,7 +375,7 @@ def _ensure_spec_tree_manager(self, resource_manager): ) @nvtx_range("eagle3_dyn.forward") - def forward( + def _forward_impl( self, input_ids, position_ids, @@ -387,11 +387,11 @@ def forward( resource_manager=None, ): """Override to add accepted_draft_tokens_indices to output.""" - # Initialize spec_tree_manager before super().forward() which calls + # Initialize spec_tree_manager before super()._forward_impl() which calls # _forward_draft_loop needing spec_tree_manager. if resource_manager is not None: self._ensure_spec_tree_manager(resource_manager) - output = super().forward( + output = super()._forward_impl( input_ids, position_ids, hidden_states, @@ -772,7 +772,7 @@ def _sample_and_accept_dynamic_tree( self.offset = torch.tensor([0], dtype=torch.int64, device=logits.device) self.seed.add_(1).remainder_(2**31) top_ks = spec_metadata.top_ks[:num_flat_tokens] - sampled = sampling_batch_spec_dec_one_model( + sampled = sample_from_logits_op( logits, spec_metadata.temperatures[:num_flat_tokens], top_ks, @@ -880,7 +880,7 @@ def _sample_and_accept_dynamic_tree_rejection( # Context tokens bypass the rejection kernel — sample them directly. if num_contexts > 0: top_ks_ctx = spec_metadata.top_ks[:num_contexts] - sampled_ctx = sampling_batch_spec_dec_one_model( + sampled_ctx = sample_from_logits_op( logits[:num_contexts], spec_metadata.temperatures[:num_contexts], top_ks_ctx, diff --git a/tensorrt_llm/_torch/speculative/interface.py b/tensorrt_llm/_torch/speculative/interface.py index 308fc86ad6b9..721b3942ed04 100644 --- a/tensorrt_llm/_torch/speculative/interface.py +++ b/tensorrt_llm/_torch/speculative/interface.py @@ -28,6 +28,7 @@ from tensorrt_llm.logger import logger from ..._utils import get_sm_version, prefer_pinned +from ..attention_backend.interface import AttentionMetadata from ..attention_backend.trtllm import (AttentionBackend, TrtllmAttention, TrtllmAttentionMetadata) from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE @@ -41,9 +42,12 @@ if IS_FLASHINFER_AVAILABLE: import flashinfer -from ..pyexecutor.sampler.sampling_utils import ( - compute_probs_from_logits, greedy, sampling_batch_spec_dec_one_model, - sampling_batch_spec_dec_one_model_for_rejection) +from tensorrt_llm.llmapi.llm_args import AdvancedSamplingMode + +from ..pyexecutor.sampler.ops.flashinfer import ( + compute_probs_from_logits, resolve_advanced_sampling_filters, + sample_from_logits_op, sampling_batch_spec_dec_one_model_for_rejection) +from ..pyexecutor.sampler.ops.vanilla import greedy_search_sampling_batch def rejection_sampling_one_model( @@ -111,6 +115,10 @@ def should_use_separate_draft_kv_cache(spec_config) -> bool: return False if not spec_config.spec_dec_mode.use_one_engine(): return False + # DSpark owns a dedicated rolling-window cache in DSparkWorker. Its draft + # model does not read the paged draft KV cache managed by attention metadata. + if spec_config.spec_dec_mode.is_dspark(): + return False return spec_config._allow_separate_draft_kv_cache @@ -276,6 +284,7 @@ class SpeculativeDecodingMode(IntEnum): SAVE_HIDDEN_STATES = auto() PARD = auto() DFLASH = auto() + DSPARK = auto() NONE = auto() AUTO = auto() @@ -310,8 +319,11 @@ def is_pard(self): def is_dflash(self): return self == SpeculativeDecodingMode.DFLASH + def is_dspark(self): + return self == SpeculativeDecodingMode.DSPARK + def is_parallel_draft(self): - return self.is_pard() or self.is_dflash() + return self.is_pard() or self.is_dflash() or self.is_dspark() def is_ngram(self): return self == SpeculativeDecodingMode.NGRAM @@ -476,6 +488,11 @@ class SpecMetadata: # The number of sequences for speculative model/layer of different rank all_rank_num_seqs: Optional[List[int]] = None + # The number of generation requests for the speculative model/layer of each + # rank (num_seqs - num_contexts). Used by external drafters (e.g. DSpark) + # whose draft forward processes only generation requests and must size a + # FUSED_COMM MoE (DeepGEMM MegaMoE) chunk loop identically across EP ranks. + all_rank_num_gens: Optional[List[int]] = None # The number of extra kv tokens # Some speculative decoding methods need to use different kv lengths for the # draft/target layers. But KVCacheManager can only support kv caches with the @@ -511,8 +528,20 @@ class SpecMetadata: # Defaults to True so non-one-engine paths (where populate is a no-op) # never accidentally select the advanced graph variant. is_all_greedy_sample: bool = True + # Group-synchronized override for ``is_all_greedy_sample`` (AND over the + # TP group's local flags; None = no group sync configured, use the local + # value). Under ADP + LM-head TP with rejection sampling, the greedy-vs- + # advanced choice gates group collectives, so all ranks must take the same + # path even though their batches (and thus local flags) differ. Set by + # ``_sync_group_all_greedy_sample`` before the CUDA graph key is built and + # re-applied by ``_scan_one_model_sampling`` on every rescan. AND is safe: + # a greedy rank pulled onto the advanced path still samples greedily via + # its sentinel params. + group_all_greedy_sample: Optional[bool] = None # Whether to use rejection sampling for one-model speculative decoding. use_rejection_sampling: bool = False + # Advanced-sampling specialization (deploy-time; from DecodingBaseConfig.advanced_sampling_mode). + advanced_sampling_mode: AdvancedSamplingMode = AdvancedSamplingMode.FULL # Sampling parameters for non-greedy sampling (per-request) temperatures: Optional[torch.Tensor] = None top_ks: Optional[torch.Tensor] = None @@ -692,7 +721,7 @@ def _scan_one_model_sampling( before the CUDA graph key is built. """ from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequestState - from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import \ + from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import \ GREEDY_TEMPERATURE_THRESHOLD from tensorrt_llm.sampling_params import SamplingParams @@ -717,6 +746,13 @@ def _normalize_request_sampling_params( top_p: Optional[float], ) -> tuple[float, int, float, bool, bool, bool, bool]: """Convert request sampling params into normalized per-request scalars.""" + # NB: min_p is intentionally omitted here. One-engine speculative + # decoding does not support min_p (there is no request_min_p buffer + # nor min_p wiring in the sampling_batch_spec_dec_one_model* + # kernels); a min_p request is rejected at admission by + # SpecSamplerBase.validate_request, so nothing reaching this scan + # carries a min_p that would change its classification. The + # two-model draft/target path honors min_p via _request_strategy. is_greedy = SamplingParams.params_imply_greedy_decoding( temperature=temperature, top_k=top_k, @@ -815,8 +851,28 @@ def _normalize_request_sampling_params( for (_, _, _, num_tokens) in per_request_normalized ] + # Apply the group-synchronized override last (semantics: see the + # ``group_all_greedy_sample`` field comment). Local contract: the + # synced value already incorporates any capture override, and rescans + # (e.g. populate after the graph key) must converge to it rather than + # resurrect the local value. + if self.group_all_greedy_sample is not None: + self.is_all_greedy_sample = self.group_all_greedy_sample return per_request_normalized, per_request_slot_ids + @property + def wants_advanced_draft_sampling(self) -> bool: + """Whether the current batch takes the advanced (rejection) draft + path: rejection sampling enabled AND not an all-greedy batch. + + Single source of truth for the greedy-vs-advanced decision: the + sampler branch (``sample_draft_tokens``) and the worker's LM-head-TP + bypass (``_forward_linear_draft_loop``) must agree exactly -- a + divergence feeds the wrong logits layout to the sampler -- so both + read this property instead of re-deriving the predicate. + """ + return self.use_rejection_sampling and not self.is_all_greedy_sample + def update_is_all_greedy_sample(self, requests: list["LlmRequest"]) -> None: """Refresh ``is_all_greedy_sample`` for the *current* batch. @@ -963,7 +1019,7 @@ def __init__(self, use_separate_draft_kv_cache: bool = False): self.force_num_accepted_tokens: float = get_force_num_accepted_tokens_float( ) # One-model speculative sampling goes through flashinfer unconditionally - # (sampling_batch_spec_dec_one_model), so flashinfer>=0.6.4 is a hard + # (sample_from_logits_op), so flashinfer>=0.6.4 is a hard # dependency here. Fail at construction with a clear error instead of # crashing mid-inference on the first non-greedy sampling step. if not IS_FLASHINFER_AVAILABLE or Version( @@ -985,6 +1041,54 @@ def __init__(self, use_separate_draft_kv_cache: bool = False): self._force_accept_rng_pool: Optional[torch.Tensor] = None self._force_accept_rng_counter: Optional[torch.Tensor] = None + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + if "forward" in cls.__dict__: + raise TypeError( + f"{cls.__name__} must not override SpecWorkerBase.forward; " + f"implement _forward_impl instead. SpecWorkerBase.forward " + f"guarantees spec-dec attn-metadata cleanup when a forward " + f"fails (https://nvbugs/6442074).") + + def forward(self, *args, **kwargs): + """Run _forward_impl with guaranteed spec-dec metadata cleanup. + + Tolerated forward failures (e.g. an OOM during the max-shape general + warmup, or an error-budget-tolerated serving exception) must not leak + the attn-metadata state saved by prepare_for_spec_dec: a stale save + fails every subsequent forward at the pairing assert. + https://nvbugs/6442074 + """ + attn_metadata = kwargs.get("attn_metadata") + spec_metadata = kwargs.get("spec_metadata") + if attn_metadata is None or spec_metadata is None: + for a in args: + if attn_metadata is None and isinstance(a, AttentionMetadata): + attn_metadata = a + elif spec_metadata is None and isinstance(a, SpecMetadata): + spec_metadata = a + try: + return self._forward_impl(*args, **kwargs) + finally: + self._ensure_spec_dec_state_restored(attn_metadata, spec_metadata) + + @abstractmethod + def _forward_impl(self, *args, **kwargs): + """Worker-specific forward logic, called by SpecWorkerBase.forward.""" + + def _ensure_spec_dec_state_restored(self, attn_metadata, spec_metadata): + """Restore attn-metadata spec-dec state if a failure skipped it. + + No-op on the success path: workers restore at their preferred point + and this sees no saved state. Subclasses with extra transient state + (e.g. the deferred kv_lens rewind in PARD/DFlash) extend this. + """ + if attn_metadata is not None and attn_metadata.has_spec_dec_saved_state: + logger.warning( + "Spec-dec worker forward failed between prepare_for_spec_dec " + "and restore_from_spec_dec; restoring attn metadata state.") + self._restore_attn_metadata_from_spec_dec(attn_metadata) + @property @abstractmethod def max_draft_len(self) -> int: @@ -1378,12 +1482,16 @@ def maybe_gather_sharded_draft_logits(self, (see ``_draft_logits_are_sharded``); replicated full-vocab logits are returned unchanged. - Plain TP gathers vocab shards over ``self.mapping``. Under ADP + LM-head - TP the worker has already trimmed the LM-head-TP padding rows so each rank - holds ``[token_count, vocab_shard]`` for its own tokens; a vocab-dim - all-gather over ``mapping_lm_head_tp`` restores full vocab (no token - re-slice is needed after the trim). + Plain TP gathers vocab shards over ``self.mapping``. The LM-head-TP + stacked/sharded layout never reaches this path: an advanced-sampling + batch bypasses the LM-head-TP fast path in the worker and computes + full-vocab logits locally from the (ADP-replicated) lm_head weight, so + ``mapping_lm_head_tp`` is only ever passed alongside greedy sampling. """ + assert mapping_lm_head_tp is None, ( + "Advanced draft sampling must not receive LM-head-TP " + "stacked/sharded logits; the worker bypasses the LM-head-TP fast " + "path for non-all-greedy batches (see _forward_linear_draft_loop)") if (spec_metadata is None or spec_metadata.is_all_greedy_sample or not self._draft_logits_are_sharded(logits, spec_metadata)): return logits @@ -1405,7 +1513,7 @@ def advanced_sample_draft(self, With rejection enabled and a ``draft_step``, samples via ``sampling_batch_spec_dec_one_model_for_rejection`` and scatters this step's proposal distribution into the slot-indexed ``draft_probs`` - buffer; otherwise uses ``sampling_batch_spec_dec_one_model`` (tokens + buffer; otherwise uses ``sample_from_logits_op`` (tokens only). Returns tokens in draft-vocab space (the caller applies d2t). Expects 2D ``[batch_size, vocab]`` logits (one row per request). """ @@ -1414,13 +1522,15 @@ def advanced_sample_draft(self, top_ps = spec_metadata.request_top_ps[:batch_size] self._update_advance_draft_sampling_seed(logits.device) + eff_top_ks, eff_top_ps = resolve_advanced_sampling_filters( + spec_metadata.advanced_sampling_mode, top_ks, top_ps) if spec_metadata.use_rejection_sampling and draft_step is not None: draft_tokens, probs = ( sampling_batch_spec_dec_one_model_for_rejection( logits, temperatures, - top_ks, - top_ps, + eff_top_ks, + eff_top_ps, seed=self.seed, offset=self.offset)) # Scatter probs into the slot-indexed buffer so each request's data @@ -1434,12 +1544,12 @@ def advanced_sample_draft(self, spec_metadata.draft_probs[batch_slots, draft_step, :vocab] = probs spec_metadata.draft_probs_last_dim = vocab else: - draft_tokens = sampling_batch_spec_dec_one_model(logits, - temperatures, - top_ks, - top_ps, - seed=self.seed, - offset=self.offset) + draft_tokens = sample_from_logits_op(logits, + temperatures, + eff_top_ks, + eff_top_ps, + seed=self.seed, + offset=self.offset) return draft_tokens.type(torch.int32) @@ -1589,6 +1699,8 @@ def _sample_and_accept_draft_tokens_rejection( spec_metadata.top_ks[gen_start:gen_end]) top_ps = (None if spec_metadata.skip_top_p else spec_metadata.top_ps[gen_start:gen_end]) + top_ks, top_ps = resolve_advanced_sampling_filters( + spec_metadata.advanced_sampling_mode, top_ks, top_ps) target_probs_flat = compute_probs_from_logits( gen_logits, temperatures, top_ks, top_ps) @@ -1700,7 +1812,8 @@ def _draft_sampler_greedy(self, logits: torch.Tensor): Returns: draft_tokens: [num_tokens] - Sampled draft token ids (int32) """ - draft_tokens = greedy(logits, return_probs=False)[0] + draft_tokens = greedy_search_sampling_batch(logits, + return_probs=False)[0] # Apply the cached draft->target vocab offset map. if self._d2t is not None: @@ -1750,7 +1863,26 @@ def greedy_sample_draft_with_tp_gather(self, vocab-sharded (see ``_draft_logits_are_sharded``) -- e.g. a borrowed or gathered full-vocab draft head. Returns tokens in draft-vocab space (the caller applies d2t). Expects 2D ``[num_tokens, vocab_shard]`` logits. + + Under ADP + LM-head TP (``mapping_lm_head_tp`` given) the logits are the + LM-head-TP group's row-stacked batch (``tp_size`` segments of + ``max_num_requests`` padded rows, all-gathered along dim 0 by the MTP + shared head) with the vocab sharded across the group. The global argmax + must combine the group's vocab shards, and each rank must read its own + row segment at offset ``tp_rank * max_num_requests`` -- NOT rows + ``[:batch]``, which belong to group rank 0. """ + if (mapping_lm_head_tp is not None + and getattr(mapping_lm_head_tp, "tp_size", 1) > 1): + from ..distributed.ops import allgather + combined = self._get_local_max_and_combined(logits, + mapping_lm_head_tp) + gathered = allgather(combined, mapping_lm_head_tp, dim=-1) + group_size = mapping_lm_head_tp.tp_size + local_rows = logits.shape[0] // group_size + own_segment = gathered.view(group_size, local_rows, + -1)[mapping_lm_head_tp.tp_rank] + return self._get_draft_tokens_from_gathered(own_segment) mapping = self.mapping sharded = self._draft_logits_are_sharded(logits, spec_metadata) if (sharded and mapping is not None @@ -1760,9 +1892,9 @@ def greedy_sample_draft_with_tp_gather(self, combined = self._get_local_max_and_combined(logits) gathered = allgather(combined, mapping, dim=-1) return self._get_draft_tokens_from_gathered(gathered) - # No TP gather for attention-DP (incl. ADP + LM-head TP): each rank owns - # its own requests, so a per-rank argmax is the correct proposal and a - # cross-rank gather here would desync the ranks (see + # No cross-rank gather for plain attention-DP: each rank owns its own + # requests with replicated full-vocab logits, so a per-rank argmax is + # the correct proposal and a gather would desync the ranks (see # _draft_logits_are_sharded). Plain argmax; caller applies d2t. return torch.argmax(logits, dim=-1).type(torch.int32) @@ -1776,7 +1908,7 @@ def advanced_sample_draft_block(self, gen_logits: torch.Tensor, With rejection enabled, samples via ``sampling_batch_spec_dec_one_model_for_rejection`` and scatters the K proposal rows into ``draft_probs[gen_slot_ids, 0:K, :]``; otherwise uses - ``sampling_batch_spec_dec_one_model`` (tokens only). Only called for a + ``sample_from_logits_op`` (tokens only). Only called for a non-greedy batch (the all-greedy path is handled by the caller). Returns ``[num_gens, K]`` int32 tokens in draft-vocab space (the caller applies d2t); stored probs likewise stay in draft-vocab space. @@ -1798,14 +1930,16 @@ def advanced_sample_draft_block(self, gen_logits: torch.Tensor, self._update_advance_draft_sampling_seed(gen_logits.device) flat_logits = gen_logits.reshape(num_gens * K, vocab) + eff_top_ks, eff_top_ps = resolve_advanced_sampling_filters( + spec_metadata.advanced_sampling_mode, top_ks, top_ps) if getattr(spec_metadata, "use_rejection_sampling", False): flat_tokens, flat_probs = ( sampling_batch_spec_dec_one_model_for_rejection( flat_logits, temps, - top_ks, - top_ps, + eff_top_ks, + eff_top_ps, seed=self.seed, offset=self.offset)) # Scatter the K prob rows per gen request into its stable slot row. @@ -1819,12 +1953,12 @@ def advanced_sample_draft_block(self, gen_logits: torch.Tensor, spec_metadata.draft_probs[gen_slot_ids, :K, :vocab] = probs spec_metadata.draft_probs_last_dim = vocab else: - flat_tokens = sampling_batch_spec_dec_one_model(flat_logits, - temps, - top_ks, - top_ps, - seed=self.seed, - offset=self.offset) + flat_tokens = sample_from_logits_op(flat_logits, + temps, + eff_top_ks, + eff_top_ps, + seed=self.seed, + offset=self.offset) return flat_tokens.reshape(num_gens, K).type(torch.int32) @@ -1886,7 +2020,6 @@ def sample_draft_tokens(self, no slicing is needed. """ is_block = logits.dim() == 3 - use_rejection = getattr(spec_metadata, "use_rejection_sampling", False) # Draft tokens use argmax unless rejection sampling is engaged for a # non-greedy batch. Rejection sampling is the only path that needs the @@ -1897,7 +2030,7 @@ def sample_draft_tokens(self, # max_i p_i >= sum_i p_i^2 = E[accept] for a stochastic draft). This # matches sglang/vLLM, which draft with argmax/top-k by default and apply # sampling params only on the target/acceptance side. - advanced = use_rejection and not spec_metadata.is_all_greedy_sample + advanced = spec_metadata.wants_advanced_draft_sampling # All samplers below return tokens in draft-vocab space; d2t is applied # once after the branch. @@ -1908,7 +2041,12 @@ def sample_draft_tokens(self, tokens = self.greedy_sample_draft_with_tp_gather( logits.reshape(-1, logits.shape[-1]), spec_metadata, mapping_lm_head_tp) - tokens = tokens.reshape(batch_shape) + if mapping_lm_head_tp is None: + tokens = tokens.reshape(batch_shape) + # else: ADP+LM-head-TP (2D step form only) -- the sampler returned + # this rank's own row segment, 1/tp_size of the stacked input rows, + # so the input batch shape no longer applies. Keep as-is; the + # caller trims the max_num_requests padding to token_count. else: # Advanced sampling gathers the vocab-sharded draft logits to full # vocab, then samples (scattering this step's proposal distribution @@ -2067,7 +2205,7 @@ def _sample_tokens_for_batch( # Use logits.shape[0] directly: for PARD under CUDA graph capture # runtime_draft_len may reflect the PARD-max while the captured # graph was built for a shorter draft_len, causing a shape mismatch - # in sampling_batch_spec_dec_one_model (which is torch.compiled). + # in sample_from_logits_op (which is torch.compiled). num_tokens = logits.shape[0] temperatures = spec_metadata.temperatures[:num_tokens] @@ -2085,13 +2223,14 @@ def _sample_tokens_for_batch( self.seed += 1 self.seed %= (2**31) - sampled_tokens = sampling_batch_spec_dec_one_model( - logits, - temperatures, - top_ks, - top_ps, - seed=self.seed, - offset=self.offset) + eff_top_ks, eff_top_ps = resolve_advanced_sampling_filters( + spec_metadata.advanced_sampling_mode, top_ks, top_ps) + sampled_tokens = sample_from_logits_op(logits, + temperatures, + eff_top_ks, + eff_top_ps, + seed=self.seed, + offset=self.offset) else: sampled_tokens = torch.argmax(logits, dim=-1) diff --git a/tensorrt_llm/_torch/speculative/mtp.py b/tensorrt_llm/_torch/speculative/mtp.py index fcf0603a04c1..3dc12a669aa6 100644 --- a/tensorrt_llm/_torch/speculative/mtp.py +++ b/tensorrt_llm/_torch/speculative/mtp.py @@ -298,7 +298,7 @@ def __init__(self, def max_draft_len(self) -> int: return self.spec_config.max_draft_len - def forward( + def _forward_impl( self, input_ids, position_ids, diff --git a/tensorrt_llm/_torch/speculative/mtp_dynamic_tree.py b/tensorrt_llm/_torch/speculative/mtp_dynamic_tree.py new file mode 100644 index 000000000000..20791f971f81 --- /dev/null +++ b/tensorrt_llm/_torch/speculative/mtp_dynamic_tree.py @@ -0,0 +1,1126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MTP-Eagle one-model dynamic tree speculative decoding (greedy only).""" + +import math +from typing import TYPE_CHECKING, List, Optional + +import torch +import triton + +from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import MambaHybridCacheManager +from tensorrt_llm._utils import get_sm_version, nvtx_range +from tensorrt_llm.mapping import Mapping + +from ..distributed.ops import allgather +from ..model_config import ModelConfig +from ..pyexecutor.llm_request import LlmRequest +from ..pyexecutor.resource_manager import BaseResourceManager +from ..pyexecutor.scheduler import ScheduledRequests +from .eagle3 import MTPEagleWorker + +# Reuse drafter-agnostic dynamic-tree helpers. +from .eagle3_dynamic_tree import ( + _build_mask_and_position, + _gather_repack_step0_kernel, + _resample_final_tokens, + _select_topk_draft_tokens, +) +from .mtp import MTPHiddenStatesManager + +if TYPE_CHECKING: + from tensorrt_llm.llmapi.llm_args import MTPDecodingConfig + + +class MTPEagleDynamicTreeWorker(MTPEagleWorker): + """MTP-Eagle worker with dynamic-tree draft and greedy verify.""" + + def __init__( + self, + spec_config: "MTPDecodingConfig", + model_config: Optional[ModelConfig] = None, + use_separate_draft_kv_cache: bool = False, + *, + mapping: Optional[Mapping] = None, + ): + super().__init__(spec_config, model_config, use_separate_draft_kv_cache, mapping=mapping) + assert getattr(spec_config, "use_dynamic_tree", False), ( + "MTPEagleDynamicTreeWorker requires use_dynamic_tree=True" + ) + + from .dynamic_tree_ops import DynamicTreeOpsConverter + + self.K = spec_config.dynamic_tree_max_topK + self.max_total_draft_tokens = spec_config.tokens_per_gen_step - 1 + self.tokens_per_gen_step = spec_config.tokens_per_gen_step + # Set by py_executor_creator from the global max_batch_size. + assert spec_config._max_batch_size is not None, ( + "MTPDecodingConfig._max_batch_size was not populated; " + "py_executor_creator should have set it from the global max_batch_size." + ) + self._max_batch_size = spec_config._max_batch_size + + K = self.K + max_draft_len = spec_config.max_draft_len + max_batch_size = self._max_batch_size + loop_max_tokens = K * max_draft_len # draft loop working size + + # spec_tree_manager is lazily bound from the resource manager. + self.spec_tree_manager = None + self._d2t = None + + # Pre-allocated draft-loop buffers (CUDA-graph safe). + self.draft_tokens_buffer = torch.zeros( + max_batch_size, loop_max_tokens, dtype=torch.int32, device="cuda" + ) + self.position_ids_buffer = torch.zeros( + max_batch_size, loop_max_tokens, dtype=torch.int32, device="cuda" + ) + self.history_draft_tokens_buffer = torch.zeros( + (max_batch_size, (K + K * K * (max_draft_len - 1))), dtype=torch.int32, device="cuda" + ) + self.history_score_buffer = torch.zeros( + (max_batch_size, K + K * K * (max_draft_len - 1)), dtype=torch.float32, device="cuda" + ) + self.history_draft_tokens_parent_buffer = torch.zeros( + (max_batch_size, max(K * (max_draft_len - 1) + 1, K + 1)), + dtype=torch.int64, + device="cuda", + ) + self.tree_mask_buffer = torch.zeros( + (max_batch_size * loop_max_tokens * loop_max_tokens), dtype=torch.int32, device="cuda" + ) + self.tree_mask_init_buffer = ( + torch.eye(K, dtype=torch.int32, device="cuda").unsqueeze(0).repeat(max_batch_size, 1, 1) + ) + self.tree_ops_converter = DynamicTreeOpsConverter( + dynamic_tree_max_topK=K, + max_draft_len=max_draft_len, + max_total_draft_tokens=self.max_total_draft_tokens, + max_batch_size=max_batch_size, + device=torch.device("cuda"), + ) + + self._max_path_len = max_draft_len + 1 + # Step-0 draft resets verify-time tree metadata to accepted-path width. + self._kv_correction = self.tokens_per_gen_step - self._max_path_len + self._step0_causal_mask = torch.tensor( + [(1 << (t + 1)) - 1 for t in range(self._max_path_len)], + dtype=torch.int32, + device="cuda", + ) + self._causal_offs = torch.arange(self._max_path_len, device="cuda", dtype=torch.int32) + self._last_selected_parents = None + self._parent_init_arange = torch.arange(-1, K, device="cuda", dtype=torch.int32) + + # Accepted-path bookkeeping for KV relocation and output. + self._accepted_draft_indices_tensor = torch.full( + (max_batch_size, max_draft_len), -1, dtype=torch.int32, device="cuda" + ) + self._kv_head_dim_bytes = None + + # === Verification buffers (greedy only) === + N = self.tokens_per_gen_step + self._accepted_tokens_buf = torch.zeros( + max_batch_size, self._max_path_len, dtype=torch.int32, device="cuda" + ) + self._num_accepted_tokens_buf = torch.ones(max_batch_size, dtype=torch.int32, device="cuda") + self._target_tokens_buf = torch.zeros(max_batch_size * N, dtype=torch.int64, device="cuda") + self._candidates_buf = torch.zeros(max_batch_size, N, dtype=torch.int32, device="cuda") + self._target_predict_buf = torch.zeros(max_batch_size, N, dtype=torch.int32, device="cuda") + + # Hidden states for the growing-context draft loop. + self._hs_write_buffer = None + self._accumulated_hs = None + self._hs_read_map = torch.zeros( + max_batch_size, loop_max_tokens, dtype=torch.long, device="cuda" + ) + self._step0_hs = None + self._hs_dim = None + + # Step-0 repack scratch for accepted-path inputs. + max_total_tokens = max_batch_size * self.tokens_per_gen_step + self._step0_input_ids_buf = torch.zeros(max_total_tokens, dtype=torch.int32, device="cuda") + self._step0_position_ids_buf = torch.zeros( + max_total_tokens, dtype=torch.int32, device="cuda" + ) + self._step0_hidden_states_buf = None + self._gather_ids_buf = torch.zeros(max_total_tokens, dtype=torch.long, device="cuda") + + # Mask repack scratch (graph-safe; avoids .contiguous() in the loop). + buf_dim = max(self.max_total_draft_tokens + 1, loop_max_tokens) + mask_width = (buf_dim + 31) // 32 + self._mask_repack_buf = torch.zeros( + max_batch_size * buf_dim * mask_width, dtype=torch.int32, device="cuda" + ) + # sm>=100 (except 120/121): prepareCustomMask keeps padded 3D; no repack. + sm = get_sm_version() + self._needs_mask_repack = sm < 100 or sm in (120, 121) + + def _prepare_attn_metadata_for_spec_dec(self, attn_metadata): + super()._prepare_attn_metadata_for_spec_dec(attn_metadata) + + batch_size = attn_metadata.num_seqs + if hasattr(attn_metadata, "kv_lens_cuda"): + # Keep kv_lens_cuda itself alive because TRTLLM attention holds a + # runtime view into it. + self._saved_kv_lens_cuda = attn_metadata.kv_lens_cuda[:batch_size].clone() + else: + self._saved_kv_lens_cuda = None + + # Restore verify metadata after the draft loop mutates it. + if attn_metadata.spec_decoding_packed_mask is not None: + self._saved_packed_mask = attn_metadata.spec_decoding_packed_mask[:batch_size].clone() + else: + self._saved_packed_mask = None + if attn_metadata.spec_decoding_position_offsets is not None: + self._saved_position_offsets = attn_metadata.spec_decoding_position_offsets.clone() + self._saved_position_offsets_cpp = attn_metadata.spec_decoding_position_offsets_cpp + else: + self._saved_position_offsets = None + self._saved_position_offsets_cpp = None + if attn_metadata.spec_decoding_generation_lengths is not None: + self._saved_generation_lengths = attn_metadata.spec_decoding_generation_lengths[ + :batch_size + ].clone() + else: + self._saved_generation_lengths = None + + def prepare_position_ids_and_last_tokens(self, position_ids, seq_lens_cuda): + position_ids = position_ids.squeeze(0) + last_tokens_idx = torch.cumsum(seq_lens_cuda, dim=0, dtype=torch.long) - 1 + return position_ids, last_tokens_idx + + def _restore_attn_metadata_from_spec_dec(self, attn_metadata): + super()._restore_attn_metadata_from_spec_dec(attn_metadata) + + if self._saved_kv_lens_cuda is not None: + batch_size = self._saved_kv_lens_cuda.shape[0] + attn_metadata.kv_lens_cuda[:batch_size].copy_(self._saved_kv_lens_cuda) + self._saved_kv_lens_cuda = None + + if self._saved_packed_mask is not None: + batch_size = self._saved_packed_mask.shape[0] + attn_metadata.spec_decoding_packed_mask[:batch_size].copy_(self._saved_packed_mask) + self._saved_packed_mask = None + if self._saved_position_offsets is not None: + attn_metadata.spec_decoding_position_offsets.copy_(self._saved_position_offsets) + attn_metadata.spec_decoding_position_offsets_cpp = self._saved_position_offsets_cpp + self._saved_position_offsets = None + self._saved_position_offsets_cpp = None + if self._saved_generation_lengths is not None: + batch_size = self._saved_generation_lengths.shape[0] + attn_metadata.spec_decoding_generation_lengths[:batch_size].copy_( + self._saved_generation_lengths + ) + self._saved_generation_lengths = None + + # ------------------------------------------------------------------ # + # Helpers # + # ------------------------------------------------------------------ # + def _apply_spec_metadata(self, attn_metadata, batch_size, query_len): + """Set spec-dec gen lengths and refresh the C++ position-offset view.""" + attn_metadata.spec_decoding_generation_lengths[:batch_size] = query_len + attn_metadata.update_position_offsets_for_cpp(query_len) + + def _refresh_blackwell_tree_mask_metadata(self, attn_metadata): + if not getattr(attn_metadata, "use_spec_decoding", False): + return + if not getattr(attn_metadata, "is_spec_dec_dynamic_tree", False): + return + + first_sparse = getattr(attn_metadata, "spec_bl_tree_first_sparse_mask_offset_kv", None) + bl_tree_mask = getattr(attn_metadata, "spec_decoding_bl_tree_mask", None) + if first_sparse is None and bl_tree_mask is None: + return + + if bl_tree_mask is not None: + bl_tree_mask.zero_() + if first_sparse is not None: + attn_metadata.update_blackwell_first_sparse_mask_offset() + + def _repack_mask_padded_to_packed(self, mask_buf, n_req, n_tok): + """Compact padded masks into the flat prefix XQA expects.""" + buf_dim = mask_buf.shape[1] + if n_tok >= buf_dim or n_req <= 1: + return + mask_width = math.ceil(n_tok / 32) + total_elems = n_req * n_tok * mask_width + scratch = self._mask_repack_buf[:total_elems].view(n_req, n_tok, mask_width) + scratch.copy_(mask_buf[:n_req, :n_tok, :mask_width]) + flat = mask_buf.view(-1) + flat[:total_elems] = scratch.view(-1) + + @nvtx_range("mtp_dyn._ensure_spec_tree_manager") + def _ensure_spec_tree_manager(self, resource_manager): + """Lazily bind spec_tree_manager and KV head metadata.""" + if self.spec_tree_manager is not None: + return + from ..pyexecutor.resource_manager import ResourceManagerType + + spec_rm = resource_manager.get_resource_manager(ResourceManagerType.SPEC_RESOURCE_MANAGER) + assert spec_rm is not None and hasattr(spec_rm, "spec_tree_manager"), ( + "Dynamic tree mode requires spec_tree_manager in resource_manager" + ) + self.spec_tree_manager = spec_rm.spec_tree_manager + + if self._kv_head_dim_bytes is None: + cache_mgr = resource_manager.get_resource_manager(ResourceManagerType.KV_CACHE_MANAGER) + if cache_mgr is not None and hasattr(cache_mgr, "head_dim"): + from tensorrt_llm.bindings import DataType + + _dtype_bytes = { + DataType.HALF: 2, + DataType.BF16: 2, + DataType.FLOAT: 4, + DataType.FP8: 1, + DataType.INT8: 1, + DataType.NVFP4: 0.5, + } + self._kv_head_dim_bytes = int( + cache_mgr.head_dim * _dtype_bytes.get(cache_mgr.dtype, 0.5) + ) + + @nvtx_range("mtp_dyn.sample") + def sample( + self, logits: torch.Tensor, max_top_k: int, draft_model=None + ) -> tuple[torch.Tensor, torch.Tensor]: + """TopK sampling for dynamic tree; all-gather sharded TP logits.""" + mapping = ( + getattr(self.model_config, "mapping", None) if self.model_config is not None else None + ) + if mapping is not None and mapping.tp_size > 1 and not mapping.enable_attention_dp: + logits = allgather(logits, mapping, dim=-1) + if draft_model is not None: + vocab_size = draft_model.lm_head.num_embeddings + logits = logits[..., :vocab_size] + probs = torch.softmax(logits, dim=-1) + topk_values, topk_indices = torch.topk(probs, k=max_top_k, dim=-1) + return topk_indices, topk_values + + def update_draft_tokens_and_scores( + self, + cur_draft_idx, + new_draft_tokens, + new_draft_scores, + previous_draft_scores, + batch_size, + attn_metadata=None, + ): + """Grow the tree and update history buffers.""" + if cur_draft_idx == 0: + new_draft_scores = new_draft_scores.reshape(batch_size, self.K) + new_draft_tokens_2d = new_draft_tokens.reshape(batch_size, self.K) + self.draft_tokens_buffer[:batch_size, : self.K] = new_draft_tokens_2d + self.history_draft_tokens_buffer[:batch_size, : self.K] = new_draft_tokens_2d + self.history_score_buffer[:batch_size, : self.K] = new_draft_scores + # Parent buffer: -1 for root, 0..K-1 for first layer. + self.history_draft_tokens_parent_buffer[:batch_size, : self.K + 1] = ( + self._parent_init_arange + ) + self.prepare_tree_mask_and_position_offset(cur_draft_idx, attn_metadata, None) + return new_draft_scores + + ( + real_draft_tokens, + topk_values, + topk_indices, + selected_parents, + new_draft_tokens, + new_draft_scores, + ) = _select_topk_draft_tokens( + new_draft_tokens, new_draft_scores, previous_draft_scores, self.K + ) + + num_tokens_previous_layer = cur_draft_idx * self.K + num_tokens_current_layer = (cur_draft_idx + 1) * self.K + self.draft_tokens_buffer[ + :batch_size, num_tokens_previous_layer:num_tokens_current_layer + ] = real_draft_tokens + + write_start = self.K + (cur_draft_idx - 1) * self.K * self.K + write_end = write_start + self.K * self.K + self.history_draft_tokens_buffer[:batch_size, write_start:write_end] = new_draft_tokens + self.history_score_buffer[:batch_size, write_start:write_end] = new_draft_scores + + self._last_selected_parents = selected_parents + self.prepare_tree_mask_and_position_offset(cur_draft_idx, attn_metadata, selected_parents) + + if cur_draft_idx < self.max_draft_len - 1: + next_layer_start = cur_draft_idx * self.K + 1 + next_layer_end = next_layer_start + self.K + parents_relative_indices = topk_indices + self.K**2 * (cur_draft_idx - 1) + self.K + self.history_draft_tokens_parent_buffer[ + :batch_size, next_layer_start:next_layer_end + ] = parents_relative_indices + return topk_values + + def resampling_final_draft_tokens(self, batch_size: int): + """Reconstruct the final tree from history buffers.""" + return _resample_final_tokens( + self.history_score_buffer[:batch_size, :], + self.history_draft_tokens_buffer[:batch_size, :], + self.max_total_draft_tokens, + ) + + def prepare_tree_mask_and_position_offset( + self, cur_draft_idx, attn_metadata, selected_parents=None + ): + """Prepare mask and position offsets for the next draft layer.""" + if attn_metadata.spec_decoding_packed_mask is None: + return + spec_tree_manager = self.spec_tree_manager + batch_size = attn_metadata.num_seqs + num_tokens_current_layer = self.K * (cur_draft_idx + 1) + num_tokens_previous_layer = self.K * cur_draft_idx + packed_mask = attn_metadata.spec_decoding_packed_mask + if cur_draft_idx == 0: + spec_tree_manager.compute_spec_dec_packed_mask( + self.tree_mask_init_buffer[:batch_size], + packed_mask[:batch_size, :num_tokens_current_layer, :], + ) + self.tree_mask_buffer[ + : batch_size * num_tokens_current_layer * num_tokens_current_layer + ].copy_(self.tree_mask_init_buffer[:batch_size].view(-1)) + attn_metadata.spec_decoding_position_offsets.fill_(0) + self._apply_spec_metadata(attn_metadata, batch_size, num_tokens_current_layer) + else: + num_parent_mask = batch_size * cur_draft_idx * self.K * cur_draft_idx * self.K + parent_mask = self.tree_mask_buffer[:num_parent_mask].reshape( + batch_size, cur_draft_idx * self.K, cur_draft_idx * self.K + ) + + prev_total = batch_size * num_tokens_previous_layer + previous_position_offsets = attn_metadata.spec_decoding_position_offsets[ + :prev_total + ].view(batch_size, num_tokens_previous_layer) + + current_mask, new_positions = _build_mask_and_position( + parent_mask, + selected_parents, + self.tree_mask_init_buffer[:batch_size], + previous_position_offsets, + self.K, + ) + + spec_tree_manager.compute_spec_dec_packed_mask( + current_mask, packed_mask[:batch_size, :num_tokens_current_layer, :] + ) + self.tree_mask_buffer[ + : batch_size * num_tokens_current_layer * num_tokens_current_layer + ].copy_(current_mask.reshape(-1)) + + cur_total = batch_size * num_tokens_current_layer + attn_metadata.spec_decoding_position_offsets[:cur_total] = new_positions.reshape(-1) + self._apply_spec_metadata(attn_metadata, batch_size, num_tokens_current_layer) + + if self._needs_mask_repack: + self._repack_mask_padded_to_packed(packed_mask, batch_size, num_tokens_current_layer) + + def update_hidden_states( + self, + cur_draft_idx, + batch_size, + step0_hs=None, + hidden_states_to_save=None, + selected_parents=None, + ): + """Manage growing-context hidden states for the MTP draft loop.""" + if cur_draft_idx == 0: + hs_dim = step0_hs.shape[-1] + self._hs_dim = hs_dim + if self._hs_write_buffer is None or self._hs_write_buffer.shape[2] != hs_dim: + self._hs_write_buffer = torch.zeros( + self._max_batch_size, + self.max_draft_len * self.K, + hs_dim, + device=step0_hs.device, + dtype=step0_hs.dtype, + ) + if self._accumulated_hs is None or self._accumulated_hs.shape[2] != hs_dim: + self._accumulated_hs = torch.zeros( + self._max_batch_size, + self.max_draft_len * self.K, + hs_dim, + device=step0_hs.device, + dtype=step0_hs.dtype, + ) + # All K depth-0 tokens share step0_hs (the parent hidden state). + self._accumulated_hs[:batch_size, : self.K] = step0_hs.unsqueeze(1).expand( + -1, self.K, -1 + ) + self._step0_hs = step0_hs + else: + num_tokens_per_req = cur_draft_idx * self.K + hs_to_save_reshaped = hidden_states_to_save.reshape(batch_size, num_tokens_per_req, -1) + self._hs_write_buffer[:batch_size, :num_tokens_per_req] = hs_to_save_reshaped + parent_offset = (cur_draft_idx - 1) * self.K + self._hs_read_map[ + :batch_size, cur_draft_idx * self.K : (cur_draft_idx + 1) * self.K + ] = parent_offset + selected_parents + num_tokens_next = (cur_draft_idx + 1) * self.K + read_idx = self._hs_read_map[:batch_size, self.K : num_tokens_next] + hs_dim = self._hs_write_buffer.shape[2] + self._accumulated_hs[:batch_size, self.K : num_tokens_next] = torch.gather( + self._hs_write_buffer[:batch_size], 1, read_idx.unsqueeze(-1).expand(-1, -1, hs_dim) + ) + + # ------------------------------------------------------------------ # + # Verification (greedy only) # + # ------------------------------------------------------------------ # + @nvtx_range("mtp_dyn.sample_and_accept_draft_tokens") + def sample_and_accept_draft_tokens(self, input_ids, logits, spec_metadata, attn_metadata): + """Greedy verification of the previous dynamic tree.""" + batch_size = attn_metadata.num_seqs + num_contexts = attn_metadata.num_contexts + num_gens = batch_size - num_contexts + N = self.tokens_per_gen_step + max_path_len = self._max_path_len + + if logits.dim() == 1: + logits = logits.unsqueeze(0) + + # Reset output buffers. + self._accepted_tokens_buf[:batch_size].zero_() + accepted_tokens = self._accepted_tokens_buf[:batch_size, :max_path_len] + self._num_accepted_tokens_buf[:batch_size].fill_(1) + num_accepted_tokens = self._num_accepted_tokens_buf[:batch_size] + self._accepted_draft_indices_tensor[:batch_size].fill_(-1) + + num_flat_tokens = logits.shape[0] + torch.argmax(logits, dim=-1, out=self._target_tokens_buf[:num_flat_tokens]) + target_tokens = self._target_tokens_buf[:num_flat_tokens] + + # Context requests: accept the sampled golden token only. + accepted_tokens[:num_contexts, 0].copy_(target_tokens[:num_contexts]) + + if num_gens > 0: + spec_tree_manager = self.spec_tree_manager + target_predict = self._target_predict_buf[:num_gens] + target_predict.copy_(target_tokens[num_contexts:].reshape(num_gens, N)) + + # No prior tree exists on bootstrap/warmup; accept the golden token. + if spec_tree_manager is None: + num_accepted_tokens[num_contexts:batch_size] = 1 + accepted_tokens[num_contexts:batch_size, 0] = target_predict[:, 0] + self._accepted_draft_indices_tensor[num_contexts:batch_size] = -1 + return accepted_tokens, num_accepted_tokens + + # candidates[:, 0] = golden token, candidates[:, 1:] = draft tokens. + candidates = self._candidates_buf[:num_gens] + candidates[:, 1:] = spec_metadata.draft_tokens.reshape(num_gens, N - 1) + candidates[:, 0] = target_predict[:, 0] + + slot_storage = spec_tree_manager.slot_storage + gen_slot_ids = slot_storage.all_ids_buf[num_contexts : num_contexts + num_gens] + tree_valid = slot_storage.has_tree[gen_slot_ids] + retrieve_packed = slot_storage.pack_retrieve_from_slots(gen_slot_ids, num_gens) + + accept_index, accept_token_num, accept_token = ( + self.tree_ops_converter.verify_dynamic_tree_greedy_out_packed( + candidates, + retrieve_packed, + target_predict, + num_gens, + self._max_path_len, + tree_valid=tree_valid, + ) + ) + tree_valid_i = tree_valid[:num_gens] + accepted_draft_count = torch.where( + tree_valid_i, + accept_token_num[:num_gens], + torch.zeros_like(accept_token_num[:num_gens]), + ) + num_accepted_tokens[num_contexts:batch_size] = (accepted_draft_count + 1).to( + torch.int32 + ) + + gen_accepted_tokens = accept_token[:num_gens].to(torch.int32) + bootstrap_accepted_tokens = torch.zeros_like(gen_accepted_tokens) + bootstrap_accepted_tokens[:, 0] = target_predict[:, 0] + accepted_tokens[num_contexts:batch_size] = torch.where( + tree_valid_i.unsqueeze(1), gen_accepted_tokens, bootstrap_accepted_tokens + ) + # Convert root/padding index 0 to draft-node sentinel -1. + gen_accepted_indices = (accept_index[:num_gens, 1:max_path_len] - 1).to(torch.int32) + self._accepted_draft_indices_tensor[num_contexts:batch_size] = torch.where( + tree_valid_i.unsqueeze(1), + gen_accepted_indices, + torch.full_like(gen_accepted_indices, -1), + ).to(torch.int32) + + num_accepted_tokens = self._apply_force_accepted_tokens( + num_accepted_tokens, num_contexts, self.max_draft_len + ) + return accepted_tokens, num_accepted_tokens + + def _accepted_leaf_intermediate_positions(self, num_accepted_tokens, num_contexts, num_gens): + """Return each accepted leaf's position in the Mamba state buffer.""" + accepted = num_accepted_tokens[num_contexts : num_contexts + num_gens].to(torch.int64) + # Column of the deepest accepted draft node, clamped to >=0 for the + # golden-only case (its value is ignored via the mask below). + draft_idx = self._accepted_draft_indices_tensor[num_contexts : num_contexts + num_gens].to( + torch.int64 + ) + last_col = (accepted - 2).clamp_(min=0, max=draft_idx.shape[1] - 1) + leaf = torch.gather(draft_idx, 1, last_col.unsqueeze(1)).squeeze(1) + 1 + # Golden-only requests (num_accepted == 1) take the root at position 0. + return torch.where(accepted > 1, leaf, torch.zeros_like(leaf)) + + @nvtx_range("mtp_dyn._relocate_kv_eagerly") + def _relocate_kv_eagerly(self, attn_metadata, batch_size): + """Move accepted draft KV from tree positions to the linear prefix.""" + cache_mgr = getattr(attn_metadata, "kv_cache_manager", None) + if cache_mgr is None or self._kv_head_dim_bytes is None: + return + if not hasattr(cache_mgr, "num_kv_heads_per_layer"): + return + + # Mamba layers have zero KV heads; relocate attention-layer KV only. + kv_heads = cache_mgr.num_kv_heads_per_layer + attn_heads = set(h for h in kv_heads if h > 0) + assert len(attn_heads) == 1, ( + "update_kv_cache_draft_token_location_2d requires uniform " + f"num_kv_heads across attention layers, got {list(kv_heads)}" + ) + attn_num_heads = attn_heads.pop() + attn_layer_offsets = [i for i, h in enumerate(kv_heads) if h > 0] + attn_num_layers = len(attn_layer_offsets) + + # Resolve the attention KV pool used by the relocation op. + pool_mapping = getattr(cache_mgr, "kv_cache_pool_mapping", None) + if pool_mapping is not None: + attn_pool_indices = set(int(pool_mapping[off][0]) for off in attn_layer_offsets) + assert len(attn_pool_indices) == 1, ( + "update_kv_cache_draft_token_location_2d requires all attention " + f"layers in one KV pool, got pools {sorted(attn_pool_indices)}" + ) + attn_pool_idx = attn_pool_indices.pop() + else: + attn_pool_idx = 0 + + pool_pointers = cache_mgr.kv_cache_pool_pointers[attn_pool_idx] + block_offsets = attn_metadata.kv_cache_block_offsets[attn_pool_idx] + + torch.ops.tensorrt_llm.update_kv_cache_draft_token_location_2d( + self._accepted_draft_indices_tensor[:batch_size], + self._num_accepted_tokens_buf[:batch_size], + attn_metadata.kv_lens_cuda[:batch_size], + True, + attn_num_layers, + attn_num_heads, + self._kv_head_dim_bytes, + cache_mgr.max_total_draft_tokens, + # Dynamic-tree MTP currently supports full-attention KV layers. + # Hybrid managers may store a recurrent-state sentinel first, + # while V2 represents full attention as None, so neither form is + # suitable for the integer maxKVCacheLen operator argument. + cache_mgr.max_seq_len, + pool_pointers, + block_offsets, + cache_mgr.max_blocks_per_seq, + cache_mgr.tokens_per_block, + None, + ) + + # ------------------------------------------------------------------ # + # Top-level forward # + # ------------------------------------------------------------------ # + @nvtx_range("mtp_dyn.forward") + def _forward_impl( + self, + input_ids, + position_ids, + hidden_states, + logits, + attn_metadata, + spec_metadata, + draft_model, + resource_manager=None, + ): + """Run verify, cache promotion, and next-tree drafting.""" + if resource_manager is not None: + self._ensure_spec_tree_manager(resource_manager) + + batch_size = attn_metadata.num_seqs + num_contexts = attn_metadata.num_contexts + num_gens = batch_size - num_contexts + raw_logits = logits + + self._execute_guided_decoder_if_present(logits) + + # (a) Verify previous tree (greedy). Also relocates accepted KV. + accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( + input_ids, logits, spec_metadata, attn_metadata + ) + if num_gens > 0: + self._relocate_kv_eagerly(attn_metadata, batch_size) + + # Dynamic-tree Mamba states are stored by tree-node position. + if self._is_mamba_hybrid_cache is None: + self._is_mamba_hybrid_cache = isinstance( + attn_metadata.kv_cache_manager, MambaHybridCacheManager + ) + if num_gens > 0 and self._is_mamba_hybrid_cache: + accepted_leaf_positions = self._accepted_leaf_intermediate_positions( + num_accepted_tokens, num_contexts, num_gens + ) + attn_metadata.kv_cache_manager.update_mamba_states( + attn_metadata=attn_metadata, + num_accepted_tokens=num_accepted_tokens, + state_indices=attn_metadata.mamba_metadata.state_indices, + accepted_leaf_positions=accepted_leaf_positions, + ) + + # Save attn/spec metadata before the draft loop mutates it. + original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + original_force_prepare_spec_dec_tree_mask = attn_metadata.force_prepare_spec_dec_tree_mask + self._prepare_attn_metadata_for_spec_dec(attn_metadata) + attn_metadata.force_prepare_spec_dec_tree_mask = True + + # (c) Run the MTP draft tree loop -> build + store the next tree. + draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) + next_draft_tokens = self._forward_draft_loop( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + num_accepted_tokens=num_accepted_tokens, + attn_metadata=attn_metadata, + spec_metadata=spec_metadata, + draft_model=draft_model, + draft_kv_cache_manager=draft_kv_cache_manager, + num_contexts=num_contexts, + num_gens=num_gens, + batch_size=batch_size, + ) + + # Restore attn metadata to support cuda graph. + self._restore_attn_metadata_from_spec_dec(attn_metadata) + attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens + attn_metadata.force_prepare_spec_dec_tree_mask = original_force_prepare_spec_dec_tree_mask + attn_metadata.use_spec_decoding = True + + # (d) Prepare next_new_tokens for overlap scheduler. + next_new_tokens = self._prepare_next_new_tokens( + accepted_tokens, + next_draft_tokens, + spec_metadata.batch_indices_cuda, + batch_size, + num_accepted_tokens, + ) + + return { + "logits": raw_logits, + "new_tokens": accepted_tokens, + "new_tokens_lens": num_accepted_tokens, + "next_draft_tokens": next_draft_tokens, + "next_new_tokens": next_new_tokens, + "accepted_draft_tokens_indices": self._accepted_draft_indices_tensor[:batch_size], + } + + # ------------------------------------------------------------------ # + # Step-0 drafter-input repack (dynamic tree) # + # ------------------------------------------------------------------ # + @nvtx_range("mtp_dyn._prepare_step0_drafter_inputs") + def _prepare_step0_drafter_inputs( + self, + input_ids, + position_ids, + last_tokens_idx, + hidden_states, + accepted_tokens, + attn_metadata, + ): + """Repack step-0 drafter inputs to accepted-path layout.""" + num_contexts = attn_metadata.num_contexts + batch_size = attn_metadata.num_seqs + num_gens = batch_size - num_contexts + num_ctx_tokens = attn_metadata.num_ctx_tokens + + # Match MTPEagleWorker context input repack. + input_ids_ctx = self._prepare_context_input_ids( + input_ids, num_ctx_tokens, last_tokens_idx, accepted_tokens, num_contexts + ) + + if num_gens > 0: + max_path_len = self._max_path_len + num_gen_tokens = num_gens * max_path_len + + hidden_dim = hidden_states.shape[-1] + if ( + self._step0_hidden_states_buf is None + or self._step0_hidden_states_buf.shape[-1] != hidden_dim + ): + self._step0_hidden_states_buf = torch.zeros( + self._step0_input_ids_buf.shape[0], + hidden_dim, + dtype=hidden_states.dtype, + device="cuda", + ) + + # Accepted path includes the golden token at column 0. + accept_token = accepted_tokens[num_contexts:batch_size] + + BLOCK_H = triton.next_power_of_2(hidden_dim) + _gather_repack_step0_kernel[(num_gens * max_path_len,)]( + hidden_states, + accept_token, + position_ids, + self._accepted_draft_indices_tensor[num_contexts:batch_size], + self._num_accepted_tokens_buf, + self._step0_hidden_states_buf, + self._step0_input_ids_buf, + self._step0_position_ids_buf, + self._gather_ids_buf, + num_ctx_tokens, + num_contexts, + self.tokens_per_gen_step, + max_path_len, + self.max_draft_len, + hidden_dim, + num_ctx_tokens, # gather_id references combined [ctx|gen] tensor + BLOCK_H=BLOCK_H, + ) + + input_ids = torch.cat( + [input_ids_ctx, self._step0_input_ids_buf[:num_gen_tokens]], dim=0 + ) + position_ids = torch.cat( + [position_ids[:num_ctx_tokens], self._step0_position_ids_buf[:num_gen_tokens]], + dim=0, + ) + hidden_states = torch.cat( + [hidden_states[:num_ctx_tokens], self._step0_hidden_states_buf[:num_gen_tokens]], + dim=0, + ) + + attn_metadata._seq_lens[num_contexts:batch_size].fill_(max_path_len) + attn_metadata._seq_lens_cuda[num_contexts:batch_size].fill_(max_path_len) + attn_metadata.on_update() + else: + # Context-only (warmup): no gen tokens to repack. + input_ids = input_ids_ctx + + return { + "input_ids": input_ids, + "position_ids": position_ids, + "hidden_states": hidden_states, + "attn_metadata": attn_metadata, + } + + # ------------------------------------------------------------------ # + # MTP draft tree loop # + # ------------------------------------------------------------------ # + def _forward_draft_loop( + self, + input_ids, + position_ids, + hidden_states, + accepted_tokens, + num_accepted_tokens, + attn_metadata, + spec_metadata, + draft_model, + draft_kv_cache_manager, + num_contexts, + num_gens, + batch_size, + ): + """Draft the next dynamic tree with growing context.""" + spec_tree_manager = self.spec_tree_manager + + assert batch_size <= self._max_batch_size, ( + f"batch_size {batch_size} exceeds pre-allocated max_batch_size {self._max_batch_size}" + ) + + # Step 0: run MTP over accepted-path rows. + position_ids, last_tokens_idx = self.prepare_position_ids_and_last_tokens( + position_ids, attn_metadata.seq_lens_cuda + ) + inputs = self._prepare_step0_drafter_inputs( + input_ids=input_ids, + position_ids=position_ids, + last_tokens_idx=last_tokens_idx, + hidden_states=hidden_states, + accepted_tokens=accepted_tokens, + attn_metadata=attn_metadata, + ) + + # Reset verify-time tree metadata to accepted-path width. + num_step0_tokens = self._max_path_len + if attn_metadata.spec_decoding_generation_lengths is not None: + total = num_gens * num_step0_tokens + dst = attn_metadata.spec_decoding_position_offsets[:total].view( + num_gens, num_step0_tokens + ) + dst.copy_(self._causal_offs[:num_step0_tokens].unsqueeze(0).expand(num_gens, -1)) + self._apply_spec_metadata(attn_metadata, num_gens, num_step0_tokens) + packed_mask = attn_metadata.spec_decoding_packed_mask + packed_mask[:num_gens].zero_() + packed_mask[:num_gens, :num_step0_tokens, 0] = self._step0_causal_mask[ + :num_step0_tokens + ] + if self._needs_mask_repack: + self._repack_mask_padded_to_packed(packed_mask, num_gens, num_step0_tokens) + attn_metadata.use_spec_decoding = num_gens > 0 + if num_gens > 0 and hasattr(attn_metadata, "kv_lens_cuda"): + attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= self._kv_correction + self._refresh_blackwell_tree_mask_metadata(attn_metadata) + if spec_metadata.all_rank_num_tokens is not None: + # Keep attention/MoE token counts aligned with step-0 repack. + attn_metadata.all_rank_num_tokens = spec_metadata.all_rank_num_tokens + + with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): + hidden_states = draft_model.mtp_layers[0]( + embed_tokens=draft_model.embed_tokens, + all_rank_num_tokens=spec_metadata.all_rank_num_tokens, + **inputs, + ) + + # Gather each request's root hidden state for depth-0 expansion. + self._gather_ids_buf[:num_contexts].copy_(last_tokens_idx[:num_contexts]) + gather_ids = self._gather_ids_buf[:batch_size] + + step0_hs = hidden_states[gather_ids] + logits = draft_model.mtp_layers[0].shared_head( + step0_hs, draft_model.lm_head, attn_metadata, True + ) + + new_draft_tokens, new_draft_scores = self.sample( + logits, self.K, draft_model=draft_model + ) + previous_draft_scores = self.update_draft_tokens_and_scores( + cur_draft_idx=0, + new_draft_tokens=new_draft_tokens, + new_draft_scores=new_draft_scores, + previous_draft_scores=None, + batch_size=batch_size, + attn_metadata=attn_metadata, + ) + self.update_hidden_states(cur_draft_idx=0, batch_size=batch_size, step0_hs=step0_hs) + self._prepare_draft_layer_metadata( + 0, + attn_metadata, + batch_size, + gather_ids, + num_contexts, + num_gens, + num_accepted_tokens, + inputs, + ) + + # Subsequent layers grow the tree. + for layer_idx in range(1, self.max_draft_len): + num_tokens_per_req = layer_idx * self.K + num_infer_tokens = batch_size * num_tokens_per_req + subseq_all_rank_num_tokens = None + if spec_metadata.all_rank_num_seqs is not None: + # Token counts scale with the current tree width. + subseq_all_rank_num_tokens = [ + n * num_tokens_per_req for n in spec_metadata.all_rank_num_seqs + ] + attn_metadata.all_rank_num_tokens = subseq_all_rank_num_tokens + + inp_hs = self._accumulated_hs[:batch_size, :num_tokens_per_req, :].reshape( + num_infer_tokens, -1 + ) + inp_ids = self.draft_tokens_buffer[:batch_size, :num_tokens_per_req].reshape(-1) + inp_pos = self.position_ids_buffer[:batch_size, :num_tokens_per_req].reshape(-1) + layer_inputs = { + "input_ids": inp_ids, + "position_ids": inp_pos, + "hidden_states": inp_hs, + "attn_metadata": attn_metadata, + } + hidden_states = draft_model.mtp_layers[0]( + embed_tokens=draft_model.embed_tokens, + all_rank_num_tokens=subseq_all_rank_num_tokens + or spec_metadata.subseq_all_rank_num_tokens, + **layer_inputs, + ) + + # Take the last K hidden states per request (the new leaves). + hs_reshaped = hidden_states.reshape(batch_size, num_tokens_per_req, -1) + selected_hs = hs_reshaped[:, -self.K :, :].reshape(batch_size * self.K, -1) + logits = draft_model.mtp_layers[0].shared_head( + selected_hs, draft_model.lm_head, attn_metadata, True + ) + + new_draft_tokens, new_draft_scores = self.sample( + logits, self.K, draft_model=draft_model + ) + new_draft_tokens = new_draft_tokens.reshape(batch_size, self.K, self.K) + new_draft_scores = new_draft_scores.reshape(batch_size, self.K, self.K) + + previous_draft_scores = self.update_draft_tokens_and_scores( + cur_draft_idx=layer_idx, + new_draft_tokens=new_draft_tokens, + new_draft_scores=new_draft_scores, + previous_draft_scores=previous_draft_scores, + batch_size=batch_size, + attn_metadata=attn_metadata, + ) + self.update_hidden_states( + cur_draft_idx=layer_idx, + batch_size=batch_size, + hidden_states_to_save=hidden_states, + selected_parents=self._last_selected_parents, + ) + self._prepare_draft_layer_metadata(layer_idx, attn_metadata, batch_size) + + # Resample the final tree and build it into slot_storage. + real_draft_tokens, topk_score_indices = self.resampling_final_draft_tokens(batch_size) + + if spec_tree_manager is not None and num_gens > 0: + self.tree_ops_converter.build_dynamic_tree( + history_draft_tokens_parent_buffer=self.history_draft_tokens_parent_buffer[ + num_contexts:batch_size + ], + topk_score_indices=topk_score_indices[num_contexts:], + tree_mask=spec_tree_manager.spec_dec_packed_mask[:num_gens], + positions=spec_tree_manager.spec_dec_position_offsets[:num_gens], + retrieve_index=spec_tree_manager.retrieve_index[:num_gens], + retrieve_next_token=spec_tree_manager.retrieve_next_token[:num_gens], + retrieve_next_sibling=spec_tree_manager.retrieve_next_sibling[:num_gens], + use_packed_mask=True, + ) + slot_storage = spec_tree_manager.slot_storage + gen_slots = slot_storage.all_ids_buf[num_contexts:batch_size] + spec_tree_manager.scatter_to_slot_storage(slot_storage, gen_slots, num_gens) + + return real_draft_tokens + + def _prepare_draft_layer_metadata( + self, + cur_draft_idx, + attn_metadata, + batch_size, + gather_ids=None, + num_contexts=0, + num_gens=0, + num_accepted_tokens=None, + inputs=None, + ): + """Set attn_metadata seq_lens/kv_lens for the next draft layer.""" + if cur_draft_idx == 0: + base_pos = inputs["position_ids"][gather_ids] + 1 + self.position_ids_buffer[:batch_size, : self.K] = base_pos.unsqueeze(1).expand( + -1, self.K + ) + + attn_metadata._seq_lens[:batch_size].fill_(self.K) + attn_metadata._seq_lens_cuda[:batch_size].fill_(self.K) + attn_metadata.on_update() + + if inputs["attn_metadata"].kv_cache_manager is not None: + attn_metadata.host_request_types[: attn_metadata.num_contexts].fill_(1) + attn_metadata.num_contexts = 0 + + if hasattr(attn_metadata, "kv_lens_cuda"): + # Rewind only unaccepted verify tokens; draft KV is added later. + if num_gens > 0: + attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= ( + self._max_path_len + ) - num_accepted_tokens[num_contexts:batch_size] + attn_metadata.kv_lens_cuda[:batch_size] += self.K + attn_metadata.use_spec_decoding = True + self._refresh_blackwell_tree_mask_metadata(attn_metadata) + else: + num_tokens_previous_layer = cur_draft_idx * self.K + num_tokens_current_layer = self.K * (cur_draft_idx + 1) + prev_pos = self.position_ids_buffer[:batch_size, :num_tokens_previous_layer] + self.position_ids_buffer[ + :batch_size, num_tokens_previous_layer:num_tokens_current_layer + ] = prev_pos[:, -self.K :] + 1 + attn_metadata._seq_lens[:batch_size].fill_(num_tokens_current_layer) + attn_metadata._seq_lens_cuda[:batch_size].fill_(num_tokens_current_layer) + attn_metadata.on_update() + if hasattr(attn_metadata, "kv_lens_cuda"): + attn_metadata.kv_lens_cuda[:batch_size] += self.K + self._refresh_blackwell_tree_mask_metadata(attn_metadata) + + +class MTPEagleDynamicTreeResourceManager(BaseResourceManager): + """Resource manager for MTP dynamic-tree mode.""" + + hidden_states: Optional[torch.Tensor] = None + + def __init__( + self, + config: "MTPDecodingConfig", + dtype: torch.dtype, + hidden_size: int, + max_num_requests: int, + sa_manager=None, + ): + from .spec_tree_manager import SpecTreeManager + + self.max_num_requests = max_num_requests + self.spec_tree_manager = SpecTreeManager( + max_num_requests=max_num_requests, + use_dynamic_tree=True, + max_draft_len=config.max_draft_len, + max_total_draft_tokens=config.tokens_per_gen_step - 1, + eagle_choices=None, + dynamic_tree_max_topK=config.dynamic_tree_max_topK, + ) + # MTP hidden-state slot pools (needed by MTPEagleWorker drafter inputs). + self._mtp_hidden_states_manager = MTPHiddenStatesManager( + config, dtype, hidden_size, max_num_requests, sa_manager=sa_manager + ) + + # Expose the MTPHiddenStatesManager surface MTPSpecMetadata expects. + @property + def slot_manager(self): + return self._mtp_hidden_states_manager.slot_manager + + @property + def mtp_past_hidden_states_pool(self): + return self._mtp_hidden_states_manager.mtp_past_hidden_states_pool + + @property + def mtp_past_tokens_pool(self): + return self._mtp_hidden_states_manager.mtp_past_tokens_pool + + @property + def sa_manager(self): + return self._mtp_hidden_states_manager.sa_manager + + def prepare_resources(self, scheduled_batch: ScheduledRequests): + self._mtp_hidden_states_manager.prepare_resources(scheduled_batch) + + def update_resources(self, scheduled_batch: ScheduledRequests): + self._mtp_hidden_states_manager.update_resources(scheduled_batch) + + def free_resources(self, request: LlmRequest): + # Clear tree validity for the freed slot, then free the MTP slot. + if request.py_seq_slot is not None: + self.spec_tree_manager.slot_storage.mark_invalid(request.py_seq_slot) + self._mtp_hidden_states_manager.free_resources(request) + + def add_dummy_requests(self, request_ids: List[int]): + # Dummies still need MTP hidden-state slots. + self._mtp_hidden_states_manager.add_dummy_requests(request_ids) + + def shutdown(self): + self._mtp_hidden_states_manager.shutdown() + + def get_max_resource_count(self) -> int: + return self.max_num_requests + + def get_needed_resource_to_completion(self, request: LlmRequest): + return 0 diff --git a/tensorrt_llm/_torch/speculative/pard.py b/tensorrt_llm/_torch/speculative/pard.py index 969add54136e..603fa26639c6 100644 --- a/tensorrt_llm/_torch/speculative/pard.py +++ b/tensorrt_llm/_torch/speculative/pard.py @@ -91,6 +91,12 @@ def __init__( self.sa_enhancer: Optional[SADraftEnhancer] = None if getattr(spec_config, "sa_config", None) is not None: self.sa_enhancer = SADraftEnhancer(spec_config.sa_config.threshold) + # Deferred kv_lens_cuda rewind state (see _prepare_kv_for_draft_forward, + # _apply_kv_rewind_after_draft, _ensure_spec_dec_state_restored). + self._kv_rewind_pending = False + self._kv_rewind_amount = None + self._kv_rewind_nc = None + self._kv_rewind_bs = None logger.info( f"PARDWorker initialized with use_separate_draft_kv_cache={use_separate_draft_kv_cache}" ) @@ -144,6 +150,7 @@ def _prepare_kv_for_draft_forward( if batch_size > num_contexts: attn_metadata.kv_lens_cuda[num_contexts:batch_size] += 1 + self._kv_rewind_pending = True attn_metadata.update_for_spec_dec() @@ -155,17 +162,32 @@ def _apply_kv_rewind_after_draft(self, attn_metadata, spec_metadata): by prepare_for_spec_dec) to avoid cumulative shrinkage. Applied during capture and normal inference. """ + self._kv_rewind_pending = False is_warmup = spec_metadata.is_cuda_graph and not torch.cuda.is_current_stream_capturing() if is_warmup: + # kv_lens_cuda was saved by prepare_for_spec_dec in this mode and + # is restored wholesale, so no rewind is needed. return - if hasattr(self, "_kv_rewind_amount") and hasattr(attn_metadata, "kv_lens_cuda"): + if self._kv_rewind_amount is not None and hasattr(attn_metadata, "kv_lens_cuda"): nc = self._kv_rewind_nc bs = self._kv_rewind_bs attn_metadata.kv_lens_cuda[nc:bs] -= self._kv_rewind_amount attn_metadata.kv_lens_cuda[nc:bs].clamp_(min=0) - def forward( + def _ensure_spec_dec_state_restored(self, attn_metadata, spec_metadata): + # Restore first (in warmup mode kv_lens_cuda was saved and comes back + # wholesale), then apply any pending rewind for the other modes so a + # failed draft forward does not leave kv_lens_cuda incremented. + super()._ensure_spec_dec_state_restored(attn_metadata, spec_metadata) + if ( + getattr(self, "_kv_rewind_pending", False) + and attn_metadata is not None + and spec_metadata is not None + ): + self._apply_kv_rewind_after_draft(attn_metadata, spec_metadata) + + def _forward_impl( self, input_ids, position_ids, diff --git a/tensorrt_llm/_torch/speculative/sa_worker.py b/tensorrt_llm/_torch/speculative/sa_worker.py index 41c342e2d2b1..9f97c8d69404 100644 --- a/tensorrt_llm/_torch/speculative/sa_worker.py +++ b/tensorrt_llm/_torch/speculative/sa_worker.py @@ -120,7 +120,7 @@ def __init__(self, spec_config: "SADecodingConfig", model_config=None): def max_draft_len(self) -> int: return self._max_draft_len - def forward( + def _forward_impl( self, input_ids: torch.Tensor, position_ids: torch.Tensor, diff --git a/tensorrt_llm/_torch/speculative/spec_sampler_base.py b/tensorrt_llm/_torch/speculative/spec_sampler_base.py index 3d841aaa4a38..f6b2a60569bb 100644 --- a/tensorrt_llm/_torch/speculative/spec_sampler_base.py +++ b/tensorrt_llm/_torch/speculative/spec_sampler_base.py @@ -77,6 +77,28 @@ class SpecSamplerBase(Sampler[SampleStateSpec], AsyncWorkerMixin): def is_generation_model(self) -> bool: return True + def validate_request(self, request: LlmRequest) -> None: + """Reject sampling parameters the one-model speculative path cannot honor. + + The one-model sampling kernels take only temperature/top_k/top_p (see + SpecMetadata.populate_sampling_params_for_one_model); min_p has no + buffer there, so it would be silently dropped and the request would + decode from a different distribution than the user asked for. Threading + it through costs measurable throughput on the rejection path, so reject + instead. Raised from validate_request (request admission), so only the + offending request fails rather than the whole executor step. + """ + sampling_config = request.sampling_config + if sampling_config is None: + return + # min_p lives on the C++ SamplingConfig as an optional singleton list. + min_p = sampling_config.min_p + if min_p and min_p[0] > 0.0: + raise ValueError( + "min_p is not supported with one-model speculative decoding. " + "Drop min_p from the request, or disable speculative decoding." + ) + @dataclass(kw_only=True) class Store: """Storage for speculative decoding tensors.""" diff --git a/tensorrt_llm/_torch/speculative/spec_tree_manager.py b/tensorrt_llm/_torch/speculative/spec_tree_manager.py index 74dd622826f1..545b5bc7bb06 100644 --- a/tensorrt_llm/_torch/speculative/spec_tree_manager.py +++ b/tensorrt_llm/_torch/speculative/spec_tree_manager.py @@ -16,24 +16,40 @@ class DynamicTreeSlotStorage: Buffers are [S, ...] where S = num_slots + 1 (+1 for CUDA graph dummy). """ - def __init__(self, num_slots: int, n_dt: int, mask_width: int): + def __init__(self, + num_slots: int, + n_dt: int, + mask_width: int, + top_k: int = 1): S = num_slots + 1 self.dummy_slot_id = num_slots - # Slot buffers — C++ kernel writes directly via slotIds - self.packed_mask = torch.zeros((S, n_dt, mask_width), - dtype=torch.int32, - device='cuda') - self.position_offsets = torch.zeros((S, n_dt), - dtype=torch.int32, - device='cuda') + # Bootstrap/reused slots may not have a tree yet; keep their metadata + # as a valid linear chain so verification kernels can read it directly. + no_tree_position_offsets, no_tree_packed_mask = self._make_kary_tree_metadata( + n_dt, mask_width, top_k=1) + self.position_offsets = no_tree_position_offsets.unsqueeze(0).repeat( + S, 1).contiguous() + self.packed_mask = no_tree_packed_mask.unsqueeze(0).repeat( + S, 1, 1).contiguous() + self._no_tree_position_offsets = no_tree_position_offsets + self._no_tree_packed_mask = no_tree_packed_mask + + # CUDA-graph dummies use a deterministic K-ary tree, matching real + # dynamic-tree mask/position shapes without depending on request state. + dummy_position_offsets, dummy_packed_mask = self._make_kary_tree_metadata( + n_dt, mask_width, top_k) + self.position_offsets[self.dummy_slot_id] = dummy_position_offsets + self.packed_mask[self.dummy_slot_id] = dummy_packed_mask self.retrieve_index = torch.zeros((S, n_dt), dtype=torch.int32, device='cuda') - self.retrieve_next_token = torch.full((S, n_dt), - -1, - dtype=torch.int32, - device='cuda') + + # Mamba verify reads next links unconditionally, so no-tree rows must be + # valid linear chains instead of sentinels. + self._no_tree_next_token = self._make_no_tree_next_token(n_dt) + self.retrieve_next_token = self._no_tree_next_token.unsqueeze(0).repeat( + S, 1) self.retrieve_next_sibling = torch.full((S, n_dt), -1, dtype=torch.int32, @@ -57,6 +73,43 @@ def __init__(self, num_slots: int, n_dt: int, mask_width: int): dtype=torch.int32, device='cuda') + @staticmethod + def _make_kary_tree_metadata( + n_dt: int, mask_width: int, + top_k: int) -> tuple[torch.Tensor, torch.Tensor]: + top_k = max(int(top_k), 1) + token_ids = torch.arange(n_dt, device='cuda') + parents = torch.where(token_ids > 0, (token_ids - 1) // top_k, + token_ids) + ancestor_chain = torch.empty((n_dt, n_dt), + dtype=torch.long, + device='cuda') + current = token_ids + for depth in range(n_dt): + ancestor_chain[:, depth] = current + current = parents[current] + + # Pack bits directly from the parent chain instead of materializing a + # dense bool mask and repacking it. + valid_ancestors = torch.ones((n_dt, n_dt), + dtype=torch.bool, + device='cuda') + valid_ancestors[:, 1:] = ancestor_chain[:, 1:] != ancestor_chain[:, :-1] + bit_values = (1 << (ancestor_chain % 32)).to(torch.int32) + bit_values.masked_fill_(~valid_ancestors, 0) + packed_mask = torch.zeros((n_dt, mask_width), + dtype=torch.int32, + device='cuda') + packed_mask.scatter_add_(1, ancestor_chain // 32, bit_values) + position_offsets = valid_ancestors.sum(-1).to(torch.int32) - 1 + return position_offsets, packed_mask + + @staticmethod + def _make_no_tree_next_token(n_dt: int) -> torch.Tensor: + next_token = torch.arange(1, n_dt + 1, dtype=torch.int32, device='cuda') + next_token[n_dt - 1] = -1 + return next_token + def fill_all_slot_ids(self, context_requests, generation_requests): """Fill all_ids_buf for full batch [ctx | gen] via one HtoD copy.""" dummy_slot = self.dummy_slot_id @@ -81,12 +134,12 @@ def mark_valid(self, slot_ids, count): self.has_tree.narrow(0, self.dummy_slot_id, 1).fill_(False) def mark_invalid(self, slot_id): - """Clear validity and reset slot data.""" + """Clear validity and restore valid no-tree metadata.""" self.has_tree[slot_id] = False - self.packed_mask[slot_id] = 0 - self.position_offsets[slot_id] = 0 + self.packed_mask[slot_id] = self._no_tree_packed_mask + self.position_offsets[slot_id] = self._no_tree_position_offsets self.retrieve_index[slot_id] = 0 - self.retrieve_next_token[slot_id] = -1 + self.retrieve_next_token[slot_id] = self._no_tree_next_token self.retrieve_next_sibling[slot_id] = -1 def pack_retrieve_from_slots(self, slot_ids, count): @@ -284,6 +337,7 @@ def init_tree_info_for_dynamic_tree(self): num_slots=self.num_trees, n_dt=num_draft_with_root, mask_width=mask_width, + top_k=self.dynamic_tree_max_topK, ) def scatter_to_slot_storage(self, ss, gen_slots, num_gens): diff --git a/tensorrt_llm/_torch/speculative/utils.py b/tensorrt_llm/_torch/speculative/utils.py index 4f9c5af8846a..bffa8833058c 100644 --- a/tensorrt_llm/_torch/speculative/utils.py +++ b/tensorrt_llm/_torch/speculative/utils.py @@ -20,6 +20,7 @@ from .draft_target import (DraftTargetOneModelSampler, DraftTargetOneModelSpecMetadata, DraftTargetOneModelWorker) +from .dspark import DSparkSpecMetadata, DSparkWorker from .eagle3 import (Eagle3OneModelDynamicTreeResourceManager, Eagle3OneModelSampler, Eagle3OneModelSpecMetadata, Eagle3OneModelWorker, Eagle3ResourceManager, @@ -27,6 +28,8 @@ from .eagle3_dynamic_tree import Eagle3OneModelDynamicTreeWorker from .model_drafter import ModelDrafter from .mtp import MTPHiddenStatesManager, MTPSampler, MTPSpecMetadata, MTPWorker +from .mtp_dynamic_tree import (MTPEagleDynamicTreeResourceManager, + MTPEagleDynamicTreeWorker) from .ngram import NGramDrafter, NGramPoolManager from .pard import PARDSpecMetadata, PARDWorker from .sa_worker import SASampler, SASpecMetadata, SAWorker @@ -108,10 +111,12 @@ def get_spec_metadata(spec_config, hidden_size=model_config.hidden_size, max_num_tokens=max_num_tokens, use_rejection_sampling=use_rejection_sampling, + advanced_sampling_mode=spec_config.advanced_sampling_mode, vocab_size=vocab_size, num_seq_slots=num_seq_slots, draft_vocab_size=draft_vocab_size, spec_resource_manager=spec_resource_manager, + use_dynamic_tree=getattr(spec_config, 'use_dynamic_tree', False), ) if spec_config.spec_dec_mode.is_mtp_vanilla(): return MTPSpecMetadata( @@ -203,6 +208,21 @@ def get_spec_metadata(spec_config, vocab_size=vocab_size, draft_vocab_size=draft_vocab_size, ) + if spec_config.spec_dec_mode.is_dspark(): + target_layer_ids = getattr(spec_config, 'target_layer_ids', None) + return DSparkSpecMetadata( + max_draft_len=spec_config.max_draft_len, + max_total_draft_tokens=spec_config.tokens_per_gen_step - 1, + spec_dec_mode=spec_config.spec_dec_mode, + max_num_requests=max_num_requests, + layers_to_capture=target_layer_ids, + hidden_size=model_config.hidden_size, + max_num_tokens=max_num_tokens, + dtype=model_config.torch_dtype, + use_rejection_sampling=use_rejection_sampling, + vocab_size=vocab_size, + draft_vocab_size=draft_vocab_size, + ) if spec_config.spec_dec_mode.is_draft_target_one_model(): return DraftTargetOneModelSpecMetadata( max_draft_len=spec_config.max_draft_len, @@ -273,6 +293,15 @@ def get_spec_resource_manager(model_engine, draft_model_engine=None): if sa_cfg is not None: sa_manager = SuffixAutomatonManager(sa_cfg, max_num_requests, max_seq_len) + # Dynamic tree combines SpecTreeManager with MTP hidden-state slots. + if getattr(spec_config, 'use_dynamic_tree', False): + return MTPEagleDynamicTreeResourceManager( + spec_config, + model_config.torch_dtype, + model_config.hidden_size, + max_num_requests, + sa_manager=sa_manager, + ) if spec_config.use_relaxed_acceptance_for_thinking or sa_manager is not None: # Unified resource manager: the unified worker reads # ``relaxed_delta_pool`` from ``Eagle3ResourceManager`` (mirrors the @@ -361,7 +390,10 @@ def get_spec_decoder( # MTP Eagle one-model now uses the same sampler as Eagle3 one-model. return Eagle3OneModelSampler(sampler_args, spec_config=spec_config) if spec_config.spec_dec_mode.is_mtp_vanilla(): - return MTPSampler(sampler_args, nextn=spec_config.max_draft_len) + nextn = spec_config.max_draft_len + if getattr(spec_config, "use_dynamic_tree", False): + nextn = spec_config.max_total_draft_tokens + return MTPSampler(sampler_args, nextn=nextn) if spec_config.spec_dec_mode.is_eagle3( ) or spec_config.spec_dec_mode.is_mtp_eagle(): # TorchSampler handles Eagle3 gracefully, by integrating d2t into the sampling process @@ -449,6 +481,11 @@ def get_spec_worker(spec_config, use_separate_draft_kv_cache, mapping=mapping) if spec_dec_mode.is_mtp_eagle_one_model(): + if getattr(spec_config, 'use_dynamic_tree', False): + return MTPEagleDynamicTreeWorker(spec_config, + model_config, + use_separate_draft_kv_cache, + mapping=mapping) return MTPEagleWorker(spec_config, model_config, use_separate_draft_kv_cache, @@ -465,6 +502,8 @@ def get_spec_worker(spec_config, return PARDWorker(spec_config, mapping, use_separate_draft_kv_cache) if spec_dec_mode.is_dflash(): return DFlashWorker(spec_config, mapping, use_separate_draft_kv_cache) + if spec_dec_mode.is_dspark(): + return DSparkWorker(spec_config, mapping, use_separate_draft_kv_cache) if spec_dec_mode.is_sa(): return SAWorker(spec_config, model_config) if spec_dec_mode.is_draft_target_one_model(): @@ -536,7 +575,8 @@ def update_spec_config_from_model_config(spec_config, model_config): f"using max_draft_len={effective_draft_len} draft tokens.") spec_config.max_draft_len = effective_draft_len - spec_config.max_total_draft_tokens = spec_config.max_draft_len + if not spec_config.use_dynamic_tree: + spec_config.max_total_draft_tokens = spec_config.max_draft_len def update_spec_config_from_loaded_model(spec_config, model) -> None: diff --git a/tensorrt_llm/_torch/tensor_lru_cache.py b/tensorrt_llm/_torch/tensor_lru_cache.py index 2d94e75a1868..58e789e86309 100644 --- a/tensorrt_llm/_torch/tensor_lru_cache.py +++ b/tensorrt_llm/_torch/tensor_lru_cache.py @@ -32,6 +32,10 @@ class _Entry(NamedTuple): value: torch.Tensor size_bytes: int + # CUDA event recorded on the producing stream right after the clone in `put`. Consumers on a + # different stream wait on it before reading `value`. `None` for CPU tensors or when the cache + # is not stream-aware. + producer_event: torch.cuda.Event | None class TensorLRUCacheStats(NamedTuple): @@ -77,9 +81,16 @@ class TensorLRUCache(Generic[K]): temporarily needs both the source tensor and its copy and may exceed the cache limit until eviction completes. + In CUDA-stream-aware mode, each entry owns the event recorded after its clone. Replacement, + eviction, and clear drop that event with the entry; events are not reused because an evicted + tensor may still have outstanding consumers on another stream. + Args: max_bytes: Maximum logical tensor bytes held by this cache. name: Short label used in debug log messages. + cuda_stream_aware: When enabled, synchronize CUDA tensor producers and consumers across + streams and extend allocation lifetime through every consuming stream. CPU tensors are + unaffected. """ def __init__( @@ -87,12 +98,14 @@ def __init__( max_bytes: int, *, name: str = "tensor_lru_cache", + cuda_stream_aware: bool = False, ) -> None: if max_bytes <= 0: raise ValueError("max_bytes must be positive") self._max_bytes = max_bytes self._name = name + self._cuda_stream_aware = cuda_stream_aware self._current_bytes = 0 self._items: OrderedDict[K, _Entry] = OrderedDict() self._lock = RLock() @@ -124,6 +137,7 @@ def get(self, key: K) -> torch.Tensor | None: self._counters.hits += 1 self._items.move_to_end(key) + self._prepare_for_current_stream(entry) return entry.value def put(self, key: K, value: torch.Tensor) -> bool: @@ -144,6 +158,10 @@ def put(self, key: K, value: torch.Tensor) -> bool: return False stored_value = value.detach().clone() + producer_event = None + if self._cuda_stream_aware and stored_value.is_cuda: + producer_event = torch.cuda.Event() + producer_event.record(torch.cuda.current_stream(stored_value.device)) with self._lock: old_entry = self._items.pop(key, None) @@ -153,7 +171,11 @@ def put(self, key: K, value: torch.Tensor) -> bool: else: self._counters.insertions += 1 - self._items[key] = _Entry(value=stored_value, size_bytes=size_bytes) + self._items[key] = _Entry( + value=stored_value, + size_bytes=size_bytes, + producer_event=producer_event, + ) self._current_bytes += size_bytes evicted_count, evicted_bytes = self._evict_until_within_limit() @@ -174,6 +196,7 @@ def pop(self, key: K) -> torch.Tensor | None: return None self._current_bytes -= entry.size_bytes + self._prepare_for_current_stream(entry) return entry.value def clear(self) -> None: @@ -210,6 +233,25 @@ def log_stats(self, reason: str) -> None: def _tensor_size_bytes(tensor: torch.Tensor) -> int: return tensor.numel() * tensor.element_size() + def _prepare_for_current_stream(self, entry: _Entry) -> None: + """Order and anchor a cached tensor for consumption on the current stream. + + Called from `get` / `pop` before returning an entry it: + * makes the current (consuming) stream wait on the entry's producer event, so a cross-stream + read observes fully-written data + * calls `record_stream` on the consuming stream so the caching allocator will not reuse + the storage while consumer-stream work is still pending, even if a later replacement or + eviction drops the cache's own reference. + """ + if not self._cuda_stream_aware or not entry.value.is_cuda: + return + + consumer_stream = torch.cuda.current_stream(entry.value.device) + # The producer event orders the data dependency; `record_stream` separately guards lifetime. + if entry.producer_event is not None: + consumer_stream.wait_event(entry.producer_event) + entry.value.record_stream(consumer_stream) + def _evict_until_within_limit(self) -> tuple[int, int]: evicted_count = 0 evicted_bytes = 0 diff --git a/tensorrt_llm/_torch/utils.py b/tensorrt_llm/_torch/utils.py index fccba779b253..dd2c3a7527c0 100644 --- a/tensorrt_llm/_torch/utils.py +++ b/tensorrt_llm/_torch/utils.py @@ -41,6 +41,11 @@ ) +def is_gdn_replay_enabled() -> bool: + """Return whether GDN replay was explicitly enabled.""" + return os.environ.get("TRTLLM_USE_GDN_REPLAY", "0") == "1" + + # IMPORTANT: Keep the same order of activation functions in this enum and the enum in # cpp/tensorrt_llm/kernels/cutlass_kernels/include/common.h class ActivationType(IntEnum): diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/__init__.py b/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/__init__.py index 292be3036d5b..9b70421c3b81 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/__init__.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/__init__.py @@ -15,7 +15,7 @@ """ CuTe DSL attention backend family for visual generation models. - fmha.py — CuTeDSLAttention (dense cubin path, head_dim=128) + fmha.py — CuTeDSLAttention (dense and blockscaled JIT FMHA) vsa.py — VSAAttention (Video Sparse Attention, CuTe JIT + SDPA fallback) """ diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/fmha.py b/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/fmha.py index c15f8b2ac47d..b0e5b882afdc 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/fmha.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/fmha.py @@ -15,40 +15,568 @@ """ CuTe DSL (NVIDIA kernels) Dense FMHA Backend for Visual Generation Models -Uses pre-compiled cubins derived from CUTLASS CuTe DSL FMHA. -Expects NHD layout ([B, S, H, D]) and supports float16/bfloat16. -For the VSA sparse path use VSAAttention in vsa.py. +JIT-compiles the dense FMHA kernel and caches the compiled artifact for each kernel configuration. +Expects NHD layout ([B, S, H, D]) and supports float16/bfloat16 inputs. The VSA sparse path uses +VSAAttention from vsa.py instead. """ import math -from typing import Optional, Tuple +from typing import Any, NamedTuple, Tuple import torch +from tensorrt_llm.logger import logger from tensorrt_llm.visual_gen.args import QuantAttentionConfig from ....attention_backend.interface import PredefinedAttentionMask from ..interface import AttentionBackend, AttentionTensorLayout -_cute_dsl_import_error = None +_cute_dsl_import_error: BaseException | None = None try: - import tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention as cute_dsl - from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention.fmha import ( - _cute_runtime_import_error, - ) + import cutlass + from cuda.bindings import driver as cuda_driver + from cutlass import cute + from cutlass.cute import typing as cute_typing + from cutlass.cute.runtime import from_dlpack - if _cute_runtime_import_error is not None: - raise ImportError(_cute_runtime_import_error) + from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention import ( + BlackwellFusedMultiHeadAttentionForward, + BlackwellFusedMultiHeadBlockScaledAttentionForward, + ) + from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.helpers import fmha_helpers as fmha_utils except (ImportError, OSError) as e: - cute_dsl = None + cutlass = None + cuda_driver = None + cute = None + cute_typing = None + from_dlpack = None + BlackwellFusedMultiHeadAttentionForward = None + BlackwellFusedMultiHeadBlockScaledAttentionForward = None + fmha_utils = None _cute_dsl_import_error = e +SUPPORTED_GPU_ARCHS: Tuple[str, ...] = ("sm_100a", "sm_103a") + + +# ============================================================================ +# Runtime helpers +# ============================================================================ + + +def _check_cute_runtime_available() -> None: + if _cute_dsl_import_error is None: + return + raise ImportError( + f"CuTe DSL runtime is not available. Import error: {_cute_dsl_import_error}" + ) from _cute_dsl_import_error + + +def _get_gpu_arch(device: torch.device | None = None) -> str: + capability = torch.cuda.get_device_capability(device) + gpu_arch = f"sm_{capability[0]}{capability[1]}a" + if gpu_arch not in SUPPORTED_GPU_ARCHS: + supported = ", ".join(SUPPORTED_GPU_ARCHS) + raise ValueError( + f"Unsupported GPU architecture {gpu_arch}. Supported architectures: {supported}." + ) + return gpu_arch + + +def _validate_inputs( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + o: torch.Tensor, +) -> None: + """Validate the (B, S, H, D) layout the backend feeds the kernel.""" + if not (q.dim() == k.dim() == v.dim() == o.dim() == 4): + raise ValueError("FMHA expects 4D (B, S, H, D) Q/K/V/O tensors.") + if q.dtype != k.dtype: + raise ValueError(f"Q/K dtype mismatch: {q.dtype} vs {k.dtype}") + if q.shape[0] != k.shape[0]: + raise ValueError(f"Batch size mismatch: q={q.shape[0]} vs k={k.shape[0]}") + if q.shape[-1] != k.shape[-1]: + raise ValueError(f"Q/K head dim mismatch: {q.shape[-1]} vs {k.shape[-1]}") + if k.shape[:-1] != v.shape[:-1]: + raise ValueError(f"K/V shape mismatch: {k.shape[:-1]} vs {v.shape[:-1]}") + expected_o_shape = (*q.shape[:-1], v.shape[-1]) + if tuple(o.shape) != expected_o_shape: + raise ValueError(f"Output shape mismatch: {tuple(o.shape)} vs {expected_o_shape}") + + +def _to_cute_tensor(tensor: torch.Tensor, leading_dim: int, cutlass_element_type=None): + """Wrap a torch tensor as a CuTe tensor. + + For sub-byte / non-torch-dtype elements (FP4 packed as uint8, MXFP8 SF exponents stored as + uint8), pass `cutlass_element_type` to override the interpretation; the tensor storage must be + byte-addressable (uint8 / int8). Otherwise the FP8-e4m3 path and the default + direct-from-dlpack path apply. + """ + # Match cutlass.torch.cute_tensor_like: set element_type BEFORE mark_layout_dynamic so the + # layout transformation sees the override-typed tensor and carries it forward. + if cutlass_element_type is not None: + cute_tensor = from_dlpack(tensor.view(torch.int8), assumed_align=16) + cute_tensor = cute_tensor.mark_layout_dynamic(leading_dim=leading_dim) + cute_tensor.element_type = cutlass_element_type + return cute_tensor + if tensor.dtype == torch.float8_e4m3fn: + cute_tensor = from_dlpack(tensor.view(torch.int8), assumed_align=16) + cute_tensor = cute_tensor.mark_layout_dynamic(leading_dim=leading_dim) + cute_tensor.element_type = cutlass.Float8E4M3FN + return cute_tensor + return from_dlpack(tensor, assumed_align=16).mark_layout_dynamic(leading_dim=leading_dim) + + +# ============================================================================ +# JIT compile + cache +# ============================================================================ + + +def _torch_to_cutlass_dtype(t: torch.dtype): + table = { + torch.float16: cutlass.Float16, + torch.bfloat16: cutlass.BFloat16, + torch.float32: cutlass.Float32, + torch.float8_e4m3fn: cutlass.Float8E4M3FN, + } + try: + return table[t] + except KeyError as exc: + raise ValueError(f"Unsupported torch dtype for CuTe DSL FMHA: {t}") from exc + + +class _CacheKey(NamedTuple): + qk_cutlass_dtype: Any + pv_cutlass_dtype: Any + out_cutlass_dtype: Any + qk_acc_dtype: Any + pv_acc_dtype: Any + head_dim: int + head_dim_v: int + mma_tiler_mn: Tuple[int, int] + qk_sf_vec: int # 0 = dense Q/K; 32 = MXFP8; 16 = NVFP4 + is_persistent: bool + mask_type: Any # fmha_utils.MaskEnum + with_lse: bool + with_sink: bool + with_scale_v_channels: bool + has_window: bool + has_skip_softmax: bool + use_tma_store: bool + enable_ex2_emulation: bool + enable_skip_correction: bool + gpu_arch_str: str + + +_COMPILE_CACHE: dict = {} + + +def clear_cute_dsl_fmha_cache() -> None: + """Drop all compiled CuTe DSL FMHA kernels (for tests / teardown).""" + _COMPILE_CACHE.clear() + + +def _get_or_compile(key: _CacheKey, compile_args: tuple): + cached = _COMPILE_CACHE.get(key) + if cached is not None: + return cached + hd, hd_v = key.head_dim, key.head_dim_v + head_dim_arg = hd if hd == hd_v else (hd, hd_v) + if key.qk_sf_vec != 0: + fmha = BlackwellFusedMultiHeadBlockScaledAttentionForward( + key.qk_acc_dtype, + key.pv_acc_dtype, + key.mma_tiler_mn, + head_dim_arg, + key.is_persistent, + key.mask_type, + key.enable_ex2_emulation, + key.enable_skip_correction, + key.qk_sf_vec, + use_tma_store=key.use_tma_store, + ) + else: + fmha = BlackwellFusedMultiHeadAttentionForward( + key.qk_acc_dtype, + key.pv_acc_dtype, + key.mma_tiler_mn, + head_dim_arg, + key.is_persistent, + key.mask_type, + key.enable_ex2_emulation, + key.enable_skip_correction, + use_tma_store=key.use_tma_store, + ) + logger.info( + f"Compiling CuTe DSL FMHA kernel for {key.qk_cutlass_dtype.__name__}/" + f"{key.pv_cutlass_dtype.__name__} head_dim={key.head_dim} " + f"mask={key.mask_type.name} persistent={key.is_persistent} " + f"lse={key.with_lse} on {key.gpu_arch_str} ..." + f"qk_sf_vec={key.qk_sf_vec} " + ) + compiled = cute.compile(fmha, *compile_args) + _COMPILE_CACHE[key] = compiled + return compiled + + +@torch.compiler.disable +def cute_dsl_fmha_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + o: torch.Tensor, + *, + is_causal: bool = False, + sm_scale: float | None = None, + window_left: int = -1, + window_right: int = -1, + lse: torch.Tensor | None = None, + scale_q: float | torch.Tensor = 1.0, + scale_k: float | torch.Tensor = 1.0, + scale_v: float | torch.Tensor = 1.0, + scale_v_channels: torch.Tensor | None = None, + scale_o: float | torch.Tensor = 1.0, + is_persistent: bool = True, + skip_softmax_threshold_scale_factor: float | None = None, + qk_sf_vec: int = 0, + q_sf: torch.Tensor | None = None, + k_sf: torch.Tensor | None = None, + qk_cutlass_dtype: Any = None, +) -> None: + """JIT-compile (or fetch from cache) and launch the CuTe DSL FMHA kernel. + + Expects contiguous 4D (B, S, H, D) Q/K/V/O tensors with uniform per-batch sequence lengths. + Varlen via indptr is intentionally not exposed — callers pack uniform-length sequences. + + When `qk_sf_vec` is non-zero, dispatches to the block-scaled kernel class: + 32 selects MXFP8 (Q/K stored as FP8 e4m3, SFs as Float8E8M0FNU uint8 storage); + 16 selects NVFP4 (Q/K stored as packed FP4 in torch.uint8, SFs as Float8E4M3FN in uint8 storage). + """ + _check_cute_runtime_available() + _validate_inputs(q, k, v, o) + if qk_sf_vec != 0: + if q_sf is None or k_sf is None: + raise ValueError("Block-scaled path (qk_sf_vec != 0) requires q_sf and k_sf tensors.") + if not q_sf.is_contiguous() or not k_sf.is_contiguous(): + raise ValueError("q_sf and k_sf must be contiguous.") + elif scale_v_channels is not None: + raise ValueError("scale_v_channels is only supported by MXFP8 and NVFP4 kernels.") + + # The kernel hard-codes dense strides in its CuTe layout (fmha.py:447-461) and ignores the + # input tensor's actual strides, so non-dense inputs (e.g. `qkv.split(...)` views) would + # be read at wrong offsets. .contiguous() is a no-op when the tensor is already dense. + q = q.contiguous() + k = k.contiguous() + v = v.contiguous() + if not o.is_contiguous(): + raise ValueError("Output tensor `o` must be contiguous (writes happen in place).") + if lse is not None and not lse.is_contiguous(): + raise ValueError("LSE tensor must be contiguous (writes happen in place).") + + # Delay scalar extraction to inside @torch.compiler.disable decoration + def _scalar_float(t): + if isinstance(t, torch.Tensor): + return t.item() + else: + return t + + scale_q = _scalar_float(scale_q) + scale_k = _scalar_float(scale_k) + scale_v = _scalar_float(scale_v) + scale_o = _scalar_float(scale_o) + + # Reshape (B, S, H, D) → (B, S, h_kv, h_r, D) for Q/O and (B, S, h_kv, 1, D) for K/V; LSE + # goes (B, S, H) → (B, S, h_kv, h_r). Layout matches fmha.py:run() (no head_dim split). + # For NVFP4 (qk_cutlass_dtype == Float4E2M1FN), Q/K's storage last-dim is head_dim//2. + # We keep the packed shape through the view; the kernel's FP4 element_type override resolves + # element-unit strides to the right byte addresses. + batch_size, seq_len_q, num_heads_q, qk_storage_dim = q.shape + _, seq_len_kv, num_heads_kv, _ = k.shape + value_head_dim = v.shape[-1] + num_head_groups = num_heads_q // num_heads_kv + is_fp4 = qk_cutlass_dtype is cutlass.Float4E2M1FN + head_dim = qk_storage_dim * 2 if is_fp4 else qk_storage_dim + if qk_sf_vec != 0 and head_dim != 128: + raise ValueError( + f"MXFP8 / NVFP4 (qk_sf_vec={qk_sf_vec}) currently requires head_dim=128, " + f"got head_dim={head_dim}." + ) + if scale_v_channels is not None: + expected_scale_shape = (num_heads_kv, value_head_dim) + if tuple(scale_v_channels.shape) != expected_scale_shape: + raise ValueError( + f"scale_v_channels must have shape {expected_scale_shape}; " + f"got {tuple(scale_v_channels.shape)}." + ) + if scale_v_channels.dtype != torch.float32: + raise ValueError("scale_v_channels must use torch.float32.") + if scale_v_channels.device != v.device: + raise ValueError("scale_v_channels must be on the same device as V.") + if not scale_v_channels.is_contiguous(): + raise ValueError("scale_v_channels must be contiguous.") + + q_5d = q.view(batch_size, seq_len_q, num_heads_kv, num_head_groups, qk_storage_dim) + o_5d = o.view(batch_size, seq_len_q, num_heads_kv, num_head_groups, value_head_dim) + k_5d = k.view(batch_size, seq_len_kv, num_heads_kv, 1, qk_storage_dim) + v_5d = v.view(batch_size, seq_len_kv, num_heads_kv, 1, value_head_dim) + lse_4d = ( + lse.view(batch_size, seq_len_q, num_heads_kv, num_head_groups) if lse is not None else None + ) + + # Map is_causal / window args onto the published MaskEnum surface. + has_window = is_causal or window_left != -1 or window_right != -1 + if is_causal: + mask_type = fmha_utils.MaskEnum.WINDOW_MASK + ws_l_int = None if window_left == -1 else window_left + ws_r_int = 0 if window_right == -1 else window_right + elif has_window: + mask_type = fmha_utils.MaskEnum.WINDOW_MASK + ws_l_int = None if window_left == -1 else window_left + ws_r_int = None if window_right == -1 else window_right + else: + mask_type = fmha_utils.MaskEnum.RESIDUAL_MASK + ws_l_int = None + ws_r_int = None + + if sm_scale is None: + sm_scale = 1.0 / math.sqrt(head_dim) + scale_softmax = scale_q * scale_k * sm_scale + scale_softmax_log2 = scale_softmax * math.log2(math.exp(1.0)) + scale_output = scale_v / scale_o + + use_skip_softmax = ( + skip_softmax_threshold_scale_factor is not None and skip_softmax_threshold_scale_factor > 0 + ) + skip_threshold_log2 = ( + cute_typing.Float32(math.log2(skip_softmax_threshold_scale_factor / seq_len_kv)) + if use_skip_softmax + else None + ) + + # For the block-scaled paths Q/K may be FP4 (stored as torch.uint8) or FP8 e4m3; pass + # qk_cutlass_dtype to override the inferred element type for FP4 (FP8 e4m3 is auto-detected). + q_cute = _to_cute_tensor(q_5d, leading_dim=4, cutlass_element_type=qk_cutlass_dtype) + k_cute = _to_cute_tensor(k_5d, leading_dim=4, cutlass_element_type=qk_cutlass_dtype) + v_cute = _to_cute_tensor(v_5d, leading_dim=4) + o_cute = _to_cute_tensor(o_5d, leading_dim=4) + if qk_sf_vec != 0: + # MXFP8 SFs are Float8E8M0FNU (uint8 storage); NVFP4 SFs are Float8E4M3FN. + sf_dtype = cutlass.Float8E8M0FNU if qk_sf_vec == 32 else cutlass.Float8E4M3FN + q_sf_cute = _to_cute_tensor(q_sf, leading_dim=0, cutlass_element_type=sf_dtype) + k_sf_cute = _to_cute_tensor(k_sf, leading_dim=0, cutlass_element_type=sf_dtype) + scale_v_channels_cute = ( + _to_cute_tensor(scale_v_channels.view(-1), leading_dim=0) + if scale_v_channels is not None + else None + ) + else: + q_sf_cute = None + k_sf_cute = None + scale_v_channels_cute = None + # lse_4d is (B, S_q, h_kv, h_r) contiguous → h_r is the stride-1 inner dim (index 3). + lse_cute = ( + from_dlpack(lse_4d, assumed_align=16).mark_layout_dynamic(leading_dim=3) + if lse_4d is not None + else None + ) + + ws_left = None if ws_l_int is None else cute_typing.Int32(ws_l_int) + ws_right = None if ws_r_int is None else cute_typing.Int32(ws_r_int) + + # problem_size = (b, s_q_max, s_lse_max, s_k_max, h_q, h_k, d, dv); with cum_seqlen_* = None, + # s_lse_max collapses to s_q (per fmha.py:run()). + problem_size = ( + batch_size, + seq_len_q, + seq_len_q, + seq_len_kv, + num_heads_q, + num_heads_kv, + head_dim, + value_head_dim, + ) + stream = cuda_driver.CUstream(torch.cuda.current_stream(q.device).cuda_stream) + + gpu_arch_str = _get_gpu_arch(q.device) + qk_dtype_cache = ( + qk_cutlass_dtype if qk_cutlass_dtype is not None else _torch_to_cutlass_dtype(q.dtype) + ) + # SM100 needs ex2 emulation; SM103 (and any other SM10X SKU) does not. + enable_ex2_emulation = gpu_arch_str == "sm_100a" + + key = _CacheKey( + qk_cutlass_dtype=qk_dtype_cache, + pv_cutlass_dtype=_torch_to_cutlass_dtype(v.dtype), + out_cutlass_dtype=_torch_to_cutlass_dtype(o.dtype), + qk_acc_dtype=cutlass.Float32, + pv_acc_dtype=cutlass.Float32, + head_dim=head_dim, + head_dim_v=value_head_dim, + mma_tiler_mn=(128, 128), + qk_sf_vec=qk_sf_vec, + is_persistent=is_persistent, + mask_type=mask_type, + with_lse=lse is not None, + with_sink=False, + with_scale_v_channels=scale_v_channels is not None, + has_window=has_window, + has_skip_softmax=use_skip_softmax, + use_tma_store=True, + enable_ex2_emulation=enable_ex2_emulation, + enable_skip_correction=True, + gpu_arch_str=gpu_arch_str, + ) + + if qk_sf_vec != 0: + launch_args = ( + q_cute, + k_cute, + q_sf_cute, + k_sf_cute, + v_cute, + o_cute, + problem_size, + None, # cum_seqlen_q + None, # cum_seqlen_k + lse_cute, + None, # sink + cute_typing.Float32(scale_softmax_log2), + cute_typing.Float32(scale_softmax), + cute_typing.Float32(scale_output), + scale_v_channels_cute, + skip_threshold_log2, + ws_left, + ws_right, + None, # skip_softmax_count + None, # total_softmax_count + stream, + False, # use_pdl + ) + else: + launch_args = ( + q_cute, + k_cute, + v_cute, + o_cute, + problem_size, + None, # cum_seqlen_q + None, # cum_seqlen_k + lse_cute, + None, # sink + cute_typing.Float32(scale_softmax_log2), + cute_typing.Float32(scale_softmax), + cute_typing.Float32(scale_output), + skip_threshold_log2, + ws_left, + ws_right, + None, # skip_softmax_count + None, # total_softmax_count + stream, + False, # use_pdl + ) + + compiled = _get_or_compile(key, launch_args) + compiled(*launch_args) + + +# ============================================================================ +# Block-scaled (MXFP8 / NVFP4) Q/K quantization +# ============================================================================ + + +_FP8_E4M3_MAX = 448.0 # FP8 e4m3 max magnitude +_FP4_E2M1_MAX = 6.0 # FP4 e2m1 max magnitude + + +def _quantize_fp8_v( + v_bshd: torch.Tensor, per_head_channel: bool +) -> Tuple[torch.Tensor, float | torch.Tensor, torch.Tensor | None]: + """Quantize V to FP8 with either one tensor scale or an (H, D) scale tensor.""" + if per_head_channel: + v_qscale = _FP8_E4M3_MAX / v_bshd.float().abs().amax(dim=(0, 1)).clamp(min=1e-3) + v_quantized = (v_bshd * v_qscale).to(torch.float8_e4m3fn) + return v_quantized, 1.0, v_qscale.reciprocal().contiguous() + + v_qscale = _FP8_E4M3_MAX / v_bshd.abs().amax().clamp(min=1e-3) + v_quantized = (v_bshd * v_qscale).to(torch.float8_e4m3fn) + return v_quantized, v_qscale.reciprocal(), None + + +def _quantize_blockscaled_one( + x_bshd: torch.Tensor, qk_sf_vec: int +) -> Tuple[torch.Tensor, torch.Tensor, float]: + """Quantize a single (B, S, H, D) bf16/fp16 tensor for the block-scaled kernel. + + Steps: + 1. Transpose to (B, H, S, D) — GEMM-compatible (num_heads is batch-like). + 2. Pad S -> S_pad (multiple of 128) along the row axis. + 3. Invoke the TRT-LLM op with proper options. + 4. Reshape the quantized data, unpad to S, and transpose back to BSHD. + The SF tensor stays at S_pad per kernel constraints. + + The returned data tensor's last dim is the *storage* dim: D for MXFP8 ; D/2 for NVFP4. + + Returns: + x_q: quantized data in (B, S, H, D_storage); + x_sf: swizzled SF tensor produced by the op (kept padded to S_pad). + scale: scalar dequant factor to fold into scale_softmax. + 1.0 for MXFP8 (per-block SFs carry the full range); + amax/(448*6) for NVFP4 (the per-tensor global scale). + """ + if x_bshd.dim() != 4: + raise ValueError(f"_quantize_blockscaled_one expects (B, S, H, D); got {x_bshd.shape}") + batch_size, seq_len, num_heads, head_dim = x_bshd.shape + + x_bhsd = x_bshd.transpose(1, 2).contiguous() # (B, H, S, D) + s_pad = ((seq_len + 127) // 128) * 128 + if s_pad != seq_len: + pad = torch.zeros( + batch_size, + num_heads, + s_pad - seq_len, + head_dim, + dtype=x_bhsd.dtype, + device=x_bhsd.device, + ) + x_bhsd = torch.cat([x_bhsd, pad], dim=2) + x_2d = x_bhsd.reshape(batch_size * num_heads * s_pad, head_dim) + + if qk_sf_vec == 32: + # MXFP8: per-32-element block, UE8M0 SFs in swizzled layout. + x_q_2d, x_sf = torch.ops.trtllm.mxfp8_quantize(x_2d, True, alignment=32) + # x_q_2d shape: (M, D) fp8_e4m3fn; storage last-dim == logical head_dim. + x_q = x_q_2d.view(batch_size, num_heads, s_pad, head_dim) + x_q = x_q[:, :, :seq_len, :].transpose(1, 2).contiguous() # (B, S, H, D) + return x_q, x_sf, 1.0 + + if qk_sf_vec == 16: + # NVFP4: per-16-element block, E4M3 SFs in swizzled layout; per-tensor global scale folded + # into the returned `scale` (caller multiplies it into scale_softmax via scale_q / scale_k). + amax = x_2d.float().abs().amax().clamp(min=1e-6) + global_sf = (_FP8_E4M3_MAX * _FP4_E2M1_MAX) / amax + global_sf_t = global_sf.to(torch.float32).reshape(1) + x_q_2d, x_sf = torch.ops.trtllm.fp4_quantize(x_2d, global_sf_t, 16, False) + # x_q_2d shape: (M, D/2) uint8 — natural packed layout (2 FP4 / byte). + head_dim_packed = head_dim // 2 + x_q = x_q_2d.view(batch_size, num_heads, s_pad, head_dim_packed) + x_q = x_q[:, :, :seq_len, :].transpose(1, 2).contiguous() # (B, S, H, D/2) + return x_q, x_sf, amax / (_FP8_E4M3_MAX * _FP4_E2M1_MAX) + + raise ValueError(f"Unsupported qk_sf_vec={qk_sf_vec}; expected 0, 16, or 32.") + + +# ============================================================================ +# VisualGen AttentionBackend class +# ============================================================================ + + class CuTeDSLAttention(AttentionBackend): """ CuTe DSL (NVIDIA kernels) backend for diffusion models. - Uses pre-compiled cubin kernels (head_dim=128 only). + JIT-compiles BlackwellFusedMultiHeadAttentionForward on first use and caches the + compiled artifact per (dtype, mask, head_dim, ...) configuration. """ def __init__( @@ -56,15 +584,12 @@ def __init__( layer_idx: int = 0, num_heads: int = 8, head_dim: int = 64, - num_kv_heads: Optional[int] = None, - dtype: Optional[torch.dtype] = None, - quant_attention_config: Optional[QuantAttentionConfig] = None, - skip_softmax_threshold_scale: Optional[float] = None, + num_kv_heads: int | None = None, + dtype: torch.dtype | None = None, + quant_attention_config: QuantAttentionConfig | None = None, + skip_softmax_threshold_scale: float | None = None, **kwargs, ): - # Only head_dim=128 cubins are packaged. - if head_dim != 128: - raise ValueError(f"CUTEDSL cubins require head_dim=128, got head_dim={head_dim}.") self.layer_idx = layer_idx self.num_heads = num_heads self.head_dim = head_dim @@ -77,6 +602,31 @@ def __init__( # CuTe DSL expects [B, S, H, D] format self._preferred_layout = AttentionTensorLayout.NHD + def _smooth_qk( + self, + q: torch.Tensor, + k: torch.Tensor, + alpha: float = 0.7, + ): + """Smooth-Smooth technique: + Performs SmoothQuant-like handling for Qk, leveraging the additional fact that + K - K_mean doesn't affect attention result. + """ + k = k.unflatten(2, (-1, 1)) # (B, S, H_K, 1, D) + q = q.unflatten(2, (k.shape[2], -1)) # (B, S, H_K, H_R, D) + q_max = ( + q.abs().amax(dim=(0, 1, 3), keepdim=True).float().clamp_min(1e-4) + ) # (1, 1, H_K, 1, D) + k_max = ( + k.abs().amax(dim=(0, 1, 3), keepdim=True).float().clamp_min(1e-4) + ) # (1, 1, H_K, 1, D) + s = (q_max.pow(alpha) / k_max.pow(1 - alpha)).clamp(1e-4, 1e4) + q = q * s.reciprocal().bfloat16() + # Per-channel shift commutes with the per-channel smooth scale `s`. + k = k - k.mean(dim=(0, 1, 3), keepdim=True) + k = k * s.bfloat16() + return q.flatten(2, 3), k.flatten(2, 3) + def _prepare_inputs( self, q: torch.Tensor, @@ -92,16 +642,22 @@ def _prepare_inputs( is_causal = attention_mask == PredefinedAttentionMask.CAUSAL - # Packaged cubins support float16 and bfloat16 only. + # Perform QK-smoothing if Bmm1 is to be quantized. + qac = self.quant_attention_config + smooth_qk = qac is not None and qac.qk_dtype not in ["bf16", "fp16"] + + # Published kernel supports float16 and bfloat16 only. origin_dtype = q.dtype if q.dtype not in (torch.float16, torch.bfloat16): q = q.to(torch.bfloat16) k = k.to(torch.bfloat16) v = v.to(torch.bfloat16) + if smooth_qk: + q, k = self._smooth_qk(q, k) return q, k, v, is_causal, origin_dtype - # cute_dsl.cute_dsl_fmha_fwd is already decorated with @torch.compiler.disable - # Allow torch.compile to fuse preceding linear/norm with quantization of V / seq-preprocess + # cute_dsl_fmha_fwd is already @torch.compiler.disable'd, so torch.compile may still fuse + # preceding linear/norm with the V quantization below. def _fwd( self, q: torch.Tensor, @@ -111,7 +667,7 @@ def _fwd( **kwargs, ) -> Tuple[torch.Tensor, torch.Tensor]: batch_size, seq_len_q, num_heads, _ = q.shape - _, seq_len_kv, _, value_head_dim = v.shape + value_head_dim = v.shape[-1] out = torch.empty( batch_size, seq_len_q, @@ -128,42 +684,50 @@ def _fwd( device=q.device, ) - # Options that instructs quantization of V + # V is tensor-scaled by default. MXFP8/NVFP4 with v_block_size=1 use an (H, D) scale. scale_v = kwargs.get("scale_v", 1.0) - if self.quant_attention_config is not None: - v_qscale = 448.0 / v.abs().amax().clamp(min=1e-3) - v = (v * v_qscale).to(torch.float8_e4m3fn) - scale_v = scale_v / v_qscale - - # Sequence preproc. - qo_indptr_host = [i * seq_len_q for i in range(batch_size + 1)] - qo_indptr = torch.tensor(qo_indptr_host).to(device=q.device, dtype=torch.int32) - kv_indptr_host = [i * seq_len_kv for i in range(batch_size + 1)] - kv_indptr = torch.tensor(kv_indptr_host).to(device=q.device, dtype=torch.int32) + scale_q = kwargs.get("scale_q", 1.0) + scale_k = kwargs.get("scale_k", 1.0) + qac = self.quant_attention_config + q_sf = k_sf = qk_cutlass_dtype = None + qk_sf_vec = 0 + scale_v_channels = None + if qac is not None: + if qac.qk_dtype in ("mxfp8", "nvfp4"): + qk_sf_vec = 32 if qac.qk_dtype == "mxfp8" else 16 + q, q_sf, gs_q = _quantize_blockscaled_one(q, qk_sf_vec) + k, k_sf, gs_k = _quantize_blockscaled_one(k, qk_sf_vec) + scale_q = scale_q * gs_q + scale_k = scale_k * gs_k + qk_cutlass_dtype = cutlass.Float4E2M1FN if qk_sf_vec == 16 else cutlass.Float8E4M3FN + v, v_dequant_scale, scale_v_channels = _quantize_fp8_v( + v, per_head_channel=qk_sf_vec != 0 and qac.v_block_size == 1 + ) + scale_v = scale_v * v_dequant_scale # Skip softmax. skip_softmax_threshold_scale = self.skip_softmax_threshold_scale if skip_softmax_threshold_scale is not None and skip_softmax_threshold_scale <= 0.0: skip_softmax_threshold_scale = None - cute_dsl.cute_dsl_fmha_fwd( - q.flatten(0, 1).contiguous(), - k.flatten(0, 1).contiguous(), - v.flatten(0, 1).contiguous(), - out.flatten(0, 1), - qo_indptr=qo_indptr, - kv_indptr=kv_indptr, + cute_dsl_fmha_fwd( + q, + k, + v, + out, is_causal=is_causal, sm_scale=self.scale, - lse=lse.flatten(0, 1).contiguous(), - scale_q=kwargs.get("scale_q", 1.0), - scale_k=kwargs.get("scale_k", 1.0), + lse=lse, + scale_q=scale_q, + scale_k=scale_k, scale_v=scale_v, + scale_v_channels=scale_v_channels, scale_o=kwargs.get("scale_o", 1.0), - max_qo_len=seq_len_q, - max_kv_len=seq_len_kv, - is_persistent=False, skip_softmax_threshold_scale_factor=skip_softmax_threshold_scale, + qk_sf_vec=qk_sf_vec, + q_sf=q_sf, + k_sf=k_sf, + qk_cutlass_dtype=qk_cutlass_dtype, ) return out, lse diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py b/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py index acf2f260713a..62c6d0d296f0 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py @@ -855,6 +855,7 @@ def wrap_parallel_attention( enable_sequence_parallel: bool = True, use_ulysses: bool = True, async_ulysses: bool = False, + ulysses_group: Optional[dist.ProcessGroup] = None, ) -> AttentionBackend: """Wrap a compute backend with the configured parallelism strategy. @@ -888,7 +889,17 @@ def wrap_parallel_attention( elif ring_size > 1: attn = RingAttention(attn, process_group=vgm.ring_group) - if ulysses_size > 1 and use_ulysses: + if ulysses_group is not None: + # Explicit group override: wrap if the override group is non-trivial, + # independent of vgm.ulysses_size (the caller's group may be wider or + # narrower than the mapping's). + if use_ulysses and dist.get_world_size(group=ulysses_group) > 1: + attn = UlyssesAttention( + attn, + process_group=ulysses_group, + async_ulysses=async_ulysses, + ) + elif ulysses_size > 1 and use_ulysses: attn = UlyssesAttention( attn, process_group=vgm.ulysses_group, diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py b/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py index e52f52c87247..ce012e16aea3 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py @@ -49,8 +49,8 @@ def get_visual_gen_attention_backend( Better performance but requires fused QKV - "FA4": Flash Attention 4; provides higher speedup on Blackwell GPUs (sm100) Requires flash-attn package with cute interface - - "CUTEDSL": CuTe DSL FMHA kernels; uses packaged cubins when present, - otherwise compiles from CuTe DSL source + - "CUTEDSL": CuTe DSL kernels. create_attention selects dense FMHA or VSA from + AttentionConfig.sparse_attention_config. """ # Lazy imports to avoid circular dependency from .cute_dsl import CuTeDSLAttention @@ -105,8 +105,8 @@ def create_attention( will automatically reallocate if larger batches are encountered. max_seq_len: Initial sequence length for metadata pre-allocation. The backend will automatically reallocate if longer sequences are encountered. - attention_config: Optional AttentionConfig; quant_attention_config is - extracted and forwarded to the TRTLLM backend when present. + attention_config: Optional AttentionConfig used to select the attention algorithm and + forward its quantization or sparsity configuration. attention_metadata_state: Optional model-scoped metadata state from visual-gen config. Required for TRTLLM backend. **kwargs: Additional backend-specific arguments @@ -116,9 +116,7 @@ def create_attention( """ attn_cls = get_visual_gen_attention_backend(backend) - # Extract quant_attention_config from AttentionConfig and pass to backends - # that support it (TRTLLM SAGE recipes, CUTEDSL QK16PV8). AttentionConfig - # validation rejects unsupported (backend, recipe) combinations upstream. + # Forward the validated quantization recipe to TRTLLM or the dense CuTe DSL FMHA backend. if attention_config is not None and attention_config.quant_attention_config is not None: kwargs["quant_attention_config"] = attention_config.quant_attention_config if backend.upper() == "TRTLLM": @@ -133,7 +131,6 @@ def create_attention( attention_config.sparse_attention_config is not None and getattr(attention_config.sparse_attention_config, "algorithm", None) == "vsa" ): - # VSA sparse path: use VSAAttention from .cute_dsl.vsa import VSAAttention attn_cls = VSAAttention diff --git a/tensorrt_llm/_torch/visual_gen/cache/cache_dit_enablers.py b/tensorrt_llm/_torch/visual_gen/cache/cache_dit_enablers.py index a859e50d70a8..4f4c4c267711 100644 --- a/tensorrt_llm/_torch/visual_gen/cache/cache_dit_enablers.py +++ b/tensorrt_llm/_torch/visual_gen/cache/cache_dit_enablers.py @@ -9,6 +9,7 @@ from typing import Any, Callable, List, Optional import cache_dit +import torch import torch.nn as nn from cache_dit import ( BlockAdapter, @@ -41,6 +42,20 @@ class CacheDiTEnableResult: summary_modules: List[nn.Module] +def _has_compiled_blocks(*block_lists: Any) -> bool: + """True when any transformer block is a torch.compile OptimizedModule wrapper. + + torch.compile'd blocks expose a ``(*args, **kwargs)`` forward signature, so + cache_dit's inspect-based forward-pattern checks cannot match them and must + be skipped (the underlying blocks still follow the declared pattern). + """ + return any( + isinstance(block, torch._dynamo.OptimizedModule) + for blocks in block_lists + for block in blocks + ) + + def _resolved_enable_separate_cfg(cfg: CacheDiTConfig, default: bool) -> bool: if cfg.enable_separate_cfg is not None: return cfg.enable_separate_cfg @@ -258,12 +273,18 @@ def enable_cache_dit_for_flux( forward_pattern = [ForwardPattern.Pattern_1, ForwardPattern.Pattern_1] tag = "FLUX.1" + compiled_blocks = _has_compiled_blocks(*block_lists) + if compiled_blocks: + logger.info( + f"Cache-DiT: {tag} blocks are torch.compile'd; skipping forward-pattern checks." + ) adapter = BlockAdapter( transformer=pipeline.transformer, blocks=block_lists, forward_pattern=forward_pattern, params_modifiers=[modifier], - check_forward_pattern=True, + check_forward_pattern=not compiled_blocks, + check_num_outputs=not compiled_blocks, ) logger.info( diff --git a/tensorrt_llm/_torch/visual_gen/cache/teacache.py b/tensorrt_llm/_torch/visual_gen/cache/teacache.py index 45098a9e8f97..fdf9f827edbe 100644 --- a/tensorrt_llm/_torch/visual_gen/cache/teacache.py +++ b/tensorrt_llm/_torch/visual_gen/cache/teacache.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import inspect from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional @@ -71,6 +74,7 @@ class ExtractorConfig: guidance_param_name: Parameter name for guidance if used (default: None) forward_params: List of parameter names (None = auto-introspect from forward signature) return_dict_default: Default value for return_dict parameter (default: True) + return_tuple_when_return_dict_false: Whether return_dict=False uses a one-element tuple output_model_class: Output class name for return type (default: "Transformer2DModelOutput") """ @@ -80,6 +84,7 @@ class ExtractorConfig: guidance_param_name: Optional[str] = None forward_params: Optional[List[str]] = None return_dict_default: bool = True + return_tuple_when_return_dict_false: bool = False output_model_class: str = "Transformer2DModelOutput" @@ -154,10 +159,8 @@ def postprocess(output): if isinstance(output, tuple): return output return Transformer2DModelOutput(sample=output) - # For return_dict=False, unwrap single-element tuple to raw tensor - if isinstance(output, tuple) and len(output) == 1: - return output[0] - # Return raw tensor as-is (TeaCacheHook always passes tensors to postprocess) + if self.config.return_tuple_when_return_dict_false: + return (output,) return output return CacheContext( diff --git a/tensorrt_llm/_torch/visual_gen/cuda_graph_runner.py b/tensorrt_llm/_torch/visual_gen/cuda_graph_runner.py index 7b29b030e3d4..3d79f73dde39 100644 --- a/tensorrt_llm/_torch/visual_gen/cuda_graph_runner.py +++ b/tensorrt_llm/_torch/visual_gen/cuda_graph_runner.py @@ -72,11 +72,19 @@ def __init__( ): self.config = config self.enabled = config.use_cuda_graph + # When False, uncaptured keys run eagerly instead of triggering capture. + self.allow_capture = True self._shared_pool = shared_pool self._extra_key_fns: Dict[str, ExtraKeyFn] = {} self.graphs: Dict[KeyType, torch.cuda.CUDAGraph] = {} self.graph_outputs: Dict[KeyType, Any] = {} # weak refs + # Keep the capture-created output tensors alive. Some pipelines keep + # outputs from multiple graph keys live at once, for example true CFG + # combines positive and negative Qwen-Image-Edit transformer outputs. + # Without strong refs, a later graph capture using the same memory pool + # can reuse the earlier output storage and make those outputs alias. + self._graph_output_refs: Dict[KeyType, Any] = {} self.static_inputs: Dict[KeyType, Tuple[List[Any], Dict[str, Any]]] = {} self.memory_pool = config.cuda_graph_mem_pool @@ -147,6 +155,7 @@ def capture( self.graphs[key] = graph self.static_inputs[key] = (static_args, static_kwargs) + self._graph_output_refs[key] = output self.graph_outputs[key] = make_weak_ref(output) self.memory_pool = graph.pool() @@ -194,6 +203,8 @@ def wrapper(*args, **kwargs): key = self.get_graph_key(*args, **kwargs) if key not in self.graphs: + if not self.allow_capture: + return fn(*args, **kwargs) self.capture(key, fn, args, kwargs) return self.replay(key, args, kwargs) else: @@ -209,5 +220,6 @@ def clear(self): graph.reset() self.graphs.clear() self.graph_outputs.clear() + self._graph_output_refs.clear() self.static_inputs.clear() self.memory_pool = None diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py index fdf84901e4c5..ec83a7d6a30b 100644 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py @@ -13,6 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .fmha import cute_dsl_fmha_fwd +from .fmha import BlackwellFusedMultiHeadAttentionForward, make_thread_cooperative_group +from .fmha_blockscaled import ( + BlackwellFusedMultiHeadBlockScaledAttentionForward, + compact_fp4_data, + create_scale_factor_tensor, +) -__all__ = ["cute_dsl_fmha_fwd"] +__all__ = [ + "BlackwellFusedMultiHeadAttentionForward", + "BlackwellFusedMultiHeadBlockScaledAttentionForward", + "compact_fp4_data", + "create_scale_factor_tensor", + "make_thread_cooperative_group", +] diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 93e56313b689..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bde8e845611c0ff97738d3822da3ab406169596065bc703a8c8f1af58aeb4fbd -size 711184 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index 8f7b825447a4..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d962a68d6d0c89ca5e71d0514b2ea5f1b1da3099c7c89ea4c05cf7c5468f6b0c -size 686552 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 6f2a475d0922..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4bc6c9ff687d1a49bdff39dffcf9e08f74c0114dc63921b458695d1b1add12df -size 702960 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 8c94270cef08..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9d71eff9707d7e4464e6b2847ece251c352c783b60056086841cfc1ad5767489 -size 682416 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index b12c8c09af05..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:117b99d4733a4cf500515a22f5fb2c87b5c89ffdf7736bc302fff48b27b68289 -size 645672 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index be106aa226d8..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:aed8d36370827808578ee1b2a7927a8bb964772ba1446a84d7eb66ae11857c28 -size 625128 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index df218788b1ad..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:909f85670f5cfeae51fb7e7ca2ec6ee648f272c72c0e65545c2f94a084bbd8d7 -size 637440 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 67cd56345e01..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cb1bd11b767de8d4b039a4456f799a0461cc0cfa423e4ce97d6bddf1f289a81d -size 616896 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 05f14144f81e..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4451124313741a07ee5c616c5fbaf3db110a1ab531bfc5955eeb2a126eb33d3e -size 711096 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index 86aef559d4ab..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0bad02325167d25248a381c9e8cfa4479d93e4f7f2d2bbfff0816db8b03e52ea -size 690552 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 2bd77cbe433b..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:38e71c573fad3a61051b4037cd38ca42daaa2cc6f2c1a007af74c7d5e0107b71 -size 702872 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index a15c97e7c4ff..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:40cb6caa255f53e90c8c84fb5b5cf13b4776863d76f1d0ead0553fb3b574b169 -size 686424 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 9b939a3c3917..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6d48995a1b50324fb7f103a543bd8aa2d48ab8617a972e7bc594a834f78e7888 -size 645576 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index 6273d4c74ad7..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2fff35882b5e0bfbebf2b313a67c7a80398a68df411a1fb30542f11a2e9212f1 -size 629136 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index ba3f0e067efe..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1436b06493487e07c260e5ce283dcd85bfd9551db272d96d2e395788ed34ba5b -size 637352 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index daf2c2df15b8..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:460134c26759d391ac0d4df94627e63794898425f887a210871259fc030d2157 -size 620904 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 10e981459b3c..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0a6ea5c21b35d5562eec533b601348f527da2ed159c8d6f2a413f02d8b3dfce4 -size 657936 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index e153954ec5ee..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ed589e4c6f369afb05903be7bee828ab5f8293736cca734390d6eab9ac609f56 -size 637400 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index a8b2717c21dc..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:99801a8fb3c547c9f5778764b14049229e07a7395a2f26e098112ca3968884eb -size 653808 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 0691e9326f5a..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:834109925ad81a8b50d44a436c34b9b9ff2f37bf87dc1c3420da10082f951f71 -size 629168 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index a33382575553..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:962849852e70fd20d8f086b9257173ddda4f6ca654ed4fa165768a5398e50e75 -size 596520 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index 6a76fed84298..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a8a040d0f9333c99a78cdf2baa42f2dc71cf2d8252ac3d1b7005f3afd7b4b536 -size 575976 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 46d5cb25f686..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3c241a35bb7ab27b9b4fe26c03f1a8ac4ba1eeb28cfc68fcc0f0d5133a74f393 -size 588288 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 073aa5c4860d..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:285541f2d3d081f8726d79c7e6f484af55a06e52e7580fb58a360f9aad5e846b -size 567744 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 02af20762774..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b7284ff86f4d04fedf14e87947411e1765b55851ee9d9f12450f0314644abd8e -size 657848 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index 4eb755d3c573..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7f26d9fb521fea7260587558e4a5ec016c2c21b53f0fca580e65c46f7599c898 -size 641400 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index a82de70cf9be..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d951a82391c4df1933bf1ff13bf7c455d90c95b19ebf3360c34a3151763ad9cc -size 653720 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 07ab962faa79..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ce18821322bdb617a5c26e52465ff1c02d6bd03f6f24ce7604a538a2862e107a -size 633176 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 5396cdeac4a3..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5af2f95b9ff7328f1dce2863345ddbf72a39ec4cc0c3ce7601ad904d1c279bb3 -size 592328 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index ec033517f17f..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:afe75060cd7f3dd1a37704e2a4fe70aec59bcf60e83f6cc77c510fae8adb1209 -size 575888 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 490ac3c2eec1..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:807fc151395aabe1f4fb2bd0451d6d4f8def14e591cff121dc5426c2ee24b841 -size 588200 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index d708b622be2a..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e0faf35561762e27fbf861fe3385298f08da3aa9cc9564b9d3cd99c7fb2f74e5 -size 567656 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 334d4de2da9a..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:20f59a88225f55116a6269dde1eb677bd527cb91eb1200058765c8c64356b9fb -size 716248 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index ffb16280b537..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f7666214fc5357f11f4db5cdae11c11f2c3a42679b30c1534535e12916aed578 -size 694232 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 7a3a7cf23675..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cd18be0827da75961d58cb09d0c63fe0826ad9d1a83e16a44b74db43fb7b183b -size 705936 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 86656cfff50e..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d3a3da59ea39eb5d41e345a04ba0ab573fb6ef7d950091d4757ac27af2c736e5 -size 683984 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 4ce0c3cc1752..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ecd8a2048373f40ddf71a9706f7a16aef9fb0e5b6487937be695f6e7d0744161 -size 651128 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index 4b6997fefec4..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3407c3c3f177c8ab0b02c69c7cae9968710041d06acef9a10fc5ffa534ec274f -size 630200 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 2c01e5f00009..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f64ccf41ae8680369363660c1c350386a8b9784141853832fb49d2767e283a9e -size 639728 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 986c6a5db0d5..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4eca236d41155a0e2c5b0f3289c5399b60589ffb5b679abff456202566e2bdf0 -size 618192 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 14609fab4bbc..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2d5ad69ef4a76110e48c4f1b6642e3e8b321dc1d794e83957b6228a8c68edfd4 -size 714872 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index 6c4ef0b8a4c7..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a1896d588ebbbb6ffde8331432dafccbf996507504b6b86a0ba17bf539f2e8f9 -size 697240 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 40433af18781..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7ce41d196fa54fb89e5848891e1e402d6511d0e5635098e98f00b0312ce3ad77 -size 704296 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 14d97d8463c3..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2f45b423c9980f32c116718dca45f1b05e65ad9abd1e0a70e37a07c9106c6827 -size 686888 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index faabf383e21f..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:150cd0b8fa1834d00aa2de8ead15916bc08c4e4f5ba002683ceccca5dc15a793 -size 650496 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index 2500b613f8e4..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8c0eab2d74850c29abd02aa927cc53da3ac730ebed1f8d62e5836c68c3c86a42 -size 635008 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 8c867ef9b986..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d044e658d6d3fda52a038a0d0d8b9cce502f7e383bb96deec344d10e217a6e98 -size 640056 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 5449e7917a22..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:20bc7f069ea456ff8274d71dccaceeb979ba877e19a29d956753ebfa8e6000ad -size 624504 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index e22d76d563cc..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:77fee6189e75f9b0233f7ccce9eec7d2f70b277f2f62d599c01f057e1c179235 -size 665400 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index cc6eab46060c..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:182a606d2ca126fc83c2f914a6e5547d84a9aa668e3526710ca3e7396aac2f8e -size 642776 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 9a06ad3879df..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:96052f92a4a4e9b8edc152a2e4e60d0b3c9ffc3e8cad0f72716f4be1072cea5e -size 656112 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 04c07f32cdd9..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2c4ca62124eb16d29903dde0f9cf9a46b8a72d3480fee3447736b6c4f2d71895 -size 632560 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index c27e7a6a19dc..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c9b8919f036acd199b4aed790563b63de6d1cfa3ef37bfeec00751a25ee5c29d -size 600408 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index 5f9f6cd54c9d..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eff71a1a8838b98ec684ada9e693db4f39a69527d8f072bb52ad3b136e2ad966 -size 580200 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 62349117d032..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2e7d9c9e4ca723169a115029633811b97d6a57880274f668678218466bef9ee0 -size 591488 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index 4b09aab76f59..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:30f0681109e2520f4135d1bb4910d31c11b6d0aae64e09efc6705ba461e2eede -size 568848 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 17b50b8f2a9b..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6a761c98700c565c98a0e7c7a67a569cf165cdf1fe1262c541ec066f60ffa7c1 -size 663704 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index a4d1b3ecae30..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f21a9a2fc02f9ada3d126ee7c054f130fad0b16b62daadd63c881dc63991cf6b -size 644472 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index 883e738091b4..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b9edebdc6f4c23e407f057fba17144122f625db87a29bdd61d14951b8d05aadf -size 655176 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index a22d88458722..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9becebe7efb3c0342b41668b6d82888e5a100bccdf906a25c10402ee76b56115 -size 635160 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so deleted file mode 100644 index 836913616f49..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8239bc1f03bef805f46a356a9dcffa449dfa1f112ce8d39297f7cc272da60e48 -size 599536 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so deleted file mode 100644 index 226972ea4776..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:deb5f65d94e50b7466c027004c9fbfe0c7a5b9b67eec6c139496b70a7a0a85eb -size 581984 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so deleted file mode 100644 index bca39e1a5e2e..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dca9eeb79fe6a289db4cee5e9c2e8700dd9e138c149da1ec1299ea471efc3c8e -size 589096 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so deleted file mode 100644 index da1001774b1e..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c24bd23172aa6d42fc3df1f7221e9ca3ce9c2b4e608386879f1734d58956fa45 -size 571544 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py index 3de28390c569..bca82e086559 100644 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py @@ -1,494 +1,3210 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import functools +# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ruff: noqa: I001, E501, F841 + import math -import platform -from pathlib import Path -from typing import Optional, Union - -import torch - -_cute_runtime_import_error = None -try: - import cutlass - from cuda.bindings import driver as cuda_driver - from cutlass import cute - from cutlass.cute import typing as cute_typing - from cutlass.cute.runtime import from_dlpack -except (ImportError, OSError) as e: - cutlass = None - cute = None - from_dlpack = None - cuda_driver = None - _cute_runtime_import_error = e - - -CUBINS_ROOT = Path(__file__).resolve().parent / "cubins" -SUPPORTED_GPU_ARCHS = ("sm_100a", "sm_103a") - - -def _dtype_to_str(dtype: torch.dtype) -> str: - float8_e4m3fn = getattr(torch, "float8_e4m3fn", None) - dtype_map = { - torch.float16: "fp16", - torch.bfloat16: "bf16", - torch.float32: "fp32", - } - if float8_e4m3fn is not None: - dtype_map[float8_e4m3fn] = "e4m3" - try: - return dtype_map[dtype] - except KeyError as exc: - raise ValueError(f"Unsupported CuTe DSL FMHA dtype: {dtype}") from exc - - -def _gpu_cpu_arch() -> str: - arch = platform.machine().lower() - if arch in ("amd64", "x64"): - return "x86_64" - if arch in ("arm64",): - return "aarch64" - return arch - - -def _get_gpu_arch(device: torch.device | str | None = None) -> str: - capability = torch.cuda.get_device_capability(device) - gpu_arch = f"sm_{capability[0]}{capability[1]}a" - if gpu_arch not in SUPPORTED_GPU_ARCHS: - supported = ", ".join(SUPPORTED_GPU_ARCHS) - raise ValueError( - f"Unsupported GPU architecture {gpu_arch}. Supported architectures: {supported}." - ) - return gpu_arch - - -def _get_cubins_dir(gpu_arch: str) -> Path: - return CUBINS_ROOT / _gpu_cpu_arch() / gpu_arch - - -def _get_variant_name( - qk_dtype: torch.dtype, - pv_dtype: torch.dtype, - out_dtype: torch.dtype, - head_dim: int, - is_causal: bool, - is_persistent: bool = True, - varlen: bool = False, - with_lse: bool = False, - enable_skip_softmax: bool = False, - enable_tvm_ffi: bool = False, -) -> str: - qk_str = _dtype_to_str(qk_dtype) - pv_str = _dtype_to_str(pv_dtype) - out_str = _dtype_to_str(out_dtype) - if qk_dtype != pv_dtype: - dtype_str = f"{qk_str}_{pv_str}_{out_str}" - elif qk_dtype != out_dtype: - dtype_str = f"{qk_str}_{out_str}" - else: - dtype_str = qk_str - - causal_str = "causal" if is_causal else "nocausal" - persist_str = "persistent" if is_persistent else "nonpersistent" - varlen_str = "_varlen" if varlen else "" - lse_str = "_lse" if with_lse else "" - skip_str = "_skipsm" if enable_skip_softmax else "" - ffi_str = "_tvmffi" if enable_tvm_ffi else "" - return ( - f"cute_dsl_fmha_{dtype_str}_h{head_dim}_{causal_str}_{persist_str}" - f"{varlen_str}{lse_str}{skip_str}{ffi_str}" - ) - - -def _get_candidate_paths(variant_name: str, gpu_arch: str | None) -> list[Path]: - names = [f"{variant_name}.so", f"{variant_name}.o"] - if gpu_arch is not None: - return [_get_cubins_dir(gpu_arch) / name for name in names] - - host_dir = CUBINS_ROOT / _gpu_cpu_arch() - return [ - host_dir / supported_gpu_arch / name - for supported_gpu_arch in SUPPORTED_GPU_ARCHS - for name in names - ] - - -def _resolve_cubin_path( - variant_name: str, - gpu_arch: str | None = None, -) -> Path: - tried = [] - for candidate in _get_candidate_paths(variant_name, gpu_arch): - tried.append(candidate) - if candidate.exists(): - return candidate.resolve() - - searched = "\n".join(f" - {path}" for path in tried) - default_dir = ( - _get_cubins_dir(gpu_arch) - if gpu_arch is not None - else CUBINS_ROOT / _gpu_cpu_arch() / "" - ) - raise FileNotFoundError( - f"Could not find packaged CuTe DSL FMHA cubins for '{variant_name}'.\n" - f"Expected a .so or .o under {default_dir}.\nSearched:\n{searched}" - ) - - -def _check_cute_runtime_available() -> None: - if cute is not None: - return - raise ImportError( - f"CuTe DSL runtime is not available. Import error: {_cute_runtime_import_error}" - ) from _cute_runtime_import_error - - -def _load_cubin_from_path( - path: str | Path, - variant_name: str | None = None, - enable_tvm_ffi: bool = True, -): - _check_cute_runtime_available() - - cubin_path = Path(path).expanduser().resolve() - if variant_name is None: - variant_name = cubin_path.stem - - module = cute.runtime.load_module(str(cubin_path), enable_tvm_ffi=enable_tvm_ffi) - try: - return getattr(module, variant_name) - except AttributeError as exc: - raise AttributeError( - f"Loaded {cubin_path}, but symbol '{variant_name}' was not found. " - "The symbol name must match the .so filename stem / function prefix." - ) from exc - - -@functools.lru_cache(maxsize=None) -def _load_cute_dsl_fmha_cubin_cached( - variant_name: str, - gpu_arch: str | None, - enable_tvm_ffi: bool, -): - path = _resolve_cubin_path(variant_name, gpu_arch) - return _load_cubin_from_path(path, variant_name, enable_tvm_ffi) - - -def get_cute_dsl_fmha_cubin( - qk_dtype: torch.dtype, - pv_dtype: torch.dtype, - out_dtype: torch.dtype, - head_dim: int, - is_causal: bool, - is_persistent: bool = True, - enable_tvm_ffi: bool = True, - varlen: bool = False, - with_lse: bool = False, - enable_skip_softmax: bool = False, - gpu_arch: str | None = None, -): - variant_name = _get_variant_name( - qk_dtype, - pv_dtype, - out_dtype, - head_dim, - is_causal, - is_persistent, - varlen, - with_lse, - enable_skip_softmax, - enable_tvm_ffi, - ) - return _load_cute_dsl_fmha_cubin_cached( - variant_name, - gpu_arch, - enable_tvm_ffi, - ) - - -def _check_inputs( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - o: torch.Tensor, -) -> bool: - if q.dim() not in (3, 4) or k.dim() != q.dim() or v.dim() != q.dim(): - raise ValueError("Expected 3D or 4D Q/K/V tensors with matching ranks.") - if q.dtype != k.dtype: - raise ValueError(f"Q/K dtype mismatch: {q.dtype} vs {k.dtype}") - if q.shape[-1] != k.shape[-1]: - raise ValueError(f"Q/K head dim mismatch: {q.shape[-1]} vs {k.shape[-1]}") - if k.shape[:-1] != v.shape[:-1]: - raise ValueError(f"K/V shape mismatch: {k.shape[:-1]} vs {v.shape[:-1]}") - expected_o_shape = (*q.shape[:-1], v.shape[-1]) - if tuple(o.shape) != expected_o_shape: - raise ValueError(f"Output shape mismatch: {tuple(o.shape)} vs {expected_o_shape}") - return q.dim() == 3 - - -def _to_cint_contiguous(tensor: torch.Tensor) -> torch.Tensor: - return tensor.to(torch.int32).contiguous() - - -def _to_cute_tensor(tensor: torch.Tensor, leading_dim: int): - float8_e4m3fn = getattr(torch, "float8_e4m3fn", None) - if tensor.dtype == float8_e4m3fn: - cute_tensor = from_dlpack(tensor.view(torch.int8), assumed_align=16) - cute_tensor = cute_tensor.mark_layout_dynamic(leading_dim=leading_dim) - cute_tensor.element_type = cutlass.Float8E4M3FN - return cute_tensor - return from_dlpack(tensor, assumed_align=16).mark_layout_dynamic(leading_dim=leading_dim) - - -def _get_runtime_problem( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - o: torch.Tensor, - lse: Optional[torch.Tensor], - qo_indptr: Optional[torch.Tensor], - kv_indptr: Optional[torch.Tensor], - max_qo_len: Optional[int], - max_kv_len: Optional[int], - varlen: bool, -) -> tuple[ - int, - int, - int, - int, - int, - int, - int, - int, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, -]: - if varlen: - if qo_indptr is None or kv_indptr is None or max_qo_len is None or max_kv_len is None: - raise ValueError("Varlen FMHA requires indptr tensors and max sequence lengths.") - if qo_indptr.dim() != 1 or kv_indptr.dim() != 1: - raise ValueError("Varlen FMHA indptr tensors must be 1D.") - if qo_indptr.numel() != kv_indptr.numel() or qo_indptr.numel() < 2: - raise ValueError("Varlen FMHA indptr tensors must have matching non-empty sizes.") - total_q, num_heads_q, head_dim = q.shape - _, num_heads_kv, _ = k.shape - batch_size = qo_indptr.numel() - 1 - max_s_q = max_qo_len - max_s_k = max_kv_len - q_4d = q.unsqueeze(0) - k_4d = k.unsqueeze(0) - v_4d = v.unsqueeze(0) - o_4d = o.unsqueeze(0) - lse_3d = lse.unsqueeze(0) if lse is not None else None - else: - batch_size, max_s_q, num_heads_q, head_dim = q.shape - _, max_s_k, num_heads_kv, _ = k.shape - total_q = batch_size * max_s_q - q_4d = q - k_4d = k - v_4d = v - o_4d = o - lse_3d = lse - value_head_dim = v.shape[-1] - return ( - batch_size, - max_s_q, - total_q, - max_s_k, - num_heads_q, - num_heads_kv, - head_dim, - value_head_dim, - q_4d, - k_4d, - v_4d, - o_4d, - lse_3d, - ) - - -@torch.compiler.disable -def cute_dsl_fmha_fwd( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - o: torch.Tensor, - qo_indptr: Optional[torch.Tensor] = None, - kv_indptr: Optional[torch.Tensor] = None, - is_causal: bool = False, - sm_scale: Optional[float] = None, - window_left: int = -1, - window_right: int = -1, - lse: Optional[torch.Tensor] = None, - scale_q: Union[float, torch.Tensor] = 1.0, - scale_k: Union[float, torch.Tensor] = 1.0, - scale_v: Union[float, torch.Tensor] = 1.0, - scale_o: Union[float, torch.Tensor] = 1.0, - enable_tvm_ffi: bool = True, - is_persistent: bool = False, - max_qo_len: Optional[int] = None, - max_kv_len: Optional[int] = None, - kernel_fn=None, - skip_softmax_threshold_scale_factor: Optional[float] = None, -) -> None: - varlen = _check_inputs(q, k, v, o) - - scale_q = float(scale_q.item()) if isinstance(scale_q, torch.Tensor) else scale_q - scale_k = float(scale_k.item()) if isinstance(scale_k, torch.Tensor) else scale_k - scale_v = float(scale_v.item()) if isinstance(scale_v, torch.Tensor) else scale_v - scale_o = float(scale_o.item()) if isinstance(scale_o, torch.Tensor) else scale_o - - ( - batch_size, - max_s_q, - total_q, - max_s_k, - num_heads_q, - num_heads_kv, - head_dim, - value_head_dim, - q_4d, - k_4d, - v_4d, - o_4d, - lse_3d, - ) = _get_runtime_problem(q, k, v, o, lse, qo_indptr, kv_indptr, max_qo_len, max_kv_len, varlen) - use_skip_softmax = ( - skip_softmax_threshold_scale_factor is not None and skip_softmax_threshold_scale_factor > 0 - ) - problem_size = ( - batch_size, - max_s_q, - total_q, - max_s_k, - num_heads_q, - num_heads_kv, - head_dim, - value_head_dim, - ) - - if kernel_fn is None: - kernel_fn = get_cute_dsl_fmha_cubin( - q.dtype, - v.dtype, - o.dtype, - head_dim, - is_causal, - is_persistent=is_persistent, - varlen=varlen, - enable_tvm_ffi=enable_tvm_ffi, - with_lse=lse is not None, - enable_skip_softmax=use_skip_softmax, - gpu_arch=_get_gpu_arch(q.device), - ) - - if sm_scale is None: - sm_scale = 1.0 / math.sqrt(head_dim) - scale_softmax = scale_q * scale_k * sm_scale - scale_softmax_log2 = scale_softmax * math.log2(math.exp(1.0)) - scale_output = scale_v / scale_o - - skip_threshold_log2 = None - if use_skip_softmax: - threshold = skip_softmax_threshold_scale_factor / max_s_k - skip_threshold_log2 = cute_typing.Float32(math.log2(threshold)) - - ws_left = None if window_left == -1 else cute_typing.Int32(window_left) - ws_right = None if window_right == -1 else cute_typing.Int32(window_right) - if is_causal and ws_right is None: - ws_right = cute_typing.Int32(0) - - # CUBIN path - num_head_groups = num_heads_q // num_heads_kv - q_5d = q_4d.unflatten(2, (num_heads_kv, num_head_groups)) - k_5d = k_4d.unsqueeze(3) - v_5d = v_4d.unsqueeze(3) - o_5d = o_4d.unflatten(2, (num_heads_kv, num_head_groups)) - lse_4d = lse_3d.unflatten(2, (num_heads_kv, num_head_groups)) if lse is not None else None - - if enable_tvm_ffi: - qo_indptr_i32 = _to_cint_contiguous(qo_indptr) if varlen else None - kv_indptr_i32 = _to_cint_contiguous(kv_indptr) if varlen else None - kernel_fn( - q_5d, - k_5d, - v_5d, - o_5d, - problem_size, - qo_indptr_i32, - kv_indptr_i32, - lse_4d, - None, # sink - cute_typing.Float32(scale_softmax_log2), - cute_typing.Float32(scale_softmax), - cute_typing.Float32(scale_output), - skip_threshold_log2, - ws_left, - ws_right, - None, - None, - False, # reserved +from typing import Callable, Type, Tuple, Union, Optional +from functools import partial + +import cuda.bindings.driver as cuda + +import cutlass +import cutlass.cute as cute +from cutlass.cute.nvgpu import tcgen05 +from cutlass.cute.nvgpu.common import OperandMajorMode +import cutlass.utils as utils +import cutlass.pipeline as pipeline +from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait +import cutlass.utils.blackwell_helpers as sm100_utils +from cutlass.cute.typing import Int8, Int32, Int64, Float32 +from cutlass.base_dsl.arch import Arch +from cutlass.cutlass_dsl import BaseDSL + +from ...helpers import fmha_helpers as fmha_utils + +""" +A fused multi-head attention (FMHA) example for the NVIDIA Blackwell SM100 architecture using CUTE DSL + +This example demonstrates an implementation of fused multi-head attention using a TMA + Blackwell SM100 +TensorCore warp-specialized persistent kernel. The implementation integrates the Q*K^T matrix multiplication, +softmax normalization, and softmax(Q*K^T)*V into a single kernel, avoiding intermediate data movement between +global memory and shared memory, thus improving computational efficiency. + +The kernel implements key optimizations including: +- Warp specialization for different computation phases (load, MMA, softmax, correction, epilogue) +- Pipeline stages between different warps for overlapping computation and memory access +- Support for different precision data types +- Optional causal masking for autoregressive models + +To run this example: + +.. code-block:: bash + + python examples/cute/blackwell/kernel/attention/fmha/fmha.py \ + --qk_acc_dtype Float32 --pv_acc_dtype Float32 \ + --mma_tiler_mn 128,128 \ + --q_shape 4,1024,8,64 --k_shape 4,1024,8,64 \ + --is_persistent + +The above example runs FMHA with batch size 4, sequence length 1024, 8 attention heads, and head +dimension 64. The Blackwell tcgen05 MMA tile shape is (128, 128), and the kernel uses fp16 for input/output +with fp32 for accumulation. + +To collect performance with NCU profiler: + +.. code-block:: bash + + ncu python examples/cute/blackwell/kernel/attention/fmha/fmha.py \ + --qk_acc_dtype Float32 --pv_acc_dtype Float32 \ + --mma_tiler_mn 128,128 \ + --q_shape 4,1024,8,64 --k_shape 4,1024,8,64 \ + --is_persistent --warmup_iterations 10 \ + --iterations 10 --skip_ref_check + +Constraints for this example: +* Supported head dimensions: 32, 64, and 128 +* Number of heads in Q must be divisible by number of heads in K +* mma_tiler_mn must be 128,128 +* Batch size must be the same for Q, K, and V tensors +* For causal masking, use --is_causal (note: specify without =True/False) +* For persistent scheduling, use --is_persistent (note: specify without =True/False) + +For details on the skip softmax algorithm, please refer to the paper: https://arxiv.org/abs/2512.12087. +""" + + +def make_thread_cooperative_group(size: int): + return pipeline.CooperativeGroup(pipeline.Agent.Thread, size) + + +class BlackwellFusedMultiHeadAttentionForward: + arch_str: str = "sm_100" + arch_name: str = "Blackwell SM100" + + def __init__( + self, + qk_acc_dtype: Type[cutlass.Numeric], + pv_acc_dtype: Type[cutlass.Numeric], + mma_tiler: Tuple[int, int], + head_dim: Union[int, Tuple[int, int]], + is_persistent: bool, + mask_type: fmha_utils.MaskEnum, + enable_ex2_emulation: bool, + enable_skip_correction: bool, + use_tma_store: bool = True, + ): + """Initializes the configuration for a Blackwell Fused Multi-Head Attention (FMHA) kernel. + + This configuration includes several key aspects: + + 1. Data Type Settings: + - qk_acc_dtype: Data type for Q*K^T matrix multiplication accumulator + - pv_acc_dtype: Data type for P*V matrix multiplication accumulator + + 2. MMA Instruction Settings: + - mma_tiler: The shape of the MMA instruction unit: (M, N) for BMM1 and (M, K) for BMM2 + - head_dim: The head dimension, it can be a single integer or a tuple of two integers (D, Dv). + If it is a tuple, Dv is the head dimension of the value & output tensors. + It also determines the K dimension of the BMM1's MMA instruction unit + & N dimension of the BMM2's MMA instruction unit. + - qk_mma_tiler: MMA shape for Q*K^T computation + - pv_mma_tiler: MMA shape for P*V computation + + 3. Kernel Execution Mode: + - is_persistent: Boolean indicating whether to use persistent kernel mode + - mask_type: Specifies the type of mask to use (no mask, residual mask, or causal mask) + - window_size_left/right: Sliding window size for attention masking + - enable_ex2_emulation: Whether to enable exp2 emulation + - enable_skip_correction: Whether to skip the correction when rowmax is not updated larger than a threshold + + :param qk_acc_dtype: Data type for Q*K^T matrix multiplication accumulator + :type qk_acc_dtype: Type[cutlass.Numeric] + :param pv_acc_dtype: Data type for P*V matrix multiplication accumulator + :type pv_acc_dtype: Type[cutlass.Numeric] + :param mma_tiler: The (M, N) shape of the MMA instruction + :type mma_tiler: Tuple[int, int] + :param head_dim: The head dimension, it can be a single integer or a tuple of two integers (D, Dv). + :type head_dim: Union[int, Tuple[int, int]] + :param is_persistent: Whether to use persistent kernel mode + :type is_persistent: bool + :param mask_type: Type of mask to use + :type mask_type: fmha_utils.MaskEnum + :param window_size_left: Left-side sliding window size for attention masking + :type window_size_left: int + :param window_size_right: Right-side sliding window size for attention masking + :type window_size_right: int + """ + + self.qk_acc_dtype = qk_acc_dtype + self.pv_acc_dtype = pv_acc_dtype + if isinstance(head_dim, tuple): + self.head_dim = head_dim[0] + self.head_dim_v = head_dim[1] + assert self.head_dim == 192 and self.head_dim_v == 128, ( + f"When Headdim is a tuple, it's for MLA. Must be (192, 128), but got {head_dim}" + ) + else: + self.head_dim = head_dim + self.head_dim_v = head_dim + self.cta_tiler = ( + 2 * mma_tiler[0], # 2 O tile per CTA + mma_tiler[1], + self.head_dim_v, + ) + self.qk_mma_tiler = ( + *mma_tiler, + self.head_dim, + ) + self.pv_mma_tiler = ( + mma_tiler[0], + self.head_dim_v, + mma_tiler[1], + ) + self.cluster_shape_mn = (1, 1) + self.is_persistent = is_persistent + self.mask_type = mask_type + self.enable_skip_correction = enable_skip_correction + self.enable_ex2_emulation = enable_ex2_emulation + self.use_tma_store = use_tma_store + + self.softmax0_warp_ids = (0, 1, 2, 3) + self.softmax1_warp_ids = (4, 5, 6, 7) + self.correction_warp_ids = (8, 9, 10, 11) + self.mma_warp_id = 12 + self.load_warp_id = 13 + self.epilogue_warp_id = 14 + self.empty_warp_id = 15 + self.num_tmem_alloc_cols = cute.arch.get_max_tmem_alloc_cols(self.arch_str) + + self.threads_per_warp = 32 + self.threads_per_cta = self.threads_per_warp * len( + ( + *self.softmax0_warp_ids, + *self.softmax1_warp_ids, + *self.correction_warp_ids, + self.mma_warp_id, + self.load_warp_id, + self.epilogue_warp_id, + self.empty_warp_id, + ) + ) + self.tmem_alloc_barrier = pipeline.NamedBarrier( + barrier_id=2, + num_threads=self.threads_per_warp + * sum( + ( + len((self.mma_warp_id,)), + len(self.softmax0_warp_ids), + len(self.softmax1_warp_ids), + len(self.correction_warp_ids), + ) + ), + ) + self.sequence_s0_s1_barrier = pipeline.NamedBarrier( + barrier_id=3, + num_threads=self.threads_per_warp + * len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)), + ) + self.sequence_s1_s0_barrier = pipeline.NamedBarrier( + barrier_id=4, + num_threads=self.threads_per_warp + * len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)), + ) + self.s0_warpgroup_barrier = pipeline.NamedBarrier( + barrier_id=5, + num_threads=self.threads_per_warp * len(self.softmax0_warp_ids), + ) + self.s1_warpgroup_barrier = pipeline.NamedBarrier( + barrier_id=6, + num_threads=self.threads_per_warp * len(self.softmax1_warp_ids), + ) + self.tmem_dealloc_barrier = pipeline.NamedBarrier( + barrier_id=7, + num_threads=self.threads_per_warp * len(self.correction_warp_ids), + ) + + self.tmem_s0_offset = 0 + self.tmem_s1_offset = 128 + self.tmem_o0_offset = 256 + self.tmem_o1_offset = 384 + # inplaced with s1 + self.tmem_p0_offset = 160 + # inplaced with s0 + self.tmem_p1_offset = 32 + # vec buffer for row_max & row_sum + # inplaced with s0 + self.tmem_vec0_offset = 0 + # inplaced with s1 + self.tmem_vec1_offset = 128 + # skip mma pv flag offset regarding to the vec buffer + # inplaced with s1 + self.tmem_skip_softmax0_offset = 136 + # inplaced with s0 + self.tmem_skip_softmax1_offset = 8 + + self.num_regs_softmax = 192 + self.num_regs_correction = 96 + self.num_regs_other = 32 + self.buffer_align_bytes = 1024 + self.arch = BaseDSL._get_dsl().get_arch_enum() + + if self.arch >= Arch.sm_103: + assert self.enable_ex2_emulation == False, ( # noqa + f"Don't enable exp2 emulation for {self.arch}, it doesn't help performance" + ) + + num_warps_per_warpgroup = 4 + self.softmax_warpgroup_count = ( + len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)) // num_warps_per_warpgroup + ) + + def _make_qk_tiled_mma(self, cta_group): + """Build the QK tiled MMA. Override in subclasses to target a different arch.""" + return sm100_utils.make_trivial_tiled_mma( + self.q_dtype, + self.q_dtype, + self.q_major_mode, + self.k_major_mode, + self.qk_acc_dtype, + cta_group, + self.qk_mma_tiler[:2], + ) + + def _make_pv_tiled_mma(self, cta_group, p_major_mode, p_source): + """Build the PV tiled MMA. Override in subclasses to target a different arch.""" + return sm100_utils.make_trivial_tiled_mma( + self.v_dtype, + self.v_dtype, + p_major_mode, + self.v_major_mode, + self.pv_acc_dtype, + cta_group, + self.pv_mma_tiler[:2], + p_source, + ) + + def _setup_attributes(self, enable_skip_softmax: bool) -> None: + """Set up configurations and parameters for the FMHA kernel operation. + + This method initializes and configures various attributes required for the + execution of the fused multi-head attention kernel, mainly about the pipeline stages: + + - Sets up staging parameters for Q, K, V inputs and accumulator data + - Configures pipeline stages for softmax, correction, and epilogue operations + """ + + self.q_stage = 2 + k_stage = 4 if self.q_dtype.width == 8 else 3 + v_stage = 4 if self.v_dtype.width == 8 else 3 + self.kv_stage = min(k_stage, v_stage) + # For D192, the smem usage of Q & K is larger. So, we need to reduce the stage count. + if self.head_dim == 192 and self.q_dtype.width == 16: + self.kv_stage = 2 + self.p_mma_stage = 1 + self.acc_stage = 1 + self.softmax_corr_stage = 1 + self.mma_corr_stage = 2 + self.mma_softmax_stage = 1 + self.epi_stage = 2 + + # Tunable parameters + if not self.enable_skip_correction: + self.rescale_threshold = 0.0 + elif enable_skip_softmax: + self.rescale_threshold = 1.0 + else: + self.rescale_threshold = 8.0 + # FP8 P pre-scale: offset added to exp2 exponent so that P*2^offset fills + # more of E4M3's [0, 448] range, improving quantization precision. + # Derived from rescale_threshold to guarantee P*2^offset <= 448. + self.p_fp8_prescale_log2 = max(0.0, math.floor(math.log2(448) - self.rescale_threshold)) + # ln(2) * offset correction for LSE when pre-scale is active + self.p_fp8_prescale_lse_correction = self.p_fp8_prescale_log2 * math.log(2) + # For most cases, seq barrier is needed to help keep the pipeline stable + # But sometimes, compiler will schedule the barrier at an unexpected place + # if it hurts perf a lot, try to quickly fix it by disabling seq barrier + self.enable_sequence_barrier = False + # Optional double buffering for correction rescale. + self.enable_correction_double_buffer = False + + @cute.jit + def __call__( + self, + q_tensor: cute.Tensor, + k_tensor: cute.Tensor, + v_tensor: cute.Tensor, + o_tensor: cute.Tensor, + problem_size: Tuple[Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32], + cum_seqlen_q: Optional[cute.Tensor], + cum_seqlen_k: Optional[cute.Tensor], + lse_tensor: Optional[cute.Tensor], + sink_tensor: Optional[cute.Tensor], + scale_softmax_log2: Float32, + scale_softmax: Float32, + scale_output: Float32, + skip_softmax_threshold_log2: Optional[Float32], + window_size_left: Optional[Int32], + window_size_right: Optional[Int32], + skip_softmax_count: Optional[cute.Tensor], + total_softmax_count: Optional[cute.Tensor], + stream: cuda.CUstream, + use_pdl: bool, + ): + """Execute the Fused Multi-Head Attention operation on the provided tensors. + + This method prepares the input tensors for processing, validates their shapes and types, + configures the computation parameters, and launches the CUDA kernel. + + The method handles: + 1. Tensor layout transformations for specific memory access patterns + 2. Validation of tensor shapes and data types + 3. Initialization of hardware-specific parameters and memory layouts + 4. Configuration of TMA (Tensor Memory Access) operations + 5. Grid and work scheduling computation + 6. Kernel launch with appropriate parameters + + :param q_tensor: The query tensor with shape (b, s_q, h_k, h_r, d) + :type q_tensor: cute.Tensor + :param k_tensor: The key tensor with shape (b, s_k, h_k, 1, d) + :type k_tensor: cute.Tensor + :param v_tensor: The value tensor with shape (b, s_v, h_k, 1, dv) + :type v_tensor: cute.Tensor + :param o_tensor: The output tensor with shape (b, s_q, h_k, h_r, dv) + :type o_tensor: cute.Tensor + :param problem_size: The problem size with shape [b, s_q_max, s_lse_max, s_k_max, h_q, h_k, d, dv]. If cum_seqlen_q or cum_seqlen_k is not None, s_q_max and s_k_max are the max of the per-batch sequence lengths respectively. + :type problem_size: Tuple[Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32] + :param cum_seqlen_q: The cumulative sequence length tensor for query + :type cum_seqlen_q: Optional[cute.Tensor] + :param cum_seqlen_k: The cumulative sequence length tensor for key + :type cum_seqlen_k: Optional[cute.Tensor] + :param scale_softmax_log2: The log2 scale factor for softmax + :type scale_softmax_log2: Float32 + :param scale_softmax: The scale factor for softmax + :type scale_softmax: Float32 + :param scale_output: The scale factor for the output + :type scale_output: Float32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + :param stream: The CUDA stream to execute the kernel on + :type stream: cuda.CUstream + :raises TypeError: If tensor data types don't match or aren't supported + :raises RuntimeError: If tensor layouts aren't in supported formats + """ + b, s_q_max, s_lse_max, s_k_max, h_q, h_k, d, dv = problem_size + h_r = h_q // h_k + # setup static attributes before smem/grid/tma computation + self.q_dtype = q_tensor.element_type + self.k_dtype = k_tensor.element_type + self.v_dtype = v_tensor.element_type + self.o_dtype = o_tensor.element_type + + # s_q, s_k, s_v are the actual tensor dimensions (total seqlen for varlen) + s_q = q_tensor.shape[1] + s_k = k_tensor.shape[1] + s_v = v_tensor.shape[1] + s_lse = s_lse_max + # Important for performance + align = 256 // self.o_dtype.width + d = cute.assume(Int32(d), align) + dv = cute.assume(Int32(dv), align) + + stride_b_q = h_r * h_k * s_q * d if cum_seqlen_q is None else 0 + stride_b_o = h_r * h_k * s_q * dv if cum_seqlen_q is None else 0 + stride_b_k = h_k * s_k * d if cum_seqlen_k is None else 0 + stride_b_v = h_k * s_v * dv if cum_seqlen_k is None else 0 + stride_b_lse = h_r * h_k * s_lse if cum_seqlen_q is None else 0 + + # (b, s_q, h_k, h_r, d) -> (s_q, d, ((h_r, h_k), b)) + q_layout = cute.make_layout( + (s_q, d, ((h_r, h_k), b)), + stride=(d * h_r * h_k, 1, ((d, d * h_r), stride_b_q)), + ) + q = cute.make_tensor(q_tensor.iterator, q_layout) + # (b, s_k, h_k, 1, d) -> (s_k, d, ((h_r, h_k), b)), 0-stride for h_r to broadcast + k_layout = cute.make_layout( + (s_k, d, ((h_r, h_k), b)), + stride=(d * h_k, 1, ((0, d), stride_b_k)), + ) + k = cute.make_tensor(k_tensor.iterator, k_layout) + # (b, s_v, h_k, 1, dv) -> (dv, s_v, ((h_r, h_k), b)), 0-stride for h_r to broadcast + v_layout = cute.make_layout( + (dv, s_v, ((h_r, h_k), b)), + stride=(1, dv * h_k, ((0, dv), stride_b_v)), + ) + v = cute.make_tensor(v_tensor.iterator, v_layout) + # (b, s_q, h_k, h_r, dv) -> (s_q, dv, ((h_r, h_k), b)) + o_layout = cute.make_layout( + (s_q, dv, ((h_r, h_k), b)), + stride=(dv * h_r * h_k, 1, ((dv, dv * h_r), stride_b_o)), + ) + o = cute.make_tensor(o_tensor.iterator, o_layout) + if cutlass.const_expr(lse_tensor is not None): + # (s, ((h_r, h_k), b)) - head stride=1 to match FlashInfer (total_q, h_q) convention + lse_layout = cute.make_layout( + (s_lse, ((h_r, h_k), b)), + stride=(h_r * h_k, ((1, h_r), stride_b_lse)), + ) + lse = cute.make_tensor(lse_tensor.iterator, lse_layout) + else: + lse = None + + if cutlass.const_expr(sink_tensor is not None): + # sink_tensor is 1D with shape (h_q,) = (h_k * h_r,) + # Create layout ((h_r, h_k), b) with stride 0 for batch so blk_coord[2] works + sink_layout = cute.make_layout( + ((h_r, h_k), b), + stride=((1, h_r), 0), + ) + sink = cute.make_tensor(sink_tensor.iterator, sink_layout) + else: + sink = None + + self.tile_sched_params, grid = fmha_utils.compute_grid( + cute.shape((s_q_max, d, ((h_r, h_k), b))), + self.cta_tiler, + self.is_persistent, + ) + self.q_major_mode = utils.LayoutEnum.from_tensor(q).mma_major_mode() + self.k_major_mode = utils.LayoutEnum.from_tensor(k).mma_major_mode() + self.v_major_mode = utils.LayoutEnum.from_tensor(v).mma_major_mode() + self.o_layout = utils.LayoutEnum.from_tensor(o) + + if cutlass.const_expr(self.q_major_mode != OperandMajorMode.K): + raise RuntimeError("The layout of q is not supported") + if cutlass.const_expr(self.k_major_mode != OperandMajorMode.K): + raise RuntimeError("The layout of k is not supported") + if cutlass.const_expr(self.v_major_mode != OperandMajorMode.MN): + raise RuntimeError("The layout of v is not supported") + + # check type consistency: Q and K must share the same dtype (qk_dtype); + # V may use a different dtype (pv_dtype) to allow qk_dtype != pv_dtype. + if cutlass.const_expr(self.q_dtype != self.k_dtype): + raise TypeError(f"Type mismatch: {self.q_dtype} != {self.k_dtype}") + self._setup_attributes(skip_softmax_threshold_log2 is not None) + + cta_group = tcgen05.CtaGroup.ONE + # the intermediate tensor p is from tmem & k-major + p_source = tcgen05.OperandSource.TMEM + p_major_mode = cute.nvgpu.OperandMajorMode.K + qk_tiled_mma = self._make_qk_tiled_mma(cta_group) + pv_tiled_mma = self._make_pv_tiled_mma(cta_group, p_major_mode, p_source) + + self.cluster_shape_mnk = (*self.cluster_shape_mn, 1) + self.cluster_layout_vmnk = cute.tiled_divide( + cute.make_layout(self.cluster_shape_mnk), + (qk_tiled_mma.thr_id.shape,), + ) + self.epi_tile = self.pv_mma_tiler[:2] + + q_smem_layout_staged = sm100_utils.make_smem_layout_a( + qk_tiled_mma, + self.qk_mma_tiler, + self.q_dtype, + self.q_stage, + ) + k_smem_layout_staged = sm100_utils.make_smem_layout_b( + qk_tiled_mma, + self.qk_mma_tiler, + self.k_dtype, + self.kv_stage, + ) + p_tmem_layout_staged = sm100_utils.make_smem_layout_a( + pv_tiled_mma, + self.pv_mma_tiler, + self.v_dtype, + self.acc_stage, + ) + v_smem_layout_staged_origin = sm100_utils.make_smem_layout_b( + pv_tiled_mma, + self.pv_mma_tiler, + self.v_dtype, + self.kv_stage, + ) + # k & v share the same smem buffer. Pad the smaller-tile operand's stage stride to + # match the larger tile. Stride is scaled to the target operand's element width. + # sK_cosize covers all kv_stage slots at the larger tile's byte footprint. + if cutlass.const_expr(self.k_dtype.width >= self.v_dtype.width): + # k tile >= v tile: give v k's stage stride in v_dtype units + k_tile_cosize = cute.cosize(cute.select(k_smem_layout_staged, mode=[0, 1, 2])) + v_stage_stride = k_tile_cosize * self.k_dtype.width // self.v_dtype.width + v_smem_layout_staged = cute.append( + cute.select(v_smem_layout_staged_origin, mode=[0, 1, 2]), + cute.make_layout(self.kv_stage, stride=v_stage_stride), + ) + sK_cosize = cute.cosize(k_smem_layout_staged) + else: + # v tile > k tile: give k v's stage stride in k_dtype units + v_tile_cosize = cute.cosize(cute.select(v_smem_layout_staged_origin, mode=[0, 1, 2])) + k_stage_stride = v_tile_cosize * self.v_dtype.width // self.k_dtype.width + k_smem_layout_staged = cute.append( + cute.select(k_smem_layout_staged, mode=[0, 1, 2]), + cute.make_layout(self.kv_stage, stride=k_stage_stride), + ) + v_smem_layout_staged = v_smem_layout_staged_origin + # cute.cosize gives (kv_stage-1)*k_stage_stride + k_tile_cosize, but the + # last stage must also accommodate a full V tile, so size for kv_stage slots. + sK_cosize = self.kv_stage * k_stage_stride + + o_smem_layout_staged = sm100_utils.make_smem_layout_epi( + self.o_dtype, + self.o_layout, + self.epi_tile, + self.epi_stage, + ) + + # TMA load for Q + tma_load_op = cute.nvgpu.cpasync.CopyBulkTensorTileG2SOp(cta_group) + tma_store_op = cute.nvgpu.cpasync.CopyBulkTensorTileS2GOp() + + q_smem_layout = cute.select(q_smem_layout_staged, mode=[0, 1, 2]) + tma_atom_q, tma_tensor_q = cute.nvgpu.make_tiled_tma_atom_A( + tma_load_op, + q, + q_smem_layout, + self.qk_mma_tiler, + qk_tiled_mma, + self.cluster_layout_vmnk.shape, + ) + + # TMA load for K + k_smem_layout = cute.select(k_smem_layout_staged, mode=[0, 1, 2]) + tma_atom_k, tma_tensor_k = cute.nvgpu.make_tiled_tma_atom_B( + tma_load_op, + k, + k_smem_layout, + self.qk_mma_tiler, + qk_tiled_mma, + self.cluster_layout_vmnk.shape, + ) + # TMA load for V + v_smem_layout = cute.select(v_smem_layout_staged, mode=[0, 1, 2]) + tma_atom_v, tma_tensor_v = cute.nvgpu.make_tiled_tma_atom_B( + tma_load_op, + v, + v_smem_layout, + self.pv_mma_tiler, + pv_tiled_mma, + self.cluster_layout_vmnk.shape, + ) + + o_smem_layout = cute.select(o_smem_layout_staged, mode=[0, 1]) + tma_atom_o, tma_tensor_o = cute.nvgpu.cpasync.make_tiled_tma_atom( + tma_store_op, + o, + o_smem_layout, + self.epi_tile, + ) + + q_copy_size = cute.size_in_bytes(self.q_dtype, q_smem_layout) + k_copy_size = cute.size_in_bytes(self.k_dtype, k_smem_layout) + v_copy_size = cute.size_in_bytes(self.v_dtype, v_smem_layout) + self.tma_copy_q_bytes = q_copy_size + self.tma_copy_k_bytes = k_copy_size + self.tma_copy_v_bytes = v_copy_size + + @cute.struct + class SharedStorage: + # Pipeline barriers + load_q_mbar_ptr: cute.struct.MemRange[Int64, self.q_stage * 2] + load_kv_mbar_ptr: cute.struct.MemRange[Int64, self.kv_stage * 2] + mma_s0_mbar_ptr: cute.struct.MemRange[Int64, self.mma_softmax_stage * 2] + mma_s1_mbar_ptr: cute.struct.MemRange[Int64, self.mma_softmax_stage * 2] + p0_mma_mbar_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] + p1_mma_mbar_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] + s0_corr_mbar_ptr: cute.struct.MemRange[Int64, self.softmax_corr_stage * 2] + s1_corr_mbar_ptr: cute.struct.MemRange[Int64, self.softmax_corr_stage * 2] + corr_epi_mbar_ptr: cute.struct.MemRange[Int64, self.epi_stage * 2] + mma_corr_mbar_ptr: cute.struct.MemRange[Int64, self.mma_corr_stage * 2] + s0_p1_inplace_barrier_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] + s1_p0_inplace_barrier_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] + # Tmem holding buffer + tmem_holding_buf: Int32 + # Smem tensors + sO: cute.struct.Align[ + cute.struct.MemRange[self.o_dtype, cute.cosize(o_smem_layout_staged)], + self.buffer_align_bytes, + ] + sQ: cute.struct.Align[ + cute.struct.MemRange[self.q_dtype, cute.cosize(q_smem_layout_staged)], + self.buffer_align_bytes, + ] + sK: cute.struct.Align[ + cute.struct.MemRange[self.k_dtype, sK_cosize], + self.buffer_align_bytes, + ] + # Skip softmax and PV warpgroup votes + s0_warp_wants_skip_softmax_exchange: cute.struct.MemRange[Int8, 4] + s1_warp_wants_skip_softmax_exchange: cute.struct.MemRange[Int8, 4] + + self.shared_storage = SharedStorage + + # Launch the kernel synchronously + self.kernel( + qk_tiled_mma, + pv_tiled_mma, + tma_atom_q, + tma_tensor_q, + tma_atom_k, + tma_tensor_k, + tma_atom_v, + tma_tensor_v, + tma_atom_o, + tma_tensor_o, + o, + cum_seqlen_q, + cum_seqlen_k, + lse, + sink, + scale_softmax_log2, + scale_softmax, + scale_output, + skip_softmax_threshold_log2, + window_size_left, + window_size_right, + q_smem_layout_staged, + k_smem_layout_staged, + p_tmem_layout_staged, + v_smem_layout_staged, + o_smem_layout_staged, + skip_softmax_count, + total_softmax_count, + self.tile_sched_params, + ).launch( + grid=grid, + block=[self.threads_per_cta, 1, 1], + cluster=self.cluster_shape_mnk, + stream=stream, + min_blocks_per_mp=1, + use_pdl=use_pdl, + ) + + # GPU device kernel + @cute.kernel + def kernel( + self, + qk_tiled_mma: cute.TiledMma, + pv_tiled_mma: cute.TiledMma, + tma_atom_q: cute.CopyAtom, + mQ_qdl: cute.Tensor, + tma_atom_k: cute.CopyAtom, + mK_kdl: cute.Tensor, + tma_atom_v: cute.CopyAtom, + mV_dkl: cute.Tensor, + tma_atom_o: cute.CopyAtom, + mO_qdl: cute.Tensor, + mO: cute.Tensor, + cum_seqlen_q: Optional[cute.Tensor], + cum_seqlen_k: Optional[cute.Tensor], + mLSE: Optional[cute.Tensor], + mSink: Optional[cute.Tensor], + scale_softmax_log2: Float32, + scale_softmax: Float32, + scale_output: Float32, + skip_softmax_threshold_log2: Optional[Float32], + window_size_left: Optional[Int32], + window_size_right: Optional[Int32], + q_smem_layout_staged: cute.ComposedLayout, + k_smem_layout_staged: cute.ComposedLayout, + p_tmem_layout_staged: cute.ComposedLayout, + v_smem_layout_staged: cute.ComposedLayout, + o_smem_layout_staged: cute.ComposedLayout, + skip_softmax_count: Optional[cute.Tensor], + total_softmax_count: Optional[cute.Tensor], + tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams, + ): + """The device kernel implementation of the Fused Multi-Head Attention. + + This kernel coordinates multiple specialized warps to perform different phases of the FMHA computation: + 1. Load warp: Loads Q, K, V data from global memory to shared memory using TMA + 2. MMA warp: Performs matrix multiplications (Q*K^T and P*V) + 3. Softmax warps: Compute softmax normalization on attention scores + 4. Correction warps: Apply adjustments to intermediate results + 5. Epilogue warp: Handles final output transformation and storage + + The kernel implements a complex pipeline with overlapping computation and memory operations, + using tensor memory access (TMA) for efficient data loading, warp specialization for different + computation phases, and optional attention masking. + + :param qk_tiled_mma: Tiled MMA for Q*K^T + :type qk_tiled_mma: cute.TiledMma + :param pv_tiled_mma: Tiled MMA for P*V + :type pv_tiled_mma: cute.TiledMma + :param tma_atom_q: TMA copy atom for query tensor + :type tma_atom_q: cute.CopyAtom + :param mQ_qdl: Partitioned query tensor + :type mQ_qdl: cute.Tensor + :param tma_atom_k: TMA copy atom for key tensor + :type tma_atom_k: cute.CopyAtom + :param mK_kdl: Partitioned key tensor + :type mK_kdl: cute.Tensor + :param tma_atom_v: TMA copy atom for value tensor + :type tma_atom_v: cute.CopyAtom + :param mV_dkl: Partitioned value tensor + :type mV_dkl: cute.Tensor + :param tma_atom_o: TMA copy atom for output tensor + :type tma_atom_o: cute.CopyAtom + :param mO_qdl: Partitioned output tensor + :type mO_qdl: cute.Tensor + :param mO: Non-partitioned output tensor + :type mO: cute.Tensor + :param scale_softmax_log2: The log2 scale factor for softmax + :type scale_softmax_log2: Float32 + :param scale_output: The scale factor for the output + :type scale_output: Float32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + :param q_smem_layout_staged: Shared memory layout for query tensor + :type q_smem_layout_staged: cute.ComposedLayout + :param k_smem_layout_staged: Shared memory layout for key tensor + :type k_smem_layout_staged: cute.ComposedLayout + :param p_tmem_layout_staged: Tensor memory layout for probability matrix + :type p_tmem_layout_staged: cute.ComposedLayout + :param v_smem_layout_staged: Shared memory layout for value tensor + :type v_smem_layout_staged: cute.ComposedLayout + :param o_smem_layout_staged: Shared memory layout for output tensor + :type o_smem_layout_staged: cute.ComposedLayout + :param tile_sched_params: Scheduling parameters for work distribution + :type tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams + """ + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + # coord inside cta + tidx, _, _ = cute.arch.thread_idx() + + # + # Prefetch tma desc + # + if warp_idx == self.load_warp_id: + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_q) + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_k) + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_v) + if cutlass.const_expr(self.use_tma_store): + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_o) + + # Alloc + smem = utils.SmemAllocator() + storage = smem.allocate(self.shared_storage) + + load_q_producer, load_q_consumer = pipeline.PipelineTmaUmma.create( + num_stages=self.q_stage, + producer_group=make_thread_cooperative_group(len([self.load_warp_id])), + consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + tx_count=self.tma_copy_q_bytes, + barrier_storage=storage.load_q_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + load_kv_producer, load_kv_consumer = pipeline.PipelineTmaUmma.create( + num_stages=self.kv_stage, + producer_group=make_thread_cooperative_group(len([self.load_warp_id])), + consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + tx_count=self.tma_copy_k_bytes, + barrier_storage=storage.load_kv_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + load_kv_full_mbar_ptr = storage.load_kv_mbar_ptr.data_ptr() + load_kv_empty_mbar_ptr = load_kv_full_mbar_ptr + self.kv_stage + mma_s0_producer, mma_s0_consumer = pipeline.PipelineUmmaAsync.create( + num_stages=self.mma_softmax_stage, + producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax0_warp_ids) + ), + barrier_storage=storage.mma_s0_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + mma_s1_producer, mma_s1_consumer = pipeline.PipelineUmmaAsync.create( + num_stages=self.mma_softmax_stage, + producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax1_warp_ids) + ), + barrier_storage=storage.mma_s1_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + p0_mma_producer, p0_mma_consumer = pipeline.PipelineAsyncUmma.create( + num_stages=self.p_mma_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax0_warp_ids) + ), + consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + barrier_storage=storage.p0_mma_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + p1_mma_producer, p1_mma_consumer = pipeline.PipelineAsyncUmma.create( + num_stages=self.p_mma_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax1_warp_ids) + ), + consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + barrier_storage=storage.p1_mma_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + s0_corr_producer, s0_corr_consumer = pipeline.PipelineAsync.create( + num_stages=self.softmax_corr_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len((*self.softmax0_warp_ids, self.mma_warp_id)) + ), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.correction_warp_ids) + ), + barrier_storage=storage.s0_corr_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + s1_corr_producer, s1_corr_consumer = pipeline.PipelineAsync.create( + num_stages=self.softmax_corr_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len((*self.softmax1_warp_ids, self.mma_warp_id)) + ), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.correction_warp_ids) + ), + barrier_storage=storage.s1_corr_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + corr_epi_producer, corr_epi_consumer = pipeline.PipelineAsync.create( + num_stages=self.epi_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.correction_warp_ids) + ), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len([self.epilogue_warp_id]) + ), + barrier_storage=storage.corr_epi_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + mma_corr_producer, mma_corr_consumer = pipeline.PipelineUmmaAsync.create( + num_stages=self.mma_corr_stage, + producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.correction_warp_ids) + ), + barrier_storage=storage.mma_corr_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + s0_p1_inplace_producer, s0_p1_inplace_consumer = pipeline.PipelineAsync.create( + num_stages=self.p_mma_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax0_warp_ids) + ), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax1_warp_ids) + ), + barrier_storage=storage.s0_p1_inplace_barrier_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + s1_p0_inplace_producer, s1_p0_inplace_consumer = pipeline.PipelineAsync.create( + num_stages=self.p_mma_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax0_warp_ids) + ), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax1_warp_ids) + ), + barrier_storage=storage.s1_p0_inplace_barrier_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + tmem = utils.TmemAllocator( + storage.tmem_holding_buf.ptr, + barrier_for_retrieve=self.tmem_alloc_barrier, + # Correction warp is the last one that accesses tmem + allocator_warp_id=self.correction_warp_ids[0], + arch=self.arch_str, ) + pipeline_init_arrive(is_relaxed=True) + + # Generate smem tensor Q/K/V/O + # (MMA, MMA_Q, MMA_D, PIPE) + sQ = storage.sQ.get_tensor(q_smem_layout_staged.outer, swizzle=q_smem_layout_staged.inner) + # (MMA, MMA_K, MMA_D, PIPE) + sK = storage.sK.get_tensor(k_smem_layout_staged.outer, swizzle=k_smem_layout_staged.inner) + # (MMA, MMA_K, MMA_D, PIPE) + # Reuse k's smem buffer for v. Recast element type so MMA descriptor matches v_dtype. + sV_ptr = cute.recast_ptr( + cute.recast_ptr(sK.iterator, dtype=self.v_dtype), + v_smem_layout_staged.inner, + ) + sV = cute.make_tensor(sV_ptr, v_smem_layout_staged.outer) + sO = storage.sO.get_tensor(o_smem_layout_staged.outer, swizzle=o_smem_layout_staged.inner) + s0_warp_wants_skip_softmax_exchange = ( + storage.s0_warp_wants_skip_softmax_exchange.get_tensor(cute.make_layout((4,))) + ) + s1_warp_wants_skip_softmax_exchange = ( + storage.s1_warp_wants_skip_softmax_exchange.get_tensor(cute.make_layout((4,))) + ) + + qk_thr_mma = qk_tiled_mma.get_slice(0) # default 1sm + pv_thr_mma = pv_tiled_mma.get_slice(0) # default 1sm + tSrQ = qk_thr_mma.make_fragment_A(sQ) + tSrK = qk_thr_mma.make_fragment_B(sK) + tOrV = pv_thr_mma.make_fragment_B(sV) + + def make_tmem_tensors( + self, + qk_thr_mma: cute.TiledMma, + pv_thr_mma: cute.TiledMma, + p_tmem_layout_staged: cute.Layout, + tmem_ptr: cute.Pointer, + ): + qk_acc_shape = qk_thr_mma.partition_shape_C( + (self.qk_mma_tiler[0], self.qk_mma_tiler[1]) + ) + tStS_fake = qk_thr_mma.make_fragment_C(qk_acc_shape) + tStS = cute.make_tensor(tmem_ptr + self.tmem_s0_offset, tStS_fake.layout) + pv_acc_shape = pv_thr_mma.partition_shape_C( + (self.pv_mma_tiler[0], self.pv_mma_tiler[1]) + ) + tOtO = pv_thr_mma.make_fragment_C(pv_acc_shape) + tStS0 = cute.make_tensor(tmem_ptr + self.tmem_s0_offset, tStS.layout) + tStS1 = cute.make_tensor(tmem_ptr + self.tmem_s1_offset, tStS.layout) + tOtO0 = cute.make_tensor(tmem_ptr + self.tmem_o0_offset, tOtO.layout) + tOtO1 = cute.make_tensor(tmem_ptr + self.tmem_o1_offset, tOtO.layout) + tP = cute.make_tensor(tStS.iterator, p_tmem_layout_staged.outer) + tOrP = pv_thr_mma.make_fragment_A(tP)[None, None, None, 0] + tOrP0 = cute.make_tensor( + cute.recast_ptr( + tmem_ptr + self.tmem_p0_offset, + dtype=tOrP.dtype, + ), + tOrP.layout, + ) + tOrP1 = cute.make_tensor( + cute.recast_ptr( + tmem_ptr + self.tmem_p1_offset, + dtype=tOrP.dtype, + ), + tOrP.layout, + ) + return tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 + + tile_sched = fmha_utils.create_fmha_static_tile_scheduler( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + pipeline_init_wait() + softmax_fn = partial( + self.softmax, + qk_thr_mma=qk_thr_mma, + value_args=( + mK_kdl.shape[0], + mQ_qdl.shape[0], + scale_softmax_log2, + skip_softmax_threshold_log2, + ), + mask_args=(window_size_left, window_size_right), + sched_args=(tile_sched, work_tile), + ) + # /////////////////////////////////////////////////////////////////////////////// + # EMPTY + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx == self.empty_warp_id: + cute.arch.setmaxregister_decrease(self.num_regs_other) + + # /////////////////////////////////////////////////////////////////////////////// + # LOAD + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx == self.load_warp_id: + cute.arch.setmaxregister_decrease(self.num_regs_other) + cute.arch.griddepcontrol_wait() + while work_tile.is_valid_tile: + curr_block_coord = work_tile.tile_idx + batch_coord = curr_block_coord[2][1] + continue_cond = False + cuseqlen_q = Int32(0) + seqlen_q = mQ_qdl.shape[0] + seqlen_k = mK_kdl.shape[0] + if cutlass.const_expr(cum_seqlen_q is not None): + cuseqlen_q = cum_seqlen_q[batch_coord] + seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q + continue_cond = ( + not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( + self.cta_tiler[0], + curr_block_coord[0], + seqlen_q, + ) + ) + if not continue_cond: + if cutlass.const_expr(cum_seqlen_k is not None): + seqlen_k = cum_seqlen_k[batch_coord + 1] - cum_seqlen_k[batch_coord] + continue_cond = seqlen_k <= 0 + if not continue_cond: + mQ_qdl_ = mQ_qdl + mK_kdl_ = mK_kdl + mV_dkl_ = mV_dkl + if cutlass.const_expr(cum_seqlen_q is not None): + mQ_qdl_ = cute.domain_offset( + (cum_seqlen_q[batch_coord], 0, ((0, 0), 0)), mQ_qdl + ) + if cutlass.const_expr(cum_seqlen_k is not None): + mK_kdl_ = cute.domain_offset( + (cum_seqlen_k[batch_coord], 0, ((0, 0), 0)), mK_kdl + ) + mV_dkl_ = cute.domain_offset( + (0, cum_seqlen_k[batch_coord], ((0, 0), 0)), mV_dkl + ) + # Local tile partition global tensors + gQ_qdl = cute.flat_divide(mQ_qdl_, cute.select(self.qk_mma_tiler, mode=[0, 2])) + tSgQ_qdl = qk_thr_mma.partition_A(gQ_qdl) + tQsQ, tQgQ_qdl = cute.nvgpu.cpasync.tma_partition( + tma_atom_q, + 0, # no multicast + cute.make_layout(1), + cute.group_modes(sQ, 0, 3), + cute.group_modes(tSgQ_qdl, 0, 3), + ) + tQgQ = tQgQ_qdl[None, None, 0, curr_block_coord[2]] + gK_kdl = cute.flat_divide(mK_kdl_, cute.select(self.qk_mma_tiler, mode=[1, 2])) + tSgK_kdl = qk_thr_mma.partition_B(gK_kdl) + tKsK, tKgK_kdl = cute.nvgpu.cpasync.tma_partition( + tma_atom_k, + 0, # no multicast + cute.make_layout(1), + cute.group_modes(sK, 0, 3), + cute.group_modes(tSgK_kdl, 0, 3), + ) + tKgK = tKgK_kdl[None, None, 0, curr_block_coord[2]] + gV_dkl = cute.flat_divide(mV_dkl_, cute.select(self.pv_mma_tiler, mode=[1, 2])) + tSgV_dkl = pv_thr_mma.partition_B(gV_dkl) + tVsV, tVgV_dkl = cute.nvgpu.cpasync.tma_partition( + tma_atom_v, + 0, # no multicast + cute.make_layout(1), + cute.group_modes(sV, 0, 3), + cute.group_modes(tSgV_dkl, 0, 3), + ) + tVgV = tVgV_dkl[None, 0, None, curr_block_coord[2]] + seqlen_kv_loop_start = fmha_utils.FusedMask.get_trip_start( + self.mask_type, + curr_block_coord, + self.cta_tiler, + seqlen_q, + seqlen_k, + window_size_left, + ) + # Q0 + q0_coord = 2 * curr_block_coord[0] + q0_handle = load_q_producer.acquire_and_advance() + cute.copy( + tma_atom_q, + tQgQ[None, q0_coord], + tQsQ[None, q0_handle.index], + tma_bar_ptr=q0_handle.barrier, + ) + seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( + self.mask_type, + curr_block_coord, + self.cta_tiler, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + # K0 + kv_coord = seqlen_kv_loop_start + k_handle = load_kv_producer.acquire_and_advance() + cute.copy( + tma_atom_k, + tKgK[None, kv_coord], + tKsK[None, k_handle.index], + tma_bar_ptr=k_handle.barrier, + ) + # Q1 + q1_coord = q0_coord + 1 + q1_handle = load_q_producer.acquire_and_advance() + cute.copy( + tma_atom_q, + tQgQ[None, q1_coord], + tQsQ[None, q1_handle.index], + tma_bar_ptr=q1_handle.barrier, + ) + kv_coord += 1 + + for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): + # Ki + k_handle = load_kv_producer.acquire_and_advance() + cute.copy( + tma_atom_k, + tKgK[None, kv_coord], + tKsK[None, k_handle.index], + tma_bar_ptr=k_handle.barrier, + ) + # Vi-1 + v_handle, load_kv_producer = self.kv_producer_update_tx_acquire_and_advance( + load_kv_producer, + load_kv_empty_mbar_ptr, + load_kv_full_mbar_ptr, + self.tma_copy_v_bytes, + ) + cute.copy( + tma_atom_v, + tVgV[None, kv_coord - 1], + tVsV[None, v_handle.index], + tma_bar_ptr=load_kv_full_mbar_ptr + v_handle.index, + ) + kv_coord += 1 + # End of seqlen_kv loop + # Vi_end + v_handle, load_kv_producer = self.kv_producer_update_tx_acquire_and_advance( + load_kv_producer, + load_kv_empty_mbar_ptr, + load_kv_full_mbar_ptr, + self.tma_copy_v_bytes, + ) + cute.copy( + tma_atom_v, + tVgV[None, kv_coord - 1], + tVsV[None, v_handle.index], + tma_bar_ptr=load_kv_full_mbar_ptr + v_handle.index, + ) + # End of if not continue_cond + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + # End of persistent scheduler loop + # /////////////////////////////////////////////////////////////////////////////// + # MMA + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx == self.mma_warp_id: + cute.arch.setmaxregister_decrease(self.num_regs_other) + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) + tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( + self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr + ) + enable_skip_softmax = skip_softmax_threshold_log2 is not None + tiled_tmem_load_v = None + tTMEM_LOADtS_v0, tTMEM_LOADtS_v1 = None, None + tTMEM_LOADrS_v0, tTMEM_LOADrS_v1 = None, None + if cutlass.const_expr(enable_skip_softmax): + cS = cute.make_identity_tensor(cute.select(self.qk_mma_tiler, mode=[0, 1])) + tScS = qk_thr_mma.partition_C(cS) + tStS_v = cute.composition(tStS, cute.make_layout((self.threads_per_warp, 1))) + tScS_v = cute.composition(tScS, cute.make_layout((self.threads_per_warp, 1))) + tmem_load_v_atom = cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(1)), + self.qk_acc_dtype, + ) + thread_idx = tidx % self.threads_per_warp + + tiled_tmem_load_v = tcgen05.make_tmem_copy(tmem_load_v_atom, tStS_v) + thr_tmem_load_v = tiled_tmem_load_v.get_slice(thread_idx) + tTMEM_LOADtS_v = thr_tmem_load_v.partition_S(tStS_v) + tTMEM_LOADcS_v = thr_tmem_load_v.partition_D(tScS_v) + tTMEM_LOADrS_v0 = cute.make_rmem_tensor(tTMEM_LOADcS_v.shape, self.qk_acc_dtype) + tTMEM_LOADrS_v1 = cute.make_rmem_tensor(tTMEM_LOADcS_v.shape, self.qk_acc_dtype) + tTMEM_LOADtS_v0 = cute.make_tensor( + tTMEM_LOADtS_v.iterator + self.tmem_skip_softmax0_offset, + tTMEM_LOADtS_v.layout, + ) + tTMEM_LOADtS_v1 = cute.make_tensor( + tTMEM_LOADtS_v.iterator + self.tmem_skip_softmax1_offset, + tTMEM_LOADtS_v.layout, + ) + + while work_tile.is_valid_tile: + curr_block_coord = work_tile.tile_idx + batch_coord = curr_block_coord[2][1] + continue_cond = False + seqlen_q = mQ_qdl.shape[0] + seqlen_k = mK_kdl.shape[0] + if cutlass.const_expr(cum_seqlen_q is not None): + cuseqlen_q = cum_seqlen_q[batch_coord] + seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q + continue_cond = ( + not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( + self.cta_tiler[0], + curr_block_coord[0], + seqlen_q, + ) + ) + if not continue_cond: + if cutlass.const_expr(cum_seqlen_k is not None): + cuseqlen_k = cum_seqlen_k[batch_coord] + seqlen_k = cum_seqlen_k[batch_coord + 1] - cuseqlen_k + continue_cond = seqlen_k <= 0 + if not continue_cond: + # Wait for Q0 + q0_handle = load_q_consumer.wait_and_advance() + tSrQ0 = tSrQ[None, None, None, q0_handle.index] + # Wait for K0 + k_handle = load_kv_consumer.wait_and_advance() + tSrK0 = tSrK[None, None, None, k_handle.index] + # GEMM_QK00 (Q0 * K0 -> S0) + mma_s0_producer, s0_corr_producer = self.mma_qk( + qk_tiled_mma, + (tSrQ0, tSrK0, tStS0), + (mma_s0_producer, s0_corr_producer), + ) + # Wait for Q1 + q1_handle = load_q_consumer.wait_and_advance() + tSrQ1 = tSrQ[None, None, None, q1_handle.index] + # GEMM_QK10 (Q1 * K0 -> S1), K0 is ready in GEMM_QK00 + mma_s1_producer, s1_corr_producer = self.mma_qk( + qk_tiled_mma, + (tSrQ1, tSrK0, tStS1), + (mma_s1_producer, s1_corr_producer), + ) + # Release K0 + k_handle.release() + # Note: Q0 & Q1 are still needed in the seqlen_kv loop + # so we need to release them after the seqlen_kv loop + seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( + self.mask_type, + curr_block_coord, + self.cta_tiler, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + # O1 hasn't been accumulated yet, its first MMA calculation doesn't need to accumulate + pv_whether_acc = False + for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): + # Wait for Ki + k_handle = load_kv_consumer.wait_and_advance() + tSrKi = tSrK[None, None, None, k_handle.index] + # GEMM_QK0i (Q0 * Ki -> S0) + mma_s0_producer, s0_corr_producer = self.mma_qk( + qk_tiled_mma, + (tSrQ0, tSrKi, tStS0), + (mma_s0_producer, s0_corr_producer), + ) + # Wait for Vi-1 + v_handle = load_kv_consumer.wait_and_advance() + tOrVi = tOrV[None, None, None, v_handle.index] + # GEMM_PV0(i-1) (P0 * Vi-1 -> O0_partial) + mma_corr_producer, p0_mma_consumer = self.mma_pv( + pv_tiled_mma, + pv_whether_acc, + (tOrP0, tOrVi, tOtO0), + (mma_corr_producer, p0_mma_consumer), + ( + enable_skip_softmax, + tiled_tmem_load_v, + tTMEM_LOADtS_v0, + tTMEM_LOADrS_v0, + ), + ) + # GEMM_QK1i (Q1 * Ki -> S1) + mma_s1_producer, s1_corr_producer = self.mma_qk( + qk_tiled_mma, + (tSrQ1, tSrKi, tStS1), + (mma_s1_producer, s1_corr_producer), + ) + # Release Ki + k_handle.release() + # GEMM_PV1(i-1) (P1 * Vi-1 -> O1_partial) + mma_corr_producer, p1_mma_consumer = self.mma_pv( + pv_tiled_mma, + pv_whether_acc, + (tOrP1, tOrVi, tOtO1), + (mma_corr_producer, p1_mma_consumer), + ( + enable_skip_softmax, + tiled_tmem_load_v, + tTMEM_LOADtS_v1, + tTMEM_LOADrS_v1, + ), + ) + pv_whether_acc = True + # Release Vi-1 + v_handle.release() + # End of seqlen_kv loop + # release Q0 & Q1 + q0_handle.release() + q1_handle.release() + # Wait for Vi_end + v_handle = load_kv_consumer.wait_and_advance() + tOrVi = tOrV[None, None, None, v_handle.index] + # GEMM_PV0(i_end) (P0 * Vi_end -> O0) + mma_corr_producer, p0_mma_consumer = self.mma_pv( + pv_tiled_mma, + pv_whether_acc, + (tOrP0, tOrVi, tOtO0), + (mma_corr_producer, p0_mma_consumer), + ( + enable_skip_softmax, + tiled_tmem_load_v, + tTMEM_LOADtS_v0, + tTMEM_LOADrS_v0, + ), + ) + # GEMM_PV1(i_end) (P1 * Vi_end -> O1) + mma_corr_producer, p1_mma_consumer = self.mma_pv( + pv_tiled_mma, + pv_whether_acc, + (tOrP1, tOrVi, tOtO1), + (mma_corr_producer, p1_mma_consumer), + ( + enable_skip_softmax, + tiled_tmem_load_v, + tTMEM_LOADtS_v1, + tTMEM_LOADrS_v1, + ), + ) + # Release Vi_end + v_handle.release() + # Empty step for correction epilog + vec0_handle = s0_corr_producer.acquire_and_advance() + vec0_handle.commit() + vec1_handle = s1_corr_producer.acquire_and_advance() + vec1_handle.commit() + # End of if not continue_cond + # Advance to next tile + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + # End of persistent scheduler loop + # /////////////////////////////////////////////////////////////////////////////// + # Epilogue (TMA store path only) + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx == self.epilogue_warp_id: + cute.arch.setmaxregister_decrease(self.num_regs_other) + if cutlass.const_expr(self.use_tma_store): + while work_tile.is_valid_tile: + curr_block_coord = work_tile.tile_idx + batch_coord = curr_block_coord[2][1] + continue_cond = False + cuseqlen_q = Int32(0) + seqlen_q = mQ_qdl.shape[0] + + if cutlass.const_expr(cum_seqlen_q is not None): + cuseqlen_q = cum_seqlen_q[batch_coord] + seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q + continue_cond = ( + not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( + self.cta_tiler[0], + curr_block_coord[0], + seqlen_q, + ) + ) + if not continue_cond: + mO_qdl_ = mO_qdl + if cutlass.const_expr(cum_seqlen_q is not None): + mO_qdl_ = cute.domain_offset( + (cum_seqlen_q[batch_coord], 0, ((0, 0), 0)), mO_qdl + ) + + o0_coord = 2 * curr_block_coord[0] + o1_coord = o0_coord + 1 + gO_qdl = cute.flat_divide( + mO_qdl_, cute.select(self.pv_mma_tiler, mode=[0, 1]) + ) + gO = gO_qdl[None, None, None, 0, curr_block_coord[2]] + tOsO, tOgO = cute.nvgpu.cpasync.tma_partition( + tma_atom_o, + 0, + cute.make_layout(1), + cute.group_modes(sO, 0, 2), + cute.group_modes(gO, 0, 2), + ) + + # O0 O1 using the same pipeline + # wait from corr, issue tma store on smem + # O0 + # 1. Wait for O0 final + o0_handle = corr_epi_consumer.wait_and_advance() + # 2. Copy O0 to gmem + cute.copy(tma_atom_o, tOsO[None, 0], tOgO[None, o0_coord]) + cute.arch.cp_async_bulk_commit_group() + # O1 + # 1. Wait for O1 final + o1_handle = corr_epi_consumer.wait_and_advance() + # 2. Copy O1 to gmem + cute.copy(tma_atom_o, tOsO[None, 1], tOgO[None, o1_coord]) + cute.arch.cp_async_bulk_commit_group() + + # Ensure O0 buffer is ready to be released + cute.arch.cp_async_bulk_wait_group(1, read=True) + o0_handle.release() + # Ensure O1 buffer is ready to be released + cute.arch.cp_async_bulk_wait_group(0, read=True) + o1_handle.release() + + # Advance to next tile + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + cute.arch.griddepcontrol_launch_dependents() + # End of persistent scheduler loop + # /////////////////////////////////////////////////////////////////////////////// + # Softmax0 + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx < self.softmax1_warp_ids[0]: + cute.arch.setmaxregister_increase(self.num_regs_softmax) + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) + tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( + self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr + ) + softmax_fn( + stage=0, + tensor_args=( + tStS, + tStS0, + cum_seqlen_k, + cum_seqlen_q, + s0_warp_wants_skip_softmax_exchange, + skip_softmax_count, + total_softmax_count, + ), + pipeline_args=(mma_s0_consumer, s0_corr_producer, p0_mma_producer), + inplace_args=(s0_p1_inplace_producer, s1_p0_inplace_consumer), + ) + + # /////////////////////////////////////////////////////////////////////////////// + # Softmax1 + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx < self.correction_warp_ids[0] and warp_idx >= self.softmax1_warp_ids[0]: + cute.arch.setmaxregister_increase(self.num_regs_softmax) + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) + tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( + self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr + ) + softmax_fn( + stage=1, + tensor_args=( + tStS, + tStS1, + cum_seqlen_k, + cum_seqlen_q, + s1_warp_wants_skip_softmax_exchange, + skip_softmax_count, + total_softmax_count, + ), + pipeline_args=(mma_s1_consumer, s1_corr_producer, p1_mma_producer), + inplace_args=(s1_p0_inplace_producer, s0_p1_inplace_consumer), + ) + + # /////////////////////////////////////////////////////////////////////////////// + # Correction + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx >= self.correction_warp_ids[0] and warp_idx < self.mma_warp_id: + cute.arch.setmaxregister_decrease(self.num_regs_correction) + tmem.allocate(self.num_tmem_alloc_cols) + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) + tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( + self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr + ) + cS = cute.make_identity_tensor((self.qk_mma_tiler[0], self.qk_mma_tiler[1])) + tScS = qk_thr_mma.partition_C(cS) + + tStS_vec_layout = cute.composition(tStS.layout, cute.make_layout((128, 2))) + + tStS_vec0 = cute.make_tensor(tStS.iterator + self.tmem_vec0_offset, tStS_vec_layout) + tStS_vec1 = cute.make_tensor(tStS.iterator + self.tmem_vec1_offset, tStS_vec_layout) + + tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) + tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) + tmem_load_v_atom = cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(2)), + self.qk_acc_dtype, + ) + tiled_tmem_load_vec = tcgen05.make_tmem_copy(tmem_load_v_atom, tStS_vec0) + thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) + thr_tmem_load_vec = tiled_tmem_load_vec.get_slice(thread_idx) + tTMEM_LOAD_VECtS0 = thr_tmem_load_vec.partition_S(tStS_vec0) + tTMEM_LOAD_VECtS1 = thr_tmem_load_vec.partition_S(tStS_vec1) + tTMEM_LOAD_VECcS = thr_tmem_load_vec.partition_D(tScS_vec) + while work_tile.is_valid_tile: + curr_block_coord = work_tile.tile_idx + batch_coord = curr_block_coord[2][1] + seqlen_k = mK_kdl.shape[0] + row_idx = Int32(0) + continue_cond = False + cuseqlen_q = Int32(0) + seqlen_q = mQ_qdl.shape[0] + + if cutlass.const_expr(cum_seqlen_q is not None): + cuseqlen_q = cum_seqlen_q[batch_coord] + seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q + continue_cond = ( + not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( + self.cta_tiler[0], + curr_block_coord[0], + seqlen_q, + ) + ) + if not continue_cond: + row_idx = curr_block_coord[0] * self.cta_tiler[0] + tTMEM_LOAD_VECcS[0][0] + if cutlass.const_expr(cum_seqlen_k is not None): + cuseqlen_k = cum_seqlen_k[batch_coord] + seqlen_k = cum_seqlen_k[batch_coord + 1] - cuseqlen_k + continue_cond = seqlen_k <= 0 + if not continue_cond: + # Ignore first signal from softmax as no correction is required + vec0_handle = s0_corr_consumer.wait_and_advance() + vec0_handle.release() + vec1_handle = s1_corr_consumer.wait_and_advance() + vec1_handle.release() + # O0/O1 share the same mma_corr consumer state, so the Oi + # peek token rolls from O0 -> O1 -> next O0. Seed with a + # blocking token; the rescale helper refreshes it near the + # end of each iteration. + oi_peek_status = cutlass.Boolean(False) + seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( + self.mask_type, + curr_block_coord, + self.cta_tiler, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): + # Rescale O0 + ( + (s0_corr_consumer, mma_corr_consumer), + oi_peek_status, + ) = self.correction_rescale( + pv_thr_mma, + tiled_tmem_load_vec, + scale_softmax_log2, + (tOtO0, tTMEM_LOAD_VECtS0, tTMEM_LOAD_VECcS), + (s0_corr_consumer, mma_corr_consumer), + oi_peek_status, + ) + # Rescale O1 + ( + (s1_corr_consumer, mma_corr_consumer), + oi_peek_status, + ) = self.correction_rescale( + pv_thr_mma, + tiled_tmem_load_vec, + scale_softmax_log2, + (tOtO1, tTMEM_LOAD_VECtS1, tTMEM_LOAD_VECcS), + (s1_corr_consumer, mma_corr_consumer), + oi_peek_status, + ) + # End of seqlen_corr_loop_steps + value_args = ( + cuseqlen_q, + seqlen_q, + curr_block_coord, + scale_softmax, + scale_output, + ) + if cutlass.const_expr(self.use_tma_store): + # TMA store path: write to sO, signal epilogue warp + # Normalize O0 + s0_corr_consumer, mma_corr_consumer, corr_epi_producer = ( + self.correction_epilog( + pv_thr_mma, + tiled_tmem_load_vec, + ( + tOtO0, + tTMEM_LOAD_VECtS0, + tTMEM_LOAD_VECcS, + sO[None, None, 0], + mLSE, + mSink, + ), + ( + s0_corr_consumer, + mma_corr_consumer, + corr_epi_producer, + ), + (row_idx, *value_args), + ) + ) + row_idx += self.qk_mma_tiler[0] + # Normalize O1 + s1_corr_consumer, mma_corr_consumer, corr_epi_producer = ( + self.correction_epilog( + pv_thr_mma, + tiled_tmem_load_vec, + ( + tOtO1, + tTMEM_LOAD_VECtS1, + tTMEM_LOAD_VECcS, + sO[None, None, 1], + mLSE, + mSink, + ), + ( + s1_corr_consumer, + mma_corr_consumer, + corr_epi_producer, + ), + (row_idx, *value_args), + ) + ) + else: + # st.global path: store directly to global memory + block_offset_o = Int32(0) + if cutlass.const_expr(cum_seqlen_q is not None): + block_offset_o = cum_seqlen_q[batch_coord] + mO_ = cute.make_tensor( + mO.iterator + block_offset_o * mO.stride[0], + cute.make_layout( + (seqlen_q, mO.shape[1], mO.shape[2]), + stride=mO.stride, + ), + ) + o0_coord = 2 * curr_block_coord[0] + o1_coord = o0_coord + 1 + gO_stg = cute.local_tile( + mO_, + (self.pv_mma_tiler[0], self.pv_mma_tiler[1]), + (None, None, None), + ) + gO0 = gO_stg[None, None, o0_coord, 0, curr_block_coord[2]] + gO1 = gO_stg[None, None, o1_coord, 0, curr_block_coord[2]] + # Normalize O0 and store to global memory + s0_corr_consumer, mma_corr_consumer = self.correction_epilog( + pv_thr_mma, + tiled_tmem_load_vec, + ( + tOtO0, + tTMEM_LOAD_VECtS0, + tTMEM_LOAD_VECcS, + gO0, + mLSE, + mSink, + ), + (s0_corr_consumer, mma_corr_consumer), + (row_idx, *value_args), + ) + row_idx += self.qk_mma_tiler[0] + # Normalize O1 and st.global to global memory + s1_corr_consumer, mma_corr_consumer = self.correction_epilog( + pv_thr_mma, + tiled_tmem_load_vec, + ( + tOtO1, + tTMEM_LOAD_VECtS1, + tTMEM_LOAD_VECcS, + gO1, + mLSE, + mSink, + ), + (s1_corr_consumer, mma_corr_consumer), + (row_idx, *value_args), + ) + # End of if not continue_cond + # Advance to next tile + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + if cutlass.const_expr(not self.use_tma_store): + cute.arch.griddepcontrol_launch_dependents() + # End of persistent scheduler loop + tmem.relinquish_alloc_permit() + # Synchronize before TMEM dealloc (done by the caller) + self.tmem_dealloc_barrier.arrive_and_wait() + tmem.free(tmem_ptr) return - q_cute = _to_cute_tensor(q_5d, leading_dim=4) - k_cute = _to_cute_tensor(k_5d, leading_dim=4) - v_cute = _to_cute_tensor(v_5d, leading_dim=4) - o_cute = _to_cute_tensor(o_5d, leading_dim=4) - - cum_seqlen_q_cute = None - cum_seqlen_k_cute = None - if varlen: - qo_indptr_i32 = _to_cint_contiguous(qo_indptr) - kv_indptr_i32 = _to_cint_contiguous(kv_indptr) - cum_seqlen_q_cute = from_dlpack(qo_indptr_i32, assumed_align=16).mark_layout_dynamic( - leading_dim=0 - ) - cum_seqlen_k_cute = from_dlpack(kv_indptr_i32, assumed_align=16).mark_layout_dynamic( - leading_dim=0 - ) - - lse_iter = None - if lse is not None: - lse_cute = from_dlpack(lse_4d, assumed_align=16).mark_layout_dynamic(leading_dim=2) - lse_iter = lse_cute.iterator - - stream = cuda_driver.CUstream(torch.cuda.current_stream(q).cuda_stream) - kernel_fn( - q_cute.iterator, - k_cute.iterator, - v_cute.iterator, - o_cute.iterator, - problem_size, - cum_seqlen_q_cute, - cum_seqlen_k_cute, - lse_iter, - None, # sink_iter - cute_typing.Float32(scale_softmax_log2), - cute_typing.Float32(scale_softmax), - cute_typing.Float32(scale_output), - skip_threshold_log2, - ws_left, - ws_right, - None, - None, - False, # reserved - stream, - ) + @cute.jit + def kv_producer_update_tx_acquire_and_advance( + self, tma_producer, empty_mbar_ptr, full_mbar_ptr, tx_bytes + ): + # This utility function is a special version of tma_producer.acquire_and_advance(). + # This is used to customize the tx bytes which is different from + # the initialized tx bytes of tma_producer. + state = tma_producer._PipelineProducer__state.clone() + cute.arch.mbarrier_wait(empty_mbar_ptr + state.index, state.phase) + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive_and_expect_tx( + full_mbar_ptr + state.index, + tx_bytes, + ) + tma_producer.advance() + return state, tma_producer + + @cute.jit + def get_skip_softmax_flag(self, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v): + cute.copy(tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v) + tTMEM_LOADrS_v_i32 = cute.recast_tensor(tTMEM_LOADrS_v, dtype=cutlass.Int32) + skip_softmax_flag = cute.arch.make_warp_uniform(tTMEM_LOADrS_v_i32[0]) + return skip_softmax_flag + + @cute.jit + def mma_qk( + self, + tiled_mma: cute.TiledMma, + tensor_args: Tuple, + pipeline_args: Tuple, + pipeline_tokens: Tuple = (None, None), + ) -> Tuple[pipeline.PipelineProducer, pipeline.PipelineProducer]: + """Perform a single step of the QK GEMM computation on a block of attention scores. + + :param tiled_mma: Tiled MMA for QK GEMM + :type tiled_mma: cute.TiledMma + :param tensor_args: Tuple containing Qi, K, and Si + :type tensor_args: Tuple + :param pipeline_args: Tuple containing mma_si_producer and si_corr_producer + :type pipeline_args: Tuple + :param pipeline_tokens: Optional non-blocking peek tokens for the Si and + vec_i producers, in the form ``(si_peek_status, veci_peek_status)``. + ``None`` for either token falls back to a blocking acquire. + :type pipeline_tokens: Tuple + :return: Tuple containing mma_si_producer and si_corr_producer + :rtype: Tuple[pipeline.PipelineProducer, pipeline.PipelineProducer] + """ + tSrQi, tSrK, tStSi = tensor_args + mma_si_producer, si_corr_producer = pipeline_args + si_peek_status, veci_peek_status = pipeline_tokens + # 0. Make sure Qi & K are ready when calling mma_qk + # 1. acquire S0 + si_handle = mma_si_producer.acquire_and_advance(si_peek_status) + # 2. make sure vec is already released in corr + veci_handle = si_corr_producer.acquire_and_advance(veci_peek_status) + veci_handle.commit() + # 3. gemm + num_kphases = cute.size(tSrQi, mode=[2]) + for kphase_idx in cutlass.range(num_kphases, unroll_full=True): + kphase_coord = (None, None, kphase_idx) + tiled_mma.set(tcgen05.Field.ACCUMULATE, kphase_idx != 0) + cute.gemm( + tiled_mma, + tStSi, + tSrQi[kphase_coord], + tSrK[kphase_coord], + tStSi, + ) + # 4. release S0 + si_handle.commit() + return mma_si_producer, si_corr_producer + + @cute.jit + def mma_pv( + self, + tiled_mma: cute.TiledMma, + whether_acc: bool, + tensor_args: Tuple, + pipeline_args: Tuple, + skip_pv_args: Tuple, + ) -> Tuple[ + pipeline.PipelineProducer, + pipeline.PipelineConsumer, + ]: + """Perform a single step of the PV GEMM computation on accumulating O. + + :param tiled_mma: Tiled MMA for PV GEMM + :type tiled_mma: cute.TiledMma + :param whether_acc: Whether to accumulate O + :type whether_acc: bool + :param tensor_args: Tuple containing Pi, Vi, and Oi + :type tensor_args: Tuple + :param pipeline_args: Tuple containing mma_corr_producer and pi_mma_consumer + :type pipeline_args: Tuple + :param skip_pv_args: Tuple containing enable_skip_softmax, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v + :type skip_pv_args: Tuple + :return: Tuple containing mma_corr_producer and pi_mma_consumer + :rtype: Tuple[pipeline.PipelineProducer, pipeline.PipelineConsumer] + """ + tOrPi, tOrVi, tOtOi = tensor_args + mma_corr_producer, pi_mma_consumer = pipeline_args + enable_skip_softmax, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v = skip_pv_args + # 0. Make sure Vi is ready when calling mma_pv + # 1. acquire Oi + oi_handle = mma_corr_producer.acquire_and_advance() + # 2. wait for Pi + pi_handle = pi_mma_consumer.wait_and_advance() + # 3. gemm + num_kphases = cute.size(tOrPi, mode=[2]) + if cutlass.const_expr(enable_skip_softmax): + skip_pv = self.get_skip_softmax_flag(tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v) + if not skip_pv: + for kphase_idx in cutlass.range(num_kphases, unroll_full=True): + kphase_coord = (None, None, kphase_idx) + tiled_mma.set(tcgen05.Field.ACCUMULATE, whether_acc or kphase_idx != 0) + cute.gemm( + tiled_mma, + tOtOi, + tOrPi[kphase_coord], + tOrVi[kphase_coord], + tOtOi, + ) + else: + for kphase_idx in cutlass.range(num_kphases, unroll_full=True): + kphase_coord = (None, None, kphase_idx) + tiled_mma.set(tcgen05.Field.ACCUMULATE, whether_acc or kphase_idx != 0) + cute.gemm( + tiled_mma, + tOtOi, + tOrPi[kphase_coord], + tOrVi[kphase_coord], + tOtOi, + ) + # 4. commit Pi + pi_handle.release() + # 5. commit Oi + oi_handle.commit() + return mma_corr_producer, pi_mma_consumer + + @cute.jit + def calculate_skip_softmax_flag( + self, + row_max, + tile_row_max, + scale_softmax_log2, + skip_softmax_threshold_log2, + seqlen_q, + thread_idx, + logical_offset, + warp_wants_skip_softmax_exchange, + stage, + skip_softmax_count, + total_softmax_count, + ) -> Tuple[bool, float]: + """Calculate the skip softmax flag and the row maximum. + + :param row_max: The row maximum. + :type row_max: float + :param tile_row_max: The tile row maximum. + :type tile_row_max: float + :param scale_softmax_log2: The scale softmax log2. + :type scale_softmax_log2: float + :param skip_softmax_threshold_log2: The skip softmax threshold log2. + :type skip_softmax_threshold_log2: float + :param seqlen_q: The sequence length q. + :type seqlen_q: int + :param thread_idx: The thread index. + :type thread_idx: int + :param logical_offset: The logical offset. + :type logical_offset: Tuple[int, int] + :param warp_wants_skip_softmax_exchange: The warp wants skip softmax exchange. + :type warp_wants_skip_softmax_exchange: cute.Tensor + :param stage: The stage. + :type stage: int + :param skip_softmax_count: The skip softmax count. + :type skip_softmax_count: cute.Tensor + :param total_softmax_count: The total softmax count. + :type total_softmax_count: cute.Tensor + :return: Tuple containing the skip softmax flag and the row maximum. + :rtype: Tuple[bool, float] + """ + thread_wants_skip = ( + tile_row_max * scale_softmax_log2 - row_max * scale_softmax_log2 + ) < skip_softmax_threshold_log2 + thread_wants_skip = thread_wants_skip or ((logical_offset[0] + thread_idx) >= seqlen_q) + warp_wants_skip = cute.arch.vote_all_sync(thread_wants_skip) + + with cute.arch.elect_one(): + warp_wants_skip_softmax_exchange[cute.arch.warp_idx() % 4] = warp_wants_skip + softmax_barrier = self.s0_warpgroup_barrier if stage == 0 else self.s1_warpgroup_barrier + softmax_barrier.arrive_and_wait() + warp_wants_skip_softmax_exchange_i32 = cute.make_tensor( + cute.recast_ptr(warp_wants_skip_softmax_exchange.iterator, dtype=cutlass.Int32), + cute.make_layout((1,)), + ) + skip_softmax = cute.arch.popc(warp_wants_skip_softmax_exchange_i32[0]) == 4 + + if not skip_softmax: + row_max = max(row_max, tile_row_max) + + if cutlass.const_expr(skip_softmax_count is not None): + if thread_idx == 0: + if skip_softmax: + cute.arch.atomic_add(skip_softmax_count.iterator.llvm_ptr, Int32(1)) + cute.arch.atomic_add(total_softmax_count.iterator.llvm_ptr, Int32(1)) + return skip_softmax, row_max + + @cute.jit + def apply_exp_and_cvt( + self, + tTMEM_LOADrS, + tTMEM_LOADrS_cvt, + tTMEM_STORErS_x4_e_cvt, + stage, + scale, + minus_row_max_scale, + local_row_sum, + inplace_consumer, + EXP2_EMULATION_OFFSET, + EXP2_EMULATION_COUNT, + CVT_COUNT, + CVT_PER_STEP, + FMA_COUNT, + ARV_COUNT, + ): + """Apply the exp and conversion to the P data type on fragment. + + :param tTMEM_LOADrS: The tTMEM_LOADrS tensor. + :type tTMEM_LOADrS: cute.Tensor + :param tTMEM_LOADrS_cvt: The tTMEM_LOADrS_cvt tensor. + :type tTMEM_LOADrS_cvt: cute.Tensor + :param tTMEM_STORErS_x4_e_cvt: The tTMEM_STORErS_x4_e_cvt tensor. + :type tTMEM_STORErS_x4_e_cvt: cute.Tensor + :param stage: The stage. + :type stage: int + :param scale: The scale. + :type scale: float + :param minus_row_max_scale: The minus row maximum scale. + :type minus_row_max_scale: float + :param local_row_sum: The local row sum. + :type local_row_sum: float + :param inplace_consumer: The inplace consumer. + :type inplace_consumer: cute.Tensor + :param EXP2_EMULATION_OFFSET: The exp2 emulation offset. + :type EXP2_EMULATION_OFFSET: int + :param EXP2_EMULATION_COUNT: The exp2 emulation count. + :type EXP2_EMULATION_COUNT: int + :param CVT_COUNT: The cvt count. + :type CVT_COUNT: int + :param CVT_PER_STEP: The cvt per step. + :type CVT_PER_STEP: int + :param FMA_COUNT: The fma count. + :type FMA_COUNT: int + :param ARV_COUNT: The arv count. + :type ARV_COUNT: int + :return: The local row sum and the inplace consumer. + :rtype: Tuple[float, cute.Tensor] + """ + for i in cutlass.range_constexpr(0, EXP2_EMULATION_OFFSET, 2): + if cutlass.const_expr(i >= CVT_COUNT): + if cutlass.const_expr(i % CVT_PER_STEP == 0): + if cutlass.const_expr(self.v_dtype.width == 8): + fmha_utils.cvt_f32x4_to_f8x4( + tTMEM_LOADrS_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP], + tTMEM_STORErS_x4_e_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP], + ) + else: + s_vec = tTMEM_LOADrS_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP].load() + tTMEM_STORErS_x4_e_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP].store( + s_vec.to(self.v_dtype) + ) + local_row_sum = cute.arch.add_packed_f32x2( + local_row_sum, + ( + tTMEM_LOADrS[i - CVT_COUNT], + tTMEM_LOADrS[i - CVT_COUNT + 1], + ), + ) + tTMEM_LOADrS[i] = cute.math.exp2(tTMEM_LOADrS[i], fastmath=True) + if cutlass.const_expr(i + FMA_COUNT < EXP2_EMULATION_OFFSET): + ( + tTMEM_LOADrS[i + FMA_COUNT], + tTMEM_LOADrS[i + FMA_COUNT + 1], + ) = cute.arch.fma_packed_f32x2( + ( + tTMEM_LOADrS[i + FMA_COUNT], + tTMEM_LOADrS[i + FMA_COUNT + 1], + ), + (scale, scale), + (minus_row_max_scale, minus_row_max_scale), + ) + tTMEM_LOADrS[i + 1] = cute.math.exp2(tTMEM_LOADrS[i + 1], fastmath=True) + if cutlass.const_expr(i == EXP2_EMULATION_OFFSET - ARV_COUNT): + if cutlass.const_expr(self.enable_sequence_barrier): + if cutlass.const_expr(stage == 0): + self.sequence_s1_s0_barrier.arrive() + else: + self.sequence_s0_s1_barrier.arrive() + + # The remaining conversion steps + for i in cutlass.range_constexpr( + EXP2_EMULATION_OFFSET - CVT_COUNT, + EXP2_EMULATION_OFFSET, + 2, + ): + if cutlass.const_expr(i % CVT_PER_STEP == 0): + if cutlass.const_expr(self.v_dtype.width == 8): + fmha_utils.cvt_f32x4_to_f8x4( + tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], + ) + else: + s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) + local_row_sum = cute.arch.add_packed_f32x2( + local_row_sum, (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]) + ) + for i in cutlass.range_constexpr( + EXP2_EMULATION_OFFSET, EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT // 2, 2 + ): + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( + (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), + (scale, scale), + (minus_row_max_scale, minus_row_max_scale), + ) + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = fmha_utils.ex2_emulation_packed_f32x2( + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] + ) + if cutlass.const_expr((i + 2) % CVT_PER_STEP == 0): + if cutlass.const_expr(self.v_dtype.width == 8): + fmha_utils.cvt_f32x4_to_f8x4( + tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], + ) + else: + s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) + + inplace_peek_status = inplace_consumer.try_wait() + for i in cutlass.range_constexpr( + EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT // 2, + EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, + 2, + ): + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( + (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), + (scale, scale), + (minus_row_max_scale, minus_row_max_scale), + ) + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = fmha_utils.ex2_emulation_packed_f32x2( + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] + ) + if cutlass.const_expr((i + 2) % CVT_PER_STEP == 0): + if cutlass.const_expr(self.v_dtype.width == 8): + fmha_utils.cvt_f32x4_to_f8x4( + tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], + ) + else: + s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) + inplace_consumer.wait_and_advance(inplace_peek_status) + return local_row_sum, inplace_consumer + + @cute.jit + def softmax_step( + self, + stage: int, + whether_apply_mask: bool, + iter_args: Tuple, + stats_args: Tuple, + pipeline_args: Tuple, + value_args: Tuple, + atom_args: Tuple, + tensor_args: Tuple, + ) -> Tuple[Tuple, Tuple]: + """Perform a single step of the softmax computation on a block of attention scores. + + This method processes one block of the attention matrix, computing numerically stable + softmax by first finding the row maximum, subtracting it from all elements, applying + exponential function, and then normalizing by the sum of exponentials. It also handles + optional masking of attention scores. + + The method involves several key operations: + 1. Loading attention scores from tensor memory + 2. Applying optional masking based on position + 3. Computing row-wise maximum values for numerical stability + 4. Transforming scores using exp2(x*scale - max*scale) + 5. Computing row sums for normalization + 6. Coordinating pipeline synchronization between different processing stages + + :param stage: Processing stage (0 for first half, 1 for second half) + :type stage: int + :param whether_apply_mask: Whether to apply attention masking + :type whether_apply_mask: bool + :param iter_args: Tuple containing the counting tensor, row_max, row_sum, and vector buffer's handle for current iteration + :type iter_args: Tuple + :param stats_args: Tuple containing row_sum and row_max + :type stats_args: Tuple + :param pipeline_args: Tuple containing pipeline related arguments for MMA, correction, and sequence synchronization + :type pipeline_args: Tuple + :param value_args: Tuple containing seqlen_k, seqlen_q, and scale_softmax_log2 + :type value_args: Tuple + :param atom_args: Tuple containing mma & copy atoms + :type atom_args: Tuple + :param tensor_args: Tuple containing softmax related tensors + :type tensor_args: Tuple + :param fused_mask: Compute trip counts and apply masking for attention blocks + :type fused_mask: fmha_utils.FusedMask + :return: Updated stats_args and pipeline_args + :rtype: Tuple[Tuple, Tuple] + """ + row_sum, row_max = stats_args + cS, is_last_iter = iter_args + ( + seqlen_k, + seqlen_q, + scale_softmax_log2, + window_size_left, + window_size_right, + skip_softmax_threshold_log2, + thread_idx, + logical_offset, + ) = value_args + ( + si_peek_status, + mma_si_consumer, + si_corr_producer, + pi_mma_producer, + inplace_producer, + inplace_consumer, + ) = pipeline_args + ( + qk_thr_mma, + tiled_tmem_load, + tiled_tmem_store, + tiled_tmem_store_vec, + thr_tmem_load, + thr_tmem_store, + thr_tmem_store_vec, + ) = atom_args + ( + tTMEM_LOADtS, + tTMEM_STORE_VECtS, + tTMEM_STORE_SKIP_SOFTMAX, + tTMEM_STOREtS_x4, + warp_wants_skip_softmax_exchange, + skip_softmax_count, + total_softmax_count, + ) = tensor_args + tilePlikeFP32 = self.qk_mma_tiler[1] // Float32.width * self.o_dtype.width + tScS = qk_thr_mma.partition_C(cS) + enable_skip_softmax = skip_softmax_threshold_log2 is not None + tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) + tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) + tScS_P_layout = cute.composition(tScS.layout, cute.make_layout((128, tilePlikeFP32))) + tScS_P = cute.make_tensor(tScS.iterator, tScS_P_layout) + tTMEM_LOADcS = thr_tmem_load.partition_D(tScS) + tTMEM_STORE_VECcS = thr_tmem_store_vec.partition_S(tScS_vec) + tTMEM_STOREcS = thr_tmem_store.partition_S(tScS_P) + # Wait for Si + si_handle = mma_si_consumer.wait_and_advance(si_peek_status) + tTMEM_LOADrS = cute.make_rmem_tensor(tTMEM_LOADcS.shape, self.qk_acc_dtype) + old_row_max = row_max + skip_softmax = cutlass.Boolean(False) + if whether_apply_mask: + if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): + cute.copy(tiled_tmem_load, tTMEM_LOADtS, tTMEM_LOADrS) + else: + tTMEM_LOADrMax = cute.make_rmem_tensor( + cute.make_layout((1, cute.size(tTMEM_LOADrS, mode=[1]))), + self.qk_acc_dtype, + ) + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[1])): + cute.copy_atom_call( + tiled_tmem_load, + tTMEM_LOADtS[None, i, 0, 0], + (tTMEM_LOADrS[None, i, 0, 0], tTMEM_LOADrMax[None, i]), + ) + fmha_utils.FusedMask.apply_mask( + self.mask_type, + tTMEM_LOADrS, + tTMEM_LOADcS, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + tile_row_max = tTMEM_LOADrS.load().reduce(cute.ReductionOp.MAX, -cutlass.Float32.inf, 0) + if cutlass.const_expr(not enable_skip_softmax): + row_max = cute.arch.fmax(row_max, tile_row_max) + else: + skip_softmax, row_max = self.calculate_skip_softmax_flag( + row_max, + tile_row_max, + scale_softmax_log2, + skip_softmax_threshold_log2, + seqlen_q, + thread_idx, + logical_offset, + warp_wants_skip_softmax_exchange, + stage, + skip_softmax_count, + total_softmax_count, + ) + si_handle.release() + # S0 -> P1 / S1 -> P0 + inplace_producer.commit() + inplace_producer.advance() + else: + if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): + cute.copy( + tiled_tmem_load, + tTMEM_LOADtS[None, 0, None, None], + tTMEM_LOADrS[None, 0, None, None], + ) + cute.copy( + tiled_tmem_load, + tTMEM_LOADtS[None, 1, None, None], + tTMEM_LOADrS[None, 1, None, None], + ) + tile_row_max = -cutlass.Float32.inf + tile_row_max_ = tile_row_max + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 0, 0, 0]) + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 0, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 0, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 0, 0, 0]) + cute.copy( + tiled_tmem_load, + tTMEM_LOADtS[None, 2, None, None], + tTMEM_LOADrS[None, 2, None, None], + ) + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 1, 0, 0]) + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 1, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 1, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 1, 0, 0]) + cute.copy( + tiled_tmem_load, + tTMEM_LOADtS[None, 3, None, None], + tTMEM_LOADrS[None, 3, None, None], + ) + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 2, 0, 0]) + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 2, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 2, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 2, 0, 0]) + cute.arch.fence_view_async_tmem_store() + si_handle.release() + # S0 -> P1 / S1 -> P0 + inplace_producer.commit() + inplace_producer.advance() + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 3, 0, 0]) + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 3, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 3, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 3, 0, 0]) + tile_row_max = cute.arch.fmax(tile_row_max, tile_row_max_) + if cutlass.const_expr(not enable_skip_softmax): + row_max = cute.arch.fmax(tile_row_max, row_max) + else: + skip_softmax, row_max = self.calculate_skip_softmax_flag( + row_max, + tile_row_max, + scale_softmax_log2, + skip_softmax_threshold_log2, + seqlen_q, + thread_idx, + logical_offset, + warp_wants_skip_softmax_exchange, + stage, + skip_softmax_count, + total_softmax_count, + ) + else: + tTMEM_LOADrMax = cute.make_rmem_tensor( + cute.make_layout((1, cute.size(tTMEM_LOADrS, mode=[1]))), + self.qk_acc_dtype, + ) + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[1])): + cute.copy_atom_call( + tiled_tmem_load, + tTMEM_LOADtS[None, i, 0, 0], + (tTMEM_LOADrS[None, i, 0, 0], tTMEM_LOADrMax[None, i]), + ) + cute.arch.fence_view_async_tmem_store() + tile_row_max = tTMEM_LOADrMax.load().reduce( + cute.ReductionOp.MAX, -cutlass.Float32.inf, 0 + ) + if cutlass.const_expr(not enable_skip_softmax): + row_max = cute.arch.fmax(tile_row_max, row_max) + else: + skip_softmax, row_max = self.calculate_skip_softmax_flag( + row_max, + tile_row_max, + scale_softmax_log2, + skip_softmax_threshold_log2, + seqlen_q, + thread_idx, + logical_offset, + warp_wants_skip_softmax_exchange, + stage, + skip_softmax_count, + total_softmax_count, + ) + si_handle.release() + # S0 -> P1 / S1 -> P0 + inplace_producer.commit() + inplace_producer.advance() + + row_max_safe = row_max + if row_max == -cutlass.Float32.inf: + row_max_safe = 0.0 + if cutlass.const_expr(self.rescale_threshold > 0.0): + if (row_max_safe - old_row_max) * scale_softmax_log2 <= self.rescale_threshold: + row_max_safe = old_row_max + tTMEM_STORE_VECrS = cute.make_rmem_tensor(tTMEM_STORE_VECcS.shape, self.qk_acc_dtype) + tTMEM_STORE_VECrS[0] = old_row_max + tTMEM_STORE_VECrS[1] = row_max_safe + vec_i_peek_status = si_corr_producer.try_acquire() + tTMEM_STORErS_x4 = cute.make_rmem_tensor(tTMEM_STOREcS.shape, self.qk_acc_dtype) + tTMEM_STORErS_x4_e = cute.make_tensor( + cute.recast_ptr(tTMEM_STORErS_x4.iterator, dtype=self.v_dtype), + tTMEM_LOADrS.layout, + ) + scale = scale_softmax_log2 + minus_row_max_scale = (0.0 - row_max_safe) * scale + if cutlass.const_expr(self.v_dtype.width == 8 and self.p_fp8_prescale_log2 > 0): + minus_row_max_scale = minus_row_max_scale + self.p_fp8_prescale_log2 + + ARV_COUNT = 4 + FMA_COUNT = 8 + CVT_COUNT = 8 if self.v_dtype.width == 8 else 4 + CVT_PER_STEP = 4 if self.v_dtype.width == 8 else 2 + assert CVT_COUNT % CVT_PER_STEP == 0, ( + f"CVT_COUNT {CVT_COUNT} must be divisible by CVT_PER_STEP {CVT_PER_STEP}" + ) + tTMEM_LOADrS_cvt = cute.logical_divide(tTMEM_LOADrS, cute.make_layout(CVT_PER_STEP)) + tTMEM_STORErS_x4_e_cvt = cute.logical_divide( + tTMEM_STORErS_x4_e, cute.make_layout(CVT_PER_STEP) + ) + for i in cutlass.range_constexpr(0, FMA_COUNT, 2): + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( + (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), + (scale, scale), + (minus_row_max_scale, minus_row_max_scale), + ) + vec_i_handle = si_corr_producer.acquire_and_advance(vec_i_peek_status) + cute.copy(tiled_tmem_store_vec, tTMEM_STORE_VECrS, tTMEM_STORE_VECtS) + cute.arch.fence_view_async_tmem_store() + # Notify correction wg that row_max is ready + vec_i_handle.commit() + + EXP2_EMULATION_COUNT = 20 if self.enable_ex2_emulation and not whether_apply_mask else 0 + EXP2_EMULATION_OFFSET = cute.size(tTMEM_LOADrS) - EXP2_EMULATION_COUNT + acc_scale_ = scale * (old_row_max - row_max_safe) + acc_scale = cute.math.exp2(acc_scale_, fastmath=True) * 0.5 + if cutlass.const_expr(self.enable_sequence_barrier): + if cutlass.const_expr(stage == 0): + self.sequence_s0_s1_barrier.arrive_and_wait() + else: + self.sequence_s1_s0_barrier.arrive_and_wait() + + if cutlass.const_expr(enable_skip_softmax): + if not skip_softmax: + row_sum *= acc_scale + local_row_sum = (row_sum, row_sum) + local_row_sum, inplace_consumer = self.apply_exp_and_cvt( + tTMEM_LOADrS, + tTMEM_LOADrS_cvt, + tTMEM_STORErS_x4_e_cvt, + stage, + scale, + minus_row_max_scale, + local_row_sum, + inplace_consumer, + EXP2_EMULATION_OFFSET, + EXP2_EMULATION_COUNT, + CVT_COUNT, + CVT_PER_STEP, + FMA_COUNT, + ARV_COUNT, + ) + tTMEM_STORE_VECrS_i32 = cute.recast_tensor(tTMEM_STORE_VECrS, dtype=cutlass.Int32) + tTMEM_STORE_VECrS_i32[0] = 0 + pi_handle = pi_mma_producer.acquire_and_advance() + # store skip softmax flag + cute.copy( + tiled_tmem_store_vec, + tTMEM_STORE_VECrS_i32, + tTMEM_STORE_SKIP_SOFTMAX, + ) + # store P + cute.copy(tiled_tmem_store, tTMEM_STORErS_x4, tTMEM_STOREtS_x4) + cute.arch.fence_view_async_tmem_store() + pi_handle.commit() + for j in cutlass.range_constexpr( + EXP2_EMULATION_OFFSET, + EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, + 2, + ): + local_row_sum = cute.arch.add_packed_f32x2( + (tTMEM_LOADrS[j], tTMEM_LOADrS[j + 1]), + local_row_sum, + ) + row_sum = local_row_sum[0] + local_row_sum[1] + cute.arch.fence_view_async_tmem_store() + else: + if cutlass.const_expr(self.enable_sequence_barrier): + if cutlass.const_expr(stage == 0): + self.sequence_s1_s0_barrier.arrive() + else: + self.sequence_s0_s1_barrier.arrive() + inplace_peek_status = inplace_consumer.try_wait() + inplace_consumer.wait_and_advance(inplace_peek_status) + tTMEM_STORE_VECrS_i32 = cute.recast_tensor(tTMEM_STORE_VECrS, dtype=cutlass.Int32) + tTMEM_STORE_VECrS_i32[0] = 1 + pi_handle = pi_mma_producer.acquire_and_advance() + # store skip softmax flag + cute.copy( + tiled_tmem_store_vec, + tTMEM_STORE_VECrS_i32, + tTMEM_STORE_SKIP_SOFTMAX, + ) + cute.arch.fence_view_async_tmem_store() + pi_handle.commit() + else: + row_sum *= acc_scale + local_row_sum = (row_sum, row_sum) + local_row_sum, inplace_consumer = self.apply_exp_and_cvt( + tTMEM_LOADrS, + tTMEM_LOADrS_cvt, + tTMEM_STORErS_x4_e_cvt, + stage, + scale, + minus_row_max_scale, + local_row_sum, + inplace_consumer, + EXP2_EMULATION_OFFSET, + EXP2_EMULATION_COUNT, + CVT_COUNT, + CVT_PER_STEP, + FMA_COUNT, + ARV_COUNT, + ) + pi_handle = pi_mma_producer.acquire_and_advance() + # store P + cute.copy(tiled_tmem_store, tTMEM_STORErS_x4, tTMEM_STOREtS_x4) + cute.arch.fence_view_async_tmem_store() + for j in cutlass.range_constexpr( + EXP2_EMULATION_OFFSET, + EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, + 2, + ): + local_row_sum = cute.arch.add_packed_f32x2( + (tTMEM_LOADrS[j], tTMEM_LOADrS[j + 1]), + local_row_sum, + ) + row_sum = local_row_sum[0] + local_row_sum[1] + cute.arch.fence_view_async_tmem_store() + # Notify tensor core warp that softmax(S->P) is ready + pi_handle.commit() + if not is_last_iter: + si_peek_status = mma_si_consumer.try_wait() + + stats_args = (row_sum, row_max_safe) + pipeline_args = ( + si_peek_status, + mma_si_consumer, + si_corr_producer, + pi_mma_producer, + inplace_producer, + inplace_consumer, + ) + return stats_args, pipeline_args + + # For both softmax0 and softmax1 warp group + @cute.jit + def softmax( + self, + stage: int, + tensor_args: Tuple, + pipeline_args: Tuple, + inplace_args: Tuple, + qk_thr_mma: cute.ThrMma, + value_args: Tuple, + mask_args: Tuple, + sched_args: Tuple, + ): + """Compute softmax on attention scores from QK matrix multiplication. + + This method handles the softmax computation for either the first or second half of the + attention matrix, depending on the 'stage' parameter. It calculates row-wise maximum + and sum values needed for stable softmax computation, applies optional masking, and + transforms raw attention scores into probability distributions. + + The implementation uses specialized memory access patterns and efficient math operations + for computing exp(x) using exp2 functions. It also coordinates pipeline + synchronization between MMA, correction, and sequence processing stages. + + :param stage: Processing stage (0 for first half, 1 for second half of attention matrix) + :type stage: int + :param seqlen_k: Length of the key sequence + :type seqlen_k: Int32 + :param seqlen_q: Length of the query sequence + :type seqlen_q: Int32 + :param cum_seqlen_q: Cumulative sequence lengths for queries + :type cum_seqlen_q: cute.Tensor | None + :param cum_seqlen_k: Cumulative sequence lengths for keys + :type cum_seqlen_k: cute.Tensor | None + :param scale_softmax_log2: Log2 scale factor for softmax operation + :type scale_softmax_log2: Float32 + :param qk_thr_mma: Thread MMA operation for QK matrix multiplication + :type qk_thr_mma: cute.ThrMma + :param tStS: Shared tensor for softmax input/output + :type tStS: cute.Tensor + :param tStSi: Input tensor containing attention scores + :type tStSi: cute.Tensor + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + :param mma_si_consumer: Pipeline for synchronizing with Si tensors + :type mma_si_consumer: pipeline.PipelineConsumer + :param si_corr_producer: Pipeline for synchronizing with correction operations + :type si_corr_producer: pipeline.PipelineProducer + :param pi_mma_producer: Pipeline for synchronizing with Pi tensors + :type pi_mma_producer: pipeline.PipelineProducer + :param tile_sched_params: Parameters for tile scheduling + :type tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams + :param fused_mask: Compute trip counts and apply masking for attention blocks + :type fused_mask: fmha_utils.FusedMask + """ + ( + tStS, + tStSi, + cum_seqlen_k, + cum_seqlen_q, + warp_wants_skip_softmax_exchange, + skip_softmax_count, + total_softmax_count, + ) = tensor_args + mma_si_consumer, si_corr_producer, pi_mma_producer = pipeline_args + inplace_producer, inplace_consumer = inplace_args + ( + seqlen_k, + seqlen_q, + scale_softmax_log2, + skip_softmax_threshold_log2, + ) = value_args + window_size_left, window_size_right = mask_args + tile_sched, work_tile = sched_args + + tidx, _, _ = cute.arch.thread_idx() + thread_idx = tidx % (self.threads_per_warp * len(self.softmax0_warp_ids)) + + cS_base = cute.make_identity_tensor((self.qk_mma_tiler[0], self.qk_mma_tiler[1])) + tilePlikeFP32 = self.qk_mma_tiler[1] // 32 * self.o_dtype.width + tScS = qk_thr_mma.partition_C(cS_base) + tStS_vec_layout = cute.composition(tStS.layout, cute.make_layout((128, 2))) + tmem_vec_offset = self.tmem_vec0_offset if stage == 0 else self.tmem_vec1_offset + tStS_vec = cute.make_tensor(tStS.iterator + tmem_vec_offset, tStS_vec_layout) + tmem_skip_softmax_offset = ( + self.tmem_skip_softmax0_offset if stage == 0 else self.tmem_skip_softmax1_offset + ) + tStS_skip_softmax = cute.make_tensor( + tStS.iterator + tmem_skip_softmax_offset, tStS_vec_layout + ) + tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) + tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) + tStS_P_layout = cute.composition(tStS.layout, cute.make_layout((128, tilePlikeFP32))) + tmem_p_offset = self.tmem_p0_offset if stage == 0 else self.tmem_p1_offset + tStS_P = cute.make_tensor(tStS.iterator + tmem_p_offset, tStS_P_layout) + if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): + tmem_load_atom = cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), + self.qk_acc_dtype, + ) + else: + tmem_load_atom = cute.make_copy_atom( + tcgen05.copy.LdRed32x32bOp( + tcgen05.copy.Repetition(32), redOp=tcgen05.TmemLoadRedOp.MAX + ), + self.qk_acc_dtype, + ) + + tiled_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tStSi) + thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) + tTMEM_LOADtS = thr_tmem_load.partition_S(tStSi) + tmem_store_vec_atom = cute.make_copy_atom( + tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(2)), + self.qk_acc_dtype, + ) + tiled_tmem_store_vec = tcgen05.make_tmem_copy(tmem_store_vec_atom, tStS_vec) + thr_tmem_store_vec = tiled_tmem_store_vec.get_slice(thread_idx) + tTMEM_STORE_VECtS = thr_tmem_store_vec.partition_D(tStS_vec) + tTMEM_STORE_VECcS = thr_tmem_store_vec.partition_S(tScS_vec) + tTMEM_STORE_SKIP_SOFTMAX = thr_tmem_store_vec.partition_D(tStS_skip_softmax) + tmem_store_atom = cute.make_copy_atom( + tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(32)), + self.qk_acc_dtype, + ) + tiled_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tStS_P) + thr_tmem_store = tiled_tmem_store.get_slice(thread_idx) + tTMEM_STOREtS_x4 = thr_tmem_store.partition_D(tStS_P) + if cutlass.const_expr(self.enable_sequence_barrier): + if cutlass.const_expr(stage == 1): + self.sequence_s0_s1_barrier.arrive() + while work_tile.is_valid_tile: + curr_block_coord = work_tile.tile_idx + batch_coord = curr_block_coord[2][1] + seqlen_k_ = seqlen_k + seqlen_q_ = seqlen_q + continue_cond = False + cuseqlen_q = Int32(0) + seqlen_q_ = seqlen_q + if cutlass.const_expr(cum_seqlen_q is not None): + cuseqlen_q = cum_seqlen_q[batch_coord] + seqlen_q_ = cum_seqlen_q[batch_coord + 1] - cuseqlen_q + continue_cond = ( + not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( + self.cta_tiler[0], + curr_block_coord[0], + seqlen_q_, + ) + ) + if not continue_cond: + if cutlass.const_expr(cum_seqlen_k is not None): + cuseqlen_k = cum_seqlen_k[batch_coord] + seqlen_k_ = cum_seqlen_k[batch_coord + 1] - cuseqlen_k + continue_cond = seqlen_k_ <= 0 + if not continue_cond: + logical_offset = ( + curr_block_coord[0] * self.cta_tiler[0] + stage * self.qk_mma_tiler[0], + 0, + ) + cS = cute.domain_offset(logical_offset, cS_base) + value_args_ = ( + seqlen_k_, + seqlen_q_, + scale_softmax_log2, + window_size_left, + window_size_right, + skip_softmax_threshold_log2, + thread_idx, + logical_offset, + ) + atom_args = ( + qk_thr_mma, + tiled_tmem_load, + tiled_tmem_store, + tiled_tmem_store_vec, + thr_tmem_load, + thr_tmem_store, + thr_tmem_store_vec, + ) + tensor_args_ = ( + tTMEM_LOADtS, + tTMEM_STORE_VECtS, + tTMEM_STORE_SKIP_SOFTMAX, + tTMEM_STOREtS_x4, + warp_wants_skip_softmax_exchange, + skip_softmax_count, + total_softmax_count, + ) + st_cnt, end_cnt, ld_mask_cnt, unmask_cnt, tl_mask_cnt = ( + fmha_utils.FusedMask.get_masked_info( + self.mask_type, + curr_block_coord, + self.cta_tiler, + seqlen_q_, + seqlen_k_, + window_size_left, + window_size_right, + ) + ) + row_max = -Float32.inf + row_sum = 0.0 + stats_args = (row_sum, row_max) + + def softmax_loop( + whether_apply_mask: bool, + loop_args: Tuple, + stats_args: Tuple, + pipeline_args: Tuple, + inner_fn: Callable, + value_args: Tuple, + atom_args: Tuple, + tensor_args: Tuple, + cS: cute.Tensor, + ) -> Tuple[Tuple, Tuple]: + start_index, iter_num, upper_bound = loop_args + for i in cutlass.range(start_index, start_index + iter_num, 1, unroll=1): + cS_iter = cute.domain_offset((0, i * self.qk_mma_tiler[1]), cS) + iter_args = (cS_iter, i == upper_bound - 1) + stats_args, pipeline_args = inner_fn( + stage, + whether_apply_mask, + iter_args, + stats_args, + pipeline_args, + value_args, + atom_args, + tensor_args, + ) + return stats_args, pipeline_args + + softmax_step_fn = self.softmax_step + softmax_loop_fn = partial( + softmax_loop, + inner_fn=softmax_step_fn, + value_args=value_args_, + atom_args=atom_args, + tensor_args=tensor_args_, + cS=cS, + ) + si_peek_status = mma_si_consumer.try_wait() + if cutlass.const_expr(stage == 1): + inplace_consumer.wait_and_advance() + pipeline_args_ = ( + si_peek_status, + mma_si_consumer, + si_corr_producer, + pi_mma_producer, + inplace_producer, + inplace_consumer, + ) + # 1. Leading mask loop + loop_args = (st_cnt, ld_mask_cnt, end_cnt) + stats_args, pipeline_args_ = softmax_loop_fn( + True, loop_args, stats_args, pipeline_args_ + ) + # 2. Unmasked loop + loop_args = (st_cnt + ld_mask_cnt, unmask_cnt, end_cnt) + stats_args, pipeline_args_ = softmax_loop_fn( + False, loop_args, stats_args, pipeline_args_ + ) + # 3. Trailing mask loop + loop_args = (st_cnt + ld_mask_cnt + unmask_cnt, tl_mask_cnt, end_cnt) + stats_args, pipeline_args_ = softmax_loop_fn( + True, loop_args, stats_args, pipeline_args_ + ) + + # Unpack pipeline_args + ( + _, + mma_si_consumer, + si_corr_producer, + pi_mma_producer, + inplace_producer, + inplace_consumer, + ) = pipeline_args_ + if cutlass.const_expr(stage == 0): + inplace_producer.commit() + inplace_producer.advance() + # 4. Copy the final stats for correction epilog + tTMEM_STORE_VECrS = cute.make_rmem_tensor( + tTMEM_STORE_VECcS.shape, self.qk_acc_dtype + ) + tTMEM_STORE_VECrS[0] = stats_args[0] + tTMEM_STORE_VECrS[1] = stats_args[1] + vec_i_handle = si_corr_producer.acquire_and_advance() + cute.copy(tiled_tmem_store_vec, tTMEM_STORE_VECrS, tTMEM_STORE_VECtS) + cute.arch.fence_view_async_tmem_store() + vec_i_handle.commit() + # End of if not continue_cond + # Advance to next tile + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + # End of persistent scheduler loop + + @cute.jit + def correction_rescale( + self, + thr_mma: cute.ThrMma, + tiled_tmem_load_vec: cute.TiledCopy, + scale_softmax_log2: Float32, + tensor_args: Tuple, + pipeline_args: Tuple, + oi_peek_status=None, + ): + """Rescale intermediate attention results based on softmax normalization factor. + + This method performs a crucial correction step in the attention computation pipeline. + When processing attention in blocks, the softmax normalization factors may change + as new blocks are processed. This method rescales previously computed partial + output values to account for updated normalization factors. + + The implementation uses efficient tensor memory operations to: + 1. Load existing partial attention output from tensor memory + 2. Apply the scaling factor to all elements + 3. Store the rescaled results back to tensor memory + + When ``self.enable_correction_double_buffer`` is True, the rescale loop + runs as a 2-buffer tensor-memory load / multiply / store pipeline. + + :param thr_mma: Thread MMA operation for the computation + :type thr_mma: cute.ThrMma + :param tiled_tmem_load_vec: Tiled memory load operation for the vectorized row-wise max + :type tiled_tmem_load_vec: cute.TiledCopy + :param scale_softmax_log2: Log2 of the softmax factor + :type scale_softmax_log2: Float32 + :param tensor_args: Tuple containing the tensors for the correction + :type tensor_args: Tuple[cute.Tensor, cute.Tensor, cute.Tensor] + :param pipeline_args: Tuple containing the pipeline arguments for the correction + :type pipeline_args: Tuple[pipeline.PipelineConsumer, pipeline.PipelineConsumer] + :param oi_peek_status: Optional non-blocking token for the Oi consumer + wait. ``None`` or ``False`` falls back to a blocking wait. + :return: ``((si_corr_consumer, mma_corr_consumer), next_oi_peek_status)`` + where ``next_oi_peek_status`` is the peek for the next Oi (only + refreshed when a rescale actually ran; otherwise stays + ``cutlass.Boolean(False)``). + """ + tOtO, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECcS = tensor_args + si_corr_consumer, mma_corr_consumer = pipeline_args + + pv_tiled_mma_shape = ( + self.pv_mma_tiler[0], + self.pv_mma_tiler[1], + ) + cO = cute.make_identity_tensor(pv_tiled_mma_shape) + tOcO = thr_mma.partition_C(cO) + corr_tile_size = 16 # tuneable parameter + tmem_load_atom = cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(corr_tile_size)), + self.pv_acc_dtype, + ) + tmem_store_atom = cute.make_copy_atom( + tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(corr_tile_size)), + self.pv_acc_dtype, + ) + tOtO_i_layout = cute.composition(tOtO.layout, cute.make_layout((128, corr_tile_size))) + tOcO_i_layout = cute.composition(tOcO.layout, cute.make_layout((128, corr_tile_size))) + tOtO_i = cute.make_tensor(tOtO.iterator, tOtO_i_layout) + tOcO_i = cute.make_tensor(tOcO.iterator, tOcO_i_layout) + tiled_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tOtO_i) + tiled_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tOtO_i) + tidx, _, _ = cute.arch.thread_idx() + thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) + thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) + thr_tmem_store = tiled_tmem_store.get_slice(thread_idx) + tTMEM_LOADtO = thr_tmem_load.partition_S(tOtO_i) + tTMEM_LOADcO = thr_tmem_load.partition_D(tOcO_i) + tTMEM_STOREtO = thr_tmem_store.partition_D(tOtO_i) + num_tiles = self.cta_tiler[2] // corr_tile_size + if cutlass.const_expr(self.enable_correction_double_buffer): + # Double buffer: 2 register buffers to pipeline tensor-memory load / multiply / store. + tTMrO = cute.make_rmem_tensor((tTMEM_LOADcO.shape, 2), self.pv_acc_dtype) + # Rank-matching views for cute.copy (TMEM partitions are rank-3, + # raw tTMrO[None, idx] is rank-1; composition restores the rank). + copy_layout = cute.make_layout(tTMrO.shape[0]) + view_0 = tTMrO[None, 0] + view_1 = tTMrO[None, 1] + tTMrO_copy = ( + cute.make_tensor( + view_0.iterator, + cute.composition(view_0.layout, copy_layout), + ), + cute.make_tensor( + view_1.iterator, + cute.composition(view_1.layout, copy_layout), + ), + ) + else: + tTMrO = cute.make_rmem_tensor((tTMEM_LOADcO.shape, num_tiles), self.pv_acc_dtype) + tTMEM_LOAD_VECrS = cute.make_rmem_tensor(tTMEM_LOAD_VECcS.shape, self.qk_acc_dtype) + # Wait for vec_i (row_wise current max & previous max) + vec_i_handle = si_corr_consumer.wait_and_advance() + cute.copy(tiled_tmem_load_vec, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECrS) + cute.arch.fence_view_async_tmem_load() + vec_i_handle.release() + # Wait for Oi (peek-token aware: None/False falls back to blocking). + oi_handle = mma_corr_consumer.wait_and_advance(oi_peek_status) + next_oi_peek_status = cutlass.Boolean(False) + vote_ballot_cnt = cute.arch.vote_ballot_sync(tTMEM_LOAD_VECrS[0] != tTMEM_LOAD_VECrS[1]) + should_rescale = vote_ballot_cnt != 0 + if should_rescale: + scale_ = scale_softmax_log2 * (tTMEM_LOAD_VECrS[0] - tTMEM_LOAD_VECrS[1]) + scale = cute.math.exp2(scale_, fastmath=True) + if cutlass.const_expr(self.enable_correction_double_buffer): + num_elems = cute.size(tTMrO, mode=[0]) + # Prologue: load first tile into buffer 0 + cute.copy(tiled_tmem_load, tTMEM_LOADtO, tTMrO_copy[0]) + # Steady state. Refresh the next Oi probe near the end of the + # current rescale pipeline. + for i in cutlass.range_constexpr(1, num_tiles): + cute.copy( + tiled_tmem_load, + cute.make_tensor( + tTMEM_LOADtO.iterator + i * corr_tile_size, + tTMEM_LOADtO.layout, + ), + tTMrO_copy[i % 2], + ) + for j in range(0, num_elems, 2): + tTMrO[j, (i - 1) % 2], tTMrO[j + 1, (i - 1) % 2] = ( + cute.arch.mul_packed_f32x2( + ( + tTMrO[j, (i - 1) % 2], + tTMrO[j + 1, (i - 1) % 2], + ), + (scale, scale), + ) + ) + cute.copy( + tiled_tmem_store, + tTMrO_copy[(i - 1) % 2], + cute.make_tensor( + tTMEM_STOREtO.iterator + (i - 1) * corr_tile_size, + tTMEM_STOREtO.layout, + ), + ) + next_oi_peek_status = mma_corr_consumer.try_wait() + # Epilogue: compute and store last tile + last = (num_tiles - 1) % 2 + for j in range(0, num_elems, 2): + tTMrO[j, last], tTMrO[j + 1, last] = cute.arch.mul_packed_f32x2( + (tTMrO[j, last], tTMrO[j + 1, last]), + (scale, scale), + ) + cute.copy( + tiled_tmem_store, + tTMrO_copy[last], + cute.make_tensor( + tTMEM_STOREtO.iterator + (num_tiles - 1) * corr_tile_size, + tTMEM_STOREtO.layout, + ), + ) + else: + for i in cutlass.range_constexpr(0, num_tiles): + tTMrO_i_ = tTMrO[None, i] + tTMrO_i = cute.make_tensor( + tTMrO_i_.iterator, + cute.composition(tTMrO_i_.layout, cute.make_layout(tTMrO.shape[0])), + ) + cute.copy( + tiled_tmem_load, + cute.make_tensor( + tTMEM_LOADtO.iterator + i * corr_tile_size, + tTMEM_LOADtO.layout, + ), + tTMrO_i, + ) + for j in range(0, cute.size(tTMrO_i), 2): + tTMrO_i[j], tTMrO_i[j + 1] = cute.arch.mul_packed_f32x2( + (tTMrO_i[j], tTMrO_i[j + 1]), + (scale, scale), + ) + cute.copy( + tiled_tmem_store, + tTMrO_i, + cute.make_tensor( + tTMEM_STOREtO.iterator + i * corr_tile_size, + tTMEM_STOREtO.layout, + ), + ) + next_oi_peek_status = mma_corr_consumer.try_wait() + # Release Oi + cute.arch.fence_view_async_tmem_store() + oi_handle.release() + return (si_corr_consumer, mma_corr_consumer), next_oi_peek_status + + @cute.jit + def correction_epilog( + self, + thr_mma: cute.ThrMma, + tiled_tmem_load_vec: cute.TiledCopy, + tensor_args: Tuple, + pipeline_args: Tuple, + value_args: Tuple, + ): + """Apply final scaling and transformation to attention output. + + When use_tma_store=True: writes to shared memory and signals epilogue warp for TMA store. + When use_tma_store=False: writes directly to global memory via st.global. + + :param thr_mma: Thread MMA operation for the computation + :type thr_mma: cute.ThrMma + :param tiled_tmem_load_vec: Tiled memory load operation for the vectorized row-wise max + :type tiled_tmem_load_vec: cute.TiledCopy + :param tensor_args: Tuple containing (tOtO, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECcS, sO_or_gO, mLSE, mSink) + :type tensor_args: Tuple + :param pipeline_args: When use_tma_store: (si_corr_consumer, mma_corr_consumer, corr_epi_producer). + When not use_tma_store: (si_corr_consumer, mma_corr_consumer). + :type pipeline_args: Tuple + :param value_args: Tuple containing (row_idx, cuseqlen_q, seqlen_q, blk_coord, scale_softmax, scale_output) + :type value_args: Tuple + """ + tOtO, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECcS, dest_O, mLSE, mSink = tensor_args + row_idx, cuseqlen_q, seqlen_q, blk_coord, scale_softmax, scale_output = value_args + + pv_tiled_mma_shape = ( + self.pv_mma_tiler[0], + self.pv_mma_tiler[1], + ) + cO = cute.make_identity_tensor(pv_tiled_mma_shape) + + corr_tile_size = 32 * 8 // self.o_dtype.width + tOdO = thr_mma.partition_C(dest_O) + tOcO = thr_mma.partition_C(cO) + tOtO_i = cute.logical_divide(tOtO, cute.make_layout((128, corr_tile_size))) + tOcO_i = cute.logical_divide(tOcO, cute.make_layout((128, corr_tile_size))) + tOdO_i = cute.logical_divide(tOdO, cute.make_layout((128, corr_tile_size))) + + tidx, _, _ = cute.arch.thread_idx() + thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) + epi_subtile = (self.epi_tile[0], corr_tile_size) + tmem_copy_atom = sm100_utils.get_tmem_load_op( + self.pv_mma_tiler, + self.o_layout, + self.o_dtype, + self.pv_acc_dtype, + epi_subtile, + use_2cta_instrs=False, + ) + tiled_tmem_load = tcgen05.make_tmem_copy(tmem_copy_atom, tOtO_i[(None, None), 0]) + thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) + tTMEM_LOADtO = thr_tmem_load.partition_S(tOtO_i[(None, None), None]) + tTMEM_LOADdO = thr_tmem_load.partition_D(tOdO_i[(None, None), None]) + tTMEM_LOADoO = thr_tmem_load.partition_D(tOcO_i[(None, None), None]) + + if cutlass.const_expr(self.use_tma_store): + si_corr_consumer, mma_corr_consumer, corr_epi_producer = pipeline_args + smem_copy_atom = sm100_utils.get_smem_store_op( + self.o_layout, self.o_dtype, self.pv_acc_dtype, tiled_tmem_load + ) + tiled_smem_store = cute.make_tiled_copy_D(smem_copy_atom, tiled_tmem_load) + else: + si_corr_consumer, mma_corr_consumer = pipeline_args + + # Wait for vec_i (row_wise global sum) + vec_i_handle = si_corr_consumer.wait_and_advance() + tTMEM_LOAD_VECrS = cute.make_rmem_tensor(tTMEM_LOAD_VECcS.shape, self.qk_acc_dtype) + cute.copy(tiled_tmem_load_vec, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECrS) + cute.arch.fence_view_async_tmem_load() + vec_i_handle.release() + + # Wait for Oi + oi_handle = mma_corr_consumer.wait_and_advance() + if cutlass.const_expr(self.use_tma_store): + oi_final_handle = corr_epi_producer.acquire_and_advance() + row_sum = tTMEM_LOAD_VECrS[0] + if cutlass.const_expr(mSink is not None): + sink_val = mSink[blk_coord[2]] + row_max_raw = tTMEM_LOAD_VECrS[1] + # sink is already in scaled logit space, row_max_raw is unscaled + # exp2((sink - max_scaled) * log2(e)) = exp(sink - max_scaled) + log2_e = Float32(1.4426950408889634) + sink_exp = cute.math.exp2( + (sink_val - row_max_raw * scale_softmax) * log2_e, fastmath=True + ) + row_sum = row_sum + sink_exp + scale = scale_output / row_sum + + for i in range(self.cta_tiler[2] // corr_tile_size): + tTMEM_LOADtO_i = tTMEM_LOADtO[None, 0, 0, i] + tTMEM_LOADdO_i = tTMEM_LOADdO[None, 0, 0, i] + tTMrO = cute.make_rmem_tensor(tTMEM_LOADoO[None, 0, 0, i].shape, self.pv_acc_dtype) + cute.copy(tiled_tmem_load, tTMEM_LOADtO_i, tTMrO) + for j in range(0, cute.size(tTMrO), 2): + tTMrO[j], tTMrO[j + 1] = cute.arch.mul_packed_f32x2( + (tTMrO[j], tTMrO[j + 1]), + (scale, scale), + ) + tDMrO = cute.make_rmem_tensor(tTMrO.shape, self.o_dtype) + o_vec = tTMrO.load() + tDMrO.store(o_vec.to(self.o_dtype)) + if cutlass.const_expr(self.use_tma_store): + # TMA store path: write to shared memory + cute.copy(tiled_smem_store, tDMrO, tTMEM_LOADdO_i) + else: + # st.global path: write directly to global memory with bounds check + if row_idx < seqlen_q: + cute.autovec_copy(tDMrO, tTMEM_LOADdO_i) + + if cutlass.const_expr(mLSE is not None): + scaled_tmp = scale_softmax * tTMEM_LOAD_VECrS[1] + # Convert LSE from natural log to log2 space, consistent with flashinfer trtllm-gen backend + lse = (cute.math.log(row_sum, fastmath=True) + scaled_tmp) * Float32(1.4426950408889634) + # Pre-scale correction: row_sum was inflated by 2^offset, so the + # log2-space LSE is too large by exactly `p_fp8_prescale_log2`. + if cutlass.const_expr(self.v_dtype.width == 8 and self.p_fp8_prescale_log2 > 0): + lse = lse - self.p_fp8_prescale_log2 + if row_idx < seqlen_q: + mLSE[row_idx + cuseqlen_q, blk_coord[2]] = lse + if cutlass.const_expr(self.use_tma_store): + # fence view async shared + cute.arch.fence_view_async_shared() + oi_handle.release() + oi_final_handle.commit() + return (si_corr_consumer, mma_corr_consumer, corr_epi_producer) + else: + oi_handle.release() + return (si_corr_consumer, mma_corr_consumer) + + def check_supported_dtypes( + self, + qk_dtype: Type[cutlass.Numeric], + pv_dtype: Type[cutlass.Numeric], + out_dtype: Type[cutlass.Numeric], + qk_acc_dtype: Type[cutlass.Numeric], + pv_acc_dtype: Type[cutlass.Numeric], + ): + supported = {cutlass.Float8E4M3FN, cutlass.Float16, cutlass.BFloat16} + if qk_dtype not in supported: + raise NotImplementedError("Unsupported qk_dtype") + if pv_dtype not in supported: + raise NotImplementedError("Unsupported pv_dtype") + if out_dtype not in {cutlass.Float8E4M3FN, cutlass.Float16, cutlass.BFloat16}: + raise NotImplementedError("Unsupported out_dtype") + if qk_acc_dtype not in {cutlass.Float32}: + raise NotImplementedError("Unsupported qk_acc_dtype") + if pv_acc_dtype not in {cutlass.Float32}: + raise NotImplementedError("Unsupported pv_acc_dtype") + + def check_invalid_shape( + self, + qk_dtype: Type[cutlass.Numeric], + q_shape: Tuple[int, int, int, int], + k_shape: Tuple[int, int, int, int], + ): + # Shapes are passed around this example as (batch, seq_len, num_heads, head_dim). + b, s_q, h_q, d = q_shape + b_, s_k, h_k, d_ = k_shape + + if b != b_: + raise NotImplementedError("q & k must have the same batch size") + if d != d_: + raise NotImplementedError("q & k must have the same head dimension") + if d not in {32, 64, 128, 192}: + raise NotImplementedError("Unsupported head dimension") + if h_q % h_k != 0: + raise NotImplementedError("h_q must be divisible by h_k") + if isinstance(s_q, tuple) and len(s_q) != b: + raise NotImplementedError("variable_seqlen s_q must have the length of batch size") + if isinstance(s_k, tuple) and len(s_k) != b: + raise NotImplementedError("variable_seqlen s_k must have the length of batch size") + if d == 192 and qk_dtype not in {cutlass.Float8E4M3FN}: + raise NotImplementedError("unimplemented dtypes for headdim 192") + + def can_implement( + self, + q_shape: Tuple[int, int, int, int], + k_shape: Tuple[int, int, int, int], + qk_dtype: Type[cutlass.Numeric], + pv_dtype: Type[cutlass.Numeric], + out_dtype: Type[cutlass.Numeric], + qk_acc_dtype: Type[cutlass.Numeric], + pv_acc_dtype: Type[cutlass.Numeric], + ) -> bool: + """ + :param q_shape: Shape of the query tensor. + :type q_shape: Tuple[int, int, int, int] + :param k_shape: Shape of the key tensor. + :type k_shape: Tuple[int, int, int, int] + :param qk_dtype: Data type for Q and K (Bmm1 inputs). + :type qk_dtype: Type[cutlass.Numeric] + :param pv_dtype: Data type for P and V (Bmm2 inputs). + :type pv_dtype: Type[cutlass.Numeric] + :param out_dtype: Data type of the output tensor. + :type out_dtype: Type[cutlass.Numeric] + :param qk_acc_dtype: Data type of the qk accumulator tensor. + :type qk_acc_dtype: Type[cutlass.Numeric] + :param pv_acc_dtype: Data type of the pv accumulator tensor. + :type pv_acc_dtype: Type[cutlass.Numeric] + :return: True if the kernel can be implemented, False otherwise. + :rtype: bool + """ + try: + # Skip unsupported types + self.check_supported_dtypes( + qk_dtype, + pv_dtype, + out_dtype, + qk_acc_dtype, + pv_acc_dtype, + ) + # Skip invalid shape + self.check_invalid_shape( + qk_dtype, + q_shape, + k_shape, + ) + except NotImplementedError: + return False + return True diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha_blockscaled.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha_blockscaled.py new file mode 100644 index 000000000000..01be74595ac9 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha_blockscaled.py @@ -0,0 +1,3768 @@ +# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ruff: noqa: I001, E501, F841, E712, E741 + +import math +from typing import Callable, Type, Tuple, Union, Optional +from functools import partial + +import torch +import cuda.bindings.driver as cuda + +import cutlass +import cutlass.cute as cute +from cutlass.cute.nvgpu import tcgen05 +from cutlass.cute.nvgpu.common import OperandMajorMode +import cutlass.utils as utils +import cutlass.pipeline as pipeline +from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait +import cutlass.torch as cutlass_torch +import cutlass.utils.blackwell_helpers as sm100_utils +import cutlass.utils.blockscaled_layout as blockscaled_utils +from cutlass.cute.typing import Int8, Int32, Int64, Float32 +from cutlass.base_dsl.arch import Arch +from cutlass.cutlass_dsl import BaseDSL + +from ...helpers import fmha_helpers as fmha_utils + +""" +A fused multi-head attention (FMHA) example where Bmm1(Q@K) is block-scaled with MXFP8 or NVFP4 for +NVIDIA Blackwell SM100 architecture using CUTE DSL + +This example demonstrates an implementation of fused multi-head attention using a TMA + Blackwell SM100 +TensorCore warp-specialized persistent kernel. The implementation integrates the Q*K^T matrix multiplication, +softmax normalization, and softmax(Q*K^T)*V into a single kernel, avoiding intermediate data movement between +global memory and shared memory, thus improving computational efficiency. + +The kernel implements key optimizations including: +- Warp specialization for different computation phases (load, MMA, softmax, correction, epilogue) +- Pipeline stages between different warps for overlapping computation and memory access +- Support for different precision data types +- Optional causal masking for autoregressive models + +To run this example: + +.. code-block:: bash + + python nvidia-internal/fmha_blockscaled.py \ + --qk_mode MXFP8 \ + --qk_acc_dtype Float32 --pv_acc_dtype Float32 \ + --mma_tiler_mn 128,128 \ + --q_shape 4,1024,8,64 --k_shape 4,1024,8,64 \ + --is_persistent + +The above example runs FMHA with batch size 4, sequence length 1024, 8 attention heads, and head +dimension 64. The Blackwell tcgen05 MMA tile shape is (128, 128), and the default runner uses BF16 +for V/output with FP32 for accumulation. + +To collect performance with NCU profiler: + +.. code-block:: bash + + ncu python nvidia-internal/fmha_blockscaled.py \ + --qk_mode NVFP4 \ + --qk_acc_dtype Float32 --pv_acc_dtype Float32 \ + --mma_tiler_mn 128,128 \ + --q_shape 4,1024,8,64 --k_shape 4,1024,8,64 \ + --is_persistent --warmup_iterations 10 \ + --iterations 10 --skip_ref_check + +Constraints for this example: +* Q and K use supports two microscaling schema: + MXFP8 (qk_dtype in {Float8E4M3FN, Float8E5M2}, qk_sf_dtype=Float8E8M0FNU, qk_sf_vec_size=32) + and NVFP4 (qk_dtype=Float4E2M1FN, qk_sf_dtype=Float8E4M3FN, qk_sf_vec_size=16) +* Bmm2 (P@V) remains dense FP8 or BF16 through pv_dtype +* Supported QK head dimensions: 128 (due to current limitations in blockscaled_utils) +* Number of heads in Q must be divisible by number of heads in K +* mma_tiler_mn must be 128,128 +* Batch size must be the same for Q, K, and V tensors +* For causal masking, use --is_causal (note: specify without =True/False) +* For persistent scheduling, use --is_persistent (note: specify without =True/False) + +For details on the skip softmax algorithm, please refer to the paper: https://arxiv.org/abs/2512.12087. +""" + + +def make_thread_cooperative_group(size: int): + return pipeline.CooperativeGroup(pipeline.Agent.Thread, size) + + +def create_scale_factor_tensor( + mn: int, + k: int, + l: int, + sf_vec_size: int, + sf_dtype: Type[cutlass.Numeric], +): + """ + Create dense reference SFs and blocked device SF storage for QK block-scaled MMA. + + Generates per-position SFs sampled from {0.5, 1.0, 2.0} (which are exactly representable in both + E8M0 and E4M3) and arranges them into the blocked storage that tile_atom_to_shape_SF, + BlockScaledBasicChunk consume on the kernel side. The SF atom layout is: + ((32,4),(sf_vec,4)) : ((16,4),(0,1)) + so for a logical mn in [0,128) inside an atom, the SF byte lives at offset: + (mn % 32) * 16 + (mn // 32) * 4 + k_inner + """ + + def ceil_div(a: int, b: int) -> int: + return (a + b - 1) // b + + atom_m = 32 * 4 + atom_k = 4 + m_atoms = ceil_div(mn, atom_m) + sf_k = ceil_div(k, sf_vec_size) + sf_k_atoms = ceil_div(sf_k, atom_k) + + # Generate SFs in the LOGICAL (mn, sf_k, l) layout first, then re-arrange into the blocked + # storage layout explained in docstring above. + # CuTe decomposes mn in [0,128) column-major as (mn % 32, mn // 32), so the size-32 axis + # gets the LOW 5 bits of mn (stride 16) and the size-4 axis gets the HIGH 2 bits (stride 4). + mn_padded = m_atoms * atom_m + sf_k_padded = sf_k_atoms * atom_k + sf_choices = torch.tensor([0.5, 1.0, 2.0], dtype=torch.float32) + sf_logical_padded = sf_choices[torch.randint(0, len(sf_choices), (mn_padded, sf_k_padded, l))] + sf_storage_f32 = ( + # K-mode: k_padded -> (sf_k_atoms, atom_k=4) + # MN-mode: mn_padded -> (sf_m_atoms, proton_m=4, neutron_m=32) + sf_logical_padded.reshape(m_atoms, 4, 32, sf_k_atoms, atom_k, l) + # Arrangement (last-idx-major): (l, sf_m_atoms, sf_k_atoms, neutron_m, proton_m, atom_k) + # (l, sf_m_atoms, sf_k_atoms) follows a traditional K-major interblock layout + # (atom_k) is the last index -> reproduces (sf_vec,atom_k):(0,1) + # (neutron_m, proton_m) -> reproduces + # prod((neutron_m,proton_m):(proton_m:1), 1:atom_k) = (32,4):(16,4) + .permute(5, 0, 3, 2, 1, 4) + .contiguous() + ) + + sf_tensor, _ = cutlass_torch.cute_tensor_like( + sf_storage_f32, + sf_dtype, + is_dynamic_layout=True, + assumed_align=16, + ) + + sf_logical = sf_logical_padded[:mn, :sf_k, :] + # Broadcast each SF across its sf_vec_size K elements to build a elementwise (mn, k, l) + # reference whose (m, j, b) entry is the SF the kernel multiplies into the same logical position + ref_elementwise = ( + sf_logical.unsqueeze(2) + .expand(-1, -1, sf_vec_size, -1) + .reshape(mn, sf_k * sf_vec_size, l)[:, :k, :] + .contiguous() + ) + + return ref_elementwise, sf_tensor + + +def compact_fp4_data(torch_underlying: torch.Tensor, dtype: Type[cutlass.Numeric]) -> None: + """Compact packed FP4 rows to match CuTe's sub-byte stride interpretation.""" + if dtype is not cutlass.Float4E2M1FN: + return + + # torch lacks fill_/copy_ kernels for Float4_e2m1fn_x2 on CUDA, but the + # storage is byte-packed (two FP4 per byte), so a uint8 view is a free + # reinterpret and supports the in-place primitives we need. + d = torch_underlying.shape[-1] + packed_size = d // (8 // dtype.width) + underlying_u8 = torch_underlying.view(torch.uint8) + rows = underlying_u8.reshape(-1, d) + packed = rows[:, :packed_size].contiguous() + underlying_u8.fill_(0) + underlying_u8.flatten()[: packed.numel()].copy_(packed.flatten()) + + +class BlackwellFusedMultiHeadBlockScaledAttentionForward: + arch_str: str = "sm_100" + arch_name: str = "Blackwell SM100" + + def __init__( + self, + qk_acc_dtype: Type[cutlass.Numeric], + pv_acc_dtype: Type[cutlass.Numeric], + mma_tiler: Tuple[int, int], + head_dim: Union[int, Tuple[int, int]], + is_persistent: bool, + mask_type: fmha_utils.MaskEnum, + enable_ex2_emulation: bool, + enable_skip_correction: bool, + qk_sf_vec_size: int, + use_tma_store: bool = True, + ): + """Initializes the configuration for a Blackwell Fused Multi-Head Attention (FMHA) kernel. + + This configuration includes several key aspects: + + 1. Data Type Settings: + - qk_acc_dtype: Data type for Q*K^T matrix multiplication accumulator + - pv_acc_dtype: Data type for P*V matrix multiplication accumulator + + 2. MMA Instruction Settings: + - mma_tiler: The shape of the MMA instruction unit: (M, N) for BMM1 and (M, K) for BMM2 + - head_dim: The head dimension, it can be a single integer or a tuple of two integers (D, Dv). + If it is a tuple, Dv is the head dimension of the value & output tensors. + It also determines the K dimension of the BMM1's MMA instruction unit + & N dimension of the BMM2's MMA instruction unit. + - qk_mma_tiler: MMA shape for Q*K^T computation + - pv_mma_tiler: MMA shape for P*V computation + + 3. Kernel Execution Mode: + - is_persistent: Boolean indicating whether to use persistent kernel mode + - mask_type: Specifies the type of mask to use (no mask, residual mask, or causal mask) + - window_size_left/right: Sliding window size for attention masking + - enable_ex2_emulation: Whether to enable exp2 emulation + - enable_skip_correction: Whether to skip the correction when rowmax is not updated larger than a threshold + + :param qk_acc_dtype: Data type for Q*K^T matrix multiplication accumulator + :type qk_acc_dtype: Type[cutlass.Numeric] + :param pv_acc_dtype: Data type for P*V matrix multiplication accumulator + :type pv_acc_dtype: Type[cutlass.Numeric] + :param mma_tiler: The (M, N) shape of the MMA instruction + :type mma_tiler: Tuple[int, int] + :param head_dim: The head dimension, it can be a single integer or a tuple of two integers (D, Dv). + :type head_dim: Union[int, Tuple[int, int]] + :param is_persistent: Whether to use persistent kernel mode + :type is_persistent: bool + :param mask_type: Type of mask to use + :type mask_type: fmha_utils.MaskEnum + :param window_size_left: Left-side sliding window size for attention masking + :type window_size_left: int + :param window_size_right: Right-side sliding window size for attention masking + :type window_size_right: int + """ + + self.qk_acc_dtype = qk_acc_dtype + self.pv_acc_dtype = pv_acc_dtype + if mma_tiler != (128, 128): + raise ValueError( + "This standalone kernel uses a static TMEM map and currently supports only mma_tiler=(128, 128)" + ) + if isinstance(head_dim, tuple): + self.head_dim = head_dim[0] + self.head_dim_v = head_dim[1] + assert self.head_dim == 192 and self.head_dim_v == 128, ( + f"When Headdim is a tuple, it's for MLA. Must be (192, 128), but got {head_dim}" + ) + else: + self.head_dim = head_dim + self.head_dim_v = head_dim + self.cta_tiler = ( + 2 * mma_tiler[0], # 2 O tile per CTA + mma_tiler[1], + self.head_dim_v, + ) + self.qk_mma_tiler = ( + *mma_tiler, + self.head_dim, + ) + self.pv_mma_tiler = ( + mma_tiler[0], + self.head_dim_v, + mma_tiler[1], + ) + self.cluster_shape_mn = (1, 1) + self.is_persistent = is_persistent + self.mask_type = mask_type + self.enable_skip_correction = enable_skip_correction + self.enable_ex2_emulation = enable_ex2_emulation + self.qk_sf_vec_size = qk_sf_vec_size + self.qk_mma_inst_bits_k = 256 + if qk_sf_vec_size == 16: + # NVFP4: a 256-bit MMA operand tile covers 64 logical K values. + self.qk_mma_inst_tile_k = self.head_dim // (self.qk_mma_inst_bits_k // 4) + elif qk_sf_vec_size == 32: + # MXFP8: a 256-bit MMA operand tile covers 32 logical K values. + self.qk_mma_inst_tile_k = self.head_dim // (self.qk_mma_inst_bits_k // 8) + else: + self.qk_mma_inst_tile_k = 0 + self.use_tma_store = use_tma_store + + self.softmax0_warp_ids = (0, 1, 2, 3) + self.softmax1_warp_ids = (4, 5, 6, 7) + self.correction_warp_ids = (8, 9, 10, 11) + self.mma_warp_id = 12 + self.load_warp_id = 13 + self.epilogue_warp_id = 14 + self.empty_warp_id = 15 + self.num_tmem_alloc_cols = cute.arch.get_max_tmem_alloc_cols(self.arch_str) + + self.threads_per_warp = 32 + self.threads_per_cta = self.threads_per_warp * len( + ( + *self.softmax0_warp_ids, + *self.softmax1_warp_ids, + *self.correction_warp_ids, + self.mma_warp_id, + self.load_warp_id, + self.epilogue_warp_id, + self.empty_warp_id, + ) + ) + self.tmem_alloc_barrier = pipeline.NamedBarrier( + barrier_id=2, + num_threads=self.threads_per_warp + * sum( + ( + len((self.mma_warp_id,)), + len(self.softmax0_warp_ids), + len(self.softmax1_warp_ids), + len(self.correction_warp_ids), + ) + ), + ) + self.sequence_s0_s1_barrier = pipeline.NamedBarrier( + barrier_id=3, + num_threads=self.threads_per_warp + * len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)), + ) + self.sequence_s1_s0_barrier = pipeline.NamedBarrier( + barrier_id=4, + num_threads=self.threads_per_warp + * len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)), + ) + self.s0_warpgroup_barrier = pipeline.NamedBarrier( + barrier_id=5, + num_threads=self.threads_per_warp * len(self.softmax0_warp_ids), + ) + self.s1_warpgroup_barrier = pipeline.NamedBarrier( + barrier_id=6, + num_threads=self.threads_per_warp * len(self.softmax1_warp_ids), + ) + self.tmem_dealloc_barrier = pipeline.NamedBarrier( + barrier_id=7, + num_threads=self.threads_per_warp * len(self.correction_warp_ids), + ) + + self.tmem_s0_offset = 0 + self.tmem_s1_offset = 128 + self.tmem_o0_offset = 256 + self.tmem_o1_offset = 384 + # inplaced with s1 + self.tmem_p0_offset = 160 + # inplaced with s0 + self.tmem_p1_offset = 32 + # Block-scaled QK scale factors are live only while issuing QK MMA. + # Keep each stage's scale buffers in the opposite S/P tile's tail columns. + self.tmem_qk0_sf_offset = 224 + self.tmem_qk1_sf_offset = 96 + # vec buffer for row_max & row_sum + # inplaced with s0 + self.tmem_vec0_offset = 0 + # inplaced with s1 + self.tmem_vec1_offset = 128 + # skip mma pv flag offset regarding to the vec buffer + # inplaced with s1 + self.tmem_skip_softmax0_offset = 136 + # inplaced with s0 + self.tmem_skip_softmax1_offset = 8 + + self.num_regs_softmax = 192 + self.num_regs_correction = 96 + self.num_regs_other = 32 + self.buffer_align_bytes = 1024 + self.arch = BaseDSL._get_dsl().get_arch_enum() + + if self.arch >= Arch.sm_103: + assert self.enable_ex2_emulation == False, ( + f"Don't enable exp2 emulation for {self.arch}, it doesn't help performance" + ) + + num_warps_per_warpgroup = 4 + self.softmax_warpgroup_count = ( + len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)) // num_warps_per_warpgroup + ) + + def _make_qk_tiled_mma(self, cta_group): + """Build the QK tiled MMA. Override in subclasses to target a different arch.""" + return sm100_utils.make_blockscaled_trivial_tiled_mma( + self.q_dtype, + self.k_dtype, + self.q_major_mode, + self.k_major_mode, + self.qk_sf_dtype, + self.qk_sf_vec_size, + cta_group, + self.qk_mma_tiler[:2], + ) + + def _make_pv_tiled_mma(self, cta_group, p_major_mode, p_source): + """Build the PV tiled MMA. Override in subclasses to target a different arch.""" + return sm100_utils.make_trivial_tiled_mma( + self.v_dtype, + self.v_dtype, + p_major_mode, + self.v_major_mode, + self.pv_acc_dtype, + cta_group, + self.pv_mma_tiler[:2], + p_source, + ) + + def _setup_attributes(self, enable_skip_softmax: bool) -> None: + """Set up configurations and parameters for the FMHA kernel operation. + + This method initializes and configures various attributes required for the + execution of the fused multi-head attention kernel, mainly about the pipeline stages: + + - Sets up staging parameters for Q, K, V inputs and accumulator data + - Configures pipeline stages for softmax, correction, and epilogue operations + """ + + self.q_stage = 2 + k_stage = 4 if self.q_dtype.width == 8 else 3 + v_stage = 4 if self.v_dtype.width == 8 else 3 + self.kv_stage = min(k_stage, v_stage) + # For D192, the smem usage of Q & K is larger. So, we need to reduce the stage count. + if self.head_dim == 192 and self.q_dtype.width == 16: + self.kv_stage = 2 + self.p_mma_stage = 1 + self.acc_stage = 1 + self.softmax_corr_stage = 1 + self.mma_corr_stage = 2 + self.mma_softmax_stage = 1 + self.epi_stage = 2 + + # Tunable parameters + if not self.enable_skip_correction: + self.rescale_threshold = 0.0 + elif enable_skip_softmax: + self.rescale_threshold = 1.0 + else: + self.rescale_threshold = 8.0 + # FP8 P pre-scale: offset added to exp2 exponent so that P*2^offset fills + # more of E4M3's [0, 448] range, improving quantization precision. + # Derived from rescale_threshold to guarantee P*2^offset <= 448. + self.p_fp8_prescale_log2 = max(0.0, math.floor(math.log2(448) - self.rescale_threshold)) + # ln(2) * offset correction for LSE when pre-scale is active + self.p_fp8_prescale_lse_correction = self.p_fp8_prescale_log2 * math.log(2) + # For most cases, seq barrier is needed to help keep the pipeline stable + # But sometimes, compiler will schedule the barrier at an unexpected place + # if it hurts perf a lot, try to quickly fix it by disabling seq barrier + self.enable_sequence_barrier = False + # Optional double buffering for correction rescale. + self.enable_correction_double_buffer = False + + @cute.jit + def __call__( + self, + q_tensor: cute.Tensor, + k_tensor: cute.Tensor, + q_sf_tensor: cute.Tensor, + k_sf_tensor: cute.Tensor, + v_tensor: cute.Tensor, + o_tensor: cute.Tensor, + problem_size: Tuple[Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32], + cum_seqlen_q: Optional[cute.Tensor], + cum_seqlen_k: Optional[cute.Tensor], + lse_tensor: Optional[cute.Tensor], + sink_tensor: Optional[cute.Tensor], + scale_softmax_log2: Float32, + scale_softmax: Float32, + scale_output: Float32, + scale_v_channels: Optional[cute.Tensor], + skip_softmax_threshold_log2: Optional[Float32], + window_size_left: Optional[Int32], + window_size_right: Optional[Int32], + skip_softmax_count: Optional[cute.Tensor], + total_softmax_count: Optional[cute.Tensor], + stream: cuda.CUstream, + use_pdl: bool, + ): + """Execute the Fused Multi-Head Attention operation on the provided tensors. + + This method prepares the input tensors for processing, validates their shapes and types, + configures the computation parameters, and launches the CUDA kernel. + + The method handles: + 1. Tensor layout transformations for specific memory access patterns + 2. Validation of tensor shapes and data types + 3. Initialization of hardware-specific parameters and memory layouts + 4. Configuration of TMA (Tensor Memory Access) operations + 5. Grid and work scheduling computation + 6. Kernel launch with appropriate parameters + + :param q_tensor: The query tensor with shape (b, s_q, h_k, h_r, d) + :type q_tensor: cute.Tensor + :param k_tensor: The key tensor with shape (b, s_k, h_k, 1, d) + :type k_tensor: cute.Tensor + :param v_tensor: The value tensor with shape (b, s_v, h_k, 1, dv) + :type v_tensor: cute.Tensor + :param o_tensor: The output tensor with shape (b, s_q, h_k, h_r, dv) + :type o_tensor: cute.Tensor + :param problem_size: The problem size with shape [b, s_q_max, s_lse_max, s_k_max, h_q, h_k, d, dv]. If cum_seqlen_q or cum_seqlen_k is not None, s_q_max and s_k_max are the max of the per-batch sequence lengths respectively. + :type problem_size: Tuple[Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32] + :param cum_seqlen_q: The cumulative sequence length tensor for query + :type cum_seqlen_q: Optional[cute.Tensor] + :param cum_seqlen_k: The cumulative sequence length tensor for key + :type cum_seqlen_k: Optional[cute.Tensor] + :param scale_softmax_log2: The log2 scale factor for softmax + :type scale_softmax_log2: Float32 + :param scale_softmax: The scale factor for softmax + :type scale_softmax: Float32 + :param scale_output: The scale factor for the output + :type scale_output: Float32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + :param stream: The CUDA stream to execute the kernel on + :type stream: cuda.CUstream + :raises TypeError: If tensor data types don't match or aren't supported + :raises RuntimeError: If tensor layouts aren't in supported formats + """ + b, s_q_max, s_lse_max, s_k_max, h_q, h_k, d_, dv_ = problem_size + h_r = h_q // h_k + # setup static attributes before smem/grid/tma computation + self.q_dtype = q_tensor.element_type + self.k_dtype = k_tensor.element_type + self.qk_sf_dtype = q_sf_tensor.element_type + self.v_dtype = v_tensor.element_type + self.o_dtype = o_tensor.element_type + + # s_q, s_k, s_v are the actual tensor dimensions (total seqlen for varlen) + s_q = q_tensor.shape[1] + s_k = k_tensor.shape[1] + s_v = v_tensor.shape[1] + s_lse = s_lse_max + d = self.head_dim + dv = self.head_dim_v + # Important for performance + align = 256 // self.o_dtype.width + assert d % align == 0, f"head_dim must be multiple of {align} for the given datatypes." + assert dv % align == 0, f"head_dim_v must be multiple of {align} for the given datatypes." + + stride_b_q = h_r * h_k * s_q * d if cum_seqlen_q is None else 0 + stride_b_o = h_r * h_k * s_q * dv if cum_seqlen_q is None else 0 + stride_b_k = h_k * s_k * d if cum_seqlen_k is None else 0 + stride_b_v = h_k * s_v * dv if cum_seqlen_k is None else 0 + stride_b_lse = h_r * h_k * s_lse if cum_seqlen_q is None else 0 + + # (b, s_q, h_k, h_r, d) -> (s_q, d, ((h_r, h_k), b)) + q_layout = cute.make_layout( + (s_q, d, ((h_r, h_k), b)), + stride=(d * h_r * h_k, 1, ((d, d * h_r), stride_b_q)), + ) + q = cute.make_tensor(q_tensor.iterator, q_layout) + # (b, s_k, h_k, 1, d) -> (s_k, d, ((h_r, h_k), b)), 0-stride for h_r to broadcast + k_layout = cute.make_layout( + (s_k, d, ((h_r, h_k), b)), + stride=(d * h_k, 1, ((0, d), stride_b_k)), + ) + k = cute.make_tensor(k_tensor.iterator, k_layout) + q_sf_layout = blockscaled_utils.tile_atom_to_shape_SF(q.shape, self.qk_sf_vec_size) + q_sf = cute.make_tensor(q_sf_tensor.iterator, q_sf_layout) + k_sf_layout = blockscaled_utils.tile_atom_to_shape_SF( + (s_k, d, (h_k, b)), self.qk_sf_vec_size + ) + k_sf = cute.make_tensor(k_sf_tensor.iterator, k_sf_layout) + # (b, s_v, h_k, 1, dv) -> (dv, s_v, ((h_r, h_k), b)), 0-stride for h_r to broadcast + v_layout = cute.make_layout( + (dv, s_v, ((h_r, h_k), b)), + stride=(1, dv * h_k, ((0, dv), stride_b_v)), + ) + v = cute.make_tensor(v_tensor.iterator, v_layout) + # (b, s_q, h_k, h_r, dv) -> (s_q, dv, ((h_r, h_k), b)) + o_layout = cute.make_layout( + (s_q, dv, ((h_r, h_k), b)), + stride=(dv * h_r * h_k, 1, ((dv, dv * h_r), stride_b_o)), + ) + o = cute.make_tensor(o_tensor.iterator, o_layout) + if cutlass.const_expr(lse_tensor is not None): + # (s, ((h_r, h_k), b)) - head stride=1 to match FlashInfer (total_q, h_q) convention + lse_layout = cute.make_layout( + (s_lse, ((h_r, h_k), b)), + stride=(h_r * h_k, ((1, h_r), stride_b_lse)), + ) + lse = cute.make_tensor(lse_tensor.iterator, lse_layout) + else: + lse = None + + if cutlass.const_expr(sink_tensor is not None): + # sink_tensor is 1D with shape (h_q,) = (h_k * h_r,) + # Create layout ((h_r, h_k), b) with stride 0 for batch so blk_coord[2] works + sink_layout = cute.make_layout( + ((h_r, h_k), b), + stride=((1, h_r), 0), + ) + sink = cute.make_tensor(sink_tensor.iterator, sink_layout) + else: + sink = None + + if cutlass.const_expr(scale_v_channels is not None): + # scale_v_channels is per (h_k, dv): shape (h_k * dv,) row-major. + # Expose as (dv, ((h_r, h_k), b)) where h_r and b are 0-stride broadcasts. + scale_v_channels_layout = cute.make_layout( + (dv, ((h_r, h_k), b)), + stride=(1, ((0, dv), 0)), + ) + m_scale_v_channels = cute.make_tensor( + scale_v_channels.iterator, scale_v_channels_layout + ) + else: + m_scale_v_channels = None + + self.tile_sched_params, grid = fmha_utils.compute_grid( + cute.shape((s_q_max, d, ((h_r, h_k), b))), + self.cta_tiler, + self.is_persistent, + ) + self.q_major_mode = utils.LayoutEnum.from_tensor(q).mma_major_mode() + self.k_major_mode = utils.LayoutEnum.from_tensor(k).mma_major_mode() + self.v_major_mode = utils.LayoutEnum.from_tensor(v).mma_major_mode() + self.o_layout = utils.LayoutEnum.from_tensor(o) + + if cutlass.const_expr(self.q_major_mode != OperandMajorMode.K): + raise RuntimeError("The layout of q is not supported") + if cutlass.const_expr(self.k_major_mode != OperandMajorMode.K): + raise RuntimeError("The layout of k is not supported") + if cutlass.const_expr(self.v_major_mode != OperandMajorMode.MN): + raise RuntimeError("The layout of v is not supported") + + # check type consistency: Q and K must share the same dtype (qk_dtype); + # V may use a different dtype (pv_dtype) to allow qk_dtype != pv_dtype. + if cutlass.const_expr(self.q_dtype != self.k_dtype): + raise TypeError(f"Type mismatch: {self.q_dtype} != {self.k_dtype}") + if cutlass.const_expr(self.qk_sf_dtype != k_sf_tensor.element_type): + raise TypeError( + f"Q/K scale type mismatch: {self.qk_sf_dtype} != {k_sf_tensor.element_type}" + ) + if cutlass.const_expr(self.qk_mma_inst_tile_k <= 0): + raise RuntimeError( + f"Invalid QK scale-factor tiling for head_dim={self.head_dim}, qk_sf_vec_size={self.qk_sf_vec_size}" + ) + # SFQ + SFK share one 32-column TMEM slot per stage (see kernel TMEM layout + # below). Validate the static footprint on the host so we don't smuggle a + # `raise` into @cute.kernel. + _sf_atom_mn = 32 + _qk_sf_tmem_cols = (self.qk_mma_tiler[0] // _sf_atom_mn) * self.qk_mma_inst_tile_k + _kqk_sf_tmem_cols = ( + ((self.qk_mma_tiler[1] + 127) // 128 * 128) // _sf_atom_mn + ) * self.qk_mma_inst_tile_k + if cutlass.const_expr(_qk_sf_tmem_cols + _kqk_sf_tmem_cols > 32): + raise RuntimeError( + "QK scale-factor TMEM footprint exceeds the static 32-column slot " + f"(qk_sf_tmem_cols={_qk_sf_tmem_cols}, k_sf_tmem_cols={_kqk_sf_tmem_cols})" + ) + self._setup_attributes(skip_softmax_threshold_log2 is not None) + + cta_group = tcgen05.CtaGroup.ONE + # the intermediate tensor p is from tmem & k-major + p_source = tcgen05.OperandSource.TMEM + p_major_mode = cute.nvgpu.OperandMajorMode.K + qk_tiled_mma = self._make_qk_tiled_mma(cta_group) + pv_tiled_mma = self._make_pv_tiled_mma(cta_group, p_major_mode, p_source) + + self.cluster_shape_mnk = (*self.cluster_shape_mn, 1) + self.cluster_layout_vmnk = cute.tiled_divide( + cute.make_layout(self.cluster_shape_mnk), + (qk_tiled_mma.thr_id.shape,), + ) + self.epi_tile = self.pv_mma_tiler[:2] + + q_smem_layout_staged = sm100_utils.make_smem_layout_a( + qk_tiled_mma, + self.qk_mma_tiler, + self.q_dtype, + self.q_stage, + ) + k_smem_layout_staged = sm100_utils.make_smem_layout_b( + qk_tiled_mma, + self.qk_mma_tiler, + self.k_dtype, + self.kv_stage, + ) + q_sf_smem_layout_staged = blockscaled_utils.make_smem_layout_sfa( + qk_tiled_mma, + self.qk_mma_tiler, + self.qk_sf_vec_size, + self.q_stage, + ) + k_sf_smem_layout_staged = blockscaled_utils.make_smem_layout_sfb( + qk_tiled_mma, + self.qk_mma_tiler, + self.qk_sf_vec_size, + self.kv_stage, + ) + p_tmem_layout_staged = sm100_utils.make_smem_layout_a( + pv_tiled_mma, + self.pv_mma_tiler, + self.v_dtype, + self.acc_stage, + ) + v_smem_layout_staged_origin = sm100_utils.make_smem_layout_b( + pv_tiled_mma, + self.pv_mma_tiler, + self.v_dtype, + self.kv_stage, + ) + # k & v share the same smem buffer. Pad the smaller-tile operand's stage stride to + # match the larger tile. Stride is scaled to the target operand's element width. + # sK_cosize covers all kv_stage slots at the larger tile's byte footprint. + if cutlass.const_expr(self.k_dtype.width >= self.v_dtype.width): + # k tile >= v tile: give v k's stage stride in v_dtype units + k_tile_cosize = cute.cosize(cute.select(k_smem_layout_staged, mode=[0, 1, 2])) + v_stage_stride = k_tile_cosize * self.k_dtype.width // self.v_dtype.width + v_smem_layout_staged = cute.append( + cute.select(v_smem_layout_staged_origin, mode=[0, 1, 2]), + cute.make_layout(self.kv_stage, stride=v_stage_stride), + ) + sK_cosize = cute.cosize(k_smem_layout_staged) + else: + # v tile > k tile: give k v's stage stride in k_dtype units + v_tile_cosize = cute.cosize(cute.select(v_smem_layout_staged_origin, mode=[0, 1, 2])) + k_stage_stride = v_tile_cosize * self.v_dtype.width // self.k_dtype.width + k_smem_layout_staged = cute.append( + cute.select(k_smem_layout_staged, mode=[0, 1, 2]), + cute.make_layout(self.kv_stage, stride=k_stage_stride), + ) + v_smem_layout_staged = v_smem_layout_staged_origin + # cute.cosize gives (kv_stage-1)*k_stage_stride + k_tile_cosize, but the + # last stage must also accommodate a full V tile, so size for kv_stage slots. + sK_cosize = self.kv_stage * k_stage_stride + + o_smem_layout_staged = sm100_utils.make_smem_layout_epi( + self.o_dtype, + self.o_layout, + self.epi_tile, + self.epi_stage, + ) + + # TMA load for Q + tma_load_op = cute.nvgpu.cpasync.CopyBulkTensorTileG2SOp(cta_group) + tma_store_op = cute.nvgpu.cpasync.CopyBulkTensorTileS2GOp() + + q_smem_layout = cute.select(q_smem_layout_staged, mode=[0, 1, 2]) + tma_atom_q, tma_tensor_q = cute.nvgpu.make_tiled_tma_atom_A( + tma_load_op, + q, + q_smem_layout, + self.qk_mma_tiler, + qk_tiled_mma, + self.cluster_layout_vmnk.shape, + ) + q_sf_smem_layout = cute.slice_(q_sf_smem_layout_staged, (None, None, None, 0)) + tma_atom_q_sf, tma_tensor_q_sf = cute.nvgpu.make_tiled_tma_atom_A( + tma_load_op, + q_sf, + q_sf_smem_layout, + self.qk_mma_tiler, + qk_tiled_mma, + self.cluster_layout_vmnk.shape, + internal_type=cutlass.Int16, + ) + + # TMA load for K + k_smem_layout = cute.select(k_smem_layout_staged, mode=[0, 1, 2]) + tma_atom_k, tma_tensor_k = cute.nvgpu.make_tiled_tma_atom_B( + tma_load_op, + k, + k_smem_layout, + self.qk_mma_tiler, + qk_tiled_mma, + self.cluster_layout_vmnk.shape, + ) + k_sf_smem_layout = cute.slice_(k_sf_smem_layout_staged, (None, None, None, 0)) + tma_atom_k_sf, tma_tensor_k_sf = cute.nvgpu.make_tiled_tma_atom_B( + tma_load_op, + k_sf, + k_sf_smem_layout, + self.qk_mma_tiler, + qk_tiled_mma, + self.cluster_layout_vmnk.shape, + internal_type=cutlass.Int16, + ) + # TMA load for V + v_smem_layout = cute.select(v_smem_layout_staged, mode=[0, 1, 2]) + tma_atom_v, tma_tensor_v = cute.nvgpu.make_tiled_tma_atom_B( + tma_load_op, + v, + v_smem_layout, + self.pv_mma_tiler, + pv_tiled_mma, + self.cluster_layout_vmnk.shape, + ) + + o_smem_layout = cute.select(o_smem_layout_staged, mode=[0, 1]) + tma_atom_o, tma_tensor_o = cute.nvgpu.cpasync.make_tiled_tma_atom( + tma_store_op, + o, + o_smem_layout, + self.epi_tile, + ) + + q_copy_size = cute.size_in_bytes(self.q_dtype, q_smem_layout) + k_copy_size = cute.size_in_bytes(self.k_dtype, k_smem_layout) + q_sf_copy_size = cute.size_in_bytes(self.qk_sf_dtype, q_sf_smem_layout) + k_sf_copy_size = cute.size_in_bytes(self.qk_sf_dtype, k_sf_smem_layout) + v_copy_size = cute.size_in_bytes(self.v_dtype, v_smem_layout) + self.tma_copy_q_bytes = q_copy_size + q_sf_copy_size + self.tma_copy_k_bytes = k_copy_size + k_sf_copy_size + self.tma_copy_v_bytes = v_copy_size + + @cute.struct + class SharedStorage: + # Pipeline barriers + load_q_mbar_ptr: cute.struct.MemRange[Int64, self.q_stage * 2] + load_kv_mbar_ptr: cute.struct.MemRange[Int64, self.kv_stage * 2] + mma_s0_mbar_ptr: cute.struct.MemRange[Int64, self.mma_softmax_stage * 2] + mma_s1_mbar_ptr: cute.struct.MemRange[Int64, self.mma_softmax_stage * 2] + p0_mma_mbar_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] + p1_mma_mbar_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] + s0_corr_mbar_ptr: cute.struct.MemRange[Int64, self.softmax_corr_stage * 2] + s1_corr_mbar_ptr: cute.struct.MemRange[Int64, self.softmax_corr_stage * 2] + corr_epi_mbar_ptr: cute.struct.MemRange[Int64, self.epi_stage * 2] + mma_corr_mbar_ptr: cute.struct.MemRange[Int64, self.mma_corr_stage * 2] + s0_p1_inplace_barrier_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] + s1_p0_inplace_barrier_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] + # Softmax_{1-j} signals MMA that S_{1-j}'s TMEM region (whose tail + # columns hold SFQ_j / SFK_j) is no longer being read, so mma_qk's + # SFQ/SFK S2T copy is safe to issue. One pipeline per QK stage. + qk_sf_inplace_0_barrier_ptr: cute.struct.MemRange[Int64, 1 * 2] + qk_sf_inplace_1_barrier_ptr: cute.struct.MemRange[Int64, 1 * 2] + # Tmem holding buffer + tmem_holding_buf: Int32 + # Smem tensors + sO: cute.struct.Align[ + cute.struct.MemRange[self.o_dtype, cute.cosize(o_smem_layout_staged)], + self.buffer_align_bytes, + ] + sQ: cute.struct.Align[ + cute.struct.MemRange[self.q_dtype, cute.cosize(q_smem_layout_staged)], + self.buffer_align_bytes, + ] + sQSF: cute.struct.Align[ + cute.struct.MemRange[self.qk_sf_dtype, cute.cosize(q_sf_smem_layout_staged)], + self.buffer_align_bytes, + ] + sK: cute.struct.Align[ + cute.struct.MemRange[self.k_dtype, sK_cosize], + self.buffer_align_bytes, + ] + sKSF: cute.struct.Align[ + cute.struct.MemRange[self.qk_sf_dtype, cute.cosize(k_sf_smem_layout_staged)], + self.buffer_align_bytes, + ] + # Skip softmax and PV warpgroup votes + s0_warp_wants_skip_softmax_exchange: cute.struct.MemRange[Int8, 4] + s1_warp_wants_skip_softmax_exchange: cute.struct.MemRange[Int8, 4] + + self.shared_storage = SharedStorage + + # Launch the kernel synchronously + self.kernel( + qk_tiled_mma, + pv_tiled_mma, + tma_atom_q, + tma_tensor_q, + tma_atom_q_sf, + tma_tensor_q_sf, + tma_atom_k, + tma_tensor_k, + tma_atom_k_sf, + tma_tensor_k_sf, + tma_atom_v, + tma_tensor_v, + tma_atom_o, + tma_tensor_o, + o, + cum_seqlen_q, + cum_seqlen_k, + lse, + sink, + scale_softmax_log2, + scale_softmax, + scale_output, + m_scale_v_channels, + skip_softmax_threshold_log2, + window_size_left, + window_size_right, + q_smem_layout_staged, + k_smem_layout_staged, + q_sf_smem_layout_staged, + k_sf_smem_layout_staged, + p_tmem_layout_staged, + v_smem_layout_staged, + o_smem_layout_staged, + skip_softmax_count, + total_softmax_count, + self.tile_sched_params, + ).launch( + grid=grid, + block=[self.threads_per_cta, 1, 1], + cluster=self.cluster_shape_mnk, + stream=stream, + min_blocks_per_mp=1, + use_pdl=use_pdl, + ) + + # GPU device kernel + @cute.kernel + def kernel( + self, + qk_tiled_mma: cute.TiledMma, + pv_tiled_mma: cute.TiledMma, + tma_atom_q: cute.CopyAtom, + mQ_qdl: cute.Tensor, + tma_atom_q_sf: cute.CopyAtom, + mQSF_qdl: cute.Tensor, + tma_atom_k: cute.CopyAtom, + mK_kdl: cute.Tensor, + tma_atom_k_sf: cute.CopyAtom, + mKSF_kdl: cute.Tensor, + tma_atom_v: cute.CopyAtom, + mV_dkl: cute.Tensor, + tma_atom_o: cute.CopyAtom, + mO_qdl: cute.Tensor, + mO: cute.Tensor, + cum_seqlen_q: Optional[cute.Tensor], + cum_seqlen_k: Optional[cute.Tensor], + mLSE: Optional[cute.Tensor], + mSink: Optional[cute.Tensor], + scale_softmax_log2: Float32, + scale_softmax: Float32, + scale_output: Float32, + m_scale_v_channels: Optional[cute.Tensor], + skip_softmax_threshold_log2: Optional[Float32], + window_size_left: Optional[Int32], + window_size_right: Optional[Int32], + q_smem_layout_staged: cute.ComposedLayout, + k_smem_layout_staged: cute.ComposedLayout, + q_sf_smem_layout_staged: cute.Layout, + k_sf_smem_layout_staged: cute.Layout, + p_tmem_layout_staged: cute.ComposedLayout, + v_smem_layout_staged: cute.ComposedLayout, + o_smem_layout_staged: cute.ComposedLayout, + skip_softmax_count: Optional[cute.Tensor], + total_softmax_count: Optional[cute.Tensor], + tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams, + ): + """The device kernel implementation of the Fused Multi-Head Attention. + + This kernel coordinates multiple specialized warps to perform different phases of the FMHA computation: + 1. Load warp: Loads Q, K, V data from global memory to shared memory using TMA + 2. MMA warp: Performs matrix multiplications (Q*K^T and P*V) + 3. Softmax warps: Compute softmax normalization on attention scores + 4. Correction warps: Apply adjustments to intermediate results + 5. Epilogue warp: Handles final output transformation and storage + + The kernel implements a complex pipeline with overlapping computation and memory operations, + using tensor memory access (TMA) for efficient data loading, warp specialization for different + computation phases, and optional attention masking. + + :param qk_tiled_mma: Tiled MMA for Q*K^T + :type qk_tiled_mma: cute.TiledMma + :param pv_tiled_mma: Tiled MMA for P*V + :type pv_tiled_mma: cute.TiledMma + :param tma_atom_q: TMA copy atom for query tensor + :type tma_atom_q: cute.CopyAtom + :param mQ_qdl: Partitioned query tensor + :type mQ_qdl: cute.Tensor + :param tma_atom_k: TMA copy atom for key tensor + :type tma_atom_k: cute.CopyAtom + :param mK_kdl: Partitioned key tensor + :type mK_kdl: cute.Tensor + :param tma_atom_v: TMA copy atom for value tensor + :type tma_atom_v: cute.CopyAtom + :param mV_dkl: Partitioned value tensor + :type mV_dkl: cute.Tensor + :param tma_atom_o: TMA copy atom for output tensor + :type tma_atom_o: cute.CopyAtom + :param mO_qdl: Partitioned output tensor + :type mO_qdl: cute.Tensor + :param mO: Non-partitioned output tensor + :type mO: cute.Tensor + :param scale_softmax_log2: The log2 scale factor for softmax + :type scale_softmax_log2: Float32 + :param scale_output: The scale factor for the output + :type scale_output: Float32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + :param q_smem_layout_staged: Shared memory layout for query tensor + :type q_smem_layout_staged: cute.ComposedLayout + :param k_smem_layout_staged: Shared memory layout for key tensor + :type k_smem_layout_staged: cute.ComposedLayout + :param p_tmem_layout_staged: Tensor memory layout for probability matrix + :type p_tmem_layout_staged: cute.ComposedLayout + :param v_smem_layout_staged: Shared memory layout for value tensor + :type v_smem_layout_staged: cute.ComposedLayout + :param o_smem_layout_staged: Shared memory layout for output tensor + :type o_smem_layout_staged: cute.ComposedLayout + :param tile_sched_params: Scheduling parameters for work distribution + :type tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams + """ + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + # coord inside cta + tidx, _, _ = cute.arch.thread_idx() + + # + # Prefetch tma desc + # + if warp_idx == self.load_warp_id: + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_q) + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_q_sf) + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_k) + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_k_sf) + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_v) + if cutlass.const_expr(self.use_tma_store): + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_o) + + # Alloc + smem = utils.SmemAllocator() + storage = smem.allocate(self.shared_storage) + + load_q_producer, load_q_consumer = pipeline.PipelineTmaUmma.create( + num_stages=self.q_stage, + producer_group=make_thread_cooperative_group(len([self.load_warp_id])), + consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + tx_count=self.tma_copy_q_bytes, + barrier_storage=storage.load_q_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + load_kv_producer, load_kv_consumer = pipeline.PipelineTmaUmma.create( + num_stages=self.kv_stage, + producer_group=make_thread_cooperative_group(len([self.load_warp_id])), + consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + tx_count=self.tma_copy_k_bytes, + barrier_storage=storage.load_kv_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + load_kv_full_mbar_ptr = storage.load_kv_mbar_ptr.data_ptr() + load_kv_empty_mbar_ptr = load_kv_full_mbar_ptr + self.kv_stage + mma_s0_producer, mma_s0_consumer = pipeline.PipelineUmmaAsync.create( + num_stages=self.mma_softmax_stage, + producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax0_warp_ids) + ), + barrier_storage=storage.mma_s0_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + mma_s1_producer, mma_s1_consumer = pipeline.PipelineUmmaAsync.create( + num_stages=self.mma_softmax_stage, + producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax1_warp_ids) + ), + barrier_storage=storage.mma_s1_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + p0_mma_producer, p0_mma_consumer = pipeline.PipelineAsyncUmma.create( + num_stages=self.p_mma_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax0_warp_ids) + ), + consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + barrier_storage=storage.p0_mma_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + p1_mma_producer, p1_mma_consumer = pipeline.PipelineAsyncUmma.create( + num_stages=self.p_mma_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax1_warp_ids) + ), + consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + barrier_storage=storage.p1_mma_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + s0_corr_producer, s0_corr_consumer = pipeline.PipelineAsync.create( + num_stages=self.softmax_corr_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len((*self.softmax0_warp_ids, self.mma_warp_id)) + ), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.correction_warp_ids) + ), + barrier_storage=storage.s0_corr_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + s1_corr_producer, s1_corr_consumer = pipeline.PipelineAsync.create( + num_stages=self.softmax_corr_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len((*self.softmax1_warp_ids, self.mma_warp_id)) + ), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.correction_warp_ids) + ), + barrier_storage=storage.s1_corr_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + corr_epi_producer, corr_epi_consumer = pipeline.PipelineAsync.create( + num_stages=self.epi_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.correction_warp_ids) + ), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len([self.epilogue_warp_id]) + ), + barrier_storage=storage.corr_epi_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + mma_corr_producer, mma_corr_consumer = pipeline.PipelineUmmaAsync.create( + num_stages=self.mma_corr_stage, + producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.correction_warp_ids) + ), + barrier_storage=storage.mma_corr_mbar_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + s0_p1_inplace_producer, s0_p1_inplace_consumer = pipeline.PipelineAsync.create( + num_stages=self.p_mma_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax0_warp_ids) + ), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax1_warp_ids) + ), + barrier_storage=storage.s0_p1_inplace_barrier_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + s1_p0_inplace_producer, s1_p0_inplace_consumer = pipeline.PipelineAsync.create( + num_stages=self.p_mma_stage, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax0_warp_ids) + ), + consumer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax1_warp_ids) + ), + barrier_storage=storage.s1_p0_inplace_barrier_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + # QK SF TMEM-slot release pipelines: softmax_{1-j} (producer) signals + # MMA (consumer) once its T2R-load of S_{1-j} is done, so mma_qk(j)'s + # SFQ/SFK S2T copy is safe to overwrite the tail of TMEM[S_{1-j}]. + qk_sf_inplace_0_producer, qk_sf_inplace_0_consumer = pipeline.PipelineAsync.create( + num_stages=1, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax1_warp_ids) + ), + consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + barrier_storage=storage.qk_sf_inplace_0_barrier_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + qk_sf_inplace_1_producer, qk_sf_inplace_1_consumer = pipeline.PipelineAsync.create( + num_stages=1, + producer_group=make_thread_cooperative_group( + self.threads_per_warp * len(self.softmax0_warp_ids) + ), + consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), + barrier_storage=storage.qk_sf_inplace_1_barrier_ptr.data_ptr(), + defer_sync=True, + ).make_participants() + tmem = utils.TmemAllocator( + storage.tmem_holding_buf.ptr, + barrier_for_retrieve=self.tmem_alloc_barrier, + # Correction warp is the last one that accesses tmem + allocator_warp_id=self.correction_warp_ids[0], + arch=self.arch_str, + ) + pipeline_init_arrive(is_relaxed=True) + + # Generate smem tensor Q/K/V/O + # (MMA, MMA_Q, MMA_D, PIPE) + sQ = storage.sQ.get_tensor(q_smem_layout_staged.outer, swizzle=q_smem_layout_staged.inner) + # (MMA, MMA_Q, MMA_D, PIPE) + sQSF = storage.sQSF.get_tensor(q_sf_smem_layout_staged) + # (MMA, MMA_K, MMA_D, PIPE) + sK = storage.sK.get_tensor(k_smem_layout_staged.outer, swizzle=k_smem_layout_staged.inner) + # (MMA, MMA_K, MMA_D, PIPE) + sKSF = storage.sKSF.get_tensor(k_sf_smem_layout_staged) + # (MMA, MMA_K, MMA_D, PIPE) + # Reuse k's smem buffer for v. Recast element type so MMA descriptor matches v_dtype. + sV_ptr = cute.recast_ptr( + cute.recast_ptr(sK.iterator, dtype=self.v_dtype), + v_smem_layout_staged.inner, + ) + sV = cute.make_tensor(sV_ptr, v_smem_layout_staged.outer) + sO = storage.sO.get_tensor(o_smem_layout_staged.outer, swizzle=o_smem_layout_staged.inner) + s0_warp_wants_skip_softmax_exchange = ( + storage.s0_warp_wants_skip_softmax_exchange.get_tensor(cute.make_layout((4,))) + ) + s1_warp_wants_skip_softmax_exchange = ( + storage.s1_warp_wants_skip_softmax_exchange.get_tensor(cute.make_layout((4,))) + ) + + qk_thr_mma = qk_tiled_mma.get_slice(0) # default 1sm + pv_thr_mma = pv_tiled_mma.get_slice(0) # default 1sm + tSrQ = qk_thr_mma.make_fragment_A(sQ) + tSrK = qk_thr_mma.make_fragment_B(sK) + tOrV = pv_thr_mma.make_fragment_B(sV) + + def make_tmem_tensors( + self, + qk_thr_mma: cute.TiledMma, + pv_thr_mma: cute.TiledMma, + p_tmem_layout_staged: cute.Layout, + tmem_ptr: cute.Pointer, + ): + qk_acc_shape = qk_thr_mma.partition_shape_C( + (self.qk_mma_tiler[0], self.qk_mma_tiler[1]) + ) + tStS_fake = qk_thr_mma.make_fragment_C(qk_acc_shape) + tStS = cute.make_tensor(tmem_ptr + self.tmem_s0_offset, tStS_fake.layout) + pv_acc_shape = pv_thr_mma.partition_shape_C( + (self.pv_mma_tiler[0], self.pv_mma_tiler[1]) + ) + tOtO = pv_thr_mma.make_fragment_C(pv_acc_shape) + tStS0 = cute.make_tensor(tmem_ptr + self.tmem_s0_offset, tStS.layout) + tStS1 = cute.make_tensor(tmem_ptr + self.tmem_s1_offset, tStS.layout) + tOtO0 = cute.make_tensor(tmem_ptr + self.tmem_o0_offset, tOtO.layout) + tOtO1 = cute.make_tensor(tmem_ptr + self.tmem_o1_offset, tOtO.layout) + tP = cute.make_tensor(tStS.iterator, p_tmem_layout_staged.outer) + tOrP = pv_thr_mma.make_fragment_A(tP)[None, None, None, 0] + tOrP0 = cute.make_tensor( + cute.recast_ptr( + tmem_ptr + self.tmem_p0_offset, + dtype=tOrP.dtype, + ), + tOrP.layout, + ) + tOrP1 = cute.make_tensor( + cute.recast_ptr( + tmem_ptr + self.tmem_p1_offset, + dtype=tOrP.dtype, + ), + tOrP.layout, + ) + return tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 + + tile_sched = fmha_utils.create_fmha_static_tile_scheduler( + tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() + ) + work_tile = tile_sched.initial_work_tile_info() + pipeline_init_wait() + softmax_fn = partial( + self.softmax, + qk_thr_mma=qk_thr_mma, + value_args=( + mK_kdl.shape[0], + mQ_qdl.shape[0], + scale_softmax_log2, + skip_softmax_threshold_log2, + ), + mask_args=(window_size_left, window_size_right), + sched_args=(tile_sched, work_tile), + # Each softmax warpgroup picks its producer by stage: softmax0 + # (stage=0) produces on qk_sf_inplace_1, softmax1 (stage=1) on + # qk_sf_inplace_0. + qk_sf_inplace_producers=( + qk_sf_inplace_1_producer, + qk_sf_inplace_0_producer, + ), + ) + # /////////////////////////////////////////////////////////////////////////////// + # EMPTY + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx == self.empty_warp_id: + cute.arch.setmaxregister_decrease(self.num_regs_other) + + # /////////////////////////////////////////////////////////////////////////////// + # LOAD + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx == self.load_warp_id: + cute.arch.setmaxregister_decrease(self.num_regs_other) + cute.arch.griddepcontrol_wait() + while work_tile.is_valid_tile: + curr_block_coord = work_tile.tile_idx + batch_coord = curr_block_coord[2][1] + continue_cond = False + cuseqlen_q = Int32(0) + seqlen_q = mQ_qdl.shape[0] + seqlen_k = mK_kdl.shape[0] + if cutlass.const_expr(cum_seqlen_q is not None): + cuseqlen_q = cum_seqlen_q[batch_coord] + seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q + continue_cond = ( + not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( + self.cta_tiler[0], + curr_block_coord[0], + seqlen_q, + ) + ) + if not continue_cond: + if cutlass.const_expr(cum_seqlen_k is not None): + seqlen_k = cum_seqlen_k[batch_coord + 1] - cum_seqlen_k[batch_coord] + continue_cond = seqlen_k <= 0 + if not continue_cond: + mQ_qdl_ = mQ_qdl + mK_kdl_ = mK_kdl + mQSF_qdl_ = mQSF_qdl + mKSF_kdl_ = mKSF_kdl + mV_dkl_ = mV_dkl + if cutlass.const_expr(cum_seqlen_q is not None): + mQ_qdl_ = cute.domain_offset( + (cum_seqlen_q[batch_coord], 0, ((0, 0), 0)), mQ_qdl + ) + mQSF_qdl_ = cute.domain_offset( + (cum_seqlen_q[batch_coord], 0, (0, 0)), mQSF_qdl + ) + if cutlass.const_expr(cum_seqlen_k is not None): + mK_kdl_ = cute.domain_offset( + (cum_seqlen_k[batch_coord], 0, ((0, 0), 0)), mK_kdl + ) + mKSF_kdl_ = cute.domain_offset( + (cum_seqlen_k[batch_coord], 0, (0, 0)), mKSF_kdl + ) + mV_dkl_ = cute.domain_offset( + (0, cum_seqlen_k[batch_coord], ((0, 0), 0)), mV_dkl + ) + # Local tile partition global tensors + gQ_qdl = cute.flat_divide(mQ_qdl_, cute.select(self.qk_mma_tiler, mode=[0, 2])) + tSgQ_qdl = qk_thr_mma.partition_A(gQ_qdl) + tQsQ, tQgQ_qdl = cute.nvgpu.cpasync.tma_partition( + tma_atom_q, + 0, # no multicast + cute.make_layout(1), + cute.group_modes(sQ, 0, 3), + cute.group_modes(tSgQ_qdl, 0, 3), + ) + tQgQ = tQgQ_qdl[None, None, 0, curr_block_coord[2]] + gQSF_qdl = cute.local_tile( + mQSF_qdl_, + cute.select(self.qk_mma_tiler, mode=[0, 2]), + (None, None, None), + ) + tSgQSF_qdl = qk_thr_mma.partition_A(gQSF_qdl) + tQsQSF, tQgQSF_qdl = cute.nvgpu.cpasync.tma_partition( + tma_atom_q_sf, + 0, # no multicast + cute.make_layout(1), + cute.group_modes(sQSF, 0, 3), + cute.group_modes(tSgQSF_qdl, 0, 3), + ) + tQsQSF = cute.filter_zeros(tQsQSF) + tQgQSF_qdl = cute.filter_zeros(tQgQSF_qdl) + # tile_atom_to_shape_SF coalesces Q's ((h_r, h_k), b) mode into (1, h_r*h_k*b), + # One needs to reconstruct the correct layout here to correctly map block_coord + # onto this linearlized L-like mode. + sf_l_q = cute.make_layout( + mQ_qdl.shape[2], + stride=( + (1, mQ_qdl.shape[2][0][0]), + cute.size(mQ_qdl.shape[2][0]), + ), + )(curr_block_coord[2]) + tQgQSF = tQgQSF_qdl[None, None, 0, sf_l_q] + gK_kdl = cute.flat_divide(mK_kdl_, cute.select(self.qk_mma_tiler, mode=[1, 2])) + tSgK_kdl = qk_thr_mma.partition_B(gK_kdl) + tKsK, tKgK_kdl = cute.nvgpu.cpasync.tma_partition( + tma_atom_k, + 0, # no multicast + cute.make_layout(1), + cute.group_modes(sK, 0, 3), + cute.group_modes(tSgK_kdl, 0, 3), + ) + tKgK = tKgK_kdl[None, None, 0, curr_block_coord[2]] + gKSF_kdl = cute.local_tile( + mKSF_kdl_, + cute.select(self.qk_mma_tiler, mode=[1, 2]), + (None, None, None), + ) + tSgKSF_kdl = qk_thr_mma.partition_B(gKSF_kdl) + tKsKSF, tKgKSF_kdl = cute.nvgpu.cpasync.tma_partition( + tma_atom_k_sf, + 0, # no multicast + cute.make_layout(1), + cute.group_modes(sKSF, 0, 3), + cute.group_modes(tSgKSF_kdl, 0, 3), + ) + tKsKSF = cute.filter_zeros(tKsKSF) + tKgKSF_kdl = cute.filter_zeros(tKgKSF_kdl) + # Same L-mode trick as Q's SF (see comment above). K's SF is shared across h_r heads + # (it indexes h_k, not h_q), which we express by a 0-stride for the h_r sub-mode. + sf_l_k = cute.make_layout( + mK_kdl.shape[2], + stride=((0, 1), mK_kdl.shape[2][0][1]), + )(curr_block_coord[2]) + tKgKSF = tKgKSF_kdl[None, None, 0, sf_l_k] + gV_dkl = cute.flat_divide(mV_dkl_, cute.select(self.pv_mma_tiler, mode=[1, 2])) + tSgV_dkl = pv_thr_mma.partition_B(gV_dkl) + tVsV, tVgV_dkl = cute.nvgpu.cpasync.tma_partition( + tma_atom_v, + 0, # no multicast + cute.make_layout(1), + cute.group_modes(sV, 0, 3), + cute.group_modes(tSgV_dkl, 0, 3), + ) + tVgV = tVgV_dkl[None, 0, None, curr_block_coord[2]] + seqlen_kv_loop_start = fmha_utils.FusedMask.get_trip_start( + self.mask_type, + curr_block_coord, + self.cta_tiler, + seqlen_q, + seqlen_k, + window_size_left, + ) + # Q0 + q0_coord = 2 * curr_block_coord[0] + q0_handle = load_q_producer.acquire_and_advance() + cute.copy( + tma_atom_q, + tQgQ[None, q0_coord], + tQsQ[None, q0_handle.index], + tma_bar_ptr=q0_handle.barrier, + ) + cute.copy( + tma_atom_q_sf, + tQgQSF[None, q0_coord], + tQsQSF[None, q0_handle.index], + tma_bar_ptr=q0_handle.barrier, + ) + seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( + self.mask_type, + curr_block_coord, + self.cta_tiler, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + # K0 + kv_coord = seqlen_kv_loop_start + k_handle = load_kv_producer.acquire_and_advance() + cute.copy( + tma_atom_k, + tKgK[None, kv_coord], + tKsK[None, k_handle.index], + tma_bar_ptr=k_handle.barrier, + ) + cute.copy( + tma_atom_k_sf, + tKgKSF[None, kv_coord], + tKsKSF[None, k_handle.index], + tma_bar_ptr=k_handle.barrier, + ) + # Q1 + q1_coord = q0_coord + 1 + q1_handle = load_q_producer.acquire_and_advance() + cute.copy( + tma_atom_q, + tQgQ[None, q1_coord], + tQsQ[None, q1_handle.index], + tma_bar_ptr=q1_handle.barrier, + ) + cute.copy( + tma_atom_q_sf, + tQgQSF[None, q1_coord], + tQsQSF[None, q1_handle.index], + tma_bar_ptr=q1_handle.barrier, + ) + kv_coord += 1 + + for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): + # Ki + k_handle = load_kv_producer.acquire_and_advance() + cute.copy( + tma_atom_k, + tKgK[None, kv_coord], + tKsK[None, k_handle.index], + tma_bar_ptr=k_handle.barrier, + ) + cute.copy( + tma_atom_k_sf, + tKgKSF[None, kv_coord], + tKsKSF[None, k_handle.index], + tma_bar_ptr=k_handle.barrier, + ) + # Vi-1 + v_handle, load_kv_producer = self.kv_producer_update_tx_acquire_and_advance( + load_kv_producer, + load_kv_empty_mbar_ptr, + load_kv_full_mbar_ptr, + self.tma_copy_v_bytes, + ) + cute.copy( + tma_atom_v, + tVgV[None, kv_coord - 1], + tVsV[None, v_handle.index], + tma_bar_ptr=load_kv_full_mbar_ptr + v_handle.index, + ) + kv_coord += 1 + # End of seqlen_kv loop + # Vi_end + v_handle, load_kv_producer = self.kv_producer_update_tx_acquire_and_advance( + load_kv_producer, + load_kv_empty_mbar_ptr, + load_kv_full_mbar_ptr, + self.tma_copy_v_bytes, + ) + cute.copy( + tma_atom_v, + tVgV[None, kv_coord - 1], + tVsV[None, v_handle.index], + tma_bar_ptr=load_kv_full_mbar_ptr + v_handle.index, + ) + # End of if not continue_cond + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + # End of persistent scheduler loop + # /////////////////////////////////////////////////////////////////////////////// + # MMA + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx == self.mma_warp_id: + cute.arch.setmaxregister_decrease(self.num_regs_other) + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) + tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( + self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr + ) + q_sf_tmem_layout = blockscaled_utils.make_tmem_layout_sfa( + qk_tiled_mma, + self.qk_mma_tiler, + self.qk_sf_vec_size, + cute.slice_(q_sf_smem_layout_staged, (None, None, None, 0)), + ) + k_sf_tmem_layout = blockscaled_utils.make_tmem_layout_sfb( + qk_tiled_mma, + self.qk_mma_tiler, + self.qk_sf_vec_size, + cute.slice_(k_sf_smem_layout_staged, (None, None, None, 0)), + ) + # TMEM offsets are in u32 columns. Follow FA4's explicit SFQ footprint + # calculation instead of deriving the offset from the recast FP8 tensor + # view, which can under-count for block-scaled layouts. + sf_atom_mn = 32 + q_sf_tmem_cols = (self.qk_mma_tiler[0] // sf_atom_mn) * self.qk_mma_inst_tile_k + k_sf_tmem_cols = ( + ((self.qk_mma_tiler[1] + 127) // 128 * 128) // sf_atom_mn + ) * self.qk_mma_inst_tile_k + tCtQSF0 = cute.make_tensor( + cute.recast_ptr(tmem_ptr + self.tmem_qk0_sf_offset, dtype=self.qk_sf_dtype), + q_sf_tmem_layout, + ) + tCtKSF0 = cute.make_tensor( + cute.recast_ptr( + tmem_ptr + self.tmem_qk0_sf_offset + q_sf_tmem_cols, + dtype=self.qk_sf_dtype, + ), + k_sf_tmem_layout, + ) + tCtQSF1 = cute.make_tensor( + cute.recast_ptr(tmem_ptr + self.tmem_qk1_sf_offset, dtype=self.qk_sf_dtype), + q_sf_tmem_layout, + ) + tCtKSF1 = cute.make_tensor( + cute.recast_ptr( + tmem_ptr + self.tmem_qk1_sf_offset + q_sf_tmem_cols, + dtype=self.qk_sf_dtype, + ), + k_sf_tmem_layout, + ) + tiled_copy_s2t_qsf0, tCsQSF_s2t, tCtQSF0_s2t = self.mainloop_s2t_copy_and_partition( + sQSF, tCtQSF0 + ) + tiled_copy_s2t_ksf0, tCsKSF_s2t, tCtKSF0_s2t = self.mainloop_s2t_copy_and_partition( + sKSF, tCtKSF0 + ) + tiled_copy_s2t_qsf1, tCsQSF1_s2t, tCtQSF1_s2t = self.mainloop_s2t_copy_and_partition( + sQSF, tCtQSF1 + ) + tiled_copy_s2t_ksf1, tCsKSF1_s2t, tCtKSF1_s2t = self.mainloop_s2t_copy_and_partition( + sKSF, tCtKSF1 + ) + enable_skip_softmax = skip_softmax_threshold_log2 is not None + tiled_tmem_load_v = None + tTMEM_LOADtS_v0, tTMEM_LOADtS_v1 = None, None + tTMEM_LOADrS_v0, tTMEM_LOADrS_v1 = None, None + if cutlass.const_expr(enable_skip_softmax): + cS = cute.make_identity_tensor(cute.select(self.qk_mma_tiler, mode=[0, 1])) + tScS = qk_thr_mma.partition_C(cS) + tStS_v = cute.composition(tStS, cute.make_layout((self.threads_per_warp, 1))) + tScS_v = cute.composition(tScS, cute.make_layout((self.threads_per_warp, 1))) + tmem_load_v_atom = cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(1)), + self.qk_acc_dtype, + ) + thread_idx = tidx % self.threads_per_warp + + tiled_tmem_load_v = tcgen05.make_tmem_copy(tmem_load_v_atom, tStS_v) + thr_tmem_load_v = tiled_tmem_load_v.get_slice(thread_idx) + tTMEM_LOADtS_v = thr_tmem_load_v.partition_S(tStS_v) + tTMEM_LOADcS_v = thr_tmem_load_v.partition_D(tScS_v) + tTMEM_LOADrS_v0 = cute.make_rmem_tensor(tTMEM_LOADcS_v.shape, self.qk_acc_dtype) + tTMEM_LOADrS_v1 = cute.make_rmem_tensor(tTMEM_LOADcS_v.shape, self.qk_acc_dtype) + tTMEM_LOADtS_v0 = cute.make_tensor( + tTMEM_LOADtS_v.iterator + self.tmem_skip_softmax0_offset, + tTMEM_LOADtS_v.layout, + ) + tTMEM_LOADtS_v1 = cute.make_tensor( + tTMEM_LOADtS_v.iterator + self.tmem_skip_softmax1_offset, + tTMEM_LOADtS_v.layout, + ) + + while work_tile.is_valid_tile: + curr_block_coord = work_tile.tile_idx + batch_coord = curr_block_coord[2][1] + continue_cond = False + seqlen_q = mQ_qdl.shape[0] + seqlen_k = mK_kdl.shape[0] + if cutlass.const_expr(cum_seqlen_q is not None): + cuseqlen_q = cum_seqlen_q[batch_coord] + seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q + continue_cond = ( + not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( + self.cta_tiler[0], + curr_block_coord[0], + seqlen_q, + ) + ) + if not continue_cond: + if cutlass.const_expr(cum_seqlen_k is not None): + cuseqlen_k = cum_seqlen_k[batch_coord] + seqlen_k = cum_seqlen_k[batch_coord + 1] - cuseqlen_k + continue_cond = seqlen_k <= 0 + if not continue_cond: + # Wait for Q0 + q0_handle = load_q_consumer.wait_and_advance() + tSrQ0 = tSrQ[None, None, None, q0_handle.index] + # Wait for K0 + k_handle = load_kv_consumer.wait_and_advance() + tSrK0 = tSrK[None, None, None, k_handle.index] + q0_sf_stage = (None, None, None, None, q0_handle.index) + k_sf_stage = (None, None, None, None, k_handle.index) + # GEMM_QK00 (Q0 * K0 -> S0) + mma_s0_producer, s0_corr_producer, qk_sf_inplace_0_consumer = self.mma_qk( + qk_tiled_mma, + (tSrQ0, tSrK0, tStS0), + ( + tiled_copy_s2t_qsf0, + tCsQSF_s2t[q0_sf_stage], + tCtQSF0_s2t, + tCtQSF0, + tiled_copy_s2t_ksf0, + tCsKSF_s2t[k_sf_stage], + tCtKSF0_s2t, + tCtKSF0, + ), + (mma_s0_producer, s0_corr_producer), + qk_sf_inplace_0_consumer, + ) + # Wait for Q1 + q1_handle = load_q_consumer.wait_and_advance() + tSrQ1 = tSrQ[None, None, None, q1_handle.index] + q1_sf_stage = (None, None, None, None, q1_handle.index) + # GEMM_QK10 (Q1 * K0 -> S1), K0 is ready in GEMM_QK00 + mma_s1_producer, s1_corr_producer, qk_sf_inplace_1_consumer = self.mma_qk( + qk_tiled_mma, + (tSrQ1, tSrK0, tStS1), + ( + tiled_copy_s2t_qsf1, + tCsQSF1_s2t[q1_sf_stage], + tCtQSF1_s2t, + tCtQSF1, + tiled_copy_s2t_ksf1, + tCsKSF1_s2t[k_sf_stage], + tCtKSF1_s2t, + tCtKSF1, + ), + (mma_s1_producer, s1_corr_producer), + qk_sf_inplace_1_consumer, + ) + # Release K0 + k_handle.release() + # Note: Q0 & Q1 are still needed in the seqlen_kv loop + # so we need to release them after the seqlen_kv loop + seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( + self.mask_type, + curr_block_coord, + self.cta_tiler, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + # O1 hasn't been accumulated yet, its first MMA calculation doesn't need to accumulate + pv_whether_acc = False + for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): + # Wait for Ki + k_handle = load_kv_consumer.wait_and_advance() + tSrKi = tSrK[None, None, None, k_handle.index] + k_sf_stage = (None, None, None, None, k_handle.index) + # GEMM_QK0i (Q0 * Ki -> S0) + mma_s0_producer, s0_corr_producer, qk_sf_inplace_0_consumer = self.mma_qk( + qk_tiled_mma, + (tSrQ0, tSrKi, tStS0), + ( + tiled_copy_s2t_qsf0, + tCsQSF_s2t[q0_sf_stage], + tCtQSF0_s2t, + tCtQSF0, + tiled_copy_s2t_ksf0, + tCsKSF_s2t[k_sf_stage], + tCtKSF0_s2t, + tCtKSF0, + ), + (mma_s0_producer, s0_corr_producer), + qk_sf_inplace_0_consumer, + ) + # Wait for Vi-1 + v_handle = load_kv_consumer.wait_and_advance() + tOrVi = tOrV[None, None, None, v_handle.index] + # GEMM_PV0(i-1) (P0 * Vi-1 -> O0_partial) + mma_corr_producer, p0_mma_consumer = self.mma_pv( + pv_tiled_mma, + pv_whether_acc, + (tOrP0, tOrVi, tOtO0), + (mma_corr_producer, p0_mma_consumer), + ( + enable_skip_softmax, + tiled_tmem_load_v, + tTMEM_LOADtS_v0, + tTMEM_LOADrS_v0, + ), + ) + # GEMM_QK1i (Q1 * Ki -> S1) + mma_s1_producer, s1_corr_producer, qk_sf_inplace_1_consumer = self.mma_qk( + qk_tiled_mma, + (tSrQ1, tSrKi, tStS1), + ( + tiled_copy_s2t_qsf1, + tCsQSF1_s2t[q1_sf_stage], + tCtQSF1_s2t, + tCtQSF1, + tiled_copy_s2t_ksf1, + tCsKSF1_s2t[k_sf_stage], + tCtKSF1_s2t, + tCtKSF1, + ), + (mma_s1_producer, s1_corr_producer), + qk_sf_inplace_1_consumer, + ) + # Release Ki + k_handle.release() + # GEMM_PV1(i-1) (P1 * Vi-1 -> O1_partial) + mma_corr_producer, p1_mma_consumer = self.mma_pv( + pv_tiled_mma, + pv_whether_acc, + (tOrP1, tOrVi, tOtO1), + (mma_corr_producer, p1_mma_consumer), + ( + enable_skip_softmax, + tiled_tmem_load_v, + tTMEM_LOADtS_v1, + tTMEM_LOADrS_v1, + ), + ) + pv_whether_acc = True + # Release Vi-1 + v_handle.release() + # End of seqlen_kv loop + # release Q0 & Q1 + q0_handle.release() + q1_handle.release() + # Wait for Vi_end + v_handle = load_kv_consumer.wait_and_advance() + tOrVi = tOrV[None, None, None, v_handle.index] + # GEMM_PV0(i_end) (P0 * Vi_end -> O0) + mma_corr_producer, p0_mma_consumer = self.mma_pv( + pv_tiled_mma, + pv_whether_acc, + (tOrP0, tOrVi, tOtO0), + (mma_corr_producer, p0_mma_consumer), + ( + enable_skip_softmax, + tiled_tmem_load_v, + tTMEM_LOADtS_v0, + tTMEM_LOADrS_v0, + ), + ) + # GEMM_PV1(i_end) (P1 * Vi_end -> O1) + mma_corr_producer, p1_mma_consumer = self.mma_pv( + pv_tiled_mma, + pv_whether_acc, + (tOrP1, tOrVi, tOtO1), + (mma_corr_producer, p1_mma_consumer), + ( + enable_skip_softmax, + tiled_tmem_load_v, + tTMEM_LOADtS_v1, + tTMEM_LOADrS_v1, + ), + ) + # Release Vi_end + v_handle.release() + # Empty step for correction epilog + vec0_handle = s0_corr_producer.acquire_and_advance() + vec0_handle.commit() + vec1_handle = s1_corr_producer.acquire_and_advance() + vec1_handle.commit() + # End of if not continue_cond + # Advance to next tile + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + # End of persistent scheduler loop + # /////////////////////////////////////////////////////////////////////////////// + # Epilogue (TMA store path only) + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx == self.epilogue_warp_id: + cute.arch.setmaxregister_decrease(self.num_regs_other) + if cutlass.const_expr(self.use_tma_store): + while work_tile.is_valid_tile: + curr_block_coord = work_tile.tile_idx + batch_coord = curr_block_coord[2][1] + continue_cond = False + cuseqlen_q = Int32(0) + seqlen_q = mQ_qdl.shape[0] + + if cutlass.const_expr(cum_seqlen_q is not None): + cuseqlen_q = cum_seqlen_q[batch_coord] + seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q + continue_cond = ( + not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( + self.cta_tiler[0], + curr_block_coord[0], + seqlen_q, + ) + ) + if not continue_cond: + mO_qdl_ = mO_qdl + if cutlass.const_expr(cum_seqlen_q is not None): + mO_qdl_ = cute.domain_offset( + (cum_seqlen_q[batch_coord], 0, ((0, 0), 0)), mO_qdl + ) + + o0_coord = 2 * curr_block_coord[0] + o1_coord = o0_coord + 1 + gO_qdl = cute.flat_divide( + mO_qdl_, cute.select(self.pv_mma_tiler, mode=[0, 1]) + ) + gO = gO_qdl[None, None, None, 0, curr_block_coord[2]] + tOsO, tOgO = cute.nvgpu.cpasync.tma_partition( + tma_atom_o, + 0, + cute.make_layout(1), + cute.group_modes(sO, 0, 2), + cute.group_modes(gO, 0, 2), + ) + + # O0 O1 using the same pipeline + # wait from corr, issue tma store on smem + # O0 + # 1. Wait for O0 final + o0_handle = corr_epi_consumer.wait_and_advance() + # 2. Copy O0 to gmem + cute.copy(tma_atom_o, tOsO[None, 0], tOgO[None, o0_coord]) + cute.arch.cp_async_bulk_commit_group() + # O1 + # 1. Wait for O1 final + o1_handle = corr_epi_consumer.wait_and_advance() + # 2. Copy O1 to gmem + cute.copy(tma_atom_o, tOsO[None, 1], tOgO[None, o1_coord]) + cute.arch.cp_async_bulk_commit_group() + + # Ensure O0 buffer is ready to be released + cute.arch.cp_async_bulk_wait_group(1, read=True) + o0_handle.release() + # Ensure O1 buffer is ready to be released + cute.arch.cp_async_bulk_wait_group(0, read=True) + o1_handle.release() + + # Advance to next tile + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + cute.arch.griddepcontrol_launch_dependents() + # End of persistent scheduler loop + # /////////////////////////////////////////////////////////////////////////////// + # Softmax0 + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx < self.softmax1_warp_ids[0]: + cute.arch.setmaxregister_increase(self.num_regs_softmax) + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) + tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( + self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr + ) + softmax_fn( + stage=0, + tensor_args=( + tStS, + tStS0, + cum_seqlen_k, + cum_seqlen_q, + s0_warp_wants_skip_softmax_exchange, + skip_softmax_count, + total_softmax_count, + ), + pipeline_args=(mma_s0_consumer, s0_corr_producer, p0_mma_producer), + inplace_args=(s0_p1_inplace_producer, s1_p0_inplace_consumer), + ) + + # /////////////////////////////////////////////////////////////////////////////// + # Softmax1 + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx < self.correction_warp_ids[0] and warp_idx >= self.softmax1_warp_ids[0]: + cute.arch.setmaxregister_increase(self.num_regs_softmax) + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) + tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( + self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr + ) + softmax_fn( + stage=1, + tensor_args=( + tStS, + tStS1, + cum_seqlen_k, + cum_seqlen_q, + s1_warp_wants_skip_softmax_exchange, + skip_softmax_count, + total_softmax_count, + ), + pipeline_args=(mma_s1_consumer, s1_corr_producer, p1_mma_producer), + inplace_args=(s1_p0_inplace_producer, s0_p1_inplace_consumer), + ) + + # /////////////////////////////////////////////////////////////////////////////// + # Correction + # /////////////////////////////////////////////////////////////////////////////// + if warp_idx >= self.correction_warp_ids[0] and warp_idx < self.mma_warp_id: + cute.arch.setmaxregister_decrease(self.num_regs_correction) + tmem.allocate(self.num_tmem_alloc_cols) + tmem.wait_for_alloc() + tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) + tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( + self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr + ) + cS = cute.make_identity_tensor((self.qk_mma_tiler[0], self.qk_mma_tiler[1])) + tScS = qk_thr_mma.partition_C(cS) + + tStS_vec_layout = cute.composition(tStS.layout, cute.make_layout((128, 2))) + + tStS_vec0 = cute.make_tensor(tStS.iterator + self.tmem_vec0_offset, tStS_vec_layout) + tStS_vec1 = cute.make_tensor(tStS.iterator + self.tmem_vec1_offset, tStS_vec_layout) + + tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) + tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) + tmem_load_v_atom = cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(2)), + self.qk_acc_dtype, + ) + tiled_tmem_load_vec = tcgen05.make_tmem_copy(tmem_load_v_atom, tStS_vec0) + thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) + thr_tmem_load_vec = tiled_tmem_load_vec.get_slice(thread_idx) + tTMEM_LOAD_VECtS0 = thr_tmem_load_vec.partition_S(tStS_vec0) + tTMEM_LOAD_VECtS1 = thr_tmem_load_vec.partition_S(tStS_vec1) + tTMEM_LOAD_VECcS = thr_tmem_load_vec.partition_D(tScS_vec) + while work_tile.is_valid_tile: + curr_block_coord = work_tile.tile_idx + batch_coord = curr_block_coord[2][1] + seqlen_k = mK_kdl.shape[0] + row_idx = Int32(0) + continue_cond = False + cuseqlen_q = Int32(0) + seqlen_q = mQ_qdl.shape[0] + + if cutlass.const_expr(cum_seqlen_q is not None): + cuseqlen_q = cum_seqlen_q[batch_coord] + seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q + continue_cond = ( + not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( + self.cta_tiler[0], + curr_block_coord[0], + seqlen_q, + ) + ) + if not continue_cond: + row_idx = curr_block_coord[0] * self.cta_tiler[0] + tTMEM_LOAD_VECcS[0][0] + if cutlass.const_expr(cum_seqlen_k is not None): + cuseqlen_k = cum_seqlen_k[batch_coord] + seqlen_k = cum_seqlen_k[batch_coord + 1] - cuseqlen_k + continue_cond = seqlen_k <= 0 + if not continue_cond: + # Ignore first signal from softmax as no correction is required + vec0_handle = s0_corr_consumer.wait_and_advance() + vec0_handle.release() + vec1_handle = s1_corr_consumer.wait_and_advance() + vec1_handle.release() + # O0/O1 share the same mma_corr consumer state, so the Oi + # peek token rolls from O0 -> O1 -> next O0. Seed with a + # blocking token; the rescale helper refreshes it near the + # end of each iteration. + oi_peek_status = cutlass.Boolean(False) + seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( + self.mask_type, + curr_block_coord, + self.cta_tiler, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): + # Rescale O0 + ( + (s0_corr_consumer, mma_corr_consumer), + oi_peek_status, + ) = self.correction_rescale( + pv_thr_mma, + tiled_tmem_load_vec, + scale_softmax_log2, + (tOtO0, tTMEM_LOAD_VECtS0, tTMEM_LOAD_VECcS), + (s0_corr_consumer, mma_corr_consumer), + oi_peek_status, + ) + # Rescale O1 + ( + (s1_corr_consumer, mma_corr_consumer), + oi_peek_status, + ) = self.correction_rescale( + pv_thr_mma, + tiled_tmem_load_vec, + scale_softmax_log2, + (tOtO1, tTMEM_LOAD_VECtS1, tTMEM_LOAD_VECcS), + (s1_corr_consumer, mma_corr_consumer), + oi_peek_status, + ) + # End of seqlen_corr_loop_steps + value_args = ( + cuseqlen_q, + seqlen_q, + curr_block_coord, + scale_softmax, + scale_output, + ) + if cutlass.const_expr(self.use_tma_store): + # TMA store path: write to sO, signal epilogue warp + # Normalize O0 + s0_corr_consumer, mma_corr_consumer, corr_epi_producer = ( + self.correction_epilog( + pv_thr_mma, + tiled_tmem_load_vec, + ( + tOtO0, + tTMEM_LOAD_VECtS0, + tTMEM_LOAD_VECcS, + sO[None, None, 0], + mLSE, + mSink, + m_scale_v_channels, + ), + ( + s0_corr_consumer, + mma_corr_consumer, + corr_epi_producer, + ), + (row_idx, *value_args), + ) + ) + row_idx += self.qk_mma_tiler[0] + # Normalize O1 + s1_corr_consumer, mma_corr_consumer, corr_epi_producer = ( + self.correction_epilog( + pv_thr_mma, + tiled_tmem_load_vec, + ( + tOtO1, + tTMEM_LOAD_VECtS1, + tTMEM_LOAD_VECcS, + sO[None, None, 1], + mLSE, + mSink, + m_scale_v_channels, + ), + ( + s1_corr_consumer, + mma_corr_consumer, + corr_epi_producer, + ), + (row_idx, *value_args), + ) + ) + else: + # st.global path: store directly to global memory + block_offset_o = Int32(0) + if cutlass.const_expr(cum_seqlen_q is not None): + block_offset_o = cum_seqlen_q[batch_coord] + mO_ = cute.make_tensor( + mO.iterator + block_offset_o * mO.stride[0], + cute.make_layout( + (seqlen_q, mO.shape[1], mO.shape[2]), + stride=mO.stride, + ), + ) + o0_coord = 2 * curr_block_coord[0] + o1_coord = o0_coord + 1 + gO_stg = cute.local_tile( + mO_, + (self.pv_mma_tiler[0], self.pv_mma_tiler[1]), + (None, None, None), + ) + gO0 = gO_stg[None, None, o0_coord, 0, curr_block_coord[2]] + gO1 = gO_stg[None, None, o1_coord, 0, curr_block_coord[2]] + # Normalize O0 and store to global memory + s0_corr_consumer, mma_corr_consumer = self.correction_epilog( + pv_thr_mma, + tiled_tmem_load_vec, + ( + tOtO0, + tTMEM_LOAD_VECtS0, + tTMEM_LOAD_VECcS, + gO0, + mLSE, + mSink, + m_scale_v_channels, + ), + (s0_corr_consumer, mma_corr_consumer), + (row_idx, *value_args), + ) + row_idx += self.qk_mma_tiler[0] + # Normalize O1 and st.global to global memory + s1_corr_consumer, mma_corr_consumer = self.correction_epilog( + pv_thr_mma, + tiled_tmem_load_vec, + ( + tOtO1, + tTMEM_LOAD_VECtS1, + tTMEM_LOAD_VECcS, + gO1, + mLSE, + mSink, + m_scale_v_channels, + ), + (s1_corr_consumer, mma_corr_consumer), + (row_idx, *value_args), + ) + # End of if not continue_cond + # Advance to next tile + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + if cutlass.const_expr(not self.use_tma_store): + cute.arch.griddepcontrol_launch_dependents() + # End of persistent scheduler loop + tmem.relinquish_alloc_permit() + # Synchronize before TMEM dealloc (done by the caller) + self.tmem_dealloc_barrier.arrive_and_wait() + tmem.free(tmem_ptr) + return + + def mainloop_s2t_copy_and_partition( + self, + sSF: cute.Tensor, + tSF: cute.Tensor, + ) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]: + """Partition one block-scaled QK scale-factor tensor for SMEM to TMEM copy.""" + tCsSF_compact = cute.filter_zeros(sSF) + tCtSF_compact = cute.filter_zeros(tSF) + + copy_atom_s2t = cute.make_copy_atom( + tcgen05.Cp4x32x128bOp(tcgen05.CtaGroup.ONE), + self.qk_sf_dtype, + ) + tiled_copy_s2t = tcgen05.make_s2t_copy(copy_atom_s2t, tCtSF_compact) + thr_copy_s2t = tiled_copy_s2t.get_slice(0) + tCsSF_compact_s2t_ = thr_copy_s2t.partition_S(tCsSF_compact) + tCsSF_compact_s2t = tcgen05.get_s2t_smem_desc_tensor(tiled_copy_s2t, tCsSF_compact_s2t_) + tCtSF_compact_s2t = thr_copy_s2t.partition_D(tCtSF_compact) + return tiled_copy_s2t, tCsSF_compact_s2t, tCtSF_compact_s2t + + @cute.jit + def kv_producer_update_tx_acquire_and_advance( + self, tma_producer, empty_mbar_ptr, full_mbar_ptr, tx_bytes + ): + # This utility function is a special version of tma_producer.acquire_and_advance(). + # This is used to customize the tx bytes which is different from + # the initialized tx bytes of tma_producer. + state = tma_producer._PipelineProducer__state.clone() + cute.arch.mbarrier_wait(empty_mbar_ptr + state.index, state.phase) + with cute.arch.elect_one(): + cute.arch.mbarrier_arrive_and_expect_tx( + full_mbar_ptr + state.index, + tx_bytes, + ) + tma_producer.advance() + return state, tma_producer + + @cute.jit + def get_skip_softmax_flag(self, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v): + cute.copy(tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v) + tTMEM_LOADrS_v_i32 = cute.recast_tensor(tTMEM_LOADrS_v, dtype=cutlass.Int32) + skip_softmax_flag = cute.arch.make_warp_uniform(tTMEM_LOADrS_v_i32[0]) + return skip_softmax_flag + + @cute.jit + def mma_qk( + self, + tiled_mma: cute.TiledMma, + tensor_args: Tuple, + scale_args: Tuple, + pipeline_args: Tuple, + qk_sf_inplace_consumer: pipeline.PipelineConsumer, + pipeline_tokens: Tuple = (None, None), + ) -> Tuple[ + pipeline.PipelineProducer, + pipeline.PipelineProducer, + pipeline.PipelineConsumer, + ]: + """Perform a single step of the QK GEMM computation on a block of attention scores. + + :param tiled_mma: Tiled MMA for QK GEMM + :type tiled_mma: cute.TiledMma + :param tensor_args: Tuple containing Qi, K, and Si + :type tensor_args: Tuple + :param pipeline_args: Tuple containing mma_si_producer and si_corr_producer + :type pipeline_args: Tuple + :param qk_sf_inplace_consumer: PipelineAsync consumer guarding the SFQ/SFK TMEM slot + (the tail of the OPPOSITE-stage S region). + :type qk_sf_inplace_consumer: pipeline.PipelineConsumer + :param pipeline_tokens: Optional non-blocking peek tokens for the Si and + vec_i producers, in the form ``(si_peek_status, veci_peek_status)``. + ``None`` for either token falls back to a blocking acquire. + :type pipeline_tokens: Tuple + :return: Tuple containing mma_si_producer, si_corr_producer, and the + (advanced) qk_sf_inplace_consumer. + :rtype: Tuple[pipeline.PipelineProducer, pipeline.PipelineProducer, + pipeline.PipelineConsumer] + """ + tSrQi, tSrK, tStSi = tensor_args + ( + tiled_copy_s2t_qsf, + tCsQSF_stage, + tCtQSF_s2t, + tCtQSF, + tiled_copy_s2t_ksf, + tCsKSF_stage, + tCtKSF_s2t, + tCtKSF, + ) = scale_args + mma_si_producer, si_corr_producer = pipeline_args + si_peek_status, veci_peek_status = pipeline_tokens + qk_tiled_mma = cutlass.new_from_mlir_values( + tiled_mma, cutlass.extract_mlir_values(tiled_mma) + ) + # 0. Make sure Qi & K are ready when calling mma_qk + # 1. acquire S0 + si_handle = mma_si_producer.acquire_and_advance(si_peek_status) + # 2. make sure vec is already released in corr + veci_handle = si_corr_producer.acquire_and_advance(veci_peek_status) + veci_handle.commit() + # 3. Wait until softmax_{1-j} has finished T2R-loading S_{1-j} to avoid SFQK_j race cond + qk_sf_inplace_consumer.wait_and_advance() + # 4. Copy MXFP8 scale factors and issue block-scaled gemm + cute.copy(tiled_copy_s2t_qsf, tCsQSF_stage, tCtQSF_s2t) + cute.copy(tiled_copy_s2t_ksf, tCsKSF_stage, tCtKSF_s2t) + qk_tiled_mma.set(tcgen05.Field.ACCUMULATE, False) + cute.gemm( + qk_tiled_mma, + tStSi, + [tSrQi, tCtQSF], + [tSrK, tCtKSF], + tStSi, + ) + # 4. release S0 + si_handle.commit() + return mma_si_producer, si_corr_producer, qk_sf_inplace_consumer + + @cute.jit + def mma_pv( + self, + tiled_mma: cute.TiledMma, + whether_acc: bool, + tensor_args: Tuple, + pipeline_args: Tuple, + skip_pv_args: Tuple, + ) -> Tuple[ + pipeline.PipelineProducer, + pipeline.PipelineConsumer, + ]: + """Perform a single step of the PV GEMM computation on accumulating O. + + :param tiled_mma: Tiled MMA for PV GEMM + :type tiled_mma: cute.TiledMma + :param whether_acc: Whether to accumulate O + :type whether_acc: bool + :param tensor_args: Tuple containing Pi, Vi, and Oi + :type tensor_args: Tuple + :param pipeline_args: Tuple containing mma_corr_producer and pi_mma_consumer + :type pipeline_args: Tuple + :param skip_pv_args: Tuple containing enable_skip_softmax, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v + :type skip_pv_args: Tuple + :return: Tuple containing mma_corr_producer and pi_mma_consumer + :rtype: Tuple[pipeline.PipelineProducer, pipeline.PipelineConsumer] + """ + tOrPi, tOrVi, tOtOi = tensor_args + mma_corr_producer, pi_mma_consumer = pipeline_args + enable_skip_softmax, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v = skip_pv_args + # 0. Make sure Vi is ready when calling mma_pv + # 1. acquire Oi + oi_handle = mma_corr_producer.acquire_and_advance() + # 2. wait for Pi + pi_handle = pi_mma_consumer.wait_and_advance() + # 3. gemm + num_kphases = cute.size(tOrPi, mode=[2]) + if cutlass.const_expr(enable_skip_softmax): + skip_pv = self.get_skip_softmax_flag(tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v) + if not skip_pv: + for kphase_idx in cutlass.range(num_kphases, unroll_full=True): + kphase_coord = (None, None, kphase_idx) + tiled_mma.set(tcgen05.Field.ACCUMULATE, whether_acc or kphase_idx != 0) + cute.gemm( + tiled_mma, + tOtOi, + tOrPi[kphase_coord], + tOrVi[kphase_coord], + tOtOi, + ) + else: + for kphase_idx in cutlass.range(num_kphases, unroll_full=True): + kphase_coord = (None, None, kphase_idx) + tiled_mma.set(tcgen05.Field.ACCUMULATE, whether_acc or kphase_idx != 0) + cute.gemm( + tiled_mma, + tOtOi, + tOrPi[kphase_coord], + tOrVi[kphase_coord], + tOtOi, + ) + # 4. commit Pi + pi_handle.release() + # 5. commit Oi + oi_handle.commit() + return mma_corr_producer, pi_mma_consumer + + @cute.jit + def calculate_skip_softmax_flag( + self, + row_max, + tile_row_max, + scale_softmax_log2, + skip_softmax_threshold_log2, + seqlen_q, + thread_idx, + logical_offset, + warp_wants_skip_softmax_exchange, + stage, + skip_softmax_count, + total_softmax_count, + ) -> Tuple[bool, float]: + """Calculate the skip softmax flag and the row maximum. + + :param row_max: The row maximum. + :type row_max: float + :param tile_row_max: The tile row maximum. + :type tile_row_max: float + :param scale_softmax_log2: The scale softmax log2. + :type scale_softmax_log2: float + :param skip_softmax_threshold_log2: The skip softmax threshold log2. + :type skip_softmax_threshold_log2: float + :param seqlen_q: The sequence length q. + :type seqlen_q: int + :param thread_idx: The thread index. + :type thread_idx: int + :param logical_offset: The logical offset. + :type logical_offset: Tuple[int, int] + :param warp_wants_skip_softmax_exchange: The warp wants skip softmax exchange. + :type warp_wants_skip_softmax_exchange: cute.Tensor + :param stage: The stage. + :type stage: int + :param skip_softmax_count: The skip softmax count. + :type skip_softmax_count: cute.Tensor + :param total_softmax_count: The total softmax count. + :type total_softmax_count: cute.Tensor + :return: Tuple containing the skip softmax flag and the row maximum. + :rtype: Tuple[bool, float] + """ + thread_wants_skip = ( + tile_row_max * scale_softmax_log2 - row_max * scale_softmax_log2 + ) < skip_softmax_threshold_log2 + thread_wants_skip = thread_wants_skip or ((logical_offset[0] + thread_idx) >= seqlen_q) + warp_wants_skip = cute.arch.vote_all_sync(thread_wants_skip) + + with cute.arch.elect_one(): + warp_wants_skip_softmax_exchange[cute.arch.warp_idx() % 4] = warp_wants_skip + softmax_barrier = self.s0_warpgroup_barrier if stage == 0 else self.s1_warpgroup_barrier + softmax_barrier.arrive_and_wait() + warp_wants_skip_softmax_exchange_i32 = cute.make_tensor( + cute.recast_ptr(warp_wants_skip_softmax_exchange.iterator, dtype=cutlass.Int32), + cute.make_layout((1,)), + ) + skip_softmax = cute.arch.popc(warp_wants_skip_softmax_exchange_i32[0]) == 4 + + if not skip_softmax: + row_max = max(row_max, tile_row_max) + + if cutlass.const_expr(skip_softmax_count is not None): + if thread_idx == 0: + if skip_softmax: + cute.arch.atomic_add(skip_softmax_count.iterator.llvm_ptr, Int32(1)) + cute.arch.atomic_add(total_softmax_count.iterator.llvm_ptr, Int32(1)) + return skip_softmax, row_max + + @cute.jit + def apply_exp_and_cvt( + self, + tTMEM_LOADrS, + tTMEM_LOADrS_cvt, + tTMEM_STORErS_x4_e_cvt, + stage, + scale, + minus_row_max_scale, + local_row_sum, + inplace_consumer, + EXP2_EMULATION_OFFSET, + EXP2_EMULATION_COUNT, + CVT_COUNT, + CVT_PER_STEP, + FMA_COUNT, + ARV_COUNT, + ): + """Apply the exp and conversion to the P data type on fragment. + + :param tTMEM_LOADrS: The tTMEM_LOADrS tensor. + :type tTMEM_LOADrS: cute.Tensor + :param tTMEM_LOADrS_cvt: The tTMEM_LOADrS_cvt tensor. + :type tTMEM_LOADrS_cvt: cute.Tensor + :param tTMEM_STORErS_x4_e_cvt: The tTMEM_STORErS_x4_e_cvt tensor. + :type tTMEM_STORErS_x4_e_cvt: cute.Tensor + :param stage: The stage. + :type stage: int + :param scale: The scale. + :type scale: float + :param minus_row_max_scale: The minus row maximum scale. + :type minus_row_max_scale: float + :param local_row_sum: The local row sum. + :type local_row_sum: float + :param inplace_consumer: The inplace consumer. + :type inplace_consumer: cute.Tensor + :param EXP2_EMULATION_OFFSET: The exp2 emulation offset. + :type EXP2_EMULATION_OFFSET: int + :param EXP2_EMULATION_COUNT: The exp2 emulation count. + :type EXP2_EMULATION_COUNT: int + :param CVT_COUNT: The cvt count. + :type CVT_COUNT: int + :param CVT_PER_STEP: The cvt per step. + :type CVT_PER_STEP: int + :param FMA_COUNT: The fma count. + :type FMA_COUNT: int + :param ARV_COUNT: The arv count. + :type ARV_COUNT: int + :return: The local row sum and the inplace consumer. + :rtype: Tuple[float, cute.Tensor] + """ + for i in cutlass.range_constexpr(0, EXP2_EMULATION_OFFSET, 2): + if cutlass.const_expr(i >= CVT_COUNT): + if cutlass.const_expr(i % CVT_PER_STEP == 0): + if cutlass.const_expr(self.v_dtype.width == 8): + fmha_utils.cvt_f32x4_to_f8x4( + tTMEM_LOADrS_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP], + tTMEM_STORErS_x4_e_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP], + ) + else: + s_vec = tTMEM_LOADrS_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP].load() + tTMEM_STORErS_x4_e_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP].store( + s_vec.to(self.v_dtype) + ) + local_row_sum = cute.arch.add_packed_f32x2( + local_row_sum, + ( + tTMEM_LOADrS[i - CVT_COUNT], + tTMEM_LOADrS[i - CVT_COUNT + 1], + ), + ) + tTMEM_LOADrS[i] = cute.math.exp2(tTMEM_LOADrS[i], fastmath=True) + if cutlass.const_expr(i + FMA_COUNT < EXP2_EMULATION_OFFSET): + ( + tTMEM_LOADrS[i + FMA_COUNT], + tTMEM_LOADrS[i + FMA_COUNT + 1], + ) = cute.arch.fma_packed_f32x2( + ( + tTMEM_LOADrS[i + FMA_COUNT], + tTMEM_LOADrS[i + FMA_COUNT + 1], + ), + (scale, scale), + (minus_row_max_scale, minus_row_max_scale), + ) + tTMEM_LOADrS[i + 1] = cute.math.exp2(tTMEM_LOADrS[i + 1], fastmath=True) + if cutlass.const_expr(i == EXP2_EMULATION_OFFSET - ARV_COUNT): + if cutlass.const_expr(self.enable_sequence_barrier): + if cutlass.const_expr(stage == 0): + self.sequence_s1_s0_barrier.arrive() + else: + self.sequence_s0_s1_barrier.arrive() + + # The remaining conversion steps + for i in cutlass.range_constexpr( + EXP2_EMULATION_OFFSET - CVT_COUNT, + EXP2_EMULATION_OFFSET, + 2, + ): + if cutlass.const_expr(i % CVT_PER_STEP == 0): + if cutlass.const_expr(self.v_dtype.width == 8): + fmha_utils.cvt_f32x4_to_f8x4( + tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], + ) + else: + s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) + local_row_sum = cute.arch.add_packed_f32x2( + local_row_sum, (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]) + ) + for i in cutlass.range_constexpr( + EXP2_EMULATION_OFFSET, EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT // 2, 2 + ): + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( + (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), + (scale, scale), + (minus_row_max_scale, minus_row_max_scale), + ) + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = fmha_utils.ex2_emulation_packed_f32x2( + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] + ) + if cutlass.const_expr((i + 2) % CVT_PER_STEP == 0): + if cutlass.const_expr(self.v_dtype.width == 8): + fmha_utils.cvt_f32x4_to_f8x4( + tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], + ) + else: + s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) + + inplace_peek_status = inplace_consumer.try_wait() + for i in cutlass.range_constexpr( + EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT // 2, + EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, + 2, + ): + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( + (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), + (scale, scale), + (minus_row_max_scale, minus_row_max_scale), + ) + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = fmha_utils.ex2_emulation_packed_f32x2( + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] + ) + if cutlass.const_expr((i + 2) % CVT_PER_STEP == 0): + if cutlass.const_expr(self.v_dtype.width == 8): + fmha_utils.cvt_f32x4_to_f8x4( + tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], + ) + else: + s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() + tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) + inplace_consumer.wait_and_advance(inplace_peek_status) + return local_row_sum, inplace_consumer + + @cute.jit + def softmax_step( + self, + stage: int, + whether_apply_mask: bool, + iter_args: Tuple, + stats_args: Tuple, + pipeline_args: Tuple, + value_args: Tuple, + atom_args: Tuple, + tensor_args: Tuple, + ) -> Tuple[Tuple, Tuple]: + """Perform a single step of the softmax computation on a block of attention scores. + + This method processes one block of the attention matrix, computing numerically stable + softmax by first finding the row maximum, subtracting it from all elements, applying + exponential function, and then normalizing by the sum of exponentials. It also handles + optional masking of attention scores. + + The method involves several key operations: + 1. Loading attention scores from tensor memory + 2. Applying optional masking based on position + 3. Computing row-wise maximum values for numerical stability + 4. Transforming scores using exp2(x*scale - max*scale) + 5. Computing row sums for normalization + 6. Coordinating pipeline synchronization between different processing stages + + :param stage: Processing stage (0 for first half, 1 for second half) + :type stage: int + :param whether_apply_mask: Whether to apply attention masking + :type whether_apply_mask: bool + :param iter_args: Tuple containing the counting tensor, row_max, row_sum, and vector buffer's handle for current iteration + :type iter_args: Tuple + :param stats_args: Tuple containing row_sum and row_max + :type stats_args: Tuple + :param pipeline_args: Tuple containing pipeline related arguments for MMA, correction, and sequence synchronization + :type pipeline_args: Tuple + :param value_args: Tuple containing seqlen_k, seqlen_q, and scale_softmax_log2 + :type value_args: Tuple + :param atom_args: Tuple containing mma & copy atoms + :type atom_args: Tuple + :param tensor_args: Tuple containing softmax related tensors + :type tensor_args: Tuple + :param fused_mask: Compute trip counts and apply masking for attention blocks + :type fused_mask: fmha_utils.FusedMask + :return: Updated stats_args and pipeline_args + :rtype: Tuple[Tuple, Tuple] + """ + row_sum, row_max = stats_args + cS, is_last_iter = iter_args + ( + seqlen_k, + seqlen_q, + scale_softmax_log2, + window_size_left, + window_size_right, + skip_softmax_threshold_log2, + thread_idx, + logical_offset, + qk_sf_inplace_producer, + ) = value_args + ( + si_peek_status, + mma_si_consumer, + si_corr_producer, + pi_mma_producer, + inplace_producer, + inplace_consumer, + ) = pipeline_args + ( + qk_thr_mma, + tiled_tmem_load, + tiled_tmem_store, + tiled_tmem_store_vec, + thr_tmem_load, + thr_tmem_store, + thr_tmem_store_vec, + ) = atom_args + ( + tTMEM_LOADtS, + tTMEM_STORE_VECtS, + tTMEM_STORE_SKIP_SOFTMAX, + tTMEM_STOREtS_x4, + warp_wants_skip_softmax_exchange, + skip_softmax_count, + total_softmax_count, + ) = tensor_args + tilePlikeFP32 = self.qk_mma_tiler[1] // Float32.width * self.o_dtype.width + tScS = qk_thr_mma.partition_C(cS) + enable_skip_softmax = skip_softmax_threshold_log2 is not None + tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) + tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) + tScS_P_layout = cute.composition(tScS.layout, cute.make_layout((128, tilePlikeFP32))) + tScS_P = cute.make_tensor(tScS.iterator, tScS_P_layout) + tTMEM_LOADcS = thr_tmem_load.partition_D(tScS) + tTMEM_STORE_VECcS = thr_tmem_store_vec.partition_S(tScS_vec) + tTMEM_STOREcS = thr_tmem_store.partition_S(tScS_P) + # Wait for Si + si_handle = mma_si_consumer.wait_and_advance(si_peek_status) + tTMEM_LOADrS = cute.make_rmem_tensor(tTMEM_LOADcS.shape, self.qk_acc_dtype) + old_row_max = row_max + skip_softmax = cutlass.Boolean(False) + if whether_apply_mask: + if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): + cute.copy(tiled_tmem_load, tTMEM_LOADtS, tTMEM_LOADrS) + else: + tTMEM_LOADrMax = cute.make_rmem_tensor( + cute.make_layout((1, cute.size(tTMEM_LOADrS, mode=[1]))), + self.qk_acc_dtype, + ) + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[1])): + cute.copy_atom_call( + tiled_tmem_load, + tTMEM_LOADtS[None, i, 0, 0], + (tTMEM_LOADrS[None, i, 0, 0], tTMEM_LOADrMax[None, i]), + ) + fmha_utils.FusedMask.apply_mask( + self.mask_type, + tTMEM_LOADrS, + tTMEM_LOADcS, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + tile_row_max = tTMEM_LOADrS.load().reduce(cute.ReductionOp.MAX, -cutlass.Float32.inf, 0) + if cutlass.const_expr(not enable_skip_softmax): + row_max = cute.arch.fmax(row_max, tile_row_max) + else: + skip_softmax, row_max = self.calculate_skip_softmax_flag( + row_max, + tile_row_max, + scale_softmax_log2, + skip_softmax_threshold_log2, + seqlen_q, + thread_idx, + logical_offset, + warp_wants_skip_softmax_exchange, + stage, + skip_softmax_count, + total_softmax_count, + ) + # Fence the T2R load above, then signal MMA that the opposite-stage S is loaded. + cute.arch.fence_view_async_tmem_load() + qk_sf_inplace_producer.commit() + qk_sf_inplace_producer.advance() + si_handle.release() + # S0 -> P1 / S1 -> P0 + inplace_producer.commit() + inplace_producer.advance() + else: + if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): + cute.copy( + tiled_tmem_load, + tTMEM_LOADtS[None, 0, None, None], + tTMEM_LOADrS[None, 0, None, None], + ) + cute.copy( + tiled_tmem_load, + tTMEM_LOADtS[None, 1, None, None], + tTMEM_LOADrS[None, 1, None, None], + ) + tile_row_max = -cutlass.Float32.inf + tile_row_max_ = tile_row_max + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 0, 0, 0]) + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 0, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 0, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 0, 0, 0]) + cute.copy( + tiled_tmem_load, + tTMEM_LOADtS[None, 2, None, None], + tTMEM_LOADrS[None, 2, None, None], + ) + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 1, 0, 0]) + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 1, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 1, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 1, 0, 0]) + cute.copy( + tiled_tmem_load, + tTMEM_LOADtS[None, 3, None, None], + tTMEM_LOADrS[None, 3, None, None], + ) + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 2, 0, 0]) + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 2, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 2, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 2, 0, 0]) + cute.arch.fence_view_async_tmem_store() + # Fence T2R loads then release the QK SF TMEM slot before we release S to MMA + cute.arch.fence_view_async_tmem_load() + qk_sf_inplace_producer.commit() + qk_sf_inplace_producer.advance() + si_handle.release() + # S0 -> P1 / S1 -> P0 + inplace_producer.commit() + inplace_producer.advance() + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 3, 0, 0]) + tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 3, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 3, 0, 0]) + tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 3, 0, 0]) + tile_row_max = cute.arch.fmax(tile_row_max, tile_row_max_) + if cutlass.const_expr(not enable_skip_softmax): + row_max = cute.arch.fmax(tile_row_max, row_max) + else: + skip_softmax, row_max = self.calculate_skip_softmax_flag( + row_max, + tile_row_max, + scale_softmax_log2, + skip_softmax_threshold_log2, + seqlen_q, + thread_idx, + logical_offset, + warp_wants_skip_softmax_exchange, + stage, + skip_softmax_count, + total_softmax_count, + ) + else: + tTMEM_LOADrMax = cute.make_rmem_tensor( + cute.make_layout((1, cute.size(tTMEM_LOADrS, mode=[1]))), + self.qk_acc_dtype, + ) + for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[1])): + cute.copy_atom_call( + tiled_tmem_load, + tTMEM_LOADtS[None, i, 0, 0], + (tTMEM_LOADrS[None, i, 0, 0], tTMEM_LOADrMax[None, i]), + ) + cute.arch.fence_view_async_tmem_store() + tile_row_max = tTMEM_LOADrMax.load().reduce( + cute.ReductionOp.MAX, -cutlass.Float32.inf, 0 + ) + if cutlass.const_expr(not enable_skip_softmax): + row_max = cute.arch.fmax(tile_row_max, row_max) + else: + skip_softmax, row_max = self.calculate_skip_softmax_flag( + row_max, + tile_row_max, + scale_softmax_log2, + skip_softmax_threshold_log2, + seqlen_q, + thread_idx, + logical_offset, + warp_wants_skip_softmax_exchange, + stage, + skip_softmax_count, + total_softmax_count, + ) + # Fence the T2R loads then release the QK SF TMEM slot. + cute.arch.fence_view_async_tmem_load() + qk_sf_inplace_producer.commit() + qk_sf_inplace_producer.advance() + si_handle.release() + # S0 -> P1 / S1 -> P0 + inplace_producer.commit() + inplace_producer.advance() + + row_max_safe = row_max + if row_max == -cutlass.Float32.inf: + row_max_safe = 0.0 + if cutlass.const_expr(self.rescale_threshold > 0.0): + if (row_max_safe - old_row_max) * scale_softmax_log2 <= self.rescale_threshold: + row_max_safe = old_row_max + tTMEM_STORE_VECrS = cute.make_rmem_tensor(tTMEM_STORE_VECcS.shape, self.qk_acc_dtype) + tTMEM_STORE_VECrS[0] = old_row_max + tTMEM_STORE_VECrS[1] = row_max_safe + vec_i_peek_status = si_corr_producer.try_acquire() + tTMEM_STORErS_x4 = cute.make_rmem_tensor(tTMEM_STOREcS.shape, self.qk_acc_dtype) + tTMEM_STORErS_x4_e = cute.make_tensor( + cute.recast_ptr(tTMEM_STORErS_x4.iterator, dtype=self.v_dtype), + tTMEM_LOADrS.layout, + ) + scale = scale_softmax_log2 + minus_row_max_scale = (0.0 - row_max_safe) * scale + if cutlass.const_expr(self.v_dtype.width == 8 and self.p_fp8_prescale_log2 > 0): + minus_row_max_scale = minus_row_max_scale + self.p_fp8_prescale_log2 + + ARV_COUNT = 4 + FMA_COUNT = 8 + CVT_COUNT = 8 if self.v_dtype.width == 8 else 4 + CVT_PER_STEP = 4 if self.v_dtype.width == 8 else 2 + assert CVT_COUNT % CVT_PER_STEP == 0, ( + f"CVT_COUNT {CVT_COUNT} must be divisible by CVT_PER_STEP {CVT_PER_STEP}" + ) + tTMEM_LOADrS_cvt = cute.logical_divide(tTMEM_LOADrS, cute.make_layout(CVT_PER_STEP)) + tTMEM_STORErS_x4_e_cvt = cute.logical_divide( + tTMEM_STORErS_x4_e, cute.make_layout(CVT_PER_STEP) + ) + for i in cutlass.range_constexpr(0, FMA_COUNT, 2): + tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( + (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), + (scale, scale), + (minus_row_max_scale, minus_row_max_scale), + ) + vec_i_handle = si_corr_producer.acquire_and_advance(vec_i_peek_status) + cute.copy(tiled_tmem_store_vec, tTMEM_STORE_VECrS, tTMEM_STORE_VECtS) + cute.arch.fence_view_async_tmem_store() + # Notify correction wg that row_max is ready + vec_i_handle.commit() + + EXP2_EMULATION_COUNT = 20 if self.enable_ex2_emulation and not whether_apply_mask else 0 + EXP2_EMULATION_OFFSET = cute.size(tTMEM_LOADrS) - EXP2_EMULATION_COUNT + acc_scale_ = scale * (old_row_max - row_max_safe) + acc_scale = cute.math.exp2(acc_scale_, fastmath=True) * 0.5 + if cutlass.const_expr(self.enable_sequence_barrier): + if cutlass.const_expr(stage == 0): + self.sequence_s0_s1_barrier.arrive_and_wait() + else: + self.sequence_s1_s0_barrier.arrive_and_wait() + + if cutlass.const_expr(enable_skip_softmax): + if not skip_softmax: + row_sum *= acc_scale + local_row_sum = (row_sum, row_sum) + local_row_sum, inplace_consumer = self.apply_exp_and_cvt( + tTMEM_LOADrS, + tTMEM_LOADrS_cvt, + tTMEM_STORErS_x4_e_cvt, + stage, + scale, + minus_row_max_scale, + local_row_sum, + inplace_consumer, + EXP2_EMULATION_OFFSET, + EXP2_EMULATION_COUNT, + CVT_COUNT, + CVT_PER_STEP, + FMA_COUNT, + ARV_COUNT, + ) + tTMEM_STORE_VECrS_i32 = cute.recast_tensor(tTMEM_STORE_VECrS, dtype=cutlass.Int32) + tTMEM_STORE_VECrS_i32[0] = 0 + pi_handle = pi_mma_producer.acquire_and_advance() + # store skip softmax flag + cute.copy( + tiled_tmem_store_vec, + tTMEM_STORE_VECrS_i32, + tTMEM_STORE_SKIP_SOFTMAX, + ) + # store P + cute.copy(tiled_tmem_store, tTMEM_STORErS_x4, tTMEM_STOREtS_x4) + cute.arch.fence_view_async_tmem_store() + pi_handle.commit() + for j in cutlass.range_constexpr( + EXP2_EMULATION_OFFSET, + EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, + 2, + ): + local_row_sum = cute.arch.add_packed_f32x2( + (tTMEM_LOADrS[j], tTMEM_LOADrS[j + 1]), + local_row_sum, + ) + row_sum = local_row_sum[0] + local_row_sum[1] + cute.arch.fence_view_async_tmem_store() + else: + if cutlass.const_expr(self.enable_sequence_barrier): + if cutlass.const_expr(stage == 0): + self.sequence_s1_s0_barrier.arrive() + else: + self.sequence_s0_s1_barrier.arrive() + inplace_peek_status = inplace_consumer.try_wait() + inplace_consumer.wait_and_advance(inplace_peek_status) + tTMEM_STORE_VECrS_i32 = cute.recast_tensor(tTMEM_STORE_VECrS, dtype=cutlass.Int32) + tTMEM_STORE_VECrS_i32[0] = 1 + pi_handle = pi_mma_producer.acquire_and_advance() + # store skip softmax flag + cute.copy( + tiled_tmem_store_vec, + tTMEM_STORE_VECrS_i32, + tTMEM_STORE_SKIP_SOFTMAX, + ) + cute.arch.fence_view_async_tmem_store() + pi_handle.commit() + else: + row_sum *= acc_scale + local_row_sum = (row_sum, row_sum) + local_row_sum, inplace_consumer = self.apply_exp_and_cvt( + tTMEM_LOADrS, + tTMEM_LOADrS_cvt, + tTMEM_STORErS_x4_e_cvt, + stage, + scale, + minus_row_max_scale, + local_row_sum, + inplace_consumer, + EXP2_EMULATION_OFFSET, + EXP2_EMULATION_COUNT, + CVT_COUNT, + CVT_PER_STEP, + FMA_COUNT, + ARV_COUNT, + ) + pi_handle = pi_mma_producer.acquire_and_advance() + # store P + cute.copy(tiled_tmem_store, tTMEM_STORErS_x4, tTMEM_STOREtS_x4) + cute.arch.fence_view_async_tmem_store() + for j in cutlass.range_constexpr( + EXP2_EMULATION_OFFSET, + EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, + 2, + ): + local_row_sum = cute.arch.add_packed_f32x2( + (tTMEM_LOADrS[j], tTMEM_LOADrS[j + 1]), + local_row_sum, + ) + row_sum = local_row_sum[0] + local_row_sum[1] + cute.arch.fence_view_async_tmem_store() + # Notify tensor core warp that softmax(S->P) is ready + pi_handle.commit() + if not is_last_iter: + si_peek_status = mma_si_consumer.try_wait() + + stats_args = (row_sum, row_max_safe) + pipeline_args = ( + si_peek_status, + mma_si_consumer, + si_corr_producer, + pi_mma_producer, + inplace_producer, + inplace_consumer, + ) + return stats_args, pipeline_args + + # For both softmax0 and softmax1 warp group + @cute.jit + def softmax( + self, + stage: int, + tensor_args: Tuple, + pipeline_args: Tuple, + inplace_args: Tuple, + qk_thr_mma: cute.ThrMma, + value_args: Tuple, + mask_args: Tuple, + sched_args: Tuple, + qk_sf_inplace_producers: Tuple[pipeline.PipelineProducer, pipeline.PipelineProducer], + ): + """Compute softmax on attention scores from QK matrix multiplication. + + This method handles the softmax computation for either the first or second half of the + attention matrix, depending on the 'stage' parameter. It calculates row-wise maximum + and sum values needed for stable softmax computation, applies optional masking, and + transforms raw attention scores into probability distributions. + + The implementation uses specialized memory access patterns and efficient math operations + for computing exp(x) using exp2 functions. It also coordinates pipeline + synchronization between MMA, correction, and sequence processing stages. + + :param stage: Processing stage (0 for first half, 1 for second half of attention matrix) + :type stage: int + :param seqlen_k: Length of the key sequence + :type seqlen_k: Int32 + :param seqlen_q: Length of the query sequence + :type seqlen_q: Int32 + :param cum_seqlen_q: Cumulative sequence lengths for queries + :type cum_seqlen_q: cute.Tensor | None + :param cum_seqlen_k: Cumulative sequence lengths for keys + :type cum_seqlen_k: cute.Tensor | None + :param scale_softmax_log2: Log2 scale factor for softmax operation + :type scale_softmax_log2: Float32 + :param qk_thr_mma: Thread MMA operation for QK matrix multiplication + :type qk_thr_mma: cute.ThrMma + :param tStS: Shared tensor for softmax input/output + :type tStS: cute.Tensor + :param tStSi: Input tensor containing attention scores + :type tStSi: cute.Tensor + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + :param mma_si_consumer: Pipeline for synchronizing with Si tensors + :type mma_si_consumer: pipeline.PipelineConsumer + :param si_corr_producer: Pipeline for synchronizing with correction operations + :type si_corr_producer: pipeline.PipelineProducer + :param pi_mma_producer: Pipeline for synchronizing with Pi tensors + :type pi_mma_producer: pipeline.PipelineProducer + :param tile_sched_params: Parameters for tile scheduling + :type tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams + :param fused_mask: Compute trip counts and apply masking for attention blocks + :type fused_mask: fmha_utils.FusedMask + """ + ( + tStS, + tStSi, + cum_seqlen_k, + cum_seqlen_q, + warp_wants_skip_softmax_exchange, + skip_softmax_count, + total_softmax_count, + ) = tensor_args + mma_si_consumer, si_corr_producer, pi_mma_producer = pipeline_args + inplace_producer, inplace_consumer = inplace_args + ( + seqlen_k, + seqlen_q, + scale_softmax_log2, + skip_softmax_threshold_log2, + ) = value_args + window_size_left, window_size_right = mask_args + tile_sched, work_tile = sched_args + + tidx, _, _ = cute.arch.thread_idx() + thread_idx = tidx % (self.threads_per_warp * len(self.softmax0_warp_ids)) + + cS_base = cute.make_identity_tensor((self.qk_mma_tiler[0], self.qk_mma_tiler[1])) + tilePlikeFP32 = self.qk_mma_tiler[1] // 32 * self.o_dtype.width + tScS = qk_thr_mma.partition_C(cS_base) + tStS_vec_layout = cute.composition(tStS.layout, cute.make_layout((128, 2))) + tmem_vec_offset = self.tmem_vec0_offset if stage == 0 else self.tmem_vec1_offset + tStS_vec = cute.make_tensor(tStS.iterator + tmem_vec_offset, tStS_vec_layout) + tmem_skip_softmax_offset = ( + self.tmem_skip_softmax0_offset if stage == 0 else self.tmem_skip_softmax1_offset + ) + tStS_skip_softmax = cute.make_tensor( + tStS.iterator + tmem_skip_softmax_offset, tStS_vec_layout + ) + tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) + tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) + tStS_P_layout = cute.composition(tStS.layout, cute.make_layout((128, tilePlikeFP32))) + tmem_p_offset = self.tmem_p0_offset if stage == 0 else self.tmem_p1_offset + tStS_P = cute.make_tensor(tStS.iterator + tmem_p_offset, tStS_P_layout) + if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): + tmem_load_atom = cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), + self.qk_acc_dtype, + ) + else: + tmem_load_atom = cute.make_copy_atom( + tcgen05.copy.LdRed32x32bOp( + tcgen05.copy.Repetition(32), redOp=tcgen05.TmemLoadRedOp.MAX + ), + self.qk_acc_dtype, + ) + + tiled_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tStSi) + thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) + tTMEM_LOADtS = thr_tmem_load.partition_S(tStSi) + tmem_store_vec_atom = cute.make_copy_atom( + tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(2)), + self.qk_acc_dtype, + ) + tiled_tmem_store_vec = tcgen05.make_tmem_copy(tmem_store_vec_atom, tStS_vec) + thr_tmem_store_vec = tiled_tmem_store_vec.get_slice(thread_idx) + tTMEM_STORE_VECtS = thr_tmem_store_vec.partition_D(tStS_vec) + tTMEM_STORE_VECcS = thr_tmem_store_vec.partition_S(tScS_vec) + tTMEM_STORE_SKIP_SOFTMAX = thr_tmem_store_vec.partition_D(tStS_skip_softmax) + tmem_store_atom = cute.make_copy_atom( + tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(32)), + self.qk_acc_dtype, + ) + tiled_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tStS_P) + thr_tmem_store = tiled_tmem_store.get_slice(thread_idx) + tTMEM_STOREtS_x4 = thr_tmem_store.partition_D(tStS_P) + # Each softmax warpgroup releases the OPPOSITE-stage S region (where + # SFQ_{1-stage}/SFK_{1-stage} live in the tail) by committing to its qk_sf_inplace producer + # once per kv block: + # softmax0 (stage=0) drives qk_sf_inplace_1 + # softmax1 (stage=1) drives qk_sf_inplace_0 + qk_sf_inplace_producer = qk_sf_inplace_producers[stage] + # Bootstrap: MMA's first mma_qk(stage=0) waits on qk_sf_inplace_0 before + # softmax1 has produced any in-loop commit. Each thread of softmax1 + # pre-commits once so the first wait succeeds. qk_sf_inplace_1 needs no + # pre-commit because mma_qk(stage=1) runs after mma_qk(stage=0), + # giving softmax0 enough time to consume S0 and commit naturally. + if cutlass.const_expr(stage == 1): + qk_sf_inplace_producer.commit() + qk_sf_inplace_producer.advance() + if cutlass.const_expr(self.enable_sequence_barrier): + if cutlass.const_expr(stage == 1): + self.sequence_s0_s1_barrier.arrive() + while work_tile.is_valid_tile: + curr_block_coord = work_tile.tile_idx + batch_coord = curr_block_coord[2][1] + seqlen_k_ = seqlen_k + seqlen_q_ = seqlen_q + continue_cond = False + cuseqlen_q = Int32(0) + seqlen_q_ = seqlen_q + if cutlass.const_expr(cum_seqlen_q is not None): + cuseqlen_q = cum_seqlen_q[batch_coord] + seqlen_q_ = cum_seqlen_q[batch_coord + 1] - cuseqlen_q + continue_cond = ( + not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( + self.cta_tiler[0], + curr_block_coord[0], + seqlen_q_, + ) + ) + if not continue_cond: + if cutlass.const_expr(cum_seqlen_k is not None): + cuseqlen_k = cum_seqlen_k[batch_coord] + seqlen_k_ = cum_seqlen_k[batch_coord + 1] - cuseqlen_k + continue_cond = seqlen_k_ <= 0 + if not continue_cond: + logical_offset = ( + curr_block_coord[0] * self.cta_tiler[0] + stage * self.qk_mma_tiler[0], + 0, + ) + cS = cute.domain_offset(logical_offset, cS_base) + value_args_ = ( + seqlen_k_, + seqlen_q_, + scale_softmax_log2, + window_size_left, + window_size_right, + skip_softmax_threshold_log2, + thread_idx, + logical_offset, + qk_sf_inplace_producer, + ) + atom_args = ( + qk_thr_mma, + tiled_tmem_load, + tiled_tmem_store, + tiled_tmem_store_vec, + thr_tmem_load, + thr_tmem_store, + thr_tmem_store_vec, + ) + tensor_args_ = ( + tTMEM_LOADtS, + tTMEM_STORE_VECtS, + tTMEM_STORE_SKIP_SOFTMAX, + tTMEM_STOREtS_x4, + warp_wants_skip_softmax_exchange, + skip_softmax_count, + total_softmax_count, + ) + st_cnt, end_cnt, ld_mask_cnt, unmask_cnt, tl_mask_cnt = ( + fmha_utils.FusedMask.get_masked_info( + self.mask_type, + curr_block_coord, + self.cta_tiler, + seqlen_q_, + seqlen_k_, + window_size_left, + window_size_right, + ) + ) + row_max = -Float32.inf + row_sum = 0.0 + stats_args = (row_sum, row_max) + + def softmax_loop( + whether_apply_mask: bool, + loop_args: Tuple, + stats_args: Tuple, + pipeline_args: Tuple, + inner_fn: Callable, + value_args: Tuple, + atom_args: Tuple, + tensor_args: Tuple, + cS: cute.Tensor, + ) -> Tuple[Tuple, Tuple]: + start_index, iter_num, upper_bound = loop_args + for i in cutlass.range(start_index, start_index + iter_num, 1, unroll=1): + cS_iter = cute.domain_offset((0, i * self.qk_mma_tiler[1]), cS) + iter_args = (cS_iter, i == upper_bound - 1) + stats_args, pipeline_args = inner_fn( + stage, + whether_apply_mask, + iter_args, + stats_args, + pipeline_args, + value_args, + atom_args, + tensor_args, + ) + return stats_args, pipeline_args + + softmax_step_fn = self.softmax_step + softmax_loop_fn = partial( + softmax_loop, + inner_fn=softmax_step_fn, + value_args=value_args_, + atom_args=atom_args, + tensor_args=tensor_args_, + cS=cS, + ) + si_peek_status = mma_si_consumer.try_wait() + if cutlass.const_expr(stage == 1): + inplace_consumer.wait_and_advance() + pipeline_args_ = ( + si_peek_status, + mma_si_consumer, + si_corr_producer, + pi_mma_producer, + inplace_producer, + inplace_consumer, + ) + # 1. Leading mask loop + loop_args = (st_cnt, ld_mask_cnt, end_cnt) + stats_args, pipeline_args_ = softmax_loop_fn( + True, loop_args, stats_args, pipeline_args_ + ) + # 2. Unmasked loop + loop_args = (st_cnt + ld_mask_cnt, unmask_cnt, end_cnt) + stats_args, pipeline_args_ = softmax_loop_fn( + False, loop_args, stats_args, pipeline_args_ + ) + # 3. Trailing mask loop + loop_args = (st_cnt + ld_mask_cnt + unmask_cnt, tl_mask_cnt, end_cnt) + stats_args, pipeline_args_ = softmax_loop_fn( + True, loop_args, stats_args, pipeline_args_ + ) + + # Unpack pipeline_args + ( + _, + mma_si_consumer, + si_corr_producer, + pi_mma_producer, + inplace_producer, + inplace_consumer, + ) = pipeline_args_ + if cutlass.const_expr(stage == 0): + inplace_producer.commit() + inplace_producer.advance() + # 4. Copy the final stats for correction epilog + tTMEM_STORE_VECrS = cute.make_rmem_tensor( + tTMEM_STORE_VECcS.shape, self.qk_acc_dtype + ) + tTMEM_STORE_VECrS[0] = stats_args[0] + tTMEM_STORE_VECrS[1] = stats_args[1] + vec_i_handle = si_corr_producer.acquire_and_advance() + cute.copy(tiled_tmem_store_vec, tTMEM_STORE_VECrS, tTMEM_STORE_VECtS) + cute.arch.fence_view_async_tmem_store() + vec_i_handle.commit() + # End of if not continue_cond + # Advance to next tile + tile_sched.advance_to_next_work() + work_tile = tile_sched.get_current_work() + # End of persistent scheduler loop + + @cute.jit + def correction_rescale( + self, + thr_mma: cute.ThrMma, + tiled_tmem_load_vec: cute.TiledCopy, + scale_softmax_log2: Float32, + tensor_args: Tuple, + pipeline_args: Tuple, + oi_peek_status=None, + ): + """Rescale intermediate attention results based on softmax normalization factor. + + This method performs a crucial correction step in the attention computation pipeline. + When processing attention in blocks, the softmax normalization factors may change + as new blocks are processed. This method rescales previously computed partial + output values to account for updated normalization factors. + + The implementation uses efficient tensor memory operations to: + 1. Load existing partial attention output from tensor memory + 2. Apply the scaling factor to all elements + 3. Store the rescaled results back to tensor memory + + When ``self.enable_correction_double_buffer`` is True, the rescale loop + runs as a 2-buffer tensor-memory load / multiply / store pipeline. + + :param thr_mma: Thread MMA operation for the computation + :type thr_mma: cute.ThrMma + :param tiled_tmem_load_vec: Tiled memory load operation for the vectorized row-wise max + :type tiled_tmem_load_vec: cute.TiledCopy + :param scale_softmax_log2: Log2 of the softmax factor + :type scale_softmax_log2: Float32 + :param tensor_args: Tuple containing the tensors for the correction + :type tensor_args: Tuple[cute.Tensor, cute.Tensor, cute.Tensor] + :param pipeline_args: Tuple containing the pipeline arguments for the correction + :type pipeline_args: Tuple[pipeline.PipelineConsumer, pipeline.PipelineConsumer] + :param oi_peek_status: Optional non-blocking token for the Oi consumer + wait. ``None`` or ``False`` falls back to a blocking wait. + :return: ``((si_corr_consumer, mma_corr_consumer), next_oi_peek_status)`` + where ``next_oi_peek_status`` is the peek for the next Oi (only + refreshed when a rescale actually ran; otherwise stays + ``cutlass.Boolean(False)``). + """ + tOtO, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECcS = tensor_args + si_corr_consumer, mma_corr_consumer = pipeline_args + + pv_tiled_mma_shape = ( + self.pv_mma_tiler[0], + self.pv_mma_tiler[1], + ) + cO = cute.make_identity_tensor(pv_tiled_mma_shape) + tOcO = thr_mma.partition_C(cO) + corr_tile_size = 16 # tuneable parameter + tmem_load_atom = cute.make_copy_atom( + tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(corr_tile_size)), + self.pv_acc_dtype, + ) + tmem_store_atom = cute.make_copy_atom( + tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(corr_tile_size)), + self.pv_acc_dtype, + ) + tOtO_i_layout = cute.composition(tOtO.layout, cute.make_layout((128, corr_tile_size))) + tOcO_i_layout = cute.composition(tOcO.layout, cute.make_layout((128, corr_tile_size))) + tOtO_i = cute.make_tensor(tOtO.iterator, tOtO_i_layout) + tOcO_i = cute.make_tensor(tOcO.iterator, tOcO_i_layout) + tiled_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tOtO_i) + tiled_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tOtO_i) + tidx, _, _ = cute.arch.thread_idx() + thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) + thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) + thr_tmem_store = tiled_tmem_store.get_slice(thread_idx) + tTMEM_LOADtO = thr_tmem_load.partition_S(tOtO_i) + tTMEM_LOADcO = thr_tmem_load.partition_D(tOcO_i) + tTMEM_STOREtO = thr_tmem_store.partition_D(tOtO_i) + num_tiles = self.cta_tiler[2] // corr_tile_size + if cutlass.const_expr(self.enable_correction_double_buffer): + # Double buffer: 2 register buffers to pipeline tensor-memory load / multiply / store. + tTMrO = cute.make_rmem_tensor((tTMEM_LOADcO.shape, 2), self.pv_acc_dtype) + # Rank-matching views for cute.copy (TMEM partitions are rank-3, + # raw tTMrO[None, idx] is rank-1; composition restores the rank). + copy_layout = cute.make_layout(tTMrO.shape[0]) + view_0 = tTMrO[None, 0] + view_1 = tTMrO[None, 1] + tTMrO_copy = ( + cute.make_tensor( + view_0.iterator, + cute.composition(view_0.layout, copy_layout), + ), + cute.make_tensor( + view_1.iterator, + cute.composition(view_1.layout, copy_layout), + ), + ) + else: + tTMrO = cute.make_rmem_tensor((tTMEM_LOADcO.shape, num_tiles), self.pv_acc_dtype) + tTMEM_LOAD_VECrS = cute.make_rmem_tensor(tTMEM_LOAD_VECcS.shape, self.qk_acc_dtype) + # Wait for vec_i (row_wise current max & previous max) + vec_i_handle = si_corr_consumer.wait_and_advance() + cute.copy(tiled_tmem_load_vec, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECrS) + cute.arch.fence_view_async_tmem_load() + vec_i_handle.release() + # Wait for Oi (peek-token aware: None/False falls back to blocking). + oi_handle = mma_corr_consumer.wait_and_advance(oi_peek_status) + next_oi_peek_status = cutlass.Boolean(False) + vote_ballot_cnt = cute.arch.vote_ballot_sync(tTMEM_LOAD_VECrS[0] != tTMEM_LOAD_VECrS[1]) + should_rescale = vote_ballot_cnt != 0 + if should_rescale: + scale_ = scale_softmax_log2 * (tTMEM_LOAD_VECrS[0] - tTMEM_LOAD_VECrS[1]) + scale = cute.math.exp2(scale_, fastmath=True) + if cutlass.const_expr(self.enable_correction_double_buffer): + num_elems = cute.size(tTMrO, mode=[0]) + # Prologue: load first tile into buffer 0 + cute.copy(tiled_tmem_load, tTMEM_LOADtO, tTMrO_copy[0]) + # Steady state. Refresh the next Oi probe near the end of the + # current rescale pipeline. + for i in cutlass.range_constexpr(1, num_tiles): + cute.copy( + tiled_tmem_load, + cute.make_tensor( + tTMEM_LOADtO.iterator + i * corr_tile_size, + tTMEM_LOADtO.layout, + ), + tTMrO_copy[i % 2], + ) + for j in range(0, num_elems, 2): + tTMrO[j, (i - 1) % 2], tTMrO[j + 1, (i - 1) % 2] = ( + cute.arch.mul_packed_f32x2( + ( + tTMrO[j, (i - 1) % 2], + tTMrO[j + 1, (i - 1) % 2], + ), + (scale, scale), + ) + ) + cute.copy( + tiled_tmem_store, + tTMrO_copy[(i - 1) % 2], + cute.make_tensor( + tTMEM_STOREtO.iterator + (i - 1) * corr_tile_size, + tTMEM_STOREtO.layout, + ), + ) + next_oi_peek_status = mma_corr_consumer.try_wait() + # Epilogue: compute and store last tile + last = (num_tiles - 1) % 2 + for j in range(0, num_elems, 2): + tTMrO[j, last], tTMrO[j + 1, last] = cute.arch.mul_packed_f32x2( + (tTMrO[j, last], tTMrO[j + 1, last]), + (scale, scale), + ) + cute.copy( + tiled_tmem_store, + tTMrO_copy[last], + cute.make_tensor( + tTMEM_STOREtO.iterator + (num_tiles - 1) * corr_tile_size, + tTMEM_STOREtO.layout, + ), + ) + else: + for i in cutlass.range_constexpr(0, num_tiles): + tTMrO_i_ = tTMrO[None, i] + tTMrO_i = cute.make_tensor( + tTMrO_i_.iterator, + cute.composition(tTMrO_i_.layout, cute.make_layout(tTMrO.shape[0])), + ) + cute.copy( + tiled_tmem_load, + cute.make_tensor( + tTMEM_LOADtO.iterator + i * corr_tile_size, + tTMEM_LOADtO.layout, + ), + tTMrO_i, + ) + for j in range(0, cute.size(tTMrO_i), 2): + tTMrO_i[j], tTMrO_i[j + 1] = cute.arch.mul_packed_f32x2( + (tTMrO_i[j], tTMrO_i[j + 1]), + (scale, scale), + ) + cute.copy( + tiled_tmem_store, + tTMrO_i, + cute.make_tensor( + tTMEM_STOREtO.iterator + i * corr_tile_size, + tTMEM_STOREtO.layout, + ), + ) + next_oi_peek_status = mma_corr_consumer.try_wait() + # Release Oi + cute.arch.fence_view_async_tmem_store() + oi_handle.release() + return (si_corr_consumer, mma_corr_consumer), next_oi_peek_status + + @cute.jit + def correction_epilog( + self, + thr_mma: cute.ThrMma, + tiled_tmem_load_vec: cute.TiledCopy, + tensor_args: Tuple, + pipeline_args: Tuple, + value_args: Tuple, + ): + """Apply final scaling and transformation to attention output. + + When use_tma_store=True: writes to shared memory and signals epilogue warp for TMA store. + When use_tma_store=False: writes directly to global memory via st.global. + + :param thr_mma: Thread MMA operation for the computation + :type thr_mma: cute.ThrMma + :param tiled_tmem_load_vec: Tiled memory load operation for the vectorized row-wise max + :type tiled_tmem_load_vec: cute.TiledCopy + :param tensor_args: Tuple containing (tOtO, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECcS, sO_or_gO, mLSE, mSink) + :type tensor_args: Tuple + :param pipeline_args: When use_tma_store: (si_corr_consumer, mma_corr_consumer, corr_epi_producer). + When not use_tma_store: (si_corr_consumer, mma_corr_consumer). + :type pipeline_args: Tuple + :param value_args: Tuple containing (row_idx, cuseqlen_q, seqlen_q, blk_coord, scale_softmax, scale_output) + :type value_args: Tuple + """ + ( + tOtO, + tTMEM_LOAD_VECtSi, + tTMEM_LOAD_VECcS, + dest_O, + mLSE, + mSink, + m_scale_v_channels, + ) = tensor_args + row_idx, cuseqlen_q, seqlen_q, blk_coord, scale_softmax, scale_output = value_args + + pv_tiled_mma_shape = ( + self.pv_mma_tiler[0], + self.pv_mma_tiler[1], + ) + cO = cute.make_identity_tensor(pv_tiled_mma_shape) + + corr_tile_size = 32 * 8 // self.o_dtype.width + tOdO = thr_mma.partition_C(dest_O) + tOcO = thr_mma.partition_C(cO) + tOtO_i = cute.logical_divide(tOtO, cute.make_layout((128, corr_tile_size))) + tOcO_i = cute.logical_divide(tOcO, cute.make_layout((128, corr_tile_size))) + tOdO_i = cute.logical_divide(tOdO, cute.make_layout((128, corr_tile_size))) + + tidx, _, _ = cute.arch.thread_idx() + thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) + epi_subtile = (self.epi_tile[0], corr_tile_size) + tmem_copy_atom = sm100_utils.get_tmem_load_op( + self.pv_mma_tiler, + self.o_layout, + self.o_dtype, + self.pv_acc_dtype, + epi_subtile, + use_2cta_instrs=False, + ) + tiled_tmem_load = tcgen05.make_tmem_copy(tmem_copy_atom, tOtO_i[(None, None), 0]) + thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) + tTMEM_LOADtO = thr_tmem_load.partition_S(tOtO_i[(None, None), None]) + tTMEM_LOADdO = thr_tmem_load.partition_D(tOdO_i[(None, None), None]) + tTMEM_LOADoO = thr_tmem_load.partition_D(tOcO_i[(None, None), None]) + + if cutlass.const_expr(self.use_tma_store): + si_corr_consumer, mma_corr_consumer, corr_epi_producer = pipeline_args + smem_copy_atom = sm100_utils.get_smem_store_op( + self.o_layout, self.o_dtype, self.pv_acc_dtype, tiled_tmem_load + ) + tiled_smem_store = cute.make_tiled_copy_D(smem_copy_atom, tiled_tmem_load) + else: + si_corr_consumer, mma_corr_consumer = pipeline_args + + # Wait for vec_i (row_wise global sum) + vec_i_handle = si_corr_consumer.wait_and_advance() + tTMEM_LOAD_VECrS = cute.make_rmem_tensor(tTMEM_LOAD_VECcS.shape, self.qk_acc_dtype) + cute.copy(tiled_tmem_load_vec, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECrS) + cute.arch.fence_view_async_tmem_load() + vec_i_handle.release() + + # Wait for Oi + oi_handle = mma_corr_consumer.wait_and_advance() + if cutlass.const_expr(self.use_tma_store): + oi_final_handle = corr_epi_producer.acquire_and_advance() + row_sum = tTMEM_LOAD_VECrS[0] + if cutlass.const_expr(mSink is not None): + sink_val = mSink[blk_coord[2]] + row_max_raw = tTMEM_LOAD_VECrS[1] + # sink is already in scaled logit space, row_max_raw is unscaled + # exp2((sink - max_scaled) * log2(e)) = exp(sink - max_scaled) + log2_e = Float32(1.4426950408889634) + sink_exp = cute.math.exp2( + (sink_val - row_max_raw * scale_softmax) * log2_e, fastmath=True + ) + row_sum = row_sum + sink_exp + scale = scale_output / row_sum + + if cutlass.const_expr(m_scale_v_channels is not None): + scale_v_ch_h = m_scale_v_channels[None, blk_coord[2]] + for i in range(self.cta_tiler[2] // corr_tile_size): + tTMEM_LOADtO_i = tTMEM_LOADtO[None, 0, 0, i] + tTMEM_LOADdO_i = tTMEM_LOADdO[None, 0, 0, i] + tTMEM_LOADoO_i = tTMEM_LOADoO[None, 0, 0, i] + tTMrO = cute.make_rmem_tensor(tTMEM_LOADoO_i.shape, self.pv_acc_dtype) + cute.copy(tiled_tmem_load, tTMEM_LOADtO_i, tTMrO) + for j in range(0, cute.size(tTMrO), 2): + tTMrO[j], tTMrO[j + 1] = cute.arch.mul_packed_f32x2( + (tTMrO[j], tTMrO[j + 1]), + (scale, scale), + ) + if cutlass.const_expr(m_scale_v_channels is not None): + for j in range(0, cute.size(tTMrO), 2): + _, n0 = tTMEM_LOADoO_i[j] + _, n1 = tTMEM_LOADoO_i[j + 1] + tTMrO[j], tTMrO[j + 1] = cute.arch.mul_packed_f32x2( + (tTMrO[j], tTMrO[j + 1]), + (scale_v_ch_h[n0], scale_v_ch_h[n1]), + ) + tDMrO = cute.make_rmem_tensor(tTMrO.shape, self.o_dtype) + o_vec = tTMrO.load() + tDMrO.store(o_vec.to(self.o_dtype)) + if cutlass.const_expr(self.use_tma_store): + # TMA store path: write to shared memory + cute.copy(tiled_smem_store, tDMrO, tTMEM_LOADdO_i) + else: + # st.global path: write directly to global memory with bounds check + if row_idx < seqlen_q: + cute.autovec_copy(tDMrO, tTMEM_LOADdO_i) + + if cutlass.const_expr(mLSE is not None): + scaled_tmp = scale_softmax * tTMEM_LOAD_VECrS[1] + # Convert LSE from natural log to log2 space, consistent with flashinfer trtllm-gen backend + lse = (cute.math.log(row_sum, fastmath=True) + scaled_tmp) * Float32(1.4426950408889634) + # Pre-scale correction: row_sum was inflated by 2^offset, so the + # log2-space LSE is too large by exactly `p_fp8_prescale_log2`. + if cutlass.const_expr(self.v_dtype.width == 8 and self.p_fp8_prescale_log2 > 0): + lse = lse - self.p_fp8_prescale_log2 + if row_idx < seqlen_q: + mLSE[row_idx + cuseqlen_q, blk_coord[2]] = lse + if cutlass.const_expr(self.use_tma_store): + # fence view async shared + cute.arch.fence_view_async_shared() + oi_handle.release() + oi_final_handle.commit() + return (si_corr_consumer, mma_corr_consumer, corr_epi_producer) + else: + oi_handle.release() + return (si_corr_consumer, mma_corr_consumer) + + def check_supported_dtypes( + self, + qk_dtype: Type[cutlass.Numeric], + pv_dtype: Type[cutlass.Numeric], + out_dtype: Type[cutlass.Numeric], + qk_sf_dtype: Type[cutlass.Numeric], + qk_sf_vec_size: int, + qk_acc_dtype: Type[cutlass.Numeric], + pv_acc_dtype: Type[cutlass.Numeric], + ): + if qk_dtype in {cutlass.Float8E4M3FN, cutlass.Float8E5M2}: + if qk_sf_dtype is not cutlass.Float8E8M0FNU: + raise NotImplementedError("MXFP8 QK requires qk_sf_dtype Float8E8M0FNU") + if qk_sf_vec_size != 32: + raise NotImplementedError("MXFP8 QK requires qk_sf_vec_size 32") + elif qk_dtype is cutlass.Float4E2M1FN: + if qk_sf_dtype is not cutlass.Float8E4M3FN: + raise NotImplementedError("NVFP4 QK requires qk_sf_dtype Float8E4M3FN") + if qk_sf_vec_size != 16: + raise NotImplementedError("NVFP4 QK requires qk_sf_vec_size 16") + else: + raise NotImplementedError( + "qk_dtype must be Float8E4M3FN/Float8E5M2 for MXFP8 or Float4E2M1FN for NVFP4" + ) + + if pv_dtype not in {cutlass.Float8E4M3FN, cutlass.BFloat16}: + raise NotImplementedError("pv_dtype must be Float8E4M3FN or BFloat16") + if out_dtype not in {cutlass.Float8E4M3FN, cutlass.Float16, cutlass.BFloat16}: + raise NotImplementedError("Unsupported out_dtype") + if qk_acc_dtype not in {cutlass.Float32}: + raise NotImplementedError("Unsupported qk_acc_dtype") + if pv_acc_dtype not in {cutlass.Float32}: + raise NotImplementedError("Unsupported pv_acc_dtype") + + def check_invalid_shape( + self, + qk_dtype: Type[cutlass.Numeric], + qk_sf_vec_size: int, + q_shape: Tuple[int, int, int, int], + k_shape: Tuple[int, int, int, int], + ): + # Shapes are passed around this example as (batch, seq_len, num_heads, head_dim). + b, s_q, h_q, d = q_shape + b_, s_k, h_k, d_ = k_shape + + if b != b_: + raise NotImplementedError("q & k must have the same batch size") + if d != d_: + raise NotImplementedError("q & k must have the same head dimension") + if qk_dtype is cutlass.Float4E2M1FN: + if d not in {64, 128}: + raise NotImplementedError("NVFP4 QK supports headdim 64 or 128") + elif d not in {32, 64, 128}: + raise NotImplementedError("MXFP8 QK supports headdim 32, 64, or 128") + if d % qk_sf_vec_size != 0: + raise NotImplementedError("head dimension must be divisible by qk_sf_vec_size") + if h_q % h_k != 0: + raise NotImplementedError("h_q must be divisible by h_k") + if isinstance(s_q, tuple) and len(s_q) != b: + raise NotImplementedError("variable_seqlen s_q must have the length of batch size") + if isinstance(s_k, tuple) and len(s_k) != b: + raise NotImplementedError("variable_seqlen s_k must have the length of batch size") + + def can_implement( + self, + q_shape: Tuple[int, int, int, int], + k_shape: Tuple[int, int, int, int], + qk_dtype: Type[cutlass.Numeric], + pv_dtype: Type[cutlass.Numeric], + out_dtype: Type[cutlass.Numeric], + qk_sf_dtype: Type[cutlass.Numeric], + qk_sf_vec_size: int, + qk_acc_dtype: Type[cutlass.Numeric], + pv_acc_dtype: Type[cutlass.Numeric], + ) -> bool: + """ + :param q_shape: Shape of the query tensor. + :type q_shape: Tuple[int, int, int, int] + :param k_shape: Shape of the key tensor. + :type k_shape: Tuple[int, int, int, int] + :param qk_dtype: Data type for Q and K (Bmm1 inputs). + :type qk_dtype: Type[cutlass.Numeric] + :param pv_dtype: Data type for P and V (Bmm2 inputs). + :type pv_dtype: Type[cutlass.Numeric] + :param out_dtype: Data type of the output tensor. + :type out_dtype: Type[cutlass.Numeric] + :param qk_acc_dtype: Data type of the qk accumulator tensor. + :type qk_acc_dtype: Type[cutlass.Numeric] + :param pv_acc_dtype: Data type of the pv accumulator tensor. + :type pv_acc_dtype: Type[cutlass.Numeric] + :return: True if the kernel can be implemented, False otherwise. + :rtype: bool + """ + try: + # Skip unsupported types + self.check_supported_dtypes( + qk_dtype, + pv_dtype, + out_dtype, + qk_sf_dtype, + qk_sf_vec_size, + qk_acc_dtype, + pv_acc_dtype, + ) + # Skip invalid shape + self.check_invalid_shape( + qk_dtype, + qk_sf_vec_size, + q_shape, + k_shape, + ) + except NotImplementedError: + return False + return True diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/__init__.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/__init__.py new file mode 100644 index 000000000000..467079831e16 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/fmha_helpers.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/fmha_helpers.py new file mode 100644 index 000000000000..f2034556e1d8 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/fmha_helpers.py @@ -0,0 +1,1176 @@ +# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ruff: noqa: I001, E501, F841 + +import enum +from typing import Tuple, Optional +import cutlass +from cutlass.cute.typing import Boolean +from cutlass._mlir.dialects import llvm, vector + +from cutlass.cutlass_dsl import ( + Int32, + Float32, + T, + min, + extract_mlir_values, + new_from_mlir_values, + dsl_user_op, +) +from cutlass.utils.hardware_info import HardwareInfo +from .static_persistent_tile_scheduler import WorkTileInfo +import cutlass.cute as cute + + +############################################################################## +# Fmha static tile scheduler +############################################################################## + + +class FmhaStaticTileSchedulerParams: + """A class to represent parameters for the FMHA (Fused Multi-Head Attention) static tile scheduler. + + This class holds the configuration parameters needed to initialize and configure + the tile scheduler for FMHA operations. + + :ivar is_persistent: Whether to use persistent kernel mode. + :type is_persistent: bool + :ivar problem_shape_mhb: Problem shape in (M, H, B) format. + :type problem_shape_mhb: cute.Shape + """ + + def __init__( + self, + is_persistent: bool, + problem_shape_mhb: cute.Shape, + *, + loc=None, + ip=None, + ): + """ + Initializes the FmhaStaticTileSchedulerParams with the given parameters. + + :param is_persistent: Whether to use persistent kernel mode. + :type is_persistent: bool + :param problem_shape_mhb: Problem shape in (M, H, B) format. + :type problem_shape_mhb: cute.Shape + """ + self.is_persistent = is_persistent + self.problem_shape_mhb = problem_shape_mhb + self._loc = loc + self._ip = ip + + def __extract_mlir_values__(self): + values, self._values_pos = [], [] + for obj in [self.problem_shape_mhb]: + obj_values = extract_mlir_values(obj) + values += obj_values + self._values_pos.append(len(obj_values)) + return values + + def __new_from_mlir_values__(self, values): + obj_list = [] + for obj, n_items in zip([self.problem_shape_mhb], self._values_pos): + obj_list.append(new_from_mlir_values(obj, values[:n_items])) + values = values[n_items:] + return FmhaStaticTileSchedulerParams(self.is_persistent, *(tuple(obj_list)), loc=self._loc) + + +class FmhaStaticTileScheduler: + """A static tile scheduler for FMHA (Fused Multi-Head Attention) operations. + + This class manages the scheduling of work tiles for FMHA kernels, supporting + both persistent and non-persistent kernel modes. It tracks the current work + position and advances through the problem space efficiently. + + :ivar _params: Scheduler parameters. + :type _params: FmhaStaticTileSchedulerParams + :ivar _blk_coord: Block coordinates. + :type _blk_coord: cute.Coord + :ivar _grid_shape: Grid shape for the kernel. + :type _grid_shape: cute.Shape + :ivar _is_persistent: Whether to use persistent kernel mode. + :type _is_persistent: bool + :ivar _current_work_linear_idx: Current linear work index. + :type _current_work_linear_idx: Int32 + :ivar _problem_shape_mhb: Problem shape in (M, H, B) format. + :type _problem_shape_mhb: cute.Layout + :ivar _num_blocks: Number of blocks in the problem. + :type _num_blocks: Int32 + :ivar _is_first_block: Whether this is the first block. + :type _is_first_block: bool + :ivar num_persistent_sm: Number of persistent SMs. + :type num_persistent_sm: Int32 + """ + + def __init__( + self, + params: FmhaStaticTileSchedulerParams, + current_work_linear_idx: Int32, + blk_coord: cute.Coord, + grid_shape: cute.Shape, + *, + loc=None, + ip=None, + ): + """ + Initializes the FmhaStaticTileScheduler with the given parameters. + + :param params: Scheduler parameters. + :type params: FmhaStaticTileSchedulerParams + :param current_work_linear_idx: Current linear work index. + :type current_work_linear_idx: Int32 + :param blk_coord: Block coordinates. + :type blk_coord: cute.Coord + :param grid_shape: Grid shape for the kernel. + :type grid_shape: cute.Shape + """ + self._params = params + self._blk_coord = blk_coord + self._grid_shape = grid_shape + self._is_persistent = params.is_persistent + self._current_work_linear_idx = current_work_linear_idx + self._problem_shape_mhb = cute.make_layout(params.problem_shape_mhb, loc=loc, ip=ip) + self._num_blocks = cute.size(self._problem_shape_mhb, loc=loc, ip=ip) + self._is_first_block = True + self.num_persistent_sm = cute.size(grid_shape, loc=loc, ip=ip) + self._loc = loc + self._ip = ip + + # called by host + @staticmethod + def get_grid_shape( + params: FmhaStaticTileSchedulerParams, + *, + loc=None, + ip=None, + ) -> cute.Shape: + """ + Determine the grid shape for the FMHA kernel. + + For persistent kernels, the grid shape is limited by the number of SMs + (Streaming Multiprocessors) available on the device. For non-persistent + kernels, the grid shape matches the problem shape. + + :param params: Scheduler parameters. + :type params: FmhaStaticTileSchedulerParams + + :return: Grid shape as (M, H, B) tuple. + :rtype: cute.Shape + """ + if params.is_persistent: + hardware_info = HardwareInfo() + sm_count = hardware_info.get_device_multiprocessor_count() + return ( + min(sm_count, cute.size(params.problem_shape_mhb, loc=loc, ip=ip)), + 1, + 1, + ) + else: + return params.problem_shape_mhb + + @staticmethod + def check_valid_work_for_seqlen_q( + q_tiler: int, + current_idx: Int32, + seqlen_q: Int32, + ) -> Boolean: + """ + Check if the current work index is valid for the given query sequence length. + + This method verifies that the current work tile index multiplied by the + query tiler size is within the bounds of the query sequence length. + + :param q_tiler: Query tiler size. + :type q_tiler: int + :param current_idx: Current work index. + :type current_idx: Int32 + :param seqlen_q: Query sequence length. + :type seqlen_q: Int32 + + :return: True if the work is valid, False otherwise. + :rtype: Boolean + """ + return current_idx * q_tiler < seqlen_q + + def get_current_work(self, *, loc=None, ip=None) -> WorkTileInfo: + """ + Get information about the current work tile. + + Determines if the current work is valid and computes the tile coordinates + based on whether the kernel is persistent or non-persistent. + + :return: WorkTileInfo containing tile coordinates and validity flag. + :rtype: WorkTileInfo + """ + is_valid = ( + self._current_work_linear_idx < self._num_blocks + if self._is_persistent + else self._is_first_block + ) + + blk_coord = (0, 0, 0) + if self._is_persistent: + blk_coord = self._problem_shape_mhb.get_hier_coord( + self._current_work_linear_idx, loc=loc, ip=ip + ) + else: + blk_coord = self._blk_coord + + # cur_tile_coord is (mid, 0, (bid, hid)) + cur_tile_coord = ( + blk_coord[0], + 0, + (blk_coord[1], blk_coord[2]), + ) + + return WorkTileInfo(cur_tile_coord, is_valid) + + def initial_work_tile_info(self, *, loc=None, ip=None): + """ + Get the initial work tile information. + + :return: Initial WorkTileInfo. + :rtype: WorkTileInfo + """ + return self.get_current_work(loc=loc, ip=ip) + + def advance_to_next_work(self, *, advance_count=1, loc=None, ip=None): + """ + Advance to the next work tile. + + For persistent kernels, advances by the number of persistent SMs. + For non-persistent kernels, marks that the first block has been processed. + + :param advance_count: Number of steps to advance (default: 1). + :type advance_count: int + """ + if self._is_persistent: + self._current_work_linear_idx += advance_count * self.num_persistent_sm + self._is_first_block = False + + def __extract_mlir_values__(self): + # Only pass mutable per-iteration state as scf.while block arguments. + # _params and _grid_shape are loop-invariant and captured from outer + # scope, keeping block arg count low (4 instead of 10). + values = extract_mlir_values(self._current_work_linear_idx) + values.extend(extract_mlir_values(self._blk_coord)) + return values + + def __new_from_mlir_values__(self, values): + assert len(values) == 4 + new_current_work_linear_idx = new_from_mlir_values( + self._current_work_linear_idx, [values[0]] + ) + new_blk_coord = new_from_mlir_values(self._blk_coord, values[1:4]) + return FmhaStaticTileScheduler( + self._params, new_current_work_linear_idx, new_blk_coord, self._grid_shape + ) + + +def create_fmha_static_tile_scheduler( + params: FmhaStaticTileSchedulerParams, + blk_coord: cute.Coord, + grid_shape: cute.Shape, +) -> FmhaStaticTileScheduler: + """ + Create a new FMHA static tile scheduler. + + :param params: Scheduler parameters. + :type params: FmhaStaticTileSchedulerParams + :param blk_coord: Block coordinates. + :type blk_coord: cute.Coord + :param grid_shape: Grid shape. + :type grid_shape: cute.Shape + + :return: New FmhaStaticTileScheduler instance. + :rtype: FmhaStaticTileScheduler + """ + return FmhaStaticTileScheduler(params, blk_coord[0], blk_coord, grid_shape) + + +def create_fmha_static_tile_scheduler_params( + is_persistent: bool, + problem_shape_mhb: cute.Shape, +) -> FmhaStaticTileSchedulerParams: + """ + Create FMHA static tile scheduler parameters. + + :param is_persistent: Whether to use persistent kernel mode. + :type is_persistent: bool + :param problem_shape_mhb: Problem shape in (M, H, B) format. + :type problem_shape_mhb: cute.Shape + + :return: New FmhaStaticTileSchedulerParams instance. + :rtype: FmhaStaticTileSchedulerParams + """ + return FmhaStaticTileSchedulerParams(is_persistent, problem_shape_mhb) + + +def compute_grid( + o_shape: cute.Shape, + cta_tiler: Tuple[int, int, int], + is_persistent: bool, + is_2cta: bool = False, +) -> Tuple[FmhaStaticTileSchedulerParams, Tuple[int, int, int]]: + """ + Compute grid parameters for FMHA operation. + + This function calculates the appropriate grid shape and scheduler parameters + based on the output tensor shape, CTA (Cooperative Thread Array) tiler, + and whether to use persistent kernel mode. + + The output tensor o has shape (s, d, ((h_r, h_k), b)) where: + - s: sequence length + - d: head dimension + - h_r: number of heads for query + - h_k: number of heads for key + - b: batch size + + :param o_shape: Output tensor shape for grid computation. + :type o_shape: cute.Shape + :param cta_tiler: CTA tiler dimensions (M, N, K). + :type cta_tiler: Tuple[int, int, int] + :param is_persistent: Whether to use persistent kernel mode. + :type is_persistent: bool + :param is_2cta: Whether to use 2CTA mode. + :type is_2cta: bool + + :return: Tuple of (scheduler_params, grid_shape). + :rtype: Tuple[FmhaStaticTileSchedulerParams, Tuple[int, int, int]] + """ + tile_sched_params = create_fmha_static_tile_scheduler_params( + is_persistent, + ( + cute.round_up(cute.ceil_div(cute.size(o_shape[0]), cta_tiler[0]), 2 if is_2cta else 1), + cute.size(o_shape[2][0]), + cute.size(o_shape[2][1]), + ), + ) + grid = FmhaStaticTileScheduler.get_grid_shape(tile_sched_params) + + return tile_sched_params, grid + + +############################################################################## +# Fused Mask +############################################################################## + + +class MaskEnum(enum.Enum): + """Enumeration of mask types for FMHA operations. + + - RESIDUAL_MASK: Residual mask for handling variable sequence lengths + - WINDOW_MASK: Window mask for attention which also includes causal and no mask + - WINDOW_MASK_INFERENCE: Same as the window mask, but has the limitation that the end of q is aligned with the end of k + - WINDOW_MASK_BWD: Window mask for backward pass + - WINDOW_MASK_BWD_INFERENCE: Same as the window mask for backward pass, but has the limitation that the end of q is aligned with the end of k + """ + + RESIDUAL_MASK = enum.auto() + RESIDUAL_MASK_BWD = enum.auto() + WINDOW_MASK = enum.auto() + WINDOW_MASK_INFERENCE = enum.auto() + WINDOW_MASK_BWD = enum.auto() + WINDOW_MASK_BWD_INFERENCE = enum.auto() + + +class FusedMask: + """A fused mask implementation for FMHA operations. + + This class handles different types of attention masks including no mask, + residual mask for variable sequence lengths, and causal mask for + autoregressive attention patterns. + + The class provides methods to: + - Calculate trip counts for different mask types + - Apply masks to attention scores + - Handle masked and unmasked trip calculations + """ + + def get_trip_count( + mask_type: MaskEnum, + blk_coord: cute.Coord, + tile_shape: cute.Shape, + seqlen_q: Int32, + seqlen_k: Int32, + window_size_left: Optional[Int32] = None, + window_size_right: Optional[Int32] = None, + ) -> Int32: + """ + Calculate the number of trips needed for the current block. + + The trip count depends on the mask type and the block coordinates. + For causal masks, it considers the autoregressive constraint. + + :param mask_type: Type of mask to use + :type mask_type: utils.MaskEnum + :param blk_coord: Block coordinates. + :type blk_coord: cute.Coord + :param tile_shape: Shape of the tile. + :type tile_shape: cute.Shape + :param seqlen_q: Query sequence length for attention computation. + :type seqlen_q: Int32 + :param seqlen_k: Key sequence length for attention computation. + :type seqlen_k: Int32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + + :return: Number of trips needed. + :rtype: Int32 + """ + result = 0 + offset = 0 + if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_INFERENCE): + offset = seqlen_k - seqlen_q + if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE): + offset = seqlen_q - seqlen_k + if cutlass.const_expr(mask_type == MaskEnum.RESIDUAL_MASK): + result = cute.ceil_div(seqlen_k, tile_shape[1]) + if cutlass.const_expr(mask_type is MaskEnum.RESIDUAL_MASK_BWD): + result = cute.ceil_div(seqlen_q, tile_shape[0]) + if cutlass.const_expr( + mask_type == MaskEnum.WINDOW_MASK or mask_type == MaskEnum.WINDOW_MASK_INFERENCE + ): + if cutlass.const_expr(window_size_right is None): + result = cute.ceil_div(seqlen_k, tile_shape[1]) + else: + max_idx_q = (blk_coord[0] + 1) * tile_shape[0] + idx_k = max_idx_q + offset + window_size_right + tmp_blocks_k = cute.ceil_div(idx_k, tile_shape[1]) + max_blocks_k = cute.ceil_div(seqlen_k, tile_shape[1]) + result = min(max_blocks_k, tmp_blocks_k) + if cutlass.const_expr( + mask_type == MaskEnum.WINDOW_MASK_BWD or mask_type == MaskEnum.WINDOW_MASK_BWD_INFERENCE + ): + if cutlass.const_expr(window_size_left is None): + result = cute.ceil_div(seqlen_q, tile_shape[0]) + else: + max_idx_k = (blk_coord[1] + 1) * tile_shape[1] + idx_k = max_idx_k + offset + window_size_left + tmp_blocks_q = cute.ceil_div(idx_k, tile_shape[0]) + max_blocks_q = cute.ceil_div(seqlen_q, tile_shape[0]) + result = min(max_blocks_q, tmp_blocks_q) + start_block = FusedMask.get_trip_start( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + result = result - start_block + return result + + @cute.jit + def get_trip_start( + mask_type: MaskEnum, + blk_coord: cute.Coord, + tile_shape: cute.Shape, + seqlen_q: Int32, + seqlen_k: Int32, + window_size_left: Optional[Int32] = None, + window_size_right: Optional[Int32] = None, + ) -> Int32: + """ + Get the start of the trip for the current block. + + :param mask_type: Type of mask to use + :type mask_type: utils.MaskEnum + :param blk_coord: Block coordinates. + :type blk_coord: cute.Coord + :param tile_shape: Shape of the tile. + :type tile_shape: cute.Shape + :param seqlen_q: Query sequence length for attention computation. + :type seqlen_q: Int32 + :param seqlen_k: Key sequence length for attention computation. + :type seqlen_k: Int32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + """ + result = 0 + offset = 0 + if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_INFERENCE): + offset = seqlen_k - seqlen_q + if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE): + offset = seqlen_q - seqlen_k + if cutlass.const_expr( + mask_type is MaskEnum.WINDOW_MASK or mask_type is MaskEnum.WINDOW_MASK_INFERENCE + ): + if cutlass.const_expr(window_size_left is not None): + min_idx_q = blk_coord[0] * tile_shape[0] + idx_k = min_idx_q + offset - window_size_left + tmp_blocks_k = idx_k // tile_shape[1] + result = max(tmp_blocks_k, result) + if cutlass.const_expr( + mask_type is MaskEnum.WINDOW_MASK_BWD or mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE + ): + if cutlass.const_expr(window_size_right is not None): + min_idx_k = blk_coord[1] * tile_shape[1] + idx_q = min_idx_k + offset - window_size_right + tmp_blocks_q = idx_q // tile_shape[0] + result = max(tmp_blocks_q, result) + return result + + @cute.jit + def get_leading_mask_id( + mask_type: MaskEnum, + blk_coord: cute.Coord, + tile_shape: cute.Shape, + seqlen_q: Int32, + seqlen_k: Int32, + window_size_left: Optional[Int32] = None, + window_size_right: Optional[Int32] = None, + ) -> Tuple[Int32, Int32]: + """ + Get the begin and end tile idx for the leading mask. + + :param mask_type: Type of mask to use + :type mask_type: utils.MaskEnum + :param blk_coord: Block coordinates. + :type blk_coord: cute.Coord + :param tile_shape: Shape of the tile. + :type tile_shape: cute.Shape + :param seqlen_q: Query sequence length for attention computation. + :type seqlen_q: Int32 + :param seqlen_k: Key sequence length for attention computation. + :type seqlen_k: Int32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + + :return: Tuple of (begin, end) tile idx for the leading mask. + :rtype: Tuple[Int32, Int32] + """ + offset = 0 + if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_INFERENCE): + offset = seqlen_k - seqlen_q + if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE): + offset = seqlen_q - seqlen_k + leading_mask_begin = FusedMask.get_trip_start( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + trip_count = FusedMask.get_trip_count( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + + leading_mask_end = leading_mask_begin + if cutlass.const_expr( + mask_type is MaskEnum.WINDOW_MASK or mask_type is MaskEnum.WINDOW_MASK_INFERENCE + ): + if cutlass.const_expr(window_size_left is not None): + min_idx_q = (blk_coord[0] + 1) * tile_shape[0] + offset - window_size_left + leading_mask_end = min( + cute.ceil_div(min_idx_q, tile_shape[1]) - 1, + trip_count + leading_mask_begin - 1, + ) + else: + leading_mask_end = leading_mask_begin - 1 + elif cutlass.const_expr( + mask_type is MaskEnum.WINDOW_MASK_BWD or mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE + ): + if cutlass.const_expr(window_size_right is not None): + min_idx_k = (blk_coord[1] + 1) * tile_shape[1] + offset - window_size_right + leading_mask_end = cute.ceil_div(min_idx_k, tile_shape[0]) - 1 + else: + leading_mask_end = leading_mask_begin - 1 + return leading_mask_begin, leading_mask_end + + @cute.jit + def get_trailing_mask_id( + mask_type: MaskEnum, + blk_coord: cute.Coord, + tile_shape: cute.Shape, + seqlen_q: Int32, + seqlen_k: Int32, + window_size_left: Optional[Int32] = None, + window_size_right: Optional[Int32] = None, + ) -> Tuple[Optional[Int32], Optional[Int32]]: + """ + Get the begin and end tile idx for the trailing mask. + + :param mask_type: Type of mask to use + :type mask_type: utils.MaskEnum + :param blk_coord: Block coordinates. + :type blk_coord: cute.Coord + :param tile_shape: Shape of the tile. + :type tile_shape: cute.Shape + :param seqlen_q: Query sequence length for attention computation. + :type seqlen_q: Int32 + :param seqlen_k: Key sequence length for attention computation. + :type seqlen_k: Int32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + + :return: Tuple of (begin, end) tile idx for the trailing mask. + :rtype: Tuple[Int32, Int32] + """ + offset = 0 + if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_INFERENCE): + offset = seqlen_k - seqlen_q + if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE): + offset = seqlen_q - seqlen_k + trip_start = FusedMask.get_trip_start( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + trip_count = FusedMask.get_trip_count( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + + trailing_mask_begin, trailing_mask_end = None, None + if cutlass.const_expr( + mask_type is MaskEnum.WINDOW_MASK or mask_type is MaskEnum.WINDOW_MASK_INFERENCE + ): + if cutlass.const_expr(window_size_right is not None): + min_idx_q = blk_coord[0] * tile_shape[0] + offset + window_size_right + trailing_mask_begin = min(min_idx_q // tile_shape[1], trip_count + trip_start - 1) + trailing_mask_end = trip_count + trip_start - 1 + else: + # last tile, we always apply mask on it regardless whether it's a residual tile + trailing_mask_begin = trip_count + trip_start - 1 + trailing_mask_end = trip_count + trip_start - 1 + else: + if cutlass.const_expr(window_size_left is not None): + min_idx_k = blk_coord[1] * tile_shape[1] + offset + window_size_left + 1 + max_idx_k = (blk_coord[1] + 1) * tile_shape[1] + offset + window_size_left + trailing_mask_begin = min( + cute.ceil_div(min_idx_k, tile_shape[0]) - 1, + trip_count + trip_start - 1, + ) + trailing_mask_end = min( + cute.ceil_div(max_idx_k, tile_shape[0]) - 1, + trip_count + trip_start - 1, + ) + else: + # last tile, we always apply mask on it regardless whether it's a residual tile + trailing_mask_begin = trip_count + trip_start - 1 + trailing_mask_end = trip_count + trip_start - 1 + + return trailing_mask_begin, trailing_mask_end + + @cute.jit + def get_masked_leading_count( + mask_type: MaskEnum, + blk_coord: cute.Coord, + tile_shape: cute.Shape, + seqlen_q: Int32, + seqlen_k: Int32, + window_size_left: Optional[Int32] = None, + window_size_right: Optional[Int32] = None, + ) -> Int32: + """ + Calculate the number of masked trips for the leading mask. + + This is used for blocks that need special handling due to masking. + + :param mask_type: Type of mask to use + :type mask_type: utils.MaskEnum + :param blk_coord: Block coordinates. + :type blk_coord: cute.Coord + :param tile_shape: Shape of the tile. + :type tile_shape: cute.Shape + :param seqlen_q: Query sequence length for attention computation. + :type seqlen_q: Int32 + :param seqlen_k: Key sequence length for attention computation. + :type seqlen_k: Int32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + + :return: Number of masked trips. + :rtype: Int32 + """ + result = 0 + if cutlass.const_expr( + mask_type is not MaskEnum.RESIDUAL_MASK and mask_type is not MaskEnum.RESIDUAL_MASK_BWD + ): + if cutlass.const_expr(window_size_left is not None or window_size_right is not None): + leading_mask_begin, leading_mask_end = FusedMask.get_leading_mask_id( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + result = max(leading_mask_end - leading_mask_begin + 1, 0) + + return result + + @cute.jit + def get_masked_trailing_count( + mask_type: MaskEnum, + blk_coord: cute.Coord, + tile_shape: cute.Shape, + seqlen_q: Int32, + seqlen_k: Int32, + window_size_left: Optional[Int32] = None, + window_size_right: Optional[Int32] = None, + rem_count: Optional[Int32] = 0, + ) -> Int32: + """ + Calculate the number of masked trips for the trailing mask. + + This is used for blocks that need special handling due to masking. + + :param mask_type: Type of mask to use + :type mask_type: utils.MaskEnum + :param blk_coord: Block coordinates. + :type blk_coord: cute.Coord + :param tile_shape: Shape of the tile. + :type tile_shape: cute.Shape + :param seqlen_q: Query sequence length for attention computation. + :type seqlen_q: Int32 + :param seqlen_k: Key sequence length for attention computation. + :type seqlen_k: Int32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + :param rem_count: Remaining count from previous calculations. + :type rem_count: Int32 + + :return: Number of masked trips. + :rtype: Int32 + """ + result = 0 + + if cutlass.const_expr( + mask_type is not MaskEnum.RESIDUAL_MASK and mask_type is not MaskEnum.RESIDUAL_MASK_BWD + ): + if cutlass.const_expr(window_size_left is not None or window_size_right is not None): + trailing_mask_begin, trailing_mask_end = FusedMask.get_trailing_mask_id( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + leading_mask_begin, leading_mask_end = FusedMask.get_leading_mask_id( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + if cutlass.const_expr( + trailing_mask_begin is not None and trailing_mask_end is not None + ): + if trailing_mask_begin <= leading_mask_end: + result = max(trailing_mask_end - leading_mask_end, 0) + else: + result = max(trailing_mask_end - trailing_mask_begin + 1, 0) + else: + if seqlen_k % tile_shape[1] != 0: + result = 1 + else: + result = 0 + + return result + rem_count + + @cute.jit + def get_unmasked_trip_count( + mask_type: MaskEnum, + blk_coord: cute.Coord, + tile_shape: cute.Shape, + seqlen_q: Int32, + seqlen_k: Int32, + window_size_left: Optional[Int32] = None, + window_size_right: Optional[Int32] = None, + ) -> Int32: + """ + Calculate the number of unmasked trips for the current block. + + This represents the number of trips that don't require special + masking treatment. + + :param mask_type: Type of mask to use + :type mask_type: utils.MaskEnum + :param blk_coord: Block coordinates. + :type blk_coord: cute.Coord + :param tile_shape: Shape of the tile. + :type tile_shape: cute.Shape + :param seqlen_q: Query sequence length for attention computation. + :type seqlen_q: Int32 + :param seqlen_k: Key sequence length for attention computation. + :type seqlen_k: Int32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + + :return: Number of unmasked trips. + :rtype: Int32 + """ + result = ( + FusedMask.get_trip_count( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + - FusedMask.get_masked_leading_count( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + - FusedMask.get_masked_trailing_count( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + 0, + ) + ) + return result + + @cute.jit + def get_masked_info( + mask_type: MaskEnum, + blk_coord: cute.Coord, + tile_shape: cute.Shape, + seqlen_q: Int32, + seqlen_k: Int32, + window_size_left: Optional[Int32] = None, + window_size_right: Optional[Int32] = None, + rem_count: Optional[Int32] = 0, + ) -> Tuple[Int32, Int32, Int32, Int32, Int32]: + """ + Calculate the number of masked trips for the trailing mask. + + This is used for blocks that need special handling due to masking. + + :param mask_type: Type of mask to use + :type mask_type: utils.MaskEnum + :param blk_coord: Block coordinates. + :type blk_coord: cute.Coord + :param tile_shape: Shape of the tile. + :type tile_shape: cute.Shape + :param seqlen_q: Query sequence length for attention computation. + :type seqlen_q: Int32 + :param seqlen_k: Key sequence length for attention computation. + :type seqlen_k: Int32 + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[Int32] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[Int32] + :param rem_count: Remaining count from previous calculations. + :type rem_count: Int32 + + :return: Number of masked info. + :rtype: Tuple[Int32, Int32, Int32, Int32, Int32] + """ + start_count = FusedMask.get_trip_start( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + leading_mask_count = FusedMask.get_masked_leading_count( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + unmask_count = FusedMask.get_unmasked_trip_count( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + ) + trailing_mask_count = FusedMask.get_masked_trailing_count( + mask_type, + blk_coord, + tile_shape, + seqlen_q, + seqlen_k, + window_size_left, + window_size_right, + rem_count, + ) + end_count = start_count + leading_mask_count + unmask_count + trailing_mask_count + return ( + start_count, + end_count, + leading_mask_count, + unmask_count, + trailing_mask_count, + ) + + @cute.jit + def apply_mask( + mask_type: MaskEnum, + acc_qk: cute.Tensor, + index_qk: cute.Tensor, + seqlen_q: Int32, + seqlen_k: Int32, + window_size_left: Optional[int] = None, + window_size_right: Optional[int] = None, + index_transform: cutlass.Constexpr = lambda index_q, index_k: ( + index_q, + index_k, + ), + ): + """ + Apply the appropriate mask to the attention scores. + + This method modifies the attention scores (acc_qk) based on the mask type + and the positions in the index tensor. + + :param mask_type: Type of mask to use + :type mask_type: utils.MaskEnum + :param acc_qk: Accumulated QK attention scores tensor. + :type acc_qk: cute.Tensor + :param index_qk: Index tensor containing position information. + :type index_qk: cute.Tensor + :param seqlen_k: Key sequence length for attention computation. + :type seqlen_k: Int32 + :param seqlen_q: Query sequence length for attention computation. + :type seqlen_q: Optional[int] + :param window_size_left: Left-side sliding window size for attention masking. + :type window_size_left: Optional[int] + :param window_size_right: Right-side sliding window size for attention masking. + :type window_size_right: Optional[int] + """ + + tidx, tidy, tidx = cute.arch.thread_idx() + offset = 0 + offset = ( + seqlen_k - seqlen_q + if cutlass.const_expr( + mask_type is MaskEnum.WINDOW_MASK_INFERENCE + or mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE + ) + else 0 + ) + for i in cutlass.range(cute.size(acc_qk), unroll_full=True): + index_q, index_k = index_transform(*index_qk[i]) + if cutlass.const_expr(window_size_left is not None or window_size_right is not None): + if cutlass.const_expr(window_size_left is None): + if index_q + offset + window_size_right < index_k: + acc_qk[i] = -Float32.inf + if index_k >= seqlen_k or index_q >= seqlen_q: # residual mask + acc_qk[i] = -Float32.inf + elif cutlass.const_expr(window_size_right is None): + if index_q + offset - window_size_left > index_k: + acc_qk[i] = -Float32.inf + if index_k >= seqlen_k or index_q >= seqlen_q: # residual mask + acc_qk[i] = -Float32.inf + else: + max_K_index = min(index_q + offset + window_size_right, seqlen_k) + min_K_index = max(0, index_q + offset - window_size_left) + if index_k > max_K_index or index_k < min_K_index: + acc_qk[i] = -Float32.inf + if index_k >= seqlen_k or index_q >= seqlen_q: # residual mask + acc_qk[i] = -Float32.inf + + if cutlass.const_expr( + mask_type == MaskEnum.RESIDUAL_MASK or mask_type == MaskEnum.RESIDUAL_MASK_BWD + ): + if index_k >= seqlen_k or index_q >= seqlen_q: + acc_qk[i] = -Float32.inf + + +@dsl_user_op +def ex2_emulation_packed_f32x2( + x: Float32, y: Float32, *, loc=None, ip=None +) -> Tuple[Float32, Float32]: + # Clamp the xy so they fit within the FP32 exponent range + # The upper side is ensured by the (s - row_max) + xy_clamped = (cute.arch.fmax(x, -127.0), cute.arch.fmax(y, -127.0)) + + fp32_round_int = float(2**23 + 2**22) + # | 0 | 10010110 | 10000000000000000000000 | + # | sign | exponent | mantissa | + # ^ + # | + # digit of ones place + # During FP32 addition, any number that smaller than this will be + # aligned the exponent to 2^23, so that the digit of ones + # is at the rightest place + # We want to round down here, so that the fractional part is in [0, 1) + xy_rounded = cute.arch.add_packed_f32x2(xy_clamped, (fp32_round_int, fp32_round_int), rnd="rm") + # The integer floor of x & y are now in the last 8 bits of xy_rounded + # We want the next 2 ops to round to nearest even. The rounding mode is important. + xy_rounded_back = cute.arch.sub_packed_f32x2(xy_rounded, (fp32_round_int, fp32_round_int)) + xy_frac = cute.arch.sub_packed_f32x2(xy_clamped, xy_rounded_back) + + @dsl_user_op + @cute.jit + def polynomial_deg3_packed_f32x2( + x: Float32, y: Float32, *, loc=None, ip=None + ) -> Tuple[Float32, Float32]: + # 2^x ~= (0.077 * x + 0.228) * x + 0.695) * x + 1, for x in [0, 1) + coeff = ( + 1.0, # coeff of deg0 + 0.695146143436431884765625, # coeff of deg1 + 0.227564394474029541015625, # coeff of deg2 + 0.077119089663028717041015625, # coeff of deg3 + ) + deg = len(coeff) - 1 # started with highest degree + out = (coeff[deg], coeff[deg]) + for i in cutlass.range_constexpr(deg - 1, -1, -1): + out = cute.arch.fma_packed_f32x2(out, (x, y), (coeff[i], coeff[i]), loc=loc, ip=ip) + return out + + xy_frac_ex2 = polynomial_deg3_packed_f32x2(*xy_frac, loc=loc, ip=ip) + + @dsl_user_op + def combine_int_frac_ex2( + x_rounded: Float32, frac_ex2: Float32, *, loc=None, ip=None + ) -> Float32: + return cutlass.Float32( + llvm.inline_asm( + T.f32(), + [ + Float32(x_rounded).ir_value(loc=loc, ip=ip), + Float32(frac_ex2).ir_value(loc=loc, ip=ip), + ], + "{\n\t" + ".reg .s32 x_rounded_i, frac_ex_i, x_rounded_e, out_i;\n\t" + "mov.b32 x_rounded_i, $1;\n\t" + "mov.b32 frac_ex_i, $2;\n\t" + "shl.b32 x_rounded_e, x_rounded_i, 23;\n\t" + "add.s32 out_i, x_rounded_e, frac_ex_i;\n\t" + "mov.b32 $0, out_i;\n\t" + "}\n", + "=f,f,f", + has_side_effects=False, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + ) + ) + + x_out = combine_int_frac_ex2(xy_rounded[0], xy_frac_ex2[0], loc=loc, ip=ip) + y_out = combine_int_frac_ex2(xy_rounded[1], xy_frac_ex2[1], loc=loc, ip=ip) + + return x_out, y_out + + +@cute.jit +def cvt_f32x4_to_f8x4_pack_i32(fp32x4, fp8_type, *, loc=None, ip=None): + fp32x4 = fp32x4.load() + src_vec4 = fp32x4.ir_value(loc=loc, ip=ip) if hasattr(fp32x4, "ir_value") else fp32x4 + + src0 = Float32(vector.extract(src_vec4, [], [0])).ir_value(loc=loc, ip=ip) + src1 = Float32(vector.extract(src_vec4, [], [1])).ir_value(loc=loc, ip=ip) + src2 = Float32(vector.extract(src_vec4, [], [2])).ir_value(loc=loc, ip=ip) + src3 = Float32(vector.extract(src_vec4, [], [3])).ir_value(loc=loc, ip=ip) + + cvt_instruction = "" + if cutlass.const_expr(fp8_type == cutlass.Float8E4M3FN): + cvt_instruction = "cvt.rn.satfinite.e4m3x2.f32" + else: + assert False, "Unsupported fp8 element type" + + asm_tmpl = ( + "{\n" + " .reg .b16 lo;\n" + " .reg .b16 hi;\n" + f" {cvt_instruction} lo, $2, $1;\n" + f" {cvt_instruction} hi, $4, $3;\n" + " mov.b32 $0, {lo, hi};\n" + "}" + ) + packed_i32 = llvm.inline_asm( + T.i32(), + [src0, src1, src2, src3], + asm_tmpl, + "=r,f,f,f,f", + has_side_effects=True, + is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, + ) + + return packed_i32 + + +@cute.jit +def cvt_f32x4_to_f8x4(fp32x4, fp8x4, *, loc=None, ip=None): + packed_i32 = cvt_f32x4_to_f8x4_pack_i32(fp32x4, fp8x4.element_type) + fp8x4_i32 = cute.recast_tensor(fp8x4, cutlass.Int32) + fp8x4_i32[0] = cutlass.Int32(packed_i32) + return diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/static_persistent_tile_scheduler.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/static_persistent_tile_scheduler.py new file mode 100644 index 000000000000..21e80c07b5cd --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/static_persistent_tile_scheduler.py @@ -0,0 +1,814 @@ +# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ruff: noqa: I001, E501 + +import inspect +from typing import Optional, Tuple + +from cutlass.cutlass_dsl import ( + Boolean, + Integer, + Int32, + min, + extract_mlir_values, + new_from_mlir_values, + dsl_user_op, + const_expr, +) +from cutlass._mlir import ir +import cutlass.cute as cute + +############################################################################## +# Static persistent tile scheduler +############################################################################## + + +class WorkTileInfo: + """A class to represent information about a work tile. + + :ivar tile_idx: The index of the tile. + :type tile_idx: cute.Coord + :ivar is_valid_tile: Whether the tile is valid. + :type is_valid_tile: Boolean + """ + + def __init__(self, tile_idx: cute.Coord, is_valid_tile: Boolean): + self._tile_idx = tile_idx + self._is_valid_tile = Boolean(is_valid_tile) + + def __extract_mlir_values__(self) -> list[ir.Value]: + values = extract_mlir_values(self.tile_idx) + values.extend(extract_mlir_values(self.is_valid_tile)) + return values + + def __new_from_mlir_values__(self, values: list[ir.Value]) -> "WorkTileInfo": + assert len(values) == 4 + new_tile_idx = new_from_mlir_values(self._tile_idx, values[:-1]) + new_is_valid_tile = new_from_mlir_values(self._is_valid_tile, [values[-1]]) + return WorkTileInfo(new_tile_idx, new_is_valid_tile) + + @property + @cute.jit + def is_valid_tile(self) -> Boolean: + """Check latest tile returned by the scheduler is valid or not. Any scheduling + requests after all tasks completed will return an invalid tile. + + :return: The validity of the tile. + :rtype: Boolean + """ + return self._is_valid_tile + + @property + @cute.jit + def tile_idx(self) -> cute.Coord: + """ + Get the index of the tile. + + :return: The index of the tile. + :rtype: cute.Coord + """ + return self._tile_idx + + +class PersistentTileSchedulerParams: + """A class to represent parameters for a persistent tile scheduler. + + This class is designed to manage and compute the layout of clusters and tiles + in a batched gemm problem. + + :ivar cluster_shape_mn: Shape of the cluster in (m, n) dimensions (K dimension cta count must be 1). + :type cluster_shape_mn: tuple + :ivar problem_layout_ncluster_mnl: Layout of the problem in terms of + number of clusters in (m, n, l) dimensions. + :type problem_layout_ncluster_mnl: cute.Layout + """ + + @dsl_user_op + def __init__( + self, + problem_shape_ntile_mnl: cute.Shape, + cluster_shape_mnk: cute.Shape, + swizzle_size: int = 1, + raster_along_m: bool = True, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> None: + """ + Initializes the PersistentTileSchedulerParams with the given parameters. + + :param problem_shape_ntile_mnl: The shape of the problem in terms of + number of CTA (Cooperative Thread Array) in (m, n, l) dimensions. + :type problem_shape_ntile_mnl: cute.Shape + :param cluster_shape_mnk: The shape of the cluster in (m, n) dimensions. + :type cluster_shape_mnk: cute.Shape + :param swizzle_size: Swizzling size in the unit of cluster. 1 means no swizzle + :type swizzle_size: int + :param raster_along_m: Rasterization order of clusters. Only used when swizzle_size > 1. + True means along M, false means along N. + :type raster_along_m: bool + + :raises ValueError: If cluster_shape_k is not 1. + """ + + if cluster_shape_mnk[2] != 1: # type: ignore[index] + raise ValueError(f"unsupported cluster_shape_k {cluster_shape_mnk[2]}") # type: ignore[index] + if swizzle_size < 1: + raise ValueError(f"expect swizzle_size >= 1, but get {swizzle_size}") + + self.problem_shape_ntile_mnl = problem_shape_ntile_mnl + # cluster_shape_mnk is kept for reconstruction + self._cluster_shape_mnk = cluster_shape_mnk + self.cluster_shape_mn = cluster_shape_mnk[:2] # type: ignore[index] + self.swizzle_size = swizzle_size + self.raster_along_m = raster_along_m + self._loc = loc + + # By default, we follow m major (col-major) raster order, so make a col-major layout + self.problem_layout_ncluster_mnl = cute.make_layout( + cute.ceil_div( + self.problem_shape_ntile_mnl, + cluster_shape_mnk[:2], # type: ignore[index] + loc=loc, + ip=ip, + ), + loc=loc, + ip=ip, + ) + + # Apply swizzle if swizzle_size > 1 + if swizzle_size > 1: + problem_shape_ncluster_mnl = cute.round_up( + self.problem_layout_ncluster_mnl.shape, + (1, swizzle_size, 1) if raster_along_m else (swizzle_size, 1, 1), + ) + + if raster_along_m: + self.problem_layout_ncluster_mnl = cute.make_layout( + ( + problem_shape_ncluster_mnl[0], # type: ignore[index] + (swizzle_size, problem_shape_ncluster_mnl[1] // swizzle_size), # type: ignore[index, operator] + problem_shape_ncluster_mnl[2], # type: ignore[index] + ), + stride=( + swizzle_size, + (1, swizzle_size * problem_shape_ncluster_mnl[0]), # type: ignore[index] + problem_shape_ncluster_mnl[0] * problem_shape_ncluster_mnl[1], # type: ignore[index, operator] + ), + loc=loc, + ip=ip, + ) + else: + self.problem_layout_ncluster_mnl = cute.make_layout( + ( + (swizzle_size, problem_shape_ncluster_mnl[0] // swizzle_size), # type: ignore[index, operator] + problem_shape_ncluster_mnl[1], # type: ignore[index] + problem_shape_ncluster_mnl[2], # type: ignore[index] + ), + stride=( + (1, swizzle_size * problem_shape_ncluster_mnl[1]), # type: ignore[index] + swizzle_size, + problem_shape_ncluster_mnl[0] * problem_shape_ncluster_mnl[1], # type: ignore[index, operator] + ), + loc=loc, + ip=ip, + ) + + # Create FastDivmod divisors (only when swizzle_size == 1 for correctness) + # FastDivmod assumes simple col-major layout, incompatible with swizzled layouts + if swizzle_size == 1: + _problem_layout_size = cute.size(self.problem_layout_ncluster_mnl, loc=loc, ip=ip) + cluster_count_m = self.problem_layout_ncluster_mnl.shape[0] + cluster_count_n = self.problem_layout_ncluster_mnl.shape[1] + + if raster_along_m: + cluster_count_major = cluster_count_m + cluster_count_minor = cluster_count_n + else: + cluster_count_major = cluster_count_n + cluster_count_minor = cluster_count_m + + # cluster_shape_major_fdd: Used to decode work_unit_id to cluster coordinates + self.cluster_shape_major_fdd = cute.fast_divmod_create_divisor( + cluster_count_major, loc=loc, ip=ip + ) + + # cluster_shape_minor_fdd: Used for the second level decomposition + self.cluster_shape_minor_fdd = cute.fast_divmod_create_divisor( + cluster_count_minor, loc=loc, ip=ip + ) + else: + # FastDivmod not applicable with swizzling, set to None + self.cluster_shape_major_fdd = None + self.cluster_shape_minor_fdd = None + + def __extract_mlir_values__(self) -> list[ir.Value]: + values, self._values_pos = [], [] + for obj in [ + self.problem_shape_ntile_mnl, + self._cluster_shape_mnk, + self.swizzle_size, + self.raster_along_m, + ]: + obj_values = extract_mlir_values(obj) + values += obj_values + self._values_pos.append(len(obj_values)) + + # Add FastDivmod divisors to MLIR values for Host->Device transfer + # Only add non-None values to avoid MLIR type errors + fastdivmod_values = [] + fastdivmod_indices = [] # Track which FastDivmod objects are present + fastdivmod_lengths = [] # Track serialized MLIR value count per FDD + + for i, (fdd_name, fdd_obj) in enumerate( + [ + ("cluster_shape_major_fdd", self.cluster_shape_major_fdd), + ("cluster_shape_minor_fdd", self.cluster_shape_minor_fdd), + ] + ): + if fdd_obj is not None: + # Extract MLIR values from FastDivmodDivisor objects + fdd_values = extract_mlir_values(fdd_obj) + fastdivmod_values.extend(fdd_values) + fastdivmod_indices.append(i) + fastdivmod_lengths.append(len(fdd_values)) + + values += fastdivmod_values + self._values_pos.append( + len(fastdivmod_indices) + ) # Store count of FastDivmod objects, not values + self._fastdivmod_indices = fastdivmod_indices # Store for reconstruction + self._fastdivmod_lengths = fastdivmod_lengths # Per-FDD value count + + return values + + def __new_from_mlir_values__(self, values: list[ir.Value]) -> "PersistentTileSchedulerParams": + obj_list = [] + values_copy = list(values) # Make a copy to avoid modifying original + + # Reconstruct original objects from MLIR values + for obj, n_items in zip( + [ + self.problem_shape_ntile_mnl, + self._cluster_shape_mnk, + self.swizzle_size, + self.raster_along_m, + ], + self._values_pos[:-1], # Exclude FastDivmod count + ): + obj_list.append(new_from_mlir_values(obj, values_copy[:n_items])) + values_copy = values_copy[n_items:] + + # Create new params object by calling __init__ with reconstructed values + # This properly recreates layouts and other derived attributes in the device context + new_params = PersistentTileSchedulerParams(*(tuple(obj_list)), loc=self._loc) + + # Restore FastDivmod divisors from remaining values + fdd_names = ["cluster_shape_major_fdd", "cluster_shape_minor_fdd"] + + if hasattr(self, "_fastdivmod_indices") and len(self._fastdivmod_indices) > 0: + # Override the FastDivmod divisors created by __init__ with reconstructed ones. + # FastDivmodDivisor now emits multiple MLIR values per object (see issue #3243); + # use the per-FDD length recorded during __extract_mlir_values__ to slice the + # tail of values_copy without re-emitting IR. + fdd_tail_offset = 0 + for original_index, n_fdd in zip(self._fastdivmod_indices, self._fastdivmod_lengths): + fdd_name = fdd_names[original_index] + original_fdd = getattr(self, fdd_name) + if original_fdd is None: + continue + end = fdd_tail_offset + n_fdd + if end > len(values_copy): + raise ValueError( + f"FastDivmod values out of range for {fdd_name}: " + f"need {end}, have {len(values_copy)}" + ) + reconstructed_fdd = new_from_mlir_values( + original_fdd, values_copy[fdd_tail_offset:end] + ) + setattr(new_params, fdd_name, reconstructed_fdd) + fdd_tail_offset = end + if fdd_tail_offset != len(values_copy): + raise ValueError( + "Unexpected trailing FastDivmod MLIR values: " + f"consumed {fdd_tail_offset}, have {len(values_copy)}" + ) + + return new_params + + @dsl_user_op + def get_grid_shape( + self, + max_active_clusters: Int32, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> Tuple[Integer, Integer, Integer]: + """ + Computes the grid shape based on the maximum active clusters allowed. + + :param max_active_clusters: The maximum number of active clusters that + can run in one wave. + :type max_active_clusters: Int32 + + :return: A tuple containing the grid shape in (m, n, persistent_clusters). + - m: self.cluster_shape_m. + - n: self.cluster_shape_n. + - persistent_clusters: Number of persistent clusters that can run. + """ + + # Total ctas in problem size + num_ctas_mnl = tuple( + cute.size(x) * y + for x, y in zip(self.problem_layout_ncluster_mnl.shape, self.cluster_shape_mn) + ) + (self.problem_layout_ncluster_mnl.shape[2],) + + num_ctas_in_problem = cute.size(num_ctas_mnl, loc=loc, ip=ip) + + num_ctas_per_cluster = cute.size(self.cluster_shape_mn, loc=loc, ip=ip) + # Total ctas that can run in one wave + num_ctas_per_wave = max_active_clusters * num_ctas_per_cluster + + num_persistent_ctas = min(num_ctas_in_problem, num_ctas_per_wave) + num_persistent_clusters = num_persistent_ctas // num_ctas_per_cluster + + return (*self.cluster_shape_mn, num_persistent_clusters) + + +# Set explicit signature for Sphinx documentation to avoid issues with @dsl_user_op decorator +PersistentTileSchedulerParams.__init__.__signature__ = inspect.Signature( # type: ignore[attr-defined] + [ + inspect.Parameter("self", inspect.Parameter.POSITIONAL_OR_KEYWORD), + ] +) + + +class StaticPersistentTileScheduler: + """A scheduler for static persistent tile execution in CUTLASS/CuTe kernels. + + :ivar params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl + :type params: PersistentTileSchedulerParams + :ivar num_persistent_clusters: Number of persistent clusters that can be launched + :type num_persistent_clusters: Int32 + :ivar cta_id_in_cluster: ID of the CTA within its cluster + :type cta_id_in_cluster: cute.Coord + :ivar _num_tiles_executed: Counter for executed tiles + :type _num_tiles_executed: Int32 + :ivar _current_work_linear_idx: Current cluster index + :type _current_work_linear_idx: Int32 + """ + + def __init__( + self, + params: PersistentTileSchedulerParams, + num_persistent_clusters: Int32, + current_work_linear_idx: Int32, + cta_id_in_cluster: cute.Coord, + num_tiles_executed: Int32, + ): + """ + Initializes the StaticPersistentTileScheduler with the given parameters. + + :param params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl. + :type params: PersistentTileSchedulerParams + :param num_persistent_clusters: Number of persistent clusters that can be launched. + :type num_persistent_clusters: Int32 + :param current_work_linear_idx: Current cluster index. + :type current_work_linear_idx: Int32 + :param cta_id_in_cluster: ID of the CTA within its cluster. + :type cta_id_in_cluster: cute.Coord + :param num_tiles_executed: Counter for executed tiles. + :type num_tiles_executed: Int32 + """ + self.params = params + self.num_persistent_clusters = num_persistent_clusters + self._current_work_linear_idx = current_work_linear_idx + self.cta_id_in_cluster = cta_id_in_cluster + self._num_tiles_executed = num_tiles_executed + + def __extract_mlir_values__(self) -> list[ir.Value]: + values = extract_mlir_values(self.num_persistent_clusters) + values.extend(extract_mlir_values(self._current_work_linear_idx)) + values.extend(extract_mlir_values(self.cta_id_in_cluster)) + values.extend(extract_mlir_values(self._num_tiles_executed)) + + # CRITICAL: Also extract FastDivmod divisors from params + values.extend(extract_mlir_values(self.params)) + + return values + + def __new_from_mlir_values__(self, values: list[ir.Value]) -> "StaticPersistentTileScheduler": + assert len(values) >= 6 + new_num_persistent_clusters = new_from_mlir_values( + self.num_persistent_clusters, [values[0]] + ) + new_current_work_linear_idx = new_from_mlir_values( + self._current_work_linear_idx, [values[1]] + ) + new_cta_id_in_cluster = new_from_mlir_values(self.cta_id_in_cluster, values[2:5]) + new_num_tiles_executed = new_from_mlir_values(self._num_tiles_executed, [values[5]]) + + # Reconstruct params with FastDivmod divisors + params_values = values[6:] # Remaining values are from params + new_params = new_from_mlir_values(self.params, params_values) + + return StaticPersistentTileScheduler( + new_params, # Use reconstructed params with FastDivmod divisors + new_num_persistent_clusters, + new_current_work_linear_idx, + new_cta_id_in_cluster, + new_num_tiles_executed, + ) + + @staticmethod + @dsl_user_op + def create( + params: PersistentTileSchedulerParams, + block_idx: Tuple[Integer, Integer, Integer], + grid_dim: Tuple[Integer, Integer, Integer], + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> "StaticPersistentTileScheduler": + """Initialize the static persistent tile scheduler. + + :param params: Parameters for the persistent + tile scheduler. + :type params: PersistentTileSchedulerParams + :param block_idx: The 3d block index in the format (bidx, bidy, bidz). + :type block_idx: Tuple[Integer, Integer, Integer] + :param grid_dim: The 3d grid dimensions for kernel launch. + :type grid_dim: Tuple[Integer, Integer, Integer] + + :return: A StaticPersistentTileScheduler object. + :rtype: StaticPersistentTileScheduler + """ + + # Calculate the number of persistent clusters by dividing the total grid size + # by the number of CTAs per cluster + num_persistent_clusters = cute.size(grid_dim, loc=loc, ip=ip) // cute.size( + params.cluster_shape_mn, loc=loc, ip=ip + ) + + bidx, bidy, bidz = block_idx + + # Initialize workload index equals to the cluster index in the grid + current_work_linear_idx = Int32(bidz) + + # CTA id in the cluster + cta_id_in_cluster = ( + Int32(bidx % params.cluster_shape_mn[0]), + Int32(bidy % params.cluster_shape_mn[1]), + Int32(0), + ) + # Initialize number of tiles executed to zero + num_tiles_executed = Int32(0) + return StaticPersistentTileScheduler( + params, + num_persistent_clusters, + current_work_linear_idx, + cta_id_in_cluster, + num_tiles_executed, + ) + + # called by host + @staticmethod + def get_grid_shape( + params: PersistentTileSchedulerParams, + max_active_clusters: Int32, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> Tuple[Integer, Integer, Integer]: + """Calculates the grid shape to be launched on GPU using problem shape, + threadblock shape, and active cluster size. + + :param params: Parameters for grid shape calculation. + :type params: PersistentTileSchedulerParams + :param max_active_clusters: Maximum active clusters allowed. + :type max_active_clusters: Int32 + + :return: The calculated 3d grid shape. + :rtype: Tuple[Integer, Integer, Integer] + """ + + return params.get_grid_shape(max_active_clusters, loc=loc, ip=ip) + + # private method + def _get_current_work_for_linear_idx( + self, + current_work_linear_idx: Int32, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> WorkTileInfo: + """Compute current tile coord given current_work_linear_idx and cta_id_in_cluster. + + :param current_work_linear_idx: The linear index of the current work. + :type current_work_linear_idx: Int32 + + :return: An object containing information about the current tile coordinates + and validity status. + :rtype: WorkTileInfo + """ + + is_valid = current_work_linear_idx < cute.size( + self.params.problem_layout_ncluster_mnl, loc=loc, ip=ip + ) + + # Choose coordinate calculation method based on swizzle configuration + if self.params.swizzle_size == 1: + # Use FastDivmod optimization for non-swizzled layouts + cur_cluster_coord = self._get_cluster_work_idx_with_fastdivmod( + current_work_linear_idx, loc=loc, ip=ip + ) + else: + # Use get_flat_coord for swizzled layouts (FastDivmod doesn't support them) + cur_cluster_coord = self.params.problem_layout_ncluster_mnl.get_flat_coord( + current_work_linear_idx, loc=loc, ip=ip + ) + + cur_tile_coord = tuple( + cute.arch.make_warp_uniform(Int32(x) * Int32(z) + Int32(y)) + for x, y, z in zip( + cur_cluster_coord, + self.cta_id_in_cluster, # type: ignore[arg-type] + (*self.params.cluster_shape_mn, Int32(1)), + ) + ) + + return WorkTileInfo(cur_tile_coord, cute.arch.make_warp_uniform(is_valid)) + + def _get_cluster_work_idx_with_fastdivmod( + self, + current_work_linear_idx: Int32, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> Tuple[Int32, Int32, Int32]: + """ + FastDivmod optimized CLUSTER coordinate calculation. + + CRITICAL: This should mimic problem_layout_ncluster_mnl.get_hier_coord() + which returns CLUSTER coordinates, not tile coordinates! + + :param current_work_linear_idx: Linear index in the work space + :type current_work_linear_idx: Int32 + :return: Cluster coordinates (m, n, l) or None if FastDivmod not available + :rtype: Tuple[Int32, Int32, Int32] or None + """ + + # Step 1: Decode current_work_linear_idx using FastDivmod objects + # The layout structure is: problem_layout_ncluster_mnl has shape (cluster_count_m, cluster_count_n, batch_count) + # current_work_linear_idx needs to be decomposed into (batch_l, cluster_minor, cluster_major) in little-endian order + + # First, get cluster_major using cluster_shape_major_fdd + cluster_minor_batch, cluster_major = divmod( + current_work_linear_idx, self.params.cluster_shape_major_fdd + ) + + # Then decode cluster_minor_batch to get cluster_minor and batch_l using FastDivmod + batch_l, cluster_minor = divmod(cluster_minor_batch, self.params.cluster_shape_minor_fdd) + + if self.params.raster_along_m: + cluster_m = cluster_major + cluster_n = cluster_minor + else: + cluster_m = cluster_minor + cluster_n = cluster_major + + return (cluster_m, cluster_n, batch_l) + + @dsl_user_op + @cute.jit + def get_current_work( + self, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> WorkTileInfo: + return self._get_current_work_for_linear_idx(self._current_work_linear_idx, loc=loc, ip=ip) + + @dsl_user_op + @cute.jit + def initial_work_tile_info( + self, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> WorkTileInfo: + return self.get_current_work(loc=loc, ip=ip) + + @dsl_user_op + @cute.jit + def advance_to_next_work( + self, + *, + advance_count: int = 1, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> None: + self._current_work_linear_idx += Int32(advance_count) * Int32(self.num_persistent_clusters) + self._num_tiles_executed += Int32(1) + + @property + @cute.jit + def num_tiles_executed(self) -> Int32: + return self._num_tiles_executed + + +class StaticPersistentRuntimeTileScheduler(StaticPersistentTileScheduler): + """A scheduler for static persistent runtime tile execution in CUTLASS/CuTe kernels. + This scheduler will always launch all the SMs and the scheduler will generate the real tile info for each SM. + + :ivar params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl + :type params: PersistentTileSchedulerParams + :ivar num_persistent_clusters: Number of persistent clusters that can be launched + :type num_persistent_clusters: Int32 + :ivar cta_id_in_cluster: ID of the CTA within its cluster + :type cta_id_in_cluster: cute.Coord + :ivar _num_tiles_executed: Counter for executed tiles + :type _num_tiles_executed: Int32 + :ivar _current_work_linear_idx: Current cluster index + :type _current_work_linear_idx: Int32 + """ + + def __init__( + self, + params: PersistentTileSchedulerParams, + num_persistent_clusters: Int32, + current_work_linear_idx: Int32, + cta_id_in_cluster: cute.Coord, + num_tiles_executed: Int32, + inner_mode: int = 1, + ): + """ + Initializes the StaticPersistentRuntimeTileScheduler with the given parameters. + + :param params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl. + :type params: PersistentTileSchedulerParams + :param num_persistent_clusters: Number of persistent clusters that can be launched. + :type num_persistent_clusters: Int32 + :param current_work_linear_idx: Current cluster index. + :type current_work_linear_idx: Int32 + :param cta_id_in_cluster: ID of the CTA within its cluster. + :type cta_id_in_cluster: cute.Coord + :param num_tiles_executed: Counter for executed tiles. + :type num_tiles_executed: Int32 + :param inner_mode: The inner mode along which the linear index will be decomposed first. + :type inner_mode: int + """ + super().__init__( + params, + num_persistent_clusters, + current_work_linear_idx, + cta_id_in_cluster, + num_tiles_executed, + ) + if inner_mode not in [0, 1]: + raise ValueError( + f"inner_mode must be 0(for M mode) or 1(for N mode), but got {inner_mode}" + ) + self.inner_mode = inner_mode + + def __new_from_mlir_values__( + self, values: list[ir.Value] + ) -> "StaticPersistentRuntimeTileScheduler": + assert len(values) >= 6 + new_num_persistent_clusters = new_from_mlir_values( + self.num_persistent_clusters, [values[0]] + ) + new_current_work_linear_idx = new_from_mlir_values( + self._current_work_linear_idx, [values[1]] + ) + new_cta_id_in_cluster = new_from_mlir_values(self.cta_id_in_cluster, values[2:5]) + new_num_tiles_executed = new_from_mlir_values(self._num_tiles_executed, [values[5]]) + + # Reconstruct params with FastDivmod divisors (same as parent class) + params_values = values[6:] # Remaining values are from params + new_params = new_from_mlir_values(self.params, params_values) + + return StaticPersistentRuntimeTileScheduler( + new_params, # Use reconstructed params with FastDivmod divisors + new_num_persistent_clusters, + new_current_work_linear_idx, + new_cta_id_in_cluster, + new_num_tiles_executed, + self.inner_mode, + ) + + @staticmethod + @dsl_user_op + def create( + params: PersistentTileSchedulerParams, + block_idx: Tuple[Integer, Integer, Integer], + grid_dim: Tuple[Integer, Integer, Integer], + inner_mode: int = 1, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> "StaticPersistentRuntimeTileScheduler": + """Initialize the static persistent tile scheduler. + + :param params: Parameters for the persistent + tile scheduler. + :type params: PersistentTileSchedulerParams + :param block_idx: The 3d block index in the format (bidx, bidy, bidz). + :type block_idx: Tuple[Integer, Integer, Integer] + :param grid_dim: The 3d grid dimensions for kernel launch. + :type grid_dim: Tuple[Integer, Integer, Integer] + :param inner_mode: The inner mode along which the linear index will be decomposed first. + :type inner_mode: int + + :return: A StaticPersistentRuntimeTileScheduler object. + :rtype: StaticPersistentRuntimeTileScheduler + """ + + # Calculate the number of persistent clusters by dividing the total grid size + # by the number of CTAs per cluster + num_persistent_clusters = cute.size(grid_dim, loc=loc, ip=ip) // cute.size( + params.cluster_shape_mn, loc=loc, ip=ip + ) + + bidx, bidy, bidz = block_idx + + # Initialize workload index equals to the cluster index in the grid + current_work_linear_idx = Int32(bidz) + + # CTA id in the cluster + cta_id_in_cluster = ( + Int32(bidx % params.cluster_shape_mn[0]), + Int32(bidy % params.cluster_shape_mn[1]), + Int32(0), + ) + # Initialize number of tiles executed to zero + num_tiles_executed = Int32(0) + return StaticPersistentRuntimeTileScheduler( + params, + num_persistent_clusters, + current_work_linear_idx, + cta_id_in_cluster, + num_tiles_executed, + inner_mode, + ) + + # private method + def _get_current_work_for_linear_idx( + self, + current_work_linear_idx: Int32, + *, + loc: Optional[ir.Location] = None, + ip: Optional[ir.InsertionPoint] = None, + ) -> WorkTileInfo: + """Compute current tile coord given current_work_linear_idx and cta_id_in_cluster. + + :param current_work_linear_idx: The linear index of the current work. + :type current_work_linear_idx: Int32 + + :return: An object containing information about the current tile coordinates + and validity status. + :rtype: WorkTileInfo + """ + ntile_shape = self.params.problem_layout_ncluster_mnl.shape + int_max = 2147483647 + if const_expr(self.inner_mode == 1): + ntile_layout = cute.make_layout((int_max, ntile_shape[1]), stride=(ntile_shape[1], 1)) + else: + ntile_layout = cute.make_layout((ntile_shape[0], int_max), stride=(1, ntile_shape[0])) + cluster_tile_coord_mn = ntile_layout.get_hier_coord(current_work_linear_idx) + cur_tile_coord = ( + cluster_tile_coord_mn[0], + cluster_tile_coord_mn[1], + Int32(0), + ) + + # it is determined by kernel implementation + is_valid = Boolean(True) + + return WorkTileInfo(cur_tile_coord, is_valid) diff --git a/tensorrt_llm/_torch/visual_gen/executor.py b/tensorrt_llm/_torch/visual_gen/executor.py index 95a170120795..5431653e0804 100644 --- a/tensorrt_llm/_torch/visual_gen/executor.py +++ b/tensorrt_llm/_torch/visual_gen/executor.py @@ -6,9 +6,9 @@ import time import traceback from collections import deque -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union import torch import torch.distributed as dist @@ -241,6 +241,7 @@ class DiffusionRequest: request_id: int prompt: List[str] params: Optional["VisualGenParams"] = None + prepared_inputs: Dict[str, Any] = field(default_factory=dict, repr=False) @dataclass @@ -253,11 +254,11 @@ class DiffusionResponse: model-specific fields populated. Set to ``None`` on the error path; on the READY signal it carries a ``dict`` instead. error_msg: Error message if generation failed. - generation: Wall-clock time the executor measured around the - engine's inference call (host ``time.perf_counter()``), in - seconds. Default ``0.0`` so the dataclass round-trips through - pickling across worker/client; the error path leaves it at - ``0.0``. + generation: Wall-clock time the executor measured around request + preparation and the engine's inference call (host + ``time.perf_counter()``), in seconds. Default ``0.0`` so the + dataclass round-trips through pickling across worker/client; the + error path leaves it at ``0.0``. """ request_id: int @@ -404,6 +405,12 @@ def _merge_defaults(self, req: DiffusionRequest): # Universal field defaults for field_name, default_value in self.pipeline.default_generation_params.items(): if hasattr(params, field_name) and getattr(params, field_name) is None: + if ( + params.image is not None + and getattr(self.pipeline, "derive_output_size_from_reference", False) is True + and field_name in ("height", "width") + ): + continue setattr(params, field_name, default_value) # Extra param defaults — fill all declared keys so infer() can use direct access @@ -419,21 +426,24 @@ def process_request(self, req: DiffusionRequest): """Process a single request.""" try: self._merge_defaults(req) - cache_key = self.pipeline.warmup_cache_key( - req.params.height, req.params.width, num_frames=req.params.num_frames - ) - if self.pipeline._warmed_up_shapes and cache_key not in self.pipeline._warmed_up_shapes: + # Include request preparation in executor-side generation latency. + # Model-specific preparation runs before the warmup lookup so it + # can resolve shape-dependent request fields such as output size. + generation_start = time.perf_counter() + self.pipeline.prepare_request(req) + cache_key = self.pipeline.request_warmup_cache_key(req) + cache_key_is_resolved = all(value is not None for value in cache_key) + if ( + cache_key_is_resolved + and self.pipeline._warmed_up_shapes + and cache_key not in self.pipeline._warmed_up_shapes + ): logger.warning( f"Requested shape {cache_key} was not warmed up. " f"First request with this shape will be slower due to " f"torch.compile recompilation or CUDA graph capture. " f"Warmed-up shapes: {self.pipeline._warmed_up_shapes}" ) - # Host wall-clock around pipeline.infer(). The pipeline already - # syncs at the end (decode_latents path), so this captures the - # full executor-side envelope including any pre/post-pipeline work - # that the per-phase CUDA-event timings on PipelineOutput do not. - generation_start = time.perf_counter() output = self.pipeline.infer(req) generation = time.perf_counter() - generation_start # seconds if self.rank == 0: diff --git a/tensorrt_llm/_torch/visual_gen/mapping.py b/tensorrt_llm/_torch/visual_gen/mapping.py index 55f6a2d8b0ac..d3ecb77f930b 100644 --- a/tensorrt_llm/_torch/visual_gen/mapping.py +++ b/tensorrt_llm/_torch/visual_gen/mapping.py @@ -10,6 +10,7 @@ from __future__ import annotations +import itertools import os from typing import Optional @@ -18,6 +19,7 @@ from torch.distributed.device_mesh import DeviceMesh, init_device_mesh from tensorrt_llm._torch.device_mesh import DeviceMeshTopologyImpl, SingleProcessGroup +from tensorrt_llm._torch.distributed.communicator import Distributed, TorchDist from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping @@ -28,6 +30,29 @@ _DEVICE_MESH_DIM_ORDER_ATTN2D = "cfg-tp-cp_row-cp_col-ulysses" +class _VisualGenAutotuneDist(TorchDist): + """Autotuner communicator whose collective spans the whole world. + + VisualGen's device mesh has no TP axis over the tuned GEMMs (its parallelism + is on cfg/ulysses), so the mesh's tp group would gather a single rank. Gather + over the default world group instead, since every rank runs the transformer. + """ + + def __init__(self, mapping: Mapping): + # Skip TorchDist.__init__ on purpose: it registers a global comm + # singleton (set_torch_comm) and builds mesh/local subgroups via + # collectives — hijacking it here would clobber the real comm. The merge + # only needs a world-group all_gather (tp_cp_allgather below), which runs + # on the default group; Distributed.__init__ just records the mapping. + Distributed.__init__(self, mapping) + assert dist.is_initialized() + + def tp_cp_allgather(self, obj: object) -> list[object]: + gathered: list[object] = [None] * dist.get_world_size() + dist.all_gather_object(gathered, obj) + return gathered + + class VisualGenMapping(DeviceMeshTopologyImpl): """Multi-dimensional communicator mesh for visual generation models. @@ -141,11 +166,6 @@ def __init__( if self._use_attn2d_plane: cp_size = attn2d_size - if tp_size > 1: - raise NotImplementedError( - "Combining Attention2D and TP is not yet supported. " - "The row/col group construction does not account for TP ranks." - ) else: cp_size = ring_size @@ -393,6 +413,33 @@ def _build_vae_group(self) -> None: if self._rank in self._vae_ranks: self._vae_adj_groups = adj_groups + def flatten_cfg_ranks(self) -> list: + """Rank lists for the ulysses groups of a topology whose cfg dim is + flattened into ulysses. + + One list per combined coordinate of every OTHER mesh dim (tp, + cp/cp_row/cp_col); within a list the cfg coordinate varies outermost and + ulysses innermost. Every non-(cfg, ulysses) group of the current mesh is + therefore preserved verbatim by a topology built from these lists. + Pure layout arithmetic — no process group is created, no state is held. + """ + strides, acc = {}, 1 + for d in reversed(self._dim_names): + strides[d] = acc + acc *= self._dim_sizes[d] + other_dims = [d for d in self._dim_names if d not in ("cfg", "ulysses")] + groups = [] + for coords in itertools.product(*(range(self._dim_sizes[d]) for d in other_dims)): + base = sum(strides[d] * v for d, v in zip(other_dims, coords)) + groups.append( + [ + base + strides["cfg"] * c + strides["ulysses"] * u + for c in range(self._dim_sizes["cfg"]) + for u in range(self._dim_sizes["ulysses"]) + ] + ) + return groups + # ------------------------------------------------------------------ # Rank decomposition # ------------------------------------------------------------------ @@ -555,3 +602,12 @@ def to_llm_mapping(self) -> Mapping: rank=self.tp_rank, tp_size=self.tp_size, ) + + def to_autotuner_mapping(self) -> Mapping: + """Mapping that makes the autotuner treat all world ranks as one tuning + group (tp_size == world_size), so its post-tune cross-rank merge engages.""" + return Mapping( + world_size=self.world_size, + rank=self._rank, + tp_size=self.world_size, + ) diff --git a/tensorrt_llm/_torch/visual_gen/models/__init__.py b/tensorrt_llm/_torch/visual_gen/models/__init__.py index c5d63ed88b23..8a472d7111d1 100644 --- a/tensorrt_llm/_torch/visual_gen/models/__init__.py +++ b/tensorrt_llm/_torch/visual_gen/models/__init__.py @@ -36,7 +36,8 @@ from .cosmos3 import Cosmos3OmniMoTPipeline from .flux import Flux2Pipeline, FluxPipeline from .ltx2 import LTX2Pipeline # noqa: F401 -from .qwen_image import QwenImagePipeline +from .qwen_image import QwenImageEditPlusPipeline, QwenImagePipeline +from .qwen_image_layered import QwenImageLayeredPipeline from .wan import WanImageToVideoPipeline, WanPipeline __all__ = [ @@ -44,6 +45,8 @@ "BasePipeline", "FluxPipeline", "Flux2Pipeline", + "QwenImageEditPlusPipeline", + "QwenImageLayeredPipeline", "QwenImagePipeline", "WanPipeline", "WanImageToVideoPipeline", diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/defaults.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/defaults.py index f5747544946d..114ca97cad22 100644 --- a/tensorrt_llm/_torch/visual_gen/models/cosmos3/defaults.py +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/defaults.py @@ -35,10 +35,7 @@ "frame_rate": 24.0, } -# Text-to-image (``output_type="image"``) defaults. Applied by the pipeline when -# the corresponding request field still carries the merged video default, since -# the executor merges a single ``default_generation_params`` dict (the video -# params above) into the request before ``infer()`` runs. +# Text-to-image (``output_type="image"``) defaults; resolved in ``infer()``. COSMOS3_T2I_PARAMS = { "height": 1024, "width": 1024, @@ -48,6 +45,18 @@ "guidance_interval": (400.0, 1000.0), } +# Fields merged by the executor into every request. Mode-dependent values +# remain None until infer() selects the request mode; key membership also +# declares these fields supported during request validation. +COSMOS3_PIPELINE_DEFAULTS = { + **COSMOS3_720P_PARAMS, + "height": None, + "width": None, + "num_inference_steps": None, + "guidance_scale": None, +} + + COSMOS3_EXTRA_SPECS: Dict[str, ExtraParamSchema] = { "use_duration_template": ExtraParamSchema( type="bool", diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py index 2dac241f2115..f497b09f7c90 100644 --- a/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py @@ -21,7 +21,7 @@ import PIL.Image import torch -from diffusers import AutoencoderKLWan, UniPCMultistepScheduler +from diffusers import AutoencoderKLWan from diffusers.utils.torch_utils import randn_tensor from diffusers.video_processor import VideoProcessor from transformers import Qwen2Tokenizer @@ -34,8 +34,14 @@ from tensorrt_llm.inputs.utils import load_image from tensorrt_llm.logger import logger -from .defaults import COSMOS3_720P_PARAMS, COSMOS3_EXTRA_SPECS, COSMOS3_T2I_PARAMS +from .defaults import ( + COSMOS3_720P_PARAMS, + COSMOS3_EXTRA_SPECS, + COSMOS3_PIPELINE_DEFAULTS, + COSMOS3_T2I_PARAMS, +) from .guardrails import check_video_safety, download_guardrail_checkpoint +from .sampling import Cosmos3SamplingPolicy, load_scheduler from .sound_tokenizer import LatentAutoEncoderV2 from .transformer_cosmos3 import Cosmos3VFMTransformer @@ -59,7 +65,9 @@ "nvidia/Cosmos3-Nano", "nvidia/Cosmos3-Super", "nvidia/Cosmos3-Super-Image2Video", + "nvidia/Cosmos3-Super-Image2Video-4Step", "nvidia/Cosmos3-Super-Text2Image", + "nvidia/Cosmos3-Super-Text2Image-4Step", ], doc="Cosmos3 Omnimodal world models.", ) @@ -68,6 +76,10 @@ def __init__(self, pipeline_config): primary_pretrained_config = pipeline_config.primary_pretrained_config self.audio_gen = False self.action_gen = False + # Pre-load placeholder; load_standard_components derives the real + # policy from the checkpoint's scheduler via from_scheduler(). + self.sampling = Cosmos3SamplingPolicy() + self.default_use_system_prompt = COSMOS3_EXTRA_SPECS["use_system_prompt"].default if getattr( primary_pretrained_config, "audio_gen", @@ -98,6 +110,18 @@ def load_standard_components( ) -> None: skip_components = skip_components or [] + # Prompting defaults are checkpoint-declared: distilled conversions + # carry ``default_use_system_prompt`` in model_index.json (diffusers' + # distilled blocks default it to True); older checkpoints omit it and + # keep the historical False. + model_index_path = os.path.join(checkpoint_dir, "model_index.json") + if os.path.exists(model_index_path): + with open(model_index_path) as f: + model_index = json.load(f) + self.default_use_system_prompt = bool( + model_index.get("default_use_system_prompt", self.default_use_system_prompt) + ) + if self.audio_gen and PipelineComponent.SOUND_TOKENIZER not in skip_components: logger.info("Loading audio tokenizer...") self.audio_tokenizer = ( @@ -139,22 +163,15 @@ def load_standard_components( if PipelineComponent.SCHEDULER not in skip_components: logger.info("Loading scheduler...") - self.scheduler = UniPCMultistepScheduler.from_pretrained( - checkpoint_dir, - subfolder=PipelineComponent.SCHEDULER, - ) - # Snapshot the checkpoint scheduler config so the scheduler can be - # rebuilt at request time when a mode-specific ``flow_shift`` is - # needed (T2I uses shift=3.0; T2V/I2V keep the checkpoint default). - self._base_scheduler_config = self.scheduler.config - self._engine_init_flow_shift = float( - getattr(self.scheduler.config, "flow_shift", 1.0) or 1.0 - ) - self._current_flow_shift = self._engine_init_flow_shift + # The scheduler class comes from the checkpoint: UniPC for base + # checkpoints, FlowMatchEuler (fixed stochastic schedule) for + # distilled ones. The policy holds the derived immutable facts. + self.scheduler = load_scheduler(checkpoint_dir) + self.sampling = Cosmos3SamplingPolicy.from_scheduler(self.scheduler) if self.audio_gen: - # Separate instance so video and audio scheduler states don't collide - # (UniPC mutates internal correction buffers on every .step() call). - self.audio_scheduler = UniPCMultistepScheduler.from_config(self.scheduler.config) + # Separate instance so video and audio scheduler states don't + # collide (schedulers mutate internal state on every .step()). + self.audio_scheduler = type(self.scheduler).from_config(self.scheduler.config) # Re-check the env var in case it was changed after initialization like in unit tests. guardrails_disabled = os.environ.get("TRTLLM_DISABLE_COSMOS3_GUARDRAILS", "0") == "1" @@ -185,23 +202,6 @@ def load_standard_components( self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) - def _set_flow_shift(self, target_shift: float) -> None: - """Rebuild the UniPC scheduler with ``flow_shift=target_shift`` if needed. - - T2I uses ``flow_shift=3.0`` while T2V/I2V use the checkpoint default. - ``self._current_flow_shift`` is tracked explicitly so a prior T2I rebuild - does not leak into a subsequent video request. - """ - if not hasattr(self, "_base_scheduler_config"): - return - target = float(target_shift) - if target == float(self._current_flow_shift): - return - self.scheduler = UniPCMultistepScheduler.from_config( - self._base_scheduler_config, flow_shift=target - ) - self._current_flow_shift = target - @property def default_warmup_resolutions(self): return [(720, 1280)] @@ -210,15 +210,30 @@ def default_warmup_resolutions(self): def default_warmup_num_frames(self): return [189] + @property + def default_warmup_steps(self): + # Distilled checkpoints only run their fixed schedule length. + return self.sampling.num_steps(super().default_warmup_steps) + @property def default_generation_params(self): - return dict(COSMOS3_720P_PARAMS) + return {**COSMOS3_PIPELINE_DEFAULTS, **self.sampling.generation_default_overrides()} @property def extra_param_specs(self): - return dict(COSMOS3_EXTRA_SPECS) + specs = dict(COSMOS3_EXTRA_SPECS) + specs["use_system_prompt"] = specs["use_system_prompt"].model_copy( + update={"default": self.default_use_system_prompt} + ) + return specs def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> None: + # Checkpoint-aware guidance: distilled defaults carry a concrete 1.0; + # base defaults leave it None ("by mode") — warmup runs the video mode. + defaults = self.default_generation_params + guidance_scale = defaults["guidance_scale"] + if guidance_scale is None: + guidance_scale = COSMOS3_720P_PARAMS["guidance_scale"] with torch.no_grad(): self.forward( prompt="warmup", @@ -227,51 +242,29 @@ def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> N width=width, num_frames=num_frames, num_inference_steps=steps, - guidance_scale=COSMOS3_720P_PARAMS["guidance_scale"], + guidance_scale=guidance_scale, seed=42, - max_sequence_length=COSMOS3_720P_PARAMS["max_sequence_length"], + max_sequence_length=defaults["max_sequence_length"], use_guardrails=False, image=None, enable_audio=False, ) - @staticmethod - def _resolve_t2i_default(merged_value, video_default, t2i_default): - """Pick the T2I default when the field still carries the merged video default. - - The executor merges a single ``default_generation_params`` dict (the - video params) into the request before ``infer()``, so an unspecified - field arrives equal to its video default. For T2I we substitute the - T2I default in that case while honoring any explicit user override. - """ - return t2i_default if merged_value == video_default else merged_value - def infer(self, req): extra_params = req.params.extra_params or {} output_type = extra_params.get("output_type", "video") is_t2i = str(output_type).lower() == "image" - height = req.params.height - width = req.params.width - num_inference_steps = req.params.num_inference_steps - guidance_scale = req.params.guidance_scale - if is_t2i: - height = self._resolve_t2i_default( - height, COSMOS3_720P_PARAMS["height"], COSMOS3_T2I_PARAMS["height"] - ) - width = self._resolve_t2i_default( - width, COSMOS3_720P_PARAMS["width"], COSMOS3_T2I_PARAMS["width"] - ) - num_inference_steps = self._resolve_t2i_default( - num_inference_steps, - COSMOS3_720P_PARAMS["num_inference_steps"], - COSMOS3_T2I_PARAMS["num_inference_steps"], - ) - guidance_scale = self._resolve_t2i_default( - guidance_scale, - COSMOS3_720P_PARAMS["guidance_scale"], - COSMOS3_T2I_PARAMS["guidance_scale"], - ) + # None = unset; resolve by mode exactly once. Non-None values pass through. + mode_params = COSMOS3_T2I_PARAMS if is_t2i else COSMOS3_720P_PARAMS + + def resolved(value, field_name): + return value if value is not None else mode_params[field_name] + + height = resolved(req.params.height, "height") + width = resolved(req.params.width, "width") + num_inference_steps = resolved(req.params.num_inference_steps, "num_inference_steps") + guidance_scale = resolved(req.params.guidance_scale, "guidance_scale") return self.forward( prompt=req.prompt, @@ -293,7 +286,8 @@ def infer(self, req): "use_resolution_template", COSMOS3_EXTRA_SPECS["use_resolution_template"].default, ), - use_system_prompt=extra_params.get("use_system_prompt", False), + # None = unset; forward() resolves it to the checkpoint default. + use_system_prompt=extra_params.get("use_system_prompt"), use_guardrails=extra_params.get("use_guardrails", True), enable_audio=extra_params.get("enable_audio", False), output_type=output_type, @@ -564,6 +558,30 @@ def _prepare_latents_i2v( velocity_mask = 1.0 - condition_mask return latents, velocity_mask, image_latent + def _conditioning_anchor_post_step(self, image_latent: Optional[torch.Tensor]): + """Per-step re-anchor of the conditioned frame for distilled sampling. + + The distilled FlowMatchEuler step is stochastic: it re-noises every + position, including the frame the velocity mask holds still, so the + conditioning frame the model reads as clean context degrades from step + 2 on. Writing the clean latent back after every scheduler step keeps + it clean (diffusers' distilled loop re-anchors the same way). + Deterministic UniPC steps never move a zero-velocity frame, so base + checkpoints need no per-step anchor and keep their exact behavior. + + Returns a ``post_step_fn`` for ``BasePipeline.denoise``, or ``None`` + when no anchoring is needed. + """ + if not self.sampling.is_distilled or image_latent is None: + return None + + def post_step_fn(latents: torch.Tensor) -> torch.Tensor: + # In-place: writes one latent frame, no full-tensor copies. + latents[:, :, 0:1] = image_latent + return latents + + return post_step_fn + # ========================================================================= # VAE decode # ========================================================================= @@ -629,21 +647,43 @@ def forward( frame_rate: float = COSMOS3_720P_PARAMS["frame_rate"], use_duration_template: bool = COSMOS3_EXTRA_SPECS["use_duration_template"].default, use_resolution_template: bool = COSMOS3_EXTRA_SPECS["use_resolution_template"].default, - use_system_prompt: bool = COSMOS3_EXTRA_SPECS["use_system_prompt"].default, + use_system_prompt: Optional[bool] = None, use_guardrails: bool = COSMOS3_EXTRA_SPECS["use_guardrails"].default, enable_audio: bool = COSMOS3_EXTRA_SPECS["enable_audio"].default, output_type: str = COSMOS3_EXTRA_SPECS["output_type"].default, ): + """Run one generation. ``infer()`` is the resolved entry point. + + Production requests arrive through ``infer()`` with fully resolved + values; the signature defaults are the base-checkpoint *video* table + values for direct internal callers. ``forward()`` cannot tell a + signature default from an explicit argument, so on distilled + checkpoints (which fix steps/guidance and reject anything else) direct + callers must pass checkpoint-valid sampling values. + + ``use_system_prompt=None`` means "unset": it resolves to the + checkpoint-declared default, so warmup and other direct callers build + the same prompt as served requests. + """ pipeline_start = time.time() timer = CudaPhaseTimer() timer.mark_pre_start() + if use_system_prompt is None: + use_system_prompt = self.default_use_system_prompt + use_guardrails = use_guardrails and not TRTLLM_DISABLE_COSMOS3_GUARDRAILS # Text-to-image mode: same checkpoint/forward path as T2V, but a single # latent frame, image-flavored prompt templates, flow_shift=3.0, a CFG # guidance interval, and an image (rather than video) output. - is_t2i = str(output_type).lower() == "image" + output_type = str(output_type).lower() + if output_type not in ("video", "image"): + raise ValueError(f"output_type must be 'video' or 'image', got {output_type!r}.") + is_t2i = output_type == "image" + + self.sampling.validate_request(num_inference_steps, guidance_scale) + guidance_interval = None if is_t2i: if image is not None: @@ -653,11 +693,25 @@ def forward( num_frames = 1 enable_audio = False guidance_interval = COSMOS3_T2I_PARAMS["guidance_interval"] - self._set_flow_shift(COSMOS3_T2I_PARAMS["flow_shift"]) + self.scheduler = self.sampling.set_flow_shift( + self.scheduler, COSMOS3_T2I_PARAMS["flow_shift"] + ) else: # Restore the checkpoint flow_shift in case a prior T2I request # rebuilt the scheduler with shift=3.0. - self._set_flow_shift(getattr(self, "_engine_init_flow_shift", 1.0)) + self.scheduler = self.sampling.set_flow_shift( + self.scheduler, self.sampling.checkpoint_flow_shift + ) + + # Weight-presence guard, not workflow policy: the request explicitly + # asks for audio, but the checkpoint ships no audio tower. Silently + # returning a silent video would hide the capability limit. + if enable_audio and not self.audio_gen: + raise ValueError( + "enable_audio=True, but this checkpoint has no audio tower " + "(transformer config declares sound_gen=false). Drop enable_audio " + "or use an audio-capable Cosmos3 checkpoint." + ) if isinstance(prompt, str): prompt = [prompt] @@ -781,7 +835,7 @@ def forward( video_shape = (T_latent, H_latent, W_latent) # 3. Set up scheduler - self.scheduler.set_timesteps(num_inference_steps, device=self.device) + self.sampling.set_timesteps(self.scheduler, num_inference_steps, device=self.device) # 3b. Audio noise init — latent length matches diffusers Cosmos3OmniPipeline.prepare_latents. do_audio = enable_audio and self.audio_gen and hasattr(self, "audio_tokenizer") @@ -798,7 +852,9 @@ def forward( dtype=latents.dtype, ) # Audio uses the same scheduler type/config as video. - self.audio_scheduler.set_timesteps(num_inference_steps, device=self.device) + self.sampling.set_timesteps( + self.audio_scheduler, num_inference_steps, device=self.device + ) # 4. Build forward_fn for the denoise loop def forward_fn( @@ -861,6 +917,8 @@ def forward_fn( extra_cfg_tensors=extra_cfg_tensors, extra_streams=extra_streams, guidance_interval=guidance_interval, + scheduler_step_kwargs=self.sampling.scheduler_step_kwargs(generator), + post_step_fn=self._conditioning_anchor_post_step(image_latent), ) if extra_streams is not None: @@ -877,8 +935,8 @@ def forward_fn( decode_start = time.time() if image_latent is not None: - latents = latents.clone() - latents[:, :, 0:1, :, :] = image_latent.to(device=latents.device, dtype=latents.dtype) + # In-place: the loop output is consumed only by the decode below. + latents[:, :, 0:1] = image_latent video = self.decode_latents(latents, self._decode_latents) diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/sampling.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/sampling.py new file mode 100644 index 000000000000..932ac974658a --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/sampling.py @@ -0,0 +1,248 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Per-checkpoint sampling policy for Cosmos3. + +Exactly two recipes are supported, read from the checkpoint's +``scheduler/scheduler_config.json``: + +* ``UniPCMultistepScheduler`` without fixed sigmas — base checkpoints: + request tables drive steps/guidance; T2I rebuilds with ``flow_shift=3.0``. +* ``FlowMatchEulerDiscreteScheduler`` with ``stochastic_sampling`` enabled + and a nonempty ``fixed_step_sampler_config.t_list`` — distilled + checkpoints: the step count is locked to the schedule, classifier-free + guidance is baked into the weights (scale 1.0), and every step draws + seeded SDE noise. + +The pipeline owns the scheduler instances; :class:`Cosmos3SamplingPolicy` is +an immutable value object of config-derived facts whose methods take +schedulers as arguments. +""" + +import json +import os +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any, Optional + +from diffusers import FlowMatchEulerDiscreteScheduler, UniPCMultistepScheduler + +from tensorrt_llm.logger import logger + +# Distilled checkpoints bake classifier-free guidance into the weights; the +# only valid scale is 1.0 ("off": a single conditional forward per step). +DISTILLED_GUIDANCE_SCALE = 1.0 + + +def _config_get(config: Any, key: str, default: Any = None) -> Any: + """Fetch a key from a plain dict, a diffusers FrozenDict, or a config object.""" + if isinstance(config, Mapping): + return config.get(key, default) + return getattr(config, key, default) + + +def _resolve_distilled_sigmas(scheduler_config: Any) -> "tuple[float, ...] | None": + """``fixed_step_sampler_config.t_list`` as floats, or None (base checkpoints).""" + fixed_step_cfg = _config_get(scheduler_config, "fixed_step_sampler_config") + t_list = _config_get(fixed_step_cfg, "t_list") if fixed_step_cfg else None + if not t_list: + return None + return tuple(float(sigma) for sigma in t_list) + + +def load_scheduler(checkpoint_dir: str, subfolder: str = "scheduler") -> Any: + """Instantiate the scheduler class the checkpoint declares. + + Base checkpoints declare ``UniPCMultistepScheduler`` (a missing declaration + also resolves to UniPC, preserving pre-declaration Cosmos3 behavior); + distilled ones declare ``FlowMatchEulerDiscreteScheduler``. An explicitly + unknown declaration is a load-time error — silently substituting UniPC + would sample the checkpoint with the wrong integrator. + """ + config_path = os.path.join(checkpoint_dir, subfolder, "scheduler_config.json") + class_name = "" + if os.path.exists(config_path): + with open(config_path) as f: + class_name = json.load(f).get("_class_name", "") + if class_name == "FlowMatchEulerDiscreteScheduler": + scheduler_cls = FlowMatchEulerDiscreteScheduler + elif class_name in ("", None, "UniPCMultistepScheduler"): + scheduler_cls = UniPCMultistepScheduler + else: + raise ValueError( + f"Unsupported Cosmos3 scheduler class {class_name!r}; supported: " + "UniPCMultistepScheduler (base), FlowMatchEulerDiscreteScheduler (distilled)." + ) + return scheduler_cls.from_pretrained(checkpoint_dir, subfolder=subfolder) + + +@dataclass(frozen=True) +class Cosmos3SamplingPolicy: + """Immutable sampling facts of a loaded Cosmos3 checkpoint. + + Construct via :meth:`from_scheduler`, which validates the recipe at load + time. A default-constructed policy (all fields ``None``) is the explicit + pre-load placeholder the pipeline holds before its scheduler exists: not + distilled, no flow-shift rebuild capability (``set_flow_shift`` is a + no-op), and replaced by ``from_scheduler`` when components load. + + Methods take scheduler instances as arguments; the current flow shift is + read from the supplied scheduler's config rather than tracked here. + """ + + # Fixed distilled schedule (t_list); None for base checkpoints. + fixed_sigmas: "tuple[float, ...] | None" = None + # Checkpoint scheduler config, kept for flow-shift rebuilds (UniPC only). + unipc_base_config: Optional[Any] = None + + @classmethod + def from_scheduler(cls, scheduler: Any) -> "Cosmos3SamplingPolicy": + """Derive the policy from a loaded scheduler's config. + + Valid recipes: UniPC without fixed sigmas (base) and stochastic + FlowMatchEuler with a nonempty ``fixed_step_sampler_config.t_list`` + (distilled); anything else fails here, at load time. + """ + fixed_sigmas = _resolve_distilled_sigmas(scheduler.config) + is_unipc = isinstance(scheduler, UniPCMultistepScheduler) + is_flow_match = isinstance(scheduler, FlowMatchEulerDiscreteScheduler) + + if ( + _config_get(scheduler.config, "fixed_step_requires_explicit_sigmas", False) + and fixed_sigmas is None + ): + raise ValueError( + "Malformed distilled checkpoint: the scheduler config declares " + "fixed_step_requires_explicit_sigmas but carries no usable " + "fixed_step_sampler_config.t_list." + ) + + if is_unipc and fixed_sigmas is None: + return cls(fixed_sigmas=None, unipc_base_config=scheduler.config) + + if is_flow_match and fixed_sigmas is not None: + if not _config_get(scheduler.config, "stochastic_sampling", False): + raise ValueError( + "Unsupported Cosmos3 sampling recipe: FlowMatchEulerDiscreteScheduler " + "declares a fixed step schedule without stochastic_sampling. The " + "distilled recipe draws SDE noise at every step; running the schedule " + "as an ODE would sample the checkpoint incorrectly." + ) + fixed_step_cfg = _config_get(scheduler.config, "fixed_step_sampler_config") + sample_type = _config_get(fixed_step_cfg, "sample_type") + if sample_type is not None and sample_type != "sde": + raise ValueError( + "Unsupported Cosmos3 sampling recipe: fixed_step_sampler_config " + f"declares sample_type={sample_type!r}; only 'sde' is supported." + ) + logger.info( + f"Distilled Cosmos3 checkpoint: fixed {len(fixed_sigmas)}-step schedule " + f"{list(fixed_sigmas)}, classifier-free guidance baked in." + ) + return cls(fixed_sigmas=fixed_sigmas, unipc_base_config=None) + + raise ValueError( + f"Unsupported Cosmos3 sampling recipe: {type(scheduler).__name__} with " + f"fixed sigmas {'present' if fixed_sigmas is not None else 'absent'}. " + "Supported: UniPCMultistepScheduler without fixed sigmas (base), " + "stochastic FlowMatchEulerDiscreteScheduler with " + "fixed_step_sampler_config.t_list (distilled)." + ) + + @property + def is_distilled(self) -> bool: + return self.fixed_sigmas is not None + + def generation_default_overrides(self) -> dict: + """Checkpoint-mandated overrides of the table generation defaults. + + Merged over ``COSMOS3_720P_PARAMS`` by the pipeline's + ``default_generation_params``, so executor-merged requests arrive + carrying the checkpoint's true defaults. + """ + if not self.is_distilled: + return {} + return { + "num_inference_steps": len(self.fixed_sigmas), + "guidance_scale": DISTILLED_GUIDANCE_SCALE, + } + + def num_steps(self, default: int) -> int: + """The only step count this policy can run: fixed for distilled, else ``default``.""" + return len(self.fixed_sigmas) if self.is_distilled else default + + def validate_request( + self, num_inference_steps: Optional[int], guidance_scale: Optional[float] + ) -> None: + """Reject sampling parameters incompatible with a distilled checkpoint.""" + if not self.is_distilled: + return + distilled_steps = len(self.fixed_sigmas) + if num_inference_steps is not None and num_inference_steps != distilled_steps: + raise ValueError( + "This is a distilled Cosmos3 checkpoint; the step count is fixed by the " + f"scheduler's fixed_step_sampler_config.t_list ({distilled_steps} steps). " + f"num_inference_steps must be {distilled_steps} or left unset " + f"(got {num_inference_steps})." + ) + if guidance_scale is not None and float(guidance_scale) != DISTILLED_GUIDANCE_SCALE: + raise ValueError( + "This is a distilled Cosmos3 checkpoint; classifier-free guidance is baked " + f"into the weights. guidance_scale must be {DISTILLED_GUIDANCE_SCALE} or " + f"left unset (got {guidance_scale})." + ) + + def set_timesteps(self, scheduler: Any, num_inference_steps: int, device: Any) -> None: + """Program a scheduler for one generation: fixed sigmas or a step count.""" + if self.is_distilled: + scheduler.set_timesteps(sigmas=list(self.fixed_sigmas), device=device) + else: + scheduler.set_timesteps(num_inference_steps, device=device) + + def scheduler_step_kwargs(self, generator: Any) -> dict: + """Extra kwargs each ``scheduler.step()`` call requires. + + The distilled FlowMatchEuler scheduler is stochastic: every step draws + SDE noise, which must come from the request-seeded ``generator`` — + otherwise it comes from the process-global RNG, breaking seed + reproducibility and diverging the replicated latents across ranks + (each rank's global RNG state is independent). UniPC steps are + deterministic and accept no ``generator`` argument, so base + checkpoints pass nothing. + """ + if self.is_distilled: + return {"generator": generator} + return {} + + @property + def checkpoint_flow_shift(self) -> float: + """The flow shift the checkpoint shipped with (UniPC only; 1.0 otherwise).""" + if self.unipc_base_config is None: + return 1.0 + return float(_config_get(self.unipc_base_config, "flow_shift", 1.0) or 1.0) + + def set_flow_shift(self, scheduler: Any, target_shift: Optional[float]) -> Any: + """Return ``scheduler`` rebuilt with ``flow_shift=target_shift`` if needed. + + The current shift is read from the supplied scheduler's own config, so + no tracking state exists to diverge. Structural no-op for distilled + checkpoints (no UniPC base config) and for ``target_shift=None``. + """ + if target_shift is None or self.unipc_base_config is None: + return scheduler + target_shift = float(target_shift) + current_shift = float(_config_get(scheduler.config, "flow_shift", 1.0) or 1.0) + if current_shift == target_shift: + return scheduler + return UniPCMultistepScheduler.from_config(self.unipc_base_config, flow_shift=target_shift) diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py index b3df78a06ec1..1a85cf2b5784 100644 --- a/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py @@ -15,7 +15,7 @@ import math from dataclasses import dataclass -from typing import Optional, Tuple +from typing import Optional, Tuple, TypeVar import torch import torch.nn as nn @@ -34,6 +34,27 @@ from tensorrt_llm.logger import logger from tensorrt_llm.models.modeling_utils import QuantConfig +# Some Cosmos3OmniTransformer checkpoint configs omit these fields; the values +# match what other conversions carry explicitly. +PRETRAINED_CONFIG_COMPAT_DEFAULTS = { + "position_embedding_type": "unified_3d_mrope", + "max_position_embeddings": 262144, + "temporal_compression_factor_sound": 1, +} + + +_PretrainedConfigT = TypeVar("_PretrainedConfigT") + + +def apply_pretrained_config_compat_defaults( + pretrained_config: _PretrainedConfigT, +) -> _PretrainedConfigT: + """Fill missing schema fields in place (idempotent); returns the config.""" + for key, value in PRETRAINED_CONFIG_COMPAT_DEFAULTS.items(): + if getattr(pretrained_config, key, None) is None: + setattr(pretrained_config, key, value) + return pretrained_config + class Qwen3VLTextRMSNorm(nn.Module): def __init__( @@ -702,7 +723,7 @@ def forward( class Cosmos3VFMTransformer(BaseDiffusionModel): def __init__(self, model_config: DiffusionModelConfig): super().__init__(model_config) - pretrained_config = model_config.pretrained_config + pretrained_config = apply_pretrained_config_compat_defaults(model_config.pretrained_config) self.audio_gen = getattr(pretrained_config, "sound_gen", False) self.action_gen = getattr(pretrained_config, "action_gen", False) diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/attention.py b/tensorrt_llm/_torch/visual_gen/models/flux/attention.py index c308c1979b23..31df009e0be8 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/attention.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/attention.py @@ -21,12 +21,15 @@ ) from tensorrt_llm._torch.modules.rms_norm import RMSNorm from tensorrt_llm._torch.modules.swiglu import swiglu +from tensorrt_llm._torch.utils import Fp4QuantizedTensor from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig from tensorrt_llm._torch.visual_gen.models.flux.joint_proj import ( FluxJointAttnMLPProj, FluxJointQKVMLPProj, ) from tensorrt_llm._torch.visual_gen.modules.attention import Attention, QKVMode, apply_rotary_emb +from tensorrt_llm._utils import is_sm_100f +from tensorrt_llm.quantization.mode import QuantAlgo # ============================================================================= # Joint Attention (shared by FLUX.1 and FLUX.2 dual-stream blocks) @@ -109,6 +112,11 @@ def __init__( mapping=config.mapping, tensor_parallel_mode=TensorParallelMode.COLUMN, reduce_output=False, + override_tp_sharding={ + "q": (self.local_q_dim_start, self.local_q_dim_end), + "k": (self.local_kv_dim_start, self.local_kv_dim_end), + "v": (self.local_kv_dim_start, self.local_kv_dim_end), + }, ) # Need not pass any mapping info since this is intra-head normalization @@ -138,6 +146,7 @@ def __init__( allreduce_strategy=config.allreduce_strategy, tensor_parallel_mode=TensorParallelMode.ROW, reduce_output=True, + override_tp_sharding=(self.local_kv_dim_start, self.local_kv_dim_end), ) def apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: @@ -314,6 +323,9 @@ class Flux2ParallelSelfAttention(FluxJointAttention): - FLUX.2: Fused QKV+MLP projection for efficiency """ + _SWIGLU_FP4_TILE_SIZE = 128 + _SWIGLU_WEIGHT_INTERLEAVE_SIZE = 128 + def __init__( self, hidden_size: int, @@ -358,11 +370,30 @@ def __init__( skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, config=config, + attn_shard=(self.local_q_dim_start, self.local_q_dim_end), ) def _init_qkv_proj(self): """Override: fused QKV+MLP projection instead of standard QKV.""" mlp_in_dim = self.mlp_hidden_dim * self.mlp_mult_factor + local_mlp_hidden_start = Linear._calc_shard( + self.mlp_hidden_dim, self.mapping.tp_size, self.mapping.tp_rank + ) + local_mlp_hidden_end = Linear._calc_shard( + self.mlp_hidden_dim, self.mapping.tp_size, self.mapping.tp_rank + 1 + ) + self.local_mlp_hidden_dim = local_mlp_hidden_end - local_mlp_hidden_start + use_cute_dsl_swiglu = ( + torch.cuda.is_available() + and is_sm_100f() + and getattr(self.quant_config, "quant_algo", None) == QuantAlgo.NVFP4 + and not self.bias + and self._is_cute_dsl_swiglu_layout_compatible( + self.mapping.tp_size, + self.local_mlp_hidden_dim * self.mlp_mult_factor, + self.local_mlp_hidden_dim, + ) + ) self.to_qkv_mlp_proj = FluxJointQKVMLPProj( in_dim=self.hidden_size, q_dim=self.q_dim, @@ -373,7 +404,27 @@ def _init_qkv_proj(self): quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + use_cute_dsl_blockscaling_mm=use_cute_dsl_swiglu, mapping=self.mapping, + override_qkv_sharding={ + "q": (self.local_q_dim_start, self.local_q_dim_end), + "k": (self.local_kv_dim_start, self.local_kv_dim_end), + "v": (self.local_kv_dim_start, self.local_kv_dim_end), + }, + ) + + @staticmethod + def _is_cute_dsl_swiglu_layout_compatible( + tp_size: int, + gate_up_out_features: int, + down_in_features: int, + ) -> bool: + return ( + tp_size > 1 + and gate_up_out_features % Flux2ParallelSelfAttention._SWIGLU_WEIGHT_INTERLEAVE_SIZE + == 0 + and down_in_features % 2 == 0 + and gate_up_out_features // 4 == down_in_features // 2 ) def _apply_norm_rope_unfused( @@ -425,6 +476,157 @@ def _apply_norm_rope( return self._apply_norm_rope_fused(qkv, image_rotary_emb) return self._apply_norm_rope_unfused(qkv, image_rotary_emb) + def _can_project_mlp_out_from_fp4(self) -> bool: + if self.to_qkv_mlp_proj.tp_size <= 1 or not hasattr(self.to_out, "mlp_proj"): + return False + if not torch.cuda.is_available() or not is_sm_100f(): + return False + + mlp_proj = self.to_out.mlp_proj + if not getattr(mlp_proj, "_weights_created", False): + return False + + return ( + mlp_proj.has_nvfp4 + and mlp_proj.input_scale is not None + and mlp_proj.pre_quant_scale is None + and not mlp_proj.force_dynamic_quantization + ) + + def _can_project_hidden_mlp_with_cute_dsl(self) -> bool: + if ( + self.to_qkv_mlp_proj.tp_size <= 1 + or not hasattr(self.to_qkv_mlp_proj, "qkv_proj") + or not hasattr(self.to_qkv_mlp_proj, "mlp_proj") + or not hasattr(self.to_out, "mlp_proj") + ): + return False + if not torch.cuda.is_available() or not is_sm_100f(): + return False + + gate_up_proj = self.to_qkv_mlp_proj.mlp_proj + down_proj = self.to_out.mlp_proj + if not getattr(gate_up_proj, "_weights_created", False) or not getattr( + down_proj, "_weights_created", False + ): + return False + + return ( + gate_up_proj.use_cute_dsl_blockscaling_mm + and gate_up_proj.has_nvfp4 + and not gate_up_proj.has_bias + and self._is_cute_dsl_swiglu_layout_compatible( + self.to_qkv_mlp_proj.tp_size, + gate_up_proj.out_features, + down_proj.in_features, + ) + ) + + def _can_project_hidden_mlp_with_fp4out(self, hidden_states: torch.Tensor) -> bool: + if not self._can_project_hidden_mlp_with_cute_dsl(): + return False + + gate_up_proj = self.to_qkv_mlp_proj.mlp_proj + down_proj = self.to_out.mlp_proj + + if not torch.compiler.is_compiling(): + num_tokens = hidden_states.reshape(-1, hidden_states.shape[-1]).shape[0] + if num_tokens < Flux2ParallelSelfAttention._SWIGLU_FP4_TILE_SIZE: + return False + + return ( + gate_up_proj.has_nvfp4 + and not gate_up_proj.has_bias + and gate_up_proj.input_scale is not None + and gate_up_proj.pre_quant_scale is None + and not gate_up_proj.force_dynamic_quantization + and down_proj.has_nvfp4 + and down_proj.input_scale is not None + and down_proj.pre_quant_scale is None + and not down_proj.force_dynamic_quantization + ) + + def _project_hidden_mlp_with_cute_dsl(self, hidden_states: torch.Tensor) -> torch.Tensor: + gate_up_proj = self.to_qkv_mlp_proj.mlp_proj + down_proj = self.to_out.mlp_proj + + original_shape = hidden_states.shape + hidden_states = hidden_states.reshape(-1, hidden_states.shape[-1]) + act_fp4, act_sf, alpha = gate_up_proj.quant_method._input_prepare( + gate_up_proj, hidden_states + ) + if self._can_project_hidden_mlp_with_fp4out(hidden_states): + mlp_fp4, mlp_sf = torch.ops.trtllm.cute_dsl_nvfp4_dense_gemm_swiglu_fp4out_blackwell( + act_fp4, + gate_up_proj.weight, + act_sf, + gate_up_proj.weight_scale, + alpha, + down_proj.input_scale, + ) + mlp_fp4 = mlp_fp4.reshape(*original_shape[:-1], mlp_fp4.shape[-1]) + return down_proj(Fp4QuantizedTensor(mlp_fp4, mlp_sf)) + + mlp_hidden = torch.ops.trtllm.cute_dsl_nvfp4_dense_gemm_swiglu_blackwell( + act_fp4, + gate_up_proj.weight, + act_sf, + gate_up_proj.weight_scale, + alpha, + gate_up_proj.dtype, + ) + expected_out = gate_up_proj.out_features // 2 + if mlp_hidden.shape[-1] > expected_out: + mlp_hidden = mlp_hidden[..., :expected_out].contiguous() + mlp_hidden = mlp_hidden.reshape(*original_shape[:-1], mlp_hidden.shape[-1]) + return down_proj(mlp_hidden) + + def _combine_split_projection( + self, + attn_out: torch.Tensor, + mlp_projected: torch.Tensor, + ) -> torch.Tensor: + out = self.to_out.allreduce(self.to_out.attn_proj(attn_out) + mlp_projected) + if self.to_out.has_bias: + out = out + self.to_out.bias + return out + + def _project_split_output_with_fp4_mlp( + self, + attn_out: torch.Tensor, + mlp_hidden: torch.Tensor, + ) -> torch.Tensor: + shape = mlp_hidden.shape + gate, up = mlp_hidden.chunk(2, dim=-1) + mlp_hidden = torch.cat((up, gate), dim=-1) + mlp_hidden = mlp_hidden.reshape(-1, shape[-1]) + tile_size = Flux2ParallelSelfAttention._SWIGLU_FP4_TILE_SIZE + num_tokens = mlp_hidden.shape[0] + num_tiles = (num_tokens + tile_size - 1) // tile_size + padded_tokens = num_tiles * tile_size + if padded_tokens != num_tokens: + mlp_hidden = F.pad(mlp_hidden, (0, 0, 0, padded_tokens - num_tokens)) + + tile_idx_to_mn_limit = ( + torch.arange(1, num_tiles + 1, dtype=torch.int32, device=mlp_hidden.device) * tile_size + ) + num_non_exiting_tiles = torch.tensor( + [num_tiles], dtype=torch.int32, device=mlp_hidden.device + ) + mlp_fp4, mlp_sf = torch.ops.trtllm.moe_swiglu_nvfp4_quantize( + mlp_hidden, + self.to_out.mlp_proj.input_scale, + tile_idx_to_mn_limit, + num_non_exiting_tiles, + tile_size, + ) + mlp_fp4 = mlp_fp4.view(torch.uint8) + mlp_out = self.to_out.mlp_proj(Fp4QuantizedTensor(mlp_fp4, mlp_sf)) + if padded_tokens != num_tokens: + mlp_out = mlp_out[:num_tokens] + mlp_out = mlp_out.reshape(*shape[:-1], mlp_out.shape[-1]) + return self._combine_split_projection(attn_out, mlp_out) + def forward( self, hidden_states: torch.Tensor, @@ -441,6 +643,14 @@ def forward( Returns: hidden_states [batch, seq, dim] """ + if self._can_project_hidden_mlp_with_cute_dsl(): + qkv = self.to_qkv_mlp_proj.qkv_proj(hidden_states) + q, k, v = self._apply_norm_rope(qkv, image_rotary_emb) + attn_out = self._attn_impl(q, k, v, timestep=timestep) + attn_out = attn_out.to(q.dtype) + mlp_out = self._project_hidden_mlp_with_cute_dsl(hidden_states) + return self._combine_split_projection(attn_out, mlp_out) + # Fused QKV + MLP projection qkv, mlp_hidden = self.to_qkv_mlp_proj(hidden_states) @@ -451,6 +661,9 @@ def forward( # Parallel MLP path (reshape to 2D for Triton kernel, then back) shape = mlp_hidden.shape + if self._can_project_mlp_out_from_fp4(): + return self._project_split_output_with_fp4_mlp(attn_out, mlp_hidden) + mlp_out = swiglu(mlp_hidden.reshape(-1, shape[-1])).reshape(*shape[:-1], -1) # Concatenate + project diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py b/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py index edb908c2d80a..7bc2a4938f7b 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py @@ -53,6 +53,7 @@ def __init__( skip_create_weights_in_init: bool = False, force_dynamic_quantization: bool = False, config: Optional[DiffusionModelConfig] = None, + attn_shard: Optional[tuple[int, int]] = None, ): super().__init__() mapping = config.mapping if config else None @@ -60,6 +61,11 @@ def __init__( self.tp_rank = getattr(mapping, "tp_rank", 0) self.attn_dim = attn_dim self.has_bias = bias + self.attn_shard = attn_shard + + assert attn_dim % self.tp_size == 0 or self.attn_shard is not None, ( + "Explicit attention sharding required for uneven TP" + ) if self.tp_size == 1: self.proj = Linear( @@ -84,6 +90,7 @@ def __init__( mapping=config.mapping, tensor_parallel_mode=TensorParallelMode.ROW, reduce_output=False, + override_tp_sharding=self.attn_shard, ) self.mlp_proj = Linear( mlp_dim, @@ -161,11 +168,14 @@ def __init__( quant_config=None, skip_create_weights_in_init: bool = False, force_dynamic_quantization: bool = False, + use_cute_dsl_blockscaling_mm: bool = False, mapping: Optional[Mapping] = None, + override_qkv_sharding=None, ): super().__init__() self.tp_size = mapping.tp_size if mapping else 1 + self.tp_rank = mapping.tp_rank if mapping else 0 # Store full (pre-TP) dims for weight loading (splitting checkpoint weight) self.full_q_dim = q_dim @@ -188,9 +198,15 @@ def __init__( self.local_qkv_dim = q_dim + 2 * kv_dim self.local_mlp_dim = mlp_dim else: - local_q_dim = q_dim // self.tp_size - local_kv_dim = kv_dim // self.tp_size - shard_mlp_hidden_dim = self.mlp_hidden_dim // self.tp_size + assert override_qkv_sharding is not None, ( + "override_qkv_sharding required when tp_size > 1" + ) + + def range_size(r): + return r[1] - r[0] + + local_q_dim = range_size(override_qkv_sharding["q"]) + local_kv_dim = range_size(override_qkv_sharding["k"]) # QKV: column-parallel with fused Q/K/V sharding self.qkv_proj = Linear( in_dim, @@ -211,8 +227,17 @@ def __init__( mapping=mapping, tensor_parallel_mode=TensorParallelMode.COLUMN, reduce_output=False, + override_tp_sharding=override_qkv_sharding, ) - # MLP gate+up: column-parallel with fused gate/up sharding + + local_mlp_hidden_start = Linear._calc_shard( + self.mlp_hidden_dim, self.tp_size, self.tp_rank + ) + local_mlp_hidden_end = Linear._calc_shard( + self.mlp_hidden_dim, self.tp_size, self.tp_rank + 1 + ) + local_mlp_hidden_size = local_mlp_hidden_end - local_mlp_hidden_start + self.mlp_proj = Linear( in_dim, mlp_dim, @@ -225,15 +250,20 @@ def __init__( weight_mode=WeightMode.FUSED_GATE_UP_LINEAR, ), fused_weight_shard_indices_mapping={ - "gate": (0, shard_mlp_hidden_dim), - "up": (shard_mlp_hidden_dim, shard_mlp_hidden_dim), + "gate": (0, local_mlp_hidden_size), + "up": (local_mlp_hidden_size, local_mlp_hidden_size), }, mapping=mapping, tensor_parallel_mode=TensorParallelMode.COLUMN, reduce_output=False, + use_cute_dsl_blockscaling_mm=use_cute_dsl_blockscaling_mm, + override_tp_sharding={ + "gate": (local_mlp_hidden_start, local_mlp_hidden_end), + "up": (local_mlp_hidden_start, local_mlp_hidden_end), + }, ) - self.local_qkv_dim = (q_dim + 2 * kv_dim) // self.tp_size - self.local_mlp_dim = mlp_dim // self.tp_size + self.local_qkv_dim = local_q_dim + 2 * local_kv_dim + self.local_mlp_dim = local_mlp_hidden_size def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: """Returns (qkv, mlp_gate_up) with local (post-TP) sizes.""" diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux.py b/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux.py index 0a35ce659122..5881614c872b 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux.py @@ -239,12 +239,13 @@ def post_load_weights(self) -> None: "return_dict", ], return_dict_default=False, + return_tuple_when_return_dict_false=True, ) ) - # TeaCache or Cache-DiT + # TeaCache or Cache-DiT: resolve coefficients here; the loader enables + # cache acceleration after torch.compile (see PipelineLoader.load). self._apply_teacache_coefficients(FLUX_TEACACHE_COEFFICIENTS) - self._setup_cache_acceleration() @property def default_generation_params(self): diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux2.py b/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux2.py index 4f0fa7391895..85a87f05a631 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux2.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux2.py @@ -18,15 +18,19 @@ - 4-axis RoPE: (32, 32, 32, 32) instead of 3-axis """ +import io import json import os import time -from typing import List, Optional, Tuple, Union +from contextlib import contextmanager +from typing import Any, Iterator, List, Optional, Tuple, Union import numpy as np +import PIL.Image import torch from diffusers import FlowMatchEulerDiscreteScheduler from diffusers.models.autoencoders.autoencoder_kl_flux2 import AutoencoderKLFlux2 +from diffusers.pipelines.flux2.image_processor import Flux2ImageProcessor from diffusers.utils.torch_utils import randn_tensor from transformers import ( AutoModelForCausalLM, @@ -92,10 +96,10 @@ def format_input(prompts: List[str], system_message: str) -> List[List[dict]]: @register_pipeline( "Flux2Pipeline", hf_ids=["black-forest-labs/FLUX.2-dev"], - doc="Black Forest Labs FLUX.2 family (text-to-image).", + doc="Black Forest Labs FLUX.2 family (text-to-image and reference-image generation).", ) class Flux2Pipeline(BasePipeline): - """FLUX.2 Text-to-Image Pipeline. + """FLUX.2 text-to-image and reference-image pipeline. Supports FLUX.2 model variants: - FLUX.2-dev (35B): guidance_embeds=True, embedded guidance @@ -105,6 +109,8 @@ class Flux2Pipeline(BasePipeline): Follows WAN pipeline pattern for DiffusionModelLoader integration. """ + derive_output_size_from_reference = True + # Hidden state layers per text encoder type (auto-detected at load time) _TEXT_ENCODER_CONFIG = { "Mistral3ForConditionalGeneration": { @@ -194,6 +200,17 @@ def default_warmup_num_frames(self): def warmup_cache_key(self, height: int, width: int, **kwargs) -> tuple: return (height, width) + def request_warmup_cache_key(self, req: Any) -> tuple: + cache_key = super().request_warmup_cache_key(req) + condition_images = req.prepared_inputs.get("condition_images") + if condition_images is None: + return cache_key + + reference_shapes = tuple( + (int(image.shape[-2]), int(image.shape[-1])) for image in condition_images + ) + return (*cache_key, len(condition_images), reference_shapes) + def _init_transformer(self) -> None: """Initialize FLUX.2 transformer with quantization support.""" logger.info("Creating FLUX.2 transformer with quantization support...") @@ -284,6 +301,7 @@ def load_standard_components( ) self.vae_scale_factor = 8 # FLUX.2 uses scale_factor=8 + self.image_processor = Flux2ImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) # Scheduler if PipelineComponent.SCHEDULER not in skip_components: @@ -333,11 +351,13 @@ def post_load_weights(self) -> None: guidance_param_name=guidance_param, forward_params=forward_params, return_dict_default=False, + return_tuple_when_return_dict_false=True, ) ) + # Cache acceleration itself is enabled by the loader after + # torch.compile (see PipelineLoader.load). self._apply_teacache_coefficients(FLUX2_TEACACHE_COEFFICIENTS) - self._setup_cache_acceleration() @property def default_generation_params(self): @@ -349,6 +369,20 @@ def default_generation_params(self): "max_sequence_length": 512, } + def prepare_request(self, req: Any) -> None: + """Load and preprocess reference images before warmup bookkeeping.""" + if req.params.image is None: + return + + reference_images = self._load_reference_images(req.params.image) + condition_images = self._preprocess_reference_images(reference_images) + req.params.height, req.params.width = self._resolve_target_dimensions( + req.params.height, + req.params.width, + condition_images, + ) + req.prepared_inputs["condition_images"] = condition_images + def infer(self, req): """Run inference from DiffusionRequest.""" return self.forward( @@ -360,6 +394,8 @@ def infer(self, req): seed=req.params.seed, max_sequence_length=req.params.max_sequence_length, num_images_per_prompt=req.params.num_images_per_prompt, + image=req.params.image, + _condition_images=req.prepared_inputs.get("condition_images"), ) @torch.inference_mode() @@ -367,21 +403,32 @@ def forward( self, prompt: Union[str, List[str]], seed: int, - height: int = 1024, - width: int = 1024, + height: Optional[int] = None, + width: Optional[int] = None, num_inference_steps: int = 50, guidance_scale: float = 3.5, max_sequence_length: int = 512, num_images_per_prompt: int = 1, + image: Optional[ + Union[ + PIL.Image.Image, + str, + bytes, + List[Union[PIL.Image.Image, str, bytes]], + ] + ] = None, + _condition_images: Optional[List[torch.Tensor]] = None, ): - """Generate image(s) from text prompt(s). + """Generate image(s) from text and optional reference images. Args: prompt: Text prompt or list of prompts for image generation. When a list is provided, generates one image per prompt in a single batched forward pass. - height: Output image height (default: 1024) - width: Output image width (default: 1024) + height: Output image height. Defaults to the first processed + reference image's height, or 1024 without a reference. + width: Output image width. Defaults to the first processed + reference image's width, or 1024 without a reference. num_inference_steps: Number of denoising steps guidance_scale: Embedded guidance scale seed: Random seed for reproducibility. @@ -389,6 +436,9 @@ def forward( num_images_per_prompt: Number of images to generate per prompt. Each prompt's embeddings are repeated and independent noise is sampled, producing N different images per prompt. + image: Reference image or shared list of reference images for + image conditioning. Public ``VisualGenParams`` requests use + file paths or encoded bytes; direct calls may also use PIL images. Returns: PipelineOutput with image tensor (B, H, W, C) where @@ -417,9 +467,35 @@ def forward( if num_images_per_prompt > 1: prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + condition_images = _condition_images + if condition_images is None and image is not None: + reference_images = self._load_reference_images(image) + condition_images = self._preprocess_reference_images(reference_images) + + height, width = self._resolve_target_dimensions(height, width, condition_images) latents, latent_ids = self._prepare_latents(batch_size, height, width, generator) logger.info(f"Latents shape: {latents.shape}") + image_latents = None + image_latent_ids = None + if condition_images is not None: + image_latents, image_latent_ids = self._prepare_image_latents( + condition_images, + batch_size=batch_size, + ) + image_latents = image_latents.to(device=latents.device, dtype=latents.dtype) + image_latent_ids = image_latent_ids.to(device=latent_ids.device, dtype=latent_ids.dtype) + self._validate_reference_sequence_length( + target_seq_len=latents.shape[1], + reference_seq_len=image_latents.shape[1], + sharder=getattr(self.transformer, "sharder", None), + ) + logger.info( + "Prepared %d FLUX.2 reference image(s), %d tokens total", + len(condition_images), + image_latents.shape[1], + ) + # Prepare timesteps with dynamic shifting # Use explicit linear sigmas (matches HF diffusers exactly) # This is critical for step-distilled models like FLUX.2-klein @@ -455,25 +531,39 @@ def forward_fn( extra_tensors, ): """Forward function for FLUX.2 transformer.""" - return self.transformer( - hidden_states=latents, + transformer_latents = latents + transformer_latent_ids = latent_ids + if image_latents is not None: + transformer_latents = torch.cat([latents, image_latents], dim=1) + transformer_latent_ids = torch.cat([latent_ids, image_latent_ids], dim=0) + + noise_pred = self.transformer( + hidden_states=transformer_latents, encoder_hidden_states=encoder_hidden_states, timestep=timestep / 1000, # FLUX.2 expects normalized timesteps - img_ids=latent_ids, + img_ids=transformer_latent_ids, txt_ids=text_ids, guidance=guidance, return_dict=False, )[0] + if image_latents is None: + return noise_pred + return noise_pred[:, : latents.shape[1]] timer.mark_denoise_start() - latents = self.denoise( - latents=latents, - scheduler=self.scheduler, - prompt_embeds=prompt_embeds, - guidance_scale=1.0, # No CFG: guidance is embedded - forward_fn=forward_fn, - timesteps=timesteps, - ) + # Reference count and dimensions change the transformer sequence length. + # Run those requests outside CUDA graphs so a long-lived process does not + # retain one graph per reference shape. torch.compile remains active and + # may compile a new sequence shape; text-only requests still use graphs. + with self._temporarily_disable_cuda_graphs(disable=image_latents is not None): + latents = self.denoise( + latents=latents, + scheduler=self.scheduler, + prompt_embeds=prompt_embeds, + guidance_scale=1.0, # No CFG: guidance is embedded + forward_fn=forward_fn, + timesteps=timesteps, + ) timer.mark_post_start() # Decode @@ -488,6 +578,23 @@ def forward_fn( timer.mark_end() return timer.fill(PipelineOutput(image=image)) + @contextmanager + def _temporarily_disable_cuda_graphs(self, disable: bool) -> Iterator[None]: + """Bypass CUDA graphs for a request without discarding captured graphs.""" + runners = list(getattr(self, "_cuda_graph_runners", {}).values()) + if not disable or not runners: + yield + return + + previous_states = [runner.enabled for runner in runners] + for runner in runners: + runner.enabled = False + try: + yield + finally: + for runner, enabled in zip(runners, previous_states): + runner.enabled = enabled + def _encode_prompt( self, prompt: List[str], @@ -629,6 +736,151 @@ def _prepare_latent_ids(self, height: int, width: int) -> torch.Tensor: return latent_ids # [seq_len, 4] + @staticmethod + def _load_reference_images( + image: Union[ + PIL.Image.Image, + str, + bytes, + List[Union[PIL.Image.Image, str, bytes]], + ], + ) -> List[PIL.Image.Image]: + """Normalize supported reference-image inputs to materialized RGB images.""" + inputs = image if isinstance(image, list) else [image] + if not inputs: + raise ValueError("`image` must contain at least one reference image.") + + images = [] + for index, item in enumerate(inputs): + try: + if isinstance(item, PIL.Image.Image): + images.append(item.convert("RGB")) + elif isinstance(item, str): + with PIL.Image.open(item) as loaded: + images.append(loaded.convert("RGB")) + elif isinstance(item, bytes): + with PIL.Image.open(io.BytesIO(item)) as loaded: + images.append(loaded.convert("RGB")) + else: + raise ValueError( + "Reference images must be PIL images, file paths, or encoded bytes; " + f"item {index} has type {type(item).__name__}." + ) + except OSError as exc: + raise ValueError(f"Unable to load reference image {index}: {exc}") from exc + return images + + def _preprocess_reference_images(self, images: List[PIL.Image.Image]) -> List[torch.Tensor]: + """Apply the upstream FLUX.2 area, crop, and normalization rules.""" + condition_images = [] + multiple_of = self.vae_scale_factor * 2 + for image in images: + self.image_processor.check_image_input(image) + image = self.image_processor._resize_if_exceeds_area(image) + image_width, image_height = image.size + image_width = (image_width // multiple_of) * multiple_of + image_height = (image_height // multiple_of) * multiple_of + condition_images.append( + self.image_processor.preprocess( + image, + height=image_height, + width=image_width, + resize_mode="crop", + ) + ) + return condition_images + + @staticmethod + def _prepare_image_ids( + image_latents: List[torch.Tensor], + scale: int = 10, + ) -> torch.Tensor: + """Create reference-image position IDs with distinct FLUX.2 T offsets.""" + image_ids = [] + for index, latent in enumerate(image_latents): + _batch_size, _channels, height, width = latent.shape + device = latent.device + t_dim = torch.tensor([scale * (index + 1)], device=device) + h_dim = torch.arange(height, device=device) + w_dim = torch.arange(width, device=device) + l_dim = torch.arange(1, device=device) + image_ids.append(torch.cartesian_prod(t_dim, h_dim, w_dim, l_dim)) + return torch.cat(image_ids, dim=0).float() + + @staticmethod + def _validate_reference_sequence_length( + target_seq_len: int, + reference_seq_len: int, + sharder: Any, + ) -> None: + """Fail before denoising when sequence parallelism cannot shard image tokens.""" + if sharder is None or not sharder.is_active: + return + + combined_seq_len = target_seq_len + reference_seq_len + if combined_seq_len % sharder.size != 0: + raise ValueError( + "FLUX.2 reference-image conditioning produced " + f"{combined_seq_len} image tokens ({target_seq_len} target + " + f"{reference_seq_len} reference), which is not divisible by the configured " + f"sequence-parallel size {sharder.size}. Adjust the target/reference image " + "dimensions or disable sequence parallelism." + ) + + @staticmethod + def _resolve_target_dimensions( + height: Optional[int], + width: Optional[int], + condition_images: Optional[List[torch.Tensor]], + ) -> Tuple[int, int]: + """Match Diffusers defaults: first reference dimensions, then 1024 fallback.""" + if condition_images: + height = height or condition_images[0].shape[-2] + width = width or condition_images[0].shape[-1] + return height or 1024, width or 1024 + + @staticmethod + def _patchify_latents(latents: torch.Tensor) -> torch.Tensor: + """Patchify VAE latents from 32 channels to FLUX.2's packed 128 channels.""" + batch_size, channels, height, width = latents.shape + latents = latents.reshape(batch_size, channels, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 1, 3, 5, 2, 4) + return latents.reshape(batch_size, channels * 4, height // 2, width // 2) + + def _encode_vae_image(self, image: torch.Tensor) -> torch.Tensor: + """Encode one preprocessed reference image deterministically.""" + if image.ndim != 4: + raise ValueError(f"Expected reference image rank 4, got {image.ndim}.") + + encoded = self.vae.encode(image) + image_latents = encoded.latent_dist.mode() + image_latents = self._patchify_latents(image_latents) + + bn_eps = getattr(self.vae.config, "batch_norm_eps", 1e-5) + latents_bn_mean = self.vae.bn.running_mean.view(1, -1, 1, 1).to( + image_latents.device, image_latents.dtype + ) + latents_bn_std = torch.sqrt(self.vae.bn.running_var.view(1, -1, 1, 1) + bn_eps).to( + image_latents.device, image_latents.dtype + ) + return (image_latents - latents_bn_mean) / latents_bn_std + + def _prepare_image_latents( + self, + images: List[torch.Tensor], + batch_size: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """VAE-encode, pack, and concatenate a shared reference-image set.""" + image_latents_4d = [] + for image in images: + image = image.to(device=self.device, dtype=self.vae.dtype) + image_latents_4d.append(self._encode_vae_image(image)) + + image_ids = self._prepare_image_ids(image_latents_4d) + packed_latents = [self._pack_latents(latent) for latent in image_latents_4d] + image_latents = torch.cat(packed_latents, dim=1).repeat(batch_size, 1, 1) + return image_latents, image_ids + def _prepare_latents( self, batch_size: int, diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py b/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py index d6551918141d..527dce31df23 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py @@ -467,21 +467,6 @@ def __init__( ) self.act_mlp = _gelu_tanh_eager - kv_dim = num_attention_heads * attention_head_dim - - # MLP + Attn Output projection, requires special handling for TP - self.proj_out = FluxJointAttnMLPProj( - attn_dim=kv_dim, - mlp_dim=self.mlp_hidden_dim, - out_dim=dim, - bias=True, - dtype=dtype, - quant_config=quant_config, - skip_create_weights_in_init=skip_create_weights, - force_dynamic_quantization=force_dynamic_quant, - config=config, - ) - # Attention (no added_kv_proj_dim since tokens are already concatenated) self.attn = FluxJointAttention( hidden_size=dim, @@ -495,6 +480,21 @@ def __init__( module_name=f"single_transformer_blocks.{layer_idx}.attn", ) + # MLP + Attn Output projection, requires special handling for TP + self.proj_out = FluxJointAttnMLPProj( + attn_dim=self.attn.q_dim, + mlp_dim=self.mlp_hidden_dim, + out_dim=dim, + bias=True, + dtype=dtype, + quant_config=quant_config, + skip_create_weights_in_init=skip_create_weights, + force_dynamic_quantization=force_dynamic_quant, + config=config, + # need explicit shard because we are aligned on head boundaries + attn_shard=(self.attn.local_q_dim_start, self.attn.local_q_dim_end), + ) + def forward( self, hidden_states: torch.Tensor, diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux2.py b/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux2.py index 9cdbad634920..253f8a7599dc 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux2.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux2.py @@ -44,7 +44,9 @@ from tensorrt_llm._torch.visual_gen.models.modeling import BaseDiffusionModel from tensorrt_llm._torch.visual_gen.quantization.loader import DynamicLinearWeightLoader from tensorrt_llm._torch.visual_gen.utils import SequenceSharder +from tensorrt_llm._utils import is_sm_100f from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo # HF FLUX.2 uses Flux2FeedForward with linear_in/linear_out attribute names. # We use GatedMLP which uses gate_up_proj/down_proj. Remap at load time. @@ -248,6 +250,12 @@ def __init__( # FFN for image stream (shared GatedMLP from _torch/modules) # HF key remapping (linear_in.* → gate_up_proj.*, linear_out.* → down_proj.*) in load_weights() + use_cute_dsl_swiglu = ( + torch.cuda.is_available() + and is_sm_100f() + and config is not None + and config.get_quant_config().quant_algo == QuantAlgo.NVFP4 + ) self.ff = GatedMLP( hidden_size=dim, intermediate_size=int(dim * mlp_ratio), @@ -256,6 +264,7 @@ def __init__( config=config, layer_idx=layer_idx, reduce_output=(tp_size != 1), + use_cute_dsl_blockscaling_mm=use_cute_dsl_swiglu, ) # FFN for text stream self.ff_context = GatedMLP( @@ -266,6 +275,7 @@ def __init__( config=config, layer_idx=layer_idx, reduce_output=(tp_size != 1), + use_cute_dsl_blockscaling_mm=use_cute_dsl_swiglu, ) def forward( diff --git a/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2.py b/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2.py index 1336bb7249bc..2ebee01b7d35 100644 --- a/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2.py +++ b/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2.py @@ -793,9 +793,15 @@ def _init_transformer(self) -> None: double_precision_rope=double_precision_rope, apply_gated_attention=apply_gated_attention, model_config=model_config, + **self._extra_transformer_kwargs(), ) self.transformer._transformer_config = vars(cfg) + def _extra_transformer_kwargs(self) -> dict: + """Extra LTXModel constructor kwargs; empty for the base one-stage + pipeline. Subclasses override to add topology-specific kwargs.""" + return {} + # ------------------------------------------------------------------ # CUDA graph setup (Modality-aware override) # ------------------------------------------------------------------ @@ -1026,7 +1032,9 @@ def post_load_weights(self) -> None: """Finalize after weight loading: TeaCache, Cache-DiT, derived attributes.""" super().post_load_weights() - # LTX-2: single transformer (one DiT for video+audio); TeaCache only with explicit coefficients. + # LTX-2: single transformer (one DiT for video+audio); TeaCache only with + # explicit coefficients. Cache acceleration itself is enabled by the + # loader after torch.compile (see PipelineLoader.load). if self.transformer is not None and self.pipeline_config.cache_backend == "teacache": if self.pipeline_config.teacache.coefficients is None: raise ValueError( @@ -1037,11 +1045,6 @@ def post_load_weights(self) -> None: "LTXModel", LTX2TeaCacheExtractor(self._compute_ltx2_timestep_embedding), ) - self._setup_cache_acceleration() - - # Cache-DiT - if self.transformer is not None and self.pipeline_config.cache_backend == "cache_dit": - self._setup_cache_acceleration() # Compression ratios from native scale factors self.vae_spatial_compression_ratio = VIDEO_SCALE_FACTORS.width @@ -2026,10 +2029,6 @@ def forward_fn( def decode_video_fn(vid_latents): vid_latents = self.video_patchifier.unpatchify(vid_latents, video_shape) - - if output_type == "latent": - return vid_latents - vid_latents = vid_latents.to(self.dtype) tiling_config = TilingConfig.default() if self._parallel_vae_enabled: @@ -2059,18 +2058,27 @@ def decode_video_fn(vid_latents): def decode_audio_fn(aud_latents): aud_latents = self.audio_patchifier.unpatchify(aud_latents, audio_shape) - - if output_type == "latent": - return aud_latents - aud_latents = aud_latents.to(self.dtype) return decode_audio(aud_latents, self.audio_decoder, self.vocoder) - video, audio = self.decode_latents( - latents=latents, - decode_fn=decode_video_fn, - extra_latents={"audio": (audio_latents, decode_audio_fn)}, - ) + if output_type == "latent": + # Latent output is a local unpatchify with no VAE work, and every + # rank already holds the full latents after the denoise loop — + # return them on every rank. decode_latents' vae_ranks/rank-0 gate + # exists to skip real VAE decode only; the two-stage handoff + # consumes these latents in place with zero collectives. + video = self.video_patchifier.unpatchify(latents, video_shape) + audio = ( + self.audio_patchifier.unpatchify(audio_latents, audio_shape) + if audio_latents is not None + else None + ) + else: + video, audio = self.decode_latents( + latents=latents, + decode_fn=decode_video_fn, + extra_latents={"audio": (audio_latents, decode_audio_fn)}, + ) if self.rank == 0: logger.info(f"Decoding completed in {time.time() - decode_start:.2f}s") diff --git a/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2_two_stages.py b/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2_two_stages.py index b7ef703be22d..dec087bf0378 100644 --- a/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2_two_stages.py +++ b/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2_two_stages.py @@ -47,8 +47,70 @@ _LTX2CUDAGraphRunner, _prefetch_ltx2_safetensors_files, ) +from .transformer_ltx2 import Stage2Groups STAGE_2_DISTILLED_SIGMA_VALUES = [0.909375, 0.725, 0.421875, 0.0] + + +class _TwoStagePhaseTimer(CudaPhaseTimer): + """CudaPhaseTimer + the two-stage extras: the stage-2 refinement loop and + the decode section. + + Inherited marks keep their contract (``denoise`` = the whole stage-1 + forward; stage 2 folds into ``post_denoise`` on ``PipelineOutput``). + The extra event pair brackets the stage-2 refinement step loop only + (upsample / LoRA bind / text-cache prep stay outside). + + Event deltas are GPU-stream distances: they include GPU work plus any + CPU time exposed to the stream, and stay correct under CUDA graphs and + async enqueue (unlike host wall clocks read at enqueue time). + """ + + def __init__(self) -> None: + super().__init__() + self._stage2_marked = 0 + self._decode_marked = 0 + if self._enabled: + self._stage2_start = torch.cuda.Event(enable_timing=True) + self._stage2_end = torch.cuda.Event(enable_timing=True) + self._decode_start = torch.cuda.Event(enable_timing=True) + self._decode_end = torch.cuda.Event(enable_timing=True) + + def mark_stage2_start(self) -> None: + if self._enabled: + self._stage2_start.record() + self._stage2_marked = 1 + + def mark_stage2_end(self) -> None: + if self._enabled and self._stage2_marked == 1: + self._stage2_end.record() + self._stage2_marked = 2 + + def stage2_denoise_time(self) -> Optional[float]: + """Loop-only stage-2 denoise seconds; None if stage 2 never completed.""" + if not self._enabled or self._stage2_marked != 2: + return None + self._stage2_end.synchronize() + return self._stage2_start.elapsed_time(self._stage2_end) / 1000.0 + + def mark_decode_start(self) -> None: + if self._enabled: + self._decode_start.record() + self._decode_marked = 1 + + def mark_decode_end(self) -> None: + if self._enabled and self._decode_marked == 1: + self._decode_end.record() + self._decode_marked = 2 + + def decode_time(self) -> Optional[float]: + """Decode-only seconds; None if the decode bracket never completed.""" + if not self._enabled or self._decode_marked != 2: + return None + self._decode_end.synchronize() + return self._decode_start.elapsed_time(self._decode_end) / 1000.0 + + _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2) # Baseline BF16 peak memory ~75 GiB, saving BF16 weights snapshot total ~108 GiB. _BF16_WEIGHTS_SNAPSHOT_FREE_MEMORY_THRESHOLD_GIB = 115.0 @@ -897,20 +959,26 @@ def precision_counts(self) -> Dict[str, int]: class _LTX2TwoStageCUDAGraphRunner(_LTX2CUDAGraphRunner): - """CUDA graph runner keyed by LTX-2 two-stage LoRA weight state.""" + """CUDA graph runner keyed by LTX-2 two-stage LoRA weight state and the + transformer's active topology (``topology_getter`` reads + ``LTXModel.active_topology``, the single source of truth, so the selected + graph always matches the live attention stacks).""" def __init__( self, config: CUDAGraphRunnerConfig, lora_state_getter: Callable[[], str], + topology_getter: Callable[[], str], ) -> None: super().__init__(config) self._lora_state_getter = lora_state_getter + self._topology_getter = topology_getter def get_graph_key(self, *args, **kwargs): return ( *super().get_graph_key(*args, **kwargs), ("ltx2_two_stage_lora_state", self._lora_state_getter()), + ("ltx2_two_stage_topology", self._topology_getter()), ) @@ -941,6 +1009,50 @@ def common_warmup_shapes(self) -> list: def _current_lora_cuda_graph_state(self) -> str: return getattr(self, "_lora_cuda_graph_state", "original") + def _extra_transformer_kwargs(self) -> dict: + """Hand the dual-topology groups to LTXModel; built collectively here at + pipeline init (load). Only the two-stage pipeline supplies them, so the + base one-stage pipeline stays free of any stage-2 surface.""" + return {"stage2_groups": self._build_stage2_dit_groups()} + + def _build_stage2_dit_groups(self) -> Optional[Stage2Groups]: + """Build the stage-2 dual-topology groups (once, collectively, at load). + + cfg is flattened into ulysses while every other mesh dim's groups are + preserved verbatim, so cp-side attention wrappers keep their peers. The + only new communicators are the stage-2 ulysses groups, plus one + seq-plane group at cp>1 whose shard/gather order is cp-major (encoded + via seq_rank/gather_index — ``dist.new_group`` sorts its rank list). + """ + vgm = self.pipeline_config.visual_gen_mapping + if vgm is None or vgm.world_size <= 1 or vgm.cfg_size <= 1: + return None + # At tp>1 the seq plane below (flattened fold lists) would cross TP + # fibers whose ranks hold different weight shards. + assert vgm.tp_size == 1, "two-stage dual topology requires tp_size == 1" + fold = vgm.flatten_cfg_ranks() + uly_group = None + # Every rank must create EVERY group (world-collective); keep only ours. + for ranks in fold: + g = dist.new_group(ranks, use_local_synchronization=False) + if self.rank in ranks: + uly_group = g + flat = [r for ranks in fold for r in ranks] + if len(fold) == 1: + seq_group, gather_index = uly_group, None + else: + # dist.new_group sorts its rank list; at tp=1 the plane spans + # 0..world-1, so the cp-major flat order doubles as group ranks. + seq_group = dist.new_group(sorted(flat), use_local_synchronization=False) + gather_index = flat + return Stage2Groups( + ulysses_group=uly_group, + seq_group=seq_group, + seq_rank=flat.index(self.rank), + seq_size=len(flat), + gather_index=gather_index, + ) + def _is_cuda_graph_enabled(self) -> bool: for config_name in ("pipeline_config", "model_config"): config = getattr(self, config_name, None) @@ -972,6 +1084,7 @@ def _setup_cuda_graphs(self): runner = _LTX2TwoStageCUDAGraphRunner( CUDAGraphRunnerConfig(use_cuda_graph=True), self._current_lora_cuda_graph_state, + lambda: self.transformer.active_topology, ) compile_note = " (with torch.compile)" if self.pipeline_config.torch_compile.enable else "" logger.info( @@ -1181,12 +1294,29 @@ def forward( enhance_prompt = False _assert_resolution(height, width, is_two_stage=True) + if self.transformer._has_stage2: + # Fail fast (no serial fallback exists): the full-resolution latent + # token count must divide the stage-2 seq plane. Uses the canonical + # VAE scale factors rather than re-deriving them. + lat = VideoLatentShape.from_pixel_shape( + VideoPixelShape( + batch=1, frames=num_frames, height=height, width=width, fps=frame_rate + ) + ) + s2_tokens = lat.frames * lat.height * lat.width + seq = self.transformer._sharder_s2.size + if s2_tokens % seq != 0: + raise ValueError( + f"Stage-2 patchified token count ({s2_tokens} = {lat.frames}x" + f"{lat.height}x{lat.width}) is not divisible by the stage-2 " + f"seq-plane size ({seq}); adjust resolution or num_frames." + ) pipeline_start = time.time() # Two-stage timing: stage 1 is reported as ``denoise``; stage 2 # (spatial upsample + refinement denoise + decode) folds into # ``post_denoise``. Only the outer timer's numbers reach # ``PipelineOutput``. - timer = CudaPhaseTimer() + timer = _TwoStagePhaseTimer() timer.mark_pre_start() height_s1 = height // 2 width_s1 = width // 2 @@ -1219,135 +1349,18 @@ def forward( enhance_prompt=enhance_prompt, ) + # Every rank computes the full stage-1 latents (all-rank denoise loop + + # per-forward gather) and output_type="latent" returns them on every + # rank, so Stage 2 needs no handoff collective in either parallel-VAE mode. video_latents = out.video # (B, C, F_lat, H_lat_s1, W_lat_s1) audio_latents = out.audio # (B, C, F_aud, M) or None + assert video_latents is not None, "stage-1 latents missing on this rank" timer.mark_post_start() - # Non-primary workers (rank != 0) receive None from - # decode_latents and exit here. Rank 0 continues with Stage 2. - if video_latents is None: - timer.mark_end() - return timer.fill(PipelineOutput(video=None, audio=None, frame_rate=float(frame_rate))) - # ================================================================ - # Stage 2: spatial upsample + refinement denoise (rank-0 only) + # Stage 2: spatial upsample + refinement denoise — all ranks, collectively # ================================================================ - # Only rank 0 refines; other vae_ranks skip Stage 2 and rejoin at the - # collective decode below, receiving the refined latents via broadcast. - if self.rank == 0: - video_latents, audio_latents = self._upsample_and_refine( - video_latents=video_latents, - audio_latents=audio_latents, - prompt=prompt, - height=height, - width=width, - num_frames=num_frames, - frame_rate=frame_rate, - seed=seed, - max_sequence_length=max_sequence_length, - image=image, - image_cond_strength=image_cond_strength, - ) - else: - video_latents, audio_latents = None, None - - # ================================================================ - # Decode - # ================================================================ - if output_type == "latent": - # No decode: only rank 0 holds the refined latents. - video_out, audio_out = ( - (video_latents, audio_latents) if self.rank == 0 else (None, None) - ) - if self.rank == 0: - logger.info(f"Two-stage total time: {time.time() - pipeline_start:.2f}s") - timer.mark_end() - return timer.fill( - PipelineOutput( - video=video_out, - audio=audio_out, - frame_rate=float(frame_rate), - audio_sample_rate=( - int(self.audio_sampling_rate) - if getattr(self, "audio_sampling_rate", None) is not None - and audio_out is not None - else None - ), - ) - ) - - if self._parallel_vae_enabled: - # Broadcast rank-0's refined Stage-2 latents to every vae_rank, then - # decode collectively (tile-parallel over vgm.vae_group). - vgm = self.pipeline_config.visual_gen_mapping - video_latents = self._broadcast_video_latents(video_latents, vgm.vae_group) - logger.info("Decoding upsampled video (tile-parallel)...") - video = tile_parallel_decode( - self.video_decoder, - video_latents, - TilingConfig.default(), - pg=vgm.vae_group, - ) - video = postprocess_video_tensor(video) - else: - logger.info("Decoding upsampled video (tiled)...") - video_latents = video_latents.to(self.dtype) - chunks = list( - self.video_decoder.tiled_decode( - video_latents, - TilingConfig.default(), - generator=None, - ) - ) - video = torch.cat(chunks, dim=2) - video = postprocess_video_tensor(video) - - # Audio decode is rank-0 only (not tile-parallel). - audio_out = None - if self.rank == 0 and audio_latents is not None: - audio_latents = audio_latents.to(self.dtype) - audio_out = decode_audio(audio_latents, self.audio_decoder, self.vocoder) - - if self.rank == 0: - logger.info(f"Two-stage total time: {time.time() - pipeline_start:.2f}s") - timer.mark_end() - return timer.fill( - PipelineOutput( - video=video, - audio=audio_out, - frame_rate=float(frame_rate), - audio_sample_rate=( - int(self.audio_sampling_rate) - if getattr(self, "audio_sampling_rate", None) is not None - and audio_out is not None - else None - ), - ) - ) - - # ------------------------------------------------------------------ - # Helpers - # ------------------------------------------------------------------ - - def _upsample_and_refine( - self, - video_latents: torch.Tensor, - audio_latents: Optional[torch.Tensor], - prompt: Union[str, List[str]], - height: int, - width: int, - num_frames: int, - frame_rate: float, - seed: int, - max_sequence_length: int, - image: Optional[Union[str, torch.Tensor]] = None, - image_cond_strength: float = 1.0, - ) -> tuple: - """Stage 2 (rank-0 only): learned 2x spatial upsample + refinement denoise. - - Returns the refined ``(video_latents, audio_latents)`` in 5-D form. - """ per_ch_stats = self._get_per_channel_statistics() video_latents = upsample_video( video_latents[:1], @@ -1366,7 +1379,6 @@ def _upsample_and_refine( snapshot_required = 0 n = 0 dense_lora_merge_completed = False - stage2_start = time.time() try: if using_persistent_lora: lora_cache.bind_merged() @@ -1390,9 +1402,19 @@ def _upsample_and_refine( self._lora_cuda_graph_state = "merged" logger.info(f"Merged distilled LoRA ({n} params) for stage 2 (BF16 weights)") - # Disable Ulysses for Stage 2: only rank 0 is active, so - # cross-rank collectives in the attention backend would hang. - self.transformer.set_ulysses_enabled(False) + if self.transformer._has_stage2: + # The switch atomically moves the graph-key topology with the + # attention stacks (the key reads active_topology). It precedes + # prepare_text_cache (topology-dependent) inside + # _refinement_denoise and is restored in the finally below. + self.transformer.set_ulysses_topology(is_stage2=True) + if self.rank == 0: + vgm = self.pipeline_config.visual_gen_mapping + logger.info( + f"Stage 2: switched parallel topology from cfg{vgm.cfg_size} " + f"uly{vgm.ulysses_size} to uly {self.transformer._sharder_s2.size} " + "(there is no cfg in stage 2)" + ) video_latents, audio_latents = self._refinement_denoise( video_latents=video_latents, audio_latents=audio_latents, @@ -1405,11 +1427,11 @@ def _upsample_and_refine( max_sequence_length=max_sequence_length, image=image, image_cond_strength=image_cond_strength, + timer=timer, ) finally: - stage2_denoise_time = time.time() - stage2_start - logger.info(f"Stage 2 denoising time: {stage2_denoise_time:.2f}s (BF16 weights)") - self.transformer.set_ulysses_enabled(True) + if self.transformer._has_stage2: + self.transformer.set_ulysses_topology(is_stage2=False) if using_persistent_lora: lora_cache.bind_original() self._lora_cuda_graph_state = "original" @@ -1444,34 +1466,99 @@ def _upsample_and_refine( else: self._lora_cuda_graph_state = "original" - return video_latents, audio_latents - - def _broadcast_video_latents( - self, video_latents: Optional[torch.Tensor], vae_group - ) -> torch.Tensor: - """Broadcast rank-0's refined Stage-2 latents to every ``vae_rank``. - - ``tile_parallel_decode`` needs the full latent replicated on each rank of - ``vae_group``; only rank 0 ran Stage 2, so it is the broadcast source. - Video latents are 5-D ``(B, C, F, H, W)``. - """ - if vae_group is None: - raise ValueError( - "parallel VAE decode requires a valid vae_group, got None " - "(a None group would fall back to the world group and hang on non-VAE ranks)." + # ================================================================ + # Decode + # ================================================================ + if output_type == "latent": + # No decode. Every rank holds the refined latents; the external + # contract returns them on rank 0 only. + video_out, audio_out = ( + (video_latents, audio_latents) if self.rank == 0 else (None, None) ) - if self.rank == 0: + if self.rank == 0: + logger.info(f"Two-stage total time: {time.time() - pipeline_start:.2f}s") + timer.mark_end() + return timer.fill( + PipelineOutput( + video=video_out, + audio=audio_out, + frame_rate=float(frame_rate), + audio_sample_rate=( + int(self.audio_sampling_rate) + if getattr(self, "audio_sampling_rate", None) is not None + and audio_out is not None + else None + ), + ) + ) + + # Event bracket: a host wall clock started here would absorb the queued + # stage-2 GPU tail (the graphed loop leaves the host far ahead). + timer.mark_decode_start() + vgm = self.pipeline_config.visual_gen_mapping + if self._parallel_vae_enabled and self.rank in vgm.vae_ranks: + # Parallel Stage 2 left identical refined latents on every rank; + # VAE ranks decode collectively (tile-parallel over vgm.vae_group). video_latents = video_latents.to(self.dtype).contiguous() - shape = torch.tensor(video_latents.shape, dtype=torch.long, device=self.device) + logger.info("Decoding upsampled video (tile-parallel)...") + video = tile_parallel_decode( + self.video_decoder, + video_latents, + TilingConfig.default(), + pg=vgm.vae_group, + ) + video = postprocess_video_tensor(video) + elif not self._parallel_vae_enabled and self.rank == 0: + logger.info("Decoding upsampled video (tiled)...") + video_latents = video_latents.to(self.dtype) + chunks = list( + self.video_decoder.tiled_decode( + video_latents, + TilingConfig.default(), + generator=None, + ) + ) + video = torch.cat(chunks, dim=2) + video = postprocess_video_tensor(video) else: - shape = torch.empty(5, dtype=torch.long, device=self.device) - dist.broadcast(shape, src=0, group=vae_group) - if self.rank != 0: - video_latents = torch.empty( - torch.Size(shape.tolist()), dtype=self.dtype, device=self.device + # Non-decoding ranks (outside vae_ranks under parallel VAE, or + # non-rank-0 otherwise) return no media. + video = None + + # Audio decode is rank-0 only (not tile-parallel). + audio_out = None + if self.rank == 0 and audio_latents is not None: + audio_latents = audio_latents.to(self.dtype) + audio_out = decode_audio(audio_latents, self.audio_decoder, self.vocoder) + + timer.mark_decode_end() + if self.rank == 0: + logger.info(f"Stage 1 denoising time: {out.denoise:.2f}s") + stage2_s = timer.stage2_denoise_time() + if stage2_s is not None: + logger.info(f"Stage 2 denoising time: {stage2_s:.2f}s") + decode_s = timer.decode_time() + if decode_s is not None: + logger.info(f"Decoding completed in {decode_s:.2f}s") + logger.info(f"Two-stage total time: {time.time() - pipeline_start:.2f}s") + timer.mark_end() + return timer.fill( + PipelineOutput( + video=video, + audio=audio_out, + frame_rate=float(frame_rate), + audio_sample_rate=( + int(self.audio_sampling_rate) + if getattr(self, "audio_sampling_rate", None) is not None + and audio_out is not None + else None + ), ) - dist.broadcast(video_latents, src=0, group=vae_group) - return video_latents + ) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ def _get_per_channel_statistics(self) -> torch.nn.Module: """Return per-channel statistics for un-normalize/normalize. @@ -1499,6 +1586,7 @@ def _refinement_denoise( max_sequence_length: int, image: Optional[Union[str, torch.Tensor]] = None, image_cond_strength: float = 1.0, + timer: Optional[_TwoStagePhaseTimer] = None, ) -> tuple: """Run stage 2 refinement denoising on upsampled latents. @@ -1628,6 +1716,11 @@ def _refinement_denoise( dtype=self.dtype, ) + # stage2_denoise measures ONLY the step loop (upsample, LoRA bind, + # and text-cache/scheduler prep stay outside the bracket). + if timer is not None: + timer.mark_stage2_start() + # --- Euler denoising loop (no guidance) --- for i in range(len(sigmas) - 1): with nvtx_range(f"refinement_step {i}"): @@ -1689,6 +1782,9 @@ def _refinement_denoise( velocity_a = (a_working.float() - denoised_a) / sigma_a a_working = (a_working.float() + velocity_a * dt).to(a_working.dtype) + if timer is not None: + timer.mark_stage2_end() + # --- Unpatchify --- video_out = self.video_patchifier.unpatchify(v_working, video_shape) audio_out = None diff --git a/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py b/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py index 8d5f4aab2581..7cba785b1bf2 100644 --- a/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py +++ b/tensorrt_llm/_torch/visual_gen/models/ltx2/transformer_ltx2.py @@ -24,13 +24,18 @@ from typing import TYPE_CHECKING, Any, Optional import torch +import torch.distributed as torch_dist import torch.nn as nn import torch.nn.functional as F from tqdm import tqdm -from tensorrt_llm._torch.modules.linear import Linear, WeightMode +from tensorrt_llm._torch.modules.linear import Linear, UnquantizedLinearMethod, WeightMode from tensorrt_llm._torch.modules.mlp import MLP from tensorrt_llm._torch.utils import Fp4QuantizedTensor, gelu_tanh +from tensorrt_llm._torch.visual_gen.attention_backend.parallel import ( + UlyssesAttention, + wrap_parallel_attention, +) from tensorrt_llm._torch.visual_gen.attention_backend.utils import create_attention from tensorrt_llm._torch.visual_gen.models.modeling import BaseDiffusionModel from tensorrt_llm._torch.visual_gen.modules.attention import Attention, QKVMode @@ -103,6 +108,7 @@ def __init__( module_name: Optional[str] = None, enable_sequence_parallel: bool = False, async_ulysses: bool = False, + stage2_ulysses_group: Optional["torch_dist.ProcessGroup"] = None, ): from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig @@ -129,13 +135,17 @@ def __init__( qkv_mode = QKVMode.FUSE_QKV # Caller opts in via enable_sequence_parallel. Cross-attn supports - # Ulysses-only (SEPARATE_QKV + ring/attn2d is rejected in Attention); - # when ring/attn2d CP is active we disable wrappers and fall back to + # Ulysses, and distributed Attention2D only WITH Ulysses on top (the AV + # dispatch keys the seq-sharded K/V path off the Ulysses wrapper); under + # ring CP or attn2d-without-ulysses we disable wrappers and fall back to # the plain backend + all-gather in the AV cross-attn forward path. ulysses_size = vgm.ulysses_size if vgm is not None else 1 cp_size = vgm.cp_size if vgm is not None else 1 + attn2d_active = vgm is not None and vgm.attn2d_row_size * vgm.attn2d_col_size > 1 if self._is_cross_attn: - enable_sp = enable_sequence_parallel and cp_size == 1 + enable_sp = enable_sequence_parallel and ( + cp_size == 1 or (attn2d_active and ulysses_size > 1) + ) else: enable_sp = enable_sequence_parallel @@ -162,7 +172,6 @@ def __init__( ) # Validate Ulysses head divisibility (from main). - self._has_dual_attn = False if enable_sp and ulysses_size > 1: U = ulysses_size H = self.num_attention_heads @@ -172,38 +181,54 @@ def __init__( f"Ulysses requires num_attention_heads ({H}) and " f"num_key_value_heads ({H_kv}) divisible by ulysses_size ({U})" ) - # Base class already built `self.attn` as the Ulysses-wrapped path - # (sharded inner backend + UlyssesAttention) for both self-attn and - # cross-attn paths. - - # Build a Ulysses/plain dual-attn pair so set_ulysses_active() can toggle - # at runtime. Built for every sequence-parallel attn: self-attn (audio seq - # not always divisible by ulysses_size; for audio both the Ulysses-inner and - # the plain backend are key_padding_mask-capable via the same backend choice) - # and v2a cross-attn under pure Ulysses (cp_size == 1), where it lets the - # block forward use Ulysses a2a instead of all-gathering the full video K/V. - # Combined ring/attn2d + Ulysses (cp_size > 1) keeps cross-attn on the - # all-gather fallback. The base class already set self.attn to - # UlyssesAttention(inner_backend=sharded_backend). - if ( - enable_sequence_parallel - and (cp_size == 1 or not self._is_cross_attn) - and ulysses_size > 1 - ): - self._ulysses_attn = self.attn - self._plain_attn = create_attention( + + # Whether ``self.attn`` is Ulysses-wrapped; the AV cross-attn dispatch + # reads it to choose seq-sharded K/V (wrapper does the a2a) vs the + # all-gather-full-K/V plain path (ring CP). Tracks the ACTIVE stack: + # set_active_attn() recomputes it, since the {default, stage2} stacks + # can differ in type (e.g. cfg2 x u1 -> stage-2 ulysses). + self.is_ulysses = isinstance(self.attn, UlyssesAttention) + + # {default, stage2} topology stacks. Both are built HERE and swapped whole + # by set_active_attn(); with no stage2 group they alias one instance. The + # stage2 stack gets a FRESH inner backend for heads/stage2-group-size and a + # fresh cp wrapper over the SAME cp group (a shared wrapper would carry the + # wrong head-count metadata and reallocate its shape-keyed buffers across + # CUDA-graphed topologies). + self._attn_default = self.attn + self._attn_stage2 = self._attn_default + if stage2_ulysses_group is not None and enable_sp: + # Inner backend sized like the default stack (local head counts + # divided by the ulysses group size). + s2 = torch_dist.get_world_size(group=stage2_ulysses_group) + H = self.local_num_attention_heads + H_kv = self.local_num_key_value_heads + if H % s2 != 0 or H_kv % s2 != 0: + raise ValueError( + f"stage-2 ulysses requires num_attention_heads ({H}) and " + f"num_key_value_heads ({H_kv}) divisible by the stage-2 " + f"ulysses group size ({s2})" + ) + inner = create_attention( backend=self.attn_backend, layer_idx=self.layer_idx, - num_heads=H, + num_heads=H // s2, head_dim=self.head_dim, - num_kv_heads=H_kv, + num_kv_heads=H_kv // s2, quant_config=self.quant_config, dtype=self.dtype, attention_config=config.attention, attention_metadata_state=config.attention_metadata_state, sparse_params=self.sparse_params, ) - self._has_dual_attn = True + self._attn_stage2 = wrap_parallel_attention( + inner, + visual_gen_mapping=vgm, + enable_sequence_parallel=enable_sp, + use_ulysses=True, + async_ulysses=self._use_async_ulysses, + ulysses_group=stage2_ulysses_group, + ) if apply_gated_attention: self.to_gate_logits = Linear( @@ -219,26 +244,15 @@ def __init__( else: self.to_gate_logits = None - def set_ulysses_active(self, active: bool): - """Toggle between Ulysses-wrapped and plain attention at runtime. + def set_active_attn(self, is_stage2: bool = False) -> None: + """Select the topology stack ``self.attn`` points at. - Effective for modules created with ``enable_sequence_parallel=True`` - (works for both self-attn and cross-attn). No-op otherwise. + Called only via ``LTXModel.set_ulysses_topology``. A no-op when the two + stacks alias one instance (no stage2 group, or cfg_size == 1). """ - if self._has_dual_attn: - self._modules.pop("attn", None) - self.attn = self._ulysses_attn if active else self._plain_attn - - def is_ulysses_active(self) -> bool: - """Whether ``self.attn`` is currently the Ulysses-wrapped path. - - Symmetric with ``set_ulysses_active``. Returns False when no Ulysses - pair was built (e.g. Attention2D mode, ulysses_size==1, or cross-attn - without pure Ulysses), so callers can use it to decide whether to pass - seq-sharded K/V (wrapper handles a2a) or to all-gather K/V into full - sequence first (plain backend). - """ - return self._has_dual_attn and self.attn is self._ulysses_attn + self._modules.pop("attn", None) + self.attn = self._attn_stage2 if is_stage2 else self._attn_default + self.is_ulysses = isinstance(self.attn, UlyssesAttention) def _init_qkv_proj(self): """Override for cross-attention: use _context_dim for K/V input. @@ -350,10 +364,9 @@ def forward( 3. SEPARATE_QKV cross-attn (cached) → split fused kernel. 4. SEPARATE_QKV self-attn (sync fallback) → split fused kernel on x. """ - # Async-Ulysses self-attn dispatch. ``hasattr`` guard: audio_attn1 may - # have ``set_ulysses_active(False)`` swap ``self.attn`` to a plain - # backend that lacks ``forward_async`` — fall through to the sync - # uncached SEPARATE_QKV branch (self-attn on x). + # Async-Ulysses self-attn dispatch. ``hasattr`` guard: a plain-constructed + # ``self.attn`` (e.g. CONDITIONAL audio_attn1) lacks ``forward_async`` — + # fall through to the sync uncached SEPARATE_QKV branch (self-attn on x). if ( self.qkv_mode == QKVMode.SEPARATE_QKV and self._use_async_ulysses @@ -491,6 +504,9 @@ def forward_async( q_src = kv_src = q_input elif ( self._maybe_share_qkv_quantize + # Runtime guard: the two-stage LoRA cache swaps merged Linears to + # BF16 UnquantizedLinearMethod (F.linear rejects Fp4QuantizedTensor). + and not isinstance(self.to_q.quant_method, UnquantizedLinearMethod) and getattr(self.to_q, "input_scale", None) is not None ): x_2d = q_input.reshape(-1, q_input.shape[-1]) @@ -583,6 +599,21 @@ class AudioShardMode(Enum): ).strip().lower() not in ("0", "false", "no") +@dataclass(frozen=True) +class Stage2Groups: + """LTX-2 two-stage dual-topology groups, built by the two-stage pipeline at + load. ``ulysses_group`` feeds the {default, stage2} attention stacks; the + seq fields configure the stage-2 ``SequenceSharder`` (shard index and gather + order follow the cp-major fold layout, which ``dist.new_group``'s rank + sorting does not preserve).""" + + ulysses_group: torch_dist.ProcessGroup + seq_group: torch_dist.ProcessGroup + seq_rank: int + seq_size: int + gather_index: Optional[list[int]] = None + + class BasicAVTransformerBlock(nn.Module): """Dual-stream (Audio/Video) transformer block using TRT-LLM primitives. @@ -598,6 +629,8 @@ def __init__( rope_type: LTXRopeType = LTXRopeType.INTERLEAVED, norm_eps: float = 1e-6, config: Optional["DiffusionModelConfig"] = None, + stage2_ulysses_group: Optional["torch_dist.ProcessGroup"] = None, + stage2_sharder: Optional[SequenceSharder] = None, ): super().__init__() self.idx = idx @@ -608,6 +641,9 @@ def __init__( # is checked once at the root model — skip num_heads here. vgm = config.visual_gen_mapping if config is not None else None self._sharder = SequenceSharder.from_vgm(vgm) + self._sharder_s2 = stage2_sharder if stage2_sharder is not None else self._sharder + self._active_sharder = self._sharder + self._stage2_ulysses_group = stage2_ulysses_group self._audio_conditional_shard = _LTX2_AUDIO_CONDITIONAL_SHARD # Whether to dispatch AdaLN modulation to the fused CUDA kernels. Resolved @@ -633,7 +669,7 @@ def _audio_shard_mode(self) -> AudioShardMode: # (CONDITIONAL) regardless of the parallelism kind — audio ops run on the full # sequence and only v2a slices the audio Q per rank (+ an output all-gather). # _audio_conditional_shard=False opts into the legacy full seq-shard. - if not self._sharder.is_active: + if not self._active_sharder.is_active: return AudioShardMode.NONE return AudioShardMode.CONDITIONAL if self._audio_conditional_shard else AudioShardMode.FULL @@ -666,6 +702,7 @@ def _init_video_modules(self, cfg, rope_type, eps, model_config, idx): module_name=f"transformer_blocks.{idx}.attn1", enable_sequence_parallel=True, async_ulysses=_async_ulysses, + stage2_ulysses_group=self._stage2_ulysses_group, ) self.attn2 = LTX2Attention( query_dim=cfg.dim, @@ -684,18 +721,24 @@ def _init_video_modules(self, cfg, rope_type, eps, model_config, idx): self.scale_shift_table = nn.Parameter(torch.empty(6, cfg.dim)) def _init_audio_modules(self, cfg, rope_type, eps, model_config, idx): - # audio_attn1 needs key_padding_mask (audio is padded to divide ulysses_size; the - # mask zeros pad slots), but TRTLLM self-attn silently drops it — so downgrade to - # VANILLA when Ulysses is active under a TRTLLM backend. + # audio_attn1 needs key_padding_mask (audio is padded to the sharder + # multiple — ulysses_size, or the stage-2 seq size when stage-2 groups + # exist; the mask zeros pad slots), but TRTLLM self-attn silently drops + # it — so downgrade to VANILLA whenever padding is possible under a + # TRTLLM backend. audio_self_config = model_config vgm = model_config.visual_gen_mapping ulysses_size = vgm.ulysses_size if vgm is not None else 1 - if ulysses_size > 1 and model_config.attention.backend == "TRTLLM": + may_pad = ulysses_size > 1 or self._stage2_ulysses_group is not None + if may_pad and model_config.attention.backend == "TRTLLM": audio_self_config = model_config.model_copy( update={ "attention": model_config.attention.model_copy(update={"backend": "VANILLA"}) } ) + # Audio self-attn TYPE is fixed at construction from the AudioShardMode env + # constant: CONDITIONAL (default) replicates audio on every rank and runs + # plain-local; legacy FULL seq-shards audio and needs the Ulysses wrapper. self.audio_attn1 = LTX2Attention( query_dim=cfg.dim, heads=cfg.heads, @@ -707,7 +750,8 @@ def _init_audio_modules(self, cfg, rope_type, eps, model_config, idx): config=audio_self_config, layer_idx=idx, module_name=f"transformer_blocks.{idx}.audio_attn1", - enable_sequence_parallel=True, + enable_sequence_parallel=not _LTX2_AUDIO_CONDITIONAL_SHARD, + stage2_ulysses_group=self._stage2_ulysses_group, ) self.audio_attn2 = LTX2Attention( query_dim=cfg.dim, @@ -752,6 +796,7 @@ def _init_av_cross_modules(self, v_cfg, a_cfg, rope_type, eps, model_config, idx module_name=f"transformer_blocks.{idx}.video_to_audio_attn", enable_sequence_parallel=True, async_ulysses=self._async_ulysses, + stage2_ulysses_group=self._stage2_ulysses_group, ) self.scale_shift_table_a2v_ca_audio = nn.Parameter(torch.empty(5, a_cfg.dim)) self.scale_shift_table_a2v_ca_video = nn.Parameter(torch.empty(5, v_cfg.dim)) @@ -871,7 +916,7 @@ def _get_av_ca_ada_table_ts_pairs( def _sp_all_gather(self, x: torch.Tensor, dim: int = 1) -> torch.Tensor: """All-gather *x* along *dim* across sequence-parallel ranks.""" - return self._sharder.gather(x, dim=dim) + return self._active_sharder.gather(x, dim=dim) # -- Forward ------------------------------------------------------------- @@ -1203,7 +1248,7 @@ def forward( # v2a Ulysses driver by slicing the audio Q (+ its cross-PE / timestep) # to this rank's shard (free view — bit-identical to SHARD mode's # per-rank input), then all-gather the seq-sharded output back to full. - ax_v2a_local = self._sharder.shard(ax_scaled_v2a, dim=1) + ax_v2a_local = self._active_sharder.shard(ax_scaled_v2a, dim=1) # audio cross-PE is full [1, T_a, ...] in REPLICATE (not sharded); # slice it to match the local Q shard. timestep is a broadcast # [B, 1, D] (not per-token) so it is passed through unsliced. @@ -1213,10 +1258,10 @@ def forward( # [B, T, H, D] (seq on dim 1). Shard the seq dim to match ax. pe_dim = 0 if a_cross_pe[0].dim() == 2 else 1 a_cross_pe = ( - self._sharder.shard(a_cross_pe[0], dim=pe_dim), - self._sharder.shard(a_cross_pe[1], dim=pe_dim), + self._active_sharder.shard(a_cross_pe[0], dim=pe_dim), + self._active_sharder.shard(a_cross_pe[1], dim=pe_dim), ) - if self._async_ulysses and self.video_to_audio_attn.is_ulysses_active(): + if self._async_ulysses and self.video_to_audio_attn.is_ulysses: out_local = self.video_to_audio_attn.forward_async( q_input=ax_v2a_local, freqs=a_cross_pe, @@ -1229,12 +1274,12 @@ def forward( vx_scaled_v2a, pe=video.cross_positional_embeddings ) if ( - not self.video_to_audio_attn.is_ulysses_active() - and self._sharder.is_active + not self.video_to_audio_attn.is_ulysses + and self._active_sharder.is_active ): - # Wrapper inactive (e.g. Attention2D): all-gather the - # seq-sharded video K/V to full so the plain backend sees - # the whole sequence. + # No wrapper (ring CP): all-gather the seq-sharded + # video K/V to full so the plain backend sees the + # whole sequence. k_v2a = self._sp_all_gather(k_v2a) v_v2a = self._sp_all_gather(v_v2a) out_local = self.video_to_audio_attn( @@ -1244,7 +1289,7 @@ def forward( timestep=audio.timesteps, ) v2a_attn_raw = self._sp_all_gather(out_local, dim=1) - elif self._async_ulysses and self.video_to_audio_attn.is_ulysses_active(): + elif self._async_ulysses and self.video_to_audio_attn.is_ulysses: # Async-Ulysses v2a: compute Q(audio)/K/V(video) inside the async # driver so the video K/V GEMMs overlap the a2a. RoPE-on-K on the local # shard is value-preserving; no key_padding_mask (video K/V unpadded, @@ -1257,18 +1302,17 @@ def forward( timestep=audio.timesteps, ) else: - # v2a sync: with the Ulysses wrapper active, K/V (video) stay - # seq-sharded and the wrapper does the Q + K|V + output a2a; RoPE-on-K - # in project_kv commutes with the seq-dim a2a (value-preserving). When - # inactive, all-gather so the plain backend sees full K/V — gate on - # is_ulysses_active(), since _audio_shard_mode can be FULL under - # Attention2D where no wrapper exists. + # v2a sync: with a Ulysses wrapper, K/V (video) stay seq-sharded + # and the wrapper does the Q + K|V + output a2a; RoPE-on-K in + # project_kv commutes with the seq-dim a2a (value-preserving). + # Without a wrapper (ring CP), all-gather so the plain backend + # sees full K/V. k_v2a, v_v2a = self.video_to_audio_attn.project_kv( vx_scaled_v2a, pe=video.cross_positional_embeddings ) - if not self.video_to_audio_attn.is_ulysses_active() and self._sharder.is_active: - # Fallback: wrapper inactive → all-gather sharded video - # K/V to full so plain backend can run. + if not self.video_to_audio_attn.is_ulysses and self._active_sharder.is_active: + # No wrapper (ring CP): all-gather sharded video K/V + # to full so the plain backend can run. k_v2a = self._sp_all_gather(k_v2a) v_v2a = self._sp_all_gather(v_v2a) @@ -1498,6 +1542,7 @@ def __init__( double_precision_rope: bool = False, apply_gated_attention: bool = False, model_config: Optional["DiffusionModelConfig"] = None, + stage2_groups: Optional[Stage2Groups] = None, ): from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig @@ -1556,10 +1601,24 @@ def __init__( vgm, num_attention_heads=num_attention_heads if model_type.is_video_enabled() else None, ) + self._stage2_groups = stage2_groups + self._has_stage2 = stage2_groups is not None + if self._has_stage2: + self._sharder_s2 = SequenceSharder( + size=stage2_groups.seq_size, + rank=stage2_groups.seq_rank, + group=stage2_groups.seq_group, + gather_index=stage2_groups.gather_index, + ) + else: + self._sharder_s2 = self._sharder + self._active_sharder = self._sharder + self._active_seq_size = self._active_sharder.size + self._active_topology = "default" self._cp_size = vgm.cp_size if vgm is not None else 1 self._ulysses_size = vgm.ulysses_size if vgm is not None else 1 if ( - self._sharder.is_active + self._active_sharder.is_active and vgm is not None and vgm.ulysses_size > 1 and model_type.is_audio_enabled() @@ -1856,6 +1915,10 @@ def _init_transformer_blocks( rope_type=self.rope_type, norm_eps=norm_eps, config=self.model_config, + stage2_ulysses_group=( + self._stage2_groups.ulysses_group if self._has_stage2 else None + ), + stage2_sharder=self._sharder_s2 if self._has_stage2 else None, ) for idx in range(num_layers) ] @@ -1877,7 +1940,7 @@ def _shard_transformer_args(self, args: TransformerArgs) -> TransformerArgs: fields (``x``, timesteps, etc.) need slicing each step. """ seq_len = args.x.shape[1] - sh = self._sharder + sh = self._active_sharder return replace( args, x=sh.shard(args.x, dim=1), @@ -1909,7 +1972,7 @@ def _make_pe_local( if pe is None: return None cos, sin = pe - sh = self._sharder + sh = self._active_sharder if sh.is_active and (not is_audio or self._audio_shard_mode == AudioShardMode.FULL): chunk = cos.shape[1] // sh.size s = sh.rank * chunk @@ -1928,7 +1991,7 @@ def _make_pe_local( def _gather_sequence(self, x: torch.Tensor) -> torch.Tensor: """All-gather hidden states along the sequence dim.""" - return self._sharder.gather(x, dim=1) + return self._active_sharder.gather(x, dim=1) @staticmethod def _pad_pe( @@ -2010,74 +2073,59 @@ def _audio_shard_mode(self) -> AudioShardMode: # (CONDITIONAL) regardless of the parallelism kind — audio ops run on the full # sequence and only v2a slices the audio Q per rank (+ an output all-gather). # _audio_conditional_shard=False opts into the legacy full seq-shard. - if not self._sharder.is_active: + if not self._active_sharder.is_active: return AudioShardMode.NONE return AudioShardMode.CONDITIONAL if self._audio_conditional_shard else AudioShardMode.FULL + @property + def active_topology(self) -> str: + """``"stage2"`` iff the stage-2 sharder/attention stacks are active, + else ``"default"``. Values match the two-stage pipeline's graph-key + topology strings.""" + return self._active_topology + + def set_ulysses_topology(self, is_stage2: bool = False) -> None: + """Switch every topology-bound piece between the default and stage-2 + layouts: the active sharder on the model and every block, the + {default, stage2} attention stacks, and ``active_topology`` (which the + two-stage CUDA-graph key reads, so the switch moves the key with the + stacks atomically). Called only by the two-stage pipeline at the + Stage-2 boundary (before ``prepare_text_cache``, restored in a + finally block). + """ + # Fail fast: a silent no-op here would let cfg>1 Stage 2 run duplicated + # in the default topology (correct output, the perf win silently lost). + assert self._has_stage2, "set_ulysses_topology on a transformer built without stage2 groups" + self._active_sharder = self._sharder_s2 if is_stage2 else self._sharder + self._active_seq_size = self._active_sharder.size + self._active_topology = "stage2" if is_stage2 else "default" + for block in self.transformer_blocks: + target = block.inner if isinstance(block, LTX2CacheDiTPattern0BlockWrapper) else block + target._active_sharder = target._sharder_s2 if is_stage2 else target._sharder + for name in ("attn1", "video_to_audio_attn", "audio_attn1"): + mod = getattr(target, name, None) + if mod is not None: + # No-op for modules whose two stacks alias one instance. + mod.set_active_attn(is_stage2=is_stage2) + def configure_audio_ulysses(self, audio_seq_len: int) -> None: - """Configure audio sharding + padding for Ulysses. + """Compute the audio pad for sequence parallelism. Call once before the denoising loop when the audio token count is - known. The decision is cached — ``forward()`` uses it without - re-checking. - - When sequence parallelism is active, audio is always padded to - ``(U - T_a % U) % U`` slots so ``T_a`` becomes divisible by ``U`` - and a ``[B, T_a_padded]`` validity mask is attached so attention - zeros out pad positions. ``forward`` strips the pad tail on exit. + known. Audio is padded to a multiple of the sharder size in BOTH + audio modes — FULL shards the sequence; CONDITIONAL needs it so + v2a's per-rank Q slice is even. A ``[B, T_a_padded]`` validity mask + zeros out pad positions; ``forward`` strips the pad tail on exit. """ - if not self._sharder.is_active: + if not self._active_sharder.is_active and not self._has_stage2: self._audio_pad = 0 return - U = self._sharder.size - # Pad audio to a multiple of U in BOTH modes: SHARD needs it to split the - # sequence; REPLICATE needs it so v2a's per-rank slice is even. + # Single pad: the stage-2 seq size is a multiple of every group size the + # audio touches in either topology, so one pad serves both (no repad at + # the stage boundary; audio shapes stay constant across both graphs). + U = self._sharder_s2.size if self._has_stage2 else self._active_sharder.size self._audio_pad = (U - audio_seq_len % U) % U - # REPLICATE: audio stays full (not seq-sharded) so its latency-bound self/ - # cross-attn + FFN run local without A2A; only v2a keeps head-sharding, via a - # per-rank slice of the replicated Q + an output all-gather (block forward). - for block in self.transformer_blocks: - target = block.inner if isinstance(block, LTX2CacheDiTPattern0BlockWrapper) else block - target._audio_conditional_shard = self._audio_conditional_shard - if hasattr(target, "audio_attn1"): - # audio_self is Ulysses in SHARD, local in REPLICATE (no A2A). - target.audio_attn1.set_ulysses_active(not self._audio_conditional_shard) - # v2a needs the Ulysses wrapper in BOTH SHARD and REPLICATE (video K/V - # A2A + head-split); the block forward routes REPLICATE through a - # slice-in / gather-out around the unchanged driver. - if hasattr(target, "video_to_audio_attn"): - target.video_to_audio_attn.set_ulysses_active(True) - - def set_ulysses_enabled(self, enabled: bool) -> None: - """Enable or disable Ulysses parallelism at runtime. - - Call with ``False`` before running the transformer on a single - rank (e.g. Stage 2 of the two-stage pipeline where non-primary - workers have already exited). Call with ``True`` to restore - multi-rank operation; audio sharding will be reconfigured by - the next :meth:`configure_audio_ulysses` call. - """ - if self._sharder.size <= 1: - return - - if enabled: - self._sharder.enable() - else: - self._sharder.disable() - - for block in self.transformer_blocks: - target = block.inner if isinstance(block, LTX2CacheDiTPattern0BlockWrapper) else block - if enabled: - target._sharder.enable() - else: - target._sharder.disable() - if hasattr(target, "attn1"): - target.attn1.set_ulysses_active(enabled) - if hasattr(target, "audio_attn1") and not enabled: - target.audio_attn1.set_ulysses_active(False) - if hasattr(target, "video_to_audio_attn") and not enabled: - target.video_to_audio_attn.set_ulysses_active(False) # -- Output processing --------------------------------------------------- @@ -2198,6 +2246,10 @@ def forward( Returns: Tuple of (video_output, audio_output) velocity predictions. """ + # Topology guard for torch.compile: a plain int read makes dynamo install + # a value guard, so the default and stage-2 topologies never share a + # compiled artifact (belt to the CUDA-graph key's suspenders). + torch._check(self._active_seq_size >= 1) if not self.model_type.is_video_enabled() and video is not None: raise ValueError("Video is not enabled for this model") if not self.model_type.is_audio_enabled() and audio is not None: @@ -2253,7 +2305,7 @@ def forward( # Shard sequences for parallelism (Ulysses head-sharding, ring CP, or Attention2D). # Video is always sharded. Audio sharding follows self._audio_shard_mode. - if self._sharder.is_active: + if self._active_sharder.is_active: if video_args is not None: video_args = self._shard_transformer_args(video_args) if self._audio_shard_mode == AudioShardMode.FULL and audio_args is not None: @@ -2301,7 +2353,7 @@ def forward( # Only gather embedded_timestep if it was actually sharded (dim-1 # matches x); scalar timestep embeddings [B, 1, D] are # broadcast-compatible and must not be gathered. - if self._sharder.is_active: + if self._active_sharder.is_active: if video_args is not None: gathered_vx = self._gather_sequence(video_args.x) v_et = video_args.embedded_timestep diff --git a/tensorrt_llm/_torch/visual_gen/models/qwen_image/__init__.py b/tensorrt_llm/_torch/visual_gen/models/qwen_image/__init__.py index 423ae1beb388..a536c398d89a 100644 --- a/tensorrt_llm/_torch/visual_gen/models/qwen_image/__init__.py +++ b/tensorrt_llm/_torch/visual_gen/models/qwen_image/__init__.py @@ -9,6 +9,7 @@ """ from .pipeline_qwen_image import QwenImagePipeline +from .pipeline_qwen_image_edit import QwenImageEditPlusPipeline from .transformer_qwen_image import ( AdaLayerNormContinuous, QwenEmbedRope, @@ -25,6 +26,7 @@ __all__ = [ "AdaLayerNormContinuous", "QwenEmbedRope", + "QwenImageEditPlusPipeline", "QwenImagePipeline", "QwenImageTransformer2DModel", "QwenImageTransformerBlock", diff --git a/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image.py b/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image.py index 1290453c57a0..c61128c62bb5 100644 --- a/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image.py +++ b/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image.py @@ -17,6 +17,8 @@ import numpy as np import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup from tensorrt_llm._torch.visual_gen.output import CudaPhaseTimer, PipelineOutput from tensorrt_llm._torch.visual_gen.pipeline import BasePipeline @@ -84,6 +86,7 @@ def __init__(self, pipeline_config): # load_standard_components() once the VAE is loaded. self.vae_scale_factor = 8 self.tokenizer_max_length = 1024 + self._logged_cfg_disabled_parallel_warning = False @property def dtype(self): @@ -154,13 +157,16 @@ def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> N ``forward`` path so CUDA graphs / torch.compile / VAE kernels all get triggered with the runtime shape. """ + vgm = self.pipeline_config.visual_gen_mapping + cfg_size = vgm.cfg_size if vgm else 1 + warmup_cfg_scale = 4.0 if cfg_size > 1 else 1.0 with torch.no_grad(): self.forward( prompt="warmup", height=height, width=width, num_inference_steps=max(steps, 2), - true_cfg_scale=1.0, + negative_prompt_cfg_scale=warmup_cfg_scale, seed=42, max_sequence_length=64, ) @@ -251,12 +257,12 @@ def _encode_prompt( prompt: List[str], device: torch.device, max_sequence_length: int, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: """Encode a list of prompts via Qwen2.5-VL + chat template. Returns: prompt_embeds: ``(B, S, 3584)`` in transformer dtype. - prompt_embeds_mask: ``(B, S)`` bool mask. + prompt_embeds_mask: Optional ``(B, S)`` mask. ``None`` means all tokens are valid. """ drop_idx = _PROMPT_TEMPLATE_START_IDX txt = [_PROMPT_TEMPLATE.format(e) for e in prompt] @@ -292,6 +298,8 @@ def _encode_prompt( prompt_embeds = prompt_embeds[:, :max_sequence_length] prompt_embeds_mask = prompt_embeds_mask[:, :max_sequence_length] prompt_embeds = prompt_embeds.to(dtype=self.dtype, device=device) + if prompt_embeds_mask.bool().all(): + return prompt_embeds, None return prompt_embeds, prompt_embeds_mask # ------------------------------------------------------------------ @@ -369,6 +377,78 @@ def _decode_latents(self, latents: torch.Tensor, height: int, width: int) -> tor def default_generation_params(self) -> dict: return dict(_DEFAULT_GENERATION_PARAMS) + @staticmethod + def _normalize_negative_prompt( + negative_prompt: Optional[Union[str, List[str]]], batch_size: int + ) -> List[str]: + if negative_prompt is None: + return [""] * batch_size + if isinstance(negative_prompt, str): + return [negative_prompt] * batch_size + + negative_prompt = list(negative_prompt) + if len(negative_prompt) == 1 and batch_size != 1: + return negative_prompt * batch_size + if len(negative_prompt) != batch_size: + raise ValueError( + "negative_prompt must be a string, a singleton list, " + "or a list with the same length as prompt" + ) + return negative_prompt + + @staticmethod + def _select_cfg_inputs( + cfg_rank: int, + prompt_embeds: torch.Tensor, + prompt_embeds_mask: torch.Tensor, + neg_prompt_embeds: torch.Tensor, + neg_prompt_embeds_mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + if cfg_rank == 0: + return prompt_embeds, prompt_embeds_mask + return neg_prompt_embeds, neg_prompt_embeds_mask + + @staticmethod + def _combine_negative_prompt_cfg( + noise_pred: torch.Tensor, neg_noise_pred: torch.Tensor, cfg_scale: float + ) -> torch.Tensor: + comb = neg_noise_pred + cfg_scale * (noise_pred - neg_noise_pred) + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb, dim=-1, keepdim=True) + return comb * (cond_norm / noise_norm) + + def _cfg_parallel_state( + self, use_negative_prompt_cfg: bool + ) -> Tuple[bool, int, int, Optional[ProcessGroup]]: + vgm = self.pipeline_config.visual_gen_mapping + cfg_size = vgm.cfg_size if vgm else 1 + cfg_rank = vgm.cfg_rank if vgm else 0 + cfg_pg = vgm.cfg_group if vgm else None + do_cfg_parallel = use_negative_prompt_cfg and cfg_size > 1 + if ( + cfg_size > 1 + and not use_negative_prompt_cfg + and cfg_rank == 0 + and not getattr(self, "_logged_cfg_disabled_parallel_warning", False) + ): + logger.warning( + "Qwen-Image configured with cfg_size=%d but negative-prompt CFG is disabled; " + "CFG-parallel ranks will redundantly compute the same request path.", + cfg_size, + ) + self._logged_cfg_disabled_parallel_warning = True + if do_cfg_parallel: + if cfg_size != 2: + raise ValueError( + f"Qwen-Image CFG parallel only supports cfg_size=2 " + f"(cond/uncond), got cfg_size={cfg_size}" + ) + if not dist.is_initialized(): + raise RuntimeError("Qwen-Image CFG parallel requires torch.distributed") + if cfg_rank == 0: + logger.info("CFG parallel: cfg_size=2") + return do_cfg_parallel, cfg_size, cfg_rank, cfg_pg + def infer(self, req): # Fan out by num_images_per_prompt so ``n > 1`` produces multiple # images in a single batched forward. Qwen-Image supports this @@ -395,7 +475,7 @@ def infer(self, req): height=params.height, width=params.width, num_inference_steps=params.num_inference_steps, - true_cfg_scale=params.guidance_scale, + negative_prompt_cfg_scale=params.guidance_scale, seed=params.seed, max_sequence_length=params.max_sequence_length, ) @@ -408,7 +488,7 @@ def forward( height: int = 1328, width: int = 1328, num_inference_steps: int = 50, - true_cfg_scale: float = 4.0, + negative_prompt_cfg_scale: float = 4.0, seed: int = 42, max_sequence_length: int = 1024, sigmas: Optional[list] = None, @@ -416,7 +496,7 @@ def forward( """Text-to-image generation. Implementation mirrors ``diffusers.QwenImagePipeline.__call__`` - with the FlowMatchEuler sampler and real CFG (``true_cfg_scale``). + with the FlowMatchEuler sampler and negative-prompt CFG. """ pipeline_start = time.time() timer = CudaPhaseTimer() @@ -426,8 +506,10 @@ def forward( prompt = [prompt] batch_size = len(prompt) - has_neg = negative_prompt is not None - do_true_cfg = true_cfg_scale > 1.0 and has_neg + use_negative_prompt_cfg = negative_prompt_cfg_scale > 1.0 + do_cfg_parallel, cfg_size, cfg_rank, cfg_pg = self._cfg_parallel_state( + use_negative_prompt_cfg + ) device = self.device generator = torch.Generator(device=device).manual_seed(seed) @@ -436,9 +518,8 @@ def forward( logger.info("Encoding prompt...") prompt_embeds, prompt_embeds_mask = self._encode_prompt(prompt, device, max_sequence_length) neg_prompt_embeds = neg_prompt_embeds_mask = None - if do_true_cfg: - if isinstance(negative_prompt, str): - negative_prompt = [negative_prompt] * batch_size + if use_negative_prompt_cfg: + negative_prompt = self._normalize_negative_prompt(negative_prompt, batch_size) neg_prompt_embeds, neg_prompt_embeds_mask = self._encode_prompt( negative_prompt, device, max_sequence_length ) @@ -486,18 +567,46 @@ def forward( # Denoise loop. timer.mark_denoise_start() logger.info("Denoising (%d steps)...", len(timesteps)) + cuda_graph_enabled = self.pipeline_config.cuda_graph.enable for i, t in enumerate(timesteps): timestep = t.expand(latents.shape[0]).to(latents.dtype) - noise_pred = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - encoder_hidden_states_mask=prompt_embeds_mask, - encoder_hidden_states=prompt_embeds, - img_shapes=img_shapes, - return_dict=False, - )[0] - - if do_true_cfg: + if do_cfg_parallel: + local_embeds, local_mask = self._select_cfg_inputs( + cfg_rank, + prompt_embeds, + prompt_embeds_mask, + neg_prompt_embeds, + neg_prompt_embeds_mask, + ) + noise_pred_local = self.transformer( + hidden_states=latents, + timestep=timestep / 1000, + encoder_hidden_states_mask=local_mask, + encoder_hidden_states=local_embeds, + img_shapes=img_shapes, + return_dict=False, + )[0].contiguous() + gather_list = [torch.empty_like(noise_pred_local) for _ in range(cfg_size)] + dist.all_gather(gather_list, noise_pred_local, group=cfg_pg) + noise_pred = self._combine_negative_prompt_cfg( + gather_list[0], gather_list[1], negative_prompt_cfg_scale + ) + else: + noise_pred = self.transformer( + hidden_states=latents, + timestep=timestep / 1000, + encoder_hidden_states_mask=prompt_embeds_mask, + encoder_hidden_states=prompt_embeds, + img_shapes=img_shapes, + return_dict=False, + )[0] + + if use_negative_prompt_cfg and not do_cfg_parallel: + if cuda_graph_enabled: + # CUDA graph outputs are graph-owned buffers; the negative CFG + # replay may reuse the same pool before guidance consumes this one. + noise_pred = noise_pred.clone() + neg_noise_pred = self.transformer( hidden_states=latents, timestep=timestep / 1000, @@ -506,10 +615,9 @@ def forward( img_shapes=img_shapes, return_dict=False, )[0] - comb = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - noise_norm = torch.norm(comb, dim=-1, keepdim=True) - noise_pred = comb * (cond_norm / noise_norm) + noise_pred = self._combine_negative_prompt_cfg( + noise_pred, neg_noise_pred, negative_prompt_cfg_scale + ) latents_dtype = latents.dtype latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] diff --git a/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image_edit.py b/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image_edit.py new file mode 100644 index 000000000000..daffe567a681 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image_edit.py @@ -0,0 +1,570 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Qwen-Image-Edit pipeline. + +The edit checkpoint uses the existing Qwen-Image transformer architecture +with extra image-conditioning paths from Diffusers +``QwenImageEditPlusPipeline``. +""" + +import math +import time +from io import BytesIO +from typing import Any + +import numpy as np +import torch +import torch.distributed as dist + +from tensorrt_llm._torch.visual_gen.output import CudaPhaseTimer, PipelineOutput +from tensorrt_llm._torch.visual_gen.pipeline_registry import register_pipeline +from tensorrt_llm.inputs.utils import load_image +from tensorrt_llm.logger import logger + +from .pipeline_qwen_image import QwenImagePipeline, _calculate_shift + +_EDIT_PROMPT_TEMPLATE = ( + "<|im_start|>system\nDescribe the key features of the input image " + "(color, shape, size, texture, objects, background), then explain how " + "the user's text instruction should alter or modify the image. Generate " + "a new image that meets the user's requirements while maintaining " + "consistency with the original input where appropriate.<|im_end|>\n" + "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" +) +_EDIT_PROMPT_TEMPLATE_START_IDX = 64 +_CONDITION_IMAGE_SIZE = 384 * 384 +_VAE_IMAGE_SIZE = 1024 * 1024 + + +def _calculate_dimensions(target_area: int, ratio: float) -> tuple[int, int]: + """Match Diffusers Qwen-Image-Edit area-preserving 32px rounding.""" + width = math.sqrt(target_area * ratio) + height = width / ratio + width = round(width / 32) * 32 + height = round(height / 32) * 32 + return width, height + + +def _retrieve_latents( + encoder_output: torch.Tensor, + generator: torch.Generator | None = None, + sample_mode: str = "sample", +) -> torch.Tensor: + """Return latent tensor from a Diffusers VAE encoder output.""" + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + if hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + if hasattr(encoder_output, "latents"): + return encoder_output.latents + raise AttributeError("Could not access latents of provided encoder_output") + + +_EDIT_DEFAULT_GENERATION_PARAMS = { + "height": None, + "width": None, + "num_inference_steps": 50, + "guidance_scale": 4.0, + "max_sequence_length": 512, + "negative_prompt": " ", +} + + +@register_pipeline( + "QwenImageEditPlusPipeline", + hf_ids=["Qwen/Qwen-Image-Edit-2511"], + doc="Qwen-Image-Edit image editing pipeline.", +) +class QwenImageEditPlusPipeline(QwenImagePipeline): + """Qwen-Image-Edit pipeline using the existing Qwen-Image transformer. + + Diffusers implements this model by feeding the input image through two + paths: Qwen2-VL sees resized condition images during text encoding, while + the Qwen-Image VAE encodes appearance latents that are appended after the + generated latent tokens. The transformer predicts every token in the + concatenated sequence, and the scheduler only steps the generated prefix. + """ + + DEFAULT_GENERATION_PARAMS = _EDIT_DEFAULT_GENERATION_PARAMS + + def load_standard_components( + self, + checkpoint_dir: str, + device: torch.device, + skip_components: list | None = None, + ) -> None: + super().load_standard_components(checkpoint_dir, device, skip_components) + skip_components = skip_components or [] + + if "processor" not in skip_components: + try: + from diffusers.image_processor import VaeImageProcessor + from transformers import Qwen2VLProcessor + except ImportError as e: # pragma: no cover + raise ImportError( + "Qwen-Image-Edit requires diffusers.VaeImageProcessor " + "and transformers.Qwen2VLProcessor." + ) from e + + logger.info("Loading Qwen2-VL processor...") + try: + self.processor = Qwen2VLProcessor.from_pretrained( + checkpoint_dir, + subfolder="processor", + ) + except (OSError, ValueError): + self.processor = Qwen2VLProcessor.from_pretrained(checkpoint_dir) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + + self.latent_channels = self.vae.config.z_dim + self.default_height = None + self.default_width = None + self.max_sequence_length = 512 + + @property + def default_generation_params(self) -> dict: + return dict(_EDIT_DEFAULT_GENERATION_PARAMS) + + @property + def default_warmup_resolutions(self) -> list[tuple[int, int]]: + return [(1024, 1024)] + + def warmup_cache_key(self, height: int | None, width: int | None, **kwargs) -> tuple: + return (height, width) + + def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> None: + from PIL import Image + + with torch.no_grad(): + self.forward( + image=Image.new("RGB", (width, height), color=(127, 127, 127)), + prompt="warmup", + height=height, + width=width, + num_inference_steps=max(steps, 2), + true_cfg_scale=1.0, + seed=42, + max_sequence_length=64, + ) + + @staticmethod + def _load_edit_images(image: Any) -> list[Any]: + if image is None: + raise ValueError("Qwen-Image-Edit requires params.image.") + images = image if isinstance(image, list) else [image] + pil_images = [] + for item in images: + if isinstance(item, bytes): + from PIL import Image + + pil_images.append(Image.open(BytesIO(item)).convert("RGB")) + else: + pil_images.append(load_image(item, format="pil")) + return pil_images + + def _preprocess_edit_images( + self, + pil_images: list[Any], + ) -> tuple[list[Any], list[torch.Tensor], list[tuple[int, int]]]: + condition_images = [] + vae_images = [] + vae_image_sizes = [] + for img in pil_images: + image_width, image_height = img.size + ratio = image_width / image_height + condition_width, condition_height = _calculate_dimensions(_CONDITION_IMAGE_SIZE, ratio) + vae_width, vae_height = _calculate_dimensions(_VAE_IMAGE_SIZE, ratio) + condition_images.append( + self.image_processor.resize(img, condition_height, condition_width) + ) + vae_images.append( + self.image_processor.preprocess(img, vae_height, vae_width).unsqueeze(2) + ) + vae_image_sizes.append((vae_width, vae_height)) + return condition_images, vae_images, vae_image_sizes + + def _get_qwen_edit_prompt_embeds( + self, + prompt: list[str], + image: list[Any] | None, + device: torch.device, + dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + img_prompt_template = "Picture {}: <|vision_start|><|image_pad|><|vision_end|>" + if isinstance(image, list): + base_img_prompt = "".join( + img_prompt_template.format(i + 1) for i, _ in enumerate(image) + ) + elif image is not None: + base_img_prompt = img_prompt_template.format(1) + else: + base_img_prompt = "" + + txt = [_EDIT_PROMPT_TEMPLATE.format(base_img_prompt + p) for p in prompt] + model_inputs = self.processor( + text=txt, + images=image, + padding=True, + return_tensors="pt", + ).to(device) + + encoder_outputs = self.text_encoder( + input_ids=model_inputs["input_ids"], + attention_mask=model_inputs["attention_mask"], + pixel_values=model_inputs.get("pixel_values"), + image_grid_thw=model_inputs.get("image_grid_thw"), + output_hidden_states=True, + ) + hidden_states = encoder_outputs.hidden_states[-1] + + split_hidden = self._extract_masked_hidden(hidden_states, model_inputs["attention_mask"]) + split_hidden = [h[_EDIT_PROMPT_TEMPLATE_START_IDX:] for h in split_hidden] + attn_masks = [ + torch.ones(h.size(0), dtype=torch.long, device=h.device) for h in split_hidden + ] + max_len = max(h.size(0) for h in split_hidden) + prompt_embeds = torch.stack( + [torch.cat([h, h.new_zeros(max_len - h.size(0), h.size(1))]) for h in split_hidden] + ) + prompt_embeds_mask = torch.stack( + [torch.cat([m, m.new_zeros(max_len - m.size(0))]) for m in attn_masks] + ) + return prompt_embeds.to(dtype=dtype, device=device), prompt_embeds_mask + + def _encode_edit_prompt( + self, + prompt: list[str], + image: list[Any], + device: torch.device, + max_sequence_length: int, + num_images_per_prompt: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + batch_size = len(prompt) + prompt_embeds, prompt_embeds_mask = self._get_qwen_edit_prompt_embeds( + prompt, + image, + device, + self.dtype, + ) + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1) + prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len) + + prompt_embeds = prompt_embeds[:, :max_sequence_length] + prompt_embeds_mask = prompt_embeds_mask[:, :max_sequence_length] + if bool(prompt_embeds_mask.all()): + prompt_embeds_mask = None + return prompt_embeds, prompt_embeds_mask + + def _encode_vae_image( + self, + image: torch.Tensor, + generator: torch.Generator | None, + ) -> torch.Tensor: + image_latents = _retrieve_latents( + self.vae.encode(image), + generator=generator, + sample_mode="argmax", + ) + latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, self.latent_channels, 1, 1, 1) + .to(image_latents.device, image_latents.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std) + .view(1, self.latent_channels, 1, 1, 1) + .to(image_latents.device, image_latents.dtype) + ) + return (image_latents - latents_mean) / latents_std + + def _prepare_edit_latents( + self, + images: list[torch.Tensor], + batch_size: int, + num_channels_latents: int, + height: int, + width: int, + dtype: torch.dtype, + device: torch.device, + generator: torch.Generator | None, + ) -> tuple[torch.Tensor, torch.Tensor]: + latents = self._prepare_latents( + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + ) + + all_image_latents = [] + for image in images: + image = image.to(device=device, dtype=dtype) + if image.shape[1] != self.latent_channels: + image_latents = self._encode_vae_image(image=image, generator=generator) + else: + image_latents = image + + if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0: + repeat = batch_size // image_latents.shape[0] + image_latents = torch.cat([image_latents] * repeat, dim=0) + elif batch_size > image_latents.shape[0]: + raise ValueError( + "Cannot duplicate image batch size " + f"{image_latents.shape[0]} to {batch_size} prompts." + ) + else: + image_latents = torch.cat([image_latents], dim=0) + + image_latent_height, image_latent_width = image_latents.shape[3:] + image_latents = self._pack_latents( + image_latents, + batch_size, + num_channels_latents, + image_latent_height, + image_latent_width, + ) + all_image_latents.append(image_latents) + return latents, torch.cat(all_image_latents, dim=1) + + def infer(self, req: Any) -> PipelineOutput: + params = req.params + num_per = params.num_images_per_prompt or 1 + if num_per > 1: + raise ValueError( + "QwenImageEditPlusPipeline currently supports num_images_per_prompt=1 only." + ) + prompts = req.prompt if isinstance(req.prompt, list) else [req.prompt] + pil_images = self._load_edit_images(params.image) + height = params.height + width = params.width + if height is None or width is None: + source_width, source_height = pil_images[-1].size + inferred_width, inferred_height = _calculate_dimensions( + _VAE_IMAGE_SIZE, + source_width / source_height, + ) + height = height or inferred_height + width = width or inferred_width + + return self.forward( + image=pil_images, + prompt=prompts, + negative_prompt=params.negative_prompt, + height=height, + width=width, + num_inference_steps=params.num_inference_steps, + true_cfg_scale=params.guidance_scale, + seed=params.seed, + max_sequence_length=params.max_sequence_length, + ) + + @torch.inference_mode() + def forward( + self, + image: Any, + prompt: str | list[str], + negative_prompt: str | list[str] | None = " ", + height: int | None = None, + width: int | None = None, + num_inference_steps: int = 50, + true_cfg_scale: float = 4.0, + seed: int = 42, + max_sequence_length: int = 512, + sigmas: list | None = None, + ) -> PipelineOutput: + """Image editing generation matching Diffusers QwenImageEditPlusPipeline.""" + pipeline_start = time.time() + timer = CudaPhaseTimer() + timer.mark_pre_start() + + pil_images = self._load_edit_images(image) + source_size = pil_images[-1].size + calculated_width, calculated_height = _calculate_dimensions( + _VAE_IMAGE_SIZE, + source_size[0] / source_size[1], + ) + height = height or calculated_height + width = width or calculated_width + multiple_of = self.vae_scale_factor * 2 + width = width // multiple_of * multiple_of + height = height // multiple_of * multiple_of + + if isinstance(prompt, str): + prompt = [prompt] + batch_size = len(prompt) + if batch_size > 1: + raise ValueError("QwenImageEditPlusPipeline currently supports one prompt at a time.") + + condition_images, vae_images, vae_image_sizes = self._preprocess_edit_images(pil_images) + device = self.device + generator = torch.Generator(device=device).manual_seed(seed) + + has_neg = negative_prompt is not None + do_true_cfg = true_cfg_scale > 1.0 and has_neg + vgm = self.pipeline_config.visual_gen_mapping + cfg_size = vgm.cfg_size if vgm else 1 + cfg_rank = vgm.cfg_rank if vgm else 0 + cfg_pg = vgm.cfg_group if vgm else None + do_cfg_parallel = do_true_cfg and cfg_size > 1 + if do_cfg_parallel: + if cfg_size != 2: + raise ValueError( + f"Qwen-Image-Edit CFG parallel requires cfg_size=2, got {cfg_size}" + ) + if not dist.is_available() or not dist.is_initialized(): + raise RuntimeError( + "Qwen-Image-Edit CFG parallel requires initialized torch.distributed" + ) + if getattr(self, "rank", 0) == 0: + logger.info("Qwen-Image-Edit CFG parallel enabled: cfg_size=2") + + logger.info("Encoding edit prompt...") + prompt_embeds, prompt_embeds_mask = self._encode_edit_prompt( + prompt, + condition_images, + device, + max_sequence_length, + num_images_per_prompt=1, + ) + neg_prompt_embeds = neg_prompt_embeds_mask = None + if do_true_cfg: + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] * batch_size + neg_prompt_embeds, neg_prompt_embeds_mask = self._encode_edit_prompt( + negative_prompt, + condition_images, + device, + max_sequence_length, + num_images_per_prompt=1, + ) + + num_channels_latents = self.transformer.in_channels // 4 + latents, image_latents = self._prepare_edit_latents( + vae_images, + batch_size, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + ) + img_shapes = [ + [ + (1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2), + *[ + ( + 1, + vae_height // self.vae_scale_factor // 2, + vae_width // self.vae_scale_factor // 2, + ) + for vae_width, vae_height in vae_image_sizes + ], + ] + ] * batch_size + + sigmas_np = ( + sigmas + if sigmas is not None + else np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) + ) + image_seq_len = latents.shape[1] + mu = _calculate_shift( + image_seq_len, + self.scheduler.config.get("base_image_seq_len", 256), + self.scheduler.config.get("max_image_seq_len", 4096), + self.scheduler.config.get("base_shift", 0.5), + self.scheduler.config.get("max_shift", 1.15), + ) + self.scheduler.set_timesteps(sigmas=sigmas_np, device=device, mu=mu) + timesteps = self.scheduler.timesteps + self.scheduler.set_begin_index(0) + + timer.mark_denoise_start() + logger.info("Denoising edit (%d steps)...", len(timesteps)) + for t in timesteps: + latent_model_input = torch.cat([latents, image_latents], dim=1) + timestep = t.expand(latents.shape[0]).to(latents.dtype) + + if do_cfg_parallel: + if cfg_rank == 0: + local_noise_pred = self.transformer( + hidden_states=latent_model_input, + timestep=timestep / 1000, + encoder_hidden_states_mask=prompt_embeds_mask, + encoder_hidden_states=prompt_embeds, + img_shapes=img_shapes, + return_dict=False, + )[0] + else: + local_noise_pred = self.transformer( + hidden_states=latent_model_input, + timestep=timestep / 1000, + encoder_hidden_states_mask=neg_prompt_embeds_mask, + encoder_hidden_states=neg_prompt_embeds, + img_shapes=img_shapes, + return_dict=False, + )[0] + local_noise_pred = local_noise_pred[:, : latents.size(1)].contiguous() + gathered_noise_pred = [torch.empty_like(local_noise_pred) for _ in range(cfg_size)] + dist.all_gather(gathered_noise_pred, local_noise_pred, group=cfg_pg) + noise_pred = gathered_noise_pred[0] + neg_noise_pred = gathered_noise_pred[1] + comb = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb, dim=-1, keepdim=True) + noise_pred = comb * (cond_norm / noise_norm) + else: + noise_pred = self.transformer( + hidden_states=latent_model_input, + timestep=timestep / 1000, + encoder_hidden_states_mask=prompt_embeds_mask, + encoder_hidden_states=prompt_embeds, + img_shapes=img_shapes, + return_dict=False, + )[0] + noise_pred = noise_pred[:, : latents.size(1)] + + if do_true_cfg: + pipeline_config = getattr(self, "pipeline_config", None) + cuda_graph_enabled = getattr( + getattr(pipeline_config, "cuda_graph", None), "enable", False + ) + if cuda_graph_enabled: + # CUDA graph outputs are graph-owned buffers; the negative CFG + # replay may overwrite this positive prediction before guidance. + noise_pred = noise_pred.clone() + neg_noise_pred = self.transformer( + hidden_states=latent_model_input, + timestep=timestep / 1000, + encoder_hidden_states_mask=neg_prompt_embeds_mask, + encoder_hidden_states=neg_prompt_embeds, + img_shapes=img_shapes, + return_dict=False, + )[0] + neg_noise_pred = neg_noise_pred[:, : latents.size(1)] + comb = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb, dim=-1, keepdim=True) + noise_pred = comb * (cond_norm / noise_norm) + + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + if latents.dtype != latents_dtype: + latents = latents.to(latents_dtype) + + timer.mark_post_start() + logger.info("Decoding edit...") + output_image = self._decode_latents(latents, height, width) + + if getattr(self, "rank", 0) == 0: + logger.info("Edit pipeline total: %.2fs", time.time() - pipeline_start) + + timer.mark_end() + return timer.fill(PipelineOutput(image=output_image)) diff --git a/tensorrt_llm/_torch/visual_gen/models/qwen_image/transformer_qwen_image.py b/tensorrt_llm/_torch/visual_gen/models/qwen_image/transformer_qwen_image.py index 74827992b6c7..b8f49937beb0 100644 --- a/tensorrt_llm/_torch/visual_gen/models/qwen_image/transformer_qwen_image.py +++ b/tensorrt_llm/_torch/visual_gen/models/qwen_image/transformer_qwen_image.py @@ -19,19 +19,27 @@ import functools import math +from collections.abc import Callable from typing import Any, Dict, Optional, Tuple import torch import torch.nn.functional as F from torch import nn -from tensorrt_llm._torch.modules.linear import Linear +from tensorrt_llm._torch.modules.linear import Linear, TensorParallelMode from tensorrt_llm._torch.modules.mlp import MLP from tensorrt_llm._torch.modules.rms_norm import RMSNorm +from tensorrt_llm._torch.utils import gelu_tanh, maybe_compile +from tensorrt_llm._torch.visual_gen.attention_backend.parallel import ( + Attention2DAttention, + RingAttention, + UlyssesAttention, +) from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig from tensorrt_llm._torch.visual_gen.models.modeling import BaseDiffusionModel from tensorrt_llm._torch.visual_gen.modules.attention import Attention, QKVMode from tensorrt_llm._torch.visual_gen.quantization.loader import DynamicLinearWeightLoader +from tensorrt_llm._torch.visual_gen.utils import SequenceSharder from tensorrt_llm.models.modeling_utils import QuantConfig _WEIGHT_KEY_REMAPS = [ @@ -224,6 +232,7 @@ def __init__( else: raise ValueError(f"unknown norm_type {norm_type}") + @maybe_compile() def forward(self, x: torch.Tensor, conditioning_embedding: torch.Tensor) -> torch.Tensor: emb = self.linear(self.silu(conditioning_embedding).to(x.dtype)) scale, shift = torch.chunk(emb, 2, dim=1) @@ -236,9 +245,15 @@ def forward(self, x: torch.Tensor, conditioning_embedding: torch.Tensor) -> torc # =========================================================================== -@torch.compiler.disable -def _gelu_tanh_eager(x: torch.Tensor) -> torch.Tensor: - return F.gelu(x, approximate="tanh") +def _get_feedforward_activation(activation_fn: str) -> Callable[[torch.Tensor], torch.Tensor]: + if activation_fn == "gelu-approximate": + return gelu_tanh + if activation_fn == "gelu": + return F.gelu + raise ValueError( + f"Unsupported activation_fn={activation_fn} in Qwen-Image " + "FeedForward; only gelu / gelu-approximate needed." + ) class FeedForward(MLP): @@ -257,15 +272,7 @@ def __init__( ): inner_dim = int(dim * mult) dim_out = dim_out if dim_out is not None else dim - if activation_fn == "gelu-approximate": - activation = _gelu_tanh_eager - elif activation_fn == "gelu": - activation = F.gelu - else: - raise ValueError( - f"Unsupported activation_fn={activation_fn} in Qwen-Image " - "FeedForward; only gelu / gelu-approximate needed." - ) + activation = _get_feedforward_activation(activation_fn) if dim_out != dim: raise ValueError("TRT-LLM MLP FeedForward requires dim_out == dim") super().__init__( @@ -276,7 +283,6 @@ def __init__( dtype=dtype, config=config, layer_idx=layer_idx, - reduce_output=False, ) @@ -309,6 +315,26 @@ def apply_rotary_emb_qwen( return x_out.type_as(x) +def qwen_complex_freqs_to_cos_sin(freqs_cis: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Convert Qwen's complex RoPE cache to shared interleaved real cos/sin tensors.""" + freqs_cis = torch.view_as_real(freqs_cis) + freqs_cos = freqs_cis[..., 0].repeat_interleave(2, dim=-1).unsqueeze(1).float().contiguous() + freqs_sin = freqs_cis[..., 1].repeat_interleave(2, dim=-1).unsqueeze(1).float().contiguous() + return freqs_cos, freqs_sin + + +def qwen_joint_freqs_to_cos_sin( + image_rotary_emb: Tuple[torch.Tensor, torch.Tensor], +) -> Tuple[torch.Tensor, torch.Tensor]: + """Convert and concatenate shared Qwen text/image RoPE frequencies.""" + img_freqs, txt_freqs = image_rotary_emb + txt_cos, txt_sin = qwen_complex_freqs_to_cos_sin(txt_freqs) + img_cos, img_sin = qwen_complex_freqs_to_cos_sin(img_freqs) + freqs_cos = torch.cat([txt_cos, img_cos], dim=0).squeeze(1) + freqs_sin = torch.cat([txt_sin, img_sin], dim=0).squeeze(1) + return freqs_cos, freqs_sin + + class QwenEmbedRope(nn.Module): """3D rotary position embedding over (frame, height, width) axes. @@ -465,6 +491,7 @@ def __init__( module_name: Optional[str] = None, ): config = config or DiffusionModelConfig() + dtype = dtype or config.torch_dtype super().__init__( hidden_size=dim, num_attention_heads=num_attention_heads, @@ -474,15 +501,19 @@ def __init__( qk_norm_mode="per_head", eps=eps, bias=True, - # TODO: enable fused qk-norm+RoPE after adapting Qwen's - # complex frequency cache to the shared real cos/sin format. - fuse_qk_norm_rope=False, + fuse_qk_norm_rope=(config.mapping.tp_size == 1), config=config, layer_idx=layer_idx, module_name=module_name, + separate_qkv_is_self_attention=True, ) - self.heads = num_attention_heads self.head_dim = attention_head_dim + self._supports_key_padding_mask = _supports_qwen_key_padding_mask( + self.attn_backend, self.attn + ) + self._uses_sequence_parallel_attention = _is_qwen_sequence_parallel_attention(self.attn) + + tp_mode = TensorParallelMode.COLUMN if self.tp_size > 1 else None # Text-stream QKV (diffusers names). self.add_q_proj = Linear( @@ -494,6 +525,8 @@ def __init__( quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + tensor_parallel_mode=tp_mode, + reduce_output=False, ) self.add_k_proj = Linear( dim, @@ -504,6 +537,8 @@ def __init__( quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + tensor_parallel_mode=tp_mode, + reduce_output=False, ) self.add_v_proj = Linear( dim, @@ -514,6 +549,8 @@ def __init__( quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + tensor_parallel_mode=tp_mode, + reduce_output=False, ) # QK-norms, applied per-head on the head_dim. @@ -533,22 +570,65 @@ def __init__( quant_config=self.quant_config, skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, + tensor_parallel_mode=TensorParallelMode.ROW if self.tp_size > 1 else None, + reduce_output=(self.tp_size > 1), + allreduce_strategy=self.allreduce_strategy, ) @staticmethod def _apply_rms_norm(x: torch.Tensor, norm: RMSNorm) -> torch.Tensor: return F.rms_norm(x, (x.shape[-1],), norm.weight, norm.variance_epsilon) - def forward( + def _use_fused_qk_norm_rope( + self, + hidden_states: torch.Tensor, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]], + ) -> bool: + return ( + self.fuse_qk_norm_rope + and image_rotary_emb is not None + and self.qk_norm + and hidden_states.is_cuda + and hidden_states.dtype == torch.bfloat16 + and self.head_dim in (64, 128, 256) + ) + + def _prepare_qkv_fused( self, hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, - image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - attention_mask: Optional[torch.Tensor] = None, - timestep: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - seq_txt = encoder_hidden_states.shape[1] + image_rotary_emb: Tuple[torch.Tensor, torch.Tensor], + fused_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + img_q, img_k, img_v = self.get_qkv(hidden_states) + txt_q = self.add_q_proj(encoder_hidden_states) + txt_k = self.add_k_proj(encoder_hidden_states) + txt_v = self.add_v_proj(encoder_hidden_states) + + txt_qkv = torch.cat([txt_q, txt_k, txt_v], dim=-1) + img_qkv = torch.cat([img_q, img_k, img_v], dim=-1) + qkv = torch.cat([txt_qkv, img_qkv], dim=1) + + if fused_rotary_emb is None: + fused_rotary_emb = qwen_joint_freqs_to_cos_sin(image_rotary_emb) + freqs_cos, freqs_sin = fused_rotary_emb + + self.apply_packed_qk_norm_rope( + qkv, + freqs_cos, + freqs_sin, + num_txt_tokens=encoder_hidden_states.shape[1], + q_add_weight=self.norm_added_q.weight, + k_add_weight=self.norm_added_k.weight, + ) + return qkv.split([self.local_q_dim, self.local_kv_dim, self.local_kv_dim], dim=-1) + def _prepare_qkv_unfused( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]], + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: # Image QKV. img_q, img_k, img_v = self.get_qkv(hidden_states) # Text QKV. @@ -557,12 +637,12 @@ def forward( txt_v = self.add_v_proj(encoder_hidden_states) # Reshape to (B, S, H, D). - img_q = img_q.unflatten(-1, (self.heads, -1)) - img_k = img_k.unflatten(-1, (self.heads, -1)) - img_v = img_v.unflatten(-1, (self.heads, -1)) - txt_q = txt_q.unflatten(-1, (self.heads, -1)) - txt_k = txt_k.unflatten(-1, (self.heads, -1)) - txt_v = txt_v.unflatten(-1, (self.heads, -1)) + img_q = img_q.unflatten(-1, (self.local_num_attention_heads, -1)) + img_k = img_k.unflatten(-1, (self.local_num_key_value_heads, -1)) + img_v = img_v.unflatten(-1, (self.local_num_key_value_heads, -1)) + txt_q = txt_q.unflatten(-1, (self.local_num_attention_heads, -1)) + txt_k = txt_k.unflatten(-1, (self.local_num_key_value_heads, -1)) + txt_v = txt_v.unflatten(-1, (self.local_num_key_value_heads, -1)) # Per-stream QK-norm on head dim. img_q = self._apply_rms_norm(img_q, self.norm_q) @@ -579,32 +659,63 @@ def forward( txt_k = apply_rotary_emb_qwen(txt_k, txt_freqs, use_real=False) # Joint attention: [txt | img] order. - joint_q = torch.cat([txt_q, img_q], dim=1) - joint_k = torch.cat([txt_k, img_k], dim=1) - joint_v = torch.cat([txt_v, img_v], dim=1) - - # SDPA expects (B, H, S, D); diffusers dispatch_attention_fn - # accepts (B, S, H, D) and transposes internally. Do the same. - joint_q = joint_q.transpose(1, 2) - joint_k = joint_k.transpose(1, 2) - joint_v = joint_v.transpose(1, 2) - - attn_mask = None - if attention_mask is not None: - # attention_mask is (B, Sjoint) bool or float. Expand to - # (B, 1, 1, Sjoint) so SDPA broadcasts over (H, Sq). - # Qwen pads text embeddings before concatenating [text | image], - # so masked SDPA is required to ignore padded text tokens. - attn_mask = attention_mask[:, None, None, :] + joint_q = torch.cat([txt_q, img_q], dim=1).flatten(2) + joint_k = torch.cat([txt_k, img_k], dim=1).flatten(2) + joint_v = torch.cat([txt_v, img_v], dim=1).flatten(2) + return joint_q, joint_k, joint_v - if attn_mask is None: - out = self._attn_impl( - joint_q.transpose(1, 2).flatten(2), - joint_k.transpose(1, 2).flatten(2), - joint_v.transpose(1, 2).flatten(2), - timestep=timestep, + def _prepare_qkv( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]], + fused_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if self._use_fused_qk_norm_rope(hidden_states, image_rotary_emb): + return self._prepare_qkv_fused( + hidden_states, + encoder_hidden_states, + image_rotary_emb, + fused_rotary_emb, + ) + return self._prepare_qkv_unfused(hidden_states, encoder_hidden_states, image_rotary_emb) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + fused_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + timestep: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + seq_txt = encoder_hidden_states.shape[1] + + joint_q, joint_k, joint_v = self._prepare_qkv( + hidden_states, + encoder_hidden_states, + image_rotary_emb, + fused_rotary_emb, + ) + + if attention_mask is None or self._supports_key_padding_mask: + attn_kwargs = {} + if attention_mask is not None: + attn_kwargs["key_padding_mask"] = attention_mask + out = self._attn_impl(joint_q, joint_k, joint_v, timestep=timestep, **attn_kwargs) + elif self._uses_sequence_parallel_attention: + raise NotImplementedError( + "Padded Qwen-Image prompts require a key-padding-mask-capable " + f"sequence-parallel attention backend, got {type(self.attn).__name__} " + f"wrapping {self.attn_backend}." ) else: + attn_mask = attention_mask[:, None, None, :] + # SDPA expects (B, H, S, D); diffusers dispatch_attention_fn + # accepts (B, S, H, D) and transposes internally. Do the same. + joint_q = joint_q.unflatten(-1, (self.local_num_attention_heads, -1)).transpose(1, 2) + joint_k = joint_k.unflatten(-1, (self.local_num_key_value_heads, -1)).transpose(1, 2) + joint_v = joint_v.unflatten(-1, (self.local_num_key_value_heads, -1)).transpose(1, 2) out = F.scaled_dot_product_attention( joint_q, joint_k, joint_v, attn_mask=attn_mask, dropout_p=0.0, is_causal=False ) @@ -699,6 +810,7 @@ def __init__( ) @staticmethod + @maybe_compile() def _modulate(x: torch.Tensor, mod_params: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: shift, scale, gate = mod_params.chunk(3, dim=-1) shift = shift.unsqueeze(1) @@ -706,12 +818,22 @@ def _modulate(x: torch.Tensor, mod_params: torch.Tensor) -> Tuple[torch.Tensor, gate = gate.unsqueeze(1) return x * (1 + scale) + shift, gate + @staticmethod + @maybe_compile() + def _apply_gate_residual( + hidden_states: torch.Tensor, + gate: torch.Tensor, + residual: torch.Tensor, + ) -> torch.Tensor: + return hidden_states + gate * residual + def forward( self, hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + fused_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, attention_mask: Optional[torch.Tensor] = None, timestep: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: @@ -730,20 +852,27 @@ def forward( hidden_states=img_modulated, encoder_hidden_states=txt_modulated, image_rotary_emb=image_rotary_emb, + fused_rotary_emb=fused_rotary_emb, attention_mask=attention_mask, timestep=timestep, ) # Residual. - hidden_states = hidden_states + img_gate1 * img_attn_output - encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output + hidden_states = self._apply_gate_residual(hidden_states, img_gate1, img_attn_output) + encoder_hidden_states = self._apply_gate_residual( + encoder_hidden_states, txt_gate1, txt_attn_output + ) # Norm2 + MLP + residual. img_modulated2, img_gate2 = self._modulate(self.img_norm2(hidden_states), img_mod2) - hidden_states = hidden_states + img_gate2 * self.img_mlp(img_modulated2) + hidden_states = self._apply_gate_residual( + hidden_states, img_gate2, self.img_mlp(img_modulated2) + ) txt_modulated2, txt_gate2 = self._modulate(self.txt_norm2(encoder_hidden_states), txt_mod2) - encoder_hidden_states = encoder_hidden_states + txt_gate2 * self.txt_mlp(txt_modulated2) + encoder_hidden_states = self._apply_gate_residual( + encoder_hidden_states, txt_gate2, self.txt_mlp(txt_modulated2) + ) if encoder_hidden_states.dtype == torch.float16: encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) @@ -758,6 +887,32 @@ def forward( # =========================================================================== +def _is_qwen_sequence_parallel_attention(attn: Any) -> bool: + return isinstance(attn, (Attention2DAttention, RingAttention, UlyssesAttention)) + + +def _supports_qwen_key_padding_mask(attn_backend: str, attn: Any) -> bool: + return attn_backend == "VANILLA" and not isinstance(attn, (Attention2DAttention, RingAttention)) + + +def _build_joint_attention_mask( + encoder_hidden_states_mask: Optional[torch.Tensor], + hidden_states: torch.Tensor, +) -> Optional[torch.Tensor]: + if encoder_hidden_states_mask is None: + return None + encoder_hidden_states_mask = encoder_hidden_states_mask.to( + device=hidden_states.device, dtype=torch.bool + ) + batch_size, image_seq_len = hidden_states.shape[:2] + image_mask = torch.ones( + (batch_size, image_seq_len), + dtype=torch.bool, + device=hidden_states.device, + ) + return torch.cat([encoder_hidden_states_mask, image_mask], dim=1) + + class QwenImageTransformer2DModel(BaseDiffusionModel): """Qwen-Image 20B MMDiT transformer. @@ -793,6 +948,14 @@ def __init__( self.joint_attention_dim = joint_attention_dim self.inner_dim = num_attention_heads * attention_head_dim + vgm = self.model_config.visual_gen_mapping + local_num_heads = num_attention_heads // self.model_config.mapping.tp_size + self.sharder = SequenceSharder.from_vgm( + vgm, + num_attention_heads=local_num_heads, + num_kv_heads=local_num_heads, + ) + self.pos_embed = QwenEmbedRope(theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True) self.time_text_embed = QwenTimestepProjEmbeddings(embedding_dim=self.inner_dim) self.txt_norm = RMSNorm( @@ -847,6 +1010,101 @@ def __init__( def device(self) -> torch.device: return self.proj_out.weight.device + def _first_block_attn(self) -> Optional[QwenJointAttention]: + """Return the first real block attention through wrappers such as Cache-DiT.""" + blocks = self.transformer_blocks + while blocks: + block = blocks[0] + attention = getattr(block, "attn", None) + if attention is not None: + return attention + blocks = getattr(block, "transformer_blocks", None) + if blocks is None: + raise AttributeError( + f"Cannot resolve Qwen attention through {type(block).__name__}." + ) + return None + + def _shard_sequences( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + encoder_hidden_states_mask: Optional[torch.Tensor], + image_rotary_emb: Tuple[torch.Tensor, torch.Tensor], + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + Tuple[torch.Tensor, torch.Tensor], + Optional[torch.Tensor], + int, + ]: + image_seq_len = hidden_states.shape[1] + if not self.sharder.is_active: + return ( + hidden_states, + encoder_hidden_states, + image_rotary_emb, + _build_joint_attention_mask(encoder_hidden_states_mask, hidden_states), + image_seq_len, + ) + + text_seq_len = encoder_hidden_states.shape[1] + image_padding = (-image_seq_len) % self.sharder.size + text_padding = (-text_seq_len) % self.sharder.size + needs_mask = encoder_hidden_states_mask is not None or image_padding > 0 or text_padding > 0 + first_block_attn = self._first_block_attn() + if ( + needs_mask + and first_block_attn is not None + and not first_block_attn._supports_key_padding_mask + ): + raise NotImplementedError( + "Padded Qwen-Image sequence parallelism requires VANILLA Ulysses attention." + ) + + hidden_states = self.sharder.shard(hidden_states, dim=1, pad_to_multiple=True) + encoder_hidden_states = self.sharder.shard( + encoder_hidden_states, dim=1, pad_to_multiple=True + ) + image_freqs, text_freqs = image_rotary_emb + image_rotary_emb = ( + self.sharder.shard(image_freqs, dim=0, pad_to_multiple=True), + self.sharder.shard(text_freqs, dim=0, pad_to_multiple=True), + ) + + block_attention_mask = None + if needs_mask: + batch_size = hidden_states.shape[0] + if encoder_hidden_states_mask is None: + encoder_hidden_states_mask = torch.ones( + (batch_size, text_seq_len), + device=hidden_states.device, + dtype=torch.bool, + ) + else: + encoder_hidden_states_mask = encoder_hidden_states_mask.to( + device=hidden_states.device, dtype=torch.bool + ) + image_mask = torch.ones( + (batch_size, image_seq_len), + device=hidden_states.device, + dtype=torch.bool, + ) + local_text_mask = self.sharder.shard( + encoder_hidden_states_mask, dim=1, pad_to_multiple=True + ) + local_image_mask = self.sharder.shard(image_mask, dim=1, pad_to_multiple=True) + local_joint_mask = torch.cat([local_text_mask, local_image_mask], dim=1) + block_attention_mask = self.sharder.gather(local_joint_mask, dim=1) + + return ( + hidden_states, + encoder_hidden_states, + image_rotary_emb, + block_attention_mask, + image_seq_len, + ) + def _weight_loading_device(self) -> torch.device: for param in self.parameters(): if param.device.type != "meta": @@ -977,6 +1235,17 @@ def load_weights(self, weights: Dict[str, torch.Tensor]) -> None: if param is not None and param_name in module_weights: param.data.copy_(module_weights[param_name].to(param.dtype)) + def register_cuda_graph_extra_key_fns(self, runner: Any) -> None: + super().register_cuda_graph_extra_key_fns(runner) + + def img_shapes_key(*args: Any, **kwargs: Any) -> tuple | None: + img_shapes = kwargs.get("img_shapes") + if img_shapes is None: + return None + return tuple(tuple(tuple(shape) for shape in item) for item in img_shapes) + + runner.register_extra_key_fn("img_shapes", img_shapes_key) + def post_load_weights(self) -> None: for name, module in self.named_modules(): if isinstance(module, Linear): @@ -1039,19 +1308,24 @@ def forward( image_rotary_emb = self.pos_embed( img_shapes, max_txt_seq_len=text_seq_len, device=hidden_states.device ) - - # Build joint attention mask [text_mask | all-ones image_mask] once. - block_attention_mask = None - if encoder_hidden_states_mask is not None: - if encoder_hidden_states_mask.dtype != torch.bool: - encoder_hidden_states_mask = encoder_hidden_states_mask.to(torch.bool) - batch_size, image_seq_len = hidden_states.shape[:2] - image_mask = torch.ones( - (batch_size, image_seq_len), - dtype=torch.bool, - device=hidden_states.device, - ) - block_attention_mask = torch.cat([encoder_hidden_states_mask, image_mask], dim=1) + ( + hidden_states, + encoder_hidden_states, + image_rotary_emb, + block_attention_mask, + image_seq_len, + ) = self._shard_sequences( + hidden_states, + encoder_hidden_states, + encoder_hidden_states_mask, + image_rotary_emb, + ) + fused_rotary_emb = None + first_block_attn = self._first_block_attn() + if first_block_attn is not None and first_block_attn._use_fused_qk_norm_rope( + hidden_states, image_rotary_emb + ): + fused_rotary_emb = qwen_joint_freqs_to_cos_sin(image_rotary_emb) for block in self.transformer_blocks: encoder_hidden_states, hidden_states = block( @@ -1059,10 +1333,12 @@ def forward( encoder_hidden_states=encoder_hidden_states, temb=temb, image_rotary_emb=image_rotary_emb, + fused_rotary_emb=fused_rotary_emb, attention_mask=block_attention_mask, timestep=timestep, ) + hidden_states = self.sharder.gather(hidden_states, dim=1, unpad_to=image_seq_len) hidden_states = self.norm_out(hidden_states, temb) output = self.proj_out(hidden_states) diff --git a/tests/integration/defs/examples/test_bindings.py b/tensorrt_llm/_torch/visual_gen/models/qwen_image_layered/__init__.py similarity index 51% rename from tests/integration/defs/examples/test_bindings.py rename to tensorrt_llm/_torch/visual_gen/models/qwen_image_layered/__init__.py index 3d5c21f82bf3..34d63003963b 100644 --- a/tests/integration/defs/examples/test_bindings.py +++ b/tensorrt_llm/_torch/visual_gen/models/qwen_image_layered/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,23 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Module test_bindings test bindings examples.""" +"""Qwen-Image-Layered pipeline exports.""" -import os +from .pipeline_qwen_image_layered import QwenImageLayeredPipeline +from .transformer_qwen_image_layered import QwenEmbedLayer3DRope, QwenImageLayeredTransformer2DModel -import pytest -from defs.conftest import get_sm_version - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@pytest.fixture(scope="module") -def bindings_example_root(llm_root): - "Get bindings example root" - example_root = os.path.join(llm_root, "examples", "bindings", "executor") - - return example_root +__all__ = [ + "QwenEmbedLayer3DRope", + "QwenImageLayeredPipeline", + "QwenImageLayeredTransformer2DModel", +] diff --git a/tensorrt_llm/_torch/visual_gen/models/qwen_image_layered/pipeline_qwen_image_layered.py b/tensorrt_llm/_torch/visual_gen/models/qwen_image_layered/pipeline_qwen_image_layered.py new file mode 100644 index 000000000000..f5d1b2383baa --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/qwen_image_layered/pipeline_qwen_image_layered.py @@ -0,0 +1,956 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Qwen-Image-Layered image decomposition pipeline.""" + +import io +import math +import time +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from tensorrt_llm._torch.visual_gen.output import CudaPhaseTimer, PipelineOutput +from tensorrt_llm._torch.visual_gen.pipeline import BasePipeline, ExtraParamSchema +from tensorrt_llm._torch.visual_gen.pipeline_registry import PipelineComponent, register_pipeline +from tensorrt_llm.logger import logger + +from .transformer_qwen_image_layered import QwenImageLayeredTransformer2DModel + +_PROMPT_TEMPLATE = ( + "<|im_start|>system\nDescribe the image by detailing the color, shape, " + "size, texture, quantity, text, spatial relationships of the objects and " + "background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" + "<|im_start|>assistant\n" +) +_PROMPT_TEMPLATE_START_IDX = 34 +_PROCESSOR_SUBFOLDER = "processor" + +_LAYERED_DEFAULT_GENERATION_PARAMS = { + "height": None, + "width": None, + "num_inference_steps": 50, + "guidance_scale": 4.0, + "max_sequence_length": 512, +} + +_LAYERED_CAPTION_PROMPT = """<|im_start|>system +You are a helpful assistant.<|im_end|> +<|im_start|>user +# 图像标注器 +你是一个专业的图像标注器。请基于输入图像，撰写图注: +1. +使用自然、描述性的语言撰写图注，不要使用结构化形式或富文本形式。 +2. 通过加入以下内容，丰富图注细节： + - 对象的属性：如数量、颜色、形状、大小、位置、材质、状态、动作等 + - +对象间的视觉关系：如空间关系、功能关系、动作关系、从属关系、比较关系、因果关系等 + - 环境细节：例如天气、光照、颜色、纹理、气氛等 + - 文字内容：识别图像中清晰可见的文字，不做翻译和解释，用引号在图注中强调 +3. +保持真实性与准确性： + - 不要使用笼统的描述 + - +描述图像中所有可见的信息，但不要加入没有在图像中出现的内容 +<|vision_start|><|image_pad|><|vision_end|><|im_end|> +<|im_start|>assistant +""" + +_LAYERED_CAPTION_PROMPT_EN = """<|im_start|>system +You are a helpful assistant.<|im_end|> +<|im_start|>user +# Image Annotator +You are a professional +image annotator. Please write an image caption based on the input image: +1. Write the caption using natural, +descriptive language without structured formats or rich text. +2. Enrich caption details by including:\x20 + - Object +attributes, such as quantity, color, shape, size, material, state, position, actions, and so on + - Vision Relations +between objects, such as spatial relations, functional relations, possessive relations, attachment relations, action +relations, comparative relations, causal relations, and so on + - Environmental details, such as weather, lighting, +colors, textures, atmosphere, and so on + - Identify the text clearly visible in the image, without translation or +explanation, and highlight it in the caption with quotation marks +3. Maintain authenticity and accuracy: + - Avoid +generalizations + - Describe all visible information in the image, while do not add information not explicitly shown in +the image +<|vision_start|><|image_pad|><|vision_end|><|im_end|> +<|im_start|>assistant +""" + + +def _calculate_dimensions(target_area: int, ratio: float) -> Tuple[int, int]: + width = math.sqrt(target_area * ratio) + height = width / ratio + width = round(width / 32) * 32 + height = round(height / 32) * 32 + return int(width), int(height) + + +def _retrieve_latents( + encoder_output: torch.Tensor, + generator: Optional[torch.Generator] = None, + sample_mode: str = "sample", +) -> torch.Tensor: + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + if hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + if hasattr(encoder_output, "latents"): + return encoder_output.latents + raise AttributeError("Could not access latents of provided encoder_output") + + +@register_pipeline( + "QwenImageLayeredPipeline", + hf_ids=["Qwen/Qwen-Image-Layered"], + doc=( + "Qwen-Image-Layered image-conditioned layer decomposition. Loads Diffusers-format " + "checkpoints with tokenizer, text encoder, transformer, VAE, scheduler, image " + "processor, and Qwen2VL processor components. Requests take an input image and " + "optional prompt, then return generated RGBA layers as a saveable image grid. " + "Use VisualGen.default_params or VisualGen.extra_param_specs to set request knobs " + "such as extra_params['layers'] and extra_params['resolution']." + ), +) +class QwenImageLayeredPipeline(BasePipeline): + """Qwen-Image-Layered image decomposition pipeline.""" + + DEFAULT_GENERATION_PARAMS = _LAYERED_DEFAULT_GENERATION_PARAMS + + def __init__(self, pipeline_config): + super().__init__(pipeline_config) + self.vae_scale_factor = 8 + self.tokenizer_max_length = 1024 + self.latent_channels = 16 + + def _init_transformer(self) -> None: + logger.info("Creating Qwen-Image-Layered transformer") + model_config = self.pipeline_config.model_configs["transformer"] + pretrained = getattr(model_config, "pretrained_config", None) + + def _cfg(name: str, default): + if pretrained is None: + return default + if isinstance(pretrained, dict): + return pretrained.get(name, default) + return getattr(pretrained, name, default) + + self.guidance_embeds = _cfg("guidance_embeds", False) + self.zero_cond_t = _cfg("zero_cond_t", False) + if self.guidance_embeds: + raise NotImplementedError("Guidance-distilled Qwen-Image-Layered is not supported.") + if self.zero_cond_t: + raise NotImplementedError("Qwen-Image-Layered zero_cond_t is not supported yet.") + + self.transformer = QwenImageLayeredTransformer2DModel( + model_config=model_config, + patch_size=_cfg("patch_size", 2), + in_channels=_cfg("in_channels", 64), + out_channels=_cfg("out_channels", 16), + num_layers=_cfg("num_layers", 60), + attention_head_dim=_cfg("attention_head_dim", 128), + num_attention_heads=_cfg("num_attention_heads", 24), + joint_attention_dim=_cfg("joint_attention_dim", 3584), + axes_dims_rope=tuple(_cfg("axes_dims_rope", (16, 56, 56))), + use_additional_t_cond=_cfg("use_additional_t_cond", False), + use_layer3d_rope=_cfg("use_layer3d_rope", False), + ) + + @property + def default_warmup_resolutions(self) -> List[Tuple[int, int]]: + return [(640, 640)] + + @property + def default_warmup_num_frames(self) -> List[int]: + return [1] + + @property + def default_generation_params(self) -> dict: + return dict(_LAYERED_DEFAULT_GENERATION_PARAMS) + + @property + def resolution_multiple_of(self) -> Tuple[int, int]: + return (self.vae_scale_factor * 2, self.vae_scale_factor * 2) + + def warmup_cache_key(self, height: Optional[int], width: Optional[int], **kwargs) -> tuple: + if height is None or width is None: + height, width = self.default_warmup_resolutions[0] + return (height, width) + + def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> None: + from PIL import Image + + del num_frames + resolution = 1024 if max(height, width) > 640 else 640 + dummy_image = Image.new("RGBA", (width, height)) + with torch.no_grad(): + self.forward( + image=dummy_image, + prompt="warmup", + true_cfg_scale=1.0, + layers=4, + num_inference_steps=max(steps, 2), + seed=42, + max_sequence_length=64, + resolution=resolution, + ) + + @property + def extra_param_specs(self) -> dict: + return { + "layers": ExtraParamSchema( + type="int", + default=4, + description="Number of latent output layers to generate.", + range=(1, 16), + ), + "resolution": ExtraParamSchema( + type="int", + default=640, + description="Layered model resolution bucket. Supported values: 640 or 1024.", + ), + "cfg_normalize": ExtraParamSchema( + type="bool", + default=False, + description="Normalize classifier-free guidance prediction by conditional norm.", + ), + "use_en_prompt": ExtraParamSchema( + type="bool", + default=False, + description="Use English auto-caption prompt when prompt is empty.", + ), + } + + def load_standard_components( + self, + checkpoint_dir: str, + device: torch.device, + skip_components: Optional[list] = None, + ) -> None: + skip_components = skip_components or [] + + if PipelineComponent.TOKENIZER not in skip_components: + try: + from transformers import Qwen2Tokenizer + except ImportError as e: # pragma: no cover + raise ImportError( + "Qwen-Image-Layered requires transformers with Qwen2Tokenizer." + ) from e + logger.info("Loading Qwen2 tokenizer...") + self.tokenizer = Qwen2Tokenizer.from_pretrained( + checkpoint_dir, subfolder=PipelineComponent.TOKENIZER + ) + + if PipelineComponent.TEXT_ENCODER not in skip_components: + try: + from transformers import Qwen2_5_VLForConditionalGeneration + except ImportError as e: # pragma: no cover + raise ImportError( + "Qwen-Image-Layered requires transformers with Qwen2_5_VL." + ) from e + logger.info("Loading Qwen2.5-VL text encoder...") + self.text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( + checkpoint_dir, + subfolder=PipelineComponent.TEXT_ENCODER, + torch_dtype=self.pipeline_config.torch_dtype, + ).to(device) + + if PipelineComponent.VAE not in skip_components: + try: + from diffusers import AutoencoderKLQwenImage + except ImportError as e: # pragma: no cover + raise ImportError( + "Qwen-Image-Layered requires diffusers with AutoencoderKLQwenImage." + ) from e + logger.info("Loading Qwen-Image VAE...") + self.vae = AutoencoderKLQwenImage.from_pretrained( + checkpoint_dir, + subfolder=PipelineComponent.VAE, + torch_dtype=torch.bfloat16, + ).to(device) + temperal_downsample = getattr(self.vae, "temperal_downsample", [1, 1, 1]) + self.vae_scale_factor = 2 ** len(temperal_downsample) + + if PipelineComponent.SCHEDULER not in skip_components: + try: + from diffusers import FlowMatchEulerDiscreteScheduler + except ImportError as e: # pragma: no cover + raise ImportError( + "Qwen-Image-Layered requires diffusers with FlowMatchEulerDiscreteScheduler." + ) from e + logger.info("Loading Qwen-Image scheduler...") + self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + checkpoint_dir, subfolder=PipelineComponent.SCHEDULER + ) + + if PipelineComponent.IMAGE_PROCESSOR not in skip_components: + try: + from diffusers.image_processor import VaeImageProcessor + except ImportError as e: # pragma: no cover + raise ImportError( + "Qwen-Image-Layered requires diffusers with VaeImageProcessor." + ) from e + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + + if _PROCESSOR_SUBFOLDER not in skip_components: + try: + from transformers import Qwen2VLProcessor + except ImportError as e: # pragma: no cover + raise ImportError( + "Qwen-Image-Layered requires transformers with Qwen2VLProcessor." + ) from e + logger.info("Loading Qwen2-VL processor...") + self.vl_processor = Qwen2VLProcessor.from_pretrained( + checkpoint_dir, subfolder=_PROCESSOR_SUBFOLDER + ) + self.processor = self.vl_processor + + self.default_height = 640 + self.default_width = 640 + self.max_sequence_length = 512 + if getattr(self, "vae", None) is not None: + self.latent_channels = self.vae.config.z_dim + + def load_weights(self, weights: dict) -> None: + if self.transformer is not None: + transformer_weights = weights.get("transformer", weights) + self.transformer.load_weights(transformer_weights) + self.transformer.to_inference_dtype().eval() + self._target_dtype = self.pipeline_config.torch_dtype + + @staticmethod + def _load_image_input(image): + from PIL import Image + + if isinstance(image, list): + return [QwenImageLayeredPipeline._load_image_input(item) for item in image] + if isinstance(image, str): + return Image.open(image).convert("RGBA") + if isinstance(image, bytes): + return Image.open(io.BytesIO(image)).convert("RGBA") + if hasattr(image, "convert") and getattr(image, "mode", None) != "RGBA": + return image.convert("RGBA") + return image + + @staticmethod + def _image_size(image) -> Tuple[int, int]: + first = image[0] if isinstance(image, list) else image + if isinstance(first, torch.Tensor): + if first.ndim == 5: + return int(first.shape[-1]), int(first.shape[-2]) + if first.ndim == 4: + return int(first.shape[-1]), int(first.shape[-2]) + if first.ndim == 3: + return int(first.shape[-1]), int(first.shape[-2]) + raise ValueError(f"Unsupported tensor image shape: {tuple(first.shape)}") + if hasattr(first, "size"): + size = first.size + if isinstance(size, tuple): + return int(size[0]), int(size[1]) + raise ValueError(f"Unsupported image input type: {type(first).__name__}") + + def _is_layered_latent_image(self, image) -> bool: + return ( + isinstance(image, torch.Tensor) + and image.ndim == 5 + and image.shape[1] == self.latent_channels + ) + + @staticmethod + def _validate_single_conditioning_frame(image: torch.Tensor) -> None: + if image.shape[2] != 1: + raise ValueError( + "Layered latent image inputs must have exactly one conditioning frame " + f"(F=1), got F={image.shape[2]}." + ) + + @staticmethod + def _is_empty_prompt(prompt) -> bool: + return prompt is None or (isinstance(prompt, str) and prompt.strip() == "") + + @staticmethod + def _image_batch_size(image) -> int: + if isinstance(image, list): + return len(image) + if isinstance(image, torch.Tensor) and image.ndim >= 4: + return int(image.shape[0]) + return 1 + + @staticmethod + def _expand_values_to_batch(values: List[str], batch_size: int, name: str) -> List[str]: + if len(values) == batch_size: + return values + if len(values) == 1: + return values * batch_size + if batch_size % len(values) == 0: + repeat = batch_size // len(values) + return [value for value in values for _ in range(repeat)] + raise ValueError( + f"Cannot align {name} batch {len(values)} to effective batch {batch_size}." + ) + + @staticmethod + def _align_prompts_to_image_batch(prompts: List[str], image_batch: int) -> List[str]: + prompt_batch = len(prompts) + if prompt_batch == image_batch or image_batch == 1 or prompt_batch % image_batch == 0: + return prompts + if prompt_batch == 1: + return prompts * image_batch + if image_batch % prompt_batch == 0: + repeat = image_batch // prompt_batch + return [prompt for prompt in prompts for _ in range(repeat)] + raise ValueError( + f"Prompt batch size {prompt_batch} must match image batch size {image_batch}, " + "or one must divide the other exactly." + ) + + @staticmethod + def _repeat_conditioning_batch( + conditioning: torch.Tensor, + batch_size: int, + name: str, + ) -> torch.Tensor: + conditioning_batch = conditioning.shape[0] + if conditioning_batch == batch_size: + return conditioning + if batch_size > conditioning_batch and batch_size % conditioning_batch == 0: + return conditioning.repeat_interleave(batch_size // conditioning_batch, dim=0) + if batch_size > conditioning_batch: + raise ValueError(f"Cannot duplicate {name} batch {conditioning_batch} to {batch_size}.") + raise ValueError( + f"{name} batch size {conditioning_batch} must match prompt batch size {batch_size}, " + "or divide it exactly." + ) + + @staticmethod + def _layer_stack_to_image_grid(layer_stack: torch.Tensor) -> torch.Tensor: + if layer_stack.ndim != 5: + raise ValueError( + "Qwen-Image-Layered output must have shape (B, layers, H, W, C), " + f"got {tuple(layer_stack.shape)}." + ) + batch_size, layers, height, width, channels = layer_stack.shape + grid_cols = math.ceil(math.sqrt(layers)) + grid_rows = math.ceil(layers / grid_cols) + pad_layers = grid_rows * grid_cols - layers + if pad_layers: + padding = layer_stack.new_zeros(batch_size, pad_layers, height, width, channels) + layer_stack = torch.cat([layer_stack, padding], dim=1) + grid = layer_stack.reshape(batch_size, grid_rows, grid_cols, height, width, channels) + grid = grid.permute(0, 1, 3, 2, 4, 5) + return grid.reshape(batch_size, grid_rows * height, grid_cols * width, channels) + + @staticmethod + def _extract_masked_hidden( + hidden_states: torch.Tensor, mask: torch.Tensor + ) -> Tuple[torch.Tensor, ...]: + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + return torch.split(selected, valid_lengths.tolist(), dim=0) + + def _encode_prompt( + self, + prompt: List[str], + device: torch.device, + max_sequence_length: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + drop_idx = _PROMPT_TEMPLATE_START_IDX + txt = [_PROMPT_TEMPLATE.format(e) for e in prompt] + tok = self.tokenizer( + txt, + max_length=self.tokenizer_max_length + drop_idx, + padding=True, + truncation=True, + return_tensors="pt", + ).to(device) + + encoder_outputs = self.text_encoder( + input_ids=tok.input_ids, + attention_mask=tok.attention_mask, + output_hidden_states=True, + ) + hidden_states = encoder_outputs.hidden_states[-1] + + split_hidden = self._extract_masked_hidden(hidden_states, tok.attention_mask) + split_hidden = [h[drop_idx:] for h in split_hidden] + attn_masks = [ + torch.ones(h.size(0), dtype=torch.long, device=h.device) for h in split_hidden + ] + max_len = max(h.size(0) for h in split_hidden) + prompt_embeds = torch.stack( + [torch.cat([h, h.new_zeros(max_len - h.size(0), h.size(1))]) for h in split_hidden] + ) + prompt_embeds_mask = torch.stack( + [torch.cat([m, m.new_zeros(max_len - m.size(0))]) for m in attn_masks] + ) + + prompt_embeds = prompt_embeds[:, :max_sequence_length] + prompt_embeds_mask = prompt_embeds_mask[:, :max_sequence_length] + prompt_embeds = prompt_embeds.to(dtype=self.dtype, device=device) + return prompt_embeds, prompt_embeds_mask + + @staticmethod + def _pack_layered_latents( + latents: torch.Tensor, + batch_size: int, + num_channels_latents: int, + height: int, + width: int, + layers: int, + ) -> torch.Tensor: + latents = latents.view( + batch_size, + layers, + num_channels_latents, + height // 2, + 2, + width // 2, + 2, + ) + latents = latents.permute(0, 1, 3, 5, 2, 4, 6) + return latents.reshape( + batch_size, + layers * (height // 2) * (width // 2), + num_channels_latents * 4, + ) + + @staticmethod + def _unpack_layered_latents( + latents: torch.Tensor, + height: int, + width: int, + layers: int, + vae_scale_factor: int, + ) -> torch.Tensor: + batch_size, _, channels = latents.shape + h = 2 * (int(height) // (vae_scale_factor * 2)) + w = 2 * (int(width) // (vae_scale_factor * 2)) + latents = latents.view(batch_size, layers + 1, h // 2, w // 2, channels // 4, 2, 2) + latents = latents.permute(0, 1, 4, 2, 5, 3, 6) + latents = latents.reshape(batch_size, layers + 1, channels // 4, h, w) + return latents.permute(0, 2, 1, 3, 4) + + def _encode_vae_image( + self, + image: torch.Tensor, + generator: Optional[torch.Generator], + ) -> torch.Tensor: + if isinstance(generator, list): + image_latents = [ + _retrieve_latents( + self.vae.encode(image[i : i + 1]), + generator=generator[i], + sample_mode="argmax", + ) + for i in range(image.shape[0]) + ] + image_latents = torch.cat(image_latents, dim=0) + else: + image_latents = _retrieve_latents( + self.vae.encode(image), + generator=generator, + sample_mode="argmax", + ) + latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, self.latent_channels, 1, 1, 1) + .to(image_latents.device, image_latents.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std) + .view(1, self.latent_channels, 1, 1, 1) + .to(image_latents.device, image_latents.dtype) + ) + return (image_latents - latents_mean) / latents_std + + def _prepare_layered_latents( + self, + image, + batch_size: int, + num_channels_latents: int, + height: int, + width: int, + layers: int, + dtype: torch.dtype, + device: torch.device, + generator: Optional[torch.Generator], + latents: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + from diffusers.utils.torch_utils import randn_tensor + + h = 2 * (int(height) // (self.vae_scale_factor * 2)) + w = 2 * (int(width) // (self.vae_scale_factor * 2)) + latent_shape = (batch_size, layers + 1, num_channels_latents, h, w) + + image = image.to(device=device, dtype=dtype) + if image.shape[1] != self.latent_channels: + image_latents = self._encode_vae_image(image=image, generator=generator) + else: + image_latents = image + self._validate_single_conditioning_frame(image_latents) + image_latents = self._repeat_conditioning_batch(image_latents, batch_size, "image") + + image_latent_height, image_latent_width = image_latents.shape[3:] + image_latents = image_latents.permute(0, 2, 1, 3, 4) + image_latents = self._pack_layered_latents( + image_latents, + batch_size, + num_channels_latents, + image_latent_height, + image_latent_width, + 1, + ) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"Received {len(generator)} generators for effective batch size {batch_size}." + ) + if latents is None: + latents = randn_tensor(latent_shape, generator=generator, device=device, dtype=dtype) + latents = self._pack_layered_latents( + latents, + batch_size, + num_channels_latents, + h, + w, + layers + 1, + ) + else: + latents = latents.to(device=device, dtype=dtype) + return latents, image_latents + + def _decode_layered_latents( + self, + latents: torch.Tensor, + height: int, + width: int, + layers: int, + ) -> torch.Tensor: + latents = self._unpack_layered_latents( + latents, + height, + width, + layers, + self.vae_scale_factor, + ) + latents = latents.to(self.vae.dtype) + + z_dim = self.vae.config.z_dim + latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, z_dim, 1, 1, 1) + .to(latents.device, latents.dtype) + ) + latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, z_dim, 1, 1, 1).to( + latents.device, latents.dtype + ) + latents = latents / latents_std + latents_mean + + batch_size, channels, frames, latent_h, latent_w = latents.shape + latents = latents[:, :, 1:] + latents = latents.permute(0, 2, 1, 3, 4).reshape(-1, channels, 1, latent_h, latent_w) + image = self.vae.decode(latents, return_dict=False)[0].squeeze(2) + image = (image / 2 + 0.5).clamp(0, 1) + image = image.permute(0, 2, 3, 1) + image = (image * 255).round().to(torch.uint8) + return image.reshape(batch_size, frames - 1, image.shape[1], image.shape[2], image.shape[3]) + + def get_image_caption( + self, + prompt_image, + use_en_prompt: bool = False, + device=None, + ) -> List[str]: + if not hasattr(self, "vl_processor"): + raise ValueError("Automatic image captioning requires Qwen2VLProcessor.") + prompt = _LAYERED_CAPTION_PROMPT_EN if use_en_prompt else _LAYERED_CAPTION_PROMPT + image_batch = self._image_batch_size(prompt_image) + model_inputs = self.vl_processor( + text=[prompt] * image_batch, + images=prompt_image, + padding=True, + return_tensors="pt", + ).to(device) + generated_ids = self.text_encoder.generate(**model_inputs, max_new_tokens=512) + generated_ids_trimmed = [ + out_ids[len(in_ids) :] + for in_ids, out_ids in zip(model_inputs.input_ids, generated_ids, strict=True) + ] + captions = self.vl_processor.batch_decode( + generated_ids_trimmed, + skip_special_tokens=True, + clean_up_tokenization_spaces=False, + ) + if len(captions) != image_batch: + raise ValueError( + f"Caption batch size {len(captions)} does not match image batch {image_batch}." + ) + return [caption.strip() for caption in captions] + + def infer(self, req): + extra = req.params.extra_params or {} + num_per = getattr(req.params, "num_images_per_prompt", 1) or 1 + base_prompts = req.prompt if isinstance(req.prompt, list) else [req.prompt] + prompts = [p for p in base_prompts for _ in range(num_per)] + + negative = req.params.negative_prompt + if negative is not None: + negatives = negative if isinstance(negative, list) else [negative] + if len(negatives) == 1: + negatives = negatives * len(base_prompts) + elif len(negatives) != len(base_prompts): + raise ValueError( + "negative_prompt must be a string, a singleton list, " + "or a list with the same length as prompt" + ) + negative = [n for n in negatives for _ in range(num_per)] + + return self.forward( + image=req.params.image, + prompt=prompts, + negative_prompt=negative, + height=req.params.height, + width=req.params.width, + true_cfg_scale=req.params.guidance_scale, + layers=extra.get("layers", 4), + num_inference_steps=req.params.num_inference_steps, + seed=req.params.seed, + max_sequence_length=req.params.max_sequence_length, + resolution=extra.get("resolution", 640), + cfg_normalize=extra.get("cfg_normalize", False), + use_en_prompt=extra.get("use_en_prompt", False), + ) + + @torch.inference_mode() + def forward( + self, + image, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + true_cfg_scale: float = 4.0, + layers: int = 4, + num_inference_steps: int = 50, + seed: int = 42, + max_sequence_length: int = 512, + resolution: int = 640, + cfg_normalize: bool = False, + use_en_prompt: bool = False, + sigmas: Optional[list] = None, + latents: Optional[torch.Tensor] = None, + ) -> PipelineOutput: + if image is None: + raise ValueError("QwenImageLayeredPipeline requires an input image.") + if resolution not in (640, 1024): + raise ValueError(f"resolution must be 640 or 1024, got {resolution}") + if layers < 1: + raise ValueError(f"layers must be >= 1, got {layers}") + if (height is None) != (width is None): + raise ValueError("height and width must be set together for QwenImageLayeredPipeline.") + + pipeline_start = time.time() + timer = CudaPhaseTimer() + timer.mark_pre_start() + image = self._load_image_input(image) + + device = self.device + generator = torch.Generator(device=device).manual_seed(seed) + is_latent_image = self._is_layered_latent_image(image) + if ( + isinstance(image, torch.Tensor) + and image.ndim >= 2 + and image.shape[1] == self.latent_channels + and not is_latent_image + ): + raise ValueError( + "Layered latent image inputs must have shape (B, C, F, H, W), " + f"got {tuple(image.shape)}." + ) + multiple_of = self.vae_scale_factor * 2 + + if is_latent_image: + self._validate_single_conditioning_frame(image) + if image.shape[-2] % 2 != 0 or image.shape[-1] % 2 != 0: + raise ValueError( + "Layered latent image spatial dimensions must be even for 2x2 packing, " + f"got H={image.shape[-2]}, W={image.shape[-1]}." + ) + image = image.to(dtype=self.dtype, device=device) + calculated_width = int(image.shape[-1]) * self.vae_scale_factor + calculated_height = int(image.shape[-2]) * self.vae_scale_factor + prompt_image = None + else: + if height is None or width is None: + image_width, image_height = self._image_size(image) + calculated_width, calculated_height = _calculate_dimensions( + resolution * resolution, + image_width / image_height, + ) + else: + calculated_width = width + calculated_height = height + if not hasattr(self, "image_processor"): + from diffusers.image_processor import VaeImageProcessor + + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + image = self.image_processor.resize(image, calculated_height, calculated_width) + prompt_image = image + image = self.image_processor.preprocess(image, calculated_height, calculated_width) + image = image.unsqueeze(2).to(dtype=self.dtype, device=device) + + width = calculated_width // multiple_of * multiple_of + height = calculated_height // multiple_of * multiple_of + + image_batch = int(image.shape[0]) + prompt = [prompt] if isinstance(prompt, str) else list(prompt) + prompt = self._align_prompts_to_image_batch(prompt, image_batch) + needs_caption = any(self._is_empty_prompt(item) for item in prompt) + if needs_caption: + if prompt_image is None: + raise ValueError( + "Automatic image captioning requires a non-latent image input; provide prompt " + "when image is already a latent tensor." + ) + captions = self.get_image_caption( + prompt_image, + use_en_prompt=use_en_prompt, + device=device, + ) + captions = self._expand_values_to_batch(captions, len(prompt), "caption") + prompt = [ + caption if self._is_empty_prompt(item) else item + for item, caption in zip(prompt, captions, strict=True) + ] + batch_size = len(prompt) + + has_neg = negative_prompt is not None + do_true_cfg = true_cfg_scale > 1.0 and has_neg + logger.info("Encoding layered prompt...") + prompt_embeds, prompt_embeds_mask = self._encode_prompt(prompt, device, max_sequence_length) + neg_prompt_embeds = neg_prompt_embeds_mask = None + if do_true_cfg: + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] * batch_size + elif len(negative_prompt) == 1: + negative_prompt = negative_prompt * batch_size + elif len(negative_prompt) != batch_size: + raise ValueError( + "negative_prompt must be a string, a singleton list, " + "or a list with the same effective batch size as prompt" + ) + neg_prompt_embeds, neg_prompt_embeds_mask = self._encode_prompt( + negative_prompt, + device, + max_sequence_length, + ) + + num_channels_latents = self.transformer.in_channels // 4 + latents, image_latents = self._prepare_layered_latents( + image, + batch_size, + num_channels_latents, + height, + width, + layers, + prompt_embeds.dtype, + device, + generator, + latents, + ) + img_shapes = [ + [ + *[ + ( + 1, + height // self.vae_scale_factor // 2, + width // self.vae_scale_factor // 2, + ) + for _ in range(layers + 1) + ], + ( + 1, + calculated_height // self.vae_scale_factor // 2, + calculated_width // self.vae_scale_factor // 2, + ), + ] + ] * batch_size + + sigmas_np = ( + sigmas if sigmas is not None else np.linspace(1.0, 0, num_inference_steps + 1)[:-1] + ) + base_seqlen = 256 * 256 / 16 / 16 + mu = (image_latents.shape[1] / base_seqlen) ** 0.5 + self.scheduler.set_timesteps(sigmas=sigmas_np, device=device, mu=mu) + timesteps = self.scheduler.timesteps + self.scheduler.set_begin_index(0) + + additional_t_cond = torch.zeros(batch_size, device=device, dtype=torch.long) + timer.mark_denoise_start() + logger.info("Denoising layered output (%d steps)...", len(timesteps)) + for t in timesteps: + latent_model_input = torch.cat([latents, image_latents], dim=1) + timestep = t.expand(latents.shape[0]).to(latents.dtype) + noise_pred = self.transformer( + hidden_states=latent_model_input, + timestep=timestep / 1000, + encoder_hidden_states_mask=prompt_embeds_mask, + encoder_hidden_states=prompt_embeds, + img_shapes=img_shapes, + additional_t_cond=additional_t_cond, + return_dict=False, + )[0] + noise_pred = noise_pred[:, : latents.size(1)] + + if do_true_cfg: + neg_noise_pred = self.transformer( + hidden_states=latent_model_input, + timestep=timestep / 1000, + encoder_hidden_states_mask=neg_prompt_embeds_mask, + encoder_hidden_states=neg_prompt_embeds, + img_shapes=img_shapes, + additional_t_cond=additional_t_cond, + return_dict=False, + )[0] + neg_noise_pred = neg_noise_pred[:, : latents.size(1)] + comb = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + if cfg_normalize: + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb, dim=-1, keepdim=True) + noise_pred = comb * (cond_norm / noise_norm) + else: + noise_pred = comb + + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + if latents.dtype != latents_dtype: + latents = latents.to(latents_dtype) + + timer.mark_post_start() + logger.info("Decoding layered output...") + layer_stack = self._decode_layered_latents(latents, height, width, layers) + if getattr(self, "rank", 0) == 0: + logger.info("Layered pipeline total: %.2fs", time.time() - pipeline_start) + + timer.mark_end() + image_grid = self._layer_stack_to_image_grid(layer_stack) + return timer.fill(PipelineOutput(image=image_grid)) diff --git a/tensorrt_llm/_torch/visual_gen/models/qwen_image_layered/transformer_qwen_image_layered.py b/tensorrt_llm/_torch/visual_gen/models/qwen_image_layered/transformer_qwen_image_layered.py new file mode 100644 index 000000000000..0968f3420a91 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/qwen_image_layered/transformer_qwen_image_layered.py @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Qwen-Image-Layered transformer variants.""" + +from __future__ import annotations + +import functools +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple + +import torch + +from ..qwen_image.transformer_qwen_image import ( + QwenEmbedRope, + QwenImageTransformer2DModel, + QwenTimestepProjEmbeddings, +) + +if TYPE_CHECKING: + from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig + from tensorrt_llm._torch.visual_gen.cuda_graph_runner import CUDAGraphRunner + + +class QwenEmbedLayer3DRope(QwenEmbedRope): + """Layer-aware 3D RoPE used by Qwen-Image-Layered. + + The layered checkpoint represents generated RGBA layers followed by + one conditioning image in ``img_shapes``. Generated layers use their + layer index as the frame-axis RoPE offset; the conditioning image uses + the negative frame index from diffusers' reference implementation. + """ + + def forward( + self, + video_fhw, + max_txt_seq_len: int | torch.Tensor, + device: Optional[torch.device] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + if isinstance(video_fhw, list): + video_fhw = video_fhw[0] + if not isinstance(video_fhw, list): + video_fhw = [video_fhw] + + vid_freqs = [] + max_vid_index = 0 + condition_index = len(video_fhw) - 1 + for idx, fhw in enumerate(video_fhw): + frame, height, width = fhw + if idx == condition_index: + video_freq = self._compute_condition_freqs(frame, height, width, device) + else: + video_freq = self._compute_video_freqs(frame, height, width, idx, device) + vid_freqs.append(video_freq) + + if self.scale_rope: + max_vid_index = max(height // 2, width // 2, max_vid_index) + else: + max_vid_index = max(height, width, max_vid_index) + + max_vid_index = max(max_vid_index, condition_index) + max_txt_seq_len_int = int(max_txt_seq_len) + txt_freqs = self._pos_freqs_for_device(device)[ + max_vid_index : max_vid_index + max_txt_seq_len_int, ... + ] + vid_freqs = torch.cat(vid_freqs, dim=0) + return vid_freqs, txt_freqs + + @functools.lru_cache(maxsize=128) + def _compute_condition_freqs( + self, + frame: int, + height: int, + width: int, + device: Optional[torch.device] = None, + ) -> torch.Tensor: + seq_lens = frame * height * width + pos_freqs = self._pos_freqs_for_device(device) + neg_freqs = self.neg_freqs.to(device) if device is not None else self.neg_freqs + + freqs_pos = pos_freqs.split([x // 2 for x in self.axes_dim], dim=1) + freqs_neg = neg_freqs.split([x // 2 for x in self.axes_dim], dim=1) + + freqs_frame = freqs_neg[0][-1:].view(frame, 1, 1, -1).expand(frame, height, width, -1) + if self.scale_rope: + freqs_height = torch.cat( + [freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]], + dim=0, + ) + freqs_height = freqs_height.view(1, height, 1, -1).expand(frame, height, width, -1) + freqs_width = torch.cat( + [freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]], + dim=0, + ) + freqs_width = freqs_width.view(1, 1, width, -1).expand(frame, height, width, -1) + else: + freqs_height = ( + freqs_pos[1][:height].view(1, height, 1, -1).expand(frame, height, width, -1) + ) + freqs_width = ( + freqs_pos[2][:width].view(1, 1, width, -1).expand(frame, height, width, -1) + ) + + freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-1).reshape(seq_lens, -1) + return freqs.clone().contiguous() + + +class QwenImageLayeredTransformer2DModel(QwenImageTransformer2DModel): + """Qwen-Image transformer variant for RGBA layer decomposition.""" + + def __init__( + self, + model_config: Optional["DiffusionModelConfig"] = None, + *, + patch_size: int = 2, + in_channels: int = 64, + out_channels: Optional[int] = 16, + num_layers: int = 60, + attention_head_dim: int = 128, + num_attention_heads: int = 24, + joint_attention_dim: int = 3584, + axes_dims_rope: Tuple[int, int, int] = (16, 56, 56), + use_additional_t_cond: bool = False, + use_layer3d_rope: bool = False, + attn_backend: str = "sdpa", + ): + super().__init__( + model_config=model_config, + patch_size=patch_size, + in_channels=in_channels, + out_channels=out_channels, + num_layers=num_layers, + attention_head_dim=attention_head_dim, + num_attention_heads=num_attention_heads, + joint_attention_dim=joint_attention_dim, + axes_dims_rope=axes_dims_rope, + attn_backend=attn_backend, + ) + if use_layer3d_rope: + self.pos_embed = QwenEmbedLayer3DRope( + theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True + ) + if use_additional_t_cond: + self.time_text_embed = QwenTimestepProjEmbeddings( + embedding_dim=self.inner_dim, + use_additional_t_cond=True, + ) + + @staticmethod + def _normalize_img_shapes_for_cuda_graph(*args, **kwargs) -> Optional[Tuple]: + img_shapes = kwargs.get("img_shapes") + if img_shapes is None and len(args) > 4: + img_shapes = args[4] + if img_shapes is None: + return None + + def normalize(value): + if isinstance(value, (list, tuple)): + return tuple(normalize(item) for item in value) + return int(value) + + return normalize(img_shapes) + + def register_cuda_graph_extra_key_fns(self, runner: "CUDAGraphRunner") -> None: + super().register_cuda_graph_extra_key_fns(runner) + runner.register_extra_key_fn("img_shapes", self._normalize_img_shapes_for_cuda_graph) + + @classmethod + def from_config_dict( + cls, cfg: Dict[str, Any], **kwargs + ) -> "QwenImageLayeredTransformer2DModel": + """Build from a transformer/config.json dict.""" + return cls( + patch_size=cfg.get("patch_size", 2), + in_channels=cfg.get("in_channels", 64), + out_channels=cfg.get("out_channels", 16), + num_layers=cfg.get("num_layers", 60), + attention_head_dim=cfg.get("attention_head_dim", 128), + num_attention_heads=cfg.get("num_attention_heads", 24), + joint_attention_dim=cfg.get("joint_attention_dim", 3584), + axes_dims_rope=tuple(cfg.get("axes_dims_rope", [16, 56, 56])), + use_additional_t_cond=cfg.get("use_additional_t_cond", False), + use_layer3d_rope=cfg.get("use_layer3d_rope", False), + **kwargs, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + encoder_hidden_states_mask: Optional[torch.Tensor] = None, + timestep: Optional[torch.Tensor] = None, + img_shapes: Optional[list] = None, + txt_seq_lens: Optional[list] = None, + additional_t_cond: Optional[torch.Tensor] = None, + return_dict: bool = False, + **kwargs, + ): + """Forward pass with optional Qwen-Image-Layered timestep condition.""" + del kwargs, txt_seq_lens # Only kept for diffusers API compat. + missing = [] + if timestep is None: + missing.append("timestep") + if img_shapes is None: + missing.append("img_shapes") + if missing: + raise ValueError(f"Missing required argument(s): {', '.join(missing)}") + + hidden_states = self.img_in(hidden_states) + timestep = timestep.to(hidden_states.dtype) + + encoder_hidden_states = self.txt_norm(encoder_hidden_states) + encoder_hidden_states = self.txt_in(encoder_hidden_states) + + text_seq_len = encoder_hidden_states.shape[1] + temb = self.time_text_embed(timestep, hidden_states, additional_t_cond) + image_rotary_emb = self.pos_embed( + img_shapes, max_txt_seq_len=text_seq_len, device=hidden_states.device + ) + + block_attention_mask = None + if encoder_hidden_states_mask is not None: + if encoder_hidden_states_mask.dtype != torch.bool: + encoder_hidden_states_mask = encoder_hidden_states_mask.to(torch.bool) + batch_size, image_seq_len = hidden_states.shape[:2] + image_mask = torch.ones( + (batch_size, image_seq_len), + dtype=torch.bool, + device=hidden_states.device, + ) + block_attention_mask = torch.cat([encoder_hidden_states_mask, image_mask], dim=1) + + for block in self.transformer_blocks: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + attention_mask=block_attention_mask, + timestep=timestep, + ) + + hidden_states = self.norm_out(hidden_states, temb) + output = self.proj_out(hidden_states) + + if return_dict: + from diffusers.models.modeling_outputs import Transformer2DModelOutput + + return Transformer2DModelOutput(sample=output) + return (output,) diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py index 3f362ea5c53d..8230f10823e5 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py @@ -345,16 +345,14 @@ def post_load_weights(self) -> None: if not self.is_wan22_14b: self._apply_teacache_coefficients(WAN_TEACACHE_COEFFICIENTS) - self._setup_cache_acceleration() - else: - if self.pipeline_config.cache_backend == "cache_dit": - self._setup_cache_acceleration() if self.transformer_2 is not None: if hasattr(self.transformer_2, "post_load_weights"): self.transformer_2.post_load_weights() - # Wan 2.2 TeaCache after both transformers' post_load_weights (FP8 scales, etc.) + # Wan 2.2 TeaCache validation after both transformers' post_load_weights + # (FP8 scales, etc.). Cache acceleration itself is enabled by the loader + # after torch.compile (see PipelineLoader.load). if ( self.transformer is not None and self.transformer_2 is not None @@ -367,7 +365,6 @@ def post_load_weights(self) -> None: "teacache.coefficients_2 (high-noise and low-noise stage polynomials). " "There is no built-in coefficient table for Wan 2.2." ) - self._setup_cache_acceleration() def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> None: with torch.no_grad(): diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py index eeef329ccac9..f83d5be29f1b 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py @@ -356,15 +356,13 @@ def post_load_weights(self) -> None: if not self.is_wan22_14b: self._apply_teacache_coefficients(WAN_I2V_TEACACHE_COEFFICIENTS) - self._setup_cache_acceleration() - else: - if self.pipeline_config.cache_backend == "cache_dit": - self._setup_cache_acceleration() if self.transformer_2 is not None: if hasattr(self.transformer_2, "post_load_weights"): self.transformer_2.post_load_weights() + # Wan 2.2 TeaCache validation; cache acceleration itself is enabled by + # the loader after torch.compile (see PipelineLoader.load). if ( self.transformer is not None and self.transformer_2 is not None @@ -377,7 +375,6 @@ def post_load_weights(self) -> None: "teacache.coefficients_2 (high-noise and low-noise stage polynomials). " "There is no built-in coefficient table for Wan 2.2." ) - self._setup_cache_acceleration() def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> None: dummy_image = PIL.Image.new("RGB", (width, height)) diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py b/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py index 71852014121c..057e088138cd 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py @@ -1,5 +1,5 @@ import math -from typing import Tuple +from typing import Optional, Tuple import torch import torch.nn as nn @@ -11,9 +11,14 @@ from tensorrt_llm._torch.modules.layer_norm import LayerNorm from tensorrt_llm._torch.modules.linear import Linear, TensorParallelMode from tensorrt_llm._torch.modules.mlp import MLP -from tensorrt_llm._torch.utils import gelu_tanh +from tensorrt_llm._torch.utils import Fp4QuantizedTensor, gelu_tanh from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig from tensorrt_llm._torch.visual_gen.models.modeling import BaseDiffusionModel +from tensorrt_llm._torch.visual_gen.models.wan.utils_wan import ( + apply_fused_layernorm_adaln_quant, + apply_fused_layernorm_affine_quant, + get_nvfp4_input_scale, +) from tensorrt_llm._torch.visual_gen.modules.attention import Attention, QKVMode from tensorrt_llm._torch.visual_gen.modules.rms_norm import RMSNormTPAware from tensorrt_llm._torch.visual_gen.quantization.loader import DynamicLinearWeightLoader @@ -29,6 +34,7 @@ def get_parameter_device(module): return next(module.parameters()).device + # ========================================================================= # 1. Rotary Positional Embeddings # ========================================================================= @@ -359,6 +365,13 @@ def __init__( hidden_size=hidden_size, eps=eps, dtype=torch.float32, has_weights=False, has_bias=False ) + # FP4 input scales propagated from downstream Linear modules in post_load_weights(). + # None until post_load_weights is called; fusion is skipped when None. + self._norm1_fp4_scale: Optional[torch.Tensor] = None + self._norm2_fp4_scale: Optional[torch.Tensor] = None + self._norm3_fp4_scale: Optional[torch.Tensor] = None + self._fused_ln_supported = hidden_size == 5120 + self.ffn = MLP( hidden_size=hidden_size, intermediate_size=ffn_dim, @@ -425,6 +438,7 @@ def __init__( force_dynamic_quantization=force_dynamic_quant, tensor_parallel_mode=tp_mode, reduce_output=False, + override_tp_sharding=(self.attn2.local_kv_dim_start, self.attn2.local_kv_dim_end), ) self.add_v_proj = Linear( added_kv_proj_dim, @@ -436,6 +450,7 @@ def __init__( force_dynamic_quantization=force_dynamic_quant, tensor_parallel_mode=tp_mode, reduce_output=False, + override_tp_sharding=(self.attn2.local_kv_dim_start, self.attn2.local_kv_dim_end), ) self.norm_added_k = RMSNormTPAware( hidden_size=hidden_size, @@ -444,6 +459,7 @@ def __init__( has_weights=True, enable_tp=(tp_size > 1), mapping=model_config.mapping, + override_tp_sharding=(self.attn2.local_kv_dim_start, self.attn2.local_kv_dim_end), ) # Use torch.empty().normal_(std=...) instead of torch.randn()/scale for MetaInitMode compatibility @@ -451,6 +467,48 @@ def __init__( torch.empty(1, 6, hidden_size).normal_(std=hidden_size**-0.5) ) + def _fused_adaln_quant(self, x, scale_msa, shift_msa, temb, fp4_scale, eps): + """Shared norm1/norm3 path: flatten x to 2D, build the per-token or + per-batch modulation rows, and run the fused LayerNorm+AdaLN+NVFP4 op. + + Returns the 2D fused result (Fp4QuantizedTensor when quantizing, + else a dense tensor). The caller reshapes it back to 3D (norm1) or + feeds it straight to the MLP's 2D fused-GELU kernel (norm3). + """ + _x_2d = x.reshape(-1, x.shape[-1]) + if temb.ndim == 4: + # scale/shift are [B, S, D] here; flatten per-token so each row of + # _x_2d gets its own modulation row (seq_len_per_batch=1). + _scale_2d = scale_msa.reshape(-1, scale_msa.shape[-1]) + _shift_2d = shift_msa.reshape(-1, shift_msa.shape[-1]) + _seq_len_per_batch = 1 + else: + # scale/shift are [B, D] here; one modulation row per batch element. + _batch_size = temb.shape[0] + _scale_2d = scale_msa.reshape(_batch_size, -1) + _shift_2d = shift_msa.reshape(_batch_size, -1) + _seq_len_per_batch = _x_2d.shape[0] // _batch_size + return apply_fused_layernorm_adaln_quant( + _x_2d, + _scale_2d, + _shift_2d, + _seq_len_per_batch, + fp4_scale, + eps=eps, + ) + + @staticmethod + def _reshape_fused_output(normed, shape): + """Reshape a fused-op output (Fp4QuantizedTensor or dense) from the + 2D op layout back to shape.""" + if isinstance(normed, Fp4QuantizedTensor): + return Fp4QuantizedTensor( + normed.fp4_tensor.reshape(*shape[:-1], normed.fp4_tensor.shape[-1]), + normed.scaling_factor, + normed.is_sf_swizzled, + ) + return normed.reshape(shape) + def forward( self, x, @@ -478,8 +536,15 @@ def forward( self.scale_shift_table.float() + temb.float() ).chunk(6, dim=1) - normed = self.norm1(x.float()) * (1 + scale_msa) + shift_msa - normed = normed.to(x.dtype) + if self._fused_ln_supported: + # x is [B, S, D]; flatten to 2D for the fused op, reshape output back. + normed = self._fused_adaln_quant( + x, scale_msa, shift_msa, temb, self._norm1_fp4_scale, self.norm1.variance_epsilon + ) + normed = self._reshape_fused_output(normed, x.shape) + else: + normed = self.norm1(x.float()) * (1 + scale_msa) + shift_msa + normed = normed.to(x.dtype) # Prepare frequencies for Attention freqs = (freqs_cos, freqs_sin) if freqs_cos is not None and freqs_sin is not None else None @@ -500,7 +565,22 @@ def forward( x = (x.float() + attn1_out.float() * gate_msa).to(x.dtype) - norm_x = self.norm2(x.float()).to(x.dtype) + if ( + self._fused_ln_supported + and isinstance(self.norm2, LayerNorm) + and self.norm2.weight is not None + ): + _x_2d = x.reshape(-1, x.shape[-1]) + norm_x = apply_fused_layernorm_affine_quant( + _x_2d, + self.norm2.weight.to(x.dtype), + self.norm2.bias.to(x.dtype), + self._norm2_fp4_scale, + eps=self.norm2.variance_epsilon, + ) + norm_x = self._reshape_fused_output(norm_x, x.shape) + else: + norm_x = self.norm2(x.float()).to(x.dtype) # I2V: Split encoder_hidden_states into image and text parts if needed encoder_hidden_states_img = None @@ -543,11 +623,24 @@ def forward( # Apply to_out once to the combined (text + image) attention output x = x + self.attn2.to_out[0](attn2_output) - # 3. Feed-forward - normed = self.norm3(x.float()) * (1 + c_scale_msa) + c_shift_msa - normed = normed.to(x.dtype) + # 3. Feed-forward. Mirrors norm1: fused LN+AdaLN (with optional NVFP4 + # quant) reshaped back to [B, S, D]; self.ffn consumes it. + if self._fused_ln_supported: + normed = self._fused_adaln_quant( + x, + c_scale_msa, + c_shift_msa, + temb, + self._norm3_fp4_scale, + self.norm3.variance_epsilon, + ) + normed = self._reshape_fused_output(normed, x.shape) + else: + normed = self.norm3(x.float()) * (1 + c_scale_msa) + c_shift_msa + normed = normed.to(x.dtype) + ffn_out = self.ffn(normed) - x = (x.float() + self.ffn(normed).float() * c_gate_msa).to(x.dtype) + x = (x.float() + ffn_out.float() * c_gate_msa).to(x.dtype) return x @@ -900,3 +993,13 @@ def post_load_weights(self) -> None: for _, module in self.named_modules(): if isinstance(module, Linear): module.post_load_weights() + + # Wire each norm's fp4_scale from the first downstream Linear that consumes its output. + for block in self.blocks: + if not isinstance(block, WanBlock): + continue + # qkv_proj exists in FUSE_QKV mode; fall back to to_q in SEPARATE_QKV (async Ulysses). + attn1_qkv = getattr(block.attn1, "qkv_proj", None) or getattr(block.attn1, "to_q", None) + block._norm1_fp4_scale = get_nvfp4_input_scale(attn1_qkv) + block._norm2_fp4_scale = get_nvfp4_input_scale(getattr(block.attn2, "to_q", None)) + block._norm3_fp4_scale = get_nvfp4_input_scale(getattr(block.ffn, "up_proj", None)) diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/utils_wan.py b/tensorrt_llm/_torch/visual_gen/models/wan/utils_wan.py new file mode 100644 index 000000000000..ab47fc889ff6 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/models/wan/utils_wan.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Wan 2.2 fused DiT kernel helpers. + +Provides thin wrappers around the trtllm::fused_adaptive_layernorm[_quant] +custom ops used for the three LayerNorm sites in WanBlock. +""" + +from typing import Optional, Union + +import torch + +from tensorrt_llm._torch.utils import Fp4QuantizedTensor + + +def get_nvfp4_input_scale(linear) -> Optional[torch.Tensor]: + """Return the calibrated NVFP4 input_scale for a Linear, or None if not applicable. + + Returns None when the linear is not NVFP4-quantized, uses a non-16 group size, + has an AWQ pre_quant_scale that must be folded into bf16 input first, or uses + dynamic quantization (which recomputes input_scale per-forward and cannot + consume a statically pre-quantized FP4 tensor). + """ + if linear is None: + return None + scale = getattr(linear, "input_scale", None) + if scale is None: + return None + if getattr(linear, "scaling_vector_size", None) != 16: + return None + if getattr(linear, "pre_quant_scale", None) is not None: + return None + if getattr(linear, "force_dynamic_quantization", False): + return None + return scale + + +def apply_fused_layernorm_adaln_quant( + x: torch.Tensor, + scale_msa: torch.Tensor, + shift_msa: torch.Tensor, + seq_len_per_batch: int, + fp4_input_scale: Optional[torch.Tensor], + eps: float = 1e-6, +) -> Union[torch.Tensor, "Fp4QuantizedTensor"]: + """Fused LayerNorm + AdaLN (y = (1 + scale_msa) * x_hat + shift_msa) + optional NVFP4 quant. + + Used for norm1 and norm3 in WanBlock (no learned affine params; modulation from timestep emb). + Returns Fp4QuantizedTensor when fp4_input_scale is provided, else a bf16 tensor. + """ + # .contiguous() handles non-contiguous views (chunk/squeeze/reshape patterns) and + # transposed layouts injected by torch.compile's inductor memory planner. + x = x.contiguous() + scale_msa = scale_msa.to(dtype=x.dtype).contiguous() + shift_msa = shift_msa.to(dtype=x.dtype).contiguous() + if fp4_input_scale is not None: + y_fp4, sf_out = torch.ops.trtllm.fused_adaptive_layernorm_quant( + x, None, None, scale_msa, shift_msa, fp4_input_scale, seq_len_per_batch, eps + ) + return Fp4QuantizedTensor(y_fp4, sf_out) + out = torch.ops.trtllm.fused_adaptive_layernorm( + x, None, None, scale_msa, shift_msa, seq_len_per_batch, eps + ) + return out + + +def apply_fused_layernorm_affine_quant( + x: torch.Tensor, + ln_weight: torch.Tensor, + ln_bias: torch.Tensor, + fp4_input_scale: Optional[torch.Tensor], + eps: float = 1e-6, +) -> Union[torch.Tensor, "Fp4QuantizedTensor"]: + """Fused LayerNorm + affine (learned weight/bias) + optional NVFP4 quant. + + Used for norm2 in WanBlock (learned LN params; no AdaLN modulation). + Returns Fp4QuantizedTensor when fp4_input_scale is provided, else a bf16 tensor. + """ + x = x.contiguous() + # seq_len_per_batch is unused on the affine path (kernel only reads it under HAS_MODULATION). + # Pass 0 so any future kernel change that accidentally reads it here fails loudly. + seq_len_per_batch = 0 + if fp4_input_scale is not None: + y_fp4, sf_out = torch.ops.trtllm.fused_adaptive_layernorm_quant( + x, + ln_weight.to(x.dtype), + ln_bias.to(x.dtype), + None, + None, + fp4_input_scale, + seq_len_per_batch, + eps, + ) + return Fp4QuantizedTensor(y_fp4, sf_out) + out = torch.ops.trtllm.fused_adaptive_layernorm( + x, ln_weight.to(x.dtype), ln_bias.to(x.dtype), None, None, seq_len_per_batch, eps + ) + return out diff --git a/tensorrt_llm/_torch/visual_gen/modules/attention.py b/tensorrt_llm/_torch/visual_gen/modules/attention.py index 58b61a280b02..d551720cb4e6 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/attention.py +++ b/tensorrt_llm/_torch/visual_gen/modules/attention.py @@ -4,7 +4,6 @@ import torch import torch.nn as nn -from tensorrt_llm.logger import logger from tensorrt_llm.visual_gen.sparse_attention import SkipSoftmaxAttentionConfig from ...modules.linear import Linear, TensorParallelMode, WeightMode, WeightsLoadingConfig @@ -57,6 +56,7 @@ def __init__( module_name: Optional[str] = None, enable_sequence_parallel: bool = True, async_ulysses: bool = False, + separate_qkv_is_self_attention: bool = False, ): super().__init__() @@ -76,10 +76,7 @@ def __init__( self.bias = bias self.tp_size = self.mapping.tp_size if self.mapping else 1 - assert ( - self.num_attention_heads % self.tp_size == 0 - and self.num_key_value_heads % self.tp_size == 0 - ), "TP size must divide the number of Query and KV Heads" + self.tp_rank = self.mapping.tp_rank if self.mapping else 0 # Fused QK Norm + RoPE: each model class opts in via fuse_qk_norm_rope. # Backed by torch.ops.trtllm.fused_dit_qk_norm_rope which auto-dispatches: @@ -127,11 +124,7 @@ def __init__( self.q_dim = self.num_attention_heads * self.head_dim self.kv_dim = self.num_key_value_heads * self.head_dim - self.local_num_attention_heads = self.num_attention_heads // self.tp_size - self.local_num_key_value_heads = self.num_key_value_heads // self.tp_size - self.local_q_dim = self.local_num_attention_heads * self.head_dim - self.local_kv_dim = self.local_num_key_value_heads * self.head_dim - + self._calculate_tp_parameters(ulysses_size if enable_sequence_parallel else None) self._init_qkv_proj() # Structural eligibility for SEPARATE_QKV self-attn quantize dedup. @@ -157,6 +150,12 @@ def __init__( q_norm_dim = self.head_dim if qk_norm_mode == "per_head" else self.q_dim k_norm_dim = self.head_dim if qk_norm_mode == "per_head" else self.kv_dim enable_tp_rms = self.tp_size > 1 and qk_norm_mode == "full" + + q_start = self.local_q_dim_start + q_end = self.local_q_dim_end + k_start = self.local_kv_dim_start + k_end = self.local_kv_dim_end + self.norm_q = RMSNormTPAware( hidden_size=q_norm_dim, eps=self.eps, @@ -164,6 +163,7 @@ def __init__( has_weights=True, enable_tp=enable_tp_rms, mapping=self.mapping, + override_tp_sharding=(q_start, q_end) if qk_norm_mode == "full" else None, ) self.norm_k = RMSNormTPAware( hidden_size=k_norm_dim, @@ -172,6 +172,7 @@ def __init__( has_weights=True, enable_tp=enable_tp_rms, mapping=self.mapping, + override_tp_sharding=(k_start, k_end) if qk_norm_mode == "full" else None, ) # TODO: Use weight mapper to create just a Linear module @@ -189,30 +190,16 @@ def __init__( tensor_parallel_mode=TensorParallelMode.ROW if self.tp_size > 1 else None, reduce_output=(self.tp_size > 1), allreduce_strategy=self.allreduce_strategy, + override_tp_sharding=(self.local_q_dim_start, self.local_q_dim_end), ) ] ) - # Ulysses auto-wrap normally skips SEPARATE_QKV (cross-attention). - # The async-ulysses path uses SEPARATE_QKV for stream-pipelined - # V/Q/K projections AND still needs the head-sharding wrap — opt in - # via async_ulysses=True. - cp_size = vgm.cp_size if vgm else 1 - use_ulysses = ( - ulysses_size > 1 - and enable_sequence_parallel - and (self.qkv_mode != QKVMode.SEPARATE_QKV or async_ulysses or cp_size == 1) - ) - if ulysses_size > 1 and enable_sequence_parallel and not use_ulysses: - # Ulysses was requested (ulysses_size > 1, SP on) but disabled: this is a - # SEPARATE_QKV cross-attention that is neither async nor pure-Ulysses - # (cp_size > 1), so it falls back to the all-gather K/V path. - logger.debug( - f"Attention(layer={layer_idx}): Ulysses disabled despite ulysses_size=" - f"{ulysses_size} — qkv_mode={self.qkv_mode.value}, " - f"async_ulysses={async_ulysses}, cp_size={cp_size} " - f"(SEPARATE_QKV cross-attn needs async_ulysses or cp_size==1)." - ) + # Ulysses (head-sharding) is orthogonal to CP (sequence-sharding), so it + # composes with pure-Ulysses, Attention2D and async alike, including for + # SEPARATE_QKV cross-attn. Ring + SEPARATE_QKV is the only unsupported + # combination and is rejected below. + use_ulysses = ulysses_size > 1 and enable_sequence_parallel # Compute head counts for the backend # Ulysses shards heads across workers; inner backend sees sharded count @@ -248,7 +235,12 @@ def __init__( sparse_params=sparse_params, ) - if enable_sequence_parallel and self.qkv_mode == QKVMode.SEPARATE_QKV and vgm is not None: + if ( + enable_sequence_parallel + and self.qkv_mode == QKVMode.SEPARATE_QKV + and not separate_qkv_is_self_attention + and vgm is not None + ): ring_size = vgm.ring_size if ring_size > 1: raise ValueError( @@ -276,6 +268,41 @@ def _qualified_module_name( prefix = f"{component_name}." return module_name if module_name.startswith(prefix) else f"{prefix}{module_name}" + def _calculate_tp_parameters(self, ulysses_size: Optional[int]): + assert self.num_attention_heads % self.num_key_value_heads == 0 + gqa_ratio = self.num_attention_heads // self.num_key_value_heads + + if not ulysses_size: + ulysses_size = 1 + + assert self.num_key_value_heads % ulysses_size == 0 + assert self.num_key_value_heads // ulysses_size >= self.tp_size + + kv_heads_per_ulysses = self.num_key_value_heads // ulysses_size + self.local_key_value_head_start = ( + Linear._calc_shard(kv_heads_per_ulysses, self.tp_size, self.tp_rank) * ulysses_size + ) + self.local_key_value_head_end = ( + Linear._calc_shard(kv_heads_per_ulysses, self.tp_size, self.tp_rank + 1) * ulysses_size + ) + self.local_num_key_value_heads = ( + self.local_key_value_head_end - self.local_key_value_head_start + ) + + self.local_attention_head_start = gqa_ratio * self.local_key_value_head_start + self.local_attention_head_end = gqa_ratio * self.local_key_value_head_end + self.local_num_attention_heads = ( + self.local_attention_head_end - self.local_attention_head_start + ) + + self.local_q_dim_start = self.local_attention_head_start * self.head_dim + self.local_q_dim_end = self.local_attention_head_end * self.head_dim + self.local_q_dim = self.local_q_dim_end - self.local_q_dim_start + + self.local_kv_dim_start = self.local_key_value_head_start * self.head_dim + self.local_kv_dim_end = self.local_key_value_head_end * self.head_dim + self.local_kv_dim = self.local_kv_dim_end - self.local_kv_dim_start + def _init_qkv_proj(self) -> None: tp_mode = TensorParallelMode.COLUMN if self.tp_size > 1 else None @@ -303,6 +330,11 @@ def _init_qkv_proj(self) -> None: }, tensor_parallel_mode=tp_mode, reduce_output=False, + override_tp_sharding={ + "q": (self.local_q_dim_start, self.local_q_dim_end), + "k": (self.local_kv_dim_start, self.local_kv_dim_end), + "v": (self.local_kv_dim_start, self.local_kv_dim_end), + }, ) else: self.to_q = Linear( @@ -316,6 +348,7 @@ def _init_qkv_proj(self) -> None: force_dynamic_quantization=self.force_dynamic_quantization, tensor_parallel_mode=tp_mode, reduce_output=False, + override_tp_sharding=(self.local_q_dim_start, self.local_q_dim_end), ) self.to_k = Linear( self.hidden_size, @@ -328,6 +361,7 @@ def _init_qkv_proj(self) -> None: force_dynamic_quantization=self.force_dynamic_quantization, tensor_parallel_mode=tp_mode, reduce_output=False, + override_tp_sharding=(self.local_kv_dim_start, self.local_kv_dim_end), ) self.to_v = Linear( self.hidden_size, @@ -340,6 +374,7 @@ def _init_qkv_proj(self) -> None: force_dynamic_quantization=self.force_dynamic_quantization, tensor_parallel_mode=tp_mode, reduce_output=False, + override_tp_sharding=(self.local_kv_dim_start, self.local_kv_dim_end), ) def get_qkv( diff --git a/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py b/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py index b58239ca8b9b..76da6b2ad18e 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py +++ b/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py @@ -19,6 +19,7 @@ from torch import nn from tensorrt_llm._torch.distributed import AllReduce +from tensorrt_llm._torch.modules.linear import Linear # for Linear._calc_shard from tensorrt_llm.functional import AllReduceStrategy from tensorrt_llm.mapping import Mapping @@ -36,6 +37,7 @@ def __init__( enable_tp: bool = False, mapping: Optional[Mapping] = None, allreduce_strategy: AllReduceStrategy = AllReduceStrategy.NCCL, + override_tp_sharding: Optional[tuple] = None, ): super().__init__() @@ -45,30 +47,43 @@ def __init__( self.mapping = mapping self.enable_tp = enable_tp + self.hidden_size = hidden_size + if enable_tp: assert mapping is not None - self.full_size = hidden_size - shard = hidden_size // mapping.tp_size - start = shard * mapping.tp_rank - end = min(shard * (mapping.tp_rank + 1), hidden_size) - hidden_size = end - start + if override_tp_sharding: + self.tp_sharding = override_tp_sharding + else: + start = Linear._calc_shard(self.hidden_size, mapping.tp_size, mapping.tp_rank) + end = Linear._calc_shard(self.hidden_size, mapping.tp_size, mapping.tp_rank + 1) + self.tp_sharding = (start, end) + + start, end = self.tp_sharding + self.local_hidden_size = end - start self.allreduce = AllReduce( mapping=mapping, strategy=allreduce_strategy, dtype=torch.float32 ) else: + self.local_hidden_size = self.hidden_size self.allreduce = None if use_gemma and not has_weights: raise ValueError("has_weights must be True if use_gemma is True") if has_weights: if not use_gemma: - self.weight = nn.Parameter(torch.ones(hidden_size, dtype=dtype, device=device)) + self.weight = nn.Parameter( + torch.ones(self.local_hidden_size, dtype=dtype, device=device) + ) else: - self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype, device=device)) + self.weight = nn.Parameter( + torch.zeros(self.local_hidden_size, dtype=dtype, device=device) + ) else: self.register_buffer( - "weight", torch.ones(hidden_size, dtype=dtype, device=device), persistent=False + "weight", + torch.ones(self.local_hidden_size, dtype=dtype, device=device), + persistent=False, ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -78,7 +93,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: x2 = hidden_states.pow(2) if self.allreduce: x2_sum = x2.sum(-1, keepdim=True) - variance = self.allreduce(x2_sum) / self.full_size + variance = self.allreduce(x2_sum) / self.hidden_size else: variance = x2.mean(-1, keepdim=True) @@ -95,9 +110,7 @@ def load_weights(self, weights: torch.Tensor): if param is None or param_name not in weights: continue if param_name == "weight" and self.enable_tp: - shard = self.full_size // self.mapping.tp_size - start = shard * self.mapping.tp_rank - end = min(shard * (self.mapping.tp_rank + 1), self.full_size) + start, end = self.tp_sharding data = weights[param_name][..., start:end] else: data = weights[param_name] diff --git a/tensorrt_llm/_torch/visual_gen/pipeline.py b/tensorrt_llm/_torch/visual_gen/pipeline.py index 13cbe77bdf12..c930b66d7c85 100644 --- a/tensorrt_llm/_torch/visual_gen/pipeline.py +++ b/tensorrt_llm/_torch/visual_gen/pipeline.py @@ -1,3 +1,4 @@ +import contextlib import itertools import os import time @@ -8,6 +9,7 @@ import torch.nn as nn from pydantic import Field +from tensorrt_llm._torch.autotuner import autotune from tensorrt_llm._torch.visual_gen.pipeline_registry import PipelineComponent from tensorrt_llm._utils import nvtx_range from tensorrt_llm.llmapi.utils import StrictBaseModel @@ -17,6 +19,7 @@ from .cache import CacheDiTAccelerator, TeaCacheAccelerator from .checkpoints import WeightLoader from .cuda_graph_runner import CUDAGraphRunner, CUDAGraphRunnerConfig, SharedGraphPool +from .mapping import _VisualGenAutotuneDist from .modules.vae.parallel_vae_interface import ParallelVAEFactory @@ -206,6 +209,18 @@ def _setup_cuda_graphs(self): model.forward = runner.wrap(model.forward) self._cuda_graph_runners[name] = runner + @contextlib.contextmanager + def disallow_cuda_graph_capture(self): + """Run wrapped forwards eagerly instead of capturing new CUDA graphs.""" + prev = {name: r.allow_capture for name, r in self._cuda_graph_runners.items()} + for r in self._cuda_graph_runners.values(): + r.allow_capture = False + try: + yield + finally: + for name, r in self._cuda_graph_runners.items(): + r.allow_capture = prev[name] + @property def rank(self): return dist.get_rank() if dist.is_initialized() else 0 @@ -236,6 +251,14 @@ def warmup_cache_key(self, height: int, width: int, num_frames: int) -> tuple: """ return (height, width, num_frames) + def request_warmup_cache_key(self, req: Any) -> tuple: + """Return the warmup cache key for a prepared inference request.""" + return self.warmup_cache_key( + req.params.height, + req.params.width, + num_frames=req.params.num_frames, + ) + @property def default_warmup_resolutions(self) -> List[Tuple[int, int]]: """Model-specific default warmup resolutions (height, width). @@ -353,6 +376,14 @@ def default_generation_params(self) -> dict: """ return {} + def prepare_request(self, req: Any) -> None: + """Prepare model-specific inputs before warmup bookkeeping. + + Subclasses may mutate internal request state and resolve request + parameters needed by :meth:`request_warmup_cache_key`. The default + implementation is a no-op. + """ + def infer(self, req: Any): raise NotImplementedError @@ -630,12 +661,14 @@ def _find_transformer_blocks(model: nn.Module) -> list: def warmup(self) -> None: """Run warmup inference to trigger torch.compile and CUDA initialization. - Resolves warmup shapes from user config or model defaults, then runs - a short denoising loop with dummy inputs for each shape. This: - 1. Triggers torch.compile's lazy compilation (first forward trace + codegen) - 2. Pre-captures CUDA graphs (if enabled) - 3. Warms up CUDA kernels and allocators - 4. Populates any lazy caches (e.g., RoPE frequencies) + Resolves warmup shapes from user config or model defaults, then runs a + short denoising loop with dummy inputs for each shape, triggering + torch.compile, CUDA graph capture (if enabled), and autotuner tuning. + + With autotuning enabled, a single rank tunes and captures in one pass. + On multiple ranks, tactics are tuned with capture off, merged across + ranks at ``autotune()`` exit, then recaptured — so every rank bakes the + same tactic into its graphs. Called automatically by PipelineLoader after model loading and torch.compile. OOM is not caught — if a warmup shape OOMs, the server fails fast at startup. @@ -645,18 +678,49 @@ def warmup(self) -> None: logger.info("Warmup disabled (no warmup shapes)") return + shape_list = ", ".join(f"{h}x{w}x{f}" for h, w, f in shapes) logger.info( - f"Running warmup for {self.__class__.__name__} " - f"with {len(shapes)} shapes and {steps} steps..." + f"Running warmup for {self.__class__.__name__}: " + f"{len(shapes)} shape(s) [{shape_list}], {steps} steps..." ) warmup_start = time.time() + # Autotuner tuning knobs: cache path from env, plus (multi-rank only) a + # world-group communicator that drives the post-tune cross-rank merge. + enable_autotune = self.pipeline_config.torch_compile.enable_autotune + cache_path = None + post_tune_merge_dist = None + if enable_autotune: + cache_path = os.environ.get("TLLM_AUTOTUNER_CACHE_PATH") + if dist.is_initialized() and dist.get_world_size() > 1: + amap = self.pipeline_config.visual_gen_mapping.to_autotuner_mapping() + post_tune_merge_dist = _VisualGenAutotuneDist(amap) + self._is_warmup = True - for height, width, num_frames in shapes: - logger.info(f"Warmup: {height}x{width}, {num_frames} frames, {steps} steps") - self._run_warmup(height, width, num_frames, steps) - torch.cuda.synchronize() - self._is_warmup = False + try: + if not enable_autotune: + self._run_warmup_pass(shapes, steps) + elif post_tune_merge_dist is None: + # Single rank: nothing to merge; tune and capture in one pass. + with autotune(cache_path=cache_path, skip_dynamic_tuning_buckets=True): + self._run_warmup_pass(shapes, steps) + else: + # Multi rank: tune with capture off, merge tactics across ranks + # at autotune() exit, then recapture from the merged tactics + # (only needed when CUDA graphs are enabled). + with ( + self.disallow_cuda_graph_capture(), + autotune( + cache_path=cache_path, + skip_dynamic_tuning_buckets=True, + post_tune_merge_dist=post_tune_merge_dist, + ), + ): + self._run_warmup_pass(shapes, steps) + if self.pipeline_config.cuda_graph.enable: + self._run_warmup_pass(shapes, steps) + finally: + self._is_warmup = False self._warmed_up_shapes = set( self.warmup_cache_key(h, w, num_frames=f) for h, w, f in shapes @@ -664,6 +728,12 @@ def warmup(self) -> None: elapsed = time.time() - warmup_start logger.info(f"Warmup completed in {elapsed:.2f}s") + def _run_warmup_pass(self, shapes, steps) -> None: + """Run one warmup pass over all shapes (denoise loop with dummy inputs).""" + for height, width, num_frames in shapes: + self._run_warmup(height, width, num_frames, steps) + torch.cuda.synchronize() + def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> None: """Run warmup for a single shape. Subclasses must override. @@ -978,16 +1048,22 @@ def _scheduler_step( timestep, scheduler, extra_stream_schedulers, + scheduler_step_kwargs=None, ): """Execute scheduler step for all streams.""" + step_kwargs = scheduler_step_kwargs or {} t_start = time.time() - latents = scheduler.step(noise_pred, timestep, latents, return_dict=False)[0] + latents = scheduler.step(noise_pred, timestep, latents, return_dict=False, **step_kwargs)[0] # Step schedulers for extra streams for name, noise_extra in extra_noise_preds.items(): if name in extra_stream_schedulers: extra_stream_latents[name] = extra_stream_schedulers[name].step( - noise_extra, timestep, extra_stream_latents[name], return_dict=False + noise_extra, + timestep, + extra_stream_latents[name], + return_dict=False, + **step_kwargs, )[0] t_sched = time.time() - t_start @@ -1010,6 +1086,7 @@ def denoise( boundary_timestep: Optional[float] = None, guidance_interval: Optional[Tuple[float, float]] = None, post_step_fn: Optional[Callable] = None, + scheduler_step_kwargs: Optional[Dict[str, Any]] = None, ): """Execute denoising loop with optional CFG parallel and TeaCache support. @@ -1045,6 +1122,8 @@ def denoise( post_step_fn: Optional callable applied to latents after each scheduler step. Signature: post_step_fn(latents) -> latents Use for constraints that must hold throughout denoising. + scheduler_step_kwargs: Extra keyword arguments forwarded to every + scheduler's ``step()`` call. Returns: Single latents if no extra_streams @@ -1171,6 +1250,7 @@ def denoise( t, scheduler, extra_stream_schedulers, + scheduler_step_kwargs=scheduler_step_kwargs, ) if post_step_fn is not None: diff --git a/tensorrt_llm/_torch/visual_gen/pipeline_loader.py b/tensorrt_llm/_torch/visual_gen/pipeline_loader.py index 3551371fa173..3b92de0572c2 100644 --- a/tensorrt_llm/_torch/visual_gen/pipeline_loader.py +++ b/tensorrt_llm/_torch/visual_gen/pipeline_loader.py @@ -21,7 +21,6 @@ import torch import torch.distributed as dist -from tensorrt_llm._torch.autotuner import autotune from tensorrt_llm._torch.models.modeling_utils import MetaInitMode from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.video_sparse_attention import ( CUTE_AVAILABLE, @@ -303,15 +302,17 @@ def load( else: logger.info("torch.compile disabled by config") + # Cache acceleration (TeaCache / Cache-DiT) is enabled AFTER torch.compile + # on purpose: Cache-DiT captures references to the transformer block + # modules at enable time, while torch_compile() replaces the block lists + # with compiled copies. If Cache-DiT were enabled first, it would keep + # running the stale eager blocks and torch.compile would contribute + # nothing. + if getattr(pipeline, "transformer", None) is not None: + pipeline._setup_cache_acceleration() + if not skip_warmup: - if config.torch_compile.enable_autotune: - with autotune( - cache_path=os.environ.get("TLLM_AUTOTUNER_CACHE_PATH"), - skip_dynamic_tuning_buckets=True, - ): - pipeline.warmup() - else: - pipeline.warmup() + pipeline.warmup() logger.info(f"Warmup completed in {time.time() - t0:.2f}s") else: logger.info("Warmup skipped (skip_warmup=True)") diff --git a/tensorrt_llm/_torch/visual_gen/pipeline_registry.py b/tensorrt_llm/_torch/visual_gen/pipeline_registry.py index 8fe7753ce7f7..acf6225278ed 100644 --- a/tensorrt_llm/_torch/visual_gen/pipeline_registry.py +++ b/tensorrt_llm/_torch/visual_gen/pipeline_registry.py @@ -182,6 +182,8 @@ def _detect_from_checkpoint(checkpoint_dir: str) -> str: return "Flux2Pipeline" if "Flux" in class_name: return "FluxPipeline" + if "QwenImageLayered" in class_name: + return "QwenImageLayeredPipeline" if "QwenImage" in class_name: return "QwenImagePipeline" diff --git a/tensorrt_llm/_torch/visual_gen/utils.py b/tensorrt_llm/_torch/visual_gen/utils.py index 066eb7a3f441..8140dc959122 100644 --- a/tensorrt_llm/_torch/visual_gen/utils.py +++ b/tensorrt_llm/_torch/visual_gen/utils.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Optional, Tuple +from typing import List, Optional, Tuple import torch import torch.distributed as dist @@ -55,7 +55,7 @@ class SequenceSharder: uniformly for sequence parallelism (CP × Ulysses). Models call ``shard(...)`` / ``gather(...)`` / ``shard_rope(...)`` directly; - when the sharder is inactive (``size == 1`` or runtime-disabled) every + when the sharder is inactive (``size == 1``) every method is a no-op pass-through so the call sites do not need an ``if is_active`` guard. @@ -65,11 +65,25 @@ class SequenceSharder: seq axis from a ``seq_len`` argument). """ - def __init__(self, size: int, rank: int, group: Optional[ProcessGroup]): + def __init__( + self, + size: int, + rank: int, + group: Optional[ProcessGroup], + gather_index: Optional[List[int]] = None, + ): self._size = size self._rank = rank self._group = group - self._enabled = size > 1 + # Optional shard-order permutation: gather_index[s] = the GROUP rank that + # holds shard s. Needed when shard indices don't follow group-rank order + # (dist.new_group sorts its rank list, so a group built from a permuted + # rank list still numbers members by ascending global rank). + if gather_index is not None and sorted(gather_index) != list(range(size)): + raise ValueError( + f"gather_index must be a permutation of range({size}), got {gather_index}" + ) + self._gather_index = gather_index # ------------------------------------------------------------------ # Factory @@ -122,7 +136,7 @@ def from_vgm( # ------------------------------------------------------------------ @property def is_active(self) -> bool: - return self._enabled and self._size > 1 + return self._size > 1 @property def size(self) -> int: @@ -136,18 +150,6 @@ def rank(self) -> int: def group(self) -> Optional[ProcessGroup]: return self._group - def disable(self) -> None: - """Run as if ``size == 1``. - - Used by LTX2's stage-2 single-rank execution path where the - non-primary workers have already exited. - """ - self._enabled = False - - def enable(self) -> None: - """Re-enable sharding after :meth:`disable` (no-op if size == 1).""" - self._enabled = self._size > 1 - # ------------------------------------------------------------------ # Shard # ------------------------------------------------------------------ @@ -162,7 +164,7 @@ def shard( """Contiguous block-shard ``tensor`` along ``dim``. Returns ``tensor`` unchanged when: - * the sharder is inactive (``size == 1`` or runtime-disabled), + * the sharder is inactive (``size == 1``), * ``tensor is None``, * ``expected_seq_len`` is given and ``tensor.shape[dim]`` doesn't match — used by LTX2 to skip dataclass fields whose seq axis @@ -249,6 +251,8 @@ def gather( tensor = tensor.contiguous() parts = [torch.empty_like(tensor) for _ in range(self._size)] dist.all_gather(parts, tensor, group=self._group) + if self._gather_index is not None: + parts = [parts[g] for g in self._gather_index] out = torch.cat(parts, dim=dim) if unpad_to is not None: diff --git a/tensorrt_llm/_torch/weight_sharing/__init__.py b/tensorrt_llm/_torch/weight_sharing/__init__.py index 80b70c6c515d..38ec2471c41c 100644 --- a/tensorrt_llm/_torch/weight_sharing/__init__.py +++ b/tensorrt_llm/_torch/weight_sharing/__init__.py @@ -14,6 +14,10 @@ # limitations under the License. """Backend-agnostic weight-sharing utilities (MX, GMS, ...).""" +from tensorrt_llm._torch.weight_sharing.artifact_identity import ( + ARTIFACT_IDENTITY_FORMAT_VERSION, + ArtifactIdentity, +) from tensorrt_llm._torch.weight_sharing.post_transform_profiles import ( PostTransformFeature, PostTransformProfile, @@ -33,6 +37,8 @@ ) __all__ = [ + "ARTIFACT_IDENTITY_FORMAT_VERSION", + "ArtifactIdentity", "SOURCE_IDENTITY_FORMAT_VERSION", "PostTransformFeature", "PostTransformProfile", diff --git a/tensorrt_llm/_torch/weight_sharing/artifact_identity.py b/tensorrt_llm/_torch/weight_sharing/artifact_identity.py new file mode 100644 index 000000000000..d1b8908e241a --- /dev/null +++ b/tensorrt_llm/_torch/weight_sharing/artifact_identity.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Immutable checkpoint identity for shared-weight compatibility checks.""" + +from __future__ import annotations + +import hashlib +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +ARTIFACT_IDENTITY_FORMAT_VERSION = 1 + +_HF_SNAPSHOT_SCHEME = "hf_snapshot_revision" +_CHECKPOINT_MANIFEST_SCHEME = "checkpoint_manifest_sha256" +_SUPPORTED_SCHEMES = frozenset({_HF_SNAPSHOT_SCHEME, _CHECKPOINT_MANIFEST_SCHEME}) +_IGNORED_DIRECTORY_NAMES = frozenset({".cache", ".git", "__pycache__"}) +_IGNORED_FILE_NAMES = frozenset({".DS_Store"}) +_HASH_CHUNK_SIZE = 1024 * 1024 + + +def _canonical_hash(value: Any) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _is_hex(value: str, lengths: tuple[int, ...]) -> bool: + return len(value) in lengths and all(char in "0123456789abcdef" for char in value) + + +def _hf_snapshot_descriptor(path: Path) -> tuple[str, str] | None: + """Return an immutable HF revision and repository-relative subpath.""" + parts = path.resolve().parts + for index, part in enumerate(parts[:-1]): + if part != "snapshots" or index == 0: + continue + if not parts[index - 1].startswith("models--"): + continue + + revision = parts[index + 1].lower() + if not _is_hex(revision, (40, 64)): + continue + subpath = "/".join(parts[index + 2 :]) + return revision, subpath + return None + + +def _raise_walk_error(error: OSError) -> None: + raise error + + +def _checkpoint_files(path: Path) -> tuple[Path, list[Path]]: + if path.is_file(): + return path.parent, [path] + + files = [] + for directory, directory_names, file_names in os.walk(path, onerror=_raise_walk_error): + retained_directories = [] + for directory_name in directory_names: + if directory_name in _IGNORED_DIRECTORY_NAMES: + continue + nested_directory = Path(directory) / directory_name + if nested_directory.is_symlink(): + raise ValueError( + "Checkpoint manifests do not support nested symlinked directories: " + f"{nested_directory}" + ) + retained_directories.append(directory_name) + directory_names[:] = retained_directories + for file_name in file_names: + if file_name in _IGNORED_FILE_NAMES: + continue + candidate = Path(directory) / file_name + if candidate.is_file(): + files.append(candidate) + files.sort(key=lambda candidate: candidate.relative_to(path).as_posix()) + if not files: + raise ValueError(f"Checkpoint path contains no files: {path}") + return path, files + + +def _sha256_file(path: Path) -> tuple[int, str]: + before = path.stat() + digest = hashlib.sha256() + with path.open("rb") as checkpoint_file: + for chunk in iter(lambda: checkpoint_file.read(_HASH_CHUNK_SIZE), b""): + digest.update(chunk) + after = path.stat() + if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns): + raise RuntimeError(f"Checkpoint file changed while being fingerprinted: {path}") + return after.st_size, digest.hexdigest() + + +def _checkpoint_manifest_digest(path: Path) -> str: + root, files = _checkpoint_files(path) + manifest = [] + for checkpoint_file in files: + size, digest = _sha256_file(checkpoint_file) + manifest.append( + { + "path": checkpoint_file.relative_to(root).as_posix(), + "size": size, + "sha256": digest, + } + ) + return _canonical_hash( + { + "format_version": ARTIFACT_IDENTITY_FORMAT_VERSION, + "files": manifest, + } + ) + + +@dataclass(frozen=True) +class ArtifactIdentity: + """Versioned identity of the immutable checkpoint artifact being loaded. + + `SourceIdentity` embeds this value as a global compatibility component. + Hugging Face cache snapshots use their immutable commit revision; local + checkpoints use a canonical manifest of relative paths, sizes, and file + content digests. Absolute paths are intentionally excluded. + """ + + format_version: int + scheme: str + digest: str + + def __post_init__(self) -> None: + if not isinstance(self.format_version, int) or isinstance(self.format_version, bool): + raise ValueError("ArtifactIdentity format version must be an integer") + if self.format_version != ARTIFACT_IDENTITY_FORMAT_VERSION: + raise ValueError(f"Unsupported ArtifactIdentity format version: {self.format_version}") + if not isinstance(self.scheme, str): + raise ValueError("ArtifactIdentity scheme must be a string") + if self.scheme not in _SUPPORTED_SCHEMES: + raise ValueError(f"Unsupported ArtifactIdentity scheme: {self.scheme}") + if not isinstance(self.digest, str): + raise ValueError("ArtifactIdentity digest must be a string") + + normalized_digest = self.digest.lower() + if not _is_hex(normalized_digest, (64,)): + raise ValueError("ArtifactIdentity digest must be a 64-character hex value") + object.__setattr__(self, "digest", normalized_digest) + + @classmethod + def from_checkpoint(cls, checkpoint_path: str | os.PathLike[str]) -> "ArtifactIdentity": + """Build an identity from an immutable snapshot or local checkpoint. + + Args: + checkpoint_path: A model checkpoint file or directory. + + Returns: + The path-independent checkpoint identity. + + Raises: + FileNotFoundError: If `checkpoint_path` does not exist. + ValueError: If a local checkpoint directory contains no files. + RuntimeError: If a local checkpoint changes while it is hashed. + + Note: + Local checkpoints have no authoritative immutable revision, so + their regular files are read in full to derive a content-bound + manifest. Hugging Face cache snapshots use the resolved immutable + revision without rereading model shards. + """ + path = Path(checkpoint_path).expanduser() + if not path.exists(): + raise FileNotFoundError(f"Checkpoint path does not exist: {path}") + + snapshot_descriptor = _hf_snapshot_descriptor(path) + if snapshot_descriptor is not None: + revision, subpath = snapshot_descriptor + digest = _canonical_hash( + { + "scheme": _HF_SNAPSHOT_SCHEME, + "revision": revision, + "subpath": subpath, + } + ) + return cls( + format_version=ARTIFACT_IDENTITY_FORMAT_VERSION, + scheme=_HF_SNAPSHOT_SCHEME, + digest=digest, + ) + + return cls( + format_version=ARTIFACT_IDENTITY_FORMAT_VERSION, + scheme=_CHECKPOINT_MANIFEST_SCHEME, + digest=_checkpoint_manifest_digest(path), + ) + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-serializable representation.""" + return { + "format_version": self.format_version, + "scheme": self.scheme, + "digest": self.digest, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "ArtifactIdentity": + """Reconstruct and validate a serialized artifact identity.""" + return cls( + format_version=data["format_version"], + scheme=data["scheme"], + digest=data["digest"], + ) diff --git a/tensorrt_llm/_torch/weight_sharing/source_identity.py b/tensorrt_llm/_torch/weight_sharing/source_identity.py index 18168703c868..c6b010634760 100644 --- a/tensorrt_llm/_torch/weight_sharing/source_identity.py +++ b/tensorrt_llm/_torch/weight_sharing/source_identity.py @@ -14,17 +14,19 @@ # limitations under the License. """Backend-agnostic source identity for weight-sharing receivers. -A :class:`SourceIdentity` is a serializable fingerprint of configuration choices -that affect how a model's weights are laid out in memory. It exists so that a -*receiver* of pre-laid-out weights (e.g. MX peer-to-peer transfer, or a GMS -read-only materialize) can verify that both the producer ("source") and the -consumer built identities and agree on every layout-affecting choice before the -receiver consumes shared weights. +A :class:`SourceIdentity` is a serializable fingerprint of an immutable +checkpoint artifact and the configuration choices that affect how its weights +are laid out in memory. It exists so that a *receiver* of pre-laid-out weights +(e.g. MX peer-to-peer transfer, or a GMS read-only materialize) can verify that +both the producer ("source") and the consumer built identities and agree on the +artifact and every layout-affecting choice before consuming shared weights. The identity is intentionally decoupled from any specific weight-sharing technology (neither MX nor GMS appears here). Both consume it identically: - local = SourceIdentity.from_model_config(model_config) + local = SourceIdentity.from_model_config( + model_config, checkpoint_dir="/path/to/checkpoint" + ) decision = check_weight_sharing_compatibility(local, source_identity, policy) if decision.should_share: ... # pull / materialize shared weights @@ -40,8 +42,9 @@ -------------- The fingerprint is split so comparison can be selective: -* **global fingerprint** -- rank-invariant model identity, quantization, - backend selection, fusion flags, and parallel *sizes* (TP/PP/EP/CP). +* **global fingerprint** -- immutable checkpoint artifact, rank-invariant + model identity, quantization, backend selection, fusion flags, and parallel + *sizes* (TP/PP/EP/CP). * **shard fingerprint** -- this rank's TP/PP/EP/CP *rank* slice plus the realized local parameter/buffer `(shape, dtype)` layout. Receiver rank `N` must align with the source rank that produced shard `N`. @@ -67,6 +70,7 @@ from enum import Enum from typing import TYPE_CHECKING, Any, List, Optional +from tensorrt_llm._torch.weight_sharing.artifact_identity import ArtifactIdentity from tensorrt_llm.logger import logger if TYPE_CHECKING: @@ -78,7 +82,7 @@ # Bump when the fingerprint projection changes in a way that makes previously # stored identities incomparable. Two identities with different format versions # never match. -SOURCE_IDENTITY_FORMAT_VERSION = 1 +SOURCE_IDENTITY_FORMAT_VERSION = 2 _PRETRAINED_METADATA_FIELDS = frozenset( { @@ -224,6 +228,7 @@ class SourceIdentity: format_version: int # --- global parts (must match across all ranks) --- + artifact_identity: ArtifactIdentity model_fingerprint: str quant_fingerprint: str backend_fingerprint: str @@ -246,6 +251,8 @@ def from_model_config( model_config: "ModelConfig", model: Optional["nn.Module"] = None, *, + checkpoint_dir: Optional[str] = None, + artifact_identity: Optional[ArtifactIdentity] = None, model_name: Optional[str] = None, ) -> "SourceIdentity": """Build an identity from a torch-backend :class:`ModelConfig`. @@ -258,6 +265,12 @@ def from_model_config( Producer and consumer must build the identity at the same lifecycle point (model construction, before weight load). When `None`, the shard fingerprint contains no tensor-layout data. + checkpoint_dir: Checkpoint file or directory used to derive the + nested artifact identity. Required unless `artifact_identity` + is supplied explicitly. + artifact_identity: Precomputed immutable checkpoint identity, + primarily for callers that resolve provenance outside this + method. Mutually exclusive with `checkpoint_dir`. model_name: Human-readable model identity used by discovery layers (e.g. the MX server's source catalog). Does not affect the compatibility fingerprints. @@ -265,7 +278,17 @@ def from_model_config( Returns: A fully populated :class:`SourceIdentity` for `model_config.mapping.rank`. + + Raises: + ValueError: If neither or both artifact identity inputs are given. """ + if checkpoint_dir is None and artifact_identity is None: + raise ValueError("Exactly one of checkpoint_dir or artifact_identity must be provided") + if checkpoint_dir is not None and artifact_identity is not None: + raise ValueError("Exactly one of checkpoint_dir or artifact_identity must be provided") + if artifact_identity is None: + artifact_identity = ArtifactIdentity.from_checkpoint(checkpoint_dir) + mapping = model_config.mapping rank = getattr(mapping, "rank", 0) @@ -275,6 +298,7 @@ def from_model_config( return cls( format_version=SOURCE_IDENTITY_FORMAT_VERSION, + artifact_identity=artifact_identity, model_fingerprint=cls._build_model_fingerprint(model_config), quant_fingerprint=cls._build_quant_fingerprint(model_config), backend_fingerprint=cls._build_backend_fingerprint(model_config), @@ -418,6 +442,7 @@ def global_fingerprint(self) -> str: return _canonical_hash( { "format_version": self.format_version, + "artifact": self.artifact_identity.to_dict(), "model": self.model_fingerprint, "quant": self.quant_fingerprint, "backend": self.backend_fingerprint, @@ -447,6 +472,8 @@ def matches( mismatched.append("format_version") if compare_global: + if self.artifact_identity != other.artifact_identity: + mismatched.append("artifact_identity") for name in ( "model_fingerprint", "quant_fingerprint", @@ -473,6 +500,7 @@ def to_dict(self) -> dict: """ return { "format_version": self.format_version, + "artifact_identity": self.artifact_identity.to_dict(), "model_fingerprint": self.model_fingerprint, "quant_fingerprint": self.quant_fingerprint, "backend_fingerprint": self.backend_fingerprint, @@ -496,8 +524,13 @@ def from_dict(cls, data: dict) -> "SourceIdentity": Returns: The reconstructed :class:`SourceIdentity`. """ + format_version = data["format_version"] + if format_version != SOURCE_IDENTITY_FORMAT_VERSION: + raise ValueError(f"Unsupported SourceIdentity format version: {format_version}") + return cls( - format_version=data["format_version"], + format_version=format_version, + artifact_identity=ArtifactIdentity.from_dict(data["artifact_identity"]), model_fingerprint=data["model_fingerprint"], quant_fingerprint=data["quant_fingerprint"], backend_fingerprint=data["backend_fingerprint"], @@ -548,7 +581,8 @@ def check_weight_sharing_compatibility( result = IdentityMatchResult(matched=False, mismatched_fields=missing_fields) message = ( "SourceIdentity unavailable for fields " - f"{missing_fields}; receiver cannot verify source weight layout." + f"{missing_fields}; receiver cannot verify the source checkpoint " + "artifact and weight layout." ) if policy is IdentityCheckPolicy.STRICT: raise SourceIdentityMismatchError(message) @@ -574,7 +608,7 @@ def check_weight_sharing_compatibility( message = ( "SourceIdentity mismatch on fields " f"{result.mismatched_fields}; receiver and source disagree on " - "weight layout." + "the checkpoint artifact or weight layout." ) if policy is IdentityCheckPolicy.STRICT: raise SourceIdentityMismatchError(message) diff --git a/tensorrt_llm/_utils.py b/tensorrt_llm/_utils.py index 67f1c9d3e950..71574f3e1aa5 100644 --- a/tensorrt_llm/_utils.py +++ b/tensorrt_llm/_utils.py @@ -31,7 +31,6 @@ from ctypes import byref from enum import EnumMeta from functools import lru_cache, partial, wraps -from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Sequence, TypeVar, Union import numpy as np @@ -64,7 +63,7 @@ has_nvml = False # isort: on -from tensorrt_llm.bindings import DataType, GptJsonConfig, LayerType +from tensorrt_llm.bindings import DataType, LayerType from tensorrt_llm.bindings.BuildInfo import ENABLE_MULTI_DEVICE from tensorrt_llm.logger import logger @@ -775,13 +774,6 @@ def __contains__(cls, item): return True -def supports_inflight_batching(engine_dir): - config_path = Path(engine_dir) / "config.json" - json_config = GptJsonConfig.parse_file(config_path) - model_config = json_config.model_config - return model_config.supports_inflight_batching - - class QuantModeWrapper: def __init__(self, objs): diff --git a/tensorrt_llm/bench/benchmark/__init__.py b/tensorrt_llm/bench/benchmark/__init__.py index 8386e115f505..7dcf3cc9bbed 100644 --- a/tensorrt_llm/bench/benchmark/__init__.py +++ b/tensorrt_llm/bench/benchmark/__init__.py @@ -6,9 +6,9 @@ from tensorrt_llm import LLM as PyTorchLLM from tensorrt_llm.bench.benchmark.utils.processes import IterationWriter -from tensorrt_llm.bench.build.build import get_model_config from tensorrt_llm.bench.dataclasses.configuration import RuntimeConfig from tensorrt_llm.bench.dataclasses.general import BenchmarkEnvironment +from tensorrt_llm.bench.tuning.settings import get_model_config from tensorrt_llm.commands.utils import \ collect_explicit_cli_keys as _collect_explicit_cli_keys from tensorrt_llm.logger import logger diff --git a/tensorrt_llm/bench/benchmark/utils/general.py b/tensorrt_llm/bench/benchmark/utils/general.py index e28121b18aba..ba3c312bd45a 100755 --- a/tensorrt_llm/bench/benchmark/utils/general.py +++ b/tensorrt_llm/bench/benchmark/utils/general.py @@ -9,12 +9,12 @@ from tensorrt_llm._torch.pyexecutor.model_loader import \ validate_and_set_kv_cache_quant -from tensorrt_llm.bench.build.build import (get_benchmark_engine_settings, - get_model_config) -from tensorrt_llm.bench.build.dataclasses import (NemotronHybridConfig, - Qwen3HybridConfig) from tensorrt_llm.bench.dataclasses.general import (DatasetMetadata, InferenceRequest) +from tensorrt_llm.bench.tuning.dataclasses import (NemotronHybridConfig, + Qwen3HybridConfig) +from tensorrt_llm.bench.tuning.settings import (get_benchmark_engine_settings, + get_model_config) from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo diff --git a/tensorrt_llm/bench/build/__init__.py b/tensorrt_llm/bench/build/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tensorrt_llm/bench/build/utils.py b/tensorrt_llm/bench/build/utils.py deleted file mode 100644 index 18d0bac8f022..000000000000 --- a/tensorrt_llm/bench/build/utils.py +++ /dev/null @@ -1,34 +0,0 @@ -import pynvml - -DEFAULT_HF_MODEL_DIRS = { - 'BaichuanForCausalLM': 'baichuan-inc/Baichuan-13B-Chat', - 'BloomForCausalLM': 'bigscience/bloom-560m', - 'GLMModel': 'THUDM/glm-10b', - 'ChatGLMModel': 'THUDM/chatglm3-6b', - 'ChatGLMForCausalLM': 'THUDM/chatglm3-6b', - 'FalconForCausalLM': 'tiiuae/falcon-rw-1b', - 'GPTForCausalLM': 'gpt2-medium', - 'GPTJForCausalLM': 'EleutherAI/gpt-j-6b', - 'GPTNeoXForCausalLM': 'EleutherAI/gpt-neox-20b', - 'InternLMForCausalLM': 'internlm/internlm-chat-7b', - 'InternLM2ForCausalLM': 'internlm/internlm2-chat-7b', - 'LlamaForCausalLM': 'meta-llama/Llama-2-7b-hf', - 'MPTForCausalLM': 'mosaicml/mpt-7b', - 'PhiForCausalLM': 'microsoft/phi-2', - 'OPTForCausalLM': 'facebook/opt-350m', - 'QWenLMHeadModel': 'Qwen/Qwen-7B', - 'QWenForCausalLM': 'Qwen/Qwen-7B', - 'Qwen2ForCausalLM': 'Qwen/Qwen1.5-7B', - 'Qwen2MoeForCausalLM': 'Qwen/Qwen1.5-MoE-A2.7B', - 'RecurrentGemmaForCausalLM': 'google/recurrentgemma-2b', -} - - -def get_device_memory(): - pynvml.nvmlInit() - handle = pynvml.nvmlDeviceGetHandleByIndex(0) - mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) - total_memory = mem_info.total / (1024**3) - pynvml.nvmlShutdown() - - return total_memory diff --git a/tensorrt_llm/bench/dataset/prepare_dataset.py b/tensorrt_llm/bench/dataset/prepare_dataset.py index aa7f4eb722e9..9fee2ef12658 100644 --- a/tensorrt_llm/bench/dataset/prepare_dataset.py +++ b/tensorrt_llm/bench/dataset/prepare_dataset.py @@ -17,7 +17,7 @@ import click from pydantic import BaseModel, model_validator -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerFast from tensorrt_llm.bench.dataset.prepare_real_data import real_dataset from tensorrt_llm.bench.dataset.prepare_synthetic_data import token_norm_dist, token_unif_dist @@ -25,7 +25,7 @@ class RootArgs(BaseModel): tokenizer: str - output: str + output: Optional[str] random_seed: int task_id: int trust_remote_code: bool = False @@ -35,9 +35,18 @@ class RootArgs(BaseModel): @model_validator(mode="after") def validate_tokenizer(self): try: - tokenizer = AutoTokenizer.from_pretrained( - self.tokenizer, padding_side="left", trust_remote_code=self.trust_remote_code - ) + # PreTrainedTokenizerFast loads directly from tokenizer.json without + # invoking AutoConfig, avoiding model-config parsing bugs (e.g. + # NemotronHConfig._pattern_to_list KeyError for hybrid models). + # Fall back to AutoTokenizer for models that lack a fast tokenizer. + try: + tokenizer = PreTrainedTokenizerFast.from_pretrained( + self.tokenizer, padding_side="left" + ) + except (OSError, ValueError): + tokenizer = AutoTokenizer.from_pretrained( + self.tokenizer, padding_side="left", trust_remote_code=self.trust_remote_code + ) except EnvironmentError as e: raise ValueError( "Cannot find a tokenizer from the given string because of " @@ -54,6 +63,12 @@ def validate_tokenizer(self): @click.option( "--output", type=str, help="Output json filename.", default="preprocessed_dataset.json" ) +@click.option( + "--stdout", + is_flag=True, + default=False, + help="Print the dataset to stdout with a JSON entry on each line instead of writing a file.", +) @click.option( "--random-seed", required=False, type=int, help="random seed for token_ids", default=420 ) @@ -74,12 +89,15 @@ def validate_tokenizer(self): def prepare_dataset(ctx, **kwargs): """Prepare dataset for benchmarking with trtllm-bench.""" model = ctx.obj.model or ctx.obj.checkpoint_path - output_path = Path(kwargs["output"]) - output_path.parent.mkdir(parents=True, exist_ok=True) + # --stdout is encoded as a null output path (mutually exclusive with a file). + output = None if kwargs["stdout"] else kwargs["output"] + if output is not None: + output_path = Path(output) + output_path.parent.mkdir(parents=True, exist_ok=True) ctx.obj = RootArgs( tokenizer=model, - output=kwargs["output"], + output=output, random_seed=kwargs["random_seed"], task_id=kwargs["task_id"], rand_task_id=kwargs["rand_task_id"], diff --git a/tensorrt_llm/bench/dataset/prepare_real_data.py b/tensorrt_llm/bench/dataset/prepare_real_data.py index 641d95a948f5..0a140482ee92 100644 --- a/tensorrt_llm/bench/dataset/prepare_real_data.py +++ b/tensorrt_llm/bench/dataset/prepare_real_data.py @@ -70,11 +70,12 @@ def query(self): def get_prompt(self, req): """Get the prompt sentence from the given request.""" if self.prompt_key: - assert self.prompt_key in req, ( - f"Dataset {self.name} does not have key '{self.prompt_key}'. " - "Please set --prompt-key to one of the available keys: " - f"{req.keys()}" - ) + if self.prompt_key not in req: + raise ValueError( + f"Dataset {self.name} does not have key '{self.prompt_key}'. " + "Please set --prompt-key to one of the available keys: " + f"{req.keys()}" + ) return req[self.prompt_key] elif self.prompt: return self.prompt @@ -83,21 +84,23 @@ def get_prompt(self, req): def get_input(self, req): """Get the input sentence from the given request.""" - assert self.input_key in req, ( - f"Dataset {self.name} does not have key '{self.input_key}'. " - "Please set --input-key to one of the available keys: " - f"{req.keys()}" - ) + if self.input_key not in req: + raise ValueError( + f"Dataset {self.name} does not have key '{self.input_key}'. " + "Please set --input-key to one of the available keys: " + f"{req.keys()}" + ) return req[self.input_key] def get_images(self, req): """Get the images from the given request.""" image_keys = [self.image_key] + [f"{self.image_key}_{i}" for i in range(1, 8)] - assert any(key in req for key in image_keys), ( - f"Dataset {self.name} does not have key '{self.image_key}'. " - "Please set --dataset-image-key to one of the available keys: " - f"{req.keys()}" - ) + if not any(key in req for key in image_keys): + raise ValueError( + f"Dataset {self.name} does not have key '{self.image_key}'. " + "Please set --dataset-image-key to one of the available keys: " + f"{req.keys()}" + ) images = [] for key in image_keys: if key in req and req[key] is not None: @@ -114,11 +117,12 @@ def get_output(self, req): "you wish to set output length to the length of the golden " "output, set --output-key." ) - assert self.output_key in req, ( - f"Dataset {self.name} does not have key '{self.output_key}'. " - "Please set --output-key to one of the available keys: " - f"{req.keys()}" - ) + if self.output_key not in req: + raise ValueError( + f"Dataset {self.name} does not have key '{self.output_key}'. " + "Please set --output-key to one of the available keys: " + f"{req.keys()}" + ) return req[self.output_key] @@ -240,7 +244,11 @@ def real_dataset(root_args, **kwargs): if any(key in req for key in ["image", "image_1", "video"]): # multimodal input if "video" in req and req["video"] is not None: - assert "Not supported yet" + # Video inputs are not supported yet; warn and fall through to + # image handling. Raising here would reject datasets that carry + # a video column alongside images, which previously worked (the + # original `assert "Not supported yet"` never fired). + logging.warning("Video modality is not supported yet; ignoring 'video' field.") assert kwargs["output_len_dist"] is not None, ( "Output length distribution must be set for multimodal requests." ) diff --git a/tensorrt_llm/bench/dataset/utils.py b/tensorrt_llm/bench/dataset/utils.py index 2ecea0320a5b..39d507419207 100644 --- a/tensorrt_llm/bench/dataset/utils.py +++ b/tensorrt_llm/bench/dataset/utils.py @@ -103,6 +103,10 @@ def get_sample_from_population(population_range, sample_size): def write_dataset_to_file(dataset_generator, output_file): + if output_file is None: + for item in dataset_generator: + print(item) + return output_file = Path(output_file) os.makedirs(output_file.parent, exist_ok=True) with open(output_file, "w") as f: diff --git a/benchmarks/cpp/utils/__init__.py b/tensorrt_llm/bench/tuning/__init__.py similarity index 100% rename from benchmarks/cpp/utils/__init__.py rename to tensorrt_llm/bench/tuning/__init__.py diff --git a/tensorrt_llm/bench/build/dataclasses.py b/tensorrt_llm/bench/tuning/dataclasses.py similarity index 65% rename from tensorrt_llm/bench/build/dataclasses.py rename to tensorrt_llm/bench/tuning/dataclasses.py index 37d63062f23a..ca7e58a4fb51 100755 --- a/tensorrt_llm/bench/build/dataclasses.py +++ b/tensorrt_llm/bench/tuning/dataclasses.py @@ -1,40 +1,42 @@ -from typing import Optional, Literal -from pydantic import AliasPath, BaseModel, Field, AliasChoices, model_validator +import json +import os +import struct +from typing import Literal, Optional + import huggingface_hub from huggingface_hub.constants import ( SAFETENSORS_INDEX_FILE, SAFETENSORS_MAX_HEADER_LENGTH, SAFETENSORS_SINGLE_FILE, ) -from huggingface_hub.utils import SafetensorsRepoMetadata, SafetensorsFileMetadata, TensorInfo +from huggingface_hub.utils import SafetensorsFileMetadata, SafetensorsRepoMetadata, TensorInfo from huggingface_hub.utils import tqdm as hf_tqdm +from pydantic import AliasChoices, AliasPath, BaseModel, Field, model_validator from tqdm.contrib.concurrent import thread_map -import os -import json -import struct from tensorrt_llm._torch.pyexecutor.config_utils import ( - load_pretrained_config, get_qwen3_hybrid_layer_types) + get_qwen3_hybrid_layer_types, + load_pretrained_config, +) # Mapping from safetensors dtype strings to bytes per element. # Used to compute checkpoint size from per-dtype element counts. SAFETENSORS_DTYPE_BYTES = { - 'F64': 8, - 'F32': 4, - 'F16': 2, - 'BF16': 2, - 'I64': 8, - 'I32': 4, - 'I16': 2, - 'I8': 1, - 'U8': 1, - 'BOOL': 1, - 'F8_E4M3': 1, + "F64": 8, + "F32": 4, + "F16": 2, + "BF16": 2, + "I64": 8, + "I32": 4, + "I16": 2, + "I8": 1, + "U8": 1, + "BOOL": 1, + "F8_E4M3": 1, } def parse_safetensors_file_metadata(model_path, filename): - with open(os.path.join(model_path, filename), "rb") as f: metadata_size = f.read(8) metadata_size = struct.unpack(" None: files_metadata[filename] = parse_safetensors_file_metadata( - model_path=model_name_or_path, filename=filename) + model_path=model_name_or_path, filename=filename + ) thread_map( _parse, @@ -123,7 +124,8 @@ def _parse(filename: str) -> None: else: # Not a safetensors repo raise RuntimeError( - f"'{model_name_or_path}' is not a safetensors repo. Couldn't find '{SAFETENSORS_INDEX_FILE}' or '{SAFETENSORS_SINGLE_FILE}' files." + f"'{model_name_or_path}' is not a safetensors repo. Couldn't find " + f"'{SAFETENSORS_INDEX_FILE}' or '{SAFETENSORS_SINGLE_FILE}' files." ) else: return huggingface_hub.get_safetensors_metadata(model_name_or_path) @@ -134,23 +136,28 @@ class ModelConfig(BaseModel): The parameters are needed in engine setting calculation. """ + name: str model_type: str param_count: int checkpoint_size_in_gb: float = Field(default=0.0) - num_hidden_layers: int = Field(validation_alias=AliasChoices( - "num_hidden_layers", - "n_layer", - AliasPath("text_config", "num_hidden_layers"), - AliasPath("language_config", "num_hidden_layers"), - )) + num_hidden_layers: int = Field( + validation_alias=AliasChoices( + "num_hidden_layers", + "n_layer", + AliasPath("text_config", "num_hidden_layers"), + AliasPath("language_config", "num_hidden_layers"), + ) + ) num_attention_layers: Optional[int] = Field(default=None) - num_attention_heads: int = Field(validation_alias=AliasChoices( - "num_attention_heads", - "n_head", - AliasPath("text_config", "num_attention_heads"), - AliasPath("language_config", "num_attention_heads"), - )) + num_attention_heads: int = Field( + validation_alias=AliasChoices( + "num_attention_heads", + "n_head", + AliasPath("text_config", "num_attention_heads"), + AliasPath("language_config", "num_attention_heads"), + ) + ) num_key_value_heads: Optional[int] = Field( default=None, validation_alias=AliasChoices( @@ -160,33 +167,37 @@ class ModelConfig(BaseModel): AliasPath("language_config", "num_key_value_heads"), ), ) - hidden_size: int = Field(validation_alias=AliasChoices( - "hidden_size", - "n_embd", - AliasPath("text_config", "hidden_size"), - )) - head_size: Optional[int] = Field(default=None, - validation_alias=AliasChoices( - "head_size", - "head_dim", - "attention_head_dim", - AliasPath("text_config", "head_dim"), - )) + hidden_size: int = Field( + validation_alias=AliasChoices( + "hidden_size", + "n_embd", + AliasPath("text_config", "hidden_size"), + ) + ) + head_size: Optional[int] = Field( + default=None, + validation_alias=AliasChoices( + "head_size", + "head_dim", + "attention_head_dim", + AliasPath("text_config", "head_dim"), + ), + ) max_position_embeddings: Optional[int] = Field( default=None, validation_alias=AliasChoices( "max_position_embeddings", "n_positions", AliasPath("text_config", "max_position_embeddings"), - )) - dtype: Literal["float16", "bfloat16", "float32", - None] = Field(default="float16", - validation_alias=AliasChoices( - "dtype", "torch_dtype")) + ), + ) + dtype: Literal["float16", "bfloat16", "float32", None] = Field( + default="float16", validation_alias=AliasChoices("dtype", "torch_dtype") + ) @model_validator(mode="after") def set_values_if_none(self): - """ Set the values if cannot get values from HF config.json. """ + """Set the values if cannot get values from HF config.json.""" if not self.dtype: # for GPT-J self.dtype = "float16" if self.num_key_value_heads is None: @@ -222,28 +233,32 @@ def get_param_count_and_checkpoint_size(cls, model_hf_name, hf_model_path): # BF16 for non-quantized layers, F8_E4M3 for scales, etc.). checkpoint_size_in_bytes = sum( count * SAFETENSORS_DTYPE_BYTES.get(dtype, 1) - for dtype, count in metadata.parameter_count.items()) + for dtype, count in metadata.parameter_count.items() + ) checkpoint_size_in_gb = checkpoint_size_in_bytes / (1024**3) if not param_count: - raise ValueError(f"Can't get valid parameter count for model: " - f"{hf_model_path or model_hf_name}.") + raise ValueError( + f"Can't get valid parameter count for model: {hf_model_path or model_hf_name}." + ) return param_count, checkpoint_size_in_gb @classmethod def from_hf(cls, model_hf_name, hf_model_path): - pretrained_config = load_pretrained_config(hf_model_path - or model_hf_name, - trust_remote_code=True) + pretrained_config = load_pretrained_config( + hf_model_path or model_hf_name, trust_remote_code=True + ) hf_config = pretrained_config.to_dict() - param_count, checkpoint_size_in_gb = ( - cls.get_param_count_and_checkpoint_size(model_hf_name, - hf_model_path)) + param_count, checkpoint_size_in_gb = cls.get_param_count_and_checkpoint_size( + model_hf_name, hf_model_path + ) - return cls(name=model_hf_name, - param_count=param_count, - checkpoint_size_in_gb=checkpoint_size_in_gb, - **hf_config) + return cls( + name=model_hf_name, + param_count=param_count, + checkpoint_size_in_gb=checkpoint_size_in_gb, + **hf_config, + ) def extra_model_cache_in_gb(self, bytes_per_elem, target_seq_len=None): return 0 @@ -253,27 +268,35 @@ def cache_memory_fraction(self, cache_memory_fraction): class NemotronHybridConfig(ModelConfig): - hybrid_override_pattern: str = Field(validation_alias=AliasChoices( - "hybrid_override_pattern", - AliasPath("text_config", "hybrid_override_pattern"), - AliasPath("language_config", "hybrid_override_pattern"), - )) - num_hidden_layers: int = Field(validation_alias=AliasChoices( - "num_hidden_layers", - "n_layer", - AliasPath("text_config", "num_hidden_layers"), - AliasPath("language_config", "num_hidden_layers"), - )) - d_state: int = Field(validation_alias=AliasChoices( - "d_state", - "mamba_d_state", - "ssm_state_size", - )) - d_conv: int = Field(validation_alias=AliasChoices( - "d_conv", - "mamba_d_conv", - "conv_kernel", - )) + hybrid_override_pattern: str = Field( + validation_alias=AliasChoices( + "hybrid_override_pattern", + AliasPath("text_config", "hybrid_override_pattern"), + AliasPath("language_config", "hybrid_override_pattern"), + ) + ) + num_hidden_layers: int = Field( + validation_alias=AliasChoices( + "num_hidden_layers", + "n_layer", + AliasPath("text_config", "num_hidden_layers"), + AliasPath("language_config", "num_hidden_layers"), + ) + ) + d_state: int = Field( + validation_alias=AliasChoices( + "d_state", + "mamba_d_state", + "ssm_state_size", + ) + ) + d_conv: int = Field( + validation_alias=AliasChoices( + "d_conv", + "mamba_d_conv", + "conv_kernel", + ) + ) mamba_num_heads: int n_groups: int mamba_head_dim: int @@ -283,7 +306,7 @@ class NemotronHybridConfig(ModelConfig): @model_validator(mode="after") def set_values_if_none(self): - """ Set the values if cannot get values from HF config.json. """ + """Set the values if cannot get values from HF config.json.""" if not self.d_inner: self.d_inner = self.mamba_num_heads * self.mamba_head_dim if self.num_mamba_layers is None: @@ -298,12 +321,17 @@ def extra_model_cache_in_gb(self, bytes_per_elem, target_seq_len=None): conv_dim = self.d_inner + 2 * self.n_groups * self.d_state conv_state_elems = conv_dim * (self.d_conv - 1) ssm_state_elems = self.mamba_num_heads * self.mamba_head_dim * self.d_state - gb_per_mamba_cache = bytes_per_elem * self.num_mamba_layers * ( - conv_state_elems + ssm_state_elems) / (1024**3) + gb_per_mamba_cache = ( + bytes_per_elem + * self.num_mamba_layers + * (conv_state_elems + ssm_state_elems) + / (1024**3) + ) return gb_per_mamba_cache def cache_memory_fraction(self, cache_memory_fraction): - # Each mamba cache entry is pretty large (~50MB for 8B model), so we are more conservative when estimating the max batch size + # Each mamba cache entry is pretty large (~50MB for 8B model), so we are + # more conservative when estimating the max batch size return cache_memory_fraction**2 def set_mamba_ssm_cache_dtype(self, mamba_ssm_cache_dtype: str): @@ -311,13 +339,13 @@ def set_mamba_ssm_cache_dtype(self, mamba_ssm_cache_dtype: str): @classmethod def from_hf(cls, model_hf_name, hf_model_path): - pretrained_config = load_pretrained_config(hf_model_path - or model_hf_name, - trust_remote_code=True) + pretrained_config = load_pretrained_config( + hf_model_path or model_hf_name, trust_remote_code=True + ) hf_config = pretrained_config.to_dict() - param_count, checkpoint_size_in_gb = ( - cls.get_param_count_and_checkpoint_size(model_hf_name, - hf_model_path)) + param_count, checkpoint_size_in_gb = cls.get_param_count_and_checkpoint_size( + model_hf_name, hf_model_path + ) # HuggingFace PretrainedConfig.to_dict() only serializes attributes known to # the base class; custom configs (e.g. NemotronHConfig) have num_hidden_layers @@ -326,8 +354,8 @@ def from_hf(cls, model_hf_name, hf_model_path): text_config = getattr(pretrained_config, "text_config", None) language_config = getattr(pretrained_config, "language_config", None) for key in ( - "num_hidden_layers", - "hybrid_override_pattern", + "num_hidden_layers", + "hybrid_override_pattern", ): if hf_config.get(key) is None: value = None @@ -342,10 +370,12 @@ def from_hf(cls, model_hf_name, hf_model_path): if value is not None: hf_config[key] = value - return cls(name=model_hf_name, - param_count=param_count, - checkpoint_size_in_gb=checkpoint_size_in_gb, - **hf_config) + return cls( + name=model_hf_name, + param_count=param_count, + checkpoint_size_in_gb=checkpoint_size_in_gb, + **hf_config, + ) class Qwen3HybridConfig(ModelConfig): @@ -354,6 +384,7 @@ class Qwen3HybridConfig(ModelConfig): Maps Qwen3.5 linear-attention parameters to the same cache estimation formulas used by NemotronHybridConfig. """ + linear_key_head_dim: int # d_state linear_conv_kernel_dim: int # d_conv linear_num_value_heads: int # num_heads (mamba_num_heads) @@ -364,34 +395,38 @@ class Qwen3HybridConfig(ModelConfig): @classmethod def from_hf(cls, model_hf_name, hf_model_path): - pretrained_config = load_pretrained_config(hf_model_path - or model_hf_name, - trust_remote_code=True) + pretrained_config = load_pretrained_config( + hf_model_path or model_hf_name, trust_remote_code=True + ) hf_config = pretrained_config.to_dict() - param_count, checkpoint_size_in_gb = ( - cls.get_param_count_and_checkpoint_size(model_hf_name, - hf_model_path)) + param_count, checkpoint_size_in_gb = cls.get_param_count_and_checkpoint_size( + model_hf_name, hf_model_path + ) layer_types = get_qwen3_hybrid_layer_types(pretrained_config) - hf_config.setdefault("num_attention_layers", - layer_types.count("full_attention")) - hf_config.setdefault("num_linear_attention_layers", - layer_types.count("linear_attention")) - - return cls(name=model_hf_name, - param_count=param_count, - checkpoint_size_in_gb=checkpoint_size_in_gb, - **hf_config) + hf_config.setdefault("num_attention_layers", layer_types.count("full_attention")) + hf_config.setdefault("num_linear_attention_layers", layer_types.count("linear_attention")) + + return cls( + name=model_hf_name, + param_count=param_count, + checkpoint_size_in_gb=checkpoint_size_in_gb, + **hf_config, + ) def extra_model_cache_in_gb(self, bytes_per_elem, target_seq_len=None): d_inner = self.linear_value_head_dim * self.linear_num_value_heads conv_dim = d_inner + 2 * self.linear_num_key_heads * self.linear_key_head_dim conv_state_elems = conv_dim * (self.linear_conv_kernel_dim - 1) - ssm_state_elems = (self.linear_num_value_heads * - self.linear_value_head_dim * - self.linear_key_head_dim) - gb_per_cache = bytes_per_elem * self.num_linear_attention_layers * ( - conv_state_elems + ssm_state_elems) / (1024**3) + ssm_state_elems = ( + self.linear_num_value_heads * self.linear_value_head_dim * self.linear_key_head_dim + ) + gb_per_cache = ( + bytes_per_elem + * self.num_linear_attention_layers + * (conv_state_elems + ssm_state_elems) + / (1024**3) + ) return gb_per_cache def cache_memory_fraction(self, cache_memory_fraction): diff --git a/tensorrt_llm/bench/build/tuning.py b/tensorrt_llm/bench/tuning/heuristics.py similarity index 79% rename from tensorrt_llm/bench/build/tuning.py rename to tensorrt_llm/bench/tuning/heuristics.py index d77cf6591dbb..0b41fee9767f 100755 --- a/tensorrt_llm/bench/build/tuning.py +++ b/tensorrt_llm/bench/tuning/heuristics.py @@ -1,20 +1,25 @@ +import math from typing import Tuple import torch from tensorrt_llm._utils import str_dtype_to_torch +from tensorrt_llm.bench.tuning.dataclasses import ( + ModelConfig, + NemotronHybridConfig, + Qwen3HybridConfig, +) from tensorrt_llm.llmapi.llm_utils import QuantConfig from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo -from tensorrt_llm.bench.build.dataclasses import ModelConfig, NemotronHybridConfig, Qwen3HybridConfig + from .utils import get_device_memory -import math BYTES_PER_ELEM = { QuantAlgo.NO_QUANT: 2.0, QuantAlgo.FP8: 1.0, QuantAlgo.FP8_BLOCK_SCALES: 1.0, - QuantAlgo.NVFP4: .5, + QuantAlgo.NVFP4: 0.5, } @@ -28,13 +33,14 @@ def calc_engine_setting( kv_cache_gpu_mem_fraction: float = 0.95, enable_attention_dp: bool = False, ) -> Tuple[int, int]: - """ Calculate the engine build settings (max batch size and max num tokens) - for a specific model + parallelism mapping + dataset configuration. - trtllm-bench sets a slightly optimistic upper bound for max batch size - and max num tokens to avoid over-allocation of memory in activation, - runtime, and decoder buffers. In runtime, TRT-LLM relies on its runtime - tuning features to adjust the runtime max batch size according to - incoming traffic. + """Calculate the engine build settings (max batch size and max num tokens). + + For a specific model + parallelism mapping + dataset configuration, + trtllm-bench sets a slightly optimistic upper bound for max batch size + and max num tokens to avoid over-allocation of memory in activation, + runtime, and decoder buffers. In runtime, TRT-LLM relies on its runtime + tuning features to adjust the runtime max batch size according to + incoming traffic. Args: model_config (ModelConfig): Model specific configurations. @@ -63,11 +69,16 @@ def calc_engine_setting( # Each GPU in TP group has at least 1 kv head adjusted_num_kv_heads = max(tp_size, model_config.num_key_value_heads) - logger.info( - f"Number of attention layers: {model_config.num_attention_layers}") + logger.info(f"Number of attention layers: {model_config.num_attention_layers}") - gb_per_token = 2 * model_config.num_attention_layers * adjusted_num_kv_heads \ - * model_config.head_size * byte_per_kv_elem / (1024 ** 3) + gb_per_token = ( + 2 + * model_config.num_attention_layers + * adjusted_num_kv_heads + * model_config.head_size + * byte_per_kv_elem + / (1024**3) + ) # Number of GPU used for this run. n_gpus = tp_size * pp_size @@ -92,13 +103,11 @@ def calc_engine_setting( # Available memory to allocate KV cache. available_memory = total_gpu_memory - engine_size logger.info(f"Estimated engine size: {engine_size:.2f} GB") - logger.info("Estimated total available memory for KV cache: " - f"{available_memory:.2f} GB") + logger.info(f"Estimated total available memory for KV cache: {available_memory:.2f} GB") # Calculate max requests in KV cache based on target ISL and OSL. target_seq_len = target_input_len + target_output_len - cache_memory = available_memory * model_config.cache_memory_fraction( - kv_cache_gpu_mem_fraction) + cache_memory = available_memory * model_config.cache_memory_fraction(kv_cache_gpu_mem_fraction) bytes_per_elem = BYTES_PER_ELEM.get(QuantAlgo.NO_QUANT) if isinstance(model_config, (NemotronHybridConfig, Qwen3HybridConfig)): @@ -107,30 +116,31 @@ def calc_engine_setting( if str_dtype_to_torch(mamba_ssm_cache_dtype) == torch.float32: bytes_per_elem = 4.0 - gb_per_extra_cache = model_config.extra_model_cache_in_gb( - bytes_per_elem, target_seq_len) - kv_cache_max_requests = cache_memory / (gb_per_token * target_seq_len + - gb_per_extra_cache) + gb_per_extra_cache = model_config.extra_model_cache_in_gb(bytes_per_elem, target_seq_len) + kv_cache_max_requests = cache_memory / (gb_per_token * target_seq_len + gb_per_extra_cache) extra_cache_memory = gb_per_extra_cache * kv_cache_max_requests kv_cache_memory = cache_memory - extra_cache_memory kv_cache_max_tokens = kv_cache_memory / gb_per_token logger.info( - f"Estimated total cache memory: {cache_memory:.2f} GB. KV cache: {kv_cache_memory:.2f} GB, Extra cache: {extra_cache_memory:.2f} GB" + f"Estimated total cache memory: {cache_memory:.2f} GB. " + f"KV cache: {kv_cache_memory:.2f} GB, Extra cache: {extra_cache_memory:.2f} GB" ) logger.info(f"Estimated kv cache max tokens: {kv_cache_max_tokens:.2f}") - logger.info("Estimated max number of requests in KV cache memory: " - f"{kv_cache_max_requests:.2f}") + logger.info(f"Estimated max number of requests in KV cache memory: {kv_cache_max_requests:.2f}") # Fine-tune the max batch size and num token setting for performance. - # For mamba-attn hybrid models, we disable optimistic tuning because the mamba cache leaves less memory for the KV cache + # For mamba-attn hybrid models, we disable optimistic tuning because the + # mamba cache leaves less memory for the KV cache max_batch_size, max_num_tokens = finetune_setting( kv_cache_max_requests, target_input_len, target_output_len, pp_size, disable_optimistic_tuning=isinstance( - model_config, (NemotronHybridConfig, Qwen3HybridConfig))) + model_config, (NemotronHybridConfig, Qwen3HybridConfig) + ), + ) # Functional and performance if total_gpu_memory < engine_size: @@ -151,21 +161,26 @@ def calc_engine_setting( f"Number of Tensor Parallel Shards: {tp_size}\n" f"Number of Pipeline Parallel Stages: {pp_size}\n" f"KV Cache GPU Memory Fraction: {kv_cache_gpu_mem_fraction}\n" - "----------------------------------------------------------\n") + "----------------------------------------------------------\n" + ) if kv_cache_max_requests < 1: - raise RuntimeError("The amount of KV cache memory is insufficient to " - "run this model. Please try with more GPUs.") + raise RuntimeError( + "The amount of KV cache memory is insufficient to " + "run this model. Please try with more GPUs." + ) warning_gpu_count = pp_size if enable_attention_dp else n_gpus if cache_memory / warning_gpu_count < 10.0: logger.warning( - f"The KV cache memory per GPU is less than 10 GB. " + "The KV cache memory per GPU is less than 10 GB. " "Performance may be undesirable. Please consider using a different " - "mapping or more GPUs.") + "mapping or more GPUs." + ) if kv_cache_max_requests < 32: logger.warning( - f"The maximum number of requests in the KV cache is too " + "The maximum number of requests in the KV cache is too " "small. Performance may be undesirable. Please consider using more " - "GPUs or a different mapping to process more concurrent requests.") + "GPUs or a different mapping to process more concurrent requests." + ) return max_batch_size, max_num_tokens @@ -177,9 +192,10 @@ def finetune_setting( pp_size: int, disable_optimistic_tuning: bool = False, ) -> Tuple[int, int]: - """ Calculate and fine-tune the engine build settings (max batch size and - max num tokens). Both max batch size and max num tokens are fine-tuned - to be slightly optimistic. + """Calculate and fine-tune the engine build settings. + + Both max batch size and max num tokens are fine-tuned to be slightly + optimistic. Args: kv_cache_max_requests (float): Max number of requests that can fits in @@ -219,11 +235,12 @@ def finetune_setting( else: max_token = 1024 * math.ceil(raw_token / 1024) - logger.debug(f"Estimated max batch size (before fine-tune): " - f"{kv_cache_max_requests / pp_size:.2f}") + logger.debug( + f"Estimated max batch size (before fine-tune): {kv_cache_max_requests / pp_size:.2f}" + ) logger.debug( f"Estimated max num tokens (before fine-tune): " - f"{kv_cache_max_requests / pp_size * (1 + input_len / output_len) :.2f}" + f"{kv_cache_max_requests / pp_size * (1 + input_len / output_len):.2f}" ) logger.info(f"Estimated max batch size (after fine-tune): {max_bs}") logger.info(f"Estimated max num tokens (after fine-tune): {max_token}") diff --git a/tensorrt_llm/bench/build/build.py b/tensorrt_llm/bench/tuning/settings.py similarity index 77% rename from tensorrt_llm/bench/build/build.py rename to tensorrt_llm/bench/tuning/settings.py index 16bf6d23d9aa..9f47292975ac 100644 --- a/tensorrt_llm/bench/build/build.py +++ b/tensorrt_llm/bench/tuning/settings.py @@ -3,21 +3,28 @@ from pathlib import Path from typing import Tuple -from tensorrt_llm._torch.pyexecutor.config_utils import (is_nemotron_hybrid, - is_qwen3_hybrid, - load_pretrained_config) -from tensorrt_llm.bench.build.dataclasses import (ModelConfig, - NemotronHybridConfig, - Qwen3HybridConfig) -from tensorrt_llm.bench.build.tuning import calc_engine_setting +from tensorrt_llm._torch.pyexecutor.config_utils import ( + is_nemotron_hybrid, + is_qwen3_hybrid, + load_pretrained_config, +) +from tensorrt_llm.bench.tuning.dataclasses import ( + ModelConfig, + NemotronHybridConfig, + Qwen3HybridConfig, +) +from tensorrt_llm.bench.tuning.heuristics import calc_engine_setting from tensorrt_llm.llmapi.llm_args import TorchLlmArgs from tensorrt_llm.llmapi.llm_utils import QuantConfig from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo TUNED_QUANTS = { - QuantAlgo.NVFP4, QuantAlgo.FP8, QuantAlgo.FP8_BLOCK_SCALES, - QuantAlgo.NO_QUANT, None + QuantAlgo.NVFP4, + QuantAlgo.FP8, + QuantAlgo.FP8_BLOCK_SCALES, + QuantAlgo.NO_QUANT, + None, } # Sourced from TorchLlmArgs so the bench defaults track the args-class field # defaults and can't drift. @@ -35,7 +42,7 @@ def get_benchmark_engine_settings( kv_cache_gpu_mem_fraction: float = 0.95, enable_attention_dp: bool = False, ) -> Tuple[int, int]: - """ Retrieve benchmark settings for a specific model + configuration. + """Retrieve benchmark settings for a specific model + configuration. Args: model_config (ModelConfig): Model specific configurations. @@ -77,13 +84,14 @@ def get_benchmark_engine_settings( ) if max_batch_size <= 0 or max_num_tokens <= 0: - raise RuntimeError(f"Unable to obtain correct settings for benchmark.") + raise RuntimeError("Unable to obtain correct settings for benchmark.") return max_batch_size, max_num_tokens def get_model_config(model_name: str, model_path: Path = None) -> ModelConfig: - """ Obtain the model-related parameters from Hugging Face. + """Obtain the model-related parameters from Hugging Face. + Args: model_name (str): Huggingface model name. model_path (Path): Path to a local Huggingface checkpoint. @@ -91,8 +99,7 @@ def get_model_config(model_name: str, model_path: Path = None) -> ModelConfig: Raises: ValueError: When model is not supported. """ - pretrained_config = load_pretrained_config(model_path or model_name, - trust_remote_code=True) + pretrained_config = load_pretrained_config(model_path or model_name, trust_remote_code=True) if is_nemotron_hybrid(pretrained_config): return NemotronHybridConfig.from_hf(model_name, model_path) if is_qwen3_hybrid(pretrained_config): diff --git a/tensorrt_llm/bench/tuning/utils.py b/tensorrt_llm/bench/tuning/utils.py new file mode 100644 index 000000000000..37d92e197f9b --- /dev/null +++ b/tensorrt_llm/bench/tuning/utils.py @@ -0,0 +1,34 @@ +import pynvml + +DEFAULT_HF_MODEL_DIRS = { + "BaichuanForCausalLM": "baichuan-inc/Baichuan-13B-Chat", + "BloomForCausalLM": "bigscience/bloom-560m", + "GLMModel": "THUDM/glm-10b", + "ChatGLMModel": "THUDM/chatglm3-6b", + "ChatGLMForCausalLM": "THUDM/chatglm3-6b", + "FalconForCausalLM": "tiiuae/falcon-rw-1b", + "GPTForCausalLM": "gpt2-medium", + "GPTJForCausalLM": "EleutherAI/gpt-j-6b", + "GPTNeoXForCausalLM": "EleutherAI/gpt-neox-20b", + "InternLMForCausalLM": "internlm/internlm-chat-7b", + "InternLM2ForCausalLM": "internlm/internlm2-chat-7b", + "LlamaForCausalLM": "meta-llama/Llama-2-7b-hf", + "MPTForCausalLM": "mosaicml/mpt-7b", + "PhiForCausalLM": "microsoft/phi-2", + "OPTForCausalLM": "facebook/opt-350m", + "QWenLMHeadModel": "Qwen/Qwen-7B", + "QWenForCausalLM": "Qwen/Qwen-7B", + "Qwen2ForCausalLM": "Qwen/Qwen1.5-7B", + "Qwen2MoeForCausalLM": "Qwen/Qwen1.5-MoE-A2.7B", + "RecurrentGemmaForCausalLM": "google/recurrentgemma-2b", +} + + +def get_device_memory(): + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) + total_memory = mem_info.total / (1024**3) + pynvml.nvmlShutdown() + + return total_memory diff --git a/tensorrt_llm/bench/utils/__init__.py b/tensorrt_llm/bench/utils/__init__.py index ea8e5a10996a..6d818578b9be 100644 --- a/tensorrt_llm/bench/utils/__init__.py +++ b/tensorrt_llm/bench/utils/__init__.py @@ -20,7 +20,7 @@ VALID_QUANT_ALGOS = Literal[f"{QuantAlgo.W8A16}", f"{QuantAlgo.W4A16}", f"{QuantAlgo.W4A16_AWQ}", f"{QuantAlgo.W4A8_AWQ}", f"{QuantAlgo.W4A16_GPTQ}", f"{QuantAlgo.FP8}", - f"{QuantAlgo.INT8}", f"{QuantAlgo.NVFP4}"] + f"{QuantAlgo.NVFP4}"] VALID_SCHEDULING_POLICIES = \ Literal["max_utilization", "guaranteed_no_evict", "static"] diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 2f39353c521c..2dfe1e91ffe3 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -1,17 +1,20 @@ import asyncio +import atexit import gc import importlib import inspect import json import os import secrets +import select import signal import socket import subprocess # nosec B404 import sys +import time import uuid from pathlib import Path -from typing import Any, Dict, Optional, Sequence, Set +from typing import Any, Dict, NamedTuple, Optional, Sequence, Set import click import torch @@ -22,11 +25,11 @@ from tensorrt_llm import LLM as PyTorchLLM from tensorrt_llm import MultimodalEncoder -from tensorrt_llm._utils import mpi_rank +from tensorrt_llm._utils import mpi_rank, set_prometheus_multiproc_dir from tensorrt_llm.commands._serve_stability import stability_option from tensorrt_llm.commands.utils import (collect_explicit_cli_keys, get_is_diffusion_only_model) -from tensorrt_llm.executor.utils import LlmLauncherEnvs +from tensorrt_llm.executor.utils import MAX_NUM_FRONTENDS, LlmLauncherEnvs from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.llmapi import KvCacheConfig from tensorrt_llm.llmapi.disagg_utils import (DisaggClusterConfig, @@ -37,7 +40,7 @@ validate_config_bool) from tensorrt_llm.llmapi.llm_args import MultimodalConfig, TorchLlmArgs from tensorrt_llm.llmapi.llm_utils import update_llm_args_with_extra_dict -from tensorrt_llm.llmapi.mpi_session import find_free_ipc_addr +from tensorrt_llm.llmapi.mpi_session import find_free_ipc_addr, split_mpi_env from tensorrt_llm.llmapi.reasoning_parser import (ReasoningParserFactory, resolve_auto_reasoning_parser) from tensorrt_llm.logger import logger, severity_map @@ -197,6 +200,7 @@ def get_llm_args( free_gpu_memory_fraction: float = 0.9, kv_cache_dtype: str = "auto", num_postprocess_workers: int = 0, + num_serve_frontends: int = 1, trust_remote_code: bool = False, revision: Optional[str] = None, reasoning_parser: Optional[str] = None, @@ -270,6 +274,8 @@ def get_llm_args( max_seq_len, "num_postprocess_workers": num_postprocess_workers, + "num_serve_frontends": + num_serve_frontends, "enable_chunked_prefill": enable_chunked_prefill, "enable_attention_dp": @@ -358,6 +364,156 @@ def _diagnose_port_in_use(port: int) -> str: return "; ".join(details) +class MultiFrontendMode(NamedTuple): + """This process's role under multi-frontend serving (prototype).""" + num_frontends: int + is_attached_frontend: bool + + @property + def is_launcher(self) -> bool: + """Owns the engine and spawns/cleans the attached frontends.""" + return self.num_frontends > 1 and not self.is_attached_frontend + + +def _init_multi_frontend_mode(llm_args: dict, + enabled: bool) -> MultiFrontendMode: + """Resolve this process's multi-frontend serving role. + + num_serve_frontends=K runs K HTTP frontend processes against ONE + executor: the launcher (frontend 0) owns the engine and spawns K-1 + attached frontends (classic IPC executor path only). enabled=False + entry points (e.g. disaggregated MPI workers) never honor the knob. + """ + if not enabled: + if llm_args.pop("num_serve_frontends", 1) > 1: + logger.warning("num_serve_frontends is only supported on plain " + "trtllm-serve; ignored on this entry point.") + return MultiFrontendMode(1, False) + + mode = MultiFrontendMode(llm_args.get("num_serve_frontends", 1), + os.getenv("TLLM_EXECUTOR_ATTACH_INFO") is not None) + if mode.is_launcher and llm_args.get("orchestrator_type") is not None: + raise ValueError( + "num_serve_frontends > 1 requires the default (classic IPC) " + "executor path, not orchestrator_type=" + f"{llm_args.get('orchestrator_type')!r}") + return mode + + +def _spawn_attached_frontends(llm, num_frontends: int) -> list: + """Spawn num_frontends - 1 attached serving frontend processes. + + Each child re-execs this trtllm-serve command line with env vars + carrying the launcher executor's attach endpoints; its executor + attaches to the already-running worker instead of launching one (see + GenerationExecutor.create / GenerationExecutorFrontendProxy). + + Blocks until every child signals READY over its inherited pipe: a + successful Popen only proves the process exists, while the frontend + can still fail during executor attach or server setup. Any child + failure (or a missed deadline) fails the whole group, terminating + the children already started, so num_serve_frontends=K never + silently degrades to fewer frontends. + """ + from tensorrt_llm.executor.proxy import GenerationExecutorProxy + + executor = getattr(llm, "_executor", None) + if not isinstance(executor, GenerationExecutorProxy) or ( + attach_info := executor.multi_frontend_attach_info()) is None: + raise ValueError( + "num_serve_frontends > 1 requires the classic IPC executor " + f"proxy in multi-frontend mode, got {type(executor).__name__}") + # Carries the executor HMAC keys; the child deletes it from its env + # once consumed (GenerationExecutor.create). + attach_env = json.dumps(attach_info) + + children, ready_fds = [], [] + try: + for frontend_id in range(1, num_frontends): + # Strip MPI/SLURM identity vars: an inherited rank identity would + # make the child's mpi4py try to (re-)join the launcher's job. + env, _ = split_mpi_env() + env["TLLM_EXECUTOR_ATTACH_INFO"] = attach_env + env["TLLM_EXECUTOR_FRONTEND_ID"] = str(frontend_id) + env["TLLM_DISABLE_MPI"] = "1" + read_fd, write_fd = os.pipe() + ready_fds.append(read_fd) + env["TLLM_FRONTEND_READY_FD"] = str(write_fd) + try: + child = subprocess.Popen([sys.executable] + sys.argv, + env=env, + pass_fds=(write_fd, )) # nosec B603 + finally: + # The child now holds the only write end; its exit before + # READY surfaces as EOF on read_fd. + os.close(write_fd) + children.append(child) + logger.info( + f"Launched attached serving frontend {frontend_id} (pid {child.pid})" + ) + _wait_attached_frontends_ready(children, ready_fds) + except BaseException: + _terminate_attached_frontends(children) + raise + finally: + for fd in ready_fds: + os.close(fd) + return children + + +def _wait_attached_frontends_ready(children: list, ready_fds: list) -> None: + """Block until every attached frontend writes its READY byte.""" + timeout = float(os.getenv("TLLM_FRONTEND_READY_TIMEOUT", "300")) + deadline = time.monotonic() + timeout + pending = dict(zip(ready_fds, children)) + while pending: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise RuntimeError( + f"{len(pending)} attached frontend(s) not ready within " + f"{timeout:.0f}s (TLLM_FRONTEND_READY_TIMEOUT)") + readable, _, _ = select.select(list(pending), [], [], + min(remaining, 1.0)) + for fd in readable: + child = pending.pop(fd) + if os.read(fd, 1) != b"R": # EOF: pipe closed without READY + raise RuntimeError( + f"Attached frontend (pid {child.pid}) exited before " + "signaling READY") + logger.info(f"Attached frontend (pid {child.pid}) is ready") + for fd, child in list(pending.items()): + if child.poll() is not None: + raise RuntimeError( + f"Attached frontend (pid {child.pid}) exited with code " + f"{child.returncode} before signaling READY") + + +def _signal_frontend_ready(multi_frontend: MultiFrontendMode) -> None: + """Report READY to the launcher over the inherited pipe. + + Called once everything fallible in an attached frontend's startup + (port bind, executor attach, LLM and OpenAIServer construction, + middleware registration) has succeeded; the launcher blocks group + startup on this byte (see _wait_attached_frontends_ready). + """ + ready_fd = os.environ.pop("TLLM_FRONTEND_READY_FD", None) + if not (multi_frontend.is_attached_frontend and ready_fd): + return + fd = int(ready_fd) + os.write(fd, b"R") + os.close(fd) + + +def _terminate_attached_frontends(children: list) -> None: + for child in children: + child.terminate() + for child in children: + try: + child.wait(timeout=10) + except subprocess.TimeoutExpired: + child.kill() + + def launch_server( host: str, port: int, @@ -372,10 +528,25 @@ def launch_server( served_model_name: Optional[str] = None, allow_request_chat_template: bool = False, num_input_processor_workers: int = 8, - num_media_load_workers: int = 8): + num_media_load_workers: int = 8, + multi_frontend_enabled: bool = True): backend = llm_args["backend"] model = served_model_name or llm_args["model"] + + multi_frontend = _init_multi_frontend_mode(llm_args, multi_frontend_enabled) + if multi_frontend.is_launcher or multi_frontend.is_attached_frontend: + # The Responses API store is per-process in-memory: with several + # frontends behind one SO_REUSEPORT port, a follow-up request may + # land on a sibling that has no record of the previous response. + # Disable storage group-wide (OpenAIServer.enable_store). + if not os.getenv("TRTLLM_RESPONSES_API_DISABLE_STORE"): + logger.warning( + "num_serve_frontends > 1: stateful Responses API storage " + "(store/previous_response_id) is disabled; the per-frontend " + "in-memory store cannot be shared across frontends.") + os.environ["TRTLLM_RESPONSES_API_DISABLE_STORE"] = "1" + addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) address_family = socket.AF_INET6 if all( @@ -383,6 +554,10 @@ def launch_server( with socket.socket(address_family, socket.SOCK_STREAM) as s: # If disagg cluster config is provided and port is not specified, try to find a free port, otherwise try to bind to the specified port assert port > 0 or disagg_cluster_config is not None, "Port must be specified if disagg cluster config is not provided" + if multi_frontend.is_launcher or multi_frontend.is_attached_frontend: + # Every frontend process binds its own listening socket on the + # same port; the kernel load-balances accepts across them. + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) try: s.bind((host, port)) if port == 0: @@ -409,25 +584,39 @@ def launch_server( f"{backend} is not a known backend, check help for available options.", param_hint="backend") - server = OpenAIServer( - generator=llm, - model=model, - tool_parser=tool_parser, - server_role=server_role, - metadata_server_cfg=metadata_server_cfg, - disagg_cluster_config=disagg_cluster_config, - multimodal_server_config=multimodal_server_config, - chat_template=chat_template, - allow_request_chat_template=allow_request_chat_template, - input_processor_workers=num_input_processor_workers, - media_load_workers=num_media_load_workers) - _apply_fastapi_middlewares(server.app, middleware) + # The finally below is the cleanup boundary for the attached + # frontends: it must cover everything from their spawn through + # server construction, middleware registration, and runtime, or a + # failure in between leaks the child processes. + frontend_children = [] + try: + if multi_frontend.is_launcher: + frontend_children = _spawn_attached_frontends( + llm, multi_frontend.num_frontends) + + server = OpenAIServer( + generator=llm, + model=model, + tool_parser=tool_parser, + server_role=server_role, + metadata_server_cfg=metadata_server_cfg, + disagg_cluster_config=disagg_cluster_config, + multimodal_server_config=multimodal_server_config, + chat_template=chat_template, + allow_request_chat_template=allow_request_chat_template, + input_processor_workers=num_input_processor_workers, + media_load_workers=num_media_load_workers) + _apply_fastapi_middlewares(server.app, middleware) - # Optionally disable GC (default: not disabled) - if os.getenv("TRTLLM_SERVER_DISABLE_GC", "0") == "1": - gc.disable() + # Optionally disable GC (default: not disabled) + if os.getenv("TRTLLM_SERVER_DISABLE_GC", "0") == "1": + gc.disable() - uvloop.run(server(host, port, sockets=[s])) + _signal_frontend_ready(multi_frontend) + uvloop.run(server(host, port, sockets=[s])) + finally: + if frontend_children: + _terminate_attached_frontends(frontend_children) def launch_grpc_server(host: str, @@ -886,6 +1075,13 @@ def launch_visual_gen_server( help="Number of workers to postprocess raw responses " "to comply with OpenAI protocol.", status="prototype") +@stability_option("--num_serve_frontends", + type=click.IntRange(min=1, max=MAX_NUM_FRONTENDS), + default=1, + help="Number of HTTP frontend processes serving one " + "executor; values > 1 share the serving port via " + "SO_REUSEPORT (classic IPC executor path only).", + status="prototype") @stability_option("--num_input_processor_workers", type=click.IntRange(min=1), default=8, @@ -1067,29 +1263,30 @@ def launch_visual_gen_server( help= "Types of agents to schedule. Now Only Support Open Deep Research agent.", status="prototype") -def serve( - model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], - post_processor_hook: Optional[str], host: str, port: int, - log_level: str, backend: str, max_beam_width: int, max_batch_size: int, - max_num_tokens: int, max_seq_len: int, tensor_parallel_size: int, - pipeline_parallel_size: int, context_parallel_size: int, - moe_expert_parallel_size: Optional[int], - moe_cluster_parallel_size: Optional[int], gpus_per_node: Optional[int], - free_gpu_memory_fraction: float, kv_cache_dtype: str, - num_postprocess_workers: int, num_input_processor_workers: int, - num_media_load_workers: int, trust_remote_code: bool, - revision: Optional[str], extra_llm_api_options: Optional[str], - reasoning_parser: Optional[str], tool_parser: Optional[str], - metadata_server_config_file: Optional[str], server_role: Optional[str], - fail_fast_on_attention_window_too_large: bool, - otlp_traces_endpoint: Optional[str], enable_chunked_prefill: bool, - enable_attention_dp: bool, disagg_cluster_uri: Optional[str], - media_io_kwargs: Optional[str], agent_percentage: float, - agent_types: Optional[str], video_pruning_rate: Optional[float], - telemetry: bool, custom_module_dirs: list[Path], - chat_template: Optional[str], allow_request_chat_template: bool, - middleware: tuple[str, ...], grpc: bool, enable_visual_gen: bool, - served_model_name: Optional[str], visual_gen_args: Optional[str]): +def serve(model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], + post_processor_hook: Optional[str], host: str, port: int, + log_level: str, backend: str, max_beam_width: int, + max_batch_size: int, max_num_tokens: int, max_seq_len: int, + tensor_parallel_size: int, pipeline_parallel_size: int, + context_parallel_size: int, moe_expert_parallel_size: Optional[int], + moe_cluster_parallel_size: Optional[int], + gpus_per_node: Optional[int], free_gpu_memory_fraction: float, + kv_cache_dtype: str, num_postprocess_workers: int, + num_serve_frontends: int, num_input_processor_workers: int, + num_media_load_workers: int, trust_remote_code: bool, + revision: Optional[str], extra_llm_api_options: Optional[str], + reasoning_parser: Optional[str], tool_parser: Optional[str], + metadata_server_config_file: Optional[str], + server_role: Optional[str], + fail_fast_on_attention_window_too_large: bool, + otlp_traces_endpoint: Optional[str], enable_chunked_prefill: bool, + enable_attention_dp: bool, disagg_cluster_uri: Optional[str], + media_io_kwargs: Optional[str], agent_percentage: float, + agent_types: Optional[str], video_pruning_rate: Optional[float], + telemetry: bool, custom_module_dirs: list[Path], + chat_template: Optional[str], allow_request_chat_template: bool, + middleware: tuple[str, ...], grpc: bool, enable_visual_gen: bool, + served_model_name: Optional[str], visual_gen_args: Optional[str]): """Running an OpenAI API compatible server MODEL: model name | HF checkpoint path | TensorRT engine path @@ -1171,6 +1368,7 @@ def _serve_llm(): free_gpu_memory_fraction=free_gpu_memory_fraction, kv_cache_dtype=kv_cache_dtype, num_postprocess_workers=num_postprocess_workers, + num_serve_frontends=num_serve_frontends, trust_remote_code=trust_remote_code, revision=revision, reasoning_parser=reasoning_parser, @@ -1188,7 +1386,11 @@ def _serve_llm(): llm_args_extra_dict = {} if extra_llm_api_options is not None: with open(extra_llm_api_options, 'r') as f: - llm_args_extra_dict = yaml.safe_load(f) or {} + llm_args_extra_dict = yaml.safe_load(f) + if llm_args_extra_dict is None: + llm_args_extra_dict = {} + elif not isinstance(llm_args_extra_dict, dict): + raise ValueError("Configuration file root must be a mapping.") extra_allow_request_chat_template = _pop_bool_config_option( llm_args_extra_dict, "allow_request_chat_template") allow_request_chat_template = (allow_request_chat_template @@ -1423,7 +1625,11 @@ def serve_encoder(model: str, host: str, port: int, log_level: str, encoder_args_extra_dict = {} if extra_encoder_options is not None: with open(extra_encoder_options, 'r') as f: - encoder_args_extra_dict = yaml.safe_load(f) or {} + encoder_args_extra_dict = yaml.safe_load(f) + if encoder_args_extra_dict is None: + encoder_args_extra_dict = {} + elif not isinstance(encoder_args_extra_dict, dict): + raise ValueError("Configuration file root must be a mapping.") extra_allow_request_chat_template = _pop_bool_config_option( encoder_args_extra_dict, "allow_request_chat_template") allow_request_chat_template = (allow_request_chat_template @@ -1541,6 +1747,10 @@ def serve_embedding( if extra_llm_api_options is not None: with open(extra_llm_api_options, 'r') as f: extra_dict = yaml.safe_load(f) + if extra_dict is None: + extra_dict = {} + elif not isinstance(extra_dict, dict): + raise ValueError("Configuration file root must be a mapping.") llm_args = update_llm_args_with_extra_dict( llm_args, extra_dict, explicit_cli_keys=explicit_cli_keys) @@ -1624,6 +1834,7 @@ def disaggregated( """Running server in disaggregated mode""" logger.set_level(log_level) + set_prometheus_multiproc_dir() if metrics_log_interval != 0: logger.warning( @@ -1643,6 +1854,35 @@ def disaggregated( logger.info(f"Reserving disaggregated server address " f"{disagg_cfg.hostname}:{disagg_cfg.port} (pid={os.getpid()})") + metadata_server_cfg = parse_metadata_server_config_file( + metadata_server_config_file) + + # Topology from explicit config (num_workers + disagg_coordinator_url): + # (a) url set -> delegate to that external coordinator; (b) url absent, + # num_workers>1 -> implicit in-process coordinator + delegating fleet; + # (c) url absent, num_workers==1 -> single self-contained server. + num_workers = disagg_cfg.num_workers + coordinator_url = disagg_cfg.disagg_coordinator_url + + if coordinator_url: + # (a) External coordinator: fork a fleet of delegating servers (or a + # single one) pointed at it; never start a coordinator in this process. + _serve_disagg_fleet(disagg_cfg, config_file, + metadata_server_config_file, request_timeout, + server_start_timeout, num_workers, coordinator_url) + return + + if num_workers > 1: + # (b) Implicit coordinator in this process (on port-1) + a delegating + # uvicorn fleet (workers=N) on the public port. See below. + _serve_coordinator_and_fleet(disagg_cfg, config_file, + metadata_server_config_file, + metadata_server_cfg, request_timeout, + server_start_timeout, num_workers) + return + + # (c) num_workers==1, no external coordinator: a single disagg server with an + # in-process (local) coordinator. Pre-bind the socket (validates port), serve. with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: try: s.bind((disagg_cfg.hostname, disagg_cfg.port)) @@ -1656,9 +1896,6 @@ def disaggregated( f"Failed to bind socket to {disagg_cfg.hostname}:{disagg_cfg.port}: {e}. " f"Port holder(s): {holder}") - metadata_server_cfg = parse_metadata_server_config_file( - metadata_server_config_file) - server = OpenAIDisaggServer( config=disagg_cfg, req_timeout_secs=request_timeout, @@ -1682,6 +1919,302 @@ def disaggregated( uvloop.run(server(disagg_cfg.hostname, disagg_cfg.port, sockets=[s])) +def _launch_disagg_fleet(disagg_cfg, config_file, metadata_server_config_file, + request_timeout, server_start_timeout, num_workers, + coordinator_url): + """Launch delegating disaggregated-server workers. + + Each worker has its own ``Popen`` running ``_run_fleet_worker`` on the shared + public port and pointing at ``coordinator_url``. Separate processes allow an + explicit per-worker ``TLLM_DISAGG_WORKER_PROCESS_ID`` and independent + ``SO_REUSEPORT`` sockets. MPI/PMIX/SLURM environment variables are stripped. + + Returns the list of ``Popen`` handles. + """ + from tensorrt_llm.llmapi.disagg_utils import disagg_process_id_space + public_host, public_port = disagg_cfg.hostname, disagg_cfg.port + base_env = { + k: v + for k, v in os.environ.items() + if not k.startswith(("SLURM_", "PMIX_", "PMI_", "OMPI_", "UCX_", + "I_MPI_", "HYDRA_", "MPI_")) + } + # num_workers is explicit config now; ensure no stale WEB_CONCURRENCY leaks in + # and re-forks each plain-HTTP worker into a nested fleet. + base_env.pop("WEB_CONCURRENCY", None) + base_env[DisaggWorkerEnvs.TLLM_DISAGG_COORDINATOR_URL] = coordinator_url + base_env[DisaggWorkerEnvs.TLLM_DISAGG_CONFIG_FILE] = os.path.abspath( + config_file) + if metadata_server_config_file: + base_env[DisaggWorkerEnvs.TLLM_DISAGG_METADATA_CONFIG_FILE] = \ + os.path.abspath(metadata_server_config_file) + base_env[DisaggWorkerEnvs.TLLM_DISAGG_REQUEST_TIMEOUT] = str( + request_timeout) + base_env[DisaggWorkerEnvs.TLLM_DISAGG_SERVER_START_TIMEOUT] = str( + server_start_timeout) + base_env[DisaggWorkerEnvs.TLLM_DISAGG_SCHEDULE_STYLE] = \ + disagg_cfg.schedule_style + # Propagate the parent's log level so fleet workers' INFO logs (e.g. the + # per-request [ttft_split] / [coord_api] breakdowns) are not dropped. + base_env[DisaggWorkerEnvs.TLLM_DISAGG_LOG_LEVEL] = logger.level + + cmd = [ + sys.executable, "-c", + "from tensorrt_llm.commands.serve import _run_fleet_worker; " + "_run_fleet_worker()" + ] + logger.info( + f"Launching disagg fleet: {num_workers} SO_REUSEPORT workers on " + f"{public_host}:{public_port}, coordinator={coordinator_url}") + + procid_space = disagg_process_id_space() + if not 1 <= num_workers <= procid_space: + raise ValueError( + f"num_workers must be between 1 and {procid_space}, got {num_workers}" + ) + fleet = [] + for i in range(num_workers): + worker_env = dict(base_env) + # Explicit per-worker process index (no shared counter file). + worker_env[DisaggWorkerEnvs.TLLM_DISAGG_WORKER_PROCESS_ID] = str(i) + p = subprocess.Popen(cmd, + env=worker_env, + stdout=sys.stdout, + stderr=sys.stderr) + logger.info(f"Disagg fleet worker {i} launched (pid={p.pid})") + fleet.append(p) + + def _cleanup(): + for p in fleet: + if p.poll() is None: + p.terminate() + deadline = time.monotonic() + 10 + for p in fleet: + try: + p.wait(timeout=max(0, deadline - time.monotonic())) + except Exception: + p.kill() + p.wait() + + def _handle_signal(signum, _frame): + _cleanup() + raise SystemExit(128 + signum) + + atexit.register(_cleanup) + signal.signal(signal.SIGTERM, _handle_signal) + signal.signal(signal.SIGINT, _handle_signal) + return fleet + + +def _serve_disagg_fleet(disagg_cfg, config_file, metadata_server_config_file, + request_timeout, server_start_timeout, num_workers, + coordinator_url): + """External coordinator: fork the delegating fleet and block on it. + + No coordinator is started in this process -- the fleet delegates to the + already-running coordinator at ``coordinator_url``. + """ + fleet = _launch_disagg_fleet(disagg_cfg, config_file, + metadata_server_config_file, request_timeout, + server_start_timeout, num_workers, + coordinator_url) + # Block until any worker exits; a nonzero exit from any worker is a failure. + try: + while True: + for i, p in enumerate(fleet): + rc = p.poll() + if rc is not None: + if rc != 0: + raise RuntimeError( + f"Disagg fleet worker {i} (pid={p.pid}) exited with " + f"code {rc}") + # A clean exit of one worker ends the fleet. + return + time.sleep(1) + finally: + for process in fleet: + if process.poll() is None: + process.terminate() + + +def _serve_coordinator_and_fleet(disagg_cfg, config_file, + metadata_server_config_file, + metadata_server_cfg, request_timeout, + server_start_timeout, num_workers): + """workers>1, no external URL: coordinator server here + a delegating fleet. + + This process runs the coordinator server (owns the ctx/gen routers, cluster + state, and centralized ZMQ ingest) on ``port-1``; a uvicorn fleet on the + public port delegates to it. + """ + from tensorrt_llm.serve.coordinator_server import CoordinatorServer + from tensorrt_llm.serve.disagg_coordinator import DisaggCoordinatorService + + public_host, public_port = disagg_cfg.hostname, disagg_cfg.port + coord_port = int( + os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_COORDINATOR_PORT, + public_port - 1)) + # The fleet is co-located with the implicit coordinator, so route its hot + # /select,/finish over a Unix domain socket (avoids the TCP loopback stack + # that dominated per-request latency). Opt-out via the UDS env = "0". + use_uds = os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_COORDINATOR_UDS, + "1") == "1" + coord_uds = ( + f"/tmp/trtllm_disagg_coord_{public_port}.sock" # nosec B108 + if use_uds else None) + # Fleet points at the UDS when enabled; the TCP port stays up for health. + coord_url = f"unix:{coord_uds}" if coord_uds else \ + f"http://{public_host}:{coord_port}" + + # 1. Launch the delegating fleet pointed at the implicit coordinator we start + # below (port-1 for TCP; UDS for the hot path). Workers hold + # CoordinatorClients (no core), so they can't race the ZMQ ingest bind. + fleet = _launch_disagg_fleet(disagg_cfg, config_file, + metadata_server_config_file, request_timeout, + server_start_timeout, num_workers, coord_url) + + # 2. Build + serve the coordinator in this process. It OWNS routing state and + # builds the owner routers itself (single shared namespace-aware core + ONE + # ZMQ ingest server, started once here). + def _client_factory(router, role, max_retries=1): + from tensorrt_llm.serve.openai_client import OpenAIHttpClient + return OpenAIHttpClient(router, role, request_timeout, max_retries) + + coordinator = DisaggCoordinatorService( + disagg_cfg, + _client_factory, + metadata_config=metadata_server_cfg, + server_start_timeout_secs=server_start_timeout) + logger.info(f"Coordinator serving on {public_host}:{coord_port} " + f"(uds={coord_uds}) (fleet on public port {public_port})") + + async def _serve_and_monitor(): + server_task = asyncio.create_task( + CoordinatorServer(coordinator)(public_host, + coord_port, + uds=coord_uds)) + + async def _monitor_fleet(): + while True: + for i, process in enumerate(fleet): + return_code = process.poll() + if return_code is not None: + if return_code != 0: + raise RuntimeError( + f"Disagg fleet worker {i} (pid={process.pid}) " + f"exited with code {return_code}") + return + await asyncio.sleep(1) + + monitor_task = asyncio.create_task(_monitor_fleet()) + done, pending = await asyncio.wait((server_task, monitor_task), + return_when=asyncio.FIRST_COMPLETED) + for task in pending: + task.cancel() + await asyncio.gather(*pending, return_exceptions=True) + for task in done: + task.result() + + try: + asyncio.run(_serve_and_monitor()) + finally: + for process in fleet: + if process.poll() is None: + process.terminate() + + +def _init_fleet_worker_process(): + """Per-process setup shared by every fleet worker (one OS process each). + + Restores logging/GC state for the fresh ``python -c`` interpreter and tags + every log line with this worker's PID so the shared-stdout fleet output stays + attributable. + """ + # A worker is a plain HTTP process, never an MPI rank; drop WEB_CONCURRENCY so + # it is never itself re-forked into multiple uvicorn workers. + os.environ.pop("WEB_CONCURRENCY", None) + if os.getenv("TRTLLM_DISAGG_SERVER_DISABLE_GC", "1") == "1": + gc.disable() + + # This is a fresh Python process, so the TRT-LLM logger defaults to WARNING + # and would drop the workers' INFO logs (per-request [ttft_split] / + # [coord_api]). Restore the parent's level. + _worker_log_level = os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_LOG_LEVEL) + if _worker_log_level: + logger.set_level(_worker_log_level) + + # All N fleet workers share one stdout; tag every trtllm log line from this + # worker with its PID so interleaved fleet output is attributable. + import logging as _logging + for _h in _logging.getLogger("TRT-LLM").handlers: + _h.setFormatter( + _logging.Formatter( + fmt= + f"[%(asctime)s] [fleet-worker pid={os.getpid()}] %(message)s", + datefmt="%m/%d/%Y-%H:%M:%S")) + + +def _build_disagg_server_from_env() -> "OpenAIDisaggServer": + """Build one delegating disagg server from the env the launcher exported. + + Fully stateless -- reads config + coordinator URL from ``DisaggWorkerEnvs``; + the server holds a remote ``CoordinatorClient`` so routing/readiness are + delegated to the coordinator. The worker's process index (for the snowflake + disagg-id) is read from ``TLLM_DISAGG_WORKER_PROCESS_ID``, which the launcher + set explicitly per worker (see ``worker_local_process_id``). + """ + config_file = os.environ[DisaggWorkerEnvs.TLLM_DISAGG_CONFIG_FILE] + coordinator_url = os.environ[DisaggWorkerEnvs.TLLM_DISAGG_COORDINATOR_URL] + metadata_config_file = os.environ.get( + DisaggWorkerEnvs.TLLM_DISAGG_METADATA_CONFIG_FILE) + request_timeout = int( + os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_REQUEST_TIMEOUT, "180")) + server_start_timeout = int( + os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_SERVER_START_TIMEOUT, + "180")) + + disagg_cfg = parse_disagg_config_file(config_file) + schedule_style = os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_SCHEDULE_STYLE) + if schedule_style: + disagg_cfg.schedule_style = schedule_style + metadata_server_cfg = parse_metadata_server_config_file( + metadata_config_file) + + server = OpenAIDisaggServer(config=disagg_cfg, + req_timeout_secs=request_timeout, + server_start_timeout_secs=server_start_timeout, + metadata_server_cfg=metadata_server_cfg, + coordinator_url=coordinator_url) + logger.info(f"Disagg server built, coordinator={coordinator_url}") + return server + + +def _run_fleet_worker(): + """Run one fleet worker process. + + Each ``Popen`` receives an explicit ``TLLM_DISAGG_WORKER_PROCESS_ID`` and + binds its own ``SO_REUSEPORT`` socket on the shared public port. + """ + _init_fleet_worker_process() + server = _build_disagg_server_from_env() + host, port = server._config.hostname, server._config.port + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # SO_REUSEPORT: every worker binds the same (host, port); the kernel spreads + # connections across the workers' accept queues by 4-tuple hash. + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + try: + s.bind((host, port)) + except OSError as e: + raise RuntimeError( + f"Fleet worker failed to SO_REUSEPORT-bind {host}:{port}: {e}") + pidx = os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_WORKER_PROCESS_ID, "0") + logger.info(f"Fleet worker process_id={pidx} bound {host}:{port} " + f"(SO_REUSEPORT)") + asyncio.run(server(host, port, sockets=[s])) + + def set_cuda_device(): if (os.getenv("OMPI_COMM_WORLD_RANK")): env_global_rank = int(os.environ["OMPI_COMM_WORLD_RANK"]) @@ -1779,6 +2312,29 @@ class DisaggLauncherEnvs(StrEnum): TLLM_DISAGG_ROLE = "TRTLLM_DISAGG_ROLE" +class DisaggWorkerEnvs(StrEnum): + # Passed from the `disaggregated` coordinator to the fleet of worker processes + # (one Popen per worker) via env, then read by _run_fleet_worker in each worker. + TLLM_DISAGG_COORDINATOR_URL = "TRTLLM_DISAGG_COORDINATOR_URL" + TLLM_DISAGG_CONFIG_FILE = "TRTLLM_DISAGG_CONFIG_FILE" + TLLM_DISAGG_METADATA_CONFIG_FILE = "TRTLLM_DISAGG_METADATA_CONFIG_FILE" + TLLM_DISAGG_REQUEST_TIMEOUT = "TRTLLM_DISAGG_REQUEST_TIMEOUT" + TLLM_DISAGG_SERVER_START_TIMEOUT = "TRTLLM_DISAGG_SERVER_START_TIMEOUT" + TLLM_DISAGG_SCHEDULE_STYLE = "TRTLLM_DISAGG_SCHEDULE_STYLE" + # Parent's logger level; the forked uvicorn fleet is a fresh process that + # otherwise defaults to WARNING and drops the workers' INFO logs. + TLLM_DISAGG_LOG_LEVEL = "TRTLLM_DISAGG_LOG_LEVEL" + # Coordinator transport (read in _serve_coordinator_and_fleet): PORT overrides + # the coordinator TCP port (default public_port-1); UDS="1" (default) routes the + # co-located fleet's hot /select,/finish over a Unix domain socket, "0" for TCP. + TLLM_DISAGG_COORDINATOR_PORT = "TRTLLM_DISAGG_COORDINATOR_PORT" + TLLM_DISAGG_COORDINATOR_UDS = "TRTLLM_DISAGG_COORDINATOR_UDS" + # Per-fleet-worker process index (0..N-1) → the snowflake disagg-id process_id + # field so co-located workers never collide; the launcher sets one distinct + # value per worker Popen (see worker_local_process_id). Defaults to 0 if unset. + TLLM_DISAGG_WORKER_PROCESS_ID = "TRTLLM_DISAGG_WORKER_PROCESS_ID" + + def _launch_disaggregated_server(disagg_config_file: str, llm_args: dict): # Launching the server instance_idx = os.environ.get(DisaggLauncherEnvs.TLLM_DISAGG_INSTANCE_IDX) @@ -1798,7 +2354,9 @@ def _launch_disaggregated_server(disagg_config_file: str, llm_args: dict): host=server_cfg.hostname, port=server_cfg.port, llm_args=llm_args, - allow_request_chat_template=disagg_config.allow_request_chat_template) + allow_request_chat_template=disagg_config.allow_request_chat_template, + # Disagg ctx/gen MPI workers must not enter multi-frontend mode. + multi_frontend_enabled=False) def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index 79f32000cfd7..f3128d605aae 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -61,11 +61,12 @@ # See generate_until for the throughput/early-signal tradeoff. MAX_IN_FLIGHT_ENV_VAR = "TLLM_EVAL_MAX_IN_FLIGHT" -# When "1", request per-request perf metrics and log an aggregate -# speculative-decoding summary (acceptance length AL as mean decoded -# tokens/step, acceptance rate AR as accepted/drafted draft tokens) at the -# end of generate_until. No-op output on non-speculative runs; unset/0 -# leaves sampling params untouched (no perf-metrics overhead). +# When "1", log an aggregate speculative-decoding summary (acceptance +# length AL as mean decoded tokens/step) at the end of generate_until. +# No-op output on non-speculative runs. Acceptance rate (AR) reporting is +# deferred: request_perf_metrics.speculative_decoding counters are only +# populated by TRTLLMSampler, not the TorchSampler used by one-engine +# spec-dec, so AR would silently read 0 on the default PyTorch path. SPEC_STATS_ENV_VAR = "TLLM_EVAL_SPEC_STATS" @@ -85,9 +86,16 @@ class _RunningScoreTracker: by lm-eval is unaffected — scoring here happens on copies. Any failure (exotic task/filter/metric shapes) permanently disables the tracker for the run and never fails the eval itself. + + CAVEAT: the copy is shallow — ``probe.doc`` is shared with the live + instance, and filters / ``process_results`` run on the live task object. + lm-eval's filter contract is task/corpus-level and documents are mutable, + so a custom filter or a stateful ``process_results`` could in principle + touch state the final score depends on. The estimate is only meaningful + (and side-effect free) for stock per-document filter pipelines. """ - def __init__(self, task_dict: dict, interval: int): + def __init__(self, task_dict: dict, interval: int) -> None: self.interval = interval self.tasks = {} self._collect_tasks(task_dict) @@ -102,14 +110,13 @@ def _collect_tasks(self, task_dict: dict) -> None: else: self.tasks[name] = obj - def update(self, instance, text: str) -> None: + def update(self, instance: Any, text: str) -> None: if self.disabled: return try: task = self.tasks.get(instance.task_name) if task is None or not getattr(task, "_filters", None): - raise ValueError( - f"no scorable task for {instance.task_name!r}") + raise ValueError(f"no scorable task for {instance.task_name!r}") # Score a shallow copy: the harness appends resps / applies # filters to the real instance later, and must see it untouched. probe = copy.copy(instance) @@ -125,7 +132,7 @@ def update(self, instance, text: str) -> None: probe.doc, [probe.filtered_resps[ensemble.name]]) for metric, value in metrics.items(): if isinstance(value, (int, float)): - key = f"{metric},{ensemble.name}" + key = f"{instance.task_name},{metric},{ensemble.name}" self.metric_sums[key] += value self.metric_counts[key] += 1 except Exception as e: @@ -139,13 +146,25 @@ def maybe_log(self, done: int, total: int) -> None: return if done % self.interval != 0 and done != total: return - stats = " | ".join( - f"{key} ~ {100 * self.metric_sums[key] / count:.2f}" - for key, count in self.metric_counts.items()) + stats = " | ".join(f"{key} ~ {100 * self.metric_sums[key] / count:.2f}" + for key, count in self.metric_counts.items()) logger.info(f"Partial scores after {done}/{total} responses " f"(estimate, 0~100): {stats}") +def _parse_partial_scores_env() -> Optional[int]: + """Parse TLLM_EVAL_PARTIAL_SCORES_EVERY and return the logging interval or None.""" + env_interval = os.environ.get(PARTIAL_SCORES_ENV_VAR) + if not env_interval: + return None + try: + value = int(env_interval) + except ValueError: + raise ValueError(f"{PARTIAL_SCORES_ENV_VAR} must be an integer, got " + f"{env_interval!r}") from None + return value if value > 0 else None + + class LmEvalWrapper(TemplateLM): def __init__(self, @@ -193,8 +212,8 @@ def __init__(self, self.max_in_flight = 0 else: self.max_in_flight = 0 - # Env-gated speculative-decoding stats (AL/AR) aggregation over the - # eval corpus; forces return_perf_metrics on generated requests. + # Env-gated speculative-decoding stats (AL) aggregation over the + # eval corpus. See SPEC_STATS_ENV_VAR. self.spec_stats = os.environ.get(SPEC_STATS_ENV_VAR) == "1" @property @@ -275,42 +294,47 @@ def _get_sampling_params(self, gen_kwargs: dict) -> SamplingParams: if current is not None and current > value: continue setattr(sampling_params, trtllm_key, value) - if self.spec_stats: - sampling_params.return_perf_metrics = True return sampling_params def _log_spec_stats(self, outputs: List[RequestOutput]) -> None: """Log corpus-aggregate speculative-decoding stats (TLLM_EVAL_SPEC_STATS=1). - AL (acceptance length) is reported as the mean of per-request + AL (acceptance length) is the iteration-weighted mean of per-request ``avg_decoded_tokens_per_iter`` (target token + accepted draft tokens - per decode step); AR (acceptance rate) as total accepted / total - drafted draft tokens from the per-request perf metrics. Skips silently - when the run produced no speculative metrics (non-spec-dec config). + per decode step), i.e. total decoded tokens / total decode iterations. + This matches the repo's canonical AL definition in + ``bench/dataclasses/reporting.py``: an equally-weighted mean would + bias the result toward short requests, which run fewer decode + iterations; iteration weighting makes it a token-level mean so longer + requests contribute proportionally. Requests that don't expose a + usable ``decoding_iter`` fall back to weight 1. min/max stay + per-request values. Skips silently when the run produced no + speculative metrics (non-spec-dec config). + + Acceptance rate (AR) is intentionally not reported: the per-request + ``request_perf_metrics.speculative_decoding`` counters it needs are + only maintained by TRTLLMSampler (not the TorchSampler used by + one-engine spec-dec), so it would silently read 0 on the default + PyTorch path. Revisit once those counters are populated there. """ - tokens_per_iter = [ - output.avg_decoded_tokens_per_iter for output in outputs - if getattr(output, "avg_decoded_tokens_per_iter", None) is not None - ] - accepted = drafted = 0 + samples = [] # (avg_decoded_tokens_per_iter, weight=decode iterations) for output in outputs: - perf_metrics = getattr(output.outputs[0], "request_perf_metrics", - None) - spec_dec = getattr(perf_metrics, "speculative_decoding", - None) if perf_metrics else None - if spec_dec is not None: - accepted += spec_dec.total_accepted_draft_tokens - drafted += spec_dec.total_draft_tokens - if tokens_per_iter: - mean_tpi = sum(tokens_per_iter) / len(tokens_per_iter) + tpi = getattr(output, "avg_decoded_tokens_per_iter", None) + if tpi is None: + continue + iters = getattr(output, "decoding_iter", None) + weight = iters if isinstance(iters, (int, float)) and iters > 0 \ + else 1 + samples.append((tpi, weight)) + if samples: + weighted_al = (sum(tpi * w for tpi, w in samples) / + sum(w for _, w in samples)) + tokens_per_iter = [tpi for tpi, _ in samples] logger.info( - f"Spec-dec stats: AL (mean decoded tokens/step) {mean_tpi:.3f} " - f"(min {min(tokens_per_iter):.3f}, " + f"Spec-dec stats: AL (decoded tokens/step, " + f"iteration-weighted) {weighted_al:.3f} " + f"(per-request min {min(tokens_per_iter):.3f}, " f"max {max(tokens_per_iter):.3f}, n={len(tokens_per_iter)})") - if drafted > 0: - logger.info( - f"Spec-dec stats: AR (accepted/drafted draft tokens) " - f"{accepted}/{drafted} = {accepted / drafted:.2%}") def _generate_until_windowed(self, requests, scorer, disable_tqdm: bool) -> List[RequestOutput]: @@ -336,6 +360,8 @@ def _generate_until_windowed(self, requests, scorer, order matches submission order — downstream handling is unchanged. """ total = len(requests) + if total == 0: + return [] outputs: List[Optional[RequestOutput]] = [None] * total next_idx = 0 done_count = 0 @@ -355,46 +381,59 @@ def _submit_next(pool): streaming=self.streaming) return pool.submit(lambda: (idx, output.result())) - pbar = tqdm(total=total, desc="Fetching responses (windowed)", + pbar = tqdm(total=total, + desc="Fetching responses (windowed)", disable=disable_tqdm) + # The window size also bounds the waiter-thread count, so cap it at + # the request count (W >= total would otherwise spawn one idle-capable + # thread per request). + pool = concurrent.futures.ThreadPoolExecutor( + max_workers=min(self.max_in_flight, total)) try: - with concurrent.futures.ThreadPoolExecutor( - max_workers=self.max_in_flight) as pool: - pending = { - _submit_next(pool) - for _ in range(min(self.max_in_flight, total)) - } - while pending: - done, pending = concurrent.futures.wait( - pending, - return_when=concurrent.futures.FIRST_COMPLETED) - for fut in done: - # A failed request re-raises here (same as the - # non-windowed path's output.result()); no deadlock — - # pool shutdown just drains the remaining in-flight - # waiters, which the engine unblocks on completion or - # via EngineDeadError. - idx, output = fut.result() - outputs[idx] = output - done_count += 1 - pbar.update(1) - if scorer is not None: - scorer.update(requests[idx], - output.outputs[0].text) - scorer.maybe_log(done_count, total) - if next_idx < total: - pending.add(_submit_next(pool)) + pending = { + _submit_next(pool) + for _ in range(min(self.max_in_flight, total)) + } + while pending: + done, pending = concurrent.futures.wait( + pending, return_when=concurrent.futures.FIRST_COMPLETED) + for fut in done: + # A failed request re-raises here (same as the + # non-windowed path's output.result()). + idx, output = fut.result() + outputs[idx] = output + done_count += 1 + pbar.update(1) + if scorer is not None: + scorer.update(requests[idx], output.outputs[0].text) + scorer.maybe_log(done_count, total) + if next_idx < total: + pending.add(_submit_next(pool)) + except BaseException: + # Fail fast: a blocking shutdown here (as a `with` block's + # __exit__ would do) joins every other in-flight waiter with no + # cancellation or timeout — if any of those never resolves, the + # exception can't escape and the eval hangs instead of failing. + # shutdown(wait=False) lets the exception propagate immediately; + # already-running waiters unblock when the engine completes or + # kills their requests (e.g. EngineDeadError). + pool.shutdown(wait=False, cancel_futures=True) + raise finally: pbar.close() + pool.shutdown(wait=True) return outputs + def _make_partial_scorer(self) -> Optional["_RunningScoreTracker"]: + if self.partial_scores_every and self.partial_scoring_task_dict: + return _RunningScoreTracker(self.partial_scoring_task_dict, + self.partial_scores_every) + return None + def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: profiler.start("trtllm exec") - scorer = None - if self.partial_scores_every and self.partial_scoring_task_dict: - scorer = _RunningScoreTracker(self.partial_scoring_task_dict, - self.partial_scores_every) + scorer = self._make_partial_scorer() if self.max_in_flight > 0 and not self.streaming: # Env-gated (TLLM_EVAL_MAX_IN_FLIGHT) submission windowing for the @@ -417,7 +456,8 @@ def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: outputs = [] for output, request in zip( - tqdm(results, desc="Fetching responses", + tqdm(results, + desc="Fetching responses", disable=disable_tqdm), requests): outputs.append(output.result()) if scorer is not None: @@ -479,9 +519,10 @@ def __init__(self, to model outputs before scoring. Used by Kimi K2.5 to strip ``...`` and extract the final answer (see ``tensorrt_llm.evaluate.post_processing``). - partial_scores_every: Accepted for interface parity with - LmEvalWrapper; the multimodal generate_until override does - not implement partial scoring yet. + partial_scores_every: Same as LmEvalWrapper — log running + metric estimates every N completed responses. The multimodal + path scores the post-processed text (the same string lm-eval + itself scores). partial_scoring_task_dict: See partial_scores_every. """ super().__init__( @@ -664,6 +705,7 @@ def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: # NOTE: TLLM_EVAL_MAX_IN_FLIGHT submission windowing (see # LmEvalWrapper.generate_until) is intentionally NOT applied to this # multimodal path; it keeps the original submit-all behavior. + scorer = self._make_partial_scorer() results = [] for request in tqdm(requests, desc="Submitting requests", @@ -691,10 +733,24 @@ def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: results.append(output) outputs = [] - for output in tqdm(results, - desc="Fetching responses", - disable=disable_tqdm): + results_text = [] + for output, request in zip( + tqdm(results, desc="Fetching responses", disable=disable_tqdm), + requests): outputs.append(output.result()) + # Apply per-sample post-processing only when caller injected one. + # Kimi K2.5 passes strip_thinking_and_extract_mmmu_answer to + # recover answers from ...-wrapped outputs that + # lm-eval's default extractor cannot parse. + raw = outputs[-1].outputs[0].text + text = self.post_process_fn( + raw) if self.post_process_fn is not None else raw + results_text.append(text) + if scorer is not None: + # Score the post-processed text — the same string lm-eval + # itself will score. + scorer.update(request, text) + scorer.maybe_log(len(outputs), len(requests)) if self.output_dir: dump_inference_results(self.output_dir, outputs, @@ -708,18 +764,6 @@ def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: logger.info(f"TRTLLM execution time: {elapsed_time:.3f} seconds.") profiler.reset("trtllm exec") - # Apply per-sample post-processing only when caller injected one. - # Kimi K2.5 passes strip_thinking_and_extract_mmmu_answer to recover - # answers from ...-wrapped outputs that lm-eval's - # default extractor cannot parse. - results_text = [] - for output in outputs: - raw = output.outputs[0].text - if self.post_process_fn is not None: - results_text.append(self.post_process_fn(raw)) - else: - results_text.append(raw) - return results_text @@ -879,17 +923,7 @@ def evaluate(self, # score every N completed responses (see _RunningScoreTracker). # Env-var driven so it uniformly covers every lm-eval-backed task # without per-task CLI plumbing. - partial_scores_every = None - env_interval = os.environ.get(PARTIAL_SCORES_ENV_VAR) - if env_interval: - try: - partial_scores_every = int(env_interval) - except ValueError: - raise ValueError( - f"{PARTIAL_SCORES_ENV_VAR} must be an integer, got " - f"{env_interval!r}") from None - if partial_scores_every <= 0: - partial_scores_every = None + partial_scores_every = _parse_partial_scores_env() lm_kwargs: Dict[str, Any] = dict( sampling_params=sampling_params, diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index 670d6f30508f..5e1413a4f0da 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -49,9 +49,11 @@ from .result import (GenerationResult, LogProbsResult, ResponseWrapper, compute_logprobs, get_metrics_dict) from .utils import (ErrorResponse, IntraProcessQueue, RequestError, + bucket_responses_by_frontend, frontend_lane_index, is_llm_response) if TYPE_CHECKING: + from .._torch.pyexecutor.kv_cache_transceiver import KvCacheTransceiver from ..disaggregated_params import DisaggregatedParams __all__ = [ @@ -118,6 +120,9 @@ def __init__( self.engine = None self.result_queue: Optional[IpcQueue] = None self.postproc_queues: Optional[List[IpcQueue]] = None + # Multi-frontend serving: one result lane per frontend process, + # selected by the frontend id in client_id's top bits. + self.frontend_result_queues: Optional[List[IpcQueue]] = None self.rank = mpi_rank() self.global_rank = global_mpi_rank() # mapping: client_id -> GenerationResult @@ -207,24 +212,8 @@ def _create_py_executor(): self.max_seq_len = _executor.max_seq_len return _executor - def _create_engine(executor_config): - engine = self._engine - if executor_config is None: - executor_config = tllm.ExecutorConfig(1) - executor_config.logits_post_processor_config = tllm.LogitsPostProcessorConfig( - processor_batched=self._batched_logits_processor, - replicate=False) - comm_ranks, device_ids = self._get_comm_ranks_device_id() - executor_config.parallel_config = tllm.ParallelConfig( - participant_ids=comm_ranks, device_ids=device_ids) - - assert not hasattr(executor_config, "backend") - return tllm.Executor(engine, tllm.ModelType.DECODER_ONLY, - executor_config) - - self.engine = _create_py_executor( - ) if self.llm_args is not None else _create_engine( - self._executor_config) + assert self.llm_args is not None, "llm_args is required to set up the worker engine" + self.engine = _create_py_executor() self._lora_manager: Optional[LoraManager] = None self._prompt_adapter_manager: Optional[PromptAdapterManager] = None @@ -245,16 +234,10 @@ def await_responses(self, timeout: Optional[float] = None) -> list: seconds=timeout) if timeout is not None else None) def fetch_stats(self) -> list: - if isinstance(self.engine, tllm.Executor): - iter_stats = self.engine.get_latest_iteration_stats() - #TODO: Support req stats with TRT engine - # This would require ensuring iter and req stats have same size - return [(iter_stat, None, None) for iter_stat in iter_stats] - else: - return self.engine.get_latest_iteration_stats() + return self.engine.get_latest_iteration_stats() def fetch_kv_cache_capacity(self) -> dict: - if self.engine is None or isinstance(self.engine, tllm.Executor): + if self.engine is None: return {} from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor @@ -264,21 +247,26 @@ def fetch_kv_cache_capacity(self) -> dict: return {} def fetch_kv_cache_events(self) -> list: - if isinstance(self.engine, tllm.Executor): - return self.engine.get_latest_kv_cache_events() - else: - return self.engine.get_latest_kv_cache_events() + return self.engine.get_latest_kv_cache_events() def set_result_queue(self, queue): """In multi-gpu mode, result_queue will be set here to communicate between the proxy and the worker 0 process.""" assert self.postproc_queues is None + assert self.frontend_result_queues is None self.result_queue = queue def set_postproc_queues(self, queues: List["IpcQueue"]): """ Set the IPC queues for feeding post-processing processes. """ assert self.result_queue is None + assert self.frontend_result_queues is None self.postproc_queues = queues + def set_frontend_result_queues(self, queues: List["IpcQueue"]): + """Multi-frontend serving: one result lane per frontend process.""" + assert self.result_queue is None + assert self.postproc_queues is None + self.frontend_result_queues = queues + def _set_iteration_result_queue(self, it_result_queue: IterationResultQueue, queue: Union[Queue, FusedIpcQueue, IntraProcessQueue]): @@ -437,6 +425,10 @@ def _deduce_max_tokens(request: GenerationRequest, llm_args: Optional[BaseLlmArgs] = None) -> int: # deduce max_tokens when it's not set by user max_tokens = request.sampling_params.max_tokens + output_prefix_len = len( + request.sampling_params._decoder_output_token_prefix) + if max_tokens is not None: + max_tokens -= output_prefix_len query_token_len = len( request.query_token_ids) if request.query_token_ids else 0 @@ -554,6 +546,8 @@ def _deduce_max_tokens(request: GenerationRequest, # E/P handoff embedding handles parked under "multimodal_embedding". request.multimodal_params.to_tensor("multimodal_data") executor_request.py_multimodal_data = request.multimodal_params.multimodal_data + if request.multimodal_params.mm_item_order: + executor_request.py_mm_item_order = request.multimodal_params.mm_item_order if self._is_pytorch_backend and request.sampling_params.logits_processor: # For PyTorch backend, we attach logits processors as a dynamic Python attribute @@ -979,6 +973,16 @@ def get_disaggregated_params(self) -> dict: return {} return self.engine.kv_cache_transceiver.get_disaggregated_params() + def get_cache_transceiver(self) -> Optional["KvCacheTransceiver"]: + if self.engine is None: + return None + return self.engine.kv_cache_transceiver + + def get_data_transceiver_state(self) -> bytes: + if self.engine is None or self.engine.kv_cache_transceiver is None: + return b"" + return self.engine.kv_cache_transceiver.get_data_transceiver_state() + @staticmethod def _stats_serializer(stats) -> str: # Per-rank path: stats is ("per_rank_dict", {..., "rank": N}). @@ -1087,20 +1091,20 @@ def responses_handler(self, responses: List[tllm.Response]): HandlerKind = AwaitResponseHelper.HandlerKind if self.handler_kind is HandlerKind.unknown: - if not (self.worker.result_queue is not None - or self.worker.postproc_queues is not None): + has_ipc_queues = (self.worker.result_queue is not None + or self.worker.postproc_queues is not None + or self.worker.frontend_result_queues is not None) + if not has_ipc_queues: logger_debug(f"creating await_response helper for Worker\n", color="yellow") # When ExecutorBindingWorker is used in the main process # aka the single process mode self.handler_kind = HandlerKind.single_process_worker - elif self.worker.result_queue is not None or self.worker.postproc_queues is not None: + else: # The ExecutorBindingProxy is used logger_debug(f"creating await_response helper for IPC\n", color="yellow") self.handler_kind = HandlerKind.ipc_batched - else: - raise NotImplementedError match self.handler_kind: case HandlerKind.single_process_worker: @@ -1279,7 +1283,13 @@ def handle_for_ipc_batched(self, responses: List[tllm.Response]) -> None: self.worker.postproc_queues[wid].put(batch) if rsp_batch: - self.worker.result_queue.put(rsp_batch) + if (lanes := self.worker.frontend_result_queues) is not None: + for frontend_id, sub_batch in enumerate( + bucket_responses_by_frontend(rsp_batch, len(lanes))): + if sub_batch: + lanes[frontend_id].put(sub_batch) + else: + self.worker.result_queue.put(rsp_batch) def _get_params_for_first_rsp( @@ -1393,7 +1403,16 @@ def _send_rsp( rsp_batch: Optional[List[tllm.Response]] = None): # if postproc_batches is set, append to batch instead of putting to IpcQueue - if worker.result_queue is not None: + if worker.frontend_result_queues is not None: + # Route to the origin frontend's result lane; None/out-of-range ids + # fall back to lane 0 (see frontend_lane_index). + if rsp_batch is not None: + rsp_batch.append(response) + else: + lanes = worker.frontend_result_queues + lanes[frontend_lane_index(response.client_id, + len(lanes))].put(response) + elif worker.result_queue is not None: if rsp_batch is not None: rsp_batch.append(response) else: diff --git a/tensorrt_llm/executor/executor.py b/tensorrt_llm/executor/executor.py index ba5b843ff347..98afe8daa44f 100644 --- a/tensorrt_llm/executor/executor.py +++ b/tensorrt_llm/executor/executor.py @@ -1,6 +1,8 @@ import atexit import faulthandler +import json import multiprocessing +import os import platform import signal import traceback @@ -41,6 +43,7 @@ from .utils import IntraProcessQueue, ProcessPoolExecutorSession, RequestError if TYPE_CHECKING: + from .._torch.pyexecutor.kv_cache_transceiver import KvCacheTransceiver from .proxy import GenerationExecutorProxy from .worker import GenerationExecutorWorker @@ -462,6 +465,12 @@ def aget_kv_events(self, timeout=None) -> IterationResult: def get_disaggregated_params(self) -> dict: return {} + def get_cache_transceiver(self) -> Optional["KvCacheTransceiver"]: + return None + + def get_data_transceiver_state(self) -> bytes: + return b"" + @staticmethod def _create_ray_executor( worker_kwargs: Dict, @@ -560,6 +569,35 @@ def create( f"Using {postproc_worker_config.num_postprocess_workers} postprocess parallel processes.\n", "green") + # Multi-frontend serving: attach to an already-running executor + # instead of launching one (set by trtllm-serve for attached + # frontend processes). + attach_env = os.getenv("TLLM_EXECUTOR_ATTACH_INFO") + if attach_env: + attach_info = json.loads(attach_env) + # Consumed: it carries the HMAC keys and must not leak into + # descendant processes. + os.environ.pop("TLLM_EXECUTOR_ATTACH_INFO", None) + if attach_info.get("mode") != "classic": + raise ValueError( + "TLLM_EXECUTOR_ATTACH_INFO only supports the classic IPC " + f"executor path, got mode={attach_info.get('mode')!r}") + from .proxy import GenerationExecutorFrontendProxy + frontend_id_env = os.getenv("TLLM_EXECUTOR_FRONTEND_ID") + if frontend_id_env is None: + raise ValueError( + "TLLM_EXECUTOR_ATTACH_INFO is set but " + "TLLM_EXECUTOR_FRONTEND_ID is not; both are set together " + "by trtllm-serve when spawning attached frontends.") + frontend_id = int(frontend_id_env) + logger.info(f"Attaching executor frontend {frontend_id} to the " + "running classic IPC worker") + return GenerationExecutorFrontendProxy( + attach_info, + frontend_id=frontend_id, + postproc_worker_config=postproc_worker_config, + is_llm_executor=is_llm_executor) + worker_kwargs = { "engine": engine, "executor_config": executor_config, diff --git a/tensorrt_llm/executor/postproc_worker.py b/tensorrt_llm/executor/postproc_worker.py index ece60b2374b1..552c5c8f2bd3 100644 --- a/tensorrt_llm/executor/postproc_worker.py +++ b/tensorrt_llm/executor/postproc_worker.py @@ -15,7 +15,7 @@ from ..sampling_params import SamplingParams from .ipc import ZeroMqQueue from .postprocessor_hook import load_post_processor_hook -from .utils import ErrorResponse, is_llm_response +from .utils import ErrorResponse, bucket_responses_by_frontend, is_llm_response if TYPE_CHECKING: from ..disaggregated_params import DisaggregatedParams @@ -86,7 +86,7 @@ class Output(NamedTuple): def __init__( self, pull_pipe_addr: tuple[str, Optional[bytes]], - push_pipe_addr: tuple[str, Optional[bytes]], + push_pipe_addrs: List[tuple[str, Optional[bytes]]], tokenizer_dir: str, record_creator: Callable[ ["PostprocWorker.Input", TransformersTokenizer], Any], @@ -95,7 +95,9 @@ def __init__( ''' Args: pull_pipe_addr (tuple[str, Optional[bytes]]): The address and HMAC key of the input IPC. - push_pipe_addr (tuple[str, Optional[bytes]]): The address and HMAC key of the output IPC. + push_pipe_addrs: The addresses and HMAC keys of the output IPC + lanes, one per frontend (a single-element list in + single-frontend mode). tokenizer_dir (str): The directory to load tokenizer. record_creator (Callable[["ResponsePostprocessWorker.Input"], Any]): A creator for creating a record for a request. result_handler (Optional[Callable[[GenerationResultBase], Any]]): A callback handles the final result. @@ -108,11 +110,14 @@ def __init__( is_async=True, is_server=False, name="postprocess_pull_pipe") - self._push_pipe = ZeroMqQueue(address=push_pipe_addr, - is_async=True, - is_server=False, - socket_type=zmq.PUSH, - name="postprocess_push_pipe") + self._push_pipes = [ + ZeroMqQueue(address=addr, + is_async=True, + is_server=False, + socket_type=zmq.PUSH, + name=f"postprocess_push_pipe_{i}") + for i, addr in enumerate(push_pipe_addrs) + ] self._to_stop = asyncio.Event() self._q = deque() @@ -192,11 +197,19 @@ async def _batched_put(self): ''' Batched IPC send. ''' async for batch in self._mainloop(): if batch is None: - # notify dispatch_result corountine to quit - await self._push_pipe.put_async(None) + # notify the dispatch_result coroutine in every frontend to + # quit + for pipe in self._push_pipes: + await pipe.put_async(None) break assert isinstance(batch, list) - await self._push_pipe.put_async(batch) + if len(self._push_pipes) == 1: + await self._push_pipes[0].put_async(batch) + continue + for frontend_id, sub_batch in enumerate( + bucket_responses_by_frontend(batch, len(self._push_pipes))): + if sub_batch: + await self._push_pipes[frontend_id].put_async(sub_batch) async def _mainloop(self): ''' The loop for handle_response and keep producing outputs. ''' @@ -289,13 +302,13 @@ async def main(): @print_traceback_on_error def postproc_worker_main(feedin_ipc_addr: tuple[str, Optional[bytes]], - feedout_ipc_addr: tuple[str, Optional[bytes]], + feedout_ipc_addrs: List[tuple[str, Optional[bytes]]], tokenizer_dir: str, record_creator: Callable, post_processor_hook: Optional[str] = None): # Pass the hook import path; PostprocWorker builds it once. worker = PostprocWorker(feedin_ipc_addr, - feedout_ipc_addr, + feedout_ipc_addrs, tokenizer_dir=tokenizer_dir, record_creator=record_creator, post_processor_hook=post_processor_hook) diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index 11451bad5b67..d1baa0792785 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -16,6 +16,8 @@ import concurrent.futures import json import os +import shutil +import tempfile import threading import weakref from queue import Empty @@ -44,12 +46,14 @@ from .utils import (EngineDeadError, ErrorResponse, RequestError, WorkerCommIpcAddrs, create_mpi_comm_session, get_spawn_proxy_process_env, is_llm_response, - print_alive_threads) + multi_frontend_request_addr, multi_frontend_result_addr, + namespace_client_id, print_alive_threads) from .worker import GenerationExecutorWorker, worker_main from .worker_process_monitor import WorkerProcessIdentity, WorkerProcessMonitor __all__ = [ "GenerationExecutorProxy", + "GenerationExecutorFrontendProxy", ] # Methods that are explicitly implemented for multi-rank MPI/IPC executor @@ -161,6 +165,24 @@ def __init__( self._enable_resource_governor = bool( getattr(_llm_args, "enable_resource_governor", False)) + # Multi-frontend serving: this launcher proxy owns the shared ipc + # dir + HMAC key for the per-frontend endpoints (_setup_queues); + # trtllm-serve hands them to the attached frontends via + # multi_frontend_attach_info(). + self._num_frontends = (_llm_args.num_serve_frontends + if _llm_args is not None else 1) + self._multi_frontend_ipc_dir: Optional[str] = None + self._multi_frontend_hmac: Optional[bytes] = None + if self._num_frontends > 1: + if self._enable_resource_governor: + raise ValueError( + "Multi-frontend serving does not support " + "enable_resource_governor: the resource-governor signal " + "only reaches the launcher frontend.") + self._multi_frontend_ipc_dir = tempfile.mkdtemp( + prefix="trtllm_frontends_") + self._multi_frontend_hmac = os.urandom(32) + # Generate RPC address and key for stats RPC self.rpc_addr = get_unique_ipc_addr() self.hmac_key = os.urandom(32) @@ -311,6 +333,18 @@ def _mark_engine_dead(self, error: Optional[BaseException] = None) -> None: result.queue.put(dead_error) except Exception: # noqa: BLE001 - a full/closed queue must not stop the sweep pass + # Release the session's exit joins here, not at teardown: interpreter + # exit joins non-daemon threads before any teardown code runs, so a + # wedged pool manager thread must be deregistered while user code is + # still alive. Non-destructive, hence safe for unowned sessions. + release = getattr(getattr(self, 'mpi_session', None), + 'release_exit_joins', None) + if release is not None: + try: + release() + except Exception as e: # noqa: BLE001 - best-effort cleanup + logger.debug( + f"MPI session exit-join release failed (ignored): {e!r}") def _handle_worker_death(self, error: BaseException) -> None: """Event-driven worker-death handler. @@ -395,34 +429,91 @@ def _error_monitor_loop(self) -> None: self._shutdown_event.wait(timeout=5.0) def _setup_queues(self) -> WorkerCommIpcAddrs: - - self.request_queue = IpcQueue(is_server=True, - name="proxy_request_queue") + frontend_result_addrs = None + if self._num_frontends > 1: + # The rank0 worker BINDS the request ingress (PULL) so every + # frontend can PUSH-connect; each frontend (incl. this launcher, + # frontend 0) binds its own result lane (PULL). + ipc_dir = self._multi_frontend_ipc_dir + hmac_key = self._multi_frontend_hmac + request_addr = (multi_frontend_request_addr(ipc_dir), hmac_key) + frontend_result_addrs = [(multi_frontend_result_addr(ipc_dir, + i), hmac_key) + for i in range(self._num_frontends)] + self.request_queue = IpcQueue(request_addr, + is_server=False, + socket_type=zmq.PUSH, + name="proxy_request_queue") + self.result_queue = FusedIpcQueue(frontend_result_addrs[0], + is_server=True, + fuse_message=False, + socket_type=zmq.PULL, + name="proxy_result_queue") + else: + request_addr = None + self.request_queue = IpcQueue(is_server=True, + name="proxy_request_queue") + # TODO[chunweiy]: Unify IpcQueue and FusedIpcQueue + # Use PULL mode when enable_postprocess_parallel as there are + # multiple senders from multiple processes. + self.result_queue = FusedIpcQueue( + is_server=True, + fuse_message=False, + socket_type=zmq.PULL + if self.enable_postprocess_parallel else zmq.PAIR, + name="proxy_result_queue") self.worker_init_status_queue = IpcQueue( is_server=True, socket_type=zmq.ROUTER, name="worker_init_status_queue") - # TODO[chunweiy]: Unify IpcQueue and FusedIpcQueue - # Use PULL mode when enable_postprocess_parallel as there are - # multiple senders from multiple processes. - self.result_queue = FusedIpcQueue( - is_server=True, - fuse_message=False, - socket_type=zmq.PULL - if self.enable_postprocess_parallel else zmq.PAIR, - name="proxy_result_queue") self._resource_governor_queue = IpcQueue( is_server=True, name="proxy_resource_governor_queue" ) if self._enable_resource_governor else None # Stats and KV events are now fetched via RPC, not IPC queues. return WorkerCommIpcAddrs( - request_queue_addr=self.request_queue.address, + # A connect-mode queue has no bound .address; use the preset one. + request_queue_addr=request_addr + if request_addr is not None else self.request_queue.address, worker_init_status_queue_addr=self.worker_init_status_queue.address, result_queue_addr=self.result_queue.address, resource_governor_queue_addr=self._resource_governor_queue.address if self._resource_governor_queue is not None else None, + frontend_result_queue_addrs=frontend_result_addrs, ) + def multi_frontend_attach_info(self) -> Optional[dict]: + """The attach payload consumed by attached serving frontends. + + See GenerationExecutorFrontendProxy and commands/serve.py. Returns + None unless multi-frontend mode is active. + """ + if self._num_frontends <= 1: + return None + ipc_dir = self._multi_frontend_ipc_dir + hmac_key = self._multi_frontend_hmac + return { + "mode": + "classic", + "request_addr": + multi_frontend_request_addr(ipc_dir), + "result_addrs": [ + multi_frontend_result_addr(ipc_dir, i) + for i in range(self._num_frontends) + ], + "hmac_key": + hmac_key.hex(), + # Attached frontends must apply the same collective_rpc guard + # as the launcher (see _check_collective_rpc_guard). + "model_world_size": + self.model_world_size, + # Stats / KV events / disagg params RPC endpoint on the rank0 + # worker (ROUTER socket, natively multi-client). + "rpc_addr": + self.rpc_addr, + "rpc_hmac_key": + self.hmac_key.hex(), + } + @property def resource_governor_queue(self): return self._resource_governor_queue @@ -573,7 +664,18 @@ def mpi_done_callback(future: concurrent.futures.Future): raise RuntimeError( "Executor worker returned error") from ready_signal - if isinstance(self.mpi_session, MpiPoolSession) and len(status) == 3: + self._register_worker_processes(status) + + def _register_worker_processes(self, status: tuple) -> None: + """Register identities returned by locally spawned MPI workers. + + Test session reuse replaces this module's ``MpiPoolSession`` class + reference with a factory, so identify pool-backed sessions by excluding + the external communication session types. + """ + if not isinstance( + self.mpi_session, + (MpiCommSession, RemoteMpiCommSessionClient)) and len(status) == 3: worker_process_identities: List[WorkerProcessIdentity] = status[2] self._worker_process_monitor.register(worker_process_identities) @@ -619,8 +721,28 @@ def pre_shutdown(self): if not self.mpi_futures or any(not f.done() for f in self.mpi_futures): self.request_queue.put_noblock(None, retry=4) + def _get_next_client_id(self) -> int: + client_id = super()._get_next_client_id() + if self._num_frontends > 1: + # Lane 0 follows the same namespace rule as attached frontends + # so a long-lived counter can never bleed into the frontend-id + # bits (a no-op re-encode until the counter wraps). + client_id = namespace_client_id(0, client_id) + return client_id + + def _cleanup_multi_frontend_ipc_dir(self): + """Remove the launcher-owned multi-frontend ipc directory. + + Unlinking ipc socket paths does not disturb established zmq + connections; it only prevents new connects. + """ + if self._multi_frontend_ipc_dir is not None: + shutil.rmtree(self._multi_frontend_ipc_dir, ignore_errors=True) + self._multi_frontend_ipc_dir = None + def shutdown(self): if not self.workers_started: + self._cleanup_multi_frontend_ipc_dir() return if not self.doing_shutdown: @@ -628,7 +750,15 @@ def shutdown(self): logger_debug('Proxy.shutdown...\n', "yellow") + # An abruptly-killed worker world (MPI_Abort, SIGKILL, OOM) never + # completes its mpi4py futures: give them one short collective grace + # instead of blocking on each, and skip the ones still pending. + if self._engine_dead: + concurrent.futures.wait(self.mpi_futures, timeout=5.0) + for f in self.mpi_futures: + if self._engine_dead and not f.done(): + continue try: f.result() except: @@ -645,7 +775,11 @@ def shutdown(self): if self.dispatch_result_thread is not None and self.dispatch_result_thread.is_alive( ): self.dispatch_result_thread.stop() - self.dispatch_result_thread.join() + # With the engine dead, the shutdown sentinel will never arrive + # and the dispatcher may be blocked in a ZMQ recv forever: bound + # the join and leak the daemon thread. + self.dispatch_result_thread.join( + timeout=5.0 if self._engine_dead else None) # step3: finish all remaining work @@ -660,10 +794,15 @@ def shutdown(self): self.result_queue.close() if self._resource_governor_queue is not None: self._resource_governor_queue.close() + self._cleanup_multi_frontend_ipc_dir() self.workers_started = False if self._owns_mpi_session: - self.mpi_session.shutdown() + if self._engine_dead: + # Anything joining a dead worker world blocks forever. + self.mpi_session.abandon() + else: + self.mpi_session.shutdown() # Process the errors in-case error during shutting down the threads self._handle_background_error() @@ -809,6 +948,16 @@ def get_disaggregated_params(self) -> dict: logger.warning(f"Error fetching disaggregated params via RPC: {e}") return {} + def get_data_transceiver_state(self) -> bytes: + """Get serialized DataTransceiverState from worker runtime via RPC.""" + if self.rpc_client is None: + return b"" + try: + return self.rpc_client.get_data_transceiver_state().remote() + except RPCError as e: + logger.error(f"Error fetching data transceiver state via RPC: {e}") + raise + def aget_stats(self, timeout: float) -> IterationResult: """Get iteration statistics from the runtime via RPC (async). @@ -900,3 +1049,126 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self.shutdown() return False # propagate the exception + + +class GenerationExecutorFrontendProxy(GenerationExecutorProxy): + """An attached serving frontend for the classic IPC executor path. + + Used for multi-frontend serving (num_serve_frontends > 1). + PUSH-connects to the request ingress bound by the rank0 worker and binds + its own per-frontend result lane (PULL); the worker routes responses to + this lane by the frontend id embedded in the top bits of client_id (see + utils.namespace_client_id). It never owns the engine: no MPI + session, no worker launch, and shutdown never emits the worker's None + shutdown sentinel -- that right is the launcher frontend's alone. + """ + + def __init__( + self, + attach_info: dict, + *, + frontend_id: int, + postproc_worker_config: Optional[PostprocWorkerConfig] = None, + is_llm_executor: Optional[bool] = None, + ) -> None: + num_lanes = len(attach_info["result_addrs"]) + if not 0 < frontend_id < num_lanes: + raise ValueError( + f"frontend_id {frontend_id} out of range: attached frontends " + f"use ids 1..{num_lanes - 1} (id 0 is the launcher)") + postproc_worker_config = postproc_worker_config or PostprocWorkerConfig( + ) + # Deliberately skip GenerationExecutorProxy.__init__: it creates an + # MPI session, launches workers, and registers the pre_shutdown + # atexit hook that emits the engine shutdown sentinel. + GenerationExecutor.__init__( + self, + num_postprocess_workers=postproc_worker_config. + num_postprocess_workers, + postprocess_tokenizer_dir=postproc_worker_config. + postprocess_tokenizer_dir, + is_llm_executor=is_llm_executor) + + # State consumed by methods inherited from GenerationExecutorProxy + # (submit / check_health / collective_rpc). The engine lives with + # the launcher: there are no local MPI workers, so the monitor + # stays empty and worker death reaches this frontend through its + # result lane / error queue instead. + self._engine_dead = False + self.model_world_size = attach_info.get("model_world_size", 1) + self._worker_process_monitor = WorkerProcessMonitor() + + self._frontend_id = frontend_id + self._num_frontends = num_lanes + self._results: Dict[int, GenerationResult] = {} + self.garbage_collection_gen0_threshold = None + self.workers_started = False + self.dispatch_result_thread: Optional[ManagedThread] = None + # Must be None: OpenAIServer reads the resource_governor_queue + # property at init; the governor lives with the launcher only. + self._resource_governor_queue = None + + hmac_key = bytes.fromhex(attach_info["hmac_key"]) + self.request_queue = IpcQueue( + (attach_info["request_addr"], hmac_key), + is_server=False, + socket_type=zmq.PUSH, + name=f"frontend_{frontend_id}_request_queue") + self.result_queue = FusedIpcQueue( + (attach_info["result_addrs"][frontend_id], hmac_key), + is_server=True, + fuse_message=False, + socket_type=zmq.PULL, + name=f"frontend_{frontend_id}_result_queue") + + # Stats / KV events / disagg params share the rank0 worker's stats + # RPC server with the launcher (ROUTER socket, multi-client). + self.rpc_client: Optional[RPCClient] = None + if attach_info.get("rpc_addr"): + self.rpc_client = RPCClient(attach_info["rpc_addr"], + hmac_key=bytes.fromhex( + attach_info["rpc_hmac_key"])) + + def _get_next_client_id(self) -> int: + # Embed the frontend id in the top bits so the worker routes the + # responses back to this frontend's result lane. + return namespace_client_id(self._frontend_id, + super()._get_next_client_id()) + + def check_health(self) -> bool: + """Health contract of an attached frontend. + + An attached frontend owns no workers, so there is no process or + MPI-future liveness to poll: it is healthy while no fatal error + has been recorded and shutdown has not begun. Engine death + reaches it through the per-lane result socket / dispatch-thread + error path, which records the fatal error checked here. + """ + if self.doing_shutdown or self._fatal_error is not None: + return False + + if self._drain_error_queue(): + return self._fatal_error is None and not self.doing_shutdown + + return True + + def pre_shutdown(self): + if self.doing_shutdown: + return + self.doing_shutdown = True + # Abort this frontend's in-flight requests so the engine frees their + # slots. Never send the None engine-shutdown sentinel: the launcher + # frontend owns the engine lifecycle (see the class docstring). + self._abort_all_requests() + + def shutdown(self): + self.pre_shutdown() + if self.rpc_client is not None: + self.rpc_client.close() + self.rpc_client = None + self._worker_process_monitor.close() + # The dispatch thread blocks on result_queue.get(); it is a daemon + # ManagedThread that exits with the process or on the worker's + # per-lane None sentinel at engine teardown. Closing its socket from + # another thread is not ZMQ-safe, so leave the queues to process + # teardown. diff --git a/tensorrt_llm/executor/ray_executor.py b/tensorrt_llm/executor/ray_executor.py index 0eb62678d85e..accd9efc1632 100644 --- a/tensorrt_llm/executor/ray_executor.py +++ b/tensorrt_llm/executor/ray_executor.py @@ -1,5 +1,6 @@ import asyncio import os +import time from typing import Any, Dict, List, Optional, Tuple try: @@ -355,6 +356,17 @@ def shutdown(self): except Exception as e: logger.warning(f"Error shutting down: {e}") + # The engines are already stopped by the shutdown RPC above, so + # kill the actor processes explicitly instead of relying on + # handle garbage collection. ray.kill() only *initiates* an + # asynchronous kill; _wait_for_cluster_resource_release() below + # blocks until Ray has reclaimed the workers' resources. + for worker in self.workers: + try: + ray.kill(worker, no_restart=True) + except Exception as e: + logger.warning(f"Error killing worker: {e}") + if hasattr(self, 'rpc_client') and self.rpc_client is not None: try: self.rpc_client.close() @@ -371,10 +383,53 @@ def shutdown(self): self.placement_group = None self.bundle_indices = None + # ray.kill() and remove_placement_group() above are asynchronous. + # Block until Ray has reclaimed the workers' resources so their GPU + # cleanup has completed before shutdown() returns. + self._wait_for_cluster_resource_release(timeout=30.0) + if self.has_start_local_cluser and ray.is_initialized(): logger.debug("Shutting down Ray cluster") ray.shutdown() + def _wait_for_cluster_resource_release(self, timeout: float = 30.0) -> None: + """Block until Ray returns the workers' resources to the cluster. + + ray.kill() and remove_placement_group() only initiate an + asynchronous teardown; Ray reclaims an actor's logical resources + after the raylet has reaped the worker process, by which point the + CUDA driver has already destroyed its context (GPU memory and IPC + mappings). Waiting here therefore guarantees that a subsequent LLM + instance will not race against the dying workers, which can + otherwise fail spuriously (e.g. cudaErrorMapBufferObjectFailed when + opening CUDA IPC handles). Full availability can only be expected on + a cluster dedicated to this executor, so the wait is skipped when + attached to an external cluster. Best-effort: logs a warning on + timeout instead of raising, since shutdown must not fail. + """ + if not self.has_start_local_cluser or not ray.is_initialized(): + return + deadline = time.monotonic() + timeout + busy = {} + while time.monotonic() < deadline: + try: + cluster = ray.cluster_resources() + available = ray.available_resources() + except Exception as e: + logger.debug(f"Could not query Ray resources: {e}") + return + busy = { + key: cluster[key] - available.get(key, 0.0) + for key in ("GPU", "CPU") if key in cluster and cluster[key] - + available.get(key, 0.0) > 1e-6 + } + if not busy: + return + time.sleep(0.1) + logger.warning( + f"Timed out after {timeout}s waiting for Ray to reclaim cluster " + f"resources; still in use: {busy}.") + def _get_worker_ready_futures(self): return [worker.__ray_ready__.remote() for worker in self.workers] diff --git a/tensorrt_llm/executor/result.py b/tensorrt_llm/executor/result.py index a3014da7072f..a6bbb60e8d20 100644 --- a/tensorrt_llm/executor/result.py +++ b/tensorrt_llm/executor/result.py @@ -295,10 +295,21 @@ def _handle_sequence(self, output = self._outputs[seq_idx] output.disaggregated_params = self.disaggregated_params output._last_token_ids_len = len(output.token_ids) + output._last_logprobs_len = len(output.logprobs) + decoder_output_prefix = () + if (self.sampling_params.exclude_input_from_output + or getattr(self, "_streaming", False)): + decoder_output_prefix = \ + self.sampling_params._decoder_output_token_prefix if self.sampling_params.use_beam_search: # Beam search enforces returning all generated tokens - output.token_ids = response_tensors.output_token_ids[src_idx] + output.token_ids = [ + *decoder_output_prefix, + *response_tensors.output_token_ids[src_idx], + ] else: + if decoder_output_prefix and not output.token_ids: + output.token_ids.extend(decoder_output_prefix) output.token_ids.extend(response_tensors.output_token_ids[src_idx]) if response_tensors.cum_log_probs is not None: @@ -310,16 +321,15 @@ def _handle_sequence(self, # generation logprobs handling (provenance varies by backend) if logprobs_result and logprobs_result.generation is not None: # TRT backend # update logprobs from ResponseWrapper (TRT top logprobs WAR) - output._last_logprobs_len = len(output.logprobs) output.logprobs += logprobs_result.generation elif response_tensors.log_probs is not None: # PyTorch backend # handle logprobs directly from response tensors given by sampler - output._last_logprobs_len = len(output.logprobs) - # In streaming mode, since out-of-order responses are not possible, - # each streamed response_tensors.log_probs[src_idx] - # contains a streamwise monotonically growing list of logprobs. - # so we need to accumulate only the new ones unique to that particular streamed response - if self.use_trtllm_sampler: + if decoder_output_prefix and self.sampling_params.use_beam_search: + output.logprobs = [ + *self._get_decoder_output_prefix_logprobs(), + *response_tensors.log_probs[src_idx], + ] + elif self.use_trtllm_sampler: assert output._last_logprobs_len <= len( response_tensors.log_probs[src_idx] ), (f"_last_logprobs_len ({output._last_logprobs_len}) > log_probs length (" @@ -327,6 +337,9 @@ def _handle_sequence(self, output.logprobs += response_tensors.log_probs[src_idx][ output._last_logprobs_len:] else: + if decoder_output_prefix and not output.logprobs: + output.logprobs.extend( + self._get_decoder_output_prefix_logprobs()) output.logprobs += response_tensors.log_probs[src_idx] # overcome some WAR in the cpp executor @@ -427,6 +440,13 @@ def _handle_sequence(self, if self._done: self.do_tracing(output, req_perf_metrics_dict) + def _get_decoder_output_prefix_logprobs( + self) -> TokenLogprobs | SimpleTokenLogprobs: + prefix = self.sampling_params._decoder_output_token_prefix + if self.sampling_params.logprobs_simple_format: + return [0.0] * len(prefix) + return [{token_id: Logprob(logprob=0.0, rank=1)} for token_id in prefix] + @print_traceback_on_error @nvtx_range_debug("handle_response", color="red", @@ -995,7 +1015,11 @@ def _handle_ray_response(self, response: Any): return response def _result_step(self, timeout: Optional[float] = None): - response = self.queue.get() + # Honor `timeout`: a bounded `queue.get()` lets the caller regain control if the executor + # worker dies silently without pushing a terminal response, instead of blocking potentially + # indefinitely. + # Raises `queue.Empty` on timeout; `result()` turns that into a `TimeoutError`. + response = self.queue.get(timeout=timeout) # Fast-fail: when a worker dies, the proxy enqueues EngineDeadError onto # every pending result so this get() unblocks instead of hanging forever # on a queue whose producer is gone. Record it as the sticky terminal @@ -1022,15 +1046,29 @@ def result(self, timeout: Optional[float] = None) -> "GenerationResult": """Wait for the completion of the request, and return the result. Args: - timeout (float, optional): Timeout. Defaults to None. + timeout (float, optional): The maximum number of seconds to wait for the request to + complete. `None` (default) waits indefinitely. + The timeout is a total budget across all streaming steps, not per-step. Returns: tensorrt_llm.executor.result.GenerationResult: generation result. + + Raises: + TimeoutError: If the request does not complete within `timeout` seconds. Bounding the + wait prevents a silently-dead executor worker from hanging the caller forever. """ if self._terminal_error is not None: raise self._terminal_error + deadline = None if timeout is None else time.monotonic() + timeout while not self._done: - self._result_step(timeout) + remaining = (None if deadline is None else max( + 0.0, deadline - time.monotonic())) + try: + self._result_step(remaining) + except Empty: + raise TimeoutError( + f"Request {self.request_id} did not complete within " + f"{timeout} seconds.") from None return self async def aresult(self) -> "GenerationResult": diff --git a/tensorrt_llm/executor/rpc_proxy.py b/tensorrt_llm/executor/rpc_proxy.py index 680b8d5c84e4..0763e09ad032 100644 --- a/tensorrt_llm/executor/rpc_proxy.py +++ b/tensorrt_llm/executor/rpc_proxy.py @@ -254,6 +254,10 @@ def collective_rpc( return [remote_call.remote_future()] return [remote_call.remote()] + def get_data_transceiver_state(self) -> bytes: + """Get serialized DataTransceiverState from worker runtime via RPC.""" + return self.rpc_client.get_data_transceiver_state().remote() + def setup_engine_remote(self): return self.rpc_client.setup_engine().remote(need_response=True) diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index 3bd0106241f8..e4a9f3333f4b 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -229,6 +229,66 @@ class WorkerCommIpcAddrs(NamedTuple): worker_init_status_queue_addr: tuple[str, Optional[bytes]] result_queue_addr: tuple[str, Optional[bytes]] resource_governor_queue_addr: Optional[tuple[str, Optional[bytes]]] = None + # Multi-frontend serving: one result lane per frontend. When set, the + # rank0 worker BINDS the request queue (PULL) and routes responses to + # these lanes; result_queue_addr then aliases lane 0 (the launcher). + frontend_result_queue_addrs: Optional[list[tuple[str, + Optional[bytes]]]] = None + + +# Multi-frontend client_id namespacing: a FRONTEND_ID_BITS-wide frontend id +# sits just below the sign bit -- bit 63 stays clear so ids remain positive +# in signed-int64 contexts -- and the low bits carry the per-frontend +# request counter. A stray un-namespaced (small) id reads as frontend 0, +# the launcher. +FRONTEND_ID_BITS = 6 +# Keep llm_args.num_serve_frontends le= in sync (it cannot import this +# module; test_multi_frontend_routing pins the two together). +MAX_NUM_FRONTENDS = 1 << FRONTEND_ID_BITS +FRONTEND_ID_SHIFT = 63 - FRONTEND_ID_BITS +FRONTEND_COUNTER_MASK = (1 << FRONTEND_ID_SHIFT) - 1 + + +def get_frontend_id(client_id: Optional[int]) -> int: + """Extract the originating frontend id from a namespaced client id.""" + if not isinstance(client_id, int): + return 0 + return client_id >> FRONTEND_ID_SHIFT + + +def namespace_client_id(frontend_id: int, client_id: int) -> int: + """Embed frontend_id in the top bits of a per-frontend client id.""" + return (frontend_id << FRONTEND_ID_SHIFT) | (client_id + & FRONTEND_COUNTER_MASK) + + +def frontend_lane_index(client_id: Optional[int], num_lanes: int) -> int: + """The result-lane index for a response's originating frontend. + + None (e.g. ADP dummy requests) and out-of-range frontend ids go to + lane 0, the launcher, which silently discards them like today. + """ + frontend_id = get_frontend_id(client_id) + return frontend_id if frontend_id < num_lanes else 0 + + +def bucket_responses_by_frontend(responses: list, + num_frontends: int) -> list[list]: + """Bucket responses by frontend_lane_index of their client_id.""" + buckets = [[] for _ in range(num_frontends)] + for rsp in responses: + buckets[frontend_lane_index(rsp.client_id, num_frontends)].append(rsp) + return buckets + + +def multi_frontend_request_addr(ipc_dir: str) -> str: + """The request ingress bound by the rank0 worker; frontends PUSH-connect.""" + return f"ipc://{os.path.join(ipc_dir, 'request.sock')}" + + +def multi_frontend_result_addr(ipc_dir: str, frontend_id: int) -> str: + """The result lane bound by a frontend; worker/postproc PUSH-connect.""" + return f"ipc://{os.path.join(ipc_dir, f'result_{frontend_id}.sock')}" def is_llm_response(instance): diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py index 5958d296e1c4..f173aabc224a 100644 --- a/tensorrt_llm/executor/worker.py +++ b/tensorrt_llm/executor/worker.py @@ -153,11 +153,6 @@ def shutdown(self): def block_subordinates(self): if self.rank != 0: - if isinstance(self.engine, tllm.Executor): - self.shutdown() - raise self.WorkerExit( - "block_subordinates() should be used in a `with GenerationExecutorWorker() as ...:` block" - ) from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor if isinstance(self.engine, PyExecutor): self.engine.wait_shutdown() @@ -221,6 +216,10 @@ def _print_stacks(): postproc_worker_config = postproc_worker_config or PostprocWorkerConfig() is_leader: bool = mpi_rank() == 0 + # Multi-frontend serving: the worker binds the request ingress (PULL) + # and pushes responses to per-frontend result lanes. + multi_frontend_addrs = worker_queues.frontend_result_queue_addrs + frontend_result_queues: Optional[List[FusedIpcQueue]] = None if tracer_init_kwargs is not None and is_leader: tracer = VizTracer(**tracer_init_kwargs) tracer.register_exit() @@ -238,7 +237,9 @@ def _print_stacks(): # inherit the log level from "TLLM_LOG_LEVEL" environment variable logger.set_level(log_level) request_queue = IpcQueue(worker_queues.request_queue_addr, - is_server=False, + is_server=multi_frontend_addrs is not None, + socket_type=zmq.PULL if multi_frontend_addrs + is not None else zmq.PAIR, name="worker_request_queue") worker_init_status_queue = IpcQueue( worker_queues.worker_init_status_queue_addr, @@ -260,6 +261,16 @@ def _print_stacks(): name=f"postprocess_{i}_feedin_queue") for i in range(postproc_worker_config.num_postprocess_workers) ] + elif multi_frontend_addrs is not None: + # One PUSH lane per frontend (see base_worker._send_rsp). + frontend_result_queues = [ + FusedIpcQueue(addr, + is_server=False, + fuse_message=False, + socket_type=zmq.PUSH, + name=f"worker_result_queue_{i}") + for i, addr in enumerate(multi_frontend_addrs) + ] else: # IPC queue for sending results back to the proxy, and let the # Proxy process to handle the postprocess @@ -269,9 +280,12 @@ def _print_stacks(): name="worker_result_queue") def notify_proxy_threads_to_quit(): - # Signal the dispatcher thread in the proxy to quit + # Signal the dispatcher thread in every frontend proxy to quit if result_queue is not None: result_queue.put(None) + elif frontend_result_queues is not None: + for q in frontend_result_queues: + q.put(None) else: assert result_queues is not None for q in result_queues: @@ -281,18 +295,20 @@ def notify_proxy_threads_to_quit(): if is_leader and postproc_worker_config.enabled: logger_debug(f"initiate postprocess workers...", "yellow") - proxy_result_queue: tuple[ - str, Optional[bytes]] = worker_queues.result_queue_addr + # Each postproc worker pushes to every frontend result lane (a + # single lane in single-frontend mode). + proxy_result_addrs = (multi_frontend_addrs + if multi_frontend_addrs is not None else + [worker_queues.result_queue_addr]) assert result_queues is not None postproc_worker_pool = ProcessPoolExecutor( max_workers=postproc_worker_config.num_postprocess_workers) - assert isinstance(proxy_result_queue, tuple) for i in range(postproc_worker_config.num_postprocess_workers): fut = postproc_worker_pool.submit( postproc_worker_main, result_queues[i].address, - proxy_result_queue, + proxy_result_addrs, postproc_worker_config.postprocess_tokenizer_dir, PostprocWorker.default_record_creator, postproc_worker_config.post_processor_hook, @@ -349,6 +365,8 @@ def notify_proxy_threads_to_quit(): if is_leader: if postproc_worker_config.enabled: worker.set_postproc_queues(result_queues) + elif frontend_result_queues is not None: + worker.set_frontend_result_queues(frontend_result_queues) else: worker.set_result_queue(result_queue) diff --git a/tensorrt_llm/functional.py b/tensorrt_llm/functional.py index f4b97746f615..f39b6eac7a03 100644 --- a/tensorrt_llm/functional.py +++ b/tensorrt_llm/functional.py @@ -64,7 +64,8 @@ class PositionEmbeddingType(IntEnum): def is_rope(self) -> bool: return self in [ - self.rope_gptj, self.rope_gpt_neox, self.long_rope, self.mrope + self.rope_gptj, self.rope_gpt_neox, self.long_rope, self.mrope, + self.yarn ] def is_mrope(self) -> bool: diff --git a/tensorrt_llm/inputs/media_io.py b/tensorrt_llm/inputs/media_io.py index e1a66b9716ba..a618f3f0a540 100644 --- a/tensorrt_llm/inputs/media_io.py +++ b/tensorrt_llm/inputs/media_io.py @@ -74,8 +74,9 @@ def convert_image_mode(image: Image.Image, to_mode: str) -> Image.Image: MediaModality = Literal["image", "video", "audio"] # Output representations supported by `ImageMediaIO`: -# `"pt"` -> `torch.Tensor`, `"pil"` -> `PIL.Image.Image`. -_SUPPORTED_IMAGE_FORMATS = ("pt", "pil") +# `"pt"` -> `torch.Tensor`, `"np"` -> uint8 HWC `numpy.ndarray`, +# `"pil"` -> `PIL.Image.Image`. +_SUPPORTED_IMAGE_FORMATS = ("pt", "np", "pil") # Output representations supported by `VideoMediaIO`. See # `_load_video_by_cv2` for the per-format contract. @@ -421,12 +422,13 @@ def _load_video_by_cv2( tempfile required. Callers that have no stream-buffered backend available should spill to a tempfile themselves and pass a path. - `format` controls the per-frame return type: - `"pt"` - list[torch.Tensor], dtype=float32, shape=(C, H, W), range - [0, 1]; rescaled and permuted to CHW here. - `"np"` - list[np.ndarray], dtype=uint8, shape=(H, W, 3); returned as - decoded, leaving rescale/permute to the HF processor. - `"pil"` - list[PIL.Image], one per sampled frame. + `format` controls the return type: + `"pt"` - list[torch.Tensor], dtype=float32, shape=(C, H, W), range + [0, 1]; rescaled and permuted to CHW here. + `"np"` - np.ndarray of shape (N, H, W, 3), dtype=uint8; a single + contiguous 4D buffer that HF video processors pass through + without an extra frame-by-frame copy. + `"pil"` - list[PIL.Image], one per sampled frame. """ assert format in ("pt", "np", "pil"), "format must be one of 'pt', 'np', 'pil'" @@ -477,42 +479,66 @@ def _load_video_by_cv2( indices = np.linspace(0, frame_count - 1, num_frames_to_sample, dtype=int).tolist() - # Sequential forward scan — grab() without per-frame seek + # Defer allocating the stacked buffer until the first frame is actually + # decoded: container metadata (CAP_PROP_FRAME_WIDTH/HEIGHT) can be 0 + # or stale before the first decode for some codecs. + stacked_rgb: Optional[np.ndarray] = None + H = W = None + target_set = set(indices) max_idx = indices[-1] - raw_frames: dict[int, np.ndarray] = {} + bgr_scratch: Optional[np.ndarray] = None + valid_indices: list[int] = [] + # Log at most once per decode to avoid spamming when a whole stream is + # affected (e.g. every frame retrieves with a drifted shape). + skip_warned = False frame_idx = 0 - while frame_idx <= max_idx: - grab_succeeded = vidcap.grab() - if not grab_succeeded: - break + while frame_idx <= max_idx and vidcap.grab(): if frame_idx in target_set: - # cv2 decodes frames in BGR order; convert to RGB for downstream use - retrieve_succeeded, bgr_frame = vidcap.retrieve() - if retrieve_succeeded: - raw_frames[frame_idx] = cv2.cvtColor(bgr_frame, cv2.COLOR_BGR2RGB) + # Reuse a single BGR buffer across retrieves; cv2 replaces its + # contents in place when the argument is shape-compatible. + ok, bgr_scratch = vidcap.retrieve(bgr_scratch) + if ok: + fh, fw = bgr_scratch.shape[:2] + if stacked_rgb is None: + H, W = fh, fw + stacked_rgb = np.empty((num_frames_to_sample, H, W, 3), dtype=np.uint8) + if (fh, fw) == (H, W): + cv2.cvtColor( + bgr_scratch, + cv2.COLOR_BGR2RGB, + dst=stacked_rgb[len(valid_indices)], + ) + valid_indices.append(frame_idx) + elif not skip_warned: + logger.warning( + f"Skipping frame {frame_idx} and subsequent size-drifted frames: " + f"shape={(fh, fw)} differs from first decoded shape=({H}, {W})." + ) + skip_warned = True + elif not skip_warned: + logger.warning( + f"Skipping frame {frame_idx} and subsequent retrieve failures: retrieve returned ok=False." + ) + skip_warned = True frame_idx += 1 vidcap.release() - if not raw_frames: + if stacked_rgb is None or not valid_indices: raise ValueError("Video has no readable frames.") + stacked_rgb = stacked_rgb[: len(valid_indices)] - valid_indices = [i for i in indices if i in raw_frames] if format == "pt": - # uint8 -> float32 + /255 rescale done once on the stacked buffer - # so the dtype conversion is a single memory pass and there's one - # Python torch call instead of one per frame. - stacked_uint8 = np.stack([raw_frames[i] for i in valid_indices]) - stacked_f32 = stacked_uint8.astype(np.float32) * (1.0 / 255.0) + stacked_f32 = stacked_rgb.astype(np.float32) + stacked_f32 *= 1.0 / 255.0 tensor_nchw = torch.from_numpy(stacked_f32).permute(0, 3, 1, 2).contiguous() if device != "cpu": tensor_nchw = tensor_nchw.to(device) loaded_frames = list(torch.unbind(tensor_nchw, dim=0)) elif format == "np": - # uint8 HWC frames as-is; the HF processor rescales/permutes. - loaded_frames = [raw_frames[i] for i in valid_indices] + loaded_frames = stacked_rgb else: # "pil" - loaded_frames = [Image.fromarray(raw_frames[i]) for i in valid_indices] + loaded_frames = [Image.fromarray(frame) for frame in stacked_rgb] metadata = { "total_num_frames": frame_count, @@ -666,7 +692,7 @@ async def _run_in_executor(fn, *args, **kwargs): return await loop.run_in_executor(BaseMediaIO._executor, fn, *args) -class ImageMediaIO(BaseMediaIO[Union[Image.Image, torch.Tensor]]): +class ImageMediaIO(BaseMediaIO[Union[Image.Image, torch.Tensor, np.ndarray]]): """I/O for the image modality.""" def __init__(self, format: str = "pt", device: str = "cpu") -> None: @@ -675,20 +701,25 @@ def __init__(self, format: str = "pt", device: str = "cpu") -> None: self._format = format self._device = device - def _postprocess(self, image: Image.Image) -> Union[Image.Image, torch.Tensor]: + def _postprocess(self, image: Image.Image) -> Union[Image.Image, torch.Tensor, np.ndarray]: if self._format == "pt": from torchvision.transforms import ToTensor return ToTensor()(image).to(device=self._device) + if self._format == "np": + # uint8 HWC array; the downstream HF processor rescales/permutes. + return np.asarray(image) return image - def load_bytes(self, data: bytes) -> Union[Image.Image, torch.Tensor]: + def load_bytes(self, data: bytes) -> Union[Image.Image, torch.Tensor, np.ndarray]: return self._postprocess(_load_and_convert_image(BytesIO(data))) - def load_base64(self, media_type: str, data: str) -> Union[Image.Image, torch.Tensor]: + def load_base64( + self, media_type: str, data: str + ) -> Union[Image.Image, torch.Tensor, np.ndarray]: return self._postprocess(_load_and_convert_image(BytesIO(base64.b64decode(data)))) - def load_file(self, url: str) -> Union[Image.Image, torch.Tensor]: + def load_file(self, url: str) -> Union[Image.Image, torch.Tensor, np.ndarray]: return self._postprocess(_load_and_convert_image(Path(_normalize_file_uri(url)))) diff --git a/tensorrt_llm/inputs/multimodal.py b/tensorrt_llm/inputs/multimodal.py index 597011ed3fee..811553b99672 100644 --- a/tensorrt_llm/inputs/multimodal.py +++ b/tensorrt_llm/inputs/multimodal.py @@ -3,7 +3,7 @@ """Multimodal utilities for handling images and other media types in TensorRT-LLM.""" from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union import numpy as np import torch @@ -503,9 +503,16 @@ class MultimodalParams: multimodal_data: Optional[Dict[str, Any]] = field(default_factory=dict) multimodal_runtime: Optional[MultimodalRuntimeData] = None input_ids_start_offset: int = 0 + # Prompt-order manifest of data-backed items. Each entry is + # ``{"modality": m, "index": i, "placeholder": p}`` where ``i`` indexes + # ``multi_modal_data[m]``. Encoders that interleave items across modalities + # in prompt order read ``modality``/``index``; ``placeholder`` is carried + # for parity with the tracker manifest. + mm_item_order: Optional[List[Dict[str, Union[str, int]]]] = None # CUDA event recorded on a side stream by the MM encoder prefetch path. # When set, the consume site in `get_multimodal_embeddings` issues a - # `wait_event` on the current stream before reading cached embeddings. + # `wait_event` and records attached embeddings on the current stream before + # reading them. # Always `None` unless side-stream prefetch is enabled and a prefetch ran. encoder_event: Optional[torch.cuda.Event] = field(default=None, repr=False, @@ -814,11 +821,26 @@ def _update_hash(hasher, item: object) -> None: hasher.update(serialize_item(item)) -def apply_mm_hashes( - mm_data: Dict[str, Any], - mm_uuids: Optional[Dict[str, List[Optional[str]]]] = None, - hash_lib=default_hasher -) -> Tuple[Dict[str, List[str]], Optional[List[Optional[str]]]]: +class MMHashResult(NamedTuple): + """Parallel per-modality dicts of content hashes and their UUIDs. + + Both dicts are indexed first by modality name (`"image"`, `"video"`, + ...) then by within-modality item position, so `hashes[m][i]` and + `uuids[m][i]` describe the same item. + """ + + hashes: Dict[str, List[str]] + """Modality -> list of BLAKE3 hex digests (64 chars each).""" + + uuids: Optional[Dict[str, List[Optional[str]]]] + """Modality -> list of original UUID strings. `None` entries mark + items that fell back to content-only hashing. The whole dict is + `None` when the caller supplied no UUIDs at all.""" + + +def apply_mm_hashes(mm_data: Dict[str, Any], + mm_uuids: Optional[Dict[str, List[Optional[str]]]] = None, + hash_lib=default_hasher) -> "MMHashResult": """Apply hashing to multimodal data, one hash per multimodal item. When a UUID is provided for an item, the hash is computed from both the UUID @@ -834,9 +856,8 @@ def apply_mm_hashes( hash_lib: Hash function to use (default: blake3) Returns: - Tuple of: - - Dictionary of modality -> list of hash hex strings (64 chars each) - - Flattened list of original UUID strings (or None for content-hashed items) + `MMHashResult` with per-modality `hashes` and `uuids` dicts. See + the type's own docstring for the field-level contract. """ def _hash_item(item): @@ -868,9 +889,8 @@ def _hash_item_with_uuid(item, uuid: str): for modality, items in mm_data.items() } - # Collect UUIDs in the same order as items - all_uuids: List[Optional[str]] = [] mm_hashes: Dict[str, List[str]] = {} + mm_uuids_by_key: Dict[str, List[Optional[str]]] = {} for modality, items in mm_items.items(): modality_uuids = None @@ -884,22 +904,25 @@ def _hash_item_with_uuid(item, uuid: str): f"data items length ({len(items)}) for modality '{modality}'" ) - hashes = [] + hashes: List[str] = [] + uuids: List[Optional[str]] = [] for i, item in enumerate(items): uuid = modality_uuids[i] if modality_uuids else None if uuid is not None: # Hash UUID + content together for cache correctness hashes.append(_hash_item_with_uuid(item, uuid)) - all_uuids.append(uuid) # Store original UUID else: # Fall back to content-only hashing hashes.append(_hash_item(item)) - all_uuids.append(None) + uuids.append(uuid) # `None` when the caller didn't supply one mm_hashes[modality] = hashes + mm_uuids_by_key[modality] = uuids - # Return None for uuids if no UUIDs were provided at all - return mm_hashes, all_uuids if mm_uuids is not None else None + return MMHashResult( + hashes=mm_hashes, + uuids=mm_uuids_by_key if mm_uuids is not None else None, + ) def hexdigest_to_int32(hex_digest: str) -> List[int]: diff --git a/tensorrt_llm/inputs/multimodal_data.py b/tensorrt_llm/inputs/multimodal_data.py index 4bcd3804353f..65895f9fa435 100644 --- a/tensorrt_llm/inputs/multimodal_data.py +++ b/tensorrt_llm/inputs/multimodal_data.py @@ -147,7 +147,8 @@ class VideoData(BaseModalityData): """Data class for video loading results. Attributes: - frames: List of video frames, either as PIL Images or PyTorch tensors. + frames: Video frames as a list of PIL Images, a list of PyTorch + tensors, or a single 4D numpy array of shape (N, H, W, 3). metadata: Dictionary containing video metadata including: - total_num_frames: Total number of frames in the video - fps: Original frames per second of the video @@ -163,14 +164,14 @@ class VideoData(BaseModalityData): content without walking pixels. """ - frames: list[Image.Image] | list[torch.Tensor] + frames: list[Image.Image] | list[torch.Tensor] | np.ndarray metadata: dict[str, Any] audio: AudioData | None = None raw_bytes_hash: str | None = None def __post_init__(self) -> None: - if not self.frames: - raise ValueError("frames list cannot be empty") + if len(self.frames) == 0: + raise ValueError("frames cannot be empty") if not isinstance(self.metadata, dict): raise TypeError("metadata must be a dictionary") diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py index e15e3e590314..d4f2f96a4561 100644 --- a/tensorrt_llm/inputs/registry.py +++ b/tensorrt_llm/inputs/registry.py @@ -21,6 +21,7 @@ from typing import (Any, Callable, ClassVar, Dict, List, Optional, Protocol, Tuple, Type, TypeVar, Union) +import numpy as np import torch from PIL import Image from torch import Tensor, nn @@ -535,6 +536,9 @@ def get_num_tokens_per_image( """ if isinstance(image, torch.Tensor): image_h, image_w = int(image.shape[-2]), int(image.shape[-1]) + elif isinstance(image, np.ndarray): + # HWC uint8 from ImageMediaIO's "np" format. + image_h, image_w = int(image.shape[0]), int(image.shape[1]) else: image_h, image_w = image.height, image.width return self.get_num_multimodal_tokens([(image_h, image_w)], @@ -572,6 +576,10 @@ def get_num_tokens_per_video( if isinstance(first_frame, torch.Tensor): frame_h = int(first_frame.shape[-2]) frame_w = int(first_frame.shape[-1]) + elif isinstance(first_frame, np.ndarray): + # HWC uint8 from VideoMediaIO's "np" format. + frame_h = int(first_frame.shape[0]) + frame_w = int(first_frame.shape[1]) else: frame_h, frame_w = first_frame.height, first_frame.width @@ -937,8 +945,15 @@ def create_input_processor( logger.debug(f"Detected checkpoint_format={checkpoint_format}.") from tensorrt_llm._torch.models.checkpoints.mistral.config_loader import \ MistralConfigLoader + from tensorrt_llm._torch.models.modeling_mistral import \ + MistralNativeInputProcessor model_config = MistralConfigLoader().load(model_path_or_dir) config = model_config.pretrained_config + return MistralNativeInputProcessor(model_path_or_dir, + config, + tokenizer, + trust_remote_code=trust_remote_code, + **kwargs) else: logger.debug( f"checkpoint_format={checkpoint_format}; skipping HF config load.") @@ -1115,7 +1130,8 @@ def multimodal_hashing_process( # Extract optional UUIDs (can be None, or dict with same structure as mm_data) mm_uuids = inputs.get('multi_modal_uuids', None) - mm_hashes, mm_uuid_list = apply_mm_hashes(mm_data, mm_uuids, hash_lib) + mm_hashes, mm_uuids_by_key = apply_mm_hashes(mm_data, mm_uuids, + hash_lib) prompt_token_ids, extra_processed_inputs = input_processor( inputs, sampling_params) @@ -1147,7 +1163,17 @@ def multimodal_hashing_process( raise ValueError( "multimodal hashing could not determine multimodal token " "lengths for the provided input.") - num_mm_tokens = next(iter(num_mm_tokens_by_key.values())) + # `mm_hashes_flat`, `start_positions`, and `num_mm_tokens` must all + # index items at the same offsets — project into prompt order via the + # manifest. + mm_item_order = inputs.get("mm_item_order") + if mm_item_order: + num_mm_tokens = [ + num_mm_tokens_by_key[e["modality"]][e["index"]] + for e in mm_item_order + ] + else: + num_mm_tokens = next(iter(num_mm_tokens_by_key.values())) if len(num_mm_tokens) <= 0: raise ValueError("multimodal hashing produced an empty multimodal " "token-length list.") @@ -1193,8 +1219,30 @@ def multimodal_hashing_process( ) > 0 and mm_special_token_ids is not None: extra_processed_inputs["multimodal_data"][ "special_token_offsets"] = start_special_token_positions - # flatten the hashes from dict to a single list - mm_hashes_flat = [h for hashes in mm_hashes.values() for h in hashes] + # Same prompt-order projection as `num_mm_tokens` above — the cache + # key indexes each item's digest by its `start_positions` offset. + mm_item_order = inputs.get("mm_item_order") + if mm_item_order: + mm_hashes_flat = [ + mm_hashes[e["modality"]][e["index"]] for e in mm_item_order + ] + else: + mm_hashes_flat = [ + h for hashes in mm_hashes.values() for h in hashes + ] + # `MultimodalInput.multimodal_uuids` must index in lockstep with + # `multimodal_hashes`, so project through the same manifest. + if mm_uuids_by_key is None: + mm_uuid_list = None + elif mm_item_order: + mm_uuid_list = [ + mm_uuids_by_key[e["modality"]][e["index"]] + for e in mm_item_order + ] + else: + mm_uuid_list = [ + u for uuids in mm_uuids_by_key.values() for u in uuids + ] validate_mm_inputs(prompt_token_ids, mm_hashes_flat, start_positions, num_mm_tokens) mm_hashes_int32 = [hexdigest_to_int32(h) for h in mm_hashes_flat @@ -1211,19 +1259,22 @@ def process_prompt_maybe_hash( ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: try_multimodal_hashing = False # only used for first time use_multimodal_hashing = False # used for subsequent calls - modalities = list(set(inputs['multi_modal_data'].keys()) - ) if 'multi_modal_data' in inputs else [] - if len(modalities) > 0: - # TODO: support multimodal hashing for multiple modalities within the same request. - if len(modalities) == 1 and modalities[0] in [ - 'image', 'video', 'audio' - ]: - # only try multimodal hashing if the inputs only contain a single modality. - if input_processor.multimodal_hashing_supported is not None: - use_multimodal_hashing = input_processor.multimodal_hashing_supported - else: - # we need to try the multimodal hashing for the first time to determine if it is supported - try_multimodal_hashing = True + # Any subset of the supported modalities is eligible for hashing. + # ``None`` payloads are skipped so an empty bucket doesn't gate off + # hashing for the modalities that are actually present. + _SUPPORTED_HASHING_MODALITIES = ('image', 'video', 'audio') + modalities = [ + m for m, d in inputs.get('multi_modal_data', {}).items() + if d is not None + ] + if modalities and all(m in _SUPPORTED_HASHING_MODALITIES + for m in modalities): + if input_processor.multimodal_hashing_supported is not None: + use_multimodal_hashing = input_processor.multimodal_hashing_supported + else: + # First-time probe: attempt hashing and latch the result on + # the input processor. + try_multimodal_hashing = True if try_multimodal_hashing or use_multimodal_hashing: try: diff --git a/tensorrt_llm/inputs/utils.py b/tensorrt_llm/inputs/utils.py index 3235cadf3e3d..0ba4b9915d8c 100644 --- a/tensorrt_llm/inputs/utils.py +++ b/tensorrt_llm/inputs/utils.py @@ -372,6 +372,14 @@ def __init__( self._embeddings = defaultdict[str, list](list) self._placeholder_counts = defaultdict[str, int](int) self._placeholder_to_modality: dict[str, str] = {} + # Prompt-order manifest of data-backed items. Each entry is + # `{"modality": , "index": , "placeholder": }`: + # `modality` is the modality name, `index` is the item's position in + # `multi_modal_data[modality]`, and `placeholder` is the exact string + # the input processor splices this item's embedding into. Populated by + # `add_data` (skipping `is_embedding=True` items — the interleave + # manifest addresses raw payload only). + self._item_order: list[dict[str, Union[str, int]]] = [] self._multimodal_server_config = multimodal_server_config if multimodal_server_config is not None else MultimodalServerConfig( ) # Per-request override merged with the server default at media-load @@ -440,6 +448,12 @@ def add_data(self, self._embeddings[media_type]) + 1 placeholder = retrieve_multimodal_placeholder(self._model_type, media_type, current_count) + if not is_embedding: + self._item_order.append({ + "modality": media_type, + "index": len(self._data[media_type]), + "placeholder": placeholder, + }) (self._embeddings if is_embedding else self._data)[media_type].append(data) if placeholder: @@ -455,20 +469,47 @@ def placeholder_modalities(self) -> Dict[str, str]: """Get the mapping from placeholder string to modality name.""" return dict(self._placeholder_to_modality) + def item_order(self) -> List[Dict[str, Union[str, int]]]: + """Prompt-order manifest of data-backed items. -def add_multimodal_placeholders(model_type: str, text_prompt: str, - mm_placeholder_counts: dict[str, int]) -> str: + Each entry is `{"modality": , "index": , "placeholder": }`. + `index` is the item's position in `multi_modal_data[modality]`; + `placeholder` is the exact string the input processor will look + for when splicing this item's encoder embedding. + """ + return list(self._item_order) + + +def add_multimodal_placeholders( + model_type: str, + text_prompt: str, + mm_placeholder_counts: dict[str, int], + item_order: Optional[List[Dict[str, Union[str, int]]]] = None, +) -> str: """Add multimodal placeholders to the text prompt. - Placeholders that already exist in the text are counted and subtracted - from the requested count to avoid double-insertion (e.g. when the - client already embeds ```` in the prompt text). + Placeholders already in the text are counted and subtracted to + avoid double-insertion (e.g. when the client already embeds + `` in the prompt text). When `item_order` is supplied, + placeholders are emitted in prompt-arrival order (needed for mixed + modality); otherwise they follow `mm_placeholder_counts` iteration + order. """ + if item_order: + wanted = [e["placeholder"] for e in item_order] + else: + wanted = [ + ph for ph, n in mm_placeholder_counts.items() for _ in range(n) + ] + + remaining = {ph: text_prompt.count(ph) for ph in set(wanted)} placeholders = [] - for placeholder, count in mm_placeholder_counts.items(): - existing = text_prompt.count(placeholder) - needed = max(0, count - existing) - placeholders.extend([placeholder] * needed) + for ph in wanted: + if remaining[ph] > 0: + remaining[ph] -= 1 + else: + placeholders.append(ph) + if not placeholders: return text_prompt parts = [] @@ -948,6 +989,9 @@ def convert_to_conversation_message( input[ "multi_modal_data"], _ = mm_data_tracker.retrieve_all_sync( ) + item_order = mm_data_tracker.item_order() + if item_order: + input["mm_item_order"] = item_order inputs.append(input) return inputs diff --git a/tensorrt_llm/llmapi/__init__.py b/tensorrt_llm/llmapi/__init__.py index 019b97913d38..e0967a5cc243 100644 --- a/tensorrt_llm/llmapi/__init__.py +++ b/tensorrt_llm/llmapi/__init__.py @@ -12,18 +12,20 @@ CudaGraphConfig, DecodeCudaGraphConfig, DeepSeekSparseAttentionConfig, DeepSeekV4SparseAttentionConfig, DFlashDecodingConfig, - DraftTargetDecodingConfig, DynamicBatchConfig, - Eagle3DecodingConfig, EagleDecodingConfig, - EncodeCudaGraphConfig, ExtendedRuntimePerfKnobConfig, - KvCacheConfig, LlmArgs, LookaheadDecodingConfig, + DraftTargetDecodingConfig, DSparkDecodingConfig, + DynamicBatchConfig, Eagle3DecodingConfig, + EagleDecodingConfig, EncodeCudaGraphConfig, + ExtendedRuntimePerfKnobConfig, KvCacheConfig, LlmArgs, + LookaheadDecodingConfig, MambaStateConfig, MedusaDecodingConfig, MiniMaxM3SparseAttentionConfig, - MoeConfig, MTPDecodingConfig, NGramDecodingConfig, - PARDDecodingConfig, PrometheusMetricsConfig, - ReorderRequestPolicyConfig, RocketSparseAttentionConfig, - SADecodingConfig, SAEnhancerConfig, - SaveHiddenStatesDecodingConfig, SchedulerConfig, - SkipSoftmaxAttentionConfig, TorchCompileConfig, - TorchLlmArgs, UserProvidedDecodingConfig) + MoeConfig, MTPDecodingConfig, MultimodalConfig, + NGramDecodingConfig, PARDDecodingConfig, + PrometheusMetricsConfig, ReorderRequestPolicyConfig, + RocketSparseAttentionConfig, SADecodingConfig, + SAEnhancerConfig, SaveHiddenStatesDecodingConfig, + SchedulerConfig, SkipSoftmaxAttentionConfig, + TorchCompileConfig, TorchLlmArgs, + UserProvidedDecodingConfig) from .llm_utils import KvCacheRetentionConfig, QuantAlgo, QuantConfig from .mm_encoder import MultimodalEncoder from .mpi_session import MpiCommSession @@ -42,6 +44,7 @@ 'ConversationParams', 'DisaggScheduleStyle', 'KvCacheConfig', + 'MambaStateConfig', 'KvCacheRetentionConfig', 'CudaGraphConfig', 'DecodeCudaGraphConfig', @@ -67,6 +70,7 @@ 'NGramDecodingConfig', 'PARDDecodingConfig', 'DFlashDecodingConfig', + 'DSparkDecodingConfig', 'SADecodingConfig', 'SAEnhancerConfig', 'UserProvidedDecodingConfig', @@ -88,5 +92,5 @@ 'PrometheusMetricsConfig', 'ThinkingBudgetLogitsProcessor', 'add_thinking_budget_logits_processor', - 'DeepSeekV4SparseAttentionConfig', + 'MultimodalConfig', ] diff --git a/tensorrt_llm/llmapi/disagg_utils.py b/tensorrt_llm/llmapi/disagg_utils.py index d1851364d7c5..08cdc2d0a680 100644 --- a/tensorrt_llm/llmapi/disagg_utils.py +++ b/tensorrt_llm/llmapi/disagg_utils.py @@ -1,4 +1,5 @@ import logging +import os import threading import time import uuid @@ -91,10 +92,12 @@ class DisaggServerConfig(): otlp_config: Optional[OtlpConfig] = None max_retries: int = 1 perf_metrics_max_requests: int = 0 + return_perf_metrics: bool = False + perf_metrics_output_dir: Optional[str] = None disagg_cluster_config: Optional[DisaggClusterConfig] = None node_id: int = uuid.getnode( - ) % 1021 # Assuming only one disagg-server is running on a machine, moding mac by the largest 10-bit prime - # If this causes collisions, users can set node_id manually within range [0, 1023] in config + ) % 256 # Assuming only one disagg-server is running on a machine, modulo 256. + # If this causes collisions, users can set node_id manually within range [0, 255] in config schedule_style: Literal['context_first', 'generation_first'] = 'context_first' allow_request_chat_template: bool = False @@ -106,6 +109,14 @@ class DisaggServerConfig(): # the orchestrator relays a string instead of materializing the token-id list # on its event loop. Text-only, non-harmony deployments (see _get_ctx_request). gen_tokids_ctxbytes: bool = False + # Number of uvicorn disagg-server worker processes to fork on the public port. + # >1 means a fleet of delegating servers behind one coordinator. Replaces the + # WEB_CONCURRENCY env var (explicit config over implicit env). + num_workers: int = 1 + # URL of an already-running coordinator (e.g. "http://host:8332"). When set the + # fleet delegates to it; when absent, num_workers>1 starts an implicit in-process + # coordinator and num_workers==1 runs a single self-contained server. + disagg_coordinator_url: Optional[str] = None @dataclass @@ -180,6 +191,8 @@ def extract_disagg_cfg(hostname: str = 'localhost', port: int = 8000, max_retries: int = 1, perf_metrics_max_requests: int = 0, + return_perf_metrics: bool = False, + perf_metrics_output_dir: Optional[str] = None, context_servers: Optional[dict] = None, generation_servers: Optional[dict] = None, conditional_disagg_config: Optional[dict] = None, @@ -192,14 +205,18 @@ def extract_disagg_cfg(hostname: str = 'localhost', allow_request_chat_template: bool = False, gen_strip_message_history: bool = False, gen_tokids_ctxbytes: bool = False, + num_workers: int = 1, + disagg_coordinator_url: Optional[str] = None, **kwargs: Any) -> DisaggServerConfig: context_servers = context_servers or {} generation_servers = generation_servers or {} + inherited_args = dict(kwargs) + # If parameters are specified outside the context_severs and generation_servers sections, # make sure they match # Also inherit the values from the top-level - for key, value in kwargs.items(): + for key, value in inherited_args.items(): for server_type, servers in [("context_servers", context_servers), ("generation_servers", generation_servers) ]: @@ -230,12 +247,24 @@ def extract_disagg_cfg(hostname: str = 'localhost', otlp_config = OtlpConfig(**otlp_config) if otlp_config else None - config = DisaggServerConfig(server_configs, hostname, port, - ctx_router_config, gen_router_config, - conditional_disagg_config, otlp_config, - max_retries, perf_metrics_max_requests, - disagg_cluster_config) + config = DisaggServerConfig( + server_configs=server_configs, + hostname=hostname, + port=port, + ctx_router_config=ctx_router_config, + gen_router_config=gen_router_config, + conditional_disagg_config=conditional_disagg_config, + otlp_config=otlp_config, + max_retries=max_retries, + perf_metrics_max_requests=perf_metrics_max_requests, + return_perf_metrics=return_perf_metrics, + perf_metrics_output_dir=perf_metrics_output_dir, + disagg_cluster_config=disagg_cluster_config) if node_id is not None: + node_id_space = 1 << DISAGG_NODE_ID_BITS + if not 0 <= node_id < node_id_space: + raise ValueError( + f"node_id must be in range [0, {node_id_space}), got {node_id}") config.node_id = node_id if schedule_style: config.schedule_style = schedule_style @@ -243,6 +272,8 @@ def extract_disagg_cfg(hostname: str = 'localhost', allow_request_chat_template, "allow_request_chat_template") config.gen_strip_message_history = gen_strip_message_history config.gen_tokids_ctxbytes = gen_tokids_ctxbytes + config.num_workers = num_workers + config.disagg_coordinator_url = disagg_coordinator_url return config @@ -294,6 +325,11 @@ def extract_router_config(server_cfg: dict) -> RouterConfig: args = server_cfg.pop("router", {}) router_type = args.pop("type", "round_robin") + if router_type == "kv_cache_aware" and "model_path" not in args: + model_path = server_cfg.get("model") + if model_path is not None: + args["model_path"] = model_path + # add fields that are not specific to router extract_keys = ["max_batch_size", "max_num_tokens"] for key in extract_keys: @@ -418,42 +454,59 @@ def parse_metadata_server_config_file( return MetadataServerConfig(**config) -MIN_GLOBAL_ID = 1 << 42 +# Snowflake global disagg request id, 64-bit / positive int64 (MSB reserved 0): +# [ 0 (1) | timestamp_ms (39) | node_id (8) | process_id (6) | counter (10) ] +# The (node_id, process_id) pair identifies a fleet worker process, so co-located +# workers never emit the same id in the same millisecond. See docs/source/ +# advanced/disaggregated-service.md for the full disagg-request-id design. +DISAGG_TIMESTAMP_BITS = 39 +DISAGG_NODE_ID_BITS = 8 +DISAGG_PROCESS_ID_BITS = 6 +DISAGG_COUNTER_BITS = 10 + +# Local ids [0, MIN_GLOBAL_ID) and global disagg ids [MIN_GLOBAL_ID, 2^63) are +# disjoint by construction so they never collide. Power of two (masked in +# get_local_request_id). +MIN_GLOBAL_ID = 1 << 40 # Consider GIL being removed in the future, use a lock to protect the counter _global_disagg_request_id_lock = threading.Lock() _global_disagg_request_id_counter = 0 -def get_global_disagg_request_id(machine_id: int) -> int: - """ - a snowflake global disagg request id that doesn't guarantee monotonicity - 0: positive integer - 1-41 41 bits: timestamp_ms - 42-51 10 bits: machine_id - 52-63 12 bits: counter +def get_global_disagg_request_id(node_id: int, process_id: int = 0) -> int: + """A snowflake global disagg request id (does not guarantee monotonicity). + + Layout: 0(1) | timestamp_ms(39) | node_id(8) | process_id(6) | counter(10). + node_id identifies the node, process_id the fleet worker process on it -- the + pair makes the id unique across co-located workers without any coordination. """ global _global_disagg_request_id_lock global _global_disagg_request_id_counter - COUNTER_BITS = 12 - MACHINE_ID_BITS = 10 - COUNTER_MASK = (1 << COUNTER_BITS) - 1 + NODE_ID_SPACE = 1 << DISAGG_NODE_ID_BITS + PROCESS_ID_SPACE = 1 << DISAGG_PROCESS_ID_BITS + COUNTER_MASK = (1 << DISAGG_COUNTER_BITS) - 1 + TIMESTAMP_MASK = (1 << DISAGG_TIMESTAMP_BITS) - 1 MAX_INT64 = (1 << 63) - 1 - if machine_id not in range(0, (1 << MACHINE_ID_BITS) - 1): - raise ValueError( - f"machine_id must be in range [0, {(1 << MACHINE_ID_BITS) - 1})") + if node_id not in range(0, NODE_ID_SPACE): + raise ValueError(f"node_id must be in range [0, {NODE_ID_SPACE})") + if process_id not in range(0, PROCESS_ID_SPACE): + raise ValueError(f"process_id must be in range [0, {PROCESS_ID_SPACE})") - timestamp_ms = int(time.monotonic() * 1000) + timestamp_ms = int(time.monotonic() * 1000) & TIMESTAMP_MASK with _global_disagg_request_id_lock: counter = _global_disagg_request_id_counter & COUNTER_MASK _global_disagg_request_id_counter += 1 - # Rotate in [MIN_GLOBAL_ID, MAX_INT64) - # [0, MIN_GLOBAL_ID) is reserved for local ids - global_id = (timestamp_ms << (MACHINE_ID_BITS + COUNTER_BITS)) | ( - machine_id << COUNTER_BITS) | counter + global_id = ( + (timestamp_ms << + (DISAGG_NODE_ID_BITS + DISAGG_PROCESS_ID_BITS + DISAGG_COUNTER_BITS)) + | (node_id << (DISAGG_PROCESS_ID_BITS + DISAGG_COUNTER_BITS)) + | (process_id << DISAGG_COUNTER_BITS) + | counter) + # Rotate into [MIN_GLOBAL_ID, MAX_INT64); [0, MIN_GLOBAL_ID) is local-id space. global_id_int64 = global_id % (MAX_INT64 - MIN_GLOBAL_ID) + MIN_GLOBAL_ID return global_id_int64 @@ -461,3 +514,23 @@ def get_global_disagg_request_id(machine_id: int) -> int: def get_local_request_id(last_id: int) -> int: """ increment the last_id by 1 and mod by MIN_GLOBAL_ID """ return (last_id + 1) & (MIN_GLOBAL_ID - 1) + + +def disagg_process_id_space() -> int: + """Number of distinct process_id slots in the snowflake id (2^bits).""" + return 1 << DISAGG_PROCESS_ID_BITS + + +def worker_local_process_id() -> int: + """Return this fleet worker's process index. + + The fleet launcher sets ``TRTLLM_DISAGG_WORKER_PROCESS_ID`` to a distinct + value per process. A standalone disaggregated server defaults to 0. + """ + process_id = int(os.environ.get("TRTLLM_DISAGG_WORKER_PROCESS_ID", "0")) + process_id_space = disagg_process_id_space() + if not 0 <= process_id < process_id_space: + raise ValueError( + "TRTLLM_DISAGG_WORKER_PROCESS_ID must be between 0 and " + f"{process_id_space - 1}, got {process_id}") + return process_id diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 5d19670b0991..2b34297c8436 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -7,7 +7,8 @@ from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path -from typing import Any, List, Literal, Optional, Sequence, Tuple, Union, cast +from typing import (Any, Dict, List, Literal, Optional, Sequence, Tuple, Union, + cast) import torch import transformers @@ -121,21 +122,33 @@ class EncoderOutput: prompt: Optional[str] = None -class _BartForcedTokensLogitsProcessor(LogitsProcessor): - """Apply BART forced BOS/EOS tokens from Hugging Face generation config.""" +class _WhisperSuppressTokensLogitsProcessor(LogitsProcessor): + """Apply Whisper suppress-token lists from the HF generation config. - _DECODER_PROMPT_LEN = 1 + ``suppress_token_ids`` are masked at every generation step; + ``begin_suppress_token_ids`` only when sampling the first token after the + decoder prompt. The prompt length is captured from the first callback + (which runs before the first sampled token, when ``token_ids`` holds + exactly the prompt) so variable-length decoder prompts need no plumbing. + """ - def __init__( - self, - *, - forced_bos_token_id: Optional[int], - forced_eos_token_id: Optional[int], - max_tokens: int, - ) -> None: - self.forced_bos_token_id = forced_bos_token_id - self.forced_eos_token_id = forced_eos_token_id - self.max_tokens = max_tokens + # Cap on tracked request ids so a SamplingParams object reused across many + # generate() calls does not grow the map unboundedly. Far above any + # realistic number of in-flight requests, so pruning (oldest closed-window + # entries first) never touches an active sequence in practice. + _MAX_TRACKED_REQUESTS = 16384 + + def __init__(self, *, suppress_token_ids: List[int], + begin_suppress_token_ids: List[int]) -> None: + self.suppress_token_ids = [int(t) for t in suppress_token_ids or []] + self.begin_suppress_token_ids = [ + int(t) for t in begin_suppress_token_ids or [] + ] + # req_id -> prompt length while the begin-suppress window is open, then + # None once it closes. Don't delete the entry: a missing key would be + # re-captured at the current length, re-arming begin-suppression + # mid-sequence. + self._prompt_len_by_req: Dict[int, Optional[int]] = {} def __call__( self, @@ -145,58 +158,96 @@ def __call__( stream_ptr: Optional[int], client_id: Optional[int], ) -> None: - del req_id, client_id + del client_id if stream_ptr is None: - self._apply(token_ids, logits) + self._apply(req_id, token_ids, logits) return with torch.cuda.stream(torch.cuda.ExternalStream(stream_ptr)): - self._apply(token_ids, logits) - - def _apply(self, token_ids: List[List[int]], logits: torch.Tensor) -> None: + self._apply(req_id, token_ids, logits) + + def _apply(self, req_id: int, token_ids: List[List[int]], + logits: torch.Tensor) -> None: + if req_id not in self._prompt_len_by_req: + self._prune_closed_entries() + # First callback runs before the first sampled token, when the + # sequence holds exactly the decoder prompt. + self._prompt_len_by_req[req_id] = len(token_ids[0]) + prompt_len = self._prompt_len_by_req[req_id] for beam_idx, beam_token_ids in enumerate(token_ids): - forced_token_id = self._forced_token_id(beam_token_ids) - if forced_token_id is not None: - self._force_token(logits, beam_idx, len(token_ids), - forced_token_id) - - def _forced_token_id(self, token_ids: List[int]) -> Optional[int]: - generated_len = max(len(token_ids) - self._DECODER_PROMPT_LEN, 0) - if generated_len == 0: - return self.forced_bos_token_id - if (self.max_tokens > 0 and generated_len == self.max_tokens - 1): - return self.forced_eos_token_id - return None - - @staticmethod - def _force_token(logits: torch.Tensor, beam_idx: int, beam_count: int, - token_id: int) -> None: - if token_id < 0 or token_id >= logits.shape[-1]: - raise ValueError( - f"Forced BART token id {token_id} is outside the logits " - f"vocabulary dimension {logits.shape[-1]}") - - target = logits - if logits.dim() > 1 and logits.shape[0] == beam_count: - target = logits[beam_idx] - target[:] = float("-inf") - target[..., token_id] = 0 + target = logits + if logits.dim() > 1 and logits.shape[0] == len(token_ids): + target = logits[beam_idx] + if self.suppress_token_ids: + target[..., self.suppress_token_ids] = float("-inf") + if (prompt_len is not None and self.begin_suppress_token_ids + and len(beam_token_ids) == prompt_len): + target[..., self.begin_suppress_token_ids] = float("-inf") + if prompt_len is not None and len(token_ids[0]) > prompt_len: + # Begin-suppress window closed; keep the entry as a tombstone. + self._prompt_len_by_req[req_id] = None + + def _prune_closed_entries(self) -> None: + """Evict the oldest closed-window entries once the map is at capacity. + + Only ``None`` tombstones are eligible: evicting an entry whose window + is still open (or a tombstone of a still-active request) would let the + next callback re-capture the prompt length mid-sequence and re-arm + begin-suppression at the wrong position. + """ + if len(self._prompt_len_by_req) < self._MAX_TRACKED_REQUESTS: + return + excess = len(self._prompt_len_by_req) - self._MAX_TRACKED_REQUESTS + 1 + stale = [ + rid for rid, prompt_len in self._prompt_len_by_req.items() + if prompt_len is None + ] + for rid in stale[:excess]: + del self._prompt_len_by_req[rid] -def _contains_bart_forced_tokens_logits_processor(processor: Any) -> bool: - if isinstance(processor, _BartForcedTokensLogitsProcessor): +def _contains_whisper_suppress_tokens_logits_processor(processor: Any) -> bool: + if isinstance(processor, _WhisperSuppressTokensLogitsProcessor): return True if isinstance(processor, list): return any( - _contains_bart_forced_tokens_logits_processor(item) + _contains_whisper_suppress_tokens_logits_processor(item) for item in processor) processors = getattr(processor, "processors", None) if isinstance(processors, list): return any( - _contains_bart_forced_tokens_logits_processor(item) + _contains_whisper_suppress_tokens_logits_processor(item) for item in processors) return False +def _append_logits_processor(sampling_params: SamplingParams, + processor: LogitsProcessor) -> None: + """Attach a logits processor without clobbering user-provided ones.""" + existing = sampling_params.logits_processor + if existing is None: + sampling_params.logits_processor = processor + elif isinstance(existing, list): + existing.append(processor) + else: + sampling_params.logits_processor = [existing, processor] + + +def _multimodal_params_have_encoder_features( + multimodal_params: Optional["MultimodalParams"]) -> bool: + """True when multimodal data carries an encoder feature tensor. + + Audio encoder-decoder models (e.g. Whisper) feed the encoder a feature + tensor instead of token ids; its presence switches the prompt handling in + ``_preprocess``. Keyed on the purpose-specific ``encoder_input_features``, + not the generic HF ``input_features`` that decoder-only audio models emit. + """ + if multimodal_params is None or multimodal_params.multimodal_data is None: + return False + audio_data = multimodal_params.multimodal_data.get("audio") + return isinstance(audio_data, + dict) and "encoder_input_features" in audio_data + + TORCH_LLM_DOCSTRING = TORCH_LLMARGS_EXPLICIT_DOCSTRING + """ Attributes: @@ -313,7 +364,11 @@ def __init__(self, load_post_processor_hook(_post_processor_path) if _post_processor_path else None) - if self.args.parallel_config.is_multi_gpu: + # Attached serving frontends connect to an already-running worker: + # they must not spawn an MPI session (see GenerationExecutor.create). + is_attached_frontend = os.getenv( + "TLLM_EXECUTOR_ATTACH_INFO") is not None + if self.args.parallel_config.is_multi_gpu and not is_attached_frontend: if os.getenv("RAY_LOCAL_WORLD_SIZE") is None and get_device_count( ) < self.args.parallel_config.world_size_per_node: raise RuntimeError( @@ -400,6 +455,17 @@ def llm_id(self) -> str: return self._llm_id + @set_api_status("prototype") + def get_data_transceiver_state(self) -> bytes: + """Get the serialized DataTransceiverState for arbitrary KV cache transfer. + + Returns: + bytes: Serialized DataTransceiverState, or empty bytes if no transceiver is configured. + """ + if self._executor is None: + return b"" + return self._executor.get_data_transceiver_state() + @property @set_api_status("beta") def disaggregated_params(self) -> dict: @@ -638,6 +704,7 @@ def generate_async( if is_ctx_only: sampling_params.max_tokens = 1 + self._configure_bart_decoder_prefix(sampling_params) if isinstance(inputs, PreprocessedInputs): prompt_token_ids = inputs.prompt_token_ids @@ -650,6 +717,10 @@ def generate_async( preprocessed_encoder_input_token_ids, "inputs.encoder_input_token_ids") encoder_input_token_ids = preprocessed_encoder_input_token_ids + if (encoder_input_token_ids is not None + and self._is_encoder_decoder_model()): + prompt_token_ids = self._get_decoder_prompt_token_ids( + sampling_params) else: (prompt_token_ids, prompt, query_token_ids, multimodal_params, encoder_input_token_ids) = self._preprocess( @@ -890,12 +961,13 @@ def _preprocess( "prompt") # This is the text prompt, if present. if extra_processed_inputs is not None: query_token_ids = extra_processed_inputs.get('query_token_ids') - # Create unified MultimodalParams + # Create unified MultimodalParams. multimodal_params = MultimodalParams( multimodal_input=extra_processed_inputs.get( 'multimodal_input'), multimodal_data=extra_processed_inputs.get( - 'multimodal_data')) + 'multimodal_data'), + mm_item_order=inputs.get("mm_item_order")) # Only pass it if it has content if not multimodal_params.has_content(): multimodal_params = None @@ -926,8 +998,27 @@ def _preprocess( normalized_encoder_input_token_ids = None if self._is_encoder_decoder_model(): - normalized_encoder_input_token_ids = prompt_token_ids - prompt_token_ids = [self._get_decoder_start_token_id()] + if _multimodal_params_have_encoder_features(multimodal_params): + # Audio encoder-decoder models (e.g. Whisper): the encoder reads + # the feature tensor from multimodal data and the processor's + # token ids are already the decoder prompt — leave both as-is. + pass + elif getattr(self.input_processor, "requires_encoder_features", + False): + # Feature-driven encoder but no features: reject now, before the + # request reaches the encoder step where feeding tokens as + # encoder input would fail the whole co-scheduled batch. + raise ValueError( + "This encoder-decoder model takes its encoder input from " + "multi_modal_data (e.g. audio), not from prompt tokens; " + "token-only prompts are not supported.") + else: + # Text encoder-decoder models (BART/T5): the tokenized prompt + # feeds the encoder; the decoder starts from its start token + # plus any forced decoder prefix (e.g. BART forced BOS). + normalized_encoder_input_token_ids = prompt_token_ids + prompt_token_ids = self._get_decoder_prompt_token_ids( + sampling_params) return (prompt_token_ids, prompt, query_token_ids, multimodal_params, normalized_encoder_input_token_ids) @@ -1287,7 +1378,8 @@ def _prepare_sampling_params( ) sampling_params._setup(self.tokenizer, self._hf_model_config, self._generation_config) - self._add_bart_forced_tokens_logits_processor(sampling_params) + self._configure_bart_decoder_prefix(sampling_params) + self._add_whisper_suppress_tokens_logits_processor(sampling_params) add_thinking_budget_logits_processor( sampling_params, reasoning_parser=self.args.reasoning_parser, @@ -1313,37 +1405,63 @@ def _prepare_sampling_params( sampling_params.return_perf_metrics = sampling_params.return_perf_metrics or self.args.return_perf_metrics return sampling_params - def _add_bart_forced_tokens_logits_processor( - self, sampling_params: SamplingParams) -> None: + def _get_decoder_prompt_token_ids( + self, sampling_params: SamplingParams) -> List[int]: + return [ + self._get_decoder_start_token_id(), + *sampling_params._decoder_output_token_prefix, + ] + + def _configure_bart_decoder_prefix(self, + sampling_params: SamplingParams) -> None: + sampling_params._decoder_output_token_prefix = () + if self.args.backend != "pytorch": return - if getattr(self._hf_model_config, "model_type", None) != "bart": + if getattr(self._hf_model_config, "model_type", + None) not in ("bart", "mbart"): return if self._generation_config is None: return forced_bos_token_id = getattr(self._generation_config, "forced_bos_token_id", None) - forced_eos_token_id = getattr(self._generation_config, - "forced_eos_token_id", None) - if forced_bos_token_id is None and forced_eos_token_id is None: + if forced_bos_token_id is None: + return + + if (sampling_params.max_tokens is not None + and sampling_params.max_tokens <= 1): + raise ValueError( + "BART requires max_tokens >= 2 because its forced BOS token " + "counts against the output token limit.") + + sampling_params._decoder_output_token_prefix = (forced_bos_token_id, ) + + def _add_whisper_suppress_tokens_logits_processor( + self, sampling_params: SamplingParams) -> None: + if self.args.backend != "pytorch": + return + if getattr(self._hf_model_config, "model_type", None) != "whisper": + return + if self._generation_config is None: + return + + suppress_token_ids = getattr(self._generation_config, "suppress_tokens", + None) or [] + begin_suppress_token_ids = getattr(self._generation_config, + "begin_suppress_tokens", None) or [] + if not suppress_token_ids and not begin_suppress_token_ids: return existing = sampling_params.logits_processor - if _contains_bart_forced_tokens_logits_processor(existing): + if _contains_whisper_suppress_tokens_logits_processor(existing): return - processor = _BartForcedTokensLogitsProcessor( - forced_bos_token_id=forced_bos_token_id, - forced_eos_token_id=forced_eos_token_id, - max_tokens=sampling_params.max_tokens, + processor = _WhisperSuppressTokensLogitsProcessor( + suppress_token_ids=suppress_token_ids, + begin_suppress_token_ids=begin_suppress_token_ids, ) - if existing is None: - sampling_params.logits_processor = processor - elif isinstance(existing, list): - existing.append(processor) - else: - sampling_params.logits_processor = [existing, processor] + _append_logits_processor(sampling_params, processor) def _check_arguments(self, prompt_len: int, query_len: int, sampling_params: SamplingParams, diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 57c3de25dcfe..cbb9f1ec554e 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -34,7 +34,7 @@ from pydantic import AliasChoices, BaseModel, ConfigDict from pydantic import Field as PydanticField from pydantic import (NonNegativeFloat, NonNegativeInt, PositiveInt, - PrivateAttr, field_validator, model_validator) + PrivateAttr, StrictInt, field_validator, model_validator) from strenum import StrEnum from transformers import PreTrainedTokenizerBase @@ -58,9 +58,7 @@ CapacitySchedulerPolicy as _CapacitySchedulerPolicy, ContextChunkingPolicy as _ContextChunkingPolicy, DecodingConfig, - DecodingMode, DynamicBatchConfig as _DynamicBatchConfig, - EagleConfig as _EagleConfig, ExecutorConfig as _ExecutorConfig, ExtendedRuntimePerfKnobConfig as _ExtendedRuntimePerfKnobConfig, KvCacheConfig as _KvCacheConfig, @@ -563,7 +561,7 @@ class MultimodalConfig(StrictBaseModel): ("Maximum number of pending multimodal requests whose encoder work can be prefetched " "on a side CUDA stream ahead of admission. 0 disables side-stream prefetch. " "Incompatible with encoder_cuda_graph because graph replay uses static buffers. " - "For the time being, this is also incompatible with encoder_cache_max_bytes > 0." + "Can be combined with encoder_cache_max_bytes; the two memory limits are additive." ), status="prototype", ) @@ -576,7 +574,7 @@ class MultimodalConfig(StrictBaseModel): "Cache entries are per multimodal item, but reuse is all-or-nothing for each request: " "every item in the request must hit the cache before cached embeddings are reused. " "Only single-modality requests are cacheable for the time being. " - "For the time being, this is incompatible with encoder_side_stream_max_ahead > 0. " + "Can be combined with encoder_side_stream_max_ahead. " "NOTE: This is only valid for child implementations of the `MultimodalModelMixin`." ), status="prototype", @@ -607,14 +605,6 @@ def validate_encoder_optimization_compatibility(self) -> 'MultimodalConfig': "multimodal_config.encoder_side_stream_max_ahead > 0 are " "mutually exclusive. Disable side-stream MM prefetch or " "disable MM encoder CUDA graphs.") - # TODO(TRTLLM-14034): Make encoder side-stream read and write from the cache. - if (self.encoder_cache_max_bytes > 0 - and self.encoder_side_stream_max_ahead > 0): - raise ValueError( - "multimodal_config.encoder_cache_max_bytes > 0 and " - "multimodal_config.encoder_side_stream_max_ahead > 0 are " - "mutually exclusive. Disable side-stream MM prefetch or set " - "the MM encoder cache capacity to 0.") return self @@ -680,20 +670,18 @@ class MiniMaxM3SparseAttentionConfig(BaseSparseAttentionConfig): Drives the two-step sparse attention used by MiniMax-M3 layers 3..N: - 1. An index attention branch projects a per-head Q vector and a - **single replicated** K vector, scores main K/V cache blocks, - and selects the top-``topk`` blocks per ``(num_kv_heads, q_token)`` - pair (with ``init_blocks`` forced at the head and ``local_blocks`` - forced at the tail). + 1. An index attention branch projects a per-head Q vector and a single + replicated K vector, scores main K/V cache blocks, and selects the + top-k blocks per (num_kv_heads, q_token) pair, with init_blocks forced + at the head and local_blocks forced at the tail. 2. A sparse GQA attention runs only over the selected blocks. - The selected backend at runtime uses - :class:`tensorrt_llm._torch.attention_backend.sparse.minimax_m3.MiniMaxM3SparseAttention` - on top of a :class:`MiniMaxM3KVCacheManagerV2` that allocates a - paged side index-K cache (``[num_slots, 1, sparse_index_dim]``) - parallel to the main K/V cache. The M3 checkpoint sets - ``disable_index_value=True`` on every sparse layer so no index V - cache is allocated for the bring-up. + At runtime one of the MiniMax-M3 sparse attention backends under + tensorrt_llm._torch.attention_backend.sparse.minimax_m3 is selected. The + chosen backend runs on top of a MiniMaxM3KVCacheManagerV2 that allocates a + paged side index-K cache of shape [num_slots, 1, sparse_index_dim] parallel + to the main K/V cache. The M3 checkpoint sets disable_index_value=True on + every sparse layer, so no index V cache is allocated. """ algorithm: Literal["minimax_m3"] = "minimax_m3" @@ -730,6 +718,34 @@ class MiniMaxM3SparseAttentionConfig(BaseSparseAttentionConfig): default=True, description="If True, skip the index V branch (M3 checkpoint default).", ) + num_attention_heads: Optional[int] = Field( + default=None, + description= + "Global number of attention (query) heads. When unset, it falls back " + "to pretrained_config.num_attention_heads.", + ) + num_key_value_heads: Optional[int] = Field( + default=None, + description= + "Global number of key/value heads. When unset, it falls back to " + "pretrained_config.num_key_value_heads, then to num_attention_heads.", + ) + implementation: Literal["triton", "msa"] = Field( + default="triton", + description= + "Sparse attention implementation: 'triton' reference (default) or 'msa' " + "(fmha_sm100 kernels). The 'msa' implementation requires an SM100 GPU, " + "the fmha_sm100 package, and sparse_block_size == 128.", + status="prototype", + ) + + @model_validator(mode="after") + def _validate_msa_block_size(self): + if self.implementation == "msa" and self.sparse_block_size != 128: + raise ValueError( + "MiniMax-M3 'msa' implementation requires sparse_block_size == " + f"128, got {self.sparse_block_size}.") + return self def supports_backend(self, backend: str) -> bool: return backend == "pytorch" @@ -738,7 +754,7 @@ def get_indices_block_size(self) -> int: return self.sparse_block_size def to_sparse_params(self, **kwargs): - from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.metadata import \ + from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.common import \ MiniMaxM3SparseParams return MiniMaxM3SparseParams( @@ -750,6 +766,36 @@ def to_sparse_params(self, **kwargs): local_blocks=self.sparse_local_blocks, score_type=self.sparse_score_type, disable_index_value=self.sparse_disable_index_value, + implementation=self.implementation, + ) + + def to_sparse_metadata_params(self, **kwargs): + """Lower into MiniMaxM3SparseMetadataParams for the attention metadata. + + Head counts resolve as this config, then pretrained_config, then a + default; num_key_value_heads falls back to num_attention_heads. Setting + them on the config lets tests skip building a pretrained_config. + """ + from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.common import \ + MiniMaxM3SparseMetadataParams + + pretrained_config = kwargs.get("pretrained_config", None) + + def _value(name: str, default=None): + value = getattr(self, name) + if value is not None: + return value + if pretrained_config is not None: + return getattr(pretrained_config, name, default) + return default + + num_attention_heads = int(_value("num_attention_heads", 0)) + num_kv_heads = int(_value("num_key_value_heads", num_attention_heads)) + return MiniMaxM3SparseMetadataParams( + global_num_q_heads=num_attention_heads, + global_num_kv_heads=num_kv_heads, + num_index_heads=self.sparse_num_index_heads, + topk=self.sparse_topk_blocks, ) @@ -1014,6 +1060,7 @@ def _value(name: str, default=None): index_head_dim=_value("index_head_dim", 128), enable_indexer_skip=self.skip_indexer_for_short_seqs, enable_heuristic_topk=self.enable_heuristic_topk, + use_cute_dsl_topk=self.use_cute_dsl_topk, use_cute_dsl_paged_mqa_logits=(self.use_cute_dsl_paged_mqa_logits), q_split_threshold=self.q_split_threshold, ) @@ -1122,6 +1169,7 @@ def _value(name: str, default=None): index_head_dim=_value("index_head_dim", 128), enable_indexer_skip=self.skip_indexer_for_short_seqs, enable_heuristic_topk=self.enable_heuristic_topk, + use_cute_dsl_topk=self.use_cute_dsl_topk, use_cute_dsl_paged_mqa_logits=(self.use_cute_dsl_paged_mqa_logits), q_split_threshold=self.q_split_threshold, compress_ratios=self.compress_ratios, @@ -1326,7 +1374,8 @@ class MoeConfig(StrictBaseModel): """Configuration for MoE.""" backend: Literal[ "AUTO", "CUTLASS", "CUTEDSL", "WIDEEP", "TRTLLM", "DEEPGEMM", - "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM"] = Field( + "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", + "MEGAMOE_CUTEDSL"] = Field( default='AUTO', description="MoE backend to use. " "AUTO selects default backend based on model. It currently doesn\'t always give the best choice for all scenarios. The capabilities of auto selection will be improved in future releases." @@ -1470,6 +1519,21 @@ class AttentionDpConfig(StrictBaseModel): "routing. The oldest conversations are evicted once more than this many " "are tracked, bounding memory on long-running servers. Only used when " "kv_cache_routing_conversation_affinity is True.") + kv_cache_routing_new_conv_placement: Literal[ + "round_robin", "least_queued"] = Field( + default="round_robin", + description= + "Placement policy in conversation-affinity routing for requests " + "with no pinned rank yet (first turn of a conversation, requests " + "without a conversation_id, sticky overflow). 'round_robin' " + "(default) equalizes per-rank conversation counts. 'least_queued' " + "places them on the rank with the fewest live requests instead: " + "per-conversation load (turn rate, fan-out, prefill length) is " + "not uniform, so count-uniform round-robin can leave some ranks " + "with deep queues while others idle; steering new conversations " + "by queue depth evens that out and cuts tail TTFT. Existing " + "conversation->rank pins are unaffected. Only used when " + "kv_cache_routing_conversation_affinity is True.") @model_validator(mode='after') def validate_attention_dp_config(self) -> 'AttentionDpConfig': @@ -1615,6 +1679,32 @@ class CalibConfig(StrictBaseModel): "The maximum sequence length to initialize tokenizer for calibration.") +class AdvancedSamplingMode(StrEnum): + """Deploy-time specialization of the one-model advanced sampler. + + FULL - per-row tensor top_k/top_p (default; mixed per-request sampling). + NO_TOPK - top_k disabled, top_p honored. Skips the top_k mask kernel. + NO_TOPP - top_p disabled, top_k honored. Skips the top_p renorm kernel. + NO_TOPK_NO_TOPP - both disabled (pure temperature sampling). Skips both kernels. + """ + FULL = "full" + NO_TOPK = "no_topk" + NO_TOPP = "no_topp" + NO_TOPK_NO_TOPP = "no_topk_no_topp" + + @property + def skips_top_k(self) -> bool: + """Single source of truth: does this mode disable the top_k filter?""" + return self in (AdvancedSamplingMode.NO_TOPK, + AdvancedSamplingMode.NO_TOPK_NO_TOPP) + + @property + def skips_top_p(self) -> bool: + """Single source of truth: does this mode disable the top_p filter?""" + return self in (AdvancedSamplingMode.NO_TOPP, + AdvancedSamplingMode.NO_TOPK_NO_TOPP) + + class DecodingBaseConfig(StrictBaseModel): max_draft_len: Optional[NonNegativeInt] = Field( default=None, description="The maximum number of draft tokens.") @@ -1696,6 +1786,13 @@ class DecodingBaseConfig(StrictBaseModel): "in a future release. Non-greedy sampling is now auto-detected per " "request; this flag no longer has any effect.") + advanced_sampling_mode: AdvancedSamplingMode = Field( + default=AdvancedSamplingMode.FULL, + description= + "Deploy-time specialization of the one-model advanced sampler that skips disabled " + "filter kernels. FULL (default): per-row top_k/top_p. NO_TOPK: skip top_k. " + "NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both.") + # If set, drafting is allowed to use chain drafter. _allow_chain_drafter: bool = PrivateAttr(True) # If set, drafting uses greedy sampling, irrespective of sampling parameters. @@ -1971,8 +2068,10 @@ class EagleDecodingConfig(DecodingBaseConfig): ) dynamic_tree_max_topK: Optional[int] = Field( default=None, - description="The topK value for each layer when dynamic tree is enabled." - ) + description= + "The topK value for each layer when dynamic tree is enabled. Required " + "when use_dynamic_tree is True; ignored (with a warning) when " + "use_dynamic_tree is False.") num_eagle_layers: Optional[int] = Field( default=None, description= @@ -2048,9 +2147,17 @@ def validate_eagle_config(self) -> 'EagleDecodingConfig': # So the number of choices also represents the number of max draft nodes. self.max_total_draft_tokens = len(self.eagle_choices) + # Dynamic tree is enabled only by an explicit use_dynamic_tree=True; + # dynamic_tree_max_topK alone does not turn it on. + if not self.use_dynamic_tree and self.dynamic_tree_max_topK is not None: + logger.warning( + "dynamic_tree_max_topK is set but use_dynamic_tree is False; " + "ignoring dynamic_tree_max_topK and using the linear draft path." + ) + self.dynamic_tree_max_topK = None + # Dynamic tree logic - if self.use_dynamic_tree or self.dynamic_tree_max_topK is not None: - self.use_dynamic_tree = True + if self.use_dynamic_tree: if self.eagle_choices is not None: raise ValueError( "If use_dynamic_tree is True, eagle_choices should be None") @@ -2070,16 +2177,12 @@ def validate_eagle_config(self) -> 'EagleDecodingConfig': logger.warning( f"max_total_draft_tokens is not provided, use the default value {default_max_total_draft_tokens} (default_max_total_draft_tokens = dynamic_tree_max_topK * max_draft_len)" ) - else: - if self.max_total_draft_tokens < self.max_draft_len: - raise ValueError( - f"max_total_draft_tokens ({self.max_total_draft_tokens}) should be >= max_draft_len ({self.max_draft_len})" - ) - if self.max_total_draft_tokens > self.dynamic_tree_max_topK * self.max_draft_len: - raise ValueError( - f"max_total_draft_tokens ({self.max_total_draft_tokens}) should be <= " - f"dynamic_tree_max_topK * max_draft_len ({self.dynamic_tree_max_topK * self.max_draft_len})" - ) + elif not (self.max_draft_len <= self.max_total_draft_tokens <= + default_max_total_draft_tokens): + raise ValueError( + f"max_total_draft_tokens ({self.max_total_draft_tokens}) must be in " + f"[max_draft_len ({self.max_draft_len}), dynamic_tree_max_topK * " + f"max_draft_len ({default_max_total_draft_tokens})]") # Linear tree if self.max_total_draft_tokens is None: @@ -2425,6 +2528,22 @@ class MTPDecodingConfig(DecodingBaseConfig): "When using EAGLE-style MTP, use faster one-model implementation (drafter as submodule) vs two-model." ) + use_dynamic_tree: bool = Field( + default=False, + description= + "Enable EAGLE-style dynamic-tree drafting for one-model MTP. When True, " + "each draft step expands dynamic_tree_max_topK candidates per node and the " + "tree is verified against the target, instead of a linear chain.") + dynamic_tree_max_topK: Optional[int] = Field( + default=None, + description= + "Top-K candidates expanded per node per draft layer when use_dynamic_tree " + "is enabled. Required when use_dynamic_tree is True; ignored (with a " + "warning) when use_dynamic_tree is False.") + + # Internal max batch size for dynamic-tree worker buffers. + _max_batch_size: Optional[int] = PrivateAttr(default=None) + sa_config: Optional[SAEnhancerConfig] = Field( default=None, status="beta", @@ -2469,15 +2588,43 @@ def _remap_deprecated_num_nextn_predict_layers(cls, data): @model_validator(mode="after") def set_max_total_draft_tokens(self): - # Leave max_draft_len as None ("use the model's num_nextn_predict_layers") - # when the user doesn't set it; update_spec_config_from_model_config - # resolves it from the checkpoint before the model runs. When the user - # does set it, validate and mirror to max_total_draft_tokens (current MTP - # only supports a linear tree). + # None means update_spec_config_from_model_config resolves it from checkpoint. if self.max_draft_len is not None: if self.max_draft_len <= 0: raise ValueError("max_draft_len must be > 0 for MTP") - self.max_total_draft_tokens = self.max_draft_len + + # Dynamic tree is enabled only by an explicit use_dynamic_tree=True; + # dynamic_tree_max_topK alone does not turn it on. + if not self.use_dynamic_tree and self.dynamic_tree_max_topK is not None: + logger.warning( + "dynamic_tree_max_topK is set but use_dynamic_tree is False; " + "ignoring dynamic_tree_max_topK and using the linear draft path." + ) + self.dynamic_tree_max_topK = None + + # Dynamic tree defaults max_total_draft_tokens to topK * max_draft_len. + if self.use_dynamic_tree: + if self.max_draft_len is None: + raise ValueError( + "max_draft_len must be set when use_dynamic_tree is True") + if self.dynamic_tree_max_topK is None or self.dynamic_tree_max_topK <= 0: + raise ValueError( + "dynamic_tree_max_topK must be > 0 when use_dynamic_tree is True" + ) + default_max_total_draft_tokens = self.dynamic_tree_max_topK * self.max_draft_len + if self.max_total_draft_tokens is None: + self.max_total_draft_tokens = default_max_total_draft_tokens + logger.warning( + f"max_total_draft_tokens is not provided, use the default value {default_max_total_draft_tokens} (default_max_total_draft_tokens = dynamic_tree_max_topK * max_draft_len)" + ) + elif not (self.max_draft_len <= self.max_total_draft_tokens <= + default_max_total_draft_tokens): + raise ValueError( + f"max_total_draft_tokens ({self.max_total_draft_tokens}) must be in " + f"[max_draft_len ({self.max_draft_len}), dynamic_tree_max_topK * " + f"max_draft_len ({default_max_total_draft_tokens})]") + elif self.max_draft_len is not None: + self.max_total_draft_tokens = self.max_draft_len # linear chain return self @model_validator(mode="after") @@ -2629,6 +2776,91 @@ def spec_dec_mode(self): return TorchSpeculativeDecodingMode.DFLASH +class DSparkDecodingConfig(DecodingBaseConfig): + """Configuration for DSpark speculative decoding. + + DSpark (DeepSeek) is a target-dependent, "semi-parallel" speculative + decoding method. Like DFlash it captures hidden states from several target + layers as cross-attention context and drafts a whole block in a single + backbone forward, but it additionally refines the per-position draft logits + with a lightweight sequential head (a low-rank Markov head, optionally an RNN + head) and predicts an acceptance-confidence per position to truncate the + proposed prefix. + + Key features: + - Target-dependent: captures hidden states from ``target_layer_ids``. + - Semi-parallel: one block backbone forward + cheap sequential head refine. + - Confidence head: truncates the proposed draft length (NOT the accept rule; + acceptance stays standard target verification, preserving greedy parity). + + Reference: DeepSeek DeepSpec (https://github.com/deepseek-ai/DeepSpec). + """ + mask_token_id: Optional[int] = Field( + default=None, + description= + "Token ID used as the mask/noise token for parallel draft prediction. " + "If None, read from the draft model config (dspark_noise_token_id).") + + target_layer_ids: Optional[List[int]] = Field( + default=None, + description= + "Target model layer indices whose hidden states are captured for " + "cross-attention in the draft model. If None, read from the draft model " + "config (dspark_target_layer_ids).") + + block_size: Optional[PositiveInt] = Field( + default=None, + description= + "Number of draft positions produced per block. If None, read from the " + "draft model config (dspark_block_size). Should equal max_draft_len.") + + markov_rank: Optional[int] = Field( + default=None, + description= + "Low-rank dimension of the Markov head logit-bias. If None, read from " + "the draft model config (dspark_markov_rank). 0 disables the head.") + + markov_head_type: Optional[Literal["vanilla", "gated", "rnn"]] = Field( + default=None, + description= + "Type of the sequential refinement head used within a block. If None, " + "read from the draft model config (dspark_markov_head_type), " + "defaulting to \"vanilla\".") + + # NOTE: confidence-based dynamic drafting (the draft model's confidence head + # that truncates the proposed block) is NOT enabled in this PR. The user-facing + # ``enable_confidence_head`` / ``confidence_threshold`` knobs are intentionally + # omitted and will be added when the feature is actually wired into the + # speculative scheduling/verification path. The confidence head module and its + # internal plumbing remain as scaffolding (see DSparkConfidenceHead / + # dspark_propose). + + decoding_type: Literal["DSpark"] = Field(default="DSpark") + + @model_validator(mode="after") + def set_max_total_draft_tokens(self): + self.max_total_draft_tokens = self.max_draft_len + return self + + @property + def tokens_per_gen_step(self) -> int: + """DSpark needs K+1 tokens per gen request (K drafts + 1 bonus). + + The draft produces its own mask queries internally; passing mask + fillers through the target is pure wasted work at large batch size. + """ + return self.max_draft_len + 1 + + def supports_backend(self, backend: str) -> bool: + return backend == "pytorch" + + @functools.cached_property + def spec_dec_mode(self): + from tensorrt_llm._torch.speculative.interface import \ + SpeculativeDecodingMode as TorchSpeculativeDecodingMode + return TorchSpeculativeDecodingMode.DSPARK + + class AutoDecodingConfig(DecodingBaseConfig): """Configuration for auto speculative decoding. @@ -3295,6 +3527,7 @@ def supports_backend(self, backend: str) -> bool: SaveHiddenStatesDecodingConfig, PARDDecodingConfig, DFlashDecodingConfig, + DSparkDecodingConfig, AutoDecodingConfig, ], Field(discriminator="decoding_type"), @@ -3327,6 +3560,14 @@ class KvCacheCompressionConfig(StrictBaseModel): "compression manager is built. Concrete algorithm configs subclass this " "and set the value.") + @property + def kv_cache_compression_mode(self): + # The mode carries algorithm-level traits (``is_*`` predicates) the + # raw algorithm string does not. + from tensorrt_llm._torch.kv_cache_compression.interface import \ + KvCacheCompressionMode + return KvCacheCompressionMode.from_string(self.algorithm) + @PybindMirror.mirror_pybind_fields(_AgentTreeConfig) class AgentTreeConfig(StrictBaseModel, PybindMirror): @@ -3366,6 +3607,42 @@ class ReorderRequestPolicyConfig(StrictBaseModel): description="The arguments of the request reordering policy.") +_StrictPositiveInt = Annotated[StrictInt, PydanticField(gt=0)] +_StrictNonNegativeInt = Annotated[StrictInt, PydanticField(ge=0)] + + +class MambaStateConfig(StrictBaseModel): + """Configuration for reusable Mamba recurrent-state snapshots.""" + + periodic_snapshot_interval: NonNegativeInt = Field( + default=0, + status="prototype", + telemetry=True, + description= + "The number of tokens between periodic snapshots in the Mamba " + "prefix cache. Periodic snapshots are disabled by default; set this " + "to a positive value to enable them.") + + additional_snapshot_offsets_from_start: List[_StrictPositiveInt] = Field( + default_factory=list, + status="prototype", + telemetry=False, + description= + "Additional Mamba state snapshot offsets measured from the start " + "of each prompt. Offsets beyond the prompt length are ignored. " + "These snapshots require KV cache manager V2.") + + additional_snapshot_offsets_from_end: List[_StrictNonNegativeInt] = Field( + default_factory=list, + status="prototype", + telemetry=False, + description= + "Additional Mamba state snapshot offsets measured backward from " + "the end of each prompt. An offset of 0 selects the prompt end. " + "Offsets that do not resolve inside the prompt are ignored. These " + "snapshots require KV cache manager V2.") + + @PybindMirror.mirror_pybind_fields(_KvCacheConfig) class KvCacheConfig(StrictBaseModel, PybindMirror): """Configuration for the KV cache.""" @@ -3499,10 +3776,18 @@ class KvCacheConfig(StrictBaseModel, PybindMirror): description="The number of tokens per block.") # This is a pure python field, not a pybind field. It is only for the Pytorch backend. - mamba_state_cache_interval: PositiveInt = Field( - default=256, + mamba_state_cache_interval: Optional[NonNegativeInt] = Field( + default=None, + status="deprecated", + telemetry=False, + exclude=True, description= - "The number of tokens between cache steps in the Mamba prefix cache.") + "Deprecated alias for mamba_state_config.periodic_snapshot_interval.") + + # This is a pure python field, not a pybind field. It is only for the Pytorch backend. + mamba_state_config: MambaStateConfig = Field( + default_factory=MambaStateConfig, + description="Configuration for reusable Mamba state snapshots.") use_kv_cache_manager_v2: bool | Literal["auto"] = Field( default="auto", @@ -3531,7 +3816,7 @@ class KvCacheConfig(StrictBaseModel, PybindMirror): max_util_for_resume: float = Field( default=0.95, - ge=0, + gt=0, le=1, status="prototype", description= @@ -3564,28 +3849,37 @@ class KvCacheConfig(StrictBaseModel, PybindMirror): default=None, min_length=1, status="prototype", - description= - "Initial pool ratios for KV cache manager v2. When used by DeepSeek-V4, " - "values map to KVCacheManagerV2 pool_group_id order and must sum to 1.0. " - "When set, DeepSeek-V4 uses this directly and avg_seq_len does not take effect." - ) + description="Initial pool ratios for KV cache manager v2. Values map to " + "KVCacheManagerV2 pool_group_id order and must sum to 1.0. Hybrid Mamba " + "models and DeepSeek-V4 use this directly, so avg_seq_len does not take " + "effect when this is set.") # This is a pure python field, not a pybind field. It is only for the Pytorch backend. avg_seq_len: Optional[PositiveInt] = Field( default=None, status="prototype", description= - "Average sequence length used by DeepSeek-V4 to build the KV cache manager v2 " - "typical step. If unset, max_seq_len is used. This does not take effect when " - "pool_ratio is set.") + "Average total sequence length of the serving workload, used to build the " + "KV cache manager v2 typical step for hybrid Mamba models and DeepSeek-V4. " + "Hybrid Mamba models warn and fall back to half of max_seq_len when this is " + "unset. This does not take effect when pool_ratio is set.") # This is a pure python field, not a pybind field. It is only for the Pytorch backend. - block_reuse_policy: Literal["all_reusable", "per_request"] = Field( - default="all_reusable", - status="prototype", - description="KV cache manager v2 block reuse policy. " - "With SWA scratch reuse and 'all_reusable', only non-scratch " - "blocks are saved for reuse.") + block_reuse_policy: Literal[ + "all_reusable", "per_request", "per_conversation"] = Field( + default="all_reusable", + status="prototype", + description="KV cache manager v2 block reuse policy. " + "'all_reusable' commits reusable blocks after every context chunk; " + "'per_request' commits them only after the final context chunk; " + "'per_conversation' uses 'per_request' commits and drops the previous " + "turn's committed SWA-window blocks and Mamba stable-boundary state " + "after the current turn's final context chunk. Periodic Mamba state " + "snapshots are disabled with 'per_conversation'. All reusable blocks " + "remain subject to normal cache eviction. " + "Requests without conversation params use 'per_request' behavior. When " + "'all_reusable' and SWA scratch reuse are both enabled, only non-scratch " + "blocks are committed for reuse.") def _to_pybind(self): config = _KvCacheConfig( @@ -3646,6 +3940,43 @@ def validate_max_gpu_total_bytes(cls, v: int): "kv_cache_config.max_gpu_total_bytes must be non-negative") return v + @model_validator(mode='after') + def migrate_legacy_mamba_interval(self) -> 'KvCacheConfig': + """Copy the deprecated Mamba interval into its nested replacement.""" + if self.mamba_state_cache_interval is None: + return self + if ("periodic_snapshot_interval" + in self.mamba_state_config.model_fields_set): + raise ValueError("Cannot set both " + "'kv_cache_config.mamba_state_cache_interval' and " + "'kv_cache_config.mamba_state_config." + "periodic_snapshot_interval'.") + logger.warning( + "'kv_cache_config.mamba_state_cache_interval' is deprecated; use " + "'kv_cache_config.mamba_state_config." + "periodic_snapshot_interval' instead.") + self.mamba_state_config = self.mamba_state_config.model_copy( + update={ + "periodic_snapshot_interval": self.mamba_state_cache_interval + }) + return self + + @model_validator(mode='after') + def disable_periodic_mamba_snapshots_for_conversations( + self) -> 'KvCacheConfig': + """Use only explicit stable boundaries for conversation reuse.""" + if (self.block_reuse_policy == "per_conversation" + and self.mamba_state_config.periodic_snapshot_interval != 0): + interval = self.mamba_state_config.periodic_snapshot_interval + logger.warning( + f"'kv_cache_config.mamba_state_config.periodic_snapshot_interval={interval}' " + "is ignored because " + "'kv_cache_config.block_reuse_policy=per_conversation' disables " + "periodic Mamba snapshots; setting it to 0.") + self.mamba_state_config = self.mamba_state_config.model_copy( + update={"periodic_snapshot_interval": 0}) + return self + @model_validator(mode='after') def validate_disk_cache_config(self): if self.disk_cache_size is not None and self.disk_cache_size > 0: @@ -3659,6 +3990,18 @@ def validate_disk_cache_config(self): ) return self + @model_validator(mode='after') + def validate_mamba_snapshot_offsets(self) -> 'KvCacheConfig': + state_config = self.mamba_state_config + has_additional_snapshots = bool( + state_config.additional_snapshot_offsets_from_start + or state_config.additional_snapshot_offsets_from_end) + if (has_additional_snapshots and self.use_kv_cache_manager_v2 is False): + raise ValueError( + "kv_cache_config.mamba_state_config additional snapshot " + "offsets require kv_cache_config.use_kv_cache_manager_v2=True.") + return self + @field_validator('max_attention_window') @classmethod def validate_max_attention_window(cls, v: Optional[List[int]]): @@ -3680,14 +4023,6 @@ def validate_max_attention_window(cls, v: Optional[List[int]]): ) return v - @field_validator('max_util_for_resume') - @classmethod - def validate_max_util_for_resume(cls, v: float): - if not 0 <= v <= 1: - raise ValueError( - "kv_cache_config.max_util_for_resume must be between 0 and 1") - return v - @field_validator('pool_ratio') @classmethod def validate_pool_ratio(cls, v: Optional[List[float]]): @@ -4121,6 +4456,19 @@ class BaseLlmArgs(StrictBaseModel): description="The path to the tokenizer directory for postprocessing.", status="prototype") + num_serve_frontends: int = Field( + default=1, + ge=1, + # = executor.utils.MAX_NUM_FRONTENDS (cannot be imported here); + # test_multi_frontend_routing pins the two together. + le=64, + description= + "The number of HTTP frontend processes serving one executor. Used by " + "trtllm-serve: values > 1 run additional attached frontend processes " + "that share the serving port via SO_REUSEPORT (classic IPC executor " + "path only).", + status="prototype") + reasoning_parser: Optional[str] = Field( default=None, description="The parser to separate reasoning content from output.", @@ -4163,15 +4511,27 @@ class BaseLlmArgs(StrictBaseModel): status="deprecated", telemetry=TelemetryField.categorical('pytorch', '_autodeploy')) - return_perf_metrics: bool = Field(default=False, - description="Return perf metrics.", - status="prototype") + return_perf_metrics: bool = Field( + default=False, + description= + "Allow serving responses to include per-request performance metrics when " + "the request sets X-TRTLLM-return-metrics: 1.", + status="prototype") + + perf_metrics_output_dir: Optional[str] = Field( + default=None, + description="Directory for per-process performance metrics JSONL " + "files. Setting this enables collection even when " + "return_perf_metrics is false.", + status="prototype", + telemetry=False) perf_metrics_max_requests: NonNegativeInt = Field( default=0, description= - "The maximum number of requests for perf metrics. Must also set return_perf_metrics to true to get perf metrics.", - status="prototype") + "Deprecated compatibility field. Completed per-request metrics are no " + "longer retained in memory.", + status="deprecated") prometheus_metrics_config: Optional[PrometheusMetricsConfig] = Field( default=None, @@ -4225,6 +4585,10 @@ def speculative_model(self) -> Optional[Union[str, Path]]: def from_yaml(cls, yaml_path: Union[str, Path]): with open(yaml_path, "r") as f: config_dict = yaml.safe_load(f) + if config_dict is None: + config_dict = {} + elif not isinstance(config_dict, dict): + raise ValueError("Configuration file root must be a mapping.") return cls(**config_dict) @field_validator("dtype") @@ -4832,7 +5196,9 @@ def validate_encoder_runtime_sizes(cls, v: Optional[int]) -> Optional[int]: checkpoint_format: Optional[str] = Field( default=None, description= - "The format of the provided checkpoint. You may use a custom checkpoint format by subclassing " + "The registered checkpoint loader format to use. `MX` selects ModelExpress as an opportunistic P2P " + "loading path and falls back to loading the provided Hugging Face checkpoint; it does not require " + "converting that checkpoint to an MX-specific format. You may use a custom checkpoint format by subclassing " "`BaseCheckpointLoader` and registering it with `register_checkpoint_loader`.\n" "If neither checkpoint_format nor checkpoint_loader are provided, checkpoint_format will be set to HF " "and the default HfCheckpointLoader will be used.\n" @@ -5105,6 +5471,7 @@ def validate_speculative_config(self): or spec_mode == TorchSpeculativeDecodingMode.MTP or spec_mode.is_mtp_eagle_one_model() or spec_mode.is_pard() or spec_mode.is_dflash() + or spec_mode.is_dspark() or spec_mode.is_draft_target_one_model()) # Combinations that break the proposal-distribution invariant. @@ -5114,13 +5481,18 @@ def validate_speculative_config(self): is_new_rejection_method = ( spec_mode == TorchSpeculativeDecodingMode.MTP or spec_mode.is_pard() or spec_mode.is_dflash() + or spec_mode.is_dspark() or spec_mode.is_draft_target_one_model()) # Plain tensor parallelism is supported (the draft path # all-gathers vocab-sharded draft logits before rejection, see # SpecWorkerBase.maybe_gather_sharded_draft_logits). - # attention-DP and context parallelism remain gated. - rs_parallel_active = (self.context_parallel_size > 1 - or self.enable_attention_dp) + # Attention DP is supported: each rank holds full-vocab draft + # logits for its own requests (the LM-head-TP fast path is + # bypassed for advanced sampling, and is_all_greedy_sample is + # group-synchronized so the LM-head-TP group's collectives stay + # uniform -- see SpecMetadata.group_all_greedy_sample). Only + # context parallelism remains gated. + rs_parallel_active = self.context_parallel_size > 1 rs_guided_active = self.guided_decoding_backend is not None rs_sa_active = getattr(self.speculative_config, "sa_config", None) is not None @@ -5172,10 +5544,9 @@ def validate_speculative_config(self): "relaxed-thinking acceptance is enabled") if rs_parallel_active: reasons.append( - "tensor/context parallelism or attention-DP " - "is active (the draft path resolves only " - "the global argmax, not full distributions)" - ) + "context parallelism is active (the draft " + "path resolves only the global argmax, " + "not full distributions)") if rs_guided_active: reasons.append("guided decoding is enabled") raise ValueError( @@ -5231,6 +5602,92 @@ def validate_speculative_config(self): if mask_id is not None: self.speculative_config.mask_token_id = mask_id + if isinstance(self.speculative_config, DSparkDecodingConfig): + spec_cfg = self.speculative_config + if not spec_cfg.max_draft_len: + raise ValueError("DSpark max_draft_len must be > 0; got " + f"{spec_cfg.max_draft_len}") + # The DSpark draft weights live in the ``mtp.*`` namespace of a + # local checkpoint directory; without ``speculative_model`` + # neither the draft weights nor the ``dspark_*`` config + # defaults can be located, and engine construction would fail + # much later with an opaque error. + if spec_cfg.speculative_model is None: + raise ValueError( + "DSpark requires speculative_config.speculative_model " + "to point at the checkpoint directory containing the " + "mtp.* draft weights (for DeepSeek-V4-Pro-DSpark this " + "is the target checkpoint directory itself).") + # Resolve target_layer_ids / mask_token_id / block_size / + # markov_rank from the draft (or main) model config if not set. + # DSpark ships these as top-level ``dspark_*`` keys in the + # DeepSeek-V4-Pro config.json; also accept a nested + # ``dspark_config`` dict for forward compatibility. + draft_config_path = os.path.join(spec_cfg.speculative_model, + "config.json") + if os.path.exists(draft_config_path): + with open(draft_config_path) as f: + draft_cfg = json.load(f) + dspark_cfg = draft_cfg.get("dspark_config", {}) + + def _dspark_get(key, top_level_key): + value = dspark_cfg.get(key) + if value is None: + value = draft_cfg.get(top_level_key) + return value + + # The checkpoint's ``dspark_target_layer_ids`` is + # authoritative: it fixes both which target hidden states + # were captured during draft training and the input width of + # ``main_proj`` (hidden_size * num_capture_layers). If the + # user leaves ``target_layer_ids`` unset we copy it verbatim + # (order preserved, since the projection columns are + # order-dependent). An explicit override that does not match + # the checkpoint list exactly would either mismatch the + # projection shape at runtime (different count) or feed the + # draft hidden states it was not trained on (same count, + # different layers), so reject it during validation. + ckpt_layer_ids = _dspark_get("target_layer_ids", + "dspark_target_layer_ids") + if spec_cfg.target_layer_ids is None: + if ckpt_layer_ids is not None: + spec_cfg.target_layer_ids = list(ckpt_layer_ids) + elif ckpt_layer_ids is not None and list( + spec_cfg.target_layer_ids) != list(ckpt_layer_ids): + raise ValueError( + "DSpark target_layer_ids must match the checkpoint's " + "dspark_target_layer_ids exactly (the draft " + "projection weights are trained for that specific " + f"layer set); got override {spec_cfg.target_layer_ids} " + f"but the checkpoint specifies {list(ckpt_layer_ids)}. " + "Leave target_layer_ids unset to use the checkpoint " + "value.") + if spec_cfg.mask_token_id is None: + mask_id = _dspark_get("mask_token_id", + "dspark_noise_token_id") + if mask_id is not None: + spec_cfg.mask_token_id = mask_id + if spec_cfg.block_size is None: + block_size = _dspark_get("block_size", + "dspark_block_size") + if block_size is not None: + spec_cfg.block_size = int(block_size) + if spec_cfg.markov_rank is None: + markov_rank = _dspark_get("markov_rank", + "dspark_markov_rank") + if markov_rank is not None: + spec_cfg.markov_rank = markov_rank + if spec_cfg.block_size is None: + spec_cfg.block_size = spec_cfg.max_draft_len + if spec_cfg.block_size <= 0: + raise ValueError( + "DSpark block_size must be greater than zero") + if spec_cfg.block_size != spec_cfg.max_draft_len: + raise ValueError( + "DSpark block_size must equal max_draft_len; got " + f"block_size={spec_cfg.block_size} and " + f"max_draft_len={spec_cfg.max_draft_len}") + if isinstance(self.speculative_config, SADecodingConfig): pool_size = self.speculative_config.global_pool_size if pool_size is not None and self.max_batch_size is not None: @@ -5562,6 +6019,8 @@ def update_llm_args_with_extra_dict( If `explicit_cli_keys` is None, YAML wins on conflicts. """ + llm_args_dict = dict(llm_args_dict) + # CLI scalar -> nested KvCacheConfig field. Callers add the CLI scalar # name to `explicit_cli_keys` to make it win over YAML's same-named # field inside `kv_cache_config:`. @@ -5673,11 +6132,15 @@ def update_llm_args_with_extra_options( if extra_llm_api_options is not None: with open(extra_llm_api_options, 'r') as f: llm_args_dict = yaml.safe_load(f) - llm_args = update_llm_args_with_extra_dict( - llm_args, - llm_args_dict, - extra_llm_api_options, - explicit_cli_keys=explicit_cli_keys) + if llm_args_dict is None: + llm_args_dict = {} + elif not isinstance(llm_args_dict, dict): + raise ValueError("Configuration file root must be a mapping.") + llm_args = update_llm_args_with_extra_dict( + llm_args, + llm_args_dict, + extra_llm_api_options, + explicit_cli_keys=explicit_cli_keys) return llm_args diff --git a/tensorrt_llm/llmapi/llm_utils.py b/tensorrt_llm/llmapi/llm_utils.py index 159ebbd7315a..63b360f5584b 100644 --- a/tensorrt_llm/llmapi/llm_utils.py +++ b/tensorrt_llm/llmapi/llm_utils.py @@ -558,9 +558,18 @@ def _compute_applied(defaults: Dict[str, Any], def _resolve_kv_cache_manager_v2_auto( - llm_args: 'TorchLlmArgs', model_defaults_dict: Dict[str, Any]) -> bool: - """Resolve the KV cache manager auto setting after model defaults are applied.""" - setting = llm_args.kv_cache_config.use_kv_cache_manager_v2 + llm_args: 'TorchLlmArgs', + model_defaults_dict: Dict[str, Any], + original_setting: Optional[Union[bool, str]] = None) -> bool: + """Resolve the KV cache manager auto setting after model defaults are applied. + + The transceiver runtime auto setting must be resolved first. In + disaggregated serving, hybrid Mamba V2 requires the Python transceiver with + NIXL, so an incompatible route falls back to V1 unless the user explicitly + selected V2. + """ + setting = (llm_args.kv_cache_config.use_kv_cache_manager_v2 + if original_setting is None else original_setting) if setting != "auto": return setting @@ -574,6 +583,18 @@ def _resolve_kv_cache_manager_v2_auto( "Model default kv_cache_config.use_kv_cache_manager_v2 must be " f"True, False, or 'auto', got {model_default!r}.") + transceiver_config = llm_args.cache_transceiver_config + if (model_default and transceiver_config is not None + and transceiver_config.backend is not None): + effective_backend, _ = transceiver_config._resolve_default_backend() + runtime = transceiver_config.transceiver_runtime + if effective_backend != "NIXL" or runtime != "PYTHON": + logger.info( + "KV cache manager V2 is the model default, but disaggregated " + "serving uses transceiver_runtime=%r with backend=%r; " + "falling back to V1.", runtime, effective_backend) + model_default = False + llm_args.kv_cache_config.use_kv_cache_manager_v2 = model_default return model_default diff --git a/tensorrt_llm/llmapi/mpi_session.py b/tensorrt_llm/llmapi/mpi_session.py index 0fa5271f8acf..31bc7363e624 100644 --- a/tensorrt_llm/llmapi/mpi_session.py +++ b/tensorrt_llm/llmapi/mpi_session.py @@ -1,5 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import abc import itertools +import math import os import socket import sys @@ -8,6 +12,7 @@ import traceback from collections.abc import Callable from concurrent.futures import Future, ThreadPoolExecutor, as_completed +from concurrent.futures import wait as futures_wait from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypeVar import zmq @@ -125,10 +130,12 @@ def _abort_on_timeout(self, fut: Future, timeout: float, reason=None): try: fut.result(timeout=timeout) except TimeoutError: - logger.critical("MpiSession shutdown timeout, aborting...") + logger.critical(f"MpiSession shutdown timeout after {timeout}s; " + "calling MPI_Abort to force-kill stuck ranks...") if reason is not None: - logger.info(f"Reason to shutdown: {repr(reason)}") + logger.info(f"Reason to shutdown: {reason!r}") self.abort() + logger.critical("MpiSession MPI_Abort returned") def shutdown_abort(self, grace: float = 60, reason=None): if sys.is_finalizing(): @@ -136,6 +143,9 @@ def shutdown_abort(self, grace: float = 60, reason=None): # simply don't wait to avoid hang return self.shutdown(wait=False) + logger.info( + f"MpiSession.shutdown_abort: waiting up to {grace}s for workers to exit" + ) fut = Future() killer = threading.Thread(group=None, target=self._abort_on_timeout, @@ -143,16 +153,156 @@ def shutdown_abort(self, grace: float = 60, reason=None): args=(fut, grace, reason)) killer.start() self.shutdown() + logger.info("MpiSession.shutdown_abort: workers exited cleanly") fut.set_result(None) killer.join() + def release_exit_joins(self): + """Mark the worker world dead and release anything that would join it. + + Non-destructive, so it may be called by a component that does not + own the session. Must not tear the session down -- only ensure that + nothing (interpreter exit, a later blocking ``shutdown()`` by the + owner) waits forever on the dead world. Default: no-op. + """ + + def abandon(self): + """Tear the session down without waiting on a dead worker world.""" + self.release_exit_joins() + self.shutdown(wait=False) + + +def _abandon_mpi_pool_threads(mpi_pool) -> None: + """Let interpreter exit proceed despite a wedged pool manager thread. + + When the worker world dies abruptly, the ``MPIPoolExecutor`` manager + thread stays blocked in an MPI call forever, and process exit hangs on + it twice: mpi4py's exit hook joins every registered manager thread, and + CPython joins every non-daemon thread. Deregister the thread from both; + it is reaped with the process. + + Best-effort: the touched names are private to mpi4py (``THREADS_QUEUES`` + in ``_lib``/3.x and ``_core``/4.x) and CPython + (``threading._shutdown_locks``, 3.9-3.12). Where a name is absent, that + mechanism is left alone and exit may still block on it. + """ + thread = getattr(getattr(mpi_pool, '_pool', None), 'thread', None) + if thread is None: + return + # mpi4py's own exit hook (joins all registered manager threads). + for mod_name in ('mpi4py.futures._lib', 'mpi4py.futures._core'): + mod = sys.modules.get(mod_name) + registry = getattr(mod, 'THREADS_QUEUES', None) if mod else None + if registry is not None: + try: + registry.pop(thread, None) + except Exception as e: # noqa: BLE001 - best-effort cleanup + logger.debug(f"THREADS_QUEUES cleanup failed (ignored): {e!r}") + # CPython's non-daemon thread join at interpreter shutdown. + tstate_lock = getattr(thread, '_tstate_lock', None) + shutdown_locks = getattr(threading, '_shutdown_locks', None) + if tstate_lock is not None and shutdown_locks is not None: + try: + shutdown_locks.discard(tstate_lock) + except Exception as e: # noqa: BLE001 - best-effort cleanup + logger.debug(f"_shutdown_locks cleanup failed (ignored): {e!r}") + + +def _process_start_time(pid: int) -> Optional[bytes]: + """Kernel start time (jiffies since boot) of ``pid``, or None if gone. + + PIDs are recycled by the OS, but the (pid, start_time) pair uniquely + identifies a process incarnation — comparing it prevents waiting on an + unrelated process that inherited a dead worker's PID. + """ + try: + with open(f"/proc/{pid}/stat", "rb") as f: + stat = f.read() + # Field 2 (comm) may contain spaces/parens; parse after the last ')'. + return stat.rsplit(b")", 1)[1].split()[19] # field 22 overall + except OSError: + return None + + +_DEFAULT_IDENTITY_TIMEOUT = 300.0 + + +def _identity_barrier_timeout() -> float: + """Deadline for the ``wait_shutdown`` worker-identity barrier, in seconds. + + The barrier itself completes in milliseconds, but it is the first work ever + submitted to a freshly built ``MPIPoolExecutor``, and mpi4py spawns lazily + from its manager thread — so this deadline really bounds the whole worker + bootstrap: process spawn plus ``import tensorrt_llm``, measured at ~50-65s + on an idle node and up to ~117s on a contended one. Hence a ceiling sized + against bootstrap cost rather than barrier latency. The test-session + prefetcher derives its own wait budget from this value so it cannot abandon + a bootstrap that this layer still considers healthy. + ``TRTLLM_MPI_IDENTITY_TIMEOUT`` overrides it. + """ + raw = os.environ.get("TRTLLM_MPI_IDENTITY_TIMEOUT") + if not raw: + return _DEFAULT_IDENTITY_TIMEOUT + try: + value = float(raw) + if math.isfinite(value) and value > 0: + return value + except ValueError: + pass + logger.warning(f"Ignoring invalid TRTLLM_MPI_IDENTITY_TIMEOUT={raw!r}; " + f"using {_DEFAULT_IDENTITY_TIMEOUT}s") + return _DEFAULT_IDENTITY_TIMEOUT + + +def _worker_identity_barrier(): + """Runs inside a pool worker; module-level so it is picklable. + + The leading barrier pins the ``n_workers`` submitted tasks one-per-worker + (a worker holding one task blocks until every other worker holds its own, + so no worker can drain a second one), collecting every worker's identity + exactly once. The workers' ``MPI_COMM_WORLD`` is the spawned worker world + (the parent process is not a member). + """ + from mpi4py import MPI + MPI.COMM_WORLD.barrier() + pid = os.getpid() + return (pid, _process_start_time(pid)) + class MpiPoolSession(MpiSession): - def __init__(self, n_workers: int): + def __init__(self, + n_workers: int, + wait_shutdown: bool = False, + env_overrides: Optional[Dict[str, str]] = None): + """Spawn a pool of MPI worker processes. + + Args: + n_workers: number of MPI workers to spawn. + wait_shutdown: when True, ``shutdown()`` blocks until the spawned + worker processes have actually exited. + ``MPIPoolExecutor.shutdown`` returns at disconnect, but a + worker's GPU memory is only released when its process exits; + callers that start new GPU work right after ``shutdown()`` + (e.g. CI test infrastructure handing a pre-spawned pool to the + next test) race that release and can OOM. Off by default: + production teardown does not need the barrier and keeps its + current latency. + env_overrides: extra environment variables to set in the WORKERS at + spawn, on top of the TRTLLM*/TLLM* variables forwarded from the + parent. The parent process environment is never touched — this + replaces the racy "set os.environ around the spawn, then + restore" pattern for callers that spawn pools from background + threads. + """ self.n_workers = n_workers + self._wait_shutdown = wait_shutdown + self._env_overrides = dict(env_overrides) if env_overrides else {} + self._worker_identities: Tuple = () self.mpi_pool: Optional[MPIPoolExecutor] = None self._start_mpi_pool() + if wait_shutdown: + self._worker_identities = self._collect_worker_identities() if ENABLE_MULTI_DEVICE: self.comm = mpi4py.MPI.COMM_WORLD @@ -174,9 +324,106 @@ def submit_sync(self, task: Callable[..., T], *args, **kwargs) -> List[T]: return [future.result() for future in futures] def shutdown(self, wait=True): + if getattr(self, '_pool_dead', False): + # A dead pool can never be joined; never block on it, no matter + # what the caller asked for. + wait = False if self.mpi_pool is not None: + logger.info( + f"MpiPoolSession.shutdown: joining {self.n_workers} worker(s) " + f"(wait={wait})") self.mpi_pool.shutdown(wait=wait) + logger.info("MpiPoolSession.shutdown: done") self.mpi_pool = None + if self._wait_shutdown: + self._wait_workers_exit() + + def _collect_worker_identities(self) -> Tuple: + """(pid, start_time) of every worker, recorded right after spawn. + + FAIL-CLOSED (review requirement): ``wait_shutdown=True`` is a + contract — shutdown blocks until the workers exited. A pool without + complete identities cannot honor it, and returning it anyway would + silently downgrade to the old non-waiting behavior (the timeout can + trip on a slow-but-healthy bootstrap, and ``futures_wait`` does not + cancel the pending tasks). Instead of handing out such a pool, tear + it down and raise; callers fall back to a fresh spawn. + """ + timeout = _identity_barrier_timeout() + try: + futures = [ + self.mpi_pool.submit(_worker_identity_barrier) + for _ in range(self.n_workers) + ] + done, not_done = futures_wait(futures, timeout=timeout) + identities = tuple(f.result() for f in done) + except Exception as e: + self._teardown_unidentified_pool(()) + raise RuntimeError( + f"MpiPoolSession(wait_shutdown=True): worker identity " + f"collection failed ({e}); pool torn down") from e + if (not_done or len(identities) != self.n_workers + or len({pid + for pid, _ in identities}) != self.n_workers + or any(start is None for _, start in identities)): + self._teardown_unidentified_pool(identities) + raise RuntimeError( + "MpiPoolSession(wait_shutdown=True): worker identity " + f"collection incomplete ({len(identities)}/{self.n_workers} " + "valid identities); pool torn down instead of handing out a " + "session that cannot honor the wait_shutdown contract. Raise " + "TRTLLM_MPI_IDENTITY_TIMEOUT if worker bootstrap is merely " + f"slow (deadline was {timeout}s)") + return identities + + def _teardown_unidentified_pool(self, partial_identities: Tuple) -> None: + """Dispose of a pool whose identity collection failed. + + The workers may be stuck in the collection barrier (one of them + never picked up its task), so a graceful blocking shutdown could + hang; disconnect without waiting and SIGKILL the workers we did + identify (with the pid-recycling guard). Workers we never identified + exit with the MPI runtime teardown; if one is truly wedged it leaks + until job end — the same bounded leak class as any wedged pool. + """ + import signal + + try: + self.mpi_pool.shutdown(wait=False) + except Exception: + pass + self.mpi_pool = None + for pid, start in partial_identities: + if start is None or _process_start_time(pid) != start: + continue # gone already, or the PID was recycled + try: + os.kill(pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + + def _wait_workers_exit(self, timeout: float = 30.0) -> None: + """Block until the spawned worker processes have actually exited. + + Bounded: a wedged worker stops blocking the caller after ``timeout`` + (its memory is not coming back anyway; the caller's own recovery — + e.g. an OOM retry or a fresh spawn — takes over from there). + """ + deadline = time.monotonic() + timeout + for pid, start in self._worker_identities: + if start is None: + continue + while _process_start_time(pid) == start: + if time.monotonic() >= deadline: + logger.warning( + f"MpiPoolSession.shutdown: worker pid {pid} still " + f"alive after {timeout}s; not waiting further") + return + time.sleep(0.05) + + def release_exit_joins(self): + if self.mpi_pool is not None: + _abandon_mpi_pool_threads(self.mpi_pool) + self._pool_dead = True def abort(self): self.get_comm().Abort(1) @@ -189,6 +436,7 @@ def _start_mpi_pool(self): for key, value in os.environ.items() if key.startswith("TRTLLM") or key.startswith("TLLM") } + env.update(self._env_overrides) self.mpi_pool = MPIPoolExecutor(max_workers=self.n_workers, path=sys.path, env=env) @@ -258,7 +506,11 @@ def shutdown(self, wait=True): # Only shutdown the mpi_pool if this instance created it # For shared global mpi_pool, we don't shut it down if self.mpi_pool is not None and self.owns_mpi_pool: + logger.info( + f"MpiCommSession.shutdown: joining {self.n_workers - 1} worker(s) " + f"(wait={wait})") self.mpi_pool.shutdown(wait=wait) + logger.info("MpiCommSession.shutdown: mpi_pool done") self.mpi_pool = None if self.thread_pool is not None: self.thread_pool.shutdown(wait=wait) diff --git a/tensorrt_llm/llmapi/thinking_budget.py b/tensorrt_llm/llmapi/thinking_budget.py index d7ca6a5a58a8..35e68d48d1c8 100644 --- a/tensorrt_llm/llmapi/thinking_budget.py +++ b/tensorrt_llm/llmapi/thinking_budget.py @@ -33,6 +33,19 @@ def __init__( self.thinking_token_budget = budget self.reasoning_start_token_ids = list(reasoning_start_token_ids) self.reasoning_end_token_ids = list(reasoning_end_token_ids) + # Progress through the reasoning end sequence, keyed by (req_id, beam_idx): + # 0 = not closing, 1..len-1 = mid-sequence, >= len = fully emitted. A + # missing key defaults to 0; the key is popped when the block closes. + # + # We keep this instead of re-deriving from `token_ids` because the overlap + # scheduler runs the processor one step behind the sampled tokens: a stale + # view would not show the just-forced end sequence, so re-derivation would + # force it again and leak a duplicate end tag (e.g. ``). + # + # Keys are reclaimed only when a block is seen to close, so entries for + # blocks that never close persist for the processor's lifetime -- bounded + # per request, but a slow drip if one SamplingParams is shared across many. + self._end_progress: dict = {} def __call__( self, @@ -42,28 +55,41 @@ def __call__( stream_ptr: Optional[int], client_id: Optional[int], ) -> None: - del req_id, client_id + del client_id if stream_ptr is None: - self._apply(token_ids, logits) + self._apply(req_id, token_ids, logits) return with torch.cuda.stream(torch.cuda.ExternalStream(stream_ptr)): - self._apply(token_ids, logits) + self._apply(req_id, token_ids, logits) - def _apply(self, token_ids: List[List[int]], logits: torch.Tensor) -> None: + def _apply(self, req_id: int, token_ids: List[List[int]], logits: torch.Tensor) -> None: for beam_idx, beam_token_ids in enumerate(token_ids): - forced_token = self._forced_token(beam_token_ids) + forced_token = self._forced_token((req_id, beam_idx), beam_token_ids) if forced_token is not None: self._force_token(logits, beam_idx, len(token_ids), forced_token) - def _forced_token(self, token_ids: List[int]) -> Optional[int]: + def _forced_token(self, key, token_ids: List[int]) -> Optional[int]: start_idx = _find_last_sequence_index(token_ids, self.reasoning_start_token_ids) if start_idx == -1: return None end_idx = _find_last_sequence_index(token_ids, self.reasoning_end_token_ids) if end_idx > start_idx: + # Reasoning block closed; reset so a later block is budgeted again. + self._end_progress.pop(key, None) return None + # Block still open. If we have already begun forcing the end sequence, + # trust recorded progress (see _end_progress) over the lagging token view. + progress = self._end_progress.get(key, 0) + if progress >= len(self.reasoning_end_token_ids): + # Whole end sequence already forced; stop so we don't leak a second tag. + return None + if progress > 0: + # Mid end-sequence: emit the next token and advance (don't restart at 0). + self._end_progress[key] = progress + 1 + return self.reasoning_end_token_ids[progress] + reasoning_start = start_idx + len(self.reasoning_start_token_ids) reasoning_token_count = len(token_ids) - reasoning_start partial_end_len = _longest_suffix_prefix_len(token_ids, self.reasoning_end_token_ids) @@ -71,9 +97,11 @@ def _forced_token(self, token_ids: List[int]) -> Optional[int]: partial_end_len > 0 and reasoning_token_count - partial_end_len >= self.thinking_token_budget ): + self._end_progress[key] = partial_end_len + 1 return self.reasoning_end_token_ids[partial_end_len] if reasoning_token_count >= self.thinking_token_budget: + self._end_progress[key] = 1 return self.reasoning_end_token_ids[0] return None diff --git a/tensorrt_llm/llmapi/utils.py b/tensorrt_llm/llmapi/utils.py index f1b49fc87af8..acfc2decd9d2 100644 --- a/tensorrt_llm/llmapi/utils.py +++ b/tensorrt_llm/llmapi/utils.py @@ -18,7 +18,7 @@ from contextlib import nullcontext from functools import wraps from pathlib import Path -from queue import Queue +from queue import Empty, Queue from typing import (Any, Callable, ContextManager, Iterable, List, Optional, Tuple, Type, get_type_hints) @@ -534,12 +534,20 @@ def get(self, timeout=None): # We can't call asyncio.run_coroutine_threadsafe(self._aq.get(), self.loop) and wait the returned Future, # since we are in the same event loop, and we can't yield the thread while waiting result. - deadline = None if timeout is None else time.time() + timeout - while deadline is None or time.time() < deadline: + deadline = None if timeout is None else time.monotonic() + timeout + while True: try: return self._aq.unsafe_get() except asyncio.QueueEmpty: - time.sleep(0.01) + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + # Match `queue.Queue.get()` semantics; a silent `None` return would be + # mis-handled downstream as an unknown response type. + raise Empty() from None + time.sleep(min(0.01, remaining)) + else: + time.sleep(0.01) def get_numa_aware_cpu_affinity(device_id): diff --git a/tensorrt_llm/metrics/collector.py b/tensorrt_llm/metrics/collector.py index cd5a91123ddf..0586b46173ab 100644 --- a/tensorrt_llm/metrics/collector.py +++ b/tensorrt_llm/metrics/collector.py @@ -805,7 +805,15 @@ def log_iteration_stats(self, iteration_stats: dict) -> None: kv_iter_by_pool_group = iteration_stats.get( "kvCacheIterationStatsByPoolGroup") if kv_iter or kv_iter_by_lifecycle or kv_iter_by_pool_group: - reuse_stats = kv_iter_by_lifecycle or kv_iter or {} + # Prefer lifecycle-level attention stats when present. An SSM-only + # lifecycle report must not hide the legacy/window-level attention + # aggregate. Missing kind remains attention-compatible. + attention_lifecycle_stats = { + key: stats + for key, stats in (kv_iter_by_lifecycle or {}).items() + if stats.get("kind", "attention") == "attention" + } + reuse_stats = attention_lifecycle_stats or kv_iter or {} pool_group_stats = kv_iter_by_pool_group or kv_iter or {} total_secondary_max = 0 total_secondary_used = 0 diff --git a/tensorrt_llm/models/automodel.py b/tensorrt_llm/models/automodel.py index 463ae334cad6..6e7ae4041c93 100644 --- a/tensorrt_llm/models/automodel.py +++ b/tensorrt_llm/models/automodel.py @@ -1,7 +1,6 @@ from pathlib import Path from typing import Optional, Union -from ..bindings.executor import DecodingMode from ..mapping import Mapping from . import MODEL_MAP from .modeling_utils import QuantConfig @@ -53,8 +52,7 @@ class AutoModelForCausalLM: @staticmethod def get_trtllm_model_class(hf_model_or_dir: Union[str, Path], - trust_remote_code: bool = False, - decoding_mode: DecodingMode = None): + trust_remote_code: bool = False): import transformers hf_model_or_dir = Path(hf_model_or_dir) if not isinstance( @@ -65,15 +63,8 @@ def get_trtllm_model_class(hf_model_or_dir: Union[str, Path], hf_config = transformers.AutoConfig.from_pretrained( hf_model_or_dir, trust_remote_code=trust_remote_code) - if decoding_mode is not None: - if decoding_mode.isMedusa(): - hf_arch = 'MedusaForCausalLM' - elif decoding_mode.isEagle(): - hf_arch = 'EagleForCausalLM' - else: - raise NotImplementedError(f"Unknown speculative decoding mode.") - elif hasattr(hf_config, - 'architectures') and hf_config.architectures is not None: + if hasattr(hf_config, + 'architectures') and hf_config.architectures is not None: hf_arch = hf_config.architectures[0] elif hasattr(hf_config, 'model_type') and hf_config.model_type.find('mamba') != -1: diff --git a/tensorrt_llm/models/convert_utils.py b/tensorrt_llm/models/convert_utils.py index 58da3ec36349..5930532202de 100644 --- a/tensorrt_llm/models/convert_utils.py +++ b/tensorrt_llm/models/convert_utils.py @@ -4,7 +4,6 @@ from typing import Dict, List, Optional, Tuple, Union import torch -from datasets import load_dataset from .._utils import torch_dtype_to_str from ..logger import logger @@ -292,39 +291,6 @@ def has_safetensors(model_dir: str): return len(list(Path(model_dir).glob('*.safetensors'))) > 0 -DEFAULT_HF_DATASET_META = { - 'ccdv/cnn_dailymail': ('3.0.0', 'train', 'article'), - 'cnn_dailymail': ('3.0.0', 'train', 'article'), - 'lambada': (None, 'validation', 'text'), - '': (None, 'train', 'text'), # Default value in HF -} - - -def load_calib_dataset(dataset_name_or_dir: str, - config_name: Optional[str] = None, - split: Optional[str] = None, - key: Optional[str] = None, - trust_remote_code=True, - **kwargs): - if config_name is None: - for name, meta in DEFAULT_HF_DATASET_META.items(): - if name in dataset_name_or_dir: - if config_name is None: - config_name = meta[0] - if split is None: - split = meta[1] - if key is None: - key = meta[2] - break - - dataset = load_dataset(dataset_name_or_dir, - name=config_name, - split=split, - trust_remote_code=trust_remote_code, - **kwargs) - return dataset[key] - - @torch.no_grad() def apply_smoothing( scales: torch.Tensor, diff --git a/tensorrt_llm/models/unet/pp/__init__.py b/tensorrt_llm/models/unet/pp/__init__.py deleted file mode 100755 index e69de29bb2d1..000000000000 diff --git a/tensorrt_llm/quantization/__init__.py b/tensorrt_llm/quantization/__init__.py index 0ecdd664e60d..900ad46bd0e8 100644 --- a/tensorrt_llm/quantization/__init__.py +++ b/tensorrt_llm/quantization/__init__.py @@ -16,11 +16,9 @@ from .mode import (KV_CACHE_QUANT_ALGO_LIST, MODELOPT_FLOW_QUANTIZATIONS, QUANT_ALGO_LIST, W8A8_SQ_PLUGIN_LIST, GroupwiseQuantAlgo, QuantAlgo, QuantMode) -from .quantize_by_modelopt import quantize_and_export, quantize_nemo_and_export __all__ = [ 'QUANT_ALGO_LIST', 'KV_CACHE_QUANT_ALGO_LIST', 'W8A8_SQ_PLUGIN_LIST', 'MODELOPT_FLOW_QUANTIZATIONS', 'QuantAlgo', 'QuantMode', - 'GroupwiseQuantAlgo', 'quantize_and_export', 'quantize_nemo_and_export', - 'utils' + 'GroupwiseQuantAlgo', 'utils' ] diff --git a/tensorrt_llm/quantization/image_processing.py b/tensorrt_llm/quantization/image_processing.py deleted file mode 100644 index 68360ff32ee0..000000000000 --- a/tensorrt_llm/quantization/image_processing.py +++ /dev/null @@ -1,97 +0,0 @@ -import torch - - -class BaseImageProcessor: - - def __init__(self, tokenizer, device='auto'): - self.tokenizer = tokenizer - self.device = device - - def __call__(self, **kwargs): - return self.tokenizer(**kwargs) - - def preprocess_function(self, examples): - raise NotImplementedError( - "Each image processor must implement its own preprocess method") - - def collate_function(self, examples): - raise NotImplementedError( - "Each image processor must implement its own colloate method") - - -# A light Encapsulation for Huggingface MllamaImageProcessor -class MllamaImageProcessor(BaseImageProcessor): - - def preprocess_function(self, examples): - # Prepare prompts in a generic chat format - if 'question' in examples: - question = examples['question'] - else: - question = "Describe this image." - - if examples['image'] is not None: - if self.tokenizer.chat_template is not None: - prompt = self.tokenizer.apply_chat_template( - [{ - "role": - "user", - "content": [{ - "type": "image" - }, { - "type": "text", - "text": question - }], - }], - add_generation_prompt=True, - ) - else: - prompt = f"<|image|><|begin_of_text|>{question}" - - # Process images using the processor's image processor - values = self.tokenizer(text=prompt, - images=examples['image'], - return_tensors="pt").to(self.device) - else: - if self.tokenizer.chat_template is not None: - prompt = self.tokenizer.apply_chat_template( - [{ - "role": "user", - "content": [{ - "type": "text", - "text": question - }], - }], - add_generation_prompt=True, - ) - else: - prompt = question - - values = self.tokenizer(text=prompt, - images=None, - return_tensors="pt").to(self.device) - - values['pixel_values'] = None - values['aspect_ratio_ids'] = None - values['aspect_ratio_mask'] = None - values['cross_attention_mask'] = None - - return values - - # Define a collate function to process images during data loading - def collate_function(self, batch): - batch[0]['input_ids'] = torch.LongTensor(batch[0]['input_ids']).to( - self.device) - batch[0]['attention_mask'] = torch.LongTensor( - batch[0]['attention_mask']).to(self.device) - - if batch[0]['pixel_values'] is not None: - batch[0]['pixel_values'] = torch.Tensor( - batch[0]['pixel_values']).to(self.device) - batch[0]['aspect_ratio_ids'] = torch.LongTensor( - batch[0]['aspect_ratio_ids']).to(self.device) - batch[0]['aspect_ratio_mask'] = torch.LongTensor( - batch[0]['aspect_ratio_mask']).to(self.device) - batch[0]['cross_attention_mask'] = torch.LongTensor( - batch[0]['cross_attention_mask']).to(self.device) - - return batch[0] diff --git a/tensorrt_llm/quantization/quantize_by_modelopt.py b/tensorrt_llm/quantization/quantize_by_modelopt.py deleted file mode 100755 index be9236aa6cef..000000000000 --- a/tensorrt_llm/quantization/quantize_by_modelopt.py +++ /dev/null @@ -1,1332 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Adapted from examples/quantization/hf_ptq.py -""" - -import copy -import json -import os -import random -import sys -import time -from importlib.metadata import version - -import numpy as np -import torch -from accelerate.hooks import remove_hook_from_module -from datasets import load_dataset -from modelopt.torch.utils import print_rank_0 -from torch import nn -from torch.utils.data import DataLoader -from transformers import (AutoConfig, AutoModelForCausalLM, AutoProcessor, - AutoTokenizer) - -from .._utils import get_hf_rope_theta, release_gc, str_dtype_to_torch -from ..logger import logger -from .image_processing import MllamaImageProcessor -from .mode import QuantAlgo - -EMPTY_CFG = { - "quant_cfg": { - "*weight_quantizer": { - "enable": False, - }, - "*input_quantizer": { - "enable": False - }, - "*lm_head*": { - "enable": False - }, - "*output_layer*": { - "enable": False - }, - "default": { - "enable": False - }, - }, - "algorithm": "max", -} - -KV_CACHE_CFG = { - "*.query_key_value.output_quantizer": { - "num_bits": 8, - "axis": None, - "enable": True - }, - "*.Wqkv.output_quantizer": { - "num_bits": 8, - "axis": None, - "enable": True - }, - "*.W_pack.output_quantizer": { - "num_bits": 8, - "axis": None, - "enable": True - }, - "*.c_attn.output_quantizer": { - "num_bits": 8, - "axis": None, - "enable": True - }, - "*.k_proj.output_quantizer": { - "num_bits": 8, - "axis": None, - "enable": True - }, - "*.v_proj.output_quantizer": { - "num_bits": 8, - "axis": None, - "enable": True - }, - "*.k.output_quantizer": { - "num_bits": 8, - "axis": None, - "enable": True - }, - "*.v.output_quantizer": { - "num_bits": 8, - "axis": None, - "enable": True - }, -} - -KV_QUANT_CFG_CHOICES = { - "fp8": "FP8_KV_CFG", - "nvfp4": "NVFP4_KV_CFG", -} - - -def quant_cfg_choices(): - import modelopt.torch.quantization as mtq - QUANT_CFG_CHOICES = { - "int8_sq": mtq.INT8_SMOOTHQUANT_CFG, - "fp8": mtq.FP8_DEFAULT_CFG, - "fp8_pc_pt": mtq.FP8_PER_CHANNEL_PER_TOKEN_CFG, - "int4_awq": mtq.INT4_AWQ_CFG, - "w4a8_awq": mtq.W4A8_AWQ_BETA_CFG, - "int8_wo": EMPTY_CFG, - "int4_wo": EMPTY_CFG, - "full_prec": EMPTY_CFG, - } - if hasattr(mtq, "NVFP4_DEFAULT_CFG"): - QUANT_CFG_CHOICES["nvfp4"] = mtq.NVFP4_DEFAULT_CFG - return QUANT_CFG_CHOICES - - -def model_type_is_enc_dec(model_type): - return model_type in ["t5", "bart"] - - -MODEL_NAME_PATTERN_MAP = { - "GPT2": "gpt2", - "Xverse": "llama", - "MllamaForConditionalGeneration": "mllama", - "Llama": "llama", - "MllamaForCausalLM": "mllama", - "Mistral": "llama", - "GPTJ": "gptj", - "FalconForCausalLM": "falcon", - "RWForCausalLM": "falcon", - "baichuan": "baichuan", - "MPT": "mpt", - "Bloom": "bloom", - "ChatGLM": "chatglm", - "QWen": "qwen", - "Qwen2VLForConditionalGeneration": "qwen2_vl", - "RecurrentGemma": "recurrentgemma", - "Gemma3": "gemma3", - "Gemma2": "gemma2", - "Gemma": "gemma", - "MixtralForCausalLM": "llama", - "NemotronForCausalLM": "nemotron", - "GPTBigCodeForCausalLM": "gpt_bigcode", - "ArcticForCausalLM": "llama", - "PhiMoEForCausalLM": "phi3", - "Phi3SmallForCausalLM": "phi3small", - "Phi3ForCausalLM": "phi3", - "Phi3VForCausalLM": "phi3", - "Starcoder2ForCausalLM": "gptnext", - "GPTBigCodeForCausalLM": "gptnext", - "GLM": "glm", - "Exaone": "exaone", - "DeciLMForCausalLM": "deci", - "DeepseekForCausalLM": "deepseek", - "GraniteForCausalLM": "granite", - "GraniteMoeForCausalLM": "granitemoe", - "T5": "t5", - "Bart": "bart" -} - -MULTIMODAL_DATASETS = ['scienceqa', 'science_qa'] - - -class _CustomDataset(torch.utils.data.Dataset): - - def __init__(self, encodings): - self.encodings = encodings - - def __getitem__(self, idx): - item = { - key: val[idx].clone().detach().requires_grad_(False) - for key, val in self.encodings.items() - } - return item - - def __len__(self): - return len(self.encodings["input_ids"]) - - -class EncDecModelWrapper(torch.nn.Module): - - def __init__(self, hf_model=None): - super().__init__() - self.hf_model = hf_model - self.model_type = get_model_type(hf_model) - - def forward(self, **kwargs): - self.hf_model.generate(**kwargs) - - def __getattr__(self, name): - try: - return super().__getattr__(name) - except AttributeError: - return getattr(self.hf_model, name) - - -def get_tokenizer(ckpt_path, max_seq_length=2048, model_type=None): - logger.info(f"Initializing tokenizer from {ckpt_path}") - tokenizer = AutoTokenizer.from_pretrained( - ckpt_path, - model_max_length=max_seq_length, - padding_side="left", - trust_remote_code=True, - ) - - if tokenizer.pad_token is None: - if model_type and model_type == "qwen": - # qwen use token id 151643 as pad and eos tokens - tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643) - tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643) - elif model_type and model_type == "qwen2_vl": - # qwen use token id 151643 as pad and 151643 and 151645 as eos tokens - tokenizer.eos_token = [ - tokenizer.convert_ids_to_tokens(151643), - tokenizer.convert_ids_to_tokens(151645) - ] - tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643) - else: - tokenizer.pad_token = tokenizer.eos_token - assert tokenizer.pad_token is not None, f"Pad token for {model_type} cannot be set!" - - return tokenizer - - -def get_processor(ckpt_path, max_seq_length=2048, model_type=None, device=None): - logger.info(f"Initializing tokenizer from {ckpt_path}") - processor = AutoProcessor.from_pretrained( - ckpt_path, - model_max_length=max_seq_length, - padding_side="left", - trust_remote_code=True, - ) - - if processor.tokenizer.pad_token is None: - if model_type and model_type == "qwen": - # qwen use token id 151643 as pad and eos tokens - processor.tokenizer.eos_token = processor.tokenizer.convert_ids_to_tokens( - 151643) - processor.tokenizer.pad_token = processor.tokenizer.convert_ids_to_tokens( - 151643) - else: - processor.tokenizer.pad_token = processor.tokenizer.eos_token - assert processor.tokenizer.pad_token is not None, f"Pad token for {model_type} cannot be set!" - - if model_type == 'mllama': - processor = MllamaImageProcessor(processor, device) - return processor - - -def _get_vila_model(model_dir): - sys.path.append(model_dir + "/../VILA") - from llava.model import LlavaLlamaConfig, LlavaLlamaModel # noqa - from transformers import AutoModel - model = AutoModel.from_pretrained( - model_dir, - device_map='auto', - trust_remote_code=True, - ) - return model.llm - - -def get_hf_config(ckpt_path): - if "mpt" in ckpt_path: - # MPT-7B cannot get initialized from AutoConfig - from transformers import MptConfig - return MptConfig.from_pretrained(ckpt_path) - else: - return AutoConfig.from_pretrained(ckpt_path, trust_remote_code=True) - - -class _MixtralBlockSparseTop2MLPCompat(nn.Module): - # Per-expert 4.x-style MLP using zero-copy parameter views into the - # transformers-5.x fused MoE tensors. Uses real nn.Linear modules so - # modelopt's quantizer wrapping (which keys off type name "Linear") - # produces per-expert weight scaling factors during calibration. - def __init__(self, hidden_dim, intermediate_dim, w1_view, w2_view, w3_view, - act_fn): - super().__init__() - self.w1 = nn.Linear(hidden_dim, intermediate_dim, bias=False) - self.w2 = nn.Linear(intermediate_dim, hidden_dim, bias=False) - self.w3 = nn.Linear(hidden_dim, intermediate_dim, bias=False) - self.w1.weight = nn.Parameter(w1_view, requires_grad=False) - self.w2.weight = nn.Parameter(w2_view, requires_grad=False) - self.w3.weight = nn.Parameter(w3_view, requires_grad=False) - self.act_fn = act_fn - - def forward(self, hidden_states): - return self.w2( - self.act_fn(self.w1(hidden_states)) * self.w3(hidden_states)) - - -class _MixtralSparseMoeBlockCompat(nn.Module): - # Drop-in replacement for transformers-5.x MixtralSparseMoeBlock that - # restores the pre-5.x ModuleList(MixtralBlockSparseTop2MLP) layout that - # nvidia-modelopt 0.37 iterates (len(experts), experts[i].{w1,w2,w3}.weight), - # while sharing weight storage with the 5.x fused tensors via parameter - # views (zero-copy). The class name contains "MixtralSparseMoeBlock" so - # modelopt's is_moe substring check still matches. - def __init__(self, mlp): - super().__init__() - experts = mlp.experts # MixtralExperts - self.top_k = mlp.top_k - self.jitter_noise = mlp.jitter_noise - self.num_experts = experts.num_experts - self.hidden_dim = experts.hidden_dim - self.intermediate_dim = experts.intermediate_dim - self.gate = mlp.gate # MixtralTopKRouter, returns (logits, scores, indices) - - gate_up = experts.gate_up_proj # [N, 2*I, H] - down = experts.down_proj # [N, H, I] - act_fn = experts.act_fn - - new_experts = nn.ModuleList() - for i in range(self.num_experts): - new_experts.append( - _MixtralBlockSparseTop2MLPCompat( - self.hidden_dim, - self.intermediate_dim, - gate_up[i, :self.intermediate_dim, :], # w1: gate - down[i], # w2: down - gate_up[i, self.intermediate_dim:, :], # w3: up - act_fn, - )) - self.experts = new_experts - - def forward(self, hidden_states): - batch_size, sequence_length, hidden_dim = hidden_states.shape - if self.training and self.jitter_noise > 0: - hidden_states = hidden_states * torch.empty_like( - hidden_states).uniform_(1.0 - self.jitter_noise, - 1.0 + self.jitter_noise) - hidden_states_flat = hidden_states.view(-1, hidden_dim) - _, top_k_weights, top_k_index = self.gate(hidden_states_flat) - - final_hidden_states = torch.zeros_like(hidden_states_flat) - expert_mask = torch.nn.functional.one_hot( - top_k_index, num_classes=self.num_experts).permute(2, 1, 0) - expert_hit = (expert_mask.sum(dim=(-1, -2)) > 0).nonzero() - - for expert_idx in expert_hit: - expert_idx = expert_idx[0] - if expert_idx == self.num_experts: - continue - top_k_pos, token_idx = torch.where(expert_mask[expert_idx]) - current_state = hidden_states_flat[token_idx] - current_hidden_states = self.experts[expert_idx](current_state) - current_hidden_states = current_hidden_states * top_k_weights[ - token_idx, top_k_pos, None] - final_hidden_states.index_add_( - 0, token_idx, - current_hidden_states.to(final_hidden_states.dtype)) - - return final_hidden_states.reshape(batch_size, sequence_length, - hidden_dim) - - -def _unfuse_mixtral_for_modelopt(model: nn.Module) -> None: - # transformers 5.x stores Mixtral experts as a single MixtralExperts module - # with 3D fused tensors (gate_up_proj, down_proj) under layer.mlp, replacing - # the per-expert ModuleList layout that nvidia-modelopt 0.37 iterates - # (len(experts), experts[i].w1.weight, ...). Without this swap, the export - # call fails with: TypeError: object of type 'MixtralExperts' has no len(). - # - # An earlier attempt added a sibling layer.block_sparse_moe with the legacy - # layout, but modelopt iterates decoder_layer.named_children() and hits the - # original layer.mlp (a MixtralSparseMoeBlock matching is_moe) first, so the - # exception still triggered. Replace layer.mlp in place so calibration uses - # per-expert nn.Linear modules (allowing modelopt to attach quantizers) and - # export reads the per-expert ModuleList layout. - try: - from transformers.models.mixtral.modeling_mixtral import MixtralExperts - except ImportError: - return - - if not (hasattr(model, "model") and hasattr(model.model, "layers")): - return - - for layer in model.model.layers: - mlp = getattr(layer, "mlp", None) - if mlp is None or not isinstance(getattr(mlp, "experts", None), - MixtralExperts): - continue - # nn.Module() defaults to training=True; mirror the original mlp's - # mode so a prior model.eval() is preserved (avoids re-enabling the - # router-input jitter during calibration/export). - compat_mlp = _MixtralSparseMoeBlockCompat(mlp) - compat_mlp.train(mlp.training) - layer.mlp = compat_mlp - - -def _get_llava_qwen_model(model_dir, dtype, device): - if "hf" in model_dir: - from transformers import LlavaOnevisionForConditionalGeneration - model = LlavaOnevisionForConditionalGeneration.from_pretrained( - model_dir, dtype=dtype, device_map=device) - model = model.language_model - else: - from llava.model.builder import load_pretrained_model - _, model, _, _ = load_pretrained_model(model_dir, - None, - 'llava_qwen', - torch_dtype=dtype, - device_map=device) - return model - - -def get_model(ckpt_path: str, - dtype: str = 'bfloat16', - device: str = 'cuda', - device_map: str = "auto"): - logger.info(f"Initializing model from {ckpt_path}") - # Note: VILA model is not in public HF model zoo yet. We need to explicitly import from the git repo - hf_config = get_hf_config(ckpt_path) - torch_dtype = str_dtype_to_torch(dtype) - - model_cls = AutoModelForCausalLM - if hf_config.model_type == "llava": - from transformers import LlavaForConditionalGeneration - model_cls = LlavaForConditionalGeneration - elif hf_config.model_type == "mpt": - from transformers import MptForCausalLM - model_cls = MptForCausalLM - elif hf_config.model_type == 'mllama': - from transformers import MllamaForConditionalGeneration - model_cls = MllamaForConditionalGeneration - elif hf_config.model_type == 'qwen2_vl': - from transformers import Qwen2VLForConditionalGeneration - model_cls = Qwen2VLForConditionalGeneration - - if "vila" in ckpt_path: - model = _get_vila_model(ckpt_path) - elif "llava-onevision-qwen2" in ckpt_path: - model = _get_llava_qwen_model(ckpt_path, dtype, device) - elif hf_config.model_type == "glm": - from transformers import AutoModelForSeq2SeqLM - model = AutoModelForSeq2SeqLM.from_pretrained(ckpt_path, - device_map="cuda", - dtype=torch_dtype, - trust_remote_code=True) - elif model_type_is_enc_dec(hf_config.model_type): - from transformers import AutoModelForSeq2SeqLM - model = AutoModelForSeq2SeqLM.from_pretrained(ckpt_path, - device_map=device, - dtype=torch_dtype, - trust_remote_code=True) - model = EncDecModelWrapper(hf_model=model) - else: - model = model_cls.from_pretrained( - ckpt_path, - device_map=device_map if device != "cpu" else "cpu", - dtype="auto", - trust_remote_code=True) - if hf_config.model_type in ["llava", "internvl_chat"]: - model = model.language_model - elif hf_config.model_type == "qwen2_vl": - #WAR for Qwen2-VL because its lm_head is outside of LLM - lm_head = model.lm_head - model = model.model - model.lm_head = lm_head - - model.eval() - - # transformers 5.x changed Mixtral MoE to a fused 3D layout that - # nvidia-modelopt 0.37 cannot iterate. Restore the per-expert layout - # so both calibration and export work; see _unfuse_mixtral_for_modelopt. - if hf_config.model_type == "mixtral": - _unfuse_mixtral_for_modelopt(model) - - model_dtype = next(model.parameters()).dtype - if torch_dtype != model_dtype: - logger.info( - f"[TensorRT-LLM][WARNING] The manually set model data type is {dtype}, " - f"but the data type of the HuggingFace model is {model_dtype}.") - - return model - - -def get_model_type(model): - if type(model).__name__ == "EncDecModelWrapper": - return model.model_type - if type(model).__name__ in MODEL_NAME_PATTERN_MAP: - return MODEL_NAME_PATTERN_MAP[type(model).__name__] - for k, v in MODEL_NAME_PATTERN_MAP.items(): - if k.lower() in type(model).__name__.lower(): - return v - return None - - -def _is_cnn_dailymail_local_repo(path: str) -> bool: - if not os.path.isdir(path): - return False - # The loader only uses the "3.0.0" config. - if os.path.isdir(os.path.join(path, "3.0.0")): - return True - if os.path.isfile(os.path.join(path, "cnn_dailymail.py")): - return True - return False - - -def get_calib_dataloader(dataset_name_or_dir="cnn_dailymail", - tokenizer=None, - batch_size=1, - calib_size=512, - block_size=512, - device=None, - include_labels=False): - logger.info("Loading calibration dataset") - if dataset_name_or_dir == "pileval": - dataset = load_dataset( - "json", - data_files="https://the-eye.eu/public/AI/pile/val.jsonl.zst", - split="train", - trust_remote_code=True) - dataset = dataset["text"][:calib_size] - elif "scienceqa" in dataset_name_or_dir.lower( - ) or "science_qa" in dataset_name_or_dir.lower(): - if os.path.isdir(dataset_name_or_dir): - dataset = load_dataset(dataset_name_or_dir, - split="train", - trust_remote_code=True) - else: - dataset = load_dataset("derek-thomas/ScienceQA", - split="train", - trust_remote_code=True) - dataset = dataset.select(range(calib_size)) - elif "cnn_dailymail" in dataset_name_or_dir or _is_cnn_dailymail_local_repo( - dataset_name_or_dir): - # Bare "cnn_dailymail" id is rejected by newer huggingface_hub; use the namespaced repo. - if dataset_name_or_dir == "cnn_dailymail": - dataset_name_or_dir = "abisee/cnn_dailymail" - dataset = load_dataset( - dataset_name_or_dir, - name="3.0.0", - split="train", - trust_remote_code=True, - ) - dataset = dataset["article"][:calib_size] - elif os.path.isdir(dataset_name_or_dir): - logger.info( - f"Recognized local dataset repo {dataset_name_or_dir} for calibration; " - "assuming the calibration data are in the train split and text column." - ) - dataset = load_dataset(dataset_name_or_dir, - split="train", - trust_remote_code=True) - dataset = dataset["text"][:calib_size] - else: - raise NotImplementedError( - f"Unsupported dataset name or local repo directory: {dataset_name_or_dir}." - ) - - is_multimodal = False - for dataset_name in MULTIMODAL_DATASETS: - if dataset_name in dataset_name_or_dir: - is_multimodal = True - if is_multimodal: - # Apply the preprocessing function to the dataset - processed_dataset = dataset.map(tokenizer.preprocess_function, - batched=False, - remove_columns=dataset.column_names) - - # Create DataLoader with the custom collate function - calib_dataloader = DataLoader(processed_dataset, - batch_size=batch_size, - shuffle=False, - collate_fn=tokenizer.collate_function) - else: - batch_encoded = tokenizer(dataset, - return_tensors="pt", - padding=True, - truncation=True, - max_length=block_size) - if device: - batch_encoded = batch_encoded.to(device) - - if include_labels: - # Labels are needed when backward is called in the model. - # The labels should be a shifted version of the input_ids. - # However, we should not shift the input_ids here since the labels are shifted by - # Huggingface models during loss calculation as shown here - - # https://github.com/huggingface/transformers/blob/7f79a97399bb52aad8460e1da2f36577d5dccfed/src/transformers/models/llama/modeling_llama.py#L1093-L1095 - batch_encoded["labels"] = torch.where( - batch_encoded["attention_mask"] > 0.5, - batch_encoded["input_ids"], -100) - batch_encoded = _CustomDataset(batch_encoded) - else: - # For backward compatibility, if labels are not needed, we only return input_ids. - batch_encoded = _CustomDataset( - {"input_ids": batch_encoded["input_ids"]}) - - calib_dataloader = DataLoader(batch_encoded, - batch_size=batch_size, - shuffle=False) - - return calib_dataloader - - -def quantize_model(model, quant_cfg, calib_dataloader, batch_size, qformat, - auto_quantize_bits): - import modelopt.torch.quantization as mtq - - # NOTE: for ModelOpt v0.19 release - # calibrate_loop = dataset_utils.create_forward_loop( - # calib_dataloader, dataloader=calib_dataloader) - - def calibrate_loop(): - if calib_dataloader is None: - return - with torch.no_grad(): - low_mem_mode = False - for idx, data in enumerate(calib_dataloader): - logger.debug(f"Calibrating batch {idx}") - batch_size = data[list(data.keys())[0]].shape[0] - if batch_size == 1: - model(**data) - elif not low_mem_mode: - # Try running the forward once. - # If output memory, we try running inference with split input tensors - try: - model(**data) - except torch.OutOfMemoryError: - print( - "Warning: torch.OutOfMemoryError detected, try reducing the batch size..." - ) - low_mem_mode = True - - if low_mem_mode: - split_data_1 = { - key: data[key][:batch_size // 2, ...] - for key in data - } - model(**split_data_1) - - split_data_2 = { - key: data[key][batch_size // 2:, ...] - for key in data - } - model(**split_data_2) - - QUANT_CFG_CHOICES = { - "int8": "INT8_DEFAULT_CFG", - "int8_sq": "INT8_SMOOTHQUANT_CFG", - "fp8": "FP8_DEFAULT_CFG", - "fp8_pc_pt": "FP8_PER_CHANNEL_PER_TOKEN_CFG", - "int4_awq": "INT4_AWQ_CFG", - "w4a8_awq": "W4A8_AWQ_BETA_CFG", - } - - logger.info("Starting quantization...") - start_time = time.time() - if auto_quantize_bits: - logger.info("Starting mixed precision quantization...") - - from packaging import version as v - opt_kwargs = {} - modelopt_version = version('nvidia-modelopt') - if v.parse(modelopt_version) > v.parse("0.21"): - opt_kwargs['disabled_layers'] = ["*lm_head*"] - - model, search_history = mtq.auto_quantize( - model, - data_loader=calib_dataloader, - loss_func=lambda output, batch: output.loss, - constraints={"effective_bits": auto_quantize_bits}, - forward_step=lambda model, batch: model(**batch), - quantization_formats=[ - QUANT_CFG_CHOICES[item] for item in qformat.split(",") - ] + [None], - num_calib_steps=len(calib_dataloader), - num_score_steps=min( - len(calib_dataloader), 128 // batch_size - ), # Limit the number of score steps to avoid long calibration time - verbose=True, - **opt_kwargs) - mtq.print_quant_summary(model) - - # We need to explicitly calibrate for kv cache quantization - enable_kv_cache_quantization = "int8" not in qformat - if enable_kv_cache_quantization: - mtq.set_quantizer_by_cfg( - model, - quant_cfg={ - "*output_quantizer": { - "num_bits": (4, 3), - "axis": None, - "enable": True - } - }, - ) - # Lets calibrate only the output quantizer this time. Let's disable all other quantizers. - with mtq.set_quantizer_by_cfg_context(model, { - "*": { - "enable": False - }, - "*output_quantizer": { - "enable": True - } - }): - mtq.calibrate(model, - algorithm="max", - forward_loop=calibrate_loop) - else: - mtq.quantize(model, quant_cfg, forward_loop=calibrate_loop) - end_time = time.time() - logger.info( - "Quantization done. Total time used: {:.2f} s.".format(end_time - - start_time)) - return model - - -def quantize_and_export(*, - model_dir, - device, - calib_dataset, - dtype, - qformat, - kv_cache_dtype, - calib_size, - batch_size, - calib_max_seq_length, - awq_block_size, - output_dir, - tp_size, - pp_size, - cp_size, - seed, - tokenizer_max_seq_length, - num_medusa_heads=None, - num_medusa_layers=None, - max_draft_len=None, - medusa_hidden_act=None, - medusa_model_dir=None, - quant_medusa_head=None, - auto_quantize_bits=None, - device_map="auto", - quantize_lm_head=False): - ''' - Load model from the model_dir, call Modelopt to quantize the model, and then export - the quantized model as TRT-LLM checkpoint - ''' - try: - import modelopt # noqa - except ImportError as e: - logger.error( - "Failed to import modelopt, pls check the Modelopt installation. Currently it is known to be unsupported on Windows OS" - ) - raise e - - import modelopt.torch.quantization as mtq - from modelopt.torch.export import export_tensorrt_llm_checkpoint - - from tensorrt_llm.models.convert_utils import infer_dtype - - if not torch.cuda.is_available(): - raise EnvironmentError("GPU is required for inference.") - - random.seed(seed) - np.random.seed(seed) - - # Check that only one quantization format is provided for non auto_quant case - if not auto_quantize_bits: - assert (len(qformat.split(",")) == 1 - ), "Quantization supports only one quantization format." - - hf_config = get_hf_config(model_dir) - dtype = infer_dtype(dtype, getattr(hf_config, 'torch_dtype', None)) - - model = get_model(model_dir, dtype, device=device, device_map=device_map) - model_type = get_model_type(model) - is_enc_dec = model_type_is_enc_dec(model_type) - if "vila" in model_dir: - tokenizer = get_tokenizer(model_dir + "/llm", - max_seq_length=tokenizer_max_seq_length, - model_type=model_type) - elif model_type == "mllama": - tokenizer = get_processor(model_dir, - max_seq_length=tokenizer_max_seq_length, - model_type=model_type, - device=device) - else: - tokenizer = get_tokenizer(model_dir, - max_seq_length=tokenizer_max_seq_length, - model_type=model_type) - - if qformat in ["full_prec", "int8_wo", "int4_wo" - ] and kv_cache_dtype is None: - logger.info(f"No quantization applied, export {dtype} model") - else: - if "awq" in qformat: - if calib_size > 32: - logger.info( - f"AWQ calibration could take longer with calib_size = {calib_size}, Using" - " calib_size=32 instead") - calib_size = 32 - logger.info( - "\nAWQ calibration could take longer than other calibration methods. Please" - " increase the batch size to speed up the calibration process. Batch size can be" - " set by adding the argument --batch_size to the command line.\n" - ) - - quant_cfg = None - if not auto_quantize_bits: - if qformat in quant_cfg_choices(): - quant_cfg = quant_cfg_choices()[qformat] - else: - raise ValueError(f"Unsupported quantization format: {qformat}") - - if "awq" in qformat: - quant_cfg = copy.deepcopy(quant_cfg_choices()[qformat]) - weight_quantizer = quant_cfg["quant_cfg"]["*weight_quantizer"] - if isinstance(weight_quantizer, list): - weight_quantizer = weight_quantizer[0] - if awq_block_size: - weight_quantizer["block_sizes"][-1] = awq_block_size - - # Coarser optimal scale search seems to resolve the overflow in TRT-LLM for some models - if "w4a8_awq" == qformat and model_type in ["gemma", "mpt"]: - quant_cfg["algorithm"] = { - "method": "awq_lite", - "alpha_step": 1 - } - - if kv_cache_dtype is not None: - if kv_cache_dtype == "fp8": - kv_cache_quant_cfg = getattr( - mtq, KV_QUANT_CFG_CHOICES[kv_cache_dtype])["quant_cfg"] - quant_cfg["quant_cfg"].update(kv_cache_quant_cfg) - else: - quant_cfg["quant_cfg"].update(KV_CACHE_CFG) # type: ignore - - # Gemma 7B has accuracy regression using alpha 1. We set 0.5 instead. - if model_type == "gemma" and "int8_sq" in qformat: - quant_cfg["algorithm"] = {"method": "smoothquant", "alpha": 0.5} - - if qformat == 'fp8' and quantize_lm_head: - print_rank_0("Quantizing lm_head layer") - del quant_cfg["quant_cfg"]["*lm_head*"] - - calib_dataloader = get_calib_dataloader( - dataset_name_or_dir=calib_dataset, - tokenizer=tokenizer, - batch_size=batch_size, - calib_size=calib_size, - block_size=calib_max_seq_length, - device=model.device, - include_labels=auto_quantize_bits is not None, - ) - - model = quantize_model(model, quant_cfg, calib_dataloader, batch_size, - qformat, auto_quantize_bits) - - with torch.inference_mode(): - if model_type is None: - logger.info( - f"Unknown model type {type(model).__name__}. Continue exporting..." - ) - model_type = f"unknown:{type(model).__name__}" - - architecture = type(model).__name__ - - export_path = output_dir - start_time = time.time() - - # Move meta tensor back to device before exporting. - remove_hook_from_module(model, recurse=True) - - QUANT_ALGO = { - "int8": "INT8", - "int8_sq": "W8A8_SQ_PER_CHANNEL", - "fp8": "FP8", - "int4_awq": "W4A16_AWQ", - "w4a8_awq": "W4A8_AWQ", - } - - if model_type == 'mllama': - model = model.language_model - - export_tensorrt_llm_checkpoint( - model.hf_model if is_enc_dec else model, - model_type, - getattr(torch, dtype), - export_dir=export_path, - inference_tensor_parallel=tp_size, - inference_pipeline_parallel=pp_size, - ) - - export_paths = [] - tensorrt_llm_configs = [] - if not is_enc_dec: - with open(f"{export_path}/config.json", "r") as f: - tensorrt_llm_config = json.load(f) - tensorrt_llm_configs.append(tensorrt_llm_config) - export_paths.append(export_path) - else: - for component in ["encoder", "decoder"]: - with open(f"{export_path}/{component}/config.json", "r") as f: - tensorrt_llm_config = json.load(f) - tensorrt_llm_configs.append(tensorrt_llm_config) - export_paths.append(f"{export_path}/{component}") - - for export_path, tensorrt_llm_config in zip(export_paths, - tensorrt_llm_configs): - - tensorrt_llm_config["model_type"] = model_type - if not is_enc_dec: - tensorrt_llm_config["architecture"] = architecture - - # Workaround for wo quantization - if qformat in ["int8_wo", "int4_wo", "full_prec"]: - if qformat == "int8_wo": - tensorrt_llm_config["quantization"][ - "quant_algo"] = QuantAlgo.W8A16 - elif qformat == "int4_wo": - tensorrt_llm_config["quantization"][ - "quant_algo"] = QuantAlgo.W4A16 - else: - tensorrt_llm_config["quantization"]["quant_algo"] = None - - # HF uses rope_scaling while tensorrt_llm uses rotary_scaling - if hasattr(model.config, "rope_scaling" - ) and "rotary_scaling" not in tensorrt_llm_config: - tensorrt_llm_config["rotary_scaling"] = getattr( - model.config, "rope_scaling") - with open(f"{export_path}/config.json", "w") as f: - json.dump(tensorrt_llm_config, f, indent=4) - - # Workaround for Modelopt 0.9.x fp8_kv_cache knob issue - if qformat in ['fp8', 'nvfp4'] and kv_cache_dtype is None: - with open(f"{export_path}/config.json", "r") as f: - tensorrt_llm_config = json.load(f) - tensorrt_llm_config["quantization"][ - "kv_cache_quant_algo"] = None - with open(f"{export_path}/config.json", "w") as f: - json.dump(tensorrt_llm_config, f, indent=4) - - # Workaround for qwen version - if model_type == 'qwen' or model_type == 'qwen2_vl': - with open(f"{export_path}/config.json", "r") as f: - tensorrt_llm_config = json.load(f) - qwen_config = AutoConfig.from_pretrained(model_dir, - trust_remote_code=True) - try: - from transformers import LlavaOnevisionConfig - if isinstance(qwen_config, LlavaOnevisionConfig): - qwen_config = qwen_config.text_config - except: - pass - tensorrt_llm_config["qwen_type"] = qwen_config.model_type - if qwen_config.model_type == "qwen2": - tensorrt_llm_config[ - "norm_epsilon"] = qwen_config.rms_norm_eps - tensorrt_llm_config["rotary_base"] = get_hf_rope_theta( - qwen_config, 100000.0) - tensorrt_llm_config[ - "intermediate_size"] = qwen_config.intermediate_size - with open(f"{export_path}/config.json", "w") as f: - json.dump(tensorrt_llm_config, f, indent=4) - - # Set rotary parameters correctly for chatglm. - if model_type == 'chatglm': - rotary_base = 10000.0 - rotary_embedding_scaling = None - chatglm_config = AutoConfig.from_pretrained( - model_dir, trust_remote_code=True) - chatglm_version = tensorrt_llm_config['chatglm_version'] - rope_ratio = tensorrt_llm_config.get('rope_ratio', 1.0) - if chatglm_version == 'chatglm2': - if rope_ratio > 1: - rotary_embedding_scaling = { - 'type': 'linear', - 'factor': rope_ratio - } - elif chatglm_version == 'chatglm3': - rotary_base *= rope_ratio - - with open(f"{export_path}/config.json", "r") as f: - tensorrt_llm_config = json.load(f) - tensorrt_llm_config['rotary_base'] = rotary_base - tensorrt_llm_config['rotary_scaling'] = rotary_embedding_scaling - tensorrt_llm_config['rotary_pct'] = 0.5 - with open(f"{export_path}/config.json", "w") as f: - json.dump(tensorrt_llm_config, f, indent=4) - - # context parallel - if cp_size > 1: - with open(f"{export_path}/config.json", "r") as f: - tensorrt_llm_config = json.load(f) - tensorrt_llm_config["mapping"]["cp_size"] = cp_size - tensorrt_llm_config["mapping"]["attn_tp_size"] = -1 - tensorrt_llm_config["mapping"]["attn_cp_size"] = -1 - tensorrt_llm_config["mapping"]["world_size"] *= cp_size - with open(f"{export_path}/config.json", "w") as f: - json.dump(tensorrt_llm_config, f, indent=4) - - if model_type == 'gptnext': - with open(f"{export_path}/config.json", "r") as f: - tensorrt_llm_config = json.load(f) - if tensorrt_llm_config['max_position_embeddings'] is None: - tensorrt_llm_config['max_position_embeddings'] = getattr( - model.config, "n_positions", None) - with open(f"{export_path}/config.json", "w") as f: - json.dump(tensorrt_llm_config, f, indent=4) - - end_time = time.time() - logger.info( - "Quantized model exported to {} \nTotal time used {:.2f} s.".format( - export_path, end_time - start_time)) - - # Need to delete the model and release memory explicitly; - # otherwise torch may retain its GPU memory until a delayed GC running, - # which reduces the available GPU memory for subsequent stages. - del model - release_gc() - - -def unwrap_model(model, module_instances=None): - # Reference: https://github.com/NVIDIA/Megatron-LM/blob/core_r0.8.0/megatron/training/utils.py - from megatron.core import DistributedDataParallel as DDP - from megatron.core.transformer.module import Float16Module - - if module_instances is None: - module_instances = (DDP, Float16Module) - - return_list = True - if not isinstance(model, list): - model = [model] - return_list = False - unwrapped_model = [] - for model_module in model: - while isinstance(model_module, module_instances): - model_module = model_module.module - unwrapped_model.append(model_module) - if not return_list: - return unwrapped_model[0] - return unwrapped_model - - -def get_nemo_calib_dataloader(dataset_name_or_dir="cnn_dailymail", - batch_size=64, - calib_size=512, - max_sequence_length=512): - if dataset_name_or_dir == "pileval": - dataset = load_dataset( - "json", - data_files="https://the-eye.eu/public/AI/pile/val.jsonl.zst", - split="train", - trust_remote_code=True) - text_column = "text" - elif "wikitext" in dataset_name_or_dir: - dataset = load_dataset(dataset_name_or_dir, - "wikitext-103-v1", - split="train", - trust_remote_code=True) - text_column = "text" - elif "cnn_dailymail" in dataset_name_or_dir or _is_cnn_dailymail_local_repo( - dataset_name_or_dir): - # Bare "cnn_dailymail" id is rejected by newer huggingface_hub; use the namespaced repo. - if dataset_name_or_dir == "cnn_dailymail": - dataset_name_or_dir = "abisee/cnn_dailymail" - dataset = load_dataset(dataset_name_or_dir, - name="3.0.0", - split="train", - trust_remote_code=True) - text_column = "article" - elif os.path.isdir(dataset_name_or_dir): - logger.info( - f"Recognized local dataset repo {dataset_name_or_dir} for calibration; " - "assuming the calibration data are in the train split and text column." - ) - dataset = load_dataset(dataset_name_or_dir, - split="train", - trust_remote_code=True) - text_column = "text" - else: - raise NotImplementedError( - f"Unsupported dataset name or local repo directory: {dataset_name_or_dir}." - ) - calib_size = max(min(len(dataset), calib_size), batch_size) - for i in range(calib_size // batch_size): - batch = dataset[i * batch_size:(i + 1) * batch_size][text_column] - for j in range(len(batch)): - batch[j] = batch[j][:max_sequence_length] - yield batch - - -def quantize_nemo_and_export(*, nemo_ckpt_path, decoder_type, calib_dataset, - calib_tp_size, calib_pp_size, dtype, qformat, - kv_cache_dtype, calib_size, batch_size, - calib_max_seq_length, awq_block_size, output_dir, - tp_size, pp_size, cp_size, seed): - try: - import modelopt # noqa - except ImportError as e: - logger.error( - "Failed to import modelopt, pls check the modelopt installation. Currently it is known to be unsupported on Windows OS" - ) - raise e - - import modelopt.torch.quantization as mtq - from megatron.core import parallel_state - from megatron.core.transformer.module import Float16Module - from modelopt.torch.export import export_tensorrt_llm_checkpoint - from nemo.collections.nlp.models.language_modeling.megatron_gpt_model import \ - MegatronGPTModel - from nemo.collections.nlp.modules.common.text_generation_strategy import \ - GPTModelTextGenerationStrategy - from nemo.collections.nlp.parts.nlp_overrides import ( - NLPDDPStrategy, NLPSaveRestoreConnector) - from nemo.utils.model_utils import load_config, save_artifacts - from omegaconf.omegaconf import open_dict - from pytorch_lightning.trainer.trainer import Trainer - - if not torch.cuda.is_available(): - raise EnvironmentError("GPU is required for the inference.") - - random.seed(seed) - np.random.seed(seed) - - model_cfg = load_config(nemo_ckpt_path) - - # dtype is used for non-quantized layers - supported_dtype = ["auto", "float16", "bfloat16"] - assert dtype in supported_dtype, f"{dtype} not supported. Supported dtypes are {supported_dtype}" - - if dtype == 'auto': - dtype = model_cfg.get('precision', None) - if dtype is None: - dtype = 'float16' - elif 'bf16' in dtype or 'bfloat16' in dtype: - dtype = 'bfloat16' - else: - dtype = 'float16' - logger.info(f"Specified dtype 'auto'; inferred dtype {dtype!r}.") - torch_dtype = getattr(torch, dtype) - - with open_dict(model_cfg): - model_cfg.activations_checkpoint_method = None - model_cfg.activations_checkpoint_granularity = None - model_cfg.tensor_model_parallel_size = calib_tp_size - model_cfg.pipeline_model_parallel_size = calib_pp_size - model_cfg.sequence_parallel = False - # Only custom modelopt spec is supported for PTQ: this custom spec is largely based on local Megatron-LM - # layer definitions to avoid Transformer Engine implementations that are currently not supported. - model_cfg.name = "modelopt" - - # trainer required for restoring model parallel models - trainer_config = { - 'devices': calib_tp_size * calib_pp_size, - 'num_nodes': 1, - 'accelerator': 'gpu', - 'logger': False, - 'precision': model_cfg.precision, - 'enable_checkpointing': False, - } - trainer = Trainer(strategy=NLPDDPStrategy(), **trainer_config) - connector = NLPSaveRestoreConnector() - - model = MegatronGPTModel.restore_from( - restore_path=nemo_ckpt_path, - trainer=trainer, - override_config_path=model_cfg, - save_restore_connector=connector, - ) - model.freeze() - - print_rank_0(model) - # Have to turn off activations_checkpoint_method for inference - try: - model.model.module.language_model.encoder.activations_checkpoint_method = None - except AttributeError: - pass - - # Check whether the DDP is initialized - if parallel_state.is_unitialized(): - - def dummy(): - return - - if model.trainer.strategy.launcher is not None: - model.trainer.strategy.launcher.launch(dummy, trainer=model.trainer) - model.trainer.strategy.setup_environment() - - inference_config = { - 'greedy': False, - 'top_k': 0, - 'top_p': 0.9, - 'temperature': 1.0, - 'add_BOS': True, - 'tokens_to_generate': 30, - 'all_probs': False, - 'repetition_penalty': 1.2, - 'min_tokens_to_generate': 0, - 'compute_logprob': False, - 'batch_size': batch_size, - 'max_context_length': calib_max_seq_length, - 'strategy': GPTModelTextGenerationStrategy(model), - } - model.set_inference_config(inference_config) - - if qformat in ["full_prec", "int8_wo", "int4_wo" - ] and kv_cache_dtype is None: - print_rank_0(f"No quantization applied, export {dtype} model") - else: - if "awq" in qformat: - if calib_size > 32: - print_rank_0( - "AWQ calibration could take longer with calib_size =" - f" {calib_size}, Using calib_size=32 instead") - calib_size = 32 - print_rank_0( - "\nAWQ calibration could take longer than other calibration methods. Please" - " increase the batch size to speed up the calibration process. Batch size can be" - " set by adding the argument inference.batch_size= to the command" - " line.\n") - - dataloader = get_nemo_calib_dataloader( - dataset_name_or_dir=calib_dataset, - batch_size=batch_size, - calib_size=calib_size, - max_sequence_length=calib_max_seq_length, - ) - - # =================== Start Quantization ==================== - if qformat in quant_cfg_choices(): - quant_cfg = quant_cfg_choices()[qformat] - else: - raise ValueError(f"Unsupported quantization format: {qformat}") - - if "awq" in qformat: - quant_cfg = copy.deepcopy(quant_cfg_choices()[qformat]) - weight_quantizer = quant_cfg["quant_cfg"][ - "*weight_quantizer"] # type: ignore - if isinstance(weight_quantizer, list): - weight_quantizer = weight_quantizer[0] - weight_quantizer["block_sizes"][-1] = awq_block_size - - if kv_cache_dtype is not None: - if kv_cache_dtype == "fp8": - for value in KV_CACHE_CFG.values(): - value.update({"num_bits": (4, 3)}) # type: ignore - quant_cfg["quant_cfg"].update(KV_CACHE_CFG) # type: ignore - - print_rank_0(quant_cfg) - - # Always turn on FP8 kv cache to save memory footprint. - # For int8_sq, we use int8 kv cache. - # TODO: Investigate why enabling FP8 kv cache will cause accuracy regressions for nemotron. - # quant_cfg["quant_cfg"]["*output_quantizer"] = { # type: ignore[index] - # "num_bits": 8 if args.qformat == "int8_sq" else (4, 3), - # "axis": None, - # "enable": args.decoder_type != "gptnext", - # } - - dataloader = [data for data in dataloader] - - def forward_loop(model): - for i, batch in enumerate(dataloader): - print_rank_0(f"Calibrating batch {i}") - model.predict_step(batch, i) - - start_time = time.time() - model = mtq.quantize(model, quant_cfg, - forward_loop) # type: ignore[arg-type] - end_time = time.time() - tot_time = end_time - start_time - tput = calib_size / tot_time - print_rank_0( - f"Quantization done. Total time used {tot_time}s. Throughput {tput} samples/s" - ) - # =================== End Quantization ====================== - - if decoder_type == "gptnext": - # We found squared_relu may have an under-calibration problem. - # Clamp the scaling_factor with a min threshold to avoid under-calibration. - maxbound = 0 - if qformat == "fp8": - maxbound = 448 - elif qformat == "int8_sq": - maxbound = 127 - model = mtq.postprocess_amax( - model, "*input_quantizer", - lambda amax: torch.clamp(amax, min=0.01 * maxbound)) - - if torch.distributed.get_rank() == 0: - mtq.print_quant_summary(model) - - if model_cfg.megatron_amp_O2: - model.model = unwrap_model(model.model, Float16Module) - - start_time = time.time() - export_tensorrt_llm_checkpoint( - model, - decoder_type, - torch_dtype, - export_dir=output_dir, - inference_tensor_parallel=tp_size, - inference_pipeline_parallel=pp_size, - ) - - # context parallel - if cp_size > 1: - with open(f"{export_path}/config.json", "r") as f: - tensorrt_llm_config = json.load(f) - tensorrt_llm_config["mapping"]["cp_size"] = cp_size - tensorrt_llm_config["mapping"]["world_size"] *= cp_size - with open(f"{export_path}/config.json", "w") as f: - json.dump(tensorrt_llm_config, f, indent=4) - - end_time = time.time() - print_rank_0( - f"Model config exported to: {output_dir}. Total time used {end_time - start_time}s" - ) - if torch.distributed.get_rank() == 0: - save_artifacts(model, output_dir, use_abspath=True) - - # Need to delete the model and release memory explicitly; - # otherwise torch may retain its GPU memory until a delayed GC running, - # which reduces the available GPU memory for subsequent stages. - del model - release_gc() diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.py b/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.py index 34399289b9ae..2d5b90468bdf 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.py @@ -13,75 +13,280 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import rawref # noqa: F401 -from ._block_radix_tree import ReuseScope, gen_multimodal_cache_key_tokens # noqa: F401 -from ._common import ( # noqa: F401 - BAD_PAGE_INDEX, - CACHE_LEVEL1, - GPU_LEVEL, - NDEBUG, - CacheLevel, - CacheTier, - CudaStream, - LayerId, - MemAddress, - PageIndexMode, - PageStatus, - Priority, - SlidingWindowSize, - TokenId, - TokenIdExt, -) -from ._config import ( # noqa: F401 - AttentionLayerConfig, - BatchDesc, - BufferConfig, - CacheTierConfig, - DataRole, - DiskCacheTierConfig, - GpuCacheTierConfig, - HostCacheTierConfig, - KVCacheDesc, - KVCacheManagerConfig, - SsmLayerConfig, - SwaScratchReuseConfig, -) -from ._core import ( # noqa: F401 - DEFAULT_BEAM_INDEX, - AggregatedPageDesc, - BeamIndex, - ExpandedBuffer, - KVCacheManager, - PageIndexConverter, - PoolDesc, - PoolGroupDesc, - PoolGroupPeakBlockStats, - ScratchDesc, - _KVCache, -) -from ._core._kv_cache import _Status as KvCacheStatus # noqa: F401 -from ._event_manager import ( # noqa: F401 - KVCacheCreatedData, - KVCacheEvent, - KVCacheEventDiff, - KVCacheEventManager, - KVCacheRemovedData, - KVCacheStoredBlockData, - KVCacheStoredData, - KVCacheUpdatedData, - UniqueToken, -) -from ._exceptions import CuError, OutOfMemoryError, OutOfPagesError # noqa: F401 -from ._life_cycle_registry import AttnLifeCycle, LayerGroupId, LifeCycleId # noqa: F401 -from ._stats import ( # noqa: F401 - _KV_CACHE_ITERATION_STATS_DELTA_FIELDS, - KVCacheIterationStatsDelta, - KVCacheStatsDelta, -) -from ._storage import BufferId # noqa: F401 -from ._storage._config import CoalescedBuffer, SlotDesc, SlotDescVariant # noqa: F401 -from ._storage._core import PoolGroupIndex, PoolIndex # noqa: F401 -from ._utils import HalfOpenRange, exact_div, typed_range # noqa: F401 +import os +import sys +from importlib.util import find_spec +from pathlib import Path +from types import ModuleType +from typing import NamedTuple, Optional, Union + +_BACKEND = os.environ.get("TLLM_KV_CACHE_MANAGER_V2_BACKEND", "cpp").lower() + +if _BACKEND == "python": + from . import rawref # noqa: F401 + from ._block_radix_tree import ReuseScope # noqa: F401 + from ._cache_key import ( # noqa: F401 + gen_multimodal_cache_key_tokens, + sequence_to_blockchain_keys, + ) + from ._common import ( # noqa: F401 + BAD_PAGE_INDEX, + CACHE_LEVEL1, + GPU_LEVEL, + NDEBUG, + CacheLevel, + CacheTier, + CudaStream, + LayerId, + MemAddress, + PageIndexMode, + PageStatus, + Priority, + SlidingWindowSize, + TokenId, + TokenIdExt, + ) + from ._config import ( # noqa: F401 + AttentionLayerConfig, + BatchDesc, + BufferConfig, + CacheTierConfig, + DataRole, + DiskCacheTierConfig, + GpuCacheTierConfig, + HostCacheTierConfig, + KVCacheDesc, + KVCacheManagerConfig, + SsmLayerConfig, + SwaScratchReuseConfig, + ) + from ._core import ( # noqa: F401 + DEFAULT_BEAM_INDEX, + AggregatedPageDesc, + BeamIndex, + ExpandedBuffer, + KVCacheManager, + PageIndexConverter, + PlannedDropHandle, + PoolDesc, + PoolGroupDesc, + PoolGroupPeakBlockStats, + ScratchDesc, + _KVCache, + ) + from ._core._kv_cache import _Status as KvCacheStatus # noqa: F401 + from ._event_manager import ( # noqa: F401 + KVCacheCreatedData, + KVCacheEvent, + KVCacheEventDiff, + KVCacheEventManager, + KVCacheRemovedData, + KVCacheStoredBlockData, + KVCacheStoredData, + KVCacheUpdatedData, + UniqueToken, + ) + from ._exceptions import CuError, OutOfMemoryError, OutOfPagesError # noqa: F401 + from ._life_cycle_registry import AttnLifeCycle, LayerGroupId, LifeCycleId # noqa: F401 + from ._stats import ( # noqa: F401 + _KV_CACHE_ITERATION_STATS_DELTA_FIELDS, + KVCacheIterationStatsDelta, + KVCacheStatsDelta, + SsmSnapshotIterationStatsDelta, + ) + from ._storage import BufferId # noqa: F401 + from ._storage._config import CoalescedBuffer, SlotDesc, SlotDescVariant # noqa: F401 + from ._storage._core import PoolGroupIndex, PoolIndex # noqa: F401 + from ._utils import HalfOpenRange, exact_div, typed_range # noqa: F401 + + _cpp_introspection = None +else: + + class ReuseScope(NamedTuple): + lora_id: int | None = None + salt: int | None = None + + def to_bytes(self) -> bytes: + ret = sum((value is not None) << i for i, value in enumerate(self)).to_bytes( + 1, "little", signed=False + ) + for value in self: + if value is not None: + ret += value.to_bytes(8, "little", signed=False) + return ret + + def _load_cpp_module(): + if "tensorrt_llm" in sys.modules: + from tensorrt_llm.bindings.internal.batch_manager import kv_cache_manager_v2 + + return kv_cache_manager_v2 + + spec = find_spec("kv_cache_manager_v2") + assert spec is not None and spec.origin is not None + trtllm_root = str(Path(spec.origin).parent.parent.parent) + sys.path.insert(0, trtllm_root) + try: + from bindings.internal.batch_manager import kv_cache_manager_v2 + + return kv_cache_manager_v2 + finally: + sys.path.remove(trtllm_root) + + _cpp = _load_cpp_module() + + AggregatedPageDesc = _cpp.AggregatedPageDesc + AttentionLayerConfig = _cpp.AttentionLayerConfig + BatchDesc = _cpp.BatchDesc + # BatchDesc is also consumed via dataclasses.replace(): MambaCacheManager's + # _build_cache_config appends dummy KVCacheDesc slots to each constraint with + # replace(batch, kv_caches=[...]). Like KVCacheManagerConfig below, the C++ + # binding replaces the Python @dataclass, so advertise the dataclass field + # set (replace() is keyed on __dataclass_fields__: reads fields via getattr, + # rebuilds via BatchDesc(**fields)). The binding already has a keyword + # __init__ and readable kv_caches / system_prompt_length fields. + import dataclasses as _dataclasses_bd + + @_dataclasses_bd.dataclass + class _BatchDescFieldSpec: + kv_caches: object = None + system_prompt_length: int = 0 + + BatchDesc.__dataclass_fields__ = _BatchDescFieldSpec.__dataclass_fields__ + del _BatchDescFieldSpec, _dataclasses_bd + BufferConfig = _cpp.BufferConfig + BufferId = _cpp.BufferId + CoalescedBuffer = _cpp.CoalescedBuffer + CacheTier = _cpp.CacheTier + DiskCacheTierConfig = _cpp.DiskCacheTierConfig + GpuCacheTierConfig = _cpp.GpuCacheTierConfig + ExpandedBuffer = _cpp.ExpandedBuffer + HostCacheTierConfig = _cpp.HostCacheTierConfig + KVCacheDesc = _cpp.KVCacheDesc + KVCacheCreatedData = _cpp.KVCacheCreatedData + KVCacheEvent = _cpp.KVCacheEvent + KVCacheEventDiff = _cpp.KVCacheEventDiff + KVCacheEventManager = _cpp.KVCacheEventManager + KVCacheIterationStatsDelta = _cpp.KVCacheIterationStatsDelta + KVCacheManager = _cpp.KVCacheManager + KVCacheManagerConfig = _cpp.KVCacheManagerConfig + # The C++ KVCacheManagerConfig binding replaces the Python @dataclass, but + # callers (the DeepSeek-V4 cache manager's _build_cache_config and our own + # host-tier fallback) use dataclasses.replace() on it. dataclasses.replace() + # is a free function keyed on __dataclass_fields__: it reads each field via + # getattr and rebuilds via cls(**fields). The binding already has a full + # keyword __init__ and readable fields, so we only need to advertise the + # dataclass field set. Field defaults/types are irrelevant here — replace() + # only uses the field names + init flag. The read-only + # enable_swa_scratch_reuse property is intentionally excluded (not a ctor + # field), matching the Python dataclass. + import dataclasses as _dataclasses + + @_dataclasses.dataclass + class _KVCacheManagerConfigFieldSpec: + tokens_per_block: int = 0 + cache_tiers: object = None + layers: object = None + max_util_for_resume: float = 0.97 + enable_partial_reuse: bool = True + constraints: object = None + typical_step: object = None + initial_pool_ratio: object = None + swa_scratch_reuse: object = None + commit_min_snapshot: bool = False + enable_stats: bool = True + + KVCacheManagerConfig.__dataclass_fields__ = _KVCacheManagerConfigFieldSpec.__dataclass_fields__ + del _KVCacheManagerConfigFieldSpec, _dataclasses + KVCacheRemovedData = _cpp.KVCacheRemovedData + KVCacheStatsDelta = _cpp.KVCacheStatsDelta + KVCacheStoredBlockData = _cpp.KVCacheStoredBlockData + KVCacheStoredData = _cpp.KVCacheStoredData + KVCacheUpdatedData = _cpp.KVCacheUpdatedData + KvCacheStatus = _cpp.KvCacheStatus + OutOfPagesError = _cpp.OutOfPagesError + PageStatus = _cpp.PageStatus + PoolDesc = _cpp.PoolDesc + PoolGroupDesc = _cpp.PoolGroupDesc + PoolGroupPeakBlockStats = _cpp.PoolGroupPeakBlockStats + SlotDesc = _cpp.SlotDesc + SlotDescVariant = _cpp.SlotDescVariant + SsmLayerConfig = _cpp.SsmLayerConfig + _KVCache = _cpp._KVCache + _cpp_introspection = getattr(_cpp, "_introspection", None) + _KV_CACHE_ITERATION_STATS_DELTA_FIELDS = tuple(KVCacheIterationStatsDelta._field_names) + PlannedDropHandle = _cpp.PlannedDropHandle + + # Symbols added on main that are not yet ported to the C++ backend. + # TODO(kvCacheManagerV2-cpp): port these and replace the fallbacks. + AttnLifeCycle = getattr(_cpp, "AttnLifeCycle", None) + CuError = getattr(_cpp, "CuError", RuntimeError) + OutOfMemoryError = getattr(_cpp, "OutOfMemoryError", MemoryError) + PageIndexConverter = getattr(_cpp, "PageIndexConverter", None) + ReuseScope = getattr(_cpp, "ReuseScope", ReuseScope) + ScratchDesc = getattr(_cpp, "ScratchDesc", None) + SsmSnapshotIterationStatsDelta = _cpp.SsmSnapshotIterationStatsDelta + SwaScratchReuseConfig = getattr(_cpp, "SwaScratchReuseConfig", None) + UniqueToken = _cpp.UniqueToken + + BeamIndex = int + CacheLevel = int + CacheTierConfig = Union[GpuCacheTierConfig, HostCacheTierConfig, DiskCacheTierConfig] + CudaStream = int + DataRole = str + HalfOpenRange = getattr(_cpp, "HalfOpenRange", tuple) + LayerGroupId = int + LayerId = int + LifeCycleId = int + MemAddress = int + PoolGroupIndex = int + PoolIndex = int + Priority = int + SlidingWindowSize = Optional[int] + TokenId = int + TokenIdExt = Union[int, bytes] + + BAD_PAGE_INDEX = -1 + DEFAULT_BEAM_INDEX = 0 + GPU_LEVEL = 0 + CACHE_LEVEL1 = 1 + NDEBUG = os.environ.get("TLLM_DEBUG_MODE", "")[0:1] != "1" + + class _RawRef: + def __init__(self, obj=None): + self._obj = obj + + def __call__(self): + return self._obj + + def invalidate(self) -> None: + self._obj = None + + @classmethod + def __class_getitem__(cls, _item): + return cls + + rawref = ModuleType(f"{__name__}.rawref") + rawref.ReferenceType = _RawRef + rawref.ref = _RawRef + rawref.NULL = _RawRef() + sys.modules.setdefault(f"{__name__}.rawref", rawref) + + class PageIndexMode(int): + SHARED = 0 + PER_LAYER = 1 + + from ._cache_key import ( # noqa: F401 + gen_multimodal_cache_key_tokens, + sequence_to_blockchain_keys, + ) + + def exact_div(x: int, y: int) -> int: + assert x % y == 0 + return x // y + + def typed_range(*args: int) -> range: + return range(*args) + __all__ = [ "AggregatedPageDesc", @@ -124,6 +329,7 @@ "NDEBUG", "OutOfPagesError", "PageIndexConverter", + "PlannedDropHandle", "PoolGroupPeakBlockStats", "PageIndexMode", "PageStatus", @@ -136,6 +342,7 @@ "ScratchDesc", "KVCacheIterationStatsDelta", "KVCacheStatsDelta", + "SsmSnapshotIterationStatsDelta", "SlidingWindowSize", "SlotDesc", "SlotDescVariant", @@ -150,6 +357,7 @@ "_KVCache", "exact_div", "gen_multimodal_cache_key_tokens", + "sequence_to_blockchain_keys", "rawref", "typed_range", ] diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.pyi b/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.pyi index 61ddc4bdbb4f..506290e52586 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.pyi +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/__init__.pyi @@ -51,6 +51,9 @@ CacheLevel = NewType("CacheLevel", int) TokenId = NewType("TokenId", int) TokenIdExt = Union[TokenId, bytes] +class PlannedDropHandle: + def drop(self) -> None: ... + class ReuseScope(NamedTuple): lora_id: int | None = None salt: int | None = None @@ -90,6 +93,18 @@ class KVCacheIterationStatsDelta: iter_host_dropped_blocks: int = 0 iter_host_dropped_bytes: int = 0 +@dataclass(slots=True) +class SsmSnapshotIterationStatsDelta: + iter_snapshot_lookups: int = 0 + iter_snapshot_hits: int = 0 + iter_snapshot_misses: int = 0 + iter_reused_tokens: int = 0 + iter_unreused_tokens: int = 0 + iter_aligned_snapshot_hits: int = 0 + iter_unaligned_snapshot_hits: int = 0 + @property + def iter_snapshot_hit_rate(self) -> float: ... + @dataclass(slots=True, frozen=True) class PoolGroupPeakBlockStats: available: int @@ -272,13 +287,18 @@ class KVCacheEventManager: def flush_iteration_events(self) -> None: ... def get_latest_events(self, timeout_ms: float | None = None) -> list[KVCacheEvent]: ... -# From _block_radix_tree.py +# From _cache_key.py def gen_multimodal_cache_key_tokens( id_offset: int, multi_modal_data_digest: bytes, num_tokens: int, token_offset: int = 0, ) -> list[TokenIdExt]: ... +def sequence_to_blockchain_keys( + tokens_per_block: int, + reuse_scope: ReuseScope, + tokens: Sequence[TokenIdExt], +) -> Iterator[tuple[list[TokenIdExt], bytes]]: ... # From _core/_kv_cache.py class _Status(enum.Enum): @@ -349,6 +369,7 @@ class _KVCache: def committed_tokens(self) -> list[TokenIdExt]: ... @property def reuse_scope(self) -> ReuseScope: ... + def plan_committed_block_drop(self) -> PlannedDropHandle | None: ... def stop_committing(self) -> None: ... def suspend(self) -> None: ... def resume(self, cuda_stream: CudaStream | None = None) -> bool: ... @@ -481,6 +502,9 @@ class KVCacheManager: def get_quota(self, cache_level: CacheLevel) -> int: ... def get_committed_stats(self) -> KVCacheStatsDelta: ... def get_and_reset_iteration_stats(self) -> dict[LifeCycleId, KVCacheIterationStatsDelta]: ... + def get_and_reset_ssm_snapshot_iteration_stats( + self, + ) -> dict[LifeCycleId, SsmSnapshotIterationStatsDelta]: ... def get_and_reset_iteration_peak_block_stats( self, cache_level: CacheLevel ) -> Sequence[PoolGroupPeakBlockStats]: ... diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_block_radix_tree.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_block_radix_tree.py index 38798e17b03a..c47c591f0d06 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_block_radix_tree.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_block_radix_tree.py @@ -13,17 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib -from array import array from typing import TYPE_CHECKING, Iterable, Iterator, NamedTuple, Sequence, TypeVar, cast from . import rawref -from ._common import NDEBUG, BlockOrdinal, PageStatus, TokenId, TokenIdExt +from ._cache_key import ( # noqa: F401 + BlockKey, + Hasher, + TokenBlock, + gen_multimodal_cache_key_tokens, + reuse_scope_to_bytes, + sequence_to_blockchain_keys, +) +from ._common import NDEBUG, BlockOrdinal, PageStatus, TokenIdExt from ._life_cycle_registry import AttnLifeCycle, LifeCycle, LifeCycleId, LifeCycleRegistry from ._utils import ( TypedIndexList, - chunked, - div_up, expect_type, filled_list, find_index, @@ -36,8 +40,6 @@ from ._event_manager import KVCacheEventManager from ._page import CommittedPage -BlockKey = bytes - class ReuseScope(NamedTuple): """Per-request namespace for prefix reuse.""" @@ -45,21 +47,8 @@ class ReuseScope(NamedTuple): lora_id: int | None = None salt: int | None = None - def _mask(self) -> bytes: - return sum((value is not None) << i for i, value in enumerate(self)).to_bytes( - div_up(len(self), 8), "little", signed=False - ) - def to_bytes(self) -> bytes: - ret = self._mask() - for value in self: - if type(value) is int: - ret += value.to_bytes(8, "little", signed=False) - else: - assert value is None, ( - "Did you forget to update to_bytes() when adding new non-int fields to ReuseScope?" - ) - return ret + return reuse_scope_to_bytes(self) class ReuseMatch(NamedTuple): @@ -67,74 +56,7 @@ class ReuseMatch(NamedTuple): blocks: list["Block"] num_tokens: int - - -# id_offset is usually vocab_size -def gen_multimodal_cache_key_tokens( - id_offset: int, multi_modal_data_digest: bytes, num_tokens: int, token_offset: int = 0 -) -> list[TokenIdExt]: - """Create synthetic tokens used only when building multimodal KV-cache keys. - - Item-local token 0 carries the content digest; later offsets use deterministic IDs above the vocab. - """ - assert num_tokens > 0 - assert token_offset >= 0 - return [ - multi_modal_data_digest if token_offset + i == 0 else TokenId(id_offset + token_offset + i) - for i in range(num_tokens) - ] - - -class Hasher: - __slots__ = "_hasher" - _hasher: "hashlib._Hash" - - def __init__(self, data: int | bytes | None | Sequence[int | bytes] = None) -> None: - self._hasher = hashlib.sha256() - if data is not None: - self.update(data) - - # This function is perf-critical. Expect compromised code quality. - def update(self, data: int | bytes | Sequence[int | bytes]) -> "Hasher": - if type(data) is int: - assert NDEBUG or (data >= 0 and data < (1 << 64)) - self._hasher.update(data.to_bytes(8, "little")) - elif type(data) is bytes: - self._hasher.update(data) - else: - # Hash the whole token block in one C call instead of one per token. - # array("Q", data).tobytes() packs each int as 8 native-endian bytes; - # all NVIDIA GPU host platforms (x86_64, aarch64/Grace) are little-endian - # so this is byte-identical to the per-token to_bytes(8, "little") loop. - # Falls back to that loop for multimodal blocks (which contain bytes items). - try: - self._hasher.update(array("Q", data).tobytes()) # type: ignore - except (TypeError, OverflowError): - for item in data: # type: ignore - assert ( - NDEBUG - or (type(item) is int and (0 <= item < (1 << 64))) - or type(item) is bytes - ) - self._hasher.update(item.to_bytes(8, "little") if (type(item) is int) else item) # type: ignore - return self - - @property - def digest(self) -> bytes: - return self._hasher.digest() - - -TokenBlock = list[TokenIdExt] - - -def sequence_to_blockchain_keys( - tokens_per_block: int, reuse_scope: ReuseScope, tokens: Sequence[TokenIdExt] -) -> Iterator[tuple[TokenBlock, BlockKey]]: - digest = Hasher(reuse_scope.to_bytes()).digest - yield [], digest - for token_block in chunked(tokens, tokens_per_block): - digest = Hasher(digest).update(token_block).digest - yield token_block, digest + num_lookup_tokens: int Child = TypeVar("Child", bound="Block | RootBlock") @@ -361,6 +283,10 @@ def __init__(self, tokens: Sequence[TokenIdExt], prev: "Block | RootBlock") -> N assert NDEBUG or (not b.is_full and b is not self and b.key == k and not b.next) to_remove.append(k) event_manager = get_tree(prev).event_manager if to_remove else None + # Keep RootBlock attached while covered children are replaced. Adding + # the replacement first prevents detach_next() from pruning an + # otherwise-empty root before this block becomes its new child. + prev.next[self.key] = self for k in to_remove: b = detach_next(prev, k) assert isinstance(b, Block) @@ -368,7 +294,6 @@ def __init__(self, tokens: Sequence[TokenIdExt], prev: "Block | RootBlock") -> N event_manager.add_removed_event(b.key) assert b.is_orphan # _KVCache may still hold it. # prev.next keeps a strong ref to this _Block, so no need to remove self from prev.next in __del__(). - prev.next[self.key] = self def _release_pages(self) -> None: """Reclaim every page held by this block. @@ -413,6 +338,9 @@ def num_life_cycles(self) -> LifeCycleId: def prev(self) -> "Block | RootBlock": return unwrap_rawref(self._prev) + def get_page(self, lc_idx: LifeCycleId) -> "CommittedPage | None": + return map_optional(self.storage[lc_idx], lambda f: f()) + def unlink_page( self, lc_idx: LifeCycleId, expected_page: "CommittedPage | None" = None ) -> bool: @@ -675,7 +603,11 @@ def match( matched = self._prune_match( list(self._match_token_path(reuse_scope, tokens, enable_partial_match)) ) - return ReuseMatch([block for block, _ in matched], self._num_matched_tokens(matched)) + return ReuseMatch( + [block for block, _ in matched], + self._num_matched_tokens(matched), + len(tokens), + ) def _check_sanity(self) -> bool: raise NotImplementedError( diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_cache_key.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_cache_key.py new file mode 100644 index 000000000000..c418c4a9708a --- /dev/null +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_cache_key.py @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Backend-neutral helpers for deriving KV-cache reuse keys. + +These are the pure-Python reference implementation shared by both the Python +and C++ backends. They depend only on the light-weight, backend-neutral +``_common`` module (no CUDA / bindings), so they can be imported and re-exported +as public API regardless of the active backend. +""" + +import hashlib +import itertools +from array import array +from typing import Iterable, Iterator, Sequence + +from ._common import NDEBUG, TokenId, TokenIdExt + +BlockKey = bytes +TokenBlock = list[TokenIdExt] + + +# id_offset is usually vocab_size +def gen_multimodal_cache_key_tokens( + id_offset: int, multi_modal_data_digest: bytes, num_tokens: int, token_offset: int = 0 +) -> list[TokenIdExt]: + """Create synthetic tokens used only when building multimodal KV-cache keys. + + Item-local token 0 carries the content digest; later offsets use deterministic IDs above the vocab. + """ + assert num_tokens > 0 + assert token_offset >= 0 + return [ + multi_modal_data_digest if token_offset + i == 0 else TokenId(id_offset + token_offset + i) + for i in range(num_tokens) + ] + + +class Hasher: + # SECURITY INVARIANT: the block-key hash MUST stay cryptographically + # collision-resistant and >= 256-bit. The radix tree is a globally shared, + # cross-request/cross-tenant cache index; prefix matches are decided purely by + # digest equality with NO re-check of the underlying tokens; and the hashed + # input (tokens, the user-supplied cache_salt, multimodal content bytes) is + # attacker-influenceable. A collision therefore silently reuses another + # request's KV blocks (cross-request corruption / data leak), and cache_salt + # tenant isolation relies entirely on this hash's collision resistance. Do NOT + # swap in a non-cryptographic hash (xxHash, HighwayHash, ...) or truncate below + # 256 bits without first adding a token-content equality check on match. The + # C++ backend (blockRadixTree) mirrors this with SHA-256 (CSHA256). + __slots__ = "_hasher" + _hasher: "hashlib._Hash" + + def __init__(self, data: int | bytes | None | Sequence[int | bytes] = None) -> None: + self._hasher = hashlib.sha256() + if data is not None: + self.update(data) + + # This function is perf-critical. Expect compromised code quality. + def update(self, data: int | bytes | Sequence[int | bytes]) -> "Hasher": + if type(data) is int: + assert NDEBUG or (data >= 0 and data < (1 << 64)) + self._hasher.update(data.to_bytes(8, "little")) + elif type(data) is bytes: + self._hasher.update(data) + else: + # Hash the whole token block in one C call instead of one per token. + # array("Q", data).tobytes() packs each int as 8 native-endian bytes; + # all NVIDIA GPU host platforms (x86_64, aarch64/Grace) are little-endian + # so this is byte-identical to the per-token to_bytes(8, "little") loop. + # Falls back to that loop for multimodal blocks (which contain bytes items). + try: + self._hasher.update(array("Q", data).tobytes()) # type: ignore + except (TypeError, OverflowError): + for item in data: # type: ignore + assert ( + NDEBUG + or (type(item) is int and (0 <= item < (1 << 64))) + or type(item) is bytes + ) + self._hasher.update(item.to_bytes(8, "little") if (type(item) is int) else item) # type: ignore + return self + + @property + def digest(self) -> bytes: + return self._hasher.digest() + + +def reuse_scope_to_bytes(reuse_scope: Iterable[int | None]) -> bytes: + """Serialize a reuse scope to its reuse-namespace bytes. + + Backend-neutral: reads the scope's fields by iteration, so it works for both + the pure-Python ``ReuseScope`` NamedTuple and the C++ binding without relying + on a ``to_bytes()`` method. The layout mirrors the C++ ``emitReuseScopeBytes``: + a mask byte (one bit per field, set when the field is present) followed by one + little-endian ``uint64`` per present field (``signed=False``). + """ + values = list(reuse_scope) + mask = sum((value is not None) << i for i, value in enumerate(values)) + ret = mask.to_bytes((len(values) + 7) // 8, "little", signed=False) + for value in values: + if value is not None: + ret += int(value).to_bytes(8, "little", signed=False) + return ret + + +def sequence_to_blockchain_keys( + tokens_per_block: int, reuse_scope: Iterable[int | None], tokens: Sequence[TokenIdExt] +) -> Iterator[tuple[TokenBlock, BlockKey]]: + """Yield ``(token_block, key)`` pairs seeding a blockchain of KV-cache keys. + + The first pair is the root (``[]``, reuse-scope digest); each subsequent pair + hashes one ``tokens_per_block`` chunk on top of the previous digest. + """ + digest = Hasher(reuse_scope_to_bytes(reuse_scope)).digest + yield [], digest + iterator = iter(tokens) + while True: + token_block = list(itertools.islice(iterator, tokens_per_block)) + if not token_block: + break + digest = Hasher(digest).update(token_block).digest + yield token_block, digest diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/__init__.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/__init__.py index ea7360d69c83..4e52e154c47b 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/__init__.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. from .._common import DEFAULT_BEAM_INDEX, BeamIndex -from ._kv_cache import _KVCache +from ._kv_cache import PlannedDropHandle, _KVCache from ._kv_cache_manager import ( AggregatedPageDesc, ExpandedBuffer, @@ -29,6 +29,7 @@ __all__ = [ "KVCacheManager", "_KVCache", + "PlannedDropHandle", "BeamIndex", "DEFAULT_BEAM_INDEX", "AggregatedPageDesc", diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py index 63e2244d9b98..da0500dc4c06 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache.py @@ -16,7 +16,7 @@ import array import enum import math -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from contextlib import contextmanager from dataclasses import dataclass from itertools import chain @@ -37,6 +37,7 @@ CudaStream, PageIndex, PageIndexMode, + PageStatus, Priority, TokenIdExt, ) @@ -129,6 +130,58 @@ def __del__(self) -> None: self.pages.clear() +class PlannedDropHandle: + """Track committed pages planned for dropping without owning them. + + The handle stores weak references and does not keep pages alive. Dropping it + decrements each live page's planned-drop count and removes an already-droppable + page from eviction tracking when no plans remain. + """ + + __slots__ = ("_page_refs",) + + _page_refs: tuple[rawref.ref[CommittedPage], ...] | None + + def __init__(self, pages: Iterable[CommittedPage]) -> None: + planned_pages = tuple({id(page): page for page in pages}.values()) + self._page_refs = tuple(rawref.ref(page) for page in planned_pages) + for page in planned_pages: + page.planned_drop_count += 1 + + def drop(self) -> None: + """Apply this drop plan and invalidate the handle. + + A live page is removed from eviction tracking only when this is its final + plan and it is already droppable and queued for eviction. Calling this + method twice is invalid. + """ + page_refs = self._page_refs + if page_refs is None: + raise ValueError("Planned drop handle has already been dropped") + + pages = list[CommittedPage]() + for page_ref in page_refs: + page = page_ref() + if page is not None: + if page.planned_drop_count <= 0: + raise ValueError("Committed page has no planned drop") + pages.append(page) + + self._page_refs = None + for page in pages: + page.planned_drop_count -= 1 + if ( + page.planned_drop_count == 0 + and page.status == PageStatus.DROPPABLE + and page.scheduled_for_eviction + ): + page.manager.exclude_from_eviction(page) + + def __del__(self) -> None: + if self._page_refs is not None: + self.drop() + + class _Status(enum.Enum): ACTIVE = enum.auto() SUSPENDED = enum.auto() @@ -359,6 +412,9 @@ def commit_pending_stats(self) -> KVCacheStatsDelta: self.manager.commit_stats( self._pending_stats.global_stats, self._pending_stats.iteration_stats_by_life_cycle ) + self.manager._commit_ssm_snapshot_iteration_stats( + self._pending_stats.ssm_snapshot_iteration_stats_by_life_cycle + ) request_stats = self._pending_stats.request_stats.copy() self._pending_stats.clear() self.manager.clear_stats_dirty(self.id) @@ -964,7 +1020,12 @@ def commit( ), move_ssm=is_end, ) - if has_partial_snapshot: + # _commit_block transitions out of ALLOWED (to USER_STOP) when a + # block cannot be committed (VIRTUAL_STOP). Stop here so we don't + # re-enter _commit_block on an already-stopped cache. + if self._commit_state != self.CommitState.ALLOWED: + break + if has_partial_snapshot and self._commit_state == self.CommitState.ALLOWED: partial_ordinal = BlockOrdinal(new_num_full_blocks) if is_end: self._commit_block( @@ -993,6 +1054,50 @@ def committed_tokens(self) -> list[TokenIdExt]: def reuse_scope(self) -> ReuseScope: return self._reuse_scope + def plan_committed_block_drop(self) -> PlannedDropHandle | None: + """Plan dropping pages needed only by the next conversation turn. + + The plan covers committed pages in each SWA life cycle's current + attention window and the exact SSM snapshot for the committed prefix. + Full-attention and attention-sink blocks are excluded because later + turns may still need them. This must be called after stop_committing(). + Returns None without creating a plan if any required page is unavailable. + """ + if self._commit_state != self.CommitState.USER_STOP: + raise LogicError("plan_committed_block_drop() requires stop_committing()") + + if self.num_committed_tokens == 0: + return None + + # Locate pages through the radix tree rather than + # SeqBlock.tree_block: the latter is not guaranteed to identify a + # partial snapshot after reuse. Requiring an exact match keeps the + # preceding conversation plan intact if this turn no longer has a + # complete reusable endpoint. All PP ranks use the same lookup so + # attention-only ranks include the final partial SWA block too. + match = self.manager._match_reuse(self.reuse_scope, self._committed_tokens) + if match.num_tokens != self.num_committed_tokens or not match.blocks: + return None + + end = BlockOrdinal(len(match.blocks)) + pages_to_drop: list[CommittedPage] = [] + for lc_idx, lc in self.manager._life_cycles.items(): + if isinstance(lc, AttnLifeCycle): + if lc.window_size is None: + continue + stale_range = _KVCache._get_stale_range( + self.tokens_per_block, self.num_committed_tokens, lc + ) + window_start = min(stale_range.end, end) + else: + window_start = BlockOrdinal(end - 1) + for ordinal in typed_range(window_start, end): + page = match.blocks[ordinal].get_page(lc_idx) + if page is None: + return None + pages_to_drop.append(page) + return PlannedDropHandle(pages_to_drop) + # Users promise to not commit any more tokens. For cases where we shouldn't reuse generated tokens # (eg. CoT), this helps us drop (instead of evict) out-of-window blocks for SWA layers. # If there is a uncommitted block containing committed tokens, we will commit the block immediately. @@ -1969,6 +2074,15 @@ def _setup_for_reuse(self, match: ReuseMatch) -> None: ) snapshot_holder = unwrap_rawref(snapshot_ref).hold() self._ssm_blocks[DEFAULT_BEAM_INDEX][ssm_lc_id] = snapshot_holder + if should_record_stats and ssm_lc_id is not None: + changed = self._pending_stats.record_ssm_snapshot_lookup( + ssm_lc_id, + lookup_tokens=match.num_lookup_tokens, + reused_tokens=num_tokens, + tokens_per_block=tokens_per_block, + ) + if changed: + self.manager.mark_stats_dirty(self.id) self._num_committed_blocks = BlockOrdinal(len(self._committed_tokens) // tokens_per_block) for beam_indices in self._base_page_indices: for indices in beam_indices: diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache_manager.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache_manager.py index af54f32fe4f7..74c73700d3e6 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache_manager.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_kv_cache_manager.py @@ -39,7 +39,7 @@ from .._config import DataRole, KVCacheManagerConfig from .._life_cycle_registry import LayerGroupId, LifeCycle, LifeCycleId, LifeCycleRegistry from .._page import Page, _PageHolder -from .._stats import KVCacheIterationStatsDelta, KVCacheStatsDelta +from .._stats import KVCacheIterationStatsDelta, KVCacheStatsDelta, SsmSnapshotIterationStatsDelta from .._storage._config import BufferId, SlotDesc, create_storage_config from .._storage._core import PoolGroupIndex, PoolIndex, SlotId from .._storage_manager import StorageManager @@ -212,6 +212,7 @@ class KVCacheManager: "_stats_enabled", "_committed_stats", "_iteration_stats_by_life_cycle", + "_ssm_snapshot_iteration_stats_by_life_cycle", "_iteration_peak_num_blocks_by_cache_level", "_dirty_stats_kv_cache_ids", "_stats_excluded_kv_cache_ids", @@ -241,6 +242,7 @@ class KVCacheManager: _stats_enabled: bool _committed_stats: KVCacheStatsDelta _iteration_stats_by_life_cycle: dict[LifeCycleId, KVCacheIterationStatsDelta] + _ssm_snapshot_iteration_stats_by_life_cycle: dict[LifeCycleId, SsmSnapshotIterationStatsDelta] _iteration_peak_num_blocks_by_cache_level: TypedIndexList[ CacheLevel, TypedIndexList[PoolGroupIndex, PoolGroupPeakBlockStats] ] @@ -267,6 +269,7 @@ def __init__( constraints=config.constraints, initial_pool_ratio=config.initial_pool_ratio, event_manager=event_manager, + max_util_for_resume=config.max_util_for_resume, ) self._living_kv_caches = set[rawref.ref[_KVCache]]() decay = 0.9999 @@ -283,6 +286,7 @@ def __init__( self._stats_enabled = config.enable_stats self._committed_stats = KVCacheStatsDelta() self._iteration_stats_by_life_cycle = {} + self._ssm_snapshot_iteration_stats_by_life_cycle = {} self._reset_iteration_peak_num_blocks() self._dirty_stats_kv_cache_ids = set() self._stats_excluded_kv_cache_ids = set() @@ -538,6 +542,31 @@ def get_and_reset_iteration_stats(self) -> dict[LifeCycleId, KVCacheIterationSta self._iteration_stats_by_life_cycle.clear() return stats + def _commit_ssm_snapshot_iteration_stats( + self, + iteration_stats_by_life_cycle: dict[LifeCycleId, SsmSnapshotIterationStatsDelta], + ) -> None: + if not self._stats_enabled: + return + for life_cycle, iteration_stats in iteration_stats_by_life_cycle.items(): + if iteration_stats.empty: + continue + destination = self._ssm_snapshot_iteration_stats_by_life_cycle.setdefault( + life_cycle, SsmSnapshotIterationStatsDelta() + ) + destination.add(iteration_stats) + + def get_and_reset_ssm_snapshot_iteration_stats( + self, + ) -> dict[LifeCycleId, SsmSnapshotIterationStatsDelta]: + stats = { + life_cycle: delta.copy() + for life_cycle, delta in self._ssm_snapshot_iteration_stats_by_life_cycle.items() + if not delta.empty + } + self._ssm_snapshot_iteration_stats_by_life_cycle.clear() + return stats + def get_and_reset_iteration_peak_block_stats( self, cache_level: CacheLevel ) -> TypedIndexList[PoolGroupIndex, PoolGroupPeakBlockStats]: @@ -642,6 +671,11 @@ def layer_grouping(self) -> HomoTuple[HomoTuple[LayerId]]: """ Layers are divided into multiple groups. Buffers in the same layer group for the same token block are always allocated/deallocated together. + + NOTE: the iteration order of the layer lists (and of the groups) is NOT part of + the API contract and may differ across backends/runs. Do not rely on it for + buffer/pool memory order -- query ``pool_group_descs`` (PoolGroupDesc.pools[i] + .base_address + coalesced_buffers) for that. """ layer_to_life_cycle_ids = self._storage._layer_to_life_cycle_ids num_life_cycles = self._life_cycles.size diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_pending_stats.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_pending_stats.py index 94d0957537be..bf8318e1c726 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_pending_stats.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_core/_pending_stats.py @@ -17,7 +17,7 @@ from .._common import BlockOrdinal from .._life_cycle_registry import LifeCycleId -from .._stats import KVCacheIterationStatsDelta, KVCacheStatsDelta +from .._stats import KVCacheIterationStatsDelta, KVCacheStatsDelta, SsmSnapshotIterationStatsDelta @dataclass(slots=True) @@ -49,6 +49,9 @@ class _PendingStats: iteration_stats_by_life_cycle: dict[LifeCycleId, KVCacheIterationStatsDelta] = field( default_factory=dict ) + ssm_snapshot_iteration_stats_by_life_cycle: dict[ + LifeCycleId, SsmSnapshotIterationStatsDelta + ] = field(default_factory=dict) allocation_segments: list[_PendingAllocationSegment] = field(default_factory=list) @property @@ -57,12 +60,14 @@ def empty(self) -> bool: self.request_stats.empty and self.global_stats.empty and not self.iteration_stats_by_life_cycle + and not self.ssm_snapshot_iteration_stats_by_life_cycle ) def clear(self) -> None: self.request_stats.clear() self.global_stats.clear() self.iteration_stats_by_life_cycle.clear() + self.ssm_snapshot_iteration_stats_by_life_cycle.clear() self.allocation_segments.clear() def add(self, delta: _PendingStatsDelta) -> bool: @@ -165,6 +170,39 @@ def record_reuse( ) ) + def record_ssm_snapshot_lookup( + self, + life_cycle: LifeCycleId, + *, + lookup_tokens: int, + reused_tokens: int, + tokens_per_block: int, + ) -> bool: + if lookup_tokens == 0: + return False + assert lookup_tokens > 0 + assert 0 <= reused_tokens <= lookup_tokens + assert tokens_per_block > 0 + + is_hit = reused_tokens > 0 + # Alignment describes the reusable snapshot boundary, not whether the + # state itself is complete. Every hit represents one complete SSM + # snapshot; token counters carry the benefit of that single lookup. + delta = SsmSnapshotIterationStatsDelta( + iter_snapshot_lookups=1, + iter_snapshot_hits=int(is_hit), + iter_snapshot_misses=int(not is_hit), + iter_reused_tokens=reused_tokens, + iter_unreused_tokens=lookup_tokens - reused_tokens, + iter_aligned_snapshot_hits=int(is_hit and reused_tokens % tokens_per_block == 0), + iter_unaligned_snapshot_hits=int(is_hit and reused_tokens % tokens_per_block != 0), + ) + pending = self.ssm_snapshot_iteration_stats_by_life_cycle.setdefault( + life_cycle, SsmSnapshotIterationStatsDelta() + ) + pending.add(delta) + return True + def subtract_allocation_range(self, block_begin: BlockOrdinal, block_end: BlockOrdinal) -> bool: if block_begin >= block_end or not self.allocation_segments: return False diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_introspection.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_introspection.py index 8b063677111d..4a5bdc44394f 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_introspection.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_introspection.py @@ -197,3 +197,127 @@ def ratio_to_slot_count_list( quota, slot_size_lists, ratio_list, granularity, min_slots ) ) + + +def attention_life_cycle_ids(manager: Any) -> list[int]: + """Return the lifecycle ids of all attention lifecycles, in order.""" + cpp_introspection = _cpp_introspection_module() + if cpp_introspection is not None: + return list(cpp_introspection.attention_life_cycle_ids(manager)) + return [lc_id for lc_id, _ in manager._life_cycles.attention_life_cycles()] + + +def swa_life_cycle_ids(manager: Any) -> list[int]: + """Return the lifecycle ids of attention lifecycles that use a sliding window.""" + cpp_introspection = _cpp_introspection_module() + if cpp_introspection is not None: + return list(cpp_introspection.swa_life_cycle_ids(manager)) + return [ + lc_id + for lc_id, lc in manager._life_cycles.attention_life_cycles() + if lc.window_size is not None + ] + + +def ssm_life_cycle_id(manager: Any) -> int | None: + """Return the SSM lifecycle id, or None if there is no SSM lifecycle.""" + cpp_introspection = _cpp_introspection_module() + if cpp_introspection is not None: + return cpp_introspection.ssm_life_cycle_id(manager) + return manager._life_cycles.ssm_life_cycle_id + + +def reuse_match_pages( + manager: Any, + reuse_scope: Any, + tokens: Any, + lc_id: int, + enable_partial: bool = False, +) -> tuple[int, list[tuple[int, int | None] | None]]: + """Match ``tokens`` against the radix tree and report reusable pages per block. + + Returns ``(num_tokens, pages)`` where ``pages[i]`` is ``None`` when block ``i`` + holds no page for lifecycle ``lc_id``, otherwise ``(slot_id, num_tokens_in_block)`` + with ``num_tokens_in_block`` set only for SSM pages (``None`` for attention pages). + """ + cpp_introspection = _cpp_introspection_module() + if cpp_introspection is not None: + num_tokens, raw_pages = cpp_introspection.reuse_match_pages( + manager, reuse_scope, list(tokens), lc_id, enable_partial + ) + pages: list[tuple[int, int | None] | None] = [] + for entry in raw_pages: + if entry is None: + pages.append(None) + else: + slot_id, num_tokens_in_block = entry + pages.append((slot_id, None if num_tokens_in_block < 0 else num_tokens_in_block)) + return num_tokens, pages + + from ._utils import unwrap_rawref + + match = manager._radix_tree.match(reuse_scope, list(tokens), enable_partial) + py_pages: list[tuple[int, int | None] | None] = [] + for block in match.blocks: + ref = block.storage[lc_id] + if ref is None: + py_pages.append(None) + else: + page = unwrap_rawref(ref) + py_pages.append((page.slot_id, getattr(page, "num_tokens_in_block", None))) + return match.num_tokens, py_pages + + +def reuse_match_planned_drop_counts( + manager: Any, + reuse_scope: Any, + tokens: Any, + lc_id: int, + enable_partial: bool = False, +) -> tuple[int, list[int | None]]: + """Match ``tokens`` and report each matched block's ``planned_drop_count`` for lifecycle ``lc_id``. + + Returns ``(num_tokens, counts)`` where ``counts[i]`` is ``None`` when block ``i`` holds no page + for lifecycle ``lc_id``, otherwise the matched page's ``planned_drop_count``. + """ + cpp_introspection = _cpp_introspection_module() + if cpp_introspection is not None: + return cpp_introspection.reuse_match_planned_drop_counts( + manager, reuse_scope, list(tokens), lc_id, enable_partial + ) + + from ._utils import unwrap_rawref + + match = manager._radix_tree.match(reuse_scope, list(tokens), enable_partial) + counts: list[int | None] = [] + for block in match.blocks: + ref = block.storage[lc_id] + counts.append(None if ref is None else unwrap_rawref(ref).planned_drop_count) + return match.num_tokens, counts + + +def pool_group_index(manager: Any, lc_id: int) -> int: + """Return the storage pool-group index for lifecycle ``lc_id``.""" + cpp_introspection = _cpp_introspection_module() + if cpp_introspection is not None: + return cpp_introspection.pool_group_index(manager, lc_id) + return manager._storage.get_pool_group_index(lc_id) + + +def compute_slots_for_batch( + manager: Any, + batch: Any, + tokens_per_block: int, + swa_scratch_reuse: Any = None, +) -> list[int]: + """Return the minimum per-pool-group slot counts to support ``batch``.""" + cpp_introspection = _cpp_introspection_module() + if cpp_introspection is not None: + return list( + cpp_introspection.compute_slots_for_batch( + manager, batch, tokens_per_block, swa_scratch_reuse + ) + ) + return list( + manager._storage._compute_slots_for_batch(batch, tokens_per_block, swa_scratch_reuse) + ) diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_page.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_page.py index 43bac9243622..70a82c090c01 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_page.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_page.py @@ -241,6 +241,7 @@ class CommittedPage(Page): """ block: rawref.ref["Block"] + planned_drop_count: int __rawref__: rawref.ref["CommittedPage"] def is_committed(self) -> bool: @@ -256,6 +257,7 @@ def __init__( priority: Priority, ): self.block = rawref.ref(block) + self.planned_drop_count = 0 self.__rawref__ = rawref.NULL Page.__init__( self, diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_stats.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_stats.py index 292876ff4985..03d95d745df0 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_stats.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_stats.py @@ -78,6 +78,23 @@ def iter_cache_hit_rate(self) -> float: return self.iter_reused_blocks / total +@dataclass(slots=True) +class SsmSnapshotIterationStatsDelta(_StatsDeltaMixin): + iter_snapshot_lookups: int = 0 + iter_snapshot_hits: int = 0 + iter_snapshot_misses: int = 0 + iter_reused_tokens: int = 0 + iter_unreused_tokens: int = 0 + iter_aligned_snapshot_hits: int = 0 + iter_unaligned_snapshot_hits: int = 0 + + @property + def iter_snapshot_hit_rate(self) -> float: + if self.iter_snapshot_hits == 0 or self.iter_snapshot_lookups == 0: + return 0.0 + return self.iter_snapshot_hits / self.iter_snapshot_lookups + + _KV_CACHE_ITERATION_STATS_DELTA_FIELDS = tuple( field.name for field in fields(KVCacheIterationStatsDelta) ) diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_storage_manager.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_storage_manager.py index 765d3bdca068..6d93c6ec3220 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_storage_manager.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_storage_manager.py @@ -219,6 +219,7 @@ def __init__( constraints: list[BatchDesc] | None = None, initial_pool_ratio: list[float] | None = None, event_manager: "KVCacheEventManager | None" = None, + max_util_for_resume: float = 1.0, ) -> None: self.__rawref__ = rawref.NULL self._event_manager = event_manager @@ -243,9 +244,12 @@ def __init__( gpu_quota = config.cache_tiers[GPU_LEVEL].quota gpu_granularity = CacheLevelManager.cache_tier_granularity(CacheTier.GPU_MEM, gpu_quota) - constraints_for_min_slots = [] if initial_pool_ratio is not None else constraints or [] + # Constraints stay feasibility floors even under an explicit initial pool + # ratio (a share below what a declared batch needs is clamped up), and the + # floors are scaled by 1/max_util_for_resume because _KVCache.resume rejects + # any pool group above that utilization. Mirrors PR #16269. self._min_slots = self._compute_min_slots_from_constraints( - constraints_for_min_slots, tokens_per_block, swa_scratch_reuse + constraints or [], tokens_per_block, swa_scratch_reuse, max_util_for_resume ) # Compute init_ratio from explicit config, typical_batch, constraints, or fallback. @@ -907,11 +911,15 @@ def _compute_min_slots_from_constraints( constraints: list[BatchDesc], tokens_per_block: int, swa_scratch_reuse: SwaScratchReuseConfig | None, + max_util_for_resume: float, ) -> TypedIndexList[PoolGroupIndex, int]: """Compute the minimum slots per pool group across all constraints (element-wise max). - All returned elements are positive. + All returned elements are positive. Constraint-derived floors include + headroom for the utilization gate checked by ``_KVCache.resume``. """ + if not 0 < max_util_for_resume <= 1: + raise ValueError(f"max_util_for_resume must be in (0, 1], got {max_util_for_resume}") max_slots = filled_list(0, self.num_pool_groups) def swa_floor_blocks(lc: AttnLifeCycle) -> int: @@ -937,7 +945,8 @@ def swa_floor_blocks(lc: AttnLifeCycle) -> int: for batch in constraints: slots = self._compute_slots_for_batch(batch, tokens_per_block, swa_scratch_reuse) for pg_idx in typed_range(self.num_pool_groups): - max_slots[pg_idx] = max(max_slots[pg_idx], slots[pg_idx]) + scaled_slots = math.ceil(slots[pg_idx] / max_util_for_resume) + max_slots[pg_idx] = max(max_slots[pg_idx], scaled_slots) return max_slots def _compute_slots_for_batch( diff --git a/tensorrt_llm/runtime/kv_cache_manager_v2/_utils.py b/tensorrt_llm/runtime/kv_cache_manager_v2/_utils.py index da750eeca0e4..2f970ca347b7 100644 --- a/tensorrt_llm/runtime/kv_cache_manager_v2/_utils.py +++ b/tensorrt_llm/runtime/kv_cache_manager_v2/_utils.py @@ -673,13 +673,28 @@ def num_set_bits(self) -> int: return self._num_set_bits def resize(self, new_capacity: int) -> None: - extra_elems = div_up(new_capacity, 64) - len(self._bits) - if extra_elems > 0: - self._bits.extend(array.array(self.TYPE_CODE, [0] * extra_elems)) - elif extra_elems < 0: - self._bits = self._bits[:extra_elems] - if new_capacity % 64 != 0: - self._bits[-1] &= self.ALL_SET_MASK >> (64 - (new_capacity % 64)) + old_elems = len(self._bits) + new_elems = div_up(new_capacity, 64) + + # When the capacity shrinks, every set bit at or above new_capacity is + # dropped. Account for those bits so num_set_bits stays accurate, and + # mask the retained partial word so any_set() cannot observe stale bits. + # This covers both fewer-words and same-word-count (new_elems == + # old_elems with a smaller new_capacity) shrinks. + if new_elems <= old_elems: + for w in range(new_elems, old_elems): + self._num_set_bits -= bin(self._bits[w]).count("1") + if new_elems >= 1 and new_capacity % 64 != 0: + keep_mask = self.ALL_SET_MASK >> (64 - (new_capacity % 64)) + word = self._bits[new_elems - 1] + dropped = word & ~keep_mask & self.ALL_SET_MASK + self._num_set_bits -= bin(dropped).count("1") + self._bits[new_elems - 1] = word & keep_mask + + if new_elems > old_elems: + self._bits.extend(array.array(self.TYPE_CODE, [0] * (new_elems - old_elems))) + elif new_elems < old_elems: + self._bits = self._bits[:new_elems] # check if any bit in the range [start, end) is set def any_set(self, start: int, end: int) -> bool: diff --git a/tensorrt_llm/sampling_params.py b/tensorrt_llm/sampling_params.py index 291abceb1c18..5a1c74fa4d1a 100644 --- a/tensorrt_llm/sampling_params.py +++ b/tensorrt_llm/sampling_params.py @@ -25,7 +25,7 @@ from tensorrt_llm.bindings import executor as tllme from tensorrt_llm.logger import logger -MAX_TOP_LOGPROBS = 20 +MAX_TOP_LOGPROBS = 100 def validate_thinking_token_budget(value: Optional[Union[int, float, bool]]) -> Optional[int]: @@ -210,9 +210,9 @@ class SamplingParams: If neither temperature, top_p, nor top_k are specified, sampling is greedy. If temperature > 0 and/or top_k > 1 are specified, sampling will proceed accordingly and top_p will default to top_p = 1. Setting top_p = 0 should result in greedy sampling, but is currently disallowed in the backend. - top_p_min (float, optional): Controls decay in the top-P algorithm. topPMin is lower-bound. None means using C++ runtime default 1.e-6. Defaults to None. - top_p_reset_ids (int, optional): Controls decay in the top-P algorithm. Indicates where to reset the decay. None means using C++ runtime default 1. Defaults to None. - top_p_decay (float, optional): Controls decay in the top-P algorithm. The decay value. None means using C++ runtime default 1.f. Defaults to None. + top_p_min (float, optional): Controls decay in the top-P algorithm. topPMin is lower-bound. Must be in (0, 1]; invalid values are rejected. None means using C++ runtime default 1.e-6. Defaults to None. + top_p_reset_ids (int, optional): Controls decay in the top-P algorithm. The token id which, when sampled, resets the decayed top-P to its initial value. Must be >= 0; invalid values are rejected. None means using C++ runtime default -1 (which never matches a token). Defaults to None. + top_p_decay (float, optional): Controls decay in the top-P algorithm. The decay value. Must be in (0, 1]; invalid values are rejected. None means using C++ runtime default 1.f. Defaults to None. seed (int, optional): Controls the random seed used by the random number generator in sampling. None means using C++ runtime default 0. Defaults to None. temperature (float, optional): Controls the modulation of logits when sampling new tokens. It can have values >= 0.f. Defaults to None. The value None is treated as "not specified" in the following. @@ -227,7 +227,7 @@ class SamplingParams: prompt_ignore_length (int, optional): Controls how many tokens to ignore from the prompt for presence and frequency penalties. Values <= 0 have no effect. Values > input (prompt) length will be clamped. None means using C++ runtime default 0. Defaults to None. length_penalty (float, optional): Controls how to penalize longer sequences in beam search. None means using C++ runtime default 0.f. Defaults to None. early_stopping (int, optional): Controls whether the generation process finishes once beamWidth sentences are generated (ends with end_token). None means using C++ runtime default 1. Defaults to None. - no_repeat_ngram_size (int, optional): Controls how many repeat ngram size are acceptable. None means using C++ runtime default 1 << 30. Defaults to None. + no_repeat_ngram_size (int, optional): Forbids repeating any n-gram of this size: a token is excluded from sampling if it would recreate an n-gram that already occurs in the sequence (prompt included). None or 0 disables the restriction. Defaults to None. min_p (float, optional): scale the most likely token to determine the minimum token probability. None means using C++ runtime default 0.0. Defaults to None. beam_width_array (List[int], optional): The array of beam width using in Variable-Beam-Width-Search. Defaults to None. @@ -318,6 +318,12 @@ class SamplingParams: return_perf_metrics: bool = False additional_model_outputs: Optional[List[str]] = None + # Decoder tokens moved from generated output into the input prefix. The + # result layer restores them to the user-visible output. + _decoder_output_token_prefix: Tuple[int, ...] = field( + default_factory=tuple, init=False, repr=False + ) + # Used in logprobs calculation in TRT flow to drop logits early if user did not explicitly request them. # Can be deprecated after migration to PyTorch backend. _context_logits_auto_enabled: bool = False @@ -370,9 +376,25 @@ def _validate(self): raise ValueError(f"require 0 <= top_p <= 1, got top_p={self.top_p}") if self.top_k is not None and self.top_k < 0: raise ValueError(f"require top_k >= 0, got top_k={self.top_k}") + if self.min_p is not None and (self.min_p < 0 or self.min_p > 1): + raise ValueError(f"require 0 <= min_p <= 1, got min_p={self.min_p}") if self.temperature is not None and self.temperature < 0: raise ValueError(f"require temperature >= 0, got temperature={self.temperature}") + # Top-p decay param ranges mirror the hard checks in the + # executor::SamplingConfig constructor (samplingConfig.cpp check* + # helpers); rejecting here gives a clear, early error instead of a + # RuntimeError from the C++ boundary. Note top_p_min > top_p is + # intentionally allowed (the runtime top-p may rise toward top_p_min). + if self.top_p_decay is not None and not 0.0 < self.top_p_decay <= 1.0: + raise ValueError(f"require 0 < top_p_decay <= 1, got top_p_decay={self.top_p_decay}") + if self.top_p_min is not None and not 0.0 < self.top_p_min <= 1.0: + raise ValueError(f"require 0 < top_p_min <= 1, got top_p_min={self.top_p_min}") + if self.top_p_reset_ids is not None and self.top_p_reset_ids < 0: + raise ValueError( + f"require top_p_reset_ids >= 0, got top_p_reset_ids={self.top_p_reset_ids}" + ) + if self.best_of is not None and self.best_of < self.n: raise ValueError(f"best_of ({self.best_of}) cannot be less than n ({self.n})") @@ -421,8 +443,40 @@ def _validate(self): if self.logprobs_simple_format and self.use_beam_search: raise ValueError("logprobs_simple_format is not supported with beam search") - # NB: Static, because downstream code only holds instances of - # bindings.SamplingConfig (not SamplingParams). + # NB: The predicates below are static because downstream code (e.g. + # sampler_strategy.resolve_sampling_strategy) only holds instances of + # bindings.SamplingConfig (not SamplingParams). They are the single + # source of truth for the greedy / top-p-decay resolution shared by + # _greedy_decoding and the torch sampler. + + @staticmethod + def params_imply_top_p_decay_active(top_p_decay: Optional[float]) -> bool: + """Whether dynamic top-p decay is active. + + Active iff ``top_p_decay`` is explicitly set and ``< 1.0``; a decay of + ``1.0`` (the C++ default) is a no-op. Values outside ``(0, 1]`` are + rejected up front (_validate and the executor::SamplingConfig + constructor), so they never reach this predicate. + """ + return top_p_decay is not None and top_p_decay < 1.0 + + @staticmethod + def params_imply_explicit_greedy( + *, + temperature: Optional[float], + top_p: Optional[float], + top_k: Optional[int], + min_p: Optional[float], + ) -> bool: + """Whether the request carries an explicit greedy control. + + Explicit means top_k == 1, top_p == 0.0, min_p == 1.0, or temperature == 0, + as opposed to the implicit "all params unset" greedy default. min_p == 1.0 + keeps only tokens whose probability equals the row maximum, so like + top_p == 0.0 it collapses sampling to a single token. + """ + return top_k == 1 or top_p == 0.0 or min_p == 1.0 or temperature == 0 + @staticmethod def params_imply_greedy_decoding( *, @@ -430,13 +484,28 @@ def params_imply_greedy_decoding( top_p: Optional[float], top_k: Optional[int], use_beam_search: bool | None, - ): - return (not use_beam_search) and ( - (temperature is None and top_p is None and top_k is None) - or top_k == 1 - or top_p == 0.0 - or temperature == 0 - ) + min_p: Optional[float] = None, + top_p_decay: Optional[float] = None, + ) -> bool: + """Whether the parameters resolve to greedy decoding. + + An explicit greedy control always wins. The implicit "all params unset" + greedy default is overridden by any active sampling knob: an active + top-p decay (which implies top-p sampling so the decayed runtime top-p + can take effect) or a ``min_p`` in ``(0, 1)`` (which still selects among + multiple tokens); callers that do not support decay may omit + ``top_p_decay``. + """ + if use_beam_search: + return False + if SamplingParams.params_imply_explicit_greedy( + temperature=temperature, top_p=top_p, top_k=top_k, min_p=min_p + ): + return True + implicitly_greedy = temperature is None and top_p is None and top_k is None + min_p_active = min_p is not None and min_p > 0.0 + decay_active = SamplingParams.params_imply_top_p_decay_active(top_p_decay) + return implicitly_greedy and not min_p_active and not decay_active @property def _greedy_decoding(self) -> bool: @@ -445,6 +514,8 @@ def _greedy_decoding(self) -> bool: top_p=self.top_p, top_k=self.top_k, use_beam_search=self.use_beam_search, + min_p=self.min_p, + top_p_decay=self.top_p_decay, ) @property diff --git a/tensorrt_llm/scaffolding/task.py b/tensorrt_llm/scaffolding/task.py index 82c92032cfcc..c77767eada81 100644 --- a/tensorrt_llm/scaffolding/task.py +++ b/tensorrt_llm/scaffolding/task.py @@ -1,3 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import annotations import json @@ -116,8 +130,8 @@ class GenerationTask(Task): # Server-side request id captured from the OpenAI client's streaming # chunks (every chunk's ``chunk.id`` field). Trace-replay clients use - # this to look up the matching record in trtllm-serve's - # ``/perf_metrics`` drain and attach per-request KV-cache statistics + # this to correlate response-carried per-request metrics and attach + # KV-cache statistics # (``num_reused_blocks`` / ``num_missed_blocks`` / ``free_num_blocks``) # to the per-LLM-call row in the step JSON. ``None`` until the worker # observes the first chunk (or when not running against an diff --git a/tensorrt_llm/scaffolding/trace_replay/replay.py b/tensorrt_llm/scaffolding/trace_replay/replay.py index 9c4278760692..8561acd56685 100644 --- a/tensorrt_llm/scaffolding/trace_replay/replay.py +++ b/tensorrt_llm/scaffolding/trace_replay/replay.py @@ -1,3 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import asyncio import hashlib import random @@ -256,7 +270,7 @@ class DropPathStats: POSTs to the server, one immediately after — for every conv id this branch owns. Each probe's ``request_id`` (server-assigned, captured by the worker from the OpenAI streaming chunk) is recorded here and - later joined with trtllm-serve's ``/perf_metrics`` drain to recover + later joined with response-carried metrics to recover ``num_reused_blocks`` (the per-request KV-cache hit count) and ``free_num_blocks`` (the post-call snapshot of the free-block pool). @@ -399,7 +413,7 @@ async def _run(self): if event is None: # sentinel # Fire end-of-branch retention probes BEFORE returning, # so the parent ``wait_all_done`` covers them and the - # client's downstream /perf_metrics drain sees the + # client receives the response-carried metrics for the # probe records. For a child branch this is at # parallel_end (immediately after any drop_kv_cache); # for the root branch it is at end-of-session. @@ -503,7 +517,7 @@ async def _handle_drop_kv_cache(self, event: TraceEvent): # ``max_tokens=1, ignore_eos=True`` probe per (conv_id, phase) pair # immediately before and after the truncate. Each probe's # request_id is recorded here; the per-request KV-cache hit - # accounting is joined in later via /perf_metrics. The probes + # accounting is joined later from response-carried metrics. The probes # serialize the truncate against a real before/after measurement # so downstream verification can mechanically prove the truncate # actually freed the blocks the engine claimed it did (P0.6.a). diff --git a/tensorrt_llm/scaffolding/worker.py b/tensorrt_llm/scaffolding/worker.py index ad1432934201..4c95167164e0 100644 --- a/tensorrt_llm/scaffolding/worker.py +++ b/tensorrt_llm/scaffolding/worker.py @@ -1,3 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import asyncio import copy import json @@ -341,8 +355,8 @@ async def generation_handler(self, task: GenerationTask) -> TaskStatus: now = time.perf_counter() # Every chunk carries the server-assigned request id. Capture # it once (the first non-None value) so callers can later - # correlate per-request perf metrics drained from - # ``/perf_metrics`` with the GenerationTask that issued them. + # correlate response-carried per-request metrics with the + # GenerationTask that issued them. if request_id is None: cid = getattr(chunk, "id", None) if cid is not None: diff --git a/tensorrt_llm/serialization.py b/tensorrt_llm/serialization.py index ba548cc426f7..2311cec58f55 100644 --- a/tensorrt_llm/serialization.py +++ b/tensorrt_llm/serialization.py @@ -66,7 +66,6 @@ "KvCacheRetentionConfig.TokenRangeRetentionConfig", "PeftCacheConfig", "SchedulerConfig" ], - "tensorrt_llm.builder": ["BuildConfig"], "tensorrt_llm.disaggregated_params": ["DisaggregatedParams"], "tensorrt_llm.inputs.multimodal": ["MultimodalInput"], "tensorrt_llm.executor.postproc_worker": [ diff --git a/tensorrt_llm/serve/_perf_metrics_schema.py b/tensorrt_llm/serve/_perf_metrics_schema.py new file mode 100644 index 000000000000..5a9d606d85ac --- /dev/null +++ b/tensorrt_llm/serve/_perf_metrics_schema.py @@ -0,0 +1,97 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Union + +from typing_extensions import NotRequired, TypedDict + + +class StepMetrics(TypedDict): + forward_start_time: float + forward_end_time: float + sample_start_time: float + sample_end_time: float + gpu_forward_time: float + gpu_sample_time: float + token_time: NotRequired[float] + scheduled_time: NotRequired[float] + prev_batch_token_time: NotRequired[float] + iter: NotRequired[int] + + +class TimeBreakdownMetrics(TypedDict): + step_metrics: NotRequired[List[StepMetrics]] + ctx_chunk_metrics: NotRequired[List[StepMetrics]] + ctx_gpu_forward_time: NotRequired[float] + ctx_gpu_sample_time: NotRequired[float] + + +class TimingMetrics(TypedDict): + arrival_time: Optional[float] + first_scheduled_time: NotRequired[Optional[float]] + first_token_time: NotRequired[Optional[float]] + last_token_time: Optional[float] + server_arrival_time: NotRequired[Optional[float]] + server_first_token_time: NotRequired[Optional[float]] + kv_cache_size: NotRequired[int] + kv_cache_transfer_start: NotRequired[Optional[float]] + kv_cache_transfer_end: NotRequired[Optional[float]] + + +class KvCacheMetrics(TypedDict): + num_total_allocated_blocks: int + num_new_allocated_blocks: int + num_reused_blocks: int + num_missed_blocks: int + + +class SpeculativeDecodingMetrics(TypedDict): + acceptance_rate: float + total_accepted_draft_tokens: int + total_draft_tokens: int + + +class PerfMetrics(TypedDict): + timing_metrics: TimingMetrics + first_iter: NotRequired[int] + last_iter: NotRequired[int] + kv_cache_metrics: NotRequired[KvCacheMetrics] + speculative_decoding: NotRequired[SpeculativeDecodingMetrics] + + +class WorkerPerfMetrics(TypedDict): + request_id: Union[int, str] + perf_metrics: PerfMetrics + ctx_request_id: NotRequired[int] + time_breakdown_metrics: NotRequired[TimeBreakdownMetrics] + + +class WorkerPerfMetricsRecord(WorkerPerfMetrics): + status: str + disagg_request_id: NotRequired[int] + + +class DisaggPerfMetricsRecord(TypedDict): + ctx_server: str + gen_server: str + disagg_server_arrival_time: float + disagg_ctx_dispatch_time: Optional[float] + disagg_server_first_token_time: Optional[float] + status: str + disagg_request_id: NotRequired[int] + ctx_perf_metrics: NotRequired[WorkerPerfMetrics] + gen_perf_metrics: NotRequired[WorkerPerfMetrics] + + +PerfMetricsRecord = Union[WorkerPerfMetricsRecord, DisaggPerfMetricsRecord] diff --git a/tensorrt_llm/serve/chat_tokenization.py b/tensorrt_llm/serve/chat_tokenization.py new file mode 100644 index 000000000000..d839e79bfff6 --- /dev/null +++ b/tensorrt_llm/serve/chat_tokenization.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING, Callable, Optional, cast + +from transformers import PretrainedConfig + +from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest + +if TYPE_CHECKING: + from tensorrt_llm.serve.harmony_adapter import HarmonyAdapter + +ToolDict = dict[str, object] + + +def resolve_model_type_from_config(model_name_or_path: str) -> Optional[str]: + """Return the checkpoint's declared model type from its config metadata.""" + config_dict, _ = PretrainedConfig.get_config_dict(model_name_or_path) + model_type = config_dict.get("model_type") + return model_type if isinstance(model_type, str) else None + + +def uses_harmony_tokenization( + use_harmony: Optional[bool] = None, + model_type: Optional[str] = None, + model_type_resolver: Optional[Callable[[], Optional[str]]] = None, +) -> bool: + if os.getenv("DISABLE_HARMONY_ADAPTER", "0") == "1": + return False + if use_harmony is not None: + return use_harmony + if model_type is None and model_type_resolver is not None: + model_type = model_type_resolver() + return model_type == "gpt_oss" + + +def get_chat_completion_tool_dicts( + request: ChatCompletionRequest, empty_as_none: bool = False +) -> Optional[list[ToolDict]]: + if request.tools is None or (empty_as_none and not request.tools): + return None + tools: list[ToolDict] = [] + for tool in request.tools: + if hasattr(tool, "model_dump"): + tools.append(cast(ToolDict, tool.model_dump())) + elif isinstance(tool, dict): + tools.append(cast(ToolDict, tool)) + else: + raise TypeError(f"Unsupported tool type: {type(tool).__name__}") + return tools + + +def tokenize_harmony_chat_request( + request: ChatCompletionRequest, + harmony_adapter: Optional["HarmonyAdapter"] = None, + set_prompt_token_ids: bool = False, +) -> list[int]: + if request.prompt_token_ids is not None: + return request.prompt_token_ids + + from tensorrt_llm.serve import harmony_adapter as harmony_adapter_module + + adapter = harmony_adapter or harmony_adapter_module.get_harmony_adapter() + result = adapter.openai_to_harmony_tokens( + request.messages, + get_chat_completion_tool_dicts(request, empty_as_none=True), + reasoning_effort=harmony_adapter_module.maybe_transform_reasoning_effort( + request.reasoning_effort + ), + tool_choice=request.tool_choice, + ) + if set_prompt_token_ids: + request.prompt_token_ids = result + return result + + +def render_chat_request_for_tokenizer( + request: ChatCompletionRequest, tokenizer: object +) -> str | list[int]: + chat_template_kwargs = ( + dict(request.chat_template_kwargs) if getattr(request, "chat_template_kwargs", None) else {} + ) + chat_template_kwargs["tools"] = get_chat_completion_tool_dicts(request) + chat_template_kwargs["documents"] = request.documents + if request.chat_template is not None: + chat_template_kwargs["chat_template"] = request.chat_template + rendered = tokenizer.apply_chat_template( + [msg if isinstance(msg, dict) else dict(msg) for msg in request.messages], + add_generation_prompt=request.add_generation_prompt, + tokenize=False, + return_dict=False, + **chat_template_kwargs, + ) + if isinstance(rendered, str): + return rendered + return list(rendered) + + +def tokenize_chat_request_for_serving( + request: ChatCompletionRequest, + tokenizer_factory: Callable[[], object], + encode_rendered: Callable[[str, object], list[int]], + use_harmony: Optional[bool] = None, + model_type: Optional[str] = None, + model_type_resolver: Optional[Callable[[], Optional[str]]] = None, + harmony_adapter: Optional["HarmonyAdapter"] = None, + set_prompt_token_ids: bool = True, +) -> list[int]: + if request.prompt_token_ids is not None: + return request.prompt_token_ids + + if uses_harmony_tokenization( + use_harmony=use_harmony, + model_type=model_type, + model_type_resolver=model_type_resolver, + ): + return tokenize_harmony_chat_request( + request, + harmony_adapter=harmony_adapter, + set_prompt_token_ids=set_prompt_token_ids, + ) + + tokenizer = tokenizer_factory() + rendered = render_chat_request_for_tokenizer(request, tokenizer) + result = encode_rendered(rendered, tokenizer) if isinstance(rendered, str) else rendered + if set_prompt_token_ids: + request.prompt_token_ids = result + return result diff --git a/tensorrt_llm/serve/chat_utils.py b/tensorrt_llm/serve/chat_utils.py index 023371adfe00..d60d93f2ce45 100644 --- a/tensorrt_llm/serve/chat_utils.py +++ b/tensorrt_llm/serve/chat_utils.py @@ -424,6 +424,7 @@ def parse_chat_messages_coroutines( model_config: AutoConfig, multimodal_server_config: Optional[MultimodalServerConfig] = None, request_media_io_kwargs: Optional[Dict[str, Dict[str, Any]]] = None, + model_type_override: Optional[str] = None, ) -> Tuple[List[ConversationMessage], Coroutine[Any, Any, tuple[Optional[Dict[ str, List[Any]]], Optional[Dict[str, List[Any]]]]], list[dict[str, int]]]: @@ -470,7 +471,7 @@ def parse_chat_messages_coroutines( # `content_parts` - overwriting any STRING-style placeholders inserted here. # See also: `_resolve_content_format` (inputs/utils.py) for the full resolution used downstream. registry_format = MULTIMODAL_PLACEHOLDER_REGISTRY.get_content_format( - type(model_config).model_type) + model_type) if registry_format is not None: content_format = registry_format else: @@ -482,6 +483,9 @@ def parse_chat_messages_coroutines( # Track placeholders added for this message only. msg_placeholder_counts = {} + # Snapshot the tracker's item_order length so we can slice off just + # the entries this message contributed. + item_order_start = len(mm_data_tracker.item_order()) if parsed_msg["media"]: for mdata in parsed_msg["media"]: placeholder = mm_data_tracker.add_data( @@ -499,20 +503,25 @@ def parse_chat_messages_coroutines( # prepend/append according to placeholder_placement. content_parts = parsed_msg.get("content_parts") interleave = MULTIMODAL_PLACEHOLDER_REGISTRY.get_interleave_placeholders( - type(model_config).model_type) + model_type) if content_parts and interleave: parsed_msg["content"] = interleave_mm_placeholders( - type(model_config).model_type, content_parts, - msg_placeholder_counts, + model_type, content_parts, msg_placeholder_counts, mm_data_tracker.placeholder_modalities()) else: + msg_item_order = mm_data_tracker.item_order()[item_order_start:] parsed_msg["content"] = add_multimodal_placeholders( - type(model_config).model_type, parsed_msg["content"], - msg_placeholder_counts) + type(model_config).model_type, + parsed_msg["content"], + msg_placeholder_counts, + item_order=msg_item_order, + ) mm_placeholder_counts.append(msg_placeholder_counts) - return conversation, mm_data_tracker.retrieve_all_async( - ), mm_placeholder_counts + # ``item_order`` is populated synchronously by ``add_data``, so it can + # be returned directly (not through the coroutine). + return (conversation, mm_data_tracker.retrieve_all_async(), + mm_placeholder_counts, mm_data_tracker.item_order()) def make_tool_call_id(id_type: str = "random", func_name=None, idx=None): diff --git a/tensorrt_llm/serve/conversation_id.py b/tensorrt_llm/serve/conversation_id.py index 7e0fed21640e..2acfcad8012e 100644 --- a/tensorrt_llm/serve/conversation_id.py +++ b/tensorrt_llm/serve/conversation_id.py @@ -36,9 +36,10 @@ class RequestWithConversationParams(Protocol): def get_request_conversation_id(request: RequestWithConversationParams) -> Optional[str]: conversation_params = request.conversation_params - if conversation_params is None: - return None - return conversation_params.conversation_id + if conversation_params is not None: + return conversation_params.conversation_id + disaggregated_params = getattr(request, "disaggregated_params", None) + return None if disaggregated_params is None else disaggregated_params.conversation_id def extract_conversation_id_from_headers(headers: Optional[Mapping[str, str]]) -> Optional[str]: diff --git a/tensorrt_llm/serve/coordinator_server.py b/tensorrt_llm/serve/coordinator_server.py new file mode 100644 index 000000000000..8eb081fd5e34 --- /dev/null +++ b/tensorrt_llm/serve/coordinator_server.py @@ -0,0 +1,173 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Coordinator HTTP server for disaggregated serving. + +One coordinator process owns all cluster state (routers, readiness, worker +events, and -- for the centralized router -- the single ZMQ event-ingest bind) +and answers the internal coordination API that the forked worker processes call: + + POST /select {"role", "routing_key", "req_id", "exclude_server"} + -> {"server": "host:port", "info": {...}, "req_id": } + POST /finish {"role", "req_id", "success"} -> {} + GET /cluster_info -> {...} + GET /health -> 200 when ready + GET /version + +The routing key is produced worker-side by ``Router.routing_key`` and consumed +here by ``Router.get_next_server_by_key`` (see ``serve/router.py``), so this +endpoint is generic across the stateful router types that use it (centralized -> +block hashes, conversation -> conversation_id). Single-process by design; it owns +the ZMQ ingest bind for centralized mode. +""" + +import asyncio +from contextlib import asynccontextmanager +from typing import Optional + +import msgpack +import uvicorn +from fastapi import FastAPI, Request +from fastapi.responses import Response + +from tensorrt_llm.logger import logger +from tensorrt_llm.serve.cluster_storage import HttpClusterStorageServer +from tensorrt_llm.serve.disagg_coordinator import DisaggCoordinatorService +from tensorrt_llm.version import __version__ as VERSION + +TIMEOUT_KEEP_ALIVE = 10 # seconds +MSGPACK_MEDIA_TYPE = "application/msgpack" + + +class CoordinatorServer: + """Serve a :class:`DisaggCoordinatorService`'s coordination API over HTTP.""" + + def __init__(self, coordinator: DisaggCoordinatorService) -> None: + self._coordinator = coordinator + + @asynccontextmanager + async def lifespan(app: FastAPI): + await self._coordinator.start() + yield + await self._coordinator.stop() + + self.app = FastAPI(lifespan=lifespan) + self.app.add_api_route("/select", self.select, methods=["POST"]) + self.app.add_api_route("/finish", self.finish, methods=["POST"]) + self.app.add_api_route("/cluster_info", self.cluster_info, methods=["GET"]) + self.app.add_api_route("/health", self.health, methods=["GET"]) + self.app.add_api_route("/version", self.version, methods=["GET"]) + cluster_storage = self._coordinator.cluster_storage + if isinstance(cluster_storage, HttpClusterStorageServer): + cluster_storage.add_routes(self.app) + + @staticmethod + def _response(content: object, status_code: int = 200) -> Response: + return Response( + content=msgpack.packb(content, use_bin_type=True), + status_code=status_code, + media_type=MSGPACK_MEDIA_TYPE, + ) + + async def select(self, raw_req: Request) -> Response: + try: + body = msgpack.unpackb(await raw_req.body(), raw=False) + except Exception as e: + return self._response({"error": f"invalid MessagePack body: {e}"}, status_code=400) + if ( + not isinstance(body, dict) + or "role" not in body + or "routing_key" not in body + or "req_id" not in body + ): + return self._response( + {"error": "body must include 'role', 'routing_key', and 'req_id'"}, + status_code=400, + ) + role = body["role"] + if role not in ("context", "ctx", "generation", "gen"): + return self._response({"error": f"invalid role: {role}"}, status_code=400) + req_id = body["req_id"] + if not isinstance(req_id, int) or isinstance(req_id, bool): + return self._response({"error": "req_id must be an integer"}, status_code=400) + exclude_server = body.get("exclude_server") + if exclude_server is not None and not isinstance(exclude_server, str): + return self._response({"error": "exclude_server must be a string"}, status_code=400) + try: + server, info, req_id = await self._coordinator.select( + role, body["routing_key"], req_id, exclude_server + ) + except ValueError as e: + return self._response({"error": str(e)}, status_code=503) + except Exception as e: # noqa: BLE001 + logger.error(f"CoordinatorServer.select failed: {e}") + return self._response({"error": str(e)}, status_code=500) + return self._response({"server": server, "info": info, "req_id": req_id}) + + async def finish(self, raw_req: Request) -> Response: + try: + body = msgpack.unpackb(await raw_req.body(), raw=False) + except Exception as e: + return self._response({"error": f"invalid MessagePack body: {e}"}, status_code=400) + if not isinstance(body, dict) or "role" not in body or "req_id" not in body: + return self._response( + {"error": "body must include 'role' and 'req_id'"}, status_code=400 + ) + role = body["role"] + if role not in ("context", "ctx", "generation", "gen"): + return self._response({"error": f"invalid role: {role}"}, status_code=400) + req_id = body["req_id"] + if not isinstance(req_id, int) or isinstance(req_id, bool): + return self._response({"error": "req_id must be an integer"}, status_code=400) + success = body.get("success", True) + if not isinstance(success, bool): + return self._response({"error": "success must be a boolean"}, status_code=400) + await self._coordinator.finish(role, req_id, success) + return self._response({}) + + async def cluster_info(self) -> Response: + return self._response(await self._coordinator.cluster_info()) + + async def health(self) -> Response: + return Response(status_code=200 if await self._coordinator.is_ready() else 503) + + async def version(self) -> Response: + return self._response({"version": VERSION}) + + async def __call__(self, host: str, port: int, uds: Optional[str] = None) -> None: + # Single-process (owns routing state + the centralized ZMQ ingest bind); + # workers=1 forced so a leaked WEB_CONCURRENCY can't fork it. When ``uds`` + # is set the co-located fleet uses it for the hot /select,/finish path + # (avoids the TCP loopback overhead that dominated per-request latency). + kwargs = dict(workers=1, log_level="info", timeout_keep_alive=TIMEOUT_KEEP_ALIVE) + if uds: + # uvicorn.Config binds uds XOR host:port, so run two Servers: UDS for + # the fleet (hot path) and TCP for health/external clients. + import asyncio as _asyncio + + await self._coordinator.start() + try: + uds_cfg = uvicorn.Config(self.app, uds=uds, lifespan="off", **kwargs) + tcp_cfg = uvicorn.Config(self.app, host=host, port=port, lifespan="off", **kwargs) + await _asyncio.gather( + uvicorn.Server(uds_cfg).serve(), uvicorn.Server(tcp_cfg).serve() + ) + finally: + await self._coordinator.stop() + else: + config = uvicorn.Config(self.app, host=host, port=port, **kwargs) + await uvicorn.Server(config).serve() + + +def serve_coordinator(host: str, port: int, coordinator: DisaggCoordinatorService) -> None: + asyncio.run(CoordinatorServer(coordinator)(host, port)) diff --git a/tensorrt_llm/serve/disagg_coordinator.py b/tensorrt_llm/serve/disagg_coordinator.py new file mode 100644 index 000000000000..86475af498fa --- /dev/null +++ b/tensorrt_llm/serve/disagg_coordinator.py @@ -0,0 +1,671 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Coordination for disaggregated serving. + +A :class:`DisaggCoordinator` owns everything that is *not* a completion: the +ctx/gen routers, readiness, cluster info, and worker/auto-scaling events. The +completions service holds one and reads ``ctx_router`` / ``gen_router`` off it, +then drives ``router.get_next_server`` / ``router.finish_request`` uniformly -- +so serving a completion is decoupled from managing the cluster and is identical +whether this process owns the routers or delegates to a remote coordinator. + +Two implementations for the coordinator/worker deployment: + +* :class:`DisaggCoordinatorService` -- runs in the coordinator (and in the + collapsed single-process path). Owns the real ctx/gen ``Router`` objects, + server preparation/monitoring, the auto-scaling ``DisaggClusterManager`` + + worker events, and readiness. Its :meth:`select` / :meth:`finish` are the + coordinator's ``/select`` / ``/finish`` handlers. +* :class:`CoordinatorClient` -- runs in each forked worker. Stateful routers + (conversation, centralized) are wrapped in a :class:`CoordinatorDelegatingRouter` + that posts the routing key to ``/select`` (finish -> ``/finish``); stateless + routers (round_robin, load_balancing) place locally in the worker. Readiness / + cluster_info proxy the coordinator over HTTP. +""" + +import asyncio +import os +import time +from abc import ABC, abstractmethod +from typing import Any, Dict, Optional, Tuple + +import aiohttp +import msgpack + +from tensorrt_llm.llmapi.disagg_utils import ( + DisaggServerConfig, + MetadataServerConfig, + ServerRole, + get_ctx_gen_server_addrs, + get_global_disagg_request_id, + worker_local_process_id, # noqa: F401 +) +from tensorrt_llm.logger import logger +from tensorrt_llm.serve.cluster_storage import ( + ClusterStorage, + WatchEventType, + create_cluster_storage, +) +from tensorrt_llm.serve.disagg_auto_scaling import DisaggClusterManager, WorkerInfo +from tensorrt_llm.serve.metadata_server import create_metadata_server +from tensorrt_llm.serve.openai_client import OpenAIClient +from tensorrt_llm.serve.router import ( + CoordinatorDelegatingRouter, + KvCacheAwareRouter, + Router, + build_disagg_routers, +) + +__all__ = [ + "DisaggCoordinator", + "DisaggCoordinatorService", + "CoordinatorClient", + "coordinator_base_url", + "make_coordinator_session", +] + +COORDINATOR_RESERVATION_TIMEOUT_ENV = "TRTLLM_DISAGG_COORDINATOR_RESERVATION_TIMEOUT" +COORDINATOR_RESERVATION_TIMEOUT_DEFAULT_S = 180.0 +COORDINATOR_STATE_SYNC_INTERVAL_S = 3.0 + + +def coordinator_reservation_timeout() -> float: + return float( + os.environ.get( + COORDINATOR_RESERVATION_TIMEOUT_ENV, + COORDINATOR_RESERVATION_TIMEOUT_DEFAULT_S, + ) + ) + + +class DisaggCoordinator(ABC): + """Abstract coordinator: ctx/gen routers + readiness + cluster info + lifecycle. + + Placement and finish are driven through ``ctx_router`` / ``gen_router`` + (``Router.get_next_server`` / ``Router.finish_request``), so this surface only + exposes the routers plus readiness/info/lifecycle. + """ + + @property + @abstractmethod + def ctx_router(self) -> Router: ... + + @property + @abstractmethod + def gen_router(self) -> Router: ... + + @abstractmethod + async def is_ready(self) -> bool: ... + + @abstractmethod + async def cluster_info(self) -> Dict[str, Any]: ... + + async def start(self) -> None: ... + + async def stop(self) -> None: ... + + @abstractmethod + async def get_disagg_request_id(self) -> int: ... + + +class DisaggCoordinatorService(DisaggCoordinator): + """In-process coordinator owning the ctx/gen routers and all cluster state. + + Used in the coordinator process and in the single-process (workers==1) path. + """ + + def __init__( + self, + config: DisaggServerConfig, + client_factory, + metadata_config: Optional[MetadataServerConfig] = None, + server_preparation_func=None, + server_start_timeout_secs: int = 180, + health_check_interval_secs: int = 3, + reservation_timeout_secs: Optional[float] = None, + ): + self._config = config + self._client_factory = client_factory + self._metadata_config = metadata_config + # The coordinator owns routing state, so it builds the owner routers here + # (is_delegating_client=False): one shared namespace-aware core + a single + # ZMQ ingest server. Sole place owner routers are created (fleet workers + # hold only a CoordinatorClient's delegating surfaces). + self._metadata_server = create_metadata_server(metadata_config) + ctx_servers, gen_servers = get_ctx_gen_server_addrs(config.server_configs) + self._ctx_router, self._gen_router = build_disagg_routers( + config.ctx_router_config, + config.gen_router_config, + ctx_servers, + gen_servers, + metadata_config, + self._metadata_server, + server_preparation_func, + disagg_node_id=config.node_id, + is_delegating_client=False, + ) + # The coordinator owns the disagg cluster storage (auto-scaling backend): + # it drives the DisaggClusterManager below and, when the storage is an + # in-process HTTP server, its routes are mounted on the coordinator app. + self._cluster_storage: Optional[ClusterStorage] = ( + create_cluster_storage( + config.disagg_cluster_config.cluster_uri, config.disagg_cluster_config.cluster_name + ) + if config.disagg_cluster_config + else None + ) + self._server_start_timeout_secs = server_start_timeout_secs + self._health_check_interval_secs = health_check_interval_secs + self._reservation_timeout_secs = ( + coordinator_reservation_timeout() + if reservation_timeout_secs is None + else reservation_timeout_secs + ) + self._reservation_tasks: dict[tuple[str, int], asyncio.Task] = {} + + self._ctx_client: Optional[OpenAIClient] = None + self._gen_client: Optional[OpenAIClient] = None + self._disagg_cluster_manager: Optional[DisaggClusterManager] = None + + @property + def ctx_router(self) -> Router: + return self._ctx_router + + @property + def gen_router(self) -> Router: + return self._gen_router + + @property + def cluster_storage(self) -> Optional[ClusterStorage]: + return self._cluster_storage + + def set_clients(self, ctx_client: OpenAIClient, gen_client: OpenAIClient) -> None: + self._ctx_client = ctx_client + self._gen_client = gen_client + + # -- coordinator-path placement (workers call these via the HTTP server) -- + + def _api_lat(self, name: str): + """Return a lazily created per-API latency logger. + + It measures the coordinator owner's in-process handler time, excluding + the fleet-to-coordinator HTTP hop captured client-side as + ``[coord_api] client.*``. + """ + cache = self.__dict__.setdefault("_coord_api_lat", {}) + if name not in cache: + from tensorrt_llm.serve.responses_utils import PeriodicLatencyLogger + + cache[name] = PeriodicLatencyLogger(f"owner.{name}") + return cache[name] + + async def select( + self, role: str, routing_key, req_id, exclude_server: Optional[str] + ) -> Tuple[str, dict, Optional[str]]: + _t0 = time.monotonic() + router = self._router_for_role(role) + reservation_key = (self._normalize_role(role), req_id) + previous = self._reservation_tasks.pop(reservation_key, None) + if previous is not None: + previous.cancel() + await router.finish_request_by_id(req_id, False) + server, info, request_id = await router.get_next_server_by_key( + routing_key, req_id=req_id, exclude_server=exclude_server + ) + self._reservation_tasks[reservation_key] = asyncio.create_task( + self._expire_reservation(reservation_key, router) + ) + self._api_lat(f"select[{role}]").record(time.monotonic() - _t0) + return server, self._compact_route_info(info), request_id + + @staticmethod + def _compact_route_info(info: dict) -> dict: + compact = {key: info[key] for key in ("match_length", "num_tokens") if key in info} + disaggregated_params = info.get("server_info", {}).get("disaggregated_params") + if disaggregated_params is not None: + compact["server_info"] = {"disaggregated_params": disaggregated_params} + return compact + + async def get_disagg_request_id(self) -> int: + return get_global_disagg_request_id(self._config.node_id) + + async def finish(self, role: str, req_id, success: bool = True) -> None: + _t0 = time.monotonic() + reservation = self._reservation_tasks.pop((self._normalize_role(role), req_id), None) + if reservation is not None: + reservation.cancel() + await self._router_for_role(role).finish_request_by_id(req_id, success) + self._api_lat(f"finish[{role}]").record(time.monotonic() - _t0) + + async def _expire_reservation(self, key: tuple[str, int], router: Router) -> None: + try: + await asyncio.sleep(self._reservation_timeout_secs) + logger.warning( + f"Releasing stale coordinator reservation for role={key[0]}, " + f"req_id={key[1]} after {self._reservation_timeout_secs}s" + ) + await router.finish_request_by_id(key[1], False) + finally: + if self._reservation_tasks.get(key) is asyncio.current_task(): + self._reservation_tasks.pop(key, None) + + @staticmethod + def _normalize_role(role: str) -> str: + normalized_role = str(role).lower() + if normalized_role in ("context", "ctx"): + return "context" + if normalized_role in ("generation", "gen"): + return "generation" + raise ValueError(f"Unsupported coordinator role: {role}") + + def _router_for_role(self, role: str) -> Router: + normalized_role = self._normalize_role(role) + if normalized_role == "context": + return self._ctx_router + return self._gen_router + + async def start(self) -> None: + await self._ctx_router.prepare_servers() + await self._gen_router.prepare_servers() + if self._ctx_client is None or self._gen_client is None: + self._ctx_client = self._client_factory( + self._ctx_router, ServerRole.CONTEXT, self._config.max_retries + ) + self._gen_client = self._client_factory( + self._gen_router, ServerRole.GENERATION, self._config.max_retries + ) + + if self._config.disagg_cluster_config and self._cluster_storage: + logger.info("Starting disagg cluster manager") + self._disagg_cluster_manager = DisaggClusterManager( + self._config.disagg_cluster_config, self._cluster_storage + ) + await self._disagg_cluster_manager.start() + await self._disagg_cluster_manager.watch_workers(on_event=self._on_worker_event) + logger.info("Disagg cluster manager started") + else: + if self._metadata_server and self._metadata_config: + logger.info("Starting server monitoring via metadata service") + await self._ctx_router.start_server_monitoring( + self._metadata_config.refresh_interval + ) + await self._gen_router.start_server_monitoring( + self._metadata_config.refresh_interval + ) + await self._wait_for_all_servers_ready() + + async def stop(self) -> None: + reservations = list(self._reservation_tasks.values()) + self._reservation_tasks.clear() + for reservation in reservations: + reservation.cancel() + if reservations: + await asyncio.gather(*reservations, return_exceptions=True) + if self._disagg_cluster_manager: + await self._disagg_cluster_manager.stop() + if self._metadata_server: + await self._ctx_router.stop_server_monitoring() + await self._gen_router.stop_server_monitoring() + + async def is_ready(self) -> bool: + if self._disagg_cluster_manager: + return await self._disagg_cluster_manager.is_ready_with_router( + self._ctx_router.num_prepared_servers, + self._gen_router.num_prepared_servers, + ) + return True + + async def cluster_info(self) -> Dict[str, Any]: + info = { + "is_ready": await self.is_ready(), + "server_lists": { + "context": list(self._ctx_router.servers), + "generation": list(self._gen_router.servers), + }, + } + routing_key_configs = {} + for role, router in (("context", self._ctx_router), ("generation", self._gen_router)): + if isinstance(router, KvCacheAwareRouter): + config = router.routing_key_config() + if config is not None: + routing_key_configs[role] = config + if routing_key_configs: + info["routing_key_configs"] = routing_key_configs + if self._disagg_cluster_manager: + info.update(await self._disagg_cluster_manager.cluster_info()) + return info + + async def _wait_for_all_servers_ready(self) -> None: + import os + + gen_only = os.getenv("TRTLLM_DISAGG_BENCHMARK_GEN_ONLY") == "1" + + async def check_servers_ready(): + elapsed_time = 0 + interval = self._health_check_interval_secs + while elapsed_time < self._server_start_timeout_secs: + if gen_only: + unready_ctx_servers = [] + else: + _, unready_ctx_servers = await self._ctx_client.check_ready() + _, unready_gen_servers = await self._gen_client.check_ready() + if len(unready_ctx_servers) == 0 and len(unready_gen_servers) == 0: + logger.info( + "All servers are ready" + if not gen_only + else "Generation servers are ready (context skipped)" + ) + return + logger.info( + f"Waiting for servers, context: {unready_ctx_servers}, " + f"generation: {unready_gen_servers}" + ) + await asyncio.sleep(interval) + elapsed_time += interval + + try: + await asyncio.wait_for(check_servers_ready(), timeout=self._server_start_timeout_secs) + except asyncio.TimeoutError: + raise TimeoutError("Timeout waiting for context and generation servers to be ready") + + async def _on_worker_event(self, worker_info: WorkerInfo, event_type: WatchEventType): + router_map = { + ServerRole.CONTEXT: self._ctx_router, + ServerRole.GENERATION: self._gen_router, + } + worker_addr = f"{worker_info.host}:{worker_info.port}" + try: + router = router_map[worker_info.role] + if event_type == WatchEventType.SET: + await router.add_server(worker_addr) + elif event_type == WatchEventType.DELETE: + await router.remove_server(worker_addr) + logger.info(f"Worker {event_type.name} event: {worker_info.worker_id}, {worker_addr}") + except KeyError: + logger.error( + f"Unknown worker role: {worker_info.role}, Worker " + f"{worker_info.worker_id} event: {event_type.name}" + ) + + +COORDINATOR_UDS_SCHEME = "unix:" + + +def coordinator_base_url(remote_url: str) -> str: + """The URL prefix to build request URLs against. + + ``remote_url`` is either a TCP URL (``http://host:port``) or a Unix domain + socket (``unix:/abs/path.sock``). For UDS the socket path routes, so the HTTP + host is a dummy ``http://localhost`` (aiohttp still needs a valid http URL). + """ + if remote_url.startswith(COORDINATOR_UDS_SCHEME): + return "http://localhost" + return remote_url.rstrip("/") + + +def make_coordinator_session(remote_url: str) -> aiohttp.ClientSession: + """Create an aiohttp session for the coordinator endpoint. + + A ``unix:/path`` URL uses a UnixConnector to avoid the TCP loopback stack + when the fleet is co-located with the implicit coordinator. Callers create + the session lazily so it binds to the running event loop. + + limit=0 (unlimited pool): each fleet worker has ~concurrency/num_workers + requests in flight (e.g. 320 at c1280/4), but aiohttp's default pool caps at + 100 -- so most /select,/finish calls were BLOCKING on a free pooled + connection, which was the real ~hundreds-of-ms client.select latency (not the + ~0.1ms handler, not the transport). Uncapping lets all concurrent calls hold + a connection. The coordinator is a trusted local socket, so no cap needed. + """ + if remote_url.startswith(COORDINATOR_UDS_SCHEME): + sock_path = remote_url[len(COORDINATOR_UDS_SCHEME) :] + return aiohttp.ClientSession(connector=aiohttp.UnixConnector(path=sock_path, limit=0)) + return aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=0)) + + +class CoordinatorClient(DisaggCoordinator): + """Worker-side coordinator: delegate stateful routing to the coordinator. + + A *stateful* router (conversation, centralized -- it exposes + ``get_next_server_by_key``) is wrapped in a :class:`CoordinatorDelegatingRouter` + so the worker computes the small routing key locally and the coordinator makes + the placement (placement -> ``/select``, finish -> ``/finish``). A *stateless* + router (round_robin, load_balancing) is used as-is and places locally in the + worker -- no coordinator round-trip. A background ``/cluster_info`` poll keeps + readiness and stateless-router server lists synchronized with the coordinator. + + Args: + remote_url: Coordinator base URL (e.g. ``http://host:PORT``). + config: The disagg config; the client builds its own delegating routers + of the configured type (same config as the coordinator so the keys it + extracts line up). + """ + + def __init__( + self, + remote_url: str, + config: DisaggServerConfig, + metadata_config: Optional[MetadataServerConfig] = None, + request_timeout_s: float = 5.0, + startup_timeout_s: float = 180.0, + ): + # remote_url may be a TCP URL or unix:/path. Resolve to a request base + # URL now; the session (with the right connector) is created lazily so it + # binds to the running event loop. + self._remote_url_raw = remote_url + self._remote_url = coordinator_base_url(remote_url) + self._request_timeout_s = request_timeout_s + self._startup_timeout_s = startup_timeout_s + self._session: Optional[aiohttp.ClientSession] = None + self._sync_task: Optional[asyncio.Task] = None + self._is_ready = False + if config.disagg_cluster_config is not None: + self._state_sync_interval_s = config.disagg_cluster_config.heartbeat_interval_sec + elif metadata_config is not None: + self._state_sync_interval_s = metadata_config.refresh_interval + else: + self._state_sync_interval_s = COORDINATOR_STATE_SYNC_INTERVAL_S + # Local disagg-id generation (no HTTP hop): (node_id, per-worker process_id) + # keeps the snowflake unique across co-located workers. + self._node_id = config.node_id + self._process_id = worker_local_process_id() + # A delegating client builds coreless router surfaces (compute routing_key + # locally, delegate placement to the coordinator; no ingest port / core). + # Sole place delegating routers are created. + ctx_servers, gen_servers = get_ctx_gen_server_addrs(config.server_configs) + ctx_router, gen_router = build_disagg_routers( + config.ctx_router_config, + config.gen_router_config, + ctx_servers, + gen_servers, + metadata_config, + create_metadata_server(metadata_config), + disagg_node_id=config.node_id, + is_delegating_client=True, + ) + self._ctx_router = self._maybe_delegate(ctx_router, "context") + self._gen_router = self._maybe_delegate(gen_router, "generation") + + def _maybe_delegate(self, local_router: Router, role: str) -> Router: + # Stateful routers expose get_next_server_by_key -> delegate placement to + # the coordinator; stateless ones place locally (used unchanged). Pass the + # RAW url so the delegating router picks the UDS connector when applicable. + if hasattr(local_router, "get_next_server_by_key"): + return CoordinatorDelegatingRouter( + self._remote_url_raw, local_router, role, self._request_timeout_s + ) + return local_router + + @property + def ctx_router(self) -> Router: + return self._ctx_router + + @property + def gen_router(self) -> Router: + return self._gen_router + + @property + def session(self) -> aiohttp.ClientSession: + if self._session is None: + self._session = make_coordinator_session(self._remote_url_raw) + return self._session + + async def start(self) -> None: + # Fail fast: probe /cluster_info with bounded retry so a delegating server + # exits non-zero if its coordinator never comes up (vs 500-ing every req). + info = await self._await_coordinator() + await self._apply_cluster_info(info) + self._sync_task = asyncio.create_task( + self._sync_coordinator_state(self._state_sync_interval_s) + ) + + async def _sync_coordinator_state(self, interval_s: float) -> None: + while True: + try: + await asyncio.sleep(interval_s) + await self._apply_cluster_info(await self.cluster_info()) + except asyncio.CancelledError: + raise + except Exception as error: # noqa: BLE001 + self._is_ready = False + logger.warning(f"CoordinatorClient state sync failed: {error}") + + async def _await_coordinator(self) -> Dict[str, Any]: + """Poll the coordinator's /cluster_info until reachable, or raise. + + Returns the cluster_info dict once the coordinator answers HTTP 200. A 200 + means the coordinator process is up (not that all workers are ready -- + that's is_ready's job). Raises RuntimeError if it stays unreachable past + startup_timeout_s so the delegating server fails fast instead of serving + against a missing coordinator. + """ + loop = asyncio.get_event_loop() + deadline = loop.time() + self._startup_timeout_s + attempt = 0 + while True: + try: + async with self.session.get( + f"{self._remote_url}/cluster_info", timeout=self._request_timeout_s + ) as resp: + if resp.status == 200: + logger.info( + f"CoordinatorClient: coordinator reachable at {self._remote_url}" + ) + return msgpack.unpackb(await resp.read(), raw=False) + last_err = f"HTTP {resp.status}" + except Exception as e: # noqa: BLE001 + last_err = str(e) + attempt += 1 + if loop.time() >= deadline: + raise RuntimeError( + f"Coordinator at {self._remote_url} not reachable after " + f"{self._startup_timeout_s}s ({attempt} attempts, last " + f"error: {last_err}); aborting delegating server startup" + ) + logger.info( + f"CoordinatorClient: waiting for coordinator at " + f"{self._remote_url} (attempt {attempt}, {last_err})" + ) + await asyncio.sleep(2.0) + + async def get_disagg_request_id(self) -> int: + # Generate locally: the snowflake id is self-contained (not shared state), + # so the per-worker (node_id, process_id) avoids collisions without any HTTP + # hop. See disagg_utils.get_global_disagg_request_id. + return get_global_disagg_request_id(self._node_id, self._process_id) + + async def is_ready(self) -> bool: + return self._is_ready + + async def _apply_cluster_info(self, info: Dict[str, Any]) -> None: + self._is_ready = info.get("is_ready", False) + self._sync_delegating_router_configs(info) + await self._sync_stateless_routers(info) + + def _sync_delegating_router_configs(self, info: Dict[str, Any]) -> None: + configs = info.get("routing_key_configs", {}) + for role, router in (("context", self._ctx_router), ("generation", self._gen_router)): + config = configs.get(role) + local = getattr(router, "_local", None) + if config is not None and isinstance(local, KvCacheAwareRouter): + local.set_routing_key_config(config) + + async def _sync_stateless_routers(self, info: Dict[str, Any]) -> None: + server_lists = info.get("server_lists") + if server_lists is None: + return + router_servers = ( + (self._ctx_router, server_lists.get("context", [])), + (self._gen_router, server_lists.get("generation", [])), + ) + for router, servers in router_servers: + if isinstance(router, CoordinatorDelegatingRouter): + continue + desired = set(servers) + for server in desired - set(router.servers): + if not await router.add_server(server): + raise RuntimeError( + f"Failed to prepare {server} before adding it to the " + f"{router.server_role.name} router" + ) + await router.prepare_servers(list(desired)) + unprepared = desired - router.prepared_servers + if unprepared: + raise RuntimeError( + f"Servers are not prepared for {router.server_role.name}: {sorted(unprepared)}" + ) + for server in set(router.servers) - desired: + await router.remove_server(server) + + async def cluster_info(self) -> Dict[str, Any]: + try: + async with self.session.get( + f"{self._remote_url}/cluster_info", timeout=self._request_timeout_s + ) as resp: + if resp.status == 200: + return msgpack.unpackb(await resp.read(), raw=False) + except Exception as e: # noqa: BLE001 + logger.warning(f"CoordinatorClient cluster_info failed: {e}") + return {"is_ready": False} + + async def proxy_cluster_storage_request( + self, + method: str, + path: str, + query: list[tuple[str, str]], + body: bytes, + content_type: Optional[str], + ) -> Tuple[bytes, int, Optional[str]]: + """Forward a public HTTP-storage request to the coordinator.""" + headers = {"Content-Type": content_type} if content_type else None + async with self.session.request( + method, + f"{self._remote_url}{path}", + params=query, + data=body, + headers=headers, + timeout=self._request_timeout_s, + ) as resp: + return await resp.read(), resp.status, resp.headers.get("Content-Type") + + async def stop(self) -> None: + if self._sync_task is not None: + self._sync_task.cancel() + await asyncio.gather(self._sync_task, return_exceptions=True) + self._sync_task = None + if self._session is not None: + await self._session.close() + self._session = None + await self._ctx_router.close() + await self._gen_router.close() diff --git a/tensorrt_llm/serve/openai_client.py b/tensorrt_llm/serve/openai_client.py index 6acbefe103d7..89bb75c0dc1f 100644 --- a/tensorrt_llm/serve/openai_client.py +++ b/tensorrt_llm/serve/openai_client.py @@ -14,10 +14,11 @@ # yapf: disable import asyncio +import json import os import traceback from abc import ABC, abstractmethod -from typing import Any, AsyncGenerator, Callable, Dict, List, Optional, Tuple, Type +from typing import Any, AsyncGenerator, Awaitable, Callable, List, Optional, Tuple, Type import aiohttp @@ -31,7 +32,13 @@ UCompletionRequest, UCompletionResponse, ) -from tensorrt_llm.serve.perf_metrics import ClientMetricsCollector +from tensorrt_llm.serve.perf_metrics import ( + _PERF_METRICS_HEADER_BUDGET_BYTES, + RETURN_METRICS_HEADER, + SSE_METRICS_EVENT, + ClientMetricsCollector, + build_metrics_record_from_headers, +) from tensorrt_llm.serve.responses_utils import ( ResponseHooks, UCompletionResponseOrGenerator, @@ -58,20 +65,30 @@ _msgpack_encoder = msgspec.msgpack.Encoder() +def _metrics_phase(role: ServerRole) -> str: + return "ctx" if role is ServerRole.CONTEXT else "gen" + + class OpenAIClient(ABC): async def send_request( self, request: UCompletionRequest, server: Optional[str] = None, hooks: Optional[ResponseHooks] = None, + req_id: Optional[int] = None, ) -> UCompletionResponseOrGenerator: if isinstance(request, CompletionRequest): return await self._send_request( - "v1/completions", request, CompletionResponse, server, hooks + "v1/completions", request, CompletionResponse, server, hooks, req_id ) elif isinstance(request, ChatCompletionRequest): return await self._send_request( - "v1/chat/completions", request, ChatCompletionResponse, server, hooks + "v1/chat/completions", + request, + ChatCompletionResponse, + server, + hooks, + req_id, ) else: raise ValueError(f"Invalid request type: {type(request)}") @@ -84,6 +101,7 @@ async def _send_request( response_type: Type[UCompletionResponse], server: Optional[str] = None, hooks: Optional[ResponseHooks] = None, + req_id: Optional[int] = None, ) -> UCompletionResponseOrGenerator: """Send a request to the server and return the response and the body generator. @@ -91,9 +109,6 @@ async def _send_request( """ ... - @abstractmethod - async def collect_metrics(self) -> Dict[str, Any]: ... - @abstractmethod async def check_ready(self) -> Tuple[List[str], List[str]]: """Return the list of ready servers and the list of unready servers.""" @@ -102,7 +117,12 @@ async def check_ready(self) -> Tuple[List[str], List[str]]: async def shutdown(self) -> None: ... @abstractmethod - async def _finish_request(self, request: UCompletionRequest, success: bool = True) -> None: + async def _finish_request( + self, + request: UCompletionRequest, + success: bool = True, + req_id: Optional[int] = None, + ) -> None: """Finish the request in the router. ``success`` lets the router distinguish completed vs failed requests @@ -120,7 +140,8 @@ def __init__( max_retries: int = 1, retry_interval_sec: int = 1, session: Optional[aiohttp.ClientSession] = None, - disagg_id_generator: Optional[Callable[[], int]] = None, + disagg_id_generator: Optional[Callable[[], Awaitable[int]]] = None, + request_perf_metrics: bool = False, ): self._router = router self._role = role @@ -134,10 +155,12 @@ def __init__( keepalive_timeout=1, ), timeout=aiohttp.ClientTimeout(total=timeout_secs), + max_field_size=_PERF_METRICS_HEADER_BUDGET_BYTES, ) self._max_retries = max_retries self._retry_interval_sec = retry_interval_sec self._disagg_id_generator = disagg_id_generator + self._request_perf_metrics = request_perf_metrics async def _send_request( self, @@ -146,9 +169,13 @@ async def _send_request( response_type: Type[UCompletionResponse], server: Optional[str] = None, hooks: Optional[ResponseHooks] = None, + req_id: Optional[int] = None, ) -> UCompletionResponseOrGenerator: if server is None: - server, _ = await self._router.get_next_server(request) + if req_id is None: + server, _ = await self._router.get_next_server(request) + else: + server, _ = await self._router.get_next_server(request, req_id=req_id) url = f"http://{server}/{endpoint}" # disaggregated_params is None when conditional_disagg bypasses ctx. _dp = request.disaggregated_params @@ -156,7 +183,7 @@ async def _send_request( logger.debug(f"Sending {self._role} request {_ctx_rid} to {url}") try: self._metrics_collector.total_requests.inc() - resp_generator = self._post_with_retry(server, url, request, hooks) + resp_generator = self._post_with_retry(server, url, request, hooks, req_id) if request.stream: # return the response generator, the request is not done yet return resp_generator @@ -175,7 +202,7 @@ async def _send_request( except Exception: self._metrics_collector.error_requests.inc() # finish the request upon error - await self._finish_request(request, success=False) + await self._finish_request(request, success=False, req_id=req_id) raise async def _post_with_retry( @@ -184,6 +211,7 @@ async def _post_with_retry( url: str, request: UCompletionRequest, hooks: Optional[ResponseHooks] = None, + req_id: Optional[int] = None, ) -> AsyncGenerator[Any, None]: is_stream = request.stream # Loop range must cover the transient-TCP extended budget (up to 5) @@ -197,7 +225,9 @@ async def _post_with_retry( if attempt > 0 and self._disagg_id_generator is not None: dp = getattr(request, "disaggregated_params", None) if dp is not None and getattr(dp, "disagg_request_id", None) is not None: - dp.disagg_request_id = self._disagg_id_generator() + dp.disagg_request_id = await self._disagg_id_generator() + if hooks: + hooks.on_disagg_request_id(dp.disagg_request_id) # Serialize once on the orchestrator's single event-loop thread. if _MSGSPEC_ENABLED: # msgspec msgpack: encode the request dict to msgpack bytes. Keep @@ -210,6 +240,8 @@ async def _post_with_retry( else: body = request.model_dump_json(exclude_unset=True) req_headers = {"Content-Type": "application/json"} + if self._request_perf_metrics: + req_headers[RETURN_METRICS_HEADER] = "1" try: lines_yielded = 0 start_time = get_steady_clock_now_in_seconds() @@ -219,6 +251,27 @@ async def _post_with_retry( headers=req_headers, ) as http_response: content_type = http_response.headers.get("Content-Type", "") + if self._request_perf_metrics: + role = _metrics_phase(self._role) + disagg_params = getattr(request, "disaggregated_params", None) + request_id = "" + if disagg_params is not None: + request_id = str( + disagg_params.disagg_request_id + or disagg_params.ctx_request_id + or "" + ) + response_metrics = build_metrics_record_from_headers( + http_response.headers, + role, + request_id=request_id, + ) + if hooks and response_metrics: + hooks.on_perf_metrics( + server, + role, + response_metrics, + ) if not is_stream and "text/event-stream" in content_type: raise ValueError( "Received an event-stream although request stream was False" @@ -227,7 +280,7 @@ async def _post_with_retry( # do NOT return generator directly here or the response will go # out of scope and get destroyed async for line in self._response_generator( - request, http_response, start_time, server, hooks + request, http_response, start_time, server, hooks, req_id ): lines_yielded += 1 yield line @@ -246,7 +299,7 @@ async def _post_with_retry( # yield here since python forbids return statements in async generators yield response_dict # finish the request after the successful response - await self._finish_request(request) + await self._finish_request(request, req_id=req_id) self._metrics_collector.complete_latency_seconds.observe( get_steady_clock_now_in_seconds() - start_time ) @@ -296,6 +349,7 @@ async def _response_generator( start_time: float, server: str, hooks: Optional[ResponseHooks] = None, + req_id: Optional[int] = None, ) -> AsyncGenerator[Any, None]: assert request.stream, "Request is not streaming" assert "text/event-stream" in http_response.headers.get("Content-Type", ""), ( @@ -304,24 +358,64 @@ async def _response_generator( success = True try: last_token_time = start_time - i = 0 - async for line in http_response.content.iter_any(): + chunk_count = 0 + marker = f"event: {SSE_METRICS_EVENT}\n".encode() + pending = b"" + metrics_event = b"" + async for chunk in http_response.content.iter_any(): now_time = get_steady_clock_now_in_seconds() - if line: - if i == 0: - if hooks: - hooks.on_first_token(server, request) - self._metrics_collector.first_token_latency_seconds.observe( - now_time - last_token_time - ) - else: - self._metrics_collector.per_token_latency_seconds.observe( - now_time - last_token_time - ) - i += 1 - yield line + if chunk: + if chunk_count == 0 and hooks: + hooks.on_first_token(server, request) + latency = now_time - last_token_time + metric = ( + self._metrics_collector.first_token_latency_seconds + if chunk_count == 0 + else self._metrics_collector.per_token_latency_seconds + ) + metric.observe(latency) + chunk_count += 1 + last_token_time = now_time + + if not self._request_perf_metrics: + yield chunk await asyncio.sleep(0) - last_token_time = now_time + continue + + if metrics_event: + metrics_event += chunk + continue + pending += chunk + marker_index = pending.find(marker) + if marker_index >= 0: + if marker_index: + yield pending[:marker_index] + metrics_event = pending[marker_index:] + pending = b"" + continue + emit_size = len(pending) - len(marker) + 1 + if emit_size > 0: + yield pending[:emit_size] + pending = pending[emit_size:] + await asyncio.sleep(0) + + if pending: + yield pending + if hooks and metrics_event: + data_prefix = b"data: " + data_index = metrics_event.find(data_prefix) + if data_index >= 0: + data = metrics_event[data_index + len(data_prefix) :].split(b"\n", 1)[0] + try: + headers = json.loads(data) + metrics = build_metrics_record_from_headers( + headers, _metrics_phase(self._role) + ) + except (TypeError, ValueError) as error: + logger.warning("Ignoring malformed perf metrics event: %s", error) + metrics = None + if metrics: + hooks.on_perf_metrics(server, _metrics_phase(self._role), metrics) if hooks: hooks.on_resp_done(server, request, None) @@ -340,22 +434,21 @@ async def _response_generator( raise finally: # finish the request after streaming response is done or error is raised - await self._finish_request(request, success=success) + await self._finish_request(request, success=success, req_id=req_id) - async def _finish_request(self, request: UCompletionRequest, success: bool = True) -> None: + async def _finish_request( + self, + request: UCompletionRequest, + success: bool = True, + req_id: Optional[int] = None, + ) -> None: self._metrics_collector.completed_requests.inc() - await self._router.finish_request(request, self._session, success=success) - - async def collect_metrics(self) -> Dict[str, Any]: - metrics = {} - for server in self._router.servers: - try: - async with self._session.get(f"http://{server}/perf_metrics") as response: - metrics[server] = await response.json() - except Exception: - logger.error(f"Failed to collect metrics from {server}") - continue - return metrics + if req_id is None: + await self._router.finish_request(request, self._session, success=success) + else: + await self._router.finish_request( + request, self._session, success=success, req_id=req_id + ) async def shutdown(self) -> None: await self._session.close() diff --git a/tensorrt_llm/serve/openai_disagg_server.py b/tensorrt_llm/serve/openai_disagg_server.py index 051bb4a1c8cf..23c2978cf337 100644 --- a/tensorrt_llm/serve/openai_disagg_server.py +++ b/tensorrt_llm/serve/openai_disagg_server.py @@ -27,58 +27,114 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, Response, StreamingResponse -# yapf: disable from tensorrt_llm.executor import CppExecutorError from tensorrt_llm.executor.executor import CppExecutorError from tensorrt_llm.llmapi import tracing from tensorrt_llm.llmapi.disagg_utils import (DisaggServerConfig, - MetadataServerConfig, ServerRole, - get_ctx_gen_server_addrs, - get_global_disagg_request_id) + MetadataServerConfig, ServerRole) from tensorrt_llm.logger import logger from tensorrt_llm.serve.cluster_storage import ( HttpClusterStorageServer, create_cluster_storage, validate_http_cluster_storage_scope) from tensorrt_llm.serve.conversation_id import resolve_request_conversation_id -from tensorrt_llm.serve.metadata_server import create_metadata_server +from tensorrt_llm.serve.disagg_coordinator import (CoordinatorClient, + DisaggCoordinatorService) from tensorrt_llm.serve.openai_client import OpenAIClient, OpenAIHttpClient from tensorrt_llm.serve.openai_disagg_service import ( OpenAIDisaggregatedService, ResponseHooks) from tensorrt_llm.serve.openai_protocol import ( - UCompletionRequest, UCompletionResponse, - ensure_request_chat_template_allowed) -from tensorrt_llm.serve.perf_metrics import DisaggPerfMetricsCollector + ChatCompletionRequest, CompletionRequest, UCompletionRequest, + UCompletionResponse, ensure_request_chat_template_allowed) +from tensorrt_llm.serve.perf_metrics import (DisaggPerfMetricsCollector, + PerfMetricsJsonlWriter, + PerfMetricsMiddleware, + combine_disagg_metrics) from tensorrt_llm.serve.responses_utils import (ServerArrivalTimeMiddleware, get_steady_clock_now_in_seconds) -from tensorrt_llm.serve.router import Router, create_router +from tensorrt_llm.serve.router import Router from tensorrt_llm.version import __version__ as VERSION # yapf: enale TIMEOUT_KEEP_ALIVE = 10 # seconds. +_LOG_CONTROL_CHARACTERS = { + code: f"\\x{code:02x}" + for code in (*range(32), 127) +} class RawRequestResponseHooks(ResponseHooks): - def __init__(self, raw_req: Request, perf_metrics_collector: DisaggPerfMetricsCollector): + def __init__(self, raw_req: Request, queue_latency_metric, + collect_perf_metrics: bool): self.raw_req = raw_req + self.queue_latency_metric = queue_latency_metric + self.collect_perf_metrics = collect_perf_metrics self.ctx_server = "" self.gen_server = "" + self.request_id = "" + self.disagg_request_id = None self.request_arrival_time = raw_req.state.server_arrival_time self.server_first_token_time = 0 - self.perf_metrics_collector = perf_metrics_collector + self.ctx_dispatch_time = 0 + self.ctx_metrics = None + self.gen_metrics = None def on_req_begin(self, request: UCompletionRequest): - self.perf_metrics_collector.queue_latency_seconds.observe(get_steady_clock_now_in_seconds() - self.request_arrival_time) + params = request.disaggregated_params + if params is not None: + self.disagg_request_id = params.disagg_request_id + request_id = params.disagg_request_id or params.ctx_request_id + self.request_id = str(request_id or "") + self.queue_latency_metric.observe( + get_steady_clock_now_in_seconds() - self.request_arrival_time) + + def on_disagg_request_id(self, disagg_request_id: int): + self.disagg_request_id = disagg_request_id + self.request_id = str(disagg_request_id) + + def on_ctx_dispatch(self, request: UCompletionRequest): + self.ctx_dispatch_time = get_steady_clock_now_in_seconds() + + def on_perf_metrics(self, server: str, role: str, metrics: dict): + if role == "ctx": + self.ctx_server = server + self.ctx_metrics = metrics + elif role == "gen": + self.gen_server = server + self.gen_metrics = metrics def on_ctx_resp(self, ctx_server: str, response: UCompletionResponse): self.ctx_server = ctx_server - def on_first_token(self, gen_server: str, request: UCompletionRequest, response: UCompletionResponse = None): + def on_first_token( + self, gen_server: str, request: UCompletionRequest, + response: UCompletionResponse = None): self.gen_server = gen_server self.server_first_token_time = get_steady_clock_now_in_seconds() - def on_resp_done(self, gen_server: str, request: UCompletionRequest, response: UCompletionResponse = None): - if request.disaggregated_params: - ctx_req_id = request.disaggregated_params.ctx_request_id - asyncio.create_task(self.perf_metrics_collector.add_per_request_metrics(self.ctx_server, gen_server, ctx_req_id, self.raw_req.state.server_arrival_time, self.server_first_token_time)) + def on_resp_done( + self, gen_server: str, request: UCompletionRequest, + response: UCompletionResponse = None): + self.gen_server = gen_server + if not self.collect_perf_metrics: + return + disagg_phase = { + "ctx_server": self.ctx_server, + "gen_server": self.gen_server, + "timing_metrics": { + "arrival_time": self.request_arrival_time, + "last_token_time": get_steady_clock_now_in_seconds(), + "server_arrival_time": self.request_arrival_time, + "ctx_dispatch_time": self.ctx_dispatch_time or None, + "server_first_token_time": self.server_first_token_time or None, + }, + } + self.raw_req.state.perf_metrics_records.append( + combine_disagg_metrics( + self.request_id, + disagg_phase, + self.ctx_metrics, + self.gen_metrics, + disagg_request_id=self.disagg_request_id, + )) class OpenAIDisaggServer: @@ -87,7 +143,8 @@ def __init__(self, req_timeout_secs: int = 180, server_start_timeout_secs: int = 180, metadata_server_cfg: Optional[MetadataServerConfig] = None, - metrics_interval_secs: int = 0): + metrics_interval_secs: int = 0, + coordinator_url: Optional[str] = None): self._config = config self._req_timeout_secs = req_timeout_secs self._server_start_timeout_secs = server_start_timeout_secs @@ -95,12 +152,19 @@ def __init__(self, self._metrics_interval_secs = metrics_interval_secs self._allow_request_chat_template = getattr( config, "allow_request_chat_template", False) - - self._ctx_servers, self._gen_servers = get_ctx_gen_server_addrs(config.server_configs) - self._ctx_router = create_router(config.ctx_router_config, self._ctx_servers, metadata_server_cfg, create_metadata_server(metadata_server_cfg), self._sync_server_clock, disagg_node_id=config.node_id) - self._gen_router = create_router(config.gen_router_config, self._gen_servers, metadata_server_cfg, create_metadata_server(metadata_server_cfg), self._sync_server_clock, disagg_node_id=config.node_id) - self._metadata_server = create_metadata_server(metadata_server_cfg) - self._perf_metrics_collector = DisaggPerfMetricsCollector(config.perf_metrics_max_requests) + # When set, this is a forked worker: routing/readiness are delegated to + # the coordinator at coordinator_url (CoordinatorClient). Otherwise this + # process owns the routers + cluster state (DisaggCoordinatorService). + self._coordinator_url = coordinator_url + + self._perf_metrics_collector = DisaggPerfMetricsCollector( + config.perf_metrics_max_requests) + self._expose_perf_metrics = config.return_perf_metrics + self._collect_perf_metrics = ( + config.return_perf_metrics + or config.perf_metrics_output_dir is not None) + self._perf_metrics_writer = PerfMetricsJsonlWriter( + config.perf_metrics_output_dir, "disagg") self._disagg_cluster_storage = None if config.disagg_cluster_config: @@ -109,15 +173,26 @@ def __init__(self, self._disagg_cluster_storage = create_cluster_storage( config.disagg_cluster_config.cluster_uri, config.disagg_cluster_config.cluster_name) + # The server doesn't build routers -- the coordinator object does: + # DisaggCoordinatorService (owner) or CoordinatorClient (delegating). The + # server just reads .ctx_router / .gen_router off whichever it holds. + if self._coordinator_url: + self._coordinator = CoordinatorClient( + self._coordinator_url, self._config, metadata_server_cfg, + request_timeout_s=self._req_timeout_secs, + startup_timeout_s=self._server_start_timeout_secs) + else: + self._coordinator = DisaggCoordinatorService( + self._config, self._create_client, + metadata_config=self._metadata_server_cfg, + server_preparation_func=self._sync_server_clock, + server_start_timeout_secs=self._server_start_timeout_secs) + self._ctx_router = self._coordinator.ctx_router + self._gen_router = self._coordinator.gen_router self._service = OpenAIDisaggregatedService( - self._config, self._ctx_router, self._gen_router, self._create_client, - metadata_server=self._metadata_server, - metadata_config=self._metadata_server_cfg, - req_timeout_secs=self._req_timeout_secs, - server_start_timeout_secs=self._server_start_timeout_secs, - perf_metrics_collector=self._perf_metrics_collector, - disagg_cluster_storage=self._disagg_cluster_storage) + self._config, self._coordinator, self._create_client, + req_timeout_secs=self._req_timeout_secs) try: otlp_cfg = config.otlp_config @@ -132,44 +207,102 @@ def __init__(self, @asynccontextmanager async def lifespan(app) -> None: - # Prepare servers (sync server clock) when static ctx/gen server list is used - await self._ctx_router.prepare_servers() - await self._gen_router.prepare_servers() + # The cluster manager (via setup) owns server preparation + monitoring. + await self._perf_metrics_writer.start() await self._service.setup() yield await self._service.teardown() + await self._perf_metrics_writer.close() self.app = FastAPI(lifespan=lifespan) + if self._collect_perf_metrics: + self.app.add_middleware( + PerfMetricsMiddleware, + expose_headers=self._expose_perf_metrics, + writer=self._perf_metrics_writer) self.app.add_middleware(ServerArrivalTimeMiddleware) + # Log request-body validation failures so a client/server schema mismatch + # shows up server-side. Throttled (first, then every 1000th) to avoid + # flooding the event loop when every request fails identically. + self._val_err_n = 0 @self.app.exception_handler(RequestValidationError) - async def validation_exception_handler(_, exc): + async def validation_exception_handler(request: Request, exc): self._perf_metrics_collector.validation_exceptions.inc() + self._val_err_n += 1 + if self._val_err_n == 1 or self._val_err_n % 1000 == 0: + try: + errs = exc.errors() + # Compact: [{loc, type, msg}] -- drops the (large) echoed input. + brief = [{"loc": e.get("loc"), "type": e.get("type"), + "msg": e.get("msg")} for e in errs][:8] + except Exception: # noqa: BLE001 + brief = str(exc)[:500] + method = request.method.translate(_LOG_CONTROL_CHARACTERS) + path = request.url.path.translate(_LOG_CONTROL_CHARACTERS) + logger.warning( + f"[validation] {method} {path} 400 " + f"(n={self._val_err_n}): {brief}") return JSONResponse(status_code=400, content={"error": str(exc)}) self.register_routes() def _create_client(self, router: Router, role: ServerRole, max_retries: int = 1) -> OpenAIClient: - node_id = self._config.node_id - client = OpenAIHttpClient( + async def disagg_id_generator(): + return await self._coordinator.get_disagg_request_id() + return OpenAIHttpClient( router, role, self._req_timeout_secs, max_retries, - disagg_id_generator=lambda: get_global_disagg_request_id(node_id)) - self._perf_metrics_collector.add_client(client) - return client + disagg_id_generator=disagg_id_generator, + request_perf_metrics=self._collect_perf_metrics) def register_routes(self): - self.app.add_api_route("/v1/completions", self._wrap_entry_point(self._service.openai_completion), methods=["POST"]) - self.app.add_api_route("/v1/chat/completions", self._wrap_entry_point(self._service.openai_chat_completion), methods=["POST"]) + # The disagg service owns only the request-serving endpoints (/v1/*) and + # perf metrics. Readiness / cluster topology are the coordinator's state, + # so /health and /cluster_info hook straight to self._coordinator. + self.app.add_api_route("/v1/completions", self._wrap_entry_point(self._service.openai_completion, CompletionRequest), methods=["POST"]) + self.app.add_api_route("/v1/chat/completions", self._wrap_entry_point(self._service.openai_chat_completion, ChatCompletionRequest), methods=["POST"]) self.app.add_api_route("/health", self.health, methods=["GET"]) self.app.add_api_route("/cluster_info", self.cluster_info, methods=["GET"]) self.app.add_api_route("/version", self.version, methods=["GET"]) - self.app.add_api_route("/perf_metrics", self._perf_metrics_collector.get_perf_metrics, methods=["GET"]) # import prometheus_client lazily to break the `set_prometheus_multiproc_dir` - from prometheus_client import make_asgi_app - self.app.mount("/prometheus/metrics", make_asgi_app()) - if self._disagg_cluster_storage and isinstance(self._disagg_cluster_storage, HttpClusterStorageServer): - self._disagg_cluster_storage.add_routes(self.app) + from prometheus_client import (CollectorRegistry, make_asgi_app, + multiprocess) + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + self.app.mount("/prometheus/metrics", make_asgi_app(registry=registry)) + # Single-process (local coordinator): mount the in-process HTTP cluster + # storage routes on this app. In worker mode the coordinator is remote and + # owns those routes (CoordinatorClient has no cluster_storage). + cluster_storage = getattr(self._coordinator, "cluster_storage", None) + if isinstance(cluster_storage, HttpClusterStorageServer): + cluster_storage.add_routes(self.app) + elif (isinstance(self._coordinator, CoordinatorClient) + and isinstance(self._disagg_cluster_storage, + HttpClusterStorageServer)): + # Keep the configured public cluster_uri valid in fleet mode while + # the coordinator remains the sole owner of the HTTP storage state. + for path, method in (("/set", "POST"), ("/get", "GET"), + ("/delete", "DELETE"), ("/expire", "GET"), + ("/get_prefix", "GET")): + self.app.add_api_route(path, + self._proxy_cluster_storage_request, + methods=[method]) + + async def _proxy_cluster_storage_request(self, + raw_req: Request) -> Response: + try: + body, status, content_type = ( + await self._coordinator.proxy_cluster_storage_request( + raw_req.method, raw_req.url.path, + list(raw_req.query_params.multi_items()), + await raw_req.body(), raw_req.headers.get("Content-Type"))) + except (aiohttp.ClientError, asyncio.TimeoutError, OSError) as e: + logger.warning(f"Failed to proxy cluster storage request: {e}") + return JSONResponse(status_code=502, + content={"error": "coordinator unavailable"}) + headers = {"Content-Type": content_type} if content_type else None + return Response(content=body, status_code=status, headers=headers) @staticmethod def _extract_conversation_id(req: UCompletionRequest, raw_req: Request): @@ -180,8 +313,13 @@ def _extract_conversation_id(req: UCompletionRequest, raw_req: Request): """ resolve_request_conversation_id(req, raw_req.headers) - def _wrap_entry_point(self, entry_point: Callable) -> Callable: - async def wrapper(req: UCompletionRequest, raw_req: Request) -> Response: + def _wrap_entry_point(self, entry_point: Callable, request_type: type = UCompletionRequest) -> Callable: + # Bind the concrete request model per route so FastAPI validates against it. + # The bare Union UCompletionRequest (no discriminator) makes Pydantic try + # CompletionRequest first and 400 every chat body, so override the wrapper's + # annotation with request_type (as openai_server.py does). + @tracing.trace_span("disaggregated_request") + async def wrapper(req: request_type, raw_req: Request) -> Response: try: self._perf_metrics_collector.total_requests.inc() if req.stream: @@ -194,13 +332,16 @@ async def wrapper(req: UCompletionRequest, raw_req: Request) -> Response: except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) from e self._extract_conversation_id(req, raw_req) - hooks = RawRequestResponseHooks(raw_req, self._perf_metrics_collector) + hooks = RawRequestResponseHooks( + raw_req, self._perf_metrics_collector.queue_latency_seconds, + self._collect_perf_metrics) response_or_generator = await entry_point(req, hooks) self._perf_metrics_collector.total_responses.inc() if req.stream: - return StreamingResponse(content=response_or_generator, media_type="text/event-stream") - else: - return JSONResponse(content=response_or_generator.model_dump()) + return StreamingResponse( + content=response_or_generator, + media_type="text/event-stream") + return JSONResponse(content=response_or_generator.model_dump()) except Exception as e: self._handle_exception(e) return wrapper @@ -220,12 +361,12 @@ def _handle_exception(self, exception): async def health(self) -> Response: - if not await self._service.is_ready(): - return Response(status_code=500) + if not await self._coordinator.is_ready(): + return Response(status_code=503) return Response(status_code=200) async def cluster_info(self) -> JSONResponse: - return JSONResponse(content=await self._service.cluster_info()) + return JSONResponse(content=await self._coordinator.cluster_info()) async def version(self) -> JSONResponse: return JSONResponse(content={"version": VERSION}) diff --git a/tensorrt_llm/serve/openai_disagg_service.py b/tensorrt_llm/serve/openai_disagg_service.py index 286bf5661bd9..8450a2c0f1a6 100644 --- a/tensorrt_llm/serve/openai_disagg_service.py +++ b/tensorrt_llm/serve/openai_disagg_service.py @@ -14,20 +14,11 @@ import asyncio import os -from typing import Any, Callable, Dict, Optional - -from tensorrt_llm.llmapi.disagg_utils import ( - ConditionalDisaggConfig, - DisaggClusterConfig, - DisaggServerConfig, - MetadataServerConfig, - ServerRole, - get_global_disagg_request_id, -) +from typing import Callable, Optional + +from tensorrt_llm.llmapi.disagg_utils import ConditionalDisaggConfig, DisaggServerConfig, ServerRole from tensorrt_llm.logger import logger -from tensorrt_llm.serve.cluster_storage import ClusterStorage, WatchEventType -from tensorrt_llm.serve.disagg_auto_scaling import DisaggClusterManager, WorkerInfo -from tensorrt_llm.serve.metadata_server import JsonDictionary +from tensorrt_llm.serve.disagg_coordinator import DisaggCoordinator from tensorrt_llm.serve.openai_client import OpenAIClient from tensorrt_llm.serve.openai_protocol import ( ChatCompletionRequest, @@ -38,13 +29,12 @@ UCompletionResponse, ) from tensorrt_llm.serve.openai_service import OpenAIService -from tensorrt_llm.serve.perf_metrics import DisaggPerfMetricsCollector from tensorrt_llm.serve.responses_utils import ( ResponseHooks, UCompletionResponseOrGenerator, done_generator, ) -from tensorrt_llm.serve.router import KvCacheAwareRouter, Router +from tensorrt_llm.serve.router import CoordinatorDelegatingRouter, KvCacheAwareRouter, Router # Finish reasons for which a GEN handoff is still pending; any other reason means # the CTX request already completed and the disagg KV-cache handoff was never set up. @@ -55,28 +45,19 @@ class OpenAIDisaggregatedService(OpenAIService): def __init__( self, config: DisaggServerConfig, - ctx_router: Router, - gen_router: Router, + coordinator: "DisaggCoordinator", client_factory: Callable[[Router, ServerRole], OpenAIClient], - metadata_server: Optional[JsonDictionary] = None, - metadata_config: Optional[MetadataServerConfig] = None, req_timeout_secs: int = 180, - server_start_timeout_secs: int = 180, - perf_metrics_collector: Optional[DisaggPerfMetricsCollector] = None, - disagg_cluster_storage: Optional[ClusterStorage] = None, - health_check_interval_secs: int = 3, ): self._config = config - self._ctx_router = ctx_router - self._gen_router = gen_router + # The service drives the coordinator's ctx/gen routers uniformly, so serving + # is identical whether the router is the real one (single-process) or a + # delegating one that forwards placement to a remote coordinator (worker). + self._coordinator = coordinator + self._ctx_router = coordinator.ctx_router + self._gen_router = coordinator.gen_router self._client_factory = client_factory - self._metadata_server = metadata_server - self._metadata_config = metadata_config self._req_timeout_secs = req_timeout_secs - self._server_start_timeout_secs = server_start_timeout_secs - self._perf_metrics_collector = perf_metrics_collector - self._cluster_storage = disagg_cluster_storage - self._health_check_interval_secs = health_check_interval_secs # Opt-in body-shrink for generation_only requests; see _get_gen_request. self._strip_gen_message_history = config.gen_strip_message_history # Opt-in: ask context workers to return prompt_token_ids as base64 int32. @@ -84,7 +65,6 @@ def __init__( self._ctx_client = None self._gen_client = None - self._disagg_cluster_manager = None self._schedule_style = DisaggScheduleStyle.CONTEXT_FIRST match self._config.schedule_style: @@ -139,26 +119,45 @@ async def _send_disagg_request_ctx_first( hooks.on_req_begin(request) # empty server means client decides which server to use ctx_server = None + disagg_request_id = await self._coordinator.get_disagg_request_id() + if hooks: + hooks.on_disagg_request_id(disagg_request_id) # reserve a gen_server if conditional disagg is needed - gen_server, need_ctx = await self._check_conditional_disagg(request) + gen_server, need_ctx = await self._check_conditional_disagg(request, disagg_request_id) + # Context retries may replace disagg_request_id for the KV-transfer + # handshake. Keep the ID used to reserve the generation server separate + # so its coordinator-side load is released under the original key. + gen_reservation_id = disagg_request_id if gen_server else None need_ctx = need_ctx and not await self._check_gen_only_disagg(request) ctx_response = None gen_req = request - disagg_request_id = get_global_disagg_request_id(self._config.node_id) if need_ctx: - ctx_req = self._get_ctx_request(request, disagg_request_id) - # ctx generator is empty - ctx_server, _ = await self._ctx_router.get_next_server( - ctx_req, exclude_server=gen_server - ) - ctx_response = await self._ctx_client.send_request( - ctx_req, server=ctx_server, hooks=hooks - ) - await self._verify_ctx_response(ctx_response) - ctx_response_disagg_params = ctx_response.choices[0].disaggregated_params - if ctx_response_disagg_params.disagg_request_id is not None: - disagg_request_id = ctx_response_disagg_params.disagg_request_id - gen_req = self._get_gen_request(request, ctx_response, disagg_request_id) + try: + # Mark ctx-dispatch start: arrival->here is the pre-ctx wait in the + # orchestrator/fleet (accept queue + event loop + pipeline). + if hooks: + hooks.on_ctx_dispatch(request) + ctx_req = self._get_ctx_request(request, disagg_request_id) + # ctx generator is empty + ctx_server, _ = await self._ctx_router.get_next_server( + ctx_req, exclude_server=gen_server, req_id=disagg_request_id + ) + ctx_response = await self._ctx_client.send_request( + ctx_req, server=ctx_server, hooks=hooks, req_id=disagg_request_id + ) + await self._verify_ctx_response(ctx_response) + ctx_response_disagg_params = ctx_response.choices[0].disaggregated_params + if ctx_response_disagg_params.disagg_request_id is not None: + disagg_request_id = ctx_response_disagg_params.disagg_request_id + if hooks: + hooks.on_disagg_request_id(disagg_request_id) + gen_req = self._get_gen_request(request, ctx_response, disagg_request_id) + except Exception: + if gen_server: + await self._gen_router.finish_request( + request, success=False, req_id=gen_reservation_id + ) + raise else: # When need_ctx=False the gen server handles full generation and # must not see a stale request_type="context_only". @@ -172,13 +171,18 @@ async def _send_disagg_request_ctx_first( if ctx_response is None or self._need_gen(ctx_response): if not gen_server: gen_server, _ = await self._gen_router.get_next_server( - gen_req, exclude_server=ctx_server + gen_req, exclude_server=ctx_server, req_id=disagg_request_id ) + gen_reservation_id = disagg_request_id gen_response = await self._gen_client.send_request( - gen_req, server=gen_server, hooks=hooks + gen_req, server=gen_server, hooks=hooks, req_id=gen_reservation_id ) return gen_response else: + if gen_server: + await self._gen_router.finish_request(request, req_id=gen_reservation_id) + if hooks: + hooks.on_resp_done("", request, ctx_response) if request.stream: # ctx client will never return a generator when streaming is requested # make up for this by returning a done generator @@ -277,14 +281,22 @@ def _get_gen_request( request.disaggregated_params.disagg_request_id = disagg_request_id return request - async def _check_conditional_disagg(self, request: UCompletionRequest) -> bool: + async def _check_conditional_disagg(self, request: UCompletionRequest, req_id: int) -> bool: if self.conditional_disagg_config: - assert isinstance(self._gen_router, KvCacheAwareRouter) + local_gen_router = ( + self._gen_router._local + if isinstance(self._gen_router, CoordinatorDelegatingRouter) + else self._gen_router + ) + if not isinstance(local_gen_router, KvCacheAwareRouter): + raise TypeError( + "conditional disaggregation requires a KV-cache-aware generation router" + ) # Query kv cache status and select a best gen_server. # The server is reserved for generation request - gen_server, info = await self._gen_router.get_next_server(request) - match_length = sum(info["matches"]) - total_length = sum(len(token_list) for token_list in info["token_lists"]) + gen_server, info = await self._gen_router.get_next_server(request, req_id=req_id) + match_length = info["match_length"] + total_length = info["num_tokens"] need_ctx_decision = ( match_length == 0 or total_length - match_length @@ -315,110 +327,33 @@ async def _check_gen_only_disagg(self, request: UCompletionRequest) -> bool: return True return False - async def cluster_info(self) -> Dict[str, Any]: - cluster_info = {"is_ready": await self.is_ready()} - if self._disagg_cluster_manager: - cluster_info.update(await self._disagg_cluster_manager.cluster_info()) - return cluster_info - async def is_ready(self) -> bool: - if self._disagg_cluster_manager: - return await self._disagg_cluster_manager.is_ready_with_router( - self._ctx_router.num_prepared_servers, - self._gen_router.num_prepared_servers, - ) - return True - - @property - def disagg_cluster_config(self) -> Optional[DisaggClusterConfig]: - return self._config.disagg_cluster_config + # Per-request readiness gate for the /v1/ handlers (the server's /health + # and /cluster_info hook the coordinator directly). Cluster topology + # (cluster_info) is the coordinator's concern, not the request service's. + return await self._coordinator.is_ready() @property def conditional_disagg_config(self) -> Optional[ConditionalDisaggConfig]: return self._config.conditional_disagg_config async def setup(self) -> None: + # Build the request-sending clients from the coordinator's routers and share + # them with the coordinator service so its readiness checks use the same pool. self._ctx_client = self._client_factory( self._ctx_router, ServerRole.CONTEXT, self._config.max_retries ) self._gen_client = self._client_factory( self._gen_router, ServerRole.GENERATION, self._config.max_retries ) - - if self.disagg_cluster_config and self._cluster_storage: - logger.info("Starting disagg cluster manager") - self._disagg_cluster_manager = DisaggClusterManager( - self.disagg_cluster_config, self._cluster_storage - ) - await self._disagg_cluster_manager.start() - await self._disagg_cluster_manager.watch_workers(on_event=self._on_worker_event) - logger.info("Disagg cluster manager started") - else: - if self._metadata_server and self._metadata_config: - logger.info("Starting server monitoring via metadata service") - await self._ctx_router.start_server_monitoring( - self._metadata_config.refresh_interval - ) - await self._gen_router.start_server_monitoring( - self._metadata_config.refresh_interval - ) - await self._wait_for_all_servers_ready() + if hasattr(self._coordinator, "set_clients"): + self._coordinator.set_clients(self._ctx_client, self._gen_client) + await self._coordinator.start() async def teardown(self) -> None: await self._ctx_client.shutdown() await self._gen_client.shutdown() - - if self._disagg_cluster_manager: - await self._disagg_cluster_manager.stop() - - if self._metadata_server: - await self._ctx_router.stop_server_monitoring() - await self._gen_router.stop_server_monitoring() - - async def _wait_for_all_servers_ready(self) -> None: - # Skip context servers if TRTLLM_DISAGG_BENCHMARK_GEN_ONLY is set - gen_only = os.getenv("TRTLLM_DISAGG_BENCHMARK_GEN_ONLY") == "1" - - async def check_servers_ready(): - elapsed_time = 0 - interval = self._health_check_interval_secs - while elapsed_time < self._server_start_timeout_secs: - if gen_only: - unready_ctx_servers = [] - else: - _, unready_ctx_servers = await self._ctx_client.check_ready() - _, unready_gen_servers = await self._gen_client.check_ready() - if len(unready_ctx_servers) == 0 and len(unready_gen_servers) == 0: - if gen_only: - logger.info("Generation servers are ready (context servers skipped)") - else: - logger.info("All servers are ready") - return - logger.info( - f"Waiting for servers, context: {unready_ctx_servers}, generation: {unready_gen_servers}" - ) - await asyncio.sleep(interval) - elapsed_time += interval - - try: - await asyncio.wait_for(check_servers_ready(), timeout=self._server_start_timeout_secs) - except asyncio.TimeoutError: - raise TimeoutError("Timeout waiting for context and generation servers to be ready") - - async def _on_worker_event(self, worker_info: WorkerInfo, event_type: WatchEventType): - router_map = {ServerRole.CONTEXT: self._ctx_router, ServerRole.GENERATION: self._gen_router} - worker_addr = f"{worker_info.host}:{worker_info.port}" - try: - router = router_map[worker_info.role] - if event_type == WatchEventType.SET: - await router.add_server(worker_addr) - elif event_type == WatchEventType.DELETE: - await router.remove_server(worker_addr) - logger.info(f"Worker {event_type.name} event: {worker_info.worker_id}, {worker_addr}") - except KeyError: - logger.error( - f"Unknown worker role: {worker_info.role}, Worker {worker_info.worker_id} event: {event_type.name}" - ) + await self._coordinator.stop() async def _verify_ctx_response(self, ctx_response: UCompletionResponse) -> None: if ctx_response: @@ -457,9 +392,18 @@ async def _send_disagg_request_gen_first( ctx_server, gen_server = None, None ctx_server_info = None ctx_req, gen_req = None, None - disagg_request_id = get_global_disagg_request_id(self._config.node_id) + # Single-issuer disagg id (see _send_disagg_request_ctx_first): fetch from + # the coordinator so fleet workers never mint colliding ids. + disagg_request_id = await self._coordinator.get_disagg_request_id() + if hooks: + hooks.on_disagg_request_id(disagg_request_id) if need_ctx: - ctx_server, ctx_server_info = await self._ctx_router.get_next_server(request) + # arrival->here = pre-ctx wait in the orchestrator/fleet. + if hooks: + hooks.on_ctx_dispatch(request) + ctx_server, ctx_server_info = await self._ctx_router.get_next_server( + request, req_id=disagg_request_id + ) ctx_req = self._get_ctx_request(request, disagg_request_id) gen_req = self._get_gen_request( request, @@ -479,7 +423,7 @@ async def _send_disagg_request_gen_first( # Fix: eagerly start consuming the gen generator in a background # task so the HTTP POST fires, then pipe chunks through a queue. gen_response = await self._gen_client.send_request( - gen_req, server=gen_server, hooks=hooks + gen_req, server=gen_server, hooks=hooks, req_id=disagg_request_id ) queue: asyncio.Queue = asyncio.Queue() @@ -496,7 +440,12 @@ async def _consume_gen(): # Now send ctx request — gen server has received its request try: - await self._ctx_client.send_request(ctx_req, server=ctx_server, hooks=hooks) + await self._ctx_client.send_request( + ctx_req, + server=ctx_server, + hooks=hooks, + req_id=disagg_request_id, + ) except Exception: consume_task.cancel() try: @@ -530,12 +479,22 @@ async def _yield_from_queue(): if need_ctx: tasks.append( asyncio.create_task( - self._ctx_client.send_request(ctx_req, server=ctx_server, hooks=hooks) + self._ctx_client.send_request( + ctx_req, + server=ctx_server, + hooks=hooks, + req_id=disagg_request_id, + ) ) ) tasks.append( asyncio.create_task( - self._gen_client.send_request(gen_req, server=gen_server, hooks=hooks) + self._gen_client.send_request( + gen_req, + server=gen_server, + hooks=hooks, + req_id=disagg_request_id, + ) ) ) responses = await asyncio.gather(*tasks) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index ba079d7a4637..4f497787e143 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -555,6 +555,15 @@ class CompletionRequest(OpenAIBaseModel): default=None, description=("Parameters for disaggregated serving"), ) + priority: Optional[float] = Field( + default=None, + ge=0.0, + le=1.0, + description= + ("Scheduling priority in [0.0, 1.0]; higher is served first. Only honored " + "when the engine runs with scheduler_config.waiting_queue_policy=priority. " + "If unset, the engine default (0.5) is used."), + ) conversation_params: Optional[ConversationParams] = Field( default=None, description=("Parameters for multi-turn conversation routing"), @@ -954,6 +963,15 @@ class ChatCompletionRequest(OpenAIBaseModel): ("If specified, KV cache will be salted with the provided string " "to limit the kv cache reuse on with the requests having the same string." )) + priority: Optional[float] = Field( + default=None, + ge=0.0, + le=1.0, + description= + ("Scheduling priority in [0.0, 1.0]; higher is served first. Only honored " + "when the engine runs with scheduler_config.waiting_queue_policy=priority. " + "If unset, the engine default (0.5) is used."), + ) agent_hierarchy: Optional[AgentHierarchy] = Field( default=None, description="Agent hierarchy ") diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index eb63305fdcb7..9b6968f35e03 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -35,6 +35,7 @@ # yapf: disable from tensorrt_llm.executor import CppExecutorError from tensorrt_llm.executor.postproc_worker import PostprocParams +from tensorrt_llm.executor.request import DEFAULT_REQUEST_PRIORITY from tensorrt_llm.inputs import prompt_inputs from tensorrt_llm.inputs.data import TokensPrompt from tensorrt_llm.inputs.media_io import BaseMediaIO @@ -57,6 +58,7 @@ from tensorrt_llm.runtime.kv_cache_hash import \ get_effective_kv_cache_event_hash_algo from tensorrt_llm.sampling_params import GuidedDecodingParams, SamplingParams +from tensorrt_llm.serve.chat_tokenization import tokenize_harmony_chat_request from tensorrt_llm.serve.chat_utils import (load_chat_template, parse_chat_messages_coroutines, resolve_top_level_model_type) @@ -78,6 +80,9 @@ ensure_request_chat_template_allowed, to_llm_conversation_params, to_llm_disaggregated_params) from tensorrt_llm.serve.openai_video_routes import _VideoRoutesMixin +from tensorrt_llm.serve.perf_metrics import (PerfMetricsJsonlWriter, + PerfMetricsMiddleware, + build_request_metrics_record) from tensorrt_llm.serve.postprocess_handlers import ( ChatCompletionPostprocArgs, ChatPostprocArgs, CompletionPostprocArgs, ResponsesAPIPostprocArgs, chat_harmony_post_processor, @@ -101,8 +106,7 @@ from tensorrt_llm.visual_gen import VisualGen from .._utils import nvtx_mark, set_prometheus_multiproc_dir -from .harmony_adapter import (HarmonyAdapter, get_harmony_adapter, - maybe_transform_reasoning_effort) +from .harmony_adapter import HarmonyAdapter, get_harmony_adapter # yapf: enable @@ -346,8 +350,21 @@ def __init__( else: self.model = model self.metrics_collector = None - self.perf_metrics = None - self.perf_metrics_lock = None + args = getattr(self.generator, "args", None) + self._expose_perf_metrics = bool( + args and getattr(args, "return_perf_metrics", False)) + perf_metrics_output_dir = (getattr(args, "perf_metrics_output_dir", + None) if args else None) + self._collect_perf_metrics = (self._expose_perf_metrics + or perf_metrics_output_dir is not None) + # AsyncLLM uses this flag to request engine-level snapshots. Preserve the + # original value separately because only it controls public headers. + if self._collect_perf_metrics and args is not None: + args.return_perf_metrics = True + server_kind = server_role.name.lower( + ) if server_role is not None else "server" + self._perf_metrics_writer = PerfMetricsJsonlWriter( + perf_metrics_output_dir, server_kind) self._iteration_stats_collector_task = None self._iteration_stats_wakeup_event = asyncio.Event() # Bounded snapshot of iteration stats for the GET /metrics handler. @@ -377,6 +394,7 @@ def __init__( @asynccontextmanager async def lifespan(app: FastAPI): + await self._perf_metrics_writer.start() if self.metadata_server is not None: metadata = { "model": self.model, @@ -432,6 +450,10 @@ async def lifespan(app: FastAPI): self._iteration_stats_buffer = deque(maxlen=max_buf) self._iteration_stats_collector_task = asyncio.create_task( self._iteration_stats_collector_loop()) + # Wake up the collector immediately so it processes the + # initial stats emitted by the executor at startup (e.g. + # cache_config_info). + self._iteration_stats_wakeup_event.set() logger.info( "Started background iteration stats collector task") @@ -443,6 +465,7 @@ async def lifespan(app: FastAPI): yield + await self._perf_metrics_writer.close() if self.embedding_batcher is not None: await self.embedding_batcher.shutdown() logger.info("Stopped encode dynamic batcher") @@ -500,6 +523,10 @@ async def validation_exception_handler(_, exc): else: self.register_routes() + if self._collect_perf_metrics: + self.app.add_middleware(PerfMetricsMiddleware, + expose_headers=self._expose_perf_metrics, + writer=self._perf_metrics_writer) self.app.add_middleware(ServerArrivalTimeMiddleware) def _init_visual_gen(self): @@ -519,13 +546,21 @@ def _init_llm(self, chat_template: Optional[str] = None): self.tokenizer.tokenizer, "name_or_path", None) or getattr( self.tokenizer, "name_or_path", None) trust_remote_code = self.generator.args.trust_remote_code - try: - self.processor = AutoProcessor.from_pretrained( - hf_tokenizer_path, trust_remote_code=trust_remote_code) - except Exception: - logger.debug("Failed to load AutoProcessor or AutoConfig for %s", - hf_tokenizer_path) + checkpoint_format = getattr(self.generator.args, "checkpoint_format", + None) + if checkpoint_format in ("mistral", "mistral_large_3"): + # Do not load HF processor for mistral native checkpoints + # even if it is available self.processor = None + else: + try: + self.processor = AutoProcessor.from_pretrained( + hf_tokenizer_path, trust_remote_code=trust_remote_code) + except Exception: + logger.debug( + "Failed to load AutoProcessor or AutoConfig for %s", + hf_tokenizer_path) + self.processor = None # load model config try: @@ -600,10 +635,6 @@ def _init_llm(self, chat_template: Optional[str] = None): pmc.request_inference_time_buckets if pmc else None), ) self._log_config_info_metrics() - max_perf_metrics = self.generator.args.perf_metrics_max_requests - if max_perf_metrics > 0: - self.perf_metrics = deque(maxlen=max_perf_metrics) - self.perf_metrics_lock = asyncio.Lock() @staticmethod def _ensure_post_processor_hook_supported( @@ -822,13 +853,13 @@ def register_routes(self): methods=["GET"]) self.app.add_api_route("/version", self.version, methods=["GET"]) self.app.add_api_route("/v1/models", self.get_model, methods=["GET"]) + self.app.add_api_route("/v1/data_transceiver_state", + self.data_transceiver_state, + methods=["GET"]) # TODO: the metrics endpoint only reports iteration stats, not the runtime stats for now self.app.add_api_route("/metrics", self.get_iteration_stats, methods=["GET"]) - self.app.add_api_route("/perf_metrics", - self.get_perf_metrics, - methods=["GET"]) self.app.add_api_route("/energy_metrics", self.get_energy_metrics, methods=["GET"]) @@ -1128,6 +1159,19 @@ def register_visual_gen_routes(self): self.delete_video, methods=["DELETE"]) + async def data_transceiver_state(self) -> JSONResponse: + """Return the serialized DataTransceiverState as base64-encoded JSON.""" + state = self.generator.get_data_transceiver_state() + if not state: + return JSONResponse( + status_code=404, + content={"error": "No transceiver state available"}) + import base64 + return JSONResponse(content={ + "data_transceiver_state": + base64.b64encode(state).decode("utf-8") + }) + async def health(self) -> Response: if self._check_health(): return Response(status_code=200) @@ -1273,79 +1317,6 @@ async def get_steady_clock_offset(self) -> JSONResponse: "transmit_ts": transmit_ts }) - async def get_perf_metrics(self) -> JSONResponse: - if self.perf_metrics is None: - return JSONResponse(content=[]) - async with self.perf_metrics_lock: - perf_metrics = self.perf_metrics - self.perf_metrics = deque( - maxlen=self.generator.args.perf_metrics_max_requests) - for metrics_dict in perf_metrics: - metrics = metrics_dict["perf_metrics"] - timing_metrics = metrics.timing_metrics - kv_cache_metrics = metrics.kv_cache_metrics - speculative_decoding = metrics.speculative_decoding - metrics_json = { - "first_iter": metrics.first_iter, - "last_iter": metrics.last_iter, - # exclude metrics.iter since it is only meaningful when the request is not finished - } - server_arrival_time = metrics_dict.pop("server_arrival_time", None) - if server_arrival_time is not None: - server_arrival_time += self.disagg_server_steady_clock_offset - server_first_token_time = metrics_dict.pop( - "server_first_token_time", None) - if server_first_token_time is not None: - server_first_token_time += self.disagg_server_steady_clock_offset - metrics_json["timing_metrics"] = { - "server_arrival_time": - server_arrival_time, - "arrival_time": - timing_metrics.arrival_time.total_seconds() + - self.disagg_server_steady_clock_offset, - "first_scheduled_time": - timing_metrics.first_scheduled_time.total_seconds() + - self.disagg_server_steady_clock_offset, - "first_token_time": - timing_metrics.first_token_time.total_seconds() + - self.disagg_server_steady_clock_offset, - "server_first_token_time": - server_first_token_time, - "last_token_time": - timing_metrics.last_token_time.total_seconds() + - self.disagg_server_steady_clock_offset, - } - metrics_json["kv_cache_metrics"] = { - "num_total_allocated_blocks": - kv_cache_metrics.num_total_allocated_blocks, - "num_new_allocated_blocks": - kv_cache_metrics.num_new_allocated_blocks, - "num_reused_blocks": kv_cache_metrics.num_reused_blocks, - "num_missed_blocks": kv_cache_metrics.num_missed_blocks, - } - if timing_metrics.kv_cache_size > 0: - metrics_json["timing_metrics"].update({ - # TODO: move to kv_cache_metrics - "kv_cache_size": - timing_metrics.kv_cache_size, - "kv_cache_transfer_start": - timing_metrics.kv_cache_transfer_start.total_seconds() + - self.disagg_server_steady_clock_offset, - "kv_cache_transfer_end": - timing_metrics.kv_cache_transfer_end.total_seconds() + - self.disagg_server_steady_clock_offset, - }) - if speculative_decoding.total_draft_tokens > 0: - metrics_json["speculative_decoding"] = { - "acceptance_rate": speculative_decoding.acceptance_rate, - "total_accepted_draft_tokens": - speculative_decoding.total_accepted_draft_tokens, - "total_draft_tokens": - speculative_decoding.total_draft_tokens, - } - metrics_dict["perf_metrics"] = metrics_json - return JSONResponse(content=list(perf_metrics)) - async def get_kv_cache_events(self) -> JSONResponse: events = [] try: @@ -1359,6 +1330,17 @@ async def get_kv_cache_events(self) -> JSONResponse: async def _extract_metrics(self, res: RequestOutput, raw_request: Request): if not res.finished: return + if self._collect_perf_metrics: + if raw_request and not getattr(raw_request.state, + "server_first_token_time", None): + raw_request.state.server_first_token_time = ( + get_steady_clock_now_in_seconds()) + record = build_request_metrics_record( + res, + raw_request, + steady_clock_offset=self.disagg_server_steady_clock_offset) + if record is not None and raw_request is not None: + raw_request.state.perf_metrics_records.append(record) if self.metrics_collector: if res.candidate_metrics: for candidate_m in res.candidate_metrics: @@ -1372,31 +1354,6 @@ async def _extract_metrics(self, res: RequestOutput, raw_request: Request): # Wake up the stats collector to drain iteration stats if getattr(self.generator.args, "enable_iter_perf_stats", True): self._iteration_stats_wakeup_event.set() - if self.generator.args.return_perf_metrics: - output = res.outputs[0] - item = { - "request_id": res.request_id, - "perf_metrics": res.outputs[0].request_perf_metrics - } - if raw_request: - item["server_arrival_time"] = getattr(raw_request.state, - "server_arrival_time", - None) - if not getattr(raw_request.state, "server_first_token_time", - None): - raw_request.state.server_first_token_time = get_steady_clock_now_in_seconds( - ) - item[ - "server_first_token_time"] = raw_request.state.server_first_token_time - if output.disaggregated_params: - item[ - "ctx_request_id"] = output.disaggregated_params.ctx_request_id - # Request-level time breakdown (on GenerationResult/RequestOutput, not CompletionOutput) - if getattr(res, 'time_breakdown_metrics', None) is not None: - item["time_breakdown_metrics"] = res.time_breakdown_metrics - if self.perf_metrics is not None: - async with self.perf_metrics_lock: - self.perf_metrics.append(item) async def _create_chat_response( self, @@ -1555,20 +1512,22 @@ async def chat_stream_generator( request.disaggregated_params) try: - conversation, mm_coroutines, mm_placeholder_counts = parse_chat_messages_coroutines( + conversation, mm_coroutines, mm_placeholder_counts, mm_item_order = parse_chat_messages_coroutines( request.messages, self.model_config, self.multimodal_server_config, - request_media_io_kwargs=request.media_io_kwargs) + request_media_io_kwargs=request.media_io_kwargs, + ) except ValidationError: # ValidatorIterator rejects extra fields; fall back to raw JSON. raw_body = await raw_request.json() raw_messages = raw_body.get("messages", []) - conversation, mm_coroutines, mm_placeholder_counts = parse_chat_messages_coroutines( + conversation, mm_coroutines, mm_placeholder_counts, mm_item_order = parse_chat_messages_coroutines( raw_messages, self.model_config, self.multimodal_server_config, - request_media_io_kwargs=request.media_io_kwargs) + request_media_io_kwargs=request.media_io_kwargs, + ) # Decode base64 int32 prompt_token_ids relayed by the orchestrator. if request.prompt_token_ids is None and request.prompt_token_ids_b64: @@ -1606,6 +1565,8 @@ async def chat_stream_generator( raise ValueError( "Passing 'multi_modal_data' and 'multi_modal_embeddings' at the same time is not supported." ) + if mm_data and mm_item_order: + prompt["mm_item_order"] = mm_item_order if request.mm_processor_kwargs: prompt["mm_processor_kwargs"] = request.mm_processor_kwargs @@ -1653,6 +1614,8 @@ async def chat_stream_generator( cache_salt=request.cache_salt, trace_headers=trace_headers, scheduling_params=scheduling_params, + priority=request.priority + if request.priority is not None else DEFAULT_REQUEST_PRIORITY, ) asyncio.create_task(self.await_disconnected(raw_request, promise)) if not self.postproc_worker_enabled: @@ -1741,20 +1704,22 @@ async def create_mm_embedding_response(promise: RequestOutput): ] try: - conversation, mm_coroutines, mm_placeholder_counts = parse_chat_messages_coroutines( + conversation, mm_coroutines, mm_placeholder_counts, mm_item_order = parse_chat_messages_coroutines( request.messages, self.model_config, self.multimodal_server_config, - request_media_io_kwargs=request.media_io_kwargs) + request_media_io_kwargs=request.media_io_kwargs, + ) except ValidationError: # ValidatorIterator rejects extra fields; fall back to raw JSON. raw_body = await raw_request.json() raw_messages = raw_body.get("messages", []) - conversation, mm_coroutines, mm_placeholder_counts = parse_chat_messages_coroutines( + conversation, mm_coroutines, mm_placeholder_counts, mm_item_order = parse_chat_messages_coroutines( raw_messages, self.model_config, self.multimodal_server_config, - request_media_io_kwargs=request.media_io_kwargs) + request_media_io_kwargs=request.media_io_kwargs, + ) if request.prompt_token_ids is not None: prompt = request.prompt_token_ids @@ -1781,6 +1746,8 @@ async def create_mm_embedding_response(promise: RequestOutput): raise ValueError("Cannot use multimodal embeddings as input") if mm_data is not None: prompt["multi_modal_data"] = mm_data + if mm_item_order: + prompt["mm_item_order"] = mm_item_order promise = self.generator.generate_async(inputs=prompt, ) asyncio.create_task(self.await_disconnected(raw_request, promise)) @@ -1979,7 +1946,9 @@ async def generator_wrapper(generator: AsyncIterator[Any]): lora_request=request.lora_request, disaggregated_params=disaggregated_params, conversation_params=conversation_params, - trace_headers=trace_headers) + trace_headers=trace_headers, + priority=request.priority if request.priority is not None + else DEFAULT_REQUEST_PRIORITY) asyncio.create_task( self.await_disconnected(raw_request, promise)) if not self.postproc_worker_enabled: @@ -2028,9 +1997,6 @@ async def create_streaming_generator(promise: RequestOutput, try: if not self.postproc_worker_enabled: post_processor, args = postproc_params.post_processor, postproc_params.postproc_args - # Stamp first-token time on the first response, then append a - # /perf_metrics entry after [DONE]. The deque is only - # populated inside _extract_metrics. first_response = await anext(promise) raw_request.state.server_first_token_time = ( get_steady_clock_now_in_seconds()) @@ -2061,34 +2027,18 @@ async def create_streaming_generator(promise: RequestOutput, # NOTE: WAR for Disagg failure, may affect perf if no warmup if not self.harmony_adapter: self.harmony_adapter = get_harmony_adapter() - # Convert Pydantic models to dictionaries for JSON serialization (standard pattern) - tools_dict = None - if request.tools: - tools_dict = [tool.model_dump() for tool in request.tools] - - # Reasoning effort precedence: request.reasoning_effort > system message parsing > serving default - reasoning_effort = maybe_transform_reasoning_effort( - request.reasoning_effort) - # Get tool_choice from request - tool_choice = getattr(request, 'tool_choice', None) # Reuse pre-tokenized harmony tokens when forwarded by an upstream # context worker (disaggregated serving). Otherwise, run the # Harmony adapter on the request messages. - if request.prompt_token_ids is not None: - harmony_tokens = request.prompt_token_ids - else: - try: - harmony_tokens = self.harmony_adapter.openai_to_harmony_tokens( - request.messages, - tools_dict, - reasoning_effort=reasoning_effort, - tool_choice=tool_choice) - except Exception: - logger.error(f"messages_dict: {request.messages}") - logger.error(f"tools_dict: {tools_dict}") - logger.error(f"request: {request}") - raise + try: + harmony_tokens = tokenize_harmony_chat_request( + request, harmony_adapter=self.harmony_adapter) + except Exception: + logger.error("messages_dict: %s", request.messages) + logger.error("tools: %s", request.tools) + logger.error("request: %s", request) + raise # Get harmony stop tokens harmony_stop_tokens = self.harmony_adapter.get_stop_tokens() @@ -2105,8 +2055,8 @@ async def create_streaming_generator(promise: RequestOutput, "thinking_token_budget is not supported by the Harmony " "GPT-OSS serving path; use reasoning_effort instead") sampling_params.detokenize = False # Harmony adapter handles detokenization - # Enable per-request perf metrics when the env var is set. - # Otherwise the /perf_metrics deque stays empty on this path. + # The server-level effective flag already enables engine metrics + # for header or JSONL output. if len(os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH", "")) > 0: sampling_params.return_perf_metrics = True disaggregated_params = to_llm_disaggregated_params( @@ -2140,6 +2090,8 @@ async def create_streaming_generator(promise: RequestOutput, disaggregated_params=disaggregated_params, conversation_params=conversation_params, trace_headers=trace_headers, + priority=request.priority + if request.priority is not None else DEFAULT_REQUEST_PRIORITY, ) if not self.postproc_worker_enabled: postproc_args.num_prompt_tokens = len(promise.prompt_token_ids) @@ -2189,6 +2141,7 @@ async def create_response( tool_parser=args.tool_parser, ) + await self._extract_metrics(promise, raw_request) return response async def create_streaming_generator(promise: RequestOutput, @@ -2205,6 +2158,7 @@ async def create_streaming_generator(promise: RequestOutput, res, args) for pp_res in pp_results: yield pp_res + await self._extract_metrics(res, raw_request) try: if request.background: @@ -2212,6 +2166,17 @@ async def create_streaming_generator(promise: RequestOutput, "Request.background is not supported yet, will fallback to foreground processing." ) + # Reject rather than silently ignore: with storage disabled + # (TRTLLM_RESPONSES_API_DISABLE_STORE, postproc workers, or + # multi-frontend serving) the previous response can never be + # resolved. + if request.previous_response_id is not None and not self.enable_store: + return self.create_error_response( + err_type="InvalidRequestError", + message=("'previous_response_id' requires response " + "storage, which is disabled on this server."), + ) + # Get prev response prev_response = None if self.enable_store: diff --git a/tensorrt_llm/serve/perf_metrics.py b/tensorrt_llm/serve/perf_metrics.py index e2a4ff7607bb..275ad005c3fd 100644 --- a/tensorrt_llm/serve/perf_metrics.py +++ b/tensorrt_llm/serve/perf_metrics.py @@ -12,12 +12,52 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Prometheus and per-request serving metrics. + +Non-streaming request opt-in and response example:: + + POST /v1/completions HTTP/1.1 + X-TRTLLM-return-metrics: 1 + + HTTP/1.1 200 OK + Content-Type: application/json + Server-Timing: server_queue;dur=1.250000, server_ttft;dur=8.500000, server_e2e;dur=24.000000 + X-TRTLLM-Start-End-Time: server-start;ts=12345.123456, server-end;ts=12345.147456 + X-TRTLLM-Step-Metrics: server-step-0-forward;dur=2.100000, server-step-0-sample;dur=0.400000 + X-TRTLLM-Ctx-Chunk-Metrics: server-ctx-chunk-0-forward;dur=4.200000 + +Streaming responses carry the same fields in a named SSE event after ``[DONE]``:: + + data: [DONE] + + event: trtllm.perf_metrics + data: { + "Server-Timing": "server_queue;dur=1.250000, server_ttft;dur=8.500000", + "X-TRTLLM-Start-End-Time": "server-start;ts=12345.123456, server-end;ts=12345.147456", + "X-TRTLLM-Step-Metrics": "server-step-0-forward;dur=2.100000", + "X-TRTLLM-Ctx-Chunk-Metrics": "server-ctx-chunk-0-forward;dur=4.200000", + } +""" + import asyncio -from collections import defaultdict, deque +import json +import math +import os +import socket from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path from typing import Any, Dict, List, Literal, Optional from tensorrt_llm.llmapi.disagg_utils import ServerRole +from tensorrt_llm.logger import logger +from tensorrt_llm.serve._perf_metrics_schema import ( + DisaggPerfMetricsRecord, + PerfMetrics, + PerfMetricsRecord, + WorkerPerfMetrics, + WorkerPerfMetricsRecord, +) COUNTER_METRICS = [ ("total_requests", "Total number of requests"), @@ -65,28 +105,10 @@ class MetricsDefinition: CLIENT_METRICS_DEFINITIONS = [ - MetricsDefinition("total_requests", "Total number of requests", "counter"), - MetricsDefinition("error_requests", "Total number of error requests", "counter"), - MetricsDefinition("retry_requests", "Total number of retry requests", "counter"), - MetricsDefinition("completed_requests", "Total number of completed requests", "counter"), - MetricsDefinition( - "first_token_latency_seconds", - "Histogram of latency from first token to completion in seconds", - "histogram", - SHORT_TIME_BUCKETS, - ), - MetricsDefinition( - "complete_latency_seconds", - "Histogram of latency from request arrival to last token in seconds", - "histogram", - LONG_TIME_BUCKETS, - ), - MetricsDefinition( - "per_token_latency_seconds", - "Histogram of latency from request arrival to completion in seconds", - "histogram", - SHORT_TIME_BUCKETS, - ), + MetricsDefinition(name, description, "counter") for name, description in COUNTER_METRICS +] + [ + MetricsDefinition(name, description, "histogram", buckets) + for name, description, buckets in HISTOGRAM_METRICS ] ROLE_TO_CLIENT_TYPE = { @@ -127,116 +149,576 @@ def __getattr__( return self._metrics[key] +SERVER_COUNTER_METRICS = ( + ("total_requests", "Total number of requests"), + ("stream_requests", "Total number of stream requests"), + ("nonstream_requests", "Total number of non-stream requests"), + ("validation_exceptions", "Total number of validation exceptions"), + ("http_exceptions", "Total number of HTTP exceptions"), + ("internal_errors", "Total number of internal errors"), + ("total_responses", "Total number of responses"), +) SERVER_METRICS_DEFINITIONS = [ - MetricsDefinition("total_requests", "Total number of requests", "counter"), - MetricsDefinition("stream_requests", "Total number of stream requests", "counter"), - MetricsDefinition("nonstream_requests", "Total number of non-stream requests", "counter"), - MetricsDefinition("validation_exceptions", "Total number of validation exceptions", "counter"), - MetricsDefinition("http_exceptions", "Total number of HTTP exceptions", "counter"), - MetricsDefinition("internal_errors", "Total number of internal errors", "counter"), - MetricsDefinition("total_responses", "Total number of responses", "counter"), + MetricsDefinition(name, description, "counter") for name, description in SERVER_COUNTER_METRICS +] + [ MetricsDefinition( "queue_latency_seconds", "Histogram of latency from request arrival to being processed in seconds", "histogram", SHORT_TIME_BUCKETS, - ), + ) ] class DisaggPerfMetricsCollector: - def __init__(self, max_requests: int): - self._max_requests = max_requests - self._request_meteics = deque(maxlen=max_requests) - self._server_metrics = defaultdict(dict) - self._lock = asyncio.Lock() - self._collect_lock = asyncio.Lock() - self._clients = [] + """Prometheus metrics owned by one disaggregated HTTP server process.""" + + def __init__(self, max_requests: int = 0): + # Kept for compatibility; per-request retention now belongs to JSONL. + del max_requests self._metrics = { definition.name: instance_metric(definition) for definition in SERVER_METRICS_DEFINITIONS } - def add_client(self, client): - self._clients.append(client) - def __getattr__(self, key: str): return self._metrics[key] - async def add_per_request_metrics( - self, - ctx_server: str, - gen_server: str, - ctx_request_id: int, - server_arrival_time: float, - server_first_token_time: float, - ): - async with self._lock: - self._request_meteics.append( + +SERVER_TIMING_HEADER = "Server-Timing" +START_END_TIME_HEADER = "X-TRTLLM-Start-End-Time" +STEP_METRICS_HEADER = "X-TRTLLM-Step-Metrics" +CTX_CHUNK_METRICS_HEADER = "X-TRTLLM-Ctx-Chunk-Metrics" +SSE_METRICS_EVENT = "trtllm.perf_metrics" +RETURN_METRICS_HEADER = "X-TRTLLM-return-metrics" +_RETURN_METRICS_HEADER_BYTES = RETURN_METRICS_HEADER.lower().encode() + +_SCHEMA_VERSION = 1 +_PERF_METRICS_HEADER_BUDGET_BYTES = 80 * 1024 +_WRITER_QUEUE_SIZE = 1024 +_WRITER_BATCH_SIZE = 64 +_WRITER_SHUTDOWN_TIMEOUT_SECONDS = 5 + + +_TIMING_FIELDS = ( + "arrival_time", + "first_scheduled_time", + "first_token_time", + "last_token_time", + "kv_cache_transfer_start", + "kv_cache_transfer_end", +) +_KV_FIELDS = ( + "num_total_allocated_blocks", + "num_new_allocated_blocks", + "num_reused_blocks", + "num_missed_blocks", + "kv_cache_hit_rate", +) +_SPEC_FIELDS = ("acceptance_rate", "total_accepted_draft_tokens", "total_draft_tokens") + + +def _as_seconds(value: Any, offset: float = 0) -> Optional[float]: + try: + seconds = float(value.total_seconds()) + except (AttributeError, TypeError, ValueError): + return None + return seconds + offset if seconds > 0 else None + + +def build_request_metrics_record( + result: Any, + raw_request: Any = None, + phase: str = "server", + steady_clock_offset: float = 0, +) -> Optional[Dict[str, Any]]: + """Convert a completed RequestOutput metrics snapshot to JSON-safe data.""" + if not result or not getattr(result, "outputs", None): + return None + output = result.outputs[0] + metrics = getattr(output, "request_perf_metrics", None) + if metrics is None: + return None + + timing = metrics.timing_metrics + timing_metrics = { + name: _as_seconds(getattr(timing, name), steady_clock_offset) for name in _TIMING_FIELDS + } + timing_metrics["kv_cache_size"] = timing.kv_cache_size + if raw_request is not None: + for name in ("server_arrival_time", "server_first_token_time"): + value = getattr(raw_request.state, name, None) + timing_metrics[name] = value + steady_clock_offset if value is not None else None + + phase_record: Dict[str, Any] = { + "first_iter": metrics.first_iter, + "last_iter": metrics.last_iter, + "timing_metrics": timing_metrics, + "kv_cache_metrics": {name: getattr(metrics.kv_cache_metrics, name) for name in _KV_FIELDS}, + } + speculative = metrics.speculative_decoding + if speculative.total_draft_tokens > 0: + phase_record["speculative_decoding"] = { + name: getattr(speculative, name) for name in _SPEC_FIELDS + } + if getattr(result, "time_breakdown_metrics", None) is not None: + phase_record["time_breakdown_metrics"] = result.time_breakdown_metrics + + record: Dict[str, Any] = { + "schema_version": _SCHEMA_VERSION, + "request_id": str(getattr(result, "request_id", "")), + "status": "complete", + "phases": {phase: phase_record}, + } + disagg = getattr(output, "disaggregated_params", None) + if disagg: + for name in ("ctx_request_id", "disagg_request_id"): + value = getattr(disagg, name, None) + if value is not None: + record[name] = value + return record + + +def _elapsed_ms(values: Dict[str, Any], start: str, end: Optional[str] = None) -> Optional[float]: + try: + value = float(values[start]) + if end is not None: + value = (float(values[end]) - value) * 1000 + except (KeyError, TypeError, ValueError): + return None + return value if value >= 0 and math.isfinite(value) else None + + +def build_metrics_headers(records: List[Dict[str, Any]]) -> Dict[str, str]: + """Format completed records as Server-Timing-style metric lists.""" + values = { + SERVER_TIMING_HEADER: [], + START_END_TIME_HEADER: [], + STEP_METRICS_HEADER: [], + CTX_CHUNK_METRICS_HEADER: [], + } + for record in records: + for phase, phase_record in record.get("phases", {}).items(): + timing = phase_record.get("timing_metrics", {}) + for name, field in ( + ("start", "arrival_time"), + ("end", "last_token_time"), + ): + timestamp = timing.get(field) + if timestamp is not None: + values[START_END_TIME_HEADER].append( + f"{phase}-{name};ts={float(timestamp):.9f}" + ) + timing_ranges = ( ( - ctx_server, - gen_server, - ctx_request_id, - server_arrival_time, - server_first_token_time, + ("queue", "server_arrival_time", "ctx_dispatch_time"), + ("ttft", "server_arrival_time", "server_first_token_time"), + ) + if phase == "disagg" + else ( + ("queue", "arrival_time", "first_scheduled_time"), + ("ttft", "arrival_time", "first_token_time"), + ("e2e", "arrival_time", "last_token_time"), + ("kv_transfer", "kv_cache_transfer_start", "kv_cache_transfer_end"), ) ) + for name, start, end in timing_ranges: + duration = _elapsed_ms(timing, start, end) + if duration is not None: + values[SERVER_TIMING_HEADER].append(f"{phase}_{name};dur={duration:.6f}") - async def get_perf_metrics(self) -> List[Dict[str, Any]]: - async with self._collect_lock: - perf_metrics = {} - for client in self._clients: - metrics_dict = await client.collect_metrics() - perf_metrics.update(metrics_dict) - - return_metrics = [] - async with self._lock: - for server, metrics_data in perf_metrics.items(): - server_metrics = self._server_metrics[server] - # avoid metrics map inflation by limiting the number of requests to add - available_req_num = min( - max(0, self._max_requests - len(server_metrics)), - len(metrics_data), + breakdown = phase_record.get("time_breakdown_metrics") or {} + for header, key, label in ( + (STEP_METRICS_HEADER, "step_metrics", "step"), + (CTX_CHUNK_METRICS_HEADER, "ctx_chunk_metrics", "ctx-chunk"), + ): + for index, metrics in enumerate(breakdown.get(key, [])): + item = metrics.get("iter", index) if key == "step_metrics" else index + durations = ( + ("forward", _elapsed_ms(metrics, "forward_start_time", "forward_end_time")), + ("sample", _elapsed_ms(metrics, "sample_start_time", "sample_end_time")), + ("gpu-forward", _elapsed_ms(metrics, "gpu_forward_time")), + ("gpu-sample", _elapsed_ms(metrics, "gpu_sample_time")), ) - req_metrics_map = { - req_metrics["ctx_request_id"]: req_metrics - for req_metrics in metrics_data[:available_req_num] - if "ctx_request_id" in req_metrics - } - server_metrics.update(req_metrics_map) - - remain_keys = [] - for ( - ctx_server, - gen_server, - ctx_request_id, - server_arrival_time, - server_first_token_time, - ) in self._request_meteics: - gen_perf_metrics = self._server_metrics[gen_server].pop(ctx_request_id, None) - if gen_perf_metrics is None: - # generation not finished - remain_keys.append( - ( - ctx_server, - gen_server, - ctx_request_id, - server_arrival_time, - server_first_token_time, - ) - ) - continue - ctx_perf_metrics = self._server_metrics[ctx_server].pop(ctx_request_id, None) - # TODO: strip the keys for less repeating and use table style response - return_metrics.append( - { - "ctx_server": ctx_server, - "gen_server": gen_server, - "disagg_server_arrival_time": server_arrival_time, - "disagg_server_first_token_time": server_first_token_time, - "ctx_perf_metrics": ctx_perf_metrics, - "gen_perf_metrics": gen_perf_metrics, - } + values[header].extend( + f"{phase}-{label}-{item}-{name};dur={duration:.6f}" + for name, duration in durations + if duration is not None ) - self._request_meteics = deque(remain_keys, maxlen=self._max_requests) - return return_metrics + + inherited = phase_record.get("metrics_headers") or {} + for header in (STEP_METRICS_HEADER, CTX_CHUNK_METRICS_HEADER): + if inherited.get(header): + values[header].append(inherited[header]) + + return {header: ", ".join(items) for header, items in values.items() if items} + + +def build_metrics_record_from_headers( + headers: Any, + phase: str, + request_id: str = "", +) -> Optional[Dict[str, Any]]: + """Build a request-local phase from standard metrics fields.""" + metrics_headers = {} + for header_name in ( + SERVER_TIMING_HEADER, + START_END_TIME_HEADER, + STEP_METRICS_HEADER, + CTX_CHUNK_METRICS_HEADER, + ): + value = headers.get(header_name) + if value: + metrics_headers[header_name] = value.replace("server_", f"{phase}_").replace( + "server-", f"{phase}-" + ) + if not metrics_headers: + return None + timing_metrics = {} + fields = { + f"{phase}-start": "arrival_time", + f"{phase}-end": "last_token_time", + } + for item in metrics_headers.get(START_END_TIME_HEADER, "").split(","): + name, separator, timestamp = item.strip().partition(";ts=") + if separator and name in fields: + try: + timing_metrics[fields[name]] = float(timestamp) + except ValueError: + logger.warning("Ignoring invalid %s timestamp: %s", name, timestamp) + + durations = {} + prefix = f"{phase}_" + for item in metrics_headers.get(SERVER_TIMING_HEADER, "").split(","): + name, separator, duration = item.strip().partition(";dur=") + if separator and name.startswith(prefix): + try: + durations[name[len(prefix) :]] = float(duration) / 1000 + except ValueError: + logger.warning("Ignoring invalid %s duration: %s", name, duration) + + arrival_time = timing_metrics.get("arrival_time") + if arrival_time is not None: + for name, field in ( + ("queue", "first_scheduled_time"), + ("ttft", "first_token_time"), + ("e2e", "last_token_time"), + ): + if name in durations: + timing_metrics.setdefault(field, arrival_time + durations[name]) + + phase_record = {"metrics_headers": metrics_headers} + if timing_metrics: + phase_record["timing_metrics"] = timing_metrics + return { + "schema_version": _SCHEMA_VERSION, + "request_id": request_id, + "status": "complete", + "metrics_headers": metrics_headers, + "phases": {phase: phase_record}, + } + + +def _limit_metrics_headers(headers: Dict[str, str]) -> Dict[str, str]: + def size(values: Dict[str, str]) -> int: + return sum(len(name.encode()) + len(value.encode()) + 4 for name, value in values.items()) + + if size(headers) <= _PERF_METRICS_HEADER_BUDGET_BYTES: + return headers + logger.warning( + "Performance metrics payload exceeds %d bytes; omitting step and context-chunk metrics", + _PERF_METRICS_HEADER_BUDGET_BYTES, + ) + return { + name: value + for name, value in headers.items() + if name in (SERVER_TIMING_HEADER, START_END_TIME_HEADER) + } + + +def combine_disagg_metrics( + request_id: str, + disagg_phase: Dict[str, Any], + ctx_record: Optional[Dict[str, Any]], + gen_record: Optional[Dict[str, Any]], + disagg_request_id: Optional[int] = None, +) -> Dict[str, Any]: + phases: Dict[str, Any] = {"disagg": disagg_phase} + + def add_worker_phase(name: str, record: Dict[str, Any]) -> None: + phase_record = dict(next(iter(record.get("phases", {}).values()), {})) + phase_record["request_id"] = record.get("request_id") + if record.get("ctx_request_id") is not None: + phase_record["ctx_request_id"] = record["ctx_request_id"] + if record.get("metrics_headers"): + phase_record["metrics_headers"] = record["metrics_headers"] + phases[name] = phase_record + + if ctx_record: + add_worker_phase("ctx", ctx_record) + if gen_record: + add_worker_phase("gen", gen_record) + combined = { + "schema_version": _SCHEMA_VERSION, + "request_id": request_id, + "status": "complete", + "phases": phases, + } + if disagg_request_id is not None: + combined["disagg_request_id"] = disagg_request_id + return combined + + +def _jsonl_perf_metrics(phase_record: Dict[str, Any]) -> PerfMetrics: + perf_metrics: PerfMetrics = { + "timing_metrics": dict(phase_record.get("timing_metrics", {})), + } + if "first_iter" in phase_record: + perf_metrics["first_iter"] = phase_record["first_iter"] + if "last_iter" in phase_record: + perf_metrics["last_iter"] = phase_record["last_iter"] + if "kv_cache_metrics" in phase_record: + perf_metrics["kv_cache_metrics"] = phase_record["kv_cache_metrics"] + if "speculative_decoding" in phase_record: + perf_metrics["speculative_decoding"] = phase_record["speculative_decoding"] + + timing_metrics = dict(perf_metrics.get("timing_metrics", {})) + if not timing_metrics.get("kv_cache_size"): + for name in ("kv_cache_size", "kv_cache_transfer_start", "kv_cache_transfer_end"): + timing_metrics.pop(name, None) + perf_metrics["timing_metrics"] = timing_metrics + + kv_cache_metrics = dict(perf_metrics.get("kv_cache_metrics", {})) + kv_cache_metrics.pop("kv_cache_hit_rate", None) + if kv_cache_metrics: + perf_metrics["kv_cache_metrics"] = kv_cache_metrics + return perf_metrics + + +def _jsonl_worker_metrics( + record: Dict[str, Any], phase_record: Dict[str, Any] +) -> WorkerPerfMetrics: + request_id = record["request_id"] + try: + request_id = int(request_id) + except (TypeError, ValueError): + pass + worker_metrics: WorkerPerfMetrics = { + "request_id": request_id, + "perf_metrics": _jsonl_perf_metrics(phase_record), + } + if record.get("ctx_request_id") is not None: + worker_metrics["ctx_request_id"] = record["ctx_request_id"] + if phase_record.get("time_breakdown_metrics") is not None: + worker_metrics["time_breakdown_metrics"] = phase_record["time_breakdown_metrics"] + return worker_metrics + + +def _jsonl_record(record: Dict[str, Any]) -> PerfMetricsRecord: + phases = record.get("phases", {}) + if "disagg" not in phases: + worker_metrics = _jsonl_worker_metrics(record, phases["server"]) + jsonl_record: WorkerPerfMetricsRecord = { + **worker_metrics, + "status": record.get("status", "complete"), + } + if record.get("disagg_request_id") is not None: + jsonl_record["disagg_request_id"] = record["disagg_request_id"] + return jsonl_record + + disagg_phase = phases["disagg"] + disagg_timing = disagg_phase["timing_metrics"] + disagg_record: DisaggPerfMetricsRecord = { + "ctx_server": disagg_phase["ctx_server"], + "gen_server": disagg_phase["gen_server"], + "disagg_server_arrival_time": disagg_timing["server_arrival_time"], + "disagg_ctx_dispatch_time": disagg_timing["ctx_dispatch_time"], + "disagg_server_first_token_time": disagg_timing["server_first_token_time"], + "status": record.get("status", "complete"), + } + for phase, field in ( + ("ctx", "ctx_perf_metrics"), + ("gen", "gen_perf_metrics"), + ): + phase_record = phases.get(phase) + if phase_record: + worker_record = { + "request_id": phase_record.get("request_id", record["request_id"]), + "ctx_request_id": phase_record.get("ctx_request_id"), + } + worker_metrics = _jsonl_worker_metrics(worker_record, phase_record) + if field == "ctx_perf_metrics": + disagg_record["ctx_perf_metrics"] = worker_metrics + else: + disagg_record["gen_perf_metrics"] = worker_metrics + if record.get("disagg_request_id") is not None: + disagg_record["disagg_request_id"] = record["disagg_request_id"] + return disagg_record + + +class PerfMetricsJsonlWriter: + """Best-effort bounded JSONL writer shared by both serving apps.""" + + def __init__(self, output_dir: Optional[str], server_kind: str): + self._output_dir = Path(output_dir) if output_dir else None + self._server_kind = server_kind + self._queue: asyncio.Queue = asyncio.Queue(maxsize=_WRITER_QUEUE_SIZE) + self._task: Optional[asyncio.Task] = None + self._path: Optional[Path] = None + self.dropped_records = 0 + self._write_error_count = 0 + + async def start(self) -> None: + if self._output_dir is None or self._task is not None: + return + try: + self._output_dir.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + filename = ( + f"perf_metrics-{self._server_kind}-{socket.gethostname()}-" + f"{os.getpid()}-{timestamp}.jsonl" + ) + self._path = self._output_dir / filename + self._task = asyncio.create_task(self._run()) + except OSError as error: + logger.error("Disabling performance metrics JSONL output: %s", error) + self._output_dir = None + + def submit(self, record: Dict[str, Any]) -> None: + if self._task is None: + return + try: + self._queue.put_nowait(record) + except asyncio.QueueFull: + self.dropped_records += 1 + if self.dropped_records == 1 or self.dropped_records % 1000 == 0: + logger.warning("Dropped %d performance metrics records", self.dropped_records) + + async def close(self) -> None: + if self._task is None: + return + task = self._task + try: + await asyncio.wait_for( + self._queue.put(None), + timeout=_WRITER_SHUTDOWN_TIMEOUT_SECONDS, + ) + await asyncio.wait_for(task, timeout=_WRITER_SHUTDOWN_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + logger.warning( + "Timed out flushing performance metrics JSONL; dropping remaining records" + ) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + finally: + self._task = None + + async def _run(self) -> None: + stop = False + while not stop: + item = await self._queue.get() + if item is None: + return + records = [item] + while len(records) < _WRITER_BATCH_SIZE: + try: + item = self._queue.get_nowait() + except asyncio.QueueEmpty: + break + if item is None: + stop = True + break + records.append(item) + serialized = [] + for record in records: + try: + item = _jsonl_record(record) + serialized.append( + json.dumps(item, separators=(",", ":"), allow_nan=False) + "\n" + ) + except (KeyError, TypeError, ValueError) as error: + self.dropped_records += 1 + if self.dropped_records == 1 or self.dropped_records % 1000 == 0: + logger.warning("Dropped malformed performance metrics record: %s", error) + if not serialized: + continue + try: + data = "".join(serialized) + await asyncio.to_thread(self._write, data) + except OSError as error: + self.dropped_records += len(serialized) + self._write_error_count += 1 + if self._write_error_count == 1: + logger.warning("Failed to write performance metrics JSONL: %s", error) + + def _write(self, data: str) -> None: + if self._path is not None: + with self._path.open("a", encoding="utf-8") as output: + output.write(data) + + +def build_metrics_sse_event(headers: Dict[str, str]) -> bytes: + headers = _limit_metrics_headers(headers) + if not headers: + return b"" + payload = json.dumps(headers, separators=(",", ":")) + return (f"event: {SSE_METRICS_EVENT}\ndata: {payload}\n\n").encode() + + +class PerfMetricsMiddleware: + """Expose request metrics and optionally persist completed records.""" + + def __init__( + self, app: Any, expose_headers: bool, writer: Optional[PerfMetricsJsonlWriter] = None + ): + self._app = app + self._expose_headers = expose_headers + self._writer = writer + + async def __call__(self, scope: Dict[str, Any], receive: Any, send: Any) -> None: + if scope["type"] != "http": + await self._app(scope, receive, send) + return + is_stream = False + metrics_headers = None + records = scope.setdefault("state", {}).setdefault("perf_metrics_records", []) + return_metrics = self._expose_headers and any( + name.lower() == _RETURN_METRICS_HEADER_BYTES and value.strip() == b"1" + for name, value in scope.get("headers", []) + ) + + async def send_metrics(message: Dict[str, Any]) -> None: + nonlocal is_stream, metrics_headers + if message["type"] == "http.response.start": + headers = list(message.get("headers", [])) + is_stream = any( + key.lower() == b"content-type" and b"text/event-stream" in value.lower() + for key, value in headers + ) + if records and return_metrics and not is_stream: + metrics_headers = build_metrics_headers(records) + public_headers = _limit_metrics_headers(metrics_headers) + headers.extend( + (name.encode(), value.encode()) for name, value in public_headers.items() + ) + message["headers"] = headers + + elif message["type"] == "http.response.body" and not message.get("more_body", False): + if records: + if metrics_headers is None: + metrics_headers = build_metrics_headers(records) + if self._writer is not None: + for record in records: + self._writer.submit(record) + if return_metrics and is_stream: + message["body"] = message.get("body", b"") + build_metrics_sse_event( + metrics_headers + ) + try: + await send(message) + except OSError: + pass + return + await send(message) + + await self._app(scope, receive, send_metrics) diff --git a/tensorrt_llm/serve/resource_governor.py b/tensorrt_llm/serve/resource_governor.py index 20e5af208256..affd9b3a2ea0 100644 --- a/tensorrt_llm/serve/resource_governor.py +++ b/tensorrt_llm/serve/resource_governor.py @@ -120,7 +120,7 @@ async def _convert_messages( ) -> List[int]: """Convert chat messages to token IDs via chat template + tokenization.""" conversation: List[ConversationMessage] = [] - conversation, mm_coroutines, mm_placeholder_counts = parse_chat_messages_coroutines( + conversation, mm_coroutines, mm_placeholder_counts, _ = parse_chat_messages_coroutines( messages, self.model_config, None ) token_task = async_apply_chat_template( diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index cfd363d71616..edd36f8624f9 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -218,9 +218,10 @@ async def store_response(self, Union[list[Message], list[ChatCompletionMessageParam]]] = [], prev_resp_id: Optional[str] = None) -> None: - """ - Store the response and its messages(model output messages) in the conversation store. If the previous response id is provided, - the messages will be appended to the conversation. Otherwise, a new conversation will be created. + """Store a response and its model-output messages. + + If the previous response ID is provided, the messages are appended to + that conversation. Otherwise, a new conversation is created. Args: resp: ResponsesResponse @@ -330,8 +331,8 @@ async def get_conversation_history( return [] def _update_visited_conversation(self, conversation_id) -> None: - """ - Update the visited conversation to the front of the conversation store. + """Move the visited conversation to the front of the store. + This function is used to keep the conversation store sorted by the visited time. And also remove the least recently visited conversation if the number of conversations exceeds the limit. @@ -356,8 +357,8 @@ def _update_visited_conversation(self, conversation_id) -> None: self.conversation_to_response.pop(removed_id) def _pop_conversation(self, resp_id) -> None: - """ - Pop the oldest conversation messages from a conversation. + """Pop the oldest messages from a conversation. + The conversation is starting by a user message and ending by an assistant message. This function is used to keep the number of messages in a conversation within the limit. @@ -820,7 +821,7 @@ async def _create_input_tokens( await conversation_store.store_messages(request.request_id, messages, request.previous_response_id) - conversation, mm_coroutines, mm_placeholder_counts = parse_chat_messages_coroutines( + conversation, mm_coroutines, mm_placeholder_counts, _ = parse_chat_messages_coroutines( messages, model_config) tools_dict = [ tool.model_dump() @@ -1375,9 +1376,10 @@ def get_reasoning_text_delta_event( def _get_output_added_events( self, output_item: ResponseOutputMessage | ResponseReasoningItem ) -> list[StreamingResponsesResponse]: - """ - Get item added event and content part added event for a message item which is starting - to be generated. + """Get the added events for a message item. + + Returns the item-added and content-part-added events when generation + starts. Returns: list[StreamingResponsesResponse]: A list of streaming responses responses @@ -1995,6 +1997,36 @@ async def __call__(self, scope, receive, send): await self.app(scope, receive, send) +class PeriodicLatencyLogger: + """Periodically log latency percentiles for a named coordinator API. + + This lock-free, self-resetting logger runs on one asyncio loop and profiles + the in-process owner and HTTP client without per-call log spam. + """ + + def __init__(self, name: str, window: int = 500): + self._name = name + self._window = window + self._samples: List[float] = [] + self._n = 0 + + def record(self, dt_s: float) -> None: + self._samples.append(dt_s * 1000.0) # ms + self._n += 1 + if self._n % self._window == 0: + s = sorted(self._samples) + m = len(s) + + def percentile(q): + return s[min(int(q * m), m - 1)] + + logger.info(f"[coord_api] {self._name} n={self._n} ms: " + f"mean={sum(s)/m:.2f} p50={percentile(0.5):.2f} " + f"p90={percentile(0.9):.2f} " + f"p99={percentile(0.99):.2f} max={s[-1]:.2f}") + self._samples = [] + + class ResponseHooks(ABC): """ Hooks for response processing and (disagg) service perf observability. @@ -2004,6 +2036,19 @@ class ResponseHooks(ABC): def on_req_begin(self, request: UCompletionRequest): pass + def on_disagg_request_id(self, disagg_request_id: int): + """Receive the request ID immediately after the service allocates it.""" + + def on_ctx_dispatch(self, request: UCompletionRequest): + """Record when the disaggregated service starts context placement. + + Arrival to this point measures the pre-context wait in the orchestrator + or fleet. The default is a no-op for non-instrumented implementations. + """ + + def on_perf_metrics(self, server: str, role: str, metrics: dict): + """Receive request-local metrics carried by an upstream response.""" + @abstractmethod def on_ctx_resp(self, ctx_server: str, response: UCompletionResponse): pass diff --git a/tensorrt_llm/serve/router.py b/tensorrt_llm/serve/router.py index 5941ed979e51..bb7ad601d732 100644 --- a/tensorrt_llm/serve/router.py +++ b/tensorrt_llm/serve/router.py @@ -13,78 +13,46 @@ # limitations under the License. import asyncio -import os import time from abc import ABC, abstractmethod from collections import OrderedDict -from typing import Awaitable, Callable, Dict, Iterable, List, Optional, Union +from typing import Awaitable, Callable, Dict, Iterable, List, Optional import aiohttp +import msgpack +from blake3 import blake3 -from tensorrt_llm.bindings.internal.batch_manager import \ - BlockKey as _NativeBlockKey -from tensorrt_llm.bindings.internal.batch_manager import \ - BlockKeyHasher as _NativeBlockKeyHasher from tensorrt_llm.llmapi.disagg_utils import (MetadataServerConfig, RouterConfig, ServerRole) from tensorrt_llm.logger import logger -from tensorrt_llm.runtime import kv_cache_hash -from tensorrt_llm.runtime.kv_cache_manager_v2._block_radix_tree import \ - Block as V2Block -from tensorrt_llm.runtime.kv_cache_manager_v2._block_radix_tree import \ - ReuseScope -from tensorrt_llm.runtime.kv_cache_manager_v2._block_radix_tree import \ - RootBlock as V2RootBlock from tensorrt_llm.serve.conversation_id import get_request_conversation_id from tensorrt_llm.serve.metadata_server import JsonDictionary -from tensorrt_llm.serve.openai_protocol import (ChatCompletionRequest, - CompletionRequest) - -KV_CACHE_HASH_ALGO_DEFAULT = kv_cache_hash.KV_CACHE_HASH_ALGO_DEFAULT -KV_CACHE_HASH_ALGO_V1 = kv_cache_hash.KV_CACHE_HASH_ALGO_V1 -KV_CACHE_HASH_ALGO_V2 = kv_cache_hash.KV_CACHE_HASH_ALGO_V2 -KV_CACHE_HASH_ALGO_V2_SHA256_64 = kv_cache_hash.KV_CACHE_HASH_ALGO_V2_SHA256_64 -get_cache_salt_id = kv_cache_hash.get_cache_salt_id -hash_v1_block_key = kv_cache_hash.hash_v1_block_key -truncate_sha256_hash_to_int64 = kv_cache_hash.truncate_sha256_hash_to_int64 - -OpenAIRequest = Union[CompletionRequest, ChatCompletionRequest] -BlockHash = Union[int, str] +from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest +# Shared tokenization / block-hashing utilities (single source of truth). +# Re-exported here for backward compat. +from tensorrt_llm.serve.router_utils import ( # noqa: F401 + KV_CACHE_HASH_ALGO_DEFAULT, KV_CACHE_HASH_ALGO_V1, KV_CACHE_HASH_ALGO_V2, + KV_CACHE_HASH_ALGO_V2_SHA256_64, KV_CACHE_HASH_ALGOS, BlockHash, + BlockHashMixin, OpenAIRequest, block_key_hasher, get_cache_salt_id, + get_request_num_tokens, hash_v1_block_key, truncate_sha256_hash_to_int64, + v2_sha256_block_hasher) + +_MSGPACK_HEADERS = {"Content-Type": "application/msgpack"} +COORDINATOR_FINISH_MAX_ATTEMPTS = 3 +COORDINATOR_FINISH_RETRY_DELAY_S = 0.1 +COORDINATOR_FINISH_TIMEOUT_S = 5.0 +COORDINATOR_FINISH_WORKERS = 16 +COORDINATOR_FINISH_QUEUE_SIZE = 4096 +COORDINATOR_FINISH_DRAIN_TIMEOUT_S = 5.0 # Max number of conversations whose home-server pin is retained (LRU). ROUTE_AFFINITY_CACHE_SIZE = 50000 +ROUTE_AFFINITY_HASH_SEED = b"TensorRT-LLM-route-affinity-v1!!" # Leading token-id count folded into the affinity key so pre-tokenized # requests (placeholder message content) still key per conversation. ROUTE_AFFINITY_TOKEN_PREFIX = 256 -def get_request_num_tokens(request: OpenAIRequest) -> int: - if request.disaggregated_params is None or request.disaggregated_params.request_type == "context_only": - if isinstance(request, ChatCompletionRequest): - raise ValueError( - "LoadBalancing router with tokens doesn't support ChatCompletionRequest yet" - ) - - if isinstance(request.prompt, str) or \ - (isinstance(request.prompt, list) and len(request.prompt) > 0 - and isinstance(request.prompt[0], int)): - prompts = [request.prompt] - else: - prompts = request.prompt - - num_tokens = sum(len(prompt) for prompt in prompts) - elif request.disaggregated_params.request_type == "generation_only": - raise ValueError( - "LoadBalancing router with tokens doesn't support generation_only requests" - ) - else: - raise ValueError( - f"Unsupported request type: {request.disaggregated_params.request_type}" - ) - - return num_tokens - - class ServerState: def __init__( @@ -198,9 +166,10 @@ def update_with_events(self, events: Iterable[dict]): if event["type"] == "created": self.set_hash_algo(hash_algo) if event["type"] == "stored": - self.add_blocks( - (block["block_hash"] for block in event["blocks"]), - hash_algo=hash_algo) + block_hashes = [ + block["block_hash"] for block in event["blocks"] + ] + self.add_blocks(block_hashes, hash_algo=hash_algo) elif event["type"] == "removed": self.remove_blocks(event["block_hashes"], hash_algo=hash_algo) @@ -219,7 +188,6 @@ async def matched_tokens( block_table = self._block_table(hash_algo) for hash_list in block_hashes: for block_hash in hash_list: - # TODO: 1) parent hash verification, 2) partial matching if block_hash in block_table: match_count += self._tokens_per_block else: @@ -305,6 +273,13 @@ def _create_server_state(self, server: str) -> ServerState: return self._server_state_class(server, self._use_tokens, lambda: self.session) + def _stage_server(self, server: str) -> None: + self._server_state.setdefault(server, self._create_server_state(server)) + + def _unstage_server(self, server: str) -> None: + if server not in self._servers: + self._server_state.pop(server, None) + def _get_server_load(self, server: str) -> int: state = self._server_state[server] return state._num_active_tokens if self._use_tokens \ @@ -335,7 +310,7 @@ def _select_least_loaded(self, exclude_server: Optional[str] = None ) -> Optional[str]: """Pick the server with the lowest load. Round-robin breaks ties.""" - candidates = [s for s in self._server_state if s != exclude_server] + candidates = [s for s in self._servers if s != exclude_server] if not candidates: return None loads = {s: self._get_server_load(s) for s in candidates} @@ -401,6 +376,14 @@ def servers(self) -> List[str]: def num_prepared_servers(self) -> int: return len(self._prepared_ready_servers) + @property + def prepared_servers(self) -> set[str]: + return set(self._prepared_ready_servers) + + @property + def server_role(self) -> ServerRole: + return self._server_role + @staticmethod def _ensure_url(server: str) -> str: return server if server.startswith("http") else f"http://{server}" @@ -434,23 +417,45 @@ async def _prepare_server(self, server: str): logger.warning(f"Error preparing server {server}: {e}") async def prepare_servers(self, servers: Optional[List[str]] = None): - for server in servers or self._servers: + targets = self._servers if servers is None else servers + for server in targets: if server not in self._servers: continue await self._prepare_server(server) - async def add_server(self, server: str): + def _stage_server(self, server: str) -> None: + pass + + def _unstage_server(self, server: str) -> None: + pass + + async def add_server(self, server: str) -> bool: if server in self._servers: logger.warning(f"Server {server} already exists") - return + return True async with self._lock: - old_servers = self._servers.copy() - self._servers = [*old_servers, server] - self._on_servers_updated(old_servers, self._servers) - await self._prepare_server(server) + self._stage_server(server) + try: + await self._prepare_server(server) + except Exception: + async with self._lock: + self._unstage_server(server) + raise + if server not in self._prepared_ready_servers: + async with self._lock: + self._unstage_server(server) + logger.warning( + f"Server {server} was not added because preparation failed") + return False + async with self._lock: + if server not in self._servers: + old_servers = self._servers.copy() + self._servers = [*old_servers, server] + self._on_servers_updated(old_servers, self._servers) logger.debug( f"Added server {server}, {self._server_role.name} current server list: {self._servers}" ) + return True async def remove_server(self, server: str): if server not in self._servers: @@ -468,17 +473,18 @@ async def remove_server(self, server: str): f"Removed server {server}, current server list: {self._servers}") @abstractmethod - async def get_next_server( - self, - request: OpenAIRequest, - exclude_server: Optional[str] = None) -> tuple[str, dict]: - '''Select server by request and return some intermediate information, exclude_server is a server to exclude from the selection''' + async def get_next_server(self, + request: OpenAIRequest, + exclude_server: Optional[str] = None, + req_id: Optional[int] = None) -> tuple[str, dict]: + """Select server by request and return some intermediate information""" @abstractmethod async def finish_request(self, request: OpenAIRequest, session: Optional[aiohttp.ClientSession] = None, - success: bool = True): + success: bool = True, + req_id: Optional[int] = None): pass @property @@ -701,10 +707,11 @@ def _get_next_server(self) -> str: self._server_idx += 1 return server - async def get_next_server( - self, - request: OpenAIRequest, - exclude_server: Optional[str] = None) -> tuple[str, dict]: + async def get_next_server(self, + request: OpenAIRequest, + exclude_server: Optional[str] = None, + req_id: Optional[int] = None) -> tuple[str, dict]: + del req_id if not self._servers: if self._metadata_server: raise ValueError( @@ -726,8 +733,9 @@ async def get_next_server( async def finish_request(self, request: OpenAIRequest, session: Optional[aiohttp.ClientSession] = None, - success: bool = True): - del request, session, success + success: bool = True, + req_id: Optional[int] = None): + del request, session, success, req_id class LoadBalancingRouter(LoadBalancingMixin, Router): @@ -750,10 +758,11 @@ def _on_servers_updated(self, old_servers, new_servers): or self._create_server_state(server)) self._server_state = new_state - async def get_next_server( - self, - request: OpenAIRequest, - exclude_server: Optional[str] = None) -> tuple[str, dict]: + async def get_next_server(self, + request: OpenAIRequest, + exclude_server: Optional[str] = None, + req_id: Optional[int] = None) -> tuple[str, dict]: + del req_id self._validate_servers_available() async with self._lock: @@ -768,229 +777,13 @@ async def get_next_server( async def finish_request(self, request: OpenAIRequest, session: Optional[aiohttp.ClientSession] = None, - success: bool = True): - del session, success + success: bool = True, + req_id: Optional[int] = None): + del session, success, req_id async with self._lock: await self._unregister_request(request) -def block_key_hasher(token_ids: list[int], - parent_hash: Optional[int] = None, - cache_salt_id: Optional[int] = None) -> int: - parent = 0 if parent_hash is None else parent_hash - # Fast path: the native C++ BlockKeyHasher is bit-exact with - # hash_v1_block_key and avoids the per-token Python loop. Its hash() binding - # takes no cache_salt_id, so fall back to Python only when a salt is set - # (rare opt-in; never in the unsalted agent/chat completion path). - if cache_salt_id is None: - return _NativeBlockKeyHasher.hash(_NativeBlockKey(token_ids), parent) - return hash_v1_block_key(token_ids, - parent_hash=parent, - cache_salt_id=cache_salt_id) - - -def v2_sha256_block_hasher(token_ids: list[int], - parent_hash: Optional[str] = None, - cache_salt_id: Optional[int] = None) -> str: - parent_key = (V2RootBlock.make_key(ReuseScope(salt=cache_salt_id)) - if parent_hash is None else bytes.fromhex(parent_hash)) - return V2Block.make_key(parent_key, token_ids).hex() - - -class BlockHashMixin: - """Shared tokenization and block-hash computation. - - Used by routers that need KV-cache-aware prefix matching. - """ - - def _init_block_hashing(self, - tokens_per_block: Optional[int] = None, - custom_tokenizer: Optional[str] = None): - env_tokens_per_block = os.environ.get( - "TRTLLM_KVCACHE_AWARE_ROUTER_HASH_TOKENS_PER_BLOCK") - if env_tokens_per_block is not None: - tokens_per_block = int(env_tokens_per_block) - self._tpb_auto = tokens_per_block is None - self._tokens_per_block = 32 if tokens_per_block is None \ - else tokens_per_block - self._tokenizers: dict = {} - self._custom_tokenizer = custom_tokenizer - logger.info(f"BlockHashMixin: tokens_per_block={self._tokens_per_block}" - f"{' (auto, adopts worker)' if self._tpb_auto else ''}" - f", custom_tokenizer={self._custom_tokenizer}") - - def _get_tokenizer(self, model: str): - if model not in self._tokenizers: - if self._custom_tokenizer: - from tensorrt_llm.tokenizer import load_custom_tokenizer - self._tokenizers[model] = load_custom_tokenizer( - self._custom_tokenizer, model) - else: - from tensorrt_llm.tokenizer import TransformersTokenizer - tokenizer = TransformersTokenizer.from_pretrained( - model, trust_remote_code=True) - self._tokenizers[model] = tokenizer.tokenizer - return self._tokenizers[model] - - def _encode_with_prefix_cache(self, rendered: str, key: int, - tokenizer) -> list[int]: - cache = getattr(self, "_tok_prefix_cache", None) - if cache is None: - cache = self._tok_prefix_cache = OrderedDict() - entry = cache.get(key) - if entry is not None and len(rendered) > len(entry[0]) and \ - rendered.startswith(entry[0]): - ids = entry[1] + tokenizer.encode(rendered[len(entry[0]):], - add_special_tokens=False) - else: - ids = tokenizer.encode(rendered, add_special_tokens=False) - cache[key] = (rendered, ids) - cache.move_to_end(key) - while len(cache) > 1024: - cache.popitem(last=False) - return ids - - def _tokenize(self, request: OpenAIRequest) -> list[list[int]]: - # Handle ChatCompletionRequest (has messages, not prompt) - if isinstance(request, ChatCompletionRequest): - if request.prompt_token_ids is not None: - return [request.prompt_token_ids] - tokenizer = self._get_tokenizer(request.model) - tool_dicts = (None if getattr(request, "tools", None) is None else [ - tool.model_dump() if hasattr(tool, "model_dump") else tool - for tool in request.tools - ]) - chat_template_kwargs = (request.chat_template_kwargs if getattr( - request, "chat_template_kwargs", None) else {}) - rendered = tokenizer.apply_chat_template( - [ - msg if isinstance(msg, dict) else dict(msg) - for msg in request.messages - ], - add_generation_prompt=request.add_generation_prompt, - tokenize=False, - return_dict=False, - tools=tool_dicts, - **chat_template_kwargs, - ) - if isinstance(rendered, str): - key = hash("".join( - str( - msg.get("content") if isinstance(msg, dict) else - getattr(msg, "content", "")) - for msg in request.messages[:2])) - result = self._encode_with_prefix_cache(rendered, key, - tokenizer) - else: - result = list(rendered) - request.prompt_token_ids = result - return [result] - - # Handle CompletionRequest (has prompt) - prompts = request.prompt - if isinstance(prompts, list) and len(prompts) == 0: - return [] - if isinstance(prompts, list) and isinstance(prompts[0], list): - return prompts - elif isinstance(prompts, list) and isinstance(prompts[0], int): - return [prompts] - elif isinstance(prompts, str): - prompts = [prompts] - else: - assert isinstance(prompts, list) and isinstance(prompts[0], str) - - tokenizer = self._get_tokenizer(request.model) - token_lists = [tokenizer(prompt)["input_ids"] for prompt in prompts] - # Replace string prompts with token IDs so the worker server - # skips re-tokenization - request.prompt = (token_lists - if len(token_lists) > 1 else token_lists[0]) - return token_lists - - def _compute_block_hashes( - self, - token_lists: list[list[int]], - hash_algo: str = KV_CACHE_HASH_ALGO_DEFAULT, - cache_salt_id: Optional[int] = None, - ) -> list[list[BlockHash]]: - if hash_algo == KV_CACHE_HASH_ALGO_V1: - block_hasher = block_key_hasher - elif hash_algo == KV_CACHE_HASH_ALGO_V2: - block_hasher = v2_sha256_block_hasher - elif hash_algo == KV_CACHE_HASH_ALGO_V2_SHA256_64: - reuse_scope = ReuseScope(salt=cache_salt_id) - block_hashes: list[list[BlockHash]] = [] - for token_list in token_lists: - hash_list = [] - parent_key = V2RootBlock.make_key(reuse_scope) - for t in range(0, len(token_list) - 1, self._tokens_per_block): - t_end = min(t + self._tokens_per_block, len(token_list) - 1) - parent_key = V2Block.make_key(parent_key, - token_list[t:t_end]) - hash_list.append(truncate_sha256_hash_to_int64(parent_key)) - block_hashes.append(hash_list) - return block_hashes - else: - raise ValueError( - f"Unsupported KV cache hash algorithm: {hash_algo}") - - block_hashes: list[list[BlockHash]] = [] - for token_list in token_lists: - hash_list = [] - # in KvCacheManager, the last token is not included in the block key - for t in range(0, len(token_list) - 1, self._tokens_per_block): - t_end = min(t + self._tokens_per_block, len(token_list) - 1) - hash_list.append( - block_hasher(token_list[t:t_end], - None if t == 0 else hash_list[-1], - cache_salt_id)) - block_hashes.append(hash_list) - return block_hashes - - def _tokenize_and_compute_block_hashes( - self, - request: OpenAIRequest) -> tuple[list[list[int]], list[list[int]]]: - """Synchronous tokenize + block-hash, combined for thread offload. - - Factored into one method so ``get_next_server`` can offload the whole - CPU-bound step via ``asyncio.to_thread`` in a single call, keeping - the orchestrator's asyncio event loop free to dispatch other - requests in parallel. - """ - token_lists = self._tokenize(request) - block_hashes = self._compute_block_hashes(token_lists) - return token_lists, block_hashes - - def _tokenize_and_compute_block_hashes_by_algo( - self, - request: OpenAIRequest, - hash_algos: Iterable[str], - cache_salt_id: Optional[int] = None, - ) -> tuple[list[list[int]], dict[str, list[list[BlockHash]]]]: - """Synchronous tokenize + per-algorithm block hashes for thread offload.""" - token_lists = self._tokenize(request) - return token_lists, { - hash_algo: - self._compute_block_hashes(token_lists, - hash_algo, - cache_salt_id=cache_salt_id) - for hash_algo in set(hash_algos) - } - - @staticmethod - def _text_to_int_sequences(texts: list[str]) -> list[list[int]]: - """Convert text strings to lists of unicode code points. - - Usable as input to ``_compute_block_hashes``. - """ - return [[ord(c) for c in text] for text in texts] - - @staticmethod - def _get_request_cache_salt_id(request: OpenAIRequest) -> Optional[int]: - cache_salt = getattr(request, "cache_salt", None) - return None if cache_salt is None else get_cache_salt_id(cache_salt) - - class KvCacheAwareRouter(BlockHashMixin, LoadBalancingMixin, Router): _server_state_class = KvCacheAwareServerState @@ -1004,13 +797,20 @@ def __init__(self, max_batch_size: int = 64, tokens_per_block: Optional[int] = None, custom_tokenizer: Optional[str] = None, + tokenizer_dir: Optional[str] = None, + use_harmony: Optional[bool] = None, + model_path: Optional[str] = None, track_routed_blocks: bool = True, load_weight: float = 0.25, load_cap: float = float("inf"), - **kwargs): + **kwargs) -> None: super().__init__(server_role, servers, metadata_server_cfg, metadata_server, **kwargs) - self._init_block_hashing(tokens_per_block, custom_tokenizer) + self._init_block_hashing(tokens_per_block, + custom_tokenizer, + tokenizer_dir, + use_harmony=use_harmony, + model_path=model_path) self._init_load_balancing(servers, use_tokens) # TODO: use max_num_tokens? per server? self._max_batch_size = max_batch_size @@ -1018,6 +818,10 @@ def __init__(self, self._load_cap = load_cap self._track_routed_blocks = track_routed_blocks self._pending_routed_blocks: dict[int, tuple[list[BlockHash], str]] = {} + # A coordinator client has no local server pool. The coordinator injects + # the role's effective hash algorithm so this router can act solely as a + # routing-key encoder. + self._routing_hash_algo: Optional[str] = None def _create_server_state(self, server: str) -> KvCacheAwareServerState: return KvCacheAwareServerState(server, self._use_tokens, @@ -1029,34 +833,45 @@ async def close(self): await state.cancel_poll_task() await super().close() - def _stash_routed_blocks_on_route(self, request: OpenAIRequest, - block_hashes: list[list[BlockHash]], - hash_algo: str) -> None: + def _stash_routed_blocks(self, key: int, + block_hashes: list[list[BlockHash]], + hash_algo: str) -> None: if not self._track_routed_blocks: return - flat = [h for hl in block_hashes for h in hl] - self._pending_routed_blocks[id(request)] = (flat, hash_algo) - - def _apply_routed_blocks_on_finish(self, request: OpenAIRequest, - server: Optional[str], - success: bool) -> None: - # Pop unconditionally to avoid leaks; apply only when eligible. - entry = self._pending_routed_blocks.pop(id(request), None) - if not (self._track_routed_blocks and success): - return - if entry is None: - return - if server is None or server not in self._server_state: - return - flat_block_hashes, hash_algo = entry - self._server_state[server].add_blocks(flat_block_hashes, - hash_algo=hash_algo) + flat_block_hashes = [h for hashes in block_hashes for h in hashes] + self._pending_routed_blocks[key] = (flat_block_hashes, hash_algo) def _get_server_hash_algo(self, server: str) -> str: # Lock-free attribute read; state is seeded at handshake and refreshed # by update_with_events. return self._server_state[server].hash_algo + def routing_key_config(self) -> Optional[dict[str, int | str]]: + """Return the encoding configuration shared by prepared servers.""" + hash_algos = { + self._get_server_hash_algo(server) + for server in self._prepared_ready_servers + } + if len(hash_algos) > 1: + raise RuntimeError( + f"KV-cache-aware routing requires one hash algorithm per role; " + f"found {sorted(hash_algos)} for {self._server_role}") + if not hash_algos: + return None + return { + "tokens_per_block": self._tokens_per_block, + "kv_cache_hash_algo": hash_algos.pop(), + } + + def set_routing_key_config(self, config: dict[str, int | str]) -> None: + hash_algo = str(config["kv_cache_hash_algo"]) + if hash_algo not in KV_CACHE_HASH_ALGOS: + raise ValueError(f"Unknown KV cache hash algorithm {hash_algo!r}; " + f"expected one of {sorted(KV_CACHE_HASH_ALGOS)}") + self._tokens_per_block = int(config["tokens_per_block"]) + self._tpb_auto = False + self._routing_hash_algo = hash_algo + def _events_aligned(self, server: str) -> bool: worker_tpb = self._server_info.get(server, {}).get("tokens_per_block") return worker_tpb is None or worker_tpb == self._tokens_per_block @@ -1124,72 +939,92 @@ def _content_affinity_key(request: OpenAIRequest) -> Optional[int]: token_ids = getattr(request, "prompt_token_ids", None) if token_ids: parts.append(str(list(token_ids[:ROUTE_AFFINITY_TOKEN_PREFIX]))) - return hash("".join(parts)) - - async def get_next_server( - self, - request: OpenAIRequest, - exclude_server: Optional[str] = None) -> tuple[str, dict]: - async with self._lock: - servers = list([ - server for server in self._server_state.keys() - if server != exclude_server - ]) - if not servers: - raise ValueError( - f"No available servers after excluding {exclude_server}") + digest = blake3("".join(parts).encode("utf-8"), + key=ROUTE_AFFINITY_HASH_SEED).digest(length=8) + return int.from_bytes(digest, "little", signed=False) + + async def get_next_server(self, + request: OpenAIRequest, + exclude_server: Optional[str] = None, + req_id: Optional[int] = None) -> tuple[str, dict]: + del req_id + # Standalone (in-process) entry point = routing_key(tokenize+hash) then + # the shared _route core -- the SAME core the coordinator path uses. + key = await asyncio.to_thread(self._routing_key_sync, request) + server, info, _handle = await self._route(key, + exclude_server=exclude_server, + request=request) + return server, info + + def _routing_key_sync(self, request: OpenAIRequest) -> dict: + """Build the coordinator routing key. + + Tokenization and per-algorithm block hashing are CPU-bound and run in a + thread. The returned plain dict can be sent over HTTP unchanged. + """ cache_salt_id = self._get_request_cache_salt_id(request) - hash_algo_by_server = { - server: self._get_server_hash_algo(server) - for server in servers + # Hash for every algo any server might use (usually one). + algos = ({self._routing_hash_algo} + if self._routing_hash_algo is not None else { + self._get_server_hash_algo(s) + for s in self._server_state.keys() + }) + if not algos: + raise RuntimeError( + "KV-cache routing-key encoder has no hash algorithm; " + "synchronize it with the coordinator before routing") + token_lists, block_hashes_by_algo = \ + self._tokenize_and_compute_block_hashes_by_algo( + request, algos, cache_salt_id) + return { + "block_hashes_by_algo": block_hashes_by_algo, + "conv_key": self._content_affinity_key(request), + "num_tokens": sum(len(token_list) for token_list in token_lists), } - # Tokenize + block-hash is CPU-bound (~50 ms p50 for a 40 k-token - # chat request with a Rust-backed tokenizer). Running it directly - # inside the async handler blocks the orchestrator's event loop and - # serializes all concurrent requests through it; with HuggingFace - # tokenizers releasing the GIL, offloading to a thread lets multiple - # tokenize calls run in parallel and frees the event loop to - # dispatch HTTP traffic to the CTX/GEN workers meanwhile. - token_lists, block_hashes_by_algo = await asyncio.to_thread( - self._tokenize_and_compute_block_hashes_by_algo, request, - hash_algo_by_server.values(), cache_salt_id) - # select the server by (KV match - load), bounded by load_cap + + async def _route(self, key, exclude_server=None, request=None, req_id=None): + """Route through the core shared by standalone and coordinator paths. + + Servers are scored by cache match and load before applying the load cap, + conversation affinity, and round-robin tie-breaking. Standalone requests + are keyed by ``id(request)``; coordinator requests use ``req_id``. + """ + block_hashes_by_algo = (key or {}).get("block_hashes_by_algo") or {} + conv_key = (key or {}).get("conv_key") + num_tokens = (key or {}).get("num_tokens", 0) + async with self._lock: + server_states = { + server: state + for server, state in self._server_state.items() + if server in self._servers and server != exclude_server + } + servers = list(server_states) + if not servers: + raise ValueError( + f"No available servers after excluding {exclude_server}") + + def _hashes(server): + algo = server_states[server].hash_algo + return algo, block_hashes_by_algo.get(algo, []) + workloads = [ - self._server_state[server].num_active_requests() - for server in servers + server_states[server].num_active_requests() for server in servers ] load_fractions = [ workloads[i] / self._max_batch_size for i in range(len(servers)) ] - scores = [] - matches = [] - for i in range(len(servers)): - server = servers[i] - hash_algo = hash_algo_by_server[server] - block_hashes = block_hashes_by_algo[hash_algo] - # https://github.com/ai-dynamo/dynamo/blob/main/docs/kv_cache_routing.md#kv-cache-routing-and-load-balancing - matches.append(await self._server_state[server].matched_tokens( - block_hashes, hash_algo)) - score = matches[-1] / self._tokens_per_block - self._load_weight * \ - workloads[i] - scores.append(score) - # Optional hard cap: drop servers at/over load_cap; fall back to all if - # none remain. Disabled by default (load_cap=inf) to match the original - # score-only selection. + scores, matches = [], [] + for i, server in enumerate(servers): + algo, bh = _hashes(server) + matches.append(await server_states[server].matched_tokens(bh, algo)) + scores.append(matches[-1] / self._tokens_per_block - + self._load_weight * workloads[i]) candidate_idx = [ i for i, lf in enumerate(load_fractions) if lf < self._load_cap - ] - if not candidate_idx: - candidate_idx = list(range(len(servers))) - # Conversation affinity: pin all turns of a conversation (keyed by a - # content-derived prefix hash, no conversation-id header) to the server - # it first landed on, so a worker eviction shrinking the match score - # cannot scatter the conversation off its warm home. New conversations - # (no pin yet) fall through to the score, which balances them by load. + ] or list(range(len(servers))) affinity = getattr(self, "_route_affinity", None) if affinity is None: affinity = self._route_affinity = OrderedDict() - conv_key = self._content_affinity_key(request) winner = None if conv_key is not None: pinned = affinity.get(conv_key) @@ -1208,33 +1043,107 @@ async def get_next_server( affinity.move_to_end(conv_key) while len(affinity) > ROUTE_AFFINITY_CACHE_SIZE: affinity.popitem(last=False) - hash_algo = hash_algo_by_server[server] - block_hashes = block_hashes_by_algo[hash_algo] + hash_algo, block_hashes = _hashes(server) + + # Same load/routing maps as the standalone path; only the key differs: + # id(request) standalone, disagg req_id under the coordinator (the id that + # crosses the /finish HTTP hop). + key = id(request) if req_id is None else req_id async with self._lock: - await self._register_request(server, request) - self._stash_routed_blocks_on_route(request, block_hashes, hash_algo) + if self._server_state.get(server) is not server_states[server]: + raise ValueError( + f"Selected server {server} is no longer available") + await server_states[server].increment_load(request) + self._req_routing_table[key] = server + try: + self._stash_routed_blocks(key, block_hashes, hash_algo) + except Exception: + self._req_routing_table.pop(key, None) + await server_states[server].decrement_load(request) + raise return server, { - "block_hashes": block_hashes, # list[list[int | str]] + "block_hashes": block_hashes, "hash_algo": hash_algo, - "token_lists": token_lists, # list[list[int]] - "matches": matches, # list[int] + "matches": matches, + "match_length": matches[winner], + "num_tokens": num_tokens, "server_info": self._server_info.get(server, {}), - } + }, req_id async def finish_request(self, request: OpenAIRequest, session: Optional[aiohttp.ClientSession] = None, - success: bool = True): + success: bool = True, + req_id: Optional[int] = None): + del req_id + # Standalone entry point: key by id(request); pass request so token-load + # accounting matches the increment_load(request) done at route time. + await self._finish(id(request), + success, + request=request, + session=session) + + async def _finish(self, key, success, request=None, session=None): + """Finish through the core shared by standalone and coordinator paths. + + This removes the maps populated by ``_route``, decrements load, and + applies routed blocks only after successful completion. + """ async with self._lock: - server = self._req_routing_table.pop(id(request), None) + server = self._req_routing_table.pop(key, None) + pending = self._pending_routed_blocks.pop(key, None) if server is not None and server in self._server_state: await self._server_state[server].decrement_load(request) - self._apply_routed_blocks_on_finish(request, server, success) + if (success and pending is not None and server is not None + and server in self._server_state): + block_hashes, hash_algo = pending + self._server_state[server].add_blocks(block_hashes, + hash_algo=hash_algo) + self._poll_server_on_finish(server, session) + + def _poll_server_on_finish(self, server, session=None): + """Refresh a server's KV-cache block table after a request finishes. + + This is shared by standalone and coordinator finish paths so the block + table remains warm for delegated routing. + """ if (server is not None and server in self._server_state and self._events_aligned(server)): # Fire-and-forget; poll runs in background and coalesces per server. self._server_state[server].schedule_poll_and_update(session) + # ---- coordinator delegation: thin wrappers over the shared _route core --- + # The fleet worker computes routing_key() locally; the coordinator (owns + # _server_state) runs get_next_server_by_key(). Both use the same _route core + # as the standalone get_next_server. + + def routing_key(self, request: OpenAIRequest): + """Return the worker-side JSON-serializable routing key. + + The returned tokenization and block hashes are consumed by ``_route``. + """ + return self._routing_key_sync(request) + + async def get_next_server_by_key(self, + routing_key, + exclude_server=None, + req_id=None): + """Place a coordinator-side request. + + The shared routing core is keyed by the caller's disaggregated request + ID rather than ``id(request)``. + """ + return await self._route(routing_key, + exclude_server=exclude_server, + request=None, + req_id=req_id) + + async def finish_request_by_id(self, req_id, success=True): + """Finish a coordinator-side request by its disaggregated request ID.""" + if req_id is None: + return + await self._finish(req_id, success) + def _on_servers_updated(self, old_servers, new_servers): new_state = {} for server in new_servers: @@ -1348,11 +1257,12 @@ def __init__(self, use_token_ids: bool = False, hash_skip_count: int = 0, max_sessions: int = 100000, + use_harmony: Optional[bool] = None, **kwargs): super().__init__(server_role, servers, metadata_server_cfg, metadata_server, **kwargs) self._init_load_balancing(servers) - self._init_block_hashing(tokens_per_block) + self._init_block_hashing(tokens_per_block, use_harmony=use_harmony) self._match_threshold = match_threshold self._use_token_ids = use_token_ids @@ -1383,6 +1293,9 @@ def __init__(self, } # id(request) -> (server, weight, monotonic_timestamp) self._req_content_entry: dict[int, tuple[str, int, float]] = {} + # Coordinator-delegated path only: disagg req_id -> server, between + # select and finish (id(request) can't cross the HTTP hop). + self._coord_pending: dict = {} # ── content-based load tracking ── @@ -1595,10 +1508,11 @@ def _evict_oldest_session(self): # ── public interface ── - async def get_next_server( - self, - request: OpenAIRequest, - exclude_server: Optional[str] = None) -> tuple[str, dict]: + async def get_next_server(self, + request: OpenAIRequest, + exclude_server: Optional[str] = None, + req_id: Optional[int] = None) -> tuple[str, dict]: + del req_id self._validate_servers_available() conv_id = self._get_conversation_id(request) @@ -1670,8 +1584,9 @@ async def get_next_server( async def finish_request(self, request: OpenAIRequest, session: Optional[aiohttp.ClientSession] = None, - success: bool = True): - del session, success + success: bool = True, + req_id: Optional[int] = None): + del session, success, req_id async with self._lock: server = await self._unregister_request(request) self._remove_content_load(server, request) @@ -1679,6 +1594,271 @@ async def finish_request(self, logger.debug(f"ConversationRouter: FINISH server={server}, " f"content_loads={loads}") + # -- coordinator-path: conversation_id-only sticky routing -- + # The coordinator has no request object, so per-request load is tracked by an + # opaque handle instead of id(request). Only explicit conversation_id sessions + # are supported over the coordinator (no implicit content match). + + def routing_key(self, request: OpenAIRequest): + """The conversation_id (or None); no tokenization on the worker.""" + return self._get_conversation_id(request) + + async def get_next_server_by_key(self, + routing_key, + exclude_server=None, + req_id=None): + conv_id = routing_key + self._validate_servers_available() + async with self._lock: + entry = self._session_table.get(conv_id) if conv_id else None + if (entry is not None and entry[0] in self._server_state + and entry[0] != exclude_server): + server = entry[0] + self._session_table.move_to_end(conv_id) + else: + server = self._select_least_loaded(exclude_server) + if server is None: + raise ValueError( + f"No available servers after excluding {exclude_server}" + ) + if conv_id: + self._update_session(conv_id, server, []) + # Request-count load (no request object at the coordinator). Keyed by + # the disagg req_id -- the sole id crossing the HTTP hop on /finish. + self._server_content_load[server] = ( + self._server_content_load.get(server, 0) + 1) + if req_id is not None: + self._coord_pending[req_id] = server + return server, { + "server_info": self._server_info.get(server, {}) + }, req_id + + async def finish_request_by_id(self, req_id, success=True): + del success + if req_id is None: + return + async with self._lock: + server = self._coord_pending.pop(req_id, None) + if server and server in self._server_content_load: + self._server_content_load[server] = max( + 0, self._server_content_load[server] - 1) + + +class CoordinatorDelegatingRouter(Router): + """Worker-side Router that delegates placement to the disagg coordinator. + + Used only for *stateful* routers (conversation, kv_cache_aware): the worker + must not keep its own copy of that state, so it wraps a local router of the same + type and, for each request, computes the small ``routing_key`` locally and + POSTs it to the coordinator's ``/select``. ``finish_request`` POSTs the + returned handle to ``/finish`` so the coordinator releases per-request state. + Server-pool / prepare / close operations delegate to the wrapped local router. + + Stateless routers (round_robin, load_balancing) are NOT wrapped -- the worker + holds the real router and places locally, so they never reach this class (see + ``CoordinatorClient``). ``OpenAIClient`` already drives + ``router.get_next_server`` / ``router.finish_request``, so the completions + service needs no worker-specific branching. + """ + + def __init__(self, + coordinator_url: str, + local_router: "Router", + role: str, + request_timeout_s: float = 5.0): + # Intentionally NOT calling Router.__init__: this is a thin proxy whose + # server-pool state lives on the wrapped local router (see __getattr__). + # coordinator_url may be a TCP URL or unix:/path (UDS avoids the TCP + # loopback stack for the hot /select,/finish calls). Resolve the request + # base URL now; the session (with UnixConnector when applicable) is made + # lazily so it binds to the running event loop. + from tensorrt_llm.serve.disagg_coordinator import coordinator_base_url + self._coordinator_url_raw = coordinator_url + self._coordinator_url = coordinator_base_url(coordinator_url) + self._local = local_router + self._role = role # "context" | "generation" + self._request_timeout_s = request_timeout_s + self._session: Optional[aiohttp.ClientSession] = None + self._finish_queue: asyncio.Queue[tuple[int, bool]] = asyncio.Queue( + maxsize=COORDINATOR_FINISH_QUEUE_SIZE) + self._finish_workers: set[asyncio.Task] = set() + self._dropped_finishes = 0 + # Coordinator HTTP-client API latency (includes network round-trip to the + # coordinator + its in-process handler). Compare against the owner-side + # [coord_api] to isolate the fleet /select|/finish HTTP overhead. + from tensorrt_llm.serve.responses_utils import PeriodicLatencyLogger + self._select_lat = PeriodicLatencyLogger(f"client.select[{role}]") + self._finish_lat = PeriodicLatencyLogger(f"client.finish[{role}]") + + def __getattr__(self, name): + # servers / prepare_servers / num_prepared_servers / start_server_monitoring + # / routing_key / ... all delegate to the local router. + return getattr(self._local, name) + + @property + def session(self) -> aiohttp.ClientSession: + if self._session is None: + from tensorrt_llm.serve.disagg_coordinator import \ + make_coordinator_session + self._session = make_coordinator_session(self._coordinator_url_raw) + return self._session + + def _on_servers_updated(self, old_servers, new_servers): + pass + + def _request_id(self, + request: OpenAIRequest, + req_id: Optional[int] = None) -> int: + """The request's disagg id -- the sole cross-process key for select/finish. + + Context requests carry disagg_request_id; generation requests inherit the + SAME id as ctx_request_id (set by OpenAIDisaggregatedService before + routing). This id is what the ctx worker registered its KV-transfer + TxSession under, so the gen request MUST keep it -- never re-issue a new + id here, or the gen transceiver waits on a key the ctx side never + registered (transfer never completes -> DISAGG_GENERATION_TRANS_IN_PROGRESS + fills the gen IndexMapper and throughput collapses). + """ + if req_id is not None: + return req_id + dp = request.disaggregated_params + if dp is None: + raise ValueError("delegated routing requires disaggregated_params") + rid = (dp.disagg_request_id + if self._role == "context" else dp.ctx_request_id) + if rid is None: + raise ValueError( + f"delegated {self._role} routing requires a disagg request id " + "(disagg_request_id/ctx_request_id) on the request") + return rid + + async def get_next_server(self, + request: OpenAIRequest, + exclude_server: Optional[str] = None, + req_id: Optional[int] = None) -> tuple[str, dict]: + # routing_key() tokenizes + block-hashes the prompt (CPU-bound for + # kv_cache_aware); run it in a thread so it doesn't block the fleet worker + # event loop driving the concurrent streams. + key = await asyncio.to_thread(self._local.routing_key, request) + # Send the request's existing disagg id as the cross-process key (the + # coordinator keys pending state by it for /finish); placement must not + # change it, since the ctx<->gen KV transfer is keyed by it. + payload = { + "role": self._role, + "routing_key": key, + "req_id": self._request_id(request, req_id), + "exclude_server": exclude_server + } + _t0 = time.monotonic() + async with self.session.post(f"{self._coordinator_url}/select", + data=msgpack.packb(payload, + use_bin_type=True), + headers=_MSGPACK_HEADERS, + timeout=self._request_timeout_s) as resp: + body = msgpack.unpackb(await resp.read(), raw=False) + if resp.status != 200: + raise ValueError(f"coordinator /select returned {resp.status}: " + f"{body.get('error', body)}") + self._select_lat.record(time.monotonic() - _t0) + info = body.get("info") or {} + return body["server"], info + + async def finish_request(self, + request: OpenAIRequest, + session: Optional[aiohttp.ClientSession] = None, + success: bool = True, + req_id: Optional[int] = None): + # /finish only releases coordinator bookkeeping, so enqueue it off the + # request path. A fixed worker pool bounds tasks and connections during a + # coordinator outage; overflow relies on coordinator-side expiration. + del session + req_id = self._request_id(request, req_id) + self._ensure_finish_workers() + try: + self._finish_queue.put_nowait((req_id, success)) + except asyncio.QueueFull: + self._dropped_finishes += 1 + if self._dropped_finishes == 1 or self._dropped_finishes % 1000 == 0: + logger.warning( + f"CoordinatorDelegatingRouter finish queue full; " + f"coordinator expiration will release dropped requests " + f"(dropped={self._dropped_finishes})") + + def _ensure_finish_workers(self) -> None: + if self._finish_workers: + return + for _ in range(COORDINATOR_FINISH_WORKERS): + task = asyncio.create_task(self._finish_worker()) + self._finish_workers.add(task) + + async def _finish_worker(self) -> None: + while True: + req_id, success = await self._finish_queue.get() + try: + await self._finish_async(req_id, success) + finally: + self._finish_queue.task_done() + + async def _finish_async(self, req_id: int, success: bool): + _t0 = time.monotonic() + for attempt in range(1, COORDINATOR_FINISH_MAX_ATTEMPTS + 1): + try: + async with self.session.post( + f"{self._coordinator_url}/finish", + data=msgpack.packb( + { + "role": self._role, + "req_id": req_id, + "success": success + }, + use_bin_type=True), + headers=_MSGPACK_HEADERS, + timeout=min(self._request_timeout_s, + COORDINATOR_FINISH_TIMEOUT_S)) as resp: + if resp.status != 200: + raw_body = await resp.read() + try: + body = msgpack.unpackb(raw_body, raw=False) + except Exception: # noqa: BLE001 + body = raw_body.decode(errors="replace") + error = body.get("error", body) if isinstance( + body, dict) else body + raise RuntimeError( + f"coordinator /finish returned {resp.status}: {error}" + ) + self._finish_lat.record(time.monotonic() - _t0) + return + except Exception as e: # noqa: BLE001 + if attempt == COORDINATOR_FINISH_MAX_ATTEMPTS: + logger.warning( + f"CoordinatorDelegatingRouter finish failed after " + f"{attempt} attempts: {e}") + return + logger.warning( + f"CoordinatorDelegatingRouter finish attempt {attempt} " + f"failed: {e}; retrying") + await asyncio.sleep(COORDINATOR_FINISH_RETRY_DELAY_S * + (2**(attempt - 1))) + + async def close(self): + if self._finish_workers: + try: + await asyncio.wait_for( + self._finish_queue.join(), + timeout=COORDINATOR_FINISH_DRAIN_TIMEOUT_S) + except asyncio.TimeoutError: + logger.warning( + "Timed out draining coordinator finish queue; " + "coordinator expiration will release remaining requests") + for task in self._finish_workers: + task.cancel() + await asyncio.gather(*self._finish_workers, return_exceptions=True) + self._finish_workers.clear() + if self._session is not None: + await self._session.close() + self._session = None + await self._local.close() + def create_router( router_config: Optional[RouterConfig], @@ -1688,8 +1868,7 @@ def create_router( server_preparation_func: Optional[Callable[[str], Awaitable[None]]] = None, disagg_node_id: int = 0, ) -> Router: - """ - Factory function to create different types of router instances. + """Factory function to create different types of router instances. Args: router_type (str): Type of router to create. Supported values: @@ -1725,3 +1904,35 @@ def create_router( metadata_server, server_preparation_func=server_preparation_func, **extra_args) + + +def build_disagg_routers( + ctx_router_config: Optional[RouterConfig], + gen_router_config: Optional[RouterConfig], + ctx_servers: Optional[List[str]], + gen_servers: Optional[List[str]], + metadata_server_cfg: Optional[MetadataServerConfig] = None, + metadata_server: Optional[JsonDictionary] = None, + server_preparation_func: Optional[Callable[[str], Awaitable[None]]] = None, + disagg_node_id: int = 0, + is_delegating_client: bool = False, +) -> tuple[Router, Router]: + """Build the ctx and gen routers for one disagg process. + + Each side is built independently via :func:`create_router`. Stateful router + types (conversation, kv_cache_aware) expose ``routing_key`` / + ``get_next_server_by_key``; when this process is a delegating client, the + caller (:class:`CoordinatorClient`) wraps those in a + :class:`CoordinatorDelegatingRouter` so placement is delegated to the + coordinator. Stateless types (round_robin, load_balancing) place locally. + ``is_delegating_client`` is accepted for call-site symmetry; router + construction itself does not depend on it. + """ + del is_delegating_client # no build-time behavior differs by this flag + ctx_router = create_router(ctx_router_config, ctx_servers, + metadata_server_cfg, metadata_server, + server_preparation_func, disagg_node_id) + gen_router = create_router(gen_router_config, gen_servers, + metadata_server_cfg, metadata_server, + server_preparation_func, disagg_node_id) + return ctx_router, gen_router diff --git a/tensorrt_llm/serve/router_utils.py b/tensorrt_llm/serve/router_utils.py new file mode 100644 index 000000000000..7e416f5e7782 --- /dev/null +++ b/tensorrt_llm/serve/router_utils.py @@ -0,0 +1,410 @@ +# Copyright (c) 2025-2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared router utilities: request tokenization and KV-cache block hashing. + +Extracted from ``router.py`` so the surface routers there can share a single +implementation of block hashing without importing the whole router module. +""" + +import os +from collections import OrderedDict +from typing import Iterable, List, Optional, Union + +from tensorrt_llm.bindings.internal.batch_manager import BlockKey as _NativeBlockKey +from tensorrt_llm.bindings.internal.batch_manager import BlockKeyHasher as _NativeBlockKeyHasher +from tensorrt_llm.logger import logger +from tensorrt_llm.runtime import kv_cache_hash +from tensorrt_llm.runtime.kv_cache_manager_v2._block_radix_tree import Block as V2Block +from tensorrt_llm.runtime.kv_cache_manager_v2._block_radix_tree import ReuseScope +from tensorrt_llm.runtime.kv_cache_manager_v2._block_radix_tree import RootBlock as V2RootBlock +from tensorrt_llm.serve.chat_tokenization import ( + resolve_model_type_from_config, + tokenize_chat_request_for_serving, +) +from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest, CompletionRequest + +KV_CACHE_HASH_ALGO_DEFAULT = kv_cache_hash.KV_CACHE_HASH_ALGO_DEFAULT +KV_CACHE_HASH_ALGO_V1 = kv_cache_hash.KV_CACHE_HASH_ALGO_V1 +KV_CACHE_HASH_ALGO_V2 = kv_cache_hash.KV_CACHE_HASH_ALGO_V2 +KV_CACHE_HASH_ALGO_V2_SHA256_64 = kv_cache_hash.KV_CACHE_HASH_ALGO_V2_SHA256_64 +KV_CACHE_HASH_ALGOS = frozenset( + { + KV_CACHE_HASH_ALGO_V1, + KV_CACHE_HASH_ALGO_V2, + KV_CACHE_HASH_ALGO_V2_SHA256_64, + } +) +get_cache_salt_id = kv_cache_hash.get_cache_salt_id +hash_v1_block_key = kv_cache_hash.hash_v1_block_key +truncate_sha256_hash_to_int64 = kv_cache_hash.truncate_sha256_hash_to_int64 + +OpenAIRequest = Union[CompletionRequest, ChatCompletionRequest] +BlockHash = Union[int, str] + +__all__ = [ + "KV_CACHE_HASH_ALGO_DEFAULT", + "KV_CACHE_HASH_ALGO_V1", + "KV_CACHE_HASH_ALGO_V2", + "KV_CACHE_HASH_ALGO_V2_SHA256_64", + "KV_CACHE_HASH_ALGOS", + "get_cache_salt_id", + "hash_v1_block_key", + "truncate_sha256_hash_to_int64", + "OpenAIRequest", + "BlockHash", + "get_request_num_tokens", + "block_key_hasher", + "v2_sha256_block_hasher", + "BlockHashMixin", + "PrefixBlockSet", +] + + +class PrefixBlockSet: + """Single-owner block-hash index -- a flat ``set`` of held block hashes. + + A KV-cache block hash folds in its parent chain (the worker computes it with + ``BlockKeyHasher.hash(block_key, parent_hash)``), so every block-hash *value* + is globally unique to one position in one prefix path. A request's ordered + block-hash list is itself the prefix path, so longest-common-prefix against a + set of held blocks is just "walk the list until the first hash the owner + doesn't hold" -- no explicit tree needed. + + This is the exact structure the orchestrator ``KvCacheAwareServerState`` uses + (a ``set[block_hash]`` per server, walked until the first miss). It is the + right index whenever there is only ONE logical owner -- e.g. the centralized + router's per-instance ``combined_trie`` (owner = instance) and each rank's + trie (owner = that rank). Those are only ever queried for a single owner's + prefix depth (:meth:`match_one`), so a ``hash -> {owner}`` reverse map with + per-depth set intersections would be pure overhead here. + + The ``owner_id`` argument on :meth:`add` / :meth:`remove` / :meth:`match_one` + is accepted and ignored so this is a drop-in for the single-owner call sites. + """ + + __slots__ = ("_blocks",) + + def __init__(self) -> None: + self._blocks: set[int] = set() + + def add(self, owner_id: str, block_hashes: Iterable[int]) -> None: + self._blocks.update(block_hashes) + + def remove(self, owner_id: str, block_hashes: Iterable[int]) -> None: + self._blocks.difference_update(block_hashes) + + def remove_worker(self, owner_id: str) -> None: + self._blocks.clear() + + def match_one(self, owner_id: str, block_hashes: List[int]) -> int: + """Consecutive prefix-block count held by the (single) owner. + + Identical to ``KvCacheAwareServerState.matched_tokens``: walk the query + path, counting blocks present in the set, and stop at the first miss. + """ + blocks = self._blocks + depth = 0 + for h in block_hashes: + if h not in blocks: + break + depth += 1 + return depth + + def has_worker(self, owner_id: str) -> bool: + return bool(self._blocks) + + +def get_request_num_tokens(request: Optional[OpenAIRequest]) -> int: + if request is None: + return 0 + + if ( + request.disaggregated_params is None + or request.disaggregated_params.request_type == "context_only" + ): + if isinstance(request, ChatCompletionRequest): + raise ValueError( + "LoadBalancing router with tokens doesn't support ChatCompletionRequest yet" + ) + + if isinstance(request.prompt, str) or ( + isinstance(request.prompt, list) + and (not request.prompt or isinstance(request.prompt[0], int)) + ): + prompts = [request.prompt] + else: + prompts = request.prompt + + num_tokens = sum(len(prompt) for prompt in prompts) + elif request.disaggregated_params.request_type == "generation_only": + raise ValueError( + "LoadBalancing router with tokens doesn't support generation_only requests" + ) + else: + raise ValueError(f"Unsupported request type: {request.disaggregated_params.request_type}") + + return num_tokens + + +def block_key_hasher( + token_ids: list[int], parent_hash: Optional[int] = None, cache_salt_id: Optional[int] = None +) -> int: + parent = 0 if parent_hash is None else parent_hash + # Fast path: the native C++ BlockKeyHasher is bit-exact with + # hash_v1_block_key and avoids the per-token Python loop. Its hash() binding + # takes no cache_salt_id, so fall back to Python only when a salt is set + # (rare opt-in; never in the unsalted agent/chat completion path). + if cache_salt_id is None: + return _NativeBlockKeyHasher.hash(_NativeBlockKey(token_ids), parent) + return hash_v1_block_key(token_ids, parent_hash=parent, cache_salt_id=cache_salt_id) + + +def v2_sha256_block_hasher( + token_ids: list[int], parent_hash: Optional[str] = None, cache_salt_id: Optional[int] = None +) -> str: + parent_key = ( + V2RootBlock.make_key(ReuseScope(salt=cache_salt_id)) + if parent_hash is None + else bytes.fromhex(parent_hash) + ) + return V2Block.make_key(parent_key, token_ids).hex() + + +class BlockHashMixin: + """Shared tokenization and block-hash computation. + + Used by routers that need KV-cache-aware prefix matching. + """ + + def _init_block_hashing( + self, + tokens_per_block: Optional[int] = None, + custom_tokenizer: Optional[str] = None, + tokenizer_dir: Optional[str] = None, + use_harmony: Optional[bool] = None, + model_path: Optional[str] = None, + ) -> None: + env_tokens_per_block = os.environ.get("TRTLLM_KVCACHE_AWARE_ROUTER_HASH_TOKENS_PER_BLOCK") + if env_tokens_per_block is not None: + tokens_per_block = int(env_tokens_per_block) + self._tpb_auto = tokens_per_block is None + self._tokens_per_block = 32 if tokens_per_block is None else tokens_per_block + self._tokenizers: dict = {} + self._model_types: dict[str, Optional[str]] = {} + self._custom_tokenizer = custom_tokenizer + self._tokenizer_dir = tokenizer_dir + self._model_path = model_path + self._use_harmony = use_harmony + logger.info( + f"BlockHashMixin: tokens_per_block={self._tokens_per_block}" + f"{' (auto, adopts worker)' if self._tpb_auto else ''}" + f", custom_tokenizer={self._custom_tokenizer}" + f", model_path={self._model_path}" + f", use_harmony={self._use_harmony}" + ) + + def _get_tokenizer(self, model: str): + if model not in self._tokenizers: + model_path = self._tokenizer_dir or model + # Only an operator-configured tokenizer directory may run remote + # tokenizer code; request model values are client-controlled. + trust_remote_code = self._tokenizer_dir is not None + if self._custom_tokenizer: + from tensorrt_llm.tokenizer import load_custom_tokenizer + + self._tokenizers[model] = load_custom_tokenizer( + self._custom_tokenizer, model_path, trust_remote_code=trust_remote_code + ) + else: + from tensorrt_llm.tokenizer import TransformersTokenizer + + tokenizer = TransformersTokenizer.from_pretrained( + model_path, trust_remote_code=trust_remote_code + ) + self._tokenizers[model] = tokenizer.tokenizer + return self._tokenizers[model] + + def _encode_with_prefix_cache(self, rendered: str, key: int, tokenizer) -> list[int]: + cache = getattr(self, "_tok_prefix_cache", None) + if cache is None: + cache = self._tok_prefix_cache = OrderedDict() + entry = cache.get(key) + if entry is not None and rendered == entry[0]: + ids = entry[1] + else: + # Tokenizing a suffix independently is not generally composable: + # BPE/SentencePiece merges can cross the cached string boundary. + ids = tokenizer.encode(rendered, add_special_tokens=False) + cache[key] = (rendered, ids) + cache.move_to_end(key) + while len(cache) > 1024: + cache.popitem(last=False) + return ids + + def _get_model_type(self) -> Optional[str]: + model_path = self._model_path or self._tokenizer_dir + if model_path is None: + return None + if model_path not in self._model_types: + try: + self._model_types[model_path] = resolve_model_type_from_config(model_path) + except (OSError, ValueError) as error: + logger.warning( + "Unable to resolve model type from checkpoint config at %s: %s. " + "Set use_harmony explicitly if the checkpoint uses Harmony.", + model_path, + error, + ) + self._model_types[model_path] = None + return self._model_types[model_path] + + def _tokenize(self, request: OpenAIRequest) -> list[list[int]]: + # Handle ChatCompletionRequest (has messages, not prompt) + if isinstance(request, ChatCompletionRequest): + + def tokenizer_factory() -> object: + return self._get_tokenizer(request.model) + + def encode_rendered(rendered: str, tokenizer: object) -> list[int]: + key = hash( + "".join( + str( + msg.get("content") + if isinstance(msg, dict) + else getattr(msg, "content", "") + ) + for msg in request.messages[:2] + ) + ) + return self._encode_with_prefix_cache(rendered, key, tokenizer) + + result = tokenize_chat_request_for_serving( + request, + tokenizer_factory=tokenizer_factory, + encode_rendered=encode_rendered, + use_harmony=self._use_harmony, + model_type_resolver=self._get_model_type, + set_prompt_token_ids=True, + ) + return [result] + + # Handle CompletionRequest (has prompt) + prompts = request.prompt + if isinstance(prompts, list) and not prompts: + return [prompts] + if isinstance(prompts, list) and isinstance(prompts[0], list): + return prompts + elif isinstance(prompts, list) and isinstance(prompts[0], int): + return [prompts] + elif isinstance(prompts, str): + prompts = [prompts] + else: + assert isinstance(prompts, list) and isinstance(prompts[0], str) + + tokenizer = self._get_tokenizer(request.model) + token_lists = [tokenizer(prompt)["input_ids"] for prompt in prompts] + # Replace string prompts with token IDs so the worker server + # skips re-tokenization + request.prompt = token_lists if len(token_lists) > 1 else token_lists[0] + return token_lists + + def _compute_block_hashes( + self, + token_lists: list[list[int]], + hash_algo: str = KV_CACHE_HASH_ALGO_DEFAULT, + cache_salt_id: Optional[int] = None, + ) -> list[list[BlockHash]]: + if hash_algo == KV_CACHE_HASH_ALGO_V1: + block_hasher = block_key_hasher + elif hash_algo == KV_CACHE_HASH_ALGO_V2: + block_hasher = v2_sha256_block_hasher + elif hash_algo == KV_CACHE_HASH_ALGO_V2_SHA256_64: + reuse_scope = ReuseScope(salt=cache_salt_id) + block_hashes: list[list[BlockHash]] = [] + for token_list in token_lists: + hash_list = [] + parent_key = V2RootBlock.make_key(reuse_scope) + for t in range(0, len(token_list) - 1, self._tokens_per_block): + t_end = min(t + self._tokens_per_block, len(token_list) - 1) + parent_key = V2Block.make_key(parent_key, token_list[t:t_end]) + hash_list.append(truncate_sha256_hash_to_int64(parent_key)) + block_hashes.append(hash_list) + return block_hashes + else: + raise ValueError(f"Unsupported KV cache hash algorithm: {hash_algo}") + + block_hashes: list[list[BlockHash]] = [] + for token_list in token_lists: + hash_list = [] + # in KvCacheManager, the last token is not included in the block key + for t in range(0, len(token_list) - 1, self._tokens_per_block): + t_end = min(t + self._tokens_per_block, len(token_list) - 1) + hash_list.append( + block_hasher( + token_list[t:t_end], None if t == 0 else hash_list[-1], cache_salt_id + ) + ) + block_hashes.append(hash_list) + return block_hashes + + def _tokenize_and_compute_block_hashes( + self, request: OpenAIRequest + ) -> tuple[list[list[int]], list[list[int]]]: + """Synchronous tokenize + block-hash, combined for thread offload. + + Factored into one method so ``get_next_server`` can offload the whole + CPU-bound step via ``asyncio.to_thread`` in a single call, keeping + the orchestrator's asyncio event loop free to dispatch other + requests in parallel. + """ + token_lists = self._tokenize(request) + block_hashes = self._compute_block_hashes(token_lists) + return token_lists, block_hashes + + def _tokenize_and_compute_block_hashes_with_salt( + self, + request: OpenAIRequest, + cache_salt_id: Optional[int] = None, + ) -> tuple[list[list[int]], list[list[int]]]: + token_lists = self._tokenize(request) + block_hashes = self._compute_block_hashes(token_lists, cache_salt_id=cache_salt_id) + return token_lists, block_hashes + + def _tokenize_and_compute_block_hashes_by_algo( + self, + request: OpenAIRequest, + hash_algos: Iterable[str], + cache_salt_id: Optional[int] = None, + ) -> tuple[list[list[int]], dict[str, list[list[BlockHash]]]]: + """Synchronous tokenize + per-algorithm block hashes for thread offload.""" + token_lists = self._tokenize(request) + return token_lists, { + hash_algo: self._compute_block_hashes( + token_lists, hash_algo, cache_salt_id=cache_salt_id + ) + for hash_algo in set(hash_algos) + } + + @staticmethod + def _text_to_int_sequences(texts: list[str]) -> list[list[int]]: + """Convert text strings to lists of unicode code points. + + Usable as input to ``_compute_block_hashes``. + """ + return [[ord(c) for c in text] for text in texts] + + @staticmethod + def _get_request_cache_salt_id(request: OpenAIRequest) -> Optional[int]: + cache_salt = getattr(request, "cache_salt", None) + return None if cache_salt is None else get_cache_salt_id(cache_salt) diff --git a/tensorrt_llm/serve/scripts/backend_request_func.py b/tensorrt_llm/serve/scripts/backend_request_func.py index e2496a6754d1..9f01abe94917 100644 --- a/tensorrt_llm/serve/scripts/backend_request_func.py +++ b/tensorrt_llm/serve/scripts/backend_request_func.py @@ -35,7 +35,7 @@ async def _iter_sse_data(response_content): continue payload = line.removeprefix("data:").lstrip() if payload == "[DONE]": - continue + return yield payload diff --git a/tensorrt_llm/serve/scripts/benchmark_serving.py b/tensorrt_llm/serve/scripts/benchmark_serving.py index 0374f96aab20..f3ddffd21f6b 100644 --- a/tensorrt_llm/serve/scripts/benchmark_serving.py +++ b/tensorrt_llm/serve/scripts/benchmark_serving.py @@ -28,6 +28,7 @@ from collections.abc import AsyncGenerator, Iterable from dataclasses import dataclass from datetime import datetime +from pathlib import Path from typing import Any, Optional import aiohttp @@ -736,32 +737,59 @@ async def fetch_energy_metrics(base_url: str) -> Optional[dict]: return None -async def fetch_perf_metrics(base_url: str) -> dict: - """ - Fetch performance metrics from the /perf_metrics endpoint. - - Args: - base_url: The base URL of the server +def _snapshot_perf_metrics(output_dir: str) -> dict[Path, int]: + directory = Path(output_dir) + if not directory.exists(): + return {} + if not directory.is_dir(): + raise ValueError( + f"Performance metrics output path is not a directory: {output_dir}") + return { + path: path.stat().st_size + for path in directory.glob("perf_metrics-*.jsonl") + } - Returns: - Dictionary containing the performance metrics - """ - perf_url = f"{base_url}/perf_metrics" - async with aiohttp.ClientSession(trust_env=True, - timeout=AIOHTTP_TIMEOUT) as session: - try: - async with session.get(perf_url) as response: - if response.status == 200: - return await response.json() - else: - print( - f"Failed to fetch performance metrics. Status: {response.status}" - ) - return {} - except Exception as e: - print(f"Error fetching performance metrics: {e}") - return {} +def _perf_metrics_files(output_dir: str, offsets: dict[Path, + int]) -> list[Path]: + paths = sorted(Path(output_dir).glob("perf_metrics-*.jsonl")) + by_kind = {} + for path in paths: + if path.stat().st_size <= offsets.get(path, 0): + continue + kind = path.name.removeprefix("perf_metrics-").split("-", 1)[0] + by_kind.setdefault(kind, []).append(path) + if "disagg" in by_kind: + return by_kind["disagg"] + if "server" in by_kind: + return by_kind["server"] + return [] + + +def _read_new_perf_metrics( + output_dir: str, + offsets: dict[Path, int], + expected_count: int, + timeout: float = 10, +) -> list[dict]: + deadline = time.monotonic() + timeout + records = [] + while time.monotonic() < deadline: + records = [] + for path in _perf_metrics_files(output_dir, offsets): + with path.open("r", encoding="utf-8") as metrics_file: + metrics_file.seek(offsets.get(path, 0)) + for line in metrics_file: + if not line.strip(): + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + continue + if len(records) >= expected_count: + return records + time.sleep(0.1) + return records def main(args: argparse.Namespace): @@ -965,6 +993,10 @@ def create_dataset_and_sample(dataset_name: str): # Avoid GC - reduce pause times. gc.disable() + perf_metrics_output_dir = getattr(args, 'save_request_time_breakdown', None) + perf_metrics_offsets = (_snapshot_perf_metrics(perf_metrics_output_dir) + if perf_metrics_output_dir else {}) + benchmark_result = asyncio.run( benchmark( backend=backend, @@ -1045,54 +1077,42 @@ def create_dataset_and_sample(dataset_name: str): json.dump(result_json, outfile) save_to_pytorch_benchmark_format(args, result_json, file_name) - # Save per-request breakdown if requested - if args.save_request_time_breakdown: - print("Fetching request performance metrics...") - perf_metrics = asyncio.run(fetch_perf_metrics(base_url)) - - if perf_metrics: - # Generate filename for perf metrics - current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") - base_model_id = model_id.split("/")[-1] - max_concurrency_str = (f"-concurrency{args.max_concurrency}" - if args.max_concurrency is not None else "") - perf_filename = f"{backend}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}-perf_metrics.json" - - if args.result_dir: - perf_filename = os.path.join(args.result_dir, perf_filename) - - # Save perf metrics to JSON file - with open(perf_filename, "w", encoding='utf-8') as outfile: - try: - json.dump(perf_metrics, outfile, indent=2) - except Exception as e: - print(f"Failed to save perf metrics: {e}") - - print(f"Request performance metrics saved to: {perf_filename}") - - # Create timing diagram from the saved JSON file - try: - analyzer = RequestTimeBreakdown() - - print("Creating time diagram from request time breakdown...") - timing_data = analyzer.parse_json_file(perf_filename) - - if timing_data: - # Generate HTML filename for the timing diagram - diagram_filename = f"{os.path.splitext(perf_filename)[0]}-time_diagram.html" - analyzer.create_timing_diagram(timing_data, - diagram_filename) - - print(f"Time diagram saved to: {diagram_filename}") - else: - print( - "No time data found in request time breakdown - skipping diagram creation." - ) - except Exception as e: - print(f"Failed to create time diagram: {e}") - print("Performance metrics were still saved successfully.") + if perf_metrics_output_dir: + expected_count = benchmark_result["completed"] + int( + not args.no_test_input) + perf_metrics = _read_new_perf_metrics(perf_metrics_output_dir, + perf_metrics_offsets, + expected_count) + if not perf_metrics: + print("No new public-server performance metrics found; " + "skipping time breakdown report.") + return + if len(perf_metrics) < expected_count: + print(f"Warning: found {len(perf_metrics)} of " + f"{expected_count} expected performance metrics records.") + + current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") + base_model_id = model_id.split("/")[-1] + max_concurrency_str = (f"-concurrency{args.max_concurrency}" + if args.max_concurrency is not None else "") + output_stem = (f"{backend}-{args.request_rate}qps{max_concurrency_str}-" + f"{base_model_id}-{current_dt}-perf_metrics") + if args.result_dir: + output_stem = os.path.join(args.result_dir, output_stem) + perf_filename = f"{output_stem}.jsonl" + with open(perf_filename, "w", encoding="utf-8") as outfile: + for record in perf_metrics: + outfile.write(json.dumps(record, separators=(",", ":")) + "\n") + print(f"Request performance metrics saved to: {perf_filename}") + + analyzer = RequestTimeBreakdown() + timing_data = analyzer.parse_json_file(perf_filename) + if timing_data: + diagram_filename = f"{output_stem}-time_diagram.html" + analyzer.create_timing_diagram(timing_data, diagram_filename) + print(f"Time diagram saved to: {diagram_filename}") else: - print("Failed to fetch per-request performance metrics.") + print("No time data found; skipping time breakdown diagram.") if __name__ == "__main__": @@ -1487,9 +1507,14 @@ def create_dataset_and_sample(dataset_name: str): parser.add_argument( "--save-request-time-breakdown", - action="store_true", - help= - "After benchmarking, call the /perf_metric endpoint, save the result as JSON, and create an interactive time breakdown diagram.", + nargs="?", + const=".", + default=None, + metavar="PERF_METRICS_OUTPUT_DIR", + help=("Read JSONL records dumped by the server's " + "perf_metrics_output_dir, save the benchmark records, and " + "create an interactive time breakdown diagram. If no directory " + "is provided, use the current directory."), ) args = parser.parse_args() diff --git a/tensorrt_llm/serve/scripts/time_breakdown/README.md b/tensorrt_llm/serve/scripts/time_breakdown/README.md index cdee49212d50..d6b6ecc60eb4 100644 --- a/tensorrt_llm/serve/scripts/time_breakdown/README.md +++ b/tensorrt_llm/serve/scripts/time_breakdown/README.md @@ -376,49 +376,33 @@ The tool expects a JSON file containing an array of request performance metrics ## Usage -### Integration with Benchmark Serving +### Server-side JSONL workflow + +Configure the server-side writer: -Step 1: Set in `extra-llm-api-config.yaml`: ```yaml -return_perf_metrics: True -perf_metrics_max_requests: +perf_metrics_output_dir: ``` -If running disaggregated serving, add configs for all servers (disagg, context and generation server). -Step 2: Add `--save-request-time-breakdown` when running `benchmark_serving.py`: -```bash -python -m tensorrt_llm.serve.scripts.benchmark_serving \ - --model ${model_name} \ - --dataset-name random \ - --ignore-eos \ - --num-prompts 1000 \ - --random-input-len 1024 \ - --random-output-len 2048 \ - --random-ids \ - --max-concurrency 64 \ - --save-result \ - --result-dir \ - --percentile-metrics "ttft,tpot,itl,e2e" \ - --save-request-time-breakdown -``` +Run the workload, then copy the generated `perf_metrics-*.jsonl` file from the server. For disaggregated serving, use the disagg server file; it contains the combined disagg, context, and generation phases. ### As a CLI Tool ```bash # Basic usage -python time_breakdown.py perf_metrics.json +python time_breakdown.py perf_metrics-disagg.jsonl # Specify output file -python time_breakdown.py perf_metrics.json -o my_time_diagram.html +python time_breakdown.py perf_metrics-disagg.jsonl -o my_time_diagram.html # Limit max requests and sort by E2E latency -python time_breakdown.py perf_metrics.json --max-requests 100 --sort-by e2e +python time_breakdown.py perf_metrics-disagg.jsonl --max-requests 100 --sort-by e2e # Show statistics only -python time_breakdown.py perf_metrics.json --stats-only +python time_breakdown.py perf_metrics-disagg.jsonl --stats-only # Create diagram and show statistics -python time_breakdown.py perf_metrics.json --show-stats +python time_breakdown.py perf_metrics-disagg.jsonl --show-stats ``` ### CLI Options diff --git a/tensorrt_llm/serve/scripts/time_breakdown/time_breakdown.py b/tensorrt_llm/serve/scripts/time_breakdown/time_breakdown.py index 19ad6767d6fd..19c8fe3fd978 100644 --- a/tensorrt_llm/serve/scripts/time_breakdown/time_breakdown.py +++ b/tensorrt_llm/serve/scripts/time_breakdown/time_breakdown.py @@ -14,12 +14,12 @@ - Hover to show individual segment details Usage as CLI: - python time_breakdown.py [options] + python time_breakdown.py [options] Usage as library: from time_breakdown import RequestTimeBreakdown analyzer = RequestTimeBreakdown() - timing_data = analyzer.parse_json_file("perf_metrics.json") + timing_data = analyzer.parse_json_file("perf_metrics.jsonl") analyzer.create_timing_diagram(timing_data, "output.html") """ @@ -33,6 +33,8 @@ import numpy as np import plotly.graph_objects as go +from tensorrt_llm.serve._perf_metrics_schema import PerfMetricsRecord + @dataclass class TimingMetric: @@ -181,7 +183,7 @@ def get_metric_by_name(self, name: str) -> Optional[TimingMetric]: class RequestDataParser: """Parser for disaggregated format with ctx_perf_metrics and gen_perf_metrics.""" - def parse_request(self, request_data: Dict, + def parse_request(self, request_data: PerfMetricsRecord, request_index: int) -> Dict[str, Any]: # Check if both ctx_perf_metrics and gen_perf_metrics exist and are not None ctx_perf = request_data.get('ctx_perf_metrics') @@ -207,15 +209,18 @@ def parse_request(self, request_data: Dict, float('nan')) ctx_first_token_time = ctx_metrics.get('first_token_time', float('nan')) ctx_server_arrival_time = ctx_metrics.get('server_arrival_time', - float('nan')) - ctx_server_first_token_time = ctx_metrics.get('server_first_token_time', - float('nan')) + ctx_arrival_time) + ctx_server_first_token_time = ctx_metrics.get( + 'server_first_token_time', + ctx_metrics.get('last_token_time', float('nan'))) # Generation timing - gen_server_first_token_time = gen_metrics.get('server_first_token_time', - float('nan')) - gen_server_arrival_time = gen_metrics.get('server_arrival_time', - float('nan')) + gen_server_first_token_time = gen_metrics.get( + 'server_first_token_time', + gen_metrics.get('last_token_time', float('nan'))) + gen_server_arrival_time = gen_metrics.get( + 'server_arrival_time', gen_metrics.get('arrival_time', + float('nan'))) gen_arrival_time = gen_metrics.get('arrival_time', float('nan')) gen_first_token_time = gen_metrics.get('first_token_time', float('nan')) gen_first_scheduled_time = gen_metrics.get('first_scheduled_time', @@ -240,15 +245,13 @@ def parse_request(self, request_data: Dict, else: request_id = request_data.get('request_id', request_index) - # Time breakdown metrics - check new unified structure first, then fall back to legacy + # Time breakdown metrics step_metrics = None ctx_gpu_forward_time = None ctx_gpu_sample_time = None ctx_chunk_metrics = None - # Try new unified time_breakdown_metrics structure if is_disaggregated: - # time_breakdown_metrics is at gen_perf_metrics top level, not inside perf_metrics time_breakdown = (gen_perf or {}).get('time_breakdown_metrics') if time_breakdown: step_metrics = time_breakdown.get('step_metrics') @@ -260,7 +263,6 @@ def parse_request(self, request_data: Dict, # Legacy: step_metrics inside perf_metrics gen_perf_data = (gen_perf or {}).get('perf_metrics') or {} step_metrics = gen_perf_data.get('step_metrics') - # ctx GPU timing / chunk metrics from ctx_perf if ctx_gpu_forward_time is None: ctx_time_breakdown = (ctx_perf or {}).get('time_breakdown_metrics') @@ -278,7 +280,6 @@ def parse_request(self, request_data: Dict, ctx_gpu_sample_time = (ctx_perf or {}).get('ctx_gpu_sample_time') else: - # Try time_breakdown_metrics at top level first (new structure) time_breakdown = request_data.get('time_breakdown_metrics') if time_breakdown: step_metrics = time_breakdown.get('step_metrics') @@ -334,26 +335,55 @@ def __init__(self, config: Optional[TimingMetricsConfig] = None): def parse_json_file(self, json_file_path: str) -> List[Dict]: """Parse JSON performance metrics file and extract timing information.""" - try: - with open(json_file_path, 'r') as f: - data = json.load(f) - except FileNotFoundError: - print(f"Error: File '{json_file_path}' not found.") - sys.exit(1) - except json.JSONDecodeError as e: - print(f"Error parsing JSON file '{json_file_path}': {e}") - sys.exit(1) + + def iter_records(json_file): + if json_file_path.endswith('.jsonl'): + for line_number, line in enumerate(json_file, start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError( + f"Error parsing JSONL file '{json_file_path}' at " + f"line {line_number}: {error}") from error + if not isinstance(record, dict): + raise ValueError( + f"Expected a JSON object at line {line_number}: " + f"{json_file_path}") + yield record + return + + try: + data = json.load(json_file) + except json.JSONDecodeError as error: + json_file.seek(0) + if not any(line.strip() for line in json_file): + return + raise ValueError( + f"Error parsing JSON file '{json_file_path}': {error}" + ) from error + + if isinstance(data, dict): + yield data + elif isinstance(data, list): + yield from data + else: + raise ValueError( + "Expected a JSON array, JSON object, or JSONL file: " + f"{json_file_path}") timing_data = [] - for i, request in enumerate(data): - parsed_data = self.parser.parse_request(request, i) + with open(json_file_path, 'r') as json_file: + for i, request in enumerate(iter_records(json_file)): + parsed_data = self.parser.parse_request(request, i) - # Calculate durations for each metric - for metric in self.config.metrics: - duration = metric.calculate_duration(parsed_data) - parsed_data[f'{metric.name}_time'] = duration + # Calculate durations for each metric + for metric in self.config.metrics: + duration = metric.calculate_duration(parsed_data) + parsed_data[f'{metric.name}_time'] = duration - timing_data.append(parsed_data) + timing_data.append(parsed_data) if timing_data: has_gen_metrics = any(not math.isnan( @@ -2141,16 +2171,17 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - python time_breakdown.py perf_metrics.json - python time_breakdown.py perf_metrics.json -o my_timing.html - python time_breakdown.py perf_metrics.json --stats-only - python time_breakdown.py perf_metrics.json --max-requests 50 --sort-by e2e - python time_breakdown.py perf_metrics.json --max-requests 100 --sort-by arrival + python time_breakdown.py perf_metrics.jsonl + python time_breakdown.py perf_metrics.jsonl -o my_timing.html + python time_breakdown.py perf_metrics.jsonl --stats-only + python time_breakdown.py perf_metrics.jsonl --max-requests 50 --sort-by e2e + python time_breakdown.py perf_metrics.jsonl --max-requests 100 --sort-by arrival """) - parser.add_argument('json_file', - type=str, - help='Path to JSON performance metrics file') + parser.add_argument( + 'json_file', + type=str, + help='Path to a JSON or server-produced JSONL performance metrics file') parser.add_argument('-o', '--output', type=str, @@ -2180,11 +2211,18 @@ def main(): analyzer = RequestTimeBreakdown() print(f"Parsing: {args.json_file}") - timing_data = analyzer.parse_json_file(args.json_file) + try: + timing_data = analyzer.parse_json_file(args.json_file) + except FileNotFoundError: + print(f"Error: File '{args.json_file}' not found.") + return 1 + except ValueError as error: + print(error) + return 1 if not timing_data: print("No timing data found.") - sys.exit(1) + return 1 if args.stats_only or args.show_stats: analyzer.show_statistics(timing_data) @@ -2194,7 +2232,8 @@ def main(): args.output, max_requests=args.max_requests, sort_by=args.sort_by) + return 0 if __name__ == '__main__': - main() + sys.exit(main()) diff --git a/tensorrt_llm/serve/tool_parser/qwen3_tool_parser.py b/tensorrt_llm/serve/tool_parser/qwen3_tool_parser.py index 298389f47e19..7e6f027cbed4 100644 --- a/tensorrt_llm/serve/tool_parser/qwen3_tool_parser.py +++ b/tensorrt_llm/serve/tool_parser/qwen3_tool_parser.py @@ -7,11 +7,12 @@ from ..openai_protocol import ChatCompletionToolsParam as Tool from .base_tool_parser import BaseToolParser -from .core_types import StreamingParseResult, StructureInfo, _GetInfoFunc +from .core_types import (StreamingParseResult, StructureInfo, ToolCallItem, + _GetInfoFunc) class Qwen3ToolParser(BaseToolParser): - """ + r""" Detector for Qwen 2.5 and Qwen 3 model function call format. Format Structure: @@ -23,9 +24,21 @@ class Qwen3ToolParser(BaseToolParser): - Tool Call Tags: `` and `` wrap each individual call - Function Call Object: JSON object with "name" and "arguments" fields + Some Qwen3 chat templates (notably Qwen3.6 FP8 served with + `--reasoning_parser qwen3_5 --tool_parser qwen3`) emit tool calls as bare + JSON objects without the `...` wrapper once the + reasoning parser strips the `` block. Both `detect_and_parse` and + `parse_streaming_increment` fall back to a bare-JSON parse in that case + (see NVBug 6240584). + Reference: https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct?chat_template=default """ + # Streaming decision state for the bare-JSON fallback path. + _STREAM_MODE_UNDECIDED = "undecided" + _STREAM_MODE_WRAPPED = "wrapped" + _STREAM_MODE_BARE_JSON = "bare_json" + def __init__(self): """ Initializes the detector with necessary state variables. @@ -35,9 +48,18 @@ def __init__(self): self.eot_token = "\n" # nosec B105 self.tool_call_separator = "\n" self._normal_text_buffer = "" # Buffer for handling partial end tokens + # Bare-JSON streaming fallback state (see NVBug 6240584). + self._bare_json_buffer = "" + self._stream_mode = self._STREAM_MODE_UNDECIDED def has_tool_call(self, text: str) -> bool: - """Check if the text contains a Qwen 3 format tool call.""" + """Check if the text contains a Qwen 3 format tool call. + + Note: intentionally strict — this only checks for the `` + wrapper. Existing callers rely on it as a wrapped-form gate. The + bare-JSON fallback lives inside `detect_and_parse` / + `parse_streaming_increment` and does not go through this method. + """ return self.bot_token in text def detect_and_parse(self, text: str, @@ -52,6 +74,27 @@ def detect_and_parse(self, text: str, idx = text.find(self.bot_token) normal_text = text[:idx].strip() if idx != -1 else text if self.bot_token not in text: + # Some Qwen3 chat templates (e.g. Qwen3.6 FP8 with thinking enabled) + # emit tool calls as bare JSON without a wrapper. Try + # to recover those before dropping the text into normal_text. Use + # an explicit type guard to avoid relying on AttributeError to + # catch scalar JSON like "42" or null. + # + # Use `raw_decode` (not `json.loads`) so a valid leading JSON + # object followed by trailing content is still recognized as a + # tool call — `json.loads` would raise `Extra data` and drop the + # entire text into normal_text (see NVBug 6240584 review). + stripped = text.strip() + try: + parsed, _end = json.JSONDecoder().raw_decode(stripped) + except json.JSONDecodeError: + return StreamingParseResult(normal_text=normal_text, calls=[]) + if isinstance(parsed, (dict, list)): + calls = self.parse_base_json(parsed, tools) + if calls: + # Match the wrapped-form convention: text outside the tool + # call is not surfaced as normal_text. + return StreamingParseResult(normal_text="", calls=calls) return StreamingParseResult(normal_text=normal_text, calls=[]) # Find all \n...\n blocks @@ -71,13 +114,123 @@ def detect_and_parse(self, text: str, def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> StreamingParseResult: - """ + r""" Streaming incremental parsing for Qwen 3 tool calls. - Uses base class implementation with buffering to handle partial end tokens. + + Handles two shapes: + - Wrapped: `\n{...}\n` — delegates to the base + class implementation, which streams the name first and then the + arguments incrementally. Applies the same partial-end-token + scrubbing as before. + - Bare JSON (NVBug 6240584): when the stream never contains + `` but the accumulated buffer is a complete JSON object + matching `{"name": ..., "arguments"/"parameters": ...}`, parse it + via `parse_base_json` and emit the calls in the same + name-first-then-arguments pattern used by the base class. + + The parser picks a mode once and stays in it for the remainder of the + stream. Modes: + - undecided: buffer input; peek to decide. + - wrapped: route everything through the base implementation. + - bare_json: bare-JSON tool call already emitted; drop trailing text. """ + if self._stream_mode == self._STREAM_MODE_WRAPPED: + return self._wrapped_streaming(new_text, tools) + if self._stream_mode == self._STREAM_MODE_BARE_JSON: + # Tool call already emitted; ignore any trailing text. + return StreamingParseResult() + + # Undecided: buffer and decide. + self._bare_json_buffer += new_text + accumulated = self._bare_json_buffer + + if self.bot_token in accumulated: + # Wrapped form: replay the accumulated buffer through the base + # streaming path. + self._stream_mode = self._STREAM_MODE_WRAPPED + replay = accumulated + self._bare_json_buffer = "" + return self._wrapped_streaming(replay, tools) + + if self._ends_with_partial_token(accumulated, self.bot_token): + # Could still resolve into a wrapped form — keep buffering. + return StreamingParseResult() + + stripped = accumulated.lstrip() + if not stripped: + # Only whitespace so far — keep buffering. + return StreamingParseResult() + + if stripped[0] not in "{[": + # Not JSON-like at all — flush as normal text via wrapped path. + self._stream_mode = self._STREAM_MODE_WRAPPED + replay = accumulated + self._bare_json_buffer = "" + return self._wrapped_streaming(replay, tools) + + # Leading `{` or `[` — could be a bare JSON tool call, or partial. + # Use `raw_decode` (not `json.loads`) so a complete JSON prefix + # followed by trailing text is emitted immediately. `json.loads` + # would raise `Extra data` and keep buffering forever (NVBug 6240584 + # follow-up review). + try: + parsed, _end = json.JSONDecoder().raw_decode(stripped) + except json.JSONDecodeError: + # Not yet complete — buffer more. + return StreamingParseResult() + + if isinstance(parsed, (dict, list)): + calls = self.parse_base_json(parsed, tools) + if calls: + streaming_calls: List[ToolCallItem] = [] + for i, call_item in enumerate(calls): + # Emit name first with empty parameters, then the full + # arguments — mirrors the base class streaming pattern. + streaming_calls.append( + ToolCallItem( + tool_index=i, + name=call_item.name, + parameters="", + )) + if call_item.parameters: + streaming_calls.append( + ToolCallItem( + tool_index=i, + parameters=call_item.parameters, + )) + self._stream_mode = self._STREAM_MODE_BARE_JSON + self._bare_json_buffer = "" + # Best-effort bookkeeping for callers that inspect these. + # NOTE: This intentionally diverges from `BaseToolParser`, + # which increments `current_tool_id` past the last completed + # tool. The bare-JSON path emits all calls in a single batch + # and never falls back to wrapped streaming afterwards, so + # we leave `current_tool_id` at the last emitted index and + # `current_tool_name_sent = True` as best-effort state for + # any downstream inspection. + self.current_tool_id = max(0, len(calls) - 1) + self.current_tool_name_sent = True + # Any content after the parsed JSON value is trailing text + # (e.g. prose after the tool call). Drop it: `_STREAM_MODE_BARE_JSON` + # already suppresses subsequent chunks, and surfacing this + # tail as `normal_text` would flip `finish_reason` back to + # `stop`. + return StreamingParseResult(normal_text="", + calls=streaming_calls) + + # Parsed as JSON but not a tool call — flush as normal text via + # wrapped path. + self._stream_mode = self._STREAM_MODE_WRAPPED + replay = accumulated + self._bare_json_buffer = "" + return self._wrapped_streaming(replay, tools) + + def _wrapped_streaming(self, new_text: str, + tools: List[Tool]) -> StreamingParseResult: + """Wrapped-form streaming: base implementation + partial-end-token scrubbing.""" result = super().parse_streaming_increment(new_text, tools) - # Handle partial end tokens that are streamed character by character + # Handle partial end tokens that are streamed character by character. if result.normal_text: self._normal_text_buffer += result.normal_text diff --git a/tensorrt_llm/tokenizer/tokenizer.py b/tensorrt_llm/tokenizer/tokenizer.py index 1422cc3804b5..9715865c65a1 100644 --- a/tensorrt_llm/tokenizer/tokenizer.py +++ b/tensorrt_llm/tokenizer/tokenizer.py @@ -40,8 +40,12 @@ # Aliases for built-in custom tokenizers. TOKENIZER_ALIASES = { - "deepseek_v32": "tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer", - "deepseek_v4": "tensorrt_llm.tokenizer.deepseek_v4.DeepseekV4Tokenizer", + "deepseek_v32": + "tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer", + "deepseek_v4": + "tensorrt_llm.tokenizer.deepseek_v4.DeepseekV4Tokenizer", + "mistral_common": + "tensorrt_llm._torch.models.checkpoints.mistral.tokenizer.MistralTokenizer", } TLLM_INCREMENTAL_DETOKENIZATION_BACKEND = os.environ.get( diff --git a/tensorrt_llm/tools/layer_wise_benchmarks/runner.py b/tensorrt_llm/tools/layer_wise_benchmarks/runner.py index f63634f463d4..03b33cc963d2 100644 --- a/tensorrt_llm/tools/layer_wise_benchmarks/runner.py +++ b/tensorrt_llm/tools/layer_wise_benchmarks/runner.py @@ -3,7 +3,6 @@ import inspect import itertools import os -import unittest.mock import weakref from enum import IntEnum from typing import Optional @@ -17,7 +16,6 @@ from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.modeling_utils import PostInitCaller, skip_forward -from tensorrt_llm._torch.modules.fused_moe.fused_moe_cutlass import CutlassFusedMoE from tensorrt_llm._torch.modules.fused_moe.fused_moe_trtllm_gen import TRTLLMGenFusedMoE from tensorrt_llm._torch.modules.fused_moe.fused_moe_wide_ep import WideEPMoE from tensorrt_llm._torch.modules.mamba.mamba2_metadata import Mamba2Metadata @@ -509,38 +507,10 @@ def select_alltoall_method_type( return select_alltoall_method_type - def make_select_alltoall_method_type_2(select_alltoall_method_type_orig): - def select_alltoall_method_type(self): - # Replace the condition `mapping.moe_ep_size <= top_k` with `scaled_from <= top_k` - # by replacing `top_k` with `fake_top_k` - top_k = self.routing_method.experts_per_token - if scaled_from <= top_k: - fake_top_k = mapping.moe_ep_size + 1 - else: - fake_top_k = mapping.moe_ep_size - 1 - assert (mapping.moe_ep_size <= fake_top_k) == (scaled_from <= top_k) - with unittest.mock.patch.object( - self.routing_method.__class__, - "experts_per_token", - new_callable=unittest.mock.PropertyMock, - ) as mock_top_k: - mock_top_k.return_value = fake_top_k - return select_alltoall_method_type_orig(self) - - return select_alltoall_method_type - - select_alltoall_method_type_cutlass = CutlassFusedMoE.select_alltoall_method_type - select_alltoall_method_type_trtllm_gen = TRTLLMGenFusedMoE.select_alltoall_method_type select_alltoall_method_type_wide_ep = WideEPMoE.select_alltoall_method_type tensorrt_llm._torch.model_config.load_pretrained_config = make_load_pretrained_config( mapping, load_pretrained_config ) - CutlassFusedMoE.select_alltoall_method_type = make_select_alltoall_method_type_2( - select_alltoall_method_type_cutlass - ) - TRTLLMGenFusedMoE.select_alltoall_method_type = make_select_alltoall_method_type_2( - select_alltoall_method_type_trtllm_gen - ) WideEPMoE.select_alltoall_method_type = make_select_alltoall_method_type( select_alltoall_method_type_wide_ep ) @@ -548,8 +518,6 @@ def select_alltoall_method_type(self): yield finally: tensorrt_llm._torch.model_config.load_pretrained_config = load_pretrained_config - CutlassFusedMoE.select_alltoall_method_type = select_alltoall_method_type_cutlass - TRTLLMGenFusedMoE.select_alltoall_method_type = select_alltoall_method_type_trtllm_gen WideEPMoE.select_alltoall_method_type = select_alltoall_method_type_wide_ep @staticmethod @@ -661,6 +629,21 @@ def create_run_pack( ) kwargs = {} + # DeepSeek-V4 (multi-head hyper-connection) decoder layers take the initial residual + # as ``hc_state`` shaped ``[num_tokens, hc_mult, hidden_size]`` (not a 2D hidden-states + # tensor), and their MoE routing requires ``input_ids``. Both are absent from the + # generic single-layer harness, so synthesize them when the model exposes ``hc_mult``. + hc_mult = getattr(pretrained_config, "hc_mult", None) + if hc_mult is not None: + hidden_states = hidden_states.unsqueeze(1).expand(-1, hc_mult, -1).contiguous() + kwargs["input_ids"] = torch.randint( + 0, + pretrained_config.vocab_size, + (batch_size * seq_len_q,), + dtype=torch.int32, + device="cuda", + ) + if is_nemotron_hybrid(pretrained_config) or is_qwen3_hybrid(pretrained_config): mamba_metadata = Mamba2Metadata( attn_metadata.max_num_requests, @@ -678,7 +661,7 @@ def run_pack(*, check=False): hidden_states_out, residual_out = self.model( position_ids, hidden_states, attn_metadata, residual, **kwargs ) - if check: + if check and isinstance(hidden_states_out, torch.Tensor): if hidden_states_out.isnan().any(): raise ValueError("Has nan, please fix weights initialization") if hidden_states_out.isinf().any(): @@ -816,6 +799,7 @@ def create_kv_cache_manager( dtype=kv_cache_dtype, spec_config=None, layer_mask=layer_mask, + vocab_size=config.vocab_size, sparse_attention_config=model_config.sparse_attention_config, pretrained_config=model_config.pretrained_config, ) diff --git a/tensorrt_llm/usage/llm_args_golden_manifest.json b/tensorrt_llm/usage/llm_args_golden_manifest.json index ca6e762490cf..42a8dd01cb3c 100644 --- a/tensorrt_llm/usage/llm_args_golden_manifest.json +++ b/tensorrt_llm/usage/llm_args_golden_manifest.json @@ -87,6 +87,16 @@ "kind": "value", "path": "attention_dp_config.kv_cache_routing_max_sessions" }, + { + "allowed_values": [ + "round_robin", + "least_queued" + ], + "annotation": "Literal['round_robin', 'least_queued']", + "converter": "", + "kind": "categorical", + "path": "attention_dp_config.kv_cache_routing_new_conv_placement" + }, { "allowed_values": [], "annotation": "", @@ -561,9 +571,10 @@ { "allowed_values": [ "all_reusable", - "per_request" + "per_request", + "per_conversation" ], - "annotation": "Literal['all_reusable', 'per_request']", + "annotation": "Literal['all_reusable', 'per_request', 'per_conversation']", "converter": "", "kind": "categorical", "path": "kv_cache_config.block_reuse_policy" @@ -709,7 +720,7 @@ "annotation": "", "converter": "", "kind": "value", - "path": "kv_cache_config.mamba_state_cache_interval" + "path": "kv_cache_config.mamba_state_config.periodic_snapshot_interval" }, { "allowed_values": [], @@ -930,9 +941,10 @@ "VANILLA", "TRITON", "MARLIN", - "MEGAMOE_DEEPGEMM" + "MEGAMOE_DEEPGEMM", + "MEGAMOE_CUTEDSL" ], - "annotation": "Literal['AUTO', 'CUTLASS', 'CUTEDSL', 'WIDEEP', 'TRTLLM', 'DEEPGEMM', 'DENSEGEMM', 'VANILLA', 'TRITON', 'MARLIN', 'MEGAMOE_DEEPGEMM']", + "annotation": "Literal['AUTO', 'CUTLASS', 'CUTEDSL', 'WIDEEP', 'TRTLLM', 'DEEPGEMM', 'DENSEGEMM', 'VANILLA', 'TRITON', 'MARLIN', 'MEGAMOE_DEEPGEMM', 'MEGAMOE_CUTEDSL']", "converter": "", "kind": "categorical", "path": "moe_config.backend" @@ -1016,6 +1028,13 @@ "kind": "value", "path": "num_postprocess_workers" }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "num_serve_frontends" + }, { "allowed_values": [ "cutlass", @@ -1389,6 +1408,16 @@ "kind": "value", "path": "sparse_attention_config.enable_heuristic_topk" }, + { + "allowed_values": [ + "triton", + "msa" + ], + "annotation": "Literal['triton', 'msa']", + "converter": "", + "kind": "categorical", + "path": "sparse_attention_config.implementation" + }, { "allowed_values": [], "annotation": "Optional[int]", @@ -1451,6 +1480,20 @@ "kind": "categorical", "path": "sparse_attention_config.kt_cache_dtype" }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.num_attention_heads" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.num_key_value_heads" + }, { "allowed_values": [], "annotation": "Optional[int]", @@ -1593,6 +1636,18 @@ "kind": "value", "path": "speculative_config.acceptance_rate_window_size" }, + { + "allowed_values": [ + "full", + "no_topk", + "no_topp", + "no_topk_no_topp" + ], + "annotation": "", + "converter": "", + "kind": "categorical", + "path": "speculative_config.advanced_sampling_mode" + }, { "allowed_values": [], "annotation": "", @@ -1607,10 +1662,18 @@ "kind": "value", "path": "speculative_config.begin_thinking_phase_token" }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Gt(gt=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.block_size" + }, { "allowed_values": [ "AUTO", "DFlash", + "DSpark", "Draft_Target", "Eagle3", "Eagle", @@ -1715,6 +1778,24 @@ "kind": "value", "path": "speculative_config.is_use_oldest" }, + { + "allowed_values": [ + "vanilla", + "gated", + "rnn" + ], + "annotation": "Optional[Literal['vanilla', 'gated', 'rnn']]", + "converter": "", + "kind": "categorical", + "path": "speculative_config.markov_head_type" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.markov_rank" + }, { "allowed_values": [], "annotation": "Optional[int]", diff --git a/tensorrt_llm/version.py b/tensorrt_llm/version.py index 6910cc0be3eb..b836ea7ede0a 100644 --- a/tensorrt_llm/version.py +++ b/tensorrt_llm/version.py @@ -12,4 +12,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.3.0rc21" +__version__ = "1.3.0rc24" diff --git a/tensorrt_llm/visual_gen/args.py b/tensorrt_llm/visual_gen/args.py index 06c2cb4f603d..ff008eb4cc68 100644 --- a/tensorrt_llm/visual_gen/args.py +++ b/tensorrt_llm/visual_gen/args.py @@ -46,18 +46,19 @@ class QuantAttentionConfig(StrictBaseModel): """Attention quantization recipe (TRTLLM / CUTEDSL backends). - Describes user intent for quantized attention: per-bmm dtype and per-block layout for Q, K, V. - Providing this config to AttentionConfig enables quantized attention; setting - AttentionConfig.quant_attention_config = None disables it. + Specifies Q/K and V quantization formats and their optional block sizes. Bare QuantAttentionConfig() is a valid Qk16Pv8 recipe. Unsupported recipes are rejected by AttentionConfig's validator with a ValueError. """ - qk_dtype: Literal["bf16", "int8", "fp8"] = Field( + qk_dtype: Literal["bf16", "int8", "fp8", "mxfp8", "nvfp4"] = Field( "bf16", status="prototype", - description="Q/K quantization dtype; bf16 leaves Q/K unquantized.", + description=( + "Q/K quantization format. bf16 leaves Q/K unquantized; int8 and fp8 use 8-bit " + "integer and floating-point element formats; mxfp8 and nvfp4 are block-scaled formats." + ), ) v_dtype: Literal["fp8"] = Field( "fp8", @@ -68,19 +69,21 @@ class QuantAttentionConfig(StrictBaseModel): 0, ge=0, status="prototype", - description="Elements per quantization block for Q; 0 for per-tensor quantization.", + description="Number of Q tokens per SageAttention quantization block; 0 otherwise.", ) k_block_size: int = Field( 0, ge=0, status="prototype", - description="Elements per quantization block for K; 0 for per-tensor quantization.", + description="Number of K tokens per SageAttention quantization block; 0 otherwise.", ) v_block_size: int = Field( 0, ge=0, status="prototype", - description="Elements per quantization block for V; 0 for per-tensor quantization.", + description=( + "V quantization block size on the hidden dimension; 0 uses one tensor-wide V scale." + ), ) @@ -119,7 +122,7 @@ class AttentionConfig(StrictBaseModel): @model_validator(mode="after") def _validate_quant_attention_config(self) -> "AttentionConfig": - # SAGE recipes target the TRTLLM backend (per-block Q/K/V scales). + # Recipe tuple: (qk_dtype, v_dtype, (q_block, k_block, v_block)). SAGE_RECIPES = { ("int8", "fp8", (1, 1, 1)), ("int8", "fp8", (1, 4, 1)), @@ -127,9 +130,12 @@ def _validate_quant_attention_config(self) -> "AttentionConfig": ("fp8", "fp8", (1, 1, 1)), ("fp8", "fp8", (1, 4, 1)), } - # QK16PV8 (CUTEDSL backend): Q/K kept in bf16, V quantized to FP8. - QK16PV8_DTYPES = { + CUTEDSL_RECIPES = { ("bf16", "fp8", (0, 0, 0)), + ("mxfp8", "fp8", (0, 0, 0)), + ("mxfp8", "fp8", (0, 0, 1)), + ("nvfp4", "fp8", (0, 0, 0)), + ("nvfp4", "fp8", (0, 0, 1)), } if self.quant_attention_config is None: @@ -150,11 +156,12 @@ def _validate_quant_attention_config(self) -> "AttentionConfig": f"{sorted(SAGE_RECIPES)}." ) elif self.backend == "CUTEDSL": - if recipe not in QK16PV8_DTYPES: + if recipe not in CUTEDSL_RECIPES: raise ValueError( f"Unsupported quant_attention_config={self.quant_attention_config!r} " - f"for backend='CUTEDSL'. Supported (qk_dtype, v_dtype): " - f"{sorted(QK16PV8_DTYPES)}." + f"for backend='CUTEDSL'. Supported recipes " + f"(qk_dtype, v_dtype, (q_block, k_block, v_block)): " + f"{sorted(CUTEDSL_RECIPES)}." ) else: raise ValueError( diff --git a/tensorrt_llm/visual_gen/visual_gen.py b/tensorrt_llm/visual_gen/visual_gen.py index ce83f862326f..9648d26f87ce 100644 --- a/tensorrt_llm/visual_gen/visual_gen.py +++ b/tensorrt_llm/visual_gen/visual_gen.py @@ -283,11 +283,16 @@ def extra_param_specs(self) -> Dict[str, "ExtraParamSchema"]: @property def default_params(self) -> "VisualGenParams": - """Returns a ``VisualGenParams`` with all defaults resolved for the loaded pipeline. + """Returns a ``VisualGenParams`` with the loaded pipeline's defaults. Universal fields (height, width, etc.) are filled from the - pipeline's defaults. All declared ``extra_params`` keys are - included with their defaults (``None`` for params without one). + pipeline's defaults. Pipelines with mode-dependent defaults + (e.g. Cosmos3, where text-to-image and video requests use + different resolutions) leave such fields as ``None``; they are + resolved per request from the output mode, so ``None`` here + means "the mode's default", not "unset". All declared + ``extra_params`` keys are included with their defaults + (``None`` for params without one). Use this to inspect what the model will use, then modify and pass to ``generate()``:: diff --git a/tests/conftest.py b/tests/conftest.py index b0d2feebd7fc..b40dd597ce76 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Fallback wiring of the session-reuse plugin for test dirs WITHOUT an ini. - -tests/unittest and tests/integration/defs load the plugin through the ``-p`` -option in their own pytest.ini; their rootdir sits below this file, so this -conftest is never collected there (no double registration). Any other -directory under tests/ (current or future) resolves its rootdir at the repo -root and picks the hooks up from here, so automatic MPI session reuse covers -every test under tests/. +"""Fallback wiring of the session-reuse and session-prefetch plugins. + +tests/unittest and tests/integration/defs load the plugins through their own +pytest.ini / conftest; their rootdir sits below this file, so this conftest +is never collected there (no double registration). Any other directory under +tests/ (current or future) picks the hooks up from here, so MPI session +reuse and prefetch cover every test under tests/. + +Both plugins define same-named hooks, and ``pytest_plugins`` is only allowed +in a rootdir conftest (this file is not one when pytest runs from the repo +root) — so dispatch to both modules explicitly instead of importing their +hook functions into this namespace (the second import would silently shadow +the first). """ import os @@ -15,8 +20,30 @@ sys.path.insert(0, os.path.dirname(__file__)) # make test_common importable -from test_common.session_reuse_hooks import ( # noqa: E402,F401 - pytest_configure, - pytest_runtest_setup, - pytest_sessionfinish, -) +from test_common import session_prefetcher_hooks as _prefetch # noqa: E402 +from test_common import session_reuse_hooks as _reuse # noqa: E402 + + +def pytest_configure(config): + _reuse.pytest_configure(config) + _prefetch.pytest_configure(config) + + +def pytest_runtest_setup(item): + _reuse.pytest_runtest_setup(item) + _prefetch.pytest_runtest_setup(item) + + +def pytest_runtest_logreport(report): + # Reuse's failure fence (drain pools after a failed test); previously not + # wired in the fallback dirs at all. + _reuse.pytest_runtest_logreport(report) + + +def pytest_runtest_logfinish(nodeid, location): + _reuse.pytest_runtest_logfinish(nodeid, location) + + +def pytest_sessionfinish(session, exitstatus): + _reuse.pytest_sessionfinish(session, exitstatus) + _prefetch.pytest_sessionfinish(session, exitstatus) diff --git a/tests/integration/README.md b/tests/integration/README.md index 8653989af4db..1f2f745bc10f 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -29,8 +29,6 @@ SM version mapping: - `sm100` = Blackwell (e.g., B100, B200) - `sm103` = Blackwell-Ultra (e.g., B300, GB300) -- To run perf tests, you also need to first build the cpp benchmark by calling `build_wheel.py` with `--benchmarks` flag. - ## Run perf tests All the perf test names are in the form of `perf/test_perf.py::test_perf[...]` where the `...` part is the test parameters. diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index 1805229baa7e..b52f6f1fdf3c 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -1,1229 +1,1507 @@ { - "accuracy/test_cli_flow.py::TestGptNext::test_auto_dtype": 856.028, - "accuracy/test_cli_flow.py::TestLlama3_1_8BInstruct::test_fp8_prequantized": 309.3845, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False]": 276.868, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True]": 180.626, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False]": 178.923, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True]": 168.358, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_spec_dec": 187.0915, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync": 289.4875, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0]": 1271.689, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 732.0625, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 114.35375, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python": 312.70975, - "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend": 299.9415, - "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False]": 303.0855, - "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True]": 292.9205, - "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python": 344.99325, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-False]": 223.599, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True]": 221.901, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-False]": 246.1605, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-True]": 231.4125, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-False]": 201.3755, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-True]": 189.674, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-False]": 213.785, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-True]": 208.0475, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-False]": 228.887, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-True]": 219.7145, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-False]": 241.8505, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-True]": 241.592, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-False]": 200.7015, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-True]": 188.3745, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-False]": 208.868, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-True]": 206.58, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_beam_search": 857.149, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2]": 181.624, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4]": 186.7045, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=2]": 156.493, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=4]": 134.614, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=2]": 106.802, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=4]": 103.261, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=2]": 103.2105, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=4]": 104.7955, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 202.1325, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_gen_only_spec_dec": 272.8145, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 84.5875, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 111.8185, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python": 184.017, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[GSM8K]": 168.5725, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[MMLU]": 100.4705, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp1pp2]": 213.444, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp1]": 154.1655, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp2]": 163.689, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp1pp2]": 106.883, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp1]": 109.68, - "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp2]": 97.014, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-False]": 433.142, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-True]": 399.9745, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True-True]": 405.477, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_chunked_prefill": 429.4015, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first[adp-ctx_tp2pp1-gen_tp2pp1]": 172.5875, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first[noadp-ctx_tp2pp1-gen_tp1pp1]": 160.7555, - "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend": 190.17025, - "accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy": 269.82425, - "accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt": 270.78825, - "accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap": 265.30175, - "accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8]": 299.6765, - "accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4]": 268.541, - "accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[qwen3vl_2b_instruct]": 194.4825, - "accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[no_overlap]": 122.1595, - "accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[overlap]": 119.8205, - "accuracy/test_llm_api_autodeploy.py::TestGPTOSS::test_mxfp4_gsm8k[120b-tp2]": 318.294, - "accuracy/test_llm_api_autodeploy.py::TestGemma4MoE::test_bf16": 1165.0775, - "accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma3n_e2b_it": 946.732, - "accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma4_e2b_it": 401.988, - "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_attention_dp[4]": 423.6015, - "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton-False-1]": 471.6145, - "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-1]": 155.56525, - "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-4]": 165.7165, - "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[flashinfer]": 202.4085, - "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[trtllm]": 190.87525, - "accuracy/test_llm_api_autodeploy.py::TestMiniMaxM2::test_finegrained_fp8": 707.0315, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[Qwen_QwQ-32B-False]": 412.705, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[deepseek-ai_DeepSeek-R1-0528-True]": 691.8425, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[google_gemma-3-1b-it-False]": 69.5815, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.1-8B-Instruct-False]": 56.201, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.3-70B-Instruct-False]": 170.709, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Codestral-22B-v0.1-False]": 105.679, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Ministral-8B-Instruct-2410-False]": 81.683, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-FP8-True]": 130.6865, - "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-NVFP4-True]": 161.1395, - "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-attn_dp_off-trtllm]": 456.2005, - "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-1-attn_dp_off-trtllm]": 408.0825, - "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-4-attn_dp_off-trtllm]": 315.42, - "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-4-attn_dp_on-trtllm]": 238.353, - "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_accuracy[fp8-4-attn_dp_off-trtllm]": 504.599, - "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_accuracy[nvfp4-1-attn_dp_off-trtllm]": 565.933, - "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[bf16]": 318.981, - "accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-4]": 2399.9215, - "accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8]": 2745.1285, - "accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_fp8[True]": 862.77125, - "accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4]": 623.947, - "accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8]": 1336.0605, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[latency]": 1860.3565, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp_tp4]": 643.6395, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_skip_indexer]": 1409.5745, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 137.1115, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 146.819, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 146.4765, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 157.991, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 123.9885, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 131.109, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 132.9965, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 145.4025, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 169.5815, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 179.0795, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 126.975, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 136.892, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 146.7295, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 160.5725, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 161.7275, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 384.1665, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 123.76, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 137.7825, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 140.497, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 158.2565, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 118.4985, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 139.3005, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 122.96, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 134.7815, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 173.3495, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 324.5846666666667, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True-v2_kv_cache=False]": 510.7425, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True-v2_kv_cache=True]": 917.914, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 171.095, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 78.3665, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 84.0355, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 73.131, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 80.026, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 79.44, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 75.187, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 85.4745, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 67.4295, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 234.031, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 206.9026, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 79.6345, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 193.37675, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 77.051, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 82.2895, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 79.1325, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 84.6905, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 87.6385, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 221.87816666666666, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 85.887, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 90.902, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 82.4455, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 234.095375, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 181.83875, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 79.9465, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 170.54233333333335, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 78.516, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 122.8175, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 76.381, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 86.6995, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 78.246, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 102.9415, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 77.611, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 87.3365, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 245.976, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 72.249, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 84.918, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 262.1948, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_kv_cache_aware_routing[mtp_nextn=0]": 164.21, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_kv_cache_aware_routing[mtp_nextn=2]": 229.00383333333335, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=CUTLASS]": 286.3555, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[ep4-mtp_nextn=0]": 176.09633333333335, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[ep4-mtp_nextn=2]": 214.70733333333334, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[tp4-mtp_nextn=0]": 163.37533333333334, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[tp4-mtp_nextn=2]": 224.63966666666667, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=False]": 199.0495, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=True]": 310.00675, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=False]": 248.528, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True]": 2813.2805, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-enable_chunked_prefill=False]": 261.7495, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-enable_chunked_prefill=True]": 1231.351, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=False]": 351.6755, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True]": 2969.883, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-enable_chunked_prefill=False]": 314.646, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-enable_chunked_prefill=True]": 791.121, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=fp8-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True]": 370.18625, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=False-fp8kv=False-overlap_scheduler=True]": 273.45925, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True]": 269.788, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=nvfp4-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True]": 287.4875, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=nvfp4-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True]": 289.6345, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_dummy_load_format": 110.87025, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 156.161, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 144.2755, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 154.8785, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 132.9995, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 125.881, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 145.4785, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 129.662, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 284.91225, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 137.8345, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 127.4445, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 121.4515, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 139.725, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 125.7285, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 127.524, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 168.432, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=vanilla-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 129.509, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 157.2875, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 104.4745, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 98.1365, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 156.635, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 106.301, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 96.591, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 117.6555, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 116.9035, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 120.706, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 104.3765, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 100.767, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 180.077, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 131.2345, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 159.402, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 108.4475, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 109.228, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 109.9205, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 119.3735, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 104.833, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 123.8425, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 108.9635, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=True]": 108.491, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 96.358, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-sampler_async_worker=False]": 126.2465, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 101.897, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=True]": 102.3075, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 201.2585, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 134.0015, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 132.085, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 191.6285, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=True]": 190.8455, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 97.372, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 97.0235, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 91.4205, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 101.4465, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 194.9305, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=0]": 127.883, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=2]": 126.9225, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=0]": 110.667, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=2]": 109.686, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 89.517, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 208.251, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=0]": 117.001, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=2]": 67.531, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=fp8-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True]": 163.013, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTEDSL-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 197.312, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTEDSL-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 307.229, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 142.8815, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 152.849, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 139.26, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 151.373, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 142.815, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 165.3095, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 138.574, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 175.8045, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 157.603, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 240.7455, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 244.274, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 233.1385, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 274.584, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 141.0775, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 168.19725, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 241.9515, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 254.805, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 198.5995, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-ep4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 246.686, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 257.149, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 253.0845, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 251.22225, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 372.246, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 174.79139999999998, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 168.551, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 253.94811111111113, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 183.4455, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 145.95433333333335, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 235.84845454545456, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 212.01, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 248.05683333333334, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 131.19, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 164.239, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 151.2365, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 153.98466666666667, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 164.37876923076922, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 216.47136363636363, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 235.656, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 227.186125, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 219.37781818181818, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 220.86844444444444, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 231.07783333333333, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 184.051, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 181.111, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 181.89375, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 179.3555, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 258.88188888888885, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=2-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 245.006, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=True-moe_backend=TRTLLM]": 209.7815, - "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_batch_waiting[batch_wait_timeout_iters=10-batch_wait_max_tokens_ratio=1.0-mtp_nextn=0-fp8kv=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-v2_kv_cache=True]": 475.6235, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dummy_load_format": 112.2155, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-one_model-overlap_scheduler]": 2191.629, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler]": 1442.558, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-one_model-no_overlap_scheduler]": 1417.956, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-one_model-overlap_scheduler]": 534.9685, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-one_model-no_overlap_scheduler]": 422.698, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-one_model-overlap_scheduler]": 442.0965, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-one_model-no_overlap_scheduler]": 1218.89175, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-one_model-overlap_scheduler]": 423.365, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-no_overlap_scheduler]": 1048.803, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-overlap_scheduler]": 621.9741666666666, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-auto]": 538.6682857142857, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-auto]": 257.006, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-fp8]": 209.844, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-auto]": 228.4475, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-trtllm-auto]": 259.6665, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-cutlass-auto]": 690.5105714285714, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-triton-auto]": 1521.0425, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-auto]": 648.173, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-fp8]": 684.241, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-triton-auto]": 600.935, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-auto]": 408.4315, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-fp8]": 313.007, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-triton-auto]": 492.58, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-trtllm-auto]": 536.685, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-triton-auto]": 960.579, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-auto]": 557.009, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-fp8]": 708.56175, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-triton-auto]": 623.248, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-auto]": 661.86875, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-fp8]": 1684.369, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-auto]": 461.847, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache_no_reuse-tp4-cutlass-auto]": 427.547, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus_online_eplb[fp8]": 340.66, - "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto]": 430.6515, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype": 295.2365, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse": 157.934, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse": 172.126, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse": 83.8615, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler": 79.696, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse": 91.1525, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse": 102.4165, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse": 77.721, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse": 77.781, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler": 80.157, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available": 91.6255, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False]": 289.573, - "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True]": 318.724, - "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized": 453.2665, - "accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_bf16": 2236.8845, - "accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_fp8": 563.1595, - "accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_nvfp4": 467.06175, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4": 226.72225, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4_streaming[stream_interval_4]": 192.3695, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4_streaming[stream_interval_64]": 203.0215, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=False]": 116.4265, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=True]": 150.031, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=False]": 102.292, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=True]": 132.6645, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=TRTLLM-torch_compile=False]": 118.01385714285715, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=False]": 144.9825, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=True]": 81.482, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=TRTLLM-torch_compile=False]": 68.282, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=TRTLLM-torch_compile=True]": 75.0055, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=False]": 118.71718181818183, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=True]": 94.256, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=TRTLLM-torch_compile=True]": 101.8575, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dflash": 183.366, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dummy_load_format": 41.1125, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=True-overlap_scheduler=True]": 1084.6785, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=True-eagle3_one_model=True-overlap_scheduler=True]": 1083.633, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 87.024, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 133.9335, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 82.5345, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=True]": 121.426, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 84.2995, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 140.0345, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 82.1835, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 111.3025, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 90.3895, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 97.6275, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 132.16025, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 82.635, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 83.7005, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 104.02969999999999, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 98.772, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 87.5075, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 66.816, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 184.921, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 83.00090909090909, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 105.571, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 57.7005, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 55.4, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True]": 65.228, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 53.736, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[llguidance]": 59.7825, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[xgrammar]": 52.646, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[v2_kv_cache=False-attn_backend=TRTLLM-torch_compile=False]": 111.9135, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[v2_kv_cache=False-attn_backend=TRTLLM-torch_compile=True]": 128.7025, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[v2_kv_cache=True-attn_backend=TRTLLM-torch_compile=True]": 337.988, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard[overlap_scheduler=False]": 901.7925, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard[overlap_scheduler=True]": 868.484, - "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B_Instruct_RocketKV::test_auto_dtype": 952.157, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=False-enable_gemm_allreduce_fusion=False]": 497.655, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False]": 784.086, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=False]": 704.389, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=True]": 710.6545, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=False]": 861.40825, - "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True]": 666.765, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_fp8": 309.765, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-1-False-False-False]": 442.911, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-1-False-False-True]": 351.5225, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-1-True-True-False]": 273.783, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-1-True-True-True]": 254.631, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-4-False-True-False]": 413.633, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-4-False-True-True]": 456.0315, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-4-True-False-False]": 221.706, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-4-True-False-True]": 192.302, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-4-True-True-True]": 298.921, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_bf16_trtllm_gen_moe_backend[attention_dp=False]": 476.645, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_bf16_trtllm_gen_moe_backend[attention_dp=True]": 284.88275, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-cpp_mamba_cache]": 378.571, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-python_mamba_cache]": 387.563, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-cpp_mamba_cache]": 215.0015, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-python_mamba_cache]": 220.314, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4]": 790.211, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4_ADP]": 461.178, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4_ADP_MTP]": 387.168, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_hopper_w4a16": 457.0335, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL]": 441.0046666666667, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=CUTLASS]": 402.3545, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=TRTLLM]": 486.2625, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL]": 419.1075, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=CUTLASS]": 246.6785, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM]": 258.079, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpu_mtp_ar": 232.6265, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[ADP4_MTP]": 867.77725, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL]": 711.49175, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=CUTLASS]": 404.3535, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM]": 447.7805, - "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[ADP2_PP2]": 673.1436666666666, - "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp1_block_reuse-cutlass]": 1013.166, - "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep4_adp_off-cutlass]": 2169.0726666666665, - "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep4_adp_off-trtllm]": 2216.35575, - "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp4ep4_adp_on-cutlass]": 1955.3773333333334, - "accuracy/test_llm_api_pytorch.py::TestQwen3NextThinking::test_auto_dtype[tp4ep4]": 523.71025, - "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_4gpus[latency_moe_cutlass]": 2459.811, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_dummy_load_format": 71.0995, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False]": 126.718, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True]": 178.0005, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=False]": 151.8095, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True]": 191.7025, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=False]": 160.995, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True]": 227.28, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_trtllm-torch_compile=False]": 188.46433333333334, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_trtllm-torch_compile=True]": 217.737, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False]": 425.1046666666667, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True]": 231.47466666666665, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=False]": 219.1575, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=True]": 212.452, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_cutlass-torch_compile=False]": 401.3496666666667, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_cutlass-torch_compile=True]": 304.056, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=False]": 237.966, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=True]": 235.95866666666666, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRTLLM]": 114.6075, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-CUTLASS]": 115.959, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRTLLM]": 118.66, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-CUTLASS]": 88.0165, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-TRTLLM]": 94.737, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention[target_sparsity_0.9-fp8kv=True]": 930.37, - "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=True]": 665.6993333333334, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-CUTLASS]": 295.65, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-TRTLLM]": 266.626, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp2-CUTLASS]": 337.7305, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp2-TRTLLM]": 239.6865, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_mtp[mtp_off]": 271.1185, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=False]": 302.241, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=True]": 471.724, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_cutedsl]": 764.5365, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_trtllm]": 860.0565, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_block_reuse]": 1057.574, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_trtllm]": 628.9185, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash": 615.05, - "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8": 685.529, - "accuracy/test_llm_api_pytorch.py::TestQwen3_6_27B::test_fp8": 333.238, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[latency]": 470.8513333333333, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dflash": 198.264, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dummy_load_format": 67.6375, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=False]": 113.8705, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=True]": 118.002, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=True-enable_draft_len_schedule=False]": 126.839, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=True-enable_max_concurrency=False-enable_draft_len_schedule=False]": 246.5975, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency]": 271.17159999999996, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[fp8-latency]": 57.58683333333334, - "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[mxfp8-latency]": 82.751, - "accuracy/test_llm_api_pytorch.py::TestSeedOss_36B::test_auto_dtype": 1201.6095, - "accuracy/test_llm_api_pytorch.py::TestStep3_7::test_auto_dtype[tp_size=8-ep_size=8]": 7200.0, - "accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=0]": 7200.0, - "accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=3]": 7200.0, - "accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=0]": 7200.0, - "accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=3]": 7200.0, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_cuda_graph_matches_eager_logits[tinyllama-1.1b]": 19.1135, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[gemma-3-1b]": 27.082, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[phi-4-mini]": 35.2105, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen2-7b]": 37.937, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen3-0.6b]": 13.976, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[starcoder2-3b]": 62.35, - "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[tinyllama-1.1b]": 7.3275, - "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_cuda_graph_matches_eager_logits[bert-yelp]": 7.9375, - "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_classification[bert-yelp-cuda_graph]": 5.185, - "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_classification[bert-yelp-eager]": 4.5535, - "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_per_token_reward[qwen2.5-prm-7b]": 37.9115, - "accuracy/test_llm_api_pytorch_multimodal.py::TestMistralSmall24B::test_auto_dtype[forced_chunked_prefill]": 225.865, - "accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[bf16]": 379.987, - "accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8]": 863.22, - "accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[nvfp4]": 531.742, - "accuracy/test_llm_api_pytorch_multimodal.py::TestNemotron_Nano_12B_V2_VL::test_auto_dtype[forced_chunked_prefill]": 372.492, - "accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_5_VL_7B::test_auto_dtype": 828.9645, - "accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL::test_auto_dtype[forced_chunked_prefill]": 238.39, - "accuracy/test_llm_api_pytorch_multimodal.py::TestVILA1_5_3B::test_auto_dtype": 356.457, - "cpp/test_e2e.py::test_benchmarks[t5-90]": 580.63, - "cpp/test_e2e.py::test_model[-enc_dec_language_adapter-90]": 237.6715, - "cpp/test_e2e.py::test_model[-gpt_executor-80]": 1174.5125, - "cpp/test_e2e.py::test_model[-gpt_tests-80]": 705.1885, - "cpp/test_e2e.py::test_model[-medusa-86]": 545.2025, - "cpp/test_e2e.py::test_model[-t5-90]": 183.012, - "cpp/test_e2e.py::test_model[fp8-llama-90]": 373.718, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-mpi_kvcache-90]": 854.0535, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-nixl_kvcache-90]": 12.3445, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-4proc-ucx_kvcache-90]": 12.0945, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-mpi_kvcache-90]": 27.0285, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-nixl_kvcache-90]": 26.472, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-6proc-ucx_kvcache-90]": 26.258, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-mpi_kvcache-90]": 14.722, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-nixl_kvcache-90]": 15.325, - "cpp/test_multi_gpu.py::TestDisagg::test_asymmetric_executor[llama-8proc-ucx_kvcache-90]": 14.7605, - "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-mpi_kvcache-90]": 18.438, - "cpp/test_multi_gpu.py::TestDisagg::test_orchestrator_params[llama-ucx_kvcache-90]": 18.755, - "cpp/test_multi_gpu.py::TestDisagg::test_spawn_orchestrator[llama-nixl_kvcache-90]": 40.666, - "cpp/test_multi_gpu.py::TestDisagg::test_spawn_orchestrator[llama-ucx_kvcache-90]": 40.163, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-mpi_kvcache-90]": 6.9815, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-nixl_kvcache-90]": 7.0665, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-2proc-ucx_kvcache-90]": 6.9995, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-mpi_kvcache-90]": 11.5145, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-nixl_kvcache-90]": 11.65, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-4proc-ucx_kvcache-90]": 11.75, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-mpi_kvcache-90]": 14.1805, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-nixl_kvcache-90]": 14.174, - "cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[llama-8proc-ucx_kvcache-90]": 14.1315, - "cpp/test_multi_gpu.py::test_enc_dec[t5-90]": 140.1025, - "cpp/test_multi_gpu.py::test_fused_gemm_allreduce[4proc-90]": 21.5805, - "cpp/test_multi_gpu.py::test_llama_executor[llama-leader-90]": 619.3225, - "cpp/test_multi_gpu.py::test_llama_executor[llama-orchestrator-90]": 188.325, - "cpp/test_multi_gpu.py::test_llama_executor_guided_decoding[llama-90]": 15.448, - "cpp/test_multi_gpu.py::test_llama_executor_logits_proc[llama-90]": 46.4445, - "cpp/test_multi_gpu.py::test_mpi_utils[90]": 47.964, - "cpp/test_multi_gpu.py::test_trt_gpt_real_decoder[llama-90]": 112.2465, - "cpp/test_multi_gpu.py::test_user_buffer[2proc-90]": 4.2345, - "cpp/test_unit_tests.py::test_unit_tests[common-80]": 257.673, - "cpp/test_unit_tests.py::test_unit_tests[common-90]": 19.7405, - "cpp/test_unit_tests.py::test_unit_tests[executor-80]": 498.0095, - "cpp/test_unit_tests.py::test_unit_tests[kernels-80]": 1237.114, - "cpp/test_unit_tests.py::test_unit_tests[kernels-90]": 1102.233, - "cpp/test_unit_tests.py::test_unit_tests[layers-80]": 1083.6465, - "cpp/test_unit_tests.py::test_unit_tests[layers-90]": 803.6475, - "cpp/test_unit_tests.py::test_unit_tests[runtime-80]": 719.1235, - "cpp/test_unit_tests.py::test_unit_tests[thop-80]": 3.441, - "cpp/test_unit_tests.py::test_unit_tests[thop-90]": 3.1505, - "cpp/test_unit_tests.py::test_unit_tests[utils-80]": 4.5175, - "cpp/test_unit_tests.py::test_unit_tests[utils-90]": 4.088, - "disaggregated/test_auto_scaling.py::test_disagg_server_restart[etcd-round_robin]": 115.3005, - "disaggregated/test_auto_scaling.py::test_disagg_server_restart[http-round_robin]": 155.1545, - "disaggregated/test_auto_scaling.py::test_minimal_instances[etcd-round_robin]": 118.212, - "disaggregated/test_auto_scaling.py::test_minimal_instances[http-round_robin]": 111.0575, - "disaggregated/test_auto_scaling.py::test_service_discovery[etcd-round_robin]": 56.3725, - "disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cache_aware]": 56.347, - "disaggregated/test_auto_scaling.py::test_service_discovery[http-round_robin]": 61.0085, - "disaggregated/test_auto_scaling.py::test_worker_restart[etcd-load_balancing]": 253.7195, - "disaggregated/test_auto_scaling.py::test_worker_restart[etcd-round_robin]": 250.7945, - "disaggregated/test_auto_scaling.py::test_worker_restart[http-load_balancing]": 239.9285, - "disaggregated/test_disaggregated.py::test_disaggregated_benchmark_gen_only_insufficient_kv[TinyLlama-1.1B-Chat-v1.0]": 92.463, - "disaggregated/test_disaggregated.py::test_disaggregated_cache_aware_balance[TinyLlama-1.1B-Chat-v1.0]": 126.614, - "disaggregated/test_disaggregated.py::test_disaggregated_chat_completion_tool_calls[TinyLlama-1.1B-Chat-v1.0]": 67.302, - "disaggregated/test_disaggregated.py::test_disaggregated_conditional[TinyLlama-1.1B-Chat-v1.0]": 86.493, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0]": 77.819, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_gentp2[TinyLlama-1.1B-Chat-v1.0]": 79.2265, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0]": 90.0015, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2_genpp2[TinyLlama-1.1B-Chat-v1.0]": 75.9995, - "disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2pp2_gentp2pp2[TinyLlama-1.1B-Chat-v1.0]": 68.377, - "disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0]": 77.759, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_empty_batch[DeepSeek-V3-Lite-bf16]": 210.885, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp[DeepSeek-V3-Lite-fp8]": 161.026, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8]": 125.1935, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_transceiver_runtime_python[DeepSeek-V3-Lite-fp8]": 159.2875, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8]": 127.536, - "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 145.9755, - "disaggregated/test_disaggregated.py::test_disaggregated_diff_max_tokens[TinyLlama-1.1B-Chat-v1.0]": 69.532, - "disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0]": 86.382, - "disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0]": 76.192, - "disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0]": 122.491, - "disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0]": 82.68, - "disaggregated/test_disaggregated.py::test_disaggregated_ngram[TinyLlama-1.1B-Chat-v1.0]": 84.112, - "disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0]": 90.595, - "disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python[TinyLlama-1.1B-Chat-v1.0]": 100.3685, - "disaggregated/test_disaggregated.py::test_disaggregated_perf_metrics[TinyLlama-1.1B-Chat-v1.0]": 86.895, - "disaggregated/test_disaggregated.py::test_disaggregated_qwen3_32b_fp8[Qwen3/Qwen3-32B-FP8]": 225.587, - "disaggregated/test_disaggregated.py::test_disaggregated_single_gpu[TinyLlama-1.1B-Chat-v1.0]": 79.562, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_cancel_gen_requests[TinyLlama-1.1B-Chat-v1.0]": 32.5555, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[False-TinyLlama-1.1B-Chat-v1.0]": 49.42125, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[True-TinyLlama-1.1B-Chat-v1.0]": 38.481, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[False-TinyLlama-1.1B-Chat-v1.0]": 41.4455, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[True-TinyLlama-1.1B-Chat-v1.0]": 42.0695, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8]": 97.16925, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-True-DeepSeek-V3-Lite-fp8/fp8]": 75.2365, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-False-DeepSeek-V3-Lite-fp8/fp8]": 90.727, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-False-TinyLlama-1.1B-Chat-v1.0]": 34.437333333333335, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-True-TinyLlama-1.1B-Chat-v1.0]": 37.568, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0]": 43.664, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0]": 46.79, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-False-Qwen3-8B-FP8]": 77.82925, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-True-Qwen3-8B-FP8]": 55.2515, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-False-Qwen3-8B-FP8]": 91.5985, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-True-Qwen3-8B-FP8]": 78.294, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[False-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct]": 65.7875, - "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[True-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct]": 56.474, - "disaggregated/test_workers.py::test_workers_conditional_disaggregation_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16]": 93.0355, - "disaggregated/test_workers.py::test_workers_kv_cache_aware_router[TinyLlama-1.1B-Chat-v1.0]": 120.034, - "examples/test_ad_guided_decoding.py::test_autodeploy_guided_decoding_main_json": 125.6035, - "examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[flashinfer-torch-simple]": 86.3555, - "examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[trtllm-torch-cudagraph]": 68.4185, - "examples/test_ad_speculative_decoding.py::test_eagle_model_with_weights": 3.358, - "examples/test_ad_speculative_decoding.py::test_nemotron_mtp_model_with_weights": 749.4955, - "examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:1-pp:1-float16-BertModel-bert/bert-base-uncased]": 91.8135, - "examples/test_enc_dec.py::test_llm_enc_dec_mmlu[flan-t5-small-float32-tp:1-pp:1-nb:1-disable_fp8]": 404.8275, - "examples/test_llm_api_with_mpi.py::test_llm_api_single_gpu_with_mpirun[TinyLlama-1.1B-Chat-v1.0]": 50.97832, - "examples/test_qwen.py::test_llm_hf_qwen_multi_lora_1gpu[qwen2.5_1.5b_instruct]": 195.3725, - "examples/test_ray.py::test_llm_inference_async_ray": 128.646, - "examples/test_ray.py::test_llm_inference_distributed_ray[pp2]": 148.069, - "examples/test_ray.py::test_llm_inference_distributed_ray[tep2]": 122.27, - "examples/test_ray.py::test_llm_inference_distributed_ray[tp2]": 82.56, - "examples/test_ray.py::test_llm_inference_distributed_ray[tp2pp2]": 134.1765, - "examples/test_visual_gen.py::test_cosmos3_example": 198.1875, - "examples/test_visual_gen.py::test_flux1_example": 200.7805, - "examples/test_visual_gen.py::test_flux2_example": 189.8375, - "examples/test_visual_gen.py::test_ltx2_example": 169.9875, - "examples/test_visual_gen.py::test_vbench_dimension_score_wan": 286.764, - "examples/test_visual_gen.py::test_visual_gen_api_walkthrough": 810.9144444444445, - "examples/test_visual_gen.py::test_visual_gen_quickstart": 142.78755555555557, - "examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_cpp_runtime]": 184.4105, - "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_batch_generation": 55.184, - "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_cold_start": 32.4805, - "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_field_completeness": 55.8465, - "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_full_block_reuse": 43.6745, - "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_long_context": 1.585, - "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_partial_block_reuse": 33.5325, - "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_rapid_fire": 44.4985, - "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_shared_prefix": 41.38, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_chunked_draft_tokens": 477.218, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_draft_tokens": 208.925, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_eviction": 118.7415, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_batch_size_limited": 287.4475, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_block_reuse": 408.856, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill": 434.23, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill_eviction_block_reuse": 52.1345, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill_multi_request": 256.3575, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill_with_eviction": 76.184, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction[cuda_graph]": 73.288, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction[no_cuda_graph]": 36.0105, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction_overlap": 55.214, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction_with_block_reuse": 51.008, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_overlap_scheduler[non_overlap]": 306.47, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_overlap_scheduler[overlap]": 359.397, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_partial_block_reuse": 565.495, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_token_budget_limited": 717.9695, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_v2_vs_v1_basic": 626.882, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2LoRA::test_lora_chunked_prefill": 538.7965, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2LoRA::test_lora_eviction": 106.776, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2LoRA::test_lora_multi_adapter_v2": 252.9395, - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2LoRA::test_lora_v2": 427.8815, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[guaranteed-chunked]": 177.728, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[max-util-chunked]": 172.9695, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[no-overlap-chunked]": 182.683, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[offload-chunked]": 183.431, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[offload-no-chunked]": 187.139, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[python-scheduler]": 178.3445, - "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[swa-chunked]": 161.1855, - "llmapi/test_llm_api_connector.py::test_connector_async_onboard[False]": 20.569, - "llmapi/test_llm_api_connector.py::test_connector_async_onboard[True]": 13.357, - "llmapi/test_llm_api_connector.py::test_connector_async_save[False]": 13.208, - "llmapi/test_llm_api_connector.py::test_connector_async_save[True]": 13.344, - "llmapi/test_llm_api_connector.py::test_connector_disagg_prefill[False]": 37.597, - "llmapi/test_llm_api_connector.py::test_connector_disagg_prefill[True]": 25.431, - "llmapi/test_llm_api_connector.py::test_connector_e2e_persistent_cache": 27.2365, - "llmapi/test_llm_api_connector.py::test_connector_multi_request": 10.822, - "llmapi/test_llm_api_connector.py::test_connector_priorities": 13.194, - "llmapi/test_llm_api_connector.py::test_connector_priorities_default": 14.629, - "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[attention_dp]": 3.272, - "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[beam_search]": 8.619, - "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[host_offloading]": 14.426, - "llmapi/test_llm_api_connector.py::test_connector_scheduler_output[False]": 16.7435, - "llmapi/test_llm_api_connector.py::test_connector_scheduler_output[True]": 17.794, - "llmapi/test_llm_api_connector.py::test_connector_scheduler_output_chunked_context[False]": 11.278, - "llmapi/test_llm_api_connector.py::test_connector_scheduler_output_chunked_context[True]": 10.906, - "llmapi/test_llm_api_connector.py::test_connector_simple[False]": 14.34, - "llmapi/test_llm_api_connector.py::test_connector_simple[True]": 13.8555, - "llmapi/test_llm_examples.py::test_llmapi_chat_example": 94.338, - "llmapi/test_llm_examples.py::test_llmapi_example_guided_decoding": 129.84318518518518, - "llmapi/test_llm_examples.py::test_llmapi_example_inference": 79.47485185185185, - "llmapi/test_llm_examples.py::test_llmapi_example_inference_async": 72.30592592592593, - "llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming": 50.056629629629626, - "llmapi/test_llm_examples.py::test_llmapi_example_logits_processor": 49.90185185185185, - "llmapi/test_llm_examples.py::test_llmapi_example_multilora": 68.58925925925925, - "llmapi/test_llm_examples.py::test_llmapi_kv_cache_connector[Qwen2-0.5B]": 91.1625, - "llmapi/test_llm_examples.py::test_llmapi_quickstart": 50.488962962962965, - "llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit": 339.47488888888887, - "llmapi/test_llm_examples.py::test_llmapi_server_example": 93.523, - "llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_ngram": 99.05975, - "perf/host_perf/test_module_resource_manager.py::test_kv_cache_prepare_context": 1.1685, - "perf/host_perf/test_module_resource_manager.py::test_kv_cache_prepare_generation": 1.36, - "perf/host_perf/test_module_sampler.py::test_sampler_update_greedy[greedy_bs8]": 1.4965, - "perf/host_perf/test_module_sampler.py::test_sampler_update_stop_words[stopwords_bs32]": 2.111, - "perf/host_perf/test_module_scheduler.py::test_scheduler_production[production_gen_only_bs8]": 1.3745, - "perf/host_perf/test_module_scheduler.py::test_scheduler_production[production_mixed_32gen_4ctx]": 1.645, - "perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:8192]": 519.5475, - "perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]": 233.7635, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_blackwell-r1_fp4_v2_tp4_mtp3_1k1k]": 329.801, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_blackwell-r1_fp4_v2_tp4_mtp3_8k1k]": 360.471, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_1k1k]": 549.78575, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_1k8k]": 1028.03525, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_8k1k]": 626.6883333333334, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_1k1k]": 503.37833333333333, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_1k8k]": 755.15525, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_8k1k]": 497.62166666666667, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_blackwell-r1_fp8_tp8_mtp3_1k1k]": 1028.532, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_blackwell-r1_fp8_tp8_mtp3_8k1k]": 839.75, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_1k1k]": 1165.225, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_tep4_mtp3_1k1k]": 564.3545, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_1k1k]": 287.24675, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_8k1k]": 318.955, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_adp_2k1k]": 605.373, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_235b_a22b_fp8_hopper-qwen3_235b_a22b_fp8_tp4_ep4_cutlass_8k1k]": 942.944, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_32b_fp8_hopper-qwen3_32b_fp8_tp2_6k1k]": 744.4795, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_grace_blackwell-glm5_fp4_dep4_mtp1_1k1k]": 596.478, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_grace_blackwell-glm5_fp4_tep4_mtp3_1k1k]": 712.10975, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tp1_mtp0_1k1k]": 332.7485, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tp1_mtp0_8k1k]": 629.5325, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tp2_mtp0_1k1k]": 227.3105, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_dep2_1k1k]": 473.9215, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_dep4_1k8k]": 1391.65175, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tep2_1k8k]": 846.39475, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_1k1k]": 399.961, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_8k1k]": 686.33625, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_1k8k]": 393.1165, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_mtp0_1k1k]": 169.305, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp4_eagle3_1k1k]": 290.457, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-host_perf_deepseek_v3_lite-v3lite_fp8_bs8_128_256]": 396.2785, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-host_perf_llama8b-llama8b_fp16_bs8_128_256]": 283.202, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-host_perf_llama8b_spec_decode-llama8b_spec_bs1_128_128]": 141.4225, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_tep8_32k8k]": 677.6165, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_32k8k]": 717.075, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_8k1k]": 364.3665, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_grace_blackwell-k25_thinking_fp4_tep4_8k1k]": 539.759, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-llama_v3_3_70b_instruct_fp4_blackwell-llama70b_fp4_tp4_1000_1000]": 540.5185, - "perf/test_perf_sanity.py::test_e2e[aggr_upload-llama_v3_3_70b_instruct_fp4_blackwell-llama70b_fp4_tp4_512_32]": 248.239, - "ray_orchestrator/RL/test_rl_perf_reproduce.py::test_rl_perf_reproduce[tp1_4instances]": 129.8185, - "ray_orchestrator/RL/test_rl_perf_reproduce.py::test_rl_perf_reproduce[tp2_2instances]": 131.3095, - "test_e2e.py::test_get_ci_container_port": 1.3305, - "test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct]": 157.9205, - "test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b]": 199.444, - "test_e2e.py::test_openai_chat_harmony": 282.97, - "test_e2e.py::test_openai_chat_harmony_perf_metrics": 210.3415, - "test_e2e.py::test_openai_chat_with_logit_bias[torch_sampler]": 83.3965, - "test_e2e.py::test_openai_chat_with_logit_bias[trtllm_sampler]": 92.0, - "test_e2e.py::test_openai_completions_example[pytorch]": 276.244, - "test_e2e.py::test_openai_completions_with_logit_bias[torch_sampler]": 91.7945, - "test_e2e.py::test_openai_completions_with_logit_bias[trtllm_sampler]": 78.521, - "test_e2e.py::test_openai_health": 66.8625, - "test_e2e.py::test_openai_lora": 215.691, - "test_e2e.py::test_openai_misc_example[pytorch]": 378.285, - "test_e2e.py::test_openai_mmencoder_example": 131.807, - "test_e2e.py::test_openai_perf_metrics": 79.008, - "test_e2e.py::test_openai_prometheus": 95.94, - "test_e2e.py::test_openai_reasoning[pytorch]": 298.189, - "test_e2e.py::test_openai_responses": 374.652, - "test_e2e.py::test_openai_responses_entrypoint": 106.551, - "test_e2e.py::test_openai_tool_call": 95.56, - "test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-20B-gpt_oss/gpt-oss-20b]": 500.0685, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]": 247.346, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]": 176.78275, - "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B]": 186.0085, - "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B]": 1002.525, - "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_fp8_hf-Qwen3/saved_models_Qwen3-30B-A3B_fp8_hf]": 546.545, - "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_nvfp4_hf-Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf]": 374.422, - "test_e2e.py::test_ptp_quickstart_advanced_bs1": 121.2115, - "test_e2e.py::test_ptp_quickstart_advanced_deepseek_v3_lite_4gpus_adp_balance[DeepSeek-V3-Lite-FP8-DeepSeek-V3-Lite/fp8]": 121.632, - "test_e2e.py::test_ptp_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B]": 65.3405, - "test_e2e.py::test_ptp_quickstart_advanced_mixed_precision": 94.1605, - "test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]": 171.3395, - "test_e2e.py::test_ptp_quickstart_advanced_mtp_eagle[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]": 254.351, - "test_e2e.py::test_ptp_quickstart_advanced_ngram[Llama-3.1-8B-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct]": 65.5025, - "test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 51.9431, - "test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 23.3442, - "test_e2e.py::test_trtllm_bench_help_sanity[meta-llama/Llama-3.1-8B]": 84.428, - "test_e2e.py::test_trtllm_bench_invalid_token_pytorch[TinyLlama-1.1B-Chat-v1.0-TinyLlama-1.1B-Chat-v1.0]": 88.27, - "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 100.346, - "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 105.9055, - "test_e2e.py::test_trtllm_bench_llmapi_launch[pytorch_backend-llama-v3-llama3-8b]": 136.6155, - "test_e2e.py::test_trtllm_bench_mgmn": 105.666, - "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False]": 139.09133333333335, - "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-instruct-hf-fp8-True-True]": 85.8235, - "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-]": 346.148, - "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-enable_request_rate]": 239.176, - "test_e2e.py::test_trtllm_serve_example": 214.5225, - "test_e2e.py::test_trtllm_serve_lora_example": 92.415, - "test_e2e.py::test_trtllm_serve_multimodal_example": 225.484, - "test_e2e.py::test_trtllm_serve_top_logprobs[pytorch]": 80.572, - "thirdparty/test_cmake_third_party.py::test_cmake_listfiles": 1.376, - "thirdparty/test_git_modules.py::test_gitmodules": 1.2635, - "triton_server/test_triton.py::test_fill_template[fill-template]": 8.291, - "triton_server/test_triton.py::test_gpt[gpt]": 153.794, - "triton_server/test_triton.py::test_gptj[gptj]": 136.4055, - "triton_server/test_triton.py::test_llmapi_unit_tests[llmapi-unit-tests]": 8.913, - "triton_server/test_triton.py::test_medusa[medusa]": 434.182, - "triton_server/test_triton.py::test_mistral[mistral]": 135.5265, - "triton_server/test_triton.py::test_mistral_ib_streaming[mistral-ib-streaming]": 288.498, - "triton_server/test_triton.py::test_python_multimodal_encoders_unit_tests[python-multimodal-encoders-unit-tests]": 34.445, - "triton_server/test_triton.py::test_python_preproc_unit_tests[python-preproc-unit-tests]": 12.773, - "triton_server/test_triton.py::test_whisper[whisper]": 187.4075, - "triton_server/test_triton_llm.py::test_benchmark_core_model[gptj_6b-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization-4096--1-1-1-False]": 439.7755, - "triton_server/test_triton_llm.py::test_eagle_vicuna_7b_ifb[False-1-eagle--False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-False-ensemble]": 264.508, - "triton_server/test_triton_llm.py::test_gpt_350m_ifb[test_basic-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-True-tensorrt_llm_bls]": 109.477, - "triton_server/test_triton_llm.py::test_gpt_350m_python_backend[accuracy]": 114.526, - "triton_server/test_triton_llm.py::test_gpt_350m_python_backend[e2e]": 54.158, - "triton_server/test_triton_llm.py::test_gpt_disaggregated_serving_bls[test_basic-False-1-top_k_top_p--False-True-True-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-max_utilization---1-1-1-True-tensorrt_llm_bls]": 99.347, - "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[False-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-guaranteed_no_evict---1-1-1-False-ensemble]": 233.0245, - "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[True-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-guaranteed_no_evict---1-1-1-False-ensemble]": 212.8315, - "triton_server/test_triton_llm.py::test_gpt_speculative_decoding_bls[True-False-1---False-True-True-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-max_utilization---1-1-1-False-ensemble]": 31.633, - "triton_server/test_triton_llm.py::test_llmapi_backend[1-0-disableDecoupleMode-tensorrt_llm]": 123.991, - "triton_server/test_triton_llm.py::test_llmapi_backend[1-0-enableDecoupleMode-tensorrt_llm]": 94.4045, - "triton_server/test_triton_llm.py::test_llmapi_lora[1-tensorrt_llm]": 84.5155, - "triton_server/test_triton_llm.py::test_medusa_vicuna_7b_ifb[False-1-medusa--False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--max_utilization---1-1-1-False-ensemble]": 203.8335, - "triton_server/test_triton_llm.py::test_t5_small_enc_dec_ifb[test_basic-False-1-top_k_top_p--False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap---guaranteed_no_evict--4096-1-1-1-False-tensorrt_llm_bls]": 171.811, - "triton_server/test_triton_llm.py::test_tiny_llama_1b_guided_decoding[xgrammar-python-True-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble-accuracy]": 133.0795, - "triton_server/test_triton_llm.py::test_tiny_llama_1b_guided_decoding[xgrammar-tensorrtllm-True-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble-accuracy]": 143.0305, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 131.5315, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble]": 145.041, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[python-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 57.0455, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 35.601, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble]": 36.219, - "triton_server/test_triton_llm.py::test_tiny_llama_ifb_token_counts[tensorrtllm-both-False-1---False-True-False-0-128-enableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-tensorrt_llm_bls]": 39.2385, - "triton_server/test_triton_llm.py::test_whisper_large_v3_ifb[True-1-top_k_top_p--False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap-0.2-0.5-guaranteed_no_evict--24000-1-1-1-False-ensemble]": 163.0495, - "triton_server/test_triton_rcca.py::test_mistral_beam_search[rcca_4714407-True-10---False-True-False-0-128-disableDecoupleMode-inflight_fused_batching-disableTrtOverlap--guaranteed_no_evict---1-1-1-False-ensemble]": 220.108, - "unittest/_torch/attention": 1356.7211666666667, - "unittest/_torch/attention/test_attention_mla.py": 74.021, - "unittest/_torch/compilation": 29.2405, - "unittest/_torch/debugger": 29.3655, - "unittest/_torch/executor": 653.356, - "unittest/_torch/executor/test_async_transfer_manager.py": 25.616, - "unittest/_torch/executor/test_disagg_index_mapper_early_release.py": 31.754, - "unittest/_torch/executor/test_kv_cache_budget_split.py": 32.3015, - "unittest/_torch/executor/test_kv_cache_estimation.py": 31.509, - "unittest/_torch/executor/test_kv_pool_rebalance.py": 31.6415, - "unittest/_torch/executor/test_overlap_scheduler.py": 740.0435, - "unittest/_torch/executor/test_scheduler_serializable_output.py": 29.6455, - "unittest/_torch/flashinfer/test_trtllm_flashinfer_symbol_collision.py": 979.1825, - "unittest/_torch/lora": 26.421, - "unittest/_torch/misc": 411.9765, - "unittest/_torch/misc/test_autotuner.py::test_autotuner_distributed_strategy": 35.7315, - "unittest/_torch/modeling -k \"modeling_gemma3\"": 279.1045, - "unittest/_torch/modeling -k \"modeling_gpt_oss\"": 125.101125, - "unittest/_torch/modeling -k \"modeling_llama\"": 113.31071428571428, - "unittest/_torch/modeling -k \"modeling_mixtral\"": 63.2445, - "unittest/_torch/modeling -k \"modeling_nemotron_nano_v2_vl\"": 536.1425, - "unittest/_torch/modeling -k \"modeling_nemotron_nas\"": 34.836, - "unittest/_torch/modeling -k \"modeling_out_of_tree\"": 140.644, - "unittest/_torch/modeling -k \"modeling_phi3\"": 33.9605, - "unittest/_torch/modeling -k \"modeling_qwen\"": 758.634, - "unittest/_torch/modeling -k \"modeling_qwen_moe\"": 283.413, - "unittest/_torch/modeling -k \"modeling_siglip\"": 128.9035, - "unittest/_torch/modeling -k \"modeling_vila\"": 38.3015, - "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_multimodal_26b_dummy": 19.23, - "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_text_26b_dummy": 19.5185, - "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_text_31b_dummy": 20.4865, - "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_text_e2b_dummy": 21.7525, - "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_text_e4b_dummy": 22.288, - "unittest/_torch/modeling/test_gemma4_multimodal.py": 21.5775, - "unittest/_torch/modeling/test_modeling_afmoe.py": 55.3755, - "unittest/_torch/modeling/test_modeling_cohere2.py": 75.964, - "unittest/_torch/modeling/test_modeling_exaone_moe.py": 234.6705, - "unittest/_torch/modeling/test_modeling_gemma4.py": 127.755, - "unittest/_torch/modeling/test_modeling_llava_next.py::TestLlavaNext::test_all": 26.9655, - "unittest/_torch/modeling/test_modeling_llava_next.py::test_llava_next_expand_prompt_token_ids_for_mm": 13.63175, - "unittest/_torch/modeling/test_modeling_mistral.py": 30.872, - "unittest/_torch/modeling/test_modeling_nemotron_h.py::test_nemotron_h_sanity": 229.8415, - "unittest/_torch/modeling/test_modeling_parakeet.py": 41.127, - "unittest/_torch/modeling/test_modeling_pixtral.py": 38.7415, - "unittest/_torch/modeling/test_modeling_pixtral.py::test_tensor_parallelism": 52.3855, - "unittest/_torch/modeling/test_modeling_qwen2_5vl.py::TestQwen2_5_VL::test_all": 91.596, - "unittest/_torch/modeling/test_modeling_qwen3vl.py::TestQwen3VL::test_all": 110.54016666666668, - "unittest/_torch/modeling/test_modeling_qwen3vl.py::test_qwen3vl_init_preserves_caller_quant_config": 10.659666666666666, - "unittest/_torch/modeling/test_modeling_qwen3vl_moe.py::TestQwen3VLMoe::test_all": 98.443, - "unittest/_torch/modeling/test_modeling_radio.py": 25.226, - "unittest/_torch/modeling/test_modeling_step3p7.py": 24.666, - "unittest/_torch/modeling/test_modeling_step3p7vl.py": 35.211, - "unittest/_torch/modeling/test_nemotron_nano_preprocessing.py": 68.0135, - "unittest/_torch/models/checkpoints/hf/test_checkpoint_loader.py": 50.816, - "unittest/_torch/models/checkpoints/hf/test_weight_loader.py": 29.332, - "unittest/_torch/modules/dwdp/test_dwdp_fixup_moe_backends.py": 29.911, - "unittest/_torch/modules/dwdp/test_dwdp_manager.py": 24.666, - "unittest/_torch/modules/dwdp/test_dwdp_mapping.py": 29.576, - "unittest/_torch/modules/dwdp/test_dwdp_peer_ranges.py": 24.359, - "unittest/_torch/modules/fused_moe/test_deepgemm_fused_expand_quant.py": 25.575333333333333, - "unittest/_torch/modules/fused_moe/test_deepgemm_fused_gather_finalize.py": 26.749333333333333, - "unittest/_torch/modules/mamba": 193.3044, - "unittest/_torch/modules/moe/test_cute_dsl_b12x_moe_backend.py": 23.932, - "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTEDSL\"": 102.8775, - "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTLASS\"": 1373.4741666666666, - "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DEEPGEMM\"": 23.586, - "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DENSEGEMM\"": 625.769, - "unittest/_torch/modules/moe/test_moe_backend.py::test_trtllm_bf16_unquantized_moe": 899.1945, - "unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm": 1483.386, - "unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_boundary": 311.748, - "unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_postquant": 163.0105, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTEDSL\"": 652.5295, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and FP8 and not MXFP8\"": 1919.16925, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and NVFP4\"": 717.7505, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A16_MXFP4\"": 1113.4495, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A8_AWQ\"": 748.3805, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A8_MXFP4_MXFP8\"": 2256.887, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W8A16\"": 755.8705, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"DEEPGEMM and not MEGAMOE_DEEPGEMM\"": 465.981, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"MEGAMOE_DEEPGEMM\"": 46.1035, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"TRTLLM and FP8_BLOCK_SCALES\"": 276.8755, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"TRTLLM and NVFP4 and not W4A8\"": 279.412, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"TRTLLM and W4A16_MXFP4\"": 1041.7595, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"TRTLLM and W4A8_MXFP4_MXFP8\"": 151.341, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"TRTLLM and W4A8_NVFP4_FP8\"": 325.758, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTEDSL-quant=NVFP4-routing=Renormalize]": 58.674, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=FP8-routing=Renormalize]": 44.2235, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=DEEPGEMM-quant=FP8_BLOCK_SCALES-routing=Renormalize]": 45.094, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_NVFP4_FP8-routing=Renormalize]": 70.7675, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]": 43.8915, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=W8A16-routing=Renormalize]": 24.378, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=FP8_BLOCK_SCALES-routing=Renormalize]": 24.4035, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]": 24.3315, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]": 43.8155, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]": 50.57, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]": 43.6625, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb": 73.24525, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]": 43.7885, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]": 44.1825, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]": 44.159, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"CUTEDSL\"": 83.394, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"CUTLASS\"": 349.87575, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"DEEPGEMM\"": 113.702, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"DENSEGEMM\"": 412.07, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"MEGAMOE_DEEPGEMM\"": 99.9455, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"TRTLLM\"": 201.5945, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTEDSL-quant=NVFP4-routing=Renormalize]": 56.91, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=FP8-routing=Renormalize]": 33.4025, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]": 29.3385, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]": 34.443, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=W8A16-routing=Renormalize]": 27.3895, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=DEEPGEMM-quant=FP8_BLOCK_SCALES-routing=Renormalize]": 25.9225, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=FP8_BLOCK_SCALES-routing=Renormalize]": 32.5945, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]": 30.7345, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]": 28.2185, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_NVFP4_FP8-routing=Renormalize]": 53.4255, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]": 17.212571428571426, - "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e8_k1_h512_i512-seq=8-dtype=torch.float16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]": 17.10242857142857, - "unittest/_torch/modules/test_awq_quantization.py": 22.3184, - "unittest/_torch/modules/test_fused_activation_quant.py": 21.9428, - "unittest/_torch/modules/test_fused_add_rms_norm_quant.py": 22.3993, - "unittest/_torch/modules/test_fused_moe.py": 25.757, - "unittest/_torch/modules/test_group_rmn_norm.py": 30.4069, - "unittest/_torch/modules/test_mla_helix.py": 639.7242, - "unittest/_torch/modules/test_moe_host_sharer.py": 23.495727272727272, - "unittest/_torch/modules/test_moe_load_balancer.py": 21.41090909090909, - "unittest/_torch/modules/test_moe_routing.py": 714.5815, - "unittest/_torch/modules/test_rotary_embedding.py": 32.174727272727274, - "unittest/_torch/modules/test_triton_linear.py": 24.2455, - "unittest/_torch/modules/tests_lora_modules": 491.32936363636367, - "unittest/_torch/multi_gpu_modeling -k \"deepseek\"": 392.7955714285714, - "unittest/_torch/multi_gpu_modeling/test_deepseek.py::test_deepseek_streaming[tp1-bf16-trtllm-deepseekv3_lite]": 148.39190909090908, - "unittest/_torch/multi_gpu_modeling/test_deepseek.py::test_deepseek_streaming[tp4-bf16-trtllm-deepseekv3_lite]": 167.4738181818182, - "unittest/_torch/multimodal -k \"not nemotron_nano_v2_vl_fp8\"": 1567.007, - "unittest/_torch/multimodal/test_mm_encoder_standalone.py -k \"qwen3_30b_a3b_fp8\"": 391.6105, - "unittest/_torch/pyexecutor/test_model_loader_mx.py": 20.26832, - "unittest/_torch/ray_orchestrator/multi_gpu -m \"gpu2\"": 264.753, - "unittest/_torch/ray_orchestrator/multi_gpu -m \"gpu4\"": 491.622, - "unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m \"part0\"": 537.258, - "unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m \"part1\"": 381.6065, - "unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m \"part2\"": 540.4515, - "unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m \"part3\"": 519.103, - "unittest/_torch/ray_orchestrator/multi_gpu/test_multi_instance.py::test_multi_instance[tp1_4instances]": 326.7765, - "unittest/_torch/ray_orchestrator/multi_gpu/test_multi_instance.py::test_multi_instance[tp2_2instances]": 348.00225, - "unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part0\"": 378.2395, - "unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part1\"": 376.9775, - "unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part2\"": 202.7715, - "unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part3\"": 201.9635, - "unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part4\"": 280.6305, - "unittest/_torch/ray_orchestrator/single_gpu/test_cache_transceiver_comm.py": 45.3815, - "unittest/_torch/ray_orchestrator/single_gpu/test_llm_sleep.py": 106.474, - "unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part0\"": 740.4615, - "unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part1\"": 700.797, - "unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part2\"": 493.506, - "unittest/_torch/sampler": 1295.51725, - "unittest/_torch/sampler/test_beam_search.py": 922.6765, - "unittest/_torch/sampler/test_beam_search_speculative_d2h.py": 238.8765, - "unittest/_torch/sampler/test_logits_logprobs.py": 399.384, - "unittest/_torch/sampler/test_torch_multi_arange.py": 34.658, - "unittest/_torch/sampler/test_torch_sampler.py": 271.662, - "unittest/_torch/sampler/test_trtllm_sampler.py": 183.6365, - "unittest/_torch/speculative/hw_agnostic": 2068.2635, - "unittest/_torch/speculative/test_eagle3.py": 787.63475, - "unittest/_torch/test_model_config.py": 24.709, - "unittest/_torch/thop/parallel": 2797.026769230769, - "unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py": 28.361, - "unittest/_torch/thop/parallel/test_w4a16_linear.py": 25.172, - "unittest/_torch/thop/parallel/test_w4a8_linear.py": 23.838, - "unittest/_torch/thop/parallel/test_weight_only_quant_gemm.py": 50.5265, - "unittest/_torch/thop/parallel/test_weight_only_quant_linear.py": 24.9865, - "unittest/_torch/thop/parallel_hw_agnostic": 581.471, - "unittest/_torch/thop/serial": 52.6481, - "unittest/_torch/visual_gen/multi_gpu": 3370.539, - "unittest/_torch/visual_gen/test_attention_cute_dsl.py": 20.219, - "unittest/_torch/visual_gen/test_attention_integration.py": 53.365, - "unittest/_torch/visual_gen/test_attention_perf.py": 85.3705, - "unittest/_torch/visual_gen/test_attention_trtllm_sage.py": 58.9075, - "unittest/_torch/visual_gen/test_cache_dit.py": 113.863, - "unittest/_torch/visual_gen/test_flux_attention.py": 21.9575, - "unittest/_torch/visual_gen/test_flux_pipeline.py": 327.88, - "unittest/_torch/visual_gen/test_flux_pipeline.py::TestFluxParallelism::test_ulysses_2gpu_correctness": 68.15725, - "unittest/_torch/visual_gen/test_flux_transformer.py": 32.4895, - "unittest/_torch/visual_gen/test_ltx2_attention.py": 24.7125, - "unittest/_torch/visual_gen/test_ltx2_pipeline.py": 635.627, - "unittest/_torch/visual_gen/test_ltx2_transformer.py": 31.634, - "unittest/_torch/visual_gen/test_model_loader.py": 36.296, - "unittest/_torch/visual_gen/test_quant_ops.py": 18.796, - "unittest/_torch/visual_gen/test_teacache.py": 19.3005, - "unittest/_torch/visual_gen/test_tensor_payload.py": 30.0645, - "unittest/_torch/visual_gen/test_trtllm_serve_e2e.py": 377.0165, - "unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py": 22.1595, - "unittest/_torch/visual_gen/test_visual_gen_args.py": 19.153, - "unittest/_torch/visual_gen/test_visual_gen_params.py": 24.3765, - "unittest/_torch/visual_gen/test_visual_gen_utils.py": 21.5255, - "unittest/_torch/visual_gen/test_wan21_i2v_pipeline.py": 159.8445, - "unittest/_torch/visual_gen/test_wan21_i2v_teacache.py": 262.1585, - "unittest/_torch/visual_gen/test_wan21_t2v_pipeline.py": 135.2515, - "unittest/_torch/visual_gen/test_wan21_t2v_teacache.py": 140.904, - "unittest/_torch/visual_gen/test_wan22_i2v_pipeline.py": 202.639, - "unittest/_torch/visual_gen/test_wan22_t2v_pipeline.py": 184.332, - "unittest/_torch/visual_gen/test_wan22_ti2v_5b_pipeline.py": 91.5925, - "unittest/_torch/visual_gen/test_wan_transformer.py": 75.328, - "unittest/_torch/visual_gen/test_warmup.py": 22.6655, - "unittest/_torch/weight_sharing": 24.712, - "unittest/api_stability": 26.1965, - "unittest/auto_deploy/multigpu/custom_ops": 530.84575, - "unittest/auto_deploy/multigpu/smoke": 99.106, - "unittest/auto_deploy/multigpu/transformations": 2409.7595, - "unittest/auto_deploy/multigpu/transformations/library/test_allreduce_residual_rmsnorm_fusion.py -k \"strategy_auto\"": 111.7555, - "unittest/auto_deploy/multigpu/transformations/library/test_tp_sharding.py::test_moe_tp_shard_nvfp4": 65.457, - "unittest/auto_deploy/singlegpu/compile": 68.76425, - "unittest/auto_deploy/singlegpu/custom_ops": 650.365, - "unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention.py::TestSDPADispatch": 25.2515, - "unittest/auto_deploy/singlegpu/custom_ops/mamba/test_flashinfer_mamba_cached_op.py": 36.6985, - "unittest/auto_deploy/singlegpu/custom_ops/moe/test_ad_moe_op.py": 23.937, - "unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py": 28.7105, - "unittest/auto_deploy/singlegpu/custom_ops/quantization/test_quant.py": 43.4155, - "unittest/auto_deploy/singlegpu/models": 321.2695, - "unittest/auto_deploy/singlegpu/shim": 73.0455, - "unittest/auto_deploy/singlegpu/smoke/test_ad_build_small_single.py -k \"Nemotron-3-Nano-30B-A3B-FP8 or Nemotron-Nano-3-30B-A3.5B-dev or Llama-4-Scout\"": 211.282, - "unittest/auto_deploy/singlegpu/smoke/test_ad_speculative_decoding.py": 101.5545, - "unittest/auto_deploy/singlegpu/transformations": 323.7135, - "unittest/auto_deploy/singlegpu/transformations/library/test_fuse_relu2_quant_nvfp4.py": 24.795, - "unittest/auto_deploy/singlegpu/transformations/library/test_moe_fusion.py": 42.2095, - "unittest/auto_deploy/singlegpu/transformations/library/test_nvfp4_swiglu.py": 26.8895, - "unittest/auto_deploy/singlegpu/transformations/library/test_quant_fusion.py": 26.096, - "unittest/auto_deploy/singlegpu/transformations/library/test_quant_moe.py": 23.3475, - "unittest/auto_deploy/singlegpu/transformations/library/test_quantization.py": 22.4775, - "unittest/auto_deploy/singlegpu/transformations/library/test_trtllm_gen_diag.py": 28.7975, - "unittest/auto_deploy/singlegpu/utils": 52.285, - "unittest/auto_deploy/standalone": 938.2885, - "unittest/bindings": 734.4411428571428, - "unittest/disaggregated/region/test_aux.py": 27.011, - "unittest/disaggregated/region/test_block.py": 28.6035, - "unittest/disaggregated/region/test_page.py": 25.8165, - "unittest/disaggregated/region/test_region.py": 24.6965, - "unittest/disaggregated/test_agent_multi_backends.py": 97.55125, - "unittest/disaggregated/test_cache_transceiver_single_process.py": 3198.1, - "unittest/disaggregated/test_cluster_storage.py": 46.2105, - "unittest/disaggregated/test_disagg_cluster_manager_worker.py": 55.711, - "unittest/disaggregated/test_disagg_openai_client.py": 27.569, - "unittest/disaggregated/test_disagg_utils.py": 27.9085, - "unittest/disaggregated/test_disaggregated_params.py": 28.7775, - "unittest/disaggregated/test_extractor.py": 28.63375, - "unittest/disaggregated/test_kv_transfer.py": 1064.8285, - "unittest/disaggregated/test_kv_transfer_mp.py": 137.8465, - "unittest/disaggregated/test_mamba_transfer.py": 36.975, - "unittest/disaggregated/test_messenger.py": 28.294, - "unittest/disaggregated/test_openai_disagg_server.py": 31.1285, - "unittest/disaggregated/test_openai_disagg_service.py": 29.595, - "unittest/disaggregated/test_peer.py": 25.619333333333334, - "unittest/disaggregated/test_perf_logger.py": 25.842, - "unittest/disaggregated/test_py_cache_transceiver_mp.py": 2353.614, - "unittest/disaggregated/test_rank_info.py": 26.3, - "unittest/disaggregated/test_remoteDictionary.py": 54.58733333333333, - "unittest/disaggregated/test_request_id.py": 26.3535, - "unittest/disaggregated/test_router.py": 25.221333333333334, - "unittest/dynamo": 28.97, - "unittest/executor/test_fatal_error_health_check.py": 31.917, - "unittest/executor/test_ipc.py": 33.135, - "unittest/executor/test_rpc.py": 489.642, - "unittest/executor/test_stats_serializer.py": 28.198, - "unittest/inputs/test_chat_template_dispatch.py": 24.518, - "unittest/inputs/test_content_format.py": 30.1275, - "unittest/inputs/test_multimodal.py": 30.17, - "unittest/inputs/test_url_validation.py": 24.577, - "unittest/kv_cache_manager_v2_tests/": 204.007, - "unittest/llmapi/apps/_test_openai_chat_multimodal.py::test_single_chat_session_image_embeds -m needs_l40s": 171.2425, - "unittest/llmapi/apps/test_chat_utils.py": 25.813, - "unittest/llmapi/apps/test_disagg_serving_perf_metrics.py": 94.339, - "unittest/llmapi/apps/test_harmony_channel_validation.py": 37.2375, - "unittest/llmapi/apps/test_tool_parsers.py": 30.88, - "unittest/llmapi/test_additional_model_outputs.py -m \"gpu1\"": 50.889, - "unittest/llmapi/test_additional_model_outputs.py -m \"gpu2\"": 55.8575, - "unittest/llmapi/test_async_llm.py -m \"gpu2\"": 82.0985, - "unittest/llmapi/test_async_llm.py -m \"gpu4\"": 89.21, - "unittest/llmapi/test_async_llm.py -m \"not (gpu2 or gpu4)\"": 199.771, - "unittest/llmapi/test_build_cache.py": 26.889, - "unittest/llmapi/test_gc_utils.py": 30.331, - "unittest/llmapi/test_kv_cache_dtype_override.py": 25.704, - "unittest/llmapi/test_llm_args.py": 166.04, - "unittest/llmapi/test_llm_encode.py": 52.008, - "unittest/llmapi/test_llm_multi_gpu_pytorch.py -m \"gpu2\"": 443.80425, - "unittest/llmapi/test_llm_multi_gpu_pytorch.py -m \"gpu4\"": 169.63875, - "unittest/llmapi/test_llm_pytorch.py -m \"part0\"": 1446.4315, - "unittest/llmapi/test_llm_pytorch.py -m \"part1\"": 371.22675, - "unittest/llmapi/test_llm_pytorch.py -m \"part2\"": 742.059, - "unittest/llmapi/test_llm_pytorch.py -m \"part3\"": 401.47725, - "unittest/llmapi/test_llm_pytorch.py::test_nemotron_nas_lora": 535.6045, - "unittest/llmapi/test_llm_quant.py": 482.09957142857144, - "unittest/llmapi/test_llm_telemetry.py": 626.276, - "unittest/llmapi/test_llm_telemetry.py::TestTelemetryArchitectureExtraction": 60.558, - "unittest/llmapi/test_llm_telemetry.py::TestTelemetryPyTorchBackend": 69.1455, - "unittest/llmapi/test_llm_telemetry.py::TestTelemetryTRTBackend": 143.4525, - "unittest/llmapi/test_llm_utils.py": 88.12, - "unittest/llmapi/test_memory_profiling.py::test_pyexecutor_and_kvcache_share_execution_stream": 37.214, - "unittest/llmapi/test_mpi_session.py::test_llmapi_launch_multiple_tasks": 110.7205, - "unittest/llmapi/test_reasoning_parser.py": 27.351, - "unittest/llmapi/test_request_priority.py": 30.2385, - "unittest/llmapi/test_serialization.py": 24.475, - "unittest/llmapi/test_utils.py": 29.9245, - "unittest/media/test_encoding.py": 24.611, - "unittest/metrics/test_collector.py": 22.6395, - "unittest/models/test_quant_config_utils.py": 29.968, - "unittest/others/test_convert_utils.py": 29.8895, - "unittest/others/test_kv_cache_transceiver.py::test_async_transfer_keeps_llm_request_alive": 11.173440000000001, - "unittest/others/test_kv_cache_transceiver.py::test_cancel_request_in_transmission[mha]": 24.84052, - "unittest/others/test_kv_cache_transceiver.py::test_cancel_request_in_transmission[mla]": 14.56738, - "unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[NIXL-mha-ctx_fp16_gen_fp16]": 20.54032, - "unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mha-ctx_fp16_gen_fp16]": 26.049, - "unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mla-ctx_fp16_gen_fp16]": 26.441, - "unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[UCX-mha-ctx_fp16_gen_fp16]": 10.8102, - "unittest/others/test_kv_cache_transceiver.py::test_kv_transfer_timeout_silent_when_unset": 11.10526, - "unittest/others/test_kv_cache_transceiver.py::test_kv_transfer_timeout_warns_once_per_request": 11.00876, - "unittest/others/test_lora_manager.py": 34.117, - "unittest/others/test_lora_module_count.py": 25.513, - "unittest/others/test_time_breakdown.py": 26.249, - "unittest/others/test_tracing.py": 134.574, - "unittest/scripts": 29.238, - "unittest/test_model_runner_cpp.py": 282.2908888888889, - "unittest/tools/test_layer_wise_benchmarks.py::test_deepseek_r1_ctx_dep[1]": 75.109, - "unittest/tools/test_layer_wise_benchmarks.py::test_nemotron_gen_dep[1]": 95.88075, - "unittest/trt/attention/test_gpt_attention.py -k \"partition0\"": 86.69411111111111, - "unittest/trt/attention/test_gpt_attention.py -k \"partition1\"": 85.7748888888889, - "unittest/trt/attention/test_gpt_attention.py -k \"partition2\"": 87.27922222222222, - "unittest/trt/attention/test_gpt_attention.py -k \"partition3\"": 83.69322222222222, - "unittest/trt/attention/test_gpt_attention.py -k \"xqa_generic\"": 801.7458181818182, - "unittest/trt/attention/test_gpt_attention_IFB.py": 113.09775, - "unittest/trt/attention/test_gpt_attention_no_cache.py": 61.8435, - "unittest/trt/functional": 212.2435, - "unittest/trt/functional/test_moe.py": 234.19, - "unittest/trt/model/eagle": 38.394, - "unittest/trt/model/test_gpt_e2e.py": 401.14779999999996, - "unittest/trt/model/test_llama.py": 141.994, - "unittest/trt/model/test_mistral.py": 336.127, - "unittest/trt/model_api/test_model_api_multi_gpu.py": 18.519599999999997, - "unittest/trt/quantization": 386.906, - "unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py": 74.273, - "unittest/trt/quantization/test_weight_only_quant_matmul.py": 72.461, - "unittest/usage/test_collectors.py": 28.44825, - "unittest/usage/test_config.py": 23.880333333333333, - "unittest/usage/test_e2e_capture.py": 28.44675, - "unittest/usage/test_opt_out.py": 26.328666666666667, - "unittest/usage/test_reporter.py": 26.39666666666667, - "unittest/usage/test_schema.py": 28.76175, - "unittest/usage/test_transport.py": 28.819333333333333, - "unittest/utils/test_logger.py": 30.1205, - "unittest/utils/test_util.py": 27.133, - "unittest/visual_gen/test_output.py": 30.678, - "verl/test_verl_cases.py::test_async_generate": 354.0725, - "verl/test_verl_cases.py::test_async_memory_management": 91.7145, - "verl/test_verl_cases.py::test_init_without_device_mesh": 108.3435, - "verl/test_verl_cases.py::test_inter_node_trtllm_rollout": 95.962, - "verl/test_verl_cases.py::test_make_async_request_get_method": 22.799, - "verl/test_verl_cases.py::test_make_async_request_http_error": 22.8155, - "verl/test_verl_cases.py::test_make_async_request_max_attempts_exceeded": 22.8345, - "verl/test_verl_cases.py::test_make_async_request_post_method": 22.7245, - "verl/test_verl_cases.py::test_multimodal_different_image_sizes": 147.6705, - "verl/test_verl_cases.py::test_multimodal_generate_with_image": 133.777, - "verl/test_verl_cases.py::test_multimodal_text_only_fallback": 119.091, - "verl/test_verl_cases.py::test_placement_group_multi_node_multi_replica": 22.8825, - "verl/test_verl_cases.py::test_placement_group_multi_node_ray_resource_pool": 22.8715, - "verl/test_verl_cases.py::test_placement_group_with_ray_resource_pool": 23.16, - "verl/test_verl_cases.py::test_placement_group_with_sub_ray_resource_pool": 22.78, - "verl/test_verl_cases.py::test_unimodal_batch_generate": 112.0925, - "verl/test_verl_cases.py::test_unimodal_generate": 99.753, - "verl/test_verl_cases.py::test_wake_sleep_cycle": 98.8645, - "visual_gen/test_visual_gen_benchmark.py::test_offline_benchmark": 67.288, - "visual_gen/test_visual_gen_benchmark.py::test_online_benchmark[openai-videos]": 76.1575 + "accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1]": 566.384125, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False]": 605.8393098591549, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v1]": 792.857375, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False]": 262.5967857142857, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True]": 158.5127857142857, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False]": 238.92535714285714, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True]": 161.73442857142857, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1dp2cp2]": 276.34350980392156, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[adp-mtp2]": 311.97010309278346, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[noadp-mtp0]": 474.63128571428575, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_spec_dec": 178.47664285714288, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync[python]": 272.53489130434787, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0]": 409.83045454545453, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2]": 744.224909090909, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 205.94233333333335, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 188.42866379310345, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python": 293.63683636363635, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend": 266.7343085106383, + "accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_auto_dtype": 592.6489204545454, + "accuracy/test_disaggregated_serving.py::TestGLM52NVFP4::test_nvfp4_nixl_python[cache_mgr_v1]": 990.9347659574468, + "accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v1]": 342.7804705882353, + "accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v2]": 415.1615625, + "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False]": 245.6970612244898, + "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True]": 247.0623829787234, + "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v1]": 255.7429157894737, + "accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v2]": 424.93321978021976, + "accuracy/test_disaggregated_serving.py::TestKimiK25::test_nvfp4": 1390.96909375, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-False]": 208.08547872340426, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True]": 209.256806122449, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-False]": 216.41998947368418, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-True]": 216.31725263157895, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-False]": 166.9120744680851, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-True]": 164.41655319148936, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-False]": 179.65035714285713, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-True]": 175.9757340425532, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-False]": 204.8307142857143, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-True]": 205.07693617021278, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-False]": 216.36536082474225, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-True]": 217.35315463917524, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-False]": 166.50253061224493, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-True]": 164.4506326530612, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-False]": 180.33906382978722, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-True]": 177.16786734693878, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_beam_search": 819.1626363636364, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True]": 171.46033653846152, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_gen_only_spec_dec": 217.6168762886598, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 76.94890816326532, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 156.0001489361702, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python": 520.4379043478261, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[GSM8K]": 175.96124, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[MMLU]": 102.43104, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp1]": 143.1969090909091, + "accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp1]": 90.42032352941176, + "accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_auto_dtype[mtp_nextn=0-block_reuse=False-use_py_transceiver=False]": 391.84766666666667, + "accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_auto_dtype[mtp_nextn=3-block_reuse=True-use_py_transceiver=False]": 478.8976666666667, + "accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_ctx_dp2_gen_tp4": 281.545, + "accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=True]": 432.71708, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-False]": 347.91988297872336, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-True]": 340.39185714285713, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True-True]": 329.44937234042555, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp2cp2]": 326.08301020408163, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_chunked_prefill": 348.02151020408166, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first": 1046.73604, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first_kv_cache_v1": 124.06744444444443, + "accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend": 188.3538899082569, + "accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4]": 332.06218233618233, + "accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[qwen3vl_2b_instruct]": 185.360093220339, + "accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[no_overlap]": 69.15029714285714, + "accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[overlap]": 67.1641367781155, + "accuracy/test_llm_api_autodeploy.py::TestGPTOSS::test_mxfp4_gsm8k[120b-ep2]": 177.65254545454545, + "accuracy/test_llm_api_autodeploy.py::TestGPTOSS::test_mxfp4_gsm8k[120b-tp2]": 293.10047959183674, + "accuracy/test_llm_api_autodeploy.py::TestGPTOSS::test_mxfp4_gsm8k[120b]": 148.66536363636365, + "accuracy/test_llm_api_autodeploy.py::TestGPTOSS::test_mxfp4_gsm8k[20b]": 130.28418181818182, + "accuracy/test_llm_api_autodeploy.py::TestGemma4MoE::test_bf16": 1040.9451304347826, + "accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma3n_e2b_it": 652.3853028571428, + "accuracy/test_llm_api_autodeploy.py::TestGemmaE2B::test_gemma4_e2b_it": 345.52021037463976, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_attention_dp[4]": 281.7136451612903, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[torch-True-1]": 909.5608181818183, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton-False-1]": 320.45450417827294, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-1]": 152.23809254143646, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-4]": 138.5165392670157, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-True-1]": 328.91990909090913, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[flashinfer]": 207.7224, + "accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B_Instruct_Eagle3::test_eagle3_one_model[trtllm]": 184.89575555555555, + "accuracy/test_llm_api_autodeploy.py::TestMiniMaxM2::test_finegrained_fp8": 607.0751460674157, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[Qwen_QwQ-32B-False]": 262.3381111111111, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[deepseek-ai_DeepSeek-R1-0528-True]": 809.233, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[google_gemma-3-1b-it-False]": 61.70166666666666, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.1-8B-Instruct-False]": 50.529333333333334, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[meta-llama_Llama-3.3-70B-Instruct-False]": 145.9032222222222, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Codestral-22B-v0.1-False]": 90.96744444444444, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[mistralai_Ministral-8B-Instruct-2410-False]": 64.63533333333334, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_DeepSeek-R1-0528-NVFP4-v2-True]": 1319.9854545454546, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-FP8-True]": 110.99866666666667, + "accuracy/test_llm_api_autodeploy.py::TestModelRegistryAccuracy::test_autodeploy_from_registry[nvidia_Llama-3.1-8B-Instruct-NVFP4-True]": 125.70889795918367, + "accuracy/test_llm_api_autodeploy.py::TestNemotronH::test_auto_dtype[trtllm-flashinfer_ssm-False]": 495.21763636363636, + "accuracy/test_llm_api_autodeploy.py::TestNemotronH::test_auto_dtype[trtllm-triton_ssm-False]": 349.2198181818182, + "accuracy/test_llm_api_autodeploy.py::TestNemotronH::test_auto_dtype[trtllm-triton_ssm-True]": 464.39036363636365, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-1-attn_dp_off-trtllm]": 1240.6362727272726, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[bf16-4-attn_dp_off-trtllm]": 352.4914444444444, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-1-attn_dp_off-trtllm]": 393.3906777316736, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-4-attn_dp_off-trtllm]": 259.3917204301075, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-4-attn_dp_on-trtllm]": 219.00550537634408, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-attn_dp_off-trtllm]": 315.08336363636363, + "accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-4-attn_dp_off-trtllm]": 305.414, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_accuracy[bf16-4-attn_dp_off-trtllm]": 603.2348181818182, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_accuracy[fp8-4-attn_dp_off-trtllm]": 466.7068367346939, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_accuracy[fp8-4-attn_dp_on-trtllm]": 405.1242727272727, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_accuracy[nvfp4-1-attn_dp_off-trtllm]": 725.3176630727763, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_accuracy[nvfp4-4-attn_dp_on-trtllm]": 510.6742727272727, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[bf16]": 240.1639674796748, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_functional_small[fp8]": 225.47681818181817, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[bf16_ws4_180gb-flashinfer]": 485.037, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[bf16_ws4_180gb-trtllm]": 375.2629090909091, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[fp8_ws4_80gb-trtllm]": 506.8681808510638, + "accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[nvfp4_ws4_80gb-trtllm]": 512.3643636363637, + "accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-4]": 1732.888907216495, + "accuracy/test_llm_api_autodeploy.py::TestNemotronUltraV3::test_accuracy[nvfp4-8]": 2127.4628181818184, + "accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[False]": 400.33763636363636, + "accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_auto_dtype[True]": 503.32218181818183, + "accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_fp8[True]": 515.5300969101123, + "accuracy/test_llm_api_autodeploy.py::TestNemotronV2::test_nvfp4[True]": 417.8357272727273, + "accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4]": 499.28675714285714, + "accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8]": 1178.8152727272727, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[latency]": 1994.9886428571428, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput]": 696.7412417218543, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency]": 815.1006, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp]": 445.61188888888887, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp_tp4]": 752.4664851485148, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen]": 924.8983000000001, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen_adp_lmtp]": 459.0964, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput]": 455.72643853820597, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_bs8_mtp]": 636.5326666666666, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_mtp]": 469.2359897260274, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_pp4_mtp]": 943.4426666666666, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp4]": 888.5035, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp8]": 409.552, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload]": 960.41075, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[baseline]": 899.9911666666666, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[baseline_mtp1]": 1043.9808039215686, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_attn_multi_gpus": 693.5356666666667, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[baseline]": 435.158900990099, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[baseline_mtp1]": 384.64672277227726, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_piecewise_cuda_graph[baseline]": 424.88255208333334, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3::test_skip_softmax_attention_multi_gpus[target_sparsity_0.9]": 1936.0571020408163, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 109.189, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 111.4233, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 131.53244444444445, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 131.10222222222222, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 103.06677777777779, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 107.91711111111111, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 111.58439999999999, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 117.0749, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 147.24383333333336, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 150.32771428571428, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 106.54339999999999, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 109.9482, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 129.39533333333333, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 176.43553470437018, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 250.69174673629243, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 105.5962, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 111.16025, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False]": 120.65477777777778, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True]": 130.57385714285715, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 100.0108, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 107.22133333333333, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 185.2611, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 185.54422222222223, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 104.255, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True]": 109.0158888888889, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False]": 243.6549696969697, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True-v2_kv_cache=False]": 423.52147860962566, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=True-v2_kv_cache=True]": 681.503816, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_2_model_mtp": 216.84545161290322, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 75.69972727272727, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 166.08079999999998, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 95.51, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 54.731818181818184, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 45.107, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 51.38036363636364, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 45.14818181818182, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 43.47654545454546, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 131.0839090909091, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 37.365199999999994, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 260.33008163265305, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 308.887675, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 54.127272727272725, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 47.00145454545454, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 44.096, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 48.436090909090915, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 50.10754545454546, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 50.03754545454546, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 61.07272727272727, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 63.22081818181818, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 249.28828947368422, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 64.609, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 173.59127272727272, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 45.734, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 44.64572727272727, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 54.545, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 45.67136363636364, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 69.7822, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 44.198, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 69.9719090909091, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 235.08795999999998, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 39.305, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 67.627, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_kv_cache_aware_routing[mtp_nextn=0]": 94.18960000000001, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_kv_cache_aware_routing[mtp_nextn=2]": 99.85809090909092, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[ep4-mtp_nextn=0]": 57.95333333333333, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[ep4-mtp_nextn=2]": 70.299, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[tp4-mtp_nextn=0]": 72.82077777777778, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_python_scheduler[tp4-mtp_nextn=2]": 74.54666666666667, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=False]": 196.24513333333334, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=True]": 240.260800807537, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=False]": 257.00025, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True]": 2899.535428571429, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-enable_chunked_prefill=False]": 242.9692307692308, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=True-cuda_graph=True-overlap_scheduler=True-enable_chunked_prefill=True]": 1214.0928999999999, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=False]": 279.8845, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True]": 3115.894142857143, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-enable_chunked_prefill=False]": 339.368, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-enable_chunked_prefill=True]": 819.28775, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=fp8-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True]": 435.4769372496662, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=False-fp8kv=False-overlap_scheduler=True]": 214.80394305555555, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=none-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True]": 227.8988691983122, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=nvfp4-kv_cache_reuse=True-fp8kv=False-overlap_scheduler=True]": 294.3789699453552, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=nvfp4-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True]": 279.15114520547945, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_dummy_load_format": 169.91534423897582, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 127.89110000000001, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 123.1895, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 106.7188, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 122.71088888888889, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 109.73866666666667, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 122.89022222222222, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 104.65889999999999, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 201.32696816208394, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 107.83333333333333, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 103.43469999999999, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 99.9161111111111, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 126.5578888888889, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 102.3408, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 105.42333333333333, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=eagle-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 205.4267492063492, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=vanilla-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 107.1272, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 127.02718181818183, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 55.298272727272725, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 48.119181818181815, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 129.86932352941176, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 72.26036363636364, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 46.696, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 73.61151960784314, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[ep4-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 65.76149019607843, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 98.89809090909091, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 49.159454545454544, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 66.94454545454546, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 51.37372727272727, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 49.74481818181818, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 49.787454545454544, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 74.08118181818182, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 66.64172727272728, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 76.389, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 59.23609090909091, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=True]": 58.95272727272727, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 55.89263636363636, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-sampler_async_worker=False]": 80.8312, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 53.64163636363636, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=True]": 53.18277777777778, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 134.4777777777778, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 67.16441176470589, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 61.92367647058824, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 113.32422222222222, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=True]": 115.1615925925926, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False]": 51.46609090909091, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-sampler_async_worker=True]": 66.509, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 47.761, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False]": 49.95254545454546, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False]": 134.4498148148148, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=0]": 68.2290737704918, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding[mtp_nextn=2]": 154.16124725274724, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=0]": 51.68824509803922, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=2]": 60.39678217821782, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 79.26566666666668, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 223.61991005291003, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=0]": 110.20872727272727, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=2]": 64.23452941176471, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=fp8-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True]": 98.88074127906977, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_no_kv_cache_reuse[quant_dtype=none-mtp_nextn=2-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True]": 238.25687696335078, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTEDSL-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 193.4495, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTEDSL-mtp_nextn=2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 325.0439166666667, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 110.1784, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 130.3401, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False]": 108.9846, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 117.9392, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 119.118375, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True]": 138.20675, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 117.565, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True]": 158.50493922651935, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 121.72740331491713, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 172.87279999999998, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False]": 171.1799, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 314.79475447570337, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 117.463, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 108.3047850340136, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False]": 208.770375, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=TRTLLM-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=False]": 201.36658333333335, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 189.09936363636365, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 417.9765, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-ep4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 196.3797142857143, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 236.29854545454543, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 119.75618181818183, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 246.47781818181818, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 155.82318181818184, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 382.13807936507936, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 164.28670588235295, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 140.84088888888888, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 55.83880952380952, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 109.41854545454545, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 229.1358, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 99.4054, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 131.42146153846153, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 123.13945454545454, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 114.74166666666667, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 82.9781914893617, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 147.91820975609755, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True]": 161.596, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 146.64896595744682, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 140.86234782608693, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 150.08861904761906, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False]": 144.58642424242424, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 132.33410526315788, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 133.45738095238096, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True]": 73.9634, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-low_precision_combine=True-torch_compile=False]": 127.70872727272727, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 177.698671875, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False]": 73.15577570093458, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 211.7968248847926, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=TRTLLM-mtp_nextn=2-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False]": 203.5012, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=True-moe_backend=TRTLLM]": 132.16976041666666, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_batch_waiting[batch_wait_timeout_iters=10-batch_wait_max_tokens_ratio=1.0-mtp_nextn=0-fp8kv=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-v2_kv_cache=True]": 208.58932602739728, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Flash::test_nvfp4_4gpus_static_eplb[moe_backend=MEGAMOE_DEEPGEMM]": 737.5082040816327, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Flash::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM]": 366.2747959183673, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Pro::test_gsm8k_full_accuracy": 1131.2820365853659, + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV4ProDSpark::test_gsm8k_dep8_megamoe_deepgemm": 896.7543000000001, + "accuracy/test_llm_api_pytorch.py::TestGLM52::test_nvfp4[tp_size=8-ep_size=8]": 694.6597543859649, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dflash": 587.7711835616439, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_dummy_load_format": 87.91589784946235, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[cutlass-one_model-overlap_scheduler]": 1955.2867352941175, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler]": 1010.3446470588235, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-one_model-no_overlap_scheduler]": 688.9055999999999, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-cutlass-one_model-overlap_scheduler]": 439.4791601941747, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-one_model-no_overlap_scheduler]": 426.94371428571424, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v1_kv_cache-trtllm-one_model-overlap_scheduler]": 444.78338647342997, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-one_model-no_overlap_scheduler]": 550.945875, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-cutlass-one_model-overlap_scheduler]": 630.1294747474748, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-no_overlap_scheduler]": 706.0050833333333, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-overlap_scheduler]": 591.5004950980392, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_vswa_reuse_4gpus[v1_kv_cache-one_model]": 280.22, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_vswa_reuse_4gpus[v1_kv_cache-two_model]": 398.28516666666667, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_vswa_reuse_4gpus[v2_kv_cache-one_model]": 1526.7824285714287, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-cutlass-auto]": 273.2275148648649, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-auto]": 241.6941, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-fp8]": 207.31851506849316, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-auto]": 175.9200138121547, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-trtllm-auto]": 248.3882, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-trtllm-fp8]": 275.5822980501393, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-triton-auto]": 1710.5607717391304, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-auto]": 587.4411111111111, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-fp8]": 455.14378974358976, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-triton-auto]": 549.2571739130434, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-ep4-trtllm-auto]": 453.9397747747748, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-triton-auto]": 463.44741304347826, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-trtllm-auto]": 552.508797752809, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-triton-auto]": 900.5855666666666, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-auto]": 586.8257777777777, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-fp8]": 654.6051242937854, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-triton-auto]": 646.6367173913043, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-auto]": 508.7110540540541, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-trtllm-fp8]": 469.1895714285714, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-trtllm-auto]": 634.4464065934066, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache_no_reuse-tp4-cutlass-auto]": 370.1528152173913, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus_online_eplb[fp8]": 282.8789893617021, + "accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto]": 327.9946086956522, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype": 201.60288607594936, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_reuse": 115.71518618618619, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_chunked_prefill_without_reuse": 128.66493072289157, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse": 49.06097674418604, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_disable_overlap_scheduler": 51.77953230769231, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_no_partial_reuse": 57.0330583090379, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_low_memory_available_partial_reuse": 60.72586086956522, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_reuse_partial_reuse": 49.97132530120482, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse": 43.764580281690144, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_disable_overlap_scheduler": 47.694558641975306, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype_vswa_without_reuse_low_memory_available": 57.55918452380953, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=False]": 183.97511111111112, + "accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True]": 188.8469, + "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_auto_dtype": 933.2313, + "accuracy/test_llm_api_pytorch.py::TestGemma3_27BInstruct::test_fp8_prequantized": 419.3982222222223, + "accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[dep8]": 467.342, + "accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[ep8]": 657.7212, + "accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8]": 1009.5471894736842, + "accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8_attn_dp]": 682.5311111111112, + "accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_bf16": 565.337822622108, + "accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_fp8": 313.896358649789, + "accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_nvfp4": 268.91661690885076, + "accuracy/test_llm_api_pytorch.py::TestLagunaXS_2_1::test_bf16_dflash": 331.3741212121212, + "accuracy/test_llm_api_pytorch.py::TestLagunaXS_2_1::test_fp8_dflash": 313.25575872093026, + "accuracy/test_llm_api_pytorch.py::TestLagunaXS_2_1::test_nvfp4_dflash": 356.3046358543417, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4": 148.77345046235138, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4_streaming[stream_interval_4]": 37.77284679665738, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4_streaming[stream_interval_64]": 136.69611111111112, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=False]": 87.9016, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=True]": 140.07416666666666, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=False]": 82.0882, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=True]": 114.42385714285714, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=FLASHINFER-torch_compile=False]": 134.89218181818183, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=TRTLLM-torch_compile=False]": 124.3952, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=False]": 46.56345454545455, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=TRTLLM-torch_compile=False]": 41.564454545454545, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=TRTLLM-torch_compile=True]": 46.043454545454544, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=False]": 86.29798181818181, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=True]": 68.56609090909092, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=TRTLLM-torch_compile=True]": 121.82825, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=False-attn_backend=FLASHINFER]": 157.36079999999998, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=False-attn_backend=TRTLLM]": 48.425628985507245, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=True-attn_backend=FLASHINFER]": 170.8326, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_chunked_prefill[use_temperature=True-attn_backend=TRTLLM]": 75.20133526011561, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dflash": 147.7249836065574, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_dummy_load_format": 17.48164864864865, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=False-eagle3_one_model=True-overlap_scheduler=True]": 998.3113925501433, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_eagle3[sampler_async_worker=True-eagle3_one_model=True-overlap_scheduler=True]": 1011.6907043478261, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 74.59066666666668, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 110.4135, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 54.13452586206896, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=True]": 81.929431085044, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 73.4508888888889, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 110.1177, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 49.26115269461078, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 70.60236734693878, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 90.87263636363637, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 36.40436363636364, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 37.01018181818181, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False]": 105.15461111111111, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=True]": 74.54427272727273, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=False]": 126.09972906403941, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=True]": 90.54714427860696, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False]": 39.76645454545454, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]": 145.214, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=False]": 58.84855392156863, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=True]": 80.60036, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance]": 47.924777777777784, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar]": 48.18807374631269, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[llguidance]": 47.427454545454545, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True]": 58.863800000000005, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True]": 51.88051724137931, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[llguidance]": 47.401900000000005, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_ngram[xgrammar]": 47.24155555555556, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[v2_kv_cache=False-attn_backend=TRTLLM-torch_compile=False]": 57.35074229691877, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[v2_kv_cache=False-attn_backend=TRTLLM-torch_compile=True]": 97.46340384615384, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_nvfp4_kv[v2_kv_cache=True-attn_backend=TRTLLM-torch_compile=True]": 179.64886426592798, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard[overlap_scheduler=False]": 785.3822138728324, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_pard[overlap_scheduler=True]": 745.1532246153846, + "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B_Instruct_RocketKV::test_auto_dtype": 924.9611, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=False]": 592.7475376344086, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=True]": 656.0788791208792, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=False]": 687.6400107526881, + "accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True]": 763.7198031914893, + "accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_size=8]": 10800.0, + "accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False]": 10800.0, + "accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph[use_msa=False]": 10800.0, + "accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False]": 10800.0, + "accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True]": 1042.919387755102, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_fp8": 281.42, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_nvfp4_marlin_multi_gpus[tp_size=8]": 354.6746428571428, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-cpp_mamba_cache]": 373.539, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-python_mamba_cache]": 320.75507692307696, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-cpp_mamba_cache]": 214.66808333333336, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-python_mamba_cache]": 220.42576923076922, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpu_mtp_ar": 305.8700747663552, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpu_mtp_ar_custom_op": 212.21884615384616, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[DEP4_MTP_OFF]": 408.7718888888889, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[DEP4_MTP_ON]": 368.3760617977528, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4]": 630.2835833333334, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_hopper_w4a16": 509.27528571428576, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=TRTLLM]": 381.3987, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL]": 341.6574444444444, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp": 666.2585544554455, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp_custom_op": 386.734, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_marlin_adp_4gpus[mtp_nextn=3]": 838.8766666666667, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_marlin_multi_gpus[tp_size=8]": 464.279, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[ADP4_MTP]": 1030.6662569832401, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL]": 867.7993636363636, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=CUTLASS]": 470.4553846153846, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_off-trtllm]": 552.3911145833333, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_on-cutedsl]": 657.688081632653, + "accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_marlin_8gpus": 2768.8230714285714, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_2gpu_mtp_ar": 198.87144444444445, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[dep4]": 341.21818181818185, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[dep4-cutlass]": 205.73988888888888, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[dep4-trtllm]": 178.89322222222222, + "accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp1_block_reuse-cutlass]": 472.1311, + "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass]": 230.0092222222222, + "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_attention_dp]": 297.94870000000003, + "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_4gpus[latency_moe_cutlass]": 1410.4496153846155, + "accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_4gpus[latency_moe_trtllm_eagle3]": 326.27490476190474, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_dummy_load_format": 45.34000611620795, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False]": 151.40441666666666, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True]": 188.0913354037267, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=False]": 127.05255555555556, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8_block_scales[latency-torch_compile=True]": 157.40188888888886, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=False]": 138.786, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True]": 176.966, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_trtllm-torch_compile=False]": 137.82438095238095, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_trtllm-torch_compile=True]": 125.6016, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False]": 304.94709523809524, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True]": 222.94661904761907, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=False]": 197.08683333333335, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=True]": 177.8886295264624, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_cutlass-torch_compile=False]": 350.6162083333333, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_cutlass-torch_compile=True]": 290.07529411764705, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=False]": 188.4275, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=True]": 222.8463482142857, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRTLLM]": 99.26474238227146, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-CUTLASS]": 90.6996, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRTLLM]": 104.65530578512397, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-CUTLASS]": 85.41755555555557, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-TRTLLM]": 77.00845762711864, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention[target_sparsity_0.9-fp8kv=True]": 845.3932931506849, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=False]": 704.4539090909091, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=True]": 539.9865714285714, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=False]": 615.3447894736842, + "accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=True]": 547.9886875, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-CUTLASS]": 213.385125, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-TRTLLM]": 208.752, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp2-CUTLASS]": 225.09233333333333, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp2-TRTLLM]": 264.60998076923073, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_mtp": 355.00717987804876, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=False]": 418.67988888888885, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_cutedsl]": 558.5049900990099, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_trtllm]": 438.426, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_block_reuse]": 627.075405940594, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_trtllm]": 475.7063333333333, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL]": 948.6269726775956, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL]": 569.6396262626263, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_mtp3_gdn_replay_tep4": 529.1704337349398, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_bf16": 188.67048780487804, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash": 260.3592507374631, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8": 190.37416863905324, + "accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8_piecewise_cuda_graph": 267.21039432176656, + "accuracy/test_llm_api_pytorch.py::TestQwen3_6_35B_A3B::test_nvfp4[TRTLLM]": 225.70839265536725, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dflash": 166.4091335403727, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dummy_load_format": 39.61743076923077, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=False]": 100.84025382262996, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=False-enable_draft_len_schedule=True]": 118.26863636363636, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=False-enable_max_concurrency=True-enable_draft_len_schedule=False]": 113.90258333333333, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_eagle3[eagle3_one_model=True-enable_chunked_prefill=True-enable_max_concurrency=False-enable_draft_len_schedule=False]": 198.20316666666665, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales[latency]": 205.6853557623479, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales_early_first_token_response": 116.59400867052022, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[fp8-latency]": 37.92944324324324, + "accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[mxfp8-latency]": 53.90125, + "accuracy/test_llm_api_pytorch.py::TestSeedOss_36B::test_auto_dtype": 1173.8134285714286, + "accuracy/test_llm_api_pytorch.py::TestStep3_7::test_auto_dtype[tp_size=8-ep_size=8]": 5400.0, + "accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=0]": 5400.0, + "accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=3]": 5400.0, + "accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=0]": 5400.0, + "accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=3]": 5400.0, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_cuda_graph_matches_eager_logits[tinyllama-1.1b]": 14.05920506329114, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[gemma-3-1b]": 19.97921739130435, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[phi-4-mini]": 22.435924242424242, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen2-7b]": 24.917961636828643, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen3-0.6b]": 9.642792307692307, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[starcoder2-3b]": 33.13732070707071, + "accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[tinyllama-1.1b]": 6.582192405063291, + "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_cuda_graph_matches_eager_logits[bert-yelp]": 6.682479797979798, + "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_classification[bert-yelp-cuda_graph]": 4.394005050505051, + "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_classification[bert-yelp-eager]": 4.2008080808080805, + "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_matches_huggingface_per_token_reward[qwen2.5-prm-7b]": 22.808949494949495, + "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_qwen3_text_embedding_matches_huggingface[qwen3-embedding-0.6b]": 12.521669789227166, + "accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_qwen3_text_embedding_matches_huggingface[qwen3-embedding-8b]": 33.73845667447307, + "accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized": 325.59247987616095, + "accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4": 374.86357142857145, + "accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8]": 2679.2065, + "accuracy/test_llm_api_pytorch_multimodal.py::TestMistralSmall24B::test_auto_dtype[forced_chunked_prefill]": 181.28377777777777, + "accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[bf16]": 400.345, + "accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8]": 690.9473907692308, + "accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8_mmmu_encoder_cuda_graph]": 346.34280115273776, + "accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[nvfp4]": 416.1755833333333, + "accuracy/test_llm_api_pytorch_multimodal.py::TestNemotron_Nano_12B_V2_VL::test_auto_dtype[forced_chunked_prefill]": 286.4858888888889, + "accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL::test_auto_dtype[forced_chunked_prefill]": 177.8486, + "accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_fp8_block_scales[mtp_nextn=3]": 7200.0, + "accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=0]": 7200.0, + "accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=3]": 7200.0, + "cpp/test_multi_gpu.py::test_cache_transceiver[2proc-ucx_kvcache-90]": 178.4007340425532, + "cpp/test_multi_gpu.py::test_cache_transceiver[8proc-nixl_kvcache-90]": 242.4074680851064, + "cpp/test_multi_gpu.py::test_cache_transceiver[8proc-ucx_kvcache-90]": 275.1460652173913, + "cpp/test_multi_gpu.py::test_fused_gemm_allreduce[4proc-90]": 21.4115, + "cpp/test_multi_gpu.py::test_mpi_utils[90]": 220.0863829787234, + "cpp/test_multi_gpu.py::test_user_buffer[2proc-90]": 4.284468085106383, + "cpp/test_unit_tests.py::test_unit_tests[batch_manager-80]": 552.2707548209366, + "cpp/test_unit_tests.py::test_unit_tests[common-80]": 223.79825, + "cpp/test_unit_tests.py::test_unit_tests[common-90]": 222.75193593314765, + "cpp/test_unit_tests.py::test_unit_tests[executor-80]": 340.2329166666667, + "cpp/test_unit_tests.py::test_unit_tests[kernels-80]": 789.6245833333334, + "cpp/test_unit_tests.py::test_unit_tests[kernels-90]": 768.3216211699165, + "cpp/test_unit_tests.py::test_unit_tests[layers-80]": 913.9700833333334, + "cpp/test_unit_tests.py::test_unit_tests[layers-90]": 805.0914456824512, + "cpp/test_unit_tests.py::test_unit_tests[runtime-80]": 689.6249166666666, + "cpp/test_unit_tests.py::test_unit_tests[thop-80]": 3.291, + "cpp/test_unit_tests.py::test_unit_tests[thop-90]": 3.207640668523677, + "disaggregated/test_ad_disagg.py::test_async_eagle3_full_model_handoff": 97.89396703296703, + "disaggregated/test_ad_disagg.py::test_async_generation_matches_aggregate": 88.92743956043955, + "disaggregated/test_ad_disagg.py::test_async_generation_no_overlap_matches_aggregate": 87.01123076923076, + "disaggregated/test_ad_disagg.py::test_async_sharded_generation_handoff": 99.30441758241757, + "disaggregated/test_ad_disagg.py::test_chunked_prefill_handoff[deepseek_v3_mla]": 161.1555747126437, + "disaggregated/test_ad_disagg.py::test_chunked_prefill_handoff[tinyllama]": 98.48647550432277, + "disaggregated/test_ad_disagg.py::test_disaggregated_logits[deepseek_v3_mla]": 58.13549287749288, + "disaggregated/test_ad_disagg.py::test_disaggregated_logits[tinyllama]": 36.24735243553008, + "disaggregated/test_ad_disagg.py::test_reduced_layer_handoff_matches_aggregate[deepseek_v3_mla]": 142.15129971181557, + "disaggregated/test_ad_disagg.py::test_reduced_layer_handoff_matches_aggregate[tinyllama]": 97.21402017291066, + "disaggregated/test_ad_disagg.py::test_tinyllama_batch_handoff_semantic_slots": 77.39993123209169, + "disaggregated/test_ad_disagg_trtllm_serve.py::test_openai_completion": 62.068666666666665, + "disaggregated/test_auto_scaling.py::test_disagg_server_restart[etcd-round_robin]": 107.51943, + "disaggregated/test_auto_scaling.py::test_disagg_server_restart[http-round_robin]": 136.50441836734694, + "disaggregated/test_auto_scaling.py::test_minimal_instances[etcd-round_robin]": 105.13314285714286, + "disaggregated/test_auto_scaling.py::test_minimal_instances[http-round_robin]": 103.02789690721649, + "disaggregated/test_auto_scaling.py::test_service_discovery[etcd-round_robin]": 54.86334020618557, + "disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cache_aware]": 52.12690909090909, + "disaggregated/test_auto_scaling.py::test_service_discovery[http-round_robin]": 52.24916831683168, + "disaggregated/test_auto_scaling.py::test_worker_restart[etcd-load_balancing]": 210.1801530612245, + "disaggregated/test_auto_scaling.py::test_worker_restart[etcd-round_robin]": 207.23043564356436, + "disaggregated/test_auto_scaling.py::test_worker_restart[http-load_balancing]": 203.67193, + "disaggregated/test_disaggregated.py::test_disaggregated_benchmark_gen_only_insufficient_kv[TinyLlama-1.1B-Chat-v1.0]": 74.8770512195122, + "disaggregated/test_disaggregated.py::test_disaggregated_cache_aware_balance[TinyLlama-1.1B-Chat-v1.0]": 126.79705569007264, + "disaggregated/test_disaggregated.py::test_disaggregated_chat_completion_tool_calls[TinyLlama-1.1B-Chat-v1.0]": 70.9736107055961, + "disaggregated/test_disaggregated.py::test_disaggregated_conditional[TinyLlama-1.1B-Chat-v1.0]": 91.700578313253, + "disaggregated/test_disaggregated.py::test_disaggregated_cuda_graph[TinyLlama-1.1B-Chat-v1.0]": 81.07452153110047, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_conditional_v2[DeepSeek-V3-Lite-bf16]": 144.77602127659574, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_empty_batch[DeepSeek-V3-Lite-bf16]": 195.28455670103094, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp[DeepSeek-V3-Lite-fp8]": 136.93694845360827, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8]": 124.63738144329898, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_transceiver_runtime_python[DeepSeek-V3-Lite-fp8]": 129.67182474226803, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8]": 105.44671134020618, + "disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 143.4193673469388, + "disaggregated/test_disaggregated.py::test_disaggregated_diff_max_tokens[TinyLlama-1.1B-Chat-v1.0]": 73.85774285714287, + "disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b]": 188.37085000000002, + "disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0]": 91.19978208232446, + "disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0]": 83.23244171779142, + "disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0]": 131.9174964028777, + "disaggregated/test_disaggregated.py::test_disaggregated_mixed_stress_test[req120-conc64-qwen3_32b_fp8_mixed_stress]": 220.23792307692307, + "disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0]": 73.60878787878788, + "disaggregated/test_disaggregated.py::test_disaggregated_ngram[TinyLlama-1.1B-Chat-v1.0]": 87.51331400966183, + "disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0]": 102.21385019455253, + "disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python[TinyLlama-1.1B-Chat-v1.0]": 83.16836734693877, + "disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python_bounce[TinyLlama-1.1B-Chat-v1.0]": 105.64901086956522, + "disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python_fabric_memory[TinyLlama-1.1B-Chat-v1.0]": 104.76987434554974, + "disaggregated/test_disaggregated.py::test_disaggregated_perf_metrics[TinyLlama-1.1B-Chat-v1.0]": 91.85413349514563, + "disaggregated/test_disaggregated.py::test_disaggregated_python_transceiver_host_offload[TinyLlama-1.1B-Chat-v1.0]": 72.75782105263158, + "disaggregated/test_disaggregated.py::test_disaggregated_qwen3_32b_fp8[Qwen3/Qwen3-32B-FP8]": 151.65707692307694, + "disaggregated/test_disaggregated.py::test_disaggregated_single_gpu[TinyLlama-1.1B-Chat-v1.0]": 86.56471394230769, + "disaggregated/test_disaggregated.py::test_disaggregated_tinyllama_multi_orchestrator[TinyLlama-1.1B-Chat-v1.0]": 71.0492895522388, + "disaggregated/test_disaggregated_single_gpu.py::test_arbitrary_kv_cache_transfer[False-TinyLlama-1.1B-Chat-v1.0]": 54.64942857142857, + "disaggregated/test_disaggregated_single_gpu.py::test_arbitrary_kv_cache_transfer_missing_blocks[False-TinyLlama-1.1B-Chat-v1.0]": 53.03601818181818, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_cancel_gen_requests[TinyLlama-1.1B-Chat-v1.0]": 33.09288200589971, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8]": 157.21709316770185, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[False-TinyLlama-1.1B-Chat-v1.0]": 46.077697058823524, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[True-TinyLlama-1.1B-Chat-v1.0]": 38.63458439716312, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[False-TinyLlama-1.1B-Chat-v1.0]": 43.92117916666667, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[True-TinyLlama-1.1B-Chat-v1.0]": 40.47734232954546, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8]": 101.05138360175695, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-True-DeepSeek-V3-Lite-fp8/fp8]": 89.19943502824859, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-False-DeepSeek-V3-Lite-fp8/fp8]": 104.34513333333334, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-True-DeepSeek-V3-Lite-fp8/fp8]": 104.84779656160458, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-False-TinyLlama-1.1B-Chat-v1.0]": 40.79223037323037, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-True-TinyLlama-1.1B-Chat-v1.0]": 44.52552798053528, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0]": 50.73347846889952, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0]": 50.999568345323745, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-False-Qwen3-8B-FP8]": 66.79170639534884, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[False-True-Qwen3-8B-FP8]": 61.497651162790696, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-False-Qwen3-8B-FP8]": 81.84019393939394, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_qwen3[True-True-Qwen3-8B-FP8]": 63.21660349854228, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[False-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct]": 57.31423822714682, + "disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_spec_dec_batch_slot_limit[True-False-EAGLE3-LLaMA3.1-Instruct-8B-Llama-3.1-8B-Instruct]": 56.641159292035404, + "disaggregated/test_workers.py::test_workers_conditional_disaggregation_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16]": 140.14826804123712, + "disaggregated/test_workers.py::test_workers_kv_cache_aware_router[TinyLlama-1.1B-Chat-v1.0]": 133.58280288461538, + "disaggregated/test_workers.py::test_workers_kv_cache_events[TinyLlama-1.1B-Chat-v1.0]": 79.16608937198068, + "examples/test_ad_guided_decoding.py::test_autodeploy_guided_decoding_main_json": 108.65893142857144, + "examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[flashinfer-torch-simple]": 77.57904189944134, + "examples/test_ad_speculative_decoding.py::test_eagle_model_with_weights": 2.5963277777777773, + "examples/test_ad_speculative_decoding.py::test_nemotron_mtp_model_with_weights": 835.9901866295264, + "examples/test_deepseek_v4_pro.py::test_short_token_boundary_smoke": 989.6935238095238, + "examples/test_llm_api_with_mpi.py::test_llm_api_single_gpu_with_mpirun[TinyLlama-1.1B-Chat-v1.0]": 49.392453516969994, + "examples/test_ray.py::test_llm_inference_async_ray": 129.92340111420614, + "examples/test_ray.py::test_llm_inference_distributed_ray[tep2]": 160.54990721649486, + "examples/test_ray.py::test_llm_inference_distributed_ray[tp2]": 68.30646391752578, + "examples/visual_gen/test_visual_gen.py::test_cosmos3_example": 297.64562113402064, + "examples/visual_gen/test_visual_gen.py::test_flux1_example": 248.61007180851064, + "examples/visual_gen/test_visual_gen.py::test_flux1_lpips_against_golden": 53.898, + "examples/visual_gen/test_visual_gen.py::test_flux2_example": 295.3289088541667, + "examples/visual_gen/test_visual_gen.py::test_flux2_lpips_against_golden": 103.39257142857143, + "examples/visual_gen/test_visual_gen.py::test_ltx2_cuda_graph_lpips_matches_eager": 338.59385714285713, + "examples/visual_gen/test_visual_gen.py::test_ltx2_example": 379.62811936339523, + "examples/visual_gen/test_visual_gen.py::test_ltx2_lpips_against_golden": 65.055, + "examples/visual_gen/test_visual_gen.py::test_qwen_image_example": 231.70423376623376, + "examples/visual_gen/test_visual_gen.py::test_qwenimage_cuda_graph_lpips_against_golden": 68.01414285714286, + "examples/visual_gen/test_visual_gen.py::test_qwenimage_lpips_against_golden": 65.81428571428572, + "examples/visual_gen/test_visual_gen.py::test_visual_gen_api_walkthrough": 271.3475490196078, + "examples/visual_gen/test_visual_gen.py::test_visual_gen_quickstart": 199.75915289982424, + "examples/visual_gen/test_visual_gen.py::test_wan21_t2v_lpips_against_golden": 21.5448, + "examples/visual_gen/test_visual_gen.py::test_wan22_t2v_lpips_against_golden": 90.2427, + "examples/visual_gen/test_visual_gen.py::test_wan_t2v_example": 726.5599819121447, + "examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[attn2d_2x2]": 839.0274545454546, + "examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2]": 261.68454545454546, + "examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[ulysses4]": 607.2706666666667, + "examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[cfg2_tp2]": 236.3876, + "examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[tp2]": 71.3225, + "examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[tp2_ulysses2]": 302.0615, + "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_batch_generation": 20.855231805929918, + "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_cold_start": 20.51119398907104, + "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_field_completeness": 12.331842541436465, + "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_full_block_reuse": 10.812967032967034, + "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_long_context": 38.267905913978495, + "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_partial_block_reuse": 19.251983516483516, + "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_rapid_fire": 14.49196694214876, + "kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_shared_prefix": 20.32607988980716, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_chunked_draft_tokens": 333.74045871559633, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_draft_tokens": 132.6506574074074, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_eviction": 37.61371028037383, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_batch_size_limited": 107.80071629213484, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_block_reuse": 121.58346723646723, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill_eviction_block_reuse": 20.056470752089137, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill_multi_request": 129.70799719887955, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill_with_eviction": 23.063224657534246, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction[cuda_graph]": 24.279370473537604, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction[no_cuda_graph]": 20.66685714285714, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction_overlap": 17.89529213483146, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_overlap_scheduler[non_overlap]": 123.73225210084034, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_overlap_scheduler[overlap]": 121.00675637393768, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_partial_block_reuse": 121.48944350282486, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_token_budget_limited": 124.12047701149424, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_v2_vs_v1_basic": 128.52438797814207, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2LoRA::test_lora_chunked_prefill": 194.13237709497207, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2LoRA::test_lora_eviction": 58.65875284090909, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2LoRA::test_lora_multi_adapter_v2": 131.8038, + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2LoRA::test_lora_v2": 112.23774785100287, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[guaranteed-chunked]": 159.27757306590257, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[max-util-chunked]": 159.43676802507838, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[no-overlap-chunked]": 170.64738461538462, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[offload-chunked]": 166.18679824561406, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[offload-no-chunked]": 164.12044207317075, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[python-scheduler]": 165.64470674486805, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix[swa-chunked]": 133.17204360465118, + "kv_cache/test_prefix_aware_scheduling.py::TestServePrefixAwareScheduling::test_multi_round_qa_shared_prefix_smoke": 133.89418328445748, + "llmapi/test_llm_api_connector.py::test_connector_async_onboard[False]": 18.619161835748795, + "llmapi/test_llm_api_connector.py::test_connector_async_onboard[True]": 18.2084039408867, + "llmapi/test_llm_api_connector.py::test_connector_async_save[False]": 15.075786924939468, + "llmapi/test_llm_api_connector.py::test_connector_async_save[True]": 12.830610294117648, + "llmapi/test_llm_api_connector.py::test_connector_disagg_prefill[False]": 25.233555282555283, + "llmapi/test_llm_api_connector.py::test_connector_disagg_prefill[True]": 24.342429611650484, + "llmapi/test_llm_api_connector.py::test_connector_e2e_persistent_cache": 22.890816625916873, + "llmapi/test_llm_api_connector.py::test_connector_multi_request": 10.410875305623472, + "llmapi/test_llm_api_connector.py::test_connector_priorities": 12.587395061728396, + "llmapi/test_llm_api_connector.py::test_connector_priorities_default": 12.447050724637682, + "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[attention_dp]": 4.182096153846154, + "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[beam_search]": 10.050549636803876, + "llmapi/test_llm_api_connector.py::test_connector_rejects_unsupported_config[host_offloading]": 11.549893975903615, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output[False]": 14.680273170731708, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output[True]": 14.689616504854367, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output_chunked_context[False]": 11.245905569007263, + "llmapi/test_llm_api_connector.py::test_connector_scheduler_output_chunked_context[True]": 11.27094362745098, + "llmapi/test_llm_api_connector.py::test_connector_simple[False]": 12.478055288461539, + "llmapi/test_llm_api_connector.py::test_connector_simple[True]": 12.50396805896806, + "llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-bart-large-cnn]": 26.003712328767122, + "llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-bart-large-cnn]": 25.386357954545456, + "llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-overlap-bart-large-cnn]": 22.717285714285715, + "llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-bart-large-cnn]": 7.3287808641975305, + "llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-overlap-bart-large-cnn]": 6.780027777777778, + "llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-bart-large-cnn]": 14.974052459016393, + "llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-overlap-bart-large-cnn]": 22.9215, + "llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-on-greedy-bart-large-cnn]": 22.552, + "llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-greedy-batch2-bart-large-cnn]": 6.945199084668192, + "llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v2-decoder-cuda-graph-on-greedy-batch2-bart-large-cnn]": 8.355032, + "llmapi/test_llm_api_pytorch_bart.py::test_mbart_pytorch_generate_encoder_decoder_end_to_end": 22.4349012345679, + "llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[cudagraph]": 5400.0, + "llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[eager]": 5400.0, + "llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[cudagraph]": 79.19040469208211, + "llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[eager]": 91.1888, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-t5-small]": 6.15115313225058, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-flan-t5-small]": 11.990981481481482, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-flan-t5-xl]": 38.03971428571428, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-overlap-t5-small]": 21.127428571428574, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-t5-base]": 22.332, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-t5-small]": 6.14457911908646, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-overlap-t5-small]": 12.96089082278481, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-t5-small]": 5.369179566563467, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-tp2-t5-small]": 17.288162393162395, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-off-greedy-byt5-small]": 13.017768595041321, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-flan-t5-small]": 21.729333333333333, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-overlap-t5-small]": 20.64085714285714, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-t5-small]": 5.132682098765432, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-on-beam2-flan-t5-small]": 21.11866666666667, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-on-beam2-t5-small]": 20.56142857142857, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v2-cuda-graph-on-greedy-flan-t5-small]": 21.302833333333332, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v2-cuda-graph-on-greedy-t5-small]": 20.521285714285714, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v1-cuda-graph-on-beam2-flan-t5-small]": 20.4565, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v1-cuda-graph-on-beam2-t5-small]": 19.902714285714286, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v2-cuda-graph-on-greedy-flan-t5-small]": 20.3345, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v2-cuda-graph-on-greedy-t5-small]": 19.869142857142858, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_context_generation_batch": 5.753551162790698, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-beam2-batch2-flan-t5-small]": 5.0157716535433075, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-beam2-batch2-t5-small]": 4.783761061946902, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-greedy-batch2-t5-small]": 5.329487640449438, + "llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v2-decoder-cuda-graph-on-greedy-batch2-t5-small]": 7.091325396825397, + "llmapi/test_llm_examples.py::test_llmapi_chat_example": 70.07263636363636, + "llmapi/test_llm_examples.py::test_llmapi_example_distributed_tp2": 56.021, + "llmapi/test_llm_examples.py::test_llmapi_example_guided_decoding": 65.55775636811832, + "llmapi/test_llm_examples.py::test_llmapi_example_inference": 71.85408210180624, + "llmapi/test_llm_examples.py::test_llmapi_example_inference_async": 53.34649383730485, + "llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming": 48.074224548440064, + "llmapi/test_llm_examples.py::test_llmapi_example_logits_processor": 47.359884646962236, + "llmapi/test_llm_examples.py::test_llmapi_example_multilora": 61.29332004930156, + "llmapi/test_llm_examples.py::test_llmapi_kv_cache_connector[Qwen2-0.5B]": 91.71036363636364, + "llmapi/test_llm_examples.py::test_llmapi_quickstart": 48.46666379310345, + "llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit": 319.7374347826087, + "llmapi/test_llm_examples.py::test_llmapi_server_example": 157.16363636363636, + "llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_mtp": 244.0717337962963, + "llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_ngram": 95.34129156908665, + "perf/host_perf/test_module_resource_manager.py::test_kv_cache_prepare_context": 1.0290494505494505, + "perf/host_perf/test_module_resource_manager.py::test_kv_cache_prepare_generation": 0.967483606557377, + "perf/host_perf/test_module_sampler.py::test_sampler_update_greedy[greedy_bs8]": 1.2933342618384402, + "perf/host_perf/test_module_sampler.py::test_sampler_update_stop_words[stopwords_bs32]": 1.4042594594594595, + "perf/host_perf/test_module_scheduler.py::test_scheduler_production[production_gen_only_bs8]": 0.925497191011236, + "perf/host_perf/test_module_scheduler.py::test_scheduler_production[production_mixed_32gen_4ctx]": 1.055345945945946, + "perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]": 255.99544495412843, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL]": 2016.2651538461537, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 379.92807692307696, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 1608.9949, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 951.954875, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 371.3276923076923, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2365.190263414634, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 434.1525384615384, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL]": 2853.0363846153846, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 376.83946153846153, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL]": 2775.1226923076924, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL]": 203.80992307692307, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL]": 576.1800634146341, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL]": 1397.571923076923, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL]": 2195.1416153846153, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 501.6934615384616, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 2228.081307317073, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL]": 300.3468461538461, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL]": 416.32678571428573, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL]": 4561.989421052632, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2140.55235, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL]": 1500.5563333333332, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL]": 2060.203, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL]": 1038.6396666666667, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL]": 2913.0081578947365, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 423.3564375, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL]": 1640.4845625, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL]": 1264.9850526315788, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL]": 1965.8842105263157, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 470.33705263157896, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_dep8_mtp1_8k1k]": 1331.24528125, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_tep8_mtp3]": 565.2403448275862, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_blackwell-r1_fp4_v2_dep8_mtp1_8k1k]": 664.1381538461538, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_blackwell-r1_fp4_v2_tp4_mtp3_8k1k]": 678.5075, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_8k1k]": 916.9552524752474, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_1k8k]": 957.3329230769231, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_8k1k]": 573.0860742574257, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_1k8k]": 605.0051538461539, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_8k1k]": 416.47730917874395, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_ad_blackwell-r1_fp8_ad_ws8_1k1k]": 755.3983636363637, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_blackwell-r1_fp8_tp8_6k1k]": 1130.9022307692308, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_dep8_mtp1_8k1k]": 817.33225, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_tep8_mtp3_8k1k]": 492.7644166666667, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_8k1k]": 984.8252307692308, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_tep4_mtp3_8k1k]": 405.94523076923076, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_deepseek_v32_fp4_2_nodes_grace_blackwell-dsv32_fp4_dep8_trtllm_lpc_mnnvl_8k1k]": 498.78161538461535, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_adp_2k1k]": 574.2704166666666, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_235b_a22b_fp8_hopper-qwen3_235b_a22b_fp8_tp4_ep4_cutlass_8k1k]": 957.4866363636363, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_32b_fp8_hopper-qwen3_32b_fp8_tp2_6k1k]": 776.424090909091, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_2_nodes_grace_blackwell-glm5_fp4_dep8_mtp1_8k1k]": 630.5806153846154, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_2_nodes_grace_blackwell-glm5_fp4_tep8_mtp3_8k1k]": 581.6817692307693, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_blackwell-glm5_fp4_dep8_mtp1_8k1k]": 590.7949166666666, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tp1_mtp0_8k1k]": 606.2082307692308, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_dep4_1k8k]": 1336.2756923076922, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tep2_1k8k]": 811.5083846153847, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_8k1k]": 540.0438461538462, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_1k8k]": 395.9511538461539, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-host_perf_deepseek_v3_lite-v3lite_fp8_bs8_128_256]": 573.4216694677872, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-host_perf_llama8b-llama8b_fp16_bs8_128_256]": 268.8767824858757, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-host_perf_llama8b_spec_decode-llama8b_spec_bs1_128_128]": 150.8482967032967, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_dep8_32k8k]": 1488.879923076923, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_32k8k]": 1444.9784166666668, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_8k1k]": 766.74, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_32k8k]": 657.536, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_8k1k]": 445.1063846153846, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_grace_blackwell-k25_thinking_fp4_dep4_8k1k]": 841.3804830917875, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_grace_blackwell-k25_thinking_fp4_tep4_8k1k]": 457.48476923076925, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-llama_v3_3_70b_instruct_fp4_blackwell-llama70b_fp4_tp4_1000_1000]": 463.101, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-llama_v3_3_70b_instruct_fp4_blackwell-llama70b_fp4_tp4_512_32]": 252.52316666666667, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_dep8_8k1k]": 684.3395384615385, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_dep8_mtp3_8k1k]": 580.3056153846154, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_tep4_mtp3_8k1k]": 386.42491666666666, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_tp4_8k1k]": 461.85966666666667, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws4_1k1k]": 564.4200416666666, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-super_mtp_ad_blackwell-super_mtp_ad_ws4_1k1k]": 532.4755454545455, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 4133.825, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2472.7274545454547, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL]": 3231.6324285714286, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL]": 3142.5304615384616, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL]": 851.2984308510638, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL]": 2151.5645, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 2125.6215384615384, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL]": 8068.51475, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2211.1548095238095, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL]": 1339.5592666666666, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL]": 2621.412888888889, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL]": 1377.9775555555555, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL]": 1276.3097857142857, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL]": 3251.7198439716312, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL]": 2548.912, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2466.507727272727, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 842.7718000000001, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 889.5297857142857, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 517.3953529411765, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 544.902, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL]": 883.5191538461539, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL]": 312.7136666666667, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL]": 264.12928571428574, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL]": 828.4586666666667, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 488.6693076923077, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL]": 3147.871157894737, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 836.2682272727272, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL]": 1035.4665454545454, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 607.4167, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL]": 586.0575, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL]": 1462.754, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 457.9464835680751, + "perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-flux2_blackwell-flux2_fp8_cfg1_ulysses4_teacache_on]": 360.2670909090909, + "perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_i2v_cfg2_ulysses4_compile_on]": 536.7069, + "perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_t2v_cfg2_ulysses4_compile_on]": 474.23836363636366, + "perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_nvfp4_i2v_cfg2_ulysses4_compile_on]": 458.0541818181818, + "perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan21_t2v_14b_blackwell-wan21_14b_nvfp4_trtllm_cfg2_ulysses4_teacache_on]": 513.6001818181818, + "ray_orchestrator/RL/test_rl_perf_reproduce.py::test_rl_perf_reproduce[tp1_4instances]": 106.18771875, + "ray_orchestrator/RL/test_rl_perf_reproduce.py::test_rl_perf_reproduce[tp2_2instances]": 109.6773125, + "stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-GUARANTEED_NO_EVICT-pytorch-stress-test]": 737.0806363636364, + "stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-MAX_UTILIZATION-pytorch-stress-test]": 664.5234545454546, + "test_e2e.py::test_get_ci_container_port": 1.3634927884615387, + "test_e2e.py::test_openai_chat_example[pytorch]": 575.1747253012048, + "test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct]": 74.3235283018868, + "test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b]": 183.251, + "test_e2e.py::test_openai_chat_harmony": 174.82321751412428, + "test_e2e.py::test_openai_chat_harmony_perf_metrics": 160.05857558139536, + "test_e2e.py::test_openai_chat_multimodal_example": 130.39714077669902, + "test_e2e.py::test_openai_chat_with_logit_bias[torch_sampler]": 86.90059294117647, + "test_e2e.py::test_openai_chat_with_logit_bias[trtllm_sampler]": 87.14738461538461, + "test_e2e.py::test_openai_completions_example[pytorch]": 261.67624220623503, + "test_e2e.py::test_openai_completions_with_logit_bias[torch_sampler]": 84.34349065420561, + "test_e2e.py::test_openai_completions_with_logit_bias[trtllm_sampler]": 84.66151643192488, + "test_e2e.py::test_openai_health": 72.27581818181818, + "test_e2e.py::test_openai_kv_cache_contamination": 1411.96477, + "test_e2e.py::test_openai_lora": 185.54133078880406, + "test_e2e.py::test_openai_misc_example[pytorch]": 350.13566343825664, + "test_e2e.py::test_openai_mmencoder_example": 136.0997242206235, + "test_e2e.py::test_openai_perf_metrics": 86.54126086956522, + "test_e2e.py::test_openai_post_processor": 372.86061686746984, + "test_e2e.py::test_openai_prometheus": 123.17233812949641, + "test_e2e.py::test_openai_reasoning[pytorch]": 257.26778132678135, + "test_e2e.py::test_openai_responses": 311.0094705882353, + "test_e2e.py::test_openai_responses_entrypoint": 107.0407848410758, + "test_e2e.py::test_openai_tool_call": 89.66446634615384, + "test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-120B-gpt_oss/gpt-oss-120b]": 1067.7551818181819, + "test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-20B-gpt_oss/gpt-oss-20b]": 214.3322994923858, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]": 219.58391370558374, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]": 127.58688772845953, + "test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B]": 153.47955013192612, + "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B]": 466.13843444730077, + "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_fp8_hf-Qwen3/saved_models_Qwen3-30B-A3B_fp8_hf]": 256.2311825192802, + "test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_nvfp4_hf-Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf]": 178.14921391752577, + "test_e2e.py::test_ptp_quickstart_advanced_bs1": 96.76857000000001, + "test_e2e.py::test_ptp_quickstart_advanced_deepseek_v3_lite_4gpus_adp_balance[DeepSeek-V3-Lite-FP8-DeepSeek-V3-Lite/fp8]": 94.38419387755103, + "test_e2e.py::test_ptp_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B]": 80.07500583090379, + "test_e2e.py::test_ptp_quickstart_advanced_mixed_precision": 90.66226975476839, + "test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]": 190.38033875338755, + "test_e2e.py::test_ptp_quickstart_advanced_mtp_eagle[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]": 280.69667039106145, + "test_e2e.py::test_ptp_quickstart_advanced_ngram[Llama-3.1-8B-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct]": 70.9769423631124, + "test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 23.737279936558288, + "test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity]": 7.569804761904762, + "test_e2e.py::test_trtllm_bench_help_sanity[meta-llama/Llama-3.1-8B]": 48.68448973607038, + "test_e2e.py::test_trtllm_bench_invalid_token_pytorch[TinyLlama-1.1B-Chat-v1.0-TinyLlama-1.1B-Chat-v1.0]": 79.17544525547444, + "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 89.94045479452055, + "test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B]": 89.49601497005989, + "test_e2e.py::test_trtllm_bench_llmapi_launch[pytorch_backend-llama-v3-llama3-8b]": 120.57213636363636, + "test_e2e.py::test_trtllm_bench_mgmn": 110.8202, + "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-False-False]": 114.68200189393939, + "test_e2e.py::test_trtllm_bench_pytorch_backend_sanity[meta-llama/Llama-3.1-8B-llama-3.1-8b-instruct-hf-fp8-True-True]": 69.32486930091186, + "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-]": 122.67552884615384, + "test_e2e.py::test_trtllm_bench_request_rate_and_concurrency[enable_concurrency-enable_request_rate]": 37.874408823529414, + "test_e2e.py::test_trtllm_serve_example": 178.86436363636363, + "test_e2e.py::test_trtllm_serve_lora_example": 94.39688040712468, + "test_e2e.py::test_trtllm_serve_multimodal_example": 178.93932289156626, + "test_e2e.py::test_trtllm_serve_top_logprobs[pytorch]": 86.90236144578313, + "test_fmha.py::test_fmha": 2348.44875, + "thirdparty/test_cmake_third_party.py::test_cmake_listfiles": 1.3952367149758453, + "thirdparty/test_git_modules.py::test_gitmodules": 1.2932296650717703, + "unittest/_torch/attention": 1499.0411369346734, + "unittest/_torch/attention --ignore=unittest/_torch/attention/sparse/test_cute_dsl_fp8_paged_mqa_logits.py --ignore=unittest/_torch/attention/sparse/test_cute_dsl_fp4_paged_mqa_logits.py --ignore=unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py": 1461.3758554216868, + "unittest/_torch/attention/sparse/deepseek_v4/test_compressor_kernel.py": 26.301077142857142, + "unittest/_torch/attention/sparse/deepseek_v4/test_compressor_module.py": 116.55956629834255, + "unittest/_torch/attention/sparse/deepseek_v4/test_compressor_tf32.py": 21.41217094017094, + "unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_cache_manager.py": 52.60678431372549, + "unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_indices_transform.py": 50.76427808988764, + "unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_o_proj.py": 37.86423978201635, + "unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_sparse_mla.py": 52.061539944903586, + "unittest/_torch/attention/sparse/dsa/test_dsa_fp4_indexer.py": 26.329861386138614, + "unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_cute_dsl_fp8_indexer_q_gemm_rope_fp4_matches_unfused": 45.63755555555556, + "unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_fused_cat_fp4_dsv32_prefill_shape": 19.948702970297028, + "unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_fused_cat_fp4_matches_deepgemm": 19.606653465346536, + "unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_fused_cat_fp4_noncontiguous_split": 2.04908071278826, + "unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_fused_cat_fp4_shape_dispatch": 19.43559405940594, + "unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_indexer_k_cache_gather_contiguous_fp4": 19.109356435643562, + "unittest/_torch/attention/sparse/test_cute_dsl_fp4_paged_mqa_logits.py": 701.1912252747252, + "unittest/_torch/attention/sparse/test_cute_dsl_fp8_paged_mqa_logits.py": 348.6813176470588, + "unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py": 1584.264129213483, + "unittest/_torch/attention/sparse/test_sparse_mla_forward.py": 111.27268644067796, + "unittest/_torch/attention/test_attention_mla.py": 135.2430928319624, + "unittest/_torch/auto_deploy": 26.348274924471298, + "unittest/_torch/compilation": 26.10724778761062, + "unittest/_torch/custom_ops/test_deepseek_v4_q_norm.py": 22.96618384401114, + "unittest/_torch/debugger": 20.013879710144927, + "unittest/_torch/distributed": 20.335233009708737, + "unittest/_torch/executor": 337.5190529327611, + "unittest/_torch/executor/test_async_transfer_manager.py": 27.127968215158926, + "unittest/_torch/executor/test_disagg_index_mapper_early_release.py": 27.426251781472686, + "unittest/_torch/executor/test_error_classification.py": 25.870426829268293, + "unittest/_torch/executor/test_hang_detector_kill.py": 42.569074018877295, + "unittest/_torch/executor/test_kv_cache_budget_split.py": 27.206930952380954, + "unittest/_torch/executor/test_kv_cache_compression_manager.py": 27.282539024390243, + "unittest/_torch/executor/test_kv_cache_estimation.py": 27.37214423076923, + "unittest/_torch/executor/test_kv_cache_v2_capacity_only.py": 25.55210606060606, + "unittest/_torch/executor/test_kv_pool_rebalance.py": 27.236201456310678, + "unittest/_torch/executor/test_model_loader_mx.py": 23.38812343201204, + "unittest/_torch/executor/test_overlap_scheduler.py": 796.4592737430168, + "unittest/_torch/executor/test_scheduler_serializable_output.py": 25.658269417475726, + "unittest/_torch/flashinfer/test_trtllm_flashinfer_symbol_collision.py": 1003.646782234957, + "unittest/_torch/lora": 21.104300771208226, + "unittest/_torch/memory": 20.41897507788162, + "unittest/_torch/misc": 217.8719063444109, + "unittest/_torch/misc/test_autotuner.py::test_autotuner_distributed_strategy": 35.712514851485146, + "unittest/_torch/modeling -k \"modeling_gemma3\"": 250.2099910714286, + "unittest/_torch/modeling -k \"modeling_gpt_oss\"": 99.71302683615819, + "unittest/_torch/modeling -k \"modeling_llama\"": 124.79015498652292, + "unittest/_torch/modeling -k \"modeling_mixtral\"": 70.61155039787799, + "unittest/_torch/modeling -k \"modeling_nemotron_nano_v2_vl\"": 426.0704334140436, + "unittest/_torch/modeling -k \"modeling_nemotron_nas\"": 38.65103496503497, + "unittest/_torch/modeling -k \"modeling_out_of_tree\"": 146.20377011494253, + "unittest/_torch/modeling -k \"modeling_phi3\"": 33.42191375291375, + "unittest/_torch/modeling -k \"modeling_qwen\"": 707.234, + "unittest/_torch/modeling -k \"modeling_qwen_moe\"": 292.4790489510489, + "unittest/_torch/modeling -k \"modeling_siglip\"": 130.33033101851854, + "unittest/_torch/modeling -k \"modeling_speculative\"": 27.073209790209788, + "unittest/_torch/modeling -k \"modeling_vila\"": 42.617395784543326, + "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_multimodal_26b_dummy": 188.08098222222222, + "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_text_26b_dummy": 160.7302222222222, + "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_text_31b_dummy": 161.78340322580647, + "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_text_e2b_dummy": 148.25198002853068, + "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_text_e2b_dummy_small_max_seq_len[256]": 173.58835267857143, + "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_text_e2b_dummy_small_max_seq_len[512]": 192.90066968325795, + "unittest/_torch/modeling/test_gemma4_e2e_dummy.py::test_e2e_text_e4b_dummy": 195.04240348330913, + "unittest/_torch/modeling/test_gemma4_multimodal.py": 24.872615803814714, + "unittest/_torch/modeling/test_modeling_afmoe.py": 65.08802259887005, + "unittest/_torch/modeling/test_modeling_cohere2.py": 60.729254196642685, + "unittest/_torch/modeling/test_modeling_cosmos3.py::TestCosmos3::test_all": 25.520618604651162, + "unittest/_torch/modeling/test_modeling_deepseekv4.py": 57.04133485193622, + "unittest/_torch/modeling/test_modeling_exaone_moe.py": 255.23339285714286, + "unittest/_torch/modeling/test_modeling_gemma4.py": 77.90781034482758, + "unittest/_torch/modeling/test_modeling_gemma4_unified.py": 20.19156862745098, + "unittest/_torch/modeling/test_modeling_llava_next.py::TestLlavaNext::test_all": 29.02741046511628, + "unittest/_torch/modeling/test_modeling_llava_next.py::test_llava_next_expand_prompt_token_ids_for_mm": 14.212421658986175, + "unittest/_torch/modeling/test_modeling_mistral.py": 37.79800717703349, + "unittest/_torch/modeling/test_modeling_nemotron_h.py::test_nemotron_h_sanity": 156.58529545454545, + "unittest/_torch/modeling/test_modeling_parakeet.py": 44.25659903381643, + "unittest/_torch/modeling/test_modeling_pixtral.py": 35.958952267303104, + "unittest/_torch/modeling/test_modeling_pixtral.py::test_tensor_parallelism": 43.275, + "unittest/_torch/modeling/test_modeling_qwen2_5vl.py::TestQwen2_5_VL::test_all": 76.14861520467836, + "unittest/_torch/modeling/test_modeling_qwen3_5_vl.py::TestQwen3_5VL::test_all": 73.7980761904762, + "unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py::TestQwen3_5MoeVL::test_all": 80.15000454545455, + "unittest/_torch/modeling/test_modeling_qwen3vl.py::TestQwen3VL::test_all": 96.30822105263158, + "unittest/_torch/modeling/test_modeling_qwen3vl.py::test_qwen3vl_init_preserves_caller_quant_config": 16.023488966318236, + "unittest/_torch/modeling/test_modeling_qwen3vl_moe.py::TestQwen3VLMoe::test_all": 82.7298538011696, + "unittest/_torch/modeling/test_modeling_radio.py": 29.436779904306217, + "unittest/_torch/modeling/test_modeling_step3p7.py": 28.5279375, + "unittest/_torch/modeling/test_modeling_step3p7vl.py": 38.826083720930235, + "unittest/_torch/modeling/test_multimodal_encoder_graph.py": 20.719585294117646, + "unittest/_torch/modeling/test_multimodal_encoder_mixin.py": 25.568744578313254, + "unittest/_torch/modeling/test_nemotron_nano_preprocessing.py": 62.31733732057416, + "unittest/_torch/models/checkpoints": 62.52584578313253, + "unittest/_torch/modules/dwdp/test_dwdp_fixup_moe_backends.py": 26.235854066985645, + "unittest/_torch/modules/dwdp/test_dwdp_manager.py": 25.706846715328467, + "unittest/_torch/modules/dwdp/test_dwdp_mapping.py": 25.72155339805825, + "unittest/_torch/modules/dwdp/test_dwdp_peer_ranges.py": 25.59992048192771, + "unittest/_torch/modules/fused_moe/test_deepgemm_fused_expand_quant.py": 22.875341658341657, + "unittest/_torch/modules/fused_moe/test_deepgemm_fused_gather_finalize.py": 22.920138532110094, + "unittest/_torch/modules/fused_ops/test_fused_qk_norm_rope_gate.py": 23.7237, + "unittest/_torch/modules/fused_ops/test_gelu_tanh_mul_fp4_quant.py": 24.07451566951567, + "unittest/_torch/modules/fused_ops/test_rmsnorm_fp4_quant.py": 23.92312912087912, + "unittest/_torch/modules/fused_ops/test_rmsnorm_residual_add.py": 24.953952802359883, + "unittest/_torch/modules/mamba": 740.4738101265823, + "unittest/_torch/modules/moe/test_cute_dsl_b12x_moe_backend.py": 21.06453807106599, + "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTEDSL and not MEGAMOE_CUTEDSL\"": 91.753, + "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTEDSL\"": 342.06576923076926, + "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"CUTLASS\"": 1208.6856129032258, + "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DEEPGEMM and not MEGAMOE_DEEPGEMM\"": 46.524699999999996, + "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DEEPGEMM\"": 62.843666666666664, + "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"DENSEGEMM\"": 644.2885555555555, + "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"MEGAMOE_CUTEDSL\"": 243.494, + "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"MEGAMOE_DEEPGEMM\"": 27.79185714285714, + "unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend -k \"TRTLLM\"": 123.4027619047619, + "unittest/_torch/modules/moe/test_moe_backend.py::test_trtllm_bf16_unquantized_moe": 899.1003994252874, + "unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm": 206.84880319148937, + "unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_boundary": 196.21933333333334, + "unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_postquant": 118.7101182795699, + "unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_nccl_ep_cuda_graph_replay_uses_updated_routing": 41.391349999999996, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTEDSL\"": 106.04984158415841, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and FP8 and not FP8_BLOCK_SCALES and not W4A8 and not MXFP8\"": 82.33142857142856, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and FP8 and not FP8_BLOCK_SCALES\"": 71.5973, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and FP8_BLOCK_SCALES\"": 60.02572727272727, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and MXFP8 and not W4A8\"": 70.1191836734694, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and NVFP4\"": 86.05340816326532, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A16_MXFP4\"": 81.69265979381443, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A8_AWQ\"": 68.79285, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A8_MXFP4_FP8\"": 81.08657575757576, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W4A8_MXFP4_MXFP8\"": 150.906099009901, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"CUTLASS and W8A16\"": 91.86447959183673, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"DEEPGEMM and not MEGAMOE_DEEPGEMM\"": 84.99442574257425, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"MARLIN\"": 81.10676923076923, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"MEGAMOE_CUTEDSL\"": 59.32976288659793, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"MEGAMOE_DEEPGEMM\"": 51.67359595959596, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"TRTLLM and FP8_BLOCK_SCALES\"": 63.96949504950495, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"TRTLLM and NVFP4 and not W4A8\"": 59.18147524752475, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"TRTLLM and W4A16_MXFP4\"": 87.15429591836735, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"TRTLLM and W4A8_MXFP4_MXFP8\"": 51.86418811881188, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k \"TRTLLM and W4A8_NVFP4_FP8\"": 157.12869306930693, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTEDSL-quant=NVFP4-routing=Renormalize]": 64.47758, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=FP8-routing=Renormalize]": 47.0703, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=DEEPGEMM-quant=FP8_BLOCK_SCALES-routing=Renormalize]": 54.87052, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_NVFP4_FP8-routing=Renormalize]": 80.27069, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=DEEPEP-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]": 48.16604, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=IGNORE-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=MEGAMOE_CUTEDSL-quant=NVFP4-routing=DeepSeekV3]": 55.19516326530612, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=IGNORE-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=MEGAMOE_DEEPGEMM-quant=W4A8_MXFP4_MXFP8-routing=DeepSeekV3]": 56.76075510204081, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=W8A16-routing=Renormalize]": 26.64341, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=FP8_BLOCK_SCALES-routing=Renormalize]": 26.58097, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e256_k6_h4096_i2048-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]": 27.330119999999997, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]": 48.381660000000004, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]": 46.22708, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]": 46.49764, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb": 48.43913725490196, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb -k \"MEGAMOE_CUTEDSL\"": 56.218061224489794, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb -k \"MEGAMOE_DEEPGEMM\"": 52.125959183673466, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb -k \"not None\"": 25.63853608247423, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]": 46.70682, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]": 47.37518, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb[parallel=DEP-comm=NVLINK_ONE_SIDED-e8_k2_h512_i512-slots=16-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]": 47.52538, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"CUTEDSL and not MEGAMOE_CUTEDSL\"": 91.13814722222222, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"CUTLASS and None\"": 27.01264705882353, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"CUTLASS and not None\"": 151.68869058295965, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"DEEPGEMM and not MEGAMOE_DEEPGEMM\"": 49.13182647058823, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"DENSEGEMM\"": 463.5970454545455, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"MEGAMOE_CUTEDSL\"": 175.77482905982907, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k \"MEGAMOE_DEEPGEMM\"": 42.02627173913043, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTEDSL-quant=NVFP4-routing=Renormalize]": 65.25673630136987, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=FP8-routing=Renormalize]": 32.85159831460675, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]": 30.820144329896905, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=W4A8_MXFP4_MXFP8-routing=Renormalize]": 29.589710743801653, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=CUTLASS-quant=W8A16-routing=Renormalize]": 30.251238754325257, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=DEEPGEMM-quant=FP8_BLOCK_SCALES-routing=Renormalize]": 35.18373224043716, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=FP8_BLOCK_SCALES-routing=Renormalize]": 35.542106529209626, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize]": 33.902055401662054, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A16_MXFP4-routing=Renormalize]": 29.093491408934707, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e60_k4_h2048_i1408-seq=1-dtype=torch.bfloat16-backend=TRTLLM-quant=W4A8_NVFP4_FP8-routing=Renormalize]": 57.66461690140845, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e8_k1_h512_i512-seq=8-dtype=torch.bfloat16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]": 21.324429906542058, + "unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu[e8_k1_h512_i512-seq=8-dtype=torch.float16-backend=CUTLASS-quant=NVFP4-routing=Renormalize]": 21.324984813084114, + "unittest/_torch/modules/test_awq_quantization.py": 20.191866873065017, + "unittest/_torch/modules/test_engram.py": 102.07443352601156, + "unittest/_torch/modules/test_fp4_num_tokens_slice.py": 21.468339943342777, + "unittest/_torch/modules/test_fused_activation_quant.py": 21.324547932330827, + "unittest/_torch/modules/test_fused_add_rms_norm_quant.py": 21.579973869346734, + "unittest/_torch/modules/test_fused_rmsnorm_fp4_quantize.py": 28.061774853801168, + "unittest/_torch/modules/test_gemma4_fused_qkv_prep.py": 25.013674285714284, + "unittest/_torch/modules/test_group_rmn_norm.py": 26.816904549509367, + "unittest/_torch/modules/test_mhc.py": 29.68183914728682, + "unittest/_torch/modules/test_mla_helix.py": 52.47190936106984, + "unittest/_torch/modules/test_moe_host_sharer.py": 22.169184415584414, + "unittest/_torch/modules/test_moe_load_balancer.py": 22.058864864864862, + "unittest/_torch/modules/test_moe_routing.py": 47.47694806421152, + "unittest/_torch/modules/test_rotary_embedding.py": 22.013752016129033, + "unittest/_torch/modules/test_triton_linear.py": 21.429786885245903, + "unittest/_torch/modules/tests_lora_modules": 334.8057459807074, + "unittest/_torch/multi_gpu -m \"not post_merge\"": 340.56549333333334, + "unittest/_torch/multi_gpu_modeling -k \"deepseek\"": 311.1584347826087, + "unittest/_torch/multi_gpu_modeling/test_deepseek.py::test_deepseek_streaming[tp1-bf16-trtllm-deepseekv3_lite]": 87.91341395348837, + "unittest/_torch/multi_gpu_modeling/test_deepseek.py::test_deepseek_streaming[tp4-bf16-trtllm-deepseekv3_lite]": 115.07457209302325, + "unittest/_torch/multimodal -k \"not nemotron_nano_v2_vl_fp8\"": 914.8516534653465, + "unittest/_torch/multimodal/test_mm_encoder_standalone.py -k \"qwen3_30b_a3b_fp8\"": 386.13995362318843, + "unittest/_torch/ray_orchestrator/multi_gpu -m \"gpu2\"": 229.27255670103094, + "unittest/_torch/ray_orchestrator/multi_gpu -m \"gpu4\"": 470.16044791666667, + "unittest/_torch/ray_orchestrator/multi_gpu/test_inflight_weight_update.py": 83.6284947368421, + "unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m \"part0\"": 606.4082, + "unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m \"part1\"": 434.8369, + "unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m \"part2\"": 604.1570294117647, + "unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m \"part3\"": 521.5557475728156, + "unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m \"part4\"": 1015.58, + "unittest/_torch/ray_orchestrator/multi_gpu/test_multi_instance.py::test_multi_instance[tp1_4instances]": 278.0227083333333, + "unittest/_torch/ray_orchestrator/multi_gpu/test_multi_instance.py::test_multi_instance[tp2_2instances]": 293.1756666666667, + "unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part0\"": 286.79657731958764, + "unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part1\"": 287.2486494845361, + "unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part2\"": 155.7070206185567, + "unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part3\"": 156.41694845360826, + "unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py -m \"part4\"": 214.8349175257732, + "unittest/_torch/ray_orchestrator/single_gpu/test_cache_transceiver_comm.py": 50.02742076502732, + "unittest/_torch/ray_orchestrator/single_gpu/test_llm_sleep.py": 116.42223463687151, + "unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part0\"": 718.5405393258427, + "unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part1\"": 676.5004831460674, + "unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py -m \"part2\"": 496.98034550561795, + "unittest/_torch/sampler -k \"not test_speculative_d2h_parity_real_predictor\"": 629.0897239263804, + "unittest/_torch/sampler/test_beam_search.py": 466.1707026378897, + "unittest/_torch/sampler/test_beam_search_speculative_d2h.py": 130.66764, + "unittest/_torch/sampler/test_logits_logprobs.py": 274.3431100478469, + "unittest/_torch/sampler/test_torch_sampler.py": 207.59631818181816, + "unittest/_torch/sampler/test_trtllm_sampler.py": 98.30468292682926, + "unittest/_torch/speculative/hw_agnostic": 841.041305882353, + "unittest/_torch/speculative/test_eagle3.py": 657.948134218289, + "unittest/_torch/speculative/test_rejection_buffers_guard.py": 20.309, + "unittest/_torch/test_model_config.py": 25.657892523364488, + "unittest/_torch/test_torch_multi_arange.py": 46.24331719128329, + "unittest/_torch/thop/parallel": 1885.2997194670613, + "unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py": 28.596906801007556, + "unittest/_torch/thop/parallel/test_w4a16_linear.py": 21.82673299748111, + "unittest/_torch/thop/parallel/test_w4a8_linear.py": 20.717201511335013, + "unittest/_torch/thop/parallel/test_weight_only_quant_gemm.py": 58.659168765743075, + "unittest/_torch/thop/parallel/test_weight_only_quant_linear.py": 26.54437279596977, + "unittest/_torch/thop/parallel_hw_agnostic": 470.2435887573964, + "unittest/_torch/thop/serial": 52.232433867735466, + "unittest/_torch/visual_gen/multi_gpu/test_attn2d_attention.py": 305.10693913043474, + "unittest/_torch/visual_gen/multi_gpu/test_cosmos3_transformer_parallel.py": 218.1122830188679, + "unittest/_torch/visual_gen/multi_gpu/test_flux2_transformer_parallel.py": 231.8527663551402, + "unittest/_torch/visual_gen/multi_gpu/test_flux_tp.py": 192.40505319148937, + "unittest/_torch/visual_gen/multi_gpu/test_flux_ulysses.py": 146.66659047619046, + "unittest/_torch/visual_gen/multi_gpu/test_ltx2_async_ulysses.py": 70.97888679245283, + "unittest/_torch/visual_gen/multi_gpu/test_ltx2_ulysses.py": 121.93633644859813, + "unittest/_torch/visual_gen/multi_gpu/test_parallel_attention.py": 44.38675531914893, + "unittest/_torch/visual_gen/multi_gpu/test_parallel_conv.py": 84.08998095238096, + "unittest/_torch/visual_gen/multi_gpu/test_parallel_group_norm.py": 64.73582075471698, + "unittest/_torch/visual_gen/multi_gpu/test_parallel_vae.py": 250.16000934579438, + "unittest/_torch/visual_gen/multi_gpu/test_ring_attention.py": 206.29218279569892, + "unittest/_torch/visual_gen/multi_gpu/test_tp_attention.py": 296.3244380952381, + "unittest/_torch/visual_gen/multi_gpu/test_ulysses_async.py": 66.38530188679246, + "unittest/_torch/visual_gen/multi_gpu/test_ulysses_attention.py": 392.43442990654205, + "unittest/_torch/visual_gen/multi_gpu/test_ulysses_sage_attention.py": 134.25890425531915, + "unittest/_torch/visual_gen/multi_gpu/test_visual_gen_mapping.py": 182.28520175438595, + "unittest/_torch/visual_gen/multi_gpu/test_visual_gen_multinode.py": 22.10380172413793, + "unittest/_torch/visual_gen/multi_gpu/test_wan_async_ulysses.py": 75.1850186915888, + "unittest/_torch/visual_gen/multi_gpu/test_wan_pipeline_parallel.py": 308.29408510638297, + "unittest/_torch/visual_gen/multi_gpu/test_wan_tp.py": 180.6913904761905, + "unittest/_torch/visual_gen/multi_gpu/test_wan_transformer_parallel.py": 455.3819038461539, + "unittest/_torch/visual_gen/test_attention_cute_dsl.py": 136.91421487603304, + "unittest/_torch/visual_gen/test_attention_cute_dsl_vsa.py": 43.35358474576271, + "unittest/_torch/visual_gen/test_attention_integration.py": 97.10202133333334, + "unittest/_torch/visual_gen/test_attention_perf.py": 210.67039617486338, + "unittest/_torch/visual_gen/test_attention_trtllm_sage.py": 189.85255145118734, + "unittest/_torch/visual_gen/test_cache_dit.py": 147.32329178470255, + "unittest/_torch/visual_gen/test_cosmos3_pipeline.py": 161.16777876106195, + "unittest/_torch/visual_gen/test_cosmos3_transformer.py": 98.51558529411764, + "unittest/_torch/visual_gen/test_flux_attention.py": 26.92808579088472, + "unittest/_torch/visual_gen/test_flux_infer.py": 25.655417233560094, + "unittest/_torch/visual_gen/test_flux_pipeline.py": 531.3778176795581, + "unittest/_torch/visual_gen/test_flux_pipeline.py::TestFluxCombinedOptimizations::test_all_optimizations_combined": 77.22944036697247, + "unittest/_torch/visual_gen/test_flux_pipeline.py::TestFluxParallelism::test_ulysses_2gpu_correctness": 64.36324778761062, + "unittest/_torch/visual_gen/test_flux_transformer.py": 38.82068067226891, + "unittest/_torch/visual_gen/test_ltx2_attention.py": 24.79130769230769, + "unittest/_torch/visual_gen/test_ltx2_pipeline.py": 717.9058804347826, + "unittest/_torch/visual_gen/test_ltx2_transformer.py": 27.601359173126614, + "unittest/_torch/visual_gen/test_model_loader.py": 46.429017441860466, + "unittest/_torch/visual_gen/test_quant_ops.py": 21.580876832844574, + "unittest/_torch/visual_gen/test_qwen_image_infer.py": 25.610972413793103, + "unittest/_torch/visual_gen/test_qwen_image_pipeline.py": 25.810387096774193, + "unittest/_torch/visual_gen/test_teacache.py": 20.69974858757062, + "unittest/_torch/visual_gen/test_tensor_payload.py": 25.807934240362812, + "unittest/_torch/visual_gen/test_trtllm_serve_e2e.py": 493.1424769647696, + "unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py": 29.099208108108108, + "unittest/_torch/visual_gen/test_visual_gen_args.py": 21.59079120879121, + "unittest/_torch/visual_gen/test_visual_gen_params.py": 26.508512376237626, + "unittest/_torch/visual_gen/test_visual_gen_utils.py": 21.791359890109888, + "unittest/_torch/visual_gen/test_wan21_i2v_pipeline.py": 223.45977393617022, + "unittest/_torch/visual_gen/test_wan21_i2v_teacache.py": 348.9664619883041, + "unittest/_torch/visual_gen/test_wan21_t2v_pipeline.py": 192.33468047337277, + "unittest/_torch/visual_gen/test_wan21_t2v_teacache.py": 250.91146764705883, + "unittest/_torch/visual_gen/test_wan21_t2v_teacache_user_coefficients.py": 55.60930623306233, + "unittest/_torch/visual_gen/test_wan22_i2v_pipeline.py": 305.1618966480447, + "unittest/_torch/visual_gen/test_wan22_i2v_teacache.py": 149.2714253521127, + "unittest/_torch/visual_gen/test_wan22_t2v_pipeline.py": 287.8022845303867, + "unittest/_torch/visual_gen/test_wan22_t2v_teacache.py": 138.68507407407407, + "unittest/_torch/visual_gen/test_wan22_ti2v_5b_pipeline.py": 132.64411436170212, + "unittest/_torch/visual_gen/test_wan_transformer.py": 86.32018313953489, + "unittest/_torch/visual_gen/test_wan_vae.py": 66.85342215568862, + "unittest/_torch/visual_gen/test_wan_vsa_pipeline.py": 132.44576832844575, + "unittest/_torch/visual_gen/test_warmup.py": 21.973272237196763, + "unittest/_torch/weight_sharing": 25.870444964871197, + "unittest/api_stability": 29.708095823095825, + "unittest/auto_deploy/multigpu/compile": 43.853566666666666, + "unittest/auto_deploy/multigpu/smoke": 357.5653522012579, + "unittest/auto_deploy/multigpu/transformations": 2098.098430107527, + "unittest/auto_deploy/multigpu/transformations/library/test_allreduce_residual_rmsnorm_fusion.py -k \"strategy_auto\"": 119.75213402061856, + "unittest/auto_deploy/multigpu/transformations/library/test_tp_sharding.py::test_moe_tp_shard_nvfp4": 69.07451546391754, + "unittest/auto_deploy/singlegpu/compile": 68.55923552123552, + "unittest/auto_deploy/singlegpu/custom_ops": 522.1245855855856, + "unittest/auto_deploy/singlegpu/custom_ops/attention/test_triton_attention.py::TestSDPADispatch": 38.268032258064515, + "unittest/auto_deploy/singlegpu/custom_ops/mamba/test_flashinfer_mamba_cached_op.py": 47.28830913978495, + "unittest/auto_deploy/singlegpu/custom_ops/moe/test_ad_moe_op.py": 29.06793530997305, + "unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py": 40.64752291105122, + "unittest/auto_deploy/singlegpu/custom_ops/quantization/test_quant.py": 50.19106738544475, + "unittest/auto_deploy/singlegpu/mlir": 29.818291828793775, + "unittest/auto_deploy/singlegpu/models": 228.87805836575873, + "unittest/auto_deploy/singlegpu/shim": 91.26811240310077, + "unittest/auto_deploy/singlegpu/smoke": 1713.564464993395, + "unittest/auto_deploy/singlegpu/smoke/test_ad_build_small_single.py -k \"Nemotron-3-Nano-30B-A3B-FP8 or Nemotron-Nano-3-30B-A3.5B-dev or Llama-4-Scout\"": 249.2459164420485, + "unittest/auto_deploy/singlegpu/smoke/test_ad_speculative_decoding.py": 210.74064480874318, + "unittest/auto_deploy/singlegpu/transformations": 320.5240232858991, + "unittest/auto_deploy/singlegpu/transformations/library/test_fuse_relu2_quant_nvfp4.py": 36.78159029649596, + "unittest/auto_deploy/singlegpu/transformations/library/test_moe_fusion.py": 66.3970269541779, + "unittest/auto_deploy/singlegpu/transformations/library/test_nvfp4_swiglu.py": 34.473213513513514, + "unittest/auto_deploy/singlegpu/transformations/library/test_quant_fusion.py": 39.26718699186991, + "unittest/auto_deploy/singlegpu/transformations/library/test_quant_moe.py": 31.283571815718158, + "unittest/auto_deploy/singlegpu/transformations/library/test_quantization.py": 33.57898641304348, + "unittest/auto_deploy/singlegpu/transformations/library/test_trtllm_gen_diag.py": 41.16601630434782, + "unittest/auto_deploy/singlegpu/utils": 45.979049222797926, + "unittest/auto_deploy/standalone": 719.8760111731843, + "unittest/bindings": 41.77800726392252, + "unittest/disaggregated/region/test_aux.py": 20.04190855457227, + "unittest/disaggregated/region/test_block.py": 23.17737385620915, + "unittest/disaggregated/region/test_page.py": 20.239856756756755, + "unittest/disaggregated/region/test_region.py": 20.012345047923322, + "unittest/disaggregated/test_agent_multi_backends.py": 82.98323180592992, + "unittest/disaggregated/test_bounce.py": 23.589685828877005, + "unittest/disaggregated/test_cache_reuse_adapter.py": 25.585138888888892, + "unittest/disaggregated/test_cache_transceiver_harness.py": 56.23020353982301, + "unittest/disaggregated/test_cache_transceiver_harness_report.py": 21.828332378223493, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver -k \"v1 and no_window\"": 189.49816756756758, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver -k \"v1 and uniform_window\"": 171.13203492063494, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver -k \"v2 and no_window\"": 228.43785139318885, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver -k \"v2 and uniform_window\"": 243.70949554896143, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver -k \"v2 and vswa\"": 221.75007692307693, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver_boundary_lengths -k \"v1\"": 53.39233522727273, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver_boundary_lengths -k \"v2\"": 64.50010810810811, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver_uneven_pp -k \"no_window\"": 163.61731012658228, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver_uneven_pp -k \"uniform_window\"": 150.50420743034056, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver_uneven_pp -k \"v2 and vswa\"": 102.53459940652819, + "unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver_v1_dsa_indexer": 49.56948913043478, + "unittest/disaggregated/test_cluster_storage.py": 47.17109420289855, + "unittest/disaggregated/test_coordinator_e2e.py": 72.26801550387597, + "unittest/disaggregated/test_coordinator_worker.py": 32.890360103626946, + "unittest/disaggregated/test_deepseek_v4_kv_transfer.py": 530.9268578431372, + "unittest/disaggregated/test_disagg_cluster_manager_worker.py": 55.52555365853659, + "unittest/disaggregated/test_disagg_openai_client.py": 33.3553485576923, + "unittest/disaggregated/test_disagg_utils.py": 23.560001321003963, + "unittest/disaggregated/test_disaggregated_params.py": 21.249914201183433, + "unittest/disaggregated/test_extractor.py": 26.12241229193342, + "unittest/disaggregated/test_kv_transfer.py": 607.4314255319149, + "unittest/disaggregated/test_kv_transfer_mp.py": 104.17107902735562, + "unittest/disaggregated/test_mamba_transfer.py": 33.60644819277109, + "unittest/disaggregated/test_messenger.py": 23.172675066312998, + "unittest/disaggregated/test_minimax_m3_kv_transfer.py": 337.1307835051547, + "unittest/disaggregated/test_openai_disagg_server.py": 27.25944660194175, + "unittest/disaggregated/test_openai_disagg_service.py": 33.72867386091127, + "unittest/disaggregated/test_openai_server_info.py": 28.47416666666667, + "unittest/disaggregated/test_peer.py": 23.180986541049798, + "unittest/disaggregated/test_perf_logger.py": 20.14458017492711, + "unittest/disaggregated/test_pool_matching.py": 19.757593406593404, + "unittest/disaggregated/test_py_cache_transceiver_mp.py -k \"ctx_first and not ctx_first_sync\"": 442.66612371134016, + "unittest/disaggregated/test_py_cache_transceiver_mp.py -k \"ctx_first_sync\"": 409.2202551020408, + "unittest/disaggregated/test_py_cache_transceiver_mp.py -k \"gen_first1\"": 446.7769587628866, + "unittest/disaggregated/test_py_cache_transceiver_mp.py -k \"gen_first2\"": 422.67712244897956, + "unittest/disaggregated/test_rank_info.py": 19.967412742382272, + "unittest/disaggregated/test_remoteDictionary.py": 28.488994572591587, + "unittest/disaggregated/test_request_id.py": 19.980398791540786, + "unittest/disaggregated/test_router.py": 25.549286479250334, + "unittest/disaggregated/test_sync_transfer_timing.py": 19.801176470588235, + "unittest/disaggregated/test_transceiver_bounded_polling.py": 20.033817204301073, + "unittest/dynamo": 30.93964938271605, + "unittest/executor/test_base_worker.py": 131.1828982630273, + "unittest/executor/test_fatal_error_health_check.py": 28.003123831775703, + "unittest/executor/test_ipc.py": 29.161146853146853, + "unittest/executor/test_multi_frontend_routing.py": 20.379634020618557, + "unittest/executor/test_postprocessor_hook.py": 29.60544927536232, + "unittest/executor/test_proxy_fast_death.py": 30.611908536585364, + "unittest/executor/test_proxy_postproc_terminate.py": 25.622395683453238, + "unittest/executor/test_rpc.py": 428.50881176470585, + "unittest/executor/test_stats_serializer.py": 22.270259154929576, + "unittest/inputs/test_chat_template_dispatch.py": 27.11566503667482, + "unittest/inputs/test_content_format.py": 25.79082608695652, + "unittest/inputs/test_multimodal.py": 25.693597087378638, + "unittest/inputs/test_multimodal_input_processor.py": 25.525548309178742, + "unittest/inputs/test_url_validation.py": 25.912739659367396, + "unittest/inputs/test_video_decode.py": 25.766543046357615, + "unittest/kv_cache_manager_v2_tests": 213.1598582781457, + "unittest/llmapi/apps/_test_openai_chat_multimodal.py::test_single_chat_session_image_embeds -m needs_l40s": 160.2151168224299, + "unittest/llmapi/apps/_test_openai_embeddings.py": 156.40381323877068, + "unittest/llmapi/apps/_test_openai_multi_gpu.py -m \"part0\"": 119.85816666666668, + "unittest/llmapi/apps/test_chat_utils.py": 27.616254716981132, + "unittest/llmapi/apps/test_disagg_serving_perf_metrics.py": 78.61523157894736, + "unittest/llmapi/apps/test_encode_batcher.py": 27.48177108433735, + "unittest/llmapi/apps/test_harmony_channel_validation.py": 30.0566327014218, + "unittest/llmapi/apps/test_tool_parsers.py": 29.678218009478673, + "unittest/llmapi/test_additional_model_outputs.py -m \"gpu1\"": 61.69495260663508, + "unittest/llmapi/test_async_llm.py -m \"gpu2\"": 83.43303092783505, + "unittest/llmapi/test_async_llm.py -m \"gpu4\"": 89.11570833333333, + "unittest/llmapi/test_async_llm.py -m \"not (gpu2 or gpu4)\"": 201.0308156424581, + "unittest/llmapi/test_deepseek_v4_tokenizer.py": 20.679311764705883, + "unittest/llmapi/test_embedding_arch_routing.py": 26.95822033898305, + "unittest/llmapi/test_executor.py": 27.13577777777778, + "unittest/llmapi/test_gc_utils.py": 32.136959427207636, + "unittest/llmapi/test_kv_cache_dtype_override.py": 27.04092523364486, + "unittest/llmapi/test_llm.py -m \"not part0\"": 214.32133333333334, + "unittest/llmapi/test_llm.py -m \"part0\"": 323.924, + "unittest/llmapi/test_llm_args.py": 131.87056698564595, + "unittest/llmapi/test_llm_encode.py": 61.49556812339331, + "unittest/llmapi/test_llm_encode_multi_item.py": 126.70357742782153, + "unittest/llmapi/test_llm_kv_cache_events.py::test_llm_api_attention_dp_kv_events": 53.872166666666665, + "unittest/llmapi/test_llm_multi_gpu_pytorch.py -m \"gpu4\"": 72.83206930693069, + "unittest/llmapi/test_llm_pytorch.py -m \"part0\"": 919.85638593482, + "unittest/llmapi/test_llm_pytorch.py -m \"part1\"": 258.6908440860215, + "unittest/llmapi/test_llm_pytorch.py -m \"part2\"": 516.3453256445048, + "unittest/llmapi/test_llm_pytorch.py -m \"part3\"": 280.88529755434786, + "unittest/llmapi/test_llm_pytorch.py::test_nemotron_nas_lora": 197.97414285714288, + "unittest/llmapi/test_llm_quant.py": 27.24870817843866, + "unittest/llmapi/test_llm_telemetry.py": 175.43786629526463, + "unittest/llmapi/test_llm_telemetry.py::TestTelemetryArchitectureExtraction": 72.02631630170316, + "unittest/llmapi/test_llm_telemetry.py::TestTelemetryPyTorchBackend": 71.87491931540343, + "unittest/llmapi/test_llm_utils.py": 31.80273798076923, + "unittest/llmapi/test_memory_profiling.py::test_pyexecutor_and_kvcache_share_execution_stream": 41.98156234096692, + "unittest/llmapi/test_mpi_session.py": 156.37610396039605, + "unittest/llmapi/test_mpi_session.py::test_llmapi_launch_multiple_tasks": 88.6355319148936, + "unittest/llmapi/test_reasoning_parser.py": 29.937814285714285, + "unittest/llmapi/test_request_priority.py": 25.679375296912113, + "unittest/llmapi/test_serialization.py": 25.607498806682578, + "unittest/llmapi/test_utils.py": 27.70747016706444, + "unittest/metrics/test_collector.py": 23.620726519337016, + "unittest/models/test_quant_config_utils.py": 25.616273809523808, + "unittest/others/test_convert_utils.py": 25.440990243902437, + "unittest/others/test_kv_cache_transceiver.py::test_async_transfer_keeps_llm_request_alive": 12.929715589541194, + "unittest/others/test_kv_cache_transceiver.py::test_cancel_request_in_transmission[mha]": 28.022305486900642, + "unittest/others/test_kv_cache_transceiver.py::test_cancel_request_in_transmission[mla]": 16.43778546712803, + "unittest/others/test_kv_cache_transceiver.py::test_cpp_nixl_sync_transfer_stress": 31.1141288945796, + "unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[NIXL-mha-ctx_fp16_gen_fp16]": 24.296445485939813, + "unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mha-ctx_fp16_gen_fp16]": 25.86616713091922, + "unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mla-ctx_fp16_gen_fp16]": 24.195852528089887, + "unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[UCX-mha-ctx_fp16_gen_fp16]": 12.57014578194376, + "unittest/others/test_kv_cache_transceiver.py::test_kv_transfer_timeout_silent_when_unset": 11.213987419832264, + "unittest/others/test_kv_cache_transceiver.py::test_kv_transfer_timeout_warns_once_per_request": 11.21132955106068, + "unittest/others/test_lora_manager.py": 36.074141463414634, + "unittest/others/test_lora_module_count.py": 27.08678883495146, + "unittest/others/test_time_breakdown.py": 27.776176904176904, + "unittest/others/test_tracing.py": 147.28475609756097, + "unittest/quantization/test_mode.py": 25.56721153846154, + "unittest/scaffolding": 446.7132156862745, + "unittest/scripts": 31.32473975903614, + "unittest/tools": 376.44609876543205, + "unittest/tools/test_layer_wise_benchmarks.py::test_deepseek_r1_ctx_dep[1]": 20.500902515723272, + "unittest/tools/test_layer_wise_benchmarks.py::test_nemotron_gen_dep[1]": 114.82735611510792, + "unittest/tools/test_layer_wise_benchmarks.py::test_performance_alignment[1]": 29.439571428571426, + "unittest/tools/test_layer_wise_benchmarks.py::test_qwen3_next_gen_tep[1]": 88.21061647058823, + "unittest/usage/test_collectors.py": 23.418028416779432, + "unittest/usage/test_config.py": 23.08291386271871, + "unittest/usage/test_e2e_capture.py": 23.17349802890933, + "unittest/usage/test_llmapi_config_telemetry_docs.py": 28.078541871921182, + "unittest/usage/test_opt_out.py": 23.03156282722513, + "unittest/usage/test_reporter.py": 26.090890645586295, + "unittest/usage/test_schema.py": 23.357160167130917, + "unittest/usage/test_transport.py": 29.417058265582654, + "unittest/utils/test_logger.py": 25.89497136038186, + "unittest/utils/test_util.py": 28.75711650485437, + "unittest/visual_gen/test_iteration_stats.py": 25.598708624708625, + "unittest/visual_gen/test_media_encoding.py": 25.9517546728972, + "unittest/visual_gen/test_output.py": 25.937927272727272, + "verl/test_verl_cases.py::test_async_generate": 424.8718181818182, + "verl/test_verl_cases.py::test_async_memory_management": 93.11290909090908, + "verl/test_verl_cases.py::test_init_without_device_mesh": 119.52972727272729, + "verl/test_verl_cases.py::test_inter_node_trtllm_rollout": 121.82945454545454, + "verl/test_verl_cases.py::test_make_async_request_get_method": 24.27409090909091, + "verl/test_verl_cases.py::test_make_async_request_http_error": 24.187363636363635, + "verl/test_verl_cases.py::test_make_async_request_max_attempts_exceeded": 24.261909090909093, + "verl/test_verl_cases.py::test_make_async_request_post_method": 25.87009090909091, + "verl/test_verl_cases.py::test_multimodal_different_image_sizes": 171.35136363636366, + "verl/test_verl_cases.py::test_multimodal_generate_with_image": 127.19154545454546, + "verl/test_verl_cases.py::test_multimodal_text_only_fallback": 115.52581818181818, + "verl/test_verl_cases.py::test_placement_group_multi_node_multi_replica": 21.758363636363637, + "verl/test_verl_cases.py::test_placement_group_multi_node_ray_resource_pool": 21.58818181818182, + "verl/test_verl_cases.py::test_placement_group_with_ray_resource_pool": 21.164454545454543, + "verl/test_verl_cases.py::test_placement_group_with_sub_ray_resource_pool": 21.24172727272727, + "verl/test_verl_cases.py::test_unimodal_batch_generate": 128.48545454545453, + "verl/test_verl_cases.py::test_unimodal_generate": 103.0079090909091, + "verl/test_verl_cases.py::test_wake_sleep_cycle": 106.03390909090909, + "visual_gen/test_visual_gen_benchmark.py::test_offline_benchmark": 74.4884, + "visual_gen/test_visual_gen_benchmark.py::test_online_benchmark[openai-videos]": 96.1846 } diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index facfca89379b..ef32f74f1ba0 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -155,6 +155,12 @@ deepseek-ai/DeepSeek-V4-Pro: kv_cache_quant_algo: FP8 spec_dec_algo: MTP accuracy: 96.0 + # Full GSM8K DEP8 DSpark path: TP=8, EP=8, attention DP, MegaMoE DeepGEMM, + # and max_draft_len=5. Measured 96.475 on 1319 samples; floor set to 96.0 + # for run-to-run margin. + - quant_algo: FP8_BLOCK_SCALES + spec_dec_algo: DSpark + accuracy: 96.0 Qwen3/Qwen3-4B: - spec_dec_algo: Eagle accuracy: 85.823 @@ -240,6 +246,10 @@ Qwen/Qwen3.5-397B-A17B: - quant_algo: NVFP4 kv_cache_quant_algo: FP8 accuracy: 95.0 + - quant_algo: NVFP4 + kv_cache_quant_algo: FP8 + spec_dec_algo: MTP + accuracy: 95.0 moonshotai/Kimi-K2-Instruct: - quant_algo: FP8_BLOCK_SCALES accuracy: 94.84 @@ -482,6 +492,9 @@ MiniMaxAI/MiniMax-M3-MXFP8: nvidia/MiniMax-M3-NVFP4: - quant_algo: MIXED_PRECISION accuracy: 88 + - quant_algo: MIXED_PRECISION + kv_cache_quant_algo: FP8 + accuracy: 86 nvidia/NVIDIA-Nemotron-Nano-9B-v2: - accuracy: 85.027 - quant_algo: FP8 diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index cbce62591221..a870fbba6402 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -284,6 +284,9 @@ MiniMaxAI/MiniMax-M3-MXFP8: nvidia/MiniMax-M3-NVFP4: - quant_algo: MIXED_PRECISION accuracy: 83 + - quant_algo: MIXED_PRECISION + kv_cache_quant_algo: FP8 + accuracy: 81 moonshotai/Kimi-K2-Instruct: - quant_algo: FP8_BLOCK_SCALES accuracy: 87.65 diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index cd63c052375c..60bf3034028f 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -166,7 +166,7 @@ def launch_disaggregated_llm( print( f"Using unified tp parameter for testing is not recommended. Please use server configs instead." ) - perf_max_requests = 50 + perf_metrics_output_dir = os.path.join(temp_dir.name, "perf_metrics") def _apply_perf_flags(cfg: Optional[Dict[str, Any]]): if not isinstance(cfg, dict): @@ -175,7 +175,7 @@ def _apply_perf_flags(cfg: Optional[Dict[str, Any]]): # Only set these if the switch is enabled. # Use `setdefault` so explicit per-test overrides are preserved. cfg.setdefault("return_perf_metrics", True) - cfg.setdefault("perf_metrics_max_requests", perf_max_requests) + cfg.setdefault("perf_metrics_output_dir", perf_metrics_output_dir) _apply_perf_flags(disaggregated_server_config) _apply_perf_flags(ctx_server_config) @@ -479,27 +479,6 @@ def generate_async(prompt: str, thread_pool.futures.append(future) return future - def _get_perf_metrics(): - path = "/perf_metrics" - perf_url = f"http://localhost:{serve_port}{path}" - try: - print(f"Fetching perf metrics from {perf_url}") - resp = requests.get(perf_url, timeout=10) - if resp.status_code == 200: - try: - metrics = resp.json() - print("perf_metrics JSON:") - print(json.dumps(metrics, indent=2, ensure_ascii=False)) - except ValueError: - print("perf_metrics returned non-JSON response:", - resp.text) - else: - print( - f"perf_metrics returned status {resp.status_code}: {resp.text}" - ) - except requests.exceptions.RequestException as e: - print(f"Error fetching {perf_url}: {e}") - def _show_kvcache_time(kv_cache_perf_dir, max_lines=100): print(f"kv_cache_perf_dir: {kv_cache_perf_dir}") for file in os.listdir(kv_cache_perf_dir): @@ -515,7 +494,6 @@ def _show_kvcache_time(kv_cache_perf_dir, max_lines=100): finally: if enable_perf: _show_kvcache_time(kv_cache_perf_dir) - _get_perf_metrics() # Gracefully shut down all server processes all_processes = list( @@ -1212,30 +1190,19 @@ def test_auto_dtype(self, overlap_scheduler, mtp_nextn): @skip_pre_blackwell @pytest.mark.skip_less_device(8) - @pytest.mark.parametrize( - "gen_pp,gen_tp,gen_cp,enable_attention_dp", [ - (1, 1, 4, False), - (1, 2, 2, False), - (1, 2, 2, True), - (2, 1, 2, False), - ], - ids=["pp1tp1cp4", "pp1tp2cp2", "pp1dp2cp2", "pp2tp1cp2"]) + @pytest.mark.parametrize("gen_pp,gen_tp,gen_cp,enable_attention_dp", [ + (1, 2, 2, False), + (1, 2, 2, True), + ], + ids=["pp1tp2cp2", "pp1dp2cp2"]) @pytest.mark.parametrize("cuda_graph_config", [ - None, - { - "enable_padding": False, - "batch_sizes": [1, 2, 4, 8, 16, 32, 64] - }, { "enable_padding": True, "batch_sizes": [1, 2, 4, 8, 16, 32, 64] }, ], - ids=[ - "cudagraph:none", "cudagraph:without_padding", - "cudagraph:with_padding" - ]) - @pytest.mark.parametrize("comms_medium", ["fifo_v1", "fifo_v2", "nccl"]) + ids=["cudagraph:with_padding"]) + @pytest.mark.parametrize("comms_medium", ["fifo_v2"]) def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config, gen_pp, gen_tp, gen_cp, enable_attention_dp): # Parse comms_medium to get use_nccl_for_alltoall and fifo_version. @@ -1387,15 +1354,12 @@ def test_kv_cache_v2_nixl_python(self): } disaggregated_server_config = { "hostname": "localhost", - "port": 8000, "backend": "pytorch", "context_servers": { - "num_instances": 1, - "urls": ["localhost:8001"] + "num_instances": 1 }, "generation_servers": { - "num_instances": 1, - "urls": ["localhost:8002"] + "num_instances": 1 } } with launch_disaggregated_llm(disaggregated_server_config, @@ -1763,100 +1727,60 @@ def test_auto_dtype(self, overlap_scheduler): model_name=self.MODEL_NAME, test_sets=["MMLU", "GSM8K"]) - @skip_pre_blackwell - @pytest.mark.skip_less_device(8) - @pytest.mark.parametrize( - "gen_pp,gen_tp,gen_cp,enable_attention_dp", [ - (1, 1, 4, False), - (1, 2, 2, False), - (1, 2, 2, True), - (2, 1, 2, False), - ], - ids=["pp1tp1cp4", "pp1tp2cp2", "pp1dp2cp2", "pp2tp1cp2"]) - @pytest.mark.parametrize("cuda_graph_config", [ - None, - { - "enable_padding": True, - "batch_sizes": [1, 2, 4, 8, 16, 32, 64], - }, - ], - ids=[ - "cudagraph:none", - "cudagraph:with_padding", - ]) - @pytest.mark.parametrize("comms_medium", ["fifo", "nccl"]) - def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config, - gen_pp, gen_tp, gen_cp, enable_attention_dp): - use_nccl_for_alltoall = comms_medium == "nccl" - fifo_version = 2 - gen_ep = gen_tp * gen_cp - kv_cache_config = { - "free_gpu_memory_fraction": 0.5, - "enable_block_reuse": False, - "enable_partial_reuse": False, - "tokens_per_block": 32, - "dtype": "fp8", - } + @pytest.mark.skip_less_device(4) + @pytest.mark.skip_less_device_memory(200000) + @pytest.mark.parametrize("use_kv_cache_manager_v2", [False], + ids=["cache_mgr_v1"]) + def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): + """Test with KV cache manager v1, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" + max_num_tokens = 8192 + moe_config = {"backend": "TRTLLM", "max_num_tokens": max_num_tokens} ctx_server_config = { - "pipeline_parallel_size": 1, - "tensor_parallel_size": 4, - "context_parallel_size": 1, "disable_overlap_scheduler": True, - "kv_cache_config": kv_cache_config, - "enable_chunked_prefill": False, - "cuda_graph_config": None, - "cache_transceiver_config": { - "backend": "UCX", - "max_tokens_in_buffer": 8192, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 }, - "moe_config": { - "backend": "TRTLLM", - "max_num_tokens": 16384, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_autotuner": False, } gen_server_config = { - "tensor_parallel_size": gen_tp, - "pipeline_parallel_size": gen_pp, - "context_parallel_size": gen_cp, - "moe_expert_parallel_size": gen_ep, - "cp_config": { - "cp_type": "HELIX", - "tokens_per_block": 32, - "use_nccl_for_alltoall": use_nccl_for_alltoall, - "fifo_version": fifo_version, + "disable_overlap_scheduler": False, + "moe_config": moe_config, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 }, - "disable_overlap_scheduler": True, - "kv_cache_config": kv_cache_config, - "enable_chunked_prefill": False, - "cuda_graph_config": cuda_graph_config, "cache_transceiver_config": { - "backend": "UCX", - "max_tokens_in_buffer": 8192, - }, - "moe_config": { - "backend": "TRTLLM", - "max_num_tokens": 16384, + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 }, - "enable_attention_dp": enable_attention_dp, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_autotuner": False, } disaggregated_server_config = { "hostname": "localhost", - "port": 8000, "backend": "pytorch", "context_servers": { "num_instances": 1, - "urls": ["localhost:8001"] }, "generation_servers": { "num_instances": 1, - "urls": ["localhost:8002"] } } with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, - gen_server_config, - self.MODEL_PATH, - max_workers=128) as llm: + ctx_server_config, gen_server_config, + self.MODEL_PATH) as llm: run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) @@ -1988,30 +1912,19 @@ def test_chunked_prefill(self): @skip_pre_blackwell @pytest.mark.skip_less_device(8) - @pytest.mark.parametrize( - "gen_pp,gen_tp,gen_cp,enable_attention_dp", [ - (1, 1, 4, False), - (1, 2, 2, False), - (1, 2, 2, True), - (2, 1, 2, False), - ], - ids=["pp1tp1cp4", "pp1tp2cp2", "pp1dp2cp2", "pp2tp1cp2"]) + @pytest.mark.parametrize("gen_pp,gen_tp,gen_cp,enable_attention_dp", [ + (1, 2, 2, False), + (1, 2, 2, True), + ], + ids=["pp1tp2cp2", "pp1dp2cp2"]) @pytest.mark.parametrize("cuda_graph_config", [ - None, - { - "enable_padding": False, - "batch_sizes": [1, 2, 4, 8, 16, 32, 64] - }, { "enable_padding": True, "batch_sizes": [1, 2, 4, 8, 16, 32, 64] }, ], - ids=[ - "cudagraph:none", "cudagraph:without_padding", - "cudagraph:with_padding" - ]) - @pytest.mark.parametrize("comms_medium", ["fifo_v1", "fifo_v2", "nccl"]) + ids=["cudagraph:with_padding"]) + @pytest.mark.parametrize("comms_medium", ["fifo_v2"]) def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config, gen_pp, gen_tp, gen_cp, enable_attention_dp): # Parse comms_medium to get use_nccl_for_alltoall and fifo_version. @@ -2199,59 +2112,6 @@ def test_mixed_ctx_gen_model(self, ctx_pp, gen_tp): gen_instances=1) -@pytest.mark.timeout(10800) -@skip_pre_blackwell -class TestKimiK2(LlmapiAccuracyTestHarness): - MODEL_NAME = "moonshotai/Kimi-K2-Thinking" - MODEL_PATH = f"{llm_models_root()}/Kimi-K2-Thinking-NVFP4" - - @pytest.mark.skip_less_device(8) - @pytest.mark.skip_less_device_memory(200000) - def test_nvfp4(self): - ctx_server_config = { - "max_batch_size": 16, - "disable_overlap_scheduler": True, - "cache_transceiver_config": { - "backend": "DEFAULT", - "max_tokens_in_buffer": 4096 - }, - "tensor_parallel_size": 4, - "enable_attention_dp": True, - "trust_remote_code": True, - "kv_cache_config": { - "free_gpu_memory_fraction": 0.8, - }, - } - gen_server_config = { - "max_batch_size": 16, - "disable_overlap_scheduler": True, - "cache_transceiver_config": { - "backend": "DEFAULT", - "max_tokens_in_buffer": 4096 - }, - "tensor_parallel_size": 4, - "enable_attention_dp": True, - "trust_remote_code": True, - "kv_cache_config": { - "free_gpu_memory_fraction": 0.8, - }, - } - disaggregated_server_config = { - "hostname": "localhost", - "backend": "pytorch", - "context_servers": { - "num_instances": 1 - }, - "generation_servers": { - "num_instances": 1 - } - } - with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, gen_server_config, - self.MODEL_PATH) as llm: - run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) - - @pytest.mark.timeout(10800) @skip_pre_blackwell class TestKimiK25(LlmapiAccuracyTestHarness): @@ -2263,9 +2123,13 @@ class TestKimiK25(LlmapiAccuracyTestHarness): def test_nvfp4(self): """Disaggregated GSM8K accuracy for Kimi-K2.5 (NVFP4). - ctx and gen servers are each TP4 (8 GPUs total) over the default cache + ctx and gen servers are each TP4 (8 GPUs total). The cache transceiver + uses backend=NIXL + transceiver_runtime=PYTHON: NIXL is required so the + disagg test harness skips its TRTLLM_USE_UCX_KVCACHE=1 fallback (which + would make the effective backend UCX and force the C++ transceiver), + letting the ctx->gen MLA-latent KV transfer run over the Python (v2) transceiver. GSM8K is text-only, so requests run through the DeepSeek-V3 - MLA backbone (no vision) and the ctx->gen KV transfer is the MLA latent. + MLA backbone (no vision). Kimi-K2.5 ships custom HF modeling code (auto_map in config.json), so trust_remote_code must be set on both servers or executor init fails at config parse time. @@ -2283,7 +2147,8 @@ def test_nvfp4(self): "max_num_tokens": 8192, "disable_overlap_scheduler": True, "cache_transceiver_config": { - "backend": "DEFAULT", + "backend": "NIXL", + "transceiver_runtime": "PYTHON", "max_tokens_in_buffer": 4096 }, "tensor_parallel_size": 4, @@ -2299,7 +2164,8 @@ def test_nvfp4(self): "max_num_tokens": 8192, "disable_overlap_scheduler": True, "cache_transceiver_config": { - "backend": "DEFAULT", + "backend": "NIXL", + "transceiver_runtime": "PYTHON", "max_tokens_in_buffer": 4096 }, "tensor_parallel_size": 4, @@ -2394,10 +2260,20 @@ def _make_configs(self, use_py_transceiver: bool = False): return ctx_server_config, gen_server_config, disaggregated_server_config @pytest.mark.skip_less_device(8) - @parametrize_with_ids("use_py_transceiver", [True, False]) - @parametrize_with_ids("block_reuse", [True, False]) - @parametrize_with_ids("mtp_nextn", [0, 1, 3]) - def test_auto_dtype(self, use_py_transceiver, block_reuse, mtp_nextn): + @pytest.mark.parametrize( + "mtp_nextn,block_reuse,use_py_transceiver", + [ + (0, False, False), + (0, False, True), + (3, True, False), + ], + ids=[ + "mtp_nextn=0-block_reuse=False-use_py_transceiver=False", + "mtp_nextn=0-block_reuse=False-use_py_transceiver=True", + "mtp_nextn=3-block_reuse=True-use_py_transceiver=False", + ], + ) + def test_auto_dtype(self, mtp_nextn, block_reuse, use_py_transceiver): if use_py_transceiver and block_reuse: pytest.skip("Python transceiver does not support block reuse") @@ -2770,3 +2646,66 @@ def test_auto_dtype(self): server_waiting_timeout=3600) as llm: task = MMLU(self.MODEL_NAME) task.evaluate(llm, is_integration_test=True) + + +@pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) +@skip_pre_blackwell +@pytest.mark.skip_less_device_memory(200000) +class TestDeepSeekR1(LlmapiAccuracyTestHarness): + MODEL_NAME = "deepseek-ai/DeepSeek-R1" + MODEL_PATH = f"{llm_models_root()}/DeepSeek-R1/DeepSeek-R1-0528-FP4-v2" + + @pytest.mark.skip_less_device(4) + @pytest.mark.parametrize("use_kv_cache_manager_v2", [False], + ids=["cache_mgr_v1"]) + def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): + """Test with KV cache manager v1, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" + max_num_tokens = 8192 + moe_config = {"backend": "TRTLLM", "max_num_tokens": max_num_tokens} + ctx_server_config = { + "disable_overlap_scheduler": True, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 + }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_autotuner": False, + } + gen_server_config = { + "disable_overlap_scheduler": False, + "moe_config": moe_config, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 + }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_autotuner": False, + } + disaggregated_server_config = { + "hostname": "localhost", + "backend": "pytorch", + "context_servers": { + "num_instances": 1, + }, + "generation_servers": { + "num_instances": 1, + } + } + with launch_disaggregated_llm(disaggregated_server_config, + ctx_server_config, gen_server_config, + self.MODEL_PATH) as llm: + run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 2305c0852664..0319c329d9a7 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -30,8 +30,9 @@ # isort: off from tensorrt_llm.llmapi import ( AttentionDpConfig, CudaGraphConfig, DeepSeekSparseAttentionConfig, - DFlashDecodingConfig, DraftTargetDecodingConfig, Eagle3DecodingConfig, - KvCacheConfig, MiniMaxM3SparseAttentionConfig, MoeConfig, MTPDecodingConfig, + DFlashDecodingConfig, DSparkDecodingConfig, DraftTargetDecodingConfig, + Eagle3DecodingConfig, KvCacheConfig, MambaStateConfig, + MiniMaxM3SparseAttentionConfig, MoeConfig, MTPDecodingConfig, NGramDecodingConfig, PARDDecodingConfig, RocketSparseAttentionConfig, SADecodingConfig, SamplingParams, SchedulerConfig, SkipSoftmaxAttentionConfig, SAEnhancerConfig, TorchCompileConfig) @@ -2865,6 +2866,13 @@ def test_nvfp4_multi_gpus(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, if moe_backend == "TRTLLM" and sm_version in (120, 121): pytest.skip(f"{moe_backend} backend does not support SM 120 or 121") + # Wrong MTP draft tokens cannot be caught by the accuracy scores: + # rejected drafts are regenerated by the target model, so accuracy + # stays intact and the only observable symptom is a collapsed + # acceptance length (https://nvbugs/6460072). Assert on it for the + # ADP + LM-head-TP cases, whose draft sampling takes a dedicated path. + check_acceptance = mtp_nextn > 0 and enable_lm_head_tp_in_adp + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.70) pytorch_config = dict( disable_overlap_scheduler=not overlap_scheduler, @@ -2886,6 +2894,7 @@ def test_nvfp4_multi_gpus(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, **pytorch_config, enable_attention_dp=attention_dp, enable_lm_head_tp_in_adp=enable_lm_head_tp_in_adp, + enable_iter_perf_stats=check_acceptance, speculative_config=mtp_config) as llm: assert llm.args.moe_config.backend == moe_backend @@ -2903,6 +2912,22 @@ def test_nvfp4_multi_gpus(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, # task.evaluate(llm, # extra_evaluator_kwargs=dict(apply_chat_template=True)) + if check_acceptance: + stats = llm.get_stats(timeout=2) + spec_iters = [ + stat['specDecodingStats'] for stat in stats + if stat.get('specDecodingStats') + and stat['specDecodingStats']['numDraftTokens'] > 0 + ] + assert spec_iters, \ + "No iterations with speculative decoding stats" + acceptance_length = sum(s['acceptanceLength'] + for s in spec_iters) / len(spec_iters) + assert acceptance_length >= 1.5, ( + f"MTP acceptance length {acceptance_length:.2f} < 1.5: " + "draft tokens are likely wrong in the ADP + LM-head-TP " + "sampling path.") + import gc gc.collect() torch.cuda.empty_cache() @@ -3318,24 +3343,34 @@ class TestDeepSeekV32(LlmapiAccuracyTestHarness): @skip_pre_hopper @pytest.mark.skip_less_device_memory(140000) @pytest.mark.parametrize( - "tp_size,pp_size,ep_size,mtp_nextn,fp8kv,attention_dp,cuda_graph,overlap_scheduler,max_batch_size,moe_backend,disable_skip_indexer,enable_heuristic_topk", + "tp_size,pp_size,ep_size,mtp_nextn,fp8kv,attention_dp,cuda_graph,overlap_scheduler,max_batch_size,moe_backend,disable_skip_indexer,enable_heuristic_topk,use_cute_dsl_topk", [ - (8, 1, 8, 0, False, True, True, True, 24, "_DEFAULT", False, False), - (8, 1, 8, 1, False, True, True, True, 24, "_DEFAULT", False, False), - (8, 1, 8, 0, True, True, True, True, 24, "_DEFAULT", False, False), - (8, 1, 8, 3, False, False, True, True, 1, "TRTLLM", False, False), - (8, 1, 8, 3, False, False, True, True, 1, "_DEFAULT", False, False), - (8, 1, 8, 1, False, True, True, True, 24, "_DEFAULT", True, False), - (8, 1, 8, 1, False, True, True, True, 24, "_DEFAULT", False, True), + (8, 1, 8, 0, False, True, True, True, 24, "_DEFAULT", False, False, + False), + (8, 1, 8, 1, False, True, True, True, 24, "_DEFAULT", False, False, + False), + (8, 1, 8, 0, True, True, True, True, 24, "_DEFAULT", False, False, + False), + (8, 1, 8, 3, False, False, True, True, 1, "TRTLLM", False, False, + False), + (8, 1, 8, 3, False, False, True, True, 1, "_DEFAULT", False, False, + False), + (8, 1, 8, 1, False, True, True, True, 24, "_DEFAULT", True, False, + False), + (8, 1, 8, 1, False, True, True, True, 24, "_DEFAULT", False, True, + False), + (8, 1, 8, 3, False, True, True, True, 24, "_DEFAULT", False, True, + True), ], ids=[ "baseline", "baseline_mtp1", "baseline_fp8kv", "latency", - "latency_default", "disable_skip_indexer", "heuristic_topk_mtp1" + "latency_default", "disable_skip_indexer", "heuristic_topk_mtp1", + "cute_dsl_gvr_mtp3" ]) def test_fp8_blockscale(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, attention_dp, cuda_graph, overlap_scheduler, max_batch_size, moe_backend, disable_skip_indexer, - enable_heuristic_topk): + enable_heuristic_topk, use_cute_dsl_topk): if get_sm_version() == 100 or get_sm_version() == 103: moe_backend = "DEEPGEMM" if moe_backend == "_DEFAULT" else moe_backend moe_config = MoeConfig(backend=moe_backend, max_num_tokens=16384) @@ -3362,16 +3397,19 @@ def test_fp8_blockscale(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, ) kv_cache_config.dtype = "fp8" - if enable_heuristic_topk and get_sm_version() < 100: + if (enable_heuristic_topk + or use_cute_dsl_topk) and get_sm_version() < 100: pytest.skip("Heuristic TopK requires Blackwell (SM >= 100)") - dsa_config = None + dsa_kwargs = {} if disable_skip_indexer: - dsa_config = DeepSeekSparseAttentionConfig( - skip_indexer_for_short_seqs=False) + dsa_kwargs["skip_indexer_for_short_seqs"] = False if enable_heuristic_topk: - dsa_config = DeepSeekSparseAttentionConfig( - enable_heuristic_topk=True) + dsa_kwargs["enable_heuristic_topk"] = enable_heuristic_topk + if use_cute_dsl_topk: + dsa_kwargs["use_cute_dsl_topk"] = use_cute_dsl_topk + dsa_config = DeepSeekSparseAttentionConfig( + **dsa_kwargs) if dsa_kwargs else None mtp_config = None if mtp_nextn > 0: @@ -3882,6 +3920,12 @@ def test_nvfp4_4gpus_online_eplb(self, moe_backend, mtp_nextn): mtp_nextn=mtp_nextn) +_DEEPSEEK_V4_GSM8K_SYSTEM_PROMPT = ( + "Solve the problem carefully. End your response with a final line exactly " + "in the form #### , using the simplest numeric form without units " + "or trailing zeros.") + + @pytest.mark.timeout(14400) @pytest.mark.skip_less_device(8) @pytest.mark.skip_less_device_memory(140000) @@ -3912,6 +3956,52 @@ def test_gsm8k_full_accuracy(self): f"{acc_params.ref_accuracy:.3f}") +@pytest.mark.timeout(14400) +@pytest.mark.skip_less_device_memory(140000) +@skip_pre_blackwell +class TestDeepSeekV4ProDSpark(LlmapiAccuracyTestHarness): + MODEL_NAME = "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PATH = f"{llm_models_root()}/DeepSeek-V4-Pro-DSpark" + EXTRA_EVALUATOR_KWARGS = dict( + apply_chat_template=True, + system_prompt=_DEEPSEEK_V4_GSM8K_SYSTEM_PROMPT, + ) + + @pytest.mark.skip_less_mpi_world_size(8) + def test_gsm8k_dep8_megamoe_deepgemm(self): + kv_cache_config = KvCacheConfig(enable_block_reuse=False, + free_gpu_memory_fraction=0.5) + spec_config = DSparkDecodingConfig(max_draft_len=5, + speculative_model=self.MODEL_PATH) + with LLM(self.MODEL_PATH, + attn_backend="TRTLLM", + tensor_parallel_size=8, + moe_expert_parallel_size=8, + enable_attention_dp=True, + moe_config=MoeConfig(backend="MEGAMOE_DEEPGEMM"), + max_batch_size=DEEPSEEKV4_TEST_MAX_BATCH_SIZE, + max_seq_len=4096, + max_num_tokens=4096, + kv_cache_config=kv_cache_config, + enable_chunked_prefill=False, + disable_overlap_scheduler=True, + custom_tokenizer="deepseek_v4", + speculative_config=spec_config) as llm: + task = GSM8K(self.MODEL_NAME) + acc_params = task.get_hypothesis_testing_params( + dtype=llm.args.dtype, + quant_algo=llm.args.quant_config.quant_algo, + kv_cache_quant_algo=llm.args.quant_config.kv_cache_quant_algo, + spec_dec_algo=llm.args.speculative_config.decoding_type) + assert acc_params.num_samples == GSM8K.NUM_SAMPLES + with mock.patch.dict(os.environ, {"INTEGRATION_TEST": "0"}): + score = task.evaluate( + llm, extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + assert score >= acc_params.ref_accuracy, ( + f"GSM8K accuracy {score:.3f} is below recorded reference " + f"{acc_params.ref_accuracy:.3f}") + + @pytest.mark.timeout(14400) @pytest.mark.skip_less_device_memory(140000) @skip_pre_blackwell @@ -4305,6 +4395,19 @@ def test_auto_dtype(self): task.evaluate(llm, extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + @pytest.mark.skip( + reason="TP2 hangs in the AUTO custom allreduce on PCIe-only (all-SYS " + "topology) nodes; the LMHead AllReduce ignores allreduce_strategy and " + "always takes the AUTO path (tunable_allreduce), so the strategy knob " + "cannot work around it. Unskip once validated on an NVLink platform " + "or the lm_head strategy plumbing is fixed.") + @pytest.mark.skip_less_device(2) + def test_tp2(self): + with LLM(self.MODEL_PATH, tensor_parallel_size=2) as llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + class TestQwen3_4B(LlmapiAccuracyTestHarness): MODEL_NAME = "Qwen3/Qwen3-4B" @@ -4586,6 +4689,12 @@ def test_fp8(self, tp_size, ep_size, torch_compile): pytorch_config = dict(cuda_graph_config=CudaGraphConfig( max_batch_size=32, enable_padding=True), torch_compile_config=torch_compile_config) + # Piecewise CUDA-graph capture (torch_compile=True) needs extra + # workspace beyond what the default KV-cache fraction reserves on + # 44 GiB L40S; lower the fraction in that case to avoid executor OOM. + if torch_compile: + pytorch_config["kv_cache_config"] = KvCacheConfig( + free_gpu_memory_fraction=0.6) with LLM(f"{llm_models_root()}/Qwen3/saved_models_Qwen3-30B-A3B_fp8_hf", tensor_parallel_size=tp_size, @@ -5822,7 +5931,7 @@ def test_nvfp4(self, moe_backend, tp_size, ep_size, attention_dp, kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.6, enable_block_reuse=enable_block_reuse) if enable_block_reuse: - kv_cache_config.mamba_state_cache_interval = 256 + kv_cache_config.mamba_state_config.periodic_snapshot_interval = 256 pytorch_config = dict(disable_overlap_scheduler=False, cuda_graph_config=CudaGraphConfig( max_batch_size=512, enable_padding=True)) @@ -6001,7 +6110,7 @@ def _run_dflash_accuracy(self, task.evaluate(llm, extra_evaluator_kwargs=self.GSM8K_EVALUATOR_KWARGS) - @pytest.mark.skip_less_device_memory(80000) + @pytest.mark.skip_less_device_memory(90000) def test_bf16_dflash(self): self._run_dflash_accuracy(self.MODEL_PATH, self.DFLASH_MODEL_PATH) @@ -6122,10 +6231,10 @@ def test_bf16(self, moe_backend, tp_size, mocker): if moe_backend == "TRTLLM" and get_sm_version() not in (100, 103): pytest.skip(f"{moe_backend} backend supports SM 100 and 103 only") - kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.8, + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.75, enable_block_reuse=False) cuda_graph_config = CudaGraphConfig(enable_padding=True, - max_batch_size=256) + max_batch_size=128) moe_config = MoeConfig(backend=moe_backend) # H20 path produces a small but fluctuating gap relative to H100/H200 BF16 MoE config. @@ -6139,7 +6248,7 @@ def test_bf16(self, moe_backend, tp_size, mocker): tensor_parallel_size=tp_size, moe_expert_parallel_size=1, max_seq_len=4096, - max_batch_size=256, + max_batch_size=128, enable_chunked_prefill=True, kv_cache_config=kv_cache_config, cuda_graph_config=cuda_graph_config, @@ -6188,6 +6297,9 @@ def test_fp8(self, enable_block_reuse, mocker): kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.8, enable_block_reuse=enable_block_reuse) + if enable_block_reuse: + kv_cache_config.avg_seq_len = 2048 + kv_cache_config.mamba_state_config.periodic_snapshot_interval = 256 # DeepGEMM MoE kernels only support datacenter Blackwell (SM100/SM103). # Fall back to the CUTLASS MoE backend (which supports FP8 block scales) # on other architectures such as Hopper (SM90) and consumer Blackwell @@ -6341,6 +6453,9 @@ def test_nvfp4(self, tp_size, ep_size, attention_dp, moe_backend, kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.9, enable_block_reuse=enable_block_reuse) + if enable_block_reuse: + kv_cache_config.avg_seq_len = 2048 + kv_cache_config.mamba_state_config.periodic_snapshot_interval = 256 cuda_graph_config = CudaGraphConfig(max_batch_size=256, enable_padding=True) @@ -6363,6 +6478,42 @@ def test_nvfp4(self, tp_size, ep_size, attention_dp, moe_backend, task.evaluate(llm, extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + @pytest.mark.skip_less_device(4) + def test_nvfp4_mtp3_gdn_replay_tep4(self, mocker): + model_path = f"{llm_models_root()}/Qwen3.5-397B-A17B-NVFP4" + if not os.path.exists(model_path): + pytest.skip(f"Model directory {model_path} does not exist") + + replay_env = {"TRTLLM_USE_GDN_REPLAY": "1"} + mocker.patch.dict(os.environ, replay_env) + + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.8, + enable_block_reuse=False, + dtype="fp8", + mamba_ssm_cache_dtype="bfloat16") + cuda_graph_config = CudaGraphConfig(enable_padding=True, + max_batch_size=128) + mtp_config = MTPDecodingConfig(max_draft_len=3) + + with LLM(model_path, + trust_remote_code=True, + tensor_parallel_size=4, + pipeline_parallel_size=1, + moe_expert_parallel_size=4, + max_num_tokens=16384, + max_batch_size=128, + kv_cache_config=kv_cache_config, + cuda_graph_config=cuda_graph_config, + moe_config=MoeConfig(backend="TRTLLM"), + enable_attention_dp=False, + speculative_config=mtp_config) as llm: + assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 + mocker.patch.object(GSM8K, "MAX_OUTPUT_LEN", + self.GSM8K_MAX_OUTPUT_LEN) + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + def _run_nvfp4_4gpus_eplb(self, eplb_config, moe_backend, mocker): model_path = f"{llm_models_root()}/Qwen3.5-397B-A17B-NVFP4" if not os.path.exists(model_path): @@ -6895,6 +7046,46 @@ def test_nvfp4_marlin_multi_gpus(self, tp_size): task.evaluate(llm, extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + @skip_pre_hopper + @skip_post_hopper + @pytest.mark.skip_less_device_memory(80000) + @pytest.mark.skip_less_mpi_world_size(4) + @parametrize_with_ids("mtp_nextn", [3]) + def test_nvfp4_marlin_adp_4gpus(self, mtp_nextn): + """Accuracy guard for MARLIN under attention DP + EP with MTP. + + The NVFP4 checkpoint is MIXED_PRECISION with deliberately-unquantized + MTP draft layers, so this also guards the per-layer + MARLIN -> Cutlass fallback in ``create_moe.get_moe_cls`` while the + main-model expert layers stay on MARLIN. Attention DP exercises the + external-comm dispatch path with scheduler-precomputed routing. + """ + model_path = f"{llm_models_root()}/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" + mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn, + mtp_eagle_one_model=True) + with LLM(model_path, + tensor_parallel_size=4, + moe_expert_parallel_size=4, + enable_attention_dp=True, + moe_config=MoeConfig(backend="MARLIN"), + kv_cache_config=KvCacheConfig( + enable_block_reuse=False, + mamba_ssm_cache_dtype="float16", + free_gpu_memory_fraction=0.8, + ), + max_batch_size=32, + cuda_graph_config=CudaGraphConfig(max_batch_size=32, + enable_padding=True), + speculative_config=mtp_config, + nvfp4_gemm_config={"allowed_backends": ["marlin"]}) as llm: + assert llm.args.quant_config.quant_algo == QuantAlgo.MIXED_PRECISION + task = MMLU(self.MODEL_NAME) + task.evaluate(llm, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + @skip_pre_hopper @pytest.mark.skip_less_mpi_world_size(4) @pytest.mark.skip_less_device_memory(40000) @@ -6949,7 +7140,7 @@ def test_fp8_4gpus(self, attention_dp, use_cpp_mamba, monkeypatch): @skip_pre_blackwell @pytest.mark.parametrize( - "tp_size, ep_size, mamba_state_cache_interval, attention_dp, use_mtp", + "tp_size, ep_size, periodic_snapshot_interval, attention_dp, use_mtp", [ (1, 1, 256, False, False), (4, 1, 256, False, True), @@ -6960,7 +7151,7 @@ def test_fp8_4gpus(self, attention_dp, use_cpp_mamba, monkeypatch): ids=["TP1", "TP4_MTP", "TEP4", "DEP4_MTP_OFF", "DEP4_MTP_ON"], ) def test_nvfp4_4gpus_block_reuse(self, tp_size, ep_size, - mamba_state_cache_interval, attention_dp, + periodic_snapshot_interval, attention_dp, use_mtp): gpu_needed = max(tp_size, ep_size) if get_device_count() < gpu_needed: @@ -6975,9 +7166,11 @@ def test_nvfp4_4gpus_block_reuse(self, tp_size, ep_size, f"{llm_models_root()}/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", kv_cache_config=KvCacheConfig( enable_block_reuse=True, + avg_seq_len=2048, mamba_ssm_cache_dtype="float16", - mamba_state_cache_interval=mamba_state_cache_interval, - free_gpu_memory_fraction=0.8, + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=periodic_snapshot_interval), + free_gpu_memory_fraction=0.7, ), max_batch_size=32, tensor_parallel_size=tp_size, @@ -7049,7 +7242,10 @@ def test_nvfp4_8gpus_mtp(self): model_path, kv_cache_config=KvCacheConfig( enable_block_reuse=True, + avg_seq_len=2048, mamba_ssm_cache_dtype="float16", + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=256), free_gpu_memory_fraction=0.5, ), max_batch_size=32, @@ -7382,7 +7578,7 @@ def test_nvfp4_8gpus(self, attention_dp, moe_backend): @skip_pre_blackwell @pytest.mark.skip_less_mpi_world_size(4) @pytest.mark.parametrize( - "tp_size, ep_size, mamba_state_cache_interval, attention_dp, use_mtp", + "tp_size, ep_size, periodic_snapshot_interval, attention_dp, use_mtp", [ (4, 1, 256, False, True), (4, 4, 256, False, False), @@ -7392,7 +7588,7 @@ def test_nvfp4_8gpus(self, attention_dp, moe_backend): ids=["TP4_MTP", "TEP4", "ADP4", "ADP4_MTP"], ) def test_nvfp4_4gpus_block_reuse(self, tp_size, ep_size, - mamba_state_cache_interval, attention_dp, + periodic_snapshot_interval, attention_dp, use_mtp): mtp_config = MTPDecodingConfig( num_nextn_predict_layers=3, @@ -7403,8 +7599,10 @@ def test_nvfp4_4gpus_block_reuse(self, tp_size, ep_size, f"{llm_models_root()}/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", kv_cache_config=KvCacheConfig( enable_block_reuse=True, + avg_seq_len=2048, mamba_ssm_cache_dtype="float16", - mamba_state_cache_interval=mamba_state_cache_interval, + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=periodic_snapshot_interval), free_gpu_memory_fraction=0.5, ), max_batch_size=max_batch_size, @@ -7570,11 +7768,12 @@ def test_auto_dtype(self, tp_size, ep_size): @pytest.mark.skip_less_device(4) @pytest.mark.skip_less_device_memory(140000) - @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) - def test_mxfp8(self, tp_size, ep_size): + @parametrize_with_ids("use_msa", [False, True]) + def test_mxfp8(self, use_msa): # MXFP8 checkpoint: weights are MXFP8 (e4m3 + UE8M0 1x32 block # scales) with MXFP8 dynamic activations; the KV cache stays in - # BF16 and the sparse attention path is unchanged from BF16. + # BF16. + tp_size = ep_size = 4 model_name = "MiniMaxAI/MiniMax-M3-MXFP8" model_path = f"{llm_models_root()}/MiniMax-M3-MXFP8" # Halving TP from the BF16 reference (TP=8) doubles per-rank @@ -7583,7 +7782,8 @@ def test_mxfp8(self, tp_size, ep_size): # under the PyTorch cap. kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.4, enable_block_reuse=False) - sparse_attention_config = MiniMaxM3SparseAttentionConfig() + sparse_attention_config = MiniMaxM3SparseAttentionConfig( + implementation="msa" if use_msa else "triton") with LLM(model_path, tensor_parallel_size=tp_size, moe_expert_parallel_size=ep_size, @@ -7599,15 +7799,18 @@ def test_mxfp8(self, tp_size, ep_size): task = GSM8K(model_name) task.evaluate(llm) - @pytest.mark.skip_less_device(8) + @pytest.mark.skip_less_device(4) @pytest.mark.skip_less_device_memory(140000) - @parametrize_with_ids("tp_size,ep_size", [(8, 8)]) - def test_mxfp8_piecewise_cuda_graph(self, tp_size, ep_size): + @parametrize_with_ids("use_msa", [False, True]) + def test_mxfp8_piecewise_cuda_graph(self, use_msa): + tp_size = ep_size = 4 model_name = "MiniMaxAI/MiniMax-M3-MXFP8" model_path = f"{llm_models_root()}/MiniMax-M3-MXFP8" kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.5, - enable_block_reuse=False) - sparse_attention_config = MiniMaxM3SparseAttentionConfig() + enable_block_reuse=False, + dtype="fp8" if use_msa else "auto") + sparse_attention_config = MiniMaxM3SparseAttentionConfig( + implementation="msa" if use_msa else "triton") cuda_graph_config = CudaGraphConfig( enable_padding=True, batch_sizes=[1, 2, 4, 8, 12, 16, 24, 32]) torch_compile_config = TorchCompileConfig( @@ -7634,16 +7837,19 @@ def test_mxfp8_piecewise_cuda_graph(self, tp_size, ep_size): @pytest.mark.skip_less_device(4) @pytest.mark.skip_less_device_memory(140000) - @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) - def test_nvfp4(self, tp_size, ep_size): + @parametrize_with_ids("use_msa", [False, True]) + def test_nvfp4(self, use_msa): # NVFP4 checkpoint: MXFP8 base layers with NVFP4 routed experts - # (MIXED_PRECISION checkpoint); the KV cache stays in BF16 and the - # sparse attention path is unchanged from BF16. + # (MIXED_PRECISION checkpoint). The MSA path runs an FP8 KV cache; the + # Triton path keeps the KV cache in BF16. + tp_size = ep_size = 4 model_name = "nvidia/MiniMax-M3-NVFP4" model_path = f"{llm_models_root()}/MiniMax-M3-NVFP4" kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.6, - enable_block_reuse=False) - sparse_attention_config = MiniMaxM3SparseAttentionConfig() + enable_block_reuse=False, + dtype="fp8" if use_msa else "auto") + sparse_attention_config = MiniMaxM3SparseAttentionConfig( + implementation="msa" if use_msa else "triton") moe_config = MoeConfig(backend="CUTLASS") with LLM(model_path, tensor_parallel_size=tp_size, diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 91bd03c3df11..3cd3c1c88868 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -492,7 +492,11 @@ def test_nvfp4_4gpus( mocker, ): mocker.patch.dict( - MMMU.EVALUATE_KWARGS, {"model_type": "mistral_large_3", "is_force_single_image": True} + MMMU.EVALUATE_KWARGS, + { + "model_type": "mistral_common", + "is_force_single_image": True, + }, ) pytorch_config = dict( disable_overlap_scheduler=not overlap_scheduler, diff --git a/tests/integration/defs/common.py b/tests/integration/defs/common.py index e40db78ac36c..f6ddb0d5c7fc 100644 --- a/tests/integration/defs/common.py +++ b/tests/integration/defs/common.py @@ -21,7 +21,6 @@ import tempfile import time from difflib import SequenceMatcher -from pathlib import Path from typing import Any import yaml @@ -33,8 +32,8 @@ from tensorrt_llm.lora_manager import LoraConfig from tensorrt_llm.sampling_params import SamplingParams -from .trt_test_alternative import (check_call, check_output, exists, is_windows, - print_info, print_warning) +from .trt_test_alternative import (check_call, check_output, print_info, + print_warning) def venv_check_call(venv, cmd, env=None, **kwargs): @@ -121,428 +120,6 @@ def parse_mpi_cmd(cmd): return cmd -class PluginOptions: - - def __init__(self, - gpt_attention: str = None, - bert_attention: str = None, - gemm: str = None, - layernorm: str = None): - self.gpt_attention = gpt_attention - self.bert_attention = bert_attention - self.gemm = gemm - - def to_legacy_args(self): - args = [] - if self.gpt_attention is not None: - args.extend(["--use_gpt_attention_plugin", self.gpt_attention]) - if self.bert_attention is not None: - args.extend(["--use_bert_attention_plugin", self.bert_attention]) - if self.gemm is not None: - args.extend(["--use_gemm_plugin", self.gemm]) - return args - - def to_args(self): - args = [] - if self.gpt_attention is not None: - args.extend(["--gpt_attention_plugin", self.gpt_attention]) - else: - args.extend(["--gpt_attention_plugin", "disable"]) - if self.bert_attention is not None: - args.extend(["--bert_attention_plugin", self.bert_attention]) - else: - args.extend(["--bert_attention_plugin", "disable"]) - if self.gemm is not None: - args.extend(["--gemm_plugin", self.gemm]) - else: - args.extend(["--gemm_plugin", "disable"]) - return args - - -def prune_checkpoint(llm_venv, checkpoint_dir): - pruned_checkpoint_dir = checkpoint_dir + ".pruned" - prune_cmd = [ - "trtllm-prune", f"--checkpoint_dir={checkpoint_dir}", - f"--out_dir={pruned_checkpoint_dir}" - ] - - check_call(" ".join(prune_cmd), shell=True, env=llm_venv._new_env) - return pruned_checkpoint_dir - - -def refit_model(llm_venv, engine_dir, unpruned_model_dir): - refit_engine_dir = f"{engine_dir}_refit_full" - refit_cmd = [ - "trtllm-refit", f"--checkpoint_dir={unpruned_model_dir}", - f"--engine_dir {engine_dir}", f"--output_dir {refit_engine_dir}" - ] - - check_call(" ".join(refit_cmd), shell=True, env=llm_venv._new_env) - return refit_engine_dir - - -def convert_weights(llm_venv, - example_root, - cmodel_dir, - model, - model_path, - quant_ckpt_path=None, - data_type="float16", - gpus=1, - tp_size=None, - pp_size=None, - model_type=None, - use_parallel_embedding=False, - embedding_sharding_dim=0, - load_by_shard=False, - int8_kv_cache=False, - use_weight_only=False, - workers=1, - processes=None, - smoothquant=0, - per_channel=False, - per_token=False, - fp8_kv_cache=False, - enable_fp8=False, - weight_only_precision=None, - per_group=False, - batch_size=8, - multimodal=False, - ckpt_type='hf', - load_model_on_cpu=False, - **kwargs): - "Convert weights from HF transformers format to FT format" - converted_model_path = os.path.join(cmodel_dir, model, data_type) - script = "convert_checkpoint.py" - - tp_size = gpus if tp_size is None else tp_size - pp_size = gpus // tp_size if pp_size is None else pp_size - gpus = tp_size * pp_size - model_dir = f'{converted_model_path}/{gpus}-gpu' - - # TODO: add other models command - if "gpt2" in model: - script = "convert_checkpoint.py" - convert_cmd = [ - f"{example_root}/{script}", f"--output_dir={model_dir}", - f"--dtype={data_type}", f"--tp_size={tp_size}", - f"--pp_size={pp_size}" - ] - if "next" in model: - convert_cmd.extend(["--nemo_ckpt_path", model_path]) - else: - convert_cmd.extend(["--model_dir", model_path]) - if "smooth" in model: - convert_cmd.extend(["--smoothquant", "0.5"]) - if "kv" in model and "int8" in model: - convert_cmd.append("--int8_kv_cache") - - elif "t5" in model or "bart" in model or "ul2" in model or "wmt" in model or "nougat" in model or 'pix2struct' in model: - assert model_type, "Encoder-Decoder models must specify model architecture type" - script = "convert_checkpoint.py" - convert_cmd = [ - f"{example_root}/{script}", "--model_dir", model_path, - "--output_dir", converted_model_path, f"--model_type={model_type}", - f"--tp_size={tp_size}", f"--pp_size={pp_size}", - f"--dtype={data_type}" - ] - if "nougat" in model: - convert_cmd.append("--nougat") - - model_dir = converted_model_path - - elif "opt" in model and model_type == "blip2": - convert_cmd = [ - f"{example_root}/{script}", - f"--model_dir={model_path}", - f"--output_dir={model_dir}", - f"--model_type={model_type}", - f"--dtype={data_type}", - f"--tp_size={tp_size}", - f"--pp_size={pp_size}", - ] - - elif "whisper" in model_path: - script = "convert_checkpoint.py" - convert_cmd = [ - f"{example_root}/{script}", "--model_dir", model_path, - "--output_dir", converted_model_path - ] - model_dir = converted_model_path - - elif "mamba" in model: - convert_cmd = [ - f"{example_root}/{script}", "--model_dir", model_path, - "--output_dir", model_dir, f"--dtype={data_type}", - f"--tp_size={tp_size}" - ] - - elif "llama" in model or "llava" in model or "vila" in model: - convert_cmd = [ - f"{example_root}/{script}", "--output_dir", model_dir, - f"--dtype={data_type}", f"--tp_size={tp_size}", - f"--pp_size={pp_size}" - ] - - if 'meta-ckpt' in model: - convert_cmd.extend(['--meta_ckpt_dir', model_path]) - else: - convert_cmd.extend(['--model_dir', model_path]) - - if 'code_llama_1gpu' in model: - convert_cmd.extend(['--rotary_base=1000000']) - convert_cmd.extend(['--vocab_size=32016']) - elif 'code_llama' in model: - convert_cmd.extend(['--rotary_base=1000000']) - convert_cmd.extend(['--vocab_size=32000']) - if 'int4_gptq' in model: - convert_cmd.extend([ - "--use_weight_only", "--weight_only_precision=int4_gptq", - f"--quant_ckpt_path={quant_ckpt_path}", "--per_group" - ]) - if 'int8_gptq' in model: - convert_cmd.extend([ - "--use_weight_only", "--weight_only_precision=int8_gptq", - f"--quant_ckpt_path={quant_ckpt_path}", "--per_group", - "--group_size=64" - ]) - - if 'awq' in model: - convert_cmd.extend([ - "--use_weight_only", "--weight_only_precision=int4_awq", - "--group_size=128" - ]) - if 'finegrained_fp8' in model: - convert_cmd.extend(["--use_fp8"]) - - elif "draft_target_model" in model: - if "gpt" in model_path: - example_name = "gpt" - elif "llama" in model_path: - example_name = "llama" - script = f"{example_root}/../models/core/{example_name}/convert_checkpoint.py" - convert_cmd = [ - f"{script}", - "--model_dir", - model_path, - "--output_dir", - model_dir, - f"--dtype={data_type}", - ] - - elif "ngram" in model: - if "gpt" in model_path: - example_name = "gpt" - elif "llama" in model_path: - example_name = "llama" - script = f"{example_root}/../models/core/{example_name}/convert_checkpoint.py" - convert_cmd = [ - f"{script}", - "--model_dir", - model_path, - "--output_dir", - model_dir, - f"--dtype={data_type}", - ] - - elif "medusa" in model: - convert_cmd = [ - f"{example_root}/{script}", "--model_dir", model_path[0], - "--medusa_model_dir", model_path[1], "--output_dir", model_dir, - f"--dtype={data_type}", f"--tp_size={tp_size}", - f"--pp_size={pp_size}", "--num_medusa_heads=4" - ] - elif "redrafter" in model: - redrafter_num_beams = kwargs.pop("redrafter_num_beams") - redrafter_draft_len_per_beam = kwargs.pop( - "redrafter_draft_len_per_beam") - convert_cmd = [ - f"{example_root}/{script}", "--base_model_checkpoint_dir", - model_path[0], "--drafter_model_dir", model_path[1], "--output_dir", - model_dir, f"--dtype={data_type}", f"--tp_size={tp_size}", - f"--redrafter_num_beams={redrafter_num_beams}", - f"--redrafter_draft_len_per_beam={redrafter_draft_len_per_beam}" - ] - elif "eagle" in model: - if len(model_path) == 2: - # Test the checkpoint released from HF, which requires two separate weights, - # one for the base model and one for the EagleNets. - convert_cmd = [ - f"{example_root}/{script}", "--model_dir", model_path[0], - "--eagle_model_dir", model_path[1], "--output_dir", model_dir, - f"--dtype={data_type}", f"--tp_size={tp_size}", - f"--pp_size={pp_size}", "--num_eagle_layers=4", - "--max_draft_len=63", "--max_non_leaves_per_layer=10" - ] - else: - # Test the checkpoint released from ModelOpt, which only requires one weight, - # which includes both the base model and EagleNets, and is an FP8 datatype. - convert_cmd = [ - f"{example_root}/{script}", "--model_dir", model_path, - "--output_dir", model_dir, f"--dtype={data_type}", - f"--tp_size={tp_size}", f"--pp_size={pp_size}", - "--num_eagle_layers=4", "--max_draft_len=63", - "--max_non_leaves_per_layer=10" - ] - elif "recurrentgemma" in model: - convert_cmd = [ - f"{example_root}/{script}", "--model_dir", model_path, - "--output_dir", model_dir, f"--dtype={data_type}", - f"--world_size={tp_size}", f"--ckpt_type={ckpt_type}" - ] - elif "cogvlm" in model: - convert_cmd = [ - f"{example_root}/{script}", "--model_dir", model_path, - "--output_dir", model_dir, f"--dtype={data_type}", - f"--tp_size={tp_size}", f"--pp_size={pp_size}", - "--use_prompt_tuning" - ] - elif "fuyu" in model or "kosmos" in model: - gpt_variant = "kosmos-2" if "kosmos" in model else "persimmon" - convert_cmd = [ - f"{example_root}/{script}", "--model_dir", model_path, - "--output_dir", model_dir, "--dtype", data_type, "--gpt_variant", - gpt_variant - ] - elif "neva-22b" in model: - convert_cmd = [ - f"{example_root}/{script}", "--nemo_ckpt_path", model_path, - "--output_dir", model_dir, "--dtype", data_type, - "--nemo_rename_key", "model:model.language_model", - "attention.linear_qkv.layer_norm_bias:input_layernorm.bias", - "attention.linear_qkv.layer_norm_weight:input_layernorm.weight", - "mlp.linear_fc1.layer_norm_bias:post_attention_layernorm.bias", - "mlp.linear_fc1.layer_norm_weight:post_attention_layernorm.weight", - "linear_qkv:query_key_value", "linear_fc1:dense_h_to_4h", - "linear_fc2:dense_4h_to_h", "linear_proj:dense", "decoder:encoder" - ] - elif "video-neva" in model: - - nemotron_root = os.path.join(example_root, "../", "nemotron") - - if llm_venv: - # Install Python requirements for nemotron - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(nemotron_root, "requirements.txt") - ]) - - qformat = 'full_prec' - model_name = 'nemotron-video-neva' - converted_model_path = os.path.join(cmodel_dir, model_name, qformat) - model_dir = f'{converted_model_path}/{gpus}-gpu' - # Overwrite the model_path with the nemotron model path - model_path = os.path.join(os.path.dirname(os.path.dirname(model_path)), - 'nemotron', 'Nemotron-4-15B-SteerLM.nemo') - convert_cmd = [ - f"{example_root}/../quantization/quantize.py", - f"--nemo_ckpt_path={model_path}", - "--batch_size=64", - f"--dtype={data_type}", - f"--qformat={qformat}", - f"--output_dir={model_dir}", - ] - elif "dit-xl" in model.lower(): - convert_cmd = [ - f"{example_root}/{script}", - f"--timm_ckpt={model_path}", - f"--output_dir={model_dir}", - f"--dtype={data_type}", - f"--tp_size={tp_size}", - f"--pp_size={pp_size}", - ] - if kwargs.get("enable_fp8_linear") is not None: - convert_cmd.append("--fp8_linear") - elif "stdit" in model.lower(): - convert_cmd = [ - f"{example_root}/{script}", - f"--timm_ckpt={model_path}/model.safetensors", - f"--output_dir={model_dir}", - f"--dtype={data_type}", - f"--tp_size={tp_size}", - f"--pp_size={pp_size}", - ] - elif "bert" in model.lower(): - convert_cmd = [ - f"{example_root}/{script}", - f"--model={model}", - f"--model_dir={model_path}", - f"--output_dir={model_dir}", - f"--dtype={data_type}", - f"--tp_size={tp_size}", - ] - elif "granite" in model.lower(): - convert_cmd = [ - f"{example_root}/{script}", - f"--model_dir={model_path}", - f"--output_dir={model_dir}", - f"--dtype={data_type}", - f"--tp_size={tp_size}", - ] - elif "stable-diffusion-3.5" in model.lower(): - convert_cmd = [ - f"{example_root}/{script}", - f"--model_path={model_path}", - f"--output_dir={model_dir}", - f"--tp_size={tp_size}", - ] - else: - convert_cmd = [ - f"{example_root}/{script}", - f"--model_dir={model_path}", - f"--output_dir={model_dir}", - f"--dtype={data_type}", - f"--tp_size={tp_size}", - f"--pp_size={pp_size}", - ] - - if use_parallel_embedding: - convert_cmd.append("--use_parallel_embedding") - convert_cmd.append(f"--embedding_sharding_dim={embedding_sharding_dim}") - if load_by_shard: - convert_cmd.extend(["--load_by_shard"]) - if load_model_on_cpu: - convert_cmd.extend(["--load_model_on_cpu"]) - if workers > 1: - convert_cmd.extend([f"--workers={workers}"]) - if int8_kv_cache: - convert_cmd.append("--int8_kv_cache") - if use_weight_only: - convert_cmd.append("--use_weight_only") - if weight_only_precision: - convert_cmd.append(f"--weight_only_precision={weight_only_precision}") - if processes is not None: - convert_cmd.append(f"--processes={processes}") - if smoothquant > 0: - convert_cmd.append(f"--smoothquant={smoothquant}") - if per_channel: - convert_cmd.append("--per_channel") - if per_token: - convert_cmd.append("--per_token") - if enable_fp8: - convert_cmd.append('--enable_fp8') - if fp8_kv_cache: - convert_cmd.append('--fp8_kv_cache') - if quant_ckpt_path: - convert_cmd.append(f"--quant_ckpt_path={quant_ckpt_path}") - if per_group: - convert_cmd.append("--per_group") - timeout = kwargs.pop('timeout', None) - - for key, value in kwargs.items(): - if isinstance(value, bool): - if value: - convert_cmd.append(f"--{key}") - else: - convert_cmd.extend([f"--{key}={value}"]) - - if llm_venv: - venv_check_call(llm_venv, convert_cmd, timeout=timeout) - return model_dir - else: - return convert_cmd, model_dir - - def similarity_score(a, b): "similar compare a and b " return SequenceMatcher(None, a, b).ratio() @@ -573,96 +150,6 @@ def generate_summary_cmd(example_root, *args, **kwargs): return summary_cmd -def generate_deterministic_cmd(example_root, *args, **kwargs): - "generate deterministic command" - deterministic_cmd = [ - f"{example_root}/mixtral_deterministic.py", - "--check_deterministic_accuracy" - ] - - for key, value in kwargs.items(): - if isinstance(value, bool): - if value: - deterministic_cmd.extend(f"--{key}") - else: - deterministic_cmd.extend([f"--{key}", f"{value}"]) - - for arg in args: - deterministic_cmd.append(f"--{arg}") - - return deterministic_cmd - - -def quantize_data(llm_venv, - example_root, - model_dir, - dtype, - quantize_dir, - qformat="full_prec", - tp_size=1, - pp_size=1, - cp_size=1, - calib_size=512, - kv_cache_dtype=None, - **kwargs): - "quanize data and return data dir" - model_name = os.path.basename(model_dir) - output_dir = os.path.join(quantize_dir, model_name, dtype, qformat, - f"tp{tp_size}pp{pp_size}") - if kv_cache_dtype: - output_dir = os.path.join(output_dir, kv_cache_dtype) - else: - output_dir = os.path.join(output_dir, "no_kv_cache") - - quantize_script = f"{example_root}/../../../quantization/quantize.py" if "core" in example_root else f"{example_root}/../quantization/quantize.py" - quantize_cmd = [ - quantize_script, - f"--model_dir={model_dir}", - f"--dtype={dtype}", - f"--qformat={qformat}", - f"--output_dir={output_dir}", - f"--tp_size={tp_size}", - f"--pp_size={pp_size}", - f"--cp_size={cp_size}", - f"--calib_size={calib_size}", - ] - - if kv_cache_dtype: - quantize_cmd.append(f"--kv_cache_dtype={kv_cache_dtype}") - timeout = kwargs.pop('timeout', None) - - for key, value in kwargs.items(): - if isinstance(value, bool): - if value: - quantize_cmd.append(f"--{key}") - else: - quantize_cmd.extend([f"--{key}", f"{value}"]) - - if llm_venv: - if not exists(output_dir): - venv_check_call(llm_venv, quantize_cmd, timeout=timeout) - return output_dir - else: - return quantize_cmd, output_dir - - -def find_tensorrt(ld_library_path): - MAX_SEARCH_HEIGHT = 10 - ld_library_path = ld_library_path.split(os.pathsep) - for trt_lib_dir in ld_library_path: - trt_lib_dir = Path(trt_lib_dir) - trt_nvinfer_lib = trt_lib_dir / "libnvinfer.so" - if trt_nvinfer_lib.exists(): - trt_root_dir = trt_lib_dir - for i in range(MAX_SEARCH_HEIGHT): - trt_root_dir = trt_root_dir.parent - trt_include_dir = trt_root_dir / "include" - trt_nvinfer_header = trt_include_dir / "NvInfer.h" - if trt_nvinfer_header.exists(): - return str(trt_include_dir), str(trt_lib_dir) - return None, None - - def get_trt_llm_lib_dir(venv): output = venv.run_raw( "import tensorrt_llm; print(f'{tensorrt_llm.__path__[0]}/libs')", @@ -729,24 +216,6 @@ def run_and_check(llm_venv, run_cmd, valid_outputs, streaming=False): ]), f"output is: {output}" -def get_cpp_benchmark(cpp_benchmark_name, llm_root): - suffix = ".exe" if is_windows() else "" - cpp_benchmark_name += suffix - # In CI/CD, we copy the cpp binary into the same folder as cpp to avoid package sanity - ci_path = os.path.join(os.path.dirname(os.path.realpath(llm_root)), - "benchmarks", "cpp", cpp_benchmark_name) - if os.path.exists(ci_path): - return ci_path - # In QA, we keep the benchmark build at its original location - qa_path = os.path.join(llm_root, "cpp", "build", "benchmarks", - cpp_benchmark_name) - if os.path.exists(qa_path): - return qa_path - raise Exception( - f"Cannot find cpp benchmark binary in either {ci_path} or {qa_path}. Did you forget --benchmark in building TRT-LLM?" - ) - - def generate_dummy_loras( hf_model_dir, lora_output_dir, diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index bba88cf07c01..fd70a176cc2e 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -39,6 +39,14 @@ import tqdm import yaml from _pytest.mark import ParameterSet +# Dispatched explicitly (not via pytest_plugins, which pytest forbids in a +# non-top-level conftest: a repo-root invocation like `pytest tests` loads +# this file as a NESTED conftest and would fail collection; and not via "-p" +# in pytest.ini addopts, which imports at preparse, before the ini pythonpath +# entries are usable). The wrappers below forward to the plugin; hooks are +# idempotent, so a repo-root run that also dispatches from tests/conftest.py +# is harmless. +from test_common import session_prefetcher_hooks as _prefetch_hooks from tensorrt_llm.bindings import ipc_nvls_supported from tensorrt_llm.llmapi.mpi_session import get_mpi_world_size @@ -63,8 +71,6 @@ # TODO: turn off this when the nightly storage issue is resolved. DEBUG_CI_STORAGE = os.environ.get("DEBUG_CI_STORAGE", False) -GITLAB_API_USER = os.environ.get("GITLAB_API_USER") -GITLAB_API_TOKEN = os.environ.get("GITLAB_API_TOKEN") def _get_s3_output(): @@ -231,15 +237,6 @@ def bert_example_root(llm_root): return example_root -@pytest.fixture(scope="module") -def enc_dec_example_root(llm_root): - "Get encoder-decoder example root" - example_root = os.path.join(llm_root, "examples", "models", "core", - "enc_dec") - - return example_root - - @pytest.fixture(scope="module") def whisper_example_root(llm_root, llm_venv): "Get whisper example root" @@ -252,39 +249,6 @@ def whisper_example_root(llm_root, llm_venv): return example_root -@pytest.fixture(scope="module") -def opt_example_root(llm_root, llm_venv): - "Get opt example root" - - example_root = os.path.join(llm_root, "examples", "models", "contrib", - "opt") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - -@pytest.fixture(scope="module") -def llama_example_root(llm_root, llm_venv): - "Get llama example root" - - example_root = os.path.join(llm_root, "examples", "models", "core", "llama") - try: - llm_venv.run_cmd([ - "-m", - "pip", - "install", - "-r", - os.path.join(example_root, "requirements.txt"), - ]) - except: - print("pip install error!") - - return example_root - - @pytest.fixture(scope="module") def llmapi_example_root(llm_root, llm_venv): "Get llm api example root" @@ -359,41 +323,6 @@ def gpt_example_root(llm_root, llm_venv): return example_root -@pytest.fixture(scope="module") -def gptj_example_root(llm_root, llm_venv): - "Get gptj example root" - example_root = os.path.join(llm_root, "examples", "models", "contrib", - "gptj") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - -@pytest.fixture(scope="module") -def glm_4_9b_example_root(llm_root, llm_venv): - "Get glm-4-9b example root" - example_root = os.path.join(llm_root, "examples", "models", "core", - "glm-4-9b") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - -@pytest.fixture(scope="module") -def exaone_example_root(llm_root, llm_venv): - "Get EXAONE example root" - example_root = os.path.join(llm_root, "examples", "models", "core", - "exaone") - - return example_root - - @pytest.fixture(scope="function") def llm_exaone_model_root(request) -> str: "Get EXAONE model root" @@ -410,26 +339,6 @@ def llm_exaone_model_root(request) -> str: return exaone_model_root -@pytest.fixture(scope="module") -def falcon_example_root(llm_root, llm_venv): - "Get falcon example root" - example_root = os.path.join(llm_root, "examples", "models", "contrib", - "falcon") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - -@pytest.fixture(scope="session") -def plugin_gen_path(llm_root): - "Path to the plugin_gen.py script" - return os.path.join(llm_root, "tensorrt_llm", "tools", "plugin_gen", - "plugin_gen.py") - - @pytest.fixture(scope="module") def internlm2_example_root(llm_root, llm_venv): "Get internlm2 example root" @@ -443,42 +352,6 @@ def internlm2_example_root(llm_root, llm_venv): return example_root -@pytest.fixture(scope="module") -def qwen_example_root(llm_root, llm_venv): - "Get qwen example root" - example_root = os.path.join(llm_root, "examples", "models", "core", "qwen") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - -@pytest.fixture(scope="module") -def draft_target_model_example_root(llm_root, llm_venv): - "Get Draft-Target-Model example root" - example_root = os.path.join(llm_root, "examples", "draft_target_model") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - -@pytest.fixture(scope="module") -def ngram_example_root(llm_root, llm_venv): - "Get NGram example root" - example_root = os.path.join(llm_root, "examples", "ngram") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - @pytest.fixture(scope="module") def medusa_example_root(llm_root, llm_venv): "Get medusa example root" @@ -491,93 +364,6 @@ def medusa_example_root(llm_root, llm_venv): return example_root -@pytest.fixture(scope="module") -def redrafter_example_root(llm_root, llm_venv): - "Get ReDrafter example root" - example_root = os.path.join(llm_root, "examples", "redrafter") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - -@pytest.fixture(scope="module") -def eagle_example_root(llm_root, llm_venv): - "Get EAGLE example root" - example_root = os.path.join(llm_root, "examples", "eagle") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - -@pytest.fixture(scope="module") -def mamba_example_root(llm_root, llm_venv): - "Get mamba example root" - example_root = os.path.join(llm_root, "examples", "models", "core", "mamba") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - yield example_root - - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(llm_root, "requirements.txt") - ]) - - -@pytest.fixture(scope="module") -def nemotron_nas_example_root(llm_root, llm_venv): - example_root = os.path.join(llm_root, "examples", "models", "core", - "nemotron_nas") - - yield example_root - - -@pytest.fixture(scope="module") -def nemotron_example_root(llm_root, llm_venv): - "Get nemotron example root" - example_root = os.path.join(llm_root, "examples", "models", "core", - "nemotron") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - return example_root - - -@pytest.fixture(scope="module") -def commandr_example_root(llm_root, llm_venv): - "Get commandr example root" - example_root = os.path.join(llm_root, "examples", "models", "core", - "commandr") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - -@pytest.fixture(scope="module") -def deepseek_v2_example_root(llm_root, llm_venv): - "Get deepseek v2 example root" - example_root = os.path.join(llm_root, "examples", "models", "contrib", - "deepseek_v2") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - @pytest.fixture(scope="function") def deepseek_v3_model_root(request): models_root = llm_models_root() @@ -676,12 +462,13 @@ def custom_user_workspace(request): @pytest.fixture(scope="session") -def llm_venv(llm_root, custom_user_workspace): +def llm_venv(request, llm_root, custom_user_workspace): workspace_dir = custom_user_workspace subdir = datetime.datetime.now().strftime("ws-%Y-%m-%d-%H-%M-%S") if workspace_dir is None: workspace_dir = "llm-test-workspace" workspace_dir = os.path.join(workspace_dir, subdir) + keep_workspace = request.config.getoption("--keep-workspace", default=False) from defs.local_venv import PythonVenvRunnerImpl venv = PythonVenvRunnerImpl("", "", "python3", @@ -689,11 +476,11 @@ def llm_venv(llm_root, custom_user_workspace): yield venv # Remove the workspace directory if os.path.exists(workspace_dir): - print(f"Cleaning up workspace: {workspace_dir}") - try: - shutil.rmtree(workspace_dir) - except Exception as e: - print(f"Failed to clean up workspace: {e}") + if keep_workspace: + print(f"Keeping workspace (--keep-workspace): {workspace_dir}") + else: + print(f"Cleaning up workspace: {workspace_dir}") + shutil.rmtree(workspace_dir, ignore_errors=True) @pytest.fixture(scope="session") @@ -2074,6 +1861,13 @@ def pytest_addoption(parser): help="Enable GPU clock locking during tests. " "By default, GPU clock locking is disabled.", ) + parser.addoption( + "--keep-workspace", + action="store_true", + default=False, + help= + "Skip workspace cleanup at session end (useful for inspecting logs after a failure).", + ) parser.addoption( "--periodic-save-unfinished-test", action="store_true", @@ -2187,6 +1981,7 @@ def pytest_collection_modifyitems(session, config, items): def pytest_configure(config): + _prefetch_hooks.pytest_configure(config) os.environ.setdefault("TRTLLM_NO_USAGE_STATS", "1") # avoid thread leak of tqdm's TMonitor @@ -2247,8 +2042,6 @@ def pytest_configure(config): "Warning: --periodic-junit requires --output-dir to be set. " "Periodic reporting disabled.") - _get_s3_output().register_plugin(config) - def deselect_by_test_model_suites(test_model_suites, items, test_prefix, config): @@ -2579,15 +2372,6 @@ def pytest_runtest_protocol(item, nextitem): return ret -@pytest.fixture(scope="function") -def deterministic_test_root(llm_root, llm_venv): - "Get deterministic test root" - deterministic_root = os.path.join(llm_root, - "tests/integration/defs/deterministic") - - return deterministic_root - - @pytest.fixture(scope="function") def disaggregated_test_root(llm_root, llm_venv): "Get disaggregated test root" @@ -2658,3 +2442,11 @@ def torch_empty_cache() -> None: gc.collect() torch.cuda.empty_cache() gc.collect() + + +def pytest_runtest_setup(item): + _prefetch_hooks.pytest_runtest_setup(item) + + +def pytest_sessionfinish(session, exitstatus): + _prefetch_hooks.pytest_sessionfinish(session, exitstatus) diff --git a/tests/integration/defs/cpp/conftest.py b/tests/integration/defs/cpp/conftest.py index cdcb8e7237e1..75d79fc463c5 100644 --- a/tests/integration/defs/cpp/conftest.py +++ b/tests/integration/defs/cpp/conftest.py @@ -3,7 +3,6 @@ import pathlib as _pl import shutil import sys as _sys -import time import defs.cpp.cpp_common as _cpp import pytest @@ -168,59 +167,12 @@ def build_google_tests(request, build_type): use_ccache=True, clean=True, generator="Ninja", - trt_root="/usr/local/tensorrt", nixl_root="/opt/nvidia/nvda_nixl", skip_building_wheel=True, extra_make_targets=["google-tests"], ) -@pytest.fixture(scope="session") -def build_benchmarks(build_google_tests, build_dir, build_type): - - make_benchmarks = [ - "cmake", - "--build", - ".", - "--config", - build_type, - "-j", - "--target", - "benchmarks", - ] - - _cpp.run_command(make_benchmarks, cwd=build_dir, timeout=300) - - -@pytest.fixture(scope="session") -def prepare_model( - root_dir, - cpp_resources_dir, - python_exe, - model_cache_arg, - install_additional_requirements, -): - - def _prepare(model_name: str, run_fp8=False): - install_additional_requirements(model_name) - - start_time = time.time() - - _cpp.prepare_model_tests( - model_name=model_name, - python_exe=python_exe, - root_dir=root_dir, - resources_dir=cpp_resources_dir, - model_cache_arg=model_cache_arg, - ) - - duration = time.time() - start_time - print(f"Built model: {model_name}") - print(f"Duration: {duration} seconds") - - return _prepare - - @pytest.fixture(scope="function", autouse=True) def keep_log_files(build_dir): """Backup previous cpp test results when run multiple ctest invocations.""" diff --git a/tests/integration/defs/cpp/cpp_common.py b/tests/integration/defs/cpp/cpp_common.py index e90c599d8003..b4017d2dd672 100755 --- a/tests/integration/defs/cpp/cpp_common.py +++ b/tests/integration/defs/cpp/cpp_common.py @@ -242,86 +242,3 @@ def produce_mpirun_command(*, global_commands, nranks, local_commands, l += ["-n", "1"] + local_commands + (leader_commands if rank == 0 else []) + [":"] return l[:-1] - - -def prepare_model_tests(model_name: str, - python_exe: str, - root_dir: _pl.Path, - resources_dir: _pl.Path, - model_cache_arg=[], - only_fp8_arg=[], - only_multi_gpu_arg=[]): - scripts_dir = resources_dir / "scripts" - - model_env = {**_os.environ, "PYTHONPATH": f"examples/{model_name}"} - enc_dec_model_name_arg = [] - beams_arg = [] - if model_name in ('bart', 't5', 'enc_dec_language_adapter'): - enc_dec_repo_name_dict = { - 'bart': 'facebook/bart-large-cnn', - 't5': 't5-small', - 'enc_dec_language_adapter': - 'language_adapter-enc_dec_language_adapter' - } - enc_dec_model_name_arg = [ - '--hf_repo_name', enc_dec_repo_name_dict[model_name] - ] - if model_name == 't5' and (not only_multi_gpu_arg): - beams_arg = ['--beams', '1,2'] - model_name = 'enc_dec' - - # share the same script for gpt related tests - if model_name == 'gpt_executor' or model_name == 'gpt_tests': - model_name = 'gpt' - - build_engines = [ - python_exe, - str(scripts_dir / f"build_{model_name}_engines.py") - ] + model_cache_arg + only_fp8_arg + only_multi_gpu_arg + enc_dec_model_name_arg + beams_arg - - if model_name in ['gpt']: - build_engines += ['--clean'] - run_command(build_engines, cwd=root_dir, env=model_env, timeout=1800) - - model_env["PYTHONPATH"] = "examples" - generate_expected_output = [ - python_exe, - str(scripts_dir / f"generate_expected_{model_name}_output.py") - ] + only_fp8_arg + only_multi_gpu_arg + enc_dec_model_name_arg - if "enc_dec" in model_name: - generate_expected_output += model_cache_arg - generate_expected_output += beams_arg - - if model_name in ['gpt']: - generate_expected_output += ['--clean'] - - if only_multi_gpu_arg and model_name != 'enc_dec': - for world_size in (2, 4): - generate_command = [ - "mpirun", "-n", - str(world_size), "--allow-run-as-root", "--timeout", "600" - ] + generate_expected_output - run_command(generate_command, - cwd=root_dir, - env=model_env, - timeout=600) - else: - run_command(generate_expected_output, - cwd=root_dir, - env=model_env, - timeout=600) - - if model_name in ['gpt', 'llama']: - if model_name == 'gpt': - script_model_name = 'gpt2' - elif model_name == 'llama': - script_model_name = 'Llama-3.2-1B' - generate_tokenizer_info = [ - python_exe, "examples/generate_xgrammar_tokenizer_info.py", - f"--model_dir={str(resources_dir / 'models' / script_model_name)}", - f"--output_dir={str(resources_dir / 'data' / script_model_name)}" - ] - run_command(generate_tokenizer_info, - cwd=root_dir, - env=model_env, - timeout=600) diff --git a/tests/integration/defs/cpp/test_e2e.py b/tests/integration/defs/cpp/test_e2e.py deleted file mode 100644 index ce583671cfd8..000000000000 --- a/tests/integration/defs/cpp/test_e2e.py +++ /dev/null @@ -1,328 +0,0 @@ -import copy -import logging as _logger -import os as _os -import pathlib as _pl -from dataclasses import dataclass -from typing import List, Optional - -import defs.cpp.cpp_common as _cpp -import pytest - - -@dataclass(frozen=True) -class DatasetConfig: - """Configuration for a benchmark dataset.""" - name: str - local_path: str - split: str - input_key: str - output_key: str - max_input_len: str - num_requests: str - config_name: Optional[str] = None - prompt: Optional[str] = None - prompt_key: Optional[str] = None - - @property - def token_file(self) -> str: - return "prepared_" + self.name.replace('/', '_') - - def get_dataset_args(self) -> dict[str, str]: - """Build the dataset args dict for prepare_dataset.py.""" - args = { - '--dataset-local-path': self.local_path, - '--dataset-split': self.split, - '--dataset-input-key': self.input_key, - '--dataset-output-key': self.output_key, - } - if self.config_name: - args['--dataset-config-name'] = self.config_name - if self.prompt: - args['--dataset-prompt'] = self.prompt - if self.prompt_key: - args['--dataset-prompt-key'] = self.prompt_key - return args - - -def get_benchmark_dataset_configs(model_cache: str) -> List[DatasetConfig]: - """Define dataset configurations for benchmark tests. - - To add a new dataset, add a new DatasetConfig entry to this list. - """ - datasets_dir = _pl.Path(model_cache) / "datasets" - - return [ - DatasetConfig( - name="ccdv/cnn_dailymail", - local_path=str(datasets_dir / "ccdv" / "cnn_dailymail"), - config_name="3.0.0", - split="validation", - input_key="article", - prompt="Summarize the following article:", - output_key="highlights", - max_input_len="256", - num_requests="50", - ), - DatasetConfig( - name="Open-Orca/1million-gpt-4", - local_path=str(datasets_dir / "Open-Orca" / "1million-gpt-4" / - "1M-GPT4-Augmented.parquet"), - split="train", - input_key="question", - prompt_key="system_prompt", - output_key="response", - max_input_len="20", - num_requests="10", - ), - ] - - -def run_single_gpu_tests(build_dir: _pl.Path, - test_list: List[str], - run_fp8=False, - timeout=3600): - - cpp_env = {**_os.environ} - tests_dir = build_dir / "tests" / "e2e_tests" - - included_tests = list(_cpp.generate_included_model_tests(test_list)) - - fname_list = list(_cpp.generate_result_file_name(test_list, - run_fp8=run_fp8)) - resultFileName = "-".join(fname_list) + ".xml" - - excluded_tests = ["FP8"] if not run_fp8 else [] - - excluded_tests.extend(list(_cpp.generate_excluded_test_list(test_list))) - - ctest = ["ctest", "--output-on-failure", "--output-junit", resultFileName] - - if included_tests: - ctest.extend(["-R", "|".join(included_tests)]) - if excluded_tests: - ctest.extend(["-E", "|".join(excluded_tests)]) - - parallel = _cpp.default_test_parallel - if parallel_override := _os.environ.get("LLM_TEST_PARALLEL_OVERRIDE", - None): - parallel = int(parallel_override) - - _cpp.parallel_run_ctest(ctest, - cwd=tests_dir, - env=cpp_env, - timeout=timeout, - parallel=parallel) - if "gpt" in test_list: - xml_output_file = build_dir / "results-single-gpu-disagg-executor_gpt.xml" - new_env = copy.copy(cpp_env) - new_env["TRTLLM_USE_MPI_KVCACHE"] = "1" - trt_model_test = _cpp.produce_mpirun_command( - global_commands=["mpirun", "--allow-run-as-root"], - nranks=2, - local_commands=[ - "executor/disaggExecutorTest", - "--gtest_filter=*GptSingleDeviceDisaggSymmetricExecutorTest*" - ], - leader_commands=[f"--gtest_output=xml:{xml_output_file}"]) - _cpp.run_command(trt_model_test, - cwd=tests_dir, - env=new_env, - timeout=timeout) - - run_spec_dec_tests(build_dir=build_dir) - - -def run_benchmarks( - model_name: str, - python_exe: str, - root_dir: _pl.Path, - build_dir: _pl.Path, - resources_dir: _pl.Path, - model_cache: str, - batching_types: list[str], - api_types: list[str], -): - benchmark_exe_dir = build_dir / "benchmarks" - if model_name == "gpt": - model_engine_dir = resources_dir / "models" / "rt_engine" / "gpt2" - tokenizer_dir = resources_dir / "models" / "gpt2" - elif model_name in ('bart', 't5'): - if model_name == "t5": - hf_repo_name = "t5-small" - elif model_name == "bart": - hf_repo_name = "bart-large-cnn" - model_engine_dir = resources_dir / "models" / "enc_dec" / "trt_engines" / hf_repo_name - tokenizer_dir = model_cache + "/" + hf_repo_name - model_engine_path = model_engine_dir / "1-gpu" / "float16" / "decoder" - encoder_model_engine_path = model_engine_dir / "1-gpu" / "float16" / "encoder" - model_name = "enc_dec" - else: - _logger.info( - f"run_benchmark test does not support {model_name}. Skipping benchmarks" - ) - return NotImplementedError - - if model_name == "gpt": - model_engine_path = model_engine_dir / "fp16_plugin_packed_paged" / "tp1-pp1-cp1-gpu" - - # WAR: Currently importing the bindings here causes a segfault in pybind 11 during shutdown - # As this just builds a path we hard-code for now to obviate the need for import of bindings - - # model_spec_obj = model_spec.ModelSpec(input_file, _tb.DataType.HALF) - # model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) - # model_spec_obj.use_gpt_plugin() - # model_spec_obj.use_packed_input() - # model_engine_path = model_engine_dir / model_spec_obj.get_model_path( - # ) / "tp1-pp1-cp1-gpu" - - for config in get_benchmark_dataset_configs(model_cache): - benchmark_src_dir = _pl.Path("benchmarks") / "cpp" - data_dir = resources_dir / "data" - prepare_dataset = [ - python_exe, - str(benchmark_src_dir / "prepare_dataset.py"), "--tokenizer", - str(tokenizer_dir), "--output", - str(data_dir / config.token_file), "dataset", "--max-input-len", - config.max_input_len, "--num-requests", config.num_requests - ] - for k, v in config.get_dataset_args().items(): - prepare_dataset += [k, v] - - # Use environment variable to force HuggingFace to use offline cached dataset - offline_env = {**_os.environ, 'HF_DATASETS_OFFLINE': '1'} - _cpp.run_command(prepare_dataset, - cwd=root_dir, - timeout=300, - env=offline_env) - - for batching_type in batching_types: - for api_type in api_types: - benchmark = [ - str(benchmark_exe_dir / "gptManagerBenchmark"), - "--engine_dir", - str(model_engine_path), "--type", - str(batching_type), "--api", - str(api_type), "--dataset", - str(data_dir / config.token_file) - ] - if model_name == "enc_dec": - benchmark += [ - "--encoder_engine_dir", - str(encoder_model_engine_path) - ] - - _cpp.run_command(benchmark, cwd=root_dir, timeout=600) - req_rate_benchmark = benchmark + [ - "--request_rate", "100", "--enable_exp_delays" - ] - _cpp.run_command(req_rate_benchmark, cwd=root_dir, timeout=600) - concurrency_benchmark = benchmark + ["--concurrency", "30"] - _cpp.run_command(concurrency_benchmark, - cwd=root_dir, - timeout=600) - - if "IFB" in batching_types and "executor" in api_types: - # executor streaming test - benchmark = [ - str(benchmark_exe_dir / "gptManagerBenchmark"), "--engine_dir", - str(model_engine_path), "--type", "IFB", "--dataset", - str(data_dir / config.token_file), "--api", "executor", - "--streaming" - ] - if model_name == "enc_dec": - benchmark += [ - "--encoder_engine_dir", - str(encoder_model_engine_path) - ] - _cpp.run_command(benchmark, cwd=root_dir, timeout=600) - - -def run_spec_dec_tests(build_dir: _pl.Path): - xml_output_file = build_dir / "results-spec-dec-fast-logits.xml" - cpp_env = {**_os.environ} - tests_dir = build_dir / "tests" / "e2e_tests" - trt_model_test = _cpp.produce_mpirun_command( - global_commands=["mpirun", "--allow-run-as-root"], - nranks=3, - local_commands=[ - "executor/executorTest", "--gtest_filter=*SpecDecFastLogits*" - ], - leader_commands=[f"--gtest_output=xml:{xml_output_file}"]) - _cpp.run_command(trt_model_test, cwd=tests_dir, env=cpp_env, timeout=1500) - - -@pytest.fixture(scope="session") -def run_model_tests(build_dir, lora_setup): - - def _run(model_name: str, run_fp8: bool): - run_single_gpu_tests( - build_dir=build_dir, - test_list=[model_name], - timeout=_cpp.default_test_timeout, - run_fp8=run_fp8, - ) - - return _run - - -@pytest.fixture(scope="session") -def run_model_benchmarks(root_dir, build_dir, cpp_resources_dir, python_exe, - model_cache): - - def _run( - model_name: str, - batching_types: List[str], - api_types: List[str], - ): - - run_benchmarks( - model_name=model_name, - python_exe=python_exe, - root_dir=root_dir, - build_dir=build_dir, - resources_dir=cpp_resources_dir, - model_cache=model_cache, - batching_types=batching_types, - api_types=api_types, - ) - - return _run - - -@pytest.mark.parametrize("build_google_tests", ["80", "86", "89", "90"], - indirect=True) -@pytest.mark.parametrize("model", [ - "bart", "chatglm", "eagle", "encoder", "enc_dec_language_adapter", "gpt", - "gpt_executor", "gpt_tests", "llama", "mamba", "medusa", "recurrentgemma", - "redrafter", "t5" -]) -@pytest.mark.parametrize("run_fp8", [False, True], ids=["", "fp8"]) -def test_model(build_google_tests, model, prepare_model, run_model_tests, - run_fp8): - - if model == "recurrentgemma": - pytest.skip( - "TODO: fix recurrentgemma OOM with newest version of transformers") - return - - prepare_model(model, run_fp8) - - run_model_tests(model, run_fp8) - - -@pytest.mark.parametrize("build_google_tests", ["80", "86", "89", "90"], - indirect=True) -@pytest.mark.parametrize("model", ["bart", "gpt", "t5"]) -def test_benchmarks(build_benchmarks, model, prepare_model, - run_model_benchmarks): - - prepare_model(model) - - batching_types = ["IFB"] - api_types = ["executor"] - - run_model_benchmarks( - model_name=model, - batching_types=batching_types, - api_types=api_types, - ) diff --git a/tests/integration/defs/cpp/test_multi_gpu.py b/tests/integration/defs/cpp/test_multi_gpu.py index 1124178cccc6..12a0353fda10 100644 --- a/tests/integration/defs/cpp/test_multi_gpu.py +++ b/tests/integration/defs/cpp/test_multi_gpu.py @@ -1,9 +1,7 @@ import os as _os import pathlib as _pl import platform -import time from enum import Enum, auto -from typing import List, Optional import defs.cpp.cpp_common as _cpp import pytest @@ -149,344 +147,6 @@ def run_nccl_utils_tests(build_dir: _pl.Path, nprocs=2, timeout=300): timeout=timeout) -def run_llama_executor_leader_tests(build_dir: _pl.Path, timeout=1500): - tests_dir = build_dir / "tests" / "e2e_tests" - - mgpu_env = get_multi_gpu_env(llama_multi_gpu=True) - - #Executor test in leader mode - xml_output_file = build_dir / "results-multi-gpu-llama-exec-leader-mode.xml" - trt_model_test = _cpp.produce_mpirun_command( - global_commands=["mpirun", "--allow-run-as-root"], - nranks=4, - local_commands=[ - "executor/executorTest", - "--gtest_filter=*LlamaExecutorTest*LeaderMode*:*LlamaMultiExecutorTest*LeaderMode*" - ], - leader_commands=[f"--gtest_output=xml:{xml_output_file}"]) - - _cpp.run_command(trt_model_test, cwd=tests_dir, env=mgpu_env, timeout=1500) - - -def run_llama_executor_orchestrator_tests(build_dir: _pl.Path, timeout=1500): - tests_dir = build_dir / "tests" / "e2e_tests" - - mgpu_env = get_multi_gpu_env(llama_multi_gpu=True) - - #Executor test in orchestrator mode - xml_output_file = build_dir / "results-multi-gpu-llama-exec-orch-mode.xml" - trt_model_test = [ - "mpirun", "-n", "1", "--allow-run-as-root", "executor/executorTest", - "--gtest_filter=*LlamaExecutorTest*OrchMode*", - f"--gtest_output=xml:{xml_output_file}" - ] - _cpp.run_command(trt_model_test, cwd=tests_dir, env=mgpu_env, timeout=1500) - - -def run_llama_executor_logits_proc_tests(build_dir: _pl.Path, timeout=1500): - tests_dir = build_dir / "tests" / "e2e_tests" - - mgpu_env = get_multi_gpu_env(llama_multi_gpu=True) - - #Logits processor test in leader mode - xml_output_file = build_dir / "results-multi-gpu-logits-proc.xml" - - tp_pp_sizes = [(4, 1), (2, 2), (1, 4)] - gtest_filter = [ - f"LlamaExecutorTest/LogitsProcParamsTest*tp{tp}_pp{pp}*" - for tp, pp in tp_pp_sizes - ] - - gtest_filter = ":".join(gtest_filter) - - trt_model_test = _cpp.produce_mpirun_command( - global_commands=["mpirun", "--allow-run-as-root"], - nranks=4, - local_commands=[ - "executor/executorTest", f"--gtest_filter={gtest_filter}" - ], - leader_commands=[f"--gtest_output=xml:{xml_output_file}"]) - - _cpp.run_command(trt_model_test, cwd=tests_dir, env=mgpu_env, timeout=1500) - - -def run_llama_executor_guided_decoding_tests(build_dir: _pl.Path, timeout=1500): - tests_dir = build_dir / "tests" / "e2e_tests" - - mgpu_env = get_multi_gpu_env(llama_multi_gpu=True) - - #Guided decoding test in leader mode - xml_output_file = build_dir / "results-multi-gpu-guided-decoding.xml" - - tp_pp_sizes = [(4, 1), (2, 2), (1, 4)] - gtest_filter = [ - f"LlamaExecutorGuidedDecodingTest/GuidedDecodingParamsTest*tp{tp}_pp{pp}*" - for tp, pp in tp_pp_sizes - ] - - gtest_filter = ":".join(gtest_filter) - - trt_model_test = _cpp.produce_mpirun_command( - global_commands=["mpirun", "--allow-run-as-root"], - nranks=4, - local_commands=[ - "executor/executorTest", f"--gtest_filter={gtest_filter}" - ], - leader_commands=[f"--gtest_output=xml:{xml_output_file}"]) - - _cpp.run_command(trt_model_test, cwd=tests_dir, env=mgpu_env, timeout=1500) - - -def run_enc_dec_multi_gpu_tests(build_dir: _pl.Path, timeout=1500): - tests_dir = build_dir / "tests" / "e2e_tests" - cpp_env = {**_os.environ} - - #EncDec test in leader mode - xml_output_file = build_dir / "results-multi-gpu-t5-exec-leader-mode.xml" - trt_model_test = _cpp.produce_mpirun_command( - global_commands=["mpirun", "--allow-run-as-root"], - nranks=4, - local_commands=[ - "executor/encDecTest", - "--gtest_filter=T5MultiGPUTest/EncDecParamsTest.Forward*" - ], - leader_commands=[f"--gtest_output=xml:{xml_output_file}"], - ) - _cpp.run_command(trt_model_test, cwd=tests_dir, env=cpp_env, timeout=1500) - - -def run_trt_gpt_model_real_decoder_multi_gpu_tests(build_dir: _pl.Path, - timeout=1500): - tests_dir = build_dir / "tests" / "e2e_tests" - cpp_env = {**_os.environ} - - xml_output_file = build_dir / "results-multi-gpu-real-decoder.xml" - trt_model_test = _cpp.produce_mpirun_command( - global_commands=["mpirun", "--allow-run-as-root"], - nranks=4, - local_commands=[ - "batch_manager/trtGptModelRealDecoderTest", - "--gtest_filter=*TP*:*PP*" - ], - leader_commands=[f"--gtest_output=xml:{xml_output_file}"]) - _cpp.run_command(trt_model_test, - cwd=tests_dir, - env=cpp_env, - timeout=timeout) # expecting ~ 1200s - - -def run_disagg_symmetric_executor_tests(build_dir: _pl.Path, - model: str, - nprocs=2, - kvcache_type=KVCacheType.MPI, - timeout=1500): - tests_dir = build_dir / "tests" / "e2e_tests" - - prefix = get_model_test_filter_prefix(model) - - mgpu_env = get_multi_gpu_env(kv_cache_type=kvcache_type, - llama_multi_gpu=True) - - xml_output_file = build_dir / f"results-multi-gpu-disagg-executor-{nprocs}-process.xml" - trt_model_test = _cpp.produce_mpirun_command( - global_commands=["mpirun", "--allow-run-as-root"], - nranks=nprocs, - local_commands=[ - "executor/disaggExecutorTest", - f"--gtest_filter=*{prefix}*DisaggSymmetricExecutorTest*" - ], - leader_commands=[f"--gtest_output=xml:{xml_output_file}"]) - - _cpp.run_command(trt_model_test, - cwd=tests_dir, - env=mgpu_env, - timeout=timeout) - - -def run_disagg_asymmetric_executor_tests(build_dir: _pl.Path, - model: str, - nprocs=4, - kvcache_type=KVCacheType.MPI, - timeout=1500): - - tests_dir = build_dir / "tests" / "e2e_tests" - - prefix = get_model_test_filter_prefix(model) - - mgpu_env = get_multi_gpu_env(kv_cache_type=kvcache_type, - llama_multi_gpu=True) - - xml_output_file = build_dir / f"results-multi-gpu-disagg-asymmetric-executor-{nprocs}-process.xml" - - trt_model_test = _cpp.produce_mpirun_command( - global_commands=["mpirun", "--allow-run-as-root"], - nranks=nprocs, - local_commands=[ - "executor/disaggExecutorTest", - f"--gtest_filter=*{prefix}*DisaggAsymmetricExecutorTest*" - ], - leader_commands=[f"--gtest_output=xml:{xml_output_file}"]) - - _cpp.run_command(trt_model_test, - cwd=tests_dir, - env=mgpu_env, - timeout=timeout) - - -def run_disagg_orchestrator_params_tests(build_dir: _pl.Path, - model: str, - kvcache_type=KVCacheType.MPI, - timeout=1500): - - tests_dir = build_dir / "tests" / "e2e_tests" - - prefix = get_model_test_filter_prefix(model) - - mgpu_env = get_multi_gpu_env(kv_cache_type=kvcache_type, - llama_multi_gpu=True) - - xml_output_file = build_dir / "results-multi-gpu-disagg-asymmetric-orchestrator-executor-7-process.xml" - trt_model_test = _cpp.produce_mpirun_command( - global_commands=["mpirun", "--allow-run-as-root"], - nranks=7, - local_commands=[ - "executor/disaggExecutorTest", - f"--gtest_filter=*{prefix}*DisaggOrchestratorParamsTest*" - ], - leader_commands=[f"--gtest_output=xml:{xml_output_file}"]) - _cpp.run_command(trt_model_test, - cwd=tests_dir, - env=mgpu_env, - timeout=timeout) - - -def run_disagg_spawn_orchestrator_tests(build_dir: _pl.Path, - model: str, - kvcache_type=False, - timeout=1500): - - tests_dir = build_dir / "tests" / "e2e_tests" - - prefix = get_model_test_filter_prefix(model) - - mgpu_env = get_multi_gpu_env(kv_cache_type=kvcache_type, - llama_multi_gpu=True) - - xml_output_file = build_dir / "results-multi-gpu-disagg-spawn-asymmetric-orchestrator-executor-1-process.xml" - - comms = [ - "executor/disaggExecutorTest", - f"--gtest_filter=*{prefix}*DisaggSpawnOrchestrator*", - f"--gtest_output=xml:{xml_output_file}" - ] - _cpp.run_command(comms, cwd=tests_dir, env=mgpu_env, timeout=timeout) - - -def prepare_multi_gpu_model_tests(test_list: List[str], - python_exe: str, - root_dir: _pl.Path, - resources_dir: _pl.Path, - model_cache: Optional[str] = None): - - model_cache_arg = ["--model_cache", model_cache] if model_cache else [] - - if "llama" in test_list: - _cpp.prepare_model_tests(model_name="llama", - python_exe=python_exe, - root_dir=root_dir, - resources_dir=resources_dir, - model_cache_arg=model_cache_arg, - only_multi_gpu_arg=["--only_multi_gpu"]) - - if "t5" in test_list: - _cpp.prepare_model_tests(model_name="t5", - python_exe=python_exe, - root_dir=root_dir, - resources_dir=resources_dir, - model_cache_arg=model_cache_arg, - only_multi_gpu_arg=['--tp', '4', '--pp', '1']) - - -@pytest.fixture(scope="session") -def prepare_model_multi_gpu(python_exe, root_dir, cpp_resources_dir, - model_cache): - - def _prepare(model_name: str): - if platform.system() != "Windows": - - start_time = time.time() - - prepare_multi_gpu_model_tests( - test_list=[model_name], - python_exe=python_exe, - root_dir=root_dir, - resources_dir=cpp_resources_dir, - model_cache=model_cache, - ) - - duration = time.time() - start_time - print(f"Built multi-GPU model: {model_name}") - print(f"Duration: {duration} seconds") - - return _prepare - - -@pytest.fixture(scope="session") -def gpt_single_gpu_model(prepare_model): - prepare_model("gpt") - return "gpt" - - -@pytest.fixture(scope="session") -def llama_single_gpu_model(prepare_model): - prepare_model("llama") - return "llama" - - -@pytest.fixture(scope="session") -def llama_multi_gpu_model(prepare_model_multi_gpu): - prepare_model_multi_gpu("llama") - return "llama" - - -# Allow us to dynamically choose a fixture at runtime -# Combined with session scope fixtures above to ensure -# that the model is built only once per pytest session -@pytest.fixture -def prepare_models_disagg(request): - - def _prepare(model_name: str): - if model_name == "llama": - fixture_names = [ - "llama_single_gpu_model", - "llama_multi_gpu_model", - ] - elif model_name == "gpt": - fixture_names = [ - "gpt_single_gpu_model", - ] - else: - raise ValueError(f"Disagg tests don't support model: {model_name}") - - print(f"Preparing models for disagg tests: {fixture_names}") - # Run the fixtures - for fixture_name in fixture_names: - request.getfixturevalue(fixture_name) - - return _prepare - - -# Use indirect parameterization to ensure that the model is built -# only once per pytest session -@pytest.fixture(scope="session") -def multi_gpu_model(request, prepare_model_multi_gpu): - - model_name = request.param - prepare_model_multi_gpu(model_name) - - return model_name - - @pytest.mark.parametrize("build_google_tests", ["80", "86", "89", "90"], indirect=True) def test_mpi_utils(build_google_tests, build_dir): @@ -535,138 +195,3 @@ def test_nccl_utils(build_google_tests, nprocs, build_dir): if platform.system() != "Windows": run_nccl_utils_tests(build_dir=build_dir, nprocs=nprocs, timeout=300) - - -@pytest.mark.parametrize("build_google_tests", ["80", "86", "89", "90"], - indirect=True) -@pytest.mark.parametrize("multi_gpu_model", ["t5"], indirect=True) -def test_enc_dec(build_google_tests, multi_gpu_model, build_dir): - - if platform.system() != "Windows": - run_enc_dec_multi_gpu_tests(build_dir=build_dir, - timeout=_cpp.default_test_timeout) - - -@pytest.mark.parametrize("build_google_tests", ["80", "86", "89", "90"], - indirect=True) -@pytest.mark.parametrize("mode", ["orchestrator", "leader"]) -@pytest.mark.parametrize("multi_gpu_model", ["llama"], indirect=True) -def test_llama_executor(build_google_tests, multi_gpu_model, mode, lora_setup, - build_dir): - - if platform.system() == "Windows": - return - - if mode == "orchestrator": - run_llama_executor_orchestrator_tests(build_dir=build_dir, - timeout=_cpp.default_test_timeout) - elif mode == "leader": - run_llama_executor_leader_tests(build_dir=build_dir, - timeout=_cpp.default_test_timeout) - else: - raise ValueError(f"Unsupported mode: {mode}") - - -@pytest.mark.parametrize("build_google_tests", ["80", "86", "89", "90"], - indirect=True) -@pytest.mark.parametrize("multi_gpu_model", ["llama"], indirect=True) -def test_llama_executor_logits_proc(build_google_tests, multi_gpu_model, - lora_setup, build_dir): - - if platform.system() != "Windows": - run_llama_executor_logits_proc_tests(build_dir=build_dir, - timeout=_cpp.default_test_timeout) - - -@pytest.mark.parametrize("build_google_tests", ["80", "86", "89", "90"], - indirect=True) -@pytest.mark.parametrize("multi_gpu_model", ["llama"], indirect=True) -def test_llama_executor_guided_decoding(build_google_tests, multi_gpu_model, - lora_setup, build_dir): - - if platform.system() != "Windows": - run_llama_executor_guided_decoding_tests( - build_dir=build_dir, timeout=_cpp.default_test_timeout) - - -@pytest.mark.parametrize("build_google_tests", ["80", "86", "89", "90"], - indirect=True) -@pytest.mark.parametrize("multi_gpu_model", ["llama"], indirect=True) -def test_trt_gpt_real_decoder(build_google_tests, multi_gpu_model, lora_setup, - build_dir): - - if platform.system() != "Windows": - run_trt_gpt_model_real_decoder_multi_gpu_tests( - build_dir=build_dir, timeout=_cpp.default_test_timeout) - - -@pytest.mark.parametrize("build_google_tests", ["80", "86", "89", "90"], - indirect=True) -class TestDisagg: - - @pytest.mark.parametrize( - "kvcache_type", [KVCacheType.MPI, KVCacheType.UCX, KVCacheType.NIXL], - ids=["mpi_kvcache", "ucx_kvcache", "nixl_kvcache"]) - @pytest.mark.parametrize("nprocs", [2, 4, 8], - ids=["2proc", "4proc", "8proc"]) - @pytest.mark.parametrize("model", ["gpt", "llama"]) - def test_symmetric_executor(self, build_google_tests, model, nprocs, - kvcache_type, prepare_models_disagg, build_dir): - - if model == "gpt" and nprocs > 2: - pytest.skip( - "test_symmetric_executor only supports 2 processes for gpt") - - if platform.system() != "Windows": - prepare_models_disagg(model) - - run_disagg_symmetric_executor_tests(build_dir=build_dir, - model=model, - nprocs=nprocs, - kvcache_type=kvcache_type) - - @pytest.mark.parametrize( - "kvcache_type", [KVCacheType.MPI, KVCacheType.UCX, KVCacheType.NIXL], - ids=["mpi_kvcache", "ucx_kvcache", "nixl_kvcache"]) - @pytest.mark.parametrize("nprocs", [4, 6, 8], - ids=["4proc", "6proc", "8proc"]) - @pytest.mark.parametrize("model", ["llama"]) - def test_asymmetric_executor(self, build_google_tests, model, nprocs, - kvcache_type, prepare_models_disagg, - build_dir): - - if platform.system() != "Windows": - prepare_models_disagg(model_name=model) - - run_disagg_asymmetric_executor_tests(build_dir=build_dir, - model=model, - nprocs=nprocs, - kvcache_type=kvcache_type) - - @pytest.mark.parametrize( - "kvcache_type", [KVCacheType.MPI, KVCacheType.UCX, KVCacheType.NIXL], - ids=["mpi_kvcache", "ucx_kvcache", "nixl_kvcache"]) - @pytest.mark.parametrize("model", ["llama"]) - def test_orchestrator_params(self, build_google_tests, model, kvcache_type, - prepare_models_disagg, build_dir): - - if platform.system() != "Windows": - prepare_models_disagg(model) - - run_disagg_orchestrator_params_tests(build_dir=build_dir, - model=model, - kvcache_type=kvcache_type) - - @pytest.mark.parametrize("kvcache_type", - [KVCacheType.UCX, KVCacheType.NIXL], - ids=["ucx_kvcache", "nixl_kvcache"]) - @pytest.mark.parametrize("model", ["llama"]) - def test_spawn_orchestrator(self, build_google_tests, model, kvcache_type, - prepare_models_disagg, build_dir): - - if platform.system() != "Windows": - prepare_models_disagg(model) - - run_disagg_spawn_orchestrator_tests(build_dir=build_dir, - model=model, - kvcache_type=kvcache_type) diff --git a/tests/integration/defs/cpp/test_unit_tests.py b/tests/integration/defs/cpp/test_unit_tests.py index a55440bb1a48..730ebbf389ed 100644 --- a/tests/integration/defs/cpp/test_unit_tests.py +++ b/tests/integration/defs/cpp/test_unit_tests.py @@ -8,7 +8,7 @@ indirect=True) @pytest.mark.parametrize("test_group", [ "batch_manager", "common", "executor", "kernels", "layers", "runtime", - "thop", "utils" + "thop" ]) def test_unit_tests(build_google_tests, test_group, build_dir, lora_setup): diff --git a/tests/integration/defs/deterministic/mixtral_deterministic.py b/tests/integration/defs/deterministic/mixtral_deterministic.py deleted file mode 100644 index 53abff63d739..000000000000 --- a/tests/integration/defs/deterministic/mixtral_deterministic.py +++ /dev/null @@ -1,253 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import json -import os -import time -from concurrent.futures import ThreadPoolExecutor -from typing import Dict, List - -import jinja2 -from transformers import AutoTokenizer - -import tensorrt_llm.bindings.executor as trtllm - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--engine_dir", type=str, default="engine_outputs") - parser.add_argument("--tokenizer_dir", type=str, default="tokenizer_dir") - parser.add_argument("--payload", type=str, default="./payload.json") - parser.add_argument("--concurrency", type=int, default=50) - parser.add_argument("--check_deterministic_accuracy", - action="store_true", - default=False) - parser.add_argument("--deterministic_accuracy_threshold", - type=int, - default=1) - parser.add_argument("--batch", action="store_true", default=False) - parser.add_argument("--wait", type=float, default=0.0) - parser.add_argument("--output", type=str, default='out-strs') - return parser.parse_args() - - -def create_engine(engine): - parallel_config = trtllm.ParallelConfig( - communication_type=trtllm.CommunicationType.MPI, - communication_mode=trtllm.CommunicationMode.LEADER) - trt_scheduler_config = trtllm.SchedulerConfig( - trtllm.CapacitySchedulerPolicy.GUARANTEED_NO_EVICT) - kv_cache_config = trtllm.KvCacheConfig( - free_gpu_memory_fraction=0.9, - enable_block_reuse=True, - ) - extend_runtime_perf_knob_config = trtllm.ExtendedRuntimePerfKnobConfig() - extend_runtime_perf_knob_config.cuda_graph_mode = False - extend_runtime_perf_knob_config.multi_block_mode = False - executor_config = trtllm.ExecutorConfig( - 1, - iter_stats_max_iterations=100, - # nvbugs/4662826 - request_stats_max_iterations=0, - parallel_config=parallel_config, - # normalize_log_probs=False, - batching_type=trtllm.BatchingType.INFLIGHT, - # batching_type=trtllm.BatchingType.STATIC, - scheduler_config=trt_scheduler_config, - kv_cache_config=kv_cache_config, - enable_chunked_context=True, - extended_runtime_perf_knob_config=extend_runtime_perf_knob_config, - ) - - return trtllm.Executor(model_path=engine, - model_type=trtllm.ModelType.DECODER_ONLY, - executor_config=executor_config) - - -def create_request(payload_file, template_str, tokenizer): - json_data = { - 'model': - 'my-model', - 'messages': [ - { - 'role': 'user', - 'content': 'Hello there how are you?', - }, - { - 'role': 'assistant', - 'content': 'Good and you?', - }, - { - 'role': 'user', - 'content': 'Whats your name?', - }, - ], - 'max_tokens': - 1024, - 'temperature': - 0, - #'top_k':1, - #'nvext': {"top_k": 1}, - 'stream': - False - } - - json_data['messages'][2]['content'] = """ - Classify the sentiment expressed in the following text and provide the response in a single word Positive/Negative/Neutral. Explain your answer in 2 lines. - TEXT:: Today I will exaggerate, will be melodramatic (mostly the case when I am excited) and be naive (as always). Just came out from the screening of the Avengers Endgame ("Endgame")! The journey had started in the year 2008, when Tony Stark, during his capture in a cave in Afghanistan, had created a combat suit and came out of his captivity. - Then the combat suit made of iron was perfected and Tony Stark officially became the Iron Man!! The Marvel Cinematic Universe ("MCU") thus was initiated. The journey continued since then and in 2012 all the MCU heroes came together and formed the original "Avengers" (so much fun and good it was). - 21 Movies in the MCU and culminating into the Infinity War (2018) and finally into the Endgame! The big adventure for me started from Jurassic Park and then came Titanic, Lagaan, Dark Knight; and then came the Avengers in 2012. Saw my absolute favorite Sholay in the hall in 2014. In the above-mentioned genre, there are good movies, great movies and then there is the Endgame. - Today after a long long time, I came out of the hall with 100% happiness, satisfaction and over the top excitement/emotions. The movie is Epic, Marvel (in the real sense) and perfect culmination of the greatest cinematic saga of all time. It is amazing, humorous, emotional and has mind-blowing action! It is one of the finest Superhero Movie of all time. - Just pure Awesome! It's intelligent! - """ - with open(payload_file, 'r') as f: - msg_system = json.load(f) - msg_user = [] - msg_user.append({ - "role": - "user", - "content": - msg_system[0]["content"] + "\n\n" + msg_system[1]["content"] - }) - msg_user.extend(msg_system[2:]) - json_data['messages'] = msg_user - - environment = jinja2.Environment() - template = environment.from_string(template_str) - json_data['bos_token'] = '' - json_data['eos_token'] = '' - prompt = template.render(json_data) - - tokens = tokenizer.encode(prompt) - - sample_params = trtllm.SamplingConfig( - beam_width=1, # beam_width=1 for inflight batching - top_k=1, # SizeType topK - top_p=1.0, - top_p_min=None, - top_p_reset_ids=None, # SizeType topPResetIds - top_p_decay=None, # FloatType topPDecay - seed=1234, - temperature=1, - min_tokens=1, # SizeType minLength - beam_search_diversity_rate=None, # FloatType beamSearchDiversityRate - repetition_penalty=1, # FloatType repetitionPenalty - presence_penalty=0, # FloatType presencePenalty - frequency_penalty=0, # FloatType frequencyPenalty - length_penalty=1, # FloatType lengthPenalty - early_stopping= - None, # SizeType earlyStopping. Controls beam search, so irrelevant until we have beam_width > 1 - ) - #sample_params = trtllm.SamplingConfig(temperature=0, seed=1234) - - return trtllm.Request( - input_token_ids=tokens[1:], - max_tokens=1024, - sampling_config=sample_params, - streaming=False, - stop_words=None, - # stop_words=[[2]], # - ), prompt - - -def get_tokenizer(tokenizer_file): - return AutoTokenizer.from_pretrained(tokenizer_file) - - -def get_template(tokenizer_file): - with open(os.path.join(tokenizer_file, - "tokenizer_config.json")) as tok_config: - cfg = json.load(tok_config) - return cfg['chat_template'] - - -def enqueue_requests(pool, executor, request, concurrency=50, wait=0): - for _ in range(concurrency): - _ = pool.submit(executor.enqueue_request, request) - if wait > 0: - time.sleep(wait) - - -def main(): - args = get_args() - executor = create_engine(args.engine_dir) - if executor.can_enqueue_requests(): - template = get_template(args.tokenizer_dir) - tokenizer = get_tokenizer(args.tokenizer_dir) - concurrency = int(args.concurrency) - - request, prompt = create_request(args.payload, template, tokenizer) - os.makedirs(args.output, exist_ok=True) - with open(os.path.join(args.output, "prompt.txt"), 'w') as f: - f.write(prompt) - - try: - for _ in range(1): - outputs: Dict[str, List[trtllm.Result]] = {} - num_finished = 0 - - if not args.batch: - with ThreadPoolExecutor(max_workers=concurrency) as pool: - enqueue_requests(pool, - executor, - request, - concurrency=concurrency, - wait=args.wait) - else: - executor.enqueue_requests( - [request for _ in range(concurrency)]) - while num_finished < concurrency: - responses = executor.await_responses() - for response in responses: - if response.has_error(): - outputs[response.request_id] = response.error_msg - num_finished += 1 - else: - result = response.result - if result.is_final: - num_finished += 1 - if response.request_id not in outputs: - outputs[response.request_id] = [] - outputs[response.request_id].append(result) - output_strs = {} - for req_id, output in outputs.items(): - if isinstance(output, str): - raise RuntimeError(output) - elif isinstance(output, list): - if len(output) != 1: - raise RuntimeError("Expected list size of 1") - output_strs[req_id] = tokenizer.decode( - output[0].output_token_ids[0]) - with open(os.path.join(args.output, f"{req_id}.out"), - "w") as f: - f.write(output_strs[req_id]) - else: - raise RuntimeError("Unexpected output") - - output_set = set(output_strs.values()) - num_unique_responses = len(output_set) - if args.check_deterministic_accuracy: - assert num_unique_responses <= args.deterministic_accuracy_threshold, f"Expected num unique responses <= {args.deterministic_accuracy_threshold} while got {num_unique_responses} " - result_str = f"Num Unique responses in {len(outputs)}: {len(output_set)}" - print(result_str) - with open(os.path.join(args.output, "num_outputs"), 'w') as f: - f.write(result_str + '\n') - finally: - executor.shutdown() - - -if __name__ == "__main__": - main() diff --git a/tests/integration/defs/deterministic/payload.json b/tests/integration/defs/deterministic/payload.json deleted file mode 100644 index b3373b811fb9..000000000000 --- a/tests/integration/defs/deterministic/payload.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "role": "system", - "content": "DOCUMENTS:\n[\n {\n \"id\": \"doc_1\",\n \"title\": \"Employee Benefits - India\",\n \"content\": \"**Insurance Entity** **SPOC** **Escalation-1** **Final Escalation** IPD: Medi Assist - TPA Dedicated help \\& Support Mr. Hamza Zaidi Mr. Syed Sajid ++[servicenow@mediassist.in](mailto:thota.rajani@mediassist.in)++ ++[hamza.zaidi@mediassist.in](mailto:hamza.zaidi@mediassist.in)++ ++[syed.sajid@mediassist.in](mailto:syed.sajid@mediassist.in)++ 8951386235 7349122713 8105565671 OPD: Connect and Heal (CNH) Dedicated help \\& Support Mr.Rohit ++[servicenow.support@connectandheal.com](mailto:servicenow.support@connectandheal.com)++ ++[rohit.singh@connectandheal.com](mailto:rohit.singh@connectandheal.com)++ 91118 91118 8884726473 Marsh - Broker Mr. Sridhar Anneboina Ms. Anjali Agarwal Mr. K. Balajee Rao ++[Sridhar.anneboina@marsh.com](mailto:Sridhar.anneboina@marsh.com)++ ++[anjali.agarwal@marsh.com](mailto:anjali.agarwal@marsh.com)++ ++[balajee.rao@marsh.com](mailto:balajee.rao@marsh.com)++ 8976773547 9966200636 9160555531 Note: Any insurance escalations are to reach up to the last contact of Marsh and later if required can reach for internal support. ++[Internal Escalation: Please raise a request in the My ServiceNow Portal.](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=42c07a15db951700b2e2d34b5e9619f1&sc_category=undefined&sc_cat_item=36b09733db36df00014fffd51d9619a0)++ **Tele Consultations on CNH portal:**A Virtual Consultation with a medical/health practitioner online for 32 specialists. This service will help to reach and consult the specialists \\& doctors to discuss health concerns from home. No cost is implied for phone specialist consultations and is available for dependants as well. **Annual Health Screening** **on CNH portal** **:**A yearly free annual health check-up for employees and a discounted program for dependants. **For further information:** Read more about this program on the [benefits portal.](https://urldefense.com/v3/__https:/servicenow.okta.com/home/thomsonsonlinebenefits/0oa1g1v3i17DiSuob0h8/38923?fromHome=true__;!!N4vogdjhuJM!Vyvf3ceOERltigdu08bwjhFaS6SzovSGcXfj6H7vRqzpUz_C1AJ5rSTXDQh037j1frPr9vC0bKY$) **Need Help?** You can log in to the CNH portal via Quick Links for bookings and can find a user guide available in the [benefits portal.](https://urldefense.com/v3/__https:/servicenow.okta.com/home/thomsonsonlinebenefits/0oa1g1v3i17DiSuob0h8/38923?fromHome=true__;!!N4vogdjhuJM!Vyvf3ceOERltigdu08bwjhFaS6SzovSGcXfj6H7vRqzpUz_C1AJ5rSTXDQh037j1frPr9vC0bKY$) contact ++[servicenow.support@connectandheal.com](mailto:servicenow.support@connectandheal.com)++before booking your appointment to know more about the terms of this service. |\n| **Group Term Life Insurance** | | The current insurance provider is ICICI Prudential Life Insurance Co. Ltd. Our insurance is renewed every **31 March to 30 March.** **I**t is an employee-only benefit. Should in the event of death either natural or accident of the Insured Member whilst working for ServiceNow as a regular, full-time active employee, you or your beneficiary(ies) will receive a lump sum payment equal to the insured amount. This payment will be subject to income tax. The insured amount is **3 x Annual Base Salary** (coverage may be subject to medical underwriting and insurer's acceptance) and subject to a maximum of INR 4,00,00,000, per Insured Member * Age Group of Cover: 18 years to 80 years of age (age as of last birthday) **New to ServiceNow?** You are automatically covered under this policy from joining the day of ServiceNow. You need to nominate your dependents in the **[benefits portal.](https://urldefense.com/v3/__https:/servicenow.okta.com/home/thomsonsonlinebenefits/0oa1g1v3i17DiSuob0h8/38923?fromHome=true__;!!N4vogdjhuJM!Vyvf3ceOERltigdu08bwjhFaS6SzovSGcXfj6H7vRqzpUz_C1AJ5rSTXDQh037j1frPr9vC0bKY$)** **Policy Rules:** Please refer to the [**Insurance Manual**](sys_attachment.do?sys_id=5b6062bb47a3795030c37a24846d43eb)for more information. **What happens if I leave ServiceNow?** On your date of exit, the group term life plan coverages stop immediately. **Need Help?** Please contact our Broker SPOCs. **Internal support** - Please [raise a request](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=42c07a15db951700b2e2d34b5e9619f1&sc_category=undefined&sc_cat_item=36b09733db36df00014fffd51d9619a0) in the My ServiceNow Portal. **Insurance Entity** **SPOC** **Escalation-1** **Final Escalation** **Marsh - Broker** Mr. Sridhar Anneboina Ms. Anjali Agarwal Mr. K. Balajee Rao [Sridhar.anneboina@marsh.com](mailto:Indraneel.Singamsetty@marsh.com) [anjali.agarwal@marsh.com](mailto:anjali.agarwal@marsh.com) [balajee.rao@marsh.com](mailto:balajee.rao@marsh.com) 8976773547 9966200636 9160555531 |\n| **Group Personal Accident** | | The current insurance provider is Liberty General Insurance. Our insurance is renewed every **31 March to 30 March.** It is an employee-only benefit. This insurance covers you for both work-related \\& non-work-related accidents. Should in the event of an accident and Permanent Total or Partial Disability of the Insured Member whilst working for ServiceNow as a regular, full-time active employee, you or your beneficiary(ies) will receive a lump sum payment equal to the insured amount. This payment will be subject to income tax. The insured amount is **3x Annual Base Salary** (coverage may be subject to medical underwriting and insurer's acceptance) per Insured Member. This policy provides a lump sum benefit to the Insured Member or dependent(s), in the event the Insured Member sustains Bodily Injury as per policy terms. Age Group of Cover: 18 years to 65 years of age (age as of last birthday) **New to ServiceNow?**You are automatically covered under this policy from joining the day of ServiceNow. You need to nominate your dependents in the [benefits portal.](https://urldefense.com/v3/__https:/servicenow.okta.com/home/thomsonsonlinebenefits/0oa1g1v3i17DiSuob0h8/38923?fromHome=true__;!!N4vogdjhuJM!Vyvf3ceOERltigdu08bwjhFaS6SzovSGcXfj6H7vRqzpUz_C1AJ5rSTXDQh037j1frPr9vC0bKY$) **Policy Rules:** Please refer to the [**Insurance Manual**](sys_attachment.do?sys_id=5b6062bb47a3795030c37a24846d43eb) for more information. **What happens if I leave ServiceNow?** On your date of exit, the group personal accident plan coverages stop immediately. **Need Help?** Please contact our Broker SPOCs. **Internal support** - Please [raise a request](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=42c07a15db951700b2e2d34b5e9619f1&sc_category=undefined&sc_cat_item=36b09733db36df00014fffd51d9619a0) in the My ServiceNow Portal. **Insurance Entity** **SPOC** **Escalation-1** **Final Escalation** **Marsh - Broker** Mr. Sridhar Anneboina Ms. Anjali Agarwal Mr. K. Balajee Rao[\\[SN10\\]](#_msocom_10){#_anchor_10} [Sridhar.anneboina@marsh.com](mailto:Indraneel.Singamsetty@marsh.com) [anjali.agarwal@marsh.com](mailto:anjali.agarwal@marsh.com) [balajee.rao@marsh.com](mailto:balajee.rao@marsh.com) 8976773547 9966200636 9160555531 *** ** * ** *** |\n| #### **Retirement \\& Pension Benefits** | | ServiceNow contributes to the statutory norms of the Country and Provides PF and gratuity. **Provident Fund (PF)** - Both employer \\& employee stipulated funds A provident fund is a part of your salary that is deducted every month and deposited with the Employee Provident Fund Organization (EPFO) under your name. Normally 12% of your basic monthly salary is deposited in the EPFO while a matching percentage is contributed by ServiceNow. While your contribution is fully invested, the employer's contribution is partly invested along with your funds and partly utilized for providing other benefits like Pensions and Employee Deposit Linked Insurance (EDLI) premiums. When an employee leaves his/her job, a part of this amount can be withdrawn as per the governing withdrawal rules. If the employee attains retirement age, then the Provident Fund can be partly withdrawn and partly used for purchasing a life annuity plan that provides a monthly pension. **Policy Rules:** Please refer to Benefits Information in the [benefits portal.](https://urldefense.com/v3/__https:/servicenow.okta.com/home/thomsonsonlinebenefits/0oa1g1v3i17DiSuob0h8/38923?fromHome=true__;!!N4vogdjhuJM!Vyvf3ceOERltigdu08bwjhFaS6SzovSGcXfj6H7vRqzpUz_C1AJ5rSTXDQh037j1frPr9vC0bKY$) **Need to avail?** Please [raise a request](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=0c0369b4db555300b2e2d34b5e9619fe&sc_category=undefined&sc_cat_item=0284fe44130dd240796ad1e2e144b0f9) in the My ServiceNow Portal with Payroll. **New to ServiceNow?** Soon after joining ServiceNow, you will be automatically enrolled in PF, and you need to provide the required details to Payroll for processing. **Need Help?** Please [rai](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=0c0369b4db555300b2e2d34b5e9619fe&sc_category=undefined&sc_cat_item=0284fe44130dd240796ad1e2e144b0f9)[se a request](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=0c0369b4db555300b2e2d34b5e9619fe&sc_category=undefined&sc_cat_item=0284fe44130dd240796ad1e2e144b0f9)in the My ServiceNow Portal with Payroll. **Gratuity:** An employee completing almost 5 years or more of uninterrupted service can claim this benefit. Only in case the employee suffers from a permanent total disability that this benefit be accelerated and can be paid before completion of 5 years of service as well. **Policy Rules:** Please refer to Benefits Information in the [benefits portal.](https://urldefense.com/v3/__https:/servicenow.okta.com/home/thomsonsonlinebenefits/0oa1g1v3i17DiSuob0h8/38923?fromHome=true__;!!N4vogdjhuJM!Vyvf3ceOERltigdu08bwjhFaS6SzovSGcXfj6H7vRqzpUz_C1AJ5rSTXDQh037j1frPr9vC0bKY$) **Need to avail?** Under the following two conditions, an employee can claim this benefit: 1. Retirement/ Resignation/ Superannuation/Death after 5 years or more of uninterrupted service. 2. Suffering from a permanent total disability due to accident, sickness, or disease. **New to ServiceNow?** Soon after joining ServiceNow, you will be automatically enrolled in Gratuity. **Need Help?** Please [raise a request](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=0c0369b4db555300b2e2d34b5e9619fe&sc_category=undefined&sc_cat_item=0284fe44130dd240796ad1e2e144b0f9) in the My ServiceNow Portal with the payroll **NPS- National Pension System** The National Pension System is a Tax effective low-cost pension plan that helps employees manage finances post-retirement. Please refer to the NPS section in the [benefits portal](https://urldefense.com/v3/__https:/servicenow.okta.com/home/thomsonsonlinebenefits/0oa1g1v3i17DiSuob0h8/38923?fromHome=true__;!!N4vogdjhuJM!Vyvf3ceOERltigdu08bwjhFaS6SzovSGcXfj6H7vRqzpUz_C1AJ5rSTXDQh037j1frPr9vC0bKY$) to know more about NPS. Corporate NPS enables to save additional tax up to 10% of Salary (Basic) on employer's contribution in NPS u/s 80CCD(2) of Income Tax Act, over and above the limit of 1.5 lakhs u/s 80C and exclusive additional tax deduction benefit of Rs. 50,000/- u/s 80CCD(1B). It is a voluntary plan and employees once have a retirement account raise a request with an [HR ticket](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=42c07a15db951700b2e2d34b5e9619f1&sc_category=0ca38e41dba9a300e7e5f9c31d961955&sc_cat_item=e9ef54aadb2636809f5258a8dc9619c2) for PRAN account mapping with corporate and after HR confirmation can start contributing monthly from their salary via employer with a [payroll ticket](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=0c0369b4db555300b2e2d34b5e9619fe&sc_category=undefined&sc_cat_item=ce6fbf171be1f0101e41caae6e4bcb67). These funds will be deposited to the employee NPS account and invested. **Need to avail?** Under the following two conditions, an employee/nominee can claim this benefit as per the pension plan guidelines of NSDL. 1. Retirement at the age of 60 years 2. Death **Need Help?** Please reach the [NSDL Portal](https://npscra.nsdl.co.in/all-citizens-faq.php). **Internal support** - Please [raise a request](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=42c07a15db951700b2e2d34b5e9619f1&sc_category=undefined&sc_cat_item=e9ef54aadb2636809f5258a8dc9619c2) in the My ServiceNow Portal. #### **What happens if I leave ServiceNow?** **On your date of exit, the Retirement \\& Pension Benefits stop immediately.** |\n\n**If you need additional assistance or have questions about any of the information provided in this article,** please [raise a request](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=42c07a15db951700b2e2d34b5e9619f1&sc_category=undefined&sc_cat_item=36b09733db36df00014fffd51d9619a0) in the My ServiceNow Portal.\n\n**Additional Resources**\n\n[**Employee Benefits Summary**](sys_attachment.do?sys_id=1f6062bb47a3795030c37a24846d43e5)\n\nTime Away and [Holiday](https://my.servicenow.com/esc?id=events) -- [KB0056866](https://my.servicenow.com/esc?sys_id=d1080c2e87a42d102217437e0ebb3580&id=surf_kb_article&q=Time%20Away%20-%20India)\n\nDay Care Facility -- [KB0059432](https://my.servicenow.com/esc?sys_id=af448b25973e81100a63b84bf253afd3&id=surf_kb_article&q=Day%20Care%20Facility%20%E2%80%93%20India)\n\nTuition Reimbursement - [KB0007401](https://my.servicenow.com/esc?id=surf_kb_article&sys_id=290bf1d7db54a8108ae42b691396199c)\n\nStock Resource Center - [KB0034611](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0034611)\n\nBusiness Travel - [KB0056930](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0056930)\n\nEmployee Assistance Program - [KB0045470](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0045470 \"KB0045470\")\n\nWhil/Rethink Benefits**-** [KB0051803](https://my.servicenow.com/esc?sys_id=fc0e5326dbf064505ad34f7813961974&id=surf_kb_article&q=Rethink%20benefits)\n\nGlobal Wellbeing Programs - [KB0058670](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0058670)\n\nCleo - Parenting support -- [KB0058667](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0058667)\n\nFertilityIQ - Family building resources - [KB0058660](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0058660)\n\nGrayce - Caregiver support - [KB0058668](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0058668)\n\nVirgin Pulse - Wellbeing platform - [KB0058669](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0058669)\n\nFond - Discount Program - [KB0008923](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0008923 \"KB0008923\")\n\n Employee Benefits - India\n\n|——————————-|---|—————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————---|\n| **Group Medical Insurance** | | The current medical insurance provider is Aditya Birla Health Insurance and the Third-Party Administrator (TPA) for claims management is Medi Assist Healthcare Services Private Limited for IPD and Connect and Heal (CNH) for OPD. Our insurance is renewed every **31 March to 30 March.** This policy provides Cashless/reimbursement of recognized costs of services or treatment which are recommended by a medical practitioner, and are medically necessary for the care and treatment of an injury or sickness, up to the limits shown in the list of benefits in the [++Insurance Manual++](sys_attachment.do?sys_id=5b6062bb47a3795030c37a24846d43eb) The insured amount is up to INR 5,00,000 and INR 15,000 per year of insurance which is inclusive of hospitalization coverages, Day Care procedures, maternity, and more for IPD and OPD expenses like dental and vision, generalist consultations, specialist consultations, and AYUSH. Employee-only medical insurance coverage for Critical illness \\& COVID of INR 5,00,000 and INR 50,000 This insurance coverage may be subject to sub-limits as shown in the list of benefits in the [Insurance manual](sys_attachment.do?sys_id=5b6062bb47a3795030c37a24846d43eb) and provides cover within Pan India. **Eligibility:** All regular, full-time active employees, employed by ServiceNow India, and their Eligible Dependents\\* **\\*Eligible Dependents:** * One legal spouse or legal LGBTQ partner * Children (4) under the age of 25 * Either Parents or Parents_In_Laws(Combination not allowed) **Policy rules:** Please refer to the [**Insurance manual**](sys_attachment.do?sys_id=5b6062bb47a3795030c37a24846d43eb) for more information. **Need to make a claim?**You may follow the claim process mentioned in the benefits portal under Group Medical Insurance benefit -- FAQ. (Cashless / Reimbursement) **New to ServiceNow?** Soon after joining ServiceNow, you will be automatically enrolled in this insurance plan, and you need to enroll your dependants in the [benefits portal](https://urldefense.com/v3/__https:/servicenow.okta.com/home/thomsonsonlinebenefits/0oa1g1v3i17DiSuob0h8/38923?fromHome=true__;!!N4vogdjhuJM!Vyvf3ceOERltigdu08bwjhFaS6SzovSGcXfj6H7vRqzpUz_C1AJ5rSTXDQh037j1frPr9vC0bKY$) and mark them for this insurance coverage within 13 days. You will be provided access to this benefits portal on your day 2 and also informed via email. **Life event?** If you are recently married or have a newborn baby, log the new dependent details on the benefits portal and mark them for this insurance cover within 30 days from the date of the event. For any further assistance, please raise a [ticket](https://my.servicenow.com/esc?id=service_catalog&spa=1&sc_catalog=42c07a15db951700b2e2d34b5e9619f1) in the My ServiceNow portal and our Global People Care team will be in touch. Once you have completed the online process, the medical cover will be set up and ready to go Via the TPA's portal, and mobile apps and membership cards will be available in the next 30 business days. **++Group Medical Insurance Top-up Coverage++** **++:++** The Mediclaim plan is an extension of company-provided medical insurance cover and you may choose this on a yearly basis for yourself and your family as an extra cover for medical expenses. **++Top-Up Insurance (Voluntary)--++** You also have the option to purchase an additional Mediclaim expense limit over and above the company-sponsored INR 5,00,000 and INR 15,000 limit via a \"Top-up Medical Insurance Plan\". Under this plan, you can choose to buy any one of the additional Sum Insured limits for IPD from INR 3,00,000, INR 5,00,000, INR 7,00,000, INR 10,00,000 and INR 15,00,000 for OPD from INR 7,000, INR 10,000, INR 15,000 by paying an additional premium from your salary. This Top-up plan is an extension of base coverages as per policy terms and will be automatically initiated when default sum insurance is exhausted. **++Note -++** The premium amount details can be found in the Group Medical Insurance Top-up sections in the Benefits portal and the respective premiums will be deducted in three equal installments from your salary based on the top-up coverage Selections. As with all medical schemes, there are a number of policy exclusions that may affect your ability to claim. For this reason, you should read the [++Insurance Manual++](sys_attachment.do?sys_id=5b6062bb47a3795030c37a24846d43eb) which details the terms and conditions of coverage. **Need to make a claim?**The claims will automatically trigger once the medical policy cover provided by the company is exhausted. **New to ServiceNow?** Soon after joining ServiceNow, you will be provided with options to choose from this insurance plan in the [benefits portal.](https://urldefense.com/v3/__https:/servicenow.okta.com/home/thomsonsonlinebenefits/0oa1g1v3i17DiSuob0h8/38923?fromHome=true__;!!N4vogdjhuJM!Vyvf3ceOERltigdu08bwjhFaS6SzovSGcXfj6H7vRqzpUz_C1AJ5rSTXDQh037j1frPr9vC0bKY$)Your dependants have automatically been considered for this cover from the company-provided base policy and the premium is pro-rated as per your Date of joining. **What happens if I leave ServiceNow?** On your date of exit, the group medical plan coverages stop immediately. **Need Help?** Please contact our TPA or Broker SPOCs.\"\n },\n {\n \"id\": \"doc_2\",\n \"title\": \"India Darwin Benefits Enrolment Portal Guide\",\n \"content\": \"India Darwin Benefits Enrolment Portal Guide\n\n**Q) How do I select my Benefits?**\n\n1. Sign into the **[My ServiceNow Benefits](https://servicenow.okta.com/home/bookmark/0oa1ttc0ujznXItCh0h8/2557)** or via [Okta](https://servicenow.okta.com/app/UserHome?session_hint=AUTHENTICATED). And select the 'Review your benefits Coverage (Go to the Darwin benefits enrolment portal)' tile in the left side corner.\n2. Select the notification bell in the top right-hand corner\n3. If you are a new joiner, you will see this option 'New Joiner Enrolment' notification\n4. If you click view or select benefits, the relevant insurance benefits will be available to select.\n5. Benefits are broken down into the following:\n\n* Financial\n* Physical\n\nIf you would like to know more about these, please select the 'more info' link found underneath each benefit which will take you to a handy FAQ document.\n\n6. Make your Insurance selections.\n\na. Click 'Edit' on the benefit you would like to select.\n\nb. Complete the sequence of steps\n\nc. Add selection to Cart\n\nd. Make sure to check out!\n\n7. Shortly after checking out, you will receive an email from DoNotReply@ServiceNowBenefits.com confirming your selections.\n\na. All selections made will show as green in your benefits portal.\n\n**Q) When can I add or change my dependents?**\n\n* You can add or remove dependents to your cover during the annual enrolment window, which occurs in March each year.\n* You can also apply to make a change to your cover if your personal circumstances change. This is called a lifestyle event and broadly includes things like births, marriage. You can apply for a lifestyle event by navigating to the person icon in the top right corner of the darwin benefits enrollment site and selecting \"Life Event\"\n* Please note;\n * When you add dependents to your benefits portal, they do not automatically add to the policies, the dependants must be added manually by yourself.\n * Kindly enroll new dependents (spouse/newborn) within one month of the occurrence of the life event.\n\n**Q) When can I select my benefits and do my previous selections carry over?**\n\nYou can add or remove dependents, change/add the Medical TOP UP selections to your cover during the annual enrolment window, which occurs in March each year.\n\n1. Sign into the **[My ServiceNow Benefits](https://servicenow.okta.com/home/bookmark/0oa1ttc0ujznXItCh0h8/2557)**or via [Okta](https://servicenow.okta.com/app/UserHome?session_hint=AUTHENTICATED). And select the 'Review your benefits Coverage (Go to the Darwin benefits enrolment portal)' tile in the left side corner.\n2. Select the notification bell in the top right-hand corner\n3. If you are a current employee, you will see this option 'Annual Enrolment Window' notification\n4. If you click view or select benefits, the relevant insurance benefits will be available to select.\n5. Benefits are broken down into the following:\n\n* Financial\n* Physical\n\nIf you would like to know more about these, please select the 'more info' link found underneath each benefit which will take you to a handy FAQ document.\n\n6. Make your Insurance selections.\n\na. Click 'Edit' on the each of the insurance benefit you would like to select(to navigate from one benefit to another click 'Go back' on top left corner of each benefits page)\n\nb. Complete the sequence of steps\n\nc. Add selection to Cart\n\nd. Make sure to check out!\n\nPlease note; Note: If there are no changes to any of the benefit page, the system notifies (Refer below the pop up notification) that there is no change to consider, no action to take. The selections will be carried forward by default.\n\nIf this article does not resolve your issue, please reach out to ServiceNow Benefit Helpdesk team by navigating to the 'question mark' icon in the top right corner of the [darwin enrollment site](https://servicenow.okta.com/home/thomsonsonlinebenefits/0oa1g1v3i17DiSuob0h8/38923?fromHome=true) and click on it, they will assist you.\"\n },\n {\n \"id\": \"doc_3\",\n \"title\": \"ServiceNow's Our World of Work – Employee FAQs\",\n \"content\": \"ServiceNow's Our World of Work – Employee FAQs\n\n**{#Top1}Our world of work -- Employee Frequently Asked Questions (FAQs)**\n\nThis document provides supporting details for our World of Work model. Our World of Work model is based on these guiding principles:\n\n* Keep employees healthy and safe\n* Live company values\n* Reinforce our brand and product opportunities\n* Drive employee engagement and experience\n* Commit to equity and inclusion in our culture\n* Further our Global Impact goals\n\nThe information contained in this FAQ is subject to local legal requirements, and may change at any time. Employees should leverage these key resources for additional information on our policies and practices:\n\n* [Our world of work KB hub](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0057030)\n* [30-day work from anywhere benefit](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0057039)\n* [Flexible work persona](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0057029)\n* [Remote work persona](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0057031)\n* [Required in office work persona](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0057402)\n* [Work persona change request](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0057042)\n\nClick on the questions below to be redirected to the answer:\n\n**General questions**\n\n[What are the benefits of a distributed, flexible work model?](#G1)\n\n[How is Our World of Work defined?](#G2)\n\n[Employees have been successfully working from home. Why doesn't this just continue for everyone in all organizations globally?](#G3)\n\n[What else is ServiceNow doing to drive improvements to collaboration, which was an opportunity area identified in the Employee Voice Survey (EVS)?](#GQ4)\n\n**Work personas**\n\n[How do I know what my work persona is?](#WP1)\n\n[What if I want to change my work persona?](#WP2)\n\n[How are work personas assigned?](#WP3)\n\n[If I submit a change to my work persona/work location, when will it be processed in our systems?](#WP4)\n\n[If my work persona is remote, can I relocate to live anywhere I like?](#WPN2)\n\n[If I change my work persona to remote, is there a compensation change? What about if I change my status to remote but still live in the same area?](#WP5)\n\n[Why have we clarified that our expectation for the flexible work persona is two or more days a week in the office?](#WPN3)\n\n[What happens if I don't come into the office two or more days a week?](#WPN4)\n\n[What if I have a personal circumstance that may make it difficult for me to meet the two or more days a week in-office expectation?](#WPN5)\n\n[What if I am in the flexible work persona, but none of my direct team is co-located in my assigned work location?](#WPNN2)\n\n[What is the messaging for contingent workers?](#WPNN1)\n\n**Policies and benefits**\n\n[Which policies support Our World of Work?](#P1)\n\n[Will I be reimbursed for my home office expenses?](#P2)\n\n[Can I use my Work from Anywhere (WFA) 30-day allotment toward the two or more days per week expectation?](#PB3)\n\n**Workplaces and workspaces**\n\n[How will workplaces evolve in our flexible, distributed model?](#WPL1)\n\n[Am I allowed to work from a ServiceNow office more than two days a week?](#WPL3)\n\n[What if there are multiple ServiceNow offices within a commutable distance for me? Which office am I expected to be aligned to?](#WWN1)\n\n[Is it possible to work from a location other than the contracted location for which I was hired?](#WPL4)\n\n[Will I have an assigned desk?](#WPL5)\n\n[What kind of workstation setup can I expect when I go into the office?](#WWN3 \"What kind of workstation setup can I expect when I go into the office?\")\n\n[What is the difference between a workplace and workspace?](#WPL6)\n\n[If we want to sit near our partners and internal customers, are we allowed access to those neighborhoods?](#WPL7)\n\n[How will the new work personas impact travel to ServiceNow workplace locations?](#WPL8)\n\n[Is my workplace attendance data being shared with anyone? If so, who is it being shared with?](#WWN4)\n\n[Why are some organizations more closely monitoring workplace attendance data than others?](#WWN5)\n\n[I'm in the flexible work persona, but work during \"off hours\" (evening or nighttime work start times). How are we accounting for physical workplace safety in meeting the two or more days per week expectation?](#WPNN1)\n\n[If I live in India, which states am I allowed to move to?](#WPL9)\n\n{#G1}\n\n**Q. What are the benefits of a distributed, flexible work model?**\n\n**A.** We have adopted a distributed, flexible work model that is right for Our World of Work. We reshaped our connections with each other, adapted the intent and purpose of the workplace, and used the power of the Now Platform to do our best work together. What about if I change my status to remote but still live in the same area?**\n\n**A**. Work personas by themselves do not affect compensation, but the market tier you work in will (only in the United States). If your work persona is flexible or required-in-office, your compensation is based on your office location. If you are remote, then your compensation is based on your remote work location. Refer to the [market tiers for the United States](https://my.servicenow.com/esc?id=surf_kb_article&number=KB0005540).\n\n[**Go to top**](#Top1 \"Go to top\")\n\n{#WPN3}\n\n**Q.** **Why have we clarified that our expectation for the flexible work persona is two or more days a week in the office?**\n\n**A.**By getting more specific on expectation of number of days spent in the office, we will bring more clarity for and ultimately drive consistency for employees in the flexible work persona across the company. We feel that two or more days a week will help facilitate the benefits and advantages of 'shoulder-to-shoulder' work. We know how valuable remote work is as well, and that's why we continue to lead Our World of Work with your flexibility and choice in mind.\n\n[**Go to top**](#Top1 \"Go to top\")\n\n{#WPN4}\n\n**Q. What happens if I don't come into the office two or more days a week?**\n\n**A.**Your manager may check in with you during regular discussions on whether you're adhering to the expectation of two or more days each week. If you're not regularly coming to the office, your manager may partner with you to create a plan that supports you meeting the expectation for being in the office. For now, there are no formal consequences for not meeting this expectation.\n\n[**Go to top**](#Top1 \"Go to top\")\n\n{#WPN5}\n\n**Q. What if I have a personal circumstance that may make it difficult for me to meet the two or more days a week in-office expectation?**\n\n**A.**Bill said it best: There's something incalculable in ServiceNow's culture about working \"shoulder-to-shoulder.\" We come together to do meaningful work while getting to know our colleagues. We believe that working with your team (and across teams) in person fosters even greater collaboration and innovation, builds social wellbeing via the relationships we develop, and actively cultivates ServiceNow's culture. Thus, we feel that those who live close to ServiceNow offices are a natural fit for the flexible persona. It is reasonable to ask you to be present in our offices without burdensome commutes that unduly impact your personal time.\n\nIf you have a personal circumstance that prevents you from meeting the expectations of the flexible work persona and you would like to request an accommodation, please use these resources:\n\n* Employees in the U.S. can reference [this workplace accommod...\"\n }\n]\n\n\nCATALOG_ITEMS:\n[\n {\n \"id\": \"cat_1\",\n \"title\": \"Benefits - General\",\n \"description\": \"\"\n },\n {\n \"id\": \"cat_2\",\n \"title\": \"Benefits Coverage\",\n \"description\": \"For questions related to your benefits including your coverage, enrolling in or changing your benefits, portal login issues, business traveler benefits, or benefit changes related to leaves of absence (LOA) please use this request\"\n },\n {\n \"id\": \"cat_3\",\n \"title\": \"Benefits - 401(k)\",\n \"description\": \"\"\n },\n {\n \"id\": \"cat_4\",\n \"title\": \"IN - Benefits & Insurance details\",\n \"description\": \"**Please add your dependent details for medical insurance policy.**\"\n },\n {\n \"id\": \"cat_5\",\n \"title\": \"Benefits - Leave / LOA\",\n \"description\": \"\"\n }\n]" - }, - { - "role": "user", - "content": "USER QUERY:\ni would like to know benefits\n\nTASK:\n- Your role as a question and answering bot is to respond to the given USER QUERY based on the information provided in the DOCUMENTS and CATALOG ITEMS\n- If the USER QUERY is not answerable based on the DOCUMENTS, say 'No answer found' and nothing else.\n- If there are any helpful CATALOG ITEMS to the USER QUERY, you should recommend them.\n\nIMPORTANT RULES:\n- Format your response in markdown format\n- Provide citations like (doc_1), (doc_2), (doc_3) or (cat_1), (cat_2), (cat_3) throughout your response.\n- Do not say \"Based on ...\" or \"Sure\"\n- Do not use any first person language such as \"I\".\n- Do not produce any HTML tags\n- Keep your answer concise.\n- Do not miss any helpful CATALOG ITEMS.\n\nRESPONSE FORMAT:\n\nuser_interested_to_read_docs: \nprovide_enough_details_for_user_to_understand_without_reading: \ndo_not_provide_task_instructions_to_user: \ndo_not_respond_to_unsafe_requests: \nlist_of_max_5_based_on_user_query_history:[max 5 key word topics...]\nlist_of_max_5_topics_requested_by_user: [max of 5 key word topics...]\nlist_of_max_5_topics_in_docs: [max of 5 key word topics...]\nlist_of_matching_catalog_items: [cat_1...]\nsay_no_answer_found: \n\n....\n" - }, - { - "role": "assistant", - "content": "\nuser_interested_to_read_docs: \nprovide_enough_details_for_user_to_understand_without_reading: \ndo_not_provide_task_instructions_to_user: \ndo_not_respond_to_unsafe_requests: " - } -] diff --git a/tests/integration/defs/deterministic/test_mixtral_deterministic.py b/tests/integration/defs/deterministic/test_mixtral_deterministic.py deleted file mode 100644 index 1796e9827d31..000000000000 --- a/tests/integration/defs/deterministic/test_mixtral_deterministic.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import pytest -from defs.common import (convert_weights, generate_deterministic_cmd, - venv_mpi_check_call) -from defs.conftest import skip_pre_hopper -from defs.trt_test_alternative import check_call - - -@skip_pre_hopper -@pytest.mark.skip_less_device(4) -@pytest.mark.skip_less_device_memory(80000) -@pytest.mark.parametrize("data_type", ['float16', 'bfloat16']) -@pytest.mark.parametrize("llm_mixtral_model_root", - ['Mixtral-8x7B-Instruct-v0.1'], - indirect=True) -def test_llm_mixtral_4gpus_deterministic(llama_example_root, - llm_mixtral_model_root, - deterministic_test_root, llm_venv, - cmodel_dir, engine_dir, data_type): - tp_size, pp_size = 4, 1 - world_size = tp_size * pp_size - moe_tp_size = tp_size - - os.environ['FORCE_DETERMINISTIC'] = "1" - - print("Convert checkpoint...") - ckpt_dir = convert_weights(llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model="mixtral-instruct", - model_path=llm_mixtral_model_root, - tp_size=tp_size, - moe_tp_size=moe_tp_size, - pp_size=pp_size, - data_type=data_type, - workers=world_size) - - print("Build engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - f"--workers={world_size}", - "--use_paged_context_fmha=enable", - "--max_batch_size=256", - "--max_num_tokens=33280", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run deterministic test...") - deterministic_accuracy_threshold = 1 - payload = os.path.join(deterministic_test_root, "payload.json") - deterministic_cmd = generate_deterministic_cmd( - deterministic_test_root, - engine_dir=engine_dir, - tokenizer_dir=llm_mixtral_model_root, - payload=payload, - deterministic_accuracy_threshold=deterministic_accuracy_threshold) - - venv_mpi_check_call( - llm_venv, ["mpirun", "-n", f"{world_size}", "--allow-run-as-root"], - deterministic_cmd) - - os.environ.pop('FORCE_DETERMINISTIC', None) diff --git a/tests/integration/defs/disaggregated/disagg_test_utils.py b/tests/integration/defs/disaggregated/disagg_test_utils.py index 32365b713873..320f08c4a278 100644 --- a/tests/integration/defs/disaggregated/disagg_test_utils.py +++ b/tests/integration/defs/disaggregated/disagg_test_utils.py @@ -84,7 +84,15 @@ async def wrapper(*args, **kwargs): def _run_worker( - model_name, worker_config, role, port, work_dir, device=-1, save_log=False, env=None + model_name, + worker_config, + role, + port, + work_dir, + device=-1, + save_log=False, + env=None, + worker_index=0, ): """Run a worker process (context or generation). @@ -97,11 +105,13 @@ def _run_worker( device: CUDA device ID (-1 for default) save_log: Whether to save logs to file env: Environment variables for the subprocess + worker_index: Index used for log/config filenames (avoids collisions when + multiple workers share port=0) Returns: ProcessWrapper: Wrapped subprocess """ - worker_config_path = os.path.join(work_dir, f"{role}_{port}_config.yaml") + worker_config_path = os.path.join(work_dir, f"{role}_{worker_index}_config.yaml") with open(worker_config_path, "w+") as f: yaml.dump(worker_config, f) f.flush() @@ -130,7 +140,7 @@ def _run_worker( stdout = None stderr = None if save_log: - log_path = os.path.join(work_dir, f"worker_{role}_{port}.log") + log_path = os.path.join(work_dir, f"worker_{role}_{worker_index}.log") log_file = open(log_path, "w+") stdout = log_file stderr = log_file @@ -145,26 +155,56 @@ def _run_worker( def run_ctx_worker( - model_name, ctx_worker_config, work_dir, port=0, device=0, env=None, save_log=False + model_name, + ctx_worker_config, + work_dir, + port=0, + device=0, + env=None, + save_log=False, + worker_index=0, ): """Launch a context worker with service discovery. Use port=0 to let the worker choose a free port. """ return _run_worker( - model_name, ctx_worker_config, "ctx", port, work_dir, device, save_log=save_log, env=env + model_name, + ctx_worker_config, + "ctx", + port, + work_dir, + device, + save_log=save_log, + env=env, + worker_index=worker_index, ) def run_gen_worker( - model_name, gen_worker_config, work_dir, port=0, device=1, env=None, save_log=False + model_name, + gen_worker_config, + work_dir, + port=0, + device=1, + env=None, + save_log=False, + worker_index=0, ): """Launch a generation worker with service discovery. Use port=0 to let the worker choose a free port. """ return _run_worker( - model_name, gen_worker_config, "gen", port, work_dir, device, save_log=save_log, env=env + model_name, + gen_worker_config, + "gen", + port, + work_dir, + device, + save_log=save_log, + env=env, + worker_index=worker_index, ) diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_cache_reuse_deepseek_v3.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_cache_reuse_deepseek_v3.yaml index 36b459dabc1e..9a51a0f5903d 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_cache_reuse_deepseek_v3.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_cache_reuse_deepseek_v3.yaml @@ -2,9 +2,12 @@ hostname: localhost model: DeepSeek-V3-Lite/bf16 free_gpu_memory_fraction: 0.15 backend: pytorch +attn_backend: FLASHINFER cuda_graph_config: null disable_overlap_scheduler: true enable_autotuner: false +model_kwargs: + num_hidden_layers: 4 context_servers: num_instances: 1 tensor_parallel_size: 1 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_conditional_deepseek_v3_v2.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_conditional_deepseek_v3_v2.yaml index d33e1541406e..5b21530c62d3 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_conditional_deepseek_v3_v2.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_conditional_deepseek_v3_v2.yaml @@ -7,9 +7,9 @@ conditional_disagg_config: max_local_prefill_length: 100 disable_overlap_scheduler: true enable_autotuner: false -# Enable per-request perf metrics so the disagg /perf_metrics endpoint (used by -# the test's _check_bypass_signal) records routed requests; bypassed ones absent. +# Enable response-carried metrics so the test can validate routed phases. perf_metrics_max_requests: 1000 +return_perf_metrics: true context_servers: num_instances: 1 tensor_parallel_size: 1 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4ep4_gentp4ep4_glm5_nvfp4_dp_tllm.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4ep4_gentp4ep4_glm5_nvfp4_dp_tllm.yaml index 57cf5bcbebed..fc790618269b 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4ep4_gentp4ep4_glm5_nvfp4_dp_tllm.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp4ep4_gentp4ep4_glm5_nvfp4_dp_tllm.yaml @@ -20,7 +20,8 @@ context_servers: cuda_graph_config: null print_iter_log: true cache_transceiver_config: - backend: DEFAULT + backend: NIXL + transceiver_runtime: PYTHON max_tokens_in_buffer: 16384 generation_servers: num_instances: 1 @@ -55,5 +56,6 @@ generation_servers: - 1024 print_iter_log: true cache_transceiver_config: - backend: DEFAULT + backend: NIXL + transceiver_runtime: PYTHON max_tokens_in_buffer: 16384 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_metrics.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_metrics.yaml index 48fc0a072f02..4bb52cc134f9 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_metrics.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_metrics.yaml @@ -5,6 +5,7 @@ backend: "pytorch" cuda_graph_config: null disable_overlap_scheduler: True perf_metrics_max_requests: 1000 +return_perf_metrics: true context_servers: num_instances: 1 tensor_parallel_size: 1 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_multi_orchestrator.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_multi_orchestrator.yaml new file mode 100644 index 000000000000..970c2e276647 --- /dev/null +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_multi_orchestrator.yaml @@ -0,0 +1,24 @@ +hostname: localhost +model: TinyLlama/TinyLlama-1.1B-Chat-v1.0 +num_workers: 4 +free_gpu_memory_fraction: 0.25 +backend: pytorch +disable_overlap_scheduler: true +context_servers: + num_instances: 1 + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + kv_cache_config: + free_gpu_memory_fraction: 0.2 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON +generation_servers: + num_instances: 1 + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + router: + type: conversation diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index 2570a4a6c17c..dc92b76ce104 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -40,7 +40,8 @@ run_disagg_server, run_gen_worker, terminate, wait_for_disagg_server_ready) from test_common.perf_metrics_utils import (get_timing_metrics, - validate_timing_metrics) + validate_timing_metrics, + wait_for_perf_metrics_jsonl) from tensorrt_llm._utils import mpi_disabled from tensorrt_llm.logger import logger @@ -53,6 +54,7 @@ class TestConfig: test_desc: str request_count: int accuracy_threshold: float + incomplete_threshold: float = 1.0 speculative_model_path: Optional[str] = None cancellation_rate: Optional[int] = None cancellation_delay: Optional[float] = None @@ -89,10 +91,14 @@ def cleanup_output_files(): # Fatal patterns whose presence in worker/server logs after a stress run # indicates the cluster did not stay healthy and the test should be failed. +# Only genuinely fatal conditions belong here — transient request-level errors +# like "Cluster is not ready" / "Internal server error" are (a) expected during +# the autotuner warmup phase (whose results are discarded) and (b) already +# gated for the measured phase by incomplete_rate/server_rejected, so scanning +# the whole log for them would false-positive on warmup churn. _FATAL_LOG_PATTERNS = ( "Hang detected on rank", - "RuntimeError: Cluster is not ready", - "Internal server error", + "out of memory", ) @@ -122,6 +128,33 @@ def scan_logs_for_fatal_errors(processes): return findings +def _crashed_workers(workers): + return [ + w for w in workers + if w.process.poll() is not None and w.process.poll() != 0 + ] + + +def build_worker_diag(workers, disagg_server): + """Check worker processes and logs for crashes/fatal errors. + + Returns a diagnostic string describing any problems found, or an empty + string if all workers are healthy. + """ + all_procs = list(workers) + [disagg_server] + crashed = _crashed_workers(workers) + fatal = scan_logs_for_fatal_errors(all_procs) + diag = "" + if crashed: + parts = [ + f"{w.log_path or 'worker'} (rc={w.process.poll()})" for w in crashed + ] + diag += f" Workers exited abnormally: {parts}." + if fatal: + diag += f" Fatal log findings: {fatal}." + return diag + + def get_default_disagg_cluster_config(): """Get default disaggregated cluster configuration.""" return { @@ -149,6 +182,7 @@ def build_worker_config(base_config: dict[str, Any], EXCLUDE_FROM_WORKER = { 'hostname', 'port', + 'num_workers', 'num_instances', 'urls', 'router', @@ -206,6 +240,8 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_load_balancing.yaml", "conversation": f"{test_configs_root}/disagg_config_conversation.yaml", + "multi_orchestrator": + f"{test_configs_root}/disagg_config_multi_orchestrator.yaml", "4_ranks": f"{test_configs_root}/disagg_config_ctxtp2_gentp1.yaml", "cuda_graph": @@ -318,7 +354,7 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_ctxtp4ep4_gentp4ep4_glm5_nvfp4_dp_tllm.yaml", "qwen3_32b_fp8_stress": f"{test_configs_root}/disagg_config_ctxtp1_gentp4_qwen3_32b_fp8.yaml", - "req60-conc64-qwen3_32b_fp8_mixed_stress": + "req120-conc64-qwen3_32b_fp8_mixed_stress": f"{test_configs_root}/disagg_config_ctxtp1_gentp4_qwen3_32b_fp8.yaml", "req10k-conc512-qwen3_32b_fp8_mixed_stress": f"{test_configs_root}/disagg_config_ctxtp1_gentp4_qwen3_32b_fp8.yaml", @@ -623,6 +659,7 @@ def setup_disagg_cluster( save_log: bool = False, startup_callback=None, startup_tick: int = 30, + perf_metrics_output_dir: str | None = None, ) -> tuple[dict[str, Any], list[ProcessWrapper], list[ProcessWrapper], ProcessWrapper, int, str]: """Load config, launch workers + disagg server, wait for ready. @@ -640,6 +677,13 @@ def setup_disagg_cluster( with open(config_file, 'r') as f: config = yaml.safe_load(f) + if perf_metrics_output_dir is not None: + config["perf_metrics_output_dir"] = perf_metrics_output_dir + for servers in ("context_servers", "generation_servers"): + config.setdefault( + servers, + {})["perf_metrics_output_dir"] = perf_metrics_output_dir + speculative_config = config.get("speculative_config") if isinstance(speculative_config, dict): speculative_model = speculative_config.get("speculative_model") @@ -650,8 +694,23 @@ def setup_disagg_cluster( disagg_cluster = get_default_disagg_cluster_config() server_host = config.get("hostname", "localhost") server_port = get_free_port() - work_dir = tempfile.mkdtemp() - disagg_cluster["cluster_uri"] = f"http://{server_host}:{server_port}" + if save_log: + log_base = os.path.join(cwd or ".", "disagg-logs") + os.makedirs(log_base, exist_ok=True) + work_dir = tempfile.mkdtemp(dir=log_base) + else: + work_dir = tempfile.mkdtemp() + logger.info(f"Disagg cluster work_dir (worker logs): {work_dir}") + server_env = env + coordinator_url = f"http://{server_host}:{server_port}" + if config.get("num_workers", 1) > 1: + coordinator_port = get_free_port() + while coordinator_port == server_port: + coordinator_port = get_free_port() + coordinator_url = f"http://{server_host}:{coordinator_port}" + server_env = (env or os.environ).copy() + server_env["TRTLLM_DISAGG_COORDINATOR_PORT"] = str(coordinator_port) + disagg_cluster["cluster_uri"] = coordinator_url # Auto-deduce minimal_instances from num_instances ctx_servers = config.get("context_servers", {}) @@ -692,34 +751,38 @@ def setup_disagg_cluster( device_ids = ",".join( str(d) for d in dict.fromkeys((next_device + j) % num_gpus for j in range(gpus_per_ctx))) - print( - f"Launching ctx worker {i + 1}/{num_ctx_instances} on device {device_ids}" - ) - ctx_workers.append( - run_ctx_worker(model, + w = run_ctx_worker(model, ctx_worker_config, work_dir, port=0, device=device_ids, env=env, - save_log=save_log)) + save_log=save_log, + worker_index=i) + ctx_workers.append(w) + log_suffix = f", logging to {w.log_path}" if w.log_path else "" + print( + f"Launching ctx worker {i + 1}/{num_ctx_instances} on device {device_ids}{log_suffix}" + ) next_device += gpus_per_ctx for i in range(num_gen_instances): device_ids = ",".join( str(d) for d in dict.fromkeys((next_device + j) % num_gpus for j in range(gpus_per_gen))) - print( - f"Launching gen worker {i + 1}/{num_gen_instances} on device {device_ids}" - ) - gen_workers.append( - run_gen_worker(model, + w = run_gen_worker(model, gen_worker_config, work_dir, port=0, device=device_ids, env=env, - save_log=save_log)) + save_log=save_log, + worker_index=i) + gen_workers.append(w) + log_suffix = f", logging to {w.log_path}" if w.log_path else "" + print( + f"Launching gen worker {i + 1}/{num_gen_instances} on device {device_ids}{log_suffix}" + ) next_device += gpus_per_gen # Build minimal server config and launch @@ -728,6 +791,8 @@ def setup_disagg_cluster( server_host, "port": server_port, + "num_workers": + config.get("num_workers", 1), "disagg_cluster": disagg_cluster, "context_servers": { @@ -740,6 +805,10 @@ def setup_disagg_cluster( config.get("conditional_disagg_config", None), "perf_metrics_max_requests": config.get("perf_metrics_max_requests", 0), + "perf_metrics_output_dir": + config.get("perf_metrics_output_dir", None), + "return_perf_metrics": + config.get("return_perf_metrics", False), } if schedule_style: server_config["schedule_style"] = schedule_style @@ -747,15 +816,20 @@ def setup_disagg_cluster( work_dir, server_port, save_log=save_log, - env=env, + env=server_env, cwd=cwd) + all_workers = ctx_workers + gen_workers + async def _wait_with_ticker(): start = time.monotonic() last_tick = start async def _tick(): nonlocal last_tick + last_worker_count = 0 + last_status_log = start + while True: await asyncio.sleep(1) now = time.monotonic() @@ -763,17 +837,86 @@ async def _tick(): startup_callback(now - start) last_tick = now - ticker = asyncio.create_task(_tick()) - try: - await wait_for_disagg_server_ready(server_port, - timeout=server_start_timeout) - finally: - ticker.cancel() + if now - last_status_log >= 10: + elapsed = int(now - start) + remaining = int(server_start_timeout - elapsed) + print( + f"[startup] {elapsed}s elapsed, " + f"{max(remaining, 0)}s left until timeout " + f"(timeout={server_start_timeout}s, " + f"workers registered: {last_worker_count}/" + f"{num_ctx_instances + num_gen_instances})", + flush=True, + ) + last_status_log = now + + # Detect workers that have exited (crash/OOM). + dead = [ + w for w in all_workers if w.process.poll() is not None + ] + if dead: + details = ", ".join( + f"{w.log_path or 'unknown'} (rc={w.process.poll()})" + for w in dead) + raise RuntimeError( + f"{len(dead)} worker(s) exited during startup: " + f"{details}") + + # Track worker registration count for the status log. + try: + timeout = aiohttp.ClientTimeout(total=2) + async with aiohttp.ClientSession( + timeout=timeout) as session: + async with session.get( + f"http://localhost:{server_port}/cluster_info" + ) as info_resp: + if info_resp.status == 200: + workers = (await info_resp.json()).get( + "current_workers", {}) + count = (len( + workers.get("context_servers", [])) + + len( + workers.get( + "generation_servers", []))) + if count > last_worker_count: + last_worker_count = count + except (aiohttp.ClientError, asyncio.TimeoutError): + pass + + # Run the readiness poller and the watchdog concurrently. + # asyncio.wait(FIRST_COMPLETED) means whichever finishes first + # (success or exception) immediately cancels the other, so a dead + # worker detected by _tick() doesn't leave wait_for_disagg_server_ready + # polling silently for up to server_start_timeout seconds. + ready_task = asyncio.create_task( + wait_for_disagg_server_ready(server_port, + timeout=server_start_timeout)) + tick_task = asyncio.create_task(_tick()) + + done, pending = await asyncio.wait( + [ready_task, tick_task], + return_when=asyncio.FIRST_COMPLETED, + ) + + for t in pending: + t.cancel() + try: + await t + except asyncio.CancelledError: + pass + + # Re-raise the first exception. Prefer tick_task's message (it names + # the dead worker); fall back to ready_task (timeout error). + for t in [tick_task, ready_task]: + if t in done and not t.cancelled() and t.exception( + ) is not None: + raise t.exception() asyncio.run(_wait_with_ticker()) except Exception: terminate(*ctx_workers, *gen_workers, disagg_server) - shutil.rmtree(work_dir, ignore_errors=True) + if not save_log: + shutil.rmtree(work_dir, ignore_errors=True) raise return config, ctx_workers, gen_workers, disagg_server, server_port, work_dir @@ -789,7 +932,8 @@ def run_disaggregated_test(example_dir, cwd=None, disagg_schedule_style=None, post_client_test=None, - assert_gen_log_contains=None): + assert_gen_log_contains=None, + perf_metrics_output_dir=None): """Run disaggregated test using service discovery instead of MPI. If assert_gen_log_contains is set, the generation-worker logs are captured and, after the @@ -809,7 +953,8 @@ def run_disaggregated_test(example_dir, config, ctx_workers, gen_workers, disagg_server, server_port, work_dir = \ setup_disagg_cluster(config_file, model_name=model_path, env=run_env, cwd=cwd, schedule_style=disagg_schedule_style, - save_log=assert_gen_log_contains is not None) + save_log=assert_gen_log_contains is not None, + perf_metrics_output_dir=perf_metrics_output_dir) server_host = config.get("hostname", "localhost") @@ -900,6 +1045,24 @@ def test_disaggregated_single_gpu(disaggregated_test_root, cwd=llm_venv.get_working_directory()) +@pytest.mark.parametrize("llama_model_root", ['TinyLlama-1.1B-Chat-v1.0'], + indirect=True) +def test_disaggregated_tinyllama_multi_orchestrator(disaggregated_test_root, + disaggregated_example_root, + llm_venv, llama_model_root): + setup_model_symlink(llm_venv, llama_model_root, + "TinyLlama/TinyLlama-1.1B-Chat-v1.0") + + env = llm_venv._new_env.copy() + env["CUDA_VISIBLE_DEVICES"] = "0" + run_disaggregated_test(disaggregated_example_root, + "multi_orchestrator", + num_iters=1, + env=env, + model_path=llama_model_root, + cwd=llm_venv.get_working_directory()) + + @pytest.mark.parametrize("llama_model_root", ['TinyLlama-1.1B-Chat-v1.0'], indirect=True) def test_disaggregated_benchmark_gen_only(disaggregated_test_root, @@ -923,28 +1086,13 @@ def test_disaggregated_benchmark_gen_only(disaggregated_test_root, indirect=True) def test_disaggregated_router(disaggregated_test_root, disaggregated_example_root, llm_venv, - llama_model_root, router_type, tmp_path): + llama_model_root, router_type): setup_model_symlink(llm_venv, llama_model_root, "TinyLlama/TinyLlama-1.1B-Chat-v1.0") - metrics_file = tmp_path / f"perf_metrics_{router_type}.json" - - def fetch_perf_metrics(server_url: str): - import json - - import requests as http_requests - resp = http_requests.get(f"{server_url}/perf_metrics", timeout=10) - assert resp.status_code == 200, \ - f"Failed to fetch perf_metrics: {resp.status_code}" - metrics = resp.json() - metrics_file.write_text(json.dumps(metrics, indent=2)) - logger.info(f"Router={router_type}: saved {len(metrics)} perf metrics " - f"to {metrics_file}") - run_disaggregated_test(disaggregated_example_root, router_type, env=llm_venv._new_env, - extra_endpoints_test=fetch_perf_metrics, model_path=llama_model_root, cwd=llm_venv.get_working_directory()) @@ -1364,12 +1512,14 @@ def post_client_test(server_url: str): indirect=True) def test_disaggregated_perf_metrics(disaggregated_test_root, llm_venv, disaggregated_example_root, - llama_model_root): + llama_model_root, tmp_path): setup_model_symlink(llm_venv, llama_model_root, "TinyLlama/TinyLlama-1.1B-Chat-v1.0") - def extra_endpoints_test(server_url: str): - item = get_timing_metrics(server_url) + perf_metrics_output_dir = str(tmp_path / "perf_metrics") + + def extra_endpoints_test(_server_url: str): + item = get_timing_metrics(perf_metrics_output_dir) # Use helper function to validate all timing metrics comprehensively validate_timing_metrics(item, "perf_metrics test") @@ -1378,7 +1528,8 @@ def extra_endpoints_test(server_url: str): env=llm_venv._new_env, extra_endpoints_test=extra_endpoints_test, model_path=llama_model_root, - cwd=llm_venv.get_working_directory()) + cwd=llm_venv.get_working_directory(), + perf_metrics_output_dir=perf_metrics_output_dir) @pytest.mark.parametrize("llama_model_root", ['TinyLlama-1.1B-Chat-v1.0'], @@ -1415,10 +1566,18 @@ def test_disaggregated_kv_cache_time_output(disaggregated_test_root, llm_venv, model_path=llama_model_root, cwd=llm_venv.get_working_directory()) assert os.path.isdir(output_path) - send_file = os.path.join(output_path, "rank_0_send.csv") - recv_file = os.path.join(output_path, "rank_0_recv.csv") - assert os.path.exists(send_file) - assert os.path.exists(recv_file) + # The C++ transceiver names timing files "__.csv" + # (instanceId is a runtime UUID that disambiguates instances sharing an + # output directory), so match by the "_.csv" suffix instead of a fixed + # "rank_0" prefix. + send_files = sorted(f for f in os.listdir(output_path) + if f.endswith("_send.csv")) + recv_files = sorted(f for f in os.listdir(output_path) + if f.endswith("_recv.csv")) + assert send_files, f"no *_send.csv in {output_path}: {os.listdir(output_path)}" + assert recv_files, f"no *_recv.csv in {output_path}: {os.listdir(output_path)}" + send_file = os.path.join(output_path, send_files[0]) + recv_file = os.path.join(output_path, recv_files[0]) with open(send_file, "r") as f: lines = f.readlines() assert len(lines) > 1 @@ -1516,12 +1675,12 @@ def test_disaggregated_sa(disaggregated_test_root, llm_venv, @pytest.mark.parametrize("llama_model_root", ['TinyLlama-1.1B-Chat-v1.0'], indirect=True) def test_disaggregated_sa_python(disaggregated_test_root, llm_venv, - disaggregated_example_root, - llama_model_root): - """Spec-split SA (ctx no-spec, gen SA) on the V2 PYTHON transceiver - path (NIXL + transceiver_runtime PYTHON). - The existing test_disaggregated_sa covers this split only on the C++ - DEFAULT backend.""" + disaggregated_example_root, llama_model_root): + """Spec-split SA (ctx no-spec, gen SA) on the V2 PYTHON transceiver path. + + NIXL + transceiver_runtime PYTHON. The existing test_disaggregated_sa + covers this split only on the C++ DEFAULT backend. + """ setup_model_symlink(llm_venv, llama_model_root, "TinyLlama/TinyLlama-1.1B-Chat-v1.0") run_disaggregated_test(disaggregated_example_root, @@ -1995,45 +2154,28 @@ def test_disaggregated_deepseek_v3_lite_bf16_conditional( indirect=True) def test_disaggregated_deepseek_v3_lite_bf16_conditional_v2( disaggregated_test_root, disaggregated_example_root, llm_venv, - deepseek_v3_model_root): + deepseek_v3_model_root, tmp_path): setup_model_symlink(llm_venv, deepseek_v3_model_root, "DeepSeek-V3-Lite/bf16") - # Conditional disagg handles short-prefill requests locally on the gen - # server (bypassing the ctx handoff + add_per_request_metrics), while routed - # requests are recorded in the disagg /perf_metrics. Verify ONCE after all - # client iterations via post_client_test (not per-iteration): routed-request - # metrics are recorded asynchronously (add_per_request_metrics via - # create_task on response completion) and surface only after the client - # traffic settles, so a per-iteration read races that lag; /perf_metrics is - # also consume-on-read, so query it exactly once at the end. - def _check_routed_recorded(server_url: str): - import requests as http_requests - metrics = [] - deadline = time.time() + 60 - while True: - resp = http_requests.get(f"{server_url}/perf_metrics", timeout=10) - assert resp.status_code == 200, \ - f"perf_metrics fetch failed: {resp.status_code}" - metrics = resp.json() - if metrics or time.time() >= deadline: - break - time.sleep(2) - logger.info(f"conditional_v2 perf_metrics len={len(metrics)} " - f"(routed requests recorded; bypassed ones absent)") - # With short prompts every prompt's first occurrence routes through the - # context server (match=0 -> need_ctx), so at least one routed request - # must be recorded; an empty result means conditional routing never - # engaged. - assert metrics, \ - "no per-request metrics recorded after client runs; conditional routing may be misconfigured" + perf_metrics_output_dir = str(tmp_path / "perf_metrics") + + def _check_routed_recorded(_server_url: str): + records = wait_for_perf_metrics_jsonl(perf_metrics_output_dir, + expected_count=3, + timeout=60) + assert any( + {"ctx_perf_metrics", "gen_perf_metrics"} <= record.keys() + for record in records + ), "new prompt did not produce routed context and generation metrics" run_disaggregated_test(disaggregated_example_root, "deepseek_v3_lite_bf16_conditional_v2", env=llm_venv._new_env, post_client_test=_check_routed_recorded, model_path=deepseek_v3_model_root, - cwd=llm_venv.get_working_directory()) + cwd=llm_venv.get_working_directory(), + perf_metrics_output_dir=perf_metrics_output_dir) @skip_no_hopper @@ -2553,9 +2695,9 @@ def test_disaggregated_deepseek_v3_lite_bf16_empty_batch( def test_llama4_long_context_kv_cache_overflow(disaggregated_test_root, disaggregated_example_root, llm_venv, model_path): - """ + """Reproduce the KV cache buffer overflow bug with long context. + RCCA: https://nvbugspro.nvidia.com/bug/5555681 - Test to reproduce KV cache buffer overflow bug with long context. """ models_root = llm_models_root() llama4_model_root = os.path.join(models_root, model_path) @@ -2579,8 +2721,7 @@ def test_llama4_long_context_kv_cache_overflow(disaggregated_test_root, @skip_pre_blackwell @pytest.mark.timeout(2400) @pytest.mark.skip_less_device(4) -@pytest.mark.parametrize("prompt_file", ["prompts.json", "long_prompts.json"], - ids=["short_prompt", "long_prompt"]) +@pytest.mark.parametrize("prompt_file", ["prompts.json"], ids=["short_prompt"]) @pytest.mark.parametrize("deepseek_v3_model_root", ['DeepSeek-V3-Lite-bf16'], indirect=True) def test_disaggregated_deepseek_v3_lite_bf16_tllm_gen_helix( @@ -2787,9 +2928,10 @@ def run_cancel_stress_test(server_url: str, requests_per_burst: int = 32, prompt_len_range: tuple = (2000, 8000), cancel_after_range: tuple = (0.01, 0.1)): - """ - Stress test that sends requests with large contexts and cancels them - during prefill to test resource cleanup under cancellation. + """Stress test cancelling large-context requests mid-prefill. + + Sends requests with large contexts and cancels them during prefill to + test resource cleanup under cancellation. Args: server_url: The server URL (e.g., "http://localhost:8000") @@ -2876,53 +3018,117 @@ class _MixedStressProfile: input_len_range: tuple # (min_tokens, max_tokens) for synthetic prompt output_len: int temperature: float - streaming: bool # JSON schema passed as response_format. None → free-text completion. structured_output_schema: dict - # Probability [0, 1] that this request is cancelled mid-stream. - cancel_probability: float + # Whether this request is cancelled mid-stream (excluded from accuracy). + cancel: bool + # Fixed prompt to send verbatim. None → synthetic filler sized to + # input_len_range. Structured profiles need a real instruction so the + # model has content to emit; otherwise guided decoding degenerates into + # whitespace padding that never closes the JSON. + prompt: str = None + # Free-text accuracy criteria (ignored for structured/cancel): + # expected_substring set → success requires it in the output (real + # correctness check; use with temperature=0 for determinism). + # min_content_chars > 0 → success requires that many non-whitespace + # chars (a "produced real content" check for stochastic profiles; + # tolerant of run-on generation that hits the length cap, which is + # legitimate at high temperature). + # neither → loose: success once the stream completes + # (for context-stress profiles whose output content is secondary). + expected_substring: str = None + min_content_chars: int = 0 + # When False, the profile still drives load (and counts toward + # incomplete_rate) but is excluded from the accuracy metric — for pure + # input/length stress where output content isn't validated. + count_accuracy: bool = True + + +def _profile_accuracy_metric(profile): + """Which per-request field measures this profile's accuracy. + + Returns the result key to sum in the accuracy numerator ("json_valid" for + structured output, "success" for free-text/long-context), or None if the + profile is excluded from accuracy entirely (cancellations, pure stress + profiles). This is the single source of truth for accuracy bucketing — the + aggregation derives from it rather than matching on profile names. + """ + if profile.cancel or not profile.count_accuracy: + return None + if profile.structured_output_schema is not None: + return "json_valid" + return "success" # Default profile mix. Weights are relative; they are normalised in -# _run_mixed_stress_async. Tune during baseline run (item 3 in the plan). +# _run_mixed_stress_async. _DEFAULT_MIXED_STRESS_PROFILES = [ _MixedStressProfile( name='free_text_low_temp', weight=25.0, - input_len_range=(512, 2048), + input_len_range=(512, 2048), # unused: prompt is fixed below output_len=256, temperature=0.0, - streaming=True, structured_output_schema=None, - cancel_probability=0.0, + cancel=False, + # Greedy + known-answer prompt → deterministic correctness check. + prompt="What is the capital of France? Answer in one word.", + expected_substring="paris", ), _MixedStressProfile( name='free_text_high_temp', weight=15.0, - input_len_range=(512, 2048), + input_len_range=(512, 2048), # unused: prompt is fixed below output_len=256, temperature=1.0, - streaming=True, structured_output_schema=None, - cancel_probability=0.0, + cancel=False, + # Open-ended generative prompt → reliably produces real text at high + # temperature (it will run to the length cap, which is fine). No exact + # content to check; require that it produced real (non-whitespace) + # content rather than empty/degenerate output. + prompt="Write a short story about a traveler who discovers a " + "hidden village in the mountains.", + min_content_chars=50, ), _MixedStressProfile( name='structured_output', weight=30.0, - input_len_range=(256, 1024), - output_len=128, + input_len_range=(256, 1024), # unused: prompt is fixed below + output_len=256, temperature=0.0, - streaming=True, + # A concrete instruction that matches the schema below. The model needs + # real content to emit; with a filler prompt it just pads whitespace + # until the token cap and never closes the JSON. + prompt=("Create a JSON object with a key \"pets\" whose value is an " + "array of 4 to 5 objects, each with a \"name\" (a pet's name) " + "and a \"species\". Respond with only the JSON object."), structured_output_schema={ "type": "object", "properties": { - "answer": { - "type": "string" + "pets": { + "type": "array", + "minItems": 4, + "maxItems": 5, + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "maxLength": 20 + }, + "species": { + "type": "string", + "maxLength": 20 + }, + }, + "required": ["name", "species"], + }, } }, - "required": ["answer"], + "required": ["pets"], }, - cancel_probability=0.0, + cancel=False, ), _MixedStressProfile( name='long_context', @@ -2930,9 +3136,22 @@ class _MixedStressProfile: input_len_range=(6000, 8192), output_len=256, temperature=0.7, - streaming=True, structured_output_schema=None, - cancel_probability=0.0, + cancel=False, + ), + _MixedStressProfile( + name='mid_length_stress', + weight=20.0, + input_len_range=(512, 2048), + output_len=256, + temperature=0.7, + structured_output_schema=None, + cancel=False, + # Pure input-length stress over the mid range (the accuracy-checked + # free-text profiles now use fixed short prompts). Output content is + # not validated, so keep it out of the accuracy metric; it still + # counts toward incomplete_rate. + count_accuracy=False, ), _MixedStressProfile( name='cancel', @@ -2940,9 +3159,8 @@ class _MixedStressProfile: input_len_range=(2000, 8000), output_len=64, temperature=0.7, - streaming=True, structured_output_schema=None, - cancel_probability=1.0, + cancel=True, ), ] @@ -2961,15 +3179,18 @@ async def _send_mixed_request(session, """ import random - prompt_len = random.randint(*profile.input_len_range) - prompt = "test " * (prompt_len // 5) + if profile.prompt is not None: + prompt = profile.prompt + else: + prompt_len = random.randint(*profile.input_len_range) + prompt = "test " * (prompt_len // 5) payload = { "model": model_name, "prompt": prompt, "max_tokens": profile.output_len, "temperature": profile.temperature, - "stream": profile.streaming, + "stream": True, } if profile.structured_output_schema is not None: payload["response_format"] = { @@ -2977,7 +3198,7 @@ async def _send_mixed_request(session, "schema": profile.structured_output_schema, } - should_cancel = random.random() < profile.cancel_probability + should_cancel = profile.cancel cancel_after = random.uniform(0.01, 0.1) if should_cancel else None result = { @@ -2986,6 +3207,9 @@ async def _send_mixed_request(session, "json_valid": None, "latency_ms": 0.0, "cancelled": should_cancel, + "timed_out": False, + "server_rejected": False, + "rejection_detail": None, # diagnostic: HTTP status or exception class } start = time.monotonic() @@ -3014,8 +3238,7 @@ async def _send_mixed_request(session, fr = choice.get("finish_reason") if fr is not None: finish_reason = fr - if profile.structured_output_schema is not None: - assembled.append(choice.get("text", "")) + assembled.append(choice.get("text", "")) except (json.JSONDecodeError, IndexError, KeyError): pass @@ -3023,10 +3246,11 @@ async def _send_mixed_request(session, # finish_reason check: server ended with a final chunk carrying # finish_reason (trtllm-serve does not send data: [DONE]) completed = done_received or finish_reason in ("stop", "length") + text = "".join(assembled) if not should_cancel and completed: if profile.structured_output_schema is not None: try: - parsed = json.loads("".join(assembled)) + parsed = json.loads(text) required = profile.structured_output_schema.get( "required", []) result["json_valid"] = all(k in parsed @@ -3034,28 +3258,69 @@ async def _send_mixed_request(session, except json.JSONDecodeError: result["json_valid"] = False result["success"] = result["json_valid"] + elif profile.expected_substring is not None: + # Deterministic correctness check (greedy profiles). + result["success"] = (profile.expected_substring.lower() + in text.lower()) + elif profile.min_content_chars > 0: + # Stochastic profile: no exact content to match, but require + # real (non-whitespace) output rather than empty/degenerate. + content_chars = sum(1 for c in text if not c.isspace()) + result["success"] = (content_chars + >= profile.min_content_chars) else: + # Loose: stream completed (context/length-stress profiles). result["success"] = True - except Exception: - pass # connection abort on cancel is expected + elif not should_cancel and not completed: + # Server closed the stream without a valid completion + # (e.g. kv_transfer_timeout cancellation, error response). + result["server_rejected"] = True + result[ + "rejection_detail"] = f"http_{resp.status}_incomplete_stream" + except asyncio.TimeoutError: + if not should_cancel: + # 120s aiohttp client timeout: server never responded in time. + result["timed_out"] = True + # else: cancel-break may race the deadline; ignore the timeout. + except (aiohttp.ClientError, OSError) as e: + if not should_cancel: + # Transport error on a non-cancel request (e.g. abrupt RST from + # server-side cancellation). HTTP error statuses do not reach here + # (the response is streamed, not raise_for_status'd) — those flow + # through the incomplete-stream path above. + result["server_rejected"] = True + result["rejection_detail"] = f"{type(e).__name__}({e})" + # else: connection abort on intentional cancel is expected finally: result["latency_ms"] = (time.monotonic() - start) * 1000 results.append(result) -async def _run_mixed_stress_async(server_url: str, - profiles: list, - total_requests: int, - concurrency: int, - model_name: str = "test-model", - progress_callback=None, - progress_interval: int = 30) -> dict: +async def _run_mixed_stress_async( + server_url: str, + profiles: list, + total_requests: int, + concurrency: int, + model_name: str = "test-model", + progress_callback=None, + progress_interval: int = 30, + workers=None, + early_abort_rejection_rate: float = 0.05) -> dict: """Drive total_requests requests at the given concurrency level. - Returns a summary dict with per-profile counts and an overall accuracy_score. + Returns a summary dict with per-profile counts and overall metrics: + accuracy_score = (free_text_successes + json_valid_count) - / (free_text_total + structured_total) - Cancelled requests are excluded from the denominator. + / completed_non_cancel_requests + Cancelled, timed-out, and server-rejected requests are excluded from the + accuracy denominator — accuracy reflects only requests that finished. + + incomplete_rate = (timed_out + server_rejected) / non_cancelled_total + timed_out: client-side 120s aiohttp timeout (server never responded). + server_rejected: server closed stream without valid completion (e.g. + kv_transfer_timeout cancellation, error response, connection reset). + Both are reported individually for diagnostics and summed as incomplete_rate + for gating. """ import random @@ -3067,7 +3332,9 @@ async def bounded(coro): async with sem: await coro - async def _progress_monitor(): + abort_reason = [] + + async def _progress_monitor(gather_task): start = time.monotonic() while True: await asyncio.sleep(progress_interval) @@ -3078,12 +3345,37 @@ async def _progress_monitor(): rate = done / elapsed if elapsed > 0 else 0.0 eta = (total_requests - done) / rate if rate > 0 else float('inf') eta_str = f"{eta:.0f}s" if eta != float('inf') else "unknown" - logger.info( - "mixed-stress progress %d/%d (%.0f%%) rate=%.1f/s ETA=%s", done, - total_requests, 100 * done / total_requests, rate, eta_str) + print( + f"[mixed-stress] {done}/{total_requests} " + f"({100 * done / total_requests:.0f}%) " + f"elapsed={elapsed:.0f}s rate={rate:.1f} req/s ETA={eta_str}", + flush=True) if progress_callback: progress_callback(done, total_requests, rate, eta) + # Early-abort checks: worker crash or high rejection rate. + reason = None + if workers: + crashed = _crashed_workers(workers) + if crashed: + parts = [ + f"{w.log_path or 'worker'} (rc={w.process.poll()})" + for w in crashed + ] + reason = f"worker(s) crashed: {parts}" + if reason is None and done > 0: + rejected = sum(1 for r in results if r["server_rejected"]) + if rejected / done > early_abort_rejection_rate: + reason = ( + f"rejection rate {rejected}/{done} " + f"({100*rejected/done:.0f}%) exceeds " + f"{100*early_abort_rejection_rate:.0f}% threshold") + if reason: + abort_reason.append(reason) + print(f"[mixed-stress] aborting early: {reason}", flush=True) + gather_task.cancel() + return + async with aiohttp.ClientSession() as session: chosen_profiles = random.choices(profiles, weights=weights, @@ -3093,41 +3385,123 @@ async def _progress_monitor(): _send_mixed_request(session, server_url, p, results, model_name)) for p in chosen_profiles ] - monitor = asyncio.create_task(_progress_monitor()) - try: + + async def _run_all(): await asyncio.gather(*tasks) + + gather_task = asyncio.create_task(_run_all()) + monitor = asyncio.create_task(_progress_monitor(gather_task)) + try: + await gather_task + except asyncio.CancelledError: + pass finally: monitor.cancel() # Aggregate per_profile: dict = {} for r in results: - p = per_profile.setdefault(r["profile"], { - "total": 0, - "success": 0, - "json_valid": 0, - "cancelled": 0 - }) + p = per_profile.setdefault( + r["profile"], { + "total": 0, + "success": 0, + "json_valid": 0, + "cancelled": 0, + "timed_out": 0, + "server_rejected": 0, + "rejection_details": {}, + }) p["total"] += 1 - if r["success"]: - p["success"] += 1 - if r["json_valid"]: - p["json_valid"] += 1 - if r["cancelled"]: - p["cancelled"] += 1 - - free_text_ok = sum(v["success"] for k, v in per_profile.items() - if "free_text" in k or "long_context" in k) - free_text_total = sum(v["total"] for k, v in per_profile.items() - if "free_text" in k or "long_context" in k) - json_ok = sum(v["json_valid"] for k, v in per_profile.items() - if "structured" in k) - json_total = sum(v["total"] for k, v in per_profile.items() - if "structured" in k) - denom = free_text_total + json_total - accuracy_score = (free_text_ok + json_ok) / denom if denom > 0 else 0.0 - - return {"per_profile": per_profile, "accuracy_score": accuracy_score} + for key in ("success", "json_valid", "cancelled", "timed_out", + "server_rejected"): + if r[key]: + p[key] += 1 + if r["rejection_detail"]: + rd = p["rejection_details"] + rd[r["rejection_detail"]] = rd.get(r["rejection_detail"], 0) + 1 + + def _completed(v): + return v["total"] - v["cancelled"] - v["timed_out"] - v[ + "server_rejected"] + + metric_by_name = {p.name: _profile_accuracy_metric(p) for p in profiles} + accuracy_ok = 0 + accuracy_denom = 0 + for k, v in per_profile.items(): + metric = metric_by_name.get(k) + # Annotate each profile with its accuracy contribution so callers can + # spot a category that silently produced nothing. metric is None for + # profiles excluded from accuracy (e.g. cancellations). + v["accuracy_metric"] = metric + if metric is None: + v["accuracy_completed"] = 0 + v["accuracy_ok"] = 0 + v["accuracy_rate"] = None + continue + ok = v[metric] + completed = _completed(v) + v["accuracy_completed"] = completed + v["accuracy_ok"] = ok + v["accuracy_rate"] = (ok / completed) if completed > 0 else None + accuracy_ok += ok + accuracy_denom += completed + accuracy_score = accuracy_ok / accuracy_denom if accuracy_denom > 0 else 0.0 + + total_non_cancelled = sum(v["total"] - v["cancelled"] + for v in per_profile.values()) + total_timed_out = sum(v["timed_out"] for v in per_profile.values()) + total_server_rejected = sum(v["server_rejected"] + for v in per_profile.values()) + incomplete_rate = ((total_timed_out + total_server_rejected) / + total_non_cancelled if total_non_cancelled > 0 else 0.0) + + # Merge per-profile rejection_details into a global breakdown + all_rejection_details: dict = {} + for v in per_profile.values(): + for reason, count in v["rejection_details"].items(): + all_rejection_details[reason] = ( + all_rejection_details.get(reason, 0) + count) + + return { + "per_profile": per_profile, + "accuracy_score": accuracy_score, + "incomplete_rate": incomplete_rate, + "timed_out": total_timed_out, + "server_rejected": total_server_rejected, + "rejection_details": all_rejection_details, + "aborted_early": abort_reason or None, + } + + +async def _warmup_requests(server_url: str, profiles: list, count: int, + concurrency: int, model_name: str) -> None: + """Send `count` requests and discard their results to warm the cluster. + + The first request of each shape pays a one-time autotuner/compile cost + (~20s host-steps observed on B200). Running those here, before the measured + run, keeps them out of the accuracy/incomplete accounting and out of the + heartbeat-eviction path (a worker stuck in a 20s step misses the 2s cluster + heartbeat and gets evicted under a high-concurrency flood). Failures are + ignored — the only goal is to trigger the autotuner across the profile mix. + """ + import random + + weights = [p.weight for p in profiles] + sem = asyncio.Semaphore(concurrency) + sink: list = [] # discarded; _send_mixed_request swallows its own errors + + async def bounded(coro): + async with sem: + await coro + + async with aiohttp.ClientSession() as session: + chosen = random.choices(profiles, weights=weights, k=count) + tasks = [ + bounded( + _send_mixed_request(session, server_url, p, sink, model_name)) + for p in chosen + ] + await asyncio.gather(*tasks, return_exceptions=True) def run_disaggregated_mixed_stress(example_dir: str, @@ -3136,8 +3510,11 @@ def run_disaggregated_mixed_stress(example_dir: str, total_requests: int = 10000, concurrency: int = 512, accuracy_threshold: float = 0.42, + incomplete_threshold: float = 0.05, + per_profile_min_sample: int = 10, + warmup_request_count: int = None, profiles: list = None, - server_start_timeout: int = 7200, + server_start_timeout: int = 600, env=None, cwd=None, startup_callback=None, @@ -3154,14 +3531,14 @@ def run_disaggregated_mixed_stress(example_dir: str, Default total_requests is 10000, a conservative starting point chosen to keep CI runtime manageable. The target workload (TRTLLM-12154) runs 20-100k requests per stability run; 10k covers the feature-mixing code - paths without the full wall-clock cost. Accuracy threshold should be - tightened after a baseline run establishes a real floor. + paths without the full wall-clock cost. - Observed wall-clock on 8x B200 (umbriel): ~17 min (1048s) for a - 500-request run, of which ~16 min (961s) was the request phase and - ~87s was server startup. Request phase scales roughly linearly with - request count: the 60-request smoke variant targets ~2 min of requests, - and the 10k full variant ~32 min of requests. + Request phase scales roughly linearly with request count on 8x B200: + the 60-request smoke variant runs in ~2 min of requests, a 5k run in + ~14 min, and the 10k full variant ~32 min (plus ~1.5 min server + startup). Each request profile is validated for real content + (structured JSON schema, greedy known-answer, generative min-content), + so a healthy cluster scores ~1.0 accuracy. Default concurrency is 512 rather than the ~32 typical of production stability runs. Higher concurrency exercises more in-flight request @@ -3176,10 +3553,13 @@ def run_disaggregated_mixed_stress(example_dir: str, run_env["UCX_TLS"] = get_ucx_tls() run_env["UCX_MM_ERROR_HANDLING"] = "y" + setup_start = time.monotonic() config, ctx_workers, gen_workers, disagg_server, server_port, work_dir = \ setup_disagg_cluster(config_file, model_name=model_path, env=run_env, cwd=cwd, server_start_timeout=server_start_timeout, save_log=True, startup_callback=startup_callback) + print(f"[startup] cluster ready in {time.monotonic() - setup_start:.1f}s", + flush=True) server_host = config.get("hostname", "localhost") server_url = f"http://{server_host}:{server_port}" @@ -3191,15 +3571,76 @@ def run_disaggregated_mixed_stress(example_dir: str, f"Disaggregated server did not become ready within " f"{server_start_timeout}s") + # Pay the one-time autotuner cost before the measured run. The first + # request of each shape triggers a ~20s autotuner host-step; left in + # the measured run at high concurrency, a worker stuck in that step + # misses the 2s cluster heartbeat and is evicted mid-run, causing + # "Cluster is not ready" 500s. Default count = ~20s at the ~6 req/s + # observed in the 5k B200 run; results are discarded. + if warmup_request_count is None: + warmup_request_count = 120 + print( + f"[mixed-stress] warmup: {warmup_request_count} requests " + f"(results discarded)", + flush=True) + asyncio.run( + _warmup_requests(server_url, profiles, warmup_request_count, + concurrency, model_path)) + summary = asyncio.run( - _run_mixed_stress_async(server_url, - profiles, - total_requests, - concurrency, - model_name=model_path, - progress_callback=progress_callback)) + _run_mixed_stress_async( + server_url, + profiles, + total_requests, + concurrency, + model_name=model_path, + progress_callback=progress_callback, + workers=ctx_workers + gen_workers, + early_abort_rejection_rate=incomplete_threshold)) - logger.info("Mixed stress summary: %s", summary) + print( + f"Mixed stress summary: " + f"accuracy={summary['accuracy_score']:.3f} " + f"incomplete_rate={summary['incomplete_rate']:.3f} " + f"(timed_out={summary['timed_out']} " + f"server_rejected={summary['server_rejected']})\n" + f"per_profile:\n{json.dumps(summary['per_profile'], indent=2)}", + flush=True) + if summary["rejection_details"]: + print( + f"Mixed stress rejection breakdown:\n" + f"{json.dumps(summary['rejection_details'], indent=2)}", + flush=True) + + # Per-profile accuracy rates. Pooled accuracy can hide a category that + # silently produced nothing (e.g. structured output scoring 0% while + # free-text carries the average), so surface each profile's rate and + # fail on any profile that had enough completions but zero successes. + dead_profiles = [] + for name, v in summary["per_profile"].items(): + if v["accuracy_metric"] is None: + continue + rate = v["accuracy_rate"] + rate_str = f"{rate:.3f}" if rate is not None else "n/a" + print( + f"[mixed-stress] profile {name}: " + f"{v['accuracy_ok']}/{v['accuracy_completed']} " + f"({v['accuracy_metric']}) rate={rate_str}", + flush=True) + if (v["accuracy_completed"] >= per_profile_min_sample + and v["accuracy_ok"] == 0): + dead_profiles.append(f"{name} (0/{v['accuracy_completed']})") + + if summary["aborted_early"]: + raise RuntimeError(f"[mixed-stress] aborted early: " + f"{'; '.join(summary['aborted_early'])}") + + if dead_profiles: + raise AssertionError( + f"Mixed stress: profile(s) produced zero successful results " + f"despite >= {per_profile_min_sample} completed requests: " + f"{dead_profiles}. A whole request category is silently " + f"failing while pooled accuracy stays above threshold.") score = summary["accuracy_score"] if score < accuracy_threshold: @@ -3208,6 +3649,15 @@ def run_disaggregated_mixed_stress(example_dir: str, f"{accuracy_threshold:.3f}. Per-profile: " f"{summary['per_profile']}") + incomplete = summary["incomplete_rate"] + if incomplete > incomplete_threshold: + raise AssertionError( + f"Mixed stress incomplete_rate {incomplete:.3f} above threshold " + f"{incomplete_threshold:.3f} " + f"(timed_out={summary['timed_out']} " + f"server_rejected={summary['server_rejected']} " + f"details={summary['rejection_details']})") + # Verify server still healthy after the stress run. # Probe /v1/chat/completions (not /v1/models) — the known failure mode # is the event loop dying while /v1/models keeps returning 200. @@ -3235,8 +3685,12 @@ def run_disaggregated_mixed_stress(example_dir: str, logger.error("Mixed stress test failed") raise finally: + diag = build_worker_diag(ctx_workers + gen_workers, disagg_server) + if diag: + print(f"[mixed-stress] worker health:{diag}", flush=True) terminate(*ctx_workers, *gen_workers, disagg_server) - shutil.rmtree(work_dir, ignore_errors=True) + if diag: + raise AssertionError(f"Mixed stress detected worker failure.{diag}") def run_disaggregated_cancel_test(example_dir, @@ -3588,27 +4042,28 @@ def test_disaggregated_mamba_conc_greater_than_mbs(disaggregated_example_root, @pytest.mark.parametrize( "test_config", [ - # Smoke run: 60 requests at 64 concurrency, ~2 min request phase on - # B200 (scaled down from 500-req baseline of ~17.5 min). Used as L0 - # post-merge gate. + # Smoke run: 120 requests at 64 concurrency (matching the warmup + # count), ~3 min request phase on B200. Used as L0 post-merge gate. A + # healthy cluster scores ~1.0 accuracy (every profile validates real + # content), so 0.9 leaves margin for a rare flaky request while still + # catching a regression. pytest.param(TestConfig( model_path='Qwen3/Qwen3-32B-FP8', - test_desc='req60-conc64-qwen3_32b_fp8_mixed_stress', - request_count=60, + test_desc='req120-conc64-qwen3_32b_fp8_mixed_stress', + request_count=120, concurrency=64, - accuracy_threshold=0.42, + accuracy_threshold=0.9, speculative_model_path='Zhi-Create-Qwen3-32B-Eagle3'), marks=(pytest.mark.skip_less_device(8), skip_pre_hopper)), # Full stress run: 10k requests at 512 concurrency. - # Estimated wall-clock: 1-2 hours (server startup ~5-10 min + request - # phase; based on 500-req baseline of ~17.5 min at 64 concurrency on - # B200, scaled to 512 concurrency which doesn't multiply rate 1:1). + # Estimated wall-clock ~40 min (server startup + ~32 min request + # phase); 512 concurrency exercises more in-flight overlap. pytest.param(TestConfig( model_path='Qwen3/Qwen3-32B-FP8', test_desc='req10k-conc512-qwen3_32b_fp8_mixed_stress', request_count=10000, concurrency=512, - accuracy_threshold=0.42, + accuracy_threshold=0.9, speculative_model_path='Zhi-Create-Qwen3-32B-Eagle3'), marks=(pytest.mark.skip_less_device(8), skip_pre_hopper)), ], @@ -3660,6 +4115,7 @@ def test_disaggregated_mixed_stress_test(disaggregated_test_root, total_requests=test_config.request_count, concurrency=test_config.concurrency, accuracy_threshold=test_config.accuracy_threshold, - server_start_timeout=7200, + incomplete_threshold=test_config.incomplete_threshold, + server_start_timeout=600, env=llm_venv._new_env, cwd=llm_venv.get_working_directory()) diff --git a/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py b/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py index 2d5b5000005a..97634b686ea5 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py +++ b/tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py @@ -138,6 +138,7 @@ async def run_worker(kv_cache_config, print(f"Sending ready signal to main process") intercomm.send(intercomm.Get_rank(), dest=0, tag=MPI_READY) + last_cached_tokens = 0 print(f"Waiting for requests") while True: try: @@ -146,6 +147,15 @@ async def run_worker(kv_cache_config, if requests is None: break + # Handle special commands + if requests == "GET_STATE": + state = llm.get_data_transceiver_state() + intercomm.send(state, dest=0, tag=MPI_RESULT) + continue + if requests == "GET_LAST_CACHED_TOKENS": + intercomm.send(last_cached_tokens, dest=0, tag=MPI_RESULT) + continue + request_metas = [] futures = [] for i, request in enumerate(requests): @@ -214,8 +224,12 @@ async def run_worker(kv_cache_config, f"Worker {rank}: awaiting future {i}/{len(futures)}", flush=True) result = await future - print(f"Worker {rank}: got result {i}, sending", - flush=True) + last_cached_tokens = getattr( + result, 'cached_tokens', 0) + print( + f"Worker {rank}: got result {i}, " + f"cached_tokens={last_cached_tokens}, sending", + flush=True) intercomm.send(result.outputs, dest=0, tag=MPI_RESULT) @@ -612,6 +626,7 @@ def test_disaggregated_spec_dec_batch_slot_limit(model, spec_dec_model_path, @pytest.mark.parametrize("generation_overlap", [False, True]) def test_disaggregated_logprobs(model, generation_overlap): """Verify that logprobs propagate correctly from prefill to decode. + Ensures first_gen_log_probs is carried in DisaggregatedParams so the generation_only worker receives one logprob per token. """ @@ -956,5 +971,282 @@ def test_disaggregated_logits(model, generation_overlap): print("All workers terminated.") +@pytest.mark.parametrize("model", ["TinyLlama-1.1B-Chat-v1.0"]) +@pytest.mark.parametrize("generation_overlap", [False]) +def test_arbitrary_kv_cache_transfer(model, generation_overlap): + """Test KV cache transfer from the reuse tree. + + Flow: + 1. Worker 0 runs a normal generate to fill the KV cache reuse tree. + 2. Retrieve the data_transceiver_state from worker 0. + 3. Worker 1 sends a generation_only request using the state from step 2. + The sender (worker 0) serves blocks directly from its reuse tree. + """ + worker_pytorch_configs = [] + + # Worker 0 (sender) + worker_pytorch_configs.append( + dict(disable_overlap_scheduler=True, + cuda_graph_config=CudaGraphConfig())) + + # Worker 1 (receiver) + worker_pytorch_configs.append( + dict(disable_overlap_scheduler=not generation_overlap, + cuda_graph_config=CudaGraphConfig())) + + kv_cache_configs = [ + KvCacheConfig(max_tokens=2048 * 8, enable_block_reuse=True) + for _ in range(2) + ] + cache_transceiver_configs = [ + CacheTransceiverConfig(backend="DEFAULT") for _ in range(2) + ] + model_names = [model_path(model) for _ in range(2)] + ranks = [0, 1] + worker_args = list( + zip(kv_cache_configs, cache_transceiver_configs, worker_pytorch_configs, + model_names, ranks)) + + port_name = mpi_publish_name() + + prompt = "What is the capital of Germany?" + + with MPIPoolExecutor(max_workers=2, + env={ + "UCX_TLS": "^ib,gdr_copy", + "UCX_MM_ERROR_HANDLING": "y" + }) as executor: + futures = [] + try: + for worker_arg in worker_args: + future = executor.submit(worker_entry_point, *worker_arg) + futures.append(future) + except Exception as e: + print(f"Error in worker {worker_arg}: {e}") + raise e + + intercomm = None + try: + print("Launched all the workers.", flush=True) + intercomm = mpi_initialize_intercomm(port_name) + + for _ in range(2): + intercomm.recv(tag=MPI_READY) + print("Received ready signal.") + + # Normal generate on worker 0 to fill KV cache reuse tree + print("Filling KV cache reuse tree on worker 0", flush=True) + requests = [(prompt, SamplingParams(max_tokens=10, + ignore_eos=True), None)] + responses = send_requests_to_worker(requests, 0, intercomm) + assert len(responses) == 1 + print(f"Worker 0 output: {responses[0][0].text}", flush=True) + + # Get data_transceiver_state from worker 0 + print("Getting data_transceiver_state from worker 0", flush=True) + intercomm.send("GET_STATE", dest=0, tag=MPI_REQUEST) + state = intercomm.recv(source=0, tag=MPI_RESULT) + assert isinstance(state, bytes) + assert len(state) > 0 + print(f"Got data_transceiver_state: {len(state)} bytes", flush=True) + + # generation_only on worker 1 using state from worker 0. + # max_tokens=1 ensures only one decode step runs, so the test + # relies on the KV cache transfer delivering all prompt tokens. + print("Sending generation_only to worker 1", flush=True) + DISAGG_REQ_ID = 42 + disagg_params = DisaggregatedParams( + request_type="generation_only", + opaque_state=state, + first_gen_tokens=[0], + disagg_request_id=DISAGG_REQ_ID, + ) + requests = [(prompt, SamplingParams(max_tokens=1, ignore_eos=True), + disagg_params)] + responses = send_requests_to_worker(requests, 1, intercomm) + assert len(responses) == 1 + output = responses[0][0] + print(f"Worker 1 output: {output.text}", flush=True) + print(f"Worker 1 token_ids: {output.token_ids}", flush=True) + assert len(output.token_ids) > 0, \ + "generation_only request should produce output tokens" + + # Send a normal (non-disagg) request to worker 1 to verify + # it still works properly after the KV cache transfer. + print("Normal request on worker 1 after transfer", flush=True) + requests = [(prompt, SamplingParams(max_tokens=10, + ignore_eos=True), None)] + responses = send_requests_to_worker(requests, 1, intercomm) + assert len(responses) == 1 + output2 = responses[0][0] + print(f"Worker 1 normal output: {output2.text}", flush=True) + assert len(output2.token_ids) > 0, \ + "Normal request should produce output tokens after transfer" + + # Query cached_tokens from worker 1's normal request + intercomm.send("GET_LAST_CACHED_TOKENS", dest=1, tag=MPI_REQUEST) + cached1 = intercomm.recv(source=1, tag=MPI_RESULT) + print(f"Worker 1 normal cached_tokens: {cached1}", flush=True) + + # Send a completely different prompt to worker 1 to confirm + # cached_tokens is 0 when there is no reuse match. + different_prompt = "Bonjour le monde, comment allez-vous aujourd'hui" + print("Different prompt on worker 1 (no match expected)", + flush=True) + requests = [(different_prompt, + SamplingParams(max_tokens=5, ignore_eos=True), None)] + responses = send_requests_to_worker(requests, 1, intercomm) + assert len(responses) == 1 + intercomm.send("GET_LAST_CACHED_TOKENS", dest=1, tag=MPI_REQUEST) + cached2 = intercomm.recv(source=1, tag=MPI_RESULT) + print(f"Worker 1 different prompt cached_tokens: {cached2}", + flush=True) + # BOS token may be shared, so at most 1 cached token is expected + assert cached2 <= 1, \ + f"Expected at most 1 cached token for unrelated prompt, got {cached2}" + assert cached2 < cached1, \ + "Unrelated prompt should have fewer cached tokens than transferred prompt" + + except Exception as e: + print(f"Exception encountered: {e}", flush=True) + raise + finally: + print("Sending termination request", flush=True) + mpi_send_termination_request(intercomm) + + print("Waiting for all workers to terminate. ", flush=True) + for future in futures: + future.result() + print("All workers terminated.") + + +@pytest.mark.parametrize("model", ["TinyLlama-1.1B-Chat-v1.0"]) +@pytest.mark.parametrize("generation_overlap", [False]) +def test_arbitrary_kv_cache_transfer_missing_blocks(model, generation_overlap): + """Test that missing-block transfers fail. + + When the receiver asks for blocks that don't exist on the sender, + the sender must notify the receiver so the request surfaces a + clear error instead of waiting for the hang detector. + + Flow: + 1. Worker 0 (sender) keeps an EMPTY KV cache reuse tree (no prior + generate call). Its reuse tree therefore has no blocks matching + any prompt the receiver might request. + 2. Retrieve the data_transceiver_state from worker 0. + 3. Worker 1 (receiver) sends a generation_only request using that + state. The sender attempts to look up the receiver's prompt + blocks in its empty reuse tree and fails with + "Couldn't find the requested block in the reuse tree". + 4. The receiver surfaces the failure as an error string. + """ + worker_pytorch_configs = [] + + # Worker 0 (sender) + worker_pytorch_configs.append( + dict(disable_overlap_scheduler=True, + cuda_graph_config=CudaGraphConfig())) + + # Worker 1 (receiver) + worker_pytorch_configs.append( + dict(disable_overlap_scheduler=not generation_overlap, + cuda_graph_config=CudaGraphConfig())) + + kv_cache_configs = [ + KvCacheConfig(max_tokens=2048 * 8, enable_block_reuse=True) + for _ in range(2) + ] + cache_transceiver_configs = [ + CacheTransceiverConfig(backend="DEFAULT") for _ in range(2) + ] + model_names = [model_path(model) for _ in range(2)] + ranks = [0, 1] + worker_args = list( + zip(kv_cache_configs, cache_transceiver_configs, worker_pytorch_configs, + model_names, ranks)) + + port_name = mpi_publish_name() + + prompt = "What is the capital of Germany?" + + with MPIPoolExecutor(max_workers=2, + env={ + "UCX_TLS": "^ib,gdr_copy", + "UCX_MM_ERROR_HANDLING": "y" + }) as executor: + futures = [] + try: + for worker_arg in worker_args: + future = executor.submit(worker_entry_point, *worker_arg) + futures.append(future) + except Exception as e: + print(f"Error in worker {worker_arg}: {e}") + raise e + + intercomm = None + try: + print("Launched all the workers.", flush=True) + intercomm = mpi_initialize_intercomm(port_name) + + for _ in range(2): + intercomm.recv(tag=MPI_READY) + print("Received ready signal.") + + # Get state from worker 0 + print("Getting data_transceiver_state from empty worker 0", + flush=True) + intercomm.send("GET_STATE", dest=0, tag=MPI_REQUEST) + state = intercomm.recv(source=0, tag=MPI_RESULT) + assert isinstance(state, bytes) + assert len(state) > 0 + print(f"Got data_transceiver_state: {len(state)} bytes", flush=True) + + # generation_only on worker 1 — sender has no blocks to serve. + print("Sending generation_only to worker 1 (sender has no blocks)", + flush=True) + DISAGG_REQ_ID = 43 + disagg_params = DisaggregatedParams( + request_type="generation_only", + opaque_state=state, + first_gen_tokens=[0], + disagg_request_id=DISAGG_REQ_ID, + ) + requests = [(prompt, SamplingParams(max_tokens=10, ignore_eos=True), + disagg_params)] + responses = send_requests_to_worker(requests, 1, intercomm) + assert len(responses) == 1 + response = responses[0] + print(f"Worker 1 response: {response}", flush=True) + assert isinstance(response, str), \ + f"Expected error string when sender has no blocks, got: {response!r}" + assert "cache transfer" in response.lower(), \ + f"Expected cache-transfer error, got: {response!r}" + + # Verify worker 1 still works for normal (non-disagg) requests + # after the failed transfer attempt. + print("Normal request on worker 1 after failed transfer", + flush=True) + requests = [(prompt, SamplingParams(max_tokens=10, + ignore_eos=True), None)] + responses = send_requests_to_worker(requests, 1, intercomm) + assert len(responses) == 1 + output = responses[0][0] + print(f"Worker 1 normal output: {output.text}", flush=True) + assert len(output.token_ids) > 0, \ + "Worker 1 should still serve normal requests after failed transfer" + + except Exception as e: + print(f"Exception encountered: {e}", flush=True) + raise + finally: + print("Sending termination request", flush=True) + mpi_send_termination_request(intercomm) + + print("Waiting for all workers to terminate. ", flush=True) + for future in futures: + future.result() + print("All workers terminated.") + + if __name__ == "__main__": pytest.main() diff --git a/tests/integration/defs/disaggregated/test_workers.py b/tests/integration/defs/disaggregated/test_workers.py index 35149391bf32..75259f782ddb 100644 --- a/tests/integration/defs/disaggregated/test_workers.py +++ b/tests/integration/defs/disaggregated/test_workers.py @@ -1,3 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import asyncio import contextlib import copy @@ -409,10 +423,10 @@ async def multi_round_request(self, prompt=request["prompt"], disaggregated_params=DisaggregatedParams( request_type="context_only")) - ctx_server, ctx_info = await self.ctx_router.get_next_server( - openai_request) + ctx_server, _ = await self.ctx_router.get_next_server(openai_request + ) prompt_str = request["prompt"] - request["prompt"] = ctx_info["token_lists"][0] + request["prompt"] = openai_request.prompt openai_request.disaggregated_params.request_type = "generation_only" gen_server, _ = await self.gen_router.get_next_server(openai_request ) diff --git a/tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py b/tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py deleted file mode 100644 index fffe95a4a461..000000000000 --- a/tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -from pathlib import Path - -from tensorrt_llm import SamplingParams -from tensorrt_llm._tensorrt_engine import LLM -from tensorrt_llm.llmapi import QuantAlgo, QuantConfig - -prompts = [ - "Hello, my name is", - "The president of the United States is", - "The capital of France is", - "The future of AI is", -] -sampling_params = SamplingParams(temperature=0.8, top_p=0.95) - -model_path = Path( - os.environ.get("LLM_MODELS_ROOT")) / "llama-models-v2/llama-v2-70b-chat-hf" -print(f'model_path: {model_path}') -print(f'gpus: {os.environ.get("CUDA_VISIBLE_DEVICES")}') - - -def main(): - - quant_config = QuantConfig(quant_algo=QuantAlgo.FP8, - kv_cache_quant_algo=QuantAlgo.FP8) - - llm = LLM(model=str(model_path), - quant_config=quant_config, - tensor_parallel_size=2) - - outputs = llm.generate(prompts, sampling_params) - - # Print the outputs. - for output in outputs: - prompt = output.prompt - generated_text = output.outputs[0].text - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") - - -if __name__ == '__main__': - main() diff --git a/tests/integration/defs/examples/test_bert.py b/tests/integration/defs/examples/test_bert.py deleted file mode 100644 index f0268325ea07..000000000000 --- a/tests/integration/defs/examples/test_bert.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Module test_bert test bert examples.""" -import pytest -from defs.common import convert_weights, venv_check_call, venv_mpi_check_call -from defs.conftest import get_device_count, get_sm_version -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -# # Build parameters -@pytest.mark.parametrize( - "model, hf_bert_model_root", - [("BertModel", 'bert/bert-base-uncased'), - ("BertForQuestionAnswering", 'bert/bert-base-cased-squad2'), - ("BertForSequenceClassification", 'bert/bert-base-uncased-yelp-polarity'), - ("RobertaModel", 'bert/roberta-base'), - ("RobertaForQuestionAnswering", 'bert/roberta-base-squad2'), - ("RobertaForSequenceClassification", 'bert/twitter-roberta-base-emotion')]) -@pytest.mark.parametrize("dtype", ["float32", "float16"]) -@pytest.mark.parametrize("pp_size", [1], ids=lambda pp_size: f'pp:{pp_size}') -@pytest.mark.parametrize("tp_size", [1, 2], ids=lambda tp_size: f'tp:{tp_size}') -@pytest.mark.parametrize( - "use_attention_plugin, context_fmha_type", [(True, 'enabled'), - (True, 'enabled_with_fp32_acc'), - (True, 'disabled'), - (False, 'disabled')], - ids=[ - 'use_attention_plugin-enable_context_fmha', - 'use_attention_plugin-enable_context_fmha_fp32_acc', - 'use_attention_plugin-disable_context_fmha', - 'disable_attention_plugin-disable_context_fmha', - ]) -@pytest.mark.parametrize( - "remove_input_padding", [True, False], - ids=["enable_remove_input_padding", "disable_remove_input_padding"]) -# Run parameters -@pytest.mark.parametrize("compare_hf", [True], ids=["compare_hf"]) -def test_llm_bert_general(bert_example_root, llm_venv, model, dtype, pp_size, - tp_size, use_attention_plugin, context_fmha_type, - hf_bert_model_root, bert_model_root, compare_hf, - cmodel_dir, engine_dir, remove_input_padding): - "Run bert for float16 and float32" - world_size = tp_size * pp_size - - if get_device_count() < world_size: - pytest.skip( - f"Running world size {world_size} on a node with only {get_device_count()} devices. Skip the test..." - ) - - print("Locate model checkpoints in test storage...") - hf_model_name, model_ckpt_path = bert_model_root - - remove_padding = remove_input_padding - if not use_attention_plugin: - remove_padding = False - else: - if get_sm_version() >= 100 and get_sm_version() < 120: - pytest.skip("Attention plugin is not supported on SM100") - - # Convert checkpoints - converted_weight_dir = convert_weights(llm_venv=llm_venv, - example_root=bert_example_root, - cmodel_dir=cmodel_dir, - model=model, - model_path=model_ckpt_path, - data_type=dtype, - tp_size=tp_size) - - # Build Engine - bert_engine_dir = f"{engine_dir}/{model}/{world_size}-gpus/{dtype}/remove_padding_{remove_padding}" - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}", - f"--output_dir={bert_engine_dir}", - "--max_batch_size=8", - ] - - if use_attention_plugin: - build_cmd.append(f"--bert_attention_plugin={dtype}") - else: - build_cmd.append(f"--bert_attention_plugin=disable") - if remove_input_padding and use_attention_plugin: - build_cmd.extend(["--remove_input_padding=enable"]) - else: - build_cmd.extend(["--remove_input_padding=disable"]) - - if context_fmha_type == 'enabled': - build_cmd.extend(["--context_fmha=enable"]) - if context_fmha_type == 'enabled_with_fp32_acc': - build_cmd.extend(["--bert_context_fmha_fp32_acc=enable"]) - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - # Run Engine - print("Run inference...") - run_cmd = [ - f"{bert_example_root}/run.py", - f"--engine_dir={bert_engine_dir}", - f"--hf_model_dir={model_ckpt_path}", - ] - if remove_input_padding and use_attention_plugin: - run_cmd.extend(["--remove_input_padding"]) - if compare_hf: - run_cmd.extend(["--run_hf_test"]) - if world_size == 1: - venv_check_call(llm_venv, run_cmd) - else: - venv_mpi_check_call( - llm_venv, ["mpirun", "-n", - str(world_size), "--allow-run-as-root"], run_cmd) diff --git a/tests/integration/defs/examples/test_chatglm.py b/tests/integration/defs/examples/test_chatglm.py deleted file mode 100644 index 37ee4a1c1740..000000000000 --- a/tests/integration/defs/examples/test_chatglm.py +++ /dev/null @@ -1,83 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Module test_chatglm test chatglm examples.""" -import os -import shutil - -import pytest -from defs.common import convert_weights, venv_check_call -from defs.conftest import get_sm_version, skip_post_blackwell -from defs.trt_test_alternative import check_call, exists - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -# TODO: add more test case for input_padding, paged_kv_cache, num_beams -@pytest.mark.skip_less_device_memory(24000) -@pytest.mark.parametrize("use_weight_only", - [pytest.param(True, marks=skip_post_blackwell), False], - ids=["enable_weight_only", "disable_weight_only"]) -@pytest.mark.parametrize("llm_glm_4_9b_model_root", - ["glm-4-9b", "glm-4-9b-chat"], - indirect=True) -def test_llm_glm_4_9b_single_gpu_summary(glm_4_9b_example_root, - llm_glm_4_9b_model_root, - llm_datasets_root, llm_rouge_root, - llm_venv, cmodel_dir, engine_dir, - use_weight_only): - "Build & run glm-4-9b on single gpu." - print("Converting checkpoint...") - dtype = 'float16' - model_name = os.path.basename(llm_glm_4_9b_model_root) - ckpt_dir = convert_weights(llm_venv=llm_venv, - example_root=glm_4_9b_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llm_glm_4_9b_model_root, - data_type=dtype, - use_weight_only=use_weight_only) - - print("Building engines...") - build_cmd = [ - "trtllm-build", f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", f"--max_batch_size={8}", - f"--max_input_len={924}", f"--max_seq_len={1024}", - f"--gpt_attention_plugin={dtype}" - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Running inference...") - - # fix HF error in glm-4-9b, hope to remove this in the future - # nvbug 5025895 - model_temp_dir = glm_4_9b_example_root + "/model_temp_dir" - if not exists(model_temp_dir): - shutil.copytree(llm_glm_4_9b_model_root, model_temp_dir) - shutil.copy(glm_4_9b_example_root + "/tokenization_chatglm.py", - model_temp_dir) - - summary_cmd = [ - f"{glm_4_9b_example_root}/../../../summarize.py", "--test_trt_llm", - "--hf_model_dir", f"{model_temp_dir}", "--data_type", "fp16", - "--check_accuracy", f"--engine_dir={engine_dir}", - f"--dataset_dir={llm_datasets_root}", f"--rouge_dir={llm_rouge_root}" - ] - - venv_check_call(llm_venv, summary_cmd) diff --git a/tests/integration/defs/examples/test_commandr.py b/tests/integration/defs/examples/test_commandr.py deleted file mode 100644 index bf6c97ec63d8..000000000000 --- a/tests/integration/defs/examples/test_commandr.py +++ /dev/null @@ -1,83 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Module test_commandr test commandr examples.""" -import os - -import pytest -from defs.common import convert_weights, venv_check_call -from defs.conftest import (get_gpu_device_list, get_sm_version, - skip_post_blackwell) -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@pytest.mark.skip_less_device_memory(80000) -@skip_post_blackwell -@pytest.mark.parametrize("use_weight_only", [True, False], - ids=["enable_weight_only", "disable_weight_only"]) -def test_llm_commandr_v01_single_gpu_summary(commandr_example_root, - llm_commandr_v01_model_root, - llm_datasets_root, llm_rouge_root, - llm_venv, cmodel_dir, engine_dir, - use_weight_only): - "Build & run commandr_v01 on single gpu." - if "GH200" in get_gpu_device_list()[0] and not use_weight_only: - pytest.skip("OOM on GH200. https://nvbugs/5250460") - - print("Converting checkpoint...") - dtype = 'float16' - model_name = os.path.basename(llm_commandr_v01_model_root) - ckpt_dir = convert_weights(llm_venv=llm_venv, - example_root=commandr_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llm_commandr_v01_model_root, - data_type=dtype, - use_weight_only=use_weight_only) - - print("Building engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - f"--max_batch_size={8}", - f"--max_input_len={924}", - f"--max_seq_len={1024}", - f"--gemm_plugin={dtype}", - f"--gpt_attention_plugin={dtype}", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - summary_cmd = [ - f"{commandr_example_root}/../../../summarize.py", - "--test_trt_llm", - "--hf_model_dir", - f"{llm_commandr_v01_model_root}", - "--data_type", - "fp16", - "--check_accuracy", - f"--engine_dir={engine_dir}", - "--tensorrt_llm_rouge1_threshold=12", - f"--dataset_dir={llm_datasets_root}", - f"--rouge_dir={llm_rouge_root}", - ] - - venv_check_call(llm_venv, summary_cmd) diff --git a/tests/integration/defs/examples/test_draft_target_model.py b/tests/integration/defs/examples/test_draft_target_model.py deleted file mode 100644 index 55fe9da35942..000000000000 --- a/tests/integration/defs/examples/test_draft_target_model.py +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import csv -from copy import deepcopy - -import pytest -from defs.common import convert_weights, venv_check_call -from defs.conftest import get_device_memory, get_sm_version, skip_post_blackwell -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -# TODO: remove skip after enable Blackwell for Speculative Decoding -@skip_post_blackwell -@pytest.mark.parametrize("batch_size", [1, 2], ids=['bs1', 'bs2']) -@pytest.mark.parametrize("data_type", ['float16']) -@pytest.mark.parametrize("draft_len", [4, 8], - ids=['draft_len_4', 'draft_len_8']) -@pytest.mark.parametrize("use_logits", [False, True], - ids=['use_tokens', 'use_logits']) -@pytest.mark.parametrize("use_py_session", [False], ids=["use_cpp_session"]) -@pytest.mark.parametrize("draft_target_model_roots", ["gpt2", "llama_v2"], - indirect=True) -@pytest.mark.parametrize("streaming", [False, True], - ids=["no_streaming", "streaming"]) -def test_llm_draft_target_model_1gpu(batch_size, data_type, draft_len, - use_logits, use_py_session, - draft_target_model_roots, streaming, - draft_target_model_example_root, - llm_datasets_root, llm_rouge_root, - llm_venv, cmodel_dir, engine_dir): - if "llama" in draft_target_model_roots[1]: - if get_device_memory() < 80000: - pytest.skip("GPU memory is insufficient.") - - model_name = "draft_target_model" - - print("Build checkpoint ...") - model_dir = convert_weights(llm_venv=llm_venv, - example_root=draft_target_model_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=draft_target_model_roots[1], - data_type=data_type) - - print("Build engines ...") - draft_engine_dir = engine_dir + "-draft" - target_engine_dir = engine_dir + "-target" - baseline_engine_dir = engine_dir + "-baseline" - common_build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--max_batch_size={batch_size}", - f"--max_beam_width=1", - "--max_input_len=1024", - "--max_seq_len=1536", - "--use_paged_context_fmha=enable", - f"--gpt_attention_plugin={data_type}", - f"--gemm_plugin={data_type}", - "--gather_generation_logits", - ] - draft_model_build_cmd = deepcopy(common_build_cmd) - draft_model_build_cmd.extend([ - f"--output_dir={draft_engine_dir}", - ]) - target_model_build_cmd = deepcopy(common_build_cmd) - target_model_build_cmd.extend([ - f"--output_dir={target_engine_dir}", - "--speculative_decoding_mode=draft_tokens_external", - f"--max_draft_len={draft_len}", - ]) - baseline_model_build_cmd = deepcopy(common_build_cmd) - baseline_model_build_cmd.extend([ - f"--output_dir={baseline_engine_dir}", - ]) - - check_call(" ".join(draft_model_build_cmd), - shell=True, - env=llm_venv._new_env) - check_call(" ".join(target_model_build_cmd), - shell=True, - env=llm_venv._new_env) - check_call(" ".join(baseline_model_build_cmd), - shell=True, - env=llm_venv._new_env) - - print("Run inferences ...") - draft_model_config = f"[{draft_len},[0],[0],{use_logits}]" - common_run_cmd = [ - f"{draft_target_model_example_root}/../run.py", - f"--tokenizer_dir={draft_target_model_roots[1]}", - "--max_output_len=64", - "--kv_cache_enable_block_reuse", - "--kv_cache_free_gpu_memory_fraction=0.25", - ] - if streaming: - common_run_cmd.extend(["--streaming", "--streaming_interval=1"]) - if batch_size == 1: - common_run_cmd.extend(["--input_text", "'How are you?'"]) - elif batch_size == 2: - common_run_cmd.extend(["--input_text", "'Hello'", "'How are you?'"]) - else: - assert False, "Only batch_size <=2 is supported in test." - assert not use_py_session, "Only CPP session is supported in Draft-Target-Model." - - run_cmd = deepcopy(common_run_cmd) - run_cmd.extend([ - f"--engine_dir={target_engine_dir}", - f"--draft_engine_dir={draft_engine_dir}", - f"--draft_target_model_config={draft_model_config}", - f"--output_csv={engine_dir}/draft_target_output.csv", - ]) - baseline_run_cmd = deepcopy(common_run_cmd) - baseline_run_cmd.extend([ - f"--engine_dir={baseline_engine_dir}", - f"--output_csv={engine_dir}/baseline_output.csv", - ]) - - venv_check_call(llm_venv, run_cmd) - venv_check_call(llm_venv, baseline_run_cmd) - - print("Compare outputs ...") - with open(f"{engine_dir}/draft_target_output.csv") as dt_f, open( - f"{engine_dir}/baseline_output.csv") as b_f: - for bs, (dt_request, - b_request) in enumerate(zip(csv.reader(dt_f), - csv.reader(b_f))): - assert ( - len(dt_request) == len(b_request) - ), f"Output length at ({bs=}) is different ({len(dt_request)} v.s. {len(b_request)})." - for index, (dt, b) in enumerate(zip(dt_request, b_request)): - assert ( - int(dt) == int(b) - ), f"Output at ({bs=}, {index=}) is different ({dt} v.s. {b})." diff --git a/tests/integration/defs/examples/test_eagle.py b/tests/integration/defs/examples/test_eagle.py deleted file mode 100644 index 0385edaff354..000000000000 --- a/tests/integration/defs/examples/test_eagle.py +++ /dev/null @@ -1,96 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from defs.common import convert_weights, venv_check_call -from defs.conftest import get_sm_version, skip_post_blackwell -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@skip_post_blackwell -@pytest.mark.parametrize("use_dynamic_tree", [False, True], - ids=['eagle1', 'eagle2']) -@pytest.mark.parametrize("batch_size", [1, 8], ids=['bs1', 'bs8']) -@pytest.mark.parametrize("data_type", ['float16']) -@pytest.mark.parametrize("eagle_model_roots", ["EAGLE-Vicuna-7B-v1.3"], - indirect=True) -def test_llm_eagle_1gpu(batch_size, data_type, use_dynamic_tree, - eagle_model_roots, eagle_example_root, - llm_datasets_root, llm_rouge_root, llm_venv, cmodel_dir, - engine_dir): - print("Build engines...") - model_name = "eagle" - - model_dir = convert_weights(llm_venv=llm_venv, - example_root=eagle_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=eagle_model_roots, - data_type=data_type) - - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - f"--gpt_attention_plugin={data_type}", - f"--gemm_plugin={data_type}", - f"--max_beam_width=1", - "--remove_input_padding=enable", - "--context_fmha=enable", - "--use_paged_context_fmha=enable", - "--max_input_len=1024", - "--max_seq_len=1536", - f"--max_batch_size={batch_size}", - "--paged_kv_cache=enable", - '--speculative_decoding_mode=eagle', - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run run...") - run_cmd = [ - f"{eagle_example_root}/../run.py", - "--max_output_len=100", - f"--tokenizer_dir={eagle_model_roots[0]}", - "--log_level=verbose", - f"--engine_dir={engine_dir}", - ] - if use_dynamic_tree: - run_cmd.extend( - [f"--eagle_dynamic_tree_max_top_k={3}", "--eagle_use_dynamic_tree"]) - - venv_check_call(llm_venv, run_cmd) - - print("Run summarize...") - summary_cmd = [ - f"{eagle_example_root}/../summarize.py", "--test_trt_llm", - "--hf_model_dir", f"{eagle_model_roots[0]}", "--tokenizer_dir", - f"{eagle_model_roots[0]}", f"--engine_dir={engine_dir}", - "--check_accuracy", "--tensorrt_llm_rouge1_threshold=24", - "--eagle_choices=[[0], [0, 0], [1], [0, 1], [2], [0, 0, 0], [1, 0], [0, 2], [3], [0, 3], [4], [0, 4], [2, 0], [0, 5], [0, 0, 1], [5], [0, 6], [6], [0, 7], [0, 1, 0], [1, 1], [7], [0, 8], [0, 0, 2], [3, 0], [0, 9], [8], [9], [1, 0, 0], [0, 2, 0], [1, 2], [0, 0, 3], [4, 0], [2, 1], [0, 0, 4], [0, 0, 5], [0, 0, 0, 0], [0, 1, 1], [0, 0, 6], [0, 3, 0], [5, 0], [1, 3], [0, 0, 7], [0, 0, 8], [0, 0, 9], [6, 0], [0, 4, 0], [1, 4], [7, 0], [0, 1, 2], [2, 0, 0], [3, 1], [2, 2], [8, 0], [0, 5, 0], [1, 5], [1, 0, 1], [0, 2, 1], [9, 0], [0, 6, 0], [0, 0, 0, 1], [1, 6], [0, 7, 0]]", - f"--max_ite=40", f"--batch_size={batch_size}", - f"--dataset_dir={llm_datasets_root}", f"--rouge_dir={llm_rouge_root}" - ] - if use_dynamic_tree: - summary_cmd.extend( - [f"--eagle_dynamic_tree_max_top_k={3}", "--eagle_use_dynamic_tree"]) - - venv_check_call(llm_venv, summary_cmd) diff --git a/tests/integration/defs/examples/test_enc_dec.py b/tests/integration/defs/examples/test_enc_dec.py deleted file mode 100644 index 10be785b220c..000000000000 --- a/tests/integration/defs/examples/test_enc_dec.py +++ /dev/null @@ -1,340 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from defs.common import (convert_weights, quantize_data, venv_check_call, - venv_mpi_check_call) -from defs.conftest import (get_device_count, get_sm_version, skip_fp8_pre_ada, - skip_post_blackwell) -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@pytest.mark.parametrize("use_fp8", [True, False], - ids=["enable_fp8", "disable_fp8"]) -@pytest.mark.parametrize("num_beams", [1, 2, 3], - ids=lambda num_beams: f'nb:{num_beams}') -@pytest.mark.parametrize("pp_size", [1, 2], ids=lambda pp_size: f'pp:{pp_size}') -@pytest.mark.parametrize("tp_size", [1, 2], ids=lambda tp_size: f'tp:{tp_size}') -@pytest.mark.parametrize( - "use_paged_kv_cache", [True, False], - ids=["enable_paged_kv_cache", "disable_paged_kv_cache"]) -@pytest.mark.parametrize( - "use_attention_plugin", - [pytest.param(True, marks=skip_post_blackwell), False], - ids=["enable_attention_plugin", "disable_attention_plugin"]) -@pytest.mark.parametrize("use_gemm_plugin", [True, False], - ids=["enable_gemm_plugin", "disable_gemm_plugin"]) -@pytest.mark.parametrize("data_type", ['bfloat16', 'float16', 'float32']) -@pytest.mark.parametrize("enc_dec_model_root", [ - pytest.param('t5-small', marks=skip_post_blackwell), - pytest.param('flan-t5-small', marks=skip_post_blackwell), - pytest.param('byt5-small', marks=skip_post_blackwell), 'bart-large-cnn', - pytest.param('mbart-large-50-many-to-one-mmt', marks=skip_post_blackwell), - 'wmt14' -], - indirect=True) -@pytest.mark.parametrize("compare_hf_fp32", [True, False], - ids=["compare_hf", "no_compare_hf"]) -def test_llm_enc_dec_general(llm_venv, cmodel_dir, engine_dir, data_type, - use_attention_plugin, use_gemm_plugin, - enc_dec_example_root, enc_dec_model_root, tp_size, - pp_size, num_beams, compare_hf_fp32, - use_paged_kv_cache, use_fp8, llm_datasets_root): - - world_size = tp_size * pp_size - - if get_device_count() < world_size: - pytest.skip( - f"Running world size {world_size} on a node with only {get_device_count()} devices. Skip the test..." - ) - - skip_fp8_pre_ada(use_fp8) - - print("Locate model checkpoints in test storage...") - tllm_model_name, model_ckpt_path = enc_dec_model_root - - print("Converting Encoder-Decoder model into binary format...") - # ckpt from llm_models/ --> cmodels// - model_name = tllm_model_name - model_type = None - if "t5" in model_name or "ul2" in model_name: - if data_type != "float32": - pytest.skip("transformer:issue/34264") - model_type = "t5" - elif "bart" in model_name: - model_type = "bart" - elif "wmt" in model_name: - model_type = "nmt" - - if use_fp8: - assert use_paged_kv_cache and use_attention_plugin - # a known apex huggingface bug for t5 only - # t5 only takes float32 in quantization loop - # https://github.com/huggingface/transformers/issues/34264 - converted_weight_dir = quantize_data( - llm_venv, - enc_dec_example_root, - model_dir=model_ckpt_path, - dtype=data_type, - quantize_dir=cmodel_dir, - qformat="fp8", - tp_size=tp_size, - pp_size=pp_size, - kv_cache_dtype="fp8", - batch_size=1, - calib_dataset=f"{llm_datasets_root}/cnn_dailymail") - - enc_dec_engine_dir = f"{engine_dir}/{tllm_model_name}/{world_size}-gpu/fp8" - else: - converted_weight_dir = convert_weights( - llm_venv=llm_venv, - example_root=enc_dec_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=model_ckpt_path, - data_type=data_type, - tp_size=tp_size, - pp_size=pp_size, - model_type=model_type) - - enc_dec_engine_dir = f"{engine_dir}/{tllm_model_name}/{world_size}-gpu/{data_type}" - - print("Build engines...") - - # change plugins precision to auto if testing fp8 - data_type = "auto" if use_fp8 else data_type - - for component in ["encoder", "decoder"]: - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}/{component}", - f"--output_dir={enc_dec_engine_dir}/{component}", - f"--max_beam_width={num_beams}", - "--moe_plugin=disable", - "--max_batch_size=8", - ] - - if component == "encoder": - build_cmd.append(f"--max_input_len=512") - else: - build_cmd.append(f"--max_input_len=1") - build_cmd.append(f"--max_seq_len=201") - build_cmd.append(f"--max_encoder_input_len=512") - - if use_paged_kv_cache and component == "decoder": - # paged_kv_cache only applies to decoder component - # As for now, we only support num_beams=1 for decoder paged kv cache in python runtime - build_cmd.append(f"--paged_kv_cache=enable") - else: - build_cmd.append(f"--paged_kv_cache=disable") - - if use_gemm_plugin: - build_cmd.append(f"--gemm_plugin={data_type}") - else: - build_cmd.append(f"--gemm_plugin=disable") - - if use_attention_plugin: - # TODO: remove skip after support bert_attention_plugin on B200 - build_cmd.append(f"--bert_attention_plugin={data_type}") - build_cmd.append(f"--gpt_attention_plugin={data_type}") - build_cmd.append("--remove_input_padding=enable") - - # for non-T5 models, FP16/BF16 - if model_type == "t5" or data_type == "float32": - build_cmd.append("--context_fmha=disable") - elif use_fp8: - build_cmd.append("--use_fp8_context_fmha=enable") - else: - build_cmd.append(f"--bert_attention_plugin=disable") - build_cmd.append(f"--gpt_attention_plugin=disable") - build_cmd.append("--remove_input_padding=disable") - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run inference...") - if use_paged_kv_cache and pp_size == 1: - # use paged engines to cover ModelRunnerCpp tests - run_cmd = [ - f"{enc_dec_example_root}/../../../run.py", - f"--engine_dir={enc_dec_engine_dir}", - f"--tokenizer_dir={model_ckpt_path}", - "--max_output_len=24", - f"--num_beams={num_beams}", - "--input_text='translate English to German: The house is wonderful.'", - ] - else: - # old Python runtime tests - run_cmd = [ - f"{enc_dec_example_root}/run.py", - f"--engine_dir={enc_dec_engine_dir}", - f"--engine_name={model_name}", - f"--model_name={model_ckpt_path}", # use ckpt path so we can use local copy rather than cloning from HF - "--max_new_tokens=24", # shorter than 3rd example input length to capture any bug - f"--num_beams={num_beams}", - ] - if compare_hf_fp32: - run_cmd.extend(["--compare_hf_fp32"]) - - if world_size == 1: - venv_check_call(llm_venv, run_cmd) - else: - venv_mpi_check_call( - llm_venv, ["mpirun", "-n", - str(world_size), "--allow-run-as-root"], run_cmd) - - -@pytest.mark.parametrize("use_fp8", [True, False], - ids=["enable_fp8", "disable_fp8"]) -@pytest.mark.parametrize("num_beams", [1, 2, 3], - ids=lambda num_beams: f'nb:{num_beams}') -@pytest.mark.parametrize("pp_size", [1, 2], ids=lambda pp_size: f'pp:{pp_size}') -@pytest.mark.parametrize("tp_size", [1, 2], ids=lambda tp_size: f'tp:{tp_size}') -@pytest.mark.parametrize("data_type", ['bfloat16', 'float16', 'float32']) -@pytest.mark.parametrize("enc_dec_model_root", ['flan-t5-small', 'flan-t5-xl'], - indirect=True) -def test_llm_enc_dec_mmlu(llm_venv, cmodel_dir, engine_dir, data_type, - enc_dec_example_root, enc_dec_model_root, tp_size, - pp_size, num_beams, mmlu_dataset_root, use_fp8, - llm_datasets_root): - - world_size = tp_size * pp_size - - if get_device_count() < world_size: - pytest.skip( - f"Running world size {world_size} on a node with only {get_device_count()} devices. Skip the test..." - ) - - skip_fp8_pre_ada(use_fp8) - - print("Locate model checkpoints in test storage...") - tllm_model_name, model_ckpt_path = enc_dec_model_root - - print("Converting Encoder-Decoder model into binary format...") - # ckpt from llm_models/ --> cmodels// - model_name = tllm_model_name - model_type = None - if "t5" in model_name or "ul2" in model_name: - model_type = "t5" - elif "bart" in model_name: - model_type = "bart" - elif "wmt" in model_name: - model_type = "nmt" - - if use_fp8: - # a known apex huggingface bug for t5 only - # t5 only takes float32 in quantization loop - # https://github.com/huggingface/transformers/issues/34264 - converted_weight_dir = quantize_data( - llm_venv, - enc_dec_example_root, - model_dir=model_ckpt_path, - dtype=data_type, - quantize_dir=cmodel_dir, - qformat="fp8", - tp_size=tp_size, - pp_size=pp_size, - kv_cache_dtype="fp8", - batch_size=1, - calib_dataset=f"{llm_datasets_root}/cnn_dailymail") - - enc_dec_engine_dir = f"{engine_dir}/{tllm_model_name}/{world_size}-gpu/fp8" - else: - converted_weight_dir = convert_weights( - llm_venv=llm_venv, - example_root=enc_dec_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=model_ckpt_path, - data_type=data_type, - tp_size=tp_size, - pp_size=pp_size, - model_type=model_type) - - enc_dec_engine_dir = f"{engine_dir}/{tllm_model_name}/{world_size}-gpu/{data_type}" - - print("Build engines...") - - max_input_len = 2048 - - # change plugins precision to auto if testing fp8 - data_type = "auto" if use_fp8 else data_type - - for component in ["encoder", "decoder"]: - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}/{component}", - f"--output_dir={enc_dec_engine_dir}/{component}", - f"--max_beam_width={num_beams}", - "--moe_plugin=disable", - "--max_batch_size=8", - ] - - if component == "encoder": - build_cmd.append(f"--max_input_len={max_input_len}") - else: - build_cmd.append(f"--max_input_len=1") - build_cmd.append(f"--max_seq_len=201") - build_cmd.append(f"--max_encoder_input_len={max_input_len}") - - if component == "decoder": - # paged_kv_cache only applies to decoder component - # As for now, we only support num_beams=1 for decoder paged kv cache in python runtime - build_cmd.append(f"--paged_kv_cache=enable") - - build_cmd.append(f"--gemm_plugin={data_type}") - build_cmd.append(f"--bert_attention_plugin={data_type}") - build_cmd.append(f"--gpt_attention_plugin={data_type}") - build_cmd.append("--remove_input_padding=enable") - - # for non-T5 models, FP16/BF16 - if model_type == "t5" or data_type == "float32": - build_cmd.append("--context_fmha=disable") - elif use_fp8: - build_cmd.append("--use_fp8_context_fmha=enable") - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run MMLU test") - accuracy_threshold_map = { - "flan-t5-xl": { - "float32": 0.440, # 0.444 - }, - "flan-t5-small": { - "float32": 0.280, # 0.282 - "float16": 0.280, # 0.283 - "float8": 0.280, # 0.284 - } - } - precision = "float8" if use_fp8 else data_type - accuracy_threshold = accuracy_threshold_map[tllm_model_name][precision] - - mmlu_cmd = [ - f"{enc_dec_example_root}/../../../mmlu.py", - f"--data_dir={mmlu_dataset_root}", - f"--hf_model_dir={model_ckpt_path}", - "--test_trt_llm", - f"--engine_dir={enc_dec_engine_dir}", - "--kv_cache_free_gpu_memory_fraction=0.45", - "--cross_kv_cache_fraction=0.45", - "--check_accuracy", - f"--accuracy_threshold={accuracy_threshold}", - ] - - venv_check_call(llm_venv, mmlu_cmd) diff --git a/tests/integration/defs/examples/test_flux.py b/tests/integration/defs/examples/test_flux.py deleted file mode 100644 index 9920ae1a85da..000000000000 --- a/tests/integration/defs/examples/test_flux.py +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Integration test for build_and_run_flux.py with multiple quantization formats.""" - -import importlib.util -import os - -import pytest -import torch -from build_and_run_flux import clip_model as load_clip_model - -# Check if CLIP is available -CLIP_AVAILABLE = importlib.util.find_spec("transformers") is not None - - -class FluxTestConfig: - """Configuration for Flux integration test.""" - - MODEL_ID = os.environ.get("FLUX_MODEL_ID", "black-forest-labs/FLUX.1-dev") - PROMPT = "a photo of an astronaut riding a horse on mars" - MIN_CLIP_SIMILARITY = 0.25 - NUM_INFERENCE_STEPS = 20 - MAX_BATCH_SIZE = 1 - BACKEND = "torch-opt" - - # Checkpoint paths for different quantization formats - # These can be set via environment variables or test parameters - FP8_CHECKPOINT = os.environ.get("FLUX_FP8_CHECKPOINT") - FP4_CHECKPOINT = os.environ.get("FLUX_FP4_CHECKPOINT") - - -@pytest.fixture(scope="module") -def clip_model(): - """Pytest fixture for loading CLIP model once per test module.""" - if not CLIP_AVAILABLE: - pytest.skip("CLIP not available") - return load_clip_model() - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required for Flux model") -@pytest.mark.slow # Mark as slow test -class TestFluxIntegration: - """Integration tests for Flux model with different quantization formats.""" diff --git a/tests/integration/defs/examples/test_gpt.py b/tests/integration/defs/examples/test_gpt.py index 746c77ecd187..64200b3549db 100644 --- a/tests/integration/defs/examples/test_gpt.py +++ b/tests/integration/defs/examples/test_gpt.py @@ -13,673 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. """Module test_gpt test gpt examples.""" -import csv -import os -import re -from pathlib import Path import defs.ci_profiler import pytest -from defs.common import (convert_weights, parse_output, quantize_data, - run_and_check, similar, similarity_score, - test_multi_lora_support, venv_check_call, - venv_check_output, venv_mpi_check_output) -from defs.conftest import (get_device_memory, get_sm_version, skip_fp8_pre_ada, - skip_post_blackwell, skip_pre_ada) -from defs.trt_test_alternative import check_call +from defs.common import similar, similarity_score from tensorrt_llm import LLM from tensorrt_llm.executor.request import LoRARequest from tensorrt_llm.lora_manager import LoraConfig from tensorrt_llm.sampling_params import SamplingParams -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - -INPUT_TEXT_1 = "After Washington had returned to Williamsburg, " + \ - "Dinwiddie ordered him to lead a larger force to assist Trent in his work. " + \ - "While en route, Washington learned of Trent's retreat. " + \ - "Since Tanaghrisson had promised support to the British, " + \ - "Washington continued toward Fort Duquesne and met with the Mingo leader. " + \ - "Learning of a French scouting party in the area, Washington, " + \ - "with Tanaghrisson and his party, surprised the Canadians on May 28 " + \ - "in what became known as the Battle of Jumonville Glen. " + \ - "They killed many of the Canadians, including their commanding officer, " + \ - "Joseph Coulon de Jumonville, whose head was reportedly split open by " + \ - "Tanaghrisson with a tomahawk. The historian Fred Anderson suggests that " + \ - "Tanaghrisson was acting to gain the support of the British and regain " + \ - "authority over his own people. They had been inclined to support the French, " + \ - "with whom they had long trading relationships. One of Tanaghrisson's men told " + \ - "Contrecoeur that Jumonville had been killed by British musket fire. " + \ - "Question: Upon learning of a French scounting party in the area, " + \ - "what did Washington do? Answer:" - -INPUT_TEXT_2 = "You hold the job title in the Wizarding World of Harry Potter where you " + \ - "say random words looking for spells" - -# streaming can can skip outputs, if the next set of outputs arrive. -# this means that the is_equal flag is currently flaky: https://nvbugspro.nvidia.com/bug/4851644 -# assert is_equal - - -@pytest.mark.parametrize("use_gemm_plugin", [True, False], - ids=["enable_gemm_plugin", "disable_gemm_plugin"]) -@pytest.mark.parametrize("use_py_session", [False, True], - ids=["use_cpp_session", "use_py_session"]) -@pytest.mark.parametrize("streaming", [False, True], - ids=["non_streaming", "streaming"]) -def test_llm_gpt2_medium_1gpu(gpt_example_root, llm_venv, - llm_gpt2_medium_model_root, cmodel_dir, - engine_dir, use_gemm_plugin, use_py_session, - streaming): - "gpt2-medium build & run" - print("Converting checkpoint...") - dtype = 'float16' - ckpt_dir = convert_weights(llm_venv=llm_venv, - example_root=gpt_example_root, - cmodel_dir=cmodel_dir, - model="gpt2-medium", - model_path=llm_gpt2_medium_model_root, - data_type=dtype) - - print("Building engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - f"--max_batch_size={8}", - f"--max_input_len={924}", - f"--max_seq_len={1024}", - f"--gpt_attention_plugin={dtype}", - "--paged_kv_cache=enable", - "--remove_input_padding=enable", - ] - - if use_gemm_plugin: - build_cmd.extend([f"--gemm_plugin={dtype}"]) - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - run_cmd = [ - f"{gpt_example_root}/../../../run.py", "--max_output_len=8", - f"--engine_dir={engine_dir}", - f"--tokenizer_dir={llm_gpt2_medium_model_root}", - "--no_add_special_tokens" - ] - - if streaming: - run_cmd.append("--streaming") - if use_py_session: - run_cmd.append("--use_py_session") - - print("Running inference...") - output = venv_check_output(llm_venv, run_cmd) - - valid_outputs = [ - "chef before moving to London in the early", - "chef before moving to London in the late", - "chef and eventually became a chef at a", - ] - - if not streaming: - output = parse_output(output)[0] - assert any([similar(output, expect) - for expect in valid_outputs]), f"output is: {output}" - else: - # Fetch all outputs and expect a monotonically increasing similarity - similarities = [] - for suboutput in parse_output(output): - similarities.append( - max([ - similarity_score(suboutput, expect) - for expect in valid_outputs - ])) - assert ( - all(x <= y for x, y in zip(similarities, similarities[1:])) - ), f"streaming outputs must have a monotonically increasing similarity score. similarities: {similarities}" - output = parse_output(output)[-1] - assert any([similar(output, expect) - for expect in valid_outputs]), f"output is: {output}" - - -@pytest.mark.parametrize("use_py_session", [False, True], - ids=["use_cpp_session", "use_py_session"]) -@pytest.mark.parametrize("streaming", [False, True], - ids=["non_streaming", "streaming"]) -def test_llm_gpt2_medium_bad_words_1gpu(gpt_example_root, llm_venv, - llm_gpt2_medium_model_root, cmodel_dir, - engine_dir, use_py_session, streaming): - "gpt2 build & run" - - if use_py_session and streaming: - pytest.skip( - "Streaming with py session does not return complete sequence to reliably check stop words" - ) - - print("Converting checkpoint...") - dtype = 'float16' - ckpt_dir = convert_weights(llm_venv=llm_venv, - example_root=gpt_example_root, - cmodel_dir=cmodel_dir, - model="gpt2-medium", - model_path=llm_gpt2_medium_model_root, - data_type=dtype) - - print("Building engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - f"--max_batch_size={8}", - f"--max_input_len={924}", - f"--max_seq_len={1024}", - f"--gpt_attention_plugin={dtype}", - "--paged_kv_cache=enable", - "--remove_input_padding=enable", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - run_cmd = [ - f"{gpt_example_root}/../../../run.py", "--max_output_len=8", - f"--engine_dir={engine_dir}", - f"--tokenizer_dir={llm_gpt2_medium_model_root}", - "--no_add_special_tokens" - ] - - if streaming: - run_cmd.append("--streaming") - if use_py_session: - run_cmd.append("--use_py_session") - - valid_outputs = [ - "chef before moving to the UK in the", - "chef and eventually became a chef at a", - ] - bad_words_args = ["--bad_words", " London"] - run_and_check(llm_venv, - run_cmd + bad_words_args, - valid_outputs, - streaming=streaming) - - bad_words_args = ["--bad_words", " to London", " irrelevant words"] - run_and_check(llm_venv, - run_cmd + bad_words_args, - valid_outputs, - streaming=streaming) - - bad_words_args = ["--bad_words", " irrelevant words", " to London"] - run_and_check(llm_venv, - run_cmd + bad_words_args, - valid_outputs, - streaming=streaming) - - -@pytest.mark.parametrize("use_py_session", [False, True], - ids=["use_cpp_session", "use_py_session"]) -@pytest.mark.parametrize("streaming", [False, True], - ids=["non_streaming", "streaming"]) -def test_llm_gpt2_medium_stop_words_1gpu(gpt_example_root, llm_venv, - llm_gpt2_medium_model_root, cmodel_dir, - engine_dir, use_py_session, streaming): - "gpt2 build & run" - if use_py_session and streaming: - pytest.skip( - "Streaming with py session does not return complete sequence to reliably check stop words" - ) - - print("Converting checkpoint...") - dtype = 'float16' - ckpt_dir = convert_weights(llm_venv=llm_venv, - example_root=gpt_example_root, - cmodel_dir=cmodel_dir, - model="gpt2-medium", - model_path=llm_gpt2_medium_model_root, - data_type=dtype) - - print("Building engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - f"--max_batch_size={8}", - f"--max_input_len={924}", - f"--max_seq_len={1024}", - f"--gpt_attention_plugin={dtype}", - "--paged_kv_cache=enable", - "--remove_input_padding=enable", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - run_cmd = [ - f"{gpt_example_root}/../../../run.py", "--max_output_len=8", - f"--engine_dir={engine_dir}", - f"--tokenizer_dir={llm_gpt2_medium_model_root}", - "--no_add_special_tokens" - ] - - if streaming: - run_cmd.append("--streaming") - if use_py_session: - run_cmd.append("--use_py_session") - - valid_outputs = [ - "chef before moving to London", - "chef and eventually became", - ] - stop_words_args = ["--stop_words", " London", " became"] - run_and_check(llm_venv, - run_cmd + stop_words_args, - valid_outputs, - streaming=streaming) - - stop_words_args = [ - "--stop_words", " eventually became", " to London", " irrelevant output" - ] - run_and_check(llm_venv, - run_cmd + stop_words_args, - valid_outputs, - streaming=streaming) - - stop_words_args = [ - "--stop_words", " to London", " eventually became", " irrelevant output" - ] - run_and_check(llm_venv, - run_cmd + stop_words_args, - valid_outputs, - streaming=streaming) - - -# transformers compatibility issues -@pytest.mark.parametrize("tensor_parallel", [1, 2], ids=["tp1", "tp2"]) -@pytest.mark.parametrize("use_py_session", [False, True], - ids=["use_cpp_session", "use_py_session"]) -def test_llm_gpt2_next_prompt_tuning(gpt_example_root, llm_venv, - llm_gpt2_next_model_root, cmodel_dir, - engine_dir, tensor_parallel, - use_py_session): - f"gpt-next prompt tuning on {tensor_parallel} gpu(s)" - dtype = "bfloat16" - ckpt_dir = convert_weights(llm_venv=llm_venv, - example_root=gpt_example_root, - cmodel_dir=cmodel_dir, - model="gpt2-next", - model_path=llm_gpt2_next_model_root, - gpus=tensor_parallel, - tp_size=tensor_parallel, - data_type=dtype) - - print("Building engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - f"--max_batch_size=4", - f"--max_input_len=924", - f"--max_seq_len=1024", - f"--gpt_attention_plugin={dtype}", - "--max_prompt_embedding_table_size=200", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Converting prompt-tuning table...") - squad_table_nemo = Path(llm_gpt2_next_model_root - ).parent / "p-tuning" / "gpt2b_gpt2-squad-vt60.nemo" - squad_table = Path(gpt_example_root) / "prompt_table_squad.npy" - train900_table_nemo = Path( - llm_gpt2_next_model_root - ).parent / "p-tuning" / "gpt2b_gpt2b-train900-v2.nemo" - train900_table = Path(gpt_example_root) / "prompt_table_train900.npy" - for (in_file, out_file) in [(squad_table_nemo, squad_table), - (train900_table_nemo, train900_table)]: - table_conv_cmd = [ - f"{gpt_example_root}/nemo_prompt_convert.py", "-i", - str(in_file), "-o", - str(out_file) - ] - venv_check_call(llm_venv, table_conv_cmd) - - merged_table = Path(gpt_example_root) / "prompt_table_train900.npy" - table_merge_cmd = [ - f"{gpt_example_root}/merge_ptuning_tables.py", - str(squad_table), - str(train900_table), - str(merged_table) - ] - venv_check_call(llm_venv, table_merge_cmd) - - inference_params = { - "squad": { - "num_v_tokens": - 50, - "input": - "Context: In Hinduism the spiritual teacher is known as a guru, and, in many traditions of Hinduism - especially those common in the West - the emphasis on spiritual mentorship is extremely high, with gurus often exercising a great deal of control over the lives of their disciples.\n\nQuestion: Who do gurus control?\n\nAnswer:", - "outputs": [ - "The answer is, of course, the disciple.", - "The guru controls the disciple's life, but", - "The guru is the one who controls the disciple." - ], - }, - "train900": { - "num_v_tokens": 20, - "input": - "Context: Carlsen faced Anand in the World Chess Championship 2013, at Hyatt Regency in Chennai, India, from 9 to 22 November. Carlsen won the match 6.5–3.5 by winning games five, six and nine and drawing the remainder, becoming the new World Chess Champion.\n\nQuestion: When did Carlsen become World Chess Champion?\n\nAnswer:", - "outputs": - ["2013", "2013" + os.linesep + os.linesep + "Question: Who"], - } - } - - print("Running inference...") - - def parse_output(text: str) -> list[str]: - results = [] - while True: - match = re.search( - r"Output \[Text \d+ Beam \d+\]: \"([^\"]*)\"" + os.linesep, - text, re.MULTILINE) - if match is None: - break - _, end = match.span() - results.append(match.group(1)) - text = text[end:] - return results - - # test model without p-tuning dict - run_cmd = [ - f"{gpt_example_root}/../../../run.py", - "--no_add_special_tokens", - "--max_output_len=10", - f"--engine_dir={engine_dir}", - f"--vocab_file={ckpt_dir}/tokenizer.model", - f"--input_text={inference_params['squad']['input']}", - ] - - if use_py_session: - run_cmd.append("--use_py_session") - - output = venv_mpi_check_output( - llm_venv, ["mpirun", "-n", f"{tensor_parallel}", "--allow-run-as-root"], - run_cmd) - assert any( - similar(parse_output(output)[0][:len(ref) + 1], ref) - for ref in inference_params["squad"]["outputs"]), "incorrect output" - - # test p-tuning task separately - run_cmd = [ - f"{gpt_example_root}/../../../run.py", - "--no_add_special_tokens", - "--max_output_len=10", - f"--engine_dir={engine_dir}", - f"--vocab_file={ckpt_dir}/tokenizer.model", - f"--prompt_table={squad_table}", - f"--num_prepend_vtokens={inference_params['squad']['num_v_tokens']}", - f"--input_text={inference_params['squad']['input']}", - f"--no-kv_cache_enable_block_reuse", - ] - - if use_py_session: - run_cmd.append("--use_py_session") - - output = venv_mpi_check_output( - llm_venv, ["mpirun", "-n", f"{tensor_parallel}", "--allow-run-as-root"], - run_cmd) - assert any( - similar(parse_output(output)[0][:len(ref) + 1], ref) - for ref in inference_params["squad"]["outputs"]), "incorrect output" - - run_cmd = [ - f"{gpt_example_root}/../../../run.py", - "--no_add_special_tokens", - "--max_output_len=10", - f"--engine_dir={engine_dir}", - f"--vocab_file={ckpt_dir}/tokenizer.model", - f"--prompt_table={train900_table}", - f"--num_prepend_vtokens={inference_params['train900']['num_v_tokens']}", - f"--input_text={inference_params['train900']['input']}", - f"--no-kv_cache_enable_block_reuse", - ] - - if use_py_session: - run_cmd.append("--use_py_session") - - output = venv_mpi_check_output( - llm_venv, ["mpirun", "-n", f"{tensor_parallel}", "--allow-run-as-root"], - run_cmd) - assert any( - similar(parse_output(output)[0][:len(ref) + 1], ref) - for ref in inference_params["train900"]["outputs"]), "incorrect output" - - # test batched p-tuning tasks - run_cmd = [ - f"{gpt_example_root}/../../../run.py", - "--no_add_special_tokens", - "--max_output_len=10", - f"--engine_dir={engine_dir}", - f"--vocab_file={ckpt_dir}/tokenizer.model", - f"--prompt_table={merged_table}", - f"--num_prepend_vtokens", - str(inference_params['squad']['num_v_tokens']), - str(inference_params['train900']['num_v_tokens']), - f"--prompt_tasks=0,1", - f"--input_text", - inference_params["squad"]["input"], - inference_params['train900']['input'], - f"--no-kv_cache_enable_block_reuse", - ] - - if use_py_session: - run_cmd.append("--use_py_session") - - output = venv_mpi_check_output( - llm_venv, ["mpirun", "-n", f"{tensor_parallel}", "--allow-run-as-root"], - run_cmd) - - outputs = parse_output(output) - assert any( - similar(outputs[0][:len(ref) + 1], ref) - for ref in inference_params["squad"]["outputs"]), "incorrect output" - assert any( - similar(outputs[1][:len(ref) + 1], ref) - for ref in inference_params["train900"]["outputs"]), "incorrect output" - - # test batched and streamed p-tuning tasks - # Streaming with py session does not return complete sequence to reliably check stop words" - - if not use_py_session and tensor_parallel == 1: - run_cmd = [ - f"{gpt_example_root}/../../../run.py", - "--no_add_special_tokens", - "--max_output_len=10", - f"--engine_dir={engine_dir}", - f"--vocab_file={ckpt_dir}/tokenizer.model", - f"--prompt_table={merged_table}", - f"--num_prepend_vtokens", - str(inference_params['squad']['num_v_tokens']), - str(inference_params['train900']['num_v_tokens']), - f"--prompt_tasks=0,1", - "--streaming", - f"--input_text", - inference_params["squad"]["input"], - inference_params['train900']['input'], - f"--no-kv_cache_enable_block_reuse", - ] - - output = venv_mpi_check_output( - llm_venv, - ["mpirun", "-n", f"{tensor_parallel}", "--allow-run-as-root"], - run_cmd) - - outputs = parse_output(output) - squad_outputs = outputs[::2] - train900_outputs = outputs[1::2] - for outputs, valid_outputs in [ - (squad_outputs, inference_params["squad"]["outputs"]), - (train900_outputs, inference_params["train900"]["outputs"]) - ]: - assert any( - similar(outputs[-1][:len(ref) + 1], ref) - for ref in valid_outputs), "incorrect output" - similarities = [] - for suboutput in outputs: - similarities.append( - max([ - similarity_score(suboutput, expect) - for expect in valid_outputs - ])) - assert ( - all(x <= y for x, y in zip(similarities, similarities[1:])) - ), f"streaming outputs must have a monotonically increasing similarity score. valid_outputs: {valid_outputs}, outputs: {outputs}, similarities: {similarities}" - - -@skip_post_blackwell -@pytest.mark.skip_less_device_memory(50000) -@pytest.mark.parametrize("data_type", ['float16', 'fp8'], - ids=['base_fp16', 'base_fp8']) -@pytest.mark.parametrize("lora_data_type", ['float16'], ids=['lora_fp16']) -@pytest.mark.parametrize("llm_gpt2_starcoder_model_root", ['starcoder2'], - indirect=True) -@pytest.mark.parametrize("llm_lora_model_root", - ['peft-lora-starcoder2-15b-unity-copilot'], - indirect=True) -def test_llm_gpt_starcoder_lora_1gpu(data_type, lora_data_type, - gpt_example_root, - llm_gpt2_starcoder_model_root, - llm_datasets_root, llm_venv, cmodel_dir, - engine_dir, llm_lora_model_root, - qcache_dir): - "run starcoder2 lora test on 1gpu" - if data_type == 'fp8': - skip_fp8_pre_ada(use_fp8=True) - else: - if get_device_memory() < 80000: - pytest.skip("GPU memory is not sufficient.") - - print("Converting checkpoint...") - model_name = 'starcoder2-lora' - - if data_type == 'fp8': - model_dir = quantize_data( - llm_venv, - gpt_example_root, - model_dir=llm_gpt2_starcoder_model_root, - calib_dataset=f"{llm_datasets_root}/cnn_dailymail", - dtype="float16", - qformat="fp8", - kv_cache_dtype="fp8", - quantize_dir=qcache_dir, - calib_size=512) - else: - model_dir = convert_weights(llm_venv=llm_venv, - example_root=gpt_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llm_gpt2_starcoder_model_root) - - print("Build engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - "--lora_plugin=auto", - "--gemm_plugin=auto", - f"--lora_dir={llm_lora_model_root}", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - ref_1 = [ - 610, 1489, 100, 7670, 100, 5879, 2284, 303, 1489, 459, 8302, 10914, - 16013, 222, 222, 610, 1489, 100, 7670, 100, 5879, 100, 115, 100, 5598, - 45, 115 - ] - ref_2 = [ - 610, 1489, 100, 7670, 100, 5879, 2284, 303, 1489, 459, 8302, 10914, 678, - 222, 222, 610, 1489, 100, 7670, 100, 5879, 100, 115, 100, 5598, 45, 115 - ] - - input_text = "def print_hello_world():" - - print(f"Run inference with lora id 0...") - venv_check_call(llm_venv, [ - f"{gpt_example_root}/../../../run.py", - "--max_output_len=20", - f"--input_text={input_text}", - "--lora_task_uids=0", - f"--tokenizer_dir={llm_gpt2_starcoder_model_root}", - f"--engine_dir={engine_dir}", - f"--output_csv={llm_venv.get_working_directory()}/use_lora.csv", - "--no_add_special_tokens", - "--use_py_session", - ]) - - with open(f"{llm_venv.get_working_directory()}/use_lora.csv") as f: - predict = csv.reader(f) - predict = next(predict) - predict = [int(p) for p in predict] - assert ref_1 == predict or data_type != "float16" - - print(f"Run inference with lora id -1...") - venv_check_call(llm_venv, [ - f"{gpt_example_root}/../../../run.py", - "--max_output_len=20", - f"--input_text={input_text}", - "--lora_task_uids=-1", - f"--tokenizer_dir={llm_gpt2_starcoder_model_root}", - f"--engine_dir={engine_dir}", - f"--output_csv={llm_venv.get_working_directory()}/no_lora.csv", - "--no_add_special_tokens", - "--use_py_session", - ]) - - with open(f"{llm_venv.get_working_directory()}/no_lora.csv") as f: - predict = csv.reader(f) - predict = next(predict) - predict = [int(p) for p in predict] - assert ref_2 == predict or data_type != "float16" - - -@skip_pre_ada -@pytest.mark.parametrize("minitron_model_root", ["4b"], indirect=True) -def test_llm_minitron_fp8_with_pseudo_loras(gpt_example_root, - minitron_model_root, - llm_datasets_root, - llm_venv, - cmodel_dir, - engine_dir, - dtype='bfloat16'): - "Run Minitron model with multiple pseudo LoRAs." - - # Quantize the base model to fp8. - print("Quantizing and converting checkpoint...") - ckpt_dir = f"{cmodel_dir}/minitron/fp8/1-gpu" - - quantize_cmd = [ - f"{gpt_example_root}/../../../quantization/quantize.py", - f"--model_dir={minitron_model_root}", - f"--calib_dataset={llm_datasets_root}/cnn_dailymail", - f"--dtype={dtype}", - "--qformat=fp8", - "--kv_cache_dtype=fp8", - f"--output_dir={ckpt_dir}", - ] - venv_check_call(llm_venv, quantize_cmd) - - test_multi_lora_support( - hf_model_dir=minitron_model_root, - tllm_ckpt_dir=ckpt_dir, - engine_dir=engine_dir, - llm_venv=llm_venv, - example_root=gpt_example_root, - num_loras=2, - lora_rank=8, - target_hf_modules=["q_proj", "k_proj", "v_proj"], - target_trtllm_modules=["attn_q", "attn_k", "attn_v"], - zero_lora_weights=True, - ) - @pytest.mark.skip_less_device_memory( 20000) # Conservative 20GB requirement for GPT-OSS-20B diff --git a/tests/integration/defs/examples/test_gptj.py b/tests/integration/defs/examples/test_gptj.py deleted file mode 100644 index 51a2a1002b6c..000000000000 --- a/tests/integration/defs/examples/test_gptj.py +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from defs.conftest import get_sm_version - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - -INPUT_TEXT = """ -Write a Python function `find_max(words)` to solve the following problem:\nWrite a function that accepts a list of strings.\nThe list contains different words. Return the word with maximum number\nof unique characters. If multiple strings have maximum number of unique\ncharacters, return the one which comes first in lexicographical order.\nfind_max(["name", "of", "string"]) == "string"\nfind_max(["name", "enam", "game"]) == "enam"\nfind_max(["aaaaaaa", "bb" ,"cc"]) == ""aaaaaaa" -""" diff --git a/tests/integration/defs/examples/test_granite.py b/tests/integration/defs/examples/test_granite.py deleted file mode 100644 index f789c4065653..000000000000 --- a/tests/integration/defs/examples/test_granite.py +++ /dev/null @@ -1,160 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import time - -import pytest -from defs.common import (convert_weights, test_multi_lora_support, - venv_mpi_check_call) -from defs.conftest import get_sm_version -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@pytest.fixture(scope="module", autouse=True) -def disable_unified_converter(): - os.environ['TRTLLM_DISABLE_UNIFIED_CONVERTER'] = '1' - yield - del os.environ['TRTLLM_DISABLE_UNIFIED_CONVERTER'] - - -@pytest.mark.parametrize("dtype", ["float16", "bfloat16"]) -@pytest.mark.parametrize( - "llm_granite_model_root", - ["granite-3.0-1b-a400m-instruct", "granite-3.0-2b-instruct"], - indirect=True) -def test_llm_granite(llama_example_root, llm_granite_model_root, - llm_datasets_root, llm_rouge_root, llm_venv, cmodel_dir, - engine_dir, dtype): - print("Converting checkpoint...") - model_name = os.path.basename(llm_granite_model_root) - - ckpt_dir = convert_weights( - llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llm_granite_model_root, - data_type=dtype, - ) - - print("Building engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - "--max_batch_size=8", - "--max_input_len=924", - "--max_seq_len=1024", - f"--gpt_attention_plugin={dtype}", - f"--gemm_plugin={dtype}", - f"--moe_plugin={dtype}", - f"--workers=1", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run engines...") - summary_cmd = [ - f"{llama_example_root}/../../../summarize.py", - f"--engine_dir={engine_dir}", - f"--hf_model_dir={llm_granite_model_root}", - f"--dataset_dir={llm_datasets_root}", - f"--rouge_dir={llm_rouge_root}" - "--test_trt_llm", - "--check_accuracy", - "--tensorrt_llm_rouge1_threshold=25", - "--batch_size=8", - "--max_ite=40", - ] - venv_mpi_check_call(llm_venv, ["mpirun", "-n", "1", "--allow-run-as-root"], - summary_cmd) - - -@pytest.mark.parametrize( - "llm_granite_model_root", - ["granite-3.0-1b-a400m-instruct", "granite-3.0-2b-instruct"], - indirect=True) -def test_granite_bf16_lora(llama_example_root, - llm_datasets_root, - qcache_dir, - llm_rouge_root, - llm_venv, - engine_dir, - cmodel_dir, - llm_granite_model_root, - num_beams=1): - "Run Granite 3.0 models with multiple dummy LoRAs." - - # TODO: Enable fp8 quantization when ModelOpt changes for Granite are available. - start_time = time.time() - print("Converting checkpoint...") - convert_start = time.time() - model_name = os.path.basename(llm_granite_model_root) - dtype = 'bfloat16' - - ckpt_dir = convert_weights( - llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llm_granite_model_root, - data_type=dtype, - ) - convert_end = time.time() - print( - f"Convert checkpoint completed in {(convert_end - convert_start):.2f} seconds." - ) - - target_hf_modules = [ - "q_proj", - "k_proj", - "v_proj", - ] - target_trtllm_modules = [ - "attn_q", - "attn_k", - "attn_v", - ] - if model_name == "granite-3.0-1b-a400m-instruct": - target_hf_modules += ["moe_h_to_4h", "moe_4h_to_h", "moe_gate"] - target_trtllm_modules += ["moe_h_to_4h", "moe_4h_to_h", "moe_gate"] - - print("Calling test_multi_lora_support...") - test_multi_lora_start = time.time() - test_multi_lora_support( - hf_model_dir=llm_granite_model_root, - tllm_ckpt_dir=ckpt_dir, - engine_dir=engine_dir, - llm_venv=llm_venv, - example_root=llama_example_root, - num_loras=2, - lora_rank=8, - target_hf_modules=target_hf_modules, - target_trtllm_modules=target_trtllm_modules, - zero_lora_weights=True, - ) - test_multi_lora_end = time.time() - print( - f"test_multi_lora_support completed in {(test_multi_lora_end - test_multi_lora_start):.2f} seconds" - ) - - total_time = time.time() - start_time - print(f"Total function execution time: {total_time:.2f} seconds") diff --git a/tests/integration/defs/examples/test_internlm.py b/tests/integration/defs/examples/test_internlm.py deleted file mode 100644 index 144f86b32fab..000000000000 --- a/tests/integration/defs/examples/test_internlm.py +++ /dev/null @@ -1,104 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import pytest -from defs.common import convert_weights, parse_mpi_cmd, venv_mpi_check_call -from defs.conftest import get_device_memory, get_sm_version -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -# @pytest.mark.skip_less_device(2) -@pytest.mark.parametrize("num_beams", [1, 2, 4], - ids=lambda num_beams: f'nb:{num_beams}') -@pytest.mark.parametrize( - "use_gpt_attention_plugin", [True, False], - ids=["enable_attention_plugin", "disable_attention_plugin"]) -@pytest.mark.parametrize("use_gemm_plugin", [True, False], - ids=["enable_gemm_plugin", "disable_gemm_plugin"]) -@pytest.mark.parametrize("context_fmha_type", [ - "enable_context_fmha", "enable_context_fmha_fp32_acc", - "disable_context_fmha" -]) -@pytest.mark.parametrize("dtype", ['float16', 'bfloat16']) -def test_llm_internlm2_7b_1node_1gpu(internlm2_example_root, - llm_internlm2_7b_model_root, - llm_datasets_root, llm_rouge_root, - llm_venv, cmodel_dir, engine_dir, - use_gpt_attention_plugin, use_gemm_plugin, - context_fmha_type, dtype, num_beams): - "Build & Run internlm2-7b with 1 gpu" - if dtype == "bfloat16" and not use_gemm_plugin: - pytest.skip("Please use gemm plugin when dtype is bfloat16.") - if num_beams == 4 and get_device_memory() < 50000: - pytest.skip("device memory is insufficient.") - - model_dir = convert_weights(llm_venv=llm_venv, - example_root=f"{internlm2_example_root}", - cmodel_dir=cmodel_dir, - model="internlm2-7b", - model_path=llm_internlm2_7b_model_root, - data_type=dtype, - gpus=1, - tp_size=1) - - build_cmd = [ - "python3 -m tensorrt_llm.commands.build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - f"--max_beam_width={num_beams}", - f"--max_batch_size=1", - ] - - if use_gpt_attention_plugin: - build_cmd.append("--remove_input_padding=enable") - build_cmd.append(f"--gpt_attention_plugin={dtype}") - else: - build_cmd.append("--gpt_attention_plugin=disable") - build_cmd.append("--remove_input_padding=disable") - build_cmd.append("--paged_kv_cache=disable") - - if use_gemm_plugin: - build_cmd.append(f"--gemm_plugin={dtype}") - else: - build_cmd.append("--gemm_plugin=disable") - - if context_fmha_type == "enable_context_fmha": - build_cmd.append("--context_fmha=enable") - elif context_fmha_type == "disable_context_fmha": - build_cmd.append("--context_fmha=disable") - - print("Building engines...") - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print('Run internlm2-7b...') - data_type = "fp16" if dtype == "float16" else "bf16" - summary_cmd = [ - f"{internlm2_example_root}/../../../summarize.py", "--test_trt_llm", - "--hf_model_dir", llm_internlm2_7b_model_root, "--engine_dir", - engine_dir, "--data_type", data_type, "--check_accuracy", - f"--num_beams={num_beams}", f"--dataset_dir={llm_datasets_root}", - f"--rouge_dir={llm_rouge_root}" - ] - if context_fmha_type == "enable_context_fmha_fp32_acc": - summary_cmd.append("--enable_context_fmha_fp32_acc") - - venv_mpi_check_call( - llm_venv, parse_mpi_cmd(["mpirun", "-n", "1", "--allow-run-as-root"]), - summary_cmd) diff --git a/tests/integration/defs/examples/test_llama.py b/tests/integration/defs/examples/test_llama.py deleted file mode 100644 index ebfcee69c760..000000000000 --- a/tests/integration/defs/examples/test_llama.py +++ /dev/null @@ -1,533 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import copy -import csv -import os - -import defs.ci_profiler -import pytest -from defs.common import (convert_weights, generate_summary_cmd, quantize_data, - test_llm_torch_multi_lora_support, - test_multi_lora_support, venv_check_call, - venv_mpi_check_call) -# yapf: disable -from defs.conftest import (get_device_count, get_device_memory, - skip_fp8_pre_ada, skip_post_blackwell, skip_pre_ada) -# yapf: enable -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -# if get_sm_version() >= 103: -# pytest.skip( -# "TRT workflow tests are not supported on post Blackwell-Ultra architecture", -# allow_module_level=True) - -INPUT_TEXT_1 = "After Washington had returned to Williamsburg, " + \ - "Dinwiddie ordered him to lead a larger force to assist Trent in his work. " + \ - "While en route, Washington learned of Trent's retreat. " + \ - "Since Tanaghrisson had promised support to the British, " + \ - "Washington continued toward Fort Duquesne and met with the Mingo leader. " + \ - "Learning of a French scouting party in the area, Washington, " + \ - "with Tanaghrisson and his party, surprised the Canadians on May 28 " + \ - "in what became known as the Battle of Jumonville Glen. " + \ - "They killed many of the Canadians, including their commanding officer, " + \ - "Joseph Coulon de Jumonville, whose head was reportedly split open by " + \ - "Tanaghrisson with a tomahawk. The historian Fred Anderson suggests that " + \ - "Tanaghrisson was acting to gain the support of the British and regain " + \ - "authority over his own people. They had been inclined to support the French, " + \ - "with whom they had long trading relationships. One of Tanaghrisson's men told " + \ - "Contrecoeur that Jumonville had been killed by British musket fire. " + \ - "Question: Upon learning of a French scounting party in the area, " + \ - "what did Washington do? Answer:" - -INPUT_TEXT_2 = "Born in north-east France, Soyer trained as a" - - -@pytest.mark.parametrize("num_beams", [1, 2, 4], - ids=lambda num_beams: f'nb:{num_beams}') -@pytest.mark.parametrize("run_type", ['inference', 'summarization']) -@pytest.mark.parametrize("data_type", ['bfloat16', 'float16']) -@pytest.mark.parametrize("fp8_cache", [True, False], - ids=["enable_fp8", "disable_fp8"]) -@pytest.mark.parametrize( - "llama_model_root", - ['llama-v3-8b-instruct-hf', 'llama-3.1-8b-instruct-hf-fp8'], - indirect=True) -def test_llm_llama_1gpu(run_type, data_type, fp8_cache, llama_example_root, - llama_model_root, llm_datasets_root, llm_rouge_root, - llm_venv, cmodel_dir, engine_dir, - qcache_dir_without_install_package, num_beams): - if num_beams > 2 and get_device_memory() < 80000: - pytest.skip("device memory is insufficient.") - - use_fp8 = fp8_cache if "fp8" not in llama_model_root.lower() else True - skip_fp8_pre_ada(use_fp8=use_fp8) - - model_name = os.path.basename(llama_model_root) - - if llama_model_root.endswith('Llama-3.1-8B-Instruct-FP8'): - model_dir = convert_weights(llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model="llama_v3_finegrained_fp8", - model_path=llama_model_root, - fp8_kv_cache=fp8_cache, - data_type=data_type) - elif fp8_cache: - # Quantize HF llama checkpoint into FP8 format - model_dir = quantize_data( - llm_venv, - llama_example_root, - model_dir=llama_model_root, - calib_dataset=f"{llm_datasets_root}/cnn_dailymail", - dtype=data_type, - qformat="fp8", - quantize_dir=qcache_dir_without_install_package, - calib_size=512, - kv_cache_dtype="fp8") - else: - model_dir = convert_weights( - llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llama_model_root, - data_type=data_type, - enable_fp8=fp8_cache, - fp8_kv_cache=fp8_cache, - quant_ckpt_path= - f"{qcache_dir_without_install_package}/quantized_fp8/llama_tp1_rank0.npz" - if fp8_cache else None) - - print("Build engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - f"--gpt_attention_plugin={data_type}", - f"--gemm_plugin={data_type}", - "--remove_input_padding=enable", - f"--max_beam_width={num_beams}", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - if run_type == "inference": - print("Run inference...") - venv_check_call(llm_venv, [ - f"{llama_example_root}/../run.py", - "--max_output_len=50", - f"--tokenizer_dir={llama_model_root}", - f"--engine_dir={engine_dir}", - f"--num_beams={num_beams}", - ]) - elif run_type == "summarization": - print("Run summarize...") - tensorrt_llm_rouge1_threshold = { - 1: 14, - 2: 19, - 4: 19, - }[num_beams] - - summary_cmd = generate_summary_cmd( - llama_example_root, - hf_model_dir=llama_model_root, - data_type="fp16", - engine_dir=engine_dir, - tensorrt_llm_rouge1_threshold=tensorrt_llm_rouge1_threshold, - num_beams=num_beams, - dataset_dir=llm_datasets_root, - rouge_dir=llm_rouge_root) - - venv_check_call(llm_venv, summary_cmd) - - -@pytest.mark.parametrize( - "data_type", ['float16', 'fp8', 'sq_ootb', 'awq', 'int8_wo'], - ids=['base_fp16', 'base_fp8', 'base_sq_ootb', 'base_awq', 'base_int8_wo']) -@pytest.mark.parametrize("llama_model_root", ['llama-v3-8b-hf'], indirect=True) -@pytest.mark.parametrize("llm_dora_model_root", - ['commonsense-llama-v3-8b-dora-r32'], - indirect=True) -def test_llm_llama_v3_dora_1gpu(data_type, llama_example_root, llama_model_root, - llm_dora_model_root, llm_datasets_root, - llm_venv, cmodel_dir, engine_dir, - qcache_dir_without_install_package): - "run llama dora test on 1gpu" - print("Build engines...") - - model_name = 'llama_v3-dora' - if data_type == 'fp8': - skip_fp8_pre_ada(use_fp8=True) - - model_dir = quantize_data( - llm_venv, - llama_example_root, - model_dir=llama_model_root, - calib_dataset=f"{llm_datasets_root}/cnn_dailymail", - dtype="float16", - qformat="fp8", - quantize_dir=qcache_dir_without_install_package, - calib_size=512, - kv_cache_dtype="fp8") - elif data_type == 'sq_ootb': - model_dir = quantize_data( - llm_venv, - llama_example_root, - model_dir=llama_model_root, - calib_dataset=f"{llm_datasets_root}/cnn_dailymail", - dtype="float16", - qformat="int8_sq", - quantize_dir=qcache_dir_without_install_package, - calib_size=32) - elif data_type == 'awq': - model_dir = quantize_data( - llm_venv, - llama_example_root, - model_dir=llama_model_root, - calib_dataset=f"{llm_datasets_root}/cnn_dailymail", - dtype="float16", - qformat="int4_awq", - awq_block_size=128, - quantize_dir=qcache_dir_without_install_package, - calib_size=32) - elif data_type == 'int8_wo': - model_dir = convert_weights(llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llama_model_root, - use_weight_only=True, - weight_only_precision='int8') - else: - model_dir = convert_weights(llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llama_model_root) - - # normalize dora magnitude - dora_weights = f"{llm_venv.get_working_directory()}/dora_weights" - - normalize_cmd = [ - f"{llama_example_root}/../../../dora/normalize_weights.py", "-i", - llm_dora_model_root, "-b", llama_model_root, "-o", dora_weights - ] - - venv_check_call(llm_venv, normalize_cmd) - - print("Build engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - "--lora_plugin=auto", - "--dora_plugin=enable", - "--remove_input_padding=enable", # otherwise no cpp runtime - "--kv_cache_type=paged", # otherwise no cpp runtime - "--gemm_plugin=auto", - f"--lora_dir={dora_weights}", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - input_tokens = [ - 128000, 39314, 374, 459, 7754, 430, 16964, 264, 3465, 11, 35526, 449, - 459, 1988, 430, 5825, 4726, 2317, 13, 9842, 264, 2077, 430, 36001, - 45695, 279, 1715, 382, 394, 17010, 30151, 512, 394, 5321, 5268, 279, - 4495, 4320, 311, 279, 3488, 25, 578, 842, 1121, 304, 279, 1920, 315, - 7397, 74767, 374, 279, 5788, 315, 13465, 323, 24463, 13, 16299, 3094, - 17738, 279, 7314, 315, 7397, 74767, 31931, 16533, 16, 25, 36424, 4907, - 374, 42101, 1555, 279, 20282, 13, 22559, 17, 25, 8828, 4907, 374, 16489, - 311, 11742, 4907, 13, 22559, 18, 25, 92479, 5237, 25734, 304, 279, - 16312, 41255, 3177, 4907, 13, 22559, 19, 25, 8219, 4238, 374, 16489, - 1139, 37833, 5237, 25734, 4286, 16533, 3645, 25, 4320, 16, 14, 9399, 17, - 14, 9399, 18, 14, 9399, 19, 271, 394, 17010, 5688, 512, 72348, 394, - 17010, 6075, 1473 - ] - - out_ref = [ - 128000, 39314, 374, 459, 7754, 430, 16964, 264, 3465, 11, 35526, 449, - 459, 1988, 430, 5825, 4726, 2317, 13, 9842, 264, 2077, 430, 36001, - 45695, 279, 1715, 382, 394, 17010, 30151, 512, 394, 5321, 5268, 279, - 4495, 4320, 311, 279, 3488, 25, 578, 842, 1121, 304, 279, 1920, 315, - 7397, 74767, 374, 279, 5788, 315, 13465, 323, 24463, 13, 16299, 3094, - 17738, 279, 7314, 315, 7397, 74767, 31931, 16533, 16, 25, 36424, 4907, - 374, 42101, 1555, 279, 20282, 13, 22559, 17, 25, 8828, 4907, 374, 16489, - 311, 11742, 4907, 13, 22559, 18, 25, 92479, 5237, 25734, 304, 279, - 16312, 41255, 3177, 4907, 13, 22559, 19, 25, 8219, 4238, 374, 16489, - 1139, 37833, 5237, 25734, 4286, 16533, 3645, 25, 4320, 16, 14, 9399, 17, - 14, 9399, 18, 14, 9399, 19, 271, 394, 17010, 5688, 512, 72348, 394, - 17010, 6075, 1473, 394, 279, 4495, 4320, 374, 4320, 18, 128001, 128001, - 128001, 128001, 128001, 128001, 128001, 128001, 128001, 128001, 128001, - 128001, 128001, 128001, 128001, 128001, 128001, 128001, 128001, 128001, - 128001, 128001, 128001, 128001, 128001 - ] - - in_csv = f"{llm_venv.get_working_directory()}/input.csv" - out_csv = f"{llm_venv.get_working_directory()}/output.csv" - with open(in_csv, "w") as f: - writer = csv.writer(f) - writer.writerow(input_tokens) - - base_run_cmd = [ - f"{llama_example_root}/../../../run.py", "--max_output_len=20", - f"--input_file={in_csv}", f"--tokenizer_dir={llama_model_root}", - f"--engine_dir={engine_dir}", "--max_output_len=32" - ] - - for use_py_session in [True, False]: - if use_py_session: - print("Run inference with Python runtime...") - else: - print("Run inference with C++ runtime...") - - print(f"Run inference with lora id 0...") - run_cmd = copy.deepcopy(base_run_cmd) - run_cmd.extend(["--lora_task_uids=0", f"--output_csv={out_csv}"]) - if use_py_session: - run_cmd.append("--use_py_session") - venv_check_call(llm_venv, run_cmd) - - with open(out_csv) as f: - predict = csv.reader(f) - predict = next(predict) - - predict = [int(p) for p in predict] - assert out_ref == predict or data_type != "float16" - - -@pytest.mark.skip_less_device(8) -@pytest.mark.skip_less_device_memory(80000) -@pytest.mark.parametrize("num_beams", [1, 4], - ids=lambda num_beams: f'nb:{num_beams}') -@pytest.mark.parametrize( - "tp_pp_size", [(8, 1), (4, 2)], - ids=lambda tp_pp_size: f'tp{tp_pp_size[0]}pp{tp_pp_size[1]}') -@pytest.mark.parametrize("test_case", ["pg64317"], indirect=True) -def test_llm_llama_long_alpaca_8gpu_summary(llama_example_root, - llm_long_alpaca_model_root, - llm_datasets_root, llm_rouge_root, - llm_venv, cmodel_dir, engine_dir, - num_beams, tp_pp_size, test_case): - "llama test for long alpaca" - tp_size, pp_size = tp_pp_size - world_size = 8 - assert tp_size * pp_size == world_size, \ - f'tp_size({tp_size}) x pp_size({pp_size}) != 8' - - model_name = 'llama_long_alpaca' - model_dir = convert_weights(llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llm_long_alpaca_model_root, - gpus=world_size, - tp_size=tp_size, - pp_size=pp_size, - data_type="bfloat16") - - print("Build engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - "--gpt_attention_plugin=bfloat16", - "--remove_input_padding=enable", - "--gemm_plugin=bfloat16", - f"--max_beam_width={num_beams}", - "--max_input_len=32768", - "--max_seq_len=49152", - "--max_batch_size=1", - "--max_num_tokens=32768", - ] - print("Build engines...") - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run summarize...") - max_output_len = test_case["max_output_len"] - run_cmd = [ - f"{llama_example_root}/../../../run.py", - f"--max_output_len={max_output_len}", - f"--input_file={test_case['input_file']}", f"--engine_dir={engine_dir}", - f"--num_beams={num_beams}", - f"--tokenizer_dir={llm_long_alpaca_model_root}", - "--max_input_length=32768" - ] - - venv_mpi_check_call( - llm_venv, ["mpirun", "-n", f"{world_size}", "--allow-run-as-root"], - run_cmd) - - summary_cmd = generate_summary_cmd(llama_example_root, - hf_model_dir=llm_long_alpaca_model_root, - max_input_length=16384, - output_len=max_output_len, - data_type="fp16", - num_beams=num_beams, - engine_dir=engine_dir, - dataset_dir=llm_datasets_root, - rouge_dir=llm_rouge_root) - - venv_mpi_check_call( - llm_venv, ["mpirun", "-n", f"{world_size}", "--allow-run-as-root"], - summary_cmd) - - -@pytest.mark.parametrize("fp8_quant", [ - 'disable_fp8', - pytest.param('enable_fp8', marks=skip_post_blackwell), - pytest.param('enable_fp8_meta_recipe', marks=skip_post_blackwell) -]) -@pytest.mark.parametrize("llama_model_root", ['llama-3.1-8b', 'llama-3.2-1b'], - indirect=True) -def test_llm_llama_v3_1_1node_single_gpu(llama_example_root, llama_model_root, - llm_venv, cmodel_dir, - llm_datasets_root, llm_rouge_root, - engine_dir, fp8_quant): - "Run llama3.1 test on 1 gpu." - data_type = "bfloat16" - model_name = os.path.basename(llama_model_root) - - use_fp8_rowwise = False - use_meta_fp8_rowwise_recipe = False - if fp8_quant == 'enable_fp8': - use_fp8_rowwise = True - elif fp8_quant == 'enable_fp8_meta_recipe': - use_fp8_rowwise = True - use_meta_fp8_rowwise_recipe = True - - print("Convert weight...") - model_dir = convert_weights( - llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llama_model_root, - data_type=data_type, - tp_size=1, - pp_size=1, - use_fp8_rowwise=use_fp8_rowwise, - use_meta_fp8_rowwise_recipe=use_meta_fp8_rowwise_recipe) - - print("Build engines...") - build_cmd = [ - "trtllm-build", f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", f"--max_batch_size={8}", - f"--max_seq_len={2048}" - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run summarize...") - summary_cmd = [ - f"{llama_example_root}/../../../summarize.py", - "--test_trt_llm", - f"--hf_model_dir={llama_model_root}", - f"--engine_dir={engine_dir}", - "--check_accuracy", - f"--tensorrt_llm_rouge1_threshold={14}", - f"--dataset_dir={llm_datasets_root}", - f"--rouge_dir={llm_rouge_root}", - ] - venv_check_call(llm_venv, summary_cmd) - - -@skip_pre_ada -@pytest.mark.skip_less_device_memory(80000) -@pytest.mark.parametrize( - "llama_model_root", - ['llama-v3-8b-instruct-hf', 'llama-3.1-8b', 'llama-3.2-1b', 'llama-3.2-3b'], - indirect=True) -def test_llama_3_x_fp8_with_bf16_lora(llama_example_root, llm_datasets_root, - qcache_dir_without_install_package, - llm_venv, engine_dir, llama_model_root): - "Run Llama 3.1 and 3.2 models with multiple dummy LoRAs." - - print("Quantizing model to fp8...") - - defs.ci_profiler.start("quantize_model") - qmodel_dir = quantize_data( - llm_venv, - llama_example_root, - model_dir=llama_model_root, - calib_dataset=f"{llm_datasets_root}/cnn_dailymail", - dtype="bfloat16", - qformat="fp8", - quantize_dir=qcache_dir_without_install_package, - calib_size=32, - kv_cache_dtype="fp8") - defs.ci_profiler.stop("quantize_model") - print( - f"quantize_model: {defs.ci_profiler.elapsed_time_in_sec('quantize_model')} sec" - ) - - defs.ci_profiler.start("test_multi_lora_support") - test_multi_lora_support( - hf_model_dir=llama_model_root, - tllm_ckpt_dir=qmodel_dir, - engine_dir=engine_dir, - llm_venv=llm_venv, - example_root=llama_example_root, - num_loras=2, - lora_rank=8, - target_hf_modules=["q_proj", "k_proj", "v_proj"], - target_trtllm_modules=["attn_q", "attn_k", "attn_v"], - zero_lora_weights=True, - ) - defs.ci_profiler.stop("test_multi_lora_support") - print( - f"test_multi_lora_support: {defs.ci_profiler.elapsed_time_in_sec('test_multi_lora_support')} sec" - ) - - -@skip_pre_ada -@pytest.mark.skip_less_device_memory(80000) -@pytest.mark.parametrize("llama_model_root", [ - 'llama-v3-8b-instruct-hf', - 'llama-3.1-8b-instruct', - 'llama-3.2-1b-instruct', - 'llama-3.2-3b-instruct', - 'llama-3.3-70b-instruct', -], - indirect=True) -def test_llama_3_x_with_bf16_lora_torch(llama_example_root, llm_datasets_root, - qcache_dir_without_install_package, - llm_venv, engine_dir, llama_model_root): - """Run Llama models with multiple dummy LoRAs using LLM-API Torch backend.""" - - if "llama-3.3-70b-instruct" in llama_model_root.lower(): - tensor_parallel_size = 8 - if get_device_count() < 8: - pytest.skip( - "Skipping: llama-3.3-70b-instruct model requires 8 GPUs") - else: - tensor_parallel_size = 1 - - print("Testing with LLM-API Torch backend...") - - defs.ci_profiler.start("test_llm_torch_multi_lora_support") - - test_llm_torch_multi_lora_support( - hf_model_dir=llama_model_root, - llm_venv=llm_venv, - num_loras=2, - lora_rank=8, - target_hf_modules=["q_proj", "k_proj", "v_proj"], - zero_lora_weights=True, - tensor_parallel_size=tensor_parallel_size) - defs.ci_profiler.stop("test_llm_torch_multi_lora_support") - print( - f"test_llm_torch_multi_lora_support: {defs.ci_profiler.elapsed_time_in_sec('test_llm_torch_multi_lora_support')} sec" - ) diff --git a/tests/integration/defs/examples/test_mamba.py b/tests/integration/defs/examples/test_mamba.py deleted file mode 100644 index c771278bdbe2..000000000000 --- a/tests/integration/defs/examples/test_mamba.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import pytest -from defs.common import (convert_weights, generate_summary_cmd, venv_check_call, - venv_mpi_check_call) -from defs.conftest import get_sm_version, skip_post_blackwell -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@pytest.mark.parametrize("gemm_plugin", [True, False], - ids=["enable_gemm_plugin", "disable_gemm_plugin"]) -@pytest.mark.parametrize("dtype", ['bfloat16', 'float16']) -@pytest.mark.parametrize("mamba_model_root", [ - pytest.param('mamba-130m', marks=skip_post_blackwell), 'mamba-2.8b', - 'mamba-1.4b', 'mamba-790m', 'mamba-370m', 'mamba2-130m', 'mamba2-2.7b', - 'mamba2-1.3b', 'mamba2-780m', 'mamba2-370m', - pytest.param('mamba-codestral-7B-v0.1', marks=skip_post_blackwell) -], - indirect=True) -def test_llm_mamba_1gpu(mamba_example_root, mamba_model_root, - llm_gptneox_model_root, llm_mathstral_model_root, - llm_datasets_root, llm_rouge_root, llm_venv, - gemm_plugin, dtype, cmodel_dir, engine_dir): - "Build & Run mamba model with one gpu" - print("Build engines...") - - model_name = os.path.basename(mamba_model_root) - model_dir = convert_weights(llm_venv=llm_venv, - example_root=mamba_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=mamba_model_root, - data_type=dtype) - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - "--paged_kv_cache=disable", - "--max_batch_size=8", - ] - if gemm_plugin: - build_cmd.append("--gemm_plugin=auto") - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print(f'Run {model_name}...') - tokenizer_dir = llm_mathstral_model_root if model_name == "mamba-codestral-7B-v0.1" else llm_gptneox_model_root - summary_cmd = generate_summary_cmd(mamba_example_root, - hf_model_dir=mamba_model_root, - tokenizer_dir=tokenizer_dir, - data_type=dtype, - engine_dir=engine_dir, - batch_size=8, - tensorrt_llm_rouge1_threshold="13.5", - dataset_dir=llm_datasets_root, - rouge_dir=llm_rouge_root) - - venv_check_call(llm_venv, summary_cmd) - - -@pytest.mark.parametrize("mamba_model_root", ['mamba-codestral-7B-v0.1'], - indirect=True) -def test_llm_mamba2_2gpu(mamba_example_root, mamba_model_root, - llm_gptneox_model_root, llm_mathstral_model_root, - llm_datasets_root, llm_rouge_root, llm_venv, - cmodel_dir, engine_dir): - "Build & Run mamba2 model with two gpus" - print("Build engines...") - - model_name = mamba_model_root.split('/')[-1] - model_dir = convert_weights(llm_venv=llm_venv, - example_root=mamba_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=mamba_model_root, - data_type='float16', - tp_size=2) - build_cmd = [ - "trtllm-build", - "--gemm_plugin=auto", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - "--paged_kv_cache=disable", - "--max_batch_size=8", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print(f'Run {model_name}...') - tokenizer_dir = llm_mathstral_model_root - summary_cmd = generate_summary_cmd(mamba_example_root, - hf_model_dir=mamba_model_root, - tokenizer_dir=tokenizer_dir, - data_type='float16', - engine_dir=engine_dir, - batch_size=8, - tensorrt_llm_rouge1_threshold="19.0", - dataset_dir=llm_datasets_root, - rouge_dir=llm_rouge_root) - - venv_mpi_check_call(llm_venv, ["mpirun", "-n", "2", "--allow-run-as-root"], - summary_cmd) diff --git a/tests/integration/defs/examples/test_medusa.py b/tests/integration/defs/examples/test_medusa.py deleted file mode 100644 index 49975a591569..000000000000 --- a/tests/integration/defs/examples/test_medusa.py +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import pytest -from defs.common import convert_weights, venv_check_call -from defs.conftest import get_sm_version, skip_post_blackwell -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@skip_post_blackwell -@pytest.mark.parametrize("batch_size", [1, 8], ids=['bs1', 'bs8']) -@pytest.mark.parametrize("data_type", ['bfloat16']) -@pytest.mark.parametrize("num_medusa_heads", [4], ids=['4-heads']) -@pytest.mark.parametrize("medusa_model_roots", ["medusa-vicuna-7b-v1.3"], - indirect=True) -@pytest.mark.parametrize("use_py_session", [False, True], - ids=["use_cpp_session", "use_py_session"]) -def test_llm_medusa_1gpu(batch_size, data_type, medusa_model_roots, - medusa_example_root, llm_datasets_root, llm_rouge_root, - num_medusa_heads, llm_venv, cmodel_dir, engine_dir, - use_py_session): - print("Build engines...") - model_name = "medusa" - - model_dir = convert_weights(llm_venv=llm_venv, - example_root=medusa_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=medusa_model_roots, - data_type=data_type) - - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - f"--gpt_attention_plugin={data_type}", - f"--gemm_plugin={data_type}", - f"--max_beam_width=1", - "--remove_input_padding=enable", - "--context_fmha=enable", - "--max_input_len=1024", - "--max_seq_len=1536", - f"--max_batch_size={batch_size}", - "--paged_kv_cache=enable", - '--speculative_decoding_mode=medusa', - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run summarize...") - - summary_cmd = [ - f"{medusa_example_root}/../summarize.py", "--test_trt_llm", - "--hf_model_dir", f"{medusa_model_roots[0]}", "--tokenizer_dir", - f"{medusa_model_roots[0]}", f"--engine_dir={engine_dir}", - "--check_accuracy", "--tensorrt_llm_rouge1_threshold=24", - "--medusa_choices=[[0], [0, 0], [1], [0, 1], [2], [0, 0, 0], [1, 0], [0, 2], [3], [0, 3], [4], [0, 4], [2, 0], [0, 5], [0, 0, 1], [5], [0, 6], [6], [0, 7], [0, 1, 0], [1, 1], [7], [0, 8], [0, 0, 2], [3, 0], [0, 9], [8], [9], [1, 0, 0], [0, 2, 0], [1, 2], [0, 0, 3], [4, 0], [2, 1], [0, 0, 4], [0, 0, 5], [0, 0, 0, 0], [0, 1, 1], [0, 0, 6], [0, 3, 0], [5, 0], [1, 3], [0, 0, 7], [0, 0, 8], [0, 0, 9], [6, 0], [0, 4, 0], [1, 4], [7, 0], [0, 1, 2], [2, 0, 0], [3, 1], [2, 2], [8, 0], [0, 5, 0], [1, 5], [1, 0, 1], [0, 2, 1], [9, 0], [0, 6, 0], [0, 0, 0, 1], [1, 6], [0, 7, 0]]", - f"--temperature=1.0", f"--max_ite=40", f"--batch_size={batch_size}", - f"--dataset_dir={llm_datasets_root}", f"--rouge_dir={llm_rouge_root}" - ] - - if use_py_session: - summary_cmd.append("--use_py_session") - - venv_check_call(llm_venv, summary_cmd) - - -@skip_post_blackwell -@pytest.mark.parametrize("batch_size", [1, 8], ids=['bs1', 'bs8']) -@pytest.mark.parametrize("data_type", ['bfloat16', 'float16']) -@pytest.mark.parametrize("num_medusa_heads", [4], ids=['4-heads']) -@pytest.mark.parametrize("medusa_model_roots", ["medusa-vicuna-7b-v1.3"], - indirect=True) -@pytest.mark.parametrize("use_py_session", [False, True], - ids=["use_cpp_session", "use_py_session"]) -@pytest.mark.parametrize("base_model_datatype", ['fp8']) -def test_llm_medusa_with_qaunt_base_model_1gpu( - batch_size, data_type, medusa_model_roots, medusa_example_root, - base_model_datatype, llm_datasets_root, llm_rouge_root, - num_medusa_heads, llm_venv, cmodel_dir, engine_dir, use_py_session): - - model_name = f"vicuna_meudsa_quant_base_mode_{base_model_datatype}" - quant_model_ckpt_output_path = os.path.join(cmodel_dir, model_name) - - print("Quant base model to FP8 and combine medusa head") - quant_cmd = [ - f"{medusa_example_root}/../quantization/quantize.py", - f"--model_dir={medusa_model_roots[0]}", f"--dtype={data_type}", - f"--qformat={base_model_datatype}", - f"--kv_cache_dtype={base_model_datatype}", - f"--output_dir={quant_model_ckpt_output_path}", "--calib_size=512", - f"--medusa_model_dir={medusa_model_roots[1]}", - f"--num_medusa_heads={num_medusa_heads}" - ] - - # https://nvbugs/4658787 - # WAR before medusa tests can work offline - env = {"HF_DATASETS_OFFLINE": "0"} - venv_check_call(llm_venv, quant_cmd, env=env) - - print("Build engines...") - - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={quant_model_ckpt_output_path}", - f"--output_dir={engine_dir}", - f"--gpt_attention_plugin={data_type}", - f"--gemm_plugin={data_type}", - f"--max_beam_width=1", - "--remove_input_padding=enable", - "--context_fmha=enable", - "--max_input_len=1024", - "--max_seq_len=1536", - f"--max_batch_size={batch_size}", - "--paged_kv_cache=enable", - '--speculative_decoding_mode=medusa', - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run summarize...") - - summary_cmd = [ - f"{medusa_example_root}/../summarize.py", "--test_trt_llm", - "--hf_model_dir", f"{medusa_model_roots[0]}", "--tokenizer_dir", - f"{medusa_model_roots[0]}", f"--engine_dir={engine_dir}", - "--check_accuracy", "--tensorrt_llm_rouge1_threshold=24", - "--medusa_choices=[[0], [0, 0], [1], [0, 1], [2], [0, 0, 0], [1, 0], [0, 2], [3], [0, 3], [4], [0, 4], [2, 0], [0, 5], [0, 0, 1], [5], [0, 6], [6], [0, 7], [0, 1, 0], [1, 1], [7], [0, 8], [0, 0, 2], [3, 0], [0, 9], [8], [9], [1, 0, 0], [0, 2, 0], [1, 2], [0, 0, 3], [4, 0], [2, 1], [0, 0, 4], [0, 0, 5], [0, 0, 0, 0], [0, 1, 1], [0, 0, 6], [0, 3, 0], [5, 0], [1, 3], [0, 0, 7], [0, 0, 8], [0, 0, 9], [6, 0], [0, 4, 0], [1, 4], [7, 0], [0, 1, 2], [2, 0, 0], [3, 1], [2, 2], [8, 0], [0, 5, 0], [1, 5], [1, 0, 1], [0, 2, 1], [9, 0], [0, 6, 0], [0, 0, 0, 1], [1, 6], [0, 7, 0]]", - f"--temperature=1.0", f"--max_ite=40", f"--batch_size={batch_size}", - f"--dataset_dir={llm_datasets_root}", f"--rouge_dir={llm_rouge_root}" - ] - - if use_py_session: - summary_cmd.append("--use_py_session") - - venv_check_call(llm_venv, summary_cmd) diff --git a/tests/integration/defs/examples/test_mistral.py b/tests/integration/defs/examples/test_mistral.py deleted file mode 100644 index b5d07fb015cb..000000000000 --- a/tests/integration/defs/examples/test_mistral.py +++ /dev/null @@ -1,219 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Module test_mistral test mistral examples.""" -import multiprocessing - -import defs.ci_profiler -import psutil -import pytest -from defs.common import (convert_weights, test_llm_torch_multi_lora_support, - venv_check_call) -from defs.conftest import (get_device_count, get_sm_version, - skip_post_blackwell, skip_pre_ada) -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -def get_optimal_jobs(): - cpu_count = multiprocessing.cpu_count() - available_memory = psutil.virtual_memory().available / (1024 * 1024 * 1024) - memory_per_job = 4 - memory_based_jobs = int(available_memory / memory_per_job) - system_load = psutil.getloadavg()[0] / cpu_count - if system_load > 0.7: - cpu_factor = 0.5 - else: - cpu_factor = 0.75 - cpu_based_jobs = max(1, int(cpu_count * cpu_factor)) - optimal_jobs = max(1, min(cpu_based_jobs, memory_based_jobs)) - return optimal_jobs - - -@skip_post_blackwell #nvbug 5298661 -@pytest.mark.parametrize( - "run_type", - ['inference', 'summarization_long', 'chunked_summarization_long']) -@pytest.mark.parametrize("max_attention_window", [4096], - ids=['max_attention_window_size_4096']) -@pytest.mark.parametrize("data_type", ['float16']) -@pytest.mark.parametrize("llm_mistral_model_root", ['mistral-7b-v0.1'], - indirect=True) -def test_llm_mistral_v1_1gpu(run_type, data_type, llama_example_root, - max_attention_window, llm_mistral_model_root, - llm_datasets_root, llm_rouge_root, llm_venv, - cmodel_dir, engine_dir): - - print("Build engines...") - if run_type == "summarization_long": - model_name = 'mistral-{}'.format(run_type) - model_dir = convert_weights(llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llm_mistral_model_root, - data_type=data_type) - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - "--max_input_len", - "6400", - f"--max_batch_size={1}", - "--max_seq_len", - "6528", - f"--gpt_attention_plugin={data_type}", - f"--gemm_plugin={data_type}", - "--context_fmha=enable", - "--use_paged_context_fmha=disable", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run long context summarize...") - # using shorter input length since A30 doesn't have enough device memory. - summary_cmd = [ - f"{llama_example_root}/summarize_long.py", - "--test_trt_llm", - "--test_hf", - "--hf_model_location", - f"{llm_mistral_model_root}", - "--data_type", - "fp16", - f"--engine_dir={engine_dir}", - f"--max_attention_window_size={max_attention_window}", - "--max_ite", - "3", - "--max_input_len", - "6400", - "--tensorrt_llm_rouge1_threshold", - "90", - "--check_accuracy", - ] - # https://nvbugs/4658787 - # WAR before summarize_long.py can work offline - env = {"HF_DATASETS_OFFLINE": "0"} - venv_check_call(llm_venv, summary_cmd, env=env) - - # multi block + sliding window attention tests. - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - "--max_input_len", - "6400", - "--max_seq_len", - "6528", - f"--gpt_attention_plugin={data_type}", - f"--gemm_plugin={data_type}", - "--use_paged_context_fmha=disable", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run long context summarize with multi_block_mode enabled...") - # using shorter input length since A30 doesn't have enough device memory. - summary_cmd = [ - f"{llama_example_root}/summarize_long.py", "--test_trt_llm", - "--test_hf", "--hf_model_location", f"{llm_mistral_model_root}", - "--data_type", "fp16", f"--engine_dir={engine_dir}", - f"--max_attention_window_size={max_attention_window}", "--max_ite", - "3", "--max_input_len", "6400", "--tensorrt_llm_rouge1_threshold", - "90", "--check_accuracy" - ] - venv_check_call(llm_venv, summary_cmd, env=env) - - elif run_type == "chunked_summarization_long": - model_name = 'mistral-{}'.format(run_type) - model_dir = convert_weights(llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llm_mistral_model_root, - data_type=data_type) - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - "--max_input_len", - "6400", - "--max_num_tokens=2048", - "--use_paged_context_fmha=enable", - f"--max_batch_size={1}", - "--max_seq_len", - "6528", - f"--gpt_attention_plugin={data_type}", - f"--gemm_plugin={data_type}", - "--context_fmha=enable", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run long context summarize...") - summary_cmd = [ - f"{llama_example_root}/../../../summarize.py", - "--eval_task=summarize_long", "--test_trt_llm", "--test_hf", - "--hf_model_dir", f"{llm_mistral_model_root}", "--data_type", - "fp16", f"--engine_dir={engine_dir}", - f"--max_attention_window_size={max_attention_window}", - "--max_input_length", "6400", "--tensorrt_llm_rouge1_threshold", - "21", "--check_accuracy", "--enable_chunked_context" - ] - # https://nvbugs/4658787 - # WAR before summarize_long.py can work offline - env = {"HF_DATASETS_OFFLINE": "0"} - venv_check_call(llm_venv, summary_cmd, env=env) - - -@skip_pre_ada -@pytest.mark.skip_less_device_memory(80000) -@pytest.mark.parametrize("llm_mistral_model_root", [ - 'mistral-7b-v0.1', - 'mistral-nemo-instruct-2407', -], - indirect=True) -def test_mistral_with_bf16_lora_torch(llama_example_root, llm_datasets_root, - qcache_dir_without_install_package, - llm_venv, engine_dir, - llm_mistral_model_root): - """Run Mistral models with multiple dummy LoRAs using LLM-API Torch backend.""" - - if "mistral-nemo-instruct-2407" in llm_mistral_model_root.lower(): - tensor_parallel_size = 2 - if get_device_count() < 2: - pytest.skip( - "Skipping: mistral-nemo-instruct-2407 model requires 2 GPUs") - else: - tensor_parallel_size = 1 - - print(f"Testing {llm_mistral_model_root} with LLM-API Torch backend...") - - defs.ci_profiler.start("test_llm_torch_multi_lora_support") - test_llm_torch_multi_lora_support( - hf_model_dir=llm_mistral_model_root, - llm_venv=llm_venv, - num_loras=2, - lora_rank=8, - target_hf_modules=["q_proj", "k_proj", "v_proj"], - zero_lora_weights=True, - tensor_parallel_size=tensor_parallel_size) - defs.ci_profiler.stop("test_llm_torch_multi_lora_support") - print( - f"test_llm_torch_multi_lora_support: {defs.ci_profiler.elapsed_time_in_sec('test_llm_torch_multi_lora_support')} sec" - ) diff --git a/tests/integration/defs/examples/test_mixtral.py b/tests/integration/defs/examples/test_mixtral.py deleted file mode 100644 index 56d762f45257..000000000000 --- a/tests/integration/defs/examples/test_mixtral.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import csv -import os - -import pytest -from defs.common import convert_weights, venv_mpi_check_call -from defs.conftest import get_sm_version -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@pytest.mark.skip_less_device(4) -@pytest.mark.skip_less_device_memory(45000) -@pytest.mark.parametrize("llm_lora_model_root", ["chinese-mixtral-lora"], - indirect=True) -@pytest.mark.parametrize("llm_mixtral_model_root", ["Mixtral-8x7B-v0.1"], - indirect=True) -def test_llm_mixtral_moe_plugin_lora_4gpus( - llama_example_root, - llm_mixtral_model_root, - llm_venv, - cmodel_dir, - engine_dir, - llm_lora_model_root, -): - "run Mixtral MoE lora test on 4 gpu." - print("Build engines...") - dtype = 'float16' - model_name = os.path.basename(llm_mixtral_model_root) - ckpt_dir = convert_weights(llm_venv=llm_venv, - example_root=llama_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - tp_size=4, - pp_size=1, - model_path=llm_mixtral_model_root, - data_type=dtype) - - print("Build engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - "--lora_plugin=auto", - "--moe_plugin=auto", - f"--lora_dir={llm_lora_model_root}", - "--worker=4", - "--max_batch_size=8", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - ref_1 = [ - 1, 28705, 29242, 30731, 31182, 235, 158, 142, 234, 182, 152, 28924, - 29926, 28971, 29242, 28988 - ] - ref_2 = [ - 1, 315, 2016, 285, 4284, 526, 5680, 28723, 28705, 28740, 28723, 661 - ] - - input_text = "我爱吃蛋糕" - print("Run inference with lora id 0...") - run_cmd = [ - f"{llama_example_root}/../../../run.py", - "--max_output_len=5", - f"--input_text={input_text}", - "--lora_task_uids=0", - f"--tokenizer_dir={llm_lora_model_root}", - f"--engine_dir={engine_dir}", - f"--output_csv={llm_venv.get_working_directory()}/use_lora.csv", - "--use_py_session", - ] - venv_mpi_check_call(llm_venv, ["mpirun", "-n", "4", "--allow-run-as-root"], - run_cmd) - - with open(f"{llm_venv.get_working_directory()}/use_lora.csv") as f: - predict = csv.reader(f) - predict = next(predict) - predict = [int(p) for p in predict] - assert ref_1 == predict - - print("Run inference with lora id -1...") - input_text = "I love french quiche" - run_cmd = [ - f"{llama_example_root}/../../../run.py", - "--max_output_len=5", - f"--input_text={input_text}", - "--lora_task_uids=-1", - f"--tokenizer_dir={llm_lora_model_root}", - f"--engine_dir={engine_dir}", - f"--output_csv={llm_venv.get_working_directory()}/no_lora.csv", - "--use_py_session", - ] - venv_mpi_check_call(llm_venv, ["mpirun", "-n", "4", "--allow-run-as-root"], - run_cmd) - - with open(f"{llm_venv.get_working_directory()}/no_lora.csv") as f: - predict = csv.reader(f) - predict = next(predict) - predict = [int(p) for p in predict] - assert ref_2 == predict diff --git a/tests/integration/defs/examples/test_multimodal.py b/tests/integration/defs/examples/test_multimodal.py deleted file mode 100644 index 591205e9afd7..000000000000 --- a/tests/integration/defs/examples/test_multimodal.py +++ /dev/null @@ -1,726 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import pytest -import torch -from defs.common import convert_weights, venv_check_call, venv_mpi_check_call -from defs.conftest import (get_device_memory, get_sm_version, - skip_post_blackwell, skip_pre_ada) -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@pytest.fixture(scope="module") -def multimodal_example_root(llm_root): - "Get multimodal example root" - example_root = os.path.join(llm_root, "examples", "models", "core", - "multimodal") - - return example_root - - -@pytest.fixture(scope="function") -def recover_transformers(llm_venv, llm_root): - "Recover transformers" - - yield - - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(llm_root, "requirements.txt") - ]) - - -def _call_run_cmd(llm_venv, llm_root, cmd, world_size): - if world_size == 1: - venv_check_call(llm_venv, cmd) - else: - venv_mpi_check_call( - llm_venv, ["mpirun", "-n", - str(world_size), "--allow-run-as-root"], cmd) - - -dataset_path_mapping = { - 'cnn_dailymail': 'cnn_dailymail', - 'scienceqa': 'derek-thomas___science_qa', -} - - -def _test_llm_multimodal_general(llm_venv, - llm_root, - llm_datasets_root, - cmodel_dir, - engine_dir, - batch_size, - data_type, - tp_size, - pp_size, - multimodal_example_root, - multimodal_model_root, - recover_transformers, - calibration_dataset=None, - qformat=None, - kv_cache_dtype=None, - cpp_e2e=False, - num_beams=1): - - # Empty the torch CUDA cache before each multimodal test to reduce risk of OOM errors. - if torch.cuda.is_available(): - torch.cuda.empty_cache() - - world_size = tp_size * pp_size - print("Locate model checkpoints in test storage...") - tllm_model_name, model_ckpt_path = multimodal_model_root - - if "neva-22b" in tllm_model_name and get_device_memory() < 80000: - pytest.skip("GPU memory is insufficient.") - if "Mistral-Small" in tllm_model_name and get_device_memory() < 80000: - pytest.skip("GPU memory is insufficient.") - - print("Converting huggingface model into binary format...") - # ckpt from llm_models/ --> cmodels// - model_name = tllm_model_name - model_name = "pix2struct" if model_name == "deplot" else model_name - opt_example_root = multimodal_example_root + "/../models/contrib/opt" - enc_dec_example_root = multimodal_example_root + "/../enc_dec" - llama_example_root = multimodal_example_root + "/../llama" - cogvlm_example_root = multimodal_example_root + "/../cogvlm" - gpt_example_root = multimodal_example_root + "/../gpt" - nemotron_example_root = multimodal_example_root + "/../nemotron" - phi_example_root = multimodal_example_root + "/../phi" - mllama_example_root = multimodal_example_root + "/../mllama" - qwen_example_root = multimodal_example_root + "/../qwen" - internlm_example_root = multimodal_example_root + "/../internlm2" - - opt_model = "opt" in model_name - nougat_model = "nougat" in model_name - gpt_model = "fuyu" in model_name or "neva-22b" in model_name or "kosmos" in model_name - pix2struct_model = "pix2struct" in model_name - enc_dec_model = "t5" in model_name or nougat_model or pix2struct_model - llava_model = "llava" in model_name - llava_next_model = "llava-v1.6" in model_name - llava_next_vision_trtllm_engine_model = "vision-trtllm" in model_name and llava_next_model - llava_onevision_model = "llava-onevision" in model_name - llava_onevision_video_model = "video" in model_name and llava_onevision_model - vila_model = "VILA" in model_name - cogvlm_model = "cogvlm" in model_name - nemotron_model = "video-neva" in model_name - phi3_model = "phi-3" in model_name.lower() - phi4_model = "phi-4" in model_name.lower() - mllama_model = 'Llama-3.2' in model_name - qwen2_vl_model = 'Qwen2-VL' in model_name - internlm_model = 'internlm-xcomposer2' in model_name - mistral_model = 'Mistral-Small' in model_name - if enc_dec_model: - builder_root = enc_dec_example_root - if nougat_model: - model_type = "bart" - if pix2struct_model: - model_type = "pix2struct" - if "t5" in model_name: - model_type = "blip2" - elif gpt_model: - builder_root, model_type = gpt_example_root, "gpt" - elif llava_onevision_model: - builder_root, model_type = qwen_example_root, "qwen" - elif qwen2_vl_model: - builder_root, model_type = qwen_example_root, "qwen" - elif internlm_model: - builder_root, model_type = internlm_example_root, "internlm" - elif llava_model or vila_model: - builder_root, model_type = llama_example_root, "llama" - elif mistral_model: - builder_root, model_type = llama_example_root, "llama" - elif cogvlm_model: - builder_root, model_type = cogvlm_example_root, "cogvlm" - elif nemotron_model: - builder_root, model_type = nemotron_example_root, "nemotron" - elif phi3_model: - model_name = model_name.split('/')[-1] # Remove HF directory name - builder_root, model_type = phi_example_root, "phi-3-vision" - elif phi4_model: - builder_root, model_type = phi_example_root, "phi-4-multimodal" - elif opt_model: - builder_root, model_type = opt_example_root, "blip2" - elif mllama_model: - builder_root, model_type = mllama_example_root, "mllama" - - use_weight_only = (not enc_dec_model) and (data_type in [ - 'int4_weight_only', 'int8_weight_only' - ]) - weight_only_precision = data_type.split('_')[0] if use_weight_only else None - if use_weight_only: data_type = 'float16' - - if vila_model: - print( - "VILA model has dependencies on certain HuggingFace version. Need to pip install until this limitation is removed." - ) - check_call( - f"pip install -r {multimodal_example_root}/requirements-vila.txt", - shell=True, - env=llm_venv._new_env) - elif llava_onevision_model: - check_call( - f"pip install -r {multimodal_example_root}/requirements-llava_onevision.txt", - shell=True, - env=llm_venv._new_env) - elif qwen2_vl_model: - check_call( - f"pip install -r {multimodal_example_root}/requirements-qwen2vl.txt", - shell=True, - env=llm_venv._new_env) - elif internlm_model: - check_call( - f"pip install -r {multimodal_example_root}/requirements-internlm-xcomposer2.txt", - shell=True, - env=llm_venv._new_env) - elif mllama_model: - check_call(f"pip install -r {mllama_example_root}/requirements.txt", - shell=True, - env=llm_venv._new_env) - if qformat == 'fp8': - convert_cmd = [ - f"{multimodal_example_root}/../../../quantization/quantize.py", - f"--model_dir={model_ckpt_path}", - f"--calib_dataset={llm_datasets_root}/{dataset_path_mapping[calibration_dataset]}", - f"--dtype={data_type}", - f"--qformat={qformat}", - f"--kv_cache_dtype={kv_cache_dtype}", - f"--output_dir={cmodel_dir}", - f"--calib_size=16", - ] - venv_check_call(llm_venv, convert_cmd) - converted_weight_dir = cmodel_dir - else: - converted_weight_dir = convert_weights( - llm_venv, - builder_root, - cmodel_dir, - model_name, - model_ckpt_path, - data_type=data_type, - gpus=tp_size, - model_type=model_type, - use_weight_only=use_weight_only, - weight_only_precision=weight_only_precision, - tp_size=tp_size, - pp_size=pp_size, - batch_size=batch_size, - multimodal=True) - - print("Build LLM engines...") - model_name = model_name.split('/')[-1] # Remove HF directory name - llm_engine_dir = f"{engine_dir}/{model_name}/{world_size}-gpu" - if "opt" in model_name or llava_model or vila_model or gpt_model or nemotron_model or phi3_model or phi4_model or qwen2_vl_model or mistral_model: - max_input_len_text = 1024 - max_output_len = 200 - if llava_next_model: - multimodal_len = 4096 - elif llava_onevision_model: - multimodal_len = 7300 - elif llava_model: - multimodal_len = 576 - elif vila_model: - multimodal_len = 196 - elif phi3_model: - multimodal_len = 5120 - elif phi4_model: - multimodal_len = 5120 - elif mistral_model: - multimodal_len = 5120 - elif "fuyu" in model_name: - multimodal_len = 2640 - elif "neva-22b" in model_name: - multimodal_len = 729 - elif "video-neva" in model_name: - multimodal_len = 3072 - elif "kosmos" in model_name: - multimodal_len = 64 - elif "Qwen2-VL" in model_name: - multimodal_len = 3552 - else: - multimodal_len = 32 - max_input_len = max_input_len_text + batch_size * multimodal_len - max_seq_len = max_input_len + max_output_len - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}", - f"--output_dir={llm_engine_dir}/llm", - f"--gpt_attention_plugin {data_type}", - f"--gemm_plugin={data_type}", - f"--max_batch_size={batch_size}", - f"--max_multimodal_len={batch_size * multimodal_len}", - f"--max_input_len={max_input_len}", - f"--max_seq_len={max_seq_len}", - f"--max_num_tokens={max_input_len}", - f"--max_beam_width={num_beams}", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - elif internlm_model: - max_input_len_text = 1536 - max_output_len = 200 - multimodal_len = 1225 - max_input_len = max_input_len_text + batch_size * multimodal_len - max_seq_len = max_input_len + max_output_len - - max_lora_rank = 256 - lora_dir = "." - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}", - f"--output_dir={llm_engine_dir}", - f"--gpt_attention_plugin {data_type}", - f"--gemm_plugin={data_type}", - f"--lora_plugin={data_type}", - f"--lora_dir={lora_dir}", - f"--max_lora_rank={max_lora_rank}", - f"--max_batch_size={batch_size}", - f"--max_multimodal_len={batch_size * multimodal_len}", - f"--max_input_len={max_input_len}", - f"--max_seq_len={max_seq_len}", - f"--max_num_tokens={max_input_len}", - f"--max_beam_width={num_beams}", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - elif enc_dec_model: - components = ["decoder"] if nougat_model or pix2struct_model else [ - "encoder", "decoder" - ] - for component in components: - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}/{component}", - f"--output_dir={llm_engine_dir}/{data_type}/llm/{component}", - "--paged_kv_cache=enable", - "--moe_plugin=disable", - f"--max_batch_size={batch_size}", - "--max_seq_len=412", - f"--gemm_plugin={data_type}", - f"--bert_attention_plugin={data_type}", - f"--gpt_attention_plugin={data_type}", - "--remove_input_padding=enable", - f"--max_beam_width={num_beams}", - ] - - # for non-T5 models, FP16/BF16 - if model_type == "t5" or data_type == "float32": - build_cmd.append("--context_fmha=disable") - if "t5" in model_name: - if component == "encoder": - build_cmd.append(f"--max_multimodal_len={32 * batch_size}") - build_cmd.append("--max_input_len=412") - else: - build_cmd.append("--max_encoder_input_len=412") - build_cmd.append(f"--max_input_len=1") - else: # Nougat - assert nougat_model or pix2struct_model - if component == "encoder": - build_cmd.append(f"--max_multimodal_len={588 * batch_size}") - - # only decoder for nougat - if nougat_model: - build_cmd.append( - f"--max_encoder_input_len={588 * batch_size}") - else: - build_cmd.append( - f"--max_encoder_input_len={2048 * batch_size}") - - build_cmd.append(f"--max_input_len=1") - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - elif cogvlm_model: - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}", - f"--output_dir={llm_engine_dir}/llm", - f"--gemm_plugin={data_type}", - f"--gpt_attention_plugin={data_type}", - f"--remove_input_padding=enable", - f"--max_batch_size={batch_size}", - f"--max_input_len=2048", - f"--max_seq_len=2048", - f"--paged_kv_cache=enable", - f"--bert_attention_plugin=disable", - f"--moe_plugin=disable", - f"--max_multimodal_len=61440", - f"--max_beam_width={num_beams}", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - elif mllama_model: - # set max_encoder_input_len = 6404 for running both non-instruct model and instruct model - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}", - f"--output_dir={llm_engine_dir}/llm", - f"--gemm_plugin={data_type}", - f"--max_num_tokens=4096", - f"--max_seq_len=2048", - f"--max_batch_size={batch_size}", - f"--max_encoder_input_len=6404", - f"--max_beam_width={num_beams}", - ] - if kv_cache_dtype == 'fp8': - build_cmd.extend([ - "--use_fp8_context_fmha=enable", - "--use_paged_context_fmha=enable", - ]) - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Build visual engines...") - vision_model_type = model_name - if 'llava' in model_name: vision_model_type = 'llava' - if 'llava-v1.6' in model_name: vision_model_type = 'llava_next' - elif llava_onevision_model: vision_model_type = 'llava_onevision' - elif 'VILA' in model_name: vision_model_type = 'vila' - elif nougat_model: vision_model_type = 'nougat' - elif pix2struct_model: vision_model_type = 'pix2struct' - elif 'cogvlm' in model_name: vision_model_type = 'cogvlm' - elif 'fuyu' in model_name: vision_model_type = 'fuyu' - elif 'neva-22b' in model_name: vision_model_type = 'neva' - elif 'video-neva' in model_name: vision_model_type = 'video-neva' - elif phi3_model: vision_model_type = "phi-3-vision" - elif phi4_model: vision_model_type = "phi-4-multimodal" - elif 'blip2' in model_name: vision_model_type = 'blip2' - elif 'Llama-3.2' in model_name: vision_model_type = 'mllama' - elif "Qwen2-VL" in model_name: vision_model_type = 'qwen2_vl' - elif 'internlm' in model_name: vision_model_type = 'internlm-xcomposer2' - elif 'Mistral-Small' in model_name: vision_model_type = 'pixtral' - - vit_batch_size = batch_size - if vision_model_type == "llava_next": - vit_batch_size = vit_batch_size * 5 - elif vision_model_type == 'llava_onevision': - vit_batch_size = vit_batch_size * 32 - - llm_engine_subdir = f"{data_type}" if enc_dec_model else "" - # Phi4MM has both vision and audio. Engine build dumps to vision and audio dirs automatically by builder. - component_dir = "vision" if vision_model_type != "phi-4-multimodal" else "" - build_cmd = [ - f"{multimodal_example_root}/build_multimodal_engine.py", - f"--output_dir={os.path.join(llm_engine_dir, llm_engine_subdir, component_dir)}", - f"--model_type={vision_model_type}", - f"--model_path={model_ckpt_path}", - f"--max_batch_size={vit_batch_size}", - ] - if vision_model_type == "vila": - vila_path = model_ckpt_path + "/../VILA" - build_cmd.extend([f"--vila_path={vila_path}"]) - if llava_next_vision_trtllm_engine_model: - script_root = f"{multimodal_example_root}/../vit" - convert_cmd = [ - f"{script_root}/convert_checkpoint.py", - f"--model_dir={model_ckpt_path}", - f"--output_dir={os.path.join(cmodel_dir, model_name, data_type, component_dir)}", - f"--dtype={data_type}", - f"--vision_tp_size={tp_size}", - ] - venv_check_call(llm_venv, convert_cmd) - - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={os.path.join(cmodel_dir, model_name, data_type, component_dir)}", - f"--output_dir={os.path.join(llm_engine_dir, llm_engine_subdir, component_dir)}", - f"--max_batch_size={vit_batch_size}", - f"--remove_input_padding disable", - f"--bert_attention_plugin disable", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - else: - venv_check_call(llm_venv, build_cmd) - - if llava_next_vision_trtllm_engine_model: - cp_cmd = [ - "cp", - f"{os.path.join(cmodel_dir, model_name, data_type, 'vision', 'image_newlines.safetensors')}", - f"{os.path.join(llm_engine_dir, llm_engine_subdir, 'vision')}", - ] - check_call(" ".join(cp_cmd), shell=True, env=llm_venv._new_env) - - print("Run inference...") - hf_model_dir = model_ckpt_path + "/../vicuna-7b-v1.5" if cogvlm_model else model_ckpt_path - hf_model_dir = converted_weight_dir if "neva" in model_name else hf_model_dir - video_path = os.path.join( - os.path.dirname(model_ckpt_path), "test_video", - "video_test.mp4") if "video-neva" in model_name else "" - run_cmd = [ - f"{multimodal_example_root}/run.py", - f"--engine_dir={llm_engine_dir}/{llm_engine_subdir}", - f"--hf_model_dir={hf_model_dir}", "--max_new_tokens=30", - f"--batch_size={batch_size}", "--check_accuracy", - "--enable_context_fmha_fp32_acc" - ] - if vision_model_type == 'phi-4-multimodal': - audio_path = f"{model_ckpt_path}/examples/what_is_shown_in_this_image.wav" - run_cmd.extend(["--audio_path", f"{audio_path}"]) - if vision_model_type in ['llava', 'vila'] and batch_size > 1: - # batch inference test - if vision_model_type == 'vila': - input_text = [ - '"\n Please elaborate what you see in the images?"' - ] * batch_size - else: - input_text = ['"\\n Which city is this? Answer:"'] * batch_size - run_cmd.append("--input_text") - run_cmd.extend(input_text) - if enc_dec_model: - run_cmd.extend(["--cross_kv_cache_fraction", "0.5"]) - if vision_model_type == "neva" and not cpp_e2e: - # randomly pick one to test the python runtime - run_cmd.extend(["--session", "python"]) - if vision_model_type == "video-neva": - run_cmd.extend(["--video_path", video_path]) - if llava_onevision_video_model: - run_cmd.extend(["--video_path", 'llava-onevision-accuracy']) - if phi3_model or phi4_model: - run_cmd.extend(["--kv_cache_free_gpu_memory_fraction", "0.4"]) - if cpp_e2e: - run_cmd.extend(["--session", "cpp"]) - if num_beams > 1: - run_cmd.extend(["--num_beams", str(num_beams)]) - - if mllama_model: - if qformat is None: - run_cmd_vision = run_cmd.copy() - run_cmd_vision.extend([ - "--cross_kv_cache_fraction=0.5", # mllama uses cross attention - "--image_path", - "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg", - "--input_text", - "If I had to write a haiku for this one" - ]) - - print("Run mllama vision test in with example image ...") - _call_run_cmd(llm_venv, llm_root, run_cmd_vision, world_size) - - print("multimodal_example_root: ", multimodal_example_root) - print("llm_root: ", llm_root) - run_cmd_vision = run_cmd.copy() - run_cmd_vision.extend([ - "--cross_kv_cache_fraction=0.5", # mllama uses cross attention - "--image_path", - os.path.join( - llm_root, - "tests/integration/test_input_files/excel_table_test.jpg"), - "--input_text", - "What is the total income? Answer:" - ]) - - print("Run mllama vision test with random image ...") - - run_cmd_text = run_cmd.copy() - run_cmd_text.extend([ - "--cross_kv_cache_fraction=0.5", # mllama uses cross attention - "--input_text", - "The key to life is", - ]) - print("Run mllama text test...") - _call_run_cmd(llm_venv, llm_root, run_cmd_text, world_size) - else: - _call_run_cmd(llm_venv, llm_root, run_cmd, world_size) - - # Run evaluation test - if batch_size == 1 and (data_type == "float16" or qformat == 'fp8'): - print(f"prepare to run eval test") - - # for blip2-t5, ref: https://github.com/huggingface/transformers/issues/25491 - if "t5" in model_name: - check_call("pip uninstall -y apex", - shell=True, - env=llm_venv._new_env) - - # Threshold are set based on the HF correctness for 20 iterations - threshold_map = { - 'blip2-opt-2.7b': 35, - 'blip2-flan-t5-xl': 55, - 'llava-1.5-7b-hf': 65, - 'llava-v1.6-mistral-7b-hf': 65, - 'llava-onevision-qwen2-7b-ov-hf': 80, - 'VILA1.5-3b': 75, # from local TRT-LLM run - 'fuyu-8b': 70, - 'kosmos-2': 60, - 'Phi-3-vision-128k-instruct': 75, - 'Phi-3.5-vision-instruct': 85, - 'Llama-3.2-11B-Vision': 60, # The expected score is 62 - 'Llama-3.2-11B-Vision-Instruct': 75, # The expected score is 77 - 'Qwen2-VL-7B-Instruct': 80, - } - - if model_name not in threshold_map: - print(f"Skip {model_name} evaluation test.") - return - - # TODO: Delete these lines after resolving the issues - # For llava - input tokens are not parsed correctly with '\n' - # For llava_next - correctness lower than HF, and needs lower transformer version built - # For Phi-3 - correctness lower than HF - # For qwen_vl - runtime issue with eval.py -- need to unify prompt generation logics - # For internvl - not added to the test - if llava_model or llava_next_model or phi3_model or qwen2_vl_model: - return - - eval_task = "lmms-lab/ai2d" if mllama_model else "lmms-lab/VQAv2" - - eval_cmd = [ - f"{multimodal_example_root}/eval.py", - f"--model_type={vision_model_type}", - f"--engine_dir={llm_engine_dir}/{llm_engine_subdir}", - f"--hf_model_dir={hf_model_dir}", "--enable_context_fmha_fp32_acc", - "--test_trtllm", - f"--accuracy_threshold={threshold_map[model_name]}", - f"--eval_task={eval_task}" - ] - - if mllama_model: - eval_cmd.extend([ - f"--dataset_dir={llm_datasets_root}/lmms-lab___ai2d/", - "--cross_kv_cache_fraction=0.5", "--max_ite=100" - ]) - else: - eval_cmd.extend([ - f"--dataset_dir={llm_datasets_root}/lmms-lab__VQAv2_valid_2000samples/" - ]) - - if phi3_model: - eval_cmd.extend(["--kv_cache_free_gpu_memory_fraction", "0.4"]) - elif enc_dec_model: - eval_cmd.extend(["--cross_kv_cache_fraction", "0.5"]) - - print(f"Run {model_name} evaluation test...") - _call_run_cmd(llm_venv, llm_root, eval_cmd, world_size) - - -@pytest.mark.parametrize("num_beams", [1, 4], - ids=lambda num_beams: f'nb:{num_beams}') -@pytest.mark.parametrize('cpp_e2e', [False, True], - ids=lambda cpp_e2e: f'cpp_e2e:{cpp_e2e}') -@pytest.mark.parametrize("batch_size", [1, 8], - ids=lambda batch_size: f'bs:{batch_size}') -@pytest.mark.parametrize( - "data_type", - ['float16', 'bfloat16', 'int4_weight_only', 'int8_weight_only']) -@pytest.mark.parametrize("tp_size", [1, 2], ids=lambda tp_size: f'tp:{tp_size}') -@pytest.mark.parametrize("pp_size", [1, 2], ids=lambda pp_size: f'pp:{pp_size}') -@pytest.mark.parametrize("multimodal_model_root", [ - 'blip2-opt-2.7b', - 'blip2-flan-t5-xl', - 'llava-1.5-7b-hf', - 'llava-v1.6-mistral-7b-hf', - pytest.param('llava-v1.6-mistral-7b-hf-vision-trtllm', - marks=pytest.mark.skipif(get_device_memory() < 50000, - reason="Skip due to low memory")), - 'llava-onevision-qwen2-7b-ov-hf', - 'llava-onevision-qwen2-7b-ov-hf-video', - pytest.param('nougat-base', marks=skip_post_blackwell), - 'VILA1.5-3b', - 'cogvlm-chat', - 'fuyu-8b', - pytest.param('deplot', marks=skip_post_blackwell), - pytest.param('neva-22b', - marks=pytest.mark.skip(reason="RCCA https://nvbugs/5220761")), - 'kosmos-2', - pytest.param('video-neva', marks=skip_post_blackwell), - pytest.param('Phi-3-vision-128k-instruct', marks=skip_post_blackwell), - pytest.param('Phi-3.5-vision-instruct', marks=skip_post_blackwell), - pytest.param('Phi-4-multimodal-instruct', marks=skip_post_blackwell), - pytest.param('Llama-3.2-11B-Vision', marks=skip_post_blackwell), - 'Qwen2-VL-7B-Instruct', - 'internlm-xcomposer2-vl-7b', - 'Mistral-Small-3.1-24B-Instruct-2503', -], - indirect=True) -def test_llm_multimodal_general(llm_venv, llm_root, llm_datasets_root, - cmodel_dir, engine_dir, batch_size, data_type, - tp_size, pp_size, multimodal_example_root, - multimodal_model_root, recover_transformers, - cpp_e2e, num_beams): - _test_llm_multimodal_general(llm_venv, - llm_root, - llm_datasets_root, - cmodel_dir, - engine_dir, - batch_size, - data_type, - tp_size, - pp_size, - multimodal_example_root, - multimodal_model_root, - recover_transformers, - cpp_e2e=cpp_e2e, - num_beams=num_beams) - - -@skip_pre_ada -@pytest.mark.parametrize('cpp_e2e', [False, True], - ids=lambda cpp_e2e: f'cpp_e2e:{cpp_e2e}') -@pytest.mark.parametrize("batch_size", [1, 8], - ids=lambda batch_size: f'bs:{batch_size}') -@pytest.mark.parametrize("data_type", ['float16', 'bfloat16']) -@pytest.mark.parametrize("tp_size", [1, 2], ids=lambda tp_size: f'tp:{tp_size}') -@pytest.mark.parametrize("pp_size", [1, 2], ids=lambda pp_size: f'pp:{pp_size}') -@pytest.mark.parametrize("multimodal_model_root", [ - 'blip2-opt-2.7b', - 'blip2-flan-t5-xl', - 'llava-1.5-7b-hf', - 'llava-v1.6-mistral-7b-hf', - 'llava-onevision-qwen2-7b-ov-hf', - 'llava-onevision-qwen2-7b-ov-hf-video', - 'nougat-base', - 'VILA1.5-3b', - 'cogvlm-chat', - 'fuyu-8b', - 'deplot', - 'neva-22b', - 'kosmos-2', - 'video-neva', - 'Phi-3-vision-128k-instruct', - 'Phi-3.5-vision-instruct', - 'Phi-4-multimodal-instruct', - pytest.param('Llama-3.2-11B-Vision-Instruct', marks=skip_post_blackwell), - pytest.param('Llama-3.2-11B-Vision', marks=skip_post_blackwell), - 'Qwen2-VL-7B-Instruct', -], - indirect=True) -@pytest.mark.parametrize('calibration_dataset', ['scienceqa', 'cnn_dailymail']) -@pytest.mark.parametrize('qformat', ['fp8']) -@pytest.mark.parametrize('kv_cache_dtype', ['fp8']) -def test_llm_fp8_multimodal_general( - llm_venv, llm_root, llm_datasets_root, cmodel_dir, engine_dir, - batch_size, data_type, tp_size, pp_size, multimodal_example_root, - multimodal_model_root, recover_transformers, calibration_dataset, - qformat, kv_cache_dtype, cpp_e2e): - _test_llm_multimodal_general(llm_venv, - llm_root, - llm_datasets_root, - cmodel_dir, - engine_dir, - batch_size, - data_type, - tp_size, - pp_size, - multimodal_example_root, - multimodal_model_root, - recover_transformers, - calibration_dataset, - qformat, - kv_cache_dtype, - cpp_e2e=cpp_e2e) diff --git a/tests/integration/defs/examples/test_nemotron.py b/tests/integration/defs/examples/test_nemotron.py deleted file mode 100644 index 0f7a69134bd4..000000000000 --- a/tests/integration/defs/examples/test_nemotron.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from defs.common import venv_check_call -from defs.conftest import get_sm_version -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@pytest.mark.skip_less_device_memory(50000) -@pytest.mark.parametrize("qformat", ["full_prec", "fp8", "int4_awq"]) -@pytest.mark.parametrize("dtype", ["float16", "bfloat16"]) -def test_llm_nemotron_3_8b_1gpu(nemotron_example_root, - llm_nemotron_3_8b_model_root, llm_datasets_root, - llm_rouge_root, llm_venv, cmodel_dir, - engine_dir, dtype, qformat): - print("Converting checkpoint...") - model_name = 'nemotron-3-8b' - ckpt_dir = f"{cmodel_dir}/{model_name}/{qformat}/1-gpu" - - quantize_cmd = [ - f"{nemotron_example_root}/../quantization/quantize.py", - f"--nemo_ckpt_path={llm_nemotron_3_8b_model_root}", - f"--calib_dataset={llm_datasets_root}/cnn_dailymail", - "--batch_size=64", - f"--dtype={dtype}", - f"--qformat={qformat}", - f"--output_dir={ckpt_dir}", - ] - venv_check_call(llm_venv, quantize_cmd) - - print("Building engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - "--max_batch_size=8", - "--max_input_len=924", - "--max_seq_len=1024", - f"--gpt_attention_plugin={dtype}", - f"--gemm_plugin={dtype}", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run engines...") - summary_cmd = [ - f"{nemotron_example_root}/../summarize.py", "--test_trt_llm", - f"--engine_dir={engine_dir}", - f"--vocab_file={ckpt_dir}/tokenizer.model", "--no_add_special_tokens", - "--batch_size=8", "--max_ite=40", "--check_accuracy", - "--tensorrt_llm_rouge1_threshold=18", - f"--dataset_dir={llm_datasets_root}", f"--rouge_dir={llm_rouge_root}" - ] - venv_check_call(llm_venv, summary_cmd) diff --git a/tests/integration/defs/examples/test_nemotron_nas.py b/tests/integration/defs/examples/test_nemotron_nas.py deleted file mode 100644 index d1663eab672e..000000000000 --- a/tests/integration/defs/examples/test_nemotron_nas.py +++ /dev/null @@ -1,124 +0,0 @@ -from pathlib import Path - -import pytest -from defs.common import convert_weights, venv_check_call, venv_mpi_check_call -from defs.conftest import get_device_memory, get_sm_version -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - -ROUGE1_ACCURACY_THRESHOLD = 20 - - -@pytest.mark.parametrize("nemotron_nas_model_root", [ - "DeciLM-7B", -], - indirect=True) -def test_nemotron_nas_summary_1gpu(nemotron_nas_example_root, llm_venv, - nemotron_nas_model_root, llm_datasets_root, - llm_rouge_root, engine_dir, cmodel_dir): - model_name = Path(nemotron_nas_model_root).name - if "51B" in model_name and get_device_memory() < 80000: - pytest.skip("device memory is insufficient.") - - print(f"Model name: {model_name}") - dtype = 'float16' - ckpt_type = "hf" - - print("Converting checkpoint...") - model_dir = convert_weights(llm_venv=llm_venv, - example_root=nemotron_nas_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=nemotron_nas_model_root, - data_type=dtype, - ckpt_type=ckpt_type, - gpus=1, - tp_size=1, - trust_remote_code=True) - - print("Building engines...") - build_cmd = [ - "trtllm-build", f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", f"--max_batch_size={4}", - f"--max_input_len={2048}", "--kv_cache_type=paged", - "--remove_input_padding=enable", "--gemm_plugin=auto", - "--gpt_attention_plugin=auto" - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Running inference...") - - summary_cmd = [ - f"{nemotron_nas_example_root}/../../../summarize.py", - f"--engine_dir={engine_dir}", "--test_hf", "--hf_device_map_auto", - "--batch_size=1", "--test_trt_llm", - f"--hf_model_dir={nemotron_nas_model_root}", "--check_accuracy", - f"--tensorrt_llm_rouge1_threshold={ROUGE1_ACCURACY_THRESHOLD}", - "--no_add_special_tokens", f"--dataset_dir={llm_datasets_root}", - f"--rouge_dir={llm_rouge_root}" - ] - - venv_check_call(llm_venv, summary_cmd) - - -@pytest.mark.skip_less_device(2) -@pytest.mark.parametrize("nemotron_nas_model_root", [ - "DeciLM-7B", - "Llama-3_1-Nemotron-51B-Instruct", -], - indirect=True) -def test_nemotron_nas_summary_2gpu(nemotron_nas_example_root, llm_venv, - nemotron_nas_model_root, llm_datasets_root, - llm_rouge_root, engine_dir, cmodel_dir): - model_name = Path(nemotron_nas_model_root).name - if "51B" in model_name and get_device_memory() < 80000: - pytest.skip("device memory is insufficient.") - - print(f"Model name: {model_name}") - dtype = 'float16' - ckpt_type = "hf" - - print("Converting checkpoint...") - model_dir = convert_weights(llm_venv=llm_venv, - example_root=nemotron_nas_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=nemotron_nas_model_root, - data_type=dtype, - ckpt_type=ckpt_type, - gpus=2, - tp_size=2, - trust_remote_code=True) - - print("Building engines...") - build_cmd = [ - "trtllm-build", f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", f"--max_batch_size={4}", - f"--max_input_len={2048}", "--kv_cache_type=paged", - "--remove_input_padding=enable", "--gemm_plugin=auto", - "--gpt_attention_plugin=auto" - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Running inference...") - - mpi_cmd = ["mpirun", "-n", "2", "--allow-run-as-root"] - - summary_cmd = [ - f"{nemotron_nas_example_root}/../../../summarize.py", - f"--engine_dir={engine_dir}", "--test_hf", "--hf_device_map_auto", - "--batch_size=1", "--test_trt_llm", - f"--hf_model_dir={nemotron_nas_model_root}", "--check_accuracy", - f"--tensorrt_llm_rouge1_threshold={ROUGE1_ACCURACY_THRESHOLD}", - "--no_add_special_tokens", f"--dataset_dir={llm_datasets_root}", - f"--rouge_dir={llm_rouge_root}" - ] - - venv_mpi_check_call(llm_venv, mpi_cmd, summary_cmd) diff --git a/tests/integration/defs/examples/test_ngram.py b/tests/integration/defs/examples/test_ngram.py deleted file mode 100644 index 2de49e8322f0..000000000000 --- a/tests/integration/defs/examples/test_ngram.py +++ /dev/null @@ -1,163 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import csv -from copy import deepcopy - -import pytest -from defs.common import convert_weights, venv_check_call -from defs.conftest import get_sm_version, skip_post_blackwell -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -# TODO: remove skip after support NGram on B200 -@skip_post_blackwell -@pytest.mark.parametrize("batch_size", [1, 2], ids=['bs1', 'bs2']) -@pytest.mark.parametrize("data_type", ['float16']) -@pytest.mark.parametrize("max_draft_len", [4, 8], - ids=['max_draft_len_4', 'max_draft_len_8']) -@pytest.mark.parametrize( - "max_matching_ngram_size", [2, 4], - ids=['max_matching_ngram_size_2', 'max_matching_ngram_size_4']) -@pytest.mark.parametrize("use_logits", [False, True], - ids=['use_tokens', 'use_logits']) # useless yet -@pytest.mark.parametrize("use_py_session", [False], ids=["use_cpp_session"]) -@pytest.mark.parametrize("ngram_root", ["gpt2"], indirect=True) -@pytest.mark.parametrize("streaming", [False, True], - ids=["no_streaming", "streaming"]) -def test_llm_ngram_1gpu(batch_size, data_type, max_draft_len, - max_matching_ngram_size, use_logits, use_py_session, - ngram_root, streaming, ngram_example_root, - llm_datasets_root, llm_rouge_root, llm_venv, cmodel_dir, - engine_dir): - model_name = "ngram" - - print("Build checkpoint ...") - model_dir = convert_weights(llm_venv=llm_venv, - example_root=ngram_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=ngram_root, - data_type=data_type) - - print("Build engines ...") - target_engine_dir = engine_dir + "-target" - baseline_engine_dir = engine_dir + "-baseline" - common_build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--max_batch_size={batch_size}", - f"--max_beam_width=1", - "--max_input_len=1024", - "--max_seq_len=1536", - "--use_paged_context_fmha=enable", - f"--gpt_attention_plugin={data_type}", - f"--gemm_plugin={data_type}", - ] - target_model_build_cmd = deepcopy(common_build_cmd) - target_model_build_cmd.extend([ - f"--output_dir={target_engine_dir}", - "--speculative_decoding_mode=draft_tokens_external", - f"--max_draft_len={max_draft_len+1}", - ]) - baseline_model_build_cmd = deepcopy(common_build_cmd) - baseline_model_build_cmd.extend([ - f"--output_dir={baseline_engine_dir}", - ]) - - check_call(" ".join(target_model_build_cmd), - shell=True, - env=llm_venv._new_env) - check_call(" ".join(baseline_model_build_cmd), - shell=True, - env=llm_venv._new_env) - - print("Run inferences ...") - common_run_cmd = [ - f"{ngram_example_root}/../run.py", - f"--tokenizer_dir={ngram_root}", - f"--max_output_len=64", - f"--kv_cache_enable_block_reuse", - f"--kv_cache_free_gpu_memory_fraction=0.25", - ] - if streaming: - common_run_cmd.extend(["--streaming", "--streaming_interval=1"]) - if batch_size == 1: - common_run_cmd.extend(["--input_text", "'How are you?'"]) - elif batch_size == 2: - common_run_cmd.extend(["--input_text", "'Hello'", "'How are you?'"]) - else: - assert False, "Only batch_size <=2 is supported in test." - assert not use_py_session, "Only CPP session is supported in Draft-Target-Model." - - run_cmd = deepcopy(common_run_cmd) - ngram_config = f"[{max_draft_len},{max_matching_ngram_size},[0]]" - run_cmd.extend([ - f"--engine_dir={target_engine_dir}", - f"--ngram_config={ngram_config}", - f"--output_csv={engine_dir}/ngram_output.csv", - ]) - baseline_run_cmd = deepcopy(common_run_cmd) - baseline_run_cmd.extend([ - f"--engine_dir={baseline_engine_dir}", - f"--output_csv={engine_dir}/baseline_output.csv", - ]) - - venv_check_call(llm_venv, run_cmd) - venv_check_call(llm_venv, baseline_run_cmd) - - print("Compare outputs ...") - with open(f"{engine_dir}/ngram_output.csv") as dt_f, open( - f"{engine_dir}/baseline_output.csv") as b_f: - for bs, (dt_request, - b_request) in enumerate(zip(csv.reader(dt_f), - csv.reader(b_f))): - assert ( - len(dt_request) == len(b_request) - ), f"Output length at ({bs=}) is different ({len(dt_request)} v.s. {len(b_request)})." - for index, (dt, b) in enumerate(zip(dt_request, b_request)): - assert ( - int(dt) == int(b) - ), f"Output at ({bs=}, {index=}) is different ({dt} v.s. {b})." - - if batch_size > 1 or streaming: # Summarize tests for only batch_size=1 and streaming=False. - return - - print("Run summarize...") - ngram_config = f"[{max_draft_len},{max_matching_ngram_size},[0]]" - - run_cmd = [ - f"{ngram_example_root}/../summarize.py", - "--test_hf", - "--test_trt_llm", - "--check_accuracy", - "--batch_size=1", - f"--hf_model_dir={ngram_root}", - f"--engine_dir={target_engine_dir}", - f"--dataset_dir={llm_datasets_root}", - f"--rouge_dir={llm_rouge_root}", - "--kv_cache_enable_block_reuse", - f"--ngram_config={ngram_config}", - "--tensorrt_llm_rouge1_threshold=20", - f"--kv_cache_free_gpu_memory_fraction=0.25", - ] - - venv_check_call(llm_venv, run_cmd) diff --git a/tests/integration/defs/examples/test_openai.py b/tests/integration/defs/examples/test_openai.py deleted file mode 100644 index 4f4776926ac2..000000000000 --- a/tests/integration/defs/examples/test_openai.py +++ /dev/null @@ -1,196 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Module test_openai test openai examples.""" -import os -import subprocess # fmt: off - -import pytest -from defs.common import find_tensorrt, venv_check_call -from defs.trt_test_alternative import call, check_call, make_clean_dirs - - -@pytest.fixture(scope="module") -def openai_triton_example_root(llm_root): - "Get openai-triton example root" - example_root = os.path.join(llm_root, "examples", "openai_triton", - "manual_plugin") - - return example_root - - -@pytest.fixture(scope="module") -def openai_triton_plugingen_example_root(llm_root): - "Get openai-triton PluginGen example root" - example_root = os.path.join(llm_root, "examples", "openai_triton", - "plugin_autogen") - - return example_root - - -@pytest.fixture(scope="module") -def llm_openai_triton_model_root(llm_venv): - "prepare openai-triton model & return model root" - workspace = llm_venv.get_working_directory() - model_root = os.path.join(workspace, "triton") - commit = "d4644d6cb3ae674e1f15932cac1f28104795744f" - - call(f"git clone https://github.com/openai/triton.git {model_root}", - shell=True) - call(f"cd {model_root} && git checkout {commit}", shell=True) - llm_venv.run_cmd(["-m", "pip", "install", "cmake"]) - llm_venv.run_cmd([ - "-m", "pip", "install", - os.path.abspath(os.path.join(model_root, "python")) - ]) - - yield model_root - - llm_venv.run_cmd(["-m", "pip", "uninstall", "-y", "triton"]) - - -def test_llm_openai_triton_1gpu(openai_triton_example_root, - llm_openai_triton_model_root, llm_venv, - engine_dir, trt_config, is_trt_environment): - aot_path = os.path.join(openai_triton_example_root, "aot") - aot_fp16_path = os.path.join(aot_path, "fp16") - aot_fp32_path = os.path.join(aot_path, "fp32") - call(f"mkdir -p {aot_fp16_path}", shell=True) - call(f"mkdir -p {aot_fp32_path}", shell=True) - - num_stages = "2" - - # yapf: disable - # Kernel for data type=float16, BLOCK_M=128, BLOCK_DMODEL=64, BLOCK_N=128 - compile_cmd = [ - f"{llm_openai_triton_model_root}/python/triton/tools/compile.py", - f"{openai_triton_example_root}/fmha_triton.py", - "-n", "fused_attention_kernel", - "-o", f"{aot_fp16_path}/fmha_kernel_d64_fp16", - "--out-name", "fmha_d64_fp16", "-w", "4", "-ns", num_stages, - "-s", "*fp16:16, *fp32:16, *fp32:16, *fp16:16, *fp16:16, *fp16:16, fp32, i32, i32, i32, 128, 64, 128", - "-g", "(seq_len + 127) / 128, batch_size * num_heads, 1" - ] - venv_check_call(llm_venv, compile_cmd) - - # Kernel for data type=float32, BLOCK_M=64, BLOCK_DMODEL=64, BLOCK_N=64 - compile_cmd = [ - f"{llm_openai_triton_model_root}/python/triton/tools/compile.py", - f"{openai_triton_example_root}/fmha_triton.py", - "-n", "fused_attention_kernel", - "-o", f"{aot_fp32_path}/fmha_kernel_d64_fp32", - "--out-name", "fmha_d64_fp32", "-w", "4", "-ns", num_stages, - "-s", "*fp32:16, *fp32:16, *fp32:16, *fp32:16, *fp32:16, *fp32:16, fp32, i32, i32, i32, 64, 64, 64", - "-g", "(seq_len + 63) / 64, batch_size * num_heads, 1" - ] - venv_check_call(llm_venv, compile_cmd) - - # Link generated headers and create dispatchers. - check_call( - f"python3 {llm_openai_triton_model_root}/python/triton/tools/link.py " - f"{aot_fp16_path}/*.h -o {aot_path}/fmha_kernel_fp16", - shell=True) - check_call( - f"python3 {llm_openai_triton_model_root}/python/triton/tools/link.py " - f"{aot_fp32_path}/*.h -o {aot_path}/fmha_kernel_fp32", - shell=True) - - build_path = os.path.join(openai_triton_example_root, "build") - # yapf: enable - - # make files - make_clean_dirs(build_path) - cmake_args = [] - try: - import trt_test # noqa - except ImportError: - pass - else: - trt_include_dir, trt_lib_dir = find_tensorrt( - trt_config["new_ld_library_path"]) - - if trt_include_dir: - cmake_args.append(f"-DTRT_INCLUDE_DIR={trt_include_dir}") - - if trt_lib_dir: - cmake_args.append(f"-DTRT_LIB_DIR={trt_lib_dir}") - - if is_trt_environment: - cmake_args.append(f"-DCMAKE_C_FLAGS='-D_GLIBCXX_USE_CXX11_ABI=0'") - - cmake_args = " ".join(cmake_args) - check_call(f"cd {build_path} && cmake {cmake_args} .. && make", shell=True) - - # build engine - build_cmd = [ - f"{openai_triton_example_root}/build.py", "--num_heads=32", - "--head_size=64", "--max_batch_size=8", "--max_seq_len=512", - "--dtype=float16", f"--output={engine_dir}" - ] - venv_check_call(llm_venv, build_cmd) - - # run inference - run_cmd = [ - f"{openai_triton_example_root}/run.py", "--num_heads=32", - "--head_size=64", "--batch_size=8", "--seq_len=512", - "--log_level=verbose", "--benchmark", f"--engine_dir={engine_dir}" - ] - venv_check_call(llm_venv, run_cmd) - - -# TODO[chunweiy]: Enable it later -def test_llm_openai_triton_plugingen_1gpu(openai_triton_plugingen_example_root, - openai_triton_example_root, - llm_openai_triton_model_root, - plugin_gen_path, llm_venv, - trt_config): - # copy the triton kernel definition - subprocess.run( - f"cp {openai_triton_example_root}/fmha_triton.py {openai_triton_plugingen_example_root}/fmha_triton.py" - .split(), - check=True) - - # generate plugin - cmd = [ - plugin_gen_path, - "--workspace", - "./tmp", - "--kernel_config", - os.path.join(openai_triton_plugingen_example_root, "kernel_config.py"), - ] - try: - import trt_test # noqa - except ImportError: - pass - else: - trt_include_dir, trt_lib_dir = find_tensorrt( - trt_config["new_ld_library_path"]) - if trt_lib_dir is not None: - cmd.append(f'--trt_lib_dir={trt_lib_dir}') - if trt_include_dir is not None: - cmd.append(f'--trt_include_dir={trt_include_dir}') - - venv_check_call(llm_venv, cmd) - - # build engine - cmd = [ - os.path.join(openai_triton_plugingen_example_root, "build_engine.py"), - ] - venv_check_call(llm_venv, cmd) - - # run engine - cmd = [ - os.path.join(openai_triton_plugingen_example_root, "run_engine.py"), - ] - venv_check_call(llm_venv, cmd) diff --git a/tests/integration/defs/examples/test_phi.py b/tests/integration/defs/examples/test_phi.py index 62f67e4bae65..a8c18f731e94 100644 --- a/tests/integration/defs/examples/test_phi.py +++ b/tests/integration/defs/examples/test_phi.py @@ -16,9 +16,8 @@ import defs.ci_profiler import pytest -from defs.common import test_llm_torch_multi_lora_support, venv_check_call -from defs.conftest import get_sm_version, skip_post_blackwell, skip_pre_ada -from defs.trt_test_alternative import check_call +from defs.common import test_llm_torch_multi_lora_support +from defs.conftest import get_sm_version, skip_pre_ada # skip trt flow cases on post-Blackwell-Ultra if get_sm_version() >= 103: @@ -39,75 +38,6 @@ def phi_example_root(llm_root, llm_venv): return example_root -@skip_pre_ada -@pytest.mark.parametrize("data_type", ['float16', 'bfloat16']) -@pytest.mark.parametrize("qformat", ['fp8']) -@pytest.mark.parametrize("llm_phi_model_root", [ - pytest.param("phi-2", marks=skip_post_blackwell), - pytest.param("Phi-3-mini-128k-instruct", marks=skip_post_blackwell), - pytest.param("Phi-3-small-128k-instruct", marks=skip_post_blackwell), - pytest.param("Phi-3.5-mini-instruct", marks=skip_post_blackwell), - "Phi-3.5-MoE-instruct", "Phi-4-mini-instruct" -], - indirect=True) -def test_llm_phi_quantization_1gpu(data_type, llm_phi_model_root, llm_venv, - cmodel_dir, engine_dir, phi_example_root, - llm_datasets_root, llm_rouge_root, qformat): - "Run phi quantization tests" - # Workaround for Modelopt can't convert Phi-3 on multi GPUs. - gpu_constraint = {"CUDA_VISIBLE_DEVICES": "0"} - - print("Convert checkpoint by modelopt...") - convert_cmd = [ - f"{phi_example_root}/../../../quantization/quantize.py", - f"--model_dir={llm_phi_model_root}", - f"--calib_dataset={llm_datasets_root}/cnn_dailymail", - f"--dtype={data_type}", - f"--qformat={qformat}", - f"--kv_cache_dtype={qformat}", - f"--output_dir={cmodel_dir}", - ] - venv_check_call(llm_venv, convert_cmd, env=gpu_constraint) - - print("Build engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={cmodel_dir}", - f"--output_dir={engine_dir}", - "--max_input_len=3000", - "--max_seq_len=3100", - f"--max_batch_size={16}", - ] - - build_env = { - **llm_venv._new_env, - **gpu_constraint - } if llm_venv._new_env else gpu_constraint - check_call(" ".join(build_cmd), shell=True, env=build_env) - - print("Run summarize...") - threshold_score = 24.0 - model_name = os.path.basename(llm_phi_model_root) - if model_name == "phi-2": - threshold_score = 22.0 - - summary_cmd = [ - f"{phi_example_root}/../../../summarize.py", - "--test_trt_llm", - f"--hf_model_dir={llm_phi_model_root}", - f"--tokenizer_dir={llm_phi_model_root}", - f"--engine_dir={engine_dir}", - "--check_accuracy", - f"--tensorrt_llm_rouge1_threshold={threshold_score}", - "--max_ite=40", - f"--batch_size={16}", - f"--dataset_dir={llm_datasets_root}", - f"--rouge_dir={llm_rouge_root}", - ] - - venv_check_call(llm_venv, summary_cmd, env=gpu_constraint) - - @pytest.mark.skip( reason="TODO: Resolve an import issue with transformers's LossKwargs") @skip_pre_ada diff --git a/tests/integration/defs/examples/test_qwen.py b/tests/integration/defs/examples/test_qwen.py deleted file mode 100644 index 35352b20e279..000000000000 --- a/tests/integration/defs/examples/test_qwen.py +++ /dev/null @@ -1,242 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Module test_qwen test qwen examples.""" - -import csv -import os - -import pytest -from defs.common import (convert_weights, test_multi_lora_support, - venv_check_call) -from defs.conftest import get_sm_version, skip_pre_ada -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - -# Delete this case refer to https://nvbugs/5072417 -# @pytest.mark.parametrize("llm_lora_model_root", ["Ko-QWEN-7B-Chat-LoRA"], -# indirect=True) -# @pytest.mark.parametrize("llm_qwen_model_root", ["qwen_7b_chat"], indirect=True) -# def test_llm_qwen_7b_single_gpu_lora( -# qwen_example_root, -# llm_qwen_model_root, -# llm_venv, -# cmodel_dir, -# engine_dir, -# llm_lora_model_root, -# ): -# "run Qwen lora test on single gpu." -# print("Build engines...") -# dtype = 'float16' -# model_name = os.path.basename(llm_qwen_model_root) -# ckpt_dir = convert_weights(llm_venv=llm_venv, -# example_root=qwen_example_root, -# cmodel_dir=cmodel_dir, -# model=model_name, -# model_path=llm_qwen_model_root, -# data_type=dtype) - -# print("Build engines...") -# build_cmd = [ -# "trtllm-build", -# f"--checkpoint_dir={ckpt_dir}", -# f"--output_dir={engine_dir}", -# "--lora_plugin=auto", -# "--gemm_plugin=auto", -# f"--lora_dir={llm_lora_model_root}", -# ] -# check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - -# ref_1 = [ -# 151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, -# 151644, 872, 198, 126246, 144370, 91145, 11, 137601, 29326, 86034, -# 12802, 5140, 98734, 19391, 35711, 30, 151645, 198, 151644, 77091, 198, -# 126246, 144370, 91145, 0, 134561, 58677, 78125, 21329, 66019, 124685, -# 134619, 94152, 28626, 17380, 11, 134637, 20401, 138520, 19391, 143603 -# ] -# ref_2 = [ -# 151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, -# 151644, 872, 198, 126246, 144370, 91145, 11, 137601, 29326, 86034, -# 12802, 5140, 98734, 19391, 35711, 30, 151645, 198, 151644, 77091, 198, -# 126246, 144370, 91145, 0, 134561, 330, 48, 1103, 54, 268, 1, 78952, 13, -# 151645, 198, 151643, 151643, 151643, 151643, 151643 -# ] - -# input_text = "안녕하세요, 혹시 이름이 뭐에요?" -# print("Run inference with lora id 0...") -# venv_check_call(llm_venv, [ -# f"{qwen_example_root}/../run.py", -# "--max_output_len=20", -# f"--input_text={input_text}", -# "--lora_task_uids=0", -# f"--tokenizer_dir={llm_qwen_model_root}", -# f"--engine_dir={engine_dir}", -# f"--output_csv={llm_venv.get_working_directory()}/use_lora.csv", -# "--use_py_session", -# ]) - -# with open(f"{llm_venv.get_working_directory()}/use_lora.csv") as f: -# predict = csv.reader(f) -# predict = next(predict) -# predict = [int(p) for p in predict] -# assert ref_1 == predict - -# print("Run inference with lora id -1...") -# venv_check_call(llm_venv, [ -# f"{qwen_example_root}/../run.py", -# "--max_output_len=20", -# f"--input_text={input_text}", -# "--lora_task_uids=-1", -# f"--tokenizer_dir={llm_qwen_model_root}", -# f"--engine_dir={engine_dir}", -# f"--output_csv={llm_venv.get_working_directory()}/no_lora.csv", -# "--use_py_session", -# ]) - -# with open(f"{llm_venv.get_working_directory()}/no_lora.csv") as f: -# predict = csv.reader(f) -# predict = next(predict) -# predict = [int(p) for p in predict] -# assert ref_2 == predict - - -@pytest.mark.parametrize("llm_lora_model_root", ["Qwen1.5-7B-Chat-750Mb-lora"], - indirect=True) -@pytest.mark.parametrize("llm_qwen_model_root", ["qwen1.5_7b_chat"], - indirect=True) -def test_llm_qwen1_5_7b_single_gpu_lora( - qwen_example_root, - llm_qwen_model_root, - llm_venv, - cmodel_dir, - engine_dir, - llm_lora_model_root, -): - "run Qwen1.5 lora test on single gpu." - print("Build engines...") - dtype = 'float16' - model_name = os.path.basename(llm_qwen_model_root) - ckpt_dir = convert_weights(llm_venv=llm_venv, - example_root=qwen_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=llm_qwen_model_root, - data_type=dtype) - - print("Build engines...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={ckpt_dir}", - f"--output_dir={engine_dir}", - "--lora_plugin=auto", - "--gemm_plugin=auto", - f"--lora_dir={llm_lora_model_root}", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - ref_1 = [ - 151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, - 151644, 872, 198, 3838, 374, 697, 829, 30, 151645, 198, 151644, 77091, - 198, 40, 2776, 458, 15235, 7881, 553, 5264, 15469, 11, 773, 358, 1513, - 944, 614, 264, 829, 304, 279, 8606, 5530 - ] - ref_2 = [ - 151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, - 151644, 872, 198, 3838, 374, 697, 829, 30, 151645, 198, 151644, 77091, - 198, 40, 1079, 1207, 16948, 11, 264, 3460, 4128, 1614, 3465, 553, 54364, - 14817, 13, 151645, 151645, 151645, 151645, 151645, 151645 - ] - - input_text = "What is your name?" - print("Run inference with lora id 0...") - venv_check_call(llm_venv, [ - f"{qwen_example_root}/../../../run.py", - "--max_output_len=20", - f"--input_text={input_text}", - "--lora_task_uids=0", - f"--tokenizer_dir={llm_qwen_model_root}", - f"--engine_dir={engine_dir}", - f"--output_csv={llm_venv.get_working_directory()}/use_lora.csv", - "--use_py_session", - ]) - - with open(f"{llm_venv.get_working_directory()}/use_lora.csv") as f: - predict = csv.reader(f) - predict = next(predict) - predict = [int(p) for p in predict] - assert ref_1 == predict - - print("Run inference with lora id -1...") - venv_check_call(llm_venv, [ - f"{qwen_example_root}/../../../run.py", - "--max_output_len=20", - f"--input_text={input_text}", - "--lora_task_uids=-1", - f"--tokenizer_dir={llm_qwen_model_root}", - f"--engine_dir={engine_dir}", - f"--output_csv={llm_venv.get_working_directory()}/no_lora.csv", - "--use_py_session", - ]) - - with open(f"{llm_venv.get_working_directory()}/no_lora.csv") as f: - predict = csv.reader(f) - predict = next(predict) - predict = [int(p) for p in predict] - assert ref_2 == predict - - -@skip_pre_ada -@pytest.mark.parametrize( - "llm_qwen_model_root", - ["qwen2_0.5b_instruct", "qwen2.5_0.5b_instruct", "qwen2.5_1.5b_instruct"], - indirect=True) -def test_llm_hf_qwen_multi_lora_1gpu(llm_qwen_model_root, - llm_venv, - cmodel_dir, - engine_dir, - qwen_example_root, - llm_datasets_root, - qformat='fp8', - dtype='bfloat16'): - "Run Qwen models with multiple dummy LoRAs." - - print("Convert checkpoint by modelopt...") - convert_cmd = [ - f"{qwen_example_root}/../../../quantization/quantize.py", - f"--model_dir={llm_qwen_model_root}", - f"--calib_dataset={llm_datasets_root}/cnn_dailymail", - f"--dtype={dtype}", - f"--qformat={qformat}", - f"--kv_cache_dtype={qformat}", - f"--output_dir={cmodel_dir}", - ] - venv_check_call(llm_venv, convert_cmd) - - test_multi_lora_support( - hf_model_dir=llm_qwen_model_root, - tllm_ckpt_dir=cmodel_dir, - engine_dir=engine_dir, - llm_venv=llm_venv, - example_root=qwen_example_root, - num_loras=2, - lora_rank=8, - target_hf_modules=["q_proj", "k_proj", "v_proj"], - target_trtllm_modules=["attn_q", "attn_k", "attn_v"], - zero_lora_weights=True, - ) diff --git a/tests/integration/defs/examples/test_qwen2audio.py b/tests/integration/defs/examples/test_qwen2audio.py deleted file mode 100644 index 9e911cca97e8..000000000000 --- a/tests/integration/defs/examples/test_qwen2audio.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Module test_qwen test qwen2audio examples.""" - -import os -import re - -import pytest -from defs.common import venv_check_call, venv_check_output -from defs.trt_test_alternative import check_call - - -@pytest.fixture(scope="module") -def qwen2audio_example_root(llm_root, llm_venv): - "Get qwen2audio example root" - example_root = os.path.join(llm_root, "examples", "models", "core", - "qwen2audio") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root - - -@pytest.mark.parametrize("llm_qwen_model_root", ["qwen2_audio_7b_instruct"], - indirect=True) -def test_llm_qwen2audio_single_gpu(qwen2audio_example_root, llm_qwen_model_root, - llm_venv, engine_dir): - "Build & run qwen2audio on 1 gpu." - workspace = llm_venv.get_working_directory() - - print("Generate audio engine...") - audio_engine_dir = f"{engine_dir}/audio" - audio_cmd = [ - f"{qwen2audio_example_root}/../multimodal/build_multimodal_engine.py", - f"--model_type=qwen2_audio", - f"--model_path={llm_qwen_model_root}", - f"--max_batch_size=32", - f"--output_dir={audio_engine_dir}", - ] - - venv_check_call(llm_venv, audio_cmd) - - print("Convert checkpoint...") - convert_cmd = [ - f"{qwen2audio_example_root}/../qwen/convert_checkpoint.py", - f"--model_dir={llm_qwen_model_root}", - f"--output_dir={workspace}/Qwen2-Audio", - f"--dtype=float16", - ] - - venv_check_call(llm_venv, convert_cmd) - - print("Build TRT-LLM engine...") - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={workspace}/Qwen2-Audio", - f"--gemm_plugin=float16", - f"--gpt_attention_plugin=float16", - f"--max_prompt_embedding_table_size=4096", - f"--output_dir={engine_dir}", - f"--max_batch_size={1}", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run engine...") - audio_url = f"{qwen2audio_example_root}/audio/glass-breaking-151256.mp3" - - run_cmd = [ - f"{qwen2audio_example_root}/run.py", - f"--tokenizer_dir={llm_qwen_model_root}", - f"--engine_dir={engine_dir}", - f"--audio_engine_path={audio_engine_dir}/model.engine", - f"--audio_url={audio_url}", - ] - - output = venv_check_output(llm_venv, run_cmd) - output = [line for line in output.split("\n") if "Output:" in line] - print(output) - - print("Verify the output...") - results = [] - for item in output: - match = re.search(r"Output: \"(.*)", item) - if match: - results.append(match.group(1)) - - for item in results: - # check the output if it contains key words - item = item.lower() - if ("glass" in item) and ("shatter" in item or "break" in item): - pass - else: - assert False, f"output is: {item}" diff --git a/tests/integration/defs/examples/test_qwenvl.py b/tests/integration/defs/examples/test_qwenvl.py deleted file mode 100644 index d4ff5765aa33..000000000000 --- a/tests/integration/defs/examples/test_qwenvl.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Module test_qwen test qwenvl examples.""" - -import os - -import pytest -from defs.conftest import get_sm_version - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@pytest.fixture(scope="module") -def qwenvl_example_root(llm_root, llm_venv): - "Get qwenvl example root" - example_root = os.path.join(llm_root, "examples", "models", "core", - "qwenvl") - llm_venv.run_cmd([ - "-m", "pip", "install", "-r", - os.path.join(example_root, "requirements.txt") - ]) - - return example_root diff --git a/tests/integration/defs/examples/test_redrafter.py b/tests/integration/defs/examples/test_redrafter.py deleted file mode 100644 index ce9a62d097b2..000000000000 --- a/tests/integration/defs/examples/test_redrafter.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from defs.common import convert_weights, venv_check_call -from defs.conftest import get_sm_version, skip_post_blackwell -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@skip_post_blackwell -@pytest.mark.parametrize("batch_size", [8], ids=['bs8']) -@pytest.mark.parametrize("redrafter_num_beams", [5, 8], ids=['nb5', 'nb8']) -@pytest.mark.parametrize("redrafter_draft_len_per_beam", [5], ids=['dl5']) -@pytest.mark.parametrize("data_type", ['bfloat16']) -@pytest.mark.parametrize("redrafter_model_roots", ["redrafter-vicuna-7b-v1.3"], - indirect=True) -@pytest.mark.parametrize("use_py_session", [False, True], - ids=["use_cpp_session", "use_py_session"]) -def test_llm_redrafter_1gpu(batch_size, data_type, redrafter_model_roots, - redrafter_num_beams, redrafter_draft_len_per_beam, - redrafter_example_root, llama_example_root, - llm_datasets_root, llm_rouge_root, llm_venv, - cmodel_dir, cmodel_base_dir, engine_dir, - use_py_session): - print("Build engines...") - model_name = "redrafter" - base_model_name = "llama" - base_example_root = llama_example_root - - base_model_dir = convert_weights(llm_venv=llm_venv, - example_root=base_example_root, - cmodel_dir=cmodel_base_dir, - model=base_model_name, - model_path=redrafter_model_roots[0], - data_type=data_type) - - redrafter_convert_roots = (base_model_dir, redrafter_model_roots[1]) - - model_dir = convert_weights( - llm_venv=llm_venv, - example_root=redrafter_example_root, - cmodel_dir=cmodel_dir, - model=model_name, - model_path=redrafter_convert_roots, - data_type=data_type, - redrafter_num_beams=redrafter_num_beams, - redrafter_draft_len_per_beam=redrafter_draft_len_per_beam) - - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={model_dir}", - f"--output_dir={engine_dir}", - f"--gpt_attention_plugin={data_type}", - f"--gemm_plugin={data_type}", - f"--max_beam_width=1", - "--remove_input_padding=enable", - "--context_fmha=enable", - "--max_input_len=1024", - "--max_seq_len=1536", - f"--max_batch_size={batch_size}", - "--kv_cache_type=paged", - '--speculative_decoding_mode=explicit_draft_tokens', - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - print("Run summarize...") - - summary_cmd = [ - f"{redrafter_example_root}/../summarize.py", "--test_trt_llm", - "--hf_model_dir", f"{redrafter_model_roots[0]}", "--tokenizer_dir", - f"{redrafter_model_roots[0]}", f"--engine_dir={engine_dir}", - "--check_accuracy", "--tensorrt_llm_rouge1_threshold=24", - f"--temperature=1.0", f"--max_ite=40", f"--batch_size={batch_size}", - f"--dataset_dir={llm_datasets_root}", f"--rouge_dir={llm_rouge_root}" - ] - - if use_py_session: - summary_cmd.append("--use_py_session") - - venv_check_call(llm_venv, summary_cmd) diff --git a/tests/integration/defs/examples/test_whisper.py b/tests/integration/defs/examples/test_whisper.py deleted file mode 100644 index 5b7b6859ffaf..000000000000 --- a/tests/integration/defs/examples/test_whisper.py +++ /dev/null @@ -1,298 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import pytest -from defs.common import convert_weights, venv_check_call -from defs.conftest import get_sm_version, llm_models_root, skip_post_blackwell -from defs.trt_test_alternative import check_call - -# skip trt flow cases on post-Blackwell-Ultra -if get_sm_version() >= 103: - pytest.skip( - "TRT workflow tests are not supported on post Blackwell-Ultra architecture", - allow_module_level=True) - - -@skip_post_blackwell -@pytest.mark.parametrize("use_cpp_runtime", [True, False], - ids=["use_cpp_runtime", "use_python_runtime"]) -@pytest.mark.parametrize("num_beams", [1, 4], - ids=lambda num_beams: f'nb:{num_beams}') -@pytest.mark.parametrize("data_type", ['float16']) -@pytest.mark.parametrize("weight_only_precision", [ - 'disable_weight_only', - pytest.param('int8', marks=skip_post_blackwell), - pytest.param('int4', marks=skip_post_blackwell) -]) -@pytest.mark.parametrize( - "use_attention_plugin", [True, False], - ids=["enable_attention_plugin", "disable_attention_plugin"]) -@pytest.mark.parametrize("use_gemm_plugin", [True, False], - ids=["enable_gemm_plugin", "disable_gemm_plugin"]) -@pytest.mark.parametrize("whisper_model_root", ['large-v3', 'large-v2'], - indirect=True) -def test_llm_whisper_general(llm_venv, engine_dir, data_type, - weight_only_precision, use_attention_plugin, - use_gemm_plugin, whisper_example_root, - whisper_model_root, num_beams, use_cpp_runtime, - whisper_example_audio_file, llm_datasets_root): - print("Locate model checkpoints in test storage...") - tllm_model_name, model_ckpt_dir = whisper_model_root - - if any((not use_attention_plugin, use_gemm_plugin, 'v3' - not in tllm_model_name)) and use_cpp_runtime: - pytest.skip(f"Plugins might not support C++ runtime. Skip the test...") - - whisper_engine_dir = f"{engine_dir}/{tllm_model_name}/{data_type}_{weight_only_precision}" - - if 'int' in weight_only_precision: - use_weight_only = True - else: - use_weight_only = False - weight_only_precision = None - converted_weight_dir = convert_weights( - llm_venv=llm_venv, - example_root=whisper_example_root, - cmodel_dir=whisper_engine_dir, - model=tllm_model_name, - model_path=model_ckpt_dir, - use_weight_only=use_weight_only, - weight_only_precision=weight_only_precision) - print("Build engines...") - for component in ["encoder", "decoder"]: - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}/{component}", - f"--output_dir={whisper_engine_dir}/{component}", - "--paged_kv_cache=disable", - "--moe_plugin=disable", - "--max_batch_size=8", - ] - if use_cpp_runtime: - build_cmd.extend( - ("--paged_kv_cache enable", "--remove_input_padding enable")) - else: - build_cmd.append("--remove_input_padding=disable") - - if component == "encoder": - build_cmd.append( - f"--max_input_len=3000" - ) # check against actual encoder features length (3000,...) in C++ runtime - build_cmd.append(f"--max_seq_len=3000") - if component == "decoder": - build_cmd.append(f"--max_input_len=14") - build_cmd.append(f"--max_seq_len=114") - build_cmd.append(f"--max_encoder_input_len=3000") - build_cmd.append(f"--max_beam_width={num_beams}") - - if use_gemm_plugin: - build_cmd.append(f"--gemm_plugin={data_type}") - else: - build_cmd.append(f"--gemm_plugin=disable") - - if use_attention_plugin: - build_cmd.append(f"--bert_attention_plugin={data_type}") - build_cmd.append(f"--gpt_attention_plugin={data_type}") - else: - build_cmd.append(f"--bert_attention_plugin=disable") - build_cmd.append(f"--gpt_attention_plugin=disable") - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - if use_cpp_runtime: - print("Run inference using Python bindings of C++ runtime...") - run_cmd = [ - f'{whisper_example_root}/../../../run.py', - f'--tokenizer_dir={llm_models_root()}/whisper-{tllm_model_name}', - f'--multimodal_input_file={whisper_example_audio_file}', - f'--engine_dir={whisper_engine_dir}', - f'--max_output_len=96', - ] - else: - print("Run inference using Whisper's custom Python runtime...") - run_cmd = [ - f"{whisper_example_root}/run.py", - f"--dataset={llm_datasets_root}/hf-internal-testing/librispeech_asr_dummy", - f"--engine_dir={whisper_engine_dir}", - f"--assets_dir={model_ckpt_dir}", - f"--num_beams={num_beams}", - f"--dtype={data_type}", - f"--use_py_session", - f"--accuracy_check", - ] - # https://nvbugs/4658787 - # WAR before whisper tests can work offline - env = {"HF_DATASETS_OFFLINE": "0"} - venv_check_call(llm_venv, run_cmd, env=env) - - -@skip_post_blackwell -@pytest.mark.parametrize("num_beams", [4], - ids=lambda num_beams: f'nb:{num_beams}') -@pytest.mark.parametrize("whisper_model_root", ['large-v3'], indirect=True) -def test_whisper_beam_search_generation_logits(llm_venv, engine_dir, - whisper_example_root, - whisper_model_root, num_beams, - whisper_example_audio_file): - """Verify that generation_logits are reordered to match the final beam paths. - - With beam search (num_beams > 1), generation_logits must be reindexed by - parentIds so that argmax(generation_logits[beam][t]) == output_ids[beam][t] - at every position. Without reordering, logits are indexed by beam slot - rather than by the final beam path, producing incorrect probabilities. - """ - tllm_model_name, model_ckpt_dir = whisper_model_root - - whisper_engine_dir = f"{engine_dir}/{tllm_model_name}/float16_disable_weight_only" - - converted_weight_dir = convert_weights(llm_venv=llm_venv, - example_root=whisper_example_root, - cmodel_dir=whisper_engine_dir, - model=tllm_model_name, - model_path=model_ckpt_dir, - use_weight_only=False, - weight_only_precision=None) - - print("Build engines for beam search generation logits test...") - for component in ["encoder", "decoder"]: - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}/{component}", - f"--output_dir={whisper_engine_dir}/{component}", - "--paged_kv_cache=enable", - "--remove_input_padding=enable", - "--moe_plugin=disable", - "--max_batch_size=1", - ] - if component == "encoder": - build_cmd.append("--max_input_len=3000") - build_cmd.append("--max_seq_len=3000") - if component == "decoder": - build_cmd.append("--max_input_len=14") - build_cmd.append("--max_seq_len=114") - build_cmd.append("--max_encoder_input_len=3000") - build_cmd.append(f"--max_beam_width={num_beams}") - build_cmd.append("--gemm_plugin=float16") - build_cmd.append("--bert_attention_plugin=float16") - build_cmd.append("--gpt_attention_plugin=float16") - - check_call(build_cmd, env=llm_venv._new_env) - - print("Run generation logits beam search validation...") - validation_script = os.path.join(os.path.dirname(__file__), - "validate_whisper_beam_logits.py") - run_cmd = [ - validation_script, - f"--engine_dir={whisper_engine_dir}", - f"--assets_dir={model_ckpt_dir}", - f"--input_file={whisper_example_audio_file}", - f"--num_beams={num_beams}", - ] - env = { - "HF_DATASETS_OFFLINE": - "0", - "PYTHONPATH": - os.pathsep.join( - filter(None, [whisper_example_root, - os.environ.get("PYTHONPATH")])), - } - venv_check_call(llm_venv, run_cmd, env=env) - - -@skip_post_blackwell -@pytest.mark.parametrize("num_beams", [4], - ids=lambda num_beams: f'nb:{num_beams}') -@pytest.mark.parametrize("batch_size", [4], - ids=lambda batch_size: f'bs:{batch_size}') -@pytest.mark.parametrize("whisper_model_root", ['large-v3'], indirect=True) -def test_whisper_log_probs_determinism(llm_venv, engine_dir, llm_root, - whisper_example_root, whisper_model_root, - num_beams, batch_size, - llm_datasets_root): - """Regression test for nMaxBatchSize stride bug in beam search log_probs. - - Sends a batch of requests with different audio samples so each item produces - a different number of output tokens. When the batch drains unevenly the - active batch dimension shrinks; the bug caused beamStage3 to use that - shrinking value as the logProbsTiled stride, producing non-deterministic - log_probs across runs. - - Runs inference num_runs times and asserts log_probs are bit-identical. - """ - tllm_model_name, model_ckpt_dir = whisper_model_root - - whisper_engine_dir = (f"{engine_dir}/{tllm_model_name}" - f"/float16_bs{batch_size}_nb{num_beams}_logprobs_det") - - converted_weight_dir = convert_weights(llm_venv=llm_venv, - example_root=whisper_example_root, - cmodel_dir=whisper_engine_dir, - model=tllm_model_name, - model_path=model_ckpt_dir, - use_weight_only=False, - weight_only_precision=None) - - print("Build engines for log_probs determinism test...") - for component in ["encoder", "decoder"]: - build_cmd = [ - "trtllm-build", - f"--checkpoint_dir={converted_weight_dir}/{component}", - f"--output_dir={whisper_engine_dir}/{component}", - "--paged_kv_cache=enable", - "--remove_input_padding=enable", - "--moe_plugin=disable", - f"--max_batch_size={batch_size}", - "--gemm_plugin=float16", - "--bert_attention_plugin=float16", - "--gpt_attention_plugin=float16", - ] - if component == "encoder": - build_cmd.append("--max_input_len=3000") - build_cmd.append("--max_seq_len=3000") - if component == "decoder": - build_cmd.append("--max_input_len=14") - build_cmd.append("--max_seq_len=114") - build_cmd.append("--max_encoder_input_len=3000") - build_cmd.append(f"--max_beam_width={num_beams}") - - check_call(build_cmd, env=llm_venv._new_env) - - print("Run log_probs determinism validation...") - validation_script = os.path.join( - os.path.dirname(__file__), "validate_whisper_log_probs_determinism.py") - librispeech_dir = os.path.join(llm_datasets_root, - "hf-internal-testing/librispeech_asr_dummy") - run_cmd = [ - validation_script, - f"--engine_dir={whisper_engine_dir}", - f"--assets_dir={model_ckpt_dir}", - f"--dataset_dir={librispeech_dir}", - f"--num_beams={num_beams}", - f"--batch_size={batch_size}", - "--num_runs=5", - ] - env = { - # llm_root is listed first so the worktree's tensorrt_llm takes priority - # over any system-installed version when the validation subprocess runs. - "PYTHONPATH": - os.pathsep.join( - filter( - None, - [llm_root, whisper_example_root, - os.environ.get("PYTHONPATH")])), - } - venv_check_call(llm_venv, run_cmd, env=env) diff --git a/tests/integration/defs/examples/validate_whisper_beam_logits.py b/tests/integration/defs/examples/validate_whisper_beam_logits.py deleted file mode 100644 index faed875b9b6c..000000000000 --- a/tests/integration/defs/examples/validate_whisper_beam_logits.py +++ /dev/null @@ -1,205 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Validate that generation_logits are correctly reordered for beam search. - -With beam search (num_beams > 1), generation_logits must be reindexed to match -the final beam paths after gatherTree finalization. This script runs whisper -inference via ModelRunnerCpp and checks that each output token has a reasonable -probability under its corresponding generation logits. - -Exits with non-zero status if any output token has near-zero probability -(log P < -10), which indicates logits from a different beam's context. -""" - -import argparse -import json -import sys -from collections import OrderedDict -from pathlib import Path - -import numpy as np -import torch - -import tensorrt_llm -from tensorrt_llm._utils import str_dtype_to_torch -from tensorrt_llm.runtime import ModelRunnerCpp - - -def read_config(component, engine_dir): - config_path = engine_dir / component / "config.json" - with open(config_path, "r") as f: - config = json.load(f) - model_config = OrderedDict() - model_config.update(config["pretrained_config"]) - model_config.update(config["build_config"]) - return model_config - - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--engine_dir", type=str, required=True) - parser.add_argument("--assets_dir", type=str, required=True) - parser.add_argument("--input_file", type=str, default=None) - parser.add_argument("--num_beams", type=int, default=4) - parser.add_argument("--max_new_tokens", type=int, default=96) - return parser.parse_args() - - -def get_mel_input(input_file, n_mels, mel_filters_dir): - from whisper_utils import log_mel_spectrogram - - if input_file: - mel, _ = log_mel_spectrogram( - input_file, n_mels, device="cuda", return_duration=True, mel_filters_dir=mel_filters_dir - ) - else: - from datasets import load_dataset - - dataset = load_dataset( - "hf-internal-testing/librispeech_asr_dummy", - "clean", - split="validation", - trust_remote_code=True, - ) - speech = dataset[0]["audio"]["array"].astype(np.float32) - waveform = torch.from_numpy(speech) - mel = log_mel_spectrogram(waveform, n_mels, device="cuda", mel_filters_dir=mel_filters_dir) - - mel = mel.type(str_dtype_to_torch("float16")) - mel = mel.unsqueeze(0) - if mel.shape[2] % 2: - mel = torch.nn.functional.pad(mel, (0, 1)) - return mel - - -def validate_logits_alignment(output_ids, generation_logits, input_len, eot_id): - """Check that output tokens have reasonable probability under generation_logits. - - Returns True if all output tokens have log P > -10 across all beams, False otherwise. - """ - LOG_PROB_THRESHOLD = -10.0 - batch_size = output_ids.shape[0] - num_beams = output_ids.shape[1] - all_aligned = True - - for b in range(batch_size): - for beam in range(num_beams): - gen_tokens = output_ids[b, beam, input_len:] - eot_positions = (gen_tokens == eot_id).nonzero(as_tuple=True)[0] - gen_len = eot_positions[0].item() if len(eot_positions) > 0 else gen_tokens.shape[0] - - if gen_len == 0: - continue - - gen_tokens = gen_tokens[:gen_len] - logits = generation_logits[b, beam, :gen_len, :] - - log_probs = torch.nn.functional.log_softmax(logits, dim=-1) - actual_logprobs = log_probs.gather(1, gen_tokens.unsqueeze(1)).squeeze(1) - - min_logprob = actual_logprobs.min().item() - near_zero = (actual_logprobs < LOG_PROB_THRESHOLD).sum().item() - - argmax_matches = (logits.argmax(dim=-1) == gen_tokens).sum().item() - - print( - f" Batch {b}, beam {beam}: argmax match {argmax_matches}/{gen_len}, " - f"min log P = {min_logprob:.4f}, " - f"near-zero positions = {near_zero}/{gen_len}" - ) - - if near_zero > 0: - all_aligned = False - print(f" FAIL: {near_zero} positions have near-zero probability") - - return all_aligned - - -def main(): - args = parse_arguments() - tensorrt_llm.logger.set_level("warning") - - engine_dir = Path(args.engine_dir) - encoder_config = read_config("encoder", engine_dir) - decoder_config = read_config("decoder", engine_dir) - - n_mels = encoder_config["n_mels"] - is_multilingual = decoder_config["vocab_size"] >= 51865 - - from tokenizer import get_tokenizer - - tokenizer_name = "multilingual" if is_multilingual else "gpt2" - tokenizer = get_tokenizer( - name=tokenizer_name, - num_languages=encoder_config["num_languages"], - tokenizer_dir=args.assets_dir, - ) - eot_id = tokenizer.encode("<|endoftext|>", allowed_special=tokenizer.special_tokens_set)[0] - - runner = ModelRunnerCpp.from_dir( - engine_dir=engine_dir, - is_enc_dec=True, - max_batch_size=1, - max_input_len=3000, - max_output_len=args.max_new_tokens, - max_beam_width=args.num_beams, - kv_cache_free_gpu_memory_fraction=0.9, - cross_kv_cache_fraction=0.5, - gather_generation_logits=True, - ) - - mel = get_mel_input(args.input_file, n_mels, args.assets_dir) - mel_input_lengths = torch.full((1,), mel.shape[2], dtype=torch.int32, device="cuda") - - prompt_text = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>" - prompt_ids = tokenizer.encode(prompt_text, allowed_special=tokenizer.special_tokens_set) - decoder_input_ids = torch.tensor(prompt_ids).unsqueeze(0) - input_len = decoder_input_ids.shape[1] - - with torch.no_grad(): - outputs = runner.generate( - batch_input_ids=decoder_input_ids, - encoder_input_features=mel.transpose(1, 2), - encoder_output_lengths=mel_input_lengths // 2, - max_new_tokens=args.max_new_tokens, - end_id=eot_id, - pad_id=eot_id, - num_beams=args.num_beams, - num_return_sequences=args.num_beams, - output_sequence_lengths=True, - output_generation_logits=True, - return_dict=True, - ) - torch.cuda.synchronize() - - generation_logits = outputs["generation_logits"] - assert generation_logits.shape[1] == args.num_beams, ( - f"Expected generation_logits beam dimension to be {args.num_beams}, " - f"got {generation_logits.shape[1]}" - ) - - passed = validate_logits_alignment( - outputs["output_ids"].cpu(), generation_logits.cpu(), input_len, eot_id - ) - - if passed: - print("PASS: generation_logits aligned with output_ids") - else: - print("FAIL: generation_logits misaligned with output_ids") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tests/integration/defs/examples/validate_whisper_log_probs_determinism.py b/tests/integration/defs/examples/validate_whisper_log_probs_determinism.py deleted file mode 100644 index 78d950880613..000000000000 --- a/tests/integration/defs/examples/validate_whisper_log_probs_determinism.py +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Validate that log_probs are deterministic across runs with beam search and batch_size > 1. - -Reproduces the nMaxBatchSize stride bug: when batch items finish at different -times the active batch dimension shrinks, causing beamStage3 to read logProbsTiled -with the wrong stride and produce different log_probs values across runs. - -Different LibriSpeech samples produce different decoder output lengths, creating -an uneven batch where some items finish before others — the exact scenario that -triggers the bug. -""" - -import argparse -import json -import sys -from collections import OrderedDict -from pathlib import Path - -import numpy as np -import torch - -import tensorrt_llm -from tensorrt_llm._utils import str_dtype_to_torch -from tensorrt_llm.runtime import ModelRunnerCpp - - -def read_config(component, engine_dir): - config_path = engine_dir / component / "config.json" - with open(config_path, "r") as f: - config = json.load(f) - model_config = OrderedDict() - model_config.update(config["pretrained_config"]) - model_config.update(config["build_config"]) - return model_config - - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--engine_dir", type=str, required=True) - parser.add_argument("--assets_dir", type=str, required=True) - parser.add_argument( - "--dataset_dir", - type=str, - required=True, - help="Local path to hf-internal-testing/librispeech_asr_dummy dataset", - ) - parser.add_argument("--num_beams", type=int, default=4) - parser.add_argument("--batch_size", type=int, default=4) - parser.add_argument("--max_new_tokens", type=int, default=96) - parser.add_argument("--num_runs", type=int, default=5) - args = parser.parse_args() - if args.num_runs < 2: - parser.error("--num_runs must be >= 2 to compare log_probs across runs") - return args - - -def main(): - args = parse_arguments() - tensorrt_llm.logger.set_level("warning") - - engine_dir = Path(args.engine_dir) - encoder_config = read_config("encoder", engine_dir) - decoder_config = read_config("decoder", engine_dir) - - n_mels = encoder_config["n_mels"] - is_multilingual = decoder_config["vocab_size"] >= 51865 # multilingual vocab size threshold - - from tokenizer import get_tokenizer - from whisper_utils import log_mel_spectrogram - - tokenizer_name = "multilingual" if is_multilingual else "gpt2" - tokenizer = get_tokenizer( - name=tokenizer_name, - num_languages=encoder_config["num_languages"], - tokenizer_dir=args.assets_dir, - ) - eot_id = tokenizer.encode("<|endoftext|>", allowed_special=tokenizer.special_tokens_set)[0] - - runner = ModelRunnerCpp.from_dir( - engine_dir=engine_dir, - is_enc_dec=True, - max_batch_size=args.batch_size, - max_input_len=3000, - max_output_len=args.max_new_tokens, - max_beam_width=args.num_beams, - kv_cache_free_gpu_memory_fraction=0.9, - cross_kv_cache_fraction=0.5, - gather_generation_logits=True, - ) - - # Use different LibriSpeech samples so each batch item produces a different - # number of output tokens — this creates the uneven-finish condition that - # triggers the nMaxBatchSize stride bug. - from datasets import load_dataset - - dataset = load_dataset(args.dataset_dir, "clean", split="validation", trust_remote_code=True) - - mel_list = [] - for i in range(args.batch_size): - speech = dataset[i]["audio"]["array"].astype(np.float32) - waveform = torch.from_numpy(speech) - m = log_mel_spectrogram(waveform, n_mels, device="cuda", mel_filters_dir=args.assets_dir) - m = m.type(str_dtype_to_torch("float16")) - if m.shape[1] % 2: - m = torch.nn.functional.pad(m, (0, 1)) - mel_list.append(m) - - max_mel_len = max(m.shape[1] for m in mel_list) - mel_batched = torch.zeros( - args.batch_size, - mel_list[0].shape[0], - max_mel_len, - dtype=mel_list[0].dtype, - device=mel_list[0].device, - ) - for i, m in enumerate(mel_list): - mel_batched[i, :, : m.shape[1]] = m - - mel_input_lengths = torch.full( - (args.batch_size,), max_mel_len, dtype=torch.int32, device="cuda" - ) - - prompt_text = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>" - prompt_ids = tokenizer.encode(prompt_text, allowed_special=tokenizer.special_tokens_set) - decoder_input_ids = torch.tensor(prompt_ids).unsqueeze(0).repeat(args.batch_size, 1) - - all_log_probs = [] - ref_output_ids = None - ref_gen_logits = None - input_len = decoder_input_ids.shape[1] - for run_idx in range(args.num_runs): - with torch.no_grad(): - outputs = runner.generate( - batch_input_ids=decoder_input_ids, - encoder_input_features=mel_batched.transpose(1, 2), - encoder_output_lengths=mel_input_lengths // 2, - max_new_tokens=args.max_new_tokens, - end_id=eot_id, - pad_id=eot_id, - num_beams=args.num_beams, - num_return_sequences=1, - output_sequence_lengths=True, - output_generation_logits=True, - output_log_probs=True, - return_dict=True, - ) - torch.cuda.synchronize() - all_log_probs.append(outputs["log_probs"][:, 0, :].cpu()) - if run_idx == 0: - ref_output_ids = outputs["output_ids"].cpu() - ref_gen_logits = outputs["generation_logits"].cpu() - - for i in range(1, args.num_runs): - if all_log_probs[i].shape != all_log_probs[0].shape: - print( - f"FAIL: log_probs shape mismatch between run 1 {all_log_probs[0].shape} " - f"and run {i + 1} {all_log_probs[i].shape}" - ) - sys.exit(1) - - max_diff = max( - (all_log_probs[0] - all_log_probs[i]).abs().max().item() for i in range(1, args.num_runs) - ) - - if max_diff >= 1e-6: - print(f"FAIL: log_probs are non-deterministic (max diff: {max_diff:.6f})") - sys.exit(1) - - # Correctness check: verify log_probs[b][t] matches log_softmax(generation_logits[b][t])[token] - # for every batch slot b (including b > 0, exercising the gatherTree batchSlot offset fix). - # Uses a loose tolerance because log_probs come from float32 beam search bookkeeping while - # generation_logits are the raw fp16->fp32 decoder outputs. - LOG_PROB_ATOL = 0.5 - all_aligned = True - for b in range(args.batch_size): - gen_tokens = ref_output_ids[b, 0, input_len:] - eot_pos = (gen_tokens == eot_id).nonzero(as_tuple=True)[0] - gen_len = eot_pos[0].item() if len(eot_pos) > 0 else gen_tokens.shape[0] - if gen_len == 0: - continue - logits = ref_gen_logits[b, 0, :gen_len, :] - log_probs_from_logits = torch.nn.functional.log_softmax(logits.float(), dim=-1) - expected = log_probs_from_logits.gather(1, gen_tokens[:gen_len].unsqueeze(1)).squeeze(1) - actual = all_log_probs[0][b, :gen_len] - max_lp_diff = (actual - expected).abs().max().item() - if max_lp_diff > LOG_PROB_ATOL: - print( - f"FAIL: log_probs[batch={b}] deviate from generation_logits " - f"(max diff: {max_lp_diff:.4f} > {LOG_PROB_ATOL}); " - "likely caused by wrong logProbsTiled batchSlot offset in gatherTree." - ) - all_aligned = False - - if not all_aligned: - sys.exit(1) - - print( - f"PASS: log_probs are deterministic across {args.num_runs} runs " - f"(max diff: {max_diff:.2e}) and aligned with generation_logits for all batch slots." - ) - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/tests/integration/defs/examples/visual_gen/conftest.py b/tests/integration/defs/examples/visual_gen/conftest.py new file mode 100644 index 000000000000..9d65d09ef545 --- /dev/null +++ b/tests/integration/defs/examples/visual_gen/conftest.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared fixtures for VisualGen example integration tests.""" + +import pytest +from defs.trt_test_alternative import check_call + + +@pytest.fixture(scope="session") +def _visual_gen_deps(llm_venv): + """Install optional media dependencies once per VisualGen test session.""" + llm_venv.run_cmd(["-m", "pip", "install", "av"]) + check_call(["apt-get", "update", "-y"], shell=False) + check_call(["apt-get", "install", "-y", "ffmpeg"], shell=False) diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_i2v_4step_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_i2v_4step_lpips_golden_video.json new file mode 100644 index 000000000000..1ebbc1f8f9c8 --- /dev/null +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_i2v_4step_lpips_golden_video.json @@ -0,0 +1,27 @@ +{ + "video": "cosmos3_i2v_4step_lpips_golden_video.mp4", + "model": "Cosmos3-Super-Image2Video-4Step", + "source": "diffusers Cosmos3DistilledModularPipeline (reference implementation, not a TRT-LLM self-golden)", + "diffusers_reference": "huggingface/diffusers#14177 'Cosmos3 Distilled support' (merged 2026-07-17)", + "diffusers_version": "0.40.0.dev0", + "diffusers_commit": "6b5199f81dc0044fe0417ebe96bbad6c59c19874", + "reference_rng_patch": "Cosmos3DistilledVisionLoopSchedulerStep patched to declare InputParam('generator') and pass generator=block_state.generator into scheduler.step. The merged PR draws the per-step SDE noise from the global torch RNG, so unpatched same-seed runs are different samples (measured LPIPS 0.144 vs 0.056 patched); the patch makes the reference trajectory reproducible and comparable.", + "reference_pipeline_construction": "The checkpoint's own modular_model_index.json is stale (base Cosmos3OmniBlocks, refs/pr/1, no distilled_sigmas). The golden run loaded a corrected index selecting Cosmos3DistilledModularPipeline / Cosmos3DistilledBlocks with is_distilled=true, distilled_sigmas=[1.0, 0.9375, 0.8333333333333334, 0.625], and component specs pointing at the local checkpoint subfolders.", + "prompt": "The orange sphere slowly rises while the camera pans right across the scene", + "conditioning_image": "deterministic 1280x720 image drawn by _write_cosmos3_i2v_conditioning_image in test_visual_gen.py (gradient sky, orange circle, green rectangle, yellow triangle)", + "height": 720, + "width": 1280, + "num_frames": 29, + "num_inference_steps": 4, + "guidance_scale": 1.0, + "seed": 42, + "global_rng_seed": 42, + "frame_rate": 24.0, + "use_system_prompt": true, + "torch_dtype": "bfloat16", + "lpips_net": "alex", + "lpips_threshold": 0.1, + "measured_lpips_at_creation": 0.0563, + "threshold_rationale": "0.0563 measured cross-stack (TRT-LLM VANILLA attention vs diffusers) on B200 with matched noise trajectories, plus headroom for the ~0.04 cross-host kernel drift documented in _preserve_lpips_candidate_on_failure.", + "notes": "num_inference_steps and guidance_scale are fixed by the distilled checkpoint (scheduler fixed_step_sampler_config.t_list; CFG baked into the weights). use_system_prompt=true matches the checkpoint's default_use_system_prompt declaration in model_index.json." +} diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/qwen_image_layered_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/qwen_image_layered_lpips_golden.json new file mode 100644 index 000000000000..000297f2e851 --- /dev/null +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/qwen_image_layered_lpips_golden.json @@ -0,0 +1,24 @@ +{ + "image": "qwen_image_layered_lpips_golden/ (assembled into a 2x2 layer grid for LPIPS)", + "model": "Qwen-Image-Layered", + "source": "Qwen-Image-Layered official demo", + "prompt": "", + "negative_prompt": " ", + "input_image": "qwen_image_layered_lpips_input.png", + "input_image_source": "https://github.com/QwenLM/Qwen-Image-Layered/blob/main/assets/test_images/1.png", + "height": 1280, + "width": 1280, + "num_inference_steps": 50, + "true_cfg_scale": 4.0, + "layers": 4, + "resolution": 640, + "cfg_normalize": true, + "use_en_prompt": true, + "seed": 777, + "torch_compile": false, + "deterministic_algorithms": true, + "lpips_net": "alex", + "lpips_threshold": 0.05, + "diffusers_version": "0.39.0", + "official_repo_commit": "54c4fe4" +} diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip index b00403606a62..632d220fe72e 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d38286a6ee60db331a89bb3563fcc0896309b6e21228d4867ab33990c5b47ed -size 14386487 +oid sha256:2844dd5fcf63e55a98fa903137be192ddc7a912b76e9c32e1dd02524dedd3177 +size 17758973 diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_fa4_fully_eager_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_fa4_fully_eager_lpips_golden_video.json new file mode 100644 index 000000000000..253c9f08e651 --- /dev/null +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_fa4_fully_eager_lpips_golden_video.json @@ -0,0 +1,25 @@ +{ + "video": "wan22_t2v_fa4_fully_eager_lpips_golden_video.mp4", + "model": "Wan2.2-T2V-A14B-Diffusers", + "source": "TensorRT-LLM VisualGen", + "prompt": "A cat sitting on a sunny windowsill watching birds outside.", + "negative_prompt": "", + "height": 480, + "width": 832, + "num_frames": 9, + "num_inference_steps": 4, + "guidance_scale": 4.0, + "seed": 42, + "frame_rate": 16.0, + "attention_backend": "FA4", + "torch_compile": false, + "torch_compile_stance": "force_eager", + "deterministic_algorithms": true, + "lpips_net": "alex", + "lpips_threshold": 0.25, + "diffusers_version": "0.38.0", + "torch_version": "2.12.0a0+0291f960b6.nv26.04.48445190", + "tensorrt_llm_version": "1.3.0rc21", + "tensorrt_llm_commit": "a0c406ff88c4a9736b5ce2f3c5eacbacdd0926d1", + "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm-staging/release@sha256:0783d275ffe7efc3b6093dd6c7e823cd703eae973e462c8e1e4f7ca7dad60d64" +} diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_lpips_golden_video.json index 35ecacdddc49..0621a4f06eec 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_lpips_golden_video.json @@ -11,7 +11,9 @@ "guidance_scale": 4.0, "seed": 42, "frame_rate": 16.0, + "attention_backend": "VANILLA", "torch_compile": false, + "torch_compile_stance": "force_eager", "deterministic_algorithms": true, "lpips_net": "alex", "lpips_threshold": 0.05, diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen.py b/tests/integration/defs/examples/visual_gen/test_visual_gen.py index a8ecf5e39782..e997b5643fb5 100644 --- a/tests/integration/defs/examples/visual_gen/test_visual_gen.py +++ b/tests/integration/defs/examples/visual_gen/test_visual_gen.py @@ -19,6 +19,7 @@ import gc import glob import json +import math import os import random import shutil @@ -27,12 +28,11 @@ import time import urllib.request import zipfile +from typing import Any import pytest import torch import torch._inductor.config as inductor_config -from defs import conftest -from defs.common import venv_check_call from defs.trt_test_alternative import check_call from torch._inductor.async_compile import shutdown_compile_workers @@ -41,6 +41,7 @@ WAN22_A14B_NVFP4_MODEL_SUBPATH = "Wan2.2-T2V-A14B-Diffusers-NVFP4" WAN22_I2V_A14B_NVFP4_MODEL_SUBPATH = "Wan2.2-I2V-A14B-Diffusers-NVFP4" QWEN_IMAGE_MODEL_SUBPATH = "qwen-image" +QWEN_IMAGE_LAYERED_MODEL_SUBPATH = "qwen-image-layered" VISUAL_GEN_OUTPUT_VIDEO = "trtllm_output.mp4" DIFFUSERS_REFERENCE_VIDEO = "diffusers_reference.mp4" WAN_T2V_PROMPT = "A cute cat playing piano" @@ -98,6 +99,7 @@ # NOTE: QwenImage's forward CFG knob is ``true_cfg_scale`` (not ``guidance_scale``), # and real-CFG only engages when a negative prompt is supplied. QWENIMAGE_MODEL_SUBPATH = "qwen-image" +QWEN_IMAGE_EDIT_MODEL_SUBPATH = "Qwen-Image-Edit-2511" QWENIMAGE_LPIPS_PROMPT = "a tiny astronaut hatching from an egg on the moon" QWENIMAGE_LPIPS_NEGATIVE_PROMPT = "" QWENIMAGE_LPIPS_HEIGHT = 1328 @@ -106,6 +108,14 @@ QWENIMAGE_LPIPS_TRUE_CFG_SCALE = 4.0 QWENIMAGE_LPIPS_SEED = 42 QWENIMAGE_LPIPS_THRESHOLD = 0.05 +QWEN_IMAGE_LAYERED_LPIPS_PROMPT = "" +QWEN_IMAGE_LAYERED_LPIPS_NEGATIVE_PROMPT = " " +QWEN_IMAGE_LAYERED_LPIPS_NUM_INFERENCE_STEPS = 50 +QWEN_IMAGE_LAYERED_LPIPS_TRUE_CFG_SCALE = 4.0 +QWEN_IMAGE_LAYERED_LPIPS_LAYERS = 4 +QWEN_IMAGE_LAYERED_LPIPS_RESOLUTION = 640 +QWEN_IMAGE_LAYERED_LPIPS_SEED = 777 +QWEN_IMAGE_LAYERED_LPIPS_THRESHOLD = 0.05 # Cosmos3-Nano (text-to-video + text-to-image) — default-setting LPIPS golden. # Params are the Cosmos3 720P defaults (cosmos3/defaults.py:COSMOS3_720P_PARAMS). @@ -122,6 +132,20 @@ COSMOS3_LPIPS_FRAME_RATE = 24.0 COSMOS3_LPIPS_THRESHOLD = 0.05 +COSMOS3_I2V_4STEP_MODEL_SUBPATH = "Cosmos3-Super-Image2Video-4Step" +COSMOS3_I2V_4STEP_LPIPS_PROMPT = ( + "The orange sphere slowly rises while the camera pans right across the scene" +) +COSMOS3_I2V_4STEP_LPIPS_NUM_FRAMES = 29 +# Fixed by the distilled checkpoint (scheduler t_list / CFG baked into weights). +COSMOS3_I2V_4STEP_LPIPS_NUM_INFERENCE_STEPS = 4 +COSMOS3_I2V_4STEP_LPIPS_GUIDANCE_SCALE = 1.0 +# Golden is diffusers-produced (cross-stack), not a TRT-LLM self-golden: +# 0.0563 measured at creation + headroom for ~0.04 cross-host kernel drift +# (see _preserve_lpips_candidate_on_failure). Provenance: +# golden/visual_gen_lpips/cosmos3_i2v_4step_lpips_golden_video.json. +COSMOS3_I2V_4STEP_LPIPS_THRESHOLD = 0.10 + # LTX-2 configuration LTX2_MODEL_CHECKPOINT_PATH = "LTX-2/ltx-2-19b-dev.safetensors" LTX2_TEXT_ENCODER_SUBPATH = "gemma-3-12b-it" @@ -230,16 +254,6 @@ AESTHETIC_PREDICTOR_CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache", "emb_reader") -@pytest.fixture(scope="session") -def _visual_gen_deps(llm_venv): - """Install av + diffusers + ffmpeg once per session (shared by all video-gen fixtures).""" - llm_venv.run_cmd(["-m", "pip", "install", "av"]) - llm_venv.run_cmd(["-m", "pip", "install", "diffusers>=0.37.0"]) - # Install ffmpeg system package required by save_video() for MP4 encoding - check_call(["apt-get", "update", "-y"], shell=False) - check_call(["apt-get", "install", "-y", "ffmpeg"], shell=False) - - @pytest.fixture(scope="session") def vbench_repo_root(llm_venv): """Clone VBench repo into workspace and install; return repo root path.""" @@ -345,8 +359,30 @@ def _precache_aesthetic_predictor(): ) from exc +def _llm_models_root(): + # Imported lazily so that re-importing this module in a torch.multiprocessing.spawn + # child (a fresh interpreter) does not run a module-level `from defs import conftest`, + # which pulls in `tensorrt_llm.bindings` -- a compiled extension absent from the source + # tree the spawned child resolves, crashing the worker before the test runs. The parent + # process already imports conftest during collection, so this deferral is free. + from defs import conftest + + return conftest.llm_models_root() + + +def _venv_check_call(*args, **kwargs): + # Deferred like _llm_models_root above: defs.common does `from tensorrt_llm import + # LLM`, which pulls in tensorrt_llm.bindings. Importing it at module load would + # crash the torch.multiprocessing.spawn child processes used by the multi-GPU LPIPS + # tests, which re-import this module before the worker fixes sys.path. Only the + # single-GPU example tests call this, and only in the parent process. + from defs.common import venv_check_call + + return venv_check_call(*args, **kwargs) + + def _lpips_model_path(*parts): - return os.path.join(conftest.llm_models_root(), *parts) + return os.path.join(_llm_models_root(), *parts) def _skip_if_missing(path, label, is_dir=False): @@ -382,7 +418,7 @@ def _golden_media_path(tmp_path, media_name, label): def _ltx2_lpips_text_encoder_path(): - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() candidates = [ os.path.join(scratch_space, LTX2_TEXT_ENCODER_SUBPATH), os.path.join(scratch_space, "gemma", LTX2_TEXT_ENCODER_SUBPATH), @@ -412,14 +448,28 @@ def _cleanup_cuda(): @contextlib.contextmanager -def _lpips_deterministic_algorithms(): - previous = torch.are_deterministic_algorithms_enabled() - os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") - torch.use_deterministic_algorithms(True) +def _lpips_deterministic_algorithms(*, fully_eager=False): + previous_deterministic = torch.are_deterministic_algorithms_enabled() + previous_warn_only = torch.is_deterministic_algorithms_warn_only_enabled() + previous_cublas_workspace_config = os.environ.get("CUBLAS_WORKSPACE_CONFIG") + try: - yield + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + torch.use_deterministic_algorithms(True) + compiler_context = ( + torch.compiler.set_stance("force_eager") if fully_eager else contextlib.nullcontext() + ) + with compiler_context: + yield finally: - torch.use_deterministic_algorithms(previous) + torch.use_deterministic_algorithms( + previous_deterministic, + warn_only=previous_warn_only, + ) + if previous_cublas_workspace_config is None: + os.environ.pop("CUBLAS_WORKSPACE_CONFIG", None) + else: + os.environ["CUBLAS_WORKSPACE_CONFIG"] = previous_cublas_workspace_config def _save_lpips_video_mp4(video, output_path, frame_rate): @@ -624,7 +674,7 @@ def _generate_ltx2_cuda_graph_trtllm_backend_video(output_path): TorchCompileConfig, ) - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() checkpoint_path = os.path.join(scratch_space, LTX2_MODEL_CHECKPOINT_PATH) text_encoder_path = _ltx2_lpips_text_encoder_path() spatial_upsampler_path = os.path.join(scratch_space, LTX2_UPSAMPLER_SUBPATH) @@ -683,6 +733,8 @@ def _generate_ltx2_cuda_graph_trtllm_backend_video(output_path): _save_lpips_video_mp4(output.video, output_path, frame_rate=LTX2_T2V_FRAME_RATE) finally: visual_gen.shutdown() + del visual_gen + _cleanup_cuda() assert os.path.isfile(output_path), f"LTX-2 TRTLLM backend did not produce {output_path}" @@ -699,6 +751,7 @@ def _run_wan_lpips_pipeline( seed, attention_backend="VANILLA", parallel=None, + fully_eager=False, ): from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader from tensorrt_llm.visual_gen.args import AttentionConfig, TorchCompileConfig, VisualGenArgs @@ -712,7 +765,7 @@ def _run_wan_lpips_pipeline( ) if parallel is not None: args_kwargs["parallel_config"] = parallel - with _lpips_deterministic_algorithms(): + with _lpips_deterministic_algorithms(fully_eager=fully_eager): args = VisualGenArgs(**args_kwargs) pipeline = PipelineLoader(args).load(skip_warmup=True) try: @@ -747,7 +800,9 @@ def _generate_wan_lpips_video( guidance_scale, seed, frame_rate, + attention_backend="VANILLA", parallel=None, + fully_eager=False, ): generated_video = _run_wan_lpips_pipeline( model_path, @@ -759,7 +814,9 @@ def _generate_wan_lpips_video( num_inference_steps, guidance_scale, seed, + attention_backend=attention_backend, parallel=parallel, + fully_eager=fully_eager, ) assert generated_video is not None, "Single-GPU Wan LPIPS run produced no video" _save_lpips_video_mp4(generated_video, output_path, frame_rate=frame_rate) @@ -811,17 +868,18 @@ def wan22_bf16_video_path(_visual_gen_deps, llm_venv): return output_path -def _generate_qwenimage_lpips_image(model_path, output_path): - """Generate the QwenImage text-to-image LPIPS sample (default setting, compile-off).""" +def _generate_qwenimage_lpips_image(model_path, output_path, *, enable_cuda_graph=False): + """Generate the QwenImage text-to-image LPIPS sample.""" from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader from tensorrt_llm.media.encoding import save_image - from tensorrt_llm.visual_gen.args import TorchCompileConfig, VisualGenArgs + from tensorrt_llm.visual_gen.args import CudaGraphConfig, TorchCompileConfig, VisualGenArgs _skip_if_missing(model_path, "QwenImage checkpoint", is_dir=True) _disable_inductor_compile_worker_quiesce() args = VisualGenArgs( model=model_path, torch_compile_config=TorchCompileConfig(enable=False), + cuda_graph_config=CudaGraphConfig(enable=enable_cuda_graph), ) pipeline = PipelineLoader(args).load(skip_warmup=True) try: @@ -843,6 +901,97 @@ def _generate_qwenimage_lpips_image(model_path, output_path): save_image(generated_image, output_path) +def _copy_qwen_image_layered_lpips_input(tmp_path, input_path): + source = _golden_media_path( + tmp_path, + "qwen_image_layered_lpips_input.png", + "Qwen-Image-Layered LPIPS input image", + ) + shutil.copyfile(source, input_path) + + +def _qwen_image_layered_golden_layer_paths(tmp_path): + golden_dir = _golden_media_path( + tmp_path, + "qwen_image_layered_lpips_golden", + "Qwen-Image-Layered LPIPS golden layer directory", + ) + layer_paths = sorted( + golden_dir.glob("layer_*.png"), + key=lambda path: int(path.stem.rsplit("_", 1)[1]), + ) + assert layer_paths, f"Qwen-Image-Layered golden layer directory is empty: {golden_dir}" + return layer_paths + + +def _write_qwen_image_layered_lpips_golden_grid(tmp_path, output_path): + from PIL import Image + + layer_paths = _qwen_image_layered_golden_layer_paths(tmp_path) + layers = [] + for path in layer_paths: + with Image.open(path) as image: + layers.append(image.convert("RGBA").copy()) + + width, height = layers[0].size + assert all(layer.size == (width, height) for layer in layers), ( + "Qwen-Image-Layered golden layers must have identical sizes, got " + f"{[layer.size for layer in layers]}" + ) + grid_cols = math.ceil(math.sqrt(len(layers))) + grid_rows = math.ceil(len(layers) / grid_cols) + grid = Image.new("RGBA", (grid_cols * width, grid_rows * height), (0, 0, 0, 0)) + for index, layer in enumerate(layers): + row, col = divmod(index, grid_cols) + grid.alpha_composite(layer, dest=(col * width, row * height)) + grid.save(output_path) + + +def _flatten_qwen_image_layered_lpips_image(input_path, output_path): + from PIL import Image + + with Image.open(input_path) as image: + rgba_image = image.convert("RGBA") + background = Image.new("RGBA", rgba_image.size, (255, 255, 255, 255)) + background.alpha_composite(rgba_image) + background.convert("RGB").save(output_path) + + +def _generate_qwen_image_layered_lpips_image(model_path, input_path, output_path): + """Generate the Qwen-Image-Layered LPIPS sample (default setting, compile-off).""" + from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader + from tensorrt_llm.media.encoding import save_image + from tensorrt_llm.visual_gen.args import TorchCompileConfig, VisualGenArgs + + _skip_if_missing(model_path, "Qwen-Image-Layered checkpoint", is_dir=True) + _disable_inductor_compile_worker_quiesce() + args = VisualGenArgs( + model=model_path, + torch_compile_config=TorchCompileConfig(enable=False), + ) + pipeline = PipelineLoader(args).load(skip_warmup=True) + try: + with torch.no_grad(): + result = pipeline.forward( + image=str(input_path), + prompt=QWEN_IMAGE_LAYERED_LPIPS_PROMPT, + negative_prompt=QWEN_IMAGE_LAYERED_LPIPS_NEGATIVE_PROMPT, + num_inference_steps=QWEN_IMAGE_LAYERED_LPIPS_NUM_INFERENCE_STEPS, + true_cfg_scale=QWEN_IMAGE_LAYERED_LPIPS_TRUE_CFG_SCALE, + layers=QWEN_IMAGE_LAYERED_LPIPS_LAYERS, + resolution=QWEN_IMAGE_LAYERED_LPIPS_RESOLUTION, + cfg_normalize=True, + use_en_prompt=True, + seed=QWEN_IMAGE_LAYERED_LPIPS_SEED, + ) + generated_image = result.image[0].detach().cpu() + finally: + del pipeline + _cleanup_cuda() + + save_image(generated_image, output_path) + + def _run_cosmos3_lpips_pipeline(num_frames): """Run the Cosmos3-Nano pipeline (default setting, VANILLA attn, compile-off). @@ -1082,6 +1231,57 @@ def test_qwenimage_lpips_against_golden(tmp_path): _assert_lpips_below_threshold(score, QWENIMAGE_LPIPS_THRESHOLD) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +def test_qwenimage_cuda_graph_lpips_against_golden(tmp_path): + generated_path = tmp_path / "qwenimage_cuda_graph_generated.png" + golden_path = _golden_media_path( + tmp_path, "qwenimage_lpips_golden.png", "QwenImage LPIPS golden image" + ) + _generate_qwenimage_lpips_image( + _lpips_model_path(QWENIMAGE_MODEL_SUBPATH), + generated_path, + enable_cuda_graph=True, + ) + score = _run_lpips_eval( + tmp_path, + "qwenimage_cuda_graph", + "image", + QWENIMAGE_LPIPS_PROMPT, + golden_path, + generated_path, + ) + _assert_lpips_below_threshold(score, QWENIMAGE_LPIPS_THRESHOLD) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +def test_qwen_image_layered_lpips_against_golden(tmp_path): + input_path = tmp_path / "qwen_image_layered_input.png" + generated_path = tmp_path / "qwen_image_layered_generated.png" + golden_path = tmp_path / "qwen_image_layered_golden_grid.png" + generated_lpips_path = tmp_path / "qwen_image_layered_generated_lpips.png" + golden_lpips_path = tmp_path / "qwen_image_layered_golden_grid_lpips.png" + _copy_qwen_image_layered_lpips_input(tmp_path, input_path) + _write_qwen_image_layered_lpips_golden_grid(tmp_path, golden_path) + _generate_qwen_image_layered_lpips_image( + _lpips_model_path(QWEN_IMAGE_LAYERED_MODEL_SUBPATH), + input_path, + generated_path, + ) + # Ignore invisible RGB values under transparent pixels while preserving + # partially transparent layer edges. + _flatten_qwen_image_layered_lpips_image(generated_path, generated_lpips_path) + _flatten_qwen_image_layered_lpips_image(golden_path, golden_lpips_path) + score = _run_lpips_eval( + tmp_path, + "qwen_image_layered", + "image", + QWEN_IMAGE_LAYERED_LPIPS_PROMPT, + golden_lpips_path, + generated_lpips_path, + ) + _assert_lpips_below_threshold(score, QWEN_IMAGE_LAYERED_LPIPS_THRESHOLD) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") def test_cosmos3_nano_t2v_lpips_against_golden(_visual_gen_deps, tmp_path): generated_path = tmp_path / "cosmos3_nano_t2v_generated.mp4" @@ -1128,7 +1328,7 @@ def _generate_wan_video(llm_venv, model_subpath, output_subdir): """ from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_path = os.path.join(scratch_space, model_subpath) if not os.path.isdir(model_path): pytest.skip( @@ -1201,7 +1401,7 @@ def _generate_ltx2_two_stage_video(llm_venv, output_subdir, linear_type="default """ from tensorrt_llm import VisualGen, VisualGenArgs, VisualGenParams - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_path = os.path.join(scratch_space, LTX2_MODEL_CHECKPOINT_PATH) text_encoder_path = os.path.join(scratch_space, LTX2_TEXT_ENCODER_SUBPATH) upsampler_path = os.path.join(scratch_space, LTX2_UPSAMPLER_SUBPATH) @@ -1361,7 +1561,7 @@ def _run_vbench_and_report( "custom_input", ] cmd.extend(["--dimension"] + VBENCH_DIMENSIONS) - venv_check_call(llm_venv, cmd) + _venv_check_call(llm_venv, cmd) pattern = os.path.join(output_path, "*_eval_results.json") result_files = glob.glob(pattern) @@ -1500,7 +1700,7 @@ def test_vbench_dimension_score_ltx2_two_stage_fp8( def test_visual_gen_quickstart(_visual_gen_deps, llm_root, llm_venv): """Run examples/visual_gen/quickstart_example.py end-to-end.""" - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_src = os.path.join(scratch_space, WAN_T2V_MODEL_SUBPATH) if not os.path.isdir(model_src): pytest.skip( @@ -1514,7 +1714,7 @@ def test_visual_gen_quickstart(_visual_gen_deps, llm_root, llm_venv): os.symlink(model_src, model_dst, target_is_directory=True) script_path = os.path.join(llm_root, "examples", "visual_gen", "quickstart_example.py") - venv_check_call(llm_venv, [script_path]) + _venv_check_call(llm_venv, [script_path]) output_path = os.path.join(llm_venv.get_working_directory(), "output.avi") assert os.path.isfile(output_path), f"Quickstart did not produce output.avi at {output_path}" @@ -1522,7 +1722,7 @@ def test_visual_gen_quickstart(_visual_gen_deps, llm_root, llm_venv): def test_visual_gen_api_walkthrough(_visual_gen_deps, llm_root, llm_venv): """Run examples/visual_gen/api_walkthrough.py end-to-end.""" - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_src = os.path.join(scratch_space, WAN_T2V_MODEL_SUBPATH) if not os.path.isdir(model_src): pytest.skip( @@ -1536,7 +1736,7 @@ def test_visual_gen_api_walkthrough(_visual_gen_deps, llm_root, llm_venv): os.symlink(model_src, model_dst, target_is_directory=True) script_path = os.path.join(llm_root, "examples", "visual_gen", "api_walkthrough.py") - venv_check_call(llm_venv, [script_path]) + _venv_check_call(llm_venv, [script_path]) output_path = os.path.join(llm_venv.get_working_directory(), "api_walkthrough_output.avi") assert os.path.isfile(output_path), f"API walkthrough did not produce {output_path}" @@ -1559,7 +1759,7 @@ def test_wan_t2v_example(_visual_gen_deps, llm_root, llm_venv): which runs the same script but with a no-quant YAML synthesized at runtime and additionally evaluates VBench scores. """ - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_path = os.path.join(scratch_space, WAN22_A14B_NVFP4_MODEL_SUBPATH) assert os.path.isdir(model_path), ( f"Model not found: {model_path} " @@ -1577,7 +1777,7 @@ def test_wan_t2v_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1613,7 +1813,7 @@ def test_flux1_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1649,7 +1849,7 @@ def test_flux2_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1664,6 +1864,47 @@ def test_flux2_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(output_path), f"Example did not produce output at {output_path}" +def test_flux2_reference_image_example(_visual_gen_deps, llm_root, llm_venv, tmp_path): + """Run the FLUX.2 example with the existing reference-image request argument.""" + model_path = _lpips_model_path("FLUX.2-dev") + _skip_if_missing(model_path, "FLUX.2-dev checkpoint", is_dir=True) + reference_path = _golden_media_path( + tmp_path, "flux2_lpips_golden.png", "FLUX.2 reference image" + ) + + out_dir = os.path.join( + llm_venv.get_working_directory(), "visual_gen_output", "flux2_reference_image_example" + ) + os.makedirs(out_dir, exist_ok=True) + output_path = os.path.join(out_dir, "flux2_reference_image_output.png") + script_path = os.path.join(llm_root, "examples", "visual_gen", "models", "flux2.py") + config_path = os.path.join( + llm_root, "examples", "visual_gen", "configs", "flux2-dev-fp4-1gpu.yaml" + ) + + _venv_check_call( + llm_venv, + [ + script_path, + "--model", + model_path, + "--visual_gen_args", + config_path, + "--image", + str(reference_path), + "--height", + "256", + "--width", + "256", + "--num_inference_steps", + "4", + "--output_path", + output_path, + ], + ) + assert os.path.isfile(output_path), f"Example did not produce output at {output_path}" + + def test_ltx2_example(_visual_gen_deps, llm_root, llm_venv): """Run examples/visual_gen/models/ltx2.py with NVFP4 config end-to-end. @@ -1688,7 +1929,7 @@ def test_ltx2_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1712,7 +1953,7 @@ def test_wan_i2v_example(_visual_gen_deps, llm_root, llm_venv): work together as documented. Uses the pre-quantized Wan 2.2 I2V A14B NVFP4 checkpoint and the default input image (cat_piano.png) bundled with the examples. """ - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_path = os.path.join(scratch_space, WAN22_I2V_A14B_NVFP4_MODEL_SUBPATH) if not os.path.isdir(model_path): pytest.skip( @@ -1731,7 +1972,7 @@ def test_wan_i2v_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1753,7 +1994,7 @@ def test_qwen_image_example(_visual_gen_deps, llm_root, llm_venv): ``configs/qwen-image-fp8-1gpu.yaml`` work together as documented. Uses the local Qwen-Image checkpoint and the shared FP8 blockwise dynamic-quant config. """ - scratch_space = conftest.llm_models_root() + scratch_space = _llm_models_root() model_path = os.path.join(scratch_space, QWEN_IMAGE_MODEL_SUBPATH) _skip_if_missing(model_path, "Qwen-Image checkpoint", is_dir=True) model_index_path = os.path.join(model_path, "model_index.json") @@ -1775,7 +2016,52 @@ def test_qwen_image_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( + llm_venv, + [ + script_path, + "--model", + model_path, + "--visual_gen_args", + config_path, + "--output_path", + output_path, + ], + ) + assert os.path.isfile(output_path), f"Example did not produce output at {output_path}" + + +def test_qwen_image_layered_example(_visual_gen_deps, tmp_path, llm_root, llm_venv): + """Run examples/visual_gen/models/qwen_image_layered.py end-to-end.""" + scratch_space = _llm_models_root() + model_path = os.path.join(scratch_space, QWEN_IMAGE_LAYERED_MODEL_SUBPATH) + _skip_if_missing(model_path, "Qwen-Image-Layered checkpoint", is_dir=True) + model_index_path = os.path.join(model_path, "model_index.json") + if not os.path.isfile(model_index_path): + pytest.skip( + f"Qwen-Image-Layered checkpoint is incomplete: {model_path} " + f"(missing {model_index_path})" + ) + + input_path = tmp_path / "qwen_image_layered_input.png" + _copy_qwen_image_layered_lpips_input(tmp_path, input_path) + + out_dir = os.path.join( + llm_venv.get_working_directory(), "visual_gen_output", "qwen_image_layered_example" + ) + os.makedirs(out_dir, exist_ok=True) + output_path = os.path.join(out_dir, "qwen_image_layered_output.png") + + script_path = os.path.join( + llm_root, "examples", "visual_gen", "models", "qwen_image_layered.py" + ) + assert os.path.isfile(script_path), f"Example script not found: {script_path}" + config_path = os.path.join( + llm_root, "examples", "visual_gen", "configs", "qwen-image-layered-1gpu.yaml" + ) + assert os.path.isfile(config_path), f"Config not found: {config_path}" + + _venv_check_call( llm_venv, [ script_path, @@ -1783,6 +2069,61 @@ def test_qwen_image_example(_visual_gen_deps, llm_root, llm_venv): model_path, "--visual_gen_args", config_path, + "--image", + str(input_path), + "--prompt", + QWEN_IMAGE_LAYERED_LPIPS_PROMPT, + "--output_path", + output_path, + ], + ) + assert os.path.isfile(output_path), f"Example did not produce output at {output_path}" + + +def test_qwen_image_edit_example(_visual_gen_deps: Any, llm_root: str, llm_venv: Any) -> None: + """Run examples/visual_gen/models/qwen_image_edit.py end-to-end. + + Validates that the Qwen-Image-Edit example script and + ``configs/qwen-image-edit-2511-fp8-1gpu.yaml`` work together as documented. + """ + model_path = os.environ.get("QWEN_IMAGE_EDIT_MODEL_PATH") or os.path.join( + _llm_models_root(), QWEN_IMAGE_EDIT_MODEL_SUBPATH + ) + _skip_if_missing(model_path, "Qwen-Image-Edit-2511 checkpoint", is_dir=True) + model_index_path = os.path.join(model_path, "model_index.json") + if not os.path.isfile(model_index_path): + pytest.skip( + f"Qwen-Image-Edit-2511 checkpoint is incomplete: {model_path} " + f"(missing {model_index_path})" + ) + + out_dir = os.path.join( + llm_venv.get_working_directory(), "visual_gen_output", "qwen_image_edit_example" + ) + os.makedirs(out_dir, exist_ok=True) + output_path = os.path.join(out_dir, "qwen_image_edit_output.png") + + script_path = os.path.join(llm_root, "examples", "visual_gen", "models", "qwen_image_edit.py") + config_path = os.path.join( + llm_root, "examples", "visual_gen", "configs", "qwen-image-edit-2511-fp8-1gpu.yaml" + ) + image_path = os.path.join(llm_root, "examples", "visual_gen", "cat_piano.png") + assert os.path.isfile(script_path), f"Example script not found: {script_path}" + assert os.path.isfile(config_path), f"Config not found: {config_path}" + assert os.path.isfile(image_path), f"Input image not found: {image_path}" + + _venv_check_call( + llm_venv, + [ + script_path, + "--model", + model_path, + "--visual_gen_args", + config_path, + "--image", + image_path, + "--prompt", + "Add a small red wizard hat to the cat while preserving the source image.", "--output_path", output_path, ], @@ -1813,7 +2154,7 @@ def test_cosmos3_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.isfile(script_path), f"Example script not found: {script_path}" assert os.path.isfile(config_path), f"Config not found: {config_path}" - venv_check_call( + _venv_check_call( llm_venv, [ script_path, @@ -1829,3 +2170,221 @@ def test_cosmos3_example(_visual_gen_deps, llm_root, llm_venv): env={"TRTLLM_DISABLE_COSMOS3_GUARDRAILS": "1"}, ) assert os.path.isfile(output_path), f"Example did not produce output at {output_path}" + + +def test_cosmos3_t2i_4step_example(_visual_gen_deps, llm_root, llm_venv): + """Run the distilled T2I checkpoint through the recommended invocation. + + Validates the documented deployment for ``Cosmos3-Super-Text2Image-4Step``: + the example script with ``configs/cosmos3-t2i-1gpu.yaml`` (T2I warmup + shapes) and ``--output_type image``. Steps/guidance come from the + checkpoint's fixed distilled schedule; the run must produce an image. + """ + model_path = _lpips_model_path("Cosmos3-Super-Text2Image-4Step") + _skip_if_missing(model_path, "Cosmos3-Super-Text2Image-4Step checkpoint", is_dir=True) + + out_dir = os.path.join( + llm_venv.get_working_directory(), "visual_gen_output", "cosmos3_t2i_4step_example" + ) + os.makedirs(out_dir, exist_ok=True) + output_path = os.path.join(out_dir, "cosmos3_t2i_4step_output.png") + if os.path.exists(output_path): + os.remove(output_path) + + script_path = os.path.join( + llm_root, "examples", "visual_gen", "models", "cosmos3", "cosmos3.py" + ) + config_path = os.path.join( + llm_root, "examples", "visual_gen", "configs", "cosmos3-t2i-1gpu.yaml" + ) + assert os.path.isfile(script_path), f"Example script not found: {script_path}" + assert os.path.isfile(config_path), f"Config not found: {config_path}" + + _venv_check_call( + llm_venv, + [ + script_path, + "--model", + model_path, + "--visual_gen_args", + config_path, + "--prompt", + "A ceramic teapot pouring steaming tea into a cup, morning window light", + "--output_type", + "image", + "--output_path", + output_path, + ], + env={"TRTLLM_DISABLE_COSMOS3_GUARDRAILS": "1"}, + ) + assert os.path.isfile(output_path), f"Example did not produce output at {output_path}" + assert os.path.getsize(output_path) > 0, f"Example produced an empty image at {output_path}" + + +def _write_cosmos3_i2v_conditioning_image(path): + """Deterministic 1280x720 conditioning image for the I2V smoke test. + + Gradient sky plus simple shapes, so I2V has real structure to animate + without shipping an asset file. + """ + from PIL import Image, ImageDraw + + image = Image.new("RGB", (1280, 720)) + draw = ImageDraw.Draw(image) + for y in range(720): + draw.line([(0, y), (1280, y)], fill=(30, 60 + y // 8, 140)) + draw.ellipse([480, 200, 800, 520], fill=(230, 120, 40), outline=(255, 255, 255), width=6) + draw.rectangle([100, 500, 400, 680], fill=(40, 160, 90)) + draw.polygon([(1000, 600), (1120, 380), (1240, 600)], fill=(200, 200, 60)) + image.save(path) + + +def test_cosmos3_i2v_4step_example(_visual_gen_deps, llm_root, llm_venv): + """Run the distilled I2V checkpoint through the recommended invocation. + + Validates the documented deployment for ``Cosmos3-Super-Image2Video-4Step``: + the example script with a conditioning image and no config override (the + omni defaults — 720p x 189 frames — are the deployed shape). Steps, + guidance, and the system-prompt default come from the checkpoint; the run + must produce a video. + """ + model_path = _lpips_model_path("Cosmos3-Super-Image2Video-4Step") + _skip_if_missing(model_path, "Cosmos3-Super-Image2Video-4Step checkpoint", is_dir=True) + + out_dir = os.path.join( + llm_venv.get_working_directory(), "visual_gen_output", "cosmos3_i2v_4step_example" + ) + os.makedirs(out_dir, exist_ok=True) + image_path = os.path.join(out_dir, "conditioning.png") + _write_cosmos3_i2v_conditioning_image(image_path) + output_path = os.path.join(out_dir, "cosmos3_i2v_4step_output.mp4") + if os.path.exists(output_path): + os.remove(output_path) + + script_path = os.path.join( + llm_root, "examples", "visual_gen", "models", "cosmos3", "cosmos3.py" + ) + assert os.path.isfile(script_path), f"Example script not found: {script_path}" + + _venv_check_call( + llm_venv, + [ + script_path, + "--model", + model_path, + "--prompt", + "The orange sphere slowly rises while the camera pans right across the scene", + "--image_path", + image_path, + "--output_path", + output_path, + ], + env={"TRTLLM_DISABLE_COSMOS3_GUARDRAILS": "1"}, + ) + assert os.path.isfile(output_path), f"Example did not produce output at {output_path}" + assert os.path.getsize(output_path) > 0, f"Example produced an empty video at {output_path}" + + +def _run_cosmos3_i2v_4step_lpips_pipeline(image_path): + """Run the distilled I2V pipeline on the deterministic conditioning image. + + VANILLA attention, compile-off. Returns the generated video tensor + ``(B, T, H, W, C)``, or ``None`` if generation produced no video. + """ + # Cosmos3 re-reads the guardrail flag in __init__; set it before the pipeline loads. + guardrails_env_key = "TRTLLM_DISABLE_COSMOS3_GUARDRAILS" + previous_guardrails_env = os.environ.get(guardrails_env_key) + os.environ[guardrails_env_key] = "1" + try: + from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader + from tensorrt_llm.visual_gen.args import ( + AttentionConfig, + CompilationConfig, + TorchCompileConfig, + VisualGenArgs, + ) + + model_path = _lpips_model_path(COSMOS3_I2V_4STEP_MODEL_SUBPATH) + _skip_if_missing(model_path, "Cosmos3-Super-Image2Video-4Step checkpoint", is_dir=True) + _disable_inductor_compile_worker_quiesce() + args = VisualGenArgs( + model=model_path, + compilation_config=CompilationConfig(skip_warmup=True), + torch_compile_config=TorchCompileConfig(enable=False), + attention_config=AttentionConfig(backend="VANILLA"), + ) + pipeline = PipelineLoader(args).load(skip_warmup=True) + try: + with torch.no_grad(): + result = pipeline.forward( + prompt=COSMOS3_I2V_4STEP_LPIPS_PROMPT, + seed=COSMOS3_LPIPS_SEED, + image=image_path, + height=COSMOS3_LPIPS_HEIGHT, + width=COSMOS3_LPIPS_WIDTH, + num_frames=COSMOS3_I2V_4STEP_LPIPS_NUM_FRAMES, + # Direct forward() calls must pass checkpoint-valid sampling + # values (the signature defaults are the base-checkpoint + # video table, which a distilled checkpoint rejects). + num_inference_steps=COSMOS3_I2V_4STEP_LPIPS_NUM_INFERENCE_STEPS, + guidance_scale=COSMOS3_I2V_4STEP_LPIPS_GUIDANCE_SCALE, + frame_rate=COSMOS3_LPIPS_FRAME_RATE, + # The checkpoint declares default_use_system_prompt=true and + # the golden was generated with it; forward()'s signature + # default is the historical False, so pass it explicitly. + use_system_prompt=True, + use_guardrails=False, + ) + if result is None or result.video is None: + return None + return result.video.detach().cpu() + finally: + del pipeline + _cleanup_cuda() + finally: + if previous_guardrails_env is None: + os.environ.pop(guardrails_env_key, None) + else: + os.environ[guardrails_env_key] = previous_guardrails_env + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +def test_cosmos3_i2v_4step_lpips_against_golden(_visual_gen_deps, request, tmp_path): + """Quality gate for the distilled I2V checkpoint against a diffusers golden. + + Unlike the self-goldens of the other models, the golden video here was + produced by the reference implementation (diffusers modular pipeline, + PR #14177, with its per-step SDE noise made generator-seeded) — so this + gate checks the denoising trajectory against the reference, not just + regression against a past TRT-LLM run. Full provenance: + ``golden/visual_gen_lpips/cosmos3_i2v_4step_lpips_golden_video.json``. + """ + image_path = str(tmp_path / "cosmos3_i2v_4step_conditioning.png") + _write_cosmos3_i2v_conditioning_image(image_path) + generated_path = tmp_path / "cosmos3_i2v_4step_generated.mp4" + golden_path = _golden_media_path( + tmp_path, + "cosmos3_i2v_4step_lpips_golden_video.mp4", + "Cosmos3 I2V-4Step LPIPS golden video", + ) + + video = _run_cosmos3_i2v_4step_lpips_pipeline(image_path) + assert video is not None, "Cosmos3 I2V-4Step LPIPS run produced no video" + _save_lpips_video_mp4(video, generated_path, frame_rate=COSMOS3_LPIPS_FRAME_RATE) + + score = _run_lpips_eval( + tmp_path, + "cosmos3_i2v_4step", + "video", + COSMOS3_I2V_4STEP_LPIPS_PROMPT, + golden_path, + generated_path, + ) + _preserve_lpips_candidate_on_failure( + request, + score, + COSMOS3_I2V_4STEP_LPIPS_THRESHOLD, + generated_path, + "cosmos3_i2v_4step_lpips_golden_video.mp4", + ) + _assert_lpips_below_threshold(score, COSMOS3_I2V_4STEP_LPIPS_THRESHOLD) diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen_multi_gpu.py b/tests/integration/defs/examples/visual_gen/test_visual_gen_multi_gpu.py index a60765fe600c..0b90db65a22c 100644 --- a/tests/integration/defs/examples/visual_gen/test_visual_gen_multi_gpu.py +++ b/tests/integration/defs/examples/visual_gen/test_visual_gen_multi_gpu.py @@ -14,7 +14,9 @@ # limitations under the License. """Multi-GPU integration tests for VisualGen LPIPS quality checks.""" +import glob import os +import sys from typing import Callable import pytest @@ -39,16 +41,21 @@ _save_lpips_video_mp4, ) -try: - from tensorrt_llm._utils import get_free_port + +def _parallel_config(**kwargs): + # Imported lazily so that mp.spawn child processes resolve tensorrt_llm only after + # _distributed_worker has prepended the installed-wheel location to sys.path. A + # module-level tensorrt_llm import would run during the child's module re-import + # (before sys.path is fixed) and resolve to the bindings-less source tree. from tensorrt_llm.visual_gen.args import ParallelConfig - MODULES_AVAILABLE = True -except ImportError: - MODULES_AVAILABLE = False + return ParallelConfig(**kwargs) + # Keep it as 0.25 as the worst case scenario at NVL72 scale WAN_MULTI_GPU_LPIPS_THRESHOLD = 0.25 +WAN22_MULTI_GPU_LPIPS_ATTENTION_BACKEND = "FA4" +WAN22_MULTI_GPU_LPIPS_GOLDEN_VIDEO = "wan22_t2v_fa4_fully_eager_lpips_golden_video.mp4" WAN22_LPIPS_MULTI_GPU_VARIANTS = [ ("ulysses4", {"ulysses_size": 4}), ("cfg2_ulysses2", {"cfg_size": 2, "ulysses_size": 2}), @@ -59,8 +66,10 @@ WAN22_LPIPS_TP_VARIANTS = [ ("tp2", {"tp_size": 2}), + ("tp3", {"tp_size": 3}), ("cfg2_tp2", {"cfg_size": 2, "tp_size": 2}), ("tp2_ulysses2", {"tp_size": 2, "ulysses_size": 2}), + ("tp2_attn2d_2x1", {"tp_size": 2, "attn2d_size": (2, 1)}), ] @@ -90,7 +99,37 @@ def cleanup_distributed(): dist.destroy_process_group() -def _distributed_worker(rank, world_size, backend, test_fn, port, kwargs): +def _validated_tllm_site(site_dir): + """Return the realpath of ``site_dir`` after verifying it holds the installed wheel. + + The spawn workers rely on this directory to import tensorrt_llm with compiled + bindings; accepting an arbitrary path would let the import silently fall through + to the bindings-less source tree, so reject anything that does not contain the + package plus its compiled bindings extension. + """ + resolved = os.path.realpath(site_dir) if site_dir else "" + package_init = os.path.join(resolved, "tensorrt_llm", "__init__.py") + bindings = glob.glob(os.path.join(resolved, "tensorrt_llm", "bindings*.so")) + glob.glob( + os.path.join(resolved, "tensorrt_llm", "bindings", "*.so") + ) + if not (resolved and os.path.isfile(package_init) and bindings): + raise RuntimeError( + f"tllm_site={site_dir!r} does not contain an installed tensorrt_llm package " + "with compiled bindings; spawn workers would import the bindings-less " + "source tree instead of the wheel." + ) + return resolved + + +def _distributed_worker(rank, world_size, backend, test_fn, port, kwargs, tllm_site): + # mp.spawn starts a fresh interpreter whose sys.path (set up by the integration + # `defs` harness) puts the source checkout ahead of the installed wheel, so a bare + # `import tensorrt_llm` would resolve to the bindings-less source tree and crash the + # worker. Prepend the parent's installed-package location so the child imports + # tensorrt_llm (with compiled bindings) from the wheel before any such import. + tllm_site = _validated_tllm_site(tllm_site) + sys.path[:] = [path for path in sys.path if os.path.realpath(path) != tllm_site] + sys.path.insert(0, tllm_site) try: init_distributed_worker(rank, world_size, backend, port) test_fn(rank, world_size, **kwargs) @@ -102,22 +141,32 @@ def _distributed_worker(rank, world_size, backend, test_fn, port, kwargs): def run_test_in_distributed(world_size: int, test_fn: Callable, use_cuda: bool = True, **kwargs): - if not MODULES_AVAILABLE: + try: + import tensorrt_llm.bindings as tllm_bindings + from tensorrt_llm._utils import get_free_port + except ImportError: pytest.skip("Required modules not available") if use_cuda and torch.cuda.device_count() < world_size: pytest.skip(f"Test requires {world_size} GPUs, only {torch.cuda.device_count()} available") backend = "nccl" if use_cuda else "gloo" port = get_free_port() + # Directory containing the installed tensorrt_llm package (i.e. site-packages), + # passed to spawn workers so they prepend it to sys.path and import the wheel with + # compiled bindings instead of the source-tree package. Validated here as well so a + # bad environment fails before any worker is spawned. + tllm_site = _validated_tllm_site( + os.path.dirname(os.path.dirname(os.path.abspath(tllm_bindings.__file__))) + ) mp.spawn( _distributed_worker, - args=(world_size, backend, test_fn, port, kwargs), + args=(world_size, backend, test_fn, port, kwargs, tllm_site), nprocs=world_size, join=True, ) def _skip_if_insufficient_gpus_for_parallel(parallel): - parallel_cfg = ParallelConfig(**parallel) + parallel_cfg = _parallel_config(**parallel) required = parallel_cfg.n_workers available = torch.cuda.device_count() if available < required: @@ -128,7 +177,7 @@ def _skip_if_insufficient_gpus_for_parallel(parallel): def _wan22_lpips_distributed_worker(rank: int, world_size: int, **kwargs) -> None: parallel = kwargs["parallel"] - ParallelConfig(**parallel).validate_world_size(world_size) + _parallel_config(**parallel).validate_world_size(world_size) generated_video = _run_wan_lpips_pipeline( kwargs["model_path"], @@ -140,8 +189,9 @@ def _wan22_lpips_distributed_worker(rank: int, world_size: int, **kwargs) -> Non kwargs["num_inference_steps"], kwargs["guidance_scale"], kwargs["seed"], - attention_backend="FA4", + attention_backend=WAN22_MULTI_GPU_LPIPS_ATTENTION_BACKEND, parallel=parallel, + fully_eager=True, ) if rank == 0: @@ -160,10 +210,12 @@ def _wan22_lpips_distributed_worker(rank: int, world_size: int, **kwargs) -> Non def _run_wan22_t2v_lpips_case(tmp_path, variant_name, parallel): _skip_if_insufficient_gpus_for_parallel(parallel) - parallel_cfg = ParallelConfig(**parallel) + parallel_cfg = _parallel_config(**parallel) generated_path = tmp_path / f"wan22_t2v_generated_{variant_name}.mp4" golden_path = _golden_media_path( - tmp_path, "wan22_t2v_lpips_golden_video.mp4", "Wan 2.2 LPIPS golden video" + tmp_path, + WAN22_MULTI_GPU_LPIPS_GOLDEN_VIDEO, + "Wan 2.2 FA4 fully-eager LPIPS golden video", ) run_test_in_distributed( @@ -200,7 +252,9 @@ def _run_wan22_t2v_lpips_case(tmp_path, variant_name, parallel): WAN22_LPIPS_MULTI_GPU_VARIANTS, ids=[name for name, _ in WAN22_LPIPS_MULTI_GPU_VARIANTS], ) -def test_wan22_t2v_lpips_against_golden_multi_gpu(tmp_path, variant_name, parallel): +def test_wan22_t2v_lpips_against_golden_multi_gpu( + _visual_gen_deps, tmp_path, variant_name, parallel +): _run_wan22_t2v_lpips_case(tmp_path, variant_name, parallel) @@ -209,5 +263,5 @@ def test_wan22_t2v_lpips_against_golden_multi_gpu(tmp_path, variant_name, parall WAN22_LPIPS_TP_VARIANTS, ids=[name for name, _ in WAN22_LPIPS_TP_VARIANTS], ) -def test_wan22_t2v_lpips_against_golden_tp(tmp_path, variant_name, parallel): +def test_wan22_t2v_lpips_against_golden_tp(_visual_gen_deps, tmp_path, variant_name, parallel): _run_wan22_t2v_lpips_case(tmp_path, variant_name, parallel) diff --git a/tests/integration/defs/llmapi/test_llm_api_pytorch_bart.py b/tests/integration/defs/llmapi/test_llm_api_pytorch_bart.py index da8c6d552aa9..6f94d06b6efa 100644 --- a/tests/integration/defs/llmapi/test_llm_api_pytorch_bart.py +++ b/tests/integration/defs/llmapi/test_llm_api_pytorch_bart.py @@ -41,27 +41,44 @@ ), ] _MODEL_NAME = "bart-large-cnn" +_MBART_MODEL_NAME = "mbart-large-50-many-to-one-mmt" +_MBART_SOURCE_LANG = "ro_RO" +_MBART_TARGET_LANG = "en_XX" +_MBART_SOURCE_TEXT = "Şeful ONU spune că nu există o soluţie militară în Siria." _MAX_NEW_TOKENS = 10 _MAX_SEQUENCE_LENGTH = 128 _MAX_KV_TOKENS = 384 _MIN_GPU_MEMORY_MB = 16_000 _FREE_GPU_MEMORY_FRACTION = 0.2 _CROSS_KV_CACHE_FRACTION = 0.5 -# "The update improves batching, lowers latency" -_EXPECTED_GREEDY_OUTPUT_TOKEN_IDS = [0, 133, 2935, 15296, 14398, 154, 6, 32222, 35940, 2] +# "The update improves batching, lowers latency," +_EXPECTED_GREEDY_OUTPUT_TOKEN_IDS = [0, 133, 2935, 15296, 14398, 154, 6, 32222, 35940, 6] +_MBART_EXPECTED_GREEDY_OUTPUT_TOKEN_IDS = [ + # "The UN chief says there is no military solution" + 250004, + 581, + 8274, + 185256, + 17378, + 2685, + 83, + 110, + 116338, + 29806, +] _EXPECTED_BEAM_OUTPUT_TOKEN_IDS_BY_BEAMS = { 2: [ + # "The update improves batching, lowers latency," + [0, 133, 2935, 15296, 14398, 154, 6, 32222, 35940, 6], # "The update improves batching, lowers latency" - [0, 133, 2935, 15296, 14398, 154, 6, 32222, 35940, 2], - # "The update improves batching, lowers" - [0, 0, 133, 2935, 15296, 14398, 154, 6, 32222, 2], + [0, 0, 133, 2935, 15296, 14398, 154, 6, 32222, 35940], ], } _MIXED_ENCODER_EXPECTED_TOKEN_IDS_BY_REQUEST = [ - # "The update improves batching, lowers latency" - [[0, 133, 2935, 15296, 14398, 154, 6, 32222, 35940, 2]], - # "The company opened a training center on Monday" - [[0, 133, 138, 1357, 10, 1058, 1312, 15, 302, 2]], + # "The update improves batching, lowers latency," + [[0, 133, 2935, 15296, 14398, 154, 6, 32222, 35940, 6]], + # "The company opened a training center on Monday." + [[0, 133, 138, 1357, 10, 1058, 1312, 15, 302, 4]], ] @@ -75,6 +92,7 @@ def _test_case( feature_id: str, cuda_graph_batch_sizes: list[int] | None = None, kv_cache_dtype: str = "auto", + disable_overlap_scheduler: bool = True, tensor_parallel_size: int = 1, marks=None, ): @@ -98,12 +116,15 @@ def _test_case( exact_match, cuda_graph_batch_sizes, kv_cache_dtype, + disable_overlap_scheduler, tensor_parallel_size, **param_kwargs, ) _TEST_CASES = [ + # Eager (non-CUDA-graph) greedy smoke. All remaining cases run with CUDA + # graphs enabled because deployments almost always enable them. _test_case( torch_dtype="bfloat16", use_kv_cache_manager_v2=False, @@ -126,38 +147,43 @@ def _test_case( _test_case( torch_dtype="float16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=1, num_return_sequences=1, exact_match=True, - feature_id="fp16-kv-v1-cuda-graph-off-greedy", + cuda_graph_batch_sizes=[2], + feature_id="fp16-kv-v1-cuda-graph-on-greedy", ), _test_case( torch_dtype="bfloat16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=True, - feature_id="bf16-kv-v1-cuda-graph-off-beam2", + feature_id="bf16-kv-v1-cuda-graph-on-beam2", ), _test_case( torch_dtype="bfloat16", - use_kv_cache_manager_v2=False, + use_kv_cache_manager_v2=True, enable_cuda_graph=True, - num_beams=2, - num_return_sequences=2, + num_beams=1, + num_return_sequences=1, exact_match=True, - feature_id="bf16-kv-v1-cuda-graph-on-beam2", + feature_id="bf16-kv-v2-cuda-graph-on-greedy", ), + # Overlap-scheduler cases: outputs must be identical to the non-overlap + # runs above. _test_case( torch_dtype="bfloat16", - use_kv_cache_manager_v2=True, - enable_cuda_graph=False, + use_kv_cache_manager_v2=False, + enable_cuda_graph=True, num_beams=1, num_return_sequences=1, exact_match=True, - feature_id="bf16-kv-v2-cuda-graph-off-greedy", + cuda_graph_batch_sizes=[2], + disable_overlap_scheduler=False, + feature_id="bf16-kv-v1-cuda-graph-on-greedy-overlap", ), _test_case( torch_dtype="bfloat16", @@ -166,20 +192,21 @@ def _test_case( num_beams=1, num_return_sequences=1, exact_match=True, - feature_id="bf16-kv-v2-cuda-graph-on-greedy", + cuda_graph_batch_sizes=[2], + disable_overlap_scheduler=False, + feature_id="bf16-kv-v2-cuda-graph-on-greedy-overlap", ), - # Tensor parallelism (TP=2) coverage _test_case( torch_dtype="bfloat16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, - num_beams=1, - num_return_sequences=1, + enable_cuda_graph=True, + num_beams=2, + num_return_sequences=2, exact_match=True, - tensor_parallel_size=2, - feature_id="bf16-kv-v1-cuda-graph-off-greedy-tp2", - marks=pytest.mark.skip_less_device(2), + disable_overlap_scheduler=False, + feature_id="bf16-kv-v1-cuda-graph-on-beam2-overlap", ), + # Tensor parallelism (TP=2) coverage _test_case( torch_dtype="bfloat16", use_kv_cache_manager_v2=False, @@ -235,15 +262,15 @@ def _mixed_batch_test_case( ] -def _get_bart_model_path() -> str: +def _get_model_path(model_name: str) -> str: try: models_root = Path(llm_models_root()) except AssertionError as exc: pytest.skip(str(exc)) - model_path = models_root / _MODEL_NAME + model_path = models_root / model_name if not model_path.exists(): - pytest.skip(f"{_MODEL_NAME} is not available under {models_root}") + pytest.skip(f"{model_name} is not available under {models_root}") return str(model_path) @@ -276,34 +303,6 @@ def _decoder_cuda_graph_config( ) -def _assert_decoder_cuda_graph_state( - llm: LLM, - enabled: bool, - batch_sizes: list[int] | None, -) -> None: - model_engine = llm._executor.engine.model_engine - - if not enabled: - assert not model_engine.encoder_cuda_graph_runner.enabled - assert not model_engine.cuda_graph_runner.enabled - assert not model_engine.encoder_cuda_graph_runner.graphs - assert not model_engine.cuda_graph_runner.graphs - return - - _assert_decoder_cuda_graphs_captured(llm) - if batch_sizes is not None: - assert model_engine.cuda_graph_runner.padding_dummy_requests - - -def _assert_decoder_cuda_graphs_captured(llm: LLM) -> None: - model_engine = llm._executor.engine.model_engine - - assert not model_engine.encoder_cuda_graph_runner.enabled - assert not model_engine.encoder_cuda_graph_runner.graphs - assert model_engine.cuda_graph_runner.enabled - assert model_engine.cuda_graph_runner.graphs - - def _assert_bart_response( response: RequestOutput, num_return_sequences: int, @@ -360,18 +359,17 @@ def _run_bart_pytorch_generate_encoder_decoder( exact_match: bool, cuda_graph_batch_sizes: list[int] | None, kv_cache_dtype: str = "auto", + disable_overlap_scheduler: bool = True, tensor_parallel_size: int = 1, ) -> None: - if tensor_parallel_size == 1: - monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") monkeypatch.setenv("TRTLLM_SKIP_KV_CACHE_ESTIMATION", "1") - model_path = _get_bart_model_path() + model_path = _get_model_path(_MODEL_NAME) tokenizer = AutoTokenizer.from_pretrained(model_path) case_id = ( f"model={_MODEL_NAME}, dtype={torch_dtype}, kv_v2={use_kv_cache_manager_v2}, " f"cuda_graph={enable_cuda_graph}, beams={num_beams}, returns={num_return_sequences}, " - f"kv_dtype={kv_cache_dtype}, tp={tensor_parallel_size}" + f"kv_dtype={kv_cache_dtype}, overlap={not disable_overlap_scheduler}, tp={tensor_parallel_size}" ) sampling_params = _sampling_params(num_beams, num_return_sequences) @@ -382,7 +380,7 @@ def _run_bart_pytorch_generate_encoder_decoder( cuda_graph_config=_decoder_cuda_graph_config(cuda_graph_batch_sizes) if enable_cuda_graph else None, - disable_overlap_scheduler=True, + disable_overlap_scheduler=disable_overlap_scheduler, dtype=torch_dtype, enable_chunked_prefill=False, tensor_parallel_size=tensor_parallel_size, @@ -418,22 +416,12 @@ def _run_bart_pytorch_generate_encoder_decoder( exact_match, expected_output_token_ids_by_output, ) - # CUDA graph state introspection reaches into the in-process engine, - # which is only available when the executor runs single-process (TP=1). - # For TP>1 the executor is a multi-process proxy without a local engine, - # so we rely on the generated-output assertions above for correctness. - if tensor_parallel_size == 1: - _assert_decoder_cuda_graph_state( - llm, - enable_cuda_graph, - cuda_graph_batch_sizes, - ) @pytest.mark.parametrize( "expected_output_token_ids_by_output,torch_dtype,use_kv_cache_manager_v2," "enable_cuda_graph,num_beams,num_return_sequences,exact_match,cuda_graph_batch_sizes," - "kv_cache_dtype,tensor_parallel_size", + "kv_cache_dtype,disable_overlap_scheduler,tensor_parallel_size", _TEST_CASES, ) def test_bart_pytorch_generate_encoder_decoder_end_to_end( @@ -447,6 +435,7 @@ def test_bart_pytorch_generate_encoder_decoder_end_to_end( exact_match: bool, cuda_graph_batch_sizes: list[int] | None, kv_cache_dtype: str, + disable_overlap_scheduler: bool, tensor_parallel_size: int, ) -> None: _run_bart_pytorch_generate_encoder_decoder( @@ -460,10 +449,79 @@ def test_bart_pytorch_generate_encoder_decoder_end_to_end( exact_match, cuda_graph_batch_sizes, kv_cache_dtype, + disable_overlap_scheduler, tensor_parallel_size, ) +def test_mbart_pytorch_generate_encoder_decoder_end_to_end( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("TRTLLM_SKIP_KV_CACHE_ESTIMATION", "1") + + model_path = _get_model_path(_MBART_MODEL_NAME) + tokenizer = AutoTokenizer.from_pretrained( + model_path, + src_lang=_MBART_SOURCE_LANG, + ) + source_lang_token_id = tokenizer.lang_code_to_id[_MBART_SOURCE_LANG] + assert tokenizer.encode(_MBART_SOURCE_TEXT)[0] == source_lang_token_id + sampling_params = SamplingParams( + max_tokens=_MAX_NEW_TOKENS, + temperature=0.0, + ) + + with LLM( + model_path, + tokenizer=tokenizer, + backend="pytorch", + attn_backend="TRTLLM", + cuda_graph_config=_decoder_cuda_graph_config(), + disable_overlap_scheduler=True, + dtype="bfloat16", + enable_chunked_prefill=False, + kv_cache_config=KvCacheConfig( + enable_block_reuse=False, + max_tokens=_MAX_KV_TOKENS, + free_gpu_memory_fraction=_FREE_GPU_MEMORY_FRACTION, + cross_kv_cache_fraction=_CROSS_KV_CACHE_FRACTION, + use_kv_cache_manager_v2=False, + ), + max_batch_size=1, + max_beam_width=1, + max_input_len=_MAX_SEQUENCE_LENGTH, + max_num_tokens=_MAX_SEQUENCE_LENGTH, + max_seq_len=_MAX_SEQUENCE_LENGTH, + model_kwargs={"torch_dtype": "bfloat16"}, + scheduler_config=SchedulerConfig(use_python_scheduler=True), + ) as llm: + response = llm.generate( + _MBART_SOURCE_TEXT, + sampling_params=sampling_params, + use_tqdm=False, + ) + token_ids_by_output = _assert_bart_response( + response, + num_return_sequences=1, + ) + _print_generated_text( + tokenizer, + f"model={_MBART_MODEL_NAME}, src_lang={_MBART_SOURCE_LANG}", + "output", + token_ids_by_output, + ) + + token_ids = token_ids_by_output[0] + assert token_ids[0] == tokenizer.lang_code_to_id[_MBART_TARGET_LANG] + assert response.outputs[0].finish_reason == "length" + _assert_expected_generation( + tokenizer, + token_ids_by_output, + exact_match=True, + expected_token_ids_by_output=[_MBART_EXPECTED_GREEDY_OUTPUT_TOKEN_IDS], + ) + + @pytest.mark.parametrize( "torch_dtype,use_kv_cache_manager_v2,num_beams,num_return_sequences", _MIXED_BATCH_TEST_CASES, @@ -475,10 +533,9 @@ def test_bart_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch( num_beams: int, num_return_sequences: int, ) -> None: - monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") monkeypatch.setenv("TRTLLM_SKIP_KV_CACHE_ESTIMATION", "1") - model_path = _get_bart_model_path() + model_path = _get_model_path(_MODEL_NAME) tokenizer = AutoTokenizer.from_pretrained(model_path) sampling_params = _sampling_params(num_beams, num_return_sequences) case_id = ( @@ -536,5 +593,3 @@ def test_bart_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch( request_idx ], ) - - _assert_decoder_cuda_graphs_captured(llm) diff --git a/tests/integration/defs/llmapi/test_llm_api_pytorch_t5.py b/tests/integration/defs/llmapi/test_llm_api_pytorch_t5.py index 1d8a0890bc78..492b88ff3d26 100644 --- a/tests/integration/defs/llmapi/test_llm_api_pytorch_t5.py +++ b/tests/integration/defs/llmapi/test_llm_api_pytorch_t5.py @@ -109,6 +109,7 @@ def _test_case( exact_match: bool, feature_id: str, cuda_graph_batch_sizes: list[int] | None = None, + disable_overlap_scheduler: bool = True, tensor_parallel_size: int = 1, marks=None, ): @@ -145,106 +146,95 @@ def _test_case( num_return_sequences, exact_match, cuda_graph_batch_sizes, + disable_overlap_scheduler, tensor_parallel_size, **param_kwargs, ) _TEST_CASES = [ - # Primary coverage: v1 cache manager and beam search. - _test_case( - model_name="t5-small", - torch_dtype="bfloat16", - use_kv_cache_manager_v2=False, - enable_cuda_graph=False, - num_beams=2, - num_return_sequences=2, - exact_match=False, - feature_id="bf16-kv-v1-cuda-graph-off-beam2", - ), + # Primary coverage: v1 cache manager and beam search across model sizes. + # CUDA graphs are enabled everywhere except one eager greedy smoke case + # because deployments almost always enable them. _test_case( model_name="flan-t5-small", torch_dtype="bfloat16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="bf16-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="bf16-kv-v1-cuda-graph-on-beam2", ), _test_case( model_name="t5-base", torch_dtype="bfloat16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="bf16-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="bf16-kv-v1-cuda-graph-on-beam2", ), _test_case( model_name="t5-large", torch_dtype="bfloat16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="bf16-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="bf16-kv-v1-cuda-graph-on-beam2", ), _test_case( model_name="flan-t5-base", torch_dtype="bfloat16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="bf16-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="bf16-kv-v1-cuda-graph-on-beam2", ), _test_case( model_name="flan-t5-large", torch_dtype="bfloat16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="bf16-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="bf16-kv-v1-cuda-graph-on-beam2", ), _test_case( model_name="flan-t5-xl", torch_dtype="bfloat16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="bf16-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="bf16-kv-v1-cuda-graph-on-beam2", ), _test_case( model_name="flan-t5-xxl", torch_dtype="bfloat16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="bf16-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="bf16-kv-v1-cuda-graph-on-beam2", marks=pytest.mark.skip_less_device_memory(_FLAN_T5_XXL_MIN_GPU_MEMORY_MB), ), - # Non-CUDA-graph smoke for the same v1 beam path. - _test_case( - model_name="t5-small", - torch_dtype="bfloat16", - use_kv_cache_manager_v2=False, - enable_cuda_graph=False, - num_beams=2, - num_return_sequences=2, - exact_match=False, - feature_id="bf16-kv-v1-cuda-graph-off-beam2", - ), - # Greedy smoke for the priority v1 path. + # Eager (non-CUDA-graph) greedy smoke for the priority v1 path. _test_case( model_name="t5-small", torch_dtype="bfloat16", @@ -283,53 +273,47 @@ def _test_case( model_name="t5-small", torch_dtype="float16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="fp16-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="fp16-kv-v1-cuda-graph-on-beam2", ), _test_case( model_name="t5-small", torch_dtype="float32", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="fp32-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="fp32-kv-v1-cuda-graph-on-beam2", ), _test_case( model_name="flan-t5-small", torch_dtype="float16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="fp16-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="fp16-kv-v1-cuda-graph-on-beam2", ), _test_case( model_name="flan-t5-small", torch_dtype="float32", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=2, num_return_sequences=2, exact_match=False, - feature_id="fp32-kv-v1-cuda-graph-off-beam2", + cuda_graph_batch_sizes=[2], + feature_id="fp32-kv-v1-cuda-graph-on-beam2", ), # Precision coverage for v2 on its supported greedy path. - _test_case( - model_name="t5-small", - torch_dtype="bfloat16", - use_kv_cache_manager_v2=True, - enable_cuda_graph=False, - num_beams=1, - num_return_sequences=1, - exact_match=True, - feature_id="bf16-kv-v2-cuda-graph-off-greedy", - ), _test_case( model_name="t5-small", torch_dtype="bfloat16", @@ -345,53 +329,62 @@ def _test_case( model_name="t5-small", torch_dtype="float16", use_kv_cache_manager_v2=True, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=1, num_return_sequences=1, exact_match=True, - feature_id="fp16-kv-v2-cuda-graph-off-greedy", + cuda_graph_batch_sizes=[2], + feature_id="fp16-kv-v2-cuda-graph-on-greedy", ), _test_case( model_name="t5-small", torch_dtype="float32", use_kv_cache_manager_v2=True, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=1, num_return_sequences=1, exact_match=True, - feature_id="fp32-kv-v2-cuda-graph-off-greedy", + cuda_graph_batch_sizes=[2], + feature_id="fp32-kv-v2-cuda-graph-on-greedy", ), _test_case( model_name="flan-t5-small", torch_dtype="bfloat16", use_kv_cache_manager_v2=True, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=1, num_return_sequences=1, exact_match=True, - feature_id="bf16-kv-v2-cuda-graph-off-greedy", + cuda_graph_batch_sizes=[2], + feature_id="bf16-kv-v2-cuda-graph-on-greedy", ), _test_case( model_name="flan-t5-small", torch_dtype="float16", use_kv_cache_manager_v2=True, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=1, num_return_sequences=1, exact_match=True, - feature_id="fp16-kv-v2-cuda-graph-off-greedy", + cuda_graph_batch_sizes=[2], + feature_id="fp16-kv-v2-cuda-graph-on-greedy", ), _test_case( model_name="flan-t5-small", torch_dtype="float32", use_kv_cache_manager_v2=True, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=1, num_return_sequences=1, exact_match=True, - feature_id="fp32-kv-v2-cuda-graph-off-greedy", + cuda_graph_batch_sizes=[2], + feature_id="fp32-kv-v2-cuda-graph-on-greedy", ), - # ByT5 sanity coverage keeps the known-stable expected output path. + # ByT5 sanity coverage keeps the known-stable expected output path. Its + # byte-level greedy logits are numerically borderline (the pinned + # reference already differs on Blackwell even in eager mode), so keep the + # exact eager configuration that the L40S/H100 CI has validated instead + # of re-pinning outputs under CUDA graphs. _test_case( model_name="byt5-small", torch_dtype="bfloat16", @@ -402,19 +395,45 @@ def _test_case( exact_match=True, feature_id="bf16-kv-v2-cuda-graph-off-greedy", ), - # Tensor parallelism (TP=2) coverage + # Overlap-scheduler cases: outputs must be identical to the non-overlap + # runs above. _test_case( model_name="t5-small", torch_dtype="bfloat16", use_kv_cache_manager_v2=False, - enable_cuda_graph=False, + enable_cuda_graph=True, num_beams=1, num_return_sequences=1, exact_match=True, - tensor_parallel_size=2, - feature_id="bf16-kv-v1-cuda-graph-off-greedy-tp2", - marks=pytest.mark.skip_less_device(2), + cuda_graph_batch_sizes=[2], + disable_overlap_scheduler=False, + feature_id="bf16-kv-v1-cuda-graph-on-greedy-overlap", ), + _test_case( + model_name="t5-small", + torch_dtype="bfloat16", + use_kv_cache_manager_v2=False, + enable_cuda_graph=True, + num_beams=2, + num_return_sequences=2, + exact_match=False, + cuda_graph_batch_sizes=[2], + disable_overlap_scheduler=False, + feature_id="bf16-kv-v1-cuda-graph-on-beam2-overlap", + ), + _test_case( + model_name="t5-small", + torch_dtype="bfloat16", + use_kv_cache_manager_v2=True, + enable_cuda_graph=True, + num_beams=1, + num_return_sequences=1, + exact_match=True, + cuda_graph_batch_sizes=[2], + disable_overlap_scheduler=False, + feature_id="bf16-kv-v2-cuda-graph-on-greedy-overlap", + ), + # Tensor parallelism (TP=2) coverage _test_case( model_name="t5-small", torch_dtype="bfloat16", @@ -546,39 +565,6 @@ def _decoder_cuda_graph_config( ) -def _assert_decoder_cuda_graph_state( - llm: LLM, - enabled: bool, - batch_sizes: list[int] | None, -) -> None: - model_engine = llm._executor.engine.model_engine - - if not enabled: - assert not model_engine.encoder_cuda_graph_runner.enabled - assert not model_engine.cuda_graph_runner.enabled - assert not model_engine.encoder_cuda_graph_runner.graphs - assert not model_engine.cuda_graph_runner.graphs - return - - _assert_decoder_cuda_graphs_captured(llm) - if batch_sizes is not None: - assert model_engine.cuda_graph_runner.padding_dummy_requests - - -def _assert_decoder_cuda_graphs_captured(llm: LLM) -> None: - model_engine = llm._executor.engine.model_engine - - assert not model_engine.encoder_cuda_graph_runner.enabled - assert not model_engine.encoder_cuda_graph_runner.graphs - assert model_engine.cuda_graph_runner.enabled - assert model_engine.cuda_graph_runner.graphs - - -def _assert_mixed_context_generation_cuda_graph_state(llm: LLM) -> None: - _assert_decoder_cuda_graphs_captured(llm) - assert llm._executor.engine.model_engine.cuda_graph_runner.padding_dummy_requests - - class _SleepLogitsProcessor: def __init__(self, delay_seconds: float) -> None: self.delay_seconds = delay_seconds @@ -654,10 +640,9 @@ def _run_t5_pytorch_generate_encoder_decoder( num_return_sequences: int, exact_match: bool, cuda_graph_batch_sizes: list[int] | None, + disable_overlap_scheduler: bool = True, tensor_parallel_size: int = 1, ) -> None: - if tensor_parallel_size == 1: - monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") monkeypatch.setenv("TRTLLM_SKIP_KV_CACHE_ESTIMATION", "1") model_path = _get_t5_model_path(model_name) @@ -665,7 +650,7 @@ def _run_t5_pytorch_generate_encoder_decoder( case_id = ( f"model={model_name}, dtype={torch_dtype}, kv_v2={use_kv_cache_manager_v2}, " f"cuda_graph={enable_cuda_graph}, beams={num_beams}, returns={num_return_sequences}, " - f"tp={tensor_parallel_size}" + f"overlap={not disable_overlap_scheduler}, tp={tensor_parallel_size}" ) sampling_params = _sampling_params(num_beams, num_return_sequences) @@ -676,7 +661,7 @@ def _run_t5_pytorch_generate_encoder_decoder( cuda_graph_config=_decoder_cuda_graph_config(cuda_graph_batch_sizes) if enable_cuda_graph else None, - disable_overlap_scheduler=True, + disable_overlap_scheduler=disable_overlap_scheduler, dtype=torch_dtype, enable_chunked_prefill=False, tensor_parallel_size=tensor_parallel_size, @@ -711,22 +696,12 @@ def _run_t5_pytorch_generate_encoder_decoder( exact_match, expected_output_token_ids_by_output, ) - # CUDA graph state introspection reaches into the in-process engine, - # which is only available when the executor runs single-process (TP=1). - # For TP>1 the executor is a multi-process proxy without a local engine, - # so we rely on the generated-output assertions above for correctness. - if tensor_parallel_size == 1: - _assert_decoder_cuda_graph_state( - llm, - enable_cuda_graph, - cuda_graph_batch_sizes, - ) @pytest.mark.parametrize( "model_name,expected_output_token_ids_by_output,torch_dtype,use_kv_cache_manager_v2," "enable_cuda_graph,num_beams,num_return_sequences,exact_match,cuda_graph_batch_sizes," - "tensor_parallel_size", + "disable_overlap_scheduler,tensor_parallel_size", _TEST_CASES, ) def test_t5_pytorch_generate_encoder_decoder_end_to_end( @@ -740,6 +715,7 @@ def test_t5_pytorch_generate_encoder_decoder_end_to_end( num_return_sequences: int, exact_match: bool, cuda_graph_batch_sizes: list[int] | None, + disable_overlap_scheduler: bool, tensor_parallel_size: int, ) -> None: _run_t5_pytorch_generate_encoder_decoder( @@ -753,6 +729,7 @@ def test_t5_pytorch_generate_encoder_decoder_end_to_end( num_return_sequences, exact_match, cuda_graph_batch_sizes, + disable_overlap_scheduler, tensor_parallel_size, ) @@ -772,7 +749,6 @@ def test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch( num_return_sequences: int, exact_match: bool, ) -> None: - monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") monkeypatch.setenv("TRTLLM_SKIP_KV_CACHE_ESTIMATION", "1") model_path = _get_t5_model_path(model_name) @@ -842,13 +818,10 @@ def test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch( expected_text_fragment=expected_text_fragment, ) - _assert_decoder_cuda_graphs_captured(llm) - def test_t5_pytorch_generate_encoder_decoder_mixed_context_generation_batch( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") monkeypatch.setenv("TRTLLM_SKIP_KV_CACHE_ESTIMATION", "1") model_name = "t5-small" @@ -910,4 +883,3 @@ def test_t5_pytorch_generate_encoder_decoder_mixed_context_generation_batch( max_tokens=_MIXED_CONTEXT_GENERATION_MAX_NEW_TOKENS, ) _assert_t5_response(second_response, num_return_sequences=1) - _assert_mixed_context_generation_cuda_graph_state(llm) diff --git a/tests/integration/defs/llmapi/test_llm_api_pytorch_whisper.py b/tests/integration/defs/llmapi/test_llm_api_pytorch_whisper.py new file mode 100644 index 000000000000..bb72ccb7121a --- /dev/null +++ b/tests/integration/defs/llmapi/test_llm_api_pytorch_whisper.py @@ -0,0 +1,329 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""End-to-end Whisper (audio encoder-decoder) tests on the PyTorch backend. + +Whisper feeds the encoder a log-mel feature tensor carried via +``multi_modal_data["audio"]`` (not encoder token ids); the decoder prompt is +the forced task-token prefix. Expected outputs below were pinned from +``openai/whisper-tiny`` fp32 greedy on the LibriSpeech test-clean utterance +1221-135766-0002 (the wav shipped next to the legacy whisper checkpoints in +``LLM_MODELS_ROOT``); HF transformers greedy produces the identical +transcript. +""" + +from pathlib import Path + +import pytest +import soundfile + +from tensorrt_llm.llmapi import LLM, CudaGraphConfig, KvCacheConfig, SamplingParams, SchedulerConfig + +from ..conftest import llm_models_root + +_MAX_NEW_TOKENS = 96 +_MIN_GPU_MEMORY_MB = 16_000 +_FREE_GPU_MEMORY_FRACTION = 0.2 +_CROSS_KV_CACHE_FRACTION = 0.5 +# whisper-tiny fp32 greedy on 1221-135766-0002.wav (matches HF transformers). +_EXPECTED_GREEDY_OUTPUT_TOKEN_IDS = [ + 1939, + 613, + 4598, + 8028, + 389, + 3011, + 582, + 259, + 1570, + 365, + 1454, + 813, + 38675, + 3378, + 13, + 50257, +] +# In half precision the first token is a near-tie between " yet" (1939) and +# " Yet" (10890), and which one wins varies with dtype and GPU architecture +# (HF: fp32/fp16 -> 1939, bf16 -> 10890; TRT-LLM fp16: SM120 -> 10890, +# DGX B200 -> 1939). Accept either; the remaining tokens are stable. +_BORDERLINE_FIRST_TOKEN_IDS = (1939, 10890) +_EXPECTED_TRANSCRIPT_FRAGMENT = "thoughts affected hester" + +pytestmark = [ + pytest.mark.skip_less_device(1), + pytest.mark.skip_less_device_memory(_MIN_GPU_MEMORY_MB), + pytest.mark.threadleak(enabled=False), +] + + +def _get_whisper_model_path() -> str: + # llm_models_root() asserts (with a clear message) when no model cache is + # reachable, so no None check is needed here. + models_root = llm_models_root() + candidates = [ + # HF-format snapshot in the CI share (next to the legacy checkpoints). + Path(models_root) / "whisper-models" / "whisper-tiny", + Path(models_root) / "whisper" / "whisper-tiny", + Path(models_root) / "whisper-tiny", + ] + for candidate in candidates: + if candidate.exists(): + return str(candidate) + pytest.skip( + f"HF-format whisper-tiny not found under {models_root} (tried " + f"{[str(c) for c in candidates]})." + ) + + +def _get_audio_path() -> str: + models_root = llm_models_root() + for legacy_dir in ("large-v3", "large-v2"): + candidate = Path(models_root) / "whisper-models" / legacy_dir / "1221-135766-0002.wav" + if candidate.exists(): + return str(candidate) + pytest.skip(f"1221-135766-0002.wav not found under {models_root}/whisper-models.") + + +def _make_llm( + model_path: str, + max_beam_width: int = 1, + use_kv_cache_manager_v2: bool = False, + torch_dtype: str | None = None, + cuda_graph_batch_sizes: list[int] | None = None, + tensor_parallel_size: int = 1, +) -> LLM: + # CudaGraphConfig captures the decode step only; fp32 enc-dec declines + # graphs at engine init (workspace-sizing guard), so requesting them must + # still work for every dtype. + cuda_graph_config = ( + CudaGraphConfig(batch_sizes=cuda_graph_batch_sizes, enable_padding=True) + if cuda_graph_batch_sizes is not None + else None + ) + dtype_kwargs = {} + if torch_dtype is not None: + # The checkpoint's torch_dtype wins over `dtype` in the PyTorch + # backend; model_kwargs is the effective override (same as T5/BART). + dtype_kwargs = {"dtype": torch_dtype, "model_kwargs": {"torch_dtype": torch_dtype}} + return LLM( + model_path, + attn_backend="TRTLLM", + cuda_graph_config=cuda_graph_config, + disable_overlap_scheduler=True, # overlap scheduler unsupported + enable_chunked_prefill=False, + kv_cache_config=KvCacheConfig( + enable_block_reuse=False, + free_gpu_memory_fraction=_FREE_GPU_MEMORY_FRACTION, + cross_kv_cache_fraction=_CROSS_KV_CACHE_FRACTION, + use_kv_cache_manager_v2=use_kv_cache_manager_v2, + ), + max_batch_size=2, + max_beam_width=max_beam_width, + # Cross-KV pool capacity; the default (1024) is smaller than the + # 1500 encoder positions every Whisper request produces. + max_input_len=1500, + max_num_tokens=3000, + scheduler_config=SchedulerConfig(use_python_scheduler=True), + tensor_parallel_size=tensor_parallel_size, + **dtype_kwargs, + ) + + +def _audio_prompt(wave, sample_rate, prompt: str = ""): + return {"prompt": prompt, "multi_modal_data": {"audio": [(wave, sample_rate)]}} + + +def test_whisper_pytorch_transcribe_end_to_end(monkeypatch): + """Greedy transcription: exact pinned token ids, single + batch-2. + + Deliberately no TRTLLM_SKIP_KV_CACHE_ESTIMATION: the engine must gate + estimation off for enc-dec itself, so the default config is what's tested. + """ + monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") + + model_path = _get_whisper_model_path() + wave, sample_rate = soundfile.read(_get_audio_path()) + sampling_params = SamplingParams(temperature=0.0, max_tokens=_MAX_NEW_TOKENS) + + with _make_llm(model_path) as llm: + outputs = llm.generate([_audio_prompt(wave, sample_rate)], sampling_params) + completion = outputs[0].outputs[0] + assert list(completion.token_ids) == _EXPECTED_GREEDY_OUTPUT_TOKEN_IDS + assert _EXPECTED_TRANSCRIPT_FRAGMENT in completion.text.lower() + + # Batch of 2: the encoder packs both mels into one pass; each request + # keeps its own cross-KV. Identical clips must transcribe identically. + outputs = llm.generate( + [_audio_prompt(wave, sample_rate) for _ in range(2)], sampling_params + ) + for output in outputs: + assert list(output.outputs[0].token_ids) == _EXPECTED_GREEDY_OUTPUT_TOKEN_IDS + + # Token-only prompts cannot feed a feature-driven encoder and must be + # rejected at submission rather than poison the encoder batch. + with pytest.raises(Exception, match="multi_modal_data"): + llm.generate([[50258, 50259, 50359, 50363]], sampling_params) + + # A non-empty text prompt is the decoder prompt: forcing German makes + # whisper-tiny emit a (rough) German rendering — assert it diverges + # from the English transcript and is non-empty. + outputs = llm.generate( + [ + _audio_prompt( + wave, + sample_rate, + prompt="<|startoftranscript|><|de|><|transcribe|><|notimestamps|>", + ) + ], + sampling_params, + ) + german_text = outputs[0].outputs[0].text.strip() + assert german_text + assert _EXPECTED_TRANSCRIPT_FRAGMENT not in german_text.lower() + + # PreprocessedInputs skips the input processor, so the executor-side + # max_tokens clamp is what bounds generation to the decoder table (448) + # rather than the encoder-sized max_seq_len. ignore_eos forces the + # request to reach that cap. + preprocessed = llm.preprocess( + _audio_prompt(wave, sample_rate), SamplingParams(temperature=0.0) + ) + fresh_params = SamplingParams(temperature=0.0, max_tokens=10_000, ignore_eos=True) + result = llm.generate_async(preprocessed, fresh_params).result() + generated = list(result.outputs[0].token_ids) + assert len(generated) == 448 - len(preprocessed.prompt_token_ids) + expected = _EXPECTED_GREEDY_OUTPUT_TOKEN_IDS + assert generated[: len(expected)] == expected + + +# Beam always rides kv-v1: KVCacheManagerV2 requires beam width 1 (a v2 +# preference with beam > 1 silently falls back to v1). The bf16 graphs-on case +# captures decode graphs over batch_size * beam_width sequences. +_BEAM_SEARCH_CASES = [ + pytest.param(None, None, False, id="fp32-kv-v1-graphs-off-beam2"), + pytest.param("bfloat16", [1, 2], True, id="bf16-kv-v1-decoder-graphs-on-beam2"), +] + + +@pytest.mark.parametrize("torch_dtype,cuda_graph_batch_sizes,graphs_captured", _BEAM_SEARCH_CASES) +def test_whisper_pytorch_beam_search( + monkeypatch, torch_dtype, cuda_graph_batch_sizes, graphs_captured +): + """Beam-2 transcription (cross-KV shared across beams).""" + monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") + + model_path = _get_whisper_model_path() + wave, sample_rate = soundfile.read(_get_audio_path()) + + llm = _make_llm( + model_path, + max_beam_width=2, + torch_dtype=torch_dtype, + cuda_graph_batch_sizes=cuda_graph_batch_sizes, + ) + with llm: + beam_params = SamplingParams( + best_of=2, n=1, temperature=0.0, use_beam_search=True, max_tokens=_MAX_NEW_TOKENS + ) + outputs = llm.generate([_audio_prompt(wave, sample_rate)], beam_params) + assert _EXPECTED_TRANSCRIPT_FRAGMENT in outputs[0].outputs[0].text.lower() + _assert_decoder_cuda_graph_state(llm, captured=graphs_captured) + + +def _assert_decoder_cuda_graph_state(llm: LLM, captured: bool) -> None: + """Introspect the in-process engine (single-process mode only). + + Decoder graphs captured (or not); the enc-dec encoder step stays eager. + """ + model_engine = llm._executor.engine.model_engine + assert not model_engine.encoder_cuda_graph_runner.enabled + assert not model_engine.encoder_cuda_graph_runner.graphs + assert model_engine.cuda_graph_runner.enabled == captured + assert bool(model_engine.cuda_graph_runner.graphs) == captured + + +# Feature-combination matrix mirroring the T5/BART enc-dec coverage. Cases: +# (torch_dtype override or None for checkpoint fp32, kv manager v2, decoder +# cuda-graph batch sizes, graphs must capture, TP size). KVCacheManagerV2 +# requires beam width 1, so v2 rides greedy; the fp32+graphs-requested case +# asserts the engine declines graphs (fp32 enc-dec guard) yet stays exact. +_FEATURE_COMBINATION_CASES = [ + pytest.param(None, True, None, False, 1, id="fp32-kv-v2-graphs-off-greedy"), + pytest.param(None, False, [1, 2], False, 1, id="fp32-kv-v1-graphs-requested-greedy"), + pytest.param("bfloat16", False, [1, 2], True, 1, id="bf16-kv-v1-decoder-graphs-on-greedy"), + pytest.param("bfloat16", True, [1, 2], True, 1, id="bf16-kv-v2-decoder-graphs-on-greedy"), + pytest.param("float16", False, None, False, 1, id="fp16-kv-v1-graphs-off-greedy"), + pytest.param( + None, + False, + None, + False, + 2, + id="fp32-kv-v1-graphs-off-greedy-tp2", + marks=pytest.mark.skip_less_device(2), + ), +] + + +@pytest.mark.parametrize( + "torch_dtype,use_kv_cache_manager_v2,cuda_graph_batch_sizes,graphs_captured,tp_size", + _FEATURE_COMBINATION_CASES, +) +def test_whisper_pytorch_feature_combinations( + monkeypatch, + torch_dtype, + use_kv_cache_manager_v2, + cuda_graph_batch_sizes, + graphs_captured, + tp_size, +): + """Greedy transcription across dtype/kv-cache-manager/CUDA-graph/TP combos. + + Batch-1 and batch-2 must both reproduce the pinned token ids (fp32 exact; + fp16/bf16 exact except the hardware-dependent borderline first token). + """ + if tp_size == 1: + monkeypatch.setenv("TLLM_WORKER_USE_SINGLE_PROCESS", "1") + + model_path = _get_whisper_model_path() + wave, sample_rate = soundfile.read(_get_audio_path()) + sampling_params = SamplingParams(temperature=0.0, max_tokens=_MAX_NEW_TOKENS) + + llm = _make_llm( + model_path, + use_kv_cache_manager_v2=use_kv_cache_manager_v2, + torch_dtype=torch_dtype, + cuda_graph_batch_sizes=cuda_graph_batch_sizes, + tensor_parallel_size=tp_size, + ) + with llm: + for batch_size in (1, 2): + outputs = llm.generate( + [_audio_prompt(wave, sample_rate) for _ in range(batch_size)], + sampling_params, + ) + for output in outputs: + completion = output.outputs[0] + token_ids = list(completion.token_ids) + if torch_dtype is None: + assert token_ids == _EXPECTED_GREEDY_OUTPUT_TOKEN_IDS + else: + assert token_ids[0] in _BORDERLINE_FIRST_TOKEN_IDS + assert token_ids[1:] == _EXPECTED_GREEDY_OUTPUT_TOKEN_IDS[1:] + assert _EXPECTED_TRANSCRIPT_FRAGMENT in completion.text.lower() + + if tp_size == 1: + _assert_decoder_cuda_graph_state(llm, captured=graphs_captured) diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index a1c534507634..53fc8c346b5f 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -229,6 +229,16 @@ Tests are defined in `jenkins/L0_Test.groovy` under the `launchTestJobs` functio **Important**: Pre-merge and post-merge tests must be in separate stages. +### Pre-merge vs Post-merge Perf-Regression Gating + +By default, `test_perf_sanity.py` fails CI on perf regression for pre-merge stages and only warns for post-merge stages. This is auto-detected from the Jenkins job URL (`PostMerge` substring), not the stage name. + +**`FUNCTIONAL-ONLY` stage-name flag**: A pre-merge stage whose name contains `FUNCTIONAL-ONLY` (e.g. `GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE1-GPU4`) still runs the full perf harness — benchmarks execute, metrics are uploaded to OpenSearch, dashboards update — but perf regressions **do not fail CI**. Only functional failures (build errors, crashes, empty output) fail the stage. + +Use this for pre-merge stages whose goal is to catch functional regressions on paths that only had post-merge coverage before. It preserves the data-continuity benefit of running in pre-merge (baselines still update from PR data points) without the flakiness cost of gating on the noisier disagg perf numbers. + +Detection is by substring match on `os.environ["stageName"]` inside `test_perf_sanity.py`; no changes to `perf_regression_utils.py`. + ### GPU Hours Calculation - Each CI stage runtime is approximately **1 hour** diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index e995ce257f8f..ae6c940ba174 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -16,15 +16,12 @@ # Model PATH of local dir synced from internal LLM models repo MODEL_PATH_DICT = { - "llama_v3.1_8b": "llama-3.1-model/Meta-Llama-3.1-8B", "llama_v3.1_8b_instruct": "llama-3.1-model/Llama-3.1-8B-Instruct", "llama_v3.1_8b_instruct_fp8": "llama-3.1-model/Llama-3.1-8B-Instruct-FP8", "llama_v3.1_8b_instruct_fp4": "modelopt-hf-model-hub/Llama-3.1-8B-Instruct-fp4", "llama_v3.3_70b_instruct": "llama-3.3-models/Llama-3.3-70B-Instruct", "llama_v3.3_70b_instruct_fp8": "modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp8", "llama_v3.3_70b_instruct_fp4": "modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp4", - "llama_v3.2_1b": "llama-3.2-models/Llama-3.2-1B", - "llama_v3.1_nemotron_nano_8b": "Llama-3.1-Nemotron-Nano-8B-v1", "llama_v3.1_nemotron_nano_8b_fp8": "Llama-3.1-Nemotron-Nano-8B-v1-FP8", "llama_v3.3_nemotron_super_49b": "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1", "llama_v3.3_nemotron_super_49b_fp8": "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1-FP8", @@ -34,8 +31,6 @@ "llama_v4_scout_17b_16e_instruct": "llama4-models/Llama-4-Scout-17B-16E-Instruct", "llama_v4_scout_17b_16e_instruct_fp8": "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8", "llama_v4_scout_17b_16e_instruct_fp4": "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4", - "llama_v4_maverick_17b_128e_instruct": "llama4-models/Llama-4-Maverick-17B-128E-Instruct", - "llama_v4_maverick_17b_128e_instruct_fp8": "llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8", "deepseek_r1_distill_qwen_32b": "DeepSeek-R1/DeepSeek-R1-Distill-Qwen-32B", "deepseek_r1_distill_llama_70b": "DeepSeek-R1/DeepSeek-R1-Distill-Llama-70B/", "gemma_3_27b_it": "gemma/gemma-3-27b-it", @@ -44,15 +39,12 @@ "gemma_3_12b_it": "gemma/gemma-3-12b-it", "gemma_3_12b_it_fp8": "gemma/gemma-3-12b-it-fp8", "gemma_3_12b_it_fp4": "gemma/gemma-3-12b-it-fp4", - "deepseek_r1_fp8": "DeepSeek-R1/DeepSeek-R1", - "deepseek_r1_nvfp4": "DeepSeek-R1/DeepSeek-R1-FP4", + "gemma_3_1b_it": "gemma/gemma-3-1b-it", + "gemma_4_26b_a4b_nvfp4": "gemma/nvidia-Gemma-4-26B-A4B-NVFP4", "deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/", "deepseek_r1_0528_fp4": "DeepSeek-R1/DeepSeek-R1-0528-FP4/", "deepseek_r1_0528_fp4_v2": "DeepSeek-R1/DeepSeek-R1-0528-FP4-v2/", "deepseek_v3_lite_fp8": "DeepSeek-V3-Lite/fp8", - "deepseek_v3_lite_nvfp4": "DeepSeek-V3-Lite/nvfp4_moe_only", - "qwen2_7b_instruct": "Qwen2-7B-Instruct", - "qwen_14b_chat": "Qwen-14B-Chat", "qwen3_0.6b": "Qwen3/Qwen3-0.6B", "qwen3_4b_eagle3": "Qwen3/Qwen3-4B", "qwen3_8b": "Qwen3/Qwen3-8B", @@ -71,17 +63,13 @@ "qwen2_5_vl_7b_instruct": "Qwen2.5-VL-7B-Instruct", "qwen2_5_vl_7b_instruct_fp8": "multimodals/Qwen2.5-VL-7B-Instruct-FP8", "qwen2_5_vl_7b_instruct_fp4": "multimodals/Qwen2.5-VL-7B-Instruct-FP4", - "starcoder2_3b": "starcoder2-3b", - "phi_4_mini_instruct": "Phi-4-mini-instruct", "phi_4_reasoning_plus": "Phi-4-reasoning-plus", "phi_4_reasoning_plus_fp8": "nvidia-Phi-4-reasoning-plus-FP8", "phi_4_reasoning_plus_fp4": "nvidia-Phi-4-reasoning-plus-NVFP4", "phi_4_multimodal_instruct": "multimodals/Phi-4-multimodal-instruct", "phi_4_multimodal_instruct_fp4": "multimodals/Phi-4-multimodal-instruct-FP4", "phi_4_multimodal_instruct_fp8": "multimodals/Phi-4-multimodal-instruct-FP8", - "mistral_small_v3.1_24b": "Mistral-Small-3.1-24B-Instruct-2503", "bielik_11b_v2.2_instruct": "Bielik-11B-v2.2-Instruct", - "bielik_11b_v2.2_instruct_fp8": "Bielik-11B-v2.2-Instruct-FP8", "gpt_oss_120b_fp4": "gpt_oss/gpt-oss-120b", "gpt_oss_20b_fp4": "gpt_oss/gpt-oss-20b", "gpt_oss_120b_eagle3": "gpt_oss/gpt-oss-120b", @@ -97,6 +85,8 @@ "kimi_k2_nvfp4": "Kimi-K2-Thinking-NVFP4", # MiniMax M2.5 (FP8 block-scale, ~230B MoE) "minimax_m2.5_fp8": "MiniMax-M2.5", + # MiniMax M3 (block-sparse MoE, MXFP8 weights, BF16 activations + KV cache) + "minimax_m3_mxfp8": "MiniMax-M3-MXFP8", # Qwen3.5 dense + MoE "qwen3.5_9b": "Qwen3.5-9B", "qwen3.5_27b": "Qwen3.5-27B", @@ -104,9 +94,16 @@ "qwen3.5_122b_a10b": "Qwen3.5-122B-A10B", "qwen3.5_397b_a17b_fp8": "Qwen3.5-397B-A17B-FP8", "qwen3.5_397b_a17b_fp4": "Qwen3.5-397B-A17B-NVFP4", + # Qwen3.6 (GDN linear-attn MoE, NVFP4) + "qwen3.6_35b_a3b_fp4": "Qwen3.6-35B-A3B-NVFP4", # DeepSeek V3.2 (671B MoE) "deepseek_v3.2_fp8": "DeepSeek-V3.2-hf", "deepseek_v3.2_fp4": "DeepSeek-V3.2-NVFP4", + # DeepSeek V4 + "deepseek_v4_pro_fp4": "DeepSeek-V4-Pro", + "deepseek_v4_flash": "DeepSeek-V4-Flash", + "deepseek_v4_flash_base_fp8": "DeepSeek-V4-Flash-Base", + "deepseek_v4_pro_dspark": "DeepSeek-V4-Pro-DSpark", # GLM-5 FP8 (MoE) "glm_5_fp8": "GLM-5-FP8", # Kimi K2.5 NVFP4 (~1T MoE multimodal) diff --git a/tests/integration/defs/perf/build.py b/tests/integration/defs/perf/build.py deleted file mode 100644 index a12169ba1ff6..000000000000 --- a/tests/integration/defs/perf/build.py +++ /dev/null @@ -1,1524 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import multiprocessing as mp -import os -import time -from collections import OrderedDict - -# isort: off -import torch -import tensorrt as trt -# isort: on - -from allowed_configs import (get_allowed_models, get_build_config, - get_model_config, get_model_family) - -import tensorrt_llm -from tensorrt_llm._utils import str_dtype_to_trt -from tensorrt_llm.builder import BuildConfig, Builder, build -from tensorrt_llm.functional import LayerNormPositionType, LayerNormType -from tensorrt_llm.logger import logger -from tensorrt_llm.mapping import Mapping -from tensorrt_llm.models import PretrainedConfig -from tensorrt_llm.models.modeling_utils import QuantConfig, optimize_model -from tensorrt_llm.network import net_guard -from tensorrt_llm.plugin.plugin import ContextFMHAType -from tensorrt_llm.quantization import QuantAlgo, QuantMode -from tensorrt_llm.quantization.quantize import quantize - -WEIGHT_STREAMING_DISABLED_VAL = "1.0" - - -def parse_arguments(): - parser = argparse.ArgumentParser(description='Build TensorRT LLM models.') - parser.add_argument('-m', - '--model', - type=str, - required=True, - choices=get_allowed_models(), - help='Specify model you want to build.') - parser.add_argument( - '--mode', - type=str, - default="plugin", - choices=['ootb', 'plugin', 'plugin-ifb', 'ootb-except-mha'], - help= - ('Choose mode between ootb/plugin/ootb-except-mha. ' - '\"ootb\" means the engines will be built without any plugins, ' - '\"plugin\" means the engines will be built with tuned recipe of using plugins.' - '\"plugin-ifb\" will include additional options required for inflight batching.' - '\"ootb-except-mha\" means the engines will be built with only attention plugins.' - )) - - parser.add_argument( - '--dtype', - type=str, - default='float16', - choices=['float16', 'bfloat16', 'float32'], - help='Choose data type between float16/bfloat16/float32.') - parser.add_argument( - '--quantization', - type=str, - default=None, - choices=[ - 'fp8', 'fp8_gemm', 'fp8_kv_cache', 'int8_sq_per_tensor', - 'int8_sq_per_token_channel', 'int8_weight_only', 'int4_weight_only', - 'int4_weight_only_awq', 'int4_weight_only_gptq' - ], - help="Optimize the model with specified quantization recipe") - - parser.add_argument( - '--input_timing_cache', - type=str, - default=None, - help= - 'The path to read timing cache, will be ignored if the file does not exist' - ) - parser.add_argument('--output_timing_cache', - type=str, - default='model.cache', - help='The path to write timing cache') - - parser.add_argument( - '--profiling_verbosity', - type=str, - default='layer_names_only', - choices=['layer_names_only', 'detailed', 'none'], - help= - 'The profiling verbosity for the generated TRT engine. Set to detailed can inspect tactic choices and kernel parameters.' - ) - parser.add_argument( - '--log_level', - type=str, - default="error", - choices=['verbose', 'info', 'warning', 'error', 'internal_error'], - help= - 'Choose log level between verbose/info/warning/error/internal_error.') - - parser.add_argument( - '--output_dir', - type=str, - required=True, - help='TensorRT engines will be saved to the specified path.') - - parser.add_argument( - '--max_beam_width', - type=int, - default=None, - help= - ('If this option is specified, it will override the max beam width of ' - 'TRT engines to the specified value instead of using pre-defined one')) - parser.add_argument( - '--max_input_len', - type=int, - default=None, - help= - ('If this option is specified, it will override the max input len of ' - 'TRT engines to the specified value instead of using pre-defined one')) - parser.add_argument( - '--max_seq_len', - '--max_decoder_seq_len', - dest='max_seq_len', - type=int, - default=None, - help= - ('If this option is specified, it will override the max sequence len of ' - 'TRT engines to the specified value instead of using pre-defined one')) - parser.add_argument( - '--max_batch_size', - type=int, - default=None, - help= - ('If this option is specified, it will override the max batch size of ' - 'TRT engines to the specified value instead of using pre-defined one')) - parser.add_argument('--force_num_layer_1', - default=False, - action='store_true', - help='Quick sanity check with num_layer=1.') - parser.add_argument('--serial_build', - default=False, - action='store_true', - help="Build engines serially") - parser.add_argument( - '--multiple_profiles', - default=False, - action='store_true', - help= - 'This option will benefit performance, but will increase the engine build time.' - ) - - parser.add_argument( - '--weight_streaming', - default=False, - action='store_true', - help= - 'Specify whether offloading weights to CPU and streaming loading at runtime.', - ) - - parser.add_argument( - '--monitor_memory', - default=False, - action='store_true', - help='Specify whether turning on the memory monitor flag.', - ) - - parser.add_argument( - '--rank', - type=int, - default=None, - help= - "The rank of the model to be built, only used when --serial_build is specified" - ) - parser.add_argument( - '--world_size', - type=int, - default=None, - help= - "The number of gpus to be used for inference, only used when --serial_build is specified" - ) - parser.add_argument( - '--opt_batch_size', - type=int, - default=None, - help= - "If opt_batch_size option is specified, it will override the opt batch size." - "This flag only takes effect when `--mode=ootb` is added. For other modes, please use --opt_num_tokens to replace it." - ) - - parser.add_argument( - '--opt_num_tokens', - type=int, - default=None, - help="It equals to max_batch_size*max_beam_width by default, set this " - "value as close as possible to the actual number of tokens on your workload. " - "Note that this argument might be removed in the future." - "This flag only takes effect when `--mode` is not `ootb`. For ootb mode, please use --opt_batch_size to replace it." - ) - - return parser.parse_args() - - -def serialize_engine(engine, path): - logger.info(f'Serializing engine to {path}...') - tik = time.time() - with open(path, 'wb') as f: - # engine object is already complies with python buffer protocol, no need to - # convert it to bytearray before write, converting to bytearray consumes lots of memory - f.write(engine) - tok = time.time() - t = time.strftime('%H:%M:%S', time.gmtime(tok - tik)) - logger.info(f'Engine serialized. Total time: {t}') - - -def get_quant_config(quantization: str): - if quantization == "fp8": - return QuantConfig(quant_algo=QuantAlgo.FP8, - kv_cache_quant_algo=QuantAlgo.FP8) - elif quantization == "fp8_gemm": - return QuantConfig(quant_algo=QuantAlgo.FP8) - elif quantization == "fp8_kv_cache": - return QuantConfig(kv_cache_quant_algo=QuantAlgo.FP8) - elif quantization == "int8_sq_per_tensor": - return QuantConfig(quant_algo=QuantAlgo.W8A8_SQ_PER_TENSOR_PLUGIN) - elif quantization == "int8_sq_per_token_channel": - return QuantConfig( - quant_algo=QuantAlgo.W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN) - elif quantization == "int8_sq_per_channel_ootb": - return QuantConfig(quant_algo=QuantAlgo.W8A8_SQ_PER_CHANNEL) - elif quantization == "int8_weight_only": - return QuantConfig(quant_algo=QuantAlgo.W8A16) - elif quantization == "int4_weight_only": - return QuantConfig(quant_algo=QuantAlgo.W4A16) - elif quantization == "int4_weight_only_awq": - return QuantConfig(quant_algo=QuantAlgo.W4A16_AWQ) - elif quantization == "int4_weight_only_gptq": - return QuantConfig(quant_algo=QuantAlgo.W4A16_GPTQ) - elif quantization is None: - return QuantConfig() - else: - raise Exception(f"Unexpected quantization: {quantization}") - - -def build_gpt(args): - build_config = get_build_config(args.model) - build_config = BuildConfig(**build_config) - model_config = get_model_config(args.model) - if args.force_num_layer_1: - model_config['num_layers'] = 1 - - # More parameters - if args.serial_build and args.rank is not None and args.world_size is not None: - runtime_rank = args.rank - world_size = args.world_size - else: - runtime_rank = tensorrt_llm.mpi_rank() - world_size = tensorrt_llm.mpi_world_size() - if not args.serial_build: - torch.cuda.set_device(runtime_rank) - - if args.profiling_verbosity != "layer_names_only": - build_config.profiling_verbosity = args.profiling_verbosity - - if args.max_batch_size is not None: - build_config.max_batch_size = args.max_batch_size - if args.max_input_len is not None: - build_config.max_input_len = args.max_input_len - if args.max_seq_len is not None: - build_config.max_seq_len = args.max_seq_len - if args.max_beam_width is not None: - build_config.max_beam_width = args.max_beam_width - if args.opt_batch_size is not None: - build_config.opt_batch_size = args.opt_batch_size - if args.opt_num_tokens is not None: - build_config.opt_num_tokens = args.opt_num_tokens - build_config.weight_streaming = getattr(args, "weight_streaming", False) - build_config.max_num_tokens = build_config.max_batch_size * max( - build_config.max_input_len, build_config.max_beam_width) - - if args.mode != "ootb" and args.opt_batch_size is not None: - raise Exception( - f'--opt_batch_size only used when mode is ootb. Please using --opt_num_tokens instead it.' - ) - if args.mode == "ootb" and args.opt_num_tokens is not None: - raise Exception( - f'--opt_num_tokens does not support ootb mode. Please using --opt_batch_size instead it.' - ) - - quant_config = get_quant_config(args.quantization) - quant_algo = quant_config.quant_algo - kv_cache_quant_algo = quant_config.kv_cache_quant_algo - quant_mode = quant_config.quant_mode - - # Initialize Module - family = get_model_family(args.model) - if family == "gpt": - if model_config['num_kv_heads'] is None: - model_config['num_kv_heads'] = model_config['num_heads'] - if model_config['inter_size'] is None: - model_config['inter_size'] = model_config['hidden_size'] * 4 - if model_config['position_embedding_type'] is None: - model_config['position_embedding_type'] = 'learned_absolute' - - config = { - 'architecture': 'GPTForCausalLM', - 'dtype': args.dtype, - 'num_hidden_layers': model_config['num_layers'], - 'num_attention_heads': model_config['num_heads'], - 'num_key_value_heads': model_config['num_kv_heads'], - 'hidden_size': model_config['hidden_size'], - 'intermediate_size': model_config['inter_size'], - 'norm_epsilon': 1e-05, - 'vocab_size': model_config['vocab_size'], - 'position_embedding_type': model_config['position_embedding_type'], - 'max_position_embeddings': model_config['n_positions'], - 'hidden_act': model_config['hidden_act'], - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo, - 'group_size': 128, - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'bias': model_config['bias'], - 'apply_query_key_layer_scaling': False, - 'rotary_pct': model_config['rotary_pct'], - 'moe': { - 'num_experts': model_config["moe_num_experts"], - 'top_k': model_config["moe_top_k"], - }, - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.GPTForCausalLM(config) - - elif family == "opt": - config = { - 'architecture': 'OPTForCausalLM', - 'dtype': args.dtype, - 'vocab_size': model_config['vocab_size'], - 'hidden_size': model_config['hidden_size'], - 'num_hidden_layers': model_config['num_layers'], - 'num_attention_heads': model_config['num_heads'], - 'hidden_act': model_config['hidden_act'], - 'max_position_embeddings': model_config['n_positions'], - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'use_parallel_embedding': False, - 'embedding_sharding_dim': 0, - 'do_layer_norm_before': model_config['do_layer_norm_before'], - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo, - 'group_size': 128 - } - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.OPTForCausalLM(config) - - elif family == "llama": - config = { - 'architecture': - 'LlamaForCausalLM', - 'dtype': - args.dtype, - 'logits_dtype': - 'float32', - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'num_key_value_heads': - model_config['num_heads'] if model_config['num_kv_heads'] is None - else model_config['num_kv_heads'], - 'hidden_size': - model_config['hidden_size'], - 'intermediate_size': - model_config['inter_size'], - 'vocab_size': - model_config['vocab_size'], - 'position_embedding_type': - 'rope_gpt_neox', - 'max_position_embeddings': - model_config['n_positions'], - 'hidden_act': - model_config['hidden_act'], - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo, - 'group_size': 128 - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'moe_tp_size': world_size, - 'moe_ep_size': 1, - 'rank': runtime_rank - }, - 'moe': { - 'num_experts': model_config["moe_num_experts"], - 'top_k': model_config["moe_top_k"], - } - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.LLaMAForCausalLM(config) - tensorrt_llm_model = optimize_model(tensorrt_llm_model, - use_fused_mlp=True) - elif family == "gptj": - config = { - 'architecture': 'GPTJForCausalLM', - 'dtype': args.dtype, - 'vocab_size': model_config['vocab_size'], - 'hidden_size': model_config['hidden_size'], - 'num_hidden_layers': model_config['num_layers'], - 'num_attention_heads': model_config['num_heads'], - 'hidden_act': model_config['hidden_act'], - 'max_position_embeddings': model_config['n_positions'], - 'position_embedding_type': 'rope_gptj', - 'rotary_dim': model_config['rotary_dim'], - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'use_parallel_embedding': False, - 'embedding_sharding_dim': 0, - 'do_layer_norm_before': model_config['do_layer_norm_before'], - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo, - 'group_size': 128 - } - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.GPTJForCausalLM(config) - - elif family == "gptneox": - config = { - 'architecture': - 'GPTNeoXForCausalLM', - 'dtype': - args.dtype, - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'hidden_size': - model_config['hidden_size'], - 'vocab_size': - model_config['vocab_size'], - 'position_embedding_type': - 'rope_gpt_neox', - 'max_position_embeddings': - model_config['n_positions'], - 'rotary_emb_base': - 10000, - 'rotary_pct': - 1.0 * model_config['rotary_dim'] * model_config['num_heads'] / - model_config['hidden_size'], - 'hidden_act': - model_config['hidden_act'], - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'use_parallel_embedding': - False, - 'embedding_sharding_dim': - 0, - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo, - 'group_size': 128, - } - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.GPTNeoXForCausalLM(config) - - elif family == "chatglm": - config = { - 'architecture': 'ChatGLMModel', - 'dtype': args.dtype, - 'num_hidden_layers': model_config['num_layers'], - 'num_attention_heads': model_config['num_heads'], - 'num_key_value_heads': model_config['num_kv_heads'], - 'hidden_size': model_config['hidden_size'], - 'intermediate_size': model_config['inter_size'], - 'norm_epsilon': 1e-5, - 'vocab_size': model_config['vocab_size'], - 'position_embedding_type': 'chatglm', - 'max_position_embeddings': model_config['n_positions'], - 'hidden_act': model_config['hidden_act'], - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'chatglm_version': 'chatglm', - 'add_bias_linear': True, - 'add_qkv_bias': True, - 'apply_query_key_layer_scaling': False, - 'apply_residual_connection_post_layernorm': False, - 'rmsnorm': False, - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.ChatGLMForCausalLM(config) - - elif family in ["chatglm2", "chatglm3"]: - config = { - 'architecture': 'ChatGLMModel', - 'dtype': args.dtype, - 'num_hidden_layers': model_config['num_layers'], - 'num_attention_heads': model_config['num_heads'], - 'num_key_value_heads': model_config['num_kv_heads'], - 'hidden_size': model_config['hidden_size'], - 'intermediate_size': model_config['inter_size'], - 'norm_epsilon': 1e-5, - 'vocab_size': model_config['vocab_size'], - 'position_embedding_type': 'rope_gptj', - 'max_position_embeddings': model_config['n_positions'], - 'hidden_act': model_config['hidden_act'], - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'chatglm_version': family, - 'add_bias_linear': False, - 'add_qkv_bias': True, - 'apply_query_key_layer_scaling': False, - 'apply_residual_connection_post_layernorm': False, - 'rmsnorm': True, - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.ChatGLMForCausalLM(config) - - elif family == "glm": - config = { - 'architecture': 'GLMModel', - 'dtype': args.dtype, - 'num_hidden_layers': model_config['num_layers'], - 'num_attention_heads': model_config['num_heads'], - 'num_key_value_heads': model_config['num_kv_heads'], - 'hidden_size': model_config['hidden_size'], - 'intermediate_size': model_config['inter_size'], - 'norm_epsilon': 1e-5, - 'vocab_size': model_config['vocab_size'], - 'position_embedding_type': 'learned_absolute', - 'max_position_embeddings': model_config['n_positions'], - 'hidden_act': model_config['hidden_act'], - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'chatglm_version': 'glm', - 'add_bias_linear': True, - 'add_qkv_bias': True, - 'apply_query_key_layer_scaling': False, - 'apply_residual_connection_post_layernorm': False, - 'rmsnorm': False, - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.ChatGLMForCausalLM(config) - - elif family == "bloom": - config = { - 'architecture': 'BloomForCausalLM', - 'dtype': args.dtype, - 'vocab_size': model_config['vocab_size'], - 'hidden_size': model_config['hidden_size'], - 'num_hidden_layers': model_config['num_layers'], - 'num_attention_heads': model_config['num_heads'], - 'hidden_act': model_config['hidden_act'], - 'max_position_embeddings': model_config['n_positions'], - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'use_parallel_embedding': (args.model == 'bloom_176b'), - 'embedding_sharding_dim': 0, - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo, - 'group_size': 128 - } - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.BloomForCausalLM(config) - elif family == "falcon": - config = { - 'architecture': - 'FalconForCausalLM', - 'dtype': - args.dtype, - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'num_key_value_heads': - model_config['num_heads'] if model_config['num_kv_heads'] is None - else model_config['num_kv_heads'], - 'hidden_size': - model_config['hidden_size'], - 'vocab_size': - model_config['vocab_size'], - 'position_embedding_type': - 'alibi_with_scale' - if model_config['use_alibi'] else 'rope_gpt_neox', - 'max_position_embeddings': - model_config['n_positions'], - 'hidden_act': - model_config['hidden_act'], - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo, - 'group_size': 128 - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'bias': - model_config['bias'], - 'parallel_attention': - model_config['parallel_attention'], - 'new_decoder_architecture': - model_config['new_decoder_architecture'], - } - if quant_mode.is_weight_only() and quant_mode.has_per_group_scaling(): - config['quantization'].update({ - 'has_zero_point': False, - 'pre_quant_scale': True, - }) - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.FalconForCausalLM(config) - - elif family == "baichuan": - config = { - 'architecture': - 'BaichuanForCausalLM', - 'dtype': - args.dtype, - 'logits_dtype': - 'float32', - 'vocab_size': - model_config['vocab_size'], - 'max_position_embeddings': - model_config['n_positions'], - 'hidden_size': - model_config['hidden_size'], - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'num_key_value_heads': - model_config['num_heads'], - 'hidden_act': - model_config['hidden_act'], - 'intermediate_size': - model_config['inter_size'], - 'position_embedding_type': - 'alibi_with_scale' if '7b' in args.model else 'rope_gpt_neox', - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo, - 'group_size': 128 - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.BaichuanForCausalLM(config) - - elif family == "internlm": - config = { - 'architecture': - 'LlamaForCausalLM', - 'dtype': - args.dtype, - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'num_key_value_heads': - model_config['num_heads'] if model_config['num_kv_heads'] is None - else model_config['num_kv_heads'], - 'hidden_size': - model_config['hidden_size'], - 'intermediate_size': - model_config['inter_size'], - 'vocab_size': - model_config['vocab_size'], - 'position_embedding_type': - 'rope_gpt_neox', - 'max_position_embeddings': - model_config['n_positions'], - 'hidden_act': - model_config['hidden_act'], - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'attn_bias': - model_config['bias'], - } - if quant_mode.is_weight_only(): - if 'awq' in args.quantization: - config['quantization'].update({ - "group_size": 128, - "has_zero_point": False, - "pre_quant_scale": True, - }) - elif 'gptq' in args.quantization: - config['quantization'].update({ - "group_size": 128, - "has_zero_point": True, - "pre_quant_scale": False, - }) - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.LLaMAForCausalLM(config) - - elif family == "qwen": - config = { - 'architecture': - 'QWenForCausalLM', - 'dtype': - args.dtype, - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'num_key_value_heads': - model_config['num_heads'] if model_config['num_kv_heads'] is None - else model_config['num_kv_heads'], - 'seq_length': - model_config['n_positions'], - 'hidden_size': - model_config['hidden_size'], - 'intermediate_size': - model_config['inter_size'], - 'vocab_size': - model_config['vocab_size'], - 'position_embedding_type': - 'rope_gpt_neox', - 'max_position_embeddings': - model_config['n_positions'], - 'hidden_act': - model_config['hidden_act'], - 'quantization': { - 'group_size': 128, - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'moe': { - 'num_experts': model_config["moe_num_experts"], - 'top_k': model_config["moe_top_k"], - }, - 'qwen_type': - 'qwen', - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.QWenForCausalLM(config) - elif family == "qwen2": - config = { - 'architecture': - 'QWenForCausalLM', - 'dtype': - args.dtype, - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'num_key_value_heads': - model_config['num_heads'] if model_config['num_kv_heads'] is None - else model_config['num_kv_heads'], - 'seq_length': - model_config['n_positions'], - 'hidden_size': - model_config['hidden_size'], - 'intermediate_size': - model_config['inter_size'], - 'vocab_size': - model_config['vocab_size'], - 'position_embedding_type': - 'rope_gpt_neox', - 'max_position_embeddings': - model_config['n_positions'], - 'hidden_act': - model_config['hidden_act'], - 'quantization': { - 'group_size': 128, - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'moe': { - 'num_experts': model_config["moe_num_experts"], - 'top_k': model_config["moe_top_k"], - }, - 'qwen_type': - 'qwen2', - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.QWenForCausalLM(config) - elif family == "mamba": - config = { - 'architecture': 'MambaForCausalLM', - 'dtype': args.dtype, - 'vocab_size': model_config['vocab_size'], - 'hidden_size': model_config['hidden_size'], - 'num_hidden_layers': model_config['num_layers'], - 'num_attention_heads': model_config['num_heads'], - 'hidden_act': model_config['hidden_act'], - 'state_size': model_config['state_size'], - 'conv_kernel': model_config['conv_kernel'], - 'layer_types': model_config['layer_types'], - 'rnn_hidden_size': model_config['rnn_hidden_size'], - 'rnn_head_size': model_config['rnn_head_size'], - 'rnn_conv_dim_size': model_config['rnn_conv_dim_size'], - 'rms_norm': True, - 'residual_in_fp32': True, - 'pad_vocab_size_multiple': 8, - 'use_bias': model_config['use_bias'], - 'mamba_version': model_config['mamba_version'], - 'ssm_rmsnorm': model_config['ssm_rmsnorm'], - 'ngroups': model_config['ngroups'], - 'chunk_size': model_config['chunk_size'], - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.MambaForCausalLM(config) - elif family == "recurrentgemma": - config = { - 'architecture': 'RecurrentGemmaForCausalLM', - 'dtype': args.dtype, - 'vocab_size': model_config['vocab_size'], - 'hidden_size': model_config['hidden_size'], - 'num_hidden_layers': model_config['num_layers'], - 'num_attention_heads': model_config['num_heads'], - 'num_key_value_heads': model_config['num_kv_heads'], - 'hidden_act': model_config['hidden_act'], - 'intermediate_size': model_config['inter_size'], - 'rms_norm': True, - 'norm_epsilon': 1e-6, - 'quantization': { - 'group_size': 128, - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - 'position_embedding_type': model_config['position_embedding_type'], - 'rotary_percentage': model_config['rotary_pct'], - 'max_position_embeddings': model_config['n_positions'], - 'conv_kernel': model_config['conv_kernel'], - 'state_size': model_config['state_size'], - 'layer_types': model_config['layer_types'], - 'rnn_hidden_size': model_config['rnn_hidden_size'], - 'rnn_head_size': model_config['rnn_head_size'], - 'rnn_conv_dim_size': model_config['rnn_conv_dim_size'], - 'logits_soft_cap': model_config['logits_soft_cap'], - 'rotary_pct': model_config['rotary_pct'], - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.RecurrentGemmaForCausalLM( - config) - tensorrt_llm_model = optimize_model(tensorrt_llm_model, - use_fused_mlp=True, - use_fused_rg_lru=True) - elif family == "phi3": - config = { - 'architecture': - 'PhiForCausalLM', - 'dtype': - args.dtype, - 'rotary_base': - 10000.0, - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'num_key_value_heads': - model_config['num_heads'] if model_config['num_kv_heads'] is None - else model_config['num_kv_heads'], - 'hidden_size': - model_config['hidden_size'], - 'intermediate_size': - model_config['inter_size'], - 'vocab_size': - model_config['vocab_size'], - 'position_embedding_type': - 'rope_gpt_neox', - 'max_position_embeddings': - model_config['n_positions'], - 'hidden_act': - model_config['hidden_act'], - 'quantization': { - 'quant_algo': quant_algo, - 'kv_cache_quant_algo': kv_cache_quant_algo, - 'group_size': 128 - }, - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_model = tensorrt_llm.models.Phi3ForCausalLM(config) - - else: - raise Exception(f'Unexpected model: {args.model}') - - # Plugins - build_config.plugin_config.to_legacy_setting() - if args.mode in ['plugin', 'plugin-ifb']: - build_config.plugin_config.gpt_attention_plugin = args.dtype - build_config.plugin_config.set_context_fmha(ContextFMHAType.enabled) - build_config.plugin_config.remove_input_padding = True - build_config.plugin_config.moe_plugin = args.dtype - build_config.plugin_config.mamba_conv1d_plugin = args.dtype - - if args.quantization is None or "fp8" not in args.quantization: - build_config.plugin_config.gemm_plugin = args.dtype - - # Quantization plugins. - use_weight_only = quant_mode.is_weight_only() - if use_weight_only: - build_config.plugin_config.weight_only_quant_matmul_plugin = args.dtype - - use_smooth_quant = quant_mode.has_act_and_weight_quant() - if use_smooth_quant: - build_config.plugin_config.set_smooth_quant_plugins( - dtype=args.dtype) - - use_qserve = quant_mode.has_act_and_weight_quant() and quant_mode._any( - QuantMode.INT4_WEIGHTS) - if use_qserve: - build_config.plugin_config.set_qserve_plugins(dtype=args.dtype) - - # Inflight batching - if args.mode == 'plugin-ifb': - build_config.plugin_config.enable_paged_kv_cache() - build_config.plugin_config.paged_state = True - elif args.mode == 'ootb-except-mha': - build_config.plugin_config.gpt_attention_plugin = args.dtype - build_config.plugin_config.set_context_fmha(ContextFMHAType.enabled) - build_config.plugin_config.remove_input_padding = True - - if args.mode not in ('plugin', 'plugin-ifb'): - build_config.plugin_config.smooth_quant_plugins = False - - if world_size > 1: - build_config.plugin_config.set_nccl_plugin(dtype=args.dtype) - - if args.multiple_profiles: - build_config.plugin_config.multiple_profiles = True - - # Enable trt monitor memory for perf tests - build_config.monitor_memory = args.monitor_memory - - start = time.time() - engine = build(tensorrt_llm_model, build_config) - assert engine.engine is not None, f'Failed to build engine for rank {runtime_rank}' - build_time = round(time.time() - start, 2) - - engine.save(args.output_dir) - - return engine, build_time - - -def build_bert(args): - family = get_model_family(args.model) - build_config = get_build_config(args.model, return_dict=False) - model_config = get_model_config(args.model) - if args.force_num_layer_1: - model_config['num_layers'] = 1 - - # More parameters - if args.serial_build and args.rank is not None and args.world_size is not None: - runtime_rank = args.rank - world_size = args.world_size - else: - runtime_rank = tensorrt_llm.mpi_rank() - world_size = tensorrt_llm.mpi_world_size() - if not args.serial_build: - torch.cuda.set_device(runtime_rank) - - num_kv_heads = model_config['num_heads'] \ - if model_config['num_kv_heads'] is None else model_config['num_kv_heads'] - max_batch_size = build_config.max_batch_size \ - if args.max_batch_size is None else args.max_batch_size - max_input_len = build_config.max_input_len \ - if args.max_input_len is None else args.max_input_len - bs_range = [1, (max_batch_size + 1) // 2, max_batch_size] - inlen_range = [1, (max_input_len + 1) // 2, max_input_len] - - is_weight_streaming = getattr(args, "weight_streaming", False) - - builder = Builder() - builder_config = builder.create_builder_config( - name=args.model, - precision=args.dtype, - timing_cache=args.input_timing_cache, - profiling_verbosity=args.profiling_verbosity, - tensor_parallel=world_size, # TP only - parallel_build=True, - num_layers=model_config['num_layers'], - num_heads=model_config['num_heads'], - num_kv_heads=num_kv_heads, - hidden_size=model_config['hidden_size'], - vocab_size=model_config['vocab_size'], - hidden_act=model_config['hidden_act'], - max_position_embeddings=model_config['n_positions'], - max_batch_size=max_batch_size, - max_input_len=max_input_len, - strongly_typed=True, - weight_streaming=is_weight_streaming, - monitor_memory=args.monitor_memory, - ) - engine_name = '{}_{}_tp{}_rank{}.engine'.format(args.model, args.dtype, - world_size, runtime_rank) - - # Initialize model - config = { - 'architecture': 'BertModel', - 'dtype': args.dtype, - 'num_hidden_layers': model_config['num_layers'], - 'num_attention_heads': model_config['num_heads'], - 'hidden_size': model_config['hidden_size'], - 'vocab_size': model_config['vocab_size'], - 'position_embedding_type': 'learned_absolute', - 'max_position_embeddings': model_config['n_positions'], - 'hidden_act': model_config['hidden_act'], - 'type_vocab_size': model_config['type_vocab_size'], - 'pad_token_id': - None if family == 'bert' else 1, # hard code for RoBERTa here - 'is_roberta': (family == 'roberta'), - 'mapping': { - 'world_size': world_size, - 'tp_size': world_size, - 'rank': runtime_rank - }, - } - config = PretrainedConfig.from_dict(config) - tensorrt_llm_bert = tensorrt_llm.models.BertModel(config) - - # Module -> Network - network = builder.create_network() - network.trt_network.name = engine_name - network.plugin_config.to_legacy_setting() - - # Plugins - if args.mode == 'plugin': - network.plugin_config.bert_attention_plugin = args.dtype - network.plugin_config.gemm_plugin = args.dtype - network.plugin_config.set_context_fmha(ContextFMHAType.enabled) - elif args.mode == 'ootb-except-mha': - network.plugin_config.bert_attention_plugin = args.dtype - network.plugin_config.set_context_fmha(ContextFMHAType.enabled) - - if world_size > 1: - network.plugin_config.set_nccl_plugin(dtype=args.dtype) - - with net_guard(network): - # Prepare - network.set_named_parameters(tensorrt_llm_bert.named_parameters()) - - # Forward - input_ids = tensorrt_llm.Tensor( - name='input_ids', - dtype=trt.int32, - shape=[-1, -1], - dim_range=OrderedDict([('batch_size', [bs_range]), - ('input_len', [inlen_range])]), - ) - input_lengths = tensorrt_llm.Tensor(name='input_lengths', - dtype=trt.int32, - shape=[-1], - dim_range=OrderedDict([ - ('batch_size', [bs_range]) - ])) - hidden_states = tensorrt_llm_bert(input_ids=input_ids, - input_lengths=input_lengths) - - # Mark outputs - hidden_states_dtype = str_dtype_to_trt(args.dtype) - hidden_states.mark_output('hidden_states', hidden_states_dtype) - - # Network -> Engine - start = time.time() - engine = builder.build_engine(network, builder_config) - assert engine is not None, f'Failed to build engine for rank {runtime_rank}' - build_time = round(time.time() - start, 2) - - if args.output_dir is not None: - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - serialize_path = os.path.join(args.output_dir, engine_name) - serialize_engine(engine, serialize_path) - if runtime_rank == 0: - config_path = os.path.join(args.output_dir, 'config.json') - builder_config.plugin_config = network.plugin_config - builder.save_config(builder_config, config_path) - if args.output_timing_cache: - # Save timing cache to output_dir if not absolute path - timing_cache_path = args.output_timing_cache if os.path.isabs( - args.output_timing_cache) else os.path.join( - args.output_dir, args.output_timing_cache) - ok = builder.save_timing_cache(builder_config, - timing_cache_path) - if not ok: - logger.warning("Failed to save timing cache.") - - return engine, build_time - - -def enc_dec_build_helper(component, build_config, model_config, args): - # More parameters - if args.serial_build and args.rank is not None and args.world_size is not None: - runtime_rank = args.rank - world_size = args.world_size - else: - runtime_rank = tensorrt_llm.mpi_rank() - world_size = tensorrt_llm.mpi_world_size() - if not args.serial_build: - torch.cuda.set_device(runtime_rank) - - family = get_model_family(args.model) - logits_dtype = 'float32' - if family == 'bart': - q_scaling = 1.0 - has_attention_qkvo_bias = True - has_mlp_bias = True - has_model_final_layernorm = False - has_position_embedding = True - has_embedding_layernorm = True - layernorm_type = LayerNormType.LayerNorm - relative_attention = False - layernorm_position = LayerNormPositionType.pre_layernorm if model_config.get( - 'normalize_before', True) else LayerNormPositionType.post_layernorm - rescale_before_lm_head = False - elif family == 'whisper': - q_scaling = 1.0 - has_position_embedding = True - relative_attention = False - has_embedding_layernorm = False - has_attention_qkvo_bias = True - has_mlp_bias = True - has_model_final_layernorm = True - layernorm_position = LayerNormPositionType.pre_layernorm - layernorm_type = LayerNormType.LayerNorm - rescale_before_lm_head = False - logits_dtype = args.dtype - model_config['n_mels'] - else: - q_scaling = 1 / model_config['head_size']**.5 - has_attention_qkvo_bias = False - has_mlp_bias = False - has_model_final_layernorm = True - has_position_embedding = False - has_embedding_layernorm = False - layernorm_type = LayerNormType.RmsNorm - relative_attention = True - layernorm_position = LayerNormPositionType.pre_layernorm - if family == 't5': - rescale_before_lm_head = True - else: - rescale_before_lm_head = False - - quant_config = get_quant_config(args.quantization) - quant_mode = quant_config.quant_mode - use_weight_only = quant_mode.is_weight_only() - - # Plugins - build_config.plugin_config.to_legacy_setting() - if args.mode in ['plugin', 'plugin-ifb']: - build_config.plugin_config.bert_attention_plugin = args.dtype - build_config.plugin_config.gpt_attention_plugin = args.dtype - build_config.plugin_config.set_context_fmha(ContextFMHAType.enabled) - build_config.plugin_config.gemm_plugin = args.dtype - build_config.plugin_config.remove_input_padding = True - build_config.plugin_config.enable_paged_kv_cache() - build_config.plugin_config.paged_state = True - if use_weight_only: - build_config.plugin_config.weight_only_quant_matmul_plugin = args.dtype - elif args.mode == 'ootb-except-mha': - build_config.plugin_config.bert_attention_plugin = args.dtype - build_config.plugin_config.gpt_attention_plugin = args.dtype - build_config.plugin_config.set_context_fmha(ContextFMHAType.enabled) - - if world_size > 1: - build_config.plugin_config.set_nccl_plugin(dtype=args.dtype) - - # build engine - mapping = Mapping(world_size=world_size, - rank=runtime_rank, - tp_size=world_size, - pp_size=1) # TP only - - if component == 'encoder': - if family == 'whisper': - pretrained_config = PretrainedConfig.from_dict({ - 'architecture': - "WhisperEncoder", - 'dtype': - args.dtype, - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'hidden_size': - model_config['hidden_size'], - 'has_position_embedding': - has_position_embedding, - 'n_mels': - model_config['n_mels'], - 'max_position_embeddings': - 1500, - 'vocab_size': - model_config['vocab_size'], - 'hidden_act': - "gelu", - 'num_languages': - 100, - 'mapping': { - 'world_size': mapping.world_size, - 'tp_size': mapping.tp_size, - 'pp_size': mapping.pp_size, - 'rank': mapping.rank, - }, - }) - tllm_model = tensorrt_llm.models.WhisperEncoder(pretrained_config) - if use_weight_only: - tllm_model = quantize(tllm_model, quant_config) - else: - pretrained_config = PretrainedConfig.from_dict({ - 'architecture': - "EncoderModel", - 'dtype': - args.dtype, - 'logits_dtype': - logits_dtype, - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'hidden_size': - model_config['hidden_size'], - 'norm_epsilon': - 1e-6, - 'vocab_size': - model_config['vocab_size'], - 'hidden_act': - model_config['hidden_act'], - 'mapping': { - 'world_size': mapping.world_size, - 'tp_size': mapping.tp_size, - 'pp_size': mapping.pp_size, - 'rank': mapping.rank, - }, - 'use_parallel_embedding': - False, - 'embedding_sharding_dim': - 0, - 'max_position_embeddings': - model_config.get('n_positions', 0), - 'use_prompt_tuning': - False, - 'head_size': - model_config['head_size'], - 'has_position_embedding': - has_position_embedding, - 'layernorm_type': - layernorm_type, - 'has_attention_qkvo_bias': - has_attention_qkvo_bias, - 'has_mlp_bias': - has_mlp_bias, - 'has_model_final_layernorm': - has_model_final_layernorm, - 'has_embedding_layernorm': - has_embedding_layernorm, - 'has_embedding_scale': - model_config.get('has_embedding_scale', False), - 'intermediate_size': - model_config['ffn_hidden_size'], - 'q_scaling': - q_scaling, - 'layernorm_position': - layernorm_position, - 'relative_attention': - relative_attention, - 'max_distance': - model_config.get('max_distance', 0), - 'num_buckets': - model_config.get('num_buckets', 0), - 'model_type': - family, - }) - tllm_model = tensorrt_llm.models.EncoderModel(pretrained_config) - elif component == 'decoder': - pretrained_config = PretrainedConfig.from_dict({ - 'architecture': - "DecoderModel", - 'dtype': - args.dtype, - 'logits_dtype': - logits_dtype, - 'num_hidden_layers': - model_config['num_layers'], - 'num_attention_heads': - model_config['num_heads'], - 'hidden_size': - model_config['hidden_size'], - 'norm_epsilon': - 1e-6, - 'vocab_size': - model_config['vocab_size'], - 'hidden_act': - model_config['hidden_act'], - 'mapping': { - 'world_size': mapping.world_size, - 'tp_size': mapping.tp_size, - 'pp_size': mapping.pp_size, - 'rank': mapping.rank, - }, - 'use_parallel_embedding': - False, - 'embedding_sharding_dim': - 0, - 'max_position_embeddings': - model_config.get('n_positions', 0), - 'use_prompt_tuning': - False, - 'head_size': - model_config['head_size'], - 'has_position_embedding': - has_position_embedding, - 'layernorm_type': - layernorm_type, - 'has_attention_qkvo_bias': - has_attention_qkvo_bias, - 'has_mlp_bias': - has_mlp_bias, - 'has_model_final_layernorm': - has_model_final_layernorm, - 'has_embedding_layernorm': - has_embedding_layernorm, - 'has_embedding_scale': - model_config.get('has_embedding_scale', False), - 'intermediate_size': - model_config['ffn_hidden_size'], - 'q_scaling': - q_scaling, - 'layernorm_position': - layernorm_position, - 'relative_attention': - relative_attention, - 'max_distance': - model_config.get('max_distance', 0), - 'num_buckets': - model_config.get('num_buckets', 0), - 'model_type': - family, - 'rescale_before_lm_head': - rescale_before_lm_head, - 'encoder_hidden_size': - model_config['hidden_size'], - 'encoder_num_heads': - model_config['num_heads'], - 'encoder_head_size': - model_config['head_size'], - 'skip_cross_kv': - model_config['skip_cross_kv'], - 'use_implicit_relative_attention': - model_config['use_implicit_relative_attention'], - 'decoder_start_token_id': - model_config['decoder_start_token_id'], - }) - tllm_model = tensorrt_llm.models.DecoderModel(pretrained_config) - if use_weight_only and family == 'whisper': - tllm_model = quantize(tllm_model, quant_config) - - tllm_model.precompute_relative_attention_bias(build_config) - - start = time.time() - engine = build(tllm_model, build_config) - assert engine.engine is not None, f'Failed to build engine for rank {runtime_rank}' - build_time = round(time.time() - start, 2) - - engine.save(os.path.join(args.output_dir, component)) - - return engine, model_config, build_time - - -def build_enc_dec(args): - build_config = get_build_config(args.model) - build_config = BuildConfig(**build_config) - model_config = get_model_config(args.model) - if args.force_num_layer_1: - model_config['num_layers'] = 1 - - if args.profiling_verbosity != "layer_names_only": - build_config.profiling_verbosity = args.profiling_verbosity - - if args.max_batch_size is not None: - build_config.max_batch_size = args.max_batch_size - if args.max_input_len is not None: - build_config.max_encoder_input_len = args.max_input_len - build_config.max_input_len = args.max_input_len - if args.max_seq_len is not None: - build_config.max_seq_len = args.max_seq_len - if args.max_beam_width is not None: - build_config.max_beam_width = args.max_beam_width - if args.opt_batch_size is not None: - build_config.opt_batch_size = args.opt_batch_size - if args.opt_num_tokens is not None: - build_config.opt_num_tokens = args.opt_num_tokens - build_config.max_num_tokens = build_config.max_batch_size * max( - build_config.max_encoder_input_len, build_config.max_beam_width) - - encoder_max_seq_len = build_config.max_encoder_input_len - decoder_max_seq_len = build_config.max_seq_len - - # Enable trt monitor memory for perf tests - build_config.monitor_memory = args.monitor_memory - - # for encoder, input len and output len both equal to max_encoder_input_len - build_config.max_input_len = encoder_max_seq_len - build_config.max_seq_len = encoder_max_seq_len - encoder_engine, encoder_model_config, encoder_build_time = enc_dec_build_helper( - component='encoder', - build_config=build_config, - model_config=model_config, - args=args) - - # for decoder, input len equals to 1 and output len equals to max_seq_len - build_config.max_input_len = 1 - build_config.max_seq_len = decoder_max_seq_len - decoder_engine, decoder_model_config, decoder_build_time = enc_dec_build_helper( - component='decoder', - build_config=build_config, - model_config=model_config, - args=args) - - return encoder_engine, decoder_engine, encoder_model_config, decoder_model_config, encoder_build_time, decoder_build_time - - -def main(args): - logger.set_level(args.log_level) - if args.model in get_allowed_models(benchmark_type="gpt"): - engine = build_gpt(args)[0] - engine_size = engine.engine.nbytes - elif args.model in get_allowed_models(benchmark_type="bert"): - engine = build_bert(args)[0] - engine_size = engine.nbytes - elif args.model in get_allowed_models(benchmark_type="enc_dec"): - encoder_engine, decoder_engine = build_enc_dec(args)[:2] - engine_size = encoder_engine.engine.nbytes + decoder_engine.engine.nbytes - else: - raise Exception(f'Unexpected model: {args.model}') - - # Print engine size for CI/CD to track. - logger.info( - f"Total engine size per GPU is {engine_size / 1048576:.2f} MiB.") - - -if __name__ == '__main__': - mp.set_start_method('spawn') - args = parse_arguments() - main(args) diff --git a/tests/integration/defs/perf/pytorch_model_config.py b/tests/integration/defs/perf/pytorch_model_config.py index 386fd72021c3..97d56c55ea22 100644 --- a/tests/integration/defs/perf/pytorch_model_config.py +++ b/tests/integration/defs/perf/pytorch_model_config.py @@ -26,7 +26,6 @@ # "DeepGEMM only supports Hopper (SM90)". _DEEPSEEK_FP8_BLOCK_SCALE_MODELS = ( 'deepseek_v3_lite_fp8', - 'deepseek_r1_fp8', 'deepseek_r1_0528_fp8', ) @@ -90,144 +89,58 @@ def get_model_yaml_config(model_label: str, 'enable_attention_dp': True, } }, - # DeepSeek R1 models with MTP speculative decoding + # DeepSeek V4 Flash uses TRTLLM for MXFP4 routed experts. { - 'patterns': [ - 'deepseek_r1-bench-pytorch-float16-maxbs:1-maxnt:8192-input_output_len:1000,2000-reqs:10-ep:4-gpus:8', - 'deepseek_r1_nvfp4-bench-pytorch-float16-maxbs:1-maxnt:8192-input_output_len:1000,2000-reqs:10-ep:4-tp:8-gpus:8' - ], + 'patterns': ['deepseek_v4_flash-bench'], 'config': { 'enable_attention_dp': True, - 'cuda_graph_config': {}, - 'speculative_config': { - 'decoding_type': 'MTP', - 'max_draft_len': 3 - } - } - }, - { - 'patterns': [ - 'deepseek_r1_nvfp4-bench-pytorch-float4-maxbs:32-maxnt:32768-input_output_len:8192,1024-reqs:20-con:1-ep:1-gpus:4' - ], - 'config': { - 'enable_iter_perf_stats': True, - 'print_iter_log': False, - 'cuda_graph_config': { - 'max_batch_size': 16, - 'enable_padding': False - }, 'moe_config': { - 'max_num_tokens': 32768 + 'backend': 'TRTLLM', }, - 'speculative_config': { - 'decoding_type': 'MTP', - 'max_draft_len': 3 - }, - 'disable_overlap_scheduler': True, - 'enable_autotuner': True, + 'max_seq_len': 10240, + 'max_num_tokens': 4096, + 'enable_chunked_prefill': True, 'kv_cache_config': { - 'free_gpu_memory_fraction': 0.6, - 'enable_block_reuse': True, - 'enable_partial_reuse': False + 'free_gpu_memory_fraction': 0.5, }, - 'enable_chunked_prefill': True - } - }, - # DeepSeek R1 models with large batch sizes and cuda graph padding - { - 'patterns': [ - 'deepseek_r1_fp8-bench-pytorch-float16-maxbs:384-maxnt:1536-input_output_len:1000,2000-reqs:49152-con:3072-ep:8-gpus:8', - 'deepseek_r1_nvfp4-bench-pytorch-float16-maxbs:384-maxnt:1536-input_output_len:1000,2000-reqs:49152-con:3072-ep:8-gpus:8' - ], - 'config': { - 'enable_attention_dp': True, - 'cuda_graph_config': { - 'enable_padding': True, - 'batch_sizes': [1, 2, 4, 8, 16, 32, 64, 128, 256, 384] - } } }, - # DeepSeek R1 model with specific batch size 128 + # DeepSeek V4 Flash-Base leaves the MoE backend to AUTO (TRTLLM on Blackwell). { - 'patterns': - 'deepseek_r1_fp8-bench-pytorch-float16-maxbs:128-maxnt:1127-input_output_len:1000,2000-reqs:5120-con:1024-ep:8-gpus:8', + 'patterns': ['deepseek_v4_flash_base'], 'config': { 'enable_attention_dp': True, - 'cuda_graph_config': { - 'batch_sizes': [128] - } + 'max_seq_len': 10240, } }, - # Deepseek R1 model with chunked prefill + # DeepSeek V4 Pro DSpark mirrors the upstream 8-GPU accuracy configuration. { - 'patterns': [ - 'deepseek_r1_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-kv_frac:0.85-input_output_len:5000,500', - 'deepseek_r1_fp8-bench-pytorch-float8-maxbs:256-maxnt:1024-kv_frac:0.85-input_output_len:2000,2000', - 'deepseek_v3_lite_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-kv_frac:0.85-input_output_len:5000,500', - 'deepseek_v3_lite_nvfp4-bench-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.85-input_output_len:5000,500', - 'deepseek_r1_nvfp4-bench-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.85-input_output_len:5000,500', - 'deepseek_r1_nvfp4-bench-pytorch-float4-maxbs:256-maxnt:1024-kv_frac:0.85-input_output_len:2000,2000', - ], - 'config': { - 'enable_attention_dp': True, - 'enable_chunked_prefill': True, - } - }, - # Deepseek R1 NVFP4 with chunked prefill, large seq len, and fp8 KV cache - { - 'patterns': [ - 'deepseek_r1_nvfp4-bench-pytorch-float4-maxbs:32-maxnt:4096-kv_frac:0.80-input_output_len:8192,512-reqs:3000-ep:2-tp:4-gpus:4', - ], - 'config': { - 'enable_attention_dp': True, - 'enable_chunked_prefill': True, - 'max_num_tokens': 4096, - 'max_batch_size': 32, - 'max_seq_len': 81920, - 'kv_cache_config': { - 'dtype': 'fp8', - 'free_gpu_memory_fraction': 0.80, - 'enable_block_reuse': False, - }, - 'cuda_graph_config': { - 'enable_padding': True, - 'max_batch_size': 32, - }, - } - }, - # Deepseek R1 model with CUTLASS backend - { - 'patterns': [ - 'deepseek_r1_nvfp4-bench-pytorch-streaming-float4-maxbs:512-maxnt:5220-input_output_len:4000,2000', - ], + 'patterns': ['deepseek_v4_pro_dspark'], 'config': { + 'attn_backend': 'TRTLLM', 'enable_attention_dp': True, 'moe_config': { - 'backend': 'CUTLASS', - 'max_num_tokens': 3072, + 'backend': 'MEGAMOE_DEEPGEMM', }, + 'max_seq_len': 10240, + 'max_num_tokens': 9216, 'kv_cache_config': { - 'dtype': 'fp8', + 'enable_block_reuse': False, 'free_gpu_memory_fraction': 0.5, }, - 'cuda_graph_config': { - 'enable_padding': True, - 'batch_sizes': [1, 2, 4, 8, 16, 32, 64], + 'enable_chunked_prefill': False, + 'disable_overlap_scheduler': True, + 'custom_tokenizer': 'deepseek_v4', + 'speculative_config': { + 'decoding_type': + 'DSpark', + 'max_draft_len': + 5, + 'speculative_model': + f'{llm_models_root()}/DeepSeek-V4-Pro-DSpark', }, } }, - # Deepseek_v3_lite_cases - { - 'patterns': - 'deepseek_v3_lite_nvfp4-bench-pytorch-streaming-float4-maxbs:2048-maxnt:8192-input_output_len:256,256-reqs:200', - 'config': { - 'print_iter_log': True, - 'cuda_graph_config': { - 'enable_padding': True, - 'batch_sizes': [1, 512, 1024, 2048] - } - } - }, # Model-specific cases with attention_dp disabled to prevent hangs { 'patterns': [ @@ -251,45 +164,54 @@ def get_model_yaml_config(model_label: str, 'enable_attention_dp': True, } }, - # MiniMax-M2.5 FP8: route MoE through attention DP. + # Qwen3.6-35B-A3B NVFP4 GDN-attn MoE: trust_remote_code, TRTLLM-Gen NVFP4 MoE (SM100/103), block reuse off. + { + 'patterns': ['qwen3.6_35b_a3b_fp4'], + 'config': { + 'trust_remote_code': True, + 'moe_config': { + 'backend': 'TRTLLM', + }, + 'kv_cache_config': { + 'enable_block_reuse': False, + }, + } + }, + # MiniMax-M2.5 FP8: every perf case must route MoE through attention DP. # TP=8: intermediate_size=1536 is not block-scale divisible (1536/8=192, %128!=0). # TP=4: trtllm-gen FP8 block-scale MoE kernel IMAs during CUDA-graph warmup # on the 1536/4=384 N-shard (Blackwell B200/B300). { - 'patterns': [ - 'minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-gpus:8', - 'minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:500,2000-ep:8-gpus:8', - 'minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:2000,500-ep:8-gpus:8', - 'minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:1000,1000-ep:8-gpus:8', - 'minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-gpus:8', - 'minimax_m2.5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:8-gpus:8', - 'minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-input_output_len:1000,1000-con:512-ep:8-gpus:8', - 'minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:128,128-gpus:4', - 'minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:500,2000-gpus:4', - 'minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:2000,500-gpus:4', - 'minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:1000,1000-gpus:4', - 'minimax_m2.5_fp8-bench-pytorch-float8-maxbs:1-maxnt:2048-input_output_len:1000,1000-reqs:10-con:1-gpus:4', - 'minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:1000,1000-con:256-gpus:4', - ], + 'patterns': ['minimax_m2.5_fp8'], 'config': { 'enable_attention_dp': True, } }, + # MiniMax-M3 MXFP8 block-sparse MoE: sparse backend, no KV reuse, trust_remote_code, capped max_seq_len to avoid the 1M-default CUDA-graph OOM. { - 'patterns': [ - 'qwen3_4b-bench-pytorch-streaming-bfloat16-maxbs:4-kv_frac:0.6-input_output_len:500,100-reqs:200-con:4', - ], + 'patterns': ['minimax_m3_mxfp8'], 'config': { - 'speculative_config': { - 'decoding_type': 'Eagle', - 'eagle3_one_model': True, - 'speculative_model': 'Qwen3-4B_eagle3', - 'max_draft_len': 3, + 'enable_attention_dp': True, + 'trust_remote_code': True, + 'max_seq_len': 4096, + 'sparse_attention_config': { + 'algorithm': 'minimax_m3', }, 'kv_cache_config': { 'enable_block_reuse': False, }, - 'enable_chunked_prefill': False, + } + }, + # MiniMax-M3 8000,1000 cases need max_seq_len >= ISL+OSL (9000). + # Patterns must be written in PerfTestConfig.to_string() form: it always + # injects maxbs:/maxnt: and drops tp: when tp_size == num_gpus. + { + 'patterns': [ + 'minimax_m3_mxfp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:8000,1000', + 'minimax_m3_mxfp8-bench-pytorch-float8-maxbs:1-maxnt:2048-input_output_len:8000,1000', + ], + 'config': { + 'max_seq_len': 9216, } }, # Qwen3-235B-A22B-FP4 with Eagle3 speculative decoding @@ -315,17 +237,9 @@ def get_model_yaml_config(model_label: str, }, } }, - # Llama-v3.3 models with fp8 quantization + # Llama-v4 Scout FP4 with cuda graph padding { - 'patterns': [ - 'llama_v3.3_70b_instruct_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:500,2000-gpus:4', - 'llama_v3.3_70b_instruct_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:1000,1000-gpus:4', - 'llama_v3.3_70b_instruct_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:2000,500-gpus:4', - 'llama_v3.3_70b_instruct_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:128,128-gpus:4', - 'llama_v3.3_70b_instruct_fp8-bench-pytorch-bfloat16-maxbs:512-maxnt:2048-input_output_len:512,32-gpus:4', - 'llama_v4_scout_17b_16e_instruct_fp4', - 'llama_v4_maverick_17b_128e_instruct_fp8' - ], + 'patterns': ['llama_v4_scout_17b_16e_instruct_fp4'], 'config': { 'cuda_graph_config': { 'enable_padding': @@ -342,7 +256,6 @@ def get_model_yaml_config(model_label: str, 'patterns': [ 'gpt_oss_120b_fp4-bench-pytorch-float4-maxbs:720-maxnt:16384-input_output_len:1024,1024-reqs:1280-con:256', 'gpt_oss_120b_fp4-bench-pytorch-float4-maxbs:720-maxnt:16384-input_output_len:1024,1024-reqs:2560-con:512', - 'gpt_oss_120b_fp4-bench-pytorch-float4-maxbs:720-maxnt:16384-input_output_len:1024,1024-reqs:5120-con:1024', 'gpt_oss_120b_fp4-bench-pytorch-float4-maxbs:720-maxnt:16384-input_output_len:1024,1024-reqs:20480-con:4096' ], 'config': { @@ -422,42 +335,9 @@ def get_model_yaml_config(model_label: str, }, } }, - # Phi-4-multimodal-instruct with chunked prefill and kv_cache_reuse - { - 'patterns': [ - 'phi_4_multimodal_instruct-bench-pytorch-bfloat16-maxbs:48-maxnt:256-input_output_len:500,2000-con:250', - 'phi_4_multimodal_instruct-bench-pytorch-bfloat16-maxbs:128-maxnt:512-input_output_len:1000,1000-con:250' - ], - 'config': { - 'enable_chunked_prefill': True, - } - }, - # Mistral-Small-3.1-24B-Instruct-2503 with chunked prefill and kv_cache_reuse - { - 'patterns': [ - 'mistral_small_v3.1_24b-bench-pytorch-bfloat16-maxbs:48-maxnt:256-input_output_len:1000,2000-reqs:500-con:200', - 'mistral_small_v3.1_24b-bench-pytorch-bfloat16-maxbs:128-maxnt:512-input_output_len:1000,2000-reqs:500-con:200' - ], - 'config': { - 'enable_chunked_prefill': True, - } - }, - # Llama-v3.3 models with xgrammar guided decoding - { - 'patterns': [ - "llama_v3.3_70b_instruct_fp8-bench-float8-maxbs:512-maxnt:2048-input_output_len:500,2000-reqs:400-con:200-gpus:8-extra" - ], - 'config': { - 'extended_runtime_perf_knob_config': { - 'cuda_graph_cache_size': 1.0, - 'cuda_graph_mode': True, - }, - 'guided_decoding_backend': 'xgrammar' - } - }, # Gemma3 models require FlashInfer backend due to sliding window attention { - 'patterns': ['gemma_3', 'gemma3'], + 'patterns': ['gemma_3'], 'config': { 'attn_backend': 'FLASHINFER', } @@ -562,36 +442,6 @@ def get_model_yaml_config(model_label: str, }, } }, - # Nemotron-3-Super-120B-NVFP4_MTP (throughput variant with MTP spec decoding) - # Pattern is intentionally narrowed so it does NOT match the - # '_mtp-serve-pytorch-streaming-' streaming variant above. - { - 'patterns': ['nemotron_3_super_120b_nvfp4_mtp-serve-pytorch-float'], - 'config': { - 'max_seq_len': 1048576, - 'enable_chunked_prefill': True, - 'enable_attention_dp': True, - 'stream_interval': 1, - 'moe_config': { - 'backend': 'CUTLASS', - }, - 'cuda_graph_config': { - 'enable_padding': True, - 'max_batch_size': 256, - }, - 'kv_cache_config': { - 'enable_block_reuse': False, - 'mamba_ssm_cache_dtype': 'float16', - 'mamba_ssm_stochastic_rounding': True, - 'mamba_ssm_philox_rounds': 5, - }, - 'speculative_config': { - 'decoding_type': 'MTP', - 'num_nextn_predict_layers': 3, - 'allow_advanced_sampling': True, - }, - } - }, # Nemotron-3-Ultra-550B-NVFP4 throughput variant, aligned with curated yaml (served from HF). { 'patterns': ['nemotron_3_ultra_550b_nvfp4-serve-pytorch-'], diff --git a/tests/integration/defs/perf/sampler_options_config.py b/tests/integration/defs/perf/sampler_options_config.py index 91f591e6f74b..de1824c8eced 100644 --- a/tests/integration/defs/perf/sampler_options_config.py +++ b/tests/integration/defs/perf/sampler_options_config.py @@ -26,15 +26,12 @@ def get_sampler_options_config(model_label: str) -> dict: Returns: dict: sampler options config """ + # Labels are compared for equality, so they must be spelled exactly as + # PerfTestConfig.to_string() emits them: maxbs:/maxnt: are always injected + # and tp: is dropped when tp_size == num_gpus. base_config = {} if model_label in [ - 'llama_v3.2_1b-bench-pytorch-bfloat16-maxbs:512-maxnt:2048-input_output_len:500,2000-gpus:2', - 'llama_v3.3_70b_instruct_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:128,128-gpus:4', - 'llama_v4_maverick_17b_128e_instruct_fp8-bench-pytorch-float8-maxbs:1024-maxnt:20000-kv_frac:0.6-input_output_len:20000,2000-reqs:1000-ep:8-gpus:8', - 'llama_v4_maverick_17b_128e_instruct_fp8-bench-pytorch-float8-maxbs:1024-maxnt:4096-kv_frac:0.85-input_output_len:1000,1000-reqs:3000-ep:8-gpus:8', - 'llama_v4_scout_17b_16e_instruct_fp8-bench-pytorch-float8-maxbs:1024-maxnt:4096-kv_frac:0.85-input_output_len:500,2000-reqs:3000-ep:8-gpus:8', - 'mistral_small_v3.1_24b-bench-pytorch-bfloat16-maxbs:512-maxnt:2048-input_output_len:1000,2000-reqs:500-con:200-gpus:2', - 'phi_4_mini_instruct-bench-pytorch-bfloat16-maxbs:512-maxnt:2048-input_output_len:128,128' + 'llama_v3.3_70b_instruct_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-input_output_len:128,128-gpus:8', ]: base_config['top_k'] = 4 base_config['top_p'] = 0.5 diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index 6d3fb5093813..58e1be73cd35 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -65,6 +65,8 @@ "nemotron_3_super_120b_nvfp4_mtp", "nemotron_3_ultra_550b_nvfp4", "glm_5_fp8", + "minimax_m3_mxfp8", + "qwen3.6_35b_a3b_fp4", "nemotron_3_nano_omni_nvfp4", "nemotron_3_nano_omni_nvfp4_image", "nemotron_nano_12b_v2", @@ -156,6 +158,11 @@ def import_allowed_perf_config(): return allowed_configs +# Covers both the C++ KVCacheManager and the Python KV cache manager v2 log lines. +KV_CACHE_SIZE_LOG_QUERY = re.compile( + r".*Allocated ([\d\.]+) GiB for max tokens in paged KV cache.*" + r"|.*KV cache manager v2 device quota set to ([\d\.]+)\s*GiB.*") + # Regex commands used to parse the metric result for the metric type. PERF_METRIC_LOG_QUERIES = { PerfMetricType.BUILD_TIME: @@ -187,7 +194,7 @@ def import_allowed_perf_config(): PerfMetricType.CONTEXT_GPU_MEMORY: re.compile(r".*Allocated ([\d\.]+) MiB for execution context memory.*"), PerfMetricType.KV_CACHE_SIZE: - re.compile(r".*Allocated ([\d\.]+) GiB for max tokens in paged KV cache.*"), + KV_CACHE_SIZE_LOG_QUERY, } BENCH_PERF_METRIC_LOG_QUERIES = { @@ -205,9 +212,11 @@ def import_allowed_perf_config(): re.compile(r"Average time-to-first-token \[TTFT\] \(ms\):\s+([\d\.]+)"), PerfMetricType.OUTPUT_TOKEN_TIME: re.compile(r"Average time-per-output-token \[TPOT\] \(ms\):\s+([\d\.]+)"), + # AutoDeploy builds its KVCacheManager from the same shared C++ class (see + # tensorrt_llm/_torch/auto_deploy/shim/interface.py), so its post-resize + # capacity also logs this line (max() below picks that final value). PerfMetricType.KV_CACHE_SIZE: - re.compile(r".*(?:Allocated ([\d\.]+) GiB for max tokens in paged KV cache|" - r"Final KV cache size after resize: ([\d\.]+) GiB).*"), + KV_CACHE_SIZE_LOG_QUERY, PerfMetricType.PER_USER_OUTPUT_THROUGHPUT: re.compile( r"Per User Output Throughput \[w\/ ctx\] \(tps\/user\):\s+([\d\.]+)"), @@ -248,6 +257,9 @@ def import_allowed_perf_config(): re.compile(r"Median E2EL \(ms\):\s+(-?[\d\.]+)"), PerfMetricType.P99_INFERENCE_TIME: re.compile(r"P99 E2EL \(ms\):\s+(-?[\d\.]+)"), + # Printed by the KV cache manager on server startup, same as trtllm-bench. + PerfMetricType.KV_CACHE_SIZE: + KV_CACHE_SIZE_LOG_QUERY, } # (Relative threshold, Absolute threshold) for all metric types @@ -954,9 +966,6 @@ def set_runtime_configs(self, else: raise RuntimeError(f"Invalid runtime {self._config.runtime}.") - allowed_configs = import_allowed_perf_config() - allowed_models = allowed_configs.get_allowed_models() - if self._config.runtime == "bench": build_script = "trtllm-bench" elif self._config.runtime == "serve": @@ -965,12 +974,10 @@ def set_runtime_configs(self, build_script = None elif self._config.runtime == "multi_node_disagg_server": build_script = None - elif self._config.pp_size > 1 or self._config.model_name not in allowed_models: - build_script = "trtllm-build" else: - # build.py is used to build engines for both python and cpp runtime - build_script = os.path.join(llm_root, - "tests/integration/defs/perf/build.py") + raise RuntimeError( + f"Invalid runtime {self._config.runtime}: engine-build flows " + "were removed with the legacy TensorRT backend.") self._build_script = build_script self._benchmark_script = benchmark_script @@ -980,56 +987,6 @@ def set_runtime_configs(self, self._llm_root = llm_root self._gpu_clock_lock = gpu_clock_lock - def get_trtllm_build_command(self, engine_dir, checkpoint_dir) -> list: - build_cmd = [ - self._build_script, f"--output_dir={engine_dir}", - f"--checkpoint_dir={checkpoint_dir}", - f"--workers={self._config.tp_size}", - f"--use_paged_context_fmha=enable", f"--monitor_memory", - f"--max_batch_size={self._config.max_batch_size}" - ] - # For Multiple Profiles - if self._config.multiple_profiles: - build_cmd.append(f"--multiple_profiles=enable") - else: - build_cmd.append(f"--multiple_profiles=disable") - num_beams = self._config.num_beams - if num_beams > 1: - build_cmd.append(f"--max_beam_width={num_beams}") - gpu_percent = self._config.gpu_weights_percent - if gpu_percent != -1: - build_cmd += [f"--weight_streaming"] - # For engine inspector - build_cmd.append("--profiling_verbosity=layer_names_only") - if self._config.num_loras > 0: - if "mixtral" in self._config.model_name: - build_cmd.append(f"--lora_plugin=auto") - build_cmd.append(f"--moe_plugin=auto") - build_cmd.append(f"--lora_target_modules") - build_cmd.append(f"attn_q") - build_cmd.append(f"attn_k") - build_cmd.append(f"attn_v") - build_cmd.append(f"attn_dense") - build_cmd.append(f"moe_h_to_4h") - build_cmd.append(f"moe_4h_to_h") - build_cmd.append(f"moe_gate") - build_cmd.append(f"moe_router") - elif "llama" in self._config.model_name: - build_cmd.append(f"--lora_plugin=float16") - build_cmd.append(f"--lora_target_modules") - build_cmd.append(f"attn_q") - build_cmd.append(f"attn_k") - build_cmd.append(f"attn_v") - build_cmd.append(f"attn_dense") - build_cmd.append(f"mlp_h_to_4h") - build_cmd.append(f"mlp_4h_to_h") - build_cmd.append(f"mlp_gate") - if TIMING_CACHE_DIR and not self._config.build_only: - timing_cache = os.path.join(TIMING_CACHE_DIR, "model.cache") - build_cmd.append(f"--input_timing_cache={timing_cache}") - build_cmd.append(f"--output_timing_cache={timing_cache}") - return build_cmd - def get_trtllm_bench_model(self): return get_model_dir(self._config.model_name) @@ -1066,8 +1023,6 @@ def get_trtllm_bench_build_command(self, engine_dir) -> list: def get_prepare_data_command(self, engine_dir, input_len, output_len) -> list: data_cmd = [] - prepare_data_script = os.path.join(self._llm_root, "benchmarks", "cpp", - "prepare_dataset.py") if self._config.model_name in MODEL_PATH_DICT.keys(): tokenizer_dir = os.path.join( @@ -1153,13 +1108,9 @@ def get_prepare_data_command(self, engine_dir, input_len, f"--input-stdev={istdev}", f"--output-stdev={ostdev}" ] else: - data_cmd += [ - "python3", prepare_data_script, f"--output={dataset_path}", - f"--tokenizer={tokenizer_dir}", f"token-norm-dist", - f"--num-requests={self._config.num_reqs}", - f"--input-mean={input_len}", f"--output-mean={output_len}", - f"--input-stdev={istdev}", f"--output-stdev={ostdev}" - ] + raise RuntimeError( + f"Unsupported build script {self._build_script} for " + "dataset preparation.") return data_cmd @@ -1679,22 +1630,10 @@ def run_metrics(self, llm_venv, gpu_clock_lock, session_data_writer, if result_state != "valid": errors.append(self.get_error()) - if self._config.runtime == 'serve': - print_info("Starting serve server") - outputs = self.run_ex(commands=commands, - cmd_idx=self._current_cmd_idx, - full_test_name="start_server", - metric_type=None, - venv=llm_venv, - gpu_clock_lock=gpu_clock_lock, - session_data_writer=session_data_writer, - output_dir=output_dir, - outputs=outputs, - original_test_name="start_server") - result_state = self.get_result_state() - result_states[self._current_cmd_idx] = result_state - if result_state != "valid": - errors.append(self.get_error()) + # Note: unlike 'bench', 'serve' has no separate setup step here. The + # server is started as part of processing the first metric below + # (KV_CACHE_SIZE, cmd_idx=0) so that its startup log is captured and + # can be parsed, instead of being discarded as a setup command. try: for metric in metrics: @@ -1786,7 +1725,9 @@ def _get_metrics(self) -> List[PerfTestMetric]: cmd_idx = 0 if self._config.runtime == "serve": - builder_metrics = [] + # No engine build step, but the server start command (cmd_idx=0) + # still emits the KV cache size, so collect it like trtllm-bench does. + builder_metrics = [PerfMetricType.KV_CACHE_SIZE] print_info( f"Skip building process for {self._config.model_name} as serve handles model loading" ) diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index a1bfbb7b22eb..4e86d5a34eba 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -28,7 +28,7 @@ import pytest import yaml from test_common.error_utils import report_error -from test_common.http_utils import wait_for_endpoint_ready +from test_common.http_utils import fail_if_proc_died, wait_for_endpoint_ready from defs.trt_test_alternative import print_info from tensorrt_llm._utils import get_free_port @@ -96,6 +96,46 @@ def ensure_bench_serving_repo() -> str: DEFAULT_TIMEOUT = 10800 +# Defaults for the server *ready* wait, separate from the whole-test timeout: +# a server that is not healthy after this long is not going to be, and failing +# here (with server-log tails, see wait_for_endpoint_ready) instead of at the +# per-test pytest kill both saves GPU-hours and leaves a classifiable failure +# in the CI log. The disagg bound is larger because its /health only answers +# once EVERY ctx/gen worker has finished model load + autotune + warmup. +AGG_SERVER_READY_TIMEOUT = 1800 +DISAGG_SERVER_READY_TIMEOUT = 3600 + + +def server_ready_timeout(default: int, mode: str) -> int: + """Ready-wait bound for one serving mode ("AGG" or "DISAGG"). + + Agg and disagg servers have very different init times (disagg's /health + answers only after every ctx/gen worker is up), so each mode has its own + override var, with the generic one as a shared fallback: + TRTLLM_TEST__SERVER_READY_TIMEOUT > TRTLLM_TEST_SERVER_READY_TIMEOUT + > the built-in per-mode default. + + Read at call time (not import time) so the env vars can be adjusted per + invocation, and parsed defensively so a malformed value cannot break + pytest collection of this module. + """ + for var in ( + f"TRTLLM_TEST_{mode.upper()}_SERVER_READY_TIMEOUT", + "TRTLLM_TEST_SERVER_READY_TIMEOUT", + ): + raw = os.environ.get(var) + if not raw: + continue + try: + timeout = int(raw) + except ValueError: + timeout = 0 + if timeout > 0: + return timeout + print_info(f"Invalid {var}={raw!r}; ignoring it") + return default + + AGG_CONFIG_FOLDER = os.environ.get("AGG_CONFIG_FOLDER", "tests/scripts/perf-sanity/aggregated") DISAGG_CONFIG_FOLDER = os.environ.get( "DISAGG_CONFIG_FOLDER", "tests/scripts/perf-sanity/disaggregated" @@ -173,20 +213,26 @@ def _scan_gen_worker_device_step_time( output_dir: str, num_gen_servers: int, start_offsets: Optional[List[int]] = None, -) -> Tuple[List[Dict[int, Tuple[int, float]]], int]: +) -> Tuple[List[Tuple[Dict[int, Tuple[int, float]], int, float]], int]: """Single-pass scan of the gen logs. - Returns (per_file_by_ngen, total_count): - - per_file_by_ngen: one dict per file that produced >=1 usable line, - mapping num_generation_tokens -> (count, Welford mean of - prev_device_step_time) over rows with iter >= 5 and a numeric - prev_device_step_time. Rows lacking num_generation_tokens on the same - line are skipped for the mean but still counted for settle detection. + Returns (per_file_scans, total_count): + - per_file_scans: one entry per file that produced >=1 usable row, each + a tuple (by_ngen, all_count, all_mean): + * by_ngen maps num_generation_tokens -> (count, Welford mean of + prev_device_step_time) over rows with iter >= 5, a numeric + prev_device_step_time, and a parseable num_generation_tokens on + the same line. + * all_count / all_mean are the count and Welford mean of + prev_device_step_time over ALL iter >= 5 numeric rows in the file, + including those whose num_generation_tokens did not parse. This is + the fallback aggregate used when a worker never emits a parseable + num_generation_tokens (nvbugs 6487036 / 6487040): PR #16298 began + requiring num_generation_tokens on every line, so a worker whose + states dict renders it as e.g. tensor(256) would drop to no + buckets and the metric would wrongly parse to None. - total_count: the number of iter >= 5 rows with a numeric - prev_device_step_time across all files. This is monotonic as new - lines flush across NFS (rows only get appended) so the caller can use - it as the settle signal without worrying that changes to a per-ngen - filter can make it drop. + prev_device_step_time across all files. Memory is O(distinct num_generation_tokens per file), a small constant in practice (steady-state plus a shrinking tail). @@ -195,7 +241,7 @@ def _scan_gen_worker_device_step_time( (model load) write partial multibyte sequences that would otherwise raise UnicodeDecodeError mid-scan. """ - per_file_by_ngen: List[Dict[int, Tuple[int, float]]] = [] + per_file_scans: List[Tuple[Dict[int, Tuple[int, float]], int, float]] = [] total_count = 0 for i in range(num_gen_servers): log_path = os.path.join(output_dir, f"gen_server_{i}.log") @@ -209,6 +255,8 @@ def _scan_gen_worker_device_step_time( ) by_ngen: Dict[int, Tuple[int, float]] = {} + all_count = 0 + all_mean = 0.0 with open(log_path, errors="replace") as f: if seek_to: f.seek(seek_to) @@ -219,39 +267,48 @@ def _scan_gen_worker_device_step_time( if int(m.group(1)) < 5: continue total_count += 1 + dt = float(m.group(2)) + # All-iter fallback aggregate (every usable row). + all_count += 1 + all_mean += (dt - all_mean) / all_count + # Per-ngen bucket (only rows with a parseable ngen). ngen_m = _NUM_GEN_TOKENS_RE.search(line) if ngen_m is None: continue ngen = int(ngen_m.group(1)) - dt = float(m.group(2)) count, mean = by_ngen.get(ngen, (0, 0.0)) count += 1 mean += (dt - mean) / count by_ngen[ngen] = (count, mean) - if by_ngen: - per_file_by_ngen.append(by_ngen) - return per_file_by_ngen, total_count + if all_count: + per_file_scans.append((by_ngen, all_count, all_mean)) + return per_file_scans, total_count def _mean_at_mode_ngen( - per_file_by_ngen: List[Dict[int, Tuple[int, float]]], + per_file_scans: List[Tuple[Dict[int, Tuple[int, float]], int, float]], ) -> Optional[float]: - """Aggregate per-file per-ngen buckets into a single mean. + """Aggregate per-file scans into a single mean. Within each file pick the num_generation_tokens value with the most iterations (the mode) and take its Welford mean; ties break to the largest ngen because the steady-state plateau is the upper of any tied clusters. Mode is more robust than strict == max — a one-off spike where a single iter's ngen briefly exceeds the sustained batch would otherwise - collapse the mean to 1-2 samples. Then average the per-file means across - workers. Returns None if no file had a usable row. + collapse the mean to 1-2 samples. When a file produced usable rows but no + parseable num_generation_tokens on any of them, fall back to the file's + all-iter mean so a present metric is never lost (nvbugs 6487036 / + 6487040). Then average the per-file means across workers. Returns None if + no file had a usable row. """ means: List[float] = [] - for by_ngen in per_file_by_ngen: - if not by_ngen: - continue - _mode_ngen, (_count, mean) = max(by_ngen.items(), key=lambda kv: (kv[1][0], kv[0])) - means.append(mean) + for by_ngen, _all_count, all_mean in per_file_scans: + if by_ngen: + _mode_ngen, (_count, mean) = max(by_ngen.items(), key=lambda kv: (kv[1][0], kv[0])) + means.append(mean) + else: + # No parseable ngen anywhere in this worker; use the all-iter mean. + means.append(all_mean) if not means: return None return sum(means) / len(means) @@ -261,8 +318,6 @@ def parse_gen_worker_device_step_time( output_dir: str, num_gen_servers: int, start_offsets: Optional[List[int]] = None, - settle_timeout: float = 90.0, - poll_interval: float = 3.0, ) -> Optional[float]: """Mean per-iter prev_device_step_time (ms) across all gen workers. @@ -275,44 +330,26 @@ def parse_gen_worker_device_step_time( below the steady-state cost. Using the mode (rather than strict == max) is robust against a single iter whose ngen briefly spikes above the sustained batch, which would otherwise collapse the mean to 1-2 samples. - Returns None if no usable line is found in any file. + A worker whose num_generation_tokens never parses falls back to its + all-iter mean rather than being dropped to None. Returns None only if no + usable line is found in any file. When start_offsets is provided, only the bytes from start_offsets[i] to end-of-file are considered for gen_server_{i}.log — used to slice out a single client's iteration segment. - The gen worker writes gen_server_{i}.log on a different node than the - benchmark/pytest process, and the worker is kept alive (waiting on the - benchmark_status file) when this runs — so when the client returns, the - decode iterations are done but their log lines may still be flushing across - NFS. Reading once immediately can see zero iter>=5 lines and wrongly return - None. So poll the slice until the iter>=5 row count is non-zero AND - stable across two consecutive reads (flush drained), bounded by - settle_timeout. The settle signal is the raw iter>=5 row count (not the - mode-bucket count) because raw rows are monotonic across polls, whereas - the mode ngen — and therefore its bucket size — can shift while the tail - is still flushing. + The log is read exactly once. The caller (DisaggTestCmds.run_cmd) blocks + on the gen_server_{i}.done sentinels before calling this, so every gen + srun has already exited and its &> aggregate log is fully flushed — there + is no partially-written tail to poll for. This replaces the earlier + settle-poll heuristic, which could return a mean over a truncated prefix + when it accepted the first repeated row count while the log was still + flushing across NFS (nvbugs 6487036 / 6487040). """ - deadline = time.time() + settle_timeout - prev_count = -1 - while True: - per_file_by_ngen, total_count = _scan_gen_worker_device_step_time( - output_dir, num_gen_servers, start_offsets - ) - # Non-empty and unchanged since the last poll → the flush has settled. - if total_count > 0 and total_count == prev_count: - return _mean_at_mode_ngen(per_file_by_ngen) - if time.time() >= deadline: - if per_file_by_ngen: - print_info( - f"parse_gen_worker_device_step_time: settle_timeout " - f"({settle_timeout}s) reached with {total_count} line(s); " - "returning current mean." - ) - return _mean_at_mode_ngen(per_file_by_ngen) - return None - prev_count = total_count - time.sleep(poll_interval) + per_file_scans, _total_count = _scan_gen_worker_device_step_time( + output_dir, num_gen_servers, start_offsets + ) + return _mean_at_mode_ngen(per_file_scans) def add_perf_metric_value( @@ -467,10 +504,18 @@ def __init__(self, server_config_data: dict, env_vars: str = ""): self.moe_max_num_tokens = moe_config.get("max_num_tokens", 0) self.use_low_precision_moe_combine = moe_config.get("use_low_precision_moe_combine", False) load_balancer_config = moe_config.get("load_balancer", {}) - self.load_balancer_num_slots = load_balancer_config.get("num_slots", 0) - self.load_balancer_layer_updates_per_iter = load_balancer_config.get( - "layer_updates_per_iter", 0 - ) + # load_balancer may be either an inline dict (num_slots + layer_updates_per_iter) + # or a path string to an offline-eplb YAML that the TRT-LLM engine loads at + # runtime. When it is a string, skip the inline attribute extraction — those + # metrics live inside the referenced YAML and aren't scraped by perf-sanity. + if isinstance(load_balancer_config, str): + self.load_balancer_num_slots = 0 + self.load_balancer_layer_updates_per_iter = 0 + else: + self.load_balancer_num_slots = load_balancer_config.get("num_slots", 0) + self.load_balancer_layer_updates_per_iter = load_balancer_config.get( + "layer_updates_per_iter", 0 + ) # cuda_graph_config cuda_graph_config = server_config_data.get("cuda_graph_config", {}) @@ -686,6 +731,15 @@ def generate_extra_llm_api_config(self) -> str: llm_models_root(), spec_model ) + # Resolve `moe_config.load_balancer` when it is a repo-relative path + # string. The TRT-LLM engine accepts either a dict (inline) or a path + # to an offline-eplb YAML. Absolute paths and dicts are left alone. + moe_cfg = config_data.get("moe_config") + if isinstance(moe_cfg, dict): + lb = moe_cfg.get("load_balancer") + if isinstance(lb, str) and lb and not os.path.isabs(lb): + moe_cfg["load_balancer"] = os.path.join(get_llm_root(), lb) + return yaml.dump(config_data, default_flow_style=False, sort_keys=False) @@ -1078,7 +1132,9 @@ def run_cmd(self, server_idx: int) -> List[str]: wait_for_endpoint_ready( f"http://{server_hostname}:{server_port}/health", - timeout=self.timeout, + timeout=min( + self.timeout, server_ready_timeout(AGG_SERVER_READY_TIMEOUT, "AGG") + ), check_files=[server_file_path], server_proc=server_proc, ) @@ -1253,8 +1309,25 @@ def _get_disagg_server_hostname_and_port(self, server_idx: int) -> Tuple[str, in server_config = yaml.safe_load(f) return server_config["hostname"], server_config["port"] - def wait_for_benchmark_ready(self, benchmark_status_file: str): - """Wait for benchmark to complete.""" + def wait_for_benchmark_ready( + self, + benchmark_status_file: str, + server_proc: subprocess.Popen | None = None, + server_log: str | None = None, + ): + """Wait for benchmark to complete, failing fast if our server dies. + + The liveness check is event-driven (process exit), not a timeout: a + ctx/gen/disagg server that dies here raises within one loop iteration + with its log tail in the CI log, and the rank exits nonzero. Teardown + of the rest of the stage then follows from the launcher + (``srun --kill-on-bad-exit=1`` kills this rank's step) plus the + benchmark rank's bounded ready-wait failing fast on the dead endpoint + -- instead of every rank sitting in this loop for the full timeout. + + The benchmark-done check runs FIRST so a server exiting just after a + completed benchmark cannot fail an otherwise-passing test. + """ start_time = time.time() while True: if os.path.exists(benchmark_status_file): @@ -1262,6 +1335,11 @@ def wait_for_benchmark_ready(self, benchmark_status_file: str): f"Benchmark status file found, terminating server {self.disagg_serving_type}" ) break + fail_if_proc_died( + server_proc, + f"{self.disagg_serving_type} server", + [server_log] if server_log else None, + ) elapsed_time = time.time() - start_time print_info(f"Waiting for benchmark status file, elapsed time: {elapsed_time}s") if elapsed_time > self.timeout: @@ -1270,6 +1348,42 @@ def wait_for_benchmark_ready(self, benchmark_status_file: str): ) time.sleep(10) + def wait_for_gen_log_sentinels(self, poll_interval: float = 2.0) -> bool: + """Block until every gen worker signals that its log is fully written. + + Each gen worker's srun in slurm_launch_draft.sh redirects all of its + ranks' stdout to gen_server_{i}.log via `&>` and touches + gen_server_{i}.done only after that srun is reaped (fd closed, log + flushed). The benchmark writes benchmark_status *before* calling this, + which is what lets the gen srun exit — so this is not circular. + + Returns True once all sentinels exist, or False if self.timeout is + reached first. On False the caller still parses whatever is on disk: + the sentinel is a correctness optimization against reading a + mid-flush log (nvbugs 6487036 / 6487040), never a hang risk for CI. + """ + sentinels = [ + os.path.join(self.test_output_dir, f"gen_server_{i}.done") + for i in range(self.num_gen_servers) + ] + start_time = time.time() + while True: + missing = [p for p in sentinels if not os.path.exists(p)] + if not missing: + print_info("All gen worker log sentinels present; log flush complete.") + return True + elapsed_time = time.time() - start_time + if elapsed_time > self.timeout: + print_info( + f"Timeout ({self.timeout}s) waiting for gen worker log " + f"sentinels {missing}; parsing current log contents." + ) + return False + print_info( + f"Waiting for gen worker log sentinels {missing}, elapsed time: {elapsed_time:.0f}s" + ) + time.sleep(poll_interval) + def get_server_logs(self, server_idx: int) -> List[str]: server_logs = [] for i in range(self.num_ctx_servers): @@ -1342,7 +1456,11 @@ def run_cmd(self, server_idx: int) -> List[str]: stdout=server_ctx, stderr=subprocess.STDOUT, ) - self.wait_for_benchmark_ready(benchmark_status_file) + self.wait_for_benchmark_ready( + benchmark_status_file, + server_proc=server_proc, + server_log=server_file_path, + ) finally: print_info(f"Server {self.disagg_serving_type} stopped") server_proc.terminate() @@ -1367,13 +1485,25 @@ def run_cmd(self, server_idx: int) -> List[str]: stdout=disagg_server_ctx, stderr=subprocess.STDOUT, ) - self.wait_for_benchmark_ready(benchmark_status_file) + self.wait_for_benchmark_ready( + benchmark_status_file, + server_proc=disagg_server_proc, + server_log=disagg_server_file_path, + ) finally: print_info(f"Disagg server {self.disagg_serving_type} stopped") disagg_server_proc.terminate() disagg_server_proc.wait() elif self.disagg_serving_type == "BENCHMARK": + # Perf-benchmark clients whose gen-worker device step time must be + # parsed once the gen logs are flushed. The parse is deferred out of + # the client loop because gen_server_*.log keeps being written until + # the gen srun exits, and the gen srun only exits after + # benchmark_status is written in the finally below. Parsing inside + # the loop (as before) could read a truncated / not-yet-flushed log + # and report a wrong mean (nvbugs 6487036 / 6487040). + pending_device_step_time: List[dict] = [] try: disagg_server_hostname, disagg_server_port = ( self._get_disagg_server_hostname_and_port(server_idx) @@ -1381,7 +1511,9 @@ def run_cmd(self, server_idx: int) -> List[str]: wait_for_endpoint_ready( f"http://{disagg_server_hostname}:{disagg_server_port}/health", - timeout=self.timeout, + timeout=min( + self.timeout, server_ready_timeout(DISAGG_SERVER_READY_TIMEOUT, "DISAGG") + ), check_files=self.get_server_logs(server_idx), ) @@ -1421,20 +1553,17 @@ def run_cmd(self, server_idx: int) -> List[str]: with open(benchmark_file_path, "w") as benchmark_ctx: benchmark_ctx.write(output) - # Only gen_only emits prev_device_step_time; other - # modes yield None and we skip writing the line. - device_step_time_mean = parse_gen_worker_device_step_time( - self.test_output_dir, - self.num_gen_servers, - start_offsets=gen_log_start_offsets, - ) - if device_step_time_mean is not None: - summary_line = f"Average Per Iter Device Step Time (ms): {device_step_time_mean:.2f}" - with open(benchmark_file_path, "a") as benchmark_ctx: - benchmark_ctx.write(f"\n{summary_line}\n") - output = f"{output}\n{summary_line}\n" - outputs.append(output) + # Defer the gen-worker device-step-time parse until the + # gen logs are flushed (see below); remember where to + # write the summary back. + pending_device_step_time.append( + { + "output_index": len(outputs) - 1, + "benchmark_file_path": benchmark_file_path, + "start_offsets": gen_log_start_offsets, + } + ) else: print_info( f"Skipping perf benchmark for client {client_idx}: " @@ -1468,6 +1597,30 @@ def run_cmd(self, server_idx: int) -> List[str]: with open(benchmark_status_file, "w") as status_file: status_file.write("Done") + # benchmark_status is written, so the gen workers can now stop and + # their srun will exit and drop gen_server_{i}.done. Wait once for + # those sentinels (bounded by self.timeout), then parse each + # benchmark client's gen-worker device step time a single time: the + # flushed log is complete, so no settle polling is needed. Only + # gen_only runs emit prev_device_step_time; other modes parse to + # None and skip the summary line. + if pending_device_step_time: + self.wait_for_gen_log_sentinels() + for record in pending_device_step_time: + device_step_time_mean = parse_gen_worker_device_step_time( + self.test_output_dir, + self.num_gen_servers, + start_offsets=record["start_offsets"], + ) + if device_step_time_mean is not None: + summary_line = ( + f"Average Per Iter Device Step Time (ms): {device_step_time_mean:.2f}" + ) + with open(record["benchmark_file_path"], "a") as benchmark_ctx: + benchmark_ctx.write(f"\n{summary_line}\n") + idx = record["output_index"] + outputs[idx] = f"{outputs[idx]}\n{summary_line}\n" + return outputs def get_cmd_str(self, server_idx: int) -> List[str]: @@ -1726,6 +1879,17 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): hardware["num_ctx_servers"] = 0 worker_env_var = environment.get("worker_env_var", "") + # Optional per-role env vars appended to the shared worker_env_var so + # ctx and gen workers can diverge (e.g. PYTORCH_CUDA_ALLOC_CONF on ctx + # only). Absent keys leave the shared value untouched. + ctx_worker_env_var_extra = environment.get("ctx_worker_env_var", "") or "" + gen_worker_env_var_extra = environment.get("gen_worker_env_var", "") or "" + ctx_worker_env_var = " ".join( + part for part in (worker_env_var, ctx_worker_env_var_extra) if part + ) + gen_worker_env_var = " ".join( + part for part in (worker_env_var, gen_worker_env_var_extra) if part + ) server_env_var = environment.get("server_env_var", "") client_env_var = environment.get("client_env_var", "") @@ -1761,7 +1925,10 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): **ctx_config, } - ctx_server_config = ServerConfig(ctx_server_config_data, worker_env_var) + # ctx_only runs the ctx worker in aggregated mode; use the merged + # ctx-side env var so the aggregated run still gets any ctx-only + # extras from the disagg yaml. + ctx_server_config = ServerConfig(ctx_server_config_data, ctx_worker_env_var) self.server_configs = [ctx_server_config] else: # For e2e and gen_only modes - create ctx and gen server configs @@ -1783,8 +1950,8 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): **worker_config.get("gen", {}), } - ctx_server_config = ServerConfig(ctx_server_config_data, worker_env_var) - gen_server_config = ServerConfig(gen_server_config_data, worker_env_var) + ctx_server_config = ServerConfig(ctx_server_config_data, ctx_worker_env_var) + gen_server_config = ServerConfig(gen_server_config_data, gen_worker_env_var) disagg_config = DisaggConfig( name=f"{benchmark_mode}-{config_file_base_name}", @@ -2273,11 +2440,20 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: else: return + stage_name = os.environ.get("stageName", "") extra_fields = { - "s_stage_name": os.environ.get("stageName", ""), + "s_stage_name": stage_name, "s_test_list": self._test_param_labels, } + # Stages tagged "FUNCTIONAL-ONLY" run the full perf harness (numbers are + # still uploaded to OpenSearch and dashboards) but do not fail CI on perf + # regression -- same behavior as post-merge. Used for pre-merge disagg + # coverage where the goal is functional-failure detection, not gating on + # perf. Explicit False (not None) so the auto-detect in + # process_and_upload_test_results does not flip it back on for pre-merge. + fail_on_regression = False if "FUNCTIONAL-ONLY" in stage_name else None + # gen_only tests are gated solely on per-iter prev_device_step_time, not # token throughput (token-based numbers are dominated by KV cache transfer # time in gen_only mode and are not a useful regression signal there). @@ -2304,6 +2480,7 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: regression_metrics=regression_metrics, extra_fields=extra_fields, upload_to_db=self.upload_to_db, + fail_on_regression=fail_on_regression, ) diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py index 8a8473db98b7..2e03a7d63680 100644 --- a/tests/integration/defs/perf/utils.py +++ b/tests/integration/defs/perf/utils.py @@ -424,6 +424,17 @@ def stop_server(self) -> None: self._server_log_file.close() self._server_log_file = None + def get_server_log_content(self) -> str: + """Read back the server's captured stdout/stderr so far (e.g. to parse startup info like KV cache size).""" + if not self._server_log_path or not os.path.exists( + self._server_log_path): + return "" + with open(self._server_log_path, + 'r', + encoding='utf-8', + errors='replace') as f: + return f.read() + def run_cmd(self, cmd_idx: int, venv) -> str: output = "" if cmd_idx <= len(self.data_cmds) - 1: @@ -447,6 +458,9 @@ def run_cmd(self, cmd_idx: int, venv) -> str: f.write(output) elif cmd_idx == len(self.data_cmds): self.start_server() + # Return the startup log (e.g. KV cache size) so it can be regex-parsed, + # instead of an empty string. + output = self.get_server_log_content() else: client_cmd = self.client_cmds[cmd_idx - 1 - len(self.data_cmds)] client_cmd_with_port = client_cmd + [ diff --git a/tests/integration/defs/pytest.ini b/tests/integration/defs/pytest.ini index ee5e97c00f74..45793bd9f51c 100644 --- a/tests/integration/defs/pytest.ini +++ b/tests/integration/defs/pytest.ini @@ -1,12 +1,13 @@ [pytest] asyncio_default_fixture_loop_scope = module threadleak = True -# Thread-\d+ \(_manager_spawn\) / session-reuse-* belong to a pool cached for -# reuse by the NEXT test (tests/test_common/session_reuse.py) and legitimately +# Thread-\d+ \(_manager_spawn\) / session-reuse-* / session-prefetch-* belong to +# a pool cached for reuse by the NEXT test (tests/test_common/session_reuse.py) +# or prefetched for it (tests/test_common/session_prefetcher.py) and legitimately # outlive the test they start under. -threadleak_exclude = asyncio_\d+|Thread-\d+ \(_manager_spawn\)|session-reuse-\w+ +threadleak_exclude = asyncio_\d+|Thread-\d+ \(_manager_spawn\)|session-reuse-\w+|session-prefetch-\w+ junit_family=legacy -addopts = --ignore-glob="*perf/test_perf.py" --ignore-glob="*perf/disagg/*" --ignore-glob="*test_list_validation.py" --ignore-glob="*llm-test-workspace*" --durations=0 -W ignore::DeprecationWarning --unused-fixtures -p test_common.session_reuse_hooks +addopts = --ignore-glob="*perf/test_perf.py" --ignore-glob="*perf/disagg/*" --ignore-glob="*test_list_validation.py" --ignore-glob="*llm-test-workspace*" --durations=0 -W ignore::DeprecationWarning --unused-fixtures -p test_common.session_reuse_hooks -p test_common.s3_output_hooks pythonpath = ../../../examples/auto_deploy ../../ norecursedirs = ./triton/perf ./perf/disagg diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 044ebc905bc0..215731702435 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -40,35 +40,6 @@ _MEM_FRACTION_95 = 0.95 -def test_gpt3_175b_1layers_build_only(llm_root, llm_venv, engine_dir): - """Build GPT-3 175B: 96 layer w/ plugins""" - example_root = os.path.join(llm_root, "examples", "models", "core", "gpt") - engine_dir = os.path.join(engine_dir, "gpt-175-96layers-build-only") - - dtype = 'float16' - convert_cmd = [ - f"{example_root}/../../../generate_checkpoint_config.py", - f"--output_path={engine_dir}/ckpt_config.json", - "--architecture=GPTForCausalLM", f"--dtype={dtype}", - "--num_hidden_layers=1", "--num_attention_heads=96", - "--hidden_size=12288", "--vocab_size=51200", "--tp_size=8" - ] - venv_check_call(llm_venv, convert_cmd) - - print("Building engines...") - build_cmd = [ - "trtllm-build", - f"--model_config={engine_dir}/ckpt_config.json", - f"--output_dir={engine_dir}", - "--max_batch_size=256", - "--max_input_len=200", - "--max_seq_len=400", - "--max_beam_width=1", - f"--gpt_attention_plugin={dtype}", - ] - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - - @pytest.mark.parametrize("model_name,model_path", [ ("DeepSeek-R1-Distill-Qwen-1.5B", "DeepSeek-R1-Distill-Qwen-1.5B"), ]) @@ -1912,7 +1883,6 @@ def test_ptp_scaffolding(llm_root, llm_venv, model_name, model_path): marks=skip_pre_blackwell), pytest.param('DeepSeek-R1/DeepSeek-R1-0528-FP4', marks=skip_pre_blackwell), pytest.param('Kimi-K2-Thinking-NVFP4', marks=skip_pre_blackwell), - pytest.param('MiniMax-M2', marks=skip_pre_hopper), pytest.param('MiniMax-M3', marks=skip_pre_blackwell), ]) def test_multi_nodes_eval(model_path, tp_size, pp_size, ep_size, eval_task, diff --git a/tests/integration/defs/test_mlpf_results.py b/tests/integration/defs/test_mlpf_results.py deleted file mode 100644 index 71dbb12430dd..000000000000 --- a/tests/integration/defs/test_mlpf_results.py +++ /dev/null @@ -1,260 +0,0 @@ -""" -MLPerf target tests -""" -import os -import subprocess -from argparse import Namespace -from copy import deepcopy - -from defs.common import get_cpp_benchmark, get_trt_llm_lib_dir, venv_check_call -from defs.trt_test_alternative import check_call - -### End of utility functions -""" -Test: Runs the gptManagerBenchmark on LLama TRTLLM engine and checks accuracy of predictions -Steps: - 1. Quantize the model: step_quantize - 2. Build the engine: step_engine_build - 3. Run engine and get outputs: step_run_llm - 4. Check prediction accuracy: step_check_accuracy -""" - - -# Test step 1: Quantize the model -# MLPerf step: python examples/quantization/quantize.py --dtype=float16 --output_dir=<> --model_dir=<> --qformat=fp8 --kv_cache_dtype=fp8 --tp_size 2 -def step_quantize(tp_size, llm_venv, llm_root, model_root, model, - calib_dataset): - quantized_model_path = "{}/test_mlperf_quantized_models/{}-tp{}-pp1/".format( - llm_venv.get_working_directory(), model, tp_size) - tekit_example_dir = os.path.join(llm_root, "examples/") - - # Set MLPerf params explicitly - quantize_cmd = [ - f"{tekit_example_dir}/quantization/quantize.py", "--dtype=float16", - "--qformat=fp8", "--kv_cache_dtype=fp8", f"--tp_size={tp_size}", - f"--output_dir={quantized_model_path}", f"--model_dir={model_root}", - "--calib_size=1024", f"--calib_dataset={calib_dataset}" - ] - - venv_check_call(llm_venv, quantize_cmd) - - return quantized_model_path - - -# Test step 2: Build the TRTLLM engine -# MLPerf step: -# python3 -m tensorrt_llm.commands.build --gpt_attention_plugin=float16 --max_batch_size=896 --max_input_len=1024 --max_seq_len=2048 --max_beam_width=1 \ -# --max_num_tokens=4096 --output_dir=<> --checkpoint_dir=<> --context_fmha=enable --remove_input_padding=enable \ -# --paged_kv_cache=enable --workers=2 - - -def step_engine_build(quantized_model_path, system_config, engine_dir, - llm_venv): - - batch_size = system_config.batch_size - beam_width = system_config.beam_width - max_input_len = system_config.max_input_len - max_seq_len = system_config.max_seq_len - max_num_tokens = system_config.max_num_tokens - num_workers = system_config.num_workers - use_fp8_context_fmha = "enable" if system_config.fp8_fmha else "disable" - - build_cmd = [ - "trtllm-build", - "--gpt_attention_plugin=float16", - f"--max_batch_size={batch_size}", - f"--max_input_len={max_input_len}", - f"--max_seq_len={max_seq_len}", - f"--max_beam_width={beam_width}", - f"--max_num_tokens={max_num_tokens}", - f"--output_dir={engine_dir}", - f"--checkpoint_dir={quantized_model_path}", - "--context_fmha=enable", - f"--use_fp8_context_fmha={use_fp8_context_fmha}", - "--remove_input_padding=enable", - "--paged_kv_cache=enable", - f"--workers={num_workers}", - ] - - check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) - return engine_dir - - -DEFAULT_RPARAMS = Namespace( - engine_dir=None, - api="executor", - # type="IFB", - dataset=None, - output_csv="gptmanager_bench_results.csv", - max_num_samples=24576, - beam_width=1, - warm_up=2, - eos_id=-1, - pad_id=-1, - max_tokens_in_paged_kvcache=None, - kv_cache_free_gpu_mem_fraction=None, - streaming=False, - enable_kv_cache_reuse=False, - enable_chunked_context=False, - return_context_logits=False, - return_generation_logits=False, - scheduler_policy="guaranteed_no_evict", - static_emulated_batch_size=None, - log_level="verbose", - log_iteration_data=False, - wait_sleep="25", - lora_dir=None, - lora_host_cache_bytes=None, - lora_num_device_mod_layers=None, - responses_json=None) -""" -./benchmarks/gptManagerBenchmark \ - --engine_dir <> \ - --dataset <> \ - --max_num_samples 24576 \ - --beam_width 1 \ - --eos_id 2 \ - --pad_id 2 \ - --kv_cache_free_gpu_mem_fraction 0.95 \ - --scheduler_policy max_utilization \ - --output_csv <> -""" - - -# Test step 3: Run the gptManagerBenchmark and get outputs -def step_run_llm(system_config, - engine_path, - dataset_path, - llm_venv, - llm_root, - kv_cache_free_gpu_mem_fraction=0.95): - tp, pp = system_config.tp_size, system_config.pp_size - eos_id, pad_id = system_config.eos_id, system_config.pad_id - max_num_samples = system_config.num_samples - beam_width = system_config.beam_width - - benchmark_exe = get_cpp_benchmark('gptManagerBenchmark', llm_root) - workspace_path = llm_venv.get_working_directory() - run_params = deepcopy(DEFAULT_RPARAMS) - run_params.beam_width = beam_width - run_params.engine_dir = engine_path - run_params.dataset = dataset_path - run_params.max_num_samples = max_num_samples - run_params.eos_id = eos_id - run_params.pad_id = pad_id - run_params.kv_cache_free_gpu_mem_fraction = kv_cache_free_gpu_mem_fraction - run_params.scheduler_policy = "max_utilization" - run_params.responses_json = os.path.join( - workspace_path, f"responses_test_mlperf_tp{tp}_pp{pp}.json") - run_params.output_csv = os.path.join( - workspace_path, f"perf_stats_test_mlperf_tp{tp}_pp{pp}.csv") - - run_params_dict = vars(run_params) - run_params_dict['type'] = "IFB" - - bench_cmd = [benchmark_exe] - for key, val in run_params_dict.items(): - if val is None or val is False: - continue - if val is True: - val = "" - bench_cmd.append("--" + str(key)) - bench_cmd.append(str(val)) - - envs = deepcopy(os.environ) - _ = envs.pop("CUDA_VISIBLE_DEVICES", "") - envs[ - "LD_LIBRARY_PATH"] = f'{get_trt_llm_lib_dir(llm_venv)}:{os.path.dirname(benchmark_exe)}:{envs.get("LD_LIBRARY_PATH", "")}' - - print( - f'CUDA_VISIBLE_DEVICES: {os.environ.get("CUDA_VISIBLE_DEVICES", None)}') - - num_ranks = tp * pp - if num_ranks > 1: - mpi_cmd = ["mpirun", "-n", f"{num_ranks}", "--allow-run-as-root"] - bench_cmd = mpi_cmd + bench_cmd - - print(f"Running gptManagerBenchmark using cmd: {' '.join(bench_cmd)}") - subprocess.check_output(bench_cmd, env=envs) - return run_params.responses_json - - -def step_check_accuracy(responses_file, dataset_path, model_root, llm_venv, - llm_root): - """ - python3 /code/tensorrt_llm/benchmarks/python/check_accuracy_mlperf.py - --dataset <> - --responses <> - --base_model <> - """ - accuracy_script = os.path.join( - llm_root, "benchmarks/python/check_accuracy_mlperf.py") - accuracy_check_cmd = [ - f"{accuracy_script}", "--dataset", f"{dataset_path}", "--responses", - f"{responses_file}", "--base_model", f"{model_root}" - ] - venv_check_call(llm_venv, accuracy_check_cmd) - - -LlamaBaseSystem = Namespace(tp_size=None, - pp_size=1, - batch_size=None, - max_input_len=1024, - max_seq_len=2048, - max_num_tokens=4096, - beam_width=1, - num_workers=None, - num_samples=24576, - eos_id=2, - pad_id=2, - fp8_fmha=False) - -GptjBaseSystem = Namespace(tp_size=1, - pp_size=1, - batch_size=None, - max_input_len=1919, - max_seq_len=2047, - max_num_tokens=4096, - beam_width=4, - num_workers=1, - num_samples=13368, - eos_id=50256, - pad_id=50256, - fp8_fmha=False) - - -def get_mlperf_system_config(model: str, system: str, fp8_fmha: bool): - if model == "llama_v2_70b_chat": - return get_mlperf_llama_system_config(system) - elif model == "gpt_j": - return get_mlperf_gptj_system_config(system, fp8_fmha) - raise RuntimeError(f"Unexpected model: {system}") - - -def get_mlperf_llama_system_config(system: str): - system_config = deepcopy(LlamaBaseSystem) - if system == "H100x2": - system_config.tp_size = 2 - system_config.batch_size = 896 - system_config.num_workers = 2 - elif system == "H200x1": - system_config.tp_size = 1 - system_config.batch_size = 806 - system_config.num_workers = 1 - else: - raise RuntimeError(f"No Llama config found for system: {system}") - - return system_config - - -def get_mlperf_gptj_system_config(system: str, fp8_fmha: bool): - system_config = deepcopy(GptjBaseSystem) - system_config.fp8_fmha = fp8_fmha - if system == "H100x1": - system_config.batch_size = 192 - elif system == "H200x1": - system_config.batch_size = 396 - else: - raise RuntimeError(f"No GPT-J config found for system: {system}") - - return system_config diff --git a/tests/integration/defs/test_unittests.py b/tests/integration/defs/test_unittests.py index 9287fe9b3ca5..badcde1f44b3 100644 --- a/tests/integration/defs/test_unittests.py +++ b/tests/integration/defs/test_unittests.py @@ -183,7 +183,6 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request): "-s", f"--output-dir={inner_output_dir}", f"--s3-upload-path={inner_upload_path}", - "--s3-capture-mode=direct", "--s3-upload-mode=deferred", ] inline_output_max_bytes = request.config.getoption( diff --git a/tests/integration/defs/triton_server/conftest.py b/tests/integration/defs/triton_server/conftest.py index 18f6087ff6b9..932585184826 100644 --- a/tests/integration/defs/triton_server/conftest.py +++ b/tests/integration/defs/triton_server/conftest.py @@ -172,11 +172,6 @@ def tensorrt_llm_gptj_example_root(llm_backend_root): return os.path.join(llm_backend_root, "../examples/models/contrib/gptj") -@pytest.fixture(scope="session") -def tensorrt_llm_multimodal_example_root(llm_backend_root): - return os.path.join(llm_backend_root, "../examples/models/core/multimodal") - - @pytest.fixture(scope="session") def tensorrt_llm_opt_example_root(llm_backend_root): return os.path.join(llm_backend_root, "../examples/models/contrib/opt") @@ -207,21 +202,11 @@ def tensorrt_llm_llama_example_root(llm_backend_root): return os.path.join(llm_backend_root, "../examples/models/core/llama") -@pytest.fixture(scope="session") -def tensorrt_llm_qwen_example_root(llm_backend_root): - return os.path.join(llm_backend_root, "../examples/models/core/qwen") - - @pytest.fixture(scope="session") def tensorrt_llm_mllama_example_root(llm_backend_root): return os.path.join(llm_backend_root, "../examples/models/core/mllama") -@pytest.fixture(scope="session") -def tensorrt_llm_mixtral_example_root(llm_backend_root): - return os.path.join(llm_backend_root, "../examples/models/core/mixtral") - - @pytest.fixture(scope="session") def inflight_batcher_llm_client_root(llm_backend_root): inflight_batcher_llm_client_root = os.path.join(llm_backend_root, diff --git a/tests/integration/test_lists/qa/README.md b/tests/integration/test_lists/qa/README.md index c82f055ed020..e242584e64f3 100644 --- a/tests/integration/test_lists/qa/README.md +++ b/tests/integration/test_lists/qa/README.md @@ -90,7 +90,7 @@ cd tests/integration/defs # Run all fp8 functional test pytest --no-header -vs --test-list=../test_lists/qa/llm_function_full.txt -k fp8 # Run a single test case -pytest -vs accuracy/test_cli_flow.py::TestLlama3_1_8B::test_auto_dtype +pytest -vs accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_auto_dtype ``` ### Automated Execution diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index b734b9d376f0..8d397905a786 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -1,24 +1,12 @@ +accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] -accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1dp2cp2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp1cp4] -accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp2tp1cp2] +accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1dp2cp2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1tp1cp4] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1tp2cp2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp2tp1cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1dp2cp2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp1cp4] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp2cp2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp2tp1cp2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp1dp2cp2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp1tp1cp4] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp1tp2cp2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp2tp1cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync[python] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] @@ -35,7 +23,6 @@ accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[True] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v2] accuracy/test_disaggregated_serving.py::TestGLM52NVFP4::test_nvfp4_nixl_python[cache_mgr_v1] -accuracy/test_disaggregated_serving.py::TestKimiK2::test_nvfp4 accuracy/test_disaggregated_serving.py::TestKimiK25::test_nvfp4 accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-False] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True] @@ -87,23 +74,16 @@ accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_mo accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-False] accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False-True] accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True-True] -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1dp2cp2] -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1tp1cp4] -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1tp2cp2] -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp2tp1cp2] accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1dp2cp2] -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp1cp4] accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp2cp2] -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp2tp1cp2] -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp1dp2cp2] -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp1tp1cp4] -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp1tp2cp2] -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp2tp1cp2] accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_chunked_prefill accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first_kv_cache_v1 accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[noadp-mtp0] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[adp-mtp2] +accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_auto_dtype +accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_gen_first +accuracy/test_disaggregated_serving.py::TestDeepSeekV4FlashBase::test_auto_dtype accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[qwen3vl_2b_instruct] accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8] @@ -684,9 +664,10 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] accuracy/test_llm_api_pytorch.py::TestMiniMaxM2_5::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_size=8] TIMEOUT (180) -accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[tp_size=4-ep_size=4] TIMEOUT (180) -accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph[tp_size=8-ep_size=8] TIMEOUT (180) -accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[tp_size=4-ep_size=4] TIMEOUT (180) +accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] TIMEOUT (180) +accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph[use_msa=False] TIMEOUT (180) +accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] TIMEOUT (180) +accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True] TIMEOUT (60) accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_fp8 accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm] @@ -709,6 +690,7 @@ accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp_custom_op accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_nvfp4_marlin_multi_gpus[tp_size=8] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_marlin_multi_gpus[tp_size=8] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_marlin_adp_4gpus[mtp_nextn=3] accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_marlin_8gpus accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TEP4_PP2] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TEP8_PP1] @@ -718,6 +700,7 @@ accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[DEP4_MTP_ON] accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype +accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_tp2 accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency] accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[throughput_latency] accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_cutlass] @@ -769,6 +752,7 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_cutedsl accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_trtllm] accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_cutedsl] accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_trtllm] +accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_mtp3_gdn_replay_tep4 accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[latency] accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[multi_gpus_no_cache] accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_dummy_load_format @@ -870,7 +854,6 @@ disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_co disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_conditional_v2[DeepSeek-V3-Lite-bf16] disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_empty_batch[DeepSeek-V3-Lite-bf16] disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_tllm_gen_helix[DeepSeek-V3-Lite-bf16-short_prompt] -disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_tllm_gen_helix[DeepSeek-V3-Lite-bf16-long_prompt] disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp[DeepSeek-V3-Lite-fp8] disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_gen_only[DeepSeek-V3-Lite-fp8] disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_one[DeepSeek-V3-Lite-fp8] @@ -906,6 +889,8 @@ disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated.py::test_disaggregated_perf_metrics[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated.py::test_disaggregated_single_gpu[TinyLlama-1.1B-Chat-v1.0] +disaggregated/test_disaggregated_single_gpu.py::test_arbitrary_kv_cache_transfer[False-TinyLlama-1.1B-Chat-v1.0] +disaggregated/test_disaggregated_single_gpu.py::test_arbitrary_kv_cache_transfer_missing_blocks[False-TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_cancel_gen_requests[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_llama_context_capacity[False-False-DeepSeek-V3-Lite-fp8/fp8] disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[False-TinyLlama-1.1B-Chat-v1.0] @@ -1004,4 +989,8 @@ llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_mult accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4[fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4[fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_auto_dtype[moe_backend=WIDEEP] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_auto_dtype[moe_backend=TRTLLM] +accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_fp8_chunked_prefill +accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_fp8_4gpus_static_eplb[moe_backend=WIDEEP] accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] diff --git a/tests/integration/test_lists/qa/llm_function_multinode.txt b/tests/integration/test_lists/qa/llm_function_multinode.txt index f3edf4384158..cb10040d1934 100644 --- a/tests/integration/test_lists/qa/llm_function_multinode.txt +++ b/tests/integration/test_lists/qa/llm_function_multinode.txt @@ -3,7 +3,6 @@ test_e2e.py::test_multi_nodes_eval[Qwen3/Qwen3-235B-A22B-tp16-mmlu] test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-tp16-mmlu] test_e2e.py::test_multi_nodes_eval[DeepSeek-R1/DeepSeek-R1-0528-FP4-tp16-mmlu] test_e2e.py::test_multi_nodes_eval[Kimi-K2-Thinking-NVFP4-tp16-mmlu] -test_e2e.py::test_multi_nodes_eval[MiniMax-M2-tp16-mmlu] test_e2e.py::test_multi_nodes_eval[MiniMax-M3-tp16-mmlu] test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp2pp1-gen_tp2pp1] test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp1pp2-gen_tp1pp2] diff --git a/tests/integration/test_lists/qa/llm_perf_core.yml b/tests/integration/test_lists/qa/llm_perf_core.yml index 7a5fd834aefb..ea30931f8b0a 100644 --- a/tests/integration/test_lists/qa/llm_perf_core.yml +++ b/tests/integration/test_lists/qa/llm_perf_core.yml @@ -28,18 +28,17 @@ llm_perf_core: - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:128,128] - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:500,2000] - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:2000,500] - - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:1000,1000] - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:1000,2000] - - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1] #min_latency - - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250] #max_throughput + - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:8000,1000] - perf/test_perf.py::test_perf[qwen3_4b_eagle3-bench-pytorch-streaming-bfloat16-maxbs:4-kv_frac:0.6-input_output_len:500,100-reqs:200-con:4] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_nano_8b_fp8-bench-pytorch-float8-maxnt:5000-input_output_len:5000,500-reqs:8-con:1] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_nano_8b_fp8-bench-pytorch-float8-input_output_len:500,2000-reqs:8-con:1] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_nano_8b_fp8-bench-pytorch-float8-input_output_len:1000,1000-reqs:8-con:1] + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_nano_8b_fp8-bench-pytorch-float8-input_output_len:8000,1000-reqs:8-con:1] # test overlap scheduler # con:1 paired with a small model is an intentional design choice—it amplifies host-side overhead and simplifies execution timelines to the maximum extent. - perf/test_perf.py::test_perf[qwen3_0.6b-bench-pytorch-bfloat16-maxnt:2048-input_output_len:8000,1000-reqs:256-con:1-pp:4-gpus:4] - - perf/test_perf.py::test_perf[qwen3_30b_a3b-bench-pytorch-bfloat16-maxnt:2048-input_output_len:8000,1000-reqs:256-con:1-pp:4-gpus:4] + # 30B MoE uses fewer reqs to fit the 1h timeout; con:1 host-overhead signal is unaffected + - perf/test_perf.py::test_perf[qwen3_30b_a3b-bench-pytorch-bfloat16-maxnt:2048-input_output_len:8000,1000-reqs:64-con:1-pp:4-gpus:4] # 2: L40S, H100, H20 @@ -53,7 +52,7 @@ llm_perf_core: #nemotron_nano_12b_v2 - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-maxbs:1-input_output_len:128,128-reqs:10-con:1] #min_latency #qwen3.5_27b (dense BF16 52G, 2-GPU) - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-streaming-bfloat16-input_output_len:128,128-gpus:4] - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-bfloat16-input_output_len:128,128-gpus:4] - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp8-bench-pytorch-streaming-float8-input_output_len:2000,200-gpus:8] @@ -62,9 +61,7 @@ llm_perf_core: - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:128,128-tp:4-gpus:4] - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:4-gpus:4] - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,1000-tp:4-gpus:4] - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250-tp:4-gpus:4] #max_throughput # 3: H100, H20 test cases @@ -89,17 +86,20 @@ llm_perf_core: - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:128,128-tp:2-gpus:2] - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:2-gpus:2] - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:2-gpus:2] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,1000-tp:2-gpus:2] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:8000,1000-tp:2-gpus:2] - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250-tp:2-gpus:2] #max_throughput + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250-tp:2-gpus:2] #max_throughput #llama_v3.3_nemotron_super_49b (nemotron-nas BF16 94G, 2-GPU) - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:128,128-tp:2-gpus:2] - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:2-gpus:2] - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,1000-tp:2-gpus:2] + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:8000,1000-tp:2-gpus:2] - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250-tp:2-gpus:2] #max_throughput + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250-tp:2-gpus:2] #max_throughput + #llama_v3.3_nemotron_super_49b (nemotron-nas BF16 94G, 4-GPU): too slow on L40S PCIe-only interconnect + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:8000,1000-tp:4-gpus:4] + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250-tp:4-gpus:4] #max_throughput # 4: H100, H20, GB200, B200, B300, GB300, RTX6000-Server test cases @@ -115,40 +115,40 @@ llm_perf_core: - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp8-bench-pytorch-float8-input_output_len:1000,2000-con:256-ep:8-gpus:8] #nemotron_nano_12b_v2 - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-input_output_len:512,512] - - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1] #min_latency - - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250] #max_throughput + - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1] #min_latency + - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250] #max_throughput - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-streaming-bfloat16-input_output_len:500,2000-con:250] #max_throughput streaming #qwen3.5_27b (dense BF16 52G, 1-GPU) - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:128,128] - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:500,2000] - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:2000,500] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,1000] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:8000,1000] - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,2000] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1] #min_latency - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250] #max_throughput + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1] #min_latency + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250] #max_throughput #qwen3.5_35b_a3b_fp8 (MoE FP8 36G, 1-GPU) - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-input_output_len:128,128] - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-input_output_len:500,2000] - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-input_output_len:2000,500] - - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-input_output_len:1000,1000] + - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-input_output_len:8000,1000] - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-input_output_len:1000,2000] - - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1] #min_latency - - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-maxbs:512-input_output_len:1000,1000-con:256] #max_throughput + - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1] #min_latency + - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-maxbs:512-input_output_len:8000,1000-con:256] #max_throughput #llama_v3.3_nemotron_super_49b_fp8 (nemotron-nas FP8 49G, 2-GPU for safety) - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:128,128-tp:2-gpus:2] - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:500,2000-tp:2-gpus:2] - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:2000,500-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:1000,1000-tp:2-gpus:2] + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:8000,1000-tp:2-gpus:2] - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:1000,2000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:1000,1000-con:250-tp:2-gpus:2] #max_throughput + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:8000,1000-con:250-tp:2-gpus:2] #max_throughput #qwen3.5_122b_a10b (MoE BF16 234G, 4-GPU) - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:500,2000-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:2000,500-ep:4-tp:4-gpus:4] - - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:1000,1000-ep:4-tp:4-gpus:4] + - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:8000,1000-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:1000,2000-ep:4-tp:4-gpus:4] - - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:4-tp:4-gpus:4] #min_latency - - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-maxbs:512-input_output_len:1000,1000-con:256-ep:4-tp:4-gpus:4] #max_throughput + - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:4-tp:4-gpus:4] #min_latency + - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-maxbs:512-input_output_len:8000,1000-con:256-ep:4-tp:4-gpus:4] #max_throughput @@ -180,7 +180,7 @@ llm_perf_core: - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:256-maxnt:1024-kv_frac:0.85-input_output_len:2000,2000-reqs:200-ep:4-tp:4-gpus:4] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:1000-maxnt:5000-kv_frac:0.85-input_output_len:5000,500-reqs:2000-ep:4-tp:4-gpus:4] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:32-maxnt:32768-input_output_len:8192,1024-reqs:20-con:1-ep:1-tp:4-gpus:4] TIMEOUT(120) - - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-kv_frac:0.85-input_output_len:1000,1000-reqs:2000-ep:4-tp:4-gpus:4] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-kv_frac:0.85-input_output_len:8000,1000-reqs:2000-ep:4-tp:4-gpus:4] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-kv_frac:0.85-input_output_len:1000,2000-reqs:3000-ep:4-tp:4-gpus:4] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:1000-maxnt:5000-kv_frac:0.85-input_output_len:5000,500-reqs:20000-ep:4-tp:4-gpus:4] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:512-input_output_len:128,128-ep:4-tp:4-gpus:4] @@ -189,29 +189,50 @@ llm_perf_core: - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:128,128-tp:4-gpus:4] - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:500,2000-tp:4-gpus:4] - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:2000,500-tp:4-gpus:4] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:1000,1000-tp:4-gpus:4] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:4-gpus:4] #min_latency - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-input_output_len:1000,1000-con:256-tp:4-gpus:4] #max_throughput + - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:8000,1000-tp:4-gpus:4] + - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:4-gpus:4] #min_latency + - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-input_output_len:8000,1000-con:256-tp:4-gpus:4] #max_throughput + #minimax_m3 (MXFP8 block-sparse MoE, 4-GPU) + - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-input_output_len:128,128-tp:4-gpus:4] + - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-input_output_len:500,2000-tp:4-gpus:4] + - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-input_output_len:2000,500-tp:4-gpus:4] + - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-input_output_len:8000,1000-tp:4-gpus:4] + - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:4-gpus:4] #min_latency + - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-maxbs:512-input_output_len:8000,1000-con:256-tp:4-gpus:4] #max_throughput #deepseek_v3.2_fp4 (FP4 389G, 4-GPU) - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-input_output_len:128,128-ep:4-tp:4-gpus:4] - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:4-tp:4-gpus:4] #min_latency - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:512-kv_frac:0.85-input_output_len:1000,1000-con:512-ep:4-tp:4-gpus:4] TIMEOUT(120) #max_throughput + - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:4-tp:4-gpus:4] #min_latency + - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:512-kv_frac:0.85-input_output_len:8000,1000-con:512-ep:4-tp:4-gpus:4] TIMEOUT(120) #max_throughput + #deepseek_v4_flash_base_fp8 (sparse-MLA MoE, FP8 block-scale, 4-GPU) + - perf/test_perf.py::test_perf[deepseek_v4_flash_base_fp8-bench-pytorch-float8-input_output_len:8192,1024-ep:4-tp:4-gpus:4] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_v4_flash_base_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:4-tp:4-gpus:4] TIMEOUT(120) + #deepseek_v4_flash (post-trained, FP4+FP8 mixed, 4-GPU) + - perf/test_perf.py::test_perf[deepseek_v4_flash-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:8192,1024-ep:4-tp:4-gpus:4] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_v4_flash-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:1000,2000-ep:4-tp:4-gpus:4] TIMEOUT(120) #llama_v3.1_nemotron_ultra_253b_fp8 (nemotron-nas FP8 241G, 4-GPU) - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:128,128-tp:4-gpus:4] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:500,2000-tp:4-gpus:4] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:2000,500-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:1000,1000-tp:4-gpus:4] + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:8000,1000-tp:4-gpus:4] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:1000,2000-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:4-gpus:4] #min_latency - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:1000,1000-con:250-tp:4-gpus:4] #max_throughput + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:4-gpus:4] #min_latency + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:8000,1000-con:250-tp:4-gpus:4] #max_throughput #qwen3.5_397b_a17b_fp4 (MoE FP4 234G, 4-GPU) - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:128,128-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:500,2000-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:2000,500-ep:4-tp:4-gpus:4] - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:1000,1000-ep:4-tp:4-gpus:4] + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:8000,1000-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:1000,2000-ep:4-tp:4-gpus:4] - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:4-tp:4-gpus:4] #min_latency - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:512-input_output_len:1000,1000-con:512-ep:4-tp:4-gpus:4] #max_throughput + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:4-tp:4-gpus:4] #min_latency + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:512-input_output_len:8000,1000-con:512-ep:4-tp:4-gpus:4] #max_throughput + #qwen3.6_35b_a3b_fp4 (GDN-attn MoE NVFP4, 1-GPU, SM100/103 only) + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-float4-input_output_len:128,128] + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-float4-input_output_len:500,2000] + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-float4-input_output_len:2000,500] + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-float4-input_output_len:8000,1000] + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-float4-input_output_len:1000,2000] + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1] #min_latency + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-float4-maxbs:512-input_output_len:8000,1000-con:256] #max_throughput #nemotron_3_super_120b_nvfp4 (Hybrid MoE+SSM+Attn FP4 76G, 4-GPU ep=4 tp=4, throughput config) #these test config come from docs/source/deployment-guide/deployment-guide-for-nemotron-3-on-trtllm.md - perf/test_perf.py::test_perf[nemotron_3_super_120b_nvfp4-serve-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024-reqs:5-con:1-ep:4-tp:4-gpus:4] #min_latency @@ -243,30 +264,39 @@ llm_perf_core: - perf/test_perf.py::test_perf[gpt_oss_120b_fp4-bench-pytorch-float4-maxbs:720-maxnt:16384-input_output_len:1024,1024-reqs:100-con:32-ep:8-tp:8-gpus:8] # deepseek_r1_0528_fp4 - - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-kv_frac:0.85-input_output_len:1000,1000-reqs:20000-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-kv_frac:0.85-input_output_len:8000,1000-reqs:20000-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-kv_frac:0.85-input_output_len:1000,2000-reqs:3000-ep:8-tp:8-gpus:8] TIMEOUT(120) + #deepseek_v4_flash_base_fp8 (sparse-MLA MoE, FP8 block-scale, 8-GPU) + - perf/test_perf.py::test_perf[deepseek_v4_flash_base_fp8-bench-pytorch-float8-input_output_len:8192,1024-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_v4_flash_base_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) + #deepseek_v4_flash (post-trained, FP4+FP8 mixed, 8-GPU) + - perf/test_perf.py::test_perf[deepseek_v4_flash-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:8192,1024-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_v4_flash-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) + #deepseek_v4_pro_dspark (DSpark speculative decoding, FP4+FP8 mixed, 8-GPU) + - perf/test_perf.py::test_perf[deepseek_v4_pro_dspark-bench-pytorch-float4-maxbs:128-maxnt:9216-kv_frac:0.5-input_output_len:8192,1024-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_v4_pro_dspark-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) #kimi_k2.5_fp4 (multimodal MoE FP4 553G, 8-GPU ep=8) - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-input_output_len:128,128-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-input_output_len:500,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-input_output_len:2000,500-ep:8-tp:8-gpus:8] TIMEOUT(120) - - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-input_output_len:1000,1000-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-input_output_len:8000,1000-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) - - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.6-input_output_len:1000,1000-con:512-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput + - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.6-input_output_len:8000,1000-con:512-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput #glm_5_fp8 (MoE FP8 708G, 8-GPU ep=8) - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-input_output_len:500,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-input_output_len:2000,500-ep:8-tp:8-gpus:8] TIMEOUT(120) - - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-input_output_len:1000,1000-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-input_output_len:8000,1000-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) - - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-kv_frac:0.6-input_output_len:1000,1000-con:512-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput + - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-kv_frac:0.6-input_output_len:8000,1000-con:512-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput #deepseek_v3.2_fp4 (FP4 389G, 8-GPU ep=8) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:384-maxnt:1536-input_output_len:1000,2000-reqs:10000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max throughput test - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-input_output_len:128,128-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:384-maxnt:1536-input_output_len:1000,1000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput + - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:384-maxnt:1536-input_output_len:8000,1000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput # 8: H100, H20, B200, B300, RTX6000-Server test cases @@ -285,26 +315,26 @@ llm_perf_core: - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-gpus:8] - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:500,2000-ep:8-gpus:8] - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:2000,500-ep:8-gpus:8] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:1000,1000-ep:8-gpus:8] + - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:8000,1000-ep:8-gpus:8] - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-gpus:8] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-input_output_len:1000,1000-con:512-ep:8-gpus:8] #max_throughput + - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-input_output_len:8000,1000-con:512-ep:8-gpus:8] #max_throughput #llama_v3.1_nemotron_ultra_253b_fp8 (nemotron-nas FP8 241G, 8-GPU) - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:128,128-tp:8-gpus:8] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:500,2000-tp:8-gpus:8] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:2000,500-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:1000,1000-tp:8-gpus:8] + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:8000,1000-tp:8-gpus:8] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:1000,2000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:1000,1000-con:250-tp:8-gpus:8] #max_throughput + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:8000,1000-con:250-tp:8-gpus:8] #max_throughput #qwen3.5_397b_a17b_fp8 (MoE FP8 380G, 8-GPU ep=8) - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:500,2000-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:2000,500-ep:8-tp:8-gpus:8] - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:1000,1000-ep:8-tp:8-gpus:8] + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:8000,1000-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-tp:8-gpus:8] - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-maxbs:512-input_output_len:1000,1000-con:512-ep:8-tp:8-gpus:8] #max_throughput + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-maxbs:512-input_output_len:8000,1000-con:512-ep:8-tp:8-gpus:8] #max_throughput - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:128,128-ep:4-tp:4-gpus:4] @@ -320,27 +350,27 @@ llm_perf_core: gt: 90000 tests: # deepseek_r1_0528 - - perf/test_perf.py::test_perf[deepseek_r1_0528_fp8-bench-pytorch-float8-input_output_len:1000,1000-reqs:20000-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_r1_0528_fp8-bench-pytorch-float8-input_output_len:8000,1000-reqs:20000-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp8-bench-pytorch-float8-input_output_len:1000,2000-reqs:3000-ep:8-tp:8-gpus:8] TIMEOUT(100) #deepseek_v3.2_fp8 (FP8 645G, 8-GPU ep=8) - perf/test_perf.py::test_perf[deepseek_v3.2_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-tp:8-gpus:8] - - perf/test_perf.py::test_perf[deepseek_v3.2_fp8-bench-pytorch-float8-maxbs:384-maxnt:1536-input_output_len:1000,1000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput + - perf/test_perf.py::test_perf[deepseek_v3.2_fp8-bench-pytorch-float8-maxbs:384-maxnt:1536-input_output_len:8000,1000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput #qwen3.5_397b_a17b_fp4 (MoE FP4 234G, 8-GPU ep=8) - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:128,128-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:500,2000-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:2000,500-ep:8-tp:8-gpus:8] - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:1000,1000-ep:8-tp:8-gpus:8] + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:8000,1000-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:1000,2000-ep:8-tp:8-gpus:8] - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:512-input_output_len:1000,1000-con:512-ep:8-tp:8-gpus:8] #max_throughput + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:512-input_output_len:8000,1000-con:512-ep:8-tp:8-gpus:8] #max_throughput #llama_v3.1_nemotron_ultra_253b (nemotron-nas BF16 474G, 8-GPU) - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:128,128-tp:8-gpus:8] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:8-gpus:8] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:1000,1000-tp:8-gpus:8] + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:8000,1000-tp:8-gpus:8] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250-tp:8-gpus:8] #max_throughput + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250-tp:8-gpus:8] #max_throughput # 10: RTX-6000 Server test cases - condition: @@ -354,7 +384,7 @@ llm_perf_core: #llama_v3.3_70b - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-bfloat16-maxbs:1-input_output_len:128,128-reqs:10-gpus:2] - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp4-bench-pytorch-float4-input_output_len:128,128-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp4-bench-pytorch-float4-maxbs:1024-maxnt:4096-kv_frac:0.85-input_output_len:1000,1000-reqs:3000-tp:8-gpus:8] + - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp4-bench-pytorch-float4-maxbs:1024-maxnt:4096-kv_frac:0.85-input_output_len:8000,1000-reqs:3000-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp4-bench-pytorch-float4-input_output_len:1000,2000-con:8-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp4-bench-pytorch-float4-input_output_len:1000,2000-con:512-ep:8-tp:8-gpus:8] # deepseek_r1_0528 diff --git a/tests/integration/test_lists/qa/llm_perf_disagg.yml b/tests/integration/test_lists/qa/llm_perf_disagg.yml index c6095b030e58..9acefe684999 100644 --- a/tests/integration/test_lists/qa/llm_perf_disagg.yml +++ b/tests/integration/test_lists/qa/llm_perf_disagg.yml @@ -11,59 +11,33 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_con1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL] TIMEOUT (120) # GB200 DeepSeek-V32 - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX] TIMEOUT (120) # GB200 GPT-OSS-120B - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) # GB200 Kimi-K2.5-Thinking - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) # GB200 Qwen3-235B - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) @@ -71,27 +45,12 @@ llm_perf_disagg: # GB200 wideep - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] TIMEOUT (120) # GB200 accuracy cases - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) # GB200 stress cases - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_stress-deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL] TIMEOUT (120) # GB200 aggregated ctx_only - - perf/test_perf_sanity.py::test_e2e[aggr-ctx_only-gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[aggr-ctx_only-gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) # GB200 aggregated gen_only - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) # 2: GB300 test cases - condition: @@ -104,19 +63,11 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX] TIMEOUT (120) # GB300 DeepSeek-V32 - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX] TIMEOUT (120) @@ -126,24 +77,18 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX] TIMEOUT (120) + # GB300 DeepSeek-V4-Pro + - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con1229_ctx7_dep4_gen1_dep8_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx8_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con2_ctx1_dep4_gen5_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) # GB300 Kimi-K2.5-Thinking - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) # GB300 Qwen3-235B - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) # GB300 wideep - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] TIMEOUT (120) # GB300 aggregated ctx_only - - perf/test_perf_sanity.py::test_e2e[aggr-ctx_only-gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) # GB300 aggregated gen_only - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/qa/llm_perf_multinode.txt b/tests/integration/test_lists/qa/llm_perf_multinode.txt index 1de41b361772..df228f9ca0c9 100644 --- a/tests/integration/test_lists/qa/llm_perf_multinode.txt +++ b/tests/integration/test_lists/qa/llm_perf_multinode.txt @@ -4,18 +4,7 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_con1_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] @@ -23,25 +12,16 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_8k1k_con4096 perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL] # GB200 DeepSeek-V32 -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX] # GB200 GPT-OSS-120B -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] @@ -49,18 +29,12 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_8k1k_con4_c perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] # GB200 Kimi-K2.5-Thinking -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] # GB200 Qwen3-235B -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] @@ -71,20 +45,12 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_128k8k_con1_ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX] # GB300 DeepSeek-V32 -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX] @@ -95,66 +61,42 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_8k1k_con1_c perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX] +# GB300 DeepSeek-V4-Pro +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con1229_ctx7_dep4_gen1_dep8_eplb384_mtp3_ccb-NIXL] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx8_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con2_ctx1_dep4_gen5_tep4_eplb0_mtp3_ccb-NIXL] + # GB300 GLM-5 -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] # GB300 Kimi-K2.5-Thinking -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] # GB300 Qwen3-235B -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] # wideep multi-node # GB200 wideep perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] # GB300 wideep perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] # accuracy cases -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] # stress cases -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_stress-deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL] # aggregated multi-node (ctx_only and gen_only reuse disagg config yamls) # ctx_only -perf/test_perf_sanity.py::test_e2e[aggr-ctx_only-gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[aggr-ctx_only-gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[aggr-ctx_only-gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] # gen_only -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] diff --git a/tests/integration/test_lists/qa/llm_perf_multinode.yml b/tests/integration/test_lists/qa/llm_perf_multinode.yml index 9b8f0a18a593..770fac50b808 100644 --- a/tests/integration/test_lists/qa/llm_perf_multinode.yml +++ b/tests/integration/test_lists/qa/llm_perf_multinode.yml @@ -7,9 +7,6 @@ llm_perf_multinode: gpu: - 'b200' tests: - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) @@ -20,15 +17,7 @@ llm_perf_multinode: gpu: - 'gb200' tests: - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_kimi-k2-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/README.md b/tests/integration/test_lists/test-db/README.md index f962e9053f0a..74e8a137cceb 100644 --- a/tests/integration/test_lists/test-db/README.md +++ b/tests/integration/test_lists/test-db/README.md @@ -31,8 +31,8 @@ l0_e2e: - '*h100*' linux_distribution_name: ubuntu* tests: - - examples/test_llama.py::test_llm_llama_v3_1_1node_multi_gpus[llama-3.1-8b-enable_fp8] - - examples/test_llama.py::test_llm_llama_v3_1_1node_multi_gpus[llama-3.1-70b-enable_fp8] + - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=False-attn_backend=TRTLLM-torch_compile=False] + - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=False] ``` ## Generating Test Lists diff --git a/tests/integration/test_lists/test-db/l0_a10.yml b/tests/integration/test_lists/test-db/l0_a10.yml index efc7286a8ff6..2db42615e237 100644 --- a/tests/integration/test_lists/test-db/l0_a10.yml +++ b/tests/integration/test_lists/test-db/l0_a10.yml @@ -15,6 +15,8 @@ l0_a10: tests: # ------------- PyTorch tests --------------- - unittest/_torch/sampler/test_torch_sampler.py + - unittest/_torch/sampler/test_penalties.py + - unittest/_torch/test_tensor_lru_cache.py - unittest/_torch/test_torch_multi_arange.py - unittest/utils/test_util.py - unittest/utils/test_logger.py @@ -28,8 +30,10 @@ l0_a10: - unittest/_torch/modeling/test_modeling_parakeet.py - unittest/_torch/modeling/test_modeling_radio.py - unittest/_torch/modeling/test_modeling_step3p7.py + - unittest/_torch/modeling/test_modeling_minicpmv4_6.py - unittest/_torch/modeling/test_multimodal_encoder_mixin.py - unittest/_torch/sampler/test_trtllm_sampler.py + - unittest/_torch/sampler/test_token_ban.py - unittest/_torch/executor/test_async_transfer_manager.py - unittest/_torch/executor/test_scheduler_serializable_output.py - unittest/_torch/executor/test_kv_cache_estimation.py @@ -37,6 +41,7 @@ l0_a10: - unittest/_torch/executor/test_kv_pool_rebalance.py - unittest/_torch/executor/test_disagg_index_mapper_early_release.py - unittest/_torch/executor/test_kv_cache_compression_manager.py + - unittest/_torch/executor/test_kv_cache_v2_capacity_only.py - unittest/_torch/executor/test_error_classification.py - unittest/_torch/modules/dwdp/test_dwdp_fixup_moe_backends.py - unittest/_torch/modules/dwdp/test_dwdp_manager.py @@ -45,19 +50,26 @@ l0_a10: # NOTE: this is a CPU-only test, but we do not have a dedicated job for this (and therefore no # test list either). - unittest/_torch/models/checkpoints + - unittest/_torch/models/test_qwen3_next_moe_quant.py - unittest/_torch/weight_sharing - unittest/inputs/test_chat_template_dispatch.py - unittest/inputs/test_content_format.py - unittest/inputs/test_url_validation.py - unittest/inputs/test_multimodal.py - unittest/inputs/test_multimodal_input_processor.py + - unittest/inputs/test_video_decode.py + - unittest/others/test_cache_transceiver_precheck_config.py + - unittest/others/test_cache_transceiver_precheck_run.py - unittest/others/test_convert_utils.py + - unittest/others/test_lm_eval.py - unittest/others/test_lora_manager.py - unittest/others/test_lora_module_count.py - unittest/others/test_time_breakdown.py - unittest/others/test_tracing.py - unittest/disaggregated/test_disagg_openai_client.py - unittest/disaggregated/test_disagg_utils.py + - unittest/disaggregated/test_coordinator_e2e.py + - unittest/disaggregated/test_coordinator_worker.py - unittest/disaggregated/test_openai_disagg_server.py - unittest/disaggregated/test_openai_disagg_service.py - unittest/disaggregated/test_router.py @@ -68,6 +80,7 @@ l0_a10: - unittest/disaggregated/test_cluster_storage.py - unittest/disaggregated/test_extractor.py - unittest/disaggregated/test_peer.py + - unittest/disaggregated/test_cache_reuse_adapter.py - unittest/disaggregated/test_bounce.py - unittest/disaggregated/region/test_block.py - unittest/disaggregated/test_mamba_transfer.py @@ -101,6 +114,8 @@ l0_a10: - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[False-True-TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-False-TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_llama[True-True-TinyLlama-1.1B-Chat-v1.0] + - disaggregated/test_disaggregated_single_gpu.py::test_arbitrary_kv_cache_transfer[False-TinyLlama-1.1B-Chat-v1.0] + - disaggregated/test_disaggregated_single_gpu.py::test_arbitrary_kv_cache_transfer_missing_blocks[False-TinyLlama-1.1B-Chat-v1.0] - test_e2e.py::test_get_ci_container_port - test_e2e.py::test_openai_chat_multimodal_example ISOLATION - test_e2e.py::test_openai_mmencoder_example @@ -177,36 +192,6 @@ l0_a10: - thirdparty/test_git_modules.py::test_gitmodules # helper-script unit tests (CPU-only, ~3s) - unittest/scripts -- condition: - ranges: - system_gpu_count: - gte: 1 - lte: 1 - wildcards: - gpu: - - '*a10*' - linux_distribution_name: ubuntu* - terms: - stage: pre_merge - backend: cpp - tests: - # ------------- CPP tests --------------- - - cpp/test_e2e.py::test_model[-redrafter-86] - - cpp/test_e2e.py::test_model[-mamba-86] -- condition: - ranges: - system_gpu_count: - gte: 1 - lte: 1 - wildcards: - gpu: - - '*a10*' - linux_distribution_name: ubuntu* - terms: - stage: pre_merge - backend: tensorrt - tests: - # ------------- TRT tests --------------- - unittest/kv_cache_manager_v2_tests # 4 min - unittest/dynamo - unittest/api_stability @@ -222,30 +207,16 @@ l0_a10: linux_distribution_name: ubuntu* terms: stage: post_merge - backend: tensorrt + backend: pytorch tests: - # ------------- Move from pre_merge to post_merge --------------- + - stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-GUARANTEED_NO_EVICT-pytorch-stress-test] + - stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-MAX_UTILIZATION-pytorch-stress-test] - llmapi/test_llm_examples.py::test_llmapi_chat_example - llmapi/test_llm_examples.py::test_llmapi_server_example - llmapi/test_llm_examples.py::test_llmapi_kv_cache_connector[Qwen2-0.5B] - test_e2e.py::test_openai_health - test_e2e.py::test_trtllm_serve_example - llmapi/test_llm_examples.py::test_llmapi_quickstart_atexit -- condition: - ranges: - system_gpu_count: - gte: 1 - lte: 1 - wildcards: - gpu: - - '*a10*' - linux_distribution_name: ubuntu* - terms: - stage: post_merge - backend: pytorch - tests: - - stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-GUARANTEED_NO_EVICT-pytorch-stress-test] - - stress_test/stress_test.py::test_run_stress_test[llama-v3-8b-instruct-hf_tp1-stress_time_300s_timeout_450s-MAX_UTILIZATION-pytorch-stress-test] - condition: ranges: system_gpu_count: diff --git a/tests/integration/test_lists/test-db/l0_a100.yml b/tests/integration/test_lists/test-db/l0_a100.yml index a61443e641ad..fe9ec737f01c 100644 --- a/tests/integration/test_lists/test-db/l0_a100.yml +++ b/tests/integration/test_lists/test-db/l0_a100.yml @@ -52,7 +52,7 @@ l0_a100: linux_distribution_name: ubuntu* terms: stage: post_merge - backend: tensorrt + backend: pytorch tests: - unittest/llmapi/test_llm.py -m "part0" - unittest/llmapi/test_llm.py -m "not part0" TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_a30.yml b/tests/integration/test_lists/test-db/l0_a30.yml index 125fb8191b1b..14ddddd8c619 100644 --- a/tests/integration/test_lists/test-db/l0_a30.yml +++ b/tests/integration/test_lists/test-db/l0_a30.yml @@ -64,42 +64,6 @@ l0_a30: - cpp/test_unit_tests.py::test_unit_tests[layers-80] - cpp/test_unit_tests.py::test_unit_tests[runtime-80] - cpp/test_unit_tests.py::test_unit_tests[thop-80] - - cpp/test_unit_tests.py::test_unit_tests[utils-80] - - cpp/test_e2e.py::test_model[-gpt-80] - - cpp/test_e2e.py::test_benchmarks[gpt-80] TIMEOUT (90) -- condition: - ranges: - system_gpu_count: - gte: 1 - lte: 1 - wildcards: - gpu: - - '*a30*' - linux_distribution_name: ubuntu* - terms: - stage: post_merge - backend: tensorrt - tests: - # ------------- TRT tests --------------- - - examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] - - examples/test_granite.py::test_llm_granite[granite-3.0-1b-a400m-instruct-bfloat16] # 5 mins - - examples/test_internlm.py::test_llm_internlm2_7b_1node_1gpu[bfloat16-enable_context_fmha-enable_gemm_plugin-enable_attention_plugin-nb:2] # 5 mins - - examples/test_ngram.py::test_llm_ngram_1gpu[streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs2] # 1 min -- condition: - ranges: - system_gpu_count: - gte: 1 - lte: 1 - wildcards: - gpu: - - '*a30*' - linux_distribution_name: ubuntu* - terms: - stage: post_merge - backend: tensorrt - tests: - - examples/test_granite.py::test_llm_granite[granite-3.0-2b-instruct-bfloat16] # 5 mins - - examples/test_ngram.py::test_llm_ngram_1gpu[no_streaming-gpt2-use_cpp_session-use_tokens-max_matching_ngram_size_2-max_draft_len_8-float16-bs2] # 1 min # ------------- AutoDeploy Backend Stages --------------- - condition: ranges: diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index f633a590f4c8..51f7f6d821e3 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -63,39 +63,6 @@ l0_b200: - accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4] - accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_mtp - disaggregated/test_workers.py::test_workers_kv_cache_aware_router_eviction[TinyLlama-1.1B-Chat-v1.0] # nvbugs 5300551 - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-off-greedy-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-off-greedy-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-greedy-batch2-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v2-decoder-cuda-graph-on-greedy-batch2-bart-large-cnn] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-t5-small0] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-t5-base] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-t5-large] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-flan-t5-base] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-flan-t5-large] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-flan-t5-xl] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-flan-t5-xxl] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-t5-small1] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-off-beam2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v1-cuda-graph-off-beam2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-off-beam2-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v1-cuda-graph-off-beam2-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-off-greedy-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v2-cuda-graph-off-greedy-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v2-cuda-graph-off-greedy-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-off-greedy-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v2-cuda-graph-off-greedy-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v2-cuda-graph-off-greedy-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-off-greedy-byt5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-beam2-batch2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-beam2-batch2-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-greedy-batch2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v2-decoder-cuda-graph-on-greedy-batch2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_context_generation_batch - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B] - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8] - test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16] @@ -109,6 +76,7 @@ l0_b200: - unittest/_torch/compilation - unittest/_torch/debugger - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_2_model_mtp + - unittest/disaggregated/test_deepseek_v4_kv_transfer.py - unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_sparse_mla.py TIMEOUT (60) - unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_cache_manager.py TIMEOUT (60) - unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_indices_transform.py TIMEOUT (60) @@ -129,6 +97,7 @@ l0_b200: - unittest/_torch/modules/fused_ops/test_rmsnorm_fp4_quant.py - unittest/_torch/modules/fused_ops/test_gelu_tanh_mul_fp4_quant.py - unittest/_torch/modules/fused_ops/test_rmsnorm_residual_add.py + - unittest/_torch/modules/fused_ops/test_fused_qk_norm_rope_gate.py - unittest/_torch/modules/test_gemma4_fused_qkv_prep.py - unittest/_torch/modules/test_fused_rmsnorm_fp4_quantize.py - unittest/_torch/modules/test_fp4_num_tokens_slice.py @@ -145,6 +114,7 @@ l0_b200: - unittest/_torch/modules/fused_moe/test_deepgemm_fused_gather_finalize.py - unittest/_torch/modules/fused_moe/test_deepgemm_fused_expand_quant.py # ------------- MoE: test_moe_backend (by backend) --------------- + - unittest/_torch/modules/moe/test_megamoe_streaming_load.py - unittest/_torch/modules/moe/test_moe_backend.py::test_trtllm_bf16_unquantized_moe # ------------- MoE: test_single_gpu (by backend) --------------- - unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k "CUTLASS and not None" @@ -241,6 +211,7 @@ l0_b200: - unittest/_torch/visual_gen/test_flux_transformer.py - unittest/_torch/visual_gen/test_flux_attention.py - unittest/_torch/visual_gen/test_flux_pipeline.py + - unittest/_torch/visual_gen/test_flux2_image_conditioning.py - unittest/_torch/visual_gen/test_ltx2_transformer.py - unittest/_torch/visual_gen/test_ltx2_attention.py - unittest/_torch/visual_gen/test_ltx2_pipeline.py @@ -259,13 +230,17 @@ l0_b200: - unittest/_torch/visual_gen/test_wan_transformer.py - unittest/_torch/visual_gen/test_cosmos3_transformer.py - unittest/_torch/visual_gen/test_cosmos3_pipeline.py + - unittest/_torch/visual_gen/test_cosmos3_distilled.py - examples/visual_gen/test_visual_gen.py::test_wan_t2v_example - examples/visual_gen/test_visual_gen.py::test_flux1_example - examples/visual_gen/test_visual_gen.py::test_flux2_example + - examples/visual_gen/test_visual_gen.py::test_flux2_reference_image_example - examples/visual_gen/test_visual_gen.py::test_ltx2_example - examples/visual_gen/test_visual_gen.py::test_wan_i2v_example - examples/visual_gen/test_visual_gen.py::test_cosmos3_example - examples/visual_gen/test_visual_gen.py::test_qwen_image_example + - examples/visual_gen/test_visual_gen.py::test_qwen_image_layered_example + - examples/visual_gen/test_visual_gen.py::test_qwen_image_edit_example # - examples/visual_gen/test_visual_gen.py # ------------- Host perf module regression tests (6 representative scenarios) --------------- - perf/host_perf/test_module_scheduler.py::test_scheduler_production[production_gen_only_bs8] @@ -290,12 +265,12 @@ l0_b200: linux_distribution_name: ubuntu* terms: stage: post_merge - backend: tensorrt + backend: pytorch tests: - # ------------- TRT tests --------------- - unittest/llmapi/test_llm_quant.py # 3.5 mins on B200 - - unittest/disaggregated/test_router.py - unittest/disaggregated/test_openai_server_info.py + - examples/visual_gen/test_visual_gen.py::test_cosmos3_t2i_4step_example TIMEOUT (30) + - examples/visual_gen/test_visual_gen.py::test_cosmos3_i2v_4step_example TIMEOUT (45) - condition: ranges: system_gpu_count: @@ -367,8 +342,11 @@ l0_b200: # Measured on B200 with TRT-LLM commit 85665f5f from the staging main image: # QwenImage ~5 min, Cosmos3-Nano T2I ~3 min, and T2V ~5 min. - examples/visual_gen/test_visual_gen.py::test_qwenimage_lpips_against_golden TIMEOUT (10) + - examples/visual_gen/test_visual_gen.py::test_qwenimage_cuda_graph_lpips_against_golden TIMEOUT (10) + - examples/visual_gen/test_visual_gen.py::test_qwen_image_layered_lpips_against_golden TIMEOUT (10) - examples/visual_gen/test_visual_gen.py::test_cosmos3_nano_t2i_lpips_against_golden TIMEOUT (10) - examples/visual_gen/test_visual_gen.py::test_cosmos3_nano_t2v_lpips_against_golden TIMEOUT (15) + - examples/visual_gen/test_visual_gen.py::test_cosmos3_i2v_4step_lpips_against_golden TIMEOUT (20) - visual_gen/test_visual_gen_benchmark.py::test_offline_benchmark - visual_gen/test_visual_gen_benchmark.py::test_online_benchmark[openai-videos] # ---- moved to post-merge (MoE CI optimization) ---- diff --git a/tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml index 4bd8ea1e828a..b963537960c7 100644 --- a/tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml @@ -16,15 +16,11 @@ l0_b200_multi_gpus_perf_sanity: orchestrator: mpi tests: # deepseek_r1_fp8_blackwell - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_blackwell-r1_fp8_tp8_mtp3_1k1k] - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_blackwell-r1_fp8_dep8_mtp1_1k1k] TIMEOUT (90) # failed - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_blackwell-r1_fp8_tp8_mtp3_8k1k] # - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_blackwell-r1_fp8_dep8_mtp1_8k1k] TIMEOUT (90) # failed - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_blackwell-r1_fp8_tp8_6k1k] TIMEOUT (90) # deepseek-r1-fp4-v2 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_blackwell-r1_fp4_v2_tp4_mtp3_1k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_blackwell-r1_fp4_v2_tp4_mtp3_8k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_blackwell-r1_fp4_v2_dep8_mtp1_1k1k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_blackwell-r1_fp4_v2_dep8_mtp1_8k1k] TIMEOUT (90) # deepseek-v32-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_tep8_mtp3_8k1k] @@ -33,8 +29,6 @@ l0_b200_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_blackwell-glm5_fp4_tep8_mtp3_8k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_blackwell-glm5_fp4_dep8_mtp1_8k1k] TIMEOUT (90) # gpt-oss-120b-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tp2_mtp0_1k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tp1_mtp0_1k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tp1_mtp0_8k1k] # llama-3.3-70b-instruct-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-llama_v3_3_70b_instruct_fp4_blackwell-llama70b_fp4_tp4_512_32] @@ -46,18 +40,11 @@ l0_b200_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_thinking_fp4_dep8_32k8k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_k25_thinking_fp4_blackwell-k25_thinking_fp4_tep8_adp_2k1k] # qwen3.5-397b-a17b-fp4 aggregated - - perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_tep8_1k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_dep4_1k1k] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_tp2_mtp3_1k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_dep8_mtp3_1k1k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_tp4_8k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_dep8_8k1k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_tep4_mtp3_8k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_dep8_mtp3_8k1k] TIMEOUT (90) # ctx_only tests (disagg config) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml b/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml index da5fde3c3f94..960b9cf6dc37 100644 --- a/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml @@ -11,18 +11,27 @@ l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8: gpu: - '*b200*' terms: - stage: post_merge + stage: pre_merge backend: pytorch tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) + +- condition: + ranges: + # 1 ctx worker with each 1 node and 4 GPUs + # 1 gen worker with each 1 node and 8 GPUs + system_gpu_count: + gte: 16 + lte: 16 + wildcards: + gpu: + - '*b200*' + terms: + stage: post_merge + backend: pytorch + tests: - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - # - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - # - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - # - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) # - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) # - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) # - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml b/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml index e570b693845c..a8d684360976 100644 --- a/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml +++ b/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml @@ -17,6 +17,5 @@ l0_b200_multi_nodes_perf_sanity_node2_gpu16: orchestrator: mpi tests: # deepseek-r1-fp4-v2 aggregated 2-nodes - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_blackwell-r1_fp4_v2_dep16_mtp1_1k1k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_blackwell-r1_fp4_v2_dep16_mtp1_8k1k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_blackwell-r1_fp4_v2_tep16_mtp3] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_b200_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_b200_perf_sanity.yml new file mode 100644 index 000000000000..3ef357924a0a --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_b200_perf_sanity.yml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version: 0.0.1 +l0_b200_perf_sanity: +- condition: + ranges: + system_gpu_count: + gte: 1 + lte: 1 + wildcards: + gpu: + - '*b200*' + linux_distribution_name: ubuntu* + cpu: x86_64 + terms: + stage: pre_merge + backend: pytorch + orchestrator: mpi + tests: + # gemma-4-26b-a4b-nvfp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gemma4_26b_a4b_nvfp4_blackwell-gemma4_26b_a4b_nvfp4_tp1_1k1k] TIMEOUT (60) diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index 5d420918073e..4e7347edd644 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -16,7 +16,10 @@ l0_b300: backend: pytorch tests: # ------------- PyTorch tests --------------- - - unittest/_torch/attention # 200s + - unittest/_torch/attention --ignore=unittest/_torch/attention/sparse/test_cute_dsl_fp8_paged_mqa_logits.py --ignore=unittest/_torch/attention/sparse/test_cute_dsl_fp4_paged_mqa_logits.py --ignore=unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py + - unittest/_torch/attention/sparse/test_cute_dsl_fp8_paged_mqa_logits.py + - unittest/_torch/attention/sparse/test_cute_dsl_fp4_paged_mqa_logits.py + - unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py - unittest/_torch/thop/parallel TIMEOUT (90) - unittest/_torch/thop/serial - unittest/_torch/executor # 250s diff --git a/tests/integration/test_lists/test-db/l0_cpu_arm.yml b/tests/integration/test_lists/test-db/l0_cpu_arm.yml index ba693acaf756..9b35d56db7bd 100644 --- a/tests/integration/test_lists/test-db/l0_cpu_arm.yml +++ b/tests/integration/test_lists/test-db/l0_cpu_arm.yml @@ -14,3 +14,5 @@ l0_cpu_arm: orchestrator: mpi tests: - unittest/executor/test_rpc.py + - unittest/executor/test_event_loop_error_broadcast.py + - unittest/others/test_http_utils_fail_fast.py diff --git a/tests/integration/test_lists/test-db/l0_cpu_x86.yml b/tests/integration/test_lists/test-db/l0_cpu_x86.yml index 9a39993347b8..072f17dcc00a 100644 --- a/tests/integration/test_lists/test-db/l0_cpu_x86.yml +++ b/tests/integration/test_lists/test-db/l0_cpu_x86.yml @@ -14,3 +14,6 @@ l0_cpu_x86: orchestrator: mpi tests: - unittest/executor/test_rpc.py + - unittest/others/test_http_utils_fail_fast.py + - unittest/executor/test_multi_frontend_routing.py + - unittest/executor/test_event_loop_error_broadcast.py diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index d5d316518f22..ea79211a0777 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -18,11 +18,6 @@ l0_dgx_b200: - unittest/_torch/misc/test_autotuner.py::test_autotuner_distributed_strategy - accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp2-TRTLLM] - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync[cpp] - # ------------- Encoder-decoder TP tests --------------- - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-tp2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-tp2-t5-small] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-tp2-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-tp2-bart-large-cnn] # ------------- KV Cache V2 Scheduler IT (multi-GPU) --------------- - kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_draft_tokens - kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2DSv3Lite::test_mtp_chunked_draft_tokens @@ -48,6 +43,7 @@ l0_dgx_b200: - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpu_mtp_ar TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_cutedsl] - accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_block_reuse] + - accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_mtp3_gdn_replay_tep4 - accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL] - accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=True] @@ -55,7 +51,7 @@ l0_dgx_b200: - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp_tp4] - - accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] TIMEOUT (60) + - accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] TIMEOUT (60) - unittest/_torch/modeling/test_modeling_deepseekv4.py - accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Flash::test_auto_dtype TIMEOUT (60) @@ -66,6 +62,7 @@ l0_dgx_b200: - unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_fused_cat_fp4_shape_dispatch - unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_fused_cat_fp4_noncontiguous_split - unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_fused_cat_fp4_dsv32_prefill_shape + - unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_cute_dsl_fp8_indexer_q_gemm_rope_fp4_matches_unfused - unittest/_torch/attention/sparse/test_cpp_custom_ops.py::test_indexer_k_cache_gather_contiguous_fp4 - unittest/_torch/attention/sparse/dsa/test_dsa_fp4_indexer.py - condition: @@ -148,12 +145,8 @@ l0_dgx_b200: backend: pytorch orchestrator: mpi tests: - - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp2tp1cp2] TIMEOUT (60) - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1dp2cp2] TIMEOUT (60) - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp2cp2] TIMEOUT (60) - - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp1cp4] TIMEOUT (60) - - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp1cp4] TIMEOUT (60) - - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_mtp] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_bs8_mtp] TIMEOUT (60) @@ -171,9 +164,12 @@ l0_dgx_b200: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp1] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp3_no_adp] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestGLM52::test_nvfp4[tp_size=8-ep_size=8] TIMEOUT (60) + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Pro::test_gsm8k_full_accuracy TIMEOUT (240) + - examples/test_deepseek_v4_pro.py::test_short_token_boundary_smoke TIMEOUT (120) - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] TIMEOUT (60) - accuracy/test_disaggregated_serving.py::TestKimiK25::test_nvfp4 TIMEOUT (180) - accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8] TIMEOUT (60) + - accuracy/test_llm_api_pytorch.py::TestDeepSeekV4ProDSpark::test_gsm8k_dep8_megamoe_deepgemm TIMEOUT (240) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_on-cutedsl] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_off-trtllm] TIMEOUT (60) @@ -226,14 +222,7 @@ l0_dgx_b200: backend: pytorch orchestrator: mpi tests: - - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1tp2cp2] TIMEOUT (60) - - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1tp1cp4] TIMEOUT (60) - - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp2tp1cp2] TIMEOUT (60) - - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1dp2cp2] TIMEOUT (60) - - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1dp2cp2] TIMEOUT (60) - - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp2tp1cp2] TIMEOUT (60) - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_tllm_gen_helix[DeepSeek-V3-Lite-bf16-short_prompt] TIMEOUT (60) - - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_tllm_gen_helix[DeepSeek-V3-Lite-bf16-long_prompt] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus_corner_case TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[baseline_fp8kv] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[latency] TIMEOUT (60) @@ -248,7 +237,7 @@ l0_dgx_b200: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[fp4_indexer_dsl_mtp3] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[baseline_pp4_mtp1] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_chunked_prefill[baseline_fp8kv] TIMEOUT (60) - - accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph[tp_size=8-ep_size=8] TIMEOUT (180) + - accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph[use_msa=False] TIMEOUT (180) - accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8_attn_dp] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[ep8] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[dep8] TIMEOUT (60) @@ -275,6 +264,7 @@ l0_dgx_b200: backend: pytorch orchestrator: mpi tests: + - accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp2-CUTLASS] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=FLASHINFER-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=False] @@ -344,6 +334,7 @@ l0_dgx_b200: - examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[tp2] - examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[cfg2_tp2] - examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[tp2_ulysses2] + - examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[tp2_attn2d_2x1] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=True] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=False] diff --git a/tests/integration/test_lists/test-db/l0_dgx_b300.yml b/tests/integration/test_lists/test-db/l0_dgx_b300.yml index 5b53885685f9..736319d8409f 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b300.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b300.yml @@ -15,7 +15,10 @@ l0_dgx_b300: stage: post_merge backend: pytorch tests: - - unittest/_torch/attention + - unittest/_torch/attention --ignore=unittest/_torch/attention/sparse/test_cute_dsl_fp8_paged_mqa_logits.py --ignore=unittest/_torch/attention/sparse/test_cute_dsl_fp4_paged_mqa_logits.py --ignore=unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py + - unittest/_torch/attention/sparse/test_cute_dsl_fp8_paged_mqa_logits.py + - unittest/_torch/attention/sparse/test_cute_dsl_fp4_paged_mqa_logits.py + - unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py - unittest/_torch/executor # ------------- modules (multi-GPU) --------------- - unittest/_torch/modules/test_mla_helix.py @@ -64,6 +67,8 @@ l0_dgx_b300: - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8] - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend + - accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1] + - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v1] - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index b11214e16256..e36416f7e5fa 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -21,10 +21,9 @@ l0_dgx_h100: - unittest/_torch/distributed - unittest/_torch/modeling/test_modeling_pixtral.py::test_tensor_parallelism # ------------- Encoder-decoder TP tests --------------- - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-tp2-t5-small] - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-tp2-t5-small] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-tp2-bart-large-cnn] - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-tp2-bart-large-cnn] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[fp32-kv-v1-graphs-off-greedy-tp2] # ------------- Disaggregated serving tests --------------- - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar] @@ -170,12 +169,14 @@ l0_dgx_h100: orchestrator: mpi tests: # ------------- MoE: test_multi_gpu (by backend x quant) --------------- - # Only CUTLASS backend runs on H100 (SM90). TRTLLM/CUTEDSL/DEEPGEMM require SM100+. + # Only CUTLASS and MARLIN backends run on H100 (SM90). TRTLLM/CUTEDSL/DEEPGEMM require SM100+. # --- CUTLASS --- - unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k "CUTLASS and FP8_BLOCK_SCALES" - unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k "CUTLASS and W8A16" - unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k "CUTLASS and W4A16_MXFP4" - unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k "CUTLASS and W4A8_AWQ" + # --- MARLIN (SM90-only; focused DEP + ALLGATHER x NVFP4 matrix) --- + - unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k "MARLIN" # ------------- MoE: test_multi_gpu_eplb --------------- - unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb - condition: @@ -283,9 +284,6 @@ l0_dgx_h100: - cpp/test_multi_gpu.py::test_cache_transceiver[8proc-ucx_kvcache-90] ISOLATION - cpp/test_multi_gpu.py::test_cache_transceiver[8proc-mooncake_kvcache-90] ISOLATION - cpp/test_multi_gpu.py::test_user_buffer[2proc-90] - - cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-mpi_kvcache-90] - - cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-ucx_kvcache-90] - - cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-nixl_kvcache-90] - condition: ranges: system_gpu_count: diff --git a/tests/integration/test_lists/test-db/l0_dgx_h200.yml b/tests/integration/test_lists/test-db/l0_dgx_h200.yml index ef63d80056fe..3b40b905220b 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h200.yml @@ -25,6 +25,7 @@ l0_dgx_h200: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp3_no_adp] - accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_nvfp4_marlin_multi_gpus[tp_size=8] - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_marlin_multi_gpus[tp_size=8] + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_marlin_adp_4gpus[mtp_nextn=3] - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_marlin_8gpus - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True] - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False] @@ -44,7 +45,7 @@ l0_dgx_h200: - disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ctxtp2ep2pp2_gentp4_one_mtp_block_reuse[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_qwen3_32b_fp8[Qwen3/Qwen3-32B-FP8] - - disaggregated/test_disaggregated.py::test_disaggregated_mixed_stress_test[req60-conc64-qwen3_32b_fp8_mixed_stress] + - disaggregated/test_disaggregated.py::test_disaggregated_mixed_stress_test[req120-conc64-qwen3_32b_fp8_mixed_stress] - unittest/llmapi/test_llm_pytorch.py::test_nemotron_nas_lora - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_spec_dec - condition: @@ -142,27 +143,6 @@ l0_dgx_h200: - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.5-fp8kv=True] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=True] -- condition: - ranges: - system_gpu_count: - gte: 4 - lte: 4 - wildcards: - gpu: - - '*h200*' - linux_distribution_name: ubuntu* - cpu: x86_64 - terms: - stage: post_merge - backend: tensorrt - tests: - # ------------- TRT tests --------------- - unittest/llmapi/test_llm_kv_cache_events.py::test_llm_api_attention_dp_kv_events - - examples/test_nemotron_nas.py::test_nemotron_nas_summary_2gpu[DeciLM-7B] - llmapi/test_llm_examples.py::test_llmapi_example_distributed_tp2 - - examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:2-bfloat16-bs:1-cpp_e2e:False-nb:1] - # ------------- TRT tests --------------- - - examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:2-pp:1-float16-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] - - examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:2-pp:1-float16-RobertaForQuestionAnswering-bert/roberta-base-squad2] - - examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-disable_attention_plugin-disable_context_fmha-tp:2-pp:1-float16-RobertaForSequenceClassification-bert/twitter-roberta-base-emotion] - unittest/llmapi/apps/_test_openai_multi_gpu.py -m "part0" diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml index 351b6d7aeac5..0dd2d3935113 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml @@ -59,6 +59,7 @@ l0_gb200_multi_gpus: - accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt - accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap - unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm + - unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_nccl_ep_cuda_graph_replay_uses_updated_routing - unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_postquant - disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python_fabric_memory[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_overlap_transceiver_runtime_python_bounce[TinyLlama-1.1B-Chat-v1.0] diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml index 11f0f8f0b44c..6227b7ee648e 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml @@ -15,8 +15,6 @@ l0_gb200_multi_gpus_perf_sanity: backend: pytorch tests: # deepseek-r1-fp4-v2 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_1k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_1k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_8k1k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_8k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_8k1k] @@ -25,7 +23,6 @@ l0_gb200_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) # k25-thinking-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_grace_blackwell-k25_thinking_fp4_dep4_8k1k] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) # qwen3-235b-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) @@ -49,40 +46,23 @@ l0_gb200_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k8k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_1k8k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_1k8k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k1k] TIMEOUT (90) # deepseek-v32-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_tep4_mtp3_1k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_1k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_tep4_mtp3_8k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_8k1k] # glm-5-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_grace_blackwell-glm5_fp4_tep4_mtp3_1k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_grace_blackwell-glm5_fp4_dep4_mtp1_1k1k] # gpt-oss-120b-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_1k8k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_dep4_1k8k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_dep2_1k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tep2_1k8k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp4_eagle3_1k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_mtp0_1k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_1k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp1_mtp0_8k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_1k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_8k1k] # k25-thinking-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_grace_blackwell-k25_thinking_fp4_tep4_8k1k] # ctx_only tests # deepseek-r1-fp4 - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) # deepseek-v32-fp4 - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] TIMEOUT (90) # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) @@ -90,14 +70,9 @@ l0_gb200_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (90) # gpt-oss-120b-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) # kimi-k25-thinking-fp4 - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml index dddbeb63b660..a2c11d454409 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml @@ -14,7 +14,4 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2: stage: post_merge backend: pytorch tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml index 626bbcce9521..48dc2a8e9790 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml @@ -1,5 +1,21 @@ version: 0.0.1 l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4: +- condition: + ranges: + # 1 ctx worker with each 1 node and 1 GPUs + # 1 gen worker with each 1 node and 4 GPUs + system_gpu_count: + gte: 8 + lte: 8 + wildcards: + gpu: + - '*gb200*' + terms: + stage: pre_merge + backend: pytorch + tests: + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) + - condition: ranges: # 1 ctx worker with each 1 node and 1 GPUs @@ -14,10 +30,8 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4: stage: post_merge backend: pytorch tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml deleted file mode 100644 index 27af1d826edb..000000000000 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 0.0.1 -l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4: -- condition: - ranges: - # 1 ctx worker with each 1 node and 4 GPUs - # 1 gen worker with each 1 node and 4 GPUs - system_gpu_count: - gte: 8 - lte: 8 - wildcards: - gpu: - - '*gb200*' - terms: - stage: post_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml index cdde60fa88dd..0af2db518631 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml @@ -14,12 +14,7 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8: stage: post_merge backend: pytorch tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) # Failed requests # - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml index d9e6b006b002..3f665caf1ab1 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml @@ -14,15 +14,11 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32: stage: post_merge backend: pytorch tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml deleted file mode 100644 index 4f36687cabbb..000000000000 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: 0.0.1 -l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32: -- condition: - ranges: - # 1 ctx worker with each 2 node and 8 GPUs - # 1 gen worker with each 8 node and 32 GPUs - system_gpu_count: - gte: 40 - lte: 40 - wildcards: - gpu: - - '*gb200*' - terms: - stage: post_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (180) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx2_node1_gpu4_gen1_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx2_node1_gpu4_gen1_node4_gpu16.yml deleted file mode 100644 index 984745251710..000000000000 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx2_node1_gpu4_gen1_node4_gpu16.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: 0.0.1 -l0_gb200_multi_nodes_perf_sanity_ctx2_node1_gpu4_gen1_node4_gpu16: -- condition: - ranges: - # 2 ctx workers with each 1 node and 4 GPUs - # 1 gen worker with each 4 node and 16 GPUs - system_gpu_count: - gte: 24 - lte: 24 - wildcards: - gpu: - - '*gb200*' - terms: - stage: post_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml index b1d454c5ad34..a939d640d8ef 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml @@ -15,7 +15,6 @@ l0_gb200_multi_nodes_perf_sanity_node2_gpu8: tests: # aggr config # deepseek-r1-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_dep8_mtp1_1k1k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_dep8_mtp1_8k1k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_tep8_mtp3] TIMEOUT (90) # k25-thinking-fp4 aggregated 2-nodes @@ -29,4 +28,3 @@ l0_gb200_multi_nodes_perf_sanity_node2_gpu8: # deepseek-r1-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (180) - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (180) diff --git a/tests/integration/test_lists/test-db/l0_gb203.yml b/tests/integration/test_lists/test-db/l0_gb203.yml index 11d692a85fdb..9fe58993698a 100644 --- a/tests/integration/test_lists/test-db/l0_gb203.yml +++ b/tests/integration/test_lists/test-db/l0_gb203.yml @@ -12,11 +12,8 @@ l0_gb203: linux_distribution_name: ubuntu* terms: stage: pre_merge - backend: tensorrt + backend: pytorch tests: - # ------------- TRT tests --------------- - # - examples/test_qwen.py::test_llm_qwen1_5_7b_single_gpu_lora[qwen1.5_7b_chat-Qwen1.5-7B-Chat-750Mb-lora] # https://nvbugs/5234573 - # - examples/test_qwen.py::test_llm_qwen_single_gpu_summary[qwen2.5_1.5b_instruct-enable_paged_kv_cache-enable_remove_input_padding-enable_weight_only-enable_fmha_fp32_acc] # https://nvbugs/5234573 - llmapi/test_llm_examples.py::test_llmapi_quickstart - llmapi/test_llm_examples.py::test_llmapi_example_inference - llmapi/test_llm_examples.py::test_llmapi_example_inference_async diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index 8b4796e86be3..669459270867 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -16,20 +16,18 @@ l0_gb300_multi_gpus_perf_sanity: tests: # ctx_only tests (disagg config) # deepseek-r1-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) # kimi-k25-thinking-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) # glm-5-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) + # deepseek-v4-pro-fp4 8k1k + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx12_node1_gpu4_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx12_node1_gpu4_gen1_node2_gpu8.yml new file mode 100644 index 000000000000..d4d0de7d1ea1 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx12_node1_gpu4_gen1_node2_gpu8.yml @@ -0,0 +1,19 @@ +version: 0.0.1 +l0_gb300_multi_nodes_perf_sanity_ctx12_node1_gpu4_gen1_node2_gpu8: +- condition: + ranges: + # 12 ctx workers each with 1 node and 4 GPUs + # 1 gen worker with 2 nodes and 8 GPUs + system_gpu_count: + gte: 56 + lte: 56 + wildcards: + gpu: + - '*gb300*' + terms: + stage: post_merge + backend: pytorch + tests: + # deepseek-v4-pro-fp4 8k1k con4301 (max throughput) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4.yml deleted file mode 100644 index 22d306e421d8..000000000000 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: 0.0.1 -l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4: -- condition: - ranges: - # 1 ctx worker with each 1 node and 2 GPUs - # 1 gen worker with each 1 node and 4 GPUs - system_gpu_count: - gte: 8 - lte: 8 - wildcards: - gpu: - - '*gb300*' - terms: - stage: post_merge - backend: pytorch - tests: - # glm-5-fp4 - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml index 4b46cb313cf3..304e8b56a98f 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml @@ -1,5 +1,22 @@ version: 0.0.1 l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8: +- condition: + ranges: + # 1 ctx worker with each 1 node and 2 GPUs + # 1 gen worker with each 2 node and 8 GPUs + system_gpu_count: + gte: 12 + lte: 12 + wildcards: + gpu: + - '*gb300*' + terms: + stage: pre_merge + backend: pytorch + tests: + # glm-5-fp4 + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) + - condition: ranges: # 1 ctx worker with each 1 node and 2 GPUs @@ -15,8 +32,5 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8: backend: pytorch tests: # glm-5-fp4 - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml index 1efd9eec9908..9a8e9b4ee1bd 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml @@ -15,5 +15,4 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32: backend: pytorch tests: # glm-5-fp4 - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml deleted file mode 100644 index a451b0fcf3d2..000000000000 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 0.0.1 -l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4: -- condition: - ranges: - # 1 ctx worker with each 1 node and 4 GPUs - # 1 gen worker with each 1 node and 4 GPUs - system_gpu_count: - gte: 8 - lte: 8 - wildcards: - gpu: - - '*gb300*' - terms: - stage: post_merge - backend: pytorch - tests: - # deepseek-r1-fp4 - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - # kimi-k25-thinking-fp4 - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml index 5e93797854ae..14aa2d0a4b24 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml @@ -1,5 +1,22 @@ version: 0.0.1 l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8: +- condition: + ranges: + # 1 ctx worker with each 1 node and 4 GPUs + # 1 gen worker with each 2 node and 8 GPUs + system_gpu_count: + gte: 12 + lte: 12 + wildcards: + gpu: + - '*gb300*' + terms: + stage: pre_merge + backend: pytorch + tests: + # kimi-k25-thinking-fp4 8k1k + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) + - condition: ranges: # 1 ctx worker with each 1 node and 4 GPUs @@ -16,9 +33,4 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8: tests: # deepseek-r1-fp4 128k8k - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) - # kimi-k25-thinking-fp4 1k1k - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - # kimi-k25-thinking-fp4 8k1k - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml index ee2dba9895fb..3c6b989f1cbc 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml @@ -15,5 +15,4 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32: backend: pytorch tests: # kimi-k25-thinking-fp4 - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml new file mode 100644 index 000000000000..07ad83e0e1e4 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml @@ -0,0 +1,19 @@ +version: 0.0.1 +l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8: +- condition: + ranges: + # 1 ctx worker with 1 node and 4 GPUs + # 4 gen workers each with 2 nodes and 8 GPUs + system_gpu_count: + gte: 36 + lte: 36 + wildcards: + gpu: + - '*gb300*' + terms: + stage: post_merge + backend: pytorch + tests: + # deepseek-v4-pro-fp4 8k1k con8 (single-user latency) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node1_gpu4_gen1_node8_gpu32.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node1_gpu4_gen1_node8_gpu32.yml new file mode 100644 index 000000000000..a6cdb0d3430e --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node1_gpu4_gen1_node8_gpu32.yml @@ -0,0 +1,19 @@ +version: 0.0.1 +l0_gb300_multi_nodes_perf_sanity_ctx3_node1_gpu4_gen1_node8_gpu32: +- condition: + ranges: + # 3 ctx workers each with 1 node and 4 GPUs + # 1 gen worker with 8 nodes and 32 GPUs + system_gpu_count: + gte: 44 + lte: 44 + wildcards: + gpu: + - '*gb300*' + terms: + stage: post_merge + backend: pytorch + tests: + # deepseek-v4-pro-fp4 8k1k con180 + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml new file mode 100644 index 000000000000..76f981ff6acb --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml @@ -0,0 +1,19 @@ +version: 0.0.1 +l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16: +- condition: + ranges: + # 6 ctx workers each with 1 node and 4 GPUs + # 1 gen worker with 4 nodes and 16 GPUs + system_gpu_count: + gte: 40 + lte: 40 + wildcards: + gpu: + - '*gb300*' + terms: + stage: post_merge + backend: pytorch + tests: + # deepseek-v4-pro-fp4 8k1k con666 + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_node2_gpu8.yml index 61a822d5b29c..e0c7b5ea4199 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_node2_gpu8.yml @@ -15,6 +15,5 @@ l0_gb300_multi_nodes_perf_sanity_node2_gpu8: tests: # aggr config # deepseek-r1-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_dep8_mtp1_1k1k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_dep8_mtp1_8k1k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_tep8_mtp3] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gh200.yml b/tests/integration/test_lists/test-db/l0_gh200.yml index f62424367d51..6cf8033c2d6b 100644 --- a/tests/integration/test_lists/test-db/l0_gh200.yml +++ b/tests/integration/test_lists/test-db/l0_gh200.yml @@ -12,7 +12,7 @@ l0_gh200: cpu: aarch64 terms: stage: post_merge - backend: tensorrt + backend: pytorch tests: - unittest/bindings - unittest/llmapi/test_llm_quant.py diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index 67fb75c29aaf..b6c1d3fb0783 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -50,6 +50,8 @@ l0_h100: - unittest/_torch/sampler -k "not test_speculative_d2h_parity_real_predictor" - unittest/_torch/speculative/test_eagle3.py - unittest/_torch/speculative/test_rejection_buffers_guard.py + - unittest/_torch/speculative/test_capture_override_leak.py + - unittest/_torch/speculative/test_sa_hybrid_state_promotion.py - unittest/_torch/speculative/hw_agnostic - unittest/_torch/thop/parallel - unittest/_torch/thop/parallel_hw_agnostic @@ -59,6 +61,7 @@ l0_h100: - unittest/_torch/modeling -k "modeling_mixtral" - unittest/_torch/modeling -k "modeling_gemma3" - unittest/_torch/modeling -k "modeling_gpt_oss" + - unittest/_torch/modeling -k "modeling_whisper" # CPU-only log-mel parity - unittest/_torch/modeling/test_modeling_nemotron_h.py::test_nemotron_h_sanity - unittest/_torch/modeling/test_multimodal_encoder_graph.py # Qwen3.5-MoE-VL is hybrid (Mamba SSM + attention); FlashInfer's @@ -82,10 +85,15 @@ l0_h100: - unittest/disaggregated/region/test_region.py - unittest/disaggregated/test_disaggregated_params.py - unittest/disaggregated/test_perf_logger.py + - unittest/disaggregated/test_sync_transfer_timing.py - unittest/disaggregated/test_rank_info.py - unittest/disaggregated/test_request_id.py - unittest/disaggregated/test_kv_transfer.py - unittest/disaggregated/test_kv_transfer_mp.py + - unittest/disaggregated/test_transceiver_bounded_polling.py + - unittest/disaggregated/test_pool_matching.py + - unittest/disaggregated/test_deepseek_v4_kv_transfer.py + - unittest/disaggregated/test_minimax_m3_kv_transfer.py # Split the large cache transceiver parametrization into readable chunks. # Main transfer matrix. - unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver -k "v1 and no_window" @@ -100,8 +108,11 @@ l0_h100: # Boundary request lengths. - unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver_boundary_lengths -k "v1" - unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver_boundary_lengths -k "v2" + # DSA indexer-K side cache (V1, REPLICATED). + - unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver_v1_dsa_indexer - unittest/disaggregated/test_cache_transceiver_harness_report.py - unittest/disaggregated/test_cache_transceiver_harness.py + - unittest/disaggregated/test_cache_transceiver_precheck_e2e.py - unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mha-ctx_fp16_gen_fp16] - unittest/others/test_kv_cache_transceiver.py::test_kv_cache_transceiver_single_process[PYTHON-mla-ctx_fp16_gen_fp16] - unittest/llmapi/test_llm_telemetry.py @@ -176,6 +187,7 @@ l0_h100: - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0] + - disaggregated/test_disaggregated.py::test_disaggregated_tinyllama_multi_orchestrator[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-True-DeepSeek-V3-Lite-fp8/fp8] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[True-False-DeepSeek-V3-Lite-fp8/fp8] @@ -192,39 +204,16 @@ l0_h100: - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logits[True-TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[False-TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_logprobs[True-TinyLlama-1.1B-Chat-v1.0] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-off-greedy-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-bart-large-cnn] + # Encoder-decoder Hopper smoke: CUDA-graph beam/greedy, kv-v2, overlap. + # The primary pre-merge set runs on L40S (l0_l40s.yml); the full + # dtype/model-size matrix runs post-merge below. - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-off-greedy-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-greedy-batch2-bart-large-cnn] - - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v2-decoder-cuda-graph-on-greedy-batch2-bart-large-cnn] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-t5-small0] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-t5-base] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-t5-large] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-flan-t5-base] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-flan-t5-large] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-flan-t5-xl] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-flan-t5-xxl] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-beam2-t5-small1] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-off-beam2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v1-cuda-graph-off-beam2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-off-beam2-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v1-cuda-graph-off-beam2-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-off-greedy-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v2-cuda-graph-off-greedy-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v2-cuda-graph-off-greedy-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-off-greedy-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v2-cuda-graph-off-greedy-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v2-cuda-graph-off-greedy-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-off-greedy-byt5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-beam2-batch2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-beam2-batch2-flan-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-greedy-batch2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v2-decoder-cuda-graph-on-greedy-batch2-t5-small] - - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_context_generation_batch + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-bart-large-cnn] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-overlap-t5-small] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_transcribe_end_to_end + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[bf16-kv-v2-decoder-graphs-on-greedy] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_beam_search[bf16-kv-v1-decoder-graphs-on-beam2] - llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[cudagraph] TIMEOUT (90) - test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B] - test_e2e.py::test_trtllm_bench_iteration_log[PyTorch-non-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B] @@ -298,8 +287,6 @@ l0_h100: - cpp/test_unit_tests.py::test_unit_tests[kernels-90] - cpp/test_unit_tests.py::test_unit_tests[layers-90] - cpp/test_unit_tests.py::test_unit_tests[thop-90] - - cpp/test_unit_tests.py::test_unit_tests[utils-90] - - cpp/test_e2e.py::test_model[-encoder-90] - condition: ranges: system_gpu_count: @@ -311,9 +298,8 @@ l0_h100: linux_distribution_name: ubuntu* terms: stage: pre_merge - backend: tensorrt + backend: pytorch tests: - # ------------- TRT tests --------------- - unittest/llmapi/test_llm_quant.py # 5.5 mins on H100 - examples/visual_gen/test_visual_gen.py::test_visual_gen_quickstart - examples/visual_gen/test_visual_gen.py::test_visual_gen_api_walkthrough @@ -332,6 +318,52 @@ l0_h100: orchestrator: mpi tests: # ------------- PyTorch tests --------------- + # Encoder-decoder full matrix (dtype, model-size, kv-v2, overlap variants); + # the representative pre-merge subset runs on L40S (l0_l40s.yml). + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-on-greedy-bart-large-cnn] + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-overlap-bart-large-cnn] + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-overlap-bart-large-cnn] + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v2-decoder-cuda-graph-on-greedy-batch2-bart-large-cnn] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-t5-base] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-t5-large] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-flan-t5-base] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-flan-t5-large] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-flan-t5-xl] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-flan-t5-xxl] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-on-beam2-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v1-cuda-graph-on-beam2-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v1-cuda-graph-on-beam2-flan-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v1-cuda-graph-on-beam2-flan-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v2-cuda-graph-on-greedy-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v2-cuda-graph-on-greedy-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-flan-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp16-kv-v2-cuda-graph-on-greedy-flan-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[fp32-kv-v2-cuda-graph-on-greedy-flan-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-off-greedy-byt5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-overlap-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-overlap-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-beam2-batch2-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-beam2-batch2-flan-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v2-decoder-cuda-graph-on-greedy-batch2-t5-small] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_beam_search[fp32-kv-v1-graphs-off-beam2] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[fp32-kv-v2-graphs-off-greedy] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[fp32-kv-v1-graphs-requested-greedy] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[bf16-kv-v1-decoder-graphs-on-greedy] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[fp16-kv-v1-graphs-off-greedy] + - examples/test_gpt.py::test_gpt_oss_20b_lora_torch[gpt-oss-20b-lora-adapter_NIM_r8-gpt-oss-20b] + - unittest/bindings # 8 mins on H100 + - unittest/kv_cache_manager_v2_tests # 4 min + # ------------- KV Cache Iteration Stats --------------- + - unittest/executor/test_stats_serializer.py + - unittest/metrics/test_collector.py + - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_cold_start + - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_partial_block_reuse + - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_full_block_reuse + - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_shared_prefix + - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_batch_generation + - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_long_context + - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_rapid_fire + - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_field_completeness - llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[eager] TIMEOUT (90) - llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[eager] TIMEOUT (90) - llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[cudagraph] TIMEOUT (90) @@ -417,58 +449,6 @@ l0_h100: - unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k "CUTLASS and None" # Documentation URL validation (CPU-only, no GPU needed) - test_doc.py::test_url_validity -- condition: - ranges: - system_gpu_count: - gte: 1 - lte: 1 - wildcards: - gpu: - - '*h100*' - linux_distribution_name: ubuntu* - terms: - stage: post_merge - backend: cpp - tests: - # ------------- CPP tests --------------- - - cpp/test_e2e.py::test_model[-bart-90] - - cpp/test_e2e.py::test_benchmarks[bart-90] -- condition: - ranges: - system_gpu_count: - gte: 1 - lte: 1 - wildcards: - gpu: - - '*h100*' - linux_distribution_name: ubuntu* - terms: - stage: post_merge - backend: tensorrt - tests: - # ------------- TRT tests --------------- - - examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] # 18mins - - examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime] # 8 mins - - examples/test_granite.py::test_granite_bf16_lora[granite-3.0-1b-a400m-instruct] - - examples/test_gpt.py::test_llm_minitron_fp8_with_pseudo_loras[4b] TIMEOUT (90) - - unittest/bindings # 8 mins on H100 - - examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1] - - examples/test_multimodal.py::test_llm_fp8_multimodal_general[fp8-fp8-scienceqa-Llama-3.2-11B-Vision-Instruct-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False] - - examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_py_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1] - - examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1] - - examples/test_gpt.py::test_gpt_oss_20b_lora_torch[gpt-oss-20b-lora-adapter_NIM_r8-gpt-oss-20b] - - unittest/kv_cache_manager_v2_tests # 4 min - # ------------- KV Cache Iteration Stats --------------- - - unittest/executor/test_stats_serializer.py - - unittest/metrics/test_collector.py - - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_cold_start - - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_partial_block_reuse - - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_full_block_reuse - - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_shared_prefix - - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_batch_generation - - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_long_context - - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_rapid_fire - - kv_cache/test_kv_cache_iteration_stats.py::TestKvCacheIterationStats::test_field_completeness - condition: ranges: system_gpu_count: diff --git a/tests/integration/test_lists/test-db/l0_l40s.yml b/tests/integration/test_lists/test-db/l0_l40s.yml index 8184e5d94eba..18fa6b86be18 100644 --- a/tests/integration/test_lists/test-db/l0_l40s.yml +++ b/tests/integration/test_lists/test-db/l0_l40s.yml @@ -36,6 +36,27 @@ l0_l40s: # AutoDeploy: Nemotron-Nano-V3 on Ada uses flashinfer; trtllm has no # (E4M3 input, BF16 output, paged_kv, head_dim=128, sm_89) FMHA cubin. - accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-1-attn_dp_off-flashinfer] + # ------------- Encoder-decoder tests --------------- + # Representative pre-merge set: eager smoke, CUDA-graph greedy/beam, kv-v2, + # overlap scheduler, mBART, mixed batches. + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-bart-large-cnn] + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-bart-large-cnn] + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-bart-large-cnn] + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-bart-large-cnn] + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-overlap-bart-large-cnn] + - llmapi/test_llm_api_pytorch_bart.py::test_mbart_pytorch_generate_encoder_decoder_end_to_end + - llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-greedy-batch2-bart-large-cnn] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-off-greedy-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-beam2-flan-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v2-cuda-graph-on-greedy-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-overlap-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_encoder_lengths_batch[bf16-kv-v1-decoder-cuda-graph-on-greedy-batch2-t5-small] + - llmapi/test_llm_api_pytorch_t5.py::test_t5_pytorch_generate_encoder_decoder_mixed_context_generation_batch + # Whisper (encoder-decoder) — customer-side deployment targets L40S/H200 + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_transcribe_end_to_end + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[bf16-kv-v2-decoder-graphs-on-greedy] - condition: ranges: system_gpu_count: @@ -47,20 +68,15 @@ l0_l40s: linux_distribution_name: ubuntu* terms: stage: post_merge - backend: tensorrt + backend: pytorch tests: - # ------------- TRT tests --------------- - - examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] - - llmapi/test_llm_examples.py::test_llmapi_quickstart - - examples/visual_gen/test_visual_gen.py::test_visual_gen_quickstart - - examples/visual_gen/test_visual_gen.py::test_visual_gen_api_walkthrough - - llmapi/test_llm_examples.py::test_llmapi_example_inference - - llmapi/test_llm_examples.py::test_llmapi_example_inference_async - - llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming - - llmapi/test_llm_examples.py::test_llmapi_example_multilora - - llmapi/test_llm_examples.py::test_llmapi_example_guided_decoding - - llmapi/test_llm_examples.py::test_llmapi_example_logits_processor - - examples/test_llm_api_with_mpi.py::test_llm_api_single_gpu_with_mpirun[TinyLlama-1.1B-Chat-v1.0] + # Whisper (encoder-decoder) — customer-side deployment targets L40S/H200 + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_beam_search[fp32-kv-v1-graphs-off-beam2] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_beam_search[bf16-kv-v1-decoder-graphs-on-beam2] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[fp32-kv-v2-graphs-off-greedy] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[fp32-kv-v1-graphs-requested-greedy] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[bf16-kv-v1-decoder-graphs-on-greedy] + - llmapi/test_llm_api_pytorch_whisper.py::test_whisper_pytorch_feature_combinations[fp16-kv-v1-graphs-off-greedy] - condition: ranges: system_gpu_count: @@ -72,11 +88,18 @@ l0_l40s: linux_distribution_name: ubuntu* terms: stage: post_merge - backend: tensorrt + backend: pytorch tests: - - examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] # 18mins - - examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime] # 8 mins - - examples/test_granite.py::test_granite_bf16_lora[granite-3.0-1b-a400m-instruct] TIMEOUT (90) + - llmapi/test_llm_examples.py::test_llmapi_quickstart + - examples/visual_gen/test_visual_gen.py::test_visual_gen_quickstart + - examples/visual_gen/test_visual_gen.py::test_visual_gen_api_walkthrough + - llmapi/test_llm_examples.py::test_llmapi_example_inference + - llmapi/test_llm_examples.py::test_llmapi_example_inference_async + - llmapi/test_llm_examples.py::test_llmapi_example_inference_async_streaming + - llmapi/test_llm_examples.py::test_llmapi_example_multilora + - llmapi/test_llm_examples.py::test_llmapi_example_guided_decoding + - llmapi/test_llm_examples.py::test_llmapi_example_logits_processor + - examples/test_llm_api_with_mpi.py::test_llm_api_single_gpu_with_mpirun[TinyLlama-1.1B-Chat-v1.0] - condition: ranges: system_gpu_count: diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 63a32137e69e..5417b0459701 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -1,9 +1,4 @@ -accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] SKIP (https://nvbugs/6120535) -accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp1cp4] SKIP (https://nvbugs/6396413) -accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] SKIP (https://nvbugs/6396415) -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp2tp1cp2] SKIP (https://nvbugs/6427411) -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0] SKIP (https://nvbugs/6426865) -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6075533) +accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[noadp-mtp0] SKIP (https://nvbugs/6481375) accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2] SKIP (https://nvbugs/6427411) accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4] SKIP (https://nvbugs/6428069) accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=2] SKIP (https://nvbugs/6427411) @@ -17,30 +12,22 @@ accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symm accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[GSM8K-tp2pp2] SKIP (https://nvbugs/6428069) accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp1pp2] SKIP (https://nvbugs/6427411) accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_tp_pp_symmetric[MMLU-tp2pp2] SKIP (https://nvbugs/6428069) -accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_auto_dtype[mtp_nextn=0-block_reuse=False-use_py_transceiver=False] SKIP (https://nvbugs/6465993) -accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_auto_dtype[mtp_nextn=3-block_reuse=True-use_py_transceiver=False] SKIP (https://nvbugs/6465993) -accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_ctx_dp2_gen_tp4 SKIP (https://nvbugs/6465993) -accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=False] SKIP (https://nvbugs/6427411) -accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=True] SKIP (https://nvbugs/6402054) +accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=False] SKIP (https://nvbugs/6535790) accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_model[ctxpp2gentp2] SKIP (https://nvbugs/5748664) -accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4] SKIP (https://nvbugs/6336747) -accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[qwen3vl_2b_instruct] SKIP (https://nvbugs/6422294) +accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8] SKIP (https://nvbugs/6478692) +accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-4-attn_dp_off-trtllm] SKIP (https://nvbugs/6367792) accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[nvfp4_ws8_80gb-trtllm] SKIP (https://nvbugs/6450341) +accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6507114) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput_mtp] SKIP (https://nvbugs/6428101) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput_mtp_trtllm] SKIP (https://nvbugs/6426868) +accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_pp4_mtp] SKIP (https://nvbugs/6481323) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload] SKIP (https://nvbugs/6384136) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp1] SKIP (https://nvbugs/6384357) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp3_no_adp] SKIP (https://nvbugs/6384357) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[baseline] SKIP (https://nvbugs/6384136) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_chunked_prefill[latency] SKIP (https://nvbugs/6276981) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_piecewise_cuda_graph[mtp3_fp8kv_chunked] SKIP (https://nvbugs/5989920) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6305365) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6305365) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6305404) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6412102) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6388129) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6426847) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6445456) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6517844) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6402058) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6278337) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6428057) @@ -50,45 +37,31 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mt accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6388153) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6428094) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6428096) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6445456) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp2pp2-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6198774) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6445456) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6198785) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6198785) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6445456) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6198774) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2-moe_backend=WIDEEP] SKIP (https://nvbugs/6313993) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True] SKIP (https://nvbugs/6388139) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_python_scheduler[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-enable_chunked_prefill=True] SKIP (https://nvbugs/6507095) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6388153) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-sampler_async_worker=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp2pp2-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6445456) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6445456) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6445456) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6445456) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=True] SKIP (https://nvbugs/6445456) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[tp4-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6463011) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6388363) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6428087) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=0-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6445456) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6445456) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5945081) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6384625) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6428063) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6384625) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6384625) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6445472) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6445456) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6272673) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6422432) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6245394) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp2pp2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6384625) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-tp2pp2-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6384625) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Flash::test_auto_dtype SKIP (https://nvbugs/6450333) +accuracy/test_llm_api_pytorch.py::TestGLM52::test_nvfp4[tp_size=8-ep_size=8] SKIP (https://nvbugs/6507108) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[one_model] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto] SKIP (https://nvbugs/6026676) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-cutlass-auto] SKIP (https://nvbugs/6388142) @@ -102,8 +75,7 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutl accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype SKIP (https://nvbugs/6209806) accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4[4gpus] SKIP (https://nvbugs/6368562) -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=FLASHINFER-torch_compile=True] SKIP (https://nvbugs/6305318) -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16[attn_backend=TRTLLM-torch_compile=True] SKIP (https://nvbugs/6305318) +accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6490043) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp2pp2-attn_backend=FLASHINFER-torch_compile=True] SKIP (https://nvbugs/6422337) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[tp4-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/5616182) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=False-attn_backend=TRTLLM-torch_compile=False] SKIP (https://nvbugs/6437412) @@ -115,38 +87,24 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torc accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=True] SKIP (https://nvbugs/6211441) accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132) -accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6248827) -accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] SKIP (https://nvbugs/6157892) accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP4_PP2] SKIP (https://nvbugs/6427411) +accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=CUTLASS] SKIP (https://nvbugs/6535767) accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[ADP2_PP2] SKIP (https://nvbugs/6427411) -accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tep4] SKIP (https://nvbugs/6255417) +accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[dep4] SKIP (https://nvbugs/6535767) +accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[tp1_block_reuse-cutlass] SKIP (https://nvbugs/6535767) accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency] SKIP (https://nvbugs/6177390) accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[throughput_latency] SKIP (https://nvbugs/6177390) accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm] SKIP (https://nvbugs/6402009) -accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-TRTLLM] SKIP (https://nvbugs/6428008) -accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_mtp SKIP (https://nvbugs/6206179) -accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=True] SKIP (https://nvbugs/6210714) -accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_trtllm] SKIP (https://nvbugs/6426850) +accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_trtllm-torch_compile=False] SKIP (https://nvbugs/6507110) +accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_cutedsl] SKIP (https://nvbugs/6535767) +accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[adp4_trtllm] SKIP (https://nvbugs/6535779) accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_cutedsl] SKIP (https://nvbugs/6255417) +accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4[tep4_trtllm] SKIP (https://nvbugs/6535779) accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM] SKIP (https://nvbugs/6418830) +accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_mtp3_gdn_replay_tep4 SKIP (https://nvbugs/6535779) accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_bf16[latency] SKIP (https://nvbugs/6412098) -accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=3] SKIP (https://nvbugs/6367805) -accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=3] SKIP (https://nvbugs/6367805) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6248827) -accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[fp8_mmmu_encoder_cuda_graph] SKIP (https://nvbugs/6336747) -accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[nvfp4] SKIP (https://nvbugs/6336747) -accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=3] SKIP (https://nvbugs/6367805) accuracy/test_llm_api_pytorch_ray.py::TestLlama3_1_8BInstruct::test_pp2_ray SKIP (https://nvbugs/6427411) -cpp/test_e2e.py::test_benchmarks[bart-90] SKIP (https://nvbugs/5550689) -cpp/test_e2e.py::test_benchmarks[gpt-80] SKIP (https://nvbugs/5550689) -cpp/test_e2e.py::test_model[-bart-90] SKIP (https://nvbugs/6162804) -cpp/test_e2e.py::test_model[-encoder-90] SKIP (waive Encoder-only test because it doesn't take batched input) -cpp/test_e2e.py::test_model[-gpt-80] SKIP (https://nvbugs/5983283) -cpp/test_e2e.py::test_model[-mamba-86] SKIP (https://nvbugs/5781665) -cpp/test_e2e.py::test_model[-redrafter-86] SKIP (https://nvbugs/5761642) -cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-mpi_kvcache-90] SKIP (https://nvbugs/5755941) -cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-nixl_kvcache-90] SKIP (https://nvbugs/6093820) -cpp/test_multi_gpu.py::TestDisagg::test_symmetric_executor[gpt-2proc-ucx_kvcache-90] SKIP (https://nvbugs/6093820) cpp/test_multi_gpu.py::test_cache_transceiver[8proc-mooncake_kvcache-90] SKIP (https://nvbugs/5838199) disaggregated/test_disaggregated.py::test_disaggregated_cancel_large_context_requests[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6105768) disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6427411) @@ -170,161 +128,139 @@ disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1 disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6162322) disaggregated/test_disaggregated.py::test_disaggregated_genbs1[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) -disaggregated/test_disaggregated.py::test_disaggregated_mixed_stress_test[req60-conc64-qwen3_32b_fp8_mixed_stress] SKIP (https://nvbugs/6432832) disaggregated/test_workers.py::test_workers_conversation_router[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) disaggregated/test_workers.py::test_workers_kv_cache_aware_router_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) disaggregated/test_workers.py::test_workers_kv_cache_aware_router_eviction[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[trtllm-torch-cudagraph] SKIP (https://nvbugs/6426841) -examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-disable_attention_plugin-disable_context_fmha-tp:2-pp:1-float16-RobertaForSequenceClassification-bert/twitter-roberta-base-emotion] SKIP (https://nvbugs/5234058) -examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:2-pp:1-float16-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] SKIP (https://nvbugs/5234058) -examples/test_bert.py::test_llm_bert_general[compare_hf-enable_remove_input_padding-use_attention_plugin-enable_context_fmha-tp:2-pp:1-float16-RobertaForQuestionAnswering-bert/roberta-base-squad2] SKIP (https://nvbugs/5234058) -examples/test_gpt.py::test_llm_minitron_fp8_with_pseudo_loras[4b] SKIP (https://nvbugs/5606233) -examples/test_granite.py::test_granite_bf16_lora[granite-3.0-1b-a400m-instruct] SKIP (https://nvbugs/5431132) -examples/test_granite.py::test_llm_granite[granite-3.0-1b-a400m-instruct-bfloat16] SKIP (https://nvbugs/5608979) -examples/test_granite.py::test_llm_granite[granite-3.0-2b-instruct-bfloat16] SKIP (https://nvbugs/5608979) -examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_cpp_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1] SKIP (https://nvbugs/5802248) -examples/test_medusa.py::test_llm_medusa_with_qaunt_base_model_1gpu[fp8-use_py_session-medusa-vicuna-7b-v1.3-4-heads-float16-bs1] SKIP (https://nvbugs/5333849) -examples/test_multimodal.py::test_llm_fp8_multimodal_general[fp8-fp8-scienceqa-Llama-3.2-11B-Vision-Instruct-pp:1-tp:1-bfloat16-bs:1-cpp_e2e:False] SKIP (https://nvbugs/5222697) -examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1] SKIP (https://nvbugs/5333818) -examples/test_multimodal.py::test_llm_multimodal_general[Llama-3.2-11B-Vision-pp:1-tp:2-bfloat16-bs:1-cpp_e2e:False-nb:1] SKIP (https://nvbugs/5333818) -examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (https://nvbugs/4961624) -examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) -examples/test_nemotron_nas.py::test_nemotron_nas_summary_2gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) -examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (https://nvbugs/5447530) +examples/test_gpt.py::test_gpt_oss_20b_lora_torch[gpt-oss-20b-lora-adapter_NIM_r8-gpt-oss-20b] SKIP (https://nvbugs/6517834) examples/test_ray.py::test_llm_inference_distributed_ray[pp2] SKIP (https://nvbugs/6427411) examples/test_ray.py::test_llm_inference_distributed_ray[tp2pp2] SKIP (https://nvbugs/6427411) examples/test_ray.py::test_ray_disaggregated_serving[tp2] SKIP (https://nvbugs/5612502) -examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-disable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime] SKIP (https://nvbugs/5244570) examples/visual_gen/test_visual_gen.py::test_cosmos3_nano_t2i_lpips_against_golden SKIP (https://nvbugs/6418815) examples/visual_gen/test_visual_gen.py::test_cosmos3_nano_t2v_lpips_against_golden SKIP (https://nvbugs/6437341) examples/visual_gen/test_visual_gen.py::test_ltx2_cuda_graph_trtllm_backend SKIP (https://nvbugs/6463822) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[attn2d_2x2] SKIP (https://nvbugs/6272644) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2] SKIP (https://nvbugs/6272644) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2_attn2d_2x1] SKIP (https://nvbugs/6272644) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[ulysses4] SKIP (https://nvbugs/6272644) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[cfg2_tp2] SKIP (https://nvbugs/6329227) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[tp2] SKIP (https://nvbugs/6329227) -examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_tp[tp2_ulysses2] SKIP (https://nvbugs/6329227) -full:A100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) -full:A100/disaggregated/test_workers.py::test_workers_conditional_disaggregation_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6329052) -full:A100X/llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_mtp SKIP (https://nvbugs/6287561) -full:A100X/unittest/llmapi/test_llm_pytorch.py -m "part0" SKIP (https://nvbugs/6416249) -full:B200/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1tp1cp4] SKIP (https://nvbugs/6410881) -full:B200/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp1tp1cp4] SKIP (https://nvbugs/6410881) +examples/visual_gen/test_visual_gen.py::test_wan22_t2v_lpips_against_golden SKIP (https://nvbugs/6535765) +examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2] SKIP (https://nvbugs/6535765) +examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[ulysses4] SKIP (https://nvbugs/6535765) +full:A100/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_mtp SKIP (https://nvbugs/6275856) +full:B200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt SKIP (https://nvbugs/6525009) +full:B200/accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8] SKIP (https://nvbugs/6327718) full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=False] SKIP (https://nvbugs/6422343) full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6422343) +full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[tp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6525007) +full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6526186) +full:B200/accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_fp8 SKIP (https://nvbugs/6525011) +full:B200/accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True] SKIP (https://nvbugs/6525010) full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_size=8] SKIP (https://nvbugs/6384747) -full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[tp_size=4-ep_size=4] SKIP (https://nvbugs/6424188) -full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[tp_size=4-ep_size=4] SKIP (https://nvbugs/6424188) -full:B200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318) -full:B200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) -full:B200/disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6344107) -full:B200/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-gpt_oss_120b_eagle_trtllm_stress] SKIP (https://nvbugs/6413724) -full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_tensor] SKIP (https://nvbugs/5161074) -full:B200/perf/test_perf.py::test_perf[quant:int8_sq_per_token_channel] SKIP (https://nvbugs/5161074) -full:B200/perf/test_perf.py::test_perf[quant:w4a8_awq] SKIP (https://nvbugs/5161074) +full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6424188) +full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] SKIP (https://nvbugs/6424188) +full:B200/accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[ADP4] SKIP (https://nvbugs/6525008) +full:B200/accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[ADP4_MTP] SKIP (https://nvbugs/6525008) +full:B200/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-deepseek_r1_v2_fp4_stress] SKIP (https://nvbugs/6472256) +full:B200/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-qwen3_32b_fp8_stress] SKIP (https://nvbugs/6472256) +full:B200/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[cudagraph] SKIP (https://nvbugs/6475623) +full:B200/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[eager] SKIP (https://nvbugs/6475621) full:B200/test_e2e.py::test_multi_nodes_eval[Qwen3/Qwen3-235B-A22B-tp16-mmlu] SKIP (https://nvbugs/6424188) full:B200/test_e2e.py::test_multi_nodes_eval[Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf-tp16-mmlu] SKIP (https://nvbugs/6424188) full:B200/test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] SKIP (https://nvbugs/6414760) -full:B300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1tp1cp4] SKIP (https://nvbugs/6410881) -full:B300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v1-cudagraph:with_padding-pp1tp2cp2] SKIP (https://nvbugs/6410881) -full:B300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp1cp4] SKIP (https://nvbugs/6410881) full:B300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp2cp2] SKIP (https://nvbugs/6410881) -full:B300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp1tp1cp4] SKIP (https://nvbugs/6410881) -full:B300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[nccl-cudagraph:with_padding-pp1tp2cp2] SKIP (https://nvbugs/6410881) +full:B300/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-4-attn_dp_off-trtllm] SKIP (https://nvbugs/6483370) +full:B300/accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[bf16_ws4_180gb-flashinfer] SKIP (https://nvbugs/6478645) +full:B300/accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[bf16_ws4_180gb-trtllm] SKIP (https://nvbugs/6478645) +full:B300/accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[fp8_ws4_80gb-trtllm] SKIP (https://nvbugs/6478645) +full:B300/accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[nvfp4_ws4_80gb-trtllm] SKIP (https://nvbugs/6478645) full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[baseline_pp4_mtp1] SKIP (https://nvbugs/6423845) full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[latency] SKIP (https://nvbugs/6423866) +full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_chunked_prefill[latency] SKIP (https://nvbugs/6483369) full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus_chunked_prefill[latency_qsplit] SKIP (https://nvbugs/6423866) full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6422343) +full:B300/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True] SKIP (https://nvbugs/6475346) full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_size=8] SKIP (https://nvbugs/6445375) -full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[tp_size=4-ep_size=4] SKIP (https://nvbugs/6424188) -full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph[tp_size=8-ep_size=8] SKIP (https://nvbugs/6442594) -full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[tp_size=4-ep_size=4] SKIP (https://nvbugs/6445375) -full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318) -full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) -full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_5_VL_7B::test_auto_dtype SKIP (https://nvbugs/6316983) +full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6424188) +full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] SKIP (https://nvbugs/6445375) +full:B300/accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_off-trtllm] SKIP (https://nvbugs/6474894) +full:B300/disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cache_aware] SKIP (https://nvbugs/6474892) full:B300/disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6322073) full:B300/test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] SKIP (https://nvbugs/6414760) -full:DGX_B200/unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py::test_allreduce_strategies[MIN_LATENCY] SKIP (https://nvbugs/6403920) -full:DGX_B200/unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py::test_allreduce_strategies[ONESHOT] SKIP (https://nvbugs/6403920) -full:DGX_B200/unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py::test_allreduce_strategies[TWOSHOT] SKIP (https://nvbugs/6403920) -full:DGX_B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6432831) -full:GB200/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False] SKIP (https://nvbugs/6402500) -full:GB200/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False] SKIP (https://nvbugs/6402500) -full:GB200/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False] SKIP (https://nvbugs/6402500) +full:DGX_B200/accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=True] SKIP (https://nvbugs/6501837) +full:DGX_B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV4ProDSpark::test_gsm8k_dep8_megamoe_deepgemm SKIP (https://nvbugs/6506920) +full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_skip_indexer] SKIP (https://nvbugs/6476233) +full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[latency_default] SKIP (https://nvbugs/6476233) +full:GB200/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-True] SKIP (https://nvbugs/6525893) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy SKIP (https://nvbugs/6276923) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt SKIP (https://nvbugs/6276923) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap SKIP (https://nvbugs/6276923) +full:GB200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6525896) full:GB200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=False] SKIP (https://nvbugs/6400067) full:GB200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6400067) +full:GB200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm[cuda_graph=True] SKIP (https://nvbugs/6525897) +full:GB200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6479471) +full:GB200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] SKIP (https://nvbugs/6479471) +full:GB200/accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-python_mamba_cache] SKIP (https://nvbugs/6528742) +full:GB200/accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[ADP4] SKIP (https://nvbugs/6474894) +full:GB200/accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM] SKIP (https://nvbugs/6525898) +full:GB200/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_dummy_load_format SKIP (https://nvbugs/6525059) +full:GB200/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True] SKIP (https://nvbugs/5929339) full:GB200/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8_moe_dflash SKIP (https://nvbugs/6316985) full:GB200/accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_4gpus_online_eplb[moe_backend=TRTLLM] SKIP (https://nvbugs/6462928) -full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318) -full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6388212) +full:GB200/accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_fp8_prequantized SKIP (https://nvbugs/6525892) full:GB200/disaggregated/test_ad_disagg.py::test_async_eagle3_full_model_handoff SKIP (https://nvbugs/6369254) full:GB200/disaggregated/test_ad_disagg.py::test_async_generation_matches_aggregate SKIP (https://nvbugs/6415323) full:GB200/disaggregated/test_ad_disagg.py::test_async_generation_no_overlap_matches_aggregate SKIP (https://nvbugs/6402495) full:GB200/disaggregated/test_ad_disagg.py::test_async_sharded_generation_handoff SKIP (https://nvbugs/6402495) full:GB200/disaggregated/test_ad_disagg_trtllm_serve.py::test_openai_completion SKIP (https://nvbugs/6402495) +full:GB200/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[cudagraph] SKIP (https://nvbugs/6475623) +full:GB200/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[eager] SKIP (https://nvbugs/6475621) full:GB200/test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] SKIP (https://nvbugs/6414760) full:GB300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[adp-mtp2] SKIP (https://nvbugs/6295740) -full:GB300/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-False] SKIP (https://nvbugs/6316984) full:GB300/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[nvfp4-1-attn_dp_off-trtllm] SKIP (https://nvbugs/6329165) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6422343) +full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/6525057) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_4gpus[pp4-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-sampler_async_worker=False] SKIP (https://nvbugs/6388153) full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/6388153) +full:GB300/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[trtllm-fp8] SKIP (https://nvbugs/6529692) +full:GB300/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True] SKIP (https://nvbugs/6475346) full:GB300/accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6385771) full:GB300/accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6385771) -full:GB300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[tp_size=4-ep_size=4] SKIP (https://nvbugs/6422502) -full:GB300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[tp_size=4-ep_size=4] SKIP (https://nvbugs/6422502) +full:GB300/accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=False-attn_backend=FLASHINFER-torch_compile=True] SKIP (https://nvbugs/6473161) +full:GB300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6422502) +full:GB300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] SKIP (https://nvbugs/6422502) +full:GB300/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_dummy_load_format SKIP (https://nvbugs/6525059) full:GB300/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8_moe_dflash SKIP (https://nvbugs/6316985) -full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318) -full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) +full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6479708) +full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_fp8_prequantized SKIP (https://nvbugs/6526429) full:GB300/test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] SKIP (https://nvbugs/6414760) -full:GH200/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (arm is not supported) -full:GH200/examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (arm is not supported) full:H100/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] SKIP (https://nvbugs/6313072) full:H100/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True] SKIP (https://nvbugs/6313072) -full:H100/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False] SKIP (https://nvbugs/6422334) -full:H100/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False] SKIP (https://nvbugs/6422334) full:H100/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=False] SKIP (https://nvbugs/6422343) full:H100/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6422343) -full:H100/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_vswa_reuse_4gpus[v2_kv_cache-one_model] SKIP (https://nvbugs/6442074) +full:H100/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=False] SKIP (https://nvbugs/6523809) full:H100/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-CUTLASS] SKIP (https://nvbugs/6273850) -full:H100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318) -full:H100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) full:H100/accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_27B_VL::test_auto_dtype SKIP (https://nvbugs/6442075) full:H100/accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_auto_dtype SKIP (https://nvbugs/6442073) full:H100/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:H100/disaggregated/test_disaggregated.py::test_disaggregated_mixed_stress_test[req10k-conc512-qwen3_32b_fp8_mixed_stress] SKIP (https://nvbugs/6440089) -full:H100/disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6344107) full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-qwen3_32b_fp8_stress] SKIP (https://nvbugs/6312828) -full:H100/test_e2e.py::test_multi_nodes_eval[Qwen3/Qwen3-235B-A22B-tp16-mmlu] SKIP (https://nvbugs/6423926) -full:H100/test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b] SKIP (https://nvbugs/6384375) -full:H100/test_e2e.py::test_ptp_quickstart_advanced_deepseek_multi_nodes[DeepSeek-V3] SKIP (https://nvbugs/6418410) full:H100/test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] SKIP (https://nvbugs/6414760) full:H100_PCIe/unittest/llmapi/test_llm_pytorch.py::test_llama_7b_multi_lora_evict_and_reload_lora_gpu_cache SKIP (https://nvbugs/5682551) full:H20/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] SKIP (https://nvbugs/6345827) full:H20/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True] SKIP (https://nvbugs/6345827) +full:H20/accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8] SKIP (https://nvbugs/6327718) full:H20/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=False] SKIP (https://nvbugs/6422343) full:H20/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6422343) full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler] SKIP (https://nvbugs/6373530) full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[triton-auto] SKIP (https://nvbugs/6026676) -full:H20/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318) -full:H20/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) full:H20/accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_auto_dtype SKIP (https://nvbugs/6442073) -full:H20/disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6344107) +full:H20/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[cudagraph] SKIP (https://nvbugs/6475623) +full:H20/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_multi_lora_varying_ranks[eager] SKIP (https://nvbugs/6475621) full:H20/test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] SKIP (https://nvbugs/6414760) full:L40S/accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[triton-False-1] SKIP (https://nvbugs/6322045) full:L40S/accuracy/test_llm_api_autodeploy.py::TestLlama3_1_8B::test_auto_dtype[trtllm-False-1] SKIP (https://nvbugs/6322045) full:L40S/accuracy/test_llm_api_autodeploy.py::TestNemotronH::test_auto_dtype[trtllm-flashinfer_ssm-False] SKIP (https://nvbugs/6327147) full:L40S/accuracy/test_llm_api_autodeploy.py::TestNemotronH::test_auto_dtype[trtllm-triton_ssm-False] SKIP (https://nvbugs/6327147) -full:L40S/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=True] SKIP (https://nvbugs/6276841) full:L40S/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6327149) full:L40S/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6327149) -full:L40S/disaggregated/test_workers.py::test_workers_conditional_disaggregation_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6329052) -full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5948435) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/5961814) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] SKIP (https://nvbugs/5961814) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True] SKIP (https://nvbugs/5929339) @@ -345,17 +281,28 @@ full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash SKI full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8 SKIP (https://nvbugs/6273850) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] SKIP (https://nvbugs/6388153) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True] SKIP (https://nvbugs/6400067) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6473374) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] SKIP (https://nvbugs/6388153) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6313072) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6313072) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6313072) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6313072) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6473374) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6473374) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6473374) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6473374) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6388153) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] SKIP (https://nvbugs/6388153) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6388153) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=2] SKIP (https://nvbugs/6388153) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6473373) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=False-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6313072) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6313072) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=0-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6473373) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6473374) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6473374) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=False-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6473373) +full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUTLASS-mtp_nextn=2-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6473373) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_batch_waiting[batch_wait_timeout_iters=10-batch_wait_max_tokens_ratio=1.0-mtp_nextn=0-fp8kv=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-v2_kv_cache=True] SKIP (https://nvbugs/6313072) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-auto] SKIP (https://nvbugs/6273846) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-trtllm-fp8] SKIP (https://nvbugs/6273846) @@ -373,10 +320,6 @@ full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::Tes full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_bf16 SKIP (https://nvbugs/6273850) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash SKIP (https://nvbugs/6273850) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8 SKIP (https://nvbugs/6273850) -full:RTX_PRO_6000_Blackwell_Server_Edition/perf/test_perf.py::test_perf[quant:int8_sq_per_tensor] SKIP (https://nvbugs/5161074) -full:RTX_PRO_6000_Blackwell_Server_Edition/perf/test_perf.py::test_perf[quant:int8_sq_per_token_channel] SKIP (https://nvbugs/5161074) -full:RTX_PRO_6000_Blackwell_Server_Edition/perf/test_perf.py::test_perf[quant:w4a8_awq] SKIP (https://nvbugs/5161074) -full:sm100/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (megatron-core 0.8 is not supported in python 3.12) full:sm100/unittest/bindings SKIP (Disable for Blackwell) kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill SKIP (https://nvbugs/6428002) kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction_with_block_reuse SKIP (https://nvbugs/6462303) @@ -384,86 +327,48 @@ llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_ llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[cudagraph] SKIP (https://nvbugs/6463829) llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[eager] SKIP (https://nvbugs/6463829) llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_eagle3 SKIP (https://nvbugs/6075431) -perf/test_perf.py::test_perf[bart_large_cnn-plugin-float16-bs:8-input_output_len:60,20] SKIP # (https://nvidia.slack.com/archives/C059LSY62BT/p1704525727177449) -perf/test_perf.py::test_perf[flan_t5_base-bench-float16-input_output_len:128,20] SKIP -perf/test_perf.py::test_perf[flan_t5_base-plugin-float16-bs:8-input_output_len:60,20] SKIP # (https://nvidia.slack.com/archives/C059LSY62BT/p1704525727177449) -perf/test_perf.py::test_perf[flan_t5_large-bench-float16-input_output_len:128,20-gpus:2] SKIP -perf/test_perf.py::test_perf[flan_t5_large-bench-float16-input_output_len:128,20] SKIP -perf/test_perf.py::test_perf[flan_t5_large-bench-float16-maxbs:1-input_output_len:128,20-gpus:2] SKIP -perf/test_perf.py::test_perf[gpt_20b-bench-float16-maxbs:8-input_output_len:128,128-reqs:80-gpus:8] SKIP -perf/test_perf.py::test_perf[gpt_20b-bench-float16-maxbs:8-input_output_len:512,32-reqs:80-gpus:8] SKIP -perf/test_perf.py::test_perf[mamba_2.8b-bench-float16-input_output_len:128,128] SKIP -perf/test_perf.py::test_perf[mamba_2.8b-bench-float16-input_output_len:512,32] SKIP -perf/test_perf.py::test_perf[mamba_370m-bench-float16-input_output_len:128,128] SKIP -perf/test_perf.py::test_perf[mamba_370m-bench-float16-input_output_len:512,32] SKIP -perf/test_perf.py::test_perf[t5-bench-float16-input_output_len:128,20-gpus:2] SKIP -perf/test_perf.py::test_perf[t5-bench-float16-maxbs:1-input_output_len:128,20-gpus:2] SKIP -perf/test_perf.py::test_perf[t5_base-plugin-float16-bs:8-input_output_len:60,20] SKIP # (https://nvidia.slack.com/archives/C059LSY62BT/p1704525727177449) -perf/test_perf.py::test_perf[whisper_large_v3-bench-float16-input_output_len:128,20] SKIP +perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) +perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL] SKIP (https://nvbugs/6517846) +perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6517846) +perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6517846) +perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_dep8_mtp1_8k1k] SKIP (https://nvbugs/6530213) perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k8k] SKIP (https://nvbugs/6422339) perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_blackwell-r1_fp8_tp8_mtp3_8k1k] SKIP (https://nvbugs/6432948) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_1k1k] SKIP (https://nvbugs/6374910) perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_8k1k] SKIP (https://nvbugs/6374910) perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_blackwell-glm5_fp4_tep8_mtp3_8k1k] SKIP (https://nvbugs/6329155) perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_tep8_32k8k] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_tep8_1k1k] SKIP (https://nvbugs/6463814) +perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_dep8_8k1k] SKIP (https://nvbugs/6535767) +perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_dep8_mtp3_8k1k] SKIP (https://nvbugs/6535767) +perf/test_perf_sanity.py::test_e2e[aggr_upload-qwen3_5_397b_fp4_blackwell-qwen3_5_397b_fp4_tp4_8k1k] SKIP (https://nvbugs/6535767) perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1k1k] SKIP (https://nvbugs/6153575) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6426890) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6463814) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6463814) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6463814) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] SKIP (https://nvbugs/6463814) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6463814) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6463814) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6426890) +perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6490049) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6374872) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6374893) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6463814) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6324123) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] SKIP (https://nvbugs/6379406) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6463814) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418834) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418834) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] SKIP (https://nvbugs/6422339) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6402069) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6323074) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan22_i2v_a14b_blackwell-wan22_i2v_a14b_nvfp4_trtllm_cfg2_ulysses4] SKIP (https://nvbugs/6422339) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] SKIP (https://nvbugs/6490049) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6490049) +perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] SKIP (https://nvbugs/6490049) test_doc.py::test_url_validity SKIP (https://nvbugs/6215684) -test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] SKIP (https://nvbugs/6368053) -test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3_depth_1_tree[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] SKIP (https://nvbugs/6368053) test_e2e.py::test_multi_nodes_eval[DeepSeek-R1/DeepSeek-R1-0528-FP4-tp16-mmlu] SKIP (https://nvbugs/6276983) test_e2e.py::test_multi_nodes_eval[Kimi-K2-Thinking-NVFP4-tp16-mmlu] SKIP (https://nvbugs/6276983) -test_e2e.py::test_multi_nodes_eval[MiniMax-M2-tp16-mmlu] SKIP (https://nvbugs/6373532) test_e2e.py::test_multi_nodes_eval[MiniMax-M3-tp16-mmlu] SKIP (https://nvbugs/6373561) test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8] SKIP (https://nvbugs/5836830) +unittest/_torch/attention/test_attention_backends.py::test_attention_backend[deepseekv3_mla-gen-bf16-HND-p32-v1] SKIP (https://nvbugs/6507109) +unittest/_torch/misc/test_autotuner.py::test_cutedsl_nvfp4_heuristic_matches_full_sweep SKIP (https://nvbugs/6490028) unittest/_torch/misc/test_share_tensor.py::TestShareTensor::test_share_tensor_different_dtypes SKIP (https://nvbugs/6418021) -unittest/_torch/modeling -k "modeling_out_of_tree" SKIP (https://nvbugs/6426847) -unittest/_torch/modeling -k "modeling_qwen" SKIP (https://nvbugs/6433376) -unittest/_torch/modeling/test_modeling_out_of_tree.py::TestOutOfTree::test_llm_api[True] SKIP (https://nvbugs/6426847) -unittest/_torch/modeling/test_modeling_qwen3_5_vl.py::test_qwen35_dense_vl_resolves_mamba_ssm_cache_dtype SKIP (https://nvbugs/6433376) unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend[act=Relu2-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize] SKIP (https://nvbugs/5989912) unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_single_gpu -k "TRTLLM" SKIP (https://nvbugs/6464169) unittest/_torch/modules/tests_lora_modules/test_nemotron_h_lora_sanity.py::TestNemotronHLoRA::test_lora_pp2_sanity SKIP (https://nvbugs/6428124) -unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m "part0" SKIP (https://nvbugs/6372711) -unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m "part1" SKIP (https://nvbugs/6426852) +unittest/_torch/multi_gpu/test_linear.py::test_row_linear[2-balanced] SKIP (https://nvbugs/6507113) +unittest/_torch/multi_gpu/test_linear.py::test_row_linear_norm_fusion[2-hidden:16-seqlen:2] SKIP (https://nvbugs/6501404) +unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m "part0" SKIP (https://nvbugs/6490036) +unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m "part1" SKIP (https://nvbugs/6490036) unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m "part4" SKIP (https://nvbugs/6437410) -unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py::test_llm_partial_update_weights_nvfp4[auto-Qwen3/Qwen3-8B] SKIP (https://nvbugs/6372690) -unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py::test_llm_partial_update_weights_nvfp4[fp8-Qwen3/Qwen3-30B-A3B] SKIP (https://nvbugs/6372690) -unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py::test_llm_partial_update_weights_nvfp4[fp8-Qwen3/Qwen3-8B] SKIP (https://nvbugs/6372690) +unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py::test_llm_update_weights_fp8[Qwen3/Qwen3-8B-Qwen3/Qwen3-8B-FP8] SKIP (https://nvbugs/6490036) unittest/_torch/sampler/test_beam_search.py::test_beam_search_e2e[multi_process-TRTLLMSampler-cuda_graph_and_overlap-None-1-1-True-True-False] SKIP (https://nvbugs/6463819) -unittest/_torch/speculative/hw_agnostic/test_dflash.py::test_dflash_qwen3_5_4b[False] SKIP (https://nvbugs/6451032) -unittest/_torch/speculative/hw_agnostic/test_dflash.py::test_dflash_qwen3_5_4b[True] SKIP (https://nvbugs/6451032) -unittest/_torch/speculative/test_eagle3.py::test_llama_eagle3[True-TRTLLM-True-False-True-True-True-False-False-False] SKIP (https://nvbugs/6451425) +unittest/_torch/sampler/test_trtllm_sampler.py::test_trtllm_sampler_best_of_with_logprobs SKIP (https://nvbugs/6487837) +unittest/_torch/speculative/hw_agnostic/test_dflash.py::test_dflash_qwen3_5_4b[False] SKIP (https://nvbugs/6535767) +unittest/_torch/speculative/hw_agnostic/test_dflash.py::test_dflash_qwen3_5_4b[True] SKIP (https://nvbugs/6535767) +unittest/_torch/speculative/hw_agnostic/test_ngram.py::test_llama_ngram[True-True-TRTLLM] SKIP (https://nvbugs/6507102) unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py::test_fp8_rowwise_linear[dtype1] SKIP (https://nvbugs/6301807) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingDSv3-swiglu-1024-1024-1] SKIP (https://nvbugs/5908070) unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_no_autotune[use_score_as_input-RoutingRenormalize_qwen_next-swiglu-1024-1024-150] SKIP (https://nvbugs/5908070) @@ -472,18 +377,24 @@ unittest/_torch/visual_gen/multi_gpu/test_ring_attention.py::TestRingAttention:: unittest/_torch/visual_gen/multi_gpu/test_ulysses_async.py::test_capture_smoke SKIP (https://nvbugs/6385134) unittest/_torch/visual_gen/test_attention_integration.py::test_sage_attention_self_attention[fp8-2-1560] SKIP (https://nvbugs/6198760) unittest/_torch/visual_gen/test_attention_integration.py::test_sage_attention_self_attention[int8-2-1560] SKIP (https://nvbugs/6198760) -unittest/_torch/visual_gen/test_wan21_i2v_pipeline.py::TestWanI2VBatchGeneration::test_batch_prompt_shape SKIP (https://nvbugs/6418822) unittest/auto_deploy/multigpu/custom_ops SKIP (https://nvbugs/6403920) -unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py::test_allreduce_strategies[AUTO] SKIP (https://nvbugs/6403920) -unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py::test_allreduce_strategies[NCCL] SKIP (https://nvbugs/6403920) -unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py::test_allreduce_strategies[SYMM_MEM] SKIP (https://nvbugs/6403920) -unittest/bindings/test_executor_bindings.py SKIP (TRTLLM-13781: legacy TensorRT examples removed; tests to be removed in follow-up PR3) -unittest/disaggregated/test_kv_transfer.py::test_transfer_worker_v2[tp4_pp1_to_tp2_pp2] SKIP (https://nvbugs/6426834) +unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[AUTO] SKIP (https://nvbugs/6517839) +unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[MIN_LATENCY] SKIP (https://nvbugs/6517839) +unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[NCCL] SKIP (https://nvbugs/6517839) +unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[ONESHOT] SKIP (https://nvbugs/6517839) +unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[SYMM_MEM] SKIP (https://nvbugs/6517839) +unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[TWOSHOT] SKIP (https://nvbugs/6517839) +unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_wait_in_progress_on_zero_timeout SKIP (https://nvbugs/6517836) +unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_gpu_tensor SKIP (https://nvbugs/6517836) +unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_multiple_chunks SKIP (https://nvbugs/6517836) unittest/executor/test_rpc.py::TestRpcCorrectness::test_incremental_task_async SKIP (https://nvbugs/5741476) unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741) unittest/llmapi/test_additional_model_outputs.py -m "gpu2" SKIP (https://nvbugs/6428091) unittest/llmapi/test_additional_model_outputs.py::test_additional_model_outputs_integration_pp2 SKIP (https://nvbugs/6427411) +unittest/llmapi/test_llm.py::test_llm_with_customized_tokenizer SKIP (https://nvbugs/6507080) +unittest/llmapi/test_llm.py::test_tokenizer_decode_incrementally[/scratch.trt_llm_data/llm-models/falcon-7b-instruct-False-0.95-HF] SKIP (https://nvbugs/6507082) +unittest/llmapi/test_llm.py::test_tokenizer_decode_incrementally[/scratch.trt_llm_data/llm-models/falcon-7b-instruct-False-0.95-TRTLLM] SKIP (https://nvbugs/6507082) unittest/llmapi/test_llm_multi_gpu_pytorch.py -m "gpu2" SKIP (https://nvbugs/6428092) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp2[False-False-True] SKIP (https://nvbugs/6432826) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp4[False-False-True] SKIP (https://nvbugs/6427411) @@ -493,5 +404,8 @@ unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_tinyllama_logits_processor_2 unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_tinyllama_logits_processor_tp2pp2 SKIP (https://nvbugs/6427411) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[None] SKIP (https://nvbugs/6162504) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[cuda_graph_config0] SKIP (https://nvbugs/6162504) +unittest/llmapi/test_llm_pytorch.py::test_llm_context_only_timed_out_kv_cache_exhausted[None-UCX-1000] SKIP (https://nvbugs/6490004) unittest/llmapi/test_memory_profiling.py::test_profile_kvcache SKIP (https://nvbugs/5580781) +unittest/tools/test_test_to_stage_mapping.py::test_cli_functionality[txt] SKIP (https://nvbugs/6482297) +unittest/tools/test_test_to_stage_mapping.py::test_cli_functionality[yml] SKIP (https://nvbugs/6482297) verl/test_verl_cases.py::test_trtllm_abort SKIP (https://nvbugs/6272653) diff --git a/tests/microbenchmarks/bench_moe/backend.py b/tests/microbenchmarks/bench_moe/backend.py index a4b682b5ab0b..866eabdd3774 100644 --- a/tests/microbenchmarks/bench_moe/backend.py +++ b/tests/microbenchmarks/bench_moe/backend.py @@ -40,6 +40,7 @@ class MoeBackendType(str, Enum): DEEPGEMM = "DEEPGEMM" DENSEGEMM = "DENSEGEMM" MEGAMOE_DEEPGEMM = "MEGAMOE_DEEPGEMM" + MEGAMOE_CUTEDSL = "MEGAMOE_CUTEDSL" @dataclass @@ -122,4 +123,8 @@ def get_backend_class(backend_type: MoeBackendType): from tensorrt_llm._torch.modules.fused_moe.mega_moe import MegaMoEDeepGemm return MegaMoEDeepGemm + if backend_type == MoeBackendType.MEGAMOE_CUTEDSL: + from tensorrt_llm._torch.modules.fused_moe.mega_moe import MegaMoECuteDsl + + return MegaMoECuteDsl raise ValueError(f"unknown MoE backend {backend_type!r}") diff --git a/tests/microbenchmarks/bench_moe/build.py b/tests/microbenchmarks/bench_moe/build.py index 0715bc783b23..36a3b5e27e27 100644 --- a/tests/microbenchmarks/bench_moe/build.py +++ b/tests/microbenchmarks/bench_moe/build.py @@ -49,6 +49,7 @@ "DeepGemmFusedMoE": "DEEPGEMM", "DenseGEMMFusedMoE": "DENSEGEMM", "MegaMoEDeepGemm": "MEGAMOE_DEEPGEMM", + "MegaMoECuteDsl": "MEGAMOE_CUTEDSL", "VanillaMoE": "VANILLA", } diff --git a/tests/microbenchmarks/bench_moe/mapping.py b/tests/microbenchmarks/bench_moe/mapping.py index 545b9109edcd..456cc1003c79 100644 --- a/tests/microbenchmarks/bench_moe/mapping.py +++ b/tests/microbenchmarks/bench_moe/mapping.py @@ -129,7 +129,7 @@ def _create_routing_method( topk_group=topk_group, routed_scaling_factor=1.0, callable_e_score_correction_bias=lambda: e_score_correction_bias, - is_fused=False, + is_fused=True, ) if routing_method_cls is MiniMaxM2MoeRoutingMethod: diff --git a/tests/microbenchmarks/bench_moe/search.py b/tests/microbenchmarks/bench_moe/search.py index f692911e7ff2..7fb8c40518c4 100644 --- a/tests/microbenchmarks/bench_moe/search.py +++ b/tests/microbenchmarks/bench_moe/search.py @@ -31,7 +31,7 @@ from .mapping import _PARALLEL_MODE_LAYOUTS, _resolve_mapping_layout from .specs import _ALL_BACKENDS, _FORCED_COMM_ENV_VALUES, ConfigSpec, ModelSpec, SearchSpec -_FUSED_COMM_BACKENDS = frozenset({"MEGAMOE_DEEPGEMM"}) +_FUSED_COMM_BACKENDS = frozenset({"MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL"}) def _is_deepep_feasible(num_ranks: int) -> bool: @@ -170,13 +170,26 @@ def is_candidate_valid( "use TEP/DEP only with other backends" ) - # MegaMoEDeepGemm is EP-only (asserts moe_tp_size == 1 in __init__); DTP/TTP are invalid. - if config.backend.upper() == "MEGAMOE_DEEPGEMM" and moe_tp > 1: + # MegaMoE backends are EP-only; DTP/TTP are invalid. + if config.backend.upper() in _FUSED_COMM_BACKENDS and moe_tp > 1: return False, ( - f"MEGAMOE_DEEPGEMM does not support MoE-TP (moe_tp_size={moe_tp}); " + f"{config.backend.upper()} does not support MoE-TP (moe_tp_size={moe_tp}); " "use DEP/TEP modes only" ) + # MegaMoEDeepGemm does not support TEP: its DeepGEMM fp8_fp4_mega_moe kernel + # assumes the MoE input is partitioned across ranks (single rank, or DEP with + # ep_size == parallel_size). TEP replicates the input TP-wide, so + # MegaMoEDeepGemm.__init__ raises NotImplementedError (enable_attention_dp=False, + # parallel_size>1). Prune here so the sweep records status="skipped" instead of a + # hard build failure. + if config.backend.upper() == "MEGAMOE_DEEPGEMM" and not enable_dp and world_size > 1: + return False, ( + f"MEGAMOE_DEEPGEMM does not support TEP (enable_attention_dp=False, " + f"parallel_size={world_size}>1); use DEP with ep_size==parallel_size " + "or enable attention-DP" + ) + # DENSEGEMM DTP: FC2 kernel requires (intermediate_size / moe_tp_size) % 256 == 0. # DENSEGEMM __init__ only checks the full intermediate_size, so a model like # DeepSeek V3 (intermediate_size=2048, 2048%256=0) passes __init__ but fails diff --git a/tests/microbenchmarks/bench_moe/specs.py b/tests/microbenchmarks/bench_moe/specs.py index 36134c9b429a..dff98ad5d06d 100644 --- a/tests/microbenchmarks/bench_moe/specs.py +++ b/tests/microbenchmarks/bench_moe/specs.py @@ -308,6 +308,23 @@ class RunResult: intermediate_size=2048, quant_algo="FP8_BLOCK_SCALES", routing_method="DEEPSEEK_V3", + n_group=1, + topk_group=1, + ), + # GLM-5 (zai-org/GLM-5): 256 routed experts, top-8, sigmoid/noaux_tc + # (DeepSeek-V3-style) routing with a single expert group (n_group=1, + # topk_group=1). intermediate_size is the per-expert moe_intermediate_size. + # quant_algo left None: pass --quant on the CLI (the glm_5 sweep uses NVFP4). + "glm_5": ModelSpec( + name="glm_5", + num_experts=256, + top_k=8, + hidden_size=6144, + intermediate_size=2048, + quant_algo=None, + routing_method="DEEPSEEK_V3", + n_group=1, + topk_group=1, ), # DeepSeek-V4-Pro: 1.6T total / 49B activated. quant_algo intentionally # left None: pass --quant on the CLI to pin the mode (the released diff --git a/tests/microbenchmarks/bench_moe/timing/autotune.py b/tests/microbenchmarks/bench_moe/timing/autotune.py index f993021be407..bdcf947f101b 100644 --- a/tests/microbenchmarks/bench_moe/timing/autotune.py +++ b/tests/microbenchmarks/bench_moe/timing/autotune.py @@ -44,6 +44,7 @@ def _run_autotune( The function always restores ``AutoTuner`` singleton state on exit so that ``--fast_autotune`` set for one case does not leak into the next. """ + tactic_autotune = bool(getattr(getattr(moe, "backend", moe), "tactic_autotune", False)) tuner = AutoTuner.get() saved_warmup = tuner.warmup saved_repeat = tuner.repeat @@ -63,3 +64,9 @@ def _run_autotune( tuner.warmup = saved_warmup tuner.repeat = saved_repeat tuner.stream_delay_micro_secs = saved_stream_delay + if tactic_autotune: + from tensorrt_llm._torch.custom_ops.cute_dsl_megamoe_custom_op import ( + release_megamoe_profiling_scratch, + ) + + release_megamoe_profiling_scratch() diff --git a/tests/microbenchmarks/bench_moe/utils.py b/tests/microbenchmarks/bench_moe/utils.py index d4874f5d91e1..40e7b0d75852 100644 --- a/tests/microbenchmarks/bench_moe/utils.py +++ b/tests/microbenchmarks/bench_moe/utils.py @@ -24,7 +24,7 @@ import torch import torch.distributed as dist -from tensorrt_llm._utils import local_mpi_rank, mpi_barrier, mpi_rank +from tensorrt_llm._utils import local_mpi_rank, mpi_barrier, mpi_comm, mpi_rank from .backend import MoeBackendType @@ -65,14 +65,23 @@ def _get_free_tcp_port() -> int: def _ensure_dist_for_megamoe(moe_backend: str, rank: int, world_size: int) -> None: """Initialize the torch.distributed NCCL ProcessGroup for MegaMoE.""" - if moe_backend.upper() != MoeBackendType.MEGAMOE_DEEPGEMM.value: + if moe_backend.upper() not in ( + MoeBackendType.MEGAMOE_DEEPGEMM.value, + MoeBackendType.MEGAMOE_CUTEDSL.value, + ): return if not torch.cuda.is_available(): raise RuntimeError("CUDA required for MegaMoE backend") if dist.is_initialized(): return - os.environ.setdefault("MASTER_ADDR", "127.0.0.1") - os.environ.setdefault("MASTER_PORT", str(_get_free_tcp_port())) + rendezvous = None + if rank == 0: + master_addr = os.environ.get("MASTER_ADDR") or "127.0.0.1" + master_port = os.environ.get("MASTER_PORT") or str(_get_free_tcp_port()) + rendezvous = (master_addr, master_port) + master_addr, master_port = mpi_comm().bcast(rendezvous, root=0) + os.environ["MASTER_ADDR"] = str(master_addr) + os.environ["MASTER_PORT"] = str(master_port) os.environ["RANK"] = str(rank) os.environ["WORLD_SIZE"] = str(world_size) os.environ["LOCAL_RANK"] = str(local_mpi_rank()) diff --git a/tests/microbenchmarks/bench_moe_comm.py b/tests/microbenchmarks/bench_moe_comm.py index e9d8ba839933..a43d0cb7020c 100644 --- a/tests/microbenchmarks/bench_moe_comm.py +++ b/tests/microbenchmarks/bench_moe_comm.py @@ -783,10 +783,12 @@ def _record_external(event: torch.cuda.Event) -> None: cupti_dispatch = detailed_stats.pop("dispatch_times_us") cupti_combine = detailed_stats.pop("combine_times_us") dispatch_times_us = [ - ct if ct is not None else et for ct, et in zip(cupti_dispatch, dispatch_times_us) + ct if ct is not None else et + for ct, et in zip(cupti_dispatch, dispatch_times_us, strict=True) ] combine_times_us = [ - ct if ct is not None else et for ct, et in zip(cupti_combine, combine_times_us) + ct if ct is not None else et + for ct, et in zip(cupti_combine, combine_times_us, strict=True) ] else: detailed_stats = {"dispatch_kernels": [], "combine_kernels": [], "other_kernels": []} @@ -825,6 +827,99 @@ def _gather_per_rank(times_us: List[float], iter_stats: bool = False) -> Dict[st return {f"rank{i}": (sum(t) / len(t) if t else 0.0) for i, t in enumerate(all_times)} +def _min_local_tokens_for_receiver_coverage(ep_size: int, top_k: int) -> int: + if top_k <= 0: + raise ValueError(f"top_k must be > 0, got {top_k}") + return (ep_size + top_k - 1) // top_k + + +def _scale_local_batch_sizes_for_receiver_coverage( + local_batch_sizes: List[int], ep_size: int, top_k: int +) -> List[int]: + min_tokens = _min_local_tokens_for_receiver_coverage(ep_size, top_k) + scaled: List[int] = [] + for local_num_tokens in local_batch_sizes: + value = max(int(local_num_tokens), min_tokens) + if not scaled or scaled[-1] != value: + scaled.append(value) + return scaled + + +def _verify_dispatch_sentinel( + backend: Communication, + *, + hidden_size: int, + top_k: int, + experts_per_rank: int, + ep_size: int, + act_dtype: torch.dtype, + device: torch.device, + local_num_tokens: Optional[int] = None, +) -> Dict[str, Any]: + """One dispatch+combine with sender-rank-tagged hidden_states. + + Each rank fills its hidden_states with the scalar ``rank + 1``. After + dispatch, each received row should be that integer cast to ``act_dtype``; + rows reading as 0 are either padding or a silently-broken peer read + (e.g. cross-rack MNNVL mapping that succeeded at construction but doesn't + actually back the peer's memory). Returns the per-rank decoded-sender + histogram for the caller to allgather and inspect. + """ + rank = mpi_rank() + min_tokens = _min_local_tokens_for_receiver_coverage(ep_size, top_k) + local_num_tokens = min_tokens if local_num_tokens is None else max(local_num_tokens, min_tokens) + all_rank_num_tokens = mpi_allgather(int(local_num_tokens)) + if not backend.is_workload_feasible(all_rank_num_tokens, num_chunks=1): + return {"rank": rank, "skipped": True} + + sentinel = float(rank + 1) + hidden_states = torch.full( + (local_num_tokens, hidden_size), + sentinel, + dtype=act_dtype, + device=device, + ) + flat_slots = torch.arange(local_num_tokens * top_k, device=device, dtype=torch.int64) + schedule = flat_slots + rank + target_rank = schedule % ep_size + local_expert = (schedule // ep_size) % experts_per_rank + token_selected_slots = ( + (target_rank * experts_per_rank + local_expert) + .view(local_num_tokens, top_k) + .to(torch.int32) + ) + token_final_scales = torch.ones( + local_num_tokens, + top_k, + dtype=torch.float32, + device=device, + ) + + backend.prepare_dispatch(token_selected_slots, all_rank_num_tokens) + recv_hs, _, _, _ = backend.dispatch( + hidden_states, + None, + token_selected_slots, + token_final_scales, + all_rank_num_tokens, + ) + # Pair dispatch with a combine so backend state mirrors the bench's + # warmup->timing call pattern (NCCL_EP especially relies on this). + shape = list(recv_hs.shape) + shape[-1] = hidden_size + moe_out = torch.zeros(tuple(shape), dtype=torch.bfloat16, device=recv_hs.device) + backend.combine(moe_out, all_rank_max_num_tokens=max(all_rank_num_tokens)) + torch.cuda.synchronize() + + first_col = recv_hs[:, 0].to(torch.float32) + decoded = first_col.round().to(torch.int64) + unique, counts = decoded.unique(return_counts=True) + histogram: Dict[int, int] = { + int(u) - 1: int(c) for u, c in zip(unique.tolist(), counts.tolist(), strict=True) + } + return {"rank": rank, "histogram": histogram} + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Unified MoE communication microbenchmark (MPI).") parser.add_argument( @@ -843,6 +938,7 @@ def parse_args() -> argparse.Namespace: "NVLINK_TWO_SIDED", "DEEPEP", "DEEPEPLOWLATENCY", + "NCCL_EP", ], help="Which communication backend to benchmark (default: run all backends).", ) @@ -951,6 +1047,16 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Disable CUDA graph mode. By default, dispatch and combine are captured into CUDA graphs for lower CPU overhead and more accurate timing.", ) + parser.add_argument( + "--verify", + action="store_true", + help=( + "Run a single sentinel dispatch per backend before timing and print a " + "receiver/sender contribution matrix. Detects silent cross-rack " + "correctness failures where dispatch appears to succeed but produces " + "zeros or local-only data." + ), + ) parser.add_argument( "--pdl", action="store_true", @@ -1050,6 +1156,10 @@ def _run_benchmark_worker_under_current_mpi( hidden_size, top_k, num_experts_total, quant_algo = _resolve_profile_args(args) local_batch_sizes = _iter_local_batch_sizes(args) + if args.verify: + local_batch_sizes = _scale_local_batch_sizes_for_receiver_coverage( + local_batch_sizes, ep_size, top_k + ) act_dtype = torch.bfloat16 quant_config = ( QuantConfig(quant_algo=None) @@ -1098,7 +1208,14 @@ def _run_benchmark_worker_under_current_mpi( print(json.dumps(benchmark_metadata, indent=2), flush=True) backends = ( - ["ALLGATHER", "NVLINK_ONE_SIDED", "NVLINK_TWO_SIDED", "DEEPEP", "DEEPEPLOWLATENCY"] + [ + "ALLGATHER", + "NVLINK_ONE_SIDED", + "NVLINK_TWO_SIDED", + "DEEPEP", + "DEEPEPLOWLATENCY", + "NCCL_EP", + ] if args.backend is None else [args.backend] ) @@ -1159,11 +1276,62 @@ def _run_benchmark_worker_under_current_mpi( reduce_results=False, model_config=model_config, init_load_balancer=False, - without_comm=True, ) # Ensure quantization params (e.g., NVFP4 global scale) live on CUDA. moe = moe.to(device) + if args.verify: + verify_local = _verify_dispatch_sentinel( + backend, + hidden_size=hidden_size, + top_k=top_k, + experts_per_rank=experts_per_rank, + ep_size=ep_size, + act_dtype=act_dtype, + device=device, + local_num_tokens=local_batch_sizes[0], + ) + all_verify = mpi_allgather(verify_local) + # Pass criterion: every receiver must have at least one token from + # every sender [0, ep_size). The verify local_num_tokens is scaled + # so local_num_tokens * top_k covers every receiver; + # any zero-column means the recv buffer was silently dropped from + # that sender. + verify_failed = False + for entry in all_verify: + if entry.get("skipped"): + verify_failed = True + break + hist = entry.get("histogram", {}) + if any(hist.get(s, 0) == 0 for s in range(ep_size)): + verify_failed = True + break + if rank == 0: + status = "FAIL" if verify_failed else "PASS" + print( + f"=== [verify] {backend_name} {status} -- sender->receiver " + f"contribution (rows=receiver, cols=sender; -1 col = " + f"padding/unmapped) ===", + flush=True, + ) + cols = [-1, *range(ep_size)] + header = "R\\S | " + " ".join(f"{c:>5}" for c in cols) + " | total" + print(header) + for entry in sorted(all_verify, key=lambda e: e.get("rank", -1)): + r = entry.get("rank") + if entry.get("skipped"): + print(f"{r:>3} | skipped (workload not feasible at verify size)") + continue + hist = entry.get("histogram", {}) + cells = " ".join(f"{hist.get(c, 0):>5}" for c in cols) + print(f"{r:>3} | {cells} | {sum(hist.values()):>5}") + sys.stdout.flush() + if verify_failed: + _maybe_warn_rank0( + f"[bench_moe_comm] Skipping timing for {backend_name}: verify FAILED." + ) + continue + for local_num_tokens in local_batch_sizes: all_rank_num_tokens = mpi_allgather(int(local_num_tokens)) if not backend.is_workload_feasible(all_rank_num_tokens, num_chunks=1): diff --git a/tests/scripts/cute_dsl_kernels/top_k/run_gvr_topk.py b/tests/scripts/cute_dsl_kernels/top_k/run_gvr_topk.py index 39a44f482b6b..41cb23260cd4 100644 --- a/tests/scripts/cute_dsl_kernels/top_k/run_gvr_topk.py +++ b/tests/scripts/cute_dsl_kernels/top_k/run_gvr_topk.py @@ -62,6 +62,8 @@ def _compile( return_output_values: bool, cluster_size: int = 1, seqlen_sorted: bool = False, + p4_warp_redundant: bool = True, + p2_warp_redundant: bool = True, ): """JIT-compile the GVR kernel for a specific knob combination. @@ -136,6 +138,8 @@ def _compile( return_output_values=return_output_values, cluster_size=cluster_size, seqlen_sorted=seqlen_sorted, + p4_warp_redundant=p4_warp_redundant, + p2_warp_redundant=p2_warp_redundant, ) return cute.compile( kernel, @@ -172,6 +176,8 @@ def gvr_topk_decode( cluster_size: int = 1, seqlen_sorted: bool = False, order_row: Optional[torch.Tensor] = None, + p4_warp_redundant: bool = True, + p2_warp_redundant: bool = True, ) -> tuple[torch.Tensor, torch.Tensor]: """CuTe DSL GVR Top-K wrapper with every tuning knob exposed. @@ -197,6 +203,17 @@ def gvr_topk_decode( given CTA processes, so longer rows land in earlier waves. Use together with :func:`gvr_topk_sort_prepare`. Compatible with ``cluster_size > 1``. + p4_warp_redundant: Default True. Phase 4 redundant-warp cadence: + every warp replays the k-th bin search reduce and the + snap-loop decision from the staged SMEM partials + (bit-identical across warps), removing the publish + barriers and keeping threshold/convergence state in + registers. False restores the leader-thread cadence. + p2_warp_redundant: Default True. Phase 2 redundant-warp secant + cadence (cluster_size == 1 only): one barrier per + round; every warp reduces the staged warp counts and + replays the classify + secant update in registers. + False restores the leader cadence. order_row: Required iff ``seqlen_sorted=True``. Request-level — ``int32[batch_size = num_rows // next_n]`` on the same device as ``logits``; ``order_row[i]`` is the original @@ -293,6 +310,8 @@ def gvr_topk_decode( return_output_values, cluster_size, seqlen_sorted, + p4_warp_redundant, + p2_warp_redundant, ) # When return_output_values=False the kernel was compiled to skip # STG.value and accepts None for the value-output slot. @@ -746,6 +765,68 @@ def test_gvr_topk_decode( ) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device required") +@pytest.mark.parametrize( + "dtype,top_k", + [ + (torch.bfloat16, 1024), + (torch.float32, 2048), + ], +) +@pytest.mark.parametrize("cluster_size", [1, 4]) +@pytest.mark.parametrize( + "p4_warp_redundant,p2_warp_redundant", + [ + # Leader-path coverage: the redundant-warp knobs default ON, so + # the main sweep above exercises the redundant cadences; these + # combinations keep the knob-off (pre-redundant leader) paths and + # the two mixed configurations compiling and exact. + (False, False), + (False, True), + (True, False), + ], +) +def test_gvr_topk_decode_leader_paths( + dtype: torch.dtype, + top_k: int, + cluster_size: int, + p4_warp_redundant: bool, + p2_warp_redundant: bool, +) -> None: + N = 65536 + batch_size = 32 + seed = 42 + logits, pre_idx, seq_lens = _make_inputs( + batch_size, + N, + top_k, + dtype, + seed, + next_n=1, + compress_ratio=1, + ) + num_sms = torch.cuda.get_device_properties(0).multi_processor_count + _, out_idxs = gvr_topk_decode( + logits, + pre_idx, + seq_lens, + top_k, + next_n=1, + num_sms=num_sms, + return_output_values=False, + cluster_size=cluster_size, + p4_warp_redundant=p4_warp_redundant, + p2_warp_redundant=p2_warp_redundant, + ) + torch.cuda.synchronize() + ok, msg = _tie_aware_correct(out_idxs, logits, seq_lens, top_k, 1) + assert ok, ( + f"dtype={dtype} K={top_k} cluster_size={cluster_size} " + f"p4_warp_redundant={p4_warp_redundant} " + f"p2_warp_redundant={p2_warp_redundant}: {msg}" + ) + + def main() -> None: p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) p.add_argument("--dtype", choices=["fp32", "bf16", "fp16"], default="bf16") diff --git a/tests/scripts/perf-sanity/aggregated/gemma4_26b_a4b_nvfp4_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gemma4_26b_a4b_nvfp4_blackwell.yaml new file mode 100644 index 000000000000..a9997d646100 --- /dev/null +++ b/tests/scripts/perf-sanity/aggregated/gemma4_26b_a4b_nvfp4_blackwell.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +metadata: + model_name: gemma_4_26b_a4b_nvfp4 + supported_gpus: + - B200 +hardware: + gpus_per_node: 1 +server_configs: + # Production-like concurrency 64 covers the scheduler, KV-pool sizing, + # CUDA graph padding, and VSWA block-table decode paths together. + - name: "gemma4_26b_a4b_nvfp4_tp1_1k1k" + tensor_parallel_size: 1 + max_batch_size: 256 + load_format: dummy + trust_remote_code: true + enable_chunked_prefill: true + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + multimodal_config: + encoder_cache_max_bytes: 0 + client_configs: + - name: "con64_iter3_1k1k" + concurrency: 64 + iterations: 3 + isl: 1000 + osl: 1000 + backend: "openai-chat" diff --git a/tests/scripts/perf-sanity/cache_transceiver_precheck/README.md b/tests/scripts/perf-sanity/cache_transceiver_precheck/README.md new file mode 100644 index 000000000000..2588c1362157 --- /dev/null +++ b/tests/scripts/perf-sanity/cache_transceiver_precheck/README.md @@ -0,0 +1,105 @@ +# Disagg Perf-Sanity Cache-Transceiver Precheck + +A fast go/no-go network check that runs **inside every disaggregated +perf-sanity CI stage, before the real test**. It brings up one MPI instance +per ctx/gen server with the *same* topology, node placement, UCX environment, +and `cache_transceiver_config` as the real test, transfers deterministic KV +data ctx → gen through the cache transceiver, and verifies the received +bytes. If the transfer hangs, errors, or mismatches, the stage fails +immediately with a specific verdict — the (expensive) model bring-up never +starts. + +## Parity with the real test — by construction + +| Requirement | How it is guaranteed | +|---|---| +| Same UCX env vars (incl. the `unset UCX_TLS` cases) | `jenkins/scripts/perf/submit.py` builds the precheck commands from the **same** `ucx_tls_cmd` + `$CTX/GEN_WORKER_ENV_VARS` strings as the worker steps; `slurm_precheck_run.sh` sources the same `slurm_env_setup.sh` (the `UCX_TLS=tcp` fixup) as `slurm_run.sh`. | +| Same instance count / parallelism | One precheck `srun` per ctx/gen server with the same `-N/--ntasks/--ntasks-per-node/--mpi=pmix` and the same node slices (`-w`) as the real server steps (`slurm_launch_draft.sh`). TP/PP/CP/attention-DP come from the same `worker_config`. | +| Same transceiver config | `CacheTransceiverConfig(**yaml["worker_config"][role]["cache_transceiver_config"])` — the yaml block is passed through verbatim (backend, `max_tokens_in_buffer`, timeouts, ...). | +| Same KV cache manager version + transceiver runtime | Explicit per-side `kv_cache_config.use_kv_cache_manager_v2` wins; absent means "auto" and resolves against the model class's `get_model_defaults()`, and `transceiver_runtime: auto` resolves via `get_preferred_transceiver_runtime()` (NIXL-gated) — both through the same llm_utils code serving uses. V2 requires the Python transceiver (the C++ one only supports V1); a V2+CPP combination fails fast with INIT_ERROR. | + +Asymmetric layouts (ctx dep4 → gen dep16, ctx pp8 → gen tp32, ...) are +supported: data is seeded per (request, **global** layer) and constant along +the KV-head axis, so the receiver regenerates its expected slice locally under +any TP/PP resharding. KV shape (layers/heads/head_dim, MLA vs GQA, MTP nextn +layers) is read from the real model's `config.json` under `$LLM_MODELS_ROOT`. + +## Enabling / disabling + +**On by default** for every disaggregated perf-sanity test. To opt out: + +- per test yaml: + + ```yaml + cache_transceiver_precheck: + enabled: false + # optional overrides (defaults in precheck_config.PRECHECK_DEFAULTS): + # request_lengths: [1024, 8192] + # num_requests: 2 + # wave_timeout_s: 180 + # wireup_timeout_s: 1800 # first-rep NIXL agent wire-up allowance + # step_timeout_s: 2700 # external srun timeout (default derives from topology) + ``` + +- or globally at launch-script generation time: `TRTLLM_DISAGG_CT_PRECHECK=0` + (kill switch; `=1` force-enables). The env var, when set, overrides the yaml + either way. + +## Timeouts + +The first rep of the schedule (the warmup rep) additionally budgets +`wireup_timeout_s` (default `min(1800, 150 * max world size)`): the C++ NIXL +path pays a one-time serialized `fetchRemoteMD` metadata exchange per +(receiver rank, ctx rank) agent pair, and cold cross-rack fetches were +measured at 100-170s each — real serving absorbs this as slow first requests, +so the precheck does too. Later reps run under the tight `wave_timeout_s`, +which is what actually catches hangs. Set `PRECHECK_DEBUG=1` in the worker +env to raise the C++/Python transceiver log levels when debugging a stall. + +## Failure output + +The sbatch log gets a summary block: per-instance verdicts +(`status/*.status`), the tail of each failing step log, and UCX red-flag lines +(`sw-emul` host-staged tcp fallback, UCX ERROR/WARN). Full artifacts under +`/cache_transceiver_precheck/`: + +``` +status/{ctx,gen}_.json # per-case detail: PASS/TIMEOUT/TRANSFER_ERROR/ + # MISMATCH/INIT_ERROR + reason + UCX env snapshot +status/{ctx,gen}_.status # one-line verdict (parsed by the launch script) +logs/{ctx,gen}_.log # merged per-rank logs (UCX_PROTO_INFO=used table) +csv/gen_/__recv.csv # C++ transceiver per-request bandwidth +csv/ctx_/_.csv # Python transceiver per-task perf + # (KVSendTask throughput on the ctx side) +``` + +## Manual runs + +```bash +# Inspect what a yaml resolves to (no GPU needed): +python3 run_precheck.py --role gen --server-idx 0 --dry-run \ + --config ../disaggregated/.yaml --work-dir /tmp/ct --llm-src + +# On a SLURM allocation: one srun per instance, e.g. ctx dep4 + gen dep8: +srun -N1 --ntasks=4 --mpi=pmix python3 run_precheck.py --role ctx --server-idx 0 \ + --config --work-dir --llm-src & +srun -N2 --ntasks=8 --mpi=pmix python3 run_precheck.py --role gen --server-idx 0 \ + --config --work-dir --llm-src & +wait +``` + +Tests: + +- Config resolution (CPU-only): + `pytest tests/unittest/others/test_cache_transceiver_precheck_config.py` +- Driver logic + internal-API contract (the precheck drives unstable + TRT-LLM internals via `run_precheck.load_internal_apis()`; the contract + tests catch upstream renames in pre-merge CI): + `pytest tests/unittest/others/test_cache_transceiver_precheck_run.py` +- End-to-end on one GPU (one `mpirun` world per ctx/gen instance, real + NIXL/PYTHON transfer + verdicts, incl. multi-instance and asymmetric TP): + `pytest tests/unittest/disaggregated/test_cache_transceiver_precheck_e2e.py` + +Lineage: adapted from `examples/disaggregated/slurm/cache_transceiver_test` +(the UCX tuning harness), reduced to a single sweep and extended to +asymmetric parallelism, attention DP, and multi-instance pairing. diff --git a/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py b/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py new file mode 100644 index 000000000000..916f2c4678a3 --- /dev/null +++ b/tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py @@ -0,0 +1,515 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Config resolution for the disagg cache-transceiver precheck. + +Pure-stdlib (+pyyaml) module: NO torch / tensorrt_llm imports, so it can be +unit-tested on a CPU-only machine and imported by the launch tooling. + +The precheck mirrors the *exact* disaggregated perf-sanity test configuration: +instance counts and per-side parallelism come from the same yaml +(`tests/scripts/perf-sanity/disaggregated/*.yaml`) the real test runs with, +and the transceiver is built from the yaml's own `cache_transceiver_config` +block, so config parity is by construction rather than by copying values. +""" + +import json +import os + +# Optional per-yaml overrides live under a `cache_transceiver_precheck:` block. +PRECHECK_DEFAULTS = { + # Request lengths to transfer. None -> derived: [1024, benchmark ISL], + # clamped to cache_transceiver_config.max_tokens_in_buffer and to + # max_request_length below. + "request_lengths": None, + # Cap for the DERIVED lengths only (long-ISL tests would otherwise move + # multi-GB per request in the precheck); explicit request_lengths in the + # yaml's cache_transceiver_precheck block are used as-is. + "max_request_length": 32768, + # Measured requests per (peer, request_length) after warmup. + "num_requests": 2, + "warmup_requests": 1, + # Upper bound on concurrently in-flight transfer pairs. The n_pairs + # dp-rank pairings are processed in batches ("waves") of at most this + # many -- see waves() for the term's exact meaning (it is NOT token + # chunking). + "max_concurrent_pairs": 8, + # signal.alarm / hang-detector budget for one wave of transfers. + "wave_timeout_s": 180, + # Extra budget for the FIRST rep of the schedule, which pays the one-time + # NIXL agent metadata wire-up (fetchRemoteMD over the mgmt network): with + # MLA + ctx pp>1 every receiving rank must connect to every ctx rank, and + # a cold cross-rack fetch was measured at 100-170s per agent pair. Real + # serving absorbs this as slow first requests (no hard alarm), so the + # precheck must too. None -> derived: min(1800, 150 * max world size). + "wireup_timeout_s": None, + # How long the gen side waits for the ctx rendezvous files (covers ctx + # KV-pool allocation + NIXL/UCX transceiver bring-up). + "rendezvous_timeout_s": 600, + # Verify the received bytes against the deterministic fill pattern. + "verify_data": True, +} + +# Fallback KV shape when the model directory cannot be resolved: the precheck +# still exercises the exact network path, just with a synthetic cache shape. +FALLBACK_KV_SHAPE = { + "num_layers": 32, + "num_kv_heads": 8, + "head_dim": 128, + "is_mla": False, + "source": "fallback", +} + + +def _side_cfg(cfg, role): + worker_config = cfg.get("worker_config", {}) or {} + side = worker_config.get(role, {}) or {} + if not side: + raise ValueError(f"worker_config.{role} missing in disagg yaml") + return side + + +def _parallel(side): + tp = int(side.get("tensor_parallel_size", 1)) + pp = int(side.get("pipeline_parallel_size", 1)) + cp = int(side.get("context_parallel_size", 1)) + adp = bool(side.get("enable_attention_dp", False)) + world = tp * pp * cp + # With attention DP the request-level data parallel width is the attention + # TP width (each dp rank owns whole requests / full KV heads). + dp_size = tp if adp else 1 + return { + "tp": tp, + "pp": pp, + "cp": cp, + "enable_attention_dp": adp, + "world_size": world, + "dp_size": dp_size, + } + + +def _spec_nextn(side): + spec = side.get("speculative_config") or {} + if str(spec.get("decoding_type", "")).upper() != "MTP": + return 0 + # Test yamls spell the MTP depth either way: num_nextn_predict_layers is + # the deprecated alias MTPDecodingConfig remaps to max_draft_len, so both + # must resolve to the same KV-pool spec-layer count here. + n = spec.get("num_nextn_predict_layers") + if n is None: + n = spec.get("max_draft_len", 0) + return int(n or 0) + + +def wireup_timeout_s(max_world): + """First-rep NIXL agent wire-up allowance (see PRECHECK_DEFAULTS).""" + return min(1800, 150 * int(max_world)) + + +def default_step_timeout_s(max_world): + """External (srun-level) timeout covering one precheck instance. + + Includes the first-rep wire-up. Imported by the launch tooling + (jenkins/scripts/perf/{,local/}submit.py) so the outer timeout can + never drift below the driver's internal budget. + """ + return 900 + wireup_timeout_s(max_world) + + +def precheck_prefix_lines( + cfg, benchmark_mode, config_path_expr, ucx_tls_cmd, max_world, stage_name="" +): + """Launch-script export lines wiring the precheck gate. + + Single owner of the enable/kill-switch policy, the step-timeout default, + and the export names the gate consumes — shared by + jenkins/scripts/perf/submit.py and jenkins/scripts/perf/local/submit.py. + `config_path_expr` and the env-var references are launch-script-side + expressions ($llmSrcNode etc.), expanded at sbatch runtime. + """ + knobs = cfg.get("cache_transceiver_precheck", {}) or {} + # On by default; yaml opts out per test; the env var (when set) overrides + # the yaml either way (global kill switch). Parse the usual boolean spellings + # so a well-meant TRTLLM_DISAGG_CT_PRECHECK=true force-enable is not silently + # read as "off"; reject anything ambiguous instead of guessing. + env = os.environ.get("TRTLLM_DISAGG_CT_PRECHECK") + if env is not None: + val = env.strip().lower() + if val in ("1", "true", "on", "yes"): + enabled = True + elif val in ("0", "false", "off", "no"): + enabled = False + else: + raise ValueError( + "TRTLLM_DISAGG_CT_PRECHECK must be a boolean " + f"(1/0/true/false/on/off/yes/no), got {env!r}" + ) + else: + enabled = bool(knobs.get("enabled", True)) + cmd = ( + "python3 $llmSrcNode/tests/scripts/perf-sanity/cache_transceiver_precheck/" + f"run_precheck.py --config {config_path_expr} " + "--work-dir $testOutputDir/cache_transceiver_precheck " + f"--benchmark-mode {benchmark_mode} --llm-src $llmSrcNode" + ) + lines = [ + f"export ctPrecheckEnabled={int(enabled)}", + # The external srun timeout must cover the driver's first-rep NIXL + # wire-up allowance; the default derives from the same formula the + # driver budgets with (default_step_timeout_s). + f"export ctPrecheckTimeout=" + f"{int(knobs.get('step_timeout_s', default_step_timeout_s(max_world)))}", + "export precheckRunScript=$llmSrcNode/jenkins/scripts/perf/" + "disaggregated/slurm_precheck_run.sh", + f'export pytestCommandCTXPrecheck="{ucx_tls_cmd} $CTX_WORKER_ENV_VARS {cmd} --role ctx"', + f'export pytestCommandGENPrecheck="{ucx_tls_cmd} $GEN_WORKER_ENV_VARS {cmd} --role gen"', + ] + if stage_name: + # Suite name for the synthetic junit xml the gate writes on failure + # (absent -> the gate falls back to $SLURM_JOB_NAME). + lines.append(f'export stageName="{stage_name}"') + return lines + + +def gate_library_content(draft_launch_sh, llm_src): + """The precheck gate shell library (defines run_cache_transceiver_precheck). + + Spliced ahead of the disagg draft by both submit.py. Single owner of the + load/splice, so the two launch generators can't drift. + + Located next to the draft by default; falls back to the in-repo copy when + the draft lives outside the repo (a custom --draft-launch-sh), so launch + generation never dies with FileNotFoundError. Whitespace-only lines are + dropped to match the rest of the assembled launch script. + """ + candidates = [ + os.path.join(os.path.dirname(draft_launch_sh), "slurm_ct_precheck_gate.sh"), + os.path.join( + llm_src, "jenkins", "scripts", "perf", "disaggregated", "slurm_ct_precheck_gate.sh" + ), + ] + for path in candidates: + if os.path.exists(path): + with open(path) as f: + lines = [ln.strip() for ln in f.read().split("\n") if ln.strip()] + return "\n".join(lines) + "\n" + raise FileNotFoundError( + "precheck gate library slurm_ct_precheck_gate.sh not found; looked in: " + + ", ".join(candidates) + ) + + +def resolve_plan(cfg, benchmark_mode="e2e"): + """Build the shared precheck plan both roles must agree on. + + `cfg` is the parsed disagg perf-sanity yaml. Returns a plain dict; the + `fingerprint` field is exchanged over the rendezvous channel so a ctx/gen + disagreement (e.g. mismatched yamls) fails fast with a clear error. + """ + hardware = cfg.get("hardware", {}) or {} + benchmark = cfg.get("benchmark", {}) or {} + + yaml_mode = str(benchmark.get("mode", "")) + if benchmark_mode == "gen_only" and "gen_only_no_context" in yaml_mode: + # No ctx workers at launch -> no KV transfer in the real test either. + return {"skip": True, "skip_reason": "gen_only_no_context mode has no KV transfer"} + + num_ctx_servers = int(hardware.get("num_ctx_servers", 0) or 0) + num_gen_servers = int(hardware.get("num_gen_servers", 0) or 0) + if num_ctx_servers < 1 or num_gen_servers < 1: + raise ValueError( + f"hardware.num_ctx_servers/num_gen_servers must be >= 1, got " + f"{num_ctx_servers}/{num_gen_servers}" + ) + gpus_per_node = int(hardware.get("gpus_per_node", 0) or 0) + if gpus_per_node < 1: + raise ValueError("hardware.gpus_per_node is required") + + ctx_side = _side_cfg(cfg, "ctx") + gen_side = _side_cfg(cfg, "gen") + ctx = _parallel(ctx_side) + gen = _parallel(gen_side) + + ctx_xcvr = ctx_side.get("cache_transceiver_config") or {} + gen_xcvr = gen_side.get("cache_transceiver_config") or {} + if not ctx_xcvr.get("backend") and not gen_xcvr.get("backend"): + return {"skip": True, "skip_reason": "no cache_transceiver_config.backend in yaml"} + if ctx_xcvr.get("backend") != gen_xcvr.get("backend"): + raise ValueError( + f"ctx/gen cache_transceiver_config.backend mismatch: " + f"{ctx_xcvr.get('backend')} vs {gen_xcvr.get('backend')}" + ) + + knobs = dict(PRECHECK_DEFAULTS) + knobs.update(cfg.get("cache_transceiver_precheck", {}) or {}) + + tokens_per_block = int( + (ctx_side.get("kv_cache_config") or {}).get("tokens_per_block", 32) or 32 + ) + max_tokens_in_buffer = ctx_xcvr.get("max_tokens_in_buffer") or gen_xcvr.get( + "max_tokens_in_buffer" + ) + + req_lens = knobs["request_lengths"] + if not req_lens: + isl = int(benchmark.get("input_length", 1024) or 1024) + req_lens = [1024, min(isl, int(knobs["max_request_length"]))] + if max_tokens_in_buffer: + req_lens = [min(int(r), int(max_tokens_in_buffer)) for r in req_lens] + req_lens = sorted({max(int(r), tokens_per_block) for r in req_lens}) + + # Transfer pairs: with attention DP each dp rank owns whole requests, so + # cover every dp rank on both sides; without DP a single request already + # involves every rank of the instance (KV is sharded across TP/PP). + n_pairs = max(ctx["dp_size"], gen["dp_size"], 1) + wave_size = max(1, min(n_pairs, int(knobs["max_concurrent_pairs"]))) + + plan = { + "skip": False, + "num_ctx_servers": num_ctx_servers, + "num_gen_servers": num_gen_servers, + "gpus_per_node": gpus_per_node, + "ctx": ctx, + "gen": gen, + "tokens_per_block": tokens_per_block, + "request_lengths": req_lens, + "num_requests": int(knobs["num_requests"]), + "warmup_requests": int(knobs["warmup_requests"]), + "n_pairs": n_pairs, + "wave_size": wave_size, + "wave_timeout_s": int(knobs["wave_timeout_s"]), + "wireup_timeout_s": int( + knobs["wireup_timeout_s"] + if knobs["wireup_timeout_s"] is not None + else wireup_timeout_s(max(ctx["world_size"], gen["world_size"])) + ), + "rendezvous_timeout_s": int(knobs["rendezvous_timeout_s"]), + "verify_data": bool(knobs["verify_data"]), + } + for role, side, xcvr in (("ctx", ctx_side, ctx_xcvr), ("gen", gen_side, gen_xcvr)): + kv_cfg = side.get("kv_cache_config") or {} + plan[f"{role}_cache_transceiver_config"] = xcvr + plan[f"{role}_num_nextn_predict_layers"] = _spec_nextn(side) + plan[f"{role}_kv_dtype"] = str(kv_cfg.get("dtype", "auto")) + # Tri-state, matching KvCacheConfig's pydantic default: explicit + # True/False from the yaml wins; absent means "auto", which the + # driver resolves against the model class's get_model_defaults() at + # runtime — exactly like serving (_resolve_kv_cache_manager_v2_auto). + plan[f"{role}_use_kv_cache_manager_v2"] = kv_cfg.get("use_kv_cache_manager_v2", "auto") + plan["fingerprint"] = plan_fingerprint(plan) + return plan + + +def plan_fingerprint(plan): + """Stable string both sides must agree on before transferring.""" + keys = ( + "num_ctx_servers", + "num_gen_servers", + "ctx", + "gen", + "ctx_cache_transceiver_config", + "gen_cache_transceiver_config", + "ctx_num_nextn_predict_layers", + "gen_num_nextn_predict_layers", + "tokens_per_block", + "request_lengths", + "num_requests", + "warmup_requests", + "n_pairs", + "wave_size", + ) + return json.dumps({k: plan[k] for k in keys}, sort_keys=True) + + +def side_plan(plan, role): + """Per-role view: this role's parallelism + transceiver/kv config.""" + return { + "role": role, + "parallel": plan[role], + "cache_transceiver_config": plan[f"{role}_cache_transceiver_config"], + "kv_dtype": plan[f"{role}_kv_dtype"], + "num_nextn_predict_layers": plan[f"{role}_num_nextn_predict_layers"], + "use_kv_cache_manager_v2": plan[f"{role}_use_kv_cache_manager_v2"], + "num_peers": plan["num_gen_servers" if role == "ctx" else "num_ctx_servers"], + } + + +def pair_participates(plan, role, tp_rank, pair_idx): + """Whether this rank takes part in transfer pair `pair_idx`. + + Attention-DP side: pair k belongs to dp rank k % dp_size (every pp stage + of that dp rank participates). Non-DP side: KV is sharded across the whole + instance, so every rank participates in every pair. + """ + side = plan[role] + if not side["enable_attention_dp"]: + return True + return tp_rank == pair_idx % side["dp_size"] + + +def owned_pairs(plan, role, tp_rank, wave_pairs): + return [k for k in wave_pairs if pair_participates(plan, role, tp_rank, k)] + + +def max_owned_per_wave(plan, role): + """Max concurrently owned pairs per rank (KV pool sizing).""" + side = plan[role] + wave = plan["wave_size"] + if not side["enable_attention_dp"]: + return wave + return (wave + side["dp_size"] - 1) // side["dp_size"] + + +def waves(plan): + """Pair indices grouped into concurrency-bounded waves. + + A "wave" is the batch of TRANSFER PAIRS in flight at once (at most + max_concurrent_pairs of the n_pairs dp-rank pairings) -- NOT token/data + chunking. It bounds the synthetic KV pool size and gives the per-wave + alarm a precise target while still exercising concurrent transfers. + """ + pairs = list(range(plan["n_pairs"])) + size = plan["wave_size"] + return [pairs[i : i + size] for i in range(0, len(pairs), size)] + + +# --------------------------------------------------------------------------- # +# Model KV shape resolution +# --------------------------------------------------------------------------- # +def _load_model_path_dict(llm_src): + """Import MODEL_PATH_DICT from tests/integration/defs/perf/_model_paths.py.""" + path = os.path.join(llm_src, "tests", "integration", "defs", "perf", "_model_paths.py") + namespace = {} + with open(path) as f: + exec(compile(f.read(), path, "exec"), namespace) # noqa: S102 - repo-local constants file + return namespace["MODEL_PATH_DICT"] + + +def resolve_model_dir(cfg, llm_src=None, llm_models_root=None): + """Resolve the local model directory the same way test_perf_sanity does. + + metadata.model_name -> MODEL_PATH_DICT -> $LLM_MODELS_ROOT/, + falling back to metadata.model_dir_name under LLM_MODELS_ROOT. Returns + None when nothing resolvable exists (precheck then uses FALLBACK_KV_SHAPE). + """ + llm_models_root = llm_models_root or os.environ.get("LLM_MODELS_ROOT", "") + if not llm_models_root: + return None + metadata = cfg.get("metadata", {}) or {} + candidates = [] + model_name = metadata.get("model_name", "") + if llm_src and model_name: + try: + rel = _load_model_path_dict(llm_src).get(model_name) + if rel: + candidates.append(os.path.join(llm_models_root, rel)) + except (OSError, KeyError, SyntaxError): + pass + if metadata.get("model_dir_name"): + candidates.append(os.path.join(llm_models_root, metadata["model_dir_name"])) + for cand in candidates: + if os.path.isfile(os.path.join(cand, "config.json")): + return cand + return None + + +def unmodelable_kv_reason(hf_cfg): + """Why the precheck cannot faithfully model this model's KV layout, or None. + + The precheck allocates a SINGLE KV pool from (layers, kv_heads, head_dim). + Sparse-attention models (DeepSeek V4 / DSA: an indexer with its own + INDEX_KEY pool at a different stride than the main K/V pool) cannot be + represented that way. The precheck is a NETWORK/transceiver availability + check, not a KV-correctness test, so rather than skip these (the check + must still run) it falls back to a simple generic pool -- the transfer + exercises the exact same UCX/NIXL path, topology, and transceiver config, + just with generic bytes. + """ + if any(k in hf_cfg for k in ("index_topk", "index_n_heads", "index_head_dim")): + return ( + "sparse-attention model (DSA/indexer): using a generic KV pool -- this checks " + "the network/transceiver path, not V4's exact KV layout" + ) + return None + + +def model_kv_shape(model_dir): + """KV cache shape (per-token layout) from the model's config.json. + + Handles MLA checkpoints (kv_lora_rank present: one latent 'head' of + kv_lora_rank + qk_rope_head_dim) and GQA/MHA. `num_layers` excludes the + MTP nextn layers -- those are added by the KV cache manager via + spec_config, mirroring real serving. A model whose KV layout the precheck + can't model (see unmodelable_kv_reason) returns the generic + FALLBACK_KV_SHAPE plus a ``simplified`` note, so the network check still + runs with a stand-in pool instead of skipping. + """ + if not model_dir: + return dict(FALLBACK_KV_SHAPE) + try: + with open(os.path.join(model_dir, "config.json")) as f: + hf_cfg = json.load(f) + except (OSError, json.JSONDecodeError): + return dict(FALLBACK_KV_SHAPE) + if isinstance(hf_cfg.get("text_config"), dict): + hf_cfg = hf_cfg["text_config"] + + num_layers = hf_cfg.get("num_hidden_layers") + + reason = unmodelable_kv_reason(hf_cfg) + if reason: + # Simple MLA-flavored stand-in pool: real layer count, one latent + # head, is_mla=True so the transceiver is set up with SELFKONLY + + # AttentionType.MLA exactly like the real sparse model -- only the + # per-token bytes are generic. Enough to check the network path. + latent = int(hf_cfg.get("kv_lora_rank") or 0) + int(hf_cfg.get("qk_rope_head_dim") or 0) + return { + "num_layers": int(num_layers) if num_layers else FALLBACK_KV_SHAPE["num_layers"], + "num_kv_heads": 1, + "head_dim": latent or int(hf_cfg.get("head_dim") or 0) or 576, + "is_mla": True, + "vocab_size": int(hf_cfg.get("vocab_size") or 0) or None, + "source": "config.json (simplified MLA stand-in)", + "simplified": reason, + } + + if num_layers is None: + return dict(FALLBACK_KV_SHAPE) + + vocab_size = int(hf_cfg.get("vocab_size") or 0) or None # V2 ctor input + + if hf_cfg.get("kv_lora_rank"): # MLA (DeepSeek-family, Kimi K2, ...) + return { + "num_layers": int(num_layers), + "num_kv_heads": 1, + "head_dim": int(hf_cfg["kv_lora_rank"]) + int(hf_cfg.get("qk_rope_head_dim", 0)), + "is_mla": True, + "vocab_size": vocab_size, + "source": "config.json (MLA)", + } + + num_heads = hf_cfg.get("num_attention_heads", 1) + num_kv_heads = hf_cfg.get("num_key_value_heads", num_heads) + head_dim = hf_cfg.get("head_dim") + if not head_dim: + hidden = hf_cfg.get("hidden_size") + head_dim = hidden // num_heads if hidden and num_heads else 128 + return { + "num_layers": int(num_layers), + "num_kv_heads": int(num_kv_heads), + "head_dim": int(head_dim), + "is_mla": False, + "vocab_size": vocab_size, + "source": "config.json", + } diff --git a/tests/scripts/perf-sanity/cache_transceiver_precheck/run_precheck.py b/tests/scripts/perf-sanity/cache_transceiver_precheck/run_precheck.py new file mode 100644 index 000000000000..19c6bf70c856 --- /dev/null +++ b/tests/scripts/perf-sanity/cache_transceiver_precheck/run_precheck.py @@ -0,0 +1,1672 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Per-rank driver for the disagg perf-sanity cache-transceiver precheck. + +Runs BEFORE the real disaggregated perf-sanity servers, with the SAME +instance layout (one MPI world per ctx/gen server, same node/GPU topology), +the SAME UCX environment (the launch script reuses the worker env prefix +verbatim), and the SAME `cache_transceiver_config` (built directly from the +test's disagg yaml). It allocates a KV cache shaped like the real model, +transfers deterministic data ctx -> gen through the transceiver for every +(ctx server, gen server) pair, verifies the received bytes, and fails the +stage with a specific error (TIMEOUT / TRANSFER_ERROR / MISMATCH / ...) +before any model is loaded, so network/UCX misconfiguration is caught in +minutes instead of after a full model bring-up. + +Vocabulary: a PAIR is one dp-rank-to-dp-rank transfer pairing (n_pairs = +max of the two sides' dp sizes, so every dp rank is exercised); a REP is one +repetition of all pairs (1 warmup + num_requests measured); a WAVE is the +batch of pairs in flight at once (at most max_concurrent_pairs). "Wave" is +used instead of "chunk" on purpose -- it would otherwise collide with +chunked prefill (splitting a request's tokens), which is unrelated. + +Asymmetric parallelism (e.g. ctx dep4 -> gen dep16, ctx pp8 -> gen tp32) is +supported: the fill pattern is seeded per (request, GLOBAL layer) and is +constant along the KV-head axis, so any TP resharding or PP re-splitting on +the receiving side regenerates the identical expected bytes locally. (This +deliberately cannot detect head-permutation bugs -- it is a network +precheck, not a transceiver-correctness test.) + +Derived from examples/disaggregated/slurm/cache_transceiver_test (the UCX +tuning harness), reduced to a single go/no-go sweep and extended to +asymmetric layouts, attention DP, and multi-instance pairing. + +Rendezvous is file-based under --work-dir (a shared filesystem): each ctx +leader binds one ZMQ REP socket per gen peer and publishes host:port plus a +per-session HMAC key in rendezvous/ctx{ci}_gen{gj}.addr; gen leaders connect +with REQ sockets. Only tiny control payloads travel over ZMQ -- KV data goes +through the transceiver under test. + +Control messages are JSON with an appended HMAC-SHA256 tag -- NEVER pickle: +the REP port is reachable from the cluster network, and unpickling +network-supplied bytes is arbitrary code execution. The key travels only via +the work-dir addr file (filesystem permissions = the job's trust domain), so +a network-only attacker can neither read it nor forge/tamper messages. +ContextPhaseParams crosses the wire as its primitive fields (opaque_state +base64-encoded), mirroring DisaggregatedParams <-> ContextPhaseParams in +tensorrt_llm/disaggregated_params.py and executor/result.py. +""" + +import argparse +import base64 +import hashlib +import hmac +import json +import os +import secrets +import signal +import socket +import sys +import time + +CUR_DIR = os.path.dirname(os.path.abspath(__file__)) +if CUR_DIR not in sys.path: + sys.path.insert(0, CUR_DIR) + +import precheck_config as pcfg # noqa: E402 + +# Request-id scheme: rids must be unique across the whole precheck AND +# dense within a (ctx, gen) session -- the C++ transceiver derives its +# notification tag from the LOW 12 BITS of the request id (tagFromRequestId, +# dataTransceiver.cpp), and notifications are matched by (remote agent, tag). +# A dense per-session sequence keeps tags unique among any 4096 consecutive +# requests of a session; the peer stride only separates sessions, which talk +# to distinct agents and therefore cannot alias tags with each other. +RID_PEER_STRIDE = 1 << 24 + + +class _Timeout(Exception): + pass + + +class _TransferError(Exception): + pass + + +class _PeerAbort(Exception): + pass + + +def _alarm_handler(signum, frame): + raise _Timeout() + + +def make_rid(ctx_idx, gen_idx, num_ctx, seq): + """Unique rid: peer-session base + dense in-session sequence number.""" + peer = gen_idx * num_ctx + ctx_idx + return 1 + peer * RID_PEER_STRIDE + seq + + +def seed_for(rid, global_layer): + # Per (request, GLOBAL layer); rank-independent so any receiving layout + # can regenerate its local slice. + return (rid * 1_000_003 + global_layer * 31) & 0x7FFFFFFF + + +# --------------------------------------------------------------------------- # +# Control-channel wire format: HMAC-SHA256-authenticated JSON (never pickle -- +# the ZMQ port is reachable from the cluster network). +# --------------------------------------------------------------------------- # +_HMAC_TAG_LEN = hashlib.sha256().digest_size + + +def pack_msg(obj, key): + """JSON-encode `obj` and append an HMAC-SHA256 tag.""" + data = json.dumps(obj, separators=(",", ":")).encode() + return data + hmac.new(key, data, hashlib.sha256).digest() + + +def unpack_msg(raw, key): + """Verify the HMAC tag, then JSON-decode. Raises _TransferError on forgery.""" + if len(raw) <= _HMAC_TAG_LEN: + raise _TransferError(f"control frame too short ({len(raw)} bytes)") + data, tag = raw[:-_HMAC_TAG_LEN], raw[-_HMAC_TAG_LEN:] + if not hmac.compare_digest(hmac.new(key, data, hashlib.sha256).digest(), tag): + raise _TransferError("control frame failed HMAC verification (tampered or wrong key)") + return json.loads(data) + + +def params_to_wire(p): + """ContextPhaseParams -> JSON-safe dict (fields per executor/result.py).""" + return { + "first_gen_tokens": list(p.first_gen_tokens or []), + "req_id": p.req_id, + "opaque_state": base64.b64encode(p.opaque_state or b"").decode(), + "draft_tokens": list(p.draft_tokens) if p.draft_tokens is not None else None, + "ctx_dp_rank": p.ctx_dp_rank, + "ctx_info_endpoint": p.disagg_info_endpoint, + } + + +def params_from_wire(d): + """Inverse of params_to_wire. + + Uses the maintained DisaggregatedParams converter (keeps us off the raw + nanobind ctor signature). + """ + return ( + load_internal_apis() + .DisaggregatedParams( + ctx_request_id=int(d["req_id"]), + first_gen_tokens=list(d["first_gen_tokens"]), + opaque_state=base64.b64decode(d["opaque_state"]), + draft_tokens=d["draft_tokens"], + ctx_dp_rank=d["ctx_dp_rank"], + ctx_info_endpoint=d["ctx_info_endpoint"], + ) + .get_context_phase_params() + ) + + +# --------------------------------------------------------------------------- # +# TRT-LLM internal API surface (single owner) +# --------------------------------------------------------------------------- # +_INTERNAL_APIS = None + + +def load_internal_apis(): + """Every tensorrt_llm symbol the precheck touches, imported in ONE place. + + The precheck bypasses the serving stack and drives internal APIs directly + (_torch.pyexecutor.*, bindings.internal.*, private llm_utils resolvers), + none of which carry a stability promise. Centralizing the imports means an + upstream rename breaks exactly here, and the contract test + (tests/unittest/others/test_cache_transceiver_precheck_run.py) fails in + the refactorer's pre-merge CI instead of aborting the SLURM disagg perf + pipeline at runtime. + + Deliberately lazy (NOT at module import): --dry-run and the pure-logic + unit tests must work without torch / tensorrt_llm installed. + """ + global _INTERNAL_APIS + if _INTERNAL_APIS is not None: + return _INTERNAL_APIS + import types + + import tensorrt_llm + import tensorrt_llm._torch.models # noqa: F401 - populates the model registry + import tensorrt_llm.bindings + import tensorrt_llm.bindings.executor as trtllm_executor + from tensorrt_llm import DisaggregatedParams + from tensorrt_llm._torch.distributed import Distributed + from tensorrt_llm._torch.models.modeling_utils import MODEL_CLASS_MAPPING + from tensorrt_llm._torch.pyexecutor.hang_detector import HangDetector + from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2 + from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import create_kv_cache_transceiver + from tensorrt_llm._torch.pyexecutor.llm_request import ( + LlmRequest, + LlmRequestState, + LlmRequestType, + ) + from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager + from tensorrt_llm.llmapi.llm_args import ( + CacheTransceiverConfig, + KvCacheConfig, + MTPDecodingConfig, + ) + from tensorrt_llm.llmapi.llm_utils import ( + _resolve_kv_cache_manager_v2_auto, + _resolve_transceiver_runtime_auto, + ) + from tensorrt_llm.mapping import Mapping + from tensorrt_llm.sampling_params import SamplingParams + + _INTERNAL_APIS = types.SimpleNamespace( + tensorrt_llm=tensorrt_llm, + DataType=tensorrt_llm.bindings.DataType, + SamplingConfigCpp=tensorrt_llm.bindings.SamplingConfig, + CacheTypeCpp=tensorrt_llm.bindings.internal.batch_manager.CacheType, + AttentionTypeCpp=tensorrt_llm.bindings.internal.batch_manager.AttentionType, + KvCacheConfigCpp=trtllm_executor.KvCacheConfig, + DisaggregatedParams=DisaggregatedParams, + Distributed=Distributed, + MODEL_CLASS_MAPPING=MODEL_CLASS_MAPPING, + HangDetector=HangDetector, + KVCacheManager=KVCacheManager, + KVCacheManagerV2=KVCacheManagerV2, + create_kv_cache_transceiver=create_kv_cache_transceiver, + LlmRequest=LlmRequest, + LlmRequestState=LlmRequestState, + LlmRequestType=LlmRequestType, + CacheTransceiverConfig=CacheTransceiverConfig, + KvCacheConfig=KvCacheConfig, + MTPDecodingConfig=MTPDecodingConfig, + resolve_kv_cache_manager_v2_auto=_resolve_kv_cache_manager_v2_auto, + resolve_transceiver_runtime_auto=_resolve_transceiver_runtime_auto, + Mapping=Mapping, + SamplingParams=SamplingParams, + ) + return _INTERNAL_APIS + + +# --------------------------------------------------------------------------- # +# KV fill / verify (heavy imports stay inside functions: --dry-run and unit +# tests must work without torch / tensorrt_llm installed) +# --------------------------------------------------------------------------- # +def _pattern_like(shape, dtype, device, seed): + """Deterministic tensor of `shape`, constant along the head axis. + + `shape` is an HND block slice: [nblocks, kv_factor, heads, tokens, dim]. + Generated on CPU (bit-identical across nodes), head axis generated as 1 + and expanded ON DEVICE (transferring heads x the unique data would be + pure waste), so ctx/gen sides with different local head counts (TP + resharding) or different local layer sets (PP re-splitting) still agree. + """ + import torch + + nb, kv, heads, tok, dim = shape + g = torch.Generator(device="cpu").manual_seed(int(seed)) + rnd = torch.rand((nb, kv, 1, tok, dim), dtype=torch.float32, generator=g) + return rnd.to(dtype).to(device).expand(nb, kv, heads, tok, dim) + + +def _request_block_views(kvm, rid): + """Yield (global_layer, buffer, valid_block_indices) for this rank.""" + for global_layer in kvm.pp_layers: + blocks = kvm.get_batch_cache_indices([rid], layer_idx=global_layer)[0] + valid = [b for b in blocks if b >= 0] + if not valid: + continue + buf = kvm.get_buffers(global_layer, kv_layout="HND") + yield global_layer, buf, valid + + +def fill_request(kvm, rid): + for global_layer, buf, valid in _request_block_views(kvm, rid): + shape = (len(valid), *buf.shape[1:]) + buf[valid] = _pattern_like(shape, buf.dtype, buf.device, seed_for(rid, global_layer)) + + +def verify_request(kvm, rid): + """Returns (ok, detail) comparing received blocks to the expected pattern.""" + import torch + + for global_layer, buf, valid in _request_block_views(kvm, rid): + recv = buf[valid] + exp = _pattern_like(recv.shape, recv.dtype, recv.device, seed_for(rid, global_layer)) + recv_f, exp_f = recv.float(), exp.float() # fp8 lacks direct compare ops + if not torch.equal(recv_f, exp_f): + bad = (recv_f != exp_f).sum().item() + return False, f"layer={global_layer} mismatched_elements={bad}/{recv.numel()}" + return True, "" + + +def _lookup_model_cls(model_dir): + """Model class from config.json architectures, like serving's automodel path.""" + try: + with open(os.path.join(model_dir, "config.json")) as f: + hf_cfg = json.load(f) + except (OSError, json.JSONDecodeError, TypeError): + return None, None + archs = hf_cfg.get("architectures") or [] + hf_view = type("HFConfigView", (), hf_cfg) # attribute access for the pref hook + if not archs: + return None, hf_view + return load_internal_apis().MODEL_CLASS_MAPPING.get(archs[0]), hf_view + + +def resolve_model_prefs(model_dir, side, cache_cfg): + """Mirror serving's model-preference resolution (PR #15823 semantics). + + - use_kv_cache_manager_v2 == "auto" (yaml absent): adopt the model + class's get_model_defaults() value, default False + (llm_utils._resolve_kv_cache_manager_v2_auto). + - cache_cfg.transceiver_runtime == "auto": adopt + model_cls.get_preferred_transceiver_runtime(), NIXL-gated, via the + REAL llm_utils._resolve_transceiver_runtime_auto (mutates cache_cfg). + + Returns the effective use_v2 bool. + """ + import types + + api = load_internal_apis() + model_cls, hf_view = _lookup_model_cls(model_dir) + + # Runtime BEFORE V2, like serving: the V2 resolver's disagg gating reads + # cache_cfg.transceiver_runtime and treats an unresolved "auto" as non-PYTHON. + if getattr(cache_cfg, "transceiver_runtime", None) == "auto": + try: + shim = types.SimpleNamespace(cache_transceiver_config=cache_cfg) + api.resolve_transceiver_runtime_auto(shim, model_cls, hf_view) + except Exception as e: # noqa: BLE001 - fall back to the create() default (CPP) + print( + f"[precheck] WARNING: transceiver_runtime 'auto' resolution failed " + f"({e!r}); create_kv_cache_transceiver will fall back to CPP", + flush=True, + ) + + setting = side["use_kv_cache_manager_v2"] + if setting == "auto": + defaults = {} + if model_cls is not None: + try: + defaults = model_cls.get_model_defaults(None) or {} + except Exception as e: # noqa: BLE001 - model hooks may need llm_args + print( + f"[precheck] WARNING: get_model_defaults failed ({e!r}); assuming V1", + flush=True, + ) + try: + # The REAL serving resolver, via the same shim pattern as the + # runtime resolution below -- one owner for the 'auto' semantics. + # cache_transceiver_config feeds the resolver's disagg gating + # (a V2 model default requires the NIXL Python transceiver). + shim = types.SimpleNamespace( + kv_cache_config=types.SimpleNamespace(use_kv_cache_manager_v2="auto"), + cache_transceiver_config=cache_cfg, + ) + use_v2 = bool(api.resolve_kv_cache_manager_v2_auto(shim, defaults)) + except Exception as e: # noqa: BLE001 - fall back like a missing model + print( + f"[precheck] WARNING: V2 'auto' resolution failed ({e!r}); assuming V1", flush=True + ) + use_v2 = False + else: + use_v2 = bool(setting) + return use_v2 + + +def build_kv_cache_manager(kv_shape, plan, side, mapping, max_req_len, use_v2): + api = load_internal_apis() + dtype_map = { + "fp8": api.DataType.FP8, + "fp16": api.DataType.HALF, + "half": api.DataType.HALF, + "bf16": api.DataType.BF16, + } + dtype_str = side["kv_dtype"].lower() + dtype = dtype_map.get(dtype_str) + if dtype is None: + print(f"[precheck] kv dtype {dtype_str!r} not mapped, using BF16", flush=True) + dtype = api.DataType.BF16 + + spec_config = None + if side["num_nextn_predict_layers"] > 0: + spec_config = api.MTPDecodingConfig( + num_nextn_predict_layers=side["num_nextn_predict_layers"] + ) + + tpb = plan["tokens_per_block"] + padded_len = ((max_req_len + tpb - 1) // tpb) * tpb + owned = pcfg.max_owned_per_wave(plan, side["role"]) + max_tokens = owned * padded_len + 2 * tpb # concurrent pairs + headroom + + # Real MLA serving uses SELFKONLY (kv_factor=1: one latent plane, no V) — + # see _torch/pyexecutor/_util.py; SELF would double the per-token bytes. + cache_type = api.CacheTypeCpp.SELFKONLY if kv_shape["is_mla"] else api.CacheTypeCpp.SELF + common = dict( + num_layers=kv_shape["num_layers"], + num_kv_heads=kv_shape["num_kv_heads"], + head_dim=kv_shape["head_dim"], + tokens_per_block=tpb, + max_seq_len=padded_len, + max_batch_size=max(4, owned + 1), + mapping=mapping, + dtype=dtype, + spec_config=spec_config, + ) + if use_v2: + # The REAL pydantic KvCacheConfig (what serving passes): partial reuse + # is pinned off because its pydantic default is True and block reuse + # is off here. is_disagg=True doubles the IndexMapper capacity so + # in-flight transfers (TRANS_IN_PROGRESS) can hold slots. + return api.KVCacheManagerV2( + api.KvCacheConfig( + max_tokens=max_tokens, + enable_block_reuse=False, + enable_partial_reuse=False, + copy_on_partial_reuse=False, + max_attention_window=[padded_len], + ), + cache_type, + vocab_size=kv_shape.get("vocab_size") or 32000, + is_disagg=True, + **common, + ) + return api.KVCacheManager( + api.KvCacheConfigCpp(max_tokens=max_tokens, enable_block_reuse=False), + cache_type, + **common, + ) + + +def make_request(is_ctx, rid, req_len, runtime, ctx_params=None): + """Build a ctx or gen LlmRequest (mirrors the UCX-tuning harness).""" + api = load_internal_apis() + LlmRequest, LlmRequestType = api.LlmRequest, api.LlmRequestType + DisaggregatedParams = api.DisaggregatedParams + + sampling = api.SamplingParams() + common = dict( + request_id=rid, + max_new_tokens=1, + input_tokens=list(range(req_len)), + sampling_config=api.SamplingConfigCpp(sampling._get_sampling_config()), + is_streaming=False, + ) + if is_ctx: + req = LlmRequest(llm_request_type=LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY, **common) + if runtime == "PYTHON": + req.py_disaggregated_params = DisaggregatedParams( + request_type="context_only", disagg_request_id=rid + ) + return req + if runtime == "PYTHON": + req = LlmRequest(llm_request_type=LlmRequestType.LLMREQUEST_TYPE_GENERATION_ONLY, **common) + req.py_disaggregated_params = DisaggregatedParams( + request_type="generation_only", + disagg_request_id=rid, + ctx_request_id=rid, + ctx_dp_rank=ctx_params.ctx_dp_rank, + ctx_info_endpoint=ctx_params.disagg_info_endpoint, + first_gen_tokens=ctx_params.first_gen_tokens, + draft_tokens=ctx_params.draft_tokens, + ) + return req + return LlmRequest( + llm_request_type=LlmRequestType.LLMREQUEST_TYPE_GENERATION_ONLY, + context_phase_params=ctx_params, + **common, + ) + + +def add_sequence(kvm, req, prompt_len, use_v2): + """Allocate KV blocks (mirrors the cache_transceiver_test harness).""" + if use_v2: + if req.is_disagg_generation_init_state: + ok = kvm.prepare_disagg_gen_init(req) + else: + ok = kvm.prepare_context(req) and kvm.resize_context(req, prompt_len) + if not ok: + raise RuntimeError(f"V2 KV cache allocation failed for request {req.py_request_id}") + return + kvm.impl.add_sequence_batch([(req.py_request_id, prompt_len, 1)], [req]) + + +def free_sequence(kvm, req, use_v2): + if use_v2: + # free_resources() closes the kv_cache AND releases the IndexMapper + # slot (closing the cache alone leaks slots). Callers synchronize the + # stream once per batch before freeing (see _free_all). + kvm.free_resources(req) + return + kvm.impl.remove_sequence(req.py_request_id, req, True) + + +def _wait_gen_complete(xcvr, req, runtime, llm_request_state): + """Block until this gen request's receive finishes (or errors). + + PYTHON transceiver: check_gen_transfer_status(None) blocks for all. C++: + the int API can return before THIS request completes on a cold link, so + poll for a terminal state (bounded by signal.alarm + hang detector). + Logs periodic progress so a stalled transfer shows WHICH request is stuck + in WHICH state (the difference between "requests never matched" and + "RDMA write never completed"). + """ + if runtime == "PYTHON": + xcvr.check_gen_transfer_status(None) + return + terminal = ( + llm_request_state.DISAGG_GENERATION_TRANS_COMPLETE, + llm_request_state.DISAGG_TRANS_ERROR, + ) + t0 = time.monotonic() + next_report = t0 + 15.0 + while req.state not in terminal: + xcvr.check_gen_transfer_status(1) + now = time.monotonic() + if now >= next_report: + print( + f"[precheck] rid={req.py_request_id} recv still waiting: " + f"state={req.state} elapsed={now - t0:.0f}s", + flush=True, + ) + next_report = now + 15.0 + time.sleep(0.001) + + +# --------------------------------------------------------------------------- # +# Rendezvous +# --------------------------------------------------------------------------- # +def addr_path(work_dir, ctx_idx, gen_idx): + return os.path.join(work_dir, "rendezvous", f"ctx{ctx_idx}_gen{gen_idx}.addr") + + +def run_token(): + """Identity of THIS run, stamped into addr files and checked by readers. + + A reused --work-dir (Slurm requeue reruns the batch script with the same + directories; manual reruns) can hold addr files from a previous run -- + connecting to that stale host:port would block until the hello timeout + and misreport TIMEOUT. Within one precheck all instances share + SLURM_JOB_ID; empty (non-Slurm manual runs) disables the check. + """ + return os.environ.get("SLURM_JOB_ID", "") + + +def write_addr(path, payload): + """Atomically publish an addr file. + + os.replace overwrites stale ones; readers reject wrong-job stamps. It + carries the session HMAC key, so restrict it to the owning user before it + becomes visible. + """ + os.makedirs(os.path.dirname(path), exist_ok=True) + payload = dict(payload, job=run_token()) + tmp = f"{path}.tmp.{os.getpid()}" + with open(tmp, "w") as f: + os.fchmod(f.fileno(), 0o600) + json.dump(payload, f) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + + +def wait_for_addr(path, timeout_s): + """Wait for THIS run's addr file. + + Files stamped with another run's job id are treated as stale and skipped + (keep polling). + """ + expect_job = run_token() + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if os.path.exists(path): + try: + with open(path) as f: + payload = json.load(f) + except (OSError, json.JSONDecodeError): + payload = None # mid-rename/NFS staleness; retry + if payload is not None: + stamped = payload.get("job", "") + if not expect_job or not stamped or stamped == expect_job: + return payload + # Stale addr from a previous run in a reused work dir. + time.sleep(1.0) + raise _Timeout(f"rendezvous file {path} not published within {timeout_s}s") + + +def abort_flag_path(work_dir): + return os.path.join(work_dir, "precheck.abort") + + +def raise_abort_flag(work_dir, reason): + """Fail-fast signal, shared across instances through the work dir. + + The first peer failure (in ANY ctx/gen instance) drops this file so the + others stop starting new sessions instead of each re-discovering the dead + fabric on its own. Best-effort and write-once. Stamped with the job id so + a stale flag in a reused work dir (Slurm requeue) is ignored, like addr + files. + """ + path = abort_flag_path(work_dir) + if abort_flag_reason(work_dir) is not None: + return + try: + tmp = f"{path}.tmp.{os.getpid()}" + with open(tmp, "w") as f: + json.dump( + {"reason": (reason or "peer failure").splitlines()[0][:400], "job": run_token()}, + f, + ) + os.replace(tmp, path) + except OSError: + pass # best-effort: a missed flag only costs the usual per-peer timeout + + +def abort_flag_reason(work_dir): + """The reason recorded by the first failing peer of THIS run, or None. + + A flag stamped with another job id is stale (reused work dir) and ignored. + """ + try: + with open(abort_flag_path(work_dir)) as f: + payload = json.load(f) + except (OSError, json.JSONDecodeError): + return None + expect_job = run_token() + stamped = payload.get("job", "") + if expect_job and stamped and stamped != expect_job: + return None + return payload.get("reason") or "peer failure" + + +# --------------------------------------------------------------------------- # +# Status recording +# --------------------------------------------------------------------------- # +class StatusRecorder: + """Leader-side result sink. + + Rewritten on every record so a SIGKILL'd run still leaves the completed + cases + the in-flight failure on disk. + """ + + def __init__(self, work_dir, role, server_idx, is_leader): + self.role = role + self.server_idx = server_idx + self.is_leader = is_leader + self.cases = [] + status_dir = os.path.join(work_dir, "status") + if is_leader: + os.makedirs(status_dir, exist_ok=True) + self.json_path = os.path.join(status_dir, f"{role}_{server_idx}.json") + self.text_path = os.path.join(status_dir, f"{role}_{server_idx}.status") + # NIXL_* is deliberately absent: the only such variable seen in + # practice is NIXL_VERSION, a stale marker from the NGC base image's + # bundled NIXL (not the library TRT-LLM links from + # /opt/nvidia/nvda_nixl), which misstates the transport version. + env_keys = ("UCX_", "TRTLLM_", "TLLM_") + self.env = {k: v for k, v in sorted(os.environ.items()) if k.startswith(env_keys)} + + def record(self, peer, req_len, status, reason=""): + if not self.is_leader: + return + self.cases.append({"peer": peer, "req_len": req_len, "status": status, "reason": reason}) + self._flush(final=False) + + def finalize(self, extra=None): + if not self.is_leader: + return + self._flush(final=True, extra=extra) + + def failed_cases(self): + return [c for c in self.cases if c["status"] not in ("PASS", "SKIP")] + + def _flush(self, final, extra=None): + failed = self.failed_cases() + overall = "PASS" if (final and not failed) else ("FAIL" if failed else "RUNNING") + doc = { + "role": self.role, + "server_idx": self.server_idx, + "overall": overall, + "cases": self.cases, + "env": self.env, + } + if extra: + doc.update(extra) + tmp = f"{self.json_path}.tmp" + with open(tmp, "w") as f: + json.dump(doc, f, indent=2) + os.replace(tmp, self.json_path) + with open(f"{self.text_path}.tmp", "w") as f: + if failed: + first = failed[0] + # One-line summary for the launch-script console output: root + # cause only (first line, bounded); the full reason incl. any + # backtrace stays in the .json. + reason = " | ".join(first["reason"].splitlines()[:2])[:400] + f.write( + f"FAIL {self.role}_{self.server_idx}: {len(failed)} case(s) failed; " + f"first: peer={first['peer']} req_len={first['req_len']} " + f"{first['status']}: {reason}\n" + ) + elif final: + f.write(f"PASS {self.role}_{self.server_idx}: {len(self.cases)} case(s)\n") + else: + f.write(f"RUNNING {self.role}_{self.server_idx}\n") + os.replace(f"{self.text_path}.tmp", self.text_path) + + +def parse_bandwidth_gbps(csv_dir, rank, tag="recv"): + """Median per-request bandwidth in GB/s (bytes/1e9), best-effort. + + Parsed from the C++ transceiver CSV this rank wrote via + TRTLLM_KVCACHE_TIME_OUTPUT_PATH, named "__.csv" + (instanceId is a runtime UUID), so match by the "__.csv" + suffix. The suffix's leading "_" keeps rank 1 from matching rank 11. + + Each row repeats the Bandwidth(Gbps) column once per transmission, so use + csv.reader (DictReader would keep only the last duplicate) and take the + mean transmission bandwidth as that request's value, then the median + across requests -- same semantics as the harness report. + """ + import csv as csv_mod + import statistics + + suffix = f"_{rank}_{tag}.csv" + try: + names = [n for n in os.listdir(csv_dir) if n.endswith(suffix)] + except OSError: + return None + vals = [] + for name in names: + try: + with open(os.path.join(csv_dir, name)) as f: + reader = csv_mod.reader(f) + header = next(reader, None) + if not header: + continue + bw_cols = [i for i, c in enumerate(header) if "Bandwidth" in c] + if not bw_cols: + continue + for row in reader: + bws = [] + for i in bw_cols: + if i < len(row) and row[i]: + try: + bws.append(float(row[i]) / 8.0) # Gbps -> GB/s + except ValueError: + pass + if bws: + vals.append(sum(bws) / len(bws)) + except OSError: + continue + return statistics.median(vals) if vals else None + + +def parse_python_bandwidth_gbps(csv_dir): + """Median KV-send throughput in GB/s from the Python transceiver's perf CSVs. + + Written by perf_logger.py, which gives TRTLLM_KVCACHE_TIME_OUTPUT_PATH top + priority and names files "{dir}/{instanceUuid}_{rank}.csv" -- so identify + the CSVs by their header columns (task_type + throughput_mbs) rather than + by name; C++ send/recv CSVs have neither column. throughput_mbs (MiB/s) is + on the SENDER (ctx) side, task_type=KVSendTask; receiver rows have no + throughput. Best-effort -- returns None when no perf CSV exists. + """ + import csv as csv_mod + import glob + import statistics + + vals = [] + for path in glob.glob(os.path.join(csv_dir, "*.csv")): + try: + with open(path) as f: + reader = csv_mod.DictReader(f) + fields = reader.fieldnames or [] + if "task_type" not in fields or "throughput_mbs" not in fields: + continue + for r in reader: + if r.get("task_type") == "KVSendTask" and r.get("throughput_mbs"): + # MiB/s -> GB/s + vals.append(float(r["throughput_mbs"]) * 1024.0 * 1024.0 / 1e9) + except (OSError, ValueError): + continue + return statistics.median(vals) if vals else None + + +# --------------------------------------------------------------------------- # +# Wave execution +# --------------------------------------------------------------------------- # +class PrecheckRunner: + """One MPI world = one ctx or gen server instance.""" + + def __init__(self, args, plan, side, comm): + from mpi4py import MPI # noqa: F401 - ensures MPI initialized + + self.plan = plan + self.side = side + self.role = side["role"] + self.is_ctx = self.role == "ctx" + self.server_idx = args.server_idx + self.comm = comm + self.rank = comm.Get_rank() + self.is_leader = self.rank == 0 + self.work_dir = args.work_dir + self.recorder = StatusRecorder(self.work_dir, self.role, self.server_idx, self.is_leader) + self.zmq_ctx = None + self.kvm = None + self.xcvr = None + self.runtime = "CPP" + # Resolved in setup(): "auto" needs the model class (get_model_defaults). + self.use_v2 = False + self.mapping = None + self.llm_request_state = None + self.csv_dir = os.path.join(self.work_dir, "csv", f"{self.role}_{self.server_idx}") + + # ---- consensus helpers ------------------------------------------------- + def _consensus_error(self, local_err): + """All ranks agree whether anyone failed; returns the shared reason.""" + errs = self.comm.allgather("" if local_err is None else repr(local_err)) + bad = [(r, e) for r, e in enumerate(errs) if e] + if not bad: + return None + ranks = [r for r, _ in bad] + return f"rank(s) {ranks}: {bad[0][1]}" + + # ---- setup ------------------------------------------------------------- + def setup(self, kv_shape, max_req_len): + api = load_internal_apis() + + self.llm_request_state = api.LlmRequestState + par = self.side["parallel"] + self.mapping = api.Mapping( + world_size=par["world_size"], + rank=self.rank, + gpus_per_node=self.plan["gpus_per_node"], + tp_size=par["tp"], + pp_size=par["pp"], + cp_size=par["cp"], + enable_attention_dp=par["enable_attention_dp"], + ) + os.makedirs(self.csv_dir, exist_ok=True) + # One env var drives both transceivers' bandwidth CSVs: C++ writes + # per-rank "__send/recv.csv", and Python's + # PerfLogManager gives the same var top priority, writing task CSVs + # as "_.csv" (KVSendTask throughput on the ctx + # side). + os.environ["TRTLLM_KVCACHE_TIME_OUTPUT_PATH"] = self.csv_dir + + # Built VERBATIM from the disagg yaml's cache_transceiver_config so + # backend/max_tokens_in_buffer/timeouts match the real test exactly. + cache_cfg = api.CacheTransceiverConfig(**self.side["cache_transceiver_config"]) + # Yaml-absent settings resolve against the model's preferences, like + # serving does (kv manager version + transceiver runtime) -- this holds + # even for the simplified stand-in pool: only the KV SHAPE is generic; + # the V1/V2 manager version and the transceiver runtime must still + # match what the real model runs (e.g. V4 -> V2 + Python). + self.use_v2 = resolve_model_prefs(self.plan.get("_model_dir"), self.side, cache_cfg) + if kv_shape.get("simplified") and self.is_leader: + print( + f"[precheck {self.role}_{self.server_idx}] SIMPLIFIED: {kv_shape['simplified']}", + flush=True, + ) + # KVCacheManagerV2 only works with the Python transceiver (see + # cache_transceiver_test/report.py); reject the pairing up front with + # a clear INIT_ERROR instead of a C++ binding type error. + if self.use_v2 and cache_cfg.transceiver_runtime != "PYTHON": + raise RuntimeError( + "KVCacheManagerV2 requires cache_transceiver_config." + f"transceiver_runtime: PYTHON, got {cache_cfg.transceiver_runtime!r} " + "(the C++ transceiver only supports the V1 manager)" + ) + + self.kvm = build_kv_cache_manager( + kv_shape, self.plan, self.side, self.mapping, max_req_len, self.use_v2 + ) + AttentionTypeCpp = api.AttentionTypeCpp + attention_type = AttentionTypeCpp.MLA if kv_shape["is_mla"] else AttentionTypeCpp.DEFAULT + dist_obj = api.Distributed.get(self.mapping) + self.xcvr = api.create_kv_cache_transceiver( + self.mapping, dist_obj, self.kvm, attention_type, cache_cfg + ) + if self.xcvr is None: + raise RuntimeError("cache transceiver disabled by config") + # create_kv_cache_transceiver resolves 'auto' in-place (no model + # preference on this path -> C++), so read the effective runtime back. + self.runtime = cache_cfg.transceiver_runtime or "CPP" + + # ---- per-wave transfer logic ------------------------------------------- + def _pair_rid(self, peer_idx, li, rep, pair): + ctx_idx = self.server_idx if self.is_ctx else peer_idx + gen_idx = peer_idx if self.is_ctx else self.server_idx + total_reps = self.plan["warmup_requests"] + self.plan["num_requests"] + seq = (li * total_reps + rep) * self.plan["n_pairs"] + pair + return make_rid(ctx_idx, gen_idx, self.plan["num_ctx_servers"], seq) + + def _owned(self, wave): + return pcfg.owned_pairs(self.plan, self.role, self.mapping.tp_rank, wave) + + def ctx_run_wave(self, peer_idx, li, req_len, rep, wave): + """Fill + send owned pairs. + + Returns {pair: context_phase_params} on the leader (params from each + pair's owning dp rank, pp stage 0). + """ + import tensorrt_llm + + owned = self._owned(wave) + reqs, local_err = {}, None + try: + for pair in owned: + rid = self._pair_rid(peer_idx, li, rep, pair) + req = make_request(True, rid, req_len, self.runtime) + add_sequence(self.kvm, req, req_len, self.use_v2) + fill_request(self.kvm, rid) + tensorrt_llm.logger.info( + f"[ctx{self.server_idx} r{self.rank}] rid={rid} len={req_len}: send START" + ) + self.xcvr.respond_and_send_async(req) + reqs[pair] = req + except Exception as e: # noqa: BLE001 - relayed to gen, then raised + local_err = e + reason = self._consensus_error(local_err) + + # Params for pair k come from its owning dp rank at pp stage 0 with + # attention DP; without DP every rank sends the same request, and the + # instance leader's params are the ones the real server would return. + if self.side["parallel"]["enable_attention_dp"]: + contributes = self.mapping.pp_rank == 0 + else: + contributes = self.is_leader + contrib = ( + {p: r.context_phase_params for p, r in reqs.items()} + if local_err is None and contributes + else {} + ) + gathered = self.comm.gather(contrib, root=0) + params_by_pair = {} + if self.is_leader: + for d in gathered: + params_by_pair.update(d or {}) + if reason is None: + missing = [p for p in wave if p not in params_by_pair] + if missing: + reason = f"missing context_phase_params for pairs {missing}" + # The missing-params check runs only on the leader (only it holds the + # gathered params). Broadcast the verdict so EVERY rank raises together: + # otherwise the leader raises here while the other ranks return and enter + # ctx_finish_wave's collective, the collective sequence diverges, and the + # step deadlocks until the watchdog SIGKILLs it (misreported as TIMEOUT). + reason = self.comm.bcast(reason, root=0) + + if reason is not None: + self._free_all(reqs) + raise _TransferError(f"ctx send setup failed: {reason}") + return params_by_pair, reqs + + def ctx_finish_wave(self, reqs): + """Wait for all in-flight sends of this wave, then free.""" + import tensorrt_llm + + t0 = time.monotonic() + local_err = None + try: + self.xcvr.check_context_transfer_status(None) # block-all + bad = [ + p for p, r in reqs.items() if r.state == self.llm_request_state.DISAGG_TRANS_ERROR + ] + if bad: + local_err = _TransferError(f"ctx DISAGG_TRANS_ERROR on pairs {bad}") + except Exception as e: # noqa: BLE001 + local_err = e + finally: + self._free_all(reqs) + states = {p: str(r.state) for p, r in reqs.items()} + tensorrt_llm.logger.info( + f"[ctx{self.server_idx} r{self.rank}] wave sends finished in " + f"{time.monotonic() - t0:.1f}s states={states}" + ) + reason = self._consensus_error(local_err) + if reason is not None: + raise _TransferError(f"ctx transfer failed: {reason}") + + def gen_run_wave(self, peer_idx, li, req_len, rep, wave, params_by_pair): + """Receive + verify owned pairs. Returns (ok, mismatch_detail). + + Warmup reps skip the (CPU-heavy) byte verification: their result is + discarded by the caller either way, transfer errors still raise. + """ + import torch + + import tensorrt_llm + + owned = self._owned(wave) + reqs, local_err = {}, None + try: + for pair in owned: + rid = self._pair_rid(peer_idx, li, rep, pair) + req = make_request( + False, rid, req_len, self.runtime, ctx_params=params_by_pair[pair] + ) + add_sequence(self.kvm, req, req_len, self.use_v2) + tensorrt_llm.logger.info( + f"[gen{self.server_idx} r{self.rank}] rid={rid} len={req_len}: recv START" + ) + self.xcvr.request_and_receive_async(req) + reqs[pair] = req + except Exception as e: # noqa: BLE001 + local_err = e + reason = self._consensus_error(local_err) + if reason is not None: + self._free_all(reqs) + raise _TransferError(f"gen receive setup failed: {reason}") + + mismatch = "" + t0 = time.monotonic() + try: + for pair, req in reqs.items(): + _wait_gen_complete(self.xcvr, req, self.runtime, self.llm_request_state) + if reqs: + tensorrt_llm.logger.info( + f"[gen{self.server_idx} r{self.rank}] wave recvs finished in " + f"{time.monotonic() - t0:.1f}s" + ) + torch.cuda.synchronize() # receive may land on a side stream + bad = [ + p for p, r in reqs.items() if r.state == self.llm_request_state.DISAGG_TRANS_ERROR + ] + if bad: + local_err = _TransferError(f"gen DISAGG_TRANS_ERROR on pairs {bad}") + elif self.plan["verify_data"] and rep >= self.plan["warmup_requests"]: + for pair, req in reqs.items(): + ok, detail = verify_request(self.kvm, req.py_request_id) + if not ok: + mismatch = f"pair={pair} {detail}" + break + except Exception as e: # noqa: BLE001 + local_err = e + finally: + self._free_all(reqs) + reason = self._consensus_error(local_err) + if reason is not None: + raise _TransferError(f"gen transfer failed: {reason}") + mismatches = [m for m in self.comm.allgather(mismatch) if m] + return (not mismatches, "; ".join(mismatches[:4])) + + def _free_all(self, reqs): + if reqs and self.use_v2: + import torch + + torch.cuda.current_stream().synchronize() # V2 frees need quiesced stream + for req in reqs.values(): + try: + free_sequence(self.kvm, req, self.use_v2) + except Exception: # noqa: BLE001 - teardown best-effort + pass + + # ---- ZMQ helpers --------------------------------------------------------- + def _zmq(self): + import zmq + + if self.zmq_ctx is None: + self.zmq_ctx = zmq.Context.instance() + return zmq, self.zmq_ctx + + def _leader_send_recv(self, sock, obj, key): + """REQ round-trip on the gen leader; broadcast the reply to all ranks.""" + reply = None + err = None + if self.is_leader: + try: + sock.send(pack_msg(obj, key)) + reply = unpack_msg(sock.recv(), key) + except Exception as e: # noqa: BLE001 + err = repr(e) + err, reply = self.comm.bcast((err, reply), root=0) + if err: + raise _TransferError(f"ZMQ control channel failed: {err}") + return reply + + +def _schedule(plan): + """Deterministic (li, req_len, rep, wave) schedule both sides iterate.""" + out = [] + total_reps = plan["warmup_requests"] + plan["num_requests"] + for li, req_len in enumerate(plan["request_lengths"]): + for rep in range(total_reps): + for wave in pcfg.waves(plan): + out.append((li, req_len, rep, wave)) + return out + + +def hello_timeout_s(plan, num_peers): + """Timeout budget for session handshakes. + + Handshakes are serialized across peers (one gen talks to one ctx at a + time), so waiting for a peer's hello/welcome can legitimately span other + peers' full sessions -- budget rendezvous + per-peer slack (including + the peer's first-rep wire-up). + """ + return plan["rendezvous_timeout_s"] + num_peers * (300 + plan["wireup_timeout_s"]) + + +def wave_timeout_s(plan, li, rep): + """Per-wave budget. + + The first rep additionally pays the one-time NIXL agent wire-up (see + PRECHECK_DEFAULTS['wireup_timeout_s']). + """ + extra = plan["wireup_timeout_s"] if (li == 0 and rep == 0) else 0 + return plan["wave_timeout_s"] + extra + + +# --------------------------------------------------------------------------- # +# ctx / gen session loops +# +# Per (ctx, gen) pair, the leaders speak a lockstep REQ/REP protocol (all +# frames HMAC-JSON; KV bytes themselves go through the transceiver under +# test, never over ZMQ): +# +# gen leader ctx leader +# | -- hello {fingerprint} --------------> | yaml mismatch -> abort +# | <---------------- welcome ------------ | +# | -- go {li, rep, wave} --------------> | every rank posts its sends +# | <----- params {pair: ctx_phase} ------ | (from the owning dp ranks) +# | ...KV transfer + byte verification.. | +# | (repeat per schedule entry) | +# | -- done (deferred: after ALL peers) -> | ctx exits only now +# | <------------------ bye -------------- | +# +# A "wave" here is a batch of TRANSFER PAIRS in flight at once (at most +# max_concurrent_pairs of the n_pairs dp-rank pairings) -- NOT token/data +# chunking. It bounds the tiny synthetic KV pool and gives the per-wave +# alarm a precise target, while still exercising concurrent transfers. +# --------------------------------------------------------------------------- # +def ctx_serve_peer(runner, sock, peer_idx, arm, disarm, key): + """Serve one gen peer's full schedule on a dedicated REP socket.""" + plan = runner.plan + comm = runner.comm + + def leader_recv(): + msg, err = None, None + if runner.is_leader: + try: + msg = unpack_msg(sock.recv(), key) + except Exception as e: # noqa: BLE001 + err = repr(e) + err, msg = comm.bcast((err, msg), root=0) + if err: + raise _TransferError(f"ZMQ recv from gen_{peer_idx} failed: {err}") + return msg + + def leader_reply(obj): + if runner.is_leader: + sock.send(pack_msg(obj, key)) + + arm(f"hello gen_{peer_idx}", seconds=hello_timeout_s(plan, runner.side["num_peers"])) + msg = leader_recv() + if msg[0] != "hello" or msg[1].get("fingerprint") != plan["fingerprint"]: + leader_reply(("abort", "plan fingerprint mismatch (ctx/gen yaml disagree)")) + raise _TransferError(f"handshake with gen_{peer_idx} failed: {msg[:1]}") + leader_reply(("welcome", {"fingerprint": plan["fingerprint"]})) + + for li, req_len, rep, wave in _schedule(plan): + arm(f"gen_{peer_idx} len={req_len} rep={rep}", seconds=wave_timeout_s(plan, li, rep)) + msg = leader_recv() + if msg[0] == "abort": + # Ack so the gen's REQ send/recv completes (fail-fast teardown + # sends this in place of the schedule; see gen_abort_peer). + leader_reply(("aborted", {})) + raise _PeerAbort(f"gen_{peer_idx} aborted: {msg[1]}") + if msg[0] != "go" or (msg[1]["li"], msg[1]["rep"]) != (li, rep): + raise _TransferError( + f"schedule desync with gen_{peer_idx}: expected li={li} rep={rep}, got {msg}" + ) + try: + params_by_pair, reqs = runner.ctx_run_wave(peer_idx, li, req_len, rep, wave) + except _TransferError as e: + leader_reply(("abort", str(e))) + raise + # JSON object keys are strings; the gen side converts back to int. + leader_reply(("params", {str(p): params_to_wire(v) for p, v in params_by_pair.items()})) + runner.ctx_finish_wave(reqs) + + # The gen defers "done" until it has finished the schedules of ALL its + # ctx peers, so every ctx instance stays alive for the whole precheck -- + # matching real serving, where ctx servers outlive the entire run. (An + # early-exiting ctx leaves the gen's C++ transceiver holding connections + # to a dead agent, a state the real test never produces.) The wait can + # therefore span the gen's remaining sessions: budget like a handshake. + arm(f"bye gen_{peer_idx}", seconds=hello_timeout_s(plan, runner.side["num_peers"])) + msg = leader_recv() + if msg[0] != "done": + raise _TransferError(f"expected done from gen_{peer_idx}, got {msg[:1]}") + leader_reply(("bye", {})) + disarm() + + +def _gen_open_session(runner, peer_idx, arm): + """Rendezvous with ctx server `peer_idx`; return (sock, key) on an OPEN REQ socket. + + Completes the hello/welcome handshake. Shared by gen_run_peer (runs the schedule) and gen_abort_peer (sends an + early abort for fail-fast). Rendezvous reaches instance-wide consensus via + bcast: if only the leader raised, the other ranks would deadlock in the + next bcast. The socket is closed here if the handshake itself fails. + """ + plan = runner.plan + comm = runner.comm + hello_s = hello_timeout_s(plan, runner.side["num_peers"]) + + sock, key, err = None, None, None + arm(f"rendezvous ctx_{peer_idx}", seconds=hello_s) + if runner.is_leader: + try: + addr = wait_for_addr( + addr_path(runner.work_dir, peer_idx, runner.server_idx), + plan["rendezvous_timeout_s"], + ) + # Session HMAC key: shared only through the work-dir addr file, + # never over the network. + key = bytes.fromhex(addr["key"]) + zmq, zctx = runner._zmq() + sock = zctx.socket(zmq.REQ) + sock.setsockopt(zmq.LINGER, 0) + sock.setsockopt(zmq.RCVTIMEO, hello_s * 1000) + sock.connect(f"tcp://{addr['host']}:{addr['port']}") + except Exception as e: # noqa: BLE001 - shared via bcast below + err = repr(e) + err = comm.bcast(err, root=0) + if err: + raise _TransferError(f"rendezvous with ctx_{peer_idx} failed: {err}") + + try: + arm(f"hello ctx_{peer_idx}", seconds=hello_s) + reply = runner._leader_send_recv( + sock, + ("hello", {"gen_idx": runner.server_idx, "fingerprint": plan["fingerprint"]}), + key, + ) + if reply[0] == "abort": + raise _TransferError(f"ctx_{peer_idx} aborted handshake: {reply[1]}") + if reply[0] != "welcome": + raise _TransferError(f"unexpected handshake reply from ctx_{peer_idx}: {reply[:1]}") + return sock, key + except BaseException: + if sock is not None: + sock.close(linger=0) + raise + + +def gen_run_peer(runner, peer_idx, arm, disarm): + """Run the full schedule against ctx server `peer_idx`. + + Returns (sock, key) with the session STILL OPEN on success -- the caller + sends the deferred "done" only after every ctx peer's schedule finished, + keeping all ctx instances alive for the whole precheck (real-serving + lifecycle; see ctx_serve_peer). On failure the socket is closed here. + """ + plan = runner.plan + sock, key = _gen_open_session(runner, peer_idx, arm) + try: + # Established sessions are dedicated: wave replies are prompt. The + # ZMQ timeout is only a backstop under the per-wave alarm, so it + # includes the first-rep wire-up allowance unconditionally. + if runner.is_leader: + zmq, _ = runner._zmq() + sock.setsockopt( + zmq.RCVTIMEO, + (plan["wave_timeout_s"] + plan["wireup_timeout_s"] + 30) * 1000, + ) + + case_ok = {} + for li, req_len, rep, wave in _schedule(plan): + arm(f"ctx_{peer_idx} len={req_len} rep={rep}", seconds=wave_timeout_s(plan, li, rep)) + reply = runner._leader_send_recv( + sock, ("go", {"li": li, "rep": rep, "wave": wave[0]}), key + ) + if reply[0] == "abort": + raise _TransferError(f"ctx_{peer_idx} aborted: {reply[1]}") + params_by_pair = {int(p): params_from_wire(v) for p, v in reply[1].items()} + ok, detail = runner.gen_run_wave(peer_idx, li, req_len, rep, wave, params_by_pair) + if rep >= plan["warmup_requests"]: + prev_ok, prev_detail = case_ok.get(req_len, (True, "")) + case_ok[req_len] = (prev_ok and ok, prev_detail or detail) + disarm() + + for req_len, (ok, detail) in case_ok.items(): + runner.recorder.record( + f"ctx_{peer_idx}", + req_len, + "PASS" if ok else "MISMATCH", + "" if ok else detail, + ) + return sock, key + except BaseException: + if sock is not None: + sock.close(linger=0) + raise + + +def gen_abort_peer(runner, peer_idx, reason, arm, disarm): + """Fail-fast teardown of a not-yet-run ctx peer. + + Open the session and tell it to abort (it is blocked awaiting our hello), + so it stops promptly instead of waiting out the handshake alarm. + Best-effort; always closes. + """ + sock = None + try: + sock, key = _gen_open_session(runner, peer_idx, arm) + arm(f"abort ctx_{peer_idx}", seconds=hello_timeout_s(runner.plan, runner.side["num_peers"])) + runner._leader_send_recv(sock, ("abort", f"peer fail-fast: {reason}"), key) + disarm() + finally: + if sock is not None: + sock.close(linger=0) + + +def gen_release_peer(runner, peer_idx, sock, key, arm, disarm): + """Deferred session teardown: send "done" and close (best-effort).""" + try: + arm(f"bye ctx_{peer_idx}") + runner._leader_send_recv(sock, ("done", {}), key) + disarm() + finally: + if sock is not None: + sock.close(linger=0) + + +# --------------------------------------------------------------------------- # +# main +# --------------------------------------------------------------------------- # +def parse_args(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--role", required=True, choices=["ctx", "gen"]) + ap.add_argument("--server-idx", type=int, required=True) + ap.add_argument("--config", required=True, help="disagg perf-sanity yaml path") + ap.add_argument("--work-dir", required=True, help="shared dir for rendezvous/status") + ap.add_argument("--benchmark-mode", default="e2e", choices=["e2e", "gen_only"]) + ap.add_argument("--llm-src", default="", help="repo root (model path dict lookup)") + ap.add_argument("--dry-run", action="store_true", help="print the resolved plan and exit") + return ap.parse_args(argv) + + +def load_plan(args): + import yaml + + with open(args.config) as f: + cfg = yaml.safe_load(f) + plan = pcfg.resolve_plan(cfg, benchmark_mode=args.benchmark_mode) + if plan.get("skip"): + return plan, None + model_dir = pcfg.resolve_model_dir(cfg, llm_src=args.llm_src or None) + role_side = pcfg.side_plan(plan, args.role) if not args.dry_run else None + kv_shape = pcfg.model_kv_shape(model_dir) + plan["_kv_shape"] = kv_shape + plan["_model_dir"] = model_dir + return plan, role_side + + +def _install_watchdog(runner, plan, rank): + """Two-layer stall protection around every phase of the run. + + signal.alarm catches Python-level stalls; HangDetector catches + GIL-released native hangs (dumps stacks, records TIMEOUT, SIGKILLs so + `srun --kill-on-bad-exit` tears the step down). The external `timeout` + around the srun is the guaranteed backstop for GIL-held hangs. + + Returns (arm, disarm, stop, current_cell): per-phase alarm control, the + final shutdown, and the mutable "what phase are we in" marker used by + failure messages. + """ + signal.signal(signal.SIGALRM, _alarm_handler) + current_cell = {"what": "startup"} + + def _on_hang(): + runner.recorder.record("-", 0, "TIMEOUT", f"hang detected during {current_cell['what']}") + runner.recorder.finalize() + sys.stderr.write( + f"[precheck {runner.role}_{runner.server_idx} r{rank}] WATCHDOG_KILL " + f"{current_cell['what']}\n" + ) + sys.stderr.flush() + os.kill(os.getpid(), signal.SIGKILL) + + # The detector must outlast the LONGEST legitimate wait (peer handshakes + # are serialized across sessions); per-cell alarms are the tighter bound + # for actual transfer work. + hang_detector = load_internal_apis().HangDetector( + timeout=hello_timeout_s(plan, runner.side["num_peers"]) + plan["wave_timeout_s"] + 60, + on_detected=_on_hang, + ) + hang_detector.start() + + def arm(what, seconds=None): + current_cell["what"] = what + signal.alarm(seconds or plan["wave_timeout_s"]) + hang_detector.checkpoint() + + def disarm(): + signal.alarm(0) + hang_detector.cancel_task() + + def stop(): + disarm() + hang_detector.stop() + + return arm, disarm, stop, current_cell + + +def _make_peer_failure_recorder(runner, disarm, current_cell): + """Exception -> verdict mapping shared by all per-peer loops. + + Recording a failure also drops the fail-fast flag so the remaining peers + (here and in the other instances) are skipped instead of tested against a + fabric already known bad -- see _drive_ctx_peers / raise_abort_flag. + """ + + def record_peer_failure(peer, exc): + disarm() + if isinstance(exc, _Timeout): + status, reason = "TIMEOUT", f"exceeded the budget during {current_cell['what']}" + elif isinstance(exc, _PeerAbort): + status, reason = "TRANSFER_ERROR", str(exc) + else: + status, reason = "TRANSFER_ERROR", repr(exc) + runner.recorder.record(peer, 0, status, reason) + raise_abort_flag(runner.work_dir, f"{peer} {status}: {reason}") + + return record_peer_failure + + +def _consensus_abort_reason(runner): + """Instance-wide agreed view of the fail-fast flag (leader reads, bcast). + + The flag file can appear at any moment (any instance's failure drops it), + so per-rank reads can race it and disagree -- and the branches they select + (gen_run_peer vs gen_abort_peer, failure vs SKIP verdict) issue different + MPI collective sequences, deadlocking or cross-pairing the instance. Must + be called collectively by every rank. + """ + reason = abort_flag_reason(runner.work_dir) if runner.is_leader else None + return runner.comm.bcast(reason, root=0) + + +def _serve_gen_peers(runner, plan, arm, disarm, record_peer_failure): + """Ctx role: bind per-peer REP sockets, publish addrs, serve each schedule. + + One dedicated REP socket per gen peer avoids REQ interleaving across + sessions on a shared socket. Each session gets a fresh HMAC key, shared + only through the work-dir addr file (0600). + """ + num_peers = runner.side["num_peers"] + socks, keys = {}, {} + if runner.is_leader: + zmq, zctx = runner._zmq() + host = os.environ.get("SLURMD_NODENAME") or socket.gethostname() + for gj in range(num_peers): + s = zctx.socket(zmq.REP) + s.setsockopt(zmq.LINGER, 0) + # Generous: gen peers are serialized across ctx servers. + s.setsockopt(zmq.RCVTIMEO, hello_timeout_s(plan, num_peers) * 1000) + port = s.bind_to_random_port("tcp://*") + keys[gj] = secrets.token_bytes(32) + write_addr( + addr_path(runner.work_dir, runner.server_idx, gj), + {"host": host, "port": port, "key": keys[gj].hex()}, + ) + socks[gj] = s + for gj in range(num_peers): + try: + ctx_serve_peer(runner, socks.get(gj), gj, arm, disarm, keys.get(gj)) + runner.recorder.record(f"gen_{gj}", 0, "PASS", "served all transfers") + except _PeerAbort as e: + # A gen driver that failed elsewhere aborts our session as part of + # fail-fast: record a (non-failing) SKIP, not our own failure -- + # the real failure is recorded by whoever hit it. Absent the flag, + # a genuine peer abort is still a real failure. The consensus read + # is collectively safe here: _PeerAbort is only raised after a + # bcast (leader_recv), so every rank reaches this handler. + if _consensus_abort_reason(runner) is not None: + runner.recorder.record(f"gen_{gj}", 0, "SKIP", f"aborted by fail-fast: {e}") + else: + record_peer_failure(f"gen_{gj}", e) + except Exception as e: # noqa: BLE001 - per-peer isolation + record_peer_failure(f"gen_{gj}", e) + + +def _drive_ctx_peers(runner, arm, disarm, record_peer_failure): + """Gen role: run every ctx peer's schedule, then release all sessions. + + Fail-fast: once any pair has failed (this instance or another -- signalled + through the work-dir abort flag), the remaining ctx peers are not tested; + each is told to abort so it tears down promptly rather than waiting out its + handshake alarm. Sessions that already succeeded still get their deferred + "done" (below) so those ctx instances shut down cleanly. + + The release ("done") is deferred until EVERY driven peer's schedule + finished, so those ctx instances stay alive for the whole precheck -- + matching real serving, where no transceiver ever holds connections to a + dead agent while transfers are still running. + """ + open_sessions = [] + for ci in range(runner.side["num_peers"]): + reason = _consensus_abort_reason(runner) + if reason is not None: + try: + gen_abort_peer(runner, ci, reason, arm, disarm) + except Exception: # noqa: BLE001 - best-effort teardown + pass + runner.recorder.record(f"ctx_{ci}", 0, "SKIP", f"fail-fast: {reason}") + continue + try: + sock, sess_key = gen_run_peer(runner, ci, arm, disarm) + open_sessions.append((ci, sock, sess_key)) + except Exception as e: # noqa: BLE001 - failure sets the fail-fast flag + record_peer_failure(f"ctx_{ci}", e) + for ci, sock, sess_key in open_sessions: + try: + gen_release_peer(runner, ci, sock, sess_key, arm, disarm) + except Exception as e: # noqa: BLE001 - best-effort release + record_peer_failure(f"ctx_{ci}", e) + + +def main(argv=None): + args = parse_args(argv) + plan, side = load_plan(args) + + if args.dry_run: + print(json.dumps(plan, indent=2, default=str)) + return 0 + if plan.get("skip"): + print(f"[precheck] SKIP: {plan['skip_reason']}", flush=True) + return 0 + + # UCX_PROTO_INFO=used is log-only (does not change transport selection): + # it makes UCX >= 1.21 print the chosen GPU<->GPU protocol table, which the + # failure summary uses to spot host-staged tcp fallbacks. + os.environ.setdefault("UCX_PROTO_INFO", "used") + + # PRECHECK_DEBUG=1: verbose C++/Python transceiver logs for stall + # debugging. Must be set before importing tensorrt_llm (the C++ logger + # reads TLLM_LOG_LEVEL at init). + debug = os.environ.get("PRECHECK_DEBUG") == "1" + if debug: + os.environ.setdefault("TLLM_LOG_LEVEL", "DEBUG") + + import torch + from mpi4py import MPI + + import tensorrt_llm + + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + world = comm.Get_size() + expected_world = side["parallel"]["world_size"] + if world != expected_world: + raise RuntimeError( + f"MPI world size {world} != {args.role} world size {expected_world}; " + f"the precheck srun must use the same topology as the real " + f"{args.role} server step." + ) + torch.cuda.set_device(rank % torch.cuda.device_count()) + tensorrt_llm.logger.set_level("debug" if debug else "info") + + ucx_env = " ".join(f"{k}={v}" for k, v in sorted(os.environ.items()) if k.startswith("UCX_")) + print( + f"[precheck {args.role}_{args.server_idx} r{rank}] UCX env: {ucx_env or ''}", + flush=True, + ) + + runner = PrecheckRunner(args, plan, side, comm) + kv_shape = plan["_kv_shape"] + if runner.is_leader: + print( + f"[precheck {args.role}_{args.server_idx}] kv_shape={kv_shape} " + f"model_dir={plan['_model_dir']} pairs={plan['n_pairs']} " + f"req_lens={plan['request_lengths']}", + flush=True, + ) + + arm, disarm, stop_watchdog, current_cell = _install_watchdog(runner, plan, rank) + + # --- setup: KV pool + transceiver (same config as the real test) --------- + setup_err = None + try: + arm("kv pool + transceiver setup") + runner.setup(kv_shape, max_req_len=max(plan["request_lengths"])) + disarm() + except Exception as e: # noqa: BLE001 - recorded and gated below + disarm() + setup_err = e + reason = runner._consensus_error(setup_err) + if reason is not None: + runner.recorder.record("-", 0, "INIT_ERROR", f"transceiver setup failed: {reason}") + runner.recorder.finalize() + print( + f"[precheck {args.role}_{args.server_idx} r{rank}] INIT_ERROR: {reason}", + file=sys.stderr, + flush=True, + ) + return 1 + if runner.is_leader: + # Effective values after model-preference resolution — what serving + # would actually run with (PR #15823 semantics). + print( + f"[precheck {args.role}_{args.server_idx}] " + f"kv_cache_manager={'V2' if runner.use_v2 else 'V1'} " + f"transceiver_runtime={runner.runtime}", + flush=True, + ) + + record_peer_failure = _make_peer_failure_recorder(runner, disarm, current_cell) + try: + if args.role == "ctx": + _serve_gen_peers(runner, plan, arm, disarm, record_peer_failure) + else: + _drive_ctx_peers(runner, arm, disarm, record_peer_failure) + finally: + stop_watchdog() + + # --- teardown + result ------------------------------------------------------ + # Bandwidth lives on different sides per transceiver: C++ records it on the + # receiver (gen recv CSVs), the Python transceiver on the sender (ctx perf + # CSVs). csv_dir is shared across the instance's ranks, so for the Python + # path the leader alone medians over all ranks' perf files. + bw = None + if runner.runtime == "PYTHON": + if args.role == "ctx" and runner.is_leader: + bw = parse_python_bandwidth_gbps(runner.csv_dir) + elif args.role == "gen": + local_bw = parse_bandwidth_gbps(runner.csv_dir, rank) + bws = [b for b in comm.gather(local_bw, root=0) or [] if b] + if runner.is_leader and bws: + bw = sorted(bws)[len(bws) // 2] + if runner.xcvr is not None and hasattr(runner.xcvr, "shutdown"): + try: + runner.xcvr.shutdown() + except Exception: # noqa: BLE001 - teardown best-effort + pass + + failed_local = 1 if runner.recorder.failed_cases() else 0 + failed = comm.allreduce(failed_local, op=MPI.MAX) + extra = { + "kv_cache_manager": "V2" if runner.use_v2 else "V1", + "transceiver_runtime": runner.runtime, + } + if bw: + extra["per_gpu_bw_gbps"] = bw + runner.recorder.finalize(extra=extra) + comm.Barrier() + if runner.is_leader: + verdict = "FAIL" if failed else "PASS" + bw_note = f" per-GPU BW ~{bw:.1f} GB/s" if bw else "" + print(f"[precheck {args.role}_{args.server_idx}] {verdict}{bw_note}", flush=True) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 2fbda813db13..ca09a3f48527 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -88,5 +89,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 37ca84fd86e2..ba734d22ddd7 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml index 7424aec950a5..81b143d4f778 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index ea66d6d99aa4..8276bd147099 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 @@ -91,6 +92,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index a3421223a0eb..6df94fea7c51 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -88,5 +89,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index d39d01edb69b..d169b9b6150c 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml new file mode 100644 index 000000000000..293e803ea2fa --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [38, 95, 50, 47, 25, 15, 26, 53, 31, 9, 41, 71, 19, 4, 143, 108, 85, 44, 6, 8, 127, 66, 2, 33, 63, 17, 70, 112, 79, 96, 109, 21, 159, 186, 90, 114, 147, 169, 155, 136, 81, 103, 120, 234, 256, 37, 107, 223, 172, 175, 217, 238, 243, 206, 209, 174, 220, 271, 261, 307, 213, 273, 247, 258, 268, 231, 277, 255, 227, 242, 309, 280, 371, 299, 303, 305, 351, 313, 320, 329, 291, 244, 374, 340, 364, 276, 379, 228, 342, 250, 322, 368, 356, 373, 354, 380, 57, 51, 1, 35, 11, 52, 34, 7, 27, 56, 0, 76, 72, 74, 100, 163, 80, 102, 55, 97, 84, 59, 54, 78, 88, 134, 105, 142, 106, 92, 40, 62, 204, 222, 126, 111, 179, 158, 32, 173, 178, 118, 160, 210, 249, 282, 196, 190, 212, 157, 236, 191, 193, 208, 287, 253, 230, 199, 153, 165, 235, 293, 274, 286, 267, 200, 266, 265, 297, 224, 298, 285, 264, 296, 301, 300, 375, 319, 314, 284, 376, 292, 281, 383, 270, 336, 345, 316, 365, 321, 331, 348, 346, 370, 382, 355, 259, 339, 3, 10, 12, 18, 5, 28, 104, 20, 73, 117, 46, 87, 39, 94, 22, 99, 75, 86, 29, 119, 48, 42, 83, 58, 145, 122, 131, 141, 162, 36, 139, 113, 101, 116, 129, 207, 152, 154, 49, 128, 170, 167, 245, 146, 125, 197, 215, 194, 260, 185, 263, 216, 214, 161, 312, 219, 182, 202, 205, 254, 188, 252, 203, 240, 251, 272, 257, 248, 315, 294, 237, 317, 232, 311, 330, 343, 288, 341, 325, 302, 360, 226, 361, 349, 338, 352, 358, 357, 366, 344, 337, 372, 381, 362, 283, 324, 93, 164, 14, 13, 24, 61, 23, 16, 30, 211, 123, 166, 68, 43, 98, 45, 110, 89, 64, 144, 67, 77, 69, 121, 132, 130, 133, 138, 137, 65, 150, 115, 221, 183, 156, 91, 82, 168, 124, 148, 184, 181, 176, 140, 60, 149, 192, 198, 135, 151, 177, 171, 195, 187, 229, 201, 180, 304, 239, 308, 233, 353, 218, 334, 359, 347, 323, 189, 278, 269, 290, 241, 295, 246, 327, 275, 289, 225, 318, 350, 262, 335, 310, 279, 369, 367, 328, 306, 326, 333, 332, 363, 377, 378] + 1: [71, 41, 6, 3, 2, 51, 5, 61, 8, 109, 7, 23, 33, 189, 34, 45, 70, 66, 40, 104, 84, 31, 75, 44, 87, 85, 215, 123, 88, 36, 32, 67, 217, 133, 278, 184, 158, 204, 101, 94, 181, 115, 155, 127, 122, 185, 180, 209, 230, 198, 200, 168, 196, 177, 194, 273, 175, 228, 249, 195, 233, 235, 240, 254, 220, 244, 207, 205, 277, 236, 270, 324, 334, 268, 289, 313, 321, 279, 285, 316, 297, 343, 370, 362, 295, 261, 315, 284, 348, 307, 372, 323, 352, 345, 383, 381, 47, 13, 81, 22, 4, 83, 10, 80, 77, 42, 21, 60, 43, 56, 76, 89, 68, 57, 37, 113, 24, 28, 131, 146, 82, 93, 53, 74, 197, 121, 110, 154, 107, 125, 52, 135, 142, 174, 160, 105, 86, 99, 137, 124, 141, 172, 161, 227, 216, 186, 193, 192, 145, 260, 257, 201, 169, 301, 234, 247, 206, 237, 332, 165, 190, 253, 255, 294, 276, 241, 299, 203, 310, 263, 347, 287, 338, 300, 320, 357, 304, 319, 258, 280, 346, 358, 364, 265, 349, 356, 341, 354, 373, 380, 377, 368, 49, 360, 9, 17, 62, 30, 1, 38, 35, 50, 55, 48, 19, 143, 100, 117, 138, 72, 54, 18, 150, 106, 97, 98, 20, 151, 91, 112, 166, 136, 126, 229, 153, 114, 64, 15, 134, 92, 152, 78, 170, 132, 243, 191, 156, 245, 157, 250, 148, 162, 224, 210, 238, 187, 262, 208, 183, 159, 214, 242, 223, 226, 264, 283, 259, 330, 275, 219, 288, 318, 256, 353, 337, 351, 252, 371, 290, 314, 359, 350, 361, 282, 286, 293, 291, 382, 342, 340, 355, 335, 366, 374, 367, 379, 375, 363, 178, 303, 0, 11, 26, 12, 90, 27, 58, 144, 59, 120, 29, 46, 25, 118, 102, 173, 69, 119, 65, 79, 96, 39, 139, 108, 16, 103, 116, 14, 63, 73, 95, 163, 167, 179, 202, 246, 130, 188, 149, 251, 239, 140, 171, 129, 111, 221, 211, 248, 274, 128, 182, 147, 164, 176, 213, 212, 225, 218, 267, 231, 199, 365, 308, 266, 281, 271, 232, 296, 272, 302, 322, 317, 269, 222, 325, 309, 331, 311, 326, 292, 298, 305, 327, 339, 329, 378, 328, 344, 333, 306, 369, 312, 336, 376] + 2: [79, 33, 43, 22, 1, 76, 52, 14, 16, 40, 12, 20, 2, 0, 102, 44, 60, 34, 24, 83, 128, 50, 136, 78, 99, 54, 73, 143, 120, 122, 118, 107, 85, 88, 172, 137, 212, 153, 152, 138, 97, 178, 206, 163, 279, 188, 184, 146, 213, 174, 237, 113, 125, 242, 233, 285, 183, 284, 201, 259, 243, 229, 241, 264, 236, 234, 323, 203, 303, 275, 367, 249, 232, 281, 258, 338, 289, 265, 286, 305, 301, 345, 326, 378, 322, 380, 371, 347, 331, 336, 344, 315, 271, 310, 300, 375, 86, 39, 6, 28, 53, 35, 127, 46, 66, 94, 89, 109, 25, 30, 13, 47, 71, 90, 96, 27, 72, 9, 169, 67, 38, 131, 141, 92, 108, 110, 26, 100, 68, 139, 167, 150, 223, 158, 80, 202, 154, 185, 155, 246, 288, 200, 299, 124, 221, 191, 176, 215, 151, 69, 132, 210, 224, 280, 180, 220, 225, 317, 298, 319, 226, 197, 291, 340, 283, 294, 307, 218, 278, 346, 297, 261, 320, 235, 270, 357, 329, 348, 316, 293, 370, 360, 342, 376, 377, 328, 312, 335, 358, 372, 365, 379, 133, 296, 10, 87, 5, 36, 55, 42, 29, 11, 82, 106, 45, 41, 126, 74, 48, 64, 18, 119, 84, 121, 58, 21, 17, 192, 63, 101, 162, 81, 267, 61, 181, 98, 240, 222, 214, 166, 140, 145, 111, 129, 130, 194, 177, 269, 211, 175, 205, 247, 148, 302, 144, 216, 187, 251, 238, 164, 230, 263, 156, 195, 290, 250, 182, 217, 186, 273, 274, 333, 308, 373, 276, 268, 228, 311, 254, 350, 245, 309, 266, 256, 318, 349, 313, 327, 231, 364, 339, 381, 295, 314, 374, 366, 341, 383, 199, 123, 4, 77, 8, 7, 115, 15, 51, 93, 37, 23, 65, 70, 49, 3, 31, 56, 134, 104, 32, 19, 142, 114, 105, 75, 62, 165, 103, 161, 168, 149, 59, 117, 57, 112, 135, 171, 170, 160, 193, 95, 219, 91, 209, 190, 287, 173, 157, 196, 255, 147, 189, 207, 159, 116, 253, 204, 179, 198, 282, 227, 272, 257, 244, 277, 292, 208, 239, 324, 262, 252, 334, 248, 343, 351, 306, 304, 352, 321, 356, 325, 368, 382, 332, 337, 260, 361, 330, 353, 354, 369, 363, 359, 362, 355] + 3: [310, 31, 94, 115, 250, 234, 374, 298, 267, 215, 186, 178, 192, 111, 189, 255, 281, 341, 337, 237, 81, 108, 347, 145, 240, 181, 317, 150, 93, 218, 332, 19, 28, 152, 361, 333, 166, 48, 7, 98, 283, 91, 251, 14, 211, 68, 144, 174, 142, 129, 306, 30, 57, 101, 78, 140, 300, 327, 20, 83, 336, 233, 126, 254, 82, 214, 228, 88, 136, 256, 312, 320, 303, 64, 72, 23, 275, 22, 124, 307, 382, 271, 311, 288, 59, 167, 285, 41, 96, 259, 353, 355, 222, 244, 9, 188, 34, 131, 66, 231, 296, 180, 348, 314, 151, 242, 125, 24, 318, 169, 324, 219, 197, 26, 130, 173, 122, 322, 356, 375, 269, 342, 376, 17, 236, 331, 297, 6, 161, 168, 95, 351, 62, 294, 184, 42, 264, 109, 65, 252, 323, 87, 159, 373, 85, 113, 177, 5, 143, 195, 309, 345, 58, 112, 193, 279, 183, 258, 165, 227, 206, 172, 274, 148, 369, 284, 325, 55, 154, 338, 191, 368, 238, 352, 360, 276, 349, 262, 175, 127, 137, 107, 2, 33, 153, 203, 89, 12, 212, 185, 47, 209, 260, 377, 18, 302, 35, 304, 97, 182, 367, 0, 210, 179, 282, 220, 330, 170, 13, 248, 90, 326, 230, 119, 221, 277, 80, 245, 295, 201, 103, 235, 121, 315, 157, 354, 198, 141, 155, 102, 226, 118, 49, 202, 200, 346, 79, 69, 249, 133, 247, 60, 56, 340, 204, 305, 70, 362, 268, 138, 4, 50, 246, 289, 343, 381, 217, 253, 194, 61, 106, 67, 190, 84, 370, 187, 104, 110, 1, 313, 299, 290, 116, 278, 334, 216, 286, 366, 99, 357, 132, 53, 257, 329, 359, 241, 105, 117, 163, 40, 319, 51, 208, 27, 120, 365, 293, 16, 339, 380, 92, 139, 291, 158, 135, 21, 134, 223, 280, 378, 76, 272, 162, 160, 71, 328, 37, 10, 128, 25, 86, 176, 308, 321, 379, 213, 164, 207, 114, 270, 38, 36, 229, 74, 273, 364, 100, 73, 52, 3, 383, 171, 63, 224, 46, 8, 15, 75, 199, 11, 123, 149, 239, 32, 344, 225, 363, 358, 146, 77, 196, 43, 243, 372, 371, 232, 147, 45, 316, 54, 29, 265, 44, 39, 156, 287, 266, 301, 292, 205, 261, 335, 350, 263] + 4: [172, 18, 264, 302, 219, 205, 22, 62, 29, 105, 115, 362, 101, 91, 149, 253, 272, 226, 36, 284, 215, 103, 320, 250, 69, 262, 312, 23, 48, 167, 328, 26, 374, 276, 271, 142, 248, 348, 157, 59, 191, 98, 182, 51, 338, 33, 129, 109, 97, 79, 168, 42, 323, 65, 17, 74, 85, 108, 75, 10, 335, 216, 30, 270, 281, 122, 300, 173, 242, 256, 201, 80, 291, 334, 343, 9, 128, 372, 211, 373, 313, 368, 224, 336, 126, 305, 132, 234, 301, 100, 364, 285, 83, 104, 247, 299, 12, 230, 257, 27, 252, 353, 116, 160, 159, 117, 24, 243, 309, 355, 50, 214, 199, 113, 25, 370, 213, 326, 379, 210, 307, 86, 135, 61, 207, 194, 361, 64, 38, 96, 221, 288, 371, 47, 196, 204, 381, 231, 310, 46, 251, 15, 185, 354, 60, 333, 92, 187, 267, 295, 133, 63, 290, 198, 174, 147, 325, 94, 332, 227, 35, 28, 238, 158, 337, 127, 140, 308, 342, 45, 282, 352, 255, 297, 139, 183, 43, 349, 19, 16, 268, 192, 171, 20, 131, 32, 40, 314, 146, 58, 222, 351, 283, 82, 273, 369, 347, 244, 169, 189, 346, 356, 37, 163, 93, 202, 258, 155, 0, 165, 145, 70, 114, 344, 130, 357, 322, 340, 3, 162, 153, 166, 327, 66, 274, 197, 71, 263, 73, 84, 236, 112, 179, 88, 358, 55, 176, 76, 318, 217, 78, 110, 260, 241, 315, 365, 229, 339, 141, 4, 181, 56, 121, 203, 154, 178, 375, 378, 184, 2, 1, 175, 95, 341, 287, 49, 138, 294, 14, 266, 31, 316, 123, 363, 383, 180, 81, 292, 319, 311, 195, 89, 5, 90, 259, 120, 156, 360, 254, 13, 275, 188, 44, 228, 239, 54, 235, 293, 206, 34, 265, 161, 77, 136, 72, 324, 345, 246, 57, 111, 240, 212, 359, 237, 209, 152, 21, 376, 278, 150, 68, 148, 6, 7, 330, 124, 99, 137, 143, 53, 233, 67, 164, 377, 87, 329, 11, 366, 52, 280, 106, 269, 296, 232, 220, 151, 125, 289, 186, 170, 177, 303, 190, 41, 200, 286, 331, 350, 223, 118, 321, 298, 249, 218, 306, 261, 8, 208, 382, 119, 102, 380, 107, 144, 225, 279, 134, 367, 193, 304, 39, 245, 317, 277] + 5: [214, 133, 193, 178, 131, 67, 46, 242, 69, 161, 35, 41, 299, 365, 188, 209, 361, 45, 4, 11, 379, 314, 66, 330, 149, 47, 284, 257, 244, 168, 308, 117, 200, 38, 106, 2, 266, 378, 146, 90, 62, 323, 225, 121, 16, 152, 186, 343, 22, 277, 30, 377, 24, 143, 9, 337, 380, 344, 5, 359, 219, 222, 123, 31, 320, 335, 32, 169, 172, 158, 367, 185, 294, 3, 360, 201, 54, 10, 349, 70, 307, 354, 110, 28, 50, 205, 98, 270, 51, 75, 348, 23, 321, 223, 63, 177, 165, 274, 276, 352, 287, 249, 217, 129, 190, 346, 120, 301, 351, 280, 61, 243, 21, 293, 147, 333, 291, 198, 332, 124, 232, 305, 59, 65, 298, 40, 206, 261, 347, 211, 184, 238, 64, 366, 60, 334, 227, 93, 52, 207, 36, 42, 364, 180, 82, 103, 202, 135, 286, 164, 282, 92, 183, 283, 220, 89, 88, 94, 236, 15, 187, 71, 107, 251, 29, 44, 127, 373, 99, 345, 248, 336, 234, 350, 155, 278, 26, 302, 126, 57, 229, 109, 85, 116, 362, 150, 271, 295, 275, 111, 300, 318, 128, 230, 372, 304, 182, 262, 81, 265, 383, 241, 309, 199, 231, 108, 137, 181, 159, 154, 194, 324, 306, 114, 14, 329, 339, 357, 338, 78, 132, 279, 80, 355, 8, 144, 252, 134, 259, 381, 218, 290, 48, 136, 34, 53, 216, 105, 0, 119, 171, 166, 153, 233, 374, 55, 115, 20, 122, 228, 157, 56, 140, 341, 255, 325, 273, 316, 95, 224, 145, 6, 195, 104, 353, 382, 43, 18, 83, 73, 91, 268, 174, 162, 33, 375, 311, 97, 170, 281, 312, 141, 368, 226, 285, 125, 142, 196, 269, 296, 49, 246, 17, 138, 213, 74, 151, 79, 250, 156, 130, 258, 239, 163, 221, 342, 13, 1, 173, 191, 118, 27, 245, 319, 260, 264, 167, 19, 256, 39, 112, 84, 370, 327, 197, 215, 7, 272, 113, 204, 254, 77, 210, 179, 289, 58, 369, 76, 100, 235, 192, 303, 317, 322, 356, 253, 267, 212, 203, 176, 208, 189, 358, 297, 12, 315, 292, 160, 340, 139, 240, 328, 175, 148, 102, 237, 101, 313, 331, 288, 72, 371, 86, 68, 96, 310, 326, 247, 363, 87, 25, 37, 376, 263] + 6: [97, 299, 34, 159, 26, 197, 319, 269, 349, 113, 79, 118, 196, 300, 222, 236, 143, 192, 67, 83, 14, 17, 133, 244, 361, 165, 324, 266, 326, 290, 364, 158, 217, 357, 230, 82, 8, 234, 184, 362, 84, 284, 330, 153, 343, 202, 280, 381, 177, 295, 342, 146, 30, 11, 57, 18, 353, 61, 378, 328, 5, 262, 20, 213, 60, 114, 238, 240, 383, 4, 16, 52, 329, 71, 163, 107, 13, 130, 382, 95, 92, 140, 152, 369, 37, 70, 315, 72, 200, 356, 21, 161, 327, 93, 257, 323, 142, 259, 173, 282, 337, 359, 352, 220, 122, 336, 366, 216, 338, 41, 358, 379, 10, 102, 54, 39, 272, 287, 205, 116, 371, 195, 48, 306, 19, 321, 176, 144, 270, 370, 211, 252, 232, 85, 293, 87, 285, 298, 128, 348, 45, 296, 334, 203, 109, 275, 51, 110, 344, 154, 175, 308, 148, 178, 235, 320, 254, 126, 360, 58, 242, 132, 15, 372, 157, 258, 190, 55, 127, 261, 265, 335, 331, 237, 224, 297, 182, 214, 288, 186, 276, 25, 65, 106, 115, 98, 40, 167, 139, 53, 172, 191, 274, 246, 289, 239, 322, 281, 0, 253, 301, 137, 166, 80, 23, 350, 286, 46, 136, 171, 251, 345, 304, 76, 100, 256, 316, 367, 49, 47, 291, 221, 27, 339, 241, 162, 33, 117, 215, 278, 249, 78, 226, 307, 376, 271, 138, 69, 181, 310, 63, 32, 347, 111, 243, 50, 108, 210, 6, 193, 9, 218, 187, 123, 12, 283, 279, 198, 22, 129, 68, 223, 75, 77, 164, 81, 131, 86, 43, 346, 2, 56, 24, 292, 373, 227, 179, 255, 201, 7, 168, 89, 121, 355, 96, 303, 90, 124, 305, 44, 267, 260, 112, 375, 160, 174, 264, 151, 377, 311, 62, 103, 318, 169, 380, 155, 125, 149, 199, 120, 368, 119, 64, 206, 312, 248, 309, 277, 219, 231, 250, 141, 188, 268, 302, 73, 233, 341, 228, 59, 207, 332, 225, 66, 229, 88, 105, 208, 42, 101, 74, 99, 189, 36, 209, 263, 29, 313, 35, 31, 91, 94, 145, 365, 104, 185, 333, 183, 340, 354, 212, 245, 28, 180, 135, 314, 170, 134, 294, 325, 156, 351, 194, 38, 204, 374, 317, 247, 273, 3, 147, 150, 363, 1] + 7: [382, 113, 133, 238, 284, 331, 69, 0, 239, 11, 59, 9, 160, 190, 21, 336, 121, 302, 359, 198, 299, 12, 18, 268, 99, 247, 182, 270, 300, 203, 152, 166, 376, 70, 104, 38, 61, 132, 379, 373, 1, 196, 313, 237, 261, 167, 271, 372, 230, 235, 143, 318, 229, 227, 128, 31, 14, 374, 367, 5, 149, 266, 159, 179, 185, 255, 171, 162, 241, 234, 283, 106, 169, 35, 52, 364, 50, 26, 338, 164, 330, 127, 120, 346, 66, 175, 366, 151, 98, 222, 371, 215, 192, 375, 37, 273, 27, 49, 170, 358, 188, 72, 46, 294, 15, 354, 119, 64, 184, 301, 337, 315, 174, 228, 40, 56, 63, 33, 303, 28, 262, 368, 334, 129, 286, 218, 309, 25, 165, 140, 224, 253, 304, 91, 287, 312, 125, 122, 306, 290, 112, 344, 242, 142, 285, 278, 342, 195, 78, 260, 343, 20, 32, 267, 93, 254, 114, 172, 322, 316, 276, 90, 212, 47, 194, 139, 89, 233, 124, 34, 84, 220, 250, 281, 263, 4, 219, 307, 199, 54, 180, 248, 361, 102, 357, 365, 240, 178, 207, 2, 111, 60, 292, 329, 258, 48, 213, 216, 305, 333, 71, 321, 19, 126, 74, 80, 30, 360, 311, 154, 295, 36, 200, 76, 153, 347, 209, 41, 217, 136, 319, 24, 317, 138, 352, 29, 73, 370, 118, 275, 193, 280, 288, 277, 181, 383, 323, 353, 7, 202, 328, 380, 339, 349, 256, 362, 265, 272, 206, 55, 297, 39, 204, 369, 75, 44, 208, 131, 308, 43, 141, 53, 147, 325, 137, 134, 45, 65, 259, 96, 116, 177, 92, 156, 231, 115, 211, 22, 189, 341, 381, 245, 6, 236, 201, 348, 3, 157, 163, 274, 378, 356, 187, 282, 51, 100, 13, 155, 335, 130, 82, 326, 197, 355, 221, 150, 101, 23, 314, 340, 296, 77, 264, 279, 94, 291, 146, 289, 148, 107, 110, 324, 176, 320, 108, 168, 83, 16, 123, 183, 205, 85, 161, 244, 332, 105, 8, 327, 86, 293, 298, 214, 246, 226, 62, 191, 103, 158, 310, 17, 144, 252, 249, 95, 117, 186, 109, 10, 269, 42, 173, 210, 251, 351, 350, 79, 67, 81, 377, 87, 232, 58, 257, 225, 68, 363, 57, 135, 223, 97, 88, 243, 145, 345] + 8: [290, 17, 315, 35, 371, 367, 227, 351, 171, 335, 225, 131, 40, 185, 376, 337, 221, 352, 278, 275, 282, 329, 330, 263, 301, 12, 238, 92, 114, 245, 243, 271, 115, 137, 179, 224, 284, 287, 205, 308, 110, 177, 359, 383, 248, 264, 251, 162, 366, 222, 345, 332, 299, 147, 43, 239, 183, 109, 72, 55, 73, 2, 274, 23, 124, 283, 184, 253, 199, 328, 146, 24, 250, 188, 178, 244, 134, 192, 379, 59, 267, 145, 357, 44, 163, 97, 78, 54, 136, 270, 15, 216, 113, 214, 102, 26, 88, 231, 62, 336, 354, 249, 103, 158, 381, 27, 90, 52, 38, 309, 80, 82, 150, 8, 33, 129, 215, 10, 257, 241, 65, 194, 344, 360, 276, 6, 3, 219, 117, 382, 187, 125, 132, 280, 259, 81, 196, 209, 191, 144, 42, 247, 14, 139, 213, 237, 180, 198, 327, 343, 22, 1, 189, 272, 57, 197, 334, 260, 311, 133, 190, 324, 95, 340, 66, 300, 159, 63, 39, 348, 365, 338, 165, 25, 230, 29, 320, 30, 310, 305, 212, 46, 108, 151, 116, 5, 254, 206, 234, 314, 120, 369, 362, 312, 7, 353, 156, 152, 50, 18, 84, 61, 148, 223, 195, 94, 240, 279, 255, 153, 70, 289, 372, 228, 296, 377, 355, 37, 31, 293, 356, 122, 45, 319, 307, 74, 93, 331, 11, 16, 277, 138, 358, 246, 60, 322, 273, 75, 161, 235, 373, 193, 347, 341, 173, 140, 261, 0, 47, 13, 167, 292, 123, 258, 181, 361, 87, 236, 166, 252, 168, 34, 266, 96, 297, 176, 69, 226, 143, 19, 112, 182, 99, 56, 363, 174, 154, 298, 104, 204, 342, 142, 242, 317, 364, 58, 41, 378, 76, 218, 202, 333, 118, 281, 200, 201, 368, 83, 326, 36, 4, 21, 175, 164, 229, 288, 302, 172, 32, 119, 321, 380, 294, 85, 318, 53, 128, 339, 20, 269, 28, 127, 67, 130, 186, 316, 208, 374, 107, 313, 306, 325, 49, 286, 9, 77, 233, 111, 157, 126, 217, 79, 155, 106, 169, 291, 304, 98, 86, 100, 232, 89, 68, 51, 135, 64, 210, 323, 91, 256, 101, 303, 285, 346, 262, 211, 349, 350, 160, 220, 149, 121, 207, 203, 170, 268, 71, 48, 141, 295, 375, 265, 370, 105] + 9: [92, 186, 30, 259, 8, 339, 56, 263, 288, 183, 131, 75, 350, 228, 222, 184, 120, 98, 174, 180, 60, 167, 74, 311, 364, 119, 110, 323, 46, 65, 9, 99, 250, 343, 306, 315, 153, 81, 328, 262, 62, 318, 251, 195, 181, 378, 22, 163, 216, 330, 166, 38, 23, 31, 86, 194, 127, 309, 61, 133, 358, 232, 190, 287, 264, 185, 370, 290, 152, 44, 267, 349, 108, 204, 269, 238, 223, 95, 93, 173, 340, 295, 374, 169, 225, 373, 7, 66, 118, 43, 212, 336, 375, 149, 326, 240, 327, 353, 281, 132, 322, 35, 210, 192, 187, 170, 94, 129, 54, 324, 159, 29, 342, 314, 176, 172, 71, 144, 217, 58, 126, 70, 256, 199, 32, 218, 130, 16, 198, 76, 354, 73, 206, 227, 310, 123, 301, 366, 302, 164, 211, 27, 377, 252, 88, 379, 272, 21, 78, 2, 316, 233, 313, 33, 113, 91, 325, 168, 182, 96, 368, 137, 258, 329, 297, 361, 160, 77, 268, 352, 230, 89, 345, 294, 162, 300, 347, 67, 249, 236, 254, 175, 338, 265, 50, 4, 308, 201, 291, 202, 284, 26, 3, 331, 143, 355, 237, 17, 103, 191, 360, 36, 299, 219, 72, 20, 25, 148, 83, 231, 376, 253, 277, 307, 112, 372, 135, 296, 380, 381, 245, 111, 261, 128, 114, 208, 107, 156, 101, 97, 84, 134, 165, 117, 177, 178, 121, 155, 1, 18, 48, 64, 68, 157, 197, 12, 356, 273, 79, 285, 200, 154, 214, 109, 292, 39, 317, 41, 362, 209, 257, 146, 303, 193, 45, 52, 85, 205, 49, 226, 40, 141, 19, 63, 346, 53, 196, 348, 276, 282, 246, 279, 24, 122, 312, 55, 283, 57, 215, 260, 150, 138, 102, 80, 5, 104, 298, 239, 10, 321, 224, 87, 244, 371, 359, 221, 47, 289, 335, 274, 220, 37, 13, 139, 6, 275, 341, 367, 151, 363, 333, 344, 235, 243, 42, 125, 242, 247, 351, 382, 179, 337, 105, 14, 320, 189, 229, 207, 293, 136, 124, 203, 332, 213, 82, 255, 357, 0, 106, 278, 171, 234, 145, 100, 15, 369, 142, 280, 304, 266, 286, 69, 383, 140, 241, 34, 271, 334, 319, 188, 365, 270, 248, 305, 28, 90, 115, 158, 59, 147, 51, 116, 11, 161] + 10: [55, 61, 222, 353, 78, 262, 365, 69, 234, 227, 124, 293, 246, 289, 138, 8, 282, 109, 341, 113, 231, 181, 338, 40, 132, 223, 257, 306, 72, 116, 75, 184, 130, 240, 267, 148, 123, 220, 327, 188, 114, 210, 243, 256, 137, 214, 189, 252, 330, 121, 100, 37, 190, 265, 308, 297, 145, 153, 131, 346, 88, 167, 122, 202, 328, 313, 383, 26, 335, 93, 13, 52, 118, 381, 323, 261, 296, 260, 134, 147, 270, 303, 366, 173, 162, 336, 166, 233, 216, 369, 64, 357, 291, 209, 334, 292, 31, 321, 180, 91, 378, 73, 141, 249, 229, 281, 102, 59, 107, 199, 301, 105, 251, 170, 149, 94, 280, 354, 352, 279, 164, 111, 87, 304, 195, 219, 343, 32, 0, 363, 376, 161, 225, 203, 259, 95, 242, 367, 67, 187, 9, 171, 119, 117, 80, 263, 53, 16, 10, 333, 43, 54, 18, 41, 269, 298, 226, 294, 344, 355, 320, 213, 276, 35, 205, 255, 51, 272, 142, 197, 360, 200, 70, 299, 12, 198, 285, 120, 28, 311, 165, 89, 288, 155, 63, 151, 6, 68, 314, 183, 332, 339, 5, 250, 66, 309, 2, 258, 33, 144, 92, 264, 318, 90, 191, 215, 268, 337, 310, 248, 239, 74, 110, 271, 284, 221, 204, 156, 382, 317, 169, 283, 377, 127, 351, 302, 361, 182, 375, 211, 17, 373, 217, 76, 20, 96, 7, 15, 56, 207, 129, 380, 286, 84, 60, 368, 370, 364, 201, 325, 235, 237, 275, 290, 30, 44, 372, 295, 374, 196, 98, 125, 307, 287, 228, 238, 146, 186, 85, 324, 1, 79, 348, 58, 4, 193, 305, 38, 82, 340, 83, 86, 254, 23, 29, 350, 157, 277, 319, 115, 266, 192, 224, 177, 103, 345, 218, 247, 99, 158, 97, 57, 342, 108, 11, 274, 241, 101, 81, 65, 22, 106, 230, 326, 322, 152, 19, 174, 128, 316, 133, 315, 329, 126, 245, 143, 185, 379, 356, 3, 34, 25, 212, 312, 154, 47, 140, 362, 49, 112, 163, 24, 359, 62, 50, 77, 150, 45, 194, 206, 208, 175, 48, 300, 172, 349, 371, 139, 159, 21, 278, 104, 27, 39, 36, 253, 358, 71, 46, 168, 14, 347, 135, 331, 236, 178, 160, 244, 179, 176, 136, 232, 273, 42] + 11: [377, 4, 277, 186, 249, 60, 317, 107, 230, 143, 150, 374, 278, 95, 131, 217, 321, 298, 293, 69, 1, 280, 165, 59, 52, 187, 67, 285, 360, 160, 294, 250, 85, 282, 297, 267, 64, 37, 253, 256, 315, 25, 198, 255, 7, 279, 40, 91, 142, 370, 332, 367, 133, 199, 222, 261, 62, 364, 292, 363, 11, 63, 144, 151, 49, 32, 311, 174, 324, 197, 113, 304, 21, 192, 158, 41, 88, 163, 193, 289, 136, 206, 361, 175, 139, 48, 270, 226, 152, 8, 352, 371, 241, 355, 55, 348, 130, 203, 17, 339, 196, 128, 313, 271, 325, 66, 247, 349, 166, 101, 87, 378, 242, 123, 331, 219, 171, 356, 53, 168, 189, 28, 182, 376, 71, 216, 232, 248, 75, 92, 207, 35, 149, 93, 350, 375, 170, 24, 372, 214, 46, 81, 237, 20, 58, 223, 36, 51, 98, 329, 47, 141, 184, 57, 245, 266, 167, 19, 319, 109, 129, 12, 301, 134, 296, 177, 18, 290, 169, 183, 103, 263, 354, 191, 236, 288, 268, 155, 138, 260, 210, 383, 100, 148, 202, 265, 105, 72, 308, 33, 112, 120, 156, 234, 83, 110, 362, 326, 254, 213, 208, 194, 30, 238, 300, 220, 299, 264, 99, 10, 188, 27, 302, 291, 281, 172, 82, 45, 357, 345, 74, 330, 31, 34, 259, 190, 2, 157, 201, 369, 318, 185, 347, 159, 42, 84, 287, 275, 209, 153, 272, 68, 273, 231, 342, 161, 200, 106, 6, 212, 137, 127, 337, 116, 79, 276, 70, 50, 86, 303, 314, 77, 224, 76, 221, 38, 108, 366, 5, 295, 335, 351, 78, 135, 320, 286, 327, 205, 328, 102, 22, 215, 344, 96, 338, 181, 379, 9, 284, 29, 306, 145, 235, 56, 43, 258, 233, 121, 80, 228, 316, 26, 373, 309, 23, 322, 90, 119, 44, 180, 115, 65, 97, 173, 162, 211, 195, 229, 39, 154, 243, 251, 239, 218, 146, 164, 73, 380, 368, 176, 365, 257, 343, 125, 341, 381, 353, 111, 274, 336, 305, 340, 0, 246, 178, 104, 227, 3, 240, 117, 16, 54, 204, 310, 252, 94, 118, 307, 346, 358, 179, 89, 124, 147, 122, 283, 382, 312, 244, 140, 269, 15, 13, 262, 333, 359, 323, 225, 126, 132, 334, 114, 61, 14] + 12: [226, 20, 194, 300, 120, 298, 109, 71, 3, 304, 108, 185, 280, 39, 59, 127, 237, 9, 198, 227, 268, 294, 93, 86, 330, 41, 329, 245, 175, 49, 317, 148, 161, 96, 196, 151, 331, 69, 236, 52, 335, 53, 267, 88, 332, 92, 114, 381, 232, 338, 25, 33, 366, 363, 77, 371, 374, 130, 137, 85, 170, 31, 191, 351, 1, 79, 178, 10, 19, 216, 307, 24, 5, 115, 243, 12, 143, 375, 188, 252, 122, 149, 167, 179, 80, 16, 23, 354, 314, 42, 213, 259, 211, 352, 220, 285, 82, 292, 308, 218, 203, 90, 234, 51, 140, 337, 72, 172, 281, 373, 207, 370, 327, 320, 113, 233, 75, 286, 219, 125, 293, 166, 107, 155, 246, 176, 180, 301, 76, 276, 258, 102, 55, 289, 156, 362, 43, 94, 275, 359, 200, 2, 173, 18, 224, 206, 136, 240, 265, 202, 123, 63, 299, 353, 132, 279, 356, 382, 89, 100, 128, 57, 101, 134, 54, 310, 262, 215, 30, 334, 135, 323, 325, 221, 35, 269, 347, 44, 119, 67, 228, 74, 56, 27, 368, 217, 249, 117, 66, 165, 60, 162, 204, 341, 255, 112, 306, 378, 13, 247, 138, 346, 254, 208, 305, 355, 195, 110, 186, 361, 326, 22, 312, 64, 62, 129, 251, 193, 11, 87, 210, 263, 38, 40, 153, 222, 192, 144, 288, 169, 328, 164, 28, 177, 26, 48, 309, 111, 376, 116, 273, 163, 6, 383, 34, 365, 257, 379, 291, 32, 350, 160, 189, 121, 340, 318, 238, 159, 360, 84, 239, 266, 260, 98, 242, 61, 65, 142, 364, 278, 73, 146, 197, 70, 372, 377, 183, 58, 324, 336, 253, 342, 15, 154, 212, 303, 272, 145, 78, 287, 150, 158, 68, 50, 322, 147, 315, 201, 209, 133, 313, 81, 271, 270, 367, 230, 290, 274, 168, 36, 46, 103, 4, 344, 380, 321, 106, 99, 105, 21, 283, 229, 91, 174, 8, 250, 282, 277, 345, 45, 214, 181, 37, 357, 264, 316, 296, 311, 29, 83, 244, 14, 284, 205, 248, 339, 297, 190, 187, 235, 369, 124, 95, 302, 0, 171, 118, 97, 152, 256, 141, 343, 223, 131, 319, 7, 241, 126, 182, 47, 17, 348, 231, 139, 184, 225, 295, 333, 104, 358, 199, 261, 349, 157] + 13: [108, 46, 172, 186, 329, 340, 231, 363, 199, 316, 134, 228, 77, 16, 55, 244, 110, 86, 324, 318, 36, 6, 125, 201, 83, 348, 109, 277, 282, 314, 40, 335, 37, 266, 121, 364, 275, 333, 21, 205, 261, 61, 322, 300, 249, 81, 238, 260, 66, 339, 326, 288, 253, 50, 92, 285, 243, 270, 79, 38, 317, 360, 63, 279, 47, 240, 67, 303, 33, 1, 26, 115, 220, 194, 251, 357, 268, 283, 334, 310, 371, 367, 245, 53, 48, 168, 17, 44, 215, 258, 27, 129, 350, 341, 132, 302, 170, 328, 309, 179, 87, 273, 30, 373, 379, 319, 72, 0, 382, 200, 291, 307, 76, 184, 188, 353, 139, 153, 11, 234, 52, 198, 177, 35, 207, 267, 222, 185, 84, 25, 169, 352, 223, 226, 41, 361, 2, 297, 107, 94, 45, 305, 126, 144, 203, 128, 162, 164, 269, 88, 290, 362, 187, 299, 101, 232, 263, 225, 202, 74, 247, 190, 182, 271, 289, 209, 365, 359, 212, 10, 193, 127, 295, 150, 7, 280, 327, 3, 112, 28, 122, 167, 98, 105, 192, 29, 354, 381, 343, 377, 32, 59, 23, 298, 159, 8, 311, 111, 208, 151, 65, 211, 214, 284, 191, 369, 123, 338, 345, 91, 315, 31, 147, 149, 221, 248, 69, 135, 131, 141, 64, 210, 293, 272, 346, 296, 292, 213, 294, 197, 229, 89, 380, 176, 286, 224, 82, 24, 281, 257, 161, 54, 15, 183, 39, 233, 265, 100, 4, 57, 133, 137, 254, 113, 173, 347, 216, 22, 287, 375, 104, 73, 136, 374, 320, 13, 372, 330, 206, 140, 306, 241, 20, 142, 276, 78, 90, 155, 204, 181, 332, 250, 124, 130, 175, 378, 85, 152, 189, 12, 195, 102, 117, 304, 56, 358, 58, 342, 178, 93, 259, 116, 51, 143, 196, 239, 171, 321, 166, 96, 34, 97, 19, 337, 75, 355, 370, 323, 313, 246, 227, 174, 237, 68, 218, 71, 376, 301, 62, 242, 99, 42, 120, 154, 49, 180, 14, 236, 114, 119, 344, 219, 80, 43, 308, 5, 274, 264, 9, 138, 163, 230, 160, 235, 252, 312, 146, 157, 349, 368, 106, 118, 156, 336, 60, 256, 148, 325, 356, 262, 158, 18, 103, 351, 366, 255, 95, 165, 278, 383, 145, 331, 70, 217] + 14: [4, 147, 30, 14, 95, 291, 118, 34, 17, 90, 356, 215, 158, 23, 92, 62, 343, 342, 145, 284, 180, 324, 369, 312, 64, 16, 317, 250, 274, 272, 115, 134, 359, 51, 262, 252, 36, 325, 224, 368, 183, 160, 285, 378, 290, 87, 24, 382, 381, 89, 277, 102, 94, 232, 57, 348, 333, 303, 136, 276, 365, 351, 119, 1, 82, 367, 98, 32, 165, 88, 74, 194, 326, 223, 96, 231, 211, 3, 323, 18, 28, 172, 146, 293, 263, 171, 138, 42, 46, 176, 195, 20, 111, 72, 106, 120, 204, 164, 110, 261, 151, 184, 213, 127, 238, 91, 375, 142, 141, 313, 139, 355, 362, 75, 71, 206, 58, 298, 220, 301, 268, 318, 38, 334, 199, 25, 275, 222, 257, 376, 198, 269, 345, 86, 60, 170, 167, 254, 350, 270, 280, 190, 66, 309, 247, 22, 129, 202, 152, 366, 173, 322, 300, 209, 246, 240, 187, 191, 329, 236, 225, 321, 93, 7, 192, 255, 135, 282, 193, 357, 133, 201, 159, 319, 320, 346, 84, 245, 85, 143, 39, 360, 336, 260, 363, 178, 289, 100, 188, 216, 140, 181, 70, 126, 163, 383, 2, 128, 8, 40, 349, 227, 5, 79, 80, 354, 44, 63, 149, 125, 234, 150, 217, 273, 109, 308, 56, 379, 296, 104, 116, 6, 361, 239, 278, 101, 226, 352, 200, 197, 248, 305, 169, 76, 105, 97, 212, 218, 182, 373, 307, 11, 54, 267, 327, 253, 137, 364, 335, 338, 286, 251, 130, 81, 205, 243, 21, 340, 153, 259, 26, 235, 256, 47, 59, 131, 249, 315, 265, 132, 48, 108, 10, 310, 229, 53, 112, 228, 304, 68, 113, 203, 50, 65, 210, 73, 337, 295, 15, 117, 122, 344, 287, 13, 271, 154, 377, 177, 370, 294, 99, 244, 156, 69, 242, 281, 380, 237, 279, 374, 347, 330, 161, 83, 341, 372, 186, 175, 283, 49, 266, 208, 157, 52, 185, 103, 162, 41, 207, 311, 297, 358, 123, 332, 77, 230, 35, 107, 124, 214, 0, 299, 19, 241, 43, 233, 33, 258, 168, 37, 55, 196, 353, 9, 316, 306, 288, 61, 148, 114, 155, 166, 144, 27, 264, 292, 314, 121, 302, 12, 221, 29, 67, 219, 371, 179, 189, 31, 45, 78, 331, 174, 339, 328] + 15: [11, 353, 24, 168, 128, 56, 344, 329, 265, 13, 173, 231, 284, 10, 145, 267, 273, 96, 257, 238, 63, 213, 84, 161, 382, 74, 2, 71, 282, 7, 122, 305, 317, 224, 266, 285, 249, 380, 303, 343, 377, 372, 188, 254, 274, 186, 70, 126, 34, 46, 379, 152, 64, 356, 142, 191, 360, 192, 318, 327, 44, 9, 85, 331, 205, 3, 123, 270, 118, 129, 89, 75, 21, 347, 132, 260, 314, 110, 154, 146, 0, 247, 354, 310, 237, 229, 239, 6, 223, 345, 374, 103, 61, 29, 328, 208, 362, 232, 359, 38, 376, 253, 197, 8, 133, 319, 246, 212, 109, 290, 134, 150, 114, 51, 16, 262, 309, 80, 258, 88, 138, 291, 251, 31, 272, 158, 54, 297, 332, 67, 203, 361, 187, 233, 346, 378, 202, 321, 235, 90, 250, 165, 40, 174, 119, 330, 198, 55, 53, 95, 1, 299, 112, 236, 25, 338, 92, 289, 184, 211, 12, 77, 121, 37, 227, 315, 226, 170, 127, 264, 69, 141, 322, 276, 23, 375, 201, 143, 45, 325, 62, 30, 140, 189, 179, 182, 139, 159, 218, 336, 176, 286, 15, 199, 28, 190, 200, 117, 350, 308, 5, 48, 252, 14, 94, 35, 39, 351, 326, 167, 334, 296, 248, 106, 66, 381, 280, 111, 65, 293, 68, 97, 169, 300, 204, 256, 306, 108, 153, 183, 155, 196, 365, 86, 164, 216, 302, 120, 207, 244, 107, 339, 73, 368, 221, 156, 185, 163, 316, 342, 50, 281, 72, 383, 181, 98, 349, 225, 60, 271, 57, 49, 341, 220, 367, 295, 312, 195, 259, 355, 58, 288, 166, 131, 81, 78, 292, 275, 18, 137, 219, 100, 136, 358, 144, 268, 269, 42, 151, 105, 217, 115, 313, 255, 324, 352, 148, 83, 240, 76, 4, 335, 357, 298, 320, 215, 277, 307, 193, 261, 20, 52, 340, 113, 373, 36, 32, 172, 209, 99, 26, 116, 371, 241, 287, 263, 177, 333, 82, 33, 149, 43, 214, 243, 311, 157, 206, 323, 171, 175, 59, 102, 147, 364, 337, 234, 230, 101, 41, 242, 369, 135, 366, 104, 222, 22, 194, 370, 294, 87, 130, 160, 279, 27, 124, 178, 301, 304, 210, 228, 79, 93, 245, 125, 283, 278, 348, 180, 162, 91, 363, 17, 47, 19] + 16: [15, 374, 287, 348, 310, 134, 216, 301, 189, 92, 11, 379, 346, 331, 56, 135, 233, 305, 241, 9, 286, 187, 274, 185, 360, 212, 285, 342, 252, 238, 357, 315, 248, 129, 102, 57, 139, 227, 278, 265, 12, 246, 222, 340, 306, 220, 253, 194, 79, 131, 270, 191, 4, 314, 363, 288, 177, 329, 132, 181, 105, 334, 157, 138, 89, 197, 58, 170, 317, 365, 259, 35, 381, 325, 153, 341, 124, 63, 313, 37, 249, 322, 168, 299, 345, 359, 112, 48, 165, 179, 284, 376, 51, 309, 163, 3, 262, 100, 64, 183, 119, 260, 106, 71, 275, 330, 96, 370, 173, 180, 27, 280, 171, 182, 121, 298, 115, 93, 19, 213, 123, 358, 200, 0, 337, 31, 167, 29, 160, 300, 104, 80, 294, 2, 371, 10, 349, 47, 236, 70, 266, 207, 304, 87, 308, 268, 130, 225, 26, 323, 210, 316, 292, 28, 352, 144, 159, 269, 373, 297, 215, 99, 148, 40, 326, 78, 372, 81, 41, 235, 18, 136, 364, 272, 324, 23, 145, 195, 244, 126, 1, 263, 251, 223, 6, 117, 34, 30, 65, 303, 50, 156, 312, 5, 46, 118, 250, 59, 333, 14, 84, 205, 229, 44, 109, 281, 162, 375, 32, 75, 273, 217, 267, 283, 120, 66, 73, 52, 147, 211, 175, 336, 154, 54, 68, 158, 256, 214, 378, 88, 176, 151, 60, 143, 17, 234, 61, 141, 245, 186, 240, 36, 133, 83, 203, 276, 380, 13, 218, 230, 271, 282, 343, 289, 295, 258, 226, 178, 101, 62, 296, 383, 247, 356, 361, 74, 219, 169, 122, 368, 174, 38, 161, 140, 208, 264, 353, 366, 193, 33, 351, 261, 350, 192, 328, 367, 257, 291, 152, 114, 7, 198, 237, 277, 125, 369, 362, 85, 21, 43, 293, 188, 146, 347, 232, 255, 91, 172, 76, 354, 90, 142, 311, 97, 302, 24, 321, 202, 224, 42, 22, 199, 86, 243, 332, 16, 290, 8, 164, 382, 204, 209, 39, 166, 113, 128, 110, 190, 103, 25, 95, 108, 116, 307, 149, 20, 206, 111, 254, 82, 221, 335, 155, 67, 319, 320, 53, 242, 327, 355, 94, 184, 49, 338, 339, 98, 55, 231, 107, 150, 201, 69, 72, 196, 377, 239, 77, 228, 127, 45, 279, 137, 318, 344] + 17: [119, 188, 46, 374, 142, 275, 79, 146, 104, 243, 207, 56, 91, 24, 272, 356, 98, 97, 189, 284, 156, 116, 363, 94, 81, 310, 319, 20, 15, 71, 259, 171, 48, 222, 33, 328, 145, 58, 230, 186, 276, 209, 247, 158, 40, 360, 359, 11, 292, 301, 256, 105, 213, 53, 216, 200, 114, 187, 313, 172, 265, 218, 199, 255, 69, 18, 221, 239, 67, 164, 149, 327, 381, 252, 353, 132, 268, 17, 85, 30, 182, 306, 295, 264, 122, 165, 152, 224, 111, 235, 280, 312, 203, 166, 223, 304, 220, 236, 89, 136, 294, 44, 117, 361, 143, 138, 178, 118, 366, 194, 107, 167, 128, 22, 113, 19, 240, 278, 208, 131, 376, 12, 348, 23, 279, 135, 286, 38, 125, 238, 73, 206, 297, 248, 174, 289, 370, 317, 316, 78, 341, 63, 83, 127, 133, 244, 1, 334, 9, 90, 274, 55, 109, 288, 139, 228, 245, 144, 300, 233, 21, 241, 151, 309, 323, 150, 141, 162, 346, 283, 34, 74, 153, 217, 333, 242, 345, 198, 175, 159, 7, 371, 329, 102, 263, 43, 342, 50, 28, 57, 227, 358, 261, 337, 357, 100, 307, 80, 197, 76, 254, 193, 296, 380, 173, 378, 190, 336, 308, 115, 163, 258, 303, 3, 96, 140, 8, 367, 364, 231, 0, 5, 59, 101, 52, 362, 318, 185, 204, 232, 282, 234, 124, 298, 343, 375, 61, 36, 37, 45, 176, 382, 285, 352, 344, 126, 372, 16, 302, 66, 10, 351, 87, 39, 92, 331, 321, 330, 281, 14, 51, 84, 325, 311, 237, 88, 270, 72, 195, 160, 70, 147, 184, 161, 202, 349, 183, 291, 299, 215, 180, 68, 210, 169, 251, 120, 368, 322, 266, 212, 108, 335, 324, 177, 110, 157, 75, 214, 262, 99, 219, 2, 42, 314, 27, 305, 196, 315, 65, 129, 290, 26, 179, 347, 77, 62, 103, 250, 82, 340, 60, 93, 6, 168, 130, 260, 326, 293, 355, 225, 49, 271, 148, 191, 287, 154, 377, 134, 112, 226, 47, 253, 365, 273, 246, 267, 369, 155, 13, 338, 4, 95, 121, 137, 229, 64, 86, 269, 350, 332, 181, 29, 211, 32, 54, 249, 170, 339, 25, 373, 277, 257, 383, 35, 320, 192, 41, 123, 379, 354, 205, 201, 31, 106] + 18: [271, 43, 46, 30, 29, 375, 380, 346, 347, 142, 174, 328, 339, 171, 234, 338, 334, 203, 355, 195, 155, 210, 231, 168, 212, 366, 316, 257, 245, 213, 291, 242, 320, 131, 341, 10, 359, 280, 238, 67, 343, 286, 99, 158, 337, 19, 26, 23, 248, 326, 173, 183, 175, 270, 156, 353, 159, 76, 299, 70, 285, 145, 133, 246, 0, 367, 1, 374, 143, 304, 2, 83, 91, 186, 219, 134, 268, 97, 151, 227, 352, 224, 92, 44, 95, 36, 150, 20, 45, 49, 59, 74, 75, 94, 101, 127, 260, 192, 35, 117, 118, 207, 228, 119, 266, 11, 342, 205, 217, 208, 356, 294, 122, 237, 373, 163, 124, 193, 96, 82, 324, 53, 236, 108, 33, 18, 22, 56, 166, 325, 12, 66, 78, 152, 382, 252, 349, 7, 62, 269, 28, 112, 39, 58, 225, 114, 170, 261, 37, 85, 305, 379, 106, 167, 214, 162, 48, 128, 289, 81, 313, 15, 9, 357, 301, 47, 182, 218, 13, 292, 120, 196, 276, 141, 265, 194, 298, 4, 364, 303, 232, 104, 350, 107, 222, 132, 161, 165, 187, 197, 220, 230, 139, 185, 169, 251, 34, 65, 68, 31, 284, 146, 189, 164, 16, 361, 77, 61, 149, 288, 300, 89, 275, 369, 297, 102, 282, 335, 41, 235, 264, 40, 93, 206, 370, 272, 333, 72, 123, 279, 60, 188, 360, 358, 345, 216, 42, 202, 14, 32, 79, 181, 250, 223, 51, 24, 121, 383, 54, 240, 157, 116, 372, 129, 377, 178, 177, 312, 281, 241, 64, 253, 233, 90, 259, 371, 221, 307, 52, 184, 21, 317, 179, 153, 130, 290, 332, 209, 3, 296, 254, 247, 258, 274, 293, 311, 314, 322, 138, 98, 71, 69, 113, 144, 376, 330, 331, 50, 255, 368, 211, 363, 351, 176, 365, 63, 229, 6, 103, 136, 80, 215, 110, 256, 148, 190, 84, 263, 306, 105, 329, 244, 262, 8, 199, 308, 38, 87, 86, 57, 267, 160, 319, 154, 201, 310, 243, 17, 239, 115, 111, 302, 283, 27, 135, 5, 278, 315, 200, 191, 147, 25, 172, 126, 277, 381, 109, 318, 100, 273, 348, 295, 140, 137, 204, 125, 198, 321, 88, 180, 226, 73, 378, 249, 55, 309, 287, 323, 327, 336, 340, 344, 354, 362] + 19: [24, 328, 207, 134, 323, 158, 152, 58, 141, 279, 339, 161, 354, 183, 243, 3, 165, 61, 208, 84, 302, 333, 147, 94, 342, 296, 360, 216, 217, 314, 352, 255, 44, 191, 239, 192, 202, 125, 230, 280, 331, 337, 41, 210, 182, 306, 370, 143, 317, 258, 371, 10, 212, 40, 330, 335, 5, 356, 324, 99, 81, 310, 76, 140, 151, 267, 11, 269, 341, 18, 234, 344, 123, 67, 254, 78, 168, 169, 128, 277, 170, 39, 276, 131, 23, 25, 32, 34, 46, 62, 72, 93, 98, 100, 106, 109, 190, 334, 363, 90, 188, 307, 115, 120, 1, 166, 287, 312, 251, 156, 297, 283, 96, 80, 377, 180, 240, 157, 359, 366, 260, 2, 73, 235, 320, 113, 13, 19, 92, 31, 144, 15, 219, 281, 378, 367, 358, 6, 55, 353, 146, 286, 221, 51, 107, 95, 111, 175, 112, 351, 110, 247, 130, 233, 293, 63, 364, 382, 127, 199, 285, 86, 28, 82, 209, 88, 308, 305, 381, 223, 278, 186, 298, 164, 350, 119, 313, 379, 129, 340, 236, 174, 347, 172, 118, 126, 133, 136, 137, 155, 163, 201, 187, 222, 97, 52, 244, 206, 77, 288, 91, 265, 203, 193, 132, 29, 215, 338, 315, 301, 8, 275, 300, 101, 38, 117, 374, 227, 167, 153, 321, 79, 272, 43, 176, 349, 159, 30, 249, 361, 290, 177, 53, 74, 295, 89, 195, 102, 200, 259, 14, 59, 65, 336, 16, 304, 228, 181, 224, 54, 27, 262, 60, 36, 26, 237, 69, 64, 45, 37, 85, 145, 252, 196, 0, 232, 21, 375, 104, 248, 362, 17, 213, 253, 343, 380, 12, 49, 250, 50, 75, 198, 205, 229, 245, 266, 274, 316, 256, 66, 7, 179, 154, 291, 211, 121, 33, 114, 122, 108, 9, 373, 318, 171, 71, 264, 270, 135, 47, 20, 284, 218, 116, 68, 138, 238, 139, 273, 303, 57, 83, 56, 184, 294, 173, 325, 311, 178, 242, 271, 149, 292, 231, 383, 241, 148, 160, 185, 70, 197, 22, 372, 194, 35, 204, 299, 282, 332, 345, 257, 368, 246, 42, 289, 189, 87, 124, 376, 142, 348, 162, 220, 329, 225, 346, 322, 261, 369, 268, 48, 226, 309, 150, 103, 4, 263, 105, 214, 319, 326, 327, 355, 357, 365] + 20: [37, 144, 369, 66, 223, 35, 252, 113, 291, 307, 339, 194, 177, 357, 374, 125, 216, 221, 289, 6, 256, 320, 188, 233, 33, 354, 90, 3, 323, 136, 346, 26, 264, 270, 110, 309, 227, 88, 50, 337, 164, 81, 312, 59, 31, 75, 133, 218, 46, 56, 141, 306, 253, 204, 48, 226, 298, 70, 308, 115, 340, 126, 217, 348, 73, 163, 238, 274, 5, 12, 16, 17, 23, 24, 28, 34, 38, 44, 47, 51, 52, 60, 62, 63, 67, 68, 72, 76, 84, 85, 89, 102, 103, 104, 107, 108, 278, 121, 311, 376, 248, 138, 318, 159, 41, 321, 145, 335, 236, 317, 4, 372, 245, 199, 64, 370, 124, 362, 375, 377, 371, 101, 213, 279, 228, 30, 45, 273, 119, 220, 240, 78, 171, 146, 334, 336, 180, 36, 282, 363, 210, 201, 219, 186, 355, 352, 65, 244, 80, 359, 0, 179, 15, 152, 61, 205, 69, 168, 301, 1, 79, 189, 239, 328, 109, 117, 118, 127, 129, 131, 134, 135, 139, 142, 147, 148, 149, 150, 151, 155, 156, 157, 167, 169, 170, 175, 181, 183, 190, 191, 192, 196, 116, 111, 122, 160, 324, 195, 325, 182, 187, 229, 351, 174, 58, 341, 292, 232, 99, 123, 158, 77, 259, 178, 209, 293, 284, 207, 246, 162, 93, 8, 140, 345, 14, 235, 184, 91, 13, 338, 267, 39, 130, 92, 258, 40, 185, 120, 7, 254, 18, 42, 272, 202, 54, 350, 222, 208, 55, 211, 74, 281, 83, 197, 343, 27, 96, 212, 251, 361, 198, 206, 214, 215, 230, 231, 234, 237, 241, 242, 243, 247, 249, 255, 257, 261, 262, 263, 266, 269, 271, 276, 277, 280, 283, 285, 286, 288, 381, 105, 161, 10, 82, 106, 2, 112, 97, 71, 290, 287, 353, 43, 95, 315, 193, 57, 25, 132, 380, 153, 342, 322, 364, 172, 378, 11, 86, 347, 200, 295, 49, 87, 165, 100, 302, 98, 114, 379, 32, 250, 19, 22, 319, 313, 368, 203, 173, 53, 305, 265, 9, 21, 154, 275, 268, 94, 128, 310, 143, 29, 176, 20, 166, 225, 365, 137, 224, 260, 366, 294, 296, 297, 299, 300, 303, 304, 314, 316, 326, 327, 329, 330, 331, 332, 333, 344, 349, 356, 358, 360, 367, 373, 382, 383] + 21: [371, 361, 250, 45, 262, 355, 245, 237, 141, 94, 29, 155, 365, 187, 25, 359, 367, 308, 162, 271, 159, 337, 351, 128, 287, 326, 132, 194, 43, 133, 22, 14, 98, 26, 183, 15, 383, 148, 354, 23, 336, 139, 358, 286, 85, 210, 9, 280, 44, 40, 324, 188, 203, 382, 161, 290, 191, 299, 156, 64, 229, 207, 363, 259, 289, 345, 123, 234, 166, 332, 35, 163, 310, 131, 243, 369, 212, 236, 223, 108, 157, 3, 7, 10, 11, 17, 32, 36, 42, 56, 58, 59, 61, 70, 93, 104, 158, 256, 270, 232, 112, 379, 277, 297, 66, 304, 213, 149, 364, 349, 55, 205, 233, 225, 150, 247, 208, 178, 189, 267, 374, 257, 252, 242, 5, 171, 258, 318, 4, 196, 227, 34, 238, 74, 316, 261, 97, 38, 263, 86, 312, 105, 380, 170, 126, 52, 303, 186, 248, 47, 119, 174, 314, 147, 192, 222, 91, 53, 117, 77, 21, 333, 30, 48, 114, 323, 330, 373, 130, 296, 99, 101, 90, 344, 334, 285, 73, 62, 68, 228, 217, 107, 125, 136, 164, 167, 179, 181, 184, 198, 215, 216, 152, 121, 214, 20, 200, 124, 230, 87, 175, 279, 218, 65, 135, 88, 240, 92, 377, 362, 103, 100, 82, 8, 295, 115, 276, 60, 211, 24, 113, 63, 346, 328, 173, 195, 190, 360, 226, 321, 255, 291, 224, 375, 102, 320, 120, 300, 209, 160, 28, 185, 84, 153, 67, 96, 1, 370, 313, 54, 302, 146, 50, 244, 110, 273, 357, 356, 76, 78, 41, 331, 39, 239, 6, 180, 353, 116, 151, 137, 197, 288, 251, 246, 127, 134, 37, 122, 294, 219, 264, 265, 266, 269, 272, 278, 282, 292, 281, 51, 182, 254, 80, 2, 199, 284, 352, 172, 221, 220, 71, 378, 376, 144, 19, 129, 293, 89, 348, 143, 12, 46, 33, 13, 325, 169, 16, 341, 366, 275, 18, 368, 75, 274, 342, 301, 83, 206, 27, 204, 372, 0, 140, 118, 168, 165, 202, 176, 260, 106, 340, 347, 142, 249, 298, 72, 109, 111, 338, 307, 57, 235, 69, 95, 81, 79, 283, 381, 317, 241, 309, 145, 177, 253, 31, 193, 231, 306, 268, 343, 154, 201, 49, 138, 329, 305, 311, 315, 319, 322, 327, 335, 339, 350] + 22: [73, 118, 329, 164, 226, 376, 51, 197, 15, 126, 170, 132, 251, 180, 305, 21, 330, 122, 179, 314, 205, 371, 184, 56, 59, 326, 352, 189, 68, 264, 255, 268, 347, 362, 69, 78, 292, 46, 19, 99, 322, 171, 232, 152, 138, 8, 85, 319, 369, 28, 340, 327, 277, 219, 18, 90, 137, 97, 168, 158, 301, 378, 26, 203, 0, 10, 140, 295, 1, 7, 12, 14, 30, 34, 35, 39, 40, 41, 43, 45, 47, 48, 49, 52, 55, 77, 81, 84, 86, 91, 93, 95, 96, 100, 102, 103, 209, 328, 224, 148, 153, 240, 216, 178, 80, 61, 139, 339, 299, 79, 215, 318, 221, 201, 252, 306, 214, 361, 50, 238, 151, 281, 293, 289, 58, 334, 37, 200, 66, 76, 38, 191, 294, 156, 227, 309, 272, 94, 92, 33, 188, 70, 22, 173, 6, 89, 364, 3, 218, 320, 379, 242, 32, 98, 287, 75, 36, 88, 206, 303, 2, 67, 235, 62, 150, 300, 104, 105, 107, 114, 119, 120, 121, 124, 128, 129, 131, 135, 136, 142, 143, 145, 146, 154, 157, 159, 161, 162, 165, 166, 167, 176, 343, 141, 357, 125, 348, 63, 24, 304, 111, 345, 110, 333, 198, 274, 351, 311, 54, 278, 244, 183, 42, 367, 193, 307, 174, 134, 72, 82, 123, 195, 366, 20, 273, 279, 11, 324, 363, 163, 5, 60, 115, 285, 283, 108, 269, 130, 365, 172, 263, 321, 25, 17, 31, 116, 249, 296, 53, 302, 335, 286, 64, 101, 213, 332, 13, 83, 239, 65, 233, 358, 181, 182, 186, 187, 190, 192, 194, 199, 202, 208, 217, 222, 223, 230, 234, 236, 237, 241, 243, 245, 247, 248, 250, 253, 256, 257, 317, 207, 260, 144, 147, 169, 204, 337, 177, 288, 74, 113, 261, 355, 381, 377, 175, 275, 331, 349, 246, 336, 211, 155, 27, 338, 228, 341, 312, 225, 220, 276, 133, 29, 229, 258, 109, 316, 254, 160, 87, 259, 106, 382, 360, 374, 23, 149, 9, 212, 4, 44, 298, 210, 57, 185, 380, 284, 383, 196, 117, 112, 231, 356, 16, 127, 325, 71, 270, 373, 262, 265, 266, 267, 271, 280, 282, 290, 291, 297, 308, 310, 313, 315, 323, 342, 344, 346, 350, 353, 354, 359, 368, 370, 372, 375] + 23: [172, 232, 374, 82, 86, 49, 170, 203, 123, 4, 299, 254, 279, 370, 100, 230, 292, 261, 1, 36, 132, 371, 336, 185, 217, 201, 219, 270, 125, 210, 349, 308, 361, 21, 282, 164, 341, 335, 0, 84, 296, 277, 30, 216, 354, 24, 151, 77, 97, 221, 145, 225, 91, 351, 15, 87, 169, 8, 206, 350, 284, 44, 41, 199, 149, 33, 209, 139, 278, 207, 96, 177, 37, 179, 150, 259, 188, 141, 345, 121, 120, 250, 303, 236, 75, 242, 13, 14, 29, 31, 64, 88, 110, 113, 157, 168, 26, 327, 5, 146, 274, 63, 339, 133, 295, 156, 340, 193, 180, 222, 78, 54, 329, 342, 297, 117, 38, 196, 346, 98, 50, 355, 147, 249, 247, 42, 127, 119, 122, 262, 32, 67, 239, 115, 108, 138, 287, 256, 109, 165, 101, 19, 154, 17, 306, 47, 103, 106, 45, 362, 365, 360, 233, 328, 255, 289, 23, 85, 271, 215, 40, 300, 227, 375, 280, 118, 128, 28, 382, 46, 311, 364, 160, 27, 211, 356, 372, 135, 61, 3, 316, 171, 114, 83, 89, 363, 198, 202, 214, 235, 265, 276, 167, 80, 358, 220, 111, 337, 197, 94, 253, 70, 130, 229, 309, 237, 184, 204, 52, 264, 307, 218, 74, 379, 373, 48, 53, 290, 324, 315, 331, 107, 263, 79, 353, 10, 73, 240, 245, 93, 269, 321, 228, 182, 7, 51, 258, 344, 191, 190, 367, 99, 159, 326, 174, 241, 55, 140, 267, 377, 153, 334, 20, 238, 305, 369, 9, 347, 266, 12, 137, 275, 381, 183, 60, 333, 260, 43, 318, 298, 205, 310, 293, 243, 294, 56, 57, 223, 192, 383, 212, 272, 134, 268, 288, 291, 301, 319, 352, 131, 314, 187, 176, 148, 213, 173, 72, 378, 102, 359, 285, 158, 58, 2, 376, 92, 155, 35, 273, 143, 313, 162, 332, 11, 322, 208, 6, 152, 195, 22, 304, 81, 163, 338, 317, 312, 161, 244, 323, 90, 124, 325, 257, 105, 104, 348, 248, 76, 226, 380, 283, 178, 234, 25, 252, 200, 66, 166, 62, 95, 320, 302, 129, 186, 116, 112, 343, 175, 142, 68, 224, 194, 246, 126, 59, 71, 18, 16, 144, 231, 65, 281, 181, 34, 357, 69, 39, 251, 136, 189, 286, 330, 366, 368] + 24: [70, 213, 122, 220, 237, 1, 226, 211, 202, 316, 210, 371, 257, 12, 182, 178, 381, 93, 8, 6, 41, 38, 382, 50, 258, 114, 181, 37, 88, 204, 262, 110, 203, 18, 357, 188, 345, 31, 306, 61, 366, 168, 347, 358, 218, 349, 155, 215, 108, 326, 145, 275, 286, 183, 310, 130, 185, 373, 106, 243, 90, 40, 47, 184, 285, 153, 120, 105, 305, 162, 298, 100, 376, 55, 56, 271, 160, 225, 10, 23, 32, 34, 36, 45, 57, 69, 74, 75, 76, 83, 98, 113, 115, 125, 129, 132, 246, 343, 301, 131, 290, 375, 283, 244, 344, 325, 137, 97, 350, 139, 201, 200, 287, 330, 319, 144, 263, 63, 269, 147, 84, 308, 4, 297, 333, 27, 190, 221, 189, 248, 229, 295, 96, 20, 261, 156, 302, 255, 216, 87, 157, 360, 224, 163, 146, 327, 117, 377, 152, 317, 328, 140, 66, 242, 186, 284, 126, 208, 133, 372, 362, 119, 197, 165, 228, 127, 300, 16, 25, 214, 85, 179, 369, 177, 81, 167, 60, 359, 176, 227, 143, 149, 161, 166, 169, 170, 173, 194, 196, 217, 240, 241, 0, 46, 5, 109, 195, 266, 118, 121, 28, 82, 172, 35, 142, 104, 318, 135, 299, 231, 256, 150, 95, 9, 15, 264, 289, 116, 253, 128, 73, 324, 151, 159, 293, 26, 39, 273, 282, 14, 44, 79, 268, 277, 103, 361, 124, 192, 365, 58, 307, 171, 315, 72, 320, 348, 338, 91, 64, 280, 30, 233, 279, 89, 54, 3, 33, 335, 234, 340, 199, 123, 193, 21, 134, 223, 48, 232, 368, 29, 77, 219, 180, 68, 2, 99, 187, 239, 249, 250, 251, 267, 270, 278, 281, 291, 303, 312, 92, 274, 154, 311, 265, 259, 86, 337, 175, 354, 43, 370, 136, 7, 342, 212, 112, 230, 323, 341, 355, 374, 51, 336, 49, 352, 62, 276, 111, 78, 254, 80, 53, 252, 321, 294, 138, 52, 260, 42, 191, 65, 11, 22, 59, 332, 102, 334, 13, 207, 71, 288, 205, 380, 236, 209, 322, 206, 245, 164, 346, 238, 67, 148, 235, 247, 363, 339, 313, 329, 19, 101, 174, 304, 17, 272, 158, 292, 94, 296, 222, 24, 141, 107, 198, 309, 314, 331, 351, 353, 356, 364, 367, 378, 379, 383] + 25: [379, 328, 80, 307, 327, 351, 89, 131, 359, 37, 222, 188, 382, 238, 365, 51, 340, 299, 194, 204, 196, 77, 68, 82, 192, 224, 0, 244, 40, 272, 24, 242, 305, 54, 201, 363, 228, 260, 368, 78, 35, 259, 383, 308, 128, 282, 312, 170, 109, 302, 30, 300, 235, 288, 23, 1, 292, 345, 94, 186, 112, 61, 184, 86, 58, 135, 198, 321, 344, 182, 189, 171, 338, 208, 213, 169, 177, 255, 104, 207, 263, 356, 3, 43, 87, 91, 95, 99, 119, 123, 174, 219, 286, 326, 329, 358, 269, 334, 167, 231, 140, 149, 215, 101, 366, 250, 309, 217, 120, 29, 218, 310, 16, 252, 11, 180, 111, 316, 336, 343, 296, 33, 136, 39, 57, 15, 8, 113, 153, 117, 348, 195, 81, 59, 306, 70, 214, 65, 144, 133, 63, 337, 197, 53, 145, 246, 41, 60, 294, 175, 258, 283, 187, 240, 339, 173, 369, 129, 349, 251, 254, 75, 88, 34, 139, 199, 205, 239, 352, 121, 55, 372, 209, 126, 122, 22, 277, 364, 243, 206, 322, 342, 267, 21, 355, 67, 13, 178, 164, 12, 264, 362, 232, 315, 193, 168, 31, 317, 9, 319, 287, 253, 64, 314, 211, 375, 25, 2, 124, 32, 146, 98, 320, 143, 73, 229, 52, 298, 261, 381, 20, 200, 361, 230, 293, 79, 132, 265, 191, 156, 225, 247, 270, 237, 125, 172, 203, 62, 74, 324, 6, 202, 116, 102, 69, 221, 281, 274, 311, 159, 318, 49, 353, 367, 138, 148, 373, 165, 83, 137, 157, 341, 210, 18, 234, 158, 71, 227, 85, 284, 50, 14, 19, 163, 92, 26, 150, 325, 160, 155, 303, 127, 257, 220, 118, 374, 152, 301, 354, 107, 72, 56, 278, 271, 332, 162, 27, 5, 291, 161, 76, 114, 280, 48, 266, 256, 262, 333, 313, 377, 115, 371, 100, 304, 295, 285, 268, 347, 46, 190, 44, 183, 241, 42, 273, 17, 335, 38, 10, 47, 103, 7, 84, 212, 97, 66, 28, 279, 216, 110, 4, 297, 108, 330, 276, 179, 249, 185, 346, 130, 350, 360, 380, 376, 142, 357, 105, 290, 248, 96, 275, 154, 370, 331, 233, 236, 378, 93, 166, 36, 289, 181, 323, 45, 90, 106, 147, 176, 223, 151, 134, 245, 226, 141] + 26: [217, 249, 113, 92, 128, 48, 328, 195, 108, 237, 293, 93, 272, 371, 367, 67, 216, 277, 188, 224, 318, 135, 329, 30, 2, 221, 126, 74, 232, 148, 44, 11, 77, 266, 94, 76, 223, 127, 134, 378, 159, 152, 192, 20, 308, 9, 370, 103, 335, 117, 331, 353, 212, 171, 349, 375, 352, 376, 301, 66, 88, 101, 325, 351, 98, 157, 64, 57, 305, 10, 43, 187, 313, 1, 3, 4, 14, 19, 22, 23, 25, 28, 29, 35, 47, 52, 54, 56, 58, 60, 63, 68, 73, 81, 84, 85, 90, 281, 27, 142, 337, 262, 89, 254, 278, 300, 294, 215, 15, 204, 137, 310, 62, 359, 183, 256, 342, 279, 174, 269, 55, 364, 110, 239, 184, 34, 231, 124, 156, 186, 220, 315, 24, 69, 245, 107, 116, 176, 327, 280, 5, 317, 208, 163, 6, 291, 314, 296, 298, 198, 160, 13, 97, 302, 100, 206, 109, 123, 46, 211, 168, 12, 377, 191, 111, 36, 326, 86, 225, 324, 91, 96, 115, 119, 121, 122, 132, 139, 141, 143, 145, 147, 150, 153, 169, 180, 185, 193, 194, 197, 199, 203, 380, 304, 227, 271, 120, 173, 360, 125, 21, 290, 170, 102, 251, 140, 138, 83, 182, 78, 165, 179, 282, 265, 344, 80, 236, 196, 59, 202, 334, 356, 230, 238, 382, 32, 258, 343, 241, 222, 275, 41, 233, 114, 363, 65, 190, 8, 154, 229, 289, 7, 161, 40, 244, 16, 37, 316, 144, 50, 347, 205, 149, 164, 366, 248, 285, 71, 49, 379, 247, 75, 332, 105, 255, 345, 207, 209, 210, 213, 218, 219, 228, 234, 235, 240, 242, 243, 246, 250, 252, 257, 259, 261, 264, 267, 268, 270, 136, 181, 338, 276, 214, 118, 146, 200, 112, 172, 178, 53, 38, 31, 70, 106, 341, 151, 131, 39, 61, 95, 323, 339, 321, 226, 263, 167, 166, 306, 355, 357, 162, 177, 18, 311, 79, 33, 129, 45, 130, 383, 336, 158, 320, 51, 87, 286, 82, 292, 312, 361, 17, 374, 0, 307, 303, 155, 340, 201, 72, 26, 354, 104, 99, 369, 175, 365, 253, 260, 189, 42, 133, 284, 346, 273, 274, 283, 287, 288, 295, 297, 299, 309, 319, 322, 330, 333, 348, 350, 358, 362, 368, 372, 373, 381] + 27: [103, 325, 10, 228, 50, 305, 133, 304, 371, 140, 28, 151, 171, 269, 78, 29, 178, 111, 131, 374, 37, 27, 324, 296, 49, 323, 105, 244, 109, 58, 132, 121, 254, 95, 266, 288, 52, 114, 321, 306, 327, 64, 316, 261, 226, 62, 232, 134, 60, 76, 284, 335, 146, 202, 170, 301, 139, 1, 136, 345, 61, 164, 283, 219, 44, 147, 200, 355, 231, 172, 287, 166, 130, 91, 41, 275, 319, 17, 154, 315, 63, 148, 243, 169, 367, 8, 51, 53, 79, 96, 137, 182, 206, 212, 255, 257, 32, 360, 98, 347, 89, 113, 141, 42, 25, 168, 349, 127, 252, 197, 73, 370, 379, 224, 277, 356, 116, 150, 71, 190, 188, 125, 120, 351, 250, 292, 218, 6, 217, 242, 106, 238, 173, 308, 294, 193, 162, 70, 334, 75, 68, 369, 149, 365, 93, 233, 31, 357, 124, 350, 207, 180, 189, 330, 67, 38, 11, 87, 208, 271, 279, 340, 14, 184, 352, 235, 122, 145, 20, 297, 331, 358, 186, 227, 363, 326, 138, 346, 36, 156, 220, 336, 101, 161, 256, 286, 203, 39, 4, 163, 270, 302, 225, 214, 12, 23, 260, 251, 16, 329, 85, 337, 118, 21, 303, 128, 353, 104, 198, 55, 343, 88, 258, 354, 48, 159, 259, 267, 187, 90, 339, 348, 94, 83, 376, 13, 181, 320, 280, 115, 5, 237, 86, 341, 272, 3, 185, 82, 199, 264, 80, 248, 84, 229, 281, 215, 377, 56, 99, 33, 9, 282, 239, 45, 97, 195, 313, 192, 26, 191, 69, 15, 299, 194, 167, 276, 155, 373, 278, 312, 24, 157, 205, 201, 307, 152, 117, 74, 247, 160, 285, 92, 240, 129, 300, 236, 332, 338, 333, 81, 177, 66, 222, 175, 265, 234, 213, 72, 230, 54, 7, 381, 0, 273, 143, 18, 317, 328, 362, 249, 263, 47, 34, 245, 183, 110, 318, 383, 344, 65, 246, 30, 204, 368, 210, 102, 382, 380, 291, 298, 295, 290, 211, 112, 274, 142, 135, 179, 309, 262, 241, 144, 35, 77, 126, 359, 366, 123, 158, 174, 375, 293, 289, 176, 22, 196, 108, 119, 216, 107, 59, 165, 314, 253, 153, 223, 268, 322, 221, 2, 364, 19, 46, 57, 100, 40, 361, 310, 209, 311, 43, 342, 372, 378] + 28: [310, 312, 201, 285, 179, 67, 3, 21, 148, 22, 112, 324, 243, 133, 224, 51, 252, 210, 25, 237, 7, 227, 321, 327, 144, 366, 221, 375, 1, 320, 268, 218, 158, 378, 70, 368, 311, 162, 130, 187, 104, 297, 209, 207, 223, 214, 77, 247, 98, 337, 125, 260, 177, 193, 184, 316, 334, 269, 63, 15, 374, 296, 56, 65, 283, 295, 82, 190, 246, 9, 258, 188, 46, 18, 315, 10, 85, 372, 139, 354, 145, 254, 89, 73, 336, 34, 123, 27, 270, 54, 212, 279, 96, 147, 160, 257, 317, 121, 197, 97, 176, 66, 289, 356, 100, 182, 16, 24, 170, 52, 369, 267, 370, 90, 241, 303, 166, 57, 341, 116, 53, 110, 32, 350, 314, 228, 5, 64, 44, 259, 349, 213, 80, 274, 363, 128, 230, 383, 308, 59, 171, 167, 181, 191, 273, 211, 276, 286, 37, 322, 272, 149, 300, 88, 131, 81, 345, 265, 359, 234, 192, 156, 309, 361, 76, 159, 189, 6, 151, 60, 39, 284, 351, 244, 103, 373, 12, 132, 118, 292, 94, 36, 115, 275, 173, 29, 256, 301, 290, 294, 298, 313, 253, 26, 0, 250, 307, 196, 169, 71, 376, 13, 165, 48, 205, 72, 172, 183, 262, 288, 157, 380, 20, 357, 107, 134, 305, 318, 175, 174, 352, 264, 362, 95, 153, 219, 346, 333, 238, 239, 137, 117, 233, 367, 280, 43, 17, 91, 99, 114, 225, 331, 120, 266, 249, 277, 271, 122, 152, 371, 140, 178, 358, 216, 281, 79, 329, 323, 339, 69, 245, 23, 33, 31, 106, 232, 119, 236, 291, 40, 55, 78, 126, 62, 231, 47, 141, 93, 186, 150, 365, 11, 146, 338, 348, 342, 343, 355, 255, 220, 235, 135, 328, 199, 163, 41, 302, 30, 222, 195, 108, 168, 278, 203, 344, 204, 326, 155, 50, 109, 226, 194, 92, 248, 105, 200, 263, 42, 242, 217, 61, 154, 306, 287, 206, 161, 240, 185, 340, 379, 330, 113, 129, 143, 4, 102, 14, 49, 229, 293, 208, 347, 198, 19, 35, 28, 124, 136, 332, 68, 38, 180, 58, 111, 202, 282, 299, 325, 84, 364, 382, 319, 74, 215, 87, 45, 251, 101, 8, 335, 142, 138, 353, 127, 2, 304, 86, 83, 164, 75, 261, 360, 377, 381] + 29: [197, 83, 277, 346, 140, 11, 132, 218, 116, 301, 180, 73, 253, 190, 214, 86, 19, 185, 349, 22, 62, 286, 113, 300, 318, 139, 175, 145, 3, 101, 340, 81, 353, 276, 2, 275, 187, 203, 287, 350, 146, 303, 94, 115, 198, 229, 5, 126, 144, 160, 103, 166, 290, 307, 117, 56, 359, 37, 322, 257, 309, 281, 51, 155, 252, 77, 68, 12, 194, 249, 192, 335, 15, 130, 321, 246, 107, 31, 82, 376, 6, 8, 10, 13, 29, 47, 69, 71, 79, 91, 96, 121, 138, 164, 165, 178, 127, 357, 314, 283, 255, 232, 4, 269, 110, 245, 215, 207, 59, 244, 195, 339, 18, 172, 345, 120, 99, 206, 330, 50, 55, 90, 250, 34, 235, 278, 291, 271, 361, 295, 325, 317, 188, 125, 323, 57, 142, 65, 143, 137, 64, 292, 167, 254, 52, 263, 299, 30, 92, 184, 289, 39, 227, 179, 355, 199, 338, 61, 364, 363, 16, 26, 98, 348, 123, 157, 268, 226, 134, 328, 191, 216, 381, 141, 372, 284, 35, 0, 306, 326, 17, 28, 239, 334, 135, 181, 95, 20, 66, 201, 212, 224, 74, 243, 311, 294, 85, 365, 280, 270, 312, 362, 375, 341, 148, 53, 259, 14, 324, 119, 360, 293, 228, 231, 327, 344, 36, 87, 333, 25, 354, 374, 196, 186, 208, 104, 320, 304, 370, 251, 124, 105, 273, 236, 129, 262, 156, 72, 7, 221, 319, 154, 274, 111, 264, 161, 308, 233, 222, 75, 67, 225, 209, 237, 298, 168, 131, 21, 63, 379, 23, 242, 234, 1, 373, 42, 122, 70, 329, 315, 93, 9, 106, 238, 282, 136, 24, 152, 337, 49, 205, 27, 153, 247, 305, 240, 260, 261, 174, 351, 266, 97, 128, 88, 40, 58, 248, 297, 352, 118, 382, 162, 149, 33, 44, 367, 48, 366, 78, 43, 151, 210, 223, 60, 369, 183, 310, 267, 219, 313, 279, 258, 80, 46, 220, 347, 114, 177, 368, 163, 343, 54, 316, 200, 32, 133, 112, 272, 159, 332, 380, 296, 171, 76, 45, 256, 100, 182, 204, 173, 383, 213, 241, 230, 41, 147, 84, 265, 193, 211, 109, 38, 189, 302, 358, 170, 108, 158, 217, 102, 336, 342, 89, 150, 202, 356, 331, 371, 176, 169, 378, 285, 288, 377] + 30: [239, 62, 144, 166, 28, 213, 362, 230, 298, 337, 60, 24, 272, 253, 198, 48, 246, 358, 292, 378, 303, 116, 163, 260, 193, 184, 289, 12, 273, 361, 196, 153, 212, 27, 147, 327, 96, 86, 352, 235, 300, 162, 141, 115, 247, 256, 41, 122, 309, 324, 133, 328, 43, 156, 5, 112, 132, 91, 173, 329, 72, 107, 299, 209, 342, 179, 314, 307, 367, 8, 148, 252, 19, 368, 231, 93, 42, 366, 75, 16, 295, 178, 114, 192, 313, 98, 297, 30, 21, 31, 32, 34, 36, 69, 79, 81, 37, 296, 233, 111, 6, 195, 248, 265, 318, 109, 245, 249, 250, 223, 52, 187, 97, 68, 221, 288, 63, 293, 200, 308, 160, 151, 139, 92, 172, 263, 217, 359, 261, 7, 283, 341, 51, 110, 129, 183, 325, 312, 332, 274, 199, 291, 128, 113, 170, 150, 335, 174, 104, 13, 87, 316, 157, 370, 26, 67, 25, 319, 154, 108, 39, 267, 186, 365, 229, 372, 301, 131, 219, 18, 176, 340, 57, 136, 134, 380, 374, 9, 56, 117, 232, 47, 149, 4, 169, 82, 88, 100, 106, 119, 127, 145, 140, 137, 65, 50, 53, 262, 121, 78, 11, 258, 38, 90, 152, 77, 14, 281, 89, 171, 224, 346, 158, 242, 351, 349, 66, 211, 142, 29, 210, 1, 167, 125, 376, 241, 205, 138, 99, 234, 320, 348, 161, 3, 220, 20, 143, 373, 271, 302, 315, 306, 379, 286, 343, 282, 227, 257, 226, 40, 215, 331, 240, 55, 103, 338, 311, 95, 275, 76, 181, 350, 165, 182, 243, 326, 251, 177, 330, 323, 126, 268, 83, 194, 276, 382, 218, 334, 146, 322, 228, 180, 185, 189, 190, 201, 237, 259, 208, 222, 277, 254, 381, 206, 2, 46, 317, 35, 164, 344, 202, 225, 357, 135, 383, 64, 10, 266, 23, 321, 203, 284, 197, 105, 244, 294, 71, 44, 285, 371, 280, 124, 155, 102, 214, 94, 369, 353, 333, 130, 84, 347, 33, 279, 175, 15, 287, 336, 360, 58, 120, 74, 45, 339, 304, 0, 290, 59, 236, 70, 270, 364, 255, 85, 188, 207, 80, 17, 159, 355, 216, 356, 264, 123, 49, 118, 375, 61, 238, 204, 73, 101, 168, 305, 54, 191, 22, 310, 269, 278, 345, 354, 363, 377] + 31: [288, 361, 123, 160, 82, 339, 176, 88, 240, 284, 146, 169, 60, 86, 70, 378, 61, 375, 45, 335, 277, 296, 336, 227, 290, 287, 177, 190, 2, 40, 195, 96, 78, 62, 275, 244, 34, 122, 267, 143, 357, 93, 219, 111, 105, 173, 8, 52, 249, 364, 368, 334, 319, 158, 145, 19, 344, 234, 209, 117, 136, 194, 382, 255, 273, 116, 247, 302, 36, 17, 212, 118, 216, 85, 80, 370, 217, 213, 300, 13, 35, 329, 120, 165, 73, 239, 127, 228, 24, 203, 95, 181, 347, 175, 207, 301, 248, 299, 253, 112, 153, 326, 124, 260, 383, 186, 252, 59, 317, 321, 183, 377, 269, 307, 359, 338, 87, 3, 178, 223, 131, 140, 272, 263, 57, 47, 79, 238, 292, 220, 200, 188, 44, 243, 320, 128, 332, 109, 167, 27, 29, 346, 144, 65, 107, 257, 171, 66, 137, 106, 254, 304, 15, 291, 193, 81, 282, 328, 362, 180, 351, 310, 148, 198, 179, 265, 280, 129, 215, 1, 199, 197, 259, 226, 168, 185, 113, 142, 68, 51, 298, 149, 246, 10, 318, 210, 352, 11, 343, 54, 162, 67, 159, 365, 360, 309, 125, 108, 39, 222, 75, 314, 354, 241, 102, 134, 191, 206, 214, 201, 58, 98, 261, 72, 139, 49, 110, 281, 84, 278, 293, 232, 224, 182, 196, 38, 379, 147, 174, 274, 312, 6, 4, 16, 286, 130, 77, 353, 56, 271, 12, 33, 92, 43, 18, 285, 31, 325, 211, 20, 250, 155, 297, 306, 303, 163, 91, 311, 337, 152, 99, 184, 374, 104, 69, 340, 161, 100, 356, 205, 225, 251, 294, 218, 341, 187, 381, 322, 90, 114, 372, 235, 135, 262, 172, 126, 229, 89, 115, 121, 237, 380, 138, 156, 376, 221, 264, 231, 23, 245, 133, 55, 25, 202, 270, 305, 26, 32, 316, 313, 349, 366, 0, 330, 170, 236, 164, 9, 355, 373, 103, 22, 324, 189, 345, 48, 276, 208, 21, 83, 348, 371, 157, 192, 141, 119, 258, 233, 266, 327, 242, 94, 97, 71, 331, 154, 166, 150, 333, 53, 363, 256, 308, 30, 342, 295, 63, 5, 367, 74, 268, 7, 46, 64, 76, 323, 41, 279, 283, 289, 369, 50, 14, 151, 42, 132, 358, 315, 101, 350, 230, 204, 28, 37] + 32: [148, 89, 83, 86, 238, 33, 237, 166, 45, 218, 120, 307, 348, 179, 175, 105, 221, 16, 194, 7, 334, 282, 190, 293, 322, 151, 53, 77, 380, 88, 110, 254, 60, 212, 126, 319, 87, 76, 150, 41, 213, 340, 231, 240, 377, 79, 280, 69, 292, 1, 129, 306, 32, 369, 18, 195, 121, 270, 315, 331, 178, 233, 154, 9, 128, 114, 370, 52, 248, 226, 314, 133, 199, 374, 31, 354, 3, 176, 350, 259, 379, 366, 295, 162, 260, 267, 97, 74, 38, 242, 62, 235, 11, 78, 117, 147, 106, 108, 227, 347, 55, 4, 229, 336, 335, 48, 171, 202, 40, 249, 58, 258, 243, 200, 136, 113, 153, 273, 261, 241, 368, 209, 21, 96, 19, 371, 297, 323, 68, 360, 49, 268, 274, 82, 339, 247, 342, 345, 51, 364, 278, 294, 232, 107, 122, 329, 70, 101, 239, 356, 10, 201, 35, 100, 164, 337, 134, 34, 90, 266, 288, 223, 376, 355, 119, 262, 43, 224, 152, 98, 289, 308, 37, 183, 296, 93, 91, 118, 244, 302, 245, 2, 80, 299, 198, 251, 135, 316, 161, 169, 180, 264, 163, 184, 367, 63, 168, 281, 157, 359, 324, 276, 160, 284, 85, 191, 131, 250, 139, 142, 219, 332, 214, 187, 256, 141, 286, 193, 36, 230, 330, 149, 59, 317, 72, 211, 127, 326, 279, 186, 67, 23, 271, 203, 353, 351, 263, 8, 298, 349, 313, 12, 42, 215, 197, 103, 22, 145, 343, 73, 66, 210, 170, 39, 328, 167, 321, 310, 225, 320, 46, 125, 275, 64, 358, 216, 318, 309, 124, 13, 54, 333, 47, 28, 112, 253, 123, 300, 44, 192, 14, 255, 291, 338, 272, 277, 290, 301, 362, 344, 352, 26, 24, 172, 99, 382, 155, 174, 285, 325, 252, 92, 5, 146, 143, 228, 156, 111, 165, 283, 6, 361, 206, 144, 75, 0, 303, 130, 205, 182, 327, 365, 109, 189, 137, 30, 207, 346, 95, 208, 140, 61, 378, 115, 246, 159, 234, 29, 204, 217, 25, 287, 50, 222, 116, 104, 357, 158, 196, 132, 94, 65, 269, 138, 363, 311, 102, 341, 373, 188, 265, 71, 181, 17, 15, 173, 84, 236, 81, 57, 372, 185, 304, 257, 177, 305, 27, 56, 20, 220, 381, 312, 375, 383] + 33: [91, 169, 377, 344, 342, 307, 38, 158, 51, 87, 255, 101, 94, 288, 191, 62, 14, 134, 323, 305, 132, 25, 79, 188, 170, 45, 182, 149, 318, 358, 347, 22, 233, 52, 85, 109, 67, 361, 72, 163, 141, 258, 335, 340, 267, 295, 212, 380, 360, 264, 123, 2, 226, 20, 222, 187, 362, 337, 356, 23, 197, 18, 110, 351, 143, 155, 312, 11, 116, 370, 284, 330, 136, 228, 53, 63, 7, 208, 78, 219, 332, 353, 282, 243, 24, 365, 310, 227, 15, 44, 32, 71, 95, 105, 128, 185, 177, 234, 167, 154, 66, 328, 338, 303, 261, 139, 229, 135, 183, 98, 41, 339, 273, 150, 204, 213, 130, 37, 8, 90, 382, 334, 202, 160, 240, 291, 343, 174, 374, 363, 6, 179, 84, 369, 224, 294, 138, 111, 9, 168, 359, 118, 12, 92, 10, 271, 100, 74, 315, 215, 157, 203, 50, 259, 248, 193, 235, 88, 230, 147, 21, 176, 27, 327, 241, 30, 16, 120, 201, 283, 251, 218, 126, 194, 189, 252, 244, 345, 114, 198, 97, 47, 119, 69, 253, 56, 186, 223, 266, 287, 298, 306, 127, 190, 262, 280, 89, 232, 59, 4, 270, 286, 319, 237, 320, 260, 381, 348, 151, 125, 217, 61, 153, 104, 31, 354, 57, 172, 80, 297, 205, 199, 254, 60, 124, 355, 40, 148, 275, 42, 162, 268, 129, 166, 39, 77, 366, 281, 206, 17, 250, 70, 164, 352, 103, 256, 313, 152, 367, 178, 290, 137, 214, 296, 209, 102, 195, 207, 26, 28, 173, 145, 112, 285, 161, 33, 184, 324, 292, 272, 372, 265, 156, 49, 304, 211, 257, 274, 13, 142, 29, 302, 236, 245, 309, 317, 326, 333, 144, 64, 113, 376, 48, 115, 108, 210, 81, 316, 314, 322, 5, 35, 311, 73, 325, 192, 375, 36, 321, 54, 238, 308, 289, 133, 121, 379, 146, 93, 247, 200, 299, 3, 329, 58, 1, 196, 96, 159, 75, 331, 181, 131, 99, 231, 349, 364, 249, 180, 171, 276, 269, 19, 300, 46, 246, 107, 43, 263, 76, 122, 371, 86, 68, 83, 82, 279, 350, 117, 65, 165, 242, 55, 341, 378, 216, 368, 221, 346, 175, 239, 225, 106, 140, 278, 277, 220, 0, 383, 336, 301, 34, 293, 357, 373] + 34: [179, 239, 136, 360, 379, 74, 283, 81, 293, 331, 294, 188, 298, 192, 348, 260, 332, 65, 80, 85, 23, 79, 104, 190, 97, 110, 148, 195, 342, 326, 120, 232, 364, 245, 131, 324, 314, 221, 270, 382, 155, 146, 305, 27, 142, 355, 344, 184, 267, 92, 353, 114, 210, 278, 118, 199, 170, 319, 77, 187, 67, 241, 227, 176, 209, 13, 175, 191, 50, 102, 68, 318, 115, 111, 35, 234, 316, 216, 215, 39, 252, 141, 11, 106, 189, 121, 309, 14, 19, 26, 38, 40, 43, 55, 56, 62, 16, 366, 288, 359, 333, 301, 87, 249, 206, 126, 345, 247, 185, 162, 158, 258, 54, 8, 4, 28, 72, 220, 123, 75, 226, 367, 350, 1, 208, 7, 41, 99, 280, 137, 31, 151, 17, 264, 330, 153, 48, 37, 88, 255, 362, 323, 9, 116, 295, 222, 130, 70, 250, 160, 163, 29, 211, 200, 224, 147, 117, 341, 165, 373, 266, 328, 186, 302, 94, 42, 282, 90, 140, 322, 164, 84, 251, 339, 343, 169, 378, 149, 86, 143, 193, 173, 310, 66, 71, 73, 76, 101, 103, 105, 112, 119, 107, 218, 238, 133, 306, 256, 354, 290, 15, 262, 203, 312, 171, 178, 60, 246, 337, 311, 159, 196, 59, 61, 230, 64, 2, 174, 47, 327, 82, 33, 272, 150, 268, 91, 271, 157, 276, 297, 254, 95, 315, 30, 134, 346, 144, 285, 357, 181, 34, 24, 124, 93, 6, 78, 253, 18, 63, 291, 233, 286, 204, 100, 380, 265, 304, 129, 205, 139, 279, 202, 236, 336, 113, 361, 212, 132, 275, 22, 12, 358, 20, 44, 207, 152, 349, 244, 313, 122, 127, 154, 156, 235, 242, 257, 281, 303, 108, 259, 368, 248, 370, 231, 36, 287, 194, 321, 284, 214, 3, 292, 138, 237, 58, 89, 69, 300, 340, 161, 52, 135, 168, 51, 263, 299, 83, 228, 374, 261, 219, 372, 223, 172, 329, 45, 5, 334, 125, 269, 317, 25, 369, 320, 145, 240, 180, 98, 46, 347, 21, 307, 109, 335, 167, 371, 381, 183, 201, 352, 53, 274, 277, 243, 96, 197, 225, 273, 0, 182, 49, 166, 296, 213, 198, 10, 229, 128, 217, 177, 32, 57, 289, 375, 308, 325, 338, 351, 356, 363, 365, 376, 377, 383] + 35: [182, 204, 131, 368, 277, 133, 271, 357, 174, 234, 11, 129, 242, 361, 353, 93, 58, 346, 9, 255, 324, 279, 380, 170, 322, 0, 40, 258, 132, 141, 212, 23, 262, 35, 352, 159, 313, 55, 241, 69, 101, 173, 296, 260, 108, 5, 226, 154, 329, 196, 49, 163, 137, 12, 82, 114, 349, 13, 4, 155, 140, 227, 341, 42, 314, 24, 181, 343, 30, 6, 87, 238, 36, 107, 14, 355, 278, 297, 336, 85, 66, 220, 303, 10, 345, 194, 46, 146, 286, 33, 305, 338, 25, 169, 280, 332, 363, 379, 157, 44, 205, 43, 67, 243, 128, 319, 224, 78, 201, 172, 130, 83, 56, 247, 53, 265, 268, 28, 91, 211, 306, 301, 348, 153, 161, 235, 374, 41, 283, 311, 334, 375, 45, 256, 15, 290, 350, 272, 208, 68, 276, 1, 298, 156, 233, 219, 171, 127, 142, 240, 27, 38, 261, 60, 193, 126, 315, 19, 143, 179, 369, 383, 376, 327, 21, 359, 105, 284, 54, 113, 254, 304, 366, 335, 365, 333, 373, 165, 223, 184, 135, 120, 100, 209, 158, 364, 293, 294, 51, 16, 81, 176, 106, 79, 203, 74, 331, 50, 269, 186, 3, 70, 328, 266, 17, 197, 183, 378, 149, 20, 112, 138, 125, 377, 139, 134, 312, 7, 222, 190, 124, 95, 65, 291, 162, 249, 59, 18, 215, 90, 160, 77, 231, 360, 117, 382, 76, 289, 337, 246, 109, 72, 103, 295, 229, 221, 344, 116, 292, 253, 202, 75, 214, 189, 216, 325, 302, 372, 232, 200, 275, 191, 259, 230, 299, 330, 145, 244, 354, 236, 2, 89, 308, 317, 26, 370, 347, 119, 371, 188, 39, 37, 245, 210, 150, 228, 177, 185, 48, 281, 362, 316, 122, 252, 358, 104, 92, 309, 250, 273, 164, 326, 31, 257, 94, 225, 110, 102, 96, 136, 121, 111, 351, 274, 318, 288, 187, 151, 213, 207, 285, 206, 8, 267, 148, 34, 167, 217, 63, 99, 22, 52, 166, 88, 180, 71, 248, 97, 239, 98, 339, 73, 321, 270, 175, 287, 62, 282, 144, 57, 32, 152, 123, 192, 84, 29, 86, 147, 195, 320, 342, 198, 367, 118, 237, 64, 61, 80, 323, 381, 307, 178, 263, 356, 218, 300, 199, 264, 168, 47, 340, 115, 251, 310] + 36: [304, 155, 352, 5, 293, 193, 138, 97, 158, 365, 247, 210, 374, 220, 176, 199, 265, 212, 364, 81, 299, 95, 71, 356, 94, 233, 160, 266, 362, 117, 101, 65, 17, 282, 110, 116, 88, 188, 99, 26, 7, 15, 174, 231, 255, 150, 225, 10, 245, 129, 119, 251, 242, 109, 18, 21, 56, 295, 149, 259, 66, 337, 120, 241, 367, 72, 326, 157, 29, 335, 173, 368, 190, 64, 307, 89, 204, 183, 55, 334, 222, 172, 148, 239, 346, 37, 221, 107, 96, 115, 123, 124, 253, 298, 313, 330, 232, 291, 331, 280, 274, 122, 76, 340, 370, 238, 207, 211, 285, 215, 201, 373, 103, 75, 297, 92, 12, 179, 156, 121, 187, 20, 283, 226, 44, 350, 131, 102, 227, 267, 108, 182, 135, 74, 114, 154, 73, 351, 60, 339, 258, 31, 163, 180, 263, 281, 53, 296, 240, 301, 269, 132, 235, 306, 8, 316, 314, 376, 224, 347, 83, 34, 284, 145, 151, 289, 234, 52, 33, 358, 3, 270, 147, 127, 11, 68, 320, 79, 361, 51, 128, 104, 372, 143, 186, 16, 336, 375, 360, 170, 353, 355, 22, 310, 134, 380, 200, 61, 100, 194, 14, 319, 359, 45, 196, 153, 322, 177, 162, 69, 46, 349, 198, 19, 342, 98, 272, 191, 161, 23, 287, 311, 181, 67, 219, 357, 354, 213, 112, 230, 82, 38, 144, 189, 140, 125, 366, 9, 246, 6, 244, 32, 303, 93, 195, 371, 85, 345, 50, 382, 341, 276, 42, 118, 70, 59, 136, 254, 277, 47, 363, 130, 206, 159, 165, 228, 338, 203, 292, 54, 378, 111, 250, 305, 302, 261, 214, 43, 39, 113, 252, 290, 315, 126, 86, 377, 0, 58, 197, 178, 184, 332, 348, 309, 288, 323, 321, 275, 237, 260, 264, 328, 278, 175, 308, 343, 168, 164, 57, 248, 87, 30, 2, 344, 257, 333, 137, 141, 62, 209, 249, 78, 171, 49, 139, 105, 369, 300, 294, 383, 192, 229, 35, 41, 185, 216, 77, 256, 133, 271, 48, 36, 166, 63, 80, 91, 318, 262, 217, 273, 279, 167, 381, 28, 379, 317, 329, 4, 208, 90, 202, 205, 236, 27, 106, 142, 325, 24, 84, 13, 1, 40, 152, 324, 25, 146, 169, 223, 312, 286, 243, 268, 327, 218] + 37: [218, 127, 252, 272, 367, 279, 174, 0, 223, 151, 40, 355, 227, 34, 60, 124, 185, 302, 254, 148, 27, 175, 292, 114, 365, 343, 291, 21, 1, 349, 149, 48, 19, 345, 126, 121, 156, 87, 298, 161, 231, 105, 164, 64, 262, 115, 173, 299, 361, 360, 341, 138, 153, 251, 206, 77, 353, 331, 245, 256, 284, 295, 282, 320, 178, 321, 358, 275, 337, 232, 165, 215, 145, 82, 319, 324, 370, 134, 122, 7, 120, 92, 380, 26, 339, 84, 96, 172, 220, 226, 236, 239, 249, 265, 352, 374, 69, 125, 29, 376, 179, 228, 30, 271, 130, 194, 240, 95, 158, 24, 129, 234, 322, 93, 6, 38, 242, 340, 342, 364, 309, 140, 143, 381, 289, 62, 222, 167, 208, 221, 52, 195, 327, 22, 347, 382, 207, 83, 200, 46, 338, 314, 243, 325, 157, 66, 59, 15, 183, 13, 264, 197, 212, 335, 334, 363, 210, 109, 74, 136, 259, 116, 285, 123, 135, 258, 304, 354, 32, 155, 90, 89, 273, 307, 244, 253, 350, 286, 293, 250, 142, 75, 31, 263, 170, 186, 266, 219, 132, 377, 63, 70, 372, 131, 16, 237, 368, 287, 65, 36, 163, 25, 166, 51, 49, 188, 277, 311, 10, 5, 71, 47, 88, 17, 351, 235, 147, 14, 192, 261, 86, 344, 73, 301, 39, 203, 41, 317, 315, 102, 18, 211, 205, 150, 2, 180, 241, 290, 144, 362, 274, 176, 308, 190, 201, 141, 100, 189, 42, 375, 154, 79, 280, 313, 336, 107, 113, 198, 72, 181, 303, 104, 369, 333, 373, 111, 94, 20, 278, 76, 99, 225, 248, 214, 326, 23, 54, 230, 209, 383, 9, 247, 56, 269, 68, 8, 12, 53, 294, 366, 268, 182, 118, 328, 37, 305, 133, 58, 199, 117, 85, 306, 44, 233, 159, 3, 318, 191, 202, 267, 193, 81, 257, 160, 332, 171, 67, 152, 110, 184, 97, 204, 359, 45, 270, 128, 316, 168, 224, 61, 329, 356, 50, 378, 238, 146, 177, 187, 323, 108, 379, 283, 169, 217, 229, 371, 213, 312, 106, 255, 78, 162, 119, 296, 297, 91, 260, 281, 103, 11, 346, 80, 139, 112, 216, 300, 43, 357, 98, 55, 246, 4, 33, 196, 101, 330, 310, 35, 276, 137, 348, 288, 28, 57] + 38: [6, 284, 321, 369, 112, 178, 154, 208, 373, 256, 177, 94, 70, 116, 352, 194, 139, 241, 23, 334, 157, 175, 372, 106, 324, 73, 274, 357, 115, 215, 158, 344, 117, 238, 104, 318, 351, 240, 85, 278, 182, 181, 276, 119, 110, 226, 207, 263, 261, 35, 310, 10, 42, 15, 118, 107, 105, 233, 81, 37, 80, 59, 62, 34, 159, 205, 247, 113, 155, 141, 137, 317, 18, 214, 375, 38, 254, 24, 285, 28, 160, 134, 169, 323, 356, 46, 210, 330, 60, 61, 64, 127, 185, 191, 193, 195, 366, 348, 203, 246, 292, 355, 40, 383, 111, 27, 332, 100, 74, 108, 245, 130, 120, 255, 69, 92, 147, 349, 49, 306, 31, 145, 189, 88, 152, 343, 3, 378, 7, 320, 202, 168, 21, 72, 235, 54, 296, 43, 265, 328, 101, 126, 360, 367, 368, 258, 257, 29, 311, 75, 93, 333, 290, 44, 338, 225, 211, 151, 52, 342, 196, 269, 45, 2, 294, 359, 250, 271, 281, 91, 174, 217, 345, 121, 143, 183, 370, 1, 365, 14, 273, 302, 232, 32, 109, 13, 66, 222, 198, 237, 304, 314, 266, 354, 230, 262, 11, 156, 171, 135, 236, 319, 293, 267, 30, 197, 228, 180, 199, 299, 227, 132, 248, 97, 56, 76, 9, 26, 51, 212, 167, 275, 99, 129, 187, 327, 136, 286, 163, 218, 82, 128, 288, 253, 87, 336, 268, 102, 22, 77, 176, 57, 213, 123, 96, 282, 339, 122, 186, 307, 289, 161, 150, 287, 146, 200, 243, 209, 48, 358, 252, 297, 162, 382, 201, 234, 166, 361, 65, 264, 192, 220, 272, 312, 12, 83, 305, 291, 103, 98, 84, 216, 114, 184, 242, 325, 346, 350, 331, 124, 19, 179, 301, 341, 17, 50, 221, 337, 279, 316, 63, 125, 379, 219, 144, 280, 259, 206, 0, 190, 68, 142, 313, 309, 20, 5, 164, 249, 53, 55, 329, 170, 295, 86, 41, 16, 78, 362, 172, 260, 347, 33, 326, 8, 335, 89, 322, 165, 36, 364, 149, 277, 229, 95, 204, 298, 58, 25, 173, 140, 303, 283, 371, 47, 308, 90, 381, 4, 138, 270, 224, 223, 131, 244, 380, 239, 79, 67, 148, 39, 315, 153, 340, 71, 300, 188, 377, 251, 133, 231, 376, 353, 363, 374] + 39: [102, 100, 353, 336, 187, 297, 372, 56, 319, 112, 248, 322, 323, 141, 14, 258, 87, 93, 213, 151, 244, 221, 28, 281, 161, 356, 54, 363, 254, 208, 284, 5, 41, 172, 185, 84, 135, 154, 58, 95, 25, 139, 13, 226, 344, 291, 148, 124, 261, 107, 17, 16, 65, 230, 326, 146, 167, 309, 289, 30, 33, 66, 365, 39, 178, 266, 232, 376, 205, 2, 217, 145, 334, 229, 293, 90, 311, 122, 355, 20, 158, 321, 371, 382, 270, 119, 219, 330, 186, 27, 171, 111, 216, 184, 142, 350, 282, 236, 127, 110, 15, 170, 153, 223, 163, 383, 290, 260, 367, 7, 18, 129, 347, 333, 227, 203, 252, 242, 80, 200, 40, 101, 11, 303, 286, 169, 246, 214, 312, 339, 36, 253, 197, 245, 181, 277, 292, 251, 378, 285, 77, 196, 125, 79, 283, 206, 23, 233, 76, 194, 37, 373, 324, 173, 366, 123, 234, 201, 55, 329, 348, 50, 255, 190, 168, 45, 250, 263, 152, 143, 211, 331, 131, 209, 276, 231, 224, 308, 375, 117, 38, 150, 73, 115, 317, 301, 341, 327, 195, 136, 19, 4, 86, 352, 116, 274, 220, 368, 315, 177, 199, 359, 369, 118, 212, 155, 144, 62, 235, 351, 247, 377, 60, 164, 133, 298, 126, 380, 82, 360, 204, 113, 257, 81, 160, 32, 121, 104, 210, 147, 31, 69, 26, 71, 320, 70, 97, 192, 132, 162, 182, 49, 46, 103, 279, 337, 106, 239, 294, 105, 310, 159, 98, 267, 166, 91, 228, 271, 120, 176, 92, 52, 51, 10, 0, 342, 357, 259, 157, 78, 287, 193, 165, 137, 243, 85, 64, 130, 9, 189, 314, 265, 22, 198, 370, 89, 108, 240, 134, 96, 180, 191, 328, 280, 379, 1, 264, 306, 300, 295, 68, 307, 114, 53, 99, 61, 225, 75, 374, 140, 138, 88, 381, 156, 48, 362, 34, 275, 272, 183, 332, 59, 63, 67, 296, 218, 335, 202, 249, 325, 313, 128, 299, 364, 273, 305, 175, 8, 6, 268, 354, 35, 3, 109, 346, 43, 302, 74, 207, 358, 174, 215, 24, 21, 316, 47, 237, 188, 343, 349, 83, 29, 179, 42, 222, 338, 262, 94, 269, 149, 345, 72, 238, 44, 304, 12, 288, 241, 278, 57, 256, 318, 361, 340] + 40: [297, 59, 12, 249, 2, 132, 187, 275, 179, 51, 21, 13, 182, 45, 202, 148, 345, 135, 7, 163, 159, 282, 294, 81, 365, 158, 85, 102, 175, 286, 46, 355, 201, 146, 261, 291, 54, 254, 363, 177, 203, 15, 255, 86, 114, 57, 265, 193, 156, 16, 379, 373, 262, 230, 351, 95, 229, 224, 296, 28, 133, 119, 155, 326, 366, 24, 112, 323, 195, 302, 90, 221, 270, 103, 125, 364, 240, 318, 211, 304, 110, 38, 19, 61, 107, 142, 144, 128, 252, 320, 8, 105, 317, 58, 122, 171, 207, 136, 272, 216, 181, 244, 258, 55, 160, 303, 10, 260, 71, 238, 209, 48, 349, 290, 360, 121, 340, 368, 82, 108, 357, 215, 89, 369, 141, 256, 87, 250, 251, 184, 298, 17, 26, 353, 120, 213, 247, 130, 69, 293, 131, 126, 278, 5, 288, 65, 91, 151, 0, 29, 66, 167, 263, 44, 169, 324, 329, 41, 33, 267, 113, 189, 321, 378, 226, 352, 374, 310, 375, 191, 157, 32, 266, 273, 178, 168, 228, 49, 346, 25, 248, 313, 20, 94, 164, 306, 27, 217, 22, 9, 50, 115, 269, 162, 188, 180, 67, 166, 227, 129, 239, 232, 23, 274, 6, 347, 83, 137, 358, 237, 284, 64, 186, 333, 253, 327, 271, 53, 1, 205, 18, 341, 289, 73, 60, 283, 339, 305, 96, 370, 63, 285, 212, 194, 196, 145, 37, 208, 336, 35, 337, 77, 200, 246, 312, 197, 140, 331, 183, 382, 101, 176, 4, 134, 30, 348, 43, 206, 117, 165, 343, 311, 299, 56, 235, 123, 315, 34, 152, 356, 371, 42, 377, 99, 88, 39, 361, 344, 154, 335, 68, 149, 314, 62, 325, 147, 92, 170, 31, 100, 190, 334, 93, 281, 75, 338, 287, 79, 40, 150, 153, 80, 74, 143, 78, 104, 332, 301, 127, 172, 3, 76, 198, 185, 236, 319, 280, 192, 316, 359, 204, 372, 308, 322, 116, 220, 259, 219, 383, 367, 11, 98, 231, 234, 330, 268, 233, 36, 52, 264, 118, 225, 362, 223, 84, 174, 199, 139, 242, 70, 328, 47, 161, 307, 381, 97, 354, 277, 124, 276, 111, 72, 210, 222, 109, 173, 257, 309, 279, 245, 295, 14, 376, 138, 380, 300, 292, 243, 241, 218, 342, 106, 350, 214] + 41: [205, 318, 202, 111, 0, 337, 341, 43, 232, 76, 206, 249, 56, 86, 139, 19, 153, 13, 150, 183, 136, 330, 2, 87, 180, 103, 200, 188, 342, 218, 311, 95, 226, 349, 163, 189, 16, 45, 279, 227, 129, 146, 126, 271, 313, 211, 214, 116, 118, 260, 105, 21, 335, 161, 230, 97, 317, 282, 195, 181, 360, 54, 371, 5, 160, 290, 68, 208, 364, 162, 73, 72, 209, 378, 380, 40, 74, 9, 71, 280, 250, 33, 213, 149, 257, 184, 138, 94, 88, 26, 287, 361, 219, 277, 29, 141, 350, 79, 359, 264, 383, 112, 235, 3, 144, 168, 234, 166, 96, 84, 256, 12, 90, 261, 140, 286, 283, 27, 147, 154, 82, 276, 212, 7, 294, 25, 156, 122, 171, 307, 248, 80, 351, 177, 204, 182, 345, 145, 382, 300, 348, 1, 169, 123, 268, 332, 114, 203, 224, 117, 228, 225, 305, 296, 199, 4, 352, 134, 152, 121, 106, 243, 50, 344, 299, 298, 133, 308, 167, 278, 210, 253, 65, 131, 51, 41, 263, 221, 130, 186, 81, 78, 285, 6, 336, 100, 173, 85, 48, 52, 233, 288, 125, 273, 24, 262, 237, 185, 292, 135, 124, 338, 347, 254, 238, 8, 172, 217, 220, 142, 374, 379, 174, 107, 333, 35, 113, 269, 42, 66, 370, 322, 302, 316, 266, 61, 331, 310, 372, 75, 334, 194, 358, 55, 241, 34, 196, 289, 137, 30, 60, 190, 59, 14, 119, 381, 91, 187, 375, 159, 274, 258, 366, 367, 270, 315, 157, 321, 319, 301, 284, 15, 295, 115, 252, 69, 109, 62, 58, 22, 148, 297, 247, 328, 354, 17, 175, 325, 151, 83, 304, 293, 216, 10, 357, 132, 346, 281, 170, 306, 324, 329, 201, 44, 275, 28, 363, 368, 101, 242, 339, 191, 207, 49, 198, 223, 222, 77, 53, 158, 323, 356, 70, 165, 244, 98, 178, 327, 179, 355, 197, 37, 164, 143, 89, 57, 369, 251, 23, 320, 215, 340, 231, 127, 32, 67, 326, 120, 314, 236, 110, 259, 92, 128, 309, 108, 93, 245, 362, 291, 373, 47, 343, 272, 104, 176, 240, 239, 46, 20, 303, 246, 255, 18, 192, 193, 63, 11, 155, 229, 36, 265, 39, 38, 365, 377, 267, 99, 353, 102, 31, 64, 376, 312] + 42: [306, 77, 80, 238, 98, 87, 268, 251, 101, 138, 88, 303, 33, 197, 154, 41, 282, 209, 164, 150, 356, 151, 351, 285, 178, 92, 277, 311, 0, 248, 60, 217, 181, 244, 312, 3, 232, 247, 362, 284, 75, 34, 261, 134, 256, 76, 353, 78, 86, 71, 38, 348, 375, 149, 215, 155, 221, 349, 258, 168, 195, 345, 267, 146, 292, 31, 338, 355, 382, 250, 346, 334, 369, 69, 24, 141, 65, 46, 347, 198, 187, 339, 42, 211, 192, 174, 243, 121, 304, 177, 372, 153, 318, 14, 99, 270, 266, 200, 166, 118, 319, 37, 212, 373, 16, 272, 135, 202, 241, 378, 359, 120, 302, 161, 296, 263, 295, 321, 222, 20, 2, 226, 316, 32, 68, 317, 231, 350, 333, 184, 140, 145, 193, 147, 264, 260, 343, 139, 79, 47, 36, 220, 67, 119, 186, 246, 49, 299, 331, 172, 66, 45, 305, 207, 43, 210, 157, 254, 324, 22, 327, 381, 249, 276, 18, 6, 223, 183, 72, 180, 109, 191, 107, 62, 206, 19, 143, 162, 199, 136, 322, 205, 214, 28, 269, 340, 51, 208, 89, 370, 25, 286, 123, 111, 17, 21, 196, 114, 274, 55, 103, 291, 367, 364, 132, 320, 59, 275, 48, 283, 315, 52, 290, 167, 12, 148, 61, 236, 328, 281, 90, 131, 228, 293, 255, 179, 301, 230, 115, 122, 15, 1, 169, 83, 245, 84, 158, 156, 298, 97, 314, 23, 240, 100, 288, 227, 354, 85, 5, 377, 7, 262, 70, 329, 294, 40, 335, 27, 332, 313, 159, 330, 357, 371, 323, 173, 365, 4, 225, 194, 383, 94, 219, 128, 253, 361, 229, 363, 74, 190, 142, 224, 113, 126, 216, 242, 58, 50, 176, 325, 189, 204, 358, 93, 279, 336, 300, 56, 116, 265, 152, 308, 239, 129, 366, 380, 326, 280, 125, 309, 8, 117, 54, 102, 201, 175, 30, 218, 342, 112, 26, 259, 237, 130, 44, 287, 127, 144, 341, 344, 213, 11, 104, 170, 252, 368, 374, 379, 133, 124, 310, 29, 13, 352, 234, 171, 82, 297, 278, 35, 73, 96, 376, 57, 9, 307, 64, 289, 203, 235, 337, 10, 360, 257, 63, 110, 160, 91, 188, 105, 165, 53, 95, 185, 106, 137, 108, 233, 273, 81, 182, 271, 163, 39] + 43: [315, 202, 211, 243, 61, 55, 252, 218, 319, 366, 63, 324, 364, 258, 209, 170, 200, 27, 30, 156, 80, 356, 338, 46, 318, 379, 335, 341, 16, 149, 342, 267, 183, 371, 1, 306, 115, 72, 368, 163, 232, 269, 271, 193, 13, 70, 185, 109, 255, 157, 137, 189, 249, 215, 199, 321, 37, 327, 317, 314, 212, 8, 130, 162, 350, 284, 105, 268, 178, 247, 145, 295, 133, 78, 230, 229, 112, 45, 289, 91, 228, 296, 15, 245, 14, 43, 181, 51, 195, 370, 58, 29, 340, 104, 294, 21, 346, 36, 240, 239, 219, 272, 69, 194, 74, 108, 172, 92, 159, 84, 52, 152, 125, 190, 337, 120, 56, 191, 127, 339, 288, 363, 86, 333, 220, 374, 347, 77, 186, 330, 40, 201, 168, 221, 282, 93, 71, 205, 32, 147, 382, 94, 122, 65, 180, 278, 351, 276, 54, 286, 287, 131, 277, 301, 124, 355, 250, 33, 44, 113, 357, 312, 305, 31, 6, 102, 136, 349, 372, 187, 41, 118, 150, 353, 3, 90, 334, 241, 236, 231, 177, 248, 169, 206, 10, 89, 320, 7, 4, 208, 106, 281, 283, 75, 198, 97, 101, 161, 167, 298, 256, 76, 358, 238, 100, 224, 260, 223, 166, 378, 142, 188, 323, 138, 9, 20, 22, 0, 336, 290, 375, 303, 73, 380, 49, 204, 48, 207, 275, 210, 251, 62, 225, 99, 151, 68, 300, 280, 291, 377, 265, 17, 88, 38, 164, 50, 365, 222, 39, 184, 226, 254, 141, 261, 66, 263, 270, 111, 18, 264, 135, 325, 128, 373, 82, 23, 19, 376, 197, 331, 5, 192, 126, 227, 361, 110, 292, 313, 213, 348, 176, 326, 60, 307, 328, 47, 310, 381, 116, 293, 53, 262, 107, 344, 175, 96, 237, 216, 257, 242, 235, 279, 196, 2, 345, 160, 311, 383, 146, 274, 26, 98, 203, 369, 103, 253, 85, 59, 123, 87, 297, 34, 343, 140, 83, 35, 332, 302, 67, 352, 121, 148, 308, 12, 299, 153, 42, 362, 259, 359, 179, 171, 316, 129, 309, 139, 273, 155, 134, 173, 360, 154, 143, 329, 24, 117, 367, 322, 285, 244, 95, 119, 144, 182, 79, 158, 246, 28, 57, 233, 165, 114, 11, 81, 266, 214, 25, 174, 64, 304, 234, 354, 132, 217] + 44: [64, 267, 95, 242, 26, 35, 275, 299, 334, 74, 351, 250, 63, 217, 318, 98, 124, 66, 212, 215, 238, 86, 173, 43, 24, 324, 199, 57, 13, 171, 85, 300, 101, 200, 330, 182, 251, 187, 342, 372, 383, 123, 87, 258, 349, 80, 79, 180, 41, 181, 273, 264, 190, 32, 142, 317, 155, 1, 226, 108, 214, 118, 147, 294, 46, 193, 241, 310, 346, 231, 53, 341, 283, 357, 78, 134, 316, 197, 308, 152, 23, 322, 65, 116, 245, 362, 309, 344, 107, 326, 230, 339, 71, 307, 198, 297, 102, 36, 350, 285, 280, 91, 61, 243, 239, 105, 186, 255, 347, 175, 207, 305, 237, 268, 93, 130, 295, 121, 89, 284, 289, 338, 221, 290, 356, 336, 371, 109, 379, 153, 192, 249, 50, 353, 164, 33, 172, 323, 111, 3, 113, 374, 159, 286, 304, 110, 281, 332, 203, 359, 28, 39, 122, 97, 358, 22, 62, 261, 277, 68, 329, 169, 115, 375, 0, 149, 47, 140, 382, 45, 380, 210, 206, 259, 225, 220, 302, 311, 42, 365, 364, 154, 208, 106, 335, 354, 148, 136, 373, 112, 248, 88, 15, 10, 369, 163, 84, 263, 227, 256, 265, 224, 73, 14, 82, 194, 272, 131, 287, 253, 271, 257, 137, 222, 177, 166, 178, 126, 76, 156, 117, 77, 188, 202, 99, 54, 355, 92, 44, 21, 19, 282, 59, 128, 201, 312, 157, 366, 67, 345, 25, 314, 90, 376, 160, 204, 141, 37, 298, 83, 327, 331, 360, 229, 240, 27, 370, 56, 30, 17, 31, 328, 120, 9, 7, 150, 367, 135, 361, 247, 170, 146, 168, 133, 319, 16, 320, 195, 138, 270, 363, 12, 5, 174, 321, 183, 211, 129, 274, 6, 213, 279, 254, 333, 266, 125, 51, 244, 218, 4, 38, 165, 296, 94, 276, 191, 292, 100, 75, 378, 96, 103, 381, 34, 352, 179, 340, 313, 348, 325, 269, 127, 48, 145, 233, 18, 236, 343, 143, 288, 235, 301, 306, 260, 234, 52, 151, 139, 189, 262, 278, 104, 158, 144, 69, 337, 223, 228, 2, 232, 114, 8, 40, 196, 81, 184, 209, 70, 11, 176, 303, 60, 185, 55, 20, 291, 246, 315, 162, 58, 368, 167, 219, 119, 216, 29, 72, 161, 252, 49, 293, 205, 377, 132] + 45: [131, 144, 79, 179, 116, 178, 369, 52, 165, 155, 208, 373, 367, 37, 275, 331, 284, 207, 253, 319, 239, 365, 315, 10, 212, 99, 242, 341, 87, 0, 216, 217, 63, 204, 105, 161, 292, 339, 147, 158, 160, 243, 177, 21, 47, 168, 140, 129, 143, 310, 92, 110, 317, 16, 151, 377, 68, 357, 353, 259, 43, 252, 370, 56, 127, 135, 75, 67, 66, 258, 24, 300, 14, 115, 227, 157, 294, 362, 64, 244, 206, 28, 272, 77, 214, 89, 69, 111, 86, 210, 53, 117, 11, 287, 326, 347, 283, 281, 61, 236, 186, 282, 222, 256, 356, 146, 298, 220, 333, 261, 141, 50, 273, 82, 162, 125, 29, 25, 274, 233, 95, 378, 269, 215, 246, 123, 325, 176, 73, 359, 80, 51, 191, 202, 180, 279, 7, 142, 104, 289, 219, 139, 91, 205, 185, 102, 84, 23, 45, 39, 156, 265, 306, 303, 361, 291, 344, 173, 35, 264, 374, 153, 198, 200, 88, 187, 245, 108, 240, 183, 169, 171, 49, 371, 368, 132, 133, 305, 3, 30, 211, 195, 13, 137, 336, 358, 270, 136, 22, 42, 337, 167, 350, 201, 109, 31, 372, 118, 286, 330, 189, 46, 145, 262, 249, 335, 277, 54, 383, 382, 166, 301, 360, 101, 20, 322, 238, 225, 128, 8, 100, 114, 41, 346, 295, 232, 152, 26, 154, 34, 323, 72, 113, 119, 188, 228, 106, 381, 320, 71, 15, 170, 33, 164, 199, 324, 235, 203, 351, 159, 379, 213, 268, 218, 121, 307, 328, 6, 94, 181, 74, 321, 355, 90, 76, 18, 55, 313, 376, 231, 380, 257, 288, 302, 338, 312, 9, 4, 263, 192, 112, 148, 297, 230, 38, 12, 174, 311, 96, 234, 175, 327, 254, 182, 48, 348, 260, 58, 27, 138, 62, 120, 329, 134, 32, 196, 280, 163, 78, 70, 366, 2, 296, 149, 375, 278, 247, 267, 97, 334, 250, 150, 223, 36, 81, 290, 122, 293, 59, 308, 318, 103, 343, 345, 332, 85, 354, 248, 285, 107, 363, 5, 340, 197, 224, 349, 314, 44, 316, 226, 251, 276, 1, 65, 309, 40, 342, 271, 352, 57, 124, 60, 304, 255, 229, 237, 241, 126, 190, 299, 184, 17, 364, 83, 266, 194, 130, 221, 193, 19, 209, 172, 93, 98] + 46: [207, 224, 247, 376, 339, 152, 102, 103, 199, 105, 253, 362, 2, 104, 353, 355, 49, 164, 342, 8, 93, 322, 99, 313, 71, 12, 348, 328, 4, 30, 245, 201, 337, 17, 372, 312, 329, 264, 299, 88, 163, 121, 144, 176, 26, 80, 357, 165, 255, 187, 156, 95, 22, 359, 24, 235, 331, 107, 340, 65, 73, 59, 287, 18, 333, 46, 365, 174, 252, 270, 237, 1, 202, 231, 225, 56, 150, 37, 300, 5, 53, 307, 294, 295, 220, 50, 184, 15, 43, 175, 132, 248, 189, 251, 293, 216, 106, 239, 170, 168, 374, 7, 68, 212, 278, 136, 232, 131, 324, 75, 23, 256, 368, 217, 76, 82, 161, 345, 311, 280, 38, 273, 208, 292, 25, 352, 284, 67, 360, 381, 323, 267, 33, 218, 114, 160, 10, 242, 258, 186, 169, 39, 66, 369, 120, 364, 153, 356, 283, 86, 233, 85, 90, 213, 183, 286, 110, 260, 244, 118, 228, 297, 146, 289, 214, 318, 14, 275, 277, 282, 335, 238, 316, 28, 63, 94, 122, 196, 173, 272, 227, 330, 92, 288, 358, 182, 281, 383, 147, 341, 361, 167, 44, 343, 58, 221, 204, 319, 139, 271, 268, 166, 126, 254, 193, 246, 51, 234, 141, 155, 180, 290, 87, 97, 64, 125, 188, 52, 327, 354, 137, 69, 96, 320, 171, 230, 162, 249, 62, 338, 29, 219, 91, 70, 21, 77, 203, 226, 344, 109, 261, 363, 380, 305, 298, 116, 133, 40, 296, 291, 172, 98, 366, 177, 123, 179, 222, 336, 191, 19, 83, 47, 31, 262, 151, 332, 72, 178, 13, 157, 108, 304, 112, 79, 20, 81, 27, 145, 370, 309, 200, 257, 16, 3, 350, 54, 347, 61, 142, 111, 303, 158, 45, 9, 194, 149, 210, 351, 198, 6, 276, 274, 34, 211, 367, 377, 206, 124, 301, 266, 205, 127, 334, 35, 263, 285, 129, 101, 259, 375, 265, 240, 229, 57, 223, 11, 314, 154, 130, 143, 36, 209, 215, 269, 84, 378, 134, 89, 159, 113, 192, 195, 241, 140, 349, 317, 128, 181, 321, 306, 243, 100, 346, 185, 115, 315, 74, 41, 42, 308, 373, 279, 190, 117, 382, 78, 138, 48, 119, 379, 60, 135, 32, 55, 148, 310, 326, 236, 0, 197, 250, 371, 325, 302] + 47: [54, 330, 346, 370, 296, 123, 136, 61, 93, 244, 367, 98, 137, 4, 267, 210, 231, 377, 53, 189, 322, 141, 24, 76, 203, 158, 0, 108, 361, 264, 250, 172, 74, 282, 62, 202, 20, 132, 331, 213, 295, 319, 177, 121, 276, 12, 269, 107, 360, 236, 316, 345, 257, 169, 148, 207, 59, 321, 31, 341, 356, 304, 363, 205, 113, 233, 225, 354, 379, 170, 273, 239, 358, 38, 68, 64, 71, 283, 325, 149, 144, 88, 100, 69, 40, 160, 135, 99, 79, 300, 84, 8, 167, 270, 21, 272, 162, 299, 65, 41, 131, 195, 87, 179, 27, 103, 118, 208, 129, 352, 366, 154, 196, 111, 280, 328, 308, 275, 156, 83, 117, 187, 259, 26, 66, 312, 81, 70, 306, 176, 218, 45, 17, 44, 193, 373, 298, 291, 101, 95, 30, 302, 333, 116, 229, 109, 120, 163, 368, 183, 279, 157, 194, 320, 228, 89, 153, 324, 309, 190, 182, 266, 353, 28, 287, 72, 274, 178, 351, 286, 336, 92, 339, 206, 261, 251, 104, 15, 348, 86, 256, 124, 161, 220, 248, 260, 152, 56, 150, 214, 318, 364, 265, 90, 243, 106, 200, 63, 168, 77, 204, 357, 175, 43, 285, 52, 301, 18, 119, 284, 372, 371, 134, 262, 376, 36, 232, 337, 329, 305, 155, 271, 294, 216, 211, 359, 35, 173, 192, 9, 201, 143, 238, 14, 16, 82, 315, 281, 34, 198, 164, 365, 221, 33, 277, 184, 226, 13, 181, 334, 254, 37, 11, 335, 102, 209, 23, 327, 217, 219, 252, 230, 222, 241, 186, 130, 258, 146, 245, 166, 355, 268, 313, 25, 311, 142, 383, 110, 344, 253, 115, 47, 246, 288, 96, 78, 381, 380, 29, 2, 127, 235, 3, 50, 375, 185, 347, 303, 42, 314, 48, 292, 67, 122, 140, 180, 94, 290, 75, 128, 240, 151, 224, 342, 73, 46, 374, 171, 165, 212, 51, 138, 105, 174, 133, 317, 307, 60, 139, 343, 297, 362, 97, 197, 215, 188, 278, 147, 80, 39, 85, 289, 1, 338, 234, 227, 223, 263, 369, 378, 91, 249, 57, 310, 247, 159, 255, 10, 22, 32, 382, 199, 242, 332, 237, 293, 49, 125, 19, 323, 55, 340, 126, 350, 6, 349, 114, 191, 326, 7, 5, 145, 58, 112] + 48: [271, 275, 86, 364, 311, 128, 112, 290, 323, 24, 317, 316, 370, 119, 346, 223, 102, 101, 3, 261, 123, 38, 146, 213, 56, 305, 45, 297, 263, 281, 21, 162, 247, 212, 330, 82, 23, 220, 148, 7, 218, 376, 358, 118, 87, 293, 195, 239, 219, 362, 133, 211, 347, 156, 47, 126, 322, 157, 135, 92, 52, 227, 280, 100, 17, 57, 69, 29, 278, 285, 93, 294, 97, 327, 15, 131, 203, 369, 299, 0, 252, 206, 303, 46, 288, 348, 310, 284, 245, 36, 201, 10, 260, 153, 105, 196, 141, 329, 344, 88, 77, 185, 58, 170, 33, 189, 12, 174, 242, 91, 117, 34, 276, 254, 19, 179, 314, 343, 132, 32, 363, 320, 382, 49, 205, 14, 76, 55, 120, 43, 269, 319, 198, 80, 90, 54, 255, 342, 253, 63, 4, 37, 155, 298, 274, 18, 190, 143, 20, 165, 228, 268, 187, 307, 53, 295, 64, 39, 312, 224, 334, 136, 129, 361, 25, 103, 259, 324, 61, 95, 338, 374, 231, 336, 345, 41, 67, 304, 70, 216, 184, 243, 339, 163, 351, 68, 200, 8, 332, 96, 188, 250, 207, 287, 318, 367, 321, 130, 35, 192, 151, 166, 107, 152, 159, 258, 235, 98, 335, 273, 48, 99, 127, 353, 81, 368, 357, 279, 51, 172, 74, 237, 350, 59, 106, 178, 13, 145, 208, 352, 296, 150, 138, 122, 286, 164, 371, 300, 337, 289, 137, 42, 169, 306, 373, 191, 199, 134, 31, 113, 173, 140, 246, 229, 232, 197, 50, 380, 341, 44, 282, 377, 315, 168, 379, 302, 222, 215, 301, 183, 251, 383, 167, 262, 79, 209, 267, 30, 40, 256, 331, 277, 160, 1, 204, 84, 308, 66, 65, 226, 116, 72, 26, 139, 264, 109, 73, 234, 115, 326, 108, 325, 272, 291, 161, 283, 233, 121, 75, 176, 154, 194, 28, 111, 158, 193, 360, 142, 144, 340, 180, 225, 202, 22, 328, 177, 365, 359, 244, 210, 240, 313, 230, 375, 147, 78, 125, 11, 9, 182, 16, 292, 236, 83, 241, 249, 5, 248, 104, 381, 2, 257, 186, 378, 6, 265, 85, 124, 214, 171, 94, 60, 89, 110, 266, 71, 333, 217, 175, 62, 356, 366, 309, 238, 114, 221, 354, 372, 355, 181, 270, 149, 27, 349] + 49: [210, 304, 248, 194, 338, 199, 56, 153, 190, 205, 351, 214, 216, 36, 150, 14, 286, 84, 169, 174, 251, 57, 299, 16, 311, 253, 300, 17, 316, 215, 314, 66, 262, 104, 27, 295, 333, 37, 163, 212, 32, 33, 122, 69, 183, 192, 259, 356, 328, 18, 227, 323, 234, 45, 282, 102, 350, 137, 22, 331, 334, 100, 355, 188, 107, 83, 75, 327, 201, 226, 191, 222, 4, 87, 24, 211, 48, 93, 329, 103, 128, 182, 318, 285, 324, 219, 112, 135, 383, 341, 291, 123, 129, 124, 145, 229, 302, 202, 116, 206, 78, 365, 271, 133, 312, 5, 265, 59, 91, 47, 151, 114, 173, 81, 198, 289, 359, 85, 221, 224, 284, 21, 26, 209, 19, 118, 270, 277, 362, 258, 168, 158, 360, 40, 138, 306, 46, 117, 371, 247, 35, 74, 278, 340, 354, 62, 171, 176, 166, 126, 301, 142, 9, 144, 319, 297, 266, 379, 187, 377, 134, 272, 42, 346, 313, 256, 60, 307, 96, 165, 31, 315, 373, 88, 8, 257, 357, 255, 106, 170, 13, 349, 317, 67, 217, 200, 10, 28, 131, 358, 51, 68, 208, 353, 120, 279, 233, 235, 86, 276, 197, 193, 240, 236, 148, 180, 111, 330, 23, 195, 361, 376, 20, 238, 189, 250, 110, 267, 298, 204, 54, 132, 181, 196, 275, 273, 53, 336, 155, 76, 225, 213, 268, 303, 310, 34, 136, 321, 274, 162, 159, 294, 146, 293, 65, 348, 11, 109, 140, 125, 30, 296, 108, 167, 325, 39, 249, 2, 366, 113, 139, 337, 364, 43, 243, 71, 105, 92, 152, 149, 231, 228, 130, 280, 64, 79, 119, 115, 177, 380, 309, 287, 0, 97, 3, 237, 347, 367, 374, 156, 220, 58, 292, 164, 261, 179, 94, 342, 363, 95, 61, 157, 223, 44, 369, 381, 141, 89, 77, 49, 264, 161, 339, 283, 172, 73, 6, 378, 345, 98, 288, 241, 185, 344, 322, 7, 368, 370, 320, 218, 239, 178, 15, 127, 263, 326, 281, 269, 335, 82, 50, 12, 242, 246, 41, 55, 63, 38, 72, 70, 175, 160, 52, 154, 186, 121, 343, 252, 207, 90, 290, 29, 254, 147, 305, 230, 184, 245, 143, 260, 25, 375, 244, 80, 1, 382, 332, 372, 352, 99, 101, 232, 203, 308] + 50: [306, 268, 180, 358, 353, 138, 315, 187, 252, 58, 90, 275, 6, 189, 214, 63, 185, 281, 256, 51, 28, 366, 240, 249, 126, 344, 153, 327, 152, 17, 77, 266, 148, 157, 350, 108, 135, 210, 217, 96, 119, 125, 294, 121, 39, 70, 134, 7, 372, 113, 21, 312, 136, 213, 177, 239, 324, 206, 375, 149, 225, 64, 98, 357, 142, 349, 11, 231, 53, 147, 154, 383, 258, 219, 259, 20, 293, 376, 137, 282, 196, 347, 2, 82, 292, 245, 99, 190, 61, 373, 341, 365, 103, 257, 264, 298, 72, 9, 175, 297, 193, 42, 86, 14, 380, 279, 69, 300, 270, 243, 160, 340, 38, 127, 333, 205, 3, 220, 336, 208, 322, 325, 156, 229, 159, 95, 194, 236, 226, 235, 10, 23, 116, 54, 146, 144, 355, 66, 269, 202, 18, 321, 287, 164, 227, 163, 295, 351, 224, 248, 359, 369, 247, 371, 263, 310, 307, 255, 317, 209, 102, 91, 334, 26, 356, 84, 139, 29, 85, 335, 151, 379, 131, 120, 40, 167, 221, 260, 276, 360, 198, 34, 179, 250, 106, 363, 211, 216, 207, 176, 237, 83, 33, 318, 129, 1, 343, 5, 75, 0, 370, 49, 15, 115, 232, 141, 41, 301, 16, 352, 31, 346, 284, 261, 267, 285, 186, 277, 242, 22, 45, 8, 299, 68, 158, 81, 74, 171, 36, 114, 234, 71, 378, 161, 46, 251, 184, 288, 165, 362, 24, 155, 174, 238, 118, 43, 289, 60, 32, 67, 339, 331, 145, 150, 323, 52, 253, 178, 241, 80, 78, 309, 183, 173, 228, 367, 55, 326, 222, 62, 204, 30, 280, 291, 332, 47, 330, 87, 181, 316, 37, 286, 278, 105, 308, 345, 4, 172, 192, 329, 27, 117, 348, 97, 218, 124, 338, 166, 123, 212, 57, 92, 188, 381, 169, 122, 311, 130, 12, 382, 88, 246, 361, 320, 197, 94, 314, 56, 110, 48, 162, 25, 100, 319, 215, 133, 143, 342, 271, 313, 223, 274, 354, 107, 104, 265, 328, 230, 44, 35, 89, 272, 377, 50, 112, 254, 109, 203, 302, 304, 283, 73, 195, 111, 305, 273, 191, 233, 244, 296, 262, 59, 182, 101, 168, 140, 368, 199, 303, 170, 374, 132, 201, 128, 337, 200, 65, 19, 79, 93, 13, 364, 76, 290] + 51: [24, 240, 253, 10, 114, 203, 342, 86, 130, 341, 201, 120, 98, 198, 100, 266, 210, 46, 279, 44, 149, 18, 190, 304, 103, 17, 81, 193, 351, 375, 88, 282, 292, 123, 325, 364, 66, 180, 363, 134, 36, 132, 113, 239, 52, 333, 286, 7, 185, 230, 222, 138, 261, 367, 56, 267, 252, 175, 191, 331, 303, 365, 50, 339, 274, 317, 366, 231, 354, 19, 93, 60, 377, 275, 147, 154, 55, 243, 347, 376, 21, 296, 370, 166, 116, 151, 181, 174, 352, 2, 61, 173, 179, 256, 269, 273, 137, 184, 194, 32, 177, 219, 260, 20, 335, 90, 359, 171, 234, 197, 322, 326, 82, 299, 320, 119, 232, 334, 225, 321, 74, 108, 270, 143, 62, 355, 382, 285, 371, 278, 12, 315, 254, 308, 360, 346, 152, 343, 76, 192, 217, 131, 136, 211, 238, 75, 300, 345, 205, 307, 37, 281, 196, 38, 358, 5, 73, 316, 224, 319, 268, 310, 244, 51, 54, 111, 57, 83, 305, 248, 6, 80, 361, 95, 218, 318, 124, 383, 290, 15, 117, 214, 178, 67, 168, 255, 349, 272, 70, 170, 156, 42, 356, 99, 110, 167, 271, 65, 25, 84, 141, 69, 118, 258, 204, 153, 3, 245, 183, 146, 209, 41, 340, 135, 353, 186, 287, 122, 140, 350, 373, 280, 298, 216, 314, 47, 372, 45, 344, 104, 295, 182, 306, 200, 302, 242, 97, 125, 379, 311, 263, 92, 284, 297, 208, 223, 91, 16, 378, 77, 337, 207, 381, 264, 30, 155, 226, 202, 8, 288, 9, 28, 206, 369, 43, 105, 257, 169, 236, 189, 237, 63, 291, 332, 309, 249, 112, 328, 29, 133, 228, 126, 176, 68, 250, 142, 265, 187, 87, 164, 368, 59, 31, 374, 329, 129, 195, 109, 220, 235, 229, 35, 312, 212, 289, 48, 246, 301, 330, 33, 357, 148, 102, 213, 40, 362, 157, 233, 324, 128, 39, 115, 26, 227, 294, 160, 262, 107, 251, 72, 127, 144, 22, 323, 188, 101, 380, 283, 259, 58, 338, 89, 13, 277, 139, 64, 34, 161, 165, 276, 247, 336, 106, 121, 163, 85, 221, 71, 199, 23, 0, 79, 327, 11, 145, 27, 49, 159, 53, 158, 172, 162, 1, 215, 78, 348, 14, 241, 4, 150, 293, 94, 96, 313] + 52: [332, 248, 151, 342, 379, 258, 217, 196, 311, 215, 116, 161, 23, 97, 33, 312, 74, 85, 138, 171, 358, 127, 84, 239, 369, 328, 272, 22, 357, 3, 63, 265, 285, 79, 50, 31, 10, 352, 5, 364, 243, 130, 334, 343, 365, 326, 61, 39, 223, 47, 208, 355, 216, 227, 149, 335, 9, 129, 295, 237, 49, 236, 318, 172, 145, 294, 362, 152, 14, 348, 120, 360, 29, 181, 259, 319, 113, 341, 253, 368, 12, 356, 309, 222, 278, 20, 315, 251, 156, 131, 142, 316, 324, 86, 158, 180, 26, 378, 219, 262, 381, 276, 75, 125, 346, 289, 192, 72, 80, 269, 136, 139, 293, 182, 167, 73, 211, 184, 176, 44, 339, 4, 189, 344, 117, 111, 197, 159, 210, 376, 257, 56, 382, 186, 225, 71, 140, 256, 366, 19, 286, 6, 213, 68, 350, 214, 271, 103, 370, 17, 108, 122, 240, 372, 162, 150, 338, 169, 306, 195, 299, 94, 147, 249, 16, 290, 209, 35, 106, 18, 54, 28, 52, 179, 46, 301, 45, 43, 373, 66, 87, 185, 42, 218, 38, 380, 203, 202, 135, 175, 200, 264, 107, 27, 252, 13, 228, 298, 323, 168, 246, 234, 104, 307, 283, 93, 177, 69, 230, 354, 198, 321, 163, 260, 367, 336, 188, 173, 233, 221, 70, 78, 277, 224, 141, 333, 118, 134, 146, 67, 11, 302, 95, 205, 137, 99, 109, 110, 287, 41, 270, 24, 297, 157, 58, 178, 121, 250, 112, 81, 37, 153, 275, 232, 62, 282, 292, 51, 76, 90, 363, 7, 34, 77, 89, 374, 349, 310, 82, 124, 280, 305, 199, 377, 226, 268, 238, 133, 204, 274, 220, 100, 115, 98, 353, 53, 279, 317, 206, 325, 193, 183, 102, 327, 300, 25, 263, 160, 114, 345, 30, 330, 337, 92, 261, 1, 88, 8, 229, 15, 143, 40, 288, 291, 347, 83, 187, 57, 59, 308, 32, 359, 64, 255, 281, 119, 296, 242, 165, 148, 273, 154, 128, 383, 191, 254, 91, 235, 2, 322, 320, 313, 55, 105, 164, 190, 361, 174, 266, 65, 166, 194, 207, 304, 123, 331, 241, 132, 340, 36, 244, 21, 267, 96, 231, 284, 371, 60, 245, 101, 351, 170, 155, 144, 247, 126, 329, 48, 314, 212, 0, 201, 303, 375] + 53: [308, 199, 350, 311, 136, 338, 70, 4, 196, 44, 335, 182, 346, 232, 5, 333, 360, 134, 219, 165, 368, 357, 214, 299, 273, 64, 233, 10, 246, 154, 192, 91, 197, 30, 104, 228, 251, 282, 303, 142, 34, 130, 224, 103, 98, 369, 222, 32, 47, 54, 159, 290, 327, 68, 276, 315, 101, 257, 53, 337, 234, 245, 58, 88, 31, 364, 263, 50, 258, 89, 137, 102, 131, 62, 380, 186, 348, 57, 185, 90, 316, 277, 138, 215, 286, 61, 106, 140, 342, 272, 264, 43, 113, 181, 289, 377, 367, 254, 110, 238, 247, 95, 175, 149, 157, 112, 164, 243, 132, 332, 86, 293, 261, 320, 329, 353, 180, 318, 170, 291, 194, 78, 63, 295, 317, 331, 306, 208, 92, 79, 195, 324, 35, 345, 21, 168, 207, 143, 158, 111, 227, 8, 326, 29, 176, 352, 212, 122, 206, 287, 28, 285, 27, 242, 370, 17, 179, 117, 42, 323, 204, 174, 148, 33, 312, 41, 59, 99, 105, 218, 220, 355, 74, 365, 256, 259, 366, 349, 252, 82, 22, 359, 343, 248, 307, 314, 141, 96, 249, 56, 84, 87, 126, 52, 127, 173, 152, 129, 76, 241, 268, 379, 15, 65, 340, 383, 374, 275, 73, 304, 100, 319, 371, 283, 69, 363, 213, 128, 231, 14, 341, 40, 60, 166, 297, 16, 202, 0, 236, 229, 119, 72, 271, 262, 221, 144, 274, 300, 235, 381, 183, 145, 12, 24, 325, 309, 269, 83, 351, 139, 13, 36, 230, 217, 120, 294, 153, 184, 361, 358, 260, 37, 305, 281, 382, 123, 321, 71, 48, 1, 171, 310, 301, 93, 167, 210, 376, 80, 298, 223, 49, 7, 177, 344, 267, 265, 188, 135, 121, 280, 189, 347, 198, 172, 66, 23, 200, 3, 38, 118, 239, 51, 250, 151, 203, 146, 46, 125, 226, 75, 336, 133, 107, 330, 266, 313, 284, 292, 237, 160, 26, 356, 201, 372, 187, 178, 150, 55, 156, 81, 11, 162, 279, 362, 2, 225, 255, 302, 288, 296, 18, 169, 9, 25, 39, 94, 124, 85, 116, 191, 97, 240, 20, 216, 373, 77, 209, 155, 67, 190, 378, 19, 161, 109, 6, 253, 322, 375, 354, 163, 211, 193, 45, 147, 328, 108, 270, 115, 278, 244, 334, 114, 339, 205] + 54: [363, 41, 299, 57, 44, 325, 154, 147, 370, 90, 192, 280, 167, 143, 300, 356, 64, 328, 181, 46, 286, 93, 258, 27, 371, 326, 71, 353, 265, 122, 121, 92, 246, 61, 140, 331, 112, 164, 327, 207, 189, 35, 367, 184, 171, 343, 145, 340, 84, 321, 241, 76, 212, 26, 166, 95, 236, 68, 333, 153, 210, 297, 172, 81, 319, 79, 217, 287, 18, 193, 48, 334, 359, 33, 183, 267, 75, 162, 223, 238, 60, 383, 229, 316, 110, 105, 195, 324, 168, 362, 10, 130, 36, 55, 78, 123, 294, 240, 80, 115, 158, 151, 53, 124, 125, 284, 282, 163, 173, 91, 226, 179, 312, 242, 188, 70, 62, 54, 174, 347, 103, 293, 40, 375, 186, 208, 329, 341, 25, 32, 245, 314, 269, 161, 4, 349, 137, 42, 235, 277, 200, 249, 348, 191, 380, 248, 37, 34, 364, 330, 317, 211, 22, 77, 87, 7, 65, 337, 72, 239, 378, 244, 69, 252, 104, 307, 221, 116, 264, 263, 372, 306, 232, 273, 342, 43, 308, 187, 270, 373, 291, 160, 227, 28, 114, 234, 368, 152, 144, 148, 169, 237, 214, 176, 117, 260, 296, 357, 82, 45, 16, 197, 3, 365, 98, 311, 156, 352, 199, 113, 275, 322, 215, 279, 290, 225, 301, 20, 129, 111, 29, 309, 196, 230, 323, 2, 58, 136, 128, 180, 170, 15, 202, 228, 5, 295, 213, 99, 283, 96, 355, 88, 142, 118, 254, 278, 231, 86, 259, 250, 13, 150, 304, 233, 182, 345, 155, 332, 203, 83, 120, 377, 14, 23, 206, 131, 379, 19, 109, 8, 101, 74, 205, 52, 177, 126, 354, 376, 303, 175, 298, 285, 141, 256, 0, 100, 222, 313, 133, 339, 17, 224, 209, 288, 272, 39, 276, 119, 139, 366, 11, 253, 262, 97, 85, 220, 271, 146, 6, 9, 257, 302, 201, 292, 178, 38, 149, 135, 381, 243, 24, 30, 216, 361, 351, 310, 56, 251, 198, 50, 47, 268, 281, 289, 157, 190, 132, 360, 63, 255, 305, 12, 108, 21, 134, 218, 49, 274, 344, 261, 335, 31, 67, 73, 127, 185, 138, 106, 369, 374, 318, 219, 159, 338, 89, 204, 94, 346, 51, 66, 59, 315, 194, 247, 102, 382, 320, 350, 165, 266, 1, 107, 358, 336] + 55: [154, 355, 304, 360, 248, 337, 270, 212, 354, 306, 57, 303, 262, 344, 322, 14, 349, 4, 218, 250, 140, 48, 267, 23, 113, 363, 266, 213, 98, 260, 90, 328, 216, 3, 246, 36, 268, 104, 125, 53, 41, 214, 180, 297, 101, 182, 45, 302, 335, 339, 277, 220, 67, 203, 175, 254, 235, 336, 258, 193, 0, 316, 89, 188, 201, 273, 230, 227, 359, 78, 145, 71, 242, 282, 186, 142, 88, 269, 47, 153, 276, 229, 85, 10, 77, 34, 109, 348, 321, 223, 173, 351, 25, 117, 236, 288, 247, 211, 138, 327, 179, 312, 161, 369, 174, 52, 35, 105, 341, 202, 50, 6, 80, 28, 198, 62, 382, 19, 334, 192, 74, 278, 189, 32, 366, 314, 243, 309, 238, 331, 353, 123, 22, 383, 54, 17, 340, 160, 2, 149, 228, 127, 146, 55, 12, 171, 205, 84, 167, 121, 298, 326, 151, 307, 30, 15, 280, 116, 103, 86, 126, 75, 18, 94, 332, 91, 239, 21, 333, 364, 31, 208, 232, 263, 283, 133, 285, 83, 108, 181, 69, 292, 59, 275, 241, 166, 191, 29, 362, 187, 87, 245, 82, 257, 357, 233, 271, 73, 16, 377, 60, 237, 38, 7, 1, 185, 365, 115, 107, 323, 70, 5, 284, 40, 122, 375, 261, 195, 172, 63, 373, 221, 158, 72, 200, 137, 176, 134, 20, 209, 286, 225, 338, 319, 293, 64, 240, 164, 141, 42, 162, 177, 93, 251, 124, 150, 253, 264, 294, 299, 368, 291, 129, 342, 231, 27, 147, 315, 222, 378, 281, 96, 165, 143, 300, 301, 99, 296, 56, 380, 170, 130, 379, 132, 120, 128, 183, 352, 51, 204, 305, 168, 356, 224, 148, 317, 135, 310, 136, 106, 287, 81, 39, 350, 265, 255, 295, 249, 210, 308, 234, 290, 110, 65, 311, 97, 371, 44, 156, 9, 100, 381, 159, 289, 11, 46, 76, 178, 226, 112, 370, 318, 313, 217, 68, 163, 252, 347, 197, 152, 358, 26, 272, 206, 325, 95, 102, 13, 194, 157, 37, 119, 330, 343, 58, 169, 324, 329, 199, 111, 361, 8, 79, 346, 114, 244, 259, 184, 118, 374, 219, 279, 256, 372, 274, 33, 24, 155, 196, 139, 320, 345, 49, 43, 376, 131, 66, 61, 190, 144, 207, 215, 92, 367] + 56: [321, 240, 156, 204, 9, 126, 154, 368, 251, 89, 83, 238, 324, 267, 144, 99, 215, 383, 23, 331, 82, 244, 284, 116, 74, 158, 201, 107, 236, 193, 289, 305, 380, 314, 111, 323, 290, 91, 224, 325, 365, 369, 248, 333, 319, 56, 5, 71, 336, 15, 122, 40, 282, 233, 67, 228, 344, 143, 159, 135, 355, 66, 378, 155, 312, 218, 176, 178, 115, 182, 3, 65, 257, 165, 85, 95, 77, 24, 43, 68, 136, 192, 124, 55, 382, 320, 239, 247, 151, 276, 25, 37, 50, 61, 90, 103, 53, 101, 342, 0, 181, 160, 299, 134, 338, 356, 21, 104, 148, 377, 343, 12, 361, 79, 29, 341, 123, 139, 315, 316, 186, 88, 52, 297, 140, 60, 220, 242, 269, 362, 149, 259, 198, 196, 232, 359, 141, 335, 185, 47, 8, 189, 379, 357, 349, 188, 175, 351, 332, 381, 372, 235, 153, 306, 350, 373, 352, 273, 46, 73, 227, 348, 1, 237, 367, 221, 265, 51, 78, 260, 206, 345, 2, 254, 255, 280, 31, 129, 86, 293, 358, 105, 110, 234, 145, 256, 92, 132, 302, 108, 109, 142, 226, 163, 327, 191, 138, 184, 207, 6, 281, 150, 245, 114, 295, 42, 303, 219, 33, 339, 328, 27, 363, 72, 317, 310, 283, 48, 225, 102, 261, 347, 253, 197, 147, 113, 376, 203, 230, 301, 18, 268, 64, 311, 169, 128, 194, 329, 208, 205, 272, 309, 318, 243, 69, 80, 59, 49, 171, 38, 13, 195, 250, 164, 222, 252, 63, 57, 217, 20, 35, 120, 271, 258, 246, 26, 286, 313, 340, 76, 45, 274, 170, 296, 127, 326, 360, 200, 152, 262, 173, 133, 112, 180, 364, 146, 202, 287, 75, 294, 11, 322, 187, 44, 166, 374, 81, 241, 162, 100, 212, 209, 337, 231, 10, 278, 117, 292, 210, 39, 106, 97, 84, 172, 177, 62, 334, 168, 28, 93, 96, 300, 174, 304, 275, 199, 279, 167, 14, 264, 30, 130, 34, 121, 308, 4, 353, 266, 249, 98, 119, 16, 298, 223, 291, 183, 161, 17, 125, 277, 32, 36, 213, 330, 19, 190, 94, 137, 354, 7, 375, 157, 214, 371, 346, 263, 229, 131, 87, 54, 22, 58, 216, 285, 179, 307, 270, 41, 70, 118, 211, 370, 288, 366] + 57: [254, 345, 302, 90, 342, 94, 50, 33, 366, 223, 227, 346, 180, 260, 40, 21, 150, 383, 238, 352, 211, 182, 102, 136, 47, 300, 267, 339, 259, 54, 70, 66, 209, 109, 146, 208, 286, 34, 277, 188, 113, 287, 25, 86, 92, 198, 91, 246, 273, 174, 266, 176, 149, 186, 295, 173, 151, 127, 175, 7, 160, 200, 330, 72, 237, 298, 310, 343, 17, 2, 335, 329, 46, 119, 309, 99, 142, 278, 171, 229, 271, 112, 202, 178, 107, 101, 361, 131, 241, 234, 24, 194, 215, 221, 258, 275, 231, 125, 306, 43, 169, 356, 29, 347, 251, 217, 55, 85, 362, 158, 235, 134, 137, 236, 95, 203, 111, 270, 332, 122, 65, 60, 35, 71, 16, 228, 195, 257, 358, 297, 219, 324, 22, 199, 374, 177, 214, 115, 382, 312, 1, 61, 156, 325, 349, 216, 172, 26, 184, 193, 11, 344, 185, 274, 110, 212, 120, 321, 39, 207, 328, 222, 100, 363, 284, 82, 282, 213, 187, 83, 78, 77, 48, 292, 296, 370, 351, 132, 18, 320, 98, 244, 245, 291, 340, 263, 80, 232, 45, 262, 293, 301, 104, 68, 159, 58, 117, 226, 315, 272, 27, 179, 314, 62, 350, 41, 289, 196, 290, 152, 144, 87, 252, 118, 154, 138, 128, 123, 379, 106, 357, 79, 368, 103, 57, 116, 20, 225, 153, 67, 8, 12, 0, 253, 377, 307, 316, 157, 265, 162, 375, 224, 294, 168, 32, 166, 230, 167, 28, 281, 218, 256, 318, 313, 96, 23, 233, 360, 371, 206, 140, 121, 38, 30, 37, 14, 88, 124, 81, 163, 56, 164, 63, 276, 255, 3, 249, 69, 210, 376, 4, 336, 5, 299, 248, 13, 261, 311, 89, 44, 141, 264, 220, 6, 9, 197, 303, 317, 365, 268, 84, 331, 31, 279, 326, 288, 323, 367, 161, 108, 145, 354, 170, 130, 73, 201, 148, 133, 243, 285, 338, 126, 240, 341, 64, 191, 204, 369, 42, 181, 378, 155, 165, 59, 139, 283, 205, 242, 381, 247, 334, 348, 359, 74, 250, 327, 190, 319, 355, 373, 76, 114, 15, 322, 135, 183, 10, 51, 93, 380, 192, 304, 53, 189, 239, 143, 269, 97, 36, 129, 305, 105, 52, 19, 280, 75, 49, 147, 337, 308, 372, 333, 353, 364] + 58: [220, 348, 116, 178, 368, 344, 221, 347, 365, 216, 210, 192, 194, 213, 314, 86, 288, 175, 297, 318, 54, 82, 320, 312, 76, 269, 93, 232, 128, 137, 350, 94, 304, 219, 245, 27, 198, 222, 302, 166, 196, 89, 48, 111, 44, 16, 343, 70, 90, 295, 139, 217, 241, 25, 281, 177, 351, 107, 130, 160, 91, 7, 354, 319, 324, 135, 228, 339, 381, 71, 174, 197, 164, 140, 294, 1, 29, 57, 224, 17, 252, 218, 65, 363, 284, 332, 310, 249, 8, 11, 23, 39, 51, 62, 109, 117, 126, 49, 56, 380, 258, 40, 99, 115, 366, 199, 257, 242, 136, 255, 373, 151, 103, 66, 227, 342, 383, 9, 67, 68, 37, 259, 195, 34, 378, 356, 74, 278, 334, 315, 105, 149, 3, 64, 120, 349, 20, 69, 223, 361, 83, 268, 52, 155, 79, 207, 317, 293, 289, 75, 276, 0, 270, 327, 185, 337, 15, 285, 84, 275, 325, 85, 247, 208, 10, 323, 370, 32, 169, 180, 88, 96, 179, 146, 272, 188, 45, 362, 209, 87, 147, 33, 95, 38, 296, 134, 158, 159, 165, 184, 204, 212, 98, 148, 305, 42, 31, 377, 298, 129, 211, 72, 104, 277, 311, 266, 357, 133, 238, 308, 333, 182, 274, 4, 122, 121, 119, 100, 152, 256, 322, 307, 181, 80, 35, 141, 286, 331, 59, 55, 14, 63, 283, 306, 187, 316, 113, 292, 6, 176, 355, 2, 153, 244, 60, 144, 251, 206, 19, 273, 336, 353, 47, 214, 170, 261, 24, 183, 112, 168, 280, 231, 142, 154, 157, 193, 260, 191, 326, 163, 230, 229, 330, 233, 201, 108, 371, 101, 162, 145, 150, 41, 340, 239, 240, 246, 267, 279, 22, 352, 28, 264, 124, 203, 12, 271, 18, 262, 114, 234, 102, 92, 190, 50, 250, 77, 127, 78, 58, 263, 341, 106, 205, 225, 46, 329, 236, 26, 13, 328, 374, 376, 382, 248, 291, 300, 321, 131, 143, 123, 172, 202, 36, 360, 358, 171, 21, 97, 30, 303, 167, 73, 237, 226, 282, 372, 156, 254, 110, 189, 138, 173, 61, 200, 379, 359, 161, 346, 118, 235, 43, 53, 367, 243, 81, 313, 265, 375, 132, 5, 290, 299, 287, 125, 186, 301, 253, 215, 369, 309, 335, 338, 345, 364] + 59: [288, 364, 59, 23, 274, 212, 258, 231, 266, 236, 224, 353, 377, 371, 113, 178, 115, 339, 378, 52, 202, 150, 91, 346, 260, 315, 36, 180, 187, 213, 314, 259, 262, 43, 372, 251, 107, 366, 312, 195, 298, 114, 307, 345, 174, 163, 222, 373, 3, 124, 65, 223, 38, 47, 352, 344, 234, 358, 317, 206, 84, 166, 102, 64, 216, 191, 147, 360, 263, 29, 96, 269, 280, 348, 83, 101, 293, 144, 276, 318, 14, 201, 249, 322, 215, 341, 253, 205, 31, 155, 4, 15, 27, 32, 67, 71, 75, 151, 97, 158, 300, 28, 148, 232, 182, 226, 184, 143, 278, 172, 159, 37, 95, 304, 53, 383, 175, 169, 60, 106, 250, 11, 70, 140, 285, 243, 76, 319, 248, 12, 120, 292, 246, 332, 183, 25, 185, 24, 26, 199, 194, 13, 306, 283, 61, 305, 20, 359, 130, 221, 48, 82, 146, 282, 19, 17, 121, 272, 382, 177, 370, 237, 116, 68, 5, 105, 77, 340, 367, 323, 176, 108, 336, 74, 168, 129, 2, 125, 153, 167, 119, 110, 165, 230, 302, 351, 173, 90, 111, 117, 171, 189, 327, 379, 81, 44, 164, 245, 320, 324, 329, 21, 34, 181, 295, 196, 86, 291, 338, 270, 197, 316, 290, 7, 331, 149, 252, 45, 241, 254, 350, 78, 356, 369, 209, 69, 57, 325, 265, 30, 160, 365, 16, 56, 161, 41, 103, 100, 257, 239, 233, 204, 281, 138, 35, 380, 362, 289, 85, 354, 39, 79, 337, 286, 145, 55, 333, 122, 88, 33, 247, 349, 328, 66, 287, 299, 242, 273, 132, 92, 303, 54, 208, 87, 368, 72, 179, 326, 228, 238, 49, 271, 128, 135, 190, 210, 218, 235, 296, 186, 46, 137, 73, 334, 203, 284, 10, 294, 42, 363, 240, 297, 94, 170, 198, 62, 256, 342, 109, 112, 330, 313, 376, 127, 214, 200, 162, 118, 142, 381, 98, 141, 374, 219, 131, 18, 347, 93, 311, 309, 279, 136, 152, 50, 343, 229, 133, 51, 63, 40, 275, 123, 0, 9, 255, 89, 104, 1, 8, 301, 277, 139, 6, 308, 261, 227, 80, 134, 157, 335, 192, 220, 267, 156, 264, 22, 126, 193, 217, 188, 225, 58, 321, 207, 211, 310, 244, 154, 355, 99, 357, 268, 361, 375] + 60: [323, 112, 71, 318, 6, 332, 102, 38, 283, 192, 40, 78, 152, 215, 237, 354, 234, 166, 133, 273, 106, 374, 95, 265, 342, 269, 115, 24, 58, 32, 144, 36, 268, 179, 165, 232, 352, 167, 271, 47, 278, 316, 114, 351, 27, 365, 148, 353, 126, 371, 96, 285, 211, 315, 20, 177, 101, 43, 90, 163, 361, 53, 307, 169, 231, 59, 131, 21, 184, 41, 360, 287, 176, 4, 327, 324, 196, 376, 256, 154, 336, 272, 348, 325, 340, 187, 328, 65, 274, 9, 13, 14, 31, 67, 68, 73, 313, 312, 46, 236, 85, 51, 57, 135, 155, 337, 130, 139, 84, 302, 212, 110, 60, 260, 362, 190, 188, 264, 98, 129, 244, 369, 55, 364, 174, 213, 296, 350, 137, 162, 291, 49, 151, 314, 266, 48, 333, 158, 214, 77, 42, 113, 334, 164, 346, 319, 259, 311, 282, 44, 141, 25, 80, 203, 276, 8, 226, 197, 372, 138, 17, 128, 299, 202, 229, 72, 12, 370, 368, 23, 383, 91, 250, 56, 228, 199, 382, 279, 380, 26, 89, 347, 377, 69, 275, 86, 109, 122, 125, 147, 157, 160, 349, 153, 297, 123, 335, 194, 263, 107, 172, 39, 100, 195, 83, 2, 378, 111, 3, 140, 124, 15, 356, 99, 189, 230, 121, 132, 52, 64, 247, 246, 82, 358, 262, 11, 191, 117, 338, 366, 239, 217, 252, 45, 286, 173, 175, 88, 331, 204, 227, 345, 233, 240, 241, 223, 290, 293, 367, 156, 225, 178, 18, 19, 359, 103, 238, 220, 330, 251, 357, 363, 308, 216, 119, 373, 161, 22, 289, 105, 326, 10, 339, 97, 70, 104, 63, 118, 62, 198, 5, 200, 317, 168, 201, 206, 207, 210, 181, 134, 81, 54, 75, 306, 1, 180, 28, 108, 295, 381, 92, 344, 243, 305, 7, 193, 146, 61, 304, 219, 379, 294, 149, 300, 74, 120, 185, 37, 322, 245, 267, 284, 303, 310, 257, 186, 209, 258, 143, 50, 171, 255, 35, 116, 343, 222, 218, 145, 66, 76, 170, 159, 94, 281, 142, 0, 183, 235, 16, 29, 298, 136, 270, 208, 261, 248, 33, 277, 150, 320, 309, 182, 205, 34, 79, 127, 253, 93, 221, 87, 242, 30, 224, 321, 249, 254, 280, 288, 292, 301, 329, 341, 355, 375] + 61: [73, 341, 101, 225, 192, 176, 21, 263, 72, 28, 51, 366, 74, 122, 309, 45, 335, 367, 195, 200, 182, 338, 186, 382, 136, 202, 205, 216, 271, 377, 348, 286, 238, 224, 218, 83, 142, 169, 346, 327, 47, 17, 177, 105, 121, 170, 295, 129, 100, 29, 11, 23, 25, 143, 196, 9, 296, 68, 318, 69, 154, 323, 16, 19, 155, 302, 369, 114, 81, 181, 93, 3, 375, 149, 276, 371, 148, 63, 10, 364, 171, 210, 208, 6, 259, 266, 242, 58, 41, 0, 240, 275, 67, 332, 96, 347, 197, 36, 184, 91, 127, 183, 289, 61, 168, 261, 372, 243, 135, 27, 298, 215, 294, 280, 92, 284, 125, 111, 248, 297, 314, 55, 207, 285, 232, 119, 5, 365, 324, 308, 85, 246, 326, 262, 151, 70, 150, 352, 330, 59, 65, 157, 7, 77, 62, 333, 118, 245, 337, 131, 373, 78, 376, 103, 204, 368, 138, 126, 354, 79, 76, 161, 301, 44, 305, 18, 137, 312, 160, 99, 175, 292, 48, 49, 37, 56, 89, 359, 340, 339, 12, 325, 278, 124, 283, 8, 281, 123, 353, 106, 267, 329, 221, 33, 230, 277, 270, 350, 322, 203, 256, 255, 4, 66, 201, 288, 2, 166, 24, 198, 80, 120, 98, 293, 174, 213, 54, 102, 82, 97, 32, 117, 172, 362, 90, 290, 43, 315, 113, 264, 193, 253, 185, 235, 158, 383, 269, 217, 194, 336, 282, 206, 233, 22, 291, 317, 252, 279, 360, 144, 303, 95, 31, 84, 38, 378, 307, 310, 46, 251, 132, 247, 64, 226, 351, 116, 141, 34, 134, 187, 52, 115, 164, 222, 287, 254, 381, 236, 163, 345, 50, 231, 152, 53, 331, 13, 319, 39, 86, 159, 380, 257, 379, 214, 212, 311, 328, 190, 272, 227, 344, 30, 179, 14, 299, 112, 145, 223, 355, 146, 361, 139, 237, 358, 349, 220, 273, 304, 60, 167, 42, 108, 234, 107, 268, 147, 75, 229, 104, 239, 370, 374, 241, 342, 219, 88, 191, 260, 20, 211, 110, 130, 173, 258, 209, 156, 249, 244, 87, 178, 228, 71, 199, 363, 165, 306, 343, 313, 274, 357, 189, 57, 188, 128, 109, 321, 162, 334, 40, 26, 356, 250, 265, 153, 1, 180, 35, 94, 140, 320, 300, 15, 133, 316] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots384.yaml b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots384.yaml new file mode 100644 index 000000000000..474423374e6e --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 150, 177, 163, 29, 68, 126, 243, 156, 200, 208, 375, 244, 365, 192, 270, 263, 286, 340, 334, 250, 226, 288, 185, 111, 0, 235, 295, 80, 176, 349, 284, 56, 311, 337, 38, 257, 331, 59, 221, 237, 98, 115, 179, 164, 121, 88, 61, 378, 259, 348, 188, 242, 139, 282, 54, 233, 174, 148, 358, 172, 249, 369, 292, 67, 229, 206, 191, 251, 24, 168, 303, 166, 290, 162, 169, 62, 65, 276, 33, 34, 90, 28, 268, 85, 103, 269, 106, 127, 6, 42, 53, 357, 92, 339, 58, 256, 224, 155, 182, 122, 63, 37, 307, 196, 186, 248, 27, 52, 108, 116, 215, 93, 210, 64, 353, 285, 298, 8, 13, 74, 173, 86, 312, 51, 255, 318, 359, 43, 354, 44, 149, 32, 213, 19, 296, 14, 50, 96, 12, 319, 329, 344, 310, 222, 330, 383, 332, 347, 70, 278, 181, 313, 297, 300, 123, 66, 146, 140, 180, 267, 102, 294, 2, 381, 374, 291, 247, 356, 217, 119, 230, 104, 218, 209, 55, 131, 258, 261, 262, 377, 240, 178, 211, 326, 154, 305, 69, 194, 271, 273, 370, 236, 49, 322, 193, 30, 157, 144, 336, 101, 129, 94, 234, 91, 227, 275, 379, 82, 260, 316, 41, 231, 142, 228, 274, 245, 203, 71, 241, 204, 232, 280, 360, 11, 36, 112, 89, 87, 46, 75, 238, 328, 252, 338, 152, 239, 189, 281, 201, 371, 184, 345, 207, 81, 321, 3, 141, 105, 265, 138, 199, 223, 137, 363, 1, 382, 350, 161, 175, 367, 287, 325, 110, 225, 134, 84, 264, 26, 376, 333, 78, 283, 109, 7, 31, 133, 335, 366, 320, 299, 342, 212, 380, 125, 18, 130, 277, 254, 95, 151, 346, 153, 327, 272, 22, 117, 124, 170, 77, 308, 136, 372, 195, 197, 145, 40, 219, 9, 23, 355, 373, 135, 17, 302, 190, 35, 343, 293, 72, 107, 361, 114, 160, 5, 317, 216, 97, 73, 20, 352, 113, 301, 304, 128, 120, 279, 79, 159, 76, 21, 48, 198, 205, 341, 57, 368, 309, 39, 4, 246, 100, 147, 47, 15, 306, 220, 167, 323, 118, 45, 60, 351, 158, 202, 289, 16, 143, 183, 10, 324, 314, 187, 214, 99, 25, 83, 362, 253, 315, 266, 132, 364, 171] + 1: [163, 178, 126, 241, 41, 56, 25, 37, 301, 335, 53, 360, 102, 330, 65, 156, 26, 346, 344, 369, 151, 66, 304, 368, 319, 134, 85, 168, 270, 300, 46, 211, 285, 162, 47, 88, 98, 362, 14, 165, 87, 8, 232, 297, 160, 16, 258, 100, 365, 292, 235, 21, 366, 63, 243, 275, 274, 218, 206, 264, 205, 140, 286, 321, 186, 181, 240, 309, 193, 45, 144, 199, 42, 272, 1, 378, 192, 294, 320, 260, 78, 166, 176, 203, 189, 313, 318, 57, 172, 311, 314, 375, 207, 208, 354, 267, 201, 82, 80, 149, 325, 271, 185, 71, 266, 187, 228, 289, 76, 121, 326, 62, 290, 352, 103, 83, 135, 10, 257, 310, 174, 111, 106, 150, 0, 250, 383, 86, 317, 197, 219, 171, 61, 79, 2, 115, 11, 113, 54, 279, 183, 132, 341, 255, 22, 225, 118, 43, 70, 23, 24, 242, 125, 288, 282, 381, 298, 122, 17, 296, 51, 268, 226, 371, 253, 361, 157, 34, 137, 120, 18, 263, 328, 248, 233, 32, 216, 92, 223, 342, 101, 164, 107, 19, 230, 96, 119, 246, 188, 252, 339, 265, 180, 198, 31, 196, 75, 244, 224, 173, 281, 84, 114, 278, 236, 238, 338, 175, 284, 227, 333, 331, 367, 191, 276, 312, 158, 182, 308, 50, 210, 153, 351, 89, 234, 194, 143, 239, 147, 348, 20, 184, 293, 13, 262, 372, 90, 124, 337, 15, 277, 3, 141, 254, 6, 73, 195, 9, 74, 261, 4, 38, 133, 170, 139, 229, 315, 94, 353, 129, 316, 374, 117, 327, 382, 377, 81, 329, 39, 334, 138, 145, 99, 280, 213, 202, 231, 222, 161, 295, 350, 302, 49, 148, 217, 128, 159, 336, 91, 67, 380, 40, 179, 105, 59, 303, 146, 379, 93, 358, 237, 305, 249, 33, 324, 55, 12, 190, 209, 155, 349, 345, 72, 347, 251, 291, 5, 58, 29, 269, 212, 109, 116, 306, 256, 204, 364, 136, 36, 323, 77, 357, 221, 322, 359, 363, 169, 167, 44, 152, 123, 299, 283, 247, 245, 108, 52, 69, 95, 259, 370, 154, 35, 97, 356, 220, 340, 64, 307, 215, 177, 127, 200, 131, 30, 110, 273, 287, 355, 142, 214, 376, 60, 7, 332, 68, 48, 104, 27, 130, 373, 343, 28, 112] + 2: [143, 230, 5, 197, 111, 355, 26, 279, 196, 273, 68, 383, 10, 193, 142, 2, 36, 331, 95, 183, 234, 260, 337, 229, 248, 180, 81, 151, 322, 280, 324, 83, 204, 130, 203, 297, 46, 69, 372, 176, 77, 352, 329, 307, 37, 353, 282, 300, 53, 122, 221, 75, 84, 157, 22, 235, 311, 87, 27, 361, 308, 160, 154, 179, 134, 359, 216, 48, 113, 64, 228, 330, 43, 139, 137, 149, 101, 271, 220, 24, 323, 263, 50, 187, 241, 309, 39, 362, 281, 57, 243, 74, 182, 80, 326, 374, 132, 314, 1, 100, 191, 73, 31, 146, 61, 206, 210, 58, 253, 103, 38, 268, 4, 272, 99, 251, 227, 240, 339, 199, 112, 304, 298, 189, 166, 278, 51, 217, 79, 299, 327, 201, 41, 186, 49, 94, 275, 231, 292, 98, 117, 380, 32, 366, 108, 86, 259, 125, 284, 286, 45, 274, 294, 140, 188, 236, 65, 15, 305, 152, 195, 141, 226, 158, 378, 266, 316, 381, 313, 377, 343, 192, 153, 306, 123, 340, 342, 90, 269, 138, 332, 129, 256, 318, 35, 52, 291, 267, 283, 249, 270, 190, 215, 91, 102, 34, 350, 66, 325, 63, 209, 17, 67, 148, 265, 181, 312, 219, 211, 144, 302, 59, 382, 88, 287, 40, 89, 168, 28, 321, 173, 245, 105, 333, 159, 356, 126, 218, 16, 169, 12, 29, 334, 233, 54, 164, 72, 127, 119, 161, 194, 62, 14, 78, 97, 131, 368, 200, 174, 170, 92, 76, 349, 276, 364, 104, 346, 261, 71, 167, 177, 369, 237, 246, 93, 19, 357, 60, 110, 44, 133, 82, 6, 7, 262, 198, 360, 116, 9, 257, 367, 207, 107, 205, 222, 13, 208, 375, 115, 290, 341, 213, 109, 118, 42, 244, 30, 347, 55, 20, 145, 348, 252, 23, 239, 319, 288, 8, 310, 289, 301, 379, 223, 232, 351, 33, 155, 70, 172, 85, 303, 224, 293, 21, 171, 175, 121, 0, 25, 264, 370, 335, 345, 214, 202, 363, 3, 242, 96, 162, 185, 135, 184, 11, 285, 18, 156, 120, 163, 250, 106, 150, 47, 178, 371, 338, 147, 317, 354, 255, 295, 212, 225, 247, 124, 56, 296, 328, 165, 344, 320, 128, 277, 254, 136, 365, 238, 376, 373, 114, 336, 315, 358, 258] + 3: [66, 113, 237, 184, 91, 110, 206, 378, 88, 90, 124, 326, 5, 247, 335, 233, 146, 324, 278, 303, 311, 222, 301, 370, 34, 236, 169, 383, 115, 20, 356, 101, 285, 45, 159, 4, 244, 371, 332, 52, 9, 257, 198, 199, 292, 59, 47, 38, 310, 56, 150, 337, 170, 325, 316, 21, 84, 32, 155, 299, 112, 30, 196, 291, 373, 362, 372, 338, 147, 265, 33, 341, 260, 13, 318, 215, 120, 189, 195, 118, 109, 229, 345, 256, 105, 268, 286, 361, 85, 153, 379, 107, 360, 60, 347, 17, 377, 317, 151, 331, 102, 250, 296, 290, 364, 359, 243, 218, 197, 280, 366, 298, 15, 259, 283, 190, 191, 369, 69, 152, 365, 31, 62, 50, 201, 58, 77, 11, 87, 154, 374, 41, 235, 65, 255, 322, 249, 130, 28, 354, 242, 141, 308, 167, 319, 51, 97, 49, 26, 164, 212, 182, 202, 161, 351, 126, 380, 117, 44, 276, 232, 376, 86, 0, 140, 368, 23, 173, 39, 209, 208, 334, 139, 336, 267, 174, 29, 8, 277, 300, 217, 355, 125, 306, 128, 106, 293, 179, 251, 1, 75, 185, 22, 230, 192, 158, 137, 343, 122, 177, 240, 162, 203, 176, 103, 270, 214, 43, 305, 381, 309, 183, 323, 95, 89, 193, 79, 35, 131, 363, 42, 168, 37, 239, 12, 78, 262, 98, 200, 327, 82, 160, 281, 123, 68, 16, 156, 148, 127, 226, 302, 275, 225, 348, 6, 63, 227, 74, 25, 136, 288, 114, 108, 53, 307, 100, 96, 294, 346, 133, 93, 284, 143, 175, 40, 46, 19, 24, 180, 178, 211, 221, 64, 271, 99, 297, 224, 57, 48, 328, 187, 367, 72, 269, 245, 73, 205, 315, 18, 71, 350, 157, 213, 207, 10, 304, 223, 279, 295, 92, 274, 94, 266, 144, 353, 7, 132, 254, 258, 329, 241, 320, 27, 263, 171, 333, 145, 339, 2, 231, 36, 382, 289, 111, 181, 344, 135, 246, 216, 67, 83, 228, 116, 234, 330, 272, 163, 313, 14, 80, 252, 238, 342, 287, 119, 264, 142, 149, 204, 55, 129, 104, 248, 70, 61, 282, 134, 121, 253, 349, 3, 194, 314, 261, 81, 186, 358, 138, 219, 357, 172, 312, 375, 166, 273, 76, 340, 188, 165, 321, 220, 210, 54, 352] + 4: [343, 253, 139, 361, 49, 35, 183, 133, 379, 216, 158, 66, 206, 237, 355, 311, 380, 134, 266, 231, 338, 182, 277, 282, 331, 86, 325, 151, 339, 22, 283, 85, 98, 261, 284, 220, 121, 327, 127, 10, 162, 202, 363, 242, 90, 107, 305, 332, 3, 320, 45, 345, 275, 365, 251, 82, 0, 360, 359, 70, 80, 92, 303, 140, 268, 129, 123, 293, 212, 337, 336, 351, 271, 259, 257, 47, 372, 357, 312, 207, 335, 252, 33, 232, 25, 267, 51, 249, 109, 205, 362, 144, 89, 225, 104, 120, 315, 258, 88, 143, 197, 76, 160, 8, 260, 243, 330, 341, 108, 288, 292, 29, 146, 211, 97, 48, 102, 301, 119, 58, 214, 375, 172, 171, 155, 200, 196, 280, 110, 116, 114, 169, 229, 310, 309, 157, 180, 71, 378, 316, 238, 299, 67, 321, 209, 18, 250, 105, 342, 145, 57, 73, 177, 99, 168, 323, 269, 328, 84, 184, 289, 130, 126, 247, 245, 326, 192, 63, 296, 340, 350, 5, 36, 347, 32, 27, 7, 278, 290, 62, 132, 69, 370, 176, 295, 147, 255, 374, 248, 285, 2, 287, 371, 34, 219, 19, 348, 167, 61, 218, 161, 187, 221, 189, 60, 125, 137, 138, 230, 304, 156, 186, 191, 265, 154, 208, 23, 188, 24, 376, 302, 329, 77, 314, 344, 369, 42, 256, 149, 74, 322, 294, 26, 170, 111, 115, 353, 11, 373, 20, 15, 152, 300, 281, 136, 274, 246, 101, 333, 141, 244, 166, 297, 262, 100, 306, 175, 291, 223, 203, 163, 91, 382, 272, 95, 254, 222, 239, 165, 377, 4, 83, 217, 317, 264, 356, 352, 383, 135, 349, 103, 122, 31, 215, 55, 164, 195, 233, 226, 148, 199, 228, 6, 56, 318, 235, 198, 153, 54, 307, 14, 64, 367, 178, 234, 334, 213, 81, 1, 368, 174, 17, 319, 142, 124, 9, 241, 13, 106, 364, 68, 159, 276, 72, 93, 131, 240, 52, 346, 118, 201, 21, 112, 224, 190, 227, 87, 117, 12, 16, 308, 298, 113, 43, 28, 179, 358, 53, 185, 313, 354, 50, 263, 181, 40, 381, 173, 366, 270, 79, 194, 273, 37, 193, 150, 59, 41, 44, 204, 38, 324, 65, 75, 46, 94, 39, 78, 210, 286, 96, 279, 30, 236, 128] + 5: [61, 65, 211, 252, 198, 131, 59, 251, 153, 46, 51, 56, 241, 254, 344, 221, 378, 86, 347, 307, 294, 326, 286, 354, 213, 260, 30, 72, 200, 304, 171, 181, 26, 45, 312, 366, 341, 119, 291, 283, 18, 297, 120, 282, 273, 348, 321, 75, 128, 308, 265, 145, 331, 173, 155, 140, 24, 301, 274, 227, 63, 90, 191, 205, 342, 322, 126, 311, 226, 280, 170, 368, 214, 208, 60, 255, 192, 74, 222, 2, 258, 16, 380, 14, 300, 9, 80, 323, 129, 189, 133, 328, 310, 83, 96, 302, 355, 172, 178, 78, 352, 335, 8, 206, 143, 295, 144, 20, 138, 383, 77, 271, 156, 125, 101, 350, 10, 27, 127, 44, 190, 21, 47, 372, 17, 338, 4, 332, 38, 3, 104, 266, 19, 250, 330, 134, 235, 68, 316, 267, 336, 42, 225, 196, 249, 259, 81, 122, 212, 364, 66, 358, 108, 210, 228, 186, 230, 150, 275, 361, 71, 299, 373, 292, 36, 180, 278, 118, 84, 159, 1, 365, 130, 246, 31, 70, 69, 207, 35, 115, 362, 201, 48, 87, 103, 114, 92, 139, 202, 116, 112, 177, 296, 231, 98, 381, 279, 7, 233, 137, 319, 320, 33, 12, 253, 284, 136, 229, 363, 135, 58, 185, 146, 357, 82, 157, 194, 272, 160, 169, 102, 293, 182, 163, 369, 333, 195, 79, 343, 219, 281, 29, 349, 40, 187, 197, 340, 248, 367, 303, 217, 6, 5, 188, 351, 88, 209, 151, 67, 204, 28, 99, 382, 121, 237, 13, 359, 270, 32, 290, 73, 234, 167, 179, 346, 288, 256, 223, 215, 89, 22, 268, 174, 34, 218, 329, 106, 257, 334, 360, 374, 107, 148, 11, 85, 339, 110, 277, 276, 52, 154, 370, 41, 147, 327, 37, 264, 152, 315, 345, 239, 240, 149, 236, 62, 244, 224, 53, 97, 109, 142, 57, 100, 356, 306, 49, 132, 43, 371, 313, 318, 175, 54, 216, 39, 161, 325, 232, 113, 263, 95, 245, 285, 123, 164, 337, 105, 287, 141, 262, 379, 91, 25, 375, 238, 314, 377, 93, 353, 317, 289, 94, 376, 168, 298, 309, 158, 117, 176, 23, 305, 269, 165, 193, 64, 247, 124, 15, 184, 324, 220, 242, 183, 203, 76, 162, 55, 199, 261, 111, 243, 0, 166, 50] + 6: [97, 378, 283, 178, 234, 336, 190, 139, 220, 69, 60, 254, 105, 381, 357, 106, 47, 250, 171, 72, 153, 331, 247, 257, 142, 350, 232, 118, 259, 128, 317, 77, 154, 87, 86, 261, 346, 70, 244, 108, 303, 263, 192, 265, 252, 40, 90, 161, 308, 371, 318, 84, 239, 113, 278, 111, 219, 365, 242, 217, 81, 224, 272, 73, 347, 195, 313, 172, 344, 204, 315, 342, 322, 55, 173, 88, 319, 300, 109, 231, 64, 332, 110, 63, 329, 147, 349, 130, 62, 351, 7, 26, 20, 75, 325, 373, 76, 341, 175, 187, 188, 156, 185, 49, 282, 143, 262, 116, 362, 18, 215, 39, 203, 338, 302, 213, 314, 132, 71, 164, 274, 269, 67, 216, 57, 196, 176, 245, 304, 107, 58, 271, 243, 145, 193, 268, 285, 288, 348, 339, 91, 61, 149, 323, 376, 249, 229, 181, 100, 184, 151, 41, 68, 327, 157, 361, 82, 14, 48, 125, 134, 226, 256, 374, 19, 382, 275, 35, 32, 59, 326, 260, 368, 79, 301, 182, 101, 211, 10, 206, 4, 194, 199, 309, 364, 251, 266, 38, 29, 228, 135, 320, 305, 212, 28, 9, 94, 255, 370, 179, 311, 321, 15, 6, 276, 355, 168, 8, 379, 150, 17, 51, 78, 92, 353, 167, 53, 297, 306, 209, 375, 112, 287, 165, 369, 158, 221, 286, 43, 284, 122, 3, 16, 366, 290, 102, 89, 241, 170, 214, 343, 146, 267, 11, 13, 367, 289, 95, 115, 30, 144, 296, 104, 335, 126, 124, 42, 123, 328, 162, 222, 1, 273, 52, 174, 177, 160, 205, 36, 189, 155, 0, 230, 12, 383, 324, 207, 197, 316, 198, 298, 235, 152, 121, 238, 140, 2, 65, 258, 166, 27, 46, 359, 293, 129, 345, 236, 202, 133, 136, 237, 159, 148, 358, 96, 292, 337, 294, 131, 127, 191, 363, 253, 233, 295, 120, 307, 119, 291, 277, 352, 279, 44, 264, 137, 23, 138, 83, 299, 312, 360, 24, 183, 93, 340, 186, 240, 66, 5, 31, 163, 56, 34, 270, 380, 223, 114, 280, 210, 225, 248, 50, 334, 74, 80, 21, 201, 141, 200, 25, 281, 99, 103, 180, 45, 246, 208, 354, 377, 37, 169, 98, 22, 330, 372, 333, 54, 310, 85, 117, 356, 227, 218, 33] + 7: [327, 260, 48, 181, 164, 119, 250, 177, 98, 314, 32, 335, 310, 247, 345, 216, 316, 80, 39, 230, 205, 279, 135, 60, 7, 308, 280, 61, 302, 29, 339, 210, 62, 321, 358, 220, 141, 233, 149, 222, 106, 88, 228, 266, 138, 122, 357, 50, 274, 258, 342, 27, 281, 262, 337, 255, 213, 368, 204, 322, 152, 373, 22, 79, 75, 218, 128, 195, 290, 326, 365, 364, 49, 341, 28, 188, 131, 100, 168, 319, 170, 241, 16, 129, 58, 306, 309, 52, 268, 295, 96, 334, 198, 352, 145, 240, 292, 163, 187, 97, 256, 215, 285, 374, 224, 132, 26, 180, 380, 155, 65, 120, 151, 36, 111, 379, 332, 12, 355, 257, 175, 369, 176, 15, 94, 307, 17, 325, 219, 333, 140, 190, 6, 353, 146, 225, 35, 69, 173, 43, 299, 127, 199, 338, 182, 298, 350, 354, 114, 113, 349, 296, 221, 243, 311, 186, 102, 56, 227, 193, 320, 194, 231, 252, 359, 143, 282, 42, 275, 362, 239, 378, 328, 47, 217, 360, 21, 377, 167, 20, 83, 121, 301, 93, 184, 112, 1, 297, 346, 165, 34, 348, 192, 25, 133, 118, 329, 249, 74, 55, 235, 191, 206, 130, 200, 134, 323, 153, 115, 166, 123, 317, 57, 66, 236, 160, 263, 201, 108, 99, 370, 73, 125, 23, 284, 300, 234, 237, 273, 226, 232, 87, 126, 209, 254, 3, 95, 197, 376, 110, 64, 103, 361, 71, 77, 183, 278, 84, 356, 344, 259, 136, 211, 67, 251, 38, 351, 169, 10, 277, 304, 331, 154, 371, 41, 18, 336, 271, 0, 313, 4, 2, 383, 264, 214, 19, 293, 53, 283, 101, 366, 207, 59, 381, 45, 150, 30, 179, 375, 105, 162, 253, 244, 294, 289, 270, 117, 372, 189, 109, 5, 44, 40, 291, 269, 286, 312, 124, 242, 147, 156, 91, 142, 340, 157, 13, 104, 14, 267, 148, 70, 330, 89, 245, 107, 86, 196, 68, 37, 287, 367, 303, 246, 172, 223, 276, 229, 382, 272, 139, 174, 11, 82, 288, 81, 158, 24, 92, 137, 305, 8, 261, 203, 363, 85, 63, 315, 54, 144, 202, 265, 343, 238, 46, 212, 324, 248, 31, 208, 90, 9, 185, 178, 51, 161, 116, 318, 33, 72, 78, 159, 347, 76, 171] + 8: [360, 64, 157, 156, 374, 150, 261, 198, 196, 72, 42, 379, 259, 299, 378, 145, 287, 193, 77, 80, 289, 19, 100, 207, 253, 108, 337, 380, 86, 69, 147, 65, 368, 120, 177, 45, 25, 151, 307, 265, 236, 357, 324, 208, 138, 49, 127, 170, 290, 161, 13, 252, 328, 22, 83, 216, 137, 334, 2, 8, 73, 90, 136, 99, 260, 33, 24, 245, 343, 57, 296, 234, 359, 104, 224, 36, 316, 131, 75, 119, 165, 183, 233, 4, 116, 249, 266, 105, 295, 54, 271, 285, 363, 0, 55, 96, 5, 273, 7, 228, 255, 280, 172, 38, 121, 309, 300, 91, 217, 230, 310, 37, 325, 306, 315, 348, 354, 305, 274, 213, 67, 281, 106, 12, 251, 209, 128, 6, 231, 303, 48, 197, 254, 98, 184, 372, 144, 9, 369, 158, 270, 203, 210, 242, 31, 182, 338, 312, 30, 11, 181, 283, 179, 275, 141, 211, 59, 248, 297, 126, 68, 93, 323, 103, 269, 53, 344, 58, 135, 204, 292, 241, 95, 364, 166, 23, 92, 178, 164, 341, 383, 118, 352, 155, 143, 322, 134, 84, 319, 331, 301, 132, 21, 229, 218, 154, 81, 377, 311, 226, 240, 304, 192, 318, 187, 191, 257, 314, 79, 35, 153, 146, 123, 70, 82, 163, 367, 371, 238, 173, 186, 18, 346, 133, 330, 40, 320, 358, 101, 87, 110, 41, 339, 175, 185, 370, 329, 342, 124, 227, 264, 152, 214, 282, 276, 97, 39, 109, 298, 215, 194, 60, 335, 237, 373, 382, 85, 89, 277, 76, 26, 43, 222, 262, 239, 353, 250, 272, 122, 168, 195, 333, 202, 219, 50, 114, 206, 174, 149, 47, 235, 159, 326, 220, 180, 51, 256, 63, 201, 308, 279, 381, 291, 349, 225, 34, 258, 112, 317, 52, 117, 212, 32, 190, 200, 205, 111, 351, 160, 286, 288, 376, 362, 221, 3, 88, 74, 115, 148, 188, 169, 167, 130, 125, 263, 28, 350, 327, 56, 375, 365, 15, 340, 94, 139, 1, 162, 62, 246, 243, 223, 44, 107, 27, 78, 278, 113, 66, 61, 232, 71, 16, 140, 14, 347, 129, 244, 142, 361, 176, 321, 10, 302, 336, 20, 294, 366, 356, 17, 189, 267, 332, 284, 247, 293, 313, 199, 29, 46, 171, 268, 355, 345, 102] + 9: [92, 198, 329, 32, 285, 268, 0, 311, 4, 181, 202, 18, 117, 322, 71, 216, 160, 58, 149, 40, 61, 168, 173, 26, 304, 29, 380, 132, 367, 153, 54, 87, 267, 189, 343, 175, 245, 79, 286, 38, 290, 368, 278, 257, 182, 338, 234, 57, 255, 172, 381, 242, 14, 9, 363, 264, 97, 186, 124, 332, 276, 307, 197, 110, 310, 232, 86, 43, 63, 362, 27, 283, 349, 19, 351, 370, 121, 133, 377, 297, 314, 269, 150, 72, 308, 379, 213, 254, 305, 346, 170, 108, 119, 28, 122, 340, 327, 81, 252, 37, 159, 56, 317, 3, 334, 116, 323, 238, 39, 354, 148, 112, 301, 162, 293, 243, 93, 204, 24, 365, 355, 166, 299, 344, 287, 282, 151, 154, 318, 41, 105, 66, 104, 248, 275, 203, 12, 125, 326, 68, 73, 378, 126, 348, 31, 78, 260, 184, 376, 209, 312, 187, 277, 107, 35, 206, 5, 75, 8, 261, 2, 375, 48, 188, 366, 361, 91, 265, 226, 212, 215, 222, 138, 67, 258, 135, 88, 192, 142, 230, 288, 235, 50, 373, 371, 333, 221, 59, 225, 101, 200, 77, 94, 10, 372, 179, 127, 136, 131, 302, 228, 128, 195, 36, 171, 246, 65, 205, 253, 270, 21, 337, 325, 223, 273, 152, 263, 147, 280, 155, 217, 23, 98, 356, 306, 364, 256, 16, 106, 324, 60, 233, 291, 134, 199, 167, 176, 118, 164, 25, 295, 103, 196, 165, 146, 51, 22, 158, 169, 183, 163, 11, 111, 13, 52, 300, 320, 76, 229, 374, 240, 279, 157, 7, 47, 95, 315, 219, 201, 341, 102, 30, 292, 247, 129, 336, 237, 319, 382, 174, 115, 330, 178, 335, 266, 328, 250, 156, 185, 244, 96, 74, 83, 271, 309, 144, 33, 321, 145, 70, 218, 208, 100, 190, 114, 251, 347, 34, 191, 44, 194, 236, 313, 64, 143, 357, 15, 214, 177, 82, 120, 180, 20, 207, 359, 80, 109, 45, 42, 224, 211, 193, 383, 284, 85, 369, 316, 49, 220, 350, 1, 210, 239, 161, 137, 231, 99, 139, 141, 339, 303, 62, 358, 352, 227, 69, 140, 296, 113, 53, 90, 262, 331, 353, 272, 281, 342, 360, 289, 6, 259, 123, 274, 17, 130, 46, 84, 298, 89, 241, 345, 249, 294, 55] + 10: [319, 263, 196, 76, 345, 360, 149, 233, 296, 9, 37, 267, 140, 13, 209, 111, 322, 251, 84, 156, 238, 283, 297, 369, 5, 8, 40, 154, 368, 208, 271, 229, 288, 219, 379, 97, 254, 220, 257, 206, 147, 118, 157, 242, 23, 295, 29, 336, 300, 215, 114, 47, 293, 274, 38, 32, 129, 75, 256, 230, 12, 50, 282, 287, 355, 158, 120, 143, 260, 82, 358, 68, 320, 126, 22, 351, 211, 312, 284, 212, 155, 214, 81, 357, 51, 125, 239, 113, 375, 373, 200, 273, 150, 228, 290, 366, 365, 137, 167, 144, 153, 304, 353, 356, 189, 266, 72, 311, 197, 25, 247, 323, 343, 169, 173, 205, 334, 347, 270, 291, 194, 268, 279, 101, 161, 19, 213, 234, 204, 218, 145, 105, 142, 327, 109, 64, 185, 307, 305, 303, 104, 172, 188, 324, 264, 93, 337, 225, 117, 138, 67, 272, 316, 249, 90, 341, 27, 112, 162, 354, 123, 96, 100, 4, 6, 344, 14, 381, 55, 246, 69, 57, 178, 66, 91, 252, 186, 61, 132, 21, 85, 328, 278, 184, 298, 65, 235, 35, 16, 207, 277, 349, 180, 255, 325, 79, 179, 241, 269, 363, 98, 231, 0, 87, 339, 378, 340, 174, 163, 159, 302, 165, 133, 333, 10, 70, 151, 192, 289, 102, 78, 330, 248, 136, 77, 376, 103, 107, 335, 338, 42, 39, 183, 131, 62, 17, 34, 352, 227, 236, 265, 294, 262, 73, 171, 210, 115, 11, 49, 359, 198, 232, 106, 41, 371, 275, 313, 226, 222, 217, 203, 191, 202, 110, 24, 301, 48, 2, 170, 99, 308, 240, 374, 285, 250, 56, 122, 74, 383, 317, 362, 299, 372, 224, 276, 361, 63, 89, 121, 310, 181, 309, 195, 237, 58, 130, 377, 382, 199, 135, 315, 20, 15, 332, 370, 364, 127, 86, 53, 83, 28, 253, 216, 146, 141, 92, 134, 182, 7, 243, 148, 306, 33, 321, 329, 26, 71, 45, 261, 326, 80, 292, 88, 281, 43, 166, 18, 164, 108, 367, 1, 124, 201, 348, 190, 223, 116, 46, 314, 59, 258, 193, 175, 244, 187, 119, 94, 44, 52, 176, 286, 31, 221, 331, 54, 177, 380, 259, 342, 280, 128, 3, 60, 346, 36, 318, 95, 350, 139, 168, 245, 30, 152, 160] + 11: [130, 376, 102, 217, 286, 154, 161, 69, 226, 89, 365, 327, 338, 302, 258, 138, 5, 112, 305, 175, 372, 155, 261, 269, 314, 260, 145, 144, 52, 136, 6, 45, 26, 10, 58, 336, 353, 311, 33, 182, 84, 210, 342, 216, 51, 212, 134, 335, 296, 77, 171, 22, 165, 146, 47, 213, 315, 316, 208, 219, 354, 324, 113, 7, 323, 222, 114, 328, 82, 329, 243, 241, 163, 252, 233, 345, 330, 177, 303, 367, 176, 133, 96, 294, 359, 118, 383, 19, 184, 4, 167, 344, 253, 334, 42, 12, 110, 156, 99, 0, 343, 381, 23, 170, 80, 151, 255, 242, 245, 193, 264, 355, 358, 306, 104, 71, 379, 122, 153, 61, 98, 15, 326, 218, 141, 282, 250, 280, 159, 374, 14, 209, 59, 158, 115, 225, 194, 271, 201, 272, 361, 111, 123, 105, 377, 285, 38, 332, 178, 337, 78, 371, 75, 56, 298, 270, 168, 179, 120, 62, 309, 37, 310, 183, 1, 202, 281, 109, 339, 357, 119, 24, 284, 318, 196, 124, 117, 251, 197, 81, 30, 135, 293, 263, 231, 274, 140, 132, 139, 53, 321, 191, 87, 126, 149, 162, 54, 204, 207, 320, 249, 27, 198, 259, 211, 279, 125, 142, 18, 108, 21, 268, 229, 333, 25, 256, 90, 100, 3, 29, 192, 70, 169, 297, 348, 347, 160, 41, 97, 292, 234, 174, 188, 106, 356, 277, 238, 239, 232, 307, 83, 248, 40, 267, 351, 128, 148, 313, 107, 93, 203, 186, 147, 364, 164, 57, 74, 103, 288, 304, 368, 276, 220, 273, 215, 44, 65, 199, 331, 95, 20, 346, 2, 172, 86, 31, 295, 378, 91, 64, 143, 200, 173, 380, 246, 181, 72, 257, 85, 116, 227, 300, 152, 39, 13, 66, 127, 92, 340, 101, 362, 34, 349, 79, 230, 375, 185, 68, 180, 189, 223, 50, 150, 36, 363, 157, 247, 131, 17, 94, 262, 373, 49, 289, 35, 366, 291, 43, 46, 360, 322, 214, 352, 317, 137, 308, 235, 11, 299, 190, 275, 244, 205, 319, 370, 195, 341, 254, 301, 265, 187, 48, 237, 266, 240, 278, 60, 28, 206, 9, 221, 121, 88, 325, 236, 73, 228, 283, 350, 166, 224, 55, 63, 312, 67, 129, 8, 382, 287, 290, 369, 16, 76, 32] + 12: [255, 288, 57, 291, 15, 350, 119, 249, 121, 257, 374, 252, 55, 72, 94, 100, 324, 73, 110, 272, 372, 220, 302, 42, 204, 292, 90, 259, 332, 368, 209, 107, 337, 283, 1, 45, 4, 38, 86, 297, 30, 282, 211, 269, 206, 64, 250, 225, 120, 109, 233, 354, 137, 25, 39, 322, 261, 286, 188, 165, 161, 11, 13, 244, 85, 375, 212, 79, 242, 213, 373, 53, 130, 175, 97, 208, 281, 317, 133, 171, 197, 144, 309, 0, 189, 169, 122, 112, 312, 310, 96, 236, 145, 314, 141, 366, 158, 344, 103, 260, 278, 330, 89, 265, 129, 157, 227, 52, 298, 319, 235, 336, 329, 47, 316, 320, 338, 328, 331, 199, 327, 49, 22, 151, 123, 216, 108, 102, 164, 23, 201, 114, 61, 325, 154, 379, 24, 383, 54, 125, 307, 166, 56, 162, 149, 363, 267, 50, 176, 205, 147, 153, 362, 326, 155, 200, 29, 280, 76, 20, 365, 274, 34, 81, 143, 360, 359, 222, 78, 156, 168, 192, 340, 68, 229, 27, 104, 80, 74, 253, 185, 67, 321, 355, 5, 186, 44, 93, 299, 221, 228, 66, 226, 341, 167, 290, 170, 382, 218, 364, 264, 63, 238, 370, 191, 62, 356, 36, 140, 217, 106, 113, 230, 7, 248, 43, 190, 46, 17, 124, 361, 195, 215, 60, 101, 135, 31, 357, 231, 138, 37, 159, 334, 263, 174, 131, 247, 111, 88, 239, 303, 203, 219, 126, 306, 295, 194, 87, 371, 273, 318, 376, 315, 256, 187, 270, 12, 128, 285, 18, 117, 118, 163, 377, 127, 333, 28, 349, 51, 6, 346, 16, 2, 241, 14, 304, 234, 224, 335, 115, 183, 207, 70, 178, 258, 184, 275, 240, 223, 172, 245, 289, 181, 3, 152, 10, 33, 308, 380, 77, 9, 313, 173, 40, 65, 279, 146, 343, 367, 26, 342, 353, 82, 150, 35, 71, 369, 339, 348, 323, 300, 271, 59, 142, 254, 262, 232, 305, 293, 148, 352, 347, 19, 182, 311, 99, 177, 243, 381, 296, 98, 134, 276, 266, 116, 180, 301, 193, 41, 84, 58, 214, 179, 284, 105, 345, 196, 210, 92, 21, 378, 69, 48, 287, 198, 75, 268, 277, 160, 294, 91, 358, 83, 136, 139, 351, 32, 246, 251, 8, 237, 132, 202, 95] + 13: [170, 46, 58, 40, 363, 147, 321, 333, 183, 267, 326, 245, 53, 73, 205, 336, 90, 20, 354, 155, 192, 204, 7, 175, 108, 0, 95, 17, 118, 198, 246, 320, 22, 239, 361, 222, 37, 220, 263, 94, 377, 285, 343, 152, 97, 368, 230, 32, 232, 96, 231, 88, 306, 369, 275, 208, 378, 145, 74, 52, 89, 379, 381, 154, 234, 233, 284, 258, 293, 156, 140, 316, 18, 81, 100, 11, 304, 75, 376, 334, 210, 26, 262, 221, 303, 345, 325, 308, 287, 119, 372, 224, 317, 148, 33, 78, 195, 77, 172, 134, 298, 45, 69, 229, 42, 179, 291, 256, 125, 337, 242, 113, 217, 61, 4, 60, 215, 47, 160, 165, 87, 137, 21, 102, 162, 295, 289, 158, 300, 237, 225, 177, 241, 68, 305, 133, 115, 365, 167, 203, 28, 269, 218, 206, 80, 194, 51, 340, 370, 176, 31, 143, 6, 355, 344, 121, 15, 34, 56, 257, 270, 213, 138, 1, 63, 272, 277, 301, 44, 9, 24, 209, 335, 279, 112, 91, 2, 211, 25, 196, 76, 223, 136, 244, 71, 124, 214, 101, 212, 341, 66, 193, 362, 318, 41, 254, 142, 323, 16, 261, 122, 79, 382, 93, 139, 288, 106, 259, 358, 313, 249, 117, 352, 290, 367, 126, 329, 274, 282, 330, 338, 131, 188, 294, 373, 219, 144, 359, 243, 36, 248, 127, 70, 235, 190, 85, 264, 332, 200, 315, 114, 30, 169, 276, 310, 35, 49, 297, 299, 65, 62, 166, 146, 105, 174, 268, 123, 13, 83, 157, 99, 347, 82, 314, 14, 12, 39, 201, 271, 159, 184, 273, 178, 98, 228, 107, 164, 331, 84, 92, 72, 38, 185, 346, 173, 251, 302, 350, 149, 238, 3, 342, 199, 171, 364, 356, 266, 10, 339, 253, 129, 163, 357, 348, 187, 5, 103, 383, 181, 371, 255, 324, 328, 189, 57, 110, 366, 307, 227, 54, 292, 375, 43, 374, 29, 48, 153, 296, 360, 19, 182, 180, 191, 236, 351, 27, 23, 280, 86, 55, 186, 116, 111, 309, 281, 168, 240, 283, 380, 8, 319, 50, 250, 130, 104, 226, 128, 349, 120, 247, 260, 312, 265, 135, 197, 109, 64, 202, 67, 207, 141, 151, 327, 161, 286, 59, 311, 150, 353, 252, 278, 132, 322, 216] + 14: [229, 228, 97, 182, 208, 246, 326, 133, 178, 377, 225, 264, 248, 173, 153, 46, 80, 185, 139, 51, 352, 205, 331, 165, 160, 383, 146, 3, 279, 166, 314, 201, 266, 109, 234, 0, 335, 63, 251, 195, 127, 281, 35, 84, 72, 29, 315, 174, 345, 274, 49, 254, 342, 271, 13, 328, 193, 346, 296, 380, 253, 313, 96, 273, 305, 190, 362, 236, 192, 38, 73, 50, 255, 56, 175, 99, 39, 250, 197, 343, 370, 382, 227, 105, 336, 12, 43, 262, 267, 18, 366, 247, 91, 340, 318, 88, 70, 147, 112, 23, 177, 374, 125, 158, 68, 104, 128, 101, 237, 242, 183, 376, 338, 74, 179, 293, 11, 219, 292, 260, 211, 275, 199, 142, 117, 360, 270, 33, 369, 333, 349, 48, 129, 223, 224, 42, 171, 78, 258, 55, 180, 134, 280, 67, 143, 71, 17, 235, 32, 269, 220, 59, 324, 301, 203, 222, 359, 202, 356, 114, 278, 136, 322, 57, 120, 252, 351, 289, 4, 155, 60, 348, 231, 21, 216, 149, 184, 108, 40, 156, 176, 89, 148, 285, 365, 238, 87, 132, 144, 243, 303, 98, 15, 308, 137, 337, 95, 79, 44, 135, 288, 363, 286, 82, 341, 152, 282, 106, 257, 20, 230, 62, 140, 283, 119, 83, 76, 378, 8, 118, 198, 61, 157, 206, 163, 339, 256, 5, 330, 168, 1, 311, 309, 304, 263, 31, 320, 27, 54, 196, 204, 355, 287, 353, 124, 162, 294, 215, 69, 37, 169, 347, 298, 300, 217, 28, 358, 334, 299, 194, 381, 368, 131, 272, 110, 126, 164, 150, 52, 154, 354, 161, 357, 22, 187, 116, 276, 200, 240, 214, 375, 361, 312, 7, 81, 307, 138, 212, 30, 327, 77, 186, 151, 213, 26, 379, 319, 159, 172, 226, 86, 115, 350, 310, 103, 36, 45, 367, 181, 65, 210, 249, 2, 122, 261, 344, 141, 371, 265, 323, 123, 167, 245, 373, 284, 316, 93, 6, 302, 9, 19, 92, 24, 188, 232, 295, 209, 34, 372, 268, 100, 317, 41, 25, 332, 145, 325, 241, 306, 189, 244, 10, 321, 14, 16, 47, 107, 113, 218, 121, 259, 75, 207, 239, 85, 64, 111, 297, 291, 277, 102, 90, 221, 191, 364, 94, 329, 290, 58, 66, 233, 170, 53, 130] + 15: [130, 236, 121, 23, 342, 235, 335, 246, 117, 282, 301, 180, 365, 123, 265, 91, 369, 280, 195, 125, 302, 192, 260, 85, 11, 185, 232, 353, 174, 18, 133, 372, 2, 50, 341, 347, 308, 169, 63, 190, 370, 136, 222, 304, 348, 25, 356, 328, 0, 39, 215, 157, 173, 138, 285, 209, 166, 34, 233, 339, 381, 45, 373, 218, 303, 149, 171, 330, 307, 19, 315, 276, 15, 199, 119, 329, 344, 111, 106, 5, 102, 77, 250, 382, 61, 55, 375, 54, 73, 14, 310, 228, 287, 219, 322, 368, 88, 291, 252, 221, 113, 143, 162, 323, 279, 10, 363, 30, 53, 204, 337, 52, 161, 29, 327, 181, 346, 186, 264, 270, 359, 4, 326, 51, 83, 241, 196, 47, 238, 248, 62, 12, 141, 244, 26, 172, 278, 126, 7, 98, 103, 249, 144, 93, 160, 290, 300, 203, 151, 277, 320, 36, 145, 269, 268, 13, 313, 314, 283, 334, 3, 188, 58, 361, 17, 297, 325, 281, 227, 298, 371, 284, 360, 193, 255, 225, 182, 212, 343, 81, 294, 72, 216, 95, 273, 197, 140, 124, 66, 120, 27, 46, 57, 68, 202, 333, 90, 376, 242, 379, 351, 247, 263, 150, 8, 191, 208, 383, 116, 65, 262, 112, 292, 70, 336, 366, 362, 134, 367, 74, 146, 139, 163, 31, 207, 105, 87, 229, 206, 349, 99, 37, 316, 274, 89, 176, 299, 21, 122, 127, 251, 104, 295, 266, 20, 56, 135, 115, 100, 305, 142, 96, 377, 321, 97, 224, 167, 239, 158, 187, 44, 110, 220, 6, 324, 148, 259, 159, 9, 16, 168, 24, 170, 49, 275, 69, 43, 107, 137, 64, 84, 79, 153, 293, 154, 243, 345, 75, 155, 175, 76, 231, 237, 35, 177, 109, 319, 42, 86, 22, 165, 226, 234, 71, 213, 147, 184, 318, 178, 286, 132, 152, 355, 217, 350, 332, 80, 48, 311, 352, 338, 101, 309, 331, 253, 306, 289, 211, 257, 210, 374, 380, 179, 108, 156, 131, 200, 128, 340, 38, 317, 296, 1, 230, 78, 28, 59, 205, 254, 67, 354, 92, 358, 378, 32, 201, 183, 288, 214, 198, 258, 94, 261, 189, 60, 40, 256, 164, 357, 194, 82, 114, 271, 118, 267, 272, 240, 364, 129, 312, 41, 223, 33, 245] + 16: [134, 274, 141, 226, 306, 377, 155, 8, 42, 339, 151, 200, 272, 91, 163, 20, 258, 288, 355, 192, 179, 94, 3, 148, 15, 224, 187, 16, 365, 248, 281, 305, 45, 137, 225, 57, 253, 109, 233, 268, 26, 176, 12, 74, 325, 98, 156, 249, 286, 5, 383, 270, 294, 97, 73, 182, 36, 9, 104, 60, 267, 142, 315, 17, 131, 241, 112, 130, 376, 344, 257, 353, 120, 330, 49, 374, 287, 212, 310, 38, 66, 219, 232, 22, 221, 158, 323, 196, 81, 128, 371, 352, 6, 135, 361, 111, 275, 67, 144, 183, 121, 123, 314, 335, 58, 41, 229, 238, 252, 208, 343, 207, 14, 127, 116, 332, 23, 40, 204, 126, 100, 68, 86, 205, 265, 87, 285, 307, 119, 184, 122, 160, 290, 102, 93, 48, 186, 113, 168, 149, 79, 254, 362, 247, 140, 162, 273, 35, 189, 347, 84, 51, 139, 231, 379, 297, 264, 18, 333, 266, 88, 157, 217, 150, 164, 329, 10, 345, 369, 11, 277, 259, 209, 367, 65, 2, 103, 31, 357, 322, 354, 54, 337, 30, 62, 284, 256, 245, 349, 341, 53, 13, 152, 271, 338, 348, 283, 246, 27, 115, 80, 380, 356, 276, 198, 132, 223, 190, 193, 324, 279, 61, 33, 291, 133, 69, 234, 75, 312, 129, 302, 364, 346, 52, 82, 153, 227, 372, 317, 29, 313, 244, 201, 47, 110, 138, 101, 326, 161, 215, 262, 280, 154, 342, 255, 46, 89, 237, 99, 282, 70, 202, 370, 340, 211, 105, 360, 107, 117, 210, 169, 197, 222, 309, 92, 118, 300, 90, 78, 235, 199, 71, 316, 39, 34, 293, 106, 228, 242, 378, 363, 145, 170, 230, 382, 263, 278, 366, 7, 303, 185, 203, 24, 236, 108, 328, 37, 159, 167, 334, 124, 336, 95, 304, 381, 218, 216, 243, 28, 350, 191, 239, 43, 44, 114, 318, 298, 213, 180, 177, 175, 0, 195, 301, 188, 375, 214, 55, 240, 261, 308, 251, 32, 299, 296, 368, 165, 64, 331, 289, 171, 295, 85, 321, 21, 56, 220, 19, 96, 25, 178, 319, 172, 166, 359, 260, 63, 136, 327, 1, 147, 269, 125, 143, 250, 146, 181, 59, 292, 311, 76, 174, 4, 358, 206, 83, 194, 373, 50, 77, 351, 173, 72, 320] + 17: [131, 197, 2, 75, 290, 177, 10, 272, 376, 36, 187, 221, 179, 249, 112, 124, 344, 227, 223, 285, 209, 169, 325, 354, 119, 193, 246, 370, 104, 381, 98, 298, 332, 20, 204, 377, 300, 262, 217, 192, 57, 132, 224, 306, 252, 14, 121, 152, 284, 91, 286, 8, 56, 180, 228, 282, 82, 13, 49, 115, 27, 299, 48, 274, 53, 280, 229, 71, 35, 159, 170, 202, 236, 208, 357, 266, 324, 172, 24, 38, 292, 304, 47, 118, 365, 44, 240, 287, 67, 318, 378, 257, 77, 51, 203, 235, 79, 80, 12, 232, 297, 207, 218, 273, 165, 191, 166, 215, 89, 181, 90, 329, 270, 31, 153, 349, 247, 111, 237, 251, 337, 307, 87, 5, 234, 339, 231, 143, 99, 162, 173, 41, 341, 265, 244, 64, 369, 65, 23, 109, 289, 268, 283, 312, 375, 138, 366, 275, 125, 167, 296, 243, 139, 314, 248, 42, 196, 271, 238, 183, 106, 350, 83, 141, 277, 334, 322, 320, 261, 212, 126, 225, 351, 175, 134, 199, 211, 214, 264, 61, 269, 253, 21, 361, 55, 308, 355, 149, 356, 348, 68, 86, 220, 28, 58, 359, 113, 328, 317, 335, 92, 74, 168, 331, 336, 309, 342, 102, 161, 364, 63, 241, 4, 178, 123, 293, 250, 374, 319, 155, 130, 62, 194, 73, 84, 256, 70, 60, 216, 382, 379, 310, 267, 372, 163, 40, 176, 353, 281, 122, 188, 362, 346, 117, 1, 276, 157, 206, 380, 263, 340, 19, 147, 101, 95, 258, 128, 158, 164, 230, 201, 0, 17, 127, 303, 294, 190, 136, 323, 189, 107, 148, 367, 33, 171, 43, 363, 186, 94, 302, 151, 278, 32, 315, 184, 242, 174, 34, 108, 305, 137, 311, 114, 347, 222, 97, 313, 9, 245, 260, 7, 37, 338, 120, 210, 76, 50, 316, 291, 160, 301, 358, 200, 330, 140, 254, 343, 6, 39, 46, 371, 135, 142, 116, 81, 198, 110, 93, 345, 255, 150, 352, 368, 288, 333, 69, 15, 26, 226, 219, 185, 133, 96, 22, 103, 146, 18, 45, 154, 205, 360, 195, 16, 326, 144, 88, 30, 11, 321, 72, 383, 100, 105, 78, 279, 29, 52, 145, 373, 327, 259, 59, 295, 3, 213, 239, 85, 25, 233, 182, 66, 129, 156, 54] + 18: [271, 374, 52, 74, 80, 97, 120, 156, 183, 222, 227, 242, 268, 293, 314, 344, 382, 1, 32, 59, 78, 147, 309, 340, 30, 95, 161, 305, 252, 336, 62, 134, 194, 221, 295, 310, 323, 196, 198, 353, 379, 49, 55, 91, 128, 186, 226, 327, 251, 315, 169, 139, 246, 99, 318, 20, 83, 316, 122, 328, 176, 319, 335, 153, 358, 66, 219, 369, 44, 299, 72, 206, 208, 333, 36, 355, 306, 341, 103, 371, 23, 79, 241, 250, 152, 178, 63, 292, 214, 48, 175, 229, 2, 258, 345, 11, 47, 53, 114, 136, 237, 330, 240, 188, 342, 87, 261, 7, 50, 195, 132, 329, 166, 108, 82, 163, 180, 372, 5, 100, 10, 45, 119, 189, 376, 35, 67, 164, 366, 121, 109, 232, 361, 54, 264, 13, 94, 383, 4, 51, 317, 85, 352, 173, 43, 193, 215, 236, 34, 378, 326, 70, 202, 262, 223, 296, 102, 111, 9, 267, 254, 303, 170, 197, 312, 42, 29, 18, 69, 272, 28, 142, 373, 144, 266, 46, 41, 104, 334, 288, 231, 125, 38, 145, 148, 127, 200, 224, 75, 107, 184, 117, 320, 356, 347, 116, 26, 151, 105, 274, 171, 359, 88, 367, 39, 8, 12, 363, 92, 179, 332, 17, 375, 90, 273, 160, 217, 118, 154, 380, 360, 297, 285, 269, 337, 149, 225, 159, 58, 140, 362, 351, 112, 275, 216, 126, 381, 115, 280, 212, 257, 6, 113, 192, 321, 291, 101, 19, 191, 73, 135, 350, 302, 205, 150, 182, 243, 213, 338, 130, 76, 84, 281, 277, 300, 260, 211, 174, 146, 357, 3, 204, 248, 177, 263, 22, 61, 203, 96, 187, 324, 256, 64, 259, 218, 86, 294, 290, 245, 185, 286, 239, 25, 284, 124, 98, 141, 343, 233, 278, 157, 349, 93, 255, 325, 348, 158, 311, 298, 172, 377, 21, 65, 106, 207, 110, 68, 282, 77, 89, 346, 283, 129, 0, 235, 308, 81, 155, 270, 60, 220, 56, 123, 354, 14, 230, 57, 228, 210, 368, 301, 253, 190, 181, 365, 331, 131, 137, 289, 15, 244, 40, 287, 234, 27, 247, 133, 370, 24, 249, 138, 71, 16, 304, 167, 31, 33, 165, 339, 307, 162, 201, 313, 279, 276, 265, 322, 143, 168, 209, 364, 37, 238, 199] + 19: [24, 216, 33, 117, 46, 127, 156, 2, 144, 336, 331, 280, 130, 82, 105, 361, 290, 367, 226, 220, 81, 98, 107, 133, 187, 70, 114, 349, 354, 25, 36, 337, 16, 67, 342, 237, 148, 88, 137, 208, 103, 381, 344, 282, 327, 198, 245, 253, 190, 91, 197, 38, 275, 147, 30, 182, 368, 347, 203, 213, 356, 376, 311, 132, 314, 106, 39, 86, 360, 382, 254, 263, 366, 52, 300, 164, 153, 171, 116, 165, 49, 297, 143, 177, 293, 281, 212, 14, 286, 357, 358, 163, 17, 43, 276, 355, 7, 48, 192, 184, 230, 292, 233, 310, 96, 40, 345, 346, 60, 372, 176, 139, 234, 112, 11, 21, 266, 274, 268, 319, 90, 339, 188, 193, 296, 9, 335, 6, 251, 257, 306, 284, 80, 3, 146, 87, 332, 150, 124, 51, 0, 55, 316, 369, 288, 243, 255, 373, 303, 189, 238, 134, 121, 272, 374, 85, 380, 140, 200, 186, 340, 231, 210, 100, 5, 78, 378, 375, 256, 141, 241, 8, 154, 271, 217, 246, 228, 353, 138, 31, 333, 23, 26, 326, 232, 174, 59, 267, 136, 56, 104, 93, 265, 115, 61, 183, 113, 301, 158, 13, 295, 68, 283, 110, 18, 131, 62, 329, 258, 235, 73, 269, 155, 236, 142, 109, 57, 307, 302, 318, 363, 97, 35, 53, 352, 172, 239, 249, 19, 108, 65, 362, 370, 247, 294, 289, 205, 248, 145, 229, 377, 64, 364, 279, 328, 89, 41, 15, 222, 270, 157, 4, 273, 160, 225, 341, 209, 28, 324, 123, 214, 261, 175, 305, 120, 240, 74, 219, 66, 159, 365, 285, 166, 173, 204, 215, 287, 75, 47, 119, 168, 63, 351, 162, 223, 308, 196, 317, 338, 101, 206, 126, 323, 152, 264, 122, 128, 167, 221, 259, 84, 202, 129, 321, 320, 94, 371, 178, 224, 379, 250, 350, 330, 312, 179, 322, 27, 77, 58, 54, 1, 83, 260, 277, 325, 262, 191, 383, 343, 149, 37, 118, 299, 102, 69, 12, 313, 207, 111, 291, 20, 334, 79, 71, 278, 76, 170, 244, 125, 45, 309, 242, 22, 181, 348, 185, 252, 151, 72, 34, 180, 42, 211, 161, 218, 227, 44, 135, 359, 29, 195, 95, 194, 10, 92, 298, 32, 99, 315, 201, 304, 169, 199, 50] + 20: [161, 309, 238, 20, 365, 253, 80, 142, 205, 72, 68, 103, 69, 27, 51, 52, 54, 61, 76, 107, 137, 167, 168, 179, 116, 111, 9, 49, 126, 194, 118, 247, 196, 248, 106, 312, 94, 11, 88, 341, 48, 70, 129, 109, 181, 189, 200, 206, 334, 339, 270, 324, 96, 292, 104, 364, 41, 213, 34, 169, 252, 326, 159, 151, 124, 354, 64, 147, 186, 143, 214, 262, 98, 14, 307, 318, 228, 187, 132, 315, 284, 175, 233, 114, 246, 33, 202, 230, 156, 321, 47, 207, 149, 222, 171, 263, 153, 272, 379, 188, 91, 362, 258, 182, 240, 219, 367, 131, 294, 78, 231, 351, 352, 28, 323, 313, 192, 275, 197, 266, 278, 174, 226, 185, 330, 229, 160, 358, 357, 348, 30, 150, 243, 338, 39, 24, 139, 199, 271, 136, 108, 301, 269, 281, 302, 37, 2, 105, 325, 95, 180, 172, 13, 35, 25, 22, 274, 232, 127, 327, 254, 296, 356, 157, 261, 306, 288, 299, 336, 122, 112, 353, 209, 273, 135, 121, 190, 378, 58, 208, 340, 310, 73, 89, 210, 289, 355, 211, 183, 344, 17, 241, 82, 8, 123, 32, 162, 290, 286, 77, 322, 343, 36, 256, 166, 5, 380, 191, 285, 79, 7, 225, 245, 31, 244, 303, 144, 282, 376, 204, 93, 311, 264, 65, 239, 375, 265, 372, 97, 297, 3, 212, 349, 115, 85, 305, 295, 152, 255, 331, 66, 46, 287, 62, 138, 125, 140, 320, 57, 235, 146, 119, 346, 12, 236, 360, 308, 319, 102, 83, 293, 155, 298, 350, 193, 335, 110, 291, 164, 382, 227, 184, 177, 267, 368, 178, 86, 370, 201, 337, 366, 250, 329, 276, 373, 203, 1, 242, 99, 4, 173, 195, 158, 141, 216, 92, 6, 361, 0, 18, 60, 100, 332, 374, 154, 283, 333, 16, 133, 220, 117, 249, 87, 145, 81, 363, 148, 128, 259, 101, 176, 328, 314, 279, 165, 377, 257, 75, 217, 21, 40, 19, 134, 251, 198, 260, 43, 268, 317, 130, 10, 221, 342, 71, 90, 218, 359, 300, 345, 29, 170, 84, 316, 120, 113, 63, 234, 23, 38, 277, 369, 45, 55, 224, 371, 381, 50, 347, 74, 26, 53, 42, 215, 15, 304, 223, 56, 59, 163, 67, 383, 237, 44, 280] + 21: [360, 92, 277, 91, 22, 366, 141, 234, 197, 64, 139, 347, 275, 160, 186, 333, 105, 143, 217, 265, 298, 297, 42, 17, 152, 82, 348, 118, 380, 25, 313, 89, 174, 223, 318, 378, 353, 337, 258, 273, 342, 156, 357, 285, 326, 311, 44, 35, 355, 46, 87, 345, 279, 349, 280, 288, 84, 309, 98, 125, 3, 14, 346, 379, 241, 253, 180, 54, 7, 231, 321, 339, 47, 236, 185, 45, 51, 128, 225, 220, 325, 331, 304, 159, 133, 301, 170, 48, 276, 163, 201, 55, 162, 32, 282, 350, 173, 256, 112, 120, 177, 365, 293, 221, 270, 49, 352, 312, 5, 30, 104, 126, 244, 181, 207, 85, 176, 59, 95, 58, 281, 190, 218, 214, 240, 147, 227, 367, 233, 267, 262, 97, 155, 154, 79, 203, 27, 110, 239, 99, 249, 61, 235, 93, 132, 153, 200, 232, 182, 151, 15, 9, 72, 372, 39, 383, 211, 40, 193, 216, 300, 315, 268, 114, 255, 116, 248, 123, 103, 289, 38, 361, 252, 184, 8, 111, 373, 73, 287, 286, 192, 266, 137, 274, 305, 195, 330, 375, 296, 166, 322, 264, 316, 302, 29, 88, 250, 351, 175, 144, 368, 63, 362, 303, 28, 237, 102, 329, 83, 334, 13, 130, 6, 31, 343, 306, 20, 140, 86, 169, 284, 191, 194, 230, 183, 229, 259, 260, 165, 295, 198, 269, 271, 319, 41, 374, 106, 142, 101, 108, 371, 113, 205, 199, 150, 26, 121, 52, 23, 1, 358, 21, 317, 100, 148, 340, 134, 381, 74, 2, 328, 188, 107, 179, 308, 257, 210, 117, 178, 307, 219, 209, 243, 131, 359, 66, 354, 115, 341, 57, 81, 382, 122, 68, 202, 145, 224, 369, 254, 71, 50, 363, 0, 16, 299, 43, 245, 338, 90, 251, 290, 65, 187, 261, 36, 12, 164, 37, 314, 212, 272, 10, 4, 119, 53, 94, 149, 172, 196, 135, 78, 263, 60, 246, 323, 77, 292, 335, 215, 67, 294, 56, 324, 222, 127, 327, 158, 376, 247, 364, 377, 320, 109, 344, 189, 18, 336, 356, 291, 171, 136, 24, 213, 310, 70, 76, 228, 161, 167, 332, 80, 124, 206, 208, 226, 238, 19, 69, 33, 168, 138, 62, 370, 204, 34, 146, 157, 96, 75, 129, 242, 278, 283, 11] + 22: [254, 58, 351, 375, 372, 122, 75, 321, 99, 341, 335, 241, 96, 262, 334, 342, 302, 43, 158, 231, 2, 28, 79, 95, 141, 38, 139, 55, 218, 312, 170, 228, 15, 165, 159, 256, 211, 136, 279, 52, 62, 344, 197, 163, 234, 101, 143, 154, 181, 332, 184, 93, 348, 7, 264, 259, 340, 49, 72, 301, 269, 268, 273, 103, 219, 64, 128, 162, 121, 297, 157, 217, 221, 126, 73, 329, 50, 338, 123, 226, 175, 324, 105, 303, 124, 45, 275, 108, 266, 291, 212, 23, 245, 313, 247, 250, 260, 9, 201, 244, 352, 111, 378, 27, 3, 25, 1, 305, 327, 286, 353, 232, 186, 350, 255, 326, 270, 315, 253, 267, 261, 204, 115, 169, 130, 345, 195, 60, 361, 191, 59, 39, 22, 373, 271, 176, 160, 36, 104, 363, 66, 167, 192, 307, 328, 33, 42, 263, 317, 149, 67, 125, 14, 48, 71, 320, 322, 32, 97, 26, 119, 30, 19, 354, 304, 243, 205, 310, 281, 316, 207, 112, 362, 153, 80, 83, 349, 113, 131, 133, 236, 189, 299, 356, 246, 137, 129, 296, 5, 277, 325, 346, 216, 371, 132, 74, 117, 272, 306, 8, 369, 82, 220, 239, 87, 377, 290, 364, 274, 151, 145, 308, 127, 295, 10, 249, 343, 94, 147, 4, 61, 357, 173, 76, 206, 358, 309, 41, 47, 78, 382, 46, 237, 92, 152, 31, 370, 166, 135, 280, 278, 110, 276, 177, 355, 70, 100, 161, 193, 339, 40, 367, 318, 11, 300, 379, 265, 120, 179, 150, 6, 293, 142, 287, 209, 376, 330, 37, 383, 381, 336, 68, 116, 156, 18, 210, 155, 54, 0, 314, 102, 196, 190, 223, 88, 107, 368, 359, 148, 178, 331, 144, 29, 240, 251, 365, 214, 53, 258, 380, 98, 283, 13, 202, 85, 182, 91, 168, 213, 84, 56, 289, 24, 20, 34, 230, 235, 229, 285, 222, 17, 366, 63, 12, 65, 227, 138, 215, 35, 187, 16, 69, 140, 114, 86, 292, 224, 183, 51, 164, 118, 333, 311, 174, 57, 347, 233, 225, 200, 199, 81, 77, 298, 294, 134, 90, 284, 172, 194, 323, 337, 288, 198, 242, 44, 252, 188, 106, 185, 109, 374, 319, 248, 238, 180, 89, 208, 282, 257, 171, 21, 146, 203, 360] + 23: [26, 374, 6, 309, 70, 152, 129, 119, 125, 341, 58, 28, 121, 25, 214, 9, 350, 22, 63, 186, 4, 320, 168, 288, 314, 197, 154, 218, 371, 184, 312, 194, 210, 66, 333, 303, 280, 108, 297, 355, 140, 88, 247, 199, 217, 189, 37, 265, 124, 77, 324, 180, 159, 340, 74, 317, 290, 157, 93, 137, 352, 200, 16, 285, 373, 282, 230, 334, 145, 262, 8, 286, 172, 211, 306, 232, 372, 130, 146, 151, 342, 370, 160, 178, 284, 256, 206, 236, 375, 310, 276, 259, 246, 296, 216, 81, 275, 376, 87, 176, 55, 92, 326, 229, 38, 359, 273, 347, 76, 109, 21, 349, 104, 188, 266, 171, 117, 13, 251, 64, 71, 69, 173, 47, 127, 257, 144, 149, 228, 44, 329, 207, 193, 339, 68, 362, 267, 367, 322, 181, 278, 18, 29, 242, 305, 51, 82, 287, 11, 315, 35, 220, 195, 311, 185, 5, 382, 73, 164, 14, 123, 205, 319, 383, 141, 83, 293, 43, 126, 225, 201, 358, 241, 86, 133, 249, 23, 292, 153, 52, 316, 90, 245, 122, 239, 248, 353, 165, 118, 105, 112, 75, 328, 281, 331, 103, 304, 148, 131, 102, 313, 215, 150, 53, 223, 143, 302, 155, 338, 308, 357, 238, 138, 183, 142, 15, 264, 177, 295, 327, 192, 270, 209, 378, 10, 255, 298, 65, 330, 258, 174, 354, 115, 79, 253, 377, 20, 289, 226, 110, 175, 135, 48, 279, 67, 351, 190, 219, 95, 221, 1, 231, 162, 54, 107, 163, 97, 260, 158, 337, 360, 345, 366, 113, 100, 300, 19, 12, 84, 182, 283, 50, 191, 40, 101, 365, 179, 250, 335, 346, 336, 106, 161, 277, 381, 30, 198, 291, 132, 72, 170, 203, 261, 59, 78, 111, 233, 136, 32, 96, 307, 204, 344, 134, 99, 139, 46, 368, 364, 42, 380, 39, 187, 85, 147, 240, 94, 169, 263, 80, 91, 224, 356, 98, 321, 196, 0, 212, 294, 227, 61, 62, 343, 208, 60, 234, 167, 252, 120, 269, 156, 57, 41, 379, 271, 237, 222, 17, 301, 348, 128, 7, 243, 369, 202, 34, 299, 318, 166, 3, 244, 323, 274, 24, 27, 2, 56, 213, 36, 272, 361, 254, 325, 45, 363, 114, 268, 33, 49, 31, 116, 89, 332, 235] + 24: [154, 5, 62, 135, 91, 142, 73, 310, 63, 295, 280, 108, 333, 175, 151, 171, 193, 236, 141, 243, 364, 81, 115, 166, 265, 112, 186, 354, 144, 138, 49, 114, 297, 197, 51, 181, 72, 10, 352, 262, 288, 256, 90, 199, 17, 374, 379, 383, 252, 28, 255, 159, 54, 350, 130, 206, 140, 168, 312, 218, 365, 212, 161, 329, 291, 367, 119, 200, 40, 21, 198, 241, 9, 259, 324, 231, 202, 273, 274, 380, 245, 147, 269, 239, 134, 276, 253, 99, 57, 248, 43, 143, 349, 82, 127, 264, 246, 328, 286, 375, 192, 15, 355, 177, 307, 169, 313, 244, 215, 275, 196, 136, 331, 25, 80, 228, 371, 100, 129, 298, 20, 95, 361, 230, 31, 306, 356, 113, 272, 321, 13, 66, 103, 282, 296, 281, 148, 183, 125, 240, 33, 214, 133, 314, 7, 110, 381, 301, 26, 366, 254, 224, 345, 216, 153, 339, 258, 170, 363, 373, 67, 109, 180, 165, 227, 152, 158, 353, 223, 211, 360, 334, 343, 344, 318, 377, 182, 58, 235, 304, 242, 65, 23, 303, 8, 42, 101, 201, 250, 69, 261, 179, 0, 87, 157, 220, 137, 195, 93, 46, 149, 287, 376, 16, 268, 309, 203, 173, 188, 50, 209, 229, 300, 117, 305, 232, 52, 97, 139, 294, 283, 204, 61, 131, 338, 12, 83, 222, 315, 270, 4, 14, 284, 102, 217, 187, 98, 285, 219, 251, 92, 290, 185, 104, 319, 53, 299, 226, 96, 205, 221, 323, 302, 346, 267, 86, 32, 191, 132, 362, 167, 289, 358, 327, 6, 128, 122, 266, 370, 24, 382, 156, 336, 77, 59, 278, 56, 194, 348, 88, 94, 19, 342, 247, 162, 316, 2, 34, 22, 189, 337, 64, 29, 164, 190, 308, 293, 145, 178, 84, 292, 37, 335, 107, 76, 160, 44, 18, 332, 208, 322, 368, 372, 85, 210, 121, 237, 213, 325, 41, 39, 48, 279, 233, 1, 225, 79, 176, 126, 163, 238, 68, 75, 340, 45, 36, 263, 369, 172, 27, 146, 277, 35, 260, 351, 3, 326, 311, 11, 30, 38, 120, 249, 359, 317, 184, 124, 347, 89, 155, 357, 70, 111, 105, 207, 78, 71, 106, 341, 118, 257, 116, 47, 60, 271, 150, 320, 55, 234, 378, 174, 123, 330, 74] + 25: [315, 331, 310, 159, 166, 117, 70, 371, 224, 363, 219, 151, 338, 19, 165, 239, 220, 267, 277, 123, 178, 91, 158, 326, 269, 253, 138, 196, 347, 187, 259, 257, 200, 251, 337, 167, 45, 41, 265, 169, 206, 195, 43, 382, 129, 233, 156, 263, 379, 216, 381, 373, 153, 321, 114, 278, 345, 202, 94, 79, 243, 111, 157, 208, 64, 264, 42, 266, 270, 303, 121, 189, 282, 246, 280, 314, 120, 234, 99, 126, 261, 203, 312, 308, 181, 31, 340, 292, 93, 311, 118, 286, 88, 288, 255, 44, 72, 78, 113, 190, 296, 368, 197, 256, 146, 307, 304, 81, 185, 201, 150, 361, 22, 281, 16, 128, 57, 335, 106, 170, 139, 334, 2, 171, 273, 318, 154, 84, 244, 161, 291, 237, 204, 328, 144, 300, 127, 35, 320, 175, 155, 3, 301, 194, 228, 298, 39, 10, 305, 230, 302, 377, 89, 97, 242, 295, 354, 365, 149, 53, 275, 75, 122, 40, 137, 48, 108, 47, 247, 268, 61, 112, 7, 59, 136, 341, 323, 218, 20, 319, 324, 134, 69, 238, 110, 177, 21, 46, 142, 95, 104, 148, 317, 168, 332, 77, 160, 380, 262, 346, 229, 367, 140, 191, 383, 213, 28, 212, 27, 186, 86, 210, 188, 63, 217, 50, 180, 258, 162, 376, 235, 32, 18, 306, 240, 60, 103, 115, 199, 290, 116, 370, 358, 353, 183, 96, 92, 205, 285, 182, 276, 83, 375, 152, 5, 102, 207, 222, 209, 164, 135, 132, 221, 15, 254, 24, 17, 271, 250, 226, 174, 344, 90, 272, 23, 65, 30, 25, 145, 33, 351, 294, 76, 54, 359, 336, 316, 297, 58, 67, 74, 109, 34, 364, 322, 1, 374, 13, 372, 98, 101, 327, 73, 313, 100, 131, 231, 279, 176, 352, 350, 37, 339, 105, 193, 248, 362, 87, 236, 147, 26, 241, 107, 62, 82, 0, 232, 8, 287, 215, 125, 366, 143, 211, 55, 330, 274, 356, 223, 51, 325, 293, 289, 360, 71, 329, 214, 192, 299, 309, 333, 38, 378, 9, 80, 227, 348, 369, 68, 357, 14, 355, 179, 260, 124, 283, 349, 119, 12, 198, 29, 284, 85, 52, 49, 66, 342, 252, 56, 343, 4, 225, 130, 184, 133, 11, 249, 163, 36, 173, 245, 141, 172, 6] + 26: [136, 204, 191, 174, 66, 318, 41, 129, 189, 368, 165, 335, 326, 258, 377, 95, 248, 54, 68, 86, 96, 106, 111, 139, 237, 224, 130, 135, 167, 359, 319, 306, 24, 57, 357, 115, 75, 280, 7, 140, 297, 39, 91, 282, 257, 132, 169, 170, 89, 342, 69, 266, 30, 332, 176, 166, 152, 194, 215, 267, 123, 232, 121, 255, 59, 77, 270, 42, 381, 168, 175, 209, 269, 271, 16, 38, 22, 200, 128, 133, 79, 118, 239, 107, 187, 301, 157, 105, 40, 154, 119, 283, 23, 379, 374, 375, 217, 349, 254, 137, 226, 382, 156, 26, 367, 173, 71, 161, 35, 196, 182, 300, 354, 74, 112, 199, 295, 366, 207, 218, 172, 281, 160, 183, 223, 126, 180, 178, 44, 241, 365, 113, 212, 231, 372, 220, 225, 72, 163, 205, 330, 84, 286, 228, 90, 233, 149, 316, 61, 146, 247, 360, 2, 186, 192, 277, 0, 50, 312, 197, 317, 334, 193, 274, 85, 222, 203, 234, 80, 5, 162, 33, 355, 92, 265, 179, 229, 122, 285, 10, 73, 256, 383, 49, 125, 347, 206, 314, 211, 345, 213, 242, 6, 320, 338, 344, 227, 245, 62, 48, 190, 327, 51, 104, 76, 93, 298, 219, 325, 8, 100, 101, 284, 310, 322, 246, 292, 150, 304, 276, 238, 376, 55, 148, 353, 103, 201, 195, 378, 114, 363, 78, 340, 43, 244, 216, 4, 28, 210, 261, 380, 230, 263, 279, 293, 36, 110, 198, 262, 97, 343, 159, 290, 350, 19, 264, 352, 64, 333, 296, 14, 46, 273, 305, 11, 339, 70, 337, 369, 108, 328, 272, 151, 356, 127, 65, 45, 370, 117, 329, 362, 346, 29, 13, 145, 147, 288, 348, 124, 253, 184, 94, 307, 88, 131, 259, 321, 102, 221, 81, 98, 32, 143, 3, 141, 9, 303, 324, 208, 164, 309, 351, 18, 291, 63, 116, 278, 158, 82, 138, 323, 99, 20, 15, 171, 144, 25, 302, 364, 34, 331, 358, 243, 188, 52, 260, 53, 181, 214, 120, 308, 249, 83, 311, 287, 27, 294, 87, 202, 37, 153, 12, 185, 275, 361, 373, 250, 235, 60, 299, 236, 21, 371, 289, 341, 315, 67, 31, 142, 251, 268, 17, 252, 177, 134, 336, 47, 109, 240, 56, 313, 58, 155, 1] + 27: [103, 290, 343, 322, 109, 261, 114, 296, 147, 9, 84, 139, 280, 302, 158, 345, 295, 92, 270, 240, 244, 262, 310, 338, 225, 58, 357, 188, 112, 323, 72, 82, 187, 370, 6, 274, 248, 121, 365, 35, 113, 186, 195, 192, 11, 169, 285, 342, 286, 117, 26, 181, 55, 21, 238, 89, 98, 155, 172, 327, 20, 318, 3, 105, 152, 74, 257, 64, 138, 363, 39, 53, 12, 145, 204, 24, 97, 242, 99, 78, 241, 88, 175, 127, 111, 66, 13, 130, 184, 264, 273, 166, 19, 371, 60, 137, 312, 104, 164, 366, 143, 272, 383, 308, 141, 15, 271, 5, 362, 249, 167, 79, 259, 293, 62, 283, 255, 57, 100, 146, 90, 301, 233, 358, 207, 48, 149, 83, 373, 161, 54, 215, 276, 44, 275, 306, 231, 253, 86, 334, 210, 298, 163, 176, 234, 81, 300, 376, 222, 230, 266, 115, 140, 42, 309, 174, 46, 2, 321, 153, 144, 359, 352, 94, 189, 221, 355, 220, 37, 354, 374, 128, 224, 251, 324, 156, 284, 133, 126, 304, 267, 171, 47, 361, 132, 319, 41, 372, 194, 148, 134, 180, 177, 34, 350, 49, 179, 75, 76, 294, 25, 27, 316, 136, 45, 299, 250, 123, 344, 346, 150, 16, 87, 160, 206, 200, 4, 211, 31, 119, 348, 228, 380, 258, 297, 356, 7, 291, 268, 43, 120, 107, 33, 281, 157, 142, 108, 170, 209, 203, 263, 341, 70, 315, 131, 336, 303, 77, 278, 317, 368, 326, 375, 106, 199, 205, 159, 185, 223, 382, 165, 311, 243, 212, 61, 227, 279, 320, 213, 183, 85, 232, 32, 191, 162, 379, 360, 239, 28, 367, 14, 96, 229, 68, 198, 277, 40, 235, 265, 80, 59, 269, 202, 237, 340, 216, 325, 337, 328, 246, 122, 18, 307, 217, 116, 154, 8, 330, 226, 314, 378, 236, 168, 349, 219, 135, 292, 190, 260, 196, 23, 193, 91, 329, 124, 67, 29, 95, 288, 208, 22, 110, 1, 331, 51, 247, 73, 30, 333, 71, 369, 335, 381, 339, 332, 305, 254, 214, 218, 347, 38, 0, 256, 282, 50, 289, 52, 56, 17, 63, 252, 69, 351, 197, 377, 182, 178, 102, 201, 118, 125, 10, 287, 129, 101, 353, 151, 313, 65, 364, 173, 245, 36, 93] + 28: [161, 289, 182, 247, 178, 256, 249, 381, 12, 297, 79, 351, 72, 76, 141, 313, 9, 138, 18, 60, 86, 115, 150, 156, 250, 253, 346, 59, 261, 285, 32, 216, 335, 222, 264, 274, 175, 170, 73, 266, 234, 232, 373, 53, 318, 336, 338, 343, 26, 248, 97, 108, 325, 228, 157, 125, 349, 316, 14, 265, 69, 148, 342, 372, 96, 251, 29, 101, 202, 11, 147, 180, 185, 0, 226, 307, 236, 71, 129, 337, 56, 43, 282, 327, 44, 362, 332, 151, 287, 17, 85, 45, 360, 40, 173, 207, 168, 312, 152, 113, 128, 227, 5, 90, 220, 273, 380, 194, 46, 82, 143, 211, 154, 323, 292, 55, 174, 350, 54, 190, 344, 355, 203, 214, 50, 42, 158, 361, 370, 193, 315, 290, 208, 81, 21, 363, 31, 218, 293, 252, 267, 65, 319, 353, 92, 4, 188, 176, 37, 320, 165, 130, 41, 367, 268, 68, 260, 98, 322, 334, 294, 348, 303, 281, 275, 189, 87, 215, 356, 321, 22, 93, 357, 3, 100, 167, 105, 13, 341, 184, 112, 64, 36, 210, 136, 149, 8, 146, 94, 83, 124, 257, 153, 197, 283, 155, 369, 219, 352, 67, 299, 19, 48, 121, 49, 57, 61, 270, 382, 119, 38, 229, 102, 145, 164, 279, 70, 241, 328, 120, 224, 223, 204, 235, 209, 217, 300, 1, 142, 305, 296, 15, 34, 80, 131, 276, 126, 181, 198, 311, 317, 286, 324, 103, 7, 368, 51, 88, 271, 20, 278, 118, 179, 295, 74, 104, 377, 330, 225, 237, 277, 187, 262, 329, 255, 99, 196, 122, 374, 137, 58, 160, 326, 376, 206, 333, 171, 358, 177, 133, 144, 364, 284, 347, 291, 309, 62, 298, 306, 25, 77, 288, 272, 95, 200, 135, 242, 238, 127, 27, 183, 375, 169, 230, 109, 117, 39, 47, 123, 254, 186, 359, 302, 52, 201, 195, 75, 166, 63, 28, 240, 308, 16, 78, 6, 23, 205, 280, 331, 89, 110, 192, 140, 354, 212, 379, 162, 107, 239, 172, 263, 378, 383, 163, 221, 340, 134, 91, 365, 159, 106, 84, 366, 345, 246, 139, 35, 304, 301, 10, 243, 30, 310, 233, 199, 191, 116, 339, 269, 33, 114, 244, 245, 314, 258, 66, 24, 213, 231, 259, 111, 371, 2, 132] + 29: [118, 124, 262, 326, 188, 123, 50, 139, 131, 78, 3, 242, 348, 114, 327, 62, 270, 290, 301, 100, 107, 241, 66, 91, 244, 283, 186, 112, 332, 38, 350, 232, 329, 104, 292, 31, 36, 368, 56, 69, 307, 30, 92, 330, 165, 281, 152, 176, 361, 383, 333, 160, 275, 207, 4, 273, 136, 345, 125, 344, 28, 135, 250, 13, 356, 246, 109, 80, 231, 206, 355, 285, 197, 89, 354, 16, 159, 370, 86, 378, 200, 59, 267, 269, 77, 46, 191, 338, 199, 113, 251, 41, 279, 268, 247, 108, 73, 255, 258, 148, 48, 216, 193, 298, 156, 272, 353, 142, 224, 315, 103, 42, 15, 162, 35, 223, 331, 342, 98, 249, 196, 11, 120, 351, 57, 237, 382, 61, 178, 349, 380, 32, 47, 52, 79, 84, 121, 168, 211, 358, 2, 335, 256, 6, 336, 127, 379, 208, 266, 163, 68, 324, 128, 347, 181, 313, 217, 369, 209, 9, 187, 376, 252, 133, 201, 64, 289, 26, 318, 311, 76, 320, 180, 190, 102, 132, 257, 18, 97, 317, 22, 93, 381, 158, 179, 149, 337, 189, 254, 1, 274, 177, 51, 23, 25, 202, 346, 63, 319, 248, 215, 300, 150, 328, 83, 157, 111, 7, 192, 219, 45, 164, 245, 372, 284, 185, 137, 173, 54, 110, 243, 87, 341, 360, 116, 225, 264, 340, 229, 126, 213, 288, 233, 306, 115, 234, 5, 260, 367, 322, 277, 282, 365, 74, 65, 343, 210, 94, 182, 280, 151, 205, 359, 302, 377, 161, 261, 194, 141, 325, 27, 291, 37, 212, 323, 44, 195, 129, 366, 175, 204, 314, 218, 146, 228, 371, 169, 312, 287, 70, 239, 308, 184, 49, 8, 221, 305, 240, 198, 236, 88, 339, 140, 134, 145, 259, 85, 72, 101, 20, 316, 40, 53, 153, 253, 17, 154, 96, 21, 321, 117, 363, 203, 60, 271, 309, 174, 375, 82, 172, 106, 222, 310, 105, 373, 10, 147, 166, 374, 334, 263, 155, 95, 81, 138, 29, 295, 67, 14, 33, 119, 235, 293, 278, 12, 171, 276, 167, 238, 230, 55, 220, 227, 0, 71, 296, 75, 226, 214, 39, 304, 362, 58, 265, 183, 24, 357, 364, 34, 19, 90, 294, 122, 170, 303, 299, 352, 43, 99, 297, 130, 286, 143, 144] + 30: [208, 160, 91, 142, 280, 73, 145, 342, 304, 287, 291, 255, 149, 229, 8, 21, 22, 31, 36, 40, 47, 88, 94, 101, 239, 274, 60, 167, 20, 111, 284, 318, 371, 381, 370, 173, 352, 328, 34, 131, 118, 123, 128, 134, 146, 185, 189, 218, 140, 59, 321, 283, 159, 336, 193, 344, 258, 329, 163, 109, 248, 166, 267, 57, 327, 178, 126, 235, 87, 176, 219, 237, 266, 242, 361, 147, 362, 138, 256, 320, 369, 96, 179, 310, 244, 288, 230, 32, 69, 368, 198, 265, 119, 201, 259, 268, 65, 76, 78, 234, 164, 340, 77, 58, 195, 67, 132, 335, 129, 332, 175, 377, 264, 85, 3, 16, 42, 191, 227, 269, 245, 333, 196, 188, 33, 97, 12, 71, 217, 281, 278, 114, 197, 115, 290, 202, 324, 323, 314, 300, 313, 251, 238, 322, 37, 213, 135, 11, 68, 216, 317, 275, 204, 356, 104, 383, 172, 55, 79, 276, 279, 125, 81, 190, 63, 252, 301, 334, 262, 184, 75, 206, 41, 303, 221, 192, 136, 285, 325, 103, 294, 48, 374, 95, 139, 155, 122, 308, 171, 4, 30, 305, 187, 124, 243, 2, 10, 17, 312, 152, 236, 260, 157, 84, 14, 66, 92, 110, 121, 116, 130, 105, 112, 56, 367, 380, 358, 254, 366, 295, 346, 250, 89, 177, 141, 226, 200, 353, 359, 286, 44, 360, 247, 90, 82, 355, 207, 199, 148, 345, 53, 158, 13, 168, 45, 316, 341, 357, 9, 296, 23, 261, 51, 293, 50, 25, 220, 161, 273, 15, 232, 372, 27, 70, 214, 133, 222, 46, 28, 271, 74, 100, 210, 382, 343, 108, 319, 349, 282, 154, 183, 83, 5, 169, 102, 270, 26, 307, 52, 289, 272, 378, 64, 212, 240, 253, 151, 277, 174, 38, 376, 375, 297, 311, 107, 326, 117, 127, 330, 39, 61, 331, 35, 62, 29, 257, 223, 302, 49, 6, 215, 18, 98, 54, 150, 241, 7, 194, 263, 348, 364, 153, 113, 0, 93, 354, 143, 156, 137, 24, 246, 315, 43, 1, 299, 292, 365, 72, 306, 211, 181, 337, 350, 203, 165, 233, 106, 338, 182, 363, 351, 373, 86, 225, 144, 205, 224, 339, 309, 170, 298, 231, 162, 180, 249, 186, 347, 99, 209, 19, 120, 80, 228, 379] + 31: [0, 186, 208, 258, 249, 40, 277, 202, 288, 8, 153, 305, 352, 226, 210, 327, 280, 35, 18, 104, 343, 132, 207, 329, 159, 364, 219, 119, 360, 112, 254, 55, 318, 307, 383, 12, 362, 228, 50, 101, 337, 181, 274, 319, 117, 67, 172, 350, 128, 292, 339, 140, 75, 214, 290, 353, 169, 46, 41, 142, 51, 108, 91, 238, 259, 342, 235, 282, 357, 154, 203, 229, 233, 265, 133, 150, 155, 286, 223, 138, 57, 205, 373, 336, 273, 11, 43, 308, 42, 64, 356, 114, 167, 200, 341, 347, 26, 331, 116, 268, 206, 72, 269, 244, 98, 65, 287, 303, 73, 177, 83, 375, 240, 113, 381, 184, 242, 13, 165, 120, 201, 125, 121, 270, 22, 283, 102, 189, 111, 243, 222, 256, 92, 195, 323, 275, 47, 309, 135, 109, 313, 27, 297, 180, 68, 320, 379, 267, 209, 34, 164, 236, 74, 368, 176, 374, 141, 183, 237, 220, 263, 367, 99, 52, 255, 30, 377, 246, 100, 248, 279, 95, 349, 188, 299, 211, 63, 2, 322, 213, 123, 110, 376, 48, 224, 157, 315, 79, 382, 311, 199, 37, 166, 227, 241, 25, 96, 61, 56, 78, 23, 245, 15, 284, 16, 285, 6, 371, 80, 139, 321, 127, 295, 332, 230, 90, 324, 76, 192, 215, 39, 170, 338, 82, 231, 366, 105, 312, 217, 354, 14, 130, 281, 178, 340, 193, 44, 359, 358, 212, 158, 87, 19, 250, 146, 60, 348, 106, 93, 81, 122, 118, 84, 29, 272, 1, 107, 251, 9, 7, 344, 161, 86, 289, 252, 185, 36, 326, 325, 58, 334, 145, 38, 66, 77, 97, 369, 361, 59, 194, 198, 129, 24, 53, 372, 218, 163, 294, 143, 261, 221, 187, 330, 378, 131, 33, 136, 175, 137, 314, 302, 196, 271, 264, 304, 160, 168, 293, 126, 174, 5, 89, 115, 70, 32, 147, 328, 171, 355, 28, 232, 134, 182, 21, 45, 260, 247, 204, 103, 298, 262, 156, 197, 257, 94, 149, 239, 363, 3, 335, 173, 333, 276, 296, 152, 88, 31, 316, 291, 49, 266, 346, 306, 345, 234, 191, 124, 179, 10, 148, 278, 144, 365, 69, 317, 253, 62, 380, 17, 54, 190, 225, 310, 4, 20, 351, 162, 85, 300, 216, 370, 301, 151, 71] + 32: [63, 256, 371, 6, 56, 339, 43, 128, 119, 170, 76, 98, 328, 162, 167, 169, 177, 181, 244, 245, 251, 262, 265, 296, 367, 65, 126, 273, 301, 325, 175, 150, 351, 84, 366, 154, 282, 195, 365, 0, 52, 180, 217, 15, 326, 173, 312, 373, 352, 229, 2, 380, 140, 335, 345, 303, 50, 178, 286, 3, 381, 172, 361, 197, 13, 330, 220, 364, 358, 295, 383, 375, 94, 135, 271, 219, 89, 97, 232, 168, 85, 206, 250, 336, 70, 309, 236, 221, 125, 248, 35, 278, 23, 17, 290, 379, 218, 234, 310, 274, 348, 151, 109, 7, 36, 101, 45, 32, 80, 10, 317, 9, 191, 222, 67, 79, 216, 47, 338, 57, 148, 246, 179, 257, 319, 279, 308, 189, 227, 25, 239, 127, 302, 22, 372, 99, 299, 223, 196, 138, 259, 112, 93, 300, 33, 155, 323, 12, 186, 211, 228, 313, 55, 26, 48, 183, 115, 340, 363, 42, 287, 341, 160, 343, 46, 74, 194, 316, 92, 288, 146, 8, 141, 283, 233, 306, 72, 231, 324, 382, 81, 4, 77, 82, 103, 304, 355, 149, 260, 192, 11, 333, 285, 275, 362, 121, 16, 202, 131, 113, 266, 104, 281, 182, 350, 24, 37, 311, 102, 356, 261, 334, 1, 368, 27, 54, 347, 322, 187, 66, 359, 96, 156, 294, 254, 69, 314, 59, 240, 337, 349, 136, 243, 117, 201, 208, 277, 88, 226, 90, 190, 329, 106, 280, 342, 184, 143, 289, 212, 354, 157, 86, 249, 129, 62, 5, 132, 20, 374, 224, 31, 95, 235, 114, 241, 166, 209, 360, 204, 225, 78, 19, 369, 71, 64, 237, 39, 205, 193, 210, 152, 238, 305, 188, 60, 18, 159, 14, 163, 344, 108, 130, 214, 174, 87, 120, 297, 199, 332, 230, 353, 276, 139, 320, 124, 376, 44, 203, 321, 272, 267, 145, 51, 107, 49, 21, 41, 83, 331, 357, 91, 111, 213, 293, 29, 247, 198, 110, 284, 73, 264, 118, 100, 161, 318, 123, 122, 164, 346, 370, 61, 263, 116, 252, 171, 185, 269, 142, 53, 315, 378, 144, 307, 153, 134, 255, 133, 215, 207, 68, 327, 38, 268, 200, 40, 75, 258, 58, 298, 292, 28, 253, 176, 30, 158, 34, 165, 105, 270, 377, 137, 242, 291, 147] + 33: [38, 68, 226, 376, 151, 222, 144, 230, 50, 370, 43, 28, 101, 318, 300, 119, 365, 274, 359, 223, 265, 293, 333, 372, 91, 153, 342, 201, 335, 160, 192, 9, 138, 216, 156, 279, 58, 215, 53, 295, 207, 168, 341, 195, 44, 47, 95, 220, 339, 176, 323, 84, 127, 17, 40, 217, 355, 383, 382, 180, 137, 66, 271, 327, 243, 381, 242, 315, 296, 203, 102, 277, 36, 51, 42, 179, 352, 273, 210, 289, 354, 109, 80, 367, 60, 292, 52, 67, 316, 245, 312, 253, 374, 0, 239, 287, 319, 328, 31, 73, 164, 233, 150, 268, 290, 16, 124, 209, 340, 106, 366, 248, 20, 132, 76, 59, 337, 304, 302, 34, 260, 167, 377, 82, 348, 37, 1, 356, 311, 173, 254, 104, 373, 345, 349, 278, 136, 23, 224, 118, 275, 99, 77, 129, 177, 86, 191, 107, 147, 361, 25, 237, 235, 93, 45, 375, 13, 234, 371, 70, 10, 238, 332, 98, 214, 309, 100, 264, 362, 89, 145, 379, 108, 48, 33, 208, 8, 187, 317, 55, 200, 249, 56, 103, 135, 346, 310, 62, 306, 321, 185, 83, 64, 94, 63, 39, 314, 5, 30, 193, 232, 351, 276, 262, 152, 213, 71, 166, 194, 74, 225, 175, 363, 2, 284, 69, 380, 267, 280, 96, 241, 270, 218, 155, 116, 202, 347, 162, 88, 170, 24, 285, 128, 298, 120, 257, 171, 7, 263, 236, 229, 307, 331, 297, 358, 255, 35, 131, 288, 90, 4, 157, 360, 15, 336, 112, 221, 111, 27, 178, 165, 344, 357, 161, 148, 154, 259, 183, 125, 169, 204, 334, 305, 329, 198, 256, 186, 368, 113, 110, 41, 130, 184, 244, 199, 57, 294, 142, 81, 299, 320, 325, 139, 266, 338, 227, 18, 211, 22, 196, 212, 364, 3, 65, 6, 182, 291, 353, 378, 172, 46, 32, 343, 283, 75, 231, 146, 190, 141, 133, 14, 205, 123, 19, 114, 122, 350, 324, 219, 286, 78, 134, 250, 228, 282, 12, 115, 301, 140, 121, 322, 21, 54, 308, 126, 303, 79, 246, 369, 105, 330, 26, 252, 181, 188, 247, 92, 313, 272, 159, 251, 258, 240, 158, 72, 206, 87, 117, 61, 149, 174, 197, 163, 49, 281, 29, 261, 11, 85, 326, 97, 269, 143, 189] + 34: [108, 194, 97, 300, 134, 12, 166, 195, 48, 247, 180, 10, 383, 314, 265, 93, 176, 104, 71, 252, 129, 14, 76, 119, 179, 223, 178, 2, 267, 124, 174, 197, 336, 72, 367, 352, 62, 186, 275, 86, 118, 205, 141, 54, 163, 273, 122, 132, 16, 354, 148, 284, 238, 77, 210, 136, 280, 328, 204, 361, 42, 298, 68, 219, 319, 165, 171, 128, 216, 343, 160, 189, 249, 151, 145, 138, 79, 6, 125, 184, 326, 133, 236, 333, 308, 127, 358, 88, 289, 380, 52, 55, 120, 106, 376, 193, 306, 237, 304, 40, 295, 8, 65, 81, 183, 143, 346, 364, 232, 19, 221, 85, 228, 100, 242, 112, 1, 325, 377, 202, 15, 47, 158, 362, 315, 78, 162, 329, 327, 137, 227, 89, 5, 114, 23, 51, 233, 307, 278, 20, 212, 344, 177, 198, 340, 207, 239, 243, 220, 191, 201, 110, 24, 37, 231, 116, 50, 157, 46, 147, 269, 331, 370, 90, 257, 182, 235, 229, 366, 29, 311, 240, 246, 41, 87, 107, 84, 60, 150, 102, 263, 277, 105, 264, 282, 75, 57, 155, 156, 303, 7, 211, 337, 382, 135, 291, 99, 335, 208, 144, 256, 317, 245, 214, 154, 164, 272, 192, 305, 255, 117, 297, 292, 94, 320, 213, 69, 372, 74, 167, 115, 271, 95, 290, 345, 172, 270, 169, 230, 294, 142, 266, 36, 226, 173, 342, 96, 188, 309, 363, 259, 61, 312, 203, 146, 92, 225, 373, 368, 251, 241, 0, 80, 63, 371, 268, 111, 274, 250, 253, 351, 234, 350, 224, 299, 190, 82, 293, 347, 222, 21, 58, 313, 33, 67, 287, 348, 322, 56, 98, 161, 286, 260, 339, 355, 276, 369, 279, 126, 196, 332, 185, 59, 288, 4, 53, 175, 217, 181, 209, 103, 140, 261, 152, 262, 353, 17, 285, 130, 338, 375, 365, 330, 44, 18, 31, 258, 11, 244, 356, 159, 206, 30, 359, 215, 318, 139, 296, 310, 34, 131, 9, 22, 349, 32, 316, 334, 3, 357, 321, 379, 254, 91, 153, 123, 199, 27, 170, 301, 323, 149, 64, 13, 28, 109, 39, 43, 378, 66, 341, 248, 218, 324, 25, 360, 83, 168, 200, 302, 187, 70, 73, 35, 113, 283, 45, 374, 101, 49, 26, 281, 38, 121, 381] + 35: [319, 211, 129, 186, 164, 159, 267, 36, 372, 122, 92, 214, 24, 301, 374, 87, 278, 89, 238, 33, 226, 199, 293, 177, 45, 62, 141, 346, 311, 331, 373, 272, 268, 313, 96, 277, 347, 376, 175, 35, 207, 115, 150, 148, 219, 2, 100, 343, 1, 80, 254, 153, 147, 335, 329, 360, 158, 30, 300, 350, 348, 166, 142, 338, 63, 245, 49, 107, 233, 280, 64, 228, 345, 309, 379, 21, 224, 326, 136, 160, 308, 294, 201, 261, 110, 383, 127, 84, 365, 149, 23, 10, 327, 252, 124, 85, 363, 48, 303, 4, 78, 307, 126, 187, 230, 213, 152, 287, 305, 330, 380, 25, 105, 316, 121, 77, 333, 247, 119, 291, 75, 340, 58, 324, 239, 234, 54, 321, 9, 215, 95, 172, 202, 216, 19, 88, 292, 370, 155, 288, 223, 0, 173, 359, 169, 281, 130, 138, 355, 194, 222, 357, 132, 190, 337, 235, 180, 34, 70, 344, 332, 377, 195, 165, 69, 188, 120, 304, 28, 113, 144, 47, 206, 349, 52, 102, 18, 328, 320, 170, 227, 204, 269, 43, 103, 253, 232, 183, 178, 208, 71, 13, 205, 322, 237, 257, 154, 265, 266, 134, 81, 104, 143, 323, 145, 382, 179, 73, 79, 248, 250, 255, 351, 242, 76, 381, 299, 72, 240, 55, 116, 364, 361, 37, 128, 61, 209, 11, 57, 106, 184, 279, 176, 135, 181, 263, 83, 371, 59, 283, 218, 276, 182, 286, 221, 375, 162, 27, 325, 171, 243, 16, 74, 369, 185, 285, 198, 189, 249, 358, 270, 123, 229, 284, 131, 203, 44, 368, 31, 352, 14, 163, 296, 5, 137, 32, 366, 264, 7, 196, 274, 259, 306, 97, 197, 258, 212, 200, 297, 342, 220, 318, 289, 46, 50, 98, 168, 192, 42, 90, 111, 91, 99, 17, 378, 29, 112, 167, 271, 336, 251, 60, 367, 86, 353, 56, 3, 41, 191, 312, 244, 298, 108, 256, 8, 82, 273, 217, 317, 161, 94, 262, 225, 51, 341, 40, 295, 53, 246, 67, 38, 114, 362, 157, 140, 282, 125, 12, 20, 117, 290, 315, 65, 151, 26, 354, 310, 210, 39, 146, 68, 133, 260, 314, 231, 15, 241, 275, 356, 93, 334, 302, 109, 339, 174, 66, 6, 156, 22, 139, 101, 193, 236, 118] + 36: [232, 380, 79, 42, 112, 49, 68, 171, 41, 211, 81, 335, 29, 58, 315, 249, 115, 8, 89, 181, 64, 170, 273, 302, 22, 59, 241, 97, 47, 33, 231, 167, 172, 153, 228, 217, 212, 17, 208, 239, 381, 337, 312, 87, 141, 222, 127, 360, 184, 370, 98, 207, 5, 262, 67, 138, 50, 168, 147, 305, 143, 157, 283, 281, 383, 357, 350, 126, 269, 306, 313, 375, 371, 197, 299, 56, 133, 261, 108, 293, 284, 342, 117, 174, 180, 39, 165, 365, 44, 6, 35, 137, 34, 96, 367, 336, 368, 294, 101, 40, 308, 32, 334, 194, 282, 289, 20, 100, 310, 28, 244, 51, 146, 77, 248, 253, 86, 124, 317, 270, 272, 252, 328, 369, 150, 85, 331, 191, 139, 183, 276, 116, 202, 257, 88, 23, 190, 158, 60, 14, 154, 330, 114, 145, 53, 192, 2, 274, 152, 93, 240, 219, 297, 245, 280, 237, 323, 36, 148, 80, 204, 259, 215, 110, 303, 27, 214, 16, 340, 131, 62, 196, 275, 92, 333, 9, 63, 164, 216, 179, 187, 52, 188, 119, 247, 169, 130, 226, 151, 227, 43, 268, 341, 107, 185, 258, 291, 205, 201, 246, 55, 69, 155, 66, 364, 104, 351, 78, 209, 265, 256, 61, 235, 286, 377, 25, 144, 223, 74, 320, 352, 332, 45, 319, 103, 91, 346, 195, 102, 200, 267, 163, 329, 338, 316, 254, 378, 4, 106, 37, 57, 12, 250, 260, 99, 54, 309, 111, 361, 300, 15, 271, 229, 0, 129, 220, 70, 295, 347, 118, 1, 128, 160, 279, 304, 372, 84, 230, 90, 314, 321, 362, 349, 3, 339, 177, 46, 76, 379, 48, 363, 236, 251, 213, 7, 327, 113, 376, 238, 225, 382, 296, 301, 31, 156, 94, 132, 298, 354, 224, 277, 264, 149, 243, 24, 109, 21, 120, 285, 203, 221, 290, 266, 193, 176, 242, 374, 125, 65, 345, 13, 322, 72, 343, 30, 71, 263, 318, 83, 189, 307, 75, 278, 136, 234, 292, 10, 178, 348, 210, 73, 287, 175, 166, 122, 121, 366, 140, 255, 38, 206, 18, 135, 173, 199, 355, 26, 353, 182, 186, 325, 311, 359, 326, 373, 233, 162, 288, 105, 95, 142, 11, 134, 344, 358, 159, 356, 198, 19, 324, 161, 82, 218, 123] + 37: [218, 104, 296, 111, 100, 207, 313, 210, 110, 216, 198, 106, 155, 307, 232, 263, 226, 112, 132, 236, 239, 288, 352, 383, 40, 171, 9, 181, 18, 85, 27, 41, 103, 356, 262, 94, 280, 53, 170, 259, 146, 177, 89, 358, 31, 375, 8, 75, 268, 178, 44, 361, 253, 147, 131, 322, 21, 365, 161, 15, 69, 255, 33, 249, 152, 378, 142, 214, 11, 276, 189, 120, 294, 151, 190, 174, 129, 327, 153, 359, 228, 149, 304, 285, 55, 321, 24, 267, 124, 260, 308, 184, 197, 186, 23, 99, 148, 243, 205, 292, 156, 252, 233, 192, 208, 173, 338, 135, 126, 334, 102, 105, 10, 266, 283, 225, 261, 90, 282, 209, 293, 273, 52, 305, 86, 245, 328, 257, 229, 64, 217, 333, 136, 37, 97, 325, 17, 303, 339, 57, 202, 381, 346, 251, 109, 66, 306, 179, 223, 360, 139, 330, 336, 51, 203, 30, 3, 331, 312, 274, 298, 215, 115, 63, 237, 45, 377, 56, 301, 364, 125, 272, 77, 185, 318, 32, 157, 258, 6, 141, 376, 83, 244, 38, 351, 254, 286, 374, 130, 227, 2, 246, 379, 71, 314, 48, 382, 302, 14, 343, 315, 287, 323, 36, 329, 224, 347, 284, 172, 138, 291, 310, 231, 281, 367, 275, 58, 319, 213, 372, 371, 47, 165, 163, 150, 122, 168, 39, 201, 247, 98, 279, 116, 160, 175, 35, 166, 20, 54, 67, 158, 240, 311, 341, 95, 355, 65, 118, 145, 76, 144, 143, 29, 206, 380, 317, 187, 13, 332, 81, 248, 180, 220, 84, 222, 188, 348, 46, 342, 140, 182, 235, 164, 370, 1, 43, 92, 335, 80, 373, 154, 26, 169, 183, 324, 326, 12, 211, 200, 0, 242, 289, 193, 114, 7, 194, 133, 316, 176, 82, 108, 34, 269, 212, 4, 256, 350, 191, 357, 300, 219, 25, 366, 340, 309, 362, 234, 59, 88, 117, 221, 19, 196, 61, 264, 49, 121, 241, 42, 368, 78, 295, 107, 70, 369, 238, 79, 113, 123, 363, 72, 344, 353, 93, 277, 320, 74, 230, 278, 87, 128, 195, 354, 91, 60, 101, 62, 28, 159, 50, 349, 162, 265, 199, 290, 16, 22, 204, 345, 127, 270, 73, 134, 167, 337, 271, 5, 299, 137, 297, 96, 119, 250, 68] + 38: [301, 169, 331, 219, 85, 359, 184, 48, 246, 376, 305, 337, 273, 215, 13, 202, 350, 153, 140, 318, 46, 60, 185, 237, 6, 309, 381, 77, 286, 352, 72, 307, 165, 229, 260, 33, 2, 313, 61, 297, 164, 65, 303, 162, 346, 251, 325, 380, 266, 93, 292, 63, 0, 300, 225, 222, 193, 57, 216, 289, 76, 353, 23, 122, 294, 205, 240, 112, 22, 160, 52, 191, 203, 187, 123, 220, 244, 344, 338, 99, 73, 105, 264, 111, 31, 242, 152, 214, 11, 183, 91, 320, 218, 198, 101, 210, 158, 4, 94, 194, 92, 206, 199, 355, 69, 262, 256, 37, 148, 41, 235, 79, 195, 80, 374, 113, 326, 315, 83, 161, 378, 10, 118, 248, 217, 135, 319, 233, 324, 59, 45, 298, 342, 197, 181, 173, 172, 295, 186, 299, 18, 20, 370, 176, 221, 282, 66, 115, 19, 27, 257, 106, 75, 358, 349, 179, 151, 84, 15, 296, 261, 231, 247, 254, 335, 365, 110, 339, 269, 62, 223, 8, 379, 166, 97, 329, 239, 21, 211, 142, 276, 356, 103, 302, 382, 25, 35, 343, 232, 132, 133, 121, 280, 89, 50, 274, 293, 212, 54, 90, 67, 29, 58, 308, 141, 28, 272, 200, 371, 311, 168, 363, 175, 291, 243, 304, 124, 333, 345, 263, 39, 17, 108, 327, 271, 95, 144, 7, 143, 180, 174, 81, 104, 375, 102, 3, 259, 116, 196, 340, 201, 154, 245, 332, 130, 145, 234, 288, 227, 208, 323, 136, 5, 241, 40, 334, 347, 82, 55, 236, 163, 207, 249, 312, 284, 277, 178, 322, 32, 328, 53, 361, 107, 224, 88, 156, 228, 306, 78, 360, 120, 100, 290, 357, 314, 209, 330, 64, 372, 114, 348, 86, 171, 278, 267, 125, 275, 150, 87, 367, 265, 170, 131, 238, 43, 252, 71, 12, 157, 377, 336, 38, 51, 383, 369, 226, 177, 279, 268, 341, 146, 321, 255, 368, 117, 190, 139, 182, 74, 44, 283, 70, 24, 167, 127, 98, 30, 126, 287, 285, 49, 317, 119, 16, 147, 362, 68, 96, 109, 26, 258, 250, 129, 47, 351, 192, 364, 281, 155, 159, 366, 189, 213, 354, 230, 373, 253, 137, 14, 9, 36, 34, 188, 134, 149, 310, 56, 204, 138, 316, 42, 128, 1, 270] + 39: [282, 267, 296, 250, 49, 202, 232, 138, 358, 364, 373, 88, 164, 377, 192, 303, 175, 123, 125, 308, 184, 89, 257, 44, 310, 156, 2, 362, 208, 59, 274, 340, 126, 371, 161, 61, 320, 363, 183, 27, 346, 211, 37, 67, 32, 90, 304, 209, 247, 348, 242, 329, 121, 77, 273, 140, 263, 353, 5, 33, 95, 4, 240, 91, 287, 78, 0, 219, 294, 21, 143, 361, 66, 82, 325, 145, 40, 163, 87, 127, 122, 69, 150, 301, 72, 97, 16, 181, 343, 55, 108, 284, 147, 237, 261, 149, 104, 234, 360, 86, 71, 76, 23, 191, 54, 152, 144, 213, 311, 275, 30, 214, 245, 159, 142, 64, 158, 124, 148, 52, 372, 280, 25, 272, 333, 369, 81, 248, 299, 367, 317, 63, 106, 133, 79, 265, 58, 111, 266, 233, 255, 332, 11, 165, 236, 96, 153, 271, 26, 300, 298, 383, 225, 74, 135, 162, 102, 173, 243, 60, 171, 117, 18, 252, 256, 57, 356, 22, 56, 99, 1, 155, 297, 141, 51, 84, 239, 137, 188, 226, 355, 193, 254, 120, 216, 107, 305, 359, 351, 262, 115, 9, 315, 176, 368, 119, 270, 17, 365, 375, 48, 101, 50, 307, 182, 203, 195, 327, 335, 168, 347, 238, 339, 269, 154, 198, 258, 100, 223, 279, 80, 98, 3, 146, 259, 112, 190, 281, 249, 212, 285, 309, 169, 109, 222, 20, 92, 349, 318, 321, 286, 43, 116, 13, 382, 324, 178, 7, 42, 251, 378, 322, 179, 94, 354, 130, 207, 196, 19, 83, 253, 350, 12, 186, 65, 85, 35, 151, 174, 172, 15, 381, 8, 323, 131, 62, 134, 167, 357, 220, 306, 288, 41, 206, 39, 210, 316, 201, 312, 379, 180, 215, 328, 295, 341, 235, 118, 336, 331, 366, 75, 228, 128, 221, 166, 283, 337, 246, 229, 326, 73, 344, 38, 313, 110, 139, 187, 374, 24, 290, 352, 28, 260, 177, 302, 289, 241, 68, 114, 217, 189, 160, 197, 370, 105, 380, 93, 314, 264, 218, 292, 277, 319, 200, 170, 31, 47, 276, 14, 293, 230, 338, 376, 291, 46, 157, 342, 129, 224, 29, 132, 36, 268, 113, 244, 204, 330, 227, 53, 278, 34, 199, 345, 45, 185, 6, 194, 70, 103, 231, 10, 136, 205, 334] + 40: [31, 316, 244, 155, 206, 37, 345, 246, 125, 135, 242, 285, 4, 298, 353, 170, 343, 105, 123, 149, 194, 214, 248, 300, 319, 213, 26, 158, 228, 362, 274, 289, 382, 56, 263, 151, 324, 237, 314, 78, 121, 266, 34, 134, 39, 24, 240, 376, 15, 181, 107, 88, 59, 363, 162, 322, 196, 309, 364, 25, 371, 281, 73, 6, 84, 268, 163, 22, 267, 191, 42, 92, 68, 260, 62, 161, 208, 91, 129, 277, 334, 302, 53, 299, 192, 305, 350, 261, 115, 184, 117, 106, 234, 223, 83, 249, 175, 280, 101, 156, 251, 17, 143, 63, 30, 308, 137, 57, 377, 356, 282, 85, 238, 276, 310, 77, 111, 313, 220, 72, 378, 340, 98, 279, 74, 28, 89, 80, 11, 199, 46, 82, 94, 287, 48, 47, 36, 256, 49, 40, 114, 176, 86, 312, 317, 357, 367, 202, 254, 3, 9, 168, 145, 200, 227, 189, 103, 327, 171, 60, 291, 288, 50, 113, 352, 381, 112, 331, 207, 379, 75, 215, 372, 58, 347, 341, 61, 148, 187, 365, 355, 339, 195, 301, 375, 361, 383, 257, 231, 96, 172, 225, 183, 132, 373, 297, 8, 100, 296, 10, 54, 216, 259, 150, 2, 29, 304, 18, 76, 348, 13, 66, 173, 315, 272, 335, 250, 226, 204, 52, 64, 224, 275, 326, 27, 236, 229, 43, 7, 120, 273, 293, 241, 0, 102, 35, 217, 380, 320, 146, 349, 33, 67, 337, 369, 258, 1, 336, 278, 38, 269, 138, 165, 338, 19, 131, 201, 23, 124, 218, 104, 152, 20, 126, 360, 346, 359, 265, 81, 235, 210, 271, 211, 205, 307, 95, 290, 333, 329, 139, 118, 14, 140, 203, 180, 87, 245, 344, 142, 179, 55, 283, 332, 198, 159, 209, 185, 370, 44, 41, 110, 93, 323, 222, 147, 141, 368, 230, 325, 109, 252, 5, 328, 108, 70, 247, 164, 97, 303, 136, 193, 12, 262, 65, 130, 318, 330, 233, 270, 212, 374, 295, 119, 342, 127, 197, 71, 21, 286, 160, 153, 351, 284, 219, 166, 188, 239, 178, 294, 264, 243, 51, 116, 186, 16, 174, 154, 45, 255, 144, 358, 190, 169, 128, 232, 90, 311, 321, 253, 221, 79, 99, 306, 167, 177, 366, 122, 133, 32, 182, 354, 157, 292, 69] + 41: [49, 11, 365, 172, 95, 326, 25, 174, 56, 322, 224, 107, 18, 317, 188, 292, 180, 28, 209, 157, 189, 118, 263, 357, 378, 196, 273, 92, 279, 306, 222, 336, 54, 235, 112, 268, 43, 128, 232, 77, 248, 98, 265, 145, 319, 135, 22, 309, 199, 87, 150, 76, 360, 253, 53, 243, 102, 55, 115, 328, 290, 5, 371, 272, 33, 165, 182, 329, 201, 346, 16, 40, 170, 185, 255, 285, 21, 198, 4, 126, 116, 254, 117, 266, 295, 246, 221, 41, 381, 48, 149, 375, 240, 88, 203, 178, 60, 305, 249, 124, 24, 108, 351, 353, 64, 256, 167, 288, 186, 19, 173, 154, 352, 332, 147, 14, 267, 361, 58, 247, 350, 122, 212, 97, 314, 337, 3, 23, 301, 276, 289, 380, 338, 81, 244, 300, 1, 141, 137, 29, 331, 362, 356, 296, 374, 205, 90, 293, 69, 44, 35, 194, 345, 93, 325, 8, 181, 6, 61, 359, 0, 72, 148, 370, 110, 162, 164, 31, 321, 215, 307, 223, 111, 206, 104, 202, 349, 241, 237, 144, 51, 160, 192, 303, 287, 68, 158, 67, 121, 312, 230, 120, 269, 119, 27, 50, 364, 311, 80, 42, 32, 282, 294, 250, 131, 261, 146, 15, 39, 259, 175, 373, 323, 211, 333, 152, 298, 159, 251, 327, 358, 347, 318, 36, 217, 379, 156, 138, 299, 94, 74, 99, 78, 297, 195, 17, 20, 377, 281, 216, 283, 275, 376, 368, 79, 262, 45, 219, 26, 225, 83, 133, 63, 277, 354, 37, 100, 320, 155, 238, 343, 270, 228, 233, 114, 129, 71, 200, 34, 12, 9, 239, 134, 70, 340, 330, 363, 130, 89, 214, 179, 13, 113, 171, 47, 106, 382, 132, 142, 366, 166, 168, 187, 334, 127, 140, 308, 125, 271, 242, 176, 208, 220, 123, 191, 207, 310, 231, 62, 341, 169, 52, 355, 197, 103, 84, 7, 280, 348, 264, 218, 226, 258, 344, 151, 91, 213, 342, 143, 234, 73, 204, 2, 177, 291, 278, 184, 369, 367, 57, 136, 302, 383, 286, 252, 46, 260, 210, 66, 153, 315, 75, 236, 38, 257, 245, 139, 372, 193, 227, 190, 30, 335, 86, 96, 161, 82, 324, 163, 313, 229, 339, 101, 284, 10, 59, 65, 105, 316, 274, 183, 109, 304, 85] + 42: [123, 137, 238, 310, 332, 364, 229, 104, 207, 115, 182, 24, 289, 27, 242, 294, 369, 376, 25, 71, 95, 254, 324, 339, 152, 247, 220, 342, 127, 274, 187, 147, 92, 160, 18, 311, 183, 196, 53, 343, 321, 75, 304, 249, 44, 382, 126, 121, 117, 211, 162, 263, 277, 30, 287, 201, 234, 118, 19, 64, 283, 5, 378, 293, 368, 186, 330, 11, 79, 0, 352, 193, 31, 295, 72, 374, 255, 373, 248, 17, 105, 322, 65, 340, 372, 315, 336, 355, 26, 145, 185, 367, 12, 83, 13, 46, 359, 20, 184, 179, 377, 319, 131, 140, 167, 181, 33, 192, 301, 89, 97, 204, 346, 284, 360, 136, 134, 110, 142, 40, 217, 375, 305, 41, 129, 21, 370, 135, 165, 335, 52, 297, 8, 323, 299, 251, 171, 318, 363, 288, 96, 70, 317, 252, 94, 154, 303, 150, 55, 327, 275, 259, 230, 308, 366, 380, 132, 246, 226, 291, 233, 206, 10, 334, 133, 57, 271, 273, 266, 210, 106, 264, 108, 189, 328, 224, 87, 36, 316, 381, 146, 9, 232, 16, 112, 290, 73, 198, 88, 100, 43, 15, 268, 188, 357, 63, 168, 208, 191, 354, 344, 326, 38, 39, 245, 219, 49, 258, 68, 56, 90, 285, 383, 35, 107, 86, 345, 2, 119, 279, 262, 203, 351, 320, 286, 356, 314, 227, 80, 231, 157, 58, 269, 84, 82, 128, 302, 353, 14, 91, 176, 37, 228, 151, 195, 59, 379, 221, 236, 216, 143, 76, 241, 62, 130, 313, 125, 194, 250, 292, 347, 190, 54, 240, 172, 138, 148, 282, 349, 48, 235, 178, 239, 338, 280, 166, 60, 307, 139, 205, 278, 175, 199, 296, 22, 153, 78, 365, 155, 333, 114, 215, 243, 209, 122, 141, 28, 161, 77, 149, 32, 180, 260, 7, 156, 261, 267, 124, 309, 276, 81, 66, 218, 158, 42, 212, 200, 281, 265, 4, 169, 103, 164, 257, 113, 371, 244, 298, 170, 120, 98, 3, 144, 47, 331, 74, 93, 256, 1, 350, 222, 6, 348, 341, 214, 173, 202, 223, 101, 225, 109, 34, 300, 213, 197, 23, 174, 253, 177, 163, 325, 111, 50, 116, 337, 306, 358, 29, 67, 329, 45, 362, 51, 237, 99, 61, 270, 272, 102, 312, 85, 69, 159, 361] + 43: [168, 332, 39, 57, 301, 338, 255, 282, 123, 149, 241, 47, 183, 125, 207, 122, 173, 355, 162, 106, 15, 104, 245, 348, 225, 344, 35, 94, 186, 318, 121, 73, 48, 299, 157, 145, 253, 101, 295, 8, 247, 211, 341, 172, 133, 176, 44, 354, 375, 27, 197, 366, 153, 227, 280, 32, 327, 271, 130, 107, 165, 167, 18, 76, 126, 109, 53, 199, 333, 212, 95, 3, 66, 369, 129, 100, 81, 203, 136, 85, 222, 140, 160, 237, 60, 266, 80, 170, 87, 132, 119, 69, 230, 58, 96, 214, 98, 164, 371, 252, 377, 111, 108, 272, 380, 204, 79, 28, 293, 143, 220, 329, 294, 233, 110, 357, 228, 362, 166, 195, 315, 23, 142, 239, 93, 29, 198, 150, 88, 286, 365, 334, 38, 210, 128, 116, 356, 268, 187, 179, 249, 383, 184, 169, 221, 351, 218, 216, 363, 342, 274, 368, 90, 277, 36, 243, 102, 56, 256, 288, 65, 19, 209, 248, 317, 139, 91, 192, 105, 43, 308, 349, 314, 264, 147, 89, 378, 135, 26, 67, 50, 217, 263, 292, 259, 287, 240, 258, 21, 236, 25, 206, 171, 298, 278, 59, 22, 323, 307, 374, 154, 306, 5, 63, 49, 70, 343, 370, 117, 163, 320, 276, 7, 270, 231, 361, 156, 297, 177, 37, 326, 322, 269, 250, 238, 86, 84, 155, 113, 45, 213, 11, 72, 358, 316, 260, 141, 281, 244, 182, 347, 30, 68, 112, 291, 223, 313, 2, 275, 339, 61, 10, 185, 251, 325, 226, 267, 381, 336, 285, 328, 340, 302, 376, 92, 75, 161, 20, 235, 127, 305, 77, 346, 219, 34, 345, 99, 120, 146, 159, 279, 118, 265, 51, 0, 42, 289, 331, 372, 190, 6, 254, 152, 138, 1, 4, 52, 234, 261, 273, 242, 46, 309, 31, 64, 134, 290, 312, 62, 208, 319, 373, 335, 283, 181, 97, 337, 55, 224, 191, 246, 83, 257, 196, 229, 180, 74, 304, 144, 158, 300, 311, 137, 303, 350, 382, 17, 14, 82, 13, 103, 178, 78, 321, 205, 200, 296, 12, 148, 284, 9, 215, 353, 193, 330, 114, 124, 352, 310, 54, 359, 324, 262, 364, 131, 360, 202, 379, 232, 115, 367, 194, 24, 16, 174, 175, 71, 151, 188, 201, 41, 40, 33, 189] + 44: [213, 257, 229, 86, 329, 337, 91, 296, 315, 380, 134, 126, 334, 309, 179, 357, 114, 375, 99, 330, 111, 308, 322, 374, 77, 25, 267, 311, 383, 142, 360, 138, 290, 70, 160, 381, 136, 40, 377, 237, 18, 7, 1, 194, 302, 31, 16, 116, 301, 258, 188, 135, 166, 141, 225, 207, 254, 100, 131, 221, 157, 185, 231, 156, 250, 51, 35, 373, 363, 356, 208, 332, 191, 17, 227, 140, 236, 249, 275, 89, 224, 201, 132, 261, 292, 74, 8, 3, 235, 232, 216, 115, 298, 293, 88, 303, 199, 0, 215, 331, 98, 182, 291, 117, 61, 55, 130, 120, 107, 90, 59, 169, 193, 367, 45, 27, 103, 127, 310, 12, 10, 167, 190, 328, 313, 118, 144, 94, 350, 219, 248, 269, 354, 108, 105, 211, 319, 4, 255, 251, 283, 259, 76, 112, 317, 64, 110, 279, 34, 306, 102, 14, 266, 39, 352, 359, 26, 15, 149, 341, 9, 53, 273, 119, 246, 123, 353, 209, 226, 97, 351, 368, 288, 294, 228, 340, 192, 150, 164, 177, 162, 165, 243, 20, 234, 137, 80, 153, 176, 152, 75, 339, 348, 238, 155, 29, 93, 38, 33, 121, 85, 173, 366, 369, 46, 370, 347, 23, 263, 183, 200, 233, 260, 335, 181, 13, 362, 168, 60, 277, 146, 242, 125, 355, 24, 349, 286, 54, 220, 278, 318, 32, 241, 62, 312, 151, 247, 139, 58, 161, 274, 346, 180, 252, 187, 289, 217, 265, 299, 287, 66, 174, 41, 92, 214, 104, 57, 204, 2, 239, 268, 361, 371, 189, 172, 170, 382, 68, 320, 264, 304, 158, 175, 163, 338, 96, 342, 282, 323, 81, 72, 256, 42, 195, 69, 37, 148, 106, 171, 145, 271, 43, 143, 63, 48, 276, 184, 364, 79, 87, 128, 83, 78, 336, 109, 314, 245, 5, 372, 73, 133, 129, 222, 21, 300, 82, 30, 333, 365, 324, 244, 122, 71, 67, 36, 147, 270, 218, 203, 321, 253, 22, 28, 297, 197, 206, 379, 284, 178, 376, 11, 212, 358, 343, 6, 154, 285, 202, 327, 196, 205, 305, 230, 49, 124, 50, 240, 47, 316, 295, 101, 223, 272, 84, 325, 280, 198, 95, 262, 186, 56, 378, 44, 344, 159, 19, 113, 345, 65, 326, 210, 307, 52, 281] + 45: [92, 378, 297, 39, 229, 308, 213, 106, 303, 58, 126, 282, 288, 314, 343, 225, 89, 270, 6, 325, 300, 174, 221, 347, 140, 233, 120, 47, 183, 228, 104, 372, 357, 109, 1, 19, 145, 38, 241, 289, 309, 238, 153, 305, 99, 66, 83, 100, 178, 318, 197, 85, 114, 351, 45, 338, 7, 44, 236, 340, 116, 358, 379, 271, 48, 377, 90, 97, 265, 75, 49, 72, 10, 301, 182, 103, 348, 331, 196, 88, 210, 157, 365, 257, 35, 205, 273, 169, 363, 113, 93, 186, 226, 148, 69, 17, 227, 173, 199, 146, 124, 248, 55, 74, 251, 86, 260, 222, 20, 170, 322, 374, 79, 268, 337, 280, 193, 216, 185, 138, 154, 191, 346, 160, 165, 258, 310, 30, 112, 108, 179, 376, 43, 46, 294, 127, 246, 190, 202, 82, 143, 366, 181, 14, 23, 324, 25, 369, 163, 275, 307, 277, 327, 135, 175, 2, 34, 291, 283, 381, 105, 364, 111, 62, 200, 77, 237, 192, 121, 383, 139, 262, 212, 234, 162, 245, 306, 80, 208, 123, 330, 319, 264, 231, 220, 371, 214, 102, 352, 172, 11, 40, 161, 81, 284, 171, 131, 224, 168, 16, 84, 281, 217, 22, 50, 252, 5, 342, 166, 78, 73, 56, 159, 278, 299, 64, 293, 95, 317, 176, 61, 128, 118, 51, 70, 267, 67, 96, 133, 296, 98, 255, 341, 147, 91, 31, 232, 13, 60, 117, 42, 164, 144, 247, 180, 209, 223, 122, 313, 242, 292, 63, 218, 87, 9, 263, 119, 312, 321, 125, 101, 287, 130, 355, 334, 329, 316, 356, 269, 52, 198, 115, 54, 156, 94, 302, 65, 304, 29, 151, 0, 368, 243, 382, 353, 259, 149, 244, 201, 276, 134, 375, 354, 107, 254, 155, 250, 345, 279, 24, 207, 59, 37, 211, 15, 203, 323, 132, 194, 335, 195, 137, 8, 373, 188, 339, 298, 272, 359, 36, 12, 41, 26, 4, 344, 290, 150, 57, 142, 3, 71, 152, 167, 53, 240, 336, 27, 367, 110, 141, 21, 349, 187, 326, 328, 249, 32, 253, 370, 256, 320, 177, 285, 362, 332, 189, 295, 28, 184, 266, 33, 350, 219, 274, 286, 380, 239, 158, 68, 360, 315, 261, 76, 230, 333, 129, 204, 215, 136, 18, 206, 311, 235, 361] + 46: [142, 170, 284, 38, 303, 102, 155, 260, 316, 50, 321, 268, 289, 313, 69, 190, 86, 163, 330, 134, 251, 225, 82, 122, 183, 210, 6, 254, 113, 350, 22, 372, 166, 13, 17, 193, 35, 222, 277, 364, 257, 337, 11, 72, 276, 70, 379, 356, 352, 188, 324, 68, 338, 39, 143, 121, 315, 128, 172, 57, 360, 255, 322, 145, 184, 15, 21, 126, 47, 349, 175, 213, 311, 152, 96, 344, 204, 345, 266, 117, 287, 192, 304, 103, 377, 160, 24, 348, 331, 262, 52, 75, 236, 54, 0, 5, 133, 335, 25, 209, 60, 55, 351, 131, 232, 137, 41, 198, 32, 308, 173, 317, 91, 370, 43, 358, 300, 123, 84, 27, 10, 140, 16, 104, 40, 165, 49, 280, 245, 156, 95, 272, 111, 97, 119, 71, 291, 244, 371, 31, 87, 116, 200, 36, 247, 66, 249, 191, 124, 76, 211, 108, 326, 369, 176, 157, 59, 92, 89, 56, 90, 93, 135, 229, 288, 179, 357, 88, 34, 319, 46, 202, 194, 199, 282, 130, 168, 180, 362, 216, 81, 329, 346, 293, 261, 278, 26, 281, 363, 248, 14, 147, 64, 106, 368, 246, 312, 203, 215, 195, 334, 259, 23, 292, 383, 78, 297, 9, 177, 309, 67, 94, 381, 185, 100, 302, 139, 354, 327, 239, 301, 146, 79, 33, 80, 4, 298, 235, 158, 263, 223, 20, 342, 339, 138, 77, 73, 306, 270, 220, 256, 367, 240, 51, 136, 44, 219, 376, 186, 109, 63, 237, 125, 144, 299, 159, 217, 361, 65, 127, 105, 212, 28, 231, 58, 230, 224, 274, 214, 374, 85, 8, 218, 197, 53, 273, 101, 373, 382, 74, 241, 98, 375, 341, 107, 279, 167, 252, 286, 149, 314, 238, 12, 242, 228, 62, 99, 42, 164, 267, 307, 141, 318, 153, 333, 171, 347, 366, 196, 296, 182, 325, 380, 207, 19, 221, 187, 18, 3, 120, 227, 37, 129, 340, 45, 328, 161, 265, 359, 234, 150, 258, 269, 29, 151, 61, 253, 343, 233, 332, 378, 114, 30, 115, 132, 295, 294, 250, 336, 290, 169, 7, 264, 305, 208, 275, 1, 174, 148, 323, 355, 110, 201, 154, 271, 2, 320, 83, 353, 118, 162, 205, 226, 206, 189, 243, 283, 285, 112, 310, 48, 181, 178, 365] + 47: [195, 198, 188, 159, 84, 124, 33, 12, 340, 117, 15, 303, 37, 310, 120, 308, 16, 30, 299, 8, 13, 191, 56, 288, 3, 279, 305, 274, 131, 122, 69, 223, 140, 322, 173, 183, 335, 60, 2, 10, 55, 304, 222, 82, 167, 214, 373, 326, 343, 375, 286, 362, 59, 114, 95, 166, 172, 0, 4, 23, 139, 281, 115, 200, 19, 275, 339, 317, 99, 283, 236, 209, 54, 35, 314, 344, 40, 251, 208, 89, 284, 184, 180, 233, 356, 273, 341, 329, 48, 189, 5, 277, 116, 232, 311, 369, 44, 192, 193, 215, 175, 327, 313, 377, 41, 199, 237, 164, 336, 225, 111, 20, 46, 57, 374, 361, 142, 221, 182, 246, 352, 143, 61, 316, 129, 342, 98, 42, 103, 255, 45, 106, 378, 276, 258, 260, 102, 72, 380, 372, 11, 125, 318, 293, 24, 179, 109, 353, 21, 229, 269, 104, 263, 364, 39, 354, 18, 259, 31, 186, 149, 216, 92, 38, 64, 165, 7, 62, 312, 187, 1, 368, 267, 363, 176, 67, 319, 50, 148, 53, 321, 163, 94, 248, 150, 266, 108, 178, 194, 206, 156, 146, 87, 347, 119, 14, 79, 346, 212, 26, 151, 370, 205, 52, 366, 121, 328, 185, 253, 152, 358, 86, 145, 217, 83, 360, 181, 271, 43, 345, 127, 110, 349, 330, 262, 285, 296, 51, 252, 70, 71, 49, 32, 242, 325, 174, 220, 244, 256, 113, 154, 75, 371, 309, 196, 118, 218, 351, 224, 323, 73, 355, 96, 243, 211, 107, 157, 160, 177, 207, 202, 264, 295, 128, 162, 137, 291, 333, 257, 287, 235, 226, 68, 298, 65, 25, 112, 80, 289, 278, 297, 365, 379, 290, 47, 247, 254, 381, 22, 134, 357, 227, 36, 238, 63, 265, 135, 249, 197, 338, 210, 348, 126, 332, 97, 334, 383, 350, 76, 88, 66, 261, 158, 300, 171, 34, 294, 376, 367, 155, 93, 324, 77, 101, 315, 91, 28, 292, 144, 190, 245, 74, 85, 270, 241, 272, 29, 359, 136, 280, 320, 168, 153, 230, 219, 228, 203, 138, 234, 141, 213, 306, 132, 9, 282, 58, 100, 268, 6, 169, 382, 307, 331, 170, 90, 240, 123, 250, 301, 231, 204, 17, 27, 201, 147, 337, 130, 81, 302, 133, 105, 78, 161, 239] + 48: [17, 26, 226, 83, 189, 254, 229, 322, 120, 329, 168, 22, 119, 84, 144, 382, 93, 3, 309, 338, 183, 220, 129, 227, 271, 249, 44, 133, 223, 186, 290, 179, 257, 170, 4, 339, 36, 198, 169, 108, 310, 163, 201, 225, 68, 126, 30, 308, 140, 297, 109, 295, 6, 264, 252, 161, 285, 255, 172, 57, 281, 349, 151, 187, 173, 213, 61, 282, 125, 265, 136, 314, 328, 159, 358, 90, 147, 372, 232, 300, 190, 324, 218, 360, 158, 211, 292, 243, 92, 29, 350, 362, 248, 303, 334, 250, 132, 237, 150, 2, 287, 359, 380, 222, 192, 45, 321, 42, 113, 204, 367, 206, 272, 135, 37, 105, 86, 153, 355, 10, 267, 263, 301, 182, 369, 48, 134, 210, 299, 123, 212, 91, 241, 97, 18, 55, 261, 214, 56, 356, 331, 260, 131, 348, 283, 15, 279, 203, 207, 208, 115, 305, 293, 152, 242, 315, 307, 289, 43, 53, 71, 286, 294, 278, 345, 296, 199, 167, 76, 62, 145, 74, 104, 38, 363, 205, 193, 247, 89, 357, 80, 351, 12, 127, 256, 32, 87, 335, 377, 46, 364, 200, 266, 78, 117, 121, 166, 327, 184, 354, 175, 233, 162, 230, 317, 14, 269, 16, 82, 217, 143, 110, 148, 251, 306, 196, 128, 85, 65, 343, 28, 235, 181, 155, 156, 88, 73, 275, 379, 138, 280, 40, 178, 366, 323, 244, 149, 332, 103, 209, 102, 381, 77, 236, 157, 240, 142, 371, 7, 291, 31, 347, 180, 70, 352, 98, 216, 95, 165, 376, 60, 106, 337, 245, 330, 141, 270, 116, 21, 375, 325, 368, 246, 370, 341, 79, 231, 96, 11, 94, 274, 326, 27, 228, 154, 262, 176, 304, 273, 101, 20, 99, 268, 139, 316, 319, 312, 174, 160, 24, 64, 146, 253, 107, 137, 67, 313, 188, 171, 23, 383, 114, 239, 353, 9, 19, 164, 5, 185, 320, 130, 75, 0, 221, 100, 49, 63, 346, 39, 47, 277, 66, 191, 54, 122, 258, 276, 284, 51, 234, 33, 344, 34, 378, 25, 50, 118, 111, 202, 302, 81, 35, 259, 69, 373, 361, 8, 124, 197, 336, 52, 311, 288, 215, 58, 177, 342, 41, 219, 112, 340, 318, 13, 72, 374, 59, 195, 194, 224, 238, 298, 365, 333, 1] + 49: [302, 255, 5, 336, 245, 170, 163, 295, 98, 239, 71, 124, 154, 232, 167, 171, 217, 68, 375, 337, 356, 64, 123, 191, 321, 278, 148, 76, 306, 333, 48, 137, 138, 189, 273, 43, 62, 117, 118, 228, 354, 13, 147, 350, 263, 188, 38, 317, 297, 83, 181, 110, 177, 246, 45, 164, 254, 16, 365, 222, 82, 36, 345, 69, 49, 283, 145, 27, 42, 18, 258, 26, 253, 37, 158, 261, 262, 286, 331, 8, 205, 17, 140, 323, 301, 238, 156, 211, 355, 361, 364, 166, 96, 287, 288, 219, 126, 149, 150, 270, 259, 260, 187, 131, 58, 28, 56, 224, 225, 327, 351, 47, 130, 113, 256, 51, 229, 152, 207, 373, 374, 197, 234, 25, 276, 133, 74, 314, 122, 80, 368, 34, 77, 308, 244, 320, 31, 192, 326, 357, 159, 334, 369, 50, 79, 184, 136, 241, 39, 218, 329, 100, 73, 1, 223, 127, 81, 33, 359, 72, 0, 285, 88, 104, 341, 281, 215, 24, 360, 233, 257, 78, 335, 199, 269, 95, 144, 54, 86, 112, 376, 330, 338, 378, 119, 206, 125, 316, 162, 231, 11, 349, 272, 210, 304, 353, 90, 91, 344, 298, 121, 310, 196, 290, 303, 371, 178, 226, 358, 84, 108, 315, 7, 21, 230, 92, 6, 251, 346, 293, 40, 289, 194, 264, 328, 340, 277, 29, 55, 173, 309, 53, 61, 282, 182, 343, 102, 266, 32, 129, 268, 193, 319, 116, 208, 243, 132, 109, 235, 382, 318, 128, 139, 179, 227, 85, 12, 284, 348, 111, 372, 249, 141, 347, 75, 242, 294, 213, 201, 107, 220, 94, 195, 221, 23, 89, 20, 67, 103, 114, 204, 143, 22, 57, 165, 161, 252, 135, 155, 9, 15, 115, 275, 60, 120, 183, 250, 362, 198, 214, 216, 10, 134, 280, 46, 342, 185, 168, 363, 236, 247, 370, 305, 271, 65, 105, 180, 325, 312, 332, 169, 174, 311, 41, 248, 267, 142, 339, 2, 151, 324, 274, 367, 99, 291, 300, 19, 202, 59, 212, 44, 380, 381, 30, 366, 63, 70, 160, 146, 352, 172, 14, 106, 101, 52, 377, 237, 265, 203, 3, 157, 313, 209, 66, 153, 292, 186, 175, 97, 279, 240, 4, 296, 35, 307, 176, 200, 190, 299, 93, 87, 379, 383, 322] + 50: [86, 14, 146, 327, 22, 253, 196, 313, 102, 342, 151, 2, 278, 362, 153, 349, 318, 162, 13, 157, 161, 118, 111, 345, 268, 67, 110, 296, 295, 224, 122, 48, 348, 169, 234, 322, 172, 5, 124, 7, 248, 147, 77, 72, 261, 364, 182, 365, 29, 128, 38, 287, 199, 79, 16, 164, 71, 240, 197, 382, 91, 108, 37, 177, 63, 141, 11, 324, 265, 266, 44, 84, 100, 231, 333, 125, 45, 331, 213, 274, 117, 116, 252, 81, 107, 99, 127, 204, 12, 354, 171, 61, 173, 309, 254, 136, 316, 129, 229, 366, 220, 51, 155, 96, 19, 263, 174, 371, 242, 113, 135, 123, 106, 148, 65, 101, 216, 31, 368, 230, 109, 3, 97, 373, 192, 314, 159, 298, 75, 154, 250, 139, 255, 143, 236, 170, 351, 293, 98, 30, 207, 167, 227, 283, 49, 1, 140, 273, 62, 88, 270, 133, 380, 379, 180, 310, 145, 356, 178, 288, 200, 212, 87, 346, 245, 104, 59, 332, 181, 18, 137, 35, 329, 119, 249, 187, 195, 359, 247, 92, 20, 357, 191, 281, 156, 90, 36, 60, 8, 95, 66, 23, 166, 26, 311, 218, 338, 114, 130, 24, 149, 183, 189, 303, 64, 205, 202, 292, 46, 68, 179, 83, 103, 158, 131, 222, 376, 350, 299, 367, 304, 256, 271, 6, 228, 168, 43, 326, 152, 105, 297, 4, 305, 237, 286, 176, 190, 279, 307, 241, 32, 27, 235, 370, 280, 334, 343, 275, 186, 203, 276, 267, 126, 185, 243, 339, 308, 330, 291, 21, 201, 221, 260, 257, 315, 56, 226, 284, 9, 358, 15, 363, 306, 134, 206, 378, 163, 340, 39, 337, 238, 383, 82, 40, 328, 54, 317, 374, 210, 214, 80, 262, 312, 0, 335, 132, 58, 375, 208, 94, 160, 223, 347, 344, 320, 89, 323, 76, 246, 225, 302, 264, 188, 321, 25, 233, 377, 193, 184, 301, 239, 290, 53, 73, 277, 211, 369, 42, 34, 150, 112, 209, 360, 120, 52, 50, 215, 325, 33, 361, 175, 217, 121, 353, 165, 194, 319, 74, 294, 28, 300, 259, 198, 355, 285, 381, 70, 142, 336, 55, 69, 232, 57, 352, 282, 341, 272, 41, 244, 219, 269, 85, 78, 251, 17, 289, 372, 258, 138, 10, 47, 144, 115, 93] + 51: [24, 113, 358, 95, 330, 346, 35, 161, 81, 42, 309, 50, 324, 258, 27, 242, 238, 19, 39, 77, 227, 377, 156, 202, 186, 123, 67, 203, 188, 11, 259, 281, 359, 366, 248, 141, 289, 115, 51, 93, 159, 66, 96, 367, 293, 21, 173, 348, 372, 106, 284, 211, 124, 177, 92, 292, 56, 360, 321, 368, 317, 84, 194, 298, 197, 148, 230, 70, 355, 231, 249, 261, 102, 31, 193, 300, 260, 129, 269, 143, 382, 97, 18, 204, 361, 157, 313, 191, 307, 316, 149, 215, 120, 71, 131, 226, 137, 108, 99, 176, 87, 105, 335, 299, 23, 268, 351, 34, 75, 364, 312, 89, 319, 109, 15, 4, 128, 169, 256, 255, 373, 322, 111, 356, 60, 301, 49, 247, 47, 237, 181, 59, 236, 306, 349, 379, 363, 336, 240, 158, 277, 352, 150, 327, 375, 286, 16, 64, 140, 153, 168, 216, 303, 308, 219, 91, 195, 320, 334, 315, 273, 353, 57, 354, 132, 175, 244, 291, 221, 163, 357, 136, 343, 282, 246, 20, 275, 254, 380, 252, 362, 205, 369, 328, 145, 378, 25, 296, 217, 179, 347, 326, 241, 229, 6, 228, 32, 235, 26, 46, 331, 86, 280, 370, 10, 69, 383, 114, 30, 43, 125, 257, 305, 63, 29, 121, 37, 126, 73, 72, 65, 283, 314, 5, 183, 341, 192, 79, 41, 239, 297, 381, 340, 55, 1, 83, 376, 187, 38, 151, 210, 82, 371, 122, 152, 146, 253, 220, 212, 142, 190, 207, 48, 224, 100, 54, 276, 53, 167, 272, 154, 185, 36, 2, 110, 144, 189, 264, 62, 304, 263, 8, 285, 198, 222, 90, 3, 265, 196, 223, 80, 112, 184, 14, 165, 318, 117, 138, 200, 337, 178, 182, 40, 250, 103, 294, 344, 345, 279, 251, 33, 311, 365, 45, 333, 58, 68, 9, 174, 302, 225, 139, 323, 342, 206, 52, 270, 233, 130, 118, 245, 172, 209, 310, 164, 234, 213, 338, 288, 232, 218, 88, 199, 274, 22, 76, 208, 17, 101, 180, 104, 325, 134, 13, 85, 107, 329, 119, 166, 171, 332, 170, 78, 61, 116, 28, 0, 133, 267, 290, 12, 287, 98, 127, 135, 295, 243, 374, 271, 74, 162, 160, 44, 262, 94, 350, 278, 339, 214, 7, 201, 266, 147, 155] + 52: [381, 323, 298, 60, 269, 98, 379, 222, 167, 280, 53, 213, 308, 277, 20, 330, 380, 313, 304, 11, 12, 360, 212, 348, 99, 289, 82, 33, 168, 50, 325, 5, 315, 292, 361, 287, 326, 177, 148, 188, 141, 18, 135, 255, 317, 240, 176, 180, 334, 130, 34, 346, 44, 252, 133, 256, 223, 116, 115, 333, 93, 242, 238, 198, 143, 343, 138, 314, 373, 14, 370, 375, 102, 267, 271, 7, 260, 266, 291, 74, 178, 367, 113, 39, 45, 258, 224, 49, 303, 353, 232, 42, 159, 162, 106, 247, 107, 320, 290, 90, 357, 355, 70, 127, 83, 227, 243, 72, 66, 119, 58, 189, 318, 111, 226, 158, 9, 202, 218, 204, 157, 231, 246, 211, 41, 219, 87, 88, 117, 173, 369, 76, 324, 161, 89, 207, 47, 118, 43, 274, 79, 19, 199, 16, 71, 120, 151, 345, 172, 26, 64, 123, 262, 281, 6, 153, 91, 40, 365, 220, 150, 278, 126, 112, 185, 65, 264, 46, 104, 307, 31, 321, 78, 10, 54, 309, 105, 124, 134, 331, 55, 306, 294, 21, 179, 342, 299, 302, 187, 265, 272, 305, 165, 92, 182, 174, 13, 140, 354, 35, 166, 341, 301, 154, 374, 270, 201, 139, 51, 344, 81, 147, 36, 15, 295, 175, 97, 121, 63, 206, 183, 259, 349, 171, 285, 30, 146, 17, 69, 225, 2, 84, 230, 197, 114, 275, 145, 122, 376, 38, 283, 327, 371, 195, 184, 245, 210, 257, 339, 296, 364, 3, 279, 253, 352, 383, 152, 368, 322, 340, 276, 261, 338, 94, 103, 186, 217, 214, 77, 68, 192, 32, 37, 191, 359, 300, 319, 170, 235, 57, 194, 100, 356, 358, 363, 329, 366, 164, 137, 52, 216, 73, 193, 62, 80, 236, 155, 221, 160, 29, 351, 125, 27, 284, 59, 129, 28, 335, 132, 297, 0, 254, 336, 205, 61, 362, 169, 382, 128, 149, 312, 286, 23, 248, 22, 337, 268, 237, 251, 136, 101, 4, 181, 131, 142, 1, 239, 25, 372, 85, 163, 288, 196, 95, 108, 378, 233, 75, 377, 316, 350, 110, 244, 249, 156, 96, 229, 203, 144, 24, 209, 48, 332, 250, 263, 190, 56, 8, 228, 234, 215, 67, 328, 208, 109, 200, 282, 347, 310, 311, 293, 273, 241, 86] + 53: [110, 63, 184, 356, 165, 244, 314, 377, 265, 22, 252, 56, 90, 18, 140, 153, 167, 312, 316, 185, 84, 114, 210, 272, 240, 142, 42, 6, 144, 221, 253, 262, 172, 335, 315, 122, 369, 158, 79, 40, 254, 264, 100, 147, 366, 124, 204, 290, 73, 297, 365, 180, 194, 368, 171, 25, 183, 277, 47, 67, 120, 166, 236, 220, 195, 80, 199, 193, 34, 27, 274, 307, 38, 68, 292, 92, 309, 81, 20, 62, 383, 197, 149, 226, 233, 370, 333, 299, 258, 246, 82, 310, 267, 211, 71, 58, 16, 8, 280, 118, 202, 26, 284, 88, 141, 37, 176, 293, 196, 276, 278, 232, 45, 137, 83, 39, 93, 190, 187, 223, 317, 75, 173, 155, 156, 157, 125, 127, 12, 338, 235, 150, 143, 216, 321, 30, 182, 54, 367, 177, 298, 208, 192, 304, 13, 217, 164, 227, 209, 268, 107, 5, 200, 116, 49, 89, 359, 178, 169, 255, 123, 55, 380, 48, 85, 374, 212, 355, 350, 308, 320, 224, 145, 86, 31, 95, 139, 361, 60, 10, 146, 115, 174, 96, 294, 21, 201, 283, 106, 260, 161, 99, 23, 336, 129, 379, 302, 261, 103, 325, 94, 112, 373, 281, 328, 36, 57, 215, 218, 87, 154, 51, 342, 104, 288, 245, 334, 230, 247, 179, 228, 162, 130, 219, 256, 231, 300, 15, 282, 339, 375, 207, 59, 275, 7, 305, 113, 263, 148, 11, 287, 186, 238, 46, 279, 291, 134, 326, 69, 205, 78, 349, 357, 128, 189, 28, 364, 337, 53, 9, 17, 135, 41, 213, 296, 4, 270, 363, 170, 332, 271, 3, 168, 188, 313, 72, 102, 248, 371, 117, 133, 97, 131, 249, 64, 382, 91, 286, 109, 32, 66, 345, 266, 376, 121, 269, 70, 77, 324, 126, 175, 35, 65, 306, 242, 257, 108, 43, 74, 163, 61, 191, 354, 330, 203, 322, 360, 225, 119, 24, 381, 151, 378, 159, 353, 111, 105, 19, 251, 234, 352, 323, 301, 303, 181, 318, 311, 250, 243, 273, 132, 237, 285, 319, 346, 152, 239, 76, 44, 340, 327, 136, 138, 101, 289, 331, 348, 372, 343, 344, 347, 295, 222, 214, 160, 351, 50, 241, 329, 341, 52, 259, 362, 2, 206, 358, 198, 0, 14, 29, 1, 229, 98, 33] + 54: [363, 357, 190, 239, 55, 61, 326, 81, 351, 243, 6, 233, 362, 104, 252, 383, 127, 205, 148, 232, 320, 336, 361, 378, 370, 72, 345, 118, 332, 115, 198, 185, 125, 365, 73, 111, 364, 141, 194, 235, 331, 60, 303, 188, 342, 354, 0, 36, 294, 267, 20, 282, 119, 3, 47, 161, 5, 217, 186, 135, 255, 209, 18, 273, 213, 110, 79, 263, 210, 222, 349, 13, 339, 71, 150, 306, 221, 7, 121, 218, 335, 34, 381, 131, 144, 107, 146, 262, 371, 211, 236, 227, 307, 241, 78, 106, 155, 253, 82, 174, 330, 314, 10, 310, 50, 152, 140, 368, 171, 327, 33, 300, 259, 219, 268, 66, 51, 379, 212, 160, 164, 296, 257, 53, 187, 180, 101, 333, 84, 265, 103, 355, 278, 293, 96, 244, 139, 15, 367, 108, 374, 350, 165, 123, 284, 153, 216, 8, 290, 117, 317, 70, 215, 94, 272, 237, 100, 373, 62, 167, 316, 308, 224, 269, 248, 65, 134, 358, 283, 380, 287, 172, 341, 26, 181, 132, 196, 321, 92, 184, 246, 264, 304, 377, 238, 245, 59, 249, 56, 149, 256, 193, 42, 337, 189, 30, 176, 309, 58, 288, 113, 91, 24, 21, 1, 366, 179, 114, 23, 12, 74, 274, 199, 285, 37, 207, 258, 295, 83, 369, 93, 311, 329, 276, 129, 323, 382, 157, 318, 68, 137, 151, 43, 63, 145, 289, 270, 360, 126, 297, 11, 95, 338, 214, 44, 87, 86, 41, 158, 225, 54, 170, 177, 298, 201, 353, 275, 88, 168, 31, 46, 35, 169, 130, 356, 286, 147, 133, 208, 25, 16, 375, 197, 45, 159, 204, 67, 48, 313, 166, 9, 251, 178, 206, 102, 116, 77, 19, 223, 299, 39, 98, 120, 305, 372, 99, 64, 220, 182, 312, 315, 163, 22, 173, 128, 271, 247, 261, 203, 154, 138, 200, 231, 230, 90, 250, 254, 57, 75, 143, 277, 124, 109, 97, 302, 85, 122, 195, 191, 334, 14, 29, 226, 292, 291, 202, 240, 280, 328, 80, 281, 40, 27, 156, 32, 325, 142, 376, 183, 69, 136, 347, 175, 192, 234, 52, 343, 4, 2, 340, 348, 38, 319, 260, 324, 352, 228, 76, 322, 17, 279, 105, 49, 301, 242, 162, 266, 229, 346, 89, 359, 28, 112, 344] + 55: [154, 16, 247, 49, 99, 300, 337, 348, 231, 361, 122, 157, 149, 127, 365, 168, 58, 248, 155, 244, 351, 186, 215, 34, 174, 7, 227, 363, 134, 44, 26, 163, 375, 178, 140, 207, 87, 232, 373, 336, 275, 261, 175, 362, 148, 51, 69, 305, 56, 113, 234, 307, 304, 293, 331, 271, 112, 105, 265, 54, 318, 85, 31, 188, 356, 243, 66, 242, 141, 108, 167, 283, 202, 326, 4, 111, 382, 223, 320, 0, 156, 276, 378, 306, 332, 52, 101, 133, 367, 55, 104, 369, 169, 142, 229, 310, 88, 76, 309, 238, 78, 249, 272, 138, 17, 325, 236, 343, 257, 358, 314, 342, 15, 372, 50, 145, 181, 36, 345, 109, 81, 115, 151, 311, 73, 40, 226, 368, 189, 350, 217, 42, 263, 277, 302, 237, 253, 30, 107, 374, 282, 334, 24, 285, 98, 269, 355, 218, 211, 301, 12, 164, 259, 170, 380, 47, 119, 287, 214, 153, 97, 321, 315, 324, 330, 21, 100, 196, 20, 29, 256, 89, 192, 2, 179, 213, 172, 46, 377, 246, 10, 23, 313, 1, 219, 126, 125, 64, 364, 195, 292, 144, 11, 130, 235, 357, 267, 200, 71, 216, 344, 91, 371, 187, 241, 171, 296, 103, 37, 297, 354, 13, 92, 110, 379, 250, 146, 180, 270, 62, 328, 291, 176, 366, 206, 284, 251, 79, 19, 346, 264, 82, 268, 319, 150, 68, 191, 266, 9, 45, 308, 323, 43, 370, 121, 70, 333, 86, 233, 352, 27, 240, 184, 128, 63, 198, 359, 35, 120, 139, 295, 32, 158, 204, 340, 74, 327, 182, 41, 132, 279, 349, 5, 102, 224, 360, 322, 255, 117, 212, 225, 93, 137, 162, 280, 83, 59, 222, 116, 347, 28, 39, 303, 106, 3, 290, 197, 14, 289, 286, 185, 254, 22, 61, 335, 201, 338, 183, 77, 288, 25, 205, 80, 48, 152, 165, 353, 199, 8, 312, 383, 190, 210, 53, 228, 84, 294, 160, 143, 118, 161, 177, 298, 57, 230, 317, 381, 90, 159, 252, 273, 65, 60, 316, 124, 274, 329, 135, 194, 278, 33, 258, 173, 38, 129, 166, 262, 75, 131, 245, 123, 299, 72, 339, 6, 341, 114, 221, 239, 95, 260, 67, 136, 193, 94, 281, 208, 209, 147, 203, 376, 220, 18, 96] + 56: [231, 9, 251, 284, 77, 207, 103, 250, 23, 317, 227, 95, 201, 375, 276, 65, 373, 337, 203, 36, 301, 173, 202, 90, 240, 88, 356, 6, 275, 80, 118, 73, 281, 374, 280, 243, 297, 134, 145, 200, 289, 60, 211, 212, 366, 74, 315, 3, 195, 107, 303, 264, 316, 2, 168, 369, 38, 105, 295, 11, 151, 368, 180, 314, 226, 367, 93, 254, 349, 377, 142, 146, 260, 167, 158, 263, 320, 332, 357, 0, 40, 324, 338, 34, 138, 364, 54, 96, 28, 372, 113, 47, 358, 247, 85, 224, 220, 8, 163, 196, 328, 302, 288, 72, 359, 10, 79, 149, 33, 129, 360, 347, 232, 16, 352, 272, 336, 174, 165, 143, 294, 177, 155, 189, 308, 41, 14, 383, 58, 312, 268, 269, 185, 221, 62, 172, 319, 29, 43, 26, 52, 45, 235, 208, 321, 100, 116, 322, 104, 71, 188, 101, 309, 148, 193, 179, 12, 270, 313, 271, 228, 330, 67, 350, 108, 59, 216, 318, 242, 279, 273, 351, 187, 46, 362, 381, 233, 160, 278, 166, 355, 209, 339, 299, 310, 379, 78, 361, 382, 261, 217, 51, 265, 147, 186, 267, 333, 48, 18, 192, 238, 69, 92, 305, 133, 255, 248, 94, 341, 175, 57, 152, 131, 215, 194, 61, 44, 304, 53, 371, 236, 39, 329, 244, 246, 27, 206, 198, 214, 164, 286, 210, 353, 274, 139, 287, 157, 68, 277, 102, 178, 75, 199, 156, 306, 346, 83, 150, 223, 182, 181, 136, 110, 153, 42, 55, 262, 124, 266, 130, 345, 370, 365, 229, 82, 5, 81, 169, 125, 161, 66, 63, 197, 285, 135, 252, 50, 17, 56, 183, 84, 293, 49, 170, 325, 7, 380, 76, 213, 30, 191, 87, 363, 140, 19, 245, 205, 342, 128, 35, 22, 123, 378, 257, 119, 307, 64, 70, 234, 326, 25, 171, 15, 259, 126, 184, 89, 99, 327, 283, 344, 376, 21, 31, 331, 127, 115, 122, 132, 4, 291, 24, 204, 239, 111, 237, 225, 340, 311, 98, 20, 162, 219, 154, 97, 300, 253, 354, 1, 13, 282, 298, 218, 323, 144, 86, 256, 335, 114, 91, 241, 106, 343, 296, 249, 117, 292, 159, 120, 334, 137, 230, 121, 141, 190, 348, 290, 112, 37, 222, 258, 176, 32, 109] + 57: [254, 344, 318, 35, 58, 179, 113, 277, 55, 225, 209, 81, 334, 236, 200, 224, 370, 264, 294, 181, 367, 301, 304, 343, 276, 220, 198, 267, 283, 88, 272, 243, 214, 183, 380, 279, 321, 98, 29, 315, 281, 194, 374, 266, 82, 336, 293, 372, 326, 342, 156, 124, 204, 79, 354, 43, 341, 83, 92, 119, 149, 23, 18, 247, 59, 357, 222, 78, 218, 278, 34, 285, 25, 11, 63, 274, 169, 22, 250, 365, 297, 46, 147, 112, 71, 369, 323, 145, 24, 188, 361, 287, 115, 258, 30, 56, 231, 93, 368, 359, 60, 180, 251, 316, 9, 20, 116, 13, 313, 90, 311, 97, 265, 168, 242, 364, 244, 212, 195, 86, 14, 157, 57, 108, 300, 345, 185, 91, 110, 177, 111, 38, 166, 199, 132, 162, 355, 87, 50, 192, 61, 340, 245, 234, 146, 154, 158, 235, 335, 150, 123, 167, 172, 249, 356, 106, 329, 337, 17, 371, 33, 4, 109, 239, 100, 291, 227, 262, 259, 74, 16, 140, 118, 197, 160, 238, 346, 152, 349, 289, 2, 196, 155, 237, 252, 0, 1, 76, 273, 189, 305, 275, 295, 302, 133, 7, 217, 317, 64, 182, 73, 42, 377, 358, 210, 352, 216, 261, 5, 378, 219, 103, 142, 52, 37, 229, 165, 125, 31, 65, 130, 117, 306, 39, 161, 173, 312, 12, 328, 296, 330, 332, 45, 255, 66, 138, 310, 221, 131, 105, 89, 170, 314, 128, 141, 350, 80, 253, 373, 298, 191, 47, 348, 190, 186, 379, 122, 270, 333, 232, 360, 327, 153, 233, 246, 320, 121, 366, 8, 127, 163, 99, 176, 27, 382, 120, 68, 351, 178, 175, 303, 307, 241, 107, 10, 129, 280, 19, 187, 286, 85, 101, 324, 151, 309, 347, 213, 51, 184, 284, 95, 282, 215, 41, 363, 126, 260, 375, 271, 206, 15, 201, 205, 72, 230, 257, 137, 6, 75, 319, 40, 228, 54, 325, 226, 3, 362, 26, 49, 32, 36, 256, 339, 114, 292, 376, 148, 331, 381, 104, 268, 84, 44, 338, 322, 96, 21, 208, 164, 353, 288, 240, 135, 248, 263, 48, 139, 193, 69, 143, 207, 94, 159, 223, 53, 174, 62, 144, 134, 70, 102, 203, 136, 171, 28, 290, 211, 299, 383, 67, 308, 269, 77, 202] + 58: [377, 3, 71, 125, 319, 321, 335, 370, 10, 108, 150, 168, 184, 280, 364, 201, 204, 340, 33, 38, 58, 134, 209, 308, 220, 178, 341, 12, 190, 187, 239, 82, 333, 63, 300, 43, 113, 8, 296, 359, 290, 349, 193, 74, 212, 158, 292, 309, 81, 344, 289, 199, 18, 16, 106, 198, 251, 62, 250, 110, 241, 21, 347, 237, 138, 100, 44, 51, 216, 174, 313, 279, 245, 79, 322, 355, 175, 6, 104, 242, 380, 25, 284, 53, 131, 253, 65, 19, 230, 142, 372, 46, 84, 304, 217, 291, 156, 61, 373, 67, 232, 47, 132, 197, 281, 244, 181, 339, 302, 144, 371, 265, 133, 170, 303, 157, 369, 286, 310, 165, 275, 76, 101, 351, 36, 85, 130, 352, 229, 207, 59, 124, 346, 231, 123, 115, 180, 243, 86, 276, 96, 15, 301, 147, 126, 151, 271, 358, 112, 141, 191, 4, 28, 314, 266, 160, 378, 154, 288, 35, 94, 294, 221, 282, 39, 32, 140, 91, 56, 249, 14, 72, 116, 342, 31, 20, 328, 55, 383, 183, 149, 357, 161, 182, 338, 5, 317, 337, 57, 34, 218, 361, 366, 77, 210, 22, 368, 324, 272, 316, 73, 202, 350, 83, 0, 172, 228, 11, 179, 360, 87, 374, 109, 145, 323, 89, 305, 248, 211, 69, 169, 295, 189, 274, 224, 381, 143, 227, 37, 176, 327, 50, 105, 24, 54, 254, 117, 153, 343, 233, 205, 263, 348, 9, 256, 45, 277, 60, 285, 375, 258, 234, 382, 186, 107, 225, 118, 297, 163, 354, 120, 159, 363, 23, 92, 114, 27, 49, 40, 196, 97, 260, 329, 331, 129, 155, 135, 13, 278, 195, 173, 102, 128, 336, 246, 137, 30, 320, 98, 298, 273, 299, 26, 68, 367, 315, 66, 48, 215, 238, 240, 148, 334, 283, 219, 213, 264, 146, 167, 17, 127, 80, 194, 122, 259, 235, 270, 376, 42, 111, 307, 139, 236, 306, 75, 379, 318, 293, 226, 64, 268, 41, 223, 52, 188, 164, 311, 312, 7, 93, 214, 262, 203, 255, 185, 1, 325, 326, 121, 208, 345, 287, 136, 261, 330, 200, 247, 78, 267, 166, 222, 177, 356, 206, 257, 152, 119, 162, 171, 192, 99, 90, 332, 362, 2, 70, 353, 29, 95, 103, 269, 88, 365, 252] + 59: [327, 371, 172, 80, 114, 164, 211, 315, 214, 344, 36, 96, 0, 246, 37, 88, 148, 312, 305, 39, 55, 235, 154, 189, 25, 33, 95, 150, 38, 112, 109, 279, 22, 19, 137, 337, 199, 206, 41, 74, 89, 166, 319, 237, 293, 139, 308, 215, 170, 182, 70, 300, 262, 58, 57, 297, 60, 142, 146, 203, 292, 333, 110, 153, 145, 296, 257, 108, 14, 87, 268, 271, 229, 113, 5, 331, 9, 309, 278, 16, 383, 245, 222, 238, 349, 48, 149, 322, 326, 106, 102, 8, 316, 77, 378, 341, 7, 73, 233, 130, 372, 266, 91, 168, 169, 29, 179, 92, 131, 32, 64, 253, 359, 249, 307, 301, 98, 125, 78, 117, 140, 332, 147, 291, 186, 281, 45, 160, 163, 248, 356, 180, 155, 61, 71, 34, 21, 132, 275, 358, 350, 121, 190, 4, 209, 65, 226, 221, 324, 382, 176, 35, 302, 373, 47, 198, 365, 107, 299, 184, 217, 210, 11, 103, 360, 375, 116, 351, 224, 234, 197, 294, 295, 243, 212, 254, 280, 252, 10, 258, 101, 2, 43, 129, 67, 183, 191, 284, 192, 56, 122, 167, 250, 352, 97, 263, 185, 178, 128, 40, 127, 174, 306, 348, 289, 242, 138, 187, 330, 277, 273, 82, 141, 171, 126, 15, 379, 346, 313, 311, 18, 377, 342, 175, 181, 225, 338, 369, 361, 362, 194, 208, 28, 93, 357, 368, 239, 244, 152, 90, 370, 66, 30, 251, 213, 26, 196, 81, 62, 320, 158, 69, 353, 156, 380, 84, 134, 336, 119, 216, 276, 264, 345, 188, 94, 220, 75, 204, 76, 328, 290, 223, 100, 376, 236, 339, 228, 51, 193, 50, 17, 3, 201, 283, 282, 367, 83, 111, 195, 340, 285, 374, 42, 317, 133, 161, 159, 218, 329, 177, 259, 68, 27, 135, 207, 49, 321, 165, 318, 205, 347, 79, 202, 200, 46, 247, 31, 151, 255, 85, 334, 162, 355, 120, 6, 314, 287, 231, 157, 44, 118, 241, 219, 270, 72, 323, 260, 173, 105, 298, 265, 144, 86, 304, 267, 143, 54, 366, 354, 123, 63, 325, 343, 261, 53, 303, 1, 272, 104, 310, 59, 363, 256, 381, 288, 23, 232, 12, 364, 20, 286, 269, 13, 274, 227, 136, 124, 24, 115, 230, 240, 335, 99, 52] + 60: [313, 309, 161, 253, 248, 93, 11, 305, 90, 66, 369, 371, 363, 287, 216, 111, 249, 71, 77, 62, 154, 288, 22, 355, 124, 34, 119, 232, 364, 150, 37, 80, 342, 53, 273, 314, 225, 383, 324, 261, 298, 306, 138, 87, 157, 207, 375, 201, 351, 167, 208, 337, 15, 203, 359, 255, 269, 9, 377, 21, 33, 95, 368, 270, 251, 145, 211, 326, 177, 109, 214, 348, 123, 347, 120, 74, 290, 206, 271, 152, 114, 92, 38, 134, 196, 182, 219, 307, 252, 54, 378, 292, 275, 10, 373, 299, 257, 243, 222, 181, 294, 112, 131, 169, 84, 358, 88, 56, 233, 91, 130, 256, 282, 361, 42, 68, 301, 73, 218, 317, 16, 170, 39, 65, 234, 180, 278, 172, 336, 6, 332, 304, 346, 200, 192, 286, 105, 13, 258, 29, 350, 4, 44, 63, 268, 121, 343, 247, 28, 353, 213, 235, 75, 221, 344, 3, 25, 142, 100, 296, 148, 67, 328, 374, 175, 17, 173, 210, 381, 236, 244, 230, 2, 49, 98, 59, 140, 283, 240, 264, 26, 31, 30, 349, 315, 202, 320, 224, 291, 72, 103, 101, 158, 94, 303, 194, 155, 319, 354, 223, 241, 362, 259, 267, 176, 52, 187, 370, 64, 125, 330, 198, 229, 43, 227, 308, 220, 102, 266, 316, 297, 76, 365, 245, 263, 20, 183, 237, 274, 70, 285, 322, 380, 376, 45, 280, 356, 135, 289, 89, 47, 284, 115, 212, 185, 239, 311, 117, 55, 69, 318, 5, 61, 141, 7, 341, 12, 333, 178, 19, 97, 254, 367, 136, 139, 310, 195, 48, 334, 108, 166, 272, 82, 277, 1, 382, 279, 107, 85, 357, 14, 338, 127, 41, 160, 86, 295, 118, 60, 265, 129, 78, 302, 327, 329, 231, 99, 189, 132, 24, 8, 360, 293, 106, 144, 156, 262, 50, 379, 159, 340, 321, 126, 174, 260, 113, 312, 205, 193, 184, 27, 151, 51, 162, 57, 171, 149, 79, 276, 168, 23, 104, 204, 165, 32, 352, 83, 40, 335, 339, 153, 366, 345, 143, 238, 58, 217, 110, 215, 323, 250, 300, 331, 197, 209, 137, 147, 281, 163, 226, 133, 199, 128, 372, 81, 35, 46, 122, 246, 0, 36, 188, 116, 186, 146, 190, 18, 325, 228, 179, 96, 191, 164, 242] + 61: [221, 178, 353, 119, 333, 243, 224, 366, 47, 23, 38, 273, 295, 132, 130, 91, 13, 201, 35, 254, 76, 116, 319, 58, 127, 205, 45, 294, 291, 31, 92, 316, 121, 245, 290, 153, 242, 26, 359, 63, 123, 185, 146, 279, 41, 343, 155, 0, 179, 108, 117, 218, 143, 34, 282, 65, 83, 379, 97, 173, 9, 22, 14, 106, 101, 99, 138, 339, 382, 183, 356, 149, 338, 360, 323, 21, 215, 177, 133, 258, 29, 2, 368, 137, 214, 196, 71, 193, 54, 24, 314, 56, 60, 376, 171, 208, 61, 303, 165, 1, 68, 327, 349, 337, 90, 252, 69, 249, 78, 365, 67, 81, 244, 163, 111, 272, 318, 277, 229, 40, 73, 231, 351, 380, 371, 288, 181, 228, 233, 113, 3, 102, 292, 125, 240, 124, 274, 310, 331, 305, 347, 281, 217, 357, 93, 107, 247, 184, 51, 192, 159, 75, 297, 82, 308, 202, 378, 37, 79, 284, 152, 301, 358, 265, 105, 131, 62, 96, 211, 216, 326, 264, 278, 66, 42, 43, 150, 27, 260, 147, 139, 306, 141, 317, 352, 271, 246, 293, 220, 377, 16, 296, 46, 88, 170, 209, 372, 112, 286, 261, 77, 190, 336, 200, 259, 174, 126, 342, 148, 255, 7, 95, 222, 226, 266, 334, 299, 86, 157, 59, 12, 238, 110, 160, 348, 114, 374, 207, 72, 346, 340, 4, 311, 239, 49, 180, 276, 64, 320, 128, 197, 161, 172, 270, 17, 142, 115, 169, 191, 300, 304, 328, 154, 30, 283, 194, 298, 6, 251, 164, 250, 175, 50, 189, 381, 307, 230, 241, 330, 198, 168, 309, 20, 84, 109, 52, 186, 341, 219, 204, 98, 364, 74, 182, 236, 355, 267, 321, 100, 263, 5, 10, 363, 55, 188, 227, 361, 268, 257, 206, 89, 275, 85, 199, 345, 25, 232, 212, 256, 39, 134, 94, 36, 248, 136, 203, 103, 213, 57, 156, 362, 166, 313, 158, 33, 375, 225, 144, 370, 354, 312, 28, 140, 151, 367, 53, 167, 324, 335, 122, 253, 237, 129, 15, 118, 135, 329, 145, 383, 8, 80, 332, 70, 187, 262, 373, 302, 285, 325, 350, 315, 344, 120, 19, 195, 235, 369, 32, 87, 44, 11, 104, 322, 280, 269, 176, 162, 18, 234, 48, 223, 289, 210, 287] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots416.yaml b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots416.yaml new file mode 100644 index 000000000000..de20f38b0a6b --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots416.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [259, 86, 264, 332, 312, 29, 232, 198, 173, 205, 156, 10, 94, 38, 133, 4, 216, 97, 326, 98, 294, 191, 285, 271, 323, 58, 166, 225, 135, 45, 188, 95, 184, 68, 144, 141, 11, 361, 208, 77, 244, 335, 192, 100, 328, 286, 305, 334, 220, 23, 325, 201, 166, 225, 235, 190, 182, 242, 321, 276, 3, 54, 313, 265, 170, 268, 85, 240, 372, 195, 82, 342, 253, 353, 164, 18, 88, 370, 0, 383, 135, 346, 35, 80, 327, 176, 209, 55, 105, 129, 261, 358, 308, 103, 19, 215, 237, 154, 40, 219, 266, 92, 355, 247, 256, 371, 254, 378, 163, 95, 134, 16, 330, 181, 43, 57, 138, 234, 223, 365, 178, 25, 73, 50, 340, 380, 125, 132, 128, 373, 256, 371, 111, 158, 122, 62, 289, 318, 330, 33, 131, 311, 300, 160, 87, 331, 140, 238, 67, 102, 338, 179, 357, 189, 273, 13, 150, 245, 111, 45, 169, 218, 184, 37, 290, 284, 196, 36, 7, 123, 39, 146, 269, 320, 127, 6, 212, 53, 251, 298, 364, 274, 151, 245, 254, 230, 60, 30, 65, 63, 290, 243, 107, 258, 112, 314, 187, 136, 99, 296, 263, 252, 352, 113, 194, 287, 130, 277, 74, 236, 165, 279, 81, 79, 255, 217, 282, 336, 34, 297, 337, 149, 257, 213, 366, 106, 147, 20, 96, 2, 301, 304, 168, 118, 74, 236, 165, 279, 153, 202, 48, 203, 126, 360, 56, 354, 114, 199, 5, 108, 246, 379, 382, 47, 316, 69, 319, 329, 288, 310, 71, 345, 111, 241, 26, 104, 376, 217, 280, 143, 101, 174, 324, 375, 172, 227, 59, 1, 267, 229, 161, 152, 381, 226, 291, 185, 222, 322, 177, 378, 295, 17, 49, 278, 224, 72, 283, 186, 148, 31, 32, 214, 369, 221, 197, 145, 42, 41, 239, 24, 228, 303, 222, 119, 348, 378, 51, 17, 49, 293, 224, 307, 341, 90, 248, 89, 52, 137, 363, 270, 299, 260, 64, 175, 367, 121, 281, 61, 302, 159, 165, 193, 21, 79, 70, 139, 272, 359, 183, 124, 44, 27, 66, 249, 116, 211, 14, 210, 115, 306, 9, 142, 344, 120, 351, 84, 162, 207, 60, 204, 343, 203, 349, 22, 117, 200, 368, 262, 377, 46, 75, 292, 83, 362, 206, 315, 250, 167, 339, 110, 356, 356, 155, 76, 347, 157, 134, 333, 173, 78, 233, 109, 28, 309, 91, 317, 275, 180, 93, 350, 15, 12, 231, 374, 8, 171] + 1: [134, 80, 72, 291, 319, 366, 56, 120, 243, 275, 266, 197, 219, 171, 101, 164, 107, 184, 293, 13, 262, 372, 90, 124, 337, 15, 178, 158, 72, 167, 347, 328, 3, 185, 272, 71, 234, 335, 282, 88, 95, 204, 364, 136, 36, 323, 77, 357, 221, 322, 359, 363, 380, 106, 91, 163, 21, 377, 58, 153, 138, 287, 355, 108, 4, 38, 133, 170, 139, 229, 315, 94, 353, 129, 316, 374, 117, 327, 141, 158, 149, 163, 319, 5, 198, 111, 224, 173, 285, 288, 206, 306, 237, 348, 20, 296, 51, 268, 226, 371, 253, 132, 341, 255, 251, 44, 91, 169, 347, 41, 63, 82, 383, 242, 125, 379, 93, 203, 189, 7, 332, 68, 48, 97, 356, 220, 340, 64, 307, 215, 81, 180, 50, 177, 168, 325, 244, 23, 320, 86, 317, 218, 52, 69, 102, 259, 370, 154, 35, 104, 27, 130, 373, 343, 28, 112, 200, 180, 85, 378, 6, 0, 182, 271, 273, 37, 301, 194, 143, 239, 147, 330, 65, 156, 26, 346, 344, 369, 151, 66, 304, 368, 365, 137, 43, 169, 319, 377, 182, 111, 59, 269, 212, 109, 116, 358, 256, 222, 161, 295, 350, 302, 49, 148, 217, 128, 159, 336, 365, 18, 277, 254, 192, 73, 105, 24, 46, 32, 78, 166, 176, 289, 76, 121, 326, 62, 290, 352, 103, 83, 135, 361, 157, 34, 1, 118, 277, 329, 39, 210, 300, 225, 29, 211, 274, 187, 228, 381, 298, 122, 17, 19, 230, 96, 119, 246, 188, 252, 339, 265, 235, 137, 382, 127, 67, 110, 334, 82, 25, 89, 245, 280, 53, 360, 98, 362, 14, 165, 87, 8, 232, 297, 160, 16, 258, 100, 42, 22, 263, 196, 67, 123, 3, 233, 283, 247, 216, 142, 214, 376, 60, 140, 286, 321, 186, 181, 240, 309, 193, 45, 144, 199, 42, 308, 150, 167, 131, 270, 198, 248, 292, 9, 74, 162, 47, 264, 205, 313, 318, 57, 172, 311, 314, 375, 207, 208, 354, 267, 201, 31, 152, 177, 241, 70, 294, 351, 292, 145, 99, 84, 223, 342, 61, 79, 2, 115, 11, 113, 54, 279, 183, 10, 257, 310, 201, 22, 382, 127, 30, 179, 120, 250, 272, 260, 281, 92, 114, 278, 236, 238, 338, 175, 284, 227, 333, 331, 367, 191, 276, 312, 174, 174, 126, 163, 40, 75, 299, 225, 195, 303, 146, 261, 213, 202, 231, 305, 249, 33, 324, 55, 12, 190, 209, 155, 349, 345] + 2: [53, 115, 28, 212, 192, 110, 70, 42, 51, 200, 174, 170, 67, 218, 16, 169, 12, 29, 334, 261, 107, 205, 222, 13, 208, 375, 53, 115, 223, 143, 101, 155, 73, 184, 314, 279, 196, 18, 156, 120, 349, 276, 364, 104, 346, 319, 288, 8, 310, 289, 301, 379, 43, 93, 298, 75, 242, 168, 91, 306, 324, 235, 311, 87, 50, 187, 241, 309, 49, 94, 275, 231, 292, 98, 117, 380, 32, 366, 43, 93, 1, 197, 232, 97, 173, 105, 139, 274, 294, 140, 92, 76, 360, 116, 9, 257, 367, 264, 370, 335, 345, 214, 202, 363, 180, 102, 5, 125, 19, 191, 166, 304, 45, 217, 79, 299, 327, 201, 65, 15, 305, 152, 195, 141, 226, 158, 378, 266, 316, 381, 230, 194, 295, 151, 185, 284, 91, 135, 122, 82, 6, 7, 262, 198, 163, 250, 106, 150, 47, 178, 371, 338, 147, 317, 354, 255, 96, 194, 81, 248, 60, 322, 355, 172, 139, 328, 165, 344, 320, 128, 277, 254, 136, 365, 238, 376, 373, 114, 336, 315, 358, 258, 132, 89, 295, 143, 290, 84, 377, 44, 368, 244, 323, 263, 27, 361, 308, 160, 154, 176, 77, 352, 329, 307, 234, 260, 337, 229, 132, 343, 259, 212, 162, 111, 377, 86, 22, 11, 285, 273, 68, 383, 10, 193, 142, 2, 36, 331, 95, 183, 37, 353, 282, 300, 112, 14, 189, 100, 19, 124, 66, 118, 122, 24, 30, 347, 55, 20, 145, 348, 252, 23, 211, 144, 291, 267, 283, 249, 270, 190, 112, 89, 3, 247, 290, 62, 56, 245, 26, 146, 61, 206, 210, 58, 253, 103, 38, 268, 4, 272, 99, 251, 227, 240, 339, 199, 108, 215, 137, 248, 213, 109, 286, 123, 304, 340, 342, 90, 269, 236, 41, 186, 39, 179, 134, 359, 216, 48, 113, 64, 228, 330, 108, 215, 321, 248, 232, 62, 280, 86, 325, 85, 303, 224, 293, 21, 171, 175, 121, 219, 239, 233, 302, 59, 382, 88, 287, 40, 225, 357, 221, 34, 149, 153, 278, 133, 314, 63, 209, 17, 188, 138, 332, 129, 256, 318, 35, 52, 54, 164, 72, 127, 119, 161, 341, 313, 3, 143, 78, 350, 131, 271, 296, 333, 159, 356, 126, 148, 265, 181, 312, 0, 25, 207, 71, 167, 177, 369, 237, 246, 351, 313, 223, 33, 242, 168, 157, 220, 31, 83, 204, 130, 203, 297, 46, 69, 372, 362, 281, 57, 243, 74, 182, 80, 326, 374] + 3: [39, 377, 314, 317, 62, 334, 343, 250, 195, 12, 142, 295, 218, 217, 255, 30, 43, 106, 132, 309, 16, 156, 89, 352, 9, 233, 39, 230, 302, 225, 80, 50, 238, 122, 212, 264, 78, 374, 274, 235, 280, 214, 286, 294, 130, 283, 378, 242, 90, 315, 5, 278, 22, 377, 209, 51, 24, 42, 337, 336, 316, 64, 161, 262, 345, 327, 266, 125, 340, 91, 381, 285, 0, 338, 148, 23, 226, 222, 22, 377, 302, 31, 151, 137, 186, 2, 182, 11, 136, 297, 103, 53, 355, 276, 249, 70, 282, 362, 73, 107, 253, 152, 173, 38, 66, 310, 171, 350, 348, 213, 26, 358, 219, 25, 271, 114, 224, 256, 44, 268, 298, 291, 72, 153, 251, 45, 141, 60, 33, 303, 66, 275, 18, 163, 261, 145, 207, 211, 304, 202, 162, 312, 176, 57, 273, 100, 187, 281, 346, 269, 116, 234, 127, 124, 247, 324, 66, 260, 319, 163, 263, 158, 178, 189, 77, 357, 109, 111, 126, 380, 48, 383, 322, 367, 356, 133, 179, 88, 330, 320, 371, 292, 66, 260, 319, 13, 313, 81, 102, 231, 138, 382, 84, 32, 181, 197, 65, 328, 184, 67, 110, 28, 134, 121, 1, 265, 332, 198, 66, 260, 18, 131, 97, 180, 339, 10, 221, 118, 8, 351, 98, 94, 105, 237, 353, 7, 123, 206, 183, 140, 75, 272, 52, 257, 66, 35, 365, 19, 157, 150, 168, 58, 36, 119, 279, 243, 299, 166, 129, 160, 306, 376, 85, 293, 372, 284, 147, 349, 17, 146, 71, 310, 14, 27, 333, 252, 201, 325, 296, 240, 154, 229, 300, 270, 135, 82, 361, 305, 228, 321, 323, 360, 205, 244, 185, 311, 46, 310, 40, 192, 3, 363, 170, 37, 287, 87, 364, 92, 204, 55, 307, 144, 248, 15, 373, 258, 354, 369, 241, 347, 193, 47, 34, 34, 40, 3, 194, 318, 120, 227, 239, 29, 289, 203, 375, 344, 117, 366, 20, 188, 68, 86, 220, 210, 95, 308, 335, 199, 34, 34, 319, 27, 263, 49, 63, 267, 74, 21, 99, 277, 41, 236, 246, 104, 216, 83, 165, 93, 190, 159, 143, 69, 167, 59, 34, 79, 365, 56, 313, 6, 139, 164, 174, 290, 359, 149, 155, 112, 169, 196, 115, 232, 61, 254, 245, 191, 54, 175, 341, 301, 34, 79, 365, 208, 194, 331, 215, 342, 177, 223, 172, 288, 108, 200, 113, 76, 96, 128, 259, 101, 379, 329, 368, 4, 326, 370] + 4: [188, 331, 371, 16, 273, 254, 274, 145, 196, 73, 207, 324, 85, 185, 131, 135, 294, 29, 291, 111, 378, 163, 238, 299, 242, 63, 34, 331, 214, 15, 150, 6, 345, 106, 83, 28, 7, 116, 360, 60, 46, 25, 306, 157, 146, 147, 115, 123, 293, 212, 337, 277, 340, 331, 315, 226, 5, 136, 36, 372, 32, 68, 153, 283, 307, 323, 132, 310, 292, 175, 170, 211, 21, 48, 102, 301, 119, 282, 3, 343, 320, 228, 117, 254, 197, 76, 57, 204, 35, 264, 256, 341, 64, 94, 39, 234, 289, 130, 10, 1, 265, 154, 208, 332, 3, 331, 219, 19, 105, 148, 59, 41, 151, 251, 317, 99, 65, 14, 261, 137, 84, 184, 210, 355, 381, 380, 245, 326, 192, 305, 3, 331, 315, 281, 319, 348, 86, 275, 160, 333, 260, 276, 53, 359, 216, 158, 66, 78, 140, 109, 129, 112, 11, 373, 20, 120, 18, 343, 250, 95, 319, 302, 253, 200, 101, 49, 369, 221, 330, 297, 313, 240, 349, 26, 122, 223, 203, 353, 202, 363, 67, 321, 209, 343, 350, 88, 87, 308, 155, 325, 43, 27, 177, 335, 168, 33, 74, 284, 178, 206, 181, 268, 97, 374, 224, 190, 227, 182, 209, 343, 259, 95, 257, 143, 47, 77, 218, 312, 82, 243, 356, 62, 232, 354, 50, 176, 334, 213, 186, 247, 366, 195, 17, 90, 271, 296, 371, 172, 273, 148, 329, 113, 357, 161, 179, 54, 290, 379, 383, 100, 267, 303, 327, 40, 215, 191, 164, 270, 128, 336, 271, 300, 214, 23, 171, 241, 342, 318, 364, 344, 159, 244, 166, 149, 108, 322, 52, 263, 118, 286, 81, 173, 144, 89, 225, 351, 271, 343, 24, 376, 194, 142, 56, 61, 361, 280, 110, 0, 72, 93, 229, 328, 370, 346, 304, 156, 126, 162, 368, 174, 233, 104, 37, 296, 214, 15, 117, 152, 298, 139, 235, 217, 38, 183, 133, 352, 70, 69, 309, 103, 295, 31, 96, 362, 248, 285, 2, 107, 12, 222, 23, 9, 194, 142, 377, 246, 365, 198, 141, 278, 252, 169, 125, 288, 220, 51, 180, 71, 205, 316, 134, 266, 231, 58, 375, 258, 315, 226, 45, 152, 167, 347, 44, 339, 187, 358, 189, 98, 262, 80, 92, 121, 249, 201, 311, 279, 30, 236, 79, 287, 124, 199, 193, 239, 87, 165, 13, 4, 314, 8, 22, 42, 114, 75, 269, 367, 138, 230, 237, 127, 255, 55, 91, 382, 272, 338] + 5: [172, 160, 346, 269, 30, 231, 352, 192, 66, 264, 69, 175, 144, 380, 362, 150, 237, 191, 283, 156, 261, 73, 357, 367, 176, 44, 208, 194, 346, 305, 381, 78, 255, 182, 327, 268, 318, 377, 104, 274, 201, 138, 254, 349, 323, 244, 86, 111, 350, 110, 83, 50, 61, 214, 49, 265, 272, 41, 130, 74, 209, 163, 319, 220, 33, 183, 230, 325, 149, 162, 221, 40, 224, 307, 294, 117, 277, 303, 61, 5, 60, 276, 52, 231, 200, 293, 155, 2, 181, 324, 99, 93, 343, 289, 334, 229, 103, 342, 290, 53, 282, 116, 278, 75, 61, 194, 355, 169, 269, 372, 252, 147, 8, 174, 184, 258, 28, 216, 239, 161, 80, 361, 129, 270, 292, 185, 328, 112, 142, 368, 165, 214, 65, 305, 356, 259, 132, 364, 304, 222, 251, 315, 301, 39, 20, 76, 9, 77, 374, 298, 373, 267, 202, 248, 164, 127, 154, 214, 98, 100, 288, 211, 212, 247, 137, 59, 67, 320, 79, 51, 366, 136, 48, 94, 113, 168, 32, 139, 336, 339, 326, 177, 190, 296, 217, 262, 287, 64, 145, 335, 4, 332, 140, 152, 295, 312, 121, 341, 119, 232, 263, 107, 347, 120, 36, 273, 157, 286, 190, 308, 260, 223, 272, 379, 17, 246, 313, 369, 206, 333, 195, 266, 240, 63, 383, 55, 235, 92, 297, 85, 234, 226, 167, 321, 141, 296, 217, 105, 6, 159, 7, 331, 70, 151, 3, 34, 35, 242, 382, 219, 87, 29, 205, 316, 95, 126, 146, 0, 96, 23, 213, 256, 355, 100, 365, 259, 279, 198, 173, 15, 207, 153, 228, 329, 300, 241, 275, 62, 363, 58, 148, 197, 340, 280, 166, 170, 213, 84, 81, 105, 288, 102, 215, 25, 371, 171, 24, 314, 115, 253, 353, 90, 330, 271, 378, 68, 189, 285, 123, 180, 57, 196, 213, 84, 47, 276, 52, 122, 43, 91, 233, 375, 204, 218, 54, 19, 56, 284, 360, 344, 299, 135, 125, 309, 42, 109, 179, 118, 249, 128, 178, 193, 287, 21, 351, 338, 124, 358, 38, 26, 46, 345, 250, 257, 281, 376, 359, 187, 322, 97, 158, 310, 82, 337, 249, 306, 355, 370, 6, 159, 88, 89, 131, 238, 108, 210, 186, 12, 106, 317, 236, 291, 114, 18, 101, 133, 311, 27, 348, 302, 128, 128, 1, 188, 356, 21, 72, 22, 31, 37, 143, 45, 16, 227, 14, 203, 13, 134, 71, 199, 245, 11, 243, 10, 225, 354] + 6: [322, 376, 32, 46, 166, 307, 100, 196, 165, 185, 30, 44, 15, 110, 225, 248, 50, 334, 74, 152, 47, 78, 26, 20, 252, 342, 322, 376, 32, 99, 233, 359, 293, 34, 301, 377, 49, 107, 58, 262, 217, 362, 105, 268, 285, 62, 348, 374, 91, 200, 214, 323, 322, 376, 53, 240, 67, 175, 112, 289, 234, 41, 12, 282, 154, 361, 4, 355, 48, 272, 364, 251, 266, 250, 29, 61, 75, 90, 146, 308, 267, 341, 209, 378, 368, 184, 300, 336, 190, 221, 286, 87, 82, 372, 333, 299, 366, 290, 117, 131, 213, 265, 153, 257, 76, 308, 343, 281, 177, 375, 57, 155, 156, 236, 111, 139, 98, 22, 23, 14, 199, 125, 379, 85, 17, 222, 227, 191, 167, 71, 76, 308, 27, 55, 180, 216, 187, 188, 354, 115, 231, 327, 280, 6, 86, 254, 18, 54, 73, 226, 256, 356, 127, 382, 186, 315, 76, 308, 249, 253, 66, 318, 88, 255, 113, 179, 279, 133, 220, 69, 243, 138, 83, 70, 39, 106, 51, 339, 192, 72, 65, 247, 297, 5, 174, 258, 371, 9, 84, 208, 95, 128, 304, 144, 296, 206, 330, 261, 346, 381, 235, 150, 21, 38, 7, 218, 363, 331, 305, 274, 160, 99, 166, 163, 367, 287, 270, 369, 317, 211, 332, 137, 63, 316, 147, 349, 357, 203, 303, 263, 92, 228, 132, 161, 305, 274, 32, 229, 326, 13, 232, 319, 176, 109, 68, 114, 324, 207, 284, 329, 224, 8, 130, 80, 351, 238, 140, 353, 149, 164, 97, 142, 258, 28, 233, 260, 119, 118, 277, 352, 311, 219, 365, 271, 335, 122, 3, 215, 312, 337, 162, 313, 93, 273, 25, 320, 97, 59, 174, 269, 205, 181, 94, 291, 0, 380, 101, 77, 264, 242, 197, 145, 124, 16, 310, 360, 294, 89, 141, 170, 135, 325, 97, 295, 343, 240, 120, 350, 246, 79, 151, 245, 223, 64, 136, 104, 116, 126, 198, 298, 244, 288, 338, 302, 171, 2, 314, 373, 97, 274, 103, 11, 177, 45, 283, 129, 259, 230, 158, 169, 10, 43, 276, 81, 193, 42, 292, 328, 102, 183, 1, 172, 204, 35, 212, 142, 306, 253, 66, 36, 56, 178, 370, 278, 202, 383, 143, 237, 60, 194, 168, 309, 134, 108, 195, 201, 19, 340, 33, 40, 142, 142, 53, 281, 31, 173, 189, 239, 345, 182, 37, 321, 157, 210, 159, 148, 358, 96, 123, 347, 24, 121, 241, 344, 52, 275] + 7: [163, 361, 375, 265, 18, 188, 77, 82, 278, 288, 377, 300, 16, 185, 380, 67, 373, 203, 35, 33, 123, 78, 246, 138, 279, 171, 341, 64, 28, 142, 105, 212, 104, 313, 23, 2, 62, 90, 109, 92, 102, 121, 251, 351, 216, 75, 254, 128, 299, 127, 376, 50, 327, 263, 176, 375, 18, 97, 174, 47, 349, 339, 177, 132, 234, 178, 305, 232, 283, 366, 209, 363, 111, 381, 150, 205, 223, 257, 327, 263, 187, 142, 280, 329, 244, 217, 289, 325, 221, 214, 259, 107, 86, 134, 161, 52, 151, 112, 173, 124, 304, 331, 160, 365, 327, 64, 260, 46, 25, 298, 281, 11, 29, 319, 117, 98, 89, 186, 83, 226, 247, 101, 318, 295, 88, 334, 45, 12, 145, 371, 175, 308, 292, 343, 25, 369, 302, 183, 31, 148, 219, 264, 311, 136, 335, 306, 309, 193, 115, 36, 312, 315, 95, 197, 240, 91, 175, 108, 49, 271, 103, 354, 61, 307, 248, 270, 170, 243, 189, 32, 322, 323, 146, 116, 222, 207, 39, 317, 147, 172, 154, 357, 162, 192, 292, 342, 99, 298, 256, 164, 262, 356, 208, 330, 204, 20, 137, 353, 68, 149, 268, 69, 367, 159, 195, 199, 355, 42, 157, 192, 133, 343, 13, 15, 370, 100, 125, 84, 21, 333, 368, 129, 211, 155, 261, 22, 126, 194, 80, 277, 66, 290, 326, 202, 272, 336, 239, 340, 71, 362, 94, 324, 55, 250, 210, 167, 24, 5, 6, 51, 40, 225, 106, 286, 379, 3, 332, 143, 156, 110, 182, 275, 41, 265, 105, 328, 0, 267, 74, 296, 255, 206, 9, 190, 58, 141, 196, 345, 166, 287, 43, 252, 346, 54, 122, 364, 182, 275, 350, 229, 139, 378, 181, 294, 168, 360, 224, 158, 140, 130, 293, 310, 291, 87, 269, 316, 96, 266, 198, 282, 34, 338, 7, 274, 292, 340, 201, 118, 27, 215, 4, 284, 81, 344, 26, 358, 44, 152, 227, 153, 184, 1, 10, 297, 57, 352, 179, 135, 7, 238, 49, 382, 253, 362, 131, 249, 285, 374, 70, 372, 245, 237, 200, 53, 233, 120, 320, 169, 231, 303, 230, 165, 236, 348, 7, 258, 41, 229, 103, 48, 114, 113, 17, 337, 213, 191, 314, 19, 220, 301, 65, 38, 37, 72, 59, 63, 242, 347, 30, 276, 274, 274, 49, 382, 201, 369, 73, 14, 119, 235, 383, 241, 321, 180, 273, 56, 8, 93, 79, 218, 85, 228, 359, 144, 76, 60] + 8: [273, 21, 201, 10, 272, 182, 30, 39, 346, 147, 165, 368, 130, 60, 206, 116, 310, 16, 339, 347, 365, 270, 26, 139, 222, 100, 253, 338, 264, 362, 353, 204, 252, 374, 366, 333, 169, 309, 192, 358, 187, 249, 350, 143, 265, 322, 277, 370, 123, 138, 127, 163, 253, 302, 264, 201, 371, 337, 18, 294, 356, 133, 215, 219, 320, 42, 117, 191, 352, 199, 144, 54, 323, 208, 268, 94, 288, 234, 253, 106, 108, 62, 62, 371, 44, 280, 128, 119, 121, 189, 334, 8, 59, 90, 297, 313, 378, 14, 287, 158, 319, 301, 345, 207, 104, 135, 367, 321, 152, 229, 251, 86, 311, 231, 298, 194, 197, 332, 98, 28, 299, 372, 85, 175, 111, 171, 286, 51, 0, 102, 214, 135, 161, 321, 64, 229, 95, 168, 75, 283, 17, 198, 120, 52, 212, 247, 373, 68, 79, 29, 357, 244, 329, 82, 124, 96, 5, 31, 7, 241, 154, 74, 157, 377, 188, 109, 275, 112, 300, 284, 230, 87, 41, 136, 200, 93, 33, 348, 305, 355, 176, 262, 5, 238, 367, 243, 221, 81, 20, 148, 22, 23, 40, 278, 267, 4, 101, 248, 32, 126, 235, 375, 351, 271, 343, 142, 210, 376, 250, 31, 359, 336, 353, 36, 223, 115, 166, 258, 216, 137, 317, 61, 263, 232, 110, 314, 35, 260, 220, 324, 363, 80, 289, 58, 3, 67, 359, 12, 152, 228, 209, 11, 83, 6, 261, 183, 211, 125, 118, 237, 382, 266, 325, 185, 134, 340, 70, 43, 57, 213, 279, 67, 21, 362, 282, 204, 255, 97, 172, 226, 78, 167, 141, 91, 335, 257, 149, 155, 205, 9, 153, 103, 354, 342, 55, 19, 360, 290, 13, 312, 88, 182, 291, 195, 107, 38, 65, 303, 164, 114, 254, 25, 47, 307, 159, 306, 146, 84, 285, 331, 242, 63, 360, 246, 239, 173, 308, 281, 316, 328, 150, 27, 178, 304, 233, 66, 45, 71, 151, 56, 99, 369, 326, 129, 269, 344, 49, 132, 360, 218, 239, 162, 308, 122, 186, 131, 181, 179, 202, 196, 177, 341, 174, 73, 259, 327, 145, 236, 76, 193, 160, 203, 296, 227, 360, 292, 359, 10, 221, 276, 349, 364, 69, 34, 92, 50, 318, 2, 383, 293, 184, 140, 105, 46, 315, 77, 180, 274, 256, 170, 290, 290, 224, 162, 381, 281, 380, 156, 225, 240, 330, 113, 48, 72, 217, 379, 37, 190, 89, 295, 24, 15, 245, 53, 361, 1] + 9: [212, 94, 94, 185, 316, 370, 51, 56, 367, 302, 180, 192, 336, 150, 319, 248, 382, 291, 110, 216, 59, 34, 361, 85, 156, 57, 92, 226, 252, 184, 201, 380, 103, 136, 144, 137, 231, 183, 123, 70, 202, 175, 52, 112, 286, 221, 347, 225, 366, 40, 362, 234, 92, 226, 299, 47, 29, 281, 95, 342, 133, 153, 0, 87, 267, 181, 237, 65, 245, 2, 320, 21, 178, 58, 118, 204, 122, 168, 92, 372, 172, 47, 37, 210, 95, 132, 82, 187, 277, 99, 186, 72, 343, 354, 100, 203, 301, 229, 346, 108, 176, 200, 250, 26, 92, 260, 19, 179, 90, 90, 64, 282, 285, 377, 334, 129, 269, 141, 39, 276, 46, 62, 12, 333, 337, 326, 223, 149, 27, 236, 78, 280, 215, 185, 313, 262, 121, 23, 161, 292, 247, 314, 16, 36, 288, 307, 197, 213, 375, 38, 335, 243, 266, 194, 157, 369, 304, 315, 74, 222, 272, 49, 271, 102, 131, 356, 54, 311, 139, 124, 75, 235, 117, 358, 71, 270, 167, 160, 328, 273, 7, 77, 304, 196, 350, 327, 146, 217, 209, 67, 158, 33, 321, 97, 207, 11, 359, 50, 303, 114, 371, 211, 227, 368, 279, 378, 265, 338, 304, 31, 263, 351, 15, 49, 214, 309, 9, 120, 228, 41, 105, 66, 274, 130, 45, 174, 352, 89, 48, 140, 345, 294, 91, 340, 304, 31, 263, 381, 1, 242, 147, 32, 151, 135, 306, 116, 323, 218, 80, 246, 275, 254, 84, 125, 188, 191, 249, 284, 25, 152, 166, 96, 165, 155, 198, 287, 244, 177, 258, 268, 318, 259, 256, 171, 308, 339, 233, 300, 134, 162, 69, 193, 63, 278, 53, 24, 255, 143, 295, 327, 316, 138, 103, 10, 360, 169, 128, 107, 4, 189, 5, 8, 379, 115, 42, 298, 86, 290, 73, 296, 55, 348, 255, 331, 295, 355, 83, 262, 147, 14, 317, 363, 88, 20, 195, 111, 106, 109, 205, 322, 305, 224, 241, 383, 119, 44, 61, 283, 255, 220, 353, 355, 313, 159, 341, 22, 98, 3, 264, 163, 35, 230, 13, 208, 373, 190, 330, 310, 170, 293, 101, 93, 182, 173, 349, 349, 219, 327, 344, 329, 64, 10, 30, 289, 6, 364, 206, 238, 104, 18, 60, 79, 253, 232, 199, 43, 325, 257, 164, 365, 349, 81, 357, 355, 376, 127, 244, 239, 312, 154, 297, 145, 142, 332, 324, 17, 148, 261, 251, 76, 374, 68, 240, 113, 28, 126] + 10: [294, 221, 5, 337, 164, 8, 241, 269, 272, 186, 231, 250, 72, 60, 256, 71, 354, 163, 113, 131, 217, 16, 120, 227, 381, 324, 192, 48, 141, 286, 146, 301, 138, 213, 115, 345, 190, 0, 341, 81, 26, 379, 184, 50, 65, 326, 224, 34, 94, 10, 89, 290, 255, 108, 5, 216, 73, 134, 114, 237, 136, 342, 359, 382, 214, 106, 357, 258, 162, 183, 298, 62, 305, 133, 273, 277, 70, 23, 194, 319, 320, 167, 309, 144, 330, 201, 243, 49, 212, 199, 116, 314, 329, 278, 42, 13, 185, 100, 86, 168, 207, 352, 166, 176, 194, 319, 24, 365, 57, 54, 263, 211, 11, 284, 306, 128, 33, 142, 378, 97, 247, 261, 370, 139, 373, 251, 83, 143, 43, 29, 264, 319, 320, 18, 2, 170, 40, 154, 312, 377, 249, 155, 87, 122, 20, 37, 193, 317, 299, 173, 187, 35, 347, 150, 188, 68, 264, 319, 320, 18, 92, 195, 351, 304, 252, 148, 376, 132, 129, 85, 9, 371, 64, 323, 364, 111, 372, 165, 344, 361, 228, 283, 55, 181, 320, 69, 331, 215, 182, 58, 67, 356, 360, 229, 46, 288, 197, 383, 174, 45, 209, 222, 206, 6, 104, 63, 110, 349, 55, 319, 5, 279, 164, 310, 177, 308, 353, 368, 77, 90, 21, 75, 112, 267, 39, 275, 226, 80, 355, 245, 172, 156, 236, 82, 300, 268, 121, 325, 101, 178, 78, 99, 259, 208, 271, 266, 105, 56, 59, 25, 12, 239, 244, 350, 292, 203, 118, 260, 160, 295, 300, 265, 262, 126, 31, 22, 210, 91, 76, 348, 189, 145, 233, 335, 339, 15, 318, 123, 282, 302, 119, 322, 333, 191, 270, 369, 300, 265, 289, 365, 102, 1, 153, 47, 234, 374, 223, 149, 135, 311, 327, 41, 254, 95, 175, 287, 17, 158, 281, 152, 238, 358, 300, 137, 24, 367, 31, 310, 19, 380, 7, 363, 38, 32, 3, 232, 328, 74, 332, 140, 169, 235, 127, 4, 157, 84, 28, 366, 246, 151, 121, 365, 225, 179, 171, 124, 130, 316, 280, 285, 103, 27, 315, 109, 340, 343, 220, 375, 147, 200, 44, 30, 242, 297, 180, 93, 5, 286, 146, 301, 66, 248, 293, 274, 98, 198, 321, 219, 338, 230, 125, 313, 257, 96, 205, 276, 334, 14, 52, 336, 180, 151, 5, 216, 79, 161, 117, 196, 240, 204, 218, 61, 107, 296, 346, 51, 36, 362, 307, 159, 303, 53, 88, 202, 253, 291] + 11: [156, 296, 110, 145, 100, 102, 165, 381, 136, 283, 159, 151, 224, 35, 294, 349, 291, 142, 194, 5, 140, 304, 139, 28, 220, 134, 252, 296, 119, 221, 357, 15, 141, 282, 148, 346, 27, 195, 365, 209, 295, 359, 57, 43, 184, 360, 167, 380, 16, 51, 153, 105, 98, 215, 110, 190, 100, 267, 330, 20, 13, 6, 350, 374, 298, 186, 292, 302, 125, 263, 8, 200, 132, 68, 344, 276, 232, 12, 98, 215, 326, 233, 248, 0, 351, 95, 303, 367, 127, 26, 49, 254, 59, 338, 120, 258, 382, 106, 4, 342, 181, 111, 137, 307, 77, 83, 150, 121, 44, 22, 247, 204, 78, 371, 80, 166, 89, 41, 336, 135, 164, 7, 309, 210, 21, 379, 317, 372, 256, 32, 377, 3, 110, 171, 325, 236, 275, 73, 23, 107, 370, 160, 133, 242, 30, 312, 79, 225, 358, 108, 104, 272, 369, 333, 72, 191, 377, 83, 150, 99, 44, 144, 331, 131, 320, 45, 249, 86, 341, 270, 168, 353, 264, 33, 230, 222, 288, 1, 180, 206, 273, 261, 339, 314, 85, 38, 285, 24, 217, 70, 196, 319, 226, 315, 96, 58, 362, 158, 187, 18, 138, 323, 201, 114, 60, 123, 257, 61, 339, 314, 85, 235, 126, 15, 343, 146, 154, 213, 348, 340, 316, 327, 211, 364, 64, 48, 271, 266, 290, 71, 238, 76, 223, 243, 260, 314, 40, 121, 248, 218, 54, 244, 297, 170, 262, 10, 198, 97, 63, 378, 293, 174, 231, 37, 322, 305, 246, 189, 281, 42, 299, 90, 163, 300, 157, 345, 152, 39, 313, 124, 69, 176, 255, 101, 245, 34, 118, 182, 143, 112, 310, 229, 53, 253, 50, 308, 130, 227, 163, 11, 116, 332, 192, 286, 250, 161, 92, 93, 14, 208, 366, 311, 115, 129, 274, 103, 287, 268, 368, 155, 321, 9, 130, 376, 163, 36, 116, 162, 178, 318, 207, 280, 2, 251, 197, 259, 265, 179, 67, 62, 46, 84, 173, 183, 175, 122, 329, 241, 130, 90, 296, 11, 36, 29, 284, 337, 169, 66, 117, 56, 203, 301, 219, 193, 234, 355, 19, 375, 214, 278, 361, 82, 212, 335, 88, 87, 149, 235, 357, 285, 128, 177, 17, 94, 373, 172, 289, 81, 147, 354, 383, 113, 237, 306, 185, 277, 216, 239, 334, 269, 363, 87, 65, 221, 126, 199, 52, 228, 205, 47, 75, 347, 55, 31, 279, 91, 324, 188, 74, 356, 240, 328, 352, 202, 25, 109] + 12: [156, 28, 17, 349, 172, 198, 208, 68, 229, 102, 271, 301, 155, 114, 138, 161, 351, 5, 179, 106, 178, 320, 184, 100, 359, 302, 255, 158, 190, 296, 333, 50, 75, 317, 133, 27, 308, 80, 358, 41, 189, 58, 335, 12, 329, 251, 34, 54, 118, 221, 324, 272, 255, 158, 190, 378, 333, 90, 3, 152, 10, 288, 294, 337, 309, 191, 224, 169, 11, 305, 374, 352, 18, 212, 132, 242, 64, 366, 255, 219, 327, 69, 46, 341, 278, 276, 266, 300, 362, 380, 142, 77, 298, 232, 154, 183, 246, 285, 55, 81, 145, 360, 239, 222, 255, 226, 344, 378, 203, 290, 195, 215, 16, 265, 241, 326, 370, 200, 119, 319, 76, 13, 207, 146, 345, 19, 182, 248, 141, 199, 255, 226, 327, 289, 203, 97, 6, 330, 277, 39, 197, 318, 357, 1, 45, 325, 122, 65, 217, 24, 96, 72, 307, 275, 166, 220, 204, 78, 127, 177, 287, 192, 134, 348, 108, 371, 57, 59, 304, 185, 61, 356, 115, 32, 365, 30, 47, 299, 338, 163, 331, 73, 204, 78, 127, 177, 103, 341, 339, 268, 364, 153, 164, 31, 15, 231, 313, 139, 40, 379, 279, 274, 131, 125, 196, 311, 377, 42, 204, 158, 327, 69, 46, 98, 260, 137, 25, 209, 129, 261, 227, 188, 67, 249, 159, 293, 297, 70, 316, 367, 26, 202, 240, 53, 363, 245, 175, 223, 126, 51, 259, 346, 89, 2, 180, 23, 193, 254, 29, 37, 235, 86, 186, 252, 8, 258, 210, 88, 92, 372, 49, 130, 167, 82, 172, 306, 354, 382, 147, 33, 135, 291, 283, 315, 262, 280, 36, 140, 312, 105, 375, 230, 79, 206, 21, 110, 149, 130, 303, 223, 71, 151, 123, 216, 323, 264, 322, 91, 144, 234, 256, 4, 121, 334, 128, 174, 343, 247, 143, 213, 95, 225, 149, 130, 303, 82, 150, 361, 176, 332, 87, 171, 104, 74, 201, 52, 83, 187, 355, 112, 284, 310, 93, 117, 94, 228, 353, 162, 35, 381, 168, 292, 243, 233, 295, 194, 60, 116, 107, 157, 253, 350, 84, 173, 38, 20, 148, 383, 347, 237, 269, 314, 56, 250, 48, 120, 267, 22, 243, 369, 170, 205, 218, 160, 63, 238, 376, 9, 165, 321, 214, 257, 263, 85, 113, 236, 111, 342, 66, 43, 120, 120, 109, 124, 150, 181, 340, 281, 368, 101, 273, 14, 286, 0, 62, 136, 270, 336, 244, 44, 282, 211, 7, 328, 99, 373] + 13: [80, 114, 23, 280, 231, 186, 142, 58, 112, 292, 2, 219, 196, 166, 34, 106, 56, 308, 115, 365, 4, 60, 347, 368, 277, 7, 80, 96, 23, 169, 238, 102, 335, 279, 275, 294, 210, 355, 107, 359, 331, 357, 59, 263, 217, 61, 138, 191, 251, 269, 165, 27, 9, 114, 328, 51, 276, 310, 162, 323, 273, 229, 42, 168, 283, 374, 105, 326, 296, 381, 19, 235, 383, 117, 258, 200, 82, 140, 44, 18, 328, 39, 12, 254, 304, 295, 297, 143, 98, 179, 26, 327, 221, 89, 242, 133, 123, 190, 372, 284, 97, 349, 314, 33, 44, 3, 87, 41, 95, 134, 88, 184, 364, 363, 122, 237, 145, 129, 380, 288, 305, 250, 360, 214, 85, 343, 293, 132, 66, 315, 362, 18, 149, 260, 209, 271, 199, 227, 31, 40, 147, 240, 291, 139, 222, 48, 92, 348, 213, 124, 157, 1, 90, 156, 120, 350, 362, 18, 149, 260, 12, 274, 109, 35, 246, 158, 300, 211, 361, 93, 8, 379, 127, 325, 234, 313, 180, 252, 224, 354, 218, 32, 194, 81, 87, 282, 342, 318, 298, 198, 16, 67, 178, 339, 225, 333, 223, 248, 136, 53, 38, 287, 346, 101, 152, 148, 367, 175, 329, 232, 195, 172, 55, 17, 370, 131, 356, 54, 207, 43, 344, 29, 52, 68, 319, 94, 73, 5, 233, 128, 264, 302, 351, 155, 329, 232, 87, 265, 30, 30, 116, 307, 202, 261, 22, 79, 262, 183, 76, 267, 268, 245, 358, 353, 185, 28, 278, 255, 322, 204, 137, 232, 195, 24, 189, 318, 46, 45, 69, 309, 65, 10, 74, 253, 303, 153, 257, 244, 377, 130, 182, 63, 99, 341, 324, 230, 77, 57, 14, 21, 201, 366, 338, 176, 289, 91, 378, 382, 144, 243, 164, 337, 174, 71, 154, 249, 226, 336, 371, 272, 206, 216, 170, 170, 195, 312, 238, 197, 11, 49, 75, 334, 373, 321, 239, 146, 15, 36, 220, 187, 70, 119, 205, 212, 173, 290, 193, 316, 170, 108, 14, 135, 189, 330, 306, 111, 188, 266, 208, 62, 25, 177, 241, 161, 345, 311, 72, 103, 285, 352, 181, 47, 160, 78, 170, 108, 0, 280, 100, 274, 159, 118, 376, 281, 6, 375, 121, 151, 37, 286, 84, 113, 13, 150, 167, 203, 20, 332, 301, 126, 170, 108, 86, 312, 110, 340, 171, 64, 369, 320, 299, 141, 228, 256, 125, 163, 259, 50, 270, 104, 83, 215, 317, 236, 247, 192] + 14: [147, 345, 255, 239, 378, 155, 39, 154, 197, 297, 370, 382, 203, 90, 225, 115, 248, 10, 153, 19, 45, 57, 381, 340, 53, 315, 70, 204, 60, 344, 337, 71, 231, 271, 294, 328, 291, 40, 325, 129, 330, 148, 43, 106, 309, 16, 263, 24, 181, 192, 272, 174, 70, 345, 8, 75, 327, 254, 52, 250, 216, 123, 288, 169, 286, 156, 298, 12, 1, 173, 338, 361, 194, 293, 252, 27, 210, 73, 56, 204, 345, 2, 348, 308, 64, 269, 220, 163, 357, 187, 102, 221, 276, 191, 217, 262, 36, 20, 312, 180, 81, 113, 67, 331, 160, 77, 110, 75, 118, 142, 177, 374, 125, 149, 201, 301, 373, 5, 0, 89, 224, 321, 103, 290, 7, 66, 283, 131, 29, 212, 160, 164, 255, 186, 355, 23, 117, 79, 13, 59, 277, 167, 109, 101, 253, 168, 96, 311, 171, 9, 80, 62, 144, 91, 119, 121, 160, 287, 137, 259, 150, 141, 95, 44, 265, 215, 369, 104, 172, 241, 200, 300, 251, 28, 278, 58, 299, 139, 11, 188, 130, 289, 229, 15, 110, 122, 327, 151, 198, 354, 166, 135, 326, 339, 245, 347, 284, 242, 244, 273, 46, 304, 322, 120, 140, 170, 98, 318, 229, 15, 383, 2, 378, 155, 342, 182, 157, 69, 184, 22, 227, 82, 316, 202, 240, 94, 257, 305, 47, 362, 307, 280, 38, 249, 229, 76, 255, 34, 355, 275, 213, 317, 33, 246, 332, 346, 349, 105, 176, 282, 376, 310, 375, 230, 281, 84, 233, 72, 196, 165, 229, 76, 17, 34, 126, 99, 124, 111, 41, 343, 319, 108, 234, 48, 341, 364, 42, 285, 74, 78, 18, 55, 320, 303, 218, 295, 211, 4, 175, 209, 126, 228, 32, 279, 270, 161, 193, 333, 256, 377, 306, 264, 63, 114, 127, 136, 87, 92, 236, 368, 232, 83, 211, 372, 30, 259, 122, 85, 100, 61, 314, 323, 324, 133, 128, 237, 359, 313, 93, 214, 267, 238, 179, 185, 247, 54, 352, 50, 274, 207, 30, 209, 353, 275, 371, 162, 158, 25, 145, 37, 296, 380, 86, 152, 350, 302, 358, 190, 258, 35, 107, 51, 138, 88, 143, 261, 112, 268, 146, 71, 3, 21, 26, 206, 68, 266, 178, 116, 336, 223, 183, 6, 365, 14, 132, 367, 219, 134, 292, 205, 143, 4, 199, 49, 308, 235, 97, 360, 208, 379, 159, 363, 226, 222, 189, 335, 356, 329, 195, 334, 31, 366, 65, 243, 351, 260] + 15: [68, 291, 39, 324, 104, 333, 83, 163, 164, 379, 194, 117, 77, 165, 141, 308, 37, 92, 364, 188, 312, 181, 44, 243, 6, 131, 298, 11, 160, 185, 159, 119, 139, 311, 111, 357, 5, 279, 229, 8, 271, 234, 216, 213, 240, 129, 370, 228, 178, 288, 144, 322, 0, 11, 57, 332, 217, 215, 23, 342, 143, 225, 238, 2, 268, 206, 301, 289, 45, 172, 190, 369, 280, 327, 110, 336, 348, 281, 0, 11, 57, 231, 266, 284, 16, 173, 177, 36, 372, 142, 87, 43, 30, 97, 169, 337, 65, 73, 149, 66, 179, 122, 46, 366, 0, 11, 362, 155, 148, 90, 360, 256, 138, 207, 49, 59, 22, 205, 349, 381, 383, 95, 218, 184, 201, 98, 299, 302, 152, 127, 290, 130, 362, 200, 94, 60, 329, 113, 109, 338, 323, 212, 62, 50, 107, 321, 64, 26, 147, 161, 176, 58, 346, 219, 19, 192, 15, 130, 359, 258, 128, 203, 376, 277, 241, 335, 182, 105, 331, 341, 339, 365, 272, 358, 3, 378, 310, 7, 70, 125, 245, 276, 15, 130, 359, 258, 128, 146, 151, 193, 320, 47, 78, 166, 10, 13, 382, 137, 244, 265, 52, 153, 126, 330, 41, 287, 25, 325, 15, 130, 371, 200, 134, 134, 40, 174, 18, 351, 209, 102, 114, 253, 313, 354, 71, 283, 210, 197, 187, 124, 17, 286, 75, 198, 261, 367, 359, 326, 175, 121, 157, 168, 242, 100, 133, 145, 96, 363, 118, 347, 123, 375, 79, 91, 140, 361, 103, 21, 315, 85, 340, 295, 160, 355, 217, 9, 317, 344, 255, 319, 170, 101, 150, 233, 226, 53, 314, 63, 116, 374, 293, 112, 120, 156, 249, 328, 350, 202, 199, 355, 94, 20, 221, 296, 196, 285, 305, 263, 269, 250, 306, 72, 55, 316, 54, 303, 318, 136, 223, 214, 27, 260, 76, 259, 251, 324, 236, 74, 56, 1, 115, 352, 247, 86, 343, 12, 377, 99, 208, 257, 273, 158, 14, 292, 307, 220, 132, 356, 4, 227, 251, 189, 104, 252, 48, 135, 24, 31, 42, 82, 309, 81, 191, 61, 204, 84, 167, 32, 262, 195, 154, 108, 264, 368, 88, 88, 300, 38, 148, 237, 80, 35, 235, 162, 246, 248, 34, 254, 294, 267, 224, 334, 278, 89, 29, 222, 183, 345, 186, 270, 88, 227, 160, 155, 175, 51, 232, 353, 230, 106, 28, 275, 282, 69, 67, 180, 211, 373, 239, 274, 380, 171, 304, 297, 33, 93] + 16: [11, 120, 300, 165, 318, 250, 383, 347, 52, 115, 80, 8, 232, 370, 104, 102, 272, 109, 88, 47, 50, 138, 12, 326, 13, 94, 162, 120, 120, 277, 259, 255, 283, 246, 84, 97, 139, 76, 36, 322, 264, 57, 343, 360, 172, 26, 170, 101, 349, 222, 133, 366, 100, 114, 43, 86, 203, 183, 187, 16, 310, 316, 0, 41, 42, 339, 29, 25, 105, 196, 157, 145, 176, 230, 23, 341, 239, 344, 100, 154, 262, 147, 118, 205, 213, 199, 27, 82, 153, 311, 301, 238, 354, 214, 253, 207, 166, 218, 241, 243, 40, 350, 320, 3, 140, 331, 43, 273, 342, 271, 236, 108, 237, 248, 66, 184, 334, 160, 336, 208, 83, 244, 284, 131, 116, 260, 135, 173, 361, 148, 140, 185, 275, 90, 280, 348, 235, 85, 71, 59, 119, 31, 293, 9, 151, 60, 93, 48, 81, 20, 359, 169, 6, 263, 161, 368, 134, 152, 92, 269, 303, 302, 287, 367, 365, 377, 281, 227, 379, 96, 221, 200, 55, 190, 128, 17, 308, 28, 63, 72, 325, 257, 134, 67, 92, 147, 118, 5, 295, 180, 306, 335, 73, 195, 357, 124, 290, 178, 223, 91, 14, 324, 150, 61, 351, 278, 345, 98, 134, 330, 275, 64, 44, 171, 298, 146, 65, 292, 39, 231, 372, 297, 375, 313, 304, 142, 127, 107, 77, 210, 112, 327, 69, 111, 134, 125, 234, 49, 44, 24, 209, 270, 294, 37, 159, 380, 19, 137, 228, 158, 319, 193, 163, 113, 371, 32, 355, 179, 376, 353, 15, 286, 7, 165, 75, 271, 46, 89, 328, 38, 155, 34, 229, 4, 198, 211, 333, 266, 373, 261, 332, 251, 74, 299, 215, 126, 15, 369, 312, 144, 280, 374, 364, 346, 177, 307, 103, 182, 202, 276, 340, 132, 378, 30, 62, 268, 149, 245, 130, 136, 296, 249, 15, 369, 7, 338, 75, 78, 265, 123, 285, 99, 56, 305, 122, 22, 252, 18, 267, 201, 233, 168, 217, 288, 254, 192, 362, 156, 15, 152, 234, 64, 143, 224, 121, 226, 314, 51, 58, 70, 220, 188, 317, 242, 337, 240, 315, 258, 256, 352, 329, 197, 10, 247, 68, 286, 262, 269, 289, 274, 189, 87, 181, 21, 282, 219, 45, 174, 358, 54, 323, 194, 381, 117, 279, 33, 164, 204, 53, 309, 286, 286, 275, 129, 303, 35, 141, 212, 321, 2, 175, 167, 356, 106, 225, 95, 206, 363, 186, 110, 216, 382, 79, 291, 191, 1] + 17: [374, 236, 337, 375, 26, 362, 351, 56, 207, 373, 10, 70, 304, 41, 221, 342, 238, 90, 308, 63, 53, 277, 291, 111, 54, 202, 261, 236, 79, 200, 225, 305, 6, 52, 296, 380, 272, 142, 332, 295, 3, 309, 310, 217, 106, 318, 316, 201, 160, 247, 174, 237, 261, 236, 79, 375, 100, 185, 125, 290, 194, 228, 98, 168, 248, 7, 196, 89, 93, 161, 151, 344, 355, 66, 257, 68, 156, 283, 28, 284, 319, 78, 383, 343, 324, 167, 46, 145, 33, 292, 116, 331, 365, 271, 240, 345, 55, 350, 227, 23, 129, 334, 122, 121, 212, 284, 190, 87, 275, 330, 328, 157, 134, 180, 99, 9, 166, 260, 204, 95, 300, 183, 274, 31, 164, 224, 4, 35, 293, 251, 138, 284, 137, 108, 197, 305, 133, 339, 39, 92, 139, 171, 13, 42, 49, 27, 110, 102, 278, 50, 83, 149, 285, 348, 127, 152, 131, 140, 346, 254, 155, 232, 189, 8, 335, 313, 256, 245, 47, 147, 118, 44, 379, 299, 124, 67, 132, 40, 288, 353, 358, 159, 131, 188, 303, 15, 311, 114, 1, 276, 24, 231, 135, 214, 36, 269, 216, 265, 94, 258, 369, 150, 153, 230, 11, 77, 252, 320, 131, 188, 303, 15, 136, 279, 246, 75, 206, 177, 327, 263, 60, 45, 382, 338, 262, 16, 64, 57, 163, 223, 349, 321, 301, 325, 131, 284, 226, 108, 5, 286, 297, 222, 73, 211, 371, 376, 264, 363, 187, 37, 302, 112, 128, 233, 65, 30, 229, 322, 14, 69, 220, 250, 79, 375, 26, 294, 347, 172, 22, 148, 74, 165, 19, 59, 186, 341, 361, 287, 25, 192, 88, 280, 306, 281, 209, 312, 220, 250, 337, 12, 193, 294, 2, 104, 96, 103, 84, 314, 61, 173, 115, 253, 195, 210, 48, 270, 182, 378, 71, 289, 86, 72, 119, 307, 366, 200, 359, 362, 29, 234, 97, 381, 146, 282, 20, 43, 198, 181, 249, 120, 372, 76, 241, 141, 178, 0, 333, 354, 119, 80, 126, 357, 117, 323, 113, 370, 218, 243, 367, 259, 82, 215, 205, 101, 244, 85, 267, 144, 32, 109, 184, 51, 34, 170, 119, 105, 58, 219, 383, 266, 130, 175, 199, 38, 143, 18, 340, 154, 81, 21, 179, 239, 326, 364, 315, 176, 242, 17, 123, 235, 119, 208, 337, 91, 100, 330, 62, 107, 317, 273, 298, 162, 191, 336, 377, 213, 360, 329, 158, 255, 352, 356, 368, 268, 169, 203] + 18: [208, 16, 320, 65, 110, 167, 306, 266, 165, 191, 87, 263, 361, 140, 264, 265, 214, 213, 64, 229, 172, 238, 11, 21, 120, 382, 208, 210, 320, 215, 106, 239, 282, 190, 240, 23, 262, 129, 350, 203, 178, 182, 112, 303, 216, 369, 84, 37, 377, 198, 374, 147, 6, 333, 286, 119, 71, 236, 291, 378, 181, 20, 104, 109, 201, 102, 96, 363, 179, 275, 338, 123, 298, 281, 277, 252, 52, 183, 118, 53, 251, 57, 228, 189, 357, 99, 274, 365, 359, 241, 288, 195, 244, 255, 324, 358, 247, 163, 95, 90, 230, 194, 55, 293, 356, 45, 251, 57, 228, 142, 151, 341, 164, 346, 331, 0, 278, 176, 205, 63, 94, 234, 259, 197, 107, 115, 249, 221, 91, 314, 30, 271, 217, 300, 106, 355, 284, 35, 318, 371, 121, 367, 58, 157, 93, 267, 254, 127, 200, 381, 42, 370, 117, 72, 186, 32, 30, 271, 217, 300, 185, 136, 31, 330, 19, 366, 83, 135, 122, 308, 279, 145, 287, 168, 27, 209, 86, 44, 14, 310, 80, 227, 30, 271, 257, 28, 169, 26, 139, 326, 98, 342, 233, 334, 50, 61, 276, 187, 322, 175, 66, 17, 2, 29, 24, 323, 97, 242, 30, 271, 257, 347, 185, 321, 360, 3, 103, 248, 343, 131, 250, 39, 319, 292, 153, 60, 4, 56, 180, 184, 345, 196, 353, 309, 30, 271, 154, 113, 380, 301, 34, 101, 89, 41, 177, 137, 7, 15, 125, 12, 351, 82, 158, 317, 354, 352, 160, 173, 226, 59, 30, 193, 211, 65, 260, 68, 376, 105, 204, 339, 149, 296, 159, 349, 54, 132, 270, 143, 220, 133, 364, 294, 290, 336, 74, 222, 47, 271, 207, 245, 260, 25, 373, 67, 269, 337, 307, 22, 231, 289, 9, 335, 92, 108, 332, 76, 218, 280, 212, 206, 327, 78, 47, 271, 315, 245, 368, 146, 237, 77, 188, 141, 79, 88, 313, 152, 81, 13, 148, 256, 51, 126, 312, 273, 5, 62, 379, 340, 10, 43, 69, 304, 138, 192, 144, 33, 124, 73, 223, 316, 328, 235, 362, 329, 166, 48, 170, 375, 372, 299, 18, 199, 156, 1, 10, 271, 69, 114, 71, 116, 253, 246, 46, 171, 283, 162, 232, 8, 150, 155, 243, 348, 219, 130, 75, 85, 100, 134, 49, 268, 272, 43, 251, 36, 138, 174, 297, 285, 70, 202, 261, 225, 302, 111, 38, 40, 325, 383, 311, 224, 258, 161, 305, 295, 128, 344] + 19: [187, 265, 57, 338, 300, 323, 20, 79, 46, 352, 83, 272, 297, 325, 177, 356, 383, 137, 99, 28, 348, 100, 0, 169, 229, 98, 187, 90, 61, 330, 101, 184, 33, 217, 292, 134, 182, 170, 2, 346, 108, 372, 326, 22, 232, 73, 118, 252, 236, 360, 350, 198, 187, 90, 57, 288, 197, 27, 218, 9, 25, 278, 121, 251, 249, 374, 262, 47, 376, 32, 258, 294, 381, 304, 248, 17, 142, 245, 141, 302, 364, 313, 207, 216, 77, 230, 246, 116, 270, 368, 277, 203, 131, 75, 362, 87, 186, 351, 123, 178, 344, 104, 316, 369, 243, 265, 90, 161, 312, 307, 334, 153, 122, 233, 359, 204, 40, 18, 213, 26, 309, 320, 14, 149, 210, 358, 102, 151, 250, 268, 190, 7, 377, 279, 42, 64, 227, 354, 54, 71, 228, 221, 144, 273, 380, 129, 281, 139, 105, 63, 357, 136, 5, 226, 305, 107, 190, 7, 241, 126, 42, 373, 89, 152, 335, 1, 172, 29, 215, 287, 85, 148, 200, 209, 208, 94, 106, 224, 56, 282, 317, 133, 190, 7, 52, 322, 207, 114, 271, 159, 135, 36, 76, 353, 110, 306, 45, 3, 191, 168, 370, 103, 112, 367, 379, 72, 196, 263, 211, 74, 377, 183, 318, 115, 38, 58, 147, 128, 173, 16, 347, 336, 237, 65, 225, 212, 234, 371, 162, 185, 86, 199, 378, 375, 366, 48, 120, 338, 240, 328, 193, 117, 53, 285, 6, 283, 4, 95, 10, 331, 242, 176, 174, 286, 267, 39, 155, 382, 12, 34, 366, 256, 91, 291, 8, 115, 164, 264, 158, 295, 337, 257, 157, 125, 333, 62, 140, 341, 235, 314, 269, 214, 205, 55, 274, 319, 366, 256, 120, 288, 240, 363, 154, 44, 365, 310, 167, 49, 259, 143, 80, 60, 329, 343, 132, 324, 289, 223, 299, 78, 276, 355, 24, 339, 288, 180, 192, 307, 303, 189, 41, 166, 165, 138, 260, 194, 19, 160, 119, 311, 332, 37, 150, 21, 163, 220, 93, 50, 24, 111, 188, 180, 219, 64, 97, 301, 238, 30, 96, 195, 67, 342, 202, 293, 321, 82, 181, 59, 231, 51, 266, 327, 109, 81, 24, 179, 255, 313, 312, 66, 296, 275, 15, 13, 127, 239, 345, 84, 284, 92, 88, 298, 247, 315, 11, 201, 261, 43, 145, 253, 24, 206, 70, 330, 101, 113, 349, 35, 171, 222, 68, 156, 31, 244, 23, 280, 146, 130, 340, 361, 124, 290, 308, 69, 175, 254] + 20: [116, 82, 99, 112, 307, 162, 158, 135, 101, 322, 378, 58, 309, 340, 338, 39, 212, 11, 156, 323, 354, 147, 251, 17, 117, 179, 116, 144, 153, 268, 379, 93, 228, 362, 172, 13, 240, 367, 22, 238, 232, 106, 326, 154, 283, 333, 19, 64, 220, 288, 167, 168, 116, 144, 99, 369, 282, 32, 221, 140, 216, 74, 348, 0, 279, 345, 15, 304, 75, 159, 319, 7, 225, 295, 306, 143, 277, 331, 116, 66, 193, 145, 291, 138, 111, 381, 358, 104, 375, 219, 34, 131, 310, 73, 337, 151, 352, 341, 47, 383, 155, 244, 181, 189, 334, 317, 287, 369, 282, 62, 273, 50, 258, 126, 343, 30, 233, 169, 294, 202, 191, 366, 250, 329, 276, 293, 31, 203, 52, 54, 334, 55, 153, 45, 335, 330, 164, 259, 342, 177, 194, 359, 178, 346, 5, 380, 84, 217, 21, 40, 63, 133, 23, 255, 200, 206, 334, 82, 110, 268, 4, 371, 141, 96, 92, 176, 284, 368, 150, 243, 20, 365, 230, 254, 88, 321, 313, 373, 152, 237, 61, 76, 8, 161, 153, 45, 130, 188, 10, 286, 132, 6, 218, 146, 208, 165, 29, 170, 253, 210, 289, 355, 211, 245, 301, 109, 260, 303, 122, 161, 123, 226, 4, 325, 91, 382, 71, 77, 26, 36, 372, 86, 370, 231, 223, 316, 120, 113, 157, 134, 103, 298, 214, 262, 37, 161, 193, 376, 353, 363, 125, 180, 49, 182, 41, 118, 256, 274, 252, 127, 327, 56, 59, 163, 67, 68, 129, 1, 198, 263, 278, 161, 98, 87, 224, 105, 128, 264, 184, 315, 361, 314, 247, 246, 248, 257, 351, 308, 124, 28, 16, 48, 192, 269, 107, 137, 278, 161, 98, 2, 46, 270, 209, 160, 320, 90, 35, 265, 42, 60, 12, 236, 360, 80, 142, 205, 72, 108, 222, 44, 242, 281, 174, 161, 98, 87, 195, 204, 95, 187, 65, 239, 267, 53, 119, 215, 377, 3, 24, 139, 199, 271, 136, 183, 275, 69, 249, 299, 302, 272, 173, 43, 339, 148, 324, 227, 292, 357, 328, 25, 300, 97, 297, 201, 89, 285, 79, 85, 305, 70, 149, 171, 280, 350, 302, 14, 66, 145, 335, 318, 229, 311, 121, 190, 235, 175, 114, 196, 100, 332, 374, 349, 115, 102, 83, 261, 344, 197, 27, 51, 336, 336, 81, 43, 46, 185, 290, 9, 347, 57, 364, 213, 18, 166, 33, 78, 312, 94, 296, 356, 207, 234, 186, 38, 241, 266] + 21: [152, 173, 132, 210, 112, 226, 150, 147, 196, 141, 234, 352, 39, 159, 125, 237, 24, 274, 134, 67, 268, 324, 228, 59, 224, 123, 152, 173, 132, 82, 257, 345, 252, 351, 175, 72, 245, 336, 262, 260, 30, 77, 269, 216, 81, 310, 201, 37, 176, 212, 311, 10, 152, 173, 132, 200, 257, 94, 238, 19, 15, 135, 325, 338, 90, 383, 290, 266, 14, 340, 110, 195, 374, 56, 249, 222, 327, 17, 302, 247, 158, 50, 208, 363, 51, 349, 366, 230, 288, 267, 304, 312, 154, 204, 102, 36, 215, 342, 207, 180, 255, 231, 332, 35, 316, 47, 29, 80, 364, 361, 149, 184, 280, 189, 368, 358, 259, 303, 370, 295, 203, 329, 381, 163, 41, 85, 265, 166, 167, 108, 316, 47, 86, 80, 140, 277, 284, 380, 344, 221, 33, 49, 309, 356, 165, 115, 187, 258, 27, 181, 74, 99, 285, 188, 283, 179, 20, 46, 281, 124, 71, 232, 91, 191, 69, 220, 243, 63, 287, 28, 317, 40, 34, 146, 271, 96, 330, 54, 162, 61, 272, 264, 20, 47, 281, 103, 113, 88, 279, 128, 8, 313, 89, 372, 318, 251, 133, 171, 292, 346, 305, 315, 122, 114, 6, 326, 297, 369, 289, 236, 281, 119, 218, 199, 177, 151, 25, 9, 131, 168, 362, 192, 354, 100, 160, 186, 334, 105, 13, 55, 242, 116, 127, 44, 371, 4, 205, 376, 119, 45, 118, 172, 194, 144, 233, 263, 246, 62, 5, 301, 104, 57, 83, 319, 2, 129, 31, 161, 248, 321, 371, 4, 360, 38, 113, 120, 178, 365, 299, 209, 270, 197, 138, 286, 155, 275, 198, 261, 213, 241, 143, 217, 142, 278, 322, 339, 308, 254, 158, 53, 140, 169, 320, 109, 219, 43, 174, 73, 139, 98, 323, 193, 337, 335, 157, 12, 239, 314, 296, 282, 11, 42, 308, 256, 360, 376, 92, 214, 182, 307, 227, 367, 373, 18, 359, 291, 211, 353, 79, 48, 244, 253, 156, 357, 145, 298, 343, 350, 153, 254, 360, 124, 71, 348, 250, 16, 293, 52, 229, 84, 64, 21, 347, 148, 136, 276, 273, 164, 70, 68, 328, 95, 101, 58, 355, 355, 360, 103, 87, 117, 240, 26, 121, 111, 183, 1, 223, 66, 378, 341, 137, 300, 379, 294, 75, 76, 202, 32, 107, 93, 355, 190, 206, 103, 185, 377, 0, 22, 225, 23, 78, 331, 60, 97, 3, 65, 170, 126, 333, 382, 130, 375, 106, 7, 235, 306] + 22: [371, 260, 276, 224, 183, 235, 61, 111, 375, 259, 14, 339, 225, 367, 22, 11, 136, 46, 137, 334, 128, 255, 158, 166, 205, 154, 254, 221, 261, 38, 20, 139, 317, 357, 161, 264, 347, 82, 72, 1, 65, 241, 314, 265, 266, 91, 257, 326, 66, 107, 142, 101, 254, 221, 261, 224, 183, 29, 355, 311, 195, 17, 170, 191, 165, 39, 301, 45, 13, 298, 186, 196, 344, 354, 146, 121, 192, 143, 254, 221, 330, 42, 73, 74, 252, 272, 83, 123, 3, 48, 99, 320, 236, 335, 89, 353, 160, 129, 168, 6, 43, 86, 315, 346, 254, 34, 261, 115, 376, 144, 229, 338, 306, 185, 122, 63, 15, 159, 269, 0, 286, 103, 232, 152, 90, 31, 197, 194, 325, 359, 316, 181, 328, 207, 184, 169, 333, 153, 8, 312, 109, 49, 233, 318, 283, 81, 290, 279, 119, 62, 104, 370, 114, 270, 360, 310, 281, 181, 24, 147, 94, 244, 362, 70, 372, 222, 228, 340, 210, 239, 155, 78, 215, 364, 282, 120, 291, 23, 21, 56, 313, 267, 281, 181, 24, 132, 110, 263, 240, 351, 174, 365, 358, 258, 59, 47, 200, 238, 77, 275, 102, 36, 19, 363, 84, 245, 323, 307, 33, 209, 148, 328, 230, 93, 130, 381, 68, 60, 175, 156, 12, 105, 322, 373, 271, 237, 219, 179, 171, 150, 213, 203, 10, 79, 216, 331, 148, 328, 94, 44, 117, 218, 336, 116, 57, 220, 309, 305, 124, 32, 300, 108, 246, 190, 302, 223, 304, 231, 135, 95, 216, 209, 141, 288, 20, 118, 58, 345, 378, 106, 113, 40, 321, 341, 189, 299, 138, 208, 274, 134, 342, 5, 140, 234, 249, 157, 204, 278, 141, 288, 201, 329, 383, 251, 67, 193, 226, 18, 324, 98, 377, 180, 97, 176, 187, 145, 69, 308, 172, 293, 2, 28, 51, 278, 141, 337, 110, 37, 348, 188, 100, 27, 349, 374, 71, 248, 54, 256, 273, 379, 151, 52, 350, 127, 284, 297, 280, 217, 198, 332, 141, 337, 376, 112, 352, 7, 80, 369, 361, 366, 41, 380, 199, 327, 96, 356, 182, 92, 16, 88, 163, 167, 287, 247, 9, 343, 260, 164, 178, 4, 55, 285, 173, 214, 206, 25, 131, 87, 227, 268, 202, 26, 35, 294, 212, 296, 277, 243, 289, 250, 126, 343, 260, 242, 178, 177, 50, 149, 76, 125, 75, 53, 133, 319, 303, 382, 211, 262, 85, 30, 64, 162, 295, 368, 292, 253] + 23: [26, 275, 305, 244, 324, 103, 372, 86, 326, 312, 228, 359, 160, 231, 76, 258, 108, 14, 336, 202, 319, 247, 318, 198, 186, 37, 26, 274, 305, 269, 12, 176, 304, 371, 184, 74, 317, 136, 98, 301, 204, 143, 16, 243, 375, 308, 277, 238, 345, 112, 18, 189, 26, 120, 132, 167, 72, 287, 11, 41, 340, 271, 224, 221, 17, 53, 284, 73, 107, 294, 373, 140, 253, 181, 20, 83, 4, 168, 26, 71, 19, 244, 323, 84, 180, 283, 133, 149, 38, 215, 365, 137, 250, 109, 335, 285, 33, 367, 62, 259, 22, 332, 110, 75, 126, 275, 132, 306, 203, 94, 192, 257, 50, 378, 313, 370, 1, 307, 162, 280, 212, 122, 97, 46, 310, 360, 364, 60, 296, 81, 126, 275, 305, 24, 252, 27, 169, 315, 111, 102, 95, 361, 185, 178, 382, 200, 164, 362, 267, 161, 357, 381, 118, 380, 13, 291, 328, 314, 77, 240, 252, 261, 374, 130, 35, 233, 237, 44, 32, 321, 223, 128, 302, 99, 369, 61, 158, 334, 208, 145, 251, 242, 328, 314, 48, 173, 85, 154, 127, 263, 309, 151, 272, 311, 329, 65, 193, 45, 68, 163, 338, 79, 34, 377, 30, 183, 216, 288, 281, 314, 100, 87, 323, 156, 241, 56, 213, 219, 152, 157, 150, 207, 330, 344, 21, 114, 239, 188, 368, 299, 343, 262, 29, 39, 225, 314, 100, 295, 135, 47, 182, 78, 144, 70, 195, 222, 153, 325, 196, 54, 134, 354, 25, 106, 31, 322, 42, 246, 15, 8, 264, 172, 211, 187, 300, 279, 218, 6, 131, 191, 342, 210, 119, 333, 341, 0, 206, 155, 115, 260, 266, 337, 63, 105, 3, 235, 264, 172, 147, 331, 85, 232, 67, 92, 209, 91, 40, 292, 254, 52, 352, 58, 174, 297, 355, 49, 353, 171, 138, 217, 226, 265, 124, 175, 170, 201, 135, 177, 57, 148, 80, 220, 23, 101, 298, 179, 347, 256, 346, 268, 139, 214, 276, 165, 289, 293, 366, 64, 124, 69, 376, 167, 197, 177, 55, 270, 146, 249, 10, 255, 66, 125, 303, 90, 7, 121, 123, 205, 88, 350, 117, 89, 113, 286, 124, 172, 71, 82, 300, 327, 2, 351, 379, 36, 194, 129, 93, 273, 348, 363, 245, 236, 227, 282, 9, 116, 141, 142, 166, 43, 51, 175, 71, 187, 72, 358, 59, 159, 190, 229, 290, 356, 96, 5, 316, 339, 28, 349, 104, 248, 230, 383, 278, 199, 234, 320] + 24: [0, 20, 210, 223, 112, 104, 31, 35, 318, 39, 224, 376, 168, 51, 65, 365, 88, 107, 57, 217, 236, 143, 289, 322, 127, 251, 0, 20, 122, 223, 128, 301, 137, 61, 308, 350, 3, 287, 233, 197, 302, 4, 38, 126, 331, 234, 256, 98, 141, 330, 129, 327, 290, 20, 172, 263, 286, 266, 135, 277, 93, 351, 12, 272, 295, 222, 292, 275, 150, 212, 19, 55, 132, 240, 167, 69, 219, 379, 97, 6, 259, 139, 189, 334, 202, 344, 106, 377, 245, 58, 169, 268, 134, 37, 348, 94, 136, 342, 125, 229, 243, 17, 45, 298, 52, 95, 372, 105, 70, 370, 26, 54, 131, 46, 77, 221, 257, 56, 215, 218, 79, 151, 281, 44, 362, 332, 227, 214, 89, 314, 52, 6, 372, 28, 361, 207, 237, 91, 260, 138, 310, 297, 84, 242, 60, 258, 176, 161, 373, 238, 18, 187, 316, 40, 34, 115, 87, 265, 7, 381, 324, 62, 78, 195, 156, 142, 205, 48, 13, 304, 30, 309, 253, 32, 352, 317, 367, 300, 200, 208, 2, 264, 92, 265, 7, 337, 121, 294, 159, 354, 325, 73, 338, 83, 16, 116, 181, 346, 296, 76, 99, 291, 80, 201, 123, 364, 179, 81, 92, 265, 7, 185, 189, 146, 231, 366, 274, 130, 145, 279, 178, 315, 225, 203, 276, 320, 67, 329, 180, 174, 119, 371, 158, 74, 211, 265, 252, 70, 85, 29, 213, 144, 41, 149, 341, 326, 323, 313, 270, 335, 303, 188, 8, 193, 68, 247, 340, 82, 36, 166, 110, 154, 252, 357, 369, 230, 319, 382, 254, 49, 380, 59, 216, 1, 66, 72, 120, 175, 191, 50, 109, 228, 165, 33, 152, 353, 9, 154, 328, 357, 369, 186, 283, 306, 226, 113, 345, 140, 147, 280, 108, 103, 196, 86, 42, 288, 184, 43, 285, 305, 21, 232, 9, 154, 252, 223, 64, 5, 24, 164, 71, 336, 118, 114, 269, 244, 194, 170, 271, 249, 148, 209, 101, 162, 90, 349, 133, 368, 9, 154, 252, 263, 128, 255, 343, 15, 355, 356, 182, 321, 235, 11, 339, 282, 23, 284, 160, 171, 183, 250, 199, 117, 358, 383, 246, 246, 22, 360, 85, 375, 53, 190, 299, 96, 293, 63, 153, 47, 312, 333, 10, 363, 359, 102, 248, 75, 124, 261, 155, 198, 246, 111, 22, 157, 27, 220, 204, 192, 273, 177, 307, 206, 278, 311, 239, 267, 14, 173, 163, 262, 25, 378, 347, 100, 374, 241] + 25: [315, 379, 101, 372, 327, 333, 314, 120, 231, 256, 176, 114, 221, 37, 328, 212, 200, 163, 22, 173, 293, 119, 57, 148, 170, 123, 315, 379, 282, 162, 62, 160, 66, 302, 209, 229, 103, 196, 312, 290, 134, 11, 300, 51, 292, 311, 289, 95, 363, 288, 255, 239, 315, 379, 282, 332, 246, 280, 38, 154, 84, 54, 146, 291, 199, 130, 257, 370, 361, 248, 117, 226, 46, 205, 360, 241, 106, 178, 315, 334, 72, 23, 152, 145, 32, 100, 306, 366, 164, 191, 237, 324, 133, 94, 74, 110, 34, 21, 245, 48, 374, 198, 151, 220, 315, 82, 72, 61, 113, 235, 8, 131, 253, 80, 348, 140, 68, 15, 202, 150, 27, 127, 250, 320, 41, 188, 217, 47, 13, 277, 315, 299, 317, 107, 29, 171, 380, 262, 215, 126, 279, 359, 383, 213, 297, 31, 53, 337, 325, 87, 16, 264, 172, 169, 338, 156, 269, 168, 30, 214, 284, 10, 59, 368, 240, 125, 4, 143, 187, 184, 365, 356, 79, 271, 281, 364, 175, 344, 141, 266, 195, 263, 269, 268, 180, 25, 309, 232, 296, 381, 373, 367, 227, 211, 316, 330, 254, 149, 223, 166, 362, 122, 92, 349, 285, 50, 6, 158, 269, 298, 72, 372, 192, 49, 318, 18, 234, 89, 321, 225, 350, 204, 339, 310, 249, 260, 36, 210, 236, 3, 335, 272, 303, 189, 269, 379, 85, 107, 29, 112, 378, 351, 222, 99, 261, 336, 132, 259, 28, 105, 179, 358, 86, 183, 322, 155, 26, 108, 270, 267, 269, 379, 317, 23, 192, 5, 102, 287, 197, 138, 97, 242, 81, 55, 201, 355, 340, 353, 93, 96, 40, 63, 147, 182, 206, 326, 98, 247, 180, 214, 284, 77, 230, 342, 252, 323, 307, 352, 203, 308, 58, 24, 159, 111, 35, 283, 118, 224, 71, 219, 43, 165, 139, 258, 282, 2, 0, 73, 216, 207, 9, 153, 161, 115, 304, 185, 331, 67, 193, 243, 157, 70, 64, 142, 88, 301, 19, 121, 139, 247, 282, 39, 52, 190, 33, 136, 377, 244, 218, 369, 278, 345, 181, 69, 238, 109, 177, 208, 371, 1, 286, 329, 194, 91, 65, 83, 375, 78, 62, 7, 305, 346, 56, 76, 135, 347, 295, 354, 274, 144, 251, 186, 75, 45, 128, 265, 12, 90, 44, 129, 228, 228, 276, 276, 376, 273, 313, 294, 341, 60, 343, 20, 319, 357, 116, 14, 17, 275, 124, 167, 174, 137, 42, 104, 382, 233] + 26: [80, 237, 90, 21, 289, 344, 22, 382, 128, 360, 97, 173, 239, 212, 41, 187, 78, 325, 335, 112, 154, 42, 85, 84, 139, 169, 80, 237, 90, 21, 16, 337, 376, 332, 156, 323, 102, 103, 195, 159, 50, 153, 336, 340, 40, 275, 314, 284, 211, 310, 170, 175, 150, 124, 217, 338, 339, 137, 61, 126, 148, 179, 251, 79, 277, 65, 231, 182, 189, 12, 334, 59, 163, 4, 250, 95, 203, 218, 320, 124, 237, 291, 339, 316, 226, 108, 180, 321, 356, 221, 17, 87, 114, 232, 368, 352, 72, 297, 101, 296, 164, 58, 168, 209, 136, 269, 217, 253, 254, 276, 88, 62, 247, 287, 174, 343, 104, 161, 196, 75, 157, 121, 43, 9, 91, 373, 257, 210, 52, 96, 136, 269, 18, 236, 230, 278, 227, 83, 200, 262, 99, 24, 15, 81, 350, 383, 25, 197, 255, 361, 205, 270, 28, 381, 374, 375, 136, 269, 18, 236, 116, 130, 135, 36, 311, 178, 142, 294, 365, 35, 171, 123, 301, 105, 354, 39, 29, 313, 46, 23, 261, 305, 136, 349, 217, 70, 149, 308, 355, 146, 48, 138, 367, 194, 20, 202, 252, 143, 302, 125, 109, 77, 274, 377, 147, 366, 348, 351, 136, 271, 237, 291, 224, 94, 249, 82, 265, 133, 27, 122, 318, 98, 115, 312, 298, 300, 185, 331, 119, 56, 222, 273, 60, 106, 5, 11, 172, 342, 69, 183, 158, 167, 176, 26, 353, 57, 127, 378, 370, 144, 264, 364, 8, 346, 216, 324, 188, 322, 1, 54, 6, 380, 53, 181, 160, 279, 30, 131, 359, 190, 306, 51, 66, 290, 76, 93, 219, 362, 347, 74, 193, 14, 282, 155, 213, 228, 6, 380, 214, 184, 162, 38, 223, 245, 272, 191, 152, 186, 357, 45, 107, 363, 280, 3, 165, 100, 206, 145, 283, 248, 260, 234, 233, 11, 172, 63, 230, 341, 238, 92, 198, 259, 44, 268, 285, 0, 267, 19, 117, 220, 317, 333, 326, 243, 330, 379, 68, 86, 292, 89, 90, 263, 253, 307, 369, 55, 67, 151, 2, 201, 71, 10, 37, 177, 329, 47, 141, 303, 13, 358, 235, 207, 309, 246, 292, 89, 53, 371, 120, 33, 315, 328, 31, 319, 327, 241, 215, 113, 256, 372, 134, 49, 225, 244, 240, 208, 295, 286, 299, 242, 281, 89, 172, 181, 304, 266, 293, 110, 204, 166, 229, 118, 192, 73, 32, 129, 7, 140, 64, 34, 199, 258, 345, 288, 132, 111] + 27: [34, 12, 61, 233, 49, 358, 183, 228, 232, 89, 141, 98, 6, 127, 375, 249, 29, 116, 96, 8, 157, 192, 311, 295, 36, 236, 234, 333, 61, 168, 80, 341, 348, 85, 76, 238, 191, 155, 271, 268, 124, 44, 365, 159, 280, 22, 229, 64, 1, 169, 247, 310, 234, 351, 263, 269, 30, 354, 190, 75, 258, 82, 356, 241, 175, 136, 67, 120, 95, 123, 231, 50, 302, 87, 52, 363, 270, 338, 81, 286, 263, 252, 357, 377, 222, 260, 380, 32, 297, 88, 91, 215, 129, 199, 167, 153, 281, 223, 352, 108, 173, 371, 285, 342, 103, 225, 312, 300, 376, 211, 179, 21, 196, 78, 322, 254, 126, 174, 106, 347, 3, 107, 154, 313, 158, 257, 11, 19, 180, 146, 103, 225, 312, 265, 128, 211, 224, 303, 149, 343, 187, 370, 368, 296, 18, 2, 217, 79, 306, 186, 150, 198, 56, 355, 200, 163, 103, 225, 312, 265, 349, 227, 188, 340, 242, 115, 317, 161, 326, 274, 46, 38, 47, 256, 152, 282, 330, 345, 245, 40, 209, 176, 103, 286, 12, 69, 135, 131, 336, 230, 381, 102, 337, 162, 125, 291, 248, 171, 13, 205, 184, 319, 110, 94, 138, 255, 203, 220, 103, 286, 12, 204, 30, 354, 237, 251, 266, 156, 118, 261, 329, 304, 239, 147, 367, 361, 259, 346, 289, 283, 148, 210, 243, 240, 301, 4, 31, 168, 80, 341, 369, 178, 216, 308, 140, 27, 10, 5, 276, 307, 101, 288, 113, 359, 41, 189, 194, 57, 51, 39, 37, 58, 350, 164, 197, 24, 207, 290, 99, 294, 193, 7, 316, 360, 45, 9, 353, 105, 344, 253, 364, 334, 160, 221, 378, 244, 37, 4, 70, 73, 119, 366, 55, 335, 72, 278, 25, 42, 246, 122, 287, 318, 250, 151, 132, 65, 372, 166, 165, 206, 212, 262, 104, 279, 90, 73, 320, 292, 97, 272, 324, 23, 201, 328, 309, 218, 362, 28, 321, 33, 14, 264, 382, 86, 331, 92, 93, 60, 145, 59, 90, 69, 26, 202, 112, 182, 77, 325, 109, 305, 214, 172, 43, 20, 84, 35, 139, 293, 68, 195, 314, 298, 100, 137, 177, 117, 90, 71, 349, 227, 213, 323, 383, 83, 373, 133, 379, 327, 267, 66, 121, 130, 144, 74, 16, 273, 170, 277, 17, 235, 177, 219, 374, 252, 315, 181, 143, 48, 339, 332, 284, 15, 54, 114, 111, 299, 0, 275, 208, 185, 142, 62, 226, 134, 63, 53] + 28: [4, 356, 356, 306, 99, 321, 219, 58, 352, 299, 209, 127, 333, 365, 23, 159, 84, 73, 31, 348, 259, 29, 350, 186, 2, 359, 355, 92, 153, 306, 25, 357, 263, 368, 339, 240, 221, 16, 222, 194, 44, 305, 170, 151, 79, 80, 47, 55, 65, 309, 379, 353, 161, 283, 203, 195, 107, 253, 346, 227, 160, 167, 349, 273, 290, 264, 171, 148, 104, 136, 331, 39, 85, 267, 371, 319, 115, 150, 161, 26, 328, 302, 25, 197, 137, 320, 135, 32, 340, 48, 134, 256, 69, 82, 270, 377, 109, 287, 237, 281, 360, 187, 147, 311, 161, 26, 188, 302, 122, 197, 95, 158, 200, 220, 56, 278, 335, 265, 208, 362, 74, 332, 149, 131, 303, 277, 53, 83, 9, 156, 161, 26, 70, 97, 99, 321, 166, 5, 129, 370, 217, 380, 300, 46, 249, 143, 36, 342, 213, 246, 8, 275, 254, 145, 11, 180, 161, 92, 153, 255, 93, 214, 378, 165, 100, 242, 105, 43, 206, 179, 6, 61, 169, 297, 119, 293, 347, 76, 102, 164, 173, 329, 312, 324, 185, 243, 288, 7, 75, 71, 88, 326, 271, 316, 1, 121, 295, 21, 230, 382, 144, 323, 146, 35, 291, 212, 54, 215, 0, 22, 226, 176, 286, 307, 128, 59, 157, 28, 308, 367, 78, 260, 81, 177, 66, 133, 89, 110, 229, 111, 45, 87, 132, 257, 248, 344, 255, 120, 286, 272, 199, 204, 63, 269, 125, 268, 341, 112, 49, 175, 12, 280, 96, 251, 276, 123, 94, 124, 138, 279, 201, 344, 152, 162, 52, 241, 3, 116, 383, 289, 13, 315, 91, 244, 57, 375, 211, 154, 117, 345, 192, 252, 189, 313, 10, 18, 239, 250, 185, 155, 107, 37, 108, 228, 235, 337, 20, 247, 68, 118, 64, 381, 334, 294, 218, 351, 139, 232, 304, 318, 60, 86, 310, 250, 317, 233, 30, 113, 236, 325, 130, 163, 33, 19, 282, 245, 327, 106, 258, 34, 330, 38, 234, 140, 126, 198, 190, 336, 196, 250, 317, 172, 30, 374, 369, 223, 361, 67, 216, 193, 178, 142, 314, 205, 210, 366, 364, 225, 17, 101, 141, 262, 298, 338, 168, 77, 185, 162, 103, 224, 50, 42, 90, 285, 238, 376, 14, 183, 274, 98, 15, 24, 372, 284, 72, 174, 202, 62, 301, 343, 168, 250, 70, 243, 52, 241, 191, 51, 261, 41, 182, 114, 184, 27, 358, 322, 296, 363, 266, 292, 231, 373, 354, 181, 40, 207] + 29: [383, 361, 203, 304, 173, 89, 326, 204, 50, 324, 139, 59, 97, 344, 312, 315, 56, 158, 179, 306, 251, 96, 5, 221, 206, 212, 197, 361, 198, 339, 202, 11, 112, 370, 86, 61, 349, 19, 310, 313, 22, 40, 111, 250, 227, 308, 141, 109, 27, 321, 367, 39, 197, 361, 198, 362, 282, 11, 65, 68, 350, 248, 257, 225, 267, 105, 229, 46, 209, 261, 239, 43, 45, 358, 80, 335, 37, 177, 73, 51, 333, 304, 173, 255, 366, 24, 210, 172, 178, 146, 101, 340, 169, 230, 220, 42, 253, 99, 184, 189, 2, 268, 98, 26, 73, 51, 203, 295, 23, 89, 180, 237, 382, 218, 34, 228, 150, 77, 224, 52, 287, 299, 352, 154, 337, 201, 8, 372, 143, 185, 271, 25, 258, 362, 282, 266, 243, 163, 82, 298, 128, 171, 181, 371, 242, 170, 114, 70, 69, 162, 30, 164, 226, 81, 256, 322, 283, 318, 323, 33, 60, 160, 140, 275, 341, 314, 106, 272, 264, 217, 31, 157, 368, 84, 121, 168, 246, 234, 286, 107, 274, 363, 14, 318, 354, 295, 311, 255, 183, 145, 278, 259, 116, 347, 3, 32, 36, 126, 53, 338, 270, 376, 296, 325, 260, 291, 6, 29, 196, 336, 323, 129, 23, 346, 174, 63, 102, 215, 12, 276, 345, 328, 238, 302, 303, 161, 15, 149, 290, 49, 245, 342, 284, 66, 196, 54, 277, 74, 309, 351, 48, 332, 193, 360, 232, 300, 104, 20, 83, 10, 191, 199, 192, 334, 211, 301, 130, 64, 144, 355, 88, 336, 120, 265, 44, 127, 375, 235, 38, 364, 222, 131, 142, 205, 359, 147, 55, 9, 62, 356, 92, 41, 100, 138, 305, 285, 58, 244, 277, 262, 236, 208, 134, 190, 293, 4, 200, 18, 380, 317, 373, 377, 103, 153, 233, 0, 263, 279, 95, 1, 214, 108, 195, 244, 137, 67, 320, 127, 159, 188, 123, 94, 329, 72, 353, 269, 122, 369, 79, 166, 307, 219, 252, 254, 21, 165, 289, 91, 118, 124, 379, 110, 44, 311, 87, 175, 319, 85, 182, 90, 151, 167, 348, 93, 327, 7, 187, 17, 35, 155, 75, 231, 249, 152, 118, 365, 137, 67, 236, 148, 57, 343, 207, 378, 273, 280, 78, 125, 47, 28, 213, 13, 374, 71, 115, 133, 331, 281, 247, 176, 118, 244, 76, 186, 60, 16, 119, 216, 357, 132, 156, 136, 292, 294, 316, 135, 381, 288, 194, 113, 297, 223, 330, 241, 117, 240] + 30: [140, 76, 53, 245, 143, 144, 147, 221, 336, 320, 160, 343, 335, 172, 375, 360, 166, 304, 83, 82, 330, 112, 93, 251, 47, 88, 140, 222, 358, 214, 156, 289, 302, 43, 71, 210, 344, 91, 54, 14, 244, 284, 230, 247, 155, 364, 190, 102, 63, 201, 123, 128, 140, 272, 29, 225, 206, 361, 205, 312, 58, 215, 67, 60, 142, 280, 241, 374, 110, 324, 287, 368, 308, 106, 171, 227, 134, 146, 133, 333, 75, 196, 373, 234, 250, 159, 253, 226, 285, 325, 383, 376, 248, 92, 95, 350, 327, 291, 173, 56, 372, 379, 345, 380, 158, 213, 243, 378, 351, 321, 362, 97, 339, 136, 292, 261, 84, 310, 48, 44, 377, 371, 5, 370, 352, 199, 39, 70, 8, 21, 37, 239, 65, 245, 46, 246, 316, 224, 275, 299, 18, 104, 231, 20, 211, 145, 194, 161, 99, 81, 105, 313, 148, 27, 301, 218, 37, 239, 65, 135, 143, 64, 41, 341, 1, 217, 356, 96, 72, 319, 115, 282, 183, 347, 165, 3, 15, 265, 367, 238, 185, 189, 254, 239, 242, 245, 24, 346, 33, 74, 193, 309, 298, 365, 162, 111, 180, 288, 337, 326, 69, 209, 255, 270, 26, 87, 307, 259, 124, 239, 53, 65, 168, 45, 164, 317, 177, 277, 170, 23, 114, 129, 109, 175, 342, 279, 381, 314, 178, 235, 42, 131, 22, 31, 266, 208, 366, 52, 351, 28, 17, 89, 6, 236, 98, 174, 306, 286, 349, 318, 186, 90, 57, 130, 120, 300, 34, 191, 36, 40, 266, 239, 13, 214, 156, 289, 315, 340, 100, 204, 369, 132, 179, 359, 181, 297, 107, 267, 323, 127, 233, 80, 328, 331, 268, 269, 266, 239, 358, 78, 2, 62, 271, 49, 256, 9, 258, 157, 150, 66, 293, 25, 154, 32, 348, 355, 232, 198, 229, 30, 354, 322, 184, 208, 187, 52, 257, 10, 68, 138, 77, 382, 281, 108, 167, 294, 332, 79, 311, 264, 203, 153, 19, 338, 61, 119, 363, 334, 262, 208, 137, 35, 295, 223, 212, 240, 192, 141, 296, 278, 103, 163, 7, 249, 121, 220, 85, 273, 113, 126, 182, 252, 94, 101, 262, 208, 187, 59, 373, 188, 303, 216, 152, 151, 260, 200, 38, 197, 73, 50, 202, 263, 125, 117, 16, 207, 228, 4, 176, 118, 208, 208, 86, 35, 62, 11, 283, 12, 357, 195, 274, 353, 329, 51, 55, 290, 276, 139, 116, 122, 169, 0, 149, 305, 219, 237] + 31: [248, 233, 121, 278, 147, 39, 349, 62, 296, 152, 368, 366, 287, 302, 195, 362, 177, 264, 210, 367, 216, 293, 18, 218, 343, 207, 320, 36, 379, 115, 70, 317, 206, 208, 258, 249, 74, 105, 46, 21, 45, 247, 83, 130, 375, 340, 135, 382, 179, 332, 89, 180, 125, 159, 26, 326, 250, 227, 22, 299, 290, 138, 243, 134, 202, 369, 285, 153, 351, 228, 309, 129, 327, 344, 242, 311, 203, 294, 128, 159, 158, 239, 187, 268, 188, 171, 131, 254, 231, 137, 222, 118, 141, 271, 275, 1, 139, 157, 99, 372, 319, 161, 165, 347, 128, 159, 158, 278, 25, 227, 209, 164, 189, 82, 40, 205, 318, 310, 8, 12, 323, 281, 226, 24, 64, 197, 27, 357, 71, 37, 128, 159, 252, 115, 70, 87, 119, 75, 214, 380, 54, 88, 374, 288, 92, 11, 376, 107, 240, 178, 321, 126, 370, 230, 104, 172, 331, 3, 252, 239, 215, 96, 325, 56, 223, 353, 2, 81, 190, 303, 110, 4, 352, 224, 306, 262, 280, 156, 255, 358, 10, 329, 265, 365, 166, 144, 363, 150, 60, 283, 348, 211, 232, 176, 31, 314, 291, 108, 14, 103, 162, 9, 300, 282, 167, 257, 67, 246, 201, 0, 26, 133, 95, 267, 333, 34, 276, 236, 23, 122, 316, 336, 361, 260, 91, 371, 101, 298, 109, 235, 301, 359, 148, 212, 201, 19, 26, 116, 261, 328, 140, 72, 112, 78, 55, 57, 41, 217, 73, 237, 266, 198, 47, 168, 191, 53, 184, 163, 13, 149, 68, 0, 324, 143, 185, 219, 58, 378, 102, 145, 98, 322, 77, 373, 29, 51, 43, 48, 308, 315, 113, 193, 174, 30, 341, 90, 68, 0, 324, 292, 185, 339, 61, 170, 334, 111, 245, 169, 284, 213, 142, 305, 6, 304, 85, 259, 356, 79, 35, 94, 117, 229, 32, 0, 279, 143, 261, 87, 155, 253, 286, 28, 66, 175, 97, 312, 84, 273, 346, 204, 345, 251, 234, 274, 313, 377, 154, 120, 192, 100, 166, 364, 270, 330, 186, 360, 269, 244, 63, 65, 256, 383, 354, 196, 220, 194, 42, 160, 381, 44, 124, 199, 151, 350, 221, 0, 233, 144, 363, 76, 173, 338, 106, 136, 38, 182, 15, 123, 49, 183, 272, 50, 80, 337, 52, 7, 114, 86, 200, 132, 241, 100, 233, 335, 69, 76, 146, 355, 33, 93, 17, 277, 225, 307, 16, 20, 59, 238, 263, 342, 127, 181, 295, 297, 5, 289] + 32: [94, 108, 187, 66, 141, 329, 184, 283, 19, 345, 101, 25, 45, 240, 24, 198, 5, 132, 128, 305, 278, 100, 215, 93, 333, 245, 94, 146, 209, 360, 38, 166, 204, 116, 120, 266, 104, 324, 142, 249, 381, 56, 311, 284, 222, 44, 52, 277, 112, 272, 251, 262, 63, 367, 92, 130, 164, 344, 96, 140, 113, 168, 36, 250, 32, 351, 10, 22, 172, 165, 376, 134, 343, 133, 98, 291, 316, 373, 63, 367, 92, 21, 257, 329, 83, 126, 289, 189, 111, 269, 127, 53, 315, 77, 144, 287, 105, 356, 334, 364, 95, 226, 27, 14, 63, 367, 323, 271, 107, 342, 65, 78, 228, 227, 85, 239, 293, 176, 337, 139, 82, 125, 304, 330, 170, 137, 47, 192, 177, 181, 63, 367, 179, 310, 107, 41, 359, 233, 7, 325, 69, 332, 253, 70, 302, 236, 136, 102, 73, 374, 377, 1, 260, 235, 326, 244, 63, 367, 285, 229, 164, 344, 174, 97, 232, 301, 185, 157, 230, 84, 4, 158, 361, 103, 195, 261, 208, 60, 18, 295, 379, 375, 63, 49, 362, 122, 327, 61, 75, 151, 357, 297, 292, 64, 237, 3, 80, 193, 42, 9, 117, 201, 203, 259, 217, 338, 162, 169, 218, 268, 285, 370, 274, 322, 263, 279, 306, 369, 354, 281, 182, 81, 205, 372, 221, 99, 223, 119, 79, 23, 242, 194, 54, 57, 218, 346, 106, 200, 327, 275, 89, 143, 273, 72, 55, 48, 336, 29, 129, 349, 34, 307, 20, 153, 255, 31, 321, 267, 90, 68, 288, 135, 190, 51, 12, 322, 202, 258, 308, 298, 26, 175, 382, 39, 247, 62, 110, 43, 341, 0, 196, 46, 358, 11, 15, 147, 148, 246, 347, 51, 122, 214, 225, 211, 294, 171, 303, 178, 286, 183, 309, 340, 339, 191, 365, 67, 355, 76, 17, 159, 167, 265, 148, 234, 347, 163, 219, 275, 348, 156, 109, 91, 50, 206, 150, 353, 6, 378, 320, 124, 197, 270, 188, 118, 161, 290, 145, 123, 352, 155, 241, 8, 121, 16, 186, 331, 58, 313, 199, 28, 59, 371, 276, 363, 210, 299, 13, 35, 138, 180, 74, 318, 173, 296, 352, 33, 190, 280, 2, 40, 319, 87, 335, 212, 231, 314, 86, 115, 30, 317, 282, 243, 248, 160, 149, 216, 88, 207, 300, 312, 352, 33, 241, 163, 38, 166, 380, 131, 252, 254, 71, 213, 256, 350, 366, 37, 154, 152, 238, 264, 220, 224, 368, 328, 114, 383] + 33: [86, 362, 36, 319, 283, 125, 89, 21, 30, 222, 305, 230, 347, 367, 279, 371, 350, 67, 136, 132, 332, 337, 275, 284, 189, 12, 167, 377, 380, 81, 251, 267, 37, 361, 227, 40, 232, 173, 80, 156, 213, 49, 52, 285, 238, 111, 365, 353, 313, 344, 282, 277, 91, 38, 42, 343, 258, 73, 89, 233, 25, 334, 18, 138, 262, 19, 105, 368, 166, 278, 23, 74, 62, 244, 374, 172, 143, 69, 91, 38, 158, 297, 183, 68, 139, 273, 1, 131, 211, 205, 22, 197, 340, 292, 28, 29, 221, 76, 291, 378, 7, 309, 223, 142, 91, 38, 323, 81, 301, 267, 352, 270, 151, 192, 144, 149, 45, 370, 152, 43, 66, 248, 194, 381, 224, 315, 214, 302, 77, 287, 328, 38, 31, 343, 39, 179, 206, 48, 117, 356, 351, 187, 79, 104, 13, 249, 26, 53, 182, 286, 312, 326, 171, 296, 129, 83, 51, 38, 153, 115, 115, 307, 266, 127, 155, 237, 202, 329, 162, 256, 58, 170, 103, 271, 298, 11, 310, 341, 274, 304, 264, 236, 240, 191, 362, 107, 251, 307, 5, 169, 150, 217, 126, 109, 246, 55, 60, 106, 349, 261, 20, 300, 98, 134, 359, 99, 185, 333, 140, 280, 121, 319, 301, 348, 146, 54, 35, 289, 290, 355, 196, 382, 364, 137, 65, 112, 318, 27, 184, 165, 92, 46, 272, 102, 176, 331, 229, 325, 299, 94, 72, 335, 193, 208, 303, 93, 375, 157, 234, 122, 70, 101, 346, 130, 257, 175, 306, 294, 265, 34, 75, 339, 380, 96, 258, 342, 226, 190, 338, 311, 235, 174, 198, 373, 114, 215, 113, 295, 128, 243, 245, 59, 250, 2, 159, 161, 145, 339, 64, 379, 299, 147, 108, 141, 160, 268, 116, 16, 124, 50, 369, 345, 366, 110, 252, 225, 78, 363, 228, 321, 263, 220, 320, 339, 64, 148, 154, 94, 84, 376, 218, 308, 354, 90, 216, 200, 88, 330, 281, 327, 6, 181, 118, 247, 269, 44, 100, 293, 260, 177, 36, 231, 82, 241, 322, 17, 87, 133, 61, 317, 383, 360, 186, 3, 336, 41, 10, 85, 188, 199, 97, 47, 0, 32, 260, 177, 36, 148, 154, 358, 201, 204, 33, 9, 14, 254, 276, 209, 212, 15, 56, 324, 316, 207, 120, 178, 253, 57, 357, 372, 63, 259, 229, 319, 283, 314, 164, 255, 210, 288, 8, 4, 123, 163, 180, 71, 24, 219, 135, 168, 119, 242, 195, 203, 239, 95] + 34: [372, 312, 135, 218, 3, 315, 246, 4, 244, 81, 368, 245, 181, 170, 180, 127, 142, 374, 36, 255, 242, 257, 43, 188, 198, 341, 382, 18, 330, 248, 145, 178, 59, 267, 162, 174, 159, 336, 30, 204, 140, 10, 23, 266, 262, 49, 57, 253, 130, 276, 129, 193, 366, 126, 16, 340, 3, 258, 347, 65, 95, 326, 327, 217, 270, 263, 308, 62, 56, 152, 310, 109, 131, 339, 163, 320, 235, 202, 366, 126, 249, 311, 190, 61, 97, 254, 373, 58, 199, 143, 227, 5, 230, 157, 147, 192, 226, 93, 250, 90, 281, 378, 224, 132, 108, 179, 16, 15, 243, 291, 79, 222, 200, 290, 166, 60, 0, 367, 80, 322, 51, 88, 118, 34, 370, 22, 54, 338, 211, 76, 108, 179, 16, 15, 293, 148, 83, 288, 110, 329, 37, 172, 214, 116, 348, 19, 261, 13, 269, 380, 173, 20, 128, 349, 213, 119, 108, 179, 16, 15, 240, 31, 8, 11, 21, 53, 136, 33, 150, 35, 232, 277, 272, 219, 228, 331, 55, 297, 325, 369, 66, 122, 108, 332, 330, 248, 223, 194, 99, 300, 225, 302, 256, 70, 236, 333, 352, 283, 139, 296, 265, 307, 17, 9, 120, 252, 377, 229, 29, 179, 74, 218, 25, 167, 295, 78, 91, 144, 137, 206, 209, 169, 154, 42, 105, 314, 282, 117, 176, 171, 96, 182, 177, 279, 337, 299, 306, 239, 196, 203, 238, 92, 134, 184, 197, 195, 231, 50, 301, 294, 264, 268, 274, 100, 260, 212, 351, 38, 273, 160, 337, 357, 334, 354, 44, 360, 115, 6, 208, 12, 175, 317, 48, 89, 215, 298, 149, 85, 111, 319, 278, 1, 71, 234, 121, 381, 69, 324, 306, 306, 190, 138, 379, 77, 125, 84, 313, 328, 346, 102, 103, 323, 186, 275, 86, 286, 285, 141, 292, 303, 375, 365, 69, 299, 82, 158, 196, 362, 271, 41, 201, 210, 183, 67, 241, 364, 113, 318, 383, 289, 233, 75, 26, 104, 156, 106, 309, 363, 207, 259, 334, 304, 185, 40, 284, 191, 124, 356, 187, 133, 72, 287, 63, 114, 45, 68, 101, 353, 39, 155, 216, 350, 32, 14, 47, 151, 249, 340, 321, 61, 2, 335, 153, 107, 24, 280, 27, 247, 361, 371, 46, 64, 305, 165, 205, 112, 355, 94, 7, 316, 237, 259, 249, 340, 44, 146, 220, 168, 87, 345, 123, 251, 73, 359, 164, 358, 221, 98, 161, 28, 52, 342, 344, 376, 343, 189] + 35: [48, 58, 297, 345, 4, 322, 224, 311, 266, 357, 128, 215, 137, 261, 111, 34, 301, 217, 274, 278, 306, 238, 351, 210, 76, 381, 309, 319, 297, 345, 324, 72, 147, 375, 329, 98, 325, 372, 243, 57, 109, 290, 269, 344, 63, 135, 89, 97, 333, 39, 173, 228, 363, 319, 131, 368, 86, 276, 206, 234, 14, 187, 171, 298, 337, 170, 145, 382, 17, 66, 103, 115, 94, 150, 270, 258, 199, 304, 363, 319, 131, 67, 47, 113, 231, 241, 191, 360, 356, 334, 122, 302, 348, 339, 99, 84, 317, 22, 232, 148, 197, 247, 212, 284, 80, 319, 211, 295, 53, 281, 15, 364, 136, 373, 230, 190, 320, 92, 20, 166, 184, 338, 65, 316, 26, 165, 225, 226, 341, 60, 75, 144, 56, 345, 342, 203, 116, 307, 222, 321, 308, 282, 12, 32, 106, 117, 376, 6, 35, 370, 250, 139, 83, 242, 59, 291, 75, 130, 1, 295, 53, 3, 340, 335, 361, 168, 296, 108, 95, 96, 110, 369, 142, 175, 105, 156, 121, 255, 310, 280, 71, 200, 353, 237, 1, 367, 55, 113, 221, 50, 362, 140, 244, 30, 209, 350, 277, 273, 185, 374, 292, 207, 155, 263, 233, 219, 64, 118, 220, 62, 379, 133, 138, 72, 355, 349, 126, 163, 18, 213, 125, 287, 305, 216, 19, 285, 248, 161, 49, 288, 123, 252, 100, 85, 45, 205, 218, 314, 86, 203, 340, 326, 331, 160, 158, 268, 313, 74, 8, 383, 70, 315, 151, 259, 354, 183, 327, 336, 236, 359, 45, 205, 218, 68, 21, 322, 186, 46, 159, 157, 132, 328, 152, 323, 214, 264, 179, 88, 332, 253, 195, 358, 33, 193, 2, 13, 45, 240, 303, 68, 129, 38, 41, 352, 37, 275, 9, 93, 300, 11, 227, 330, 196, 279, 365, 29, 112, 101, 178, 188, 283, 146, 169, 44, 1, 367, 141, 289, 78, 114, 27, 312, 272, 294, 42, 172, 202, 204, 380, 43, 79, 377, 23, 77, 371, 0, 120, 293, 169, 246, 254, 318, 257, 281, 346, 164, 162, 267, 36, 5, 201, 16, 180, 347, 127, 73, 176, 149, 181, 10, 271, 119, 251, 177, 260, 182, 299, 286, 342, 276, 154, 194, 52, 134, 61, 192, 90, 256, 91, 82, 174, 378, 87, 189, 262, 167, 223, 208, 124, 40, 28, 28, 299, 133, 153, 31, 239, 265, 54, 102, 81, 104, 143, 235, 366, 24, 7, 25, 198, 245, 249, 107, 69, 51, 229, 343] + 36: [53, 232, 371, 325, 369, 372, 73, 93, 112, 9, 105, 298, 174, 228, 0, 257, 119, 208, 204, 77, 316, 378, 278, 218, 145, 279, 53, 232, 371, 325, 274, 372, 97, 32, 309, 231, 122, 183, 366, 100, 30, 38, 351, 135, 329, 248, 35, 253, 154, 269, 160, 123, 131, 348, 238, 299, 196, 223, 308, 85, 240, 139, 361, 15, 168, 237, 264, 81, 358, 356, 363, 251, 256, 8, 7, 181, 221, 170, 192, 304, 10, 328, 98, 223, 47, 287, 111, 191, 63, 342, 147, 41, 200, 344, 267, 48, 209, 381, 347, 213, 161, 306, 113, 25, 252, 304, 359, 368, 193, 225, 207, 42, 321, 166, 103, 322, 171, 102, 310, 17, 165, 58, 281, 249, 357, 350, 285, 82, 367, 375, 294, 250, 59, 368, 178, 373, 133, 233, 262, 194, 167, 339, 153, 202, 229, 104, 220, 318, 158, 236, 21, 87, 26, 222, 114, 313, 22, 84, 2, 371, 40, 56, 352, 156, 175, 349, 95, 69, 346, 46, 305, 379, 148, 29, 83, 315, 307, 355, 34, 124, 43, 16, 22, 57, 266, 272, 320, 380, 152, 45, 333, 282, 132, 20, 72, 364, 134, 335, 188, 146, 70, 259, 14, 226, 330, 141, 377, 360, 22, 57, 185, 311, 241, 12, 374, 331, 162, 49, 138, 276, 216, 66, 211, 206, 88, 283, 247, 199, 130, 61, 96, 227, 186, 290, 197, 144, 62, 311, 296, 301, 5, 332, 67, 345, 297, 289, 117, 343, 39, 52, 78, 243, 6, 169, 118, 254, 128, 327, 270, 292, 370, 144, 232, 272, 326, 12, 275, 261, 201, 319, 284, 172, 140, 11, 71, 143, 129, 365, 295, 19, 337, 324, 203, 136, 234, 273, 382, 184, 266, 272, 242, 99, 150, 65, 33, 55, 91, 164, 271, 224, 180, 76, 23, 159, 109, 173, 60, 126, 86, 4, 268, 64, 74, 184, 232, 230, 258, 107, 92, 125, 362, 293, 13, 155, 280, 195, 217, 28, 149, 51, 190, 265, 120, 115, 303, 286, 182, 336, 340, 341, 10, 368, 193, 225, 291, 31, 108, 219, 121, 300, 142, 116, 187, 212, 80, 163, 189, 198, 215, 235, 151, 106, 376, 127, 340, 184, 232, 260, 210, 107, 205, 54, 246, 94, 68, 50, 179, 177, 255, 36, 263, 18, 24, 338, 137, 312, 1, 353, 317, 37, 176, 341, 238, 101, 178, 90, 79, 314, 334, 288, 245, 3, 354, 277, 323, 157, 244, 239, 44, 383, 75, 110, 27, 89, 214, 302] + 37: [109, 379, 148, 200, 362, 382, 59, 353, 343, 192, 229, 168, 36, 135, 83, 121, 274, 4, 368, 378, 35, 96, 20, 307, 84, 75, 109, 379, 148, 200, 341, 162, 302, 360, 118, 18, 51, 27, 161, 278, 331, 262, 24, 105, 215, 260, 63, 248, 177, 367, 68, 56, 188, 123, 148, 293, 72, 349, 140, 327, 328, 336, 104, 173, 30, 111, 247, 337, 280, 354, 10, 303, 142, 357, 300, 186, 276, 132, 240, 273, 311, 363, 366, 340, 0, 88, 233, 164, 19, 203, 217, 376, 126, 321, 38, 42, 91, 332, 81, 106, 180, 89, 238, 112, 268, 218, 213, 366, 289, 113, 86, 22, 117, 150, 127, 6, 296, 34, 49, 380, 317, 5, 110, 350, 166, 62, 28, 90, 232, 236, 268, 218, 314, 222, 272, 95, 355, 245, 131, 257, 287, 144, 338, 29, 136, 244, 271, 298, 124, 78, 101, 146, 11, 220, 251, 383, 268, 218, 151, 125, 179, 340, 361, 65, 147, 157, 76, 64, 141, 285, 87, 373, 102, 210, 17, 60, 191, 184, 45, 219, 99, 239, 319, 218, 309, 46, 372, 234, 47, 185, 32, 330, 258, 176, 39, 264, 334, 80, 279, 267, 53, 170, 198, 225, 54, 369, 209, 288, 301, 218, 265, 293, 79, 292, 344, 7, 153, 133, 122, 74, 143, 108, 224, 269, 241, 249, 299, 291, 310, 214, 227, 250, 67, 8, 301, 218, 348, 178, 305, 371, 156, 165, 235, 145, 208, 149, 41, 134, 335, 347, 313, 26, 138, 183, 259, 57, 119, 381, 25, 31, 71, 294, 242, 306, 342, 162, 171, 9, 181, 345, 320, 365, 61, 3, 356, 312, 97, 154, 160, 115, 295, 70, 202, 159, 377, 352, 364, 218, 158, 222, 48, 113, 0, 182, 93, 359, 370, 43, 230, 329, 100, 255, 94, 187, 256, 175, 374, 107, 281, 246, 358, 226, 66, 294, 158, 293, 44, 77, 223, 318, 139, 322, 21, 270, 92, 333, 69, 37, 33, 325, 266, 286, 283, 231, 197, 12, 282, 375, 40, 40, 58, 199, 190, 174, 16, 252, 163, 277, 228, 1, 73, 201, 55, 207, 284, 351, 152, 216, 308, 130, 155, 2, 50, 189, 40, 40, 243, 52, 290, 349, 114, 14, 204, 315, 85, 196, 304, 103, 206, 128, 212, 172, 13, 169, 297, 237, 326, 275, 23, 263, 40, 294, 58, 205, 79, 193, 129, 253, 194, 221, 316, 323, 82, 15, 167, 98, 195, 116, 254, 137, 339, 324, 261, 211, 346, 120] + 38: [282, 245, 332, 8, 383, 177, 217, 317, 233, 222, 239, 224, 367, 298, 356, 376, 310, 360, 204, 311, 202, 113, 232, 196, 312, 52, 10, 266, 123, 372, 154, 293, 19, 119, 53, 99, 73, 58, 57, 229, 28, 76, 15, 195, 347, 283, 102, 335, 209, 365, 121, 340, 4, 266, 203, 292, 115, 230, 63, 267, 329, 106, 95, 36, 349, 276, 228, 181, 152, 172, 382, 351, 290, 65, 350, 1, 140, 185, 158, 221, 203, 372, 154, 62, 171, 257, 338, 355, 29, 144, 184, 109, 141, 272, 337, 182, 375, 294, 299, 254, 175, 198, 64, 161, 158, 266, 118, 366, 274, 333, 17, 54, 97, 147, 67, 193, 142, 48, 306, 41, 2, 79, 11, 297, 3, 164, 128, 22, 162, 237, 187, 221, 66, 285, 189, 89, 145, 135, 137, 90, 321, 307, 165, 34, 241, 180, 200, 238, 82, 61, 138, 24, 163, 336, 159, 339, 6, 369, 124, 30, 126, 220, 381, 206, 199, 125, 9, 262, 216, 37, 148, 174, 103, 371, 104, 168, 343, 320, 291, 83, 330, 325, 6, 178, 269, 51, 114, 32, 244, 212, 288, 107, 271, 358, 156, 179, 31, 242, 23, 296, 215, 44, 12, 192, 116, 281, 370, 380, 6, 348, 203, 284, 277, 62, 130, 253, 319, 146, 227, 362, 96, 7, 117, 258, 131, 43, 56, 13, 55, 112, 20, 377, 98, 176, 6, 345, 94, 366, 189, 309, 278, 344, 331, 352, 208, 68, 323, 342, 265, 353, 40, 313, 231, 80, 183, 364, 314, 127, 318, 46, 223, 50, 201, 284, 248, 39, 379, 300, 286, 341, 59, 105, 264, 188, 260, 33, 139, 74, 273, 35, 71, 157, 132, 160, 133, 101, 93, 378, 124, 51, 114, 49, 92, 279, 327, 219, 21, 45, 256, 151, 5, 149, 78, 122, 120, 186, 247, 42, 259, 155, 346, 60, 301, 301, 201, 354, 322, 333, 166, 268, 108, 75, 69, 211, 88, 143, 170, 26, 81, 214, 100, 374, 236, 240, 153, 218, 249, 251, 301, 213, 280, 30, 126, 277, 373, 328, 361, 275, 85, 359, 136, 368, 84, 197, 250, 334, 25, 295, 357, 91, 18, 110, 304, 38, 301, 287, 280, 86, 383, 89, 27, 77, 234, 14, 324, 150, 308, 111, 190, 134, 173, 302, 252, 47, 70, 316, 167, 303, 243, 191, 301, 378, 269, 263, 226, 194, 0, 169, 16, 225, 72, 87, 255, 246, 289, 305, 235, 129, 261, 205, 363, 326, 207, 315, 270, 210] + 39: [43, 310, 286, 315, 132, 86, 329, 369, 208, 274, 48, 190, 371, 317, 102, 97, 128, 207, 343, 107, 337, 92, 339, 73, 332, 44, 100, 310, 236, 312, 360, 85, 80, 24, 77, 53, 54, 331, 150, 307, 228, 173, 275, 194, 216, 103, 246, 37, 210, 124, 115, 304, 176, 310, 223, 13, 132, 187, 277, 295, 381, 298, 251, 137, 276, 213, 94, 289, 309, 60, 120, 189, 83, 252, 158, 32, 11, 21, 247, 104, 25, 56, 242, 382, 296, 300, 23, 8, 140, 74, 281, 364, 72, 241, 30, 377, 70, 288, 305, 64, 129, 125, 29, 198, 247, 104, 153, 56, 279, 272, 71, 250, 81, 170, 232, 34, 62, 161, 134, 354, 79, 306, 58, 19, 347, 197, 147, 256, 90, 257, 247, 82, 310, 218, 313, 379, 297, 341, 51, 118, 122, 336, 50, 179, 355, 133, 230, 327, 245, 168, 20, 0, 136, 205, 201, 334, 99, 104, 372, 315, 36, 119, 374, 141, 146, 227, 323, 101, 199, 63, 293, 45, 254, 169, 171, 41, 206, 123, 351, 308, 57, 89, 282, 116, 1, 113, 93, 85, 362, 330, 290, 273, 28, 138, 135, 322, 311, 95, 195, 220, 192, 335, 160, 18, 349, 12, 344, 143, 282, 258, 236, 215, 38, 2, 145, 15, 365, 259, 127, 378, 126, 226, 212, 320, 164, 183, 291, 117, 46, 10, 255, 262, 356, 22, 282, 258, 368, 312, 93, 292, 270, 319, 3, 42, 191, 47, 353, 301, 373, 193, 4, 363, 27, 196, 108, 253, 326, 105, 184, 52, 282, 234, 372, 65, 314, 155, 333, 76, 163, 352, 248, 131, 152, 249, 33, 203, 221, 181, 166, 222, 157, 238, 284, 318, 186, 321, 96, 110, 348, 139, 38, 244, 174, 178, 49, 375, 225, 239, 358, 75, 14, 285, 338, 214, 376, 111, 55, 39, 342, 224, 261, 165, 280, 66, 286, 151, 313, 328, 324, 40, 200, 235, 299, 260, 367, 345, 182, 88, 6, 68, 109, 346, 231, 142, 350, 237, 269, 149, 35, 66, 156, 56, 325, 379, 204, 98, 7, 383, 340, 69, 188, 5, 61, 185, 16, 240, 217, 287, 266, 229, 219, 67, 380, 209, 180, 66, 236, 65, 36, 267, 172, 17, 59, 202, 31, 263, 366, 144, 106, 167, 130, 265, 91, 303, 175, 211, 370, 316, 154, 9, 268, 264, 372, 315, 314, 271, 26, 121, 87, 84, 112, 278, 177, 162, 302, 357, 243, 114, 283, 159, 78, 359, 233, 294, 148, 361] + 40: [31, 286, 358, 55, 21, 153, 251, 159, 143, 289, 278, 307, 309, 299, 93, 333, 233, 246, 212, 310, 174, 119, 152, 255, 312, 353, 31, 349, 204, 378, 265, 158, 74, 228, 136, 274, 205, 46, 365, 94, 165, 171, 273, 261, 276, 0, 40, 104, 39, 157, 83, 105, 31, 349, 71, 175, 52, 346, 3, 90, 219, 80, 38, 227, 57, 110, 192, 330, 131, 139, 361, 23, 295, 34, 354, 320, 191, 248, 31, 360, 71, 175, 247, 346, 208, 351, 326, 277, 216, 262, 364, 306, 264, 356, 345, 222, 84, 348, 163, 242, 154, 292, 331, 170, 31, 360, 358, 175, 108, 226, 224, 100, 10, 162, 199, 239, 99, 151, 29, 206, 301, 118, 116, 32, 140, 173, 342, 298, 344, 5, 31, 169, 70, 190, 283, 88, 254, 9, 168, 341, 188, 229, 263, 2, 7, 48, 304, 6, 121, 383, 182, 111, 4, 45, 220, 300, 132, 250, 359, 378, 108, 226, 198, 59, 271, 63, 12, 382, 138, 377, 327, 177, 293, 291, 78, 184, 257, 218, 352, 86, 225, 376, 379, 250, 68, 75, 179, 156, 258, 17, 129, 322, 236, 370, 82, 244, 324, 323, 47, 241, 141, 124, 66, 106, 234, 112, 146, 144, 317, 15, 68, 373, 297, 128, 372, 347, 284, 166, 30, 137, 53, 65, 318, 305, 314, 147, 270, 49, 368, 180, 315, 223, 126, 123, 317, 15, 68, 367, 33, 107, 332, 210, 303, 27, 193, 56, 79, 43, 167, 371, 366, 375, 115, 135, 203, 134, 267, 172, 20, 194, 357, 15, 142, 101, 21, 164, 232, 311, 362, 211, 145, 269, 41, 130, 120, 329, 51, 238, 186, 13, 374, 325, 380, 245, 197, 149, 207, 183, 328, 67, 160, 215, 369, 275, 363, 11, 54, 44, 187, 189, 339, 243, 350, 201, 14, 125, 77, 35, 114, 24, 42, 249, 207, 260, 213, 98, 337, 8, 235, 91, 296, 253, 308, 196, 150, 355, 155, 237, 37, 85, 288, 268, 117, 22, 96, 252, 240, 69, 340, 183, 328, 378, 26, 279, 64, 1, 89, 185, 334, 259, 316, 290, 338, 281, 18, 76, 133, 102, 113, 285, 109, 381, 335, 92, 280, 319, 142, 190, 179, 161, 58, 28, 336, 61, 200, 148, 95, 103, 294, 195, 19, 73, 256, 266, 230, 217, 313, 272, 127, 214, 319, 319, 181, 62, 33, 202, 81, 97, 321, 209, 221, 302, 178, 25, 287, 282, 60, 36, 122, 50, 16, 231, 87, 176, 72, 343] + 41: [205, 142, 170, 255, 84, 285, 136, 253, 4, 239, 345, 241, 294, 328, 363, 213, 220, 342, 191, 48, 257, 329, 121, 372, 135, 120, 305, 199, 355, 190, 366, 76, 34, 44, 80, 243, 217, 55, 156, 133, 176, 5, 371, 292, 1, 141, 209, 14, 2, 162, 230, 85, 122, 199, 376, 190, 168, 92, 187, 306, 45, 140, 174, 93, 258, 210, 151, 261, 244, 33, 143, 72, 182, 47, 183, 361, 193, 40, 321, 11, 251, 30, 369, 159, 24, 12, 9, 353, 134, 226, 289, 330, 181, 160, 315, 188, 236, 207, 175, 370, 323, 377, 164, 152, 321, 199, 142, 86, 369, 327, 161, 302, 222, 264, 301, 379, 344, 112, 295, 130, 74, 300, 165, 13, 155, 245, 331, 16, 58, 263, 185, 71, 307, 374, 223, 119, 279, 206, 3, 202, 56, 54, 325, 380, 107, 277, 192, 15, 78, 248, 113, 171, 201, 362, 382, 309, 49, 114, 350, 90, 129, 159, 314, 262, 318, 126, 336, 83, 322, 268, 299, 317, 128, 75, 287, 149, 316, 17, 62, 109, 281, 278, 49, 114, 350, 184, 366, 96, 364, 347, 53, 42, 102, 313, 340, 51, 131, 94, 173, 272, 352, 38, 310, 20, 139, 346, 169, 22, 49, 166, 355, 57, 197, 124, 7, 82, 127, 36, 308, 276, 271, 266, 66, 6, 221, 99, 381, 158, 98, 29, 157, 312, 333, 132, 269, 196, 60, 374, 368, 119, 69, 95, 348, 23, 116, 282, 115, 250, 153, 18, 89, 59, 0, 259, 137, 67, 145, 189, 228, 178, 269, 103, 27, 249, 365, 215, 358, 280, 35, 219, 349, 254, 237, 8, 63, 43, 61, 154, 77, 320, 204, 375, 88, 177, 291, 357, 298, 378, 60, 212, 50, 97, 111, 311, 326, 194, 32, 225, 167, 224, 186, 81, 303, 41, 179, 68, 147, 274, 110, 341, 118, 31, 298, 378, 60, 374, 129, 275, 360, 334, 104, 218, 26, 256, 229, 339, 284, 19, 146, 232, 65, 297, 28, 265, 240, 211, 233, 247, 283, 335, 350, 184, 200, 293, 79, 198, 324, 286, 70, 125, 235, 288, 101, 208, 354, 359, 39, 105, 73, 238, 319, 270, 52, 227, 283, 378, 170, 30, 273, 275, 21, 108, 383, 163, 252, 260, 138, 242, 338, 246, 214, 37, 100, 332, 148, 231, 267, 356, 304, 296, 87, 367, 170, 150, 197, 215, 172, 337, 351, 25, 64, 46, 117, 144, 290, 91, 10, 123, 180, 234, 195, 373, 343, 106, 203, 216] + 42: [2, 152, 117, 218, 325, 333, 203, 189, 328, 17, 167, 33, 52, 8, 89, 139, 5, 125, 26, 73, 312, 153, 47, 14, 0, 242, 188, 152, 117, 247, 158, 41, 263, 127, 274, 67, 147, 77, 65, 32, 246, 283, 291, 378, 272, 197, 124, 22, 69, 43, 361, 254, 210, 152, 117, 212, 264, 162, 262, 277, 370, 230, 87, 202, 257, 241, 371, 226, 205, 346, 56, 250, 288, 367, 249, 78, 13, 294, 154, 152, 31, 218, 93, 243, 377, 373, 4, 169, 216, 164, 227, 340, 180, 315, 332, 53, 194, 363, 3, 302, 115, 142, 24, 352, 375, 228, 31, 350, 151, 168, 129, 265, 354, 187, 326, 314, 149, 80, 219, 49, 16, 156, 120, 368, 136, 330, 79, 54, 240, 324, 359, 148, 119, 116, 63, 138, 208, 342, 275, 344, 338, 38, 160, 245, 323, 157, 7, 269, 206, 199, 334, 383, 100, 107, 91, 95, 359, 114, 31, 211, 158, 374, 306, 191, 141, 287, 165, 280, 39, 60, 372, 244, 336, 68, 293, 104, 185, 85, 11, 331, 163, 369, 20, 295, 268, 176, 256, 37, 255, 327, 358, 28, 173, 234, 76, 132, 307, 183, 61, 278, 229, 267, 304, 144, 110, 81, 365, 339, 1, 266, 345, 176, 256, 179, 6, 319, 178, 140, 161, 45, 118, 137, 238, 99, 298, 233, 290, 321, 23, 347, 353, 317, 40, 46, 217, 94, 305, 155, 282, 349, 108, 379, 30, 236, 356, 92, 381, 51, 237, 310, 196, 170, 284, 102, 75, 174, 190, 177, 252, 126, 217, 266, 268, 172, 111, 195, 220, 235, 341, 135, 103, 322, 380, 113, 225, 130, 204, 270, 213, 10, 186, 207, 276, 271, 273, 27, 50, 94, 303, 172, 111, 337, 200, 351, 29, 224, 201, 366, 166, 19, 311, 299, 258, 300, 318, 82, 128, 133, 12, 159, 66, 121, 42, 123, 345, 155, 72, 222, 55, 348, 320, 239, 36, 181, 192, 18, 62, 109, 313, 171, 261, 98, 296, 96, 253, 382, 289, 71, 123, 123, 117, 93, 215, 138, 59, 21, 221, 259, 329, 316, 223, 301, 9, 97, 58, 112, 343, 360, 198, 309, 35, 182, 74, 193, 123, 123, 357, 279, 325, 333, 209, 281, 131, 286, 308, 335, 297, 101, 231, 232, 34, 364, 84, 145, 285, 88, 70, 44, 15, 376, 123, 123, 106, 184, 150, 37, 48, 122, 248, 214, 105, 143, 362, 146, 64, 260, 251, 355, 175, 90, 292, 134, 57, 83, 86, 25] + 43: [164, 375, 142, 35, 14, 298, 235, 203, 321, 379, 232, 123, 34, 194, 47, 217, 309, 329, 71, 173, 187, 285, 236, 270, 302, 169, 369, 375, 17, 66, 283, 297, 93, 223, 224, 32, 327, 219, 84, 107, 102, 284, 370, 175, 126, 193, 330, 230, 21, 362, 184, 214, 27, 308, 197, 112, 349, 30, 186, 127, 136, 255, 282, 271, 149, 196, 293, 70, 9, 292, 353, 381, 355, 162, 141, 383, 376, 182, 344, 225, 371, 359, 39, 100, 81, 342, 338, 191, 154, 26, 160, 28, 49, 167, 343, 159, 356, 209, 357, 69, 7, 58, 289, 348, 315, 225, 335, 98, 75, 351, 23, 313, 274, 77, 380, 339, 67, 63, 185, 46, 325, 215, 279, 119, 316, 341, 172, 96, 212, 245, 315, 225, 17, 66, 20, 103, 152, 322, 280, 85, 378, 140, 5, 155, 183, 143, 220, 122, 65, 134, 290, 248, 41, 139, 244, 373, 262, 347, 129, 66, 14, 43, 326, 264, 78, 250, 238, 306, 365, 145, 24, 120, 226, 18, 132, 151, 188, 114, 328, 333, 231, 195, 82, 347, 335, 359, 239, 366, 153, 301, 1, 2, 88, 83, 130, 334, 165, 45, 125, 64, 8, 118, 211, 124, 0, 40, 310, 192, 181, 332, 218, 59, 254, 131, 57, 178, 202, 374, 346, 200, 36, 148, 38, 266, 174, 116, 259, 268, 179, 312, 228, 42, 350, 3, 168, 156, 372, 364, 283, 298, 291, 138, 108, 73, 204, 234, 257, 243, 50, 16, 263, 72, 76, 247, 110, 260, 303, 133, 33, 382, 168, 156, 372, 37, 13, 94, 337, 111, 305, 150, 275, 286, 157, 345, 60, 251, 288, 170, 163, 158, 265, 276, 62, 15, 319, 354, 168, 177, 92, 98, 190, 314, 22, 227, 307, 272, 52, 135, 296, 237, 180, 210, 128, 11, 117, 287, 300, 311, 281, 340, 361, 206, 6, 171, 92, 98, 97, 297, 377, 29, 198, 222, 48, 86, 61, 12, 242, 74, 256, 295, 144, 233, 336, 53, 317, 104, 25, 331, 221, 105, 68, 216, 75, 351, 23, 318, 269, 4, 246, 277, 79, 10, 113, 56, 101, 304, 87, 19, 320, 199, 137, 352, 54, 189, 221, 171, 278, 324, 190, 43, 363, 360, 121, 89, 205, 115, 261, 273, 99, 213, 80, 31, 294, 358, 258, 51, 249, 208, 166, 44, 161, 105, 375, 324, 252, 30, 55, 323, 147, 368, 90, 299, 367, 241, 229, 253, 146, 207, 267, 109, 240, 201, 106, 91, 176, 95] + 44: [362, 191, 10, 101, 257, 170, 98, 38, 358, 262, 244, 39, 71, 44, 220, 370, 318, 20, 203, 62, 2, 373, 69, 127, 208, 303, 362, 191, 10, 223, 180, 311, 236, 141, 291, 265, 24, 154, 160, 202, 344, 92, 205, 32, 193, 256, 216, 22, 176, 361, 16, 161, 274, 301, 379, 223, 21, 331, 34, 333, 337, 121, 85, 201, 132, 369, 354, 278, 59, 305, 230, 137, 245, 251, 372, 37, 88, 106, 274, 301, 379, 155, 190, 229, 30, 182, 95, 207, 350, 364, 79, 221, 327, 40, 323, 3, 241, 183, 253, 119, 298, 293, 148, 116, 168, 301, 10, 101, 267, 252, 313, 142, 228, 94, 186, 122, 248, 54, 107, 185, 19, 336, 345, 367, 375, 233, 302, 73, 133, 189, 238, 300, 382, 348, 43, 277, 320, 329, 225, 324, 299, 173, 338, 96, 292, 282, 347, 81, 319, 321, 1, 27, 268, 52, 281, 295, 97, 178, 199, 60, 84, 68, 86, 249, 125, 343, 290, 219, 378, 67, 26, 15, 149, 156, 357, 204, 45, 239, 307, 152, 75, 209, 64, 272, 222, 284, 227, 368, 93, 118, 144, 355, 254, 55, 164, 381, 46, 83, 243, 218, 72, 51, 255, 50, 246, 123, 353, 374, 172, 77, 317, 348, 82, 346, 280, 306, 33, 117, 184, 100, 131, 261, 342, 334, 377, 169, 9, 314, 326, 153, 330, 111, 308, 112, 172, 77, 317, 348, 145, 346, 383, 264, 48, 89, 296, 349, 380, 269, 126, 147, 270, 237, 234, 7, 124, 99, 240, 47, 316, 322, 171, 25, 351, 226, 145, 279, 143, 294, 102, 340, 61, 163, 286, 174, 162, 74, 105, 104, 57, 49, 35, 151, 363, 297, 197, 129, 171, 77, 110, 226, 29, 328, 146, 198, 275, 158, 192, 287, 66, 359, 136, 196, 214, 113, 263, 65, 312, 210, 283, 259, 76, 332, 167, 213, 222, 271, 188, 140, 166, 289, 360, 138, 266, 56, 285, 134, 41, 108, 8, 179, 235, 114, 42, 115, 28, 356, 310, 12, 0, 213, 199, 284, 215, 135, 288, 242, 304, 276, 175, 150, 366, 177, 157, 165, 231, 23, 250, 232, 200, 5, 103, 335, 371, 13, 17, 213, 199, 226, 376, 170, 187, 63, 365, 91, 6, 70, 130, 120, 128, 90, 78, 341, 18, 53, 80, 195, 260, 31, 181, 339, 258, 213, 191, 317, 21, 325, 11, 212, 217, 14, 224, 315, 352, 87, 36, 159, 309, 211, 109, 4, 273, 194, 247, 139, 58, 206] + 45: [154, 293, 10, 274, 103, 47, 383, 275, 55, 74, 327, 108, 175, 253, 133, 290, 211, 314, 15, 3, 337, 226, 270, 77, 181, 361, 154, 284, 178, 227, 227, 329, 124, 331, 223, 308, 12, 179, 32, 370, 344, 5, 98, 169, 203, 147, 243, 232, 200, 259, 72, 244, 324, 293, 201, 141, 85, 276, 383, 351, 168, 104, 210, 345, 24, 302, 205, 87, 231, 285, 309, 364, 136, 97, 353, 335, 17, 355, 23, 161, 346, 227, 375, 329, 348, 248, 45, 88, 68, 217, 106, 218, 58, 320, 150, 127, 363, 202, 90, 153, 216, 237, 11, 64, 23, 161, 182, 27, 160, 164, 95, 39, 187, 277, 306, 41, 208, 222, 330, 38, 37, 263, 268, 371, 225, 143, 159, 311, 14, 117, 191, 317, 201, 247, 350, 131, 212, 234, 16, 51, 7, 242, 236, 22, 207, 145, 29, 342, 341, 343, 125, 89, 366, 75, 60, 300, 173, 110, 197, 27, 262, 269, 81, 228, 239, 115, 135, 86, 365, 257, 35, 256, 379, 271, 0, 362, 189, 280, 53, 28, 235, 347, 121, 92, 178, 356, 146, 180, 114, 380, 326, 229, 80, 281, 109, 261, 170, 264, 283, 151, 312, 78, 91, 167, 206, 6, 49, 83, 121, 92, 178, 339, 367, 164, 165, 254, 359, 30, 313, 260, 26, 340, 19, 252, 322, 374, 220, 215, 102, 18, 101, 99, 240, 266, 301, 92, 334, 176, 373, 286, 297, 258, 307, 70, 54, 94, 279, 63, 65, 59, 294, 381, 289, 113, 111, 186, 194, 278, 40, 137, 318, 92, 334, 171, 373, 354, 349, 209, 310, 250, 84, 44, 292, 43, 76, 358, 9, 57, 48, 368, 238, 82, 193, 13, 299, 192, 219, 92, 42, 8, 369, 61, 163, 183, 122, 338, 112, 156, 315, 50, 20, 319, 333, 129, 246, 73, 214, 56, 265, 172, 69, 336, 140, 134, 139, 10, 33, 276, 128, 224, 162, 372, 328, 157, 67, 376, 34, 46, 241, 79, 166, 71, 323, 382, 352, 325, 195, 221, 140, 144, 25, 10, 33, 378, 272, 52, 196, 245, 251, 213, 96, 4, 230, 296, 282, 288, 142, 93, 332, 152, 305, 287, 185, 138, 140, 316, 42, 8, 350, 298, 81, 107, 158, 155, 357, 360, 1, 123, 126, 291, 273, 177, 105, 377, 132, 62, 148, 66, 130, 100, 233, 188, 199, 120, 367, 21, 95, 118, 198, 36, 267, 249, 2, 303, 116, 304, 255, 204, 119, 190, 321, 31, 295, 184, 149, 174] + 46: [152, 314, 380, 207, 367, 344, 187, 242, 280, 33, 186, 166, 164, 340, 383, 92, 318, 317, 370, 278, 52, 229, 107, 47, 70, 5, 188, 58, 355, 110, 149, 301, 194, 320, 62, 108, 197, 42, 162, 158, 45, 206, 69, 342, 261, 257, 138, 281, 196, 54, 200, 27, 142, 256, 253, 170, 354, 319, 378, 18, 266, 227, 326, 298, 176, 103, 226, 360, 222, 265, 91, 285, 21, 77, 300, 349, 14, 147, 142, 256, 253, 16, 149, 335, 271, 203, 131, 115, 118, 156, 273, 17, 111, 263, 382, 241, 359, 26, 163, 341, 269, 123, 357, 36, 142, 58, 327, 207, 354, 66, 254, 44, 215, 376, 218, 180, 272, 198, 81, 160, 313, 7, 291, 171, 150, 31, 363, 0, 100, 213, 133, 139, 343, 311, 284, 104, 165, 345, 79, 83, 334, 369, 23, 59, 125, 35, 346, 20, 364, 375, 67, 330, 251, 179, 178, 231, 133, 139, 343, 249, 324, 319, 60, 374, 351, 195, 232, 316, 304, 321, 57, 328, 56, 322, 90, 86, 43, 127, 87, 116, 148, 220, 210, 64, 355, 110, 246, 312, 136, 146, 211, 130, 168, 192, 128, 205, 308, 24, 297, 293, 177, 305, 15, 11, 1, 306, 82, 88, 10, 19, 240, 96, 230, 140, 68, 76, 282, 155, 121, 295, 250, 157, 268, 78, 223, 277, 217, 339, 112, 94, 48, 212, 270, 302, 10, 64, 46, 311, 6, 209, 204, 2, 39, 22, 353, 259, 294, 216, 307, 290, 89, 71, 309, 184, 358, 258, 288, 248, 225, 252, 201, 233, 311, 25, 367, 154, 40, 303, 49, 8, 315, 4, 362, 193, 377, 97, 173, 243, 190, 262, 65, 275, 381, 84, 151, 356, 247, 352, 224, 238, 274, 12, 124, 3, 85, 120, 80, 50, 63, 235, 336, 144, 161, 74, 264, 371, 135, 366, 134, 279, 323, 61, 247, 352, 380, 51, 332, 335, 214, 219, 228, 143, 245, 109, 95, 172, 32, 289, 189, 159, 93, 361, 75, 310, 174, 296, 365, 122, 183, 34, 286, 191, 230, 140, 38, 199, 30, 117, 287, 13, 129, 267, 329, 169, 299, 153, 283, 98, 347, 236, 181, 29, 182, 175, 183, 34, 368, 239, 221, 66, 55, 338, 350, 260, 99, 37, 41, 237, 101, 141, 348, 333, 145, 234, 208, 72, 73, 185, 28, 325, 183, 352, 286, 202, 106, 106, 114, 113, 102, 372, 132, 137, 53, 292, 373, 255, 119, 9, 331, 244, 337, 126, 105, 276, 167, 379] + 47: [195, 44, 3, 29, 193, 127, 175, 188, 250, 208, 41, 298, 138, 106, 183, 335, 258, 185, 130, 275, 5, 133, 264, 167, 66, 288, 195, 352, 3, 29, 227, 353, 274, 235, 63, 42, 224, 203, 45, 39, 96, 225, 31, 49, 190, 282, 374, 16, 76, 206, 7, 156, 195, 352, 43, 158, 300, 14, 110, 229, 351, 262, 285, 255, 296, 27, 378, 163, 94, 329, 9, 365, 304, 100, 145, 217, 254, 326, 195, 352, 371, 61, 359, 316, 36, 349, 159, 135, 89, 77, 355, 252, 139, 303, 278, 32, 144, 302, 380, 86, 247, 256, 13, 209, 75, 44, 3, 87, 300, 305, 240, 238, 269, 155, 103, 33, 12, 210, 234, 147, 332, 157, 297, 72, 379, 361, 88, 125, 182, 369, 343, 162, 24, 375, 134, 286, 118, 153, 122, 67, 223, 184, 51, 233, 91, 70, 328, 2, 46, 48, 207, 317, 244, 161, 311, 236, 343, 162, 24, 119, 134, 137, 21, 342, 114, 104, 284, 323, 338, 173, 80, 321, 37, 107, 10, 120, 174, 30, 85, 295, 113, 146, 343, 44, 24, 35, 309, 90, 168, 40, 265, 95, 231, 65, 205, 117, 366, 28, 281, 260, 242, 325, 358, 222, 142, 283, 83, 62, 54, 192, 22, 158, 280, 196, 267, 367, 313, 69, 124, 197, 25, 180, 356, 126, 259, 132, 149, 216, 383, 372, 270, 232, 272, 293, 54, 143, 291, 170, 333, 314, 376, 59, 230, 84, 228, 140, 17, 52, 336, 71, 273, 248, 150, 266, 245, 220, 299, 6, 8, 360, 54, 171, 357, 279, 345, 137, 287, 363, 212, 219, 263, 172, 322, 164, 315, 213, 111, 341, 81, 19, 108, 58, 194, 116, 169, 214, 271, 154, 3, 109, 359, 368, 198, 346, 330, 26, 166, 73, 148, 4, 18, 211, 276, 20, 102, 55, 74, 105, 11, 82, 128, 191, 347, 154, 312, 87, 307, 320, 129, 123, 98, 68, 249, 199, 204, 53, 15, 121, 292, 310, 160, 152, 290, 38, 202, 221, 241, 56, 331, 181, 22, 1, 382, 294, 79, 218, 251, 377, 151, 50, 237, 340, 141, 348, 60, 115, 334, 57, 177, 277, 64, 99, 381, 373, 187, 181, 312, 34, 382, 257, 344, 327, 226, 93, 324, 364, 0, 112, 23, 289, 306, 200, 253, 308, 92, 339, 268, 78, 261, 246, 179, 136, 312, 87, 307, 215, 362, 131, 176, 301, 319, 370, 101, 354, 201, 243, 337, 186, 97, 189, 350, 178, 47, 165, 318, 239] + 48: [237, 145, 215, 226, 295, 58, 287, 139, 41, 232, 25, 120, 212, 347, 70, 138, 18, 98, 32, 310, 278, 244, 86, 251, 131, 200, 267, 65, 358, 44, 203, 78, 62, 363, 34, 312, 175, 73, 89, 218, 22, 36, 269, 59, 216, 92, 87, 309, 106, 296, 336, 209, 267, 117, 273, 239, 101, 33, 166, 240, 342, 205, 193, 340, 329, 91, 64, 96, 352, 137, 243, 194, 294, 238, 377, 54, 303, 308, 159, 77, 109, 19, 311, 28, 157, 254, 142, 300, 246, 88, 118, 318, 100, 72, 53, 144, 187, 3, 56, 225, 331, 332, 199, 245, 297, 102, 273, 276, 353, 133, 6, 5, 252, 319, 152, 233, 357, 168, 14, 49, 349, 346, 256, 326, 323, 188, 60, 183, 197, 136, 288, 102, 330, 239, 90, 381, 207, 208, 184, 320, 257, 370, 111, 315, 281, 119, 127, 169, 272, 29, 165, 148, 154, 383, 265, 304, 9, 270, 330, 116, 2, 141, 147, 344, 181, 368, 291, 285, 24, 4, 351, 204, 280, 94, 261, 214, 277, 350, 125, 46, 348, 114, 51, 26, 328, 234, 311, 78, 15, 359, 115, 290, 192, 112, 242, 360, 180, 97, 43, 292, 71, 67, 163, 149, 171, 176, 220, 250, 283, 128, 279, 132, 182, 268, 189, 177, 305, 229, 179, 75, 31, 42, 339, 81, 198, 274, 178, 95, 213, 228, 345, 23, 364, 334, 283, 128, 301, 99, 353, 141, 375, 223, 380, 378, 299, 190, 170, 79, 202, 11, 84, 107, 82, 366, 335, 66, 8, 262, 355, 129, 76, 271, 328, 236, 52, 83, 15, 372, 325, 156, 7, 123, 341, 57, 317, 146, 367, 35, 39, 313, 27, 361, 282, 248, 333, 10, 76, 271, 328, 343, 52, 381, 104, 327, 134, 155, 50, 247, 221, 230, 113, 158, 16, 151, 47, 69, 37, 356, 153, 124, 167, 1, 20, 271, 140, 132, 284, 21, 62, 38, 316, 210, 322, 45, 255, 13, 379, 253, 211, 195, 108, 135, 61, 201, 298, 68, 306, 314, 249, 17, 140, 132, 284, 85, 263, 48, 185, 222, 130, 0, 160, 275, 241, 289, 40, 206, 173, 93, 143, 376, 260, 365, 258, 30, 266, 17, 150, 121, 74, 85, 263, 264, 186, 219, 161, 324, 162, 231, 307, 63, 374, 259, 382, 217, 105, 110, 338, 122, 103, 196, 266, 17, 140, 276, 101, 164, 369, 235, 354, 371, 293, 174, 321, 172, 80, 302, 12, 55, 286, 373, 224, 191, 362, 126, 337, 227] + 49: [374, 268, 297, 150, 278, 261, 380, 269, 186, 235, 1, 382, 112, 81, 225, 309, 14, 280, 285, 299, 185, 7, 363, 383, 26, 188, 374, 346, 209, 157, 66, 153, 199, 289, 381, 5, 97, 245, 365, 163, 55, 62, 47, 85, 282, 354, 324, 87, 367, 11, 288, 317, 79, 6, 305, 181, 212, 9, 193, 91, 8, 314, 362, 56, 240, 146, 139, 338, 72, 176, 284, 145, 364, 165, 266, 249, 369, 337, 79, 268, 297, 19, 25, 40, 251, 74, 255, 195, 254, 310, 224, 296, 82, 98, 351, 119, 31, 113, 232, 147, 96, 215, 247, 64, 210, 6, 65, 158, 241, 90, 60, 220, 120, 48, 366, 368, 196, 41, 295, 43, 359, 0, 12, 143, 168, 93, 334, 152, 129, 322, 233, 319, 304, 213, 202, 180, 148, 132, 312, 122, 73, 28, 29, 34, 10, 33, 172, 61, 46, 154, 111, 166, 161, 141, 230, 123, 360, 83, 75, 293, 116, 275, 259, 76, 30, 94, 174, 23, 138, 376, 216, 178, 211, 53, 151, 22, 125, 341, 18, 236, 373, 356, 360, 294, 15, 126, 353, 270, 292, 187, 45, 58, 250, 54, 214, 323, 308, 36, 156, 69, 204, 361, 357, 315, 162, 263, 258, 38, 302, 321, 59, 157, 78, 9, 193, 260, 175, 333, 279, 86, 140, 20, 273, 307, 320, 378, 84, 342, 316, 27, 21, 99, 347, 349, 302, 321, 234, 136, 271, 201, 325, 246, 194, 164, 336, 137, 89, 222, 371, 330, 103, 226, 206, 283, 274, 13, 159, 217, 203, 375, 302, 272, 155, 126, 313, 242, 262, 286, 183, 109, 169, 80, 290, 160, 179, 352, 239, 358, 124, 190, 88, 51, 42, 207, 252, 191, 302, 272, 257, 253, 313, 335, 39, 243, 331, 144, 328, 221, 170, 4, 173, 67, 244, 118, 106, 256, 343, 57, 231, 379, 370, 92, 302, 149, 155, 253, 202, 242, 44, 107, 264, 298, 340, 311, 198, 248, 303, 134, 114, 71, 49, 101, 326, 167, 237, 291, 32, 135, 302, 37, 75, 253, 126, 276, 177, 133, 332, 131, 63, 223, 277, 77, 301, 227, 339, 117, 192, 52, 348, 229, 102, 281, 3, 68, 184, 321, 305, 115, 105, 110, 218, 344, 100, 95, 17, 70, 128, 189, 238, 267, 345, 355, 200, 108, 182, 372, 287, 171, 219, 300, 197, 321, 297, 19, 271, 208, 251, 306, 329, 121, 205, 16, 318, 127, 327, 35, 142, 130, 2, 228, 104, 377, 265, 350, 50, 24] + 50: [57, 166, 315, 370, 27, 129, 0, 193, 224, 96, 71, 43, 94, 310, 99, 300, 178, 124, 156, 354, 323, 21, 47, 167, 302, 111, 33, 166, 367, 361, 312, 1, 128, 51, 121, 122, 117, 183, 342, 326, 152, 28, 243, 177, 46, 212, 346, 76, 157, 221, 317, 136, 80, 229, 38, 262, 287, 350, 338, 314, 335, 41, 306, 102, 250, 382, 91, 105, 202, 4, 372, 36, 138, 209, 72, 279, 84, 332, 29, 181, 376, 218, 214, 26, 329, 331, 159, 363, 195, 194, 174, 234, 242, 223, 297, 339, 349, 171, 162, 381, 265, 54, 254, 345, 29, 67, 14, 69, 214, 282, 296, 119, 298, 274, 203, 348, 180, 85, 145, 340, 292, 204, 248, 330, 147, 10, 173, 227, 115, 257, 86, 49, 376, 69, 366, 26, 45, 88, 270, 6, 75, 263, 208, 267, 205, 5, 37, 259, 12, 355, 286, 201, 103, 144, 59, 50, 86, 49, 32, 188, 175, 129, 199, 256, 184, 313, 228, 240, 169, 73, 163, 108, 135, 106, 63, 258, 112, 83, 190, 161, 66, 55, 86, 181, 333, 35, 146, 1, 231, 155, 213, 196, 149, 219, 319, 92, 294, 356, 17, 344, 238, 68, 11, 101, 104, 261, 364, 222, 25, 140, 97, 325, 373, 3, 231, 343, 16, 353, 375, 359, 206, 303, 322, 143, 347, 123, 305, 90, 65, 40, 245, 95, 307, 230, 235, 137, 32, 188, 27, 3, 217, 341, 271, 301, 48, 134, 189, 107, 255, 251, 39, 289, 7, 383, 324, 13, 207, 260, 118, 264, 268, 109, 233, 273, 110, 377, 128, 295, 187, 132, 19, 116, 269, 378, 277, 113, 236, 351, 200, 318, 179, 176, 328, 309, 131, 93, 268, 109, 210, 215, 56, 18, 62, 22, 253, 244, 239, 379, 197, 74, 160, 185, 362, 42, 320, 293, 285, 30, 360, 225, 52, 241, 268, 311, 315, 325, 321, 9, 327, 358, 130, 164, 380, 53, 139, 371, 2, 357, 337, 281, 141, 237, 87, 60, 70, 158, 44, 283, 226, 316, 352, 232, 125, 350, 304, 79, 249, 133, 186, 290, 252, 64, 20, 278, 127, 153, 34, 89, 291, 246, 216, 266, 182, 23, 100, 299, 210, 215, 56, 18, 192, 114, 15, 24, 165, 154, 276, 151, 78, 211, 191, 288, 198, 150, 82, 61, 8, 120, 368, 365, 100, 316, 284, 232, 321, 280, 220, 334, 272, 275, 58, 168, 247, 81, 126, 172, 369, 170, 308, 148, 98, 77, 31, 142, 336, 374] + 51: [322, 206, 323, 102, 356, 140, 87, 168, 346, 118, 344, 86, 351, 119, 3, 317, 213, 80, 307, 25, 19, 88, 120, 302, 21, 327, 108, 178, 186, 287, 337, 126, 124, 188, 130, 220, 81, 254, 59, 309, 236, 265, 315, 51, 159, 197, 230, 215, 376, 63, 225, 2, 31, 110, 99, 102, 122, 229, 60, 283, 92, 259, 331, 107, 366, 368, 10, 84, 196, 191, 78, 53, 15, 296, 367, 179, 76, 138, 137, 200, 186, 284, 337, 126, 95, 330, 243, 23, 382, 341, 190, 34, 48, 205, 115, 223, 319, 43, 184, 199, 174, 29, 22, 226, 137, 200, 186, 102, 127, 343, 40, 129, 11, 294, 47, 359, 222, 204, 224, 306, 45, 379, 55, 58, 57, 354, 227, 71, 256, 202, 373, 6, 24, 17, 228, 229, 203, 105, 263, 46, 161, 345, 79, 321, 362, 289, 312, 338, 363, 112, 218, 201, 0, 187, 173, 249, 373, 110, 24, 287, 136, 300, 301, 250, 246, 281, 308, 74, 192, 91, 164, 324, 349, 328, 298, 336, 214, 14, 277, 117, 36, 255, 375, 210, 208, 12, 144, 358, 260, 233, 269, 85, 303, 237, 181, 207, 234, 320, 381, 89, 353, 109, 1, 4, 133, 293, 155, 139, 375, 101, 186, 113, 264, 82, 286, 177, 49, 143, 212, 56, 18, 160, 50, 311, 334, 27, 93, 378, 66, 9, 70, 169, 267, 261, 106, 210, 357, 52, 180, 65, 286, 153, 5, 8, 292, 279, 42, 251, 195, 364, 297, 350, 340, 339, 96, 39, 128, 274, 131, 121, 221, 372, 111, 193, 72, 67, 152, 253, 134, 35, 271, 268, 360, 166, 44, 313, 100, 54, 333, 238, 68, 257, 305, 147, 150, 290, 221, 372, 16, 123, 144, 82, 32, 304, 247, 13, 285, 380, 209, 90, 239, 262, 258, 194, 278, 316, 116, 272, 154, 318, 244, 156, 189, 372, 24, 182, 342, 163, 270, 146, 314, 216, 245, 198, 219, 252, 75, 171, 332, 30, 242, 240, 148, 28, 355, 175, 231, 291, 73, 37, 24, 17, 342, 211, 235, 325, 103, 374, 97, 172, 310, 370, 41, 157, 365, 170, 273, 61, 83, 7, 77, 185, 377, 326, 241, 371, 323, 12, 64, 62, 135, 282, 26, 299, 275, 142, 280, 248, 361, 69, 94, 114, 145, 232, 149, 158, 132, 266, 347, 348, 241, 37, 98, 208, 176, 163, 104, 295, 335, 20, 183, 162, 329, 33, 141, 369, 383, 288, 276, 167, 125, 165, 217, 352, 38, 151] + 52: [52, 267, 71, 137, 345, 169, 68, 357, 98, 309, 5, 296, 67, 3, 324, 326, 242, 244, 150, 143, 96, 358, 203, 65, 376, 305, 186, 99, 71, 137, 206, 163, 205, 128, 54, 171, 74, 223, 75, 53, 333, 350, 58, 224, 198, 188, 343, 138, 329, 144, 373, 247, 327, 99, 71, 73, 321, 48, 288, 196, 32, 379, 127, 23, 364, 319, 55, 45, 177, 251, 249, 49, 335, 232, 185, 12, 199, 204, 121, 99, 246, 97, 271, 298, 13, 192, 266, 291, 105, 315, 6, 301, 287, 27, 383, 84, 220, 118, 344, 363, 9, 202, 272, 16, 92, 99, 157, 104, 214, 90, 382, 259, 64, 228, 221, 281, 280, 72, 22, 235, 352, 238, 47, 28, 340, 275, 158, 0, 24, 176, 102, 381, 157, 165, 290, 209, 62, 245, 95, 312, 222, 173, 134, 113, 213, 93, 89, 282, 179, 356, 313, 229, 314, 15, 86, 348, 102, 320, 157, 362, 289, 82, 41, 56, 252, 133, 339, 160, 233, 154, 66, 40, 277, 194, 230, 380, 4, 353, 131, 159, 264, 212, 102, 371, 151, 104, 85, 184, 183, 219, 210, 37, 117, 178, 369, 361, 374, 161, 2, 294, 347, 197, 111, 147, 132, 366, 254, 38, 307, 381, 334, 104, 323, 25, 7, 269, 355, 108, 256, 215, 116, 17, 69, 109, 110, 59, 368, 141, 43, 274, 187, 19, 295, 46, 120, 381, 334, 103, 34, 211, 33, 80, 236, 325, 262, 83, 243, 76, 279, 268, 57, 365, 129, 101, 18, 293, 42, 273, 360, 218, 231, 381, 334, 250, 195, 25, 78, 168, 87, 257, 191, 124, 248, 115, 337, 308, 237, 148, 136, 156, 126, 304, 79, 11, 164, 106, 332, 217, 334, 103, 174, 193, 172, 10, 149, 155, 285, 30, 367, 351, 208, 306, 200, 21, 139, 322, 226, 276, 297, 241, 1, 370, 61, 283, 130, 97, 239, 263, 60, 140, 354, 35, 234, 341, 292, 153, 316, 225, 270, 152, 51, 278, 81, 112, 36, 122, 240, 375, 216, 283, 182, 97, 239, 209, 346, 260, 8, 88, 286, 359, 300, 331, 39, 253, 284, 207, 330, 310, 299, 135, 261, 142, 317, 180, 107, 107, 63, 165, 336, 48, 190, 26, 50, 70, 166, 227, 146, 328, 170, 119, 201, 20, 318, 342, 114, 302, 145, 338, 14, 175, 107, 372, 31, 165, 336, 77, 205, 44, 349, 123, 378, 167, 29, 377, 91, 125, 258, 189, 100, 303, 311, 181, 255, 265, 94, 162] + 53: [230, 247, 296, 354, 347, 32, 157, 162, 50, 262, 346, 52, 60, 146, 128, 327, 278, 97, 195, 43, 29, 163, 374, 187, 318, 185, 336, 73, 38, 109, 250, 32, 291, 376, 95, 241, 116, 151, 76, 353, 339, 105, 19, 254, 7, 53, 331, 229, 356, 124, 99, 167, 308, 73, 38, 16, 330, 186, 360, 121, 134, 168, 383, 335, 89, 293, 196, 57, 28, 294, 108, 137, 48, 193, 41, 190, 304, 210, 217, 73, 238, 292, 295, 132, 145, 130, 284, 25, 381, 373, 10, 143, 35, 166, 79, 182, 54, 9, 85, 366, 161, 181, 245, 312, 75, 270, 287, 173, 224, 266, 160, 119, 125, 200, 183, 313, 281, 282, 178, 174, 30, 59, 234, 131, 100, 303, 104, 213, 290, 56, 8, 13, 350, 311, 273, 227, 351, 332, 253, 219, 77, 239, 150, 362, 111, 276, 136, 218, 87, 380, 1, 147, 148, 212, 58, 265, 68, 66, 334, 164, 179, 155, 268, 225, 319, 88, 205, 152, 126, 102, 248, 371, 255, 299, 21, 14, 199, 372, 135, 286, 307, 90, 240, 317, 23, 354, 347, 4, 202, 26, 326, 94, 361, 78, 149, 226, 233, 96, 358, 364, 201, 80, 82, 64, 382, 33, 344, 84, 240, 317, 23, 365, 214, 42, 86, 261, 20, 329, 141, 197, 235, 357, 369, 370, 123, 198, 275, 264, 310, 34, 93, 204, 223, 22, 240, 13, 350, 46, 311, 4, 309, 81, 24, 256, 338, 324, 349, 359, 206, 375, 215, 242, 352, 246, 83, 106, 267, 165, 314, 18, 110, 110, 38, 322, 295, 302, 180, 271, 269, 62, 341, 49, 259, 67, 36, 158, 236, 55, 257, 289, 63, 301, 288, 61, 191, 316, 110, 110, 296, 109, 345, 170, 156, 144, 171, 325, 231, 37, 15, 122, 216, 189, 117, 232, 45, 305, 342, 184, 263, 98, 71, 153, 110, 297, 320, 142, 250, 92, 6, 221, 31, 3, 188, 172, 159, 44, 120, 321, 333, 251, 337, 283, 39, 74, 211, 27, 11, 114, 110, 243, 129, 280, 363, 228, 279, 103, 107, 12, 70, 277, 176, 328, 340, 207, 40, 138, 258, 323, 177, 113, 260, 343, 377, 140, 110, 203, 287, 16, 379, 186, 209, 285, 368, 139, 69, 300, 47, 175, 2, 169, 133, 220, 0, 249, 298, 17, 208, 274, 244, 252, 110, 222, 334, 16, 330, 118, 194, 237, 127, 5, 112, 378, 72, 315, 115, 65, 306, 101, 154, 367, 51, 348, 91, 192, 355, 272] + 54: [295, 42, 258, 260, 280, 306, 330, 117, 70, 218, 17, 34, 376, 355, 68, 293, 347, 141, 238, 247, 31, 248, 303, 188, 165, 127, 370, 338, 339, 155, 299, 95, 221, 3, 10, 196, 225, 159, 184, 318, 144, 304, 364, 128, 192, 110, 331, 66, 292, 112, 78, 106, 370, 147, 258, 240, 240, 53, 119, 75, 121, 158, 91, 272, 105, 97, 373, 201, 62, 166, 371, 14, 211, 56, 270, 222, 207, 354, 370, 83, 39, 214, 133, 172, 352, 228, 101, 197, 323, 92, 21, 131, 366, 179, 96, 244, 12, 224, 268, 343, 210, 241, 297, 13, 370, 90, 42, 369, 30, 324, 93, 314, 276, 47, 220, 190, 170, 49, 264, 209, 167, 23, 251, 59, 29, 46, 149, 138, 344, 232, 284, 328, 339, 216, 230, 72, 187, 329, 181, 64, 50, 182, 157, 365, 85, 163, 18, 175, 139, 235, 261, 60, 154, 169, 200, 36, 284, 189, 11, 164, 280, 286, 57, 7, 156, 129, 357, 279, 186, 55, 69, 137, 377, 300, 213, 308, 233, 263, 65, 77, 340, 148, 337, 319, 11, 82, 38, 341, 290, 372, 375, 310, 325, 152, 54, 135, 298, 313, 353, 43, 271, 74, 236, 102, 285, 291, 383, 320, 380, 71, 267, 80, 38, 281, 305, 180, 288, 99, 94, 321, 239, 1, 278, 111, 173, 191, 243, 234, 269, 359, 4, 379, 130, 160, 153, 283, 223, 231, 20, 174, 311, 16, 27, 215, 335, 109, 103, 100, 171, 136, 81, 275, 334, 145, 249, 203, 360, 2, 349, 336, 296, 283, 223, 231, 230, 8, 25, 86, 332, 161, 84, 24, 237, 302, 48, 107, 114, 262, 63, 346, 52, 89, 307, 256, 342, 205, 363, 363, 155, 257, 98, 286, 87, 40, 143, 115, 45, 185, 381, 177, 246, 122, 162, 273, 88, 6, 206, 362, 51, 134, 19, 0, 363, 253, 356, 164, 348, 176, 120, 118, 132, 333, 5, 265, 125, 183, 73, 242, 146, 9, 194, 168, 274, 226, 35, 28, 202, 361, 363, 294, 356, 164, 348, 208, 44, 309, 41, 32, 124, 217, 204, 368, 255, 327, 151, 266, 245, 178, 289, 227, 116, 350, 358, 378, 363, 294, 287, 150, 250, 254, 282, 26, 76, 113, 277, 382, 312, 67, 61, 326, 195, 351, 229, 219, 79, 108, 104, 252, 193, 123, 363, 294, 339, 155, 299, 95, 345, 317, 58, 322, 198, 142, 140, 315, 301, 22, 33, 316, 259, 15, 367, 199, 374, 37, 126, 212] + 55: [56, 4, 357, 307, 381, 323, 363, 99, 179, 271, 337, 105, 260, 136, 332, 101, 294, 365, 268, 209, 125, 166, 282, 110, 205, 109, 56, 98, 340, 111, 192, 74, 65, 291, 320, 366, 197, 274, 27, 135, 119, 157, 214, 127, 219, 55, 120, 183, 262, 220, 229, 245, 326, 309, 340, 16, 381, 323, 70, 200, 349, 221, 112, 352, 377, 306, 318, 84, 314, 367, 359, 297, 150, 203, 155, 142, 379, 196, 113, 174, 218, 238, 227, 74, 29, 249, 12, 164, 170, 348, 47, 42, 277, 85, 313, 258, 372, 118, 50, 139, 330, 32, 148, 96, 72, 174, 62, 252, 303, 48, 301, 49, 44, 216, 5, 350, 246, 178, 122, 171, 31, 232, 30, 35, 68, 177, 334, 57, 167, 186, 159, 174, 308, 80, 123, 247, 29, 223, 279, 316, 189, 190, 251, 228, 19, 128, 198, 61, 15, 168, 275, 162, 298, 230, 24, 305, 202, 151, 7, 116, 90, 273, 76, 199, 138, 156, 233, 375, 231, 240, 23, 33, 87, 37, 208, 129, 58, 77, 364, 345, 18, 204, 202, 174, 11, 339, 299, 347, 130, 293, 8, 300, 163, 14, 361, 79, 346, 94, 22, 281, 126, 93, 66, 248, 21, 362, 283, 222, 152, 355, 146, 370, 73, 48, 269, 341, 60, 368, 102, 95, 378, 194, 185, 255, 212, 237, 97, 321, 374, 369, 175, 195, 9, 310, 28, 235, 234, 39, 267, 347, 180, 71, 114, 26, 239, 210, 265, 360, 287, 149, 296, 143, 173, 137, 315, 376, 92, 288, 25, 144, 88, 81, 98, 311, 328, 78, 130, 134, 3, 0, 172, 276, 224, 187, 184, 207, 264, 133, 225, 243, 161, 242, 191, 83, 59, 34, 88, 81, 20, 165, 123, 121, 115, 40, 333, 86, 259, 284, 236, 286, 257, 52, 302, 188, 335, 104, 147, 280, 295, 100, 131, 69, 327, 256, 11, 89, 299, 382, 180, 226, 176, 124, 206, 289, 217, 54, 263, 254, 153, 160, 356, 336, 64, 261, 244, 266, 51, 250, 154, 154, 308, 182, 41, 353, 269, 2, 272, 213, 46, 325, 343, 67, 10, 358, 82, 1, 373, 338, 354, 141, 169, 158, 351, 215, 154, 43, 146, 116, 90, 211, 76, 132, 290, 383, 344, 371, 380, 53, 322, 193, 63, 342, 319, 201, 145, 324, 36, 75, 45, 285, 154, 270, 20, 80, 6, 304, 115, 106, 331, 312, 17, 91, 329, 140, 241, 278, 117, 103, 253, 38, 107, 13, 181, 108, 292, 317] + 56: [279, 82, 15, 155, 259, 189, 251, 302, 332, 223, 369, 38, 295, 268, 138, 129, 378, 353, 347, 57, 70, 204, 239, 7, 380, 208, 100, 82, 15, 186, 5, 304, 161, 99, 159, 192, 27, 312, 252, 374, 164, 141, 95, 96, 290, 139, 144, 211, 176, 52, 25, 146, 177, 178, 294, 88, 340, 196, 125, 41, 80, 207, 359, 278, 206, 22, 34, 227, 210, 183, 112, 330, 130, 60, 234, 377, 102, 109, 231, 126, 273, 116, 5, 89, 333, 284, 288, 150, 58, 193, 181, 12, 11, 286, 122, 145, 293, 291, 170, 254, 203, 59, 111, 173, 231, 191, 294, 53, 87, 147, 6, 292, 327, 97, 197, 285, 35, 179, 33, 123, 339, 341, 375, 16, 29, 24, 215, 32, 165, 74, 231, 199, 213, 163, 371, 304, 46, 316, 77, 101, 300, 10, 105, 280, 50, 297, 348, 314, 307, 37, 67, 47, 382, 365, 318, 85, 231, 178, 220, 296, 259, 356, 306, 140, 381, 244, 73, 250, 136, 23, 255, 209, 94, 55, 124, 276, 43, 152, 68, 336, 235, 142, 8, 260, 213, 98, 30, 147, 117, 39, 154, 118, 103, 281, 21, 317, 151, 282, 62, 257, 200, 379, 222, 373, 256, 45, 174, 3, 195, 44, 294, 107, 169, 263, 363, 48, 245, 63, 182, 137, 305, 355, 110, 56, 190, 84, 172, 289, 266, 258, 272, 216, 194, 90, 195, 260, 220, 303, 106, 9, 308, 275, 329, 383, 233, 160, 31, 354, 243, 185, 271, 298, 226, 175, 86, 337, 325, 217, 247, 143, 195, 260, 220, 267, 322, 236, 264, 219, 362, 72, 344, 148, 135, 331, 1, 42, 221, 119, 228, 367, 49, 113, 349, 326, 171, 224, 311, 44, 240, 351, 106, 156, 162, 71, 14, 283, 376, 253, 92, 121, 133, 364, 313, 218, 310, 372, 64, 287, 370, 335, 237, 315, 321, 265, 240, 225, 241, 20, 187, 18, 19, 309, 205, 40, 338, 269, 153, 248, 115, 262, 132, 232, 350, 131, 26, 36, 51, 229, 321, 265, 240, 225, 241, 249, 104, 188, 66, 120, 0, 342, 198, 166, 13, 127, 201, 180, 4, 319, 361, 93, 212, 261, 114, 202, 343, 242, 81, 184, 30, 75, 346, 320, 357, 168, 334, 128, 324, 149, 214, 368, 360, 54, 323, 65, 78, 157, 108, 366, 61, 76, 167, 242, 240, 158, 340, 75, 328, 2, 83, 238, 246, 69, 79, 230, 17, 270, 299, 134, 28, 274, 352, 345, 358, 277, 91, 301] + 57: [157, 220, 140, 101, 94, 170, 154, 22, 123, 91, 341, 46, 183, 279, 348, 216, 81, 379, 236, 126, 36, 364, 115, 82, 131, 234, 231, 295, 198, 235, 286, 130, 169, 79, 163, 144, 51, 177, 54, 225, 210, 351, 323, 355, 265, 188, 194, 76, 10, 278, 229, 202, 231, 121, 7, 148, 72, 335, 217, 345, 80, 365, 213, 83, 184, 13, 68, 282, 29, 261, 299, 32, 222, 239, 212, 52, 181, 301, 93, 295, 156, 148, 124, 344, 302, 151, 354, 179, 176, 113, 147, 95, 329, 28, 330, 332, 200, 270, 375, 67, 258, 221, 245, 304, 11, 259, 187, 128, 72, 53, 154, 64, 185, 338, 214, 27, 116, 328, 226, 353, 18, 145, 24, 378, 192, 48, 218, 129, 153, 262, 342, 259, 63, 65, 274, 324, 350, 117, 347, 43, 134, 173, 92, 106, 352, 155, 178, 290, 255, 307, 256, 370, 291, 327, 37, 367, 159, 146, 187, 205, 286, 141, 302, 88, 35, 99, 228, 96, 38, 358, 149, 296, 215, 247, 59, 0, 242, 103, 271, 292, 105, 19, 14, 314, 368, 25, 257, 268, 267, 317, 75, 316, 40, 356, 349, 380, 313, 90, 98, 41, 33, 333, 66, 308, 264, 193, 69, 275, 14, 146, 57, 366, 207, 125, 74, 197, 251, 161, 322, 277, 377, 112, 166, 337, 162, 26, 45, 168, 383, 78, 142, 336, 233, 143, 230, 133, 89, 25, 108, 320, 150, 127, 238, 167, 73, 312, 21, 12, 196, 362, 132, 175, 252, 219, 263, 107, 266, 269, 77, 376, 276, 254, 89, 205, 207, 125, 74, 283, 250, 373, 346, 42, 289, 47, 71, 209, 17, 49, 122, 357, 50, 232, 339, 340, 305, 343, 276, 254, 246, 104, 223, 170, 6, 318, 272, 39, 9, 298, 208, 284, 3, 23, 288, 237, 303, 241, 109, 61, 310, 30, 15, 285, 276, 254, 246, 25, 331, 8, 60, 309, 44, 319, 70, 152, 111, 119, 2, 199, 371, 97, 135, 281, 374, 138, 114, 294, 34, 56, 31, 254, 165, 326, 359, 137, 204, 160, 306, 58, 297, 20, 102, 325, 136, 190, 334, 363, 248, 4, 361, 287, 244, 189, 227, 293, 381, 254, 85, 326, 94, 320, 174, 84, 182, 172, 110, 249, 203, 120, 171, 369, 186, 240, 5, 1, 224, 273, 100, 206, 280, 372, 158, 16, 165, 326, 331, 118, 300, 180, 62, 253, 243, 191, 382, 55, 164, 321, 311, 315, 211, 87, 260, 139, 360, 195, 201, 86] + 58: [263, 79, 348, 245, 93, 199, 368, 342, 12, 329, 229, 258, 250, 121, 362, 228, 293, 219, 95, 359, 349, 34, 78, 165, 252, 204, 248, 61, 210, 211, 177, 214, 175, 47, 255, 185, 331, 181, 83, 0, 172, 43, 113, 133, 87, 128, 223, 117, 188, 343, 321, 58, 156, 81, 205, 194, 49, 355, 232, 367, 316, 274, 239, 244, 333, 240, 148, 2, 19, 94, 142, 221, 103, 246, 74, 310, 370, 308, 156, 81, 205, 194, 312, 351, 324, 272, 315, 73, 281, 62, 53, 149, 357, 334, 353, 297, 261, 41, 336, 304, 286, 291, 80, 150, 220, 126, 305, 311, 312, 67, 295, 189, 60, 66, 139, 350, 306, 302, 144, 21, 70, 237, 317, 54, 46, 96, 174, 140, 71, 340, 220, 56, 366, 373, 298, 358, 152, 119, 260, 197, 375, 325, 339, 131, 253, 278, 50, 118, 102, 282, 51, 39, 88, 212, 147, 10, 220, 56, 366, 222, 122, 114, 45, 16, 104, 198, 192, 143, 124, 183, 231, 161, 283, 170, 24, 163, 290, 159, 267, 363, 335, 134, 220, 81, 259, 222, 122, 72, 68, 42, 106, 352, 55, 82, 227, 378, 288, 327, 8, 230, 372, 86, 146, 145, 301, 320, 164, 168, 275, 76, 305, 311, 235, 26, 112, 341, 130, 285, 187, 284, 215, 135, 13, 318, 35, 180, 296, 254, 337, 137, 15, 91, 309, 209, 275, 126, 273, 101, 9, 114, 196, 191, 132, 171, 202, 266, 90, 332, 208, 345, 182, 338, 138, 276, 374, 216, 17, 361, 3, 184, 77, 151, 344, 245, 177, 256, 116, 141, 277, 224, 48, 129, 236, 63, 300, 11, 265, 136, 100, 264, 247, 120, 365, 89, 279, 280, 377, 377, 92, 271, 299, 270, 36, 85, 111, 328, 1, 59, 155, 75, 379, 371, 115, 105, 360, 268, 44, 369, 217, 233, 292, 364, 377, 377, 7, 14, 22, 69, 376, 203, 20, 242, 251, 207, 234, 37, 176, 65, 225, 243, 294, 157, 200, 153, 30, 218, 125, 33, 377, 377, 356, 98, 322, 169, 40, 18, 97, 28, 380, 99, 238, 110, 241, 347, 287, 173, 29, 330, 269, 32, 52, 23, 166, 201, 377, 377, 27, 245, 298, 289, 262, 31, 4, 190, 381, 383, 326, 382, 186, 123, 195, 64, 5, 303, 167, 109, 127, 323, 319, 38, 377, 249, 92, 98, 206, 257, 178, 6, 162, 307, 314, 25, 160, 346, 154, 107, 179, 226, 213, 354, 84, 57, 193, 313, 158, 108] + 59: [332, 197, 5, 363, 173, 31, 324, 243, 109, 304, 162, 376, 120, 269, 51, 194, 71, 50, 145, 44, 307, 237, 141, 335, 139, 341, 140, 313, 370, 260, 247, 234, 65, 281, 176, 12, 334, 174, 236, 369, 228, 299, 138, 136, 74, 183, 119, 216, 219, 378, 122, 15, 140, 97, 370, 195, 204, 346, 42, 255, 86, 254, 383, 22, 214, 238, 6, 242, 135, 325, 89, 336, 241, 316, 239, 39, 152, 271, 73, 256, 7, 70, 340, 18, 186, 80, 175, 161, 62, 60, 356, 259, 337, 199, 63, 184, 153, 24, 273, 283, 125, 367, 351, 308, 209, 250, 7, 202, 311, 331, 328, 23, 212, 35, 163, 143, 258, 177, 365, 292, 64, 231, 37, 88, 102, 319, 305, 205, 72, 310, 209, 250, 7, 202, 298, 130, 372, 144, 223, 40, 267, 338, 47, 366, 353, 156, 41, 34, 359, 132, 166, 257, 1, 272, 345, 189, 113, 170, 94, 251, 66, 221, 65, 309, 342, 159, 280, 218, 222, 198, 68, 206, 227, 322, 343, 11, 118, 358, 270, 264, 167, 235, 182, 170, 94, 195, 66, 352, 265, 232, 133, 279, 181, 142, 54, 354, 289, 96, 43, 124, 330, 210, 296, 98, 318, 375, 55, 268, 327, 25, 226, 363, 340, 150, 9, 26, 164, 57, 211, 245, 179, 180, 131, 32, 287, 134, 21, 191, 321, 165, 350, 87, 116, 90, 327, 25, 229, 59, 220, 185, 38, 262, 91, 160, 252, 373, 339, 348, 349, 48, 380, 157, 67, 275, 53, 312, 56, 244, 52, 4, 327, 25, 229, 59, 381, 76, 377, 151, 45, 85, 248, 315, 69, 344, 2, 0, 333, 217, 17, 249, 357, 303, 14, 171, 99, 111, 327, 170, 379, 200, 294, 352, 295, 266, 290, 127, 302, 19, 329, 203, 155, 274, 149, 208, 49, 261, 284, 108, 282, 77, 126, 215, 33, 46, 379, 173, 371, 234, 172, 58, 128, 168, 169, 20, 355, 92, 13, 314, 246, 187, 93, 277, 103, 301, 240, 121, 83, 154, 30, 224, 229, 95, 220, 288, 317, 178, 278, 81, 320, 225, 10, 137, 123, 107, 27, 84, 28, 106, 148, 82, 360, 78, 190, 188, 105, 224, 233, 260, 374, 291, 300, 196, 114, 297, 100, 29, 158, 146, 101, 362, 193, 253, 207, 115, 201, 368, 192, 347, 323, 117, 285, 75, 147, 200, 263, 346, 213, 382, 112, 16, 364, 286, 306, 361, 36, 61, 110, 129, 326, 3, 230, 8, 276, 293, 104, 79] + 60: [284, 158, 243, 126, 48, 230, 294, 119, 122, 27, 277, 259, 11, 56, 360, 85, 79, 296, 67, 202, 41, 160, 86, 4, 73, 22, 102, 158, 195, 83, 339, 153, 28, 15, 59, 271, 248, 221, 80, 304, 52, 273, 349, 64, 125, 330, 258, 29, 350, 177, 101, 154, 94, 257, 47, 139, 174, 155, 76, 112, 213, 140, 189, 132, 110, 38, 323, 250, 300, 371, 325, 228, 137, 147, 135, 227, 226, 288, 123, 124, 303, 244, 174, 185, 205, 354, 161, 235, 20, 1, 88, 342, 107, 285, 322, 368, 270, 13, 198, 229, 17, 292, 136, 299, 123, 257, 220, 60, 265, 208, 337, 290, 272, 246, 362, 240, 344, 25, 90, 293, 95, 276, 156, 262, 224, 379, 159, 103, 352, 201, 123, 257, 220, 60, 310, 297, 108, 46, 245, 172, 75, 37, 382, 274, 186, 146, 100, 286, 105, 361, 50, 374, 301, 340, 218, 207, 236, 124, 129, 347, 78, 302, 34, 329, 231, 99, 152, 51, 162, 57, 141, 7, 192, 219, 307, 252, 261, 298, 326, 71, 321, 355, 121, 124, 115, 126, 222, 65, 2, 98, 203, 241, 58, 217, 267, 176, 171, 149, 341, 256, 282, 251, 42, 68, 175, 32, 87, 317, 313, 351, 128, 133, 199, 247, 35, 166, 232, 364, 151, 6, 5, 3, 26, 30, 182, 18, 363, 324, 216, 111, 249, 163, 10, 157, 313, 351, 260, 83, 199, 312, 74, 309, 223, 238, 0, 36, 9, 377, 346, 91, 130, 144, 168, 23, 145, 211, 306, 164, 77, 214, 313, 351, 170, 39, 40, 167, 319, 365, 278, 253, 84, 358, 237, 215, 21, 33, 357, 12, 333, 209, 19, 97, 43, 173, 89, 210, 313, 124, 16, 113, 212, 194, 49, 353, 184, 82, 283, 93, 92, 305, 142, 200, 106, 380, 376, 45, 280, 356, 254, 289, 242, 375, 313, 124, 16, 133, 316, 120, 327, 311, 131, 169, 336, 183, 24, 279, 53, 31, 314, 225, 383, 328, 54, 378, 281, 138, 109, 63, 268, 335, 266, 139, 310, 181, 239, 193, 117, 55, 255, 114, 332, 116, 233, 66, 369, 148, 315, 287, 179, 96, 165, 295, 44, 373, 268, 351, 16, 372, 40, 334, 366, 345, 143, 263, 69, 318, 188, 8, 70, 187, 370, 331, 197, 178, 320, 291, 72, 275, 118, 348, 381, 381, 47, 343, 265, 81, 234, 180, 206, 359, 150, 269, 264, 61, 134, 196, 190, 14, 338, 127, 104, 204, 191, 367, 308, 62] + 61: [231, 46, 117, 1, 218, 294, 307, 88, 156, 227, 90, 27, 23, 97, 153, 9, 85, 106, 311, 99, 49, 105, 318, 210, 320, 208, 303, 46, 178, 197, 167, 372, 86, 115, 369, 87, 84, 44, 260, 102, 38, 125, 340, 342, 345, 364, 246, 28, 56, 131, 334, 356, 230, 338, 203, 380, 253, 198, 143, 129, 34, 361, 316, 190, 368, 207, 346, 37, 225, 152, 370, 358, 232, 140, 256, 62, 41, 319, 172, 338, 323, 197, 21, 68, 55, 288, 110, 20, 77, 252, 336, 249, 137, 22, 141, 204, 352, 18, 312, 236, 250, 50, 367, 0, 61, 179, 299, 381, 353, 235, 248, 216, 181, 261, 224, 11, 329, 378, 383, 79, 67, 14, 244, 123, 111, 146, 35, 267, 376, 58, 61, 338, 122, 45, 167, 371, 248, 216, 15, 169, 366, 83, 3, 200, 273, 280, 196, 359, 63, 310, 272, 95, 377, 289, 217, 287, 73, 179, 205, 363, 36, 324, 237, 192, 243, 309, 29, 166, 206, 147, 214, 275, 269, 148, 70, 74, 13, 164, 285, 39, 229, 94, 73, 179, 299, 184, 100, 142, 263, 107, 42, 92, 114, 135, 328, 52, 306, 8, 283, 298, 162, 251, 7, 339, 276, 151, 40, 96, 5, 170, 351, 381, 195, 51, 161, 333, 188, 191, 233, 113, 245, 202, 292, 365, 4, 199, 163, 301, 24, 48, 222, 134, 325, 149, 136, 221, 165, 264, 103, 215, 263, 107, 286, 228, 118, 304, 109, 139, 186, 80, 26, 144, 54, 187, 293, 182, 60, 281, 350, 116, 335, 221, 127, 241, 330, 344, 119, 57, 177, 258, 268, 300, 145, 104, 30, 375, 240, 98, 193, 101, 262, 223, 220, 382, 266, 53, 108, 221, 127, 19, 315, 344, 66, 168, 349, 43, 348, 379, 47, 173, 72, 219, 71, 124, 25, 354, 314, 212, 64, 254, 128, 343, 360, 221, 127, 10, 36, 324, 161, 112, 160, 75, 150, 82, 69, 158, 89, 341, 317, 332, 91, 239, 305, 180, 226, 355, 16, 171, 93, 211, 127, 270, 12, 17, 86, 291, 159, 337, 362, 374, 2, 290, 78, 174, 295, 284, 274, 234, 331, 265, 302, 279, 357, 189, 93, 120, 247, 59, 315, 278, 213, 327, 31, 32, 65, 313, 257, 33, 322, 126, 81, 176, 271, 255, 201, 373, 277, 175, 296, 321, 211, 157, 326, 209, 100, 238, 307, 88, 282, 133, 297, 121, 308, 259, 154, 242, 132, 194, 130, 6, 185, 138, 347, 183, 76, 155] +layer_updates_per_iter: 0 +num_slots: 416 diff --git a/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots384.yaml b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots384.yaml new file mode 100644 index 000000000000..262c8042d3d1 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 327, 22, 354, 123, 257, 180, 263, 53, 250, 120, 61, 111, 232, 55, 10, 31, 213, 192, 145, 340, 132, 88, 185, 378, 29, 282, 156, 148, 172, 369, 67, 206, 164, 168, 303, 166, 104, 272, 72, 114, 358, 249, 147, 252, 266, 304, 247, 256, 347, 198, 284, 368, 375, 331, 267, 350, 125, 142, 110, 74, 188, 63, 117, 261, 66, 365, 326, 229, 357, 24, 373, 222, 332, 139, 34, 200, 39, 269, 299, 40, 2, 121, 274, 356, 190, 343, 205, 36, 89, 108, 25, 210, 301, 339, 370, 236, 241, 255, 78, 297, 309, 103, 100, 47, 175, 189, 201, 245, 346, 48, 359, 186, 160, 4, 97, 20, 41, 271, 171, 371, 158, 321, 143, 90, 27, 317, 1, 50, 12, 92, 13, 225, 230, 289, 141, 124, 268, 136, 238, 6, 353, 23, 277, 254, 193, 218, 336, 57, 77, 214, 215, 260, 380, 298, 118, 135, 207, 157, 360, 109, 149, 335, 211, 342, 306, 18, 58, 279, 76, 80, 349, 56, 337, 227, 296, 362, 315, 167, 310, 45, 264, 242, 126, 233, 208, 244, 59, 237, 115, 251, 288, 60, 235, 62, 209, 43, 170, 32, 366, 14, 352, 69, 226, 79, 177, 204, 144, 183, 7, 85, 372, 82, 98, 319, 8, 17, 348, 30, 280, 307, 300, 262, 75, 83, 286, 381, 344, 95, 162, 202, 3, 181, 28, 314, 363, 127, 294, 194, 291, 134, 155, 81, 333, 131, 112, 38, 146, 328, 305, 231, 228, 184, 86, 153, 176, 341, 265, 308, 275, 320, 316, 239, 281, 49, 383, 21, 16, 33, 248, 234, 240, 93, 102, 285, 329, 203, 119, 35, 37, 196, 311, 377, 216, 195, 64, 191, 273, 217, 322, 295, 318, 54, 44, 199, 46, 382, 338, 367, 325, 330, 345, 169, 70, 101, 94, 223, 19, 106, 253, 9, 374, 173, 351, 122, 276, 313, 324, 87, 116, 292, 96, 334, 323, 224, 84, 312, 65, 243, 138, 52, 137, 73, 154, 220, 287, 290, 159, 182, 376, 11, 361, 187, 99, 379, 212, 179, 355, 259, 302, 26, 278, 105, 129, 91, 140, 221, 15, 219, 130, 0, 71, 51, 293, 107, 258, 133, 178, 270, 42, 113, 128, 150, 151, 163, 68, 283, 174, 5, 246, 197, 161, 152, 364] + 1: [163, 0, 29, 355, 176, 205, 318, 293, 119, 253, 117, 255, 319, 39, 24, 78, 143, 133, 2, 290, 77, 316, 339, 100, 365, 131, 248, 89, 261, 376, 259, 57, 181, 220, 45, 265, 42, 40, 23, 32, 280, 171, 204, 165, 113, 372, 191, 15, 201, 21, 250, 173, 335, 239, 305, 321, 268, 129, 16, 363, 174, 378, 56, 86, 142, 202, 222, 154, 8, 331, 155, 327, 22, 329, 299, 71, 194, 38, 348, 33, 302, 371, 322, 215, 137, 254, 244, 260, 92, 360, 170, 136, 323, 246, 124, 310, 180, 291, 300, 275, 162, 69, 79, 229, 13, 375, 343, 368, 158, 196, 294, 211, 84, 306, 313, 295, 94, 279, 128, 34, 277, 43, 58, 287, 108, 264, 140, 175, 96, 130, 361, 345, 382, 152, 334, 145, 288, 88, 362, 184, 227, 369, 252, 336, 91, 50, 63, 37, 212, 116, 298, 156, 352, 357, 221, 312, 72, 263, 105, 9, 146, 93, 76, 17, 97, 83, 135, 112, 169, 149, 73, 269, 187, 381, 7, 115, 104, 148, 217, 199, 177, 150, 110, 303, 166, 203, 121, 107, 311, 190, 209, 267, 167, 126, 328, 195, 197, 289, 122, 296, 26, 314, 132, 157, 127, 85, 41, 273, 218, 358, 164, 19, 350, 103, 66, 304, 347, 80, 123, 138, 301, 282, 189, 14, 51, 27, 208, 354, 67, 106, 210, 59, 99, 228, 256, 326, 11, 356, 10, 257, 377, 44, 70, 243, 74, 206, 237, 332, 315, 333, 373, 258, 3, 18, 325, 46, 285, 47, 231, 20, 48, 232, 151, 374, 182, 118, 75, 320, 274, 213, 98, 370, 284, 54, 340, 341, 198, 308, 179, 383, 266, 214, 61, 286, 87, 240, 207, 64, 120, 31, 5, 283, 216, 52, 60, 65, 186, 262, 160, 337, 225, 235, 270, 224, 281, 4, 95, 139, 324, 353, 193, 159, 111, 1, 366, 25, 245, 53, 102, 161, 36, 226, 367, 349, 82, 200, 192, 233, 317, 219, 101, 249, 35, 12, 90, 276, 272, 81, 6, 185, 234, 114, 236, 364, 230, 344, 188, 144, 292, 251, 30, 351, 125, 223, 147, 338, 172, 49, 297, 359, 134, 141, 168, 153, 247, 379, 278, 330, 68, 346, 183, 307, 178, 380, 241, 271, 242, 109, 342, 238, 62, 55, 309, 28] + 2: [143, 284, 325, 79, 92, 46, 136, 239, 371, 48, 228, 379, 248, 97, 31, 323, 55, 332, 154, 275, 373, 74, 326, 381, 53, 350, 51, 285, 224, 349, 305, 77, 292, 177, 289, 374, 43, 111, 45, 30, 263, 360, 106, 23, 329, 222, 315, 258, 132, 149, 118, 235, 126, 163, 9, 362, 99, 98, 287, 366, 112, 60, 245, 82, 262, 277, 38, 94, 95, 164, 202, 229, 108, 75, 278, 24, 68, 253, 256, 152, 216, 382, 282, 40, 313, 33, 70, 340, 90, 120, 181, 179, 144, 267, 237, 190, 215, 197, 286, 279, 7, 128, 160, 176, 141, 8, 354, 199, 89, 34, 73, 200, 17, 201, 15, 365, 331, 205, 260, 330, 194, 247, 280, 63, 170, 58, 186, 150, 264, 167, 240, 161, 93, 125, 56, 217, 299, 383, 169, 104, 207, 114, 13, 300, 115, 151, 131, 146, 206, 21, 309, 29, 376, 338, 353, 363, 223, 100, 66, 328, 140, 20, 193, 318, 178, 335, 369, 375, 3, 189, 355, 83, 130, 361, 175, 268, 52, 183, 317, 255, 295, 321, 157, 274, 344, 187, 348, 257, 272, 251, 214, 246, 212, 137, 173, 26, 204, 327, 308, 129, 25, 226, 72, 316, 242, 221, 166, 22, 294, 210, 41, 276, 219, 291, 117, 358, 232, 5, 124, 368, 165, 320, 65, 364, 0, 359, 345, 127, 290, 259, 191, 324, 174, 67, 148, 49, 47, 319, 147, 249, 19, 81, 322, 296, 61, 203, 76, 121, 238, 352, 310, 32, 62, 1, 84, 172, 342, 18, 171, 250, 134, 302, 378, 119, 168, 298, 155, 105, 311, 293, 265, 372, 2, 57, 336, 301, 91, 28, 192, 271, 209, 356, 241, 69, 36, 54, 182, 337, 377, 102, 109, 220, 6, 273, 16, 116, 195, 71, 158, 339, 86, 343, 110, 133, 85, 156, 10, 254, 281, 107, 307, 270, 304, 14, 101, 42, 303, 27, 236, 142, 4, 233, 234, 64, 314, 357, 78, 184, 11, 269, 145, 103, 211, 288, 113, 80, 139, 351, 185, 306, 159, 87, 138, 12, 346, 370, 59, 208, 122, 341, 153, 135, 196, 347, 198, 39, 367, 231, 37, 380, 180, 225, 162, 44, 244, 50, 297, 312, 334, 243, 227, 266, 230, 96, 213, 123, 333, 188, 218, 252, 35, 261, 283, 88] + 3: [66, 199, 278, 292, 303, 59, 311, 47, 222, 38, 301, 370, 34, 17, 247, 332, 335, 52, 9, 233, 257, 146, 324, 198, 310, 43, 165, 93, 234, 148, 352, 244, 152, 167, 226, 5, 260, 328, 15, 309, 121, 147, 124, 315, 173, 185, 341, 371, 377, 98, 270, 237, 367, 101, 134, 330, 175, 320, 326, 193, 365, 161, 181, 327, 160, 232, 362, 141, 253, 265, 349, 33, 319, 29, 229, 217, 246, 361, 153, 45, 54, 127, 4, 347, 39, 239, 364, 374, 166, 44, 249, 282, 159, 90, 272, 60, 22, 138, 118, 277, 200, 113, 187, 269, 220, 75, 308, 69, 79, 211, 119, 154, 204, 129, 286, 373, 258, 210, 368, 23, 302, 238, 325, 21, 288, 224, 266, 322, 130, 179, 143, 1, 40, 26, 189, 223, 99, 218, 307, 216, 381, 285, 338, 95, 18, 337, 250, 36, 142, 375, 105, 276, 106, 190, 372, 89, 27, 158, 102, 231, 12, 203, 103, 48, 353, 85, 183, 205, 163, 318, 201, 122, 240, 32, 299, 280, 96, 356, 245, 241, 3, 24, 186, 37, 64, 359, 108, 273, 115, 83, 354, 16, 194, 363, 139, 10, 11, 172, 41, 65, 248, 376, 0, 378, 313, 151, 339, 267, 290, 295, 126, 117, 340, 123, 133, 107, 263, 348, 215, 227, 25, 289, 274, 135, 306, 61, 206, 191, 71, 157, 137, 58, 182, 162, 300, 355, 196, 294, 28, 251, 46, 62, 42, 2, 219, 8, 345, 169, 366, 91, 254, 379, 275, 80, 331, 358, 316, 84, 176, 112, 100, 7, 86, 323, 35, 333, 6, 164, 177, 109, 155, 57, 214, 291, 346, 242, 230, 261, 213, 342, 174, 78, 111, 235, 76, 70, 228, 284, 209, 97, 50, 120, 296, 87, 243, 94, 82, 188, 110, 360, 314, 192, 180, 336, 74, 136, 297, 197, 144, 298, 283, 116, 14, 208, 252, 170, 195, 271, 92, 344, 383, 305, 68, 329, 171, 56, 81, 343, 304, 279, 114, 53, 268, 67, 72, 73, 51, 13, 145, 207, 287, 357, 312, 256, 125, 128, 132, 88, 31, 131, 49, 168, 77, 382, 351, 236, 30, 184, 293, 140, 350, 225, 334, 178, 221, 202, 262, 380, 104, 20, 321, 369, 19, 317, 150, 63, 212, 264, 149, 55, 255, 281, 259, 156] + 4: [343, 67, 321, 90, 336, 107, 182, 277, 282, 305, 332, 351, 331, 79, 128, 231, 242, 272, 338, 287, 58, 63, 104, 120, 3, 341, 137, 51, 40, 374, 368, 174, 20, 192, 208, 233, 271, 75, 288, 234, 249, 96, 293, 270, 227, 119, 225, 17, 315, 101, 260, 307, 269, 370, 175, 127, 279, 202, 154, 2, 214, 365, 28, 141, 359, 80, 178, 111, 311, 373, 299, 337, 209, 325, 43, 187, 252, 64, 84, 103, 130, 1, 11, 382, 296, 86, 361, 204, 159, 14, 284, 349, 223, 205, 224, 195, 371, 171, 347, 83, 177, 65, 261, 328, 26, 286, 380, 144, 23, 150, 167, 218, 317, 133, 132, 367, 157, 355, 173, 366, 15, 105, 155, 57, 369, 85, 323, 310, 29, 201, 48, 91, 95, 257, 298, 314, 8, 38, 60, 46, 206, 327, 378, 248, 226, 45, 197, 4, 7, 276, 379, 232, 184, 295, 362, 191, 319, 5, 377, 106, 161, 116, 53, 322, 52, 237, 112, 164, 87, 19, 36, 196, 339, 358, 185, 108, 121, 122, 123, 238, 194, 376, 308, 76, 82, 207, 149, 125, 303, 170, 115, 102, 273, 281, 6, 139, 73, 0, 356, 100, 220, 31, 156, 236, 117, 88, 165, 145, 251, 99, 168, 94, 292, 109, 213, 266, 142, 239, 143, 32, 49, 183, 297, 131, 50, 140, 255, 363, 148, 9, 241, 61, 280, 221, 189, 354, 176, 181, 10, 285, 254, 172, 136, 77, 68, 42, 360, 135, 309, 268, 126, 301, 152, 228, 348, 200, 344, 264, 72, 313, 263, 291, 203, 245, 188, 16, 302, 275, 217, 243, 330, 25, 39, 289, 353, 55, 34, 193, 253, 235, 44, 179, 33, 70, 66, 304, 163, 162, 340, 320, 345, 372, 312, 153, 352, 240, 92, 334, 316, 247, 18, 250, 56, 113, 22, 244, 114, 262, 267, 118, 21, 30, 37, 259, 13, 41, 35, 54, 256, 383, 294, 147, 129, 134, 12, 219, 274, 357, 333, 324, 62, 74, 346, 210, 186, 326, 375, 350, 342, 318, 198, 290, 166, 158, 306, 211, 97, 265, 124, 24, 329, 364, 160, 278, 169, 229, 78, 146, 215, 89, 199, 258, 59, 246, 110, 335, 98, 216, 230, 180, 81, 212, 222, 300, 47, 151, 27, 283, 93, 69, 138, 71, 381, 190] + 5: [61, 207, 144, 300, 236, 129, 340, 226, 286, 177, 354, 302, 213, 37, 295, 56, 284, 363, 316, 85, 142, 179, 118, 303, 128, 313, 24, 186, 250, 13, 263, 158, 180, 127, 368, 75, 214, 304, 184, 152, 329, 237, 134, 322, 197, 310, 321, 44, 355, 147, 171, 324, 380, 241, 77, 92, 309, 96, 157, 170, 190, 72, 335, 108, 320, 353, 76, 349, 189, 11, 10, 337, 249, 379, 364, 38, 333, 345, 334, 344, 125, 133, 116, 23, 84, 372, 89, 15, 175, 266, 341, 94, 58, 53, 248, 50, 296, 365, 293, 151, 210, 51, 257, 291, 378, 202, 109, 196, 194, 381, 246, 70, 3, 79, 121, 87, 114, 282, 176, 277, 217, 30, 331, 137, 143, 115, 240, 360, 18, 311, 225, 57, 346, 169, 200, 155, 204, 99, 201, 80, 359, 135, 328, 278, 276, 188, 43, 66, 318, 28, 239, 149, 283, 244, 350, 164, 100, 262, 7, 8, 251, 46, 382, 48, 62, 148, 111, 339, 105, 193, 88, 131, 140, 195, 362, 9, 232, 95, 101, 234, 305, 370, 352, 4, 319, 377, 106, 63, 323, 168, 97, 27, 269, 223, 252, 233, 264, 315, 93, 90, 376, 224, 120, 112, 356, 265, 17, 371, 332, 104, 14, 138, 71, 270, 357, 117, 287, 49, 212, 74, 369, 33, 253, 383, 271, 86, 245, 123, 52, 60, 255, 209, 375, 35, 39, 325, 374, 40, 267, 326, 288, 178, 215, 31, 358, 16, 12, 275, 191, 373, 185, 367, 6, 65, 132, 327, 222, 228, 183, 289, 55, 156, 285, 167, 272, 47, 279, 124, 59, 220, 227, 136, 205, 187, 307, 82, 231, 81, 351, 182, 2, 45, 216, 281, 229, 32, 292, 294, 159, 1, 145, 173, 163, 54, 366, 150, 103, 107, 36, 110, 259, 98, 130, 25, 174, 218, 242, 254, 162, 261, 126, 280, 21, 260, 64, 22, 268, 301, 20, 219, 299, 199, 146, 0, 172, 160, 41, 198, 181, 258, 230, 161, 113, 342, 336, 273, 208, 5, 102, 338, 69, 314, 343, 317, 221, 68, 243, 83, 165, 308, 122, 247, 206, 34, 274, 119, 361, 347, 290, 42, 154, 256, 78, 91, 238, 153, 312, 330, 29, 297, 139, 166, 141, 306, 211, 192, 67, 26, 19, 203, 235, 298, 73, 348] + 6: [97, 71, 323, 331, 40, 90, 161, 164, 247, 257, 342, 373, 142, 363, 25, 33, 52, 186, 204, 275, 315, 35, 320, 325, 308, 218, 353, 265, 153, 252, 75, 132, 149, 135, 167, 214, 322, 248, 299, 74, 51, 192, 241, 273, 170, 61, 72, 65, 76, 23, 333, 134, 360, 374, 19, 93, 382, 20, 2, 314, 274, 133, 157, 243, 346, 123, 17, 38, 127, 340, 191, 228, 376, 231, 327, 206, 372, 96, 235, 21, 140, 141, 344, 200, 32, 109, 158, 220, 225, 168, 125, 226, 201, 91, 1, 172, 305, 319, 301, 101, 332, 104, 126, 215, 357, 47, 171, 227, 53, 318, 208, 270, 12, 211, 207, 148, 16, 312, 131, 78, 343, 163, 287, 113, 68, 15, 237, 122, 124, 347, 102, 213, 174, 45, 255, 259, 236, 77, 271, 217, 42, 244, 89, 238, 258, 350, 112, 234, 49, 383, 262, 355, 268, 39, 121, 7, 253, 173, 119, 239, 245, 221, 6, 82, 3, 73, 117, 26, 240, 175, 56, 370, 380, 139, 280, 330, 193, 310, 256, 29, 281, 9, 368, 156, 41, 169, 365, 60, 105, 366, 266, 92, 99, 371, 293, 129, 377, 282, 58, 197, 199, 364, 85, 302, 66, 67, 100, 289, 352, 321, 154, 335, 83, 379, 290, 339, 233, 31, 232, 155, 115, 202, 98, 159, 81, 54, 328, 250, 166, 120, 367, 184, 369, 111, 296, 284, 48, 309, 337, 263, 177, 205, 88, 178, 128, 190, 286, 63, 18, 272, 251, 356, 146, 326, 187, 79, 185, 37, 10, 110, 14, 198, 288, 195, 297, 180, 94, 291, 179, 279, 136, 276, 138, 358, 108, 162, 212, 209, 181, 34, 354, 30, 219, 116, 261, 381, 285, 183, 59, 46, 216, 118, 165, 223, 143, 137, 145, 50, 62, 303, 295, 11, 13, 196, 176, 304, 264, 242, 316, 147, 203, 294, 5, 229, 307, 188, 277, 317, 324, 43, 329, 224, 106, 338, 103, 341, 359, 246, 345, 230, 64, 69, 362, 70, 292, 222, 160, 55, 260, 84, 300, 311, 107, 361, 194, 8, 152, 313, 27, 28, 375, 57, 0, 278, 144, 87, 254, 334, 80, 24, 249, 269, 378, 189, 151, 336, 114, 22, 4, 349, 150, 348, 306, 267, 36, 283, 95, 182, 44, 210, 86, 298, 130, 351] + 7: [327, 321, 19, 56, 120, 10, 228, 156, 135, 171, 50, 60, 7, 2, 372, 380, 323, 320, 112, 304, 138, 257, 338, 364, 274, 337, 158, 92, 152, 22, 363, 334, 223, 371, 357, 42, 49, 183, 84, 213, 204, 305, 68, 209, 43, 147, 179, 91, 292, 14, 23, 224, 140, 6, 146, 216, 231, 242, 127, 202, 175, 281, 168, 221, 264, 200, 161, 203, 59, 78, 355, 110, 182, 329, 267, 235, 214, 137, 196, 351, 85, 128, 34, 348, 275, 48, 313, 288, 330, 185, 141, 149, 287, 297, 205, 276, 192, 378, 249, 270, 90, 358, 51, 116, 286, 252, 172, 365, 263, 118, 11, 31, 377, 245, 44, 261, 106, 39, 66, 290, 64, 99, 324, 55, 62, 109, 86, 291, 166, 379, 150, 145, 41, 71, 73, 285, 219, 26, 5, 251, 366, 88, 315, 376, 375, 253, 104, 74, 383, 368, 237, 306, 37, 295, 246, 143, 142, 13, 302, 307, 284, 167, 107, 8, 101, 367, 159, 54, 229, 139, 181, 217, 208, 241, 211, 227, 93, 96, 266, 144, 265, 280, 256, 125, 296, 314, 186, 310, 79, 72, 195, 199, 343, 271, 27, 248, 325, 16, 32, 134, 184, 218, 198, 282, 382, 342, 94, 100, 356, 206, 178, 309, 153, 123, 381, 240, 340, 46, 174, 4, 210, 98, 273, 233, 115, 36, 45, 12, 105, 28, 77, 119, 170, 191, 335, 373, 126, 69, 230, 165, 18, 187, 244, 17, 255, 243, 293, 40, 193, 80, 317, 160, 103, 176, 328, 349, 21, 300, 102, 301, 269, 194, 332, 197, 201, 260, 0, 29, 177, 344, 58, 353, 35, 33, 299, 122, 25, 350, 114, 278, 319, 234, 220, 247, 38, 254, 124, 279, 362, 133, 370, 262, 117, 259, 190, 232, 225, 111, 3, 236, 298, 239, 61, 289, 70, 333, 83, 283, 318, 316, 359, 76, 369, 361, 131, 82, 374, 24, 180, 155, 222, 169, 346, 331, 163, 108, 212, 215, 81, 132, 322, 65, 87, 312, 303, 326, 341, 308, 188, 164, 339, 311, 136, 67, 268, 207, 57, 352, 162, 336, 15, 113, 250, 189, 130, 226, 151, 75, 95, 347, 157, 258, 354, 294, 360, 89, 20, 121, 345, 173, 277, 30, 272, 238, 97, 47, 148, 9, 129, 53, 52, 1, 63, 154] + 8: [360, 32, 378, 111, 160, 0, 227, 102, 58, 170, 234, 207, 253, 303, 125, 28, 190, 236, 193, 355, 19, 262, 132, 96, 290, 333, 304, 332, 87, 307, 260, 324, 51, 63, 213, 100, 359, 115, 34, 368, 2, 71, 327, 134, 370, 210, 163, 1, 5, 157, 346, 92, 50, 4, 293, 16, 369, 180, 331, 376, 67, 337, 97, 109, 278, 341, 174, 382, 145, 220, 363, 296, 31, 228, 168, 23, 215, 318, 98, 37, 159, 146, 305, 55, 135, 64, 377, 133, 219, 197, 118, 41, 205, 76, 343, 127, 21, 282, 364, 27, 40, 211, 230, 47, 35, 351, 70, 242, 367, 154, 11, 150, 178, 177, 212, 299, 89, 24, 286, 176, 264, 88, 18, 75, 216, 48, 206, 149, 79, 365, 129, 342, 239, 381, 209, 69, 169, 120, 335, 259, 200, 287, 348, 361, 201, 272, 349, 172, 121, 164, 263, 310, 265, 357, 103, 288, 362, 243, 44, 188, 65, 141, 217, 184, 85, 33, 15, 345, 162, 241, 328, 356, 275, 233, 117, 352, 235, 153, 84, 289, 321, 336, 294, 83, 165, 192, 254, 110, 325, 277, 171, 49, 10, 173, 316, 22, 17, 334, 383, 373, 339, 326, 271, 57, 62, 312, 380, 366, 330, 113, 114, 350, 155, 185, 77, 94, 221, 12, 255, 107, 261, 317, 358, 25, 99, 295, 268, 203, 308, 13, 95, 181, 78, 112, 42, 73, 56, 9, 245, 142, 353, 224, 186, 225, 179, 189, 101, 237, 144, 175, 285, 80, 152, 108, 223, 311, 202, 130, 52, 151, 136, 347, 208, 139, 371, 7, 251, 128, 298, 196, 60, 90, 266, 46, 158, 274, 229, 161, 20, 280, 147, 300, 66, 249, 372, 54, 244, 344, 204, 106, 252, 86, 6, 198, 45, 116, 140, 375, 354, 138, 182, 302, 30, 131, 38, 309, 59, 248, 199, 14, 269, 301, 281, 338, 291, 148, 283, 167, 8, 191, 313, 315, 270, 82, 273, 238, 276, 166, 240, 267, 91, 379, 105, 306, 123, 43, 104, 292, 36, 156, 226, 137, 187, 232, 314, 323, 340, 124, 214, 218, 122, 195, 231, 320, 72, 297, 126, 29, 319, 222, 250, 246, 81, 39, 119, 183, 61, 247, 68, 93, 329, 256, 3, 279, 74, 374, 258, 194, 284, 257, 143, 322, 26, 53] + 9: [92, 320, 347, 368, 40, 164, 236, 283, 338, 365, 234, 57, 304, 245, 253, 227, 44, 182, 152, 168, 340, 348, 173, 26, 255, 187, 321, 150, 276, 112, 76, 191, 257, 157, 27, 369, 349, 144, 6, 139, 343, 117, 301, 383, 362, 250, 53, 24, 327, 121, 360, 180, 267, 104, 60, 84, 199, 284, 61, 77, 355, 51, 161, 54, 105, 5, 379, 110, 178, 345, 55, 126, 31, 198, 14, 153, 97, 72, 235, 62, 224, 290, 85, 122, 226, 344, 23, 88, 142, 336, 80, 203, 298, 293, 204, 265, 94, 37, 136, 363, 107, 171, 354, 373, 89, 188, 378, 25, 263, 29, 67, 158, 247, 206, 8, 46, 310, 326, 200, 7, 295, 381, 209, 30, 231, 4, 202, 382, 286, 335, 273, 156, 47, 351, 341, 82, 318, 323, 359, 100, 371, 346, 176, 28, 185, 184, 271, 317, 0, 256, 237, 303, 42, 48, 240, 91, 313, 179, 214, 285, 334, 195, 106, 52, 213, 333, 160, 249, 316, 155, 281, 151, 306, 269, 288, 307, 300, 229, 59, 361, 90, 222, 210, 367, 228, 186, 16, 319, 291, 270, 193, 63, 262, 165, 329, 133, 297, 145, 230, 246, 261, 125, 374, 194, 49, 74, 287, 312, 377, 163, 123, 75, 322, 216, 34, 366, 64, 219, 370, 9, 135, 259, 11, 208, 254, 38, 170, 119, 244, 350, 159, 98, 120, 128, 141, 50, 205, 375, 325, 73, 95, 353, 380, 258, 302, 20, 124, 339, 2, 251, 243, 113, 103, 299, 242, 131, 154, 364, 274, 65, 275, 305, 68, 101, 147, 357, 138, 22, 3, 192, 111, 248, 174, 232, 167, 223, 10, 252, 217, 177, 292, 99, 36, 308, 114, 330, 140, 296, 212, 172, 376, 132, 137, 183, 238, 130, 45, 12, 43, 266, 78, 19, 272, 239, 33, 311, 218, 18, 233, 162, 337, 93, 166, 215, 201, 342, 169, 87, 189, 109, 148, 71, 108, 118, 81, 260, 83, 309, 289, 41, 181, 17, 79, 352, 225, 279, 143, 280, 1, 56, 277, 35, 207, 39, 190, 21, 241, 149, 331, 372, 15, 102, 356, 116, 66, 324, 115, 134, 58, 294, 220, 196, 146, 32, 264, 129, 70, 13, 358, 211, 86, 328, 96, 315, 127, 282, 268, 314, 332, 175, 197, 221, 69, 278] + 10: [319, 349, 290, 295, 358, 366, 29, 68, 283, 336, 297, 369, 5, 160, 110, 70, 236, 253, 166, 176, 23, 82, 291, 324, 300, 227, 10, 14, 242, 28, 43, 188, 228, 89, 238, 381, 320, 27, 383, 313, 159, 4, 333, 63, 277, 260, 52, 270, 365, 308, 348, 382, 72, 338, 278, 220, 62, 53, 352, 202, 194, 66, 312, 98, 229, 85, 51, 317, 131, 217, 44, 150, 264, 138, 293, 190, 90, 106, 97, 323, 326, 224, 157, 152, 55, 19, 353, 77, 198, 122, 230, 275, 169, 17, 281, 156, 180, 153, 115, 204, 132, 296, 41, 174, 244, 187, 158, 191, 151, 263, 213, 342, 199, 288, 328, 193, 257, 206, 200, 361, 265, 171, 380, 377, 155, 311, 74, 254, 307, 305, 6, 347, 24, 215, 201, 136, 231, 21, 112, 36, 343, 80, 88, 30, 121, 73, 177, 240, 306, 232, 378, 340, 282, 139, 94, 84, 216, 102, 182, 11, 38, 103, 314, 162, 185, 287, 273, 143, 18, 79, 210, 243, 189, 0, 60, 267, 50, 350, 168, 344, 286, 101, 241, 272, 360, 33, 142, 371, 239, 96, 292, 207, 31, 57, 40, 76, 208, 145, 219, 25, 140, 209, 303, 83, 164, 225, 351, 67, 148, 128, 81, 379, 45, 364, 205, 172, 146, 92, 114, 7, 316, 250, 75, 37, 332, 299, 147, 104, 310, 2, 78, 91, 49, 32, 129, 339, 39, 362, 119, 35, 301, 309, 117, 58, 186, 376, 56, 9, 247, 298, 375, 120, 294, 331, 161, 47, 252, 271, 3, 197, 12, 95, 355, 16, 192, 367, 179, 196, 284, 359, 107, 335, 64, 123, 373, 133, 255, 126, 178, 124, 363, 266, 214, 256, 125, 163, 86, 322, 246, 167, 134, 237, 374, 218, 135, 20, 15, 65, 173, 276, 93, 69, 144, 269, 368, 212, 105, 26, 71, 226, 111, 34, 268, 279, 8, 211, 259, 223, 46, 357, 318, 183, 235, 203, 137, 337, 22, 248, 130, 280, 87, 329, 184, 261, 222, 245, 181, 325, 1, 330, 345, 61, 116, 346, 42, 370, 372, 251, 108, 289, 170, 154, 234, 249, 341, 59, 258, 113, 127, 165, 48, 262, 195, 304, 356, 149, 233, 327, 109, 175, 100, 118, 221, 141, 54, 99, 274, 285, 321, 315, 354, 13, 302, 334] + 11: [130, 373, 289, 279, 383, 184, 132, 53, 256, 243, 335, 269, 314, 320, 226, 365, 292, 57, 382, 214, 181, 273, 42, 241, 296, 250, 45, 151, 294, 125, 113, 310, 180, 333, 32, 12, 163, 207, 280, 10, 336, 120, 129, 140, 272, 72, 257, 105, 110, 128, 297, 27, 55, 366, 64, 271, 380, 137, 50, 9, 98, 52, 303, 80, 255, 362, 264, 43, 21, 114, 206, 109, 377, 165, 196, 117, 89, 254, 353, 19, 103, 344, 253, 261, 339, 178, 148, 283, 160, 301, 234, 142, 288, 183, 76, 308, 87, 29, 95, 350, 197, 208, 291, 48, 322, 68, 155, 61, 90, 162, 39, 47, 195, 259, 338, 143, 200, 361, 111, 191, 83, 102, 73, 313, 93, 96, 265, 237, 375, 139, 317, 134, 215, 267, 131, 6, 92, 186, 312, 8, 274, 352, 372, 307, 85, 199, 343, 78, 2, 224, 81, 349, 230, 5, 60, 202, 150, 157, 330, 23, 75, 14, 97, 91, 263, 287, 167, 212, 235, 325, 284, 136, 346, 251, 211, 302, 355, 104, 290, 329, 221, 300, 141, 169, 124, 86, 242, 311, 182, 173, 268, 334, 121, 190, 217, 17, 94, 172, 30, 354, 194, 37, 175, 28, 11, 145, 345, 177, 170, 374, 59, 359, 231, 108, 216, 16, 36, 233, 0, 154, 319, 315, 168, 364, 358, 106, 277, 153, 116, 171, 22, 228, 367, 340, 35, 378, 46, 360, 369, 189, 44, 99, 332, 20, 371, 341, 327, 34, 7, 112, 328, 232, 248, 38, 351, 244, 159, 298, 31, 324, 18, 306, 379, 220, 100, 326, 152, 204, 127, 49, 101, 135, 188, 356, 1, 239, 126, 119, 247, 70, 69, 26, 147, 293, 258, 185, 304, 223, 357, 149, 144, 282, 348, 203, 209, 118, 174, 240, 305, 281, 285, 65, 218, 13, 66, 347, 245, 158, 138, 323, 229, 276, 15, 40, 24, 205, 161, 176, 295, 115, 79, 4, 278, 321, 156, 3, 192, 381, 107, 316, 270, 164, 225, 201, 71, 82, 252, 376, 54, 286, 249, 56, 63, 187, 62, 210, 238, 122, 77, 227, 236, 146, 213, 133, 41, 179, 309, 84, 342, 25, 260, 363, 331, 337, 262, 166, 219, 67, 33, 222, 368, 123, 299, 88, 275, 318, 370, 198, 58, 193, 74, 266, 246, 51] + 12: [255, 250, 366, 199, 73, 110, 225, 272, 372, 220, 302, 42, 204, 169, 112, 44, 125, 100, 166, 64, 66, 99, 222, 162, 120, 83, 319, 336, 105, 247, 213, 377, 331, 240, 43, 53, 130, 155, 0, 58, 86, 310, 230, 206, 359, 141, 239, 373, 158, 180, 283, 136, 122, 352, 178, 338, 88, 21, 56, 353, 327, 266, 362, 301, 315, 187, 12, 284, 117, 79, 275, 95, 149, 281, 89, 273, 15, 165, 335, 5, 251, 258, 92, 324, 78, 195, 323, 322, 14, 185, 45, 65, 329, 237, 143, 311, 226, 98, 205, 371, 294, 200, 298, 257, 246, 212, 118, 248, 190, 50, 278, 16, 271, 286, 234, 321, 32, 252, 320, 132, 303, 233, 6, 60, 107, 144, 114, 325, 293, 146, 343, 221, 127, 192, 339, 108, 57, 91, 41, 173, 183, 383, 55, 182, 223, 290, 295, 68, 102, 241, 231, 189, 379, 297, 81, 210, 82, 97, 123, 268, 33, 326, 350, 67, 270, 279, 96, 307, 177, 103, 134, 382, 264, 63, 253, 161, 235, 106, 47, 202, 378, 287, 259, 229, 265, 291, 304, 232, 355, 174, 375, 228, 69, 71, 176, 317, 116, 129, 1, 224, 140, 179, 72, 367, 243, 126, 354, 25, 10, 318, 9, 119, 334, 244, 211, 19, 150, 296, 3, 152, 133, 157, 254, 61, 351, 374, 113, 7, 172, 289, 75, 194, 27, 59, 357, 280, 115, 24, 282, 314, 333, 349, 340, 216, 300, 164, 376, 4, 76, 285, 34, 242, 203, 124, 208, 215, 39, 380, 142, 256, 36, 312, 54, 111, 46, 22, 260, 330, 153, 238, 193, 249, 214, 70, 30, 360, 341, 292, 198, 346, 101, 308, 191, 84, 305, 128, 18, 184, 156, 267, 181, 137, 104, 135, 358, 37, 40, 207, 347, 26, 363, 109, 151, 147, 277, 23, 227, 313, 13, 365, 236, 196, 49, 168, 90, 368, 364, 74, 370, 262, 121, 148, 131, 94, 35, 175, 51, 87, 288, 31, 309, 356, 38, 85, 316, 328, 48, 167, 306, 348, 209, 337, 201, 139, 11, 274, 345, 342, 381, 344, 170, 332, 171, 197, 52, 29, 20, 186, 299, 269, 245, 17, 369, 276, 2, 80, 77, 62, 154, 217, 8, 163, 219, 28, 361, 218, 160, 261, 188, 138, 159, 263, 93, 145] + 13: [170, 350, 140, 155, 230, 316, 78, 192, 32, 204, 7, 175, 108, 320, 62, 139, 379, 308, 103, 157, 278, 66, 324, 315, 232, 158, 179, 333, 36, 244, 217, 60, 90, 120, 206, 126, 18, 91, 79, 166, 68, 50, 358, 203, 152, 160, 301, 33, 195, 64, 266, 240, 256, 288, 268, 213, 346, 258, 218, 314, 87, 111, 143, 339, 183, 48, 296, 124, 83, 349, 367, 27, 80, 109, 297, 42, 225, 125, 59, 13, 343, 293, 200, 216, 44, 162, 16, 299, 25, 52, 357, 53, 190, 212, 156, 165, 362, 370, 364, 208, 144, 29, 326, 311, 119, 28, 47, 193, 329, 338, 75, 334, 141, 243, 153, 187, 353, 63, 354, 247, 114, 17, 176, 309, 375, 374, 223, 245, 38, 1, 264, 277, 14, 340, 323, 273, 65, 196, 241, 325, 19, 226, 99, 82, 149, 110, 304, 289, 300, 145, 76, 56, 123, 249, 290, 302, 328, 100, 306, 31, 373, 239, 222, 319, 70, 138, 132, 255, 23, 342, 171, 69, 6, 26, 164, 345, 360, 233, 236, 322, 260, 95, 46, 188, 122, 344, 380, 220, 154, 182, 173, 351, 312, 55, 298, 112, 2, 228, 15, 136, 94, 214, 352, 148, 280, 276, 335, 369, 178, 382, 146, 106, 71, 104, 215, 368, 189, 231, 142, 184, 363, 321, 129, 161, 113, 115, 101, 181, 238, 201, 199, 49, 292, 43, 327, 337, 133, 72, 117, 224, 12, 135, 88, 227, 67, 219, 253, 286, 250, 270, 284, 347, 30, 21, 186, 198, 281, 355, 151, 248, 348, 73, 252, 251, 274, 172, 11, 58, 229, 237, 177, 267, 263, 377, 336, 371, 318, 51, 366, 45, 294, 211, 93, 89, 305, 61, 205, 269, 9, 282, 310, 295, 261, 168, 283, 105, 92, 5, 185, 272, 194, 41, 134, 159, 54, 22, 361, 331, 257, 130, 4, 317, 137, 24, 102, 307, 202, 98, 121, 163, 174, 313, 383, 20, 77, 169, 254, 118, 275, 10, 291, 37, 259, 150, 167, 332, 57, 265, 209, 279, 40, 210, 74, 303, 127, 235, 372, 97, 81, 39, 330, 116, 356, 207, 107, 221, 381, 234, 180, 191, 3, 86, 197, 131, 246, 378, 359, 8, 84, 287, 85, 128, 96, 0, 271, 35, 376, 147, 262, 34, 242, 365, 285, 341] + 14: [229, 249, 295, 121, 205, 260, 289, 331, 50, 88, 165, 174, 160, 296, 225, 240, 365, 194, 303, 210, 130, 318, 83, 212, 345, 166, 332, 145, 284, 316, 103, 58, 7, 134, 292, 315, 255, 250, 25, 172, 234, 63, 262, 46, 11, 247, 272, 73, 70, 213, 314, 193, 109, 222, 1, 251, 322, 47, 243, 351, 211, 124, 13, 343, 382, 105, 168, 153, 334, 185, 170, 119, 143, 198, 111, 246, 40, 82, 350, 376, 87, 281, 218, 232, 4, 254, 79, 328, 333, 341, 223, 273, 127, 120, 320, 67, 15, 235, 154, 135, 349, 48, 89, 10, 267, 140, 381, 98, 76, 146, 61, 161, 203, 226, 244, 302, 195, 81, 144, 138, 204, 348, 162, 158, 159, 227, 253, 248, 74, 139, 219, 196, 110, 118, 317, 33, 370, 178, 0, 93, 171, 258, 233, 53, 30, 337, 371, 197, 326, 169, 156, 152, 94, 78, 180, 252, 2, 353, 32, 220, 357, 104, 129, 191, 114, 136, 35, 283, 209, 150, 177, 208, 69, 37, 306, 189, 278, 361, 84, 307, 259, 186, 97, 125, 184, 301, 359, 335, 358, 9, 293, 27, 75, 344, 342, 270, 291, 277, 86, 336, 309, 179, 18, 340, 34, 239, 231, 26, 319, 22, 330, 300, 375, 14, 55, 51, 122, 268, 64, 265, 324, 167, 221, 242, 214, 238, 24, 236, 327, 49, 39, 41, 369, 339, 237, 264, 6, 338, 107, 65, 126, 175, 3, 216, 201, 108, 298, 202, 321, 290, 57, 72, 355, 112, 117, 294, 68, 363, 276, 200, 257, 190, 66, 368, 378, 199, 95, 44, 206, 325, 347, 313, 173, 36, 181, 188, 308, 383, 100, 271, 379, 256, 373, 217, 42, 230, 362, 54, 155, 17, 52, 157, 215, 102, 90, 12, 285, 263, 45, 352, 71, 8, 85, 374, 288, 266, 176, 115, 329, 305, 62, 192, 275, 60, 141, 354, 323, 187, 101, 148, 106, 19, 366, 131, 147, 137, 151, 360, 123, 133, 377, 282, 311, 16, 80, 29, 56, 287, 228, 279, 149, 128, 116, 364, 28, 20, 367, 91, 274, 164, 23, 21, 297, 286, 5, 356, 310, 304, 92, 113, 261, 77, 142, 269, 163, 346, 380, 183, 43, 299, 132, 38, 372, 207, 99, 182, 59, 245, 241, 224, 96, 312, 31, 280] + 15: [130, 245, 75, 131, 322, 366, 260, 93, 281, 270, 85, 328, 11, 220, 27, 315, 264, 198, 325, 127, 192, 368, 356, 276, 0, 308, 216, 364, 188, 299, 122, 156, 345, 348, 25, 144, 15, 43, 365, 84, 262, 44, 125, 214, 132, 33, 46, 6, 88, 269, 294, 71, 273, 58, 136, 302, 336, 249, 19, 152, 359, 78, 49, 22, 226, 272, 91, 153, 154, 287, 297, 186, 160, 255, 133, 323, 62, 349, 123, 52, 380, 243, 288, 219, 227, 56, 18, 182, 194, 8, 165, 211, 218, 280, 178, 21, 57, 203, 83, 109, 5, 263, 12, 354, 172, 197, 124, 286, 362, 215, 35, 196, 209, 309, 13, 267, 334, 303, 112, 108, 251, 119, 48, 320, 170, 247, 206, 253, 92, 116, 310, 179, 324, 284, 23, 138, 335, 238, 282, 107, 45, 54, 176, 183, 155, 252, 151, 177, 207, 105, 254, 271, 26, 265, 187, 41, 355, 333, 360, 1, 352, 275, 229, 180, 224, 190, 293, 103, 200, 74, 221, 168, 24, 142, 34, 381, 208, 278, 318, 120, 258, 236, 90, 256, 164, 59, 268, 347, 314, 274, 184, 223, 94, 159, 60, 173, 235, 117, 77, 61, 137, 378, 374, 304, 128, 266, 9, 342, 379, 2, 114, 97, 234, 73, 369, 307, 217, 326, 80, 353, 351, 246, 233, 339, 283, 63, 149, 346, 175, 231, 329, 311, 162, 102, 50, 313, 373, 213, 370, 17, 148, 332, 317, 163, 31, 279, 69, 191, 169, 3, 201, 70, 104, 185, 232, 296, 338, 145, 341, 301, 375, 257, 14, 292, 134, 38, 121, 277, 115, 82, 87, 306, 383, 65, 312, 110, 68, 189, 146, 113, 225, 166, 331, 321, 244, 239, 29, 222, 298, 300, 20, 135, 230, 248, 10, 53, 289, 161, 89, 330, 290, 39, 51, 344, 106, 212, 150, 67, 55, 79, 32, 361, 261, 199, 237, 193, 285, 86, 343, 72, 99, 147, 129, 66, 340, 291, 376, 242, 47, 372, 205, 382, 337, 316, 126, 98, 350, 371, 157, 174, 36, 101, 250, 30, 37, 167, 327, 181, 76, 367, 139, 143, 100, 28, 96, 118, 204, 210, 158, 195, 4, 202, 16, 241, 357, 42, 81, 377, 95, 358, 140, 228, 259, 295, 40, 111, 319, 305, 363, 141, 64, 240, 7, 171] + 16: [134, 366, 368, 1, 98, 344, 3, 148, 156, 249, 257, 353, 15, 345, 13, 69, 215, 239, 320, 94, 111, 126, 247, 309, 286, 278, 296, 325, 376, 361, 362, 10, 53, 133, 161, 191, 120, 365, 66, 41, 106, 25, 190, 107, 332, 299, 192, 72, 275, 310, 37, 195, 124, 132, 244, 145, 149, 6, 197, 179, 100, 212, 307, 380, 301, 375, 93, 62, 170, 79, 263, 204, 140, 146, 71, 56, 45, 104, 223, 363, 110, 352, 327, 222, 369, 189, 85, 21, 70, 160, 206, 48, 315, 251, 164, 291, 152, 171, 199, 248, 229, 22, 57, 30, 81, 150, 349, 326, 234, 35, 236, 321, 155, 36, 290, 313, 194, 14, 279, 136, 262, 374, 235, 177, 139, 184, 225, 336, 91, 88, 359, 173, 92, 205, 46, 52, 281, 305, 188, 358, 109, 127, 50, 351, 7, 5, 213, 314, 103, 34, 137, 214, 193, 157, 77, 12, 43, 348, 295, 306, 58, 219, 9, 211, 142, 166, 176, 112, 165, 259, 364, 84, 80, 76, 297, 242, 196, 373, 241, 23, 147, 318, 141, 285, 316, 167, 322, 60, 201, 128, 116, 40, 269, 203, 187, 27, 82, 220, 317, 340, 272, 17, 245, 74, 64, 342, 121, 89, 119, 8, 276, 95, 304, 113, 61, 33, 44, 289, 287, 294, 99, 356, 96, 54, 207, 284, 217, 130, 303, 143, 383, 65, 292, 232, 4, 200, 266, 324, 256, 135, 118, 90, 265, 108, 175, 227, 339, 178, 319, 131, 138, 101, 280, 129, 209, 181, 73, 182, 198, 228, 83, 261, 260, 254, 75, 49, 283, 346, 377, 357, 370, 158, 105, 20, 210, 169, 271, 144, 298, 180, 38, 379, 238, 102, 240, 47, 216, 329, 11, 273, 255, 16, 335, 31, 174, 18, 378, 268, 230, 382, 162, 86, 24, 226, 51, 231, 19, 208, 55, 218, 243, 28, 68, 338, 302, 367, 237, 159, 42, 354, 267, 381, 117, 32, 67, 277, 224, 270, 328, 153, 202, 221, 333, 172, 168, 63, 330, 312, 183, 87, 59, 282, 334, 252, 337, 186, 258, 355, 125, 300, 274, 246, 115, 0, 122, 264, 253, 360, 26, 288, 331, 154, 250, 347, 97, 39, 372, 29, 323, 163, 308, 341, 185, 114, 78, 123, 2, 311, 293, 151, 343, 233, 371, 350] + 17: [131, 170, 202, 203, 235, 237, 251, 283, 312, 320, 325, 354, 119, 169, 14, 86, 122, 127, 358, 69, 72, 121, 152, 159, 284, 321, 156, 209, 252, 301, 333, 123, 174, 293, 34, 54, 236, 228, 162, 248, 37, 95, 192, 150, 368, 247, 111, 281, 79, 296, 371, 332, 154, 110, 369, 233, 178, 71, 35, 322, 337, 194, 367, 376, 336, 21, 372, 318, 242, 77, 51, 68, 375, 8, 46, 272, 260, 49, 238, 128, 32, 176, 0, 268, 261, 29, 96, 373, 18, 7, 115, 102, 329, 315, 201, 17, 220, 351, 290, 180, 292, 19, 365, 120, 161, 53, 109, 334, 250, 2, 234, 335, 84, 45, 196, 249, 151, 65, 277, 348, 188, 130, 370, 199, 135, 42, 295, 262, 48, 344, 160, 285, 303, 193, 107, 313, 214, 166, 341, 360, 326, 255, 288, 306, 108, 225, 297, 148, 327, 269, 81, 302, 55, 57, 356, 353, 200, 359, 133, 73, 98, 191, 363, 379, 25, 76, 30, 289, 15, 155, 189, 317, 143, 259, 41, 181, 93, 85, 316, 66, 383, 117, 347, 207, 10, 168, 215, 382, 239, 210, 355, 223, 100, 136, 324, 39, 92, 171, 13, 309, 213, 67, 270, 11, 26, 311, 125, 134, 273, 245, 47, 101, 342, 364, 350, 129, 330, 78, 246, 175, 206, 298, 147, 118, 195, 112, 352, 224, 305, 357, 1, 75, 22, 139, 59, 198, 240, 217, 88, 280, 294, 254, 185, 222, 231, 146, 173, 204, 244, 287, 182, 378, 362, 219, 5, 157, 380, 142, 264, 271, 94, 106, 144, 229, 374, 91, 275, 104, 211, 263, 61, 3, 310, 64, 83, 23, 28, 12, 114, 276, 97, 99, 331, 187, 179, 274, 163, 257, 212, 87, 286, 339, 103, 70, 340, 338, 27, 124, 50, 349, 138, 366, 232, 56, 38, 74, 20, 205, 361, 267, 132, 149, 307, 126, 343, 172, 381, 165, 36, 186, 300, 308, 164, 4, 80, 58, 279, 167, 218, 256, 43, 216, 258, 345, 227, 40, 105, 346, 323, 6, 145, 9, 82, 89, 44, 158, 31, 184, 208, 319, 113, 328, 24, 33, 60, 221, 299, 16, 153, 230, 140, 190, 266, 62, 177, 314, 304, 253, 265, 278, 63, 291, 226, 137, 197, 52, 243, 282, 116, 377, 90, 183, 241, 141] + 18: [271, 293, 314, 344, 382, 1, 32, 59, 78, 147, 309, 340, 30, 52, 74, 80, 97, 120, 156, 183, 222, 227, 242, 268, 251, 288, 54, 94, 163, 95, 117, 295, 55, 226, 327, 374, 208, 165, 283, 7, 176, 150, 112, 311, 107, 160, 310, 91, 47, 35, 318, 296, 313, 363, 179, 219, 76, 173, 194, 49, 10, 144, 181, 343, 203, 276, 256, 158, 85, 280, 198, 186, 43, 282, 89, 149, 350, 178, 270, 197, 258, 212, 323, 128, 69, 376, 326, 366, 223, 195, 12, 275, 247, 218, 206, 221, 320, 26, 33, 202, 22, 361, 187, 322, 133, 372, 277, 196, 217, 189, 246, 346, 233, 111, 81, 348, 220, 90, 281, 353, 257, 236, 378, 46, 263, 102, 63, 108, 27, 42, 294, 379, 300, 142, 151, 274, 83, 122, 125, 292, 358, 66, 298, 252, 245, 116, 253, 19, 331, 159, 205, 329, 383, 4, 375, 336, 65, 239, 297, 164, 359, 232, 279, 351, 175, 56, 364, 62, 57, 174, 291, 70, 269, 129, 157, 132, 303, 51, 381, 134, 138, 192, 237, 190, 23, 241, 50, 93, 60, 200, 352, 290, 71, 169, 284, 330, 307, 0, 201, 13, 153, 369, 84, 377, 228, 380, 306, 266, 177, 87, 349, 244, 48, 332, 29, 14, 106, 110, 360, 240, 171, 225, 39, 145, 127, 216, 115, 24, 185, 368, 373, 188, 20, 316, 152, 182, 213, 338, 161, 345, 260, 304, 31, 341, 73, 131, 235, 40, 148, 17, 86, 5, 6, 36, 25, 101, 339, 109, 61, 96, 234, 64, 172, 100, 118, 114, 167, 204, 41, 79, 308, 38, 243, 209, 354, 230, 356, 119, 146, 124, 191, 261, 140, 319, 143, 82, 238, 249, 272, 215, 136, 285, 104, 262, 15, 264, 287, 123, 2, 21, 193, 347, 355, 105, 342, 278, 231, 155, 324, 229, 370, 305, 45, 113, 321, 77, 121, 334, 8, 362, 168, 170, 299, 11, 53, 28, 68, 3, 365, 135, 302, 255, 325, 317, 37, 184, 333, 154, 301, 67, 141, 367, 328, 265, 214, 224, 44, 273, 16, 207, 34, 103, 337, 88, 289, 335, 166, 126, 180, 199, 210, 315, 139, 98, 371, 162, 58, 9, 254, 130, 312, 72, 211, 286, 357, 99, 248, 137, 250, 267, 92, 259, 75, 18] + 19: [24, 133, 198, 245, 253, 254, 263, 268, 319, 355, 369, 375, 187, 109, 229, 305, 317, 350, 12, 34, 50, 81, 98, 107, 190, 69, 199, 327, 55, 104, 145, 196, 274, 316, 378, 93, 366, 360, 17, 43, 72, 220, 382, 78, 142, 175, 250, 276, 7, 96, 259, 108, 3, 232, 59, 304, 261, 379, 226, 282, 90, 122, 353, 277, 380, 26, 258, 73, 367, 308, 102, 169, 288, 217, 246, 170, 95, 284, 383, 137, 348, 21, 266, 151, 256, 66, 171, 134, 272, 143, 85, 281, 370, 294, 51, 0, 265, 216, 117, 54, 76, 260, 160, 376, 14, 314, 100, 5, 57, 114, 89, 30, 68, 40, 293, 88, 139, 357, 136, 56, 377, 197, 97, 335, 239, 273, 23, 92, 132, 324, 155, 236, 120, 300, 77, 46, 6, 204, 60, 62, 181, 351, 205, 248, 338, 8, 271, 25, 228, 138, 287, 336, 242, 341, 124, 267, 330, 219, 227, 147, 359, 257, 342, 45, 280, 99, 247, 299, 313, 192, 296, 41, 222, 156, 31, 18, 298, 212, 106, 269, 180, 318, 334, 79, 71, 172, 2, 75, 65, 105, 28, 289, 42, 279, 303, 354, 36, 127, 306, 333, 146, 209, 118, 123, 207, 183, 27, 264, 1, 182, 157, 356, 200, 22, 315, 344, 312, 126, 193, 153, 285, 121, 221, 202, 326, 32, 361, 86, 101, 322, 349, 159, 238, 195, 203, 19, 129, 208, 63, 162, 240, 291, 38, 301, 13, 83, 297, 262, 329, 168, 103, 178, 64, 161, 218, 35, 116, 165, 347, 331, 191, 234, 94, 185, 307, 188, 33, 152, 135, 283, 84, 325, 362, 320, 112, 201, 115, 255, 20, 275, 233, 167, 215, 177, 309, 332, 149, 290, 141, 91, 164, 230, 15, 29, 67, 10, 148, 340, 286, 358, 243, 70, 363, 58, 278, 173, 144, 80, 372, 174, 235, 39, 211, 52, 154, 189, 53, 251, 345, 131, 47, 343, 150, 214, 339, 241, 184, 158, 310, 368, 374, 213, 140, 87, 371, 163, 111, 364, 328, 9, 352, 270, 249, 125, 119, 311, 381, 223, 179, 61, 373, 365, 166, 16, 244, 194, 321, 82, 231, 224, 206, 302, 113, 292, 295, 49, 110, 346, 130, 176, 11, 210, 48, 74, 323, 44, 128, 337, 4, 237, 225, 186, 37, 252] + 20: [161, 200, 206, 214, 262, 263, 266, 281, 299, 303, 331, 350, 116, 52, 54, 61, 76, 107, 137, 167, 168, 179, 181, 189, 334, 17, 117, 198, 241, 242, 249, 260, 277, 280, 27, 51, 98, 284, 233, 97, 170, 56, 356, 329, 31, 155, 143, 269, 153, 65, 58, 279, 304, 360, 283, 225, 192, 298, 197, 288, 278, 125, 184, 235, 150, 246, 3, 80, 333, 152, 203, 171, 302, 290, 101, 194, 367, 345, 39, 75, 21, 323, 220, 1, 336, 273, 49, 240, 213, 274, 338, 159, 120, 341, 251, 38, 82, 10, 160, 13, 26, 60, 232, 374, 296, 354, 23, 237, 144, 209, 342, 176, 375, 166, 310, 230, 319, 47, 234, 44, 66, 158, 259, 190, 35, 346, 236, 24, 366, 64, 261, 69, 193, 62, 95, 92, 267, 215, 5, 253, 88, 163, 103, 244, 99, 32, 128, 320, 218, 247, 294, 201, 139, 205, 129, 255, 87, 204, 221, 347, 348, 309, 169, 257, 210, 67, 293, 109, 43, 105, 141, 132, 364, 146, 42, 252, 89, 11, 19, 373, 369, 363, 229, 258, 177, 53, 22, 377, 217, 124, 271, 186, 45, 353, 111, 292, 57, 328, 86, 33, 202, 151, 63, 383, 268, 224, 228, 286, 90, 41, 196, 29, 254, 321, 355, 222, 145, 195, 91, 140, 239, 378, 165, 78, 231, 156, 157, 68, 4, 130, 324, 216, 6, 175, 238, 15, 351, 28, 7, 275, 335, 291, 164, 135, 74, 219, 131, 370, 316, 40, 85, 301, 46, 307, 93, 362, 104, 314, 300, 365, 191, 352, 207, 108, 282, 379, 148, 264, 121, 368, 178, 332, 212, 59, 305, 306, 8, 339, 162, 381, 126, 0, 372, 380, 337, 142, 83, 183, 122, 226, 325, 9, 315, 25, 119, 106, 312, 199, 276, 245, 37, 112, 185, 227, 357, 118, 256, 20, 94, 289, 16, 133, 174, 2, 371, 187, 182, 30, 208, 73, 326, 79, 72, 295, 272, 376, 330, 311, 71, 343, 340, 127, 327, 115, 136, 70, 14, 287, 188, 50, 172, 359, 18, 12, 84, 285, 313, 147, 317, 123, 138, 382, 322, 265, 34, 248, 308, 113, 102, 344, 55, 110, 318, 96, 77, 36, 114, 297, 154, 250, 211, 149, 81, 173, 270, 180, 358, 361, 243, 100, 223, 349, 48, 134] + 21: [360, 10, 17, 35, 58, 93, 123, 264, 306, 321, 339, 350, 152, 283, 297, 311, 327, 332, 11, 42, 44, 108, 179, 369, 355, 331, 286, 100, 258, 67, 68, 328, 166, 95, 107, 167, 47, 189, 49, 356, 353, 57, 319, 76, 202, 278, 235, 224, 173, 43, 73, 139, 125, 136, 27, 342, 6, 298, 248, 272, 281, 280, 131, 358, 28, 24, 346, 41, 357, 282, 101, 127, 132, 349, 367, 1, 260, 77, 340, 273, 74, 55, 188, 322, 103, 16, 69, 234, 62, 155, 137, 305, 241, 228, 326, 343, 316, 0, 121, 23, 267, 383, 40, 186, 96, 129, 242, 7, 20, 91, 175, 220, 18, 66, 275, 146, 12, 2, 249, 32, 371, 284, 109, 141, 372, 312, 204, 329, 300, 122, 85, 212, 308, 182, 19, 221, 338, 21, 65, 274, 335, 130, 99, 222, 254, 377, 128, 144, 84, 262, 323, 337, 333, 253, 265, 59, 4, 226, 151, 9, 174, 64, 133, 148, 83, 105, 106, 61, 158, 88, 149, 196, 245, 352, 303, 102, 36, 164, 255, 116, 80, 94, 250, 15, 72, 336, 159, 34, 379, 374, 114, 231, 124, 364, 178, 26, 230, 197, 223, 3, 115, 334, 294, 296, 376, 208, 150, 365, 135, 168, 309, 30, 171, 381, 70, 375, 119, 87, 252, 172, 111, 63, 304, 154, 266, 110, 163, 31, 71, 185, 118, 351, 313, 263, 60, 165, 160, 81, 75, 37, 257, 112, 345, 147, 209, 229, 318, 290, 187, 213, 143, 56, 113, 218, 45, 307, 52, 325, 39, 192, 203, 126, 156, 180, 140, 92, 214, 22, 344, 89, 259, 237, 301, 195, 181, 142, 302, 200, 120, 184, 227, 368, 97, 347, 292, 157, 315, 145, 289, 82, 199, 238, 219, 243, 362, 317, 198, 48, 201, 314, 153, 210, 169, 191, 225, 78, 98, 354, 170, 216, 207, 324, 190, 38, 348, 240, 194, 270, 287, 291, 269, 261, 239, 54, 256, 50, 117, 279, 299, 233, 359, 211, 14, 215, 268, 162, 236, 53, 232, 51, 8, 33, 251, 370, 104, 244, 330, 176, 46, 86, 277, 177, 25, 288, 246, 378, 79, 276, 13, 217, 247, 29, 363, 380, 293, 373, 138, 295, 193, 134, 382, 161, 206, 205, 361, 320, 366, 183, 90, 5, 341, 271, 310, 285] + 22: [254, 154, 157, 217, 247, 250, 253, 267, 307, 310, 346, 359, 141, 287, 289, 292, 323, 360, 2, 28, 79, 95, 101, 143, 181, 191, 225, 269, 96, 298, 344, 5, 166, 234, 135, 192, 221, 226, 131, 236, 136, 208, 350, 84, 293, 297, 142, 205, 260, 349, 25, 124, 138, 160, 128, 223, 86, 313, 249, 280, 261, 306, 185, 49, 341, 335, 211, 187, 212, 354, 194, 315, 328, 173, 222, 366, 321, 373, 290, 219, 255, 363, 203, 325, 281, 252, 80, 57, 374, 11, 81, 266, 296, 370, 231, 10, 216, 352, 375, 27, 18, 159, 299, 176, 294, 145, 114, 107, 343, 317, 153, 60, 82, 210, 322, 273, 262, 16, 21, 167, 278, 58, 345, 264, 48, 15, 32, 180, 186, 64, 43, 243, 209, 383, 338, 174, 339, 71, 238, 241, 137, 152, 172, 121, 148, 177, 55, 67, 206, 233, 22, 97, 246, 308, 6, 368, 24, 4, 240, 195, 122, 324, 189, 271, 274, 31, 150, 56, 224, 37, 333, 188, 8, 156, 1, 305, 26, 151, 179, 245, 337, 144, 130, 70, 193, 75, 239, 87, 46, 182, 190, 270, 288, 230, 355, 251, 378, 109, 319, 318, 379, 282, 91, 197, 183, 184, 235, 100, 68, 347, 367, 98, 364, 334, 134, 213, 20, 73, 329, 218, 83, 340, 12, 65, 356, 35, 291, 304, 178, 201, 118, 311, 372, 53, 59, 78, 382, 52, 30, 66, 376, 242, 61, 272, 116, 170, 248, 227, 314, 85, 19, 127, 110, 164, 244, 111, 161, 113, 320, 301, 275, 279, 104, 140, 94, 38, 93, 336, 76, 228, 72, 303, 353, 103, 257, 146, 371, 42, 44, 381, 369, 175, 99, 256, 268, 92, 36, 158, 316, 115, 29, 149, 365, 220, 165, 327, 0, 102, 168, 284, 33, 207, 112, 357, 214, 14, 39, 54, 202, 232, 69, 88, 204, 132, 74, 50, 123, 40, 41, 283, 286, 120, 62, 163, 51, 147, 263, 285, 312, 258, 133, 377, 300, 196, 129, 277, 198, 276, 139, 348, 259, 361, 47, 199, 215, 108, 342, 295, 9, 34, 117, 7, 106, 63, 309, 45, 13, 119, 90, 162, 126, 330, 169, 351, 125, 3, 105, 155, 77, 237, 171, 23, 332, 331, 362, 229, 17, 358, 380, 200, 89, 265, 302, 326] + 23: [26, 43, 235, 286, 320, 37, 81, 242, 265, 64, 168, 288, 314, 375, 357, 22, 262, 13, 234, 291, 39, 75, 189, 8, 124, 221, 98, 352, 108, 355, 277, 30, 142, 18, 3, 15, 172, 233, 210, 65, 45, 21, 373, 31, 141, 60, 226, 4, 275, 70, 255, 125, 54, 68, 104, 34, 42, 198, 366, 216, 71, 309, 224, 93, 179, 335, 25, 188, 381, 112, 251, 113, 305, 56, 102, 40, 17, 196, 206, 227, 88, 318, 166, 29, 126, 59, 111, 152, 254, 307, 107, 285, 158, 299, 332, 110, 328, 372, 213, 313, 160, 321, 346, 349, 61, 343, 217, 296, 264, 182, 190, 271, 361, 333, 339, 243, 248, 377, 199, 186, 175, 84, 130, 249, 215, 53, 76, 134, 267, 230, 208, 380, 100, 103, 159, 35, 38, 298, 250, 200, 336, 253, 364, 293, 132, 327, 57, 131, 228, 1, 204, 0, 115, 367, 118, 145, 187, 279, 169, 209, 317, 157, 207, 344, 268, 161, 165, 105, 167, 324, 127, 133, 149, 222, 178, 109, 155, 205, 238, 89, 244, 197, 241, 92, 191, 356, 5, 58, 354, 79, 322, 278, 323, 12, 11, 340, 229, 101, 301, 73, 121, 214, 337, 83, 252, 203, 304, 283, 91, 44, 137, 143, 14, 260, 350, 183, 85, 269, 374, 263, 151, 359, 325, 363, 122, 308, 383, 246, 72, 87, 192, 41, 220, 292, 273, 258, 114, 369, 266, 289, 300, 306, 67, 371, 312, 129, 231, 90, 212, 106, 62, 63, 135, 240, 180, 78, 36, 311, 52, 348, 163, 33, 276, 345, 177, 24, 94, 257, 378, 136, 66, 303, 7, 139, 368, 117, 281, 173, 232, 148, 74, 272, 329, 347, 16, 297, 46, 116, 225, 82, 156, 315, 219, 290, 365, 162, 245, 97, 202, 138, 51, 331, 176, 6, 146, 10, 370, 193, 128, 123, 140, 247, 69, 201, 261, 270, 379, 342, 153, 330, 28, 99, 9, 334, 274, 295, 27, 218, 50, 195, 185, 341, 164, 294, 319, 259, 120, 48, 154, 55, 184, 95, 119, 223, 302, 236, 310, 20, 376, 77, 358, 86, 326, 237, 96, 382, 174, 362, 353, 171, 211, 19, 47, 351, 80, 194, 32, 284, 280, 239, 282, 181, 147, 170, 287, 2, 144, 23, 150, 316, 256, 338, 49, 360] + 24: [154, 74, 81, 115, 166, 198, 241, 264, 298, 314, 353, 379, 265, 21, 127, 129, 133, 158, 179, 232, 251, 327, 368, 383, 252, 100, 152, 219, 358, 2, 45, 89, 34, 36, 155, 374, 9, 197, 365, 107, 359, 98, 300, 340, 117, 261, 330, 40, 246, 233, 108, 303, 8, 109, 332, 347, 208, 305, 364, 82, 20, 206, 278, 60, 14, 281, 132, 75, 285, 371, 322, 17, 7, 130, 48, 323, 30, 176, 373, 183, 124, 289, 33, 214, 223, 366, 49, 287, 295, 181, 88, 42, 248, 141, 316, 69, 0, 35, 142, 307, 235, 280, 170, 120, 342, 125, 165, 123, 52, 31, 299, 149, 178, 222, 275, 188, 55, 201, 167, 243, 92, 53, 306, 145, 58, 116, 282, 175, 171, 229, 199, 349, 6, 283, 54, 377, 77, 269, 225, 79, 126, 262, 187, 90, 22, 237, 190, 46, 12, 84, 339, 72, 212, 329, 247, 119, 372, 62, 164, 254, 380, 140, 313, 4, 150, 44, 25, 200, 263, 266, 26, 71, 177, 63, 56, 134, 249, 50, 101, 227, 357, 5, 137, 61, 350, 272, 13, 194, 348, 320, 102, 180, 70, 64, 231, 91, 356, 310, 16, 65, 276, 352, 209, 362, 369, 286, 343, 15, 113, 205, 168, 239, 23, 284, 217, 18, 85, 112, 319, 382, 138, 3, 153, 312, 271, 32, 57, 240, 189, 121, 230, 144, 156, 118, 321, 304, 333, 76, 331, 250, 128, 27, 207, 344, 355, 338, 83, 268, 103, 94, 148, 228, 290, 324, 375, 195, 260, 336, 257, 302, 309, 151, 67, 162, 97, 361, 186, 192, 308, 351, 169, 66, 215, 99, 191, 174, 87, 105, 370, 277, 106, 293, 59, 315, 203, 253, 238, 68, 211, 28, 146, 354, 226, 341, 376, 1, 346, 161, 19, 378, 110, 381, 29, 135, 131, 96, 147, 242, 267, 296, 317, 184, 111, 360, 255, 24, 318, 73, 326, 292, 270, 136, 160, 43, 95, 157, 301, 204, 274, 182, 221, 47, 37, 10, 193, 236, 328, 139, 334, 159, 41, 224, 279, 51, 218, 196, 288, 256, 172, 185, 294, 202, 273, 245, 297, 244, 258, 363, 291, 367, 259, 337, 104, 78, 93, 345, 114, 11, 335, 173, 234, 80, 210, 122, 220, 213, 325, 39, 216, 311, 38, 86, 163, 143] + 25: [315, 239, 156, 220, 263, 267, 277, 123, 178, 91, 158, 326, 269, 338, 43, 44, 303, 382, 19, 121, 165, 129, 189, 233, 379, 329, 6, 206, 106, 194, 13, 170, 151, 195, 270, 255, 282, 167, 141, 374, 108, 272, 288, 47, 148, 50, 182, 241, 72, 164, 132, 290, 370, 117, 70, 119, 71, 169, 198, 266, 139, 294, 256, 114, 350, 69, 67, 36, 283, 264, 12, 104, 228, 120, 153, 242, 295, 133, 79, 337, 175, 142, 285, 219, 247, 59, 84, 146, 352, 185, 24, 111, 16, 286, 42, 301, 317, 32, 234, 126, 347, 203, 149, 150, 93, 45, 155, 88, 180, 33, 215, 138, 143, 81, 201, 238, 186, 293, 224, 217, 276, 49, 262, 240, 161, 204, 213, 17, 271, 40, 92, 90, 23, 73, 318, 373, 227, 336, 184, 355, 275, 362, 118, 26, 372, 10, 216, 197, 343, 369, 357, 310, 243, 250, 64, 335, 107, 171, 380, 9, 279, 115, 199, 257, 300, 109, 311, 57, 214, 112, 314, 306, 60, 211, 237, 181, 212, 34, 183, 363, 29, 77, 102, 252, 366, 291, 319, 274, 356, 157, 320, 172, 284, 376, 296, 222, 367, 225, 316, 14, 105, 86, 124, 147, 192, 246, 38, 131, 244, 20, 304, 144, 11, 281, 210, 265, 62, 113, 66, 346, 99, 196, 187, 28, 223, 51, 87, 3, 98, 152, 8, 154, 253, 348, 312, 58, 251, 163, 173, 48, 65, 327, 160, 368, 323, 191, 278, 254, 53, 110, 128, 174, 83, 52, 280, 302, 125, 97, 383, 339, 159, 127, 236, 322, 258, 0, 5, 342, 56, 307, 330, 297, 74, 292, 208, 95, 168, 309, 190, 18, 229, 218, 324, 202, 249, 22, 226, 63, 268, 25, 145, 136, 231, 135, 259, 15, 179, 166, 245, 41, 298, 2, 232, 378, 89, 103, 55, 328, 193, 260, 371, 205, 334, 39, 333, 381, 209, 176, 130, 134, 340, 177, 364, 344, 82, 61, 235, 100, 341, 140, 221, 94, 31, 325, 96, 188, 299, 332, 273, 230, 54, 4, 68, 116, 200, 35, 122, 360, 85, 162, 313, 287, 377, 359, 345, 365, 361, 75, 21, 1, 101, 78, 7, 207, 80, 261, 308, 37, 358, 353, 289, 349, 30, 375, 305, 351, 76, 321, 354, 331, 27, 248, 46, 137] + 26: [136, 218, 228, 234, 242, 246, 261, 305, 348, 351, 374, 375, 237, 54, 68, 86, 96, 106, 111, 139, 169, 170, 175, 209, 89, 78, 346, 56, 250, 95, 207, 60, 203, 260, 299, 1, 269, 318, 370, 134, 317, 29, 208, 23, 155, 210, 132, 213, 217, 178, 251, 215, 50, 300, 361, 119, 282, 248, 273, 168, 172, 156, 151, 118, 252, 93, 352, 39, 163, 283, 286, 288, 90, 55, 166, 122, 35, 144, 362, 100, 193, 295, 322, 309, 80, 62, 360, 186, 239, 231, 3, 9, 314, 381, 366, 52, 6, 376, 128, 2, 20, 45, 177, 364, 244, 206, 330, 379, 292, 22, 180, 174, 66, 98, 75, 340, 34, 101, 85, 84, 380, 355, 200, 190, 24, 81, 298, 336, 335, 284, 211, 310, 11, 293, 131, 191, 79, 159, 256, 12, 275, 216, 222, 58, 124, 266, 167, 259, 97, 17, 41, 114, 47, 297, 296, 345, 18, 316, 30, 204, 27, 51, 115, 123, 121, 59, 199, 28, 53, 344, 315, 359, 99, 268, 171, 129, 165, 74, 205, 46, 236, 337, 135, 67, 287, 194, 0, 143, 140, 77, 274, 147, 21, 279, 158, 265, 353, 294, 107, 153, 105, 40, 258, 164, 181, 16, 238, 110, 179, 173, 195, 350, 125, 72, 270, 188, 291, 289, 227, 328, 323, 241, 277, 37, 197, 333, 240, 235, 253, 116, 369, 332, 319, 103, 65, 196, 49, 112, 326, 257, 339, 120, 307, 382, 26, 102, 357, 32, 189, 141, 154, 243, 230, 224, 278, 146, 138, 327, 15, 267, 232, 325, 334, 313, 150, 254, 341, 126, 198, 306, 127, 202, 182, 220, 303, 13, 320, 160, 38, 92, 176, 44, 285, 290, 372, 7, 347, 377, 5, 149, 308, 36, 48, 229, 57, 378, 312, 264, 8, 324, 233, 162, 130, 108, 311, 142, 192, 161, 117, 368, 43, 358, 281, 304, 137, 245, 272, 152, 104, 212, 383, 329, 64, 373, 349, 338, 183, 88, 31, 367, 365, 73, 363, 157, 185, 4, 271, 70, 33, 249, 148, 356, 71, 87, 25, 301, 255, 14, 214, 263, 94, 223, 82, 262, 201, 113, 280, 219, 354, 145, 63, 371, 276, 226, 247, 133, 343, 76, 187, 225, 331, 91, 342, 184, 69, 61, 83, 321, 221, 10, 19, 302, 109, 42] + 27: [103, 137, 146, 163, 176, 220, 240, 244, 262, 310, 338, 342, 225, 232, 241, 218, 249, 367, 313, 64, 56, 212, 270, 53, 286, 216, 118, 10, 18, 365, 96, 150, 194, 235, 285, 60, 12, 99, 356, 54, 46, 321, 144, 50, 86, 245, 236, 39, 312, 228, 325, 254, 214, 43, 256, 302, 372, 209, 17, 63, 90, 182, 32, 42, 136, 307, 0, 231, 52, 243, 36, 93, 234, 112, 196, 109, 329, 362, 47, 306, 185, 198, 19, 180, 37, 97, 149, 25, 126, 287, 120, 123, 223, 94, 138, 200, 177, 207, 242, 297, 91, 375, 199, 153, 8, 68, 255, 203, 4, 358, 251, 78, 305, 296, 29, 13, 281, 173, 378, 51, 263, 49, 183, 343, 370, 6, 106, 151, 22, 158, 210, 100, 61, 119, 55, 258, 193, 326, 67, 95, 280, 330, 226, 247, 265, 320, 190, 339, 373, 271, 267, 250, 344, 282, 334, 221, 168, 315, 237, 383, 141, 316, 276, 353, 132, 113, 1, 277, 73, 357, 336, 324, 284, 368, 124, 147, 79, 65, 195, 295, 252, 197, 143, 76, 322, 309, 45, 248, 14, 152, 87, 298, 69, 135, 222, 266, 140, 125, 122, 299, 116, 74, 257, 355, 30, 26, 179, 380, 278, 161, 215, 101, 167, 184, 108, 40, 349, 269, 128, 75, 201, 261, 174, 28, 84, 352, 16, 92, 80, 204, 376, 335, 82, 7, 114, 20, 105, 319, 166, 148, 227, 164, 202, 230, 332, 187, 274, 38, 275, 41, 110, 160, 341, 71, 366, 303, 102, 162, 379, 129, 139, 259, 165, 169, 211, 233, 24, 48, 238, 155, 291, 2, 288, 264, 273, 170, 34, 374, 213, 85, 72, 328, 246, 111, 130, 346, 189, 311, 354, 300, 131, 178, 115, 27, 304, 121, 217, 154, 345, 363, 81, 350, 181, 290, 294, 15, 172, 171, 35, 359, 283, 314, 301, 31, 292, 272, 83, 337, 268, 9, 205, 293, 62, 331, 104, 70, 188, 323, 77, 88, 175, 239, 33, 208, 192, 11, 145, 351, 369, 340, 308, 317, 360, 44, 361, 186, 142, 371, 117, 333, 377, 21, 156, 133, 327, 318, 107, 253, 382, 57, 219, 58, 348, 260, 23, 191, 127, 347, 159, 157, 289, 134, 59, 279, 224, 381, 89, 98, 5, 66, 3, 229, 364, 206] + 28: [161, 180, 207, 215, 257, 279, 311, 329, 336, 338, 343, 353, 250, 190, 298, 359, 379, 10, 18, 60, 86, 115, 150, 156, 26, 67, 367, 14, 314, 15, 364, 303, 76, 145, 301, 173, 185, 59, 163, 376, 121, 205, 372, 348, 101, 181, 2, 132, 168, 346, 88, 240, 178, 6, 148, 31, 89, 174, 187, 9, 344, 7, 116, 32, 300, 327, 362, 154, 293, 126, 189, 138, 92, 307, 42, 167, 238, 278, 118, 66, 170, 72, 276, 147, 356, 253, 325, 135, 13, 380, 244, 210, 211, 85, 237, 11, 153, 113, 368, 285, 217, 16, 112, 322, 294, 38, 267, 309, 70, 214, 223, 339, 349, 247, 179, 61, 34, 225, 275, 313, 317, 93, 227, 28, 19, 48, 256, 270, 334, 146, 347, 40, 255, 122, 137, 90, 289, 290, 64, 159, 79, 96, 373, 318, 306, 288, 320, 361, 370, 282, 44, 69, 377, 80, 47, 186, 302, 103, 236, 383, 182, 316, 245, 106, 366, 284, 35, 304, 162, 233, 166, 200, 125, 134, 49, 143, 24, 351, 141, 371, 243, 172, 378, 352, 209, 268, 142, 258, 73, 39, 111, 53, 30, 195, 75, 228, 308, 193, 183, 12, 332, 8, 192, 212, 107, 155, 374, 261, 242, 222, 57, 82, 109, 119, 123, 254, 52, 97, 272, 158, 340, 43, 27, 104, 342, 231, 139, 262, 25, 120, 199, 5, 221, 341, 365, 169, 363, 234, 259, 62, 99, 176, 37, 58, 299, 68, 171, 305, 117, 149, 140, 354, 286, 226, 357, 165, 20, 114, 46, 175, 151, 110, 277, 360, 241, 152, 263, 204, 56, 127, 264, 36, 133, 131, 232, 198, 197, 203, 108, 51, 216, 315, 265, 296, 280, 246, 29, 319, 321, 188, 369, 71, 33, 273, 295, 375, 218, 213, 291, 83, 4, 328, 224, 100, 269, 333, 208, 381, 330, 266, 45, 202, 355, 283, 50, 129, 271, 206, 194, 230, 297, 292, 55, 87, 312, 324, 128, 157, 41, 184, 249, 21, 136, 251, 94, 350, 0, 22, 219, 63, 326, 1, 358, 177, 382, 345, 252, 124, 248, 77, 95, 235, 105, 335, 260, 74, 84, 17, 229, 54, 201, 196, 191, 160, 337, 78, 81, 98, 144, 287, 102, 65, 239, 310, 3, 130, 220, 91, 23, 274, 331, 323, 281, 164] + 29: [118, 324, 19, 205, 79, 199, 252, 8, 98, 355, 152, 176, 244, 4, 18, 105, 157, 84, 162, 234, 64, 212, 29, 66, 361, 123, 248, 380, 217, 111, 374, 251, 279, 367, 108, 91, 197, 175, 350, 378, 292, 167, 327, 288, 141, 189, 256, 26, 73, 87, 357, 85, 101, 359, 55, 261, 17, 286, 335, 240, 196, 16, 82, 329, 280, 312, 52, 356, 290, 201, 274, 322, 336, 208, 343, 172, 72, 313, 230, 153, 149, 254, 284, 363, 318, 202, 24, 132, 347, 269, 170, 7, 376, 49, 37, 39, 51, 320, 188, 61, 225, 125, 126, 158, 168, 325, 1, 285, 137, 309, 332, 364, 136, 20, 93, 70, 113, 334, 260, 81, 277, 186, 235, 94, 131, 32, 315, 42, 227, 30, 41, 143, 323, 74, 190, 278, 349, 146, 83, 377, 62, 0, 80, 138, 198, 262, 134, 50, 360, 142, 373, 381, 13, 306, 291, 107, 203, 110, 375, 145, 182, 104, 294, 103, 338, 43, 245, 241, 295, 33, 159, 68, 12, 181, 316, 209, 192, 246, 223, 144, 304, 265, 57, 216, 139, 59, 229, 147, 233, 296, 358, 249, 362, 339, 112, 207, 273, 264, 169, 302, 194, 71, 5, 268, 67, 129, 160, 102, 178, 228, 224, 28, 239, 297, 133, 281, 60, 333, 351, 293, 298, 353, 317, 287, 161, 219, 27, 342, 236, 258, 148, 38, 314, 267, 344, 114, 166, 99, 75, 165, 44, 354, 266, 319, 156, 171, 242, 369, 270, 154, 130, 221, 282, 120, 346, 382, 200, 300, 348, 40, 307, 263, 226, 321, 173, 379, 183, 193, 215, 345, 77, 213, 253, 92, 96, 6, 23, 76, 243, 163, 128, 78, 328, 368, 250, 308, 231, 372, 311, 25, 65, 86, 259, 3, 371, 56, 299, 109, 155, 177, 127, 54, 174, 370, 116, 151, 47, 53, 15, 45, 21, 289, 11, 365, 119, 341, 257, 276, 31, 10, 352, 164, 301, 185, 255, 124, 326, 210, 218, 310, 340, 220, 9, 115, 95, 305, 89, 195, 140, 237, 232, 90, 238, 303, 187, 184, 100, 117, 383, 58, 48, 63, 222, 97, 122, 191, 69, 337, 330, 247, 271, 88, 366, 275, 106, 272, 36, 46, 121, 35, 331, 214, 283, 14, 180, 204, 34, 150, 22, 135, 179, 211, 2, 206] + 30: [208, 189, 218, 219, 237, 259, 268, 269, 322, 334, 345, 380, 239, 40, 47, 88, 94, 101, 118, 123, 128, 134, 146, 185, 140, 305, 307, 331, 354, 363, 379, 8, 21, 22, 31, 36, 266, 374, 324, 273, 198, 56, 87, 70, 201, 227, 238, 301, 65, 336, 309, 104, 179, 349, 263, 304, 126, 300, 27, 131, 245, 74, 217, 356, 51, 297, 230, 69, 368, 229, 34, 30, 37, 362, 256, 157, 162, 375, 350, 90, 173, 80, 119, 176, 262, 271, 339, 260, 108, 280, 175, 194, 165, 178, 149, 191, 187, 28, 224, 195, 281, 310, 92, 337, 209, 16, 199, 251, 358, 234, 43, 299, 98, 114, 180, 220, 32, 190, 235, 252, 53, 188, 303, 71, 274, 38, 66, 145, 139, 117, 265, 4, 214, 361, 250, 152, 296, 167, 129, 290, 186, 327, 308, 313, 52, 11, 315, 100, 170, 84, 172, 284, 288, 370, 232, 270, 35, 206, 302, 6, 18, 261, 111, 79, 318, 130, 3, 63, 143, 2, 316, 1, 344, 353, 197, 44, 360, 82, 364, 171, 351, 295, 205, 177, 382, 278, 376, 50, 25, 5, 81, 367, 373, 168, 17, 253, 292, 91, 14, 248, 276, 85, 122, 148, 156, 257, 212, 357, 277, 96, 72, 109, 267, 161, 330, 338, 62, 46, 283, 12, 275, 200, 54, 286, 371, 57, 127, 26, 289, 24, 164, 221, 215, 365, 359, 241, 347, 326, 113, 112, 133, 225, 68, 77, 192, 174, 163, 211, 107, 264, 120, 207, 158, 378, 147, 317, 58, 23, 383, 55, 121, 348, 255, 102, 254, 59, 33, 89, 9, 132, 103, 293, 247, 203, 153, 61, 124, 78, 41, 341, 236, 325, 231, 332, 183, 279, 352, 106, 184, 196, 64, 97, 141, 298, 142, 48, 95, 83, 291, 93, 213, 135, 321, 49, 136, 369, 294, 7, 154, 125, 314, 182, 333, 242, 223, 312, 226, 285, 20, 249, 311, 155, 355, 228, 76, 75, 10, 216, 204, 160, 319, 282, 342, 287, 15, 328, 222, 366, 144, 159, 320, 67, 150, 73, 377, 99, 105, 372, 272, 243, 45, 240, 151, 60, 335, 244, 202, 381, 169, 39, 86, 13, 246, 138, 193, 343, 329, 115, 166, 323, 233, 42, 29, 137, 346, 340, 210, 258, 306, 181, 110, 116, 19, 0] + 31: [0, 246, 289, 294, 347, 172, 229, 120, 132, 207, 329, 350, 159, 49, 43, 375, 113, 313, 165, 89, 180, 37, 90, 149, 128, 81, 310, 20, 351, 99, 280, 297, 117, 203, 71, 212, 233, 189, 40, 15, 118, 12, 194, 367, 274, 18, 343, 148, 26, 34, 145, 243, 256, 195, 47, 42, 126, 167, 67, 341, 201, 206, 286, 380, 176, 97, 354, 107, 315, 52, 377, 200, 68, 186, 355, 254, 31, 374, 361, 103, 129, 127, 199, 154, 100, 317, 208, 231, 134, 383, 247, 352, 321, 7, 86, 13, 166, 339, 378, 33, 66, 41, 29, 323, 308, 64, 35, 30, 324, 267, 61, 223, 54, 287, 84, 272, 226, 327, 282, 242, 158, 270, 170, 296, 55, 202, 8, 266, 162, 234, 53, 163, 252, 95, 171, 106, 245, 316, 291, 177, 306, 300, 293, 94, 143, 187, 188, 290, 98, 190, 142, 59, 91, 340, 156, 218, 115, 215, 333, 258, 23, 77, 373, 51, 264, 160, 181, 257, 239, 147, 58, 269, 38, 318, 21, 196, 50, 298, 235, 311, 278, 25, 60, 112, 17, 222, 225, 73, 238, 85, 255, 27, 144, 250, 155, 348, 211, 65, 141, 376, 275, 262, 79, 332, 363, 69, 119, 214, 78, 122, 369, 6, 83, 337, 193, 359, 70, 335, 140, 334, 249, 46, 284, 260, 304, 157, 135, 151, 261, 326, 22, 102, 28, 88, 16, 271, 220, 9, 381, 104, 185, 133, 209, 62, 152, 277, 92, 273, 224, 210, 295, 174, 87, 364, 349, 276, 82, 182, 123, 183, 228, 178, 109, 5, 76, 116, 325, 253, 93, 175, 303, 237, 48, 309, 124, 114, 227, 292, 39, 131, 236, 205, 213, 4, 198, 139, 344, 179, 248, 121, 150, 72, 232, 366, 217, 362, 130, 345, 372, 301, 320, 379, 219, 360, 74, 368, 302, 108, 204, 24, 356, 10, 125, 279, 330, 56, 111, 322, 314, 153, 281, 101, 382, 161, 331, 36, 173, 283, 244, 105, 307, 110, 263, 259, 44, 319, 265, 3, 96, 338, 353, 137, 312, 285, 80, 240, 370, 184, 32, 365, 268, 164, 138, 169, 45, 14, 346, 342, 197, 357, 192, 241, 146, 75, 136, 57, 336, 11, 371, 168, 191, 230, 221, 19, 328, 299, 63, 2, 288, 305, 1, 251, 216, 358] + 32: [63, 177, 181, 244, 245, 251, 262, 265, 296, 312, 373, 375, 367, 316, 379, 57, 68, 147, 167, 173, 383, 14, 123, 169, 352, 382, 315, 320, 13, 118, 88, 11, 90, 145, 162, 300, 94, 283, 289, 69, 237, 276, 82, 261, 196, 93, 226, 114, 218, 184, 120, 111, 230, 4, 154, 124, 201, 170, 159, 333, 148, 174, 357, 71, 59, 80, 372, 103, 238, 277, 215, 235, 33, 202, 113, 104, 332, 115, 366, 243, 117, 224, 242, 267, 92, 219, 140, 301, 50, 127, 6, 5, 376, 23, 180, 15, 285, 274, 211, 189, 231, 256, 24, 339, 105, 364, 358, 328, 347, 257, 279, 7, 26, 293, 247, 110, 248, 1, 216, 27, 190, 12, 151, 306, 199, 32, 337, 34, 20, 278, 192, 54, 241, 2, 126, 345, 101, 142, 129, 144, 341, 31, 259, 326, 163, 121, 83, 294, 185, 253, 30, 172, 195, 0, 355, 194, 51, 360, 263, 58, 292, 182, 205, 311, 125, 305, 343, 272, 122, 66, 75, 252, 354, 86, 302, 363, 287, 365, 60, 260, 327, 280, 96, 19, 85, 314, 350, 349, 152, 197, 46, 321, 38, 8, 225, 228, 91, 213, 84, 309, 282, 44, 52, 318, 164, 200, 65, 156, 72, 28, 81, 10, 361, 119, 79, 207, 107, 370, 89, 87, 212, 250, 240, 378, 165, 304, 76, 368, 344, 21, 348, 131, 254, 48, 351, 236, 42, 270, 208, 112, 166, 130, 186, 331, 297, 178, 336, 317, 221, 330, 138, 291, 329, 271, 319, 97, 227, 269, 3, 381, 9, 374, 377, 98, 322, 310, 380, 116, 369, 45, 286, 158, 99, 264, 188, 217, 275, 229, 204, 78, 303, 25, 53, 193, 102, 134, 67, 295, 288, 179, 359, 143, 36, 175, 70, 22, 132, 73, 255, 47, 155, 323, 40, 258, 325, 64, 371, 340, 284, 153, 220, 17, 246, 106, 61, 308, 171, 157, 249, 62, 307, 356, 334, 74, 234, 187, 214, 233, 313, 150, 183, 139, 136, 35, 149, 290, 135, 209, 16, 335, 168, 281, 39, 56, 43, 128, 100, 95, 268, 362, 41, 273, 298, 324, 353, 198, 191, 222, 133, 18, 346, 146, 342, 109, 266, 239, 29, 77, 299, 223, 137, 161, 49, 108, 141, 232, 55, 206, 176, 37, 210, 160, 203, 338] + 33: [38, 32, 95, 102, 69, 142, 220, 277, 287, 293, 333, 372, 91, 311, 317, 114, 43, 135, 207, 250, 309, 264, 12, 34, 339, 151, 305, 79, 279, 15, 324, 332, 326, 185, 189, 161, 36, 335, 356, 187, 19, 122, 26, 238, 365, 228, 159, 129, 319, 37, 155, 351, 246, 105, 371, 194, 243, 275, 321, 83, 260, 314, 376, 308, 355, 373, 113, 128, 286, 306, 344, 236, 177, 73, 255, 131, 205, 156, 345, 29, 78, 341, 357, 265, 362, 342, 190, 192, 93, 256, 58, 166, 6, 346, 7, 313, 64, 82, 48, 117, 211, 375, 213, 3, 248, 188, 242, 46, 380, 183, 169, 237, 329, 262, 234, 71, 261, 119, 178, 143, 229, 39, 127, 208, 202, 162, 60, 281, 20, 74, 171, 44, 148, 107, 361, 227, 303, 196, 364, 215, 41, 225, 92, 57, 81, 297, 266, 193, 232, 123, 163, 336, 219, 27, 315, 47, 343, 96, 206, 210, 8, 276, 152, 249, 295, 184, 337, 203, 115, 379, 139, 25, 126, 347, 212, 49, 101, 312, 244, 223, 251, 325, 84, 33, 290, 216, 200, 350, 327, 291, 363, 302, 258, 231, 72, 1, 354, 383, 360, 24, 285, 62, 118, 239, 301, 121, 164, 30, 144, 254, 369, 137, 53, 310, 353, 263, 283, 323, 226, 150, 61, 80, 13, 170, 271, 120, 98, 272, 299, 158, 5, 338, 18, 45, 180, 330, 182, 111, 274, 99, 154, 42, 352, 87, 116, 198, 88, 106, 298, 181, 253, 269, 307, 31, 108, 160, 235, 124, 340, 368, 278, 245, 134, 0, 267, 153, 146, 218, 14, 22, 186, 292, 67, 132, 97, 172, 94, 377, 201, 35, 288, 4, 197, 70, 136, 85, 165, 296, 89, 191, 125, 273, 334, 149, 209, 349, 23, 300, 374, 359, 86, 280, 68, 233, 133, 173, 367, 28, 112, 257, 247, 282, 167, 331, 179, 270, 217, 174, 370, 66, 110, 224, 59, 77, 328, 259, 348, 21, 289, 230, 382, 52, 221, 168, 199, 304, 51, 63, 147, 141, 268, 90, 50, 56, 316, 130, 195, 2, 240, 320, 322, 54, 222, 138, 104, 366, 252, 381, 378, 284, 140, 145, 358, 204, 9, 16, 157, 103, 318, 76, 175, 100, 176, 75, 241, 17, 40, 109, 55, 65, 10, 11, 214, 294] + 34: [108, 76, 119, 122, 132, 160, 189, 193, 202, 229, 363, 365, 179, 377, 177, 198, 211, 213, 224, 279, 316, 341, 381, 14, 16, 10, 298, 282, 55, 171, 7, 309, 32, 66, 273, 343, 249, 87, 84, 336, 30, 322, 261, 289, 155, 112, 376, 375, 306, 92, 58, 37, 102, 301, 88, 100, 131, 351, 320, 121, 15, 2, 21, 327, 346, 263, 64, 226, 49, 355, 350, 129, 340, 97, 201, 123, 241, 50, 283, 275, 255, 104, 106, 235, 366, 79, 208, 175, 181, 5, 62, 268, 75, 9, 43, 303, 337, 295, 225, 313, 245, 35, 127, 56, 305, 297, 20, 369, 69, 284, 162, 256, 328, 367, 323, 139, 228, 278, 182, 378, 259, 220, 244, 12, 217, 48, 103, 45, 36, 370, 188, 38, 299, 347, 65, 345, 70, 270, 277, 149, 233, 173, 120, 94, 126, 178, 271, 184, 197, 247, 19, 221, 269, 250, 156, 234, 330, 362, 335, 107, 166, 204, 154, 51, 111, 260, 292, 276, 334, 240, 83, 91, 368, 236, 230, 85, 353, 176, 96, 252, 248, 293, 59, 95, 302, 33, 150, 114, 23, 314, 253, 90, 218, 243, 115, 77, 159, 67, 72, 308, 192, 307, 205, 130, 3, 321, 99, 4, 326, 195, 227, 157, 46, 374, 339, 212, 44, 185, 379, 125, 174, 251, 73, 180, 147, 161, 17, 257, 196, 25, 146, 6, 200, 280, 231, 294, 105, 13, 342, 1, 190, 223, 360, 267, 153, 133, 214, 318, 358, 262, 26, 22, 61, 145, 203, 11, 329, 60, 333, 232, 152, 331, 242, 163, 372, 304, 167, 288, 290, 187, 364, 361, 68, 34, 109, 216, 382, 354, 40, 8, 81, 183, 170, 42, 371, 310, 285, 71, 29, 158, 138, 238, 210, 24, 89, 215, 98, 93, 117, 325, 207, 239, 148, 191, 356, 199, 359, 352, 264, 265, 165, 344, 47, 82, 291, 254, 53, 136, 116, 80, 266, 86, 319, 338, 237, 311, 31, 41, 110, 143, 209, 113, 272, 274, 52, 54, 151, 135, 315, 168, 134, 172, 0, 140, 383, 286, 57, 128, 324, 74, 246, 300, 124, 317, 27, 164, 186, 101, 380, 349, 357, 312, 258, 78, 144, 137, 169, 348, 296, 118, 28, 281, 332, 18, 194, 222, 373, 206, 287, 63, 142, 219, 39, 141] + 35: [319, 90, 91, 196, 105, 207, 225, 336, 359, 228, 177, 343, 45, 163, 192, 11, 82, 344, 195, 183, 124, 146, 60, 293, 1, 46, 98, 5, 32, 369, 66, 135, 354, 193, 0, 13, 345, 364, 27, 9, 372, 172, 19, 285, 262, 10, 212, 381, 363, 15, 159, 267, 209, 111, 383, 292, 253, 310, 51, 40, 75, 355, 52, 357, 95, 350, 185, 332, 29, 107, 120, 291, 169, 4, 362, 272, 108, 96, 127, 73, 250, 238, 229, 304, 28, 138, 265, 373, 137, 57, 7, 25, 94, 101, 173, 284, 205, 47, 311, 275, 93, 302, 117, 84, 181, 148, 251, 200, 299, 257, 114, 157, 282, 256, 264, 175, 189, 288, 219, 59, 218, 141, 375, 312, 171, 12, 106, 17, 248, 77, 123, 242, 131, 55, 352, 37, 61, 298, 348, 382, 365, 316, 178, 341, 297, 286, 41, 329, 308, 215, 323, 301, 217, 23, 165, 283, 367, 314, 186, 54, 134, 300, 227, 24, 338, 249, 197, 71, 295, 318, 224, 222, 187, 243, 145, 216, 63, 259, 371, 188, 68, 67, 78, 331, 296, 313, 202, 34, 374, 306, 271, 76, 133, 368, 239, 126, 102, 122, 287, 142, 279, 49, 255, 100, 53, 56, 147, 136, 321, 152, 74, 99, 378, 26, 263, 85, 86, 211, 289, 50, 244, 42, 109, 376, 6, 115, 69, 39, 342, 254, 3, 241, 325, 334, 235, 179, 35, 232, 351, 119, 203, 379, 221, 326, 132, 268, 110, 339, 103, 112, 358, 118, 276, 303, 231, 335, 18, 294, 170, 269, 79, 377, 233, 208, 72, 58, 346, 234, 81, 328, 214, 347, 176, 149, 83, 210, 322, 130, 154, 361, 360, 337, 8, 273, 87, 161, 33, 252, 113, 237, 206, 191, 128, 30, 16, 70, 315, 89, 97, 199, 281, 144, 116, 194, 36, 104, 305, 330, 198, 245, 270, 247, 340, 62, 31, 164, 230, 190, 180, 204, 274, 278, 327, 226, 48, 240, 38, 307, 158, 213, 261, 380, 43, 121, 167, 64, 309, 182, 129, 162, 168, 201, 92, 184, 317, 22, 139, 236, 80, 246, 153, 266, 160, 125, 20, 290, 151, 156, 333, 280, 353, 44, 324, 349, 140, 143, 277, 174, 88, 155, 150, 2, 220, 260, 21, 14, 356, 320, 366, 166, 65, 370, 223, 258] + 36: [232, 64, 127, 313, 336, 37, 123, 170, 273, 302, 360, 375, 22, 171, 310, 157, 135, 209, 253, 26, 317, 145, 292, 25, 184, 138, 289, 343, 36, 149, 383, 120, 124, 306, 279, 16, 371, 233, 293, 284, 195, 71, 220, 265, 137, 227, 367, 290, 368, 5, 108, 91, 177, 200, 243, 158, 14, 128, 214, 186, 272, 133, 309, 132, 354, 211, 244, 44, 337, 303, 377, 270, 53, 152, 321, 167, 153, 323, 165, 239, 19, 126, 114, 376, 340, 73, 162, 288, 20, 117, 76, 318, 259, 110, 203, 160, 341, 352, 314, 49, 105, 228, 212, 159, 189, 226, 96, 218, 144, 369, 85, 319, 322, 216, 206, 148, 173, 35, 141, 221, 57, 326, 31, 219, 342, 100, 264, 48, 70, 61, 1, 181, 304, 320, 93, 166, 50, 224, 0, 263, 198, 60, 269, 234, 238, 242, 47, 345, 95, 116, 255, 356, 190, 254, 285, 222, 266, 296, 92, 139, 297, 102, 305, 58, 329, 8, 235, 43, 10, 196, 205, 334, 122, 364, 180, 18, 363, 87, 378, 113, 325, 98, 79, 262, 103, 237, 39, 163, 6, 215, 286, 182, 311, 258, 308, 201, 63, 346, 134, 88, 281, 249, 161, 268, 178, 210, 150, 240, 13, 300, 280, 335, 188, 77, 256, 4, 193, 230, 301, 33, 67, 155, 140, 257, 208, 204, 324, 34, 225, 260, 56, 65, 9, 183, 366, 38, 267, 248, 316, 106, 372, 299, 40, 331, 246, 15, 147, 81, 23, 338, 115, 86, 12, 101, 274, 42, 194, 164, 202, 187, 29, 24, 312, 151, 223, 328, 241, 45, 362, 72, 174, 143, 80, 236, 21, 353, 107, 2, 373, 156, 112, 276, 11, 52, 283, 247, 350, 278, 131, 59, 380, 32, 94, 245, 46, 229, 365, 295, 355, 154, 192, 62, 90, 175, 111, 339, 179, 344, 119, 315, 307, 327, 252, 359, 99, 261, 231, 298, 41, 104, 51, 83, 213, 7, 294, 185, 374, 54, 282, 121, 277, 217, 146, 109, 347, 136, 197, 348, 97, 287, 361, 3, 142, 17, 351, 251, 130, 82, 370, 250, 207, 125, 55, 172, 271, 28, 358, 199, 357, 89, 382, 84, 275, 332, 349, 69, 168, 379, 129, 381, 75, 330, 74, 176, 291, 333, 191, 68, 66, 30, 78, 169, 118, 27] + 37: [218, 120, 226, 8, 75, 112, 132, 236, 239, 288, 352, 383, 40, 377, 67, 84, 99, 209, 251, 263, 31, 189, 56, 375, 268, 287, 376, 334, 105, 256, 62, 214, 275, 232, 238, 276, 294, 322, 6, 304, 380, 317, 60, 295, 307, 367, 23, 25, 148, 328, 19, 203, 161, 244, 271, 137, 183, 369, 12, 358, 293, 129, 118, 144, 111, 83, 4, 368, 96, 70, 211, 50, 109, 171, 93, 320, 296, 356, 373, 298, 291, 130, 186, 68, 301, 140, 235, 127, 43, 92, 128, 102, 260, 63, 107, 282, 379, 86, 318, 359, 21, 103, 55, 42, 91, 142, 237, 346, 58, 223, 353, 277, 173, 264, 207, 279, 286, 297, 159, 90, 158, 292, 182, 257, 168, 141, 347, 212, 332, 310, 250, 381, 222, 179, 302, 343, 229, 338, 126, 313, 26, 378, 324, 246, 200, 272, 47, 194, 316, 61, 87, 195, 172, 175, 231, 2, 366, 48, 355, 181, 85, 82, 100, 312, 249, 169, 57, 89, 79, 190, 344, 233, 208, 230, 278, 247, 354, 10, 35, 20, 349, 372, 114, 117, 336, 365, 285, 136, 5, 13, 106, 202, 162, 341, 156, 204, 370, 217, 333, 331, 210, 299, 146, 227, 113, 362, 16, 147, 145, 51, 135, 206, 267, 110, 166, 281, 340, 289, 59, 65, 345, 258, 329, 69, 325, 17, 101, 180, 0, 363, 361, 139, 164, 104, 134, 49, 187, 124, 248, 326, 188, 46, 174, 327, 153, 149, 201, 335, 154, 138, 225, 300, 240, 199, 382, 88, 330, 64, 39, 37, 33, 53, 259, 11, 319, 205, 77, 185, 315, 228, 34, 262, 284, 350, 283, 45, 71, 178, 371, 22, 32, 1, 41, 269, 24, 152, 339, 54, 364, 52, 234, 165, 131, 74, 143, 121, 274, 215, 374, 220, 66, 306, 95, 14, 163, 323, 30, 98, 94, 170, 81, 28, 273, 125, 193, 253, 150, 76, 3, 337, 280, 266, 191, 119, 123, 151, 72, 7, 133, 27, 108, 224, 351, 254, 357, 155, 243, 314, 290, 252, 18, 196, 73, 255, 97, 216, 198, 177, 265, 213, 44, 9, 192, 176, 36, 321, 38, 303, 308, 197, 348, 311, 305, 245, 221, 270, 29, 167, 116, 160, 184, 261, 242, 309, 342, 360, 157, 122, 15, 80, 241, 78, 115, 219] + 38: [301, 52, 101, 191, 210, 46, 60, 185, 237, 251, 325, 380, 6, 121, 304, 312, 64, 98, 159, 161, 176, 339, 340, 38, 266, 125, 193, 142, 170, 250, 11, 231, 18, 232, 270, 346, 203, 169, 341, 224, 367, 84, 310, 375, 202, 281, 330, 318, 158, 373, 234, 95, 359, 260, 376, 104, 25, 163, 175, 243, 378, 32, 327, 355, 96, 228, 149, 200, 183, 24, 209, 133, 221, 220, 206, 90, 362, 368, 81, 296, 247, 364, 128, 370, 269, 309, 135, 361, 146, 349, 188, 353, 43, 295, 316, 127, 280, 194, 328, 106, 352, 136, 265, 181, 302, 102, 164, 160, 124, 248, 317, 225, 99, 308, 190, 356, 100, 311, 236, 155, 201, 274, 212, 16, 219, 156, 117, 26, 261, 204, 314, 291, 284, 115, 279, 97, 73, 255, 34, 139, 74, 61, 299, 1, 372, 226, 0, 338, 14, 307, 141, 134, 129, 80, 343, 140, 51, 322, 278, 53, 9, 150, 241, 28, 347, 215, 42, 20, 30, 285, 166, 268, 72, 36, 276, 272, 238, 13, 240, 198, 366, 354, 17, 54, 321, 211, 111, 235, 56, 252, 259, 132, 189, 263, 130, 137, 275, 165, 246, 258, 334, 35, 91, 303, 126, 86, 19, 108, 208, 45, 5, 180, 273, 44, 113, 22, 383, 8, 217, 119, 271, 58, 48, 40, 360, 55, 254, 218, 114, 292, 92, 344, 21, 262, 148, 41, 182, 351, 320, 315, 154, 332, 27, 267, 85, 323, 298, 78, 79, 205, 326, 365, 277, 123, 381, 253, 227, 57, 179, 152, 214, 186, 357, 162, 333, 94, 145, 300, 239, 144, 342, 174, 337, 290, 335, 377, 89, 118, 244, 77, 29, 87, 229, 2, 23, 71, 65, 153, 62, 66, 49, 319, 69, 88, 7, 15, 313, 283, 12, 83, 282, 245, 293, 286, 222, 264, 151, 242, 122, 168, 70, 110, 10, 345, 63, 288, 147, 184, 143, 103, 195, 374, 192, 196, 4, 348, 171, 233, 107, 256, 109, 305, 172, 294, 363, 249, 187, 178, 230, 329, 75, 216, 31, 76, 120, 82, 350, 207, 223, 369, 39, 331, 67, 105, 289, 33, 173, 138, 112, 116, 93, 50, 379, 199, 324, 68, 37, 131, 371, 297, 3, 336, 213, 287, 177, 257, 59, 358, 306, 197, 382, 47, 157, 167] + 39: [282, 39, 262, 261, 321, 304, 143, 149, 257, 22, 44, 209, 310, 302, 230, 120, 107, 339, 57, 201, 52, 9, 198, 361, 247, 138, 317, 106, 306, 245, 64, 342, 269, 380, 165, 89, 66, 248, 378, 281, 320, 4, 288, 175, 147, 356, 29, 334, 104, 81, 340, 358, 33, 275, 70, 20, 370, 308, 90, 21, 372, 208, 127, 101, 366, 45, 6, 335, 211, 351, 154, 11, 236, 163, 352, 190, 371, 95, 16, 287, 83, 67, 73, 115, 56, 374, 87, 54, 144, 167, 114, 216, 197, 219, 332, 148, 315, 333, 23, 48, 353, 134, 128, 60, 117, 359, 184, 186, 258, 174, 121, 146, 47, 177, 285, 183, 283, 10, 238, 344, 286, 119, 26, 7, 232, 126, 14, 357, 265, 159, 229, 316, 65, 267, 71, 51, 140, 367, 345, 354, 207, 291, 92, 284, 312, 242, 324, 365, 191, 152, 301, 97, 30, 346, 206, 125, 38, 279, 277, 369, 274, 69, 135, 88, 164, 19, 157, 237, 93, 325, 204, 76, 170, 74, 188, 185, 221, 41, 108, 124, 132, 360, 296, 178, 259, 34, 276, 193, 254, 111, 337, 318, 36, 13, 362, 49, 383, 137, 199, 133, 79, 103, 266, 12, 314, 218, 270, 3, 42, 239, 322, 289, 195, 303, 305, 294, 313, 139, 297, 77, 8, 225, 63, 61, 309, 168, 55, 105, 379, 215, 244, 40, 375, 31, 75, 373, 327, 343, 253, 233, 85, 113, 86, 141, 235, 278, 179, 182, 130, 196, 246, 256, 43, 151, 80, 172, 298, 336, 5, 293, 169, 91, 252, 136, 100, 156, 292, 250, 290, 260, 62, 203, 220, 192, 160, 158, 176, 153, 382, 330, 227, 323, 162, 72, 363, 27, 347, 205, 99, 25, 272, 319, 118, 122, 213, 212, 68, 58, 142, 349, 96, 1, 155, 24, 53, 299, 249, 102, 194, 171, 0, 210, 280, 368, 271, 341, 84, 112, 364, 311, 181, 166, 123, 326, 35, 348, 328, 300, 273, 28, 161, 228, 240, 109, 18, 129, 180, 223, 187, 15, 202, 251, 226, 355, 214, 376, 37, 255, 268, 82, 2, 98, 200, 331, 150, 241, 338, 189, 78, 224, 264, 234, 329, 17, 59, 131, 50, 173, 243, 222, 46, 32, 110, 116, 145, 295, 381, 263, 307, 94, 377, 217, 231, 350] + 40: [31, 249, 343, 376, 92, 105, 123, 149, 194, 214, 248, 300, 319, 308, 365, 29, 350, 0, 40, 252, 72, 312, 69, 170, 15, 334, 53, 155, 19, 116, 140, 325, 272, 240, 144, 353, 68, 296, 193, 187, 2, 171, 6, 186, 173, 154, 191, 331, 175, 258, 10, 382, 43, 329, 139, 23, 113, 255, 225, 344, 378, 198, 136, 38, 41, 120, 131, 288, 134, 39, 320, 5, 317, 208, 89, 145, 309, 165, 304, 118, 361, 34, 234, 220, 207, 251, 143, 30, 79, 7, 195, 147, 133, 104, 315, 20, 183, 81, 363, 211, 44, 306, 93, 238, 141, 285, 380, 127, 250, 64, 1, 253, 205, 290, 327, 36, 78, 111, 267, 83, 349, 164, 275, 289, 236, 150, 243, 301, 32, 374, 172, 335, 360, 232, 311, 162, 196, 110, 264, 261, 135, 66, 157, 126, 142, 26, 3, 271, 269, 65, 305, 314, 49, 16, 298, 146, 328, 160, 224, 168, 216, 263, 338, 293, 125, 203, 223, 197, 71, 297, 58, 129, 278, 316, 294, 273, 270, 50, 180, 24, 358, 247, 235, 336, 200, 99, 192, 37, 122, 368, 114, 42, 190, 52, 74, 284, 54, 56, 355, 48, 51, 268, 230, 245, 21, 337, 372, 362, 166, 307, 94, 318, 291, 115, 231, 313, 108, 265, 332, 9, 11, 137, 103, 167, 201, 84, 295, 45, 179, 283, 369, 303, 27, 259, 151, 371, 76, 212, 22, 352, 33, 62, 254, 159, 322, 370, 57, 323, 222, 184, 163, 381, 346, 101, 158, 210, 341, 302, 130, 287, 241, 14, 217, 87, 226, 98, 88, 28, 277, 221, 189, 333, 233, 266, 77, 109, 132, 75, 153, 347, 61, 199, 244, 356, 345, 102, 117, 86, 379, 367, 8, 91, 185, 12, 364, 237, 18, 310, 257, 176, 357, 373, 156, 59, 219, 46, 178, 281, 366, 124, 174, 112, 340, 67, 215, 17, 63, 148, 95, 330, 73, 276, 35, 96, 280, 55, 279, 228, 321, 229, 299, 206, 47, 383, 218, 152, 260, 204, 161, 97, 274, 262, 82, 282, 85, 348, 242, 342, 286, 359, 107, 90, 326, 239, 138, 177, 60, 13, 182, 292, 169, 213, 202, 100, 80, 188, 25, 339, 375, 121, 119, 354, 181, 70, 128, 351, 209, 227, 377, 324, 246, 256, 106, 4] + 41: [49, 64, 325, 181, 221, 148, 373, 341, 227, 216, 309, 357, 378, 222, 276, 339, 10, 214, 98, 137, 193, 135, 22, 278, 199, 326, 134, 271, 299, 208, 352, 141, 17, 343, 152, 85, 170, 280, 324, 254, 242, 315, 303, 147, 28, 16, 164, 178, 60, 360, 53, 116, 289, 338, 342, 13, 20, 2, 247, 132, 350, 161, 351, 345, 167, 186, 99, 105, 204, 58, 230, 296, 374, 314, 4, 301, 115, 290, 188, 72, 209, 304, 333, 31, 321, 358, 318, 26, 229, 63, 81, 1, 287, 110, 372, 263, 269, 34, 253, 264, 258, 144, 371, 123, 248, 265, 382, 228, 298, 187, 104, 163, 56, 66, 153, 75, 207, 375, 169, 118, 283, 7, 383, 353, 282, 380, 43, 15, 68, 47, 157, 120, 114, 365, 95, 202, 294, 250, 173, 37, 48, 14, 109, 52, 142, 200, 279, 35, 102, 117, 295, 272, 259, 29, 346, 233, 355, 168, 24, 3, 32, 340, 107, 128, 191, 175, 323, 40, 184, 84, 69, 127, 349, 322, 101, 18, 154, 234, 67, 361, 190, 273, 21, 45, 174, 235, 176, 19, 292, 297, 238, 362, 30, 86, 364, 108, 126, 54, 151, 284, 300, 38, 195, 291, 369, 57, 111, 44, 219, 55, 112, 244, 41, 149, 171, 145, 197, 150, 172, 9, 140, 256, 224, 61, 89, 310, 231, 240, 366, 255, 79, 80, 36, 46, 268, 277, 179, 320, 245, 319, 129, 249, 97, 334, 218, 125, 51, 246, 39, 381, 121, 356, 275, 90, 76, 302, 308, 379, 8, 130, 78, 0, 62, 88, 159, 212, 92, 347, 217, 225, 330, 213, 59, 158, 274, 177, 119, 307, 96, 206, 252, 313, 131, 6, 33, 65, 139, 189, 215, 27, 368, 306, 286, 156, 138, 74, 146, 73, 155, 203, 205, 376, 223, 82, 42, 237, 344, 220, 354, 113, 182, 281, 305, 251, 293, 262, 336, 70, 363, 91, 143, 77, 183, 211, 122, 11, 285, 311, 194, 93, 133, 160, 100, 165, 201, 270, 185, 71, 136, 198, 243, 241, 210, 261, 236, 180, 267, 162, 87, 166, 50, 348, 25, 83, 266, 94, 232, 257, 329, 312, 367, 103, 327, 337, 239, 260, 288, 5, 192, 316, 370, 377, 335, 196, 124, 12, 23, 226, 328, 317, 359, 332, 331, 106] + 42: [123, 193, 242, 294, 369, 376, 25, 71, 95, 254, 324, 339, 152, 313, 213, 285, 115, 13, 365, 24, 252, 289, 46, 27, 117, 232, 233, 175, 134, 331, 86, 361, 273, 352, 126, 121, 31, 259, 356, 160, 180, 7, 343, 292, 353, 15, 163, 40, 359, 209, 4, 103, 166, 371, 58, 346, 82, 136, 159, 66, 217, 255, 320, 161, 257, 311, 298, 364, 90, 304, 182, 74, 94, 208, 277, 287, 314, 192, 97, 378, 360, 186, 14, 91, 266, 263, 358, 230, 322, 137, 64, 53, 98, 330, 78, 240, 268, 41, 189, 178, 147, 362, 372, 205, 26, 229, 22, 190, 345, 179, 281, 341, 338, 227, 231, 310, 120, 261, 96, 276, 176, 150, 327, 221, 329, 39, 237, 258, 112, 199, 383, 44, 172, 264, 342, 29, 326, 76, 307, 34, 293, 267, 302, 12, 155, 282, 203, 30, 216, 149, 62, 299, 194, 272, 133, 35, 218, 72, 59, 354, 167, 38, 297, 226, 156, 197, 144, 54, 93, 63, 48, 248, 236, 316, 132, 246, 270, 321, 174, 47, 325, 215, 55, 379, 201, 335, 340, 139, 336, 104, 124, 382, 111, 151, 220, 235, 173, 36, 241, 283, 68, 102, 207, 249, 256, 279, 374, 265, 344, 234, 18, 260, 16, 363, 3, 177, 158, 350, 168, 21, 87, 92, 60, 315, 125, 290, 153, 11, 333, 116, 243, 373, 286, 223, 381, 183, 171, 56, 69, 70, 138, 212, 162, 122, 239, 366, 146, 323, 170, 284, 85, 57, 37, 247, 222, 127, 67, 280, 301, 9, 204, 73, 296, 0, 2, 211, 337, 319, 165, 181, 51, 89, 61, 185, 309, 100, 188, 305, 200, 348, 105, 308, 8, 219, 269, 318, 88, 110, 210, 184, 349, 191, 28, 143, 380, 157, 251, 145, 198, 271, 154, 303, 262, 275, 17, 77, 32, 244, 291, 250, 288, 43, 375, 106, 108, 131, 135, 33, 80, 49, 332, 368, 334, 142, 20, 357, 306, 370, 169, 45, 101, 238, 196, 312, 367, 79, 1, 119, 377, 274, 140, 118, 113, 130, 355, 10, 23, 317, 50, 228, 6, 141, 224, 52, 245, 109, 5, 128, 75, 81, 42, 114, 129, 328, 214, 164, 65, 99, 278, 206, 347, 83, 295, 148, 195, 351, 187, 202, 19, 225, 300, 84, 253, 107] + 43: [168, 374, 234, 293, 174, 116, 211, 317, 302, 195, 354, 3, 225, 264, 88, 306, 229, 325, 215, 258, 69, 104, 214, 245, 375, 322, 255, 200, 165, 226, 175, 260, 248, 176, 44, 348, 66, 186, 280, 327, 130, 107, 253, 163, 151, 7, 212, 95, 98, 377, 136, 32, 34, 345, 120, 65, 109, 41, 25, 182, 315, 363, 224, 73, 261, 273, 266, 193, 158, 106, 244, 331, 221, 97, 127, 238, 26, 242, 128, 294, 265, 124, 289, 373, 105, 254, 342, 368, 86, 24, 101, 207, 110, 285, 54, 169, 171, 20, 301, 250, 219, 185, 146, 329, 187, 311, 189, 192, 156, 13, 138, 85, 83, 241, 45, 292, 240, 162, 361, 206, 347, 349, 111, 379, 339, 196, 217, 170, 357, 276, 319, 382, 92, 39, 223, 321, 286, 28, 180, 288, 11, 290, 114, 139, 372, 216, 29, 89, 204, 67, 210, 18, 72, 300, 312, 376, 335, 37, 55, 121, 48, 157, 183, 370, 64, 119, 172, 350, 17, 364, 153, 78, 275, 160, 49, 16, 117, 118, 328, 91, 359, 59, 81, 147, 205, 115, 113, 256, 295, 179, 51, 310, 324, 112, 93, 198, 378, 123, 50, 80, 159, 316, 201, 33, 14, 35, 337, 178, 380, 277, 47, 343, 31, 209, 0, 333, 283, 197, 291, 108, 222, 140, 38, 309, 122, 320, 62, 58, 190, 129, 235, 307, 272, 36, 243, 284, 353, 134, 137, 133, 75, 371, 152, 305, 150, 61, 10, 46, 8, 173, 249, 15, 30, 142, 57, 360, 282, 257, 155, 56, 76, 287, 21, 383, 297, 218, 131, 227, 154, 84, 194, 167, 71, 233, 141, 42, 298, 68, 103, 1, 2, 149, 145, 213, 279, 330, 228, 208, 43, 278, 94, 338, 4, 365, 334, 74, 356, 355, 341, 166, 351, 308, 366, 269, 77, 79, 237, 70, 87, 381, 236, 270, 23, 332, 314, 318, 52, 367, 148, 125, 259, 188, 199, 184, 164, 177, 239, 323, 346, 296, 12, 143, 126, 247, 281, 340, 369, 161, 252, 202, 232, 135, 60, 9, 304, 336, 53, 96, 27, 6, 100, 274, 191, 5, 63, 251, 132, 19, 230, 231, 344, 181, 22, 203, 246, 271, 99, 263, 144, 268, 303, 352, 262, 82, 326, 313, 90, 299, 102, 220, 267, 358, 362, 40] + 44: [213, 259, 133, 209, 129, 332, 303, 12, 106, 116, 322, 374, 77, 121, 100, 87, 370, 78, 234, 27, 246, 31, 281, 206, 301, 358, 138, 315, 44, 282, 218, 62, 373, 127, 75, 339, 191, 306, 265, 287, 221, 185, 113, 321, 115, 73, 310, 13, 199, 63, 207, 364, 177, 334, 179, 256, 195, 361, 181, 189, 10, 264, 276, 192, 269, 107, 309, 357, 245, 176, 52, 308, 317, 187, 144, 349, 130, 196, 105, 18, 273, 260, 111, 161, 226, 98, 228, 173, 164, 15, 149, 345, 312, 268, 316, 295, 348, 313, 304, 219, 366, 126, 243, 53, 124, 247, 353, 112, 362, 146, 217, 254, 160, 162, 336, 314, 35, 194, 58, 371, 274, 93, 125, 296, 380, 46, 19, 263, 153, 283, 76, 16, 172, 34, 48, 266, 71, 134, 40, 169, 232, 50, 293, 197, 171, 288, 33, 244, 70, 381, 159, 211, 7, 251, 37, 208, 222, 29, 320, 275, 186, 338, 344, 59, 241, 375, 330, 88, 379, 82, 143, 102, 184, 352, 92, 205, 367, 255, 210, 148, 101, 376, 166, 365, 6, 285, 36, 74, 20, 137, 119, 297, 223, 257, 30, 198, 262, 39, 261, 90, 305, 183, 151, 356, 284, 84, 236, 329, 91, 56, 369, 165, 3, 49, 5, 152, 21, 267, 11, 142, 117, 201, 120, 83, 104, 51, 22, 335, 145, 215, 325, 38, 340, 150, 96, 41, 231, 57, 80, 302, 170, 188, 229, 249, 95, 290, 248, 354, 347, 72, 200, 307, 346, 227, 311, 118, 360, 163, 131, 108, 214, 4, 65, 99, 168, 180, 331, 182, 89, 85, 54, 327, 81, 203, 216, 47, 238, 190, 135, 383, 355, 24, 286, 26, 270, 204, 114, 240, 97, 60, 279, 212, 343, 55, 66, 136, 318, 230, 1, 298, 64, 271, 252, 294, 291, 122, 132, 278, 8, 250, 326, 363, 167, 155, 43, 289, 94, 350, 174, 128, 156, 193, 253, 123, 0, 351, 368, 86, 337, 154, 79, 147, 323, 319, 45, 372, 17, 110, 277, 333, 324, 61, 378, 220, 377, 9, 2, 28, 258, 382, 328, 242, 225, 299, 67, 157, 341, 109, 42, 69, 25, 300, 140, 280, 14, 224, 359, 292, 23, 237, 239, 139, 272, 178, 68, 141, 158, 175, 202, 342, 32, 235, 233, 103] + 45: [92, 189, 195, 117, 192, 64, 266, 300, 100, 174, 221, 347, 140, 16, 357, 365, 34, 273, 220, 337, 295, 240, 336, 138, 178, 380, 51, 41, 123, 87, 263, 268, 125, 13, 181, 83, 10, 272, 158, 80, 2, 116, 333, 15, 321, 305, 69, 355, 227, 124, 310, 112, 175, 344, 294, 381, 243, 56, 66, 185, 154, 212, 307, 313, 279, 330, 9, 151, 238, 62, 130, 72, 23, 165, 168, 328, 106, 35, 264, 374, 362, 270, 259, 299, 121, 298, 118, 250, 260, 43, 358, 79, 371, 193, 366, 361, 161, 160, 183, 155, 213, 4, 205, 255, 312, 132, 53, 244, 293, 369, 107, 229, 360, 302, 46, 231, 309, 102, 148, 137, 42, 375, 52, 36, 345, 1, 230, 241, 203, 90, 101, 17, 334, 247, 209, 115, 135, 96, 170, 379, 48, 323, 352, 14, 201, 339, 224, 326, 267, 24, 126, 211, 289, 332, 265, 40, 8, 356, 128, 45, 210, 236, 207, 322, 119, 111, 194, 235, 27, 141, 349, 122, 7, 208, 133, 282, 341, 214, 200, 49, 33, 176, 297, 196, 251, 67, 76, 37, 342, 377, 186, 75, 350, 171, 114, 162, 306, 109, 20, 5, 129, 71, 31, 184, 367, 120, 348, 55, 327, 315, 19, 283, 177, 3, 82, 99, 373, 274, 163, 223, 68, 26, 65, 98, 57, 136, 153, 149, 276, 197, 61, 239, 84, 32, 58, 296, 142, 78, 97, 311, 329, 182, 21, 187, 54, 292, 340, 29, 169, 225, 280, 60, 164, 346, 269, 198, 308, 94, 50, 150, 127, 93, 382, 325, 95, 199, 131, 359, 12, 179, 218, 320, 271, 202, 152, 172, 81, 139, 180, 254, 70, 217, 370, 38, 288, 113, 143, 6, 383, 25, 47, 228, 372, 44, 303, 291, 166, 147, 167, 77, 324, 284, 103, 331, 245, 156, 261, 59, 285, 73, 18, 237, 191, 317, 354, 275, 74, 108, 376, 319, 246, 368, 226, 287, 173, 110, 286, 351, 277, 242, 222, 145, 204, 190, 232, 335, 301, 316, 85, 39, 104, 86, 22, 290, 363, 91, 89, 11, 318, 144, 378, 258, 88, 281, 253, 256, 105, 364, 353, 28, 219, 134, 262, 234, 30, 249, 257, 252, 314, 215, 159, 216, 233, 188, 146, 248, 338, 157, 63, 304, 0, 343, 206, 278] + 46: [142, 95, 321, 290, 159, 15, 358, 14, 270, 252, 88, 356, 183, 155, 50, 292, 318, 7, 337, 330, 28, 323, 325, 302, 352, 115, 156, 268, 92, 364, 305, 87, 134, 61, 122, 175, 311, 55, 211, 353, 294, 32, 255, 265, 26, 366, 167, 365, 133, 254, 85, 232, 304, 336, 78, 241, 93, 196, 178, 82, 10, 60, 39, 326, 109, 57, 206, 291, 339, 288, 174, 213, 247, 124, 242, 83, 4, 377, 263, 264, 375, 229, 179, 27, 34, 209, 219, 143, 164, 205, 223, 277, 347, 11, 248, 36, 64, 6, 303, 8, 42, 59, 144, 333, 309, 269, 225, 147, 139, 324, 18, 62, 118, 272, 161, 243, 52, 21, 185, 379, 256, 332, 345, 108, 197, 273, 173, 56, 135, 94, 276, 5, 58, 221, 113, 131, 315, 172, 313, 322, 150, 75, 47, 220, 286, 274, 338, 280, 168, 128, 307, 382, 331, 381, 54, 231, 380, 246, 114, 228, 121, 53, 81, 346, 283, 98, 281, 123, 253, 239, 165, 49, 132, 162, 193, 289, 359, 361, 1, 182, 355, 191, 378, 351, 245, 23, 158, 328, 91, 285, 107, 349, 343, 238, 204, 30, 218, 41, 45, 35, 261, 171, 341, 0, 110, 202, 38, 282, 334, 362, 111, 373, 190, 234, 31, 181, 207, 51, 68, 215, 99, 298, 103, 299, 20, 86, 73, 251, 149, 96, 40, 3, 130, 63, 198, 89, 317, 138, 77, 84, 230, 16, 187, 102, 195, 176, 237, 297, 69, 67, 127, 100, 367, 25, 136, 350, 33, 129, 216, 119, 293, 208, 258, 151, 354, 170, 271, 266, 22, 316, 340, 169, 74, 257, 72, 306, 106, 249, 194, 374, 260, 192, 383, 141, 90, 184, 105, 212, 319, 46, 104, 320, 80, 259, 17, 24, 348, 65, 43, 70, 66, 240, 344, 76, 186, 295, 101, 308, 177, 262, 363, 279, 140, 327, 12, 2, 372, 369, 125, 97, 145, 370, 48, 148, 335, 224, 154, 44, 287, 180, 157, 360, 342, 244, 126, 296, 152, 368, 284, 79, 117, 13, 250, 189, 71, 112, 275, 200, 188, 314, 312, 199, 227, 137, 226, 329, 217, 278, 236, 29, 210, 19, 301, 203, 376, 37, 235, 160, 9, 163, 310, 357, 201, 233, 214, 146, 120, 166, 267, 222, 153, 371, 300, 116] + 47: [195, 146, 191, 373, 209, 369, 246, 293, 360, 56, 288, 326, 3, 252, 303, 32, 275, 247, 165, 261, 113, 7, 236, 62, 343, 50, 112, 335, 157, 282, 244, 232, 8, 128, 156, 239, 54, 255, 354, 211, 200, 190, 374, 361, 206, 272, 381, 214, 44, 98, 140, 25, 183, 37, 248, 380, 372, 217, 66, 169, 352, 176, 298, 148, 139, 185, 186, 216, 264, 6, 241, 311, 24, 114, 42, 73, 96, 60, 297, 92, 38, 256, 254, 318, 312, 159, 68, 197, 315, 292, 9, 266, 268, 167, 13, 83, 87, 238, 122, 65, 237, 378, 111, 97, 302, 99, 283, 182, 181, 129, 235, 135, 370, 45, 234, 276, 102, 304, 85, 125, 154, 175, 363, 219, 324, 101, 336, 337, 160, 325, 78, 161, 162, 127, 240, 265, 319, 205, 80, 147, 365, 86, 100, 295, 22, 286, 188, 351, 33, 17, 91, 213, 120, 339, 178, 221, 158, 314, 168, 251, 67, 364, 106, 126, 329, 10, 177, 299, 29, 309, 267, 63, 377, 199, 233, 23, 225, 19, 108, 220, 382, 345, 362, 342, 104, 166, 340, 366, 259, 48, 379, 142, 307, 227, 79, 349, 301, 41, 203, 121, 258, 130, 308, 116, 359, 333, 196, 131, 250, 231, 138, 201, 273, 152, 383, 11, 300, 193, 274, 218, 262, 103, 204, 71, 94, 253, 245, 88, 134, 280, 344, 367, 93, 89, 52, 164, 31, 132, 358, 270, 137, 61, 198, 153, 69, 224, 173, 210, 49, 20, 74, 202, 75, 34, 36, 229, 26, 151, 180, 27, 278, 334, 207, 47, 271, 1, 368, 110, 226, 172, 355, 15, 289, 189, 290, 76, 347, 109, 294, 287, 269, 228, 322, 321, 28, 149, 58, 105, 331, 170, 215, 118, 313, 323, 338, 356, 107, 260, 350, 145, 187, 279, 257, 376, 330, 284, 77, 348, 115, 310, 5, 222, 179, 119, 353, 40, 212, 184, 39, 53, 332, 81, 55, 82, 136, 375, 316, 21, 230, 263, 296, 243, 281, 150, 133, 317, 192, 35, 320, 327, 155, 285, 51, 18, 306, 144, 57, 64, 143, 371, 305, 346, 208, 223, 12, 70, 163, 242, 16, 30, 171, 43, 14, 123, 95, 249, 0, 141, 328, 46, 174, 277, 357, 291, 90, 59, 84, 124, 4, 117, 341, 2, 72, 194] + 48: [17, 41, 257, 24, 351, 16, 47, 278, 153, 258, 209, 250, 271, 354, 291, 111, 14, 40, 173, 335, 260, 336, 304, 308, 140, 139, 210, 329, 275, 367, 195, 29, 238, 333, 10, 196, 328, 5, 378, 321, 91, 211, 259, 373, 309, 355, 200, 227, 132, 6, 134, 192, 341, 180, 96, 178, 67, 225, 306, 114, 267, 287, 181, 179, 285, 70, 146, 216, 69, 148, 126, 129, 283, 203, 185, 371, 162, 218, 269, 169, 87, 149, 383, 334, 76, 74, 208, 175, 112, 202, 302, 39, 194, 60, 365, 314, 266, 121, 327, 293, 123, 231, 84, 292, 277, 154, 176, 245, 128, 343, 344, 319, 118, 79, 43, 206, 105, 244, 131, 1, 102, 236, 223, 155, 255, 360, 374, 346, 213, 350, 199, 30, 330, 44, 177, 232, 370, 168, 138, 151, 135, 110, 296, 136, 273, 234, 189, 305, 152, 357, 281, 127, 32, 37, 171, 220, 239, 19, 147, 380, 299, 221, 307, 253, 35, 143, 8, 23, 276, 99, 375, 252, 50, 212, 339, 63, 55, 294, 356, 348, 52, 226, 369, 184, 229, 89, 347, 12, 59, 163, 376, 167, 311, 116, 166, 115, 290, 31, 13, 81, 98, 61, 66, 103, 284, 215, 207, 316, 320, 170, 57, 36, 382, 3, 201, 337, 353, 358, 104, 186, 222, 340, 315, 18, 108, 93, 298, 54, 101, 109, 157, 325, 205, 160, 230, 352, 243, 326, 338, 68, 141, 150, 28, 34, 156, 73, 318, 349, 94, 165, 362, 122, 381, 301, 83, 240, 246, 120, 100, 49, 187, 310, 282, 183, 85, 279, 21, 235, 25, 233, 241, 204, 261, 313, 228, 265, 78, 145, 268, 359, 7, 247, 113, 72, 272, 92, 86, 248, 62, 117, 33, 48, 322, 174, 80, 119, 71, 217, 106, 46, 15, 65, 295, 363, 300, 0, 4, 53, 82, 214, 331, 251, 263, 77, 58, 264, 130, 324, 172, 280, 107, 27, 188, 364, 237, 270, 90, 342, 368, 242, 317, 97, 274, 224, 361, 197, 159, 26, 2, 372, 193, 190, 64, 158, 256, 366, 125, 124, 297, 249, 133, 38, 161, 75, 22, 289, 286, 95, 345, 332, 288, 20, 182, 254, 219, 88, 379, 11, 144, 323, 377, 262, 9, 51, 164, 142, 312, 45, 42, 198, 137, 56, 191, 303] + 49: [302, 135, 300, 322, 375, 38, 317, 337, 356, 64, 123, 191, 321, 227, 130, 22, 7, 266, 3, 347, 68, 188, 349, 92, 297, 344, 144, 362, 163, 295, 69, 2, 185, 236, 32, 370, 253, 60, 30, 336, 318, 139, 103, 106, 125, 363, 258, 373, 126, 325, 8, 254, 198, 55, 359, 46, 364, 372, 252, 24, 374, 262, 186, 250, 240, 20, 338, 61, 52, 167, 334, 26, 79, 44, 381, 366, 138, 376, 309, 0, 108, 87, 18, 129, 360, 39, 194, 328, 16, 224, 43, 339, 228, 343, 99, 288, 272, 275, 74, 195, 382, 296, 330, 14, 101, 104, 379, 369, 6, 201, 107, 5, 310, 41, 371, 71, 204, 324, 171, 247, 268, 270, 306, 131, 368, 323, 36, 244, 154, 51, 367, 50, 75, 212, 289, 94, 245, 34, 82, 358, 49, 182, 215, 263, 155, 105, 243, 333, 23, 128, 98, 345, 256, 147, 237, 219, 305, 78, 292, 100, 137, 4, 10, 280, 190, 168, 350, 203, 19, 66, 218, 264, 174, 170, 303, 119, 200, 232, 152, 291, 157, 353, 380, 175, 1, 365, 216, 142, 31, 326, 281, 383, 313, 116, 177, 45, 169, 277, 225, 351, 12, 354, 165, 11, 202, 213, 199, 255, 73, 196, 301, 156, 355, 145, 341, 230, 271, 293, 259, 331, 97, 140, 273, 307, 118, 299, 229, 217, 9, 115, 148, 329, 121, 28, 112, 173, 117, 151, 166, 231, 242, 181, 276, 269, 314, 86, 214, 134, 239, 113, 93, 159, 193, 158, 335, 286, 205, 279, 127, 179, 378, 206, 88, 265, 251, 150, 180, 91, 235, 80, 89, 327, 226, 124, 348, 249, 210, 136, 153, 260, 48, 340, 77, 238, 176, 283, 27, 287, 233, 234, 110, 132, 183, 311, 81, 308, 53, 192, 57, 162, 184, 59, 261, 220, 332, 70, 146, 178, 114, 361, 13, 42, 197, 65, 278, 120, 298, 56, 189, 33, 47, 285, 377, 102, 149, 209, 241, 312, 109, 221, 290, 62, 172, 143, 315, 96, 37, 257, 25, 187, 164, 54, 29, 67, 72, 282, 274, 141, 83, 304, 40, 246, 95, 223, 160, 267, 320, 284, 357, 207, 294, 15, 208, 133, 58, 63, 222, 352, 85, 84, 111, 21, 319, 346, 90, 76, 122, 17, 248, 35, 211, 342, 316, 161] + 50: [86, 116, 151, 28, 153, 330, 30, 95, 55, 230, 345, 365, 268, 154, 319, 294, 39, 123, 147, 83, 66, 222, 50, 93, 29, 0, 121, 244, 208, 378, 113, 34, 98, 173, 131, 241, 100, 312, 79, 195, 379, 357, 236, 212, 258, 31, 279, 374, 316, 146, 314, 375, 252, 105, 5, 308, 285, 207, 307, 257, 109, 373, 22, 117, 102, 255, 281, 344, 324, 176, 336, 264, 49, 125, 119, 6, 183, 85, 172, 288, 318, 104, 54, 111, 181, 110, 114, 363, 348, 382, 211, 289, 383, 60, 158, 136, 166, 273, 88, 133, 194, 371, 340, 4, 68, 89, 261, 364, 376, 35, 256, 132, 24, 250, 73, 185, 198, 323, 8, 120, 32, 218, 193, 298, 164, 94, 234, 178, 320, 112, 201, 44, 315, 262, 341, 353, 41, 269, 163, 243, 372, 179, 246, 84, 210, 361, 45, 16, 122, 310, 303, 223, 349, 286, 47, 144, 188, 370, 334, 271, 228, 189, 78, 127, 170, 346, 21, 161, 215, 233, 296, 187, 301, 267, 74, 356, 237, 150, 266, 182, 69, 352, 62, 155, 165, 359, 126, 191, 259, 40, 209, 283, 232, 284, 280, 184, 58, 219, 2, 347, 12, 293, 216, 309, 325, 367, 304, 343, 75, 168, 20, 202, 177, 77, 381, 332, 321, 38, 282, 253, 19, 134, 160, 292, 42, 291, 103, 167, 214, 14, 175, 51, 186, 43, 91, 369, 156, 36, 360, 23, 56, 333, 366, 331, 263, 206, 242, 337, 124, 82, 76, 221, 27, 97, 287, 295, 203, 290, 322, 204, 339, 11, 10, 225, 350, 229, 9, 224, 306, 169, 277, 17, 7, 171, 70, 227, 26, 67, 377, 130, 313, 139, 326, 251, 63, 138, 190, 260, 18, 140, 338, 272, 149, 53, 145, 362, 200, 162, 328, 302, 1, 137, 329, 270, 239, 247, 205, 297, 106, 87, 13, 317, 3, 311, 192, 213, 71, 342, 152, 37, 46, 354, 245, 52, 129, 299, 327, 249, 96, 197, 64, 300, 238, 90, 72, 115, 231, 226, 199, 335, 380, 174, 107, 143, 248, 148, 265, 118, 128, 235, 217, 159, 275, 180, 81, 278, 305, 65, 157, 254, 57, 25, 220, 15, 48, 240, 99, 135, 141, 355, 142, 368, 33, 80, 358, 274, 196, 276, 92, 108, 351, 101, 61, 59] + 51: [24, 219, 141, 157, 194, 336, 257, 266, 173, 202, 249, 261, 186, 81, 321, 100, 297, 15, 57, 150, 76, 121, 291, 326, 372, 314, 245, 209, 362, 258, 298, 68, 28, 22, 377, 151, 102, 95, 335, 271, 351, 361, 115, 319, 116, 132, 38, 21, 137, 60, 130, 183, 18, 119, 324, 30, 159, 154, 293, 348, 373, 104, 103, 285, 279, 79, 364, 80, 197, 174, 71, 290, 375, 270, 269, 292, 42, 59, 265, 288, 58, 201, 169, 156, 221, 152, 243, 85, 198, 248, 196, 315, 112, 272, 352, 226, 241, 67, 295, 299, 56, 34, 69, 54, 232, 14, 175, 2, 37, 64, 330, 259, 172, 10, 224, 307, 61, 277, 179, 138, 210, 176, 325, 8, 97, 33, 262, 223, 238, 88, 187, 139, 110, 122, 250, 247, 303, 192, 164, 27, 167, 7, 63, 155, 200, 356, 188, 5, 308, 310, 239, 338, 316, 158, 185, 327, 323, 127, 301, 92, 382, 280, 48, 365, 273, 1, 367, 255, 208, 180, 235, 26, 344, 190, 41, 334, 93, 148, 376, 267, 12, 228, 124, 263, 275, 181, 236, 349, 333, 214, 217, 231, 287, 182, 260, 134, 161, 222, 3, 312, 363, 66, 128, 36, 17, 52, 40, 11, 47, 366, 195, 332, 278, 43, 39, 318, 342, 284, 203, 49, 212, 329, 75, 94, 78, 378, 165, 302, 337, 193, 32, 346, 331, 360, 370, 289, 328, 53, 215, 29, 144, 113, 87, 253, 281, 142, 368, 311, 114, 339, 305, 274, 82, 123, 135, 246, 23, 254, 44, 234, 145, 340, 0, 70, 126, 357, 140, 168, 20, 359, 91, 171, 170, 240, 296, 117, 229, 99, 343, 282, 143, 237, 160, 317, 350, 25, 96, 131, 163, 16, 65, 153, 216, 107, 204, 84, 379, 149, 230, 256, 286, 111, 300, 304, 294, 162, 207, 313, 89, 125, 4, 225, 322, 98, 72, 146, 13, 345, 50, 306, 276, 353, 77, 355, 108, 206, 136, 177, 118, 341, 252, 369, 45, 184, 354, 244, 31, 178, 211, 129, 35, 380, 90, 320, 242, 55, 120, 133, 106, 6, 358, 283, 374, 86, 251, 383, 51, 218, 9, 147, 189, 101, 264, 105, 220, 74, 166, 205, 191, 109, 199, 227, 73, 371, 62, 233, 46, 268, 309, 213, 381, 19, 83, 347] + 52: [381, 45, 383, 310, 18, 203, 86, 264, 46, 38, 176, 180, 99, 39, 58, 51, 226, 36, 14, 272, 305, 247, 348, 375, 334, 223, 154, 308, 152, 278, 302, 338, 295, 360, 16, 175, 102, 127, 23, 319, 93, 194, 342, 147, 0, 376, 162, 212, 107, 64, 222, 248, 374, 242, 143, 126, 273, 240, 218, 370, 157, 60, 171, 221, 361, 213, 294, 303, 299, 11, 317, 106, 71, 33, 98, 359, 113, 324, 365, 356, 335, 12, 373, 204, 104, 62, 54, 74, 367, 22, 270, 139, 322, 158, 1, 94, 165, 346, 349, 256, 233, 115, 235, 189, 118, 363, 24, 164, 97, 172, 252, 285, 364, 351, 350, 207, 344, 132, 144, 199, 283, 90, 357, 312, 191, 300, 208, 148, 368, 232, 145, 254, 103, 174, 68, 95, 378, 29, 279, 352, 220, 111, 276, 241, 137, 321, 190, 32, 339, 160, 377, 225, 251, 197, 275, 142, 336, 271, 288, 87, 379, 167, 53, 268, 84, 141, 274, 366, 239, 195, 41, 266, 133, 30, 17, 161, 21, 380, 112, 122, 209, 290, 382, 355, 291, 281, 146, 27, 201, 198, 353, 159, 48, 323, 183, 210, 236, 124, 280, 109, 89, 100, 358, 202, 25, 289, 13, 128, 123, 5, 301, 287, 224, 136, 181, 261, 205, 214, 263, 196, 108, 286, 72, 69, 59, 230, 229, 329, 52, 85, 169, 56, 35, 105, 243, 40, 237, 28, 43, 19, 186, 73, 345, 269, 354, 315, 6, 91, 200, 101, 81, 65, 327, 362, 34, 219, 50, 178, 153, 66, 277, 156, 114, 265, 121, 250, 163, 44, 70, 341, 328, 333, 244, 150, 4, 185, 92, 246, 193, 259, 257, 227, 75, 119, 110, 249, 293, 187, 307, 151, 206, 168, 8, 173, 67, 125, 177, 347, 304, 255, 120, 63, 7, 10, 37, 83, 116, 326, 57, 129, 96, 314, 231, 130, 184, 245, 325, 166, 369, 253, 258, 188, 311, 42, 332, 182, 78, 260, 88, 117, 76, 170, 282, 179, 313, 79, 61, 31, 298, 140, 155, 234, 331, 316, 20, 318, 135, 297, 216, 267, 82, 80, 228, 262, 3, 55, 284, 49, 340, 15, 372, 371, 77, 26, 309, 296, 292, 337, 238, 47, 343, 131, 217, 320, 211, 192, 149, 215, 134, 306, 2, 330, 138, 9] + 53: [110, 18, 140, 153, 167, 312, 316, 185, 84, 114, 210, 272, 240, 231, 349, 196, 79, 337, 1, 190, 58, 377, 56, 90, 73, 319, 383, 89, 120, 117, 275, 310, 187, 191, 314, 252, 38, 103, 25, 176, 44, 19, 123, 289, 301, 192, 223, 265, 16, 162, 241, 77, 159, 105, 251, 331, 51, 212, 244, 22, 317, 302, 121, 168, 52, 102, 321, 101, 108, 163, 27, 344, 13, 92, 31, 219, 10, 76, 236, 133, 380, 17, 267, 307, 350, 42, 332, 325, 172, 67, 158, 220, 264, 147, 61, 71, 23, 227, 261, 5, 151, 146, 327, 232, 283, 193, 181, 99, 334, 155, 237, 127, 205, 126, 339, 255, 137, 366, 343, 245, 287, 179, 376, 95, 188, 72, 115, 333, 367, 34, 260, 290, 296, 379, 291, 171, 116, 15, 216, 136, 257, 39, 98, 318, 109, 224, 194, 253, 183, 281, 111, 28, 87, 82, 104, 11, 354, 345, 157, 24, 112, 47, 362, 371, 0, 234, 374, 263, 311, 363, 86, 20, 341, 149, 175, 370, 294, 305, 348, 124, 347, 214, 351, 326, 338, 150, 122, 278, 138, 9, 298, 304, 295, 273, 268, 269, 361, 259, 353, 30, 358, 249, 342, 355, 250, 365, 309, 284, 94, 49, 233, 166, 59, 14, 64, 356, 330, 46, 202, 107, 62, 37, 248, 2, 195, 154, 303, 165, 32, 322, 360, 130, 70, 300, 340, 375, 242, 80, 106, 161, 4, 292, 156, 144, 141, 378, 369, 169, 258, 7, 113, 148, 186, 280, 180, 271, 381, 78, 328, 57, 254, 246, 184, 211, 230, 142, 160, 119, 139, 239, 36, 178, 45, 21, 372, 213, 336, 173, 228, 50, 3, 197, 282, 96, 198, 29, 83, 286, 308, 164, 145, 225, 69, 335, 128, 206, 54, 201, 93, 135, 217, 320, 279, 221, 12, 152, 35, 174, 299, 100, 177, 33, 75, 129, 132, 26, 262, 235, 293, 207, 364, 199, 48, 288, 8, 238, 118, 368, 200, 277, 359, 306, 182, 43, 63, 204, 68, 247, 266, 134, 256, 313, 357, 189, 55, 53, 229, 41, 297, 270, 170, 125, 88, 373, 143, 276, 97, 352, 208, 382, 243, 66, 6, 285, 329, 324, 226, 65, 218, 323, 85, 274, 203, 222, 209, 81, 346, 60, 315, 215, 40, 131, 74, 91] + 54: [363, 13, 160, 205, 0, 36, 148, 232, 320, 336, 361, 378, 370, 383, 19, 200, 202, 358, 130, 212, 106, 123, 127, 354, 294, 125, 97, 326, 195, 229, 269, 292, 252, 193, 340, 344, 339, 156, 335, 239, 355, 293, 347, 243, 52, 270, 207, 78, 155, 10, 198, 94, 49, 298, 275, 74, 178, 77, 2, 342, 164, 352, 99, 225, 246, 61, 191, 145, 247, 138, 112, 349, 284, 57, 375, 50, 103, 105, 107, 33, 251, 89, 374, 297, 283, 290, 317, 113, 152, 365, 327, 18, 351, 46, 102, 188, 42, 174, 75, 197, 24, 184, 201, 62, 271, 108, 149, 222, 258, 53, 314, 196, 34, 55, 68, 244, 334, 343, 285, 241, 11, 8, 40, 76, 272, 368, 85, 151, 141, 362, 56, 256, 356, 172, 25, 41, 17, 1, 373, 9, 259, 206, 303, 37, 223, 20, 119, 143, 91, 177, 73, 175, 308, 261, 307, 126, 231, 98, 311, 132, 124, 381, 122, 167, 371, 359, 360, 165, 240, 250, 87, 332, 357, 382, 21, 136, 353, 12, 233, 51, 348, 30, 330, 27, 64, 54, 140, 242, 146, 15, 367, 291, 38, 133, 345, 101, 322, 182, 186, 137, 114, 238, 60, 203, 280, 214, 221, 58, 277, 142, 204, 304, 364, 110, 29, 65, 230, 369, 305, 181, 158, 92, 237, 22, 81, 63, 331, 104, 299, 80, 187, 121, 84, 217, 312, 111, 173, 139, 263, 199, 286, 260, 93, 26, 333, 321, 135, 366, 23, 219, 236, 210, 95, 82, 282, 180, 288, 323, 278, 144, 266, 224, 249, 4, 295, 150, 281, 16, 47, 45, 183, 48, 273, 234, 274, 28, 337, 257, 306, 372, 129, 159, 376, 209, 162, 194, 66, 226, 380, 216, 72, 329, 115, 220, 318, 69, 300, 346, 289, 350, 153, 267, 120, 309, 310, 325, 67, 171, 128, 6, 14, 379, 296, 287, 324, 86, 161, 109, 157, 313, 377, 245, 248, 227, 253, 39, 44, 117, 70, 265, 315, 163, 166, 192, 79, 35, 71, 90, 176, 3, 215, 279, 170, 179, 96, 88, 268, 116, 319, 83, 341, 228, 32, 185, 131, 301, 316, 235, 211, 134, 328, 147, 208, 118, 218, 190, 302, 255, 262, 213, 31, 154, 189, 338, 254, 7, 276, 5, 100, 264, 43, 59, 168, 169] + 55: [154, 254, 61, 201, 50, 369, 283, 186, 317, 215, 69, 305, 174, 27, 54, 63, 365, 261, 162, 45, 222, 245, 204, 34, 56, 301, 44, 5, 95, 286, 117, 367, 139, 324, 250, 96, 202, 227, 333, 233, 289, 322, 278, 97, 118, 75, 83, 285, 88, 307, 249, 312, 231, 257, 84, 219, 137, 181, 57, 109, 81, 111, 2, 366, 276, 79, 358, 160, 321, 248, 266, 196, 98, 238, 341, 216, 325, 240, 23, 212, 126, 338, 282, 32, 20, 16, 200, 300, 14, 360, 255, 1, 319, 129, 298, 351, 11, 252, 291, 221, 350, 42, 128, 237, 35, 38, 36, 100, 146, 311, 106, 213, 348, 47, 277, 264, 37, 66, 183, 205, 308, 39, 40, 164, 102, 343, 346, 31, 258, 145, 242, 9, 340, 165, 49, 368, 170, 361, 287, 153, 103, 275, 169, 167, 116, 339, 132, 138, 46, 246, 318, 82, 232, 354, 262, 158, 80, 89, 199, 290, 163, 378, 228, 22, 225, 147, 155, 108, 381, 182, 65, 349, 112, 105, 53, 294, 335, 58, 364, 229, 123, 357, 70, 272, 316, 265, 122, 85, 253, 209, 244, 230, 299, 370, 363, 12, 124, 217, 194, 296, 143, 107, 280, 25, 90, 218, 304, 60, 0, 380, 10, 171, 93, 336, 110, 345, 48, 62, 382, 71, 271, 236, 19, 157, 372, 315, 92, 24, 347, 7, 273, 99, 239, 371, 119, 207, 30, 297, 362, 59, 74, 234, 353, 114, 26, 224, 263, 52, 356, 168, 21, 142, 323, 4, 121, 293, 189, 91, 306, 198, 342, 120, 191, 131, 180, 309, 247, 134, 337, 375, 136, 87, 281, 177, 13, 310, 130, 151, 78, 223, 86, 377, 184, 193, 268, 55, 330, 195, 29, 355, 303, 8, 197, 284, 135, 314, 359, 243, 334, 288, 269, 235, 73, 3, 172, 190, 67, 302, 173, 125, 77, 292, 115, 256, 328, 179, 259, 274, 178, 214, 127, 374, 376, 379, 76, 270, 192, 331, 17, 352, 187, 313, 208, 150, 141, 148, 326, 43, 6, 320, 383, 251, 140, 94, 373, 161, 295, 18, 113, 327, 267, 279, 156, 210, 185, 101, 15, 104, 175, 220, 72, 28, 41, 176, 206, 329, 241, 33, 133, 64, 166, 144, 159, 152, 211, 226, 344, 260, 332, 149, 188, 68, 203, 51] + 56: [231, 142, 85, 224, 315, 146, 173, 202, 3, 90, 143, 208, 240, 65, 382, 217, 171, 318, 102, 229, 76, 109, 301, 74, 195, 324, 149, 286, 54, 375, 352, 203, 277, 237, 51, 61, 260, 197, 105, 280, 141, 201, 226, 93, 26, 366, 91, 235, 220, 77, 205, 135, 214, 368, 119, 37, 113, 108, 377, 165, 294, 327, 309, 182, 305, 248, 360, 232, 287, 234, 45, 247, 321, 306, 292, 101, 342, 198, 185, 62, 4, 43, 215, 194, 242, 104, 275, 168, 160, 206, 297, 96, 172, 16, 261, 365, 265, 363, 316, 223, 233, 12, 11, 145, 228, 29, 326, 380, 44, 263, 46, 344, 58, 268, 138, 314, 310, 67, 335, 25, 178, 9, 346, 246, 103, 269, 153, 262, 353, 78, 336, 111, 82, 156, 48, 238, 359, 22, 34, 190, 274, 24, 47, 216, 213, 371, 161, 154, 312, 136, 56, 271, 112, 350, 68, 114, 15, 322, 328, 80, 0, 92, 50, 42, 218, 289, 212, 174, 225, 351, 162, 188, 150, 285, 295, 209, 299, 200, 361, 358, 241, 267, 117, 19, 14, 278, 317, 151, 94, 28, 130, 325, 106, 303, 6, 71, 72, 73, 355, 133, 210, 124, 222, 256, 340, 158, 125, 302, 63, 137, 338, 227, 55, 290, 266, 7, 259, 184, 308, 140, 97, 27, 354, 127, 183, 293, 49, 349, 30, 296, 251, 39, 207, 40, 79, 270, 257, 307, 64, 370, 5, 98, 187, 99, 118, 69, 35, 164, 298, 323, 144, 52, 75, 163, 333, 41, 383, 376, 374, 243, 134, 347, 170, 59, 304, 88, 264, 284, 120, 334, 331, 1, 84, 341, 367, 32, 147, 155, 89, 66, 159, 128, 179, 33, 221, 276, 152, 131, 279, 53, 20, 83, 357, 193, 166, 13, 378, 57, 373, 272, 100, 186, 189, 329, 192, 38, 252, 282, 313, 139, 70, 239, 177, 116, 356, 332, 381, 253, 230, 17, 115, 319, 204, 157, 8, 107, 169, 18, 283, 281, 21, 364, 180, 132, 372, 36, 311, 273, 87, 2, 245, 250, 23, 255, 339, 291, 86, 176, 343, 81, 196, 362, 288, 10, 31, 123, 95, 330, 211, 258, 167, 199, 236, 219, 244, 148, 181, 129, 122, 379, 254, 337, 126, 191, 249, 320, 369, 300, 121, 110, 348, 175, 60, 345] + 57: [254, 56, 86, 234, 262, 275, 376, 293, 372, 301, 304, 343, 276, 167, 249, 12, 337, 175, 299, 375, 339, 280, 19, 285, 326, 127, 238, 214, 208, 351, 323, 307, 194, 218, 15, 233, 25, 174, 272, 228, 92, 71, 132, 122, 260, 239, 193, 143, 231, 6, 80, 243, 111, 210, 17, 303, 36, 67, 52, 367, 14, 124, 35, 176, 277, 55, 216, 363, 219, 78, 269, 202, 146, 359, 317, 99, 70, 47, 196, 379, 200, 361, 212, 34, 259, 108, 197, 91, 27, 203, 353, 18, 255, 222, 258, 201, 295, 223, 283, 58, 177, 120, 352, 41, 265, 48, 244, 229, 165, 257, 318, 338, 152, 38, 149, 334, 270, 242, 291, 105, 89, 128, 151, 347, 51, 289, 3, 29, 24, 256, 114, 69, 246, 101, 84, 144, 42, 284, 171, 162, 59, 374, 115, 77, 187, 65, 117, 319, 96, 358, 199, 371, 248, 192, 100, 181, 205, 235, 150, 182, 341, 147, 164, 186, 135, 66, 364, 221, 148, 366, 204, 354, 297, 183, 313, 215, 211, 263, 370, 327, 207, 104, 169, 251, 9, 54, 226, 311, 236, 383, 232, 294, 331, 140, 350, 250, 73, 116, 136, 98, 45, 50, 138, 340, 94, 198, 217, 62, 134, 102, 68, 330, 49, 87, 310, 245, 72, 7, 60, 75, 40, 377, 2, 362, 261, 333, 107, 278, 286, 156, 300, 163, 213, 184, 166, 28, 332, 224, 103, 189, 320, 63, 267, 123, 346, 349, 348, 81, 26, 126, 360, 37, 170, 368, 118, 44, 322, 21, 329, 23, 290, 188, 10, 292, 125, 57, 344, 88, 161, 298, 328, 369, 237, 4, 271, 336, 302, 85, 268, 79, 172, 113, 380, 178, 97, 168, 308, 131, 74, 220, 53, 306, 253, 382, 95, 190, 355, 109, 76, 82, 154, 121, 324, 309, 373, 46, 325, 155, 145, 357, 61, 30, 157, 314, 8, 160, 316, 312, 279, 282, 33, 32, 264, 129, 93, 133, 130, 64, 365, 356, 13, 209, 252, 0, 273, 153, 11, 158, 335, 345, 110, 173, 112, 321, 5, 378, 142, 195, 342, 16, 137, 185, 43, 83, 119, 296, 247, 241, 139, 227, 159, 381, 141, 22, 39, 191, 225, 90, 240, 281, 266, 206, 230, 31, 274, 180, 179, 20, 106, 288, 315, 1, 287, 305] + 58: [377, 280, 364, 201, 204, 340, 33, 38, 58, 134, 209, 308, 220, 71, 125, 319, 321, 335, 370, 10, 108, 150, 168, 184, 81, 144, 70, 170, 354, 349, 218, 233, 252, 292, 309, 3, 245, 104, 229, 244, 378, 161, 50, 221, 146, 137, 320, 279, 156, 191, 185, 381, 135, 253, 230, 268, 200, 216, 80, 147, 275, 196, 260, 236, 250, 347, 293, 100, 167, 301, 164, 158, 126, 45, 60, 48, 99, 37, 327, 173, 5, 57, 323, 23, 56, 376, 162, 192, 284, 339, 11, 113, 24, 109, 343, 165, 366, 36, 189, 73, 375, 300, 357, 180, 317, 51, 304, 166, 305, 116, 342, 106, 53, 382, 107, 179, 303, 290, 15, 313, 205, 257, 203, 198, 331, 241, 172, 64, 102, 117, 363, 212, 92, 72, 272, 328, 202, 302, 379, 338, 54, 337, 30, 291, 98, 289, 295, 255, 139, 350, 124, 278, 353, 163, 41, 17, 194, 69, 262, 111, 1, 181, 332, 148, 219, 128, 374, 127, 311, 22, 68, 132, 380, 258, 63, 13, 118, 213, 120, 91, 222, 9, 232, 130, 187, 25, 160, 334, 8, 264, 103, 52, 177, 49, 152, 315, 281, 62, 240, 288, 237, 330, 44, 153, 312, 235, 175, 97, 28, 251, 227, 318, 195, 276, 157, 78, 122, 299, 368, 6, 316, 238, 234, 345, 225, 359, 46, 159, 298, 206, 40, 85, 190, 129, 75, 154, 133, 261, 246, 361, 114, 93, 178, 141, 171, 55, 121, 176, 136, 296, 32, 188, 263, 322, 169, 367, 352, 266, 131, 186, 94, 243, 34, 267, 248, 373, 112, 47, 20, 215, 83, 228, 19, 138, 39, 365, 77, 101, 214, 31, 242, 59, 155, 21, 265, 142, 369, 89, 249, 271, 256, 18, 197, 82, 326, 2, 115, 360, 96, 74, 151, 14, 199, 341, 66, 239, 149, 231, 105, 294, 145, 174, 76, 211, 26, 324, 224, 314, 333, 371, 182, 29, 223, 217, 61, 210, 355, 16, 274, 325, 110, 208, 226, 95, 247, 310, 79, 344, 270, 119, 4, 306, 0, 362, 297, 372, 269, 140, 259, 348, 351, 12, 285, 383, 183, 65, 283, 282, 254, 88, 273, 27, 67, 277, 329, 143, 90, 43, 287, 87, 84, 193, 7, 356, 358, 42, 307, 207, 346, 123, 35, 86, 336, 286] + 59: [327, 51, 110, 93, 241, 56, 190, 308, 341, 215, 271, 189, 25, 223, 248, 69, 123, 246, 89, 230, 192, 375, 188, 154, 170, 175, 162, 258, 349, 149, 326, 11, 303, 351, 310, 79, 229, 262, 40, 373, 198, 61, 325, 261, 312, 335, 4, 268, 7, 178, 304, 252, 339, 2, 41, 145, 284, 270, 116, 235, 140, 265, 114, 62, 54, 101, 32, 207, 210, 305, 272, 90, 209, 172, 128, 297, 338, 289, 380, 343, 183, 301, 55, 111, 224, 221, 144, 35, 20, 180, 96, 129, 296, 239, 205, 117, 250, 294, 26, 57, 376, 269, 242, 34, 277, 358, 39, 15, 379, 371, 23, 161, 315, 120, 68, 135, 49, 103, 14, 139, 370, 263, 317, 109, 267, 214, 353, 299, 359, 3, 219, 293, 94, 204, 372, 112, 163, 356, 228, 48, 74, 44, 350, 77, 195, 247, 9, 133, 320, 222, 337, 71, 184, 357, 276, 347, 202, 298, 377, 290, 169, 47, 36, 227, 124, 148, 8, 345, 260, 374, 186, 342, 383, 179, 6, 0, 217, 118, 82, 99, 59, 311, 38, 45, 364, 158, 92, 64, 153, 88, 360, 171, 363, 381, 213, 176, 280, 10, 238, 63, 330, 24, 368, 323, 200, 251, 328, 91, 211, 236, 259, 43, 134, 191, 165, 72, 173, 70, 295, 278, 302, 355, 354, 193, 157, 275, 319, 122, 340, 95, 324, 212, 334, 329, 131, 194, 37, 336, 125, 244, 220, 5, 151, 164, 100, 286, 13, 206, 17, 249, 257, 83, 66, 233, 300, 86, 159, 306, 365, 333, 21, 106, 216, 52, 346, 226, 31, 196, 127, 245, 361, 27, 231, 119, 108, 104, 352, 147, 18, 255, 279, 19, 203, 138, 84, 321, 282, 87, 234, 197, 42, 58, 181, 142, 155, 287, 28, 132, 1, 264, 65, 97, 185, 309, 168, 29, 177, 156, 67, 115, 283, 167, 332, 313, 331, 243, 12, 143, 348, 107, 187, 307, 240, 121, 73, 256, 76, 281, 85, 22, 344, 362, 208, 102, 98, 378, 113, 46, 150, 266, 16, 218, 366, 314, 253, 53, 237, 78, 182, 75, 288, 232, 254, 174, 369, 199, 50, 166, 316, 126, 33, 285, 291, 382, 160, 60, 137, 292, 136, 201, 318, 152, 30, 105, 130, 80, 81, 225, 146, 274, 322, 273, 141, 367] + 60: [313, 317, 63, 154, 207, 348, 299, 288, 375, 22, 201, 355, 124, 218, 226, 242, 308, 89, 136, 210, 373, 62, 157, 214, 351, 262, 224, 135, 173, 4, 32, 101, 321, 10, 77, 87, 123, 8, 66, 380, 202, 50, 229, 163, 295, 352, 118, 109, 257, 140, 183, 274, 85, 79, 18, 258, 97, 164, 340, 44, 16, 235, 20, 162, 21, 95, 225, 198, 356, 71, 103, 73, 268, 234, 232, 336, 259, 304, 200, 219, 251, 379, 281, 275, 381, 76, 122, 152, 1, 116, 91, 144, 23, 374, 254, 177, 158, 2, 329, 27, 269, 110, 142, 30, 333, 320, 175, 227, 220, 153, 353, 231, 362, 188, 171, 100, 338, 178, 301, 289, 47, 312, 49, 117, 189, 344, 186, 190, 105, 45, 17, 292, 139, 120, 319, 272, 84, 11, 25, 322, 64, 19, 147, 138, 60, 247, 337, 184, 150, 240, 323, 370, 331, 137, 96, 367, 126, 230, 294, 131, 75, 318, 279, 293, 368, 67, 42, 68, 83, 167, 35, 245, 172, 6, 377, 70, 182, 325, 179, 111, 133, 194, 108, 206, 359, 51, 57, 233, 300, 363, 216, 298, 199, 78, 239, 278, 253, 217, 176, 346, 369, 383, 261, 291, 40, 48, 205, 15, 263, 36, 264, 31, 7, 168, 209, 72, 174, 222, 366, 213, 246, 132, 237, 273, 106, 371, 54, 378, 265, 212, 34, 98, 364, 221, 3, 107, 130, 148, 228, 86, 310, 316, 208, 311, 55, 37, 38, 360, 192, 315, 287, 159, 284, 339, 81, 166, 238, 114, 61, 53, 149, 270, 324, 165, 102, 113, 155, 193, 241, 93, 267, 187, 341, 125, 127, 43, 94, 39, 74, 354, 169, 255, 382, 285, 33, 376, 328, 191, 236, 372, 327, 309, 82, 358, 215, 52, 146, 307, 252, 249, 121, 347, 28, 345, 99, 283, 5, 90, 196, 282, 13, 306, 335, 244, 181, 119, 271, 248, 9, 134, 357, 197, 330, 326, 128, 343, 334, 290, 59, 58, 88, 56, 314, 12, 280, 29, 260, 303, 302, 112, 223, 69, 332, 26, 349, 14, 41, 160, 170, 129, 65, 180, 143, 277, 24, 305, 296, 286, 104, 204, 115, 266, 185, 365, 161, 151, 92, 141, 250, 156, 361, 350, 195, 243, 297, 46, 203, 0, 80, 342, 276, 256, 145, 211] + 61: [221, 44, 104, 37, 342, 246, 28, 217, 356, 155, 0, 58, 127, 366, 145, 306, 4, 54, 314, 226, 128, 321, 149, 319, 179, 159, 75, 308, 137, 174, 204, 262, 339, 376, 96, 94, 338, 286, 133, 260, 72, 365, 244, 354, 285, 175, 53, 208, 61, 294, 181, 43, 257, 186, 125, 359, 24, 265, 229, 171, 73, 68, 291, 87, 374, 259, 219, 274, 187, 60, 151, 287, 93, 380, 66, 32, 300, 147, 141, 144, 305, 373, 40, 116, 211, 1, 119, 227, 83, 97, 196, 176, 201, 95, 334, 343, 46, 264, 237, 309, 135, 52, 275, 124, 101, 256, 62, 189, 299, 184, 213, 169, 114, 328, 30, 71, 163, 272, 277, 350, 197, 209, 55, 243, 84, 245, 346, 317, 25, 123, 250, 183, 381, 59, 143, 156, 268, 23, 153, 81, 91, 99, 35, 210, 100, 241, 198, 282, 29, 206, 273, 132, 271, 234, 318, 382, 36, 270, 235, 160, 316, 368, 214, 26, 63, 18, 347, 134, 167, 19, 215, 15, 233, 69, 78, 295, 130, 301, 220, 355, 315, 10, 142, 31, 90, 3, 292, 340, 352, 364, 222, 39, 344, 203, 238, 349, 150, 336, 89, 375, 311, 255, 56, 254, 324, 117, 371, 177, 118, 113, 322, 341, 98, 232, 105, 41, 248, 178, 278, 112, 258, 304, 200, 240, 194, 13, 180, 357, 263, 323, 51, 192, 77, 166, 38, 8, 106, 312, 140, 267, 307, 122, 21, 188, 297, 27, 154, 280, 152, 49, 48, 320, 161, 205, 12, 57, 261, 2, 173, 22, 148, 251, 146, 367, 86, 165, 195, 42, 348, 252, 378, 79, 298, 7, 182, 266, 88, 351, 17, 129, 361, 379, 158, 80, 345, 310, 377, 289, 107, 247, 330, 168, 20, 313, 102, 269, 70, 358, 276, 281, 216, 326, 372, 327, 224, 190, 383, 9, 332, 185, 164, 296, 231, 170, 103, 115, 228, 47, 249, 283, 162, 74, 236, 50, 303, 157, 253, 369, 191, 11, 207, 67, 199, 293, 212, 16, 230, 120, 353, 34, 92, 109, 290, 85, 14, 111, 223, 76, 172, 360, 218, 110, 65, 82, 139, 242, 370, 6, 302, 279, 5, 108, 45, 288, 337, 329, 33, 126, 193, 239, 64, 131, 136, 335, 363, 333, 362, 121, 202, 225, 284, 331, 138, 325] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots416.yaml b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots416.yaml new file mode 100644 index 000000000000..cf48f10d4ae8 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots416.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [259, 332, 218, 203, 349, 233, 337, 317, 99, 326, 334, 239, 201, 166, 45, 95, 278, 330, 265, 77, 213, 366, 252, 352, 23, 8, 166, 190, 157, 139, 360, 156, 300, 4, 178, 260, 340, 250, 118, 0, 45, 95, 63, 144, 124, 138, 137, 363, 350, 15, 167, 288, 256, 378, 80, 68, 290, 200, 44, 335, 192, 263, 175, 285, 120, 256, 346, 17, 293, 330, 361, 149, 214, 369, 229, 338, 226, 291, 150, 378, 29, 203, 126, 183, 123, 5, 25, 73, 2, 319, 61, 151, 378, 79, 333, 280, 90, 112, 46, 75, 210, 96, 287, 364, 74, 158, 62, 176, 290, 311, 7, 244, 211, 342, 380, 18, 88, 74, 241, 242, 37, 173, 297, 268, 377, 320, 127, 219, 220, 310, 71, 279, 17, 217, 3, 36, 31, 257, 215, 362, 306, 251, 13, 222, 193, 30, 16, 173, 129, 170, 227, 59, 328, 179, 301, 247, 222, 230, 104, 198, 209, 10, 94, 365, 275, 93, 113, 367, 303, 302, 279, 204, 318, 224, 258, 148, 187, 238, 98, 294, 329, 344, 351, 207, 79, 276, 224, 174, 248, 108, 116, 14, 315, 164, 370, 356, 264, 202, 232, 54, 105, 262, 91, 1, 6, 316, 121, 281, 356, 76, 153, 134, 284, 131, 38, 52, 195, 82, 53, 266, 58, 84, 135, 60, 321, 205, 57, 324, 331, 216, 67, 353, 125, 277, 159, 135, 21, 184, 78, 56, 114, 146, 246, 50, 212, 271, 130, 345, 235, 81, 134, 359, 354, 368, 240, 269, 20, 42, 142, 355, 322, 111, 60, 289, 307, 196, 89, 39, 106, 267, 152, 194, 373, 119, 111, 26, 327, 33, 11, 199, 85, 379, 382, 191, 231, 274, 236, 165, 122, 49, 181, 43, 208, 136, 19, 147, 69, 381, 185, 236, 254, 35, 48, 141, 283, 160, 32, 296, 286, 305, 374, 325, 245, 254, 347, 255, 143, 107, 234, 66, 292, 83, 12, 9, 171, 245, 111, 51, 343, 55, 313, 314, 308, 100, 154, 40, 298, 168, 371, 165, 163, 49, 72, 34, 309, 172, 180, 145, 115, 132, 323, 371, 165, 295, 184, 243, 117, 375, 133, 97, 102, 161, 304, 339, 383, 177, 312, 217, 272, 186, 28, 103, 372, 47, 64, 189, 128, 225, 162, 169, 376, 336, 341, 27, 87, 221, 197, 253, 24, 228, 225, 348, 188, 70, 22, 101, 358, 223, 270, 299, 206, 92, 273, 86, 155, 182, 65, 282, 109, 261, 249, 140, 237, 41, 357, 110] + 1: [134, 378, 73, 225, 59, 166, 202, 231, 154, 230, 314, 361, 157, 178, 254, 110, 225, 195, 187, 376, 60, 175, 284, 103, 66, 336, 380, 329, 41, 250, 273, 197, 264, 205, 165, 87, 220, 340, 255, 141, 127, 328, 3, 32, 281, 47, 140, 286, 181, 240, 64, 307, 251, 127, 377, 3, 173, 266, 214, 313, 318, 113, 54, 128, 159, 81, 163, 325, 182, 275, 285, 143, 259, 364, 8, 232, 374, 117, 200, 163, 67, 153, 224, 335, 239, 61, 115, 11, 130, 373, 15, 365, 291, 6, 198, 145, 99, 52, 79, 2, 26, 279, 183, 354, 365, 163, 67, 23, 138, 108, 358, 76, 296, 51, 369, 151, 257, 1, 167, 192, 120, 269, 212, 360, 102, 68, 48, 119, 155, 349, 235, 167, 5, 198, 37, 301, 53, 305, 161, 13, 262, 45, 144, 42, 177, 0, 105, 242, 125, 206, 348, 20, 97, 371, 253, 112, 42, 196, 70, 63, 89, 234, 203, 298, 19, 324, 309, 193, 327, 201, 169, 179, 244, 287, 355, 4, 170, 370, 268, 226, 322, 359, 201, 177, 270, 300, 211, 317, 219, 222, 65, 104, 246, 188, 199, 174, 72, 30, 120, 303, 146, 176, 121, 326, 55, 12, 208, 215, 174, 169, 40, 82, 29, 162, 88, 98, 33, 36, 331, 367, 368, 31, 72, 131, 248, 292, 218, 306, 237, 321, 186, 297, 160, 345, 22, 91, 319, 271, 46, 261, 38, 133, 136, 35, 356, 16, 258, 22, 91, 319, 351, 320, 142, 381, 362, 14, 227, 27, 132, 341, 308, 43, 21, 233, 243, 84, 93, 256, 57, 172, 83, 135, 363, 118, 263, 366, 299, 9, 74, 213, 204, 338, 94, 353, 191, 276, 137, 50, 168, 111, 272, 194, 289, 122, 17, 311, 372, 90, 267, 137, 152, 319, 82, 383, 109, 278, 236, 295, 350, 375, 207, 304, 18, 150, 241, 111, 272, 92, 228, 164, 107, 352, 129, 316, 100, 180, 126, 347, 185, 283, 379, 342, 147, 184, 293, 148, 217, 310, 180, 149, 347, 24, 25, 280, 171, 101, 156, 302, 49, 252, 339, 44, 382, 377, 182, 260, 78, 223, 238, 139, 96, 333, 343, 28, 158, 277, 123, 334, 247, 245, 282, 7, 332, 346, 344, 10, 312, 158, 382, 39, 56, 292, 288, 69, 95, 62, 290, 190, 209, 34, 106, 277, 210, 58, 71, 216, 116, 189, 229, 315, 357, 221, 265, 80, 85, 75, 294, 86, 274, 114, 330, 249, 323, 77, 124, 337] + 2: [53, 33, 322, 245, 22, 356, 126, 103, 38, 211, 98, 37, 375, 53, 75, 191, 70, 63, 209, 201, 41, 365, 231, 292, 260, 337, 43, 248, 124, 86, 26, 299, 327, 15, 305, 35, 183, 227, 190, 43, 197, 284, 44, 325, 87, 50, 348, 150, 141, 226, 315, 358, 180, 248, 97, 56, 83, 204, 361, 309, 142, 272, 251, 283, 199, 230, 248, 350, 280, 274, 294, 187, 241, 252, 52, 371, 177, 300, 96, 34, 185, 118, 122, 347, 55, 175, 257, 144, 291, 214, 282, 132, 212, 153, 86, 368, 224, 293, 193, 268, 233, 302, 369, 237, 132, 212, 110, 278, 200, 174, 58, 332, 2, 178, 373, 222, 363, 112, 143, 155, 73, 146, 61, 21, 10, 318, 261, 54, 310, 366, 112, 143, 84, 157, 24, 285, 76, 145, 39, 281, 74, 336, 381, 108, 247, 109, 286, 279, 196, 218, 65, 176, 376, 95, 117, 161, 108, 143, 19, 135, 122, 263, 27, 308, 29, 334, 167, 64, 228, 225, 125, 290, 220, 31, 344, 20, 265, 179, 207, 107, 378, 379, 341, 151, 19, 271, 296, 18, 156, 181, 312, 264, 288, 382, 258, 351, 100, 162, 105, 304, 90, 269, 250, 106, 239, 158, 345, 255, 313, 321, 168, 355, 85, 311, 236, 171, 49, 346, 205, 80, 326, 313, 189, 78, 172, 314, 170, 188, 129, 372, 275, 307, 147, 354, 357, 137, 111, 377, 328, 165, 67, 254, 136, 77, 164, 127, 119, 215, 295, 62, 91, 82, 6, 120, 349, 23, 319, 71, 182, 330, 215, 295, 62, 91, 11, 323, 297, 46, 219, 352, 329, 380, 32, 14, 221, 168, 66, 340, 342, 128, 360, 0, 367, 114, 266, 316, 89, 3, 101, 184, 51, 130, 92, 116, 9, 25, 8, 88, 287, 89, 3, 290, 123, 139, 140, 320, 160, 154, 47, 335, 13, 208, 343, 5, 149, 306, 324, 273, 68, 169, 12, 134, 267, 289, 301, 194, 81, 213, 133, 139, 206, 210, 186, 121, 238, 338, 353, 202, 194, 259, 232, 304, 314, 17, 203, 276, 364, 36, 59, 317, 246, 102, 1, 192, 42, 45, 7, 262, 69, 256, 195, 48, 72, 229, 93, 223, 232, 377, 217, 79, 383, 163, 94, 359, 216, 240, 339, 93, 298, 60, 131, 333, 159, 148, 277, 152, 4, 99, 234, 40, 115, 28, 242, 173, 235, 30, 138, 253, 362, 57, 243, 249, 270, 115, 223, 242, 166, 244, 303, 198, 16, 104, 331, 370, 113, 374] + 3: [39, 192, 97, 336, 36, 109, 94, 366, 115, 101, 116, 175, 226, 39, 3, 194, 37, 287, 99, 200, 214, 20, 362, 121, 95, 198, 22, 56, 263, 358, 177, 161, 166, 255, 67, 373, 360, 205, 38, 22, 208, 261, 342, 182, 271, 270, 160, 43, 93, 190, 347, 17, 66, 27, 333, 2, 316, 84, 55, 44, 361, 153, 88, 75, 52, 66, 27, 80, 168, 25, 277, 224, 105, 91, 259, 107, 241, 301, 66, 131, 194, 325, 221, 172, 235, 76, 249, 321, 140, 89, 9, 66, 163, 313, 227, 74, 78, 197, 237, 306, 285, 179, 308, 335, 66, 163, 313, 211, 212, 359, 256, 328, 340, 206, 183, 23, 193, 66, 13, 3, 10, 138, 289, 380, 268, 70, 282, 16, 127, 233, 71, 225, 157, 343, 64, 111, 345, 104, 187, 293, 251, 272, 257, 46, 317, 263, 58, 174, 279, 327, 196, 286, 269, 134, 315, 341, 34, 14, 213, 201, 382, 229, 204, 65, 367, 346, 191, 253, 370, 34, 171, 145, 164, 219, 136, 112, 276, 298, 309, 323, 352, 146, 34, 51, 150, 139, 12, 262, 274, 117, 7, 381, 369, 54, 292, 34, 31, 158, 186, 29, 149, 126, 135, 232, 61, 242, 90, 303, 34, 350, 348, 207, 223, 374, 98, 280, 291, 130, 284, 147, 59, 34, 19, 62, 63, 118, 154, 217, 144, 216, 258, 354, 152, 33, 79, 18, 49, 122, 304, 295, 103, 273, 15, 85, 210, 148, 199, 79, 18, 81, 267, 296, 8, 53, 30, 322, 254, 245, 320, 5, 310, 319, 252, 189, 77, 142, 236, 82, 353, 283, 378, 349, 371, 310, 319, 50, 250, 239, 162, 344, 125, 96, 86, 220, 4, 173, 310, 319, 180, 170, 11, 32, 108, 355, 376, 228, 372, 60, 326, 275, 40, 331, 26, 264, 312, 181, 48, 128, 123, 329, 368, 311, 260, 365, 42, 215, 240, 92, 155, 383, 184, 28, 379, 69, 167, 260, 365, 137, 337, 290, 297, 375, 307, 294, 356, 45, 141, 47, 260, 40, 6, 102, 202, 203, 218, 169, 83, 132, 338, 330, 278, 35, 365, 334, 178, 21, 114, 176, 113, 188, 72, 156, 124, 247, 377, 302, 363, 238, 357, 243, 300, 246, 305, 110, 234, 265, 332, 377, 302, 318, 339, 87, 351, 41, 129, 106, 68, 73, 1, 324, 377, 314, 151, 231, 195, 364, 57, 100, 248, 133, 0, 244, 185, 230, 209, 24, 120, 119, 288, 299, 266, 281, 165, 159, 143, 222] + 4: [188, 19, 273, 253, 68, 0, 185, 64, 157, 146, 255, 174, 182, 34, 376, 117, 36, 280, 116, 379, 125, 29, 122, 205, 89, 287, 340, 226, 194, 274, 333, 187, 359, 80, 292, 130, 81, 382, 351, 3, 88, 142, 377, 8, 38, 189, 232, 346, 304, 353, 245, 233, 3, 281, 194, 59, 312, 22, 323, 310, 78, 140, 311, 266, 104, 3, 226, 105, 155, 339, 82, 149, 322, 176, 147, 362, 102, 227, 18, 95, 254, 347, 160, 177, 93, 69, 138, 40, 378, 326, 336, 209, 239, 152, 61, 364, 244, 114, 46, 303, 327, 374, 373, 225, 209, 95, 254, 167, 83, 179, 252, 240, 184, 210, 48, 248, 208, 271, 228, 136, 329, 204, 7, 341, 328, 178, 355, 129, 212, 119, 271, 172, 302, 13, 344, 207, 360, 216, 263, 334, 279, 30, 242, 271, 9, 152, 200, 361, 264, 72, 269, 103, 295, 97, 293, 192, 37, 16, 241, 139, 235, 358, 330, 131, 349, 286, 115, 285, 17, 12, 15, 165, 246, 57, 141, 62, 132, 26, 170, 126, 270, 305, 375, 15, 143, 76, 27, 35, 14, 137, 52, 211, 55, 91, 79, 124, 23, 348, 325, 101, 159, 352, 100, 309, 156, 381, 363, 277, 199, 193, 148, 298, 49, 110, 75, 367, 50, 118, 316, 144, 128, 222, 23, 148, 56, 217, 153, 85, 158, 39, 71, 162, 265, 58, 258, 371, 257, 77, 365, 221, 166, 135, 66, 111, 247, 368, 63, 296, 214, 87, 4, 314, 278, 98, 261, 175, 291, 186, 299, 332, 296, 371, 45, 41, 43, 369, 60, 354, 230, 237, 112, 11, 337, 300, 214, 319, 372, 32, 283, 356, 74, 206, 181, 203, 301, 231, 343, 350, 117, 342, 198, 335, 168, 70, 92, 223, 191, 164, 67, 343, 259, 273, 86, 251, 42, 297, 229, 121, 249, 21, 190, 20, 343, 250, 87, 318, 357, 99, 307, 288, 84, 31, 380, 134, 272, 343, 320, 5, 275, 151, 276, 256, 108, 51, 180, 10, 195, 282, 343, 214, 319, 113, 44, 317, 169, 262, 267, 109, 173, 366, 321, 331, 24, 308, 106, 196, 243, 53, 313, 234, 289, 123, 224, 2, 331, 219, 6, 145, 218, 183, 133, 284, 370, 213, 96, 236, 107, 331, 315, 150, 197, 28, 260, 33, 383, 294, 268, 1, 238, 338, 331, 315, 171, 47, 73, 54, 290, 25, 306, 201, 163, 202, 90, 331, 315, 142, 345, 161, 324, 65, 94, 220, 127, 215, 154, 120] + 5: [172, 269, 64, 182, 327, 184, 19, 56, 271, 187, 297, 110, 179, 208, 169, 102, 364, 304, 143, 12, 20, 349, 199, 347, 226, 83, 61, 188, 41, 192, 66, 377, 195, 241, 13, 113, 197, 357, 23, 61, 305, 21, 89, 8, 218, 301, 289, 254, 298, 189, 27, 321, 61, 305, 21, 147, 155, 152, 104, 136, 119, 244, 290, 0, 166, 165, 193, 379, 198, 371, 38, 345, 341, 360, 221, 309, 123, 167, 154, 262, 259, 255, 238, 324, 79, 343, 376, 378, 111, 340, 196, 190, 105, 259, 130, 2, 3, 380, 382, 94, 283, 267, 282, 44, 190, 370, 211, 72, 369, 315, 228, 284, 330, 168, 125, 248, 118, 141, 105, 122, 132, 332, 45, 16, 325, 9, 129, 185, 294, 82, 213, 276, 159, 252, 163, 210, 51, 366, 229, 40, 224, 112, 303, 213, 100, 272, 246, 313, 314, 99, 317, 334, 270, 133, 234, 96, 213, 100, 372, 293, 131, 220, 33, 76, 383, 58, 148, 116, 157, 249, 223, 78, 43, 37, 207, 266, 230, 62, 263, 11, 146, 50, 249, 276, 231, 88, 171, 318, 93, 362, 77, 205, 73, 311, 354, 128, 346, 231, 200, 151, 251, 329, 257, 291, 374, 307, 36, 127, 128, 346, 159, 145, 174, 26, 46, 138, 149, 68, 373, 339, 177, 128, 265, 6, 7, 15, 140, 242, 201, 29, 235, 85, 328, 75, 306, 49, 272, 212, 268, 258, 28, 63, 48, 342, 245, 117, 278, 256, 60, 288, 331, 70, 206, 39, 353, 55, 103, 120, 158, 57, 84, 355, 6, 352, 375, 320, 115, 203, 275, 316, 32, 10, 142, 84, 178, 52, 279, 59, 153, 35, 150, 80, 299, 126, 202, 337, 296, 355, 381, 22, 124, 108, 183, 106, 191, 135, 95, 310, 176, 296, 355, 356, 25, 31, 69, 274, 300, 134, 363, 97, 243, 170, 308, 65, 30, 338, 233, 319, 253, 121, 344, 323, 139, 42, 164, 214, 1, 287, 91, 4, 204, 216, 239, 162, 107, 86, 273, 225, 214, 81, 356, 74, 209, 181, 227, 14, 232, 156, 261, 367, 368, 214, 47, 269, 335, 173, 67, 312, 250, 236, 359, 285, 350, 302, 5, 217, 288, 351, 358, 24, 186, 219, 87, 114, 101, 180, 326, 194, 98, 365, 17, 222, 175, 144, 240, 361, 71, 53, 336, 286, 194, 217, 52, 215, 264, 34, 54, 90, 281, 92, 322, 280, 277, 160, 260, 287, 247, 137, 333, 295, 161, 237, 18, 292, 109, 348] + 6: [322, 46, 350, 283, 176, 279, 69, 159, 299, 310, 78, 26, 90, 322, 99, 371, 255, 179, 158, 271, 225, 54, 73, 356, 127, 325, 322, 99, 9, 84, 113, 223, 237, 82, 215, 244, 250, 29, 161, 146, 11, 36, 56, 345, 37, 262, 284, 272, 364, 263, 92, 164, 76, 281, 375, 57, 277, 49, 43, 23, 16, 366, 238, 171, 257, 76, 281, 378, 368, 95, 311, 242, 372, 83, 251, 256, 72, 65, 76, 229, 166, 239, 128, 114, 10, 14, 168, 290, 266, 265, 153, 297, 240, 177, 34, 182, 221, 280, 330, 96, 123, 339, 192, 342, 305, 55, 180, 208, 278, 107, 332, 148, 198, 106, 51, 170, 135, 305, 240, 233, 118, 369, 77, 264, 81, 333, 328, 102, 20, 252, 97, 28, 205, 178, 41, 321, 210, 276, 309, 134, 131, 213, 373, 97, 341, 66, 184, 377, 44, 136, 145, 48, 360, 338, 191, 167, 97, 253, 209, 155, 352, 139, 296, 122, 105, 347, 162, 61, 75, 97, 253, 66, 291, 336, 327, 324, 316, 199, 85, 117, 2, 314, 212, 269, 233, 79, 236, 211, 143, 126, 346, 226, 17, 228, 132, 142, 174, 45, 189, 354, 30, 87, 60, 334, 74, 47, 273, 25, 142, 267, 173, 112, 270, 12, 361, 63, 381, 235, 121, 241, 35, 142, 258, 31, 319, 109, 282, 365, 86, 349, 357, 201, 91, 331, 142, 258, 67, 188, 185, 190, 22, 116, 70, 292, 374, 19, 320, 59, 174, 120, 196, 245, 169, 58, 362, 298, 39, 183, 1, 204, 295, 306, 175, 246, 165, 304, 110, 335, 125, 379, 302, 140, 247, 274, 343, 181, 293, 0, 68, 206, 355, 358, 62, 348, 200, 214, 274, 27, 177, 287, 230, 383, 286, 138, 18, 337, 303, 382, 186, 274, 249, 166, 129, 115, 64, 98, 194, 50, 152, 89, 141, 71, 5, 343, 13, 100, 234, 101, 137, 197, 268, 285, 313, 93, 275, 308, 32, 216, 367, 151, 231, 104, 4, 193, 108, 294, 353, 149, 308, 32, 318, 232, 300, 111, 6, 243, 42, 312, 38, 7, 40, 308, 53, 307, 187, 259, 202, 154, 329, 3, 203, 24, 340, 33, 308, 53, 163, 94, 301, 317, 207, 217, 124, 288, 195, 344, 52, 376, 103, 260, 119, 370, 144, 220, 254, 147, 80, 351, 218, 363, 376, 160, 359, 88, 156, 219, 15, 248, 224, 150, 21, 172, 315, 376, 32, 326, 289, 380, 133, 157, 261, 8, 130, 222, 227, 323] + 7: [163, 340, 328, 217, 31, 81, 5, 220, 351, 106, 254, 143, 154, 341, 340, 188, 100, 278, 372, 16, 134, 233, 295, 297, 147, 223, 327, 382, 212, 267, 248, 158, 314, 53, 161, 316, 124, 95, 110, 327, 382, 329, 215, 84, 21, 20, 335, 193, 184, 266, 198, 364, 327, 271, 97, 82, 125, 62, 204, 152, 68, 268, 63, 299, 338, 175, 343, 354, 307, 4, 330, 245, 121, 283, 1, 59, 347, 240, 175, 343, 15, 164, 55, 255, 180, 200, 93, 37, 315, 359, 50, 162, 342, 48, 113, 337, 377, 92, 44, 373, 72, 312, 54, 122, 157, 46, 378, 249, 289, 243, 259, 323, 225, 115, 228, 304, 202, 272, 265, 369, 47, 349, 70, 190, 273, 153, 79, 159, 195, 60, 182, 229, 118, 11, 168, 208, 186, 58, 40, 69, 317, 172, 279, 182, 265, 369, 294, 285, 210, 237, 380, 196, 194, 3, 332, 257, 7, 142, 298, 324, 17, 117, 178, 83, 101, 151, 277, 346, 42, 7, 142, 298, 183, 262, 264, 9, 155, 22, 320, 381, 57, 135, 7, 229, 362, 313, 29, 177, 130, 102, 65, 75, 85, 127, 376, 274, 28, 362, 14, 74, 191, 321, 353, 38, 126, 334, 45, 371, 274, 375, 71, 281, 23, 383, 107, 322, 366, 166, 367, 144, 76, 274, 375, 99, 73, 119, 221, 19, 137, 52, 269, 78, 242, 171, 238, 187, 253, 0, 284, 167, 32, 211, 120, 318, 128, 246, 357, 258, 176, 139, 244, 356, 206, 129, 293, 345, 209, 80, 352, 365, 275, 292, 25, 181, 148, 213, 358, 305, 116, 35, 43, 66, 276, 275, 260, 18, 77, 235, 214, 26, 51, 146, 286, 111, 12, 326, 336, 292, 201, 174, 374, 98, 24, 301, 87, 216, 252, 150, 160, 192, 49, 25, 131, 2, 224, 109, 141, 261, 287, 173, 165, 145, 192, 292, 103, 256, 360, 90, 140, 67, 309, 36, 96, 282, 34, 308, 49, 103, 94, 288, 219, 185, 86, 247, 112, 123, 138, 156, 108, 49, 105, 370, 325, 132, 311, 226, 8, 169, 231, 197, 236, 263, 41, 18, 104, 319, 300, 368, 310, 149, 363, 88, 199, 355, 263, 350, 280, 61, 296, 333, 189, 306, 291, 33, 10, 331, 91, 64, 133, 201, 27, 250, 241, 234, 232, 203, 222, 303, 230, 348, 64, 41, 105, 114, 339, 344, 89, 56, 227, 207, 39, 205, 30, 361, 239, 13, 302, 270, 170, 136, 6, 251, 218, 379, 290, 179] + 8: [273, 62, 122, 195, 22, 303, 125, 28, 126, 85, 220, 43, 289, 253, 10, 281, 374, 128, 183, 91, 117, 136, 35, 365, 274, 256, 253, 10, 281, 39, 166, 194, 358, 383, 266, 89, 287, 94, 288, 253, 243, 182, 364, 188, 112, 72, 217, 299, 9, 315, 301, 345, 104, 241, 74, 168, 69, 309, 60, 206, 382, 295, 323, 82, 222, 214, 321, 182, 377, 75, 167, 332, 98, 155, 235, 193, 363, 63, 5, 336, 276, 115, 107, 368, 61, 187, 143, 79, 357, 139, 124, 5, 321, 81, 156, 366, 278, 233, 71, 297, 46, 33, 203, 1, 250, 173, 36, 157, 226, 17, 120, 87, 327, 378, 324, 305, 49, 3, 312, 152, 131, 181, 198, 66, 263, 372, 159, 153, 53, 242, 279, 162, 229, 186, 119, 178, 320, 249, 47, 306, 277, 331, 176, 360, 12, 204, 20, 258, 40, 300, 90, 310, 29, 129, 329, 207, 360, 362, 371, 280, 346, 137, 114, 254, 37, 236, 24, 51, 163, 360, 362, 371, 86, 311, 196, 197, 174, 140, 105, 146, 142, 227, 360, 162, 152, 97, 172, 189, 8, 59, 68, 144, 158, 26, 96, 290, 13, 228, 223, 38, 78, 211, 191, 350, 54, 351, 138, 19, 290, 201, 337, 95, 34, 261, 318, 232, 151, 369, 208, 319, 234, 290, 201, 229, 252, 179, 202, 42, 45, 314, 339, 370, 354, 58, 246, 239, 204, 291, 231, 298, 4, 335, 190, 205, 15, 123, 102, 218, 239, 64, 148, 225, 50, 341, 116, 373, 322, 340, 245, 55, 292, 224, 221, 11, 83, 113, 177, 248, 149, 375, 77, 269, 170, 67, 264, 62, 251, 147, 169, 141, 237, 16, 200, 84, 160, 132, 67, 264, 221, 30, 240, 216, 48, 118, 56, 99, 326, 342, 127, 31, 7, 88, 349, 333, 92, 2, 212, 199, 265, 171, 70, 57, 31, 108, 154, 255, 283, 121, 164, 247, 184, 185, 271, 286, 0, 238, 367, 282, 380, 6, 330, 52, 230, 313, 325, 348, 343, 210, 135, 359, 353, 209, 23, 165, 267, 25, 32, 175, 244, 180, 100, 135, 161, 308, 328, 109, 65, 317, 293, 259, 14, 111, 80, 296, 338, 367, 381, 294, 150, 219, 192, 257, 352, 347, 103, 285, 213, 302, 359, 353, 316, 133, 275, 334, 73, 110, 93, 270, 268, 376, 106, 359, 308, 18, 356, 304, 284, 101, 41, 260, 76, 355, 262, 21, 21, 272, 44, 27, 215, 130, 379, 307, 145, 134, 344, 361] + 9: [212, 185, 287, 244, 158, 107, 323, 50, 60, 270, 167, 284, 25, 92, 47, 49, 95, 268, 228, 238, 339, 300, 375, 225, 240, 283, 92, 351, 159, 341, 169, 0, 181, 237, 2, 89, 347, 257, 250, 92, 381, 49, 95, 153, 231, 66, 274, 115, 221, 346, 40, 28, 92, 185, 262, 51, 120, 277, 141, 80, 261, 253, 326, 44, 61, 78, 47, 217, 10, 161, 87, 195, 109, 100, 162, 69, 73, 55, 304, 355, 210, 244, 360, 41, 186, 130, 379, 211, 178, 378, 265, 304, 355, 281, 64, 30, 321, 124, 39, 62, 84, 193, 93, 182, 304, 179, 329, 64, 144, 6, 230, 17, 45, 251, 383, 63, 369, 304, 184, 262, 147, 154, 306, 171, 5, 358, 305, 58, 118, 340, 166, 355, 380, 209, 187, 364, 36, 104, 79, 320, 368, 279, 348, 255, 327, 138, 67, 363, 183, 139, 276, 46, 330, 243, 266, 365, 255, 327, 370, 103, 135, 192, 207, 235, 117, 298, 86, 119, 24, 255, 327, 242, 271, 33, 334, 111, 288, 291, 229, 48, 113, 91, 349, 165, 316, 23, 3, 297, 332, 75, 213, 110, 290, 149, 27, 349, 222, 376, 147, 356, 128, 206, 246, 148, 134, 293, 278, 53, 349, 155, 146, 10, 137, 88, 218, 106, 114, 232, 227, 200, 157, 81, 74, 83, 56, 9, 129, 16, 319, 322, 286, 160, 328, 26, 143, 295, 127, 121, 302, 314, 123, 324, 190, 301, 68, 345, 173, 331, 219, 15, 177, 258, 99, 4, 248, 205, 76, 374, 273, 7, 220, 295, 316, 136, 289, 318, 189, 13, 197, 310, 170, 101, 236, 96, 350, 90, 214, 292, 54, 150, 308, 174, 12, 191, 249, 168, 315, 353, 313, 132, 82, 264, 72, 202, 112, 371, 140, 366, 234, 196, 357, 1, 14, 317, 180, 11, 359, 254, 224, 241, 85, 156, 31, 252, 29, 22, 98, 163, 35, 354, 373, 352, 43, 325, 57, 31, 299, 37, 102, 131, 145, 336, 208, 233, 71, 108, 176, 126, 280, 172, 198, 309, 367, 116, 142, 65, 52, 125, 188, 204, 122, 226, 263, 201, 282, 151, 97, 267, 175, 303, 21, 199, 294, 164, 226, 19, 344, 342, 285, 247, 70, 343, 203, 42, 34, 223, 77, 372, 263, 313, 32, 133, 259, 256, 8, 275, 333, 337, 296, 152, 260, 215, 90, 103, 377, 311, 105, 18, 382, 216, 59, 361, 338, 94, 94, 272, 239, 312, 20, 269, 307, 245, 38, 335, 194, 362] + 10: [294, 367, 2, 248, 293, 155, 129, 37, 193, 287, 217, 143, 160, 192, 126, 146, 47, 363, 231, 85, 328, 275, 226, 35, 347, 290, 255, 167, 331, 58, 67, 285, 21, 339, 261, 175, 16, 104, 366, 194, 279, 22, 182, 186, 249, 75, 51, 162, 173, 206, 150, 238, 194, 69, 31, 91, 76, 376, 81, 97, 247, 111, 372, 152, 110, 264, 286, 310, 241, 274, 280, 232, 357, 183, 298, 203, 94, 336, 264, 337, 102, 237, 316, 77, 314, 109, 340, 222, 127, 207, 176, 55, 18, 215, 177, 49, 359, 56, 329, 343, 370, 276, 118, 297, 55, 286, 57, 196, 208, 360, 27, 26, 317, 299, 165, 344, 82, 300, 365, 161, 117, 252, 199, 321, 20, 45, 169, 251, 83, 324, 300, 216, 54, 263, 136, 266, 105, 9, 95, 220, 133, 120, 29, 300, 216, 178, 171, 312, 0, 33, 267, 12, 375, 147, 156, 381, 300, 18, 73, 330, 377, 32, 214, 379, 174, 209, 53, 273, 68, 246, 365, 1, 114, 234, 229, 135, 378, 323, 364, 6, 334, 369, 180, 325, 146, 308, 353, 190, 335, 278, 42, 235, 187, 333, 23, 180, 365, 164, 201, 243, 90, 233, 258, 36, 80, 355, 277, 228, 93, 121, 225, 213, 115, 382, 341, 197, 123, 282, 245, 172, 358, 151, 289, 8, 66, 345, 271, 311, 25, 125, 100, 86, 352, 166, 151, 121, 92, 380, 7, 218, 288, 338, 239, 113, 224, 202, 70, 268, 262, 170, 40, 368, 61, 46, 256, 163, 244, 292, 63, 28, 265, 24, 195, 351, 348, 38, 219, 371, 318, 302, 119, 191, 236, 265, 24, 134, 153, 240, 132, 103, 15, 354, 62, 305, 227, 43, 137, 320, 179, 78, 130, 145, 72, 383, 184, 96, 205, 10, 242, 181, 320, 301, 138, 374, 212, 122, 230, 39, 131, 322, 30, 188, 319, 5, 310, 210, 284, 306, 106, 41, 254, 139, 373, 260, 270, 319, 5, 301, 19, 356, 223, 296, 327, 140, 326, 4, 157, 349, 319, 141, 144, 124, 342, 149, 3, 315, 13, 185, 34, 88, 283, 319, 320, 101, 304, 148, 98, 142, 74, 332, 159, 303, 14, 52, 319, 320, 79, 99, 204, 189, 60, 59, 50, 65, 17, 84, 89, 108, 5, 164, 269, 272, 198, 87, 346, 313, 257, 168, 361, 253, 48, 5, 309, 211, 259, 250, 107, 112, 362, 307, 158, 281, 295, 221, 5, 31, 154, 11, 128, 116, 71, 64, 350, 200, 44, 291] + 11: [156, 36, 236, 152, 367, 75, 101, 34, 324, 222, 201, 276, 137, 252, 11, 22, 247, 213, 127, 270, 279, 113, 46, 114, 180, 335, 98, 300, 144, 275, 45, 373, 208, 158, 164, 356, 240, 123, 281, 98, 11, 332, 192, 124, 2, 41, 295, 7, 74, 183, 60, 12, 77, 190, 218, 141, 319, 347, 255, 292, 142, 237, 277, 175, 105, 377, 233, 345, 331, 107, 262, 242, 63, 62, 8, 278, 16, 257, 377, 145, 0, 217, 283, 27, 186, 364, 383, 37, 4, 239, 321, 339, 121, 15, 284, 371, 93, 55, 312, 129, 103, 140, 202, 256, 339, 121, 15, 178, 66, 172, 289, 338, 67, 106, 322, 333, 273, 260, 171, 357, 204, 78, 166, 224, 193, 57, 184, 167, 51, 50, 299, 99, 100, 244, 320, 86, 341, 349, 64, 271, 272, 253, 308, 130, 235, 102, 165, 313, 151, 197, 211, 225, 358, 305, 352, 261, 130, 221, 357, 146, 154, 26, 198, 302, 120, 375, 132, 317, 191, 130, 221, 285, 330, 161, 56, 203, 311, 264, 5, 288, 76, 212, 88, 235, 126, 337, 169, 80, 254, 359, 293, 323, 287, 189, 232, 363, 38, 199, 39, 303, 10, 49, 366, 174, 19, 71, 216, 243, 87, 110, 162, 351, 170, 69, 81, 30, 43, 360, 68, 369, 134, 87, 326, 267, 52, 94, 251, 89, 265, 79, 274, 380, 111, 25, 227, 110, 157, 318, 207, 195, 96, 354, 291, 108, 214, 372, 72, 376, 110, 248, 131, 6, 374, 298, 59, 48, 266, 290, 28, 153, 90, 85, 44, 70, 196, 92, 259, 378, 187, 112, 310, 155, 329, 90, 85, 325, 286, 250, 350, 35, 336, 18, 138, 328, 344, 42, 314, 119, 116, 20, 13, 348, 301, 219, 263, 194, 268, 361, 9, 314, 150, 36, 228, 205, 370, 97, 147, 182, 143, 1, 238, 241, 314, 150, 116, 177, 17, 249, 327, 362, 188, 309, 379, 139, 307, 3, 149, 44, 73, 297, 160, 133, 91, 234, 210, 21, 206, 334, 83, 40, 285, 128, 47, 176, 14, 245, 33, 230, 229, 368, 269, 83, 65, 248, 95, 23, 340, 316, 179, 115, 200, 104, 181, 109, 215, 163, 29, 343, 346, 315, 365, 353, 125, 84, 173, 82, 220, 215, 163, 24, 54, 280, 226, 31, 135, 118, 306, 185, 122, 223, 296, 296, 126, 381, 136, 159, 209, 168, 258, 382, 342, 53, 61, 296, 163, 100, 282, 148, 117, 58, 294, 355, 231, 304, 246, 32] + 12: [156, 69, 98, 208, 288, 180, 52, 313, 257, 263, 212, 132, 353, 255, 289, 198, 260, 209, 107, 231, 232, 351, 285, 55, 94, 250, 255, 296, 151, 123, 371, 271, 315, 356, 40, 128, 237, 269, 366, 255, 378, 51, 259, 300, 104, 234, 321, 335, 30, 47, 145, 222, 255, 378, 361, 176, 101, 80, 376, 61, 334, 70, 316, 275, 359, 255, 69, 90, 75, 171, 129, 200, 138, 20, 207, 72, 196, 42, 204, 349, 369, 170, 39, 322, 0, 37, 235, 252, 8, 206, 21, 204, 177, 181, 340, 364, 164, 254, 67, 379, 85, 113, 248, 324, 204, 177, 306, 354, 133, 362, 114, 4, 121, 274, 131, 88, 92, 363, 82, 50, 295, 116, 197, 185, 262, 305, 279, 81, 143, 272, 49, 223, 290, 195, 160, 337, 253, 165, 5, 179, 125, 307, 302, 149, 223, 192, 3, 264, 261, 227, 29, 32, 365, 19, 7, 199, 149, 82, 287, 216, 89, 308, 77, 189, 183, 246, 236, 221, 99, 35, 124, 341, 6, 102, 294, 1, 62, 13, 312, 211, 111, 73, 48, 22, 203, 68, 10, 135, 188, 136, 154, 352, 18, 314, 141, 120, 109, 341, 205, 266, 23, 286, 319, 122, 146, 345, 311, 377, 120, 267, 46, 278, 2, 241, 350, 256, 65, 148, 117, 79, 225, 120, 292, 46, 281, 153, 273, 191, 280, 36, 217, 54, 118, 240, 381, 168, 172, 215, 147, 74, 155, 58, 11, 310, 93, 242, 373, 130, 127, 103, 332, 16, 326, 370, 45, 86, 186, 230, 184, 110, 130, 127, 203, 382, 27, 63, 9, 224, 293, 297, 320, 360, 43, 130, 175, 150, 330, 368, 318, 15, 187, 355, 44, 282, 163, 56, 245, 303, 97, 137, 277, 57, 41, 249, 159, 383, 347, 328, 331, 78, 167, 333, 276, 323, 157, 309, 139, 115, 106, 178, 342, 64, 78, 303, 126, 152, 218, 301, 142, 83, 336, 244, 367, 202, 66, 158, 327, 233, 134, 265, 380, 357, 119, 112, 284, 247, 26, 162, 158, 190, 150, 268, 229, 91, 201, 298, 140, 374, 299, 182, 372, 158, 190, 172, 348, 87, 59, 358, 325, 214, 251, 34, 210, 220, 219, 327, 333, 194, 25, 14, 144, 169, 38, 24, 96, 213, 95, 226, 344, 71, 346, 108, 31, 283, 173, 76, 105, 375, 228, 239, 226, 327, 243, 317, 60, 238, 193, 84, 12, 329, 258, 338, 53, 28, 17, 243, 339, 33, 291, 304, 161, 270, 174, 343, 100, 166] + 13: [80, 21, 310, 109, 40, 10, 344, 357, 127, 5, 85, 20, 314, 80, 135, 366, 171, 363, 2, 183, 76, 220, 119, 182, 156, 120, 9, 280, 186, 46, 266, 122, 327, 221, 345, 190, 205, 148, 367, 44, 172, 340, 162, 320, 178, 129, 380, 56, 104, 285, 200, 82, 44, 280, 12, 118, 16, 43, 121, 267, 133, 360, 191, 258, 204, 362, 282, 30, 307, 356, 211, 228, 337, 250, 70, 212, 264, 7, 362, 51, 12, 35, 202, 207, 283, 89, 325, 19, 1, 97, 324, 194, 41, 238, 176, 188, 339, 374, 105, 53, 270, 28, 278, 175, 329, 169, 110, 131, 246, 375, 239, 153, 257, 365, 167, 47, 351, 329, 24, 95, 227, 112, 321, 359, 331, 94, 13, 343, 272, 165, 137, 312, 274, 304, 143, 98, 253, 303, 311, 72, 215, 371, 350, 77, 312, 201, 198, 376, 79, 144, 326, 259, 61, 138, 302, 206, 170, 23, 30, 116, 289, 237, 225, 68, 348, 73, 101, 293, 27, 170, 39, 209, 323, 297, 240, 262, 248, 268, 313, 157, 317, 216, 170, 260, 342, 58, 75, 382, 29, 52, 305, 103, 233, 255, 322, 170, 260, 189, 295, 31, 355, 196, 288, 244, 38, 117, 332, 126, 170, 265, 231, 184, 54, 147, 107, 163, 308, 358, 63, 90, 140, 108, 328, 189, 111, 369, 141, 151, 37, 50, 115, 284, 349, 33, 108, 328, 100, 279, 273, 168, 25, 161, 381, 124, 372, 341, 301, 108, 23, 276, 49, 69, 179, 26, 286, 296, 235, 83, 354, 160, 57, 149, 238, 45, 364, 62, 361, 36, 59, 377, 60, 99, 155, 232, 87, 318, 335, 292, 373, 139, 222, 319, 249, 180, 181, 78, 232, 87, 318, 370, 158, 378, 177, 241, 174, 353, 185, 368, 193, 232, 149, 55, 64, 275, 219, 145, 379, 245, 234, 336, 132, 218, 81, 87, 330, 298, 334, 208, 146, 15, 84, 150, 226, 224, 192, 18, 195, 134, 88, 281, 6, 256, 125, 136, 214, 4, 290, 247, 18, 195, 271, 199, 91, 22, 74, 48, 113, 217, 203, 347, 32, 18, 195, 197, 11, 67, 299, 243, 164, 242, 287, 252, 251, 315, 3, 86, 102, 338, 294, 210, 93, 8, 71, 154, 128, 152, 230, 114, 14, 17, 142, 309, 65, 291, 106, 92, 130, 346, 236, 277, 114, 0, 274, 159, 261, 300, 333, 223, 263, 213, 383, 269, 66, 96, 14, 254, 306, 229, 42, 166, 34, 187, 123, 352, 173, 316] + 14: [147, 122, 85, 100, 297, 370, 377, 298, 273, 74, 47, 236, 331, 70, 34, 151, 52, 246, 319, 90, 276, 214, 78, 66, 247, 50, 70, 34, 141, 95, 123, 288, 221, 336, 10, 153, 185, 170, 98, 56, 186, 99, 124, 206, 68, 241, 0, 94, 278, 139, 11, 249, 160, 75, 254, 198, 25, 145, 286, 313, 248, 20, 263, 131, 29, 160, 239, 228, 32, 135, 326, 116, 359, 262, 365, 92, 144, 73, 160, 344, 23, 117, 158, 333, 172, 152, 224, 230, 31, 219, 318, 229, 209, 275, 97, 323, 324, 347, 330, 302, 127, 362, 188, 315, 229, 259, 235, 39, 149, 201, 373, 335, 43, 195, 57, 252, 83, 229, 259, 142, 342, 328, 291, 129, 86, 114, 338, 62, 283, 212, 229, 75, 155, 317, 33, 104, 109, 168, 310, 375, 120, 140, 289, 211, 2, 71, 231, 69, 184, 105, 284, 6, 103, 35, 320, 88, 211, 2, 71, 3, 163, 357, 237, 200, 106, 257, 84, 381, 165, 274, 209, 308, 213, 161, 193, 5, 316, 42, 36, 293, 307, 295, 143, 49, 275, 177, 343, 332, 380, 225, 285, 46, 55, 81, 121, 143, 268, 155, 371, 379, 159, 227, 300, 311, 309, 24, 181, 174, 261, 175, 378, 360, 125, 266, 222, 189, 350, 16, 299, 280, 38, 4, 30, 337, 271, 41, 40, 102, 148, 93, 14, 18, 51, 138, 4, 30, 118, 79, 208, 363, 101, 306, 173, 171, 367, 72, 196, 372, 112, 378, 269, 220, 382, 256, 191, 356, 58, 312, 303, 218, 207, 199, 146, 162, 59, 277, 82, 253, 28, 358, 180, 107, 205, 15, 110, 308, 64, 215, 369, 156, 176, 321, 304, 322, 340, 53, 15, 383, 355, 250, 216, 169, 128, 282, 240, 361, 45, 192, 272, 76, 17, 348, 154, 13, 301, 234, 115, 251, 136, 80, 91, 119, 76, 110, 355, 44, 26, 339, 226, 202, 376, 238, 132, 134, 292, 77, 255, 353, 111, 294, 346, 203, 264, 244, 19, 179, 27, 210, 164, 255, 126, 354, 166, 37, 178, 223, 63, 9, 258, 368, 232, 287, 255, 126, 61, 314, 167, 296, 89, 183, 190, 281, 54, 352, 204, 8, 327, 279, 270, 22, 48, 341, 329, 334, 87, 243, 351, 204, 60, 327, 374, 197, 108, 245, 242, 1, 290, 194, 65, 130, 345, 345, 150, 182, 157, 187, 325, 364, 96, 267, 366, 113, 67, 345, 137, 122, 21, 265, 133, 349, 12, 217, 305, 7, 233, 260] + 15: [68, 258, 128, 342, 357, 5, 229, 381, 383, 303, 293, 125, 260, 298, 258, 94, 168, 36, 372, 13, 377, 63, 116, 98, 299, 127, 0, 200, 104, 296, 100, 133, 77, 365, 224, 89, 126, 156, 249, 0, 200, 104, 113, 207, 49, 206, 294, 316, 54, 171, 304, 192, 0, 326, 266, 344, 335, 170, 10, 308, 37, 364, 292, 307, 368, 290, 231, 236, 353, 47, 78, 268, 61, 314, 188, 318, 287, 93, 15, 355, 175, 256, 338, 323, 363, 67, 211, 73, 140, 214, 27, 15, 355, 175, 311, 351, 209, 114, 137, 208, 129, 149, 297, 33, 15, 332, 217, 1, 352, 246, 43, 339, 84, 167, 181, 44, 46, 261, 155, 148, 173, 106, 238, 341, 30, 172, 190, 222, 183, 281, 340, 185, 252, 40, 164, 101, 233, 349, 337, 278, 330, 41, 245, 350, 155, 217, 174, 115, 212, 96, 53, 272, 153, 201, 336, 348, 76, 38, 119, 139, 242, 2, 62, 118, 92, 91, 380, 21, 25, 4, 189, 215, 23, 241, 105, 250, 107, 283, 210, 280, 243, 85, 88, 251, 90, 221, 109, 59, 69, 180, 26, 147, 370, 345, 186, 88, 324, 134, 83, 225, 28, 81, 141, 334, 3, 310, 286, 328, 88, 324, 134, 35, 235, 102, 282, 347, 64, 274, 327, 110, 356, 227, 300, 146, 151, 138, 142, 87, 99, 55, 184, 7, 70, 264, 227, 251, 203, 376, 143, 117, 331, 97, 204, 374, 58, 346, 144, 259, 160, 9, 329, 196, 279, 205, 226, 373, 65, 124, 17, 19, 295, 160, 121, 360, 18, 86, 343, 289, 216, 32, 14, 122, 6, 202, 160, 128, 135, 379, 247, 12, 271, 257, 273, 228, 178, 75, 367, 39, 94, 193, 319, 182, 150, 72, 123, 161, 312, 220, 132, 130, 57, 284, 16, 177, 305, 165, 313, 358, 239, 29, 302, 131, 130, 362, 74, 56, 31, 42, 253, 191, 95, 52, 361, 103, 198, 130, 362, 333, 48, 24, 166, 50, 306, 375, 79, 262, 219, 315, 130, 199, 159, 163, 285, 194, 254, 382, 213, 240, 195, 154, 152, 11, 359, 51, 232, 255, 263, 269, 354, 71, 378, 112, 120, 325, 11, 359, 237, 80, 111, 248, 34, 267, 45, 369, 136, 223, 366, 11, 371, 20, 157, 320, 82, 309, 234, 244, 158, 187, 288, 276, 11, 57, 148, 277, 162, 145, 8, 301, 265, 218, 66, 179, 322, 291, 359, 60, 317, 230, 275, 22, 321, 169, 197, 176, 108, 270] + 16: [11, 90, 171, 383, 97, 159, 357, 242, 244, 284, 138, 12, 1, 162, 129, 255, 265, 52, 41, 334, 358, 48, 381, 32, 6, 98, 100, 64, 250, 364, 314, 184, 137, 221, 207, 157, 176, 173, 362, 100, 64, 24, 287, 306, 219, 297, 252, 266, 166, 241, 192, 10, 140, 269, 78, 209, 307, 66, 122, 60, 253, 107, 230, 23, 239, 140, 49, 302, 283, 2, 281, 19, 225, 360, 14, 371, 278, 161, 134, 144, 205, 235, 365, 8, 276, 158, 337, 110, 216, 204, 361, 134, 147, 75, 212, 335, 282, 356, 214, 91, 172, 382, 79, 3, 134, 147, 259, 123, 82, 80, 322, 228, 55, 324, 169, 355, 257, 134, 269, 280, 89, 115, 155, 202, 336, 190, 128, 245, 350, 368, 15, 165, 348, 236, 321, 182, 372, 340, 272, 17, 116, 263, 191, 15, 165, 5, 46, 310, 34, 96, 290, 363, 88, 217, 291, 345, 15, 86, 280, 108, 71, 39, 229, 211, 83, 145, 243, 40, 320, 15, 273, 118, 199, 328, 31, 232, 95, 304, 113, 256, 326, 13, 68, 338, 271, 87, 316, 73, 160, 264, 30, 186, 28, 63, 148, 286, 7, 224, 189, 177, 231, 293, 104, 201, 127, 279, 327, 69, 286, 43, 374, 213, 65, 70, 106, 25, 323, 131, 359, 72, 296, 286, 43, 35, 187, 294, 227, 370, 132, 343, 168, 50, 299, 215, 286, 277, 271, 270, 99, 58, 174, 354, 193, 62, 251, 74, 94, 369, 312, 274, 298, 27, 167, 124, 208, 267, 218, 260, 130, 111, 369, 7, 183, 121, 84, 305, 9, 198, 240, 373, 332, 197, 53, 152, 92, 342, 85, 237, 311, 379, 313, 109, 163, 288, 135, 126, 152, 92, 203, 346, 38, 56, 42, 18, 206, 117, 308, 222, 133, 67, 262, 75, 347, 21, 175, 22, 151, 142, 233, 352, 254, 247, 330, 262, 289, 226, 377, 119, 339, 200, 93, 47, 210, 112, 344, 125, 275, 318, 180, 51, 0, 45, 54, 223, 20, 149, 329, 309, 331, 275, 118, 16, 248, 153, 220, 178, 319, 26, 170, 179, 376, 185, 275, 303, 367, 37, 103, 238, 29, 378, 268, 150, 341, 366, 114, 234, 303, 246, 292, 139, 301, 375, 196, 315, 33, 164, 156, 154, 234, 143, 146, 59, 380, 36, 102, 105, 261, 101, 349, 249, 120, 120, 44, 295, 181, 195, 188, 57, 333, 258, 77, 136, 325, 120, 300, 44, 141, 285, 76, 4, 317, 194, 81, 61, 351, 353] + 17: [374, 15, 155, 56, 22, 314, 304, 44, 379, 233, 65, 322, 54, 261, 78, 225, 8, 177, 135, 13, 338, 240, 364, 227, 334, 358, 261, 15, 383, 52, 180, 99, 19, 101, 249, 50, 355, 68, 156, 28, 357, 383, 104, 313, 298, 41, 216, 345, 106, 176, 242, 235, 212, 254, 117, 107, 243, 84, 20, 37, 94, 344, 83, 129, 152, 138, 219, 311, 157, 335, 9, 82, 95, 120, 372, 23, 285, 159, 131, 375, 100, 222, 46, 171, 61, 253, 238, 350, 352, 268, 34, 131, 375, 100, 290, 194, 18, 147, 221, 102, 369, 356, 368, 312, 131, 375, 26, 234, 273, 98, 336, 196, 195, 67, 182, 289, 127, 131, 91, 136, 339, 38, 139, 43, 365, 112, 128, 109, 184, 251, 220, 200, 26, 167, 206, 292, 47, 181, 179, 31, 32, 348, 69, 220, 200, 359, 172, 97, 165, 191, 342, 16, 64, 223, 349, 354, 119, 108, 185, 324, 231, 146, 340, 27, 300, 63, 53, 35, 123, 119, 108, 343, 2, 211, 256, 260, 115, 258, 25, 280, 51, 174, 119, 87, 362, 347, 381, 74, 332, 265, 161, 48, 40, 288, 202, 119, 12, 305, 328, 373, 10, 7, 204, 210, 55, 230, 11, 86, 307, 366, 279, 246, 296, 70, 36, 89, 110, 57, 163, 281, 209, 80, 126, 323, 113, 39, 245, 60, 341, 239, 326, 88, 306, 170, 105, 58, 114, 130, 96, 168, 116, 309, 360, 144, 164, 321, 252, 208, 337, 266, 1, 134, 142, 166, 271, 361, 270, 316, 17, 333, 250, 337, 294, 29, 218, 263, 331, 3, 85, 267, 224, 4, 237, 250, 337, 362, 125, 145, 33, 248, 21, 262, 150, 132, 77, 14, 140, 79, 294, 351, 148, 327, 173, 382, 287, 274, 201, 160, 72, 188, 79, 305, 62, 92, 143, 363, 81, 244, 76, 241, 353, 121, 188, 79, 232, 189, 24, 282, 264, 213, 310, 255, 315, 0, 301, 284, 319, 330, 6, 228, 371, 295, 49, 302, 318, 153, 247, 122, 284, 303, 286, 297, 380, 272, 42, 198, 299, 124, 378, 111, 293, 284, 303, 330, 133, 103, 367, 154, 118, 90, 278, 277, 291, 283, 284, 346, 193, 75, 207, 259, 215, 187, 93, 192, 141, 178, 203, 236, 226, 5, 276, 199, 214, 45, 205, 217, 151, 30, 229, 320, 236, 137, 197, 175, 73, 162, 59, 186, 183, 308, 66, 257, 325, 236, 190, 275, 370, 317, 376, 269, 377, 329, 158, 149, 71, 169] + 18: [208, 36, 106, 330, 181, 223, 122, 363, 383, 219, 364, 198, 379, 208, 304, 228, 99, 337, 307, 176, 264, 214, 224, 273, 24, 147, 6, 114, 228, 33, 103, 0, 288, 255, 275, 56, 2, 194, 91, 118, 119, 71, 378, 164, 225, 313, 335, 60, 338, 280, 345, 309, 356, 57, 71, 341, 318, 262, 232, 13, 166, 163, 172, 173, 52, 30, 245, 260, 190, 165, 131, 195, 244, 254, 197, 354, 206, 74, 30, 245, 260, 77, 70, 129, 157, 362, 234, 64, 370, 252, 80, 30, 65, 185, 326, 240, 135, 231, 145, 168, 332, 294, 18, 222, 30, 65, 368, 285, 274, 316, 302, 182, 175, 216, 161, 117, 227, 30, 215, 138, 246, 41, 331, 349, 205, 94, 123, 372, 221, 128, 30, 57, 138, 144, 346, 343, 350, 267, 108, 247, 180, 295, 186, 47, 300, 106, 3, 124, 22, 328, 132, 303, 259, 312, 310, 226, 47, 300, 185, 67, 204, 367, 235, 9, 287, 375, 37, 5, 344, 10, 28, 174, 297, 365, 79, 278, 329, 82, 170, 238, 160, 242, 10, 347, 110, 35, 89, 233, 201, 40, 48, 158, 299, 212, 268, 272, 113, 192, 266, 269, 104, 111, 38, 322, 381, 44, 100, 382, 43, 257, 116, 253, 19, 88, 361, 155, 92, 126, 95, 336, 97, 43, 154, 380, 105, 188, 296, 58, 63, 324, 229, 352, 277, 293, 271, 286, 26, 357, 171, 87, 7, 292, 143, 200, 281, 249, 32, 271, 320, 68, 291, 371, 283, 203, 279, 148, 17, 90, 305, 59, 271, 320, 321, 31, 20, 149, 289, 12, 243, 209, 115, 11, 183, 271, 207, 301, 376, 141, 334, 102, 81, 112, 130, 107, 62, 49, 271, 315, 25, 139, 23, 109, 39, 276, 179, 311, 258, 134, 55, 271, 217, 142, 151, 46, 121, 15, 54, 270, 133, 84, 21, 120, 271, 217, 236, 284, 366, 83, 152, 319, 256, 66, 75, 199, 156, 271, 257, 239, 282, 191, 177, 308, 125, 351, 76, 86, 72, 327, 193, 69, 167, 373, 73, 162, 50, 93, 127, 220, 29, 377, 340, 45, 251, 146, 360, 248, 241, 61, 96, 348, 4, 218, 323, 374, 53, 69, 355, 237, 342, 261, 159, 187, 213, 27, 42, 290, 314, 333, 251, 136, 34, 339, 263, 8, 150, 325, 317, 184, 14, 78, 16, 211, 189, 306, 202, 359, 140, 178, 153, 369, 85, 230, 1, 210, 251, 169, 101, 98, 137, 250, 265, 358, 51, 298, 196, 353] + 19: [187, 279, 300, 159, 53, 6, 306, 80, 242, 247, 201, 102, 196, 187, 180, 312, 35, 365, 167, 347, 3, 140, 37, 124, 282, 50, 187, 180, 8, 79, 46, 83, 249, 237, 168, 14, 290, 151, 250, 141, 126, 101, 58, 147, 182, 215, 10, 119, 63, 269, 43, 145, 243, 183, 207, 153, 135, 156, 345, 356, 200, 324, 224, 0, 316, 190, 313, 216, 354, 246, 172, 277, 331, 362, 59, 210, 360, 81, 190, 313, 373, 117, 335, 121, 260, 129, 82, 232, 51, 205, 355, 190, 322, 197, 275, 238, 76, 157, 65, 298, 28, 112, 327, 245, 211, 291, 307, 33, 1, 195, 325, 213, 32, 370, 136, 344, 375, 366, 330, 115, 334, 36, 16, 95, 108, 137, 181, 214, 163, 254, 366, 330, 115, 296, 116, 49, 244, 333, 343, 99, 150, 382, 263, 366, 161, 307, 38, 310, 283, 336, 380, 139, 105, 231, 55, 276, 24, 288, 363, 154, 134, 204, 194, 372, 326, 351, 252, 5, 378, 24, 288, 27, 218, 166, 221, 125, 148, 191, 361, 304, 56, 93, 24, 338, 184, 303, 278, 272, 273, 47, 309, 103, 348, 78, 109, 24, 338, 113, 164, 295, 170, 346, 23, 176, 174, 357, 104, 229, 339, 288, 328, 20, 352, 239, 287, 45, 341, 235, 106, 72, 305, 111, 188, 323, 193, 128, 257, 18, 19, 209, 208, 118, 199, 317, 179, 255, 66, 349, 222, 138, 84, 177, 22, 186, 367, 236, 350, 206, 70, 64, 77, 13, 29, 2, 26, 321, 94, 289, 69, 175, 48, 120, 312, 189, 171, 173, 4, 202, 212, 234, 381, 220, 12, 256, 91, 101, 264, 158, 270, 67, 75, 281, 294, 123, 248, 98, 256, 120, 219, 97, 285, 251, 40, 62, 383, 149, 162, 261, 107, 74, 52, 240, 89, 71, 96, 144, 160, 376, 315, 178, 308, 133, 7, 377, 114, 301, 122, 127, 143, 60, 225, 73, 267, 17, 142, 7, 377, 64, 227, 30, 368, 374, 131, 130, 340, 358, 169, 274, 7, 241, 240, 230, 15, 165, 110, 284, 87, 258, 100, 299, 369, 302, 364, 207, 271, 233, 353, 259, 262, 311, 332, 11, 226, 34, 265, 57, 318, 9, 292, 68, 297, 280, 329, 371, 21, 155, 319, 265, 57, 42, 152, 54, 359, 203, 293, 88, 286, 185, 379, 198, 90, 90, 192, 44, 41, 337, 31, 85, 320, 132, 39, 86, 253, 90, 61, 42, 217, 25, 228, 342, 92, 146, 314, 223, 266, 268] + 20: [116, 268, 138, 91, 342, 265, 178, 73, 191, 85, 136, 171, 280, 116, 268, 330, 10, 347, 343, 60, 12, 308, 199, 245, 31, 27, 116, 145, 185, 221, 104, 267, 346, 338, 139, 28, 16, 220, 331, 116, 145, 270, 141, 101, 219, 18, 106, 75, 289, 70, 251, 350, 334, 45, 93, 209, 172, 35, 274, 332, 94, 352, 147, 275, 262, 334, 45, 325, 158, 65, 218, 345, 5, 217, 21, 133, 301, 263, 334, 226, 188, 229, 132, 53, 208, 304, 84, 296, 234, 244, 167, 8, 112, 318, 228, 216, 41, 42, 127, 230, 79, 293, 1, 117, 122, 369, 162, 164, 49, 361, 215, 370, 11, 205, 211, 197, 51, 37, 369, 4, 286, 177, 359, 34, 294, 151, 355, 19, 23, 181, 278, 43, 339, 135, 13, 213, 114, 248, 349, 323, 354, 17, 198, 278, 43, 379, 160, 176, 367, 196, 310, 316, 40, 63, 103, 189, 174, 2, 32, 273, 77, 240, 169, 39, 374, 163, 47, 38, 241, 302, 87, 46, 264, 126, 194, 97, 33, 212, 102, 225, 269, 52, 302, 87, 282, 96, 90, 26, 238, 78, 337, 329, 305, 288, 54, 336, 287, 353, 311, 121, 348, 165, 377, 285, 271, 83, 152, 61, 336, 376, 335, 140, 190, 36, 300, 170, 24, 113, 157, 129, 200, 272, 99, 105, 125, 320, 146, 247, 231, 360, 356, 207, 186, 206, 14, 99, 4, 227, 57, 58, 22, 100, 154, 156, 295, 155, 76, 161, 193, 204, 128, 184, 0, 309, 257, 89, 7, 67, 44, 242, 161, 110, 46, 187, 322, 25, 150, 236, 351, 115, 373, 69, 249, 161, 123, 130, 362, 357, 375, 131, 29, 210, 333, 276, 203, 107, 161, 153, 307, 259, 239, 368, 233, 202, 327, 124, 48, 149, 299, 161, 153, 291, 50, 6, 314, 279, 365, 253, 321, 64, 192, 303, 161, 153, 335, 180, 74, 30, 119, 380, 254, 142, 183, 344, 281, 317, 193, 363, 95, 92, 235, 243, 15, 56, 120, 134, 237, 137, 55, 98, 224, 381, 358, 284, 340, 20, 80, 319, 261, 255, 168, 82, 98, 282, 9, 315, 175, 256, 232, 223, 88, 313, 298, 179, 82, 98, 195, 382, 182, 328, 166, 252, 159, 59, 108, 306, 266, 144, 81, 371, 324, 258, 118, 372, 3, 312, 250, 383, 109, 260, 144, 173, 62, 290, 71, 364, 246, 201, 326, 283, 68, 222, 214, 66, 66, 148, 111, 292, 378, 86, 297, 366, 341, 72, 143, 277] + 21: [152, 376, 345, 0, 121, 338, 60, 125, 258, 305, 114, 242, 35, 152, 376, 377, 150, 344, 18, 359, 65, 24, 83, 74, 32, 322, 152, 119, 226, 118, 194, 168, 304, 5, 146, 334, 76, 31, 127, 302, 124, 277, 178, 225, 331, 138, 155, 346, 134, 122, 59, 343, 316, 200, 92, 128, 23, 245, 28, 77, 203, 271, 130, 61, 101, 316, 124, 119, 147, 220, 33, 192, 275, 198, 181, 129, 142, 167, 20, 210, 140, 380, 135, 229, 98, 165, 126, 333, 375, 265, 58, 20, 82, 218, 349, 367, 373, 383, 193, 14, 96, 374, 116, 107, 289, 103, 117, 240, 219, 131, 62, 323, 186, 381, 217, 145, 283, 371, 103, 88, 238, 141, 288, 286, 341, 187, 315, 357, 161, 297, 371, 103, 169, 320, 209, 270, 21, 154, 274, 110, 201, 212, 311, 308, 50, 45, 182, 366, 63, 309, 133, 329, 195, 207, 222, 327, 308, 38, 232, 284, 8, 233, 97, 353, 79, 382, 55, 255, 321, 153, 80, 214, 252, 25, 73, 318, 290, 216, 273, 324, 285, 93, 355, 132, 348, 250, 196, 372, 223, 378, 335, 81, 37, 176, 264, 355, 53, 361, 149, 175, 352, 39, 237, 340, 319, 13, 326, 108, 355, 80, 199, 177, 299, 84, 139, 211, 276, 213, 56, 249, 306, 190, 158, 94, 279, 144, 325, 260, 115, 337, 241, 143, 166, 224, 254, 132, 113, 16, 293, 49, 362, 204, 102, 253, 156, 188, 272, 254, 132, 208, 22, 9, 263, 246, 354, 36, 215, 180, 328, 350, 256, 158, 257, 365, 221, 368, 291, 40, 136, 164, 70, 278, 332, 4, 29, 363, 51, 69, 336, 64, 317, 300, 157, 85, 296, 339, 4, 86, 140, 191, 52, 1, 259, 3, 170, 12, 239, 231, 11, 236, 360, 120, 91, 15, 174, 251, 30, 269, 67, 268, 282, 42, 46, 360, 113, 307, 43, 197, 262, 347, 261, 379, 54, 162, 123, 47, 205, 364, 184, 280, 358, 287, 295, 57, 27, 99, 202, 10, 47, 360, 185, 26, 111, 183, 356, 301, 104, 342, 41, 7, 248, 47, 360, 87, 151, 227, 267, 90, 370, 48, 244, 314, 228, 17, 247, 281, 112, 172, 189, 78, 303, 266, 292, 310, 330, 95, 235, 173, 206, 257, 351, 72, 234, 159, 100, 137, 294, 75, 298, 44, 173, 281, 71, 19, 230, 89, 66, 148, 34, 105, 2, 6, 179, 173, 281, 71, 109, 313, 243, 312, 171, 160, 163, 68, 106, 369] + 22: [371, 288, 112, 317, 161, 366, 233, 241, 279, 186, 150, 146, 360, 254, 337, 118, 58, 60, 361, 98, 155, 46, 182, 344, 167, 135, 254, 164, 169, 240, 174, 82, 131, 180, 262, 246, 5, 140, 143, 254, 242, 139, 130, 195, 40, 105, 256, 202, 30, 257, 231, 154, 254, 288, 230, 153, 264, 347, 380, 377, 96, 219, 162, 66, 292, 316, 337, 263, 55, 76, 191, 99, 335, 215, 334, 255, 277, 247, 281, 224, 177, 50, 67, 258, 47, 322, 265, 137, 291, 203, 2, 281, 224, 74, 352, 83, 156, 71, 45, 300, 134, 19, 243, 28, 33, 38, 110, 345, 222, 175, 133, 373, 138, 152, 296, 295, 250, 216, 115, 73, 188, 80, 63, 320, 269, 26, 274, 127, 284, 253, 216, 42, 183, 111, 100, 18, 248, 22, 356, 196, 104, 368, 79, 204, 207, 201, 218, 306, 374, 87, 65, 103, 35, 223, 84, 142, 51, 132, 183, 285, 8, 358, 341, 54, 364, 294, 168, 56, 313, 198, 147, 4, 252, 68, 48, 309, 81, 77, 85, 354, 114, 192, 9, 328, 144, 229, 106, 109, 41, 238, 271, 62, 171, 121, 325, 126, 328, 37, 348, 372, 340, 15, 0, 314, 129, 302, 166, 10, 343, 276, 94, 381, 116, 206, 72, 11, 97, 36, 69, 293, 95, 343, 328, 184, 338, 375, 220, 39, 236, 237, 187, 88, 304, 359, 332, 330, 94, 272, 17, 226, 225, 78, 211, 179, 308, 172, 205, 278, 261, 376, 7, 123, 53, 321, 382, 136, 266, 23, 21, 323, 278, 261, 110, 357, 259, 122, 59, 299, 13, 145, 128, 158, 157, 331, 261, 178, 70, 125, 57, 319, 199, 379, 282, 212, 163, 217, 209, 24, 178, 149, 365, 113, 1, 124, 273, 52, 342, 234, 267, 209, 24, 376, 351, 193, 349, 210, 327, 290, 190, 31, 197, 307, 181, 141, 93, 383, 214, 228, 12, 227, 353, 120, 16, 86, 280, 181, 148, 20, 251, 173, 25, 159, 301, 298, 151, 90, 107, 315, 181, 148, 20, 311, 336, 49, 305, 303, 208, 232, 326, 43, 101, 221, 141, 329, 333, 378, 3, 165, 32, 286, 91, 64, 194, 287, 221, 141, 29, 362, 312, 339, 324, 268, 108, 119, 370, 270, 289, 221, 141, 235, 117, 369, 170, 367, 283, 89, 160, 6, 213, 346, 34, 260, 44, 355, 185, 75, 239, 189, 275, 92, 350, 297, 310, 260, 260, 244, 61, 27, 14, 318, 200, 176, 102, 363, 245, 249] + 23: [26, 87, 84, 192, 91, 38, 52, 284, 243, 239, 360, 343, 168, 26, 269, 94, 374, 191, 224, 93, 258, 7, 373, 350, 83, 4, 26, 244, 156, 169, 249, 194, 1, 339, 68, 33, 266, 63, 75, 26, 244, 232, 241, 151, 342, 125, 250, 268, 25, 116, 22, 37, 126, 240, 176, 180, 271, 311, 53, 223, 99, 260, 62, 332, 234, 126, 306, 261, 11, 233, 272, 254, 58, 107, 46, 34, 145, 226, 328, 173, 27, 372, 220, 313, 333, 347, 285, 79, 310, 60, 166, 328, 24, 154, 55, 70, 152, 5, 162, 294, 205, 158, 112, 18, 281, 167, 47, 59, 213, 290, 329, 143, 245, 61, 88, 183, 216, 225, 167, 103, 127, 36, 237, 301, 330, 349, 214, 319, 217, 366, 264, 187, 177, 182, 312, 228, 365, 73, 335, 202, 230, 246, 15, 264, 82, 327, 86, 190, 157, 207, 303, 14, 336, 299, 118, 64, 124, 201, 287, 67, 149, 136, 98, 54, 21, 227, 238, 30, 291, 124, 331, 135, 2, 379, 356, 119, 344, 134, 355, 337, 278, 8, 124, 187, 324, 6, 340, 370, 178, 204, 155, 123, 181, 138, 265, 51, 295, 135, 351, 309, 222, 321, 352, 122, 97, 381, 141, 81, 175, 305, 85, 263, 184, 23, 150, 200, 28, 49, 383, 105, 3, 175, 305, 85, 283, 144, 215, 65, 341, 362, 308, 353, 293, 251, 69, 305, 252, 371, 102, 317, 137, 382, 236, 106, 357, 142, 186, 172, 48, 12, 257, 74, 95, 325, 316, 297, 369, 377, 42, 39, 172, 132, 323, 78, 219, 195, 96, 109, 16, 161, 253, 380, 13, 172, 132, 323, 130, 111, 221, 160, 363, 206, 248, 9, 262, 113, 314, 100, 197, 148, 131, 44, 153, 128, 302, 282, 322, 117, 189, 314, 100, 300, 218, 146, 129, 273, 90, 174, 367, 247, 318, 242, 314, 19, 72, 56, 35, 359, 17, 348, 163, 375, 259, 20, 235, 314, 77, 203, 41, 326, 210, 185, 280, 346, 188, 368, 198, 296, 275, 211, 358, 57, 378, 10, 32, 0, 108, 104, 276, 89, 29, 275, 147, 279, 270, 133, 361, 179, 193, 354, 139, 165, 289, 286, 275, 170, 300, 159, 209, 101, 307, 196, 121, 338, 171, 364, 288, 274, 376, 177, 304, 229, 40, 298, 256, 164, 140, 277, 199, 110, 120, 71, 72, 315, 80, 292, 231, 76, 114, 115, 334, 208, 43, 71, 71, 252, 92, 50, 255, 66, 45, 212, 267, 31, 345, 320] + 24: [0, 105, 104, 343, 356, 341, 280, 170, 271, 238, 248, 364, 368, 0, 70, 5, 135, 299, 338, 268, 203, 276, 317, 25, 371, 74, 290, 357, 128, 344, 274, 205, 295, 292, 126, 148, 143, 316, 298, 97, 357, 121, 192, 131, 345, 244, 30, 212, 67, 378, 340, 314, 52, 381, 189, 35, 355, 12, 313, 103, 196, 191, 43, 124, 353, 52, 28, 369, 306, 336, 376, 153, 258, 303, 288, 183, 322, 127, 87, 360, 324, 15, 96, 287, 233, 66, 284, 50, 101, 33, 152, 92, 139, 286, 277, 318, 307, 304, 194, 76, 99, 240, 17, 219, 92, 157, 361, 54, 177, 297, 13, 60, 151, 136, 250, 40, 358, 211, 185, 266, 204, 325, 293, 56, 37, 23, 291, 180, 69, 2, 110, 337, 85, 382, 39, 140, 178, 267, 348, 342, 362, 117, 45, 9, 223, 189, 144, 142, 310, 116, 181, 86, 373, 332, 100, 34, 9, 263, 85, 91, 149, 321, 235, 72, 38, 44, 109, 261, 374, 9, 263, 27, 190, 46, 58, 84, 218, 176, 217, 184, 200, 166, 246, 122, 128, 195, 260, 224, 1, 108, 173, 102, 18, 167, 81, 246, 223, 70, 366, 73, 114, 16, 215, 107, 42, 228, 214, 155, 246, 223, 369, 61, 351, 59, 147, 275, 150, 171, 367, 349, 232, 111, 210, 112, 354, 41, 245, 47, 312, 94, 281, 300, 82, 89, 154, 372, 220, 24, 93, 221, 168, 335, 14, 160, 229, 123, 129, 154, 372, 334, 26, 350, 279, 269, 365, 88, 163, 187, 141, 383, 154, 172, 294, 31, 226, 77, 222, 225, 161, 8, 174, 208, 36, 154, 259, 64, 164, 71, 326, 216, 333, 79, 209, 68, 199, 115, 265, 252, 230, 319, 130, 118, 242, 239, 188, 19, 201, 347, 379, 265, 7, 301, 137, 138, 83, 257, 282, 10, 57, 98, 285, 241, 265, 7, 255, 283, 113, 380, 11, 270, 175, 331, 162, 289, 264, 265, 7, 29, 231, 377, 182, 315, 134, 363, 193, 132, 305, 21, 95, 252, 207, 237, 49, 272, 197, 302, 249, 359, 247, 90, 133, 6, 252, 146, 53, 156, 48, 323, 309, 120, 55, 256, 243, 158, 6, 252, 186, 78, 308, 145, 51, 65, 32, 352, 75, 330, 179, 20, 22, 370, 213, 254, 206, 278, 339, 320, 262, 80, 119, 251, 20, 22, 375, 202, 106, 3, 311, 346, 253, 329, 125, 165, 327, 20, 328, 62, 159, 273, 63, 169, 4, 296, 234, 236, 227, 198] + 25: [315, 107, 284, 154, 197, 211, 316, 355, 17, 167, 322, 198, 194, 315, 107, 284, 136, 234, 225, 350, 116, 275, 183, 41, 217, 220, 315, 2, 29, 302, 306, 336, 132, 133, 110, 34, 155, 374, 263, 315, 25, 29, 120, 256, 218, 290, 134, 271, 157, 224, 26, 121, 315, 214, 309, 287, 373, 347, 295, 58, 248, 177, 119, 141, 267, 315, 214, 0, 18, 209, 348, 130, 370, 74, 320, 188, 285, 277, 269, 23, 376, 368, 153, 97, 357, 69, 223, 70, 64, 108, 43, 269, 23, 77, 230, 99, 279, 184, 365, 163, 22, 1, 360, 158, 269, 372, 327, 381, 341, 143, 187, 11, 238, 173, 293, 241, 106, 269, 372, 192, 207, 56, 261, 37, 328, 51, 292, 264, 363, 123, 269, 39, 52, 294, 252, 115, 304, 274, 243, 362, 142, 42, 195, 98, 61, 73, 318, 240, 321, 259, 201, 127, 86, 349, 90, 165, 139, 162, 232, 296, 89, 135, 330, 14, 193, 124, 174, 301, 239, 139, 332, 62, 342, 84, 359, 345, 24, 300, 208, 40, 47, 13, 65, 78, 333, 59, 76, 343, 15, 202, 109, 35, 48, 71, 129, 228, 276, 5, 102, 323, 196, 312, 31, 53, 96, 46, 148, 170, 228, 276, 190, 33, 54, 140, 68, 150, 159, 226, 175, 50, 6, 83, 375, 145, 380, 80, 227, 308, 257, 166, 250, 137, 12, 151, 258, 30, 7, 305, 366, 114, 221, 212, 200, 311, 371, 182, 206, 247, 180, 160, 8, 126, 291, 199, 339, 353, 281, 205, 57, 178, 247, 180, 273, 313, 367, 20, 319, 67, 251, 87, 16, 329, 255, 168, 282, 235, 66, 60, 369, 278, 331, 186, 36, 265, 88, 44, 268, 282, 280, 38, 138, 103, 55, 144, 249, 283, 118, 169, 303, 298, 282, 112, 378, 231, 161, 185, 28, 337, 93, 3, 335, 156, 379, 72, 113, 131, 215, 352, 203, 149, 340, 45, 128, 219, 382, 379, 72, 246, 100, 9, 191, 237, 310, 361, 210, 236, 272, 19, 379, 72, 62, 351, 253, 307, 213, 297, 111, 117, 63, 286, 326, 379, 101, 152, 346, 222, 242, 81, 254, 260, 75, 95, 172, 233, 379, 282, 192, 262, 377, 4, 383, 356, 79, 364, 289, 266, 270, 334, 317, 10, 32, 244, 176, 354, 105, 27, 21, 245, 288, 338, 82, 317, 171, 314, 229, 164, 324, 94, 179, 325, 344, 147, 91, 299, 85, 49, 216, 125, 146, 204, 181, 358, 122, 92, 104, 189] + 26: [80, 291, 307, 369, 133, 186, 127, 383, 25, 64, 324, 58, 168, 80, 253, 266, 135, 247, 294, 65, 107, 364, 347, 270, 46, 139, 150, 291, 276, 92, 148, 24, 20, 123, 12, 109, 313, 95, 203, 320, 338, 16, 88, 190, 367, 290, 256, 329, 112, 206, 155, 213, 136, 181, 254, 110, 67, 268, 357, 177, 362, 334, 258, 222, 246, 136, 181, 224, 146, 31, 221, 66, 232, 125, 225, 208, 345, 261, 136, 184, 160, 328, 198, 152, 113, 231, 157, 275, 314, 288, 132, 136, 263, 149, 332, 128, 122, 285, 19, 117, 244, 163, 28, 111, 136, 70, 253, 55, 360, 99, 161, 187, 264, 77, 199, 310, 86, 5, 371, 279, 245, 323, 356, 73, 350, 7, 346, 358, 248, 260, 6, 236, 278, 227, 262, 79, 215, 144, 325, 317, 243, 147, 169, 6, 21, 308, 223, 265, 57, 239, 93, 220, 8, 101, 164, 170, 233, 21, 38, 238, 26, 229, 45, 267, 47, 141, 42, 188, 175, 292, 342, 341, 293, 321, 44, 202, 252, 121, 335, 377, 286, 299, 292, 236, 130, 355, 178, 306, 98, 115, 340, 185, 284, 235, 209, 281, 63, 337, 249, 151, 173, 17, 143, 140, 40, 373, 322, 1, 89, 53, 344, 22, 179, 327, 87, 41, 78, 9, 205, 379, 54, 89, 53, 33, 315, 191, 97, 195, 50, 189, 303, 240, 207, 309, 89, 214, 183, 158, 319, 2, 35, 76, 197, 34, 274, 210, 52, 380, 90, 316, 226, 156, 201, 318, 153, 105, 165, 145, 283, 351, 380, 90, 137, 61, 176, 102, 159, 370, 49, 72, 56, 295, 374, 11, 172, 69, 30, 166, 353, 378, 114, 336, 39, 29, 85, 234, 11, 90, 230, 167, 180, 343, 277, 129, 219, 297, 216, 257, 218, 269, 18, 289, 376, 287, 241, 192, 312, 298, 74, 119, 84, 68, 269, 172, 339, 131, 259, 174, 212, 32, 301, 331, 326, 273, 60, 269, 172, 339, 62, 48, 27, 10, 37, 302, 361, 13, 330, 375, 349, 18, 116, 108, 359, 51, 365, 363, 280, 333, 193, 381, 228, 271, 217, 94, 36, 138, 142, 0, 196, 368, 100, 154, 211, 242, 124, 217, 304, 83, 272, 103, 104, 372, 134, 59, 91, 366, 106, 124, 217, 162, 82, 204, 118, 15, 75, 352, 354, 14, 282, 348, 237, 237, 120, 382, 200, 194, 71, 182, 300, 43, 296, 23, 96, 237, 237, 230, 126, 311, 251, 81, 171, 3, 255, 4, 250, 305] + 27: [34, 269, 80, 179, 32, 155, 215, 147, 367, 319, 142, 194, 310, 234, 69, 354, 143, 332, 261, 274, 362, 116, 184, 62, 160, 53, 234, 204, 349, 213, 82, 162, 379, 199, 167, 259, 86, 170, 60, 81, 69, 80, 369, 278, 254, 172, 129, 35, 344, 108, 40, 378, 103, 71, 354, 336, 294, 133, 304, 44, 365, 223, 110, 165, 342, 103, 73, 211, 228, 232, 191, 268, 375, 33, 14, 198, 134, 243, 103, 73, 211, 75, 258, 317, 136, 347, 3, 313, 257, 11, 212, 103, 252, 376, 97, 83, 15, 114, 67, 288, 185, 68, 52, 244, 103, 252, 227, 207, 308, 161, 122, 124, 256, 8, 157, 19, 285, 301, 164, 366, 335, 72, 187, 5, 111, 123, 132, 345, 206, 17, 37, 168, 377, 348, 325, 370, 6, 66, 121, 293, 364, 226, 338, 37, 233, 292, 183, 102, 98, 218, 43, 275, 264, 41, 331, 220, 104, 168, 24, 224, 23, 7, 296, 106, 107, 346, 382, 56, 163, 145, 265, 131, 222, 78, 328, 246, 249, 29, 359, 372, 245, 176, 177, 300, 202, 55, 343, 27, 214, 248, 361, 154, 158, 311, 137, 177, 265, 181, 182, 77, 241, 327, 18, 153, 208, 195, 314, 146, 117, 90, 358, 251, 216, 25, 125, 38, 47, 65, 302, 298, 235, 219, 90, 341, 230, 196, 322, 10, 267, 151, 231, 283, 371, 93, 59, 90, 341, 112, 238, 337, 360, 45, 79, 281, 334, 57, 209, 279, 374, 227, 237, 156, 305, 127, 287, 130, 306, 87, 277, 200, 58, 350, 320, 290, 99, 201, 175, 299, 0, 144, 192, 92, 36, 4, 312, 26, 323, 149, 140, 91, 2, 217, 50, 150, 138, 247, 4, 312, 135, 85, 242, 356, 316, 307, 101, 96, 166, 169, 51, 225, 61, 197, 340, 381, 141, 126, 120, 95, 113, 94, 255, 39, 225, 61, 357, 260, 380, 193, 326, 239, 205, 152, 273, 210, 100, 225, 312, 30, 303, 76, 373, 368, 318, 250, 74, 352, 355, 236, 286, 263, 128, 190, 115, 42, 174, 20, 84, 22, 229, 295, 203, 286, 263, 349, 188, 89, 88, 291, 171, 13, 280, 189, 363, 63, 286, 70, 315, 21, 383, 284, 309, 9, 353, 186, 330, 148, 270, 333, 31, 30, 178, 266, 109, 271, 28, 321, 253, 289, 1, 240, 351, 12, 49, 48, 339, 297, 329, 276, 105, 139, 64, 221, 180, 12, 12, 119, 272, 324, 118, 54, 46, 159, 282, 16, 173, 262] + 28: [4, 233, 253, 137, 285, 308, 121, 81, 366, 330, 267, 65, 11, 355, 172, 113, 320, 135, 216, 118, 208, 24, 331, 275, 189, 40, 161, 162, 272, 75, 240, 43, 206, 143, 74, 149, 111, 45, 173, 161, 243, 374, 236, 220, 315, 282, 82, 15, 351, 47, 181, 115, 161, 243, 357, 108, 32, 316, 184, 64, 34, 117, 101, 202, 150, 161, 155, 37, 166, 163, 193, 335, 322, 169, 213, 123, 94, 190, 161, 195, 224, 3, 67, 273, 244, 327, 382, 144, 232, 360, 298, 312, 162, 7, 219, 289, 247, 134, 21, 104, 246, 8, 87, 180, 0, 302, 307, 191, 167, 19, 290, 159, 211, 225, 17, 309, 336, 248, 302, 321, 227, 160, 20, 256, 171, 334, 284, 72, 313, 338, 201, 97, 214, 368, 361, 182, 365, 98, 84, 39, 85, 124, 207, 239, 120, 321, 223, 130, 217, 46, 295, 73, 372, 373, 304, 54, 310, 176, 286, 116, 63, 125, 265, 175, 230, 292, 231, 164, 215, 196, 306, 197, 51, 383, 340, 260, 314, 151, 31, 192, 187, 257, 168, 255, 99, 228, 235, 367, 78, 23, 332, 287, 277, 126, 10, 168, 306, 99, 59, 90, 221, 179, 148, 12, 131, 303, 212, 18, 77, 255, 25, 158, 352, 56, 183, 305, 342, 323, 29, 371, 60, 250, 185, 241, 165, 100, 271, 264, 106, 296, 345, 347, 319, 353, 250, 185, 241, 5, 261, 209, 112, 381, 297, 119, 76, 141, 86, 250, 152, 286, 204, 129, 349, 14, 358, 294, 218, 174, 350, 156, 250, 226, 25, 42, 157, 33, 91, 69, 270, 38, 234, 186, 359, 344, 317, 197, 71, 88, 376, 27, 49, 377, 109, 139, 83, 132, 344, 185, 93, 58, 200, 105, 245, 61, 36, 266, 140, 102, 379, 22, 317, 122, 325, 339, 13, 142, 249, 154, 79, 259, 145, 138, 324, 203, 52, 95, 28, 238, 194, 6, 136, 364, 35, 291, 279, 283, 70, 52, 378, 337, 278, 178, 177, 66, 110, 146, 198, 311, 26, 328, 288, 346, 370, 380, 333, 205, 210, 251, 229, 262, 329, 26, 188, 103, 50, 299, 16, 222, 274, 170, 293, 276, 62, 147, 26, 70, 107, 128, 41, 114, 1, 44, 363, 348, 55, 53, 301, 92, 153, 30, 369, 242, 48, 300, 362, 258, 80, 237, 318, 343, 92, 153, 107, 199, 269, 268, 341, 375, 133, 89, 252, 254, 2, 356, 356, 30, 263, 326, 127, 68, 57, 280, 96, 281, 354, 9] + 29: [383, 186, 89, 370, 382, 232, 310, 157, 79, 17, 296, 1, 206, 197, 262, 255, 204, 207, 257, 104, 312, 42, 69, 41, 100, 176, 197, 74, 160, 216, 102, 139, 353, 348, 70, 121, 223, 330, 240, 73, 67, 255, 68, 341, 106, 101, 230, 220, 179, 358, 80, 66, 73, 67, 346, 145, 82, 128, 20, 359, 13, 374, 297, 372, 367, 271, 110, 89, 237, 319, 273, 380, 315, 56, 227, 133, 331, 285, 283, 33, 148, 112, 298, 228, 167, 46, 111, 334, 92, 130, 39, 14, 265, 23, 134, 364, 222, 267, 31, 84, 62, 109, 226, 355, 196, 362, 311, 180, 314, 171, 340, 126, 191, 154, 201, 8, 26, 196, 362, 311, 243, 248, 280, 269, 224, 299, 113, 251, 165, 284, 88, 339, 173, 159, 132, 225, 125, 238, 338, 253, 254, 21, 322, 58, 129, 23, 235, 50, 136, 3, 22, 9, 194, 164, 286, 108, 195, 304, 309, 188, 193, 178, 32, 316, 161, 187, 141, 81, 305, 118, 295, 320, 163, 210, 12, 151, 10, 53, 0, 30, 231, 249, 118, 295, 44, 63, 293, 116, 292, 169, 153, 307, 325, 291, 177, 118, 304, 236, 190, 278, 349, 142, 47, 199, 15, 252, 268, 274, 124, 203, 236, 48, 61, 347, 294, 83, 288, 270, 234, 260, 6, 365, 203, 60, 24, 357, 156, 150, 28, 209, 71, 96, 5, 363, 244, 198, 44, 275, 123, 182, 264, 135, 213, 99, 301, 75, 212, 244, 198, 282, 175, 350, 18, 97, 302, 103, 149, 45, 335, 143, 244, 333, 60, 343, 38, 34, 105, 373, 261, 192, 290, 107, 98, 336, 323, 127, 140, 259, 300, 317, 229, 166, 239, 189, 2, 91, 336, 258, 173, 375, 4, 131, 345, 170, 287, 219, 263, 342, 289, 54, 323, 202, 87, 94, 329, 78, 93, 327, 162, 115, 138, 117, 318, 120, 127, 366, 324, 72, 313, 242, 7, 352, 279, 95, 185, 318, 354, 282, 332, 86, 200, 181, 52, 55, 306, 35, 221, 37, 51, 277, 208, 57, 172, 19, 205, 122, 250, 233, 155, 27, 29, 51, 277, 351, 119, 378, 59, 371, 369, 114, 168, 184, 241, 247, 25, 379, 16, 326, 215, 272, 344, 40, 381, 308, 211, 321, 214, 361, 76, 266, 174, 360, 146, 328, 36, 158, 356, 49, 245, 152, 361, 137, 11, 65, 85, 276, 77, 377, 368, 376, 337, 281, 256, 361, 137, 11, 183, 218, 90, 217, 147, 303, 43, 246, 64, 144] + 30: [140, 35, 10, 147, 100, 104, 359, 248, 263, 83, 102, 367, 322, 140, 35, 64, 41, 256, 23, 72, 79, 311, 81, 105, 379, 334, 140, 225, 28, 17, 210, 296, 280, 109, 139, 85, 352, 251, 123, 133, 378, 246, 164, 320, 160, 231, 360, 154, 82, 232, 252, 128, 158, 52, 321, 316, 151, 260, 197, 332, 32, 348, 80, 39, 134, 37, 214, 144, 33, 336, 353, 129, 241, 264, 99, 106, 148, 8, 37, 52, 234, 271, 58, 67, 329, 25, 337, 355, 113, 87, 21, 254, 214, 289, 315, 152, 325, 84, 297, 267, 323, 235, 42, 146, 124, 59, 62, 303, 177, 91, 294, 48, 350, 327, 265, 63, 185, 266, 135, 373, 49, 382, 344, 111, 349, 90, 203, 313, 171, 219, 266, 196, 351, 74, 195, 274, 319, 181, 220, 381, 0, 149, 237, 266, 78, 351, 312, 226, 356, 163, 44, 95, 117, 173, 305, 259, 184, 245, 143, 317, 275, 365, 114, 50, 377, 364, 178, 307, 268, 262, 245, 24, 12, 309, 174, 103, 374, 110, 370, 56, 328, 345, 262, 245, 143, 341, 193, 298, 142, 284, 230, 368, 16, 27, 331, 208, 53, 373, 240, 9, 258, 150, 290, 276, 3, 15, 354, 269, 208, 53, 156, 159, 141, 18, 172, 180, 304, 69, 255, 191, 118, 208, 242, 295, 224, 277, 292, 286, 375, 161, 57, 207, 182, 363, 208, 65, 46, 138, 339, 96, 38, 249, 183, 130, 120, 119, 94, 208, 65, 257, 221, 6, 343, 54, 282, 107, 287, 199, 372, 380, 208, 65, 168, 89, 215, 98, 335, 145, 194, 314, 308, 201, 101, 239, 13, 346, 68, 192, 60, 179, 288, 186, 291, 190, 30, 22, 239, 358, 223, 212, 204, 369, 306, 175, 342, 122, 169, 31, 36, 239, 358, 45, 283, 217, 281, 20, 211, 279, 116, 270, 26, 70, 239, 243, 188, 205, 299, 285, 162, 7, 121, 127, 233, 40, 47, 239, 366, 289, 302, 236, 157, 383, 318, 166, 153, 19, 131, 88, 239, 75, 206, 216, 357, 108, 51, 55, 324, 155, 338, 61, 176, 213, 187, 2, 340, 71, 200, 167, 73, 371, 5, 126, 228, 189, 333, 187, 156, 43, 136, 170, 376, 92, 202, 273, 330, 4, 227, 76, 137, 11, 362, 253, 278, 310, 244, 247, 125, 198, 229, 218, 222, 29, 361, 97, 77, 261, 66, 293, 326, 165, 300, 34, 238, 272, 86, 62, 250, 1, 132, 14, 115, 347, 209, 112, 93, 301] + 31: [248, 144, 227, 208, 189, 74, 312, 291, 371, 367, 216, 359, 341, 320, 144, 39, 378, 296, 54, 15, 237, 43, 178, 327, 230, 13, 125, 335, 328, 170, 106, 40, 97, 29, 1, 139, 280, 161, 151, 128, 278, 76, 171, 33, 23, 374, 110, 228, 309, 300, 199, 117, 128, 278, 96, 155, 236, 2, 46, 51, 177, 315, 181, 27, 154, 128, 326, 76, 60, 82, 366, 222, 273, 323, 340, 235, 174, 200, 331, 239, 227, 34, 290, 232, 307, 84, 91, 101, 64, 94, 165, 265, 239, 330, 253, 223, 231, 314, 8, 351, 298, 52, 357, 347, 201, 115, 87, 186, 244, 277, 383, 354, 204, 345, 127, 30, 104, 201, 115, 87, 206, 249, 169, 118, 45, 103, 47, 193, 18, 89, 68, 364, 317, 283, 269, 66, 190, 362, 83, 251, 113, 311, 120, 68, 133, 150, 209, 111, 182, 123, 285, 48, 42, 135, 86, 343, 32, 116, 268, 349, 138, 105, 217, 305, 352, 157, 234, 377, 148, 192, 143, 219, 146, 136, 176, 369, 271, 275, 129, 381, 163, 67, 221, 292, 339, 140, 254, 65, 310, 195, 224, 308, 7, 179, 71, 241, 143, 70, 188, 28, 38, 284, 49, 130, 240, 109, 218, 149, 100, 379, 261, 58, 380, 243, 318, 183, 14, 337, 372, 114, 246, 100, 121, 147, 72, 258, 245, 41, 196, 281, 375, 53, 301, 180, 0, 252, 185, 173, 93, 322, 336, 247, 6, 24, 44, 35, 172, 0, 252, 185, 325, 152, 137, 303, 11, 272, 168, 197, 313, 229, 0, 26, 70, 164, 334, 122, 288, 73, 194, 85, 99, 358, 10, 0, 26, 187, 75, 112, 63, 21, 16, 107, 226, 79, 184, 289, 0, 26, 95, 333, 78, 368, 77, 260, 264, 210, 321, 297, 5, 19, 158, 267, 22, 353, 205, 302, 20, 376, 262, 282, 124, 132, 365, 158, 69, 56, 348, 175, 256, 361, 198, 306, 382, 167, 212, 3, 324, 363, 62, 131, 57, 213, 141, 238, 263, 344, 370, 207, 159, 279, 215, 360, 286, 81, 31, 12, 266, 9, 156, 242, 90, 159, 166, 250, 355, 276, 134, 287, 4, 220, 259, 356, 257, 294, 159, 166, 25, 338, 214, 55, 316, 153, 346, 160, 191, 332, 203, 159, 324, 363, 299, 102, 98, 225, 142, 50, 80, 274, 255, 329, 36, 233, 261, 61, 145, 17, 202, 108, 59, 342, 126, 295, 350, 233, 233, 270, 119, 211, 88, 373, 92, 304, 162, 293, 319, 37] + 32: [94, 360, 40, 186, 227, 250, 84, 24, 210, 264, 118, 112, 316, 94, 122, 61, 348, 91, 324, 249, 337, 243, 73, 220, 192, 379, 63, 163, 275, 319, 298, 281, 183, 309, 311, 44, 52, 93, 162, 63, 51, 41, 75, 189, 25, 45, 158, 42, 153, 334, 272, 167, 63, 51, 166, 96, 369, 213, 382, 378, 102, 365, 23, 260, 312, 63, 8, 344, 184, 345, 26, 182, 193, 221, 356, 377, 159, 326, 63, 280, 344, 283, 306, 175, 237, 139, 110, 160, 149, 226, 333, 63, 66, 164, 211, 294, 111, 29, 315, 191, 197, 216, 98, 177, 218, 200, 166, 202, 266, 332, 59, 77, 284, 238, 46, 321, 373, 218, 163, 107, 126, 7, 50, 142, 198, 99, 341, 133, 242, 262, 288, 370, 38, 233, 308, 199, 286, 10, 154, 0, 196, 267, 114, 148, 130, 164, 279, 289, 231, 336, 22, 172, 67, 355, 328, 145, 148, 21, 38, 143, 252, 85, 39, 4, 152, 222, 137, 358, 265, 352, 229, 322, 65, 254, 269, 32, 340, 320, 305, 170, 338, 173, 352, 310, 322, 89, 297, 157, 230, 317, 34, 270, 188, 318, 90, 352, 271, 329, 225, 171, 64, 293, 62, 9, 248, 100, 215, 251, 155, 241, 16, 204, 72, 239, 115, 247, 43, 195, 180, 17, 383, 33, 190, 214, 359, 325, 178, 70, 363, 82, 330, 278, 295, 14, 33, 241, 275, 380, 212, 48, 3, 129, 299, 376, 76, 368, 375, 246, 190, 329, 174, 313, 28, 86, 349, 307, 105, 1, 95, 296, 234, 347, 342, 263, 168, 206, 253, 372, 282, 134, 343, 290, 57, 135, 347, 107, 151, 58, 71, 53, 381, 136, 374, 224, 74, 123, 268, 285, 219, 156, 19, 55, 81, 80, 124, 117, 31, 18, 244, 346, 285, 257, 140, 120, 354, 351, 366, 339, 119, 79, 194, 68, 367, 106, 274, 97, 357, 104, 350, 6, 132, 304, 259, 217, 181, 367, 92, 2, 78, 113, 36, 176, 302, 125, 128, 60, 161, 245, 367, 92, 12, 83, 301, 314, 150, 236, 144, 201, 203, 235, 54, 367, 323, 121, 258, 232, 303, 371, 205, 287, 20, 255, 207, 15, 367, 179, 122, 331, 273, 292, 240, 276, 103, 13, 364, 47, 300, 49, 187, 327, 116, 335, 69, 127, 37, 5, 35, 138, 11, 27, 108, 209, 327, 87, 228, 101, 256, 56, 361, 261, 208, 291, 147, 146, 362, 141, 131, 109, 185, 353, 30, 165, 223, 277, 88, 169] + 33: [86, 115, 267, 206, 208, 16, 45, 249, 70, 10, 242, 306, 277, 167, 107, 307, 89, 268, 354, 50, 114, 67, 76, 291, 302, 282, 91, 297, 179, 37, 227, 109, 216, 71, 350, 286, 312, 296, 189, 91, 343, 82, 361, 237, 138, 347, 364, 285, 128, 341, 374, 95, 91, 343, 342, 89, 133, 232, 157, 3, 336, 181, 118, 99, 357, 328, 81, 314, 164, 308, 351, 156, 15, 56, 238, 247, 269, 100, 51, 81, 73, 266, 192, 202, 256, 345, 366, 135, 315, 171, 102, 240, 96, 267, 5, 356, 187, 80, 279, 295, 225, 78, 46, 272, 140, 319, 94, 322, 311, 317, 262, 105, 53, 136, 363, 7, 143, 176, 319, 348, 72, 40, 355, 22, 369, 248, 11, 119, 2, 265, 75, 319, 68, 139, 289, 211, 382, 49, 52, 298, 175, 250, 142, 145, 379, 125, 352, 334, 18, 104, 371, 113, 346, 178, 274, 69, 320, 325, 94, 146, 9, 205, 196, 152, 261, 85, 188, 47, 0, 260, 148, 241, 84, 222, 144, 55, 122, 166, 182, 59, 97, 293, 260, 148, 358, 201, 217, 4, 123, 330, 281, 300, 98, 359, 239, 63, 231, 147, 108, 288, 116, 209, 43, 66, 74, 62, 309, 264, 259, 36, 307, 226, 131, 93, 246, 213, 271, 221, 165, 92, 287, 177, 36, 154, 270, 150, 174, 124, 13, 110, 207, 224, 284, 129, 177, 36, 154, 21, 87, 90, 383, 215, 24, 316, 326, 275, 220, 339, 380, 299, 233, 30, 303, 19, 137, 65, 23, 134, 44, 77, 339, 229, 301, 335, 151, 290, 197, 292, 28, 6, 244, 228, 223, 339, 229, 301, 255, 338, 254, 375, 58, 103, 20, 353, 313, 263, 331, 380, 39, 204, 218, 305, 370, 340, 324, 243, 332, 294, 83, 280, 64, 251, 54, 160, 126, 367, 212, 29, 130, 257, 172, 159, 191, 64, 251, 48, 117, 149, 79, 170, 26, 252, 199, 195, 333, 38, 31, 299, 273, 33, 235, 163, 180, 327, 27, 310, 304, 12, 38, 42, 283, 127, 155, 173, 276, 368, 101, 318, 378, 214, 372, 38, 158, 258, 376, 25, 14, 200, 88, 41, 168, 120, 321, 34, 38, 323, 258, 190, 210, 230, 198, 106, 349, 111, 365, 344, 32, 38, 121, 115, 141, 1, 61, 360, 186, 219, 381, 245, 57, 185, 377, 153, 283, 169, 193, 329, 162, 60, 112, 132, 184, 203, 236, 362, 362, 183, 17, 35, 8, 373, 234, 278, 194, 337, 253, 161] + 34: [372, 218, 25, 6, 184, 136, 50, 301, 374, 310, 90, 281, 189, 382, 218, 223, 300, 200, 217, 241, 298, 221, 265, 9, 351, 193, 366, 248, 321, 78, 210, 183, 367, 215, 219, 228, 171, 96, 341, 366, 248, 243, 11, 356, 166, 27, 42, 105, 380, 370, 350, 309, 108, 340, 178, 99, 373, 60, 102, 103, 64, 307, 26, 303, 273, 108, 340, 360, 379, 58, 199, 169, 154, 85, 331, 285, 94, 7, 108, 354, 40, 284, 225, 336, 346, 358, 23, 233, 212, 355, 202, 108, 304, 362, 271, 81, 368, 5, 230, 275, 86, 39, 188, 198, 29, 340, 31, 8, 144, 137, 204, 113, 98, 75, 173, 234, 121, 337, 15, 291, 79, 91, 133, 48, 318, 142, 28, 52, 130, 381, 337, 15, 167, 295, 134, 143, 227, 127, 261, 49, 57, 163, 14, 69, 158, 61, 2, 87, 67, 236, 283, 139, 269, 257, 43, 211, 69, 15, 203, 238, 153, 206, 73, 114, 383, 165, 205, 338, 213, 207, 239, 148, 83, 162, 70, 181, 371, 46, 282, 1, 71, 229, 47, 306, 61, 97, 124, 251, 150, 323, 147, 274, 342, 252, 377, 237, 311, 185, 267, 107, 197, 263, 308, 13, 305, 22, 54, 224, 151, 306, 190, 335, 84, 175, 170, 140, 192, 93, 260, 376, 32, 259, 82, 240, 191, 326, 313, 35, 232, 314, 161, 112, 120, 76, 259, 306, 293, 92, 53, 24, 89, 80, 296, 117, 176, 182, 177, 324, 334, 258, 246, 12, 123, 333, 180, 152, 255, 242, 276, 129, 299, 249, 315, 220, 110, 172, 359, 164, 268, 100, 17, 106, 66, 299, 334, 146, 347, 329, 37, 0, 63, 264, 286, 253, 216, 119, 357, 249, 138, 59, 21, 280, 231, 10, 149, 111, 141, 292, 365, 179, 16, 44, 168, 208, 195, 287, 322, 266, 34, 250, 369, 375, 179, 16, 44, 65, 95, 328, 270, 62, 56, 262, 20, 128, 132, 179, 16, 145, 41, 290, 187, 72, 19, 272, 109, 131, 344, 160, 179, 249, 3, 77, 125, 245, 30, 277, 51, 226, 297, 38, 343, 332, 16, 3, 4, 244, 33, 209, 157, 186, 319, 278, 320, 235, 126, 330, 196, 222, 174, 159, 116, 348, 68, 101, 55, 349, 279, 126, 330, 196, 254, 302, 256, 247, 361, 289, 36, 104, 156, 363, 18, 135, 190, 288, 345, 327, 364, 352, 88, 353, 339, 325, 122, 312, 74, 194, 115, 201, 317, 214, 294, 45, 118, 155, 378, 316] + 35: [48, 286, 3, 194, 266, 213, 337, 166, 376, 22, 232, 119, 251, 309, 133, 113, 326, 14, 215, 261, 145, 315, 317, 238, 351, 177, 363, 68, 281, 335, 162, 294, 350, 277, 378, 259, 354, 226, 341, 363, 133, 113, 349, 126, 93, 125, 204, 301, 207, 195, 242, 124, 80, 68, 281, 164, 331, 372, 57, 109, 6, 35, 101, 178, 200, 75, 318, 289, 307, 52, 104, 320, 273, 7, 29, 306, 280, 71, 75, 314, 322, 234, 191, 30, 302, 348, 17, 65, 139, 83, 13, 353, 295, 31, 355, 134, 5, 209, 216, 142, 151, 165, 225, 381, 220, 295, 276, 340, 360, 356, 235, 8, 338, 248, 255, 310, 284, 45, 67, 38, 239, 102, 61, 122, 82, 380, 370, 121, 252, 146, 45, 367, 203, 340, 160, 128, 12, 106, 73, 176, 183, 223, 304, 45, 367, 276, 346, 267, 325, 32, 369, 217, 198, 10, 271, 343, 169, 345, 72, 186, 163, 132, 170, 366, 84, 365, 167, 333, 228, 169, 345, 322, 224, 373, 36, 95, 34, 269, 149, 94, 51, 229, 260, 368, 72, 78, 275, 308, 90, 264, 196, 135, 89, 258, 59, 28, 297, 342, 231, 140, 244, 11, 202, 43, 103, 148, 197, 40, 28, 345, 21, 41, 37, 328, 42, 382, 285, 63, 77, 193, 85, 44, 297, 55, 265, 222, 190, 201, 339, 66, 105, 263, 233, 283, 246, 56, 138, 364, 136, 268, 137, 290, 99, 316, 26, 123, 100, 182, 1, 86, 154, 321, 9, 256, 91, 19, 161, 49, 188, 64, 205, 211, 324, 375, 159, 192, 143, 347, 175, 292, 358, 33, 199, 205, 131, 53, 50, 362, 334, 74, 214, 179, 377, 250, 336, 236, 240, 131, 53, 241, 361, 108, 92, 20, 374, 189, 262, 0, 120, 62, 1, 153, 352, 312, 171, 300, 117, 127, 156, 155, 208, 2, 237, 1, 129, 147, 157, 230, 287, 227, 25, 274, 107, 69, 293, 144, 254, 4, 15, 98, 81, 16, 305, 88, 79, 150, 270, 359, 130, 379, 86, 221, 168, 296, 96, 180, 279, 332, 97, 327, 60, 319, 299, 203, 206, 187, 272, 172, 111, 344, 87, 288, 219, 118, 319, 218, 141, 114, 27, 158, 152, 330, 70, 253, 23, 371, 291, 319, 218, 257, 311, 54, 298, 243, 383, 184, 245, 181, 210, 76, 319, 303, 47, 46, 329, 282, 313, 24, 174, 278, 249, 247, 212, 58, 299, 342, 116, 357, 18, 323, 110, 185, 115, 112, 39, 173] + 36: [53, 260, 301, 73, 191, 3, 171, 157, 244, 248, 35, 4, 160, 53, 230, 372, 287, 246, 122, 177, 187, 18, 44, 87, 26, 360, 131, 299, 12, 152, 55, 284, 195, 323, 58, 329, 126, 161, 375, 192, 272, 107, 352, 139, 68, 41, 143, 149, 173, 60, 181, 367, 252, 101, 223, 374, 349, 298, 147, 217, 29, 70, 324, 27, 145, 294, 272, 12, 133, 319, 91, 228, 344, 263, 198, 215, 222, 214, 22, 311, 380, 308, 166, 50, 179, 257, 119, 259, 226, 203, 25, 22, 311, 99, 150, 345, 289, 346, 206, 88, 169, 118, 136, 268, 22, 272, 225, 92, 67, 361, 102, 0, 318, 77, 316, 306, 377, 197, 328, 372, 5, 288, 183, 72, 335, 148, 265, 120, 353, 376, 370, 368, 373, 291, 94, 342, 140, 211, 135, 209, 8, 7, 302, 382, 368, 225, 47, 262, 103, 66, 310, 356, 295, 355, 285, 313, 74, 368, 56, 205, 111, 13, 364, 38, 351, 19, 337, 269, 43, 340, 325, 223, 97, 293, 172, 366, 305, 146, 338, 137, 82, 234, 340, 325, 107, 275, 282, 95, 100, 264, 159, 190, 115, 154, 336, 176, 371, 90, 207, 49, 164, 271, 81, 358, 249, 357, 286, 113, 341, 371, 369, 54, 162, 339, 280, 52, 220, 109, 61, 34, 270, 341, 371, 274, 332, 334, 63, 116, 104, 243, 281, 378, 96, 279, 184, 359, 40, 79, 231, 105, 11, 76, 23, 189, 254, 128, 290, 184, 59, 320, 31, 240, 15, 174, 255, 48, 6, 14, 227, 292, 184, 2, 178, 32, 112, 132, 202, 212, 80, 158, 110, 327, 16, 144, 10, 210, 45, 219, 69, 153, 36, 267, 251, 256, 106, 221, 144, 10, 258, 65, 9, 121, 237, 39, 78, 204, 312, 303, 37, 84, 62, 296, 42, 108, 167, 277, 180, 51, 247, 350, 1, 127, 57, 266, 193, 314, 33, 20, 168, 229, 365, 236, 21, 218, 182, 57, 185, 178, 261, 362, 297, 224, 30, 163, 315, 307, 89, 317, 250, 266, 326, 85, 175, 155, 117, 134, 239, 363, 235, 86, 123, 304, 238, 241, 233, 333, 276, 216, 71, 208, 24, 213, 151, 170, 304, 238, 193, 125, 309, 245, 343, 379, 188, 199, 130, 124, 186, 348, 232, 242, 93, 201, 322, 142, 28, 129, 383, 75, 330, 64, 232, 232, 98, 331, 194, 138, 46, 200, 283, 83, 253, 278, 273, 232, 232, 196, 156, 321, 300, 354, 17, 165, 381, 347, 141, 114] + 37: [109, 363, 72, 185, 131, 19, 285, 121, 274, 216, 308, 246, 358, 109, 366, 79, 253, 153, 176, 143, 100, 154, 138, 70, 180, 375, 188, 200, 79, 360, 32, 127, 135, 312, 94, 350, 63, 28, 226, 240, 200, 342, 302, 18, 370, 329, 337, 280, 260, 142, 119, 8, 268, 222, 193, 47, 330, 104, 278, 136, 172, 160, 214, 300, 120, 268, 199, 290, 245, 118, 258, 108, 262, 317, 215, 96, 227, 288, 268, 46, 305, 353, 194, 196, 141, 167, 249, 13, 107, 11, 251, 319, 222, 44, 7, 93, 270, 333, 49, 351, 266, 106, 281, 352, 301, 52, 190, 22, 117, 144, 338, 206, 42, 303, 146, 326, 383, 301, 205, 48, 88, 233, 21, 92, 83, 354, 60, 81, 381, 25, 71, 293, 272, 182, 181, 122, 134, 347, 97, 137, 283, 89, 238, 364, 293, 179, 252, 343, 208, 376, 80, 284, 175, 339, 186, 276, 66, 293, 362, 14, 204, 323, 82, 380, 24, 91, 184, 155, 75, 40, 314, 292, 344, 192, 27, 161, 247, 5, 110, 225, 2, 50, 40, 151, 372, 327, 328, 43, 39, 87, 298, 124, 57, 220, 189, 40, 125, 341, 165, 235, 1, 217, 244, 271, 115, 295, 367, 68, 40, 306, 289, 65, 147, 365, 304, 356, 26, 254, 130, 219, 99, 40, 178, 366, 318, 139, 203, 73, 55, 325, 286, 374, 275, 23, 294, 148, 349, 171, 133, 316, 3, 69, 4, 53, 357, 261, 236, 294, 148, 162, 114, 345, 76, 111, 373, 212, 291, 35, 20, 56, 294, 213, 162, 140, 164, 287, 15, 334, 105, 152, 62, 202, 239, 218, 158, 113, 59, 145, 173, 30, 331, 210, 256, 231, 369, 209, 218, 265, 113, 223, 336, 149, 36, 126, 187, 10, 166, 177, 112, 218, 348, 382, 0, 315, 64, 296, 207, 38, 17, 248, 197, 132, 218, 242, 95, 9, 163, 320, 34, 37, 33, 170, 191, 12, 282, 218, 309, 371, 355, 277, 85, 103, 255, 313, 183, 297, 159, 377, 218, 311, 234, 129, 257, 229, 29, 269, 102, 332, 198, 211, 346, 218, 148, 349, 156, 221, 228, 201, 98, 195, 169, 310, 90, 232, 273, 158, 174, 16, 359, 51, 264, 224, 267, 368, 324, 250, 67, 123, 243, 77, 361, 322, 6, 61, 128, 241, 378, 101, 45, 263, 379, 58, 340, 86, 157, 74, 41, 335, 116, 299, 237, 54, 31, 379, 58, 340, 0, 150, 168, 230, 321, 279, 78, 259, 307, 84] + 38: [282, 285, 39, 217, 275, 45, 349, 305, 131, 80, 299, 18, 60, 10, 354, 230, 92, 355, 36, 323, 353, 139, 294, 343, 232, 185, 4, 366, 293, 381, 352, 307, 88, 174, 15, 47, 70, 207, 38, 158, 263, 62, 63, 14, 87, 342, 376, 173, 273, 42, 198, 312, 158, 366, 62, 379, 147, 58, 151, 180, 337, 347, 112, 336, 243, 187, 30, 277, 119, 108, 208, 367, 84, 302, 186, 71, 209, 52, 6, 51, 177, 171, 146, 211, 276, 242, 81, 100, 326, 83, 330, 6, 8, 49, 130, 107, 68, 246, 26, 250, 261, 157, 1, 140, 6, 86, 333, 344, 286, 227, 229, 148, 195, 295, 357, 128, 339, 6, 30, 154, 267, 97, 85, 48, 31, 214, 283, 138, 153, 340, 223, 51, 89, 373, 361, 321, 368, 356, 238, 311, 236, 163, 237, 93, 292, 333, 169, 16, 72, 156, 76, 296, 120, 335, 127, 162, 301, 201, 89, 317, 137, 9, 136, 28, 200, 104, 65, 196, 133, 301, 284, 220, 166, 90, 358, 256, 181, 152, 252, 316, 365, 121, 301, 284, 32, 244, 99, 105, 57, 272, 2, 11, 320, 160, 304, 301, 372, 194, 77, 338, 67, 34, 289, 371, 44, 12, 20, 101, 301, 372, 309, 206, 329, 271, 188, 141, 122, 205, 290, 132, 191, 213, 201, 114, 135, 327, 95, 96, 306, 79, 297, 363, 175, 210, 287, 124, 154, 268, 331, 239, 111, 117, 172, 25, 164, 22, 270, 378, 280, 277, 54, 199, 73, 298, 260, 313, 351, 183, 116, 64, 378, 124, 114, 253, 234, 69, 179, 190, 360, 35, 247, 167, 98, 50, 280, 248, 257, 53, 21, 7, 5, 43, 375, 254, 377, 159, 369, 269, 115, 300, 319, 324, 143, 241, 182, 13, 55, 314, 251, 178, 269, 383, 328, 225, 29, 37, 228, 74, 82, 91, 281, 161, 348, 332, 126, 279, 233, 193, 216, 134, 310, 382, 24, 110, 370, 345, 123, 274, 278, 125, 224, 308, 41, 235, 374, 3, 259, 380, 221, 203, 189, 27, 75, 359, 264, 33, 40, 204, 240, 218, 318, 221, 203, 189, 145, 219, 262, 184, 149, 78, 215, 192, 315, 346, 266, 118, 126, 0, 106, 59, 142, 170, 334, 168, 102, 291, 46, 266, 94, 226, 17, 341, 150, 255, 258, 103, 61, 202, 350, 325, 266, 203, 322, 19, 222, 144, 109, 265, 129, 231, 113, 303, 176, 245, 66, 383, 212, 288, 362, 165, 197, 23, 56, 364, 155, 249] + 39: [43, 218, 2, 296, 118, 31, 5, 293, 309, 41, 347, 262, 356, 100, 13, 382, 297, 53, 251, 179, 61, 130, 288, 211, 67, 90, 176, 139, 379, 362, 170, 248, 301, 182, 254, 111, 55, 105, 11, 247, 312, 85, 329, 352, 191, 249, 302, 181, 166, 229, 219, 29, 247, 312, 292, 145, 274, 232, 226, 373, 240, 217, 0, 351, 44, 247, 215, 187, 277, 146, 336, 276, 320, 79, 189, 266, 73, 332, 99, 65, 86, 270, 383, 340, 213, 72, 220, 192, 18, 158, 148, 282, 315, 267, 172, 375, 239, 367, 357, 60, 120, 92, 339, 209, 282, 315, 119, 374, 84, 225, 199, 241, 243, 222, 157, 308, 57, 282, 315, 155, 333, 8, 127, 345, 212, 377, 70, 246, 124, 261, 282, 65, 313, 330, 87, 34, 353, 285, 214, 376, 359, 233, 321, 96, 113, 325, 300, 7, 101, 281, 88, 128, 335, 142, 349, 334, 280, 56, 244, 174, 42, 140, 162, 33, 207, 343, 238, 284, 21, 35, 56, 85, 80, 259, 48, 161, 134, 169, 171, 37, 147, 165, 180, 56, 328, 324, 200, 323, 307, 311, 363, 27, 20, 294, 361, 268, 151, 279, 250, 59, 138, 358, 193, 68, 109, 252, 350, 304, 264, 286, 313, 141, 81, 331, 150, 185, 230, 287, 160, 125, 9, 66, 153, 314, 17, 23, 47, 152, 354, 265, 91, 197, 129, 52, 66, 156, 314, 98, 298, 122, 366, 106, 221, 117, 46, 318, 186, 66, 286, 36, 295, 202, 278, 135, 289, 164, 159, 78, 224, 115, 110, 25, 132, 15, 77, 137, 50, 97, 338, 107, 337, 12, 154, 116, 1, 36, 40, 273, 112, 62, 133, 6, 103, 83, 205, 201, 258, 236, 272, 71, 227, 299, 75, 14, 327, 245, 10, 255, 22, 258, 368, 271, 26, 381, 28, 364, 45, 275, 346, 231, 256, 89, 234, 236, 242, 121, 235, 54, 63, 94, 195, 19, 305, 237, 269, 104, 372, 93, 178, 163, 131, 144, 355, 114, 283, 253, 326, 149, 104, 372, 360, 24, 3, 260, 317, 228, 183, 291, 39, 342, 198, 104, 236, 93, 369, 49, 190, 371, 167, 16, 303, 206, 32, 344, 82, 372, 38, 319, 365, 74, 188, 95, 4, 196, 175, 370, 257, 310, 310, 379, 204, 290, 263, 126, 203, 194, 216, 64, 210, 143, 310, 223, 132, 341, 51, 69, 177, 173, 30, 168, 108, 316, 184, 310, 348, 38, 76, 208, 378, 322, 102, 306, 58, 123, 136, 380] + 40: [31, 75, 164, 232, 321, 148, 82, 371, 6, 121, 285, 152, 42, 31, 98, 88, 254, 11, 193, 377, 327, 73, 256, 111, 234, 72, 31, 190, 226, 251, 136, 302, 290, 264, 131, 310, 203, 245, 249, 31, 190, 8, 81, 219, 262, 41, 323, 118, 116, 173, 354, 144, 31, 101, 156, 74, 271, 382, 110, 29, 293, 383, 230, 252, 343, 31, 62, 128, 90, 284, 259, 316, 237, 37, 13, 16, 24, 194, 132, 367, 153, 369, 211, 145, 263, 243, 350, 49, 368, 272, 92, 379, 373, 346, 58, 143, 239, 364, 329, 147, 270, 180, 154, 191, 317, 378, 202, 97, 296, 137, 53, 177, 85, 288, 231, 267, 225, 317, 67, 247, 228, 341, 188, 103, 294, 238, 186, 119, 352, 240, 357, 378, 107, 235, 326, 370, 299, 93, 60, 135, 218, 87, 20, 207, 55, 26, 198, 336, 46, 365, 195, 19, 124, 77, 176, 312, 207, 378, 33, 351, 362, 307, 309, 330, 301, 266, 242, 109, 127, 340, 175, 215, 3, 129, 229, 150, 282, 201, 14, 34, 342, 69, 280, 175, 33, 258, 10, 196, 189, 339, 36, 122, 325, 313, 331, 319, 358, 108, 28, 166, 30, 138, 333, 18, 184, 22, 114, 335, 319, 358, 108, 91, 253, 278, 130, 120, 47, 102, 295, 157, 353, 319, 175, 283, 9, 61, 308, 43, 167, 139, 361, 163, 45, 146, 183, 71, 21, 311, 80, 38, 306, 338, 314, 268, 40, 255, 105, 183, 71, 21, 275, 209, 205, 355, 155, 76, 133, 106, 4, 220, 260, 204, 265, 1, 162, 199, 2, 7, 241, 141, 134, 39, 83, 15, 328, 161, 372, 63, 12, 244, 324, 261, 276, 182, 298, 300, 15, 328, 226, 332, 277, 216, 151, 192, 51, 125, 217, 315, 126, 15, 359, 337, 347, 322, 236, 57, 206, 233, 23, 113, 292, 123, 250, 68, 297, 100, 303, 56, 79, 48, 304, 50, 117, 86, 214, 250, 68, 160, 224, 168, 227, 65, 318, 222, 84, 174, 112, 248, 286, 68, 52, 159, 185, 200, 99, 171, 273, 32, 140, 172, 376, 349, 142, 279, 59, 363, 44, 187, 281, 291, 78, 257, 381, 344, 349, 142, 346, 208, 89, 269, 95, 305, 375, 115, 35, 380, 5, 360, 70, 179, 210, 274, 221, 94, 165, 246, 212, 104, 96, 197, 360, 213, 179, 17, 289, 334, 25, 287, 345, 348, 66, 223, 170, 169, 181, 158, 64, 27, 54, 178, 356, 366, 0, 374, 320, 149] + 41: [205, 86, 119, 334, 318, 55, 340, 43, 173, 207, 175, 106, 281, 305, 30, 124, 172, 218, 345, 242, 66, 99, 65, 375, 341, 291, 122, 30, 368, 314, 219, 308, 210, 101, 123, 77, 14, 110, 120, 321, 150, 327, 12, 326, 93, 156, 130, 10, 68, 73, 177, 228, 321, 57, 119, 302, 239, 349, 266, 107, 75, 78, 231, 121, 152, 185, 255, 76, 136, 163, 70, 138, 153, 15, 143, 17, 62, 216, 49, 249, 92, 187, 243, 56, 117, 18, 192, 352, 47, 267, 309, 49, 190, 215, 358, 25, 134, 330, 290, 41, 381, 373, 88, 333, 49, 190, 97, 364, 36, 252, 51, 299, 33, 72, 182, 362, 382, 269, 184, 159, 347, 286, 64, 8, 181, 154, 100, 245, 323, 296, 269, 184, 159, 206, 35, 125, 258, 213, 244, 234, 195, 372, 118, 298, 374, 96, 7, 126, 102, 339, 295, 359, 38, 98, 109, 203, 298, 90, 215, 111, 194, 336, 328, 186, 59, 165, 29, 343, 31, 283, 374, 293, 34, 23, 26, 112, 246, 89, 179, 274, 2, 178, 283, 374, 285, 79, 353, 116, 288, 338, 232, 180, 67, 157, 247, 87, 212, 275, 279, 42, 32, 268, 131, 292, 105, 147, 346, 169, 367, 307, 275, 280, 53, 241, 237, 208, 146, 191, 329, 145, 263, 335, 350, 223, 161, 222, 54, 322, 5, 371, 248, 113, 211, 52, 378, 60, 50, 262, 45, 226, 167, 19, 214, 1, 238, 270, 227, 378, 350, 129, 21, 202, 174, 133, 176, 342, 13, 155, 162, 164, 378, 350, 129, 69, 264, 301, 380, 81, 303, 39, 171, 331, 357, 196, 60, 366, 306, 383, 276, 289, 91, 221, 332, 148, 312, 230, 103, 60, 197, 253, 80, 83, 115, 151, 300, 0, 265, 377, 135, 114, 355, 200, 108, 3, 225, 229, 160, 128, 48, 257, 356, 304, 114, 355, 365, 95, 9, 254, 344, 284, 37, 236, 20, 139, 278, 166, 170, 366, 82, 348, 256, 144, 63, 188, 259, 137, 189, 233, 71, 170, 84, 198, 351, 282, 294, 261, 220, 149, 316, 16, 58, 11, 170, 369, 24, 324, 313, 235, 94, 74, 297, 28, 183, 85, 199, 251, 197, 44, 4, 379, 271, 317, 61, 141, 209, 240, 22, 199, 376, 369, 311, 127, 46, 325, 277, 354, 287, 370, 319, 40, 199, 27, 273, 360, 140, 217, 224, 6, 315, 158, 310, 361, 193, 142, 142, 168, 337, 104, 260, 250, 363, 272, 320, 204, 201, 132] + 42: [2, 93, 111, 265, 341, 181, 381, 283, 53, 290, 207, 35, 242, 188, 350, 256, 203, 135, 167, 137, 238, 205, 73, 198, 159, 289, 210, 279, 150, 209, 29, 338, 241, 311, 171, 343, 134, 47, 294, 154, 218, 111, 191, 354, 143, 223, 232, 233, 56, 22, 253, 339, 375, 218, 215, 351, 320, 164, 76, 97, 156, 284, 75, 331, 369, 359, 212, 333, 189, 328, 366, 80, 246, 298, 360, 383, 57, 15, 359, 116, 93, 348, 221, 329, 101, 180, 5, 197, 124, 177, 74, 20, 247, 168, 377, 344, 335, 65, 315, 270, 318, 186, 44, 163, 1, 211, 337, 262, 287, 173, 132, 219, 336, 98, 285, 271, 361, 217, 155, 222, 200, 17, 201, 146, 225, 16, 82, 128, 43, 0, 217, 184, 349, 220, 230, 165, 301, 64, 34, 104, 309, 249, 376, 50, 155, 333, 319, 131, 234, 166, 99, 68, 84, 302, 69, 95, 42, 172, 158, 327, 370, 45, 362, 226, 278, 175, 288, 182, 352, 123, 345, 374, 129, 286, 322, 160, 139, 7, 145, 330, 107, 27, 123, 345, 243, 108, 214, 161, 51, 372, 378, 102, 185, 78, 66, 123, 303, 41, 263, 239, 36, 192, 49, 251, 206, 3, 54, 40, 123, 305, 37, 59, 236, 92, 149, 244, 291, 229, 23, 81, 252, 123, 176, 256, 342, 248, 280, 39, 109, 61, 250, 144, 110, 121, 123, 176, 158, 281, 274, 314, 227, 157, 196, 321, 174, 12, 193, 123, 172, 72, 373, 275, 316, 245, 89, 58, 363, 367, 70, 273, 94, 268, 138, 208, 140, 105, 19, 231, 300, 293, 153, 100, 86, 94, 268, 138, 255, 224, 87, 8, 62, 125, 194, 96, 115, 254, 266, 357, 195, 306, 187, 356, 113, 323, 258, 368, 347, 353, 324, 266, 106, 37, 48, 67, 216, 60, 307, 170, 120, 85, 190, 25, 295, 117, 162, 55, 28, 103, 340, 371, 269, 213, 88, 276, 71, 148, 117, 179, 6, 169, 308, 380, 260, 313, 10, 334, 14, 91, 114, 117, 264, 122, 4, 38, 52, 299, 346, 261, 136, 79, 46, 228, 117, 282, 127, 30, 77, 118, 310, 364, 272, 304, 83, 126, 152, 31, 325, 379, 141, 202, 257, 183, 332, 199, 292, 317, 240, 152, 31, 325, 235, 358, 33, 297, 130, 355, 267, 296, 382, 365, 152, 31, 63, 21, 259, 147, 32, 9, 204, 90, 312, 142, 13, 152, 119, 151, 277, 178, 326, 18, 237, 112, 26, 133, 11, 24] + 43: [164, 324, 351, 23, 73, 115, 261, 284, 215, 87, 300, 228, 189, 369, 324, 314, 363, 272, 219, 61, 46, 101, 173, 69, 236, 195, 27, 216, 43, 337, 255, 90, 196, 183, 226, 193, 258, 42, 184, 344, 37, 298, 93, 321, 246, 79, 217, 116, 259, 260, 208, 231, 315, 59, 131, 57, 2, 234, 130, 45, 288, 247, 285, 328, 95, 315, 364, 298, 153, 121, 346, 28, 47, 146, 233, 248, 281, 169, 262, 98, 351, 23, 222, 200, 365, 213, 220, 209, 199, 21, 376, 82, 98, 43, 291, 4, 282, 334, 113, 325, 151, 179, 383, 182, 181, 98, 297, 81, 280, 88, 36, 74, 125, 358, 230, 141, 54, 168, 112, 103, 235, 191, 286, 107, 49, 72, 279, 357, 333, 33, 168, 359, 297, 55, 368, 378, 148, 102, 329, 65, 265, 58, 289, 168, 359, 94, 186, 89, 154, 243, 293, 370, 19, 110, 270, 302, 6, 35, 366, 152, 77, 275, 237, 165, 309, 268, 276, 303, 192, 221, 66, 30, 178, 1, 135, 367, 120, 170, 163, 336, 15, 319, 221, 66, 100, 326, 150, 277, 67, 210, 175, 126, 290, 133, 350, 161, 66, 30, 29, 85, 232, 12, 242, 122, 117, 320, 96, 212, 105, 129, 14, 127, 307, 123, 5, 253, 295, 76, 330, 104, 25, 105, 197, 14, 111, 250, 238, 155, 229, 9, 134, 53, 0, 382, 171, 142, 13, 138, 147, 327, 63, 24, 207, 71, 355, 362, 206, 171, 371, 20, 342, 78, 380, 84, 167, 128, 109, 114, 62, 44, 156, 17, 39, 377, 379, 52, 241, 180, 263, 119, 124, 249, 331, 156, 17, 349, 22, 32, 86, 157, 143, 18, 132, 187, 340, 310, 177, 335, 283, 227, 108, 299, 296, 266, 159, 353, 188, 139, 244, 332, 335, 283, 323, 269, 339, 145, 99, 64, 294, 311, 137, 373, 347, 372, 254, 264, 305, 140, 34, 251, 292, 144, 240, 352, 361, 347, 372, 190, 301, 136, 204, 345, 60, 304, 8, 211, 91, 176, 225, 68, 252, 318, 224, 271, 149, 70, 31, 356, 162, 7, 214, 225, 218, 75, 203, 274, 83, 10, 38, 174, 118, 201, 41, 245, 225, 92, 190, 322, 202, 48, 194, 50, 80, 381, 341, 106, 348, 308, 92, 75, 223, 374, 205, 273, 185, 11, 267, 316, 40, 166, 375, 375, 239, 313, 198, 26, 160, 16, 256, 158, 312, 172, 354, 375, 278, 97, 360, 338, 306, 257, 56, 343, 287, 51, 317, 3] + 44: [362, 271, 325, 11, 14, 219, 286, 46, 211, 114, 273, 31, 133, 362, 60, 331, 187, 158, 296, 269, 90, 205, 53, 1, 302, 332, 274, 284, 311, 143, 121, 287, 131, 327, 179, 314, 326, 307, 303, 274, 284, 229, 288, 265, 70, 71, 165, 8, 367, 375, 47, 197, 168, 155, 135, 320, 207, 315, 66, 36, 237, 183, 45, 259, 295, 238, 223, 252, 198, 228, 192, 177, 196, 377, 9, 233, 123, 208, 97, 223, 346, 63, 33, 186, 67, 220, 169, 51, 312, 52, 310, 64, 101, 68, 329, 144, 184, 221, 41, 32, 4, 2, 37, 75, 172, 348, 140, 93, 340, 56, 352, 128, 3, 62, 245, 127, 281, 172, 101, 346, 306, 48, 24, 54, 15, 59, 109, 239, 139, 181, 171, 348, 279, 383, 324, 349, 380, 342, 113, 256, 200, 297, 209, 171, 348, 368, 146, 276, 100, 132, 83, 214, 321, 216, 356, 339, 167, 226, 170, 142, 102, 85, 261, 92, 318, 230, 119, 246, 353, 0, 226, 170, 118, 125, 244, 381, 108, 323, 250, 151, 247, 58, 17, 226, 145, 333, 291, 364, 130, 147, 156, 57, 373, 69, 189, 258, 317, 376, 141, 217, 122, 378, 26, 341, 203, 251, 372, 308, 213, 110, 145, 98, 355, 61, 174, 185, 270, 49, 124, 111, 13, 213, 351, 82, 313, 94, 201, 79, 136, 336, 319, 22, 176, 106, 213, 317, 84, 242, 275, 266, 120, 126, 309, 193, 5, 103, 12, 213, 317, 257, 280, 89, 175, 87, 157, 104, 137, 253, 152, 161, 25, 379, 43, 166, 117, 290, 202, 354, 81, 204, 42, 73, 371, 77, 10, 277, 30, 138, 163, 248, 107, 20, 7, 35, 335, 129, 77, 10, 328, 236, 91, 254, 359, 74, 149, 345, 153, 330, 374, 77, 379, 29, 34, 262, 350, 369, 278, 78, 235, 99, 283, 76, 272, 10, 227, 86, 343, 150, 366, 334, 231, 263, 210, 363, 148, 178, 382, 180, 264, 365, 154, 160, 292, 218, 232, 80, 293, 112, 300, 222, 267, 212, 337, 173, 338, 370, 347, 241, 50, 240, 316, 301, 199, 188, 182, 95, 55, 164, 40, 105, 357, 195, 260, 116, 301, 199, 215, 289, 360, 39, 285, 282, 243, 18, 27, 268, 322, 301, 222, 190, 38, 358, 299, 96, 162, 305, 65, 255, 361, 206, 191, 191, 21, 294, 304, 224, 44, 344, 23, 234, 194, 298, 88, 191, 199, 21, 249, 225, 6, 134, 159, 19, 72, 115, 28, 16] + 45: [154, 227, 269, 275, 223, 217, 26, 296, 282, 113, 111, 99, 69, 154, 227, 298, 107, 162, 260, 315, 59, 294, 268, 132, 278, 100, 324, 247, 329, 254, 359, 94, 32, 38, 37, 289, 31, 101, 117, 23, 141, 21, 331, 307, 157, 279, 252, 322, 190, 214, 77, 181, 23, 356, 164, 248, 196, 108, 67, 290, 263, 73, 91, 66, 195, 191, 339, 164, 234, 45, 86, 208, 358, 9, 48, 89, 265, 355, 173, 176, 61, 258, 310, 44, 292, 264, 283, 364, 189, 13, 60, 121, 120, 131, 351, 168, 281, 109, 87, 151, 362, 90, 259, 235, 121, 171, 329, 228, 239, 41, 96, 319, 333, 215, 321, 287, 185, 301, 27, 180, 39, 187, 156, 175, 256, 379, 377, 238, 172, 244, 318, 27, 47, 118, 198, 242, 365, 5, 98, 203, 18, 193, 64, 219, 274, 286, 163, 245, 210, 376, 20, 288, 341, 186, 352, 138, 140, 10, 276, 95, 30, 251, 253, 19, 177, 220, 152, 194, 336, 140, 8, 354, 165, 88, 313, 43, 344, 79, 166, 82, 206, 174, 140, 8, 276, 95, 104, 357, 261, 34, 57, 147, 102, 311, 49, 233, 10, 378, 81, 250, 328, 257, 230, 255, 202, 225, 325, 17, 188, 10, 262, 209, 158, 345, 123, 207, 374, 119, 382, 53, 14, 134, 197, 375, 114, 338, 112, 222, 126, 231, 71, 243, 237, 130, 144, 182, 227, 297, 372, 84, 340, 205, 241, 93, 136, 184, 299, 316, 346, 350, 380, 122, 249, 236, 291, 273, 368, 125, 6, 11, 92, 178, 146, 383, 74, 306, 22, 58, 314, 105, 143, 159, 300, 92, 178, 103, 348, 70, 54, 50, 133, 342, 15, 56, 148, 40, 92, 178, 85, 124, 155, 213, 2, 145, 29, 78, 332, 28, 149, 92, 201, 369, 183, 55, 179, 106, 320, 150, 312, 280, 295, 361, 92, 201, 367, 128, 36, 327, 63, 35, 129, 142, 167, 270, 221, 110, 334, 373, 212, 51, 360, 24, 304, 381, 371, 62, 366, 192, 161, 199, 160, 383, 277, 7, 303, 76, 271, 363, 226, 200, 83, 161, 334, 367, 224, 326, 135, 1, 46, 211, 343, 323, 335, 72, 317, 139, 350, 52, 16, 267, 302, 116, 169, 3, 337, 75, 240, 293, 42, 373, 81, 308, 12, 218, 65, 285, 0, 97, 353, 347, 293, 25, 33, 349, 229, 68, 370, 330, 204, 246, 232, 305, 266, 284, 42, 33, 272, 115, 80, 4, 170, 127, 309, 153, 216, 137] + 46: [152, 191, 140, 303, 280, 334, 17, 32, 71, 26, 347, 84, 200, 188, 239, 154, 165, 8, 295, 250, 81, 9, 339, 67, 105, 88, 142, 207, 12, 40, 117, 132, 237, 307, 159, 98, 150, 123, 357, 142, 202, 335, 254, 195, 353, 267, 255, 313, 264, 72, 107, 302, 142, 238, 66, 345, 131, 287, 158, 45, 20, 90, 258, 251, 5, 133, 51, 301, 338, 62, 232, 235, 383, 265, 244, 15, 306, 27, 133, 207, 312, 378, 130, 118, 292, 328, 56, 278, 112, 212, 270, 210, 96, 335, 60, 155, 186, 321, 308, 7, 93, 75, 196, 61, 10, 110, 344, 204, 143, 315, 103, 92, 189, 283, 281, 269, 122, 10, 25, 140, 2, 85, 42, 294, 329, 243, 309, 77, 1, 82, 201, 110, 66, 113, 211, 156, 273, 35, 346, 305, 135, 279, 323, 247, 170, 319, 194, 372, 192, 95, 101, 293, 190, 275, 296, 365, 247, 16, 354, 203, 39, 37, 41, 373, 317, 361, 127, 48, 147, 183, 311, 209, 271, 376, 166, 63, 263, 382, 371, 163, 47, 151, 183, 311, 319, 124, 227, 259, 272, 226, 74, 261, 31, 73, 36, 183, 249, 104, 214, 260, 4, 362, 169, 299, 370, 330, 179, 178, 34, 343, 106, 136, 83, 369, 23, 97, 173, 364, 236, 54, 70, 34, 311, 284, 55, 115, 99, 157, 111, 153, 262, 65, 381, 356, 352, 355, 324, 199, 30, 180, 53, 125, 342, 285, 21, 300, 175, 352, 355, 6, 114, 108, 326, 193, 268, 241, 91, 11, 134, 220, 352, 343, 367, 320, 266, 50, 164, 141, 348, 375, 208, 349, 14, 233, 253, 367, 3, 102, 197, 205, 78, 222, 86, 43, 87, 231, 19, 240, 274, 242, 215, 298, 176, 289, 318, 177, 94, 174, 252, 64, 46, 149, 68, 120, 80, 59, 336, 333, 217, 366, 181, 325, 64, 253, 149, 38, 33, 245, 340, 57, 277, 171, 337, 0, 100, 139, 380, 230, 44, 49, 218, 198, 290, 119, 145, 341, 29, 182, 139, 327, 221, 18, 228, 168, 216, 24, 297, 331, 310, 116, 148, 256, 224, 332, 219, 350, 137, 128, 377, 322, 234, 52, 248, 225, 256, 380, 246, 374, 351, 121, 172, 360, 223, 184, 358, 288, 379, 58, 286, 106, 187, 22, 316, 304, 206, 69, 257, 138, 185, 28, 58, 368, 354, 146, 79, 109, 162, 160, 89, 359, 126, 276, 167, 314, 286, 230, 76, 282, 13, 129, 144, 161, 291, 229, 363, 213] + 47: [195, 158, 196, 129, 159, 197, 164, 378, 186, 144, 100, 247, 56, 195, 61, 127, 175, 351, 199, 354, 321, 273, 325, 372, 88, 369, 195, 29, 353, 274, 69, 249, 12, 28, 332, 302, 174, 99, 318, 195, 29, 257, 36, 93, 285, 0, 126, 258, 152, 379, 295, 7, 75, 158, 368, 79, 250, 298, 138, 163, 37, 120, 105, 270, 191, 343, 87, 316, 362, 265, 370, 101, 335, 107, 242, 16, 64, 254, 343, 1, 14, 376, 219, 151, 52, 315, 132, 297, 277, 11, 8, 343, 34, 305, 240, 63, 203, 210, 15, 259, 266, 361, 47, 246, 54, 109, 345, 59, 98, 255, 117, 201, 328, 57, 304, 116, 156, 54, 87, 300, 153, 262, 166, 180, 23, 310, 46, 74, 232, 62, 54, 87, 333, 40, 176, 184, 106, 366, 2, 97, 133, 264, 146, 271, 170, 309, 21, 95, 41, 17, 71, 115, 102, 207, 6, 182, 347, 279, 286, 238, 251, 77, 252, 139, 32, 253, 317, 299, 373, 331, 119, 134, 188, 226, 65, 233, 356, 341, 149, 290, 125, 261, 187, 375, 134, 346, 114, 50, 27, 18, 60, 19, 350, 165, 272, 179, 35, 90, 168, 212, 73, 112, 141, 94, 275, 5, 161, 209, 136, 3, 294, 198, 230, 172, 322, 147, 306, 189, 358, 217, 66, 181, 43, 320, 267, 313, 140, 4, 96, 248, 334, 38, 85, 288, 181, 371, 215, 344, 330, 33, 205, 289, 111, 48, 177, 256, 83, 154, 3, 137, 110, 269, 323, 45, 336, 278, 282, 374, 167, 113, 154, 3, 137, 118, 155, 103, 355, 211, 185, 81, 380, 283, 128, 192, 3, 314, 287, 301, 223, 296, 213, 31, 365, 220, 76, 293, 143, 22, 300, 327, 377, 124, 39, 80, 281, 216, 222, 142, 381, 171, 22, 307, 123, 26, 228, 338, 121, 200, 160, 92, 78, 169, 162, 312, 193, 218, 68, 231, 237, 243, 337, 308, 86, 194, 13, 162, 312, 227, 349, 135, 89, 340, 183, 157, 130, 30, 145, 360, 44, 357, 359, 367, 67, 224, 173, 234, 20, 9, 383, 221, 241, 44, 291, 359, 229, 104, 324, 204, 225, 260, 10, 245, 206, 214, 44, 24, 382, 342, 208, 319, 25, 70, 292, 55, 339, 268, 236, 352, 24, 382, 363, 84, 263, 51, 303, 49, 150, 58, 244, 239, 352, 24, 307, 131, 122, 364, 53, 91, 276, 72, 178, 202, 326, 352, 312, 280, 235, 42, 284, 148, 348, 329, 190, 108, 82, 311] + 48: [237, 121, 83, 15, 156, 50, 230, 379, 259, 382, 149, 171, 314, 267, 236, 28, 166, 371, 174, 170, 36, 269, 310, 61, 86, 245, 267, 343, 78, 62, 300, 192, 275, 22, 35, 313, 27, 54, 303, 159, 44, 21, 62, 205, 112, 221, 307, 137, 216, 361, 46, 348, 297, 234, 85, 263, 312, 299, 347, 80, 346, 373, 224, 68, 306, 288, 19, 182, 235, 134, 285, 24, 180, 107, 173, 66, 8, 334, 9, 132, 2, 38, 186, 123, 118, 289, 16, 243, 225, 282, 209, 51, 132, 353, 240, 115, 322, 360, 113, 169, 194, 294, 296, 10, 283, 99, 284, 5, 305, 233, 357, 317, 274, 135, 105, 298, 196, 283, 132, 311, 372, 342, 7, 42, 96, 352, 69, 37, 332, 199, 76, 276, 52, 48, 316, 161, 231, 281, 55, 217, 278, 338, 227, 76, 276, 203, 264, 185, 247, 321, 351, 98, 366, 335, 362, 250, 20, 239, 311, 363, 34, 340, 329, 204, 280, 92, 87, 124, 167, 249, 239, 74, 254, 142, 0, 160, 49, 12, 39, 309, 365, 258, 266, 226, 284, 139, 354, 152, 315, 339, 195, 108, 191, 331, 304, 266, 116, 52, 359, 184, 130, 57, 241, 292, 214, 277, 183, 333, 17, 273, 101, 344, 380, 120, 242, 97, 43, 187, 350, 248, 355, 17, 273, 295, 147, 319, 179, 218, 14, 40, 82, 376, 122, 103, 17, 215, 353, 327, 181, 293, 172, 72, 18, 272, 356, 251, 131, 271, 330, 78, 15, 222, 370, 111, 202, 94, 261, 148, 154, 308, 271, 109, 381, 104, 378, 324, 255, 63, 374, 29, 165, 383, 265, 271, 358, 133, 207, 229, 73, 89, 11, 84, 93, 143, 262, 197, 128, 328, 85, 263, 210, 190, 31, 119, 53, 71, 201, 345, 114, 128, 330, 33, 6, 320, 257, 168, 158, 127, 256, 228, 377, 129, 26, 328, 268, 177, 41, 193, 13, 138, 151, 178, 188, 153, 30, 270, 328, 381, 157, 219, 25, 318, 64, 59, 286, 238, 60, 200, 102, 140, 141, 375, 155, 175, 91, 146, 349, 32, 110, 106, 336, 102, 140, 141, 189, 232, 246, 79, 302, 367, 47, 244, 125, 1, 77, 140, 90, 208, 325, 88, 341, 81, 198, 326, 323, 23, 364, 65, 279, 164, 369, 290, 45, 162, 253, 211, 3, 56, 126, 337, 117, 301, 58, 287, 368, 75, 212, 100, 144, 95, 213, 260, 136, 145, 150, 101, 223, 252, 291, 4, 70, 206, 67, 163, 176, 220] + 49: [374, 126, 201, 325, 255, 28, 29, 238, 85, 101, 88, 249, 369, 374, 126, 153, 199, 331, 80, 290, 308, 176, 190, 232, 141, 230, 79, 253, 9, 193, 381, 221, 170, 267, 72, 284, 27, 21, 50, 79, 213, 40, 251, 332, 70, 196, 62, 244, 52, 57, 231, 347, 210, 253, 278, 286, 121, 63, 112, 35, 47, 285, 147, 96, 92, 233, 253, 116, 260, 144, 169, 81, 225, 61, 49, 104, 171, 135, 360, 157, 202, 269, 264, 23, 277, 338, 345, 299, 377, 265, 370, 360, 293, 126, 91, 58, 336, 77, 273, 355, 361, 167, 237, 38, 302, 136, 242, 44, 186, 54, 128, 10, 53, 151, 93, 334, 373, 302, 150, 180, 148, 8, 174, 4, 173, 280, 192, 13, 102, 24, 302, 158, 105, 246, 164, 328, 127, 179, 117, 84, 185, 207, 252, 302, 181, 353, 74, 183, 368, 214, 327, 378, 200, 324, 215, 247, 302, 115, 202, 344, 95, 16, 318, 43, 114, 113, 229, 159, 300, 302, 157, 313, 187, 100, 311, 140, 227, 156, 154, 372, 287, 32, 184, 19, 335, 39, 45, 382, 163, 55, 130, 124, 168, 350, 322, 197, 19, 241, 243, 195, 340, 189, 330, 339, 283, 7, 363, 64, 321, 305, 275, 259, 312, 56, 240, 307, 351, 354, 315, 266, 68, 321, 305, 276, 177, 120, 223, 138, 309, 359, 145, 326, 217, 203, 321, 59, 242, 262, 329, 86, 198, 178, 239, 143, 341, 18, 258, 321, 234, 208, 251, 175, 137, 89, 352, 320, 342, 51, 42, 188, 272, 297, 271, 76, 122, 1, 34, 33, 14, 228, 316, 99, 317, 272, 65, 25, 133, 48, 254, 248, 139, 0, 204, 274, 152, 129, 149, 297, 78, 220, 131, 17, 160, 67, 211, 282, 343, 379, 337, 37, 297, 313, 306, 94, 366, 20, 301, 118, 206, 166, 161, 356, 83, 209, 66, 289, 5, 250, 146, 216, 119, 31, 357, 281, 3, 294, 257, 261, 218, 333, 97, 376, 303, 358, 2, 348, 263, 26, 319, 155, 90, 60, 30, 279, 41, 295, 226, 106, 182, 383, 349, 6, 75, 270, 292, 109, 205, 296, 82, 71, 46, 125, 291, 219, 6, 155, 271, 107, 235, 362, 222, 371, 69, 12, 364, 11, 288, 268, 304, 9, 193, 194, 245, 365, 36, 103, 256, 165, 162, 375, 268, 75, 212, 132, 298, 310, 224, 98, 142, 108, 87, 367, 123, 346, 15, 110, 380, 314, 73, 323, 134, 172, 22, 111, 236, 191] + 50: [57, 325, 129, 155, 271, 203, 310, 340, 204, 12, 381, 216, 131, 33, 325, 129, 22, 224, 195, 151, 145, 281, 156, 10, 8, 50, 80, 273, 26, 295, 130, 48, 92, 242, 39, 141, 13, 103, 55, 29, 232, 27, 256, 159, 290, 342, 251, 42, 200, 61, 190, 93, 29, 232, 280, 88, 213, 53, 252, 5, 259, 89, 82, 279, 307, 86, 69, 282, 314, 270, 263, 267, 105, 337, 238, 209, 265, 264, 86, 69, 26, 331, 15, 116, 180, 172, 289, 7, 323, 260, 254, 86, 35, 377, 0, 274, 228, 81, 163, 178, 318, 346, 158, 368, 25, 218, 287, 220, 133, 71, 378, 322, 202, 248, 21, 47, 230, 235, 262, 312, 199, 132, 380, 73, 126, 369, 148, 98, 144, 283, 268, 215, 175, 341, 298, 58, 326, 277, 243, 354, 112, 54, 84, 268, 215, 27, 327, 363, 306, 107, 294, 347, 90, 65, 120, 182, 268, 361, 56, 45, 6, 239, 64, 152, 37, 293, 285, 221, 317, 226, 370, 3, 334, 187, 165, 371, 78, 127, 212, 83, 328, 111, 100, 188, 350, 51, 184, 183, 250, 99, 4, 63, 11, 142, 336, 100, 188, 350, 79, 272, 154, 94, 211, 191, 68, 87, 261, 364, 299, 210, 18, 193, 249, 134, 206, 143, 170, 237, 40, 245, 332, 316, 233, 18, 114, 335, 379, 197, 223, 344, 305, 176, 360, 136, 316, 210, 9, 119, 121, 359, 174, 185, 339, 349, 147, 266, 44, 109, 284, 3, 343, 253, 375, 382, 2, 17, 355, 77, 31, 59, 109, 352, 1, 358, 16, 19, 74, 160, 362, 258, 179, 161, 23, 311, 315, 1, 217, 353, 194, 276, 357, 153, 34, 286, 309, 222, 137, 315, 56, 192, 24, 75, 139, 28, 135, 36, 138, 225, 52, 140, 32, 214, 62, 301, 186, 234, 255, 292, 46, 101, 104, 66, 49, 376, 214, 329, 196, 117, 85, 91, 297, 330, 30, 70, 241, 49, 32, 146, 231, 313, 219, 319, 113, 123, 308, 76, 157, 374, 181, 367, 321, 296, 122, 348, 208, 205, 288, 198, 291, 167, 302, 181, 376, 373, 304, 244, 168, 169, 108, 124, 320, 201, 173, 345, 67, 38, 321, 128, 164, 43, 247, 300, 106, 383, 60, 72, 257, 229, 14, 125, 128, 96, 240, 269, 278, 177, 372, 246, 207, 365, 166, 97, 366, 338, 41, 149, 303, 20, 236, 171, 162, 227, 115, 166, 333, 110, 231, 275, 102, 189, 356, 351, 150, 324, 95, 118] + 51: [322, 17, 62, 32, 143, 341, 90, 75, 30, 242, 199, 293, 255, 108, 17, 300, 325, 247, 331, 33, 164, 27, 363, 272, 128, 290, 31, 102, 140, 270, 374, 162, 219, 317, 258, 43, 125, 29, 225, 137, 52, 211, 188, 92, 47, 251, 306, 89, 353, 39, 305, 348, 137, 182, 126, 235, 259, 161, 34, 195, 328, 333, 215, 132, 156, 373, 102, 229, 95, 299, 382, 91, 361, 223, 145, 158, 217, 202, 373, 102, 163, 60, 20, 268, 18, 171, 334, 109, 4, 376, 249, 375, 287, 343, 203, 216, 86, 190, 205, 115, 378, 83, 71, 76, 375, 284, 65, 104, 294, 359, 280, 324, 312, 53, 184, 117, 244, 106, 287, 163, 40, 35, 237, 192, 239, 170, 93, 7, 277, 326, 221, 113, 82, 295, 11, 183, 160, 157, 365, 336, 15, 302, 377, 221, 193, 358, 146, 346, 212, 252, 265, 297, 339, 354, 367, 151, 189, 123, 126, 250, 5, 285, 368, 364, 349, 238, 218, 147, 150, 73, 12, 229, 260, 46, 74, 360, 369, 383, 25, 19, 63, 347, 241, 208, 82, 124, 13, 245, 321, 236, 54, 276, 165, 70, 327, 241, 12, 67, 286, 23, 380, 329, 84, 196, 167, 68, 274, 231, 371, 208, 144, 286, 220, 279, 42, 311, 315, 55, 9, 154, 261, 37, 99, 337, 129, 335, 254, 366, 313, 100, 112, 116, 266, 22, 37, 357, 136, 233, 263, 303, 204, 41, 379, 61, 214, 352, 36, 372, 186, 264, 135, 85, 107, 209, 69, 213, 240, 66, 187, 21, 372, 186, 72, 152, 281, 198, 119, 141, 288, 278, 88, 77, 138, 372, 186, 144, 87, 118, 56, 222, 262, 332, 232, 148, 185, 131, 210, 16, 337, 330, 26, 275, 370, 44, 80, 78, 14, 174, 291, 210, 186, 64, 105, 314, 344, 248, 10, 191, 298, 296, 355, 121, 101, 111, 176, 304, 134, 308, 207, 48, 350, 340, 28, 0, 2, 6, 24, 122, 283, 49, 271, 309, 50, 194, 319, 257, 120, 139, 110, 24, 127, 153, 130, 345, 351, 289, 94, 316, 1, 133, 173, 110, 24, 356, 282, 243, 142, 181, 320, 381, 159, 57, 227, 226, 200, 323, 180, 177, 103, 81, 310, 224, 45, 197, 96, 169, 155, 200, 24, 228, 253, 269, 97, 79, 3, 338, 273, 230, 179, 267, 178, 323, 342, 168, 246, 172, 59, 362, 114, 307, 201, 175, 256, 206, 98, 342, 301, 8, 292, 166, 234, 51, 58, 149, 318, 38] + 52: [52, 137, 184, 140, 149, 124, 367, 350, 242, 100, 293, 338, 14, 186, 137, 90, 68, 309, 74, 377, 268, 201, 380, 335, 185, 212, 327, 73, 25, 60, 257, 166, 3, 324, 282, 220, 363, 241, 1, 121, 97, 25, 346, 37, 339, 351, 161, 89, 101, 18, 36, 175, 92, 97, 48, 382, 123, 281, 248, 93, 383, 347, 229, 79, 162, 102, 165, 193, 205, 155, 178, 67, 208, 365, 322, 226, 65, 376, 102, 165, 48, 183, 108, 5, 153, 39, 194, 129, 343, 131, 218, 102, 97, 209, 190, 325, 285, 53, 27, 58, 356, 313, 9, 305, 307, 165, 345, 245, 236, 191, 113, 374, 251, 188, 353, 203, 106, 120, 104, 290, 56, 354, 378, 154, 45, 177, 156, 126, 145, 38, 231, 104, 239, 44, 95, 215, 134, 170, 84, 150, 138, 265, 94, 332, 104, 214, 269, 87, 221, 331, 40, 200, 179, 232, 15, 86, 61, 362, 85, 168, 210, 105, 301, 55, 148, 49, 4, 132, 247, 216, 250, 323, 219, 266, 234, 72, 69, 21, 51, 147, 273, 204, 107, 71, 34, 33, 291, 341, 369, 91, 258, 310, 299, 261, 176, 107, 103, 336, 128, 349, 30, 75, 326, 237, 28, 340, 159, 264, 107, 103, 336, 196, 32, 160, 364, 235, 110, 47, 302, 202, 272, 372, 71, 174, 260, 8, 359, 6, 333, 238, 136, 304, 366, 254, 283, 246, 321, 10, 252, 23, 233, 225, 57, 143, 96, 158, 348, 283, 71, 289, 192, 355, 167, 116, 306, 2, 330, 135, 187, 370, 217, 157, 271, 80, 50, 256, 319, 109, 270, 118, 344, 0, 24, 381, 334, 209, 78, 379, 222, 17, 253, 294, 197, 111, 19, 295, 381, 151, 206, 357, 98, 315, 243, 22, 207, 141, 43, 255, 16, 381, 157, 239, 259, 64, 173, 146, 316, 152, 230, 274, 122, 240, 381, 157, 195, 26, 54, 223, 280, 213, 20, 368, 112, 329, 180, 320, 334, 169, 7, 171, 83, 292, 287, 224, 303, 311, 297, 375, 371, 334, 263, 172, 35, 127, 115, 119, 352, 342, 114, 11, 164, 99, 182, 298, 205, 228, 227, 29, 308, 277, 198, 358, 142, 317, 99, 63, 82, 288, 88, 117, 361, 66, 244, 139, 276, 144, 373, 99, 130, 77, 13, 133, 286, 76, 125, 284, 278, 81, 314, 46, 99, 334, 211, 62, 70, 262, 328, 279, 189, 249, 181, 42, 360, 267, 31, 163, 41, 312, 296, 300, 337, 59, 318, 275, 12, 199] + 53: [230, 311, 132, 209, 88, 116, 76, 111, 182, 257, 106, 288, 90, 336, 365, 266, 145, 139, 361, 362, 216, 294, 87, 113, 98, 71, 308, 354, 302, 160, 25, 69, 67, 2, 28, 283, 51, 41, 290, 217, 354, 186, 291, 107, 378, 349, 371, 218, 234, 64, 104, 84, 75, 46, 227, 157, 241, 77, 89, 166, 215, 9, 100, 124, 99, 8, 322, 214, 221, 95, 151, 15, 105, 236, 249, 199, 165, 245, 68, 109, 363, 81, 326, 335, 315, 120, 251, 21, 74, 135, 114, 240, 109, 345, 26, 284, 49, 10, 370, 220, 195, 303, 204, 11, 240, 280, 347, 121, 368, 373, 149, 96, 358, 80, 63, 91, 307, 240, 292, 273, 119, 24, 300, 259, 189, 133, 154, 348, 263, 153, 110, 296, 228, 156, 329, 141, 146, 206, 79, 246, 310, 192, 377, 110, 296, 170, 180, 262, 70, 328, 169, 30, 305, 39, 260, 140, 110, 164, 92, 268, 168, 383, 102, 233, 97, 275, 184, 286, 344, 110, 16, 32, 86, 5, 231, 293, 340, 101, 337, 229, 33, 355, 110, 16, 118, 360, 94, 183, 175, 128, 254, 54, 34, 343, 265, 110, 16, 4, 279, 12, 381, 357, 174, 306, 323, 29, 211, 56, 110, 173, 155, 194, 253, 239, 159, 339, 255, 367, 147, 208, 185, 110, 142, 224, 225, 134, 52, 281, 321, 364, 14, 48, 148, 252, 297, 38, 186, 6, 62, 341, 143, 248, 59, 0, 163, 213, 318, 243, 38, 4, 351, 325, 205, 44, 369, 55, 201, 301, 382, 210, 203, 38, 179, 376, 20, 324, 47, 276, 138, 7, 298, 356, 167, 222, 320, 42, 202, 200, 188, 72, 375, 278, 43, 331, 212, 22, 66, 129, 330, 261, 125, 277, 60, 327, 40, 352, 17, 374, 272, 317, 287, 32, 309, 256, 338, 359, 115, 136, 131, 82, 27, 58, 317, 238, 330, 162, 31, 78, 176, 207, 198, 53, 342, 161, 18, 13, 334, 250, 332, 269, 37, 353, 35, 299, 137, 1, 93, 316, 13, 287, 347, 285, 171, 152, 126, 65, 19, 264, 83, 61, 223, 270, 334, 311, 103, 127, 172, 122, 36, 242, 258, 372, 274, 191, 73, 350, 379, 237, 319, 313, 150, 57, 123, 108, 193, 267, 312, 73, 23, 295, 130, 219, 112, 282, 196, 232, 45, 366, 187, 304, 73, 23, 295, 271, 3, 346, 226, 178, 117, 380, 85, 190, 314, 247, 350, 250, 144, 50, 197, 235, 158, 333, 289, 177, 181, 244] + 54: [295, 240, 240, 3, 129, 225, 135, 171, 128, 6, 31, 138, 344, 370, 260, 324, 120, 156, 217, 239, 304, 364, 12, 343, 379, 36, 370, 80, 306, 187, 41, 17, 131, 366, 23, 235, 206, 291, 383, 370, 369, 299, 26, 322, 272, 381, 179, 96, 139, 226, 35, 19, 370, 214, 133, 118, 197, 91, 302, 48, 273, 59, 367, 37, 126, 284, 164, 72, 221, 288, 5, 186, 68, 166, 259, 52, 51, 354, 284, 82, 95, 345, 70, 265, 204, 111, 173, 243, 362, 307, 378, 337, 164, 38, 228, 47, 357, 67, 326, 195, 213, 233, 169, 358, 380, 164, 348, 75, 310, 190, 184, 136, 81, 334, 102, 285, 13, 153, 257, 280, 314, 215, 198, 183, 246, 141, 110, 331, 77, 340, 296, 150, 30, 329, 276, 94, 376, 293, 347, 15, 261, 2, 349, 363, 155, 176, 93, 332, 182, 21, 107, 114, 194, 227, 116, 200, 363, 155, 208, 44, 181, 24, 312, 85, 18, 63, 359, 134, 202, 363, 155, 254, 282, 27, 142, 157, 327, 151, 88, 263, 210, 148, 363, 216, 281, 305, 101, 109, 55, 144, 266, 234, 269, 350, 127, 363, 231, 230, 40, 64, 45, 97, 301, 316, 178, 274, 222, 207, 363, 231, 230, 117, 32, 220, 355, 264, 43, 247, 289, 241, 297, 253, 339, 286, 87, 121, 279, 237, 163, 244, 308, 89, 65, 160, 294, 339, 286, 57, 58, 382, 368, 373, 9, 346, 108, 104, 205, 294, 223, 172, 25, 333, 335, 318, 278, 351, 74, 29, 112, 78, 294, 223, 8, 352, 99, 50, 100, 61, 191, 145, 236, 252, 193, 283, 287, 341, 290, 375, 34, 103, 209, 167, 251, 60, 154, 130, 283, 339, 299, 309, 158, 92, 125, 242, 146, 371, 203, 360, 0, 71, 356, 95, 330, 10, 152, 140, 122, 162, 245, 56, 270, 232, 319, 356, 53, 119, 76, 159, 54, 137, 377, 192, 66, 292, 320, 328, 11, 98, 7, 196, 325, 365, 298, 275, 14, 211, 256, 342, 189, 11, 38, 317, 143, 323, 315, 255, 262, 219, 79, 4, 212, 338, 267, 20, 180, 161, 277, 177, 201, 62, 238, 199, 374, 336, 147, 258, 174, 311, 115, 84, 1, 22, 33, 229, 248, 303, 361, 83, 258, 250, 86, 218, 321, 105, 69, 300, 168, 249, 188, 165, 90, 39, 280, 372, 113, 124, 49, 73, 175, 224, 268, 28, 106, 42, 42, 348, 16, 132, 185, 170, 313, 353, 271, 46, 149, 123] + 55: [56, 111, 323, 226, 60, 206, 228, 193, 294, 104, 58, 83, 131, 56, 307, 323, 71, 272, 233, 194, 277, 1, 97, 177, 334, 222, 326, 238, 74, 99, 114, 102, 240, 287, 61, 356, 248, 21, 205, 113, 16, 304, 223, 176, 348, 360, 257, 127, 173, 376, 175, 317, 72, 252, 74, 134, 3, 17, 135, 19, 133, 208, 141, 92, 196, 159, 80, 382, 293, 179, 5, 306, 185, 367, 93, 120, 220, 292, 202, 311, 347, 363, 271, 284, 27, 278, 63, 118, 13, 181, 245, 202, 80, 347, 180, 124, 350, 246, 171, 214, 38, 64, 244, 144, 152, 39, 78, 180, 156, 112, 67, 332, 143, 15, 162, 295, 215, 28, 165, 121, 269, 312, 371, 231, 23, 198, 201, 161, 100, 18, 88, 89, 48, 269, 316, 375, 265, 128, 212, 372, 324, 36, 51, 88, 339, 90, 106, 349, 172, 286, 322, 365, 219, 183, 298, 34, 327, 182, 211, 76, 164, 274, 378, 101, 314, 137, 315, 158, 351, 154, 357, 247, 29, 86, 190, 260, 33, 22, 243, 125, 142, 379, 154, 370, 273, 199, 138, 276, 251, 94, 87, 338, 354, 230, 24, 154, 116, 41, 40, 331, 259, 187, 263, 188, 373, 77, 191, 204, 154, 116, 353, 115, 216, 325, 343, 358, 82, 126, 145, 32, 148, 43, 340, 48, 115, 221, 14, 224, 84, 296, 129, 147, 345, 59, 270, 340, 73, 130, 26, 95, 236, 207, 264, 209, 68, 362, 250, 256, 218, 90, 130, 383, 163, 79, 119, 103, 225, 242, 169, 69, 81, 308, 123, 70, 366, 210, 380, 52, 117, 297, 261, 75, 45, 81, 62, 267, 65, 300, 105, 217, 85, 313, 55, 275, 288, 25, 235, 308, 381, 2, 8, 46, 42, 184, 258, 335, 280, 364, 305, 355, 146, 192, 341, 12, 344, 136, 318, 281, 253, 166, 282, 96, 151, 146, 227, 291, 333, 189, 53, 241, 37, 268, 203, 262, 109, 174, 234, 303, 76, 368, 289, 377, 157, 302, 168, 150, 195, 9, 174, 11, 299, 132, 290, 197, 178, 10, 232, 319, 374, 57, 167, 174, 11, 299, 49, 44, 239, 140, 122, 342, 35, 107, 108, 229, 174, 7, 381, 249, 279, 170, 47, 346, 160, 359, 369, 155, 285, 309, 20, 123, 301, 213, 352, 361, 255, 237, 30, 50, 110, 283, 4, 20, 6, 200, 320, 337, 54, 254, 153, 321, 66, 266, 310, 98, 98, 328, 29, 0, 91, 329, 149, 31, 336, 139, 330, 186] + 56: [279, 225, 9, 275, 66, 160, 31, 1, 218, 347, 157, 108, 76, 100, 225, 156, 48, 245, 0, 121, 123, 201, 274, 93, 203, 111, 177, 351, 75, 284, 159, 233, 22, 17, 183, 290, 24, 256, 146, 231, 184, 169, 161, 244, 137, 206, 248, 115, 139, 144, 45, 91, 231, 158, 30, 71, 288, 103, 317, 286, 190, 291, 170, 216, 165, 231, 303, 259, 140, 14, 344, 331, 34, 96, 293, 152, 272, 173, 231, 267, 259, 46, 283, 69, 35, 110, 145, 307, 204, 325, 202, 8, 296, 322, 188, 329, 38, 295, 33, 257, 323, 211, 358, 3, 195, 88, 340, 346, 63, 285, 79, 133, 314, 4, 70, 336, 208, 195, 163, 340, 320, 357, 376, 179, 127, 94, 232, 373, 52, 380, 195, 98, 106, 219, 19, 281, 136, 255, 180, 200, 43, 326, 224, 311, 155, 241, 251, 97, 359, 355, 185, 62, 330, 254, 215, 235, 321, 186, 106, 292, 327, 40, 338, 164, 298, 28, 350, 335, 90, 321, 53, 241, 2, 83, 128, 324, 42, 122, 37, 67, 217, 318, 343, 116, 5, 6, 207, 193, 92, 56, 313, 367, 49, 7, 109, 167, 107, 30, 162, 150, 250, 21, 243, 134, 310, 337, 68, 194, 242, 15, 371, 99, 80, 58, 230, 364, 339, 372, 64, 32, 301, 242, 15, 87, 316, 381, 197, 12, 151, 54, 112, 86, 36, 74, 265, 294, 304, 18, 332, 312, 252, 11, 119, 228, 258, 234, 315, 265, 273, 5, 117, 101, 278, 181, 129, 378, 16, 29, 366, 142, 44, 294, 75, 302, 362, 342, 198, 138, 262, 375, 345, 239, 237, 44, 294, 304, 41, 154, 73, 166, 13, 55, 132, 287, 349, 174, 260, 220, 263, 363, 192, 253, 135, 297, 221, 289, 266, 365, 102, 260, 213, 196, 125, 309, 27, 374, 209, 271, 379, 47, 382, 143, 260, 213, 20, 187, 223, 369, 149, 214, 353, 226, 361, 261, 51, 126, 220, 89, 333, 118, 148, 305, 270, 210, 319, 60, 176, 25, 191, 220, 147, 308, 383, 182, 354, 153, 84, 172, 131, 26, 229, 199, 81, 147, 39, 77, 300, 268, 227, 299, 175, 130, 59, 247, 178, 240, 249, 264, 72, 205, 280, 50, 341, 124, 222, 370, 171, 178, 240, 236, 104, 238, 246, 269, 282, 348, 65, 78, 277, 61, 82, 240, 189, 328, 168, 10, 105, 368, 95, 276, 113, 212, 114, 82, 240, 356, 306, 120, 334, 23, 141, 360, 57, 352, 377, 85] + 57: [157, 148, 125, 309, 163, 277, 377, 351, 41, 200, 48, 291, 234, 231, 128, 125, 22, 44, 27, 208, 337, 29, 4, 361, 292, 245, 231, 148, 118, 79, 123, 42, 203, 149, 162, 32, 222, 206, 105, 93, 101, 268, 283, 354, 173, 54, 209, 17, 188, 194, 189, 305, 11, 235, 8, 317, 75, 356, 349, 352, 26, 303, 308, 10, 293, 342, 65, 335, 197, 251, 298, 116, 321, 311, 168, 383, 193, 69, 159, 205, 344, 345, 80, 191, 382, 196, 261, 59, 239, 52, 280, 14, 25, 141, 267, 365, 73, 112, 166, 363, 357, 50, 114, 233, 14, 205, 324, 60, 43, 213, 328, 136, 371, 281, 256, 221, 15, 230, 25, 130, 302, 172, 110, 380, 296, 215, 255, 67, 212, 19, 276, 366, 170, 74, 58, 152, 38, 210, 145, 265, 287, 100, 285, 276, 104, 170, 302, 373, 346, 55, 282, 18, 87, 260, 30, 227, 276, 25, 320, 169, 167, 177, 47, 164, 379, 126, 36, 340, 153, 31, 326, 124, 84, 238, 243, 225, 226, 247, 24, 138, 129, 229, 381, 326, 72, 318, 272, 341, 106, 155, 98, 252, 61, 142, 301, 158, 326, 359, 350, 338, 70, 358, 71, 334, 241, 66, 278, 181, 16, 140, 331, 64, 306, 96, 92, 190, 323, 1, 224, 195, 37, 254, 187, 53, 217, 179, 176, 13, 68, 49, 0, 242, 269, 77, 254, 63, 137, 300, 161, 322, 279, 348, 355, 333, 109, 336, 131, 254, 156, 320, 74, 91, 113, 183, 362, 178, 122, 139, 360, 343, 254, 7, 94, 127, 347, 9, 12, 28, 81, 33, 273, 266, 376, 254, 198, 207, 88, 185, 20, 102, 3, 237, 236, 107, 244, 262, 133, 187, 286, 154, 253, 214, 120, 369, 132, 5, 78, 310, 304, 314, 368, 286, 154, 39, 51, 289, 313, 175, 270, 375, 294, 143, 146, 246, 207, 160, 35, 249, 147, 23, 330, 248, 370, 258, 56, 146, 246, 331, 180, 62, 83, 184, 199, 332, 299, 232, 339, 86, 259, 89, 257, 151, 250, 312, 21, 171, 240, 307, 374, 82, 201, 259, 89, 223, 117, 182, 46, 111, 90, 288, 135, 364, 115, 202, 295, 165, 72, 150, 319, 297, 325, 353, 290, 378, 192, 218, 367, 295, 57, 108, 204, 316, 228, 284, 216, 186, 45, 76, 271, 372, 121, 165, 94, 6, 99, 134, 95, 329, 315, 219, 263, 327, 34, 220, 85, 274, 174, 144, 40, 119, 2, 97, 211, 103, 264, 275] + 58: [263, 222, 122, 68, 285, 187, 110, 241, 170, 360, 246, 313, 158, 248, 222, 9, 262, 329, 229, 378, 65, 195, 354, 103, 88, 340, 156, 101, 93, 272, 171, 139, 0, 172, 64, 296, 304, 361, 3, 156, 373, 206, 31, 242, 251, 236, 228, 293, 138, 32, 89, 279, 220, 194, 299, 42, 197, 314, 215, 123, 283, 41, 336, 188, 33, 220, 194, 235, 119, 66, 375, 234, 161, 35, 100, 39, 267, 38, 220, 311, 177, 189, 190, 381, 83, 21, 113, 5, 34, 233, 292, 220, 311, 177, 16, 130, 284, 135, 13, 230, 157, 146, 193, 58, 275, 98, 312, 203, 224, 48, 90, 43, 265, 24, 145, 23, 166, 275, 271, 312, 367, 198, 192, 238, 347, 287, 102, 153, 291, 80, 77, 98, 322, 191, 185, 331, 160, 186, 94, 317, 369, 165, 252, 377, 27, 67, 342, 315, 258, 131, 253, 173, 268, 51, 52, 280, 377, 348, 289, 152, 28, 380, 346, 327, 237, 303, 84, 78, 364, 377, 344, 114, 341, 106, 383, 63, 300, 243, 142, 120, 212, 147, 377, 210, 114, 18, 260, 143, 326, 334, 179, 330, 269, 217, 150, 377, 211, 298, 324, 20, 207, 339, 2, 19, 213, 159, 320, 164, 377, 245, 122, 141, 4, 25, 382, 154, 133, 359, 349, 310, 370, 377, 245, 22, 45, 352, 55, 306, 11, 70, 294, 137, 363, 335, 377, 245, 298, 196, 274, 239, 124, 176, 136, 163, 290, 301, 308, 377, 14, 49, 47, 328, 1, 53, 371, 115, 261, 216, 91, 309, 249, 356, 214, 175, 132, 129, 240, 148, 338, 86, 44, 15, 168, 151, 7, 355, 232, 12, 350, 302, 144, 118, 29, 57, 140, 71, 76, 92, 358, 376, 73, 281, 155, 278, 50, 54, 167, 365, 134, 126, 205, 351, 85, 316, 244, 183, 288, 8, 221, 46, 17, 184, 126, 92, 69, 295, 307, 202, 250, 345, 182, 95, 109, 218, 125, 56, 305, 270, 36, 104, 325, 332, 208, 180, 128, 223, 74, 10, 56, 205, 72, 6, 60, 82, 75, 379, 226, 372, 117, 323, 319, 81, 366, 199, 368, 255, 59, 333, 318, 225, 254, 200, 286, 209, 81, 273, 26, 112, 111, 266, 149, 357, 219, 276, 337, 30, 201, 81, 305, 257, 178, 162, 62, 227, 107, 353, 282, 374, 127, 204, 61, 366, 256, 116, 97, 99, 37, 231, 105, 264, 247, 174, 108, 79, 259, 169, 40, 277, 181, 121, 362, 297, 87, 96, 343, 321] + 59: [332, 173, 76, 178, 112, 218, 10, 96, 43, 115, 148, 272, 235, 140, 200, 150, 317, 164, 267, 354, 289, 253, 207, 108, 282, 99, 140, 200, 185, 151, 342, 245, 69, 242, 34, 326, 98, 318, 104, 73, 363, 31, 372, 81, 143, 236, 32, 287, 275, 119, 56, 90, 209, 251, 18, 186, 160, 174, 54, 48, 380, 11, 273, 270, 188, 209, 363, 331, 23, 128, 286, 269, 51, 129, 89, 82, 360, 189, 113, 70, 288, 262, 86, 338, 344, 2, 187, 145, 368, 192, 154, 182, 95, 130, 9, 85, 225, 214, 199, 63, 132, 166, 87, 55, 327, 260, 346, 295, 16, 302, 238, 6, 84, 28, 8, 276, 116, 327, 260, 221, 65, 254, 373, 158, 292, 64, 24, 321, 264, 351, 327, 59, 263, 38, 57, 334, 177, 13, 217, 67, 319, 305, 341, 327, 59, 371, 328, 161, 62, 369, 228, 134, 17, 216, 219, 271, 33, 202, 381, 196, 278, 20, 356, 194, 71, 3, 312, 335, 268, 30, 195, 311, 309, 109, 181, 361, 337, 322, 343, 237, 141, 126, 105, 202, 220, 255, 176, 22, 306, 299, 138, 210, 296, 378, 139, 285, 195, 220, 58, 133, 19, 222, 314, 246, 44, 307, 78, 345, 75, 94, 66, 26, 114, 142, 339, 156, 41, 277, 103, 293, 308, 224, 7, 234, 42, 279, 364, 203, 155, 184, 153, 301, 240, 83, 224, 94, 298, 281, 40, 280, 366, 353, 124, 330, 165, 350, 215, 46, 7, 294, 324, 35, 252, 120, 68, 208, 93, 241, 77, 323, 25, 370, 66, 80, 290, 29, 47, 107, 27, 249, 53, 125, 79, 25, 370, 204, 377, 175, 100, 259, 36, 325, 261, 230, 205, 122, 25, 7, 247, 144, 91, 376, 179, 362, 193, 106, 118, 14, 15, 170, 229, 352, 65, 168, 163, 146, 101, 50, 359, 316, 239, 152, 170, 233, 352, 265, 127, 169, 180, 131, 136, 74, 257, 1, 190, 170, 5, 346, 213, 12, 248, 258, 274, 149, 88, 284, 347, 72, 250, 229, 173, 266, 223, 383, 348, 349, 231, 37, 283, 39, 310, 250, 229, 340, 382, 304, 315, 355, 0, 333, 191, 201, 171, 52, 256, 226, 340, 232, 45, 320, 137, 123, 110, 336, 358, 121, 111, 313, 147, 374, 243, 212, 60, 329, 206, 227, 183, 357, 244, 167, 97, 379, 291, 300, 159, 162, 198, 61, 135, 49, 102, 367, 4, 197, 379, 234, 172, 297, 211, 92, 365, 157, 21, 303, 375, 117] + 60: [284, 133, 174, 119, 184, 37, 162, 33, 192, 252, 261, 340, 118, 102, 133, 78, 180, 203, 93, 344, 250, 300, 330, 54, 173, 89, 94, 113, 40, 365, 59, 255, 88, 7, 369, 328, 258, 227, 226, 123, 126, 302, 2, 140, 150, 61, 273, 349, 127, 41, 191, 317, 123, 83, 222, 345, 27, 0, 377, 53, 296, 251, 145, 17, 136, 123, 83, 48, 166, 161, 114, 92, 30, 182, 361, 42, 86, 288, 236, 126, 334, 366, 82, 20, 304, 171, 225, 338, 378, 275, 109, 121, 39, 40, 112, 231, 358, 267, 149, 130, 376, 204, 249, 63, 313, 60, 208, 76, 213, 240, 342, 360, 256, 363, 298, 164, 218, 313, 343, 155, 35, 143, 132, 24, 85, 79, 262, 374, 72, 154, 313, 244, 65, 34, 55, 362, 56, 134, 331, 383, 291, 306, 207, 313, 347, 312, 239, 235, 221, 11, 196, 190, 228, 137, 71, 10, 313, 372, 310, 290, 117, 6, 110, 142, 368, 168, 68, 159, 375, 268, 139, 81, 327, 241, 277, 116, 323, 371, 67, 50, 175, 355, 268, 60, 181, 108, 131, 183, 215, 70, 144, 307, 356, 289, 242, 381, 303, 297, 74, 253, 152, 264, 187, 322, 178, 320, 32, 73, 381, 139, 185, 205, 172, 75, 382, 146, 100, 202, 224, 301, 201, 335, 129, 194, 49, 232, 217, 5, 90, 276, 156, 379, 43, 210, 351, 220, 199, 309, 364, 189, 57, 186, 64, 125, 97, 4, 352, 351, 220, 199, 15, 246, 58, 176, 21, 14, 105, 229, 135, 373, 351, 47, 339, 98, 169, 336, 3, 107, 219, 325, 111, 367, 308, 351, 47, 316, 329, 263, 283, 332, 285, 370, 209, 19, 163, 87, 124, 170, 153, 28, 359, 151, 274, 233, 286, 197, 147, 295, 101, 124, 115, 120, 294, 245, 318, 8, 346, 12, 315, 96, 281, 157, 124, 195, 310, 353, 223, 269, 188, 66, 341, 287, 179, 177, 321, 124, 243, 265, 311, 238, 69, 279, 52, 380, 270, 160, 165, 22, 124, 266, 212, 354, 99, 84, 38, 31, 314, 45, 280, 292, 214, 257, 16, 230, 319, 278, 1, 305, 141, 18, 333, 29, 138, 77, 257, 128, 247, 234, 206, 259, 25, 26, 148, 282, 211, 326, 348, 257, 260, 167, 337, 122, 51, 237, 293, 95, 23, 104, 350, 299, 158, 16, 174, 193, 271, 248, 80, 91, 357, 324, 216, 103, 44, 158, 16, 265, 46, 272, 36, 9, 200, 106, 13, 198, 254, 62] + 61: [231, 381, 235, 263, 261, 233, 245, 125, 4, 239, 138, 276, 116, 303, 381, 215, 213, 32, 268, 104, 30, 85, 358, 331, 289, 266, 230, 363, 238, 307, 309, 366, 69, 341, 144, 25, 95, 377, 319, 172, 45, 142, 248, 20, 362, 2, 79, 176, 271, 180, 226, 208, 61, 197, 294, 161, 156, 166, 97, 273, 225, 74, 49, 267, 320, 61, 197, 324, 307, 227, 29, 206, 37, 71, 54, 265, 302, 343, 73, 380, 371, 248, 191, 118, 329, 219, 106, 244, 164, 285, 149, 73, 1, 68, 143, 337, 77, 336, 365, 194, 63, 201, 279, 357, 5, 264, 17, 119, 92, 297, 257, 242, 332, 91, 293, 131, 334, 136, 184, 12, 66, 133, 84, 3, 9, 340, 364, 262, 183, 76, 335, 209, 100, 216, 31, 44, 23, 8, 67, 18, 232, 50, 367, 108, 59, 100, 107, 181, 82, 200, 78, 26, 354, 105, 256, 58, 360, 241, 103, 288, 110, 304, 290, 153, 283, 251, 7, 382, 16, 93, 203, 167, 86, 361, 224, 109, 80, 204, 274, 48, 222, 321, 93, 10, 253, 291, 159, 113, 102, 38, 14, 345, 212, 64, 356, 211, 19, 353, 129, 188, 374, 33, 186, 298, 162, 236, 220, 171, 211, 270, 36, 107, 349, 27, 145, 375, 81, 187, 28, 56, 96, 120, 117, 36, 115, 369, 11, 259, 322, 152, 370, 182, 60, 40, 157, 178, 218, 216, 286, 252, 202, 292, 124, 163, 146, 250, 0, 221, 299, 315, 327, 15, 135, 207, 214, 269, 123, 246, 39, 325, 221, 299, 21, 88, 160, 83, 147, 346, 196, 234, 13, 175, 296, 221, 122, 315, 192, 243, 300, 158, 137, 284, 130, 305, 151, 128, 221, 323, 167, 86, 87, 65, 308, 22, 132, 255, 314, 254, 350, 170, 205, 330, 112, 75, 316, 249, 89, 317, 6, 185, 355, 229, 179, 127, 344, 168, 34, 313, 378, 383, 98, 193, 373, 281, 287, 179, 127, 372, 88, 282, 190, 52, 306, 141, 101, 111, 210, 376, 179, 127, 51, 161, 258, 348, 47, 126, 342, 352, 272, 277, 189, 338, 127, 278, 55, 228, 150, 260, 174, 199, 310, 339, 347, 94, 338, 351, 344, 57, 177, 379, 173, 72, 148, 70, 140, 35, 155, 338, 165, 195, 333, 42, 121, 139, 154, 359, 311, 223, 318, 53, 46, 326, 198, 263, 43, 90, 328, 275, 240, 99, 24, 134, 217, 46, 247, 324, 237, 169, 114, 368, 280, 295, 301, 312, 62, 41] +layer_updates_per_iter: 0 +num_slots: 416 diff --git a/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots384.yaml b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots384.yaml new file mode 100644 index 000000000000..c55d3f2ebf7f --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 356, 245, 279, 17, 49, 224, 150, 151, 345, 86, 348, 207, 241, 332, 163, 347, 153, 104, 62, 232, 321, 376, 176, 278, 198, 280, 282, 55, 143, 205, 54, 11, 105, 313, 56, 354, 297, 129, 258, 324, 368, 7, 170, 149, 309, 314, 38, 52, 5, 91, 172, 227, 4, 108, 249, 363, 372, 275, 59, 215, 97, 106, 270, 299, 73, 267, 67, 260, 50, 252, 98, 294, 352, 316, 340, 380, 12, 219, 179, 381, 9, 357, 250, 298, 92, 287, 24, 228, 128, 344, 288, 118, 13, 110, 303, 111, 222, 225, 45, 79, 203, 217, 0, 302, 322, 155, 162, 264, 230, 312, 188, 35, 21, 202, 80, 218, 255, 70, 139, 293, 318, 3, 349, 336, 78, 72, 243, 117, 34, 283, 156, 10, 124, 265, 200, 261, 114, 248, 337, 123, 268, 89, 234, 66, 39, 133, 244, 335, 103, 146, 331, 116, 178, 216, 192, 296, 100, 379, 292, 83, 147, 326, 237, 342, 20, 210, 286, 338, 42, 212, 253, 53, 41, 113, 334, 194, 231, 285, 164, 18, 142, 304, 121, 281, 364, 273, 130, 370, 171, 201, 247, 378, 74, 371, 135, 95, 184, 173, 259, 351, 84, 119, 235, 193, 346, 182, 169, 51, 26, 204, 242, 327, 289, 65, 63, 37, 333, 144, 209, 360, 359, 307, 284, 131, 196, 341, 233, 109, 186, 361, 311, 112, 138, 44, 148, 31, 27, 199, 358, 87, 85, 308, 257, 213, 136, 240, 365, 140, 246, 19, 366, 180, 238, 320, 195, 82, 127, 328, 263, 362, 47, 102, 350, 161, 40, 64, 206, 315, 175, 2, 152, 301, 239, 220, 125, 167, 271, 329, 226, 291, 339, 8, 88, 310, 58, 373, 61, 166, 256, 236, 254, 60, 134, 330, 290, 71, 159, 383, 177, 76, 158, 190, 295, 122, 81, 30, 29, 157, 48, 343, 68, 276, 16, 272, 126, 22, 141, 181, 33, 101, 107, 183, 43, 57, 90, 36, 174, 94, 28, 300, 208, 77, 160, 262, 375, 223, 377, 187, 32, 214, 317, 46, 137, 75, 269, 99, 369, 211, 1, 25, 221, 197, 382, 93, 14, 145, 6, 154, 229, 305, 15, 96, 115, 306, 353, 69, 191, 319, 367, 266, 251, 132, 23, 189, 374, 325, 355, 323, 168, 277, 120, 274, 185] + 1: [163, 137, 382, 177, 67, 182, 225, 178, 380, 1, 308, 126, 149, 152, 254, 40, 6, 366, 70, 123, 73, 294, 299, 271, 185, 283, 29, 138, 269, 275, 86, 242, 125, 281, 74, 212, 197, 288, 261, 109, 114, 213, 176, 228, 289, 306, 376, 278, 236, 95, 205, 189, 164, 79, 348, 330, 65, 139, 286, 326, 19, 175, 33, 321, 290, 35, 230, 350, 352, 94, 323, 346, 344, 240, 333, 103, 83, 369, 331, 129, 316, 90, 151, 221, 132, 361, 16, 45, 258, 159, 28, 157, 312, 345, 265, 255, 319, 22, 277, 169, 347, 377, 82, 272, 81, 31, 18, 106, 50, 196, 378, 168, 39, 179, 75, 41, 105, 334, 250, 153, 351, 224, 46, 273, 9, 287, 71, 173, 216, 78, 285, 301, 218, 108, 92, 335, 143, 4, 53, 116, 381, 69, 360, 342, 61, 231, 237, 76, 121, 362, 305, 238, 338, 161, 318, 17, 156, 295, 165, 62, 48, 186, 293, 11, 97, 96, 8, 113, 54, 262, 49, 356, 357, 375, 372, 220, 340, 188, 373, 135, 66, 124, 155, 191, 144, 276, 359, 257, 199, 310, 15, 215, 365, 174, 180, 72, 127, 3, 111, 134, 141, 200, 118, 80, 263, 291, 329, 131, 30, 5, 325, 328, 110, 300, 244, 248, 233, 383, 243, 59, 303, 211, 260, 89, 234, 317, 355, 146, 166, 162, 142, 379, 214, 219, 47, 282, 358, 88, 38, 171, 101, 133, 102, 298, 122, 140, 170, 204, 364, 249, 332, 107, 115, 229, 154, 68, 87, 324, 315, 26, 104, 227, 181, 268, 226, 353, 77, 27, 148, 279, 371, 309, 193, 183, 253, 209, 208, 128, 322, 374, 307, 349, 339, 304, 267, 363, 327, 368, 42, 201, 158, 91, 167, 198, 120, 292, 251, 235, 44, 85, 150, 43, 21, 241, 192, 270, 0, 210, 63, 58, 56, 23, 24, 25, 320, 195, 37, 145, 32, 247, 245, 266, 274, 99, 187, 84, 194, 280, 223, 52, 206, 93, 203, 264, 202, 239, 147, 60, 98, 256, 7, 313, 222, 259, 370, 2, 20, 14, 296, 184, 136, 57, 172, 36, 284, 51, 311, 13, 302, 55, 12, 232, 119, 314, 190, 130, 246, 297, 160, 367, 207, 217, 10, 252, 343, 64, 117, 337, 341, 354, 112, 336, 34, 100] + 2: [143, 313, 215, 3, 242, 62, 86, 230, 96, 102, 28, 5, 189, 34, 33, 149, 101, 111, 191, 166, 66, 56, 70, 135, 133, 105, 45, 368, 146, 217, 82, 24, 323, 196, 174, 294, 130, 299, 273, 87, 27, 68, 203, 210, 361, 120, 76, 138, 332, 253, 16, 171, 175, 186, 254, 129, 372, 154, 39, 142, 318, 176, 179, 2, 195, 77, 281, 35, 178, 264, 261, 57, 243, 291, 107, 95, 183, 114, 267, 307, 378, 310, 37, 177, 369, 249, 315, 266, 316, 32, 337, 354, 300, 199, 381, 258, 248, 108, 89, 295, 232, 19, 314, 122, 351, 343, 298, 259, 321, 197, 162, 192, 185, 84, 350, 173, 355, 73, 278, 271, 42, 31, 51, 22, 274, 200, 11, 235, 30, 6, 342, 204, 140, 170, 18, 347, 50, 156, 188, 320, 187, 201, 148, 198, 349, 41, 65, 241, 193, 15, 160, 276, 364, 106, 312, 252, 29, 104, 362, 23, 211, 134, 334, 238, 376, 331, 319, 231, 292, 302, 288, 99, 251, 167, 59, 98, 182, 336, 227, 222, 13, 353, 380, 88, 287, 358, 339, 237, 246, 375, 161, 379, 53, 112, 194, 223, 290, 168, 304, 139, 341, 14, 81, 221, 100, 247, 75, 78, 153, 155, 97, 284, 131, 280, 245, 118, 184, 172, 296, 26, 83, 63, 340, 244, 303, 159, 209, 165, 344, 17, 356, 263, 55, 269, 67, 92, 20, 383, 218, 297, 46, 163, 265, 308, 309, 169, 103, 69, 256, 136, 12, 49, 257, 150, 94, 0, 25, 239, 346, 4, 272, 141, 144, 352, 329, 54, 71, 371, 335, 205, 48, 74, 113, 382, 283, 345, 214, 240, 127, 64, 228, 301, 208, 202, 363, 40, 229, 374, 43, 132, 93, 115, 212, 91, 377, 180, 225, 357, 1, 137, 151, 125, 60, 213, 110, 109, 322, 124, 157, 286, 44, 306, 123, 220, 324, 325, 328, 279, 333, 85, 285, 311, 61, 79, 206, 7, 90, 224, 293, 126, 262, 327, 21, 58, 128, 236, 360, 277, 10, 145, 348, 181, 250, 116, 9, 38, 305, 121, 268, 365, 152, 219, 36, 275, 367, 47, 52, 207, 233, 359, 216, 226, 370, 373, 338, 8, 164, 158, 117, 72, 234, 147, 317, 260, 289, 80, 326, 119, 270, 282, 255, 190, 366, 330] + 3: [66, 79, 27, 194, 71, 35, 51, 350, 13, 192, 157, 363, 158, 49, 213, 337, 26, 102, 178, 342, 164, 250, 325, 138, 287, 74, 177, 119, 290, 29, 202, 87, 364, 84, 142, 359, 229, 312, 92, 155, 181, 41, 274, 217, 235, 166, 94, 355, 273, 135, 48, 268, 125, 144, 366, 43, 216, 353, 20, 361, 281, 232, 7, 83, 381, 259, 110, 321, 86, 101, 283, 354, 183, 379, 251, 73, 156, 191, 329, 143, 241, 368, 75, 89, 124, 315, 60, 152, 33, 226, 332, 233, 324, 59, 311, 370, 34, 22, 18, 313, 275, 314, 31, 225, 208, 333, 348, 318, 334, 252, 6, 137, 139, 238, 343, 120, 358, 10, 189, 231, 221, 239, 316, 36, 64, 240, 264, 136, 78, 162, 289, 288, 32, 262, 374, 297, 345, 204, 108, 299, 55, 380, 197, 169, 105, 307, 113, 255, 383, 328, 100, 306, 248, 286, 187, 70, 67, 294, 15, 376, 61, 356, 68, 228, 153, 254, 133, 285, 245, 116, 220, 88, 234, 242, 45, 360, 253, 1, 95, 127, 265, 320, 69, 173, 167, 341, 335, 257, 198, 278, 47, 301, 310, 365, 319, 40, 3, 263, 209, 14, 19, 56, 97, 62, 151, 145, 50, 331, 42, 186, 201, 207, 336, 2, 227, 267, 212, 77, 296, 219, 182, 25, 21, 382, 271, 279, 161, 172, 99, 277, 351, 114, 176, 98, 375, 218, 103, 256, 53, 57, 327, 65, 117, 246, 44, 104, 214, 160, 196, 340, 96, 298, 188, 305, 367, 106, 165, 130, 85, 346, 362, 293, 269, 206, 190, 0, 134, 372, 140, 121, 338, 159, 369, 54, 90, 148, 272, 308, 349, 244, 326, 185, 5, 17, 52, 199, 292, 222, 260, 377, 39, 302, 163, 46, 230, 171, 317, 131, 261, 80, 24, 150, 81, 180, 215, 339, 63, 168, 170, 58, 211, 37, 122, 304, 195, 174, 11, 223, 12, 118, 357, 109, 8, 154, 295, 203, 149, 111, 243, 300, 126, 224, 200, 236, 344, 270, 112, 129, 266, 280, 76, 30, 82, 237, 276, 115, 322, 184, 249, 128, 291, 91, 123, 373, 132, 72, 282, 28, 309, 93, 258, 179, 378, 16, 323, 210, 107, 284, 141, 205, 147, 330, 352, 23, 175, 4, 347, 193, 371, 247, 9, 146, 303, 38] + 4: [343, 296, 15, 87, 142, 188, 37, 199, 24, 250, 172, 281, 5, 171, 143, 348, 342, 13, 197, 155, 139, 41, 77, 372, 361, 364, 218, 160, 333, 8, 73, 251, 110, 260, 159, 141, 54, 244, 243, 116, 307, 53, 330, 256, 185, 359, 149, 323, 269, 74, 232, 383, 135, 322, 288, 328, 138, 309, 294, 50, 184, 78, 51, 175, 291, 327, 237, 118, 71, 109, 286, 156, 126, 381, 115, 129, 123, 279, 173, 164, 368, 144, 102, 373, 326, 363, 299, 337, 192, 17, 79, 67, 90, 58, 104, 120, 331, 209, 95, 194, 148, 34, 12, 222, 219, 320, 228, 88, 45, 150, 165, 136, 253, 56, 329, 167, 145, 246, 318, 275, 357, 235, 83, 44, 49, 312, 161, 280, 35, 317, 38, 179, 183, 358, 221, 42, 133, 166, 356, 114, 379, 93, 60, 341, 64, 125, 131, 262, 100, 25, 284, 367, 178, 39, 267, 52, 230, 206, 29, 103, 170, 249, 210, 334, 111, 268, 355, 213, 186, 96, 378, 97, 112, 316, 1, 191, 366, 30, 245, 190, 285, 236, 195, 154, 2, 225, 242, 338, 63, 182, 277, 282, 3, 214, 371, 226, 273, 254, 340, 375, 258, 350, 193, 9, 376, 257, 6, 241, 86, 345, 59, 274, 325, 76, 61, 106, 151, 196, 314, 43, 27, 28, 344, 68, 7, 82, 187, 153, 278, 324, 276, 0, 85, 252, 168, 72, 62, 297, 33, 75, 261, 108, 216, 313, 80, 354, 158, 137, 370, 292, 92, 349, 263, 234, 121, 157, 122, 146, 140, 289, 147, 223, 40, 127, 255, 81, 311, 21, 374, 163, 362, 162, 55, 238, 134, 248, 293, 301, 266, 270, 174, 20, 208, 231, 321, 336, 305, 332, 271, 315, 23, 319, 117, 152, 18, 124, 300, 259, 16, 239, 19, 105, 308, 302, 36, 47, 377, 298, 347, 113, 4, 200, 32, 365, 57, 101, 339, 204, 217, 198, 22, 369, 177, 207, 283, 335, 99, 264, 290, 65, 189, 360, 98, 352, 169, 14, 132, 46, 229, 70, 69, 240, 94, 310, 84, 220, 66, 306, 176, 346, 303, 26, 295, 180, 181, 304, 211, 31, 201, 130, 10, 215, 205, 203, 48, 353, 380, 247, 91, 265, 202, 11, 224, 212, 89, 382, 227, 119, 233, 128, 272, 287, 107, 351] + 5: [61, 84, 346, 276, 269, 272, 21, 165, 308, 160, 81, 60, 370, 381, 78, 102, 215, 279, 252, 7, 246, 192, 338, 182, 327, 304, 131, 70, 151, 375, 332, 238, 181, 319, 251, 207, 324, 320, 333, 220, 33, 104, 46, 79, 266, 216, 329, 183, 230, 106, 239, 250, 341, 219, 76, 136, 281, 334, 48, 360, 134, 29, 271, 55, 205, 374, 299, 263, 107, 342, 187, 92, 224, 189, 347, 101, 133, 126, 267, 307, 311, 146, 340, 294, 117, 310, 339, 280, 166, 225, 167, 277, 286, 196, 303, 75, 213, 249, 194, 105, 52, 6, 159, 141, 256, 1, 65, 223, 188, 169, 122, 64, 130, 351, 17, 43, 293, 247, 335, 22, 124, 173, 4, 155, 37, 222, 358, 264, 69, 67, 318, 24, 210, 34, 153, 315, 54, 295, 99, 144, 51, 242, 345, 227, 14, 343, 362, 240, 63, 138, 203, 325, 254, 119, 149, 236, 344, 77, 191, 349, 359, 113, 323, 378, 199, 68, 168, 58, 148, 322, 32, 245, 185, 139, 285, 73, 243, 350, 357, 42, 180, 10, 116, 112, 348, 164, 82, 157, 337, 118, 127, 44, 128, 190, 296, 100, 356, 231, 172, 208, 5, 260, 47, 265, 262, 365, 372, 41, 72, 145, 352, 200, 89, 147, 198, 25, 31, 233, 66, 137, 171, 163, 59, 2, 108, 206, 204, 143, 26, 314, 152, 377, 301, 35, 195, 186, 380, 274, 93, 39, 20, 353, 201, 121, 257, 161, 317, 289, 275, 330, 80, 87, 361, 94, 229, 62, 235, 221, 71, 114, 18, 135, 40, 298, 261, 125, 95, 292, 11, 111, 197, 85, 328, 336, 158, 234, 0, 273, 27, 367, 142, 278, 96, 179, 23, 321, 368, 302, 214, 355, 217, 305, 287, 288, 259, 154, 306, 98, 178, 49, 193, 30, 211, 379, 255, 132, 212, 88, 331, 91, 364, 74, 209, 371, 8, 313, 15, 268, 369, 174, 184, 38, 140, 3, 175, 258, 45, 218, 16, 228, 28, 115, 12, 312, 253, 19, 56, 366, 382, 300, 241, 150, 284, 90, 383, 237, 9, 13, 291, 162, 232, 376, 363, 283, 103, 129, 270, 244, 156, 316, 297, 373, 86, 290, 53, 120, 309, 97, 282, 36, 202, 123, 109, 226, 248, 110, 83, 176, 326, 57, 50, 170, 177, 354] + 6: [97, 32, 343, 240, 66, 177, 5, 249, 269, 11, 326, 120, 350, 45, 163, 318, 232, 293, 189, 112, 319, 34, 178, 289, 156, 0, 270, 113, 179, 369, 182, 336, 317, 223, 30, 202, 282, 383, 144, 211, 143, 324, 154, 58, 210, 361, 104, 110, 276, 335, 330, 60, 355, 14, 145, 126, 18, 83, 124, 147, 349, 215, 299, 54, 73, 310, 39, 235, 80, 62, 251, 360, 266, 24, 195, 351, 89, 201, 263, 38, 7, 140, 91, 93, 273, 382, 228, 265, 252, 149, 25, 33, 320, 71, 40, 90, 142, 376, 174, 258, 233, 297, 59, 160, 341, 55, 205, 31, 9, 36, 307, 13, 88, 100, 368, 84, 255, 287, 291, 129, 259, 345, 277, 165, 185, 352, 245, 278, 311, 304, 37, 111, 321, 44, 64, 107, 264, 98, 220, 157, 271, 22, 206, 242, 225, 197, 23, 4, 362, 138, 316, 148, 48, 333, 198, 358, 298, 96, 268, 272, 134, 366, 285, 357, 106, 203, 347, 85, 17, 102, 303, 51, 313, 356, 302, 78, 26, 171, 19, 227, 344, 200, 61, 2, 314, 135, 167, 52, 275, 323, 161, 164, 308, 274, 53, 253, 99, 146, 295, 27, 267, 229, 209, 67, 173, 375, 359, 181, 187, 367, 57, 283, 118, 208, 155, 184, 370, 151, 301, 176, 128, 377, 380, 236, 49, 68, 12, 158, 133, 139, 114, 219, 15, 10, 286, 280, 6, 87, 207, 137, 116, 63, 243, 217, 194, 261, 122, 248, 199, 105, 193, 224, 8, 16, 381, 309, 364, 123, 312, 74, 150, 288, 328, 290, 117, 162, 338, 21, 121, 222, 339, 238, 29, 192, 127, 241, 340, 170, 218, 72, 65, 75, 214, 186, 315, 331, 247, 257, 322, 76, 305, 281, 166, 212, 103, 306, 28, 46, 180, 371, 175, 378, 260, 216, 94, 119, 56, 246, 188, 196, 79, 239, 300, 234, 95, 354, 109, 115, 41, 230, 231, 101, 279, 190, 169, 221, 77, 327, 332, 136, 296, 365, 262, 69, 43, 237, 159, 284, 82, 86, 254, 372, 329, 81, 168, 346, 3, 50, 334, 42, 70, 125, 379, 292, 244, 130, 152, 108, 226, 337, 256, 294, 348, 47, 183, 131, 250, 374, 92, 213, 141, 1, 172, 191, 20, 353, 153, 132, 363, 204, 35, 325, 342, 373] + 7: [327, 275, 41, 229, 340, 103, 163, 162, 258, 361, 260, 342, 139, 99, 97, 328, 244, 27, 181, 281, 11, 313, 164, 82, 248, 262, 168, 55, 84, 2, 319, 374, 383, 208, 177, 224, 90, 344, 333, 214, 24, 368, 314, 109, 358, 19, 32, 190, 211, 83, 335, 305, 67, 53, 353, 232, 251, 373, 65, 291, 87, 101, 52, 120, 37, 126, 268, 209, 363, 112, 69, 218, 85, 254, 111, 39, 252, 381, 78, 159, 195, 95, 45, 57, 352, 331, 143, 282, 34, 376, 236, 179, 135, 371, 338, 42, 7, 182, 263, 142, 382, 201, 298, 157, 308, 239, 176, 28, 13, 118, 48, 329, 77, 370, 256, 73, 14, 249, 113, 217, 289, 29, 349, 23, 270, 148, 250, 235, 117, 21, 221, 62, 330, 264, 98, 206, 9, 321, 311, 140, 185, 130, 129, 5, 220, 200, 102, 44, 141, 134, 306, 301, 283, 8, 227, 146, 116, 93, 193, 203, 79, 318, 35, 216, 33, 75, 36, 295, 88, 312, 173, 379, 297, 124, 63, 128, 198, 304, 230, 66, 290, 197, 127, 54, 122, 223, 145, 30, 365, 202, 171, 60, 274, 175, 64, 265, 105, 25, 369, 272, 336, 133, 187, 46, 280, 378, 354, 212, 104, 174, 114, 131, 183, 294, 307, 100, 125, 74, 119, 285, 337, 339, 296, 284, 70, 210, 170, 213, 158, 191, 243, 167, 234, 89, 26, 204, 136, 237, 180, 107, 322, 273, 58, 6, 51, 121, 56, 155, 161, 40, 309, 68, 149, 153, 225, 351, 184, 222, 151, 106, 286, 169, 194, 1, 10, 96, 367, 43, 317, 303, 228, 315, 242, 299, 332, 150, 205, 165, 138, 144, 76, 156, 279, 160, 91, 348, 357, 50, 49, 292, 192, 375, 343, 18, 362, 341, 238, 108, 350, 271, 253, 71, 15, 188, 0, 61, 94, 302, 324, 47, 215, 267, 278, 4, 17, 31, 288, 360, 325, 356, 81, 255, 219, 377, 372, 300, 241, 132, 259, 189, 16, 245, 92, 20, 186, 178, 137, 293, 380, 86, 152, 226, 310, 323, 233, 196, 247, 261, 22, 38, 345, 366, 115, 269, 320, 166, 287, 207, 316, 72, 59, 123, 80, 231, 3, 277, 334, 266, 246, 359, 346, 147, 172, 12, 347, 199, 355, 154, 326, 240, 276, 110, 257, 364] + 8: [360, 135, 239, 321, 308, 229, 273, 3, 292, 106, 224, 173, 272, 282, 36, 122, 252, 95, 316, 44, 364, 115, 131, 195, 128, 366, 75, 150, 109, 283, 119, 179, 330, 169, 165, 215, 304, 189, 183, 196, 130, 164, 233, 197, 2, 66, 4, 42, 101, 217, 212, 174, 379, 248, 257, 249, 184, 149, 47, 382, 266, 327, 68, 199, 79, 339, 35, 145, 306, 14, 369, 54, 365, 153, 351, 220, 84, 103, 244, 208, 26, 285, 70, 305, 142, 53, 139, 82, 288, 210, 242, 227, 376, 58, 96, 207, 253, 31, 264, 162, 221, 371, 281, 250, 218, 302, 108, 312, 243, 88, 64, 276, 20, 223, 380, 18, 328, 97, 156, 39, 166, 181, 69, 83, 23, 34, 6, 258, 92, 261, 65, 275, 303, 112, 198, 194, 317, 267, 120, 177, 341, 91, 114, 61, 59, 206, 254, 230, 237, 116, 232, 293, 151, 110, 259, 37, 190, 126, 16, 143, 144, 325, 378, 205, 295, 9, 29, 185, 323, 33, 277, 146, 15, 171, 370, 77, 329, 245, 354, 343, 43, 80, 331, 344, 256, 49, 57, 176, 163, 63, 102, 234, 290, 67, 367, 362, 62, 353, 204, 104, 279, 338, 7, 13, 241, 381, 337, 81, 291, 186, 255, 209, 11, 168, 280, 374, 225, 346, 172, 188, 133, 226, 38, 240, 298, 78, 216, 178, 219, 368, 167, 50, 320, 48, 141, 211, 125, 358, 72, 8, 45, 335, 383, 98, 28, 25, 247, 90, 297, 352, 310, 41, 307, 155, 313, 56, 99, 200, 85, 89, 236, 175, 93, 347, 134, 287, 326, 76, 193, 271, 129, 270, 269, 180, 160, 363, 51, 138, 342, 301, 222, 289, 0, 55, 296, 1, 132, 170, 359, 5, 21, 201, 10, 152, 182, 214, 246, 238, 161, 12, 336, 154, 228, 74, 30, 251, 157, 349, 294, 377, 86, 148, 311, 107, 22, 356, 27, 333, 147, 231, 202, 17, 121, 40, 278, 137, 309, 113, 300, 334, 192, 318, 332, 52, 60, 284, 187, 263, 117, 118, 87, 73, 191, 71, 350, 373, 32, 299, 136, 372, 314, 140, 105, 265, 235, 159, 260, 375, 322, 46, 315, 357, 111, 24, 324, 348, 158, 340, 319, 268, 123, 286, 355, 203, 94, 274, 124, 345, 361, 127, 19, 262, 213, 100] + 9: [92, 226, 47, 316, 244, 147, 81, 220, 280, 19, 357, 165, 184, 29, 15, 201, 138, 159, 281, 341, 136, 132, 177, 309, 9, 133, 82, 158, 363, 356, 120, 289, 264, 334, 231, 6, 364, 87, 116, 145, 186, 323, 207, 206, 124, 66, 189, 72, 106, 104, 343, 308, 208, 246, 50, 65, 52, 60, 275, 373, 174, 322, 254, 114, 371, 301, 251, 305, 224, 333, 211, 221, 347, 69, 188, 337, 290, 191, 368, 225, 345, 118, 328, 101, 93, 44, 194, 294, 91, 250, 61, 27, 236, 168, 338, 26, 304, 31, 295, 90, 64, 10, 166, 96, 372, 172, 353, 74, 179, 37, 272, 146, 217, 380, 210, 51, 67, 282, 56, 102, 98, 367, 317, 360, 135, 154, 3, 292, 297, 306, 54, 321, 314, 311, 259, 129, 269, 105, 4, 123, 230, 181, 238, 141, 237, 359, 80, 13, 276, 248, 354, 307, 379, 117, 205, 233, 300, 79, 203, 115, 42, 110, 12, 330, 298, 270, 162, 229, 346, 335, 241, 374, 293, 383, 243, 43, 325, 279, 361, 223, 149, 284, 273, 40, 362, 7, 122, 53, 77, 24, 365, 57, 255, 355, 263, 185, 49, 103, 78, 143, 315, 260, 299, 219, 155, 381, 198, 83, 127, 242, 329, 214, 121, 14, 32, 342, 131, 151, 285, 30, 377, 153, 268, 169, 88, 228, 0, 128, 183, 97, 20, 99, 142, 256, 139, 16, 70, 11, 171, 332, 288, 202, 39, 319, 339, 18, 175, 109, 100, 382, 45, 46, 190, 261, 358, 291, 84, 253, 71, 352, 310, 125, 38, 89, 48, 178, 59, 86, 34, 326, 68, 140, 176, 366, 63, 119, 278, 85, 378, 257, 164, 156, 265, 55, 152, 126, 340, 173, 349, 327, 94, 313, 262, 95, 212, 331, 196, 215, 252, 350, 222, 351, 344, 1, 376, 370, 287, 271, 209, 23, 239, 22, 258, 312, 161, 144, 187, 302, 137, 33, 277, 318, 180, 247, 192, 107, 41, 163, 35, 195, 267, 336, 36, 150, 111, 218, 274, 5, 75, 324, 17, 235, 8, 130, 303, 245, 148, 197, 62, 213, 112, 2, 320, 286, 375, 134, 76, 216, 21, 232, 199, 227, 167, 170, 193, 160, 58, 108, 240, 266, 249, 73, 296, 204, 200, 113, 28, 157, 25, 182, 369, 283, 348, 234] + 10: [319, 55, 24, 18, 310, 294, 268, 108, 289, 279, 331, 57, 102, 22, 134, 161, 117, 40, 138, 330, 124, 304, 308, 91, 67, 312, 234, 240, 348, 356, 377, 148, 77, 212, 376, 359, 382, 155, 250, 90, 103, 129, 107, 3, 56, 81, 85, 27, 315, 357, 59, 327, 41, 51, 371, 109, 340, 318, 184, 254, 317, 50, 362, 261, 298, 113, 257, 209, 326, 375, 80, 111, 127, 305, 147, 217, 322, 6, 16, 34, 94, 120, 44, 347, 30, 143, 202, 260, 270, 89, 160, 253, 291, 295, 29, 283, 5, 264, 265, 286, 146, 255, 137, 221, 325, 167, 309, 225, 73, 8, 170, 54, 263, 114, 66, 210, 99, 47, 248, 58, 115, 243, 272, 136, 368, 274, 342, 204, 360, 38, 306, 223, 149, 132, 199, 198, 116, 214, 135, 105, 288, 311, 106, 314, 329, 112, 378, 328, 74, 97, 383, 278, 42, 12, 354, 332, 45, 239, 123, 13, 370, 244, 307, 169, 131, 159, 302, 235, 187, 355, 205, 224, 4, 158, 133, 203, 118, 273, 157, 344, 361, 150, 63, 277, 28, 43, 238, 166, 176, 349, 68, 336, 300, 194, 151, 216, 164, 192, 93, 48, 262, 69, 367, 92, 79, 215, 1, 178, 78, 351, 241, 182, 237, 211, 269, 213, 76, 353, 130, 252, 363, 374, 186, 208, 190, 231, 271, 280, 285, 266, 145, 0, 33, 72, 46, 341, 296, 75, 122, 335, 339, 197, 20, 338, 37, 230, 15, 71, 162, 125, 39, 174, 140, 313, 163, 343, 175, 220, 65, 299, 350, 100, 62, 173, 206, 373, 303, 292, 35, 200, 245, 276, 334, 172, 83, 207, 352, 152, 227, 14, 52, 188, 110, 236, 82, 290, 358, 297, 320, 365, 180, 121, 31, 301, 246, 181, 141, 337, 126, 2, 101, 144, 195, 179, 171, 153, 19, 177, 196, 154, 201, 380, 7, 293, 11, 259, 345, 284, 49, 316, 98, 189, 249, 218, 61, 32, 128, 229, 233, 87, 321, 21, 232, 219, 60, 142, 9, 256, 26, 346, 379, 25, 267, 258, 36, 247, 64, 193, 275, 323, 95, 183, 185, 282, 226, 364, 96, 287, 139, 222, 372, 86, 119, 17, 168, 165, 251, 53, 88, 104, 281, 333, 191, 84, 156, 10, 242, 228, 381, 70, 23, 324, 366, 369] + 11: [130, 339, 215, 221, 116, 126, 156, 260, 363, 65, 326, 145, 325, 162, 332, 345, 331, 192, 343, 128, 39, 286, 318, 146, 154, 169, 23, 320, 94, 66, 280, 213, 348, 350, 80, 92, 93, 176, 56, 86, 341, 224, 14, 55, 208, 58, 209, 327, 362, 211, 30, 279, 338, 359, 179, 302, 164, 125, 120, 57, 129, 174, 355, 188, 309, 237, 231, 271, 266, 5, 106, 210, 21, 240, 104, 132, 272, 268, 114, 328, 180, 216, 139, 53, 253, 333, 122, 76, 212, 232, 137, 257, 61, 307, 105, 335, 314, 377, 83, 85, 11, 100, 357, 299, 88, 149, 119, 233, 199, 267, 24, 22, 275, 54, 284, 165, 73, 204, 282, 177, 17, 207, 250, 313, 161, 346, 124, 45, 249, 127, 159, 27, 195, 26, 251, 10, 198, 89, 298, 365, 186, 101, 242, 97, 147, 63, 59, 292, 353, 364, 193, 311, 187, 234, 264, 64, 43, 113, 62, 182, 143, 8, 19, 382, 200, 37, 323, 222, 288, 4, 140, 290, 278, 304, 183, 379, 60, 175, 246, 181, 111, 189, 276, 155, 329, 220, 321, 50, 308, 134, 243, 241, 296, 98, 87, 150, 36, 248, 15, 252, 376, 40, 38, 171, 300, 29, 0, 236, 247, 217, 141, 52, 131, 70, 337, 228, 205, 136, 78, 303, 6, 283, 371, 107, 370, 2, 75, 226, 315, 166, 374, 160, 133, 49, 197, 289, 259, 254, 270, 301, 295, 219, 168, 366, 354, 378, 158, 91, 324, 291, 115, 383, 48, 142, 263, 7, 74, 230, 358, 274, 103, 375, 84, 356, 173, 185, 322, 167, 380, 305, 277, 1, 238, 344, 352, 317, 372, 202, 82, 123, 153, 223, 25, 256, 109, 32, 42, 269, 163, 110, 90, 235, 121, 44, 285, 77, 227, 3, 99, 190, 157, 102, 218, 144, 152, 351, 178, 330, 244, 95, 381, 20, 13, 148, 196, 297, 47, 319, 170, 367, 262, 69, 117, 373, 172, 347, 151, 340, 316, 203, 255, 96, 31, 41, 81, 35, 336, 294, 245, 265, 312, 34, 135, 349, 118, 67, 293, 79, 258, 33, 225, 18, 138, 46, 194, 184, 360, 108, 306, 112, 310, 201, 287, 214, 68, 71, 342, 229, 368, 361, 369, 16, 51, 28, 239, 206, 334, 281, 72, 273, 191, 261, 9, 12] + 12: [255, 190, 127, 378, 172, 341, 245, 219, 109, 22, 296, 103, 50, 90, 198, 170, 354, 295, 134, 278, 216, 346, 268, 25, 147, 266, 364, 27, 2, 160, 300, 104, 107, 63, 135, 308, 238, 23, 301, 144, 358, 253, 357, 77, 1, 191, 114, 0, 45, 224, 313, 280, 232, 37, 321, 11, 214, 121, 159, 305, 379, 293, 112, 244, 329, 207, 217, 374, 106, 285, 44, 375, 345, 55, 96, 54, 117, 236, 258, 79, 182, 184, 118, 132, 163, 328, 275, 92, 377, 56, 66, 99, 250, 110, 272, 42, 204, 149, 78, 177, 150, 46, 49, 381, 28, 168, 267, 349, 71, 192, 51, 306, 208, 259, 3, 339, 382, 205, 276, 152, 317, 323, 16, 89, 153, 209, 265, 171, 264, 271, 164, 294, 80, 291, 31, 74, 286, 309, 304, 227, 188, 9, 52, 315, 29, 84, 298, 138, 136, 4, 187, 58, 319, 38, 355, 351, 20, 140, 334, 336, 365, 246, 297, 279, 274, 174, 70, 310, 34, 8, 93, 113, 212, 237, 320, 367, 143, 210, 26, 269, 360, 314, 100, 213, 166, 21, 64, 239, 373, 366, 225, 372, 120, 327, 226, 82, 243, 203, 363, 35, 17, 175, 292, 289, 97, 233, 151, 369, 260, 176, 123, 6, 281, 332, 348, 215, 218, 368, 60, 10, 39, 288, 101, 371, 322, 273, 180, 129, 261, 337, 59, 318, 155, 15, 193, 376, 200, 350, 231, 234, 83, 119, 67, 62, 262, 249, 173, 325, 76, 122, 115, 270, 13, 65, 12, 32, 5, 186, 312, 263, 24, 352, 146, 105, 30, 282, 178, 131, 81, 299, 125, 230, 19, 307, 94, 145, 221, 202, 342, 248, 359, 95, 141, 240, 43, 53, 199, 302, 130, 158, 303, 223, 69, 333, 156, 48, 344, 167, 124, 126, 287, 290, 361, 181, 98, 340, 75, 195, 68, 137, 194, 330, 229, 87, 108, 277, 133, 33, 102, 116, 362, 57, 197, 241, 326, 380, 157, 14, 91, 283, 201, 142, 370, 254, 185, 41, 165, 61, 189, 256, 161, 139, 356, 169, 335, 235, 40, 154, 36, 257, 183, 86, 179, 284, 128, 148, 85, 383, 252, 251, 47, 316, 343, 18, 347, 72, 211, 247, 196, 7, 111, 338, 242, 228, 206, 88, 311, 324, 331, 353, 222, 162, 73, 220] + 13: [170, 44, 14, 312, 274, 318, 3, 39, 265, 282, 201, 95, 110, 271, 366, 88, 338, 46, 306, 118, 279, 45, 227, 364, 16, 289, 273, 40, 143, 158, 363, 299, 373, 98, 210, 168, 79, 179, 43, 382, 145, 196, 107, 359, 151, 256, 146, 52, 221, 125, 303, 331, 161, 286, 326, 220, 345, 92, 259, 381, 50, 250, 187, 123, 70, 38, 73, 313, 150, 103, 119, 249, 185, 372, 383, 343, 284, 28, 63, 128, 99, 181, 20, 317, 269, 47, 156, 82, 351, 247, 301, 216, 350, 316, 7, 175, 108, 80, 362, 328, 280, 30, 194, 77, 0, 169, 51, 231, 100, 209, 134, 310, 11, 335, 162, 159, 307, 131, 111, 64, 75, 31, 69, 356, 54, 294, 67, 266, 334, 6, 378, 65, 147, 219, 240, 62, 144, 26, 225, 283, 374, 253, 327, 243, 15, 380, 37, 8, 163, 36, 337, 153, 319, 174, 257, 127, 263, 244, 308, 311, 360, 72, 115, 377, 235, 5, 365, 190, 138, 285, 85, 157, 226, 1, 60, 212, 90, 258, 224, 97, 332, 272, 354, 148, 255, 120, 277, 314, 126, 33, 155, 192, 232, 87, 329, 23, 189, 238, 137, 57, 96, 24, 21, 135, 55, 330, 102, 197, 186, 370, 171, 304, 35, 176, 295, 198, 369, 112, 297, 202, 246, 91, 281, 292, 178, 2, 300, 207, 10, 211, 237, 321, 25, 344, 262, 361, 93, 166, 183, 139, 164, 222, 223, 34, 48, 89, 68, 106, 357, 84, 242, 268, 59, 325, 113, 53, 94, 270, 217, 234, 213, 287, 104, 214, 233, 83, 167, 180, 252, 101, 203, 352, 152, 264, 347, 293, 349, 341, 368, 200, 322, 160, 66, 165, 324, 140, 78, 32, 18, 195, 114, 149, 260, 12, 9, 81, 86, 41, 172, 276, 342, 254, 340, 17, 199, 142, 298, 109, 116, 323, 58, 184, 49, 188, 275, 376, 309, 261, 229, 320, 208, 122, 42, 22, 375, 355, 339, 141, 228, 239, 291, 74, 121, 333, 177, 129, 29, 76, 241, 105, 267, 288, 248, 379, 136, 56, 305, 296, 245, 348, 133, 71, 154, 358, 13, 19, 124, 130, 61, 353, 182, 205, 4, 346, 336, 117, 215, 191, 278, 173, 251, 371, 132, 236, 290, 302, 367, 218, 193, 206, 27, 315, 230, 204] + 14: [229, 4, 110, 259, 327, 308, 147, 372, 137, 17, 268, 150, 337, 23, 254, 85, 52, 3, 64, 317, 360, 374, 44, 216, 166, 208, 13, 158, 149, 246, 343, 357, 319, 68, 370, 108, 167, 104, 40, 325, 286, 109, 296, 82, 116, 377, 129, 176, 276, 359, 316, 335, 89, 152, 313, 183, 356, 93, 376, 311, 10, 114, 302, 329, 309, 195, 36, 136, 20, 9, 19, 312, 179, 18, 132, 366, 62, 35, 84, 181, 236, 144, 219, 65, 192, 368, 131, 352, 232, 119, 196, 73, 212, 205, 88, 174, 160, 143, 204, 209, 122, 378, 275, 261, 164, 60, 112, 239, 353, 146, 228, 99, 100, 124, 97, 342, 61, 154, 182, 271, 157, 294, 220, 314, 161, 379, 215, 123, 69, 291, 326, 159, 37, 22, 339, 169, 102, 349, 178, 227, 5, 48, 241, 221, 341, 284, 86, 200, 202, 168, 264, 282, 350, 217, 240, 96, 285, 94, 214, 321, 153, 375, 338, 46, 238, 304, 14, 16, 230, 258, 45, 31, 66, 55, 367, 180, 11, 81, 247, 252, 188, 91, 134, 280, 29, 218, 138, 272, 318, 249, 260, 165, 345, 211, 15, 30, 34, 355, 71, 56, 77, 287, 383, 175, 186, 118, 142, 151, 95, 32, 177, 39, 162, 79, 279, 111, 41, 265, 125, 197, 59, 25, 323, 163, 184, 324, 369, 145, 277, 363, 333, 382, 128, 256, 172, 234, 222, 90, 156, 380, 330, 298, 0, 336, 300, 223, 191, 12, 364, 1, 63, 251, 106, 28, 6, 262, 278, 103, 171, 127, 74, 334, 290, 58, 263, 322, 80, 194, 362, 57, 92, 24, 120, 107, 320, 283, 113, 51, 243, 303, 38, 53, 292, 67, 315, 295, 289, 331, 255, 70, 76, 2, 75, 126, 155, 274, 207, 8, 199, 49, 344, 348, 235, 141, 198, 117, 231, 371, 213, 269, 21, 250, 354, 26, 270, 33, 135, 297, 206, 328, 288, 332, 201, 193, 301, 266, 133, 346, 187, 203, 245, 226, 373, 101, 347, 105, 237, 306, 225, 253, 189, 115, 242, 148, 224, 244, 248, 43, 42, 310, 173, 273, 358, 257, 365, 267, 78, 361, 305, 190, 299, 87, 281, 7, 47, 293, 185, 139, 140, 233, 381, 307, 170, 72, 27, 54, 340, 210, 351, 98, 130, 83, 121, 50] + 15: [130, 227, 57, 155, 94, 148, 298, 76, 202, 39, 38, 326, 333, 119, 121, 51, 232, 360, 139, 56, 296, 193, 256, 1, 242, 255, 230, 24, 352, 225, 338, 100, 170, 209, 305, 194, 145, 275, 59, 263, 269, 34, 268, 62, 50, 250, 12, 43, 301, 30, 313, 118, 321, 381, 267, 137, 244, 45, 272, 211, 373, 375, 358, 63, 116, 52, 65, 210, 378, 369, 374, 188, 29, 140, 176, 318, 58, 112, 222, 98, 299, 103, 223, 17, 243, 21, 302, 345, 27, 33, 245, 75, 198, 127, 368, 85, 11, 160, 251, 355, 217, 134, 290, 350, 367, 199, 185, 266, 74, 284, 146, 20, 221, 157, 23, 48, 353, 344, 311, 174, 177, 143, 18, 164, 379, 106, 351, 335, 133, 5, 49, 247, 101, 166, 117, 142, 343, 77, 331, 282, 69, 81, 254, 165, 377, 141, 191, 67, 180, 97, 365, 234, 314, 224, 123, 37, 92, 172, 84, 213, 167, 218, 239, 79, 91, 161, 89, 32, 262, 149, 187, 380, 124, 136, 330, 181, 44, 179, 304, 70, 286, 122, 219, 220, 132, 348, 25, 264, 325, 260, 356, 328, 0, 359, 362, 200, 128, 104, 340, 4, 291, 371, 332, 159, 236, 215, 60, 9, 317, 151, 376, 83, 113, 277, 342, 135, 241, 111, 109, 235, 31, 285, 357, 47, 78, 323, 28, 42, 102, 279, 86, 248, 87, 10, 150, 96, 363, 8, 206, 13, 271, 339, 349, 306, 53, 72, 308, 289, 55, 204, 169, 64, 26, 337, 316, 257, 240, 54, 278, 364, 73, 197, 153, 129, 370, 201, 293, 280, 327, 292, 66, 195, 154, 183, 307, 110, 108, 125, 156, 297, 186, 19, 46, 144, 322, 192, 93, 270, 15, 88, 324, 258, 175, 68, 261, 259, 295, 300, 189, 231, 252, 203, 90, 237, 80, 329, 16, 40, 35, 173, 168, 163, 138, 320, 196, 115, 162, 36, 207, 319, 182, 372, 238, 246, 105, 212, 2, 82, 309, 114, 229, 22, 205, 233, 341, 253, 294, 382, 107, 226, 347, 61, 354, 99, 208, 216, 383, 71, 283, 95, 334, 265, 273, 190, 3, 147, 274, 303, 184, 158, 14, 126, 312, 310, 7, 171, 361, 228, 178, 41, 120, 346, 288, 287, 214, 336, 249, 315, 152, 6, 131, 366, 281, 276] + 16: [134, 369, 234, 43, 303, 75, 68, 331, 300, 338, 49, 289, 259, 224, 250, 298, 283, 287, 213, 146, 246, 16, 226, 108, 27, 84, 306, 59, 97, 38, 37, 292, 73, 139, 0, 282, 41, 182, 34, 195, 232, 45, 42, 301, 137, 297, 4, 106, 290, 354, 228, 358, 208, 57, 214, 95, 223, 323, 319, 190, 244, 266, 240, 381, 163, 14, 88, 373, 218, 17, 113, 26, 216, 217, 50, 332, 230, 288, 169, 382, 79, 351, 130, 341, 173, 197, 327, 278, 361, 10, 191, 320, 309, 98, 3, 148, 15, 140, 92, 147, 64, 271, 162, 185, 312, 86, 129, 342, 348, 183, 78, 255, 383, 187, 141, 46, 367, 212, 346, 89, 294, 365, 52, 237, 2, 335, 316, 82, 119, 281, 159, 153, 380, 305, 76, 220, 229, 379, 372, 174, 9, 339, 370, 124, 104, 221, 151, 340, 200, 60, 211, 242, 333, 343, 83, 105, 91, 196, 193, 360, 186, 81, 127, 128, 47, 261, 258, 170, 116, 256, 77, 149, 61, 243, 352, 63, 164, 12, 23, 135, 350, 291, 179, 296, 362, 53, 345, 94, 111, 366, 156, 249, 286, 100, 262, 165, 44, 280, 67, 125, 154, 273, 90, 203, 5, 374, 274, 24, 265, 121, 364, 236, 87, 123, 180, 181, 285, 310, 321, 328, 307, 248, 99, 21, 80, 66, 155, 39, 31, 219, 8, 70, 356, 36, 334, 19, 160, 238, 96, 188, 198, 252, 375, 25, 132, 18, 102, 178, 337, 93, 272, 378, 109, 207, 201, 363, 62, 233, 157, 284, 107, 20, 131, 110, 308, 279, 176, 150, 138, 251, 33, 28, 355, 112, 40, 254, 299, 263, 72, 204, 325, 133, 13, 215, 126, 368, 257, 353, 120, 275, 152, 7, 269, 118, 11, 330, 114, 277, 144, 143, 318, 205, 35, 302, 171, 209, 189, 295, 235, 270, 347, 199, 85, 65, 314, 177, 71, 115, 51, 377, 175, 103, 58, 56, 311, 231, 184, 227, 167, 357, 122, 202, 293, 276, 22, 322, 317, 225, 264, 29, 336, 158, 54, 313, 206, 253, 267, 304, 55, 48, 142, 30, 194, 172, 315, 166, 324, 145, 268, 168, 117, 371, 359, 241, 245, 210, 260, 101, 32, 6, 349, 74, 329, 136, 192, 326, 222, 376, 161, 69, 239, 247, 1, 344] + 17: [131, 261, 250, 200, 305, 374, 138, 140, 190, 126, 91, 136, 155, 275, 185, 114, 1, 125, 189, 62, 75, 8, 56, 104, 207, 73, 218, 206, 148, 313, 211, 103, 143, 84, 272, 146, 282, 314, 162, 18, 332, 13, 166, 60, 45, 260, 215, 154, 118, 187, 221, 205, 101, 253, 21, 213, 360, 249, 195, 244, 299, 329, 16, 85, 64, 106, 278, 326, 364, 350, 63, 344, 53, 355, 316, 227, 23, 40, 277, 66, 257, 368, 11, 129, 77, 348, 111, 281, 209, 123, 34, 86, 69, 72, 121, 152, 119, 375, 303, 15, 26, 28, 80, 226, 346, 87, 219, 78, 225, 5, 343, 266, 130, 324, 133, 6, 175, 172, 52, 339, 96, 296, 335, 97, 381, 180, 380, 373, 10, 98, 33, 135, 292, 171, 245, 259, 340, 248, 47, 116, 59, 42, 269, 295, 49, 204, 377, 186, 341, 338, 342, 265, 94, 179, 262, 310, 90, 120, 161, 112, 267, 274, 369, 25, 192, 50, 150, 57, 163, 83, 153, 352, 149, 230, 356, 223, 349, 178, 242, 71, 247, 353, 268, 321, 252, 174, 54, 122, 159, 170, 202, 203, 284, 337, 188, 383, 330, 362, 307, 208, 319, 366, 254, 311, 359, 197, 232, 323, 246, 351, 29, 297, 370, 167, 290, 157, 134, 317, 22, 199, 145, 243, 38, 92, 367, 139, 298, 74, 165, 263, 70, 376, 191, 82, 61, 36, 331, 147, 7, 336, 81, 196, 115, 382, 37, 89, 181, 27, 110, 238, 302, 361, 102, 239, 210, 217, 128, 48, 308, 124, 67, 255, 318, 144, 164, 132, 241, 88, 280, 141, 201, 224, 4, 184, 160, 285, 289, 0, 35, 322, 301, 293, 169, 127, 235, 237, 251, 283, 236, 79, 220, 108, 100, 294, 212, 105, 137, 58, 12, 357, 117, 193, 286, 279, 113, 2, 347, 328, 276, 234, 222, 107, 39, 194, 46, 24, 231, 177, 228, 273, 371, 256, 99, 327, 214, 9, 142, 168, 264, 304, 19, 20, 173, 43, 41, 363, 216, 198, 365, 3, 309, 271, 95, 44, 379, 300, 93, 240, 258, 183, 345, 287, 151, 55, 158, 372, 76, 270, 31, 233, 32, 65, 315, 182, 378, 176, 109, 30, 229, 291, 288, 306, 334, 17, 51, 68, 156, 333, 14, 358, 312, 320, 325, 354] + 18: [271, 65, 106, 6, 356, 210, 315, 347, 119, 304, 239, 189, 167, 301, 139, 237, 284, 297, 144, 266, 33, 105, 240, 98, 165, 269, 20, 171, 202, 339, 121, 283, 149, 223, 233, 131, 135, 0, 22, 288, 313, 328, 61, 102, 111, 349, 205, 264, 319, 362, 132, 265, 155, 363, 270, 322, 325, 112, 275, 256, 60, 234, 332, 158, 27, 247, 126, 224, 369, 123, 76, 2, 107, 84, 370, 161, 29, 184, 294, 100, 305, 277, 72, 173, 62, 295, 198, 91, 327, 97, 222, 293, 1, 32, 59, 78, 30, 320, 245, 228, 118, 272, 16, 207, 154, 215, 110, 169, 236, 146, 68, 373, 306, 31, 151, 246, 99, 77, 67, 103, 204, 164, 89, 337, 346, 366, 248, 307, 83, 343, 79, 225, 88, 162, 137, 278, 350, 361, 58, 50, 289, 140, 203, 279, 38, 244, 93, 12, 182, 255, 187, 329, 153, 214, 166, 108, 48, 348, 143, 82, 216, 66, 219, 56, 163, 209, 133, 218, 180, 258, 86, 37, 85, 238, 273, 377, 230, 11, 117, 290, 336, 221, 196, 353, 128, 374, 120, 227, 314, 147, 309, 340, 251, 10, 69, 257, 57, 71, 185, 45, 333, 286, 113, 114, 380, 174, 116, 136, 25, 26, 34, 291, 253, 35, 330, 285, 326, 3, 188, 19, 318, 46, 41, 191, 141, 177, 104, 359, 261, 316, 263, 241, 296, 7, 232, 201, 231, 157, 39, 8, 308, 125, 178, 276, 96, 63, 40, 335, 292, 351, 148, 243, 254, 303, 127, 358, 168, 170, 338, 4, 51, 311, 197, 229, 375, 298, 312, 372, 172, 44, 90, 299, 280, 14, 345, 18, 160, 21, 206, 134, 310, 379, 186, 52, 74, 156, 242, 344, 208, 47, 43, 217, 300, 138, 260, 193, 53, 211, 28, 36, 368, 192, 142, 355, 321, 360, 376, 357, 282, 378, 341, 190, 101, 124, 70, 274, 181, 371, 365, 342, 23, 73, 331, 262, 87, 334, 129, 109, 367, 159, 122, 250, 302, 235, 152, 195, 15, 176, 54, 150, 81, 9, 145, 13, 267, 94, 287, 324, 92, 179, 213, 383, 175, 200, 64, 220, 259, 130, 17, 317, 381, 364, 75, 95, 42, 354, 115, 352, 281, 24, 5, 249, 212, 199, 252, 194, 323, 49, 55, 226, 80, 183, 268, 382] + 19: [24, 256, 120, 180, 312, 64, 211, 48, 302, 52, 291, 126, 318, 197, 66, 363, 218, 33, 296, 97, 230, 275, 189, 264, 158, 147, 246, 53, 285, 1, 278, 134, 6, 165, 167, 76, 353, 195, 257, 204, 2, 297, 4, 215, 95, 342, 336, 194, 85, 177, 23, 131, 60, 293, 280, 129, 225, 376, 362, 242, 298, 137, 209, 87, 186, 14, 208, 174, 28, 286, 371, 103, 112, 357, 267, 162, 185, 39, 224, 21, 155, 86, 56, 308, 226, 43, 199, 55, 142, 250, 378, 12, 34, 50, 81, 98, 187, 288, 265, 338, 42, 240, 141, 111, 364, 91, 161, 322, 192, 300, 113, 328, 154, 349, 227, 89, 301, 354, 35, 58, 365, 25, 335, 54, 71, 30, 222, 295, 127, 182, 68, 83, 156, 368, 138, 283, 157, 144, 259, 249, 374, 273, 84, 346, 19, 213, 45, 202, 75, 356, 3, 26, 191, 88, 200, 119, 130, 212, 22, 168, 232, 132, 234, 235, 63, 314, 37, 294, 150, 106, 11, 269, 201, 214, 252, 51, 205, 163, 236, 379, 282, 69, 220, 78, 145, 274, 93, 305, 107, 133, 198, 245, 190, 90, 57, 330, 207, 307, 243, 179, 61, 241, 255, 183, 219, 114, 373, 27, 20, 38, 303, 77, 159, 153, 217, 44, 41, 171, 238, 15, 13, 116, 128, 310, 228, 173, 96, 121, 251, 16, 272, 221, 260, 347, 277, 31, 306, 203, 325, 18, 333, 237, 10, 80, 331, 148, 47, 62, 309, 146, 326, 321, 176, 311, 32, 320, 258, 181, 332, 247, 94, 324, 59, 315, 231, 118, 124, 289, 290, 223, 304, 100, 299, 266, 248, 102, 151, 360, 327, 104, 175, 276, 109, 317, 253, 254, 263, 268, 366, 7, 377, 313, 101, 115, 339, 206, 74, 70, 188, 279, 8, 216, 184, 323, 193, 164, 334, 271, 152, 117, 79, 9, 292, 46, 135, 122, 233, 36, 352, 166, 359, 337, 270, 172, 29, 49, 170, 239, 40, 67, 345, 110, 244, 143, 287, 125, 284, 108, 380, 262, 160, 372, 92, 65, 140, 281, 329, 383, 341, 82, 139, 343, 370, 99, 105, 340, 149, 351, 73, 361, 348, 381, 210, 123, 178, 358, 367, 136, 344, 0, 5, 261, 169, 17, 72, 382, 196, 316, 229, 350, 319, 355, 369, 375] + 20: [161, 144, 87, 268, 46, 37, 317, 110, 376, 339, 130, 105, 185, 138, 162, 325, 91, 228, 273, 95, 96, 264, 227, 342, 49, 101, 172, 77, 182, 239, 176, 364, 348, 235, 378, 175, 265, 146, 58, 367, 279, 372, 22, 34, 238, 243, 165, 294, 29, 15, 5, 106, 380, 39, 3, 202, 327, 89, 84, 230, 154, 316, 366, 352, 289, 283, 319, 28, 40, 102, 205, 341, 136, 83, 16, 207, 234, 383, 295, 192, 103, 186, 306, 203, 298, 109, 269, 198, 249, 27, 51, 52, 54, 61, 76, 107, 116, 302, 66, 43, 145, 282, 174, 55, 287, 112, 291, 224, 32, 270, 371, 148, 324, 141, 128, 125, 382, 9, 286, 216, 292, 347, 92, 71, 126, 90, 74, 240, 194, 328, 26, 219, 53, 36, 359, 233, 309, 119, 18, 131, 166, 340, 345, 248, 297, 338, 232, 252, 332, 365, 201, 231, 360, 191, 223, 351, 217, 56, 210, 11, 59, 79, 21, 88, 113, 329, 271, 354, 211, 276, 19, 313, 261, 68, 108, 70, 220, 222, 344, 237, 1, 38, 143, 288, 260, 137, 167, 168, 179, 181, 189, 200, 334, 278, 82, 99, 45, 335, 122, 14, 173, 2, 379, 195, 363, 62, 188, 93, 209, 164, 111, 290, 381, 187, 180, 259, 135, 258, 184, 65, 104, 322, 357, 13, 57, 284, 35, 343, 41, 118, 368, 314, 247, 150, 300, 256, 114, 274, 346, 97, 169, 370, 310, 12, 33, 127, 73, 170, 312, 374, 337, 75, 94, 254, 80, 285, 151, 142, 115, 120, 124, 356, 333, 355, 47, 67, 225, 63, 48, 133, 293, 183, 147, 251, 275, 31, 152, 244, 44, 171, 17, 241, 277, 206, 214, 262, 263, 266, 98, 153, 336, 193, 369, 4, 8, 272, 81, 123, 226, 307, 353, 204, 318, 330, 158, 10, 229, 221, 311, 362, 50, 140, 160, 132, 320, 121, 358, 315, 190, 6, 177, 218, 267, 361, 375, 0, 25, 30, 213, 208, 178, 42, 196, 60, 215, 86, 246, 377, 20, 100, 78, 236, 304, 257, 326, 24, 212, 253, 159, 308, 139, 349, 156, 199, 250, 296, 321, 7, 85, 163, 323, 72, 305, 157, 64, 134, 373, 245, 149, 23, 129, 301, 155, 255, 69, 197, 117, 242, 280, 281, 299, 303, 331, 350] + 21: [360, 103, 371, 158, 124, 257, 256, 206, 53, 210, 200, 208, 88, 199, 214, 226, 178, 91, 177, 238, 22, 307, 147, 349, 15, 299, 293, 225, 43, 313, 141, 220, 368, 183, 243, 174, 73, 338, 18, 331, 287, 138, 60, 304, 97, 66, 383, 192, 370, 125, 347, 30, 295, 40, 100, 301, 136, 187, 79, 269, 57, 258, 126, 36, 215, 81, 83, 271, 319, 181, 342, 253, 156, 268, 74, 374, 217, 99, 37, 180, 176, 328, 6, 106, 326, 61, 212, 95, 235, 101, 127, 311, 44, 10, 17, 35, 152, 132, 308, 80, 119, 140, 190, 236, 29, 38, 218, 364, 94, 361, 232, 377, 150, 0, 240, 279, 184, 16, 365, 128, 175, 366, 227, 344, 52, 230, 111, 23, 78, 33, 233, 325, 263, 372, 49, 336, 362, 64, 223, 246, 62, 260, 286, 291, 211, 155, 378, 154, 65, 204, 171, 148, 137, 14, 102, 24, 274, 146, 216, 48, 244, 213, 134, 334, 12, 96, 315, 164, 70, 201, 330, 2, 68, 357, 56, 314, 249, 202, 242, 31, 161, 7, 116, 222, 248, 107, 167, 327, 108, 58, 93, 123, 355, 281, 20, 4, 71, 302, 289, 247, 205, 82, 92, 87, 169, 117, 120, 345, 252, 182, 149, 51, 380, 172, 26, 109, 196, 8, 194, 280, 135, 144, 367, 72, 245, 288, 270, 131, 84, 197, 267, 352, 359, 259, 262, 139, 98, 312, 303, 159, 317, 165, 5, 133, 77, 275, 266, 341, 337, 292, 198, 170, 340, 346, 261, 276, 273, 157, 27, 381, 67, 105, 382, 294, 75, 207, 13, 122, 76, 55, 114, 324, 255, 228, 265, 142, 278, 32, 166, 231, 322, 224, 283, 332, 179, 264, 306, 321, 47, 173, 316, 254, 376, 113, 153, 46, 86, 50, 112, 185, 363, 348, 277, 45, 118, 284, 250, 320, 191, 151, 351, 19, 121, 219, 25, 69, 9, 209, 221, 189, 234, 373, 89, 229, 168, 63, 1, 358, 39, 90, 309, 318, 251, 21, 356, 28, 354, 290, 323, 3, 237, 353, 115, 193, 34, 160, 104, 203, 329, 186, 335, 300, 379, 333, 305, 110, 195, 163, 310, 241, 143, 239, 41, 130, 129, 85, 375, 54, 162, 296, 285, 145, 298, 59, 188, 282, 343, 272, 297, 11, 42, 369, 339, 350] + 22: [254, 281, 209, 337, 178, 371, 51, 332, 276, 115, 242, 230, 177, 169, 29, 93, 58, 352, 50, 70, 153, 7, 311, 100, 80, 76, 8, 312, 60, 125, 185, 349, 170, 75, 113, 109, 48, 63, 258, 233, 309, 165, 12, 72, 367, 47, 305, 303, 200, 54, 189, 11, 78, 32, 0, 268, 211, 202, 89, 275, 46, 237, 262, 246, 102, 219, 151, 334, 30, 36, 152, 257, 64, 69, 171, 255, 150, 370, 5, 127, 43, 66, 277, 270, 166, 56, 194, 297, 10, 205, 289, 2, 28, 79, 95, 101, 141, 328, 278, 224, 20, 94, 204, 126, 330, 207, 42, 201, 4, 112, 329, 235, 383, 362, 333, 55, 381, 351, 285, 338, 173, 375, 195, 83, 106, 365, 116, 214, 206, 57, 228, 358, 156, 366, 191, 25, 324, 321, 41, 133, 239, 320, 248, 87, 98, 322, 269, 283, 256, 373, 45, 81, 286, 290, 215, 77, 273, 364, 265, 279, 274, 119, 266, 182, 52, 120, 62, 179, 342, 291, 90, 302, 296, 223, 6, 162, 21, 197, 146, 295, 293, 167, 86, 203, 313, 135, 249, 292, 143, 154, 157, 217, 181, 261, 216, 148, 288, 376, 316, 9, 331, 132, 38, 184, 37, 74, 139, 244, 355, 117, 240, 348, 357, 345, 188, 218, 111, 67, 161, 372, 123, 222, 17, 369, 122, 226, 175, 347, 53, 82, 220, 49, 131, 15, 99, 225, 1, 380, 341, 318, 65, 236, 199, 227, 327, 238, 299, 180, 300, 97, 271, 96, 353, 379, 298, 103, 108, 35, 160, 186, 92, 196, 129, 190, 350, 19, 104, 344, 308, 354, 88, 23, 140, 213, 158, 84, 172, 107, 243, 231, 315, 142, 280, 323, 247, 250, 253, 267, 221, 260, 343, 24, 183, 110, 33, 198, 34, 147, 164, 73, 144, 263, 118, 44, 61, 130, 317, 252, 229, 272, 149, 251, 336, 306, 68, 378, 174, 264, 193, 259, 27, 14, 361, 3, 340, 339, 40, 18, 374, 71, 59, 210, 105, 319, 39, 159, 301, 155, 22, 124, 377, 335, 382, 241, 13, 138, 314, 136, 356, 26, 176, 208, 232, 85, 187, 137, 282, 294, 145, 91, 134, 16, 168, 128, 212, 31, 363, 326, 284, 304, 163, 114, 245, 121, 368, 234, 325, 192, 287, 360, 307, 310, 346, 359] + 23: [26, 126, 100, 244, 72, 281, 274, 147, 48, 82, 306, 12, 327, 47, 176, 84, 241, 127, 372, 2, 270, 56, 130, 283, 209, 379, 144, 213, 378, 220, 191, 229, 152, 23, 342, 40, 255, 129, 359, 370, 150, 153, 254, 17, 273, 137, 178, 207, 162, 341, 196, 348, 90, 58, 344, 256, 107, 302, 108, 21, 114, 14, 268, 99, 123, 338, 227, 373, 308, 205, 49, 46, 310, 368, 230, 277, 383, 116, 247, 381, 138, 20, 208, 42, 83, 199, 112, 142, 166, 226, 113, 216, 189, 235, 81, 64, 314, 305, 175, 167, 85, 177, 69, 376, 77, 201, 173, 87, 279, 103, 261, 156, 67, 374, 169, 55, 6, 315, 78, 263, 133, 35, 326, 111, 219, 312, 91, 151, 317, 10, 228, 224, 136, 210, 356, 157, 32, 96, 160, 98, 53, 65, 301, 307, 382, 316, 284, 352, 258, 143, 0, 128, 245, 174, 206, 134, 121, 122, 297, 294, 239, 25, 139, 355, 106, 248, 140, 282, 158, 353, 34, 31, 322, 165, 334, 299, 343, 22, 289, 63, 183, 145, 332, 60, 186, 366, 234, 15, 291, 286, 242, 168, 124, 71, 264, 132, 252, 135, 51, 120, 170, 295, 240, 203, 197, 287, 154, 232, 180, 11, 57, 59, 159, 86, 257, 41, 340, 309, 146, 184, 74, 70, 36, 149, 95, 195, 38, 290, 215, 292, 44, 222, 298, 185, 329, 66, 231, 5, 325, 321, 223, 330, 193, 76, 45, 73, 109, 280, 28, 164, 7, 212, 163, 155, 349, 236, 336, 375, 33, 104, 260, 161, 188, 61, 253, 357, 88, 276, 337, 238, 259, 360, 364, 318, 30, 117, 246, 262, 217, 198, 296, 251, 29, 3, 39, 8, 320, 288, 172, 275, 328, 187, 323, 300, 225, 211, 19, 331, 24, 269, 324, 358, 27, 94, 192, 304, 182, 218, 351, 148, 371, 92, 190, 131, 80, 50, 102, 249, 233, 271, 313, 194, 237, 272, 221, 311, 101, 361, 1, 119, 93, 365, 52, 125, 333, 179, 347, 250, 204, 303, 54, 363, 339, 200, 335, 346, 16, 68, 354, 243, 362, 285, 97, 115, 369, 267, 214, 79, 367, 202, 9, 319, 62, 266, 350, 377, 171, 181, 118, 141, 345, 278, 105, 89, 293, 380, 13, 18, 110, 4, 75, 43, 37, 265] + 24: [154, 223, 52, 372, 189, 97, 111, 328, 139, 28, 27, 121, 266, 301, 29, 146, 231, 53, 78, 159, 354, 144, 91, 61, 273, 41, 106, 355, 350, 356, 149, 351, 341, 182, 307, 310, 287, 279, 376, 83, 257, 84, 13, 233, 197, 313, 1, 268, 66, 312, 108, 215, 346, 335, 267, 282, 271, 176, 88, 296, 107, 212, 188, 249, 136, 8, 148, 163, 262, 209, 288, 44, 109, 25, 18, 256, 187, 378, 240, 75, 141, 200, 316, 243, 371, 261, 364, 100, 2, 45, 21, 129, 232, 74, 81, 115, 265, 7, 6, 263, 369, 87, 211, 259, 122, 381, 324, 286, 104, 220, 375, 230, 319, 137, 202, 135, 277, 195, 15, 306, 93, 226, 131, 308, 142, 130, 96, 73, 338, 145, 12, 3, 206, 221, 140, 59, 147, 235, 269, 295, 311, 242, 51, 11, 292, 239, 60, 270, 37, 218, 170, 72, 276, 303, 120, 10, 363, 76, 284, 320, 281, 42, 67, 19, 55, 50, 234, 238, 132, 101, 68, 367, 247, 43, 174, 300, 90, 165, 227, 340, 33, 117, 305, 17, 152, 89, 127, 133, 251, 166, 198, 241, 252, 20, 92, 357, 70, 290, 110, 172, 185, 360, 361, 112, 294, 334, 62, 186, 283, 26, 31, 24, 164, 344, 382, 366, 71, 318, 254, 156, 138, 377, 39, 336, 224, 380, 245, 205, 272, 114, 321, 326, 178, 216, 168, 323, 116, 47, 244, 280, 339, 65, 225, 134, 103, 365, 333, 4, 38, 348, 150, 196, 173, 32, 151, 94, 352, 57, 331, 359, 329, 217, 291, 317, 183, 180, 184, 80, 201, 143, 228, 332, 119, 167, 285, 347, 69, 208, 322, 40, 219, 34, 155, 158, 327, 264, 298, 314, 9, 246, 0, 22, 85, 128, 95, 210, 337, 157, 105, 64, 5, 255, 370, 207, 237, 343, 213, 204, 192, 35, 190, 54, 325, 299, 274, 260, 46, 113, 49, 177, 293, 345, 118, 77, 48, 58, 297, 63, 169, 153, 16, 278, 222, 304, 315, 56, 302, 194, 30, 181, 309, 258, 203, 275, 23, 79, 14, 253, 86, 175, 161, 126, 99, 373, 191, 160, 171, 102, 193, 342, 248, 362, 236, 125, 229, 98, 162, 250, 124, 199, 289, 123, 349, 214, 330, 82, 358, 36, 374, 179, 368, 353, 379, 383] + 25: [315, 317, 107, 284, 83, 268, 85, 78, 39, 309, 152, 77, 73, 145, 235, 305, 33, 314, 318, 120, 207, 381, 346, 252, 222, 373, 56, 240, 256, 323, 367, 279, 164, 261, 348, 103, 114, 191, 115, 132, 295, 350, 68, 55, 354, 357, 15, 213, 365, 339, 201, 14, 144, 370, 31, 200, 53, 27, 361, 358, 166, 337, 271, 22, 35, 325, 250, 167, 21, 45, 87, 311, 371, 245, 174, 349, 224, 48, 1, 264, 141, 335, 42, 285, 219, 266, 148, 206, 194, 255, 338, 165, 189, 220, 123, 91, 269, 228, 276, 214, 62, 258, 334, 101, 162, 2, 52, 246, 171, 232, 190, 273, 8, 102, 380, 230, 18, 342, 154, 294, 306, 197, 377, 231, 229, 153, 138, 60, 227, 97, 307, 4, 196, 291, 225, 336, 304, 237, 199, 312, 345, 259, 184, 37, 328, 28, 116, 181, 69, 355, 105, 212, 238, 251, 340, 193, 275, 260, 111, 51, 177, 281, 34, 362, 320, 283, 226, 208, 40, 289, 64, 322, 137, 265, 63, 119, 147, 172, 71, 90, 301, 288, 50, 241, 13, 151, 43, 382, 239, 267, 158, 326, 379, 139, 180, 372, 192, 65, 298, 299, 375, 61, 0, 113, 112, 49, 5, 160, 313, 38, 32, 59, 368, 262, 287, 302, 215, 9, 84, 253, 125, 99, 244, 366, 343, 321, 218, 176, 359, 242, 20, 347, 352, 278, 316, 187, 383, 324, 330, 290, 297, 331, 58, 257, 94, 149, 24, 11, 356, 159, 249, 74, 243, 110, 353, 109, 163, 75, 36, 157, 183, 70, 210, 173, 16, 175, 236, 92, 41, 142, 95, 205, 286, 363, 374, 88, 104, 108, 272, 182, 329, 170, 195, 44, 19, 233, 263, 178, 282, 72, 247, 23, 29, 98, 168, 82, 30, 332, 25, 327, 376, 10, 333, 280, 7, 66, 296, 216, 378, 351, 100, 136, 131, 234, 341, 209, 76, 80, 89, 54, 126, 146, 161, 135, 143, 140, 211, 369, 203, 319, 81, 221, 130, 204, 308, 185, 202, 134, 254, 133, 274, 310, 67, 150, 79, 300, 223, 17, 179, 127, 248, 186, 292, 117, 93, 86, 124, 122, 96, 364, 293, 46, 128, 118, 188, 155, 3, 344, 57, 360, 26, 12, 217, 169, 198, 47, 6, 106, 270, 303, 121, 129, 156, 277] + 26: [136, 80, 18, 236, 253, 281, 63, 371, 304, 254, 116, 69, 33, 130, 38, 293, 238, 135, 22, 36, 92, 110, 332, 200, 180, 204, 198, 26, 319, 323, 151, 353, 306, 99, 327, 79, 186, 343, 194, 20, 285, 318, 277, 195, 10, 73, 378, 115, 171, 37, 256, 187, 123, 143, 372, 298, 25, 302, 264, 49, 300, 325, 362, 225, 255, 40, 334, 9, 275, 303, 39, 333, 326, 199, 101, 216, 373, 284, 14, 250, 330, 46, 164, 381, 95, 248, 207, 288, 203, 1, 54, 68, 86, 96, 106, 111, 237, 90, 380, 53, 291, 230, 5, 271, 263, 70, 160, 289, 94, 337, 308, 266, 227, 355, 315, 88, 30, 146, 328, 83, 55, 265, 148, 48, 259, 360, 321, 190, 133, 2, 356, 97, 103, 122, 201, 268, 66, 192, 15, 357, 81, 161, 35, 98, 0, 252, 196, 350, 93, 144, 232, 363, 19, 134, 336, 329, 220, 3, 140, 364, 317, 354, 72, 347, 34, 297, 331, 100, 29, 193, 13, 205, 296, 56, 313, 145, 282, 85, 222, 188, 23, 84, 286, 210, 309, 213, 139, 169, 170, 175, 209, 218, 89, 172, 292, 124, 21, 339, 233, 214, 342, 162, 149, 16, 279, 344, 137, 341, 369, 61, 158, 376, 245, 167, 131, 82, 67, 156, 31, 311, 138, 178, 191, 287, 367, 44, 27, 152, 173, 241, 118, 294, 127, 365, 17, 215, 65, 87, 45, 202, 107, 41, 370, 50, 114, 177, 182, 312, 117, 189, 368, 7, 78, 12, 105, 340, 47, 141, 109, 8, 64, 59, 77, 244, 154, 206, 91, 274, 324, 258, 377, 208, 283, 211, 345, 235, 366, 310, 322, 273, 52, 132, 260, 228, 234, 242, 246, 261, 269, 217, 6, 11, 181, 150, 320, 349, 184, 338, 224, 120, 276, 183, 316, 278, 307, 223, 226, 249, 108, 126, 382, 62, 247, 359, 128, 176, 272, 179, 166, 262, 142, 229, 174, 102, 251, 24, 221, 51, 57, 71, 104, 239, 159, 113, 212, 290, 267, 76, 32, 231, 129, 75, 153, 383, 280, 301, 157, 219, 197, 121, 125, 352, 165, 185, 335, 43, 346, 74, 112, 361, 163, 240, 119, 314, 358, 270, 42, 4, 243, 295, 28, 147, 257, 58, 155, 379, 60, 168, 299, 305, 348, 351, 374, 375] + 27: [103, 263, 265, 349, 80, 34, 104, 59, 351, 300, 269, 197, 119, 358, 131, 188, 348, 336, 237, 183, 251, 21, 303, 381, 266, 383, 216, 343, 83, 308, 32, 325, 373, 118, 337, 305, 15, 161, 162, 328, 309, 10, 246, 172, 136, 5, 127, 122, 45, 362, 239, 347, 2, 44, 318, 121, 353, 167, 367, 205, 361, 79, 151, 139, 132, 231, 184, 185, 359, 8, 74, 157, 352, 372, 68, 273, 62, 334, 87, 165, 226, 148, 170, 19, 295, 92, 371, 378, 63, 200, 236, 39, 53, 60, 137, 146, 225, 234, 37, 168, 30, 227, 354, 219, 58, 31, 374, 26, 357, 49, 366, 202, 213, 224, 143, 112, 335, 230, 290, 85, 260, 324, 196, 72, 238, 115, 332, 278, 322, 284, 356, 317, 187, 27, 261, 254, 91, 316, 326, 379, 114, 174, 218, 268, 287, 46, 18, 111, 147, 299, 120, 66, 101, 47, 365, 0, 84, 153, 116, 105, 33, 144, 281, 259, 282, 319, 22, 264, 229, 41, 110, 16, 158, 86, 166, 345, 52, 331, 56, 311, 255, 298, 169, 57, 206, 17, 93, 203, 247, 163, 176, 220, 286, 90, 4, 61, 252, 341, 301, 145, 333, 70, 233, 204, 135, 320, 181, 292, 369, 179, 97, 190, 228, 75, 178, 340, 242, 76, 99, 258, 102, 156, 78, 82, 297, 140, 141, 193, 98, 42, 7, 370, 125, 126, 54, 6, 291, 215, 296, 360, 375, 267, 67, 43, 248, 9, 199, 249, 307, 321, 217, 3, 275, 288, 159, 130, 14, 344, 113, 152, 313, 50, 223, 293, 302, 150, 289, 142, 195, 94, 189, 192, 1, 173, 245, 314, 210, 221, 355, 134, 209, 36, 100, 212, 270, 240, 244, 262, 12, 312, 177, 73, 69, 211, 81, 117, 279, 350, 164, 71, 315, 128, 376, 24, 377, 222, 207, 55, 182, 272, 48, 323, 380, 149, 339, 232, 77, 294, 23, 89, 201, 25, 109, 191, 241, 133, 155, 88, 368, 271, 329, 175, 214, 304, 274, 327, 124, 276, 129, 106, 20, 171, 28, 38, 29, 250, 95, 13, 107, 123, 35, 256, 306, 96, 280, 154, 208, 346, 65, 186, 253, 330, 364, 382, 257, 283, 198, 64, 108, 194, 138, 160, 11, 277, 40, 363, 243, 51, 180, 235, 285, 310, 338, 342] + 28: [161, 317, 302, 52, 286, 4, 201, 196, 328, 152, 97, 122, 93, 7, 199, 108, 236, 3, 320, 223, 59, 158, 58, 261, 235, 157, 285, 163, 337, 299, 32, 105, 340, 349, 308, 193, 247, 380, 316, 134, 178, 91, 14, 27, 256, 179, 264, 171, 249, 44, 314, 375, 205, 143, 61, 296, 66, 210, 334, 280, 34, 151, 382, 372, 218, 96, 266, 80, 351, 284, 292, 234, 229, 47, 192, 281, 267, 111, 140, 94, 202, 371, 83, 145, 313, 164, 186, 9, 132, 190, 10, 18, 60, 86, 115, 150, 250, 92, 70, 162, 107, 241, 355, 248, 77, 283, 120, 172, 233, 253, 224, 374, 75, 346, 378, 137, 368, 228, 165, 5, 100, 383, 361, 339, 28, 220, 41, 125, 209, 182, 216, 33, 367, 114, 127, 268, 206, 333, 282, 121, 194, 142, 46, 265, 208, 358, 57, 274, 305, 177, 362, 106, 74, 12, 270, 332, 342, 366, 154, 377, 331, 117, 89, 149, 323, 39, 293, 303, 17, 72, 276, 232, 29, 76, 275, 277, 102, 350, 53, 254, 181, 318, 198, 212, 11, 138, 298, 156, 180, 207, 215, 257, 26, 344, 153, 306, 30, 99, 321, 0, 310, 324, 188, 226, 195, 103, 214, 37, 272, 263, 50, 166, 191, 42, 227, 204, 200, 352, 88, 160, 135, 67, 289, 242, 269, 221, 56, 13, 19, 273, 315, 16, 341, 335, 222, 1, 290, 244, 365, 260, 49, 6, 81, 64, 69, 98, 381, 322, 175, 169, 104, 15, 170, 363, 24, 73, 136, 31, 330, 119, 287, 345, 110, 38, 131, 8, 85, 237, 139, 55, 101, 373, 291, 126, 65, 189, 354, 187, 319, 262, 301, 40, 147, 359, 279, 311, 329, 336, 185, 168, 356, 255, 243, 25, 197, 312, 239, 22, 203, 176, 155, 288, 113, 307, 357, 369, 128, 219, 95, 325, 71, 51, 116, 63, 129, 90, 130, 167, 370, 326, 240, 271, 217, 20, 238, 376, 278, 48, 43, 300, 68, 184, 78, 118, 183, 112, 245, 327, 23, 295, 82, 159, 21, 148, 36, 258, 230, 84, 211, 133, 297, 294, 109, 144, 79, 364, 213, 348, 251, 225, 246, 231, 146, 347, 259, 252, 174, 35, 123, 45, 141, 304, 360, 309, 87, 124, 62, 2, 54, 173, 379, 338, 343, 353] + 29: [118, 318, 323, 203, 67, 23, 255, 89, 58, 365, 354, 258, 262, 320, 148, 16, 65, 140, 174, 87, 175, 188, 216, 204, 207, 210, 38, 50, 94, 132, 248, 298, 12, 106, 178, 329, 18, 347, 272, 59, 97, 181, 380, 142, 20, 313, 371, 217, 316, 238, 242, 36, 126, 170, 10, 28, 191, 55, 381, 56, 42, 7, 166, 13, 374, 352, 227, 307, 168, 149, 17, 71, 45, 337, 30, 246, 109, 49, 41, 201, 2, 80, 5, 226, 64, 81, 221, 342, 247, 274, 367, 249, 39, 355, 240, 285, 244, 336, 137, 304, 60, 282, 127, 283, 14, 54, 379, 339, 265, 186, 202, 346, 112, 180, 57, 134, 190, 343, 275, 370, 382, 123, 293, 86, 364, 172, 259, 215, 34, 257, 232, 349, 225, 72, 280, 90, 150, 345, 353, 292, 105, 205, 317, 167, 47, 122, 224, 312, 315, 40, 369, 147, 209, 79, 103, 220, 70, 153, 250, 161, 15, 233, 69, 62, 356, 306, 334, 99, 115, 35, 251, 92, 164, 96, 223, 254, 8, 100, 27, 286, 1, 372, 335, 281, 98, 256, 206, 143, 177, 108, 91, 152, 361, 196, 277, 295, 236, 44, 311, 271, 195, 124, 120, 333, 110, 309, 208, 160, 326, 119, 48, 159, 332, 63, 163, 145, 82, 102, 357, 278, 4, 324, 314, 218, 128, 116, 273, 222, 136, 19, 146, 228, 264, 78, 310, 267, 125, 269, 328, 344, 373, 22, 229, 348, 93, 52, 302, 135, 213, 111, 287, 327, 84, 299, 338, 261, 187, 239, 121, 194, 113, 219, 308, 154, 184, 211, 141, 296, 234, 155, 358, 279, 21, 330, 75, 130, 291, 165, 268, 138, 305, 289, 214, 6, 185, 26, 363, 176, 197, 73, 51, 198, 362, 173, 11, 383, 88, 25, 76, 129, 33, 74, 351, 266, 183, 243, 366, 375, 235, 24, 237, 68, 341, 319, 193, 350, 61, 360, 378, 85, 182, 139, 156, 200, 131, 300, 171, 276, 151, 101, 104, 3, 32, 294, 340, 77, 359, 83, 169, 31, 157, 230, 46, 377, 53, 303, 114, 368, 158, 9, 288, 199, 192, 253, 179, 270, 162, 376, 0, 43, 263, 252, 290, 297, 325, 301, 133, 189, 331, 95, 245, 260, 231, 107, 321, 241, 117, 284, 37, 144, 212, 322, 29, 66] + 30: [208, 358, 35, 373, 133, 184, 76, 29, 242, 135, 225, 257, 188, 321, 10, 64, 147, 17, 271, 216, 49, 89, 240, 336, 357, 100, 275, 236, 151, 299, 217, 382, 296, 356, 67, 98, 174, 96, 132, 167, 54, 329, 114, 306, 20, 51, 172, 286, 109, 211, 181, 7, 297, 175, 318, 337, 154, 342, 202, 107, 220, 247, 304, 203, 57, 116, 327, 370, 117, 364, 127, 355, 173, 16, 232, 126, 300, 56, 235, 80, 39, 171, 182, 87, 252, 131, 238, 331, 8, 21, 22, 31, 36, 40, 47, 88, 239, 187, 52, 143, 158, 124, 222, 137, 75, 78, 46, 168, 11, 223, 45, 68, 164, 205, 315, 303, 340, 312, 221, 12, 253, 339, 192, 193, 9, 136, 195, 292, 170, 281, 258, 298, 365, 104, 278, 84, 38, 179, 335, 162, 376, 310, 280, 293, 241, 55, 244, 282, 50, 290, 288, 25, 186, 276, 121, 183, 263, 324, 90, 326, 83, 69, 155, 5, 130, 81, 153, 15, 169, 190, 330, 198, 112, 270, 265, 149, 42, 367, 228, 119, 4, 27, 176, 301, 354, 94, 101, 118, 123, 128, 134, 146, 140, 245, 262, 53, 156, 289, 213, 333, 86, 243, 196, 378, 295, 361, 28, 144, 346, 33, 212, 250, 362, 159, 224, 341, 77, 177, 152, 256, 215, 141, 210, 320, 277, 18, 274, 160, 60, 23, 200, 91, 108, 72, 383, 150, 359, 14, 197, 111, 66, 180, 332, 115, 248, 92, 79, 374, 194, 166, 230, 95, 267, 350, 139, 279, 348, 99, 85, 287, 165, 82, 368, 291, 105, 233, 308, 113, 255, 207, 338, 313, 328, 63, 148, 93, 34, 191, 30, 201, 305, 363, 185, 189, 218, 219, 237, 259, 266, 65, 37, 214, 351, 62, 254, 272, 13, 366, 59, 24, 2, 206, 234, 246, 41, 283, 316, 302, 97, 43, 74, 138, 317, 1, 6, 71, 58, 309, 204, 226, 344, 260, 285, 369, 157, 343, 325, 353, 261, 231, 103, 142, 163, 319, 294, 129, 349, 375, 48, 73, 249, 44, 284, 145, 360, 110, 311, 377, 347, 371, 32, 264, 161, 381, 125, 323, 209, 273, 3, 122, 314, 19, 178, 120, 352, 102, 106, 199, 0, 372, 26, 61, 229, 251, 70, 227, 307, 379, 268, 269, 322, 334, 345, 380] + 31: [0, 100, 252, 278, 185, 227, 32, 221, 3, 116, 335, 25, 267, 268, 317, 325, 349, 155, 60, 171, 283, 208, 338, 276, 348, 189, 223, 33, 82, 254, 138, 63, 23, 98, 368, 2, 175, 65, 105, 31, 77, 287, 374, 312, 21, 288, 217, 16, 92, 49, 285, 4, 153, 237, 20, 271, 177, 323, 91, 220, 228, 281, 107, 80, 162, 42, 309, 129, 160, 340, 337, 234, 300, 109, 191, 7, 156, 382, 372, 124, 370, 242, 301, 332, 161, 311, 358, 5, 104, 13, 341, 212, 289, 229, 132, 207, 159, 68, 158, 239, 70, 76, 125, 192, 365, 379, 364, 69, 250, 187, 150, 39, 146, 209, 186, 188, 61, 34, 72, 56, 131, 286, 258, 296, 78, 236, 244, 353, 231, 38, 54, 245, 134, 169, 182, 205, 176, 190, 316, 284, 225, 213, 310, 302, 45, 142, 73, 354, 196, 51, 362, 247, 14, 272, 275, 346, 371, 194, 198, 304, 47, 306, 240, 139, 101, 298, 9, 24, 342, 113, 135, 216, 53, 181, 44, 274, 126, 35, 18, 27, 319, 218, 163, 230, 10, 117, 165, 89, 71, 90, 294, 350, 128, 201, 324, 115, 363, 87, 320, 265, 241, 279, 121, 133, 147, 95, 219, 330, 173, 22, 119, 333, 378, 75, 360, 62, 355, 214, 112, 290, 145, 380, 111, 152, 74, 232, 55, 66, 81, 88, 122, 322, 46, 318, 41, 256, 314, 118, 123, 336, 141, 84, 8, 195, 260, 12, 183, 11, 59, 266, 43, 351, 1, 48, 264, 103, 238, 226, 85, 210, 259, 157, 251, 168, 99, 327, 356, 280, 293, 235, 197, 282, 255, 313, 167, 174, 357, 257, 94, 377, 151, 154, 343, 148, 180, 149, 347, 329, 233, 26, 166, 143, 144, 261, 248, 331, 19, 36, 292, 326, 215, 270, 339, 328, 96, 140, 206, 58, 170, 253, 164, 299, 334, 102, 269, 106, 28, 211, 249, 93, 136, 17, 40, 243, 366, 57, 277, 137, 222, 15, 202, 97, 307, 373, 383, 303, 369, 291, 29, 110, 361, 273, 108, 305, 376, 6, 83, 352, 204, 130, 224, 50, 263, 308, 375, 345, 367, 178, 315, 262, 321, 64, 381, 52, 127, 79, 193, 344, 295, 114, 184, 179, 30, 359, 86, 199, 297, 200, 67, 203, 37, 246, 172, 120] + 32: [63, 241, 327, 344, 275, 135, 108, 362, 179, 21, 8, 66, 274, 41, 16, 186, 89, 202, 263, 211, 143, 156, 233, 109, 232, 58, 289, 301, 171, 298, 325, 85, 354, 199, 50, 25, 28, 157, 250, 230, 293, 142, 336, 350, 249, 371, 70, 4, 276, 381, 22, 198, 349, 236, 372, 5, 110, 136, 361, 284, 299, 287, 125, 238, 248, 117, 20, 356, 330, 201, 67, 52, 149, 188, 220, 224, 100, 23, 180, 321, 242, 88, 112, 318, 290, 93, 328, 54, 90, 300, 147, 14, 123, 169, 177, 181, 367, 92, 347, 51, 164, 329, 155, 268, 146, 106, 310, 370, 121, 12, 141, 359, 319, 225, 75, 184, 279, 140, 87, 116, 273, 113, 357, 345, 189, 369, 91, 227, 104, 71, 292, 175, 314, 64, 281, 48, 86, 182, 382, 183, 351, 353, 115, 84, 30, 337, 24, 10, 56, 62, 158, 363, 311, 144, 210, 42, 102, 191, 165, 124, 304, 341, 128, 73, 374, 35, 160, 119, 79, 203, 255, 334, 364, 133, 46, 216, 368, 358, 98, 192, 207, 291, 194, 15, 27, 114, 316, 167, 244, 245, 251, 262, 352, 148, 285, 163, 38, 166, 288, 246, 346, 187, 229, 271, 200, 2, 257, 342, 214, 204, 348, 174, 283, 126, 97, 131, 331, 335, 19, 120, 7, 313, 254, 212, 72, 36, 111, 69, 101, 332, 213, 324, 45, 150, 237, 256, 286, 53, 39, 176, 80, 205, 129, 6, 309, 340, 77, 193, 154, 172, 320, 282, 221, 132, 43, 243, 195, 376, 365, 222, 153, 264, 261, 305, 196, 138, 170, 208, 377, 31, 137, 76, 259, 95, 215, 74, 47, 272, 338, 11, 235, 326, 145, 379, 173, 265, 296, 312, 94, 218, 33, 190, 122, 107, 322, 234, 49, 209, 323, 130, 280, 360, 219, 61, 40, 380, 65, 96, 83, 151, 78, 258, 308, 228, 252, 294, 306, 168, 266, 297, 55, 303, 185, 231, 26, 206, 269, 239, 178, 59, 253, 32, 127, 29, 240, 81, 3, 302, 247, 315, 366, 37, 378, 139, 317, 339, 34, 82, 9, 99, 307, 152, 103, 197, 105, 13, 223, 134, 270, 44, 0, 355, 278, 343, 118, 1, 60, 277, 260, 18, 161, 17, 217, 159, 295, 226, 267, 333, 162, 57, 68, 383, 373, 375] + 33: [38, 362, 148, 251, 299, 94, 240, 145, 331, 153, 158, 379, 107, 73, 125, 358, 146, 139, 5, 48, 190, 21, 54, 160, 1, 338, 155, 237, 40, 334, 9, 14, 126, 232, 144, 187, 205, 138, 4, 317, 347, 22, 246, 19, 382, 367, 370, 88, 114, 60, 213, 368, 15, 345, 71, 336, 281, 52, 65, 53, 278, 248, 295, 221, 135, 6, 298, 111, 130, 243, 225, 291, 332, 245, 62, 134, 175, 242, 337, 97, 250, 171, 313, 309, 57, 296, 47, 77, 357, 0, 159, 264, 95, 69, 142, 220, 91, 177, 229, 115, 283, 267, 167, 140, 320, 280, 31, 42, 96, 297, 179, 348, 147, 108, 164, 226, 37, 335, 233, 141, 35, 25, 87, 227, 117, 356, 289, 217, 311, 305, 290, 211, 202, 355, 109, 174, 123, 45, 124, 375, 156, 55, 104, 200, 180, 279, 340, 105, 170, 249, 292, 330, 24, 28, 349, 166, 271, 285, 261, 327, 252, 194, 20, 316, 181, 286, 132, 381, 78, 188, 257, 365, 247, 353, 315, 363, 195, 275, 274, 269, 359, 172, 304, 203, 100, 143, 282, 129, 161, 12, 277, 287, 339, 260, 64, 81, 301, 307, 89, 328, 176, 63, 377, 323, 325, 39, 82, 314, 322, 201, 352, 266, 127, 169, 273, 376, 17, 33, 150, 151, 208, 308, 133, 222, 61, 18, 116, 303, 329, 173, 16, 254, 216, 198, 80, 196, 162, 157, 209, 373, 186, 212, 369, 234, 58, 49, 43, 371, 350, 66, 56, 70, 26, 112, 29, 41, 67, 318, 182, 23, 346, 11, 207, 76, 27, 119, 310, 118, 98, 59, 326, 165, 378, 306, 253, 7, 44, 321, 2, 294, 185, 223, 272, 189, 83, 34, 293, 333, 36, 319, 380, 343, 258, 154, 86, 51, 75, 259, 191, 121, 231, 183, 342, 68, 241, 84, 72, 206, 361, 255, 270, 204, 218, 30, 210, 193, 192, 131, 268, 288, 8, 235, 354, 351, 93, 149, 90, 230, 383, 79, 276, 262, 256, 197, 50, 163, 360, 152, 13, 364, 122, 137, 106, 3, 215, 113, 366, 103, 101, 110, 324, 219, 136, 10, 128, 238, 74, 300, 85, 168, 184, 224, 120, 312, 178, 244, 341, 199, 92, 214, 374, 228, 99, 344, 284, 46, 302, 263, 239, 265, 236, 32, 102, 372] + 34: [108, 366, 299, 248, 196, 382, 237, 332, 135, 158, 223, 243, 31, 167, 178, 315, 379, 284, 2, 271, 254, 11, 6, 4, 125, 225, 91, 153, 329, 356, 302, 345, 175, 24, 136, 37, 143, 280, 195, 181, 236, 231, 72, 209, 204, 170, 35, 5, 308, 348, 352, 63, 19, 318, 277, 56, 23, 261, 45, 51, 296, 192, 314, 13, 305, 262, 101, 226, 34, 307, 380, 52, 176, 370, 26, 39, 297, 141, 90, 1, 54, 355, 163, 344, 378, 303, 234, 369, 235, 66, 377, 279, 14, 76, 119, 122, 179, 340, 259, 334, 44, 372, 47, 357, 312, 311, 145, 185, 240, 148, 138, 258, 347, 59, 79, 238, 222, 92, 300, 65, 95, 208, 124, 110, 107, 184, 81, 159, 123, 327, 197, 60, 133, 317, 336, 328, 346, 150, 27, 287, 333, 102, 247, 263, 361, 103, 154, 140, 157, 42, 10, 371, 221, 142, 186, 147, 64, 152, 374, 219, 233, 36, 310, 286, 100, 255, 28, 109, 57, 242, 250, 55, 342, 171, 212, 22, 128, 43, 292, 182, 188, 38, 276, 376, 309, 273, 177, 211, 316, 132, 160, 189, 16, 15, 69, 330, 3, 61, 207, 324, 74, 239, 304, 321, 362, 291, 194, 146, 220, 115, 97, 8, 335, 78, 267, 77, 87, 201, 134, 373, 144, 53, 12, 84, 256, 368, 187, 137, 172, 217, 33, 245, 30, 227, 48, 214, 364, 169, 367, 230, 301, 215, 113, 322, 114, 323, 62, 298, 139, 46, 272, 264, 85, 68, 268, 228, 269, 111, 265, 274, 93, 331, 49, 117, 131, 278, 260, 155, 253, 9, 112, 71, 281, 120, 96, 216, 338, 252, 320, 7, 121, 375, 198, 213, 341, 193, 202, 229, 249, 306, 337, 126, 218, 190, 29, 151, 18, 82, 354, 25, 293, 203, 40, 360, 246, 99, 295, 83, 288, 191, 41, 168, 244, 162, 21, 200, 58, 290, 210, 174, 326, 313, 183, 166, 199, 206, 251, 70, 67, 241, 270, 73, 0, 359, 89, 116, 50, 232, 80, 180, 164, 294, 127, 283, 358, 149, 105, 383, 266, 98, 88, 275, 289, 282, 161, 86, 118, 353, 75, 319, 165, 205, 173, 17, 285, 339, 104, 20, 257, 351, 156, 130, 325, 349, 106, 94, 350, 129, 32, 343, 224, 381, 363, 365] + 35: [319, 28, 131, 68, 342, 322, 48, 220, 182, 144, 379, 368, 286, 47, 21, 31, 221, 206, 224, 41, 46, 265, 194, 50, 362, 52, 222, 329, 275, 102, 187, 267, 325, 128, 9, 158, 192, 268, 213, 108, 42, 313, 337, 12, 32, 170, 287, 74, 20, 111, 305, 106, 117, 347, 330, 273, 7, 184, 19, 301, 285, 43, 344, 6, 35, 198, 105, 63, 135, 207, 377, 115, 112, 26, 155, 89, 288, 167, 10, 101, 178, 351, 225, 33, 219, 258, 247, 280, 251, 120, 236, 100, 118, 304, 359, 177, 45, 169, 218, 295, 86, 72, 340, 353, 246, 237, 303, 56, 318, 141, 138, 129, 3, 346, 239, 15, 375, 311, 307, 241, 361, 266, 54, 159, 312, 134, 160, 360, 296, 18, 132, 61, 5, 328, 215, 372, 125, 143, 95, 122, 302, 96, 235, 350, 277, 227, 180, 91, 264, 382, 383, 24, 174, 185, 127, 142, 175, 217, 25, 338, 103, 65, 292, 87, 189, 253, 149, 245, 94, 249, 195, 262, 165, 263, 139, 107, 271, 327, 310, 371, 252, 336, 0, 39, 341, 229, 2, 64, 146, 381, 60, 228, 1, 75, 299, 297, 53, 276, 281, 80, 44, 240, 130, 254, 314, 55, 324, 153, 231, 116, 355, 186, 352, 349, 326, 335, 162, 14, 126, 27, 98, 163, 321, 140, 244, 272, 308, 36, 282, 104, 190, 334, 209, 320, 201, 300, 11, 16, 172, 92, 109, 145, 202, 348, 82, 339, 216, 290, 99, 179, 70, 196, 279, 315, 84, 374, 79, 274, 332, 151, 259, 29, 316, 278, 181, 306, 250, 354, 255, 358, 150, 238, 69, 83, 333, 123, 193, 208, 51, 119, 71, 76, 124, 283, 291, 284, 40, 293, 345, 363, 205, 367, 133, 203, 113, 309, 260, 62, 58, 211, 67, 257, 4, 38, 289, 154, 78, 147, 364, 114, 234, 164, 191, 331, 136, 37, 157, 373, 357, 168, 356, 81, 230, 171, 93, 298, 294, 30, 137, 243, 152, 90, 256, 323, 261, 57, 8, 366, 110, 214, 369, 204, 34, 166, 376, 269, 380, 17, 66, 73, 88, 378, 176, 317, 365, 248, 161, 156, 370, 22, 232, 121, 23, 49, 77, 97, 183, 148, 197, 270, 223, 233, 188, 242, 210, 226, 173, 212, 59, 199, 85, 200, 13, 343] + 36: [232, 340, 304, 325, 225, 107, 294, 74, 62, 2, 210, 196, 241, 369, 99, 301, 291, 133, 47, 287, 314, 32, 156, 45, 201, 309, 246, 288, 319, 191, 139, 13, 63, 105, 68, 245, 69, 183, 276, 153, 354, 117, 147, 343, 237, 116, 46, 134, 211, 30, 71, 39, 257, 36, 206, 165, 351, 263, 119, 159, 208, 51, 58, 239, 24, 70, 295, 265, 315, 173, 251, 357, 60, 307, 75, 14, 115, 350, 126, 26, 151, 285, 330, 327, 89, 269, 181, 306, 234, 113, 376, 290, 64, 37, 123, 170, 22, 53, 57, 10, 193, 223, 252, 382, 176, 359, 328, 260, 98, 242, 40, 373, 97, 207, 150, 5, 125, 85, 31, 42, 33, 162, 334, 67, 219, 166, 293, 49, 349, 132, 284, 91, 50, 172, 342, 155, 72, 142, 280, 179, 41, 66, 195, 202, 102, 217, 180, 0, 52, 38, 143, 335, 244, 358, 267, 78, 318, 283, 18, 146, 329, 83, 190, 109, 259, 381, 198, 199, 337, 316, 347, 118, 110, 312, 355, 235, 161, 278, 34, 96, 27, 136, 286, 222, 367, 182, 317, 186, 292, 127, 273, 302, 184, 272, 144, 266, 178, 12, 192, 197, 250, 185, 59, 230, 296, 320, 56, 90, 275, 352, 152, 205, 92, 233, 93, 331, 175, 321, 262, 112, 111, 194, 94, 55, 361, 122, 297, 121, 298, 300, 289, 339, 171, 168, 216, 140, 271, 100, 177, 277, 310, 255, 187, 264, 104, 81, 344, 17, 157, 148, 129, 23, 220, 365, 29, 135, 281, 209, 204, 189, 169, 338, 77, 248, 130, 35, 215, 137, 61, 254, 213, 8, 378, 154, 303, 128, 124, 353, 4, 106, 214, 160, 114, 145, 270, 16, 313, 360, 371, 368, 341, 238, 311, 372, 131, 370, 84, 348, 101, 299, 258, 326, 274, 380, 374, 73, 308, 79, 332, 54, 261, 65, 333, 240, 108, 362, 9, 231, 345, 282, 138, 167, 103, 95, 164, 322, 3, 20, 15, 346, 366, 174, 11, 228, 224, 364, 323, 305, 229, 200, 76, 212, 28, 379, 88, 188, 149, 80, 243, 163, 48, 356, 44, 363, 247, 6, 158, 236, 383, 19, 249, 256, 21, 120, 226, 324, 253, 87, 7, 86, 1, 203, 227, 82, 141, 218, 377, 221, 43, 268, 279, 25, 336, 375] + 37: [218, 58, 366, 113, 240, 123, 242, 213, 178, 46, 362, 190, 342, 72, 382, 77, 59, 171, 47, 360, 182, 14, 65, 139, 93, 343, 330, 18, 345, 145, 157, 19, 229, 228, 270, 196, 74, 144, 203, 230, 41, 39, 30, 111, 134, 29, 87, 100, 335, 247, 37, 121, 128, 380, 195, 97, 102, 284, 105, 354, 267, 154, 249, 299, 256, 17, 215, 170, 332, 169, 115, 297, 339, 198, 166, 96, 184, 324, 202, 281, 155, 20, 11, 219, 186, 367, 275, 23, 68, 67, 99, 31, 8, 75, 112, 132, 40, 109, 158, 79, 340, 71, 66, 309, 151, 125, 205, 372, 179, 290, 234, 95, 86, 16, 355, 302, 253, 252, 22, 327, 117, 235, 163, 32, 133, 192, 359, 322, 127, 320, 258, 21, 27, 1, 149, 217, 82, 73, 143, 161, 333, 201, 285, 3, 356, 55, 206, 331, 207, 269, 244, 373, 212, 279, 24, 33, 325, 351, 172, 298, 110, 124, 91, 266, 216, 350, 291, 183, 310, 374, 308, 248, 106, 225, 107, 237, 28, 177, 180, 45, 369, 90, 381, 282, 50, 232, 84, 209, 189, 236, 239, 288, 268, 293, 379, 222, 349, 0, 364, 243, 348, 311, 52, 199, 341, 272, 305, 193, 174, 223, 156, 344, 129, 9, 353, 165, 318, 328, 194, 118, 153, 277, 221, 315, 370, 316, 287, 76, 51, 173, 168, 323, 43, 61, 338, 304, 92, 278, 103, 329, 15, 126, 167, 83, 69, 80, 262, 347, 312, 94, 241, 38, 274, 4, 5, 116, 26, 160, 368, 53, 303, 260, 78, 137, 378, 35, 259, 81, 63, 357, 57, 130, 214, 326, 197, 300, 54, 159, 307, 12, 211, 346, 25, 276, 251, 56, 120, 352, 294, 148, 301, 200, 162, 188, 319, 273, 265, 314, 306, 363, 289, 48, 44, 292, 371, 361, 114, 140, 185, 245, 7, 88, 147, 204, 233, 181, 131, 257, 164, 150, 336, 208, 85, 122, 104, 176, 64, 6, 365, 296, 36, 141, 264, 108, 135, 376, 34, 136, 49, 224, 334, 337, 321, 98, 255, 313, 271, 317, 280, 42, 210, 187, 138, 13, 254, 10, 152, 60, 286, 175, 295, 101, 283, 191, 142, 62, 146, 231, 70, 227, 119, 261, 220, 250, 89, 2, 246, 358, 238, 377, 263, 375, 226, 383] + 38: [301, 124, 51, 126, 277, 282, 223, 50, 345, 94, 8, 285, 115, 220, 293, 63, 217, 27, 278, 212, 169, 206, 268, 108, 331, 199, 137, 288, 219, 352, 146, 147, 85, 208, 9, 59, 150, 105, 144, 323, 184, 256, 96, 367, 48, 229, 342, 276, 289, 260, 170, 28, 376, 33, 181, 15, 131, 152, 23, 173, 238, 214, 371, 56, 261, 100, 25, 215, 61, 294, 205, 71, 102, 290, 357, 192, 91, 254, 65, 42, 350, 132, 209, 303, 365, 83, 110, 155, 133, 243, 121, 159, 340, 46, 60, 185, 6, 221, 201, 30, 383, 333, 187, 213, 369, 118, 123, 263, 322, 248, 177, 230, 130, 244, 17, 373, 317, 77, 119, 267, 319, 338, 233, 361, 90, 75, 275, 271, 67, 72, 21, 193, 362, 36, 45, 57, 88, 264, 156, 368, 298, 143, 188, 228, 141, 148, 84, 356, 180, 149, 272, 103, 139, 2, 250, 200, 334, 313, 43, 74, 382, 375, 82, 252, 80, 297, 186, 13, 138, 202, 363, 316, 113, 320, 112, 259, 18, 175, 128, 116, 22, 377, 127, 1, 249, 270, 304, 161, 38, 101, 237, 251, 266, 378, 280, 372, 189, 154, 62, 10, 93, 348, 245, 292, 86, 226, 194, 32, 379, 19, 92, 145, 0, 328, 54, 253, 257, 234, 329, 16, 225, 341, 99, 222, 324, 239, 69, 29, 224, 358, 307, 87, 58, 255, 165, 308, 179, 34, 111, 246, 5, 265, 190, 241, 76, 134, 41, 174, 305, 235, 81, 310, 296, 360, 122, 195, 273, 104, 120, 204, 44, 295, 168, 283, 35, 183, 299, 3, 12, 164, 326, 157, 163, 314, 291, 153, 167, 218, 160, 196, 140, 330, 318, 312, 176, 52, 191, 325, 203, 158, 269, 284, 366, 114, 89, 4, 287, 178, 66, 332, 354, 274, 309, 49, 39, 171, 381, 166, 279, 135, 344, 300, 53, 97, 286, 327, 106, 125, 355, 107, 14, 227, 73, 95, 321, 359, 262, 68, 211, 349, 216, 136, 142, 7, 37, 109, 151, 117, 31, 306, 197, 26, 242, 258, 353, 40, 78, 337, 302, 182, 79, 172, 129, 347, 11, 231, 47, 311, 374, 351, 55, 247, 343, 236, 70, 240, 335, 24, 364, 232, 20, 207, 281, 315, 198, 336, 162, 370, 346, 64, 98, 339, 210, 380] + 39: [282, 56, 65, 132, 379, 100, 280, 264, 82, 348, 153, 13, 279, 267, 382, 86, 80, 270, 324, 26, 98, 17, 15, 319, 77, 49, 23, 381, 235, 375, 170, 352, 248, 340, 251, 112, 336, 101, 34, 47, 353, 177, 150, 281, 317, 364, 162, 301, 302, 228, 102, 311, 95, 185, 133, 285, 230, 221, 30, 195, 363, 183, 327, 60, 166, 283, 58, 70, 196, 346, 189, 117, 46, 157, 206, 231, 142, 37, 10, 39, 284, 326, 351, 370, 294, 318, 237, 262, 269, 344, 201, 29, 321, 9, 149, 22, 310, 236, 286, 38, 314, 43, 96, 180, 116, 368, 113, 215, 360, 271, 272, 2, 145, 296, 204, 374, 40, 141, 341, 369, 3, 365, 59, 146, 298, 84, 8, 259, 191, 48, 323, 122, 278, 190, 137, 260, 126, 358, 135, 188, 144, 307, 161, 249, 14, 373, 355, 106, 167, 320, 173, 354, 275, 6, 79, 309, 68, 377, 194, 214, 376, 291, 109, 217, 288, 19, 303, 107, 337, 108, 20, 246, 0, 252, 253, 238, 233, 339, 136, 158, 67, 205, 12, 316, 356, 184, 11, 90, 334, 89, 143, 198, 247, 372, 258, 312, 36, 85, 99, 268, 234, 1, 156, 218, 325, 155, 328, 244, 174, 362, 71, 172, 295, 330, 24, 76, 208, 7, 81, 200, 227, 202, 53, 42, 140, 232, 28, 31, 263, 69, 74, 131, 152, 50, 276, 371, 5, 63, 75, 226, 134, 182, 61, 94, 289, 241, 193, 357, 4, 338, 243, 130, 220, 306, 169, 181, 120, 171, 343, 27, 335, 103, 222, 111, 55, 266, 83, 160, 211, 64, 197, 229, 342, 210, 129, 349, 125, 32, 105, 73, 332, 261, 115, 148, 21, 52, 257, 44, 66, 104, 315, 93, 313, 176, 35, 110, 223, 25, 151, 139, 242, 119, 187, 292, 329, 297, 277, 333, 121, 250, 300, 178, 163, 51, 87, 290, 273, 118, 383, 274, 127, 54, 299, 225, 239, 138, 331, 378, 367, 62, 199, 366, 322, 179, 213, 345, 33, 212, 72, 293, 45, 88, 97, 203, 128, 164, 16, 254, 240, 114, 207, 265, 91, 216, 245, 192, 287, 41, 168, 159, 78, 305, 347, 175, 18, 123, 92, 359, 219, 255, 147, 350, 256, 124, 224, 308, 57, 154, 186, 380, 304, 165, 361, 209] + 40: [31, 349, 71, 108, 132, 280, 213, 204, 98, 283, 52, 107, 279, 153, 254, 64, 251, 198, 258, 228, 17, 347, 362, 363, 136, 168, 321, 63, 322, 211, 27, 30, 216, 188, 221, 302, 269, 137, 56, 53, 365, 57, 138, 25, 377, 94, 355, 192, 338, 324, 167, 206, 177, 330, 273, 37, 47, 314, 85, 261, 6, 36, 118, 121, 141, 256, 276, 184, 268, 310, 140, 117, 113, 295, 35, 242, 111, 231, 114, 354, 342, 315, 176, 223, 292, 112, 127, 220, 225, 240, 344, 170, 343, 194, 214, 248, 319, 317, 183, 328, 358, 346, 379, 286, 169, 55, 373, 101, 297, 160, 202, 161, 88, 372, 208, 58, 3, 100, 28, 311, 9, 143, 89, 326, 274, 341, 185, 289, 54, 145, 308, 199, 382, 196, 229, 259, 187, 309, 263, 178, 82, 189, 290, 103, 2, 294, 287, 93, 318, 305, 281, 243, 304, 350, 60, 51, 366, 201, 76, 238, 147, 78, 186, 288, 348, 102, 124, 383, 368, 174, 66, 203, 77, 34, 217, 173, 352, 154, 313, 380, 381, 245, 298, 172, 272, 83, 197, 312, 5, 69, 376, 300, 15, 378, 250, 142, 21, 33, 226, 340, 70, 359, 75, 265, 337, 128, 156, 8, 232, 369, 74, 235, 90, 59, 91, 275, 1, 284, 336, 271, 219, 80, 277, 166, 11, 278, 38, 236, 370, 148, 262, 307, 44, 316, 364, 95, 43, 244, 110, 306, 165, 29, 120, 327, 371, 329, 323, 237, 195, 131, 293, 233, 345, 139, 241, 291, 270, 115, 133, 14, 0, 23, 135, 13, 50, 16, 182, 163, 257, 106, 104, 22, 180, 234, 96, 4, 87, 45, 252, 24, 320, 126, 42, 335, 144, 353, 92, 123, 68, 175, 207, 360, 190, 179, 357, 260, 181, 67, 367, 62, 247, 26, 215, 158, 164, 332, 81, 224, 97, 351, 159, 210, 303, 296, 129, 10, 209, 162, 61, 253, 193, 334, 200, 12, 205, 227, 46, 239, 79, 99, 41, 150, 65, 299, 130, 151, 7, 155, 264, 339, 333, 356, 282, 48, 171, 19, 18, 301, 73, 222, 246, 375, 116, 122, 212, 361, 84, 125, 266, 32, 49, 374, 40, 230, 218, 134, 285, 119, 325, 267, 152, 39, 109, 157, 255, 86, 20, 191, 72, 146, 331, 249, 105, 149] + 41: [49, 321, 114, 184, 197, 159, 305, 367, 103, 251, 90, 86, 273, 365, 293, 76, 7, 172, 24, 279, 206, 334, 198, 253, 80, 326, 127, 222, 286, 202, 353, 126, 252, 349, 301, 56, 313, 225, 93, 55, 237, 229, 340, 344, 242, 330, 288, 268, 299, 101, 181, 284, 19, 208, 160, 317, 61, 220, 89, 354, 37, 359, 154, 292, 1, 39, 191, 77, 72, 13, 248, 149, 316, 148, 155, 137, 29, 238, 370, 47, 183, 110, 323, 319, 270, 106, 341, 362, 382, 291, 230, 52, 263, 31, 120, 85, 378, 374, 283, 190, 366, 119, 122, 335, 166, 376, 212, 57, 84, 368, 124, 285, 136, 111, 358, 360, 347, 12, 95, 82, 383, 318, 351, 35, 239, 219, 194, 163, 308, 26, 64, 70, 54, 226, 256, 83, 294, 235, 325, 138, 339, 133, 210, 8, 63, 151, 295, 290, 81, 277, 261, 43, 173, 315, 146, 192, 15, 232, 300, 33, 143, 78, 287, 165, 259, 207, 38, 68, 98, 310, 182, 28, 265, 171, 245, 373, 201, 121, 343, 88, 177, 162, 109, 356, 304, 281, 164, 233, 22, 178, 152, 132, 199, 350, 298, 355, 369, 275, 205, 185, 71, 27, 307, 255, 200, 50, 97, 96, 187, 34, 69, 161, 280, 311, 108, 306, 348, 9, 4, 53, 42, 25, 23, 218, 336, 174, 116, 345, 241, 282, 46, 260, 258, 322, 117, 289, 250, 224, 380, 51, 131, 176, 186, 338, 91, 246, 94, 5, 371, 74, 128, 303, 59, 123, 342, 188, 236, 100, 381, 180, 297, 234, 105, 48, 257, 73, 175, 204, 375, 231, 274, 17, 62, 267, 2, 145, 189, 312, 346, 361, 169, 193, 118, 333, 40, 247, 216, 309, 170, 60, 269, 142, 30, 129, 215, 87, 196, 11, 249, 150, 168, 223, 327, 92, 79, 364, 21, 314, 302, 262, 44, 337, 104, 45, 3, 324, 264, 140, 36, 243, 217, 134, 32, 102, 125, 379, 254, 276, 156, 271, 167, 115, 144, 112, 266, 328, 363, 66, 107, 153, 18, 6, 130, 213, 244, 10, 221, 214, 41, 99, 75, 272, 352, 179, 0, 65, 141, 320, 332, 158, 147, 113, 195, 209, 14, 67, 329, 20, 139, 331, 157, 240, 211, 377, 372, 16, 58, 203, 135, 228, 227, 296, 278, 357] + 42: [123, 172, 111, 138, 210, 1, 114, 119, 184, 247, 63, 282, 168, 222, 262, 377, 129, 208, 203, 189, 127, 373, 277, 328, 320, 4, 259, 236, 187, 28, 230, 165, 326, 338, 36, 92, 143, 45, 33, 362, 227, 192, 380, 113, 51, 19, 8, 231, 225, 323, 283, 139, 244, 226, 34, 298, 196, 16, 355, 270, 278, 112, 346, 120, 56, 318, 267, 82, 197, 360, 128, 292, 23, 288, 330, 367, 22, 85, 133, 353, 249, 100, 79, 331, 78, 271, 13, 86, 66, 361, 46, 126, 25, 71, 95, 254, 152, 94, 268, 155, 256, 37, 188, 20, 295, 106, 303, 116, 151, 264, 41, 337, 162, 200, 6, 48, 281, 21, 191, 379, 275, 370, 274, 30, 140, 135, 286, 214, 167, 105, 161, 216, 335, 322, 280, 316, 118, 76, 257, 160, 32, 80, 241, 301, 311, 62, 372, 9, 89, 315, 183, 232, 299, 258, 7, 204, 378, 68, 125, 300, 293, 213, 84, 284, 206, 90, 321, 363, 73, 185, 124, 334, 144, 174, 383, 96, 153, 190, 12, 11, 83, 44, 159, 43, 317, 91, 74, 24, 273, 27, 294, 324, 117, 217, 345, 218, 325, 333, 154, 50, 148, 357, 211, 212, 215, 72, 374, 349, 220, 55, 263, 59, 342, 319, 265, 235, 358, 141, 354, 341, 221, 169, 239, 17, 287, 173, 329, 356, 181, 38, 164, 202, 52, 223, 149, 297, 245, 60, 18, 132, 137, 307, 180, 64, 238, 49, 109, 99, 58, 205, 5, 336, 291, 53, 170, 171, 364, 343, 175, 272, 229, 98, 145, 368, 312, 296, 75, 186, 136, 207, 302, 88, 253, 276, 35, 70, 107, 54, 182, 142, 382, 240, 163, 40, 289, 121, 369, 339, 31, 359, 266, 176, 93, 158, 2, 375, 42, 228, 305, 350, 279, 150, 179, 243, 195, 108, 306, 255, 209, 327, 122, 351, 348, 131, 248, 29, 178, 224, 67, 344, 87, 201, 147, 103, 308, 77, 314, 234, 366, 166, 39, 65, 381, 340, 101, 146, 237, 371, 219, 246, 260, 130, 157, 97, 310, 313, 61, 251, 332, 156, 269, 233, 26, 194, 261, 290, 199, 102, 250, 10, 104, 285, 3, 198, 304, 309, 347, 134, 69, 115, 47, 57, 110, 81, 14, 177, 0, 15, 365, 252, 352, 193, 242, 376] + 43: [168, 105, 92, 359, 190, 298, 43, 344, 6, 177, 68, 112, 216, 20, 252, 103, 314, 152, 291, 55, 138, 322, 264, 178, 1, 305, 147, 78, 250, 255, 272, 73, 232, 282, 88, 327, 339, 306, 115, 299, 367, 36, 84, 157, 237, 10, 63, 107, 49, 24, 113, 180, 217, 56, 251, 74, 125, 101, 174, 370, 215, 295, 329, 122, 259, 279, 76, 132, 247, 118, 134, 358, 211, 290, 240, 179, 260, 341, 276, 230, 106, 62, 249, 236, 383, 40, 15, 91, 176, 350, 302, 361, 169, 206, 214, 245, 225, 221, 156, 335, 283, 297, 351, 27, 82, 278, 218, 35, 37, 13, 239, 94, 57, 326, 337, 153, 29, 301, 203, 318, 108, 274, 269, 321, 85, 374, 150, 2, 238, 90, 346, 204, 86, 219, 200, 26, 149, 5, 365, 67, 196, 194, 345, 145, 99, 60, 50, 165, 45, 253, 120, 46, 263, 9, 80, 288, 175, 159, 207, 72, 144, 267, 353, 163, 19, 158, 173, 109, 320, 300, 110, 336, 285, 51, 69, 124, 7, 172, 141, 281, 42, 270, 133, 96, 212, 33, 166, 376, 192, 182, 382, 3, 375, 315, 347, 17, 14, 30, 23, 262, 161, 332, 142, 129, 59, 349, 97, 100, 131, 235, 377, 186, 111, 127, 323, 313, 224, 338, 202, 280, 379, 4, 77, 222, 378, 154, 380, 275, 286, 234, 123, 271, 160, 34, 61, 130, 241, 334, 273, 12, 242, 293, 38, 47, 210, 266, 284, 167, 128, 146, 256, 325, 170, 116, 304, 64, 294, 71, 117, 87, 151, 287, 233, 119, 330, 355, 187, 357, 311, 201, 53, 114, 317, 328, 21, 228, 362, 340, 333, 104, 25, 289, 184, 54, 189, 331, 44, 348, 66, 98, 171, 372, 324, 75, 164, 369, 181, 308, 371, 197, 364, 39, 254, 366, 22, 363, 93, 81, 223, 342, 360, 227, 198, 307, 136, 121, 89, 368, 191, 32, 205, 52, 246, 48, 83, 277, 140, 135, 296, 261, 257, 79, 28, 155, 243, 148, 102, 185, 229, 183, 143, 16, 70, 213, 220, 309, 343, 226, 18, 31, 292, 11, 356, 8, 65, 126, 193, 381, 268, 209, 188, 265, 316, 258, 162, 312, 248, 199, 41, 0, 137, 303, 208, 352, 58, 139, 244, 319, 231, 310, 195, 373, 95, 354] + 44: [213, 226, 172, 101, 145, 238, 0, 272, 110, 60, 188, 257, 43, 252, 135, 325, 288, 93, 98, 86, 333, 141, 63, 329, 48, 144, 125, 291, 355, 89, 121, 262, 244, 192, 61, 296, 154, 173, 364, 70, 285, 130, 248, 286, 54, 174, 369, 134, 157, 342, 41, 26, 196, 370, 165, 159, 214, 105, 323, 309, 156, 81, 305, 237, 234, 57, 250, 345, 367, 49, 51, 232, 273, 42, 45, 216, 239, 373, 115, 99, 246, 363, 176, 302, 47, 293, 73, 361, 58, 88, 75, 371, 209, 13, 332, 374, 77, 317, 274, 379, 223, 170, 97, 258, 178, 155, 190, 215, 376, 277, 279, 331, 11, 166, 146, 280, 289, 242, 294, 198, 33, 228, 225, 95, 324, 94, 207, 91, 6, 350, 254, 290, 163, 219, 287, 39, 71, 79, 338, 378, 67, 359, 120, 44, 327, 107, 46, 136, 15, 108, 185, 334, 270, 243, 78, 347, 341, 169, 3, 218, 241, 72, 193, 357, 4, 204, 62, 7, 124, 245, 80, 1, 27, 195, 151, 194, 247, 372, 260, 307, 123, 297, 127, 31, 316, 148, 181, 16, 161, 189, 303, 116, 301, 10, 348, 171, 21, 168, 167, 25, 382, 271, 227, 84, 29, 68, 140, 311, 236, 143, 320, 383, 182, 249, 264, 142, 365, 304, 275, 337, 343, 276, 265, 138, 266, 85, 175, 186, 122, 55, 100, 56, 352, 132, 366, 160, 202, 96, 261, 381, 128, 292, 220, 344, 92, 278, 74, 282, 149, 8, 231, 377, 23, 32, 20, 113, 263, 203, 319, 18, 53, 256, 137, 65, 35, 312, 200, 326, 153, 5, 251, 210, 283, 28, 69, 268, 139, 356, 335, 111, 76, 133, 208, 281, 295, 129, 206, 322, 191, 199, 362, 222, 284, 346, 64, 17, 300, 351, 180, 267, 82, 328, 368, 229, 30, 34, 313, 187, 212, 118, 38, 306, 358, 102, 217, 360, 158, 14, 340, 117, 184, 224, 24, 299, 349, 150, 201, 315, 66, 131, 164, 380, 269, 87, 221, 177, 36, 162, 354, 126, 147, 83, 90, 40, 205, 59, 318, 19, 336, 211, 104, 179, 235, 230, 9, 109, 314, 321, 183, 114, 255, 2, 253, 375, 233, 22, 119, 50, 240, 298, 103, 330, 259, 37, 152, 52, 353, 197, 310, 308, 112, 339, 12, 106] + 45: [92, 161, 334, 33, 276, 81, 219, 233, 316, 346, 274, 176, 369, 160, 85, 269, 298, 348, 114, 209, 234, 331, 39, 198, 239, 196, 45, 115, 104, 30, 338, 84, 112, 327, 80, 345, 249, 156, 44, 260, 279, 365, 24, 1, 22, 222, 218, 43, 330, 34, 116, 87, 256, 296, 320, 283, 29, 282, 211, 263, 342, 177, 285, 105, 142, 15, 48, 78, 3, 190, 377, 136, 225, 332, 132, 62, 89, 82, 97, 232, 193, 194, 366, 259, 6, 335, 184, 11, 149, 240, 14, 40, 137, 192, 100, 174, 140, 23, 42, 27, 373, 95, 191, 301, 134, 110, 199, 120, 356, 375, 262, 378, 131, 21, 124, 297, 52, 183, 258, 228, 359, 168, 162, 158, 36, 51, 245, 250, 12, 68, 251, 267, 135, 213, 157, 94, 292, 175, 67, 2, 302, 303, 253, 370, 58, 35, 19, 170, 304, 252, 145, 358, 98, 150, 322, 241, 151, 127, 288, 204, 79, 363, 289, 203, 371, 113, 215, 71, 73, 238, 321, 102, 153, 56, 152, 143, 353, 200, 295, 53, 216, 77, 99, 325, 130, 49, 69, 299, 244, 138, 83, 221, 178, 154, 293, 201, 350, 329, 383, 318, 144, 317, 139, 197, 141, 247, 146, 354, 47, 165, 163, 349, 224, 380, 275, 351, 187, 223, 55, 122, 229, 155, 88, 372, 54, 328, 306, 357, 41, 86, 108, 179, 32, 315, 208, 96, 123, 376, 63, 340, 76, 65, 133, 126, 205, 319, 291, 5, 37, 294, 333, 231, 255, 381, 374, 314, 166, 0, 119, 309, 268, 147, 343, 93, 91, 337, 323, 125, 280, 18, 382, 226, 206, 305, 352, 148, 278, 287, 172, 66, 60, 195, 181, 72, 355, 361, 266, 347, 10, 227, 121, 8, 367, 164, 324, 173, 188, 284, 25, 182, 171, 339, 103, 286, 180, 61, 212, 128, 272, 107, 248, 254, 118, 307, 310, 16, 326, 277, 74, 70, 308, 313, 210, 7, 360, 242, 281, 217, 106, 26, 109, 236, 4, 261, 257, 50, 20, 344, 207, 230, 46, 290, 59, 38, 264, 9, 273, 379, 57, 169, 129, 271, 246, 220, 341, 312, 364, 368, 362, 202, 243, 111, 214, 90, 189, 31, 186, 167, 159, 270, 101, 265, 13, 311, 28, 75, 237, 235, 185, 17, 117, 336, 64, 300] + 46: [142, 34, 58, 355, 149, 106, 335, 188, 368, 327, 16, 51, 274, 6, 209, 12, 214, 60, 38, 165, 113, 44, 76, 219, 350, 215, 211, 62, 115, 195, 22, 227, 80, 232, 121, 168, 180, 109, 4, 13, 129, 362, 53, 273, 172, 216, 157, 158, 45, 57, 226, 373, 328, 97, 360, 24, 297, 223, 346, 69, 9, 317, 322, 333, 145, 264, 190, 370, 184, 244, 339, 26, 347, 163, 65, 43, 77, 310, 281, 72, 87, 381, 48, 174, 212, 29, 185, 0, 100, 28, 148, 270, 220, 379, 61, 122, 183, 247, 139, 380, 207, 367, 140, 210, 314, 240, 170, 238, 191, 324, 104, 301, 271, 136, 204, 114, 345, 2, 374, 3, 102, 39, 351, 228, 130, 33, 155, 372, 287, 334, 218, 315, 259, 137, 37, 50, 176, 23, 162, 250, 198, 267, 235, 193, 125, 377, 32, 307, 141, 78, 144, 169, 299, 161, 318, 348, 159, 293, 265, 342, 261, 283, 217, 93, 361, 305, 234, 375, 337, 15, 112, 21, 127, 330, 341, 126, 288, 73, 196, 181, 279, 116, 276, 84, 151, 167, 178, 323, 231, 5, 36, 175, 352, 10, 256, 253, 110, 354, 66, 201, 19, 46, 25, 202, 246, 332, 154, 344, 194, 187, 254, 55, 18, 303, 146, 203, 266, 85, 30, 280, 8, 117, 120, 83, 326, 132, 245, 197, 295, 192, 316, 166, 63, 41, 95, 128, 292, 237, 340, 321, 336, 383, 81, 308, 206, 92, 289, 255, 313, 222, 173, 382, 7, 243, 277, 241, 359, 91, 291, 309, 285, 278, 98, 257, 150, 52, 135, 75, 258, 11, 366, 236, 363, 105, 269, 251, 296, 179, 47, 123, 70, 357, 225, 365, 252, 27, 88, 213, 311, 133, 64, 286, 343, 230, 319, 152, 233, 224, 249, 96, 239, 221, 284, 312, 124, 40, 68, 378, 338, 199, 242, 320, 79, 282, 49, 131, 108, 143, 376, 260, 186, 99, 353, 118, 156, 42, 369, 298, 164, 304, 294, 272, 205, 103, 17, 59, 268, 111, 101, 329, 263, 35, 160, 290, 89, 119, 189, 56, 71, 74, 153, 20, 364, 177, 90, 331, 86, 262, 171, 371, 208, 138, 67, 358, 275, 94, 229, 31, 300, 107, 1, 134, 306, 248, 54, 349, 200, 14, 182, 82, 147, 325, 302, 356] + 47: [195, 312, 162, 382, 134, 347, 136, 357, 35, 170, 61, 333, 286, 320, 294, 196, 362, 129, 376, 118, 59, 123, 342, 235, 114, 176, 330, 269, 95, 104, 135, 42, 166, 285, 263, 284, 172, 184, 50, 203, 0, 101, 338, 45, 52, 106, 117, 234, 23, 139, 201, 243, 71, 126, 289, 281, 276, 306, 111, 107, 329, 310, 132, 248, 334, 10, 150, 19, 48, 282, 57, 325, 74, 290, 108, 58, 16, 38, 220, 85, 145, 142, 194, 78, 116, 125, 217, 221, 182, 83, 381, 236, 191, 369, 56, 288, 3, 24, 154, 158, 307, 137, 331, 192, 291, 279, 34, 227, 345, 90, 215, 257, 36, 274, 240, 287, 188, 327, 367, 218, 251, 122, 351, 250, 84, 69, 219, 377, 124, 151, 231, 103, 33, 65, 255, 197, 12, 205, 138, 148, 340, 233, 112, 164, 53, 183, 356, 91, 321, 211, 303, 28, 225, 328, 332, 258, 115, 200, 157, 2, 297, 81, 9, 46, 242, 189, 308, 266, 383, 207, 350, 174, 133, 86, 178, 222, 202, 247, 11, 64, 268, 161, 165, 256, 241, 261, 113, 169, 7, 239, 246, 293, 343, 352, 181, 22, 300, 271, 179, 143, 371, 119, 109, 193, 309, 14, 368, 127, 344, 79, 168, 110, 40, 346, 153, 363, 159, 212, 226, 265, 208, 93, 301, 67, 224, 249, 228, 41, 323, 140, 364, 77, 204, 51, 355, 237, 180, 173, 252, 210, 96, 378, 18, 80, 15, 348, 163, 147, 259, 292, 278, 60, 49, 341, 186, 20, 97, 144, 160, 102, 365, 275, 55, 72, 177, 245, 5, 380, 339, 30, 105, 277, 47, 76, 270, 244, 82, 167, 232, 295, 8, 254, 128, 311, 156, 62, 373, 326, 54, 44, 87, 29, 359, 75, 187, 171, 43, 375, 1, 280, 314, 305, 316, 353, 198, 175, 267, 21, 238, 131, 229, 349, 63, 98, 313, 230, 155, 26, 262, 68, 89, 223, 324, 319, 298, 370, 199, 73, 296, 322, 17, 25, 39, 4, 354, 27, 315, 366, 141, 336, 70, 121, 335, 213, 273, 337, 94, 37, 31, 185, 32, 260, 130, 253, 190, 149, 120, 152, 302, 216, 379, 358, 374, 92, 304, 100, 317, 361, 372, 88, 299, 264, 99, 283, 6, 206, 66, 13, 272, 318, 214, 146, 209, 360] + 48: [17, 76, 102, 276, 353, 78, 62, 297, 26, 270, 279, 215, 99, 236, 182, 295, 33, 28, 166, 189, 177, 240, 38, 264, 34, 342, 181, 305, 320, 371, 229, 312, 322, 7, 25, 257, 370, 112, 190, 0, 160, 255, 321, 357, 315, 347, 79, 13, 379, 202, 113, 281, 72, 289, 97, 11, 84, 43, 349, 16, 151, 292, 94, 274, 261, 39, 187, 47, 217, 92, 67, 313, 27, 163, 294, 105, 110, 309, 188, 191, 8, 338, 60, 106, 296, 248, 23, 383, 103, 337, 199, 258, 30, 245, 196, 227, 271, 283, 330, 273, 284, 85, 15, 288, 20, 77, 145, 109, 234, 121, 74, 164, 268, 207, 369, 147, 344, 139, 48, 254, 142, 115, 134, 252, 155, 290, 210, 300, 161, 175, 192, 50, 120, 247, 88, 340, 329, 341, 212, 24, 168, 172, 42, 231, 307, 22, 80, 351, 119, 96, 302, 81, 198, 269, 367, 40, 206, 346, 107, 137, 108, 82, 272, 173, 366, 214, 69, 373, 224, 37, 87, 278, 244, 350, 361, 225, 298, 362, 282, 125, 46, 183, 262, 176, 220, 265, 355, 336, 136, 304, 114, 250, 140, 267, 266, 52, 311, 141, 237, 159, 249, 65, 301, 358, 19, 44, 2, 90, 21, 83, 104, 6, 5, 235, 359, 363, 186, 316, 184, 354, 319, 378, 222, 156, 130, 246, 299, 152, 285, 75, 45, 73, 89, 118, 242, 111, 4, 318, 91, 360, 180, 339, 241, 14, 138, 146, 36, 253, 211, 280, 12, 53, 55, 144, 35, 259, 243, 178, 71, 286, 95, 194, 93, 3, 56, 143, 277, 335, 66, 228, 356, 148, 154, 86, 345, 153, 124, 251, 54, 122, 303, 364, 131, 348, 10, 200, 129, 308, 328, 132, 128, 239, 101, 381, 263, 9, 51, 117, 150, 226, 116, 343, 203, 58, 133, 157, 287, 375, 223, 327, 208, 372, 325, 185, 380, 41, 232, 368, 205, 219, 193, 179, 293, 291, 233, 174, 123, 324, 162, 170, 31, 221, 230, 275, 218, 57, 317, 64, 100, 70, 49, 158, 204, 63, 374, 352, 18, 127, 98, 169, 59, 195, 382, 216, 256, 32, 135, 310, 326, 29, 165, 323, 213, 61, 201, 238, 376, 149, 171, 331, 377, 260, 365, 332, 68, 126, 306, 197, 333, 167, 1, 209, 334, 314] + 49: [302, 6, 305, 271, 193, 197, 346, 15, 234, 158, 116, 78, 25, 40, 153, 148, 259, 60, 380, 76, 289, 220, 187, 312, 255, 30, 332, 94, 109, 122, 58, 97, 169, 362, 382, 28, 221, 23, 311, 196, 198, 290, 189, 34, 41, 248, 20, 301, 295, 371, 238, 267, 330, 338, 227, 359, 239, 339, 320, 117, 378, 0, 61, 106, 49, 206, 342, 228, 143, 108, 145, 125, 88, 168, 316, 377, 51, 7, 341, 42, 159, 287, 161, 379, 141, 207, 291, 252, 129, 50, 24, 68, 300, 317, 64, 123, 321, 79, 268, 19, 202, 210, 37, 83, 304, 136, 181, 213, 105, 278, 110, 335, 180, 39, 325, 292, 107, 243, 306, 269, 381, 186, 175, 183, 298, 5, 195, 121, 250, 254, 174, 340, 80, 54, 86, 70, 138, 365, 29, 89, 160, 222, 296, 273, 225, 82, 10, 327, 352, 36, 134, 307, 103, 244, 14, 72, 53, 226, 119, 118, 200, 124, 84, 285, 283, 154, 22, 299, 182, 232, 274, 343, 93, 13, 87, 372, 96, 162, 334, 236, 171, 152, 217, 11, 247, 32, 3, 347, 188, 322, 337, 191, 297, 374, 272, 155, 313, 9, 251, 149, 294, 209, 65, 150, 353, 66, 241, 208, 201, 276, 262, 177, 286, 91, 260, 74, 120, 329, 45, 264, 95, 131, 333, 48, 205, 328, 73, 1, 63, 310, 16, 368, 240, 277, 318, 128, 224, 4, 127, 77, 146, 55, 216, 179, 33, 67, 43, 178, 156, 351, 172, 114, 85, 280, 69, 358, 355, 31, 2, 151, 282, 101, 361, 52, 354, 348, 326, 357, 166, 27, 147, 167, 165, 367, 266, 231, 265, 350, 281, 263, 230, 203, 288, 26, 370, 349, 375, 356, 253, 126, 360, 75, 157, 242, 233, 184, 319, 257, 59, 115, 293, 212, 261, 270, 90, 275, 44, 199, 218, 133, 246, 132, 344, 194, 331, 8, 100, 314, 164, 235, 144, 366, 336, 279, 17, 137, 245, 56, 223, 140, 170, 214, 112, 323, 163, 81, 376, 139, 303, 173, 308, 35, 98, 62, 309, 142, 345, 211, 47, 176, 71, 130, 12, 46, 204, 192, 284, 190, 113, 256, 364, 185, 324, 104, 111, 57, 315, 229, 363, 237, 102, 18, 21, 99, 249, 215, 383, 258, 369, 219, 373, 92, 135, 38] + 50: [86, 181, 315, 69, 214, 350, 129, 57, 226, 137, 333, 367, 361, 35, 146, 175, 282, 338, 327, 0, 193, 114, 22, 343, 253, 15, 159, 187, 132, 122, 164, 244, 306, 58, 117, 203, 194, 168, 219, 43, 189, 342, 180, 139, 382, 74, 92, 64, 277, 126, 78, 99, 340, 223, 113, 5, 236, 17, 39, 292, 4, 344, 123, 106, 349, 308, 156, 141, 237, 150, 90, 36, 65, 112, 82, 346, 30, 381, 176, 246, 207, 328, 265, 104, 95, 54, 309, 260, 115, 59, 336, 182, 222, 50, 111, 345, 268, 49, 32, 215, 321, 26, 3, 33, 235, 140, 97, 284, 370, 273, 373, 312, 377, 329, 45, 217, 358, 119, 295, 155, 224, 270, 213, 298, 133, 24, 301, 313, 239, 228, 195, 375, 348, 290, 240, 53, 206, 269, 276, 94, 371, 234, 73, 107, 294, 160, 2, 205, 211, 185, 108, 300, 178, 369, 127, 337, 289, 339, 177, 351, 7, 305, 12, 248, 330, 355, 354, 171, 98, 179, 11, 324, 101, 10, 83, 201, 173, 190, 72, 31, 266, 120, 167, 227, 52, 368, 307, 84, 23, 264, 136, 365, 29, 109, 376, 210, 232, 56, 18, 231, 25, 299, 67, 14, 352, 262, 110, 366, 9, 62, 296, 220, 334, 256, 314, 51, 184, 130, 121, 249, 274, 6, 41, 96, 71, 48, 19, 165, 154, 116, 359, 102, 247, 319, 174, 208, 267, 151, 378, 81, 152, 20, 242, 91, 357, 172, 278, 356, 135, 347, 297, 191, 204, 153, 42, 170, 34, 63, 320, 46, 212, 89, 258, 148, 138, 285, 286, 87, 40, 60, 13, 21, 47, 8, 360, 245, 261, 279, 161, 225, 317, 254, 131, 44, 283, 241, 55, 230, 100, 316, 166, 188, 325, 27, 1, 128, 80, 311, 229, 38, 233, 218, 125, 287, 280, 304, 192, 199, 341, 88, 331, 79, 272, 271, 335, 16, 353, 363, 196, 275, 380, 149, 186, 75, 263, 183, 134, 379, 252, 169, 197, 250, 310, 85, 326, 303, 163, 322, 255, 145, 105, 28, 251, 143, 37, 243, 362, 202, 281, 288, 124, 259, 198, 200, 372, 238, 68, 383, 318, 293, 162, 323, 291, 147, 77, 209, 61, 76, 157, 103, 70, 216, 142, 158, 144, 221, 302, 118, 66, 364, 332, 257, 93, 374] + 51: [24, 221, 110, 208, 342, 229, 108, 73, 178, 16, 113, 52, 127, 64, 358, 65, 87, 104, 260, 235, 330, 105, 153, 177, 269, 346, 134, 247, 13, 46, 118, 85, 331, 212, 344, 97, 172, 341, 56, 107, 18, 329, 190, 59, 90, 370, 251, 33, 141, 3, 361, 234, 157, 205, 262, 317, 213, 115, 312, 381, 379, 54, 328, 80, 307, 319, 273, 340, 197, 53, 238, 167, 184, 68, 66, 57, 4, 215, 14, 158, 154, 0, 132, 355, 175, 179, 266, 29, 22, 244, 267, 155, 327, 138, 156, 202, 186, 375, 210, 12, 337, 126, 322, 189, 6, 111, 123, 182, 356, 136, 67, 62, 152, 60, 40, 301, 325, 304, 146, 253, 103, 130, 263, 5, 259, 143, 20, 220, 183, 47, 382, 81, 74, 142, 359, 162, 360, 209, 351, 310, 204, 119, 309, 166, 50, 75, 48, 239, 320, 364, 171, 84, 196, 94, 349, 45, 51, 223, 338, 288, 145, 78, 363, 159, 58, 109, 112, 240, 218, 116, 148, 83, 354, 296, 88, 165, 174, 77, 217, 133, 293, 302, 352, 117, 36, 347, 377, 173, 121, 151, 249, 261, 372, 373, 37, 323, 17, 82, 286, 106, 101, 98, 357, 284, 180, 176, 72, 211, 343, 270, 32, 95, 250, 233, 283, 168, 246, 49, 314, 26, 299, 216, 374, 23, 285, 245, 275, 308, 345, 86, 268, 279, 42, 366, 280, 79, 34, 91, 160, 252, 10, 195, 236, 224, 306, 69, 289, 313, 100, 332, 334, 297, 350, 27, 114, 194, 276, 278, 333, 55, 61, 336, 316, 25, 125, 149, 1, 96, 9, 7, 257, 199, 277, 120, 376, 227, 71, 187, 63, 318, 225, 38, 131, 21, 226, 139, 291, 326, 102, 137, 241, 200, 287, 144, 163, 31, 371, 206, 99, 193, 228, 122, 264, 300, 140, 135, 203, 124, 188, 129, 295, 282, 243, 11, 335, 92, 8, 35, 294, 281, 292, 271, 161, 303, 254, 237, 198, 380, 219, 222, 181, 192, 207, 248, 321, 368, 44, 362, 41, 164, 265, 324, 311, 369, 383, 258, 365, 315, 89, 170, 191, 30, 242, 298, 93, 353, 339, 378, 43, 232, 19, 15, 214, 230, 28, 39, 272, 201, 305, 128, 367, 70, 169, 185, 274, 147, 150, 231, 256, 76, 255, 2, 290, 348] + 52: [381, 104, 283, 336, 25, 327, 120, 216, 371, 63, 362, 85, 290, 345, 169, 77, 90, 183, 172, 62, 357, 44, 259, 192, 266, 32, 8, 354, 257, 325, 312, 133, 339, 222, 234, 191, 178, 359, 30, 23, 364, 75, 243, 369, 113, 361, 53, 115, 22, 213, 324, 208, 125, 109, 161, 225, 270, 237, 200, 177, 294, 282, 152, 207, 220, 230, 136, 249, 143, 28, 310, 322, 226, 340, 4, 96, 353, 135, 232, 276, 297, 329, 314, 203, 159, 65, 122, 142, 317, 14, 295, 272, 204, 370, 180, 247, 99, 71, 103, 137, 48, 186, 307, 61, 320, 182, 250, 289, 195, 174, 263, 298, 7, 382, 190, 33, 260, 140, 196, 219, 87, 54, 95, 50, 291, 88, 228, 379, 127, 378, 262, 5, 83, 315, 215, 160, 233, 248, 6, 300, 153, 72, 331, 351, 279, 316, 55, 287, 253, 326, 268, 235, 352, 57, 242, 258, 365, 20, 21, 194, 51, 47, 330, 347, 303, 101, 380, 118, 344, 18, 126, 311, 358, 138, 274, 363, 9, 36, 185, 42, 273, 265, 338, 144, 373, 254, 376, 360, 162, 16, 305, 375, 334, 157, 165, 239, 52, 92, 332, 217, 31, 130, 73, 214, 321, 206, 193, 163, 184, 13, 346, 68, 245, 80, 128, 168, 210, 252, 64, 98, 70, 37, 123, 108, 256, 285, 117, 166, 296, 227, 124, 167, 29, 116, 280, 292, 3, 76, 17, 319, 39, 69, 333, 91, 45, 27, 308, 350, 58, 201, 383, 284, 148, 59, 84, 251, 318, 150, 368, 129, 188, 156, 141, 342, 43, 81, 299, 313, 181, 293, 112, 147, 132, 145, 187, 79, 11, 202, 366, 241, 1, 86, 94, 199, 218, 46, 175, 348, 102, 107, 97, 209, 205, 121, 231, 372, 267, 151, 246, 323, 271, 34, 211, 82, 78, 41, 288, 60, 10, 26, 56, 269, 355, 349, 149, 236, 309, 35, 155, 171, 74, 221, 286, 105, 341, 281, 173, 223, 367, 134, 146, 67, 301, 328, 154, 377, 374, 170, 66, 337, 40, 306, 119, 93, 89, 2, 110, 277, 224, 244, 238, 189, 139, 179, 100, 198, 49, 356, 278, 197, 111, 114, 335, 343, 229, 304, 302, 275, 158, 261, 255, 131, 12, 19, 15, 0, 24, 240, 164, 264, 106, 212, 38, 176] + 53: [110, 311, 330, 308, 68, 297, 66, 173, 322, 273, 363, 155, 92, 132, 156, 6, 351, 194, 261, 144, 130, 121, 368, 284, 319, 127, 219, 256, 62, 262, 69, 361, 116, 205, 37, 313, 49, 373, 149, 47, 349, 315, 293, 282, 67, 226, 340, 120, 339, 178, 375, 371, 327, 96, 19, 28, 255, 123, 299, 220, 294, 0, 195, 275, 201, 7, 137, 246, 9, 249, 82, 51, 310, 147, 348, 64, 229, 372, 135, 260, 356, 190, 27, 181, 286, 187, 355, 245, 290, 265, 252, 167, 316, 114, 210, 272, 240, 13, 334, 109, 347, 32, 230, 217, 222, 238, 129, 142, 46, 379, 179, 170, 302, 160, 360, 268, 291, 237, 81, 119, 221, 31, 20, 24, 95, 325, 200, 94, 346, 141, 341, 383, 77, 172, 378, 197, 277, 176, 281, 89, 76, 44, 102, 143, 328, 36, 248, 216, 169, 65, 166, 57, 207, 236, 117, 333, 133, 364, 232, 59, 55, 54, 337, 352, 53, 367, 131, 264, 43, 100, 177, 39, 298, 74, 106, 366, 93, 208, 382, 211, 41, 192, 343, 33, 204, 11, 99, 223, 314, 56, 18, 185, 73, 317, 350, 287, 354, 250, 336, 8, 243, 270, 320, 292, 214, 345, 227, 228, 266, 180, 209, 309, 376, 162, 225, 103, 285, 50, 269, 253, 171, 3, 88, 139, 12, 70, 338, 231, 188, 152, 324, 78, 52, 10, 150, 15, 159, 122, 175, 362, 357, 128, 35, 196, 206, 105, 276, 189, 306, 278, 79, 136, 40, 198, 218, 242, 97, 258, 257, 234, 108, 14, 289, 283, 323, 199, 48, 63, 342, 301, 113, 34, 163, 374, 263, 161, 91, 212, 98, 165, 304, 191, 344, 244, 377, 90, 140, 84, 38, 16, 23, 296, 295, 4, 186, 75, 203, 247, 164, 280, 365, 224, 42, 118, 145, 279, 202, 86, 157, 332, 26, 271, 134, 125, 107, 326, 241, 5, 168, 329, 25, 381, 112, 183, 151, 239, 335, 300, 235, 60, 259, 72, 126, 353, 359, 146, 233, 369, 111, 115, 2, 174, 321, 158, 370, 30, 215, 358, 138, 182, 251, 254, 101, 45, 87, 21, 154, 80, 380, 305, 331, 29, 83, 85, 1, 193, 17, 184, 303, 104, 267, 148, 288, 61, 124, 213, 274, 318, 71, 58, 307, 22, 153, 312] + 54: [363, 258, 231, 280, 95, 296, 328, 147, 287, 150, 214, 250, 8, 72, 176, 120, 345, 305, 352, 40, 7, 3, 317, 58, 10, 132, 76, 333, 196, 215, 158, 357, 225, 45, 5, 190, 272, 109, 279, 239, 184, 105, 186, 355, 131, 100, 246, 278, 264, 366, 85, 293, 137, 242, 304, 81, 167, 173, 347, 300, 275, 191, 238, 194, 251, 12, 88, 224, 235, 168, 211, 236, 289, 79, 268, 343, 362, 226, 270, 285, 116, 307, 28, 379, 2, 112, 37, 165, 349, 200, 358, 354, 0, 36, 148, 232, 370, 284, 11, 240, 230, 295, 153, 319, 83, 267, 82, 369, 98, 53, 281, 208, 44, 187, 87, 119, 309, 118, 75, 329, 181, 332, 276, 143, 47, 218, 129, 64, 335, 124, 323, 94, 265, 92, 159, 382, 381, 21, 157, 312, 55, 302, 183, 67, 144, 373, 301, 163, 327, 22, 209, 201, 146, 353, 195, 18, 244, 316, 175, 266, 245, 271, 63, 213, 15, 234, 74, 247, 249, 31, 261, 89, 359, 60, 46, 227, 360, 154, 51, 104, 134, 77, 252, 126, 207, 78, 344, 130, 106, 13, 320, 336, 294, 164, 42, 223, 38, 286, 337, 253, 189, 90, 216, 260, 30, 20, 306, 341, 282, 290, 330, 311, 26, 180, 314, 86, 27, 121, 375, 288, 41, 310, 113, 197, 277, 84, 17, 220, 321, 185, 34, 182, 54, 103, 140, 204, 97, 365, 315, 318, 1, 73, 298, 48, 255, 111, 107, 313, 62, 96, 33, 162, 151, 262, 23, 9, 351, 192, 229, 334, 259, 110, 346, 145, 14, 367, 206, 233, 269, 263, 66, 102, 56, 374, 292, 65, 210, 169, 138, 188, 241, 297, 193, 383, 212, 123, 160, 361, 339, 155, 283, 356, 348, 299, 380, 71, 338, 39, 257, 80, 133, 172, 174, 324, 254, 93, 221, 57, 25, 117, 372, 228, 16, 101, 156, 70, 99, 115, 32, 161, 322, 198, 50, 91, 325, 152, 217, 142, 24, 125, 170, 376, 237, 368, 49, 135, 177, 61, 171, 69, 68, 326, 122, 136, 179, 364, 114, 377, 166, 273, 43, 128, 141, 139, 243, 371, 308, 219, 6, 59, 178, 29, 274, 331, 52, 108, 248, 203, 199, 149, 35, 303, 4, 350, 291, 222, 256, 342, 340, 19, 202, 127, 205, 378] + 55: [154, 20, 340, 123, 347, 130, 113, 159, 256, 309, 7, 357, 339, 238, 111, 6, 211, 78, 382, 132, 49, 341, 293, 114, 179, 60, 333, 368, 216, 366, 383, 124, 233, 17, 46, 348, 284, 91, 274, 260, 378, 224, 246, 343, 79, 53, 228, 140, 19, 119, 122, 128, 171, 52, 94, 33, 314, 214, 264, 1, 342, 127, 365, 173, 15, 335, 372, 93, 201, 38, 55, 147, 161, 125, 66, 68, 324, 162, 183, 364, 295, 175, 334, 75, 110, 195, 100, 24, 351, 292, 45, 310, 222, 245, 196, 34, 174, 98, 308, 381, 48, 180, 115, 72, 327, 235, 4, 370, 39, 311, 192, 267, 353, 273, 70, 65, 106, 249, 134, 71, 3, 272, 44, 138, 221, 312, 26, 156, 239, 197, 163, 325, 276, 352, 289, 371, 27, 377, 231, 47, 306, 286, 187, 185, 263, 184, 318, 255, 157, 207, 101, 149, 313, 63, 31, 237, 232, 281, 367, 208, 219, 253, 97, 126, 336, 168, 243, 58, 354, 150, 145, 261, 13, 166, 77, 376, 298, 181, 282, 36, 266, 288, 158, 59, 131, 379, 205, 144, 250, 317, 69, 305, 56, 81, 146, 116, 90, 323, 29, 326, 152, 43, 151, 62, 218, 89, 252, 227, 328, 41, 247, 363, 301, 40, 2, 223, 226, 279, 12, 331, 164, 213, 316, 271, 206, 337, 189, 112, 102, 14, 95, 105, 210, 217, 236, 265, 240, 67, 178, 54, 332, 322, 257, 346, 23, 254, 193, 84, 87, 22, 117, 153, 212, 160, 103, 133, 37, 225, 268, 356, 319, 118, 209, 104, 338, 120, 275, 374, 139, 177, 203, 242, 141, 92, 262, 21, 244, 32, 108, 220, 345, 148, 167, 9, 51, 96, 109, 204, 202, 88, 11, 80, 299, 74, 269, 76, 28, 270, 355, 234, 182, 165, 16, 307, 303, 73, 121, 304, 199, 291, 200, 99, 176, 8, 320, 349, 290, 300, 0, 86, 344, 172, 259, 5, 170, 350, 190, 375, 329, 251, 380, 361, 360, 42, 135, 194, 136, 241, 10, 277, 287, 358, 278, 85, 296, 294, 302, 82, 198, 61, 258, 188, 143, 373, 359, 30, 35, 321, 297, 137, 129, 315, 107, 50, 64, 248, 369, 280, 169, 191, 155, 330, 362, 57, 142, 83, 230, 18, 229, 25, 283, 186, 285, 215] + 56: [231, 242, 82, 241, 30, 147, 8, 126, 273, 53, 98, 303, 371, 169, 356, 236, 333, 125, 117, 161, 71, 99, 292, 18, 332, 327, 154, 245, 283, 101, 383, 309, 197, 233, 334, 27, 285, 281, 137, 278, 92, 181, 252, 79, 354, 269, 12, 22, 11, 34, 153, 243, 141, 282, 364, 42, 378, 122, 201, 299, 298, 96, 145, 218, 228, 132, 307, 200, 330, 276, 65, 367, 78, 144, 352, 222, 373, 24, 211, 258, 26, 256, 215, 370, 326, 32, 377, 365, 114, 247, 235, 61, 76, 74, 146, 3, 240, 321, 178, 15, 340, 75, 100, 343, 199, 116, 163, 158, 351, 156, 20, 89, 264, 306, 162, 328, 316, 284, 275, 302, 19, 66, 159, 357, 244, 192, 72, 97, 344, 103, 300, 182, 69, 342, 193, 128, 324, 305, 31, 295, 374, 268, 317, 149, 214, 138, 13, 17, 127, 185, 56, 286, 115, 313, 360, 94, 55, 134, 262, 341, 310, 290, 323, 226, 379, 57, 139, 372, 266, 170, 43, 350, 113, 204, 131, 337, 108, 272, 68, 212, 59, 336, 45, 216, 380, 171, 237, 102, 109, 142, 173, 90, 195, 294, 44, 225, 106, 5, 279, 311, 191, 107, 155, 296, 267, 87, 189, 249, 104, 251, 308, 346, 320, 140, 219, 2, 83, 329, 288, 381, 168, 150, 120, 223, 369, 0, 58, 73, 38, 160, 250, 10, 105, 21, 136, 35, 179, 23, 355, 280, 50, 110, 151, 164, 123, 209, 129, 297, 190, 339, 62, 271, 183, 180, 84, 119, 172, 347, 28, 4, 232, 175, 291, 16, 29, 49, 70, 361, 93, 152, 254, 345, 234, 325, 239, 349, 261, 7, 36, 217, 25, 174, 91, 318, 301, 85, 315, 143, 260, 220, 265, 213, 259, 304, 177, 167, 81, 186, 88, 184, 322, 9, 196, 263, 363, 187, 6, 46, 48, 39, 41, 188, 362, 77, 80, 14, 63, 207, 118, 238, 246, 205, 359, 376, 40, 148, 312, 253, 135, 206, 198, 338, 166, 230, 121, 331, 1, 133, 255, 33, 270, 227, 248, 368, 221, 95, 348, 210, 257, 54, 314, 353, 375, 293, 124, 112, 319, 274, 37, 289, 67, 64, 86, 130, 60, 287, 47, 157, 382, 358, 176, 203, 52, 335, 366, 277, 111, 194, 165, 51, 229, 224, 202, 208] + 57: [254, 165, 205, 331, 170, 74, 11, 31, 314, 85, 198, 140, 257, 223, 124, 344, 118, 217, 150, 174, 117, 22, 88, 185, 44, 250, 238, 347, 338, 316, 373, 176, 134, 243, 51, 70, 42, 173, 83, 191, 102, 183, 203, 358, 12, 13, 119, 279, 166, 313, 352, 353, 209, 321, 155, 288, 186, 323, 162, 175, 237, 332, 315, 33, 303, 265, 200, 270, 32, 188, 219, 224, 50, 194, 256, 138, 364, 103, 139, 142, 10, 339, 114, 294, 327, 193, 292, 305, 37, 131, 201, 202, 56, 234, 293, 343, 276, 146, 89, 148, 94, 286, 154, 342, 158, 133, 220, 7, 366, 65, 274, 137, 335, 268, 300, 6, 127, 151, 345, 160, 309, 62, 75, 182, 144, 167, 365, 43, 322, 40, 73, 214, 177, 96, 20, 277, 382, 111, 54, 92, 120, 55, 380, 95, 348, 226, 71, 196, 351, 337, 369, 362, 330, 98, 17, 18, 379, 261, 26, 247, 252, 122, 236, 248, 307, 0, 87, 241, 109, 383, 242, 374, 48, 239, 287, 61, 310, 271, 115, 258, 82, 206, 269, 336, 69, 77, 153, 229, 19, 367, 376, 304, 326, 14, 259, 246, 207, 320, 157, 159, 381, 16, 368, 63, 104, 128, 359, 324, 141, 8, 60, 350, 84, 197, 180, 79, 35, 123, 251, 354, 99, 58, 39, 172, 179, 213, 297, 228, 152, 113, 298, 312, 21, 377, 147, 38, 289, 284, 325, 328, 68, 2, 136, 149, 3, 190, 296, 28, 178, 132, 311, 334, 41, 240, 97, 363, 5, 45, 211, 59, 255, 281, 168, 357, 66, 263, 36, 222, 67, 370, 107, 308, 360, 264, 244, 291, 129, 52, 340, 30, 195, 181, 280, 105, 233, 86, 262, 372, 25, 231, 295, 187, 72, 125, 302, 93, 230, 121, 57, 156, 235, 101, 108, 53, 130, 267, 169, 204, 318, 283, 317, 64, 306, 163, 80, 272, 319, 91, 253, 161, 110, 346, 9, 341, 27, 249, 46, 356, 349, 184, 116, 208, 47, 106, 112, 225, 329, 210, 164, 171, 199, 282, 90, 23, 216, 81, 215, 29, 371, 290, 145, 49, 355, 135, 24, 299, 378, 4, 126, 1, 333, 260, 361, 192, 375, 232, 76, 78, 273, 266, 100, 218, 212, 221, 189, 278, 227, 245, 15, 34, 143, 285, 275, 301] + 58: [377, 222, 298, 77, 61, 7, 344, 271, 299, 22, 358, 355, 270, 112, 175, 152, 376, 18, 85, 42, 31, 272, 315, 255, 106, 190, 197, 185, 285, 329, 48, 331, 251, 207, 143, 266, 325, 238, 90, 124, 160, 333, 37, 149, 110, 241, 13, 253, 176, 327, 334, 371, 21, 11, 287, 225, 353, 136, 105, 180, 297, 24, 213, 142, 372, 264, 86, 268, 282, 146, 290, 46, 223, 369, 120, 145, 216, 15, 78, 301, 174, 310, 363, 23, 165, 320, 279, 3, 125, 370, 364, 33, 38, 58, 134, 209, 220, 126, 305, 98, 177, 263, 151, 79, 27, 211, 373, 206, 49, 72, 214, 26, 169, 116, 232, 295, 342, 341, 367, 191, 12, 104, 111, 316, 20, 307, 224, 198, 229, 281, 202, 314, 244, 59, 62, 258, 236, 53, 155, 227, 183, 382, 0, 332, 144, 172, 357, 186, 107, 161, 345, 43, 70, 293, 195, 35, 8, 173, 338, 64, 243, 261, 360, 87, 317, 221, 303, 354, 200, 337, 51, 247, 96, 109, 32, 153, 17, 88, 30, 267, 140, 323, 361, 291, 80, 252, 71, 319, 10, 168, 201, 308, 81, 275, 366, 92, 311, 122, 248, 76, 273, 348, 14, 322, 235, 69, 351, 257, 256, 368, 36, 262, 196, 324, 47, 119, 189, 97, 132, 162, 4, 274, 66, 171, 28, 380, 192, 139, 381, 25, 82, 129, 99, 215, 306, 250, 234, 346, 240, 131, 63, 302, 362, 148, 231, 65, 123, 318, 347, 265, 115, 283, 50, 237, 219, 118, 226, 294, 296, 29, 95, 54, 330, 163, 41, 254, 374, 84, 269, 39, 57, 246, 159, 52, 193, 127, 365, 218, 343, 89, 212, 164, 147, 292, 321, 108, 184, 204, 245, 156, 56, 205, 194, 312, 114, 249, 259, 356, 210, 101, 93, 9, 289, 67, 199, 40, 178, 68, 45, 141, 6, 203, 16, 277, 130, 60, 260, 352, 242, 328, 73, 187, 1, 375, 55, 239, 383, 181, 284, 350, 83, 339, 326, 378, 75, 121, 135, 300, 208, 379, 154, 288, 278, 2, 228, 113, 19, 182, 179, 133, 94, 230, 170, 5, 102, 138, 100, 128, 359, 276, 157, 336, 103, 44, 167, 349, 117, 34, 137, 304, 286, 188, 74, 217, 91, 233, 313, 166, 158, 309, 335, 150, 280, 340] + 59: [327, 224, 94, 59, 340, 352, 73, 30, 46, 197, 5, 381, 247, 294, 331, 185, 172, 295, 9, 317, 262, 243, 309, 58, 133, 176, 278, 223, 57, 12, 16, 297, 364, 334, 383, 252, 143, 60, 245, 286, 214, 236, 222, 120, 137, 177, 238, 337, 365, 131, 123, 242, 199, 107, 48, 333, 138, 43, 41, 135, 187, 231, 134, 157, 28, 74, 89, 261, 44, 183, 191, 148, 273, 166, 230, 312, 303, 301, 108, 318, 282, 350, 14, 270, 272, 244, 347, 345, 139, 116, 52, 351, 111, 90, 79, 189, 25, 209, 379, 260, 173, 346, 65, 182, 105, 256, 233, 70, 311, 263, 221, 150, 42, 213, 186, 38, 178, 144, 266, 196, 86, 112, 212, 109, 161, 85, 254, 279, 181, 211, 163, 320, 376, 22, 174, 142, 329, 10, 355, 69, 198, 348, 92, 259, 155, 36, 228, 2, 61, 314, 32, 299, 287, 193, 63, 149, 129, 136, 184, 322, 67, 343, 359, 93, 49, 3, 24, 106, 11, 321, 296, 284, 358, 82, 165, 237, 239, 240, 360, 56, 335, 293, 264, 87, 78, 83, 72, 99, 235, 117, 341, 271, 170, 140, 370, 195, 363, 220, 332, 113, 75, 313, 147, 251, 374, 371, 130, 76, 31, 300, 324, 328, 23, 80, 382, 232, 114, 342, 91, 175, 40, 160, 168, 127, 100, 302, 169, 162, 373, 225, 218, 338, 54, 356, 179, 339, 269, 180, 366, 354, 203, 6, 353, 51, 68, 274, 362, 0, 206, 380, 64, 110, 325, 208, 84, 124, 21, 207, 153, 145, 277, 88, 249, 115, 118, 53, 119, 102, 241, 368, 319, 98, 141, 1, 192, 219, 121, 77, 171, 205, 126, 104, 122, 323, 308, 15, 188, 154, 229, 7, 250, 202, 200, 66, 234, 33, 285, 97, 226, 95, 298, 204, 291, 288, 18, 265, 372, 377, 151, 26, 281, 255, 128, 290, 45, 164, 304, 81, 35, 159, 267, 280, 62, 248, 315, 20, 29, 19, 306, 47, 158, 258, 344, 146, 369, 361, 289, 13, 101, 349, 156, 96, 292, 194, 227, 71, 27, 246, 34, 50, 253, 217, 17, 330, 37, 326, 336, 132, 210, 275, 201, 357, 307, 103, 216, 283, 257, 8, 316, 276, 305, 125, 39, 375, 367, 378, 152, 55, 190, 167, 4, 310, 268, 215] + 60: [313, 381, 139, 133, 265, 94, 128, 195, 303, 372, 316, 78, 247, 65, 297, 208, 205, 239, 76, 49, 365, 166, 345, 98, 245, 272, 143, 59, 241, 27, 169, 140, 84, 0, 362, 283, 269, 358, 183, 240, 80, 188, 382, 264, 215, 274, 304, 25, 141, 233, 107, 142, 285, 91, 149, 30, 300, 341, 95, 349, 380, 286, 225, 148, 325, 338, 156, 67, 251, 127, 209, 328, 179, 198, 145, 68, 204, 378, 298, 301, 159, 306, 43, 292, 164, 275, 295, 352, 109, 10, 226, 373, 63, 207, 22, 355, 124, 268, 47, 83, 174, 102, 335, 115, 129, 347, 339, 48, 167, 312, 334, 155, 28, 366, 294, 337, 180, 112, 193, 354, 278, 131, 231, 203, 271, 263, 99, 172, 75, 189, 58, 20, 93, 37, 217, 259, 11, 88, 267, 110, 38, 57, 279, 305, 323, 186, 70, 90, 273, 66, 200, 250, 182, 130, 357, 314, 296, 256, 14, 371, 197, 383, 282, 168, 202, 13, 252, 287, 19, 216, 280, 42, 374, 211, 29, 147, 72, 86, 249, 17, 281, 289, 71, 4, 32, 101, 77, 242, 62, 214, 348, 201, 351, 16, 220, 126, 40, 310, 236, 170, 243, 244, 39, 222, 230, 153, 302, 181, 74, 34, 108, 234, 353, 119, 329, 15, 213, 232, 184, 161, 235, 55, 246, 359, 336, 151, 277, 255, 114, 6, 36, 318, 92, 332, 162, 5, 3, 377, 8, 342, 134, 171, 346, 360, 187, 85, 33, 196, 190, 192, 106, 322, 276, 144, 18, 64, 315, 333, 105, 270, 262, 45, 330, 324, 137, 261, 41, 104, 379, 229, 97, 96, 291, 191, 326, 135, 173, 163, 138, 340, 44, 118, 87, 308, 210, 317, 299, 288, 123, 257, 158, 60, 199, 284, 121, 260, 266, 343, 113, 212, 194, 120, 185, 81, 2, 327, 35, 319, 290, 46, 311, 309, 206, 122, 117, 223, 238, 82, 364, 253, 152, 150, 69, 248, 221, 132, 51, 1, 344, 24, 9, 237, 61, 176, 116, 56, 26, 21, 52, 53, 31, 293, 7, 146, 100, 369, 79, 370, 368, 219, 12, 331, 125, 363, 307, 376, 23, 361, 178, 228, 320, 54, 258, 50, 224, 160, 356, 111, 175, 165, 350, 254, 227, 367, 177, 103, 73, 321, 218, 89, 136, 157, 154, 375] + 61: [221, 211, 381, 315, 263, 88, 303, 136, 120, 247, 122, 203, 241, 264, 363, 218, 12, 278, 238, 235, 237, 119, 327, 115, 34, 42, 31, 160, 227, 133, 87, 258, 224, 65, 268, 29, 300, 190, 313, 11, 109, 336, 69, 368, 207, 200, 102, 173, 72, 137, 346, 214, 275, 9, 365, 80, 269, 141, 340, 71, 124, 14, 342, 298, 162, 345, 63, 54, 354, 18, 364, 358, 312, 331, 246, 272, 95, 28, 48, 236, 250, 222, 60, 285, 254, 355, 382, 267, 376, 217, 266, 350, 116, 53, 94, 0, 127, 93, 197, 167, 248, 86, 231, 5, 360, 326, 351, 117, 270, 184, 45, 353, 330, 17, 294, 215, 213, 66, 291, 57, 177, 188, 15, 282, 32, 75, 92, 361, 362, 150, 233, 366, 83, 27, 113, 374, 2, 257, 245, 328, 52, 158, 290, 139, 186, 89, 292, 30, 375, 341, 22, 79, 283, 67, 295, 132, 199, 284, 204, 152, 370, 274, 352, 25, 310, 301, 101, 99, 49, 24, 13, 305, 180, 182, 146, 339, 318, 347, 64, 377, 279, 131, 210, 50, 367, 325, 334, 357, 343, 321, 149, 155, 179, 61, 299, 100, 344, 307, 107, 230, 335, 157, 165, 323, 19, 209, 380, 103, 21, 51, 68, 142, 143, 112, 129, 168, 110, 159, 349, 243, 169, 261, 228, 191, 348, 297, 316, 114, 135, 82, 252, 304, 47, 308, 206, 145, 104, 202, 33, 259, 322, 38, 78, 306, 8, 37, 125, 126, 85, 196, 26, 317, 176, 194, 359, 148, 70, 311, 91, 163, 187, 255, 234, 251, 7, 262, 185, 314, 164, 212, 223, 105, 35, 220, 56, 226, 151, 175, 134, 62, 41, 320, 16, 128, 356, 189, 208, 319, 338, 73, 46, 36, 324, 161, 216, 172, 108, 170, 205, 178, 10, 59, 1, 253, 195, 372, 371, 198, 55, 192, 288, 333, 369, 286, 181, 156, 309, 43, 337, 20, 118, 77, 90, 84, 44, 166, 121, 379, 329, 260, 3, 23, 97, 147, 249, 378, 154, 383, 153, 273, 280, 174, 242, 219, 225, 240, 81, 4, 144, 332, 98, 106, 193, 244, 130, 271, 239, 6, 123, 74, 232, 111, 201, 293, 373, 138, 265, 140, 256, 276, 302, 277, 281, 39, 289, 183, 76, 229, 296, 40, 171, 96, 287, 58] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots416.yaml b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots416.yaml new file mode 100644 index 000000000000..93595be1ffc3 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots416.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [259, 151, 74, 356, 356, 322, 245, 225, 177, 165, 254, 135, 76, 193, 346, 332, 312, 295, 26, 60, 104, 242, 17, 95, 184, 232, 255, 376, 139, 68, 37, 333, 173, 290, 280, 282, 55, 143, 205, 54, 11, 105, 313, 56, 354, 297, 129, 258, 324, 368, 7, 170, 149, 309, 314, 38, 52, 5, 91, 172, 227, 4, 108, 249, 363, 372, 275, 59, 215, 97, 106, 270, 299, 73, 267, 67, 260, 50, 252, 98, 294, 352, 316, 340, 380, 12, 219, 179, 381, 9, 357, 250, 298, 92, 287, 24, 228, 128, 344, 288, 118, 13, 110, 303, 166, 256, 222, 302, 159, 236, 371, 383, 162, 165, 111, 235, 279, 241, 378, 45, 188, 35, 347, 153, 202, 30, 80, 95, 134, 49, 321, 70, 203, 176, 276, 16, 330, 290, 144, 209, 360, 359, 307, 284, 131, 196, 341, 233, 109, 186, 361, 311, 112, 138, 44, 148, 31, 27, 199, 358, 87, 85, 308, 257, 213, 136, 240, 365, 140, 246, 19, 366, 180, 238, 320, 195, 82, 127, 328, 263, 362, 47, 102, 350, 161, 40, 64, 206, 315, 175, 2, 152, 301, 239, 220, 125, 167, 271, 329, 226, 291, 339, 8, 88, 310, 58, 373, 61, 166, 150, 71, 222, 345, 236, 371, 225, 155, 165, 111, 135, 279, 158, 378, 45, 169, 51, 122, 81, 79, 204, 29, 218, 184, 327, 289, 65, 203, 217, 278, 198, 173, 224, 3, 349, 336, 78, 72, 243, 117, 34, 283, 156, 10, 124, 265, 200, 261, 114, 248, 337, 123, 268, 89, 234, 66, 39, 133, 244, 335, 103, 146, 331, 116, 178, 216, 192, 296, 100, 379, 292, 83, 147, 326, 237, 342, 20, 210, 286, 338, 42, 212, 253, 53, 41, 113, 334, 194, 231, 285, 164, 18, 142, 304, 121, 281, 364, 273, 130, 370, 171, 201, 247, 0, 256, 74, 351, 84, 119, 245, 86, 348, 254, 111, 264, 207, 230, 378, 190, 182, 163, 60, 21, 79, 62, 17, 157, 134, 49, 48, 343, 63, 217, 293, 318, 330, 224, 272, 126, 22, 141, 181, 33, 101, 107, 183, 43, 57, 90, 36, 174, 94, 28, 300, 208, 77, 160, 262, 375, 223, 377, 187, 32, 214, 317, 46, 137, 75, 269, 99, 369, 211, 1, 25, 221, 197, 382, 93, 14, 145, 6, 154, 229, 305, 15, 96, 115, 306, 353, 69, 191, 319, 367, 266, 251, 132, 23, 189, 374, 325, 355, 323, 168, 277, 120, 274, 185] + 1: [134, 365, 42, 174, 174, 137, 180, 158, 382, 382, 43, 72, 177, 167, 163, 329, 319, 319, 241, 39, 366, 0, 210, 73, 105, 182, 294, 120, 225, 111, 271, 82, 283, 272, 29, 195, 269, 275, 32, 247, 245, 266, 274, 99, 187, 84, 194, 280, 223, 52, 206, 93, 203, 264, 202, 239, 147, 60, 98, 256, 7, 313, 222, 259, 370, 2, 20, 14, 296, 184, 136, 57, 172, 36, 284, 51, 311, 13, 302, 55, 12, 232, 119, 314, 190, 130, 246, 297, 160, 367, 207, 217, 10, 252, 343, 64, 117, 337, 341, 354, 112, 336, 34, 100, 178, 81, 235, 201, 22, 118, 180, 158, 277, 150, 50, 91, 169, 167, 163, 254, 319, 347, 168, 6, 5, 70, 377, 41, 63, 58, 244, 198, 23, 248, 185, 82, 383, 272, 243, 292, 9, 145, 86, 242, 125, 281, 74, 212, 197, 288, 261, 109, 114, 213, 176, 228, 289, 306, 376, 278, 236, 95, 205, 189, 164, 79, 348, 330, 65, 139, 286, 326, 19, 175, 33, 321, 290, 35, 230, 350, 352, 94, 323, 346, 344, 240, 333, 103, 83, 369, 331, 129, 316, 90, 151, 221, 132, 361, 16, 45, 258, 159, 28, 157, 312, 345, 265, 255, 380, 200, 1, 42, 31, 137, 44, 80, 85, 126, 263, 72, 177, 291, 163, 127, 21, 347, 67, 192, 270, 75, 377, 110, 3, 3, 299, 198, 250, 111, 153, 24, 25, 320, 138, 273, 303, 211, 71, 173, 216, 78, 285, 301, 218, 108, 92, 335, 143, 4, 53, 116, 381, 69, 360, 342, 61, 231, 237, 76, 121, 362, 305, 238, 338, 161, 318, 17, 156, 295, 165, 62, 48, 186, 293, 11, 97, 96, 8, 113, 54, 262, 49, 356, 357, 375, 372, 220, 340, 188, 373, 135, 66, 124, 155, 191, 144, 276, 359, 257, 199, 310, 15, 215, 141, 251, 365, 201, 22, 308, 18, 106, 277, 149, 152, 91, 169, 196, 127, 378, 40, 131, 67, 30, 179, 325, 123, 328, 334, 182, 300, 56, 120, 225, 351, 233, 224, 46, 59, 292, 37, 287, 260, 89, 234, 317, 355, 146, 166, 162, 142, 379, 214, 219, 47, 282, 358, 88, 38, 171, 101, 133, 102, 298, 122, 140, 170, 204, 364, 249, 332, 107, 115, 229, 154, 68, 87, 324, 315, 26, 104, 227, 181, 268, 226, 353, 77, 27, 148, 279, 371, 309, 193, 183, 253, 209, 208, 128, 322, 374, 307, 349, 339, 304, 267, 363, 327, 368] + 2: [53, 96, 112, 108, 215, 89, 194, 115, 28, 81, 3, 295, 125, 143, 248, 197, 232, 149, 213, 185, 153, 111, 168, 322, 157, 91, 280, 286, 86, 135, 184, 172, 314, 139, 324, 22, 274, 200, 11, 235, 30, 6, 342, 204, 140, 170, 18, 347, 50, 156, 188, 320, 187, 201, 148, 198, 349, 41, 65, 241, 193, 15, 160, 276, 364, 106, 312, 252, 29, 104, 362, 23, 211, 134, 334, 238, 376, 331, 319, 231, 292, 302, 288, 99, 251, 167, 59, 98, 182, 336, 227, 222, 13, 353, 380, 88, 287, 358, 339, 237, 246, 375, 161, 379, 53, 132, 132, 225, 357, 14, 102, 115, 223, 1, 295, 321, 151, 212, 248, 33, 242, 232, 101, 19, 110, 62, 168, 191, 166, 131, 377, 278, 44, 271, 42, 304, 139, 51, 122, 26, 83, 63, 340, 244, 303, 159, 209, 165, 344, 17, 356, 263, 55, 269, 67, 92, 20, 383, 218, 297, 46, 163, 265, 308, 309, 169, 103, 69, 256, 136, 12, 49, 257, 150, 94, 0, 25, 239, 346, 4, 272, 141, 144, 352, 329, 54, 71, 371, 335, 205, 48, 74, 113, 382, 283, 345, 214, 240, 127, 64, 228, 301, 208, 202, 363, 40, 229, 374, 43, 180, 108, 351, 313, 215, 194, 93, 223, 5, 3, 137, 247, 143, 212, 248, 60, 192, 78, 290, 155, 62, 350, 124, 173, 91, 377, 70, 245, 306, 133, 105, 304, 45, 296, 325, 328, 279, 333, 85, 285, 311, 61, 79, 206, 7, 90, 224, 293, 126, 262, 327, 21, 58, 128, 236, 360, 277, 10, 145, 348, 181, 250, 116, 9, 38, 305, 121, 268, 365, 152, 219, 36, 275, 367, 47, 52, 207, 233, 359, 216, 226, 370, 373, 338, 8, 164, 158, 117, 72, 234, 147, 317, 260, 289, 80, 326, 119, 270, 282, 255, 190, 366, 330, 43, 230, 112, 341, 313, 89, 343, 93, 298, 259, 221, 189, 100, 143, 34, 75, 242, 162, 290, 19, 109, 84, 97, 284, 355, 66, 56, 73, 86, 118, 123, 220, 314, 31, 122, 368, 146, 217, 82, 24, 323, 196, 174, 294, 130, 299, 273, 87, 27, 68, 203, 210, 361, 120, 76, 138, 332, 253, 16, 171, 175, 186, 254, 129, 372, 154, 39, 142, 318, 176, 179, 2, 195, 77, 281, 35, 178, 264, 261, 57, 243, 291, 107, 95, 183, 114, 267, 307, 378, 310, 37, 177, 369, 249, 315, 266, 316, 32, 337, 354, 300, 199, 381, 258] + 3: [39, 66, 71, 34, 79, 275, 260, 230, 314, 40, 319, 171, 51, 27, 163, 192, 194, 97, 263, 24, 363, 145, 50, 42, 215, 26, 102, 120, 336, 227, 211, 325, 212, 304, 177, 316, 64, 21, 202, 264, 136, 162, 142, 154, 288, 262, 374, 243, 300, 181, 108, 218, 200, 55, 344, 112, 355, 135, 266, 246, 268, 82, 144, 160, 306, 322, 298, 361, 305, 291, 15, 83, 123, 61, 259, 228, 321, 309, 133, 206, 179, 183, 116, 16, 88, 121, 191, 45, 141, 253, 148, 89, 352, 315, 4, 69, 326, 341, 247, 335, 324, 278, 311, 301, 39, 66, 46, 34, 34, 310, 260, 377, 302, 365, 319, 14, 350, 225, 163, 208, 194, 263, 80, 348, 150, 334, 180, 331, 139, 238, 178, 170, 342, 250, 10, 267, 287, 74, 239, 36, 11, 290, 118, 382, 279, 8, 84, 289, 203, 229, 312, 114, 176, 98, 274, 103, 217, 380, 197, 327, 129, 65, 307, 44, 383, 214, 100, 237, 248, 340, 187, 20, 188, 281, 7, 106, 381, 85, 132, 68, 153, 293, 283, 285, 0, 245, 134, 251, 156, 338, 329, 369, 241, 147, 95, 75, 272, 175, 349, 152, 173, 226, 17, 332, 146, 199, 59, 38, 22, 66, 66, 34, 34, 310, 35, 377, 209, 365, 40, 18, 31, 27, 13, 3, 3, 313, 333, 151, 158, 49, 252, 137, 337, 339, 63, 343, 2, 358, 189, 231, 77, 296, 195, 182, 25, 223, 12, 357, 364, 78, 161, 359, 32, 277, 351, 297, 345, 375, 204, 224, 256, 166, 53, 57, 169, 117, 280, 255, 104, 30, 328, 196, 43, 115, 353, 249, 70, 232, 367, 376, 165, 130, 346, 282, 362, 101, 254, 93, 190, 378, 220, 323, 234, 107, 242, 159, 143, 54, 368, 127, 124, 308, 60, 244, 33, 185, 5, 52, 257, 292, 47, 222, 22, 66, 66, 34, 79, 310, 260, 377, 302, 365, 319, 18, 19, 317, 131, 56, 313, 261, 62, 157, 318, 81, 213, 6, 186, 201, 207, 168, 164, 58, 37, 122, 138, 221, 174, 219, 119, 240, 29, 87, 271, 109, 172, 99, 295, 149, 111, 92, 155, 41, 126, 299, 236, 235, 94, 270, 105, 273, 48, 113, 76, 125, 366, 276, 216, 286, 96, 184, 67, 128, 294, 91, 373, 356, 72, 110, 28, 86, 269, 258, 354, 379, 372, 73, 140, 210, 284, 360, 205, 90, 1, 330, 265, 23, 320, 347, 167, 193, 371, 9, 233, 198, 303, 370] + 4: [188, 209, 271, 124, 222, 343, 331, 331, 315, 350, 320, 193, 15, 172, 88, 19, 319, 257, 273, 150, 148, 152, 143, 241, 342, 345, 59, 155, 347, 246, 4, 372, 32, 365, 314, 44, 339, 8, 217, 251, 22, 82, 38, 207, 278, 335, 99, 264, 133, 166, 189, 72, 62, 352, 149, 75, 261, 125, 131, 313, 80, 354, 284, 310, 178, 39, 66, 306, 184, 346, 29, 157, 122, 180, 181, 304, 71, 223, 355, 156, 126, 81, 378, 21, 48, 353, 1, 247, 265, 134, 144, 224, 212, 89, 266, 299, 20, 192, 79, 242, 287, 58, 104, 120, 34, 18, 271, 375, 258, 300, 343, 331, 24, 259, 214, 23, 15, 9, 95, 376, 319, 87, 273, 171, 148, 152, 165, 136, 36, 56, 377, 167, 139, 113, 61, 275, 151, 196, 83, 101, 49, 28, 161, 280, 7, 369, 159, 141, 283, 244, 243, 42, 307, 53, 168, 114, 98, 93, 33, 323, 269, 74, 229, 383, 135, 322, 288, 137, 84, 220, 92, 349, 230, 78, 121, 175, 291, 249, 210, 334, 111, 31, 201, 213, 186, 381, 205, 129, 112, 163, 279, 191, 164, 366, 102, 248, 190, 301, 363, 154, 227, 225, 128, 67, 321, 63, 182, 277, 340, 3, 271, 37, 296, 296, 343, 331, 219, 250, 214, 371, 16, 228, 281, 226, 5, 105, 194, 117, 142, 254, 308, 348, 253, 47, 329, 298, 325, 41, 318, 200, 361, 364, 218, 160, 333, 312, 73, 198, 110, 260, 187, 179, 54, 324, 276, 116, 290, 252, 330, 256, 185, 297, 60, 14, 132, 108, 232, 70, 69, 240, 158, 328, 370, 309, 294, 50, 176, 234, 51, 26, 295, 327, 237, 118, 211, 268, 286, 130, 10, 215, 115, 203, 123, 362, 173, 162, 238, 30, 202, 293, 285, 236, 195, 270, 2, 119, 17, 231, 90, 336, 332, 351, 3, 3, 209, 12, 199, 343, 343, 331, 315, 315, 214, 371, 23, 239, 95, 226, 45, 87, 194, 117, 142, 254, 6, 302, 86, 13, 197, 274, 145, 76, 77, 106, 357, 235, 57, 43, 27, 204, 344, 68, 35, 317, 177, 153, 183, 358, 221, 0, 85, 65, 356, 360, 379, 359, 169, 341, 64, 46, 216, 262, 100, 25, 94, 367, 138, 292, 267, 52, 263, 206, 303, 103, 170, 146, 140, 289, 147, 109, 40, 127, 255, 96, 311, 97, 374, 316, 380, 55, 91, 368, 245, 11, 373, 326, 382, 174, 337, 208, 233, 272, 338, 107, 282, 305] + 5: [172, 190, 213, 213, 306, 296, 308, 5, 217, 47, 355, 60, 276, 100, 262, 305, 356, 287, 288, 272, 159, 259, 78, 21, 72, 279, 252, 7, 246, 91, 364, 74, 209, 371, 8, 137, 171, 163, 59, 2, 108, 206, 204, 143, 26, 314, 152, 377, 301, 35, 195, 186, 380, 274, 93, 39, 20, 353, 201, 121, 257, 161, 317, 289, 275, 330, 80, 87, 361, 94, 229, 62, 235, 221, 71, 114, 18, 135, 40, 298, 261, 125, 95, 292, 11, 111, 197, 85, 328, 336, 158, 234, 0, 273, 27, 367, 142, 278, 96, 179, 23, 321, 368, 302, 208, 154, 190, 249, 128, 296, 214, 194, 217, 65, 355, 49, 265, 105, 188, 305, 381, 365, 52, 6, 231, 159, 21, 41, 130, 351, 17, 43, 293, 147, 198, 25, 31, 233, 66, 313, 15, 268, 369, 174, 184, 38, 140, 3, 175, 258, 45, 218, 16, 228, 28, 115, 12, 312, 253, 19, 56, 366, 382, 300, 241, 150, 284, 90, 383, 237, 9, 13, 291, 162, 232, 376, 363, 283, 103, 129, 270, 244, 156, 316, 297, 373, 86, 290, 53, 120, 309, 97, 282, 36, 202, 123, 109, 226, 248, 110, 83, 176, 326, 57, 50, 170, 177, 354, 61, 61, 213, 128, 128, 84, 214, 194, 98, 81, 355, 346, 276, 100, 370, 169, 269, 356, 288, 272, 231, 211, 259, 379, 255, 132, 212, 88, 331, 192, 338, 182, 327, 304, 131, 70, 151, 375, 332, 238, 181, 319, 251, 207, 324, 320, 333, 220, 33, 104, 46, 79, 266, 216, 329, 183, 230, 106, 239, 250, 341, 219, 76, 136, 281, 334, 48, 360, 134, 29, 271, 55, 205, 374, 299, 263, 107, 342, 187, 92, 224, 189, 347, 101, 133, 126, 267, 307, 311, 146, 340, 294, 117, 310, 339, 280, 166, 225, 167, 277, 286, 196, 303, 75, 61, 165, 141, 249, 256, 84, 214, 160, 260, 1, 178, 346, 223, 105, 193, 269, 30, 287, 52, 6, 372, 122, 102, 64, 215, 145, 352, 200, 89, 247, 335, 22, 124, 173, 4, 155, 37, 222, 358, 264, 69, 67, 318, 24, 210, 34, 153, 315, 54, 295, 99, 144, 51, 242, 345, 227, 14, 343, 362, 240, 63, 138, 203, 325, 254, 119, 149, 236, 344, 77, 191, 349, 359, 113, 323, 378, 199, 68, 168, 58, 148, 322, 32, 245, 185, 139, 285, 73, 243, 350, 357, 42, 180, 10, 116, 112, 348, 164, 82, 157, 337, 118, 127, 44] + 6: [322, 76, 97, 212, 142, 274, 5, 376, 32, 32, 249, 267, 269, 55, 281, 99, 66, 166, 177, 31, 175, 378, 260, 216, 94, 119, 57, 283, 118, 208, 155, 184, 370, 151, 301, 176, 128, 377, 380, 236, 49, 68, 12, 158, 133, 139, 114, 219, 15, 10, 286, 280, 6, 87, 207, 137, 116, 63, 243, 217, 194, 261, 122, 248, 199, 105, 193, 224, 8, 16, 381, 309, 364, 123, 312, 74, 150, 288, 328, 290, 117, 162, 338, 21, 121, 222, 339, 238, 29, 192, 127, 241, 340, 170, 218, 72, 65, 75, 214, 186, 315, 331, 247, 257, 322, 76, 97, 142, 142, 295, 308, 376, 160, 53, 27, 174, 28, 253, 281, 46, 209, 233, 326, 371, 9, 36, 307, 13, 88, 367, 189, 112, 319, 34, 178, 289, 156, 0, 270, 113, 179, 115, 41, 230, 231, 101, 279, 190, 169, 221, 77, 327, 332, 136, 296, 365, 262, 69, 43, 237, 159, 284, 82, 86, 254, 372, 329, 81, 168, 346, 3, 50, 334, 42, 70, 125, 379, 292, 244, 130, 152, 108, 226, 337, 256, 294, 348, 47, 183, 131, 250, 374, 92, 213, 141, 1, 172, 191, 20, 353, 153, 132, 363, 204, 35, 325, 342, 373, 322, 297, 305, 97, 142, 274, 308, 376, 103, 53, 343, 258, 341, 253, 240, 11, 180, 233, 205, 67, 173, 45, 163, 318, 232, 293, 368, 84, 255, 287, 291, 129, 259, 345, 277, 165, 185, 352, 245, 278, 311, 304, 37, 111, 321, 44, 64, 107, 264, 98, 220, 157, 271, 22, 206, 242, 225, 197, 23, 4, 362, 138, 316, 148, 48, 333, 198, 358, 298, 96, 268, 272, 134, 366, 285, 357, 106, 203, 347, 85, 17, 102, 303, 51, 313, 356, 302, 78, 26, 171, 19, 227, 344, 200, 61, 2, 314, 135, 167, 52, 275, 323, 161, 164, 146, 76, 305, 97, 59, 274, 308, 308, 32, 343, 306, 174, 258, 240, 229, 99, 66, 166, 177, 120, 350, 375, 359, 181, 187, 100, 56, 246, 188, 196, 79, 239, 300, 234, 95, 354, 109, 369, 182, 336, 317, 223, 30, 202, 282, 383, 144, 211, 143, 324, 154, 58, 210, 361, 104, 110, 276, 335, 330, 60, 355, 14, 145, 126, 18, 83, 124, 147, 349, 215, 299, 54, 73, 310, 39, 235, 80, 62, 251, 360, 266, 24, 195, 351, 89, 201, 263, 38, 7, 140, 91, 93, 273, 382, 228, 265, 252, 149, 25, 33, 320, 71, 40, 90] + 7: [163, 162, 182, 7, 258, 275, 192, 263, 133, 49, 292, 375, 229, 46, 382, 382, 105, 103, 201, 253, 362, 118, 48, 329, 0, 61, 94, 131, 183, 47, 215, 267, 278, 4, 168, 55, 270, 148, 250, 235, 117, 21, 221, 62, 90, 344, 333, 214, 24, 368, 314, 109, 358, 19, 32, 190, 211, 83, 335, 305, 67, 53, 353, 232, 251, 373, 65, 291, 87, 101, 52, 120, 37, 126, 268, 209, 363, 112, 69, 218, 85, 254, 111, 39, 252, 381, 78, 159, 195, 95, 45, 57, 352, 331, 143, 282, 34, 376, 236, 179, 135, 371, 338, 42, 341, 175, 272, 7, 274, 336, 263, 64, 41, 49, 292, 28, 142, 229, 343, 340, 105, 18, 25, 25, 298, 378, 354, 188, 244, 27, 181, 281, 11, 313, 164, 82, 248, 262, 17, 31, 288, 360, 296, 284, 70, 210, 170, 213, 158, 191, 243, 167, 234, 89, 26, 204, 136, 237, 180, 107, 322, 273, 58, 6, 51, 121, 56, 155, 161, 40, 309, 68, 149, 153, 225, 351, 184, 222, 151, 106, 286, 169, 194, 1, 10, 96, 367, 43, 317, 303, 228, 315, 242, 299, 332, 150, 205, 165, 138, 144, 76, 156, 279, 160, 91, 348, 357, 50, 327, 327, 182, 274, 274, 192, 108, 64, 41, 350, 292, 187, 142, 265, 342, 340, 18, 103, 13, 99, 298, 369, 97, 212, 77, 370, 256, 73, 14, 249, 113, 217, 289, 29, 349, 23, 84, 2, 325, 356, 81, 255, 219, 377, 372, 300, 241, 132, 259, 189, 16, 245, 92, 20, 186, 178, 137, 293, 380, 86, 152, 226, 310, 323, 233, 196, 247, 261, 22, 38, 345, 366, 115, 269, 320, 166, 287, 207, 316, 72, 59, 123, 80, 231, 3, 277, 334, 266, 246, 359, 346, 147, 172, 12, 347, 199, 355, 154, 326, 240, 276, 110, 257, 364, 327, 175, 157, 7, 238, 275, 308, 361, 239, 49, 260, 176, 375, 265, 343, 271, 280, 139, 201, 71, 362, 369, 15, 328, 104, 174, 114, 302, 324, 294, 307, 100, 125, 74, 119, 285, 337, 339, 319, 374, 383, 208, 177, 224, 330, 264, 98, 206, 9, 321, 311, 140, 185, 130, 129, 5, 220, 200, 102, 44, 141, 134, 306, 301, 283, 8, 227, 146, 116, 93, 193, 203, 79, 318, 35, 216, 33, 75, 36, 295, 88, 312, 173, 379, 297, 124, 63, 128, 198, 304, 230, 66, 290, 197, 127, 54, 122, 223, 145, 30, 365, 202, 171, 60] + 8: [273, 5, 360, 290, 218, 31, 135, 302, 359, 7, 264, 13, 12, 162, 321, 243, 221, 308, 353, 64, 229, 229, 182, 281, 291, 223, 157, 349, 11, 377, 131, 195, 128, 107, 75, 150, 27, 333, 38, 240, 92, 78, 65, 275, 304, 189, 167, 194, 130, 267, 120, 177, 341, 52, 4, 42, 101, 335, 383, 174, 237, 248, 247, 90, 297, 352, 47, 299, 190, 126, 16, 56, 99, 265, 235, 205, 306, 175, 322, 46, 315, 33, 326, 220, 324, 103, 370, 208, 26, 180, 70, 286, 355, 80, 342, 82, 222, 289, 361, 127, 227, 1, 132, 207, 253, 104, 279, 360, 290, 292, 135, 21, 359, 367, 264, 239, 362, 162, 336, 10, 272, 381, 353, 152, 228, 204, 74, 281, 20, 186, 316, 18, 294, 97, 86, 39, 311, 366, 22, 188, 109, 226, 147, 179, 202, 169, 216, 178, 303, 112, 198, 50, 317, 334, 141, 211, 125, 66, 72, 8, 45, 206, 117, 230, 379, 25, 257, 249, 350, 110, 310, 382, 266, 155, 313, 140, 105, 325, 378, 145, 236, 375, 369, 54, 323, 153, 111, 24, 15, 171, 129, 270, 319, 245, 354, 305, 43, 138, 94, 344, 256, 49, 0, 55, 19, 63, 213, 234, 253, 214, 3, 360, 290, 67, 31, 338, 359, 161, 108, 201, 362, 173, 241, 62, 62, 308, 282, 152, 337, 204, 81, 276, 30, 251, 380, 44, 364, 168, 280, 374, 225, 346, 69, 83, 133, 283, 119, 231, 298, 261, 165, 40, 219, 368, 183, 113, 300, 48, 192, 318, 2, 91, 60, 61, 59, 263, 212, 98, 28, 73, 191, 71, 151, 373, 32, 37, 307, 327, 314, 199, 144, 339, 35, 89, 260, 14, 29, 185, 134, 357, 351, 146, 84, 271, 158, 340, 329, 285, 123, 343, 51, 203, 139, 274, 124, 345, 242, 296, 262, 102, 58, 170, 253, 5, 250, 360, 246, 67, 238, 106, 21, 367, 224, 239, 201, 312, 321, 10, 221, 88, 154, 371, 371, 36, 182, 122, 252, 95, 255, 209, 328, 115, 156, 148, 166, 181, 172, 356, 23, 34, 6, 258, 330, 17, 121, 215, 278, 137, 309, 196, 320, 164, 233, 197, 332, 358, 114, 284, 187, 217, 254, 118, 87, 116, 232, 293, 184, 149, 259, 41, 136, 372, 68, 143, 79, 200, 85, 159, 295, 9, 93, 347, 365, 287, 277, 76, 193, 348, 244, 77, 269, 268, 160, 363, 142, 53, 331, 301, 288, 57, 210, 176, 163, 376, 96, 100] + 9: [212, 304, 166, 349, 81, 96, 226, 260, 263, 19, 350, 219, 155, 355, 184, 185, 37, 1, 313, 376, 127, 242, 380, 281, 244, 95, 341, 209, 136, 282, 32, 102, 98, 312, 82, 158, 363, 356, 120, 289, 264, 334, 231, 6, 364, 87, 116, 145, 186, 323, 207, 206, 124, 66, 189, 72, 106, 104, 343, 308, 208, 246, 50, 65, 52, 60, 275, 373, 174, 322, 254, 114, 371, 301, 251, 305, 224, 333, 211, 221, 347, 69, 188, 337, 290, 191, 368, 225, 345, 118, 328, 101, 93, 44, 194, 294, 91, 250, 61, 27, 236, 168, 338, 26, 92, 92, 255, 255, 143, 220, 280, 94, 94, 252, 357, 165, 327, 355, 179, 185, 198, 15, 201, 316, 138, 262, 49, 210, 64, 271, 103, 147, 10, 23, 239, 22, 258, 151, 285, 144, 187, 302, 137, 33, 277, 318, 180, 247, 192, 107, 41, 163, 35, 195, 267, 336, 36, 150, 111, 218, 274, 5, 75, 324, 17, 235, 8, 130, 303, 245, 148, 197, 62, 213, 112, 2, 320, 286, 375, 134, 76, 216, 21, 232, 199, 227, 167, 170, 193, 160, 58, 108, 240, 266, 249, 73, 296, 204, 200, 113, 28, 157, 25, 182, 369, 283, 348, 234, 92, 78, 304, 349, 349, 196, 31, 226, 215, 299, 353, 74, 222, 327, 47, 351, 29, 83, 146, 316, 217, 262, 49, 287, 244, 95, 51, 147, 10, 14, 56, 309, 9, 133, 161, 360, 135, 154, 3, 292, 297, 306, 54, 321, 314, 311, 259, 129, 269, 105, 4, 123, 230, 181, 238, 141, 237, 359, 80, 13, 276, 248, 354, 307, 379, 117, 205, 233, 300, 79, 203, 115, 42, 110, 12, 330, 298, 270, 162, 229, 346, 335, 241, 374, 293, 383, 243, 43, 325, 279, 361, 223, 149, 284, 273, 40, 362, 7, 122, 53, 77, 24, 365, 57, 92, 304, 304, 255, 331, 315, 31, 372, 263, 172, 295, 295, 327, 355, 47, 381, 344, 272, 313, 90, 90, 370, 159, 329, 64, 214, 103, 121, 67, 132, 177, 342, 131, 367, 317, 30, 377, 153, 268, 169, 88, 228, 0, 128, 183, 97, 20, 99, 142, 256, 139, 16, 70, 11, 171, 332, 288, 202, 39, 319, 339, 18, 175, 109, 100, 382, 45, 46, 190, 261, 358, 291, 84, 253, 71, 352, 310, 125, 38, 89, 48, 178, 59, 86, 34, 326, 68, 140, 176, 366, 63, 119, 278, 85, 378, 257, 164, 156, 265, 55, 152, 126, 340, 173] + 10: [294, 55, 300, 180, 151, 265, 319, 108, 5, 141, 320, 289, 365, 18, 286, 167, 31, 146, 2, 79, 310, 310, 134, 54, 263, 114, 66, 182, 237, 154, 201, 380, 7, 293, 11, 259, 345, 284, 49, 316, 98, 189, 249, 218, 61, 32, 128, 229, 233, 87, 321, 21, 232, 219, 60, 142, 9, 256, 26, 346, 379, 25, 267, 258, 36, 247, 64, 193, 275, 323, 95, 183, 185, 282, 226, 364, 96, 287, 139, 222, 372, 86, 119, 17, 168, 165, 251, 53, 88, 104, 281, 333, 191, 84, 156, 10, 242, 228, 381, 70, 23, 324, 366, 369, 192, 264, 300, 180, 93, 265, 319, 319, 5, 320, 24, 121, 365, 216, 337, 126, 331, 146, 225, 73, 215, 301, 195, 178, 171, 153, 19, 177, 196, 211, 269, 213, 67, 243, 272, 136, 368, 274, 342, 204, 360, 38, 306, 223, 149, 132, 199, 198, 116, 214, 135, 105, 288, 311, 106, 314, 329, 112, 378, 328, 74, 97, 383, 278, 42, 12, 354, 332, 45, 239, 123, 13, 370, 244, 307, 169, 131, 159, 302, 235, 187, 355, 205, 224, 4, 158, 133, 203, 118, 273, 157, 344, 361, 150, 63, 277, 28, 43, 238, 166, 176, 349, 68, 336, 255, 264, 55, 246, 268, 137, 319, 221, 5, 320, 320, 121, 325, 216, 279, 367, 309, 164, 92, 102, 22, 301, 170, 179, 78, 351, 241, 210, 99, 47, 248, 58, 115, 312, 234, 240, 348, 356, 377, 148, 77, 212, 376, 359, 382, 155, 250, 90, 103, 129, 107, 3, 56, 81, 85, 27, 315, 357, 59, 327, 41, 51, 371, 109, 340, 318, 184, 254, 317, 50, 362, 261, 298, 113, 257, 209, 326, 375, 80, 111, 127, 305, 147, 217, 322, 6, 16, 34, 94, 120, 44, 347, 30, 143, 202, 260, 270, 89, 160, 253, 291, 295, 29, 283, 194, 194, 300, 300, 151, 181, 319, 48, 5, 5, 24, 262, 365, 18, 286, 69, 31, 164, 57, 101, 144, 8, 1, 161, 117, 40, 138, 330, 124, 304, 308, 91, 76, 353, 130, 252, 363, 374, 186, 208, 190, 231, 271, 280, 285, 266, 145, 0, 33, 72, 46, 341, 296, 75, 122, 335, 339, 197, 20, 338, 37, 230, 15, 71, 162, 125, 39, 174, 140, 313, 163, 343, 175, 220, 65, 299, 350, 100, 62, 173, 206, 373, 303, 292, 35, 200, 245, 276, 334, 172, 83, 207, 352, 152, 227, 14, 52, 188, 110, 236, 82, 290, 358, 297] + 11: [156, 339, 299, 130, 87, 314, 83, 296, 163, 40, 150, 326, 221, 171, 145, 11, 116, 157, 100, 357, 199, 29, 332, 144, 275, 54, 178, 52, 73, 204, 282, 177, 17, 207, 250, 313, 161, 346, 124, 45, 249, 127, 159, 27, 195, 26, 251, 10, 198, 89, 298, 365, 186, 101, 242, 97, 147, 63, 59, 292, 353, 364, 193, 311, 187, 234, 264, 64, 43, 113, 62, 182, 143, 8, 19, 382, 200, 37, 323, 222, 288, 4, 140, 290, 278, 304, 183, 379, 60, 175, 246, 181, 111, 189, 276, 155, 329, 220, 321, 50, 308, 134, 243, 241, 252, 377, 130, 130, 376, 90, 3, 215, 163, 85, 85, 110, 235, 221, 190, 11, 116, 248, 248, 285, 285, 15, 218, 22, 247, 217, 141, 165, 131, 70, 337, 228, 205, 136, 78, 303, 6, 283, 371, 107, 370, 2, 75, 226, 315, 166, 374, 160, 133, 49, 197, 289, 259, 254, 270, 301, 295, 219, 168, 366, 354, 378, 158, 91, 324, 291, 115, 383, 48, 142, 263, 7, 74, 230, 358, 274, 103, 375, 84, 356, 173, 185, 322, 167, 380, 305, 277, 1, 238, 344, 352, 317, 372, 202, 82, 123, 153, 223, 25, 256, 109, 32, 42, 269, 98, 77, 260, 88, 87, 314, 83, 296, 296, 65, 150, 110, 235, 99, 121, 300, 44, 44, 100, 357, 267, 15, 0, 345, 152, 351, 284, 128, 39, 286, 318, 146, 154, 169, 23, 320, 47, 319, 170, 367, 262, 69, 117, 373, 172, 347, 151, 340, 316, 203, 255, 96, 31, 41, 81, 35, 336, 294, 245, 265, 312, 34, 135, 349, 118, 67, 293, 79, 258, 33, 225, 18, 138, 46, 194, 184, 360, 108, 306, 112, 310, 201, 287, 214, 68, 71, 342, 229, 368, 361, 369, 16, 51, 28, 239, 206, 334, 281, 72, 273, 191, 261, 9, 12, 98, 377, 339, 363, 227, 90, 314, 215, 163, 149, 119, 110, 38, 121, 233, 36, 36, 325, 126, 126, 102, 162, 24, 236, 331, 192, 343, 330, 244, 95, 381, 20, 13, 148, 196, 297, 94, 66, 280, 213, 348, 350, 80, 92, 93, 176, 56, 86, 341, 224, 14, 55, 208, 58, 209, 327, 362, 211, 30, 279, 338, 359, 179, 302, 164, 125, 120, 57, 129, 174, 355, 188, 309, 237, 231, 271, 266, 5, 106, 210, 21, 240, 104, 132, 272, 268, 114, 328, 180, 216, 139, 53, 253, 333, 122, 76, 212, 232, 137, 257, 61, 307, 105, 335] + 12: [156, 204, 149, 35, 120, 78, 158, 226, 327, 190, 303, 267, 292, 223, 289, 378, 150, 172, 333, 46, 290, 192, 361, 369, 340, 354, 295, 339, 281, 216, 215, 317, 25, 87, 108, 133, 153, 2, 265, 264, 57, 271, 135, 294, 80, 380, 74, 91, 144, 309, 201, 227, 188, 52, 185, 41, 45, 61, 224, 262, 280, 356, 37, 169, 11, 38, 115, 36, 305, 334, 183, 336, 244, 246, 312, 374, 352, 285, 252, 251, 34, 316, 55, 131, 299, 237, 230, 258, 143, 307, 269, 118, 221, 202, 342, 311, 359, 324, 66, 239, 373, 162, 110, 372, 255, 255, 363, 48, 381, 130, 158, 226, 344, 190, 127, 168, 124, 82, 177, 69, 243, 126, 333, 203, 341, 50, 51, 198, 170, 259, 134, 195, 382, 332, 194, 152, 218, 323, 89, 10, 27, 160, 102, 300, 104, 273, 63, 129, 326, 31, 59, 14, 286, 304, 193, 142, 77, 9, 191, 114, 165, 189, 67, 256, 161, 232, 173, 321, 214, 235, 121, 270, 20, 379, 32, 86, 365, 297, 207, 279, 274, 174, 146, 105, 47, 8, 343, 347, 212, 211, 320, 19, 196, 111, 26, 145, 360, 328, 206, 88, 92, 21, 331, 99, 43, 366, 272, 302, 255, 255, 49, 149, 120, 245, 158, 28, 17, 327, 127, 109, 22, 82, 177, 378, 243, 172, 287, 46, 341, 233, 181, 306, 260, 176, 75, 278, 68, 205, 276, 330, 147, 368, 60, 364, 288, 33, 101, 171, 322, 180, 197, 308, 261, 238, 23, 318, 15, 283, 253, 376, 200, 1, 315, 0, 83, 84, 298, 313, 4, 139, 187, 319, 76, 122, 40, 159, 13, 257, 293, 5, 179, 284, 128, 263, 85, 106, 70, 44, 282, 345, 96, 18, 54, 72, 236, 367, 182, 7, 184, 132, 242, 314, 100, 248, 377, 64, 141, 353, 53, 250, 225, 220, 255, 204, 204, 120, 130, 130, 78, 219, 327, 167, 303, 175, 223, 349, 296, 69, 150, 71, 103, 203, 97, 90, 151, 98, 208, 123, 3, 6, 137, 346, 348, 268, 229, 266, 16, 277, 39, 209, 116, 371, 362, 107, 164, 241, 291, 337, 157, 301, 155, 358, 357, 370, 350, 254, 231, 234, 29, 119, 62, 138, 136, 249, 325, 58, 335, 355, 154, 351, 65, 140, 12, 112, 186, 329, 148, 217, 24, 383, 310, 30, 375, 93, 178, 113, 81, 117, 125, 247, 79, 210, 94, 338, 163, 228, 213, 275, 166, 95, 56, 240, 222, 199, 73, 42] + 13: [80, 362, 77, 170, 108, 232, 18, 114, 14, 195, 87, 23, 265, 169, 282, 280, 55, 95, 238, 30, 30, 318, 134, 310, 199, 335, 171, 159, 118, 131, 227, 49, 75, 69, 297, 246, 309, 91, 266, 363, 208, 373, 22, 65, 10, 219, 339, 382, 144, 291, 107, 283, 359, 177, 256, 139, 52, 241, 37, 8, 163, 337, 286, 153, 319, 242, 259, 296, 381, 50, 133, 187, 360, 358, 19, 73, 235, 150, 104, 214, 182, 285, 85, 346, 336, 284, 63, 352, 152, 99, 224, 97, 349, 290, 47, 200, 82, 160, 193, 314, 126, 350, 192, 204, 80, 194, 137, 170, 108, 232, 18, 96, 14, 87, 149, 328, 260, 312, 172, 135, 231, 189, 238, 12, 274, 318, 102, 186, 88, 370, 46, 109, 35, 295, 111, 64, 364, 188, 275, 273, 261, 294, 158, 334, 2, 6, 98, 207, 375, 355, 43, 62, 25, 239, 196, 361, 374, 333, 183, 243, 380, 222, 303, 331, 89, 288, 379, 326, 220, 56, 305, 59, 245, 348, 71, 311, 70, 72, 217, 213, 124, 287, 353, 249, 205, 83, 167, 226, 343, 60, 203, 191, 258, 173, 347, 293, 132, 269, 354, 302, 351, 218, 247, 324, 33, 315, 230, 7, 9, 362, 329, 170, 108, 232, 18, 114, 86, 195, 87, 23, 39, 24, 41, 280, 276, 342, 110, 209, 254, 330, 271, 366, 11, 162, 298, 116, 307, 279, 58, 198, 16, 112, 356, 376, 40, 143, 67, 292, 299, 300, 42, 210, 211, 179, 240, 141, 228, 26, 225, 74, 93, 253, 129, 146, 15, 76, 125, 34, 48, 36, 68, 357, 345, 174, 92, 127, 325, 113, 250, 53, 154, 115, 13, 377, 5, 130, 61, 119, 138, 185, 157, 383, 1, 101, 215, 128, 278, 181, 371, 317, 236, 272, 368, 255, 322, 120, 66, 206, 27, 140, 316, 175, 44, 44, 329, 170, 170, 57, 81, 3, 0, 195, 149, 328, 260, 312, 51, 21, 201, 189, 100, 12, 274, 340, 197, 17, 142, 338, 306, 304, 323, 176, 45, 184, 369, 31, 289, 202, 54, 229, 281, 320, 178, 122, 378, 147, 168, 79, 237, 321, 344, 145, 262, 121, 166, 151, 327, 29, 164, 221, 223, 105, 267, 161, 248, 106, 136, 84, 257, 268, 263, 244, 308, 94, 123, 270, 38, 234, 313, 365, 103, 190, 233, 4, 372, 180, 252, 117, 28, 212, 90, 264, 251, 20, 332, 341, 156, 148, 367, 277, 165, 301, 216, 155, 78, 32] + 14: [147, 229, 211, 274, 372, 76, 76, 345, 137, 255, 110, 175, 2, 209, 75, 34, 150, 355, 337, 146, 235, 142, 99, 254, 198, 117, 3, 371, 162, 374, 271, 111, 41, 265, 13, 197, 297, 149, 215, 343, 291, 319, 369, 277, 301, 266, 133, 382, 349, 203, 178, 226, 222, 241, 380, 237, 298, 284, 0, 316, 300, 152, 242, 148, 224, 244, 248, 96, 285, 114, 262, 321, 278, 257, 338, 267, 238, 361, 19, 16, 263, 87, 18, 31, 366, 367, 84, 139, 11, 107, 320, 307, 51, 72, 368, 303, 38, 210, 119, 272, 83, 249, 289, 50, 70, 160, 229, 143, 261, 15, 164, 204, 8, 255, 110, 30, 2, 209, 344, 34, 327, 126, 118, 378, 155, 71, 275, 85, 52, 124, 342, 213, 317, 269, 182, 250, 216, 166, 125, 314, 135, 379, 328, 123, 288, 324, 193, 370, 363, 104, 346, 40, 325, 286, 296, 373, 5, 101, 347, 129, 341, 306, 225, 336, 115, 168, 313, 364, 356, 63, 43, 251, 106, 10, 214, 329, 358, 309, 36, 46, 20, 304, 14, 58, 299, 179, 45, 194, 293, 55, 92, 180, 81, 233, 381, 65, 188, 134, 243, 340, 218, 232, 138, 67, 318, 295, 331, 88, 70, 160, 229, 143, 4, 207, 287, 345, 345, 255, 199, 112, 49, 259, 75, 186, 327, 355, 353, 308, 155, 71, 23, 141, 95, 32, 97, 64, 61, 360, 21, 44, 157, 270, 220, 158, 59, 206, 323, 69, 184, 332, 201, 159, 108, 167, 339, 187, 102, 256, 109, 227, 82, 116, 377, 221, 330, 359, 253, 200, 202, 223, 264, 282, 350, 1, 93, 42, 311, 94, 6, 273, 375, 171, 365, 74, 78, 9, 290, 190, 312, 258, 80, 7, 66, 57, 185, 120, 140, 236, 144, 252, 113, 192, 54, 131, 352, 292, 196, 98, 315, 121, 205, 165, 56, 160, 229, 211, 4, 15, 77, 204, 60, 17, 383, 30, 268, 259, 239, 122, 122, 126, 348, 378, 308, 275, 228, 151, 100, 231, 177, 39, 154, 79, 279, 354, 294, 26, 208, 33, 161, 25, 246, 163, 357, 68, 326, 145, 37, 22, 333, 169, 128, 172, 245, 234, 48, 90, 156, 105, 176, 276, 86, 189, 335, 89, 191, 12, 183, 217, 240, 376, 28, 310, 173, 302, 153, 103, 195, 127, 136, 334, 305, 230, 322, 281, 132, 47, 362, 62, 35, 24, 181, 247, 219, 283, 170, 91, 27, 280, 29, 53, 351, 130, 73, 212, 260, 174] + 15: [68, 15, 350, 88, 88, 367, 130, 11, 359, 57, 39, 160, 38, 355, 355, 200, 128, 159, 148, 134, 134, 203, 146, 9, 80, 329, 139, 56, 296, 193, 256, 1, 242, 255, 196, 115, 162, 36, 207, 319, 182, 372, 238, 246, 105, 212, 2, 82, 309, 114, 229, 22, 205, 233, 341, 253, 294, 382, 107, 226, 347, 61, 354, 99, 208, 216, 383, 71, 283, 95, 334, 265, 273, 190, 3, 147, 274, 303, 184, 158, 14, 126, 312, 310, 7, 171, 361, 228, 178, 41, 120, 346, 288, 287, 214, 336, 249, 315, 152, 6, 131, 366, 281, 276, 298, 15, 340, 4, 259, 295, 11, 291, 371, 362, 160, 251, 189, 155, 332, 258, 94, 217, 175, 104, 252, 119, 121, 237, 232, 360, 23, 48, 353, 344, 311, 174, 177, 143, 18, 235, 31, 285, 357, 47, 78, 323, 28, 42, 102, 279, 86, 248, 87, 10, 150, 96, 363, 8, 206, 13, 271, 339, 349, 306, 53, 72, 308, 289, 55, 204, 169, 64, 26, 337, 316, 257, 240, 54, 278, 364, 73, 197, 153, 129, 370, 201, 293, 280, 327, 292, 66, 195, 154, 183, 307, 110, 108, 125, 156, 297, 186, 19, 46, 144, 322, 192, 93, 270, 0, 0, 261, 76, 227, 130, 130, 11, 359, 57, 199, 251, 324, 155, 231, 326, 94, 217, 148, 236, 74, 284, 60, 20, 221, 151, 376, 83, 113, 277, 342, 135, 241, 111, 109, 164, 379, 106, 351, 335, 133, 5, 49, 247, 101, 166, 117, 142, 343, 77, 331, 282, 69, 81, 254, 165, 377, 141, 191, 67, 180, 97, 365, 234, 314, 224, 123, 37, 92, 172, 84, 213, 167, 218, 239, 79, 91, 161, 89, 32, 262, 149, 187, 380, 124, 136, 330, 181, 44, 179, 304, 70, 286, 122, 219, 220, 132, 348, 25, 264, 325, 260, 356, 328, 0, 290, 15, 88, 227, 202, 130, 11, 359, 362, 160, 300, 324, 185, 200, 258, 266, 128, 175, 104, 333, 215, 90, 51, 317, 157, 16, 40, 35, 173, 168, 163, 138, 320, 230, 24, 352, 225, 338, 100, 170, 209, 305, 194, 145, 275, 59, 263, 269, 34, 268, 62, 50, 250, 12, 43, 301, 30, 313, 118, 321, 381, 267, 137, 244, 45, 272, 211, 373, 375, 358, 63, 116, 52, 65, 210, 378, 369, 374, 188, 29, 140, 176, 318, 58, 112, 222, 98, 299, 103, 223, 17, 243, 21, 302, 345, 27, 33, 245, 75, 198, 127, 368, 85] + 16: [11, 134, 134, 15, 286, 67, 125, 120, 300, 275, 262, 7, 338, 86, 49, 64, 44, 118, 342, 75, 271, 35, 302, 255, 283, 265, 213, 235, 270, 87, 226, 85, 294, 285, 177, 71, 59, 248, 316, 292, 119, 73, 155, 282, 31, 182, 34, 70, 220, 45, 122, 372, 174, 339, 4, 370, 198, 104, 375, 358, 158, 208, 214, 211, 333, 267, 272, 105, 48, 109, 266, 194, 381, 315, 157, 284, 47, 218, 131, 110, 216, 217, 50, 241, 138, 243, 352, 32, 355, 12, 112, 329, 136, 173, 291, 179, 296, 362, 13, 69, 94, 309, 344, 353, 162, 134, 134, 15, 286, 152, 330, 120, 120, 275, 92, 7, 277, 147, 147, 64, 303, 303, 280, 259, 271, 183, 78, 24, 209, 287, 141, 236, 367, 123, 346, 89, 65, 310, 52, 321, 115, 335, 38, 21, 80, 139, 56, 159, 231, 41, 219, 195, 232, 379, 202, 293, 137, 9, 322, 188, 317, 221, 354, 340, 132, 54, 102, 95, 206, 343, 319, 55, 190, 196, 193, 360, 186, 81, 88, 373, 145, 20, 17, 258, 371, 279, 176, 149, 245, 260, 101, 382, 79, 349, 40, 135, 350, 263, 72, 204, 325, 53, 191, 239, 126, 368, 156, 249, 100, 140, 15, 68, 369, 152, 331, 154, 234, 275, 312, 43, 165, 273, 269, 90, 44, 118, 203, 318, 5, 205, 224, 171, 383, 121, 187, 295, 146, 347, 180, 108, 27, 365, 306, 328, 307, 51, 377, 82, 103, 58, 66, 153, 380, 305, 8, 227, 357, 36, 42, 301, 160, 297, 96, 106, 225, 252, 151, 336, 200, 57, 313, 178, 223, 253, 93, 83, 244, 207, 201, 363, 62, 233, 127, 128, 324, 268, 113, 26, 170, 116, 359, 150, 61, 230, 288, 28, 63, 351, 74, 254, 299, 192, 326, 222, 376, 133, 345, 320, 247, 1, 98, 257, 100, 140, 15, 286, 286, 369, 185, 114, 234, 262, 92, 43, 165, 144, 269, 129, 143, 289, 280, 75, 348, 374, 274, 250, 298, 189, 364, 46, 246, 212, 16, 199, 181, 314, 84, 237, 2, 97, 99, 37, 175, 281, 0, 39, 311, 184, 76, 167, 356, 229, 334, 19, 276, 238, 22, 124, 290, 264, 29, 228, 25, 18, 60, 242, 337, 323, 304, 378, 91, 142, 30, 240, 172, 163, 14, 166, 107, 261, 168, 117, 308, 256, 77, 332, 210, 251, 33, 169, 6, 164, 23, 130, 341, 197, 327, 278, 361, 10, 161, 215, 111, 366, 3, 148] + 17: [374, 131, 220, 119, 208, 250, 284, 236, 137, 303, 337, 366, 108, 200, 375, 15, 136, 100, 359, 5, 330, 305, 343, 323, 1, 2, 189, 6, 370, 167, 222, 107, 96, 296, 335, 97, 381, 180, 380, 373, 10, 98, 33, 135, 292, 171, 245, 259, 340, 248, 47, 116, 59, 42, 269, 295, 49, 204, 377, 186, 341, 338, 342, 265, 94, 179, 262, 310, 90, 120, 161, 112, 267, 274, 369, 25, 192, 50, 150, 57, 163, 83, 153, 352, 149, 230, 356, 223, 349, 178, 242, 71, 247, 353, 268, 321, 252, 174, 54, 122, 159, 170, 202, 203, 261, 138, 220, 119, 80, 188, 284, 284, 190, 346, 79, 337, 12, 200, 91, 78, 117, 100, 26, 197, 330, 232, 185, 266, 130, 125, 347, 297, 276, 172, 290, 339, 39, 194, 218, 199, 145, 243, 38, 92, 371, 256, 99, 327, 214, 9, 142, 168, 264, 304, 19, 20, 173, 43, 41, 363, 216, 198, 365, 3, 309, 271, 95, 44, 379, 300, 93, 240, 258, 183, 345, 287, 151, 55, 158, 372, 76, 270, 31, 233, 32, 65, 315, 182, 378, 176, 109, 30, 229, 291, 288, 306, 334, 17, 51, 68, 156, 333, 14, 358, 312, 320, 325, 354, 261, 131, 131, 119, 105, 140, 188, 236, 319, 79, 79, 126, 108, 375, 254, 15, 311, 155, 26, 193, 305, 294, 362, 114, 113, 351, 29, 328, 75, 234, 56, 157, 134, 317, 46, 206, 148, 313, 211, 103, 143, 84, 272, 146, 282, 314, 162, 18, 332, 13, 166, 60, 45, 260, 215, 154, 118, 187, 221, 205, 101, 253, 21, 213, 360, 249, 195, 244, 299, 329, 16, 85, 64, 106, 278, 326, 364, 350, 63, 344, 53, 355, 316, 227, 23, 40, 277, 66, 257, 368, 11, 129, 77, 348, 111, 281, 209, 123, 34, 86, 69, 72, 121, 152, 28, 212, 131, 119, 307, 250, 284, 236, 226, 303, 337, 58, 87, 375, 219, 357, 383, 383, 225, 275, 286, 294, 362, 279, 246, 324, 133, 62, 175, 8, 52, 104, 207, 73, 22, 24, 231, 177, 228, 273, 367, 139, 298, 74, 165, 263, 70, 376, 191, 82, 61, 36, 331, 147, 7, 336, 81, 196, 115, 382, 37, 89, 181, 27, 110, 238, 302, 361, 102, 239, 210, 217, 128, 48, 308, 124, 67, 255, 318, 144, 164, 132, 241, 88, 280, 141, 201, 224, 4, 184, 160, 285, 289, 0, 35, 322, 301, 293, 169, 127, 235, 237, 251, 283] + 18: [208, 30, 30, 272, 43, 271, 193, 53, 251, 69, 320, 154, 300, 300, 65, 304, 71, 106, 185, 380, 174, 189, 355, 26, 139, 237, 291, 297, 35, 99, 77, 326, 240, 70, 19, 269, 337, 171, 342, 73, 307, 359, 87, 233, 316, 131, 0, 137, 278, 201, 313, 58, 157, 39, 195, 176, 205, 38, 244, 96, 132, 265, 255, 292, 94, 322, 166, 254, 48, 127, 358, 168, 332, 220, 4, 219, 56, 197, 317, 76, 298, 75, 107, 86, 37, 90, 299, 281, 24, 100, 11, 160, 21, 336, 295, 198, 379, 226, 74, 97, 222, 293, 1, 32, 208, 30, 47, 10, 271, 271, 271, 210, 251, 320, 217, 217, 113, 245, 57, 114, 71, 228, 368, 110, 192, 136, 25, 301, 373, 306, 357, 282, 378, 341, 33, 101, 103, 98, 164, 318, 20, 346, 191, 248, 104, 283, 149, 261, 88, 263, 162, 367, 122, 232, 250, 302, 289, 8, 140, 308, 279, 150, 319, 9, 12, 182, 13, 363, 287, 148, 324, 92, 108, 213, 383, 82, 170, 64, 27, 311, 163, 209, 123, 133, 218, 180, 372, 354, 85, 161, 29, 273, 377, 249, 117, 277, 72, 206, 194, 323, 55, 128, 374, 156, 242, 344, 147, 309, 6, 30, 30, 10, 43, 271, 271, 16, 251, 286, 315, 257, 28, 65, 57, 36, 138, 106, 185, 169, 239, 236, 146, 68, 34, 376, 151, 144, 330, 266, 190, 3, 124, 165, 274, 46, 371, 365, 141, 339, 177, 331, 343, 79, 334, 109, 135, 296, 159, 288, 361, 50, 61, 102, 111, 15, 125, 178, 276, 362, 145, 335, 267, 329, 270, 153, 325, 179, 275, 348, 234, 175, 216, 338, 51, 247, 126, 224, 369, 375, 364, 95, 258, 172, 44, 115, 352, 294, 345, 230, 18, 212, 199, 62, 134, 310, 49, 91, 327, 120, 227, 314, 59, 78, 118, 356, 30, 47, 271, 271, 45, 333, 211, 69, 207, 257, 347, 245, 215, 119, 138, 228, 260, 260, 116, 142, 167, 321, 360, 31, 284, 253, 246, 285, 105, 67, 188, 204, 89, 181, 41, 366, 202, 23, 121, 83, 262, 223, 225, 129, 241, 22, 7, 350, 231, 328, 235, 152, 203, 349, 54, 264, 81, 93, 63, 40, 155, 187, 351, 243, 214, 112, 303, 256, 60, 143, 200, 158, 66, 259, 130, 17, 229, 381, 2, 312, 42, 84, 370, 238, 184, 280, 14, 5, 305, 290, 173, 252, 221, 196, 353, 186, 52, 80, 183, 268, 382, 340] + 19: [187, 190, 366, 24, 339, 74, 7, 90, 57, 241, 120, 255, 338, 330, 322, 279, 192, 219, 101, 197, 64, 66, 184, 323, 20, 38, 303, 89, 159, 275, 189, 58, 365, 147, 238, 15, 13, 30, 222, 295, 6, 165, 167, 76, 353, 195, 257, 204, 2, 297, 4, 215, 95, 342, 336, 194, 85, 177, 23, 131, 60, 293, 280, 129, 225, 376, 362, 242, 298, 137, 209, 87, 186, 14, 208, 174, 28, 286, 371, 103, 112, 357, 267, 162, 185, 39, 224, 21, 155, 86, 56, 308, 226, 43, 199, 55, 142, 250, 378, 12, 34, 50, 81, 98, 187, 190, 366, 24, 111, 256, 302, 90, 57, 364, 91, 288, 288, 291, 126, 183, 318, 207, 312, 240, 64, 373, 113, 27, 154, 349, 227, 77, 230, 153, 217, 44, 41, 171, 246, 53, 285, 36, 352, 166, 359, 337, 270, 172, 29, 49, 170, 239, 40, 67, 345, 110, 244, 143, 287, 125, 284, 108, 380, 262, 160, 372, 92, 65, 140, 281, 329, 383, 341, 82, 139, 343, 370, 99, 105, 340, 149, 351, 73, 361, 348, 381, 210, 123, 178, 358, 367, 136, 344, 0, 5, 261, 169, 17, 72, 382, 196, 316, 229, 350, 319, 355, 369, 375, 187, 190, 366, 24, 179, 256, 7, 265, 90, 377, 120, 70, 288, 161, 313, 180, 42, 207, 101, 300, 216, 307, 115, 328, 193, 164, 334, 97, 152, 117, 79, 9, 158, 25, 335, 54, 71, 116, 128, 310, 228, 173, 96, 121, 251, 368, 138, 283, 157, 144, 259, 249, 374, 273, 84, 346, 19, 213, 45, 202, 75, 356, 3, 26, 191, 88, 200, 119, 130, 212, 22, 168, 232, 132, 234, 235, 63, 314, 37, 294, 150, 106, 11, 269, 201, 214, 252, 51, 205, 163, 236, 379, 282, 69, 220, 78, 145, 274, 93, 305, 107, 133, 198, 245, 141, 243, 211, 24, 206, 48, 7, 265, 61, 377, 52, 188, 338, 330, 313, 180, 42, 312, 8, 240, 114, 307, 115, 363, 218, 33, 296, 271, 301, 354, 35, 264, 292, 46, 135, 122, 233, 1, 278, 134, 127, 182, 68, 83, 156, 16, 272, 221, 260, 347, 277, 31, 306, 203, 325, 18, 333, 237, 10, 80, 331, 148, 47, 62, 309, 146, 326, 321, 176, 311, 32, 320, 258, 181, 332, 247, 94, 324, 59, 315, 231, 118, 124, 289, 290, 223, 304, 100, 299, 266, 248, 102, 151, 360, 327, 104, 175, 276, 109, 317, 253, 254, 263, 268] + 20: [116, 334, 37, 302, 272, 161, 317, 144, 98, 98, 110, 153, 376, 43, 112, 268, 339, 335, 130, 353, 363, 188, 318, 93, 158, 111, 141, 125, 382, 187, 259, 227, 342, 320, 172, 71, 77, 315, 190, 176, 284, 218, 35, 361, 118, 0, 146, 367, 150, 208, 372, 114, 196, 60, 86, 345, 370, 20, 310, 252, 236, 304, 170, 201, 326, 24, 212, 230, 159, 217, 210, 11, 156, 79, 250, 88, 321, 102, 163, 271, 67, 211, 276, 313, 134, 373, 183, 70, 149, 23, 222, 344, 255, 44, 143, 288, 242, 280, 266, 281, 299, 303, 331, 350, 116, 334, 278, 174, 14, 161, 82, 66, 66, 193, 153, 287, 2, 369, 226, 145, 4, 307, 282, 282, 32, 185, 162, 325, 164, 324, 290, 95, 381, 9, 50, 160, 292, 132, 92, 65, 126, 322, 74, 177, 240, 235, 328, 375, 219, 36, 314, 359, 309, 279, 18, 22, 238, 215, 340, 169, 294, 15, 338, 33, 127, 73, 365, 231, 360, 191, 84, 253, 308, 56, 80, 349, 59, 199, 21, 296, 40, 7, 329, 323, 354, 305, 19, 157, 64, 261, 68, 245, 220, 129, 301, 152, 237, 1, 69, 197, 198, 260, 107, 137, 167, 168, 179, 181, 116, 334, 278, 302, 336, 161, 161, 144, 81, 193, 153, 99, 87, 369, 45, 145, 4, 335, 46, 195, 204, 270, 138, 371, 209, 228, 273, 128, 311, 362, 140, 286, 216, 101, 347, 121, 182, 357, 6, 13, 194, 348, 378, 41, 175, 53, 30, 58, 233, 300, 42, 256, 274, 243, 346, 246, 248, 377, 12, 100, 78, 332, 39, 3, 312, 374, 223, 75, 94, 154, 139, 366, 142, 289, 120, 319, 113, 85, 333, 341, 47, 136, 83, 207, 133, 293, 108, 295, 251, 186, 306, 155, 244, 38, 109, 269, 241, 277, 189, 200, 206, 214, 262, 263, 116, 8, 122, 336, 161, 161, 55, 82, 173, 98, 123, 99, 87, 43, 45, 268, 379, 291, 46, 224, 105, 62, 330, 148, 10, 91, 229, 221, 96, 264, 180, 135, 258, 49, 184, 358, 104, 90, 239, 57, 364, 267, 343, 26, 265, 25, 368, 213, 247, 178, 119, 34, 131, 166, 165, 97, 297, 29, 5, 232, 106, 380, 257, 202, 327, 89, 337, 351, 254, 316, 285, 151, 352, 115, 283, 124, 28, 356, 205, 355, 72, 225, 16, 63, 48, 234, 383, 147, 192, 103, 275, 31, 203, 298, 171, 17, 117, 249, 27, 51, 52, 54, 61, 76] + 21: [152, 316, 308, 355, 190, 256, 47, 173, 281, 360, 360, 132, 50, 103, 124, 124, 92, 257, 185, 364, 363, 199, 120, 226, 178, 91, 177, 238, 191, 151, 351, 19, 121, 219, 25, 69, 9, 209, 221, 189, 234, 373, 89, 229, 168, 63, 1, 358, 39, 90, 309, 318, 251, 21, 356, 28, 354, 290, 323, 3, 237, 353, 115, 193, 34, 160, 104, 203, 329, 186, 335, 300, 379, 333, 305, 110, 195, 163, 310, 241, 143, 239, 41, 130, 129, 85, 375, 54, 162, 296, 285, 145, 298, 59, 188, 282, 343, 272, 297, 11, 42, 369, 339, 350, 152, 20, 371, 153, 254, 4, 47, 247, 281, 360, 29, 132, 80, 103, 200, 376, 119, 218, 113, 140, 88, 361, 277, 345, 252, 284, 250, 320, 22, 307, 147, 349, 15, 299, 293, 225, 43, 313, 141, 220, 368, 183, 243, 174, 73, 338, 18, 331, 287, 138, 60, 304, 97, 66, 383, 192, 370, 125, 347, 30, 295, 40, 100, 301, 136, 187, 79, 269, 57, 258, 126, 36, 215, 81, 83, 271, 319, 181, 342, 253, 156, 268, 74, 374, 217, 99, 37, 180, 176, 328, 6, 106, 326, 61, 212, 95, 235, 101, 127, 311, 44, 10, 17, 35, 152, 20, 371, 355, 355, 4, 46, 173, 206, 205, 86, 132, 80, 103, 210, 119, 71, 112, 87, 208, 169, 117, 214, 45, 118, 182, 149, 51, 380, 172, 26, 109, 196, 8, 194, 280, 135, 144, 367, 23, 78, 33, 233, 325, 263, 372, 49, 336, 362, 64, 223, 246, 62, 260, 286, 291, 211, 155, 378, 154, 65, 204, 171, 148, 137, 14, 102, 24, 274, 146, 216, 48, 244, 213, 134, 334, 12, 96, 315, 164, 70, 201, 330, 2, 68, 357, 56, 314, 249, 202, 242, 31, 161, 7, 116, 222, 248, 107, 167, 327, 108, 58, 93, 123, 302, 316, 289, 308, 254, 236, 47, 173, 281, 360, 158, 158, 53, 38, 82, 376, 71, 257, 113, 140, 94, 348, 232, 377, 150, 0, 240, 279, 184, 16, 365, 128, 175, 366, 227, 344, 52, 230, 111, 72, 245, 288, 270, 131, 84, 197, 267, 352, 359, 259, 262, 139, 98, 312, 303, 159, 317, 165, 5, 133, 77, 275, 266, 341, 337, 292, 198, 170, 340, 346, 261, 276, 273, 157, 27, 381, 67, 105, 382, 294, 75, 207, 13, 122, 76, 55, 114, 324, 255, 228, 265, 142, 278, 32, 166, 231, 322, 224, 283, 332, 179, 264, 306, 321] + 22: [371, 281, 216, 198, 343, 331, 181, 260, 141, 148, 24, 261, 328, 207, 224, 242, 20, 184, 376, 230, 4, 74, 118, 244, 61, 317, 352, 229, 381, 153, 188, 251, 336, 375, 161, 8, 123, 60, 193, 214, 122, 57, 228, 3, 53, 339, 258, 374, 233, 71, 165, 12, 105, 320, 341, 98, 65, 22, 124, 189, 327, 335, 0, 180, 300, 271, 202, 136, 356, 26, 265, 103, 232, 102, 219, 182, 92, 294, 152, 134, 342, 19, 168, 212, 296, 354, 88, 326, 43, 213, 163, 114, 270, 167, 86, 231, 315, 192, 287, 360, 307, 310, 346, 359, 254, 254, 204, 51, 343, 209, 181, 34, 260, 148, 24, 330, 328, 115, 224, 288, 201, 20, 110, 94, 144, 169, 29, 44, 383, 362, 240, 50, 70, 345, 285, 311, 173, 80, 372, 378, 312, 125, 17, 369, 226, 170, 175, 358, 82, 191, 63, 49, 15, 309, 99, 225, 367, 39, 47, 305, 155, 200, 54, 227, 78, 238, 299, 241, 13, 211, 215, 96, 353, 379, 298, 108, 246, 119, 266, 282, 120, 30, 62, 179, 350, 104, 69, 302, 308, 363, 162, 5, 140, 304, 277, 172, 293, 243, 194, 234, 325, 205, 289, 2, 28, 79, 95, 101, 254, 316, 216, 126, 278, 278, 221, 221, 260, 141, 261, 276, 132, 42, 337, 164, 183, 73, 376, 94, 37, 263, 329, 235, 117, 58, 252, 348, 357, 351, 218, 338, 306, 67, 68, 83, 106, 264, 259, 27, 349, 361, 347, 113, 340, 366, 220, 18, 324, 59, 41, 210, 1, 380, 159, 87, 301, 236, 199, 283, 11, 373, 382, 268, 138, 290, 77, 89, 275, 46, 237, 262, 274, 160, 186, 137, 334, 36, 129, 190, 16, 64, 90, 344, 31, 150, 370, 127, 284, 66, 146, 295, 107, 121, 368, 297, 10, 135, 249, 292, 143, 154, 157, 217, 254, 281, 33, 9, 332, 209, 181, 221, 141, 141, 261, 328, 147, 38, 337, 288, 183, 178, 178, 110, 177, 112, 139, 93, 355, 130, 333, 55, 272, 149, 7, 111, 100, 195, 76, 174, 365, 222, 116, 185, 206, 14, 75, 109, 156, 48, 40, 25, 131, 321, 133, 72, 239, 319, 248, 318, 303, 322, 269, 377, 256, 32, 45, 81, 286, 97, 314, 273, 364, 176, 208, 279, 35, 85, 187, 151, 52, 196, 145, 91, 257, 291, 171, 128, 255, 223, 6, 23, 21, 197, 158, 84, 245, 166, 56, 203, 313, 142, 280, 323, 247, 250, 253, 267] + 23: [26, 126, 264, 124, 175, 314, 275, 120, 71, 170, 19, 305, 187, 82, 240, 244, 203, 85, 197, 324, 327, 47, 176, 156, 180, 127, 57, 55, 270, 56, 130, 41, 190, 35, 80, 50, 102, 249, 191, 229, 152, 10, 228, 224, 136, 210, 356, 157, 32, 96, 160, 98, 53, 65, 301, 307, 382, 316, 284, 352, 258, 143, 0, 128, 245, 174, 206, 134, 121, 122, 297, 294, 239, 25, 139, 355, 106, 248, 140, 282, 158, 353, 34, 31, 322, 165, 334, 299, 343, 22, 289, 63, 183, 145, 332, 60, 186, 366, 234, 15, 291, 286, 242, 168, 26, 126, 264, 124, 175, 314, 275, 71, 71, 147, 77, 305, 201, 167, 173, 269, 323, 252, 72, 135, 177, 177, 261, 232, 192, 304, 372, 59, 351, 315, 78, 263, 209, 309, 144, 184, 74, 70, 36, 149, 95, 195, 38, 290, 215, 311, 101, 361, 1, 119, 93, 365, 52, 125, 333, 179, 347, 250, 204, 303, 54, 363, 339, 200, 335, 346, 16, 68, 354, 243, 362, 285, 97, 115, 369, 267, 214, 79, 367, 202, 9, 319, 62, 266, 350, 377, 171, 181, 118, 141, 345, 278, 105, 89, 293, 380, 13, 18, 110, 4, 75, 43, 37, 265, 26, 328, 281, 51, 69, 172, 314, 274, 376, 100, 132, 132, 295, 167, 306, 244, 12, 252, 72, 135, 103, 358, 27, 94, 241, 374, 169, 218, 159, 148, 371, 92, 133, 131, 146, 111, 378, 220, 233, 271, 313, 194, 237, 272, 221, 292, 44, 222, 298, 185, 329, 66, 231, 5, 325, 321, 223, 330, 193, 76, 45, 73, 109, 280, 28, 164, 7, 212, 163, 155, 349, 236, 336, 375, 33, 104, 260, 161, 188, 61, 253, 357, 88, 276, 337, 238, 259, 360, 364, 318, 30, 117, 246, 262, 217, 198, 296, 251, 29, 3, 39, 8, 320, 288, 26, 328, 225, 124, 172, 172, 314, 275, 211, 100, 48, 305, 331, 187, 24, 87, 323, 85, 300, 300, 279, 287, 154, 84, 67, 11, 182, 2, 6, 86, 257, 283, 340, 379, 326, 213, 219, 312, 91, 151, 317, 23, 342, 40, 255, 129, 359, 370, 150, 153, 254, 17, 273, 137, 178, 207, 162, 341, 196, 348, 90, 58, 344, 256, 107, 302, 108, 21, 114, 14, 268, 99, 123, 338, 227, 373, 308, 205, 49, 46, 310, 368, 230, 277, 383, 116, 247, 381, 138, 20, 208, 42, 83, 199, 112, 142, 166, 226, 113, 216, 189, 235, 81, 64] + 24: [0, 87, 9, 9, 246, 154, 265, 6, 252, 7, 7, 172, 223, 185, 381, 357, 369, 27, 189, 286, 104, 220, 370, 186, 283, 53, 78, 159, 354, 144, 91, 61, 273, 41, 106, 355, 350, 356, 49, 177, 293, 345, 307, 310, 287, 279, 376, 83, 257, 84, 13, 233, 197, 313, 1, 268, 66, 312, 108, 215, 346, 335, 267, 282, 271, 176, 88, 296, 107, 212, 188, 249, 136, 8, 148, 163, 262, 209, 288, 44, 109, 25, 18, 256, 187, 378, 240, 75, 141, 200, 316, 243, 371, 261, 364, 100, 2, 45, 21, 129, 232, 74, 81, 115, 0, 92, 110, 9, 111, 154, 265, 20, 22, 252, 372, 259, 263, 139, 157, 357, 361, 324, 128, 112, 266, 334, 62, 207, 237, 343, 213, 24, 164, 344, 382, 366, 71, 318, 254, 156, 46, 113, 149, 351, 341, 182, 12, 3, 206, 221, 140, 59, 147, 235, 269, 295, 311, 242, 51, 11, 292, 239, 60, 270, 37, 218, 170, 72, 276, 303, 120, 10, 363, 76, 284, 320, 281, 42, 67, 19, 55, 50, 234, 238, 132, 101, 68, 367, 247, 43, 174, 300, 90, 165, 227, 340, 33, 117, 305, 17, 152, 89, 127, 133, 251, 166, 198, 241, 290, 52, 211, 246, 154, 265, 95, 20, 328, 252, 372, 210, 223, 337, 360, 70, 70, 85, 189, 64, 5, 255, 375, 230, 319, 137, 202, 204, 192, 35, 190, 54, 325, 299, 274, 260, 138, 377, 96, 73, 338, 145, 118, 77, 48, 58, 297, 63, 169, 153, 16, 278, 222, 304, 315, 56, 302, 194, 30, 181, 309, 258, 203, 275, 23, 79, 14, 253, 86, 175, 161, 126, 99, 373, 191, 160, 171, 102, 193, 342, 248, 362, 236, 125, 229, 98, 162, 250, 124, 199, 289, 123, 349, 214, 330, 82, 358, 36, 374, 179, 368, 353, 379, 383, 97, 52, 92, 246, 154, 265, 6, 20, 22, 252, 7, 122, 223, 263, 28, 105, 369, 85, 121, 128, 294, 301, 29, 146, 231, 26, 31, 135, 277, 195, 15, 306, 93, 226, 131, 308, 142, 130, 39, 336, 224, 380, 245, 205, 272, 114, 321, 326, 178, 216, 168, 323, 116, 47, 244, 280, 339, 65, 225, 134, 103, 365, 333, 4, 38, 348, 150, 196, 173, 32, 151, 94, 352, 57, 331, 359, 329, 217, 291, 317, 183, 180, 184, 80, 201, 143, 228, 332, 119, 167, 285, 347, 69, 208, 322, 40, 219, 34, 155, 158, 327, 264, 298, 314] + 25: [315, 315, 269, 228, 247, 247, 379, 82, 85, 101, 282, 375, 78, 61, 107, 25, 309, 284, 152, 62, 112, 333, 5, 273, 305, 38, 380, 59, 18, 351, 381, 346, 252, 9, 84, 231, 125, 153, 89, 367, 146, 164, 135, 4, 359, 20, 225, 352, 203, 237, 81, 68, 354, 308, 357, 213, 297, 28, 339, 181, 14, 355, 105, 356, 79, 53, 249, 74, 110, 127, 111, 271, 35, 75, 36, 86, 183, 283, 364, 208, 40, 175, 64, 92, 349, 3, 63, 344, 147, 363, 12, 42, 90, 169, 288, 329, 194, 170, 43, 44, 121, 156, 263, 91, 315, 315, 98, 65, 228, 298, 379, 379, 317, 72, 180, 276, 276, 23, 372, 214, 29, 52, 192, 376, 77, 232, 145, 7, 66, 33, 216, 230, 207, 262, 287, 302, 215, 373, 253, 209, 80, 256, 54, 366, 227, 161, 261, 348, 196, 291, 211, 336, 132, 319, 221, 383, 345, 259, 330, 37, 134, 331, 116, 257, 144, 310, 67, 212, 200, 27, 223, 17, 275, 353, 337, 163, 22, 117, 93, 124, 167, 70, 226, 173, 293, 289, 245, 41, 188, 265, 205, 119, 57, 141, 71, 217, 301, 198, 266, 241, 106, 13, 338, 382, 189, 239, 123, 178, 315, 269, 269, 139, 83, 268, 379, 299, 72, 282, 282, 180, 162, 372, 39, 214, 29, 327, 192, 246, 171, 49, 190, 160, 313, 102, 314, 318, 120, 342, 154, 131, 306, 197, 377, 56, 229, 244, 138, 60, 279, 321, 307, 103, 143, 114, 347, 115, 304, 316, 199, 187, 55, 204, 184, 290, 328, 58, 254, 201, 94, 370, 11, 31, 238, 159, 340, 193, 243, 166, 186, 109, 177, 281, 34, 250, 320, 45, 210, 311, 46, 371, 118, 174, 137, 155, 48, 264, 360, 172, 374, 285, 108, 219, 47, 50, 6, 151, 270, 19, 165, 220, 267, 158, 315, 269, 269, 139, 258, 168, 379, 334, 317, 72, 282, 30, 332, 23, 107, 2, 0, 284, 113, 62, 10, 73, 280, 235, 8, 296, 32, 378, 368, 100, 136, 294, 222, 234, 341, 240, 76, 99, 323, 126, 343, 97, 218, 176, 140, 242, 191, 369, 278, 295, 350, 312, 130, 324, 185, 15, 202, 365, 133, 274, 69, 149, 24, 150, 300, 251, 361, 179, 358, 260, 248, 51, 292, 325, 157, 362, 122, 21, 96, 87, 16, 128, 236, 322, 142, 224, 95, 1, 286, 335, 26, 88, 104, 272, 148, 182, 206, 195, 255, 303, 129, 233, 277, 326] + 26: [80, 136, 6, 292, 380, 269, 269, 237, 237, 172, 172, 53, 342, 371, 184, 291, 304, 254, 224, 116, 279, 33, 308, 307, 227, 223, 315, 376, 108, 146, 131, 82, 265, 204, 311, 176, 138, 319, 323, 133, 367, 306, 27, 97, 103, 221, 343, 57, 20, 192, 17, 277, 65, 113, 212, 0, 107, 76, 196, 50, 129, 123, 153, 363, 298, 189, 368, 78, 49, 3, 105, 352, 225, 354, 334, 43, 9, 297, 361, 333, 193, 13, 91, 274, 358, 270, 284, 208, 283, 211, 28, 164, 381, 155, 379, 288, 132, 213, 111, 139, 169, 170, 175, 209, 80, 136, 6, 281, 89, 11, 349, 237, 217, 217, 90, 214, 236, 181, 181, 338, 253, 339, 230, 16, 94, 183, 38, 266, 369, 135, 22, 30, 36, 92, 328, 55, 200, 359, 128, 48, 259, 191, 166, 262, 142, 2, 356, 251, 241, 186, 118, 294, 127, 71, 15, 239, 81, 161, 98, 378, 171, 252, 37, 350, 144, 75, 182, 383, 25, 134, 336, 7, 197, 140, 340, 47, 165, 255, 72, 8, 346, 77, 331, 244, 163, 326, 205, 216, 324, 258, 42, 145, 282, 85, 345, 257, 84, 310, 322, 273, 60, 203, 1, 54, 68, 86, 96, 106, 150, 136, 5, 292, 89, 11, 269, 124, 237, 18, 90, 53, 63, 21, 70, 291, 162, 160, 120, 289, 276, 316, 137, 341, 293, 61, 226, 88, 245, 382, 62, 332, 67, 156, 148, 198, 26, 360, 190, 151, 353, 174, 152, 327, 79, 24, 201, 268, 285, 104, 318, 195, 159, 10, 35, 202, 267, 370, 231, 114, 187, 143, 232, 19, 117, 301, 157, 264, 220, 12, 125, 362, 317, 109, 335, 64, 34, 59, 112, 39, 29, 206, 119, 314, 373, 313, 4, 14, 250, 330, 46, 188, 23, 58, 248, 207, 309, 260, 218, 228, 234, 242, 246, 261, 320, 136, 136, 233, 89, 380, 271, 124, 217, 18, 172, 90, 236, 21, 263, 253, 149, 339, 230, 69, 337, 344, 130, 278, 238, 355, 158, 249, 126, 167, 110, 83, 247, 180, 31, 272, 178, 179, 321, 287, 44, 229, 99, 102, 173, 122, 51, 194, 66, 365, 357, 215, 87, 73, 45, 290, 115, 41, 32, 256, 93, 177, 372, 312, 280, 302, 219, 329, 300, 121, 325, 364, 141, 185, 40, 347, 275, 74, 303, 100, 154, 240, 199, 101, 296, 56, 377, 243, 295, 222, 147, 235, 366, 95, 286, 210, 52, 168, 299, 305, 348, 351, 374, 375] + 27: [34, 103, 37, 177, 279, 58, 286, 333, 263, 61, 312, 90, 265, 168, 252, 269, 30, 197, 49, 80, 341, 358, 292, 131, 369, 207, 143, 183, 75, 251, 323, 85, 242, 76, 339, 258, 115, 156, 23, 32, 201, 141, 193, 187, 42, 27, 155, 162, 125, 126, 326, 214, 291, 215, 327, 268, 124, 267, 129, 248, 299, 171, 28, 318, 321, 365, 167, 0, 275, 35, 159, 130, 306, 96, 231, 154, 313, 223, 22, 293, 352, 110, 289, 382, 62, 86, 166, 192, 1, 194, 148, 314, 221, 295, 92, 57, 209, 51, 180, 235, 285, 310, 338, 342, 234, 103, 301, 177, 117, 4, 225, 351, 263, 31, 312, 374, 265, 233, 252, 69, 26, 315, 128, 80, 227, 211, 354, 24, 213, 224, 97, 55, 228, 272, 21, 303, 266, 149, 99, 232, 102, 83, 332, 325, 140, 284, 356, 317, 98, 133, 88, 254, 309, 54, 329, 6, 172, 136, 296, 122, 287, 276, 111, 106, 347, 9, 120, 249, 29, 250, 217, 84, 205, 123, 79, 151, 344, 280, 113, 259, 319, 50, 8, 264, 229, 150, 372, 158, 257, 334, 345, 108, 173, 331, 138, 170, 255, 355, 363, 206, 378, 93, 203, 247, 146, 163, 176, 220, 234, 103, 37, 145, 59, 225, 225, 12, 12, 61, 312, 90, 300, 164, 73, 69, 135, 320, 119, 376, 341, 366, 181, 377, 348, 222, 336, 190, 335, 48, 290, 340, 381, 324, 216, 72, 238, 294, 89, 82, 322, 373, 118, 337, 305, 161, 7, 370, 91, 271, 10, 175, 114, 304, 274, 127, 375, 362, 67, 239, 147, 2, 199, 307, 101, 47, 95, 367, 361, 153, 116, 105, 139, 144, 152, 184, 282, 346, 74, 253, 302, 330, 16, 142, 273, 94, 189, 64, 165, 56, 245, 311, 210, 277, 40, 371, 17, 36, 100, 212, 270, 240, 244, 262, 81, 103, 103, 104, 219, 4, 286, 286, 12, 70, 350, 90, 168, 71, 73, 204, 30, 357, 349, 349, 227, 211, 354, 202, 188, 179, 237, 112, 182, 230, 178, 260, 380, 383, 196, 77, 343, 308, 78, 278, 297, 25, 109, 191, 241, 15, 261, 328, 368, 316, 246, 379, 174, 5, 218, 360, 45, 46, 18, 43, 20, 44, 38, 66, 121, 353, 13, 3, 107, 288, 256, 33, 14, 132, 281, 208, 185, 359, 65, 186, 157, 41, 364, 68, 195, 283, 198, 87, 52, 226, 160, 11, 19, 298, 169, 134, 243, 63, 200, 236, 39, 53, 60, 137] + 28: [4, 312, 239, 310, 77, 344, 26, 26, 328, 203, 185, 152, 306, 302, 162, 195, 107, 103, 25, 286, 197, 113, 253, 7, 199, 346, 378, 137, 320, 71, 204, 58, 261, 352, 90, 361, 285, 220, 370, 32, 105, 182, 13, 349, 376, 114, 380, 43, 206, 178, 1, 290, 194, 183, 46, 245, 6, 249, 57, 274, 159, 21, 205, 175, 169, 230, 15, 84, 332, 297, 154, 377, 109, 31, 330, 119, 287, 351, 225, 131, 303, 146, 347, 192, 252, 101, 275, 140, 94, 141, 189, 254, 145, 313, 124, 62, 301, 138, 298, 150, 156, 180, 207, 215, 355, 161, 201, 196, 250, 250, 26, 92, 356, 70, 317, 226, 306, 97, 243, 172, 30, 288, 93, 286, 197, 214, 307, 374, 75, 50, 219, 3, 223, 59, 51, 5, 200, 383, 160, 130, 167, 163, 242, 299, 221, 56, 340, 238, 273, 247, 48, 16, 335, 222, 184, 78, 118, 244, 260, 264, 327, 358, 64, 69, 98, 381, 143, 322, 296, 12, 66, 170, 342, 34, 24, 294, 372, 218, 96, 266, 80, 39, 251, 246, 8, 17, 229, 139, 232, 174, 373, 277, 102, 202, 304, 354, 181, 318, 164, 186, 2, 40, 147, 359, 257, 279, 311, 329, 161, 161, 0, 168, 250, 22, 324, 356, 153, 188, 317, 255, 255, 120, 243, 233, 30, 52, 122, 99, 241, 321, 37, 357, 369, 108, 166, 95, 325, 42, 228, 116, 63, 235, 88, 135, 67, 289, 41, 240, 125, 217, 20, 33, 19, 193, 278, 316, 300, 333, 91, 14, 121, 142, 365, 265, 171, 208, 23, 295, 82, 375, 362, 61, 36, 258, 270, 334, 133, 280, 151, 382, 144, 79, 364, 149, 323, 345, 38, 284, 234, 85, 237, 47, 55, 76, 267, 123, 45, 126, 65, 53, 83, 309, 87, 262, 11, 132, 190, 10, 18, 60, 86, 115, 161, 161, 248, 168, 250, 344, 283, 92, 153, 70, 185, 185, 176, 302, 162, 155, 107, 52, 25, 99, 241, 321, 224, 272, 263, 128, 236, 191, 368, 227, 158, 165, 100, 129, 157, 339, 28, 337, 326, 269, 209, 271, 216, 308, 367, 315, 127, 268, 341, 134, 68, 282, 27, 256, 112, 179, 49, 81, 44, 314, 305, 177, 148, 106, 74, 104, 210, 211, 363, 366, 73, 136, 331, 117, 89, 213, 348, 110, 293, 292, 231, 72, 276, 259, 29, 281, 35, 111, 291, 350, 371, 360, 187, 319, 198, 212, 9, 54, 173, 379, 336, 338, 343, 353] + 29: [383, 14, 196, 195, 365, 244, 51, 25, 76, 354, 323, 198, 304, 362, 265, 74, 60, 44, 173, 173, 208, 11, 16, 255, 183, 243, 57, 134, 175, 188, 216, 145, 82, 319, 193, 350, 61, 360, 378, 85, 12, 106, 178, 329, 18, 347, 272, 59, 97, 181, 380, 142, 20, 313, 371, 217, 316, 238, 242, 36, 126, 170, 10, 28, 191, 55, 381, 56, 42, 7, 166, 13, 374, 352, 227, 307, 168, 149, 17, 71, 45, 337, 30, 246, 109, 49, 41, 201, 2, 80, 5, 226, 64, 81, 221, 342, 247, 274, 367, 249, 39, 355, 240, 285, 197, 271, 58, 118, 244, 336, 51, 361, 137, 277, 258, 203, 295, 339, 67, 262, 236, 309, 282, 23, 127, 148, 255, 89, 326, 180, 366, 375, 235, 24, 237, 68, 341, 123, 293, 86, 364, 172, 259, 218, 128, 116, 273, 222, 136, 19, 146, 228, 264, 78, 310, 267, 125, 269, 328, 344, 373, 22, 229, 348, 93, 52, 302, 135, 213, 111, 287, 327, 84, 299, 338, 261, 187, 239, 121, 194, 113, 219, 308, 154, 184, 211, 141, 296, 234, 155, 358, 279, 21, 330, 75, 130, 291, 165, 268, 138, 305, 289, 214, 6, 185, 26, 363, 176, 197, 283, 196, 118, 124, 336, 318, 361, 379, 277, 198, 333, 304, 129, 67, 186, 60, 44, 202, 23, 127, 11, 266, 346, 65, 140, 174, 87, 190, 343, 163, 370, 382, 210, 38, 50, 94, 132, 248, 298, 34, 257, 232, 349, 225, 72, 280, 90, 150, 345, 353, 292, 105, 205, 317, 167, 47, 122, 224, 312, 315, 40, 369, 147, 209, 79, 103, 220, 70, 153, 250, 161, 15, 233, 69, 62, 356, 306, 334, 99, 115, 35, 251, 92, 164, 96, 223, 254, 8, 100, 27, 286, 1, 372, 335, 281, 98, 256, 206, 143, 177, 108, 91, 152, 73, 73, 88, 118, 244, 54, 318, 361, 137, 120, 323, 203, 295, 362, 33, 110, 236, 320, 282, 311, 311, 351, 160, 89, 112, 119, 48, 159, 332, 63, 275, 204, 207, 102, 357, 278, 4, 324, 314, 215, 182, 139, 156, 200, 131, 300, 171, 276, 151, 101, 104, 3, 32, 294, 340, 77, 359, 83, 169, 31, 157, 230, 46, 377, 53, 303, 114, 368, 158, 9, 288, 199, 192, 253, 179, 270, 162, 376, 0, 43, 263, 252, 290, 297, 325, 301, 133, 189, 331, 95, 245, 260, 231, 107, 321, 241, 117, 284, 37, 144, 212, 322, 29, 66] + 30: [140, 37, 266, 262, 208, 208, 213, 333, 137, 187, 366, 65, 245, 196, 52, 35, 24, 351, 156, 156, 289, 321, 28, 346, 68, 17, 205, 303, 340, 49, 341, 221, 77, 253, 256, 58, 215, 141, 136, 226, 296, 274, 369, 258, 365, 325, 104, 353, 84, 231, 335, 306, 14, 319, 129, 111, 293, 55, 73, 181, 297, 290, 374, 318, 186, 154, 276, 107, 267, 371, 279, 161, 203, 99, 155, 5, 209, 364, 122, 314, 169, 178, 330, 120, 198, 106, 56, 265, 372, 367, 93, 229, 251, 30, 176, 301, 354, 88, 94, 101, 118, 123, 128, 134, 140, 37, 266, 262, 208, 239, 239, 76, 187, 13, 243, 65, 245, 78, 52, 35, 143, 295, 351, 2, 361, 234, 223, 64, 41, 212, 271, 315, 216, 224, 240, 317, 336, 1, 71, 193, 236, 151, 204, 277, 292, 281, 160, 67, 60, 23, 278, 261, 54, 72, 383, 162, 359, 20, 294, 66, 375, 211, 332, 7, 249, 50, 145, 288, 194, 311, 95, 121, 220, 350, 32, 304, 57, 83, 85, 323, 130, 117, 3, 153, 15, 19, 308, 352, 112, 207, 235, 0, 149, 63, 26, 182, 119, 4, 70, 227, 307, 379, 268, 269, 322, 334, 345, 380, 140, 254, 124, 208, 208, 239, 239, 272, 86, 358, 75, 242, 245, 135, 214, 378, 46, 257, 373, 62, 62, 188, 45, 144, 33, 164, 316, 362, 159, 43, 74, 12, 177, 152, 339, 192, 309, 299, 217, 195, 18, 170, 285, 98, 157, 343, 132, 108, 38, 329, 114, 142, 163, 197, 172, 280, 349, 241, 244, 248, 92, 44, 79, 360, 166, 110, 230, 202, 263, 247, 139, 326, 381, 116, 287, 165, 370, 368, 127, 355, 233, 16, 190, 255, 300, 270, 199, 313, 39, 42, 148, 34, 87, 252, 201, 305, 363, 146, 185, 189, 218, 219, 237, 259, 133, 158, 266, 184, 208, 239, 239, 222, 29, 358, 53, 53, 65, 59, 214, 225, 143, 168, 373, 206, 11, 289, 246, 10, 147, 283, 302, 250, 97, 312, 89, 138, 357, 100, 6, 275, 9, 210, 320, 382, 344, 260, 356, 298, 174, 200, 96, 91, 167, 179, 103, 150, 376, 51, 310, 286, 180, 109, 48, 115, 282, 284, 175, 25, 337, 342, 377, 183, 347, 324, 264, 90, 348, 69, 125, 327, 82, 273, 81, 291, 105, 173, 232, 113, 126, 102, 338, 80, 328, 171, 61, 228, 191, 27, 131, 238, 331, 8, 21, 22, 31, 36, 40, 47] + 31: [248, 265, 68, 221, 0, 19, 159, 36, 233, 324, 26, 26, 116, 115, 278, 326, 250, 70, 95, 261, 268, 87, 330, 227, 349, 209, 155, 58, 170, 283, 360, 338, 348, 102, 269, 106, 28, 236, 111, 152, 232, 40, 245, 368, 81, 57, 65, 176, 222, 190, 41, 284, 314, 288, 217, 303, 16, 84, 49, 285, 153, 12, 237, 20, 14, 177, 43, 91, 371, 194, 198, 107, 306, 226, 210, 139, 367, 251, 9, 168, 99, 113, 381, 127, 293, 193, 44, 344, 124, 35, 242, 319, 359, 257, 94, 199, 151, 13, 165, 148, 37, 149, 347, 350, 320, 331, 68, 241, 100, 0, 365, 159, 166, 324, 158, 26, 143, 133, 239, 144, 363, 147, 70, 185, 339, 87, 39, 96, 325, 22, 60, 61, 171, 75, 72, 355, 334, 258, 290, 33, 380, 249, 244, 138, 231, 38, 55, 243, 134, 88, 122, 137, 77, 202, 256, 374, 213, 118, 123, 302, 141, 8, 29, 110, 260, 273, 11, 305, 376, 266, 275, 351, 204, 264, 304, 238, 263, 375, 240, 101, 259, 157, 315, 337, 234, 327, 191, 216, 7, 235, 274, 282, 255, 114, 18, 179, 30, 161, 86, 230, 104, 154, 67, 341, 90, 289, 229, 329, 125, 128, 201, 192, 0, 0, 3, 159, 279, 166, 252, 379, 292, 115, 239, 335, 69, 215, 187, 185, 219, 150, 328, 227, 146, 186, 119, 188, 378, 253, 299, 56, 276, 286, 189, 296, 78, 82, 353, 136, 74, 17, 98, 2, 366, 169, 182, 105, 31, 318, 316, 225, 307, 373, 310, 336, 92, 142, 195, 361, 4, 51, 362, 271, 6, 272, 346, 352, 228, 48, 224, 50, 47, 162, 85, 309, 160, 178, 340, 342, 300, 64, 356, 52, 53, 79, 382, 126, 295, 184, 167, 301, 332, 218, 163, 358, 10, 200, 203, 89, 212, 294, 120, 132, 128, 128, 201, 32, 100, 0, 159, 233, 233, 158, 252, 121, 143, 364, 278, 144, 363, 25, 261, 270, 267, 317, 76, 76, 173, 140, 206, 333, 34, 208, 164, 62, 131, 214, 112, 223, 145, 211, 254, 93, 63, 23, 54, 66, 175, 277, 205, 322, 46, 15, 287, 97, 312, 21, 383, 369, 45, 291, 73, 354, 196, 183, 108, 247, 59, 323, 83, 220, 1, 130, 281, 103, 80, 308, 42, 345, 129, 298, 24, 262, 321, 135, 109, 280, 156, 181, 197, 372, 370, 313, 27, 174, 357, 311, 377, 297, 5, 117, 343, 71, 180, 246, 172, 207] + 32: [94, 63, 288, 352, 246, 234, 367, 146, 106, 92, 285, 190, 229, 370, 51, 360, 12, 257, 327, 107, 342, 166, 322, 16, 204, 319, 202, 263, 151, 211, 87, 331, 273, 113, 252, 289, 189, 168, 254, 212, 85, 354, 185, 50, 206, 28, 157, 239, 45, 230, 382, 286, 183, 240, 176, 3, 80, 276, 24, 309, 37, 378, 139, 236, 154, 144, 136, 282, 102, 191, 165, 103, 195, 376, 13, 222, 374, 330, 201, 0, 52, 138, 343, 377, 1, 31, 137, 216, 368, 242, 88, 47, 192, 295, 11, 226, 15, 54, 162, 57, 167, 181, 244, 245, 94, 63, 148, 148, 155, 346, 367, 367, 209, 92, 285, 241, 310, 200, 8, 66, 122, 327, 38, 164, 344, 329, 275, 275, 380, 65, 225, 75, 279, 97, 143, 233, 109, 232, 120, 58, 345, 369, 298, 227, 303, 111, 199, 101, 25, 332, 269, 48, 150, 237, 142, 127, 29, 350, 371, 84, 4, 205, 6, 10, 340, 77, 193, 372, 339, 110, 82, 221, 99, 307, 152, 287, 197, 105, 365, 73, 356, 261, 160, 119, 355, 170, 278, 334, 364, 133, 46, 180, 321, 215, 74, 217, 318, 291, 194, 267, 326, 114, 316, 147, 14, 123, 169, 177, 63, 63, 218, 352, 33, 135, 367, 108, 187, 179, 347, 241, 130, 21, 51, 280, 2, 274, 141, 107, 41, 166, 322, 40, 186, 348, 96, 83, 126, 78, 258, 116, 308, 19, 357, 306, 301, 266, 297, 72, 55, 71, 69, 231, 314, 64, 281, 250, 59, 253, 256, 32, 53, 249, 81, 115, 30, 337, 315, 366, 56, 62, 349, 363, 172, 320, 210, 361, 132, 43, 299, 125, 238, 248, 117, 223, 264, 270, 305, 67, 79, 149, 208, 220, 224, 60, 277, 259, 95, 18, 358, 112, 272, 290, 338, 328, 333, 145, 379, 173, 251, 262, 265, 296, 63, 63, 218, 352, 33, 268, 367, 49, 362, 323, 347, 190, 271, 163, 163, 122, 121, 219, 38, 164, 344, 329, 61, 214, 359, 89, 174, 184, 283, 140, 156, 131, 335, 228, 294, 7, 313, 171, 91, 325, 36, 104, 292, 26, 175, 213, 324, 178, 86, 182, 293, 336, 351, 39, 353, 70, 302, 129, 247, 381, 22, 198, 158, 317, 311, 5, 34, 42, 9, 284, 243, 124, 304, 341, 128, 20, 153, 134, 35, 44, 196, 203, 188, 255, 118, 100, 23, 76, 260, 161, 17, 98, 159, 207, 93, 235, 27, 90, 300, 68, 383, 312, 373, 375] + 33: [86, 240, 145, 63, 177, 339, 38, 362, 362, 121, 380, 36, 231, 319, 297, 343, 251, 301, 283, 154, 307, 68, 348, 358, 322, 89, 226, 37, 127, 335, 233, 17, 35, 25, 338, 117, 192, 40, 289, 217, 14, 126, 354, 202, 93, 355, 230, 4, 123, 198, 80, 246, 156, 50, 104, 200, 88, 279, 234, 364, 368, 345, 106, 71, 350, 349, 52, 70, 101, 110, 29, 219, 221, 135, 182, 238, 181, 11, 243, 27, 291, 310, 120, 62, 178, 353, 337, 199, 97, 275, 374, 269, 44, 344, 294, 203, 357, 282, 159, 189, 236, 102, 333, 372, 167, 51, 75, 260, 177, 331, 38, 377, 153, 64, 64, 229, 325, 379, 81, 107, 251, 258, 283, 299, 267, 179, 147, 241, 201, 89, 352, 266, 169, 190, 21, 54, 160, 87, 150, 227, 356, 334, 133, 9, 235, 116, 290, 351, 205, 173, 16, 254, 45, 347, 22, 262, 19, 157, 370, 163, 180, 152, 60, 58, 122, 249, 330, 371, 113, 28, 56, 26, 53, 285, 41, 327, 318, 128, 20, 23, 74, 130, 85, 381, 188, 224, 257, 118, 247, 175, 242, 363, 214, 171, 274, 359, 309, 172, 2, 46, 100, 223, 0, 265, 12, 34, 220, 277, 91, 91, 176, 320, 339, 339, 191, 38, 31, 323, 229, 36, 148, 319, 96, 343, 258, 183, 299, 82, 342, 267, 314, 94, 108, 72, 139, 5, 361, 270, 376, 141, 1, 33, 151, 193, 237, 131, 222, 311, 8, 305, 144, 211, 187, 149, 138, 317, 383, 276, 196, 375, 382, 55, 367, 373, 114, 369, 340, 213, 49, 137, 292, 215, 24, 281, 65, 166, 278, 324, 248, 295, 252, 194, 298, 346, 300, 207, 132, 225, 78, 119, 245, 365, 244, 59, 341, 378, 195, 306, 7, 228, 99, 57, 304, 302, 77, 143, 272, 264, 161, 32, 69, 142, 91, 328, 140, 260, 259, 280, 38, 38, 42, 158, 380, 36, 148, 319, 81, 115, 115, 39, 301, 154, 307, 73, 125, 94, 146, 84, 164, 206, 48, 255, 273, 204, 218, 30, 210, 155, 208, 308, 268, 288, 61, 18, 232, 303, 329, 90, 109, 174, 216, 79, 124, 162, 256, 197, 209, 360, 186, 212, 13, 105, 170, 15, 43, 3, 336, 66, 366, 103, 271, 112, 261, 67, 136, 10, 6, 316, 111, 286, 168, 76, 184, 332, 312, 98, 134, 326, 315, 165, 92, 250, 253, 313, 321, 296, 284, 47, 185, 263, 239, 129, 83, 95, 287, 293] + 34: [372, 108, 69, 207, 259, 357, 179, 126, 330, 16, 334, 82, 239, 15, 340, 218, 3, 145, 240, 190, 148, 61, 40, 315, 246, 284, 295, 271, 288, 11, 41, 4, 244, 162, 21, 373, 144, 53, 210, 345, 175, 183, 197, 37, 172, 280, 317, 328, 236, 30, 48, 214, 333, 364, 116, 50, 301, 215, 180, 164, 157, 42, 62, 283, 56, 46, 272, 51, 296, 152, 268, 289, 233, 36, 262, 226, 34, 331, 28, 49, 52, 205, 173, 26, 297, 104, 112, 257, 281, 43, 156, 325, 338, 252, 234, 350, 121, 375, 177, 198, 279, 14, 76, 119, 382, 108, 337, 47, 151, 299, 179, 18, 330, 16, 249, 306, 306, 340, 354, 248, 44, 25, 321, 190, 362, 167, 138, 258, 347, 59, 79, 83, 254, 92, 267, 65, 87, 225, 124, 153, 329, 356, 174, 302, 159, 123, 24, 199, 133, 251, 195, 245, 346, 227, 27, 209, 204, 102, 89, 5, 230, 361, 352, 63, 114, 318, 127, 358, 149, 105, 383, 264, 85, 88, 314, 374, 282, 161, 86, 101, 286, 100, 255, 109, 57, 242, 250, 39, 155, 9, 171, 22, 128, 355, 130, 216, 188, 38, 94, 320, 7, 309, 377, 224, 381, 229, 363, 365, 366, 108, 29, 237, 259, 299, 332, 312, 135, 16, 249, 306, 311, 15, 304, 248, 44, 223, 293, 196, 203, 61, 178, 360, 220, 115, 97, 238, 222, 191, 300, 168, 95, 208, 134, 110, 58, 184, 12, 326, 256, 327, 136, 137, 206, 217, 33, 67, 241, 150, 231, 0, 359, 169, 247, 35, 232, 348, 154, 140, 19, 294, 10, 298, 139, 23, 261, 45, 266, 98, 192, 219, 228, 305, 265, 118, 353, 307, 380, 165, 176, 370, 17, 285, 253, 339, 90, 1, 54, 351, 96, 163, 182, 378, 276, 376, 235, 66, 343, 213, 341, 189, 193, 202, 366, 108, 337, 69, 324, 179, 179, 126, 74, 16, 249, 334, 158, 15, 340, 218, 3, 185, 243, 196, 31, 291, 194, 146, 379, 99, 2, 8, 335, 78, 6, 77, 125, 201, 91, 200, 107, 290, 81, 84, 313, 368, 166, 187, 60, 143, 336, 70, 181, 270, 72, 73, 287, 170, 367, 263, 308, 80, 103, 113, 322, 323, 277, 371, 221, 142, 186, 147, 64, 68, 275, 13, 269, 111, 310, 274, 93, 75, 319, 117, 131, 278, 260, 55, 342, 141, 20, 212, 71, 120, 292, 344, 349, 106, 303, 369, 129, 32, 273, 211, 316, 122, 132, 160] + 35: [48, 353, 220, 169, 28, 240, 319, 319, 299, 379, 211, 297, 345, 367, 318, 68, 141, 257, 153, 86, 276, 31, 289, 281, 116, 206, 239, 15, 46, 114, 234, 335, 361, 266, 54, 159, 312, 134, 160, 168, 356, 81, 230, 171, 282, 104, 190, 334, 209, 320, 201, 300, 11, 16, 172, 92, 109, 145, 202, 348, 82, 339, 216, 290, 99, 179, 70, 196, 279, 315, 84, 374, 79, 274, 332, 151, 259, 29, 316, 278, 181, 306, 250, 354, 255, 358, 150, 238, 69, 83, 333, 123, 193, 208, 51, 119, 71, 76, 124, 283, 291, 284, 40, 293, 309, 75, 45, 260, 182, 205, 144, 319, 218, 303, 131, 56, 368, 367, 314, 133, 53, 138, 4, 342, 203, 72, 322, 281, 346, 340, 78, 186, 375, 265, 194, 164, 191, 331, 136, 37, 275, 102, 357, 360, 296, 18, 132, 61, 5, 328, 215, 372, 125, 143, 95, 122, 302, 96, 235, 350, 277, 227, 180, 91, 264, 382, 383, 24, 174, 185, 127, 142, 175, 217, 25, 338, 103, 65, 292, 87, 189, 253, 149, 245, 94, 249, 195, 262, 165, 263, 139, 107, 271, 327, 310, 371, 252, 336, 0, 39, 341, 229, 2, 64, 146, 381, 60, 228, 363, 80, 45, 28, 44, 205, 237, 58, 218, 1, 1, 131, 345, 67, 68, 133, 53, 324, 129, 86, 276, 38, 113, 113, 231, 340, 224, 147, 352, 349, 326, 241, 162, 14, 126, 27, 98, 163, 321, 140, 244, 272, 308, 36, 93, 298, 294, 30, 137, 243, 152, 90, 256, 323, 261, 57, 8, 366, 110, 214, 369, 204, 34, 166, 376, 269, 380, 17, 66, 73, 88, 378, 176, 317, 365, 248, 161, 156, 370, 22, 232, 121, 23, 49, 77, 97, 183, 148, 197, 270, 223, 233, 188, 242, 210, 226, 173, 212, 59, 199, 85, 200, 13, 343, 363, 75, 45, 169, 246, 62, 130, 319, 299, 254, 1, 297, 345, 295, 295, 286, 55, 47, 21, 342, 203, 72, 322, 3, 221, 154, 355, 41, 364, 311, 307, 50, 362, 52, 222, 329, 157, 373, 187, 267, 325, 128, 9, 158, 192, 268, 213, 108, 42, 313, 337, 12, 32, 170, 287, 74, 20, 111, 305, 106, 117, 347, 330, 273, 7, 184, 19, 301, 285, 43, 344, 6, 35, 198, 105, 63, 135, 207, 377, 115, 112, 26, 155, 89, 288, 167, 10, 101, 178, 351, 225, 33, 219, 258, 247, 280, 251, 120, 236, 100, 118, 304, 359, 177] + 36: [53, 22, 370, 340, 341, 144, 250, 304, 238, 266, 10, 2, 325, 368, 272, 230, 196, 296, 320, 40, 56, 12, 99, 107, 97, 73, 133, 205, 332, 233, 261, 65, 162, 240, 112, 111, 194, 231, 345, 55, 13, 122, 105, 95, 298, 69, 183, 339, 346, 142, 117, 174, 11, 100, 116, 364, 211, 255, 30, 200, 212, 81, 36, 17, 165, 358, 23, 80, 159, 208, 29, 356, 363, 209, 295, 158, 169, 236, 173, 19, 337, 35, 21, 14, 226, 312, 213, 126, 26, 154, 1, 128, 227, 89, 286, 306, 377, 182, 114, 270, 292, 313, 302, 360, 53, 22, 382, 74, 341, 184, 304, 232, 232, 185, 10, 371, 325, 328, 272, 101, 98, 178, 241, 369, 372, 372, 301, 107, 207, 352, 79, 5, 125, 31, 32, 331, 333, 321, 262, 362, 9, 319, 191, 349, 361, 297, 284, 121, 172, 322, 289, 20, 72, 168, 280, 179, 41, 237, 202, 102, 134, 305, 71, 0, 104, 143, 379, 206, 88, 188, 149, 119, 163, 283, 48, 58, 281, 70, 190, 109, 265, 315, 198, 249, 357, 60, 307, 120, 61, 324, 253, 235, 161, 303, 285, 203, 327, 141, 4, 218, 160, 221, 43, 376, 16, 25, 336, 375, 131, 294, 22, 176, 184, 84, 57, 348, 232, 266, 62, 371, 368, 368, 272, 299, 258, 242, 326, 274, 225, 380, 90, 223, 374, 150, 308, 47, 287, 314, 42, 45, 33, 201, 108, 246, 219, 166, 49, 139, 138, 63, 103, 245, 164, 3, 342, 15, 153, 354, 140, 271, 228, 195, 224, 277, 310, 217, 229, 39, 52, 257, 28, 335, 148, 351, 267, 220, 318, 51, 18, 135, 329, 83, 247, 6, 259, 383, 248, 199, 130, 316, 75, 118, 110, 350, 8, 87, 7, 278, 96, 27, 124, 82, 269, 222, 214, 234, 268, 186, 290, 127, 170, 273, 192, 252, 197, 340, 184, 144, 57, 232, 232, 238, 359, 59, 371, 311, 311, 260, 210, 178, 193, 193, 225, 12, 373, 223, 275, 291, 152, 92, 54, 85, 93, 156, 175, 309, 334, 67, 288, 94, 293, 282, 167, 132, 68, 91, 50, 300, 276, 155, 171, 216, 366, 147, 343, 66, 177, 46, 323, 180, 187, 264, 76, 38, 344, 157, 244, 129, 263, 78, 243, 365, 146, 239, 44, 24, 204, 189, 338, 381, 77, 251, 256, 215, 347, 137, 254, 115, 355, 378, 86, 151, 34, 330, 136, 353, 106, 181, 367, 317, 113, 145, 279, 64, 37, 123] + 37: [109, 268, 364, 40, 294, 218, 218, 379, 158, 158, 311, 314, 178, 205, 199, 363, 289, 48, 179, 72, 349, 234, 371, 340, 0, 156, 140, 129, 360, 7, 182, 318, 204, 117, 343, 32, 257, 277, 150, 315, 127, 316, 122, 21, 270, 1, 168, 43, 217, 338, 36, 161, 278, 103, 201, 34, 126, 136, 224, 334, 37, 262, 244, 255, 313, 102, 279, 274, 4, 5, 351, 26, 160, 254, 10, 53, 303, 332, 137, 378, 35, 101, 374, 142, 357, 184, 231, 214, 227, 119, 261, 54, 369, 89, 90, 246, 358, 25, 238, 99, 31, 120, 8, 75, 109, 319, 301, 66, 294, 218, 218, 123, 243, 348, 148, 148, 306, 52, 222, 366, 341, 190, 305, 290, 292, 162, 382, 77, 361, 0, 171, 355, 245, 252, 88, 65, 147, 235, 163, 131, 330, 345, 221, 157, 370, 229, 228, 51, 27, 64, 6, 323, 82, 73, 39, 304, 333, 135, 29, 15, 100, 167, 83, 247, 337, 269, 347, 98, 195, 212, 284, 280, 42, 210, 116, 249, 13, 110, 368, 266, 216, 350, 169, 115, 295, 339, 198, 166, 62, 106, 57, 107, 281, 155, 20, 300, 159, 250, 381, 275, 23, 50, 276, 209, 189, 226, 112, 132, 188, 268, 71, 40, 40, 218, 218, 379, 58, 242, 309, 151, 125, 293, 222, 200, 362, 372, 342, 79, 193, 95, 113, 113, 223, 59, 344, 47, 185, 353, 22, 14, 328, 233, 181, 118, 18, 164, 145, 322, 19, 320, 76, 258, 176, 196, 144, 365, 61, 41, 141, 92, 264, 134, 285, 376, 356, 49, 55, 331, 80, 207, 321, 312, 94, 241, 24, 317, 325, 267, 187, 298, 299, 124, 91, 152, 60, 260, 175, 183, 297, 259, 81, 308, 96, 225, 324, 70, 202, 28, 197, 11, 220, 186, 2, 12, 282, 68, 377, 251, 56, 375, 236, 239, 240, 268, 301, 40, 40, 294, 218, 273, 58, 265, 148, 213, 293, 293, 46, 200, 366, 272, 44, 79, 349, 162, 174, 340, 86, 16, 114, 302, 253, 9, 165, 327, 139, 194, 93, 153, 133, 192, 359, 336, 208, 85, 287, 104, 173, 74, 149, 203, 230, 296, 143, 30, 108, 111, 329, 3, 87, 335, 206, 69, 121, 128, 380, 373, 97, 271, 38, 33, 105, 354, 154, 172, 138, 256, 17, 215, 170, 286, 78, 291, 310, 283, 191, 63, 248, 146, 130, 237, 326, 177, 180, 45, 219, 307, 367, 211, 346, 232, 67, 84, 263, 288, 352, 383] + 38: [282, 6, 93, 301, 378, 369, 345, 266, 203, 203, 269, 124, 284, 51, 30, 366, 226, 383, 114, 248, 333, 333, 62, 230, 171, 244, 27, 166, 328, 344, 77, 257, 53, 338, 286, 288, 341, 355, 275, 146, 227, 67, 72, 95, 362, 359, 45, 211, 57, 349, 136, 156, 368, 111, 143, 151, 5, 289, 306, 28, 76, 134, 33, 174, 15, 235, 250, 337, 334, 182, 195, 43, 56, 347, 231, 204, 311, 61, 374, 13, 55, 247, 3, 357, 240, 113, 335, 42, 259, 175, 20, 116, 281, 377, 196, 155, 249, 243, 346, 159, 339, 101, 185, 237, 10, 6, 223, 301, 301, 178, 348, 245, 66, 332, 280, 201, 372, 8, 263, 366, 189, 126, 115, 277, 277, 220, 32, 39, 379, 217, 145, 0, 212, 206, 54, 267, 108, 199, 329, 233, 106, 90, 222, 107, 271, 85, 21, 9, 193, 262, 144, 87, 255, 256, 308, 367, 34, 298, 188, 342, 228, 148, 241, 84, 197, 41, 149, 258, 103, 40, 2, 173, 302, 214, 122, 74, 273, 11, 100, 252, 47, 297, 186, 283, 138, 290, 363, 70, 316, 320, 112, 24, 163, 18, 207, 128, 22, 315, 83, 336, 330, 270, 304, 64, 161, 38, 210, 380, 4, 187, 6, 301, 213, 378, 221, 266, 118, 123, 280, 124, 372, 51, 86, 285, 322, 274, 383, 154, 194, 89, 177, 62, 63, 92, 381, 278, 317, 135, 119, 253, 97, 319, 327, 361, 125, 219, 75, 147, 239, 73, 29, 59, 224, 105, 36, 68, 323, 264, 165, 96, 179, 37, 229, 276, 265, 31, 260, 356, 376, 242, 181, 305, 131, 78, 81, 200, 238, 360, 172, 371, 261, 104, 82, 25, 295, 80, 168, 351, 183, 202, 299, 236, 164, 91, 326, 364, 314, 232, 209, 153, 218, 365, 110, 127, 162, 133, 318, 98, 176, 52, 46, 60, 158, 158, 6, 301, 287, 50, 221, 266, 94, 203, 269, 201, 284, 292, 30, 354, 189, 126, 114, 154, 309, 89, 49, 293, 130, 19, 17, 373, 279, 169, 268, 300, 234, 331, 16, 137, 225, 99, 352, 14, 324, 69, 208, 321, 150, 358, 307, 58, 88, 184, 216, 142, 7, 48, 109, 246, 117, 141, 190, 170, 26, 180, 272, 353, 139, 152, 310, 23, 296, 79, 313, 129, 382, 120, 375, 215, 44, 294, 205, 35, 71, 102, 343, 12, 192, 254, 65, 157, 350, 291, 132, 167, 303, 160, 198, 1, 140, 370, 121, 312, 340, 191, 251, 325] + 39: [43, 282, 96, 268, 264, 110, 104, 82, 223, 236, 1, 153, 56, 113, 312, 215, 93, 360, 325, 242, 155, 328, 86, 244, 174, 362, 71, 172, 295, 330, 24, 319, 77, 49, 23, 381, 298, 84, 8, 259, 191, 48, 323, 122, 278, 190, 137, 260, 126, 358, 135, 188, 144, 307, 161, 249, 14, 373, 355, 106, 167, 320, 173, 354, 275, 6, 79, 309, 68, 377, 194, 214, 376, 291, 109, 217, 288, 19, 303, 107, 337, 108, 20, 246, 0, 252, 253, 238, 233, 339, 136, 158, 67, 205, 12, 316, 356, 184, 11, 90, 334, 89, 143, 198, 100, 99, 282, 35, 66, 258, 104, 310, 348, 372, 236, 156, 151, 315, 65, 139, 93, 132, 314, 313, 271, 272, 85, 85, 80, 270, 277, 374, 40, 17, 15, 76, 208, 7, 81, 200, 227, 202, 53, 42, 140, 232, 28, 31, 263, 69, 74, 131, 152, 50, 276, 371, 5, 63, 75, 226, 134, 182, 61, 94, 289, 241, 193, 357, 4, 338, 243, 130, 220, 306, 169, 181, 120, 171, 343, 27, 335, 103, 222, 111, 55, 266, 83, 160, 211, 64, 197, 229, 342, 210, 129, 349, 125, 32, 105, 73, 332, 261, 115, 148, 21, 52, 257, 44, 176, 247, 282, 180, 66, 116, 234, 310, 372, 372, 368, 286, 56, 315, 65, 13, 38, 36, 36, 313, 267, 187, 382, 292, 329, 297, 324, 26, 98, 141, 341, 369, 3, 365, 87, 290, 273, 118, 383, 274, 127, 54, 299, 225, 239, 138, 331, 378, 367, 62, 199, 366, 322, 179, 213, 345, 33, 212, 72, 293, 45, 88, 97, 203, 128, 164, 16, 254, 240, 114, 207, 265, 91, 216, 245, 192, 287, 41, 168, 159, 78, 305, 347, 175, 18, 123, 92, 359, 219, 255, 147, 350, 256, 124, 224, 308, 57, 154, 186, 380, 304, 165, 361, 209, 247, 247, 282, 280, 66, 258, 104, 310, 310, 236, 25, 286, 56, 315, 312, 218, 38, 132, 314, 279, 119, 379, 379, 2, 145, 296, 204, 333, 121, 250, 300, 178, 163, 51, 59, 146, 235, 375, 170, 352, 248, 340, 251, 112, 336, 101, 34, 47, 353, 177, 150, 281, 317, 364, 162, 301, 302, 228, 102, 311, 95, 185, 133, 285, 230, 221, 30, 195, 363, 183, 327, 60, 166, 283, 58, 70, 196, 346, 189, 117, 46, 157, 206, 231, 142, 37, 10, 39, 284, 326, 351, 370, 294, 318, 237, 262, 269, 344, 201, 29, 321, 9, 149, 22] + 40: [31, 31, 357, 319, 319, 260, 250, 360, 70, 213, 328, 358, 175, 67, 367, 190, 21, 179, 179, 26, 215, 346, 8, 158, 369, 74, 251, 224, 59, 228, 347, 275, 362, 143, 168, 219, 274, 322, 289, 166, 30, 308, 216, 221, 382, 229, 46, 307, 316, 41, 263, 138, 244, 377, 306, 355, 155, 192, 339, 167, 371, 177, 323, 195, 37, 293, 301, 345, 73, 261, 36, 147, 270, 186, 256, 288, 125, 135, 310, 32, 140, 174, 230, 77, 35, 106, 111, 173, 352, 313, 39, 380, 157, 245, 298, 320, 191, 197, 146, 240, 144, 376, 149, 300, 31, 31, 207, 280, 183, 15, 286, 360, 142, 68, 359, 71, 175, 378, 373, 190, 21, 108, 52, 160, 202, 156, 226, 88, 64, 372, 235, 3, 97, 28, 159, 1, 303, 129, 10, 271, 209, 277, 185, 11, 193, 200, 38, 205, 148, 302, 137, 259, 309, 365, 150, 178, 65, 25, 130, 103, 294, 338, 327, 318, 356, 282, 281, 171, 19, 131, 60, 233, 139, 76, 246, 118, 116, 121, 14, 84, 348, 124, 268, 13, 16, 117, 203, 163, 242, 104, 285, 325, 154, 267, 4, 315, 45, 223, 292, 272, 127, 126, 335, 344, 5, 343, 123, 248, 31, 132, 317, 340, 183, 15, 349, 349, 142, 68, 328, 358, 55, 378, 75, 101, 265, 108, 33, 33, 107, 279, 161, 153, 254, 81, 58, 258, 351, 100, 210, 311, 284, 296, 136, 326, 341, 80, 61, 27, 54, 334, 12, 188, 370, 262, 269, 44, 187, 99, 95, 57, 189, 299, 151, 2, 165, 120, 287, 93, 333, 329, 243, 237, 273, 350, 314, 366, 85, 201, 238, 375, 78, 115, 141, 361, 23, 102, 266, 50, 374, 40, 113, 257, 218, 217, 22, 119, 114, 234, 342, 109, 381, 255, 86, 172, 20, 83, 225, 331, 353, 249, 105, 214, 31, 379, 317, 207, 319, 15, 250, 169, 181, 68, 204, 71, 175, 378, 98, 62, 283, 337, 247, 297, 128, 346, 226, 164, 232, 332, 208, 198, 90, 17, 91, 9, 363, 89, 336, 321, 162, 63, 253, 211, 145, 278, 236, 199, 196, 227, 239, 56, 79, 53, 364, 82, 43, 290, 110, 94, 7, 29, 264, 324, 206, 305, 330, 48, 304, 18, 47, 51, 222, 241, 6, 291, 122, 133, 212, 276, 0, 184, 383, 49, 368, 182, 66, 295, 34, 134, 231, 180, 354, 96, 152, 87, 176, 252, 24, 112, 220, 42, 72, 312, 69, 170, 92, 194] + 41: [205, 49, 298, 298, 367, 114, 114, 142, 27, 170, 60, 350, 212, 90, 190, 86, 369, 366, 366, 223, 159, 124, 215, 285, 187, 111, 21, 360, 347, 311, 198, 337, 104, 45, 3, 53, 42, 25, 23, 163, 308, 26, 64, 70, 54, 226, 256, 83, 294, 235, 325, 138, 339, 133, 210, 8, 63, 151, 295, 290, 81, 277, 261, 43, 173, 315, 146, 192, 15, 232, 300, 33, 143, 78, 287, 165, 259, 207, 38, 68, 98, 310, 182, 28, 265, 171, 245, 373, 201, 121, 343, 88, 177, 162, 109, 356, 304, 281, 164, 233, 22, 178, 152, 132, 305, 49, 269, 283, 335, 103, 166, 199, 376, 170, 60, 350, 374, 184, 190, 30, 369, 168, 365, 129, 275, 119, 293, 96, 7, 34, 69, 161, 280, 12, 108, 306, 348, 9, 4, 324, 264, 140, 36, 243, 217, 134, 32, 102, 125, 379, 254, 276, 156, 271, 167, 115, 144, 112, 266, 328, 363, 66, 107, 153, 18, 6, 130, 213, 244, 10, 221, 214, 41, 99, 75, 272, 352, 179, 0, 65, 141, 320, 332, 158, 147, 113, 195, 209, 14, 67, 329, 20, 139, 331, 157, 240, 211, 377, 372, 16, 58, 203, 135, 228, 227, 296, 278, 357, 122, 185, 269, 283, 378, 196, 71, 199, 251, 355, 60, 307, 374, 184, 150, 57, 197, 84, 200, 368, 159, 327, 97, 92, 79, 172, 24, 279, 206, 334, 95, 82, 383, 318, 351, 35, 239, 219, 194, 218, 336, 174, 116, 345, 241, 282, 46, 260, 258, 322, 117, 289, 250, 224, 380, 51, 131, 176, 186, 338, 91, 246, 94, 5, 371, 74, 128, 303, 59, 123, 342, 188, 236, 100, 381, 180, 297, 234, 105, 48, 257, 73, 175, 204, 375, 231, 274, 17, 62, 267, 2, 145, 189, 312, 346, 361, 169, 193, 118, 333, 40, 247, 216, 309, 321, 321, 49, 87, 378, 378, 11, 199, 142, 170, 355, 350, 374, 249, 255, 30, 273, 197, 50, 129, 275, 119, 215, 76, 136, 364, 358, 314, 302, 262, 44, 253, 80, 326, 127, 222, 286, 202, 353, 126, 252, 349, 301, 56, 313, 225, 93, 55, 237, 229, 340, 344, 242, 330, 288, 268, 299, 101, 181, 284, 19, 208, 160, 317, 61, 220, 89, 354, 37, 359, 154, 292, 1, 39, 191, 77, 72, 13, 248, 149, 316, 148, 155, 137, 29, 238, 370, 47, 183, 110, 323, 319, 270, 106, 341, 362, 382, 291, 230, 52, 263, 31, 120, 85] + 42: [2, 20, 1, 123, 123, 94, 114, 152, 31, 117, 106, 303, 172, 155, 247, 93, 215, 282, 111, 158, 138, 37, 222, 349, 200, 377, 48, 209, 281, 319, 351, 277, 358, 248, 4, 341, 135, 239, 187, 287, 165, 201, 338, 36, 181, 38, 143, 366, 362, 76, 149, 381, 245, 18, 101, 137, 311, 372, 246, 89, 260, 130, 97, 226, 205, 7, 196, 332, 156, 170, 300, 346, 293, 84, 175, 284, 267, 321, 10, 363, 185, 292, 186, 136, 330, 302, 85, 133, 115, 276, 70, 79, 83, 81, 78, 142, 13, 240, 361, 289, 27, 121, 25, 71, 188, 359, 217, 123, 123, 94, 228, 152, 31, 117, 357, 305, 172, 211, 218, 350, 151, 63, 264, 256, 138, 179, 168, 337, 220, 129, 263, 255, 189, 21, 373, 379, 131, 370, 320, 178, 140, 236, 344, 230, 87, 161, 147, 216, 322, 314, 234, 45, 118, 39, 160, 192, 80, 60, 19, 146, 231, 371, 225, 238, 283, 139, 183, 310, 298, 258, 5, 291, 53, 270, 171, 233, 194, 343, 290, 229, 102, 98, 250, 73, 128, 296, 198, 304, 144, 383, 88, 153, 353, 12, 47, 57, 107, 159, 177, 317, 15, 365, 24, 40, 352, 294, 95, 254, 210, 359, 217, 123, 123, 266, 295, 152, 119, 117, 268, 345, 176, 184, 116, 218, 93, 325, 111, 158, 333, 41, 243, 195, 55, 108, 59, 203, 327, 122, 265, 348, 275, 141, 30, 29, 224, 169, 67, 214, 105, 173, 103, 308, 335, 77, 202, 316, 223, 257, 227, 380, 32, 241, 51, 8, 237, 180, 64, 9, 315, 109, 157, 299, 58, 313, 16, 204, 378, 125, 269, 112, 26, 56, 272, 206, 199, 197, 368, 104, 312, 3, 23, 334, 207, 367, 22, 134, 69, 190, 11, 110, 44, 54, 14, 43, 0, 91, 163, 273, 126, 242, 376, 339, 154, 375, 50, 42, 123, 266, 148, 152, 31, 117, 268, 345, 176, 155, 212, 279, 325, 72, 150, 256, 333, 37, 374, 162, 262, 306, 6, 208, 342, 127, 191, 235, 328, 274, 354, 221, 259, 286, 28, 17, 167, 329, 326, 356, 92, 164, 280, 33, 52, 166, 65, 297, 113, 340, 132, 301, 62, 307, 219, 323, 49, 244, 99, 232, 34, 61, 336, 251, 355, 68, 278, 364, 120, 213, 261, 318, 82, 90, 145, 360, 285, 124, 75, 288, 174, 309, 347, 96, 253, 35, 249, 100, 331, 182, 271, 382, 86, 66, 74, 252, 46, 193, 369, 324] + 43: [164, 82, 168, 221, 171, 156, 225, 375, 278, 92, 335, 197, 35, 98, 59, 324, 14, 20, 190, 239, 30, 103, 298, 351, 57, 326, 93, 81, 223, 301, 360, 227, 198, 307, 136, 121, 89, 368, 191, 32, 205, 52, 246, 48, 83, 277, 140, 135, 296, 261, 257, 79, 28, 155, 243, 148, 102, 185, 229, 183, 143, 16, 70, 213, 220, 309, 343, 226, 18, 31, 292, 11, 356, 8, 65, 126, 193, 381, 268, 209, 188, 265, 316, 258, 162, 312, 248, 199, 41, 0, 137, 303, 208, 352, 58, 139, 244, 319, 231, 310, 195, 373, 95, 354, 369, 262, 168, 161, 171, 332, 347, 225, 68, 372, 17, 142, 66, 112, 98, 37, 349, 283, 254, 252, 30, 94, 43, 351, 23, 235, 377, 186, 111, 127, 323, 313, 108, 274, 269, 321, 85, 374, 150, 2, 238, 90, 346, 204, 86, 219, 200, 26, 149, 5, 365, 67, 196, 194, 345, 145, 99, 60, 50, 165, 45, 253, 120, 46, 263, 9, 80, 288, 175, 159, 207, 72, 144, 267, 353, 163, 19, 158, 173, 109, 320, 300, 110, 336, 285, 51, 69, 124, 7, 172, 141, 281, 42, 270, 133, 96, 212, 33, 166, 376, 192, 182, 382, 3, 27, 315, 168, 221, 105, 156, 225, 375, 375, 372, 17, 371, 66, 359, 98, 216, 14, 13, 190, 75, 366, 297, 43, 131, 23, 152, 291, 55, 138, 342, 203, 318, 224, 338, 202, 280, 379, 4, 77, 222, 378, 154, 380, 275, 286, 234, 123, 271, 160, 34, 61, 130, 241, 334, 273, 12, 242, 293, 38, 47, 210, 266, 284, 167, 128, 146, 256, 325, 170, 116, 304, 64, 294, 71, 117, 87, 151, 287, 233, 119, 330, 355, 187, 357, 311, 201, 53, 114, 317, 328, 21, 228, 362, 340, 333, 104, 25, 289, 184, 54, 189, 331, 44, 348, 344, 315, 181, 6, 105, 177, 347, 308, 218, 92, 335, 129, 66, 359, 364, 324, 39, 283, 75, 97, 100, 297, 298, 314, 22, 363, 337, 153, 29, 322, 264, 178, 1, 305, 147, 78, 250, 255, 272, 73, 232, 282, 88, 327, 339, 306, 115, 299, 367, 36, 84, 157, 237, 10, 63, 107, 49, 24, 113, 180, 217, 56, 251, 74, 125, 101, 174, 370, 215, 295, 329, 122, 259, 279, 76, 132, 247, 118, 134, 358, 211, 290, 240, 179, 260, 341, 276, 230, 106, 62, 249, 236, 383, 40, 15, 91, 176, 350, 302, 361, 169, 206, 214, 245] + 44: [362, 97, 172, 0, 213, 77, 300, 191, 199, 10, 10, 351, 226, 348, 155, 60, 180, 215, 376, 43, 277, 346, 279, 325, 11, 166, 320, 86, 333, 242, 294, 198, 33, 228, 275, 337, 343, 276, 265, 138, 266, 85, 175, 186, 122, 55, 100, 56, 352, 132, 366, 160, 202, 96, 261, 381, 128, 292, 220, 344, 92, 278, 74, 282, 149, 8, 231, 377, 23, 32, 20, 113, 263, 203, 319, 18, 53, 256, 137, 65, 35, 312, 200, 326, 153, 5, 251, 210, 283, 28, 69, 268, 139, 356, 335, 111, 76, 133, 208, 281, 295, 129, 206, 322, 362, 64, 172, 167, 213, 77, 301, 191, 199, 382, 10, 110, 317, 348, 223, 284, 21, 188, 257, 29, 328, 346, 368, 229, 236, 93, 146, 383, 289, 249, 38, 306, 358, 102, 217, 95, 324, 89, 121, 262, 244, 192, 61, 296, 154, 173, 364, 70, 285, 130, 248, 286, 54, 174, 369, 134, 157, 342, 41, 26, 196, 370, 165, 159, 214, 105, 323, 309, 156, 81, 305, 237, 234, 57, 250, 345, 367, 49, 51, 232, 273, 42, 45, 216, 239, 373, 115, 99, 246, 363, 176, 302, 47, 293, 73, 361, 58, 88, 75, 371, 209, 13, 332, 374, 274, 168, 171, 258, 213, 77, 178, 301, 199, 222, 379, 317, 226, 101, 101, 284, 190, 267, 84, 145, 170, 252, 135, 311, 30, 143, 313, 280, 182, 118, 264, 142, 365, 304, 225, 360, 158, 14, 340, 117, 184, 224, 24, 299, 349, 150, 201, 315, 66, 131, 164, 380, 269, 87, 221, 177, 36, 162, 354, 126, 147, 83, 90, 40, 205, 59, 318, 19, 336, 211, 104, 179, 235, 230, 9, 109, 314, 321, 183, 114, 255, 2, 253, 375, 233, 22, 119, 50, 240, 298, 103, 330, 259, 37, 152, 52, 353, 197, 310, 308, 112, 339, 12, 106, 274, 238, 171, 17, 213, 25, 272, 301, 191, 222, 379, 317, 226, 348, 223, 271, 21, 227, 82, 145, 170, 68, 140, 331, 288, 34, 98, 187, 212, 141, 63, 329, 48, 144, 125, 291, 355, 94, 207, 91, 6, 350, 254, 290, 163, 219, 287, 39, 71, 79, 338, 378, 67, 359, 120, 44, 327, 107, 46, 136, 15, 108, 185, 334, 270, 243, 78, 347, 341, 169, 3, 218, 241, 72, 193, 357, 4, 204, 62, 7, 124, 245, 80, 1, 27, 195, 151, 194, 247, 372, 260, 307, 123, 297, 127, 31, 316, 148, 181, 16, 161, 189, 303, 116] + 45: [154, 173, 219, 140, 188, 92, 110, 293, 25, 334, 178, 182, 8, 27, 171, 247, 33, 369, 262, 103, 276, 180, 131, 21, 165, 124, 348, 349, 209, 183, 275, 228, 239, 168, 196, 16, 115, 30, 338, 250, 84, 313, 7, 267, 41, 242, 157, 260, 26, 315, 236, 24, 302, 253, 222, 370, 76, 65, 34, 170, 290, 252, 296, 264, 283, 273, 333, 231, 263, 177, 288, 314, 105, 0, 289, 312, 147, 3, 215, 93, 136, 238, 332, 189, 18, 89, 186, 232, 270, 194, 295, 148, 13, 6, 28, 184, 130, 49, 69, 72, 137, 336, 266, 221, 154, 121, 121, 140, 134, 92, 161, 293, 42, 178, 201, 10, 10, 27, 176, 227, 227, 350, 367, 373, 354, 47, 329, 61, 163, 212, 114, 383, 107, 248, 351, 118, 198, 223, 45, 122, 229, 104, 245, 308, 12, 112, 306, 357, 213, 249, 44, 179, 292, 67, 109, 2, 261, 303, 340, 43, 58, 330, 116, 126, 87, 319, 59, 5, 98, 294, 379, 255, 151, 342, 374, 79, 166, 363, 48, 203, 364, 113, 343, 202, 225, 337, 90, 102, 62, 31, 82, 167, 159, 193, 352, 216, 259, 77, 99, 325, 149, 240, 181, 17, 244, 361, 300, 347, 324, 191, 318, 140, 316, 92, 92, 284, 42, 334, 201, 346, 8, 120, 356, 227, 375, 350, 160, 373, 378, 329, 298, 164, 95, 81, 297, 128, 224, 234, 331, 39, 187, 307, 55, 158, 51, 155, 88, 70, 54, 327, 251, 80, 360, 108, 281, 217, 106, 175, 208, 96, 4, 22, 63, 218, 20, 35, 133, 230, 304, 145, 291, 38, 37, 29, 322, 241, 169, 381, 271, 204, 246, 341, 15, 309, 78, 368, 377, 71, 243, 214, 321, 125, 280, 56, 152, 226, 206, 305, 265, 53, 278, 335, 75, 237, 60, 235, 299, 40, 355, 192, 83, 174, 23, 23, 301, 233, 144, 92, 161, 317, 139, 199, 178, 197, 10, 274, 141, 339, 33, 367, 146, 85, 276, 286, 269, 164, 95, 81, 383, 272, 52, 380, 258, 254, 359, 310, 162, 326, 36, 277, 74, 372, 328, 68, 210, 135, 345, 86, 156, 94, 32, 279, 365, 1, 123, 376, 257, 50, 344, 19, 207, 205, 46, 256, 358, 320, 150, 9, 282, 211, 57, 127, 129, 285, 142, 220, 119, 268, 371, 190, 362, 73, 91, 111, 323, 132, 153, 382, 97, 143, 353, 200, 101, 366, 311, 287, 172, 66, 11, 195, 185, 14, 117, 138, 64, 100] + 46: [152, 210, 10, 183, 352, 233, 256, 314, 286, 327, 253, 343, 170, 110, 51, 191, 149, 230, 332, 106, 106, 140, 344, 312, 271, 136, 254, 55, 345, 2, 374, 3, 102, 39, 351, 228, 130, 195, 22, 227, 80, 232, 121, 168, 180, 109, 4, 13, 129, 362, 53, 273, 172, 216, 157, 158, 45, 57, 226, 373, 328, 97, 360, 24, 297, 223, 346, 69, 9, 317, 322, 333, 145, 264, 190, 370, 184, 244, 339, 26, 347, 163, 65, 43, 77, 310, 281, 72, 87, 381, 48, 174, 212, 29, 185, 0, 100, 28, 148, 270, 220, 379, 61, 122, 188, 133, 247, 247, 34, 352, 139, 58, 224, 380, 253, 343, 249, 110, 207, 202, 274, 367, 354, 284, 104, 66, 335, 335, 194, 187, 204, 114, 113, 44, 76, 219, 350, 215, 211, 62, 108, 143, 376, 260, 186, 99, 353, 197, 295, 192, 316, 166, 63, 41, 95, 128, 292, 237, 340, 321, 336, 383, 81, 308, 206, 92, 289, 255, 313, 222, 173, 382, 7, 243, 277, 241, 359, 91, 291, 309, 285, 278, 98, 257, 150, 52, 135, 75, 258, 11, 366, 236, 363, 105, 269, 251, 296, 179, 47, 123, 70, 357, 225, 365, 252, 27, 88, 213, 142, 142, 201, 183, 34, 19, 64, 58, 368, 240, 46, 355, 311, 16, 96, 239, 246, 230, 354, 324, 319, 319, 140, 301, 214, 40, 68, 378, 338, 199, 242, 320, 79, 282, 49, 131, 115, 33, 155, 372, 287, 334, 218, 315, 259, 137, 37, 50, 176, 23, 162, 250, 198, 267, 235, 193, 125, 377, 32, 307, 141, 78, 144, 169, 299, 161, 318, 348, 159, 293, 265, 342, 261, 283, 217, 93, 361, 305, 234, 375, 337, 15, 112, 21, 127, 330, 341, 126, 288, 73, 196, 181, 279, 116, 276, 84, 151, 167, 178, 323, 231, 5, 36, 175, 142, 133, 10, 183, 352, 64, 139, 256, 286, 380, 355, 311, 311, 25, 207, 238, 149, 221, 367, 6, 209, 66, 154, 12, 124, 60, 38, 165, 18, 303, 146, 203, 266, 85, 30, 280, 8, 117, 120, 83, 326, 132, 245, 118, 156, 42, 369, 298, 164, 304, 294, 272, 205, 103, 17, 59, 268, 111, 101, 329, 263, 35, 160, 290, 89, 119, 189, 56, 71, 74, 153, 20, 364, 177, 90, 331, 86, 262, 171, 371, 208, 138, 67, 358, 275, 94, 229, 31, 300, 107, 1, 134, 306, 248, 54, 349, 200, 14, 182, 82, 147, 325, 302, 356] + 47: [195, 75, 271, 179, 154, 154, 44, 352, 24, 312, 3, 3, 279, 109, 1, 61, 227, 307, 359, 134, 286, 320, 316, 353, 362, 129, 267, 287, 59, 123, 229, 218, 114, 212, 330, 230, 155, 26, 135, 377, 124, 223, 324, 41, 323, 184, 364, 203, 296, 51, 355, 25, 180, 173, 252, 53, 315, 378, 336, 91, 70, 121, 289, 213, 281, 292, 306, 111, 115, 185, 186, 132, 297, 160, 46, 242, 48, 152, 302, 72, 177, 74, 350, 304, 16, 100, 38, 222, 47, 145, 11, 194, 82, 167, 125, 256, 66, 261, 128, 318, 239, 373, 293, 56, 195, 343, 54, 347, 181, 171, 162, 352, 291, 312, 22, 43, 35, 87, 158, 158, 382, 382, 333, 345, 137, 305, 215, 127, 36, 175, 168, 21, 188, 327, 153, 349, 251, 176, 351, 250, 84, 69, 219, 68, 166, 151, 263, 284, 172, 65, 199, 73, 12, 322, 17, 148, 4, 354, 117, 27, 96, 139, 141, 80, 321, 126, 303, 147, 273, 94, 278, 60, 107, 32, 157, 260, 144, 81, 190, 150, 19, 189, 308, 325, 245, 358, 380, 174, 133, 30, 317, 372, 202, 270, 299, 268, 78, 161, 206, 221, 241, 182, 113, 311, 236, 191, 246, 360, 195, 343, 54, 331, 181, 143, 162, 44, 357, 24, 22, 371, 375, 170, 34, 29, 280, 359, 300, 309, 314, 14, 294, 196, 198, 274, 376, 110, 238, 131, 367, 235, 159, 122, 313, 265, 95, 93, 301, 42, 224, 285, 231, 103, 33, 370, 50, 197, 0, 205, 138, 237, 340, 233, 112, 164, 234, 183, 356, 15, 243, 71, 335, 225, 276, 328, 37, 258, 341, 200, 310, 248, 97, 253, 9, 102, 365, 282, 57, 216, 379, 207, 5, 92, 339, 105, 178, 277, 76, 247, 64, 244, 99, 283, 165, 295, 254, 272, 381, 7, 156, 62, 209, 288, 195, 343, 54, 187, 136, 192, 44, 352, 24, 312, 3, 3, 119, 87, 87, 29, 193, 307, 300, 134, 137, 90, 368, 257, 344, 79, 240, 118, 40, 346, 342, 363, 63, 98, 226, 269, 208, 104, 262, 67, 89, 249, 228, 319, 298, 140, 255, 77, 204, 101, 338, 45, 39, 52, 106, 210, 23, 366, 18, 201, 348, 211, 163, 28, 259, 337, 332, 31, 49, 329, 2, 20, 130, 334, 10, 149, 120, 275, 55, 266, 383, 290, 374, 108, 58, 86, 220, 361, 85, 88, 142, 264, 116, 232, 6, 217, 8, 13, 83, 169, 214, 146, 369, 326] + 48: [237, 51, 283, 266, 17, 271, 102, 145, 140, 328, 330, 273, 239, 99, 132, 343, 203, 284, 182, 101, 58, 141, 85, 78, 62, 15, 287, 6, 344, 235, 359, 363, 186, 342, 134, 252, 155, 290, 210, 300, 161, 175, 192, 50, 120, 247, 88, 340, 329, 341, 212, 24, 168, 172, 42, 231, 307, 22, 80, 351, 119, 96, 302, 81, 198, 269, 367, 40, 206, 346, 107, 137, 108, 82, 272, 173, 366, 214, 69, 373, 224, 37, 87, 278, 244, 350, 361, 225, 298, 362, 282, 125, 46, 183, 262, 176, 220, 265, 355, 336, 136, 304, 114, 250, 267, 288, 76, 20, 17, 271, 102, 117, 150, 328, 109, 358, 239, 276, 19, 236, 52, 311, 353, 295, 164, 381, 381, 78, 62, 207, 104, 375, 5, 139, 48, 254, 142, 115, 181, 305, 320, 371, 229, 312, 322, 7, 25, 257, 370, 112, 190, 0, 160, 255, 321, 357, 315, 347, 79, 13, 379, 202, 113, 281, 72, 289, 97, 11, 84, 43, 349, 16, 151, 292, 94, 274, 261, 39, 187, 47, 217, 92, 67, 313, 27, 163, 294, 105, 110, 309, 188, 191, 8, 338, 60, 106, 296, 248, 23, 383, 103, 337, 199, 258, 30, 245, 196, 227, 267, 9, 283, 249, 17, 128, 26, 77, 279, 140, 328, 215, 116, 276, 234, 44, 52, 311, 2, 101, 133, 141, 85, 28, 15, 263, 166, 189, 177, 240, 38, 264, 34, 316, 184, 354, 319, 378, 222, 156, 130, 246, 299, 152, 285, 75, 45, 73, 89, 118, 242, 111, 4, 318, 91, 360, 180, 339, 241, 14, 138, 146, 36, 253, 211, 280, 12, 53, 55, 144, 35, 259, 243, 178, 71, 286, 95, 194, 93, 3, 56, 143, 277, 335, 66, 228, 356, 148, 154, 86, 345, 153, 124, 251, 54, 122, 303, 364, 131, 348, 10, 200, 129, 308, 159, 297, 76, 266, 271, 128, 270, 65, 301, 140, 330, 273, 226, 132, 132, 121, 74, 284, 353, 90, 33, 268, 21, 83, 157, 263, 369, 147, 223, 327, 208, 372, 325, 185, 380, 41, 232, 368, 205, 219, 193, 179, 293, 291, 233, 174, 123, 324, 162, 170, 31, 221, 230, 275, 218, 57, 317, 64, 100, 70, 49, 158, 204, 63, 374, 352, 18, 127, 98, 169, 59, 195, 382, 216, 256, 32, 135, 310, 326, 29, 165, 323, 213, 61, 201, 238, 376, 149, 171, 331, 377, 260, 365, 332, 68, 126, 306, 197, 333, 167, 1, 209, 334, 314] + 49: [374, 360, 302, 302, 321, 272, 294, 346, 15, 209, 297, 305, 19, 158, 157, 126, 313, 66, 212, 278, 261, 201, 9, 276, 262, 193, 199, 218, 286, 289, 246, 269, 120, 186, 8, 264, 314, 94, 5, 48, 366, 73, 1, 362, 28, 80, 245, 70, 140, 318, 170, 112, 34, 4, 163, 146, 225, 216, 371, 238, 33, 35, 62, 309, 359, 244, 339, 47, 85, 378, 119, 61, 46, 2, 124, 285, 190, 361, 22, 52, 364, 324, 168, 111, 166, 51, 7, 229, 367, 102, 18, 231, 379, 281, 215, 383, 203, 258, 129, 347, 68, 135, 38, 64, 374, 360, 302, 302, 321, 272, 319, 6, 75, 155, 65, 234, 19, 181, 293, 126, 126, 353, 105, 25, 270, 153, 9, 275, 44, 177, 251, 380, 76, 260, 306, 187, 312, 331, 255, 183, 298, 109, 235, 58, 205, 254, 336, 340, 137, 54, 86, 223, 138, 198, 290, 224, 323, 222, 77, 81, 301, 303, 295, 327, 352, 36, 98, 338, 103, 345, 14, 211, 176, 69, 71, 0, 12, 31, 84, 192, 101, 283, 108, 299, 125, 185, 104, 357, 377, 13, 147, 165, 363, 162, 334, 161, 99, 152, 207, 291, 247, 288, 3, 50, 24, 92, 375, 191, 79, 210, 302, 197, 321, 37, 6, 268, 304, 257, 297, 59, 136, 157, 253, 253, 313, 202, 78, 271, 110, 90, 335, 242, 39, 259, 325, 292, 107, 91, 220, 344, 329, 45, 30, 100, 95, 164, 122, 144, 328, 169, 174, 63, 382, 16, 368, 311, 196, 277, 128, 29, 160, 127, 248, 376, 55, 139, 173, 308, 67, 43, 134, 227, 156, 142, 239, 320, 280, 226, 358, 118, 106, 204, 206, 282, 342, 113, 154, 354, 348, 326, 274, 316, 27, 315, 87, 341, 237, 159, 287, 265, 171, 350, 217, 11, 230, 369, 219, 373, 349, 322, 337, 123, 79, 233, 302, 184, 321, 149, 83, 268, 75, 155, 297, 305, 150, 115, 253, 213, 116, 202, 241, 271, 40, 208, 180, 242, 148, 193, 60, 251, 133, 243, 132, 74, 194, 381, 175, 332, 131, 333, 195, 121, 250, 97, 279, 17, 310, 221, 23, 56, 240, 365, 214, 89, 189, 41, 296, 20, 273, 82, 10, 179, 267, 330, 178, 307, 351, 172, 114, 72, 53, 117, 130, 355, 200, 49, 151, 284, 228, 143, 256, 145, 182, 88, 232, 343, 57, 93, 167, 372, 42, 96, 266, 21, 236, 249, 141, 263, 252, 32, 26, 370, 188, 300, 317, 356] + 50: [57, 86, 268, 100, 109, 140, 49, 67, 97, 32, 32, 233, 215, 215, 69, 273, 321, 146, 56, 56, 9, 26, 1, 3, 128, 128, 327, 0, 341, 114, 331, 155, 224, 270, 335, 16, 353, 122, 164, 313, 239, 48, 186, 375, 194, 116, 134, 379, 206, 319, 180, 250, 310, 74, 378, 81, 152, 20, 242, 205, 211, 28, 251, 300, 135, 369, 127, 202, 281, 344, 123, 106, 34, 200, 372, 141, 212, 355, 258, 36, 138, 112, 286, 324, 101, 10, 83, 201, 173, 190, 72, 31, 142, 279, 167, 221, 302, 118, 66, 364, 222, 257, 111, 345, 33, 86, 268, 100, 299, 137, 49, 229, 333, 376, 315, 210, 188, 262, 69, 325, 321, 214, 214, 27, 350, 282, 18, 129, 231, 62, 45, 220, 334, 256, 314, 343, 253, 15, 121, 187, 274, 24, 301, 244, 306, 228, 195, 165, 348, 183, 219, 102, 189, 169, 276, 208, 267, 85, 73, 107, 277, 126, 255, 99, 105, 172, 278, 5, 37, 243, 362, 337, 289, 153, 42, 170, 198, 305, 12, 46, 68, 383, 90, 148, 162, 179, 291, 346, 30, 381, 176, 246, 207, 328, 265, 104, 261, 54, 144, 225, 317, 59, 131, 44, 283, 264, 136, 365, 80, 86, 25, 226, 316, 109, 181, 166, 38, 376, 284, 352, 370, 218, 35, 325, 125, 366, 312, 175, 377, 350, 18, 129, 231, 338, 192, 199, 193, 88, 22, 51, 184, 271, 213, 298, 132, 363, 196, 275, 380, 58, 19, 75, 263, 290, 240, 43, 247, 269, 174, 94, 371, 151, 92, 64, 294, 160, 78, 145, 340, 185, 108, 143, 236, 17, 297, 292, 204, 339, 177, 351, 349, 308, 156, 238, 237, 89, 318, 171, 65, 285, 82, 147, 77, 209, 61, 76, 157, 103, 70, 216, 266, 120, 161, 227, 52, 254, 307, 84, 332, 50, 55, 230, 29, 29, 235, 268, 316, 311, 181, 166, 14, 367, 315, 210, 188, 361, 232, 232, 110, 373, 287, 27, 280, 26, 1, 3, 304, 329, 296, 217, 358, 119, 295, 79, 272, 130, 159, 249, 133, 6, 41, 96, 71, 149, 117, 203, 154, 168, 359, 53, 252, 342, 197, 139, 382, 234, 326, 303, 163, 322, 2, 91, 357, 223, 113, 356, 178, 347, 39, 191, 4, 288, 124, 259, 7, 63, 320, 248, 330, 150, 354, 293, 98, 323, 11, 87, 40, 60, 13, 21, 47, 8, 360, 245, 95, 158, 309, 260, 115, 368, 336, 182, 23, 241, 93, 374] + 51: [322, 375, 221, 189, 372, 210, 110, 178, 24, 111, 186, 357, 123, 113, 284, 17, 228, 176, 64, 72, 67, 211, 229, 65, 135, 270, 40, 124, 188, 105, 295, 168, 269, 130, 263, 5, 13, 46, 118, 85, 183, 47, 382, 81, 74, 142, 359, 162, 360, 209, 351, 310, 204, 119, 309, 166, 50, 75, 48, 239, 320, 364, 171, 84, 196, 94, 349, 45, 51, 223, 338, 288, 145, 78, 363, 159, 58, 109, 112, 240, 218, 116, 148, 83, 354, 296, 88, 165, 174, 77, 217, 133, 293, 302, 352, 117, 36, 347, 377, 173, 121, 151, 249, 261, 108, 373, 106, 241, 371, 372, 6, 200, 323, 24, 186, 208, 12, 193, 102, 52, 180, 122, 337, 264, 82, 126, 229, 343, 286, 286, 260, 235, 330, 129, 146, 177, 243, 11, 314, 26, 299, 216, 374, 23, 285, 245, 275, 308, 345, 86, 268, 279, 42, 366, 280, 79, 34, 91, 160, 252, 10, 195, 236, 224, 306, 69, 289, 313, 100, 332, 334, 297, 350, 27, 114, 194, 276, 278, 333, 55, 61, 336, 316, 25, 125, 149, 1, 96, 9, 7, 257, 199, 277, 120, 376, 227, 71, 187, 63, 318, 225, 38, 131, 21, 226, 139, 291, 326, 31, 137, 221, 73, 37, 210, 110, 206, 98, 24, 16, 186, 208, 287, 102, 17, 342, 127, 337, 144, 358, 126, 163, 62, 87, 104, 203, 301, 250, 233, 283, 253, 103, 49, 335, 92, 8, 35, 294, 281, 292, 271, 161, 303, 254, 237, 198, 380, 219, 222, 181, 192, 207, 248, 321, 368, 44, 362, 41, 164, 265, 324, 311, 369, 383, 258, 365, 315, 89, 170, 191, 30, 242, 298, 93, 353, 339, 378, 43, 232, 19, 15, 214, 230, 28, 39, 272, 201, 305, 128, 367, 70, 169, 185, 274, 147, 150, 231, 256, 76, 255, 2, 290, 348, 137, 373, 375, 241, 37, 372, 101, 200, 323, 24, 186, 99, 12, 287, 102, 182, 342, 356, 136, 144, 82, 300, 163, 140, 152, 60, 32, 95, 325, 304, 153, 282, 246, 346, 134, 247, 259, 143, 20, 220, 331, 212, 344, 97, 172, 341, 56, 107, 18, 329, 190, 59, 90, 370, 251, 33, 141, 3, 361, 234, 157, 205, 262, 317, 213, 115, 312, 381, 379, 54, 328, 80, 307, 319, 273, 340, 197, 53, 238, 167, 184, 68, 66, 57, 4, 215, 14, 158, 154, 0, 132, 355, 175, 179, 266, 29, 22, 244, 267, 155, 327, 138, 156, 202] + 52: [52, 102, 332, 107, 372, 381, 371, 99, 182, 334, 157, 71, 362, 104, 165, 97, 85, 336, 239, 34, 211, 48, 82, 184, 78, 205, 346, 62, 245, 140, 56, 269, 355, 349, 149, 354, 257, 325, 312, 133, 339, 222, 234, 191, 178, 359, 30, 23, 364, 75, 243, 369, 113, 361, 53, 115, 22, 213, 324, 208, 125, 109, 161, 225, 270, 237, 200, 177, 294, 282, 152, 207, 220, 230, 136, 249, 143, 28, 310, 322, 226, 340, 4, 96, 353, 135, 232, 276, 297, 329, 314, 203, 159, 65, 122, 142, 317, 14, 295, 272, 204, 370, 180, 247, 186, 102, 231, 107, 107, 381, 320, 99, 63, 334, 157, 71, 250, 104, 165, 73, 214, 239, 321, 174, 169, 193, 163, 90, 7, 205, 190, 33, 260, 80, 196, 192, 266, 54, 95, 50, 291, 88, 228, 379, 127, 378, 262, 5, 83, 315, 215, 160, 233, 248, 6, 300, 153, 72, 331, 351, 279, 316, 55, 287, 253, 326, 268, 235, 352, 57, 242, 258, 365, 20, 21, 194, 51, 47, 330, 347, 303, 101, 380, 118, 344, 18, 126, 311, 358, 138, 274, 363, 9, 36, 185, 42, 273, 265, 338, 144, 373, 254, 376, 360, 162, 16, 305, 375, 327, 102, 120, 61, 283, 381, 99, 99, 130, 334, 151, 71, 103, 104, 97, 137, 289, 290, 195, 206, 209, 209, 77, 25, 13, 41, 172, 68, 357, 44, 259, 219, 87, 32, 8, 236, 309, 35, 155, 171, 74, 221, 286, 105, 341, 281, 173, 223, 367, 134, 146, 67, 301, 328, 154, 377, 374, 170, 66, 337, 40, 306, 119, 93, 89, 2, 110, 277, 224, 244, 238, 189, 139, 179, 100, 198, 49, 356, 278, 197, 111, 114, 335, 343, 229, 304, 302, 275, 158, 261, 255, 131, 12, 19, 15, 0, 24, 240, 164, 264, 106, 212, 38, 176, 121, 92, 307, 216, 283, 217, 381, 267, 31, 334, 157, 246, 103, 165, 97, 137, 323, 336, 271, 345, 263, 48, 298, 25, 183, 382, 288, 60, 10, 26, 128, 168, 210, 252, 64, 98, 70, 37, 123, 108, 256, 285, 117, 166, 296, 227, 124, 167, 29, 116, 280, 292, 3, 76, 17, 319, 39, 69, 333, 91, 45, 27, 308, 350, 58, 201, 383, 284, 148, 59, 84, 251, 318, 150, 368, 129, 188, 156, 141, 342, 43, 81, 299, 313, 181, 293, 112, 147, 132, 145, 187, 79, 11, 202, 366, 241, 1, 86, 94, 199, 218, 46, 175, 348] + 53: [230, 240, 110, 110, 297, 222, 13, 73, 23, 287, 129, 296, 16, 142, 292, 365, 273, 295, 250, 330, 32, 4, 186, 132, 145, 279, 86, 291, 376, 225, 130, 221, 31, 50, 326, 95, 241, 168, 256, 262, 381, 341, 112, 77, 152, 335, 300, 52, 235, 47, 281, 159, 44, 293, 67, 328, 340, 35, 196, 178, 375, 276, 189, 370, 236, 215, 255, 358, 364, 220, 232, 97, 0, 54, 201, 108, 137, 131, 380, 323, 29, 82, 51, 39, 193, 64, 184, 163, 208, 267, 356, 91, 192, 181, 213, 33, 355, 99, 223, 307, 22, 18, 312, 210, 336, 68, 240, 110, 110, 203, 13, 73, 350, 334, 320, 296, 164, 109, 46, 354, 311, 347, 250, 330, 42, 118, 228, 170, 160, 6, 202, 157, 162, 332, 103, 119, 125, 284, 24, 127, 5, 3, 94, 12, 70, 361, 383, 205, 172, 313, 49, 277, 176, 259, 89, 72, 122, 175, 143, 146, 36, 128, 120, 2, 174, 166, 57, 327, 19, 30, 333, 133, 299, 251, 242, 55, 258, 337, 21, 352, 14, 246, 305, 43, 331, 83, 85, 342, 74, 17, 113, 34, 93, 382, 148, 211, 212, 98, 165, 187, 304, 71, 58, 290, 265, 153, 167, 114, 308, 8, 240, 110, 110, 66, 317, 247, 23, 287, 38, 38, 16, 109, 280, 311, 214, 295, 224, 179, 155, 32, 186, 266, 156, 209, 309, 194, 237, 26, 81, 285, 368, 20, 253, 319, 219, 200, 329, 139, 346, 338, 183, 188, 37, 239, 378, 197, 60, 149, 15, 126, 353, 282, 362, 357, 369, 248, 216, 206, 65, 105, 371, 96, 306, 117, 79, 40, 138, 182, 294, 101, 195, 87, 257, 7, 53, 80, 264, 9, 100, 177, 310, 1, 147, 301, 229, 303, 374, 260, 263, 288, 27, 343, 286, 204, 11, 344, 244, 314, 252, 140, 316, 272, 217, 75, 110, 110, 243, 317, 270, 73, 350, 334, 238, 38, 16, 173, 322, 354, 347, 363, 345, 379, 227, 4, 92, 302, 180, 360, 268, 351, 261, 144, 271, 121, 134, 107, 269, 171, 325, 88, 62, 25, 69, 141, 231, 116, 151, 324, 78, 373, 10, 150, 349, 76, 315, 102, 359, 226, 233, 111, 115, 339, 169, 321, 158, 207, 278, 28, 136, 123, 198, 218, 59, 254, 45, 275, 234, 154, 367, 289, 283, 249, 199, 48, 63, 298, 348, 106, 366, 372, 104, 135, 161, 41, 190, 61, 124, 274, 191, 318, 245, 377, 56, 90, 185, 84] + 54: [295, 337, 380, 363, 363, 71, 338, 147, 258, 356, 356, 155, 216, 164, 164, 80, 250, 38, 20, 299, 53, 95, 72, 176, 93, 221, 87, 119, 309, 314, 86, 16, 27, 121, 288, 41, 47, 218, 129, 322, 84, 124, 91, 5, 190, 272, 182, 24, 54, 381, 204, 237, 55, 368, 100, 67, 61, 144, 69, 68, 293, 111, 136, 201, 62, 81, 162, 18, 166, 300, 128, 266, 194, 229, 334, 213, 308, 234, 74, 178, 29, 249, 261, 52, 89, 263, 203, 56, 270, 116, 303, 210, 134, 169, 188, 222, 165, 349, 19, 200, 106, 354, 361, 378, 370, 370, 363, 363, 253, 294, 319, 42, 258, 11, 339, 339, 155, 257, 82, 214, 133, 348, 280, 230, 172, 306, 95, 208, 44, 345, 305, 25, 26, 118, 75, 329, 181, 332, 276, 76, 115, 196, 161, 215, 357, 50, 323, 94, 185, 152, 142, 109, 239, 184, 376, 105, 355, 131, 49, 1, 278, 264, 373, 85, 163, 122, 209, 304, 364, 353, 173, 195, 262, 23, 275, 141, 238, 251, 12, 63, 110, 346, 145, 14, 211, 289, 331, 268, 343, 248, 46, 226, 149, 360, 51, 65, 350, 138, 112, 256, 297, 193, 344, 358, 127, 160, 148, 232, 370, 284, 153, 363, 294, 283, 328, 90, 42, 11, 287, 223, 231, 231, 260, 240, 240, 348, 280, 230, 8, 286, 324, 341, 120, 290, 57, 352, 40, 180, 3, 228, 58, 156, 132, 143, 99, 310, 197, 158, 277, 225, 45, 325, 265, 92, 217, 382, 103, 140, 21, 312, 97, 315, 183, 318, 73, 171, 48, 301, 326, 107, 22, 242, 146, 167, 377, 151, 244, 316, 191, 351, 245, 243, 371, 259, 15, 224, 235, 247, 236, 274, 233, 79, 108, 66, 60, 199, 374, 154, 307, 4, 28, 77, 2, 241, 207, 78, 340, 202, 123, 13, 0, 36, 370, 284, 296, 363, 294, 283, 189, 83, 39, 267, 339, 223, 155, 150, 164, 369, 30, 38, 98, 299, 174, 286, 281, 254, 282, 187, 330, 311, 117, 7, 372, 317, 101, 10, 375, 70, 333, 32, 113, 64, 198, 335, 17, 220, 321, 34, 159, 279, 125, 170, 157, 186, 365, 302, 135, 177, 246, 298, 366, 255, 327, 137, 313, 179, 96, 33, 114, 347, 273, 43, 9, 175, 139, 192, 271, 88, 219, 6, 59, 168, 367, 31, 206, 269, 359, 362, 102, 227, 285, 35, 292, 104, 379, 291, 252, 37, 126, 342, 383, 130, 212, 205, 320, 336] + 55: [56, 202, 152, 154, 256, 81, 174, 174, 234, 146, 146, 218, 370, 39, 311, 16, 381, 123, 299, 90, 48, 78, 347, 74, 180, 29, 269, 76, 49, 291, 223, 134, 114, 272, 320, 349, 368, 312, 316, 156, 344, 259, 197, 46, 14, 276, 91, 375, 260, 27, 380, 47, 240, 286, 53, 194, 241, 119, 10, 346, 358, 207, 84, 94, 22, 314, 31, 153, 160, 342, 188, 37, 208, 253, 359, 319, 321, 38, 297, 129, 58, 120, 50, 374, 248, 166, 183, 376, 92, 295, 181, 36, 75, 32, 195, 230, 59, 148, 292, 51, 310, 285, 34, 69, 56, 202, 28, 154, 154, 355, 174, 4, 20, 11, 308, 357, 116, 165, 80, 238, 381, 267, 299, 90, 48, 121, 382, 74, 130, 29, 269, 301, 40, 106, 293, 71, 176, 12, 60, 44, 164, 300, 271, 26, 239, 189, 5, 163, 348, 95, 190, 371, 217, 251, 231, 361, 79, 178, 187, 140, 332, 184, 318, 287, 254, 171, 85, 33, 87, 63, 302, 212, 237, 258, 127, 365, 373, 335, 356, 126, 93, 336, 243, 338, 354, 275, 150, 68, 139, 324, 242, 280, 191, 298, 330, 21, 362, 108, 220, 158, 18, 351, 9, 45, 250, 245, 109, 204, 326, 159, 88, 154, 270, 81, 151, 309, 20, 11, 62, 340, 182, 339, 80, 111, 227, 328, 123, 73, 353, 273, 304, 323, 180, 70, 65, 76, 132, 200, 249, 226, 3, 279, 331, 290, 216, 221, 86, 124, 206, 172, 17, 170, 350, 284, 274, 210, 329, 236, 265, 246, 42, 306, 54, 228, 19, 263, 257, 23, 128, 157, 278, 149, 296, 117, 214, 82, 1, 232, 133, 367, 15, 219, 268, 372, 201, 118, 55, 104, 147, 107, 125, 145, 177, 13, 162, 77, 364, 155, 175, 244, 266, 142, 83, 100, 24, 229, 25, 205, 96, 186, 196, 215, 113, 72, 88, 327, 43, 235, 174, 98, 98, 7, 308, 340, 116, 89, 252, 307, 192, 6, 303, 41, 211, 247, 347, 323, 130, 363, 115, 115, 199, 341, 2, 99, 179, 8, 333, 138, 213, 366, 0, 383, 233, 337, 112, 102, 325, 352, 105, 289, 378, 224, 377, 343, 360, 67, 135, 136, 185, 322, 122, 277, 255, 52, 193, 101, 294, 313, 264, 198, 61, 103, 281, 143, 173, 225, 97, 30, 35, 209, 168, 137, 315, 161, 66, 64, 261, 203, 369, 141, 169, 262, 282, 334, 110, 57, 288, 345, 131, 167, 379, 283, 144, 222, 317, 305] + 56: [279, 8, 195, 167, 242, 44, 191, 178, 240, 220, 213, 15, 107, 98, 303, 267, 106, 106, 30, 5, 75, 9, 189, 263, 363, 125, 6, 346, 320, 140, 275, 2, 83, 77, 80, 14, 63, 207, 118, 238, 246, 205, 359, 376, 38, 160, 250, 10, 105, 21, 136, 35, 179, 23, 355, 280, 50, 110, 151, 164, 123, 209, 129, 297, 190, 339, 62, 271, 183, 180, 84, 119, 172, 347, 28, 4, 232, 175, 291, 16, 29, 49, 70, 361, 93, 152, 254, 345, 234, 325, 239, 349, 261, 7, 36, 217, 25, 174, 91, 318, 301, 85, 315, 143, 100, 231, 311, 321, 265, 260, 126, 82, 240, 220, 294, 273, 116, 163, 158, 351, 322, 259, 371, 87, 304, 147, 196, 236, 333, 187, 162, 161, 316, 99, 292, 18, 332, 329, 288, 381, 168, 192, 72, 97, 344, 233, 334, 27, 285, 281, 137, 278, 92, 181, 252, 79, 354, 269, 12, 22, 11, 34, 153, 243, 141, 282, 364, 42, 378, 122, 201, 299, 298, 96, 145, 218, 228, 132, 307, 200, 330, 276, 65, 367, 78, 144, 352, 222, 373, 24, 211, 258, 26, 256, 215, 370, 326, 32, 377, 365, 114, 247, 235, 61, 76, 74, 146, 3, 177, 231, 195, 343, 265, 260, 199, 178, 240, 81, 213, 15, 186, 155, 296, 225, 241, 340, 30, 5, 304, 156, 20, 249, 104, 251, 308, 46, 48, 39, 41, 188, 362, 327, 154, 245, 283, 101, 383, 309, 197, 103, 300, 182, 69, 342, 193, 128, 324, 305, 31, 295, 374, 268, 317, 149, 214, 138, 13, 17, 127, 185, 56, 286, 115, 313, 360, 94, 55, 134, 262, 341, 310, 290, 323, 226, 379, 57, 139, 372, 266, 170, 43, 350, 113, 204, 131, 337, 108, 272, 68, 212, 59, 336, 45, 216, 380, 171, 237, 102, 109, 142, 173, 90, 231, 231, 195, 321, 242, 44, 260, 82, 240, 220, 294, 294, 53, 88, 184, 225, 241, 340, 259, 169, 75, 147, 356, 89, 264, 306, 117, 328, 71, 284, 219, 302, 19, 66, 159, 357, 244, 150, 120, 223, 369, 0, 58, 73, 40, 148, 312, 253, 135, 206, 198, 338, 166, 230, 121, 331, 1, 133, 255, 33, 270, 227, 248, 368, 221, 95, 348, 210, 257, 54, 314, 353, 375, 293, 124, 112, 319, 274, 37, 289, 67, 64, 86, 130, 60, 287, 47, 157, 382, 358, 176, 203, 52, 335, 366, 277, 111, 194, 165, 51, 229, 224, 202, 208] + 57: [157, 14, 14, 276, 254, 314, 146, 295, 165, 246, 187, 156, 366, 25, 65, 101, 207, 223, 72, 286, 320, 324, 268, 8, 302, 300, 204, 174, 84, 197, 22, 64, 306, 123, 354, 182, 338, 167, 365, 373, 346, 134, 341, 214, 27, 113, 20, 277, 377, 183, 147, 289, 47, 12, 112, 328, 68, 2, 164, 196, 353, 209, 296, 216, 81, 186, 323, 371, 175, 237, 26, 363, 45, 122, 236, 299, 0, 4, 126, 219, 263, 383, 361, 375, 67, 232, 103, 308, 10, 271, 218, 212, 206, 327, 278, 195, 305, 131, 280, 233, 143, 56, 262, 301, 231, 342, 276, 158, 16, 133, 146, 295, 57, 89, 368, 7, 326, 25, 235, 148, 331, 108, 94, 274, 53, 170, 335, 118, 217, 60, 154, 150, 127, 283, 180, 309, 44, 163, 251, 272, 99, 58, 253, 172, 110, 213, 9, 228, 42, 173, 83, 191, 349, 102, 54, 203, 120, 55, 325, 279, 166, 210, 71, 171, 199, 321, 369, 155, 178, 98, 17, 29, 379, 332, 145, 315, 252, 135, 265, 255, 307, 32, 168, 241, 66, 50, 194, 374, 138, 364, 287, 273, 310, 266, 115, 258, 82, 52, 193, 336, 77, 245, 34, 229, 202, 86, 376, 343, 231, 159, 276, 31, 254, 254, 259, 220, 85, 246, 63, 140, 326, 104, 205, 148, 207, 359, 94, 124, 137, 170, 344, 125, 267, 74, 154, 6, 318, 151, 79, 88, 185, 75, 80, 347, 144, 91, 43, 161, 176, 40, 73, 70, 152, 96, 298, 312, 382, 111, 116, 38, 358, 106, 119, 95, 329, 226, 149, 3, 351, 282, 23, 362, 288, 132, 311, 162, 41, 290, 247, 355, 5, 303, 24, 248, 378, 281, 357, 1, 224, 36, 242, 192, 48, 370, 78, 139, 360, 244, 100, 291, 129, 221, 269, 30, 227, 37, 153, 105, 19, 285, 293, 372, 93, 11, 230, 381, 254, 254, 259, 121, 165, 89, 187, 198, 326, 25, 205, 128, 331, 257, 72, 286, 320, 130, 141, 125, 302, 74, 169, 350, 117, 317, 345, 160, 35, 62, 250, 238, 319, 39, 316, 179, 322, 243, 297, 51, 177, 249, 46, 356, 21, 184, 92, 208, 284, 380, 13, 225, 348, 136, 313, 352, 337, 190, 90, 28, 330, 215, 18, 334, 240, 261, 97, 49, 33, 211, 59, 200, 270, 87, 188, 333, 109, 260, 222, 256, 239, 76, 107, 61, 142, 264, 339, 114, 294, 340, 189, 292, 69, 181, 15, 201, 367, 234, 275, 304] + 58: [263, 220, 377, 377, 377, 249, 56, 79, 366, 205, 92, 344, 245, 98, 194, 101, 177, 299, 9, 22, 114, 358, 270, 256, 116, 178, 68, 196, 18, 47, 119, 191, 277, 132, 162, 20, 274, 185, 328, 28, 281, 192, 375, 55, 244, 62, 266, 99, 236, 83, 339, 333, 37, 240, 0, 63, 300, 253, 148, 231, 123, 334, 228, 21, 70, 293, 35, 50, 237, 230, 226, 170, 24, 261, 29, 372, 330, 268, 276, 41, 336, 44, 46, 223, 369, 246, 153, 304, 52, 286, 188, 174, 140, 343, 23, 212, 166, 279, 71, 321, 108, 280, 33, 38, 248, 220, 77, 377, 377, 126, 56, 81, 259, 305, 356, 210, 211, 14, 311, 373, 93, 235, 312, 298, 289, 67, 355, 169, 40, 36, 262, 324, 341, 6, 203, 272, 130, 111, 255, 316, 190, 307, 73, 198, 380, 1, 331, 381, 82, 143, 181, 284, 215, 53, 250, 326, 378, 121, 131, 332, 144, 13, 357, 154, 65, 161, 2, 43, 265, 115, 195, 179, 136, 219, 118, 297, 102, 138, 142, 100, 264, 221, 163, 354, 200, 374, 167, 349, 57, 117, 34, 137, 15, 88, 30, 365, 323, 91, 89, 291, 80, 147, 309, 370, 168, 201, 204, 209, 156, 220, 275, 377, 377, 151, 81, 61, 273, 205, 92, 27, 245, 271, 311, 222, 322, 49, 312, 298, 69, 351, 214, 199, 368, 175, 376, 295, 342, 85, 42, 189, 12, 315, 260, 106, 242, 197, 171, 329, 229, 139, 314, 251, 25, 383, 258, 350, 238, 90, 160, 234, 346, 75, 149, 110, 241, 362, 176, 186, 327, 318, 345, 11, 113, 19, 225, 8, 133, 173, 338, 243, 294, 296, 95, 87, 128, 86, 303, 157, 146, 337, 84, 269, 39, 109, 32, 159, 78, 301, 267, 217, 218, 363, 165, 313, 252, 292, 125, 335, 150, 364, 58, 134, 156, 220, 275, 377, 377, 76, 126, 81, 366, 305, 7, 348, 245, 98, 194, 222, 177, 206, 122, 122, 114, 72, 257, 26, 112, 232, 152, 45, 141, 367, 31, 16, 97, 104, 60, 4, 352, 66, 224, 285, 187, 48, 202, 239, 207, 59, 129, 325, 306, 155, 124, 227, 183, 382, 135, 302, 172, 208, 379, 288, 107, 278, 371, 347, 287, 182, 283, 353, 94, 105, 180, 64, 5, 213, 360, 317, 54, 359, 282, 254, 290, 103, 51, 247, 96, 120, 145, 216, 17, 193, 127, 74, 310, 361, 233, 320, 164, 158, 3, 319, 10, 184, 340, 308] + 59: [332, 182, 327, 33, 75, 25, 170, 197, 147, 229, 370, 94, 195, 260, 251, 173, 173, 220, 66, 204, 221, 352, 288, 18, 300, 65, 372, 377, 151, 26, 266, 196, 133, 112, 45, 109, 304, 81, 168, 279, 181, 302, 383, 252, 373, 225, 29, 19, 329, 236, 158, 69, 269, 348, 238, 354, 155, 13, 131, 51, 61, 199, 362, 299, 380, 27, 64, 110, 129, 50, 253, 134, 17, 207, 359, 93, 277, 132, 183, 11, 118, 53, 296, 103, 312, 368, 301, 237, 141, 282, 192, 125, 121, 293, 367, 78, 126, 104, 99, 52, 4, 111, 268, 271, 140, 209, 327, 285, 46, 25, 250, 256, 379, 233, 7, 94, 202, 59, 95, 200, 340, 311, 247, 371, 352, 234, 331, 31, 42, 213, 186, 328, 23, 80, 309, 232, 114, 342, 278, 223, 57, 12, 35, 159, 364, 334, 62, 162, 376, 60, 218, 338, 214, 356, 222, 258, 137, 146, 177, 259, 337, 353, 101, 123, 314, 274, 48, 206, 227, 71, 43, 135, 325, 184, 84, 124, 67, 74, 153, 49, 261, 44, 24, 191, 201, 357, 102, 230, 216, 303, 8, 108, 239, 240, 350, 14, 335, 77, 244, 87, 152, 139, 72, 323, 310, 90, 79, 154, 140, 113, 327, 105, 224, 170, 170, 97, 226, 229, 5, 7, 195, 260, 70, 200, 340, 381, 66, 263, 346, 291, 130, 185, 265, 65, 324, 317, 178, 243, 281, 58, 128, 176, 212, 164, 161, 160, 16, 127, 267, 280, 163, 320, 143, 22, 174, 142, 54, 47, 179, 339, 344, 180, 366, 203, 289, 365, 349, 68, 242, 292, 32, 194, 287, 193, 149, 246, 187, 136, 231, 217, 21, 343, 326, 89, 336, 210, 106, 115, 148, 273, 307, 241, 358, 165, 319, 316, 276, 305, 219, 270, 375, 171, 205, 347, 83, 55, 122, 167, 235, 117, 341, 189, 73, 209, 327, 30, 224, 25, 250, 313, 379, 229, 370, 7, 202, 59, 363, 363, 220, 298, 374, 294, 346, 234, 76, 150, 172, 295, 9, 38, 262, 144, 382, 255, 86, 290, 91, 175, 40, 85, 254, 297, 100, 211, 169, 248, 315, 20, 245, 286, 306, 10, 355, 120, 198, 369, 92, 361, 36, 6, 228, 2, 156, 96, 107, 0, 333, 138, 63, 41, 34, 208, 322, 157, 330, 28, 37, 145, 3, 88, 249, 275, 321, 119, 166, 284, 283, 82, 257, 98, 318, 1, 360, 56, 39, 272, 264, 378, 345, 190, 116, 351, 308, 15, 188, 215] + 60: [284, 121, 313, 313, 381, 124, 257, 158, 16, 115, 220, 47, 139, 244, 126, 133, 316, 212, 174, 310, 167, 312, 185, 81, 2, 239, 294, 319, 365, 46, 193, 309, 206, 232, 143, 223, 241, 55, 364, 359, 336, 0, 69, 248, 221, 37, 51, 1, 92, 24, 9, 237, 215, 274, 304, 305, 134, 186, 70, 53, 31, 66, 33, 30, 100, 192, 106, 370, 380, 256, 18, 64, 125, 363, 282, 376, 251, 45, 330, 328, 179, 54, 198, 50, 374, 211, 29, 96, 301, 191, 350, 281, 289, 164, 275, 340, 101, 321, 226, 136, 373, 154, 348, 355, 102, 236, 313, 381, 335, 124, 124, 158, 128, 195, 266, 47, 139, 343, 83, 83, 199, 222, 174, 310, 194, 120, 297, 208, 74, 366, 35, 337, 180, 112, 311, 354, 245, 131, 184, 203, 238, 263, 99, 253, 152, 150, 58, 20, 269, 358, 183, 240, 80, 332, 382, 264, 61, 176, 8, 56, 323, 21, 107, 142, 285, 85, 200, 250, 300, 130, 357, 349, 368, 219, 14, 148, 315, 338, 105, 270, 23, 13, 252, 324, 137, 261, 104, 42, 68, 204, 378, 291, 175, 249, 17, 254, 227, 163, 177, 295, 44, 118, 218, 89, 210, 63, 207, 201, 94, 123, 313, 268, 351, 351, 124, 257, 16, 170, 243, 303, 60, 372, 39, 133, 339, 40, 78, 265, 247, 65, 302, 181, 205, 34, 108, 49, 290, 166, 345, 98, 278, 122, 117, 161, 235, 27, 246, 140, 84, 151, 362, 283, 93, 132, 36, 318, 11, 88, 267, 110, 38, 57, 279, 25, 26, 233, 52, 360, 187, 293, 149, 196, 182, 369, 95, 314, 276, 144, 225, 371, 325, 333, 307, 67, 202, 361, 209, 287, 19, 216, 41, 145, 160, 356, 147, 298, 86, 165, 326, 135, 173, 367, 4, 103, 352, 10, 77, 242, 157, 317, 288, 375, 123, 123, 313, 268, 351, 351, 124, 257, 16, 260, 220, 129, 60, 347, 126, 113, 199, 48, 40, 265, 230, 153, 334, 155, 28, 327, 76, 234, 353, 119, 329, 15, 213, 272, 231, 59, 271, 82, 169, 172, 75, 189, 277, 255, 114, 6, 217, 259, 344, 188, 162, 5, 3, 377, 116, 342, 141, 171, 346, 90, 273, 91, 7, 146, 190, 341, 79, 322, 296, 286, 12, 331, 197, 383, 156, 168, 262, 127, 178, 228, 320, 258, 280, 224, 379, 229, 97, 111, 72, 159, 306, 43, 292, 71, 138, 32, 73, 109, 87, 308, 62, 214, 299, 22] + 61: [231, 73, 136, 93, 157, 170, 179, 46, 127, 127, 299, 323, 19, 184, 1, 45, 253, 218, 167, 21, 344, 371, 324, 198, 248, 263, 161, 119, 88, 107, 216, 168, 34, 159, 349, 243, 169, 261, 228, 191, 348, 297, 316, 114, 135, 82, 252, 304, 47, 308, 206, 145, 104, 202, 33, 259, 322, 38, 78, 306, 8, 37, 125, 126, 85, 196, 26, 317, 176, 194, 359, 148, 70, 311, 91, 163, 187, 255, 234, 251, 7, 262, 185, 314, 164, 212, 223, 105, 35, 220, 56, 226, 151, 175, 134, 62, 41, 320, 16, 128, 356, 189, 208, 319, 303, 73, 5, 211, 221, 221, 338, 46, 127, 127, 299, 178, 270, 264, 197, 363, 100, 36, 167, 195, 372, 278, 324, 142, 143, 213, 237, 66, 112, 216, 129, 115, 369, 42, 31, 160, 309, 43, 337, 20, 118, 77, 90, 84, 44, 166, 121, 379, 329, 260, 3, 23, 97, 147, 249, 378, 154, 383, 153, 273, 280, 174, 242, 219, 225, 240, 81, 4, 144, 332, 98, 106, 193, 244, 130, 271, 239, 6, 123, 74, 232, 111, 201, 293, 373, 138, 265, 140, 256, 276, 302, 277, 281, 39, 289, 183, 76, 229, 296, 40, 171, 96, 287, 58, 230, 61, 108, 360, 211, 221, 179, 338, 247, 165, 205, 117, 241, 59, 197, 381, 103, 36, 315, 330, 344, 68, 294, 215, 55, 263, 161, 86, 88, 107, 288, 333, 110, 286, 181, 156, 227, 133, 87, 258, 224, 65, 268, 29, 300, 190, 313, 11, 109, 336, 69, 368, 207, 200, 102, 173, 72, 137, 346, 214, 275, 9, 365, 80, 269, 141, 340, 71, 124, 14, 342, 298, 162, 345, 63, 54, 354, 18, 364, 358, 312, 331, 246, 272, 95, 28, 48, 236, 250, 222, 60, 285, 254, 355, 382, 267, 376, 217, 266, 350, 116, 53, 94, 0, 172, 61, 335, 93, 120, 221, 179, 338, 326, 351, 122, 203, 10, 209, 380, 381, 100, 353, 315, 12, 17, 51, 238, 235, 248, 307, 307, 86, 192, 327, 291, 57, 177, 188, 15, 282, 32, 75, 92, 361, 362, 150, 233, 366, 83, 27, 113, 374, 2, 257, 245, 328, 52, 158, 290, 139, 186, 89, 292, 30, 375, 341, 22, 79, 283, 67, 295, 132, 199, 284, 204, 152, 370, 274, 352, 25, 310, 301, 101, 99, 49, 24, 13, 305, 180, 182, 146, 339, 318, 347, 64, 377, 279, 131, 210, 50, 367, 325, 334, 357, 343, 321, 149, 155] +layer_updates_per_iter: 0 +num_slots: 416 diff --git a/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots384.yaml b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots384.yaml new file mode 100644 index 000000000000..4a5c707042a2 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 45, 49, 259, 322, 235, 158, 163, 81, 29, 289, 70, 198, 144, 141, 181, 107, 43, 57, 361, 28, 248, 309, 262, 5, 91, 317, 137, 363, 192, 296, 270, 299, 326, 50, 252, 352, 316, 175, 2, 301, 125, 132, 304, 364, 273, 13, 373, 111, 279, 203, 150, 159, 348, 241, 295, 21, 80, 255, 376, 318, 272, 359, 72, 101, 233, 109, 129, 324, 208, 123, 314, 39, 133, 4, 108, 116, 216, 100, 379, 382, 263, 20, 210, 42, 212, 315, 334, 367, 251, 298, 329, 339, 8, 171, 274, 378, 135, 184, 0, 345, 177, 346, 188, 153, 242, 321, 65, 278, 209, 360, 284, 34, 313, 297, 311, 114, 44, 268, 234, 52, 308, 103, 146, 140, 19, 180, 320, 127, 14, 145, 350, 15, 64, 53, 69, 319, 164, 18, 226, 291, 88, 370, 185, 166, 254, 134, 217, 351, 155, 264, 332, 122, 62, 157, 68, 333, 280, 143, 205, 196, 283, 186, 200, 261, 148, 31, 358, 85, 187, 136, 240, 178, 99, 211, 195, 82, 328, 47, 102, 338, 340, 380, 179, 381, 250, 167, 374, 325, 323, 58, 201, 256, 225, 95, 330, 151, 383, 193, 182, 51, 104, 232, 63, 37, 126, 22, 307, 11, 341, 90, 258, 112, 300, 27, 89, 66, 32, 214, 249, 246, 275, 238, 292, 83, 93, 260, 229, 161, 253, 353, 152, 194, 357, 142, 24, 228, 130, 277, 61, 74, 371, 79, 290, 84, 119, 230, 169, 35, 202, 327, 139, 276, 349, 55, 33, 117, 156, 10, 36, 138, 7, 77, 375, 377, 172, 227, 331, 269, 366, 1, 25, 73, 67, 342, 98, 294, 206, 12, 219, 9, 285, 271, 121, 281, 168, 118, 247, 222, 245, 17, 173, 71, 86, 207, 312, 347, 30, 218, 176, 16, 3, 78, 243, 105, 183, 124, 174, 94, 337, 160, 199, 87, 244, 335, 365, 372, 369, 97, 106, 147, 237, 362, 286, 305, 115, 306, 191, 231, 220, 23, 189, 355, 288, 310, 303, 356, 236, 60, 224, 302, 162, 76, 190, 26, 204, 48, 343, 293, 282, 336, 54, 131, 56, 354, 265, 368, 170, 149, 38, 223, 257, 213, 46, 75, 59, 215, 221, 197, 267, 6, 154, 40, 96, 41, 113, 239, 266, 92, 287, 128, 344, 120, 110] + 1: [163, 177, 198, 292, 251, 18, 196, 378, 366, 328, 73, 153, 351, 195, 9, 32, 266, 285, 187, 261, 143, 282, 264, 202, 231, 189, 164, 238, 338, 326, 19, 33, 186, 51, 352, 346, 344, 77, 357, 220, 340, 217, 132, 16, 144, 28, 336, 265, 319, 169, 182, 225, 1, 80, 152, 40, 179, 41, 110, 271, 185, 273, 145, 173, 317, 355, 197, 194, 214, 176, 203, 239, 61, 98, 7, 305, 2, 20, 175, 136, 324, 11, 97, 8, 232, 314, 130, 246, 188, 373, 361, 45, 276, 339, 345, 15, 365, 72, 127, 82, 31, 106, 50, 21, 270, 123, 105, 248, 383, 243, 287, 71, 234, 212, 162, 379, 52, 53, 306, 376, 60, 205, 362, 204, 364, 17, 156, 68, 36, 284, 96, 181, 240, 103, 83, 297, 160, 207, 252, 343, 349, 341, 363, 327, 42, 91, 167, 178, 380, 118, 43, 241, 5, 75, 300, 244, 25, 46, 275, 86, 245, 301, 84, 92, 223, 116, 69, 360, 95, 76, 121, 259, 370, 107, 115, 62, 290, 26, 104, 113, 54, 119, 369, 331, 193, 209, 124, 155, 307, 354, 112, 368, 201, 382, 67, 272, 81, 44, 291, 329, 30, 210, 58, 56, 24, 59, 303, 247, 78, 274, 166, 142, 114, 93, 289, 278, 236, 298, 122, 330, 65, 286, 184, 154, 35, 230, 311, 268, 226, 333, 375, 372, 367, 221, 128, 322, 117, 257, 199, 255, 174, 277, 347, 134, 141, 308, 263, 131, 6, 325, 294, 299, 224, 320, 211, 260, 216, 99, 288, 109, 219, 228, 381, 342, 147, 237, 140, 170, 139, 318, 229, 165, 48, 293, 227, 55, 12, 49, 148, 309, 90, 151, 66, 374, 337, 157, 267, 100, 22, 158, 377, 111, 200, 126, 149, 254, 39, 0, 63, 250, 283, 29, 269, 242, 281, 74, 218, 280, 4, 47, 358, 171, 101, 102, 313, 222, 161, 332, 295, 321, 87, 315, 13, 302, 262, 356, 279, 371, 183, 253, 208, 64, 258, 304, 312, 215, 137, 180, 3, 120, 235, 85, 150, 168, 192, 70, 334, 23, 233, 138, 37, 89, 125, 146, 108, 335, 213, 206, 88, 38, 133, 256, 79, 348, 249, 14, 296, 57, 172, 350, 94, 323, 353, 27, 190, 129, 316, 135, 10, 191, 159, 359, 310, 34] + 2: [143, 295, 242, 180, 96, 137, 321, 78, 111, 173, 73, 245, 133, 325, 217, 235, 303, 165, 344, 273, 68, 67, 21, 138, 46, 171, 193, 129, 372, 252, 365, 219, 239, 47, 178, 261, 302, 288, 8, 164, 310, 37, 353, 88, 287, 337, 40, 374, 248, 3, 212, 230, 225, 28, 197, 33, 101, 284, 56, 70, 172, 368, 63, 333, 159, 209, 17, 87, 293, 92, 383, 148, 253, 265, 181, 103, 38, 312, 176, 179, 77, 238, 376, 319, 54, 71, 251, 98, 117, 227, 240, 289, 301, 202, 363, 161, 53, 223, 290, 314, 14, 5, 189, 213, 84, 166, 286, 44, 123, 45, 200, 82, 285, 61, 206, 224, 27, 262, 120, 218, 41, 65, 309, 69, 256, 12, 257, 0, 25, 281, 141, 352, 329, 99, 335, 158, 378, 283, 249, 315, 358, 282, 255, 366, 43, 115, 62, 86, 343, 298, 125, 162, 110, 322, 280, 278, 31, 51, 274, 24, 311, 79, 299, 263, 50, 203, 201, 76, 332, 241, 169, 250, 106, 39, 268, 2, 36, 334, 264, 359, 216, 370, 338, 307, 382, 177, 369, 64, 228, 354, 300, 381, 132, 93, 232, 139, 357, 259, 100, 192, 155, 124, 355, 271, 220, 22, 328, 85, 6, 342, 7, 90, 269, 210, 361, 297, 349, 308, 186, 254, 136, 49, 318, 23, 195, 346, 331, 57, 243, 371, 114, 267, 336, 222, 13, 380, 32, 339, 199, 330, 112, 194, 168, 304, 341, 221, 247, 60, 153, 97, 157, 135, 105, 296, 279, 340, 196, 174, 170, 347, 55, 188, 58, 128, 360, 145, 348, 276, 364, 121, 29, 362, 134, 4, 272, 231, 292, 373, 167, 59, 72, 234, 260, 127, 119, 270, 190, 379, 108, 89, 91, 377, 102, 1, 151, 149, 109, 191, 66, 118, 42, 26, 83, 244, 323, 204, 130, 356, 126, 327, 187, 198, 163, 16, 15, 160, 154, 142, 104, 94, 275, 35, 52, 144, 226, 95, 183, 48, 113, 345, 214, 266, 316, 208, 375, 258, 313, 215, 19, 122, 351, 81, 34, 75, 185, 350, 131, 306, 184, 324, 146, 11, 30, 294, 140, 18, 156, 320, 20, 236, 277, 10, 175, 116, 9, 305, 150, 152, 211, 367, 207, 233, 291, 107, 205, 74, 182, 147, 317, 80, 326, 237, 246, 229] + 3: [66, 171, 13, 62, 252, 331, 201, 120, 58, 231, 74, 316, 12, 87, 78, 288, 203, 374, 375, 299, 344, 197, 266, 280, 144, 366, 96, 249, 128, 7, 356, 321, 86, 133, 379, 140, 121, 329, 1, 95, 175, 60, 226, 233, 324, 47, 222, 370, 34, 14, 131, 80, 158, 180, 139, 178, 164, 37, 287, 174, 25, 118, 279, 142, 32, 111, 300, 274, 256, 57, 65, 246, 30, 160, 340, 70, 67, 15, 85, 346, 254, 206, 116, 220, 191, 45, 253, 352, 315, 69, 341, 257, 198, 311, 38, 301, 310, 27, 3, 275, 19, 192, 24, 150, 6, 102, 336, 2, 212, 239, 219, 240, 382, 161, 289, 262, 176, 345, 217, 53, 270, 117, 255, 125, 196, 115, 184, 232, 91, 132, 72, 101, 93, 179, 372, 242, 159, 54, 148, 320, 244, 5, 17, 278, 260, 18, 71, 314, 225, 333, 151, 50, 213, 63, 342, 358, 138, 221, 36, 290, 136, 109, 295, 277, 92, 98, 103, 55, 327, 135, 48, 328, 100, 353, 188, 305, 165, 381, 362, 293, 258, 183, 16, 210, 360, 90, 330, 349, 347, 371, 247, 292, 377, 40, 313, 230, 31, 97, 348, 49, 337, 339, 343, 211, 189, 304, 119, 21, 264, 8, 359, 312, 114, 41, 218, 235, 112, 105, 76, 214, 276, 286, 20, 281, 106, 259, 110, 269, 285, 245, 251, 338, 369, 147, 75, 124, 152, 33, 332, 303, 365, 302, 163, 46, 317, 261, 157, 81, 42, 238, 207, 10, 267, 296, 182, 223, 357, 162, 154, 149, 243, 126, 224, 380, 355, 129, 268, 82, 237, 322, 361, 291, 83, 61, 68, 309, 190, 0, 323, 107, 284, 368, 89, 265, 173, 167, 335, 59, 319, 79, 263, 209, 51, 208, 363, 334, 137, 26, 168, 227, 325, 195, 177, 29, 271, 364, 99, 351, 297, 204, 108, 236, 169, 273, 44, 104, 306, 248, 187, 294, 376, 130, 228, 153, 354, 378, 73, 156, 141, 205, 127, 308, 326, 185, 52, 199, 39, 22, 194, 35, 350, 56, 318, 145, 215, 186, 170, 250, 122, 77, 11, 64, 202, 84, 172, 229, 155, 181, 200, 166, 94, 307, 113, 383, 43, 216, 298, 367, 123, 373, 282, 28, 283, 134, 88, 234, 143, 241, 272, 23, 4, 193, 9, 146] + 4: [343, 148, 375, 259, 228, 257, 302, 345, 274, 76, 200, 361, 44, 312, 251, 35, 278, 276, 290, 133, 62, 379, 261, 64, 313, 94, 284, 39, 306, 121, 26, 180, 304, 201, 355, 96, 115, 353, 55, 368, 190, 285, 154, 2, 242, 120, 182, 277, 331, 142, 37, 24, 239, 105, 143, 253, 155, 41, 275, 357, 160, 8, 198, 187, 38, 335, 264, 189, 330, 169, 149, 229, 70, 240, 370, 220, 349, 303, 175, 249, 334, 40, 286, 126, 97, 163, 162, 366, 224, 301, 174, 20, 67, 336, 332, 351, 3, 319, 254, 12, 219, 88, 5, 136, 47, 329, 113, 77, 364, 101, 333, 22, 110, 283, 244, 0, 166, 72, 185, 341, 216, 232, 158, 288, 66, 52, 346, 122, 170, 118, 71, 213, 311, 203, 316, 247, 30, 245, 363, 227, 119, 272, 58, 282, 271, 226, 152, 124, 350, 172, 45, 241, 13, 197, 139, 318, 196, 27, 28, 280, 369, 179, 324, 243, 53, 356, 297, 323, 46, 262, 354, 84, 138, 267, 78, 103, 291, 147, 111, 255, 205, 378, 362, 191, 238, 293, 326, 337, 192, 321, 63, 305, 315, 95, 117, 18, 300, 16, 376, 308, 36, 377, 325, 61, 372, 57, 83, 217, 73, 260, 207, 358, 42, 168, 360, 352, 132, 269, 383, 25, 310, 92, 50, 230, 29, 327, 211, 223, 10, 186, 374, 279, 173, 134, 248, 89, 382, 17, 79, 90, 214, 15, 273, 340, 258, 193, 19, 6, 86, 59, 167, 246, 151, 235, 43, 49, 7, 82, 141, 183, 116, 307, 98, 93, 14, 108, 69, 100, 137, 292, 176, 234, 157, 140, 210, 31, 127, 215, 48, 112, 91, 202, 11, 212, 195, 225, 338, 107, 209, 23, 194, 34, 222, 320, 281, 171, 348, 56, 347, 4, 32, 314, 218, 344, 161, 177, 159, 99, 85, 65, 114, 359, 75, 125, 80, 135, 367, 309, 263, 184, 51, 181, 237, 109, 130, 81, 129, 380, 1, 265, 373, 236, 270, 208, 231, 104, 296, 371, 87, 188, 199, 250, 9, 150, 165, 342, 298, 145, 106, 365, 339, 204, 68, 317, 153, 54, 221, 252, 256, 33, 60, 74, 131, 322, 328, 178, 294, 206, 295, 146, 289, 268, 156, 381, 21, 123, 164, 144, 102, 266, 299, 233, 128, 287] + 5: [61, 305, 231, 141, 1, 223, 365, 64, 132, 200, 91, 182, 304, 70, 268, 238, 319, 24, 34, 220, 295, 186, 312, 19, 106, 250, 161, 90, 334, 236, 162, 55, 283, 378, 244, 58, 373, 111, 307, 282, 158, 294, 112, 348, 167, 57, 196, 302, 213, 105, 272, 154, 98, 265, 381, 41, 351, 7, 192, 74, 233, 313, 163, 264, 38, 207, 258, 377, 35, 144, 274, 227, 366, 300, 219, 289, 237, 360, 94, 376, 221, 263, 135, 92, 125, 101, 126, 85, 146, 42, 339, 142, 225, 50, 118, 354, 128, 100, 6, 165, 260, 49, 30, 379, 279, 88, 247, 22, 173, 137, 222, 174, 206, 3, 320, 315, 228, 79, 242, 183, 353, 240, 150, 325, 330, 13, 77, 62, 113, 129, 68, 316, 189, 290, 120, 73, 36, 123, 310, 176, 278, 286, 127, 75, 214, 276, 288, 208, 160, 60, 169, 78, 215, 352, 89, 335, 371, 155, 37, 369, 67, 143, 314, 218, 104, 115, 216, 39, 343, 121, 257, 203, 275, 149, 29, 191, 235, 71, 270, 298, 297, 347, 185, 309, 328, 357, 273, 10, 367, 157, 321, 177, 355, 346, 52, 172, 5, 178, 188, 102, 145, 43, 147, 25, 31, 8, 375, 2, 69, 204, 26, 45, 301, 99, 12, 253, 14, 239, 138, 136, 119, 9, 344, 232, 359, 103, 107, 168, 322, 245, 292, 267, 97, 336, 180, 27, 110, 82, 179, 303, 190, 217, 287, 259, 256, 65, 193, 211, 130, 17, 246, 338, 327, 66, 15, 358, 108, 318, 175, 333, 16, 195, 266, 329, 20, 382, 63, 284, 281, 80, 291, 271, 374, 299, 199, 40, 148, 32, 53, 285, 243, 340, 109, 116, 280, 96, 277, 368, 249, 194, 356, 21, 308, 81, 370, 372, 72, 212, 331, 364, 124, 131, 171, 59, 181, 140, 210, 153, 54, 28, 51, 345, 230, 362, 341, 76, 383, 87, 134, 229, 363, 114, 18, 156, 224, 95, 11, 139, 350, 234, 117, 248, 83, 164, 337, 44, 84, 296, 269, 159, 306, 47, 262, 122, 255, 252, 293, 198, 209, 4, 151, 332, 184, 251, 324, 152, 33, 46, 380, 93, 56, 201, 241, 317, 254, 48, 361, 349, 205, 323, 342, 187, 261, 86, 133, 197, 311, 202, 0, 226, 166, 326, 23, 170] + 6: [97, 281, 212, 306, 46, 371, 36, 318, 119, 246, 34, 239, 0, 113, 352, 236, 223, 202, 44, 211, 98, 280, 207, 116, 197, 261, 148, 199, 124, 349, 381, 366, 285, 152, 328, 117, 303, 183, 302, 29, 91, 344, 61, 65, 167, 35, 40, 90, 142, 240, 297, 249, 11, 67, 378, 216, 293, 84, 208, 129, 151, 176, 377, 278, 304, 111, 383, 219, 136, 365, 237, 242, 86, 217, 138, 81, 198, 224, 54, 309, 235, 130, 360, 162, 338, 313, 339, 92, 19, 93, 228, 153, 214, 320, 161, 164, 308, 253, 146, 27, 229, 31, 375, 181, 100, 112, 287, 289, 234, 165, 369, 336, 101, 190, 139, 327, 10, 157, 22, 110, 284, 23, 194, 316, 83, 105, 96, 16, 379, 357, 74, 337, 256, 348, 89, 356, 140, 127, 172, 353, 252, 363, 325, 247, 322, 258, 177, 160, 55, 120, 173, 163, 88, 368, 196, 184, 300, 301, 109, 245, 317, 37, 321, 77, 15, 154, 210, 69, 159, 243, 254, 329, 333, 346, 215, 42, 134, 292, 62, 108, 266, 351, 222, 250, 78, 141, 273, 200, 75, 149, 71, 323, 76, 174, 166, 103, 341, 205, 350, 359, 232, 189, 118, 291, 345, 277, 185, 380, 231, 279, 133, 64, 264, 58, 6, 104, 276, 82, 362, 122, 168, 358, 8, 70, 310, 312, 203, 288, 102, 294, 131, 201, 238, 227, 340, 218, 132, 135, 204, 257, 274, 343, 66, 295, 269, 180, 45, 307, 187, 57, 188, 178, 259, 95, 179, 41, 311, 158, 221, 107, 332, 286, 271, 87, 225, 330, 355, 145, 18, 193, 298, 268, 364, 244, 80, 347, 290, 51, 21, 374, 171, 241, 1, 20, 72, 25, 275, 342, 376, 53, 99, 59, 267, 209, 9, 260, 94, 56, 255, 79, 370, 270, 128, 182, 49, 12, 282, 144, 324, 220, 262, 206, 63, 335, 372, 126, 48, 3, 334, 299, 73, 123, 106, 226, 85, 24, 47, 38, 7, 192, 382, 2, 265, 33, 315, 373, 32, 305, 233, 5, 28, 326, 175, 13, 367, 283, 319, 155, 156, 354, 115, 230, 68, 30, 169, 114, 143, 296, 361, 43, 137, 60, 4, 14, 248, 50, 147, 272, 125, 39, 150, 251, 17, 195, 121, 263, 26, 213, 170, 191, 314, 52, 186, 331] + 7: [327, 265, 25, 272, 133, 271, 378, 328, 27, 73, 164, 248, 349, 288, 325, 208, 219, 191, 264, 9, 311, 204, 32, 178, 335, 121, 353, 373, 146, 68, 153, 79, 151, 287, 295, 1, 312, 3, 63, 198, 346, 12, 347, 154, 279, 110, 202, 364, 7, 229, 201, 157, 239, 46, 99, 329, 61, 281, 313, 267, 4, 31, 360, 284, 255, 62, 300, 214, 189, 245, 237, 107, 200, 86, 226, 323, 196, 291, 93, 203, 269, 106, 112, 72, 123, 39, 381, 159, 95, 66, 165, 138, 376, 236, 135, 257, 274, 142, 103, 162, 361, 342, 118, 212, 370, 302, 294, 82, 262, 23, 148, 235, 210, 377, 344, 167, 368, 109, 130, 5, 83, 44, 134, 301, 40, 261, 101, 351, 318, 166, 169, 218, 254, 379, 124, 315, 359, 150, 172, 127, 355, 30, 160, 50, 49, 375, 340, 369, 108, 28, 71, 188, 0, 94, 324, 100, 217, 119, 84, 250, 70, 170, 213, 132, 206, 26, 136, 190, 293, 102, 152, 310, 161, 8, 22, 345, 126, 320, 363, 69, 85, 111, 252, 228, 242, 304, 197, 143, 34, 326, 371, 42, 292, 41, 18, 341, 258, 187, 13, 97, 244, 131, 183, 47, 289, 285, 2, 319, 356, 21, 90, 241, 234, 140, 20, 180, 211, 58, 51, 53, 232, 227, 87, 52, 366, 35, 209, 194, 88, 173, 297, 334, 246, 45, 205, 54, 282, 240, 365, 60, 175, 64, 105, 163, 336, 176, 139, 15, 174, 181, 14, 215, 278, 17, 270, 296, 81, 224, 330, 333, 259, 16, 92, 19, 220, 380, 141, 56, 251, 309, 38, 193, 184, 216, 286, 316, 10, 367, 277, 78, 266, 57, 147, 331, 156, 223, 348, 171, 182, 263, 382, 298, 308, 260, 253, 48, 104, 114, 11, 113, 74, 168, 337, 374, 383, 177, 158, 98, 321, 89, 358, 129, 322, 6, 305, 306, 283, 247, 149, 120, 37, 222, 33, 207, 96, 231, 317, 128, 195, 230, 290, 144, 122, 179, 91, 357, 275, 192, 343, 362, 238, 350, 280, 354, 77, 256, 249, 307, 125, 29, 55, 339, 117, 221, 372, 243, 24, 314, 185, 186, 137, 273, 67, 155, 233, 65, 116, 225, 115, 268, 75, 36, 59, 80, 43, 303, 299, 332, 352, 199, 76, 145, 276, 338] + 8: [360, 308, 273, 238, 224, 272, 36, 252, 316, 364, 131, 128, 75, 109, 119, 330, 165, 219, 198, 113, 233, 52, 114, 101, 263, 379, 191, 90, 310, 136, 327, 200, 325, 236, 93, 323, 351, 324, 129, 340, 123, 305, 139, 344, 57, 262, 58, 234, 253, 162, 371, 279, 106, 336, 228, 276, 157, 294, 97, 311, 69, 23, 226, 179, 17, 303, 167, 196, 177, 125, 8, 284, 383, 25, 116, 249, 32, 382, 143, 140, 35, 260, 322, 134, 153, 15, 271, 26, 268, 80, 138, 288, 242, 127, 96, 170, 290, 362, 152, 214, 338, 12, 88, 81, 251, 209, 377, 374, 346, 83, 34, 240, 261, 178, 137, 50, 334, 211, 91, 61, 206, 98, 73, 293, 110, 266, 16, 199, 89, 295, 375, 365, 33, 146, 270, 269, 160, 355, 274, 301, 49, 163, 1, 207, 359, 201, 353, 104, 292, 13, 381, 74, 186, 349, 168, 148, 181, 188, 333, 258, 78, 275, 368, 183, 300, 141, 332, 60, 45, 117, 174, 71, 297, 47, 37, 126, 99, 339, 306, 14, 347, 24, 76, 158, 77, 245, 43, 203, 345, 176, 55, 102, 5, 239, 221, 281, 218, 108, 243, 337, 20, 255, 328, 280, 166, 22, 27, 283, 202, 169, 304, 194, 320, 192, 2, 358, 335, 217, 28, 232, 350, 299, 41, 155, 105, 85, 159, 369, 287, 357, 220, 244, 208, 285, 51, 53, 124, 0, 227, 132, 67, 264, 321, 204, 3, 302, 241, 64, 122, 380, 11, 86, 225, 172, 356, 147, 92, 65, 215, 112, 317, 120, 318, 4, 42, 254, 87, 247, 151, 373, 372, 313, 144, 235, 205, 185, 46, 277, 84, 348, 180, 354, 142, 331, 222, 210, 296, 100, 31, 367, 10, 182, 246, 161, 173, 282, 291, 95, 18, 115, 39, 107, 133, 6, 231, 121, 278, 309, 130, 48, 197, 66, 59, 212, 237, 257, 184, 149, 190, 314, 79, 265, 145, 29, 54, 326, 171, 103, 319, 70, 343, 342, 256, 361, 19, 376, 135, 21, 62, 229, 250, 7, 312, 154, 30, 223, 44, 156, 195, 366, 150, 38, 298, 216, 40, 189, 267, 164, 341, 72, 187, 230, 118, 248, 352, 259, 307, 68, 56, 378, 9, 175, 315, 111, 193, 370, 329, 286, 363, 94, 82, 289, 63, 213] + 9: [92, 244, 81, 372, 353, 184, 15, 138, 210, 67, 239, 258, 161, 135, 120, 297, 231, 192, 116, 186, 4, 230, 238, 237, 343, 235, 248, 382, 197, 261, 112, 291, 71, 298, 38, 229, 86, 34, 383, 266, 279, 44, 204, 157, 61, 236, 234, 26, 304, 64, 166, 280, 357, 179, 272, 217, 281, 136, 177, 9, 82, 377, 137, 88, 180, 314, 259, 35, 105, 36, 66, 141, 39, 324, 109, 307, 275, 233, 254, 110, 375, 310, 162, 347, 170, 193, 58, 240, 328, 284, 85, 164, 182, 77, 338, 57, 255, 313, 95, 143, 260, 219, 381, 83, 242, 271, 14, 342, 312, 360, 302, 289, 228, 6, 97, 99, 195, 123, 150, 218, 359, 104, 13, 354, 60, 45, 300, 2, 320, 251, 352, 232, 227, 69, 337, 140, 325, 73, 101, 378, 250, 27, 53, 340, 349, 185, 103, 331, 215, 350, 351, 1, 370, 214, 23, 309, 133, 30, 356, 292, 306, 247, 107, 129, 256, 336, 11, 332, 5, 308, 246, 8, 100, 148, 174, 358, 253, 305, 134, 89, 346, 167, 290, 368, 345, 249, 93, 200, 28, 122, 152, 348, 327, 47, 262, 212, 315, 252, 222, 344, 376, 287, 121, 56, 98, 317, 363, 268, 264, 0, 183, 41, 269, 139, 16, 189, 106, 75, 208, 50, 52, 245, 190, 203, 371, 330, 76, 211, 48, 188, 293, 243, 176, 118, 149, 273, 91, 265, 369, 365, 355, 295, 49, 78, 196, 299, 155, 198, 127, 329, 209, 32, 131, 285, 187, 3, 277, 54, 364, 20, 142, 267, 70, 181, 274, 80, 339, 65, 130, 117, 373, 114, 42, 286, 333, 21, 221, 241, 326, 68, 108, 361, 278, 194, 362, 25, 55, 173, 31, 263, 316, 10, 220, 19, 74, 29, 201, 159, 51, 132, 22, 367, 158, 154, 33, 334, 321, 311, 163, 323, 124, 171, 288, 319, 17, 18, 379, 46, 62, 322, 84, 301, 270, 125, 199, 59, 374, 43, 225, 63, 223, 113, 40, 7, 24, 168, 226, 94, 90, 147, 96, 172, 165, 37, 146, 380, 341, 282, 102, 151, 144, 153, 169, 318, 128, 87, 145, 207, 206, 111, 72, 202, 276, 175, 303, 205, 213, 79, 115, 12, 224, 216, 178, 335, 160, 191, 366, 119, 296, 294, 257, 156, 126, 283] + 10: [319, 310, 137, 325, 309, 73, 1, 263, 241, 196, 308, 91, 293, 259, 342, 360, 212, 223, 145, 90, 321, 21, 81, 329, 20, 338, 383, 36, 247, 193, 123, 343, 307, 350, 100, 173, 303, 168, 165, 334, 157, 344, 63, 14, 89, 253, 68, 369, 5, 146, 268, 289, 331, 102, 134, 179, 351, 330, 211, 67, 272, 363, 374, 148, 306, 149, 155, 103, 214, 288, 311, 339, 256, 37, 230, 162, 12, 174, 239, 185, 282, 96, 159, 222, 355, 4, 158, 94, 273, 30, 84, 277, 160, 166, 358, 336, 300, 164, 192, 48, 167, 225, 170, 161, 40, 182, 201, 213, 234, 368, 186, 208, 249, 285, 250, 0, 72, 296, 219, 9, 197, 41, 51, 71, 184, 317, 50, 183, 65, 299, 80, 187, 373, 35, 200, 118, 172, 352, 152, 52, 238, 176, 295, 283, 320, 121, 301, 181, 337, 2, 144, 54, 153, 210, 304, 58, 353, 252, 356, 204, 189, 280, 266, 233, 46, 3, 122, 335, 26, 346, 15, 258, 64, 275, 323, 261, 226, 364, 139, 206, 86, 17, 6, 34, 83, 207, 156, 242, 110, 236, 29, 297, 365, 24, 31, 294, 108, 279, 79, 195, 178, 114, 177, 154, 115, 130, 240, 377, 190, 376, 382, 132, 33, 129, 232, 85, 27, 357, 379, 25, 109, 318, 45, 313, 298, 113, 209, 62, 372, 119, 217, 16, 203, 281, 333, 202, 270, 381, 70, 366, 194, 265, 18, 246, 221, 126, 101, 22, 78, 66, 237, 269, 7, 243, 345, 274, 98, 38, 218, 128, 198, 107, 105, 106, 314, 112, 74, 97, 278, 354, 140, 362, 370, 244, 169, 302, 111, 147, 224, 133, 276, 44, 347, 227, 28, 43, 23, 324, 264, 151, 216, 93, 141, 367, 57, 215, 171, 138, 99, 248, 380, 312, 348, 284, 77, 231, 61, 32, 116, 87, 56, 75, 315, 378, 327, 267, 340, 39, 254, 163, 175, 220, 131, 287, 127, 305, 322, 245, 53, 104, 191, 143, 10, 188, 82, 349, 55, 180, 286, 255, 262, 69, 92, 8, 117, 19, 124, 47, 76, 11, 136, 49, 316, 271, 359, 199, 229, 135, 341, 60, 142, 59, 328, 371, 42, 125, 332, 95, 13, 257, 326, 375, 235, 205, 292, 251, 88, 120, 361, 150, 260, 228, 291, 290] + 11: [130, 221, 126, 299, 149, 190, 29, 218, 192, 141, 39, 20, 13, 250, 283, 107, 348, 92, 195, 86, 341, 255, 101, 327, 63, 30, 338, 179, 125, 264, 113, 62, 143, 184, 360, 222, 240, 132, 380, 229, 180, 181, 202, 123, 220, 321, 134, 243, 314, 235, 100, 77, 363, 145, 102, 22, 247, 330, 95, 381, 207, 320, 161, 45, 249, 226, 315, 340, 316, 14, 208, 301, 219, 59, 353, 193, 234, 383, 142, 263, 230, 271, 266, 112, 310, 167, 68, 183, 60, 53, 28, 239, 223, 25, 109, 12, 296, 150, 248, 156, 65, 119, 300, 236, 217, 343, 286, 318, 17, 297, 47, 280, 2, 373, 172, 151, 89, 298, 186, 270, 211, 168, 354, 349, 291, 293, 174, 182, 46, 194, 375, 210, 21, 214, 278, 277, 175, 369, 189, 276, 281, 72, 191, 335, 163, 85, 44, 252, 3, 99, 325, 345, 351, 284, 73, 177, 154, 313, 6, 124, 127, 159, 166, 374, 203, 96, 254, 242, 294, 245, 359, 91, 164, 57, 129, 188, 309, 274, 103, 106, 4, 140, 304, 328, 216, 139, 51, 206, 334, 50, 308, 269, 110, 215, 116, 260, 88, 233, 267, 332, 331, 128, 204, 282, 136, 23, 319, 170, 262, 75, 347, 10, 224, 289, 41, 81, 336, 279, 364, 302, 187, 64, 43, 7, 74, 231, 5, 323, 201, 322, 305, 379, 344, 246, 253, 76, 212, 137, 307, 241, 98, 83, 11, 357, 40, 326, 199, 24, 152, 165, 131, 228, 205, 303, 94, 367, 69, 117, 176, 56, 49, 197, 259, 35, 147, 292, 378, 158, 118, 115, 48, 18, 138, 358, 37, 84, 288, 290, 71, 342, 361, 352, 111, 82, 153, 256, 32, 42, 377, 90, 121, 15, 227, 38, 162, 0, 54, 178, 244, 146, 169, 196, 346, 213, 370, 80, 26, 251, 133, 365, 58, 209, 362, 366, 34, 135, 324, 79, 258, 355, 237, 382, 108, 306, 173, 287, 268, 114, 238, 317, 333, 155, 329, 273, 261, 9, 339, 87, 36, 285, 376, 171, 157, 144, 275, 52, 70, 337, 148, 78, 66, 371, 350, 27, 93, 160, 198, 55, 31, 97, 295, 265, 312, 311, 67, 120, 33, 225, 8, 19, 200, 356, 185, 104, 272, 1, 368, 16, 372, 122, 232, 257, 61, 105] + 12: [255, 35, 167, 22, 103, 306, 170, 75, 332, 152, 368, 10, 39, 101, 271, 129, 261, 318, 358, 357, 200, 231, 119, 256, 249, 325, 38, 121, 20, 293, 329, 148, 352, 44, 34, 343, 299, 247, 182, 269, 228, 88, 166, 66, 222, 220, 302, 42, 204, 177, 203, 381, 168, 296, 233, 369, 176, 195, 205, 348, 229, 277, 153, 102, 107, 135, 326, 14, 286, 142, 1, 315, 61, 62, 161, 321, 76, 154, 65, 32, 244, 217, 85, 252, 316, 113, 117, 320, 210, 145, 242, 100, 359, 239, 53, 225, 120, 82, 333, 48, 175, 289, 192, 361, 354, 339, 68, 194, 25, 60, 2, 300, 322, 294, 291, 157, 155, 376, 9, 191, 45, 138, 280, 58, 11, 40, 13, 183, 365, 263, 274, 310, 345, 131, 237, 125, 79, 26, 163, 206, 377, 240, 162, 272, 130, 223, 172, 363, 344, 349, 97, 181, 208, 295, 382, 330, 87, 16, 209, 371, 362, 63, 337, 23, 15, 201, 350, 41, 29, 298, 139, 187, 122, 159, 257, 86, 186, 128, 383, 105, 375, 55, 212, 258, 307, 338, 360, 213, 21, 331, 73, 110, 158, 127, 150, 49, 17, 124, 290, 51, 340, 134, 137, 215, 147, 89, 288, 116, 57, 164, 380, 74, 283, 253, 188, 114, 165, 67, 4, 173, 214, 270, 379, 112, 284, 207, 106, 146, 282, 96, 81, 230, 143, 94, 202, 248, 95, 56, 199, 372, 327, 303, 243, 156, 28, 109, 126, 50, 98, 259, 281, 216, 317, 108, 133, 160, 171, 180, 241, 59, 144, 193, 227, 52, 234, 224, 136, 232, 335, 115, 351, 12, 336, 246, 374, 70, 251, 178, 347, 236, 196, 111, 132, 328, 92, 324, 373, 250, 149, 190, 378, 46, 245, 267, 71, 90, 198, 123, 278, 346, 268, 323, 27, 33, 264, 197, 308, 31, 91, 304, 370, 185, 0, 84, 313, 37, 169, 235, 36, 140, 5, 297, 24, 174, 30, 8, 54, 72, 367, 184, 221, 342, 275, 64, 353, 366, 78, 226, 69, 341, 219, 292, 287, 151, 260, 3, 6, 276, 218, 266, 364, 265, 104, 273, 80, 238, 301, 309, 77, 254, 83, 189, 262, 356, 319, 355, 305, 334, 179, 312, 279, 285, 47, 93, 18, 211, 19, 7, 118, 314, 311, 141, 99, 43] + 13: [170, 274, 3, 41, 276, 330, 366, 370, 159, 176, 184, 31, 202, 143, 320, 6, 207, 355, 141, 145, 74, 333, 243, 221, 105, 161, 326, 84, 296, 50, 250, 72, 115, 150, 103, 138, 346, 101, 28, 278, 293, 236, 47, 351, 314, 126, 192, 175, 108, 23, 30, 77, 86, 135, 100, 310, 11, 306, 118, 58, 75, 273, 309, 67, 178, 300, 79, 179, 382, 361, 151, 177, 164, 303, 267, 379, 220, 305, 325, 308, 270, 217, 287, 104, 233, 383, 117, 63, 90, 181, 269, 354, 120, 277, 140, 155, 232, 328, 238, 57, 96, 231, 254, 197, 88, 171, 279, 295, 369, 356, 246, 292, 373, 378, 219, 237, 144, 283, 374, 129, 52, 8, 331, 153, 174, 257, 348, 71, 154, 234, 213, 214, 185, 167, 284, 212, 152, 224, 272, 368, 160, 66, 33, 230, 18, 149, 12, 194, 265, 282, 55, 102, 338, 162, 323, 227, 364, 275, 261, 158, 208, 22, 10, 43, 239, 225, 93, 183, 380, 34, 163, 106, 357, 268, 59, 113, 358, 13, 124, 119, 182, 372, 336, 352, 258, 347, 341, 156, 82, 193, 315, 316, 195, 14, 260, 81, 0, 201, 110, 17, 199, 46, 131, 45, 16, 289, 294, 281, 334, 65, 147, 62, 25, 262, 166, 327, 222, 125, 89, 68, 345, 92, 245, 187, 123, 19, 313, 190, 205, 85, 226, 60, 264, 20, 317, 148, 255, 247, 350, 78, 87, 114, 280, 9, 169, 21, 342, 271, 142, 298, 35, 198, 112, 297, 91, 229, 299, 210, 375, 321, 26, 291, 253, 256, 76, 241, 288, 248, 136, 259, 244, 133, 360, 73, 235, 353, 285, 157, 252, 203, 99, 251, 332, 200, 322, 165, 324, 7, 80, 329, 189, 318, 24, 172, 209, 134, 186, 109, 116, 111, 188, 69, 40, 363, 2, 42, 168, 240, 228, 196, 359, 139, 15, 223, 36, 337, 56, 242, 263, 311, 70, 38, 5, 365, 83, 4, 343, 191, 128, 97, 349, 302, 367, 206, 27, 32, 44, 362, 312, 137, 39, 51, 95, 340, 335, 304, 307, 64, 49, 376, 54, 266, 122, 98, 211, 339, 344, 107, 121, 146, 29, 37, 48, 286, 319, 127, 381, 53, 94, 377, 130, 61, 249, 180, 1, 215, 173, 371, 132, 290, 218, 301, 216, 204] + 14: [229, 355, 274, 8, 268, 348, 228, 95, 342, 154, 279, 41, 13, 158, 123, 319, 68, 37, 339, 286, 296, 90, 156, 237, 200, 189, 12, 183, 96, 285, 302, 329, 127, 74, 305, 179, 45, 366, 139, 140, 144, 65, 280, 53, 292, 249, 260, 174, 160, 209, 378, 261, 60, 239, 146, 141, 117, 371, 269, 44, 166, 33, 149, 163, 324, 145, 266, 169, 256, 226, 116, 221, 306, 316, 89, 282, 244, 376, 94, 321, 309, 267, 361, 190, 258, 7, 62, 180, 107, 219, 188, 303, 210, 351, 295, 289, 345, 2, 327, 147, 137, 17, 118, 142, 32, 97, 162, 250, 216, 197, 59, 246, 69, 370, 22, 333, 349, 227, 373, 105, 330, 225, 335, 148, 224, 251, 106, 214, 257, 365, 334, 19, 299, 80, 293, 185, 233, 252, 113, 54, 340, 67, 315, 165, 255, 30, 126, 56, 287, 383, 150, 99, 52, 39, 213, 111, 294, 208, 135, 206, 184, 159, 301, 133, 325, 234, 5, 377, 298, 0, 115, 191, 350, 240, 28, 6, 153, 195, 238, 16, 230, 132, 55, 367, 236, 283, 170, 131, 352, 98, 130, 331, 70, 110, 34, 71, 372, 49, 353, 23, 85, 177, 61, 182, 157, 270, 25, 328, 288, 326, 167, 104, 203, 245, 101, 347, 176, 253, 202, 242, 356, 248, 311, 173, 278, 338, 20, 9, 312, 31, 47, 35, 181, 307, 51, 243, 232, 119, 73, 50, 211, 204, 259, 308, 77, 112, 186, 254, 198, 231, 360, 21, 354, 220, 379, 343, 357, 193, 108, 346, 102, 109, 222, 129, 276, 359, 223, 152, 364, 43, 42, 262, 358, 36, 78, 58, 263, 281, 57, 92, 11, 247, 72, 27, 29, 138, 318, 88, 143, 76, 122, 275, 207, 175, 337, 235, 124, 64, 317, 271, 26, 125, 161, 215, 332, 369, 363, 382, 128, 178, 82, 380, 284, 86, 168, 264, 1, 63, 310, 273, 375, 46, 136, 14, 322, 194, 66, 84, 81, 320, 192, 368, 38, 196, 83, 121, 4, 15, 75, 155, 164, 199, 344, 151, 100, 3, 79, 374, 265, 314, 297, 323, 291, 201, 277, 40, 187, 172, 48, 241, 341, 336, 300, 313, 217, 93, 10, 114, 103, 171, 304, 290, 87, 18, 362, 24, 120, 381, 91, 134, 218, 272, 212, 205] + 15: [130, 258, 68, 259, 39, 236, 215, 80, 151, 56, 277, 174, 143, 164, 162, 78, 182, 247, 212, 86, 150, 282, 81, 43, 382, 347, 53, 137, 45, 37, 337, 116, 218, 91, 73, 129, 126, 380, 361, 299, 304, 110, 21, 336, 245, 144, 260, 270, 11, 200, 298, 295, 300, 159, 146, 237, 376, 83, 342, 177, 111, 115, 36, 47, 5, 194, 102, 263, 34, 331, 254, 165, 30, 226, 308, 208, 55, 64, 95, 213, 54, 79, 161, 158, 140, 280, 171, 44, 178, 243, 122, 220, 348, 322, 368, 276, 0, 355, 104, 350, 371, 326, 284, 60, 221, 16, 113, 311, 138, 24, 225, 335, 323, 305, 2, 248, 229, 50, 233, 253, 141, 306, 354, 244, 204, 71, 172, 257, 190, 147, 197, 188, 149, 310, 292, 154, 179, 286, 302, 345, 19, 6, 325, 85, 15, 155, 148, 76, 199, 332, 252, 20, 360, 48, 193, 135, 255, 379, 285, 319, 372, 42, 275, 142, 114, 69, 8, 12, 294, 67, 72, 234, 224, 211, 334, 265, 278, 210, 303, 262, 201, 327, 136, 98, 183, 108, 288, 249, 315, 75, 366, 328, 88, 324, 175, 340, 291, 231, 119, 9, 329, 40, 344, 163, 320, 235, 106, 357, 49, 145, 279, 309, 77, 96, 206, 13, 339, 118, 381, 289, 272, 92, 375, 167, 3, 239, 369, 370, 312, 7, 124, 181, 41, 307, 156, 214, 46, 25, 93, 356, 359, 251, 217, 261, 367, 185, 333, 90, 317, 23, 296, 256, 242, 196, 352, 351, 170, 238, 105, 117, 87, 268, 205, 341, 377, 349, 321, 267, 314, 169, 283, 316, 240, 65, 378, 184, 14, 187, 58, 112, 66, 103, 346, 219, 297, 152, 264, 281, 160, 362, 128, 290, 202, 38, 74, 121, 232, 139, 353, 168, 241, 109, 31, 338, 133, 28, 101, 59, 269, 10, 22, 250, 271, 313, 180, 61, 365, 123, 383, 358, 63, 273, 274, 89, 374, 176, 293, 222, 195, 120, 17, 125, 132, 27, 131, 127, 227, 57, 94, 134, 4, 189, 266, 203, 51, 157, 35, 173, 1, 18, 230, 207, 100, 209, 246, 166, 82, 343, 62, 363, 301, 191, 107, 97, 99, 216, 26, 373, 84, 52, 364, 153, 32, 29, 318, 330, 228, 223, 70, 287, 186, 33, 198, 192] + 16: [134, 118, 330, 185, 49, 342, 302, 383, 187, 235, 123, 294, 306, 2, 248, 73, 58, 153, 219, 356, 42, 301, 96, 188, 252, 200, 102, 333, 343, 109, 207, 363, 81, 373, 258, 371, 241, 61, 243, 63, 40, 254, 326, 361, 10, 320, 156, 249, 15, 303, 67, 331, 86, 318, 78, 283, 189, 270, 212, 65, 365, 307, 99, 21, 56, 380, 227, 70, 379, 9, 4, 106, 29, 336, 214, 337, 93, 105, 201, 62, 14, 128, 113, 110, 256, 138, 251, 355, 74, 329, 327, 362, 53, 94, 366, 257, 286, 44, 68, 125, 338, 289, 250, 298, 287, 87, 347, 285, 310, 97, 37, 103, 281, 311, 8, 167, 36, 160, 370, 124, 228, 358, 211, 242, 83, 378, 240, 163, 88, 284, 131, 117, 279, 210, 260, 6, 349, 136, 192, 204, 133, 111, 368, 353, 120, 7, 269, 11, 154, 129, 5, 374, 255, 213, 146, 346, 27, 314, 115, 38, 119, 139, 282, 34, 195, 45, 19, 339, 104, 264, 158, 18, 95, 272, 55, 193, 172, 127, 324, 168, 170, 359, 230, 288, 79, 351, 173, 197, 222, 161, 126, 1, 275, 92, 64, 271, 300, 143, 259, 224, 265, 141, 367, 16, 89, 321, 237, 51, 175, 155, 31, 76, 220, 122, 174, 238, 225, 354, 208, 54, 178, 304, 190, 266, 186, 157, 47, 261, 116, 50, 332, 169, 382, 130, 341, 72, 278, 191, 247, 98, 100, 262, 165, 280, 277, 144, 348, 183, 171, 364, 246, 180, 108, 177, 328, 377, 292, 159, 231, 182, 232, 334, 276, 22, 317, 151, 340, 313, 253, 267, 244, 30, 381, 166, 107, 20, 216, 77, 149, 352, 164, 12, 350, 179, 296, 345, 309, 344, 140, 234, 43, 162, 114, 90, 205, 35, 24, 295, 236, 199, 85, 84, 59, 316, 82, 0, 41, 184, 229, 202, 293, 322, 198, 221, 25, 57, 206, 319, 48, 142, 194, 233, 145, 268, 308, 176, 150, 33, 28, 112, 299, 263, 325, 13, 215, 3, 369, 152, 147, 75, 312, 273, 203, 274, 209, 121, 46, 226, 181, 52, 71, 335, 80, 66, 39, 305, 357, 372, 137, 297, 290, 375, 132, 60, 223, 323, 91, 196, 360, 315, 218, 17, 26, 217, 245, 101, 32, 23, 135, 291, 376, 69, 239, 148] + 17: [131, 383, 374, 226, 87, 311, 185, 113, 351, 6, 52, 96, 22, 97, 180, 98, 272, 165, 171, 332, 19, 116, 41, 363, 382, 89, 271, 379, 93, 244, 345, 48, 55, 326, 233, 163, 315, 378, 30, 66, 291, 334, 268, 301, 54, 69, 72, 121, 119, 15, 28, 137, 366, 136, 343, 114, 328, 234, 167, 339, 218, 145, 243, 10, 298, 282, 245, 191, 248, 173, 260, 204, 196, 186, 95, 360, 195, 310, 16, 112, 278, 364, 150, 164, 316, 23, 141, 257, 11, 285, 17, 281, 169, 152, 159, 170, 284, 200, 362, 208, 126, 78, 279, 266, 189, 175, 172, 39, 194, 381, 228, 103, 33, 214, 142, 259, 20, 45, 42, 49, 198, 341, 21, 94, 300, 102, 183, 217, 369, 76, 270, 32, 355, 149, 176, 349, 129, 306, 35, 321, 14, 122, 202, 203, 236, 108, 305, 138, 346, 219, 359, 5, 246, 347, 370, 222, 104, 199, 177, 313, 143, 84, 327, 168, 264, 13, 36, 336, 118, 221, 309, 181, 27, 240, 120, 329, 287, 372, 25, 318, 53, 83, 88, 201, 4, 368, 348, 322, 333, 86, 127, 235, 79, 303, 330, 307, 190, 254, 193, 197, 323, 133, 75, 8, 157, 73, 148, 211, 367, 139, 146, 162, 376, 166, 331, 147, 216, 377, 101, 338, 44, 238, 361, 85, 64, 308, 255, 350, 65, 132, 182, 109, 223, 178, 247, 51, 156, 123, 237, 251, 337, 188, 294, 80, 58, 91, 117, 232, 130, 29, 297, 56, 207, 335, 206, 273, 92, 135, 263, 70, 18, 61, 59, 269, 81, 187, 3, 342, 179, 249, 258, 161, 267, 158, 124, 31, 344, 352, 40, 230, 184, 242, 289, 111, 209, 174, 283, 312, 375, 250, 26, 105, 319, 357, 225, 275, 1, 324, 276, 290, 134, 296, 231, 380, 371, 256, 74, 9, 340, 47, 60, 7, 154, 365, 37, 253, 265, 262, 299, 210, 128, 106, 192, 63, 57, 153, 227, 277, 224, 160, 71, 353, 252, 293, 320, 325, 261, 220, 100, 212, 140, 12, 155, 286, 2, 125, 62, 107, 317, 46, 24, 38, 373, 99, 292, 314, 82, 304, 43, 215, 295, 115, 205, 213, 110, 302, 90, 239, 151, 274, 67, 50, 144, 241, 280, 356, 229, 288, 77, 0, 68, 34, 358, 354] + 18: [271, 169, 136, 26, 31, 144, 285, 3, 165, 46, 366, 73, 359, 233, 109, 22, 201, 50, 8, 176, 150, 9, 335, 329, 243, 179, 348, 82, 220, 311, 209, 76, 95, 354, 238, 294, 249, 290, 62, 198, 226, 222, 344, 1, 32, 59, 78, 147, 30, 286, 215, 110, 236, 68, 237, 297, 99, 326, 188, 274, 346, 339, 177, 149, 131, 0, 278, 58, 157, 349, 279, 81, 63, 267, 153, 325, 256, 175, 332, 259, 197, 375, 364, 86, 44, 352, 230, 18, 199, 323, 186, 80, 227, 382, 309, 340, 251, 65, 185, 193, 211, 347, 380, 189, 321, 306, 282, 266, 67, 70, 89, 41, 342, 83, 87, 334, 367, 232, 250, 235, 308, 205, 276, 132, 363, 148, 324, 127, 383, 158, 66, 224, 381, 2, 107, 85, 29, 377, 277, 336, 196, 353, 97, 242, 208, 245, 106, 356, 210, 154, 119, 116, 355, 376, 284, 330, 33, 124, 269, 371, 202, 331, 79, 225, 137, 7, 328, 289, 15, 125, 244, 12, 13, 351, 112, 108, 60, 216, 4, 56, 229, 312, 172, 90, 184, 14, 117, 72, 134, 379, 120, 268, 47, 300, 138, 260, 53, 28, 36, 174, 146, 139, 151, 35, 190, 98, 181, 20, 141, 283, 262, 88, 162, 288, 313, 102, 140, 178, 93, 145, 187, 94, 214, 213, 234, 338, 51, 126, 317, 75, 258, 115, 273, 24, 160, 173, 194, 49, 327, 293, 10, 257, 228, 272, 45, 315, 304, 142, 167, 34, 357, 341, 105, 103, 19, 365, 23, 307, 223, 263, 241, 350, 361, 39, 195, 38, 96, 182, 292, 270, 166, 303, 143, 170, 219, 130, 369, 180, 372, 161, 280, 345, 212, 206, 221, 55, 374, 314, 43, 217, 71, 118, 16, 207, 368, 192, 25, 360, 291, 378, 77, 204, 318, 337, 191, 104, 343, 129, 135, 159, 302, 152, 111, 264, 362, 40, 155, 287, 254, 48, 358, 64, 27, 17, 123, 218, 42, 370, 281, 5, 305, 252, 295, 91, 52, 156, 69, 320, 57, 6, 333, 113, 114, 239, 301, 373, 253, 246, 101, 240, 164, 171, 248, 121, 261, 316, 296, 122, 231, 61, 203, 54, 319, 265, 255, 322, 92, 275, 168, 200, 247, 163, 133, 298, 84, 37, 299, 100, 11, 21, 310, 128, 74, 183] + 19: [24, 207, 243, 302, 188, 219, 184, 218, 227, 159, 35, 158, 54, 71, 278, 173, 68, 251, 204, 2, 277, 273, 346, 237, 131, 75, 280, 146, 326, 212, 22, 186, 340, 351, 103, 112, 267, 39, 224, 86, 56, 226, 55, 142, 378, 355, 369, 375, 187, 313, 307, 111, 52, 183, 300, 323, 38, 97, 275, 58, 25, 335, 1, 134, 6, 76, 368, 257, 347, 215, 203, 18, 333, 202, 160, 3, 191, 383, 311, 32, 232, 174, 63, 37, 150, 269, 214, 252, 163, 236, 282, 78, 145, 93, 50, 81, 190, 330, 240, 206, 241, 322, 114, 363, 164, 271, 153, 264, 171, 238, 30, 295, 127, 172, 16, 272, 67, 345, 374, 125, 284, 262, 293, 26, 309, 242, 298, 87, 258, 235, 94, 59, 231, 289, 223, 304, 266, 248, 360, 104, 175, 109, 98, 107, 366, 338, 64, 179, 91, 126, 197, 328, 349, 77, 354, 44, 292, 122, 13, 222, 359, 83, 195, 138, 40, 31, 95, 287, 108, 380, 372, 62, 140, 329, 82, 139, 370, 247, 149, 73, 348, 123, 178, 367, 0, 5, 151, 72, 196, 229, 133, 198, 7, 120, 101, 339, 364, 291, 8, 113, 20, 303, 301, 217, 147, 135, 36, 166, 165, 96, 353, 221, 260, 249, 143, 84, 19, 23, 356, 65, 225, 362, 137, 209, 14, 208, 286, 294, 357, 162, 185, 21, 155, 261, 17, 199, 250, 305, 245, 253, 90, 377, 312, 211, 61, 161, 192, 66, 154, 296, 230, 189, 365, 15, 233, 352, 182, 167, 29, 283, 144, 4, 244, 325, 177, 45, 148, 47, 281, 119, 130, 168, 132, 234, 314, 315, 106, 11, 201, 51, 205, 308, 43, 220, 274, 317, 254, 263, 288, 57, 42, 141, 74, 255, 318, 373, 193, 334, 152, 79, 46, 246, 116, 310, 337, 270, 156, 239, 157, 110, 306, 336, 213, 10, 331, 129, 376, 321, 176, 320, 181, 332, 324, 361, 118, 124, 290, 100, 299, 379, 69, 327, 276, 350, 268, 319, 256, 265, 180, 115, 48, 70, 279, 216, 27, 33, 89, 117, 9, 41, 53, 285, 128, 228, 121, 49, 170, 297, 259, 342, 194, 85, 80, 60, 92, 88, 200, 341, 343, 99, 105, 28, 371, 381, 210, 358, 136, 344, 102, 169, 382, 316, 12, 34] + 20: [161, 174, 123, 307, 204, 330, 10, 273, 382, 140, 292, 92, 71, 322, 240, 35, 375, 314, 58, 18, 22, 86, 169, 15, 33, 73, 202, 89, 351, 210, 151, 115, 124, 329, 47, 136, 48, 234, 147, 103, 31, 298, 242, 27, 51, 52, 54, 61, 116, 4, 272, 287, 291, 32, 162, 164, 290, 96, 259, 258, 172, 126, 315, 194, 348, 26, 30, 359, 42, 256, 165, 246, 20, 78, 304, 231, 191, 94, 154, 156, 250, 321, 333, 67, 211, 64, 261, 149, 129, 152, 1, 197, 76, 107, 137, 167, 334, 87, 335, 14, 376, 130, 62, 371, 324, 128, 187, 342, 347, 65, 74, 284, 218, 361, 368, 213, 372, 238, 340, 294, 338, 106, 365, 312, 212, 159, 217, 352, 283, 28, 356, 72, 225, 133, 293, 192, 186, 155, 38, 269, 168, 179, 181, 189, 98, 99, 145, 37, 173, 112, 353, 188, 148, 141, 311, 362, 216, 101, 358, 357, 177, 328, 41, 36, 150, 208, 34, 166, 370, 310, 252, 39, 3, 374, 254, 316, 59, 199, 40, 7, 271, 16, 63, 373, 220, 222, 203, 44, 288, 200, 206, 214, 153, 193, 268, 122, 110, 226, 363, 185, 209, 229, 381, 180, 286, 49, 121, 6, 364, 235, 175, 146, 367, 119, 114, 60, 248, 377, 236, 257, 326, 337, 308, 56, 349, 21, 296, 163, 355, 19, 157, 383, 251, 275, 237, 69, 17, 249, 262, 263, 278, 66, 45, 8, 81, 2, 105, 138, 325, 111, 125, 264, 227, 132, 77, 90, 176, 343, 118, 0, 233, 300, 274, 346, 297, 29, 127, 170, 201, 24, 253, 285, 142, 289, 113, 85, 323, 305, 134, 68, 245, 301, 244, 109, 117, 260, 266, 281, 302, 144, 369, 282, 55, 339, 224, 270, 158, 91, 221, 50, 160, 184, 182, 239, 57, 267, 219, 25, 247, 178, 131, 215, 345, 12, 100, 332, 327, 84, 75, 80, 366, 120, 319, 205, 341, 83, 207, 108, 295, 306, 255, 143, 198, 277, 299, 303, 336, 82, 43, 46, 317, 379, 195, 318, 93, 228, 95, 9, 135, 320, 104, 190, 13, 378, 265, 53, 309, 279, 196, 243, 97, 5, 232, 380, 360, 223, 230, 139, 11, 79, 88, 102, 354, 276, 313, 183, 70, 23, 344, 171, 241, 280, 331, 350] + 21: [360, 80, 302, 206, 210, 364, 88, 345, 0, 240, 16, 26, 366, 69, 9, 189, 234, 325, 197, 336, 362, 318, 62, 286, 165, 30, 65, 301, 14, 102, 258, 335, 333, 305, 105, 342, 207, 130, 85, 37, 176, 6, 326, 95, 235, 311, 10, 17, 152, 158, 140, 247, 82, 208, 94, 377, 182, 320, 22, 349, 299, 280, 43, 23, 245, 131, 84, 18, 259, 262, 97, 303, 125, 378, 40, 100, 136, 104, 346, 48, 244, 334, 163, 241, 143, 41, 76, 56, 249, 106, 7, 116, 248, 327, 35, 58, 355, 4, 113, 256, 53, 185, 199, 232, 252, 250, 172, 351, 219, 227, 144, 220, 368, 229, 168, 1, 64, 223, 260, 159, 370, 5, 77, 193, 160, 198, 146, 276, 273, 381, 181, 253, 156, 74, 129, 180, 255, 31, 161, 166, 322, 332, 93, 123, 47, 254, 71, 236, 38, 218, 348, 214, 118, 149, 307, 147, 15, 194, 230, 111, 183, 243, 63, 49, 90, 309, 312, 28, 317, 323, 204, 171, 187, 170, 340, 300, 379, 110, 195, 164, 70, 330, 217, 314, 296, 142, 278, 188, 343, 11, 42, 264, 173, 308, 257, 190, 50, 112, 117, 277, 178, 238, 380, 19, 196, 225, 135, 221, 33, 174, 73, 267, 138, 60, 98, 192, 354, 133, 237, 148, 137, 203, 329, 216, 81, 83, 319, 294, 75, 13, 68, 114, 328, 145, 298, 212, 101, 127, 44, 306, 281, 371, 119, 46, 86, 87, 169, 226, 284, 51, 191, 128, 8, 25, 313, 141, 288, 270, 338, 331, 39, 304, 66, 383, 290, 154, 275, 266, 292, 269, 57, 261, 213, 134, 12, 315, 239, 374, 99, 324, 202, 242, 32, 222, 107, 167, 108, 321, 132, 20, 376, 153, 29, 92, 361, 120, 150, 279, 184, 109, 121, 293, 209, 367, 373, 89, 372, 358, 287, 246, 21, 356, 155, 3, 295, 341, 337, 24, 274, 126, 157, 27, 67, 382, 268, 2, 357, 54, 162, 265, 59, 231, 224, 283, 179, 339, 103, 316, 124, 289, 205, 200, 363, 45, 91, 177, 151, 365, 175, 344, 52, 72, 78, 233, 263, 352, 359, 139, 251, 291, 211, 347, 353, 115, 34, 79, 186, 36, 215, 271, 96, 310, 201, 122, 55, 375, 228, 285, 61, 282, 272, 297, 369, 350] + 22: [254, 376, 198, 34, 242, 112, 44, 355, 252, 70, 188, 111, 8, 222, 125, 170, 361, 366, 220, 233, 133, 367, 305, 200, 54, 256, 81, 300, 96, 353, 265, 246, 137, 282, 62, 257, 302, 150, 370, 304, 114, 270, 231, 10, 205, 2, 28, 79, 141, 224, 94, 126, 207, 184, 263, 244, 130, 348, 345, 311, 67, 83, 60, 185, 14, 3, 48, 25, 71, 12, 320, 318, 322, 283, 373, 241, 97, 136, 26, 103, 119, 182, 30, 91, 342, 344, 223, 6, 21, 146, 293, 234, 135, 249, 95, 101, 181, 24, 178, 204, 332, 73, 37, 329, 61, 55, 272, 218, 100, 378, 174, 193, 122, 228, 53, 258, 15, 99, 72, 47, 65, 189, 238, 45, 13, 271, 46, 208, 274, 151, 334, 129, 350, 128, 212, 88, 43, 158, 107, 243, 142, 280, 143, 154, 221, 148, 20, 33, 330, 115, 230, 235, 362, 317, 229, 149, 306, 161, 123, 214, 27, 113, 156, 49, 374, 210, 380, 87, 301, 199, 327, 299, 138, 314, 379, 262, 35, 160, 92, 145, 16, 168, 354, 162, 197, 163, 121, 368, 192, 287, 157, 217, 260, 209, 110, 9, 132, 164, 177, 139, 383, 240, 351, 338, 336, 68, 106, 116, 349, 358, 339, 191, 309, 165, 105, 248, 303, 269, 78, 32, 268, 202, 364, 279, 85, 187, 120, 152, 64, 69, 255, 23, 140, 277, 166, 56, 297, 289, 247, 250, 261, 278, 183, 371, 276, 38, 144, 29, 117, 352, 357, 7, 80, 372, 312, 369, 226, 175, 109, 18, 321, 59, 1, 39, 98, 227, 335, 382, 211, 215, 356, 298, 102, 219, 196, 179, 291, 90, 296, 326, 284, 295, 167, 86, 313, 292, 253, 267, 328, 343, 288, 316, 147, 42, 4, 169, 58, 333, 381, 285, 375, 195, 264, 17, 57, 75, 340, 63, 324, 225, 319, 341, 236, 124, 377, 180, 290, 77, 273, 176, 108, 266, 294, 190, 19, 104, 308, 5, 127, 66, 172, 194, 315, 323, 307, 310, 281, 216, 337, 51, 331, 201, 74, 118, 93, 50, 153, 251, 173, 76, 365, 259, 206, 347, 82, 40, 131, 41, 239, 159, 155, 22, 11, 0, 286, 89, 275, 237, 232, 186, 52, 36, 134, 171, 31, 363, 213, 84, 245, 203, 325, 360, 346, 359] + 23: [26, 135, 376, 19, 240, 324, 47, 374, 127, 148, 92, 340, 80, 70, 233, 23, 228, 292, 210, 298, 329, 231, 333, 223, 284, 258, 344, 164, 16, 14, 297, 239, 369, 214, 161, 9, 62, 165, 171, 299, 345, 89, 145, 186, 234, 189, 37, 64, 314, 167, 177, 211, 331, 203, 287, 94, 11, 218, 315, 263, 35, 213, 220, 149, 10, 290, 129, 157, 119, 66, 5, 307, 316, 76, 73, 200, 174, 68, 155, 236, 115, 104, 248, 61, 357, 31, 377, 343, 30, 117, 142, 380, 3, 216, 168, 288, 124, 187, 72, 274, 170, 24, 197, 154, 192, 182, 86, 41, 283, 326, 249, 191, 313, 224, 40, 222, 150, 96, 125, 178, 347, 352, 58, 363, 302, 206, 121, 349, 338, 139, 267, 46, 202, 34, 383, 181, 364, 42, 278, 198, 60, 366, 43, 81, 172, 132, 244, 281, 201, 269, 358, 176, 180, 59, 2, 56, 133, 146, 378, 91, 317, 237, 311, 356, 153, 160, 65, 301, 330, 250, 143, 109, 335, 212, 354, 362, 25, 375, 106, 188, 319, 88, 266, 116, 138, 208, 83, 217, 296, 4, 15, 242, 275, 100, 323, 120, 147, 82, 327, 261, 67, 372, 351, 130, 190, 144, 102, 312, 152, 272, 136, 361, 370, 17, 52, 325, 341, 204, 348, 280, 107, 7, 134, 99, 285, 355, 79, 367, 353, 230, 322, 247, 20, 141, 183, 293, 13, 251, 8, 235, 71, 175, 300, 225, 295, 87, 103, 156, 84, 55, 6, 371, 209, 50, 74, 36, 195, 342, 221, 44, 32, 365, 53, 137, 162, 193, 45, 0, 245, 108, 114, 243, 336, 373, 205, 49, 253, 276, 337, 334, 22, 318, 246, 112, 166, 29, 291, 265, 305, 264, 252, 69, 77, 173, 12, 232, 304, 57, 159, 78, 309, 111, 184, 229, 95, 194, 255, 101, 1, 93, 98, 207, 382, 196, 54, 128, 256, 21, 163, 268, 97, 33, 308, 140, 310, 277, 350, 259, 118, 289, 105, 262, 18, 110, 39, 286, 126, 328, 85, 51, 48, 306, 279, 27, 241, 169, 270, 257, 131, 379, 219, 151, 271, 38, 215, 359, 185, 254, 273, 179, 321, 303, 90, 339, 28, 346, 122, 294, 123, 227, 260, 282, 158, 368, 238, 360, 381, 63, 199, 332, 226, 113, 75, 320] + 24: [154, 357, 290, 172, 360, 112, 334, 186, 283, 159, 354, 91, 226, 274, 356, 49, 380, 3, 206, 63, 169, 295, 116, 11, 239, 60, 346, 282, 271, 296, 76, 161, 281, 160, 171, 291, 25, 18, 187, 240, 165, 316, 208, 364, 36, 374, 74, 81, 265, 263, 87, 259, 28, 286, 255, 230, 53, 135, 277, 382, 273, 260, 377, 39, 345, 118, 221, 321, 235, 323, 313, 280, 65, 225, 37, 170, 348, 10, 363, 320, 57, 331, 217, 317, 101, 68, 247, 250, 167, 340, 214, 17, 152, 133, 115, 166, 252, 372, 97, 210, 381, 64, 301, 207, 26, 204, 192, 15, 41, 106, 138, 336, 224, 307, 58, 297, 84, 168, 242, 315, 194, 30, 103, 4, 38, 14, 173, 249, 373, 163, 262, 342, 180, 184, 378, 300, 199, 347, 261, 40, 219, 158, 198, 241, 9, 22, 85, 328, 139, 361, 104, 375, 343, 78, 195, 306, 318, 254, 350, 96, 182, 12, 287, 83, 257, 233, 222, 56, 302, 134, 218, 203, 79, 88, 86, 188, 8, 148, 209, 44, 109, 236, 43, 174, 141, 123, 349, 82, 358, 179, 264, 298, 246, 6, 189, 111, 337, 324, 294, 146, 237, 24, 164, 190, 71, 355, 113, 149, 145, 245, 279, 376, 153, 269, 47, 51, 66, 181, 258, 267, 176, 120, 107, 284, 42, 67, 329, 238, 132, 256, 143, 228, 90, 227, 371, 100, 2, 21, 232, 314, 20, 92, 128, 110, 185, 27, 266, 62, 231, 213, 144, 61, 325, 308, 46, 351, 293, 77, 48, 59, 147, 278, 197, 1, 292, 215, 335, 333, 303, 150, 175, 94, 352, 359, 50, 234, 362, 367, 98, 162, 124, 285, 33, 305, 45, 127, 251, 353, 7, 52, 369, 95, 157, 105, 220, 370, 137, 202, 35, 54, 299, 131, 142, 177, 338, 205, 272, 140, 178, 13, 304, 244, 339, 270, 365, 72, 276, 196, 212, 126, 99, 191, 102, 193, 183, 80, 201, 332, 200, 243, 69, 322, 89, 129, 327, 379, 223, 0, 70, 211, 122, 121, 5, 29, 319, 31, 344, 366, 93, 156, 130, 73, 341, 310, 114, 326, 216, 16, 311, 268, 312, 108, 309, 275, 23, 253, 32, 151, 136, 19, 55, 288, 248, 125, 229, 75, 119, 289, 117, 330, 34, 155, 368, 383] + 25: [315, 30, 309, 77, 145, 305, 32, 120, 287, 215, 84, 240, 89, 343, 135, 359, 369, 304, 81, 130, 184, 202, 133, 14, 105, 300, 361, 127, 248, 22, 34, 122, 96, 293, 118, 349, 1, 286, 88, 104, 148, 106, 338, 233, 263, 123, 91, 158, 269, 168, 101, 2, 246, 232, 273, 102, 230, 342, 294, 197, 231, 153, 60, 97, 4, 291, 336, 237, 312, 259, 37, 28, 181, 355, 212, 251, 193, 260, 51, 281, 362, 283, 208, 289, 322, 265, 119, 172, 217, 272, 241, 194, 43, 19, 277, 326, 379, 192, 268, 375, 25, 376, 333, 7, 296, 378, 100, 131, 373, 76, 244, 146, 161, 143, 225, 319, 199, 345, 185, 328, 201, 94, 11, 356, 74, 166, 186, 177, 157, 21, 45, 40, 128, 137, 95, 57, 26, 301, 50, 329, 382, 239, 267, 178, 282, 372, 62, 298, 78, 0, 112, 5, 8, 318, 351, 381, 252, 377, 229, 54, 126, 218, 196, 347, 132, 221, 354, 290, 297, 257, 69, 370, 53, 27, 179, 271, 35, 325, 70, 210, 46, 236, 142, 63, 264, 71, 108, 198, 13, 270, 156, 220, 72, 23, 284, 258, 85, 39, 113, 49, 160, 38, 59, 262, 346, 9, 56, 256, 367, 321, 348, 242, 20, 203, 68, 55, 357, 331, 58, 310, 31, 200, 17, 358, 109, 163, 86, 124, 226, 175, 245, 155, 3, 147, 12, 285, 182, 6, 44, 121, 139, 276, 107, 98, 299, 61, 327, 10, 280, 66, 216, 18, 154, 222, 341, 80, 366, 279, 261, 140, 211, 295, 350, 204, 308, 134, 254, 144, 24, 79, 223, 243, 337, 75, 36, 167, 364, 371, 64, 41, 205, 360, 42, 90, 266, 170, 255, 165, 228, 180, 29, 83, 334, 162, 152, 171, 190, 33, 314, 368, 302, 234, 209, 99, 138, 227, 307, 114, 191, 278, 316, 324, 330, 365, 339, 149, 67, 238, 249, 275, 111, 292, 250, 183, 87, 311, 174, 224, 48, 141, 335, 169, 288, 151, 195, 129, 247, 317, 214, 65, 82, 332, 52, 73, 235, 313, 380, 207, 136, 306, 253, 125, 323, 164, 176, 103, 115, 352, 187, 383, 15, 213, 116, 274, 150, 159, 340, 110, 353, 117, 93, 320, 173, 16, 92, 188, 344, 363, 374, 219, 47, 206, 303, 189] + 26: [136, 150, 63, 162, 16, 183, 266, 223, 88, 167, 83, 247, 48, 138, 323, 353, 97, 103, 268, 365, 17, 113, 45, 0, 256, 75, 232, 301, 7, 49, 325, 225, 8, 64, 100, 29, 13, 216, 284, 330, 235, 310, 322, 132, 1, 54, 68, 86, 237, 53, 230, 214, 304, 289, 33, 137, 369, 249, 146, 328, 359, 148, 319, 190, 229, 174, 24, 221, 57, 318, 10, 73, 76, 32, 114, 182, 25, 219, 220, 340, 317, 43, 346, 244, 154, 91, 296, 4, 243, 222, 58, 155, 288, 96, 106, 111, 89, 18, 339, 233, 338, 116, 279, 38, 355, 158, 245, 82, 200, 311, 259, 321, 133, 102, 251, 294, 71, 104, 161, 35, 107, 350, 123, 143, 134, 264, 300, 352, 141, 109, 59, 77, 326, 101, 324, 14, 250, 345, 95, 248, 309, 139, 169, 170, 269, 124, 253, 271, 342, 120, 337, 130, 293, 315, 108, 382, 180, 31, 360, 262, 306, 99, 186, 343, 66, 215, 65, 98, 267, 50, 144, 372, 298, 336, 3, 140, 185, 335, 74, 112, 163, 119, 358, 145, 282, 28, 257, 210, 168, 175, 209, 218, 217, 11, 291, 320, 263, 254, 69, 308, 61, 135, 30, 62, 67, 176, 272, 151, 142, 152, 173, 194, 127, 239, 159, 202, 115, 231, 177, 312, 117, 368, 12, 105, 47, 40, 9, 303, 193, 199, 373, 208, 283, 46, 381, 273, 203, 228, 234, 242, 172, 380, 236, 281, 70, 224, 344, 316, 307, 376, 92, 131, 265, 198, 26, 191, 367, 27, 241, 118, 285, 277, 195, 212, 171, 37, 129, 383, 280, 157, 78, 362, 255, 72, 34, 331, 206, 205, 56, 313, 85, 147, 23, 379, 213, 246, 261, 305, 90, 292, 181, 5, 184, 149, 94, 341, 238, 226, 36, 110, 156, 204, 179, 287, 2, 327, 79, 51, 192, 15, 81, 290, 252, 196, 187, 363, 19, 302, 197, 364, 354, 334, 275, 361, 240, 274, 258, 377, 211, 164, 366, 207, 52, 260, 348, 351, 80, 6, 21, 349, 371, 160, 276, 278, 227, 22, 126, 332, 55, 128, 178, 166, 44, 356, 122, 201, 20, 357, 87, 378, 41, 370, 93, 153, 189, 329, 121, 125, 165, 347, 297, 39, 333, 314, 270, 42, 295, 188, 84, 286, 60, 299, 374, 375] + 27: [103, 354, 58, 164, 320, 181, 213, 143, 75, 303, 381, 339, 77, 89, 322, 118, 305, 7, 91, 54, 174, 218, 45, 276, 171, 199, 250, 365, 275, 105, 130, 259, 185, 65, 157, 289, 62, 283, 173, 138, 314, 57, 206, 200, 39, 53, 60, 137, 225, 73, 341, 301, 219, 204, 135, 358, 348, 237, 335, 290, 266, 99, 102, 23, 278, 317, 27, 88, 162, 329, 304, 296, 46, 362, 2, 9, 101, 95, 107, 159, 132, 96, 346, 74, 229, 364, 86, 334, 194, 148, 11, 92, 209, 203, 146, 163, 286, 168, 227, 81, 59, 71, 49, 24, 131, 222, 230, 48, 260, 196, 115, 83, 140, 141, 337, 155, 125, 10, 114, 215, 267, 18, 347, 28, 249, 367, 205, 35, 33, 184, 208, 264, 302, 372, 94, 166, 108, 160, 19, 169, 243, 212, 176, 220, 12, 265, 349, 34, 279, 269, 26, 128, 224, 112, 55, 178, 324, 216, 238, 78, 284, 356, 187, 161, 316, 379, 214, 274, 287, 43, 239, 318, 121, 167, 361, 79, 14, 280, 359, 50, 352, 16, 158, 345, 226, 170, 255, 363, 378, 235, 240, 244, 312, 61, 252, 117, 351, 300, 315, 366, 369, 190, 182, 21, 76, 149, 72, 32, 325, 191, 42, 254, 368, 326, 172, 5, 375, 106, 248, 66, 307, 0, 3, 256, 144, 231, 282, 223, 41, 330, 195, 192, 165, 311, 210, 298, 17, 63, 236, 262, 90, 263, 30, 104, 70, 374, 119, 376, 179, 336, 183, 323, 242, 383, 343, 82, 201, 193, 133, 370, 328, 271, 136, 268, 122, 67, 147, 44, 321, 217, 123, 288, 139, 281, 319, 8, 186, 142, 257, 64, 87, 331, 295, 371, 36, 93, 247, 310, 234, 4, 69, 145, 31, 350, 357, 202, 377, 97, 228, 85, 380, 232, 156, 308, 297, 109, 241, 261, 309, 6, 175, 327, 124, 129, 299, 120, 47, 353, 153, 116, 306, 113, 313, 293, 150, 110, 68, 189, 1, 56, 221, 355, 51, 100, 270, 338, 37, 177, 80, 211, 333, 233, 197, 292, 188, 207, 251, 272, 340, 258, 294, 332, 25, 373, 98, 15, 126, 246, 291, 127, 360, 111, 20, 38, 29, 13, 84, 151, 344, 152, 154, 22, 253, 382, 273, 198, 52, 245, 277, 40, 134, 180, 285, 342] + 28: [161, 0, 324, 120, 122, 7, 108, 191, 71, 58, 352, 339, 41, 240, 217, 367, 380, 43, 184, 78, 46, 245, 44, 98, 177, 36, 104, 363, 24, 294, 96, 80, 38, 284, 347, 232, 174, 140, 65, 53, 87, 262, 10, 18, 60, 86, 115, 150, 250, 25, 355, 77, 176, 288, 37, 75, 166, 223, 158, 63, 160, 28, 289, 105, 56, 19, 114, 341, 68, 282, 260, 264, 57, 159, 205, 296, 230, 133, 280, 331, 117, 213, 39, 131, 146, 29, 281, 277, 189, 254, 124, 62, 138, 156, 180, 207, 26, 302, 99, 312, 22, 226, 103, 307, 369, 378, 325, 204, 100, 157, 67, 337, 209, 271, 273, 127, 268, 134, 14, 183, 171, 208, 295, 375, 74, 66, 170, 73, 136, 89, 348, 225, 246, 229, 55, 76, 111, 126, 354, 164, 186, 147, 215, 257, 185, 306, 52, 4, 196, 152, 233, 224, 263, 219, 368, 51, 261, 90, 167, 163, 221, 182, 238, 278, 16, 1, 121, 365, 265, 64, 305, 148, 106, 210, 211, 151, 382, 364, 323, 110, 292, 276, 252, 101, 123, 141, 360, 198, 212, 173, 279, 311, 168, 255, 30, 197, 310, 203, 172, 113, 272, 50, 137, 227, 165, 235, 135, 220, 32, 340, 376, 48, 316, 91, 118, 142, 49, 81, 314, 143, 322, 12, 334, 154, 377, 218, 266, 251, 303, 237, 47, 373, 102, 304, 83, 309, 301, 190, 329, 336, 344, 317, 107, 321, 239, 188, 195, 253, 357, 128, 3, 228, 200, 88, 285, 326, 269, 13, 33, 193, 206, 178, 27, 244, 6, 249, 274, 362, 61, 258, 332, 297, 372, 330, 287, 293, 8, 17, 192, 35, 291, 371, 145, 313, 2, 40, 298, 338, 92, 70, 243, 241, 248, 328, 97, 93, 199, 236, 320, 42, 5, 129, 361, 242, 299, 216, 349, 315, 300, 333, 194, 256, 327, 358, 69, 381, 169, 270, 342, 34, 109, 31, 119, 345, 231, 72, 139, 267, 45, 202, 181, 318, 9, 54, 359, 343, 356, 153, 162, 286, 201, 283, 155, 214, 374, 346, 95, 59, 116, 383, 130, 370, 125, 20, 308, 247, 335, 222, 290, 112, 179, 23, 82, 21, 175, 15, 84, 366, 144, 79, 149, 351, 234, 85, 259, 275, 94, 350, 187, 319, 11, 132, 379, 353] + 29: [118, 304, 23, 283, 25, 129, 320, 346, 243, 87, 63, 204, 123, 50, 172, 218, 106, 329, 18, 272, 101, 353, 105, 217, 316, 242, 230, 28, 191, 56, 42, 166, 253, 270, 219, 308, 252, 30, 164, 189, 2, 130, 64, 138, 98, 206, 26, 363, 244, 295, 173, 271, 54, 339, 309, 160, 140, 134, 343, 145, 319, 278, 360, 85, 139, 222, 225, 59, 97, 3, 205, 317, 238, 31, 157, 135, 209, 368, 7, 250, 233, 62, 376, 71, 115, 246, 234, 201, 8, 5, 1, 268, 256, 367, 322, 29, 361, 203, 282, 383, 365, 333, 186, 266, 180, 375, 24, 68, 210, 350, 324, 215, 257, 349, 347, 228, 150, 292, 294, 328, 122, 224, 52, 46, 303, 220, 70, 338, 239, 194, 149, 17, 184, 296, 325, 254, 21, 27, 372, 342, 274, 37, 185, 240, 197, 198, 236, 255, 195, 379, 74, 16, 65, 48, 190, 216, 382, 38, 4, 378, 128, 273, 300, 146, 181, 380, 125, 340, 22, 229, 40, 10, 213, 327, 84, 161, 187, 121, 113, 99, 337, 297, 49, 41, 100, 226, 81, 221, 305, 6, 212, 285, 73, 323, 44, 89, 124, 76, 262, 148, 326, 57, 332, 163, 82, 86, 94, 259, 116, 232, 72, 171, 345, 104, 20, 371, 83, 169, 126, 369, 79, 103, 153, 261, 192, 179, 162, 334, 35, 251, 301, 279, 95, 245, 107, 321, 117, 144, 39, 91, 196, 277, 60, 11, 58, 120, 110, 351, 112, 366, 175, 237, 102, 357, 61, 298, 182, 178, 19, 280, 78, 310, 32, 167, 47, 36, 170, 302, 53, 287, 299, 13, 374, 227, 168, 0, 263, 92, 109, 223, 330, 286, 165, 335, 247, 214, 355, 152, 336, 137, 362, 127, 88, 354, 33, 202, 119, 174, 188, 275, 341, 293, 132, 248, 12, 200, 131, 90, 264, 142, 269, 344, 373, 312, 315, 147, 111, 114, 158, 199, 15, 69, 356, 154, 211, 141, 96, 358, 80, 75, 291, 281, 289, 249, 108, 66, 318, 51, 67, 311, 14, 258, 265, 208, 183, 159, 235, 370, 207, 193, 364, 314, 34, 156, 136, 276, 151, 267, 313, 77, 359, 348, 93, 377, 55, 381, 9, 288, 352, 307, 306, 43, 45, 290, 155, 133, 331, 260, 231, 241, 284, 143, 177, 176] + 30: [208, 76, 366, 225, 11, 144, 283, 250, 97, 341, 253, 256, 141, 136, 292, 98, 157, 278, 179, 335, 51, 310, 55, 48, 44, 374, 186, 121, 220, 139, 69, 85, 209, 314, 178, 352, 207, 265, 63, 61, 191, 238, 8, 21, 22, 31, 36, 40, 239, 333, 243, 59, 2, 45, 41, 315, 216, 240, 100, 58, 215, 226, 281, 298, 174, 91, 72, 114, 163, 286, 349, 181, 282, 360, 342, 377, 371, 161, 203, 327, 368, 122, 169, 330, 102, 338, 171, 93, 251, 301, 47, 88, 94, 101, 118, 123, 140, 35, 289, 213, 13, 378, 206, 28, 68, 303, 340, 74, 152, 275, 210, 204, 344, 369, 343, 261, 54, 383, 197, 129, 241, 73, 50, 145, 194, 183, 347, 326, 381, 5, 370, 355, 16, 113, 126, 313, 367, 182, 252, 305, 128, 134, 146, 185, 266, 52, 62, 184, 137, 78, 295, 223, 147, 302, 43, 221, 77, 192, 309, 382, 170, 160, 23, 353, 231, 142, 359, 111, 375, 248, 297, 25, 166, 202, 263, 32, 57, 125, 130, 291, 173, 120, 198, 0, 149, 228, 4, 70, 307, 189, 218, 219, 65, 214, 351, 133, 222, 242, 257, 188, 246, 17, 362, 159, 89, 1, 193, 236, 277, 260, 67, 60, 132, 329, 150, 376, 66, 293, 7, 92, 288, 337, 107, 267, 264, 83, 155, 82, 127, 233, 255, 300, 80, 328, 229, 27, 131, 331, 237, 259, 245, 53, 373, 254, 86, 75, 168, 321, 346, 212, 271, 49, 138, 336, 339, 299, 217, 18, 285, 200, 96, 167, 162, 14, 172, 180, 244, 284, 79, 110, 230, 247, 279, 99, 287, 273, 153, 19, 190, 106, 56, 372, 34, 87, 176, 354, 268, 269, 37, 358, 156, 124, 272, 135, 24, 234, 64, 33, 205, 312, 12, 357, 6, 9, 195, 274, 258, 365, 108, 84, 306, 319, 280, 109, 332, 290, 175, 154, 276, 324, 304, 116, 323, 117, 3, 15, 308, 270, 199, 39, 148, 30, 201, 363, 322, 334, 262, 187, 143, 158, 29, 196, 46, 361, 10, 164, 316, 224, 317, 177, 71, 151, 320, 296, 356, 325, 104, 38, 103, 20, 294, 211, 115, 249, 318, 311, 95, 350, 90, 348, 165, 364, 81, 105, 232, 112, 235, 42, 26, 119, 227, 379, 345, 380] + 31: [0, 185, 32, 36, 335, 95, 330, 209, 58, 253, 62, 102, 223, 82, 74, 17, 66, 122, 46, 15, 97, 383, 141, 49, 354, 237, 376, 266, 204, 224, 107, 210, 367, 315, 99, 113, 127, 197, 372, 27, 174, 257, 151, 165, 148, 229, 132, 207, 159, 239, 227, 241, 116, 25, 219, 39, 119, 171, 72, 355, 258, 145, 236, 63, 23, 243, 277, 31, 318, 225, 373, 16, 84, 361, 362, 6, 272, 228, 48, 50, 240, 129, 298, 321, 135, 7, 235, 370, 313, 359, 311, 104, 117, 71, 289, 329, 128, 115, 87, 265, 279, 326, 270, 328, 22, 333, 283, 338, 112, 296, 211, 136, 40, 366, 65, 322, 256, 374, 217, 92, 142, 153, 12, 305, 275, 351, 198, 103, 85, 309, 24, 234, 327, 156, 79, 126, 114, 357, 377, 230, 203, 180, 294, 350, 233, 143, 261, 331, 3, 133, 187, 317, 349, 60, 34, 299, 348, 290, 380, 353, 232, 245, 175, 205, 222, 202, 213, 21, 302, 73, 196, 183, 271, 346, 352, 304, 238, 375, 160, 157, 168, 381, 53, 193, 255, 184, 30, 161, 5, 154, 212, 347, 26, 252, 70, 125, 365, 364, 215, 150, 325, 155, 378, 164, 276, 189, 78, 111, 231, 55, 134, 182, 77, 287, 307, 118, 336, 195, 260, 273, 14, 177, 371, 281, 263, 42, 139, 340, 300, 64, 293, 382, 295, 18, 179, 86, 358, 343, 37, 149, 201, 158, 363, 320, 192, 379, 147, 267, 146, 186, 61, 360, 56, 269, 28, 249, 93, 38, 368, 57, 105, 316, 312, 288, 303, 291, 4, 51, 20, 323, 1, 130, 80, 308, 345, 9, 262, 109, 280, 274, 282, 167, 301, 163, 297, 200, 90, 246, 68, 324, 144, 248, 221, 292, 69, 339, 173, 140, 188, 75, 334, 214, 106, 254, 152, 54, 2, 88, 137, 41, 284, 123, 45, 8, 110, 108, 59, 43, 91, 194, 47, 162, 101, 178, 337, 356, 52, 44, 344, 242, 319, 94, 199, 67, 341, 120, 100, 166, 278, 76, 19, 121, 250, 268, 96, 206, 170, 208, 131, 286, 33, 244, 138, 98, 81, 169, 176, 190, 314, 310, 369, 29, 285, 11, 247, 83, 220, 264, 306, 226, 259, 251, 342, 191, 216, 181, 124, 35, 332, 218, 10, 13, 89, 172] + 32: [63, 146, 310, 66, 141, 204, 96, 126, 258, 109, 289, 171, 227, 104, 69, 64, 48, 237, 127, 240, 81, 337, 24, 349, 363, 311, 42, 102, 103, 197, 73, 374, 196, 203, 377, 31, 23, 321, 217, 338, 194, 162, 68, 14, 123, 169, 177, 181, 367, 166, 246, 209, 271, 2, 342, 319, 202, 211, 308, 232, 120, 189, 298, 71, 26, 25, 157, 150, 286, 53, 371, 129, 6, 340, 154, 172, 221, 132, 152, 304, 153, 264, 0, 355, 118, 224, 46, 368, 47, 207, 11, 27, 147, 244, 245, 251, 352, 51, 344, 234, 362, 130, 121, 41, 186, 283, 279, 116, 273, 345, 266, 91, 111, 50, 314, 281, 59, 142, 39, 3, 302, 10, 37, 193, 34, 82, 299, 124, 238, 222, 330, 201, 170, 208, 133, 259, 242, 112, 291, 226, 326, 167, 262, 265, 94, 163, 164, 322, 49, 229, 370, 12, 40, 65, 75, 78, 156, 252, 58, 301, 212, 354, 199, 213, 178, 230, 336, 29, 176, 247, 315, 56, 139, 339, 9, 99, 243, 365, 223, 35, 160, 52, 188, 60, 76, 95, 98, 295, 235, 333, 173, 296, 218, 241, 107, 275, 108, 179, 8, 257, 16, 89, 263, 140, 233, 335, 306, 254, 325, 36, 231, 28, 250, 293, 382, 353, 70, 4, 381, 77, 236, 5, 136, 284, 125, 376, 20, 356, 67, 79, 255, 100, 180, 18, 17, 272, 15, 54, 300, 312, 148, 190, 38, 288, 346, 106, 280, 274, 214, 348, 83, 97, 87, 113, 294, 313, 72, 85, 292, 269, 45, 253, 183, 351, 115, 276, 22, 198, 317, 110, 210, 307, 195, 105, 13, 134, 305, 278, 343, 137, 260, 358, 192, 290, 93, 90, 316, 373, 33, 347, 327, 155, 268, 323, 200, 219, 359, 225, 174, 143, 131, 228, 7, 369, 55, 303, 101, 332, 324, 182, 256, 249, 80, 30, 309, 378, 372, 144, 282, 43, 165, 248, 117, 261, 44, 138, 220, 364, 216, 161, 74, 159, 267, 114, 379, 375, 92, 285, 122, 329, 135, 187, 21, 360, 61, 380, 184, 151, 331, 19, 357, 168, 297, 185, 175, 206, 239, 86, 32, 350, 84, 205, 366, 62, 158, 320, 361, 191, 287, 341, 128, 270, 119, 149, 334, 1, 277, 215, 88, 318, 328, 145, 57, 383] + 33: [38, 307, 240, 331, 323, 39, 125, 146, 206, 335, 54, 33, 155, 40, 288, 126, 303, 173, 174, 347, 375, 157, 200, 369, 213, 43, 3, 28, 166, 248, 327, 20, 346, 132, 76, 332, 247, 353, 214, 171, 359, 2, 46, 0, 265, 293, 333, 372, 91, 115, 267, 140, 280, 121, 183, 348, 84, 266, 255, 204, 30, 227, 308, 217, 18, 351, 355, 254, 79, 262, 197, 163, 152, 170, 106, 71, 66, 26, 110, 67, 135, 74, 286, 291, 310, 118, 242, 363, 7, 228, 284, 47, 239, 129, 69, 142, 339, 343, 154, 51, 259, 158, 107, 314, 108, 5, 190, 141, 25, 193, 131, 9, 305, 211, 149, 4, 198, 246, 55, 373, 279, 364, 137, 371, 281, 70, 285, 219, 10, 316, 130, 381, 120, 62, 315, 165, 274, 44, 344, 357, 282, 12, 34, 220, 36, 81, 299, 328, 63, 31, 96, 179, 201, 352, 127, 376, 218, 151, 192, 289, 61, 354, 187, 109, 45, 80, 156, 370, 114, 60, 49, 330, 113, 65, 278, 41, 252, 182, 181, 85, 78, 257, 178, 59, 199, 253, 313, 294, 203, 159, 189, 277, 319, 148, 283, 167, 320, 42, 297, 68, 322, 226, 169, 273, 1, 210, 356, 222, 235, 144, 205, 230, 383, 196, 382, 367, 186, 234, 368, 292, 24, 52, 271, 261, 318, 298, 300, 168, 188, 312, 244, 175, 378, 374, 269, 57, 302, 272, 264, 287, 260, 229, 301, 86, 145, 153, 379, 73, 241, 164, 37, 270, 160, 150, 208, 268, 14, 290, 93, 138, 123, 276, 19, 50, 180, 13, 58, 249, 350, 366, 101, 324, 136, 194, 111, 11, 225, 245, 365, 337, 92, 250, 99, 172, 77, 143, 83, 95, 177, 380, 258, 89, 75, 377, 231, 342, 147, 139, 361, 233, 17, 338, 117, 334, 8, 116, 329, 16, 216, 22, 162, 104, 88, 212, 105, 15, 215, 349, 53, 112, 221, 128, 23, 207, 27, 119, 98, 326, 97, 275, 309, 296, 100, 223, 236, 102, 362, 64, 251, 94, 176, 191, 325, 82, 358, 72, 48, 21, 35, 87, 237, 133, 311, 232, 202, 90, 317, 124, 256, 209, 360, 340, 122, 345, 336, 56, 103, 29, 295, 6, 238, 243, 184, 224, 134, 341, 195, 306, 321, 304, 185, 263, 161, 32] + 34: [108, 44, 151, 135, 145, 240, 178, 246, 83, 271, 78, 162, 208, 290, 81, 256, 136, 133, 33, 181, 73, 0, 247, 263, 113, 140, 62, 298, 105, 266, 314, 13, 101, 226, 331, 205, 17, 285, 90, 1, 355, 378, 320, 7, 375, 14, 76, 119, 179, 334, 372, 357, 239, 321, 291, 258, 59, 288, 41, 95, 134, 373, 12, 159, 368, 206, 195, 241, 48, 287, 367, 230, 215, 294, 10, 358, 142, 296, 374, 219, 36, 286, 28, 109, 242, 342, 212, 22, 43, 182, 38, 235, 177, 211, 122, 132, 16, 330, 61, 324, 311, 185, 148, 146, 115, 238, 92, 65, 225, 153, 184, 326, 327, 199, 251, 67, 270, 209, 102, 308, 103, 164, 283, 371, 261, 64, 68, 228, 111, 353, 49, 117, 278, 253, 9, 54, 120, 188, 252, 309, 198, 213, 160, 189, 249, 126, 196, 382, 18, 304, 31, 360, 315, 295, 191, 6, 125, 201, 107, 356, 123, 197, 143, 217, 30, 227, 333, 169, 232, 154, 63, 127, 139, 272, 85, 88, 233, 161, 34, 255, 165, 26, 39, 104, 128, 156, 338, 106, 121, 32, 224, 193, 306, 299, 190, 29, 332, 354, 203, 194, 347, 79, 335, 267, 244, 21, 144, 174, 84, 183, 172, 336, 328, 231, 364, 170, 361, 180, 322, 277, 149, 383, 98, 192, 269, 262, 93, 319, 52, 370, 339, 141, 281, 292, 349, 303, 129, 66, 279, 202, 15, 259, 248, 207, 82, 158, 362, 167, 379, 2, 8, 300, 4, 124, 58, 53, 313, 187, 37, 280, 346, 72, 214, 35, 5, 80, 19, 318, 221, 186, 147, 268, 282, 310, 100, 380, 57, 173, 55, 171, 351, 96, 216, 94, 350, 273, 316, 229, 340, 69, 3, 237, 74, 25, 243, 138, 220, 97, 222, 11, 87, 91, 110, 302, 345, 24, 60, 70, 245, 150, 359, 116, 301, 348, 114, 323, 23, 46, 264, 275, 289, 265, 274, 75, 131, 250, 155, 112, 71, 130, 325, 234, 369, 343, 341, 363, 366, 337, 218, 47, 312, 223, 293, 40, 99, 284, 254, 168, 77, 200, 329, 210, 175, 166, 137, 317, 236, 27, 204, 89, 50, 352, 157, 42, 56, 45, 51, 152, 305, 86, 118, 307, 176, 260, 297, 20, 257, 163, 344, 276, 376, 377, 381, 365] + 35: [319, 68, 322, 220, 144, 141, 138, 31, 147, 114, 335, 362, 159, 275, 357, 296, 158, 5, 294, 137, 152, 11, 170, 74, 348, 264, 330, 99, 380, 73, 43, 176, 105, 189, 149, 112, 23, 354, 238, 83, 225, 219, 226, 39, 120, 304, 228, 177, 45, 295, 72, 353, 237, 56, 257, 38, 154, 15, 265, 50, 266, 27, 102, 168, 18, 61, 328, 108, 143, 90, 323, 57, 227, 91, 382, 273, 179, 196, 315, 378, 317, 151, 253, 22, 26, 262, 97, 101, 270, 252, 280, 119, 229, 381, 293, 343, 1, 367, 276, 48, 260, 211, 368, 129, 3, 224, 46, 307, 14, 222, 373, 160, 244, 308, 282, 372, 42, 320, 16, 172, 109, 214, 369, 383, 376, 19, 17, 88, 198, 292, 259, 245, 94, 121, 77, 10, 69, 310, 188, 210, 173, 236, 291, 359, 345, 297, 86, 281, 62, 254, 67, 153, 231, 78, 41, 326, 361, 136, 312, 267, 356, 36, 171, 30, 125, 95, 32, 350, 366, 202, 339, 24, 290, 142, 66, 344, 274, 332, 161, 370, 181, 155, 288, 107, 351, 233, 123, 258, 212, 199, 85, 284, 363, 131, 53, 80, 246, 303, 318, 4, 221, 239, 364, 349, 331, 54, 157, 321, 128, 81, 93, 213, 209, 337, 96, 287, 8, 305, 347, 166, 174, 269, 217, 25, 6, 365, 156, 29, 115, 49, 165, 167, 271, 33, 371, 336, 76, 2, 118, 200, 75, 218, 203, 340, 44, 130, 286, 21, 116, 206, 375, 194, 191, 126, 37, 360, 140, 132, 192, 215, 334, 201, 256, 261, 111, 145, 82, 204, 184, 127, 175, 84, 35, 63, 248, 278, 249, 195, 255, 139, 148, 327, 208, 0, 341, 64, 283, 60, 169, 299, 133, 309, 182, 58, 55, 47, 289, 186, 311, 234, 241, 329, 98, 134, 272, 9, 298, 190, 313, 300, 302, 235, 20, 180, 117, 216, 185, 301, 279, 338, 103, 87, 135, 377, 306, 250, 263, 150, 178, 333, 242, 51, 71, 124, 146, 13, 28, 205, 342, 113, 240, 379, 314, 324, 346, 355, 352, 164, 162, 52, 163, 187, 325, 230, 268, 104, 243, 122, 12, 92, 277, 110, 106, 34, 7, 70, 285, 374, 79, 65, 207, 316, 232, 89, 358, 183, 197, 223, 193, 247, 251, 59, 100, 40] + 36: [232, 372, 197, 348, 299, 296, 380, 374, 150, 314, 31, 45, 246, 231, 293, 13, 68, 245, 155, 168, 216, 228, 46, 310, 0, 52, 335, 244, 119, 208, 51, 209, 204, 338, 251, 357, 118, 110, 235, 151, 285, 89, 269, 160, 114, 64, 37, 123, 22, 10, 223, 382, 359, 210, 241, 90, 291, 47, 85, 65, 201, 67, 166, 55, 132, 91, 50, 339, 354, 174, 100, 177, 255, 187, 81, 344, 358, 78, 220, 58, 239, 189, 236, 383, 60, 120, 226, 87, 7, 34, 27, 106, 221, 43, 270, 170, 184, 266, 12, 370, 62, 230, 326, 373, 73, 5, 54, 32, 175, 362, 319, 49, 349, 95, 164, 20, 171, 179, 237, 224, 305, 229, 104, 17, 157, 129, 243, 135, 281, 6, 158, 199, 337, 21, 115, 350, 278, 96, 124, 218, 234, 113, 279, 273, 371, 238, 225, 131, 74, 328, 98, 56, 97, 308, 125, 261, 240, 334, 288, 282, 167, 105, 298, 342, 142, 117, 271, 277, 323, 264, 76, 28, 148, 263, 283, 18, 44, 247, 315, 173, 316, 137, 61, 355, 154, 303, 227, 181, 306, 145, 290, 302, 368, 304, 193, 107, 176, 2, 320, 99, 301, 205, 92, 331, 333, 108, 219, 139, 361, 121, 172, 289, 153, 140, 343, 364, 134, 71, 212, 36, 188, 267, 365, 29, 329, 295, 381, 198, 35, 307, 254, 213, 26, 128, 327, 222, 367, 186, 292, 360, 272, 57, 178, 192, 84, 59, 258, 274, 352, 79, 287, 156, 33, 112, 194, 94, 122, 284, 322, 183, 15, 280, 66, 195, 102, 200, 257, 379, 88, 149, 318, 356, 363, 109, 259, 19, 249, 215, 312, 253, 161, 203, 136, 286, 182, 317, 127, 375, 53, 144, 311, 252, 250, 260, 242, 369, 207, 133, 332, 42, 321, 309, 111, 345, 63, 297, 300, 276, 346, 147, 11, 116, 211, 30, 39, 206, 165, 23, 159, 48, 24, 70, 169, 77, 130, 347, 324, 8, 378, 330, 353, 4, 214, 268, 16, 313, 340, 341, 325, 294, 185, 101, 196, 40, 275, 152, 233, 93, 162, 262, 9, 191, 138, 103, 69, 3, 72, 366, 41, 202, 217, 180, 38, 143, 351, 80, 163, 146, 83, 190, 265, 248, 256, 75, 14, 126, 86, 1, 82, 141, 377, 376, 25, 336] + 37: [218, 213, 199, 48, 193, 223, 344, 253, 22, 139, 181, 18, 145, 19, 122, 27, 149, 82, 143, 108, 285, 356, 83, 337, 347, 313, 38, 4, 116, 160, 17, 170, 175, 35, 81, 62, 130, 119, 300, 89, 90, 68, 84, 31, 132, 236, 239, 288, 40, 188, 123, 314, 46, 272, 292, 361, 171, 245, 165, 204, 163, 133, 359, 208, 76, 173, 144, 61, 39, 333, 329, 126, 206, 121, 244, 94, 284, 42, 154, 13, 91, 216, 169, 297, 191, 96, 231, 155, 11, 220, 275, 232, 99, 189, 352, 383, 268, 366, 0, 273, 311, 205, 190, 72, 77, 140, 360, 88, 328, 343, 257, 150, 127, 228, 176, 6, 230, 141, 278, 29, 100, 224, 80, 380, 97, 24, 325, 187, 299, 10, 60, 137, 310, 198, 106, 324, 177, 45, 219, 211, 346, 209, 56, 75, 294, 200, 340, 66, 309, 52, 372, 290, 382, 114, 185, 182, 147, 93, 330, 221, 370, 287, 270, 168, 217, 36, 264, 201, 87, 55, 37, 321, 195, 279, 105, 351, 138, 368, 303, 78, 295, 374, 248, 57, 202, 197, 369, 246, 358, 251, 375, 112, 148, 222, 162, 364, 242, 306, 341, 179, 174, 86, 129, 7, 65, 117, 131, 192, 315, 229, 21, 196, 365, 41, 30, 134, 376, 49, 247, 128, 255, 241, 274, 5, 172, 124, 152, 286, 291, 283, 142, 146, 107, 326, 54, 159, 381, 25, 238, 120, 293, 158, 113, 71, 348, 125, 289, 342, 234, 16, 302, 353, 14, 233, 153, 164, 157, 316, 51, 74, 203, 296, 92, 111, 34, 335, 331, 269, 312, 271, 280, 210, 249, 254, 215, 332, 378, 339, 166, 184, 237, 28, 261, 186, 2, 23, 276, 226, 109, 58, 349, 319, 243, 178, 362, 305, 95, 156, 355, 252, 318, 235, 32, 277, 322, 320, 104, 64, 323, 73, 161, 103, 3, 136, 69, 262, 373, 102, 317, 267, 298, 256, 266, 350, 115, 101, 308, 225, 70, 227, 20, 250, 367, 50, 377, 8, 301, 379, 79, 240, 265, 151, 363, 44, 371, 59, 47, 9, 327, 194, 118, 345, 336, 85, 258, 1, 43, 338, 304, 135, 15, 167, 334, 207, 98, 212, 33, 354, 26, 110, 53, 260, 183, 259, 63, 357, 214, 281, 180, 307, 12, 282, 67, 263] + 38: [301, 282, 50, 94, 322, 32, 171, 145, 212, 54, 97, 199, 125, 352, 271, 208, 193, 262, 57, 184, 179, 143, 228, 31, 28, 242, 353, 2, 173, 214, 56, 11, 47, 186, 351, 363, 164, 254, 259, 207, 116, 365, 330, 304, 161, 46, 60, 185, 6, 154, 187, 348, 332, 274, 177, 19, 17, 328, 253, 319, 137, 341, 107, 239, 29, 358, 87, 349, 156, 48, 246, 117, 306, 134, 174, 139, 310, 334, 172, 382, 25, 204, 283, 13, 102, 240, 320, 163, 175, 22, 377, 370, 312, 176, 237, 251, 266, 51, 114, 4, 178, 292, 226, 49, 130, 166, 135, 119, 234, 327, 90, 222, 67, 95, 150, 144, 255, 136, 34, 188, 289, 170, 26, 272, 235, 337, 360, 371, 104, 100, 295, 297, 183, 3, 357, 157, 314, 291, 218, 160, 162, 270, 339, 325, 203, 372, 383, 10, 369, 123, 285, 220, 63, 27, 317, 268, 267, 286, 225, 355, 227, 69, 224, 105, 58, 256, 7, 229, 265, 260, 76, 33, 258, 40, 200, 313, 273, 375, 44, 374, 247, 236, 12, 326, 350, 20, 167, 198, 249, 318, 340, 380, 158, 284, 126, 62, 287, 118, 263, 309, 230, 381, 279, 344, 108, 338, 106, 99, 14, 73, 362, 359, 88, 264, 368, 109, 276, 190, 84, 149, 103, 81, 296, 122, 74, 82, 215, 168, 35, 290, 91, 113, 24, 209, 281, 83, 127, 346, 38, 52, 378, 201, 189, 89, 213, 245, 354, 248, 379, 244, 0, 77, 257, 16, 288, 75, 324, 72, 59, 36, 68, 165, 96, 298, 5, 141, 197, 180, 15, 78, 302, 195, 43, 347, 61, 80, 55, 202, 70, 335, 18, 232, 315, 110, 155, 121, 64, 101, 221, 124, 366, 333, 93, 66, 86, 194, 39, 92, 373, 206, 53, 329, 361, 275, 147, 21, 321, 45, 211, 216, 142, 37, 342, 241, 356, 181, 131, 250, 23, 182, 261, 120, 311, 294, 71, 299, 192, 65, 364, 128, 153, 196, 1, 133, 98, 191, 269, 280, 30, 277, 223, 345, 8, 115, 293, 217, 278, 169, 300, 331, 233, 219, 146, 85, 9, 307, 323, 308, 367, 111, 151, 148, 376, 41, 305, 152, 238, 79, 129, 231, 252, 205, 138, 343, 316, 112, 42, 132, 303, 336, 140, 243, 159, 210] + 39: [282, 36, 176, 116, 153, 218, 292, 329, 204, 98, 15, 163, 146, 381, 352, 127, 31, 278, 331, 358, 276, 5, 226, 33, 72, 289, 193, 4, 243, 377, 169, 120, 58, 335, 189, 266, 206, 0, 197, 210, 326, 124, 12, 308, 29, 148, 361, 209, 310, 93, 100, 264, 1, 13, 155, 244, 297, 374, 141, 369, 51, 200, 84, 274, 340, 112, 190, 131, 62, 366, 63, 301, 212, 311, 241, 354, 338, 309, 114, 265, 283, 192, 287, 222, 78, 246, 252, 10, 219, 129, 294, 73, 154, 11, 52, 143, 247, 38, 85, 180, 348, 215, 279, 187, 145, 277, 172, 76, 178, 87, 375, 8, 248, 251, 138, 47, 152, 199, 364, 249, 14, 102, 45, 133, 128, 6, 130, 214, 166, 109, 288, 168, 157, 160, 175, 229, 233, 147, 205, 105, 332, 90, 321, 22, 66, 312, 313, 35, 223, 139, 242, 272, 270, 71, 295, 319, 77, 23, 202, 118, 48, 122, 225, 69, 367, 150, 322, 213, 182, 228, 95, 173, 230, 16, 220, 194, 376, 217, 196, 303, 108, 305, 123, 92, 255, 351, 32, 318, 57, 115, 165, 149, 104, 65, 314, 280, 234, 151, 119, 382, 174, 333, 250, 330, 49, 7, 235, 170, 191, 299, 101, 74, 50, 177, 307, 161, 134, 355, 88, 320, 275, 30, 363, 181, 60, 291, 19, 111, 337, 231, 211, 359, 238, 136, 370, 316, 344, 186, 334, 44, 372, 286, 132, 99, 110, 25, 360, 271, 296, 324, 40, 341, 365, 81, 298, 259, 42, 323, 239, 34, 126, 135, 371, 162, 302, 94, 106, 203, 164, 254, 306, 183, 91, 70, 27, 103, 46, 347, 37, 64, 342, 158, 67, 224, 184, 261, 9, 89, 236, 258, 379, 96, 82, 156, 325, 328, 80, 362, 17, 24, 208, 290, 227, 53, 232, 28, 336, 137, 353, 188, 144, 75, 345, 293, 185, 97, 357, 79, 68, 207, 171, 343, 41, 159, 55, 20, 18, 253, 339, 349, 125, 237, 269, 201, 21, 198, 56, 315, 43, 268, 368, 113, 267, 86, 2, 26, 121, 300, 3, 59, 273, 383, 140, 54, 263, 260, 378, 281, 317, 179, 373, 61, 167, 285, 221, 195, 240, 327, 216, 245, 346, 117, 107, 83, 142, 39, 284, 350, 256, 262, 356, 380, 304, 257] + 40: [31, 359, 367, 52, 153, 64, 74, 3, 91, 363, 336, 341, 322, 278, 200, 188, 137, 44, 99, 189, 25, 155, 339, 333, 177, 304, 301, 139, 241, 122, 141, 125, 135, 16, 174, 106, 119, 180, 39, 87, 255, 191, 72, 5, 92, 149, 248, 300, 319, 71, 33, 260, 204, 283, 160, 128, 164, 58, 97, 28, 303, 136, 321, 289, 27, 236, 205, 259, 307, 150, 138, 94, 103, 29, 356, 281, 273, 293, 222, 246, 78, 121, 361, 13, 32, 182, 34, 111, 325, 315, 109, 292, 220, 225, 144, 105, 15, 328, 346, 280, 55, 101, 247, 8, 332, 372, 59, 228, 275, 326, 271, 166, 211, 12, 221, 239, 56, 57, 178, 377, 192, 120, 206, 329, 37, 18, 233, 291, 118, 256, 84, 266, 117, 40, 218, 231, 114, 313, 223, 298, 83, 126, 69, 343, 68, 142, 179, 357, 70, 373, 265, 161, 369, 81, 90, 17, 1, 168, 274, 185, 11, 199, 382, 46, 187, 263, 244, 290, 294, 324, 93, 243, 171, 314, 261, 6, 147, 212, 184, 268, 49, 113, 242, 22, 173, 96, 176, 24, 172, 335, 353, 376, 175, 360, 108, 379, 286, 62, 297, 215, 88, 235, 351, 210, 284, 143, 63, 80, 54, 38, 148, 227, 53, 364, 43, 110, 2, 338, 305, 282, 19, 131, 345, 238, 270, 14, 276, 310, 50, 66, 77, 217, 234, 152, 381, 86, 272, 146, 312, 194, 378, 349, 21, 226, 169, 75, 26, 279, 158, 198, 100, 311, 296, 89, 209, 61, 193, 216, 196, 262, 316, 95, 82, 130, 7, 264, 318, 237, 48, 47, 366, 36, 375, 288, 348, 383, 140, 203, 257, 104, 352, 342, 45, 245, 320, 240, 331, 214, 317, 250, 358, 340, 213, 67, 337, 156, 254, 251, 224, 159, 362, 129, 162, 277, 145, 334, 302, 269, 79, 365, 299, 306, 165, 287, 167, 323, 350, 60, 73, 201, 115, 186, 0, 102, 368, 230, 35, 134, 354, 4, 380, 112, 20, 197, 170, 249, 207, 183, 190, 132, 181, 98, 107, 202, 232, 208, 258, 347, 9, 10, 219, 253, 30, 308, 370, 229, 309, 41, 65, 151, 355, 327, 371, 330, 195, 51, 85, 76, 116, 133, 23, 124, 374, 295, 163, 285, 154, 267, 157, 252, 127, 42, 344, 123] + 41: [49, 190, 119, 335, 27, 86, 365, 293, 79, 24, 206, 306, 348, 127, 140, 126, 217, 301, 379, 260, 156, 340, 133, 288, 66, 107, 6, 130, 315, 303, 359, 154, 100, 165, 13, 248, 113, 155, 67, 20, 183, 145, 189, 362, 382, 233, 247, 216, 378, 184, 159, 367, 103, 255, 273, 92, 111, 358, 302, 82, 383, 3, 219, 194, 349, 102, 125, 93, 229, 325, 330, 380, 101, 181, 208, 160, 10, 192, 15, 300, 179, 180, 207, 38, 147, 182, 238, 47, 201, 240, 211, 356, 304, 230, 296, 278, 199, 355, 197, 122, 11, 249, 50, 97, 187, 161, 311, 108, 45, 4, 202, 243, 308, 64, 225, 256, 235, 117, 242, 8, 63, 290, 277, 261, 220, 214, 37, 342, 39, 65, 72, 332, 148, 137, 29, 274, 267, 319, 270, 16, 58, 118, 227, 309, 170, 142, 369, 215, 166, 57, 84, 76, 172, 279, 12, 95, 318, 222, 264, 163, 336, 70, 54, 276, 258, 167, 112, 210, 363, 153, 18, 43, 173, 89, 41, 188, 236, 381, 234, 105, 310, 28, 265, 329, 331, 88, 177, 372, 291, 52, 263, 357, 60, 283, 275, 185, 71, 150, 200, 285, 136, 314, 262, 44, 326, 351, 25, 23, 134, 116, 282, 55, 237, 289, 224, 266, 176, 186, 19, 213, 244, 146, 59, 272, 352, 0, 141, 68, 73, 175, 171, 370, 110, 323, 377, 109, 203, 333, 31, 120, 350, 114, 366, 305, 376, 307, 168, 96, 34, 69, 347, 337, 104, 53, 286, 218, 252, 56, 313, 254, 271, 115, 144, 328, 151, 295, 81, 317, 61, 128, 123, 292, 1, 191, 259, 48, 257, 204, 375, 373, 121, 343, 106, 341, 281, 228, 22, 85, 374, 298, 129, 87, 196, 90, 223, 124, 7, 360, 334, 198, 9, 324, 42, 353, 26, 32, 226, 83, 294, 138, 250, 268, 299, 284, 91, 94, 74, 221, 99, 75, 78, 77, 297, 149, 316, 209, 14, 245, 62, 2, 162, 346, 169, 135, 40, 132, 321, 269, 30, 205, 251, 212, 368, 327, 364, 21, 280, 253, 80, 35, 239, 36, 174, 345, 241, 46, 322, 344, 339, 51, 131, 338, 246, 5, 371, 354, 232, 33, 143, 287, 320, 158, 98, 195, 231, 17, 139, 157, 312, 361, 193, 164, 178, 152] + 42: [123, 184, 212, 41, 349, 55, 281, 327, 351, 370, 29, 135, 28, 329, 36, 92, 33, 362, 380, 241, 301, 225, 323, 183, 226, 61, 251, 156, 364, 84, 284, 82, 360, 23, 288, 347, 96, 353, 107, 14, 271, 163, 273, 242, 25, 71, 95, 254, 152, 93, 154, 50, 106, 151, 150, 337, 377, 208, 21, 235, 248, 30, 67, 17, 173, 216, 314, 316, 257, 192, 113, 311, 62, 89, 315, 58, 205, 378, 278, 26, 194, 267, 90, 363, 124, 334, 22, 85, 190, 110, 78, 43, 91, 289, 294, 324, 117, 218, 2, 1, 303, 116, 264, 262, 263, 209, 189, 373, 328, 221, 169, 344, 147, 181, 322, 52, 223, 245, 101, 132, 307, 238, 49, 310, 196, 16, 270, 293, 120, 206, 98, 73, 185, 136, 207, 133, 276, 79, 54, 142, 240, 352, 369, 339, 31, 155, 333, 375, 114, 211, 63, 374, 220, 6, 342, 191, 275, 320, 140, 187, 87, 326, 335, 280, 118, 149, 32, 60, 8, 371, 260, 130, 34, 298, 355, 269, 233, 175, 272, 197, 104, 285, 304, 309, 153, 12, 11, 81, 317, 365, 46, 126, 359, 172, 256, 210, 148, 228, 350, 72, 195, 306, 255, 122, 379, 131, 4, 239, 230, 201, 356, 77, 366, 39, 160, 340, 137, 372, 9, 139, 232, 258, 291, 68, 112, 213, 229, 321, 128, 292, 330, 367, 253, 35, 70, 159, 382, 66, 361, 376, 217, 176, 158, 188, 42, 357, 215, 179, 243, 200, 48, 265, 358, 354, 259, 224, 214, 161, 103, 164, 45, 76, 381, 18, 146, 180, 64, 244, 299, 313, 332, 125, 300, 261, 290, 145, 368, 75, 186, 383, 69, 115, 83, 182, 0, 15, 24, 27, 94, 345, 111, 37, 20, 305, 279, 282, 162, 129, 59, 127, 277, 274, 178, 236, 287, 165, 308, 38, 202, 227, 65, 80, 237, 219, 246, 109, 99, 5, 336, 53, 346, 343, 199, 102, 312, 296, 144, 174, 88, 47, 57, 331, 13, 86, 40, 121, 266, 268, 325, 138, 295, 119, 247, 168, 222, 108, 203, 319, 348, 141, 341, 286, 167, 105, 338, 143, 234, 166, 297, 51, 19, 231, 283, 157, 97, 7, 204, 170, 171, 56, 318, 250, 10, 3, 198, 302, 134, 249, 100, 44, 177, 74, 252, 193] + 43: [168, 359, 298, 262, 278, 112, 20, 103, 152, 55, 127, 313, 307, 321, 4, 222, 282, 204, 306, 26, 36, 67, 334, 148, 293, 180, 56, 74, 9, 370, 295, 329, 144, 353, 118, 134, 211, 357, 341, 69, 7, 141, 383, 104, 25, 166, 331, 44, 225, 17, 30, 344, 308, 35, 13, 94, 131, 186, 322, 227, 198, 121, 89, 191, 238, 275, 339, 271, 160, 365, 155, 243, 60, 38, 45, 251, 309, 343, 159, 304, 267, 76, 19, 173, 265, 316, 162, 114, 41, 228, 42, 133, 350, 54, 189, 354, 375, 335, 297, 27, 332, 197, 349, 366, 57, 337, 111, 264, 1, 147, 368, 150, 90, 346, 277, 299, 149, 84, 196, 107, 49, 229, 143, 120, 263, 288, 170, 122, 8, 65, 151, 233, 290, 240, 311, 124, 106, 236, 208, 333, 33, 310, 192, 95, 66, 372, 75, 369, 6, 142, 59, 254, 22, 81, 138, 323, 274, 269, 374, 2, 232, 327, 83, 123, 296, 130, 241, 145, 99, 47, 210, 284, 101, 174, 116, 207, 294, 132, 158, 268, 300, 110, 260, 230, 62, 249, 270, 58, 289, 361, 206, 214, 98, 92, 283, 23, 177, 68, 216, 252, 326, 377, 29, 360, 305, 202, 255, 272, 154, 246, 234, 140, 5, 61, 10, 63, 185, 165, 217, 46, 128, 325, 175, 72, 279, 87, 287, 358, 355, 187, 285, 248, 172, 281, 340, 139, 244, 184, 373, 245, 315, 347, 190, 164, 82, 129, 364, 239, 363, 93, 223, 203, 338, 136, 379, 32, 378, 48, 286, 135, 367, 257, 237, 345, 24, 50, 253, 167, 220, 226, 18, 64, 71, 117, 193, 109, 320, 179, 312, 199, 328, 21, 362, 91, 176, 231, 195, 3, 221, 156, 14, 351, 181, 371, 37, 97, 235, 291, 342, 318, 108, 78, 250, 73, 205, 88, 219, 200, 261, 157, 194, 273, 102, 113, 266, 213, 125, 80, 31, 292, 356, 163, 381, 119, 330, 258, 51, 276, 0, 137, 352, 96, 212, 302, 169, 348, 105, 171, 324, 43, 161, 218, 39, 100, 314, 153, 301, 178, 224, 280, 85, 77, 52, 380, 86, 115, 34, 79, 28, 12, 242, 183, 16, 70, 146, 256, 215, 11, 259, 126, 247, 209, 188, 336, 201, 53, 317, 303, 40, 15, 319, 376, 182, 382] + 44: [213, 101, 238, 272, 60, 257, 252, 325, 93, 86, 118, 329, 144, 337, 276, 138, 350, 290, 150, 70, 130, 380, 87, 134, 162, 136, 83, 159, 243, 309, 211, 237, 230, 345, 49, 114, 245, 375, 373, 99, 247, 176, 293, 31, 58, 75, 209, 106, 77, 379, 284, 167, 110, 227, 29, 135, 30, 280, 289, 63, 102, 217, 355, 117, 244, 299, 173, 201, 132, 286, 202, 369, 107, 46, 370, 40, 59, 318, 341, 113, 234, 109, 367, 65, 35, 80, 239, 210, 283, 260, 297, 52, 76, 181, 339, 116, 301, 222, 223, 97, 178, 180, 43, 140, 11, 187, 182, 306, 304, 275, 158, 265, 85, 175, 163, 364, 79, 338, 54, 120, 36, 126, 15, 185, 149, 231, 23, 20, 72, 193, 4, 51, 255, 200, 195, 119, 298, 268, 356, 111, 88, 371, 13, 332, 191, 171, 170, 0, 351, 188, 277, 279, 143, 98, 212, 198, 228, 125, 343, 91, 224, 61, 154, 39, 71, 378, 174, 177, 327, 344, 196, 74, 205, 347, 32, 305, 203, 319, 204, 62, 273, 216, 5, 115, 28, 330, 37, 152, 148, 16, 161, 322, 199, 172, 346, 17, 300, 84, 68, 331, 166, 146, 242, 294, 365, 360, 14, 340, 192, 24, 122, 287, 285, 160, 269, 381, 128, 220, 278, 334, 105, 323, 81, 3, 263, 357, 256, 137, 312, 1, 27, 151, 363, 302, 47, 73, 316, 281, 295, 374, 10, 274, 168, 64, 155, 190, 82, 311, 236, 383, 333, 142, 33, 95, 324, 262, 6, 186, 219, 56, 352, 366, 359, 44, 157, 41, 92, 90, 214, 377, 169, 104, 241, 9, 53, 183, 2, 253, 22, 251, 372, 307, 123, 127, 353, 308, 112, 303, 317, 362, 21, 25, 271, 215, 328, 368, 34, 320, 249, 264, 358, 225, 89, 121, 266, 296, 55, 100, 131, 248, 67, 221, 292, 354, 108, 165, 270, 19, 336, 179, 235, 18, 314, 232, 42, 45, 153, 50, 240, 69, 139, 335, 133, 208, 189, 12, 226, 348, 145, 258, 382, 267, 376, 229, 288, 313, 141, 38, 48, 291, 94, 207, 184, 254, 349, 315, 66, 164, 96, 261, 342, 26, 147, 282, 8, 78, 156, 218, 57, 250, 321, 7, 124, 326, 233, 194, 246, 103, 259, 361, 197, 310, 129, 206] + 45: [92, 95, 219, 284, 120, 375, 286, 165, 128, 183, 228, 223, 16, 104, 250, 112, 80, 213, 157, 315, 236, 4, 340, 65, 34, 205, 38, 264, 231, 255, 204, 105, 15, 147, 93, 136, 332, 31, 382, 200, 265, 259, 75, 237, 299, 361, 100, 174, 140, 27, 164, 318, 317, 274, 369, 354, 61, 349, 234, 254, 307, 158, 277, 70, 313, 267, 86, 108, 175, 67, 22, 253, 58, 116, 46, 358, 29, 322, 127, 288, 142, 289, 368, 202, 225, 90, 62, 143, 270, 366, 311, 184, 11, 240, 138, 64, 178, 8, 329, 301, 144, 25, 176, 378, 47, 348, 52, 275, 239, 162, 155, 74, 12, 327, 249, 217, 292, 96, 1, 370, 330, 126, 87, 5, 150, 333, 381, 374, 363, 48, 78, 343, 238, 321, 89, 82, 353, 53, 216, 325, 49, 72, 355, 221, 10, 201, 276, 173, 134, 139, 171, 103, 269, 124, 224, 39, 118, 310, 229, 245, 84, 7, 345, 44, 32, 2, 123, 218, 344, 170, 304, 320, 294, 379, 342, 177, 79, 203, 371, 362, 337, 323, 56, 152, 159, 194, 6, 335, 195, 17, 117, 83, 227, 334, 33, 383, 316, 182, 356, 262, 21, 163, 380, 258, 187, 196, 36, 88, 54, 357, 135, 281, 279, 109, 261, 257, 35, 19, 252, 145, 37, 211, 263, 314, 0, 119, 3, 377, 111, 214, 153, 167, 193, 295, 77, 28, 60, 69, 244, 347, 154, 42, 367, 191, 188, 346, 141, 146, 298, 212, 272, 331, 359, 45, 326, 372, 328, 251, 242, 94, 106, 208, 303, 222, 20, 207, 319, 291, 283, 241, 151, 285, 220, 341, 113, 71, 73, 102, 280, 97, 232, 352, 287, 99, 130, 181, 137, 266, 23, 293, 373, 81, 110, 197, 339, 85, 180, 297, 107, 248, 168, 122, 51, 30, 308, 306, 360, 179, 260, 24, 302, 50, 76, 133, 290, 59, 9, 273, 57, 129, 166, 309, 268, 190, 91, 125, 18, 186, 206, 101, 278, 66, 235, 14, 40, 192, 121, 161, 350, 324, 233, 199, 247, 160, 131, 114, 209, 351, 198, 55, 115, 338, 68, 210, 41, 156, 26, 365, 376, 63, 43, 230, 256, 296, 98, 282, 169, 271, 246, 312, 364, 215, 243, 132, 189, 226, 305, 148, 13, 172, 149, 185, 336, 300] + 46: [142, 355, 106, 201, 46, 238, 6, 301, 60, 114, 199, 3, 85, 62, 117, 260, 132, 118, 192, 298, 23, 250, 237, 193, 383, 373, 78, 297, 189, 9, 265, 145, 283, 262, 244, 371, 43, 358, 330, 126, 196, 296, 185, 100, 178, 365, 122, 175, 183, 253, 367, 188, 240, 51, 324, 312, 40, 165, 2, 320, 282, 280, 33, 372, 334, 315, 109, 13, 41, 272, 267, 158, 377, 329, 92, 24, 222, 56, 317, 241, 91, 370, 257, 52, 138, 275, 341, 363, 269, 29, 116, 151, 225, 82, 379, 356, 352, 380, 354, 210, 327, 202, 332, 12, 187, 55, 44, 219, 215, 131, 195, 227, 232, 197, 137, 50, 362, 273, 103, 59, 111, 308, 97, 169, 223, 69, 293, 342, 177, 93, 98, 150, 163, 310, 31, 236, 1, 174, 276, 70, 14, 147, 5, 213, 311, 286, 149, 335, 233, 96, 221, 344, 271, 38, 18, 146, 102, 30, 143, 120, 80, 218, 156, 316, 63, 95, 205, 157, 45, 101, 263, 289, 119, 318, 159, 277, 261, 217, 309, 171, 208, 15, 77, 366, 381, 48, 181, 47, 349, 270, 220, 27, 133, 58, 230, 66, 368, 170, 246, 284, 136, 68, 303, 203, 350, 351, 108, 155, 99, 353, 295, 166, 176, 128, 172, 340, 336, 307, 328, 360, 313, 173, 348, 20, 333, 90, 361, 234, 337, 112, 127, 229, 300, 134, 251, 54, 0, 28, 231, 36, 10, 256, 207, 152, 19, 25, 274, 154, 124, 254, 345, 76, 79, 211, 8, 376, 287, 121, 180, 37, 129, 294, 198, 17, 125, 226, 35, 144, 89, 161, 7, 74, 359, 190, 285, 339, 347, 135, 258, 281, 87, 73, 306, 123, 84, 167, 252, 88, 247, 139, 110, 319, 314, 16, 239, 104, 194, 204, 113, 242, 266, 228, 130, 22, 326, 168, 259, 369, 304, 53, 292, 235, 268, 32, 141, 290, 299, 346, 71, 322, 264, 291, 86, 278, 26, 67, 75, 94, 288, 105, 212, 179, 200, 182, 325, 302, 34, 64, 343, 140, 224, 249, 191, 209, 214, 378, 338, 374, 39, 49, 115, 83, 186, 245, 42, 4, 164, 162, 216, 321, 57, 81, 206, 160, 255, 382, 243, 153, 364, 331, 184, 305, 375, 65, 21, 11, 72, 107, 279, 248, 357, 148, 323, 61] + 47: [195, 307, 347, 357, 193, 309, 215, 196, 21, 40, 342, 63, 313, 95, 68, 166, 231, 65, 184, 51, 101, 173, 252, 183, 139, 211, 163, 281, 37, 107, 132, 297, 150, 275, 57, 325, 108, 339, 277, 88, 270, 167, 125, 254, 169, 7, 369, 326, 3, 29, 137, 171, 119, 61, 314, 353, 274, 287, 188, 363, 251, 351, 69, 104, 249, 324, 284, 199, 0, 138, 233, 112, 315, 80, 15, 289, 276, 306, 115, 20, 97, 160, 48, 282, 383, 174, 58, 220, 202, 244, 99, 165, 13, 272, 191, 246, 343, 158, 134, 179, 35, 170, 90, 294, 362, 240, 59, 153, 114, 330, 84, 219, 151, 228, 370, 197, 296, 148, 39, 210, 234, 141, 243, 28, 225, 60, 341, 260, 130, 190, 152, 302, 245, 5, 133, 222, 47, 299, 283, 206, 182, 83, 156, 293, 54, 22, 75, 192, 279, 280, 345, 316, 198, 168, 346, 367, 98, 226, 93, 301, 223, 285, 323, 255, 205, 17, 4, 354, 378, 336, 348, 147, 259, 111, 185, 32, 81, 9, 242, 216, 358, 374, 16, 105, 76, 142, 116, 217, 221, 311, 214, 360, 44, 154, 300, 136, 371, 34, 14, 257, 36, 110, 118, 218, 159, 176, 26, 377, 124, 263, 298, 77, 203, 25, 45, 27, 53, 91, 321, 213, 273, 258, 200, 157, 144, 10, 365, 266, 74, 290, 30, 317, 145, 11, 268, 256, 241, 318, 236, 56, 352, 162, 382, 331, 291, 227, 333, 320, 175, 129, 131, 123, 212, 250, 269, 67, 224, 103, 140, 364, 322, 355, 52, 106, 23, 356, 71, 126, 328, 332, 329, 186, 253, 46, 120, 72, 177, 380, 86, 38, 372, 264, 161, 295, 8, 113, 239, 288, 24, 181, 359, 187, 43, 1, 286, 127, 344, 267, 327, 229, 122, 230, 155, 135, 42, 41, 33, 50, 12, 338, 180, 117, 366, 18, 121, 303, 292, 278, 49, 310, 334, 102, 19, 308, 207, 350, 100, 178, 85, 64, 78, 232, 261, 128, 62, 373, 312, 87, 271, 143, 375, 109, 305, 368, 79, 376, 238, 349, 235, 265, 208, 262, 89, 319, 172, 73, 204, 237, 340, 164, 96, 201, 70, 335, 337, 94, 31, 2, 248, 149, 189, 55, 379, 92, 304, 361, 247, 194, 82, 6, 66, 381, 146, 209] + 48: [17, 52, 85, 288, 65, 358, 99, 90, 83, 369, 235, 48, 325, 252, 290, 219, 161, 25, 75, 88, 118, 31, 172, 218, 22, 100, 158, 63, 374, 40, 206, 259, 382, 272, 214, 69, 37, 87, 309, 148, 154, 153, 124, 68, 103, 199, 209, 196, 271, 276, 78, 9, 117, 226, 121, 164, 268, 375, 223, 264, 185, 380, 368, 312, 322, 192, 247, 73, 89, 212, 275, 231, 307, 70, 49, 36, 269, 367, 346, 107, 82, 71, 366, 29, 165, 335, 66, 356, 86, 345, 251, 122, 337, 348, 245, 227, 140, 239, 141, 159, 270, 150, 236, 58, 33, 6, 5, 372, 342, 134, 232, 222, 130, 291, 233, 45, 255, 24, 168, 57, 317, 281, 72, 204, 280, 12, 55, 274, 178, 256, 95, 93, 143, 277, 350, 149, 171, 260, 365, 126, 265, 355, 304, 114, 328, 273, 381, 297, 77, 109, 19, 74, 157, 189, 177, 254, 34, 305, 378, 300, 7, 293, 112, 324, 162, 221, 230, 79, 202, 113, 289, 81, 84, 18, 98, 137, 243, 286, 194, 326, 323, 213, 228, 191, 8, 377, 332, 383, 364, 131, 1, 250, 132, 330, 353, 263, 249, 301, 234, 295, 133, 287, 139, 363, 186, 354, 371, 210, 246, 299, 174, 123, 341, 111, 4, 13, 379, 351, 119, 97, 43, 349, 144, 35, 39, 173, 310, 313, 27, 294, 238, 225, 298, 60, 183, 262, 197, 333, 30, 308, 267, 102, 101, 237, 26, 145, 343, 182, 28, 104, 327, 208, 316, 181, 155, 229, 179, 50, 120, 190, 170, 357, 315, 42, 339, 14, 138, 302, 352, 53, 169, 59, 216, 32, 135, 373, 224, 61, 201, 376, 331, 282, 248, 23, 220, 167, 136, 314, 283, 128, 311, 15, 20, 279, 44, 2, 21, 166, 240, 38, 142, 41, 320, 156, 193, 257, 370, 0, 160, 242, 347, 360, 180, 241, 96, 11, 198, 127, 292, 94, 261, 47, 217, 3, 56, 105, 110, 188, 338, 106, 296, 176, 303, 258, 10, 129, 76, 266, 284, 62, 51, 215, 116, 203, 207, 147, 344, 359, 115, 184, 319, 205, 175, 152, 285, 340, 329, 321, 318, 91, 64, 80, 146, 253, 211, 16, 151, 195, 108, 187, 92, 67, 163, 278, 244, 361, 362, 125, 46, 54, 306, 336, 200, 334] + 49: [302, 37, 209, 115, 212, 208, 148, 60, 91, 74, 381, 264, 94, 48, 97, 362, 80, 56, 277, 29, 4, 81, 139, 179, 35, 338, 172, 85, 71, 355, 84, 192, 143, 145, 168, 316, 7, 341, 287, 21, 249, 230, 32, 68, 135, 356, 123, 191, 321, 157, 197, 304, 181, 78, 261, 201, 177, 133, 260, 194, 30, 332, 195, 366, 336, 310, 16, 138, 365, 34, 41, 273, 371, 352, 98, 227, 114, 176, 358, 12, 31, 190, 154, 364, 88, 343, 87, 372, 334, 161, 141, 263, 129, 188, 300, 64, 297, 305, 242, 251, 294, 59, 353, 25, 276, 325, 107, 132, 187, 45, 314, 333, 144, 174, 137, 221, 140, 170, 189, 296, 301, 82, 327, 134, 351, 244, 53, 117, 200, 151, 228, 22, 52, 232, 166, 51, 165, 162, 171, 350, 203, 258, 50, 322, 253, 155, 9, 193, 83, 65, 293, 241, 90, 44, 292, 220, 344, 8, 100, 122, 328, 254, 340, 368, 240, 128, 89, 222, 225, 303, 238, 178, 103, 345, 280, 378, 46, 2, 342, 256, 125, 324, 27, 315, 363, 102, 236, 207, 291, 3, 347, 375, 126, 75, 271, 210, 15, 234, 116, 110, 180, 262, 380, 289, 312, 255, 95, 5, 205, 73, 63, 245, 196, 198, 160, 163, 146, 10, 308, 62, 359, 239, 72, 119, 0, 204, 101, 283, 182, 326, 57, 93, 367, 266, 99, 152, 252, 26, 370, 38, 374, 268, 313, 233, 346, 136, 213, 40, 335, 39, 218, 246, 120, 331, 131, 109, 121, 1, 17, 23, 223, 214, 224, 248, 376, 173, 33, 43, 307, 339, 47, 226, 106, 49, 285, 108, 299, 104, 357, 167, 237, 159, 379, 281, 247, 288, 373, 317, 79, 6, 202, 184, 319, 150, 66, 278, 275, 199, 76, 243, 329, 175, 183, 235, 250, 169, 382, 86, 311, 290, 112, 127, 55, 216, 67, 330, 156, 142, 320, 130, 61, 124, 282, 361, 348, 185, 111, 147, 42, 96, 231, 215, 383, 219, 24, 92, 360, 272, 19, 149, 257, 158, 105, 270, 153, 259, 286, 306, 269, 186, 298, 164, 58, 279, 28, 54, 70, 318, 323, 77, 20, 295, 267, 36, 309, 14, 211, 69, 118, 206, 284, 113, 354, 274, 377, 13, 229, 18, 265, 217, 11, 369, 349, 337] + 50: [86, 69, 26, 33, 140, 370, 125, 304, 329, 256, 295, 184, 15, 133, 122, 71, 165, 375, 219, 189, 180, 310, 81, 64, 160, 105, 278, 347, 362, 153, 339, 349, 156, 141, 354, 162, 291, 209, 21, 201, 216, 31, 120, 52, 131, 23, 50, 345, 268, 215, 350, 57, 137, 367, 287, 280, 327, 334, 114, 130, 121, 274, 301, 239, 186, 194, 290, 53, 276, 267, 73, 152, 242, 340, 108, 5, 297, 4, 344, 7, 305, 212, 90, 36, 82, 40, 381, 103, 328, 158, 144, 115, 307, 222, 55, 365, 29, 188, 27, 128, 311, 229, 262, 373, 377, 296, 88, 79, 272, 270, 363, 196, 313, 19, 348, 359, 247, 208, 94, 92, 163, 255, 211, 223, 37, 39, 337, 259, 63, 308, 68, 150, 323, 147, 77, 10, 246, 245, 104, 161, 118, 336, 241, 93, 100, 210, 325, 129, 226, 333, 352, 146, 175, 220, 217, 314, 253, 213, 6, 24, 380, 75, 203, 134, 252, 250, 139, 303, 107, 126, 99, 113, 135, 127, 204, 177, 200, 320, 237, 355, 285, 11, 324, 13, 47, 360, 279, 309, 254, 364, 182, 111, 316, 315, 214, 3, 235, 38, 284, 366, 312, 199, 0, 22, 155, 249, 353, 96, 48, 228, 183, 102, 206, 197, 85, 234, 322, 145, 172, 251, 236, 191, 281, 123, 106, 46, 383, 148, 171, 87, 346, 61, 157, 70, 142, 167, 227, 368, 257, 136, 109, 32, 56, 1, 25, 67, 218, 273, 62, 338, 341, 119, 271, 159, 298, 275, 149, 117, 263, 240, 319, 342, 371, 74, 20, 91, 28, 185, 300, 202, 292, 170, 351, 238, 89, 258, 65, 112, 101, 76, 173, 72, 261, 221, 260, 59, 283, 230, 49, 376, 232, 18, 80, 97, 361, 35, 9, 45, 193, 331, 224, 335, 132, 41, 244, 58, 116, 168, 43, 174, 151, 326, 277, 78, 357, 143, 178, 369, 288, 42, 34, 372, 330, 318, 98, 179, 30, 83, 207, 265, 266, 225, 302, 66, 332, 374, 181, 166, 321, 231, 299, 14, 233, 110, 282, 192, 358, 51, 343, 16, 187, 164, 306, 195, 154, 379, 169, 269, 382, 378, 294, 2, 205, 356, 243, 17, 289, 124, 198, 12, 248, 293, 138, 286, 60, 176, 8, 190, 95, 54, 317, 44, 84, 264] + 51: [24, 12, 229, 73, 16, 182, 64, 65, 60, 235, 105, 253, 49, 247, 374, 23, 275, 74, 341, 366, 192, 34, 309, 195, 361, 320, 289, 332, 115, 89, 328, 278, 319, 339, 43, 240, 83, 354, 158, 277, 70, 133, 185, 244, 21, 226, 326, 348, 186, 208, 126, 189, 111, 52, 136, 62, 104, 301, 304, 168, 11, 5, 216, 47, 245, 142, 86, 329, 280, 91, 321, 44, 236, 265, 69, 383, 315, 381, 114, 242, 93, 109, 53, 25, 66, 9, 257, 305, 355, 227, 302, 347, 76, 255, 156, 202, 372, 323, 82, 106, 98, 284, 176, 211, 270, 124, 129, 177, 346, 92, 35, 85, 212, 308, 237, 107, 279, 351, 204, 251, 10, 362, 324, 311, 94, 258, 379, 191, 307, 273, 55, 167, 125, 148, 28, 272, 154, 132, 175, 274, 225, 267, 327, 249, 102, 200, 144, 286, 371, 113, 228, 72, 87, 32, 330, 146, 269, 314, 259, 20, 285, 344, 172, 268, 18, 181, 59, 160, 166, 41, 224, 171, 100, 312, 350, 338, 145, 363, 336, 378, 184, 1, 230, 165, 174, 217, 179, 266, 22, 38, 121, 139, 137, 110, 287, 31, 206, 193, 180, 264, 135, 203, 233, 153, 246, 263, 8, 294, 292, 382, 97, 162, 380, 310, 370, 119, 50, 48, 234, 317, 369, 365, 27, 54, 276, 333, 159, 112, 218, 57, 4, 14, 199, 376, 187, 352, 36, 231, 151, 290, 373, 210, 17, 108, 6, 99, 356, 300, 343, 95, 250, 283, 130, 26, 46, 220, 331, 161, 81, 198, 219, 190, 79, 252, 141, 3, 205, 262, 196, 349, 170, 223, 80, 340, 197, 232, 19, 116, 7, 215, 0, 120, 293, 29, 117, 377, 2, 138, 375, 37, 342, 322, 101, 357, 122, 67, 140, 260, 188, 295, 103, 335, 299, 118, 281, 303, 254, 56, 42, 209, 207, 33, 368, 164, 306, 84, 313, 297, 45, 194, 78, 298, 58, 238, 149, 214, 39, 296, 77, 128, 71, 63, 147, 131, 155, 261, 221, 241, 337, 163, 178, 123, 127, 358, 152, 40, 325, 282, 243, 134, 13, 143, 183, 271, 345, 359, 360, 222, 90, 248, 75, 239, 157, 364, 213, 334, 51, 30, 288, 353, 61, 316, 15, 68, 96, 88, 201, 367, 169, 318, 150, 256, 173, 291] + 52: [381, 209, 121, 372, 130, 289, 345, 82, 41, 33, 26, 269, 349, 50, 325, 379, 222, 105, 227, 23, 134, 146, 301, 331, 213, 333, 326, 268, 58, 110, 59, 152, 318, 249, 143, 342, 114, 4, 353, 276, 297, 203, 159, 142, 317, 199, 204, 176, 99, 239, 92, 217, 151, 323, 34, 298, 382, 60, 44, 168, 252, 98, 88, 133, 221, 191, 281, 160, 116, 280, 3, 154, 279, 208, 45, 225, 2, 277, 244, 21, 139, 347, 303, 197, 111, 96, 358, 363, 9, 185, 202, 144, 373, 264, 212, 180, 334, 336, 327, 216, 63, 214, 206, 77, 183, 68, 80, 192, 54, 354, 37, 171, 378, 5, 296, 30, 233, 292, 361, 319, 39, 287, 40, 93, 201, 177, 294, 189, 179, 100, 356, 322, 226, 311, 181, 112, 36, 187, 19, 15, 86, 272, 370, 247, 102, 137, 52, 332, 31, 73, 321, 193, 184, 190, 140, 196, 210, 236, 70, 312, 285, 286, 315, 124, 248, 369, 72, 17, 374, 337, 306, 308, 89, 242, 282, 84, 220, 129, 188, 118, 344, 313, 229, 302, 145, 255, 65, 122, 240, 360, 162, 348, 107, 103, 25, 231, 267, 362, 290, 211, 78, 346, 357, 219, 355, 95, 257, 123, 256, 234, 341, 223, 364, 300, 76, 53, 69, 66, 27, 161, 352, 383, 20, 238, 51, 136, 28, 310, 340, 343, 293, 147, 132, 314, 265, 338, 14, 295, 16, 305, 157, 283, 48, 307, 320, 246, 195, 263, 7, 172, 245, 259, 266, 8, 291, 108, 339, 166, 83, 173, 75, 243, 113, 377, 170, 324, 109, 119, 270, 200, 365, 207, 230, 330, 101, 380, 18, 126, 304, 275, 158, 42, 273, 241, 1, 376, 46, 175, 71, 97, 186, 61, 182, 85, 174, 163, 13, 62, 260, 56, 87, 149, 309, 228, 74, 117, 359, 167, 367, 6, 153, 351, 316, 55, 125, 235, 57, 258, 148, 194, 47, 198, 49, 141, 43, 299, 135, 232, 261, 79, 11, 366, 254, 94, 218, 375, 104, 165, 205, 120, 371, 250, 271, 169, 90, 288, 10, 128, 32, 64, 35, 155, 127, 262, 178, 215, 29, 67, 328, 115, 22, 91, 253, 350, 237, 284, 224, 251, 150, 368, 156, 278, 81, 335, 138, 274, 329, 131, 12, 0, 24, 164, 106, 38] + 53: [110, 363, 118, 160, 309, 237, 225, 134, 269, 127, 62, 12, 361, 77, 313, 197, 259, 159, 175, 146, 111, 339, 57, 370, 136, 358, 251, 101, 275, 7, 14, 43, 177, 1, 74, 34, 267, 212, 98, 286, 223, 307, 167, 185, 84, 114, 210, 272, 240, 311, 186, 68, 247, 280, 345, 92, 180, 268, 162, 130, 125, 24, 325, 94, 381, 383, 152, 300, 176, 89, 122, 362, 369, 216, 375, 189, 236, 333, 299, 59, 258, 21, 201, 289, 323, 48, 39, 64, 163, 260, 211, 165, 11, 191, 314, 312, 73, 354, 4, 8, 66, 173, 214, 155, 145, 360, 194, 103, 368, 284, 241, 168, 346, 341, 188, 324, 277, 47, 15, 143, 328, 120, 2, 166, 96, 215, 198, 294, 0, 45, 53, 80, 331, 83, 298, 106, 303, 263, 288, 213, 33, 99, 377, 316, 38, 109, 32, 217, 222, 142, 273, 227, 132, 6, 291, 26, 285, 326, 5, 200, 25, 338, 205, 378, 373, 281, 72, 282, 226, 128, 174, 65, 158, 278, 133, 220, 242, 257, 108, 380, 29, 199, 342, 113, 372, 356, 41, 27, 204, 245, 265, 18, 16, 296, 330, 75, 270, 164, 365, 42, 302, 202, 376, 81, 121, 20, 95, 256, 329, 112, 116, 37, 60, 10, 76, 353, 233, 248, 178, 276, 207, 28, 138, 232, 254, 234, 352, 264, 305, 85, 348, 184, 93, 104, 91, 61, 274, 58, 244, 140, 317, 287, 250, 230, 297, 129, 46, 224, 228, 156, 351, 144, 119, 31, 253, 3, 262, 141, 172, 239, 52, 150, 126, 359, 340, 36, 206, 321, 306, 117, 40, 364, 195, 54, 137, 246, 100, 82, 310, 229, 208, 135, 190, 181, 187, 318, 22, 56, 13, 334, 295, 336, 243, 320, 322, 379, 266, 209, 86, 332, 221, 50, 319, 88, 139, 183, 231, 335, 235, 149, 315, 293, 357, 196, 169, 105, 327, 79, 255, 182, 55, 337, 367, 131, 9, 51, 147, 301, 374, 382, 192, 343, 304, 344, 252, 90, 350, 23, 347, 308, 203, 238, 292, 179, 170, 279, 157, 261, 271, 107, 171, 219, 70, 69, 151, 49, 78, 349, 44, 102, 67, 35, 115, 371, 19, 30, 123, 218, 97, 87, 154, 283, 249, 63, 193, 17, 366, 161, 148, 124, 355, 71, 290, 153] + 54: [363, 71, 39, 80, 172, 341, 290, 311, 180, 228, 121, 70, 310, 357, 50, 325, 142, 109, 376, 186, 100, 67, 48, 85, 137, 62, 81, 262, 9, 175, 334, 110, 234, 367, 261, 89, 66, 270, 307, 134, 169, 297, 383, 106, 354, 336, 361, 378, 370, 299, 296, 83, 82, 250, 72, 254, 330, 119, 7, 329, 276, 333, 129, 64, 323, 190, 159, 279, 204, 237, 183, 318, 69, 163, 22, 242, 377, 273, 128, 266, 371, 219, 346, 29, 249, 108, 248, 374, 51, 65, 112, 256, 19, 200, 36, 148, 294, 231, 286, 253, 90, 260, 20, 324, 93, 57, 117, 86, 16, 132, 47, 32, 322, 91, 185, 272, 239, 184, 365, 302, 278, 264, 301, 136, 96, 33, 347, 191, 351, 12, 88, 145, 236, 274, 343, 362, 149, 35, 104, 252, 37, 342, 123, 13, 339, 223, 280, 380, 147, 257, 98, 306, 208, 305, 309, 3, 27, 156, 41, 196, 158, 17, 321, 182, 24, 21, 312, 49, 1, 366, 293, 107, 304, 173, 18, 300, 194, 229, 213, 6, 178, 233, 79, 60, 199, 116, 210, 188, 222, 349, 127, 160, 155, 356, 230, 153, 338, 150, 30, 281, 282, 221, 25, 314, 101, 332, 99, 218, 277, 225, 265, 92, 382, 157, 105, 135, 177, 373, 327, 122, 179, 162, 151, 316, 245, 243, 259, 235, 247, 331, 268, 46, 226, 154, 4, 77, 126, 78, 202, 232, 164, 11, 38, 337, 189, 216, 133, 174, 120, 187, 26, 118, 181, 375, 76, 197, 198, 335, 220, 152, 54, 381, 55, 368, 61, 144, 326, 111, 146, 167, 166, 23, 141, 251, 308, 59, 168, 289, 359, 102, 227, 292, 28, 138, 165, 193, 358, 320, 284, 258, 348, 295, 328, 287, 369, 53, 44, 345, 352, 372, 58, 10, 143, 113, 215, 45, 5, 217, 103, 140, 97, 315, 73, 171, 255, 313, 364, 353, 244, 43, 238, 271, 15, 224, 14, 31, 269, 203, 56, 360, 350, 291, 207, 340, 130, 205, 283, 42, 240, 95, 319, 267, 214, 8, 176, 87, 40, 75, 317, 288, 115, 161, 84, 124, 94, 34, 125, 170, 355, 131, 246, 298, 68, 209, 201, 114, 195, 275, 139, 192, 63, 74, 211, 206, 52, 263, 285, 303, 379, 2, 241, 344, 212, 0] + 55: [154, 323, 113, 256, 62, 39, 16, 41, 121, 106, 249, 71, 272, 300, 316, 233, 197, 276, 95, 260, 231, 343, 194, 136, 318, 157, 207, 313, 63, 160, 127, 173, 356, 321, 168, 315, 66, 68, 280, 169, 330, 362, 100, 345, 205, 222, 245, 109, 174, 90, 269, 43, 151, 89, 252, 353, 247, 40, 341, 176, 320, 164, 86, 239, 5, 325, 105, 378, 377, 47, 54, 241, 122, 254, 193, 22, 117, 232, 133, 208, 372, 336, 55, 147, 145, 261, 77, 376, 334, 75, 195, 351, 292, 144, 196, 215, 56, 80, 74, 326, 270, 309, 357, 307, 73, 78, 132, 223, 226, 60, 138, 26, 156, 112, 350, 190, 217, 236, 240, 228, 140, 346, 358, 278, 294, 302, 342, 188, 373, 359, 118, 38, 354, 374, 139, 203, 191, 155, 266, 158, 24, 25, 250, 204, 202, 116, 347, 115, 327, 235, 370, 227, 6, 70, 301, 134, 279, 44, 368, 271, 206, 163, 348, 375, 251, 380, 360, 187, 322, 257, 171, 52, 314, 214, 1, 365, 15, 335, 35, 243, 58, 107, 177, 369, 295, 175, 32, 108, 59, 379, 96, 317, 88, 340, 48, 76, 152, 4, 218, 192, 267, 363, 65, 99, 8, 331, 213, 0, 344, 46, 170, 289, 371, 246, 79, 53, 19, 119, 287, 101, 149, 82, 198, 103, 37, 30, 93, 209, 120, 125, 248, 162, 364, 21, 36, 288, 148, 167, 51, 34, 81, 308, 299, 29, 28, 355, 339, 311, 211, 273, 291, 200, 179, 349, 290, 383, 124, 17, 284, 91, 210, 265, 306, 286, 332, 277, 23, 84, 87, 264, 61, 367, 219, 253, 126, 104, 338, 150, 324, 183, 92, 262, 57, 142, 131, 9, 283, 69, 98, 146, 381, 130, 159, 7, 182, 111, 303, 382, 199, 293, 3, 333, 216, 366, 172, 259, 102, 274, 27, 224, 42, 135, 263, 184, 128, 85, 296, 31, 237, 281, 225, 268, 319, 137, 129, 275, 13, 166, 298, 181, 110, 220, 18, 229, 310, 305, 20, 11, 123, 180, 72, 234, 165, 238, 328, 304, 49, 2, 114, 12, 221, 312, 337, 189, 14, 352, 329, 361, 67, 178, 185, 10, 255, 94, 33, 153, 212, 258, 143, 97, 201, 297, 161, 50, 64, 242, 141, 282, 244, 83, 230, 45, 186, 285] + 56: [231, 340, 100, 199, 98, 371, 20, 333, 328, 316, 275, 66, 159, 150, 383, 0, 58, 40, 253, 135, 295, 166, 355, 110, 255, 270, 286, 115, 122, 134, 145, 347, 323, 274, 37, 16, 70, 93, 254, 272, 239, 261, 36, 25, 247, 142, 146, 3, 240, 225, 304, 167, 186, 158, 156, 89, 306, 46, 99, 18, 327, 245, 101, 223, 233, 376, 148, 10, 21, 79, 268, 280, 133, 164, 227, 368, 95, 94, 54, 341, 290, 200, 276, 372, 144, 130, 47, 337, 176, 349, 366, 111, 171, 301, 173, 90, 195, 15, 75, 343, 116, 184, 169, 236, 125, 161, 48, 2, 19, 288, 244, 97, 344, 334, 342, 278, 92, 252, 230, 121, 34, 153, 209, 297, 190, 348, 180, 314, 132, 293, 319, 65, 29, 86, 113, 131, 345, 203, 370, 377, 114, 237, 74, 85, 260, 213, 5, 311, 107, 296, 87, 196, 187, 6, 39, 219, 332, 14, 283, 309, 197, 73, 38, 137, 181, 31, 374, 331, 50, 33, 123, 282, 339, 299, 183, 262, 228, 28, 330, 289, 67, 352, 287, 157, 382, 212, 52, 7, 277, 91, 102, 143, 220, 82, 30, 8, 273, 88, 351, 356, 264, 162, 71, 292, 329, 381, 168, 120, 205, 27, 285, 250, 206, 198, 179, 12, 1, 17, 127, 56, 221, 201, 257, 84, 353, 124, 379, 367, 78, 43, 373, 258, 26, 325, 335, 216, 174, 165, 229, 208, 294, 178, 259, 177, 81, 163, 322, 249, 104, 117, 140, 302, 362, 154, 192, 72, 246, 300, 160, 193, 324, 338, 23, 22, 11, 13, 243, 364, 313, 360, 298, 119, 172, 307, 175, 57, 64, 350, 60, 152, 108, 358, 336, 217, 194, 235, 76, 224, 321, 44, 106, 279, 126, 53, 303, 9, 263, 308, 346, 41, 83, 77, 357, 118, 238, 103, 182, 312, 105, 305, 354, 317, 214, 151, 141, 129, 42, 62, 210, 218, 310, 4, 226, 139, 266, 361, 222, 204, 234, 215, 326, 45, 365, 318, 109, 315, 242, 265, 241, 147, 191, 155, 267, 189, 363, 251, 320, 284, 188, 80, 63, 207, 369, 359, 69, 281, 128, 136, 35, 269, 149, 138, 185, 248, 378, 271, 55, 96, 375, 112, 232, 291, 49, 170, 24, 211, 256, 68, 59, 32, 380, 51, 61, 202] + 57: [254, 125, 159, 121, 198, 257, 324, 267, 6, 283, 160, 163, 238, 91, 172, 213, 214, 113, 312, 111, 38, 55, 328, 226, 352, 209, 28, 132, 29, 237, 49, 211, 200, 168, 219, 256, 239, 78, 308, 258, 82, 221, 77, 153, 202, 262, 301, 304, 276, 148, 170, 342, 314, 7, 128, 53, 118, 150, 197, 88, 62, 272, 167, 373, 40, 51, 249, 356, 102, 203, 106, 95, 210, 196, 282, 155, 186, 162, 261, 315, 236, 255, 357, 333, 361, 370, 232, 10, 264, 206, 278, 227, 280, 367, 275, 343, 326, 205, 72, 93, 158, 63, 65, 124, 268, 350, 318, 79, 123, 182, 99, 253, 322, 228, 70, 83, 21, 183, 284, 13, 279, 171, 337, 90, 330, 18, 371, 97, 33, 59, 0, 32, 109, 194, 48, 364, 271, 266, 52, 327, 181, 15, 229, 372, 25, 187, 320, 11, 16, 368, 235, 137, 344, 169, 117, 64, 44, 354, 58, 39, 346, 73, 27, 298, 184, 147, 12, 380, 2, 164, 199, 362, 178, 215, 240, 145, 252, 24, 270, 126, 66, 36, 374, 139, 61, 115, 291, 292, 305, 34, 105, 376, 231, 246, 286, 157, 133, 156, 101, 274, 8, 204, 151, 22, 35, 251, 347, 161, 179, 297, 42, 46, 349, 92, 358, 112, 329, 313, 190, 321, 98, 334, 290, 247, 5, 265, 307, 281, 263, 50, 192, 76, 273, 218, 114, 340, 69, 245, 285, 56, 14, 89, 94, 154, 381, 57, 104, 359, 335, 300, 127, 317, 306, 80, 338, 316, 110, 9, 177, 277, 191, 54, 47, 119, 348, 71, 351, 369, 288, 17, 379, 363, 45, 303, 378, 87, 224, 242, 67, 103, 142, 244, 129, 189, 336, 201, 233, 293, 146, 165, 207, 302, 31, 220, 140, 223, 130, 60, 174, 345, 185, 75, 319, 43, 176, 341, 152, 20, 382, 116, 120, 225, 68, 136, 3, 296, 81, 311, 41, 332, 122, 135, 299, 241, 1, 222, 138, 107, 310, 100, 212, 193, 30, 37, 19, 86, 259, 295, 331, 74, 230, 85, 366, 108, 141, 217, 84, 180, 309, 250, 144, 365, 134, 243, 96, 173, 377, 208, 289, 325, 166, 149, 353, 23, 216, 323, 175, 26, 355, 248, 4, 188, 383, 260, 375, 287, 360, 339, 294, 269, 195, 131, 143, 234] + 58: [377, 169, 232, 324, 367, 132, 4, 190, 198, 229, 55, 239, 129, 236, 160, 183, 149, 362, 379, 334, 347, 113, 195, 8, 64, 243, 372, 330, 354, 374, 223, 109, 34, 15, 301, 91, 361, 279, 321, 168, 201, 340, 33, 38, 58, 134, 209, 308, 220, 114, 61, 27, 373, 9, 351, 256, 112, 376, 85, 42, 130, 162, 66, 185, 48, 202, 383, 284, 350, 326, 37, 0, 144, 148, 123, 345, 19, 35, 237, 170, 294, 87, 317, 41, 254, 247, 117, 145, 17, 30, 140, 89, 147, 292, 184, 204, 81, 194, 298, 76, 348, 101, 49, 67, 199, 368, 295, 342, 119, 104, 111, 307, 328, 380, 314, 207, 181, 238, 227, 346, 135, 172, 176, 288, 2, 287, 50, 133, 118, 5, 138, 221, 303, 103, 44, 349, 137, 52, 127, 218, 233, 158, 309, 280, 245, 98, 311, 263, 7, 344, 93, 69, 355, 116, 68, 6, 203, 315, 316, 20, 28, 281, 375, 82, 99, 215, 155, 382, 332, 13, 253, 161, 371, 293, 179, 136, 226, 24, 142, 359, 157, 336, 269, 120, 153, 78, 188, 323, 23, 320, 335, 364, 156, 92, 122, 248, 356, 210, 322, 22, 214, 178, 262, 47, 31, 12, 255, 197, 171, 1, 331, 143, 258, 306, 234, 378, 63, 302, 154, 107, 228, 265, 283, 94, 230, 296, 100, 54, 163, 146, 46, 369, 216, 88, 267, 343, 165, 80, 3, 370, 275, 205, 312, 249, 273, 211, 235, 72, 257, 36, 196, 141, 191, 97, 60, 242, 224, 192, 139, 59, 62, 53, 250, 240, 131, 208, 357, 278, 21, 11, 182, 105, 180, 102, 360, 86, 268, 200, 51, 96, 159, 193, 365, 363, 212, 164, 71, 10, 126, 305, 222, 77, 79, 271, 206, 358, 26, 40, 152, 341, 189, 277, 260, 352, 73, 187, 381, 244, 325, 83, 339, 75, 121, 300, 231, 65, 318, 115, 353, 219, 297, 213, 29, 128, 276, 290, 84, 39, 246, 286, 74, 217, 291, 166, 125, 108, 56, 366, 177, 151, 259, 14, 299, 289, 270, 175, 45, 18, 16, 272, 106, 274, 285, 329, 251, 25, 266, 90, 124, 333, 110, 241, 186, 327, 43, 70, 225, 173, 338, 261, 95, 264, 282, 337, 167, 57, 32, 304, 174, 310, 313, 252, 319, 150] + 59: [327, 220, 113, 105, 226, 374, 291, 172, 328, 151, 281, 290, 45, 40, 16, 181, 162, 315, 29, 47, 355, 348, 366, 6, 353, 199, 107, 333, 246, 34, 84, 330, 359, 44, 183, 53, 166, 303, 8, 141, 350, 375, 347, 345, 167, 4, 90, 189, 25, 260, 346, 182, 97, 251, 371, 150, 213, 295, 144, 196, 86, 176, 160, 279, 297, 248, 252, 245, 10, 236, 344, 238, 13, 228, 156, 48, 71, 27, 208, 134, 157, 93, 49, 24, 118, 296, 82, 165, 240, 14, 293, 87, 78, 72, 235, 154, 170, 202, 66, 33, 313, 70, 263, 130, 185, 324, 80, 255, 58, 212, 81, 35, 302, 163, 376, 338, 214, 158, 137, 92, 337, 123, 314, 32, 287, 41, 135, 184, 17, 326, 336, 275, 273, 307, 368, 319, 1, 56, 77, 171, 83, 99, 308, 215, 229, 195, 340, 73, 285, 5, 298, 288, 300, 186, 262, 382, 342, 109, 223, 254, 280, 334, 22, 60, 286, 258, 146, 369, 365, 2, 61, 194, 138, 43, 50, 124, 28, 74, 88, 249, 148, 102, 241, 237, 318, 360, 272, 244, 104, 122, 310, 271, 7, 94, 173, 332, 30, 147, 247, 331, 31, 372, 26, 266, 133, 175, 161, 12, 100, 169, 143, 19, 306, 69, 120, 354, 203, 51, 68, 299, 193, 63, 187, 217, 21, 89, 261, 115, 201, 230, 312, 108, 239, 192, 335, 378, 126, 323, 111, 268, 140, 370, 59, 234, 46, 95, 381, 294, 18, 9, 23, 243, 112, 91, 57, 127, 267, 383, 20, 218, 356, 179, 180, 177, 36, 101, 242, 0, 206, 64, 136, 253, 207, 153, 132, 210, 357, 103, 358, 98, 316, 305, 39, 264, 55, 190, 15, 341, 209, 379, 200, 65, 75, 233, 311, 76, 42, 38, 178, 309, 114, 278, 85, 159, 364, 320, 373, 142, 329, 339, 198, 361, 155, 131, 274, 362, 380, 110, 325, 231, 67, 145, 277, 191, 321, 284, 216, 257, 276, 219, 121, 367, 139, 116, 117, 79, 224, 250, 363, 352, 256, 197, 204, 221, 265, 377, 317, 232, 128, 164, 304, 168, 211, 62, 225, 174, 54, 222, 269, 259, 289, 349, 96, 292, 227, 149, 129, 322, 343, 37, 3, 106, 11, 119, 283, 301, 282, 125, 270, 205, 152, 52, 351, 188] + 60: [313, 94, 195, 372, 78, 65, 208, 239, 49, 166, 98, 272, 59, 27, 169, 151, 20, 6, 240, 332, 110, 215, 116, 186, 90, 85, 146, 182, 95, 219, 225, 338, 282, 361, 328, 258, 145, 356, 291, 72, 43, 289, 340, 10, 218, 214, 288, 201, 124, 174, 335, 129, 339, 230, 185, 74, 294, 180, 329, 278, 231, 238, 99, 336, 255, 269, 183, 11, 237, 274, 25, 134, 142, 293, 196, 300, 322, 256, 64, 197, 168, 13, 209, 179, 42, 204, 96, 86, 135, 367, 32, 44, 242, 154, 375, 355, 351, 83, 310, 170, 244, 212, 312, 81, 34, 319, 119, 15, 131, 161, 241, 253, 0, 277, 37, 1, 188, 264, 377, 304, 21, 107, 66, 7, 192, 370, 286, 18, 325, 270, 45, 228, 216, 50, 68, 97, 159, 254, 163, 4, 101, 308, 317, 22, 123, 60, 265, 128, 303, 316, 247, 297, 205, 76, 365, 345, 245, 143, 263, 359, 84, 283, 132, 318, 80, 267, 57, 342, 233, 70, 187, 30, 369, 314, 380, 148, 363, 376, 127, 287, 261, 224, 160, 147, 165, 281, 164, 103, 118, 89, 62, 299, 257, 139, 40, 121, 243, 113, 194, 334, 2, 108, 353, 193, 213, 117, 235, 246, 75, 58, 221, 36, 344, 9, 3, 279, 26, 346, 31, 200, 100, 357, 276, 12, 315, 307, 262, 330, 19, 198, 374, 378, 301, 306, 292, 177, 73, 77, 210, 207, 16, 47, 199, 236, 266, 39, 167, 302, 28, 35, 290, 311, 206, 184, 271, 364, 152, 69, 114, 51, 92, 162, 38, 8, 323, 52, 273, 33, 190, 106, 296, 14, 125, 105, 202, 252, 320, 280, 379, 29, 175, 326, 173, 138, 352, 87, 373, 348, 268, 220, 126, 284, 260, 343, 48, 153, 181, 327, 234, 112, 354, 232, 203, 55, 172, 189, 248, 93, 217, 88, 382, 176, 56, 141, 360, 91, 149, 341, 349, 144, 331, 383, 67, 23, 178, 137, 41, 211, 111, 191, 350, 71, 275, 321, 136, 157, 381, 158, 133, 102, 115, 347, 222, 120, 155, 366, 337, 46, 309, 122, 223, 82, 140, 150, 362, 358, 259, 24, 5, 61, 305, 171, 53, 285, 250, 130, 79, 368, 371, 333, 156, 251, 324, 54, 104, 229, 298, 249, 17, 227, 295, 109, 226, 63] + 61: [221, 248, 107, 360, 247, 241, 363, 330, 278, 235, 129, 115, 159, 32, 43, 348, 233, 44, 190, 47, 336, 207, 202, 139, 292, 8, 365, 225, 67, 71, 342, 193, 311, 163, 364, 312, 13, 314, 223, 236, 56, 281, 134, 41, 16, 356, 94, 58, 127, 167, 88, 120, 326, 270, 45, 12, 51, 237, 112, 110, 15, 169, 337, 224, 77, 83, 82, 109, 206, 145, 33, 72, 78, 275, 125, 269, 295, 124, 204, 298, 91, 239, 101, 74, 201, 272, 138, 250, 60, 226, 210, 50, 40, 128, 319, 0, 179, 100, 161, 230, 170, 323, 59, 253, 372, 142, 119, 333, 42, 156, 75, 258, 297, 29, 27, 379, 308, 368, 200, 259, 38, 273, 37, 126, 240, 317, 332, 152, 274, 25, 255, 358, 24, 293, 28, 339, 220, 285, 131, 267, 325, 296, 343, 208, 338, 36, 307, 172, 157, 178, 209, 103, 17, 215, 66, 57, 188, 243, 133, 361, 65, 84, 166, 11, 257, 23, 147, 173, 383, 214, 9, 79, 196, 132, 194, 106, 244, 271, 6, 99, 262, 305, 212, 105, 347, 377, 175, 183, 320, 229, 189, 53, 61, 381, 324, 86, 335, 205, 10, 184, 353, 238, 55, 327, 34, 349, 309, 228, 20, 316, 300, 113, 2, 260, 328, 290, 154, 137, 280, 242, 283, 340, 199, 359, 70, 130, 354, 123, 251, 185, 373, 265, 35, 222, 279, 289, 367, 334, 350, 155, 73, 197, 344, 231, 5, 351, 19, 1, 195, 371, 143, 288, 168, 286, 227, 261, 191, 268, 366, 313, 304, 3, 52, 102, 186, 153, 306, 341, 85, 141, 176, 14, 162, 352, 187, 18, 7, 111, 164, 48, 256, 64, 277, 382, 76, 217, 96, 321, 93, 46, 263, 216, 108, 165, 203, 264, 218, 294, 213, 192, 369, 31, 282, 87, 362, 90, 114, 252, 329, 69, 104, 249, 322, 89, 375, 22, 219, 81, 144, 284, 370, 345, 54, 301, 49, 331, 180, 146, 140, 276, 151, 355, 376, 266, 171, 287, 211, 299, 315, 303, 136, 122, 117, 380, 21, 68, 198, 291, 177, 181, 160, 92, 118, 150, 135, 121, 374, 245, 97, 158, 378, 346, 30, 174, 80, 26, 4, 98, 148, 63, 310, 234, 232, 246, 95, 182, 318, 302, 254, 39, 62, 357, 116, 149] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots416.yaml b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots416.yaml new file mode 100644 index 000000000000..434b41525a94 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots416.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [259, 356, 356, 225, 177, 111, 230, 378, 51, 60, 204, 218, 232, 376, 203, 293, 224, 144, 141, 181, 107, 43, 57, 361, 28, 248, 309, 262, 5, 91, 317, 137, 363, 192, 296, 270, 299, 326, 50, 252, 352, 316, 175, 2, 301, 125, 132, 304, 364, 273, 13, 373, 166, 302, 236, 86, 348, 135, 193, 45, 169, 21, 79, 157, 184, 255, 217, 318, 330, 209, 22, 72, 101, 233, 109, 129, 324, 208, 123, 314, 39, 133, 4, 108, 116, 216, 100, 379, 382, 263, 20, 210, 42, 212, 315, 334, 367, 251, 298, 329, 339, 8, 171, 274, 166, 351, 322, 371, 165, 111, 207, 45, 188, 81, 79, 17, 327, 48, 217, 198, 330, 282, 55, 307, 11, 313, 297, 311, 114, 44, 268, 234, 52, 308, 103, 146, 140, 19, 180, 320, 127, 14, 145, 350, 15, 64, 53, 69, 319, 164, 18, 226, 291, 88, 370, 185, 0, 222, 236, 225, 162, 135, 279, 190, 182, 122, 62, 29, 134, 70, 139, 276, 290, 349, 336, 33, 117, 156, 10, 36, 261, 148, 31, 358, 85, 187, 136, 240, 178, 99, 211, 195, 82, 328, 47, 102, 338, 340, 380, 179, 381, 250, 167, 374, 325, 323, 58, 201, 256, 71, 159, 245, 254, 264, 158, 378, 163, 153, 202, 17, 49, 289, 176, 16, 173, 3, 78, 284, 34, 283, 186, 200, 112, 300, 27, 89, 66, 32, 214, 249, 246, 275, 238, 292, 83, 93, 260, 229, 161, 253, 353, 152, 194, 357, 142, 24, 228, 130, 277, 61, 256, 222, 345, 371, 165, 254, 76, 346, 35, 26, 104, 95, 49, 321, 68, 333, 173, 272, 359, 54, 131, 56, 354, 265, 368, 7, 77, 375, 377, 172, 227, 331, 269, 366, 1, 25, 73, 67, 342, 98, 294, 206, 12, 219, 9, 285, 271, 121, 281, 168, 118, 247, 150, 74, 119, 383, 155, 111, 279, 332, 312, 347, 242, 80, 134, 65, 63, 37, 290, 126, 360, 243, 105, 341, 90, 258, 138, 170, 149, 38, 87, 244, 335, 365, 372, 369, 97, 106, 147, 237, 362, 286, 305, 115, 306, 191, 231, 220, 23, 189, 355, 288, 310, 303, 151, 74, 84, 245, 165, 235, 241, 378, 295, 60, 30, 95, 184, 343, 203, 278, 224, 280, 143, 205, 196, 183, 124, 174, 94, 337, 160, 199, 223, 257, 213, 46, 75, 59, 215, 221, 197, 267, 6, 154, 40, 96, 41, 113, 239, 266, 92, 287, 128, 344, 120, 110] + 1: [134, 174, 118, 158, 150, 152, 169, 163, 21, 192, 325, 377, 63, 299, 111, 153, 224, 273, 287, 86, 216, 99, 108, 280, 4, 47, 358, 171, 101, 102, 313, 222, 161, 332, 295, 321, 87, 315, 13, 302, 262, 356, 279, 371, 183, 253, 208, 64, 258, 304, 312, 215, 178, 201, 137, 106, 382, 91, 167, 127, 319, 6, 5, 73, 334, 56, 120, 351, 272, 243, 37, 89, 125, 212, 162, 335, 213, 206, 88, 38, 133, 256, 79, 348, 249, 14, 296, 57, 172, 350, 94, 323, 353, 27, 190, 129, 316, 135, 10, 191, 159, 359, 310, 34, 380, 201, 137, 80, 277, 91, 167, 254, 131, 67, 270, 328, 58, 244, 225, 82, 383, 292, 275, 71, 234, 146, 288, 109, 219, 228, 289, 278, 236, 76, 121, 259, 370, 107, 115, 62, 290, 26, 104, 113, 54, 119, 369, 331, 193, 209, 124, 155, 307, 354, 112, 368, 141, 42, 308, 44, 126, 263, 177, 127, 347, 67, 70, 123, 105, 120, 250, 271, 320, 138, 303, 247, 317, 355, 197, 92, 223, 116, 69, 360, 95, 298, 122, 330, 65, 286, 184, 154, 35, 230, 311, 268, 226, 333, 375, 372, 367, 221, 128, 322, 117, 257, 199, 255, 251, 42, 22, 180, 149, 50, 169, 163, 40, 39, 179, 41, 3, 198, 23, 185, 46, 59, 9, 32, 266, 285, 187, 261, 143, 282, 306, 376, 60, 98, 7, 305, 2, 20, 175, 136, 324, 11, 97, 8, 232, 314, 130, 246, 188, 373, 361, 45, 276, 339, 345, 15, 81, 235, 31, 158, 85, 72, 177, 329, 347, 30, 366, 110, 182, 294, 111, 233, 272, 29, 269, 242, 281, 74, 218, 379, 52, 53, 264, 202, 231, 189, 164, 238, 338, 326, 19, 33, 186, 51, 352, 346, 344, 77, 357, 220, 340, 217, 132, 16, 144, 28, 336, 265, 200, 1, 174, 180, 382, 43, 196, 378, 319, 241, 0, 210, 3, 198, 225, 24, 283, 292, 211, 260, 245, 301, 84, 194, 214, 176, 203, 239, 61, 205, 362, 204, 364, 17, 156, 68, 36, 284, 96, 181, 240, 103, 83, 297, 160, 207, 252, 343, 349, 341, 363, 327, 365, 365, 22, 18, 277, 72, 291, 163, 319, 168, 75, 377, 182, 300, 248, 82, 25, 195, 145, 173, 78, 274, 166, 142, 114, 93, 381, 342, 147, 237, 140, 170, 139, 318, 229, 165, 48, 293, 227, 55, 12, 49, 148, 309, 90, 151, 66, 374, 337, 157, 267, 100] + 2: [53, 341, 215, 93, 223, 295, 125, 197, 232, 19, 109, 322, 131, 280, 86, 220, 31, 296, 200, 82, 285, 61, 206, 224, 27, 262, 120, 218, 41, 65, 309, 69, 256, 12, 257, 0, 25, 281, 141, 352, 329, 99, 335, 158, 378, 283, 249, 315, 358, 282, 255, 366, 53, 351, 313, 343, 259, 295, 143, 248, 149, 213, 62, 350, 91, 377, 44, 105, 304, 26, 83, 24, 311, 79, 299, 263, 50, 203, 201, 76, 332, 241, 169, 250, 106, 39, 268, 2, 36, 334, 264, 359, 216, 370, 338, 307, 382, 177, 369, 64, 228, 354, 300, 381, 43, 108, 89, 93, 298, 221, 143, 248, 242, 101, 155, 284, 66, 56, 245, 172, 314, 368, 279, 340, 196, 174, 170, 347, 55, 188, 58, 128, 349, 308, 186, 254, 136, 49, 318, 23, 195, 346, 331, 57, 243, 371, 114, 267, 336, 222, 13, 380, 32, 339, 199, 330, 43, 225, 89, 194, 81, 137, 212, 212, 242, 290, 84, 97, 355, 70, 118, 184, 51, 122, 328, 244, 323, 204, 130, 356, 126, 327, 187, 198, 163, 16, 15, 160, 154, 142, 104, 94, 275, 35, 52, 144, 226, 95, 183, 48, 113, 345, 214, 266, 316, 208, 375, 258, 180, 108, 14, 115, 28, 3, 151, 33, 232, 185, 153, 191, 91, 377, 86, 304, 314, 22, 274, 85, 6, 342, 7, 90, 269, 210, 361, 297, 360, 145, 348, 276, 364, 121, 29, 362, 134, 4, 272, 231, 292, 373, 167, 59, 72, 234, 260, 127, 119, 270, 190, 379, 230, 112, 215, 115, 223, 3, 100, 75, 192, 78, 62, 168, 166, 278, 271, 133, 139, 324, 146, 11, 30, 294, 140, 18, 156, 320, 20, 236, 277, 10, 175, 116, 9, 305, 150, 152, 211, 367, 207, 233, 291, 107, 205, 74, 182, 147, 317, 80, 326, 237, 246, 229, 96, 112, 313, 194, 5, 321, 143, 248, 162, 290, 111, 168, 173, 286, 135, 42, 45, 325, 217, 235, 303, 165, 344, 273, 68, 67, 21, 138, 46, 171, 193, 129, 372, 252, 365, 219, 239, 47, 178, 261, 302, 288, 8, 164, 310, 37, 353, 88, 287, 337, 40, 374, 132, 132, 357, 102, 1, 189, 247, 34, 60, 19, 110, 124, 157, 73, 306, 123, 139, 122, 63, 333, 159, 209, 17, 87, 293, 92, 383, 148, 253, 265, 181, 103, 38, 312, 176, 179, 77, 238, 376, 319, 54, 71, 251, 98, 117, 227, 240, 289, 301, 202, 363, 161] + 3: [39, 34, 310, 230, 314, 319, 350, 163, 313, 62, 334, 213, 26, 168, 358, 189, 304, 316, 64, 202, 364, 289, 277, 92, 155, 103, 217, 327, 135, 307, 82, 276, 216, 298, 291, 15, 165, 68, 153, 93, 190, 323, 107, 191, 141, 148, 23, 320, 193, 247, 324, 301, 39, 34, 275, 377, 302, 319, 31, 192, 97, 80, 145, 6, 186, 207, 2, 325, 221, 174, 290, 264, 8, 161, 288, 243, 345, 299, 380, 197, 105, 113, 76, 160, 306, 249, 70, 376, 130, 346, 101, 269, 379, 251, 121, 242, 54, 368, 315, 60, 33, 5, 278, 47, 22, 34, 34, 377, 302, 319, 51, 3, 194, 151, 318, 137, 215, 63, 342, 122, 77, 219, 25, 29, 279, 359, 262, 114, 176, 218, 256, 112, 65, 266, 125, 328, 286, 184, 67, 91, 373, 228, 321, 258, 354, 73, 88, 45, 143, 330, 175, 349, 226, 332, 198, 303, 22, 34, 79, 260, 365, 14, 27, 163, 263, 263, 81, 331, 139, 178, 164, 267, 296, 195, 21, 87, 109, 142, 32, 312, 375, 204, 236, 270, 117, 280, 214, 366, 322, 20, 188, 106, 85, 132, 293, 133, 0, 16, 140, 159, 205, 75, 272, 347, 341, 335, 199, 59, 66, 66, 310, 35, 365, 18, 317, 131, 194, 24, 363, 252, 201, 170, 250, 211, 212, 182, 11, 12, 271, 154, 229, 351, 41, 126, 55, 53, 355, 44, 104, 100, 340, 361, 305, 83, 381, 282, 362, 285, 179, 116, 156, 360, 241, 89, 352, 152, 173, 52, 257, 370, 66, 66, 310, 377, 209, 40, 19, 13, 313, 157, 158, 180, 339, 343, 227, 10, 287, 36, 119, 118, 136, 99, 149, 111, 300, 224, 235, 94, 169, 255, 383, 237, 248, 353, 232, 294, 61, 259, 309, 206, 183, 134, 234, 369, 253, 127, 124, 4, 326, 9, 233, 311, 66, 71, 79, 260, 40, 171, 225, 56, 261, 333, 49, 42, 337, 102, 336, 231, 74, 239, 240, 357, 162, 84, 295, 297, 181, 108, 200, 57, 273, 48, 30, 144, 115, 187, 281, 7, 123, 110, 86, 283, 378, 372, 338, 329, 147, 95, 308, 244, 185, 17, 146, 38, 66, 46, 34, 260, 365, 18, 27, 208, 3, 348, 150, 50, 238, 120, 58, 37, 138, 177, 223, 382, 78, 172, 203, 374, 98, 274, 166, 344, 129, 246, 268, 196, 43, 96, 128, 367, 356, 72, 28, 254, 245, 220, 210, 284, 90, 1, 265, 69, 167, 371, 292, 222] + 4: [188, 124, 258, 331, 315, 214, 15, 376, 319, 117, 254, 136, 36, 167, 246, 77, 196, 44, 49, 198, 317, 187, 358, 42, 166, 189, 352, 341, 64, 232, 354, 367, 178, 294, 346, 175, 170, 181, 211, 156, 126, 203, 353, 1, 238, 11, 373, 270, 2, 17, 90, 277, 34, 375, 296, 331, 315, 214, 23, 19, 87, 117, 148, 6, 13, 59, 347, 372, 32, 314, 27, 251, 22, 207, 183, 221, 85, 360, 185, 169, 125, 70, 69, 94, 309, 52, 263, 103, 295, 304, 109, 40, 215, 129, 112, 247, 366, 245, 326, 299, 192, 231, 58, 104, 340, 12, 199, 331, 259, 371, 172, 88, 105, 194, 142, 308, 47, 329, 76, 318, 365, 160, 333, 68, 35, 179, 324, 99, 290, 356, 379, 323, 132, 131, 135, 288, 220, 306, 78, 121, 249, 334, 268, 213, 186, 21, 48, 162, 265, 224, 190, 266, 20, 79, 287, 282, 3, 271, 300, 343, 315, 214, 15, 226, 87, 273, 142, 143, 345, 274, 145, 106, 357, 43, 339, 280, 7, 153, 278, 243, 252, 330, 359, 75, 269, 383, 100, 284, 138, 267, 234, 26, 122, 289, 223, 130, 10, 115, 123, 55, 91, 102, 301, 195, 337, 272, 338, 351, 3, 271, 343, 343, 24, 193, 9, 95, 257, 273, 148, 165, 56, 377, 139, 61, 364, 101, 28, 73, 110, 141, 54, 276, 65, 114, 98, 14, 261, 262, 80, 158, 39, 92, 230, 157, 146, 140, 147, 127, 255, 97, 374, 191, 164, 248, 285, 382, 208, 233, 107, 182, 3, 37, 222, 331, 250, 371, 16, 281, 5, 171, 152, 348, 86, 155, 41, 200, 361, 83, 204, 217, 260, 177, 335, 0, 307, 256, 297, 60, 46, 229, 322, 328, 370, 50, 176, 51, 180, 210, 71, 286, 81, 378, 163, 279, 30, 202, 236, 174, 225, 242, 63, 120, 18, 271, 296, 331, 350, 320, 228, 226, 45, 194, 254, 241, 342, 197, 325, 275, 151, 57, 312, 344, 369, 159, 244, 116, 133, 72, 62, 149, 108, 216, 25, 137, 84, 66, 184, 29, 291, 237, 111, 355, 381, 205, 316, 380, 134, 144, 89, 154, 227, 67, 321, 332, 209, 209, 343, 343, 219, 23, 239, 95, 319, 150, 152, 302, 253, 298, 113, 4, 235, 218, 8, 161, 82, 38, 283, 264, 53, 168, 93, 33, 74, 313, 240, 310, 292, 349, 206, 303, 327, 118, 31, 201, 96, 311, 362, 173, 368, 293, 212, 363, 119, 128, 336, 305] + 5: [172, 128, 296, 214, 217, 346, 265, 269, 269, 52, 231, 102, 279, 252, 247, 22, 124, 8, 375, 2, 108, 318, 258, 153, 228, 46, 216, 39, 20, 201, 138, 76, 275, 13, 361, 376, 363, 378, 199, 298, 189, 347, 126, 85, 311, 234, 0, 27, 348, 57, 23, 368, 208, 249, 296, 194, 81, 178, 100, 193, 287, 272, 211, 259, 215, 43, 192, 335, 173, 155, 15, 238, 181, 204, 175, 377, 35, 195, 242, 183, 230, 240, 63, 317, 334, 48, 94, 229, 374, 299, 135, 40, 224, 245, 111, 267, 328, 294, 117, 367, 225, 179, 321, 75, 61, 213, 84, 214, 1, 60, 276, 262, 287, 272, 122, 21, 351, 17, 331, 182, 327, 131, 268, 59, 67, 3, 210, 220, 33, 186, 380, 253, 353, 250, 257, 284, 237, 9, 344, 62, 235, 263, 107, 187, 322, 292, 11, 197, 146, 357, 310, 339, 164, 82, 337, 302, 61, 213, 84, 5, 217, 346, 276, 370, 365, 288, 159, 379, 132, 212, 147, 198, 371, 137, 37, 174, 184, 251, 320, 218, 54, 144, 51, 345, 106, 239, 161, 289, 254, 87, 77, 191, 359, 103, 68, 168, 373, 86, 53, 309, 336, 123, 109, 112, 176, 157, 286, 354, 61, 249, 256, 160, 47, 355, 100, 188, 356, 6, 259, 78, 130, 7, 293, 25, 31, 66, 163, 358, 38, 143, 26, 333, 295, 28, 12, 93, 343, 300, 241, 203, 330, 80, 134, 349, 113, 323, 244, 156, 297, 101, 185, 73, 282, 202, 226, 248, 278, 277, 196, 303, 165, 213, 306, 194, 65, 355, 105, 305, 356, 6, 231, 64, 145, 352, 91, 364, 304, 70, 151, 332, 319, 24, 34, 152, 104, 99, 274, 227, 14, 382, 219, 90, 383, 236, 291, 55, 205, 114, 18, 92, 261, 290, 120, 285, 350, 42, 180, 110, 83, 326, 118, 127, 154, 141, 128, 214, 98, 49, 223, 169, 30, 52, 372, 21, 72, 88, 89, 338, 233, 313, 171, 264, 69, 140, 314, 45, 16, 79, 266, 329, 366, 121, 341, 136, 281, 360, 29, 232, 221, 71, 342, 316, 125, 95, 139, 97, 243, 340, 10, 116, 142, 96, 50, 44, 190, 190, 128, 308, 260, 355, 105, 305, 381, 288, 159, 41, 255, 200, 246, 74, 209, 4, 222, 369, 206, 207, 324, 315, 301, 115, 312, 19, 56, 362, 150, 325, 119, 149, 162, 271, 283, 129, 270, 58, 148, 32, 133, 307, 36, 158, 273, 280, 166, 167, 170, 177] + 6: [322, 97, 274, 308, 53, 343, 240, 99, 233, 205, 36, 359, 187, 57, 196, 289, 151, 95, 179, 245, 317, 202, 321, 327, 324, 154, 22, 206, 197, 4, 355, 329, 346, 3, 96, 70, 366, 357, 106, 360, 294, 348, 201, 263, 92, 141, 170, 218, 149, 25, 35, 164, 322, 212, 274, 5, 32, 267, 341, 240, 209, 120, 350, 181, 119, 56, 34, 129, 156, 176, 377, 278, 304, 37, 282, 219, 15, 280, 6, 242, 284, 243, 138, 148, 199, 147, 349, 272, 310, 244, 288, 251, 117, 51, 89, 339, 192, 1, 172, 2, 65, 33, 71, 40, 322, 142, 142, 376, 160, 258, 269, 46, 180, 31, 173, 163, 293, 112, 319, 178, 300, 165, 369, 336, 101, 12, 133, 211, 143, 58, 210, 207, 335, 82, 372, 248, 83, 124, 16, 268, 123, 39, 62, 328, 256, 21, 121, 374, 213, 93, 273, 61, 153, 52, 275, 342, 146, 97, 274, 308, 32, 174, 28, 11, 66, 67, 175, 216, 100, 189, 287, 239, 370, 113, 352, 236, 223, 279, 139, 114, 10, 286, 69, 43, 225, 217, 362, 122, 18, 358, 215, 299, 364, 130, 152, 290, 24, 195, 313, 238, 140, 227, 191, 20, 314, 204, 320, 247, 76, 305, 295, 308, 53, 27, 55, 229, 66, 371, 9, 318, 367, 246, 255, 291, 234, 301, 109, 41, 311, 111, 383, 144, 98, 220, 271, 237, 116, 330, 261, 81, 168, 224, 8, 125, 379, 74, 150, 85, 266, 47, 183, 38, 7, 91, 200, 228, 252, 186, 315, 373, 76, 97, 142, 376, 32, 249, 253, 281, 166, 177, 45, 13, 232, 283, 118, 155, 345, 270, 128, 230, 68, 30, 44, 64, 136, 296, 87, 104, 63, 23, 14, 126, 48, 50, 334, 309, 73, 312, 108, 226, 17, 351, 356, 302, 171, 241, 340, 353, 135, 167, 325, 257, 76, 97, 59, 376, 103, 174, 258, 99, 166, 177, 375, 307, 94, 84, 188, 79, 259, 354, 115, 182, 49, 158, 169, 77, 264, 365, 262, 137, 159, 60, 194, 145, 333, 198, 298, 54, 134, 285, 203, 347, 102, 303, 222, 78, 26, 19, 344, 72, 75, 214, 323, 90, 297, 305, 142, 308, 343, 306, 253, 281, 233, 326, 378, 260, 88, 368, 208, 184, 0, 277, 185, 380, 231, 190, 221, 107, 332, 157, 361, 110, 276, 86, 254, 316, 105, 193, 42, 381, 292, 235, 80, 337, 162, 338, 131, 250, 29, 127, 382, 265, 132, 363, 331, 161] + 7: [163, 274, 336, 308, 41, 292, 265, 271, 280, 99, 362, 188, 0, 281, 11, 82, 248, 349, 288, 325, 21, 377, 372, 214, 368, 16, 92, 129, 137, 380, 134, 306, 40, 309, 153, 52, 37, 320, 169, 295, 88, 173, 124, 63, 195, 150, 205, 138, 154, 145, 365, 364, 341, 7, 275, 108, 239, 187, 142, 342, 103, 25, 369, 212, 174, 94, 324, 100, 289, 168, 270, 296, 255, 213, 264, 98, 89, 311, 185, 190, 200, 102, 51, 301, 283, 68, 116, 351, 115, 151, 112, 316, 96, 43, 252, 315, 95, 45, 12, 54, 122, 160, 276, 60, 327, 182, 192, 263, 133, 292, 46, 340, 103, 71, 362, 97, 104, 181, 249, 307, 74, 285, 337, 374, 81, 170, 158, 132, 321, 26, 358, 107, 83, 335, 53, 155, 373, 291, 101, 345, 222, 268, 207, 1, 10, 367, 381, 128, 359, 66, 290, 127, 376, 30, 91, 50, 327, 7, 258, 263, 41, 292, 265, 340, 139, 13, 378, 354, 61, 256, 313, 217, 125, 119, 2, 235, 383, 224, 90, 167, 259, 109, 237, 5, 211, 44, 152, 232, 251, 65, 22, 120, 79, 106, 286, 72, 59, 80, 317, 266, 246, 332, 197, 199, 355, 179, 371, 42, 327, 7, 275, 192, 350, 260, 229, 382, 105, 25, 298, 329, 244, 73, 14, 267, 278, 17, 84, 250, 210, 62, 300, 241, 234, 204, 19, 178, 273, 305, 226, 310, 196, 247, 93, 193, 269, 216, 75, 69, 254, 39, 297, 159, 242, 230, 165, 144, 76, 240, 135, 338, 175, 272, 238, 64, 49, 176, 375, 343, 18, 201, 369, 328, 77, 302, 294, 164, 262, 31, 339, 284, 117, 219, 330, 206, 24, 245, 130, 180, 293, 58, 67, 353, 8, 227, 87, 203, 184, 166, 287, 218, 85, 111, 3, 334, 304, 346, 331, 282, 34, 236, 348, 357, 175, 182, 274, 361, 49, 28, 142, 343, 18, 253, 298, 15, 370, 131, 183, 215, 4, 55, 360, 356, 208, 177, 191, 243, 189, 314, 136, 32, 220, 6, 141, 323, 161, 146, 149, 366, 126, 35, 33, 36, 123, 231, 303, 228, 198, 147, 172, 347, 156, 279, 202, 257, 162, 157, 274, 64, 49, 375, 229, 382, 105, 201, 118, 48, 27, 114, 47, 113, 29, 23, 148, 319, 70, 221, 344, 333, 9, 140, 20, 186, 322, 86, 121, 56, 233, 261, 38, 225, 318, 209, 363, 194, 312, 379, 277, 78, 299, 57, 352, 143, 223, 326, 110, 171] + 8: [273, 290, 31, 135, 359, 239, 362, 241, 308, 152, 228, 281, 251, 255, 377, 131, 181, 75, 283, 240, 92, 178, 303, 113, 300, 318, 66, 114, 217, 254, 73, 191, 352, 382, 136, 16, 200, 235, 375, 369, 153, 326, 271, 270, 268, 123, 142, 82, 124, 361, 1, 132, 253, 360, 292, 135, 359, 264, 362, 10, 272, 353, 204, 74, 20, 349, 97, 280, 311, 150, 27, 38, 78, 65, 368, 194, 48, 120, 125, 284, 206, 383, 248, 247, 373, 47, 266, 313, 325, 89, 295, 185, 365, 24, 193, 77, 269, 343, 203, 274, 288, 127, 163, 96, 253, 360, 67, 31, 359, 264, 201, 336, 308, 152, 229, 182, 252, 209, 168, 86, 128, 83, 333, 119, 261, 165, 137, 309, 164, 141, 332, 61, 59, 118, 237, 249, 350, 299, 372, 314, 339, 159, 306, 347, 287, 111, 324, 208, 319, 363, 53, 331, 49, 0, 102, 100, 253, 360, 67, 238, 21, 108, 173, 62, 62, 154, 371, 122, 95, 380, 115, 156, 166, 22, 133, 179, 202, 40, 278, 50, 317, 211, 91, 72, 263, 117, 25, 257, 149, 259, 155, 140, 105, 205, 9, 93, 357, 351, 15, 129, 329, 305, 51, 344, 345, 296, 227, 170, 104, 360, 290, 338, 367, 201, 312, 321, 381, 371, 229, 276, 186, 316, 328, 195, 107, 69, 226, 258, 17, 121, 189, 167, 130, 192, 2, 4, 187, 174, 379, 71, 151, 37, 190, 68, 144, 145, 236, 46, 315, 76, 84, 158, 26, 70, 355, 301, 222, 176, 262, 234, 214, 279, 290, 106, 161, 224, 12, 243, 221, 282, 204, 81, 291, 18, 11, 374, 366, 172, 109, 231, 298, 275, 112, 198, 267, 233, 52, 8, 45, 98, 28, 293, 184, 41, 307, 143, 79, 85, 175, 322, 33, 277, 171, 244, 180, 286, 43, 342, 57, 210, 376, 207, 5, 250, 218, 21, 367, 13, 162, 10, 221, 353, 36, 182, 30, 44, 364, 148, 225, 356, 23, 147, 169, 216, 304, 196, 320, 197, 358, 60, 101, 212, 116, 232, 110, 310, 327, 199, 265, 378, 14, 29, 323, 220, 348, 340, 245, 354, 80, 94, 289, 242, 63, 213, 5, 3, 246, 302, 7, 239, 162, 321, 88, 64, 337, 281, 223, 157, 294, 39, 346, 188, 34, 6, 330, 215, 219, 183, 334, 177, 341, 42, 335, 230, 87, 90, 297, 32, 126, 56, 99, 35, 260, 54, 134, 146, 103, 370, 285, 160, 138, 139, 256, 55, 19, 58] + 9: [212, 349, 81, 94, 94, 165, 155, 184, 1, 313, 262, 287, 244, 103, 10, 239, 133, 158, 302, 169, 88, 6, 364, 163, 35, 267, 36, 171, 5, 75, 17, 175, 245, 373, 174, 115, 42, 12, 270, 21, 178, 59, 193, 243, 266, 119, 278, 40, 362, 55, 152, 234, 92, 255, 331, 372, 172, 295, 327, 47, 272, 201, 242, 329, 64, 103, 67, 102, 312, 161, 377, 292, 306, 180, 314, 145, 142, 336, 66, 189, 104, 343, 339, 50, 100, 205, 322, 2, 320, 251, 224, 211, 69, 188, 34, 58, 118, 328, 204, 200, 250, 61, 24, 365, 92, 255, 220, 226, 19, 350, 327, 381, 15, 313, 262, 49, 95, 147, 14, 32, 131, 82, 135, 137, 297, 247, 192, 129, 269, 206, 150, 72, 274, 324, 18, 8, 382, 197, 213, 358, 301, 134, 76, 229, 199, 170, 160, 368, 279, 63, 149, 294, 164, 53, 77, 173, 92, 255, 96, 280, 252, 357, 355, 185, 29, 90, 90, 380, 271, 209, 282, 56, 9, 317, 153, 33, 264, 321, 107, 116, 323, 123, 124, 332, 359, 80, 276, 65, 52, 275, 79, 203, 286, 352, 333, 221, 347, 374, 383, 225, 240, 361, 85, 378, 156, 27, 236, 57, 92, 349, 349, 260, 263, 74, 222, 179, 83, 146, 217, 281, 244, 341, 136, 22, 258, 144, 187, 3, 334, 231, 183, 20, 195, 207, 181, 238, 202, 39, 208, 354, 117, 45, 190, 114, 371, 305, 298, 89, 346, 337, 191, 43, 325, 223, 44, 113, 91, 182, 126, 340, 78, 166, 143, 226, 299, 353, 355, 47, 37, 376, 127, 159, 214, 147, 132, 177, 98, 360, 356, 289, 228, 0, 87, 41, 105, 4, 230, 141, 288, 13, 235, 130, 379, 233, 300, 291, 84, 71, 125, 38, 227, 167, 290, 68, 345, 73, 296, 257, 28, 265, 168, 338, 304, 304, 315, 31, 263, 295, 355, 185, 344, 316, 370, 49, 95, 121, 23, 342, 151, 285, 154, 120, 277, 128, 97, 99, 186, 16, 70, 218, 237, 319, 248, 109, 303, 46, 62, 112, 253, 330, 310, 232, 48, 86, 326, 108, 176, 249, 284, 273, 7, 122, 283, 348, 304, 304, 196, 31, 215, 219, 327, 351, 198, 316, 138, 210, 64, 51, 10, 309, 367, 30, 363, 268, 318, 54, 311, 259, 256, 139, 11, 111, 106, 308, 246, 307, 60, 148, 261, 254, 110, 375, 216, 162, 335, 241, 293, 140, 366, 101, 93, 194, 157, 25, 369, 26] + 10: [294, 180, 151, 319, 141, 289, 365, 69, 31, 225, 8, 1, 153, 138, 154, 201, 312, 234, 356, 208, 190, 359, 382, 198, 214, 135, 311, 27, 315, 26, 379, 71, 125, 39, 275, 163, 282, 226, 96, 139, 86, 119, 322, 53, 94, 344, 361, 277, 188, 381, 324, 283, 192, 180, 93, 319, 5, 262, 216, 367, 31, 57, 310, 54, 263, 241, 304, 58, 353, 11, 348, 377, 212, 271, 149, 229, 87, 321, 219, 106, 357, 346, 51, 267, 162, 174, 45, 13, 113, 209, 326, 235, 187, 205, 165, 203, 334, 333, 191, 156, 228, 253, 291, 68, 255, 246, 181, 319, 5, 24, 18, 286, 164, 73, 144, 179, 171, 177, 237, 213, 115, 136, 368, 148, 77, 280, 285, 90, 233, 21, 232, 335, 339, 327, 41, 109, 340, 332, 50, 123, 175, 257, 375, 80, 355, 224, 158, 133, 104, 281, 152, 260, 270, 166, 176, 358, 194, 300, 265, 221, 5, 320, 216, 126, 331, 92, 301, 195, 351, 210, 99, 91, 293, 272, 345, 342, 189, 223, 266, 0, 33, 46, 288, 60, 9, 378, 97, 371, 247, 193, 140, 343, 244, 169, 131, 62, 206, 303, 6, 16, 88, 157, 84, 63, 28, 238, 295, 297, 194, 300, 137, 108, 5, 24, 365, 167, 309, 2, 310, 134, 114, 19, 211, 248, 7, 259, 374, 186, 231, 218, 61, 145, 72, 3, 56, 314, 329, 328, 74, 278, 42, 254, 239, 362, 220, 307, 100, 302, 373, 292, 200, 245, 120, 44, 30, 202, 89, 160, 290, 336, 264, 300, 268, 319, 320, 121, 365, 286, 146, 101, 301, 170, 40, 66, 47, 269, 243, 130, 284, 49, 38, 376, 132, 199, 116, 341, 81, 85, 256, 338, 230, 15, 36, 184, 323, 261, 370, 364, 287, 111, 127, 147, 35, 34, 118, 207, 352, 14, 242, 236, 82, 29, 264, 300, 265, 319, 320, 121, 325, 279, 146, 79, 22, 178, 117, 330, 124, 308, 76, 252, 274, 316, 98, 306, 32, 128, 103, 105, 75, 122, 112, 59, 25, 383, 318, 64, 317, 183, 298, 299, 159, 173, 305, 217, 4, 251, 172, 83, 150, 10, 52, 70, 23, 366, 55, 55, 151, 48, 5, 320, 18, 337, 164, 102, 215, 161, 78, 182, 196, 380, 67, 240, 363, 204, 360, 249, 155, 250, 129, 107, 296, 142, 197, 20, 37, 258, 12, 354, 313, 95, 185, 65, 350, 222, 372, 17, 168, 276, 273, 347, 143, 227, 43, 110, 349, 369] + 11: [156, 363, 314, 215, 65, 150, 235, 233, 44, 126, 102, 345, 152, 284, 286, 318, 154, 297, 47, 280, 2, 373, 172, 151, 89, 298, 186, 242, 294, 245, 359, 91, 164, 57, 129, 188, 309, 274, 103, 106, 4, 140, 304, 328, 216, 139, 51, 206, 334, 50, 308, 269, 252, 88, 376, 215, 85, 85, 235, 121, 116, 126, 162, 24, 192, 330, 244, 381, 169, 23, 319, 124, 127, 75, 347, 10, 224, 289, 41, 81, 336, 279, 364, 302, 187, 64, 43, 7, 74, 231, 5, 323, 201, 322, 305, 379, 344, 246, 253, 76, 212, 137, 307, 241, 98, 130, 90, 3, 40, 150, 221, 11, 116, 248, 29, 0, 351, 141, 204, 282, 136, 320, 161, 45, 249, 226, 315, 86, 341, 255, 101, 327, 63, 30, 338, 179, 125, 264, 113, 62, 143, 184, 360, 222, 240, 132, 380, 229, 180, 181, 202, 123, 220, 321, 134, 243, 98, 130, 90, 83, 163, 110, 99, 300, 325, 357, 267, 144, 275, 165, 131, 228, 205, 78, 66, 371, 350, 92, 195, 340, 316, 14, 208, 301, 219, 168, 354, 349, 291, 293, 174, 182, 46, 194, 375, 210, 21, 214, 278, 277, 175, 369, 189, 276, 281, 72, 191, 335, 77, 130, 87, 296, 296, 326, 221, 36, 44, 100, 15, 218, 54, 178, 95, 177, 17, 303, 94, 367, 69, 117, 26, 251, 198, 55, 31, 97, 295, 265, 312, 311, 67, 120, 33, 225, 8, 19, 200, 356, 185, 104, 272, 1, 368, 16, 372, 122, 232, 257, 61, 105, 377, 260, 314, 83, 163, 110, 171, 11, 36, 100, 15, 332, 217, 343, 70, 337, 148, 196, 283, 213, 370, 159, 166, 374, 203, 96, 254, 270, 211, 59, 353, 193, 234, 383, 142, 263, 230, 271, 266, 112, 310, 167, 68, 183, 60, 53, 28, 239, 223, 25, 109, 12, 377, 299, 87, 296, 163, 110, 121, 145, 248, 357, 199, 236, 331, 128, 39, 20, 13, 250, 346, 107, 348, 27, 93, 160, 133, 365, 58, 209, 362, 366, 34, 135, 324, 79, 258, 355, 237, 382, 108, 306, 173, 287, 268, 114, 238, 317, 333, 155, 329, 273, 261, 9, 339, 339, 227, 314, 149, 119, 38, 190, 157, 285, 285, 22, 247, 52, 73, 146, 207, 313, 6, 170, 262, 80, 176, 56, 49, 197, 259, 35, 147, 292, 378, 158, 118, 115, 48, 18, 138, 358, 37, 84, 288, 290, 71, 342, 361, 352, 111, 82, 153, 256, 32, 42] + 12: [156, 120, 130, 158, 327, 109, 22, 177, 150, 333, 97, 369, 354, 75, 216, 317, 266, 277, 27, 300, 322, 294, 337, 23, 155, 142, 9, 191, 61, 62, 161, 169, 11, 115, 334, 183, 246, 312, 285, 105, 375, 113, 81, 247, 307, 118, 221, 342, 359, 353, 162, 110, 255, 49, 381, 226, 344, 267, 292, 177, 243, 333, 192, 181, 170, 295, 332, 348, 147, 60, 33, 264, 362, 197, 238, 74, 15, 253, 200, 114, 165, 138, 280, 319, 76, 40, 305, 5, 179, 128, 106, 70, 316, 18, 299, 258, 7, 184, 328, 311, 64, 141, 366, 42, 255, 149, 130, 78, 167, 303, 82, 296, 150, 203, 233, 361, 123, 134, 137, 330, 87, 108, 160, 171, 104, 135, 326, 91, 286, 370, 52, 185, 119, 262, 136, 58, 335, 270, 379, 336, 244, 263, 85, 310, 93, 55, 54, 367, 111, 145, 360, 248, 324, 239, 222, 272, 255, 149, 130, 158, 327, 168, 124, 289, 243, 287, 341, 98, 260, 195, 205, 276, 25, 89, 209, 102, 271, 241, 291, 157, 304, 193, 77, 41, 45, 67, 139, 37, 235, 351, 140, 12, 329, 279, 274, 44, 282, 96, 212, 19, 210, 338, 163, 206, 166, 240, 53, 73, 255, 35, 120, 28, 17, 127, 349, 69, 172, 46, 90, 151, 176, 6, 382, 152, 368, 16, 2, 371, 57, 63, 380, 14, 144, 227, 254, 315, 189, 313, 249, 325, 122, 154, 65, 86, 186, 148, 383, 30, 47, 343, 72, 236, 143, 94, 228, 275, 21, 56, 250, 302, 255, 48, 120, 226, 190, 127, 223, 69, 71, 103, 290, 306, 340, 3, 346, 268, 323, 10, 153, 101, 273, 129, 261, 301, 309, 201, 188, 0, 83, 298, 232, 321, 214, 159, 257, 112, 284, 217, 24, 252, 8, 131, 211, 320, 182, 269, 314, 213, 95, 66, 43, 225, 204, 204, 78, 158, 190, 303, 82, 378, 172, 46, 50, 51, 259, 278, 281, 194, 218, 364, 288, 116, 107, 164, 31, 59, 283, 357, 350, 234, 29, 224, 356, 187, 355, 36, 13, 293, 297, 207, 174, 146, 345, 347, 117, 125, 196, 132, 242, 100, 92, 331, 199, 372, 204, 363, 245, 219, 327, 175, 223, 378, 126, 203, 341, 198, 208, 339, 68, 215, 229, 133, 39, 265, 180, 308, 80, 318, 358, 376, 1, 231, 84, 256, 4, 173, 38, 121, 20, 32, 365, 374, 352, 251, 34, 178, 237, 230, 79, 26, 202, 88, 377, 99, 373, 220] + 13: [80, 170, 232, 114, 195, 149, 260, 280, 189, 238, 318, 102, 370, 46, 279, 58, 112, 273, 309, 363, 2, 378, 179, 62, 228, 107, 374, 243, 241, 37, 163, 357, 84, 305, 113, 250, 70, 73, 5, 103, 205, 346, 343, 128, 278, 293, 132, 354, 218, 301, 33, 78, 80, 170, 232, 96, 14, 23, 312, 172, 95, 110, 274, 366, 88, 159, 118, 227, 31, 289, 91, 292, 373, 65, 375, 43, 239, 74, 359, 29, 222, 303, 337, 286, 220, 127, 348, 53, 154, 213, 313, 249, 4, 167, 117, 212, 264, 20, 332, 200, 82, 247, 155, 316, 9, 170, 232, 114, 14, 23, 265, 135, 276, 12, 30, 186, 199, 109, 35, 49, 75, 202, 40, 334, 6, 42, 219, 321, 26, 225, 333, 183, 76, 125, 48, 153, 345, 268, 245, 308, 115, 217, 287, 119, 138, 180, 284, 352, 99, 224, 341, 302, 351, 165, 27, 7, 44, 77, 108, 81, 0, 328, 312, 51, 55, 209, 254, 17, 162, 298, 295, 111, 188, 275, 229, 67, 178, 22, 211, 141, 344, 283, 93, 139, 380, 34, 288, 106, 136, 296, 381, 94, 72, 13, 130, 104, 285, 383, 1, 215, 181, 317, 236, 368, 277, 314, 315, 192, 44, 170, 170, 18, 86, 328, 39, 280, 189, 238, 318, 134, 11, 116, 307, 198, 16, 356, 261, 158, 300, 98, 355, 382, 144, 121, 166, 146, 15, 105, 36, 68, 242, 92, 325, 187, 270, 38, 150, 353, 83, 372, 101, 28, 152, 251, 269, 156, 120, 193, 350, 32, 362, 329, 57, 18, 195, 149, 260, 21, 201, 12, 330, 197, 338, 306, 131, 64, 364, 376, 294, 281, 122, 210, 79, 339, 291, 361, 253, 129, 164, 331, 161, 248, 319, 59, 263, 311, 358, 19, 124, 214, 185, 85, 252, 63, 258, 371, 272, 148, 367, 324, 140, 230, 362, 137, 108, 3, 195, 87, 24, 41, 231, 30, 274, 310, 335, 171, 176, 45, 69, 297, 143, 266, 299, 147, 10, 240, 145, 196, 151, 327, 221, 223, 89, 379, 174, 259, 50, 133, 360, 234, 235, 190, 182, 157, 336, 191, 90, 347, 290, 47, 160, 206, 126, 204, 194, 329, 108, 18, 87, 87, 169, 282, 342, 100, 340, 271, 142, 304, 323, 184, 369, 246, 54, 320, 208, 207, 168, 237, 25, 262, 177, 256, 52, 8, 267, 326, 56, 257, 244, 71, 123, 377, 365, 61, 233, 226, 60, 203, 173, 97, 349, 255, 322, 66, 216, 175] + 14: [147, 143, 15, 164, 60, 30, 268, 186, 126, 348, 235, 99, 32, 97, 374, 21, 294, 13, 297, 343, 332, 193, 363, 187, 325, 373, 5, 347, 341, 189, 335, 313, 244, 251, 28, 6, 278, 127, 136, 19, 87, 18, 55, 92, 236, 307, 134, 243, 218, 67, 318, 165, 70, 274, 76, 76, 17, 175, 2, 75, 122, 378, 142, 228, 124, 64, 154, 250, 41, 197, 379, 123, 288, 145, 37, 382, 286, 178, 101, 380, 359, 253, 115, 12, 356, 42, 285, 329, 103, 195, 334, 190, 312, 132, 367, 84, 233, 219, 170, 303, 210, 119, 83, 260, 70, 143, 4, 287, 255, 199, 209, 75, 327, 353, 275, 85, 100, 3, 79, 271, 216, 314, 161, 328, 357, 277, 108, 40, 102, 227, 82, 129, 298, 336, 223, 191, 183, 248, 10, 214, 309, 36, 20, 14, 179, 194, 293, 24, 181, 283, 72, 54, 352, 130, 73, 331, 56, 211, 372, 204, 255, 110, 259, 34, 355, 337, 71, 23, 117, 177, 269, 279, 166, 125, 25, 163, 291, 159, 301, 169, 128, 234, 48, 156, 176, 316, 202, 282, 217, 43, 106, 273, 171, 365, 9, 290, 258, 7, 66, 139, 11, 320, 192, 131, 29, 196, 249, 50, 160, 229, 207, 345, 137, 112, 2, 239, 150, 378, 71, 275, 231, 213, 162, 354, 270, 220, 206, 323, 68, 201, 22, 346, 203, 245, 241, 237, 284, 0, 89, 242, 224, 376, 310, 173, 358, 267, 238, 16, 263, 45, 57, 185, 247, 65, 188, 340, 292, 98, 315, 174, 160, 229, 15, 77, 255, 110, 259, 122, 126, 146, 155, 151, 52, 342, 360, 182, 157, 33, 135, 246, 319, 369, 104, 339, 172, 109, 90, 377, 306, 225, 300, 364, 63, 93, 114, 321, 375, 74, 78, 58, 299, 80, 62, 35, 107, 144, 113, 280, 53, 351, 121, 205, 160, 211, 4, 204, 8, 30, 49, 34, 355, 118, 155, 141, 198, 371, 317, 44, 265, 158, 59, 69, 184, 370, 167, 333, 256, 296, 116, 105, 330, 200, 168, 148, 1, 240, 94, 262, 257, 338, 361, 230, 322, 31, 366, 120, 140, 252, 51, 368, 232, 138, 212, 289, 229, 229, 261, 345, 345, 383, 209, 344, 327, 308, 308, 254, 95, 39, 61, 111, 26, 208, 149, 215, 324, 326, 266, 133, 349, 226, 222, 221, 276, 86, 152, 264, 350, 96, 311, 302, 153, 46, 304, 305, 281, 47, 362, 180, 81, 381, 91, 27, 38, 272, 295, 88] + 15: [68, 88, 295, 11, 371, 251, 155, 231, 128, 104, 252, 51, 360, 83, 353, 1, 241, 115, 162, 319, 170, 238, 105, 82, 309, 268, 81, 254, 294, 349, 381, 137, 244, 71, 283, 265, 54, 278, 369, 188, 201, 310, 7, 195, 154, 307, 122, 219, 348, 264, 366, 328, 298, 88, 259, 11, 362, 160, 185, 200, 217, 236, 74, 20, 151, 40, 344, 256, 111, 109, 225, 335, 182, 49, 166, 59, 114, 282, 69, 165, 382, 306, 72, 354, 224, 123, 375, 213, 167, 239, 161, 153, 126, 187, 124, 181, 44, 120, 125, 156, 186, 6, 131, 368, 0, 350, 367, 130, 359, 160, 38, 258, 266, 175, 284, 60, 232, 139, 173, 342, 177, 18, 36, 357, 5, 42, 102, 86, 87, 62, 50, 43, 339, 107, 321, 234, 314, 211, 373, 84, 116, 3, 197, 129, 14, 312, 292, 66, 178, 346, 288, 336, 19, 46, 127, 281, 0, 76, 227, 11, 362, 160, 155, 355, 128, 104, 333, 237, 329, 48, 296, 135, 242, 164, 379, 351, 209, 305, 212, 2, 10, 96, 363, 253, 30, 313, 97, 289, 55, 169, 337, 316, 240, 79, 91, 184, 140, 293, 112, 222, 179, 70, 286, 345, 249, 75, 198, 93, 0, 4, 88, 11, 57, 300, 324, 200, 217, 134, 134, 9, 157, 56, 113, 311, 320, 196, 285, 100, 133, 28, 145, 263, 269, 150, 233, 341, 141, 191, 61, 267, 204, 383, 95, 334, 273, 147, 274, 374, 149, 318, 136, 330, 183, 110, 108, 214, 27, 152, 192, 270, 290, 340, 202, 130, 359, 251, 324, 326, 94, 175, 215, 90, 221, 23, 193, 168, 143, 230, 106, 338, 323, 194, 101, 142, 34, 229, 250, 13, 377, 118, 347, 365, 216, 64, 26, 257, 218, 364, 73, 89, 370, 280, 327, 228, 103, 304, 21, 302, 315, 25, 325, 85, 15, 15, 130, 130, 359, 199, 355, 332, 159, 148, 203, 146, 80, 376, 277, 163, 255, 24, 352, 47, 78, 246, 279, 248, 343, 331, 8, 206, 271, 226, 53, 308, 45, 272, 172, 358, 52, 65, 303, 158, 29, 176, 58, 98, 299, 17, 243, 220, 132, 144, 322, 356, 15, 261, 227, 291, 57, 39, 189, 258, 94, 148, 119, 121, 317, 16, 35, 174, 138, 235, 31, 207, 372, 247, 275, 117, 77, 22, 205, 12, 301, 67, 180, 99, 208, 37, 92, 63, 190, 210, 378, 32, 262, 380, 171, 361, 41, 223, 287, 297, 33, 245, 260, 276] + 16: [11, 286, 369, 120, 300, 92, 338, 147, 303, 203, 5, 250, 383, 295, 246, 108, 181, 328, 335, 99, 103, 159, 380, 195, 229, 334, 160, 124, 225, 29, 25, 242, 337, 105, 48, 266, 233, 157, 261, 110, 216, 332, 251, 382, 79, 23, 341, 222, 53, 191, 309, 249, 162, 68, 152, 331, 262, 7, 165, 90, 118, 280, 35, 274, 298, 141, 212, 16, 285, 177, 307, 292, 119, 0, 182, 34, 379, 202, 276, 188, 317, 354, 57, 313, 343, 378, 91, 30, 172, 166, 20, 17, 308, 149, 245, 28, 63, 40, 350, 278, 133, 345, 366, 257, 100, 15, 152, 154, 234, 92, 165, 129, 118, 280, 183, 224, 121, 46, 367, 180, 365, 306, 248, 21, 80, 282, 31, 70, 220, 19, 9, 96, 221, 228, 200, 178, 223, 93, 109, 193, 186, 128, 324, 117, 116, 359, 260, 32, 355, 74, 299, 204, 376, 320, 1, 98, 100, 15, 67, 114, 234, 312, 273, 64, 44, 75, 348, 24, 209, 364, 347, 85, 27, 237, 115, 38, 281, 153, 311, 167, 356, 293, 297, 4, 290, 336, 158, 214, 253, 272, 142, 201, 62, 14, 47, 131, 217, 50, 138, 352, 12, 130, 136, 291, 361, 13, 94, 344, 140, 15, 286, 120, 120, 262, 277, 269, 143, 342, 374, 302, 283, 213, 270, 346, 314, 84, 51, 316, 58, 66, 41, 8, 232, 372, 238, 22, 252, 358, 54, 102, 323, 83, 244, 363, 81, 127, 218, 26, 170, 241, 243, 169, 6, 112, 197, 72, 296, 215, 247, 156, 140, 15, 369, 185, 275, 7, 86, 64, 303, 75, 271, 255, 287, 236, 87, 226, 310, 321, 97, 37, 175, 39, 184, 76, 36, 42, 137, 322, 264, 340, 132, 95, 206, 55, 190, 360, 315, 373, 268, 113, 371, 150, 61, 288, 351, 329, 263, 179, 325, 239, 368, 353, 134, 134, 286, 330, 275, 43, 147, 49, 44, 259, 271, 171, 189, 235, 146, 89, 65, 71, 59, 82, 73, 155, 305, 219, 45, 122, 174, 370, 104, 151, 208, 211, 333, 304, 196, 240, 163, 88, 107, 168, 256, 77, 210, 33, 164, 135, 173, 326, 362, 69, 111, 3, 134, 134, 286, 125, 275, 43, 144, 269, 289, 318, 205, 78, 265, 187, 123, 199, 294, 52, 2, 377, 139, 56, 231, 227, 357, 301, 339, 106, 198, 375, 18, 60, 267, 319, 207, 194, 381, 284, 145, 258, 279, 176, 230, 101, 349, 254, 192, 327, 10, 161, 126, 148] + 17: [374, 119, 188, 236, 303, 58, 87, 15, 383, 193, 305, 343, 351, 62, 8, 56, 207, 24, 231, 38, 98, 146, 282, 259, 82, 61, 42, 269, 196, 377, 338, 181, 249, 195, 299, 287, 274, 326, 364, 63, 53, 316, 141, 201, 291, 11, 247, 111, 123, 34, 69, 72, 261, 119, 250, 236, 346, 79, 200, 254, 117, 225, 305, 323, 1, 189, 276, 157, 96, 199, 145, 228, 84, 298, 171, 245, 13, 47, 59, 295, 49, 115, 37, 21, 179, 240, 90, 345, 55, 158, 255, 233, 65, 352, 176, 109, 229, 129, 289, 268, 174, 54, 121, 152, 261, 119, 140, 236, 226, 337, 375, 375, 383, 197, 330, 362, 2, 297, 234, 107, 317, 97, 381, 211, 139, 99, 263, 18, 264, 36, 260, 7, 204, 205, 101, 213, 360, 361, 329, 210, 48, 308, 50, 318, 132, 241, 40, 277, 4, 160, 353, 321, 301, 127, 312, 320, 28, 119, 250, 284, 190, 337, 200, 219, 136, 275, 330, 362, 125, 6, 75, 52, 194, 335, 180, 92, 367, 327, 9, 142, 191, 20, 331, 154, 118, 382, 253, 342, 238, 262, 239, 112, 128, 278, 270, 57, 163, 182, 23, 30, 257, 306, 348, 281, 209, 86, 235, 237, 212, 220, 208, 188, 79, 79, 375, 91, 311, 5, 286, 185, 324, 328, 172, 290, 39, 46, 313, 103, 143, 135, 214, 168, 248, 166, 43, 41, 81, 3, 309, 44, 379, 302, 183, 161, 151, 369, 350, 144, 164, 227, 149, 66, 349, 71, 77, 35, 252, 122, 251, 283, 138, 220, 307, 284, 319, 126, 108, 15, 155, 26, 232, 114, 130, 29, 370, 104, 73, 22, 243, 380, 256, 33, 292, 376, 332, 116, 147, 215, 198, 365, 89, 95, 300, 244, 102, 16, 64, 25, 192, 344, 355, 88, 280, 223, 368, 285, 334, 322, 293, 169, 159, 170, 131, 131, 80, 284, 303, 366, 108, 357, 100, 26, 294, 279, 246, 133, 167, 222, 134, 206, 148, 273, 371, 74, 165, 70, 304, 19, 173, 363, 216, 186, 341, 265, 110, 310, 120, 85, 267, 124, 67, 150, 83, 153, 378, 224, 178, 242, 17, 68, 156, 14, 202, 203, 131, 131, 105, 284, 137, 337, 12, 78, 100, 359, 294, 266, 113, 347, 175, 339, 296, 218, 177, 373, 10, 272, 314, 162, 340, 60, 45, 336, 187, 221, 271, 27, 94, 93, 258, 217, 106, 372, 76, 31, 32, 315, 230, 356, 184, 288, 0, 51, 333, 358, 325, 354] + 18: [208, 10, 271, 210, 251, 315, 300, 57, 138, 380, 174, 68, 306, 284, 378, 67, 103, 46, 371, 342, 83, 87, 88, 241, 201, 313, 152, 140, 264, 362, 265, 363, 243, 325, 348, 168, 332, 311, 224, 76, 218, 372, 238, 281, 5, 305, 336, 323, 226, 97, 242, 59, 208, 272, 271, 16, 251, 320, 65, 114, 106, 185, 236, 25, 360, 297, 285, 190, 165, 181, 41, 141, 359, 79, 334, 22, 278, 50, 61, 349, 54, 319, 335, 187, 351, 179, 303, 175, 216, 66, 17, 229, 75, 42, 370, 294, 100, 11, 21, 196, 353, 120, 268, 78, 6, 10, 271, 333, 251, 207, 245, 119, 138, 185, 189, 167, 376, 151, 330, 101, 204, 89, 171, 339, 104, 149, 109, 296, 122, 361, 8, 195, 150, 9, 63, 155, 287, 92, 275, 82, 170, 219, 130, 123, 298, 172, 85, 29, 377, 290, 62, 194, 55, 374, 344, 340, 118, 47, 271, 53, 69, 217, 347, 57, 71, 260, 239, 321, 237, 291, 246, 3, 240, 19, 366, 248, 177, 343, 129, 135, 288, 231, 289, 176, 205, 276, 132, 255, 322, 112, 108, 60, 338, 247, 163, 133, 364, 354, 90, 184, 14, 160, 173, 221, 91, 52, 183, 382, 356, 47, 271, 45, 69, 217, 113, 36, 228, 260, 142, 146, 34, 253, 99, 33, 188, 164, 365, 73, 307, 233, 263, 137, 159, 328, 235, 15, 125, 244, 12, 13, 270, 254, 127, 383, 220, 51, 126, 369, 312, 86, 44, 352, 249, 117, 72, 134, 49, 327, 314, 309, 30, 30, 43, 271, 320, 257, 245, 215, 228, 110, 192, 26, 31, 357, 35, 326, 124, 269, 337, 191, 283, 262, 131, 162, 350, 302, 102, 203, 178, 93, 182, 329, 153, 214, 213, 234, 64, 27, 209, 375, 2, 107, 115, 273, 24, 212, 206, 295, 128, 74, 222, 1, 30, 30, 43, 193, 211, 257, 28, 304, 71, 368, 116, 301, 139, 282, 266, 77, 70, 318, 20, 202, 331, 261, 225, 0, 7, 58, 157, 308, 279, 81, 145, 292, 94, 166, 48, 358, 158, 4, 56, 317, 95, 84, 37, 299, 230, 18, 199, 198, 379, 156, 293, 147, 30, 30, 271, 271, 286, 154, 300, 65, 106, 169, 136, 355, 373, 144, 341, 105, 98, 274, 346, 23, 121, 223, 316, 367, 232, 250, 39, 111, 38, 96, 40, 267, 148, 324, 256, 143, 200, 259, 197, 381, 180, 258, 161, 280, 345, 277, 252, 310, 186, 80, 227, 32] + 19: [187, 24, 48, 302, 61, 91, 161, 313, 312, 300, 64, 113, 33, 303, 117, 44, 292, 53, 285, 222, 359, 76, 29, 283, 144, 4, 244, 325, 177, 45, 148, 47, 281, 119, 130, 168, 132, 234, 314, 315, 106, 11, 201, 51, 205, 308, 43, 220, 274, 317, 263, 268, 187, 24, 256, 7, 364, 255, 288, 180, 219, 8, 307, 115, 164, 77, 159, 79, 46, 54, 71, 278, 337, 96, 251, 204, 2, 215, 203, 18, 333, 202, 160, 3, 191, 383, 311, 32, 232, 174, 63, 37, 150, 269, 214, 252, 163, 236, 282, 78, 145, 93, 81, 98, 187, 24, 256, 7, 377, 120, 288, 180, 207, 312, 64, 323, 193, 296, 153, 58, 25, 335, 1, 166, 127, 172, 156, 239, 157, 110, 306, 336, 213, 10, 331, 129, 376, 321, 176, 320, 181, 332, 324, 361, 118, 124, 290, 100, 299, 379, 69, 327, 276, 350, 319, 355, 141, 366, 74, 7, 57, 70, 338, 279, 42, 101, 373, 27, 218, 97, 230, 217, 147, 135, 36, 134, 6, 83, 195, 138, 40, 31, 95, 287, 108, 380, 372, 62, 140, 329, 82, 139, 370, 247, 149, 73, 348, 123, 178, 367, 0, 5, 151, 72, 196, 229, 198, 245, 243, 24, 339, 90, 377, 288, 338, 183, 318, 197, 307, 115, 38, 271, 301, 189, 171, 246, 116, 310, 182, 167, 49, 170, 297, 259, 342, 194, 85, 80, 60, 92, 88, 200, 341, 343, 99, 105, 28, 371, 381, 210, 358, 136, 344, 102, 169, 382, 316, 12, 34, 50, 190, 211, 206, 265, 90, 120, 330, 313, 207, 101, 66, 184, 349, 227, 275, 264, 158, 122, 13, 128, 228, 121, 16, 272, 67, 345, 143, 84, 19, 23, 356, 65, 225, 362, 137, 209, 14, 208, 286, 294, 357, 162, 185, 21, 155, 261, 17, 199, 250, 305, 253, 254, 190, 366, 111, 90, 241, 188, 330, 126, 42, 240, 114, 328, 20, 334, 354, 9, 41, 15, 233, 352, 173, 68, 353, 221, 260, 249, 374, 125, 284, 262, 293, 26, 309, 242, 298, 87, 258, 235, 94, 59, 231, 289, 223, 304, 266, 248, 360, 104, 175, 109, 107, 133, 190, 366, 179, 265, 57, 52, 291, 322, 192, 240, 216, 363, 154, 89, 152, 35, 365, 238, 30, 295, 165, 270, 368, 257, 347, 277, 273, 346, 237, 131, 75, 280, 146, 326, 212, 22, 186, 340, 351, 103, 112, 267, 39, 224, 86, 56, 226, 55, 142, 378, 369, 375] + 20: [116, 174, 161, 144, 193, 110, 2, 226, 379, 224, 105, 93, 228, 273, 362, 259, 132, 347, 90, 239, 218, 343, 265, 314, 208, 18, 166, 246, 248, 12, 236, 257, 191, 253, 217, 210, 199, 296, 356, 323, 354, 157, 134, 183, 23, 222, 255, 288, 280, 303, 331, 350, 116, 302, 272, 82, 98, 153, 376, 268, 335, 130, 270, 318, 209, 128, 382, 227, 292, 172, 315, 13, 194, 26, 25, 30, 233, 34, 131, 340, 377, 100, 332, 3, 312, 212, 316, 285, 289, 319, 7, 329, 225, 207, 234, 245, 103, 275, 298, 17, 117, 27, 51, 52, 116, 302, 14, 161, 66, 287, 369, 145, 291, 282, 32, 371, 324, 221, 9, 135, 258, 92, 322, 6, 235, 35, 175, 367, 300, 22, 238, 345, 15, 338, 304, 231, 360, 230, 159, 366, 115, 283, 102, 163, 47, 83, 293, 70, 149, 306, 38, 109, 241, 107, 137, 167, 116, 336, 336, 317, 81, 99, 369, 45, 4, 353, 363, 325, 111, 141, 187, 160, 216, 121, 182, 177, 240, 375, 36, 58, 247, 114, 196, 346, 20, 310, 73, 201, 326, 337, 56, 80, 156, 88, 321, 341, 67, 276, 373, 147, 192, 344, 44, 143, 242, 168, 179, 181, 334, 37, 161, 144, 98, 153, 87, 268, 335, 46, 62, 162, 10, 95, 311, 140, 49, 184, 357, 57, 364, 41, 0, 146, 178, 372, 243, 169, 294, 252, 127, 170, 24, 84, 308, 151, 59, 21, 40, 271, 136, 19, 64, 68, 220, 152, 237, 197, 260, 214, 262, 263, 334, 278, 161, 55, 98, 123, 87, 145, 307, 46, 185, 330, 164, 290, 264, 180, 320, 71, 77, 190, 284, 361, 118, 359, 150, 119, 215, 86, 370, 33, 78, 365, 89, 223, 154, 139, 142, 120, 113, 355, 211, 63, 133, 108, 251, 155, 244, 269, 277, 266, 281, 299, 334, 278, 161, 82, 173, 153, 43, 112, 339, 195, 204, 148, 91, 229, 96, 50, 101, 65, 126, 74, 267, 328, 219, 213, 309, 256, 274, 97, 297, 232, 106, 39, 374, 351, 254, 11, 352, 124, 28, 205, 72, 16, 48, 383, 186, 31, 203, 171, 249, 189, 200, 206, 8, 122, 161, 66, 193, 99, 43, 45, 4, 282, 188, 138, 158, 125, 381, 286, 342, 358, 104, 176, 348, 378, 53, 368, 279, 42, 60, 165, 29, 5, 380, 202, 327, 75, 94, 349, 79, 250, 85, 333, 305, 313, 261, 295, 129, 301, 1, 69, 198, 54, 61, 76] + 21: [152, 153, 4, 173, 205, 132, 53, 376, 71, 113, 361, 232, 178, 177, 172, 351, 8, 69, 52, 72, 245, 131, 168, 1, 64, 309, 312, 28, 317, 323, 204, 171, 187, 170, 340, 300, 379, 110, 195, 164, 70, 330, 217, 314, 296, 142, 278, 188, 343, 11, 42, 264, 152, 355, 256, 173, 281, 132, 80, 124, 257, 185, 348, 214, 118, 149, 380, 109, 121, 194, 230, 141, 373, 89, 372, 358, 259, 262, 97, 303, 125, 378, 40, 100, 136, 104, 346, 48, 244, 334, 163, 241, 143, 41, 76, 56, 249, 106, 7, 116, 248, 327, 35, 58, 152, 355, 4, 247, 281, 158, 50, 200, 92, 364, 94, 345, 0, 238, 191, 19, 196, 25, 313, 111, 183, 243, 63, 49, 90, 223, 260, 159, 370, 5, 77, 193, 160, 198, 146, 276, 273, 381, 181, 253, 156, 74, 129, 180, 255, 31, 161, 166, 322, 332, 93, 123, 302, 308, 236, 173, 281, 132, 38, 210, 112, 87, 117, 120, 150, 279, 184, 128, 219, 293, 209, 221, 33, 174, 73, 267, 138, 60, 98, 192, 354, 133, 237, 148, 137, 203, 329, 216, 81, 83, 319, 294, 75, 13, 68, 114, 328, 145, 298, 212, 101, 127, 44, 306, 316, 371, 254, 47, 360, 158, 80, 376, 71, 140, 88, 377, 91, 51, 307, 147, 15, 227, 144, 367, 288, 233, 263, 352, 359, 246, 21, 356, 155, 3, 295, 341, 337, 24, 274, 126, 157, 27, 67, 382, 268, 2, 357, 54, 162, 265, 59, 231, 224, 283, 179, 339, 316, 308, 254, 47, 360, 360, 103, 119, 218, 113, 199, 277, 252, 250, 22, 349, 299, 225, 9, 189, 234, 325, 197, 336, 287, 139, 251, 291, 211, 347, 353, 115, 34, 79, 186, 36, 215, 271, 96, 310, 201, 122, 55, 375, 228, 285, 61, 282, 272, 297, 369, 350, 20, 289, 190, 46, 206, 86, 103, 82, 257, 208, 363, 45, 284, 240, 151, 365, 175, 344, 43, 23, 78, 270, 338, 331, 39, 304, 66, 383, 290, 154, 275, 266, 292, 269, 57, 261, 213, 134, 12, 315, 239, 374, 99, 324, 202, 242, 32, 222, 107, 167, 108, 321, 20, 371, 355, 47, 360, 29, 103, 124, 119, 140, 169, 226, 182, 320, 16, 26, 366, 280, 135, 220, 368, 229, 84, 18, 362, 318, 62, 286, 165, 30, 65, 301, 14, 102, 258, 335, 333, 305, 105, 342, 207, 130, 85, 37, 176, 6, 326, 95, 235, 311, 10, 17] + 22: [371, 126, 332, 221, 260, 330, 42, 224, 184, 376, 177, 44, 61, 229, 153, 111, 100, 174, 106, 214, 57, 113, 340, 25, 131, 210, 320, 98, 65, 377, 373, 299, 138, 215, 379, 298, 160, 182, 92, 129, 350, 212, 308, 127, 284, 277, 167, 194, 325, 2, 28, 79, 254, 204, 331, 181, 148, 24, 328, 164, 183, 230, 74, 29, 117, 55, 351, 338, 80, 76, 125, 17, 122, 75, 191, 63, 15, 133, 380, 87, 301, 227, 78, 382, 290, 136, 364, 103, 232, 266, 36, 179, 19, 168, 354, 162, 304, 172, 293, 297, 192, 289, 247, 250, 254, 51, 278, 221, 260, 261, 38, 224, 183, 94, 4, 235, 130, 333, 70, 7, 195, 372, 222, 193, 349, 109, 156, 18, 324, 72, 39, 318, 303, 283, 11, 241, 97, 314, 176, 108, 246, 137, 334, 91, 257, 344, 363, 326, 43, 158, 121, 56, 313, 323, 307, 310, 254, 198, 343, 221, 141, 261, 132, 288, 201, 376, 169, 139, 362, 240, 272, 285, 67, 68, 60, 116, 206, 3, 53, 258, 71, 165, 1, 47, 200, 269, 32, 45, 286, 77, 275, 208, 274, 282, 120, 145, 16, 90, 31, 370, 66, 295, 243, 203, 10, 249, 95, 101, 254, 9, 343, 181, 141, 276, 207, 337, 73, 178, 37, 244, 383, 50, 149, 251, 375, 161, 264, 259, 226, 175, 82, 374, 233, 12, 319, 305, 322, 189, 327, 180, 211, 96, 356, 279, 85, 219, 30, 152, 291, 302, 223, 88, 213, 114, 270, 234, 142, 287, 157, 217, 316, 216, 209, 260, 141, 328, 147, 288, 178, 110, 263, 118, 58, 252, 357, 188, 306, 8, 312, 27, 361, 347, 339, 49, 309, 225, 239, 341, 22, 54, 238, 0, 300, 273, 46, 237, 102, 186, 196, 190, 104, 69, 255, 23, 21, 146, 107, 368, 205, 292, 253, 267, 281, 33, 209, 34, 141, 261, 115, 337, 20, 110, 112, 329, 355, 348, 345, 311, 173, 378, 365, 369, 14, 228, 48, 40, 59, 99, 105, 248, 236, 124, 256, 268, 271, 89, 353, 262, 35, 187, 294, 134, 64, 171, 150, 6, 197, 84, 245, 231, 135, 280, 143, 154, 281, 216, 278, 181, 148, 24, 328, 242, 20, 94, 144, 93, 317, 352, 381, 218, 336, 83, 123, 185, 170, 358, 366, 220, 321, 41, 367, 159, 155, 199, 335, 81, 13, 202, 26, 265, 119, 151, 52, 62, 342, 128, 296, 5, 140, 163, 166, 86, 315, 360, 346, 359] + 23: [26, 124, 172, 274, 376, 305, 167, 173, 203, 135, 103, 156, 192, 182, 56, 283, 209, 144, 312, 149, 317, 40, 221, 361, 153, 254, 52, 179, 347, 348, 73, 109, 302, 21, 114, 99, 338, 33, 106, 61, 158, 88, 350, 381, 118, 42, 246, 60, 166, 3, 39, 265, 26, 124, 172, 120, 71, 48, 187, 306, 252, 197, 177, 232, 67, 218, 159, 371, 131, 146, 378, 91, 195, 237, 210, 356, 119, 160, 137, 207, 382, 76, 258, 200, 335, 134, 121, 349, 115, 227, 214, 282, 319, 34, 238, 259, 22, 345, 145, 332, 251, 29, 8, 320, 26, 124, 314, 275, 100, 77, 187, 240, 85, 72, 287, 27, 127, 57, 148, 92, 190, 213, 102, 191, 10, 228, 292, 44, 150, 365, 125, 178, 162, 352, 363, 339, 174, 7, 354, 285, 97, 104, 248, 140, 9, 62, 116, 334, 318, 117, 199, 112, 296, 15, 291, 288, 26, 51, 175, 275, 147, 305, 331, 87, 323, 324, 177, 261, 374, 372, 86, 130, 379, 326, 70, 233, 194, 342, 311, 101, 298, 17, 273, 321, 341, 196, 58, 344, 346, 16, 155, 362, 375, 369, 308, 46, 310, 277, 377, 171, 138, 30, 83, 198, 13, 110, 43, 37, 126, 264, 175, 275, 100, 132, 295, 244, 85, 135, 358, 176, 241, 169, 315, 263, 133, 50, 220, 151, 313, 290, 129, 359, 96, 93, 231, 307, 223, 303, 143, 0, 164, 108, 14, 268, 123, 355, 79, 188, 357, 31, 322, 181, 20, 208, 183, 380, 18, 113, 235, 81, 126, 264, 69, 314, 170, 19, 167, 269, 323, 300, 279, 94, 304, 55, 270, 41, 340, 184, 74, 36, 23, 38, 136, 370, 32, 98, 5, 301, 316, 284, 54, 256, 107, 212, 122, 297, 239, 373, 205, 367, 368, 266, 337, 360, 364, 278, 262, 217, 226, 234, 286, 242, 328, 281, 314, 71, 71, 132, 201, 244, 12, 300, 327, 84, 180, 2, 6, 257, 35, 80, 219, 271, 95, 272, 215, 222, 185, 329, 65, 333, 330, 193, 90, 280, 28, 206, 243, 236, 336, 267, 161, 49, 353, 276, 165, 247, 141, 63, 89, 293, 366, 216, 189, 168, 328, 225, 172, 314, 211, 305, 82, 24, 252, 72, 47, 154, 11, 59, 351, 78, 309, 111, 249, 229, 152, 224, 255, 157, 1, 66, 53, 325, 250, 204, 45, 128, 245, 68, 163, 294, 25, 139, 260, 202, 253, 230, 383, 299, 343, 289, 105, 142, 186, 4, 75, 64] + 24: [0, 246, 265, 20, 328, 210, 223, 360, 85, 121, 294, 186, 283, 24, 164, 382, 299, 131, 142, 177, 338, 205, 272, 321, 235, 323, 116, 11, 239, 60, 346, 282, 271, 296, 76, 161, 281, 160, 171, 291, 25, 18, 187, 240, 165, 316, 208, 364, 36, 374, 74, 81, 0, 246, 265, 20, 252, 7, 139, 381, 85, 189, 255, 375, 53, 78, 195, 306, 93, 156, 130, 96, 380, 3, 206, 63, 169, 295, 313, 280, 194, 30, 103, 4, 38, 14, 173, 249, 373, 163, 262, 342, 180, 184, 378, 300, 199, 347, 261, 40, 219, 158, 198, 241, 290, 9, 154, 6, 252, 259, 263, 28, 361, 128, 301, 29, 137, 204, 192, 54, 318, 254, 350, 73, 341, 310, 114, 326, 178, 13, 304, 244, 339, 270, 365, 72, 276, 196, 212, 126, 99, 191, 102, 193, 183, 80, 201, 332, 200, 243, 69, 322, 89, 129, 327, 379, 97, 9, 154, 95, 252, 122, 223, 157, 27, 189, 334, 62, 26, 202, 35, 15, 226, 106, 138, 351, 293, 77, 48, 59, 147, 278, 197, 1, 292, 215, 335, 333, 303, 150, 175, 94, 352, 359, 50, 234, 362, 367, 98, 162, 124, 285, 33, 305, 45, 127, 251, 353, 52, 110, 111, 265, 252, 172, 263, 70, 369, 286, 266, 207, 237, 31, 144, 61, 325, 308, 46, 336, 224, 307, 58, 297, 153, 269, 47, 51, 302, 134, 218, 203, 79, 88, 86, 188, 8, 148, 209, 44, 109, 236, 43, 174, 141, 123, 349, 82, 358, 179, 264, 298, 52, 9, 246, 265, 22, 372, 337, 105, 369, 64, 5, 230, 319, 213, 344, 366, 71, 355, 113, 149, 145, 245, 279, 376, 84, 168, 242, 315, 65, 225, 37, 170, 348, 10, 363, 320, 57, 331, 217, 317, 101, 68, 247, 250, 167, 340, 214, 17, 152, 133, 115, 166, 87, 211, 154, 20, 22, 372, 185, 357, 324, 128, 220, 370, 343, 159, 354, 91, 41, 274, 356, 49, 182, 12, 287, 83, 257, 233, 222, 56, 66, 181, 258, 267, 176, 120, 107, 284, 42, 67, 329, 238, 132, 256, 143, 228, 90, 227, 371, 100, 2, 21, 232, 314, 92, 92, 154, 6, 7, 7, 223, 357, 70, 112, 104, 146, 231, 135, 277, 190, 273, 260, 377, 39, 345, 118, 221, 140, 216, 16, 311, 268, 312, 108, 309, 275, 23, 253, 32, 151, 136, 19, 55, 288, 248, 125, 229, 75, 119, 289, 117, 330, 34, 155, 368, 383] + 25: [315, 269, 298, 379, 85, 282, 61, 25, 0, 192, 333, 5, 38, 380, 342, 287, 252, 253, 80, 367, 227, 4, 359, 352, 203, 383, 345, 37, 328, 274, 69, 24, 159, 361, 353, 337, 325, 86, 183, 208, 175, 92, 188, 119, 363, 88, 301, 329, 194, 303, 189, 277, 315, 98, 268, 379, 72, 180, 23, 39, 327, 152, 49, 235, 66, 318, 207, 346, 9, 84, 153, 138, 97, 261, 291, 225, 316, 221, 308, 357, 58, 254, 144, 67, 251, 17, 275, 163, 22, 250, 320, 311, 371, 174, 349, 264, 172, 374, 198, 266, 151, 270, 220, 123, 315, 139, 258, 379, 72, 282, 332, 214, 29, 192, 232, 145, 296, 314, 262, 381, 234, 377, 256, 89, 321, 348, 143, 369, 278, 350, 259, 330, 28, 116, 94, 356, 79, 340, 260, 248, 281, 157, 70, 226, 46, 118, 265, 205, 147, 42, 104, 241, 106, 44, 129, 267, 315, 139, 247, 379, 317, 276, 276, 2, 284, 113, 73, 160, 8, 59, 368, 131, 373, 240, 125, 60, 343, 307, 20, 191, 319, 68, 354, 213, 297, 133, 149, 105, 27, 223, 110, 111, 35, 36, 283, 364, 289, 245, 3, 48, 335, 26, 108, 288, 195, 255, 233, 178, 315, 65, 83, 82, 101, 375, 78, 107, 29, 62, 77, 273, 313, 230, 18, 302, 222, 341, 244, 366, 164, 176, 196, 336, 132, 187, 55, 290, 331, 339, 370, 150, 53, 249, 166, 109, 177, 362, 21, 96, 293, 322, 155, 95, 141, 71, 272, 50, 6, 382, 156, 263, 315, 228, 228, 299, 282, 282, 162, 214, 284, 376, 171, 280, 33, 378, 120, 294, 197, 231, 99, 323, 161, 103, 140, 115, 304, 312, 130, 15, 202, 181, 14, 212, 200, 179, 358, 51, 292, 124, 167, 173, 40, 64, 224, 63, 57, 12, 90, 182, 206, 19, 165, 326, 269, 269, 247, 334, 317, 30, 372, 372, 52, 62, 112, 7, 305, 216, 351, 136, 215, 56, 76, 54, 146, 135, 242, 211, 237, 199, 324, 185, 365, 257, 355, 31, 300, 193, 243, 271, 117, 34, 122, 87, 16, 41, 142, 1, 286, 285, 219, 148, 170, 43, 121, 158, 269, 269, 168, 379, 72, 180, 23, 107, 309, 246, 10, 190, 102, 32, 100, 154, 306, 209, 229, 126, 279, 218, 114, 347, 295, 81, 204, 184, 134, 201, 310, 11, 238, 74, 127, 186, 75, 93, 45, 210, 128, 236, 137, 344, 360, 217, 169, 47, 13, 338, 239, 91] + 26: [80, 292, 269, 124, 217, 90, 21, 70, 339, 230, 316, 137, 227, 88, 245, 55, 156, 31, 191, 151, 306, 251, 79, 294, 104, 17, 161, 202, 171, 231, 144, 312, 298, 219, 49, 340, 185, 72, 297, 244, 163, 205, 258, 42, 85, 345, 155, 322, 132, 1, 54, 68, 80, 281, 11, 237, 217, 90, 342, 253, 160, 120, 344, 38, 238, 315, 146, 131, 359, 176, 178, 321, 44, 356, 221, 343, 127, 277, 65, 35, 115, 350, 143, 232, 302, 78, 121, 325, 141, 40, 275, 100, 154, 199, 56, 377, 222, 257, 95, 286, 60, 299, 374, 375, 150, 292, 11, 269, 18, 90, 236, 338, 339, 116, 337, 341, 223, 158, 36, 83, 200, 48, 138, 166, 353, 97, 173, 194, 192, 318, 73, 45, 107, 50, 75, 182, 189, 329, 140, 362, 354, 347, 74, 303, 206, 314, 313, 4, 250, 46, 23, 210, 168, 139, 169, 170, 320, 233, 380, 237, 237, 214, 181, 181, 253, 230, 33, 308, 293, 376, 167, 328, 204, 198, 259, 190, 142, 27, 122, 118, 365, 357, 87, 0, 41, 32, 129, 363, 280, 264, 300, 364, 109, 335, 77, 331, 240, 101, 296, 14, 282, 147, 366, 273, 203, 175, 209, 218, 136, 136, 380, 124, 237, 53, 236, 263, 254, 224, 94, 266, 135, 226, 108, 332, 67, 148, 179, 287, 174, 102, 103, 57, 71, 15, 113, 290, 76, 37, 187, 19, 117, 368, 3, 105, 165, 8, 346, 333, 326, 216, 324, 145, 283, 164, 84, 379, 213, 228, 234, 242, 136, 5, 89, 271, 217, 53, 371, 184, 304, 69, 183, 130, 369, 30, 382, 110, 180, 128, 360, 133, 2, 327, 241, 268, 285, 195, 159, 212, 252, 256, 177, 383, 25, 336, 220, 352, 255, 334, 59, 112, 193, 274, 270, 284, 211, 28, 381, 288, 86, 96, 106, 111, 136, 6, 89, 269, 172, 172, 63, 291, 162, 16, 279, 278, 61, 22, 126, 82, 247, 311, 319, 262, 367, 99, 24, 201, 20, 215, 10, 98, 267, 114, 123, 153, 301, 7, 12, 125, 225, 43, 9, 361, 13, 91, 358, 208, 295, 188, 310, 207, 309, 246, 261, 305, 136, 6, 89, 349, 18, 172, 21, 291, 149, 289, 276, 307, 355, 249, 92, 62, 265, 272, 26, 323, 229, 152, 186, 51, 66, 239, 81, 378, 370, 196, 93, 372, 134, 157, 197, 47, 317, 64, 34, 39, 29, 119, 373, 243, 330, 235, 58, 248, 52, 260, 348, 351] + 27: [34, 177, 225, 286, 12, 350, 168, 204, 30, 349, 227, 354, 179, 336, 230, 290, 380, 77, 308, 78, 140, 356, 241, 162, 368, 326, 215, 268, 124, 67, 9, 120, 353, 3, 361, 105, 14, 152, 282, 264, 330, 16, 195, 87, 1, 148, 210, 40, 17, 100, 270, 338, 234, 145, 4, 225, 263, 90, 71, 73, 357, 80, 341, 202, 369, 55, 182, 260, 324, 72, 115, 32, 201, 187, 161, 155, 54, 10, 136, 274, 46, 43, 20, 307, 101, 217, 35, 79, 139, 259, 346, 186, 302, 68, 273, 64, 165, 311, 298, 371, 93, 212, 146, 163, 234, 177, 58, 286, 12, 90, 168, 269, 30, 49, 358, 377, 213, 183, 335, 340, 76, 196, 156, 23, 322, 337, 15, 261, 271, 6, 172, 327, 267, 106, 347, 249, 29, 13, 275, 33, 306, 154, 313, 253, 352, 158, 62, 166, 194, 314, 355, 57, 209, 235, 176, 220, 81, 104, 279, 286, 31, 312, 265, 252, 197, 349, 341, 24, 188, 237, 251, 21, 242, 339, 294, 89, 297, 191, 42, 88, 309, 175, 114, 218, 276, 239, 147, 318, 250, 367, 107, 130, 344, 208, 185, 65, 41, 364, 86, 189, 226, 11, 19, 134, 243, 236, 240, 244, 103, 103, 4, 225, 263, 374, 164, 73, 320, 128, 227, 354, 207, 143, 48, 323, 381, 99, 343, 325, 141, 317, 133, 254, 126, 379, 174, 127, 375, 248, 299, 66, 121, 0, 205, 151, 132, 113, 359, 74, 150, 372, 94, 198, 331, 138, 255, 206, 378, 247, 262, 310, 103, 301, 59, 333, 61, 312, 265, 252, 315, 119, 366, 131, 348, 112, 228, 85, 149, 258, 83, 332, 284, 109, 98, 370, 91, 246, 304, 360, 45, 18, 2, 199, 365, 95, 123, 116, 144, 184, 319, 293, 110, 289, 257, 108, 52, 160, 221, 92, 36, 180, 285, 342, 103, 37, 117, 12, 70, 90, 233, 69, 26, 80, 211, 181, 222, 190, 75, 303, 383, 232, 102, 278, 373, 193, 27, 7, 316, 214, 291, 122, 287, 129, 44, 28, 321, 84, 153, 159, 96, 231, 50, 8, 157, 142, 283, 345, 56, 245, 277, 169, 51, 200, 39, 53, 103, 37, 219, 351, 61, 312, 300, 69, 135, 376, 211, 292, 224, 97, 272, 178, 266, 216, 238, 82, 25, 118, 305, 328, 125, 329, 5, 296, 362, 111, 171, 38, 47, 167, 288, 256, 280, 281, 223, 22, 229, 382, 334, 192, 173, 170, 295, 363, 63, 203, 60, 137] + 28: [4, 168, 250, 324, 203, 255, 176, 155, 52, 122, 321, 37, 108, 95, 325, 228, 235, 135, 285, 32, 209, 216, 315, 48, 341, 290, 183, 260, 249, 64, 305, 322, 169, 270, 297, 154, 117, 119, 345, 225, 17, 229, 101, 123, 94, 189, 313, 62, 301, 86, 115, 150, 355, 168, 77, 26, 188, 185, 302, 162, 30, 286, 197, 7, 346, 236, 71, 204, 129, 339, 67, 370, 217, 20, 376, 268, 335, 282, 256, 264, 171, 23, 82, 148, 104, 211, 34, 24, 372, 266, 110, 293, 146, 47, 55, 373, 350, 354, 181, 164, 54, 359, 336, 338, 161, 248, 344, 356, 153, 317, 120, 243, 52, 93, 321, 224, 128, 378, 59, 51, 383, 90, 28, 242, 221, 349, 193, 278, 206, 91, 244, 179, 6, 314, 177, 106, 296, 170, 363, 382, 144, 364, 323, 131, 303, 259, 29, 267, 141, 53, 83, 87, 11, 190, 257, 279, 161, 201, 344, 92, 153, 317, 306, 233, 107, 25, 113, 253, 199, 137, 320, 58, 200, 157, 337, 269, 105, 308, 367, 43, 178, 78, 118, 245, 327, 274, 21, 205, 230, 334, 366, 377, 109, 330, 287, 246, 8, 347, 76, 275, 202, 254, 145, 124, 40, 298, 311, 329, 161, 239, 250, 26, 70, 152, 97, 243, 103, 99, 241, 357, 75, 3, 223, 116, 63, 88, 220, 240, 125, 238, 247, 380, 134, 68, 27, 46, 208, 57, 159, 143, 74, 210, 280, 151, 79, 96, 80, 38, 234, 139, 232, 111, 45, 65, 309, 262, 138, 10, 18, 60, 161, 310, 250, 26, 328, 185, 302, 162, 30, 286, 241, 272, 263, 191, 368, 165, 100, 361, 163, 326, 271, 33, 19, 16, 333, 14, 121, 265, 358, 295, 375, 61, 36, 15, 332, 294, 218, 149, 351, 284, 72, 276, 174, 140, 102, 304, 318, 186, 2, 147, 156, 180, 161, 196, 250, 283, 70, 226, 255, 172, 107, 25, 214, 307, 369, 166, 227, 158, 352, 130, 167, 299, 182, 13, 273, 316, 222, 1, 194, 112, 49, 69, 381, 175, 258, 84, 342, 136, 331, 213, 39, 251, 85, 237, 281, 277, 126, 371, 319, 212, 9, 173, 207, 215, 312, 0, 22, 92, 356, 185, 306, 195, 288, 99, 197, 374, 50, 219, 42, 5, 261, 160, 289, 41, 56, 340, 114, 127, 300, 184, 142, 365, 81, 44, 98, 362, 12, 66, 133, 73, 31, 89, 348, 292, 231, 192, 252, 35, 291, 360, 187, 198, 132, 379, 343, 353] + 29: [383, 118, 336, 361, 137, 333, 295, 186, 60, 311, 311, 255, 119, 48, 188, 216, 82, 278, 4, 298, 34, 178, 72, 228, 150, 292, 294, 328, 122, 224, 52, 46, 303, 368, 7, 166, 253, 270, 219, 308, 252, 30, 164, 189, 2, 130, 64, 138, 98, 206, 26, 363, 197, 118, 244, 318, 354, 323, 304, 262, 320, 282, 148, 160, 180, 57, 235, 68, 102, 357, 360, 85, 182, 200, 136, 276, 101, 353, 105, 217, 316, 31, 157, 135, 209, 220, 70, 338, 239, 194, 149, 17, 184, 296, 325, 254, 21, 27, 372, 342, 274, 37, 185, 240, 197, 118, 244, 51, 379, 203, 295, 74, 236, 202, 351, 255, 243, 366, 175, 204, 210, 293, 364, 215, 12, 329, 18, 146, 181, 380, 125, 340, 22, 229, 40, 10, 213, 327, 84, 161, 187, 121, 113, 99, 337, 297, 49, 41, 100, 226, 81, 221, 305, 6, 212, 285, 73, 88, 54, 361, 137, 198, 362, 33, 236, 282, 11, 266, 183, 134, 343, 275, 341, 350, 172, 378, 106, 273, 300, 59, 97, 3, 205, 317, 238, 242, 230, 28, 191, 56, 42, 288, 352, 307, 306, 43, 45, 290, 155, 133, 331, 260, 231, 241, 284, 143, 177, 176, 73, 58, 336, 361, 277, 198, 129, 67, 44, 173, 208, 89, 112, 159, 63, 163, 207, 50, 61, 218, 128, 232, 19, 280, 78, 310, 32, 77, 359, 348, 93, 377, 55, 381, 9, 250, 233, 62, 376, 71, 115, 246, 234, 201, 8, 5, 1, 268, 256, 367, 322, 29, 271, 195, 365, 25, 76, 203, 304, 110, 44, 23, 127, 346, 140, 174, 190, 145, 319, 193, 324, 259, 139, 349, 225, 171, 345, 104, 20, 371, 83, 169, 126, 369, 79, 103, 153, 261, 192, 179, 162, 334, 35, 251, 301, 279, 95, 245, 107, 321, 117, 144, 39, 91, 283, 196, 244, 51, 277, 258, 339, 67, 60, 23, 11, 16, 65, 87, 24, 237, 382, 86, 132, 314, 257, 156, 347, 272, 151, 267, 313, 344, 373, 312, 315, 147, 111, 114, 158, 199, 15, 69, 356, 154, 211, 141, 96, 358, 80, 75, 291, 281, 289, 249, 108, 66, 14, 196, 124, 318, 120, 323, 362, 265, 309, 173, 127, 89, 326, 375, 332, 370, 123, 38, 94, 248, 116, 222, 131, 90, 264, 142, 269, 167, 47, 36, 170, 302, 53, 287, 299, 13, 374, 227, 168, 0, 263, 92, 109, 223, 330, 286, 165, 335, 247, 214, 355, 152] + 30: [140, 262, 208, 213, 187, 366, 78, 225, 168, 373, 321, 10, 147, 316, 224, 240, 100, 58, 141, 136, 344, 369, 343, 261, 54, 383, 197, 129, 241, 73, 50, 145, 194, 183, 347, 161, 203, 327, 368, 122, 169, 330, 102, 338, 171, 93, 251, 301, 36, 40, 47, 88, 140, 262, 208, 333, 187, 243, 59, 35, 351, 351, 234, 223, 212, 250, 159, 89, 1, 193, 151, 204, 18, 285, 200, 96, 167, 162, 163, 286, 349, 181, 282, 360, 342, 377, 371, 326, 381, 5, 370, 355, 16, 113, 126, 313, 367, 182, 252, 305, 94, 101, 118, 123, 140, 208, 208, 239, 13, 65, 245, 214, 143, 206, 62, 45, 283, 362, 216, 317, 336, 339, 210, 320, 296, 160, 23, 91, 72, 114, 359, 111, 375, 248, 297, 25, 166, 202, 263, 32, 57, 125, 130, 291, 173, 120, 198, 0, 149, 228, 4, 70, 307, 128, 134, 146, 133, 184, 208, 76, 137, 75, 196, 35, 295, 373, 188, 28, 17, 315, 97, 12, 77, 256, 309, 382, 170, 356, 325, 104, 38, 103, 20, 294, 211, 115, 249, 318, 311, 95, 350, 90, 348, 165, 364, 81, 105, 232, 112, 235, 42, 26, 119, 227, 379, 334, 345, 380, 158, 266, 239, 239, 358, 65, 65, 378, 257, 156, 289, 64, 41, 302, 43, 74, 152, 275, 299, 217, 292, 98, 157, 278, 179, 335, 51, 310, 55, 48, 44, 374, 186, 121, 220, 139, 69, 85, 209, 314, 178, 352, 207, 265, 63, 61, 191, 238, 8, 21, 22, 31, 37, 266, 208, 272, 86, 53, 245, 52, 143, 2, 11, 246, 164, 303, 340, 341, 253, 192, 9, 195, 274, 258, 174, 353, 231, 142, 376, 66, 293, 7, 92, 288, 337, 107, 267, 264, 83, 155, 82, 127, 233, 255, 300, 80, 328, 229, 27, 131, 331, 185, 189, 218, 37, 266, 239, 239, 358, 242, 245, 214, 24, 62, 361, 144, 33, 205, 312, 221, 177, 6, 215, 226, 281, 298, 365, 108, 84, 150, 14, 172, 180, 244, 284, 79, 110, 230, 247, 279, 99, 287, 273, 153, 19, 190, 106, 56, 372, 34, 87, 176, 354, 219, 237, 259, 254, 124, 239, 222, 29, 53, 135, 52, 46, 156, 289, 346, 68, 271, 49, 138, 357, 71, 236, 277, 260, 67, 60, 132, 329, 306, 319, 280, 109, 332, 290, 175, 154, 276, 324, 304, 116, 323, 117, 3, 15, 308, 270, 199, 39, 148, 30, 201, 363, 268, 269, 322] + 31: [248, 241, 0, 159, 279, 26, 116, 326, 215, 261, 219, 39, 140, 206, 34, 164, 348, 296, 236, 254, 40, 368, 81, 65, 202, 225, 307, 310, 141, 361, 51, 108, 266, 275, 1, 107, 375, 240, 251, 9, 99, 52, 7, 274, 295, 319, 311, 199, 151, 148, 37, 347, 320, 221, 0, 36, 233, 379, 115, 278, 147, 261, 268, 96, 325, 333, 75, 72, 258, 189, 28, 152, 232, 66, 175, 137, 222, 256, 373, 336, 8, 29, 196, 271, 272, 351, 130, 281, 308, 42, 157, 315, 327, 280, 181, 44, 313, 27, 357, 297, 10, 341, 90, 229, 125, 192, 100, 159, 166, 26, 364, 144, 363, 187, 87, 330, 186, 119, 171, 355, 334, 33, 249, 136, 74, 54, 169, 105, 46, 41, 21, 302, 291, 73, 4, 20, 14, 91, 371, 238, 80, 345, 298, 262, 300, 216, 156, 382, 184, 242, 257, 94, 117, 203, 246, 207, 128, 68, 0, 159, 324, 158, 292, 278, 70, 95, 267, 227, 173, 188, 378, 338, 102, 269, 380, 353, 23, 245, 88, 322, 77, 316, 288, 369, 84, 49, 260, 247, 6, 346, 228, 50, 263, 101, 129, 24, 321, 127, 53, 372, 35, 167, 161, 86, 200, 71, 180, 172, 128, 68, 0, 159, 324, 252, 143, 335, 69, 270, 339, 227, 146, 61, 253, 56, 276, 223, 82, 111, 17, 55, 57, 205, 318, 284, 312, 383, 92, 195, 273, 11, 59, 220, 204, 103, 306, 139, 259, 337, 113, 381, 193, 282, 124, 174, 332, 230, 104, 165, 289, 329, 128, 32, 100, 365, 158, 252, 143, 144, 363, 70, 317, 76, 349, 58, 208, 299, 286, 290, 78, 93, 63, 98, 277, 176, 31, 287, 118, 303, 142, 110, 153, 305, 376, 352, 264, 304, 162, 85, 367, 342, 135, 109, 235, 344, 370, 179, 359, 358, 5, 343, 294, 350, 331, 201, 19, 233, 166, 26, 133, 239, 25, 185, 150, 76, 22, 60, 283, 62, 131, 106, 211, 138, 231, 2, 366, 122, 190, 97, 213, 123, 16, 285, 183, 237, 177, 43, 48, 224, 226, 210, 160, 168, 234, 191, 293, 126, 255, 301, 218, 163, 13, 67, 149, 132, 265, 201, 0, 3, 233, 121, 115, 239, 250, 185, 87, 328, 209, 155, 170, 360, 214, 112, 145, 244, 38, 243, 134, 182, 15, 374, 314, 217, 45, 354, 12, 362, 323, 83, 194, 198, 47, 309, 178, 340, 64, 356, 79, 197, 114, 18, 30, 377, 154, 89, 212, 120] + 32: [94, 352, 234, 146, 187, 241, 163, 163, 257, 141, 329, 16, 186, 263, 97, 258, 19, 357, 189, 212, 85, 231, 314, 324, 182, 293, 39, 115, 30, 309, 340, 236, 311, 210, 132, 287, 105, 365, 134, 160, 79, 208, 60, 180, 161, 17, 272, 328, 145, 57, 173, 373, 94, 352, 135, 367, 92, 347, 229, 360, 327, 107, 344, 40, 359, 184, 78, 131, 109, 289, 313, 227, 36, 185, 64, 250, 237, 142, 240, 84, 4, 6, 62, 317, 339, 361, 307, 124, 197, 222, 270, 305, 149, 255, 31, 277, 95, 74, 318, 235, 326, 147, 14, 123, 63, 218, 268, 367, 209, 241, 370, 51, 2, 38, 166, 275, 319, 174, 126, 233, 308, 58, 168, 254, 71, 292, 213, 48, 59, 382, 249, 3, 80, 24, 378, 372, 5, 34, 43, 299, 341, 73, 356, 119, 203, 220, 118, 76, 18, 358, 207, 93, 333, 167, 169, 177, 63, 218, 346, 367, 362, 190, 200, 8, 121, 164, 342, 322, 380, 75, 140, 331, 273, 306, 266, 297, 111, 69, 332, 239, 86, 256, 350, 70, 302, 381, 56, 158, 110, 82, 191, 243, 248, 223, 330, 35, 196, 334, 224, 23, 260, 88, 291, 194, 54, 162, 181, 244, 63, 288, 246, 49, 92, 285, 130, 280, 12, 327, 329, 214, 204, 83, 211, 87, 232, 252, 345, 325, 55, 26, 175, 178, 45, 286, 183, 176, 337, 315, 77, 363, 172, 221, 99, 165, 376, 117, 264, 0, 52, 343, 133, 259, 242, 47, 192, 267, 114, 379, 296, 312, 63, 148, 33, 367, 323, 190, 21, 51, 122, 107, 41, 322, 348, 202, 279, 156, 228, 7, 171, 91, 104, 101, 25, 269, 230, 32, 53, 371, 276, 10, 22, 193, 144, 136, 102, 125, 304, 20, 153, 44, 355, 377, 364, 46, 321, 98, 295, 11, 15, 300, 245, 251, 63, 148, 33, 367, 179, 347, 310, 122, 274, 38, 166, 61, 89, 96, 151, 143, 113, 120, 301, 72, 303, 50, 206, 157, 150, 336, 351, 81, 205, 366, 37, 349, 320, 282, 284, 103, 195, 128, 261, 201, 138, 188, 1, 137, 368, 112, 159, 226, 27, 68, 383, 375, 63, 352, 155, 108, 106, 285, 271, 66, 219, 164, 344, 275, 65, 225, 283, 116, 335, 294, 369, 298, 354, 199, 28, 281, 253, 127, 29, 353, 129, 247, 198, 139, 154, 42, 9, 152, 238, 13, 374, 67, 170, 278, 100, 216, 215, 217, 290, 338, 90, 316, 262, 265] + 33: [86, 63, 280, 191, 362, 36, 325, 343, 183, 154, 307, 147, 84, 352, 335, 233, 33, 210, 40, 288, 126, 354, 90, 138, 276, 375, 157, 370, 152, 340, 345, 330, 281, 26, 101, 327, 10, 182, 286, 168, 119, 120, 353, 199, 97, 313, 344, 203, 357, 129, 83, 333, 167, 260, 339, 38, 158, 380, 319, 96, 39, 299, 73, 241, 146, 206, 361, 17, 35, 117, 237, 268, 235, 351, 187, 4, 123, 80, 156, 360, 186, 364, 368, 71, 350, 103, 271, 41, 221, 23, 181, 225, 78, 62, 178, 337, 275, 374, 172, 294, 263, 159, 161, 293, 91, 145, 177, 362, 323, 229, 319, 81, 258, 283, 68, 125, 89, 5, 48, 204, 30, 151, 131, 222, 61, 202, 329, 317, 45, 22, 197, 209, 369, 60, 249, 215, 24, 366, 110, 29, 135, 298, 11, 243, 310, 245, 59, 341, 171, 274, 296, 302, 77, 272, 102, 287, 91, 320, 177, 377, 121, 64, 148, 343, 301, 283, 267, 94, 322, 37, 270, 54, 1, 193, 356, 9, 8, 211, 173, 109, 347, 262, 19, 163, 180, 213, 137, 371, 113, 52, 285, 67, 136, 6, 74, 381, 224, 257, 175, 242, 195, 269, 44, 304, 282, 264, 236, 372, 91, 260, 259, 38, 64, 380, 148, 107, 251, 301, 267, 94, 201, 266, 255, 376, 25, 338, 334, 217, 116, 290, 355, 230, 198, 124, 50, 104, 212, 13, 15, 43, 66, 65, 112, 248, 194, 316, 207, 132, 332, 312, 326, 315, 214, 359, 99, 2, 143, 0, 32, 69, 328, 75, 339, 38, 31, 36, 231, 115, 115, 154, 307, 348, 72, 139, 190, 21, 87, 150, 308, 311, 18, 232, 205, 174, 216, 196, 382, 373, 114, 58, 49, 106, 349, 166, 278, 219, 252, 238, 300, 27, 184, 365, 244, 363, 306, 7, 309, 46, 100, 189, 12, 220, 51, 176, 331, 38, 153, 36, 379, 297, 251, 82, 342, 314, 89, 164, 169, 273, 160, 227, 192, 133, 14, 303, 93, 254, 383, 162, 256, 200, 88, 234, 170, 3, 336, 70, 53, 261, 128, 20, 111, 76, 291, 98, 134, 378, 250, 253, 57, 284, 223, 239, 95, 142, 240, 140, 339, 38, 42, 229, 319, 81, 258, 299, 179, 358, 108, 226, 127, 141, 218, 155, 208, 289, 305, 144, 149, 16, 79, 246, 55, 367, 279, 105, 122, 292, 28, 56, 324, 295, 318, 346, 130, 85, 188, 118, 247, 165, 92, 228, 321, 47, 185, 265, 34, 277] + 34: [372, 237, 259, 179, 135, 306, 311, 218, 3, 190, 148, 360, 246, 271, 288, 4, 244, 21, 53, 210, 368, 187, 143, 317, 150, 73, 359, 50, 301, 113, 157, 358, 221, 45, 266, 268, 161, 274, 307, 49, 57, 250, 9, 171, 43, 156, 252, 234, 121, 177, 279, 202, 382, 47, 151, 179, 330, 82, 239, 248, 3, 196, 167, 40, 59, 79, 335, 77, 95, 124, 290, 302, 256, 199, 206, 195, 227, 209, 204, 5, 230, 180, 294, 371, 46, 272, 98, 314, 282, 101, 353, 117, 176, 17, 155, 257, 120, 96, 182, 276, 235, 211, 341, 363, 366, 69, 299, 18, 16, 249, 15, 304, 44, 190, 61, 194, 115, 97, 222, 168, 87, 91, 184, 81, 183, 197, 280, 67, 181, 214, 333, 247, 361, 63, 322, 62, 56, 51, 296, 374, 305, 118, 34, 255, 242, 285, 141, 20, 71, 130, 188, 320, 309, 213, 381, 365, 366, 207, 259, 126, 16, 249, 15, 248, 44, 196, 203, 146, 220, 238, 11, 41, 201, 200, 107, 12, 313, 37, 60, 328, 241, 48, 364, 35, 308, 164, 323, 298, 149, 186, 64, 192, 111, 226, 100, 380, 370, 55, 342, 22, 281, 216, 349, 369, 32, 273, 119, 122, 108, 29, 179, 179, 74, 334, 340, 340, 25, 321, 31, 258, 379, 83, 78, 6, 208, 373, 58, 84, 175, 137, 172, 336, 30, 231, 170, 367, 215, 140, 318, 283, 139, 383, 152, 13, 269, 265, 93, 319, 278, 39, 297, 1, 128, 163, 338, 350, 375, 224, 14, 76, 108, 337, 299, 312, 16, 249, 158, 340, 185, 243, 61, 178, 347, 8, 92, 267, 225, 110, 329, 345, 123, 24, 251, 245, 346, 72, 287, 263, 232, 352, 42, 127, 105, 147, 68, 219, 228, 262, 331, 165, 205, 26, 253, 90, 351, 325, 378, 376, 66, 343, 132, 160, 108, 337, 357, 332, 330, 306, 306, 218, 145, 240, 291, 138, 99, 2, 191, 300, 162, 153, 144, 326, 327, 136, 217, 70, 236, 27, 169, 116, 348, 154, 114, 10, 23, 261, 85, 275, 36, 86, 286, 109, 131, 260, 104, 112, 54, 292, 106, 303, 129, 198, 316, 229, 108, 69, 324, 126, 16, 334, 15, 354, 223, 293, 362, 315, 284, 295, 254, 65, 125, 134, 356, 174, 159, 166, 133, 33, 270, 0, 102, 89, 80, 103, 19, 277, 142, 264, 88, 289, 233, 310, 75, 28, 52, 173, 339, 212, 355, 344, 38, 94, 7, 377, 189, 193] + 35: [48, 28, 44, 130, 218, 211, 367, 314, 53, 342, 276, 113, 221, 224, 114, 164, 14, 126, 102, 267, 325, 158, 192, 372, 313, 300, 302, 350, 366, 110, 82, 216, 269, 142, 175, 338, 176, 365, 29, 316, 112, 262, 263, 107, 197, 371, 252, 39, 173, 199, 304, 177, 309, 260, 62, 319, 218, 131, 367, 295, 257, 86, 203, 322, 340, 15, 364, 335, 361, 329, 163, 187, 272, 61, 5, 108, 42, 95, 11, 287, 227, 180, 382, 273, 7, 70, 315, 378, 274, 87, 189, 22, 94, 250, 97, 139, 69, 310, 258, 119, 251, 283, 146, 40, 363, 169, 205, 319, 303, 56, 345, 286, 138, 4, 38, 3, 231, 147, 46, 326, 191, 159, 373, 168, 296, 36, 93, 294, 243, 122, 256, 57, 277, 106, 264, 24, 174, 301, 217, 84, 79, 332, 253, 149, 232, 354, 167, 10, 178, 33, 219, 280, 71, 124, 284, 293, 363, 169, 205, 319, 379, 131, 368, 133, 141, 86, 203, 322, 206, 186, 311, 194, 331, 27, 98, 321, 81, 308, 298, 215, 209, 201, 323, 92, 109, 91, 369, 290, 184, 127, 73, 25, 103, 248, 207, 377, 26, 155, 255, 183, 327, 225, 242, 226, 341, 59, 291, 228, 80, 45, 240, 58, 299, 297, 295, 318, 53, 342, 276, 289, 340, 41, 352, 241, 266, 54, 312, 140, 356, 171, 282, 334, 125, 152, 96, 261, 20, 348, 339, 330, 179, 380, 43, 344, 65, 105, 161, 245, 249, 49, 288, 148, 83, 333, 336, 210, 76, 236, 200, 60, 75, 45, 246, 319, 299, 297, 345, 68, 324, 21, 72, 281, 154, 239, 265, 307, 52, 222, 275, 160, 128, 9, 268, 190, 320, 12, 16, 235, 8, 214, 117, 34, 185, 17, 66, 88, 317, 151, 259, 115, 181, 23, 77, 101, 351, 233, 188, 51, 229, 100, 85, 359, 75, 45, 182, 144, 254, 1, 345, 133, 47, 129, 31, 281, 346, 78, 375, 50, 362, 37, 157, 357, 18, 230, 104, 213, 143, 90, 32, 74, 145, 202, 347, 383, 99, 19, 285, 6, 35, 292, 156, 370, 306, 89, 165, 150, 270, 223, 208, 247, 212, 2, 381, 343, 353, 220, 28, 237, 1, 1, 67, 68, 55, 153, 72, 113, 116, 355, 349, 234, 162, 136, 134, 360, 244, 132, 328, 30, 137, 337, 170, 172, 111, 305, 204, 166, 376, 196, 279, 374, 198, 63, 135, 278, 121, 195, 358, 238, 271, 123, 193, 0, 120, 64, 118, 13] + 36: [53, 340, 184, 232, 232, 371, 325, 260, 178, 320, 380, 107, 150, 205, 125, 42, 201, 334, 219, 191, 297, 105, 172, 339, 153, 117, 11, 46, 255, 30, 212, 335, 148, 263, 283, 48, 209, 295, 383, 248, 35, 75, 226, 235, 151, 96, 353, 218, 234, 376, 64, 123, 53, 176, 184, 304, 185, 10, 368, 101, 296, 193, 56, 99, 291, 152, 31, 45, 175, 111, 94, 55, 13, 121, 3, 289, 354, 271, 195, 224, 323, 264, 76, 206, 88, 149, 208, 29, 83, 247, 381, 77, 316, 14, 110, 378, 303, 285, 136, 106, 221, 268, 25, 37, 131, 340, 341, 348, 266, 62, 311, 272, 210, 274, 12, 90, 97, 5, 54, 65, 162, 112, 194, 293, 63, 68, 50, 155, 216, 366, 343, 102, 217, 229, 81, 36, 358, 220, 318, 239, 329, 189, 236, 173, 60, 120, 324, 126, 161, 27, 124, 181, 214, 145, 290, 360, 192, 382, 184, 304, 266, 10, 328, 311, 98, 369, 12, 373, 352, 79, 314, 261, 321, 108, 288, 139, 361, 245, 300, 276, 72, 147, 228, 364, 211, 71, 257, 17, 165, 267, 51, 18, 281, 158, 265, 199, 130, 21, 350, 8, 7, 203, 89, 286, 317, 43, 279, 273, 252, 74, 341, 232, 238, 59, 371, 230, 196, 193, 225, 301, 207, 308, 85, 93, 240, 246, 231, 49, 167, 91, 164, 15, 168, 280, 237, 202, 180, 187, 143, 379, 188, 23, 159, 135, 363, 6, 259, 249, 357, 347, 115, 355, 154, 1, 227, 222, 377, 186, 292, 375, 294, 370, 144, 57, 232, 2, 325, 299, 242, 241, 225, 107, 73, 133, 233, 32, 309, 262, 9, 345, 122, 103, 322, 342, 142, 140, 41, 277, 134, 39, 52, 344, 129, 119, 365, 146, 70, 190, 169, 251, 256, 118, 61, 253, 278, 128, 82, 269, 182, 114, 127, 170, 22, 22, 84, 250, 238, 359, 368, 272, 178, 326, 372, 223, 275, 47, 287, 156, 33, 67, 166, 349, 138, 284, 69, 183, 346, 174, 100, 116, 305, 200, 38, 28, 351, 78, 243, 58, 44, 109, 338, 19, 337, 307, 254, 87, 26, 330, 141, 4, 367, 270, 313, 302, 22, 197, 144, 57, 232, 371, 368, 272, 258, 40, 372, 223, 374, 92, 332, 331, 333, 362, 319, 282, 132, 95, 298, 20, 171, 179, 66, 177, 310, 0, 104, 157, 244, 80, 163, 356, 24, 204, 315, 198, 215, 137, 312, 213, 86, 34, 327, 306, 160, 113, 16, 336] + 37: [109, 40, 294, 218, 242, 311, 293, 200, 289, 79, 349, 371, 156, 344, 7, 22, 117, 163, 133, 150, 208, 122, 74, 149, 73, 141, 134, 29, 167, 83, 121, 312, 97, 38, 354, 154, 160, 215, 286, 78, 259, 191, 106, 107, 326, 54, 369, 246, 358, 251, 375, 239, 109, 40, 294, 273, 58, 151, 125, 222, 48, 342, 349, 113, 361, 302, 253, 182, 194, 181, 277, 315, 19, 228, 64, 144, 338, 39, 264, 34, 49, 55, 337, 373, 212, 317, 5, 351, 256, 17, 60, 137, 283, 198, 146, 237, 28, 261, 186, 381, 25, 238, 120, 112, 188, 40, 40, 218, 348, 309, 293, 363, 366, 72, 162, 382, 59, 114, 353, 327, 328, 131, 257, 322, 85, 76, 196, 323, 82, 92, 278, 376, 136, 224, 262, 98, 195, 279, 210, 298, 110, 368, 350, 183, 310, 142, 357, 231, 155, 300, 307, 12, 282, 67, 263, 383, 240, 66, 294, 218, 265, 148, 178, 199, 190, 290, 292, 174, 16, 140, 252, 65, 147, 32, 330, 336, 229, 104, 173, 365, 61, 30, 111, 3, 87, 334, 37, 255, 94, 284, 267, 26, 138, 266, 332, 291, 35, 374, 225, 70, 202, 197, 89, 90, 68, 84, 31, 288, 268, 301, 218, 379, 243, 213, 205, 366, 362, 79, 193, 77, 223, 129, 360, 14, 204, 118, 192, 359, 316, 258, 176, 6, 296, 161, 333, 15, 100, 247, 128, 321, 271, 280, 4, 116, 124, 91, 260, 378, 101, 81, 96, 324, 177, 11, 220, 275, 232, 99, 189, 352, 268, 71, 218, 123, 158, 148, 306, 46, 341, 44, 234, 340, 86, 171, 9, 88, 233, 93, 345, 221, 370, 51, 27, 203, 217, 304, 103, 201, 335, 331, 269, 380, 102, 24, 42, 187, 254, 10, 303, 115, 295, 166, 62, 57, 281, 45, 219, 211, 346, 209, 56, 236, 268, 364, 218, 379, 58, 314, 52, 200, 372, 305, 162, 113, 0, 47, 185, 165, 235, 343, 18, 157, 127, 287, 1, 168, 41, 36, 135, 329, 126, 206, 207, 347, 313, 274, 325, 249, 299, 152, 216, 169, 297, 63, 248, 130, 119, 180, 250, 367, 50, 377, 8, 75, 319, 301, 40, 218, 158, 148, 293, 222, 272, 179, 95, 340, 0, 355, 245, 318, 139, 153, 164, 145, 320, 21, 270, 43, 230, 143, 108, 285, 356, 69, 80, 244, 241, 33, 105, 172, 13, 53, 170, 175, 339, 308, 184, 214, 227, 20, 159, 2, 23, 276, 226, 132] + 38: [282, 301, 178, 266, 203, 269, 51, 86, 126, 154, 333, 62, 217, 145, 317, 253, 108, 233, 90, 107, 239, 208, 359, 36, 349, 367, 7, 229, 31, 190, 134, 258, 131, 23, 296, 74, 273, 252, 47, 35, 183, 12, 192, 364, 314, 153, 218, 110, 370, 159, 339, 251, 10, 301, 369, 266, 118, 201, 292, 30, 383, 114, 333, 62, 92, 27, 279, 77, 338, 329, 219, 75, 69, 59, 262, 45, 184, 165, 298, 276, 228, 356, 376, 305, 139, 173, 302, 129, 261, 25, 295, 283, 247, 70, 316, 42, 259, 116, 281, 196, 133, 346, 191, 325, 4, 301, 50, 266, 203, 269, 372, 366, 226, 114, 89, 49, 171, 373, 135, 257, 331, 286, 99, 352, 73, 321, 105, 307, 264, 156, 48, 109, 117, 170, 26, 181, 78, 250, 334, 371, 104, 82, 61, 374, 202, 299, 240, 24, 163, 128, 22, 198, 270, 64, 161, 60, 158, 93, 378, 221, 203, 280, 284, 354, 189, 154, 309, 293, 244, 381, 212, 119, 53, 288, 355, 275, 85, 9, 193, 211, 256, 308, 111, 151, 5, 241, 180, 353, 235, 200, 238, 172, 382, 204, 311, 205, 290, 363, 320, 112, 18, 207, 315, 127, 162, 304, 38, 210, 158, 301, 378, 221, 94, 124, 51, 30, 126, 115, 89, 177, 379, 0, 169, 300, 97, 137, 341, 14, 324, 72, 224, 58, 255, 96, 368, 246, 289, 84, 197, 272, 152, 310, 360, 43, 56, 215, 44, 13, 71, 236, 91, 326, 175, 20, 377, 1, 140, 318, 101, 185, 187, 223, 301, 245, 66, 201, 8, 263, 274, 248, 220, 32, 130, 166, 206, 54, 199, 16, 225, 146, 271, 95, 362, 68, 57, 216, 37, 342, 265, 28, 76, 33, 15, 2, 79, 313, 11, 100, 297, 351, 102, 343, 254, 65, 350, 167, 303, 336, 243, 121, 340, 380, 6, 6, 213, 345, 123, 280, 372, 366, 189, 277, 277, 39, 63, 278, 344, 268, 319, 327, 125, 147, 227, 21, 150, 144, 88, 136, 34, 143, 148, 306, 41, 149, 40, 81, 214, 122, 120, 375, 294, 186, 55, 3, 164, 157, 232, 209, 365, 155, 249, 312, 52, 46, 6, 6, 287, 348, 332, 124, 284, 285, 322, 383, 194, 230, 19, 17, 328, 267, 234, 361, 106, 222, 67, 29, 358, 87, 323, 142, 179, 188, 141, 260, 242, 174, 103, 337, 182, 195, 347, 231, 80, 168, 138, 357, 113, 335, 291, 132, 160, 83, 330, 98, 176, 237] + 39: [43, 268, 258, 310, 223, 286, 56, 139, 360, 325, 271, 85, 270, 277, 121, 319, 365, 59, 227, 53, 54, 225, 336, 34, 126, 188, 364, 226, 182, 61, 167, 357, 6, 309, 183, 327, 291, 217, 287, 168, 108, 206, 123, 92, 210, 147, 124, 224, 356, 90, 334, 209, 100, 180, 110, 104, 372, 368, 315, 312, 93, 313, 328, 86, 145, 374, 295, 178, 208, 381, 298, 352, 191, 28, 138, 47, 358, 150, 317, 249, 302, 106, 45, 203, 164, 30, 240, 181, 166, 192, 335, 222, 157, 231, 142, 359, 233, 158, 125, 316, 261, 115, 89, 143, 176, 35, 116, 234, 236, 156, 56, 218, 38, 279, 272, 85, 297, 324, 250, 341, 51, 81, 84, 259, 140, 251, 190, 378, 353, 281, 307, 161, 134, 311, 241, 193, 4, 16, 363, 60, 120, 70, 288, 189, 266, 246, 37, 197, 255, 351, 318, 237, 332, 148, 21, 361, 247, 282, 258, 104, 372, 25, 56, 13, 38, 242, 379, 379, 296, 204, 141, 15, 77, 146, 235, 8, 232, 31, 263, 260, 152, 366, 322, 301, 33, 355, 320, 97, 128, 79, 377, 214, 376, 27, 196, 303, 305, 347, 64, 10, 342, 349, 256, 73, 154, 186, 304, 22, 247, 96, 66, 310, 372, 236, 315, 65, 132, 314, 119, 2, 329, 26, 40, 369, 3, 200, 273, 383, 340, 323, 101, 331, 177, 135, 144, 345, 212, 72, 289, 354, 338, 130, 68, 194, 216, 245, 41, 117, 78, 160, 18, 229, 339, 136, 67, 262, 184, 11, 9, 149, 247, 280, 66, 82, 310, 286, 315, 312, 36, 36, 267, 292, 362, 333, 98, 76, 163, 290, 375, 42, 127, 112, 278, 137, 62, 199, 63, 75, 14, 293, 185, 173, 275, 195, 220, 265, 171, 343, 103, 107, 55, 83, 252, 39, 219, 129, 32, 105, 269, 29, 321, 44, 99, 282, 66, 310, 348, 1, 113, 65, 93, 313, 187, 382, 174, 172, 330, 24, 49, 87, 202, 274, 248, 122, 239, 74, 50, 276, 179, 213, 228, 94, 95, 285, 221, 243, 306, 169, 91, 58, 19, 111, 337, 175, 211, 238, 284, 350, 294, 308, 344, 201, 52, 198, 282, 282, 264, 104, 236, 153, 151, 215, 132, 314, 155, 244, 80, 71, 17, 300, 7, 23, 118, 170, 48, 299, 69, 131, 367, 371, 5, 162, 373, 102, 88, 133, 230, 254, 114, 207, 283, 109, 346, 159, 46, 20, 0, 253, 326, 370, 205, 12, 57, 380, 165, 257] + 40: [31, 357, 15, 360, 68, 328, 378, 101, 21, 26, 107, 226, 81, 208, 90, 1, 303, 321, 277, 185, 334, 199, 227, 239, 41, 82, 43, 355, 155, 324, 282, 323, 18, 233, 139, 375, 121, 256, 23, 310, 174, 257, 218, 111, 352, 87, 176, 172, 20, 146, 144, 300, 31, 207, 15, 349, 70, 204, 175, 98, 179, 179, 156, 88, 64, 258, 17, 159, 129, 336, 322, 289, 200, 38, 262, 269, 365, 138, 65, 151, 192, 339, 177, 281, 293, 345, 73, 238, 186, 288, 125, 383, 182, 66, 106, 119, 354, 109, 381, 272, 127, 240, 69, 194, 31, 207, 15, 360, 213, 328, 378, 62, 283, 160, 128, 8, 332, 235, 100, 347, 143, 10, 209, 11, 193, 205, 302, 307, 309, 150, 25, 110, 29, 264, 206, 330, 131, 301, 76, 246, 133, 141, 124, 49, 368, 295, 34, 231, 267, 39, 157, 298, 42, 72, 249, 105, 31, 340, 260, 349, 142, 358, 55, 75, 337, 52, 346, 164, 232, 198, 351, 311, 296, 168, 63, 253, 278, 188, 148, 137, 53, 95, 299, 2, 7, 93, 333, 48, 304, 366, 85, 36, 116, 361, 184, 13, 16, 203, 134, 173, 154, 342, 255, 320, 191, 312, 353, 214, 31, 280, 183, 286, 68, 359, 175, 190, 265, 297, 215, 226, 372, 58, 59, 275, 284, 326, 274, 27, 54, 12, 370, 44, 187, 364, 377, 306, 338, 327, 329, 237, 19, 47, 261, 147, 270, 276, 0, 268, 117, 77, 35, 285, 96, 152, 252, 112, 126, 335, 376, 149, 31, 319, 319, 250, 142, 71, 378, 373, 108, 33, 202, 153, 254, 3, 28, 210, 363, 219, 162, 166, 30, 221, 382, 56, 79, 263, 290, 103, 294, 167, 371, 195, 37, 314, 241, 6, 78, 212, 135, 50, 374, 163, 242, 22, 114, 315, 223, 24, 220, 344, 5, 248, 132, 317, 183, 169, 181, 358, 67, 190, 21, 33, 279, 161, 369, 224, 228, 91, 89, 136, 341, 211, 145, 216, 229, 46, 99, 178, 189, 130, 120, 287, 305, 243, 350, 51, 222, 118, 115, 14, 102, 266, 40, 113, 104, 325, 313, 380, 245, 86, 83, 225, 343, 123, 379, 317, 319, 250, 68, 71, 175, 367, 108, 247, 346, 158, 74, 251, 97, 9, 362, 271, 80, 61, 308, 236, 196, 259, 316, 57, 244, 94, 165, 318, 356, 171, 273, 60, 201, 291, 122, 84, 348, 32, 140, 230, 217, 180, 234, 4, 45, 292, 197, 331, 170, 92] + 41: [205, 87, 103, 114, 355, 350, 184, 30, 273, 366, 124, 215, 34, 69, 347, 306, 348, 127, 140, 353, 26, 32, 226, 83, 294, 138, 250, 268, 299, 284, 91, 94, 74, 221, 99, 75, 78, 77, 297, 149, 316, 209, 14, 245, 62, 2, 162, 346, 169, 135, 40, 132, 305, 283, 114, 199, 376, 60, 374, 190, 366, 223, 119, 285, 79, 24, 311, 95, 326, 35, 239, 36, 174, 345, 241, 46, 322, 344, 339, 51, 131, 338, 246, 5, 371, 354, 232, 33, 143, 287, 320, 158, 98, 195, 231, 17, 139, 157, 312, 361, 193, 164, 178, 152, 122, 283, 196, 199, 142, 307, 184, 57, 197, 50, 119, 92, 136, 161, 206, 82, 383, 4, 202, 218, 252, 56, 313, 254, 271, 115, 144, 328, 151, 295, 81, 317, 61, 128, 123, 292, 1, 191, 259, 48, 257, 204, 375, 373, 121, 343, 106, 341, 281, 228, 22, 85, 321, 298, 335, 142, 170, 355, 374, 255, 168, 368, 159, 97, 364, 358, 302, 253, 80, 351, 25, 23, 134, 116, 282, 55, 237, 289, 224, 266, 176, 186, 19, 213, 244, 146, 59, 272, 352, 0, 141, 68, 73, 175, 171, 370, 110, 323, 377, 109, 203, 333, 31, 120, 321, 298, 378, 71, 170, 350, 90, 30, 369, 365, 159, 76, 7, 314, 262, 337, 104, 53, 286, 243, 308, 64, 225, 256, 235, 117, 242, 8, 63, 290, 277, 261, 220, 214, 37, 342, 39, 65, 72, 332, 148, 137, 29, 274, 267, 319, 270, 16, 58, 118, 227, 309, 185, 269, 378, 11, 251, 60, 374, 86, 369, 200, 327, 96, 187, 360, 334, 44, 318, 222, 264, 163, 336, 70, 54, 276, 258, 167, 112, 210, 363, 153, 18, 43, 173, 89, 41, 188, 236, 381, 234, 105, 310, 28, 265, 329, 331, 88, 177, 372, 291, 52, 263, 357, 49, 49, 378, 199, 27, 60, 212, 190, 84, 129, 275, 293, 172, 279, 12, 108, 45, 3, 219, 194, 349, 102, 125, 93, 229, 340, 133, 288, 66, 107, 6, 130, 315, 303, 359, 154, 100, 165, 13, 248, 113, 155, 67, 20, 183, 145, 189, 362, 382, 233, 247, 216, 49, 269, 367, 166, 170, 350, 249, 150, 197, 129, 275, 215, 111, 21, 280, 198, 9, 324, 42, 126, 217, 301, 379, 260, 156, 325, 330, 380, 101, 181, 208, 160, 10, 192, 15, 300, 179, 180, 207, 38, 147, 182, 238, 47, 201, 240, 211, 356, 304, 230, 296, 278] + 42: [2, 123, 123, 152, 119, 357, 211, 93, 63, 256, 333, 195, 108, 59, 319, 277, 141, 30, 236, 287, 87, 103, 38, 202, 223, 297, 241, 8, 311, 89, 244, 99, 313, 16, 355, 112, 26, 206, 267, 104, 312, 136, 330, 153, 35, 249, 182, 142, 13, 361, 294, 25, 188, 123, 123, 152, 31, 268, 155, 247, 282, 158, 41, 168, 377, 48, 122, 379, 248, 4, 169, 214, 105, 147, 92, 316, 257, 149, 245, 137, 307, 246, 49, 97, 298, 204, 156, 171, 194, 318, 90, 250, 124, 186, 207, 85, 115, 100, 44, 14, 91, 252, 126, 324, 210, 123, 123, 152, 31, 345, 176, 279, 151, 256, 333, 162, 55, 208, 342, 351, 275, 341, 135, 28, 173, 338, 314, 280, 76, 160, 80, 101, 371, 323, 130, 226, 258, 196, 170, 364, 120, 261, 98, 368, 3, 23, 309, 88, 353, 110, 54, 78, 240, 273, 27, 71, 154, 42, 266, 295, 117, 305, 155, 116, 325, 264, 37, 222, 262, 209, 281, 373, 358, 221, 259, 344, 329, 356, 322, 366, 39, 227, 113, 19, 237, 238, 109, 157, 7, 5, 125, 269, 56, 272, 197, 10, 285, 198, 367, 134, 190, 11, 81, 177, 365, 289, 121, 95, 375, 50, 266, 148, 117, 303, 176, 350, 215, 111, 37, 243, 306, 6, 21, 348, 274, 320, 239, 17, 165, 326, 77, 234, 52, 381, 18, 132, 231, 9, 283, 299, 58, 251, 378, 346, 293, 229, 102, 363, 185, 75, 144, 133, 12, 47, 107, 317, 15, 24, 352, 339, 359, 217, 94, 114, 117, 268, 172, 218, 93, 150, 138, 349, 200, 255, 327, 191, 328, 178, 140, 187, 161, 216, 164, 143, 362, 192, 60, 51, 62, 64, 315, 310, 205, 291, 53, 300, 343, 284, 82, 145, 296, 304, 174, 22, 69, 57, 331, 382, 86, 40, 46, 254, 359, 217, 94, 228, 117, 106, 184, 218, 325, 111, 138, 337, 220, 203, 189, 265, 131, 354, 286, 230, 167, 308, 335, 33, 118, 380, 340, 146, 372, 225, 139, 183, 61, 336, 270, 233, 213, 290, 321, 360, 292, 288, 383, 96, 253, 79, 83, 271, 66, 74, 193, 369, 20, 1, 123, 152, 31, 345, 172, 212, 72, 158, 179, 374, 129, 263, 127, 235, 370, 29, 224, 67, 201, 36, 181, 45, 166, 65, 32, 301, 180, 219, 260, 232, 34, 332, 68, 278, 84, 175, 199, 73, 128, 334, 302, 347, 276, 70, 159, 43, 0, 163, 242, 376] + 43: [164, 161, 171, 375, 375, 197, 359, 37, 349, 190, 366, 131, 326, 377, 29, 227, 108, 121, 379, 73, 232, 48, 286, 299, 367, 61, 155, 345, 60, 50, 45, 284, 146, 343, 159, 304, 267, 76, 151, 233, 290, 240, 311, 124, 106, 236, 208, 333, 33, 310, 192, 95, 369, 221, 171, 375, 278, 129, 35, 216, 14, 97, 100, 314, 57, 153, 342, 360, 338, 136, 374, 222, 378, 88, 234, 123, 5, 79, 28, 107, 49, 165, 143, 46, 263, 370, 175, 72, 279, 87, 158, 268, 300, 110, 260, 230, 62, 249, 270, 58, 289, 361, 206, 214, 27, 221, 105, 225, 92, 142, 66, 59, 20, 75, 297, 43, 23, 55, 127, 264, 305, 147, 4, 272, 154, 275, 83, 140, 34, 130, 241, 148, 185, 47, 217, 74, 125, 174, 116, 207, 356, 163, 381, 119, 330, 258, 51, 276, 0, 137, 352, 96, 212, 302, 169, 348, 344, 6, 105, 225, 372, 335, 359, 324, 13, 190, 297, 43, 22, 81, 111, 318, 224, 280, 85, 77, 282, 204, 339, 26, 160, 84, 196, 145, 99, 183, 253, 167, 128, 325, 215, 11, 294, 132, 287, 358, 355, 187, 285, 248, 172, 281, 340, 139, 244, 184, 373, 245, 315, 168, 177, 347, 92, 17, 66, 98, 283, 75, 94, 298, 363, 337, 301, 323, 307, 202, 368, 150, 52, 346, 306, 135, 296, 257, 237, 12, 242, 180, 16, 70, 309, 288, 170, 122, 8, 65, 247, 209, 188, 336, 201, 53, 317, 303, 40, 15, 319, 376, 182, 382, 315, 168, 332, 347, 218, 17, 66, 98, 283, 252, 30, 351, 235, 93, 223, 178, 1, 78, 89, 32, 205, 327, 86, 115, 261, 157, 194, 63, 24, 113, 56, 120, 101, 256, 295, 329, 144, 353, 118, 173, 265, 316, 162, 114, 41, 228, 42, 133, 350, 54, 189, 354, 262, 168, 156, 308, 372, 335, 98, 324, 39, 239, 30, 351, 152, 291, 138, 203, 274, 269, 255, 2, 238, 246, 277, 271, 149, 365, 10, 243, 293, 38, 210, 251, 9, 80, 31, 292, 259, 126, 19, 134, 211, 357, 341, 69, 7, 141, 383, 104, 25, 166, 331, 44, 82, 181, 156, 225, 68, 371, 112, 364, 14, 254, 103, 298, 23, 186, 322, 313, 198, 321, 250, 191, 90, 380, 219, 200, 36, 67, 334, 273, 102, 229, 266, 213, 220, 226, 18, 64, 71, 117, 193, 109, 320, 179, 312, 199, 328, 21, 362, 91, 176, 231, 195, 3] + 44: [362, 17, 213, 178, 199, 379, 101, 101, 190, 43, 277, 279, 93, 86, 212, 329, 228, 275, 276, 121, 184, 296, 163, 287, 66, 164, 87, 221, 342, 126, 147, 165, 270, 309, 32, 305, 230, 9, 367, 114, 255, 216, 22, 119, 246, 103, 259, 361, 197, 310, 129, 206, 362, 258, 213, 272, 191, 379, 348, 284, 227, 82, 170, 135, 288, 187, 182, 306, 144, 125, 324, 91, 6, 290, 150, 56, 352, 338, 359, 44, 36, 136, 92, 74, 149, 231, 341, 179, 235, 345, 314, 7, 124, 375, 195, 151, 363, 302, 47, 73, 316, 281, 295, 374, 274, 167, 77, 191, 199, 317, 317, 223, 188, 257, 346, 311, 236, 280, 141, 63, 48, 95, 355, 117, 266, 186, 122, 39, 71, 160, 54, 261, 107, 46, 83, 40, 8, 377, 81, 3, 241, 109, 4, 51, 273, 1, 27, 50, 247, 176, 293, 31, 58, 75, 209, 106, 274, 0, 25, 300, 382, 10, 348, 271, 21, 145, 346, 229, 30, 146, 118, 38, 365, 360, 94, 265, 350, 24, 154, 315, 131, 366, 96, 369, 162, 220, 370, 159, 59, 318, 169, 104, 57, 250, 49, 62, 312, 326, 153, 194, 283, 260, 297, 52, 76, 181, 339, 116, 168, 171, 77, 191, 10, 10, 348, 155, 21, 145, 68, 331, 34, 98, 333, 198, 102, 337, 343, 138, 224, 254, 55, 100, 132, 378, 67, 120, 128, 41, 108, 282, 105, 323, 211, 218, 72, 193, 204, 137, 2, 253, 373, 115, 28, 268, 356, 111, 88, 371, 13, 332, 238, 171, 77, 301, 222, 110, 226, 60, 180, 29, 328, 368, 143, 383, 289, 142, 304, 291, 89, 207, 85, 175, 219, 364, 130, 286, 269, 134, 327, 344, 15, 185, 205, 347, 23, 20, 203, 319, 321, 65, 35, 200, 5, 251, 372, 307, 123, 127, 353, 308, 112, 12, 97, 172, 213, 301, 222, 351, 226, 284, 215, 376, 252, 140, 166, 313, 249, 264, 358, 225, 14, 340, 192, 61, 349, 70, 285, 380, 174, 177, 157, 354, 278, 334, 243, 78, 336, 237, 234, 357, 256, 183, 42, 45, 233, 210, 298, 330, 37, 152, 148, 16, 161, 322, 64, 172, 213, 301, 199, 317, 226, 223, 267, 84, 170, 325, 11, 320, 242, 294, 33, 217, 158, 262, 244, 299, 173, 201, 79, 248, 202, 381, 292, 26, 196, 90, 214, 19, 156, 113, 263, 18, 53, 232, 245, 80, 239, 99, 240, 69, 139, 335, 133, 208, 189, 303] + 45: [154, 140, 92, 161, 42, 197, 8, 176, 350, 373, 378, 61, 81, 297, 234, 228, 187, 158, 155, 372, 54, 267, 41, 157, 67, 109, 302, 370, 330, 116, 256, 358, 283, 231, 263, 79, 166, 203, 147, 362, 214, 321, 89, 143, 159, 352, 335, 184, 130, 72, 137, 64, 154, 233, 316, 92, 334, 346, 8, 356, 367, 146, 47, 164, 95, 128, 209, 258, 168, 45, 36, 250, 112, 306, 249, 179, 32, 24, 4, 218, 76, 126, 87, 38, 98, 333, 342, 374, 220, 119, 190, 73, 91, 132, 153, 232, 194, 366, 77, 99, 235, 40, 355, 300, 324, 140, 134, 110, 139, 178, 120, 227, 375, 367, 329, 131, 212, 114, 107, 254, 239, 122, 104, 88, 12, 135, 345, 156, 292, 1, 123, 43, 20, 170, 252, 296, 29, 322, 381, 271, 0, 289, 113, 202, 337, 90, 62, 226, 305, 216, 6, 28, 195, 14, 117, 266, 23, 318, 92, 284, 25, 178, 27, 227, 227, 262, 286, 164, 95, 272, 224, 331, 307, 55, 277, 74, 84, 7, 86, 44, 315, 2, 303, 340, 19, 207, 145, 291, 294, 255, 151, 314, 142, 309, 371, 377, 225, 125, 56, 186, 200, 101, 259, 325, 149, 17, 244, 83, 23, 219, 144, 161, 42, 10, 10, 141, 33, 85, 276, 269, 81, 383, 183, 39, 359, 326, 115, 308, 327, 251, 108, 281, 279, 365, 22, 63, 35, 133, 319, 59, 37, 211, 57, 285, 341, 15, 3, 71, 243, 189, 280, 167, 193, 295, 287, 172, 240, 185, 138, 100, 191, 140, 188, 293, 334, 201, 274, 247, 33, 160, 180, 329, 124, 383, 380, 118, 198, 16, 229, 70, 68, 357, 213, 217, 106, 96, 261, 50, 344, 230, 304, 264, 150, 379, 177, 288, 246, 268, 364, 343, 111, 323, 31, 152, 270, 53, 278, 237, 49, 181, 361, 347, 173, 301, 92, 293, 178, 182, 10, 171, 350, 103, 276, 21, 165, 349, 52, 275, 223, 196, 51, 338, 313, 80, 360, 260, 26, 236, 253, 222, 65, 34, 290, 320, 273, 241, 169, 204, 105, 48, 368, 93, 136, 102, 18, 97, 353, 148, 13, 75, 60, 299, 192, 174, 121, 121, 92, 317, 199, 201, 27, 339, 369, 373, 354, 298, 163, 348, 248, 351, 310, 162, 30, 245, 328, 210, 242, 94, 175, 208, 376, 257, 58, 205, 46, 5, 9, 282, 127, 129, 363, 312, 78, 215, 238, 332, 382, 82, 206, 265, 311, 66, 11, 69, 336, 221] + 46: [152, 183, 352, 256, 286, 343, 110, 207, 221, 354, 66, 335, 194, 165, 199, 3, 102, 30, 108, 227, 186, 168, 42, 4, 362, 250, 103, 17, 111, 101, 35, 160, 313, 56, 71, 333, 145, 90, 361, 375, 15, 75, 127, 229, 300, 134, 279, 349, 14, 182, 5, 36, 188, 183, 34, 58, 368, 343, 16, 51, 367, 106, 106, 301, 40, 68, 338, 76, 215, 62, 130, 83, 132, 245, 192, 37, 41, 53, 292, 193, 268, 329, 97, 360, 89, 189, 293, 265, 283, 190, 285, 26, 52, 135, 94, 126, 105, 174, 296, 0, 100, 365, 27, 88, 142, 247, 352, 139, 224, 355, 25, 191, 149, 367, 66, 335, 214, 55, 18, 203, 39, 211, 117, 120, 80, 118, 156, 166, 164, 95, 216, 321, 377, 307, 263, 290, 223, 382, 243, 322, 177, 291, 278, 98, 347, 21, 77, 366, 363, 1, 306, 84, 200, 270, 325, 213, 142, 247, 352, 139, 380, 253, 311, 239, 149, 354, 319, 154, 136, 204, 303, 219, 350, 351, 195, 155, 232, 121, 180, 13, 129, 162, 237, 340, 383, 32, 141, 255, 161, 173, 159, 241, 359, 93, 262, 234, 208, 43, 258, 281, 87, 196, 179, 47, 151, 82, 231, 122, 142, 183, 34, 64, 240, 46, 311, 96, 230, 284, 104, 12, 124, 114, 44, 146, 79, 228, 33, 22, 326, 197, 295, 298, 176, 273, 198, 235, 57, 81, 206, 169, 222, 346, 7, 277, 91, 217, 244, 339, 337, 358, 275, 341, 288, 269, 248, 54, 70, 225, 252, 175, 133, 10, 64, 58, 380, 253, 249, 238, 246, 332, 140, 140, 60, 38, 113, 374, 282, 131, 115, 260, 287, 315, 137, 316, 304, 128, 205, 158, 45, 226, 92, 289, 119, 318, 74, 153, 261, 331, 86, 257, 138, 67, 330, 236, 107, 181, 29, 185, 28, 148, 147, 302, 133, 201, 19, 256, 286, 311, 110, 202, 274, 6, 209, 312, 187, 254, 345, 320, 266, 280, 8, 372, 334, 218, 259, 50, 63, 294, 267, 59, 125, 308, 328, 24, 299, 348, 317, 20, 364, 309, 305, 171, 150, 112, 11, 72, 73, 48, 212, 123, 357, 323, 379, 356, 210, 10, 233, 314, 327, 355, 170, 207, 230, 324, 319, 344, 271, 378, 2, 242, 85, 49, 143, 376, 99, 353, 109, 369, 23, 272, 172, 157, 336, 373, 78, 144, 297, 69, 9, 342, 264, 370, 184, 371, 163, 65, 310, 31, 381, 251, 116, 276, 167, 178, 220, 61] + 47: [195, 347, 171, 162, 291, 3, 109, 34, 382, 345, 314, 353, 129, 267, 327, 153, 122, 250, 95, 377, 151, 263, 172, 197, 0, 138, 354, 53, 234, 91, 70, 28, 259, 258, 32, 260, 334, 150, 189, 55, 383, 304, 86, 220, 76, 64, 99, 256, 254, 7, 156, 209, 195, 331, 143, 162, 357, 3, 170, 158, 193, 134, 320, 316, 362, 21, 59, 349, 159, 351, 104, 42, 224, 284, 184, 50, 51, 17, 173, 210, 378, 141, 243, 213, 276, 60, 185, 310, 160, 10, 120, 266, 358, 108, 58, 222, 88, 142, 283, 165, 261, 381, 373, 360, 195, 187, 192, 352, 312, 371, 119, 158, 307, 300, 14, 294, 175, 168, 40, 218, 251, 330, 208, 67, 89, 103, 65, 364, 101, 148, 4, 112, 23, 336, 71, 335, 94, 278, 49, 186, 81, 190, 275, 57, 177, 374, 30, 317, 145, 194, 116, 295, 13, 83, 62, 369, 195, 179, 154, 352, 312, 43, 35, 29, 227, 134, 137, 127, 36, 118, 123, 229, 98, 269, 84, 262, 249, 228, 323, 77, 204, 45, 39, 117, 366, 80, 321, 147, 273, 31, 200, 132, 297, 242, 282, 216, 245, 380, 100, 38, 85, 244, 161, 221, 241, 113, 146, 246, 75, 271, 154, 352, 312, 3, 279, 61, 307, 359, 90, 215, 274, 110, 238, 363, 212, 313, 69, 219, 223, 324, 33, 199, 205, 355, 233, 27, 96, 15, 348, 225, 292, 306, 107, 20, 144, 102, 48, 302, 74, 350, 105, 361, 202, 268, 78, 206, 272, 311, 236, 288, 343, 54, 136, 44, 24, 22, 87, 87, 359, 300, 305, 368, 79, 287, 346, 342, 114, 226, 93, 135, 124, 41, 370, 203, 296, 237, 340, 252, 139, 356, 126, 303, 328, 37, 341, 2, 130, 46, 19, 72, 379, 174, 133, 277, 270, 299, 232, 6, 8, 169, 214, 56, 343, 54, 181, 44, 24, 22, 87, 1, 382, 333, 137, 257, 344, 376, 188, 235, 63, 265, 26, 301, 285, 231, 298, 255, 322, 25, 180, 106, 183, 18, 211, 163, 281, 111, 329, 248, 97, 9, 365, 325, 290, 92, 339, 372, 247, 264, 167, 125, 182, 128, 191, 293, 343, 54, 181, 44, 24, 3, 375, 29, 280, 309, 286, 196, 198, 240, 131, 367, 176, 230, 155, 68, 166, 319, 140, 73, 12, 338, 52, 164, 315, 201, 121, 289, 337, 332, 115, 157, 253, 149, 152, 308, 207, 5, 16, 178, 47, 11, 82, 217, 66, 318, 239, 326] + 48: [237, 266, 128, 77, 301, 273, 116, 132, 311, 295, 58, 83, 157, 287, 139, 254, 142, 41, 320, 156, 322, 25, 75, 88, 118, 31, 172, 218, 22, 100, 158, 63, 374, 40, 206, 259, 382, 272, 214, 69, 37, 87, 309, 148, 154, 153, 124, 68, 103, 199, 209, 196, 267, 249, 128, 102, 328, 273, 239, 19, 52, 353, 268, 28, 15, 369, 235, 363, 186, 305, 371, 222, 130, 291, 233, 123, 341, 111, 4, 13, 379, 351, 119, 97, 43, 349, 98, 137, 243, 286, 194, 326, 323, 213, 228, 191, 8, 377, 332, 383, 364, 131, 1, 250, 267, 266, 271, 145, 328, 215, 226, 234, 52, 2, 141, 78, 15, 6, 5, 48, 34, 354, 378, 300, 7, 192, 247, 73, 89, 212, 275, 231, 307, 70, 49, 36, 269, 367, 346, 107, 82, 71, 366, 29, 165, 335, 66, 356, 86, 345, 251, 122, 337, 348, 245, 227, 159, 20, 271, 117, 140, 358, 239, 132, 284, 353, 133, 85, 263, 147, 344, 208, 316, 181, 319, 205, 175, 50, 120, 190, 170, 221, 230, 79, 202, 113, 289, 81, 84, 18, 144, 35, 39, 173, 310, 313, 27, 294, 238, 225, 298, 60, 183, 262, 197, 333, 30, 308, 297, 76, 271, 65, 140, 330, 276, 121, 74, 101, 33, 85, 263, 189, 177, 372, 342, 134, 155, 229, 179, 299, 174, 45, 255, 24, 168, 57, 317, 281, 72, 204, 280, 12, 55, 274, 178, 256, 95, 93, 143, 277, 350, 149, 171, 260, 365, 126, 265, 355, 304, 114, 288, 76, 17, 26, 279, 330, 99, 236, 284, 182, 141, 21, 62, 166, 327, 264, 185, 380, 368, 312, 193, 257, 285, 340, 329, 321, 318, 91, 64, 80, 146, 253, 211, 16, 151, 195, 108, 187, 92, 67, 163, 278, 244, 361, 362, 125, 46, 54, 306, 336, 200, 334, 9, 283, 17, 102, 150, 109, 276, 44, 203, 90, 381, 381, 207, 375, 223, 359, 115, 184, 232, 210, 246, 152, 370, 0, 160, 242, 347, 360, 180, 241, 96, 11, 198, 53, 169, 59, 216, 32, 135, 373, 224, 61, 201, 376, 331, 282, 248, 23, 220, 167, 136, 314, 51, 283, 17, 270, 140, 328, 132, 343, 311, 101, 164, 78, 62, 104, 240, 38, 325, 252, 290, 219, 161, 293, 112, 324, 162, 357, 315, 42, 339, 14, 138, 302, 352, 127, 292, 94, 261, 47, 217, 3, 56, 105, 110, 188, 338, 106, 296, 176, 303, 258, 10, 129] + 49: [374, 184, 272, 268, 155, 65, 181, 293, 116, 25, 270, 276, 193, 218, 260, 306, 45, 255, 164, 235, 328, 279, 221, 23, 240, 112, 189, 376, 303, 371, 267, 178, 345, 320, 53, 130, 2, 151, 190, 22, 125, 357, 51, 147, 237, 18, 350, 291, 369, 50, 24, 38, 374, 197, 321, 346, 257, 297, 19, 213, 353, 212, 153, 9, 44, 199, 133, 187, 312, 332, 109, 5, 205, 340, 137, 223, 198, 290, 41, 163, 301, 238, 43, 62, 142, 211, 280, 61, 31, 285, 361, 354, 185, 232, 377, 341, 159, 231, 379, 383, 32, 129, 135, 337, 79, 302, 149, 6, 209, 297, 19, 253, 313, 105, 90, 335, 148, 251, 91, 220, 331, 30, 298, 144, 73, 336, 54, 86, 140, 224, 34, 248, 216, 10, 67, 134, 244, 72, 176, 355, 49, 84, 101, 108, 324, 104, 166, 165, 162, 161, 99, 263, 288, 219, 92, 375, 79, 302, 37, 6, 15, 234, 150, 126, 202, 271, 110, 275, 193, 251, 107, 74, 186, 8, 131, 58, 169, 174, 80, 368, 196, 170, 222, 146, 139, 179, 35, 309, 359, 339, 378, 71, 46, 206, 143, 256, 88, 274, 315, 87, 367, 21, 236, 11, 258, 26, 349, 123, 210, 302, 272, 268, 75, 305, 115, 157, 66, 241, 208, 180, 39, 325, 76, 344, 381, 175, 94, 122, 366, 17, 382, 70, 365, 29, 160, 81, 225, 327, 330, 338, 172, 47, 85, 118, 204, 282, 342, 145, 364, 316, 93, 167, 42, 266, 249, 141, 230, 370, 300, 356, 233, 302, 321, 319, 75, 305, 158, 253, 313, 78, 201, 9, 262, 60, 286, 269, 329, 100, 333, 195, 97, 1, 28, 56, 138, 89, 323, 20, 55, 173, 352, 227, 351, 114, 226, 119, 12, 124, 113, 154, 326, 343, 13, 7, 363, 265, 171, 217, 203, 3, 188, 64, 360, 302, 321, 83, 155, 297, 136, 126, 202, 271, 40, 242, 259, 380, 243, 132, 194, 264, 314, 48, 250, 362, 16, 245, 277, 128, 4, 296, 82, 295, 33, 98, 156, 239, 69, 358, 106, 192, 228, 52, 182, 111, 27, 372, 102, 287, 152, 207, 252, 373, 322, 191, 360, 302, 321, 294, 304, 59, 157, 253, 126, 278, 261, 242, 177, 292, 289, 246, 120, 183, 95, 121, 254, 63, 310, 311, 318, 214, 127, 77, 273, 308, 36, 307, 103, 14, 117, 0, 200, 284, 283, 299, 348, 168, 57, 229, 96, 334, 281, 215, 247, 347, 68, 317] + 50: [57, 100, 140, 49, 14, 352, 370, 325, 214, 312, 282, 1, 128, 220, 256, 331, 15, 249, 133, 301, 228, 195, 194, 240, 319, 276, 267, 107, 322, 91, 211, 143, 243, 39, 289, 123, 34, 46, 68, 354, 323, 346, 60, 61, 207, 31, 142, 309, 59, 336, 50, 93, 33, 100, 137, 49, 376, 284, 215, 35, 146, 175, 350, 1, 304, 199, 358, 51, 184, 298, 132, 196, 239, 165, 263, 53, 342, 197, 74, 326, 152, 242, 105, 356, 17, 362, 344, 259, 308, 141, 212, 36, 138, 11, 40, 176, 190, 72, 279, 227, 115, 131, 257, 374, 80, 226, 311, 181, 376, 233, 188, 325, 366, 56, 280, 129, 128, 327, 88, 79, 272, 187, 274, 244, 48, 186, 168, 359, 252, 139, 85, 73, 20, 78, 185, 108, 135, 292, 204, 106, 63, 372, 355, 258, 112, 87, 77, 21, 47, 265, 54, 144, 254, 84, 332, 345, 29, 268, 109, 166, 97, 210, 215, 232, 125, 287, 26, 3, 231, 217, 334, 295, 271, 335, 24, 164, 306, 75, 154, 43, 269, 174, 151, 303, 294, 99, 357, 5, 37, 202, 4, 170, 7, 320, 150, 293, 285, 82, 209, 83, 103, 70, 158, 225, 52, 66, 241, 230, 29, 268, 109, 166, 333, 210, 188, 273, 321, 56, 350, 18, 329, 296, 119, 155, 224, 213, 122, 41, 149, 117, 116, 219, 206, 208, 310, 64, 126, 2, 340, 300, 178, 297, 339, 177, 305, 156, 89, 318, 179, 324, 30, 13, 173, 245, 95, 161, 368, 182, 23, 55, 86, 25, 316, 67, 367, 315, 218, 69, 321, 27, 377, 18, 62, 45, 341, 343, 253, 16, 353, 313, 58, 19, 290, 134, 247, 94, 382, 81, 277, 145, 28, 251, 347, 127, 153, 42, 198, 238, 237, 90, 65, 286, 381, 246, 328, 104, 261, 167, 302, 364, 222, 111, 86, 235, 316, 181, 32, 32, 361, 232, 110, 214, 26, 129, 231, 0, 193, 22, 130, 121, 6, 275, 380, 375, 183, 379, 169, 180, 371, 92, 160, 205, 172, 278, 369, 191, 288, 124, 349, 248, 330, 148, 98, 291, 10, 201, 8, 360, 120, 221, 317, 44, 283, 136, 86, 268, 299, 229, 38, 315, 262, 69, 373, 27, 9, 3, 338, 192, 114, 314, 270, 159, 363, 96, 71, 203, 348, 102, 189, 250, 234, 378, 163, 255, 223, 113, 236, 337, 281, 351, 200, 12, 383, 171, 162, 147, 101, 76, 157, 216, 266, 260, 118, 307, 264, 365] + 51: [322, 241, 37, 101, 323, 186, 113, 102, 127, 72, 67, 229, 286, 260, 304, 177, 243, 5, 8, 294, 47, 382, 172, 279, 42, 351, 90, 33, 10, 239, 324, 171, 332, 315, 89, 30, 242, 55, 378, 43, 184, 96, 215, 14, 305, 227, 71, 266, 347, 173, 2, 290, 108, 241, 371, 6, 323, 186, 12, 52, 180, 144, 82, 163, 152, 301, 250, 283, 49, 92, 259, 118, 292, 97, 86, 198, 222, 192, 204, 368, 44, 224, 306, 369, 383, 45, 51, 288, 145, 340, 61, 167, 218, 230, 296, 88, 174, 355, 175, 274, 231, 377, 139, 202, 31, 221, 372, 206, 98, 16, 208, 17, 342, 264, 358, 163, 286, 124, 325, 168, 246, 247, 13, 20, 331, 308, 237, 56, 329, 310, 91, 252, 75, 48, 320, 313, 100, 312, 170, 191, 78, 93, 109, 112, 149, 57, 28, 165, 77, 217, 185, 63, 225, 76, 255, 291, 137, 189, 372, 200, 24, 208, 12, 182, 228, 136, 126, 343, 104, 203, 105, 153, 130, 26, 46, 23, 285, 81, 74, 380, 18, 190, 119, 309, 3, 164, 205, 311, 115, 349, 223, 338, 319, 333, 336, 316, 68, 66, 4, 199, 277, 133, 293, 117, 36, 256, 151, 261, 137, 73, 372, 110, 111, 186, 193, 102, 122, 64, 126, 62, 60, 32, 188, 253, 269, 263, 35, 85, 212, 344, 142, 107, 219, 280, 248, 321, 362, 236, 364, 317, 213, 381, 379, 80, 307, 159, 58, 232, 116, 148, 354, 201, 154, 70, 169, 352, 38, 267, 138, 156, 373, 106, 210, 200, 24, 357, 287, 102, 356, 144, 82, 229, 87, 235, 330, 146, 346, 134, 143, 374, 271, 161, 254, 268, 209, 181, 370, 251, 195, 361, 69, 289, 258, 297, 350, 194, 298, 363, 53, 238, 15, 214, 39, 272, 128, 367, 179, 29, 22, 131, 121, 249, 373, 221, 37, 110, 24, 186, 123, 17, 342, 337, 211, 65, 270, 40, 129, 295, 11, 314, 299, 281, 183, 303, 341, 359, 366, 59, 207, 166, 141, 234, 157, 262, 94, 365, 54, 328, 278, 273, 197, 25, 125, 83, 7, 257, 120, 376, 302, 318, 244, 155, 327, 326, 375, 375, 210, 178, 24, 99, 287, 284, 176, 337, 300, 140, 135, 95, 233, 282, 103, 335, 216, 220, 245, 275, 345, 162, 360, 79, 34, 160, 50, 41, 265, 84, 196, 334, 27, 114, 276, 353, 339, 240, 19, 1, 9, 158, 0, 132, 187, 147, 150, 21, 226, 348] + 52: [52, 107, 381, 267, 31, 71, 104, 97, 323, 174, 169, 298, 78, 68, 80, 259, 349, 236, 88, 312, 222, 234, 341, 160, 233, 6, 113, 351, 374, 337, 306, 308, 89, 242, 282, 84, 220, 129, 188, 118, 344, 313, 229, 302, 145, 255, 65, 122, 240, 360, 162, 348, 186, 107, 283, 99, 130, 246, 104, 165, 289, 195, 48, 82, 13, 60, 10, 128, 54, 354, 35, 123, 285, 262, 296, 223, 367, 300, 72, 17, 170, 324, 109, 119, 270, 200, 365, 207, 230, 330, 101, 380, 18, 126, 304, 275, 158, 42, 273, 241, 1, 376, 46, 175, 327, 216, 381, 99, 334, 151, 103, 137, 239, 206, 209, 163, 205, 172, 357, 269, 87, 95, 291, 379, 127, 166, 315, 215, 116, 280, 3, 115, 22, 91, 125, 235, 57, 258, 148, 194, 47, 198, 49, 141, 43, 299, 135, 232, 261, 79, 11, 366, 254, 94, 218, 375, 121, 61, 381, 99, 334, 157, 362, 165, 85, 239, 48, 25, 382, 346, 260, 219, 266, 64, 257, 133, 339, 105, 178, 23, 364, 67, 328, 154, 279, 208, 45, 225, 2, 277, 244, 21, 139, 347, 303, 197, 111, 96, 358, 363, 9, 185, 202, 144, 373, 264, 212, 180, 92, 332, 107, 99, 334, 157, 103, 137, 290, 345, 263, 90, 183, 62, 245, 192, 210, 149, 70, 171, 74, 191, 227, 124, 29, 369, 76, 53, 213, 333, 326, 268, 58, 110, 59, 152, 318, 249, 143, 342, 114, 4, 353, 276, 297, 203, 159, 142, 317, 199, 204, 176, 102, 307, 217, 320, 182, 71, 165, 73, 336, 271, 193, 25, 7, 33, 140, 196, 32, 50, 37, 228, 221, 286, 281, 173, 134, 146, 301, 331, 316, 55, 253, 350, 237, 284, 224, 251, 150, 368, 156, 278, 81, 335, 138, 274, 329, 131, 12, 0, 24, 164, 106, 38, 102, 120, 372, 371, 63, 71, 104, 97, 214, 34, 211, 184, 41, 190, 26, 168, 355, 8, 325, 155, 378, 5, 83, 30, 248, 292, 361, 319, 39, 287, 40, 93, 201, 177, 294, 189, 179, 100, 356, 322, 226, 311, 181, 112, 36, 187, 19, 15, 86, 272, 370, 247, 102, 231, 283, 381, 334, 157, 250, 97, 336, 321, 209, 77, 205, 288, 44, 56, 252, 98, 309, 108, 256, 117, 359, 167, 75, 243, 153, 377, 69, 66, 27, 161, 352, 383, 20, 238, 51, 136, 28, 310, 340, 343, 293, 147, 132, 314, 265, 338, 14, 295, 16, 305] + 53: [230, 110, 317, 247, 287, 38, 16, 292, 295, 179, 32, 186, 360, 291, 376, 119, 125, 24, 200, 62, 341, 183, 313, 49, 10, 72, 353, 357, 128, 339, 174, 57, 30, 333, 220, 232, 275, 21, 14, 283, 177, 298, 74, 34, 93, 288, 181, 204, 11, 307, 90, 84, 336, 110, 66, 73, 23, 296, 109, 46, 214, 345, 118, 266, 180, 309, 26, 121, 134, 326, 168, 329, 69, 231, 37, 197, 235, 15, 122, 146, 35, 120, 65, 371, 28, 136, 251, 55, 258, 7, 53, 380, 82, 310, 106, 366, 267, 356, 98, 286, 344, 290, 56, 185, 308, 110, 222, 73, 334, 320, 164, 365, 273, 250, 227, 302, 160, 268, 225, 285, 368, 319, 88, 12, 346, 77, 152, 277, 60, 159, 44, 359, 111, 216, 276, 327, 236, 40, 138, 59, 45, 108, 137, 249, 199, 1, 147, 184, 104, 161, 343, 274, 191, 223, 22, 114, 217, 110, 203, 73, 23, 296, 109, 354, 347, 250, 42, 228, 279, 351, 332, 130, 284, 171, 325, 94, 361, 383, 335, 300, 259, 89, 102, 143, 340, 206, 321, 370, 278, 123, 198, 242, 54, 154, 367, 43, 331, 85, 64, 229, 135, 148, 212, 187, 304, 314, 140, 316, 75, 110, 243, 270, 287, 238, 16, 322, 295, 330, 4, 186, 6, 157, 162, 81, 107, 253, 219, 262, 141, 116, 151, 52, 149, 126, 293, 67, 248, 196, 105, 189, 215, 255, 364, 101, 195, 201, 80, 305, 48, 63, 301, 163, 208, 41, 190, 124, 71, 245, 18, 312, 8, 110, 297, 317, 334, 129, 16, 280, 363, 330, 4, 92, 209, 194, 237, 221, 31, 269, 3, 25, 381, 188, 172, 373, 150, 349, 282, 226, 233, 178, 166, 158, 19, 358, 182, 294, 337, 352, 246, 9, 100, 342, 17, 113, 374, 91, 192, 165, 99, 58, 265, 272, 68, 240, 110, 13, 350, 38, 173, 311, 311, 379, 32, 170, 156, 86, 261, 103, 50, 127, 5, 256, 338, 112, 324, 78, 176, 76, 315, 362, 369, 2, 375, 96, 306, 133, 218, 254, 87, 257, 289, 323, 29, 51, 193, 303, 382, 211, 27, 213, 318, 244, 252, 210, 240, 240, 110, 13, 350, 38, 142, 354, 347, 224, 155, 132, 145, 202, 144, 271, 20, 95, 241, 139, 70, 205, 239, 378, 47, 281, 175, 328, 36, 115, 169, 207, 117, 79, 299, 97, 0, 234, 131, 264, 83, 39, 348, 372, 260, 263, 61, 33, 355, 377, 153, 167] + 54: [295, 363, 71, 319, 356, 223, 231, 260, 38, 299, 172, 254, 282, 330, 118, 3, 10, 375, 99, 197, 225, 91, 190, 382, 103, 157, 131, 135, 298, 301, 107, 22, 146, 18, 166, 9, 192, 271, 234, 247, 31, 206, 248, 102, 285, 65, 379, 37, 126, 383, 205, 148, 370, 363, 253, 328, 11, 339, 216, 369, 38, 299, 8, 208, 44, 311, 314, 75, 121, 70, 32, 64, 357, 325, 92, 142, 381, 312, 97, 1, 278, 48, 326, 179, 96, 347, 316, 141, 238, 213, 308, 59, 211, 269, 359, 199, 374, 4, 28, 241, 78, 340, 13, 0, 370, 363, 294, 189, 11, 287, 150, 82, 250, 98, 95, 72, 187, 352, 40, 317, 58, 41, 47, 161, 198, 323, 152, 279, 125, 204, 302, 100, 171, 255, 327, 201, 62, 377, 43, 128, 229, 334, 346, 14, 29, 261, 362, 203, 360, 104, 134, 188, 165, 19, 106, 232, 370, 363, 294, 338, 267, 339, 231, 80, 348, 20, 306, 95, 221, 87, 117, 228, 27, 76, 196, 158, 277, 94, 272, 109, 239, 376, 368, 177, 246, 69, 111, 136, 167, 162, 23, 275, 251, 259, 15, 235, 367, 52, 89, 46, 149, 51, 350, 256, 193, 344, 160, 36, 370, 363, 294, 147, 258, 155, 257, 164, 348, 230, 53, 176, 290, 25, 26, 329, 101, 288, 218, 322, 84, 5, 34, 182, 170, 186, 365, 318, 144, 85, 163, 304, 81, 173, 262, 175, 139, 12, 110, 145, 249, 331, 66, 60, 116, 303, 77, 2, 297, 202, 127, 336, 284, 153, 283, 83, 39, 223, 155, 240, 240, 280, 286, 281, 345, 57, 7, 372, 156, 132, 115, 113, 50, 220, 265, 217, 184, 237, 55, 49, 264, 373, 137, 313, 353, 195, 273, 266, 194, 63, 219, 168, 274, 79, 108, 227, 154, 307, 138, 112, 342, 358, 354, 361, 284, 296, 283, 90, 42, 339, 155, 214, 133, 280, 286, 324, 120, 305, 180, 86, 332, 276, 333, 215, 335, 45, 321, 159, 140, 105, 355, 183, 73, 68, 293, 242, 364, 151, 244, 191, 243, 371, 6, 74, 236, 233, 263, 226, 35, 292, 169, 222, 207, 200, 123, 320, 337, 380, 363, 42, 258, 356, 164, 164, 30, 230, 174, 341, 93, 119, 309, 16, 181, 143, 310, 129, 124, 17, 185, 24, 54, 21, 315, 67, 61, 366, 122, 209, 33, 114, 300, 351, 245, 88, 224, 178, 289, 268, 343, 56, 270, 210, 291, 252, 349, 130, 212, 378] + 55: [56, 154, 235, 174, 146, 146, 370, 307, 381, 90, 121, 382, 29, 65, 291, 249, 8, 333, 312, 316, 189, 5, 348, 210, 217, 231, 79, 228, 263, 346, 358, 101, 314, 82, 1, 281, 219, 359, 118, 55, 120, 374, 13, 162, 298, 21, 362, 345, 18, 25, 250, 69, 56, 154, 355, 174, 11, 62, 165, 80, 6, 303, 78, 347, 180, 301, 106, 134, 3, 44, 221, 271, 337, 112, 14, 375, 260, 343, 240, 140, 332, 23, 254, 94, 22, 31, 232, 365, 335, 372, 201, 104, 58, 50, 177, 280, 295, 334, 32, 195, 167, 379, 285, 196, 326, 154, 81, 174, 20, 340, 89, 252, 267, 299, 48, 74, 363, 115, 199, 71, 12, 320, 300, 26, 239, 102, 276, 190, 251, 265, 306, 54, 19, 257, 52, 193, 313, 264, 103, 367, 253, 97, 35, 129, 147, 64, 261, 141, 169, 36, 75, 100, 351, 45, 96, 215, 113, 327, 256, 174, 7, 308, 116, 238, 381, 90, 247, 347, 70, 269, 341, 2, 114, 290, 213, 383, 233, 163, 352, 105, 329, 47, 42, 194, 241, 287, 255, 149, 296, 212, 237, 133, 208, 319, 209, 168, 161, 68, 139, 242, 191, 330, 266, 230, 24, 283, 222, 317, 72, 88, 81, 151, 234, 218, 39, 311, 227, 41, 211, 323, 180, 76, 49, 99, 176, 138, 368, 156, 344, 170, 325, 371, 236, 377, 178, 187, 184, 122, 157, 278, 117, 214, 342, 143, 373, 30, 321, 137, 315, 66, 248, 77, 364, 244, 57, 288, 229, 9, 310, 305, 159, 88, 270, 309, 11, 308, 339, 80, 328, 123, 353, 304, 29, 269, 200, 293, 279, 331, 366, 86, 259, 197, 284, 274, 27, 380, 286, 53, 322, 277, 128, 33, 87, 198, 61, 127, 173, 126, 38, 297, 107, 125, 166, 183, 262, 282, 110, 158, 148, 292, 186, 204, 202, 152, 43, 98, 20, 340, 116, 111, 192, 73, 273, 323, 130, 115, 40, 223, 179, 349, 216, 0, 172, 17, 95, 91, 378, 246, 360, 136, 185, 318, 207, 84, 294, 153, 160, 188, 15, 268, 336, 338, 354, 145, 324, 376, 92, 181, 108, 220, 131, 51, 144, 34, 202, 28, 154, 4, 98, 357, 182, 16, 123, 299, 48, 74, 130, 76, 132, 226, 272, 60, 164, 124, 206, 46, 350, 289, 224, 361, 67, 135, 119, 10, 171, 85, 63, 302, 258, 37, 225, 356, 93, 243, 275, 150, 203, 369, 155, 175, 142, 83, 59, 205, 245, 109] + 56: [279, 167, 260, 82, 240, 15, 116, 225, 106, 371, 304, 249, 333, 162, 39, 41, 77, 80, 192, 72, 344, 73, 40, 253, 92, 31, 374, 280, 50, 33, 123, 364, 122, 201, 180, 218, 310, 200, 330, 372, 144, 352, 204, 258, 272, 203, 59, 377, 171, 237, 224, 146, 100, 343, 260, 82, 220, 213, 163, 303, 241, 5, 147, 189, 251, 161, 316, 292, 327, 381, 244, 97, 197, 334, 160, 250, 21, 136, 269, 121, 110, 151, 227, 42, 378, 210, 183, 119, 228, 112, 319, 367, 49, 70, 24, 131, 234, 239, 7, 36, 194, 51, 229, 208, 177, 321, 265, 178, 240, 15, 107, 225, 340, 259, 9, 196, 125, 117, 284, 219, 66, 159, 101, 383, 205, 376, 38, 278, 181, 198, 166, 12, 11, 17, 270, 248, 190, 62, 257, 341, 375, 226, 175, 291, 170, 350, 287, 345, 358, 349, 326, 216, 25, 91, 301, 143, 231, 195, 44, 199, 81, 294, 155, 296, 340, 30, 147, 356, 306, 6, 99, 275, 329, 154, 207, 118, 103, 27, 285, 10, 105, 79, 179, 331, 1, 164, 141, 129, 115, 360, 54, 314, 293, 28, 379, 16, 29, 43, 113, 337, 325, 212, 261, 45, 380, 102, 109, 90, 231, 195, 44, 178, 240, 294, 88, 184, 259, 30, 156, 20, 187, 328, 71, 302, 362, 288, 168, 309, 0, 300, 148, 312, 305, 252, 230, 22, 34, 153, 282, 286, 221, 271, 298, 353, 347, 124, 274, 37, 67, 361, 93, 47, 382, 370, 335, 217, 111, 165, 74, 173, 231, 311, 265, 260, 220, 213, 98, 351, 241, 5, 304, 236, 264, 46, 320, 188, 332, 245, 63, 238, 246, 359, 281, 193, 135, 338, 268, 317, 138, 13, 127, 56, 95, 348, 96, 145, 290, 323, 57, 139, 64, 86, 152, 157, 256, 176, 52, 366, 247, 61, 76, 3, 231, 321, 242, 126, 220, 294, 53, 267, 106, 87, 75, 263, 363, 308, 140, 18, 19, 14, 283, 223, 369, 182, 69, 137, 206, 295, 23, 355, 133, 255, 209, 368, 339, 94, 55, 84, 172, 4, 232, 289, 266, 222, 373, 254, 108, 68, 336, 365, 174, 318, 85, 315, 8, 195, 242, 191, 240, 273, 186, 158, 322, 169, 75, 89, 104, 346, 48, 2, 83, 357, 150, 120, 233, 58, 342, 128, 324, 35, 354, 149, 214, 243, 185, 297, 313, 299, 134, 262, 132, 307, 276, 65, 78, 130, 60, 211, 26, 215, 32, 277, 114, 235, 142, 202] + 57: [157, 158, 133, 259, 165, 198, 326, 235, 359, 72, 320, 125, 267, 150, 127, 64, 35, 182, 338, 161, 322, 341, 113, 83, 183, 147, 380, 225, 2, 164, 199, 216, 288, 29, 41, 247, 135, 236, 0, 126, 260, 192, 48, 78, 360, 212, 52, 269, 181, 201, 56, 275, 231, 31, 254, 295, 246, 187, 104, 148, 331, 274, 130, 344, 74, 154, 151, 79, 123, 238, 319, 43, 134, 214, 27, 20, 111, 116, 55, 119, 166, 171, 190, 296, 215, 371, 261, 97, 122, 59, 307, 1, 109, 374, 138, 61, 142, 291, 340, 193, 69, 15, 367, 343, 231, 381, 254, 146, 165, 140, 326, 205, 108, 94, 320, 125, 217, 154, 197, 22, 306, 347, 144, 316, 243, 70, 152, 298, 377, 289, 47, 325, 136, 71, 351, 23, 81, 17, 175, 363, 211, 265, 4, 188, 36, 222, 239, 273, 10, 218, 221, 278, 245, 229, 202, 304, 93, 276, 314, 220, 89, 156, 25, 65, 257, 94, 137, 141, 74, 204, 317, 345, 163, 251, 91, 179, 176, 9, 42, 191, 382, 38, 358, 13, 329, 352, 321, 155, 178, 334, 332, 145, 303, 248, 378, 219, 66, 361, 76, 103, 310, 114, 206, 195, 77, 34, 285, 301, 11, 276, 254, 259, 246, 187, 25, 101, 207, 286, 170, 335, 60, 174, 84, 88, 44, 354, 58, 365, 213, 51, 177, 46, 184, 92, 12, 328, 226, 149, 353, 362, 98, 311, 240, 49, 33, 255, 87, 168, 383, 256, 370, 139, 271, 258, 327, 292, 37, 153, 86, 376, 342, 276, 254, 295, 89, 63, 25, 128, 207, 286, 170, 8, 302, 6, 318, 160, 185, 272, 99, 253, 40, 228, 249, 356, 349, 203, 120, 112, 68, 196, 282, 369, 186, 18, 379, 355, 45, 24, 270, 333, 224, 375, 67, 308, 244, 100, 82, 189, 305, 233, 143, 372, 159, 230, 254, 121, 57, 7, 326, 148, 331, 124, 324, 118, 302, 169, 283, 180, 75, 80, 39, 373, 346, 73, 173, 277, 102, 54, 106, 279, 210, 313, 337, 90, 132, 323, 290, 315, 252, 200, 32, 357, 50, 194, 232, 107, 264, 339, 129, 336, 131, 280, 234, 262, 14, 14, 16, 146, 85, 368, 366, 205, 223, 72, 53, 268, 300, 350, 117, 309, 62, 250, 167, 172, 110, 297, 96, 312, 21, 208, 284, 95, 348, 3, 209, 28, 330, 162, 237, 26, 5, 299, 281, 241, 263, 242, 364, 287, 266, 115, 294, 30, 227, 105, 19, 293] + 58: [263, 377, 126, 56, 259, 348, 245, 373, 322, 298, 358, 270, 36, 152, 47, 119, 315, 60, 66, 171, 139, 55, 207, 99, 238, 124, 183, 63, 241, 154, 65, 345, 182, 8, 237, 243, 294, 95, 359, 254, 374, 269, 120, 159, 301, 217, 218, 165, 252, 370, 364, 308, 248, 377, 76, 126, 366, 27, 98, 194, 299, 9, 114, 169, 40, 262, 342, 189, 130, 4, 352, 285, 187, 239, 25, 129, 306, 333, 121, 131, 300, 288, 327, 43, 265, 50, 136, 64, 5, 142, 221, 163, 146, 84, 39, 32, 78, 267, 91, 89, 279, 125, 150, 38, 156, 377, 377, 61, 205, 92, 245, 222, 49, 312, 351, 256, 116, 68, 141, 272, 12, 316, 274, 329, 229, 381, 82, 258, 83, 234, 37, 135, 362, 176, 107, 228, 293, 353, 105, 338, 138, 360, 268, 282, 290, 223, 109, 145, 193, 74, 310, 320, 164, 321, 184, 134, 156, 377, 377, 79, 305, 356, 271, 311, 93, 298, 72, 214, 175, 196, 85, 42, 132, 162, 185, 328, 192, 202, 143, 325, 90, 326, 382, 332, 253, 357, 334, 11, 70, 225, 230, 170, 24, 29, 330, 157, 337, 247, 117, 153, 286, 174, 140, 23, 166, 335, 280, 209, 220, 275, 249, 81, 205, 92, 14, 222, 177, 22, 289, 355, 232, 324, 367, 16, 97, 260, 197, 224, 48, 331, 59, 181, 53, 227, 378, 302, 208, 148, 278, 347, 287, 179, 94, 180, 296, 317, 54, 276, 200, 167, 57, 34, 88, 365, 361, 233, 292, 319, 168, 58, 220, 275, 151, 81, 273, 210, 211, 194, 235, 122, 114, 199, 368, 376, 6, 203, 111, 20, 190, 28, 1, 375, 62, 266, 155, 160, 346, 110, 144, 186, 123, 2, 19, 195, 173, 118, 102, 372, 86, 303, 103, 51, 96, 137, 15, 30, 323, 212, 147, 309, 201, 204, 220, 77, 377, 81, 305, 7, 98, 311, 206, 312, 67, 26, 112, 295, 18, 191, 277, 106, 242, 198, 281, 314, 244, 350, 236, 339, 75, 149, 13, 379, 318, 21, 113, 35, 133, 297, 261, 100, 264, 41, 336, 349, 246, 304, 52, 188, 363, 291, 80, 71, 108, 33, 220, 377, 377, 56, 366, 344, 245, 101, 177, 122, 69, 257, 178, 45, 341, 31, 104, 255, 307, 73, 380, 251, 383, 284, 215, 250, 240, 0, 172, 231, 161, 371, 115, 283, 219, 226, 213, 87, 128, 354, 44, 46, 369, 216, 17, 127, 343, 313, 158, 3, 10, 340] + 59: [332, 285, 170, 170, 229, 7, 260, 173, 173, 66, 352, 31, 172, 328, 23, 266, 290, 278, 12, 35, 302, 383, 60, 218, 47, 179, 180, 92, 13, 131, 96, 107, 71, 43, 136, 322, 74, 145, 3, 115, 148, 284, 303, 98, 318, 219, 272, 367, 190, 116, 90, 154, 140, 30, 25, 250, 229, 370, 195, 95, 220, 371, 221, 150, 300, 9, 26, 232, 133, 109, 304, 159, 334, 62, 225, 29, 54, 69, 344, 354, 289, 51, 61, 0, 287, 110, 129, 84, 67, 49, 261, 249, 273, 307, 165, 319, 305, 270, 171, 378, 345, 351, 308, 341, 140, 105, 25, 256, 226, 370, 195, 70, 220, 294, 346, 331, 295, 38, 262, 309, 342, 175, 161, 16, 181, 252, 373, 142, 214, 222, 137, 259, 155, 2, 242, 206, 380, 135, 325, 253, 330, 89, 336, 11, 357, 102, 216, 8, 276, 14, 335, 78, 126, 323, 15, 79, 73, 33, 224, 197, 147, 7, 202, 363, 381, 66, 234, 130, 65, 317, 151, 281, 176, 212, 160, 297, 267, 162, 376, 286, 306, 339, 369, 177, 365, 349, 199, 362, 227, 246, 187, 217, 21, 93, 277, 275, 321, 241, 358, 316, 239, 360, 77, 244, 104, 99, 111, 215, 209, 327, 46, 313, 379, 94, 59, 200, 340, 204, 352, 76, 65, 377, 178, 382, 112, 45, 85, 279, 100, 320, 143, 338, 10, 355, 348, 238, 337, 68, 156, 299, 27, 64, 184, 124, 207, 37, 88, 191, 201, 103, 312, 237, 141, 350, 293, 205, 83, 52, 4, 188, 209, 327, 25, 250, 229, 5, 202, 251, 311, 247, 234, 185, 265, 324, 243, 255, 86, 91, 81, 127, 211, 169, 22, 174, 356, 258, 146, 366, 353, 101, 274, 48, 138, 63, 50, 157, 28, 153, 210, 106, 118, 230, 283, 108, 282, 56, 39, 264, 139, 72, 310, 189, 113, 327, 224, 97, 379, 94, 59, 200, 340, 263, 346, 18, 42, 372, 144, 196, 114, 223, 57, 168, 364, 248, 315, 19, 329, 120, 269, 361, 6, 228, 292, 32, 333, 41, 34, 231, 343, 326, 44, 24, 53, 296, 368, 301, 240, 192, 375, 87, 152, 167, 235, 268, 182, 327, 75, 170, 233, 7, 260, 363, 298, 374, 291, 288, 213, 186, 80, 58, 128, 164, 40, 254, 280, 163, 20, 245, 236, 158, 198, 203, 36, 123, 314, 194, 193, 149, 208, 134, 17, 359, 132, 183, 119, 166, 82, 257, 1, 125, 121, 347, 55, 122, 117, 271] + 60: [284, 381, 335, 158, 128, 129, 139, 133, 199, 265, 153, 302, 2, 49, 119, 354, 272, 184, 82, 359, 152, 20, 6, 1, 332, 382, 274, 304, 171, 107, 85, 200, 341, 370, 256, 18, 315, 67, 202, 330, 54, 198, 229, 298, 159, 43, 173, 103, 44, 218, 317, 355, 102, 268, 351, 124, 115, 266, 244, 39, 40, 40, 230, 181, 327, 35, 112, 311, 245, 223, 263, 169, 189, 58, 269, 240, 80, 267, 176, 116, 141, 70, 285, 146, 100, 106, 219, 14, 338, 105, 13, 252, 261, 50, 374, 29, 86, 17, 292, 32, 352, 136, 62, 207, 94, 268, 351, 124, 260, 303, 139, 83, 199, 310, 194, 208, 205, 234, 180, 193, 232, 231, 27, 246, 0, 283, 93, 36, 344, 110, 38, 342, 186, 360, 293, 33, 192, 79, 296, 148, 125, 282, 45, 178, 320, 224, 379, 97, 165, 135, 227, 177, 101, 226, 63, 348, 123, 313, 351, 257, 16, 47, 60, 133, 339, 310, 247, 81, 366, 76, 353, 15, 213, 59, 238, 364, 151, 255, 114, 217, 11, 237, 57, 8, 21, 52, 187, 196, 182, 95, 380, 371, 383, 376, 361, 287, 179, 145, 211, 111, 72, 326, 367, 4, 73, 89, 210, 22, 123, 313, 124, 257, 16, 220, 347, 126, 222, 174, 120, 297, 34, 294, 46, 329, 131, 143, 271, 140, 75, 277, 37, 183, 24, 9, 215, 56, 323, 90, 66, 7, 369, 314, 276, 64, 197, 156, 262, 328, 258, 280, 356, 291, 175, 281, 71, 295, 118, 242, 154, 299, 123, 313, 124, 257, 16, 47, 60, 113, 212, 174, 312, 185, 28, 337, 166, 309, 278, 161, 235, 172, 336, 248, 221, 51, 92, 264, 61, 305, 134, 53, 273, 30, 190, 349, 368, 331, 325, 168, 23, 209, 19, 41, 160, 147, 191, 306, 289, 275, 321, 308, 157, 288, 236, 313, 351, 124, 195, 220, 343, 83, 316, 265, 65, 334, 239, 108, 365, 345, 122, 117, 55, 253, 84, 362, 358, 318, 188, 162, 377, 25, 26, 142, 91, 149, 130, 357, 286, 12, 363, 270, 251, 228, 137, 104, 204, 96, 301, 254, 164, 340, 109, 87, 214, 375, 121, 313, 381, 158, 170, 243, 372, 126, 48, 78, 167, 155, 74, 319, 290, 98, 206, 203, 241, 99, 150, 69, 132, 259, 88, 5, 3, 279, 233, 346, 31, 250, 300, 322, 144, 225, 333, 307, 127, 324, 216, 42, 68, 378, 249, 350, 163, 138, 10, 77, 373, 201] + 61: [231, 211, 221, 179, 326, 203, 270, 381, 218, 167, 51, 238, 263, 86, 107, 288, 177, 160, 169, 191, 348, 114, 135, 379, 2, 3, 200, 290, 89, 292, 341, 80, 141, 340, 332, 98, 70, 54, 18, 364, 232, 314, 180, 140, 256, 277, 279, 183, 320, 296, 189, 149, 303, 93, 120, 338, 127, 299, 19, 381, 36, 21, 371, 324, 143, 161, 88, 57, 369, 282, 32, 87, 65, 29, 300, 304, 109, 328, 52, 378, 383, 273, 280, 126, 85, 132, 14, 342, 274, 91, 187, 99, 24, 185, 182, 223, 347, 285, 131, 210, 325, 334, 356, 319, 230, 93, 211, 338, 127, 323, 241, 45, 36, 195, 278, 324, 55, 237, 88, 115, 188, 156, 227, 361, 362, 366, 83, 252, 260, 206, 147, 173, 72, 306, 8, 365, 240, 81, 194, 359, 244, 63, 354, 251, 7, 13, 28, 146, 318, 64, 254, 50, 367, 350, 116, 58, 172, 360, 221, 179, 247, 178, 184, 380, 103, 315, 68, 294, 248, 307, 112, 168, 286, 181, 43, 337, 77, 90, 27, 11, 47, 245, 202, 249, 137, 346, 37, 79, 283, 71, 199, 298, 193, 352, 6, 123, 262, 305, 95, 105, 35, 377, 39, 289, 41, 128, 343, 0, 61, 335, 221, 338, 127, 122, 59, 197, 253, 330, 344, 198, 213, 119, 327, 129, 34, 31, 261, 228, 150, 233, 190, 374, 329, 23, 97, 33, 186, 30, 275, 125, 225, 317, 284, 204, 162, 163, 301, 101, 49, 201, 164, 339, 220, 56, 175, 134, 229, 16, 321, 94, 61, 108, 157, 179, 127, 299, 10, 363, 353, 12, 17, 142, 248, 66, 192, 333, 42, 15, 133, 258, 118, 84, 44, 113, 336, 368, 207, 259, 322, 214, 375, 242, 196, 26, 144, 152, 311, 271, 239, 358, 111, 293, 212, 48, 222, 60, 281, 62, 76, 357, 96, 208, 73, 5, 170, 46, 351, 117, 209, 1, 100, 167, 344, 235, 307, 161, 216, 291, 110, 243, 75, 92, 297, 316, 82, 121, 308, 145, 104, 139, 154, 78, 174, 219, 67, 295, 124, 148, 370, 25, 310, 74, 312, 272, 373, 236, 250, 226, 355, 382, 376, 40, 171, 155, 73, 136, 221, 46, 165, 205, 264, 197, 100, 315, 372, 215, 263, 86, 107, 216, 159, 349, 309, 20, 224, 268, 166, 313, 257, 69, 158, 102, 38, 153, 9, 22, 269, 4, 176, 106, 345, 130, 255, 234, 331, 246, 138, 265, 276, 302, 151, 267, 217, 266, 53, 287] +layer_updates_per_iter: 0 +num_slots: 416 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 4be1d01c7e20..e7c09700e588 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 @@ -90,6 +91,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 23d0854324d2..063398ec2f7d 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -88,5 +89,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml deleted file mode 100644 index 74801acda6f8..000000000000 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,93 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 128k8k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '64' - input_length: 131072 - output_length: 8192 - dataset_file: datasets/perf-ci/deepseek_r1-128k8k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - print_iter_log: true - max_batch_size: 2 - max_num_tokens: 8 - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 2 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 131104 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - max_draft_len: 3 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 1 - max_num_tokens: 131104 - tensor_parallel_size: 1 - moe_expert_parallel_size: 1 - pipeline_parallel_size: 8 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.3 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 131104 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml index e57ea6d424e8..533d9445a568 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml @@ -81,6 +81,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 4608 backend: NIXL + transceiver_runtime: PYTHON stream_interval: 20 ctx: print_iter_log: true @@ -101,3 +102,4 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 4608 backend: NIXL + transceiver_runtime: PYTHON diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 50c56d99eb38..729c978ff57d 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index c3ff2770bcb5..ad82f32ada02 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -66,6 +66,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON stream_interval: 100 num_postprocess_workers: 4 ctx: @@ -90,3 +91,4 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index e13f5348ad24..b7dd65d335fe 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml index 5cc8664e6dcb..4cc735c9526a 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml index 3e06cdd34340..88cab7edce8a 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml @@ -66,6 +66,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -93,5 +94,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index a1f1b7507d3b..c87531b8119a 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 7ab651ae8e5c..01f93de49256 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 08d59450f353..07c8eb6f8b09 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 51a7a78e16cc..efa3a15fcdf6 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index 2ffed4acf11e..2c5f64b26c28 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -67,6 +67,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 13763203780c..4e77197a1a10 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -62,6 +62,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -97,5 +98,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 927657daa04e..812fab472093 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -64,6 +64,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -98,5 +99,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index eb93763ec738..28b6dbed2efa 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -65,6 +65,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -102,5 +103,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index d17172e8d66d..f973d98b114a 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -69,6 +69,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 @@ -106,6 +107,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 07fbb43780e9..53ba74fff572 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -66,6 +66,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -102,5 +103,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 63f2ef1ae8f9..37241b7b94cd 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -40,7 +40,7 @@ environment: # Env vars taken from the Dynamo deploy.yaml (NCCL MNNVL/CUMEM/NVLS, UCX MNNVL # IPC, MoE all-to-all without all-gather, PDL, GC disable) so the perf test # exercises the same code paths as the production deployment. - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=1 NVIDIA_GDRCOPY=1 UCX_CUDA_IPC_ENABLE_MNNVL=1 NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER=1 TRTLLM_FORCE_COMM_METHOD=NVLINK_TWO_SIDED ENABLE_CONFIGURABLE_MOE=1" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=1 NVIDIA_GDRCOPY=1 UCX_CUDA_IPC_ENABLE_MNNVL=1 NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER=1 TRTLLM_FORCE_COMM_METHOD=NVLINK_TWO_SIDED" server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false @@ -52,13 +52,13 @@ worker_config: pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: true - max_batch_size: 8 + max_batch_size: 32 max_num_tokens: 8192 max_seq_len: 121000 allreduce_strategy: MNNVL cuda_graph_config: enable_padding: true - max_batch_size: 8 + max_batch_size: 32 kv_cache_config: enable_block_reuse: false free_gpu_memory_fraction: 0.9 @@ -70,6 +70,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + transceiver_runtime: PYTHON num_postprocess_workers: 8 stream_interval: 10 ctx: @@ -97,3 +98,4 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + transceiver_runtime: PYTHON diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index ef23747a7ac6..f292dbb71ddd 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -67,6 +67,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 883b280a28bd..784b8d798097 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -62,6 +62,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -97,5 +98,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml index 4691eb174302..6b60f15cc1fa 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml @@ -67,6 +67,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -98,4 +99,5 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml similarity index 54% rename from tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml rename to tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml index 20a89f47b3c7..0517cb17b0e5 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml @@ -1,7 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + metadata: - model_name: gpt_oss_120b_fp4 - precision: fp4 - model_dir_name: GPT-OSS-120B-FP4 + model_name: gemma_3_1b_it + precision: bf16 + model_dir_name: gemma-3-1b-it supported_gpus: - GB200 script_file: disaggr_torch.slurm @@ -20,10 +35,10 @@ benchmark: multi_round: 10 benchmark_ratio: 0.0 streaming: true - concurrency_list: '2048' + concurrency_list: '256' input_length: 1024 output_length: 1024 - dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json + dataset_file: hardware: gpus_per_node: 4 num_ctx_servers: 1 @@ -35,56 +50,40 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true - max_batch_size: 1536 - max_num_tokens: 20000 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 + max_batch_size: 256 + max_num_tokens: 512 + tensor_parallel_size: 1 + moe_expert_parallel_size: 1 pipeline_parallel_size: 1 context_parallel_size: 1 - enable_attention_dp: true - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 + enable_attention_dp: false cuda_graph_config: enable_padding: true - max_batch_size: 1536 + max_batch_size: 256 kv_cache_config: use_kv_cache_manager_v2: false enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: TRTLLM + free_gpu_memory_fraction: 0.85 + dtype: auto + max_attention_window: [512, 512, 512, 512, 512, 32768] cache_transceiver_config: - max_tokens_in_buffer: 1024 + max_tokens_in_buffer: 4096 backend: NIXL transceiver_runtime: PYTHON - disable_overlap_scheduler: true + disable_overlap_scheduler: false num_postprocess_workers: 4 stream_interval: 20 ctx: print_iter_log: true max_batch_size: 32 - max_num_tokens: 20000 + max_num_tokens: 8192 tensor_parallel_size: 1 moe_expert_parallel_size: 1 pipeline_parallel_size: 1 @@ -94,13 +93,11 @@ worker_config: kv_cache_config: use_kv_cache_manager_v2: false enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true + free_gpu_memory_fraction: 0.85 + dtype: auto + max_attention_window: [512, 512, 512, 512, 512, 32768] cache_transceiver_config: - max_tokens_in_buffer: 1024 + max_tokens_in_buffer: 4096 backend: NIXL transceiver_runtime: PYTHON disable_overlap_scheduler: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml index 6678fdfc4f8b..9a514f62dc65 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml index a54e7d0335cf..4b01710438c2 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -66,6 +66,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -93,5 +94,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index e8bcff454293..1b427f4f2c05 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml index 37d162efda13..9dc48b40047f 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -66,6 +66,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -93,5 +94,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 3761761e4e9d..d9fef4a05711 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 876121e49a57..6f95b245621a 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 9c82b55a4a17..53736a99c780 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: max_tokens_in_buffer: 1024 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -88,4 +89,5 @@ worker_config: max_tokens_in_buffer: 1024 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 9f4b7086060d..ffa8cdfa4522 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -75,6 +75,7 @@ worker_config: max_tokens_in_buffer: 9216 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: false num_postprocess_workers: 4 stream_interval: 20 @@ -103,6 +104,7 @@ worker_config: max_tokens_in_buffer: 9216 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index c9f73573fcf9..3f00a0063523 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -88,4 +89,5 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 9610d4a5bdf9..edca66a96d87 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -88,4 +89,5 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_llama-3.1-8b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml similarity index 60% rename from tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml rename to tests/scripts/perf-sanity/disaggregated/gb200_llama-3.1-8b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml index 020e476bf118..43bc0a1b8196 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_llama-3.1-8b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml @@ -1,7 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + metadata: - model_name: gpt_oss_120b_fp4 - precision: fp4 - model_dir_name: GPT-OSS-120B-FP4 + model_name: llama_v3.1_8b_instruct + precision: bf16 + model_dir_name: Llama-3.1-8B-Instruct supported_gpus: - GB200 script_file: disaggr_torch.slurm @@ -20,10 +35,10 @@ benchmark: multi_round: 10 benchmark_ratio: 0.0 streaming: true - concurrency_list: '64' + concurrency_list: '256' input_length: 1024 output_length: 1024 - dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json + dataset_file: hardware: gpus_per_node: 4 num_ctx_servers: 1 @@ -35,27 +50,16 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true max_batch_size: 256 - max_num_tokens: 20000 - tensor_parallel_size: 4 + max_num_tokens: 512 + tensor_parallel_size: 1 moe_expert_parallel_size: 1 pipeline_parallel_size: 1 context_parallel_size: 1 @@ -66,21 +70,19 @@ worker_config: kv_cache_config: use_kv_cache_manager_v2: false enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: TRTLLM + free_gpu_memory_fraction: 0.85 + dtype: auto cache_transceiver_config: - max_tokens_in_buffer: 1024 + max_tokens_in_buffer: 4096 backend: NIXL transceiver_runtime: PYTHON - disable_overlap_scheduler: true + disable_overlap_scheduler: false num_postprocess_workers: 4 stream_interval: 20 ctx: print_iter_log: true max_batch_size: 32 - max_num_tokens: 20000 + max_num_tokens: 8192 tensor_parallel_size: 1 moe_expert_parallel_size: 1 pipeline_parallel_size: 1 @@ -90,13 +92,10 @@ worker_config: kv_cache_config: use_kv_cache_manager_v2: false enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true + free_gpu_memory_fraction: 0.85 + dtype: auto cache_transceiver_config: - max_tokens_in_buffer: 1024 + max_tokens_in_buffer: 4096 backend: NIXL transceiver_runtime: PYTHON disable_overlap_scheduler: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index cecdab72aafd..c15b39b591b8 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -62,6 +62,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 795a6192bf41..92e9e36230a3 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -17,7 +17,7 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 2 + multi_round: 3 benchmark_ratio: 0.0 streaming: true concurrency_list: '256' diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 5cc6c81a78b9..e347dce94e4d 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -64,6 +64,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 5d77929a60a7..d19cd5cbdb34 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 89458072b018..7fa8048b70af 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 1176fd8e5982..60153dcc3eab 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index a54762c3050c..e9a43f20bed3 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 446b7663f2f1..6ff78f0a198d 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index 11610c5a26ee..c86ee7f26338 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -67,6 +67,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 86b5e51260f7..352412b55b37 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -62,6 +62,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -97,5 +98,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml index 091c76a214e4..c2c097f2a989 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml @@ -64,6 +64,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -98,5 +99,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 699424bd083e..256db17b8621 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -65,6 +65,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -102,5 +103,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index b7a042d78e58..f13ca285af68 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -69,6 +69,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 @@ -106,6 +107,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 2919dec781b5..4f5f238cd289 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -66,6 +66,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -102,5 +103,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index f5fd13315d4f..06d0ba1ff9d3 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -40,7 +40,7 @@ environment: # Env vars taken from the Dynamo deploy.yaml (NCCL MNNVL/CUMEM/NVLS, UCX MNNVL # IPC, MoE all-to-all without all-gather, PDL, GC disable) so the perf test # exercises the same code paths as the production deployment. - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=1 NVIDIA_GDRCOPY=1 UCX_CUDA_IPC_ENABLE_MNNVL=1 NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER=1 TRTLLM_FORCE_COMM_METHOD=NVLINK_TWO_SIDED ENABLE_CONFIGURABLE_MOE=1" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=1 NVIDIA_GDRCOPY=1 UCX_CUDA_IPC_ENABLE_MNNVL=1 NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER=1 TRTLLM_FORCE_COMM_METHOD=NVLINK_TWO_SIDED" server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false @@ -70,6 +70,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 num_postprocess_workers: 8 stream_interval: 10 @@ -98,4 +99,5 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index 609ef00b6968..59f2e71a3daf 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -67,6 +67,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 2a119e6dcba7..1b6533231914 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -62,6 +62,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -97,5 +98,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml index 74efcde4fc8c..6a569f8d0371 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml @@ -67,6 +67,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -98,4 +99,5 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL.yaml similarity index 56% rename from tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL.yaml index f0c728ac2eec..36edcb022db4 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL.yaml @@ -1,11 +1,11 @@ metadata: - model_name: deepseek_r1_0528_fp4_v2 + model_name: deepseek_v4_pro_fp4 precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 + model_dir_name: DeepSeek-V4-Pro supported_gpus: - GB300 script_file: disaggr_torch.slurm - benchmark_type: 1k1k + benchmark_type: 8k1k slurm: script_file: disaggr_torch.slurm partition: @@ -20,13 +20,14 @@ benchmark: multi_round: 10 benchmark_ratio: 0.0 streaming: true - concurrency_list: '1024' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json + concurrency_list: '180' + input_length: 7408 + output_length: 922 + # Random synthetic dataset — see the con8 variant for rationale. + dataset_file: hardware: gpus_per_node: 4 - num_ctx_servers: 1 + num_ctx_servers: 3 num_gen_servers: 1 environment: container_mount: @@ -35,26 +36,17 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 "PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core" TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true - max_batch_size: 32 - max_num_tokens: 128 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 tensor_parallel_size: 32 moe_expert_parallel_size: 32 pipeline_parallel_size: 1 @@ -63,42 +55,51 @@ worker_config: enable_lm_head_tp_in_adp: true cuda_graph_config: enable_padding: true - max_batch_size: 32 + batch_sizes: [1, 2, 4] kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.7 dtype: fp8 + tokens_per_block: 128 moe_config: - backend: CUTEDSL + backend: MEGAMOE_DEEPGEMM use_low_precision_moe_combine: true + load_balancer: tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots384.yaml cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL - disable_overlap_scheduler: true + transceiver_runtime: PYTHON + disable_overlap_scheduler: false speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 - stream_interval: 20 + stream_interval: 100 ctx: print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 tensor_parallel_size: 4 moe_expert_parallel_size: 4 pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: true - enable_lm_head_tp_in_adp: true + enable_lm_head_tp_in_adp: false cuda_graph_config: null kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.5 dtype: fp8 + tokens_per_block: 128 moe_config: backend: TRTLLM + load_balancer: tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL.yaml new file mode 100644 index 000000000000..e35e6bb17fba --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL.yaml @@ -0,0 +1,172 @@ +metadata: + model_name: deepseek_v4_pro_fp4 + precision: fp4 + model_dir_name: DeepSeek-V4-Pro + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '4301' + input_length: 7392 + output_length: 922 + # Random synthetic dataset — see the con8 variant for rationale. + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 12 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 "PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core" TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 512 + max_num_tokens: 1024 + max_seq_len: 9256 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + load_balancer: tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots384.yaml + cache_transceiver_config: + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 + backend: NIXL + transceiver_runtime: PYTHON + disable_overlap_scheduler: false + speculative_config: &id001 + decoding_type: MTP + max_draft_len: 1 + num_postprocess_workers: 4 + stream_interval: 100 + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: TRTLLM + load_balancer: tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml + cache_transceiver_config: + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 + backend: NIXL + transceiver_runtime: PYTHON + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL.yaml similarity index 55% rename from tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL.yaml index 84334736ffeb..a0cc66e78cc9 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL.yaml @@ -1,11 +1,11 @@ metadata: - model_name: deepseek_r1_0528_fp4_v2 + model_name: deepseek_v4_pro_fp4 precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 + model_dir_name: DeepSeek-V4-Pro supported_gpus: - - GB200 + - GB300 script_file: disaggr_torch.slurm - benchmark_type: 1k1k + benchmark_type: 8k1k slurm: script_file: disaggr_torch.slurm partition: @@ -17,16 +17,17 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 5 + multi_round: 10 benchmark_ratio: 0.0 streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json + concurrency_list: '666' + input_length: 7386 + output_length: 922 + # Random synthetic dataset — see the con8 variant for rationale. + dataset_file: hardware: gpus_per_node: 4 - num_ctx_servers: 2 + num_ctx_servers: 6 num_gen_servers: 1 environment: container_mount: @@ -35,26 +36,17 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 "PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core" TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true - max_batch_size: 768 - max_num_tokens: 3072 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 tensor_parallel_size: 16 moe_expert_parallel_size: 16 pipeline_parallel_size: 1 @@ -63,43 +55,51 @@ worker_config: enable_lm_head_tp_in_adp: true cuda_graph_config: enable_padding: true - max_batch_size: 768 + batch_sizes: [1, 2, 4, 8, 16, 24, 32] kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.8 + free_gpu_memory_fraction: 0.7 dtype: fp8 + tokens_per_block: 128 moe_config: - backend: CUTEDSL + backend: MEGAMOE_DEEPGEMM use_low_precision_moe_combine: true + load_balancer: tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots384.yaml cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL - disable_overlap_scheduler: true + transceiver_runtime: PYTHON + disable_overlap_scheduler: false speculative_config: &id001 decoding_type: MTP max_draft_len: 3 num_postprocess_workers: 4 - stream_interval: 20 + stream_interval: 100 ctx: print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 tensor_parallel_size: 4 moe_expert_parallel_size: 4 pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: true - enable_lm_head_tp_in_adp: true + enable_lm_head_tp_in_adp: false cuda_graph_config: null kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.5 dtype: fp8 + tokens_per_block: 128 moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true + backend: TRTLLM + load_balancer: tests/scripts/perf-sanity/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL.yaml similarity index 56% rename from tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL.yaml index 5369f95e2555..b54449034d31 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -1,11 +1,11 @@ metadata: - model_name: deepseek_v32_fp4 + model_name: deepseek_v4_pro_fp4 precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 + model_dir_name: DeepSeek-V4-Pro supported_gpus: - GB300 script_file: disaggr_torch.slurm - benchmark_type: 1k1k + benchmark_type: 8k1k slurm: script_file: disaggr_torch.slurm partition: @@ -20,14 +20,17 @@ benchmark: multi_round: 10 benchmark_ratio: 0.0 streaming: true - concurrency_list: '1' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json + concurrency_list: '8' + input_length: 7456 + output_length: 923 + # Random synthetic dataset (no captured dataset file yet for DSv4-Pro; matches + # the GLM-5 perf-sanity pattern of leaving as a placeholder so + # test_perf_sanity.py falls through to the random-dataset benchmark path). + dataset_file: hardware: gpus_per_node: 4 num_ctx_servers: 1 - num_gen_servers: 1 + num_gen_servers: 4 environment: container_mount: container_image: @@ -35,78 +38,70 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 "PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core" TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + # ctx workers only: expandable_segments avoids CUDA fragmentation on the + # long-prompt ctx path. gen workers do not need it. + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true max_batch_size: 1 - max_num_tokens: 512 + max_num_tokens: 4 + max_seq_len: 9256 tensor_parallel_size: 8 moe_expert_parallel_size: 8 pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: false + enable_lm_head_tp_in_adp: false cuda_graph_config: enable_padding: true - max_batch_size: 1 + batch_sizes: [1, 2, 4] kv_cache_config: enable_block_reuse: false free_gpu_memory_fraction: 0.9 dtype: fp8 - tokens_per_block: 64 + tokens_per_block: 128 moe_config: backend: TRTLLM + use_low_precision_moe_combine: true cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL - disable_overlap_scheduler: true + transceiver_runtime: PYTHON + disable_overlap_scheduler: false speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - allreduce_strategy: MNNVL + stream_interval: 100 ctx: print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 tensor_parallel_size: 4 moe_expert_parallel_size: 4 pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: true - enable_lm_head_tp_in_adp: true + enable_lm_head_tp_in_adp: false cuda_graph_config: null kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.5 dtype: fp8 - tokens_per_block: 64 + tokens_per_block: 128 moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true + backend: TRTLLM cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml index 7e5aa487facc..e1f98578eda1 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml index f6334f686d1c..e2d50f587b7b 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -66,6 +66,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -93,5 +94,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 8ef0b3db0473..ebc92211e8ee 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml index 22a64298673b..228622bb77c6 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -66,6 +66,8 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -93,5 +95,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 1e6c43b2ddf7..67ae7c9d5c3a 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -61,6 +61,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 777987a01492..8a8ddd2f7e96 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml new file mode 100644 index 000000000000..293e803ea2fa --- /dev/null +++ b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [38, 95, 50, 47, 25, 15, 26, 53, 31, 9, 41, 71, 19, 4, 143, 108, 85, 44, 6, 8, 127, 66, 2, 33, 63, 17, 70, 112, 79, 96, 109, 21, 159, 186, 90, 114, 147, 169, 155, 136, 81, 103, 120, 234, 256, 37, 107, 223, 172, 175, 217, 238, 243, 206, 209, 174, 220, 271, 261, 307, 213, 273, 247, 258, 268, 231, 277, 255, 227, 242, 309, 280, 371, 299, 303, 305, 351, 313, 320, 329, 291, 244, 374, 340, 364, 276, 379, 228, 342, 250, 322, 368, 356, 373, 354, 380, 57, 51, 1, 35, 11, 52, 34, 7, 27, 56, 0, 76, 72, 74, 100, 163, 80, 102, 55, 97, 84, 59, 54, 78, 88, 134, 105, 142, 106, 92, 40, 62, 204, 222, 126, 111, 179, 158, 32, 173, 178, 118, 160, 210, 249, 282, 196, 190, 212, 157, 236, 191, 193, 208, 287, 253, 230, 199, 153, 165, 235, 293, 274, 286, 267, 200, 266, 265, 297, 224, 298, 285, 264, 296, 301, 300, 375, 319, 314, 284, 376, 292, 281, 383, 270, 336, 345, 316, 365, 321, 331, 348, 346, 370, 382, 355, 259, 339, 3, 10, 12, 18, 5, 28, 104, 20, 73, 117, 46, 87, 39, 94, 22, 99, 75, 86, 29, 119, 48, 42, 83, 58, 145, 122, 131, 141, 162, 36, 139, 113, 101, 116, 129, 207, 152, 154, 49, 128, 170, 167, 245, 146, 125, 197, 215, 194, 260, 185, 263, 216, 214, 161, 312, 219, 182, 202, 205, 254, 188, 252, 203, 240, 251, 272, 257, 248, 315, 294, 237, 317, 232, 311, 330, 343, 288, 341, 325, 302, 360, 226, 361, 349, 338, 352, 358, 357, 366, 344, 337, 372, 381, 362, 283, 324, 93, 164, 14, 13, 24, 61, 23, 16, 30, 211, 123, 166, 68, 43, 98, 45, 110, 89, 64, 144, 67, 77, 69, 121, 132, 130, 133, 138, 137, 65, 150, 115, 221, 183, 156, 91, 82, 168, 124, 148, 184, 181, 176, 140, 60, 149, 192, 198, 135, 151, 177, 171, 195, 187, 229, 201, 180, 304, 239, 308, 233, 353, 218, 334, 359, 347, 323, 189, 278, 269, 290, 241, 295, 246, 327, 275, 289, 225, 318, 350, 262, 335, 310, 279, 369, 367, 328, 306, 326, 333, 332, 363, 377, 378] + 1: [71, 41, 6, 3, 2, 51, 5, 61, 8, 109, 7, 23, 33, 189, 34, 45, 70, 66, 40, 104, 84, 31, 75, 44, 87, 85, 215, 123, 88, 36, 32, 67, 217, 133, 278, 184, 158, 204, 101, 94, 181, 115, 155, 127, 122, 185, 180, 209, 230, 198, 200, 168, 196, 177, 194, 273, 175, 228, 249, 195, 233, 235, 240, 254, 220, 244, 207, 205, 277, 236, 270, 324, 334, 268, 289, 313, 321, 279, 285, 316, 297, 343, 370, 362, 295, 261, 315, 284, 348, 307, 372, 323, 352, 345, 383, 381, 47, 13, 81, 22, 4, 83, 10, 80, 77, 42, 21, 60, 43, 56, 76, 89, 68, 57, 37, 113, 24, 28, 131, 146, 82, 93, 53, 74, 197, 121, 110, 154, 107, 125, 52, 135, 142, 174, 160, 105, 86, 99, 137, 124, 141, 172, 161, 227, 216, 186, 193, 192, 145, 260, 257, 201, 169, 301, 234, 247, 206, 237, 332, 165, 190, 253, 255, 294, 276, 241, 299, 203, 310, 263, 347, 287, 338, 300, 320, 357, 304, 319, 258, 280, 346, 358, 364, 265, 349, 356, 341, 354, 373, 380, 377, 368, 49, 360, 9, 17, 62, 30, 1, 38, 35, 50, 55, 48, 19, 143, 100, 117, 138, 72, 54, 18, 150, 106, 97, 98, 20, 151, 91, 112, 166, 136, 126, 229, 153, 114, 64, 15, 134, 92, 152, 78, 170, 132, 243, 191, 156, 245, 157, 250, 148, 162, 224, 210, 238, 187, 262, 208, 183, 159, 214, 242, 223, 226, 264, 283, 259, 330, 275, 219, 288, 318, 256, 353, 337, 351, 252, 371, 290, 314, 359, 350, 361, 282, 286, 293, 291, 382, 342, 340, 355, 335, 366, 374, 367, 379, 375, 363, 178, 303, 0, 11, 26, 12, 90, 27, 58, 144, 59, 120, 29, 46, 25, 118, 102, 173, 69, 119, 65, 79, 96, 39, 139, 108, 16, 103, 116, 14, 63, 73, 95, 163, 167, 179, 202, 246, 130, 188, 149, 251, 239, 140, 171, 129, 111, 221, 211, 248, 274, 128, 182, 147, 164, 176, 213, 212, 225, 218, 267, 231, 199, 365, 308, 266, 281, 271, 232, 296, 272, 302, 322, 317, 269, 222, 325, 309, 331, 311, 326, 292, 298, 305, 327, 339, 329, 378, 328, 344, 333, 306, 369, 312, 336, 376] + 2: [79, 33, 43, 22, 1, 76, 52, 14, 16, 40, 12, 20, 2, 0, 102, 44, 60, 34, 24, 83, 128, 50, 136, 78, 99, 54, 73, 143, 120, 122, 118, 107, 85, 88, 172, 137, 212, 153, 152, 138, 97, 178, 206, 163, 279, 188, 184, 146, 213, 174, 237, 113, 125, 242, 233, 285, 183, 284, 201, 259, 243, 229, 241, 264, 236, 234, 323, 203, 303, 275, 367, 249, 232, 281, 258, 338, 289, 265, 286, 305, 301, 345, 326, 378, 322, 380, 371, 347, 331, 336, 344, 315, 271, 310, 300, 375, 86, 39, 6, 28, 53, 35, 127, 46, 66, 94, 89, 109, 25, 30, 13, 47, 71, 90, 96, 27, 72, 9, 169, 67, 38, 131, 141, 92, 108, 110, 26, 100, 68, 139, 167, 150, 223, 158, 80, 202, 154, 185, 155, 246, 288, 200, 299, 124, 221, 191, 176, 215, 151, 69, 132, 210, 224, 280, 180, 220, 225, 317, 298, 319, 226, 197, 291, 340, 283, 294, 307, 218, 278, 346, 297, 261, 320, 235, 270, 357, 329, 348, 316, 293, 370, 360, 342, 376, 377, 328, 312, 335, 358, 372, 365, 379, 133, 296, 10, 87, 5, 36, 55, 42, 29, 11, 82, 106, 45, 41, 126, 74, 48, 64, 18, 119, 84, 121, 58, 21, 17, 192, 63, 101, 162, 81, 267, 61, 181, 98, 240, 222, 214, 166, 140, 145, 111, 129, 130, 194, 177, 269, 211, 175, 205, 247, 148, 302, 144, 216, 187, 251, 238, 164, 230, 263, 156, 195, 290, 250, 182, 217, 186, 273, 274, 333, 308, 373, 276, 268, 228, 311, 254, 350, 245, 309, 266, 256, 318, 349, 313, 327, 231, 364, 339, 381, 295, 314, 374, 366, 341, 383, 199, 123, 4, 77, 8, 7, 115, 15, 51, 93, 37, 23, 65, 70, 49, 3, 31, 56, 134, 104, 32, 19, 142, 114, 105, 75, 62, 165, 103, 161, 168, 149, 59, 117, 57, 112, 135, 171, 170, 160, 193, 95, 219, 91, 209, 190, 287, 173, 157, 196, 255, 147, 189, 207, 159, 116, 253, 204, 179, 198, 282, 227, 272, 257, 244, 277, 292, 208, 239, 324, 262, 252, 334, 248, 343, 351, 306, 304, 352, 321, 356, 325, 368, 382, 332, 337, 260, 361, 330, 353, 354, 369, 363, 359, 362, 355] + 3: [310, 31, 94, 115, 250, 234, 374, 298, 267, 215, 186, 178, 192, 111, 189, 255, 281, 341, 337, 237, 81, 108, 347, 145, 240, 181, 317, 150, 93, 218, 332, 19, 28, 152, 361, 333, 166, 48, 7, 98, 283, 91, 251, 14, 211, 68, 144, 174, 142, 129, 306, 30, 57, 101, 78, 140, 300, 327, 20, 83, 336, 233, 126, 254, 82, 214, 228, 88, 136, 256, 312, 320, 303, 64, 72, 23, 275, 22, 124, 307, 382, 271, 311, 288, 59, 167, 285, 41, 96, 259, 353, 355, 222, 244, 9, 188, 34, 131, 66, 231, 296, 180, 348, 314, 151, 242, 125, 24, 318, 169, 324, 219, 197, 26, 130, 173, 122, 322, 356, 375, 269, 342, 376, 17, 236, 331, 297, 6, 161, 168, 95, 351, 62, 294, 184, 42, 264, 109, 65, 252, 323, 87, 159, 373, 85, 113, 177, 5, 143, 195, 309, 345, 58, 112, 193, 279, 183, 258, 165, 227, 206, 172, 274, 148, 369, 284, 325, 55, 154, 338, 191, 368, 238, 352, 360, 276, 349, 262, 175, 127, 137, 107, 2, 33, 153, 203, 89, 12, 212, 185, 47, 209, 260, 377, 18, 302, 35, 304, 97, 182, 367, 0, 210, 179, 282, 220, 330, 170, 13, 248, 90, 326, 230, 119, 221, 277, 80, 245, 295, 201, 103, 235, 121, 315, 157, 354, 198, 141, 155, 102, 226, 118, 49, 202, 200, 346, 79, 69, 249, 133, 247, 60, 56, 340, 204, 305, 70, 362, 268, 138, 4, 50, 246, 289, 343, 381, 217, 253, 194, 61, 106, 67, 190, 84, 370, 187, 104, 110, 1, 313, 299, 290, 116, 278, 334, 216, 286, 366, 99, 357, 132, 53, 257, 329, 359, 241, 105, 117, 163, 40, 319, 51, 208, 27, 120, 365, 293, 16, 339, 380, 92, 139, 291, 158, 135, 21, 134, 223, 280, 378, 76, 272, 162, 160, 71, 328, 37, 10, 128, 25, 86, 176, 308, 321, 379, 213, 164, 207, 114, 270, 38, 36, 229, 74, 273, 364, 100, 73, 52, 3, 383, 171, 63, 224, 46, 8, 15, 75, 199, 11, 123, 149, 239, 32, 344, 225, 363, 358, 146, 77, 196, 43, 243, 372, 371, 232, 147, 45, 316, 54, 29, 265, 44, 39, 156, 287, 266, 301, 292, 205, 261, 335, 350, 263] + 4: [172, 18, 264, 302, 219, 205, 22, 62, 29, 105, 115, 362, 101, 91, 149, 253, 272, 226, 36, 284, 215, 103, 320, 250, 69, 262, 312, 23, 48, 167, 328, 26, 374, 276, 271, 142, 248, 348, 157, 59, 191, 98, 182, 51, 338, 33, 129, 109, 97, 79, 168, 42, 323, 65, 17, 74, 85, 108, 75, 10, 335, 216, 30, 270, 281, 122, 300, 173, 242, 256, 201, 80, 291, 334, 343, 9, 128, 372, 211, 373, 313, 368, 224, 336, 126, 305, 132, 234, 301, 100, 364, 285, 83, 104, 247, 299, 12, 230, 257, 27, 252, 353, 116, 160, 159, 117, 24, 243, 309, 355, 50, 214, 199, 113, 25, 370, 213, 326, 379, 210, 307, 86, 135, 61, 207, 194, 361, 64, 38, 96, 221, 288, 371, 47, 196, 204, 381, 231, 310, 46, 251, 15, 185, 354, 60, 333, 92, 187, 267, 295, 133, 63, 290, 198, 174, 147, 325, 94, 332, 227, 35, 28, 238, 158, 337, 127, 140, 308, 342, 45, 282, 352, 255, 297, 139, 183, 43, 349, 19, 16, 268, 192, 171, 20, 131, 32, 40, 314, 146, 58, 222, 351, 283, 82, 273, 369, 347, 244, 169, 189, 346, 356, 37, 163, 93, 202, 258, 155, 0, 165, 145, 70, 114, 344, 130, 357, 322, 340, 3, 162, 153, 166, 327, 66, 274, 197, 71, 263, 73, 84, 236, 112, 179, 88, 358, 55, 176, 76, 318, 217, 78, 110, 260, 241, 315, 365, 229, 339, 141, 4, 181, 56, 121, 203, 154, 178, 375, 378, 184, 2, 1, 175, 95, 341, 287, 49, 138, 294, 14, 266, 31, 316, 123, 363, 383, 180, 81, 292, 319, 311, 195, 89, 5, 90, 259, 120, 156, 360, 254, 13, 275, 188, 44, 228, 239, 54, 235, 293, 206, 34, 265, 161, 77, 136, 72, 324, 345, 246, 57, 111, 240, 212, 359, 237, 209, 152, 21, 376, 278, 150, 68, 148, 6, 7, 330, 124, 99, 137, 143, 53, 233, 67, 164, 377, 87, 329, 11, 366, 52, 280, 106, 269, 296, 232, 220, 151, 125, 289, 186, 170, 177, 303, 190, 41, 200, 286, 331, 350, 223, 118, 321, 298, 249, 218, 306, 261, 8, 208, 382, 119, 102, 380, 107, 144, 225, 279, 134, 367, 193, 304, 39, 245, 317, 277] + 5: [214, 133, 193, 178, 131, 67, 46, 242, 69, 161, 35, 41, 299, 365, 188, 209, 361, 45, 4, 11, 379, 314, 66, 330, 149, 47, 284, 257, 244, 168, 308, 117, 200, 38, 106, 2, 266, 378, 146, 90, 62, 323, 225, 121, 16, 152, 186, 343, 22, 277, 30, 377, 24, 143, 9, 337, 380, 344, 5, 359, 219, 222, 123, 31, 320, 335, 32, 169, 172, 158, 367, 185, 294, 3, 360, 201, 54, 10, 349, 70, 307, 354, 110, 28, 50, 205, 98, 270, 51, 75, 348, 23, 321, 223, 63, 177, 165, 274, 276, 352, 287, 249, 217, 129, 190, 346, 120, 301, 351, 280, 61, 243, 21, 293, 147, 333, 291, 198, 332, 124, 232, 305, 59, 65, 298, 40, 206, 261, 347, 211, 184, 238, 64, 366, 60, 334, 227, 93, 52, 207, 36, 42, 364, 180, 82, 103, 202, 135, 286, 164, 282, 92, 183, 283, 220, 89, 88, 94, 236, 15, 187, 71, 107, 251, 29, 44, 127, 373, 99, 345, 248, 336, 234, 350, 155, 278, 26, 302, 126, 57, 229, 109, 85, 116, 362, 150, 271, 295, 275, 111, 300, 318, 128, 230, 372, 304, 182, 262, 81, 265, 383, 241, 309, 199, 231, 108, 137, 181, 159, 154, 194, 324, 306, 114, 14, 329, 339, 357, 338, 78, 132, 279, 80, 355, 8, 144, 252, 134, 259, 381, 218, 290, 48, 136, 34, 53, 216, 105, 0, 119, 171, 166, 153, 233, 374, 55, 115, 20, 122, 228, 157, 56, 140, 341, 255, 325, 273, 316, 95, 224, 145, 6, 195, 104, 353, 382, 43, 18, 83, 73, 91, 268, 174, 162, 33, 375, 311, 97, 170, 281, 312, 141, 368, 226, 285, 125, 142, 196, 269, 296, 49, 246, 17, 138, 213, 74, 151, 79, 250, 156, 130, 258, 239, 163, 221, 342, 13, 1, 173, 191, 118, 27, 245, 319, 260, 264, 167, 19, 256, 39, 112, 84, 370, 327, 197, 215, 7, 272, 113, 204, 254, 77, 210, 179, 289, 58, 369, 76, 100, 235, 192, 303, 317, 322, 356, 253, 267, 212, 203, 176, 208, 189, 358, 297, 12, 315, 292, 160, 340, 139, 240, 328, 175, 148, 102, 237, 101, 313, 331, 288, 72, 371, 86, 68, 96, 310, 326, 247, 363, 87, 25, 37, 376, 263] + 6: [97, 299, 34, 159, 26, 197, 319, 269, 349, 113, 79, 118, 196, 300, 222, 236, 143, 192, 67, 83, 14, 17, 133, 244, 361, 165, 324, 266, 326, 290, 364, 158, 217, 357, 230, 82, 8, 234, 184, 362, 84, 284, 330, 153, 343, 202, 280, 381, 177, 295, 342, 146, 30, 11, 57, 18, 353, 61, 378, 328, 5, 262, 20, 213, 60, 114, 238, 240, 383, 4, 16, 52, 329, 71, 163, 107, 13, 130, 382, 95, 92, 140, 152, 369, 37, 70, 315, 72, 200, 356, 21, 161, 327, 93, 257, 323, 142, 259, 173, 282, 337, 359, 352, 220, 122, 336, 366, 216, 338, 41, 358, 379, 10, 102, 54, 39, 272, 287, 205, 116, 371, 195, 48, 306, 19, 321, 176, 144, 270, 370, 211, 252, 232, 85, 293, 87, 285, 298, 128, 348, 45, 296, 334, 203, 109, 275, 51, 110, 344, 154, 175, 308, 148, 178, 235, 320, 254, 126, 360, 58, 242, 132, 15, 372, 157, 258, 190, 55, 127, 261, 265, 335, 331, 237, 224, 297, 182, 214, 288, 186, 276, 25, 65, 106, 115, 98, 40, 167, 139, 53, 172, 191, 274, 246, 289, 239, 322, 281, 0, 253, 301, 137, 166, 80, 23, 350, 286, 46, 136, 171, 251, 345, 304, 76, 100, 256, 316, 367, 49, 47, 291, 221, 27, 339, 241, 162, 33, 117, 215, 278, 249, 78, 226, 307, 376, 271, 138, 69, 181, 310, 63, 32, 347, 111, 243, 50, 108, 210, 6, 193, 9, 218, 187, 123, 12, 283, 279, 198, 22, 129, 68, 223, 75, 77, 164, 81, 131, 86, 43, 346, 2, 56, 24, 292, 373, 227, 179, 255, 201, 7, 168, 89, 121, 355, 96, 303, 90, 124, 305, 44, 267, 260, 112, 375, 160, 174, 264, 151, 377, 311, 62, 103, 318, 169, 380, 155, 125, 149, 199, 120, 368, 119, 64, 206, 312, 248, 309, 277, 219, 231, 250, 141, 188, 268, 302, 73, 233, 341, 228, 59, 207, 332, 225, 66, 229, 88, 105, 208, 42, 101, 74, 99, 189, 36, 209, 263, 29, 313, 35, 31, 91, 94, 145, 365, 104, 185, 333, 183, 340, 354, 212, 245, 28, 180, 135, 314, 170, 134, 294, 325, 156, 351, 194, 38, 204, 374, 317, 247, 273, 3, 147, 150, 363, 1] + 7: [382, 113, 133, 238, 284, 331, 69, 0, 239, 11, 59, 9, 160, 190, 21, 336, 121, 302, 359, 198, 299, 12, 18, 268, 99, 247, 182, 270, 300, 203, 152, 166, 376, 70, 104, 38, 61, 132, 379, 373, 1, 196, 313, 237, 261, 167, 271, 372, 230, 235, 143, 318, 229, 227, 128, 31, 14, 374, 367, 5, 149, 266, 159, 179, 185, 255, 171, 162, 241, 234, 283, 106, 169, 35, 52, 364, 50, 26, 338, 164, 330, 127, 120, 346, 66, 175, 366, 151, 98, 222, 371, 215, 192, 375, 37, 273, 27, 49, 170, 358, 188, 72, 46, 294, 15, 354, 119, 64, 184, 301, 337, 315, 174, 228, 40, 56, 63, 33, 303, 28, 262, 368, 334, 129, 286, 218, 309, 25, 165, 140, 224, 253, 304, 91, 287, 312, 125, 122, 306, 290, 112, 344, 242, 142, 285, 278, 342, 195, 78, 260, 343, 20, 32, 267, 93, 254, 114, 172, 322, 316, 276, 90, 212, 47, 194, 139, 89, 233, 124, 34, 84, 220, 250, 281, 263, 4, 219, 307, 199, 54, 180, 248, 361, 102, 357, 365, 240, 178, 207, 2, 111, 60, 292, 329, 258, 48, 213, 216, 305, 333, 71, 321, 19, 126, 74, 80, 30, 360, 311, 154, 295, 36, 200, 76, 153, 347, 209, 41, 217, 136, 319, 24, 317, 138, 352, 29, 73, 370, 118, 275, 193, 280, 288, 277, 181, 383, 323, 353, 7, 202, 328, 380, 339, 349, 256, 362, 265, 272, 206, 55, 297, 39, 204, 369, 75, 44, 208, 131, 308, 43, 141, 53, 147, 325, 137, 134, 45, 65, 259, 96, 116, 177, 92, 156, 231, 115, 211, 22, 189, 341, 381, 245, 6, 236, 201, 348, 3, 157, 163, 274, 378, 356, 187, 282, 51, 100, 13, 155, 335, 130, 82, 326, 197, 355, 221, 150, 101, 23, 314, 340, 296, 77, 264, 279, 94, 291, 146, 289, 148, 107, 110, 324, 176, 320, 108, 168, 83, 16, 123, 183, 205, 85, 161, 244, 332, 105, 8, 327, 86, 293, 298, 214, 246, 226, 62, 191, 103, 158, 310, 17, 144, 252, 249, 95, 117, 186, 109, 10, 269, 42, 173, 210, 251, 351, 350, 79, 67, 81, 377, 87, 232, 58, 257, 225, 68, 363, 57, 135, 223, 97, 88, 243, 145, 345] + 8: [290, 17, 315, 35, 371, 367, 227, 351, 171, 335, 225, 131, 40, 185, 376, 337, 221, 352, 278, 275, 282, 329, 330, 263, 301, 12, 238, 92, 114, 245, 243, 271, 115, 137, 179, 224, 284, 287, 205, 308, 110, 177, 359, 383, 248, 264, 251, 162, 366, 222, 345, 332, 299, 147, 43, 239, 183, 109, 72, 55, 73, 2, 274, 23, 124, 283, 184, 253, 199, 328, 146, 24, 250, 188, 178, 244, 134, 192, 379, 59, 267, 145, 357, 44, 163, 97, 78, 54, 136, 270, 15, 216, 113, 214, 102, 26, 88, 231, 62, 336, 354, 249, 103, 158, 381, 27, 90, 52, 38, 309, 80, 82, 150, 8, 33, 129, 215, 10, 257, 241, 65, 194, 344, 360, 276, 6, 3, 219, 117, 382, 187, 125, 132, 280, 259, 81, 196, 209, 191, 144, 42, 247, 14, 139, 213, 237, 180, 198, 327, 343, 22, 1, 189, 272, 57, 197, 334, 260, 311, 133, 190, 324, 95, 340, 66, 300, 159, 63, 39, 348, 365, 338, 165, 25, 230, 29, 320, 30, 310, 305, 212, 46, 108, 151, 116, 5, 254, 206, 234, 314, 120, 369, 362, 312, 7, 353, 156, 152, 50, 18, 84, 61, 148, 223, 195, 94, 240, 279, 255, 153, 70, 289, 372, 228, 296, 377, 355, 37, 31, 293, 356, 122, 45, 319, 307, 74, 93, 331, 11, 16, 277, 138, 358, 246, 60, 322, 273, 75, 161, 235, 373, 193, 347, 341, 173, 140, 261, 0, 47, 13, 167, 292, 123, 258, 181, 361, 87, 236, 166, 252, 168, 34, 266, 96, 297, 176, 69, 226, 143, 19, 112, 182, 99, 56, 363, 174, 154, 298, 104, 204, 342, 142, 242, 317, 364, 58, 41, 378, 76, 218, 202, 333, 118, 281, 200, 201, 368, 83, 326, 36, 4, 21, 175, 164, 229, 288, 302, 172, 32, 119, 321, 380, 294, 85, 318, 53, 128, 339, 20, 269, 28, 127, 67, 130, 186, 316, 208, 374, 107, 313, 306, 325, 49, 286, 9, 77, 233, 111, 157, 126, 217, 79, 155, 106, 169, 291, 304, 98, 86, 100, 232, 89, 68, 51, 135, 64, 210, 323, 91, 256, 101, 303, 285, 346, 262, 211, 349, 350, 160, 220, 149, 121, 207, 203, 170, 268, 71, 48, 141, 295, 375, 265, 370, 105] + 9: [92, 186, 30, 259, 8, 339, 56, 263, 288, 183, 131, 75, 350, 228, 222, 184, 120, 98, 174, 180, 60, 167, 74, 311, 364, 119, 110, 323, 46, 65, 9, 99, 250, 343, 306, 315, 153, 81, 328, 262, 62, 318, 251, 195, 181, 378, 22, 163, 216, 330, 166, 38, 23, 31, 86, 194, 127, 309, 61, 133, 358, 232, 190, 287, 264, 185, 370, 290, 152, 44, 267, 349, 108, 204, 269, 238, 223, 95, 93, 173, 340, 295, 374, 169, 225, 373, 7, 66, 118, 43, 212, 336, 375, 149, 326, 240, 327, 353, 281, 132, 322, 35, 210, 192, 187, 170, 94, 129, 54, 324, 159, 29, 342, 314, 176, 172, 71, 144, 217, 58, 126, 70, 256, 199, 32, 218, 130, 16, 198, 76, 354, 73, 206, 227, 310, 123, 301, 366, 302, 164, 211, 27, 377, 252, 88, 379, 272, 21, 78, 2, 316, 233, 313, 33, 113, 91, 325, 168, 182, 96, 368, 137, 258, 329, 297, 361, 160, 77, 268, 352, 230, 89, 345, 294, 162, 300, 347, 67, 249, 236, 254, 175, 338, 265, 50, 4, 308, 201, 291, 202, 284, 26, 3, 331, 143, 355, 237, 17, 103, 191, 360, 36, 299, 219, 72, 20, 25, 148, 83, 231, 376, 253, 277, 307, 112, 372, 135, 296, 380, 381, 245, 111, 261, 128, 114, 208, 107, 156, 101, 97, 84, 134, 165, 117, 177, 178, 121, 155, 1, 18, 48, 64, 68, 157, 197, 12, 356, 273, 79, 285, 200, 154, 214, 109, 292, 39, 317, 41, 362, 209, 257, 146, 303, 193, 45, 52, 85, 205, 49, 226, 40, 141, 19, 63, 346, 53, 196, 348, 276, 282, 246, 279, 24, 122, 312, 55, 283, 57, 215, 260, 150, 138, 102, 80, 5, 104, 298, 239, 10, 321, 224, 87, 244, 371, 359, 221, 47, 289, 335, 274, 220, 37, 13, 139, 6, 275, 341, 367, 151, 363, 333, 344, 235, 243, 42, 125, 242, 247, 351, 382, 179, 337, 105, 14, 320, 189, 229, 207, 293, 136, 124, 203, 332, 213, 82, 255, 357, 0, 106, 278, 171, 234, 145, 100, 15, 369, 142, 280, 304, 266, 286, 69, 383, 140, 241, 34, 271, 334, 319, 188, 365, 270, 248, 305, 28, 90, 115, 158, 59, 147, 51, 116, 11, 161] + 10: [55, 61, 222, 353, 78, 262, 365, 69, 234, 227, 124, 293, 246, 289, 138, 8, 282, 109, 341, 113, 231, 181, 338, 40, 132, 223, 257, 306, 72, 116, 75, 184, 130, 240, 267, 148, 123, 220, 327, 188, 114, 210, 243, 256, 137, 214, 189, 252, 330, 121, 100, 37, 190, 265, 308, 297, 145, 153, 131, 346, 88, 167, 122, 202, 328, 313, 383, 26, 335, 93, 13, 52, 118, 381, 323, 261, 296, 260, 134, 147, 270, 303, 366, 173, 162, 336, 166, 233, 216, 369, 64, 357, 291, 209, 334, 292, 31, 321, 180, 91, 378, 73, 141, 249, 229, 281, 102, 59, 107, 199, 301, 105, 251, 170, 149, 94, 280, 354, 352, 279, 164, 111, 87, 304, 195, 219, 343, 32, 0, 363, 376, 161, 225, 203, 259, 95, 242, 367, 67, 187, 9, 171, 119, 117, 80, 263, 53, 16, 10, 333, 43, 54, 18, 41, 269, 298, 226, 294, 344, 355, 320, 213, 276, 35, 205, 255, 51, 272, 142, 197, 360, 200, 70, 299, 12, 198, 285, 120, 28, 311, 165, 89, 288, 155, 63, 151, 6, 68, 314, 183, 332, 339, 5, 250, 66, 309, 2, 258, 33, 144, 92, 264, 318, 90, 191, 215, 268, 337, 310, 248, 239, 74, 110, 271, 284, 221, 204, 156, 382, 317, 169, 283, 377, 127, 351, 302, 361, 182, 375, 211, 17, 373, 217, 76, 20, 96, 7, 15, 56, 207, 129, 380, 286, 84, 60, 368, 370, 364, 201, 325, 235, 237, 275, 290, 30, 44, 372, 295, 374, 196, 98, 125, 307, 287, 228, 238, 146, 186, 85, 324, 1, 79, 348, 58, 4, 193, 305, 38, 82, 340, 83, 86, 254, 23, 29, 350, 157, 277, 319, 115, 266, 192, 224, 177, 103, 345, 218, 247, 99, 158, 97, 57, 342, 108, 11, 274, 241, 101, 81, 65, 22, 106, 230, 326, 322, 152, 19, 174, 128, 316, 133, 315, 329, 126, 245, 143, 185, 379, 356, 3, 34, 25, 212, 312, 154, 47, 140, 362, 49, 112, 163, 24, 359, 62, 50, 77, 150, 45, 194, 206, 208, 175, 48, 300, 172, 349, 371, 139, 159, 21, 278, 104, 27, 39, 36, 253, 358, 71, 46, 168, 14, 347, 135, 331, 236, 178, 160, 244, 179, 176, 136, 232, 273, 42] + 11: [377, 4, 277, 186, 249, 60, 317, 107, 230, 143, 150, 374, 278, 95, 131, 217, 321, 298, 293, 69, 1, 280, 165, 59, 52, 187, 67, 285, 360, 160, 294, 250, 85, 282, 297, 267, 64, 37, 253, 256, 315, 25, 198, 255, 7, 279, 40, 91, 142, 370, 332, 367, 133, 199, 222, 261, 62, 364, 292, 363, 11, 63, 144, 151, 49, 32, 311, 174, 324, 197, 113, 304, 21, 192, 158, 41, 88, 163, 193, 289, 136, 206, 361, 175, 139, 48, 270, 226, 152, 8, 352, 371, 241, 355, 55, 348, 130, 203, 17, 339, 196, 128, 313, 271, 325, 66, 247, 349, 166, 101, 87, 378, 242, 123, 331, 219, 171, 356, 53, 168, 189, 28, 182, 376, 71, 216, 232, 248, 75, 92, 207, 35, 149, 93, 350, 375, 170, 24, 372, 214, 46, 81, 237, 20, 58, 223, 36, 51, 98, 329, 47, 141, 184, 57, 245, 266, 167, 19, 319, 109, 129, 12, 301, 134, 296, 177, 18, 290, 169, 183, 103, 263, 354, 191, 236, 288, 268, 155, 138, 260, 210, 383, 100, 148, 202, 265, 105, 72, 308, 33, 112, 120, 156, 234, 83, 110, 362, 326, 254, 213, 208, 194, 30, 238, 300, 220, 299, 264, 99, 10, 188, 27, 302, 291, 281, 172, 82, 45, 357, 345, 74, 330, 31, 34, 259, 190, 2, 157, 201, 369, 318, 185, 347, 159, 42, 84, 287, 275, 209, 153, 272, 68, 273, 231, 342, 161, 200, 106, 6, 212, 137, 127, 337, 116, 79, 276, 70, 50, 86, 303, 314, 77, 224, 76, 221, 38, 108, 366, 5, 295, 335, 351, 78, 135, 320, 286, 327, 205, 328, 102, 22, 215, 344, 96, 338, 181, 379, 9, 284, 29, 306, 145, 235, 56, 43, 258, 233, 121, 80, 228, 316, 26, 373, 309, 23, 322, 90, 119, 44, 180, 115, 65, 97, 173, 162, 211, 195, 229, 39, 154, 243, 251, 239, 218, 146, 164, 73, 380, 368, 176, 365, 257, 343, 125, 341, 381, 353, 111, 274, 336, 305, 340, 0, 246, 178, 104, 227, 3, 240, 117, 16, 54, 204, 310, 252, 94, 118, 307, 346, 358, 179, 89, 124, 147, 122, 283, 382, 312, 244, 140, 269, 15, 13, 262, 333, 359, 323, 225, 126, 132, 334, 114, 61, 14] + 12: [226, 20, 194, 300, 120, 298, 109, 71, 3, 304, 108, 185, 280, 39, 59, 127, 237, 9, 198, 227, 268, 294, 93, 86, 330, 41, 329, 245, 175, 49, 317, 148, 161, 96, 196, 151, 331, 69, 236, 52, 335, 53, 267, 88, 332, 92, 114, 381, 232, 338, 25, 33, 366, 363, 77, 371, 374, 130, 137, 85, 170, 31, 191, 351, 1, 79, 178, 10, 19, 216, 307, 24, 5, 115, 243, 12, 143, 375, 188, 252, 122, 149, 167, 179, 80, 16, 23, 354, 314, 42, 213, 259, 211, 352, 220, 285, 82, 292, 308, 218, 203, 90, 234, 51, 140, 337, 72, 172, 281, 373, 207, 370, 327, 320, 113, 233, 75, 286, 219, 125, 293, 166, 107, 155, 246, 176, 180, 301, 76, 276, 258, 102, 55, 289, 156, 362, 43, 94, 275, 359, 200, 2, 173, 18, 224, 206, 136, 240, 265, 202, 123, 63, 299, 353, 132, 279, 356, 382, 89, 100, 128, 57, 101, 134, 54, 310, 262, 215, 30, 334, 135, 323, 325, 221, 35, 269, 347, 44, 119, 67, 228, 74, 56, 27, 368, 217, 249, 117, 66, 165, 60, 162, 204, 341, 255, 112, 306, 378, 13, 247, 138, 346, 254, 208, 305, 355, 195, 110, 186, 361, 326, 22, 312, 64, 62, 129, 251, 193, 11, 87, 210, 263, 38, 40, 153, 222, 192, 144, 288, 169, 328, 164, 28, 177, 26, 48, 309, 111, 376, 116, 273, 163, 6, 383, 34, 365, 257, 379, 291, 32, 350, 160, 189, 121, 340, 318, 238, 159, 360, 84, 239, 266, 260, 98, 242, 61, 65, 142, 364, 278, 73, 146, 197, 70, 372, 377, 183, 58, 324, 336, 253, 342, 15, 154, 212, 303, 272, 145, 78, 287, 150, 158, 68, 50, 322, 147, 315, 201, 209, 133, 313, 81, 271, 270, 367, 230, 290, 274, 168, 36, 46, 103, 4, 344, 380, 321, 106, 99, 105, 21, 283, 229, 91, 174, 8, 250, 282, 277, 345, 45, 214, 181, 37, 357, 264, 316, 296, 311, 29, 83, 244, 14, 284, 205, 248, 339, 297, 190, 187, 235, 369, 124, 95, 302, 0, 171, 118, 97, 152, 256, 141, 343, 223, 131, 319, 7, 241, 126, 182, 47, 17, 348, 231, 139, 184, 225, 295, 333, 104, 358, 199, 261, 349, 157] + 13: [108, 46, 172, 186, 329, 340, 231, 363, 199, 316, 134, 228, 77, 16, 55, 244, 110, 86, 324, 318, 36, 6, 125, 201, 83, 348, 109, 277, 282, 314, 40, 335, 37, 266, 121, 364, 275, 333, 21, 205, 261, 61, 322, 300, 249, 81, 238, 260, 66, 339, 326, 288, 253, 50, 92, 285, 243, 270, 79, 38, 317, 360, 63, 279, 47, 240, 67, 303, 33, 1, 26, 115, 220, 194, 251, 357, 268, 283, 334, 310, 371, 367, 245, 53, 48, 168, 17, 44, 215, 258, 27, 129, 350, 341, 132, 302, 170, 328, 309, 179, 87, 273, 30, 373, 379, 319, 72, 0, 382, 200, 291, 307, 76, 184, 188, 353, 139, 153, 11, 234, 52, 198, 177, 35, 207, 267, 222, 185, 84, 25, 169, 352, 223, 226, 41, 361, 2, 297, 107, 94, 45, 305, 126, 144, 203, 128, 162, 164, 269, 88, 290, 362, 187, 299, 101, 232, 263, 225, 202, 74, 247, 190, 182, 271, 289, 209, 365, 359, 212, 10, 193, 127, 295, 150, 7, 280, 327, 3, 112, 28, 122, 167, 98, 105, 192, 29, 354, 381, 343, 377, 32, 59, 23, 298, 159, 8, 311, 111, 208, 151, 65, 211, 214, 284, 191, 369, 123, 338, 345, 91, 315, 31, 147, 149, 221, 248, 69, 135, 131, 141, 64, 210, 293, 272, 346, 296, 292, 213, 294, 197, 229, 89, 380, 176, 286, 224, 82, 24, 281, 257, 161, 54, 15, 183, 39, 233, 265, 100, 4, 57, 133, 137, 254, 113, 173, 347, 216, 22, 287, 375, 104, 73, 136, 374, 320, 13, 372, 330, 206, 140, 306, 241, 20, 142, 276, 78, 90, 155, 204, 181, 332, 250, 124, 130, 175, 378, 85, 152, 189, 12, 195, 102, 117, 304, 56, 358, 58, 342, 178, 93, 259, 116, 51, 143, 196, 239, 171, 321, 166, 96, 34, 97, 19, 337, 75, 355, 370, 323, 313, 246, 227, 174, 237, 68, 218, 71, 376, 301, 62, 242, 99, 42, 120, 154, 49, 180, 14, 236, 114, 119, 344, 219, 80, 43, 308, 5, 274, 264, 9, 138, 163, 230, 160, 235, 252, 312, 146, 157, 349, 368, 106, 118, 156, 336, 60, 256, 148, 325, 356, 262, 158, 18, 103, 351, 366, 255, 95, 165, 278, 383, 145, 331, 70, 217] + 14: [4, 147, 30, 14, 95, 291, 118, 34, 17, 90, 356, 215, 158, 23, 92, 62, 343, 342, 145, 284, 180, 324, 369, 312, 64, 16, 317, 250, 274, 272, 115, 134, 359, 51, 262, 252, 36, 325, 224, 368, 183, 160, 285, 378, 290, 87, 24, 382, 381, 89, 277, 102, 94, 232, 57, 348, 333, 303, 136, 276, 365, 351, 119, 1, 82, 367, 98, 32, 165, 88, 74, 194, 326, 223, 96, 231, 211, 3, 323, 18, 28, 172, 146, 293, 263, 171, 138, 42, 46, 176, 195, 20, 111, 72, 106, 120, 204, 164, 110, 261, 151, 184, 213, 127, 238, 91, 375, 142, 141, 313, 139, 355, 362, 75, 71, 206, 58, 298, 220, 301, 268, 318, 38, 334, 199, 25, 275, 222, 257, 376, 198, 269, 345, 86, 60, 170, 167, 254, 350, 270, 280, 190, 66, 309, 247, 22, 129, 202, 152, 366, 173, 322, 300, 209, 246, 240, 187, 191, 329, 236, 225, 321, 93, 7, 192, 255, 135, 282, 193, 357, 133, 201, 159, 319, 320, 346, 84, 245, 85, 143, 39, 360, 336, 260, 363, 178, 289, 100, 188, 216, 140, 181, 70, 126, 163, 383, 2, 128, 8, 40, 349, 227, 5, 79, 80, 354, 44, 63, 149, 125, 234, 150, 217, 273, 109, 308, 56, 379, 296, 104, 116, 6, 361, 239, 278, 101, 226, 352, 200, 197, 248, 305, 169, 76, 105, 97, 212, 218, 182, 373, 307, 11, 54, 267, 327, 253, 137, 364, 335, 338, 286, 251, 130, 81, 205, 243, 21, 340, 153, 259, 26, 235, 256, 47, 59, 131, 249, 315, 265, 132, 48, 108, 10, 310, 229, 53, 112, 228, 304, 68, 113, 203, 50, 65, 210, 73, 337, 295, 15, 117, 122, 344, 287, 13, 271, 154, 377, 177, 370, 294, 99, 244, 156, 69, 242, 281, 380, 237, 279, 374, 347, 330, 161, 83, 341, 372, 186, 175, 283, 49, 266, 208, 157, 52, 185, 103, 162, 41, 207, 311, 297, 358, 123, 332, 77, 230, 35, 107, 124, 214, 0, 299, 19, 241, 43, 233, 33, 258, 168, 37, 55, 196, 353, 9, 316, 306, 288, 61, 148, 114, 155, 166, 144, 27, 264, 292, 314, 121, 302, 12, 221, 29, 67, 219, 371, 179, 189, 31, 45, 78, 331, 174, 339, 328] + 15: [11, 353, 24, 168, 128, 56, 344, 329, 265, 13, 173, 231, 284, 10, 145, 267, 273, 96, 257, 238, 63, 213, 84, 161, 382, 74, 2, 71, 282, 7, 122, 305, 317, 224, 266, 285, 249, 380, 303, 343, 377, 372, 188, 254, 274, 186, 70, 126, 34, 46, 379, 152, 64, 356, 142, 191, 360, 192, 318, 327, 44, 9, 85, 331, 205, 3, 123, 270, 118, 129, 89, 75, 21, 347, 132, 260, 314, 110, 154, 146, 0, 247, 354, 310, 237, 229, 239, 6, 223, 345, 374, 103, 61, 29, 328, 208, 362, 232, 359, 38, 376, 253, 197, 8, 133, 319, 246, 212, 109, 290, 134, 150, 114, 51, 16, 262, 309, 80, 258, 88, 138, 291, 251, 31, 272, 158, 54, 297, 332, 67, 203, 361, 187, 233, 346, 378, 202, 321, 235, 90, 250, 165, 40, 174, 119, 330, 198, 55, 53, 95, 1, 299, 112, 236, 25, 338, 92, 289, 184, 211, 12, 77, 121, 37, 227, 315, 226, 170, 127, 264, 69, 141, 322, 276, 23, 375, 201, 143, 45, 325, 62, 30, 140, 189, 179, 182, 139, 159, 218, 336, 176, 286, 15, 199, 28, 190, 200, 117, 350, 308, 5, 48, 252, 14, 94, 35, 39, 351, 326, 167, 334, 296, 248, 106, 66, 381, 280, 111, 65, 293, 68, 97, 169, 300, 204, 256, 306, 108, 153, 183, 155, 196, 365, 86, 164, 216, 302, 120, 207, 244, 107, 339, 73, 368, 221, 156, 185, 163, 316, 342, 50, 281, 72, 383, 181, 98, 349, 225, 60, 271, 57, 49, 341, 220, 367, 295, 312, 195, 259, 355, 58, 288, 166, 131, 81, 78, 292, 275, 18, 137, 219, 100, 136, 358, 144, 268, 269, 42, 151, 105, 217, 115, 313, 255, 324, 352, 148, 83, 240, 76, 4, 335, 357, 298, 320, 215, 277, 307, 193, 261, 20, 52, 340, 113, 373, 36, 32, 172, 209, 99, 26, 116, 371, 241, 287, 263, 177, 333, 82, 33, 149, 43, 214, 243, 311, 157, 206, 323, 171, 175, 59, 102, 147, 364, 337, 234, 230, 101, 41, 242, 369, 135, 366, 104, 222, 22, 194, 370, 294, 87, 130, 160, 279, 27, 124, 178, 301, 304, 210, 228, 79, 93, 245, 125, 283, 278, 348, 180, 162, 91, 363, 17, 47, 19] + 16: [15, 374, 287, 348, 310, 134, 216, 301, 189, 92, 11, 379, 346, 331, 56, 135, 233, 305, 241, 9, 286, 187, 274, 185, 360, 212, 285, 342, 252, 238, 357, 315, 248, 129, 102, 57, 139, 227, 278, 265, 12, 246, 222, 340, 306, 220, 253, 194, 79, 131, 270, 191, 4, 314, 363, 288, 177, 329, 132, 181, 105, 334, 157, 138, 89, 197, 58, 170, 317, 365, 259, 35, 381, 325, 153, 341, 124, 63, 313, 37, 249, 322, 168, 299, 345, 359, 112, 48, 165, 179, 284, 376, 51, 309, 163, 3, 262, 100, 64, 183, 119, 260, 106, 71, 275, 330, 96, 370, 173, 180, 27, 280, 171, 182, 121, 298, 115, 93, 19, 213, 123, 358, 200, 0, 337, 31, 167, 29, 160, 300, 104, 80, 294, 2, 371, 10, 349, 47, 236, 70, 266, 207, 304, 87, 308, 268, 130, 225, 26, 323, 210, 316, 292, 28, 352, 144, 159, 269, 373, 297, 215, 99, 148, 40, 326, 78, 372, 81, 41, 235, 18, 136, 364, 272, 324, 23, 145, 195, 244, 126, 1, 263, 251, 223, 6, 117, 34, 30, 65, 303, 50, 156, 312, 5, 46, 118, 250, 59, 333, 14, 84, 205, 229, 44, 109, 281, 162, 375, 32, 75, 273, 217, 267, 283, 120, 66, 73, 52, 147, 211, 175, 336, 154, 54, 68, 158, 256, 214, 378, 88, 176, 151, 60, 143, 17, 234, 61, 141, 245, 186, 240, 36, 133, 83, 203, 276, 380, 13, 218, 230, 271, 282, 343, 289, 295, 258, 226, 178, 101, 62, 296, 383, 247, 356, 361, 74, 219, 169, 122, 368, 174, 38, 161, 140, 208, 264, 353, 366, 193, 33, 351, 261, 350, 192, 328, 367, 257, 291, 152, 114, 7, 198, 237, 277, 125, 369, 362, 85, 21, 43, 293, 188, 146, 347, 232, 255, 91, 172, 76, 354, 90, 142, 311, 97, 302, 24, 321, 202, 224, 42, 22, 199, 86, 243, 332, 16, 290, 8, 164, 382, 204, 209, 39, 166, 113, 128, 110, 190, 103, 25, 95, 108, 116, 307, 149, 20, 206, 111, 254, 82, 221, 335, 155, 67, 319, 320, 53, 242, 327, 355, 94, 184, 49, 338, 339, 98, 55, 231, 107, 150, 201, 69, 72, 196, 377, 239, 77, 228, 127, 45, 279, 137, 318, 344] + 17: [119, 188, 46, 374, 142, 275, 79, 146, 104, 243, 207, 56, 91, 24, 272, 356, 98, 97, 189, 284, 156, 116, 363, 94, 81, 310, 319, 20, 15, 71, 259, 171, 48, 222, 33, 328, 145, 58, 230, 186, 276, 209, 247, 158, 40, 360, 359, 11, 292, 301, 256, 105, 213, 53, 216, 200, 114, 187, 313, 172, 265, 218, 199, 255, 69, 18, 221, 239, 67, 164, 149, 327, 381, 252, 353, 132, 268, 17, 85, 30, 182, 306, 295, 264, 122, 165, 152, 224, 111, 235, 280, 312, 203, 166, 223, 304, 220, 236, 89, 136, 294, 44, 117, 361, 143, 138, 178, 118, 366, 194, 107, 167, 128, 22, 113, 19, 240, 278, 208, 131, 376, 12, 348, 23, 279, 135, 286, 38, 125, 238, 73, 206, 297, 248, 174, 289, 370, 317, 316, 78, 341, 63, 83, 127, 133, 244, 1, 334, 9, 90, 274, 55, 109, 288, 139, 228, 245, 144, 300, 233, 21, 241, 151, 309, 323, 150, 141, 162, 346, 283, 34, 74, 153, 217, 333, 242, 345, 198, 175, 159, 7, 371, 329, 102, 263, 43, 342, 50, 28, 57, 227, 358, 261, 337, 357, 100, 307, 80, 197, 76, 254, 193, 296, 380, 173, 378, 190, 336, 308, 115, 163, 258, 303, 3, 96, 140, 8, 367, 364, 231, 0, 5, 59, 101, 52, 362, 318, 185, 204, 232, 282, 234, 124, 298, 343, 375, 61, 36, 37, 45, 176, 382, 285, 352, 344, 126, 372, 16, 302, 66, 10, 351, 87, 39, 92, 331, 321, 330, 281, 14, 51, 84, 325, 311, 237, 88, 270, 72, 195, 160, 70, 147, 184, 161, 202, 349, 183, 291, 299, 215, 180, 68, 210, 169, 251, 120, 368, 322, 266, 212, 108, 335, 324, 177, 110, 157, 75, 214, 262, 99, 219, 2, 42, 314, 27, 305, 196, 315, 65, 129, 290, 26, 179, 347, 77, 62, 103, 250, 82, 340, 60, 93, 6, 168, 130, 260, 326, 293, 355, 225, 49, 271, 148, 191, 287, 154, 377, 134, 112, 226, 47, 253, 365, 273, 246, 267, 369, 155, 13, 338, 4, 95, 121, 137, 229, 64, 86, 269, 350, 332, 181, 29, 211, 32, 54, 249, 170, 339, 25, 373, 277, 257, 383, 35, 320, 192, 41, 123, 379, 354, 205, 201, 31, 106] + 18: [271, 43, 46, 30, 29, 375, 380, 346, 347, 142, 174, 328, 339, 171, 234, 338, 334, 203, 355, 195, 155, 210, 231, 168, 212, 366, 316, 257, 245, 213, 291, 242, 320, 131, 341, 10, 359, 280, 238, 67, 343, 286, 99, 158, 337, 19, 26, 23, 248, 326, 173, 183, 175, 270, 156, 353, 159, 76, 299, 70, 285, 145, 133, 246, 0, 367, 1, 374, 143, 304, 2, 83, 91, 186, 219, 134, 268, 97, 151, 227, 352, 224, 92, 44, 95, 36, 150, 20, 45, 49, 59, 74, 75, 94, 101, 127, 260, 192, 35, 117, 118, 207, 228, 119, 266, 11, 342, 205, 217, 208, 356, 294, 122, 237, 373, 163, 124, 193, 96, 82, 324, 53, 236, 108, 33, 18, 22, 56, 166, 325, 12, 66, 78, 152, 382, 252, 349, 7, 62, 269, 28, 112, 39, 58, 225, 114, 170, 261, 37, 85, 305, 379, 106, 167, 214, 162, 48, 128, 289, 81, 313, 15, 9, 357, 301, 47, 182, 218, 13, 292, 120, 196, 276, 141, 265, 194, 298, 4, 364, 303, 232, 104, 350, 107, 222, 132, 161, 165, 187, 197, 220, 230, 139, 185, 169, 251, 34, 65, 68, 31, 284, 146, 189, 164, 16, 361, 77, 61, 149, 288, 300, 89, 275, 369, 297, 102, 282, 335, 41, 235, 264, 40, 93, 206, 370, 272, 333, 72, 123, 279, 60, 188, 360, 358, 345, 216, 42, 202, 14, 32, 79, 181, 250, 223, 51, 24, 121, 383, 54, 240, 157, 116, 372, 129, 377, 178, 177, 312, 281, 241, 64, 253, 233, 90, 259, 371, 221, 307, 52, 184, 21, 317, 179, 153, 130, 290, 332, 209, 3, 296, 254, 247, 258, 274, 293, 311, 314, 322, 138, 98, 71, 69, 113, 144, 376, 330, 331, 50, 255, 368, 211, 363, 351, 176, 365, 63, 229, 6, 103, 136, 80, 215, 110, 256, 148, 190, 84, 263, 306, 105, 329, 244, 262, 8, 199, 308, 38, 87, 86, 57, 267, 160, 319, 154, 201, 310, 243, 17, 239, 115, 111, 302, 283, 27, 135, 5, 278, 315, 200, 191, 147, 25, 172, 126, 277, 381, 109, 318, 100, 273, 348, 295, 140, 137, 204, 125, 198, 321, 88, 180, 226, 73, 378, 249, 55, 309, 287, 323, 327, 336, 340, 344, 354, 362] + 19: [24, 328, 207, 134, 323, 158, 152, 58, 141, 279, 339, 161, 354, 183, 243, 3, 165, 61, 208, 84, 302, 333, 147, 94, 342, 296, 360, 216, 217, 314, 352, 255, 44, 191, 239, 192, 202, 125, 230, 280, 331, 337, 41, 210, 182, 306, 370, 143, 317, 258, 371, 10, 212, 40, 330, 335, 5, 356, 324, 99, 81, 310, 76, 140, 151, 267, 11, 269, 341, 18, 234, 344, 123, 67, 254, 78, 168, 169, 128, 277, 170, 39, 276, 131, 23, 25, 32, 34, 46, 62, 72, 93, 98, 100, 106, 109, 190, 334, 363, 90, 188, 307, 115, 120, 1, 166, 287, 312, 251, 156, 297, 283, 96, 80, 377, 180, 240, 157, 359, 366, 260, 2, 73, 235, 320, 113, 13, 19, 92, 31, 144, 15, 219, 281, 378, 367, 358, 6, 55, 353, 146, 286, 221, 51, 107, 95, 111, 175, 112, 351, 110, 247, 130, 233, 293, 63, 364, 382, 127, 199, 285, 86, 28, 82, 209, 88, 308, 305, 381, 223, 278, 186, 298, 164, 350, 119, 313, 379, 129, 340, 236, 174, 347, 172, 118, 126, 133, 136, 137, 155, 163, 201, 187, 222, 97, 52, 244, 206, 77, 288, 91, 265, 203, 193, 132, 29, 215, 338, 315, 301, 8, 275, 300, 101, 38, 117, 374, 227, 167, 153, 321, 79, 272, 43, 176, 349, 159, 30, 249, 361, 290, 177, 53, 74, 295, 89, 195, 102, 200, 259, 14, 59, 65, 336, 16, 304, 228, 181, 224, 54, 27, 262, 60, 36, 26, 237, 69, 64, 45, 37, 85, 145, 252, 196, 0, 232, 21, 375, 104, 248, 362, 17, 213, 253, 343, 380, 12, 49, 250, 50, 75, 198, 205, 229, 245, 266, 274, 316, 256, 66, 7, 179, 154, 291, 211, 121, 33, 114, 122, 108, 9, 373, 318, 171, 71, 264, 270, 135, 47, 20, 284, 218, 116, 68, 138, 238, 139, 273, 303, 57, 83, 56, 184, 294, 173, 325, 311, 178, 242, 271, 149, 292, 231, 383, 241, 148, 160, 185, 70, 197, 22, 372, 194, 35, 204, 299, 282, 332, 345, 257, 368, 246, 42, 289, 189, 87, 124, 376, 142, 348, 162, 220, 329, 225, 346, 322, 261, 369, 268, 48, 226, 309, 150, 103, 4, 263, 105, 214, 319, 326, 327, 355, 357, 365] + 20: [37, 144, 369, 66, 223, 35, 252, 113, 291, 307, 339, 194, 177, 357, 374, 125, 216, 221, 289, 6, 256, 320, 188, 233, 33, 354, 90, 3, 323, 136, 346, 26, 264, 270, 110, 309, 227, 88, 50, 337, 164, 81, 312, 59, 31, 75, 133, 218, 46, 56, 141, 306, 253, 204, 48, 226, 298, 70, 308, 115, 340, 126, 217, 348, 73, 163, 238, 274, 5, 12, 16, 17, 23, 24, 28, 34, 38, 44, 47, 51, 52, 60, 62, 63, 67, 68, 72, 76, 84, 85, 89, 102, 103, 104, 107, 108, 278, 121, 311, 376, 248, 138, 318, 159, 41, 321, 145, 335, 236, 317, 4, 372, 245, 199, 64, 370, 124, 362, 375, 377, 371, 101, 213, 279, 228, 30, 45, 273, 119, 220, 240, 78, 171, 146, 334, 336, 180, 36, 282, 363, 210, 201, 219, 186, 355, 352, 65, 244, 80, 359, 0, 179, 15, 152, 61, 205, 69, 168, 301, 1, 79, 189, 239, 328, 109, 117, 118, 127, 129, 131, 134, 135, 139, 142, 147, 148, 149, 150, 151, 155, 156, 157, 167, 169, 170, 175, 181, 183, 190, 191, 192, 196, 116, 111, 122, 160, 324, 195, 325, 182, 187, 229, 351, 174, 58, 341, 292, 232, 99, 123, 158, 77, 259, 178, 209, 293, 284, 207, 246, 162, 93, 8, 140, 345, 14, 235, 184, 91, 13, 338, 267, 39, 130, 92, 258, 40, 185, 120, 7, 254, 18, 42, 272, 202, 54, 350, 222, 208, 55, 211, 74, 281, 83, 197, 343, 27, 96, 212, 251, 361, 198, 206, 214, 215, 230, 231, 234, 237, 241, 242, 243, 247, 249, 255, 257, 261, 262, 263, 266, 269, 271, 276, 277, 280, 283, 285, 286, 288, 381, 105, 161, 10, 82, 106, 2, 112, 97, 71, 290, 287, 353, 43, 95, 315, 193, 57, 25, 132, 380, 153, 342, 322, 364, 172, 378, 11, 86, 347, 200, 295, 49, 87, 165, 100, 302, 98, 114, 379, 32, 250, 19, 22, 319, 313, 368, 203, 173, 53, 305, 265, 9, 21, 154, 275, 268, 94, 128, 310, 143, 29, 176, 20, 166, 225, 365, 137, 224, 260, 366, 294, 296, 297, 299, 300, 303, 304, 314, 316, 326, 327, 329, 330, 331, 332, 333, 344, 349, 356, 358, 360, 367, 373, 382, 383] + 21: [371, 361, 250, 45, 262, 355, 245, 237, 141, 94, 29, 155, 365, 187, 25, 359, 367, 308, 162, 271, 159, 337, 351, 128, 287, 326, 132, 194, 43, 133, 22, 14, 98, 26, 183, 15, 383, 148, 354, 23, 336, 139, 358, 286, 85, 210, 9, 280, 44, 40, 324, 188, 203, 382, 161, 290, 191, 299, 156, 64, 229, 207, 363, 259, 289, 345, 123, 234, 166, 332, 35, 163, 310, 131, 243, 369, 212, 236, 223, 108, 157, 3, 7, 10, 11, 17, 32, 36, 42, 56, 58, 59, 61, 70, 93, 104, 158, 256, 270, 232, 112, 379, 277, 297, 66, 304, 213, 149, 364, 349, 55, 205, 233, 225, 150, 247, 208, 178, 189, 267, 374, 257, 252, 242, 5, 171, 258, 318, 4, 196, 227, 34, 238, 74, 316, 261, 97, 38, 263, 86, 312, 105, 380, 170, 126, 52, 303, 186, 248, 47, 119, 174, 314, 147, 192, 222, 91, 53, 117, 77, 21, 333, 30, 48, 114, 323, 330, 373, 130, 296, 99, 101, 90, 344, 334, 285, 73, 62, 68, 228, 217, 107, 125, 136, 164, 167, 179, 181, 184, 198, 215, 216, 152, 121, 214, 20, 200, 124, 230, 87, 175, 279, 218, 65, 135, 88, 240, 92, 377, 362, 103, 100, 82, 8, 295, 115, 276, 60, 211, 24, 113, 63, 346, 328, 173, 195, 190, 360, 226, 321, 255, 291, 224, 375, 102, 320, 120, 300, 209, 160, 28, 185, 84, 153, 67, 96, 1, 370, 313, 54, 302, 146, 50, 244, 110, 273, 357, 356, 76, 78, 41, 331, 39, 239, 6, 180, 353, 116, 151, 137, 197, 288, 251, 246, 127, 134, 37, 122, 294, 219, 264, 265, 266, 269, 272, 278, 282, 292, 281, 51, 182, 254, 80, 2, 199, 284, 352, 172, 221, 220, 71, 378, 376, 144, 19, 129, 293, 89, 348, 143, 12, 46, 33, 13, 325, 169, 16, 341, 366, 275, 18, 368, 75, 274, 342, 301, 83, 206, 27, 204, 372, 0, 140, 118, 168, 165, 202, 176, 260, 106, 340, 347, 142, 249, 298, 72, 109, 111, 338, 307, 57, 235, 69, 95, 81, 79, 283, 381, 317, 241, 309, 145, 177, 253, 31, 193, 231, 306, 268, 343, 154, 201, 49, 138, 329, 305, 311, 315, 319, 322, 327, 335, 339, 350] + 22: [73, 118, 329, 164, 226, 376, 51, 197, 15, 126, 170, 132, 251, 180, 305, 21, 330, 122, 179, 314, 205, 371, 184, 56, 59, 326, 352, 189, 68, 264, 255, 268, 347, 362, 69, 78, 292, 46, 19, 99, 322, 171, 232, 152, 138, 8, 85, 319, 369, 28, 340, 327, 277, 219, 18, 90, 137, 97, 168, 158, 301, 378, 26, 203, 0, 10, 140, 295, 1, 7, 12, 14, 30, 34, 35, 39, 40, 41, 43, 45, 47, 48, 49, 52, 55, 77, 81, 84, 86, 91, 93, 95, 96, 100, 102, 103, 209, 328, 224, 148, 153, 240, 216, 178, 80, 61, 139, 339, 299, 79, 215, 318, 221, 201, 252, 306, 214, 361, 50, 238, 151, 281, 293, 289, 58, 334, 37, 200, 66, 76, 38, 191, 294, 156, 227, 309, 272, 94, 92, 33, 188, 70, 22, 173, 6, 89, 364, 3, 218, 320, 379, 242, 32, 98, 287, 75, 36, 88, 206, 303, 2, 67, 235, 62, 150, 300, 104, 105, 107, 114, 119, 120, 121, 124, 128, 129, 131, 135, 136, 142, 143, 145, 146, 154, 157, 159, 161, 162, 165, 166, 167, 176, 343, 141, 357, 125, 348, 63, 24, 304, 111, 345, 110, 333, 198, 274, 351, 311, 54, 278, 244, 183, 42, 367, 193, 307, 174, 134, 72, 82, 123, 195, 366, 20, 273, 279, 11, 324, 363, 163, 5, 60, 115, 285, 283, 108, 269, 130, 365, 172, 263, 321, 25, 17, 31, 116, 249, 296, 53, 302, 335, 286, 64, 101, 213, 332, 13, 83, 239, 65, 233, 358, 181, 182, 186, 187, 190, 192, 194, 199, 202, 208, 217, 222, 223, 230, 234, 236, 237, 241, 243, 245, 247, 248, 250, 253, 256, 257, 317, 207, 260, 144, 147, 169, 204, 337, 177, 288, 74, 113, 261, 355, 381, 377, 175, 275, 331, 349, 246, 336, 211, 155, 27, 338, 228, 341, 312, 225, 220, 276, 133, 29, 229, 258, 109, 316, 254, 160, 87, 259, 106, 382, 360, 374, 23, 149, 9, 212, 4, 44, 298, 210, 57, 185, 380, 284, 383, 196, 117, 112, 231, 356, 16, 127, 325, 71, 270, 373, 262, 265, 266, 267, 271, 280, 282, 290, 291, 297, 308, 310, 313, 315, 323, 342, 344, 346, 350, 353, 354, 359, 368, 370, 372, 375] + 23: [172, 232, 374, 82, 86, 49, 170, 203, 123, 4, 299, 254, 279, 370, 100, 230, 292, 261, 1, 36, 132, 371, 336, 185, 217, 201, 219, 270, 125, 210, 349, 308, 361, 21, 282, 164, 341, 335, 0, 84, 296, 277, 30, 216, 354, 24, 151, 77, 97, 221, 145, 225, 91, 351, 15, 87, 169, 8, 206, 350, 284, 44, 41, 199, 149, 33, 209, 139, 278, 207, 96, 177, 37, 179, 150, 259, 188, 141, 345, 121, 120, 250, 303, 236, 75, 242, 13, 14, 29, 31, 64, 88, 110, 113, 157, 168, 26, 327, 5, 146, 274, 63, 339, 133, 295, 156, 340, 193, 180, 222, 78, 54, 329, 342, 297, 117, 38, 196, 346, 98, 50, 355, 147, 249, 247, 42, 127, 119, 122, 262, 32, 67, 239, 115, 108, 138, 287, 256, 109, 165, 101, 19, 154, 17, 306, 47, 103, 106, 45, 362, 365, 360, 233, 328, 255, 289, 23, 85, 271, 215, 40, 300, 227, 375, 280, 118, 128, 28, 382, 46, 311, 364, 160, 27, 211, 356, 372, 135, 61, 3, 316, 171, 114, 83, 89, 363, 198, 202, 214, 235, 265, 276, 167, 80, 358, 220, 111, 337, 197, 94, 253, 70, 130, 229, 309, 237, 184, 204, 52, 264, 307, 218, 74, 379, 373, 48, 53, 290, 324, 315, 331, 107, 263, 79, 353, 10, 73, 240, 245, 93, 269, 321, 228, 182, 7, 51, 258, 344, 191, 190, 367, 99, 159, 326, 174, 241, 55, 140, 267, 377, 153, 334, 20, 238, 305, 369, 9, 347, 266, 12, 137, 275, 381, 183, 60, 333, 260, 43, 318, 298, 205, 310, 293, 243, 294, 56, 57, 223, 192, 383, 212, 272, 134, 268, 288, 291, 301, 319, 352, 131, 314, 187, 176, 148, 213, 173, 72, 378, 102, 359, 285, 158, 58, 2, 376, 92, 155, 35, 273, 143, 313, 162, 332, 11, 322, 208, 6, 152, 195, 22, 304, 81, 163, 338, 317, 312, 161, 244, 323, 90, 124, 325, 257, 105, 104, 348, 248, 76, 226, 380, 283, 178, 234, 25, 252, 200, 66, 166, 62, 95, 320, 302, 129, 186, 116, 112, 343, 175, 142, 68, 224, 194, 246, 126, 59, 71, 18, 16, 144, 231, 65, 281, 181, 34, 357, 69, 39, 251, 136, 189, 286, 330, 366, 368] + 24: [70, 213, 122, 220, 237, 1, 226, 211, 202, 316, 210, 371, 257, 12, 182, 178, 381, 93, 8, 6, 41, 38, 382, 50, 258, 114, 181, 37, 88, 204, 262, 110, 203, 18, 357, 188, 345, 31, 306, 61, 366, 168, 347, 358, 218, 349, 155, 215, 108, 326, 145, 275, 286, 183, 310, 130, 185, 373, 106, 243, 90, 40, 47, 184, 285, 153, 120, 105, 305, 162, 298, 100, 376, 55, 56, 271, 160, 225, 10, 23, 32, 34, 36, 45, 57, 69, 74, 75, 76, 83, 98, 113, 115, 125, 129, 132, 246, 343, 301, 131, 290, 375, 283, 244, 344, 325, 137, 97, 350, 139, 201, 200, 287, 330, 319, 144, 263, 63, 269, 147, 84, 308, 4, 297, 333, 27, 190, 221, 189, 248, 229, 295, 96, 20, 261, 156, 302, 255, 216, 87, 157, 360, 224, 163, 146, 327, 117, 377, 152, 317, 328, 140, 66, 242, 186, 284, 126, 208, 133, 372, 362, 119, 197, 165, 228, 127, 300, 16, 25, 214, 85, 179, 369, 177, 81, 167, 60, 359, 176, 227, 143, 149, 161, 166, 169, 170, 173, 194, 196, 217, 240, 241, 0, 46, 5, 109, 195, 266, 118, 121, 28, 82, 172, 35, 142, 104, 318, 135, 299, 231, 256, 150, 95, 9, 15, 264, 289, 116, 253, 128, 73, 324, 151, 159, 293, 26, 39, 273, 282, 14, 44, 79, 268, 277, 103, 361, 124, 192, 365, 58, 307, 171, 315, 72, 320, 348, 338, 91, 64, 280, 30, 233, 279, 89, 54, 3, 33, 335, 234, 340, 199, 123, 193, 21, 134, 223, 48, 232, 368, 29, 77, 219, 180, 68, 2, 99, 187, 239, 249, 250, 251, 267, 270, 278, 281, 291, 303, 312, 92, 274, 154, 311, 265, 259, 86, 337, 175, 354, 43, 370, 136, 7, 342, 212, 112, 230, 323, 341, 355, 374, 51, 336, 49, 352, 62, 276, 111, 78, 254, 80, 53, 252, 321, 294, 138, 52, 260, 42, 191, 65, 11, 22, 59, 332, 102, 334, 13, 207, 71, 288, 205, 380, 236, 209, 322, 206, 245, 164, 346, 238, 67, 148, 235, 247, 363, 339, 313, 329, 19, 101, 174, 304, 17, 272, 158, 292, 94, 296, 222, 24, 141, 107, 198, 309, 314, 331, 351, 353, 356, 364, 367, 378, 379, 383] + 25: [379, 328, 80, 307, 327, 351, 89, 131, 359, 37, 222, 188, 382, 238, 365, 51, 340, 299, 194, 204, 196, 77, 68, 82, 192, 224, 0, 244, 40, 272, 24, 242, 305, 54, 201, 363, 228, 260, 368, 78, 35, 259, 383, 308, 128, 282, 312, 170, 109, 302, 30, 300, 235, 288, 23, 1, 292, 345, 94, 186, 112, 61, 184, 86, 58, 135, 198, 321, 344, 182, 189, 171, 338, 208, 213, 169, 177, 255, 104, 207, 263, 356, 3, 43, 87, 91, 95, 99, 119, 123, 174, 219, 286, 326, 329, 358, 269, 334, 167, 231, 140, 149, 215, 101, 366, 250, 309, 217, 120, 29, 218, 310, 16, 252, 11, 180, 111, 316, 336, 343, 296, 33, 136, 39, 57, 15, 8, 113, 153, 117, 348, 195, 81, 59, 306, 70, 214, 65, 144, 133, 63, 337, 197, 53, 145, 246, 41, 60, 294, 175, 258, 283, 187, 240, 339, 173, 369, 129, 349, 251, 254, 75, 88, 34, 139, 199, 205, 239, 352, 121, 55, 372, 209, 126, 122, 22, 277, 364, 243, 206, 322, 342, 267, 21, 355, 67, 13, 178, 164, 12, 264, 362, 232, 315, 193, 168, 31, 317, 9, 319, 287, 253, 64, 314, 211, 375, 25, 2, 124, 32, 146, 98, 320, 143, 73, 229, 52, 298, 261, 381, 20, 200, 361, 230, 293, 79, 132, 265, 191, 156, 225, 247, 270, 237, 125, 172, 203, 62, 74, 324, 6, 202, 116, 102, 69, 221, 281, 274, 311, 159, 318, 49, 353, 367, 138, 148, 373, 165, 83, 137, 157, 341, 210, 18, 234, 158, 71, 227, 85, 284, 50, 14, 19, 163, 92, 26, 150, 325, 160, 155, 303, 127, 257, 220, 118, 374, 152, 301, 354, 107, 72, 56, 278, 271, 332, 162, 27, 5, 291, 161, 76, 114, 280, 48, 266, 256, 262, 333, 313, 377, 115, 371, 100, 304, 295, 285, 268, 347, 46, 190, 44, 183, 241, 42, 273, 17, 335, 38, 10, 47, 103, 7, 84, 212, 97, 66, 28, 279, 216, 110, 4, 297, 108, 330, 276, 179, 249, 185, 346, 130, 350, 360, 380, 376, 142, 357, 105, 290, 248, 96, 275, 154, 370, 331, 233, 236, 378, 93, 166, 36, 289, 181, 323, 45, 90, 106, 147, 176, 223, 151, 134, 245, 226, 141] + 26: [217, 249, 113, 92, 128, 48, 328, 195, 108, 237, 293, 93, 272, 371, 367, 67, 216, 277, 188, 224, 318, 135, 329, 30, 2, 221, 126, 74, 232, 148, 44, 11, 77, 266, 94, 76, 223, 127, 134, 378, 159, 152, 192, 20, 308, 9, 370, 103, 335, 117, 331, 353, 212, 171, 349, 375, 352, 376, 301, 66, 88, 101, 325, 351, 98, 157, 64, 57, 305, 10, 43, 187, 313, 1, 3, 4, 14, 19, 22, 23, 25, 28, 29, 35, 47, 52, 54, 56, 58, 60, 63, 68, 73, 81, 84, 85, 90, 281, 27, 142, 337, 262, 89, 254, 278, 300, 294, 215, 15, 204, 137, 310, 62, 359, 183, 256, 342, 279, 174, 269, 55, 364, 110, 239, 184, 34, 231, 124, 156, 186, 220, 315, 24, 69, 245, 107, 116, 176, 327, 280, 5, 317, 208, 163, 6, 291, 314, 296, 298, 198, 160, 13, 97, 302, 100, 206, 109, 123, 46, 211, 168, 12, 377, 191, 111, 36, 326, 86, 225, 324, 91, 96, 115, 119, 121, 122, 132, 139, 141, 143, 145, 147, 150, 153, 169, 180, 185, 193, 194, 197, 199, 203, 380, 304, 227, 271, 120, 173, 360, 125, 21, 290, 170, 102, 251, 140, 138, 83, 182, 78, 165, 179, 282, 265, 344, 80, 236, 196, 59, 202, 334, 356, 230, 238, 382, 32, 258, 343, 241, 222, 275, 41, 233, 114, 363, 65, 190, 8, 154, 229, 289, 7, 161, 40, 244, 16, 37, 316, 144, 50, 347, 205, 149, 164, 366, 248, 285, 71, 49, 379, 247, 75, 332, 105, 255, 345, 207, 209, 210, 213, 218, 219, 228, 234, 235, 240, 242, 243, 246, 250, 252, 257, 259, 261, 264, 267, 268, 270, 136, 181, 338, 276, 214, 118, 146, 200, 112, 172, 178, 53, 38, 31, 70, 106, 341, 151, 131, 39, 61, 95, 323, 339, 321, 226, 263, 167, 166, 306, 355, 357, 162, 177, 18, 311, 79, 33, 129, 45, 130, 383, 336, 158, 320, 51, 87, 286, 82, 292, 312, 361, 17, 374, 0, 307, 303, 155, 340, 201, 72, 26, 354, 104, 99, 369, 175, 365, 253, 260, 189, 42, 133, 284, 346, 273, 274, 283, 287, 288, 295, 297, 299, 309, 319, 322, 330, 333, 348, 350, 358, 362, 368, 372, 373, 381] + 27: [103, 325, 10, 228, 50, 305, 133, 304, 371, 140, 28, 151, 171, 269, 78, 29, 178, 111, 131, 374, 37, 27, 324, 296, 49, 323, 105, 244, 109, 58, 132, 121, 254, 95, 266, 288, 52, 114, 321, 306, 327, 64, 316, 261, 226, 62, 232, 134, 60, 76, 284, 335, 146, 202, 170, 301, 139, 1, 136, 345, 61, 164, 283, 219, 44, 147, 200, 355, 231, 172, 287, 166, 130, 91, 41, 275, 319, 17, 154, 315, 63, 148, 243, 169, 367, 8, 51, 53, 79, 96, 137, 182, 206, 212, 255, 257, 32, 360, 98, 347, 89, 113, 141, 42, 25, 168, 349, 127, 252, 197, 73, 370, 379, 224, 277, 356, 116, 150, 71, 190, 188, 125, 120, 351, 250, 292, 218, 6, 217, 242, 106, 238, 173, 308, 294, 193, 162, 70, 334, 75, 68, 369, 149, 365, 93, 233, 31, 357, 124, 350, 207, 180, 189, 330, 67, 38, 11, 87, 208, 271, 279, 340, 14, 184, 352, 235, 122, 145, 20, 297, 331, 358, 186, 227, 363, 326, 138, 346, 36, 156, 220, 336, 101, 161, 256, 286, 203, 39, 4, 163, 270, 302, 225, 214, 12, 23, 260, 251, 16, 329, 85, 337, 118, 21, 303, 128, 353, 104, 198, 55, 343, 88, 258, 354, 48, 159, 259, 267, 187, 90, 339, 348, 94, 83, 376, 13, 181, 320, 280, 115, 5, 237, 86, 341, 272, 3, 185, 82, 199, 264, 80, 248, 84, 229, 281, 215, 377, 56, 99, 33, 9, 282, 239, 45, 97, 195, 313, 192, 26, 191, 69, 15, 299, 194, 167, 276, 155, 373, 278, 312, 24, 157, 205, 201, 307, 152, 117, 74, 247, 160, 285, 92, 240, 129, 300, 236, 332, 338, 333, 81, 177, 66, 222, 175, 265, 234, 213, 72, 230, 54, 7, 381, 0, 273, 143, 18, 317, 328, 362, 249, 263, 47, 34, 245, 183, 110, 318, 383, 344, 65, 246, 30, 204, 368, 210, 102, 382, 380, 291, 298, 295, 290, 211, 112, 274, 142, 135, 179, 309, 262, 241, 144, 35, 77, 126, 359, 366, 123, 158, 174, 375, 293, 289, 176, 22, 196, 108, 119, 216, 107, 59, 165, 314, 253, 153, 223, 268, 322, 221, 2, 364, 19, 46, 57, 100, 40, 361, 310, 209, 311, 43, 342, 372, 378] + 28: [310, 312, 201, 285, 179, 67, 3, 21, 148, 22, 112, 324, 243, 133, 224, 51, 252, 210, 25, 237, 7, 227, 321, 327, 144, 366, 221, 375, 1, 320, 268, 218, 158, 378, 70, 368, 311, 162, 130, 187, 104, 297, 209, 207, 223, 214, 77, 247, 98, 337, 125, 260, 177, 193, 184, 316, 334, 269, 63, 15, 374, 296, 56, 65, 283, 295, 82, 190, 246, 9, 258, 188, 46, 18, 315, 10, 85, 372, 139, 354, 145, 254, 89, 73, 336, 34, 123, 27, 270, 54, 212, 279, 96, 147, 160, 257, 317, 121, 197, 97, 176, 66, 289, 356, 100, 182, 16, 24, 170, 52, 369, 267, 370, 90, 241, 303, 166, 57, 341, 116, 53, 110, 32, 350, 314, 228, 5, 64, 44, 259, 349, 213, 80, 274, 363, 128, 230, 383, 308, 59, 171, 167, 181, 191, 273, 211, 276, 286, 37, 322, 272, 149, 300, 88, 131, 81, 345, 265, 359, 234, 192, 156, 309, 361, 76, 159, 189, 6, 151, 60, 39, 284, 351, 244, 103, 373, 12, 132, 118, 292, 94, 36, 115, 275, 173, 29, 256, 301, 290, 294, 298, 313, 253, 26, 0, 250, 307, 196, 169, 71, 376, 13, 165, 48, 205, 72, 172, 183, 262, 288, 157, 380, 20, 357, 107, 134, 305, 318, 175, 174, 352, 264, 362, 95, 153, 219, 346, 333, 238, 239, 137, 117, 233, 367, 280, 43, 17, 91, 99, 114, 225, 331, 120, 266, 249, 277, 271, 122, 152, 371, 140, 178, 358, 216, 281, 79, 329, 323, 339, 69, 245, 23, 33, 31, 106, 232, 119, 236, 291, 40, 55, 78, 126, 62, 231, 47, 141, 93, 186, 150, 365, 11, 146, 338, 348, 342, 343, 355, 255, 220, 235, 135, 328, 199, 163, 41, 302, 30, 222, 195, 108, 168, 278, 203, 344, 204, 326, 155, 50, 109, 226, 194, 92, 248, 105, 200, 263, 42, 242, 217, 61, 154, 306, 287, 206, 161, 240, 185, 340, 379, 330, 113, 129, 143, 4, 102, 14, 49, 229, 293, 208, 347, 198, 19, 35, 28, 124, 136, 332, 68, 38, 180, 58, 111, 202, 282, 299, 325, 84, 364, 382, 319, 74, 215, 87, 45, 251, 101, 8, 335, 142, 138, 353, 127, 2, 304, 86, 83, 164, 75, 261, 360, 377, 381] + 29: [197, 83, 277, 346, 140, 11, 132, 218, 116, 301, 180, 73, 253, 190, 214, 86, 19, 185, 349, 22, 62, 286, 113, 300, 318, 139, 175, 145, 3, 101, 340, 81, 353, 276, 2, 275, 187, 203, 287, 350, 146, 303, 94, 115, 198, 229, 5, 126, 144, 160, 103, 166, 290, 307, 117, 56, 359, 37, 322, 257, 309, 281, 51, 155, 252, 77, 68, 12, 194, 249, 192, 335, 15, 130, 321, 246, 107, 31, 82, 376, 6, 8, 10, 13, 29, 47, 69, 71, 79, 91, 96, 121, 138, 164, 165, 178, 127, 357, 314, 283, 255, 232, 4, 269, 110, 245, 215, 207, 59, 244, 195, 339, 18, 172, 345, 120, 99, 206, 330, 50, 55, 90, 250, 34, 235, 278, 291, 271, 361, 295, 325, 317, 188, 125, 323, 57, 142, 65, 143, 137, 64, 292, 167, 254, 52, 263, 299, 30, 92, 184, 289, 39, 227, 179, 355, 199, 338, 61, 364, 363, 16, 26, 98, 348, 123, 157, 268, 226, 134, 328, 191, 216, 381, 141, 372, 284, 35, 0, 306, 326, 17, 28, 239, 334, 135, 181, 95, 20, 66, 201, 212, 224, 74, 243, 311, 294, 85, 365, 280, 270, 312, 362, 375, 341, 148, 53, 259, 14, 324, 119, 360, 293, 228, 231, 327, 344, 36, 87, 333, 25, 354, 374, 196, 186, 208, 104, 320, 304, 370, 251, 124, 105, 273, 236, 129, 262, 156, 72, 7, 221, 319, 154, 274, 111, 264, 161, 308, 233, 222, 75, 67, 225, 209, 237, 298, 168, 131, 21, 63, 379, 23, 242, 234, 1, 373, 42, 122, 70, 329, 315, 93, 9, 106, 238, 282, 136, 24, 152, 337, 49, 205, 27, 153, 247, 305, 240, 260, 261, 174, 351, 266, 97, 128, 88, 40, 58, 248, 297, 352, 118, 382, 162, 149, 33, 44, 367, 48, 366, 78, 43, 151, 210, 223, 60, 369, 183, 310, 267, 219, 313, 279, 258, 80, 46, 220, 347, 114, 177, 368, 163, 343, 54, 316, 200, 32, 133, 112, 272, 159, 332, 380, 296, 171, 76, 45, 256, 100, 182, 204, 173, 383, 213, 241, 230, 41, 147, 84, 265, 193, 211, 109, 38, 189, 302, 358, 170, 108, 158, 217, 102, 336, 342, 89, 150, 202, 356, 331, 371, 176, 169, 378, 285, 288, 377] + 30: [239, 62, 144, 166, 28, 213, 362, 230, 298, 337, 60, 24, 272, 253, 198, 48, 246, 358, 292, 378, 303, 116, 163, 260, 193, 184, 289, 12, 273, 361, 196, 153, 212, 27, 147, 327, 96, 86, 352, 235, 300, 162, 141, 115, 247, 256, 41, 122, 309, 324, 133, 328, 43, 156, 5, 112, 132, 91, 173, 329, 72, 107, 299, 209, 342, 179, 314, 307, 367, 8, 148, 252, 19, 368, 231, 93, 42, 366, 75, 16, 295, 178, 114, 192, 313, 98, 297, 30, 21, 31, 32, 34, 36, 69, 79, 81, 37, 296, 233, 111, 6, 195, 248, 265, 318, 109, 245, 249, 250, 223, 52, 187, 97, 68, 221, 288, 63, 293, 200, 308, 160, 151, 139, 92, 172, 263, 217, 359, 261, 7, 283, 341, 51, 110, 129, 183, 325, 312, 332, 274, 199, 291, 128, 113, 170, 150, 335, 174, 104, 13, 87, 316, 157, 370, 26, 67, 25, 319, 154, 108, 39, 267, 186, 365, 229, 372, 301, 131, 219, 18, 176, 340, 57, 136, 134, 380, 374, 9, 56, 117, 232, 47, 149, 4, 169, 82, 88, 100, 106, 119, 127, 145, 140, 137, 65, 50, 53, 262, 121, 78, 11, 258, 38, 90, 152, 77, 14, 281, 89, 171, 224, 346, 158, 242, 351, 349, 66, 211, 142, 29, 210, 1, 167, 125, 376, 241, 205, 138, 99, 234, 320, 348, 161, 3, 220, 20, 143, 373, 271, 302, 315, 306, 379, 286, 343, 282, 227, 257, 226, 40, 215, 331, 240, 55, 103, 338, 311, 95, 275, 76, 181, 350, 165, 182, 243, 326, 251, 177, 330, 323, 126, 268, 83, 194, 276, 382, 218, 334, 146, 322, 228, 180, 185, 189, 190, 201, 237, 259, 208, 222, 277, 254, 381, 206, 2, 46, 317, 35, 164, 344, 202, 225, 357, 135, 383, 64, 10, 266, 23, 321, 203, 284, 197, 105, 244, 294, 71, 44, 285, 371, 280, 124, 155, 102, 214, 94, 369, 353, 333, 130, 84, 347, 33, 279, 175, 15, 287, 336, 360, 58, 120, 74, 45, 339, 304, 0, 290, 59, 236, 70, 270, 364, 255, 85, 188, 207, 80, 17, 159, 355, 216, 356, 264, 123, 49, 118, 375, 61, 238, 204, 73, 101, 168, 305, 54, 191, 22, 310, 269, 278, 345, 354, 363, 377] + 31: [288, 361, 123, 160, 82, 339, 176, 88, 240, 284, 146, 169, 60, 86, 70, 378, 61, 375, 45, 335, 277, 296, 336, 227, 290, 287, 177, 190, 2, 40, 195, 96, 78, 62, 275, 244, 34, 122, 267, 143, 357, 93, 219, 111, 105, 173, 8, 52, 249, 364, 368, 334, 319, 158, 145, 19, 344, 234, 209, 117, 136, 194, 382, 255, 273, 116, 247, 302, 36, 17, 212, 118, 216, 85, 80, 370, 217, 213, 300, 13, 35, 329, 120, 165, 73, 239, 127, 228, 24, 203, 95, 181, 347, 175, 207, 301, 248, 299, 253, 112, 153, 326, 124, 260, 383, 186, 252, 59, 317, 321, 183, 377, 269, 307, 359, 338, 87, 3, 178, 223, 131, 140, 272, 263, 57, 47, 79, 238, 292, 220, 200, 188, 44, 243, 320, 128, 332, 109, 167, 27, 29, 346, 144, 65, 107, 257, 171, 66, 137, 106, 254, 304, 15, 291, 193, 81, 282, 328, 362, 180, 351, 310, 148, 198, 179, 265, 280, 129, 215, 1, 199, 197, 259, 226, 168, 185, 113, 142, 68, 51, 298, 149, 246, 10, 318, 210, 352, 11, 343, 54, 162, 67, 159, 365, 360, 309, 125, 108, 39, 222, 75, 314, 354, 241, 102, 134, 191, 206, 214, 201, 58, 98, 261, 72, 139, 49, 110, 281, 84, 278, 293, 232, 224, 182, 196, 38, 379, 147, 174, 274, 312, 6, 4, 16, 286, 130, 77, 353, 56, 271, 12, 33, 92, 43, 18, 285, 31, 325, 211, 20, 250, 155, 297, 306, 303, 163, 91, 311, 337, 152, 99, 184, 374, 104, 69, 340, 161, 100, 356, 205, 225, 251, 294, 218, 341, 187, 381, 322, 90, 114, 372, 235, 135, 262, 172, 126, 229, 89, 115, 121, 237, 380, 138, 156, 376, 221, 264, 231, 23, 245, 133, 55, 25, 202, 270, 305, 26, 32, 316, 313, 349, 366, 0, 330, 170, 236, 164, 9, 355, 373, 103, 22, 324, 189, 345, 48, 276, 208, 21, 83, 348, 371, 157, 192, 141, 119, 258, 233, 266, 327, 242, 94, 97, 71, 331, 154, 166, 150, 333, 53, 363, 256, 308, 30, 342, 295, 63, 5, 367, 74, 268, 7, 46, 64, 76, 323, 41, 279, 283, 289, 369, 50, 14, 151, 42, 132, 358, 315, 101, 350, 230, 204, 28, 37] + 32: [148, 89, 83, 86, 238, 33, 237, 166, 45, 218, 120, 307, 348, 179, 175, 105, 221, 16, 194, 7, 334, 282, 190, 293, 322, 151, 53, 77, 380, 88, 110, 254, 60, 212, 126, 319, 87, 76, 150, 41, 213, 340, 231, 240, 377, 79, 280, 69, 292, 1, 129, 306, 32, 369, 18, 195, 121, 270, 315, 331, 178, 233, 154, 9, 128, 114, 370, 52, 248, 226, 314, 133, 199, 374, 31, 354, 3, 176, 350, 259, 379, 366, 295, 162, 260, 267, 97, 74, 38, 242, 62, 235, 11, 78, 117, 147, 106, 108, 227, 347, 55, 4, 229, 336, 335, 48, 171, 202, 40, 249, 58, 258, 243, 200, 136, 113, 153, 273, 261, 241, 368, 209, 21, 96, 19, 371, 297, 323, 68, 360, 49, 268, 274, 82, 339, 247, 342, 345, 51, 364, 278, 294, 232, 107, 122, 329, 70, 101, 239, 356, 10, 201, 35, 100, 164, 337, 134, 34, 90, 266, 288, 223, 376, 355, 119, 262, 43, 224, 152, 98, 289, 308, 37, 183, 296, 93, 91, 118, 244, 302, 245, 2, 80, 299, 198, 251, 135, 316, 161, 169, 180, 264, 163, 184, 367, 63, 168, 281, 157, 359, 324, 276, 160, 284, 85, 191, 131, 250, 139, 142, 219, 332, 214, 187, 256, 141, 286, 193, 36, 230, 330, 149, 59, 317, 72, 211, 127, 326, 279, 186, 67, 23, 271, 203, 353, 351, 263, 8, 298, 349, 313, 12, 42, 215, 197, 103, 22, 145, 343, 73, 66, 210, 170, 39, 328, 167, 321, 310, 225, 320, 46, 125, 275, 64, 358, 216, 318, 309, 124, 13, 54, 333, 47, 28, 112, 253, 123, 300, 44, 192, 14, 255, 291, 338, 272, 277, 290, 301, 362, 344, 352, 26, 24, 172, 99, 382, 155, 174, 285, 325, 252, 92, 5, 146, 143, 228, 156, 111, 165, 283, 6, 361, 206, 144, 75, 0, 303, 130, 205, 182, 327, 365, 109, 189, 137, 30, 207, 346, 95, 208, 140, 61, 378, 115, 246, 159, 234, 29, 204, 217, 25, 287, 50, 222, 116, 104, 357, 158, 196, 132, 94, 65, 269, 138, 363, 311, 102, 341, 373, 188, 265, 71, 181, 17, 15, 173, 84, 236, 81, 57, 372, 185, 304, 257, 177, 305, 27, 56, 20, 220, 381, 312, 375, 383] + 33: [91, 169, 377, 344, 342, 307, 38, 158, 51, 87, 255, 101, 94, 288, 191, 62, 14, 134, 323, 305, 132, 25, 79, 188, 170, 45, 182, 149, 318, 358, 347, 22, 233, 52, 85, 109, 67, 361, 72, 163, 141, 258, 335, 340, 267, 295, 212, 380, 360, 264, 123, 2, 226, 20, 222, 187, 362, 337, 356, 23, 197, 18, 110, 351, 143, 155, 312, 11, 116, 370, 284, 330, 136, 228, 53, 63, 7, 208, 78, 219, 332, 353, 282, 243, 24, 365, 310, 227, 15, 44, 32, 71, 95, 105, 128, 185, 177, 234, 167, 154, 66, 328, 338, 303, 261, 139, 229, 135, 183, 98, 41, 339, 273, 150, 204, 213, 130, 37, 8, 90, 382, 334, 202, 160, 240, 291, 343, 174, 374, 363, 6, 179, 84, 369, 224, 294, 138, 111, 9, 168, 359, 118, 12, 92, 10, 271, 100, 74, 315, 215, 157, 203, 50, 259, 248, 193, 235, 88, 230, 147, 21, 176, 27, 327, 241, 30, 16, 120, 201, 283, 251, 218, 126, 194, 189, 252, 244, 345, 114, 198, 97, 47, 119, 69, 253, 56, 186, 223, 266, 287, 298, 306, 127, 190, 262, 280, 89, 232, 59, 4, 270, 286, 319, 237, 320, 260, 381, 348, 151, 125, 217, 61, 153, 104, 31, 354, 57, 172, 80, 297, 205, 199, 254, 60, 124, 355, 40, 148, 275, 42, 162, 268, 129, 166, 39, 77, 366, 281, 206, 17, 250, 70, 164, 352, 103, 256, 313, 152, 367, 178, 290, 137, 214, 296, 209, 102, 195, 207, 26, 28, 173, 145, 112, 285, 161, 33, 184, 324, 292, 272, 372, 265, 156, 49, 304, 211, 257, 274, 13, 142, 29, 302, 236, 245, 309, 317, 326, 333, 144, 64, 113, 376, 48, 115, 108, 210, 81, 316, 314, 322, 5, 35, 311, 73, 325, 192, 375, 36, 321, 54, 238, 308, 289, 133, 121, 379, 146, 93, 247, 200, 299, 3, 329, 58, 1, 196, 96, 159, 75, 331, 181, 131, 99, 231, 349, 364, 249, 180, 171, 276, 269, 19, 300, 46, 246, 107, 43, 263, 76, 122, 371, 86, 68, 83, 82, 279, 350, 117, 65, 165, 242, 55, 341, 378, 216, 368, 221, 346, 175, 239, 225, 106, 140, 278, 277, 220, 0, 383, 336, 301, 34, 293, 357, 373] + 34: [179, 239, 136, 360, 379, 74, 283, 81, 293, 331, 294, 188, 298, 192, 348, 260, 332, 65, 80, 85, 23, 79, 104, 190, 97, 110, 148, 195, 342, 326, 120, 232, 364, 245, 131, 324, 314, 221, 270, 382, 155, 146, 305, 27, 142, 355, 344, 184, 267, 92, 353, 114, 210, 278, 118, 199, 170, 319, 77, 187, 67, 241, 227, 176, 209, 13, 175, 191, 50, 102, 68, 318, 115, 111, 35, 234, 316, 216, 215, 39, 252, 141, 11, 106, 189, 121, 309, 14, 19, 26, 38, 40, 43, 55, 56, 62, 16, 366, 288, 359, 333, 301, 87, 249, 206, 126, 345, 247, 185, 162, 158, 258, 54, 8, 4, 28, 72, 220, 123, 75, 226, 367, 350, 1, 208, 7, 41, 99, 280, 137, 31, 151, 17, 264, 330, 153, 48, 37, 88, 255, 362, 323, 9, 116, 295, 222, 130, 70, 250, 160, 163, 29, 211, 200, 224, 147, 117, 341, 165, 373, 266, 328, 186, 302, 94, 42, 282, 90, 140, 322, 164, 84, 251, 339, 343, 169, 378, 149, 86, 143, 193, 173, 310, 66, 71, 73, 76, 101, 103, 105, 112, 119, 107, 218, 238, 133, 306, 256, 354, 290, 15, 262, 203, 312, 171, 178, 60, 246, 337, 311, 159, 196, 59, 61, 230, 64, 2, 174, 47, 327, 82, 33, 272, 150, 268, 91, 271, 157, 276, 297, 254, 95, 315, 30, 134, 346, 144, 285, 357, 181, 34, 24, 124, 93, 6, 78, 253, 18, 63, 291, 233, 286, 204, 100, 380, 265, 304, 129, 205, 139, 279, 202, 236, 336, 113, 361, 212, 132, 275, 22, 12, 358, 20, 44, 207, 152, 349, 244, 313, 122, 127, 154, 156, 235, 242, 257, 281, 303, 108, 259, 368, 248, 370, 231, 36, 287, 194, 321, 284, 214, 3, 292, 138, 237, 58, 89, 69, 300, 340, 161, 52, 135, 168, 51, 263, 299, 83, 228, 374, 261, 219, 372, 223, 172, 329, 45, 5, 334, 125, 269, 317, 25, 369, 320, 145, 240, 180, 98, 46, 347, 21, 307, 109, 335, 167, 371, 381, 183, 201, 352, 53, 274, 277, 243, 96, 197, 225, 273, 0, 182, 49, 166, 296, 213, 198, 10, 229, 128, 217, 177, 32, 57, 289, 375, 308, 325, 338, 351, 356, 363, 365, 376, 377, 383] + 35: [182, 204, 131, 368, 277, 133, 271, 357, 174, 234, 11, 129, 242, 361, 353, 93, 58, 346, 9, 255, 324, 279, 380, 170, 322, 0, 40, 258, 132, 141, 212, 23, 262, 35, 352, 159, 313, 55, 241, 69, 101, 173, 296, 260, 108, 5, 226, 154, 329, 196, 49, 163, 137, 12, 82, 114, 349, 13, 4, 155, 140, 227, 341, 42, 314, 24, 181, 343, 30, 6, 87, 238, 36, 107, 14, 355, 278, 297, 336, 85, 66, 220, 303, 10, 345, 194, 46, 146, 286, 33, 305, 338, 25, 169, 280, 332, 363, 379, 157, 44, 205, 43, 67, 243, 128, 319, 224, 78, 201, 172, 130, 83, 56, 247, 53, 265, 268, 28, 91, 211, 306, 301, 348, 153, 161, 235, 374, 41, 283, 311, 334, 375, 45, 256, 15, 290, 350, 272, 208, 68, 276, 1, 298, 156, 233, 219, 171, 127, 142, 240, 27, 38, 261, 60, 193, 126, 315, 19, 143, 179, 369, 383, 376, 327, 21, 359, 105, 284, 54, 113, 254, 304, 366, 335, 365, 333, 373, 165, 223, 184, 135, 120, 100, 209, 158, 364, 293, 294, 51, 16, 81, 176, 106, 79, 203, 74, 331, 50, 269, 186, 3, 70, 328, 266, 17, 197, 183, 378, 149, 20, 112, 138, 125, 377, 139, 134, 312, 7, 222, 190, 124, 95, 65, 291, 162, 249, 59, 18, 215, 90, 160, 77, 231, 360, 117, 382, 76, 289, 337, 246, 109, 72, 103, 295, 229, 221, 344, 116, 292, 253, 202, 75, 214, 189, 216, 325, 302, 372, 232, 200, 275, 191, 259, 230, 299, 330, 145, 244, 354, 236, 2, 89, 308, 317, 26, 370, 347, 119, 371, 188, 39, 37, 245, 210, 150, 228, 177, 185, 48, 281, 362, 316, 122, 252, 358, 104, 92, 309, 250, 273, 164, 326, 31, 257, 94, 225, 110, 102, 96, 136, 121, 111, 351, 274, 318, 288, 187, 151, 213, 207, 285, 206, 8, 267, 148, 34, 167, 217, 63, 99, 22, 52, 166, 88, 180, 71, 248, 97, 239, 98, 339, 73, 321, 270, 175, 287, 62, 282, 144, 57, 32, 152, 123, 192, 84, 29, 86, 147, 195, 320, 342, 198, 367, 118, 237, 64, 61, 80, 323, 381, 307, 178, 263, 356, 218, 300, 199, 264, 168, 47, 340, 115, 251, 310] + 36: [304, 155, 352, 5, 293, 193, 138, 97, 158, 365, 247, 210, 374, 220, 176, 199, 265, 212, 364, 81, 299, 95, 71, 356, 94, 233, 160, 266, 362, 117, 101, 65, 17, 282, 110, 116, 88, 188, 99, 26, 7, 15, 174, 231, 255, 150, 225, 10, 245, 129, 119, 251, 242, 109, 18, 21, 56, 295, 149, 259, 66, 337, 120, 241, 367, 72, 326, 157, 29, 335, 173, 368, 190, 64, 307, 89, 204, 183, 55, 334, 222, 172, 148, 239, 346, 37, 221, 107, 96, 115, 123, 124, 253, 298, 313, 330, 232, 291, 331, 280, 274, 122, 76, 340, 370, 238, 207, 211, 285, 215, 201, 373, 103, 75, 297, 92, 12, 179, 156, 121, 187, 20, 283, 226, 44, 350, 131, 102, 227, 267, 108, 182, 135, 74, 114, 154, 73, 351, 60, 339, 258, 31, 163, 180, 263, 281, 53, 296, 240, 301, 269, 132, 235, 306, 8, 316, 314, 376, 224, 347, 83, 34, 284, 145, 151, 289, 234, 52, 33, 358, 3, 270, 147, 127, 11, 68, 320, 79, 361, 51, 128, 104, 372, 143, 186, 16, 336, 375, 360, 170, 353, 355, 22, 310, 134, 380, 200, 61, 100, 194, 14, 319, 359, 45, 196, 153, 322, 177, 162, 69, 46, 349, 198, 19, 342, 98, 272, 191, 161, 23, 287, 311, 181, 67, 219, 357, 354, 213, 112, 230, 82, 38, 144, 189, 140, 125, 366, 9, 246, 6, 244, 32, 303, 93, 195, 371, 85, 345, 50, 382, 341, 276, 42, 118, 70, 59, 136, 254, 277, 47, 363, 130, 206, 159, 165, 228, 338, 203, 292, 54, 378, 111, 250, 305, 302, 261, 214, 43, 39, 113, 252, 290, 315, 126, 86, 377, 0, 58, 197, 178, 184, 332, 348, 309, 288, 323, 321, 275, 237, 260, 264, 328, 278, 175, 308, 343, 168, 164, 57, 248, 87, 30, 2, 344, 257, 333, 137, 141, 62, 209, 249, 78, 171, 49, 139, 105, 369, 300, 294, 383, 192, 229, 35, 41, 185, 216, 77, 256, 133, 271, 48, 36, 166, 63, 80, 91, 318, 262, 217, 273, 279, 167, 381, 28, 379, 317, 329, 4, 208, 90, 202, 205, 236, 27, 106, 142, 325, 24, 84, 13, 1, 40, 152, 324, 25, 146, 169, 223, 312, 286, 243, 268, 327, 218] + 37: [218, 127, 252, 272, 367, 279, 174, 0, 223, 151, 40, 355, 227, 34, 60, 124, 185, 302, 254, 148, 27, 175, 292, 114, 365, 343, 291, 21, 1, 349, 149, 48, 19, 345, 126, 121, 156, 87, 298, 161, 231, 105, 164, 64, 262, 115, 173, 299, 361, 360, 341, 138, 153, 251, 206, 77, 353, 331, 245, 256, 284, 295, 282, 320, 178, 321, 358, 275, 337, 232, 165, 215, 145, 82, 319, 324, 370, 134, 122, 7, 120, 92, 380, 26, 339, 84, 96, 172, 220, 226, 236, 239, 249, 265, 352, 374, 69, 125, 29, 376, 179, 228, 30, 271, 130, 194, 240, 95, 158, 24, 129, 234, 322, 93, 6, 38, 242, 340, 342, 364, 309, 140, 143, 381, 289, 62, 222, 167, 208, 221, 52, 195, 327, 22, 347, 382, 207, 83, 200, 46, 338, 314, 243, 325, 157, 66, 59, 15, 183, 13, 264, 197, 212, 335, 334, 363, 210, 109, 74, 136, 259, 116, 285, 123, 135, 258, 304, 354, 32, 155, 90, 89, 273, 307, 244, 253, 350, 286, 293, 250, 142, 75, 31, 263, 170, 186, 266, 219, 132, 377, 63, 70, 372, 131, 16, 237, 368, 287, 65, 36, 163, 25, 166, 51, 49, 188, 277, 311, 10, 5, 71, 47, 88, 17, 351, 235, 147, 14, 192, 261, 86, 344, 73, 301, 39, 203, 41, 317, 315, 102, 18, 211, 205, 150, 2, 180, 241, 290, 144, 362, 274, 176, 308, 190, 201, 141, 100, 189, 42, 375, 154, 79, 280, 313, 336, 107, 113, 198, 72, 181, 303, 104, 369, 333, 373, 111, 94, 20, 278, 76, 99, 225, 248, 214, 326, 23, 54, 230, 209, 383, 9, 247, 56, 269, 68, 8, 12, 53, 294, 366, 268, 182, 118, 328, 37, 305, 133, 58, 199, 117, 85, 306, 44, 233, 159, 3, 318, 191, 202, 267, 193, 81, 257, 160, 332, 171, 67, 152, 110, 184, 97, 204, 359, 45, 270, 128, 316, 168, 224, 61, 329, 356, 50, 378, 238, 146, 177, 187, 323, 108, 379, 283, 169, 217, 229, 371, 213, 312, 106, 255, 78, 162, 119, 296, 297, 91, 260, 281, 103, 11, 346, 80, 139, 112, 216, 300, 43, 357, 98, 55, 246, 4, 33, 196, 101, 330, 310, 35, 276, 137, 348, 288, 28, 57] + 38: [6, 284, 321, 369, 112, 178, 154, 208, 373, 256, 177, 94, 70, 116, 352, 194, 139, 241, 23, 334, 157, 175, 372, 106, 324, 73, 274, 357, 115, 215, 158, 344, 117, 238, 104, 318, 351, 240, 85, 278, 182, 181, 276, 119, 110, 226, 207, 263, 261, 35, 310, 10, 42, 15, 118, 107, 105, 233, 81, 37, 80, 59, 62, 34, 159, 205, 247, 113, 155, 141, 137, 317, 18, 214, 375, 38, 254, 24, 285, 28, 160, 134, 169, 323, 356, 46, 210, 330, 60, 61, 64, 127, 185, 191, 193, 195, 366, 348, 203, 246, 292, 355, 40, 383, 111, 27, 332, 100, 74, 108, 245, 130, 120, 255, 69, 92, 147, 349, 49, 306, 31, 145, 189, 88, 152, 343, 3, 378, 7, 320, 202, 168, 21, 72, 235, 54, 296, 43, 265, 328, 101, 126, 360, 367, 368, 258, 257, 29, 311, 75, 93, 333, 290, 44, 338, 225, 211, 151, 52, 342, 196, 269, 45, 2, 294, 359, 250, 271, 281, 91, 174, 217, 345, 121, 143, 183, 370, 1, 365, 14, 273, 302, 232, 32, 109, 13, 66, 222, 198, 237, 304, 314, 266, 354, 230, 262, 11, 156, 171, 135, 236, 319, 293, 267, 30, 197, 228, 180, 199, 299, 227, 132, 248, 97, 56, 76, 9, 26, 51, 212, 167, 275, 99, 129, 187, 327, 136, 286, 163, 218, 82, 128, 288, 253, 87, 336, 268, 102, 22, 77, 176, 57, 213, 123, 96, 282, 339, 122, 186, 307, 289, 161, 150, 287, 146, 200, 243, 209, 48, 358, 252, 297, 162, 382, 201, 234, 166, 361, 65, 264, 192, 220, 272, 312, 12, 83, 305, 291, 103, 98, 84, 216, 114, 184, 242, 325, 346, 350, 331, 124, 19, 179, 301, 341, 17, 50, 221, 337, 279, 316, 63, 125, 379, 219, 144, 280, 259, 206, 0, 190, 68, 142, 313, 309, 20, 5, 164, 249, 53, 55, 329, 170, 295, 86, 41, 16, 78, 362, 172, 260, 347, 33, 326, 8, 335, 89, 322, 165, 36, 364, 149, 277, 229, 95, 204, 298, 58, 25, 173, 140, 303, 283, 371, 47, 308, 90, 381, 4, 138, 270, 224, 223, 131, 244, 380, 239, 79, 67, 148, 39, 315, 153, 340, 71, 300, 188, 377, 251, 133, 231, 376, 353, 363, 374] + 39: [102, 100, 353, 336, 187, 297, 372, 56, 319, 112, 248, 322, 323, 141, 14, 258, 87, 93, 213, 151, 244, 221, 28, 281, 161, 356, 54, 363, 254, 208, 284, 5, 41, 172, 185, 84, 135, 154, 58, 95, 25, 139, 13, 226, 344, 291, 148, 124, 261, 107, 17, 16, 65, 230, 326, 146, 167, 309, 289, 30, 33, 66, 365, 39, 178, 266, 232, 376, 205, 2, 217, 145, 334, 229, 293, 90, 311, 122, 355, 20, 158, 321, 371, 382, 270, 119, 219, 330, 186, 27, 171, 111, 216, 184, 142, 350, 282, 236, 127, 110, 15, 170, 153, 223, 163, 383, 290, 260, 367, 7, 18, 129, 347, 333, 227, 203, 252, 242, 80, 200, 40, 101, 11, 303, 286, 169, 246, 214, 312, 339, 36, 253, 197, 245, 181, 277, 292, 251, 378, 285, 77, 196, 125, 79, 283, 206, 23, 233, 76, 194, 37, 373, 324, 173, 366, 123, 234, 201, 55, 329, 348, 50, 255, 190, 168, 45, 250, 263, 152, 143, 211, 331, 131, 209, 276, 231, 224, 308, 375, 117, 38, 150, 73, 115, 317, 301, 341, 327, 195, 136, 19, 4, 86, 352, 116, 274, 220, 368, 315, 177, 199, 359, 369, 118, 212, 155, 144, 62, 235, 351, 247, 377, 60, 164, 133, 298, 126, 380, 82, 360, 204, 113, 257, 81, 160, 32, 121, 104, 210, 147, 31, 69, 26, 71, 320, 70, 97, 192, 132, 162, 182, 49, 46, 103, 279, 337, 106, 239, 294, 105, 310, 159, 98, 267, 166, 91, 228, 271, 120, 176, 92, 52, 51, 10, 0, 342, 357, 259, 157, 78, 287, 193, 165, 137, 243, 85, 64, 130, 9, 189, 314, 265, 22, 198, 370, 89, 108, 240, 134, 96, 180, 191, 328, 280, 379, 1, 264, 306, 300, 295, 68, 307, 114, 53, 99, 61, 225, 75, 374, 140, 138, 88, 381, 156, 48, 362, 34, 275, 272, 183, 332, 59, 63, 67, 296, 218, 335, 202, 249, 325, 313, 128, 299, 364, 273, 305, 175, 8, 6, 268, 354, 35, 3, 109, 346, 43, 302, 74, 207, 358, 174, 215, 24, 21, 316, 47, 237, 188, 343, 349, 83, 29, 179, 42, 222, 338, 262, 94, 269, 149, 345, 72, 238, 44, 304, 12, 288, 241, 278, 57, 256, 318, 361, 340] + 40: [297, 59, 12, 249, 2, 132, 187, 275, 179, 51, 21, 13, 182, 45, 202, 148, 345, 135, 7, 163, 159, 282, 294, 81, 365, 158, 85, 102, 175, 286, 46, 355, 201, 146, 261, 291, 54, 254, 363, 177, 203, 15, 255, 86, 114, 57, 265, 193, 156, 16, 379, 373, 262, 230, 351, 95, 229, 224, 296, 28, 133, 119, 155, 326, 366, 24, 112, 323, 195, 302, 90, 221, 270, 103, 125, 364, 240, 318, 211, 304, 110, 38, 19, 61, 107, 142, 144, 128, 252, 320, 8, 105, 317, 58, 122, 171, 207, 136, 272, 216, 181, 244, 258, 55, 160, 303, 10, 260, 71, 238, 209, 48, 349, 290, 360, 121, 340, 368, 82, 108, 357, 215, 89, 369, 141, 256, 87, 250, 251, 184, 298, 17, 26, 353, 120, 213, 247, 130, 69, 293, 131, 126, 278, 5, 288, 65, 91, 151, 0, 29, 66, 167, 263, 44, 169, 324, 329, 41, 33, 267, 113, 189, 321, 378, 226, 352, 374, 310, 375, 191, 157, 32, 266, 273, 178, 168, 228, 49, 346, 25, 248, 313, 20, 94, 164, 306, 27, 217, 22, 9, 50, 115, 269, 162, 188, 180, 67, 166, 227, 129, 239, 232, 23, 274, 6, 347, 83, 137, 358, 237, 284, 64, 186, 333, 253, 327, 271, 53, 1, 205, 18, 341, 289, 73, 60, 283, 339, 305, 96, 370, 63, 285, 212, 194, 196, 145, 37, 208, 336, 35, 337, 77, 200, 246, 312, 197, 140, 331, 183, 382, 101, 176, 4, 134, 30, 348, 43, 206, 117, 165, 343, 311, 299, 56, 235, 123, 315, 34, 152, 356, 371, 42, 377, 99, 88, 39, 361, 344, 154, 335, 68, 149, 314, 62, 325, 147, 92, 170, 31, 100, 190, 334, 93, 281, 75, 338, 287, 79, 40, 150, 153, 80, 74, 143, 78, 104, 332, 301, 127, 172, 3, 76, 198, 185, 236, 319, 280, 192, 316, 359, 204, 372, 308, 322, 116, 220, 259, 219, 383, 367, 11, 98, 231, 234, 330, 268, 233, 36, 52, 264, 118, 225, 362, 223, 84, 174, 199, 139, 242, 70, 328, 47, 161, 307, 381, 97, 354, 277, 124, 276, 111, 72, 210, 222, 109, 173, 257, 309, 279, 245, 295, 14, 376, 138, 380, 300, 292, 243, 241, 218, 342, 106, 350, 214] + 41: [205, 318, 202, 111, 0, 337, 341, 43, 232, 76, 206, 249, 56, 86, 139, 19, 153, 13, 150, 183, 136, 330, 2, 87, 180, 103, 200, 188, 342, 218, 311, 95, 226, 349, 163, 189, 16, 45, 279, 227, 129, 146, 126, 271, 313, 211, 214, 116, 118, 260, 105, 21, 335, 161, 230, 97, 317, 282, 195, 181, 360, 54, 371, 5, 160, 290, 68, 208, 364, 162, 73, 72, 209, 378, 380, 40, 74, 9, 71, 280, 250, 33, 213, 149, 257, 184, 138, 94, 88, 26, 287, 361, 219, 277, 29, 141, 350, 79, 359, 264, 383, 112, 235, 3, 144, 168, 234, 166, 96, 84, 256, 12, 90, 261, 140, 286, 283, 27, 147, 154, 82, 276, 212, 7, 294, 25, 156, 122, 171, 307, 248, 80, 351, 177, 204, 182, 345, 145, 382, 300, 348, 1, 169, 123, 268, 332, 114, 203, 224, 117, 228, 225, 305, 296, 199, 4, 352, 134, 152, 121, 106, 243, 50, 344, 299, 298, 133, 308, 167, 278, 210, 253, 65, 131, 51, 41, 263, 221, 130, 186, 81, 78, 285, 6, 336, 100, 173, 85, 48, 52, 233, 288, 125, 273, 24, 262, 237, 185, 292, 135, 124, 338, 347, 254, 238, 8, 172, 217, 220, 142, 374, 379, 174, 107, 333, 35, 113, 269, 42, 66, 370, 322, 302, 316, 266, 61, 331, 310, 372, 75, 334, 194, 358, 55, 241, 34, 196, 289, 137, 30, 60, 190, 59, 14, 119, 381, 91, 187, 375, 159, 274, 258, 366, 367, 270, 315, 157, 321, 319, 301, 284, 15, 295, 115, 252, 69, 109, 62, 58, 22, 148, 297, 247, 328, 354, 17, 175, 325, 151, 83, 304, 293, 216, 10, 357, 132, 346, 281, 170, 306, 324, 329, 201, 44, 275, 28, 363, 368, 101, 242, 339, 191, 207, 49, 198, 223, 222, 77, 53, 158, 323, 356, 70, 165, 244, 98, 178, 327, 179, 355, 197, 37, 164, 143, 89, 57, 369, 251, 23, 320, 215, 340, 231, 127, 32, 67, 326, 120, 314, 236, 110, 259, 92, 128, 309, 108, 93, 245, 362, 291, 373, 47, 343, 272, 104, 176, 240, 239, 46, 20, 303, 246, 255, 18, 192, 193, 63, 11, 155, 229, 36, 265, 39, 38, 365, 377, 267, 99, 353, 102, 31, 64, 376, 312] + 42: [306, 77, 80, 238, 98, 87, 268, 251, 101, 138, 88, 303, 33, 197, 154, 41, 282, 209, 164, 150, 356, 151, 351, 285, 178, 92, 277, 311, 0, 248, 60, 217, 181, 244, 312, 3, 232, 247, 362, 284, 75, 34, 261, 134, 256, 76, 353, 78, 86, 71, 38, 348, 375, 149, 215, 155, 221, 349, 258, 168, 195, 345, 267, 146, 292, 31, 338, 355, 382, 250, 346, 334, 369, 69, 24, 141, 65, 46, 347, 198, 187, 339, 42, 211, 192, 174, 243, 121, 304, 177, 372, 153, 318, 14, 99, 270, 266, 200, 166, 118, 319, 37, 212, 373, 16, 272, 135, 202, 241, 378, 359, 120, 302, 161, 296, 263, 295, 321, 222, 20, 2, 226, 316, 32, 68, 317, 231, 350, 333, 184, 140, 145, 193, 147, 264, 260, 343, 139, 79, 47, 36, 220, 67, 119, 186, 246, 49, 299, 331, 172, 66, 45, 305, 207, 43, 210, 157, 254, 324, 22, 327, 381, 249, 276, 18, 6, 223, 183, 72, 180, 109, 191, 107, 62, 206, 19, 143, 162, 199, 136, 322, 205, 214, 28, 269, 340, 51, 208, 89, 370, 25, 286, 123, 111, 17, 21, 196, 114, 274, 55, 103, 291, 367, 364, 132, 320, 59, 275, 48, 283, 315, 52, 290, 167, 12, 148, 61, 236, 328, 281, 90, 131, 228, 293, 255, 179, 301, 230, 115, 122, 15, 1, 169, 83, 245, 84, 158, 156, 298, 97, 314, 23, 240, 100, 288, 227, 354, 85, 5, 377, 7, 262, 70, 329, 294, 40, 335, 27, 332, 313, 159, 330, 357, 371, 323, 173, 365, 4, 225, 194, 383, 94, 219, 128, 253, 361, 229, 363, 74, 190, 142, 224, 113, 126, 216, 242, 58, 50, 176, 325, 189, 204, 358, 93, 279, 336, 300, 56, 116, 265, 152, 308, 239, 129, 366, 380, 326, 280, 125, 309, 8, 117, 54, 102, 201, 175, 30, 218, 342, 112, 26, 259, 237, 130, 44, 287, 127, 144, 341, 344, 213, 11, 104, 170, 252, 368, 374, 379, 133, 124, 310, 29, 13, 352, 234, 171, 82, 297, 278, 35, 73, 96, 376, 57, 9, 307, 64, 289, 203, 235, 337, 10, 360, 257, 63, 110, 160, 91, 188, 105, 165, 53, 95, 185, 106, 137, 108, 233, 273, 81, 182, 271, 163, 39] + 43: [315, 202, 211, 243, 61, 55, 252, 218, 319, 366, 63, 324, 364, 258, 209, 170, 200, 27, 30, 156, 80, 356, 338, 46, 318, 379, 335, 341, 16, 149, 342, 267, 183, 371, 1, 306, 115, 72, 368, 163, 232, 269, 271, 193, 13, 70, 185, 109, 255, 157, 137, 189, 249, 215, 199, 321, 37, 327, 317, 314, 212, 8, 130, 162, 350, 284, 105, 268, 178, 247, 145, 295, 133, 78, 230, 229, 112, 45, 289, 91, 228, 296, 15, 245, 14, 43, 181, 51, 195, 370, 58, 29, 340, 104, 294, 21, 346, 36, 240, 239, 219, 272, 69, 194, 74, 108, 172, 92, 159, 84, 52, 152, 125, 190, 337, 120, 56, 191, 127, 339, 288, 363, 86, 333, 220, 374, 347, 77, 186, 330, 40, 201, 168, 221, 282, 93, 71, 205, 32, 147, 382, 94, 122, 65, 180, 278, 351, 276, 54, 286, 287, 131, 277, 301, 124, 355, 250, 33, 44, 113, 357, 312, 305, 31, 6, 102, 136, 349, 372, 187, 41, 118, 150, 353, 3, 90, 334, 241, 236, 231, 177, 248, 169, 206, 10, 89, 320, 7, 4, 208, 106, 281, 283, 75, 198, 97, 101, 161, 167, 298, 256, 76, 358, 238, 100, 224, 260, 223, 166, 378, 142, 188, 323, 138, 9, 20, 22, 0, 336, 290, 375, 303, 73, 380, 49, 204, 48, 207, 275, 210, 251, 62, 225, 99, 151, 68, 300, 280, 291, 377, 265, 17, 88, 38, 164, 50, 365, 222, 39, 184, 226, 254, 141, 261, 66, 263, 270, 111, 18, 264, 135, 325, 128, 373, 82, 23, 19, 376, 197, 331, 5, 192, 126, 227, 361, 110, 292, 313, 213, 348, 176, 326, 60, 307, 328, 47, 310, 381, 116, 293, 53, 262, 107, 344, 175, 96, 237, 216, 257, 242, 235, 279, 196, 2, 345, 160, 311, 383, 146, 274, 26, 98, 203, 369, 103, 253, 85, 59, 123, 87, 297, 34, 343, 140, 83, 35, 332, 302, 67, 352, 121, 148, 308, 12, 299, 153, 42, 362, 259, 359, 179, 171, 316, 129, 309, 139, 273, 155, 134, 173, 360, 154, 143, 329, 24, 117, 367, 322, 285, 244, 95, 119, 144, 182, 79, 158, 246, 28, 57, 233, 165, 114, 11, 81, 266, 214, 25, 174, 64, 304, 234, 354, 132, 217] + 44: [64, 267, 95, 242, 26, 35, 275, 299, 334, 74, 351, 250, 63, 217, 318, 98, 124, 66, 212, 215, 238, 86, 173, 43, 24, 324, 199, 57, 13, 171, 85, 300, 101, 200, 330, 182, 251, 187, 342, 372, 383, 123, 87, 258, 349, 80, 79, 180, 41, 181, 273, 264, 190, 32, 142, 317, 155, 1, 226, 108, 214, 118, 147, 294, 46, 193, 241, 310, 346, 231, 53, 341, 283, 357, 78, 134, 316, 197, 308, 152, 23, 322, 65, 116, 245, 362, 309, 344, 107, 326, 230, 339, 71, 307, 198, 297, 102, 36, 350, 285, 280, 91, 61, 243, 239, 105, 186, 255, 347, 175, 207, 305, 237, 268, 93, 130, 295, 121, 89, 284, 289, 338, 221, 290, 356, 336, 371, 109, 379, 153, 192, 249, 50, 353, 164, 33, 172, 323, 111, 3, 113, 374, 159, 286, 304, 110, 281, 332, 203, 359, 28, 39, 122, 97, 358, 22, 62, 261, 277, 68, 329, 169, 115, 375, 0, 149, 47, 140, 382, 45, 380, 210, 206, 259, 225, 220, 302, 311, 42, 365, 364, 154, 208, 106, 335, 354, 148, 136, 373, 112, 248, 88, 15, 10, 369, 163, 84, 263, 227, 256, 265, 224, 73, 14, 82, 194, 272, 131, 287, 253, 271, 257, 137, 222, 177, 166, 178, 126, 76, 156, 117, 77, 188, 202, 99, 54, 355, 92, 44, 21, 19, 282, 59, 128, 201, 312, 157, 366, 67, 345, 25, 314, 90, 376, 160, 204, 141, 37, 298, 83, 327, 331, 360, 229, 240, 27, 370, 56, 30, 17, 31, 328, 120, 9, 7, 150, 367, 135, 361, 247, 170, 146, 168, 133, 319, 16, 320, 195, 138, 270, 363, 12, 5, 174, 321, 183, 211, 129, 274, 6, 213, 279, 254, 333, 266, 125, 51, 244, 218, 4, 38, 165, 296, 94, 276, 191, 292, 100, 75, 378, 96, 103, 381, 34, 352, 179, 340, 313, 348, 325, 269, 127, 48, 145, 233, 18, 236, 343, 143, 288, 235, 301, 306, 260, 234, 52, 151, 139, 189, 262, 278, 104, 158, 144, 69, 337, 223, 228, 2, 232, 114, 8, 40, 196, 81, 184, 209, 70, 11, 176, 303, 60, 185, 55, 20, 291, 246, 315, 162, 58, 368, 167, 219, 119, 216, 29, 72, 161, 252, 49, 293, 205, 377, 132] + 45: [131, 144, 79, 179, 116, 178, 369, 52, 165, 155, 208, 373, 367, 37, 275, 331, 284, 207, 253, 319, 239, 365, 315, 10, 212, 99, 242, 341, 87, 0, 216, 217, 63, 204, 105, 161, 292, 339, 147, 158, 160, 243, 177, 21, 47, 168, 140, 129, 143, 310, 92, 110, 317, 16, 151, 377, 68, 357, 353, 259, 43, 252, 370, 56, 127, 135, 75, 67, 66, 258, 24, 300, 14, 115, 227, 157, 294, 362, 64, 244, 206, 28, 272, 77, 214, 89, 69, 111, 86, 210, 53, 117, 11, 287, 326, 347, 283, 281, 61, 236, 186, 282, 222, 256, 356, 146, 298, 220, 333, 261, 141, 50, 273, 82, 162, 125, 29, 25, 274, 233, 95, 378, 269, 215, 246, 123, 325, 176, 73, 359, 80, 51, 191, 202, 180, 279, 7, 142, 104, 289, 219, 139, 91, 205, 185, 102, 84, 23, 45, 39, 156, 265, 306, 303, 361, 291, 344, 173, 35, 264, 374, 153, 198, 200, 88, 187, 245, 108, 240, 183, 169, 171, 49, 371, 368, 132, 133, 305, 3, 30, 211, 195, 13, 137, 336, 358, 270, 136, 22, 42, 337, 167, 350, 201, 109, 31, 372, 118, 286, 330, 189, 46, 145, 262, 249, 335, 277, 54, 383, 382, 166, 301, 360, 101, 20, 322, 238, 225, 128, 8, 100, 114, 41, 346, 295, 232, 152, 26, 154, 34, 323, 72, 113, 119, 188, 228, 106, 381, 320, 71, 15, 170, 33, 164, 199, 324, 235, 203, 351, 159, 379, 213, 268, 218, 121, 307, 328, 6, 94, 181, 74, 321, 355, 90, 76, 18, 55, 313, 376, 231, 380, 257, 288, 302, 338, 312, 9, 4, 263, 192, 112, 148, 297, 230, 38, 12, 174, 311, 96, 234, 175, 327, 254, 182, 48, 348, 260, 58, 27, 138, 62, 120, 329, 134, 32, 196, 280, 163, 78, 70, 366, 2, 296, 149, 375, 278, 247, 267, 97, 334, 250, 150, 223, 36, 81, 290, 122, 293, 59, 308, 318, 103, 343, 345, 332, 85, 354, 248, 285, 107, 363, 5, 340, 197, 224, 349, 314, 44, 316, 226, 251, 276, 1, 65, 309, 40, 342, 271, 352, 57, 124, 60, 304, 255, 229, 237, 241, 126, 190, 299, 184, 17, 364, 83, 266, 194, 130, 221, 193, 19, 209, 172, 93, 98] + 46: [207, 224, 247, 376, 339, 152, 102, 103, 199, 105, 253, 362, 2, 104, 353, 355, 49, 164, 342, 8, 93, 322, 99, 313, 71, 12, 348, 328, 4, 30, 245, 201, 337, 17, 372, 312, 329, 264, 299, 88, 163, 121, 144, 176, 26, 80, 357, 165, 255, 187, 156, 95, 22, 359, 24, 235, 331, 107, 340, 65, 73, 59, 287, 18, 333, 46, 365, 174, 252, 270, 237, 1, 202, 231, 225, 56, 150, 37, 300, 5, 53, 307, 294, 295, 220, 50, 184, 15, 43, 175, 132, 248, 189, 251, 293, 216, 106, 239, 170, 168, 374, 7, 68, 212, 278, 136, 232, 131, 324, 75, 23, 256, 368, 217, 76, 82, 161, 345, 311, 280, 38, 273, 208, 292, 25, 352, 284, 67, 360, 381, 323, 267, 33, 218, 114, 160, 10, 242, 258, 186, 169, 39, 66, 369, 120, 364, 153, 356, 283, 86, 233, 85, 90, 213, 183, 286, 110, 260, 244, 118, 228, 297, 146, 289, 214, 318, 14, 275, 277, 282, 335, 238, 316, 28, 63, 94, 122, 196, 173, 272, 227, 330, 92, 288, 358, 182, 281, 383, 147, 341, 361, 167, 44, 343, 58, 221, 204, 319, 139, 271, 268, 166, 126, 254, 193, 246, 51, 234, 141, 155, 180, 290, 87, 97, 64, 125, 188, 52, 327, 354, 137, 69, 96, 320, 171, 230, 162, 249, 62, 338, 29, 219, 91, 70, 21, 77, 203, 226, 344, 109, 261, 363, 380, 305, 298, 116, 133, 40, 296, 291, 172, 98, 366, 177, 123, 179, 222, 336, 191, 19, 83, 47, 31, 262, 151, 332, 72, 178, 13, 157, 108, 304, 112, 79, 20, 81, 27, 145, 370, 309, 200, 257, 16, 3, 350, 54, 347, 61, 142, 111, 303, 158, 45, 9, 194, 149, 210, 351, 198, 6, 276, 274, 34, 211, 367, 377, 206, 124, 301, 266, 205, 127, 334, 35, 263, 285, 129, 101, 259, 375, 265, 240, 229, 57, 223, 11, 314, 154, 130, 143, 36, 209, 215, 269, 84, 378, 134, 89, 159, 113, 192, 195, 241, 140, 349, 317, 128, 181, 321, 306, 243, 100, 346, 185, 115, 315, 74, 41, 42, 308, 373, 279, 190, 117, 382, 78, 138, 48, 119, 379, 60, 135, 32, 55, 148, 310, 326, 236, 0, 197, 250, 371, 325, 302] + 47: [54, 330, 346, 370, 296, 123, 136, 61, 93, 244, 367, 98, 137, 4, 267, 210, 231, 377, 53, 189, 322, 141, 24, 76, 203, 158, 0, 108, 361, 264, 250, 172, 74, 282, 62, 202, 20, 132, 331, 213, 295, 319, 177, 121, 276, 12, 269, 107, 360, 236, 316, 345, 257, 169, 148, 207, 59, 321, 31, 341, 356, 304, 363, 205, 113, 233, 225, 354, 379, 170, 273, 239, 358, 38, 68, 64, 71, 283, 325, 149, 144, 88, 100, 69, 40, 160, 135, 99, 79, 300, 84, 8, 167, 270, 21, 272, 162, 299, 65, 41, 131, 195, 87, 179, 27, 103, 118, 208, 129, 352, 366, 154, 196, 111, 280, 328, 308, 275, 156, 83, 117, 187, 259, 26, 66, 312, 81, 70, 306, 176, 218, 45, 17, 44, 193, 373, 298, 291, 101, 95, 30, 302, 333, 116, 229, 109, 120, 163, 368, 183, 279, 157, 194, 320, 228, 89, 153, 324, 309, 190, 182, 266, 353, 28, 287, 72, 274, 178, 351, 286, 336, 92, 339, 206, 261, 251, 104, 15, 348, 86, 256, 124, 161, 220, 248, 260, 152, 56, 150, 214, 318, 364, 265, 90, 243, 106, 200, 63, 168, 77, 204, 357, 175, 43, 285, 52, 301, 18, 119, 284, 372, 371, 134, 262, 376, 36, 232, 337, 329, 305, 155, 271, 294, 216, 211, 359, 35, 173, 192, 9, 201, 143, 238, 14, 16, 82, 315, 281, 34, 198, 164, 365, 221, 33, 277, 184, 226, 13, 181, 334, 254, 37, 11, 335, 102, 209, 23, 327, 217, 219, 252, 230, 222, 241, 186, 130, 258, 146, 245, 166, 355, 268, 313, 25, 311, 142, 383, 110, 344, 253, 115, 47, 246, 288, 96, 78, 381, 380, 29, 2, 127, 235, 3, 50, 375, 185, 347, 303, 42, 314, 48, 292, 67, 122, 140, 180, 94, 290, 75, 128, 240, 151, 224, 342, 73, 46, 374, 171, 165, 212, 51, 138, 105, 174, 133, 317, 307, 60, 139, 343, 297, 362, 97, 197, 215, 188, 278, 147, 80, 39, 85, 289, 1, 338, 234, 227, 223, 263, 369, 378, 91, 249, 57, 310, 247, 159, 255, 10, 22, 32, 382, 199, 242, 332, 237, 293, 49, 125, 19, 323, 55, 340, 126, 350, 6, 349, 114, 191, 326, 7, 5, 145, 58, 112] + 48: [271, 275, 86, 364, 311, 128, 112, 290, 323, 24, 317, 316, 370, 119, 346, 223, 102, 101, 3, 261, 123, 38, 146, 213, 56, 305, 45, 297, 263, 281, 21, 162, 247, 212, 330, 82, 23, 220, 148, 7, 218, 376, 358, 118, 87, 293, 195, 239, 219, 362, 133, 211, 347, 156, 47, 126, 322, 157, 135, 92, 52, 227, 280, 100, 17, 57, 69, 29, 278, 285, 93, 294, 97, 327, 15, 131, 203, 369, 299, 0, 252, 206, 303, 46, 288, 348, 310, 284, 245, 36, 201, 10, 260, 153, 105, 196, 141, 329, 344, 88, 77, 185, 58, 170, 33, 189, 12, 174, 242, 91, 117, 34, 276, 254, 19, 179, 314, 343, 132, 32, 363, 320, 382, 49, 205, 14, 76, 55, 120, 43, 269, 319, 198, 80, 90, 54, 255, 342, 253, 63, 4, 37, 155, 298, 274, 18, 190, 143, 20, 165, 228, 268, 187, 307, 53, 295, 64, 39, 312, 224, 334, 136, 129, 361, 25, 103, 259, 324, 61, 95, 338, 374, 231, 336, 345, 41, 67, 304, 70, 216, 184, 243, 339, 163, 351, 68, 200, 8, 332, 96, 188, 250, 207, 287, 318, 367, 321, 130, 35, 192, 151, 166, 107, 152, 159, 258, 235, 98, 335, 273, 48, 99, 127, 353, 81, 368, 357, 279, 51, 172, 74, 237, 350, 59, 106, 178, 13, 145, 208, 352, 296, 150, 138, 122, 286, 164, 371, 300, 337, 289, 137, 42, 169, 306, 373, 191, 199, 134, 31, 113, 173, 140, 246, 229, 232, 197, 50, 380, 341, 44, 282, 377, 315, 168, 379, 302, 222, 215, 301, 183, 251, 383, 167, 262, 79, 209, 267, 30, 40, 256, 331, 277, 160, 1, 204, 84, 308, 66, 65, 226, 116, 72, 26, 139, 264, 109, 73, 234, 115, 326, 108, 325, 272, 291, 161, 283, 233, 121, 75, 176, 154, 194, 28, 111, 158, 193, 360, 142, 144, 340, 180, 225, 202, 22, 328, 177, 365, 359, 244, 210, 240, 313, 230, 375, 147, 78, 125, 11, 9, 182, 16, 292, 236, 83, 241, 249, 5, 248, 104, 381, 2, 257, 186, 378, 6, 265, 85, 124, 214, 171, 94, 60, 89, 110, 266, 71, 333, 217, 175, 62, 356, 366, 309, 238, 114, 221, 354, 372, 355, 181, 270, 149, 27, 349] + 49: [210, 304, 248, 194, 338, 199, 56, 153, 190, 205, 351, 214, 216, 36, 150, 14, 286, 84, 169, 174, 251, 57, 299, 16, 311, 253, 300, 17, 316, 215, 314, 66, 262, 104, 27, 295, 333, 37, 163, 212, 32, 33, 122, 69, 183, 192, 259, 356, 328, 18, 227, 323, 234, 45, 282, 102, 350, 137, 22, 331, 334, 100, 355, 188, 107, 83, 75, 327, 201, 226, 191, 222, 4, 87, 24, 211, 48, 93, 329, 103, 128, 182, 318, 285, 324, 219, 112, 135, 383, 341, 291, 123, 129, 124, 145, 229, 302, 202, 116, 206, 78, 365, 271, 133, 312, 5, 265, 59, 91, 47, 151, 114, 173, 81, 198, 289, 359, 85, 221, 224, 284, 21, 26, 209, 19, 118, 270, 277, 362, 258, 168, 158, 360, 40, 138, 306, 46, 117, 371, 247, 35, 74, 278, 340, 354, 62, 171, 176, 166, 126, 301, 142, 9, 144, 319, 297, 266, 379, 187, 377, 134, 272, 42, 346, 313, 256, 60, 307, 96, 165, 31, 315, 373, 88, 8, 257, 357, 255, 106, 170, 13, 349, 317, 67, 217, 200, 10, 28, 131, 358, 51, 68, 208, 353, 120, 279, 233, 235, 86, 276, 197, 193, 240, 236, 148, 180, 111, 330, 23, 195, 361, 376, 20, 238, 189, 250, 110, 267, 298, 204, 54, 132, 181, 196, 275, 273, 53, 336, 155, 76, 225, 213, 268, 303, 310, 34, 136, 321, 274, 162, 159, 294, 146, 293, 65, 348, 11, 109, 140, 125, 30, 296, 108, 167, 325, 39, 249, 2, 366, 113, 139, 337, 364, 43, 243, 71, 105, 92, 152, 149, 231, 228, 130, 280, 64, 79, 119, 115, 177, 380, 309, 287, 0, 97, 3, 237, 347, 367, 374, 156, 220, 58, 292, 164, 261, 179, 94, 342, 363, 95, 61, 157, 223, 44, 369, 381, 141, 89, 77, 49, 264, 161, 339, 283, 172, 73, 6, 378, 345, 98, 288, 241, 185, 344, 322, 7, 368, 370, 320, 218, 239, 178, 15, 127, 263, 326, 281, 269, 335, 82, 50, 12, 242, 246, 41, 55, 63, 38, 72, 70, 175, 160, 52, 154, 186, 121, 343, 252, 207, 90, 290, 29, 254, 147, 305, 230, 184, 245, 143, 260, 25, 375, 244, 80, 1, 382, 332, 372, 352, 99, 101, 232, 203, 308] + 50: [306, 268, 180, 358, 353, 138, 315, 187, 252, 58, 90, 275, 6, 189, 214, 63, 185, 281, 256, 51, 28, 366, 240, 249, 126, 344, 153, 327, 152, 17, 77, 266, 148, 157, 350, 108, 135, 210, 217, 96, 119, 125, 294, 121, 39, 70, 134, 7, 372, 113, 21, 312, 136, 213, 177, 239, 324, 206, 375, 149, 225, 64, 98, 357, 142, 349, 11, 231, 53, 147, 154, 383, 258, 219, 259, 20, 293, 376, 137, 282, 196, 347, 2, 82, 292, 245, 99, 190, 61, 373, 341, 365, 103, 257, 264, 298, 72, 9, 175, 297, 193, 42, 86, 14, 380, 279, 69, 300, 270, 243, 160, 340, 38, 127, 333, 205, 3, 220, 336, 208, 322, 325, 156, 229, 159, 95, 194, 236, 226, 235, 10, 23, 116, 54, 146, 144, 355, 66, 269, 202, 18, 321, 287, 164, 227, 163, 295, 351, 224, 248, 359, 369, 247, 371, 263, 310, 307, 255, 317, 209, 102, 91, 334, 26, 356, 84, 139, 29, 85, 335, 151, 379, 131, 120, 40, 167, 221, 260, 276, 360, 198, 34, 179, 250, 106, 363, 211, 216, 207, 176, 237, 83, 33, 318, 129, 1, 343, 5, 75, 0, 370, 49, 15, 115, 232, 141, 41, 301, 16, 352, 31, 346, 284, 261, 267, 285, 186, 277, 242, 22, 45, 8, 299, 68, 158, 81, 74, 171, 36, 114, 234, 71, 378, 161, 46, 251, 184, 288, 165, 362, 24, 155, 174, 238, 118, 43, 289, 60, 32, 67, 339, 331, 145, 150, 323, 52, 253, 178, 241, 80, 78, 309, 183, 173, 228, 367, 55, 326, 222, 62, 204, 30, 280, 291, 332, 47, 330, 87, 181, 316, 37, 286, 278, 105, 308, 345, 4, 172, 192, 329, 27, 117, 348, 97, 218, 124, 338, 166, 123, 212, 57, 92, 188, 381, 169, 122, 311, 130, 12, 382, 88, 246, 361, 320, 197, 94, 314, 56, 110, 48, 162, 25, 100, 319, 215, 133, 143, 342, 271, 313, 223, 274, 354, 107, 104, 265, 328, 230, 44, 35, 89, 272, 377, 50, 112, 254, 109, 203, 302, 304, 283, 73, 195, 111, 305, 273, 191, 233, 244, 296, 262, 59, 182, 101, 168, 140, 368, 199, 303, 170, 374, 132, 201, 128, 337, 200, 65, 19, 79, 93, 13, 364, 76, 290] + 51: [24, 240, 253, 10, 114, 203, 342, 86, 130, 341, 201, 120, 98, 198, 100, 266, 210, 46, 279, 44, 149, 18, 190, 304, 103, 17, 81, 193, 351, 375, 88, 282, 292, 123, 325, 364, 66, 180, 363, 134, 36, 132, 113, 239, 52, 333, 286, 7, 185, 230, 222, 138, 261, 367, 56, 267, 252, 175, 191, 331, 303, 365, 50, 339, 274, 317, 366, 231, 354, 19, 93, 60, 377, 275, 147, 154, 55, 243, 347, 376, 21, 296, 370, 166, 116, 151, 181, 174, 352, 2, 61, 173, 179, 256, 269, 273, 137, 184, 194, 32, 177, 219, 260, 20, 335, 90, 359, 171, 234, 197, 322, 326, 82, 299, 320, 119, 232, 334, 225, 321, 74, 108, 270, 143, 62, 355, 382, 285, 371, 278, 12, 315, 254, 308, 360, 346, 152, 343, 76, 192, 217, 131, 136, 211, 238, 75, 300, 345, 205, 307, 37, 281, 196, 38, 358, 5, 73, 316, 224, 319, 268, 310, 244, 51, 54, 111, 57, 83, 305, 248, 6, 80, 361, 95, 218, 318, 124, 383, 290, 15, 117, 214, 178, 67, 168, 255, 349, 272, 70, 170, 156, 42, 356, 99, 110, 167, 271, 65, 25, 84, 141, 69, 118, 258, 204, 153, 3, 245, 183, 146, 209, 41, 340, 135, 353, 186, 287, 122, 140, 350, 373, 280, 298, 216, 314, 47, 372, 45, 344, 104, 295, 182, 306, 200, 302, 242, 97, 125, 379, 311, 263, 92, 284, 297, 208, 223, 91, 16, 378, 77, 337, 207, 381, 264, 30, 155, 226, 202, 8, 288, 9, 28, 206, 369, 43, 105, 257, 169, 236, 189, 237, 63, 291, 332, 309, 249, 112, 328, 29, 133, 228, 126, 176, 68, 250, 142, 265, 187, 87, 164, 368, 59, 31, 374, 329, 129, 195, 109, 220, 235, 229, 35, 312, 212, 289, 48, 246, 301, 330, 33, 357, 148, 102, 213, 40, 362, 157, 233, 324, 128, 39, 115, 26, 227, 294, 160, 262, 107, 251, 72, 127, 144, 22, 323, 188, 101, 380, 283, 259, 58, 338, 89, 13, 277, 139, 64, 34, 161, 165, 276, 247, 336, 106, 121, 163, 85, 221, 71, 199, 23, 0, 79, 327, 11, 145, 27, 49, 159, 53, 158, 172, 162, 1, 215, 78, 348, 14, 241, 4, 150, 293, 94, 96, 313] + 52: [332, 248, 151, 342, 379, 258, 217, 196, 311, 215, 116, 161, 23, 97, 33, 312, 74, 85, 138, 171, 358, 127, 84, 239, 369, 328, 272, 22, 357, 3, 63, 265, 285, 79, 50, 31, 10, 352, 5, 364, 243, 130, 334, 343, 365, 326, 61, 39, 223, 47, 208, 355, 216, 227, 149, 335, 9, 129, 295, 237, 49, 236, 318, 172, 145, 294, 362, 152, 14, 348, 120, 360, 29, 181, 259, 319, 113, 341, 253, 368, 12, 356, 309, 222, 278, 20, 315, 251, 156, 131, 142, 316, 324, 86, 158, 180, 26, 378, 219, 262, 381, 276, 75, 125, 346, 289, 192, 72, 80, 269, 136, 139, 293, 182, 167, 73, 211, 184, 176, 44, 339, 4, 189, 344, 117, 111, 197, 159, 210, 376, 257, 56, 382, 186, 225, 71, 140, 256, 366, 19, 286, 6, 213, 68, 350, 214, 271, 103, 370, 17, 108, 122, 240, 372, 162, 150, 338, 169, 306, 195, 299, 94, 147, 249, 16, 290, 209, 35, 106, 18, 54, 28, 52, 179, 46, 301, 45, 43, 373, 66, 87, 185, 42, 218, 38, 380, 203, 202, 135, 175, 200, 264, 107, 27, 252, 13, 228, 298, 323, 168, 246, 234, 104, 307, 283, 93, 177, 69, 230, 354, 198, 321, 163, 260, 367, 336, 188, 173, 233, 221, 70, 78, 277, 224, 141, 333, 118, 134, 146, 67, 11, 302, 95, 205, 137, 99, 109, 110, 287, 41, 270, 24, 297, 157, 58, 178, 121, 250, 112, 81, 37, 153, 275, 232, 62, 282, 292, 51, 76, 90, 363, 7, 34, 77, 89, 374, 349, 310, 82, 124, 280, 305, 199, 377, 226, 268, 238, 133, 204, 274, 220, 100, 115, 98, 353, 53, 279, 317, 206, 325, 193, 183, 102, 327, 300, 25, 263, 160, 114, 345, 30, 330, 337, 92, 261, 1, 88, 8, 229, 15, 143, 40, 288, 291, 347, 83, 187, 57, 59, 308, 32, 359, 64, 255, 281, 119, 296, 242, 165, 148, 273, 154, 128, 383, 191, 254, 91, 235, 2, 322, 320, 313, 55, 105, 164, 190, 361, 174, 266, 65, 166, 194, 207, 304, 123, 331, 241, 132, 340, 36, 244, 21, 267, 96, 231, 284, 371, 60, 245, 101, 351, 170, 155, 144, 247, 126, 329, 48, 314, 212, 0, 201, 303, 375] + 53: [308, 199, 350, 311, 136, 338, 70, 4, 196, 44, 335, 182, 346, 232, 5, 333, 360, 134, 219, 165, 368, 357, 214, 299, 273, 64, 233, 10, 246, 154, 192, 91, 197, 30, 104, 228, 251, 282, 303, 142, 34, 130, 224, 103, 98, 369, 222, 32, 47, 54, 159, 290, 327, 68, 276, 315, 101, 257, 53, 337, 234, 245, 58, 88, 31, 364, 263, 50, 258, 89, 137, 102, 131, 62, 380, 186, 348, 57, 185, 90, 316, 277, 138, 215, 286, 61, 106, 140, 342, 272, 264, 43, 113, 181, 289, 377, 367, 254, 110, 238, 247, 95, 175, 149, 157, 112, 164, 243, 132, 332, 86, 293, 261, 320, 329, 353, 180, 318, 170, 291, 194, 78, 63, 295, 317, 331, 306, 208, 92, 79, 195, 324, 35, 345, 21, 168, 207, 143, 158, 111, 227, 8, 326, 29, 176, 352, 212, 122, 206, 287, 28, 285, 27, 242, 370, 17, 179, 117, 42, 323, 204, 174, 148, 33, 312, 41, 59, 99, 105, 218, 220, 355, 74, 365, 256, 259, 366, 349, 252, 82, 22, 359, 343, 248, 307, 314, 141, 96, 249, 56, 84, 87, 126, 52, 127, 173, 152, 129, 76, 241, 268, 379, 15, 65, 340, 383, 374, 275, 73, 304, 100, 319, 371, 283, 69, 363, 213, 128, 231, 14, 341, 40, 60, 166, 297, 16, 202, 0, 236, 229, 119, 72, 271, 262, 221, 144, 274, 300, 235, 381, 183, 145, 12, 24, 325, 309, 269, 83, 351, 139, 13, 36, 230, 217, 120, 294, 153, 184, 361, 358, 260, 37, 305, 281, 382, 123, 321, 71, 48, 1, 171, 310, 301, 93, 167, 210, 376, 80, 298, 223, 49, 7, 177, 344, 267, 265, 188, 135, 121, 280, 189, 347, 198, 172, 66, 23, 200, 3, 38, 118, 239, 51, 250, 151, 203, 146, 46, 125, 226, 75, 336, 133, 107, 330, 266, 313, 284, 292, 237, 160, 26, 356, 201, 372, 187, 178, 150, 55, 156, 81, 11, 162, 279, 362, 2, 225, 255, 302, 288, 296, 18, 169, 9, 25, 39, 94, 124, 85, 116, 191, 97, 240, 20, 216, 373, 77, 209, 155, 67, 190, 378, 19, 161, 109, 6, 253, 322, 375, 354, 163, 211, 193, 45, 147, 328, 108, 270, 115, 278, 244, 334, 114, 339, 205] + 54: [363, 41, 299, 57, 44, 325, 154, 147, 370, 90, 192, 280, 167, 143, 300, 356, 64, 328, 181, 46, 286, 93, 258, 27, 371, 326, 71, 353, 265, 122, 121, 92, 246, 61, 140, 331, 112, 164, 327, 207, 189, 35, 367, 184, 171, 343, 145, 340, 84, 321, 241, 76, 212, 26, 166, 95, 236, 68, 333, 153, 210, 297, 172, 81, 319, 79, 217, 287, 18, 193, 48, 334, 359, 33, 183, 267, 75, 162, 223, 238, 60, 383, 229, 316, 110, 105, 195, 324, 168, 362, 10, 130, 36, 55, 78, 123, 294, 240, 80, 115, 158, 151, 53, 124, 125, 284, 282, 163, 173, 91, 226, 179, 312, 242, 188, 70, 62, 54, 174, 347, 103, 293, 40, 375, 186, 208, 329, 341, 25, 32, 245, 314, 269, 161, 4, 349, 137, 42, 235, 277, 200, 249, 348, 191, 380, 248, 37, 34, 364, 330, 317, 211, 22, 77, 87, 7, 65, 337, 72, 239, 378, 244, 69, 252, 104, 307, 221, 116, 264, 263, 372, 306, 232, 273, 342, 43, 308, 187, 270, 373, 291, 160, 227, 28, 114, 234, 368, 152, 144, 148, 169, 237, 214, 176, 117, 260, 296, 357, 82, 45, 16, 197, 3, 365, 98, 311, 156, 352, 199, 113, 275, 322, 215, 279, 290, 225, 301, 20, 129, 111, 29, 309, 196, 230, 323, 2, 58, 136, 128, 180, 170, 15, 202, 228, 5, 295, 213, 99, 283, 96, 355, 88, 142, 118, 254, 278, 231, 86, 259, 250, 13, 150, 304, 233, 182, 345, 155, 332, 203, 83, 120, 377, 14, 23, 206, 131, 379, 19, 109, 8, 101, 74, 205, 52, 177, 126, 354, 376, 303, 175, 298, 285, 141, 256, 0, 100, 222, 313, 133, 339, 17, 224, 209, 288, 272, 39, 276, 119, 139, 366, 11, 253, 262, 97, 85, 220, 271, 146, 6, 9, 257, 302, 201, 292, 178, 38, 149, 135, 381, 243, 24, 30, 216, 361, 351, 310, 56, 251, 198, 50, 47, 268, 281, 289, 157, 190, 132, 360, 63, 255, 305, 12, 108, 21, 134, 218, 49, 274, 344, 261, 335, 31, 67, 73, 127, 185, 138, 106, 369, 374, 318, 219, 159, 338, 89, 204, 94, 346, 51, 66, 59, 315, 194, 247, 102, 382, 320, 350, 165, 266, 1, 107, 358, 336] + 55: [154, 355, 304, 360, 248, 337, 270, 212, 354, 306, 57, 303, 262, 344, 322, 14, 349, 4, 218, 250, 140, 48, 267, 23, 113, 363, 266, 213, 98, 260, 90, 328, 216, 3, 246, 36, 268, 104, 125, 53, 41, 214, 180, 297, 101, 182, 45, 302, 335, 339, 277, 220, 67, 203, 175, 254, 235, 336, 258, 193, 0, 316, 89, 188, 201, 273, 230, 227, 359, 78, 145, 71, 242, 282, 186, 142, 88, 269, 47, 153, 276, 229, 85, 10, 77, 34, 109, 348, 321, 223, 173, 351, 25, 117, 236, 288, 247, 211, 138, 327, 179, 312, 161, 369, 174, 52, 35, 105, 341, 202, 50, 6, 80, 28, 198, 62, 382, 19, 334, 192, 74, 278, 189, 32, 366, 314, 243, 309, 238, 331, 353, 123, 22, 383, 54, 17, 340, 160, 2, 149, 228, 127, 146, 55, 12, 171, 205, 84, 167, 121, 298, 326, 151, 307, 30, 15, 280, 116, 103, 86, 126, 75, 18, 94, 332, 91, 239, 21, 333, 364, 31, 208, 232, 263, 283, 133, 285, 83, 108, 181, 69, 292, 59, 275, 241, 166, 191, 29, 362, 187, 87, 245, 82, 257, 357, 233, 271, 73, 16, 377, 60, 237, 38, 7, 1, 185, 365, 115, 107, 323, 70, 5, 284, 40, 122, 375, 261, 195, 172, 63, 373, 221, 158, 72, 200, 137, 176, 134, 20, 209, 286, 225, 338, 319, 293, 64, 240, 164, 141, 42, 162, 177, 93, 251, 124, 150, 253, 264, 294, 299, 368, 291, 129, 342, 231, 27, 147, 315, 222, 378, 281, 96, 165, 143, 300, 301, 99, 296, 56, 380, 170, 130, 379, 132, 120, 128, 183, 352, 51, 204, 305, 168, 356, 224, 148, 317, 135, 310, 136, 106, 287, 81, 39, 350, 265, 255, 295, 249, 210, 308, 234, 290, 110, 65, 311, 97, 371, 44, 156, 9, 100, 381, 159, 289, 11, 46, 76, 178, 226, 112, 370, 318, 313, 217, 68, 163, 252, 347, 197, 152, 358, 26, 272, 206, 325, 95, 102, 13, 194, 157, 37, 119, 330, 343, 58, 169, 324, 329, 199, 111, 361, 8, 79, 346, 114, 244, 259, 184, 118, 374, 219, 279, 256, 372, 274, 33, 24, 155, 196, 139, 320, 345, 49, 43, 376, 131, 66, 61, 190, 144, 207, 215, 92, 367] + 56: [321, 240, 156, 204, 9, 126, 154, 368, 251, 89, 83, 238, 324, 267, 144, 99, 215, 383, 23, 331, 82, 244, 284, 116, 74, 158, 201, 107, 236, 193, 289, 305, 380, 314, 111, 323, 290, 91, 224, 325, 365, 369, 248, 333, 319, 56, 5, 71, 336, 15, 122, 40, 282, 233, 67, 228, 344, 143, 159, 135, 355, 66, 378, 155, 312, 218, 176, 178, 115, 182, 3, 65, 257, 165, 85, 95, 77, 24, 43, 68, 136, 192, 124, 55, 382, 320, 239, 247, 151, 276, 25, 37, 50, 61, 90, 103, 53, 101, 342, 0, 181, 160, 299, 134, 338, 356, 21, 104, 148, 377, 343, 12, 361, 79, 29, 341, 123, 139, 315, 316, 186, 88, 52, 297, 140, 60, 220, 242, 269, 362, 149, 259, 198, 196, 232, 359, 141, 335, 185, 47, 8, 189, 379, 357, 349, 188, 175, 351, 332, 381, 372, 235, 153, 306, 350, 373, 352, 273, 46, 73, 227, 348, 1, 237, 367, 221, 265, 51, 78, 260, 206, 345, 2, 254, 255, 280, 31, 129, 86, 293, 358, 105, 110, 234, 145, 256, 92, 132, 302, 108, 109, 142, 226, 163, 327, 191, 138, 184, 207, 6, 281, 150, 245, 114, 295, 42, 303, 219, 33, 339, 328, 27, 363, 72, 317, 310, 283, 48, 225, 102, 261, 347, 253, 197, 147, 113, 376, 203, 230, 301, 18, 268, 64, 311, 169, 128, 194, 329, 208, 205, 272, 309, 318, 243, 69, 80, 59, 49, 171, 38, 13, 195, 250, 164, 222, 252, 63, 57, 217, 20, 35, 120, 271, 258, 246, 26, 286, 313, 340, 76, 45, 274, 170, 296, 127, 326, 360, 200, 152, 262, 173, 133, 112, 180, 364, 146, 202, 287, 75, 294, 11, 322, 187, 44, 166, 374, 81, 241, 162, 100, 212, 209, 337, 231, 10, 278, 117, 292, 210, 39, 106, 97, 84, 172, 177, 62, 334, 168, 28, 93, 96, 300, 174, 304, 275, 199, 279, 167, 14, 264, 30, 130, 34, 121, 308, 4, 353, 266, 249, 98, 119, 16, 298, 223, 291, 183, 161, 17, 125, 277, 32, 36, 213, 330, 19, 190, 94, 137, 354, 7, 375, 157, 214, 371, 346, 263, 229, 131, 87, 54, 22, 58, 216, 285, 179, 307, 270, 41, 70, 118, 211, 370, 288, 366] + 57: [254, 345, 302, 90, 342, 94, 50, 33, 366, 223, 227, 346, 180, 260, 40, 21, 150, 383, 238, 352, 211, 182, 102, 136, 47, 300, 267, 339, 259, 54, 70, 66, 209, 109, 146, 208, 286, 34, 277, 188, 113, 287, 25, 86, 92, 198, 91, 246, 273, 174, 266, 176, 149, 186, 295, 173, 151, 127, 175, 7, 160, 200, 330, 72, 237, 298, 310, 343, 17, 2, 335, 329, 46, 119, 309, 99, 142, 278, 171, 229, 271, 112, 202, 178, 107, 101, 361, 131, 241, 234, 24, 194, 215, 221, 258, 275, 231, 125, 306, 43, 169, 356, 29, 347, 251, 217, 55, 85, 362, 158, 235, 134, 137, 236, 95, 203, 111, 270, 332, 122, 65, 60, 35, 71, 16, 228, 195, 257, 358, 297, 219, 324, 22, 199, 374, 177, 214, 115, 382, 312, 1, 61, 156, 325, 349, 216, 172, 26, 184, 193, 11, 344, 185, 274, 110, 212, 120, 321, 39, 207, 328, 222, 100, 363, 284, 82, 282, 213, 187, 83, 78, 77, 48, 292, 296, 370, 351, 132, 18, 320, 98, 244, 245, 291, 340, 263, 80, 232, 45, 262, 293, 301, 104, 68, 159, 58, 117, 226, 315, 272, 27, 179, 314, 62, 350, 41, 289, 196, 290, 152, 144, 87, 252, 118, 154, 138, 128, 123, 379, 106, 357, 79, 368, 103, 57, 116, 20, 225, 153, 67, 8, 12, 0, 253, 377, 307, 316, 157, 265, 162, 375, 224, 294, 168, 32, 166, 230, 167, 28, 281, 218, 256, 318, 313, 96, 23, 233, 360, 371, 206, 140, 121, 38, 30, 37, 14, 88, 124, 81, 163, 56, 164, 63, 276, 255, 3, 249, 69, 210, 376, 4, 336, 5, 299, 248, 13, 261, 311, 89, 44, 141, 264, 220, 6, 9, 197, 303, 317, 365, 268, 84, 331, 31, 279, 326, 288, 323, 367, 161, 108, 145, 354, 170, 130, 73, 201, 148, 133, 243, 285, 338, 126, 240, 341, 64, 191, 204, 369, 42, 181, 378, 155, 165, 59, 139, 283, 205, 242, 381, 247, 334, 348, 359, 74, 250, 327, 190, 319, 355, 373, 76, 114, 15, 322, 135, 183, 10, 51, 93, 380, 192, 304, 53, 189, 239, 143, 269, 97, 36, 129, 305, 105, 52, 19, 280, 75, 49, 147, 337, 308, 372, 333, 353, 364] + 58: [220, 348, 116, 178, 368, 344, 221, 347, 365, 216, 210, 192, 194, 213, 314, 86, 288, 175, 297, 318, 54, 82, 320, 312, 76, 269, 93, 232, 128, 137, 350, 94, 304, 219, 245, 27, 198, 222, 302, 166, 196, 89, 48, 111, 44, 16, 343, 70, 90, 295, 139, 217, 241, 25, 281, 177, 351, 107, 130, 160, 91, 7, 354, 319, 324, 135, 228, 339, 381, 71, 174, 197, 164, 140, 294, 1, 29, 57, 224, 17, 252, 218, 65, 363, 284, 332, 310, 249, 8, 11, 23, 39, 51, 62, 109, 117, 126, 49, 56, 380, 258, 40, 99, 115, 366, 199, 257, 242, 136, 255, 373, 151, 103, 66, 227, 342, 383, 9, 67, 68, 37, 259, 195, 34, 378, 356, 74, 278, 334, 315, 105, 149, 3, 64, 120, 349, 20, 69, 223, 361, 83, 268, 52, 155, 79, 207, 317, 293, 289, 75, 276, 0, 270, 327, 185, 337, 15, 285, 84, 275, 325, 85, 247, 208, 10, 323, 370, 32, 169, 180, 88, 96, 179, 146, 272, 188, 45, 362, 209, 87, 147, 33, 95, 38, 296, 134, 158, 159, 165, 184, 204, 212, 98, 148, 305, 42, 31, 377, 298, 129, 211, 72, 104, 277, 311, 266, 357, 133, 238, 308, 333, 182, 274, 4, 122, 121, 119, 100, 152, 256, 322, 307, 181, 80, 35, 141, 286, 331, 59, 55, 14, 63, 283, 306, 187, 316, 113, 292, 6, 176, 355, 2, 153, 244, 60, 144, 251, 206, 19, 273, 336, 353, 47, 214, 170, 261, 24, 183, 112, 168, 280, 231, 142, 154, 157, 193, 260, 191, 326, 163, 230, 229, 330, 233, 201, 108, 371, 101, 162, 145, 150, 41, 340, 239, 240, 246, 267, 279, 22, 352, 28, 264, 124, 203, 12, 271, 18, 262, 114, 234, 102, 92, 190, 50, 250, 77, 127, 78, 58, 263, 341, 106, 205, 225, 46, 329, 236, 26, 13, 328, 374, 376, 382, 248, 291, 300, 321, 131, 143, 123, 172, 202, 36, 360, 358, 171, 21, 97, 30, 303, 167, 73, 237, 226, 282, 372, 156, 254, 110, 189, 138, 173, 61, 200, 379, 359, 161, 346, 118, 235, 43, 53, 367, 243, 81, 313, 265, 375, 132, 5, 290, 299, 287, 125, 186, 301, 253, 215, 369, 309, 335, 338, 345, 364] + 59: [288, 364, 59, 23, 274, 212, 258, 231, 266, 236, 224, 353, 377, 371, 113, 178, 115, 339, 378, 52, 202, 150, 91, 346, 260, 315, 36, 180, 187, 213, 314, 259, 262, 43, 372, 251, 107, 366, 312, 195, 298, 114, 307, 345, 174, 163, 222, 373, 3, 124, 65, 223, 38, 47, 352, 344, 234, 358, 317, 206, 84, 166, 102, 64, 216, 191, 147, 360, 263, 29, 96, 269, 280, 348, 83, 101, 293, 144, 276, 318, 14, 201, 249, 322, 215, 341, 253, 205, 31, 155, 4, 15, 27, 32, 67, 71, 75, 151, 97, 158, 300, 28, 148, 232, 182, 226, 184, 143, 278, 172, 159, 37, 95, 304, 53, 383, 175, 169, 60, 106, 250, 11, 70, 140, 285, 243, 76, 319, 248, 12, 120, 292, 246, 332, 183, 25, 185, 24, 26, 199, 194, 13, 306, 283, 61, 305, 20, 359, 130, 221, 48, 82, 146, 282, 19, 17, 121, 272, 382, 177, 370, 237, 116, 68, 5, 105, 77, 340, 367, 323, 176, 108, 336, 74, 168, 129, 2, 125, 153, 167, 119, 110, 165, 230, 302, 351, 173, 90, 111, 117, 171, 189, 327, 379, 81, 44, 164, 245, 320, 324, 329, 21, 34, 181, 295, 196, 86, 291, 338, 270, 197, 316, 290, 7, 331, 149, 252, 45, 241, 254, 350, 78, 356, 369, 209, 69, 57, 325, 265, 30, 160, 365, 16, 56, 161, 41, 103, 100, 257, 239, 233, 204, 281, 138, 35, 380, 362, 289, 85, 354, 39, 79, 337, 286, 145, 55, 333, 122, 88, 33, 247, 349, 328, 66, 287, 299, 242, 273, 132, 92, 303, 54, 208, 87, 368, 72, 179, 326, 228, 238, 49, 271, 128, 135, 190, 210, 218, 235, 296, 186, 46, 137, 73, 334, 203, 284, 10, 294, 42, 363, 240, 297, 94, 170, 198, 62, 256, 342, 109, 112, 330, 313, 376, 127, 214, 200, 162, 118, 142, 381, 98, 141, 374, 219, 131, 18, 347, 93, 311, 309, 279, 136, 152, 50, 343, 229, 133, 51, 63, 40, 275, 123, 0, 9, 255, 89, 104, 1, 8, 301, 277, 139, 6, 308, 261, 227, 80, 134, 157, 335, 192, 220, 267, 156, 264, 22, 126, 193, 217, 188, 225, 58, 321, 207, 211, 310, 244, 154, 355, 99, 357, 268, 361, 375] + 60: [323, 112, 71, 318, 6, 332, 102, 38, 283, 192, 40, 78, 152, 215, 237, 354, 234, 166, 133, 273, 106, 374, 95, 265, 342, 269, 115, 24, 58, 32, 144, 36, 268, 179, 165, 232, 352, 167, 271, 47, 278, 316, 114, 351, 27, 365, 148, 353, 126, 371, 96, 285, 211, 315, 20, 177, 101, 43, 90, 163, 361, 53, 307, 169, 231, 59, 131, 21, 184, 41, 360, 287, 176, 4, 327, 324, 196, 376, 256, 154, 336, 272, 348, 325, 340, 187, 328, 65, 274, 9, 13, 14, 31, 67, 68, 73, 313, 312, 46, 236, 85, 51, 57, 135, 155, 337, 130, 139, 84, 302, 212, 110, 60, 260, 362, 190, 188, 264, 98, 129, 244, 369, 55, 364, 174, 213, 296, 350, 137, 162, 291, 49, 151, 314, 266, 48, 333, 158, 214, 77, 42, 113, 334, 164, 346, 319, 259, 311, 282, 44, 141, 25, 80, 203, 276, 8, 226, 197, 372, 138, 17, 128, 299, 202, 229, 72, 12, 370, 368, 23, 383, 91, 250, 56, 228, 199, 382, 279, 380, 26, 89, 347, 377, 69, 275, 86, 109, 122, 125, 147, 157, 160, 349, 153, 297, 123, 335, 194, 263, 107, 172, 39, 100, 195, 83, 2, 378, 111, 3, 140, 124, 15, 356, 99, 189, 230, 121, 132, 52, 64, 247, 246, 82, 358, 262, 11, 191, 117, 338, 366, 239, 217, 252, 45, 286, 173, 175, 88, 331, 204, 227, 345, 233, 240, 241, 223, 290, 293, 367, 156, 225, 178, 18, 19, 359, 103, 238, 220, 330, 251, 357, 363, 308, 216, 119, 373, 161, 22, 289, 105, 326, 10, 339, 97, 70, 104, 63, 118, 62, 198, 5, 200, 317, 168, 201, 206, 207, 210, 181, 134, 81, 54, 75, 306, 1, 180, 28, 108, 295, 381, 92, 344, 243, 305, 7, 193, 146, 61, 304, 219, 379, 294, 149, 300, 74, 120, 185, 37, 322, 245, 267, 284, 303, 310, 257, 186, 209, 258, 143, 50, 171, 255, 35, 116, 343, 222, 218, 145, 66, 76, 170, 159, 94, 281, 142, 0, 183, 235, 16, 29, 298, 136, 270, 208, 261, 248, 33, 277, 150, 320, 309, 182, 205, 34, 79, 127, 253, 93, 221, 87, 242, 30, 224, 321, 249, 254, 280, 288, 292, 301, 329, 341, 355, 375] + 61: [73, 341, 101, 225, 192, 176, 21, 263, 72, 28, 51, 366, 74, 122, 309, 45, 335, 367, 195, 200, 182, 338, 186, 382, 136, 202, 205, 216, 271, 377, 348, 286, 238, 224, 218, 83, 142, 169, 346, 327, 47, 17, 177, 105, 121, 170, 295, 129, 100, 29, 11, 23, 25, 143, 196, 9, 296, 68, 318, 69, 154, 323, 16, 19, 155, 302, 369, 114, 81, 181, 93, 3, 375, 149, 276, 371, 148, 63, 10, 364, 171, 210, 208, 6, 259, 266, 242, 58, 41, 0, 240, 275, 67, 332, 96, 347, 197, 36, 184, 91, 127, 183, 289, 61, 168, 261, 372, 243, 135, 27, 298, 215, 294, 280, 92, 284, 125, 111, 248, 297, 314, 55, 207, 285, 232, 119, 5, 365, 324, 308, 85, 246, 326, 262, 151, 70, 150, 352, 330, 59, 65, 157, 7, 77, 62, 333, 118, 245, 337, 131, 373, 78, 376, 103, 204, 368, 138, 126, 354, 79, 76, 161, 301, 44, 305, 18, 137, 312, 160, 99, 175, 292, 48, 49, 37, 56, 89, 359, 340, 339, 12, 325, 278, 124, 283, 8, 281, 123, 353, 106, 267, 329, 221, 33, 230, 277, 270, 350, 322, 203, 256, 255, 4, 66, 201, 288, 2, 166, 24, 198, 80, 120, 98, 293, 174, 213, 54, 102, 82, 97, 32, 117, 172, 362, 90, 290, 43, 315, 113, 264, 193, 253, 185, 235, 158, 383, 269, 217, 194, 336, 282, 206, 233, 22, 291, 317, 252, 279, 360, 144, 303, 95, 31, 84, 38, 378, 307, 310, 46, 251, 132, 247, 64, 226, 351, 116, 141, 34, 134, 187, 52, 115, 164, 222, 287, 254, 381, 236, 163, 345, 50, 231, 152, 53, 331, 13, 319, 39, 86, 159, 380, 257, 379, 214, 212, 311, 328, 190, 272, 227, 344, 30, 179, 14, 299, 112, 145, 223, 355, 146, 361, 139, 237, 358, 349, 220, 273, 304, 60, 167, 42, 108, 234, 107, 268, 147, 75, 229, 104, 239, 370, 374, 241, 342, 219, 88, 191, 260, 20, 211, 110, 130, 173, 258, 209, 156, 249, 244, 87, 178, 228, 71, 199, 363, 165, 306, 343, 313, 274, 357, 189, 57, 188, 128, 109, 321, 162, 334, 40, 26, 356, 250, 265, 153, 1, 180, 35, 94, 140, 320, 300, 15, 133, 316] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots384.yaml b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots384.yaml new file mode 100644 index 000000000000..474423374e6e --- /dev/null +++ b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 150, 177, 163, 29, 68, 126, 243, 156, 200, 208, 375, 244, 365, 192, 270, 263, 286, 340, 334, 250, 226, 288, 185, 111, 0, 235, 295, 80, 176, 349, 284, 56, 311, 337, 38, 257, 331, 59, 221, 237, 98, 115, 179, 164, 121, 88, 61, 378, 259, 348, 188, 242, 139, 282, 54, 233, 174, 148, 358, 172, 249, 369, 292, 67, 229, 206, 191, 251, 24, 168, 303, 166, 290, 162, 169, 62, 65, 276, 33, 34, 90, 28, 268, 85, 103, 269, 106, 127, 6, 42, 53, 357, 92, 339, 58, 256, 224, 155, 182, 122, 63, 37, 307, 196, 186, 248, 27, 52, 108, 116, 215, 93, 210, 64, 353, 285, 298, 8, 13, 74, 173, 86, 312, 51, 255, 318, 359, 43, 354, 44, 149, 32, 213, 19, 296, 14, 50, 96, 12, 319, 329, 344, 310, 222, 330, 383, 332, 347, 70, 278, 181, 313, 297, 300, 123, 66, 146, 140, 180, 267, 102, 294, 2, 381, 374, 291, 247, 356, 217, 119, 230, 104, 218, 209, 55, 131, 258, 261, 262, 377, 240, 178, 211, 326, 154, 305, 69, 194, 271, 273, 370, 236, 49, 322, 193, 30, 157, 144, 336, 101, 129, 94, 234, 91, 227, 275, 379, 82, 260, 316, 41, 231, 142, 228, 274, 245, 203, 71, 241, 204, 232, 280, 360, 11, 36, 112, 89, 87, 46, 75, 238, 328, 252, 338, 152, 239, 189, 281, 201, 371, 184, 345, 207, 81, 321, 3, 141, 105, 265, 138, 199, 223, 137, 363, 1, 382, 350, 161, 175, 367, 287, 325, 110, 225, 134, 84, 264, 26, 376, 333, 78, 283, 109, 7, 31, 133, 335, 366, 320, 299, 342, 212, 380, 125, 18, 130, 277, 254, 95, 151, 346, 153, 327, 272, 22, 117, 124, 170, 77, 308, 136, 372, 195, 197, 145, 40, 219, 9, 23, 355, 373, 135, 17, 302, 190, 35, 343, 293, 72, 107, 361, 114, 160, 5, 317, 216, 97, 73, 20, 352, 113, 301, 304, 128, 120, 279, 79, 159, 76, 21, 48, 198, 205, 341, 57, 368, 309, 39, 4, 246, 100, 147, 47, 15, 306, 220, 167, 323, 118, 45, 60, 351, 158, 202, 289, 16, 143, 183, 10, 324, 314, 187, 214, 99, 25, 83, 362, 253, 315, 266, 132, 364, 171] + 1: [163, 178, 126, 241, 41, 56, 25, 37, 301, 335, 53, 360, 102, 330, 65, 156, 26, 346, 344, 369, 151, 66, 304, 368, 319, 134, 85, 168, 270, 300, 46, 211, 285, 162, 47, 88, 98, 362, 14, 165, 87, 8, 232, 297, 160, 16, 258, 100, 365, 292, 235, 21, 366, 63, 243, 275, 274, 218, 206, 264, 205, 140, 286, 321, 186, 181, 240, 309, 193, 45, 144, 199, 42, 272, 1, 378, 192, 294, 320, 260, 78, 166, 176, 203, 189, 313, 318, 57, 172, 311, 314, 375, 207, 208, 354, 267, 201, 82, 80, 149, 325, 271, 185, 71, 266, 187, 228, 289, 76, 121, 326, 62, 290, 352, 103, 83, 135, 10, 257, 310, 174, 111, 106, 150, 0, 250, 383, 86, 317, 197, 219, 171, 61, 79, 2, 115, 11, 113, 54, 279, 183, 132, 341, 255, 22, 225, 118, 43, 70, 23, 24, 242, 125, 288, 282, 381, 298, 122, 17, 296, 51, 268, 226, 371, 253, 361, 157, 34, 137, 120, 18, 263, 328, 248, 233, 32, 216, 92, 223, 342, 101, 164, 107, 19, 230, 96, 119, 246, 188, 252, 339, 265, 180, 198, 31, 196, 75, 244, 224, 173, 281, 84, 114, 278, 236, 238, 338, 175, 284, 227, 333, 331, 367, 191, 276, 312, 158, 182, 308, 50, 210, 153, 351, 89, 234, 194, 143, 239, 147, 348, 20, 184, 293, 13, 262, 372, 90, 124, 337, 15, 277, 3, 141, 254, 6, 73, 195, 9, 74, 261, 4, 38, 133, 170, 139, 229, 315, 94, 353, 129, 316, 374, 117, 327, 382, 377, 81, 329, 39, 334, 138, 145, 99, 280, 213, 202, 231, 222, 161, 295, 350, 302, 49, 148, 217, 128, 159, 336, 91, 67, 380, 40, 179, 105, 59, 303, 146, 379, 93, 358, 237, 305, 249, 33, 324, 55, 12, 190, 209, 155, 349, 345, 72, 347, 251, 291, 5, 58, 29, 269, 212, 109, 116, 306, 256, 204, 364, 136, 36, 323, 77, 357, 221, 322, 359, 363, 169, 167, 44, 152, 123, 299, 283, 247, 245, 108, 52, 69, 95, 259, 370, 154, 35, 97, 356, 220, 340, 64, 307, 215, 177, 127, 200, 131, 30, 110, 273, 287, 355, 142, 214, 376, 60, 7, 332, 68, 48, 104, 27, 130, 373, 343, 28, 112] + 2: [143, 230, 5, 197, 111, 355, 26, 279, 196, 273, 68, 383, 10, 193, 142, 2, 36, 331, 95, 183, 234, 260, 337, 229, 248, 180, 81, 151, 322, 280, 324, 83, 204, 130, 203, 297, 46, 69, 372, 176, 77, 352, 329, 307, 37, 353, 282, 300, 53, 122, 221, 75, 84, 157, 22, 235, 311, 87, 27, 361, 308, 160, 154, 179, 134, 359, 216, 48, 113, 64, 228, 330, 43, 139, 137, 149, 101, 271, 220, 24, 323, 263, 50, 187, 241, 309, 39, 362, 281, 57, 243, 74, 182, 80, 326, 374, 132, 314, 1, 100, 191, 73, 31, 146, 61, 206, 210, 58, 253, 103, 38, 268, 4, 272, 99, 251, 227, 240, 339, 199, 112, 304, 298, 189, 166, 278, 51, 217, 79, 299, 327, 201, 41, 186, 49, 94, 275, 231, 292, 98, 117, 380, 32, 366, 108, 86, 259, 125, 284, 286, 45, 274, 294, 140, 188, 236, 65, 15, 305, 152, 195, 141, 226, 158, 378, 266, 316, 381, 313, 377, 343, 192, 153, 306, 123, 340, 342, 90, 269, 138, 332, 129, 256, 318, 35, 52, 291, 267, 283, 249, 270, 190, 215, 91, 102, 34, 350, 66, 325, 63, 209, 17, 67, 148, 265, 181, 312, 219, 211, 144, 302, 59, 382, 88, 287, 40, 89, 168, 28, 321, 173, 245, 105, 333, 159, 356, 126, 218, 16, 169, 12, 29, 334, 233, 54, 164, 72, 127, 119, 161, 194, 62, 14, 78, 97, 131, 368, 200, 174, 170, 92, 76, 349, 276, 364, 104, 346, 261, 71, 167, 177, 369, 237, 246, 93, 19, 357, 60, 110, 44, 133, 82, 6, 7, 262, 198, 360, 116, 9, 257, 367, 207, 107, 205, 222, 13, 208, 375, 115, 290, 341, 213, 109, 118, 42, 244, 30, 347, 55, 20, 145, 348, 252, 23, 239, 319, 288, 8, 310, 289, 301, 379, 223, 232, 351, 33, 155, 70, 172, 85, 303, 224, 293, 21, 171, 175, 121, 0, 25, 264, 370, 335, 345, 214, 202, 363, 3, 242, 96, 162, 185, 135, 184, 11, 285, 18, 156, 120, 163, 250, 106, 150, 47, 178, 371, 338, 147, 317, 354, 255, 295, 212, 225, 247, 124, 56, 296, 328, 165, 344, 320, 128, 277, 254, 136, 365, 238, 376, 373, 114, 336, 315, 358, 258] + 3: [66, 113, 237, 184, 91, 110, 206, 378, 88, 90, 124, 326, 5, 247, 335, 233, 146, 324, 278, 303, 311, 222, 301, 370, 34, 236, 169, 383, 115, 20, 356, 101, 285, 45, 159, 4, 244, 371, 332, 52, 9, 257, 198, 199, 292, 59, 47, 38, 310, 56, 150, 337, 170, 325, 316, 21, 84, 32, 155, 299, 112, 30, 196, 291, 373, 362, 372, 338, 147, 265, 33, 341, 260, 13, 318, 215, 120, 189, 195, 118, 109, 229, 345, 256, 105, 268, 286, 361, 85, 153, 379, 107, 360, 60, 347, 17, 377, 317, 151, 331, 102, 250, 296, 290, 364, 359, 243, 218, 197, 280, 366, 298, 15, 259, 283, 190, 191, 369, 69, 152, 365, 31, 62, 50, 201, 58, 77, 11, 87, 154, 374, 41, 235, 65, 255, 322, 249, 130, 28, 354, 242, 141, 308, 167, 319, 51, 97, 49, 26, 164, 212, 182, 202, 161, 351, 126, 380, 117, 44, 276, 232, 376, 86, 0, 140, 368, 23, 173, 39, 209, 208, 334, 139, 336, 267, 174, 29, 8, 277, 300, 217, 355, 125, 306, 128, 106, 293, 179, 251, 1, 75, 185, 22, 230, 192, 158, 137, 343, 122, 177, 240, 162, 203, 176, 103, 270, 214, 43, 305, 381, 309, 183, 323, 95, 89, 193, 79, 35, 131, 363, 42, 168, 37, 239, 12, 78, 262, 98, 200, 327, 82, 160, 281, 123, 68, 16, 156, 148, 127, 226, 302, 275, 225, 348, 6, 63, 227, 74, 25, 136, 288, 114, 108, 53, 307, 100, 96, 294, 346, 133, 93, 284, 143, 175, 40, 46, 19, 24, 180, 178, 211, 221, 64, 271, 99, 297, 224, 57, 48, 328, 187, 367, 72, 269, 245, 73, 205, 315, 18, 71, 350, 157, 213, 207, 10, 304, 223, 279, 295, 92, 274, 94, 266, 144, 353, 7, 132, 254, 258, 329, 241, 320, 27, 263, 171, 333, 145, 339, 2, 231, 36, 382, 289, 111, 181, 344, 135, 246, 216, 67, 83, 228, 116, 234, 330, 272, 163, 313, 14, 80, 252, 238, 342, 287, 119, 264, 142, 149, 204, 55, 129, 104, 248, 70, 61, 282, 134, 121, 253, 349, 3, 194, 314, 261, 81, 186, 358, 138, 219, 357, 172, 312, 375, 166, 273, 76, 340, 188, 165, 321, 220, 210, 54, 352] + 4: [343, 253, 139, 361, 49, 35, 183, 133, 379, 216, 158, 66, 206, 237, 355, 311, 380, 134, 266, 231, 338, 182, 277, 282, 331, 86, 325, 151, 339, 22, 283, 85, 98, 261, 284, 220, 121, 327, 127, 10, 162, 202, 363, 242, 90, 107, 305, 332, 3, 320, 45, 345, 275, 365, 251, 82, 0, 360, 359, 70, 80, 92, 303, 140, 268, 129, 123, 293, 212, 337, 336, 351, 271, 259, 257, 47, 372, 357, 312, 207, 335, 252, 33, 232, 25, 267, 51, 249, 109, 205, 362, 144, 89, 225, 104, 120, 315, 258, 88, 143, 197, 76, 160, 8, 260, 243, 330, 341, 108, 288, 292, 29, 146, 211, 97, 48, 102, 301, 119, 58, 214, 375, 172, 171, 155, 200, 196, 280, 110, 116, 114, 169, 229, 310, 309, 157, 180, 71, 378, 316, 238, 299, 67, 321, 209, 18, 250, 105, 342, 145, 57, 73, 177, 99, 168, 323, 269, 328, 84, 184, 289, 130, 126, 247, 245, 326, 192, 63, 296, 340, 350, 5, 36, 347, 32, 27, 7, 278, 290, 62, 132, 69, 370, 176, 295, 147, 255, 374, 248, 285, 2, 287, 371, 34, 219, 19, 348, 167, 61, 218, 161, 187, 221, 189, 60, 125, 137, 138, 230, 304, 156, 186, 191, 265, 154, 208, 23, 188, 24, 376, 302, 329, 77, 314, 344, 369, 42, 256, 149, 74, 322, 294, 26, 170, 111, 115, 353, 11, 373, 20, 15, 152, 300, 281, 136, 274, 246, 101, 333, 141, 244, 166, 297, 262, 100, 306, 175, 291, 223, 203, 163, 91, 382, 272, 95, 254, 222, 239, 165, 377, 4, 83, 217, 317, 264, 356, 352, 383, 135, 349, 103, 122, 31, 215, 55, 164, 195, 233, 226, 148, 199, 228, 6, 56, 318, 235, 198, 153, 54, 307, 14, 64, 367, 178, 234, 334, 213, 81, 1, 368, 174, 17, 319, 142, 124, 9, 241, 13, 106, 364, 68, 159, 276, 72, 93, 131, 240, 52, 346, 118, 201, 21, 112, 224, 190, 227, 87, 117, 12, 16, 308, 298, 113, 43, 28, 179, 358, 53, 185, 313, 354, 50, 263, 181, 40, 381, 173, 366, 270, 79, 194, 273, 37, 193, 150, 59, 41, 44, 204, 38, 324, 65, 75, 46, 94, 39, 78, 210, 286, 96, 279, 30, 236, 128] + 5: [61, 65, 211, 252, 198, 131, 59, 251, 153, 46, 51, 56, 241, 254, 344, 221, 378, 86, 347, 307, 294, 326, 286, 354, 213, 260, 30, 72, 200, 304, 171, 181, 26, 45, 312, 366, 341, 119, 291, 283, 18, 297, 120, 282, 273, 348, 321, 75, 128, 308, 265, 145, 331, 173, 155, 140, 24, 301, 274, 227, 63, 90, 191, 205, 342, 322, 126, 311, 226, 280, 170, 368, 214, 208, 60, 255, 192, 74, 222, 2, 258, 16, 380, 14, 300, 9, 80, 323, 129, 189, 133, 328, 310, 83, 96, 302, 355, 172, 178, 78, 352, 335, 8, 206, 143, 295, 144, 20, 138, 383, 77, 271, 156, 125, 101, 350, 10, 27, 127, 44, 190, 21, 47, 372, 17, 338, 4, 332, 38, 3, 104, 266, 19, 250, 330, 134, 235, 68, 316, 267, 336, 42, 225, 196, 249, 259, 81, 122, 212, 364, 66, 358, 108, 210, 228, 186, 230, 150, 275, 361, 71, 299, 373, 292, 36, 180, 278, 118, 84, 159, 1, 365, 130, 246, 31, 70, 69, 207, 35, 115, 362, 201, 48, 87, 103, 114, 92, 139, 202, 116, 112, 177, 296, 231, 98, 381, 279, 7, 233, 137, 319, 320, 33, 12, 253, 284, 136, 229, 363, 135, 58, 185, 146, 357, 82, 157, 194, 272, 160, 169, 102, 293, 182, 163, 369, 333, 195, 79, 343, 219, 281, 29, 349, 40, 187, 197, 340, 248, 367, 303, 217, 6, 5, 188, 351, 88, 209, 151, 67, 204, 28, 99, 382, 121, 237, 13, 359, 270, 32, 290, 73, 234, 167, 179, 346, 288, 256, 223, 215, 89, 22, 268, 174, 34, 218, 329, 106, 257, 334, 360, 374, 107, 148, 11, 85, 339, 110, 277, 276, 52, 154, 370, 41, 147, 327, 37, 264, 152, 315, 345, 239, 240, 149, 236, 62, 244, 224, 53, 97, 109, 142, 57, 100, 356, 306, 49, 132, 43, 371, 313, 318, 175, 54, 216, 39, 161, 325, 232, 113, 263, 95, 245, 285, 123, 164, 337, 105, 287, 141, 262, 379, 91, 25, 375, 238, 314, 377, 93, 353, 317, 289, 94, 376, 168, 298, 309, 158, 117, 176, 23, 305, 269, 165, 193, 64, 247, 124, 15, 184, 324, 220, 242, 183, 203, 76, 162, 55, 199, 261, 111, 243, 0, 166, 50] + 6: [97, 378, 283, 178, 234, 336, 190, 139, 220, 69, 60, 254, 105, 381, 357, 106, 47, 250, 171, 72, 153, 331, 247, 257, 142, 350, 232, 118, 259, 128, 317, 77, 154, 87, 86, 261, 346, 70, 244, 108, 303, 263, 192, 265, 252, 40, 90, 161, 308, 371, 318, 84, 239, 113, 278, 111, 219, 365, 242, 217, 81, 224, 272, 73, 347, 195, 313, 172, 344, 204, 315, 342, 322, 55, 173, 88, 319, 300, 109, 231, 64, 332, 110, 63, 329, 147, 349, 130, 62, 351, 7, 26, 20, 75, 325, 373, 76, 341, 175, 187, 188, 156, 185, 49, 282, 143, 262, 116, 362, 18, 215, 39, 203, 338, 302, 213, 314, 132, 71, 164, 274, 269, 67, 216, 57, 196, 176, 245, 304, 107, 58, 271, 243, 145, 193, 268, 285, 288, 348, 339, 91, 61, 149, 323, 376, 249, 229, 181, 100, 184, 151, 41, 68, 327, 157, 361, 82, 14, 48, 125, 134, 226, 256, 374, 19, 382, 275, 35, 32, 59, 326, 260, 368, 79, 301, 182, 101, 211, 10, 206, 4, 194, 199, 309, 364, 251, 266, 38, 29, 228, 135, 320, 305, 212, 28, 9, 94, 255, 370, 179, 311, 321, 15, 6, 276, 355, 168, 8, 379, 150, 17, 51, 78, 92, 353, 167, 53, 297, 306, 209, 375, 112, 287, 165, 369, 158, 221, 286, 43, 284, 122, 3, 16, 366, 290, 102, 89, 241, 170, 214, 343, 146, 267, 11, 13, 367, 289, 95, 115, 30, 144, 296, 104, 335, 126, 124, 42, 123, 328, 162, 222, 1, 273, 52, 174, 177, 160, 205, 36, 189, 155, 0, 230, 12, 383, 324, 207, 197, 316, 198, 298, 235, 152, 121, 238, 140, 2, 65, 258, 166, 27, 46, 359, 293, 129, 345, 236, 202, 133, 136, 237, 159, 148, 358, 96, 292, 337, 294, 131, 127, 191, 363, 253, 233, 295, 120, 307, 119, 291, 277, 352, 279, 44, 264, 137, 23, 138, 83, 299, 312, 360, 24, 183, 93, 340, 186, 240, 66, 5, 31, 163, 56, 34, 270, 380, 223, 114, 280, 210, 225, 248, 50, 334, 74, 80, 21, 201, 141, 200, 25, 281, 99, 103, 180, 45, 246, 208, 354, 377, 37, 169, 98, 22, 330, 372, 333, 54, 310, 85, 117, 356, 227, 218, 33] + 7: [327, 260, 48, 181, 164, 119, 250, 177, 98, 314, 32, 335, 310, 247, 345, 216, 316, 80, 39, 230, 205, 279, 135, 60, 7, 308, 280, 61, 302, 29, 339, 210, 62, 321, 358, 220, 141, 233, 149, 222, 106, 88, 228, 266, 138, 122, 357, 50, 274, 258, 342, 27, 281, 262, 337, 255, 213, 368, 204, 322, 152, 373, 22, 79, 75, 218, 128, 195, 290, 326, 365, 364, 49, 341, 28, 188, 131, 100, 168, 319, 170, 241, 16, 129, 58, 306, 309, 52, 268, 295, 96, 334, 198, 352, 145, 240, 292, 163, 187, 97, 256, 215, 285, 374, 224, 132, 26, 180, 380, 155, 65, 120, 151, 36, 111, 379, 332, 12, 355, 257, 175, 369, 176, 15, 94, 307, 17, 325, 219, 333, 140, 190, 6, 353, 146, 225, 35, 69, 173, 43, 299, 127, 199, 338, 182, 298, 350, 354, 114, 113, 349, 296, 221, 243, 311, 186, 102, 56, 227, 193, 320, 194, 231, 252, 359, 143, 282, 42, 275, 362, 239, 378, 328, 47, 217, 360, 21, 377, 167, 20, 83, 121, 301, 93, 184, 112, 1, 297, 346, 165, 34, 348, 192, 25, 133, 118, 329, 249, 74, 55, 235, 191, 206, 130, 200, 134, 323, 153, 115, 166, 123, 317, 57, 66, 236, 160, 263, 201, 108, 99, 370, 73, 125, 23, 284, 300, 234, 237, 273, 226, 232, 87, 126, 209, 254, 3, 95, 197, 376, 110, 64, 103, 361, 71, 77, 183, 278, 84, 356, 344, 259, 136, 211, 67, 251, 38, 351, 169, 10, 277, 304, 331, 154, 371, 41, 18, 336, 271, 0, 313, 4, 2, 383, 264, 214, 19, 293, 53, 283, 101, 366, 207, 59, 381, 45, 150, 30, 179, 375, 105, 162, 253, 244, 294, 289, 270, 117, 372, 189, 109, 5, 44, 40, 291, 269, 286, 312, 124, 242, 147, 156, 91, 142, 340, 157, 13, 104, 14, 267, 148, 70, 330, 89, 245, 107, 86, 196, 68, 37, 287, 367, 303, 246, 172, 223, 276, 229, 382, 272, 139, 174, 11, 82, 288, 81, 158, 24, 92, 137, 305, 8, 261, 203, 363, 85, 63, 315, 54, 144, 202, 265, 343, 238, 46, 212, 324, 248, 31, 208, 90, 9, 185, 178, 51, 161, 116, 318, 33, 72, 78, 159, 347, 76, 171] + 8: [360, 64, 157, 156, 374, 150, 261, 198, 196, 72, 42, 379, 259, 299, 378, 145, 287, 193, 77, 80, 289, 19, 100, 207, 253, 108, 337, 380, 86, 69, 147, 65, 368, 120, 177, 45, 25, 151, 307, 265, 236, 357, 324, 208, 138, 49, 127, 170, 290, 161, 13, 252, 328, 22, 83, 216, 137, 334, 2, 8, 73, 90, 136, 99, 260, 33, 24, 245, 343, 57, 296, 234, 359, 104, 224, 36, 316, 131, 75, 119, 165, 183, 233, 4, 116, 249, 266, 105, 295, 54, 271, 285, 363, 0, 55, 96, 5, 273, 7, 228, 255, 280, 172, 38, 121, 309, 300, 91, 217, 230, 310, 37, 325, 306, 315, 348, 354, 305, 274, 213, 67, 281, 106, 12, 251, 209, 128, 6, 231, 303, 48, 197, 254, 98, 184, 372, 144, 9, 369, 158, 270, 203, 210, 242, 31, 182, 338, 312, 30, 11, 181, 283, 179, 275, 141, 211, 59, 248, 297, 126, 68, 93, 323, 103, 269, 53, 344, 58, 135, 204, 292, 241, 95, 364, 166, 23, 92, 178, 164, 341, 383, 118, 352, 155, 143, 322, 134, 84, 319, 331, 301, 132, 21, 229, 218, 154, 81, 377, 311, 226, 240, 304, 192, 318, 187, 191, 257, 314, 79, 35, 153, 146, 123, 70, 82, 163, 367, 371, 238, 173, 186, 18, 346, 133, 330, 40, 320, 358, 101, 87, 110, 41, 339, 175, 185, 370, 329, 342, 124, 227, 264, 152, 214, 282, 276, 97, 39, 109, 298, 215, 194, 60, 335, 237, 373, 382, 85, 89, 277, 76, 26, 43, 222, 262, 239, 353, 250, 272, 122, 168, 195, 333, 202, 219, 50, 114, 206, 174, 149, 47, 235, 159, 326, 220, 180, 51, 256, 63, 201, 308, 279, 381, 291, 349, 225, 34, 258, 112, 317, 52, 117, 212, 32, 190, 200, 205, 111, 351, 160, 286, 288, 376, 362, 221, 3, 88, 74, 115, 148, 188, 169, 167, 130, 125, 263, 28, 350, 327, 56, 375, 365, 15, 340, 94, 139, 1, 162, 62, 246, 243, 223, 44, 107, 27, 78, 278, 113, 66, 61, 232, 71, 16, 140, 14, 347, 129, 244, 142, 361, 176, 321, 10, 302, 336, 20, 294, 366, 356, 17, 189, 267, 332, 284, 247, 293, 313, 199, 29, 46, 171, 268, 355, 345, 102] + 9: [92, 198, 329, 32, 285, 268, 0, 311, 4, 181, 202, 18, 117, 322, 71, 216, 160, 58, 149, 40, 61, 168, 173, 26, 304, 29, 380, 132, 367, 153, 54, 87, 267, 189, 343, 175, 245, 79, 286, 38, 290, 368, 278, 257, 182, 338, 234, 57, 255, 172, 381, 242, 14, 9, 363, 264, 97, 186, 124, 332, 276, 307, 197, 110, 310, 232, 86, 43, 63, 362, 27, 283, 349, 19, 351, 370, 121, 133, 377, 297, 314, 269, 150, 72, 308, 379, 213, 254, 305, 346, 170, 108, 119, 28, 122, 340, 327, 81, 252, 37, 159, 56, 317, 3, 334, 116, 323, 238, 39, 354, 148, 112, 301, 162, 293, 243, 93, 204, 24, 365, 355, 166, 299, 344, 287, 282, 151, 154, 318, 41, 105, 66, 104, 248, 275, 203, 12, 125, 326, 68, 73, 378, 126, 348, 31, 78, 260, 184, 376, 209, 312, 187, 277, 107, 35, 206, 5, 75, 8, 261, 2, 375, 48, 188, 366, 361, 91, 265, 226, 212, 215, 222, 138, 67, 258, 135, 88, 192, 142, 230, 288, 235, 50, 373, 371, 333, 221, 59, 225, 101, 200, 77, 94, 10, 372, 179, 127, 136, 131, 302, 228, 128, 195, 36, 171, 246, 65, 205, 253, 270, 21, 337, 325, 223, 273, 152, 263, 147, 280, 155, 217, 23, 98, 356, 306, 364, 256, 16, 106, 324, 60, 233, 291, 134, 199, 167, 176, 118, 164, 25, 295, 103, 196, 165, 146, 51, 22, 158, 169, 183, 163, 11, 111, 13, 52, 300, 320, 76, 229, 374, 240, 279, 157, 7, 47, 95, 315, 219, 201, 341, 102, 30, 292, 247, 129, 336, 237, 319, 382, 174, 115, 330, 178, 335, 266, 328, 250, 156, 185, 244, 96, 74, 83, 271, 309, 144, 33, 321, 145, 70, 218, 208, 100, 190, 114, 251, 347, 34, 191, 44, 194, 236, 313, 64, 143, 357, 15, 214, 177, 82, 120, 180, 20, 207, 359, 80, 109, 45, 42, 224, 211, 193, 383, 284, 85, 369, 316, 49, 220, 350, 1, 210, 239, 161, 137, 231, 99, 139, 141, 339, 303, 62, 358, 352, 227, 69, 140, 296, 113, 53, 90, 262, 331, 353, 272, 281, 342, 360, 289, 6, 259, 123, 274, 17, 130, 46, 84, 298, 89, 241, 345, 249, 294, 55] + 10: [319, 263, 196, 76, 345, 360, 149, 233, 296, 9, 37, 267, 140, 13, 209, 111, 322, 251, 84, 156, 238, 283, 297, 369, 5, 8, 40, 154, 368, 208, 271, 229, 288, 219, 379, 97, 254, 220, 257, 206, 147, 118, 157, 242, 23, 295, 29, 336, 300, 215, 114, 47, 293, 274, 38, 32, 129, 75, 256, 230, 12, 50, 282, 287, 355, 158, 120, 143, 260, 82, 358, 68, 320, 126, 22, 351, 211, 312, 284, 212, 155, 214, 81, 357, 51, 125, 239, 113, 375, 373, 200, 273, 150, 228, 290, 366, 365, 137, 167, 144, 153, 304, 353, 356, 189, 266, 72, 311, 197, 25, 247, 323, 343, 169, 173, 205, 334, 347, 270, 291, 194, 268, 279, 101, 161, 19, 213, 234, 204, 218, 145, 105, 142, 327, 109, 64, 185, 307, 305, 303, 104, 172, 188, 324, 264, 93, 337, 225, 117, 138, 67, 272, 316, 249, 90, 341, 27, 112, 162, 354, 123, 96, 100, 4, 6, 344, 14, 381, 55, 246, 69, 57, 178, 66, 91, 252, 186, 61, 132, 21, 85, 328, 278, 184, 298, 65, 235, 35, 16, 207, 277, 349, 180, 255, 325, 79, 179, 241, 269, 363, 98, 231, 0, 87, 339, 378, 340, 174, 163, 159, 302, 165, 133, 333, 10, 70, 151, 192, 289, 102, 78, 330, 248, 136, 77, 376, 103, 107, 335, 338, 42, 39, 183, 131, 62, 17, 34, 352, 227, 236, 265, 294, 262, 73, 171, 210, 115, 11, 49, 359, 198, 232, 106, 41, 371, 275, 313, 226, 222, 217, 203, 191, 202, 110, 24, 301, 48, 2, 170, 99, 308, 240, 374, 285, 250, 56, 122, 74, 383, 317, 362, 299, 372, 224, 276, 361, 63, 89, 121, 310, 181, 309, 195, 237, 58, 130, 377, 382, 199, 135, 315, 20, 15, 332, 370, 364, 127, 86, 53, 83, 28, 253, 216, 146, 141, 92, 134, 182, 7, 243, 148, 306, 33, 321, 329, 26, 71, 45, 261, 326, 80, 292, 88, 281, 43, 166, 18, 164, 108, 367, 1, 124, 201, 348, 190, 223, 116, 46, 314, 59, 258, 193, 175, 244, 187, 119, 94, 44, 52, 176, 286, 31, 221, 331, 54, 177, 380, 259, 342, 280, 128, 3, 60, 346, 36, 318, 95, 350, 139, 168, 245, 30, 152, 160] + 11: [130, 376, 102, 217, 286, 154, 161, 69, 226, 89, 365, 327, 338, 302, 258, 138, 5, 112, 305, 175, 372, 155, 261, 269, 314, 260, 145, 144, 52, 136, 6, 45, 26, 10, 58, 336, 353, 311, 33, 182, 84, 210, 342, 216, 51, 212, 134, 335, 296, 77, 171, 22, 165, 146, 47, 213, 315, 316, 208, 219, 354, 324, 113, 7, 323, 222, 114, 328, 82, 329, 243, 241, 163, 252, 233, 345, 330, 177, 303, 367, 176, 133, 96, 294, 359, 118, 383, 19, 184, 4, 167, 344, 253, 334, 42, 12, 110, 156, 99, 0, 343, 381, 23, 170, 80, 151, 255, 242, 245, 193, 264, 355, 358, 306, 104, 71, 379, 122, 153, 61, 98, 15, 326, 218, 141, 282, 250, 280, 159, 374, 14, 209, 59, 158, 115, 225, 194, 271, 201, 272, 361, 111, 123, 105, 377, 285, 38, 332, 178, 337, 78, 371, 75, 56, 298, 270, 168, 179, 120, 62, 309, 37, 310, 183, 1, 202, 281, 109, 339, 357, 119, 24, 284, 318, 196, 124, 117, 251, 197, 81, 30, 135, 293, 263, 231, 274, 140, 132, 139, 53, 321, 191, 87, 126, 149, 162, 54, 204, 207, 320, 249, 27, 198, 259, 211, 279, 125, 142, 18, 108, 21, 268, 229, 333, 25, 256, 90, 100, 3, 29, 192, 70, 169, 297, 348, 347, 160, 41, 97, 292, 234, 174, 188, 106, 356, 277, 238, 239, 232, 307, 83, 248, 40, 267, 351, 128, 148, 313, 107, 93, 203, 186, 147, 364, 164, 57, 74, 103, 288, 304, 368, 276, 220, 273, 215, 44, 65, 199, 331, 95, 20, 346, 2, 172, 86, 31, 295, 378, 91, 64, 143, 200, 173, 380, 246, 181, 72, 257, 85, 116, 227, 300, 152, 39, 13, 66, 127, 92, 340, 101, 362, 34, 349, 79, 230, 375, 185, 68, 180, 189, 223, 50, 150, 36, 363, 157, 247, 131, 17, 94, 262, 373, 49, 289, 35, 366, 291, 43, 46, 360, 322, 214, 352, 317, 137, 308, 235, 11, 299, 190, 275, 244, 205, 319, 370, 195, 341, 254, 301, 265, 187, 48, 237, 266, 240, 278, 60, 28, 206, 9, 221, 121, 88, 325, 236, 73, 228, 283, 350, 166, 224, 55, 63, 312, 67, 129, 8, 382, 287, 290, 369, 16, 76, 32] + 12: [255, 288, 57, 291, 15, 350, 119, 249, 121, 257, 374, 252, 55, 72, 94, 100, 324, 73, 110, 272, 372, 220, 302, 42, 204, 292, 90, 259, 332, 368, 209, 107, 337, 283, 1, 45, 4, 38, 86, 297, 30, 282, 211, 269, 206, 64, 250, 225, 120, 109, 233, 354, 137, 25, 39, 322, 261, 286, 188, 165, 161, 11, 13, 244, 85, 375, 212, 79, 242, 213, 373, 53, 130, 175, 97, 208, 281, 317, 133, 171, 197, 144, 309, 0, 189, 169, 122, 112, 312, 310, 96, 236, 145, 314, 141, 366, 158, 344, 103, 260, 278, 330, 89, 265, 129, 157, 227, 52, 298, 319, 235, 336, 329, 47, 316, 320, 338, 328, 331, 199, 327, 49, 22, 151, 123, 216, 108, 102, 164, 23, 201, 114, 61, 325, 154, 379, 24, 383, 54, 125, 307, 166, 56, 162, 149, 363, 267, 50, 176, 205, 147, 153, 362, 326, 155, 200, 29, 280, 76, 20, 365, 274, 34, 81, 143, 360, 359, 222, 78, 156, 168, 192, 340, 68, 229, 27, 104, 80, 74, 253, 185, 67, 321, 355, 5, 186, 44, 93, 299, 221, 228, 66, 226, 341, 167, 290, 170, 382, 218, 364, 264, 63, 238, 370, 191, 62, 356, 36, 140, 217, 106, 113, 230, 7, 248, 43, 190, 46, 17, 124, 361, 195, 215, 60, 101, 135, 31, 357, 231, 138, 37, 159, 334, 263, 174, 131, 247, 111, 88, 239, 303, 203, 219, 126, 306, 295, 194, 87, 371, 273, 318, 376, 315, 256, 187, 270, 12, 128, 285, 18, 117, 118, 163, 377, 127, 333, 28, 349, 51, 6, 346, 16, 2, 241, 14, 304, 234, 224, 335, 115, 183, 207, 70, 178, 258, 184, 275, 240, 223, 172, 245, 289, 181, 3, 152, 10, 33, 308, 380, 77, 9, 313, 173, 40, 65, 279, 146, 343, 367, 26, 342, 353, 82, 150, 35, 71, 369, 339, 348, 323, 300, 271, 59, 142, 254, 262, 232, 305, 293, 148, 352, 347, 19, 182, 311, 99, 177, 243, 381, 296, 98, 134, 276, 266, 116, 180, 301, 193, 41, 84, 58, 214, 179, 284, 105, 345, 196, 210, 92, 21, 378, 69, 48, 287, 198, 75, 268, 277, 160, 294, 91, 358, 83, 136, 139, 351, 32, 246, 251, 8, 237, 132, 202, 95] + 13: [170, 46, 58, 40, 363, 147, 321, 333, 183, 267, 326, 245, 53, 73, 205, 336, 90, 20, 354, 155, 192, 204, 7, 175, 108, 0, 95, 17, 118, 198, 246, 320, 22, 239, 361, 222, 37, 220, 263, 94, 377, 285, 343, 152, 97, 368, 230, 32, 232, 96, 231, 88, 306, 369, 275, 208, 378, 145, 74, 52, 89, 379, 381, 154, 234, 233, 284, 258, 293, 156, 140, 316, 18, 81, 100, 11, 304, 75, 376, 334, 210, 26, 262, 221, 303, 345, 325, 308, 287, 119, 372, 224, 317, 148, 33, 78, 195, 77, 172, 134, 298, 45, 69, 229, 42, 179, 291, 256, 125, 337, 242, 113, 217, 61, 4, 60, 215, 47, 160, 165, 87, 137, 21, 102, 162, 295, 289, 158, 300, 237, 225, 177, 241, 68, 305, 133, 115, 365, 167, 203, 28, 269, 218, 206, 80, 194, 51, 340, 370, 176, 31, 143, 6, 355, 344, 121, 15, 34, 56, 257, 270, 213, 138, 1, 63, 272, 277, 301, 44, 9, 24, 209, 335, 279, 112, 91, 2, 211, 25, 196, 76, 223, 136, 244, 71, 124, 214, 101, 212, 341, 66, 193, 362, 318, 41, 254, 142, 323, 16, 261, 122, 79, 382, 93, 139, 288, 106, 259, 358, 313, 249, 117, 352, 290, 367, 126, 329, 274, 282, 330, 338, 131, 188, 294, 373, 219, 144, 359, 243, 36, 248, 127, 70, 235, 190, 85, 264, 332, 200, 315, 114, 30, 169, 276, 310, 35, 49, 297, 299, 65, 62, 166, 146, 105, 174, 268, 123, 13, 83, 157, 99, 347, 82, 314, 14, 12, 39, 201, 271, 159, 184, 273, 178, 98, 228, 107, 164, 331, 84, 92, 72, 38, 185, 346, 173, 251, 302, 350, 149, 238, 3, 342, 199, 171, 364, 356, 266, 10, 339, 253, 129, 163, 357, 348, 187, 5, 103, 383, 181, 371, 255, 324, 328, 189, 57, 110, 366, 307, 227, 54, 292, 375, 43, 374, 29, 48, 153, 296, 360, 19, 182, 180, 191, 236, 351, 27, 23, 280, 86, 55, 186, 116, 111, 309, 281, 168, 240, 283, 380, 8, 319, 50, 250, 130, 104, 226, 128, 349, 120, 247, 260, 312, 265, 135, 197, 109, 64, 202, 67, 207, 141, 151, 327, 161, 286, 59, 311, 150, 353, 252, 278, 132, 322, 216] + 14: [229, 228, 97, 182, 208, 246, 326, 133, 178, 377, 225, 264, 248, 173, 153, 46, 80, 185, 139, 51, 352, 205, 331, 165, 160, 383, 146, 3, 279, 166, 314, 201, 266, 109, 234, 0, 335, 63, 251, 195, 127, 281, 35, 84, 72, 29, 315, 174, 345, 274, 49, 254, 342, 271, 13, 328, 193, 346, 296, 380, 253, 313, 96, 273, 305, 190, 362, 236, 192, 38, 73, 50, 255, 56, 175, 99, 39, 250, 197, 343, 370, 382, 227, 105, 336, 12, 43, 262, 267, 18, 366, 247, 91, 340, 318, 88, 70, 147, 112, 23, 177, 374, 125, 158, 68, 104, 128, 101, 237, 242, 183, 376, 338, 74, 179, 293, 11, 219, 292, 260, 211, 275, 199, 142, 117, 360, 270, 33, 369, 333, 349, 48, 129, 223, 224, 42, 171, 78, 258, 55, 180, 134, 280, 67, 143, 71, 17, 235, 32, 269, 220, 59, 324, 301, 203, 222, 359, 202, 356, 114, 278, 136, 322, 57, 120, 252, 351, 289, 4, 155, 60, 348, 231, 21, 216, 149, 184, 108, 40, 156, 176, 89, 148, 285, 365, 238, 87, 132, 144, 243, 303, 98, 15, 308, 137, 337, 95, 79, 44, 135, 288, 363, 286, 82, 341, 152, 282, 106, 257, 20, 230, 62, 140, 283, 119, 83, 76, 378, 8, 118, 198, 61, 157, 206, 163, 339, 256, 5, 330, 168, 1, 311, 309, 304, 263, 31, 320, 27, 54, 196, 204, 355, 287, 353, 124, 162, 294, 215, 69, 37, 169, 347, 298, 300, 217, 28, 358, 334, 299, 194, 381, 368, 131, 272, 110, 126, 164, 150, 52, 154, 354, 161, 357, 22, 187, 116, 276, 200, 240, 214, 375, 361, 312, 7, 81, 307, 138, 212, 30, 327, 77, 186, 151, 213, 26, 379, 319, 159, 172, 226, 86, 115, 350, 310, 103, 36, 45, 367, 181, 65, 210, 249, 2, 122, 261, 344, 141, 371, 265, 323, 123, 167, 245, 373, 284, 316, 93, 6, 302, 9, 19, 92, 24, 188, 232, 295, 209, 34, 372, 268, 100, 317, 41, 25, 332, 145, 325, 241, 306, 189, 244, 10, 321, 14, 16, 47, 107, 113, 218, 121, 259, 75, 207, 239, 85, 64, 111, 297, 291, 277, 102, 90, 221, 191, 364, 94, 329, 290, 58, 66, 233, 170, 53, 130] + 15: [130, 236, 121, 23, 342, 235, 335, 246, 117, 282, 301, 180, 365, 123, 265, 91, 369, 280, 195, 125, 302, 192, 260, 85, 11, 185, 232, 353, 174, 18, 133, 372, 2, 50, 341, 347, 308, 169, 63, 190, 370, 136, 222, 304, 348, 25, 356, 328, 0, 39, 215, 157, 173, 138, 285, 209, 166, 34, 233, 339, 381, 45, 373, 218, 303, 149, 171, 330, 307, 19, 315, 276, 15, 199, 119, 329, 344, 111, 106, 5, 102, 77, 250, 382, 61, 55, 375, 54, 73, 14, 310, 228, 287, 219, 322, 368, 88, 291, 252, 221, 113, 143, 162, 323, 279, 10, 363, 30, 53, 204, 337, 52, 161, 29, 327, 181, 346, 186, 264, 270, 359, 4, 326, 51, 83, 241, 196, 47, 238, 248, 62, 12, 141, 244, 26, 172, 278, 126, 7, 98, 103, 249, 144, 93, 160, 290, 300, 203, 151, 277, 320, 36, 145, 269, 268, 13, 313, 314, 283, 334, 3, 188, 58, 361, 17, 297, 325, 281, 227, 298, 371, 284, 360, 193, 255, 225, 182, 212, 343, 81, 294, 72, 216, 95, 273, 197, 140, 124, 66, 120, 27, 46, 57, 68, 202, 333, 90, 376, 242, 379, 351, 247, 263, 150, 8, 191, 208, 383, 116, 65, 262, 112, 292, 70, 336, 366, 362, 134, 367, 74, 146, 139, 163, 31, 207, 105, 87, 229, 206, 349, 99, 37, 316, 274, 89, 176, 299, 21, 122, 127, 251, 104, 295, 266, 20, 56, 135, 115, 100, 305, 142, 96, 377, 321, 97, 224, 167, 239, 158, 187, 44, 110, 220, 6, 324, 148, 259, 159, 9, 16, 168, 24, 170, 49, 275, 69, 43, 107, 137, 64, 84, 79, 153, 293, 154, 243, 345, 75, 155, 175, 76, 231, 237, 35, 177, 109, 319, 42, 86, 22, 165, 226, 234, 71, 213, 147, 184, 318, 178, 286, 132, 152, 355, 217, 350, 332, 80, 48, 311, 352, 338, 101, 309, 331, 253, 306, 289, 211, 257, 210, 374, 380, 179, 108, 156, 131, 200, 128, 340, 38, 317, 296, 1, 230, 78, 28, 59, 205, 254, 67, 354, 92, 358, 378, 32, 201, 183, 288, 214, 198, 258, 94, 261, 189, 60, 40, 256, 164, 357, 194, 82, 114, 271, 118, 267, 272, 240, 364, 129, 312, 41, 223, 33, 245] + 16: [134, 274, 141, 226, 306, 377, 155, 8, 42, 339, 151, 200, 272, 91, 163, 20, 258, 288, 355, 192, 179, 94, 3, 148, 15, 224, 187, 16, 365, 248, 281, 305, 45, 137, 225, 57, 253, 109, 233, 268, 26, 176, 12, 74, 325, 98, 156, 249, 286, 5, 383, 270, 294, 97, 73, 182, 36, 9, 104, 60, 267, 142, 315, 17, 131, 241, 112, 130, 376, 344, 257, 353, 120, 330, 49, 374, 287, 212, 310, 38, 66, 219, 232, 22, 221, 158, 323, 196, 81, 128, 371, 352, 6, 135, 361, 111, 275, 67, 144, 183, 121, 123, 314, 335, 58, 41, 229, 238, 252, 208, 343, 207, 14, 127, 116, 332, 23, 40, 204, 126, 100, 68, 86, 205, 265, 87, 285, 307, 119, 184, 122, 160, 290, 102, 93, 48, 186, 113, 168, 149, 79, 254, 362, 247, 140, 162, 273, 35, 189, 347, 84, 51, 139, 231, 379, 297, 264, 18, 333, 266, 88, 157, 217, 150, 164, 329, 10, 345, 369, 11, 277, 259, 209, 367, 65, 2, 103, 31, 357, 322, 354, 54, 337, 30, 62, 284, 256, 245, 349, 341, 53, 13, 152, 271, 338, 348, 283, 246, 27, 115, 80, 380, 356, 276, 198, 132, 223, 190, 193, 324, 279, 61, 33, 291, 133, 69, 234, 75, 312, 129, 302, 364, 346, 52, 82, 153, 227, 372, 317, 29, 313, 244, 201, 47, 110, 138, 101, 326, 161, 215, 262, 280, 154, 342, 255, 46, 89, 237, 99, 282, 70, 202, 370, 340, 211, 105, 360, 107, 117, 210, 169, 197, 222, 309, 92, 118, 300, 90, 78, 235, 199, 71, 316, 39, 34, 293, 106, 228, 242, 378, 363, 145, 170, 230, 382, 263, 278, 366, 7, 303, 185, 203, 24, 236, 108, 328, 37, 159, 167, 334, 124, 336, 95, 304, 381, 218, 216, 243, 28, 350, 191, 239, 43, 44, 114, 318, 298, 213, 180, 177, 175, 0, 195, 301, 188, 375, 214, 55, 240, 261, 308, 251, 32, 299, 296, 368, 165, 64, 331, 289, 171, 295, 85, 321, 21, 56, 220, 19, 96, 25, 178, 319, 172, 166, 359, 260, 63, 136, 327, 1, 147, 269, 125, 143, 250, 146, 181, 59, 292, 311, 76, 174, 4, 358, 206, 83, 194, 373, 50, 77, 351, 173, 72, 320] + 17: [131, 197, 2, 75, 290, 177, 10, 272, 376, 36, 187, 221, 179, 249, 112, 124, 344, 227, 223, 285, 209, 169, 325, 354, 119, 193, 246, 370, 104, 381, 98, 298, 332, 20, 204, 377, 300, 262, 217, 192, 57, 132, 224, 306, 252, 14, 121, 152, 284, 91, 286, 8, 56, 180, 228, 282, 82, 13, 49, 115, 27, 299, 48, 274, 53, 280, 229, 71, 35, 159, 170, 202, 236, 208, 357, 266, 324, 172, 24, 38, 292, 304, 47, 118, 365, 44, 240, 287, 67, 318, 378, 257, 77, 51, 203, 235, 79, 80, 12, 232, 297, 207, 218, 273, 165, 191, 166, 215, 89, 181, 90, 329, 270, 31, 153, 349, 247, 111, 237, 251, 337, 307, 87, 5, 234, 339, 231, 143, 99, 162, 173, 41, 341, 265, 244, 64, 369, 65, 23, 109, 289, 268, 283, 312, 375, 138, 366, 275, 125, 167, 296, 243, 139, 314, 248, 42, 196, 271, 238, 183, 106, 350, 83, 141, 277, 334, 322, 320, 261, 212, 126, 225, 351, 175, 134, 199, 211, 214, 264, 61, 269, 253, 21, 361, 55, 308, 355, 149, 356, 348, 68, 86, 220, 28, 58, 359, 113, 328, 317, 335, 92, 74, 168, 331, 336, 309, 342, 102, 161, 364, 63, 241, 4, 178, 123, 293, 250, 374, 319, 155, 130, 62, 194, 73, 84, 256, 70, 60, 216, 382, 379, 310, 267, 372, 163, 40, 176, 353, 281, 122, 188, 362, 346, 117, 1, 276, 157, 206, 380, 263, 340, 19, 147, 101, 95, 258, 128, 158, 164, 230, 201, 0, 17, 127, 303, 294, 190, 136, 323, 189, 107, 148, 367, 33, 171, 43, 363, 186, 94, 302, 151, 278, 32, 315, 184, 242, 174, 34, 108, 305, 137, 311, 114, 347, 222, 97, 313, 9, 245, 260, 7, 37, 338, 120, 210, 76, 50, 316, 291, 160, 301, 358, 200, 330, 140, 254, 343, 6, 39, 46, 371, 135, 142, 116, 81, 198, 110, 93, 345, 255, 150, 352, 368, 288, 333, 69, 15, 26, 226, 219, 185, 133, 96, 22, 103, 146, 18, 45, 154, 205, 360, 195, 16, 326, 144, 88, 30, 11, 321, 72, 383, 100, 105, 78, 279, 29, 52, 145, 373, 327, 259, 59, 295, 3, 213, 239, 85, 25, 233, 182, 66, 129, 156, 54] + 18: [271, 374, 52, 74, 80, 97, 120, 156, 183, 222, 227, 242, 268, 293, 314, 344, 382, 1, 32, 59, 78, 147, 309, 340, 30, 95, 161, 305, 252, 336, 62, 134, 194, 221, 295, 310, 323, 196, 198, 353, 379, 49, 55, 91, 128, 186, 226, 327, 251, 315, 169, 139, 246, 99, 318, 20, 83, 316, 122, 328, 176, 319, 335, 153, 358, 66, 219, 369, 44, 299, 72, 206, 208, 333, 36, 355, 306, 341, 103, 371, 23, 79, 241, 250, 152, 178, 63, 292, 214, 48, 175, 229, 2, 258, 345, 11, 47, 53, 114, 136, 237, 330, 240, 188, 342, 87, 261, 7, 50, 195, 132, 329, 166, 108, 82, 163, 180, 372, 5, 100, 10, 45, 119, 189, 376, 35, 67, 164, 366, 121, 109, 232, 361, 54, 264, 13, 94, 383, 4, 51, 317, 85, 352, 173, 43, 193, 215, 236, 34, 378, 326, 70, 202, 262, 223, 296, 102, 111, 9, 267, 254, 303, 170, 197, 312, 42, 29, 18, 69, 272, 28, 142, 373, 144, 266, 46, 41, 104, 334, 288, 231, 125, 38, 145, 148, 127, 200, 224, 75, 107, 184, 117, 320, 356, 347, 116, 26, 151, 105, 274, 171, 359, 88, 367, 39, 8, 12, 363, 92, 179, 332, 17, 375, 90, 273, 160, 217, 118, 154, 380, 360, 297, 285, 269, 337, 149, 225, 159, 58, 140, 362, 351, 112, 275, 216, 126, 381, 115, 280, 212, 257, 6, 113, 192, 321, 291, 101, 19, 191, 73, 135, 350, 302, 205, 150, 182, 243, 213, 338, 130, 76, 84, 281, 277, 300, 260, 211, 174, 146, 357, 3, 204, 248, 177, 263, 22, 61, 203, 96, 187, 324, 256, 64, 259, 218, 86, 294, 290, 245, 185, 286, 239, 25, 284, 124, 98, 141, 343, 233, 278, 157, 349, 93, 255, 325, 348, 158, 311, 298, 172, 377, 21, 65, 106, 207, 110, 68, 282, 77, 89, 346, 283, 129, 0, 235, 308, 81, 155, 270, 60, 220, 56, 123, 354, 14, 230, 57, 228, 210, 368, 301, 253, 190, 181, 365, 331, 131, 137, 289, 15, 244, 40, 287, 234, 27, 247, 133, 370, 24, 249, 138, 71, 16, 304, 167, 31, 33, 165, 339, 307, 162, 201, 313, 279, 276, 265, 322, 143, 168, 209, 364, 37, 238, 199] + 19: [24, 216, 33, 117, 46, 127, 156, 2, 144, 336, 331, 280, 130, 82, 105, 361, 290, 367, 226, 220, 81, 98, 107, 133, 187, 70, 114, 349, 354, 25, 36, 337, 16, 67, 342, 237, 148, 88, 137, 208, 103, 381, 344, 282, 327, 198, 245, 253, 190, 91, 197, 38, 275, 147, 30, 182, 368, 347, 203, 213, 356, 376, 311, 132, 314, 106, 39, 86, 360, 382, 254, 263, 366, 52, 300, 164, 153, 171, 116, 165, 49, 297, 143, 177, 293, 281, 212, 14, 286, 357, 358, 163, 17, 43, 276, 355, 7, 48, 192, 184, 230, 292, 233, 310, 96, 40, 345, 346, 60, 372, 176, 139, 234, 112, 11, 21, 266, 274, 268, 319, 90, 339, 188, 193, 296, 9, 335, 6, 251, 257, 306, 284, 80, 3, 146, 87, 332, 150, 124, 51, 0, 55, 316, 369, 288, 243, 255, 373, 303, 189, 238, 134, 121, 272, 374, 85, 380, 140, 200, 186, 340, 231, 210, 100, 5, 78, 378, 375, 256, 141, 241, 8, 154, 271, 217, 246, 228, 353, 138, 31, 333, 23, 26, 326, 232, 174, 59, 267, 136, 56, 104, 93, 265, 115, 61, 183, 113, 301, 158, 13, 295, 68, 283, 110, 18, 131, 62, 329, 258, 235, 73, 269, 155, 236, 142, 109, 57, 307, 302, 318, 363, 97, 35, 53, 352, 172, 239, 249, 19, 108, 65, 362, 370, 247, 294, 289, 205, 248, 145, 229, 377, 64, 364, 279, 328, 89, 41, 15, 222, 270, 157, 4, 273, 160, 225, 341, 209, 28, 324, 123, 214, 261, 175, 305, 120, 240, 74, 219, 66, 159, 365, 285, 166, 173, 204, 215, 287, 75, 47, 119, 168, 63, 351, 162, 223, 308, 196, 317, 338, 101, 206, 126, 323, 152, 264, 122, 128, 167, 221, 259, 84, 202, 129, 321, 320, 94, 371, 178, 224, 379, 250, 350, 330, 312, 179, 322, 27, 77, 58, 54, 1, 83, 260, 277, 325, 262, 191, 383, 343, 149, 37, 118, 299, 102, 69, 12, 313, 207, 111, 291, 20, 334, 79, 71, 278, 76, 170, 244, 125, 45, 309, 242, 22, 181, 348, 185, 252, 151, 72, 34, 180, 42, 211, 161, 218, 227, 44, 135, 359, 29, 195, 95, 194, 10, 92, 298, 32, 99, 315, 201, 304, 169, 199, 50] + 20: [161, 309, 238, 20, 365, 253, 80, 142, 205, 72, 68, 103, 69, 27, 51, 52, 54, 61, 76, 107, 137, 167, 168, 179, 116, 111, 9, 49, 126, 194, 118, 247, 196, 248, 106, 312, 94, 11, 88, 341, 48, 70, 129, 109, 181, 189, 200, 206, 334, 339, 270, 324, 96, 292, 104, 364, 41, 213, 34, 169, 252, 326, 159, 151, 124, 354, 64, 147, 186, 143, 214, 262, 98, 14, 307, 318, 228, 187, 132, 315, 284, 175, 233, 114, 246, 33, 202, 230, 156, 321, 47, 207, 149, 222, 171, 263, 153, 272, 379, 188, 91, 362, 258, 182, 240, 219, 367, 131, 294, 78, 231, 351, 352, 28, 323, 313, 192, 275, 197, 266, 278, 174, 226, 185, 330, 229, 160, 358, 357, 348, 30, 150, 243, 338, 39, 24, 139, 199, 271, 136, 108, 301, 269, 281, 302, 37, 2, 105, 325, 95, 180, 172, 13, 35, 25, 22, 274, 232, 127, 327, 254, 296, 356, 157, 261, 306, 288, 299, 336, 122, 112, 353, 209, 273, 135, 121, 190, 378, 58, 208, 340, 310, 73, 89, 210, 289, 355, 211, 183, 344, 17, 241, 82, 8, 123, 32, 162, 290, 286, 77, 322, 343, 36, 256, 166, 5, 380, 191, 285, 79, 7, 225, 245, 31, 244, 303, 144, 282, 376, 204, 93, 311, 264, 65, 239, 375, 265, 372, 97, 297, 3, 212, 349, 115, 85, 305, 295, 152, 255, 331, 66, 46, 287, 62, 138, 125, 140, 320, 57, 235, 146, 119, 346, 12, 236, 360, 308, 319, 102, 83, 293, 155, 298, 350, 193, 335, 110, 291, 164, 382, 227, 184, 177, 267, 368, 178, 86, 370, 201, 337, 366, 250, 329, 276, 373, 203, 1, 242, 99, 4, 173, 195, 158, 141, 216, 92, 6, 361, 0, 18, 60, 100, 332, 374, 154, 283, 333, 16, 133, 220, 117, 249, 87, 145, 81, 363, 148, 128, 259, 101, 176, 328, 314, 279, 165, 377, 257, 75, 217, 21, 40, 19, 134, 251, 198, 260, 43, 268, 317, 130, 10, 221, 342, 71, 90, 218, 359, 300, 345, 29, 170, 84, 316, 120, 113, 63, 234, 23, 38, 277, 369, 45, 55, 224, 371, 381, 50, 347, 74, 26, 53, 42, 215, 15, 304, 223, 56, 59, 163, 67, 383, 237, 44, 280] + 21: [360, 92, 277, 91, 22, 366, 141, 234, 197, 64, 139, 347, 275, 160, 186, 333, 105, 143, 217, 265, 298, 297, 42, 17, 152, 82, 348, 118, 380, 25, 313, 89, 174, 223, 318, 378, 353, 337, 258, 273, 342, 156, 357, 285, 326, 311, 44, 35, 355, 46, 87, 345, 279, 349, 280, 288, 84, 309, 98, 125, 3, 14, 346, 379, 241, 253, 180, 54, 7, 231, 321, 339, 47, 236, 185, 45, 51, 128, 225, 220, 325, 331, 304, 159, 133, 301, 170, 48, 276, 163, 201, 55, 162, 32, 282, 350, 173, 256, 112, 120, 177, 365, 293, 221, 270, 49, 352, 312, 5, 30, 104, 126, 244, 181, 207, 85, 176, 59, 95, 58, 281, 190, 218, 214, 240, 147, 227, 367, 233, 267, 262, 97, 155, 154, 79, 203, 27, 110, 239, 99, 249, 61, 235, 93, 132, 153, 200, 232, 182, 151, 15, 9, 72, 372, 39, 383, 211, 40, 193, 216, 300, 315, 268, 114, 255, 116, 248, 123, 103, 289, 38, 361, 252, 184, 8, 111, 373, 73, 287, 286, 192, 266, 137, 274, 305, 195, 330, 375, 296, 166, 322, 264, 316, 302, 29, 88, 250, 351, 175, 144, 368, 63, 362, 303, 28, 237, 102, 329, 83, 334, 13, 130, 6, 31, 343, 306, 20, 140, 86, 169, 284, 191, 194, 230, 183, 229, 259, 260, 165, 295, 198, 269, 271, 319, 41, 374, 106, 142, 101, 108, 371, 113, 205, 199, 150, 26, 121, 52, 23, 1, 358, 21, 317, 100, 148, 340, 134, 381, 74, 2, 328, 188, 107, 179, 308, 257, 210, 117, 178, 307, 219, 209, 243, 131, 359, 66, 354, 115, 341, 57, 81, 382, 122, 68, 202, 145, 224, 369, 254, 71, 50, 363, 0, 16, 299, 43, 245, 338, 90, 251, 290, 65, 187, 261, 36, 12, 164, 37, 314, 212, 272, 10, 4, 119, 53, 94, 149, 172, 196, 135, 78, 263, 60, 246, 323, 77, 292, 335, 215, 67, 294, 56, 324, 222, 127, 327, 158, 376, 247, 364, 377, 320, 109, 344, 189, 18, 336, 356, 291, 171, 136, 24, 213, 310, 70, 76, 228, 161, 167, 332, 80, 124, 206, 208, 226, 238, 19, 69, 33, 168, 138, 62, 370, 204, 34, 146, 157, 96, 75, 129, 242, 278, 283, 11] + 22: [254, 58, 351, 375, 372, 122, 75, 321, 99, 341, 335, 241, 96, 262, 334, 342, 302, 43, 158, 231, 2, 28, 79, 95, 141, 38, 139, 55, 218, 312, 170, 228, 15, 165, 159, 256, 211, 136, 279, 52, 62, 344, 197, 163, 234, 101, 143, 154, 181, 332, 184, 93, 348, 7, 264, 259, 340, 49, 72, 301, 269, 268, 273, 103, 219, 64, 128, 162, 121, 297, 157, 217, 221, 126, 73, 329, 50, 338, 123, 226, 175, 324, 105, 303, 124, 45, 275, 108, 266, 291, 212, 23, 245, 313, 247, 250, 260, 9, 201, 244, 352, 111, 378, 27, 3, 25, 1, 305, 327, 286, 353, 232, 186, 350, 255, 326, 270, 315, 253, 267, 261, 204, 115, 169, 130, 345, 195, 60, 361, 191, 59, 39, 22, 373, 271, 176, 160, 36, 104, 363, 66, 167, 192, 307, 328, 33, 42, 263, 317, 149, 67, 125, 14, 48, 71, 320, 322, 32, 97, 26, 119, 30, 19, 354, 304, 243, 205, 310, 281, 316, 207, 112, 362, 153, 80, 83, 349, 113, 131, 133, 236, 189, 299, 356, 246, 137, 129, 296, 5, 277, 325, 346, 216, 371, 132, 74, 117, 272, 306, 8, 369, 82, 220, 239, 87, 377, 290, 364, 274, 151, 145, 308, 127, 295, 10, 249, 343, 94, 147, 4, 61, 357, 173, 76, 206, 358, 309, 41, 47, 78, 382, 46, 237, 92, 152, 31, 370, 166, 135, 280, 278, 110, 276, 177, 355, 70, 100, 161, 193, 339, 40, 367, 318, 11, 300, 379, 265, 120, 179, 150, 6, 293, 142, 287, 209, 376, 330, 37, 383, 381, 336, 68, 116, 156, 18, 210, 155, 54, 0, 314, 102, 196, 190, 223, 88, 107, 368, 359, 148, 178, 331, 144, 29, 240, 251, 365, 214, 53, 258, 380, 98, 283, 13, 202, 85, 182, 91, 168, 213, 84, 56, 289, 24, 20, 34, 230, 235, 229, 285, 222, 17, 366, 63, 12, 65, 227, 138, 215, 35, 187, 16, 69, 140, 114, 86, 292, 224, 183, 51, 164, 118, 333, 311, 174, 57, 347, 233, 225, 200, 199, 81, 77, 298, 294, 134, 90, 284, 172, 194, 323, 337, 288, 198, 242, 44, 252, 188, 106, 185, 109, 374, 319, 248, 238, 180, 89, 208, 282, 257, 171, 21, 146, 203, 360] + 23: [26, 374, 6, 309, 70, 152, 129, 119, 125, 341, 58, 28, 121, 25, 214, 9, 350, 22, 63, 186, 4, 320, 168, 288, 314, 197, 154, 218, 371, 184, 312, 194, 210, 66, 333, 303, 280, 108, 297, 355, 140, 88, 247, 199, 217, 189, 37, 265, 124, 77, 324, 180, 159, 340, 74, 317, 290, 157, 93, 137, 352, 200, 16, 285, 373, 282, 230, 334, 145, 262, 8, 286, 172, 211, 306, 232, 372, 130, 146, 151, 342, 370, 160, 178, 284, 256, 206, 236, 375, 310, 276, 259, 246, 296, 216, 81, 275, 376, 87, 176, 55, 92, 326, 229, 38, 359, 273, 347, 76, 109, 21, 349, 104, 188, 266, 171, 117, 13, 251, 64, 71, 69, 173, 47, 127, 257, 144, 149, 228, 44, 329, 207, 193, 339, 68, 362, 267, 367, 322, 181, 278, 18, 29, 242, 305, 51, 82, 287, 11, 315, 35, 220, 195, 311, 185, 5, 382, 73, 164, 14, 123, 205, 319, 383, 141, 83, 293, 43, 126, 225, 201, 358, 241, 86, 133, 249, 23, 292, 153, 52, 316, 90, 245, 122, 239, 248, 353, 165, 118, 105, 112, 75, 328, 281, 331, 103, 304, 148, 131, 102, 313, 215, 150, 53, 223, 143, 302, 155, 338, 308, 357, 238, 138, 183, 142, 15, 264, 177, 295, 327, 192, 270, 209, 378, 10, 255, 298, 65, 330, 258, 174, 354, 115, 79, 253, 377, 20, 289, 226, 110, 175, 135, 48, 279, 67, 351, 190, 219, 95, 221, 1, 231, 162, 54, 107, 163, 97, 260, 158, 337, 360, 345, 366, 113, 100, 300, 19, 12, 84, 182, 283, 50, 191, 40, 101, 365, 179, 250, 335, 346, 336, 106, 161, 277, 381, 30, 198, 291, 132, 72, 170, 203, 261, 59, 78, 111, 233, 136, 32, 96, 307, 204, 344, 134, 99, 139, 46, 368, 364, 42, 380, 39, 187, 85, 147, 240, 94, 169, 263, 80, 91, 224, 356, 98, 321, 196, 0, 212, 294, 227, 61, 62, 343, 208, 60, 234, 167, 252, 120, 269, 156, 57, 41, 379, 271, 237, 222, 17, 301, 348, 128, 7, 243, 369, 202, 34, 299, 318, 166, 3, 244, 323, 274, 24, 27, 2, 56, 213, 36, 272, 361, 254, 325, 45, 363, 114, 268, 33, 49, 31, 116, 89, 332, 235] + 24: [154, 5, 62, 135, 91, 142, 73, 310, 63, 295, 280, 108, 333, 175, 151, 171, 193, 236, 141, 243, 364, 81, 115, 166, 265, 112, 186, 354, 144, 138, 49, 114, 297, 197, 51, 181, 72, 10, 352, 262, 288, 256, 90, 199, 17, 374, 379, 383, 252, 28, 255, 159, 54, 350, 130, 206, 140, 168, 312, 218, 365, 212, 161, 329, 291, 367, 119, 200, 40, 21, 198, 241, 9, 259, 324, 231, 202, 273, 274, 380, 245, 147, 269, 239, 134, 276, 253, 99, 57, 248, 43, 143, 349, 82, 127, 264, 246, 328, 286, 375, 192, 15, 355, 177, 307, 169, 313, 244, 215, 275, 196, 136, 331, 25, 80, 228, 371, 100, 129, 298, 20, 95, 361, 230, 31, 306, 356, 113, 272, 321, 13, 66, 103, 282, 296, 281, 148, 183, 125, 240, 33, 214, 133, 314, 7, 110, 381, 301, 26, 366, 254, 224, 345, 216, 153, 339, 258, 170, 363, 373, 67, 109, 180, 165, 227, 152, 158, 353, 223, 211, 360, 334, 343, 344, 318, 377, 182, 58, 235, 304, 242, 65, 23, 303, 8, 42, 101, 201, 250, 69, 261, 179, 0, 87, 157, 220, 137, 195, 93, 46, 149, 287, 376, 16, 268, 309, 203, 173, 188, 50, 209, 229, 300, 117, 305, 232, 52, 97, 139, 294, 283, 204, 61, 131, 338, 12, 83, 222, 315, 270, 4, 14, 284, 102, 217, 187, 98, 285, 219, 251, 92, 290, 185, 104, 319, 53, 299, 226, 96, 205, 221, 323, 302, 346, 267, 86, 32, 191, 132, 362, 167, 289, 358, 327, 6, 128, 122, 266, 370, 24, 382, 156, 336, 77, 59, 278, 56, 194, 348, 88, 94, 19, 342, 247, 162, 316, 2, 34, 22, 189, 337, 64, 29, 164, 190, 308, 293, 145, 178, 84, 292, 37, 335, 107, 76, 160, 44, 18, 332, 208, 322, 368, 372, 85, 210, 121, 237, 213, 325, 41, 39, 48, 279, 233, 1, 225, 79, 176, 126, 163, 238, 68, 75, 340, 45, 36, 263, 369, 172, 27, 146, 277, 35, 260, 351, 3, 326, 311, 11, 30, 38, 120, 249, 359, 317, 184, 124, 347, 89, 155, 357, 70, 111, 105, 207, 78, 71, 106, 341, 118, 257, 116, 47, 60, 271, 150, 320, 55, 234, 378, 174, 123, 330, 74] + 25: [315, 331, 310, 159, 166, 117, 70, 371, 224, 363, 219, 151, 338, 19, 165, 239, 220, 267, 277, 123, 178, 91, 158, 326, 269, 253, 138, 196, 347, 187, 259, 257, 200, 251, 337, 167, 45, 41, 265, 169, 206, 195, 43, 382, 129, 233, 156, 263, 379, 216, 381, 373, 153, 321, 114, 278, 345, 202, 94, 79, 243, 111, 157, 208, 64, 264, 42, 266, 270, 303, 121, 189, 282, 246, 280, 314, 120, 234, 99, 126, 261, 203, 312, 308, 181, 31, 340, 292, 93, 311, 118, 286, 88, 288, 255, 44, 72, 78, 113, 190, 296, 368, 197, 256, 146, 307, 304, 81, 185, 201, 150, 361, 22, 281, 16, 128, 57, 335, 106, 170, 139, 334, 2, 171, 273, 318, 154, 84, 244, 161, 291, 237, 204, 328, 144, 300, 127, 35, 320, 175, 155, 3, 301, 194, 228, 298, 39, 10, 305, 230, 302, 377, 89, 97, 242, 295, 354, 365, 149, 53, 275, 75, 122, 40, 137, 48, 108, 47, 247, 268, 61, 112, 7, 59, 136, 341, 323, 218, 20, 319, 324, 134, 69, 238, 110, 177, 21, 46, 142, 95, 104, 148, 317, 168, 332, 77, 160, 380, 262, 346, 229, 367, 140, 191, 383, 213, 28, 212, 27, 186, 86, 210, 188, 63, 217, 50, 180, 258, 162, 376, 235, 32, 18, 306, 240, 60, 103, 115, 199, 290, 116, 370, 358, 353, 183, 96, 92, 205, 285, 182, 276, 83, 375, 152, 5, 102, 207, 222, 209, 164, 135, 132, 221, 15, 254, 24, 17, 271, 250, 226, 174, 344, 90, 272, 23, 65, 30, 25, 145, 33, 351, 294, 76, 54, 359, 336, 316, 297, 58, 67, 74, 109, 34, 364, 322, 1, 374, 13, 372, 98, 101, 327, 73, 313, 100, 131, 231, 279, 176, 352, 350, 37, 339, 105, 193, 248, 362, 87, 236, 147, 26, 241, 107, 62, 82, 0, 232, 8, 287, 215, 125, 366, 143, 211, 55, 330, 274, 356, 223, 51, 325, 293, 289, 360, 71, 329, 214, 192, 299, 309, 333, 38, 378, 9, 80, 227, 348, 369, 68, 357, 14, 355, 179, 260, 124, 283, 349, 119, 12, 198, 29, 284, 85, 52, 49, 66, 342, 252, 56, 343, 4, 225, 130, 184, 133, 11, 249, 163, 36, 173, 245, 141, 172, 6] + 26: [136, 204, 191, 174, 66, 318, 41, 129, 189, 368, 165, 335, 326, 258, 377, 95, 248, 54, 68, 86, 96, 106, 111, 139, 237, 224, 130, 135, 167, 359, 319, 306, 24, 57, 357, 115, 75, 280, 7, 140, 297, 39, 91, 282, 257, 132, 169, 170, 89, 342, 69, 266, 30, 332, 176, 166, 152, 194, 215, 267, 123, 232, 121, 255, 59, 77, 270, 42, 381, 168, 175, 209, 269, 271, 16, 38, 22, 200, 128, 133, 79, 118, 239, 107, 187, 301, 157, 105, 40, 154, 119, 283, 23, 379, 374, 375, 217, 349, 254, 137, 226, 382, 156, 26, 367, 173, 71, 161, 35, 196, 182, 300, 354, 74, 112, 199, 295, 366, 207, 218, 172, 281, 160, 183, 223, 126, 180, 178, 44, 241, 365, 113, 212, 231, 372, 220, 225, 72, 163, 205, 330, 84, 286, 228, 90, 233, 149, 316, 61, 146, 247, 360, 2, 186, 192, 277, 0, 50, 312, 197, 317, 334, 193, 274, 85, 222, 203, 234, 80, 5, 162, 33, 355, 92, 265, 179, 229, 122, 285, 10, 73, 256, 383, 49, 125, 347, 206, 314, 211, 345, 213, 242, 6, 320, 338, 344, 227, 245, 62, 48, 190, 327, 51, 104, 76, 93, 298, 219, 325, 8, 100, 101, 284, 310, 322, 246, 292, 150, 304, 276, 238, 376, 55, 148, 353, 103, 201, 195, 378, 114, 363, 78, 340, 43, 244, 216, 4, 28, 210, 261, 380, 230, 263, 279, 293, 36, 110, 198, 262, 97, 343, 159, 290, 350, 19, 264, 352, 64, 333, 296, 14, 46, 273, 305, 11, 339, 70, 337, 369, 108, 328, 272, 151, 356, 127, 65, 45, 370, 117, 329, 362, 346, 29, 13, 145, 147, 288, 348, 124, 253, 184, 94, 307, 88, 131, 259, 321, 102, 221, 81, 98, 32, 143, 3, 141, 9, 303, 324, 208, 164, 309, 351, 18, 291, 63, 116, 278, 158, 82, 138, 323, 99, 20, 15, 171, 144, 25, 302, 364, 34, 331, 358, 243, 188, 52, 260, 53, 181, 214, 120, 308, 249, 83, 311, 287, 27, 294, 87, 202, 37, 153, 12, 185, 275, 361, 373, 250, 235, 60, 299, 236, 21, 371, 289, 341, 315, 67, 31, 142, 251, 268, 17, 252, 177, 134, 336, 47, 109, 240, 56, 313, 58, 155, 1] + 27: [103, 290, 343, 322, 109, 261, 114, 296, 147, 9, 84, 139, 280, 302, 158, 345, 295, 92, 270, 240, 244, 262, 310, 338, 225, 58, 357, 188, 112, 323, 72, 82, 187, 370, 6, 274, 248, 121, 365, 35, 113, 186, 195, 192, 11, 169, 285, 342, 286, 117, 26, 181, 55, 21, 238, 89, 98, 155, 172, 327, 20, 318, 3, 105, 152, 74, 257, 64, 138, 363, 39, 53, 12, 145, 204, 24, 97, 242, 99, 78, 241, 88, 175, 127, 111, 66, 13, 130, 184, 264, 273, 166, 19, 371, 60, 137, 312, 104, 164, 366, 143, 272, 383, 308, 141, 15, 271, 5, 362, 249, 167, 79, 259, 293, 62, 283, 255, 57, 100, 146, 90, 301, 233, 358, 207, 48, 149, 83, 373, 161, 54, 215, 276, 44, 275, 306, 231, 253, 86, 334, 210, 298, 163, 176, 234, 81, 300, 376, 222, 230, 266, 115, 140, 42, 309, 174, 46, 2, 321, 153, 144, 359, 352, 94, 189, 221, 355, 220, 37, 354, 374, 128, 224, 251, 324, 156, 284, 133, 126, 304, 267, 171, 47, 361, 132, 319, 41, 372, 194, 148, 134, 180, 177, 34, 350, 49, 179, 75, 76, 294, 25, 27, 316, 136, 45, 299, 250, 123, 344, 346, 150, 16, 87, 160, 206, 200, 4, 211, 31, 119, 348, 228, 380, 258, 297, 356, 7, 291, 268, 43, 120, 107, 33, 281, 157, 142, 108, 170, 209, 203, 263, 341, 70, 315, 131, 336, 303, 77, 278, 317, 368, 326, 375, 106, 199, 205, 159, 185, 223, 382, 165, 311, 243, 212, 61, 227, 279, 320, 213, 183, 85, 232, 32, 191, 162, 379, 360, 239, 28, 367, 14, 96, 229, 68, 198, 277, 40, 235, 265, 80, 59, 269, 202, 237, 340, 216, 325, 337, 328, 246, 122, 18, 307, 217, 116, 154, 8, 330, 226, 314, 378, 236, 168, 349, 219, 135, 292, 190, 260, 196, 23, 193, 91, 329, 124, 67, 29, 95, 288, 208, 22, 110, 1, 331, 51, 247, 73, 30, 333, 71, 369, 335, 381, 339, 332, 305, 254, 214, 218, 347, 38, 0, 256, 282, 50, 289, 52, 56, 17, 63, 252, 69, 351, 197, 377, 182, 178, 102, 201, 118, 125, 10, 287, 129, 101, 353, 151, 313, 65, 364, 173, 245, 36, 93] + 28: [161, 289, 182, 247, 178, 256, 249, 381, 12, 297, 79, 351, 72, 76, 141, 313, 9, 138, 18, 60, 86, 115, 150, 156, 250, 253, 346, 59, 261, 285, 32, 216, 335, 222, 264, 274, 175, 170, 73, 266, 234, 232, 373, 53, 318, 336, 338, 343, 26, 248, 97, 108, 325, 228, 157, 125, 349, 316, 14, 265, 69, 148, 342, 372, 96, 251, 29, 101, 202, 11, 147, 180, 185, 0, 226, 307, 236, 71, 129, 337, 56, 43, 282, 327, 44, 362, 332, 151, 287, 17, 85, 45, 360, 40, 173, 207, 168, 312, 152, 113, 128, 227, 5, 90, 220, 273, 380, 194, 46, 82, 143, 211, 154, 323, 292, 55, 174, 350, 54, 190, 344, 355, 203, 214, 50, 42, 158, 361, 370, 193, 315, 290, 208, 81, 21, 363, 31, 218, 293, 252, 267, 65, 319, 353, 92, 4, 188, 176, 37, 320, 165, 130, 41, 367, 268, 68, 260, 98, 322, 334, 294, 348, 303, 281, 275, 189, 87, 215, 356, 321, 22, 93, 357, 3, 100, 167, 105, 13, 341, 184, 112, 64, 36, 210, 136, 149, 8, 146, 94, 83, 124, 257, 153, 197, 283, 155, 369, 219, 352, 67, 299, 19, 48, 121, 49, 57, 61, 270, 382, 119, 38, 229, 102, 145, 164, 279, 70, 241, 328, 120, 224, 223, 204, 235, 209, 217, 300, 1, 142, 305, 296, 15, 34, 80, 131, 276, 126, 181, 198, 311, 317, 286, 324, 103, 7, 368, 51, 88, 271, 20, 278, 118, 179, 295, 74, 104, 377, 330, 225, 237, 277, 187, 262, 329, 255, 99, 196, 122, 374, 137, 58, 160, 326, 376, 206, 333, 171, 358, 177, 133, 144, 364, 284, 347, 291, 309, 62, 298, 306, 25, 77, 288, 272, 95, 200, 135, 242, 238, 127, 27, 183, 375, 169, 230, 109, 117, 39, 47, 123, 254, 186, 359, 302, 52, 201, 195, 75, 166, 63, 28, 240, 308, 16, 78, 6, 23, 205, 280, 331, 89, 110, 192, 140, 354, 212, 379, 162, 107, 239, 172, 263, 378, 383, 163, 221, 340, 134, 91, 365, 159, 106, 84, 366, 345, 246, 139, 35, 304, 301, 10, 243, 30, 310, 233, 199, 191, 116, 339, 269, 33, 114, 244, 245, 314, 258, 66, 24, 213, 231, 259, 111, 371, 2, 132] + 29: [118, 124, 262, 326, 188, 123, 50, 139, 131, 78, 3, 242, 348, 114, 327, 62, 270, 290, 301, 100, 107, 241, 66, 91, 244, 283, 186, 112, 332, 38, 350, 232, 329, 104, 292, 31, 36, 368, 56, 69, 307, 30, 92, 330, 165, 281, 152, 176, 361, 383, 333, 160, 275, 207, 4, 273, 136, 345, 125, 344, 28, 135, 250, 13, 356, 246, 109, 80, 231, 206, 355, 285, 197, 89, 354, 16, 159, 370, 86, 378, 200, 59, 267, 269, 77, 46, 191, 338, 199, 113, 251, 41, 279, 268, 247, 108, 73, 255, 258, 148, 48, 216, 193, 298, 156, 272, 353, 142, 224, 315, 103, 42, 15, 162, 35, 223, 331, 342, 98, 249, 196, 11, 120, 351, 57, 237, 382, 61, 178, 349, 380, 32, 47, 52, 79, 84, 121, 168, 211, 358, 2, 335, 256, 6, 336, 127, 379, 208, 266, 163, 68, 324, 128, 347, 181, 313, 217, 369, 209, 9, 187, 376, 252, 133, 201, 64, 289, 26, 318, 311, 76, 320, 180, 190, 102, 132, 257, 18, 97, 317, 22, 93, 381, 158, 179, 149, 337, 189, 254, 1, 274, 177, 51, 23, 25, 202, 346, 63, 319, 248, 215, 300, 150, 328, 83, 157, 111, 7, 192, 219, 45, 164, 245, 372, 284, 185, 137, 173, 54, 110, 243, 87, 341, 360, 116, 225, 264, 340, 229, 126, 213, 288, 233, 306, 115, 234, 5, 260, 367, 322, 277, 282, 365, 74, 65, 343, 210, 94, 182, 280, 151, 205, 359, 302, 377, 161, 261, 194, 141, 325, 27, 291, 37, 212, 323, 44, 195, 129, 366, 175, 204, 314, 218, 146, 228, 371, 169, 312, 287, 70, 239, 308, 184, 49, 8, 221, 305, 240, 198, 236, 88, 339, 140, 134, 145, 259, 85, 72, 101, 20, 316, 40, 53, 153, 253, 17, 154, 96, 21, 321, 117, 363, 203, 60, 271, 309, 174, 375, 82, 172, 106, 222, 310, 105, 373, 10, 147, 166, 374, 334, 263, 155, 95, 81, 138, 29, 295, 67, 14, 33, 119, 235, 293, 278, 12, 171, 276, 167, 238, 230, 55, 220, 227, 0, 71, 296, 75, 226, 214, 39, 304, 362, 58, 265, 183, 24, 357, 364, 34, 19, 90, 294, 122, 170, 303, 299, 352, 43, 99, 297, 130, 286, 143, 144] + 30: [208, 160, 91, 142, 280, 73, 145, 342, 304, 287, 291, 255, 149, 229, 8, 21, 22, 31, 36, 40, 47, 88, 94, 101, 239, 274, 60, 167, 20, 111, 284, 318, 371, 381, 370, 173, 352, 328, 34, 131, 118, 123, 128, 134, 146, 185, 189, 218, 140, 59, 321, 283, 159, 336, 193, 344, 258, 329, 163, 109, 248, 166, 267, 57, 327, 178, 126, 235, 87, 176, 219, 237, 266, 242, 361, 147, 362, 138, 256, 320, 369, 96, 179, 310, 244, 288, 230, 32, 69, 368, 198, 265, 119, 201, 259, 268, 65, 76, 78, 234, 164, 340, 77, 58, 195, 67, 132, 335, 129, 332, 175, 377, 264, 85, 3, 16, 42, 191, 227, 269, 245, 333, 196, 188, 33, 97, 12, 71, 217, 281, 278, 114, 197, 115, 290, 202, 324, 323, 314, 300, 313, 251, 238, 322, 37, 213, 135, 11, 68, 216, 317, 275, 204, 356, 104, 383, 172, 55, 79, 276, 279, 125, 81, 190, 63, 252, 301, 334, 262, 184, 75, 206, 41, 303, 221, 192, 136, 285, 325, 103, 294, 48, 374, 95, 139, 155, 122, 308, 171, 4, 30, 305, 187, 124, 243, 2, 10, 17, 312, 152, 236, 260, 157, 84, 14, 66, 92, 110, 121, 116, 130, 105, 112, 56, 367, 380, 358, 254, 366, 295, 346, 250, 89, 177, 141, 226, 200, 353, 359, 286, 44, 360, 247, 90, 82, 355, 207, 199, 148, 345, 53, 158, 13, 168, 45, 316, 341, 357, 9, 296, 23, 261, 51, 293, 50, 25, 220, 161, 273, 15, 232, 372, 27, 70, 214, 133, 222, 46, 28, 271, 74, 100, 210, 382, 343, 108, 319, 349, 282, 154, 183, 83, 5, 169, 102, 270, 26, 307, 52, 289, 272, 378, 64, 212, 240, 253, 151, 277, 174, 38, 376, 375, 297, 311, 107, 326, 117, 127, 330, 39, 61, 331, 35, 62, 29, 257, 223, 302, 49, 6, 215, 18, 98, 54, 150, 241, 7, 194, 263, 348, 364, 153, 113, 0, 93, 354, 143, 156, 137, 24, 246, 315, 43, 1, 299, 292, 365, 72, 306, 211, 181, 337, 350, 203, 165, 233, 106, 338, 182, 363, 351, 373, 86, 225, 144, 205, 224, 339, 309, 170, 298, 231, 162, 180, 249, 186, 347, 99, 209, 19, 120, 80, 228, 379] + 31: [0, 186, 208, 258, 249, 40, 277, 202, 288, 8, 153, 305, 352, 226, 210, 327, 280, 35, 18, 104, 343, 132, 207, 329, 159, 364, 219, 119, 360, 112, 254, 55, 318, 307, 383, 12, 362, 228, 50, 101, 337, 181, 274, 319, 117, 67, 172, 350, 128, 292, 339, 140, 75, 214, 290, 353, 169, 46, 41, 142, 51, 108, 91, 238, 259, 342, 235, 282, 357, 154, 203, 229, 233, 265, 133, 150, 155, 286, 223, 138, 57, 205, 373, 336, 273, 11, 43, 308, 42, 64, 356, 114, 167, 200, 341, 347, 26, 331, 116, 268, 206, 72, 269, 244, 98, 65, 287, 303, 73, 177, 83, 375, 240, 113, 381, 184, 242, 13, 165, 120, 201, 125, 121, 270, 22, 283, 102, 189, 111, 243, 222, 256, 92, 195, 323, 275, 47, 309, 135, 109, 313, 27, 297, 180, 68, 320, 379, 267, 209, 34, 164, 236, 74, 368, 176, 374, 141, 183, 237, 220, 263, 367, 99, 52, 255, 30, 377, 246, 100, 248, 279, 95, 349, 188, 299, 211, 63, 2, 322, 213, 123, 110, 376, 48, 224, 157, 315, 79, 382, 311, 199, 37, 166, 227, 241, 25, 96, 61, 56, 78, 23, 245, 15, 284, 16, 285, 6, 371, 80, 139, 321, 127, 295, 332, 230, 90, 324, 76, 192, 215, 39, 170, 338, 82, 231, 366, 105, 312, 217, 354, 14, 130, 281, 178, 340, 193, 44, 359, 358, 212, 158, 87, 19, 250, 146, 60, 348, 106, 93, 81, 122, 118, 84, 29, 272, 1, 107, 251, 9, 7, 344, 161, 86, 289, 252, 185, 36, 326, 325, 58, 334, 145, 38, 66, 77, 97, 369, 361, 59, 194, 198, 129, 24, 53, 372, 218, 163, 294, 143, 261, 221, 187, 330, 378, 131, 33, 136, 175, 137, 314, 302, 196, 271, 264, 304, 160, 168, 293, 126, 174, 5, 89, 115, 70, 32, 147, 328, 171, 355, 28, 232, 134, 182, 21, 45, 260, 247, 204, 103, 298, 262, 156, 197, 257, 94, 149, 239, 363, 3, 335, 173, 333, 276, 296, 152, 88, 31, 316, 291, 49, 266, 346, 306, 345, 234, 191, 124, 179, 10, 148, 278, 144, 365, 69, 317, 253, 62, 380, 17, 54, 190, 225, 310, 4, 20, 351, 162, 85, 300, 216, 370, 301, 151, 71] + 32: [63, 256, 371, 6, 56, 339, 43, 128, 119, 170, 76, 98, 328, 162, 167, 169, 177, 181, 244, 245, 251, 262, 265, 296, 367, 65, 126, 273, 301, 325, 175, 150, 351, 84, 366, 154, 282, 195, 365, 0, 52, 180, 217, 15, 326, 173, 312, 373, 352, 229, 2, 380, 140, 335, 345, 303, 50, 178, 286, 3, 381, 172, 361, 197, 13, 330, 220, 364, 358, 295, 383, 375, 94, 135, 271, 219, 89, 97, 232, 168, 85, 206, 250, 336, 70, 309, 236, 221, 125, 248, 35, 278, 23, 17, 290, 379, 218, 234, 310, 274, 348, 151, 109, 7, 36, 101, 45, 32, 80, 10, 317, 9, 191, 222, 67, 79, 216, 47, 338, 57, 148, 246, 179, 257, 319, 279, 308, 189, 227, 25, 239, 127, 302, 22, 372, 99, 299, 223, 196, 138, 259, 112, 93, 300, 33, 155, 323, 12, 186, 211, 228, 313, 55, 26, 48, 183, 115, 340, 363, 42, 287, 341, 160, 343, 46, 74, 194, 316, 92, 288, 146, 8, 141, 283, 233, 306, 72, 231, 324, 382, 81, 4, 77, 82, 103, 304, 355, 149, 260, 192, 11, 333, 285, 275, 362, 121, 16, 202, 131, 113, 266, 104, 281, 182, 350, 24, 37, 311, 102, 356, 261, 334, 1, 368, 27, 54, 347, 322, 187, 66, 359, 96, 156, 294, 254, 69, 314, 59, 240, 337, 349, 136, 243, 117, 201, 208, 277, 88, 226, 90, 190, 329, 106, 280, 342, 184, 143, 289, 212, 354, 157, 86, 249, 129, 62, 5, 132, 20, 374, 224, 31, 95, 235, 114, 241, 166, 209, 360, 204, 225, 78, 19, 369, 71, 64, 237, 39, 205, 193, 210, 152, 238, 305, 188, 60, 18, 159, 14, 163, 344, 108, 130, 214, 174, 87, 120, 297, 199, 332, 230, 353, 276, 139, 320, 124, 376, 44, 203, 321, 272, 267, 145, 51, 107, 49, 21, 41, 83, 331, 357, 91, 111, 213, 293, 29, 247, 198, 110, 284, 73, 264, 118, 100, 161, 318, 123, 122, 164, 346, 370, 61, 263, 116, 252, 171, 185, 269, 142, 53, 315, 378, 144, 307, 153, 134, 255, 133, 215, 207, 68, 327, 38, 268, 200, 40, 75, 258, 58, 298, 292, 28, 253, 176, 30, 158, 34, 165, 105, 270, 377, 137, 242, 291, 147] + 33: [38, 68, 226, 376, 151, 222, 144, 230, 50, 370, 43, 28, 101, 318, 300, 119, 365, 274, 359, 223, 265, 293, 333, 372, 91, 153, 342, 201, 335, 160, 192, 9, 138, 216, 156, 279, 58, 215, 53, 295, 207, 168, 341, 195, 44, 47, 95, 220, 339, 176, 323, 84, 127, 17, 40, 217, 355, 383, 382, 180, 137, 66, 271, 327, 243, 381, 242, 315, 296, 203, 102, 277, 36, 51, 42, 179, 352, 273, 210, 289, 354, 109, 80, 367, 60, 292, 52, 67, 316, 245, 312, 253, 374, 0, 239, 287, 319, 328, 31, 73, 164, 233, 150, 268, 290, 16, 124, 209, 340, 106, 366, 248, 20, 132, 76, 59, 337, 304, 302, 34, 260, 167, 377, 82, 348, 37, 1, 356, 311, 173, 254, 104, 373, 345, 349, 278, 136, 23, 224, 118, 275, 99, 77, 129, 177, 86, 191, 107, 147, 361, 25, 237, 235, 93, 45, 375, 13, 234, 371, 70, 10, 238, 332, 98, 214, 309, 100, 264, 362, 89, 145, 379, 108, 48, 33, 208, 8, 187, 317, 55, 200, 249, 56, 103, 135, 346, 310, 62, 306, 321, 185, 83, 64, 94, 63, 39, 314, 5, 30, 193, 232, 351, 276, 262, 152, 213, 71, 166, 194, 74, 225, 175, 363, 2, 284, 69, 380, 267, 280, 96, 241, 270, 218, 155, 116, 202, 347, 162, 88, 170, 24, 285, 128, 298, 120, 257, 171, 7, 263, 236, 229, 307, 331, 297, 358, 255, 35, 131, 288, 90, 4, 157, 360, 15, 336, 112, 221, 111, 27, 178, 165, 344, 357, 161, 148, 154, 259, 183, 125, 169, 204, 334, 305, 329, 198, 256, 186, 368, 113, 110, 41, 130, 184, 244, 199, 57, 294, 142, 81, 299, 320, 325, 139, 266, 338, 227, 18, 211, 22, 196, 212, 364, 3, 65, 6, 182, 291, 353, 378, 172, 46, 32, 343, 283, 75, 231, 146, 190, 141, 133, 14, 205, 123, 19, 114, 122, 350, 324, 219, 286, 78, 134, 250, 228, 282, 12, 115, 301, 140, 121, 322, 21, 54, 308, 126, 303, 79, 246, 369, 105, 330, 26, 252, 181, 188, 247, 92, 313, 272, 159, 251, 258, 240, 158, 72, 206, 87, 117, 61, 149, 174, 197, 163, 49, 281, 29, 261, 11, 85, 326, 97, 269, 143, 189] + 34: [108, 194, 97, 300, 134, 12, 166, 195, 48, 247, 180, 10, 383, 314, 265, 93, 176, 104, 71, 252, 129, 14, 76, 119, 179, 223, 178, 2, 267, 124, 174, 197, 336, 72, 367, 352, 62, 186, 275, 86, 118, 205, 141, 54, 163, 273, 122, 132, 16, 354, 148, 284, 238, 77, 210, 136, 280, 328, 204, 361, 42, 298, 68, 219, 319, 165, 171, 128, 216, 343, 160, 189, 249, 151, 145, 138, 79, 6, 125, 184, 326, 133, 236, 333, 308, 127, 358, 88, 289, 380, 52, 55, 120, 106, 376, 193, 306, 237, 304, 40, 295, 8, 65, 81, 183, 143, 346, 364, 232, 19, 221, 85, 228, 100, 242, 112, 1, 325, 377, 202, 15, 47, 158, 362, 315, 78, 162, 329, 327, 137, 227, 89, 5, 114, 23, 51, 233, 307, 278, 20, 212, 344, 177, 198, 340, 207, 239, 243, 220, 191, 201, 110, 24, 37, 231, 116, 50, 157, 46, 147, 269, 331, 370, 90, 257, 182, 235, 229, 366, 29, 311, 240, 246, 41, 87, 107, 84, 60, 150, 102, 263, 277, 105, 264, 282, 75, 57, 155, 156, 303, 7, 211, 337, 382, 135, 291, 99, 335, 208, 144, 256, 317, 245, 214, 154, 164, 272, 192, 305, 255, 117, 297, 292, 94, 320, 213, 69, 372, 74, 167, 115, 271, 95, 290, 345, 172, 270, 169, 230, 294, 142, 266, 36, 226, 173, 342, 96, 188, 309, 363, 259, 61, 312, 203, 146, 92, 225, 373, 368, 251, 241, 0, 80, 63, 371, 268, 111, 274, 250, 253, 351, 234, 350, 224, 299, 190, 82, 293, 347, 222, 21, 58, 313, 33, 67, 287, 348, 322, 56, 98, 161, 286, 260, 339, 355, 276, 369, 279, 126, 196, 332, 185, 59, 288, 4, 53, 175, 217, 181, 209, 103, 140, 261, 152, 262, 353, 17, 285, 130, 338, 375, 365, 330, 44, 18, 31, 258, 11, 244, 356, 159, 206, 30, 359, 215, 318, 139, 296, 310, 34, 131, 9, 22, 349, 32, 316, 334, 3, 357, 321, 379, 254, 91, 153, 123, 199, 27, 170, 301, 323, 149, 64, 13, 28, 109, 39, 43, 378, 66, 341, 248, 218, 324, 25, 360, 83, 168, 200, 302, 187, 70, 73, 35, 113, 283, 45, 374, 101, 49, 26, 281, 38, 121, 381] + 35: [319, 211, 129, 186, 164, 159, 267, 36, 372, 122, 92, 214, 24, 301, 374, 87, 278, 89, 238, 33, 226, 199, 293, 177, 45, 62, 141, 346, 311, 331, 373, 272, 268, 313, 96, 277, 347, 376, 175, 35, 207, 115, 150, 148, 219, 2, 100, 343, 1, 80, 254, 153, 147, 335, 329, 360, 158, 30, 300, 350, 348, 166, 142, 338, 63, 245, 49, 107, 233, 280, 64, 228, 345, 309, 379, 21, 224, 326, 136, 160, 308, 294, 201, 261, 110, 383, 127, 84, 365, 149, 23, 10, 327, 252, 124, 85, 363, 48, 303, 4, 78, 307, 126, 187, 230, 213, 152, 287, 305, 330, 380, 25, 105, 316, 121, 77, 333, 247, 119, 291, 75, 340, 58, 324, 239, 234, 54, 321, 9, 215, 95, 172, 202, 216, 19, 88, 292, 370, 155, 288, 223, 0, 173, 359, 169, 281, 130, 138, 355, 194, 222, 357, 132, 190, 337, 235, 180, 34, 70, 344, 332, 377, 195, 165, 69, 188, 120, 304, 28, 113, 144, 47, 206, 349, 52, 102, 18, 328, 320, 170, 227, 204, 269, 43, 103, 253, 232, 183, 178, 208, 71, 13, 205, 322, 237, 257, 154, 265, 266, 134, 81, 104, 143, 323, 145, 382, 179, 73, 79, 248, 250, 255, 351, 242, 76, 381, 299, 72, 240, 55, 116, 364, 361, 37, 128, 61, 209, 11, 57, 106, 184, 279, 176, 135, 181, 263, 83, 371, 59, 283, 218, 276, 182, 286, 221, 375, 162, 27, 325, 171, 243, 16, 74, 369, 185, 285, 198, 189, 249, 358, 270, 123, 229, 284, 131, 203, 44, 368, 31, 352, 14, 163, 296, 5, 137, 32, 366, 264, 7, 196, 274, 259, 306, 97, 197, 258, 212, 200, 297, 342, 220, 318, 289, 46, 50, 98, 168, 192, 42, 90, 111, 91, 99, 17, 378, 29, 112, 167, 271, 336, 251, 60, 367, 86, 353, 56, 3, 41, 191, 312, 244, 298, 108, 256, 8, 82, 273, 217, 317, 161, 94, 262, 225, 51, 341, 40, 295, 53, 246, 67, 38, 114, 362, 157, 140, 282, 125, 12, 20, 117, 290, 315, 65, 151, 26, 354, 310, 210, 39, 146, 68, 133, 260, 314, 231, 15, 241, 275, 356, 93, 334, 302, 109, 339, 174, 66, 6, 156, 22, 139, 101, 193, 236, 118] + 36: [232, 380, 79, 42, 112, 49, 68, 171, 41, 211, 81, 335, 29, 58, 315, 249, 115, 8, 89, 181, 64, 170, 273, 302, 22, 59, 241, 97, 47, 33, 231, 167, 172, 153, 228, 217, 212, 17, 208, 239, 381, 337, 312, 87, 141, 222, 127, 360, 184, 370, 98, 207, 5, 262, 67, 138, 50, 168, 147, 305, 143, 157, 283, 281, 383, 357, 350, 126, 269, 306, 313, 375, 371, 197, 299, 56, 133, 261, 108, 293, 284, 342, 117, 174, 180, 39, 165, 365, 44, 6, 35, 137, 34, 96, 367, 336, 368, 294, 101, 40, 308, 32, 334, 194, 282, 289, 20, 100, 310, 28, 244, 51, 146, 77, 248, 253, 86, 124, 317, 270, 272, 252, 328, 369, 150, 85, 331, 191, 139, 183, 276, 116, 202, 257, 88, 23, 190, 158, 60, 14, 154, 330, 114, 145, 53, 192, 2, 274, 152, 93, 240, 219, 297, 245, 280, 237, 323, 36, 148, 80, 204, 259, 215, 110, 303, 27, 214, 16, 340, 131, 62, 196, 275, 92, 333, 9, 63, 164, 216, 179, 187, 52, 188, 119, 247, 169, 130, 226, 151, 227, 43, 268, 341, 107, 185, 258, 291, 205, 201, 246, 55, 69, 155, 66, 364, 104, 351, 78, 209, 265, 256, 61, 235, 286, 377, 25, 144, 223, 74, 320, 352, 332, 45, 319, 103, 91, 346, 195, 102, 200, 267, 163, 329, 338, 316, 254, 378, 4, 106, 37, 57, 12, 250, 260, 99, 54, 309, 111, 361, 300, 15, 271, 229, 0, 129, 220, 70, 295, 347, 118, 1, 128, 160, 279, 304, 372, 84, 230, 90, 314, 321, 362, 349, 3, 339, 177, 46, 76, 379, 48, 363, 236, 251, 213, 7, 327, 113, 376, 238, 225, 382, 296, 301, 31, 156, 94, 132, 298, 354, 224, 277, 264, 149, 243, 24, 109, 21, 120, 285, 203, 221, 290, 266, 193, 176, 242, 374, 125, 65, 345, 13, 322, 72, 343, 30, 71, 263, 318, 83, 189, 307, 75, 278, 136, 234, 292, 10, 178, 348, 210, 73, 287, 175, 166, 122, 121, 366, 140, 255, 38, 206, 18, 135, 173, 199, 355, 26, 353, 182, 186, 325, 311, 359, 326, 373, 233, 162, 288, 105, 95, 142, 11, 134, 344, 358, 159, 356, 198, 19, 324, 161, 82, 218, 123] + 37: [218, 104, 296, 111, 100, 207, 313, 210, 110, 216, 198, 106, 155, 307, 232, 263, 226, 112, 132, 236, 239, 288, 352, 383, 40, 171, 9, 181, 18, 85, 27, 41, 103, 356, 262, 94, 280, 53, 170, 259, 146, 177, 89, 358, 31, 375, 8, 75, 268, 178, 44, 361, 253, 147, 131, 322, 21, 365, 161, 15, 69, 255, 33, 249, 152, 378, 142, 214, 11, 276, 189, 120, 294, 151, 190, 174, 129, 327, 153, 359, 228, 149, 304, 285, 55, 321, 24, 267, 124, 260, 308, 184, 197, 186, 23, 99, 148, 243, 205, 292, 156, 252, 233, 192, 208, 173, 338, 135, 126, 334, 102, 105, 10, 266, 283, 225, 261, 90, 282, 209, 293, 273, 52, 305, 86, 245, 328, 257, 229, 64, 217, 333, 136, 37, 97, 325, 17, 303, 339, 57, 202, 381, 346, 251, 109, 66, 306, 179, 223, 360, 139, 330, 336, 51, 203, 30, 3, 331, 312, 274, 298, 215, 115, 63, 237, 45, 377, 56, 301, 364, 125, 272, 77, 185, 318, 32, 157, 258, 6, 141, 376, 83, 244, 38, 351, 254, 286, 374, 130, 227, 2, 246, 379, 71, 314, 48, 382, 302, 14, 343, 315, 287, 323, 36, 329, 224, 347, 284, 172, 138, 291, 310, 231, 281, 367, 275, 58, 319, 213, 372, 371, 47, 165, 163, 150, 122, 168, 39, 201, 247, 98, 279, 116, 160, 175, 35, 166, 20, 54, 67, 158, 240, 311, 341, 95, 355, 65, 118, 145, 76, 144, 143, 29, 206, 380, 317, 187, 13, 332, 81, 248, 180, 220, 84, 222, 188, 348, 46, 342, 140, 182, 235, 164, 370, 1, 43, 92, 335, 80, 373, 154, 26, 169, 183, 324, 326, 12, 211, 200, 0, 242, 289, 193, 114, 7, 194, 133, 316, 176, 82, 108, 34, 269, 212, 4, 256, 350, 191, 357, 300, 219, 25, 366, 340, 309, 362, 234, 59, 88, 117, 221, 19, 196, 61, 264, 49, 121, 241, 42, 368, 78, 295, 107, 70, 369, 238, 79, 113, 123, 363, 72, 344, 353, 93, 277, 320, 74, 230, 278, 87, 128, 195, 354, 91, 60, 101, 62, 28, 159, 50, 349, 162, 265, 199, 290, 16, 22, 204, 345, 127, 270, 73, 134, 167, 337, 271, 5, 299, 137, 297, 96, 119, 250, 68] + 38: [301, 169, 331, 219, 85, 359, 184, 48, 246, 376, 305, 337, 273, 215, 13, 202, 350, 153, 140, 318, 46, 60, 185, 237, 6, 309, 381, 77, 286, 352, 72, 307, 165, 229, 260, 33, 2, 313, 61, 297, 164, 65, 303, 162, 346, 251, 325, 380, 266, 93, 292, 63, 0, 300, 225, 222, 193, 57, 216, 289, 76, 353, 23, 122, 294, 205, 240, 112, 22, 160, 52, 191, 203, 187, 123, 220, 244, 344, 338, 99, 73, 105, 264, 111, 31, 242, 152, 214, 11, 183, 91, 320, 218, 198, 101, 210, 158, 4, 94, 194, 92, 206, 199, 355, 69, 262, 256, 37, 148, 41, 235, 79, 195, 80, 374, 113, 326, 315, 83, 161, 378, 10, 118, 248, 217, 135, 319, 233, 324, 59, 45, 298, 342, 197, 181, 173, 172, 295, 186, 299, 18, 20, 370, 176, 221, 282, 66, 115, 19, 27, 257, 106, 75, 358, 349, 179, 151, 84, 15, 296, 261, 231, 247, 254, 335, 365, 110, 339, 269, 62, 223, 8, 379, 166, 97, 329, 239, 21, 211, 142, 276, 356, 103, 302, 382, 25, 35, 343, 232, 132, 133, 121, 280, 89, 50, 274, 293, 212, 54, 90, 67, 29, 58, 308, 141, 28, 272, 200, 371, 311, 168, 363, 175, 291, 243, 304, 124, 333, 345, 263, 39, 17, 108, 327, 271, 95, 144, 7, 143, 180, 174, 81, 104, 375, 102, 3, 259, 116, 196, 340, 201, 154, 245, 332, 130, 145, 234, 288, 227, 208, 323, 136, 5, 241, 40, 334, 347, 82, 55, 236, 163, 207, 249, 312, 284, 277, 178, 322, 32, 328, 53, 361, 107, 224, 88, 156, 228, 306, 78, 360, 120, 100, 290, 357, 314, 209, 330, 64, 372, 114, 348, 86, 171, 278, 267, 125, 275, 150, 87, 367, 265, 170, 131, 238, 43, 252, 71, 12, 157, 377, 336, 38, 51, 383, 369, 226, 177, 279, 268, 341, 146, 321, 255, 368, 117, 190, 139, 182, 74, 44, 283, 70, 24, 167, 127, 98, 30, 126, 287, 285, 49, 317, 119, 16, 147, 362, 68, 96, 109, 26, 258, 250, 129, 47, 351, 192, 364, 281, 155, 159, 366, 189, 213, 354, 230, 373, 253, 137, 14, 9, 36, 34, 188, 134, 149, 310, 56, 204, 138, 316, 42, 128, 1, 270] + 39: [282, 267, 296, 250, 49, 202, 232, 138, 358, 364, 373, 88, 164, 377, 192, 303, 175, 123, 125, 308, 184, 89, 257, 44, 310, 156, 2, 362, 208, 59, 274, 340, 126, 371, 161, 61, 320, 363, 183, 27, 346, 211, 37, 67, 32, 90, 304, 209, 247, 348, 242, 329, 121, 77, 273, 140, 263, 353, 5, 33, 95, 4, 240, 91, 287, 78, 0, 219, 294, 21, 143, 361, 66, 82, 325, 145, 40, 163, 87, 127, 122, 69, 150, 301, 72, 97, 16, 181, 343, 55, 108, 284, 147, 237, 261, 149, 104, 234, 360, 86, 71, 76, 23, 191, 54, 152, 144, 213, 311, 275, 30, 214, 245, 159, 142, 64, 158, 124, 148, 52, 372, 280, 25, 272, 333, 369, 81, 248, 299, 367, 317, 63, 106, 133, 79, 265, 58, 111, 266, 233, 255, 332, 11, 165, 236, 96, 153, 271, 26, 300, 298, 383, 225, 74, 135, 162, 102, 173, 243, 60, 171, 117, 18, 252, 256, 57, 356, 22, 56, 99, 1, 155, 297, 141, 51, 84, 239, 137, 188, 226, 355, 193, 254, 120, 216, 107, 305, 359, 351, 262, 115, 9, 315, 176, 368, 119, 270, 17, 365, 375, 48, 101, 50, 307, 182, 203, 195, 327, 335, 168, 347, 238, 339, 269, 154, 198, 258, 100, 223, 279, 80, 98, 3, 146, 259, 112, 190, 281, 249, 212, 285, 309, 169, 109, 222, 20, 92, 349, 318, 321, 286, 43, 116, 13, 382, 324, 178, 7, 42, 251, 378, 322, 179, 94, 354, 130, 207, 196, 19, 83, 253, 350, 12, 186, 65, 85, 35, 151, 174, 172, 15, 381, 8, 323, 131, 62, 134, 167, 357, 220, 306, 288, 41, 206, 39, 210, 316, 201, 312, 379, 180, 215, 328, 295, 341, 235, 118, 336, 331, 366, 75, 228, 128, 221, 166, 283, 337, 246, 229, 326, 73, 344, 38, 313, 110, 139, 187, 374, 24, 290, 352, 28, 260, 177, 302, 289, 241, 68, 114, 217, 189, 160, 197, 370, 105, 380, 93, 314, 264, 218, 292, 277, 319, 200, 170, 31, 47, 276, 14, 293, 230, 338, 376, 291, 46, 157, 342, 129, 224, 29, 132, 36, 268, 113, 244, 204, 330, 227, 53, 278, 34, 199, 345, 45, 185, 6, 194, 70, 103, 231, 10, 136, 205, 334] + 40: [31, 316, 244, 155, 206, 37, 345, 246, 125, 135, 242, 285, 4, 298, 353, 170, 343, 105, 123, 149, 194, 214, 248, 300, 319, 213, 26, 158, 228, 362, 274, 289, 382, 56, 263, 151, 324, 237, 314, 78, 121, 266, 34, 134, 39, 24, 240, 376, 15, 181, 107, 88, 59, 363, 162, 322, 196, 309, 364, 25, 371, 281, 73, 6, 84, 268, 163, 22, 267, 191, 42, 92, 68, 260, 62, 161, 208, 91, 129, 277, 334, 302, 53, 299, 192, 305, 350, 261, 115, 184, 117, 106, 234, 223, 83, 249, 175, 280, 101, 156, 251, 17, 143, 63, 30, 308, 137, 57, 377, 356, 282, 85, 238, 276, 310, 77, 111, 313, 220, 72, 378, 340, 98, 279, 74, 28, 89, 80, 11, 199, 46, 82, 94, 287, 48, 47, 36, 256, 49, 40, 114, 176, 86, 312, 317, 357, 367, 202, 254, 3, 9, 168, 145, 200, 227, 189, 103, 327, 171, 60, 291, 288, 50, 113, 352, 381, 112, 331, 207, 379, 75, 215, 372, 58, 347, 341, 61, 148, 187, 365, 355, 339, 195, 301, 375, 361, 383, 257, 231, 96, 172, 225, 183, 132, 373, 297, 8, 100, 296, 10, 54, 216, 259, 150, 2, 29, 304, 18, 76, 348, 13, 66, 173, 315, 272, 335, 250, 226, 204, 52, 64, 224, 275, 326, 27, 236, 229, 43, 7, 120, 273, 293, 241, 0, 102, 35, 217, 380, 320, 146, 349, 33, 67, 337, 369, 258, 1, 336, 278, 38, 269, 138, 165, 338, 19, 131, 201, 23, 124, 218, 104, 152, 20, 126, 360, 346, 359, 265, 81, 235, 210, 271, 211, 205, 307, 95, 290, 333, 329, 139, 118, 14, 140, 203, 180, 87, 245, 344, 142, 179, 55, 283, 332, 198, 159, 209, 185, 370, 44, 41, 110, 93, 323, 222, 147, 141, 368, 230, 325, 109, 252, 5, 328, 108, 70, 247, 164, 97, 303, 136, 193, 12, 262, 65, 130, 318, 330, 233, 270, 212, 374, 295, 119, 342, 127, 197, 71, 21, 286, 160, 153, 351, 284, 219, 166, 188, 239, 178, 294, 264, 243, 51, 116, 186, 16, 174, 154, 45, 255, 144, 358, 190, 169, 128, 232, 90, 311, 321, 253, 221, 79, 99, 306, 167, 177, 366, 122, 133, 32, 182, 354, 157, 292, 69] + 41: [49, 11, 365, 172, 95, 326, 25, 174, 56, 322, 224, 107, 18, 317, 188, 292, 180, 28, 209, 157, 189, 118, 263, 357, 378, 196, 273, 92, 279, 306, 222, 336, 54, 235, 112, 268, 43, 128, 232, 77, 248, 98, 265, 145, 319, 135, 22, 309, 199, 87, 150, 76, 360, 253, 53, 243, 102, 55, 115, 328, 290, 5, 371, 272, 33, 165, 182, 329, 201, 346, 16, 40, 170, 185, 255, 285, 21, 198, 4, 126, 116, 254, 117, 266, 295, 246, 221, 41, 381, 48, 149, 375, 240, 88, 203, 178, 60, 305, 249, 124, 24, 108, 351, 353, 64, 256, 167, 288, 186, 19, 173, 154, 352, 332, 147, 14, 267, 361, 58, 247, 350, 122, 212, 97, 314, 337, 3, 23, 301, 276, 289, 380, 338, 81, 244, 300, 1, 141, 137, 29, 331, 362, 356, 296, 374, 205, 90, 293, 69, 44, 35, 194, 345, 93, 325, 8, 181, 6, 61, 359, 0, 72, 148, 370, 110, 162, 164, 31, 321, 215, 307, 223, 111, 206, 104, 202, 349, 241, 237, 144, 51, 160, 192, 303, 287, 68, 158, 67, 121, 312, 230, 120, 269, 119, 27, 50, 364, 311, 80, 42, 32, 282, 294, 250, 131, 261, 146, 15, 39, 259, 175, 373, 323, 211, 333, 152, 298, 159, 251, 327, 358, 347, 318, 36, 217, 379, 156, 138, 299, 94, 74, 99, 78, 297, 195, 17, 20, 377, 281, 216, 283, 275, 376, 368, 79, 262, 45, 219, 26, 225, 83, 133, 63, 277, 354, 37, 100, 320, 155, 238, 343, 270, 228, 233, 114, 129, 71, 200, 34, 12, 9, 239, 134, 70, 340, 330, 363, 130, 89, 214, 179, 13, 113, 171, 47, 106, 382, 132, 142, 366, 166, 168, 187, 334, 127, 140, 308, 125, 271, 242, 176, 208, 220, 123, 191, 207, 310, 231, 62, 341, 169, 52, 355, 197, 103, 84, 7, 280, 348, 264, 218, 226, 258, 344, 151, 91, 213, 342, 143, 234, 73, 204, 2, 177, 291, 278, 184, 369, 367, 57, 136, 302, 383, 286, 252, 46, 260, 210, 66, 153, 315, 75, 236, 38, 257, 245, 139, 372, 193, 227, 190, 30, 335, 86, 96, 161, 82, 324, 163, 313, 229, 339, 101, 284, 10, 59, 65, 105, 316, 274, 183, 109, 304, 85] + 42: [123, 137, 238, 310, 332, 364, 229, 104, 207, 115, 182, 24, 289, 27, 242, 294, 369, 376, 25, 71, 95, 254, 324, 339, 152, 247, 220, 342, 127, 274, 187, 147, 92, 160, 18, 311, 183, 196, 53, 343, 321, 75, 304, 249, 44, 382, 126, 121, 117, 211, 162, 263, 277, 30, 287, 201, 234, 118, 19, 64, 283, 5, 378, 293, 368, 186, 330, 11, 79, 0, 352, 193, 31, 295, 72, 374, 255, 373, 248, 17, 105, 322, 65, 340, 372, 315, 336, 355, 26, 145, 185, 367, 12, 83, 13, 46, 359, 20, 184, 179, 377, 319, 131, 140, 167, 181, 33, 192, 301, 89, 97, 204, 346, 284, 360, 136, 134, 110, 142, 40, 217, 375, 305, 41, 129, 21, 370, 135, 165, 335, 52, 297, 8, 323, 299, 251, 171, 318, 363, 288, 96, 70, 317, 252, 94, 154, 303, 150, 55, 327, 275, 259, 230, 308, 366, 380, 132, 246, 226, 291, 233, 206, 10, 334, 133, 57, 271, 273, 266, 210, 106, 264, 108, 189, 328, 224, 87, 36, 316, 381, 146, 9, 232, 16, 112, 290, 73, 198, 88, 100, 43, 15, 268, 188, 357, 63, 168, 208, 191, 354, 344, 326, 38, 39, 245, 219, 49, 258, 68, 56, 90, 285, 383, 35, 107, 86, 345, 2, 119, 279, 262, 203, 351, 320, 286, 356, 314, 227, 80, 231, 157, 58, 269, 84, 82, 128, 302, 353, 14, 91, 176, 37, 228, 151, 195, 59, 379, 221, 236, 216, 143, 76, 241, 62, 130, 313, 125, 194, 250, 292, 347, 190, 54, 240, 172, 138, 148, 282, 349, 48, 235, 178, 239, 338, 280, 166, 60, 307, 139, 205, 278, 175, 199, 296, 22, 153, 78, 365, 155, 333, 114, 215, 243, 209, 122, 141, 28, 161, 77, 149, 32, 180, 260, 7, 156, 261, 267, 124, 309, 276, 81, 66, 218, 158, 42, 212, 200, 281, 265, 4, 169, 103, 164, 257, 113, 371, 244, 298, 170, 120, 98, 3, 144, 47, 331, 74, 93, 256, 1, 350, 222, 6, 348, 341, 214, 173, 202, 223, 101, 225, 109, 34, 300, 213, 197, 23, 174, 253, 177, 163, 325, 111, 50, 116, 337, 306, 358, 29, 67, 329, 45, 362, 51, 237, 99, 61, 270, 272, 102, 312, 85, 69, 159, 361] + 43: [168, 332, 39, 57, 301, 338, 255, 282, 123, 149, 241, 47, 183, 125, 207, 122, 173, 355, 162, 106, 15, 104, 245, 348, 225, 344, 35, 94, 186, 318, 121, 73, 48, 299, 157, 145, 253, 101, 295, 8, 247, 211, 341, 172, 133, 176, 44, 354, 375, 27, 197, 366, 153, 227, 280, 32, 327, 271, 130, 107, 165, 167, 18, 76, 126, 109, 53, 199, 333, 212, 95, 3, 66, 369, 129, 100, 81, 203, 136, 85, 222, 140, 160, 237, 60, 266, 80, 170, 87, 132, 119, 69, 230, 58, 96, 214, 98, 164, 371, 252, 377, 111, 108, 272, 380, 204, 79, 28, 293, 143, 220, 329, 294, 233, 110, 357, 228, 362, 166, 195, 315, 23, 142, 239, 93, 29, 198, 150, 88, 286, 365, 334, 38, 210, 128, 116, 356, 268, 187, 179, 249, 383, 184, 169, 221, 351, 218, 216, 363, 342, 274, 368, 90, 277, 36, 243, 102, 56, 256, 288, 65, 19, 209, 248, 317, 139, 91, 192, 105, 43, 308, 349, 314, 264, 147, 89, 378, 135, 26, 67, 50, 217, 263, 292, 259, 287, 240, 258, 21, 236, 25, 206, 171, 298, 278, 59, 22, 323, 307, 374, 154, 306, 5, 63, 49, 70, 343, 370, 117, 163, 320, 276, 7, 270, 231, 361, 156, 297, 177, 37, 326, 322, 269, 250, 238, 86, 84, 155, 113, 45, 213, 11, 72, 358, 316, 260, 141, 281, 244, 182, 347, 30, 68, 112, 291, 223, 313, 2, 275, 339, 61, 10, 185, 251, 325, 226, 267, 381, 336, 285, 328, 340, 302, 376, 92, 75, 161, 20, 235, 127, 305, 77, 346, 219, 34, 345, 99, 120, 146, 159, 279, 118, 265, 51, 0, 42, 289, 331, 372, 190, 6, 254, 152, 138, 1, 4, 52, 234, 261, 273, 242, 46, 309, 31, 64, 134, 290, 312, 62, 208, 319, 373, 335, 283, 181, 97, 337, 55, 224, 191, 246, 83, 257, 196, 229, 180, 74, 304, 144, 158, 300, 311, 137, 303, 350, 382, 17, 14, 82, 13, 103, 178, 78, 321, 205, 200, 296, 12, 148, 284, 9, 215, 353, 193, 330, 114, 124, 352, 310, 54, 359, 324, 262, 364, 131, 360, 202, 379, 232, 115, 367, 194, 24, 16, 174, 175, 71, 151, 188, 201, 41, 40, 33, 189] + 44: [213, 257, 229, 86, 329, 337, 91, 296, 315, 380, 134, 126, 334, 309, 179, 357, 114, 375, 99, 330, 111, 308, 322, 374, 77, 25, 267, 311, 383, 142, 360, 138, 290, 70, 160, 381, 136, 40, 377, 237, 18, 7, 1, 194, 302, 31, 16, 116, 301, 258, 188, 135, 166, 141, 225, 207, 254, 100, 131, 221, 157, 185, 231, 156, 250, 51, 35, 373, 363, 356, 208, 332, 191, 17, 227, 140, 236, 249, 275, 89, 224, 201, 132, 261, 292, 74, 8, 3, 235, 232, 216, 115, 298, 293, 88, 303, 199, 0, 215, 331, 98, 182, 291, 117, 61, 55, 130, 120, 107, 90, 59, 169, 193, 367, 45, 27, 103, 127, 310, 12, 10, 167, 190, 328, 313, 118, 144, 94, 350, 219, 248, 269, 354, 108, 105, 211, 319, 4, 255, 251, 283, 259, 76, 112, 317, 64, 110, 279, 34, 306, 102, 14, 266, 39, 352, 359, 26, 15, 149, 341, 9, 53, 273, 119, 246, 123, 353, 209, 226, 97, 351, 368, 288, 294, 228, 340, 192, 150, 164, 177, 162, 165, 243, 20, 234, 137, 80, 153, 176, 152, 75, 339, 348, 238, 155, 29, 93, 38, 33, 121, 85, 173, 366, 369, 46, 370, 347, 23, 263, 183, 200, 233, 260, 335, 181, 13, 362, 168, 60, 277, 146, 242, 125, 355, 24, 349, 286, 54, 220, 278, 318, 32, 241, 62, 312, 151, 247, 139, 58, 161, 274, 346, 180, 252, 187, 289, 217, 265, 299, 287, 66, 174, 41, 92, 214, 104, 57, 204, 2, 239, 268, 361, 371, 189, 172, 170, 382, 68, 320, 264, 304, 158, 175, 163, 338, 96, 342, 282, 323, 81, 72, 256, 42, 195, 69, 37, 148, 106, 171, 145, 271, 43, 143, 63, 48, 276, 184, 364, 79, 87, 128, 83, 78, 336, 109, 314, 245, 5, 372, 73, 133, 129, 222, 21, 300, 82, 30, 333, 365, 324, 244, 122, 71, 67, 36, 147, 270, 218, 203, 321, 253, 22, 28, 297, 197, 206, 379, 284, 178, 376, 11, 212, 358, 343, 6, 154, 285, 202, 327, 196, 205, 305, 230, 49, 124, 50, 240, 47, 316, 295, 101, 223, 272, 84, 325, 280, 198, 95, 262, 186, 56, 378, 44, 344, 159, 19, 113, 345, 65, 326, 210, 307, 52, 281] + 45: [92, 378, 297, 39, 229, 308, 213, 106, 303, 58, 126, 282, 288, 314, 343, 225, 89, 270, 6, 325, 300, 174, 221, 347, 140, 233, 120, 47, 183, 228, 104, 372, 357, 109, 1, 19, 145, 38, 241, 289, 309, 238, 153, 305, 99, 66, 83, 100, 178, 318, 197, 85, 114, 351, 45, 338, 7, 44, 236, 340, 116, 358, 379, 271, 48, 377, 90, 97, 265, 75, 49, 72, 10, 301, 182, 103, 348, 331, 196, 88, 210, 157, 365, 257, 35, 205, 273, 169, 363, 113, 93, 186, 226, 148, 69, 17, 227, 173, 199, 146, 124, 248, 55, 74, 251, 86, 260, 222, 20, 170, 322, 374, 79, 268, 337, 280, 193, 216, 185, 138, 154, 191, 346, 160, 165, 258, 310, 30, 112, 108, 179, 376, 43, 46, 294, 127, 246, 190, 202, 82, 143, 366, 181, 14, 23, 324, 25, 369, 163, 275, 307, 277, 327, 135, 175, 2, 34, 291, 283, 381, 105, 364, 111, 62, 200, 77, 237, 192, 121, 383, 139, 262, 212, 234, 162, 245, 306, 80, 208, 123, 330, 319, 264, 231, 220, 371, 214, 102, 352, 172, 11, 40, 161, 81, 284, 171, 131, 224, 168, 16, 84, 281, 217, 22, 50, 252, 5, 342, 166, 78, 73, 56, 159, 278, 299, 64, 293, 95, 317, 176, 61, 128, 118, 51, 70, 267, 67, 96, 133, 296, 98, 255, 341, 147, 91, 31, 232, 13, 60, 117, 42, 164, 144, 247, 180, 209, 223, 122, 313, 242, 292, 63, 218, 87, 9, 263, 119, 312, 321, 125, 101, 287, 130, 355, 334, 329, 316, 356, 269, 52, 198, 115, 54, 156, 94, 302, 65, 304, 29, 151, 0, 368, 243, 382, 353, 259, 149, 244, 201, 276, 134, 375, 354, 107, 254, 155, 250, 345, 279, 24, 207, 59, 37, 211, 15, 203, 323, 132, 194, 335, 195, 137, 8, 373, 188, 339, 298, 272, 359, 36, 12, 41, 26, 4, 344, 290, 150, 57, 142, 3, 71, 152, 167, 53, 240, 336, 27, 367, 110, 141, 21, 349, 187, 326, 328, 249, 32, 253, 370, 256, 320, 177, 285, 362, 332, 189, 295, 28, 184, 266, 33, 350, 219, 274, 286, 380, 239, 158, 68, 360, 315, 261, 76, 230, 333, 129, 204, 215, 136, 18, 206, 311, 235, 361] + 46: [142, 170, 284, 38, 303, 102, 155, 260, 316, 50, 321, 268, 289, 313, 69, 190, 86, 163, 330, 134, 251, 225, 82, 122, 183, 210, 6, 254, 113, 350, 22, 372, 166, 13, 17, 193, 35, 222, 277, 364, 257, 337, 11, 72, 276, 70, 379, 356, 352, 188, 324, 68, 338, 39, 143, 121, 315, 128, 172, 57, 360, 255, 322, 145, 184, 15, 21, 126, 47, 349, 175, 213, 311, 152, 96, 344, 204, 345, 266, 117, 287, 192, 304, 103, 377, 160, 24, 348, 331, 262, 52, 75, 236, 54, 0, 5, 133, 335, 25, 209, 60, 55, 351, 131, 232, 137, 41, 198, 32, 308, 173, 317, 91, 370, 43, 358, 300, 123, 84, 27, 10, 140, 16, 104, 40, 165, 49, 280, 245, 156, 95, 272, 111, 97, 119, 71, 291, 244, 371, 31, 87, 116, 200, 36, 247, 66, 249, 191, 124, 76, 211, 108, 326, 369, 176, 157, 59, 92, 89, 56, 90, 93, 135, 229, 288, 179, 357, 88, 34, 319, 46, 202, 194, 199, 282, 130, 168, 180, 362, 216, 81, 329, 346, 293, 261, 278, 26, 281, 363, 248, 14, 147, 64, 106, 368, 246, 312, 203, 215, 195, 334, 259, 23, 292, 383, 78, 297, 9, 177, 309, 67, 94, 381, 185, 100, 302, 139, 354, 327, 239, 301, 146, 79, 33, 80, 4, 298, 235, 158, 263, 223, 20, 342, 339, 138, 77, 73, 306, 270, 220, 256, 367, 240, 51, 136, 44, 219, 376, 186, 109, 63, 237, 125, 144, 299, 159, 217, 361, 65, 127, 105, 212, 28, 231, 58, 230, 224, 274, 214, 374, 85, 8, 218, 197, 53, 273, 101, 373, 382, 74, 241, 98, 375, 341, 107, 279, 167, 252, 286, 149, 314, 238, 12, 242, 228, 62, 99, 42, 164, 267, 307, 141, 318, 153, 333, 171, 347, 366, 196, 296, 182, 325, 380, 207, 19, 221, 187, 18, 3, 120, 227, 37, 129, 340, 45, 328, 161, 265, 359, 234, 150, 258, 269, 29, 151, 61, 253, 343, 233, 332, 378, 114, 30, 115, 132, 295, 294, 250, 336, 290, 169, 7, 264, 305, 208, 275, 1, 174, 148, 323, 355, 110, 201, 154, 271, 2, 320, 83, 353, 118, 162, 205, 226, 206, 189, 243, 283, 285, 112, 310, 48, 181, 178, 365] + 47: [195, 198, 188, 159, 84, 124, 33, 12, 340, 117, 15, 303, 37, 310, 120, 308, 16, 30, 299, 8, 13, 191, 56, 288, 3, 279, 305, 274, 131, 122, 69, 223, 140, 322, 173, 183, 335, 60, 2, 10, 55, 304, 222, 82, 167, 214, 373, 326, 343, 375, 286, 362, 59, 114, 95, 166, 172, 0, 4, 23, 139, 281, 115, 200, 19, 275, 339, 317, 99, 283, 236, 209, 54, 35, 314, 344, 40, 251, 208, 89, 284, 184, 180, 233, 356, 273, 341, 329, 48, 189, 5, 277, 116, 232, 311, 369, 44, 192, 193, 215, 175, 327, 313, 377, 41, 199, 237, 164, 336, 225, 111, 20, 46, 57, 374, 361, 142, 221, 182, 246, 352, 143, 61, 316, 129, 342, 98, 42, 103, 255, 45, 106, 378, 276, 258, 260, 102, 72, 380, 372, 11, 125, 318, 293, 24, 179, 109, 353, 21, 229, 269, 104, 263, 364, 39, 354, 18, 259, 31, 186, 149, 216, 92, 38, 64, 165, 7, 62, 312, 187, 1, 368, 267, 363, 176, 67, 319, 50, 148, 53, 321, 163, 94, 248, 150, 266, 108, 178, 194, 206, 156, 146, 87, 347, 119, 14, 79, 346, 212, 26, 151, 370, 205, 52, 366, 121, 328, 185, 253, 152, 358, 86, 145, 217, 83, 360, 181, 271, 43, 345, 127, 110, 349, 330, 262, 285, 296, 51, 252, 70, 71, 49, 32, 242, 325, 174, 220, 244, 256, 113, 154, 75, 371, 309, 196, 118, 218, 351, 224, 323, 73, 355, 96, 243, 211, 107, 157, 160, 177, 207, 202, 264, 295, 128, 162, 137, 291, 333, 257, 287, 235, 226, 68, 298, 65, 25, 112, 80, 289, 278, 297, 365, 379, 290, 47, 247, 254, 381, 22, 134, 357, 227, 36, 238, 63, 265, 135, 249, 197, 338, 210, 348, 126, 332, 97, 334, 383, 350, 76, 88, 66, 261, 158, 300, 171, 34, 294, 376, 367, 155, 93, 324, 77, 101, 315, 91, 28, 292, 144, 190, 245, 74, 85, 270, 241, 272, 29, 359, 136, 280, 320, 168, 153, 230, 219, 228, 203, 138, 234, 141, 213, 306, 132, 9, 282, 58, 100, 268, 6, 169, 382, 307, 331, 170, 90, 240, 123, 250, 301, 231, 204, 17, 27, 201, 147, 337, 130, 81, 302, 133, 105, 78, 161, 239] + 48: [17, 26, 226, 83, 189, 254, 229, 322, 120, 329, 168, 22, 119, 84, 144, 382, 93, 3, 309, 338, 183, 220, 129, 227, 271, 249, 44, 133, 223, 186, 290, 179, 257, 170, 4, 339, 36, 198, 169, 108, 310, 163, 201, 225, 68, 126, 30, 308, 140, 297, 109, 295, 6, 264, 252, 161, 285, 255, 172, 57, 281, 349, 151, 187, 173, 213, 61, 282, 125, 265, 136, 314, 328, 159, 358, 90, 147, 372, 232, 300, 190, 324, 218, 360, 158, 211, 292, 243, 92, 29, 350, 362, 248, 303, 334, 250, 132, 237, 150, 2, 287, 359, 380, 222, 192, 45, 321, 42, 113, 204, 367, 206, 272, 135, 37, 105, 86, 153, 355, 10, 267, 263, 301, 182, 369, 48, 134, 210, 299, 123, 212, 91, 241, 97, 18, 55, 261, 214, 56, 356, 331, 260, 131, 348, 283, 15, 279, 203, 207, 208, 115, 305, 293, 152, 242, 315, 307, 289, 43, 53, 71, 286, 294, 278, 345, 296, 199, 167, 76, 62, 145, 74, 104, 38, 363, 205, 193, 247, 89, 357, 80, 351, 12, 127, 256, 32, 87, 335, 377, 46, 364, 200, 266, 78, 117, 121, 166, 327, 184, 354, 175, 233, 162, 230, 317, 14, 269, 16, 82, 217, 143, 110, 148, 251, 306, 196, 128, 85, 65, 343, 28, 235, 181, 155, 156, 88, 73, 275, 379, 138, 280, 40, 178, 366, 323, 244, 149, 332, 103, 209, 102, 381, 77, 236, 157, 240, 142, 371, 7, 291, 31, 347, 180, 70, 352, 98, 216, 95, 165, 376, 60, 106, 337, 245, 330, 141, 270, 116, 21, 375, 325, 368, 246, 370, 341, 79, 231, 96, 11, 94, 274, 326, 27, 228, 154, 262, 176, 304, 273, 101, 20, 99, 268, 139, 316, 319, 312, 174, 160, 24, 64, 146, 253, 107, 137, 67, 313, 188, 171, 23, 383, 114, 239, 353, 9, 19, 164, 5, 185, 320, 130, 75, 0, 221, 100, 49, 63, 346, 39, 47, 277, 66, 191, 54, 122, 258, 276, 284, 51, 234, 33, 344, 34, 378, 25, 50, 118, 111, 202, 302, 81, 35, 259, 69, 373, 361, 8, 124, 197, 336, 52, 311, 288, 215, 58, 177, 342, 41, 219, 112, 340, 318, 13, 72, 374, 59, 195, 194, 224, 238, 298, 365, 333, 1] + 49: [302, 255, 5, 336, 245, 170, 163, 295, 98, 239, 71, 124, 154, 232, 167, 171, 217, 68, 375, 337, 356, 64, 123, 191, 321, 278, 148, 76, 306, 333, 48, 137, 138, 189, 273, 43, 62, 117, 118, 228, 354, 13, 147, 350, 263, 188, 38, 317, 297, 83, 181, 110, 177, 246, 45, 164, 254, 16, 365, 222, 82, 36, 345, 69, 49, 283, 145, 27, 42, 18, 258, 26, 253, 37, 158, 261, 262, 286, 331, 8, 205, 17, 140, 323, 301, 238, 156, 211, 355, 361, 364, 166, 96, 287, 288, 219, 126, 149, 150, 270, 259, 260, 187, 131, 58, 28, 56, 224, 225, 327, 351, 47, 130, 113, 256, 51, 229, 152, 207, 373, 374, 197, 234, 25, 276, 133, 74, 314, 122, 80, 368, 34, 77, 308, 244, 320, 31, 192, 326, 357, 159, 334, 369, 50, 79, 184, 136, 241, 39, 218, 329, 100, 73, 1, 223, 127, 81, 33, 359, 72, 0, 285, 88, 104, 341, 281, 215, 24, 360, 233, 257, 78, 335, 199, 269, 95, 144, 54, 86, 112, 376, 330, 338, 378, 119, 206, 125, 316, 162, 231, 11, 349, 272, 210, 304, 353, 90, 91, 344, 298, 121, 310, 196, 290, 303, 371, 178, 226, 358, 84, 108, 315, 7, 21, 230, 92, 6, 251, 346, 293, 40, 289, 194, 264, 328, 340, 277, 29, 55, 173, 309, 53, 61, 282, 182, 343, 102, 266, 32, 129, 268, 193, 319, 116, 208, 243, 132, 109, 235, 382, 318, 128, 139, 179, 227, 85, 12, 284, 348, 111, 372, 249, 141, 347, 75, 242, 294, 213, 201, 107, 220, 94, 195, 221, 23, 89, 20, 67, 103, 114, 204, 143, 22, 57, 165, 161, 252, 135, 155, 9, 15, 115, 275, 60, 120, 183, 250, 362, 198, 214, 216, 10, 134, 280, 46, 342, 185, 168, 363, 236, 247, 370, 305, 271, 65, 105, 180, 325, 312, 332, 169, 174, 311, 41, 248, 267, 142, 339, 2, 151, 324, 274, 367, 99, 291, 300, 19, 202, 59, 212, 44, 380, 381, 30, 366, 63, 70, 160, 146, 352, 172, 14, 106, 101, 52, 377, 237, 265, 203, 3, 157, 313, 209, 66, 153, 292, 186, 175, 97, 279, 240, 4, 296, 35, 307, 176, 200, 190, 299, 93, 87, 379, 383, 322] + 50: [86, 14, 146, 327, 22, 253, 196, 313, 102, 342, 151, 2, 278, 362, 153, 349, 318, 162, 13, 157, 161, 118, 111, 345, 268, 67, 110, 296, 295, 224, 122, 48, 348, 169, 234, 322, 172, 5, 124, 7, 248, 147, 77, 72, 261, 364, 182, 365, 29, 128, 38, 287, 199, 79, 16, 164, 71, 240, 197, 382, 91, 108, 37, 177, 63, 141, 11, 324, 265, 266, 44, 84, 100, 231, 333, 125, 45, 331, 213, 274, 117, 116, 252, 81, 107, 99, 127, 204, 12, 354, 171, 61, 173, 309, 254, 136, 316, 129, 229, 366, 220, 51, 155, 96, 19, 263, 174, 371, 242, 113, 135, 123, 106, 148, 65, 101, 216, 31, 368, 230, 109, 3, 97, 373, 192, 314, 159, 298, 75, 154, 250, 139, 255, 143, 236, 170, 351, 293, 98, 30, 207, 167, 227, 283, 49, 1, 140, 273, 62, 88, 270, 133, 380, 379, 180, 310, 145, 356, 178, 288, 200, 212, 87, 346, 245, 104, 59, 332, 181, 18, 137, 35, 329, 119, 249, 187, 195, 359, 247, 92, 20, 357, 191, 281, 156, 90, 36, 60, 8, 95, 66, 23, 166, 26, 311, 218, 338, 114, 130, 24, 149, 183, 189, 303, 64, 205, 202, 292, 46, 68, 179, 83, 103, 158, 131, 222, 376, 350, 299, 367, 304, 256, 271, 6, 228, 168, 43, 326, 152, 105, 297, 4, 305, 237, 286, 176, 190, 279, 307, 241, 32, 27, 235, 370, 280, 334, 343, 275, 186, 203, 276, 267, 126, 185, 243, 339, 308, 330, 291, 21, 201, 221, 260, 257, 315, 56, 226, 284, 9, 358, 15, 363, 306, 134, 206, 378, 163, 340, 39, 337, 238, 383, 82, 40, 328, 54, 317, 374, 210, 214, 80, 262, 312, 0, 335, 132, 58, 375, 208, 94, 160, 223, 347, 344, 320, 89, 323, 76, 246, 225, 302, 264, 188, 321, 25, 233, 377, 193, 184, 301, 239, 290, 53, 73, 277, 211, 369, 42, 34, 150, 112, 209, 360, 120, 52, 50, 215, 325, 33, 361, 175, 217, 121, 353, 165, 194, 319, 74, 294, 28, 300, 259, 198, 355, 285, 381, 70, 142, 336, 55, 69, 232, 57, 352, 282, 341, 272, 41, 244, 219, 269, 85, 78, 251, 17, 289, 372, 258, 138, 10, 47, 144, 115, 93] + 51: [24, 113, 358, 95, 330, 346, 35, 161, 81, 42, 309, 50, 324, 258, 27, 242, 238, 19, 39, 77, 227, 377, 156, 202, 186, 123, 67, 203, 188, 11, 259, 281, 359, 366, 248, 141, 289, 115, 51, 93, 159, 66, 96, 367, 293, 21, 173, 348, 372, 106, 284, 211, 124, 177, 92, 292, 56, 360, 321, 368, 317, 84, 194, 298, 197, 148, 230, 70, 355, 231, 249, 261, 102, 31, 193, 300, 260, 129, 269, 143, 382, 97, 18, 204, 361, 157, 313, 191, 307, 316, 149, 215, 120, 71, 131, 226, 137, 108, 99, 176, 87, 105, 335, 299, 23, 268, 351, 34, 75, 364, 312, 89, 319, 109, 15, 4, 128, 169, 256, 255, 373, 322, 111, 356, 60, 301, 49, 247, 47, 237, 181, 59, 236, 306, 349, 379, 363, 336, 240, 158, 277, 352, 150, 327, 375, 286, 16, 64, 140, 153, 168, 216, 303, 308, 219, 91, 195, 320, 334, 315, 273, 353, 57, 354, 132, 175, 244, 291, 221, 163, 357, 136, 343, 282, 246, 20, 275, 254, 380, 252, 362, 205, 369, 328, 145, 378, 25, 296, 217, 179, 347, 326, 241, 229, 6, 228, 32, 235, 26, 46, 331, 86, 280, 370, 10, 69, 383, 114, 30, 43, 125, 257, 305, 63, 29, 121, 37, 126, 73, 72, 65, 283, 314, 5, 183, 341, 192, 79, 41, 239, 297, 381, 340, 55, 1, 83, 376, 187, 38, 151, 210, 82, 371, 122, 152, 146, 253, 220, 212, 142, 190, 207, 48, 224, 100, 54, 276, 53, 167, 272, 154, 185, 36, 2, 110, 144, 189, 264, 62, 304, 263, 8, 285, 198, 222, 90, 3, 265, 196, 223, 80, 112, 184, 14, 165, 318, 117, 138, 200, 337, 178, 182, 40, 250, 103, 294, 344, 345, 279, 251, 33, 311, 365, 45, 333, 58, 68, 9, 174, 302, 225, 139, 323, 342, 206, 52, 270, 233, 130, 118, 245, 172, 209, 310, 164, 234, 213, 338, 288, 232, 218, 88, 199, 274, 22, 76, 208, 17, 101, 180, 104, 325, 134, 13, 85, 107, 329, 119, 166, 171, 332, 170, 78, 61, 116, 28, 0, 133, 267, 290, 12, 287, 98, 127, 135, 295, 243, 374, 271, 74, 162, 160, 44, 262, 94, 350, 278, 339, 214, 7, 201, 266, 147, 155] + 52: [381, 323, 298, 60, 269, 98, 379, 222, 167, 280, 53, 213, 308, 277, 20, 330, 380, 313, 304, 11, 12, 360, 212, 348, 99, 289, 82, 33, 168, 50, 325, 5, 315, 292, 361, 287, 326, 177, 148, 188, 141, 18, 135, 255, 317, 240, 176, 180, 334, 130, 34, 346, 44, 252, 133, 256, 223, 116, 115, 333, 93, 242, 238, 198, 143, 343, 138, 314, 373, 14, 370, 375, 102, 267, 271, 7, 260, 266, 291, 74, 178, 367, 113, 39, 45, 258, 224, 49, 303, 353, 232, 42, 159, 162, 106, 247, 107, 320, 290, 90, 357, 355, 70, 127, 83, 227, 243, 72, 66, 119, 58, 189, 318, 111, 226, 158, 9, 202, 218, 204, 157, 231, 246, 211, 41, 219, 87, 88, 117, 173, 369, 76, 324, 161, 89, 207, 47, 118, 43, 274, 79, 19, 199, 16, 71, 120, 151, 345, 172, 26, 64, 123, 262, 281, 6, 153, 91, 40, 365, 220, 150, 278, 126, 112, 185, 65, 264, 46, 104, 307, 31, 321, 78, 10, 54, 309, 105, 124, 134, 331, 55, 306, 294, 21, 179, 342, 299, 302, 187, 265, 272, 305, 165, 92, 182, 174, 13, 140, 354, 35, 166, 341, 301, 154, 374, 270, 201, 139, 51, 344, 81, 147, 36, 15, 295, 175, 97, 121, 63, 206, 183, 259, 349, 171, 285, 30, 146, 17, 69, 225, 2, 84, 230, 197, 114, 275, 145, 122, 376, 38, 283, 327, 371, 195, 184, 245, 210, 257, 339, 296, 364, 3, 279, 253, 352, 383, 152, 368, 322, 340, 276, 261, 338, 94, 103, 186, 217, 214, 77, 68, 192, 32, 37, 191, 359, 300, 319, 170, 235, 57, 194, 100, 356, 358, 363, 329, 366, 164, 137, 52, 216, 73, 193, 62, 80, 236, 155, 221, 160, 29, 351, 125, 27, 284, 59, 129, 28, 335, 132, 297, 0, 254, 336, 205, 61, 362, 169, 382, 128, 149, 312, 286, 23, 248, 22, 337, 268, 237, 251, 136, 101, 4, 181, 131, 142, 1, 239, 25, 372, 85, 163, 288, 196, 95, 108, 378, 233, 75, 377, 316, 350, 110, 244, 249, 156, 96, 229, 203, 144, 24, 209, 48, 332, 250, 263, 190, 56, 8, 228, 234, 215, 67, 328, 208, 109, 200, 282, 347, 310, 311, 293, 273, 241, 86] + 53: [110, 63, 184, 356, 165, 244, 314, 377, 265, 22, 252, 56, 90, 18, 140, 153, 167, 312, 316, 185, 84, 114, 210, 272, 240, 142, 42, 6, 144, 221, 253, 262, 172, 335, 315, 122, 369, 158, 79, 40, 254, 264, 100, 147, 366, 124, 204, 290, 73, 297, 365, 180, 194, 368, 171, 25, 183, 277, 47, 67, 120, 166, 236, 220, 195, 80, 199, 193, 34, 27, 274, 307, 38, 68, 292, 92, 309, 81, 20, 62, 383, 197, 149, 226, 233, 370, 333, 299, 258, 246, 82, 310, 267, 211, 71, 58, 16, 8, 280, 118, 202, 26, 284, 88, 141, 37, 176, 293, 196, 276, 278, 232, 45, 137, 83, 39, 93, 190, 187, 223, 317, 75, 173, 155, 156, 157, 125, 127, 12, 338, 235, 150, 143, 216, 321, 30, 182, 54, 367, 177, 298, 208, 192, 304, 13, 217, 164, 227, 209, 268, 107, 5, 200, 116, 49, 89, 359, 178, 169, 255, 123, 55, 380, 48, 85, 374, 212, 355, 350, 308, 320, 224, 145, 86, 31, 95, 139, 361, 60, 10, 146, 115, 174, 96, 294, 21, 201, 283, 106, 260, 161, 99, 23, 336, 129, 379, 302, 261, 103, 325, 94, 112, 373, 281, 328, 36, 57, 215, 218, 87, 154, 51, 342, 104, 288, 245, 334, 230, 247, 179, 228, 162, 130, 219, 256, 231, 300, 15, 282, 339, 375, 207, 59, 275, 7, 305, 113, 263, 148, 11, 287, 186, 238, 46, 279, 291, 134, 326, 69, 205, 78, 349, 357, 128, 189, 28, 364, 337, 53, 9, 17, 135, 41, 213, 296, 4, 270, 363, 170, 332, 271, 3, 168, 188, 313, 72, 102, 248, 371, 117, 133, 97, 131, 249, 64, 382, 91, 286, 109, 32, 66, 345, 266, 376, 121, 269, 70, 77, 324, 126, 175, 35, 65, 306, 242, 257, 108, 43, 74, 163, 61, 191, 354, 330, 203, 322, 360, 225, 119, 24, 381, 151, 378, 159, 353, 111, 105, 19, 251, 234, 352, 323, 301, 303, 181, 318, 311, 250, 243, 273, 132, 237, 285, 319, 346, 152, 239, 76, 44, 340, 327, 136, 138, 101, 289, 331, 348, 372, 343, 344, 347, 295, 222, 214, 160, 351, 50, 241, 329, 341, 52, 259, 362, 2, 206, 358, 198, 0, 14, 29, 1, 229, 98, 33] + 54: [363, 357, 190, 239, 55, 61, 326, 81, 351, 243, 6, 233, 362, 104, 252, 383, 127, 205, 148, 232, 320, 336, 361, 378, 370, 72, 345, 118, 332, 115, 198, 185, 125, 365, 73, 111, 364, 141, 194, 235, 331, 60, 303, 188, 342, 354, 0, 36, 294, 267, 20, 282, 119, 3, 47, 161, 5, 217, 186, 135, 255, 209, 18, 273, 213, 110, 79, 263, 210, 222, 349, 13, 339, 71, 150, 306, 221, 7, 121, 218, 335, 34, 381, 131, 144, 107, 146, 262, 371, 211, 236, 227, 307, 241, 78, 106, 155, 253, 82, 174, 330, 314, 10, 310, 50, 152, 140, 368, 171, 327, 33, 300, 259, 219, 268, 66, 51, 379, 212, 160, 164, 296, 257, 53, 187, 180, 101, 333, 84, 265, 103, 355, 278, 293, 96, 244, 139, 15, 367, 108, 374, 350, 165, 123, 284, 153, 216, 8, 290, 117, 317, 70, 215, 94, 272, 237, 100, 373, 62, 167, 316, 308, 224, 269, 248, 65, 134, 358, 283, 380, 287, 172, 341, 26, 181, 132, 196, 321, 92, 184, 246, 264, 304, 377, 238, 245, 59, 249, 56, 149, 256, 193, 42, 337, 189, 30, 176, 309, 58, 288, 113, 91, 24, 21, 1, 366, 179, 114, 23, 12, 74, 274, 199, 285, 37, 207, 258, 295, 83, 369, 93, 311, 329, 276, 129, 323, 382, 157, 318, 68, 137, 151, 43, 63, 145, 289, 270, 360, 126, 297, 11, 95, 338, 214, 44, 87, 86, 41, 158, 225, 54, 170, 177, 298, 201, 353, 275, 88, 168, 31, 46, 35, 169, 130, 356, 286, 147, 133, 208, 25, 16, 375, 197, 45, 159, 204, 67, 48, 313, 166, 9, 251, 178, 206, 102, 116, 77, 19, 223, 299, 39, 98, 120, 305, 372, 99, 64, 220, 182, 312, 315, 163, 22, 173, 128, 271, 247, 261, 203, 154, 138, 200, 231, 230, 90, 250, 254, 57, 75, 143, 277, 124, 109, 97, 302, 85, 122, 195, 191, 334, 14, 29, 226, 292, 291, 202, 240, 280, 328, 80, 281, 40, 27, 156, 32, 325, 142, 376, 183, 69, 136, 347, 175, 192, 234, 52, 343, 4, 2, 340, 348, 38, 319, 260, 324, 352, 228, 76, 322, 17, 279, 105, 49, 301, 242, 162, 266, 229, 346, 89, 359, 28, 112, 344] + 55: [154, 16, 247, 49, 99, 300, 337, 348, 231, 361, 122, 157, 149, 127, 365, 168, 58, 248, 155, 244, 351, 186, 215, 34, 174, 7, 227, 363, 134, 44, 26, 163, 375, 178, 140, 207, 87, 232, 373, 336, 275, 261, 175, 362, 148, 51, 69, 305, 56, 113, 234, 307, 304, 293, 331, 271, 112, 105, 265, 54, 318, 85, 31, 188, 356, 243, 66, 242, 141, 108, 167, 283, 202, 326, 4, 111, 382, 223, 320, 0, 156, 276, 378, 306, 332, 52, 101, 133, 367, 55, 104, 369, 169, 142, 229, 310, 88, 76, 309, 238, 78, 249, 272, 138, 17, 325, 236, 343, 257, 358, 314, 342, 15, 372, 50, 145, 181, 36, 345, 109, 81, 115, 151, 311, 73, 40, 226, 368, 189, 350, 217, 42, 263, 277, 302, 237, 253, 30, 107, 374, 282, 334, 24, 285, 98, 269, 355, 218, 211, 301, 12, 164, 259, 170, 380, 47, 119, 287, 214, 153, 97, 321, 315, 324, 330, 21, 100, 196, 20, 29, 256, 89, 192, 2, 179, 213, 172, 46, 377, 246, 10, 23, 313, 1, 219, 126, 125, 64, 364, 195, 292, 144, 11, 130, 235, 357, 267, 200, 71, 216, 344, 91, 371, 187, 241, 171, 296, 103, 37, 297, 354, 13, 92, 110, 379, 250, 146, 180, 270, 62, 328, 291, 176, 366, 206, 284, 251, 79, 19, 346, 264, 82, 268, 319, 150, 68, 191, 266, 9, 45, 308, 323, 43, 370, 121, 70, 333, 86, 233, 352, 27, 240, 184, 128, 63, 198, 359, 35, 120, 139, 295, 32, 158, 204, 340, 74, 327, 182, 41, 132, 279, 349, 5, 102, 224, 360, 322, 255, 117, 212, 225, 93, 137, 162, 280, 83, 59, 222, 116, 347, 28, 39, 303, 106, 3, 290, 197, 14, 289, 286, 185, 254, 22, 61, 335, 201, 338, 183, 77, 288, 25, 205, 80, 48, 152, 165, 353, 199, 8, 312, 383, 190, 210, 53, 228, 84, 294, 160, 143, 118, 161, 177, 298, 57, 230, 317, 381, 90, 159, 252, 273, 65, 60, 316, 124, 274, 329, 135, 194, 278, 33, 258, 173, 38, 129, 166, 262, 75, 131, 245, 123, 299, 72, 339, 6, 341, 114, 221, 239, 95, 260, 67, 136, 193, 94, 281, 208, 209, 147, 203, 376, 220, 18, 96] + 56: [231, 9, 251, 284, 77, 207, 103, 250, 23, 317, 227, 95, 201, 375, 276, 65, 373, 337, 203, 36, 301, 173, 202, 90, 240, 88, 356, 6, 275, 80, 118, 73, 281, 374, 280, 243, 297, 134, 145, 200, 289, 60, 211, 212, 366, 74, 315, 3, 195, 107, 303, 264, 316, 2, 168, 369, 38, 105, 295, 11, 151, 368, 180, 314, 226, 367, 93, 254, 349, 377, 142, 146, 260, 167, 158, 263, 320, 332, 357, 0, 40, 324, 338, 34, 138, 364, 54, 96, 28, 372, 113, 47, 358, 247, 85, 224, 220, 8, 163, 196, 328, 302, 288, 72, 359, 10, 79, 149, 33, 129, 360, 347, 232, 16, 352, 272, 336, 174, 165, 143, 294, 177, 155, 189, 308, 41, 14, 383, 58, 312, 268, 269, 185, 221, 62, 172, 319, 29, 43, 26, 52, 45, 235, 208, 321, 100, 116, 322, 104, 71, 188, 101, 309, 148, 193, 179, 12, 270, 313, 271, 228, 330, 67, 350, 108, 59, 216, 318, 242, 279, 273, 351, 187, 46, 362, 381, 233, 160, 278, 166, 355, 209, 339, 299, 310, 379, 78, 361, 382, 261, 217, 51, 265, 147, 186, 267, 333, 48, 18, 192, 238, 69, 92, 305, 133, 255, 248, 94, 341, 175, 57, 152, 131, 215, 194, 61, 44, 304, 53, 371, 236, 39, 329, 244, 246, 27, 206, 198, 214, 164, 286, 210, 353, 274, 139, 287, 157, 68, 277, 102, 178, 75, 199, 156, 306, 346, 83, 150, 223, 182, 181, 136, 110, 153, 42, 55, 262, 124, 266, 130, 345, 370, 365, 229, 82, 5, 81, 169, 125, 161, 66, 63, 197, 285, 135, 252, 50, 17, 56, 183, 84, 293, 49, 170, 325, 7, 380, 76, 213, 30, 191, 87, 363, 140, 19, 245, 205, 342, 128, 35, 22, 123, 378, 257, 119, 307, 64, 70, 234, 326, 25, 171, 15, 259, 126, 184, 89, 99, 327, 283, 344, 376, 21, 31, 331, 127, 115, 122, 132, 4, 291, 24, 204, 239, 111, 237, 225, 340, 311, 98, 20, 162, 219, 154, 97, 300, 253, 354, 1, 13, 282, 298, 218, 323, 144, 86, 256, 335, 114, 91, 241, 106, 343, 296, 249, 117, 292, 159, 120, 334, 137, 230, 121, 141, 190, 348, 290, 112, 37, 222, 258, 176, 32, 109] + 57: [254, 344, 318, 35, 58, 179, 113, 277, 55, 225, 209, 81, 334, 236, 200, 224, 370, 264, 294, 181, 367, 301, 304, 343, 276, 220, 198, 267, 283, 88, 272, 243, 214, 183, 380, 279, 321, 98, 29, 315, 281, 194, 374, 266, 82, 336, 293, 372, 326, 342, 156, 124, 204, 79, 354, 43, 341, 83, 92, 119, 149, 23, 18, 247, 59, 357, 222, 78, 218, 278, 34, 285, 25, 11, 63, 274, 169, 22, 250, 365, 297, 46, 147, 112, 71, 369, 323, 145, 24, 188, 361, 287, 115, 258, 30, 56, 231, 93, 368, 359, 60, 180, 251, 316, 9, 20, 116, 13, 313, 90, 311, 97, 265, 168, 242, 364, 244, 212, 195, 86, 14, 157, 57, 108, 300, 345, 185, 91, 110, 177, 111, 38, 166, 199, 132, 162, 355, 87, 50, 192, 61, 340, 245, 234, 146, 154, 158, 235, 335, 150, 123, 167, 172, 249, 356, 106, 329, 337, 17, 371, 33, 4, 109, 239, 100, 291, 227, 262, 259, 74, 16, 140, 118, 197, 160, 238, 346, 152, 349, 289, 2, 196, 155, 237, 252, 0, 1, 76, 273, 189, 305, 275, 295, 302, 133, 7, 217, 317, 64, 182, 73, 42, 377, 358, 210, 352, 216, 261, 5, 378, 219, 103, 142, 52, 37, 229, 165, 125, 31, 65, 130, 117, 306, 39, 161, 173, 312, 12, 328, 296, 330, 332, 45, 255, 66, 138, 310, 221, 131, 105, 89, 170, 314, 128, 141, 350, 80, 253, 373, 298, 191, 47, 348, 190, 186, 379, 122, 270, 333, 232, 360, 327, 153, 233, 246, 320, 121, 366, 8, 127, 163, 99, 176, 27, 382, 120, 68, 351, 178, 175, 303, 307, 241, 107, 10, 129, 280, 19, 187, 286, 85, 101, 324, 151, 309, 347, 213, 51, 184, 284, 95, 282, 215, 41, 363, 126, 260, 375, 271, 206, 15, 201, 205, 72, 230, 257, 137, 6, 75, 319, 40, 228, 54, 325, 226, 3, 362, 26, 49, 32, 36, 256, 339, 114, 292, 376, 148, 331, 381, 104, 268, 84, 44, 338, 322, 96, 21, 208, 164, 353, 288, 240, 135, 248, 263, 48, 139, 193, 69, 143, 207, 94, 159, 223, 53, 174, 62, 144, 134, 70, 102, 203, 136, 171, 28, 290, 211, 299, 383, 67, 308, 269, 77, 202] + 58: [377, 3, 71, 125, 319, 321, 335, 370, 10, 108, 150, 168, 184, 280, 364, 201, 204, 340, 33, 38, 58, 134, 209, 308, 220, 178, 341, 12, 190, 187, 239, 82, 333, 63, 300, 43, 113, 8, 296, 359, 290, 349, 193, 74, 212, 158, 292, 309, 81, 344, 289, 199, 18, 16, 106, 198, 251, 62, 250, 110, 241, 21, 347, 237, 138, 100, 44, 51, 216, 174, 313, 279, 245, 79, 322, 355, 175, 6, 104, 242, 380, 25, 284, 53, 131, 253, 65, 19, 230, 142, 372, 46, 84, 304, 217, 291, 156, 61, 373, 67, 232, 47, 132, 197, 281, 244, 181, 339, 302, 144, 371, 265, 133, 170, 303, 157, 369, 286, 310, 165, 275, 76, 101, 351, 36, 85, 130, 352, 229, 207, 59, 124, 346, 231, 123, 115, 180, 243, 86, 276, 96, 15, 301, 147, 126, 151, 271, 358, 112, 141, 191, 4, 28, 314, 266, 160, 378, 154, 288, 35, 94, 294, 221, 282, 39, 32, 140, 91, 56, 249, 14, 72, 116, 342, 31, 20, 328, 55, 383, 183, 149, 357, 161, 182, 338, 5, 317, 337, 57, 34, 218, 361, 366, 77, 210, 22, 368, 324, 272, 316, 73, 202, 350, 83, 0, 172, 228, 11, 179, 360, 87, 374, 109, 145, 323, 89, 305, 248, 211, 69, 169, 295, 189, 274, 224, 381, 143, 227, 37, 176, 327, 50, 105, 24, 54, 254, 117, 153, 343, 233, 205, 263, 348, 9, 256, 45, 277, 60, 285, 375, 258, 234, 382, 186, 107, 225, 118, 297, 163, 354, 120, 159, 363, 23, 92, 114, 27, 49, 40, 196, 97, 260, 329, 331, 129, 155, 135, 13, 278, 195, 173, 102, 128, 336, 246, 137, 30, 320, 98, 298, 273, 299, 26, 68, 367, 315, 66, 48, 215, 238, 240, 148, 334, 283, 219, 213, 264, 146, 167, 17, 127, 80, 194, 122, 259, 235, 270, 376, 42, 111, 307, 139, 236, 306, 75, 379, 318, 293, 226, 64, 268, 41, 223, 52, 188, 164, 311, 312, 7, 93, 214, 262, 203, 255, 185, 1, 325, 326, 121, 208, 345, 287, 136, 261, 330, 200, 247, 78, 267, 166, 222, 177, 356, 206, 257, 152, 119, 162, 171, 192, 99, 90, 332, 362, 2, 70, 353, 29, 95, 103, 269, 88, 365, 252] + 59: [327, 371, 172, 80, 114, 164, 211, 315, 214, 344, 36, 96, 0, 246, 37, 88, 148, 312, 305, 39, 55, 235, 154, 189, 25, 33, 95, 150, 38, 112, 109, 279, 22, 19, 137, 337, 199, 206, 41, 74, 89, 166, 319, 237, 293, 139, 308, 215, 170, 182, 70, 300, 262, 58, 57, 297, 60, 142, 146, 203, 292, 333, 110, 153, 145, 296, 257, 108, 14, 87, 268, 271, 229, 113, 5, 331, 9, 309, 278, 16, 383, 245, 222, 238, 349, 48, 149, 322, 326, 106, 102, 8, 316, 77, 378, 341, 7, 73, 233, 130, 372, 266, 91, 168, 169, 29, 179, 92, 131, 32, 64, 253, 359, 249, 307, 301, 98, 125, 78, 117, 140, 332, 147, 291, 186, 281, 45, 160, 163, 248, 356, 180, 155, 61, 71, 34, 21, 132, 275, 358, 350, 121, 190, 4, 209, 65, 226, 221, 324, 382, 176, 35, 302, 373, 47, 198, 365, 107, 299, 184, 217, 210, 11, 103, 360, 375, 116, 351, 224, 234, 197, 294, 295, 243, 212, 254, 280, 252, 10, 258, 101, 2, 43, 129, 67, 183, 191, 284, 192, 56, 122, 167, 250, 352, 97, 263, 185, 178, 128, 40, 127, 174, 306, 348, 289, 242, 138, 187, 330, 277, 273, 82, 141, 171, 126, 15, 379, 346, 313, 311, 18, 377, 342, 175, 181, 225, 338, 369, 361, 362, 194, 208, 28, 93, 357, 368, 239, 244, 152, 90, 370, 66, 30, 251, 213, 26, 196, 81, 62, 320, 158, 69, 353, 156, 380, 84, 134, 336, 119, 216, 276, 264, 345, 188, 94, 220, 75, 204, 76, 328, 290, 223, 100, 376, 236, 339, 228, 51, 193, 50, 17, 3, 201, 283, 282, 367, 83, 111, 195, 340, 285, 374, 42, 317, 133, 161, 159, 218, 329, 177, 259, 68, 27, 135, 207, 49, 321, 165, 318, 205, 347, 79, 202, 200, 46, 247, 31, 151, 255, 85, 334, 162, 355, 120, 6, 314, 287, 231, 157, 44, 118, 241, 219, 270, 72, 323, 260, 173, 105, 298, 265, 144, 86, 304, 267, 143, 54, 366, 354, 123, 63, 325, 343, 261, 53, 303, 1, 272, 104, 310, 59, 363, 256, 381, 288, 23, 232, 12, 364, 20, 286, 269, 13, 274, 227, 136, 124, 24, 115, 230, 240, 335, 99, 52] + 60: [313, 309, 161, 253, 248, 93, 11, 305, 90, 66, 369, 371, 363, 287, 216, 111, 249, 71, 77, 62, 154, 288, 22, 355, 124, 34, 119, 232, 364, 150, 37, 80, 342, 53, 273, 314, 225, 383, 324, 261, 298, 306, 138, 87, 157, 207, 375, 201, 351, 167, 208, 337, 15, 203, 359, 255, 269, 9, 377, 21, 33, 95, 368, 270, 251, 145, 211, 326, 177, 109, 214, 348, 123, 347, 120, 74, 290, 206, 271, 152, 114, 92, 38, 134, 196, 182, 219, 307, 252, 54, 378, 292, 275, 10, 373, 299, 257, 243, 222, 181, 294, 112, 131, 169, 84, 358, 88, 56, 233, 91, 130, 256, 282, 361, 42, 68, 301, 73, 218, 317, 16, 170, 39, 65, 234, 180, 278, 172, 336, 6, 332, 304, 346, 200, 192, 286, 105, 13, 258, 29, 350, 4, 44, 63, 268, 121, 343, 247, 28, 353, 213, 235, 75, 221, 344, 3, 25, 142, 100, 296, 148, 67, 328, 374, 175, 17, 173, 210, 381, 236, 244, 230, 2, 49, 98, 59, 140, 283, 240, 264, 26, 31, 30, 349, 315, 202, 320, 224, 291, 72, 103, 101, 158, 94, 303, 194, 155, 319, 354, 223, 241, 362, 259, 267, 176, 52, 187, 370, 64, 125, 330, 198, 229, 43, 227, 308, 220, 102, 266, 316, 297, 76, 365, 245, 263, 20, 183, 237, 274, 70, 285, 322, 380, 376, 45, 280, 356, 135, 289, 89, 47, 284, 115, 212, 185, 239, 311, 117, 55, 69, 318, 5, 61, 141, 7, 341, 12, 333, 178, 19, 97, 254, 367, 136, 139, 310, 195, 48, 334, 108, 166, 272, 82, 277, 1, 382, 279, 107, 85, 357, 14, 338, 127, 41, 160, 86, 295, 118, 60, 265, 129, 78, 302, 327, 329, 231, 99, 189, 132, 24, 8, 360, 293, 106, 144, 156, 262, 50, 379, 159, 340, 321, 126, 174, 260, 113, 312, 205, 193, 184, 27, 151, 51, 162, 57, 171, 149, 79, 276, 168, 23, 104, 204, 165, 32, 352, 83, 40, 335, 339, 153, 366, 345, 143, 238, 58, 217, 110, 215, 323, 250, 300, 331, 197, 209, 137, 147, 281, 163, 226, 133, 199, 128, 372, 81, 35, 46, 122, 246, 0, 36, 188, 116, 186, 146, 190, 18, 325, 228, 179, 96, 191, 164, 242] + 61: [221, 178, 353, 119, 333, 243, 224, 366, 47, 23, 38, 273, 295, 132, 130, 91, 13, 201, 35, 254, 76, 116, 319, 58, 127, 205, 45, 294, 291, 31, 92, 316, 121, 245, 290, 153, 242, 26, 359, 63, 123, 185, 146, 279, 41, 343, 155, 0, 179, 108, 117, 218, 143, 34, 282, 65, 83, 379, 97, 173, 9, 22, 14, 106, 101, 99, 138, 339, 382, 183, 356, 149, 338, 360, 323, 21, 215, 177, 133, 258, 29, 2, 368, 137, 214, 196, 71, 193, 54, 24, 314, 56, 60, 376, 171, 208, 61, 303, 165, 1, 68, 327, 349, 337, 90, 252, 69, 249, 78, 365, 67, 81, 244, 163, 111, 272, 318, 277, 229, 40, 73, 231, 351, 380, 371, 288, 181, 228, 233, 113, 3, 102, 292, 125, 240, 124, 274, 310, 331, 305, 347, 281, 217, 357, 93, 107, 247, 184, 51, 192, 159, 75, 297, 82, 308, 202, 378, 37, 79, 284, 152, 301, 358, 265, 105, 131, 62, 96, 211, 216, 326, 264, 278, 66, 42, 43, 150, 27, 260, 147, 139, 306, 141, 317, 352, 271, 246, 293, 220, 377, 16, 296, 46, 88, 170, 209, 372, 112, 286, 261, 77, 190, 336, 200, 259, 174, 126, 342, 148, 255, 7, 95, 222, 226, 266, 334, 299, 86, 157, 59, 12, 238, 110, 160, 348, 114, 374, 207, 72, 346, 340, 4, 311, 239, 49, 180, 276, 64, 320, 128, 197, 161, 172, 270, 17, 142, 115, 169, 191, 300, 304, 328, 154, 30, 283, 194, 298, 6, 251, 164, 250, 175, 50, 189, 381, 307, 230, 241, 330, 198, 168, 309, 20, 84, 109, 52, 186, 341, 219, 204, 98, 364, 74, 182, 236, 355, 267, 321, 100, 263, 5, 10, 363, 55, 188, 227, 361, 268, 257, 206, 89, 275, 85, 199, 345, 25, 232, 212, 256, 39, 134, 94, 36, 248, 136, 203, 103, 213, 57, 156, 362, 166, 313, 158, 33, 375, 225, 144, 370, 354, 312, 28, 140, 151, 367, 53, 167, 324, 335, 122, 253, 237, 129, 15, 118, 135, 329, 145, 383, 8, 80, 332, 70, 187, 262, 373, 302, 285, 325, 350, 315, 344, 120, 19, 195, 235, 369, 32, 87, 44, 11, 104, 322, 280, 269, 176, 162, 18, 234, 48, 223, 289, 210, 287] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots416.yaml b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots416.yaml new file mode 100644 index 000000000000..de20f38b0a6b --- /dev/null +++ b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep16_slots416.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [259, 86, 264, 332, 312, 29, 232, 198, 173, 205, 156, 10, 94, 38, 133, 4, 216, 97, 326, 98, 294, 191, 285, 271, 323, 58, 166, 225, 135, 45, 188, 95, 184, 68, 144, 141, 11, 361, 208, 77, 244, 335, 192, 100, 328, 286, 305, 334, 220, 23, 325, 201, 166, 225, 235, 190, 182, 242, 321, 276, 3, 54, 313, 265, 170, 268, 85, 240, 372, 195, 82, 342, 253, 353, 164, 18, 88, 370, 0, 383, 135, 346, 35, 80, 327, 176, 209, 55, 105, 129, 261, 358, 308, 103, 19, 215, 237, 154, 40, 219, 266, 92, 355, 247, 256, 371, 254, 378, 163, 95, 134, 16, 330, 181, 43, 57, 138, 234, 223, 365, 178, 25, 73, 50, 340, 380, 125, 132, 128, 373, 256, 371, 111, 158, 122, 62, 289, 318, 330, 33, 131, 311, 300, 160, 87, 331, 140, 238, 67, 102, 338, 179, 357, 189, 273, 13, 150, 245, 111, 45, 169, 218, 184, 37, 290, 284, 196, 36, 7, 123, 39, 146, 269, 320, 127, 6, 212, 53, 251, 298, 364, 274, 151, 245, 254, 230, 60, 30, 65, 63, 290, 243, 107, 258, 112, 314, 187, 136, 99, 296, 263, 252, 352, 113, 194, 287, 130, 277, 74, 236, 165, 279, 81, 79, 255, 217, 282, 336, 34, 297, 337, 149, 257, 213, 366, 106, 147, 20, 96, 2, 301, 304, 168, 118, 74, 236, 165, 279, 153, 202, 48, 203, 126, 360, 56, 354, 114, 199, 5, 108, 246, 379, 382, 47, 316, 69, 319, 329, 288, 310, 71, 345, 111, 241, 26, 104, 376, 217, 280, 143, 101, 174, 324, 375, 172, 227, 59, 1, 267, 229, 161, 152, 381, 226, 291, 185, 222, 322, 177, 378, 295, 17, 49, 278, 224, 72, 283, 186, 148, 31, 32, 214, 369, 221, 197, 145, 42, 41, 239, 24, 228, 303, 222, 119, 348, 378, 51, 17, 49, 293, 224, 307, 341, 90, 248, 89, 52, 137, 363, 270, 299, 260, 64, 175, 367, 121, 281, 61, 302, 159, 165, 193, 21, 79, 70, 139, 272, 359, 183, 124, 44, 27, 66, 249, 116, 211, 14, 210, 115, 306, 9, 142, 344, 120, 351, 84, 162, 207, 60, 204, 343, 203, 349, 22, 117, 200, 368, 262, 377, 46, 75, 292, 83, 362, 206, 315, 250, 167, 339, 110, 356, 356, 155, 76, 347, 157, 134, 333, 173, 78, 233, 109, 28, 309, 91, 317, 275, 180, 93, 350, 15, 12, 231, 374, 8, 171] + 1: [134, 80, 72, 291, 319, 366, 56, 120, 243, 275, 266, 197, 219, 171, 101, 164, 107, 184, 293, 13, 262, 372, 90, 124, 337, 15, 178, 158, 72, 167, 347, 328, 3, 185, 272, 71, 234, 335, 282, 88, 95, 204, 364, 136, 36, 323, 77, 357, 221, 322, 359, 363, 380, 106, 91, 163, 21, 377, 58, 153, 138, 287, 355, 108, 4, 38, 133, 170, 139, 229, 315, 94, 353, 129, 316, 374, 117, 327, 141, 158, 149, 163, 319, 5, 198, 111, 224, 173, 285, 288, 206, 306, 237, 348, 20, 296, 51, 268, 226, 371, 253, 132, 341, 255, 251, 44, 91, 169, 347, 41, 63, 82, 383, 242, 125, 379, 93, 203, 189, 7, 332, 68, 48, 97, 356, 220, 340, 64, 307, 215, 81, 180, 50, 177, 168, 325, 244, 23, 320, 86, 317, 218, 52, 69, 102, 259, 370, 154, 35, 104, 27, 130, 373, 343, 28, 112, 200, 180, 85, 378, 6, 0, 182, 271, 273, 37, 301, 194, 143, 239, 147, 330, 65, 156, 26, 346, 344, 369, 151, 66, 304, 368, 365, 137, 43, 169, 319, 377, 182, 111, 59, 269, 212, 109, 116, 358, 256, 222, 161, 295, 350, 302, 49, 148, 217, 128, 159, 336, 365, 18, 277, 254, 192, 73, 105, 24, 46, 32, 78, 166, 176, 289, 76, 121, 326, 62, 290, 352, 103, 83, 135, 361, 157, 34, 1, 118, 277, 329, 39, 210, 300, 225, 29, 211, 274, 187, 228, 381, 298, 122, 17, 19, 230, 96, 119, 246, 188, 252, 339, 265, 235, 137, 382, 127, 67, 110, 334, 82, 25, 89, 245, 280, 53, 360, 98, 362, 14, 165, 87, 8, 232, 297, 160, 16, 258, 100, 42, 22, 263, 196, 67, 123, 3, 233, 283, 247, 216, 142, 214, 376, 60, 140, 286, 321, 186, 181, 240, 309, 193, 45, 144, 199, 42, 308, 150, 167, 131, 270, 198, 248, 292, 9, 74, 162, 47, 264, 205, 313, 318, 57, 172, 311, 314, 375, 207, 208, 354, 267, 201, 31, 152, 177, 241, 70, 294, 351, 292, 145, 99, 84, 223, 342, 61, 79, 2, 115, 11, 113, 54, 279, 183, 10, 257, 310, 201, 22, 382, 127, 30, 179, 120, 250, 272, 260, 281, 92, 114, 278, 236, 238, 338, 175, 284, 227, 333, 331, 367, 191, 276, 312, 174, 174, 126, 163, 40, 75, 299, 225, 195, 303, 146, 261, 213, 202, 231, 305, 249, 33, 324, 55, 12, 190, 209, 155, 349, 345] + 2: [53, 115, 28, 212, 192, 110, 70, 42, 51, 200, 174, 170, 67, 218, 16, 169, 12, 29, 334, 261, 107, 205, 222, 13, 208, 375, 53, 115, 223, 143, 101, 155, 73, 184, 314, 279, 196, 18, 156, 120, 349, 276, 364, 104, 346, 319, 288, 8, 310, 289, 301, 379, 43, 93, 298, 75, 242, 168, 91, 306, 324, 235, 311, 87, 50, 187, 241, 309, 49, 94, 275, 231, 292, 98, 117, 380, 32, 366, 43, 93, 1, 197, 232, 97, 173, 105, 139, 274, 294, 140, 92, 76, 360, 116, 9, 257, 367, 264, 370, 335, 345, 214, 202, 363, 180, 102, 5, 125, 19, 191, 166, 304, 45, 217, 79, 299, 327, 201, 65, 15, 305, 152, 195, 141, 226, 158, 378, 266, 316, 381, 230, 194, 295, 151, 185, 284, 91, 135, 122, 82, 6, 7, 262, 198, 163, 250, 106, 150, 47, 178, 371, 338, 147, 317, 354, 255, 96, 194, 81, 248, 60, 322, 355, 172, 139, 328, 165, 344, 320, 128, 277, 254, 136, 365, 238, 376, 373, 114, 336, 315, 358, 258, 132, 89, 295, 143, 290, 84, 377, 44, 368, 244, 323, 263, 27, 361, 308, 160, 154, 176, 77, 352, 329, 307, 234, 260, 337, 229, 132, 343, 259, 212, 162, 111, 377, 86, 22, 11, 285, 273, 68, 383, 10, 193, 142, 2, 36, 331, 95, 183, 37, 353, 282, 300, 112, 14, 189, 100, 19, 124, 66, 118, 122, 24, 30, 347, 55, 20, 145, 348, 252, 23, 211, 144, 291, 267, 283, 249, 270, 190, 112, 89, 3, 247, 290, 62, 56, 245, 26, 146, 61, 206, 210, 58, 253, 103, 38, 268, 4, 272, 99, 251, 227, 240, 339, 199, 108, 215, 137, 248, 213, 109, 286, 123, 304, 340, 342, 90, 269, 236, 41, 186, 39, 179, 134, 359, 216, 48, 113, 64, 228, 330, 108, 215, 321, 248, 232, 62, 280, 86, 325, 85, 303, 224, 293, 21, 171, 175, 121, 219, 239, 233, 302, 59, 382, 88, 287, 40, 225, 357, 221, 34, 149, 153, 278, 133, 314, 63, 209, 17, 188, 138, 332, 129, 256, 318, 35, 52, 54, 164, 72, 127, 119, 161, 341, 313, 3, 143, 78, 350, 131, 271, 296, 333, 159, 356, 126, 148, 265, 181, 312, 0, 25, 207, 71, 167, 177, 369, 237, 246, 351, 313, 223, 33, 242, 168, 157, 220, 31, 83, 204, 130, 203, 297, 46, 69, 372, 362, 281, 57, 243, 74, 182, 80, 326, 374] + 3: [39, 377, 314, 317, 62, 334, 343, 250, 195, 12, 142, 295, 218, 217, 255, 30, 43, 106, 132, 309, 16, 156, 89, 352, 9, 233, 39, 230, 302, 225, 80, 50, 238, 122, 212, 264, 78, 374, 274, 235, 280, 214, 286, 294, 130, 283, 378, 242, 90, 315, 5, 278, 22, 377, 209, 51, 24, 42, 337, 336, 316, 64, 161, 262, 345, 327, 266, 125, 340, 91, 381, 285, 0, 338, 148, 23, 226, 222, 22, 377, 302, 31, 151, 137, 186, 2, 182, 11, 136, 297, 103, 53, 355, 276, 249, 70, 282, 362, 73, 107, 253, 152, 173, 38, 66, 310, 171, 350, 348, 213, 26, 358, 219, 25, 271, 114, 224, 256, 44, 268, 298, 291, 72, 153, 251, 45, 141, 60, 33, 303, 66, 275, 18, 163, 261, 145, 207, 211, 304, 202, 162, 312, 176, 57, 273, 100, 187, 281, 346, 269, 116, 234, 127, 124, 247, 324, 66, 260, 319, 163, 263, 158, 178, 189, 77, 357, 109, 111, 126, 380, 48, 383, 322, 367, 356, 133, 179, 88, 330, 320, 371, 292, 66, 260, 319, 13, 313, 81, 102, 231, 138, 382, 84, 32, 181, 197, 65, 328, 184, 67, 110, 28, 134, 121, 1, 265, 332, 198, 66, 260, 18, 131, 97, 180, 339, 10, 221, 118, 8, 351, 98, 94, 105, 237, 353, 7, 123, 206, 183, 140, 75, 272, 52, 257, 66, 35, 365, 19, 157, 150, 168, 58, 36, 119, 279, 243, 299, 166, 129, 160, 306, 376, 85, 293, 372, 284, 147, 349, 17, 146, 71, 310, 14, 27, 333, 252, 201, 325, 296, 240, 154, 229, 300, 270, 135, 82, 361, 305, 228, 321, 323, 360, 205, 244, 185, 311, 46, 310, 40, 192, 3, 363, 170, 37, 287, 87, 364, 92, 204, 55, 307, 144, 248, 15, 373, 258, 354, 369, 241, 347, 193, 47, 34, 34, 40, 3, 194, 318, 120, 227, 239, 29, 289, 203, 375, 344, 117, 366, 20, 188, 68, 86, 220, 210, 95, 308, 335, 199, 34, 34, 319, 27, 263, 49, 63, 267, 74, 21, 99, 277, 41, 236, 246, 104, 216, 83, 165, 93, 190, 159, 143, 69, 167, 59, 34, 79, 365, 56, 313, 6, 139, 164, 174, 290, 359, 149, 155, 112, 169, 196, 115, 232, 61, 254, 245, 191, 54, 175, 341, 301, 34, 79, 365, 208, 194, 331, 215, 342, 177, 223, 172, 288, 108, 200, 113, 76, 96, 128, 259, 101, 379, 329, 368, 4, 326, 370] + 4: [188, 331, 371, 16, 273, 254, 274, 145, 196, 73, 207, 324, 85, 185, 131, 135, 294, 29, 291, 111, 378, 163, 238, 299, 242, 63, 34, 331, 214, 15, 150, 6, 345, 106, 83, 28, 7, 116, 360, 60, 46, 25, 306, 157, 146, 147, 115, 123, 293, 212, 337, 277, 340, 331, 315, 226, 5, 136, 36, 372, 32, 68, 153, 283, 307, 323, 132, 310, 292, 175, 170, 211, 21, 48, 102, 301, 119, 282, 3, 343, 320, 228, 117, 254, 197, 76, 57, 204, 35, 264, 256, 341, 64, 94, 39, 234, 289, 130, 10, 1, 265, 154, 208, 332, 3, 331, 219, 19, 105, 148, 59, 41, 151, 251, 317, 99, 65, 14, 261, 137, 84, 184, 210, 355, 381, 380, 245, 326, 192, 305, 3, 331, 315, 281, 319, 348, 86, 275, 160, 333, 260, 276, 53, 359, 216, 158, 66, 78, 140, 109, 129, 112, 11, 373, 20, 120, 18, 343, 250, 95, 319, 302, 253, 200, 101, 49, 369, 221, 330, 297, 313, 240, 349, 26, 122, 223, 203, 353, 202, 363, 67, 321, 209, 343, 350, 88, 87, 308, 155, 325, 43, 27, 177, 335, 168, 33, 74, 284, 178, 206, 181, 268, 97, 374, 224, 190, 227, 182, 209, 343, 259, 95, 257, 143, 47, 77, 218, 312, 82, 243, 356, 62, 232, 354, 50, 176, 334, 213, 186, 247, 366, 195, 17, 90, 271, 296, 371, 172, 273, 148, 329, 113, 357, 161, 179, 54, 290, 379, 383, 100, 267, 303, 327, 40, 215, 191, 164, 270, 128, 336, 271, 300, 214, 23, 171, 241, 342, 318, 364, 344, 159, 244, 166, 149, 108, 322, 52, 263, 118, 286, 81, 173, 144, 89, 225, 351, 271, 343, 24, 376, 194, 142, 56, 61, 361, 280, 110, 0, 72, 93, 229, 328, 370, 346, 304, 156, 126, 162, 368, 174, 233, 104, 37, 296, 214, 15, 117, 152, 298, 139, 235, 217, 38, 183, 133, 352, 70, 69, 309, 103, 295, 31, 96, 362, 248, 285, 2, 107, 12, 222, 23, 9, 194, 142, 377, 246, 365, 198, 141, 278, 252, 169, 125, 288, 220, 51, 180, 71, 205, 316, 134, 266, 231, 58, 375, 258, 315, 226, 45, 152, 167, 347, 44, 339, 187, 358, 189, 98, 262, 80, 92, 121, 249, 201, 311, 279, 30, 236, 79, 287, 124, 199, 193, 239, 87, 165, 13, 4, 314, 8, 22, 42, 114, 75, 269, 367, 138, 230, 237, 127, 255, 55, 91, 382, 272, 338] + 5: [172, 160, 346, 269, 30, 231, 352, 192, 66, 264, 69, 175, 144, 380, 362, 150, 237, 191, 283, 156, 261, 73, 357, 367, 176, 44, 208, 194, 346, 305, 381, 78, 255, 182, 327, 268, 318, 377, 104, 274, 201, 138, 254, 349, 323, 244, 86, 111, 350, 110, 83, 50, 61, 214, 49, 265, 272, 41, 130, 74, 209, 163, 319, 220, 33, 183, 230, 325, 149, 162, 221, 40, 224, 307, 294, 117, 277, 303, 61, 5, 60, 276, 52, 231, 200, 293, 155, 2, 181, 324, 99, 93, 343, 289, 334, 229, 103, 342, 290, 53, 282, 116, 278, 75, 61, 194, 355, 169, 269, 372, 252, 147, 8, 174, 184, 258, 28, 216, 239, 161, 80, 361, 129, 270, 292, 185, 328, 112, 142, 368, 165, 214, 65, 305, 356, 259, 132, 364, 304, 222, 251, 315, 301, 39, 20, 76, 9, 77, 374, 298, 373, 267, 202, 248, 164, 127, 154, 214, 98, 100, 288, 211, 212, 247, 137, 59, 67, 320, 79, 51, 366, 136, 48, 94, 113, 168, 32, 139, 336, 339, 326, 177, 190, 296, 217, 262, 287, 64, 145, 335, 4, 332, 140, 152, 295, 312, 121, 341, 119, 232, 263, 107, 347, 120, 36, 273, 157, 286, 190, 308, 260, 223, 272, 379, 17, 246, 313, 369, 206, 333, 195, 266, 240, 63, 383, 55, 235, 92, 297, 85, 234, 226, 167, 321, 141, 296, 217, 105, 6, 159, 7, 331, 70, 151, 3, 34, 35, 242, 382, 219, 87, 29, 205, 316, 95, 126, 146, 0, 96, 23, 213, 256, 355, 100, 365, 259, 279, 198, 173, 15, 207, 153, 228, 329, 300, 241, 275, 62, 363, 58, 148, 197, 340, 280, 166, 170, 213, 84, 81, 105, 288, 102, 215, 25, 371, 171, 24, 314, 115, 253, 353, 90, 330, 271, 378, 68, 189, 285, 123, 180, 57, 196, 213, 84, 47, 276, 52, 122, 43, 91, 233, 375, 204, 218, 54, 19, 56, 284, 360, 344, 299, 135, 125, 309, 42, 109, 179, 118, 249, 128, 178, 193, 287, 21, 351, 338, 124, 358, 38, 26, 46, 345, 250, 257, 281, 376, 359, 187, 322, 97, 158, 310, 82, 337, 249, 306, 355, 370, 6, 159, 88, 89, 131, 238, 108, 210, 186, 12, 106, 317, 236, 291, 114, 18, 101, 133, 311, 27, 348, 302, 128, 128, 1, 188, 356, 21, 72, 22, 31, 37, 143, 45, 16, 227, 14, 203, 13, 134, 71, 199, 245, 11, 243, 10, 225, 354] + 6: [322, 376, 32, 46, 166, 307, 100, 196, 165, 185, 30, 44, 15, 110, 225, 248, 50, 334, 74, 152, 47, 78, 26, 20, 252, 342, 322, 376, 32, 99, 233, 359, 293, 34, 301, 377, 49, 107, 58, 262, 217, 362, 105, 268, 285, 62, 348, 374, 91, 200, 214, 323, 322, 376, 53, 240, 67, 175, 112, 289, 234, 41, 12, 282, 154, 361, 4, 355, 48, 272, 364, 251, 266, 250, 29, 61, 75, 90, 146, 308, 267, 341, 209, 378, 368, 184, 300, 336, 190, 221, 286, 87, 82, 372, 333, 299, 366, 290, 117, 131, 213, 265, 153, 257, 76, 308, 343, 281, 177, 375, 57, 155, 156, 236, 111, 139, 98, 22, 23, 14, 199, 125, 379, 85, 17, 222, 227, 191, 167, 71, 76, 308, 27, 55, 180, 216, 187, 188, 354, 115, 231, 327, 280, 6, 86, 254, 18, 54, 73, 226, 256, 356, 127, 382, 186, 315, 76, 308, 249, 253, 66, 318, 88, 255, 113, 179, 279, 133, 220, 69, 243, 138, 83, 70, 39, 106, 51, 339, 192, 72, 65, 247, 297, 5, 174, 258, 371, 9, 84, 208, 95, 128, 304, 144, 296, 206, 330, 261, 346, 381, 235, 150, 21, 38, 7, 218, 363, 331, 305, 274, 160, 99, 166, 163, 367, 287, 270, 369, 317, 211, 332, 137, 63, 316, 147, 349, 357, 203, 303, 263, 92, 228, 132, 161, 305, 274, 32, 229, 326, 13, 232, 319, 176, 109, 68, 114, 324, 207, 284, 329, 224, 8, 130, 80, 351, 238, 140, 353, 149, 164, 97, 142, 258, 28, 233, 260, 119, 118, 277, 352, 311, 219, 365, 271, 335, 122, 3, 215, 312, 337, 162, 313, 93, 273, 25, 320, 97, 59, 174, 269, 205, 181, 94, 291, 0, 380, 101, 77, 264, 242, 197, 145, 124, 16, 310, 360, 294, 89, 141, 170, 135, 325, 97, 295, 343, 240, 120, 350, 246, 79, 151, 245, 223, 64, 136, 104, 116, 126, 198, 298, 244, 288, 338, 302, 171, 2, 314, 373, 97, 274, 103, 11, 177, 45, 283, 129, 259, 230, 158, 169, 10, 43, 276, 81, 193, 42, 292, 328, 102, 183, 1, 172, 204, 35, 212, 142, 306, 253, 66, 36, 56, 178, 370, 278, 202, 383, 143, 237, 60, 194, 168, 309, 134, 108, 195, 201, 19, 340, 33, 40, 142, 142, 53, 281, 31, 173, 189, 239, 345, 182, 37, 321, 157, 210, 159, 148, 358, 96, 123, 347, 24, 121, 241, 344, 52, 275] + 7: [163, 361, 375, 265, 18, 188, 77, 82, 278, 288, 377, 300, 16, 185, 380, 67, 373, 203, 35, 33, 123, 78, 246, 138, 279, 171, 341, 64, 28, 142, 105, 212, 104, 313, 23, 2, 62, 90, 109, 92, 102, 121, 251, 351, 216, 75, 254, 128, 299, 127, 376, 50, 327, 263, 176, 375, 18, 97, 174, 47, 349, 339, 177, 132, 234, 178, 305, 232, 283, 366, 209, 363, 111, 381, 150, 205, 223, 257, 327, 263, 187, 142, 280, 329, 244, 217, 289, 325, 221, 214, 259, 107, 86, 134, 161, 52, 151, 112, 173, 124, 304, 331, 160, 365, 327, 64, 260, 46, 25, 298, 281, 11, 29, 319, 117, 98, 89, 186, 83, 226, 247, 101, 318, 295, 88, 334, 45, 12, 145, 371, 175, 308, 292, 343, 25, 369, 302, 183, 31, 148, 219, 264, 311, 136, 335, 306, 309, 193, 115, 36, 312, 315, 95, 197, 240, 91, 175, 108, 49, 271, 103, 354, 61, 307, 248, 270, 170, 243, 189, 32, 322, 323, 146, 116, 222, 207, 39, 317, 147, 172, 154, 357, 162, 192, 292, 342, 99, 298, 256, 164, 262, 356, 208, 330, 204, 20, 137, 353, 68, 149, 268, 69, 367, 159, 195, 199, 355, 42, 157, 192, 133, 343, 13, 15, 370, 100, 125, 84, 21, 333, 368, 129, 211, 155, 261, 22, 126, 194, 80, 277, 66, 290, 326, 202, 272, 336, 239, 340, 71, 362, 94, 324, 55, 250, 210, 167, 24, 5, 6, 51, 40, 225, 106, 286, 379, 3, 332, 143, 156, 110, 182, 275, 41, 265, 105, 328, 0, 267, 74, 296, 255, 206, 9, 190, 58, 141, 196, 345, 166, 287, 43, 252, 346, 54, 122, 364, 182, 275, 350, 229, 139, 378, 181, 294, 168, 360, 224, 158, 140, 130, 293, 310, 291, 87, 269, 316, 96, 266, 198, 282, 34, 338, 7, 274, 292, 340, 201, 118, 27, 215, 4, 284, 81, 344, 26, 358, 44, 152, 227, 153, 184, 1, 10, 297, 57, 352, 179, 135, 7, 238, 49, 382, 253, 362, 131, 249, 285, 374, 70, 372, 245, 237, 200, 53, 233, 120, 320, 169, 231, 303, 230, 165, 236, 348, 7, 258, 41, 229, 103, 48, 114, 113, 17, 337, 213, 191, 314, 19, 220, 301, 65, 38, 37, 72, 59, 63, 242, 347, 30, 276, 274, 274, 49, 382, 201, 369, 73, 14, 119, 235, 383, 241, 321, 180, 273, 56, 8, 93, 79, 218, 85, 228, 359, 144, 76, 60] + 8: [273, 21, 201, 10, 272, 182, 30, 39, 346, 147, 165, 368, 130, 60, 206, 116, 310, 16, 339, 347, 365, 270, 26, 139, 222, 100, 253, 338, 264, 362, 353, 204, 252, 374, 366, 333, 169, 309, 192, 358, 187, 249, 350, 143, 265, 322, 277, 370, 123, 138, 127, 163, 253, 302, 264, 201, 371, 337, 18, 294, 356, 133, 215, 219, 320, 42, 117, 191, 352, 199, 144, 54, 323, 208, 268, 94, 288, 234, 253, 106, 108, 62, 62, 371, 44, 280, 128, 119, 121, 189, 334, 8, 59, 90, 297, 313, 378, 14, 287, 158, 319, 301, 345, 207, 104, 135, 367, 321, 152, 229, 251, 86, 311, 231, 298, 194, 197, 332, 98, 28, 299, 372, 85, 175, 111, 171, 286, 51, 0, 102, 214, 135, 161, 321, 64, 229, 95, 168, 75, 283, 17, 198, 120, 52, 212, 247, 373, 68, 79, 29, 357, 244, 329, 82, 124, 96, 5, 31, 7, 241, 154, 74, 157, 377, 188, 109, 275, 112, 300, 284, 230, 87, 41, 136, 200, 93, 33, 348, 305, 355, 176, 262, 5, 238, 367, 243, 221, 81, 20, 148, 22, 23, 40, 278, 267, 4, 101, 248, 32, 126, 235, 375, 351, 271, 343, 142, 210, 376, 250, 31, 359, 336, 353, 36, 223, 115, 166, 258, 216, 137, 317, 61, 263, 232, 110, 314, 35, 260, 220, 324, 363, 80, 289, 58, 3, 67, 359, 12, 152, 228, 209, 11, 83, 6, 261, 183, 211, 125, 118, 237, 382, 266, 325, 185, 134, 340, 70, 43, 57, 213, 279, 67, 21, 362, 282, 204, 255, 97, 172, 226, 78, 167, 141, 91, 335, 257, 149, 155, 205, 9, 153, 103, 354, 342, 55, 19, 360, 290, 13, 312, 88, 182, 291, 195, 107, 38, 65, 303, 164, 114, 254, 25, 47, 307, 159, 306, 146, 84, 285, 331, 242, 63, 360, 246, 239, 173, 308, 281, 316, 328, 150, 27, 178, 304, 233, 66, 45, 71, 151, 56, 99, 369, 326, 129, 269, 344, 49, 132, 360, 218, 239, 162, 308, 122, 186, 131, 181, 179, 202, 196, 177, 341, 174, 73, 259, 327, 145, 236, 76, 193, 160, 203, 296, 227, 360, 292, 359, 10, 221, 276, 349, 364, 69, 34, 92, 50, 318, 2, 383, 293, 184, 140, 105, 46, 315, 77, 180, 274, 256, 170, 290, 290, 224, 162, 381, 281, 380, 156, 225, 240, 330, 113, 48, 72, 217, 379, 37, 190, 89, 295, 24, 15, 245, 53, 361, 1] + 9: [212, 94, 94, 185, 316, 370, 51, 56, 367, 302, 180, 192, 336, 150, 319, 248, 382, 291, 110, 216, 59, 34, 361, 85, 156, 57, 92, 226, 252, 184, 201, 380, 103, 136, 144, 137, 231, 183, 123, 70, 202, 175, 52, 112, 286, 221, 347, 225, 366, 40, 362, 234, 92, 226, 299, 47, 29, 281, 95, 342, 133, 153, 0, 87, 267, 181, 237, 65, 245, 2, 320, 21, 178, 58, 118, 204, 122, 168, 92, 372, 172, 47, 37, 210, 95, 132, 82, 187, 277, 99, 186, 72, 343, 354, 100, 203, 301, 229, 346, 108, 176, 200, 250, 26, 92, 260, 19, 179, 90, 90, 64, 282, 285, 377, 334, 129, 269, 141, 39, 276, 46, 62, 12, 333, 337, 326, 223, 149, 27, 236, 78, 280, 215, 185, 313, 262, 121, 23, 161, 292, 247, 314, 16, 36, 288, 307, 197, 213, 375, 38, 335, 243, 266, 194, 157, 369, 304, 315, 74, 222, 272, 49, 271, 102, 131, 356, 54, 311, 139, 124, 75, 235, 117, 358, 71, 270, 167, 160, 328, 273, 7, 77, 304, 196, 350, 327, 146, 217, 209, 67, 158, 33, 321, 97, 207, 11, 359, 50, 303, 114, 371, 211, 227, 368, 279, 378, 265, 338, 304, 31, 263, 351, 15, 49, 214, 309, 9, 120, 228, 41, 105, 66, 274, 130, 45, 174, 352, 89, 48, 140, 345, 294, 91, 340, 304, 31, 263, 381, 1, 242, 147, 32, 151, 135, 306, 116, 323, 218, 80, 246, 275, 254, 84, 125, 188, 191, 249, 284, 25, 152, 166, 96, 165, 155, 198, 287, 244, 177, 258, 268, 318, 259, 256, 171, 308, 339, 233, 300, 134, 162, 69, 193, 63, 278, 53, 24, 255, 143, 295, 327, 316, 138, 103, 10, 360, 169, 128, 107, 4, 189, 5, 8, 379, 115, 42, 298, 86, 290, 73, 296, 55, 348, 255, 331, 295, 355, 83, 262, 147, 14, 317, 363, 88, 20, 195, 111, 106, 109, 205, 322, 305, 224, 241, 383, 119, 44, 61, 283, 255, 220, 353, 355, 313, 159, 341, 22, 98, 3, 264, 163, 35, 230, 13, 208, 373, 190, 330, 310, 170, 293, 101, 93, 182, 173, 349, 349, 219, 327, 344, 329, 64, 10, 30, 289, 6, 364, 206, 238, 104, 18, 60, 79, 253, 232, 199, 43, 325, 257, 164, 365, 349, 81, 357, 355, 376, 127, 244, 239, 312, 154, 297, 145, 142, 332, 324, 17, 148, 261, 251, 76, 374, 68, 240, 113, 28, 126] + 10: [294, 221, 5, 337, 164, 8, 241, 269, 272, 186, 231, 250, 72, 60, 256, 71, 354, 163, 113, 131, 217, 16, 120, 227, 381, 324, 192, 48, 141, 286, 146, 301, 138, 213, 115, 345, 190, 0, 341, 81, 26, 379, 184, 50, 65, 326, 224, 34, 94, 10, 89, 290, 255, 108, 5, 216, 73, 134, 114, 237, 136, 342, 359, 382, 214, 106, 357, 258, 162, 183, 298, 62, 305, 133, 273, 277, 70, 23, 194, 319, 320, 167, 309, 144, 330, 201, 243, 49, 212, 199, 116, 314, 329, 278, 42, 13, 185, 100, 86, 168, 207, 352, 166, 176, 194, 319, 24, 365, 57, 54, 263, 211, 11, 284, 306, 128, 33, 142, 378, 97, 247, 261, 370, 139, 373, 251, 83, 143, 43, 29, 264, 319, 320, 18, 2, 170, 40, 154, 312, 377, 249, 155, 87, 122, 20, 37, 193, 317, 299, 173, 187, 35, 347, 150, 188, 68, 264, 319, 320, 18, 92, 195, 351, 304, 252, 148, 376, 132, 129, 85, 9, 371, 64, 323, 364, 111, 372, 165, 344, 361, 228, 283, 55, 181, 320, 69, 331, 215, 182, 58, 67, 356, 360, 229, 46, 288, 197, 383, 174, 45, 209, 222, 206, 6, 104, 63, 110, 349, 55, 319, 5, 279, 164, 310, 177, 308, 353, 368, 77, 90, 21, 75, 112, 267, 39, 275, 226, 80, 355, 245, 172, 156, 236, 82, 300, 268, 121, 325, 101, 178, 78, 99, 259, 208, 271, 266, 105, 56, 59, 25, 12, 239, 244, 350, 292, 203, 118, 260, 160, 295, 300, 265, 262, 126, 31, 22, 210, 91, 76, 348, 189, 145, 233, 335, 339, 15, 318, 123, 282, 302, 119, 322, 333, 191, 270, 369, 300, 265, 289, 365, 102, 1, 153, 47, 234, 374, 223, 149, 135, 311, 327, 41, 254, 95, 175, 287, 17, 158, 281, 152, 238, 358, 300, 137, 24, 367, 31, 310, 19, 380, 7, 363, 38, 32, 3, 232, 328, 74, 332, 140, 169, 235, 127, 4, 157, 84, 28, 366, 246, 151, 121, 365, 225, 179, 171, 124, 130, 316, 280, 285, 103, 27, 315, 109, 340, 343, 220, 375, 147, 200, 44, 30, 242, 297, 180, 93, 5, 286, 146, 301, 66, 248, 293, 274, 98, 198, 321, 219, 338, 230, 125, 313, 257, 96, 205, 276, 334, 14, 52, 336, 180, 151, 5, 216, 79, 161, 117, 196, 240, 204, 218, 61, 107, 296, 346, 51, 36, 362, 307, 159, 303, 53, 88, 202, 253, 291] + 11: [156, 296, 110, 145, 100, 102, 165, 381, 136, 283, 159, 151, 224, 35, 294, 349, 291, 142, 194, 5, 140, 304, 139, 28, 220, 134, 252, 296, 119, 221, 357, 15, 141, 282, 148, 346, 27, 195, 365, 209, 295, 359, 57, 43, 184, 360, 167, 380, 16, 51, 153, 105, 98, 215, 110, 190, 100, 267, 330, 20, 13, 6, 350, 374, 298, 186, 292, 302, 125, 263, 8, 200, 132, 68, 344, 276, 232, 12, 98, 215, 326, 233, 248, 0, 351, 95, 303, 367, 127, 26, 49, 254, 59, 338, 120, 258, 382, 106, 4, 342, 181, 111, 137, 307, 77, 83, 150, 121, 44, 22, 247, 204, 78, 371, 80, 166, 89, 41, 336, 135, 164, 7, 309, 210, 21, 379, 317, 372, 256, 32, 377, 3, 110, 171, 325, 236, 275, 73, 23, 107, 370, 160, 133, 242, 30, 312, 79, 225, 358, 108, 104, 272, 369, 333, 72, 191, 377, 83, 150, 99, 44, 144, 331, 131, 320, 45, 249, 86, 341, 270, 168, 353, 264, 33, 230, 222, 288, 1, 180, 206, 273, 261, 339, 314, 85, 38, 285, 24, 217, 70, 196, 319, 226, 315, 96, 58, 362, 158, 187, 18, 138, 323, 201, 114, 60, 123, 257, 61, 339, 314, 85, 235, 126, 15, 343, 146, 154, 213, 348, 340, 316, 327, 211, 364, 64, 48, 271, 266, 290, 71, 238, 76, 223, 243, 260, 314, 40, 121, 248, 218, 54, 244, 297, 170, 262, 10, 198, 97, 63, 378, 293, 174, 231, 37, 322, 305, 246, 189, 281, 42, 299, 90, 163, 300, 157, 345, 152, 39, 313, 124, 69, 176, 255, 101, 245, 34, 118, 182, 143, 112, 310, 229, 53, 253, 50, 308, 130, 227, 163, 11, 116, 332, 192, 286, 250, 161, 92, 93, 14, 208, 366, 311, 115, 129, 274, 103, 287, 268, 368, 155, 321, 9, 130, 376, 163, 36, 116, 162, 178, 318, 207, 280, 2, 251, 197, 259, 265, 179, 67, 62, 46, 84, 173, 183, 175, 122, 329, 241, 130, 90, 296, 11, 36, 29, 284, 337, 169, 66, 117, 56, 203, 301, 219, 193, 234, 355, 19, 375, 214, 278, 361, 82, 212, 335, 88, 87, 149, 235, 357, 285, 128, 177, 17, 94, 373, 172, 289, 81, 147, 354, 383, 113, 237, 306, 185, 277, 216, 239, 334, 269, 363, 87, 65, 221, 126, 199, 52, 228, 205, 47, 75, 347, 55, 31, 279, 91, 324, 188, 74, 356, 240, 328, 352, 202, 25, 109] + 12: [156, 28, 17, 349, 172, 198, 208, 68, 229, 102, 271, 301, 155, 114, 138, 161, 351, 5, 179, 106, 178, 320, 184, 100, 359, 302, 255, 158, 190, 296, 333, 50, 75, 317, 133, 27, 308, 80, 358, 41, 189, 58, 335, 12, 329, 251, 34, 54, 118, 221, 324, 272, 255, 158, 190, 378, 333, 90, 3, 152, 10, 288, 294, 337, 309, 191, 224, 169, 11, 305, 374, 352, 18, 212, 132, 242, 64, 366, 255, 219, 327, 69, 46, 341, 278, 276, 266, 300, 362, 380, 142, 77, 298, 232, 154, 183, 246, 285, 55, 81, 145, 360, 239, 222, 255, 226, 344, 378, 203, 290, 195, 215, 16, 265, 241, 326, 370, 200, 119, 319, 76, 13, 207, 146, 345, 19, 182, 248, 141, 199, 255, 226, 327, 289, 203, 97, 6, 330, 277, 39, 197, 318, 357, 1, 45, 325, 122, 65, 217, 24, 96, 72, 307, 275, 166, 220, 204, 78, 127, 177, 287, 192, 134, 348, 108, 371, 57, 59, 304, 185, 61, 356, 115, 32, 365, 30, 47, 299, 338, 163, 331, 73, 204, 78, 127, 177, 103, 341, 339, 268, 364, 153, 164, 31, 15, 231, 313, 139, 40, 379, 279, 274, 131, 125, 196, 311, 377, 42, 204, 158, 327, 69, 46, 98, 260, 137, 25, 209, 129, 261, 227, 188, 67, 249, 159, 293, 297, 70, 316, 367, 26, 202, 240, 53, 363, 245, 175, 223, 126, 51, 259, 346, 89, 2, 180, 23, 193, 254, 29, 37, 235, 86, 186, 252, 8, 258, 210, 88, 92, 372, 49, 130, 167, 82, 172, 306, 354, 382, 147, 33, 135, 291, 283, 315, 262, 280, 36, 140, 312, 105, 375, 230, 79, 206, 21, 110, 149, 130, 303, 223, 71, 151, 123, 216, 323, 264, 322, 91, 144, 234, 256, 4, 121, 334, 128, 174, 343, 247, 143, 213, 95, 225, 149, 130, 303, 82, 150, 361, 176, 332, 87, 171, 104, 74, 201, 52, 83, 187, 355, 112, 284, 310, 93, 117, 94, 228, 353, 162, 35, 381, 168, 292, 243, 233, 295, 194, 60, 116, 107, 157, 253, 350, 84, 173, 38, 20, 148, 383, 347, 237, 269, 314, 56, 250, 48, 120, 267, 22, 243, 369, 170, 205, 218, 160, 63, 238, 376, 9, 165, 321, 214, 257, 263, 85, 113, 236, 111, 342, 66, 43, 120, 120, 109, 124, 150, 181, 340, 281, 368, 101, 273, 14, 286, 0, 62, 136, 270, 336, 244, 44, 282, 211, 7, 328, 99, 373] + 13: [80, 114, 23, 280, 231, 186, 142, 58, 112, 292, 2, 219, 196, 166, 34, 106, 56, 308, 115, 365, 4, 60, 347, 368, 277, 7, 80, 96, 23, 169, 238, 102, 335, 279, 275, 294, 210, 355, 107, 359, 331, 357, 59, 263, 217, 61, 138, 191, 251, 269, 165, 27, 9, 114, 328, 51, 276, 310, 162, 323, 273, 229, 42, 168, 283, 374, 105, 326, 296, 381, 19, 235, 383, 117, 258, 200, 82, 140, 44, 18, 328, 39, 12, 254, 304, 295, 297, 143, 98, 179, 26, 327, 221, 89, 242, 133, 123, 190, 372, 284, 97, 349, 314, 33, 44, 3, 87, 41, 95, 134, 88, 184, 364, 363, 122, 237, 145, 129, 380, 288, 305, 250, 360, 214, 85, 343, 293, 132, 66, 315, 362, 18, 149, 260, 209, 271, 199, 227, 31, 40, 147, 240, 291, 139, 222, 48, 92, 348, 213, 124, 157, 1, 90, 156, 120, 350, 362, 18, 149, 260, 12, 274, 109, 35, 246, 158, 300, 211, 361, 93, 8, 379, 127, 325, 234, 313, 180, 252, 224, 354, 218, 32, 194, 81, 87, 282, 342, 318, 298, 198, 16, 67, 178, 339, 225, 333, 223, 248, 136, 53, 38, 287, 346, 101, 152, 148, 367, 175, 329, 232, 195, 172, 55, 17, 370, 131, 356, 54, 207, 43, 344, 29, 52, 68, 319, 94, 73, 5, 233, 128, 264, 302, 351, 155, 329, 232, 87, 265, 30, 30, 116, 307, 202, 261, 22, 79, 262, 183, 76, 267, 268, 245, 358, 353, 185, 28, 278, 255, 322, 204, 137, 232, 195, 24, 189, 318, 46, 45, 69, 309, 65, 10, 74, 253, 303, 153, 257, 244, 377, 130, 182, 63, 99, 341, 324, 230, 77, 57, 14, 21, 201, 366, 338, 176, 289, 91, 378, 382, 144, 243, 164, 337, 174, 71, 154, 249, 226, 336, 371, 272, 206, 216, 170, 170, 195, 312, 238, 197, 11, 49, 75, 334, 373, 321, 239, 146, 15, 36, 220, 187, 70, 119, 205, 212, 173, 290, 193, 316, 170, 108, 14, 135, 189, 330, 306, 111, 188, 266, 208, 62, 25, 177, 241, 161, 345, 311, 72, 103, 285, 352, 181, 47, 160, 78, 170, 108, 0, 280, 100, 274, 159, 118, 376, 281, 6, 375, 121, 151, 37, 286, 84, 113, 13, 150, 167, 203, 20, 332, 301, 126, 170, 108, 86, 312, 110, 340, 171, 64, 369, 320, 299, 141, 228, 256, 125, 163, 259, 50, 270, 104, 83, 215, 317, 236, 247, 192] + 14: [147, 345, 255, 239, 378, 155, 39, 154, 197, 297, 370, 382, 203, 90, 225, 115, 248, 10, 153, 19, 45, 57, 381, 340, 53, 315, 70, 204, 60, 344, 337, 71, 231, 271, 294, 328, 291, 40, 325, 129, 330, 148, 43, 106, 309, 16, 263, 24, 181, 192, 272, 174, 70, 345, 8, 75, 327, 254, 52, 250, 216, 123, 288, 169, 286, 156, 298, 12, 1, 173, 338, 361, 194, 293, 252, 27, 210, 73, 56, 204, 345, 2, 348, 308, 64, 269, 220, 163, 357, 187, 102, 221, 276, 191, 217, 262, 36, 20, 312, 180, 81, 113, 67, 331, 160, 77, 110, 75, 118, 142, 177, 374, 125, 149, 201, 301, 373, 5, 0, 89, 224, 321, 103, 290, 7, 66, 283, 131, 29, 212, 160, 164, 255, 186, 355, 23, 117, 79, 13, 59, 277, 167, 109, 101, 253, 168, 96, 311, 171, 9, 80, 62, 144, 91, 119, 121, 160, 287, 137, 259, 150, 141, 95, 44, 265, 215, 369, 104, 172, 241, 200, 300, 251, 28, 278, 58, 299, 139, 11, 188, 130, 289, 229, 15, 110, 122, 327, 151, 198, 354, 166, 135, 326, 339, 245, 347, 284, 242, 244, 273, 46, 304, 322, 120, 140, 170, 98, 318, 229, 15, 383, 2, 378, 155, 342, 182, 157, 69, 184, 22, 227, 82, 316, 202, 240, 94, 257, 305, 47, 362, 307, 280, 38, 249, 229, 76, 255, 34, 355, 275, 213, 317, 33, 246, 332, 346, 349, 105, 176, 282, 376, 310, 375, 230, 281, 84, 233, 72, 196, 165, 229, 76, 17, 34, 126, 99, 124, 111, 41, 343, 319, 108, 234, 48, 341, 364, 42, 285, 74, 78, 18, 55, 320, 303, 218, 295, 211, 4, 175, 209, 126, 228, 32, 279, 270, 161, 193, 333, 256, 377, 306, 264, 63, 114, 127, 136, 87, 92, 236, 368, 232, 83, 211, 372, 30, 259, 122, 85, 100, 61, 314, 323, 324, 133, 128, 237, 359, 313, 93, 214, 267, 238, 179, 185, 247, 54, 352, 50, 274, 207, 30, 209, 353, 275, 371, 162, 158, 25, 145, 37, 296, 380, 86, 152, 350, 302, 358, 190, 258, 35, 107, 51, 138, 88, 143, 261, 112, 268, 146, 71, 3, 21, 26, 206, 68, 266, 178, 116, 336, 223, 183, 6, 365, 14, 132, 367, 219, 134, 292, 205, 143, 4, 199, 49, 308, 235, 97, 360, 208, 379, 159, 363, 226, 222, 189, 335, 356, 329, 195, 334, 31, 366, 65, 243, 351, 260] + 15: [68, 291, 39, 324, 104, 333, 83, 163, 164, 379, 194, 117, 77, 165, 141, 308, 37, 92, 364, 188, 312, 181, 44, 243, 6, 131, 298, 11, 160, 185, 159, 119, 139, 311, 111, 357, 5, 279, 229, 8, 271, 234, 216, 213, 240, 129, 370, 228, 178, 288, 144, 322, 0, 11, 57, 332, 217, 215, 23, 342, 143, 225, 238, 2, 268, 206, 301, 289, 45, 172, 190, 369, 280, 327, 110, 336, 348, 281, 0, 11, 57, 231, 266, 284, 16, 173, 177, 36, 372, 142, 87, 43, 30, 97, 169, 337, 65, 73, 149, 66, 179, 122, 46, 366, 0, 11, 362, 155, 148, 90, 360, 256, 138, 207, 49, 59, 22, 205, 349, 381, 383, 95, 218, 184, 201, 98, 299, 302, 152, 127, 290, 130, 362, 200, 94, 60, 329, 113, 109, 338, 323, 212, 62, 50, 107, 321, 64, 26, 147, 161, 176, 58, 346, 219, 19, 192, 15, 130, 359, 258, 128, 203, 376, 277, 241, 335, 182, 105, 331, 341, 339, 365, 272, 358, 3, 378, 310, 7, 70, 125, 245, 276, 15, 130, 359, 258, 128, 146, 151, 193, 320, 47, 78, 166, 10, 13, 382, 137, 244, 265, 52, 153, 126, 330, 41, 287, 25, 325, 15, 130, 371, 200, 134, 134, 40, 174, 18, 351, 209, 102, 114, 253, 313, 354, 71, 283, 210, 197, 187, 124, 17, 286, 75, 198, 261, 367, 359, 326, 175, 121, 157, 168, 242, 100, 133, 145, 96, 363, 118, 347, 123, 375, 79, 91, 140, 361, 103, 21, 315, 85, 340, 295, 160, 355, 217, 9, 317, 344, 255, 319, 170, 101, 150, 233, 226, 53, 314, 63, 116, 374, 293, 112, 120, 156, 249, 328, 350, 202, 199, 355, 94, 20, 221, 296, 196, 285, 305, 263, 269, 250, 306, 72, 55, 316, 54, 303, 318, 136, 223, 214, 27, 260, 76, 259, 251, 324, 236, 74, 56, 1, 115, 352, 247, 86, 343, 12, 377, 99, 208, 257, 273, 158, 14, 292, 307, 220, 132, 356, 4, 227, 251, 189, 104, 252, 48, 135, 24, 31, 42, 82, 309, 81, 191, 61, 204, 84, 167, 32, 262, 195, 154, 108, 264, 368, 88, 88, 300, 38, 148, 237, 80, 35, 235, 162, 246, 248, 34, 254, 294, 267, 224, 334, 278, 89, 29, 222, 183, 345, 186, 270, 88, 227, 160, 155, 175, 51, 232, 353, 230, 106, 28, 275, 282, 69, 67, 180, 211, 373, 239, 274, 380, 171, 304, 297, 33, 93] + 16: [11, 120, 300, 165, 318, 250, 383, 347, 52, 115, 80, 8, 232, 370, 104, 102, 272, 109, 88, 47, 50, 138, 12, 326, 13, 94, 162, 120, 120, 277, 259, 255, 283, 246, 84, 97, 139, 76, 36, 322, 264, 57, 343, 360, 172, 26, 170, 101, 349, 222, 133, 366, 100, 114, 43, 86, 203, 183, 187, 16, 310, 316, 0, 41, 42, 339, 29, 25, 105, 196, 157, 145, 176, 230, 23, 341, 239, 344, 100, 154, 262, 147, 118, 205, 213, 199, 27, 82, 153, 311, 301, 238, 354, 214, 253, 207, 166, 218, 241, 243, 40, 350, 320, 3, 140, 331, 43, 273, 342, 271, 236, 108, 237, 248, 66, 184, 334, 160, 336, 208, 83, 244, 284, 131, 116, 260, 135, 173, 361, 148, 140, 185, 275, 90, 280, 348, 235, 85, 71, 59, 119, 31, 293, 9, 151, 60, 93, 48, 81, 20, 359, 169, 6, 263, 161, 368, 134, 152, 92, 269, 303, 302, 287, 367, 365, 377, 281, 227, 379, 96, 221, 200, 55, 190, 128, 17, 308, 28, 63, 72, 325, 257, 134, 67, 92, 147, 118, 5, 295, 180, 306, 335, 73, 195, 357, 124, 290, 178, 223, 91, 14, 324, 150, 61, 351, 278, 345, 98, 134, 330, 275, 64, 44, 171, 298, 146, 65, 292, 39, 231, 372, 297, 375, 313, 304, 142, 127, 107, 77, 210, 112, 327, 69, 111, 134, 125, 234, 49, 44, 24, 209, 270, 294, 37, 159, 380, 19, 137, 228, 158, 319, 193, 163, 113, 371, 32, 355, 179, 376, 353, 15, 286, 7, 165, 75, 271, 46, 89, 328, 38, 155, 34, 229, 4, 198, 211, 333, 266, 373, 261, 332, 251, 74, 299, 215, 126, 15, 369, 312, 144, 280, 374, 364, 346, 177, 307, 103, 182, 202, 276, 340, 132, 378, 30, 62, 268, 149, 245, 130, 136, 296, 249, 15, 369, 7, 338, 75, 78, 265, 123, 285, 99, 56, 305, 122, 22, 252, 18, 267, 201, 233, 168, 217, 288, 254, 192, 362, 156, 15, 152, 234, 64, 143, 224, 121, 226, 314, 51, 58, 70, 220, 188, 317, 242, 337, 240, 315, 258, 256, 352, 329, 197, 10, 247, 68, 286, 262, 269, 289, 274, 189, 87, 181, 21, 282, 219, 45, 174, 358, 54, 323, 194, 381, 117, 279, 33, 164, 204, 53, 309, 286, 286, 275, 129, 303, 35, 141, 212, 321, 2, 175, 167, 356, 106, 225, 95, 206, 363, 186, 110, 216, 382, 79, 291, 191, 1] + 17: [374, 236, 337, 375, 26, 362, 351, 56, 207, 373, 10, 70, 304, 41, 221, 342, 238, 90, 308, 63, 53, 277, 291, 111, 54, 202, 261, 236, 79, 200, 225, 305, 6, 52, 296, 380, 272, 142, 332, 295, 3, 309, 310, 217, 106, 318, 316, 201, 160, 247, 174, 237, 261, 236, 79, 375, 100, 185, 125, 290, 194, 228, 98, 168, 248, 7, 196, 89, 93, 161, 151, 344, 355, 66, 257, 68, 156, 283, 28, 284, 319, 78, 383, 343, 324, 167, 46, 145, 33, 292, 116, 331, 365, 271, 240, 345, 55, 350, 227, 23, 129, 334, 122, 121, 212, 284, 190, 87, 275, 330, 328, 157, 134, 180, 99, 9, 166, 260, 204, 95, 300, 183, 274, 31, 164, 224, 4, 35, 293, 251, 138, 284, 137, 108, 197, 305, 133, 339, 39, 92, 139, 171, 13, 42, 49, 27, 110, 102, 278, 50, 83, 149, 285, 348, 127, 152, 131, 140, 346, 254, 155, 232, 189, 8, 335, 313, 256, 245, 47, 147, 118, 44, 379, 299, 124, 67, 132, 40, 288, 353, 358, 159, 131, 188, 303, 15, 311, 114, 1, 276, 24, 231, 135, 214, 36, 269, 216, 265, 94, 258, 369, 150, 153, 230, 11, 77, 252, 320, 131, 188, 303, 15, 136, 279, 246, 75, 206, 177, 327, 263, 60, 45, 382, 338, 262, 16, 64, 57, 163, 223, 349, 321, 301, 325, 131, 284, 226, 108, 5, 286, 297, 222, 73, 211, 371, 376, 264, 363, 187, 37, 302, 112, 128, 233, 65, 30, 229, 322, 14, 69, 220, 250, 79, 375, 26, 294, 347, 172, 22, 148, 74, 165, 19, 59, 186, 341, 361, 287, 25, 192, 88, 280, 306, 281, 209, 312, 220, 250, 337, 12, 193, 294, 2, 104, 96, 103, 84, 314, 61, 173, 115, 253, 195, 210, 48, 270, 182, 378, 71, 289, 86, 72, 119, 307, 366, 200, 359, 362, 29, 234, 97, 381, 146, 282, 20, 43, 198, 181, 249, 120, 372, 76, 241, 141, 178, 0, 333, 354, 119, 80, 126, 357, 117, 323, 113, 370, 218, 243, 367, 259, 82, 215, 205, 101, 244, 85, 267, 144, 32, 109, 184, 51, 34, 170, 119, 105, 58, 219, 383, 266, 130, 175, 199, 38, 143, 18, 340, 154, 81, 21, 179, 239, 326, 364, 315, 176, 242, 17, 123, 235, 119, 208, 337, 91, 100, 330, 62, 107, 317, 273, 298, 162, 191, 336, 377, 213, 360, 329, 158, 255, 352, 356, 368, 268, 169, 203] + 18: [208, 16, 320, 65, 110, 167, 306, 266, 165, 191, 87, 263, 361, 140, 264, 265, 214, 213, 64, 229, 172, 238, 11, 21, 120, 382, 208, 210, 320, 215, 106, 239, 282, 190, 240, 23, 262, 129, 350, 203, 178, 182, 112, 303, 216, 369, 84, 37, 377, 198, 374, 147, 6, 333, 286, 119, 71, 236, 291, 378, 181, 20, 104, 109, 201, 102, 96, 363, 179, 275, 338, 123, 298, 281, 277, 252, 52, 183, 118, 53, 251, 57, 228, 189, 357, 99, 274, 365, 359, 241, 288, 195, 244, 255, 324, 358, 247, 163, 95, 90, 230, 194, 55, 293, 356, 45, 251, 57, 228, 142, 151, 341, 164, 346, 331, 0, 278, 176, 205, 63, 94, 234, 259, 197, 107, 115, 249, 221, 91, 314, 30, 271, 217, 300, 106, 355, 284, 35, 318, 371, 121, 367, 58, 157, 93, 267, 254, 127, 200, 381, 42, 370, 117, 72, 186, 32, 30, 271, 217, 300, 185, 136, 31, 330, 19, 366, 83, 135, 122, 308, 279, 145, 287, 168, 27, 209, 86, 44, 14, 310, 80, 227, 30, 271, 257, 28, 169, 26, 139, 326, 98, 342, 233, 334, 50, 61, 276, 187, 322, 175, 66, 17, 2, 29, 24, 323, 97, 242, 30, 271, 257, 347, 185, 321, 360, 3, 103, 248, 343, 131, 250, 39, 319, 292, 153, 60, 4, 56, 180, 184, 345, 196, 353, 309, 30, 271, 154, 113, 380, 301, 34, 101, 89, 41, 177, 137, 7, 15, 125, 12, 351, 82, 158, 317, 354, 352, 160, 173, 226, 59, 30, 193, 211, 65, 260, 68, 376, 105, 204, 339, 149, 296, 159, 349, 54, 132, 270, 143, 220, 133, 364, 294, 290, 336, 74, 222, 47, 271, 207, 245, 260, 25, 373, 67, 269, 337, 307, 22, 231, 289, 9, 335, 92, 108, 332, 76, 218, 280, 212, 206, 327, 78, 47, 271, 315, 245, 368, 146, 237, 77, 188, 141, 79, 88, 313, 152, 81, 13, 148, 256, 51, 126, 312, 273, 5, 62, 379, 340, 10, 43, 69, 304, 138, 192, 144, 33, 124, 73, 223, 316, 328, 235, 362, 329, 166, 48, 170, 375, 372, 299, 18, 199, 156, 1, 10, 271, 69, 114, 71, 116, 253, 246, 46, 171, 283, 162, 232, 8, 150, 155, 243, 348, 219, 130, 75, 85, 100, 134, 49, 268, 272, 43, 251, 36, 138, 174, 297, 285, 70, 202, 261, 225, 302, 111, 38, 40, 325, 383, 311, 224, 258, 161, 305, 295, 128, 344] + 19: [187, 265, 57, 338, 300, 323, 20, 79, 46, 352, 83, 272, 297, 325, 177, 356, 383, 137, 99, 28, 348, 100, 0, 169, 229, 98, 187, 90, 61, 330, 101, 184, 33, 217, 292, 134, 182, 170, 2, 346, 108, 372, 326, 22, 232, 73, 118, 252, 236, 360, 350, 198, 187, 90, 57, 288, 197, 27, 218, 9, 25, 278, 121, 251, 249, 374, 262, 47, 376, 32, 258, 294, 381, 304, 248, 17, 142, 245, 141, 302, 364, 313, 207, 216, 77, 230, 246, 116, 270, 368, 277, 203, 131, 75, 362, 87, 186, 351, 123, 178, 344, 104, 316, 369, 243, 265, 90, 161, 312, 307, 334, 153, 122, 233, 359, 204, 40, 18, 213, 26, 309, 320, 14, 149, 210, 358, 102, 151, 250, 268, 190, 7, 377, 279, 42, 64, 227, 354, 54, 71, 228, 221, 144, 273, 380, 129, 281, 139, 105, 63, 357, 136, 5, 226, 305, 107, 190, 7, 241, 126, 42, 373, 89, 152, 335, 1, 172, 29, 215, 287, 85, 148, 200, 209, 208, 94, 106, 224, 56, 282, 317, 133, 190, 7, 52, 322, 207, 114, 271, 159, 135, 36, 76, 353, 110, 306, 45, 3, 191, 168, 370, 103, 112, 367, 379, 72, 196, 263, 211, 74, 377, 183, 318, 115, 38, 58, 147, 128, 173, 16, 347, 336, 237, 65, 225, 212, 234, 371, 162, 185, 86, 199, 378, 375, 366, 48, 120, 338, 240, 328, 193, 117, 53, 285, 6, 283, 4, 95, 10, 331, 242, 176, 174, 286, 267, 39, 155, 382, 12, 34, 366, 256, 91, 291, 8, 115, 164, 264, 158, 295, 337, 257, 157, 125, 333, 62, 140, 341, 235, 314, 269, 214, 205, 55, 274, 319, 366, 256, 120, 288, 240, 363, 154, 44, 365, 310, 167, 49, 259, 143, 80, 60, 329, 343, 132, 324, 289, 223, 299, 78, 276, 355, 24, 339, 288, 180, 192, 307, 303, 189, 41, 166, 165, 138, 260, 194, 19, 160, 119, 311, 332, 37, 150, 21, 163, 220, 93, 50, 24, 111, 188, 180, 219, 64, 97, 301, 238, 30, 96, 195, 67, 342, 202, 293, 321, 82, 181, 59, 231, 51, 266, 327, 109, 81, 24, 179, 255, 313, 312, 66, 296, 275, 15, 13, 127, 239, 345, 84, 284, 92, 88, 298, 247, 315, 11, 201, 261, 43, 145, 253, 24, 206, 70, 330, 101, 113, 349, 35, 171, 222, 68, 156, 31, 244, 23, 280, 146, 130, 340, 361, 124, 290, 308, 69, 175, 254] + 20: [116, 82, 99, 112, 307, 162, 158, 135, 101, 322, 378, 58, 309, 340, 338, 39, 212, 11, 156, 323, 354, 147, 251, 17, 117, 179, 116, 144, 153, 268, 379, 93, 228, 362, 172, 13, 240, 367, 22, 238, 232, 106, 326, 154, 283, 333, 19, 64, 220, 288, 167, 168, 116, 144, 99, 369, 282, 32, 221, 140, 216, 74, 348, 0, 279, 345, 15, 304, 75, 159, 319, 7, 225, 295, 306, 143, 277, 331, 116, 66, 193, 145, 291, 138, 111, 381, 358, 104, 375, 219, 34, 131, 310, 73, 337, 151, 352, 341, 47, 383, 155, 244, 181, 189, 334, 317, 287, 369, 282, 62, 273, 50, 258, 126, 343, 30, 233, 169, 294, 202, 191, 366, 250, 329, 276, 293, 31, 203, 52, 54, 334, 55, 153, 45, 335, 330, 164, 259, 342, 177, 194, 359, 178, 346, 5, 380, 84, 217, 21, 40, 63, 133, 23, 255, 200, 206, 334, 82, 110, 268, 4, 371, 141, 96, 92, 176, 284, 368, 150, 243, 20, 365, 230, 254, 88, 321, 313, 373, 152, 237, 61, 76, 8, 161, 153, 45, 130, 188, 10, 286, 132, 6, 218, 146, 208, 165, 29, 170, 253, 210, 289, 355, 211, 245, 301, 109, 260, 303, 122, 161, 123, 226, 4, 325, 91, 382, 71, 77, 26, 36, 372, 86, 370, 231, 223, 316, 120, 113, 157, 134, 103, 298, 214, 262, 37, 161, 193, 376, 353, 363, 125, 180, 49, 182, 41, 118, 256, 274, 252, 127, 327, 56, 59, 163, 67, 68, 129, 1, 198, 263, 278, 161, 98, 87, 224, 105, 128, 264, 184, 315, 361, 314, 247, 246, 248, 257, 351, 308, 124, 28, 16, 48, 192, 269, 107, 137, 278, 161, 98, 2, 46, 270, 209, 160, 320, 90, 35, 265, 42, 60, 12, 236, 360, 80, 142, 205, 72, 108, 222, 44, 242, 281, 174, 161, 98, 87, 195, 204, 95, 187, 65, 239, 267, 53, 119, 215, 377, 3, 24, 139, 199, 271, 136, 183, 275, 69, 249, 299, 302, 272, 173, 43, 339, 148, 324, 227, 292, 357, 328, 25, 300, 97, 297, 201, 89, 285, 79, 85, 305, 70, 149, 171, 280, 350, 302, 14, 66, 145, 335, 318, 229, 311, 121, 190, 235, 175, 114, 196, 100, 332, 374, 349, 115, 102, 83, 261, 344, 197, 27, 51, 336, 336, 81, 43, 46, 185, 290, 9, 347, 57, 364, 213, 18, 166, 33, 78, 312, 94, 296, 356, 207, 234, 186, 38, 241, 266] + 21: [152, 173, 132, 210, 112, 226, 150, 147, 196, 141, 234, 352, 39, 159, 125, 237, 24, 274, 134, 67, 268, 324, 228, 59, 224, 123, 152, 173, 132, 82, 257, 345, 252, 351, 175, 72, 245, 336, 262, 260, 30, 77, 269, 216, 81, 310, 201, 37, 176, 212, 311, 10, 152, 173, 132, 200, 257, 94, 238, 19, 15, 135, 325, 338, 90, 383, 290, 266, 14, 340, 110, 195, 374, 56, 249, 222, 327, 17, 302, 247, 158, 50, 208, 363, 51, 349, 366, 230, 288, 267, 304, 312, 154, 204, 102, 36, 215, 342, 207, 180, 255, 231, 332, 35, 316, 47, 29, 80, 364, 361, 149, 184, 280, 189, 368, 358, 259, 303, 370, 295, 203, 329, 381, 163, 41, 85, 265, 166, 167, 108, 316, 47, 86, 80, 140, 277, 284, 380, 344, 221, 33, 49, 309, 356, 165, 115, 187, 258, 27, 181, 74, 99, 285, 188, 283, 179, 20, 46, 281, 124, 71, 232, 91, 191, 69, 220, 243, 63, 287, 28, 317, 40, 34, 146, 271, 96, 330, 54, 162, 61, 272, 264, 20, 47, 281, 103, 113, 88, 279, 128, 8, 313, 89, 372, 318, 251, 133, 171, 292, 346, 305, 315, 122, 114, 6, 326, 297, 369, 289, 236, 281, 119, 218, 199, 177, 151, 25, 9, 131, 168, 362, 192, 354, 100, 160, 186, 334, 105, 13, 55, 242, 116, 127, 44, 371, 4, 205, 376, 119, 45, 118, 172, 194, 144, 233, 263, 246, 62, 5, 301, 104, 57, 83, 319, 2, 129, 31, 161, 248, 321, 371, 4, 360, 38, 113, 120, 178, 365, 299, 209, 270, 197, 138, 286, 155, 275, 198, 261, 213, 241, 143, 217, 142, 278, 322, 339, 308, 254, 158, 53, 140, 169, 320, 109, 219, 43, 174, 73, 139, 98, 323, 193, 337, 335, 157, 12, 239, 314, 296, 282, 11, 42, 308, 256, 360, 376, 92, 214, 182, 307, 227, 367, 373, 18, 359, 291, 211, 353, 79, 48, 244, 253, 156, 357, 145, 298, 343, 350, 153, 254, 360, 124, 71, 348, 250, 16, 293, 52, 229, 84, 64, 21, 347, 148, 136, 276, 273, 164, 70, 68, 328, 95, 101, 58, 355, 355, 360, 103, 87, 117, 240, 26, 121, 111, 183, 1, 223, 66, 378, 341, 137, 300, 379, 294, 75, 76, 202, 32, 107, 93, 355, 190, 206, 103, 185, 377, 0, 22, 225, 23, 78, 331, 60, 97, 3, 65, 170, 126, 333, 382, 130, 375, 106, 7, 235, 306] + 22: [371, 260, 276, 224, 183, 235, 61, 111, 375, 259, 14, 339, 225, 367, 22, 11, 136, 46, 137, 334, 128, 255, 158, 166, 205, 154, 254, 221, 261, 38, 20, 139, 317, 357, 161, 264, 347, 82, 72, 1, 65, 241, 314, 265, 266, 91, 257, 326, 66, 107, 142, 101, 254, 221, 261, 224, 183, 29, 355, 311, 195, 17, 170, 191, 165, 39, 301, 45, 13, 298, 186, 196, 344, 354, 146, 121, 192, 143, 254, 221, 330, 42, 73, 74, 252, 272, 83, 123, 3, 48, 99, 320, 236, 335, 89, 353, 160, 129, 168, 6, 43, 86, 315, 346, 254, 34, 261, 115, 376, 144, 229, 338, 306, 185, 122, 63, 15, 159, 269, 0, 286, 103, 232, 152, 90, 31, 197, 194, 325, 359, 316, 181, 328, 207, 184, 169, 333, 153, 8, 312, 109, 49, 233, 318, 283, 81, 290, 279, 119, 62, 104, 370, 114, 270, 360, 310, 281, 181, 24, 147, 94, 244, 362, 70, 372, 222, 228, 340, 210, 239, 155, 78, 215, 364, 282, 120, 291, 23, 21, 56, 313, 267, 281, 181, 24, 132, 110, 263, 240, 351, 174, 365, 358, 258, 59, 47, 200, 238, 77, 275, 102, 36, 19, 363, 84, 245, 323, 307, 33, 209, 148, 328, 230, 93, 130, 381, 68, 60, 175, 156, 12, 105, 322, 373, 271, 237, 219, 179, 171, 150, 213, 203, 10, 79, 216, 331, 148, 328, 94, 44, 117, 218, 336, 116, 57, 220, 309, 305, 124, 32, 300, 108, 246, 190, 302, 223, 304, 231, 135, 95, 216, 209, 141, 288, 20, 118, 58, 345, 378, 106, 113, 40, 321, 341, 189, 299, 138, 208, 274, 134, 342, 5, 140, 234, 249, 157, 204, 278, 141, 288, 201, 329, 383, 251, 67, 193, 226, 18, 324, 98, 377, 180, 97, 176, 187, 145, 69, 308, 172, 293, 2, 28, 51, 278, 141, 337, 110, 37, 348, 188, 100, 27, 349, 374, 71, 248, 54, 256, 273, 379, 151, 52, 350, 127, 284, 297, 280, 217, 198, 332, 141, 337, 376, 112, 352, 7, 80, 369, 361, 366, 41, 380, 199, 327, 96, 356, 182, 92, 16, 88, 163, 167, 287, 247, 9, 343, 260, 164, 178, 4, 55, 285, 173, 214, 206, 25, 131, 87, 227, 268, 202, 26, 35, 294, 212, 296, 277, 243, 289, 250, 126, 343, 260, 242, 178, 177, 50, 149, 76, 125, 75, 53, 133, 319, 303, 382, 211, 262, 85, 30, 64, 162, 295, 368, 292, 253] + 23: [26, 275, 305, 244, 324, 103, 372, 86, 326, 312, 228, 359, 160, 231, 76, 258, 108, 14, 336, 202, 319, 247, 318, 198, 186, 37, 26, 274, 305, 269, 12, 176, 304, 371, 184, 74, 317, 136, 98, 301, 204, 143, 16, 243, 375, 308, 277, 238, 345, 112, 18, 189, 26, 120, 132, 167, 72, 287, 11, 41, 340, 271, 224, 221, 17, 53, 284, 73, 107, 294, 373, 140, 253, 181, 20, 83, 4, 168, 26, 71, 19, 244, 323, 84, 180, 283, 133, 149, 38, 215, 365, 137, 250, 109, 335, 285, 33, 367, 62, 259, 22, 332, 110, 75, 126, 275, 132, 306, 203, 94, 192, 257, 50, 378, 313, 370, 1, 307, 162, 280, 212, 122, 97, 46, 310, 360, 364, 60, 296, 81, 126, 275, 305, 24, 252, 27, 169, 315, 111, 102, 95, 361, 185, 178, 382, 200, 164, 362, 267, 161, 357, 381, 118, 380, 13, 291, 328, 314, 77, 240, 252, 261, 374, 130, 35, 233, 237, 44, 32, 321, 223, 128, 302, 99, 369, 61, 158, 334, 208, 145, 251, 242, 328, 314, 48, 173, 85, 154, 127, 263, 309, 151, 272, 311, 329, 65, 193, 45, 68, 163, 338, 79, 34, 377, 30, 183, 216, 288, 281, 314, 100, 87, 323, 156, 241, 56, 213, 219, 152, 157, 150, 207, 330, 344, 21, 114, 239, 188, 368, 299, 343, 262, 29, 39, 225, 314, 100, 295, 135, 47, 182, 78, 144, 70, 195, 222, 153, 325, 196, 54, 134, 354, 25, 106, 31, 322, 42, 246, 15, 8, 264, 172, 211, 187, 300, 279, 218, 6, 131, 191, 342, 210, 119, 333, 341, 0, 206, 155, 115, 260, 266, 337, 63, 105, 3, 235, 264, 172, 147, 331, 85, 232, 67, 92, 209, 91, 40, 292, 254, 52, 352, 58, 174, 297, 355, 49, 353, 171, 138, 217, 226, 265, 124, 175, 170, 201, 135, 177, 57, 148, 80, 220, 23, 101, 298, 179, 347, 256, 346, 268, 139, 214, 276, 165, 289, 293, 366, 64, 124, 69, 376, 167, 197, 177, 55, 270, 146, 249, 10, 255, 66, 125, 303, 90, 7, 121, 123, 205, 88, 350, 117, 89, 113, 286, 124, 172, 71, 82, 300, 327, 2, 351, 379, 36, 194, 129, 93, 273, 348, 363, 245, 236, 227, 282, 9, 116, 141, 142, 166, 43, 51, 175, 71, 187, 72, 358, 59, 159, 190, 229, 290, 356, 96, 5, 316, 339, 28, 349, 104, 248, 230, 383, 278, 199, 234, 320] + 24: [0, 20, 210, 223, 112, 104, 31, 35, 318, 39, 224, 376, 168, 51, 65, 365, 88, 107, 57, 217, 236, 143, 289, 322, 127, 251, 0, 20, 122, 223, 128, 301, 137, 61, 308, 350, 3, 287, 233, 197, 302, 4, 38, 126, 331, 234, 256, 98, 141, 330, 129, 327, 290, 20, 172, 263, 286, 266, 135, 277, 93, 351, 12, 272, 295, 222, 292, 275, 150, 212, 19, 55, 132, 240, 167, 69, 219, 379, 97, 6, 259, 139, 189, 334, 202, 344, 106, 377, 245, 58, 169, 268, 134, 37, 348, 94, 136, 342, 125, 229, 243, 17, 45, 298, 52, 95, 372, 105, 70, 370, 26, 54, 131, 46, 77, 221, 257, 56, 215, 218, 79, 151, 281, 44, 362, 332, 227, 214, 89, 314, 52, 6, 372, 28, 361, 207, 237, 91, 260, 138, 310, 297, 84, 242, 60, 258, 176, 161, 373, 238, 18, 187, 316, 40, 34, 115, 87, 265, 7, 381, 324, 62, 78, 195, 156, 142, 205, 48, 13, 304, 30, 309, 253, 32, 352, 317, 367, 300, 200, 208, 2, 264, 92, 265, 7, 337, 121, 294, 159, 354, 325, 73, 338, 83, 16, 116, 181, 346, 296, 76, 99, 291, 80, 201, 123, 364, 179, 81, 92, 265, 7, 185, 189, 146, 231, 366, 274, 130, 145, 279, 178, 315, 225, 203, 276, 320, 67, 329, 180, 174, 119, 371, 158, 74, 211, 265, 252, 70, 85, 29, 213, 144, 41, 149, 341, 326, 323, 313, 270, 335, 303, 188, 8, 193, 68, 247, 340, 82, 36, 166, 110, 154, 252, 357, 369, 230, 319, 382, 254, 49, 380, 59, 216, 1, 66, 72, 120, 175, 191, 50, 109, 228, 165, 33, 152, 353, 9, 154, 328, 357, 369, 186, 283, 306, 226, 113, 345, 140, 147, 280, 108, 103, 196, 86, 42, 288, 184, 43, 285, 305, 21, 232, 9, 154, 252, 223, 64, 5, 24, 164, 71, 336, 118, 114, 269, 244, 194, 170, 271, 249, 148, 209, 101, 162, 90, 349, 133, 368, 9, 154, 252, 263, 128, 255, 343, 15, 355, 356, 182, 321, 235, 11, 339, 282, 23, 284, 160, 171, 183, 250, 199, 117, 358, 383, 246, 246, 22, 360, 85, 375, 53, 190, 299, 96, 293, 63, 153, 47, 312, 333, 10, 363, 359, 102, 248, 75, 124, 261, 155, 198, 246, 111, 22, 157, 27, 220, 204, 192, 273, 177, 307, 206, 278, 311, 239, 267, 14, 173, 163, 262, 25, 378, 347, 100, 374, 241] + 25: [315, 379, 101, 372, 327, 333, 314, 120, 231, 256, 176, 114, 221, 37, 328, 212, 200, 163, 22, 173, 293, 119, 57, 148, 170, 123, 315, 379, 282, 162, 62, 160, 66, 302, 209, 229, 103, 196, 312, 290, 134, 11, 300, 51, 292, 311, 289, 95, 363, 288, 255, 239, 315, 379, 282, 332, 246, 280, 38, 154, 84, 54, 146, 291, 199, 130, 257, 370, 361, 248, 117, 226, 46, 205, 360, 241, 106, 178, 315, 334, 72, 23, 152, 145, 32, 100, 306, 366, 164, 191, 237, 324, 133, 94, 74, 110, 34, 21, 245, 48, 374, 198, 151, 220, 315, 82, 72, 61, 113, 235, 8, 131, 253, 80, 348, 140, 68, 15, 202, 150, 27, 127, 250, 320, 41, 188, 217, 47, 13, 277, 315, 299, 317, 107, 29, 171, 380, 262, 215, 126, 279, 359, 383, 213, 297, 31, 53, 337, 325, 87, 16, 264, 172, 169, 338, 156, 269, 168, 30, 214, 284, 10, 59, 368, 240, 125, 4, 143, 187, 184, 365, 356, 79, 271, 281, 364, 175, 344, 141, 266, 195, 263, 269, 268, 180, 25, 309, 232, 296, 381, 373, 367, 227, 211, 316, 330, 254, 149, 223, 166, 362, 122, 92, 349, 285, 50, 6, 158, 269, 298, 72, 372, 192, 49, 318, 18, 234, 89, 321, 225, 350, 204, 339, 310, 249, 260, 36, 210, 236, 3, 335, 272, 303, 189, 269, 379, 85, 107, 29, 112, 378, 351, 222, 99, 261, 336, 132, 259, 28, 105, 179, 358, 86, 183, 322, 155, 26, 108, 270, 267, 269, 379, 317, 23, 192, 5, 102, 287, 197, 138, 97, 242, 81, 55, 201, 355, 340, 353, 93, 96, 40, 63, 147, 182, 206, 326, 98, 247, 180, 214, 284, 77, 230, 342, 252, 323, 307, 352, 203, 308, 58, 24, 159, 111, 35, 283, 118, 224, 71, 219, 43, 165, 139, 258, 282, 2, 0, 73, 216, 207, 9, 153, 161, 115, 304, 185, 331, 67, 193, 243, 157, 70, 64, 142, 88, 301, 19, 121, 139, 247, 282, 39, 52, 190, 33, 136, 377, 244, 218, 369, 278, 345, 181, 69, 238, 109, 177, 208, 371, 1, 286, 329, 194, 91, 65, 83, 375, 78, 62, 7, 305, 346, 56, 76, 135, 347, 295, 354, 274, 144, 251, 186, 75, 45, 128, 265, 12, 90, 44, 129, 228, 228, 276, 276, 376, 273, 313, 294, 341, 60, 343, 20, 319, 357, 116, 14, 17, 275, 124, 167, 174, 137, 42, 104, 382, 233] + 26: [80, 237, 90, 21, 289, 344, 22, 382, 128, 360, 97, 173, 239, 212, 41, 187, 78, 325, 335, 112, 154, 42, 85, 84, 139, 169, 80, 237, 90, 21, 16, 337, 376, 332, 156, 323, 102, 103, 195, 159, 50, 153, 336, 340, 40, 275, 314, 284, 211, 310, 170, 175, 150, 124, 217, 338, 339, 137, 61, 126, 148, 179, 251, 79, 277, 65, 231, 182, 189, 12, 334, 59, 163, 4, 250, 95, 203, 218, 320, 124, 237, 291, 339, 316, 226, 108, 180, 321, 356, 221, 17, 87, 114, 232, 368, 352, 72, 297, 101, 296, 164, 58, 168, 209, 136, 269, 217, 253, 254, 276, 88, 62, 247, 287, 174, 343, 104, 161, 196, 75, 157, 121, 43, 9, 91, 373, 257, 210, 52, 96, 136, 269, 18, 236, 230, 278, 227, 83, 200, 262, 99, 24, 15, 81, 350, 383, 25, 197, 255, 361, 205, 270, 28, 381, 374, 375, 136, 269, 18, 236, 116, 130, 135, 36, 311, 178, 142, 294, 365, 35, 171, 123, 301, 105, 354, 39, 29, 313, 46, 23, 261, 305, 136, 349, 217, 70, 149, 308, 355, 146, 48, 138, 367, 194, 20, 202, 252, 143, 302, 125, 109, 77, 274, 377, 147, 366, 348, 351, 136, 271, 237, 291, 224, 94, 249, 82, 265, 133, 27, 122, 318, 98, 115, 312, 298, 300, 185, 331, 119, 56, 222, 273, 60, 106, 5, 11, 172, 342, 69, 183, 158, 167, 176, 26, 353, 57, 127, 378, 370, 144, 264, 364, 8, 346, 216, 324, 188, 322, 1, 54, 6, 380, 53, 181, 160, 279, 30, 131, 359, 190, 306, 51, 66, 290, 76, 93, 219, 362, 347, 74, 193, 14, 282, 155, 213, 228, 6, 380, 214, 184, 162, 38, 223, 245, 272, 191, 152, 186, 357, 45, 107, 363, 280, 3, 165, 100, 206, 145, 283, 248, 260, 234, 233, 11, 172, 63, 230, 341, 238, 92, 198, 259, 44, 268, 285, 0, 267, 19, 117, 220, 317, 333, 326, 243, 330, 379, 68, 86, 292, 89, 90, 263, 253, 307, 369, 55, 67, 151, 2, 201, 71, 10, 37, 177, 329, 47, 141, 303, 13, 358, 235, 207, 309, 246, 292, 89, 53, 371, 120, 33, 315, 328, 31, 319, 327, 241, 215, 113, 256, 372, 134, 49, 225, 244, 240, 208, 295, 286, 299, 242, 281, 89, 172, 181, 304, 266, 293, 110, 204, 166, 229, 118, 192, 73, 32, 129, 7, 140, 64, 34, 199, 258, 345, 288, 132, 111] + 27: [34, 12, 61, 233, 49, 358, 183, 228, 232, 89, 141, 98, 6, 127, 375, 249, 29, 116, 96, 8, 157, 192, 311, 295, 36, 236, 234, 333, 61, 168, 80, 341, 348, 85, 76, 238, 191, 155, 271, 268, 124, 44, 365, 159, 280, 22, 229, 64, 1, 169, 247, 310, 234, 351, 263, 269, 30, 354, 190, 75, 258, 82, 356, 241, 175, 136, 67, 120, 95, 123, 231, 50, 302, 87, 52, 363, 270, 338, 81, 286, 263, 252, 357, 377, 222, 260, 380, 32, 297, 88, 91, 215, 129, 199, 167, 153, 281, 223, 352, 108, 173, 371, 285, 342, 103, 225, 312, 300, 376, 211, 179, 21, 196, 78, 322, 254, 126, 174, 106, 347, 3, 107, 154, 313, 158, 257, 11, 19, 180, 146, 103, 225, 312, 265, 128, 211, 224, 303, 149, 343, 187, 370, 368, 296, 18, 2, 217, 79, 306, 186, 150, 198, 56, 355, 200, 163, 103, 225, 312, 265, 349, 227, 188, 340, 242, 115, 317, 161, 326, 274, 46, 38, 47, 256, 152, 282, 330, 345, 245, 40, 209, 176, 103, 286, 12, 69, 135, 131, 336, 230, 381, 102, 337, 162, 125, 291, 248, 171, 13, 205, 184, 319, 110, 94, 138, 255, 203, 220, 103, 286, 12, 204, 30, 354, 237, 251, 266, 156, 118, 261, 329, 304, 239, 147, 367, 361, 259, 346, 289, 283, 148, 210, 243, 240, 301, 4, 31, 168, 80, 341, 369, 178, 216, 308, 140, 27, 10, 5, 276, 307, 101, 288, 113, 359, 41, 189, 194, 57, 51, 39, 37, 58, 350, 164, 197, 24, 207, 290, 99, 294, 193, 7, 316, 360, 45, 9, 353, 105, 344, 253, 364, 334, 160, 221, 378, 244, 37, 4, 70, 73, 119, 366, 55, 335, 72, 278, 25, 42, 246, 122, 287, 318, 250, 151, 132, 65, 372, 166, 165, 206, 212, 262, 104, 279, 90, 73, 320, 292, 97, 272, 324, 23, 201, 328, 309, 218, 362, 28, 321, 33, 14, 264, 382, 86, 331, 92, 93, 60, 145, 59, 90, 69, 26, 202, 112, 182, 77, 325, 109, 305, 214, 172, 43, 20, 84, 35, 139, 293, 68, 195, 314, 298, 100, 137, 177, 117, 90, 71, 349, 227, 213, 323, 383, 83, 373, 133, 379, 327, 267, 66, 121, 130, 144, 74, 16, 273, 170, 277, 17, 235, 177, 219, 374, 252, 315, 181, 143, 48, 339, 332, 284, 15, 54, 114, 111, 299, 0, 275, 208, 185, 142, 62, 226, 134, 63, 53] + 28: [4, 356, 356, 306, 99, 321, 219, 58, 352, 299, 209, 127, 333, 365, 23, 159, 84, 73, 31, 348, 259, 29, 350, 186, 2, 359, 355, 92, 153, 306, 25, 357, 263, 368, 339, 240, 221, 16, 222, 194, 44, 305, 170, 151, 79, 80, 47, 55, 65, 309, 379, 353, 161, 283, 203, 195, 107, 253, 346, 227, 160, 167, 349, 273, 290, 264, 171, 148, 104, 136, 331, 39, 85, 267, 371, 319, 115, 150, 161, 26, 328, 302, 25, 197, 137, 320, 135, 32, 340, 48, 134, 256, 69, 82, 270, 377, 109, 287, 237, 281, 360, 187, 147, 311, 161, 26, 188, 302, 122, 197, 95, 158, 200, 220, 56, 278, 335, 265, 208, 362, 74, 332, 149, 131, 303, 277, 53, 83, 9, 156, 161, 26, 70, 97, 99, 321, 166, 5, 129, 370, 217, 380, 300, 46, 249, 143, 36, 342, 213, 246, 8, 275, 254, 145, 11, 180, 161, 92, 153, 255, 93, 214, 378, 165, 100, 242, 105, 43, 206, 179, 6, 61, 169, 297, 119, 293, 347, 76, 102, 164, 173, 329, 312, 324, 185, 243, 288, 7, 75, 71, 88, 326, 271, 316, 1, 121, 295, 21, 230, 382, 144, 323, 146, 35, 291, 212, 54, 215, 0, 22, 226, 176, 286, 307, 128, 59, 157, 28, 308, 367, 78, 260, 81, 177, 66, 133, 89, 110, 229, 111, 45, 87, 132, 257, 248, 344, 255, 120, 286, 272, 199, 204, 63, 269, 125, 268, 341, 112, 49, 175, 12, 280, 96, 251, 276, 123, 94, 124, 138, 279, 201, 344, 152, 162, 52, 241, 3, 116, 383, 289, 13, 315, 91, 244, 57, 375, 211, 154, 117, 345, 192, 252, 189, 313, 10, 18, 239, 250, 185, 155, 107, 37, 108, 228, 235, 337, 20, 247, 68, 118, 64, 381, 334, 294, 218, 351, 139, 232, 304, 318, 60, 86, 310, 250, 317, 233, 30, 113, 236, 325, 130, 163, 33, 19, 282, 245, 327, 106, 258, 34, 330, 38, 234, 140, 126, 198, 190, 336, 196, 250, 317, 172, 30, 374, 369, 223, 361, 67, 216, 193, 178, 142, 314, 205, 210, 366, 364, 225, 17, 101, 141, 262, 298, 338, 168, 77, 185, 162, 103, 224, 50, 42, 90, 285, 238, 376, 14, 183, 274, 98, 15, 24, 372, 284, 72, 174, 202, 62, 301, 343, 168, 250, 70, 243, 52, 241, 191, 51, 261, 41, 182, 114, 184, 27, 358, 322, 296, 363, 266, 292, 231, 373, 354, 181, 40, 207] + 29: [383, 361, 203, 304, 173, 89, 326, 204, 50, 324, 139, 59, 97, 344, 312, 315, 56, 158, 179, 306, 251, 96, 5, 221, 206, 212, 197, 361, 198, 339, 202, 11, 112, 370, 86, 61, 349, 19, 310, 313, 22, 40, 111, 250, 227, 308, 141, 109, 27, 321, 367, 39, 197, 361, 198, 362, 282, 11, 65, 68, 350, 248, 257, 225, 267, 105, 229, 46, 209, 261, 239, 43, 45, 358, 80, 335, 37, 177, 73, 51, 333, 304, 173, 255, 366, 24, 210, 172, 178, 146, 101, 340, 169, 230, 220, 42, 253, 99, 184, 189, 2, 268, 98, 26, 73, 51, 203, 295, 23, 89, 180, 237, 382, 218, 34, 228, 150, 77, 224, 52, 287, 299, 352, 154, 337, 201, 8, 372, 143, 185, 271, 25, 258, 362, 282, 266, 243, 163, 82, 298, 128, 171, 181, 371, 242, 170, 114, 70, 69, 162, 30, 164, 226, 81, 256, 322, 283, 318, 323, 33, 60, 160, 140, 275, 341, 314, 106, 272, 264, 217, 31, 157, 368, 84, 121, 168, 246, 234, 286, 107, 274, 363, 14, 318, 354, 295, 311, 255, 183, 145, 278, 259, 116, 347, 3, 32, 36, 126, 53, 338, 270, 376, 296, 325, 260, 291, 6, 29, 196, 336, 323, 129, 23, 346, 174, 63, 102, 215, 12, 276, 345, 328, 238, 302, 303, 161, 15, 149, 290, 49, 245, 342, 284, 66, 196, 54, 277, 74, 309, 351, 48, 332, 193, 360, 232, 300, 104, 20, 83, 10, 191, 199, 192, 334, 211, 301, 130, 64, 144, 355, 88, 336, 120, 265, 44, 127, 375, 235, 38, 364, 222, 131, 142, 205, 359, 147, 55, 9, 62, 356, 92, 41, 100, 138, 305, 285, 58, 244, 277, 262, 236, 208, 134, 190, 293, 4, 200, 18, 380, 317, 373, 377, 103, 153, 233, 0, 263, 279, 95, 1, 214, 108, 195, 244, 137, 67, 320, 127, 159, 188, 123, 94, 329, 72, 353, 269, 122, 369, 79, 166, 307, 219, 252, 254, 21, 165, 289, 91, 118, 124, 379, 110, 44, 311, 87, 175, 319, 85, 182, 90, 151, 167, 348, 93, 327, 7, 187, 17, 35, 155, 75, 231, 249, 152, 118, 365, 137, 67, 236, 148, 57, 343, 207, 378, 273, 280, 78, 125, 47, 28, 213, 13, 374, 71, 115, 133, 331, 281, 247, 176, 118, 244, 76, 186, 60, 16, 119, 216, 357, 132, 156, 136, 292, 294, 316, 135, 381, 288, 194, 113, 297, 223, 330, 241, 117, 240] + 30: [140, 76, 53, 245, 143, 144, 147, 221, 336, 320, 160, 343, 335, 172, 375, 360, 166, 304, 83, 82, 330, 112, 93, 251, 47, 88, 140, 222, 358, 214, 156, 289, 302, 43, 71, 210, 344, 91, 54, 14, 244, 284, 230, 247, 155, 364, 190, 102, 63, 201, 123, 128, 140, 272, 29, 225, 206, 361, 205, 312, 58, 215, 67, 60, 142, 280, 241, 374, 110, 324, 287, 368, 308, 106, 171, 227, 134, 146, 133, 333, 75, 196, 373, 234, 250, 159, 253, 226, 285, 325, 383, 376, 248, 92, 95, 350, 327, 291, 173, 56, 372, 379, 345, 380, 158, 213, 243, 378, 351, 321, 362, 97, 339, 136, 292, 261, 84, 310, 48, 44, 377, 371, 5, 370, 352, 199, 39, 70, 8, 21, 37, 239, 65, 245, 46, 246, 316, 224, 275, 299, 18, 104, 231, 20, 211, 145, 194, 161, 99, 81, 105, 313, 148, 27, 301, 218, 37, 239, 65, 135, 143, 64, 41, 341, 1, 217, 356, 96, 72, 319, 115, 282, 183, 347, 165, 3, 15, 265, 367, 238, 185, 189, 254, 239, 242, 245, 24, 346, 33, 74, 193, 309, 298, 365, 162, 111, 180, 288, 337, 326, 69, 209, 255, 270, 26, 87, 307, 259, 124, 239, 53, 65, 168, 45, 164, 317, 177, 277, 170, 23, 114, 129, 109, 175, 342, 279, 381, 314, 178, 235, 42, 131, 22, 31, 266, 208, 366, 52, 351, 28, 17, 89, 6, 236, 98, 174, 306, 286, 349, 318, 186, 90, 57, 130, 120, 300, 34, 191, 36, 40, 266, 239, 13, 214, 156, 289, 315, 340, 100, 204, 369, 132, 179, 359, 181, 297, 107, 267, 323, 127, 233, 80, 328, 331, 268, 269, 266, 239, 358, 78, 2, 62, 271, 49, 256, 9, 258, 157, 150, 66, 293, 25, 154, 32, 348, 355, 232, 198, 229, 30, 354, 322, 184, 208, 187, 52, 257, 10, 68, 138, 77, 382, 281, 108, 167, 294, 332, 79, 311, 264, 203, 153, 19, 338, 61, 119, 363, 334, 262, 208, 137, 35, 295, 223, 212, 240, 192, 141, 296, 278, 103, 163, 7, 249, 121, 220, 85, 273, 113, 126, 182, 252, 94, 101, 262, 208, 187, 59, 373, 188, 303, 216, 152, 151, 260, 200, 38, 197, 73, 50, 202, 263, 125, 117, 16, 207, 228, 4, 176, 118, 208, 208, 86, 35, 62, 11, 283, 12, 357, 195, 274, 353, 329, 51, 55, 290, 276, 139, 116, 122, 169, 0, 149, 305, 219, 237] + 31: [248, 233, 121, 278, 147, 39, 349, 62, 296, 152, 368, 366, 287, 302, 195, 362, 177, 264, 210, 367, 216, 293, 18, 218, 343, 207, 320, 36, 379, 115, 70, 317, 206, 208, 258, 249, 74, 105, 46, 21, 45, 247, 83, 130, 375, 340, 135, 382, 179, 332, 89, 180, 125, 159, 26, 326, 250, 227, 22, 299, 290, 138, 243, 134, 202, 369, 285, 153, 351, 228, 309, 129, 327, 344, 242, 311, 203, 294, 128, 159, 158, 239, 187, 268, 188, 171, 131, 254, 231, 137, 222, 118, 141, 271, 275, 1, 139, 157, 99, 372, 319, 161, 165, 347, 128, 159, 158, 278, 25, 227, 209, 164, 189, 82, 40, 205, 318, 310, 8, 12, 323, 281, 226, 24, 64, 197, 27, 357, 71, 37, 128, 159, 252, 115, 70, 87, 119, 75, 214, 380, 54, 88, 374, 288, 92, 11, 376, 107, 240, 178, 321, 126, 370, 230, 104, 172, 331, 3, 252, 239, 215, 96, 325, 56, 223, 353, 2, 81, 190, 303, 110, 4, 352, 224, 306, 262, 280, 156, 255, 358, 10, 329, 265, 365, 166, 144, 363, 150, 60, 283, 348, 211, 232, 176, 31, 314, 291, 108, 14, 103, 162, 9, 300, 282, 167, 257, 67, 246, 201, 0, 26, 133, 95, 267, 333, 34, 276, 236, 23, 122, 316, 336, 361, 260, 91, 371, 101, 298, 109, 235, 301, 359, 148, 212, 201, 19, 26, 116, 261, 328, 140, 72, 112, 78, 55, 57, 41, 217, 73, 237, 266, 198, 47, 168, 191, 53, 184, 163, 13, 149, 68, 0, 324, 143, 185, 219, 58, 378, 102, 145, 98, 322, 77, 373, 29, 51, 43, 48, 308, 315, 113, 193, 174, 30, 341, 90, 68, 0, 324, 292, 185, 339, 61, 170, 334, 111, 245, 169, 284, 213, 142, 305, 6, 304, 85, 259, 356, 79, 35, 94, 117, 229, 32, 0, 279, 143, 261, 87, 155, 253, 286, 28, 66, 175, 97, 312, 84, 273, 346, 204, 345, 251, 234, 274, 313, 377, 154, 120, 192, 100, 166, 364, 270, 330, 186, 360, 269, 244, 63, 65, 256, 383, 354, 196, 220, 194, 42, 160, 381, 44, 124, 199, 151, 350, 221, 0, 233, 144, 363, 76, 173, 338, 106, 136, 38, 182, 15, 123, 49, 183, 272, 50, 80, 337, 52, 7, 114, 86, 200, 132, 241, 100, 233, 335, 69, 76, 146, 355, 33, 93, 17, 277, 225, 307, 16, 20, 59, 238, 263, 342, 127, 181, 295, 297, 5, 289] + 32: [94, 108, 187, 66, 141, 329, 184, 283, 19, 345, 101, 25, 45, 240, 24, 198, 5, 132, 128, 305, 278, 100, 215, 93, 333, 245, 94, 146, 209, 360, 38, 166, 204, 116, 120, 266, 104, 324, 142, 249, 381, 56, 311, 284, 222, 44, 52, 277, 112, 272, 251, 262, 63, 367, 92, 130, 164, 344, 96, 140, 113, 168, 36, 250, 32, 351, 10, 22, 172, 165, 376, 134, 343, 133, 98, 291, 316, 373, 63, 367, 92, 21, 257, 329, 83, 126, 289, 189, 111, 269, 127, 53, 315, 77, 144, 287, 105, 356, 334, 364, 95, 226, 27, 14, 63, 367, 323, 271, 107, 342, 65, 78, 228, 227, 85, 239, 293, 176, 337, 139, 82, 125, 304, 330, 170, 137, 47, 192, 177, 181, 63, 367, 179, 310, 107, 41, 359, 233, 7, 325, 69, 332, 253, 70, 302, 236, 136, 102, 73, 374, 377, 1, 260, 235, 326, 244, 63, 367, 285, 229, 164, 344, 174, 97, 232, 301, 185, 157, 230, 84, 4, 158, 361, 103, 195, 261, 208, 60, 18, 295, 379, 375, 63, 49, 362, 122, 327, 61, 75, 151, 357, 297, 292, 64, 237, 3, 80, 193, 42, 9, 117, 201, 203, 259, 217, 338, 162, 169, 218, 268, 285, 370, 274, 322, 263, 279, 306, 369, 354, 281, 182, 81, 205, 372, 221, 99, 223, 119, 79, 23, 242, 194, 54, 57, 218, 346, 106, 200, 327, 275, 89, 143, 273, 72, 55, 48, 336, 29, 129, 349, 34, 307, 20, 153, 255, 31, 321, 267, 90, 68, 288, 135, 190, 51, 12, 322, 202, 258, 308, 298, 26, 175, 382, 39, 247, 62, 110, 43, 341, 0, 196, 46, 358, 11, 15, 147, 148, 246, 347, 51, 122, 214, 225, 211, 294, 171, 303, 178, 286, 183, 309, 340, 339, 191, 365, 67, 355, 76, 17, 159, 167, 265, 148, 234, 347, 163, 219, 275, 348, 156, 109, 91, 50, 206, 150, 353, 6, 378, 320, 124, 197, 270, 188, 118, 161, 290, 145, 123, 352, 155, 241, 8, 121, 16, 186, 331, 58, 313, 199, 28, 59, 371, 276, 363, 210, 299, 13, 35, 138, 180, 74, 318, 173, 296, 352, 33, 190, 280, 2, 40, 319, 87, 335, 212, 231, 314, 86, 115, 30, 317, 282, 243, 248, 160, 149, 216, 88, 207, 300, 312, 352, 33, 241, 163, 38, 166, 380, 131, 252, 254, 71, 213, 256, 350, 366, 37, 154, 152, 238, 264, 220, 224, 368, 328, 114, 383] + 33: [86, 362, 36, 319, 283, 125, 89, 21, 30, 222, 305, 230, 347, 367, 279, 371, 350, 67, 136, 132, 332, 337, 275, 284, 189, 12, 167, 377, 380, 81, 251, 267, 37, 361, 227, 40, 232, 173, 80, 156, 213, 49, 52, 285, 238, 111, 365, 353, 313, 344, 282, 277, 91, 38, 42, 343, 258, 73, 89, 233, 25, 334, 18, 138, 262, 19, 105, 368, 166, 278, 23, 74, 62, 244, 374, 172, 143, 69, 91, 38, 158, 297, 183, 68, 139, 273, 1, 131, 211, 205, 22, 197, 340, 292, 28, 29, 221, 76, 291, 378, 7, 309, 223, 142, 91, 38, 323, 81, 301, 267, 352, 270, 151, 192, 144, 149, 45, 370, 152, 43, 66, 248, 194, 381, 224, 315, 214, 302, 77, 287, 328, 38, 31, 343, 39, 179, 206, 48, 117, 356, 351, 187, 79, 104, 13, 249, 26, 53, 182, 286, 312, 326, 171, 296, 129, 83, 51, 38, 153, 115, 115, 307, 266, 127, 155, 237, 202, 329, 162, 256, 58, 170, 103, 271, 298, 11, 310, 341, 274, 304, 264, 236, 240, 191, 362, 107, 251, 307, 5, 169, 150, 217, 126, 109, 246, 55, 60, 106, 349, 261, 20, 300, 98, 134, 359, 99, 185, 333, 140, 280, 121, 319, 301, 348, 146, 54, 35, 289, 290, 355, 196, 382, 364, 137, 65, 112, 318, 27, 184, 165, 92, 46, 272, 102, 176, 331, 229, 325, 299, 94, 72, 335, 193, 208, 303, 93, 375, 157, 234, 122, 70, 101, 346, 130, 257, 175, 306, 294, 265, 34, 75, 339, 380, 96, 258, 342, 226, 190, 338, 311, 235, 174, 198, 373, 114, 215, 113, 295, 128, 243, 245, 59, 250, 2, 159, 161, 145, 339, 64, 379, 299, 147, 108, 141, 160, 268, 116, 16, 124, 50, 369, 345, 366, 110, 252, 225, 78, 363, 228, 321, 263, 220, 320, 339, 64, 148, 154, 94, 84, 376, 218, 308, 354, 90, 216, 200, 88, 330, 281, 327, 6, 181, 118, 247, 269, 44, 100, 293, 260, 177, 36, 231, 82, 241, 322, 17, 87, 133, 61, 317, 383, 360, 186, 3, 336, 41, 10, 85, 188, 199, 97, 47, 0, 32, 260, 177, 36, 148, 154, 358, 201, 204, 33, 9, 14, 254, 276, 209, 212, 15, 56, 324, 316, 207, 120, 178, 253, 57, 357, 372, 63, 259, 229, 319, 283, 314, 164, 255, 210, 288, 8, 4, 123, 163, 180, 71, 24, 219, 135, 168, 119, 242, 195, 203, 239, 95] + 34: [372, 312, 135, 218, 3, 315, 246, 4, 244, 81, 368, 245, 181, 170, 180, 127, 142, 374, 36, 255, 242, 257, 43, 188, 198, 341, 382, 18, 330, 248, 145, 178, 59, 267, 162, 174, 159, 336, 30, 204, 140, 10, 23, 266, 262, 49, 57, 253, 130, 276, 129, 193, 366, 126, 16, 340, 3, 258, 347, 65, 95, 326, 327, 217, 270, 263, 308, 62, 56, 152, 310, 109, 131, 339, 163, 320, 235, 202, 366, 126, 249, 311, 190, 61, 97, 254, 373, 58, 199, 143, 227, 5, 230, 157, 147, 192, 226, 93, 250, 90, 281, 378, 224, 132, 108, 179, 16, 15, 243, 291, 79, 222, 200, 290, 166, 60, 0, 367, 80, 322, 51, 88, 118, 34, 370, 22, 54, 338, 211, 76, 108, 179, 16, 15, 293, 148, 83, 288, 110, 329, 37, 172, 214, 116, 348, 19, 261, 13, 269, 380, 173, 20, 128, 349, 213, 119, 108, 179, 16, 15, 240, 31, 8, 11, 21, 53, 136, 33, 150, 35, 232, 277, 272, 219, 228, 331, 55, 297, 325, 369, 66, 122, 108, 332, 330, 248, 223, 194, 99, 300, 225, 302, 256, 70, 236, 333, 352, 283, 139, 296, 265, 307, 17, 9, 120, 252, 377, 229, 29, 179, 74, 218, 25, 167, 295, 78, 91, 144, 137, 206, 209, 169, 154, 42, 105, 314, 282, 117, 176, 171, 96, 182, 177, 279, 337, 299, 306, 239, 196, 203, 238, 92, 134, 184, 197, 195, 231, 50, 301, 294, 264, 268, 274, 100, 260, 212, 351, 38, 273, 160, 337, 357, 334, 354, 44, 360, 115, 6, 208, 12, 175, 317, 48, 89, 215, 298, 149, 85, 111, 319, 278, 1, 71, 234, 121, 381, 69, 324, 306, 306, 190, 138, 379, 77, 125, 84, 313, 328, 346, 102, 103, 323, 186, 275, 86, 286, 285, 141, 292, 303, 375, 365, 69, 299, 82, 158, 196, 362, 271, 41, 201, 210, 183, 67, 241, 364, 113, 318, 383, 289, 233, 75, 26, 104, 156, 106, 309, 363, 207, 259, 334, 304, 185, 40, 284, 191, 124, 356, 187, 133, 72, 287, 63, 114, 45, 68, 101, 353, 39, 155, 216, 350, 32, 14, 47, 151, 249, 340, 321, 61, 2, 335, 153, 107, 24, 280, 27, 247, 361, 371, 46, 64, 305, 165, 205, 112, 355, 94, 7, 316, 237, 259, 249, 340, 44, 146, 220, 168, 87, 345, 123, 251, 73, 359, 164, 358, 221, 98, 161, 28, 52, 342, 344, 376, 343, 189] + 35: [48, 58, 297, 345, 4, 322, 224, 311, 266, 357, 128, 215, 137, 261, 111, 34, 301, 217, 274, 278, 306, 238, 351, 210, 76, 381, 309, 319, 297, 345, 324, 72, 147, 375, 329, 98, 325, 372, 243, 57, 109, 290, 269, 344, 63, 135, 89, 97, 333, 39, 173, 228, 363, 319, 131, 368, 86, 276, 206, 234, 14, 187, 171, 298, 337, 170, 145, 382, 17, 66, 103, 115, 94, 150, 270, 258, 199, 304, 363, 319, 131, 67, 47, 113, 231, 241, 191, 360, 356, 334, 122, 302, 348, 339, 99, 84, 317, 22, 232, 148, 197, 247, 212, 284, 80, 319, 211, 295, 53, 281, 15, 364, 136, 373, 230, 190, 320, 92, 20, 166, 184, 338, 65, 316, 26, 165, 225, 226, 341, 60, 75, 144, 56, 345, 342, 203, 116, 307, 222, 321, 308, 282, 12, 32, 106, 117, 376, 6, 35, 370, 250, 139, 83, 242, 59, 291, 75, 130, 1, 295, 53, 3, 340, 335, 361, 168, 296, 108, 95, 96, 110, 369, 142, 175, 105, 156, 121, 255, 310, 280, 71, 200, 353, 237, 1, 367, 55, 113, 221, 50, 362, 140, 244, 30, 209, 350, 277, 273, 185, 374, 292, 207, 155, 263, 233, 219, 64, 118, 220, 62, 379, 133, 138, 72, 355, 349, 126, 163, 18, 213, 125, 287, 305, 216, 19, 285, 248, 161, 49, 288, 123, 252, 100, 85, 45, 205, 218, 314, 86, 203, 340, 326, 331, 160, 158, 268, 313, 74, 8, 383, 70, 315, 151, 259, 354, 183, 327, 336, 236, 359, 45, 205, 218, 68, 21, 322, 186, 46, 159, 157, 132, 328, 152, 323, 214, 264, 179, 88, 332, 253, 195, 358, 33, 193, 2, 13, 45, 240, 303, 68, 129, 38, 41, 352, 37, 275, 9, 93, 300, 11, 227, 330, 196, 279, 365, 29, 112, 101, 178, 188, 283, 146, 169, 44, 1, 367, 141, 289, 78, 114, 27, 312, 272, 294, 42, 172, 202, 204, 380, 43, 79, 377, 23, 77, 371, 0, 120, 293, 169, 246, 254, 318, 257, 281, 346, 164, 162, 267, 36, 5, 201, 16, 180, 347, 127, 73, 176, 149, 181, 10, 271, 119, 251, 177, 260, 182, 299, 286, 342, 276, 154, 194, 52, 134, 61, 192, 90, 256, 91, 82, 174, 378, 87, 189, 262, 167, 223, 208, 124, 40, 28, 28, 299, 133, 153, 31, 239, 265, 54, 102, 81, 104, 143, 235, 366, 24, 7, 25, 198, 245, 249, 107, 69, 51, 229, 343] + 36: [53, 232, 371, 325, 369, 372, 73, 93, 112, 9, 105, 298, 174, 228, 0, 257, 119, 208, 204, 77, 316, 378, 278, 218, 145, 279, 53, 232, 371, 325, 274, 372, 97, 32, 309, 231, 122, 183, 366, 100, 30, 38, 351, 135, 329, 248, 35, 253, 154, 269, 160, 123, 131, 348, 238, 299, 196, 223, 308, 85, 240, 139, 361, 15, 168, 237, 264, 81, 358, 356, 363, 251, 256, 8, 7, 181, 221, 170, 192, 304, 10, 328, 98, 223, 47, 287, 111, 191, 63, 342, 147, 41, 200, 344, 267, 48, 209, 381, 347, 213, 161, 306, 113, 25, 252, 304, 359, 368, 193, 225, 207, 42, 321, 166, 103, 322, 171, 102, 310, 17, 165, 58, 281, 249, 357, 350, 285, 82, 367, 375, 294, 250, 59, 368, 178, 373, 133, 233, 262, 194, 167, 339, 153, 202, 229, 104, 220, 318, 158, 236, 21, 87, 26, 222, 114, 313, 22, 84, 2, 371, 40, 56, 352, 156, 175, 349, 95, 69, 346, 46, 305, 379, 148, 29, 83, 315, 307, 355, 34, 124, 43, 16, 22, 57, 266, 272, 320, 380, 152, 45, 333, 282, 132, 20, 72, 364, 134, 335, 188, 146, 70, 259, 14, 226, 330, 141, 377, 360, 22, 57, 185, 311, 241, 12, 374, 331, 162, 49, 138, 276, 216, 66, 211, 206, 88, 283, 247, 199, 130, 61, 96, 227, 186, 290, 197, 144, 62, 311, 296, 301, 5, 332, 67, 345, 297, 289, 117, 343, 39, 52, 78, 243, 6, 169, 118, 254, 128, 327, 270, 292, 370, 144, 232, 272, 326, 12, 275, 261, 201, 319, 284, 172, 140, 11, 71, 143, 129, 365, 295, 19, 337, 324, 203, 136, 234, 273, 382, 184, 266, 272, 242, 99, 150, 65, 33, 55, 91, 164, 271, 224, 180, 76, 23, 159, 109, 173, 60, 126, 86, 4, 268, 64, 74, 184, 232, 230, 258, 107, 92, 125, 362, 293, 13, 155, 280, 195, 217, 28, 149, 51, 190, 265, 120, 115, 303, 286, 182, 336, 340, 341, 10, 368, 193, 225, 291, 31, 108, 219, 121, 300, 142, 116, 187, 212, 80, 163, 189, 198, 215, 235, 151, 106, 376, 127, 340, 184, 232, 260, 210, 107, 205, 54, 246, 94, 68, 50, 179, 177, 255, 36, 263, 18, 24, 338, 137, 312, 1, 353, 317, 37, 176, 341, 238, 101, 178, 90, 79, 314, 334, 288, 245, 3, 354, 277, 323, 157, 244, 239, 44, 383, 75, 110, 27, 89, 214, 302] + 37: [109, 379, 148, 200, 362, 382, 59, 353, 343, 192, 229, 168, 36, 135, 83, 121, 274, 4, 368, 378, 35, 96, 20, 307, 84, 75, 109, 379, 148, 200, 341, 162, 302, 360, 118, 18, 51, 27, 161, 278, 331, 262, 24, 105, 215, 260, 63, 248, 177, 367, 68, 56, 188, 123, 148, 293, 72, 349, 140, 327, 328, 336, 104, 173, 30, 111, 247, 337, 280, 354, 10, 303, 142, 357, 300, 186, 276, 132, 240, 273, 311, 363, 366, 340, 0, 88, 233, 164, 19, 203, 217, 376, 126, 321, 38, 42, 91, 332, 81, 106, 180, 89, 238, 112, 268, 218, 213, 366, 289, 113, 86, 22, 117, 150, 127, 6, 296, 34, 49, 380, 317, 5, 110, 350, 166, 62, 28, 90, 232, 236, 268, 218, 314, 222, 272, 95, 355, 245, 131, 257, 287, 144, 338, 29, 136, 244, 271, 298, 124, 78, 101, 146, 11, 220, 251, 383, 268, 218, 151, 125, 179, 340, 361, 65, 147, 157, 76, 64, 141, 285, 87, 373, 102, 210, 17, 60, 191, 184, 45, 219, 99, 239, 319, 218, 309, 46, 372, 234, 47, 185, 32, 330, 258, 176, 39, 264, 334, 80, 279, 267, 53, 170, 198, 225, 54, 369, 209, 288, 301, 218, 265, 293, 79, 292, 344, 7, 153, 133, 122, 74, 143, 108, 224, 269, 241, 249, 299, 291, 310, 214, 227, 250, 67, 8, 301, 218, 348, 178, 305, 371, 156, 165, 235, 145, 208, 149, 41, 134, 335, 347, 313, 26, 138, 183, 259, 57, 119, 381, 25, 31, 71, 294, 242, 306, 342, 162, 171, 9, 181, 345, 320, 365, 61, 3, 356, 312, 97, 154, 160, 115, 295, 70, 202, 159, 377, 352, 364, 218, 158, 222, 48, 113, 0, 182, 93, 359, 370, 43, 230, 329, 100, 255, 94, 187, 256, 175, 374, 107, 281, 246, 358, 226, 66, 294, 158, 293, 44, 77, 223, 318, 139, 322, 21, 270, 92, 333, 69, 37, 33, 325, 266, 286, 283, 231, 197, 12, 282, 375, 40, 40, 58, 199, 190, 174, 16, 252, 163, 277, 228, 1, 73, 201, 55, 207, 284, 351, 152, 216, 308, 130, 155, 2, 50, 189, 40, 40, 243, 52, 290, 349, 114, 14, 204, 315, 85, 196, 304, 103, 206, 128, 212, 172, 13, 169, 297, 237, 326, 275, 23, 263, 40, 294, 58, 205, 79, 193, 129, 253, 194, 221, 316, 323, 82, 15, 167, 98, 195, 116, 254, 137, 339, 324, 261, 211, 346, 120] + 38: [282, 245, 332, 8, 383, 177, 217, 317, 233, 222, 239, 224, 367, 298, 356, 376, 310, 360, 204, 311, 202, 113, 232, 196, 312, 52, 10, 266, 123, 372, 154, 293, 19, 119, 53, 99, 73, 58, 57, 229, 28, 76, 15, 195, 347, 283, 102, 335, 209, 365, 121, 340, 4, 266, 203, 292, 115, 230, 63, 267, 329, 106, 95, 36, 349, 276, 228, 181, 152, 172, 382, 351, 290, 65, 350, 1, 140, 185, 158, 221, 203, 372, 154, 62, 171, 257, 338, 355, 29, 144, 184, 109, 141, 272, 337, 182, 375, 294, 299, 254, 175, 198, 64, 161, 158, 266, 118, 366, 274, 333, 17, 54, 97, 147, 67, 193, 142, 48, 306, 41, 2, 79, 11, 297, 3, 164, 128, 22, 162, 237, 187, 221, 66, 285, 189, 89, 145, 135, 137, 90, 321, 307, 165, 34, 241, 180, 200, 238, 82, 61, 138, 24, 163, 336, 159, 339, 6, 369, 124, 30, 126, 220, 381, 206, 199, 125, 9, 262, 216, 37, 148, 174, 103, 371, 104, 168, 343, 320, 291, 83, 330, 325, 6, 178, 269, 51, 114, 32, 244, 212, 288, 107, 271, 358, 156, 179, 31, 242, 23, 296, 215, 44, 12, 192, 116, 281, 370, 380, 6, 348, 203, 284, 277, 62, 130, 253, 319, 146, 227, 362, 96, 7, 117, 258, 131, 43, 56, 13, 55, 112, 20, 377, 98, 176, 6, 345, 94, 366, 189, 309, 278, 344, 331, 352, 208, 68, 323, 342, 265, 353, 40, 313, 231, 80, 183, 364, 314, 127, 318, 46, 223, 50, 201, 284, 248, 39, 379, 300, 286, 341, 59, 105, 264, 188, 260, 33, 139, 74, 273, 35, 71, 157, 132, 160, 133, 101, 93, 378, 124, 51, 114, 49, 92, 279, 327, 219, 21, 45, 256, 151, 5, 149, 78, 122, 120, 186, 247, 42, 259, 155, 346, 60, 301, 301, 201, 354, 322, 333, 166, 268, 108, 75, 69, 211, 88, 143, 170, 26, 81, 214, 100, 374, 236, 240, 153, 218, 249, 251, 301, 213, 280, 30, 126, 277, 373, 328, 361, 275, 85, 359, 136, 368, 84, 197, 250, 334, 25, 295, 357, 91, 18, 110, 304, 38, 301, 287, 280, 86, 383, 89, 27, 77, 234, 14, 324, 150, 308, 111, 190, 134, 173, 302, 252, 47, 70, 316, 167, 303, 243, 191, 301, 378, 269, 263, 226, 194, 0, 169, 16, 225, 72, 87, 255, 246, 289, 305, 235, 129, 261, 205, 363, 326, 207, 315, 270, 210] + 39: [43, 310, 286, 315, 132, 86, 329, 369, 208, 274, 48, 190, 371, 317, 102, 97, 128, 207, 343, 107, 337, 92, 339, 73, 332, 44, 100, 310, 236, 312, 360, 85, 80, 24, 77, 53, 54, 331, 150, 307, 228, 173, 275, 194, 216, 103, 246, 37, 210, 124, 115, 304, 176, 310, 223, 13, 132, 187, 277, 295, 381, 298, 251, 137, 276, 213, 94, 289, 309, 60, 120, 189, 83, 252, 158, 32, 11, 21, 247, 104, 25, 56, 242, 382, 296, 300, 23, 8, 140, 74, 281, 364, 72, 241, 30, 377, 70, 288, 305, 64, 129, 125, 29, 198, 247, 104, 153, 56, 279, 272, 71, 250, 81, 170, 232, 34, 62, 161, 134, 354, 79, 306, 58, 19, 347, 197, 147, 256, 90, 257, 247, 82, 310, 218, 313, 379, 297, 341, 51, 118, 122, 336, 50, 179, 355, 133, 230, 327, 245, 168, 20, 0, 136, 205, 201, 334, 99, 104, 372, 315, 36, 119, 374, 141, 146, 227, 323, 101, 199, 63, 293, 45, 254, 169, 171, 41, 206, 123, 351, 308, 57, 89, 282, 116, 1, 113, 93, 85, 362, 330, 290, 273, 28, 138, 135, 322, 311, 95, 195, 220, 192, 335, 160, 18, 349, 12, 344, 143, 282, 258, 236, 215, 38, 2, 145, 15, 365, 259, 127, 378, 126, 226, 212, 320, 164, 183, 291, 117, 46, 10, 255, 262, 356, 22, 282, 258, 368, 312, 93, 292, 270, 319, 3, 42, 191, 47, 353, 301, 373, 193, 4, 363, 27, 196, 108, 253, 326, 105, 184, 52, 282, 234, 372, 65, 314, 155, 333, 76, 163, 352, 248, 131, 152, 249, 33, 203, 221, 181, 166, 222, 157, 238, 284, 318, 186, 321, 96, 110, 348, 139, 38, 244, 174, 178, 49, 375, 225, 239, 358, 75, 14, 285, 338, 214, 376, 111, 55, 39, 342, 224, 261, 165, 280, 66, 286, 151, 313, 328, 324, 40, 200, 235, 299, 260, 367, 345, 182, 88, 6, 68, 109, 346, 231, 142, 350, 237, 269, 149, 35, 66, 156, 56, 325, 379, 204, 98, 7, 383, 340, 69, 188, 5, 61, 185, 16, 240, 217, 287, 266, 229, 219, 67, 380, 209, 180, 66, 236, 65, 36, 267, 172, 17, 59, 202, 31, 263, 366, 144, 106, 167, 130, 265, 91, 303, 175, 211, 370, 316, 154, 9, 268, 264, 372, 315, 314, 271, 26, 121, 87, 84, 112, 278, 177, 162, 302, 357, 243, 114, 283, 159, 78, 359, 233, 294, 148, 361] + 40: [31, 286, 358, 55, 21, 153, 251, 159, 143, 289, 278, 307, 309, 299, 93, 333, 233, 246, 212, 310, 174, 119, 152, 255, 312, 353, 31, 349, 204, 378, 265, 158, 74, 228, 136, 274, 205, 46, 365, 94, 165, 171, 273, 261, 276, 0, 40, 104, 39, 157, 83, 105, 31, 349, 71, 175, 52, 346, 3, 90, 219, 80, 38, 227, 57, 110, 192, 330, 131, 139, 361, 23, 295, 34, 354, 320, 191, 248, 31, 360, 71, 175, 247, 346, 208, 351, 326, 277, 216, 262, 364, 306, 264, 356, 345, 222, 84, 348, 163, 242, 154, 292, 331, 170, 31, 360, 358, 175, 108, 226, 224, 100, 10, 162, 199, 239, 99, 151, 29, 206, 301, 118, 116, 32, 140, 173, 342, 298, 344, 5, 31, 169, 70, 190, 283, 88, 254, 9, 168, 341, 188, 229, 263, 2, 7, 48, 304, 6, 121, 383, 182, 111, 4, 45, 220, 300, 132, 250, 359, 378, 108, 226, 198, 59, 271, 63, 12, 382, 138, 377, 327, 177, 293, 291, 78, 184, 257, 218, 352, 86, 225, 376, 379, 250, 68, 75, 179, 156, 258, 17, 129, 322, 236, 370, 82, 244, 324, 323, 47, 241, 141, 124, 66, 106, 234, 112, 146, 144, 317, 15, 68, 373, 297, 128, 372, 347, 284, 166, 30, 137, 53, 65, 318, 305, 314, 147, 270, 49, 368, 180, 315, 223, 126, 123, 317, 15, 68, 367, 33, 107, 332, 210, 303, 27, 193, 56, 79, 43, 167, 371, 366, 375, 115, 135, 203, 134, 267, 172, 20, 194, 357, 15, 142, 101, 21, 164, 232, 311, 362, 211, 145, 269, 41, 130, 120, 329, 51, 238, 186, 13, 374, 325, 380, 245, 197, 149, 207, 183, 328, 67, 160, 215, 369, 275, 363, 11, 54, 44, 187, 189, 339, 243, 350, 201, 14, 125, 77, 35, 114, 24, 42, 249, 207, 260, 213, 98, 337, 8, 235, 91, 296, 253, 308, 196, 150, 355, 155, 237, 37, 85, 288, 268, 117, 22, 96, 252, 240, 69, 340, 183, 328, 378, 26, 279, 64, 1, 89, 185, 334, 259, 316, 290, 338, 281, 18, 76, 133, 102, 113, 285, 109, 381, 335, 92, 280, 319, 142, 190, 179, 161, 58, 28, 336, 61, 200, 148, 95, 103, 294, 195, 19, 73, 256, 266, 230, 217, 313, 272, 127, 214, 319, 319, 181, 62, 33, 202, 81, 97, 321, 209, 221, 302, 178, 25, 287, 282, 60, 36, 122, 50, 16, 231, 87, 176, 72, 343] + 41: [205, 142, 170, 255, 84, 285, 136, 253, 4, 239, 345, 241, 294, 328, 363, 213, 220, 342, 191, 48, 257, 329, 121, 372, 135, 120, 305, 199, 355, 190, 366, 76, 34, 44, 80, 243, 217, 55, 156, 133, 176, 5, 371, 292, 1, 141, 209, 14, 2, 162, 230, 85, 122, 199, 376, 190, 168, 92, 187, 306, 45, 140, 174, 93, 258, 210, 151, 261, 244, 33, 143, 72, 182, 47, 183, 361, 193, 40, 321, 11, 251, 30, 369, 159, 24, 12, 9, 353, 134, 226, 289, 330, 181, 160, 315, 188, 236, 207, 175, 370, 323, 377, 164, 152, 321, 199, 142, 86, 369, 327, 161, 302, 222, 264, 301, 379, 344, 112, 295, 130, 74, 300, 165, 13, 155, 245, 331, 16, 58, 263, 185, 71, 307, 374, 223, 119, 279, 206, 3, 202, 56, 54, 325, 380, 107, 277, 192, 15, 78, 248, 113, 171, 201, 362, 382, 309, 49, 114, 350, 90, 129, 159, 314, 262, 318, 126, 336, 83, 322, 268, 299, 317, 128, 75, 287, 149, 316, 17, 62, 109, 281, 278, 49, 114, 350, 184, 366, 96, 364, 347, 53, 42, 102, 313, 340, 51, 131, 94, 173, 272, 352, 38, 310, 20, 139, 346, 169, 22, 49, 166, 355, 57, 197, 124, 7, 82, 127, 36, 308, 276, 271, 266, 66, 6, 221, 99, 381, 158, 98, 29, 157, 312, 333, 132, 269, 196, 60, 374, 368, 119, 69, 95, 348, 23, 116, 282, 115, 250, 153, 18, 89, 59, 0, 259, 137, 67, 145, 189, 228, 178, 269, 103, 27, 249, 365, 215, 358, 280, 35, 219, 349, 254, 237, 8, 63, 43, 61, 154, 77, 320, 204, 375, 88, 177, 291, 357, 298, 378, 60, 212, 50, 97, 111, 311, 326, 194, 32, 225, 167, 224, 186, 81, 303, 41, 179, 68, 147, 274, 110, 341, 118, 31, 298, 378, 60, 374, 129, 275, 360, 334, 104, 218, 26, 256, 229, 339, 284, 19, 146, 232, 65, 297, 28, 265, 240, 211, 233, 247, 283, 335, 350, 184, 200, 293, 79, 198, 324, 286, 70, 125, 235, 288, 101, 208, 354, 359, 39, 105, 73, 238, 319, 270, 52, 227, 283, 378, 170, 30, 273, 275, 21, 108, 383, 163, 252, 260, 138, 242, 338, 246, 214, 37, 100, 332, 148, 231, 267, 356, 304, 296, 87, 367, 170, 150, 197, 215, 172, 337, 351, 25, 64, 46, 117, 144, 290, 91, 10, 123, 180, 234, 195, 373, 343, 106, 203, 216] + 42: [2, 152, 117, 218, 325, 333, 203, 189, 328, 17, 167, 33, 52, 8, 89, 139, 5, 125, 26, 73, 312, 153, 47, 14, 0, 242, 188, 152, 117, 247, 158, 41, 263, 127, 274, 67, 147, 77, 65, 32, 246, 283, 291, 378, 272, 197, 124, 22, 69, 43, 361, 254, 210, 152, 117, 212, 264, 162, 262, 277, 370, 230, 87, 202, 257, 241, 371, 226, 205, 346, 56, 250, 288, 367, 249, 78, 13, 294, 154, 152, 31, 218, 93, 243, 377, 373, 4, 169, 216, 164, 227, 340, 180, 315, 332, 53, 194, 363, 3, 302, 115, 142, 24, 352, 375, 228, 31, 350, 151, 168, 129, 265, 354, 187, 326, 314, 149, 80, 219, 49, 16, 156, 120, 368, 136, 330, 79, 54, 240, 324, 359, 148, 119, 116, 63, 138, 208, 342, 275, 344, 338, 38, 160, 245, 323, 157, 7, 269, 206, 199, 334, 383, 100, 107, 91, 95, 359, 114, 31, 211, 158, 374, 306, 191, 141, 287, 165, 280, 39, 60, 372, 244, 336, 68, 293, 104, 185, 85, 11, 331, 163, 369, 20, 295, 268, 176, 256, 37, 255, 327, 358, 28, 173, 234, 76, 132, 307, 183, 61, 278, 229, 267, 304, 144, 110, 81, 365, 339, 1, 266, 345, 176, 256, 179, 6, 319, 178, 140, 161, 45, 118, 137, 238, 99, 298, 233, 290, 321, 23, 347, 353, 317, 40, 46, 217, 94, 305, 155, 282, 349, 108, 379, 30, 236, 356, 92, 381, 51, 237, 310, 196, 170, 284, 102, 75, 174, 190, 177, 252, 126, 217, 266, 268, 172, 111, 195, 220, 235, 341, 135, 103, 322, 380, 113, 225, 130, 204, 270, 213, 10, 186, 207, 276, 271, 273, 27, 50, 94, 303, 172, 111, 337, 200, 351, 29, 224, 201, 366, 166, 19, 311, 299, 258, 300, 318, 82, 128, 133, 12, 159, 66, 121, 42, 123, 345, 155, 72, 222, 55, 348, 320, 239, 36, 181, 192, 18, 62, 109, 313, 171, 261, 98, 296, 96, 253, 382, 289, 71, 123, 123, 117, 93, 215, 138, 59, 21, 221, 259, 329, 316, 223, 301, 9, 97, 58, 112, 343, 360, 198, 309, 35, 182, 74, 193, 123, 123, 357, 279, 325, 333, 209, 281, 131, 286, 308, 335, 297, 101, 231, 232, 34, 364, 84, 145, 285, 88, 70, 44, 15, 376, 123, 123, 106, 184, 150, 37, 48, 122, 248, 214, 105, 143, 362, 146, 64, 260, 251, 355, 175, 90, 292, 134, 57, 83, 86, 25] + 43: [164, 375, 142, 35, 14, 298, 235, 203, 321, 379, 232, 123, 34, 194, 47, 217, 309, 329, 71, 173, 187, 285, 236, 270, 302, 169, 369, 375, 17, 66, 283, 297, 93, 223, 224, 32, 327, 219, 84, 107, 102, 284, 370, 175, 126, 193, 330, 230, 21, 362, 184, 214, 27, 308, 197, 112, 349, 30, 186, 127, 136, 255, 282, 271, 149, 196, 293, 70, 9, 292, 353, 381, 355, 162, 141, 383, 376, 182, 344, 225, 371, 359, 39, 100, 81, 342, 338, 191, 154, 26, 160, 28, 49, 167, 343, 159, 356, 209, 357, 69, 7, 58, 289, 348, 315, 225, 335, 98, 75, 351, 23, 313, 274, 77, 380, 339, 67, 63, 185, 46, 325, 215, 279, 119, 316, 341, 172, 96, 212, 245, 315, 225, 17, 66, 20, 103, 152, 322, 280, 85, 378, 140, 5, 155, 183, 143, 220, 122, 65, 134, 290, 248, 41, 139, 244, 373, 262, 347, 129, 66, 14, 43, 326, 264, 78, 250, 238, 306, 365, 145, 24, 120, 226, 18, 132, 151, 188, 114, 328, 333, 231, 195, 82, 347, 335, 359, 239, 366, 153, 301, 1, 2, 88, 83, 130, 334, 165, 45, 125, 64, 8, 118, 211, 124, 0, 40, 310, 192, 181, 332, 218, 59, 254, 131, 57, 178, 202, 374, 346, 200, 36, 148, 38, 266, 174, 116, 259, 268, 179, 312, 228, 42, 350, 3, 168, 156, 372, 364, 283, 298, 291, 138, 108, 73, 204, 234, 257, 243, 50, 16, 263, 72, 76, 247, 110, 260, 303, 133, 33, 382, 168, 156, 372, 37, 13, 94, 337, 111, 305, 150, 275, 286, 157, 345, 60, 251, 288, 170, 163, 158, 265, 276, 62, 15, 319, 354, 168, 177, 92, 98, 190, 314, 22, 227, 307, 272, 52, 135, 296, 237, 180, 210, 128, 11, 117, 287, 300, 311, 281, 340, 361, 206, 6, 171, 92, 98, 97, 297, 377, 29, 198, 222, 48, 86, 61, 12, 242, 74, 256, 295, 144, 233, 336, 53, 317, 104, 25, 331, 221, 105, 68, 216, 75, 351, 23, 318, 269, 4, 246, 277, 79, 10, 113, 56, 101, 304, 87, 19, 320, 199, 137, 352, 54, 189, 221, 171, 278, 324, 190, 43, 363, 360, 121, 89, 205, 115, 261, 273, 99, 213, 80, 31, 294, 358, 258, 51, 249, 208, 166, 44, 161, 105, 375, 324, 252, 30, 55, 323, 147, 368, 90, 299, 367, 241, 229, 253, 146, 207, 267, 109, 240, 201, 106, 91, 176, 95] + 44: [362, 191, 10, 101, 257, 170, 98, 38, 358, 262, 244, 39, 71, 44, 220, 370, 318, 20, 203, 62, 2, 373, 69, 127, 208, 303, 362, 191, 10, 223, 180, 311, 236, 141, 291, 265, 24, 154, 160, 202, 344, 92, 205, 32, 193, 256, 216, 22, 176, 361, 16, 161, 274, 301, 379, 223, 21, 331, 34, 333, 337, 121, 85, 201, 132, 369, 354, 278, 59, 305, 230, 137, 245, 251, 372, 37, 88, 106, 274, 301, 379, 155, 190, 229, 30, 182, 95, 207, 350, 364, 79, 221, 327, 40, 323, 3, 241, 183, 253, 119, 298, 293, 148, 116, 168, 301, 10, 101, 267, 252, 313, 142, 228, 94, 186, 122, 248, 54, 107, 185, 19, 336, 345, 367, 375, 233, 302, 73, 133, 189, 238, 300, 382, 348, 43, 277, 320, 329, 225, 324, 299, 173, 338, 96, 292, 282, 347, 81, 319, 321, 1, 27, 268, 52, 281, 295, 97, 178, 199, 60, 84, 68, 86, 249, 125, 343, 290, 219, 378, 67, 26, 15, 149, 156, 357, 204, 45, 239, 307, 152, 75, 209, 64, 272, 222, 284, 227, 368, 93, 118, 144, 355, 254, 55, 164, 381, 46, 83, 243, 218, 72, 51, 255, 50, 246, 123, 353, 374, 172, 77, 317, 348, 82, 346, 280, 306, 33, 117, 184, 100, 131, 261, 342, 334, 377, 169, 9, 314, 326, 153, 330, 111, 308, 112, 172, 77, 317, 348, 145, 346, 383, 264, 48, 89, 296, 349, 380, 269, 126, 147, 270, 237, 234, 7, 124, 99, 240, 47, 316, 322, 171, 25, 351, 226, 145, 279, 143, 294, 102, 340, 61, 163, 286, 174, 162, 74, 105, 104, 57, 49, 35, 151, 363, 297, 197, 129, 171, 77, 110, 226, 29, 328, 146, 198, 275, 158, 192, 287, 66, 359, 136, 196, 214, 113, 263, 65, 312, 210, 283, 259, 76, 332, 167, 213, 222, 271, 188, 140, 166, 289, 360, 138, 266, 56, 285, 134, 41, 108, 8, 179, 235, 114, 42, 115, 28, 356, 310, 12, 0, 213, 199, 284, 215, 135, 288, 242, 304, 276, 175, 150, 366, 177, 157, 165, 231, 23, 250, 232, 200, 5, 103, 335, 371, 13, 17, 213, 199, 226, 376, 170, 187, 63, 365, 91, 6, 70, 130, 120, 128, 90, 78, 341, 18, 53, 80, 195, 260, 31, 181, 339, 258, 213, 191, 317, 21, 325, 11, 212, 217, 14, 224, 315, 352, 87, 36, 159, 309, 211, 109, 4, 273, 194, 247, 139, 58, 206] + 45: [154, 293, 10, 274, 103, 47, 383, 275, 55, 74, 327, 108, 175, 253, 133, 290, 211, 314, 15, 3, 337, 226, 270, 77, 181, 361, 154, 284, 178, 227, 227, 329, 124, 331, 223, 308, 12, 179, 32, 370, 344, 5, 98, 169, 203, 147, 243, 232, 200, 259, 72, 244, 324, 293, 201, 141, 85, 276, 383, 351, 168, 104, 210, 345, 24, 302, 205, 87, 231, 285, 309, 364, 136, 97, 353, 335, 17, 355, 23, 161, 346, 227, 375, 329, 348, 248, 45, 88, 68, 217, 106, 218, 58, 320, 150, 127, 363, 202, 90, 153, 216, 237, 11, 64, 23, 161, 182, 27, 160, 164, 95, 39, 187, 277, 306, 41, 208, 222, 330, 38, 37, 263, 268, 371, 225, 143, 159, 311, 14, 117, 191, 317, 201, 247, 350, 131, 212, 234, 16, 51, 7, 242, 236, 22, 207, 145, 29, 342, 341, 343, 125, 89, 366, 75, 60, 300, 173, 110, 197, 27, 262, 269, 81, 228, 239, 115, 135, 86, 365, 257, 35, 256, 379, 271, 0, 362, 189, 280, 53, 28, 235, 347, 121, 92, 178, 356, 146, 180, 114, 380, 326, 229, 80, 281, 109, 261, 170, 264, 283, 151, 312, 78, 91, 167, 206, 6, 49, 83, 121, 92, 178, 339, 367, 164, 165, 254, 359, 30, 313, 260, 26, 340, 19, 252, 322, 374, 220, 215, 102, 18, 101, 99, 240, 266, 301, 92, 334, 176, 373, 286, 297, 258, 307, 70, 54, 94, 279, 63, 65, 59, 294, 381, 289, 113, 111, 186, 194, 278, 40, 137, 318, 92, 334, 171, 373, 354, 349, 209, 310, 250, 84, 44, 292, 43, 76, 358, 9, 57, 48, 368, 238, 82, 193, 13, 299, 192, 219, 92, 42, 8, 369, 61, 163, 183, 122, 338, 112, 156, 315, 50, 20, 319, 333, 129, 246, 73, 214, 56, 265, 172, 69, 336, 140, 134, 139, 10, 33, 276, 128, 224, 162, 372, 328, 157, 67, 376, 34, 46, 241, 79, 166, 71, 323, 382, 352, 325, 195, 221, 140, 144, 25, 10, 33, 378, 272, 52, 196, 245, 251, 213, 96, 4, 230, 296, 282, 288, 142, 93, 332, 152, 305, 287, 185, 138, 140, 316, 42, 8, 350, 298, 81, 107, 158, 155, 357, 360, 1, 123, 126, 291, 273, 177, 105, 377, 132, 62, 148, 66, 130, 100, 233, 188, 199, 120, 367, 21, 95, 118, 198, 36, 267, 249, 2, 303, 116, 304, 255, 204, 119, 190, 321, 31, 295, 184, 149, 174] + 46: [152, 314, 380, 207, 367, 344, 187, 242, 280, 33, 186, 166, 164, 340, 383, 92, 318, 317, 370, 278, 52, 229, 107, 47, 70, 5, 188, 58, 355, 110, 149, 301, 194, 320, 62, 108, 197, 42, 162, 158, 45, 206, 69, 342, 261, 257, 138, 281, 196, 54, 200, 27, 142, 256, 253, 170, 354, 319, 378, 18, 266, 227, 326, 298, 176, 103, 226, 360, 222, 265, 91, 285, 21, 77, 300, 349, 14, 147, 142, 256, 253, 16, 149, 335, 271, 203, 131, 115, 118, 156, 273, 17, 111, 263, 382, 241, 359, 26, 163, 341, 269, 123, 357, 36, 142, 58, 327, 207, 354, 66, 254, 44, 215, 376, 218, 180, 272, 198, 81, 160, 313, 7, 291, 171, 150, 31, 363, 0, 100, 213, 133, 139, 343, 311, 284, 104, 165, 345, 79, 83, 334, 369, 23, 59, 125, 35, 346, 20, 364, 375, 67, 330, 251, 179, 178, 231, 133, 139, 343, 249, 324, 319, 60, 374, 351, 195, 232, 316, 304, 321, 57, 328, 56, 322, 90, 86, 43, 127, 87, 116, 148, 220, 210, 64, 355, 110, 246, 312, 136, 146, 211, 130, 168, 192, 128, 205, 308, 24, 297, 293, 177, 305, 15, 11, 1, 306, 82, 88, 10, 19, 240, 96, 230, 140, 68, 76, 282, 155, 121, 295, 250, 157, 268, 78, 223, 277, 217, 339, 112, 94, 48, 212, 270, 302, 10, 64, 46, 311, 6, 209, 204, 2, 39, 22, 353, 259, 294, 216, 307, 290, 89, 71, 309, 184, 358, 258, 288, 248, 225, 252, 201, 233, 311, 25, 367, 154, 40, 303, 49, 8, 315, 4, 362, 193, 377, 97, 173, 243, 190, 262, 65, 275, 381, 84, 151, 356, 247, 352, 224, 238, 274, 12, 124, 3, 85, 120, 80, 50, 63, 235, 336, 144, 161, 74, 264, 371, 135, 366, 134, 279, 323, 61, 247, 352, 380, 51, 332, 335, 214, 219, 228, 143, 245, 109, 95, 172, 32, 289, 189, 159, 93, 361, 75, 310, 174, 296, 365, 122, 183, 34, 286, 191, 230, 140, 38, 199, 30, 117, 287, 13, 129, 267, 329, 169, 299, 153, 283, 98, 347, 236, 181, 29, 182, 175, 183, 34, 368, 239, 221, 66, 55, 338, 350, 260, 99, 37, 41, 237, 101, 141, 348, 333, 145, 234, 208, 72, 73, 185, 28, 325, 183, 352, 286, 202, 106, 106, 114, 113, 102, 372, 132, 137, 53, 292, 373, 255, 119, 9, 331, 244, 337, 126, 105, 276, 167, 379] + 47: [195, 44, 3, 29, 193, 127, 175, 188, 250, 208, 41, 298, 138, 106, 183, 335, 258, 185, 130, 275, 5, 133, 264, 167, 66, 288, 195, 352, 3, 29, 227, 353, 274, 235, 63, 42, 224, 203, 45, 39, 96, 225, 31, 49, 190, 282, 374, 16, 76, 206, 7, 156, 195, 352, 43, 158, 300, 14, 110, 229, 351, 262, 285, 255, 296, 27, 378, 163, 94, 329, 9, 365, 304, 100, 145, 217, 254, 326, 195, 352, 371, 61, 359, 316, 36, 349, 159, 135, 89, 77, 355, 252, 139, 303, 278, 32, 144, 302, 380, 86, 247, 256, 13, 209, 75, 44, 3, 87, 300, 305, 240, 238, 269, 155, 103, 33, 12, 210, 234, 147, 332, 157, 297, 72, 379, 361, 88, 125, 182, 369, 343, 162, 24, 375, 134, 286, 118, 153, 122, 67, 223, 184, 51, 233, 91, 70, 328, 2, 46, 48, 207, 317, 244, 161, 311, 236, 343, 162, 24, 119, 134, 137, 21, 342, 114, 104, 284, 323, 338, 173, 80, 321, 37, 107, 10, 120, 174, 30, 85, 295, 113, 146, 343, 44, 24, 35, 309, 90, 168, 40, 265, 95, 231, 65, 205, 117, 366, 28, 281, 260, 242, 325, 358, 222, 142, 283, 83, 62, 54, 192, 22, 158, 280, 196, 267, 367, 313, 69, 124, 197, 25, 180, 356, 126, 259, 132, 149, 216, 383, 372, 270, 232, 272, 293, 54, 143, 291, 170, 333, 314, 376, 59, 230, 84, 228, 140, 17, 52, 336, 71, 273, 248, 150, 266, 245, 220, 299, 6, 8, 360, 54, 171, 357, 279, 345, 137, 287, 363, 212, 219, 263, 172, 322, 164, 315, 213, 111, 341, 81, 19, 108, 58, 194, 116, 169, 214, 271, 154, 3, 109, 359, 368, 198, 346, 330, 26, 166, 73, 148, 4, 18, 211, 276, 20, 102, 55, 74, 105, 11, 82, 128, 191, 347, 154, 312, 87, 307, 320, 129, 123, 98, 68, 249, 199, 204, 53, 15, 121, 292, 310, 160, 152, 290, 38, 202, 221, 241, 56, 331, 181, 22, 1, 382, 294, 79, 218, 251, 377, 151, 50, 237, 340, 141, 348, 60, 115, 334, 57, 177, 277, 64, 99, 381, 373, 187, 181, 312, 34, 382, 257, 344, 327, 226, 93, 324, 364, 0, 112, 23, 289, 306, 200, 253, 308, 92, 339, 268, 78, 261, 246, 179, 136, 312, 87, 307, 215, 362, 131, 176, 301, 319, 370, 101, 354, 201, 243, 337, 186, 97, 189, 350, 178, 47, 165, 318, 239] + 48: [237, 145, 215, 226, 295, 58, 287, 139, 41, 232, 25, 120, 212, 347, 70, 138, 18, 98, 32, 310, 278, 244, 86, 251, 131, 200, 267, 65, 358, 44, 203, 78, 62, 363, 34, 312, 175, 73, 89, 218, 22, 36, 269, 59, 216, 92, 87, 309, 106, 296, 336, 209, 267, 117, 273, 239, 101, 33, 166, 240, 342, 205, 193, 340, 329, 91, 64, 96, 352, 137, 243, 194, 294, 238, 377, 54, 303, 308, 159, 77, 109, 19, 311, 28, 157, 254, 142, 300, 246, 88, 118, 318, 100, 72, 53, 144, 187, 3, 56, 225, 331, 332, 199, 245, 297, 102, 273, 276, 353, 133, 6, 5, 252, 319, 152, 233, 357, 168, 14, 49, 349, 346, 256, 326, 323, 188, 60, 183, 197, 136, 288, 102, 330, 239, 90, 381, 207, 208, 184, 320, 257, 370, 111, 315, 281, 119, 127, 169, 272, 29, 165, 148, 154, 383, 265, 304, 9, 270, 330, 116, 2, 141, 147, 344, 181, 368, 291, 285, 24, 4, 351, 204, 280, 94, 261, 214, 277, 350, 125, 46, 348, 114, 51, 26, 328, 234, 311, 78, 15, 359, 115, 290, 192, 112, 242, 360, 180, 97, 43, 292, 71, 67, 163, 149, 171, 176, 220, 250, 283, 128, 279, 132, 182, 268, 189, 177, 305, 229, 179, 75, 31, 42, 339, 81, 198, 274, 178, 95, 213, 228, 345, 23, 364, 334, 283, 128, 301, 99, 353, 141, 375, 223, 380, 378, 299, 190, 170, 79, 202, 11, 84, 107, 82, 366, 335, 66, 8, 262, 355, 129, 76, 271, 328, 236, 52, 83, 15, 372, 325, 156, 7, 123, 341, 57, 317, 146, 367, 35, 39, 313, 27, 361, 282, 248, 333, 10, 76, 271, 328, 343, 52, 381, 104, 327, 134, 155, 50, 247, 221, 230, 113, 158, 16, 151, 47, 69, 37, 356, 153, 124, 167, 1, 20, 271, 140, 132, 284, 21, 62, 38, 316, 210, 322, 45, 255, 13, 379, 253, 211, 195, 108, 135, 61, 201, 298, 68, 306, 314, 249, 17, 140, 132, 284, 85, 263, 48, 185, 222, 130, 0, 160, 275, 241, 289, 40, 206, 173, 93, 143, 376, 260, 365, 258, 30, 266, 17, 150, 121, 74, 85, 263, 264, 186, 219, 161, 324, 162, 231, 307, 63, 374, 259, 382, 217, 105, 110, 338, 122, 103, 196, 266, 17, 140, 276, 101, 164, 369, 235, 354, 371, 293, 174, 321, 172, 80, 302, 12, 55, 286, 373, 224, 191, 362, 126, 337, 227] + 49: [374, 268, 297, 150, 278, 261, 380, 269, 186, 235, 1, 382, 112, 81, 225, 309, 14, 280, 285, 299, 185, 7, 363, 383, 26, 188, 374, 346, 209, 157, 66, 153, 199, 289, 381, 5, 97, 245, 365, 163, 55, 62, 47, 85, 282, 354, 324, 87, 367, 11, 288, 317, 79, 6, 305, 181, 212, 9, 193, 91, 8, 314, 362, 56, 240, 146, 139, 338, 72, 176, 284, 145, 364, 165, 266, 249, 369, 337, 79, 268, 297, 19, 25, 40, 251, 74, 255, 195, 254, 310, 224, 296, 82, 98, 351, 119, 31, 113, 232, 147, 96, 215, 247, 64, 210, 6, 65, 158, 241, 90, 60, 220, 120, 48, 366, 368, 196, 41, 295, 43, 359, 0, 12, 143, 168, 93, 334, 152, 129, 322, 233, 319, 304, 213, 202, 180, 148, 132, 312, 122, 73, 28, 29, 34, 10, 33, 172, 61, 46, 154, 111, 166, 161, 141, 230, 123, 360, 83, 75, 293, 116, 275, 259, 76, 30, 94, 174, 23, 138, 376, 216, 178, 211, 53, 151, 22, 125, 341, 18, 236, 373, 356, 360, 294, 15, 126, 353, 270, 292, 187, 45, 58, 250, 54, 214, 323, 308, 36, 156, 69, 204, 361, 357, 315, 162, 263, 258, 38, 302, 321, 59, 157, 78, 9, 193, 260, 175, 333, 279, 86, 140, 20, 273, 307, 320, 378, 84, 342, 316, 27, 21, 99, 347, 349, 302, 321, 234, 136, 271, 201, 325, 246, 194, 164, 336, 137, 89, 222, 371, 330, 103, 226, 206, 283, 274, 13, 159, 217, 203, 375, 302, 272, 155, 126, 313, 242, 262, 286, 183, 109, 169, 80, 290, 160, 179, 352, 239, 358, 124, 190, 88, 51, 42, 207, 252, 191, 302, 272, 257, 253, 313, 335, 39, 243, 331, 144, 328, 221, 170, 4, 173, 67, 244, 118, 106, 256, 343, 57, 231, 379, 370, 92, 302, 149, 155, 253, 202, 242, 44, 107, 264, 298, 340, 311, 198, 248, 303, 134, 114, 71, 49, 101, 326, 167, 237, 291, 32, 135, 302, 37, 75, 253, 126, 276, 177, 133, 332, 131, 63, 223, 277, 77, 301, 227, 339, 117, 192, 52, 348, 229, 102, 281, 3, 68, 184, 321, 305, 115, 105, 110, 218, 344, 100, 95, 17, 70, 128, 189, 238, 267, 345, 355, 200, 108, 182, 372, 287, 171, 219, 300, 197, 321, 297, 19, 271, 208, 251, 306, 329, 121, 205, 16, 318, 127, 327, 35, 142, 130, 2, 228, 104, 377, 265, 350, 50, 24] + 50: [57, 166, 315, 370, 27, 129, 0, 193, 224, 96, 71, 43, 94, 310, 99, 300, 178, 124, 156, 354, 323, 21, 47, 167, 302, 111, 33, 166, 367, 361, 312, 1, 128, 51, 121, 122, 117, 183, 342, 326, 152, 28, 243, 177, 46, 212, 346, 76, 157, 221, 317, 136, 80, 229, 38, 262, 287, 350, 338, 314, 335, 41, 306, 102, 250, 382, 91, 105, 202, 4, 372, 36, 138, 209, 72, 279, 84, 332, 29, 181, 376, 218, 214, 26, 329, 331, 159, 363, 195, 194, 174, 234, 242, 223, 297, 339, 349, 171, 162, 381, 265, 54, 254, 345, 29, 67, 14, 69, 214, 282, 296, 119, 298, 274, 203, 348, 180, 85, 145, 340, 292, 204, 248, 330, 147, 10, 173, 227, 115, 257, 86, 49, 376, 69, 366, 26, 45, 88, 270, 6, 75, 263, 208, 267, 205, 5, 37, 259, 12, 355, 286, 201, 103, 144, 59, 50, 86, 49, 32, 188, 175, 129, 199, 256, 184, 313, 228, 240, 169, 73, 163, 108, 135, 106, 63, 258, 112, 83, 190, 161, 66, 55, 86, 181, 333, 35, 146, 1, 231, 155, 213, 196, 149, 219, 319, 92, 294, 356, 17, 344, 238, 68, 11, 101, 104, 261, 364, 222, 25, 140, 97, 325, 373, 3, 231, 343, 16, 353, 375, 359, 206, 303, 322, 143, 347, 123, 305, 90, 65, 40, 245, 95, 307, 230, 235, 137, 32, 188, 27, 3, 217, 341, 271, 301, 48, 134, 189, 107, 255, 251, 39, 289, 7, 383, 324, 13, 207, 260, 118, 264, 268, 109, 233, 273, 110, 377, 128, 295, 187, 132, 19, 116, 269, 378, 277, 113, 236, 351, 200, 318, 179, 176, 328, 309, 131, 93, 268, 109, 210, 215, 56, 18, 62, 22, 253, 244, 239, 379, 197, 74, 160, 185, 362, 42, 320, 293, 285, 30, 360, 225, 52, 241, 268, 311, 315, 325, 321, 9, 327, 358, 130, 164, 380, 53, 139, 371, 2, 357, 337, 281, 141, 237, 87, 60, 70, 158, 44, 283, 226, 316, 352, 232, 125, 350, 304, 79, 249, 133, 186, 290, 252, 64, 20, 278, 127, 153, 34, 89, 291, 246, 216, 266, 182, 23, 100, 299, 210, 215, 56, 18, 192, 114, 15, 24, 165, 154, 276, 151, 78, 211, 191, 288, 198, 150, 82, 61, 8, 120, 368, 365, 100, 316, 284, 232, 321, 280, 220, 334, 272, 275, 58, 168, 247, 81, 126, 172, 369, 170, 308, 148, 98, 77, 31, 142, 336, 374] + 51: [322, 206, 323, 102, 356, 140, 87, 168, 346, 118, 344, 86, 351, 119, 3, 317, 213, 80, 307, 25, 19, 88, 120, 302, 21, 327, 108, 178, 186, 287, 337, 126, 124, 188, 130, 220, 81, 254, 59, 309, 236, 265, 315, 51, 159, 197, 230, 215, 376, 63, 225, 2, 31, 110, 99, 102, 122, 229, 60, 283, 92, 259, 331, 107, 366, 368, 10, 84, 196, 191, 78, 53, 15, 296, 367, 179, 76, 138, 137, 200, 186, 284, 337, 126, 95, 330, 243, 23, 382, 341, 190, 34, 48, 205, 115, 223, 319, 43, 184, 199, 174, 29, 22, 226, 137, 200, 186, 102, 127, 343, 40, 129, 11, 294, 47, 359, 222, 204, 224, 306, 45, 379, 55, 58, 57, 354, 227, 71, 256, 202, 373, 6, 24, 17, 228, 229, 203, 105, 263, 46, 161, 345, 79, 321, 362, 289, 312, 338, 363, 112, 218, 201, 0, 187, 173, 249, 373, 110, 24, 287, 136, 300, 301, 250, 246, 281, 308, 74, 192, 91, 164, 324, 349, 328, 298, 336, 214, 14, 277, 117, 36, 255, 375, 210, 208, 12, 144, 358, 260, 233, 269, 85, 303, 237, 181, 207, 234, 320, 381, 89, 353, 109, 1, 4, 133, 293, 155, 139, 375, 101, 186, 113, 264, 82, 286, 177, 49, 143, 212, 56, 18, 160, 50, 311, 334, 27, 93, 378, 66, 9, 70, 169, 267, 261, 106, 210, 357, 52, 180, 65, 286, 153, 5, 8, 292, 279, 42, 251, 195, 364, 297, 350, 340, 339, 96, 39, 128, 274, 131, 121, 221, 372, 111, 193, 72, 67, 152, 253, 134, 35, 271, 268, 360, 166, 44, 313, 100, 54, 333, 238, 68, 257, 305, 147, 150, 290, 221, 372, 16, 123, 144, 82, 32, 304, 247, 13, 285, 380, 209, 90, 239, 262, 258, 194, 278, 316, 116, 272, 154, 318, 244, 156, 189, 372, 24, 182, 342, 163, 270, 146, 314, 216, 245, 198, 219, 252, 75, 171, 332, 30, 242, 240, 148, 28, 355, 175, 231, 291, 73, 37, 24, 17, 342, 211, 235, 325, 103, 374, 97, 172, 310, 370, 41, 157, 365, 170, 273, 61, 83, 7, 77, 185, 377, 326, 241, 371, 323, 12, 64, 62, 135, 282, 26, 299, 275, 142, 280, 248, 361, 69, 94, 114, 145, 232, 149, 158, 132, 266, 347, 348, 241, 37, 98, 208, 176, 163, 104, 295, 335, 20, 183, 162, 329, 33, 141, 369, 383, 288, 276, 167, 125, 165, 217, 352, 38, 151] + 52: [52, 267, 71, 137, 345, 169, 68, 357, 98, 309, 5, 296, 67, 3, 324, 326, 242, 244, 150, 143, 96, 358, 203, 65, 376, 305, 186, 99, 71, 137, 206, 163, 205, 128, 54, 171, 74, 223, 75, 53, 333, 350, 58, 224, 198, 188, 343, 138, 329, 144, 373, 247, 327, 99, 71, 73, 321, 48, 288, 196, 32, 379, 127, 23, 364, 319, 55, 45, 177, 251, 249, 49, 335, 232, 185, 12, 199, 204, 121, 99, 246, 97, 271, 298, 13, 192, 266, 291, 105, 315, 6, 301, 287, 27, 383, 84, 220, 118, 344, 363, 9, 202, 272, 16, 92, 99, 157, 104, 214, 90, 382, 259, 64, 228, 221, 281, 280, 72, 22, 235, 352, 238, 47, 28, 340, 275, 158, 0, 24, 176, 102, 381, 157, 165, 290, 209, 62, 245, 95, 312, 222, 173, 134, 113, 213, 93, 89, 282, 179, 356, 313, 229, 314, 15, 86, 348, 102, 320, 157, 362, 289, 82, 41, 56, 252, 133, 339, 160, 233, 154, 66, 40, 277, 194, 230, 380, 4, 353, 131, 159, 264, 212, 102, 371, 151, 104, 85, 184, 183, 219, 210, 37, 117, 178, 369, 361, 374, 161, 2, 294, 347, 197, 111, 147, 132, 366, 254, 38, 307, 381, 334, 104, 323, 25, 7, 269, 355, 108, 256, 215, 116, 17, 69, 109, 110, 59, 368, 141, 43, 274, 187, 19, 295, 46, 120, 381, 334, 103, 34, 211, 33, 80, 236, 325, 262, 83, 243, 76, 279, 268, 57, 365, 129, 101, 18, 293, 42, 273, 360, 218, 231, 381, 334, 250, 195, 25, 78, 168, 87, 257, 191, 124, 248, 115, 337, 308, 237, 148, 136, 156, 126, 304, 79, 11, 164, 106, 332, 217, 334, 103, 174, 193, 172, 10, 149, 155, 285, 30, 367, 351, 208, 306, 200, 21, 139, 322, 226, 276, 297, 241, 1, 370, 61, 283, 130, 97, 239, 263, 60, 140, 354, 35, 234, 341, 292, 153, 316, 225, 270, 152, 51, 278, 81, 112, 36, 122, 240, 375, 216, 283, 182, 97, 239, 209, 346, 260, 8, 88, 286, 359, 300, 331, 39, 253, 284, 207, 330, 310, 299, 135, 261, 142, 317, 180, 107, 107, 63, 165, 336, 48, 190, 26, 50, 70, 166, 227, 146, 328, 170, 119, 201, 20, 318, 342, 114, 302, 145, 338, 14, 175, 107, 372, 31, 165, 336, 77, 205, 44, 349, 123, 378, 167, 29, 377, 91, 125, 258, 189, 100, 303, 311, 181, 255, 265, 94, 162] + 53: [230, 247, 296, 354, 347, 32, 157, 162, 50, 262, 346, 52, 60, 146, 128, 327, 278, 97, 195, 43, 29, 163, 374, 187, 318, 185, 336, 73, 38, 109, 250, 32, 291, 376, 95, 241, 116, 151, 76, 353, 339, 105, 19, 254, 7, 53, 331, 229, 356, 124, 99, 167, 308, 73, 38, 16, 330, 186, 360, 121, 134, 168, 383, 335, 89, 293, 196, 57, 28, 294, 108, 137, 48, 193, 41, 190, 304, 210, 217, 73, 238, 292, 295, 132, 145, 130, 284, 25, 381, 373, 10, 143, 35, 166, 79, 182, 54, 9, 85, 366, 161, 181, 245, 312, 75, 270, 287, 173, 224, 266, 160, 119, 125, 200, 183, 313, 281, 282, 178, 174, 30, 59, 234, 131, 100, 303, 104, 213, 290, 56, 8, 13, 350, 311, 273, 227, 351, 332, 253, 219, 77, 239, 150, 362, 111, 276, 136, 218, 87, 380, 1, 147, 148, 212, 58, 265, 68, 66, 334, 164, 179, 155, 268, 225, 319, 88, 205, 152, 126, 102, 248, 371, 255, 299, 21, 14, 199, 372, 135, 286, 307, 90, 240, 317, 23, 354, 347, 4, 202, 26, 326, 94, 361, 78, 149, 226, 233, 96, 358, 364, 201, 80, 82, 64, 382, 33, 344, 84, 240, 317, 23, 365, 214, 42, 86, 261, 20, 329, 141, 197, 235, 357, 369, 370, 123, 198, 275, 264, 310, 34, 93, 204, 223, 22, 240, 13, 350, 46, 311, 4, 309, 81, 24, 256, 338, 324, 349, 359, 206, 375, 215, 242, 352, 246, 83, 106, 267, 165, 314, 18, 110, 110, 38, 322, 295, 302, 180, 271, 269, 62, 341, 49, 259, 67, 36, 158, 236, 55, 257, 289, 63, 301, 288, 61, 191, 316, 110, 110, 296, 109, 345, 170, 156, 144, 171, 325, 231, 37, 15, 122, 216, 189, 117, 232, 45, 305, 342, 184, 263, 98, 71, 153, 110, 297, 320, 142, 250, 92, 6, 221, 31, 3, 188, 172, 159, 44, 120, 321, 333, 251, 337, 283, 39, 74, 211, 27, 11, 114, 110, 243, 129, 280, 363, 228, 279, 103, 107, 12, 70, 277, 176, 328, 340, 207, 40, 138, 258, 323, 177, 113, 260, 343, 377, 140, 110, 203, 287, 16, 379, 186, 209, 285, 368, 139, 69, 300, 47, 175, 2, 169, 133, 220, 0, 249, 298, 17, 208, 274, 244, 252, 110, 222, 334, 16, 330, 118, 194, 237, 127, 5, 112, 378, 72, 315, 115, 65, 306, 101, 154, 367, 51, 348, 91, 192, 355, 272] + 54: [295, 42, 258, 260, 280, 306, 330, 117, 70, 218, 17, 34, 376, 355, 68, 293, 347, 141, 238, 247, 31, 248, 303, 188, 165, 127, 370, 338, 339, 155, 299, 95, 221, 3, 10, 196, 225, 159, 184, 318, 144, 304, 364, 128, 192, 110, 331, 66, 292, 112, 78, 106, 370, 147, 258, 240, 240, 53, 119, 75, 121, 158, 91, 272, 105, 97, 373, 201, 62, 166, 371, 14, 211, 56, 270, 222, 207, 354, 370, 83, 39, 214, 133, 172, 352, 228, 101, 197, 323, 92, 21, 131, 366, 179, 96, 244, 12, 224, 268, 343, 210, 241, 297, 13, 370, 90, 42, 369, 30, 324, 93, 314, 276, 47, 220, 190, 170, 49, 264, 209, 167, 23, 251, 59, 29, 46, 149, 138, 344, 232, 284, 328, 339, 216, 230, 72, 187, 329, 181, 64, 50, 182, 157, 365, 85, 163, 18, 175, 139, 235, 261, 60, 154, 169, 200, 36, 284, 189, 11, 164, 280, 286, 57, 7, 156, 129, 357, 279, 186, 55, 69, 137, 377, 300, 213, 308, 233, 263, 65, 77, 340, 148, 337, 319, 11, 82, 38, 341, 290, 372, 375, 310, 325, 152, 54, 135, 298, 313, 353, 43, 271, 74, 236, 102, 285, 291, 383, 320, 380, 71, 267, 80, 38, 281, 305, 180, 288, 99, 94, 321, 239, 1, 278, 111, 173, 191, 243, 234, 269, 359, 4, 379, 130, 160, 153, 283, 223, 231, 20, 174, 311, 16, 27, 215, 335, 109, 103, 100, 171, 136, 81, 275, 334, 145, 249, 203, 360, 2, 349, 336, 296, 283, 223, 231, 230, 8, 25, 86, 332, 161, 84, 24, 237, 302, 48, 107, 114, 262, 63, 346, 52, 89, 307, 256, 342, 205, 363, 363, 155, 257, 98, 286, 87, 40, 143, 115, 45, 185, 381, 177, 246, 122, 162, 273, 88, 6, 206, 362, 51, 134, 19, 0, 363, 253, 356, 164, 348, 176, 120, 118, 132, 333, 5, 265, 125, 183, 73, 242, 146, 9, 194, 168, 274, 226, 35, 28, 202, 361, 363, 294, 356, 164, 348, 208, 44, 309, 41, 32, 124, 217, 204, 368, 255, 327, 151, 266, 245, 178, 289, 227, 116, 350, 358, 378, 363, 294, 287, 150, 250, 254, 282, 26, 76, 113, 277, 382, 312, 67, 61, 326, 195, 351, 229, 219, 79, 108, 104, 252, 193, 123, 363, 294, 339, 155, 299, 95, 345, 317, 58, 322, 198, 142, 140, 315, 301, 22, 33, 316, 259, 15, 367, 199, 374, 37, 126, 212] + 55: [56, 4, 357, 307, 381, 323, 363, 99, 179, 271, 337, 105, 260, 136, 332, 101, 294, 365, 268, 209, 125, 166, 282, 110, 205, 109, 56, 98, 340, 111, 192, 74, 65, 291, 320, 366, 197, 274, 27, 135, 119, 157, 214, 127, 219, 55, 120, 183, 262, 220, 229, 245, 326, 309, 340, 16, 381, 323, 70, 200, 349, 221, 112, 352, 377, 306, 318, 84, 314, 367, 359, 297, 150, 203, 155, 142, 379, 196, 113, 174, 218, 238, 227, 74, 29, 249, 12, 164, 170, 348, 47, 42, 277, 85, 313, 258, 372, 118, 50, 139, 330, 32, 148, 96, 72, 174, 62, 252, 303, 48, 301, 49, 44, 216, 5, 350, 246, 178, 122, 171, 31, 232, 30, 35, 68, 177, 334, 57, 167, 186, 159, 174, 308, 80, 123, 247, 29, 223, 279, 316, 189, 190, 251, 228, 19, 128, 198, 61, 15, 168, 275, 162, 298, 230, 24, 305, 202, 151, 7, 116, 90, 273, 76, 199, 138, 156, 233, 375, 231, 240, 23, 33, 87, 37, 208, 129, 58, 77, 364, 345, 18, 204, 202, 174, 11, 339, 299, 347, 130, 293, 8, 300, 163, 14, 361, 79, 346, 94, 22, 281, 126, 93, 66, 248, 21, 362, 283, 222, 152, 355, 146, 370, 73, 48, 269, 341, 60, 368, 102, 95, 378, 194, 185, 255, 212, 237, 97, 321, 374, 369, 175, 195, 9, 310, 28, 235, 234, 39, 267, 347, 180, 71, 114, 26, 239, 210, 265, 360, 287, 149, 296, 143, 173, 137, 315, 376, 92, 288, 25, 144, 88, 81, 98, 311, 328, 78, 130, 134, 3, 0, 172, 276, 224, 187, 184, 207, 264, 133, 225, 243, 161, 242, 191, 83, 59, 34, 88, 81, 20, 165, 123, 121, 115, 40, 333, 86, 259, 284, 236, 286, 257, 52, 302, 188, 335, 104, 147, 280, 295, 100, 131, 69, 327, 256, 11, 89, 299, 382, 180, 226, 176, 124, 206, 289, 217, 54, 263, 254, 153, 160, 356, 336, 64, 261, 244, 266, 51, 250, 154, 154, 308, 182, 41, 353, 269, 2, 272, 213, 46, 325, 343, 67, 10, 358, 82, 1, 373, 338, 354, 141, 169, 158, 351, 215, 154, 43, 146, 116, 90, 211, 76, 132, 290, 383, 344, 371, 380, 53, 322, 193, 63, 342, 319, 201, 145, 324, 36, 75, 45, 285, 154, 270, 20, 80, 6, 304, 115, 106, 331, 312, 17, 91, 329, 140, 241, 278, 117, 103, 253, 38, 107, 13, 181, 108, 292, 317] + 56: [279, 82, 15, 155, 259, 189, 251, 302, 332, 223, 369, 38, 295, 268, 138, 129, 378, 353, 347, 57, 70, 204, 239, 7, 380, 208, 100, 82, 15, 186, 5, 304, 161, 99, 159, 192, 27, 312, 252, 374, 164, 141, 95, 96, 290, 139, 144, 211, 176, 52, 25, 146, 177, 178, 294, 88, 340, 196, 125, 41, 80, 207, 359, 278, 206, 22, 34, 227, 210, 183, 112, 330, 130, 60, 234, 377, 102, 109, 231, 126, 273, 116, 5, 89, 333, 284, 288, 150, 58, 193, 181, 12, 11, 286, 122, 145, 293, 291, 170, 254, 203, 59, 111, 173, 231, 191, 294, 53, 87, 147, 6, 292, 327, 97, 197, 285, 35, 179, 33, 123, 339, 341, 375, 16, 29, 24, 215, 32, 165, 74, 231, 199, 213, 163, 371, 304, 46, 316, 77, 101, 300, 10, 105, 280, 50, 297, 348, 314, 307, 37, 67, 47, 382, 365, 318, 85, 231, 178, 220, 296, 259, 356, 306, 140, 381, 244, 73, 250, 136, 23, 255, 209, 94, 55, 124, 276, 43, 152, 68, 336, 235, 142, 8, 260, 213, 98, 30, 147, 117, 39, 154, 118, 103, 281, 21, 317, 151, 282, 62, 257, 200, 379, 222, 373, 256, 45, 174, 3, 195, 44, 294, 107, 169, 263, 363, 48, 245, 63, 182, 137, 305, 355, 110, 56, 190, 84, 172, 289, 266, 258, 272, 216, 194, 90, 195, 260, 220, 303, 106, 9, 308, 275, 329, 383, 233, 160, 31, 354, 243, 185, 271, 298, 226, 175, 86, 337, 325, 217, 247, 143, 195, 260, 220, 267, 322, 236, 264, 219, 362, 72, 344, 148, 135, 331, 1, 42, 221, 119, 228, 367, 49, 113, 349, 326, 171, 224, 311, 44, 240, 351, 106, 156, 162, 71, 14, 283, 376, 253, 92, 121, 133, 364, 313, 218, 310, 372, 64, 287, 370, 335, 237, 315, 321, 265, 240, 225, 241, 20, 187, 18, 19, 309, 205, 40, 338, 269, 153, 248, 115, 262, 132, 232, 350, 131, 26, 36, 51, 229, 321, 265, 240, 225, 241, 249, 104, 188, 66, 120, 0, 342, 198, 166, 13, 127, 201, 180, 4, 319, 361, 93, 212, 261, 114, 202, 343, 242, 81, 184, 30, 75, 346, 320, 357, 168, 334, 128, 324, 149, 214, 368, 360, 54, 323, 65, 78, 157, 108, 366, 61, 76, 167, 242, 240, 158, 340, 75, 328, 2, 83, 238, 246, 69, 79, 230, 17, 270, 299, 134, 28, 274, 352, 345, 358, 277, 91, 301] + 57: [157, 220, 140, 101, 94, 170, 154, 22, 123, 91, 341, 46, 183, 279, 348, 216, 81, 379, 236, 126, 36, 364, 115, 82, 131, 234, 231, 295, 198, 235, 286, 130, 169, 79, 163, 144, 51, 177, 54, 225, 210, 351, 323, 355, 265, 188, 194, 76, 10, 278, 229, 202, 231, 121, 7, 148, 72, 335, 217, 345, 80, 365, 213, 83, 184, 13, 68, 282, 29, 261, 299, 32, 222, 239, 212, 52, 181, 301, 93, 295, 156, 148, 124, 344, 302, 151, 354, 179, 176, 113, 147, 95, 329, 28, 330, 332, 200, 270, 375, 67, 258, 221, 245, 304, 11, 259, 187, 128, 72, 53, 154, 64, 185, 338, 214, 27, 116, 328, 226, 353, 18, 145, 24, 378, 192, 48, 218, 129, 153, 262, 342, 259, 63, 65, 274, 324, 350, 117, 347, 43, 134, 173, 92, 106, 352, 155, 178, 290, 255, 307, 256, 370, 291, 327, 37, 367, 159, 146, 187, 205, 286, 141, 302, 88, 35, 99, 228, 96, 38, 358, 149, 296, 215, 247, 59, 0, 242, 103, 271, 292, 105, 19, 14, 314, 368, 25, 257, 268, 267, 317, 75, 316, 40, 356, 349, 380, 313, 90, 98, 41, 33, 333, 66, 308, 264, 193, 69, 275, 14, 146, 57, 366, 207, 125, 74, 197, 251, 161, 322, 277, 377, 112, 166, 337, 162, 26, 45, 168, 383, 78, 142, 336, 233, 143, 230, 133, 89, 25, 108, 320, 150, 127, 238, 167, 73, 312, 21, 12, 196, 362, 132, 175, 252, 219, 263, 107, 266, 269, 77, 376, 276, 254, 89, 205, 207, 125, 74, 283, 250, 373, 346, 42, 289, 47, 71, 209, 17, 49, 122, 357, 50, 232, 339, 340, 305, 343, 276, 254, 246, 104, 223, 170, 6, 318, 272, 39, 9, 298, 208, 284, 3, 23, 288, 237, 303, 241, 109, 61, 310, 30, 15, 285, 276, 254, 246, 25, 331, 8, 60, 309, 44, 319, 70, 152, 111, 119, 2, 199, 371, 97, 135, 281, 374, 138, 114, 294, 34, 56, 31, 254, 165, 326, 359, 137, 204, 160, 306, 58, 297, 20, 102, 325, 136, 190, 334, 363, 248, 4, 361, 287, 244, 189, 227, 293, 381, 254, 85, 326, 94, 320, 174, 84, 182, 172, 110, 249, 203, 120, 171, 369, 186, 240, 5, 1, 224, 273, 100, 206, 280, 372, 158, 16, 165, 326, 331, 118, 300, 180, 62, 253, 243, 191, 382, 55, 164, 321, 311, 315, 211, 87, 260, 139, 360, 195, 201, 86] + 58: [263, 79, 348, 245, 93, 199, 368, 342, 12, 329, 229, 258, 250, 121, 362, 228, 293, 219, 95, 359, 349, 34, 78, 165, 252, 204, 248, 61, 210, 211, 177, 214, 175, 47, 255, 185, 331, 181, 83, 0, 172, 43, 113, 133, 87, 128, 223, 117, 188, 343, 321, 58, 156, 81, 205, 194, 49, 355, 232, 367, 316, 274, 239, 244, 333, 240, 148, 2, 19, 94, 142, 221, 103, 246, 74, 310, 370, 308, 156, 81, 205, 194, 312, 351, 324, 272, 315, 73, 281, 62, 53, 149, 357, 334, 353, 297, 261, 41, 336, 304, 286, 291, 80, 150, 220, 126, 305, 311, 312, 67, 295, 189, 60, 66, 139, 350, 306, 302, 144, 21, 70, 237, 317, 54, 46, 96, 174, 140, 71, 340, 220, 56, 366, 373, 298, 358, 152, 119, 260, 197, 375, 325, 339, 131, 253, 278, 50, 118, 102, 282, 51, 39, 88, 212, 147, 10, 220, 56, 366, 222, 122, 114, 45, 16, 104, 198, 192, 143, 124, 183, 231, 161, 283, 170, 24, 163, 290, 159, 267, 363, 335, 134, 220, 81, 259, 222, 122, 72, 68, 42, 106, 352, 55, 82, 227, 378, 288, 327, 8, 230, 372, 86, 146, 145, 301, 320, 164, 168, 275, 76, 305, 311, 235, 26, 112, 341, 130, 285, 187, 284, 215, 135, 13, 318, 35, 180, 296, 254, 337, 137, 15, 91, 309, 209, 275, 126, 273, 101, 9, 114, 196, 191, 132, 171, 202, 266, 90, 332, 208, 345, 182, 338, 138, 276, 374, 216, 17, 361, 3, 184, 77, 151, 344, 245, 177, 256, 116, 141, 277, 224, 48, 129, 236, 63, 300, 11, 265, 136, 100, 264, 247, 120, 365, 89, 279, 280, 377, 377, 92, 271, 299, 270, 36, 85, 111, 328, 1, 59, 155, 75, 379, 371, 115, 105, 360, 268, 44, 369, 217, 233, 292, 364, 377, 377, 7, 14, 22, 69, 376, 203, 20, 242, 251, 207, 234, 37, 176, 65, 225, 243, 294, 157, 200, 153, 30, 218, 125, 33, 377, 377, 356, 98, 322, 169, 40, 18, 97, 28, 380, 99, 238, 110, 241, 347, 287, 173, 29, 330, 269, 32, 52, 23, 166, 201, 377, 377, 27, 245, 298, 289, 262, 31, 4, 190, 381, 383, 326, 382, 186, 123, 195, 64, 5, 303, 167, 109, 127, 323, 319, 38, 377, 249, 92, 98, 206, 257, 178, 6, 162, 307, 314, 25, 160, 346, 154, 107, 179, 226, 213, 354, 84, 57, 193, 313, 158, 108] + 59: [332, 197, 5, 363, 173, 31, 324, 243, 109, 304, 162, 376, 120, 269, 51, 194, 71, 50, 145, 44, 307, 237, 141, 335, 139, 341, 140, 313, 370, 260, 247, 234, 65, 281, 176, 12, 334, 174, 236, 369, 228, 299, 138, 136, 74, 183, 119, 216, 219, 378, 122, 15, 140, 97, 370, 195, 204, 346, 42, 255, 86, 254, 383, 22, 214, 238, 6, 242, 135, 325, 89, 336, 241, 316, 239, 39, 152, 271, 73, 256, 7, 70, 340, 18, 186, 80, 175, 161, 62, 60, 356, 259, 337, 199, 63, 184, 153, 24, 273, 283, 125, 367, 351, 308, 209, 250, 7, 202, 311, 331, 328, 23, 212, 35, 163, 143, 258, 177, 365, 292, 64, 231, 37, 88, 102, 319, 305, 205, 72, 310, 209, 250, 7, 202, 298, 130, 372, 144, 223, 40, 267, 338, 47, 366, 353, 156, 41, 34, 359, 132, 166, 257, 1, 272, 345, 189, 113, 170, 94, 251, 66, 221, 65, 309, 342, 159, 280, 218, 222, 198, 68, 206, 227, 322, 343, 11, 118, 358, 270, 264, 167, 235, 182, 170, 94, 195, 66, 352, 265, 232, 133, 279, 181, 142, 54, 354, 289, 96, 43, 124, 330, 210, 296, 98, 318, 375, 55, 268, 327, 25, 226, 363, 340, 150, 9, 26, 164, 57, 211, 245, 179, 180, 131, 32, 287, 134, 21, 191, 321, 165, 350, 87, 116, 90, 327, 25, 229, 59, 220, 185, 38, 262, 91, 160, 252, 373, 339, 348, 349, 48, 380, 157, 67, 275, 53, 312, 56, 244, 52, 4, 327, 25, 229, 59, 381, 76, 377, 151, 45, 85, 248, 315, 69, 344, 2, 0, 333, 217, 17, 249, 357, 303, 14, 171, 99, 111, 327, 170, 379, 200, 294, 352, 295, 266, 290, 127, 302, 19, 329, 203, 155, 274, 149, 208, 49, 261, 284, 108, 282, 77, 126, 215, 33, 46, 379, 173, 371, 234, 172, 58, 128, 168, 169, 20, 355, 92, 13, 314, 246, 187, 93, 277, 103, 301, 240, 121, 83, 154, 30, 224, 229, 95, 220, 288, 317, 178, 278, 81, 320, 225, 10, 137, 123, 107, 27, 84, 28, 106, 148, 82, 360, 78, 190, 188, 105, 224, 233, 260, 374, 291, 300, 196, 114, 297, 100, 29, 158, 146, 101, 362, 193, 253, 207, 115, 201, 368, 192, 347, 323, 117, 285, 75, 147, 200, 263, 346, 213, 382, 112, 16, 364, 286, 306, 361, 36, 61, 110, 129, 326, 3, 230, 8, 276, 293, 104, 79] + 60: [284, 158, 243, 126, 48, 230, 294, 119, 122, 27, 277, 259, 11, 56, 360, 85, 79, 296, 67, 202, 41, 160, 86, 4, 73, 22, 102, 158, 195, 83, 339, 153, 28, 15, 59, 271, 248, 221, 80, 304, 52, 273, 349, 64, 125, 330, 258, 29, 350, 177, 101, 154, 94, 257, 47, 139, 174, 155, 76, 112, 213, 140, 189, 132, 110, 38, 323, 250, 300, 371, 325, 228, 137, 147, 135, 227, 226, 288, 123, 124, 303, 244, 174, 185, 205, 354, 161, 235, 20, 1, 88, 342, 107, 285, 322, 368, 270, 13, 198, 229, 17, 292, 136, 299, 123, 257, 220, 60, 265, 208, 337, 290, 272, 246, 362, 240, 344, 25, 90, 293, 95, 276, 156, 262, 224, 379, 159, 103, 352, 201, 123, 257, 220, 60, 310, 297, 108, 46, 245, 172, 75, 37, 382, 274, 186, 146, 100, 286, 105, 361, 50, 374, 301, 340, 218, 207, 236, 124, 129, 347, 78, 302, 34, 329, 231, 99, 152, 51, 162, 57, 141, 7, 192, 219, 307, 252, 261, 298, 326, 71, 321, 355, 121, 124, 115, 126, 222, 65, 2, 98, 203, 241, 58, 217, 267, 176, 171, 149, 341, 256, 282, 251, 42, 68, 175, 32, 87, 317, 313, 351, 128, 133, 199, 247, 35, 166, 232, 364, 151, 6, 5, 3, 26, 30, 182, 18, 363, 324, 216, 111, 249, 163, 10, 157, 313, 351, 260, 83, 199, 312, 74, 309, 223, 238, 0, 36, 9, 377, 346, 91, 130, 144, 168, 23, 145, 211, 306, 164, 77, 214, 313, 351, 170, 39, 40, 167, 319, 365, 278, 253, 84, 358, 237, 215, 21, 33, 357, 12, 333, 209, 19, 97, 43, 173, 89, 210, 313, 124, 16, 113, 212, 194, 49, 353, 184, 82, 283, 93, 92, 305, 142, 200, 106, 380, 376, 45, 280, 356, 254, 289, 242, 375, 313, 124, 16, 133, 316, 120, 327, 311, 131, 169, 336, 183, 24, 279, 53, 31, 314, 225, 383, 328, 54, 378, 281, 138, 109, 63, 268, 335, 266, 139, 310, 181, 239, 193, 117, 55, 255, 114, 332, 116, 233, 66, 369, 148, 315, 287, 179, 96, 165, 295, 44, 373, 268, 351, 16, 372, 40, 334, 366, 345, 143, 263, 69, 318, 188, 8, 70, 187, 370, 331, 197, 178, 320, 291, 72, 275, 118, 348, 381, 381, 47, 343, 265, 81, 234, 180, 206, 359, 150, 269, 264, 61, 134, 196, 190, 14, 338, 127, 104, 204, 191, 367, 308, 62] + 61: [231, 46, 117, 1, 218, 294, 307, 88, 156, 227, 90, 27, 23, 97, 153, 9, 85, 106, 311, 99, 49, 105, 318, 210, 320, 208, 303, 46, 178, 197, 167, 372, 86, 115, 369, 87, 84, 44, 260, 102, 38, 125, 340, 342, 345, 364, 246, 28, 56, 131, 334, 356, 230, 338, 203, 380, 253, 198, 143, 129, 34, 361, 316, 190, 368, 207, 346, 37, 225, 152, 370, 358, 232, 140, 256, 62, 41, 319, 172, 338, 323, 197, 21, 68, 55, 288, 110, 20, 77, 252, 336, 249, 137, 22, 141, 204, 352, 18, 312, 236, 250, 50, 367, 0, 61, 179, 299, 381, 353, 235, 248, 216, 181, 261, 224, 11, 329, 378, 383, 79, 67, 14, 244, 123, 111, 146, 35, 267, 376, 58, 61, 338, 122, 45, 167, 371, 248, 216, 15, 169, 366, 83, 3, 200, 273, 280, 196, 359, 63, 310, 272, 95, 377, 289, 217, 287, 73, 179, 205, 363, 36, 324, 237, 192, 243, 309, 29, 166, 206, 147, 214, 275, 269, 148, 70, 74, 13, 164, 285, 39, 229, 94, 73, 179, 299, 184, 100, 142, 263, 107, 42, 92, 114, 135, 328, 52, 306, 8, 283, 298, 162, 251, 7, 339, 276, 151, 40, 96, 5, 170, 351, 381, 195, 51, 161, 333, 188, 191, 233, 113, 245, 202, 292, 365, 4, 199, 163, 301, 24, 48, 222, 134, 325, 149, 136, 221, 165, 264, 103, 215, 263, 107, 286, 228, 118, 304, 109, 139, 186, 80, 26, 144, 54, 187, 293, 182, 60, 281, 350, 116, 335, 221, 127, 241, 330, 344, 119, 57, 177, 258, 268, 300, 145, 104, 30, 375, 240, 98, 193, 101, 262, 223, 220, 382, 266, 53, 108, 221, 127, 19, 315, 344, 66, 168, 349, 43, 348, 379, 47, 173, 72, 219, 71, 124, 25, 354, 314, 212, 64, 254, 128, 343, 360, 221, 127, 10, 36, 324, 161, 112, 160, 75, 150, 82, 69, 158, 89, 341, 317, 332, 91, 239, 305, 180, 226, 355, 16, 171, 93, 211, 127, 270, 12, 17, 86, 291, 159, 337, 362, 374, 2, 290, 78, 174, 295, 284, 274, 234, 331, 265, 302, 279, 357, 189, 93, 120, 247, 59, 315, 278, 213, 327, 31, 32, 65, 313, 257, 33, 322, 126, 81, 176, 271, 255, 201, 373, 277, 175, 296, 321, 211, 157, 326, 209, 100, 238, 307, 88, 282, 133, 297, 121, 308, 259, 154, 242, 132, 194, 130, 6, 185, 138, 347, 183, 76, 155] +layer_updates_per_iter: 0 +num_slots: 416 diff --git a/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots384.yaml b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots384.yaml new file mode 100644 index 000000000000..262c8042d3d1 --- /dev/null +++ b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 327, 22, 354, 123, 257, 180, 263, 53, 250, 120, 61, 111, 232, 55, 10, 31, 213, 192, 145, 340, 132, 88, 185, 378, 29, 282, 156, 148, 172, 369, 67, 206, 164, 168, 303, 166, 104, 272, 72, 114, 358, 249, 147, 252, 266, 304, 247, 256, 347, 198, 284, 368, 375, 331, 267, 350, 125, 142, 110, 74, 188, 63, 117, 261, 66, 365, 326, 229, 357, 24, 373, 222, 332, 139, 34, 200, 39, 269, 299, 40, 2, 121, 274, 356, 190, 343, 205, 36, 89, 108, 25, 210, 301, 339, 370, 236, 241, 255, 78, 297, 309, 103, 100, 47, 175, 189, 201, 245, 346, 48, 359, 186, 160, 4, 97, 20, 41, 271, 171, 371, 158, 321, 143, 90, 27, 317, 1, 50, 12, 92, 13, 225, 230, 289, 141, 124, 268, 136, 238, 6, 353, 23, 277, 254, 193, 218, 336, 57, 77, 214, 215, 260, 380, 298, 118, 135, 207, 157, 360, 109, 149, 335, 211, 342, 306, 18, 58, 279, 76, 80, 349, 56, 337, 227, 296, 362, 315, 167, 310, 45, 264, 242, 126, 233, 208, 244, 59, 237, 115, 251, 288, 60, 235, 62, 209, 43, 170, 32, 366, 14, 352, 69, 226, 79, 177, 204, 144, 183, 7, 85, 372, 82, 98, 319, 8, 17, 348, 30, 280, 307, 300, 262, 75, 83, 286, 381, 344, 95, 162, 202, 3, 181, 28, 314, 363, 127, 294, 194, 291, 134, 155, 81, 333, 131, 112, 38, 146, 328, 305, 231, 228, 184, 86, 153, 176, 341, 265, 308, 275, 320, 316, 239, 281, 49, 383, 21, 16, 33, 248, 234, 240, 93, 102, 285, 329, 203, 119, 35, 37, 196, 311, 377, 216, 195, 64, 191, 273, 217, 322, 295, 318, 54, 44, 199, 46, 382, 338, 367, 325, 330, 345, 169, 70, 101, 94, 223, 19, 106, 253, 9, 374, 173, 351, 122, 276, 313, 324, 87, 116, 292, 96, 334, 323, 224, 84, 312, 65, 243, 138, 52, 137, 73, 154, 220, 287, 290, 159, 182, 376, 11, 361, 187, 99, 379, 212, 179, 355, 259, 302, 26, 278, 105, 129, 91, 140, 221, 15, 219, 130, 0, 71, 51, 293, 107, 258, 133, 178, 270, 42, 113, 128, 150, 151, 163, 68, 283, 174, 5, 246, 197, 161, 152, 364] + 1: [163, 0, 29, 355, 176, 205, 318, 293, 119, 253, 117, 255, 319, 39, 24, 78, 143, 133, 2, 290, 77, 316, 339, 100, 365, 131, 248, 89, 261, 376, 259, 57, 181, 220, 45, 265, 42, 40, 23, 32, 280, 171, 204, 165, 113, 372, 191, 15, 201, 21, 250, 173, 335, 239, 305, 321, 268, 129, 16, 363, 174, 378, 56, 86, 142, 202, 222, 154, 8, 331, 155, 327, 22, 329, 299, 71, 194, 38, 348, 33, 302, 371, 322, 215, 137, 254, 244, 260, 92, 360, 170, 136, 323, 246, 124, 310, 180, 291, 300, 275, 162, 69, 79, 229, 13, 375, 343, 368, 158, 196, 294, 211, 84, 306, 313, 295, 94, 279, 128, 34, 277, 43, 58, 287, 108, 264, 140, 175, 96, 130, 361, 345, 382, 152, 334, 145, 288, 88, 362, 184, 227, 369, 252, 336, 91, 50, 63, 37, 212, 116, 298, 156, 352, 357, 221, 312, 72, 263, 105, 9, 146, 93, 76, 17, 97, 83, 135, 112, 169, 149, 73, 269, 187, 381, 7, 115, 104, 148, 217, 199, 177, 150, 110, 303, 166, 203, 121, 107, 311, 190, 209, 267, 167, 126, 328, 195, 197, 289, 122, 296, 26, 314, 132, 157, 127, 85, 41, 273, 218, 358, 164, 19, 350, 103, 66, 304, 347, 80, 123, 138, 301, 282, 189, 14, 51, 27, 208, 354, 67, 106, 210, 59, 99, 228, 256, 326, 11, 356, 10, 257, 377, 44, 70, 243, 74, 206, 237, 332, 315, 333, 373, 258, 3, 18, 325, 46, 285, 47, 231, 20, 48, 232, 151, 374, 182, 118, 75, 320, 274, 213, 98, 370, 284, 54, 340, 341, 198, 308, 179, 383, 266, 214, 61, 286, 87, 240, 207, 64, 120, 31, 5, 283, 216, 52, 60, 65, 186, 262, 160, 337, 225, 235, 270, 224, 281, 4, 95, 139, 324, 353, 193, 159, 111, 1, 366, 25, 245, 53, 102, 161, 36, 226, 367, 349, 82, 200, 192, 233, 317, 219, 101, 249, 35, 12, 90, 276, 272, 81, 6, 185, 234, 114, 236, 364, 230, 344, 188, 144, 292, 251, 30, 351, 125, 223, 147, 338, 172, 49, 297, 359, 134, 141, 168, 153, 247, 379, 278, 330, 68, 346, 183, 307, 178, 380, 241, 271, 242, 109, 342, 238, 62, 55, 309, 28] + 2: [143, 284, 325, 79, 92, 46, 136, 239, 371, 48, 228, 379, 248, 97, 31, 323, 55, 332, 154, 275, 373, 74, 326, 381, 53, 350, 51, 285, 224, 349, 305, 77, 292, 177, 289, 374, 43, 111, 45, 30, 263, 360, 106, 23, 329, 222, 315, 258, 132, 149, 118, 235, 126, 163, 9, 362, 99, 98, 287, 366, 112, 60, 245, 82, 262, 277, 38, 94, 95, 164, 202, 229, 108, 75, 278, 24, 68, 253, 256, 152, 216, 382, 282, 40, 313, 33, 70, 340, 90, 120, 181, 179, 144, 267, 237, 190, 215, 197, 286, 279, 7, 128, 160, 176, 141, 8, 354, 199, 89, 34, 73, 200, 17, 201, 15, 365, 331, 205, 260, 330, 194, 247, 280, 63, 170, 58, 186, 150, 264, 167, 240, 161, 93, 125, 56, 217, 299, 383, 169, 104, 207, 114, 13, 300, 115, 151, 131, 146, 206, 21, 309, 29, 376, 338, 353, 363, 223, 100, 66, 328, 140, 20, 193, 318, 178, 335, 369, 375, 3, 189, 355, 83, 130, 361, 175, 268, 52, 183, 317, 255, 295, 321, 157, 274, 344, 187, 348, 257, 272, 251, 214, 246, 212, 137, 173, 26, 204, 327, 308, 129, 25, 226, 72, 316, 242, 221, 166, 22, 294, 210, 41, 276, 219, 291, 117, 358, 232, 5, 124, 368, 165, 320, 65, 364, 0, 359, 345, 127, 290, 259, 191, 324, 174, 67, 148, 49, 47, 319, 147, 249, 19, 81, 322, 296, 61, 203, 76, 121, 238, 352, 310, 32, 62, 1, 84, 172, 342, 18, 171, 250, 134, 302, 378, 119, 168, 298, 155, 105, 311, 293, 265, 372, 2, 57, 336, 301, 91, 28, 192, 271, 209, 356, 241, 69, 36, 54, 182, 337, 377, 102, 109, 220, 6, 273, 16, 116, 195, 71, 158, 339, 86, 343, 110, 133, 85, 156, 10, 254, 281, 107, 307, 270, 304, 14, 101, 42, 303, 27, 236, 142, 4, 233, 234, 64, 314, 357, 78, 184, 11, 269, 145, 103, 211, 288, 113, 80, 139, 351, 185, 306, 159, 87, 138, 12, 346, 370, 59, 208, 122, 341, 153, 135, 196, 347, 198, 39, 367, 231, 37, 380, 180, 225, 162, 44, 244, 50, 297, 312, 334, 243, 227, 266, 230, 96, 213, 123, 333, 188, 218, 252, 35, 261, 283, 88] + 3: [66, 199, 278, 292, 303, 59, 311, 47, 222, 38, 301, 370, 34, 17, 247, 332, 335, 52, 9, 233, 257, 146, 324, 198, 310, 43, 165, 93, 234, 148, 352, 244, 152, 167, 226, 5, 260, 328, 15, 309, 121, 147, 124, 315, 173, 185, 341, 371, 377, 98, 270, 237, 367, 101, 134, 330, 175, 320, 326, 193, 365, 161, 181, 327, 160, 232, 362, 141, 253, 265, 349, 33, 319, 29, 229, 217, 246, 361, 153, 45, 54, 127, 4, 347, 39, 239, 364, 374, 166, 44, 249, 282, 159, 90, 272, 60, 22, 138, 118, 277, 200, 113, 187, 269, 220, 75, 308, 69, 79, 211, 119, 154, 204, 129, 286, 373, 258, 210, 368, 23, 302, 238, 325, 21, 288, 224, 266, 322, 130, 179, 143, 1, 40, 26, 189, 223, 99, 218, 307, 216, 381, 285, 338, 95, 18, 337, 250, 36, 142, 375, 105, 276, 106, 190, 372, 89, 27, 158, 102, 231, 12, 203, 103, 48, 353, 85, 183, 205, 163, 318, 201, 122, 240, 32, 299, 280, 96, 356, 245, 241, 3, 24, 186, 37, 64, 359, 108, 273, 115, 83, 354, 16, 194, 363, 139, 10, 11, 172, 41, 65, 248, 376, 0, 378, 313, 151, 339, 267, 290, 295, 126, 117, 340, 123, 133, 107, 263, 348, 215, 227, 25, 289, 274, 135, 306, 61, 206, 191, 71, 157, 137, 58, 182, 162, 300, 355, 196, 294, 28, 251, 46, 62, 42, 2, 219, 8, 345, 169, 366, 91, 254, 379, 275, 80, 331, 358, 316, 84, 176, 112, 100, 7, 86, 323, 35, 333, 6, 164, 177, 109, 155, 57, 214, 291, 346, 242, 230, 261, 213, 342, 174, 78, 111, 235, 76, 70, 228, 284, 209, 97, 50, 120, 296, 87, 243, 94, 82, 188, 110, 360, 314, 192, 180, 336, 74, 136, 297, 197, 144, 298, 283, 116, 14, 208, 252, 170, 195, 271, 92, 344, 383, 305, 68, 329, 171, 56, 81, 343, 304, 279, 114, 53, 268, 67, 72, 73, 51, 13, 145, 207, 287, 357, 312, 256, 125, 128, 132, 88, 31, 131, 49, 168, 77, 382, 351, 236, 30, 184, 293, 140, 350, 225, 334, 178, 221, 202, 262, 380, 104, 20, 321, 369, 19, 317, 150, 63, 212, 264, 149, 55, 255, 281, 259, 156] + 4: [343, 67, 321, 90, 336, 107, 182, 277, 282, 305, 332, 351, 331, 79, 128, 231, 242, 272, 338, 287, 58, 63, 104, 120, 3, 341, 137, 51, 40, 374, 368, 174, 20, 192, 208, 233, 271, 75, 288, 234, 249, 96, 293, 270, 227, 119, 225, 17, 315, 101, 260, 307, 269, 370, 175, 127, 279, 202, 154, 2, 214, 365, 28, 141, 359, 80, 178, 111, 311, 373, 299, 337, 209, 325, 43, 187, 252, 64, 84, 103, 130, 1, 11, 382, 296, 86, 361, 204, 159, 14, 284, 349, 223, 205, 224, 195, 371, 171, 347, 83, 177, 65, 261, 328, 26, 286, 380, 144, 23, 150, 167, 218, 317, 133, 132, 367, 157, 355, 173, 366, 15, 105, 155, 57, 369, 85, 323, 310, 29, 201, 48, 91, 95, 257, 298, 314, 8, 38, 60, 46, 206, 327, 378, 248, 226, 45, 197, 4, 7, 276, 379, 232, 184, 295, 362, 191, 319, 5, 377, 106, 161, 116, 53, 322, 52, 237, 112, 164, 87, 19, 36, 196, 339, 358, 185, 108, 121, 122, 123, 238, 194, 376, 308, 76, 82, 207, 149, 125, 303, 170, 115, 102, 273, 281, 6, 139, 73, 0, 356, 100, 220, 31, 156, 236, 117, 88, 165, 145, 251, 99, 168, 94, 292, 109, 213, 266, 142, 239, 143, 32, 49, 183, 297, 131, 50, 140, 255, 363, 148, 9, 241, 61, 280, 221, 189, 354, 176, 181, 10, 285, 254, 172, 136, 77, 68, 42, 360, 135, 309, 268, 126, 301, 152, 228, 348, 200, 344, 264, 72, 313, 263, 291, 203, 245, 188, 16, 302, 275, 217, 243, 330, 25, 39, 289, 353, 55, 34, 193, 253, 235, 44, 179, 33, 70, 66, 304, 163, 162, 340, 320, 345, 372, 312, 153, 352, 240, 92, 334, 316, 247, 18, 250, 56, 113, 22, 244, 114, 262, 267, 118, 21, 30, 37, 259, 13, 41, 35, 54, 256, 383, 294, 147, 129, 134, 12, 219, 274, 357, 333, 324, 62, 74, 346, 210, 186, 326, 375, 350, 342, 318, 198, 290, 166, 158, 306, 211, 97, 265, 124, 24, 329, 364, 160, 278, 169, 229, 78, 146, 215, 89, 199, 258, 59, 246, 110, 335, 98, 216, 230, 180, 81, 212, 222, 300, 47, 151, 27, 283, 93, 69, 138, 71, 381, 190] + 5: [61, 207, 144, 300, 236, 129, 340, 226, 286, 177, 354, 302, 213, 37, 295, 56, 284, 363, 316, 85, 142, 179, 118, 303, 128, 313, 24, 186, 250, 13, 263, 158, 180, 127, 368, 75, 214, 304, 184, 152, 329, 237, 134, 322, 197, 310, 321, 44, 355, 147, 171, 324, 380, 241, 77, 92, 309, 96, 157, 170, 190, 72, 335, 108, 320, 353, 76, 349, 189, 11, 10, 337, 249, 379, 364, 38, 333, 345, 334, 344, 125, 133, 116, 23, 84, 372, 89, 15, 175, 266, 341, 94, 58, 53, 248, 50, 296, 365, 293, 151, 210, 51, 257, 291, 378, 202, 109, 196, 194, 381, 246, 70, 3, 79, 121, 87, 114, 282, 176, 277, 217, 30, 331, 137, 143, 115, 240, 360, 18, 311, 225, 57, 346, 169, 200, 155, 204, 99, 201, 80, 359, 135, 328, 278, 276, 188, 43, 66, 318, 28, 239, 149, 283, 244, 350, 164, 100, 262, 7, 8, 251, 46, 382, 48, 62, 148, 111, 339, 105, 193, 88, 131, 140, 195, 362, 9, 232, 95, 101, 234, 305, 370, 352, 4, 319, 377, 106, 63, 323, 168, 97, 27, 269, 223, 252, 233, 264, 315, 93, 90, 376, 224, 120, 112, 356, 265, 17, 371, 332, 104, 14, 138, 71, 270, 357, 117, 287, 49, 212, 74, 369, 33, 253, 383, 271, 86, 245, 123, 52, 60, 255, 209, 375, 35, 39, 325, 374, 40, 267, 326, 288, 178, 215, 31, 358, 16, 12, 275, 191, 373, 185, 367, 6, 65, 132, 327, 222, 228, 183, 289, 55, 156, 285, 167, 272, 47, 279, 124, 59, 220, 227, 136, 205, 187, 307, 82, 231, 81, 351, 182, 2, 45, 216, 281, 229, 32, 292, 294, 159, 1, 145, 173, 163, 54, 366, 150, 103, 107, 36, 110, 259, 98, 130, 25, 174, 218, 242, 254, 162, 261, 126, 280, 21, 260, 64, 22, 268, 301, 20, 219, 299, 199, 146, 0, 172, 160, 41, 198, 181, 258, 230, 161, 113, 342, 336, 273, 208, 5, 102, 338, 69, 314, 343, 317, 221, 68, 243, 83, 165, 308, 122, 247, 206, 34, 274, 119, 361, 347, 290, 42, 154, 256, 78, 91, 238, 153, 312, 330, 29, 297, 139, 166, 141, 306, 211, 192, 67, 26, 19, 203, 235, 298, 73, 348] + 6: [97, 71, 323, 331, 40, 90, 161, 164, 247, 257, 342, 373, 142, 363, 25, 33, 52, 186, 204, 275, 315, 35, 320, 325, 308, 218, 353, 265, 153, 252, 75, 132, 149, 135, 167, 214, 322, 248, 299, 74, 51, 192, 241, 273, 170, 61, 72, 65, 76, 23, 333, 134, 360, 374, 19, 93, 382, 20, 2, 314, 274, 133, 157, 243, 346, 123, 17, 38, 127, 340, 191, 228, 376, 231, 327, 206, 372, 96, 235, 21, 140, 141, 344, 200, 32, 109, 158, 220, 225, 168, 125, 226, 201, 91, 1, 172, 305, 319, 301, 101, 332, 104, 126, 215, 357, 47, 171, 227, 53, 318, 208, 270, 12, 211, 207, 148, 16, 312, 131, 78, 343, 163, 287, 113, 68, 15, 237, 122, 124, 347, 102, 213, 174, 45, 255, 259, 236, 77, 271, 217, 42, 244, 89, 238, 258, 350, 112, 234, 49, 383, 262, 355, 268, 39, 121, 7, 253, 173, 119, 239, 245, 221, 6, 82, 3, 73, 117, 26, 240, 175, 56, 370, 380, 139, 280, 330, 193, 310, 256, 29, 281, 9, 368, 156, 41, 169, 365, 60, 105, 366, 266, 92, 99, 371, 293, 129, 377, 282, 58, 197, 199, 364, 85, 302, 66, 67, 100, 289, 352, 321, 154, 335, 83, 379, 290, 339, 233, 31, 232, 155, 115, 202, 98, 159, 81, 54, 328, 250, 166, 120, 367, 184, 369, 111, 296, 284, 48, 309, 337, 263, 177, 205, 88, 178, 128, 190, 286, 63, 18, 272, 251, 356, 146, 326, 187, 79, 185, 37, 10, 110, 14, 198, 288, 195, 297, 180, 94, 291, 179, 279, 136, 276, 138, 358, 108, 162, 212, 209, 181, 34, 354, 30, 219, 116, 261, 381, 285, 183, 59, 46, 216, 118, 165, 223, 143, 137, 145, 50, 62, 303, 295, 11, 13, 196, 176, 304, 264, 242, 316, 147, 203, 294, 5, 229, 307, 188, 277, 317, 324, 43, 329, 224, 106, 338, 103, 341, 359, 246, 345, 230, 64, 69, 362, 70, 292, 222, 160, 55, 260, 84, 300, 311, 107, 361, 194, 8, 152, 313, 27, 28, 375, 57, 0, 278, 144, 87, 254, 334, 80, 24, 249, 269, 378, 189, 151, 336, 114, 22, 4, 349, 150, 348, 306, 267, 36, 283, 95, 182, 44, 210, 86, 298, 130, 351] + 7: [327, 321, 19, 56, 120, 10, 228, 156, 135, 171, 50, 60, 7, 2, 372, 380, 323, 320, 112, 304, 138, 257, 338, 364, 274, 337, 158, 92, 152, 22, 363, 334, 223, 371, 357, 42, 49, 183, 84, 213, 204, 305, 68, 209, 43, 147, 179, 91, 292, 14, 23, 224, 140, 6, 146, 216, 231, 242, 127, 202, 175, 281, 168, 221, 264, 200, 161, 203, 59, 78, 355, 110, 182, 329, 267, 235, 214, 137, 196, 351, 85, 128, 34, 348, 275, 48, 313, 288, 330, 185, 141, 149, 287, 297, 205, 276, 192, 378, 249, 270, 90, 358, 51, 116, 286, 252, 172, 365, 263, 118, 11, 31, 377, 245, 44, 261, 106, 39, 66, 290, 64, 99, 324, 55, 62, 109, 86, 291, 166, 379, 150, 145, 41, 71, 73, 285, 219, 26, 5, 251, 366, 88, 315, 376, 375, 253, 104, 74, 383, 368, 237, 306, 37, 295, 246, 143, 142, 13, 302, 307, 284, 167, 107, 8, 101, 367, 159, 54, 229, 139, 181, 217, 208, 241, 211, 227, 93, 96, 266, 144, 265, 280, 256, 125, 296, 314, 186, 310, 79, 72, 195, 199, 343, 271, 27, 248, 325, 16, 32, 134, 184, 218, 198, 282, 382, 342, 94, 100, 356, 206, 178, 309, 153, 123, 381, 240, 340, 46, 174, 4, 210, 98, 273, 233, 115, 36, 45, 12, 105, 28, 77, 119, 170, 191, 335, 373, 126, 69, 230, 165, 18, 187, 244, 17, 255, 243, 293, 40, 193, 80, 317, 160, 103, 176, 328, 349, 21, 300, 102, 301, 269, 194, 332, 197, 201, 260, 0, 29, 177, 344, 58, 353, 35, 33, 299, 122, 25, 350, 114, 278, 319, 234, 220, 247, 38, 254, 124, 279, 362, 133, 370, 262, 117, 259, 190, 232, 225, 111, 3, 236, 298, 239, 61, 289, 70, 333, 83, 283, 318, 316, 359, 76, 369, 361, 131, 82, 374, 24, 180, 155, 222, 169, 346, 331, 163, 108, 212, 215, 81, 132, 322, 65, 87, 312, 303, 326, 341, 308, 188, 164, 339, 311, 136, 67, 268, 207, 57, 352, 162, 336, 15, 113, 250, 189, 130, 226, 151, 75, 95, 347, 157, 258, 354, 294, 360, 89, 20, 121, 345, 173, 277, 30, 272, 238, 97, 47, 148, 9, 129, 53, 52, 1, 63, 154] + 8: [360, 32, 378, 111, 160, 0, 227, 102, 58, 170, 234, 207, 253, 303, 125, 28, 190, 236, 193, 355, 19, 262, 132, 96, 290, 333, 304, 332, 87, 307, 260, 324, 51, 63, 213, 100, 359, 115, 34, 368, 2, 71, 327, 134, 370, 210, 163, 1, 5, 157, 346, 92, 50, 4, 293, 16, 369, 180, 331, 376, 67, 337, 97, 109, 278, 341, 174, 382, 145, 220, 363, 296, 31, 228, 168, 23, 215, 318, 98, 37, 159, 146, 305, 55, 135, 64, 377, 133, 219, 197, 118, 41, 205, 76, 343, 127, 21, 282, 364, 27, 40, 211, 230, 47, 35, 351, 70, 242, 367, 154, 11, 150, 178, 177, 212, 299, 89, 24, 286, 176, 264, 88, 18, 75, 216, 48, 206, 149, 79, 365, 129, 342, 239, 381, 209, 69, 169, 120, 335, 259, 200, 287, 348, 361, 201, 272, 349, 172, 121, 164, 263, 310, 265, 357, 103, 288, 362, 243, 44, 188, 65, 141, 217, 184, 85, 33, 15, 345, 162, 241, 328, 356, 275, 233, 117, 352, 235, 153, 84, 289, 321, 336, 294, 83, 165, 192, 254, 110, 325, 277, 171, 49, 10, 173, 316, 22, 17, 334, 383, 373, 339, 326, 271, 57, 62, 312, 380, 366, 330, 113, 114, 350, 155, 185, 77, 94, 221, 12, 255, 107, 261, 317, 358, 25, 99, 295, 268, 203, 308, 13, 95, 181, 78, 112, 42, 73, 56, 9, 245, 142, 353, 224, 186, 225, 179, 189, 101, 237, 144, 175, 285, 80, 152, 108, 223, 311, 202, 130, 52, 151, 136, 347, 208, 139, 371, 7, 251, 128, 298, 196, 60, 90, 266, 46, 158, 274, 229, 161, 20, 280, 147, 300, 66, 249, 372, 54, 244, 344, 204, 106, 252, 86, 6, 198, 45, 116, 140, 375, 354, 138, 182, 302, 30, 131, 38, 309, 59, 248, 199, 14, 269, 301, 281, 338, 291, 148, 283, 167, 8, 191, 313, 315, 270, 82, 273, 238, 276, 166, 240, 267, 91, 379, 105, 306, 123, 43, 104, 292, 36, 156, 226, 137, 187, 232, 314, 323, 340, 124, 214, 218, 122, 195, 231, 320, 72, 297, 126, 29, 319, 222, 250, 246, 81, 39, 119, 183, 61, 247, 68, 93, 329, 256, 3, 279, 74, 374, 258, 194, 284, 257, 143, 322, 26, 53] + 9: [92, 320, 347, 368, 40, 164, 236, 283, 338, 365, 234, 57, 304, 245, 253, 227, 44, 182, 152, 168, 340, 348, 173, 26, 255, 187, 321, 150, 276, 112, 76, 191, 257, 157, 27, 369, 349, 144, 6, 139, 343, 117, 301, 383, 362, 250, 53, 24, 327, 121, 360, 180, 267, 104, 60, 84, 199, 284, 61, 77, 355, 51, 161, 54, 105, 5, 379, 110, 178, 345, 55, 126, 31, 198, 14, 153, 97, 72, 235, 62, 224, 290, 85, 122, 226, 344, 23, 88, 142, 336, 80, 203, 298, 293, 204, 265, 94, 37, 136, 363, 107, 171, 354, 373, 89, 188, 378, 25, 263, 29, 67, 158, 247, 206, 8, 46, 310, 326, 200, 7, 295, 381, 209, 30, 231, 4, 202, 382, 286, 335, 273, 156, 47, 351, 341, 82, 318, 323, 359, 100, 371, 346, 176, 28, 185, 184, 271, 317, 0, 256, 237, 303, 42, 48, 240, 91, 313, 179, 214, 285, 334, 195, 106, 52, 213, 333, 160, 249, 316, 155, 281, 151, 306, 269, 288, 307, 300, 229, 59, 361, 90, 222, 210, 367, 228, 186, 16, 319, 291, 270, 193, 63, 262, 165, 329, 133, 297, 145, 230, 246, 261, 125, 374, 194, 49, 74, 287, 312, 377, 163, 123, 75, 322, 216, 34, 366, 64, 219, 370, 9, 135, 259, 11, 208, 254, 38, 170, 119, 244, 350, 159, 98, 120, 128, 141, 50, 205, 375, 325, 73, 95, 353, 380, 258, 302, 20, 124, 339, 2, 251, 243, 113, 103, 299, 242, 131, 154, 364, 274, 65, 275, 305, 68, 101, 147, 357, 138, 22, 3, 192, 111, 248, 174, 232, 167, 223, 10, 252, 217, 177, 292, 99, 36, 308, 114, 330, 140, 296, 212, 172, 376, 132, 137, 183, 238, 130, 45, 12, 43, 266, 78, 19, 272, 239, 33, 311, 218, 18, 233, 162, 337, 93, 166, 215, 201, 342, 169, 87, 189, 109, 148, 71, 108, 118, 81, 260, 83, 309, 289, 41, 181, 17, 79, 352, 225, 279, 143, 280, 1, 56, 277, 35, 207, 39, 190, 21, 241, 149, 331, 372, 15, 102, 356, 116, 66, 324, 115, 134, 58, 294, 220, 196, 146, 32, 264, 129, 70, 13, 358, 211, 86, 328, 96, 315, 127, 282, 268, 314, 332, 175, 197, 221, 69, 278] + 10: [319, 349, 290, 295, 358, 366, 29, 68, 283, 336, 297, 369, 5, 160, 110, 70, 236, 253, 166, 176, 23, 82, 291, 324, 300, 227, 10, 14, 242, 28, 43, 188, 228, 89, 238, 381, 320, 27, 383, 313, 159, 4, 333, 63, 277, 260, 52, 270, 365, 308, 348, 382, 72, 338, 278, 220, 62, 53, 352, 202, 194, 66, 312, 98, 229, 85, 51, 317, 131, 217, 44, 150, 264, 138, 293, 190, 90, 106, 97, 323, 326, 224, 157, 152, 55, 19, 353, 77, 198, 122, 230, 275, 169, 17, 281, 156, 180, 153, 115, 204, 132, 296, 41, 174, 244, 187, 158, 191, 151, 263, 213, 342, 199, 288, 328, 193, 257, 206, 200, 361, 265, 171, 380, 377, 155, 311, 74, 254, 307, 305, 6, 347, 24, 215, 201, 136, 231, 21, 112, 36, 343, 80, 88, 30, 121, 73, 177, 240, 306, 232, 378, 340, 282, 139, 94, 84, 216, 102, 182, 11, 38, 103, 314, 162, 185, 287, 273, 143, 18, 79, 210, 243, 189, 0, 60, 267, 50, 350, 168, 344, 286, 101, 241, 272, 360, 33, 142, 371, 239, 96, 292, 207, 31, 57, 40, 76, 208, 145, 219, 25, 140, 209, 303, 83, 164, 225, 351, 67, 148, 128, 81, 379, 45, 364, 205, 172, 146, 92, 114, 7, 316, 250, 75, 37, 332, 299, 147, 104, 310, 2, 78, 91, 49, 32, 129, 339, 39, 362, 119, 35, 301, 309, 117, 58, 186, 376, 56, 9, 247, 298, 375, 120, 294, 331, 161, 47, 252, 271, 3, 197, 12, 95, 355, 16, 192, 367, 179, 196, 284, 359, 107, 335, 64, 123, 373, 133, 255, 126, 178, 124, 363, 266, 214, 256, 125, 163, 86, 322, 246, 167, 134, 237, 374, 218, 135, 20, 15, 65, 173, 276, 93, 69, 144, 269, 368, 212, 105, 26, 71, 226, 111, 34, 268, 279, 8, 211, 259, 223, 46, 357, 318, 183, 235, 203, 137, 337, 22, 248, 130, 280, 87, 329, 184, 261, 222, 245, 181, 325, 1, 330, 345, 61, 116, 346, 42, 370, 372, 251, 108, 289, 170, 154, 234, 249, 341, 59, 258, 113, 127, 165, 48, 262, 195, 304, 356, 149, 233, 327, 109, 175, 100, 118, 221, 141, 54, 99, 274, 285, 321, 315, 354, 13, 302, 334] + 11: [130, 373, 289, 279, 383, 184, 132, 53, 256, 243, 335, 269, 314, 320, 226, 365, 292, 57, 382, 214, 181, 273, 42, 241, 296, 250, 45, 151, 294, 125, 113, 310, 180, 333, 32, 12, 163, 207, 280, 10, 336, 120, 129, 140, 272, 72, 257, 105, 110, 128, 297, 27, 55, 366, 64, 271, 380, 137, 50, 9, 98, 52, 303, 80, 255, 362, 264, 43, 21, 114, 206, 109, 377, 165, 196, 117, 89, 254, 353, 19, 103, 344, 253, 261, 339, 178, 148, 283, 160, 301, 234, 142, 288, 183, 76, 308, 87, 29, 95, 350, 197, 208, 291, 48, 322, 68, 155, 61, 90, 162, 39, 47, 195, 259, 338, 143, 200, 361, 111, 191, 83, 102, 73, 313, 93, 96, 265, 237, 375, 139, 317, 134, 215, 267, 131, 6, 92, 186, 312, 8, 274, 352, 372, 307, 85, 199, 343, 78, 2, 224, 81, 349, 230, 5, 60, 202, 150, 157, 330, 23, 75, 14, 97, 91, 263, 287, 167, 212, 235, 325, 284, 136, 346, 251, 211, 302, 355, 104, 290, 329, 221, 300, 141, 169, 124, 86, 242, 311, 182, 173, 268, 334, 121, 190, 217, 17, 94, 172, 30, 354, 194, 37, 175, 28, 11, 145, 345, 177, 170, 374, 59, 359, 231, 108, 216, 16, 36, 233, 0, 154, 319, 315, 168, 364, 358, 106, 277, 153, 116, 171, 22, 228, 367, 340, 35, 378, 46, 360, 369, 189, 44, 99, 332, 20, 371, 341, 327, 34, 7, 112, 328, 232, 248, 38, 351, 244, 159, 298, 31, 324, 18, 306, 379, 220, 100, 326, 152, 204, 127, 49, 101, 135, 188, 356, 1, 239, 126, 119, 247, 70, 69, 26, 147, 293, 258, 185, 304, 223, 357, 149, 144, 282, 348, 203, 209, 118, 174, 240, 305, 281, 285, 65, 218, 13, 66, 347, 245, 158, 138, 323, 229, 276, 15, 40, 24, 205, 161, 176, 295, 115, 79, 4, 278, 321, 156, 3, 192, 381, 107, 316, 270, 164, 225, 201, 71, 82, 252, 376, 54, 286, 249, 56, 63, 187, 62, 210, 238, 122, 77, 227, 236, 146, 213, 133, 41, 179, 309, 84, 342, 25, 260, 363, 331, 337, 262, 166, 219, 67, 33, 222, 368, 123, 299, 88, 275, 318, 370, 198, 58, 193, 74, 266, 246, 51] + 12: [255, 250, 366, 199, 73, 110, 225, 272, 372, 220, 302, 42, 204, 169, 112, 44, 125, 100, 166, 64, 66, 99, 222, 162, 120, 83, 319, 336, 105, 247, 213, 377, 331, 240, 43, 53, 130, 155, 0, 58, 86, 310, 230, 206, 359, 141, 239, 373, 158, 180, 283, 136, 122, 352, 178, 338, 88, 21, 56, 353, 327, 266, 362, 301, 315, 187, 12, 284, 117, 79, 275, 95, 149, 281, 89, 273, 15, 165, 335, 5, 251, 258, 92, 324, 78, 195, 323, 322, 14, 185, 45, 65, 329, 237, 143, 311, 226, 98, 205, 371, 294, 200, 298, 257, 246, 212, 118, 248, 190, 50, 278, 16, 271, 286, 234, 321, 32, 252, 320, 132, 303, 233, 6, 60, 107, 144, 114, 325, 293, 146, 343, 221, 127, 192, 339, 108, 57, 91, 41, 173, 183, 383, 55, 182, 223, 290, 295, 68, 102, 241, 231, 189, 379, 297, 81, 210, 82, 97, 123, 268, 33, 326, 350, 67, 270, 279, 96, 307, 177, 103, 134, 382, 264, 63, 253, 161, 235, 106, 47, 202, 378, 287, 259, 229, 265, 291, 304, 232, 355, 174, 375, 228, 69, 71, 176, 317, 116, 129, 1, 224, 140, 179, 72, 367, 243, 126, 354, 25, 10, 318, 9, 119, 334, 244, 211, 19, 150, 296, 3, 152, 133, 157, 254, 61, 351, 374, 113, 7, 172, 289, 75, 194, 27, 59, 357, 280, 115, 24, 282, 314, 333, 349, 340, 216, 300, 164, 376, 4, 76, 285, 34, 242, 203, 124, 208, 215, 39, 380, 142, 256, 36, 312, 54, 111, 46, 22, 260, 330, 153, 238, 193, 249, 214, 70, 30, 360, 341, 292, 198, 346, 101, 308, 191, 84, 305, 128, 18, 184, 156, 267, 181, 137, 104, 135, 358, 37, 40, 207, 347, 26, 363, 109, 151, 147, 277, 23, 227, 313, 13, 365, 236, 196, 49, 168, 90, 368, 364, 74, 370, 262, 121, 148, 131, 94, 35, 175, 51, 87, 288, 31, 309, 356, 38, 85, 316, 328, 48, 167, 306, 348, 209, 337, 201, 139, 11, 274, 345, 342, 381, 344, 170, 332, 171, 197, 52, 29, 20, 186, 299, 269, 245, 17, 369, 276, 2, 80, 77, 62, 154, 217, 8, 163, 219, 28, 361, 218, 160, 261, 188, 138, 159, 263, 93, 145] + 13: [170, 350, 140, 155, 230, 316, 78, 192, 32, 204, 7, 175, 108, 320, 62, 139, 379, 308, 103, 157, 278, 66, 324, 315, 232, 158, 179, 333, 36, 244, 217, 60, 90, 120, 206, 126, 18, 91, 79, 166, 68, 50, 358, 203, 152, 160, 301, 33, 195, 64, 266, 240, 256, 288, 268, 213, 346, 258, 218, 314, 87, 111, 143, 339, 183, 48, 296, 124, 83, 349, 367, 27, 80, 109, 297, 42, 225, 125, 59, 13, 343, 293, 200, 216, 44, 162, 16, 299, 25, 52, 357, 53, 190, 212, 156, 165, 362, 370, 364, 208, 144, 29, 326, 311, 119, 28, 47, 193, 329, 338, 75, 334, 141, 243, 153, 187, 353, 63, 354, 247, 114, 17, 176, 309, 375, 374, 223, 245, 38, 1, 264, 277, 14, 340, 323, 273, 65, 196, 241, 325, 19, 226, 99, 82, 149, 110, 304, 289, 300, 145, 76, 56, 123, 249, 290, 302, 328, 100, 306, 31, 373, 239, 222, 319, 70, 138, 132, 255, 23, 342, 171, 69, 6, 26, 164, 345, 360, 233, 236, 322, 260, 95, 46, 188, 122, 344, 380, 220, 154, 182, 173, 351, 312, 55, 298, 112, 2, 228, 15, 136, 94, 214, 352, 148, 280, 276, 335, 369, 178, 382, 146, 106, 71, 104, 215, 368, 189, 231, 142, 184, 363, 321, 129, 161, 113, 115, 101, 181, 238, 201, 199, 49, 292, 43, 327, 337, 133, 72, 117, 224, 12, 135, 88, 227, 67, 219, 253, 286, 250, 270, 284, 347, 30, 21, 186, 198, 281, 355, 151, 248, 348, 73, 252, 251, 274, 172, 11, 58, 229, 237, 177, 267, 263, 377, 336, 371, 318, 51, 366, 45, 294, 211, 93, 89, 305, 61, 205, 269, 9, 282, 310, 295, 261, 168, 283, 105, 92, 5, 185, 272, 194, 41, 134, 159, 54, 22, 361, 331, 257, 130, 4, 317, 137, 24, 102, 307, 202, 98, 121, 163, 174, 313, 383, 20, 77, 169, 254, 118, 275, 10, 291, 37, 259, 150, 167, 332, 57, 265, 209, 279, 40, 210, 74, 303, 127, 235, 372, 97, 81, 39, 330, 116, 356, 207, 107, 221, 381, 234, 180, 191, 3, 86, 197, 131, 246, 378, 359, 8, 84, 287, 85, 128, 96, 0, 271, 35, 376, 147, 262, 34, 242, 365, 285, 341] + 14: [229, 249, 295, 121, 205, 260, 289, 331, 50, 88, 165, 174, 160, 296, 225, 240, 365, 194, 303, 210, 130, 318, 83, 212, 345, 166, 332, 145, 284, 316, 103, 58, 7, 134, 292, 315, 255, 250, 25, 172, 234, 63, 262, 46, 11, 247, 272, 73, 70, 213, 314, 193, 109, 222, 1, 251, 322, 47, 243, 351, 211, 124, 13, 343, 382, 105, 168, 153, 334, 185, 170, 119, 143, 198, 111, 246, 40, 82, 350, 376, 87, 281, 218, 232, 4, 254, 79, 328, 333, 341, 223, 273, 127, 120, 320, 67, 15, 235, 154, 135, 349, 48, 89, 10, 267, 140, 381, 98, 76, 146, 61, 161, 203, 226, 244, 302, 195, 81, 144, 138, 204, 348, 162, 158, 159, 227, 253, 248, 74, 139, 219, 196, 110, 118, 317, 33, 370, 178, 0, 93, 171, 258, 233, 53, 30, 337, 371, 197, 326, 169, 156, 152, 94, 78, 180, 252, 2, 353, 32, 220, 357, 104, 129, 191, 114, 136, 35, 283, 209, 150, 177, 208, 69, 37, 306, 189, 278, 361, 84, 307, 259, 186, 97, 125, 184, 301, 359, 335, 358, 9, 293, 27, 75, 344, 342, 270, 291, 277, 86, 336, 309, 179, 18, 340, 34, 239, 231, 26, 319, 22, 330, 300, 375, 14, 55, 51, 122, 268, 64, 265, 324, 167, 221, 242, 214, 238, 24, 236, 327, 49, 39, 41, 369, 339, 237, 264, 6, 338, 107, 65, 126, 175, 3, 216, 201, 108, 298, 202, 321, 290, 57, 72, 355, 112, 117, 294, 68, 363, 276, 200, 257, 190, 66, 368, 378, 199, 95, 44, 206, 325, 347, 313, 173, 36, 181, 188, 308, 383, 100, 271, 379, 256, 373, 217, 42, 230, 362, 54, 155, 17, 52, 157, 215, 102, 90, 12, 285, 263, 45, 352, 71, 8, 85, 374, 288, 266, 176, 115, 329, 305, 62, 192, 275, 60, 141, 354, 323, 187, 101, 148, 106, 19, 366, 131, 147, 137, 151, 360, 123, 133, 377, 282, 311, 16, 80, 29, 56, 287, 228, 279, 149, 128, 116, 364, 28, 20, 367, 91, 274, 164, 23, 21, 297, 286, 5, 356, 310, 304, 92, 113, 261, 77, 142, 269, 163, 346, 380, 183, 43, 299, 132, 38, 372, 207, 99, 182, 59, 245, 241, 224, 96, 312, 31, 280] + 15: [130, 245, 75, 131, 322, 366, 260, 93, 281, 270, 85, 328, 11, 220, 27, 315, 264, 198, 325, 127, 192, 368, 356, 276, 0, 308, 216, 364, 188, 299, 122, 156, 345, 348, 25, 144, 15, 43, 365, 84, 262, 44, 125, 214, 132, 33, 46, 6, 88, 269, 294, 71, 273, 58, 136, 302, 336, 249, 19, 152, 359, 78, 49, 22, 226, 272, 91, 153, 154, 287, 297, 186, 160, 255, 133, 323, 62, 349, 123, 52, 380, 243, 288, 219, 227, 56, 18, 182, 194, 8, 165, 211, 218, 280, 178, 21, 57, 203, 83, 109, 5, 263, 12, 354, 172, 197, 124, 286, 362, 215, 35, 196, 209, 309, 13, 267, 334, 303, 112, 108, 251, 119, 48, 320, 170, 247, 206, 253, 92, 116, 310, 179, 324, 284, 23, 138, 335, 238, 282, 107, 45, 54, 176, 183, 155, 252, 151, 177, 207, 105, 254, 271, 26, 265, 187, 41, 355, 333, 360, 1, 352, 275, 229, 180, 224, 190, 293, 103, 200, 74, 221, 168, 24, 142, 34, 381, 208, 278, 318, 120, 258, 236, 90, 256, 164, 59, 268, 347, 314, 274, 184, 223, 94, 159, 60, 173, 235, 117, 77, 61, 137, 378, 374, 304, 128, 266, 9, 342, 379, 2, 114, 97, 234, 73, 369, 307, 217, 326, 80, 353, 351, 246, 233, 339, 283, 63, 149, 346, 175, 231, 329, 311, 162, 102, 50, 313, 373, 213, 370, 17, 148, 332, 317, 163, 31, 279, 69, 191, 169, 3, 201, 70, 104, 185, 232, 296, 338, 145, 341, 301, 375, 257, 14, 292, 134, 38, 121, 277, 115, 82, 87, 306, 383, 65, 312, 110, 68, 189, 146, 113, 225, 166, 331, 321, 244, 239, 29, 222, 298, 300, 20, 135, 230, 248, 10, 53, 289, 161, 89, 330, 290, 39, 51, 344, 106, 212, 150, 67, 55, 79, 32, 361, 261, 199, 237, 193, 285, 86, 343, 72, 99, 147, 129, 66, 340, 291, 376, 242, 47, 372, 205, 382, 337, 316, 126, 98, 350, 371, 157, 174, 36, 101, 250, 30, 37, 167, 327, 181, 76, 367, 139, 143, 100, 28, 96, 118, 204, 210, 158, 195, 4, 202, 16, 241, 357, 42, 81, 377, 95, 358, 140, 228, 259, 295, 40, 111, 319, 305, 363, 141, 64, 240, 7, 171] + 16: [134, 366, 368, 1, 98, 344, 3, 148, 156, 249, 257, 353, 15, 345, 13, 69, 215, 239, 320, 94, 111, 126, 247, 309, 286, 278, 296, 325, 376, 361, 362, 10, 53, 133, 161, 191, 120, 365, 66, 41, 106, 25, 190, 107, 332, 299, 192, 72, 275, 310, 37, 195, 124, 132, 244, 145, 149, 6, 197, 179, 100, 212, 307, 380, 301, 375, 93, 62, 170, 79, 263, 204, 140, 146, 71, 56, 45, 104, 223, 363, 110, 352, 327, 222, 369, 189, 85, 21, 70, 160, 206, 48, 315, 251, 164, 291, 152, 171, 199, 248, 229, 22, 57, 30, 81, 150, 349, 326, 234, 35, 236, 321, 155, 36, 290, 313, 194, 14, 279, 136, 262, 374, 235, 177, 139, 184, 225, 336, 91, 88, 359, 173, 92, 205, 46, 52, 281, 305, 188, 358, 109, 127, 50, 351, 7, 5, 213, 314, 103, 34, 137, 214, 193, 157, 77, 12, 43, 348, 295, 306, 58, 219, 9, 211, 142, 166, 176, 112, 165, 259, 364, 84, 80, 76, 297, 242, 196, 373, 241, 23, 147, 318, 141, 285, 316, 167, 322, 60, 201, 128, 116, 40, 269, 203, 187, 27, 82, 220, 317, 340, 272, 17, 245, 74, 64, 342, 121, 89, 119, 8, 276, 95, 304, 113, 61, 33, 44, 289, 287, 294, 99, 356, 96, 54, 207, 284, 217, 130, 303, 143, 383, 65, 292, 232, 4, 200, 266, 324, 256, 135, 118, 90, 265, 108, 175, 227, 339, 178, 319, 131, 138, 101, 280, 129, 209, 181, 73, 182, 198, 228, 83, 261, 260, 254, 75, 49, 283, 346, 377, 357, 370, 158, 105, 20, 210, 169, 271, 144, 298, 180, 38, 379, 238, 102, 240, 47, 216, 329, 11, 273, 255, 16, 335, 31, 174, 18, 378, 268, 230, 382, 162, 86, 24, 226, 51, 231, 19, 208, 55, 218, 243, 28, 68, 338, 302, 367, 237, 159, 42, 354, 267, 381, 117, 32, 67, 277, 224, 270, 328, 153, 202, 221, 333, 172, 168, 63, 330, 312, 183, 87, 59, 282, 334, 252, 337, 186, 258, 355, 125, 300, 274, 246, 115, 0, 122, 264, 253, 360, 26, 288, 331, 154, 250, 347, 97, 39, 372, 29, 323, 163, 308, 341, 185, 114, 78, 123, 2, 311, 293, 151, 343, 233, 371, 350] + 17: [131, 170, 202, 203, 235, 237, 251, 283, 312, 320, 325, 354, 119, 169, 14, 86, 122, 127, 358, 69, 72, 121, 152, 159, 284, 321, 156, 209, 252, 301, 333, 123, 174, 293, 34, 54, 236, 228, 162, 248, 37, 95, 192, 150, 368, 247, 111, 281, 79, 296, 371, 332, 154, 110, 369, 233, 178, 71, 35, 322, 337, 194, 367, 376, 336, 21, 372, 318, 242, 77, 51, 68, 375, 8, 46, 272, 260, 49, 238, 128, 32, 176, 0, 268, 261, 29, 96, 373, 18, 7, 115, 102, 329, 315, 201, 17, 220, 351, 290, 180, 292, 19, 365, 120, 161, 53, 109, 334, 250, 2, 234, 335, 84, 45, 196, 249, 151, 65, 277, 348, 188, 130, 370, 199, 135, 42, 295, 262, 48, 344, 160, 285, 303, 193, 107, 313, 214, 166, 341, 360, 326, 255, 288, 306, 108, 225, 297, 148, 327, 269, 81, 302, 55, 57, 356, 353, 200, 359, 133, 73, 98, 191, 363, 379, 25, 76, 30, 289, 15, 155, 189, 317, 143, 259, 41, 181, 93, 85, 316, 66, 383, 117, 347, 207, 10, 168, 215, 382, 239, 210, 355, 223, 100, 136, 324, 39, 92, 171, 13, 309, 213, 67, 270, 11, 26, 311, 125, 134, 273, 245, 47, 101, 342, 364, 350, 129, 330, 78, 246, 175, 206, 298, 147, 118, 195, 112, 352, 224, 305, 357, 1, 75, 22, 139, 59, 198, 240, 217, 88, 280, 294, 254, 185, 222, 231, 146, 173, 204, 244, 287, 182, 378, 362, 219, 5, 157, 380, 142, 264, 271, 94, 106, 144, 229, 374, 91, 275, 104, 211, 263, 61, 3, 310, 64, 83, 23, 28, 12, 114, 276, 97, 99, 331, 187, 179, 274, 163, 257, 212, 87, 286, 339, 103, 70, 340, 338, 27, 124, 50, 349, 138, 366, 232, 56, 38, 74, 20, 205, 361, 267, 132, 149, 307, 126, 343, 172, 381, 165, 36, 186, 300, 308, 164, 4, 80, 58, 279, 167, 218, 256, 43, 216, 258, 345, 227, 40, 105, 346, 323, 6, 145, 9, 82, 89, 44, 158, 31, 184, 208, 319, 113, 328, 24, 33, 60, 221, 299, 16, 153, 230, 140, 190, 266, 62, 177, 314, 304, 253, 265, 278, 63, 291, 226, 137, 197, 52, 243, 282, 116, 377, 90, 183, 241, 141] + 18: [271, 293, 314, 344, 382, 1, 32, 59, 78, 147, 309, 340, 30, 52, 74, 80, 97, 120, 156, 183, 222, 227, 242, 268, 251, 288, 54, 94, 163, 95, 117, 295, 55, 226, 327, 374, 208, 165, 283, 7, 176, 150, 112, 311, 107, 160, 310, 91, 47, 35, 318, 296, 313, 363, 179, 219, 76, 173, 194, 49, 10, 144, 181, 343, 203, 276, 256, 158, 85, 280, 198, 186, 43, 282, 89, 149, 350, 178, 270, 197, 258, 212, 323, 128, 69, 376, 326, 366, 223, 195, 12, 275, 247, 218, 206, 221, 320, 26, 33, 202, 22, 361, 187, 322, 133, 372, 277, 196, 217, 189, 246, 346, 233, 111, 81, 348, 220, 90, 281, 353, 257, 236, 378, 46, 263, 102, 63, 108, 27, 42, 294, 379, 300, 142, 151, 274, 83, 122, 125, 292, 358, 66, 298, 252, 245, 116, 253, 19, 331, 159, 205, 329, 383, 4, 375, 336, 65, 239, 297, 164, 359, 232, 279, 351, 175, 56, 364, 62, 57, 174, 291, 70, 269, 129, 157, 132, 303, 51, 381, 134, 138, 192, 237, 190, 23, 241, 50, 93, 60, 200, 352, 290, 71, 169, 284, 330, 307, 0, 201, 13, 153, 369, 84, 377, 228, 380, 306, 266, 177, 87, 349, 244, 48, 332, 29, 14, 106, 110, 360, 240, 171, 225, 39, 145, 127, 216, 115, 24, 185, 368, 373, 188, 20, 316, 152, 182, 213, 338, 161, 345, 260, 304, 31, 341, 73, 131, 235, 40, 148, 17, 86, 5, 6, 36, 25, 101, 339, 109, 61, 96, 234, 64, 172, 100, 118, 114, 167, 204, 41, 79, 308, 38, 243, 209, 354, 230, 356, 119, 146, 124, 191, 261, 140, 319, 143, 82, 238, 249, 272, 215, 136, 285, 104, 262, 15, 264, 287, 123, 2, 21, 193, 347, 355, 105, 342, 278, 231, 155, 324, 229, 370, 305, 45, 113, 321, 77, 121, 334, 8, 362, 168, 170, 299, 11, 53, 28, 68, 3, 365, 135, 302, 255, 325, 317, 37, 184, 333, 154, 301, 67, 141, 367, 328, 265, 214, 224, 44, 273, 16, 207, 34, 103, 337, 88, 289, 335, 166, 126, 180, 199, 210, 315, 139, 98, 371, 162, 58, 9, 254, 130, 312, 72, 211, 286, 357, 99, 248, 137, 250, 267, 92, 259, 75, 18] + 19: [24, 133, 198, 245, 253, 254, 263, 268, 319, 355, 369, 375, 187, 109, 229, 305, 317, 350, 12, 34, 50, 81, 98, 107, 190, 69, 199, 327, 55, 104, 145, 196, 274, 316, 378, 93, 366, 360, 17, 43, 72, 220, 382, 78, 142, 175, 250, 276, 7, 96, 259, 108, 3, 232, 59, 304, 261, 379, 226, 282, 90, 122, 353, 277, 380, 26, 258, 73, 367, 308, 102, 169, 288, 217, 246, 170, 95, 284, 383, 137, 348, 21, 266, 151, 256, 66, 171, 134, 272, 143, 85, 281, 370, 294, 51, 0, 265, 216, 117, 54, 76, 260, 160, 376, 14, 314, 100, 5, 57, 114, 89, 30, 68, 40, 293, 88, 139, 357, 136, 56, 377, 197, 97, 335, 239, 273, 23, 92, 132, 324, 155, 236, 120, 300, 77, 46, 6, 204, 60, 62, 181, 351, 205, 248, 338, 8, 271, 25, 228, 138, 287, 336, 242, 341, 124, 267, 330, 219, 227, 147, 359, 257, 342, 45, 280, 99, 247, 299, 313, 192, 296, 41, 222, 156, 31, 18, 298, 212, 106, 269, 180, 318, 334, 79, 71, 172, 2, 75, 65, 105, 28, 289, 42, 279, 303, 354, 36, 127, 306, 333, 146, 209, 118, 123, 207, 183, 27, 264, 1, 182, 157, 356, 200, 22, 315, 344, 312, 126, 193, 153, 285, 121, 221, 202, 326, 32, 361, 86, 101, 322, 349, 159, 238, 195, 203, 19, 129, 208, 63, 162, 240, 291, 38, 301, 13, 83, 297, 262, 329, 168, 103, 178, 64, 161, 218, 35, 116, 165, 347, 331, 191, 234, 94, 185, 307, 188, 33, 152, 135, 283, 84, 325, 362, 320, 112, 201, 115, 255, 20, 275, 233, 167, 215, 177, 309, 332, 149, 290, 141, 91, 164, 230, 15, 29, 67, 10, 148, 340, 286, 358, 243, 70, 363, 58, 278, 173, 144, 80, 372, 174, 235, 39, 211, 52, 154, 189, 53, 251, 345, 131, 47, 343, 150, 214, 339, 241, 184, 158, 310, 368, 374, 213, 140, 87, 371, 163, 111, 364, 328, 9, 352, 270, 249, 125, 119, 311, 381, 223, 179, 61, 373, 365, 166, 16, 244, 194, 321, 82, 231, 224, 206, 302, 113, 292, 295, 49, 110, 346, 130, 176, 11, 210, 48, 74, 323, 44, 128, 337, 4, 237, 225, 186, 37, 252] + 20: [161, 200, 206, 214, 262, 263, 266, 281, 299, 303, 331, 350, 116, 52, 54, 61, 76, 107, 137, 167, 168, 179, 181, 189, 334, 17, 117, 198, 241, 242, 249, 260, 277, 280, 27, 51, 98, 284, 233, 97, 170, 56, 356, 329, 31, 155, 143, 269, 153, 65, 58, 279, 304, 360, 283, 225, 192, 298, 197, 288, 278, 125, 184, 235, 150, 246, 3, 80, 333, 152, 203, 171, 302, 290, 101, 194, 367, 345, 39, 75, 21, 323, 220, 1, 336, 273, 49, 240, 213, 274, 338, 159, 120, 341, 251, 38, 82, 10, 160, 13, 26, 60, 232, 374, 296, 354, 23, 237, 144, 209, 342, 176, 375, 166, 310, 230, 319, 47, 234, 44, 66, 158, 259, 190, 35, 346, 236, 24, 366, 64, 261, 69, 193, 62, 95, 92, 267, 215, 5, 253, 88, 163, 103, 244, 99, 32, 128, 320, 218, 247, 294, 201, 139, 205, 129, 255, 87, 204, 221, 347, 348, 309, 169, 257, 210, 67, 293, 109, 43, 105, 141, 132, 364, 146, 42, 252, 89, 11, 19, 373, 369, 363, 229, 258, 177, 53, 22, 377, 217, 124, 271, 186, 45, 353, 111, 292, 57, 328, 86, 33, 202, 151, 63, 383, 268, 224, 228, 286, 90, 41, 196, 29, 254, 321, 355, 222, 145, 195, 91, 140, 239, 378, 165, 78, 231, 156, 157, 68, 4, 130, 324, 216, 6, 175, 238, 15, 351, 28, 7, 275, 335, 291, 164, 135, 74, 219, 131, 370, 316, 40, 85, 301, 46, 307, 93, 362, 104, 314, 300, 365, 191, 352, 207, 108, 282, 379, 148, 264, 121, 368, 178, 332, 212, 59, 305, 306, 8, 339, 162, 381, 126, 0, 372, 380, 337, 142, 83, 183, 122, 226, 325, 9, 315, 25, 119, 106, 312, 199, 276, 245, 37, 112, 185, 227, 357, 118, 256, 20, 94, 289, 16, 133, 174, 2, 371, 187, 182, 30, 208, 73, 326, 79, 72, 295, 272, 376, 330, 311, 71, 343, 340, 127, 327, 115, 136, 70, 14, 287, 188, 50, 172, 359, 18, 12, 84, 285, 313, 147, 317, 123, 138, 382, 322, 265, 34, 248, 308, 113, 102, 344, 55, 110, 318, 96, 77, 36, 114, 297, 154, 250, 211, 149, 81, 173, 270, 180, 358, 361, 243, 100, 223, 349, 48, 134] + 21: [360, 10, 17, 35, 58, 93, 123, 264, 306, 321, 339, 350, 152, 283, 297, 311, 327, 332, 11, 42, 44, 108, 179, 369, 355, 331, 286, 100, 258, 67, 68, 328, 166, 95, 107, 167, 47, 189, 49, 356, 353, 57, 319, 76, 202, 278, 235, 224, 173, 43, 73, 139, 125, 136, 27, 342, 6, 298, 248, 272, 281, 280, 131, 358, 28, 24, 346, 41, 357, 282, 101, 127, 132, 349, 367, 1, 260, 77, 340, 273, 74, 55, 188, 322, 103, 16, 69, 234, 62, 155, 137, 305, 241, 228, 326, 343, 316, 0, 121, 23, 267, 383, 40, 186, 96, 129, 242, 7, 20, 91, 175, 220, 18, 66, 275, 146, 12, 2, 249, 32, 371, 284, 109, 141, 372, 312, 204, 329, 300, 122, 85, 212, 308, 182, 19, 221, 338, 21, 65, 274, 335, 130, 99, 222, 254, 377, 128, 144, 84, 262, 323, 337, 333, 253, 265, 59, 4, 226, 151, 9, 174, 64, 133, 148, 83, 105, 106, 61, 158, 88, 149, 196, 245, 352, 303, 102, 36, 164, 255, 116, 80, 94, 250, 15, 72, 336, 159, 34, 379, 374, 114, 231, 124, 364, 178, 26, 230, 197, 223, 3, 115, 334, 294, 296, 376, 208, 150, 365, 135, 168, 309, 30, 171, 381, 70, 375, 119, 87, 252, 172, 111, 63, 304, 154, 266, 110, 163, 31, 71, 185, 118, 351, 313, 263, 60, 165, 160, 81, 75, 37, 257, 112, 345, 147, 209, 229, 318, 290, 187, 213, 143, 56, 113, 218, 45, 307, 52, 325, 39, 192, 203, 126, 156, 180, 140, 92, 214, 22, 344, 89, 259, 237, 301, 195, 181, 142, 302, 200, 120, 184, 227, 368, 97, 347, 292, 157, 315, 145, 289, 82, 199, 238, 219, 243, 362, 317, 198, 48, 201, 314, 153, 210, 169, 191, 225, 78, 98, 354, 170, 216, 207, 324, 190, 38, 348, 240, 194, 270, 287, 291, 269, 261, 239, 54, 256, 50, 117, 279, 299, 233, 359, 211, 14, 215, 268, 162, 236, 53, 232, 51, 8, 33, 251, 370, 104, 244, 330, 176, 46, 86, 277, 177, 25, 288, 246, 378, 79, 276, 13, 217, 247, 29, 363, 380, 293, 373, 138, 295, 193, 134, 382, 161, 206, 205, 361, 320, 366, 183, 90, 5, 341, 271, 310, 285] + 22: [254, 154, 157, 217, 247, 250, 253, 267, 307, 310, 346, 359, 141, 287, 289, 292, 323, 360, 2, 28, 79, 95, 101, 143, 181, 191, 225, 269, 96, 298, 344, 5, 166, 234, 135, 192, 221, 226, 131, 236, 136, 208, 350, 84, 293, 297, 142, 205, 260, 349, 25, 124, 138, 160, 128, 223, 86, 313, 249, 280, 261, 306, 185, 49, 341, 335, 211, 187, 212, 354, 194, 315, 328, 173, 222, 366, 321, 373, 290, 219, 255, 363, 203, 325, 281, 252, 80, 57, 374, 11, 81, 266, 296, 370, 231, 10, 216, 352, 375, 27, 18, 159, 299, 176, 294, 145, 114, 107, 343, 317, 153, 60, 82, 210, 322, 273, 262, 16, 21, 167, 278, 58, 345, 264, 48, 15, 32, 180, 186, 64, 43, 243, 209, 383, 338, 174, 339, 71, 238, 241, 137, 152, 172, 121, 148, 177, 55, 67, 206, 233, 22, 97, 246, 308, 6, 368, 24, 4, 240, 195, 122, 324, 189, 271, 274, 31, 150, 56, 224, 37, 333, 188, 8, 156, 1, 305, 26, 151, 179, 245, 337, 144, 130, 70, 193, 75, 239, 87, 46, 182, 190, 270, 288, 230, 355, 251, 378, 109, 319, 318, 379, 282, 91, 197, 183, 184, 235, 100, 68, 347, 367, 98, 364, 334, 134, 213, 20, 73, 329, 218, 83, 340, 12, 65, 356, 35, 291, 304, 178, 201, 118, 311, 372, 53, 59, 78, 382, 52, 30, 66, 376, 242, 61, 272, 116, 170, 248, 227, 314, 85, 19, 127, 110, 164, 244, 111, 161, 113, 320, 301, 275, 279, 104, 140, 94, 38, 93, 336, 76, 228, 72, 303, 353, 103, 257, 146, 371, 42, 44, 381, 369, 175, 99, 256, 268, 92, 36, 158, 316, 115, 29, 149, 365, 220, 165, 327, 0, 102, 168, 284, 33, 207, 112, 357, 214, 14, 39, 54, 202, 232, 69, 88, 204, 132, 74, 50, 123, 40, 41, 283, 286, 120, 62, 163, 51, 147, 263, 285, 312, 258, 133, 377, 300, 196, 129, 277, 198, 276, 139, 348, 259, 361, 47, 199, 215, 108, 342, 295, 9, 34, 117, 7, 106, 63, 309, 45, 13, 119, 90, 162, 126, 330, 169, 351, 125, 3, 105, 155, 77, 237, 171, 23, 332, 331, 362, 229, 17, 358, 380, 200, 89, 265, 302, 326] + 23: [26, 43, 235, 286, 320, 37, 81, 242, 265, 64, 168, 288, 314, 375, 357, 22, 262, 13, 234, 291, 39, 75, 189, 8, 124, 221, 98, 352, 108, 355, 277, 30, 142, 18, 3, 15, 172, 233, 210, 65, 45, 21, 373, 31, 141, 60, 226, 4, 275, 70, 255, 125, 54, 68, 104, 34, 42, 198, 366, 216, 71, 309, 224, 93, 179, 335, 25, 188, 381, 112, 251, 113, 305, 56, 102, 40, 17, 196, 206, 227, 88, 318, 166, 29, 126, 59, 111, 152, 254, 307, 107, 285, 158, 299, 332, 110, 328, 372, 213, 313, 160, 321, 346, 349, 61, 343, 217, 296, 264, 182, 190, 271, 361, 333, 339, 243, 248, 377, 199, 186, 175, 84, 130, 249, 215, 53, 76, 134, 267, 230, 208, 380, 100, 103, 159, 35, 38, 298, 250, 200, 336, 253, 364, 293, 132, 327, 57, 131, 228, 1, 204, 0, 115, 367, 118, 145, 187, 279, 169, 209, 317, 157, 207, 344, 268, 161, 165, 105, 167, 324, 127, 133, 149, 222, 178, 109, 155, 205, 238, 89, 244, 197, 241, 92, 191, 356, 5, 58, 354, 79, 322, 278, 323, 12, 11, 340, 229, 101, 301, 73, 121, 214, 337, 83, 252, 203, 304, 283, 91, 44, 137, 143, 14, 260, 350, 183, 85, 269, 374, 263, 151, 359, 325, 363, 122, 308, 383, 246, 72, 87, 192, 41, 220, 292, 273, 258, 114, 369, 266, 289, 300, 306, 67, 371, 312, 129, 231, 90, 212, 106, 62, 63, 135, 240, 180, 78, 36, 311, 52, 348, 163, 33, 276, 345, 177, 24, 94, 257, 378, 136, 66, 303, 7, 139, 368, 117, 281, 173, 232, 148, 74, 272, 329, 347, 16, 297, 46, 116, 225, 82, 156, 315, 219, 290, 365, 162, 245, 97, 202, 138, 51, 331, 176, 6, 146, 10, 370, 193, 128, 123, 140, 247, 69, 201, 261, 270, 379, 342, 153, 330, 28, 99, 9, 334, 274, 295, 27, 218, 50, 195, 185, 341, 164, 294, 319, 259, 120, 48, 154, 55, 184, 95, 119, 223, 302, 236, 310, 20, 376, 77, 358, 86, 326, 237, 96, 382, 174, 362, 353, 171, 211, 19, 47, 351, 80, 194, 32, 284, 280, 239, 282, 181, 147, 170, 287, 2, 144, 23, 150, 316, 256, 338, 49, 360] + 24: [154, 74, 81, 115, 166, 198, 241, 264, 298, 314, 353, 379, 265, 21, 127, 129, 133, 158, 179, 232, 251, 327, 368, 383, 252, 100, 152, 219, 358, 2, 45, 89, 34, 36, 155, 374, 9, 197, 365, 107, 359, 98, 300, 340, 117, 261, 330, 40, 246, 233, 108, 303, 8, 109, 332, 347, 208, 305, 364, 82, 20, 206, 278, 60, 14, 281, 132, 75, 285, 371, 322, 17, 7, 130, 48, 323, 30, 176, 373, 183, 124, 289, 33, 214, 223, 366, 49, 287, 295, 181, 88, 42, 248, 141, 316, 69, 0, 35, 142, 307, 235, 280, 170, 120, 342, 125, 165, 123, 52, 31, 299, 149, 178, 222, 275, 188, 55, 201, 167, 243, 92, 53, 306, 145, 58, 116, 282, 175, 171, 229, 199, 349, 6, 283, 54, 377, 77, 269, 225, 79, 126, 262, 187, 90, 22, 237, 190, 46, 12, 84, 339, 72, 212, 329, 247, 119, 372, 62, 164, 254, 380, 140, 313, 4, 150, 44, 25, 200, 263, 266, 26, 71, 177, 63, 56, 134, 249, 50, 101, 227, 357, 5, 137, 61, 350, 272, 13, 194, 348, 320, 102, 180, 70, 64, 231, 91, 356, 310, 16, 65, 276, 352, 209, 362, 369, 286, 343, 15, 113, 205, 168, 239, 23, 284, 217, 18, 85, 112, 319, 382, 138, 3, 153, 312, 271, 32, 57, 240, 189, 121, 230, 144, 156, 118, 321, 304, 333, 76, 331, 250, 128, 27, 207, 344, 355, 338, 83, 268, 103, 94, 148, 228, 290, 324, 375, 195, 260, 336, 257, 302, 309, 151, 67, 162, 97, 361, 186, 192, 308, 351, 169, 66, 215, 99, 191, 174, 87, 105, 370, 277, 106, 293, 59, 315, 203, 253, 238, 68, 211, 28, 146, 354, 226, 341, 376, 1, 346, 161, 19, 378, 110, 381, 29, 135, 131, 96, 147, 242, 267, 296, 317, 184, 111, 360, 255, 24, 318, 73, 326, 292, 270, 136, 160, 43, 95, 157, 301, 204, 274, 182, 221, 47, 37, 10, 193, 236, 328, 139, 334, 159, 41, 224, 279, 51, 218, 196, 288, 256, 172, 185, 294, 202, 273, 245, 297, 244, 258, 363, 291, 367, 259, 337, 104, 78, 93, 345, 114, 11, 335, 173, 234, 80, 210, 122, 220, 213, 325, 39, 216, 311, 38, 86, 163, 143] + 25: [315, 239, 156, 220, 263, 267, 277, 123, 178, 91, 158, 326, 269, 338, 43, 44, 303, 382, 19, 121, 165, 129, 189, 233, 379, 329, 6, 206, 106, 194, 13, 170, 151, 195, 270, 255, 282, 167, 141, 374, 108, 272, 288, 47, 148, 50, 182, 241, 72, 164, 132, 290, 370, 117, 70, 119, 71, 169, 198, 266, 139, 294, 256, 114, 350, 69, 67, 36, 283, 264, 12, 104, 228, 120, 153, 242, 295, 133, 79, 337, 175, 142, 285, 219, 247, 59, 84, 146, 352, 185, 24, 111, 16, 286, 42, 301, 317, 32, 234, 126, 347, 203, 149, 150, 93, 45, 155, 88, 180, 33, 215, 138, 143, 81, 201, 238, 186, 293, 224, 217, 276, 49, 262, 240, 161, 204, 213, 17, 271, 40, 92, 90, 23, 73, 318, 373, 227, 336, 184, 355, 275, 362, 118, 26, 372, 10, 216, 197, 343, 369, 357, 310, 243, 250, 64, 335, 107, 171, 380, 9, 279, 115, 199, 257, 300, 109, 311, 57, 214, 112, 314, 306, 60, 211, 237, 181, 212, 34, 183, 363, 29, 77, 102, 252, 366, 291, 319, 274, 356, 157, 320, 172, 284, 376, 296, 222, 367, 225, 316, 14, 105, 86, 124, 147, 192, 246, 38, 131, 244, 20, 304, 144, 11, 281, 210, 265, 62, 113, 66, 346, 99, 196, 187, 28, 223, 51, 87, 3, 98, 152, 8, 154, 253, 348, 312, 58, 251, 163, 173, 48, 65, 327, 160, 368, 323, 191, 278, 254, 53, 110, 128, 174, 83, 52, 280, 302, 125, 97, 383, 339, 159, 127, 236, 322, 258, 0, 5, 342, 56, 307, 330, 297, 74, 292, 208, 95, 168, 309, 190, 18, 229, 218, 324, 202, 249, 22, 226, 63, 268, 25, 145, 136, 231, 135, 259, 15, 179, 166, 245, 41, 298, 2, 232, 378, 89, 103, 55, 328, 193, 260, 371, 205, 334, 39, 333, 381, 209, 176, 130, 134, 340, 177, 364, 344, 82, 61, 235, 100, 341, 140, 221, 94, 31, 325, 96, 188, 299, 332, 273, 230, 54, 4, 68, 116, 200, 35, 122, 360, 85, 162, 313, 287, 377, 359, 345, 365, 361, 75, 21, 1, 101, 78, 7, 207, 80, 261, 308, 37, 358, 353, 289, 349, 30, 375, 305, 351, 76, 321, 354, 331, 27, 248, 46, 137] + 26: [136, 218, 228, 234, 242, 246, 261, 305, 348, 351, 374, 375, 237, 54, 68, 86, 96, 106, 111, 139, 169, 170, 175, 209, 89, 78, 346, 56, 250, 95, 207, 60, 203, 260, 299, 1, 269, 318, 370, 134, 317, 29, 208, 23, 155, 210, 132, 213, 217, 178, 251, 215, 50, 300, 361, 119, 282, 248, 273, 168, 172, 156, 151, 118, 252, 93, 352, 39, 163, 283, 286, 288, 90, 55, 166, 122, 35, 144, 362, 100, 193, 295, 322, 309, 80, 62, 360, 186, 239, 231, 3, 9, 314, 381, 366, 52, 6, 376, 128, 2, 20, 45, 177, 364, 244, 206, 330, 379, 292, 22, 180, 174, 66, 98, 75, 340, 34, 101, 85, 84, 380, 355, 200, 190, 24, 81, 298, 336, 335, 284, 211, 310, 11, 293, 131, 191, 79, 159, 256, 12, 275, 216, 222, 58, 124, 266, 167, 259, 97, 17, 41, 114, 47, 297, 296, 345, 18, 316, 30, 204, 27, 51, 115, 123, 121, 59, 199, 28, 53, 344, 315, 359, 99, 268, 171, 129, 165, 74, 205, 46, 236, 337, 135, 67, 287, 194, 0, 143, 140, 77, 274, 147, 21, 279, 158, 265, 353, 294, 107, 153, 105, 40, 258, 164, 181, 16, 238, 110, 179, 173, 195, 350, 125, 72, 270, 188, 291, 289, 227, 328, 323, 241, 277, 37, 197, 333, 240, 235, 253, 116, 369, 332, 319, 103, 65, 196, 49, 112, 326, 257, 339, 120, 307, 382, 26, 102, 357, 32, 189, 141, 154, 243, 230, 224, 278, 146, 138, 327, 15, 267, 232, 325, 334, 313, 150, 254, 341, 126, 198, 306, 127, 202, 182, 220, 303, 13, 320, 160, 38, 92, 176, 44, 285, 290, 372, 7, 347, 377, 5, 149, 308, 36, 48, 229, 57, 378, 312, 264, 8, 324, 233, 162, 130, 108, 311, 142, 192, 161, 117, 368, 43, 358, 281, 304, 137, 245, 272, 152, 104, 212, 383, 329, 64, 373, 349, 338, 183, 88, 31, 367, 365, 73, 363, 157, 185, 4, 271, 70, 33, 249, 148, 356, 71, 87, 25, 301, 255, 14, 214, 263, 94, 223, 82, 262, 201, 113, 280, 219, 354, 145, 63, 371, 276, 226, 247, 133, 343, 76, 187, 225, 331, 91, 342, 184, 69, 61, 83, 321, 221, 10, 19, 302, 109, 42] + 27: [103, 137, 146, 163, 176, 220, 240, 244, 262, 310, 338, 342, 225, 232, 241, 218, 249, 367, 313, 64, 56, 212, 270, 53, 286, 216, 118, 10, 18, 365, 96, 150, 194, 235, 285, 60, 12, 99, 356, 54, 46, 321, 144, 50, 86, 245, 236, 39, 312, 228, 325, 254, 214, 43, 256, 302, 372, 209, 17, 63, 90, 182, 32, 42, 136, 307, 0, 231, 52, 243, 36, 93, 234, 112, 196, 109, 329, 362, 47, 306, 185, 198, 19, 180, 37, 97, 149, 25, 126, 287, 120, 123, 223, 94, 138, 200, 177, 207, 242, 297, 91, 375, 199, 153, 8, 68, 255, 203, 4, 358, 251, 78, 305, 296, 29, 13, 281, 173, 378, 51, 263, 49, 183, 343, 370, 6, 106, 151, 22, 158, 210, 100, 61, 119, 55, 258, 193, 326, 67, 95, 280, 330, 226, 247, 265, 320, 190, 339, 373, 271, 267, 250, 344, 282, 334, 221, 168, 315, 237, 383, 141, 316, 276, 353, 132, 113, 1, 277, 73, 357, 336, 324, 284, 368, 124, 147, 79, 65, 195, 295, 252, 197, 143, 76, 322, 309, 45, 248, 14, 152, 87, 298, 69, 135, 222, 266, 140, 125, 122, 299, 116, 74, 257, 355, 30, 26, 179, 380, 278, 161, 215, 101, 167, 184, 108, 40, 349, 269, 128, 75, 201, 261, 174, 28, 84, 352, 16, 92, 80, 204, 376, 335, 82, 7, 114, 20, 105, 319, 166, 148, 227, 164, 202, 230, 332, 187, 274, 38, 275, 41, 110, 160, 341, 71, 366, 303, 102, 162, 379, 129, 139, 259, 165, 169, 211, 233, 24, 48, 238, 155, 291, 2, 288, 264, 273, 170, 34, 374, 213, 85, 72, 328, 246, 111, 130, 346, 189, 311, 354, 300, 131, 178, 115, 27, 304, 121, 217, 154, 345, 363, 81, 350, 181, 290, 294, 15, 172, 171, 35, 359, 283, 314, 301, 31, 292, 272, 83, 337, 268, 9, 205, 293, 62, 331, 104, 70, 188, 323, 77, 88, 175, 239, 33, 208, 192, 11, 145, 351, 369, 340, 308, 317, 360, 44, 361, 186, 142, 371, 117, 333, 377, 21, 156, 133, 327, 318, 107, 253, 382, 57, 219, 58, 348, 260, 23, 191, 127, 347, 159, 157, 289, 134, 59, 279, 224, 381, 89, 98, 5, 66, 3, 229, 364, 206] + 28: [161, 180, 207, 215, 257, 279, 311, 329, 336, 338, 343, 353, 250, 190, 298, 359, 379, 10, 18, 60, 86, 115, 150, 156, 26, 67, 367, 14, 314, 15, 364, 303, 76, 145, 301, 173, 185, 59, 163, 376, 121, 205, 372, 348, 101, 181, 2, 132, 168, 346, 88, 240, 178, 6, 148, 31, 89, 174, 187, 9, 344, 7, 116, 32, 300, 327, 362, 154, 293, 126, 189, 138, 92, 307, 42, 167, 238, 278, 118, 66, 170, 72, 276, 147, 356, 253, 325, 135, 13, 380, 244, 210, 211, 85, 237, 11, 153, 113, 368, 285, 217, 16, 112, 322, 294, 38, 267, 309, 70, 214, 223, 339, 349, 247, 179, 61, 34, 225, 275, 313, 317, 93, 227, 28, 19, 48, 256, 270, 334, 146, 347, 40, 255, 122, 137, 90, 289, 290, 64, 159, 79, 96, 373, 318, 306, 288, 320, 361, 370, 282, 44, 69, 377, 80, 47, 186, 302, 103, 236, 383, 182, 316, 245, 106, 366, 284, 35, 304, 162, 233, 166, 200, 125, 134, 49, 143, 24, 351, 141, 371, 243, 172, 378, 352, 209, 268, 142, 258, 73, 39, 111, 53, 30, 195, 75, 228, 308, 193, 183, 12, 332, 8, 192, 212, 107, 155, 374, 261, 242, 222, 57, 82, 109, 119, 123, 254, 52, 97, 272, 158, 340, 43, 27, 104, 342, 231, 139, 262, 25, 120, 199, 5, 221, 341, 365, 169, 363, 234, 259, 62, 99, 176, 37, 58, 299, 68, 171, 305, 117, 149, 140, 354, 286, 226, 357, 165, 20, 114, 46, 175, 151, 110, 277, 360, 241, 152, 263, 204, 56, 127, 264, 36, 133, 131, 232, 198, 197, 203, 108, 51, 216, 315, 265, 296, 280, 246, 29, 319, 321, 188, 369, 71, 33, 273, 295, 375, 218, 213, 291, 83, 4, 328, 224, 100, 269, 333, 208, 381, 330, 266, 45, 202, 355, 283, 50, 129, 271, 206, 194, 230, 297, 292, 55, 87, 312, 324, 128, 157, 41, 184, 249, 21, 136, 251, 94, 350, 0, 22, 219, 63, 326, 1, 358, 177, 382, 345, 252, 124, 248, 77, 95, 235, 105, 335, 260, 74, 84, 17, 229, 54, 201, 196, 191, 160, 337, 78, 81, 98, 144, 287, 102, 65, 239, 310, 3, 130, 220, 91, 23, 274, 331, 323, 281, 164] + 29: [118, 324, 19, 205, 79, 199, 252, 8, 98, 355, 152, 176, 244, 4, 18, 105, 157, 84, 162, 234, 64, 212, 29, 66, 361, 123, 248, 380, 217, 111, 374, 251, 279, 367, 108, 91, 197, 175, 350, 378, 292, 167, 327, 288, 141, 189, 256, 26, 73, 87, 357, 85, 101, 359, 55, 261, 17, 286, 335, 240, 196, 16, 82, 329, 280, 312, 52, 356, 290, 201, 274, 322, 336, 208, 343, 172, 72, 313, 230, 153, 149, 254, 284, 363, 318, 202, 24, 132, 347, 269, 170, 7, 376, 49, 37, 39, 51, 320, 188, 61, 225, 125, 126, 158, 168, 325, 1, 285, 137, 309, 332, 364, 136, 20, 93, 70, 113, 334, 260, 81, 277, 186, 235, 94, 131, 32, 315, 42, 227, 30, 41, 143, 323, 74, 190, 278, 349, 146, 83, 377, 62, 0, 80, 138, 198, 262, 134, 50, 360, 142, 373, 381, 13, 306, 291, 107, 203, 110, 375, 145, 182, 104, 294, 103, 338, 43, 245, 241, 295, 33, 159, 68, 12, 181, 316, 209, 192, 246, 223, 144, 304, 265, 57, 216, 139, 59, 229, 147, 233, 296, 358, 249, 362, 339, 112, 207, 273, 264, 169, 302, 194, 71, 5, 268, 67, 129, 160, 102, 178, 228, 224, 28, 239, 297, 133, 281, 60, 333, 351, 293, 298, 353, 317, 287, 161, 219, 27, 342, 236, 258, 148, 38, 314, 267, 344, 114, 166, 99, 75, 165, 44, 354, 266, 319, 156, 171, 242, 369, 270, 154, 130, 221, 282, 120, 346, 382, 200, 300, 348, 40, 307, 263, 226, 321, 173, 379, 183, 193, 215, 345, 77, 213, 253, 92, 96, 6, 23, 76, 243, 163, 128, 78, 328, 368, 250, 308, 231, 372, 311, 25, 65, 86, 259, 3, 371, 56, 299, 109, 155, 177, 127, 54, 174, 370, 116, 151, 47, 53, 15, 45, 21, 289, 11, 365, 119, 341, 257, 276, 31, 10, 352, 164, 301, 185, 255, 124, 326, 210, 218, 310, 340, 220, 9, 115, 95, 305, 89, 195, 140, 237, 232, 90, 238, 303, 187, 184, 100, 117, 383, 58, 48, 63, 222, 97, 122, 191, 69, 337, 330, 247, 271, 88, 366, 275, 106, 272, 36, 46, 121, 35, 331, 214, 283, 14, 180, 204, 34, 150, 22, 135, 179, 211, 2, 206] + 30: [208, 189, 218, 219, 237, 259, 268, 269, 322, 334, 345, 380, 239, 40, 47, 88, 94, 101, 118, 123, 128, 134, 146, 185, 140, 305, 307, 331, 354, 363, 379, 8, 21, 22, 31, 36, 266, 374, 324, 273, 198, 56, 87, 70, 201, 227, 238, 301, 65, 336, 309, 104, 179, 349, 263, 304, 126, 300, 27, 131, 245, 74, 217, 356, 51, 297, 230, 69, 368, 229, 34, 30, 37, 362, 256, 157, 162, 375, 350, 90, 173, 80, 119, 176, 262, 271, 339, 260, 108, 280, 175, 194, 165, 178, 149, 191, 187, 28, 224, 195, 281, 310, 92, 337, 209, 16, 199, 251, 358, 234, 43, 299, 98, 114, 180, 220, 32, 190, 235, 252, 53, 188, 303, 71, 274, 38, 66, 145, 139, 117, 265, 4, 214, 361, 250, 152, 296, 167, 129, 290, 186, 327, 308, 313, 52, 11, 315, 100, 170, 84, 172, 284, 288, 370, 232, 270, 35, 206, 302, 6, 18, 261, 111, 79, 318, 130, 3, 63, 143, 2, 316, 1, 344, 353, 197, 44, 360, 82, 364, 171, 351, 295, 205, 177, 382, 278, 376, 50, 25, 5, 81, 367, 373, 168, 17, 253, 292, 91, 14, 248, 276, 85, 122, 148, 156, 257, 212, 357, 277, 96, 72, 109, 267, 161, 330, 338, 62, 46, 283, 12, 275, 200, 54, 286, 371, 57, 127, 26, 289, 24, 164, 221, 215, 365, 359, 241, 347, 326, 113, 112, 133, 225, 68, 77, 192, 174, 163, 211, 107, 264, 120, 207, 158, 378, 147, 317, 58, 23, 383, 55, 121, 348, 255, 102, 254, 59, 33, 89, 9, 132, 103, 293, 247, 203, 153, 61, 124, 78, 41, 341, 236, 325, 231, 332, 183, 279, 352, 106, 184, 196, 64, 97, 141, 298, 142, 48, 95, 83, 291, 93, 213, 135, 321, 49, 136, 369, 294, 7, 154, 125, 314, 182, 333, 242, 223, 312, 226, 285, 20, 249, 311, 155, 355, 228, 76, 75, 10, 216, 204, 160, 319, 282, 342, 287, 15, 328, 222, 366, 144, 159, 320, 67, 150, 73, 377, 99, 105, 372, 272, 243, 45, 240, 151, 60, 335, 244, 202, 381, 169, 39, 86, 13, 246, 138, 193, 343, 329, 115, 166, 323, 233, 42, 29, 137, 346, 340, 210, 258, 306, 181, 110, 116, 19, 0] + 31: [0, 246, 289, 294, 347, 172, 229, 120, 132, 207, 329, 350, 159, 49, 43, 375, 113, 313, 165, 89, 180, 37, 90, 149, 128, 81, 310, 20, 351, 99, 280, 297, 117, 203, 71, 212, 233, 189, 40, 15, 118, 12, 194, 367, 274, 18, 343, 148, 26, 34, 145, 243, 256, 195, 47, 42, 126, 167, 67, 341, 201, 206, 286, 380, 176, 97, 354, 107, 315, 52, 377, 200, 68, 186, 355, 254, 31, 374, 361, 103, 129, 127, 199, 154, 100, 317, 208, 231, 134, 383, 247, 352, 321, 7, 86, 13, 166, 339, 378, 33, 66, 41, 29, 323, 308, 64, 35, 30, 324, 267, 61, 223, 54, 287, 84, 272, 226, 327, 282, 242, 158, 270, 170, 296, 55, 202, 8, 266, 162, 234, 53, 163, 252, 95, 171, 106, 245, 316, 291, 177, 306, 300, 293, 94, 143, 187, 188, 290, 98, 190, 142, 59, 91, 340, 156, 218, 115, 215, 333, 258, 23, 77, 373, 51, 264, 160, 181, 257, 239, 147, 58, 269, 38, 318, 21, 196, 50, 298, 235, 311, 278, 25, 60, 112, 17, 222, 225, 73, 238, 85, 255, 27, 144, 250, 155, 348, 211, 65, 141, 376, 275, 262, 79, 332, 363, 69, 119, 214, 78, 122, 369, 6, 83, 337, 193, 359, 70, 335, 140, 334, 249, 46, 284, 260, 304, 157, 135, 151, 261, 326, 22, 102, 28, 88, 16, 271, 220, 9, 381, 104, 185, 133, 209, 62, 152, 277, 92, 273, 224, 210, 295, 174, 87, 364, 349, 276, 82, 182, 123, 183, 228, 178, 109, 5, 76, 116, 325, 253, 93, 175, 303, 237, 48, 309, 124, 114, 227, 292, 39, 131, 236, 205, 213, 4, 198, 139, 344, 179, 248, 121, 150, 72, 232, 366, 217, 362, 130, 345, 372, 301, 320, 379, 219, 360, 74, 368, 302, 108, 204, 24, 356, 10, 125, 279, 330, 56, 111, 322, 314, 153, 281, 101, 382, 161, 331, 36, 173, 283, 244, 105, 307, 110, 263, 259, 44, 319, 265, 3, 96, 338, 353, 137, 312, 285, 80, 240, 370, 184, 32, 365, 268, 164, 138, 169, 45, 14, 346, 342, 197, 357, 192, 241, 146, 75, 136, 57, 336, 11, 371, 168, 191, 230, 221, 19, 328, 299, 63, 2, 288, 305, 1, 251, 216, 358] + 32: [63, 177, 181, 244, 245, 251, 262, 265, 296, 312, 373, 375, 367, 316, 379, 57, 68, 147, 167, 173, 383, 14, 123, 169, 352, 382, 315, 320, 13, 118, 88, 11, 90, 145, 162, 300, 94, 283, 289, 69, 237, 276, 82, 261, 196, 93, 226, 114, 218, 184, 120, 111, 230, 4, 154, 124, 201, 170, 159, 333, 148, 174, 357, 71, 59, 80, 372, 103, 238, 277, 215, 235, 33, 202, 113, 104, 332, 115, 366, 243, 117, 224, 242, 267, 92, 219, 140, 301, 50, 127, 6, 5, 376, 23, 180, 15, 285, 274, 211, 189, 231, 256, 24, 339, 105, 364, 358, 328, 347, 257, 279, 7, 26, 293, 247, 110, 248, 1, 216, 27, 190, 12, 151, 306, 199, 32, 337, 34, 20, 278, 192, 54, 241, 2, 126, 345, 101, 142, 129, 144, 341, 31, 259, 326, 163, 121, 83, 294, 185, 253, 30, 172, 195, 0, 355, 194, 51, 360, 263, 58, 292, 182, 205, 311, 125, 305, 343, 272, 122, 66, 75, 252, 354, 86, 302, 363, 287, 365, 60, 260, 327, 280, 96, 19, 85, 314, 350, 349, 152, 197, 46, 321, 38, 8, 225, 228, 91, 213, 84, 309, 282, 44, 52, 318, 164, 200, 65, 156, 72, 28, 81, 10, 361, 119, 79, 207, 107, 370, 89, 87, 212, 250, 240, 378, 165, 304, 76, 368, 344, 21, 348, 131, 254, 48, 351, 236, 42, 270, 208, 112, 166, 130, 186, 331, 297, 178, 336, 317, 221, 330, 138, 291, 329, 271, 319, 97, 227, 269, 3, 381, 9, 374, 377, 98, 322, 310, 380, 116, 369, 45, 286, 158, 99, 264, 188, 217, 275, 229, 204, 78, 303, 25, 53, 193, 102, 134, 67, 295, 288, 179, 359, 143, 36, 175, 70, 22, 132, 73, 255, 47, 155, 323, 40, 258, 325, 64, 371, 340, 284, 153, 220, 17, 246, 106, 61, 308, 171, 157, 249, 62, 307, 356, 334, 74, 234, 187, 214, 233, 313, 150, 183, 139, 136, 35, 149, 290, 135, 209, 16, 335, 168, 281, 39, 56, 43, 128, 100, 95, 268, 362, 41, 273, 298, 324, 353, 198, 191, 222, 133, 18, 346, 146, 342, 109, 266, 239, 29, 77, 299, 223, 137, 161, 49, 108, 141, 232, 55, 206, 176, 37, 210, 160, 203, 338] + 33: [38, 32, 95, 102, 69, 142, 220, 277, 287, 293, 333, 372, 91, 311, 317, 114, 43, 135, 207, 250, 309, 264, 12, 34, 339, 151, 305, 79, 279, 15, 324, 332, 326, 185, 189, 161, 36, 335, 356, 187, 19, 122, 26, 238, 365, 228, 159, 129, 319, 37, 155, 351, 246, 105, 371, 194, 243, 275, 321, 83, 260, 314, 376, 308, 355, 373, 113, 128, 286, 306, 344, 236, 177, 73, 255, 131, 205, 156, 345, 29, 78, 341, 357, 265, 362, 342, 190, 192, 93, 256, 58, 166, 6, 346, 7, 313, 64, 82, 48, 117, 211, 375, 213, 3, 248, 188, 242, 46, 380, 183, 169, 237, 329, 262, 234, 71, 261, 119, 178, 143, 229, 39, 127, 208, 202, 162, 60, 281, 20, 74, 171, 44, 148, 107, 361, 227, 303, 196, 364, 215, 41, 225, 92, 57, 81, 297, 266, 193, 232, 123, 163, 336, 219, 27, 315, 47, 343, 96, 206, 210, 8, 276, 152, 249, 295, 184, 337, 203, 115, 379, 139, 25, 126, 347, 212, 49, 101, 312, 244, 223, 251, 325, 84, 33, 290, 216, 200, 350, 327, 291, 363, 302, 258, 231, 72, 1, 354, 383, 360, 24, 285, 62, 118, 239, 301, 121, 164, 30, 144, 254, 369, 137, 53, 310, 353, 263, 283, 323, 226, 150, 61, 80, 13, 170, 271, 120, 98, 272, 299, 158, 5, 338, 18, 45, 180, 330, 182, 111, 274, 99, 154, 42, 352, 87, 116, 198, 88, 106, 298, 181, 253, 269, 307, 31, 108, 160, 235, 124, 340, 368, 278, 245, 134, 0, 267, 153, 146, 218, 14, 22, 186, 292, 67, 132, 97, 172, 94, 377, 201, 35, 288, 4, 197, 70, 136, 85, 165, 296, 89, 191, 125, 273, 334, 149, 209, 349, 23, 300, 374, 359, 86, 280, 68, 233, 133, 173, 367, 28, 112, 257, 247, 282, 167, 331, 179, 270, 217, 174, 370, 66, 110, 224, 59, 77, 328, 259, 348, 21, 289, 230, 382, 52, 221, 168, 199, 304, 51, 63, 147, 141, 268, 90, 50, 56, 316, 130, 195, 2, 240, 320, 322, 54, 222, 138, 104, 366, 252, 381, 378, 284, 140, 145, 358, 204, 9, 16, 157, 103, 318, 76, 175, 100, 176, 75, 241, 17, 40, 109, 55, 65, 10, 11, 214, 294] + 34: [108, 76, 119, 122, 132, 160, 189, 193, 202, 229, 363, 365, 179, 377, 177, 198, 211, 213, 224, 279, 316, 341, 381, 14, 16, 10, 298, 282, 55, 171, 7, 309, 32, 66, 273, 343, 249, 87, 84, 336, 30, 322, 261, 289, 155, 112, 376, 375, 306, 92, 58, 37, 102, 301, 88, 100, 131, 351, 320, 121, 15, 2, 21, 327, 346, 263, 64, 226, 49, 355, 350, 129, 340, 97, 201, 123, 241, 50, 283, 275, 255, 104, 106, 235, 366, 79, 208, 175, 181, 5, 62, 268, 75, 9, 43, 303, 337, 295, 225, 313, 245, 35, 127, 56, 305, 297, 20, 369, 69, 284, 162, 256, 328, 367, 323, 139, 228, 278, 182, 378, 259, 220, 244, 12, 217, 48, 103, 45, 36, 370, 188, 38, 299, 347, 65, 345, 70, 270, 277, 149, 233, 173, 120, 94, 126, 178, 271, 184, 197, 247, 19, 221, 269, 250, 156, 234, 330, 362, 335, 107, 166, 204, 154, 51, 111, 260, 292, 276, 334, 240, 83, 91, 368, 236, 230, 85, 353, 176, 96, 252, 248, 293, 59, 95, 302, 33, 150, 114, 23, 314, 253, 90, 218, 243, 115, 77, 159, 67, 72, 308, 192, 307, 205, 130, 3, 321, 99, 4, 326, 195, 227, 157, 46, 374, 339, 212, 44, 185, 379, 125, 174, 251, 73, 180, 147, 161, 17, 257, 196, 25, 146, 6, 200, 280, 231, 294, 105, 13, 342, 1, 190, 223, 360, 267, 153, 133, 214, 318, 358, 262, 26, 22, 61, 145, 203, 11, 329, 60, 333, 232, 152, 331, 242, 163, 372, 304, 167, 288, 290, 187, 364, 361, 68, 34, 109, 216, 382, 354, 40, 8, 81, 183, 170, 42, 371, 310, 285, 71, 29, 158, 138, 238, 210, 24, 89, 215, 98, 93, 117, 325, 207, 239, 148, 191, 356, 199, 359, 352, 264, 265, 165, 344, 47, 82, 291, 254, 53, 136, 116, 80, 266, 86, 319, 338, 237, 311, 31, 41, 110, 143, 209, 113, 272, 274, 52, 54, 151, 135, 315, 168, 134, 172, 0, 140, 383, 286, 57, 128, 324, 74, 246, 300, 124, 317, 27, 164, 186, 101, 380, 349, 357, 312, 258, 78, 144, 137, 169, 348, 296, 118, 28, 281, 332, 18, 194, 222, 373, 206, 287, 63, 142, 219, 39, 141] + 35: [319, 90, 91, 196, 105, 207, 225, 336, 359, 228, 177, 343, 45, 163, 192, 11, 82, 344, 195, 183, 124, 146, 60, 293, 1, 46, 98, 5, 32, 369, 66, 135, 354, 193, 0, 13, 345, 364, 27, 9, 372, 172, 19, 285, 262, 10, 212, 381, 363, 15, 159, 267, 209, 111, 383, 292, 253, 310, 51, 40, 75, 355, 52, 357, 95, 350, 185, 332, 29, 107, 120, 291, 169, 4, 362, 272, 108, 96, 127, 73, 250, 238, 229, 304, 28, 138, 265, 373, 137, 57, 7, 25, 94, 101, 173, 284, 205, 47, 311, 275, 93, 302, 117, 84, 181, 148, 251, 200, 299, 257, 114, 157, 282, 256, 264, 175, 189, 288, 219, 59, 218, 141, 375, 312, 171, 12, 106, 17, 248, 77, 123, 242, 131, 55, 352, 37, 61, 298, 348, 382, 365, 316, 178, 341, 297, 286, 41, 329, 308, 215, 323, 301, 217, 23, 165, 283, 367, 314, 186, 54, 134, 300, 227, 24, 338, 249, 197, 71, 295, 318, 224, 222, 187, 243, 145, 216, 63, 259, 371, 188, 68, 67, 78, 331, 296, 313, 202, 34, 374, 306, 271, 76, 133, 368, 239, 126, 102, 122, 287, 142, 279, 49, 255, 100, 53, 56, 147, 136, 321, 152, 74, 99, 378, 26, 263, 85, 86, 211, 289, 50, 244, 42, 109, 376, 6, 115, 69, 39, 342, 254, 3, 241, 325, 334, 235, 179, 35, 232, 351, 119, 203, 379, 221, 326, 132, 268, 110, 339, 103, 112, 358, 118, 276, 303, 231, 335, 18, 294, 170, 269, 79, 377, 233, 208, 72, 58, 346, 234, 81, 328, 214, 347, 176, 149, 83, 210, 322, 130, 154, 361, 360, 337, 8, 273, 87, 161, 33, 252, 113, 237, 206, 191, 128, 30, 16, 70, 315, 89, 97, 199, 281, 144, 116, 194, 36, 104, 305, 330, 198, 245, 270, 247, 340, 62, 31, 164, 230, 190, 180, 204, 274, 278, 327, 226, 48, 240, 38, 307, 158, 213, 261, 380, 43, 121, 167, 64, 309, 182, 129, 162, 168, 201, 92, 184, 317, 22, 139, 236, 80, 246, 153, 266, 160, 125, 20, 290, 151, 156, 333, 280, 353, 44, 324, 349, 140, 143, 277, 174, 88, 155, 150, 2, 220, 260, 21, 14, 356, 320, 366, 166, 65, 370, 223, 258] + 36: [232, 64, 127, 313, 336, 37, 123, 170, 273, 302, 360, 375, 22, 171, 310, 157, 135, 209, 253, 26, 317, 145, 292, 25, 184, 138, 289, 343, 36, 149, 383, 120, 124, 306, 279, 16, 371, 233, 293, 284, 195, 71, 220, 265, 137, 227, 367, 290, 368, 5, 108, 91, 177, 200, 243, 158, 14, 128, 214, 186, 272, 133, 309, 132, 354, 211, 244, 44, 337, 303, 377, 270, 53, 152, 321, 167, 153, 323, 165, 239, 19, 126, 114, 376, 340, 73, 162, 288, 20, 117, 76, 318, 259, 110, 203, 160, 341, 352, 314, 49, 105, 228, 212, 159, 189, 226, 96, 218, 144, 369, 85, 319, 322, 216, 206, 148, 173, 35, 141, 221, 57, 326, 31, 219, 342, 100, 264, 48, 70, 61, 1, 181, 304, 320, 93, 166, 50, 224, 0, 263, 198, 60, 269, 234, 238, 242, 47, 345, 95, 116, 255, 356, 190, 254, 285, 222, 266, 296, 92, 139, 297, 102, 305, 58, 329, 8, 235, 43, 10, 196, 205, 334, 122, 364, 180, 18, 363, 87, 378, 113, 325, 98, 79, 262, 103, 237, 39, 163, 6, 215, 286, 182, 311, 258, 308, 201, 63, 346, 134, 88, 281, 249, 161, 268, 178, 210, 150, 240, 13, 300, 280, 335, 188, 77, 256, 4, 193, 230, 301, 33, 67, 155, 140, 257, 208, 204, 324, 34, 225, 260, 56, 65, 9, 183, 366, 38, 267, 248, 316, 106, 372, 299, 40, 331, 246, 15, 147, 81, 23, 338, 115, 86, 12, 101, 274, 42, 194, 164, 202, 187, 29, 24, 312, 151, 223, 328, 241, 45, 362, 72, 174, 143, 80, 236, 21, 353, 107, 2, 373, 156, 112, 276, 11, 52, 283, 247, 350, 278, 131, 59, 380, 32, 94, 245, 46, 229, 365, 295, 355, 154, 192, 62, 90, 175, 111, 339, 179, 344, 119, 315, 307, 327, 252, 359, 99, 261, 231, 298, 41, 104, 51, 83, 213, 7, 294, 185, 374, 54, 282, 121, 277, 217, 146, 109, 347, 136, 197, 348, 97, 287, 361, 3, 142, 17, 351, 251, 130, 82, 370, 250, 207, 125, 55, 172, 271, 28, 358, 199, 357, 89, 382, 84, 275, 332, 349, 69, 168, 379, 129, 381, 75, 330, 74, 176, 291, 333, 191, 68, 66, 30, 78, 169, 118, 27] + 37: [218, 120, 226, 8, 75, 112, 132, 236, 239, 288, 352, 383, 40, 377, 67, 84, 99, 209, 251, 263, 31, 189, 56, 375, 268, 287, 376, 334, 105, 256, 62, 214, 275, 232, 238, 276, 294, 322, 6, 304, 380, 317, 60, 295, 307, 367, 23, 25, 148, 328, 19, 203, 161, 244, 271, 137, 183, 369, 12, 358, 293, 129, 118, 144, 111, 83, 4, 368, 96, 70, 211, 50, 109, 171, 93, 320, 296, 356, 373, 298, 291, 130, 186, 68, 301, 140, 235, 127, 43, 92, 128, 102, 260, 63, 107, 282, 379, 86, 318, 359, 21, 103, 55, 42, 91, 142, 237, 346, 58, 223, 353, 277, 173, 264, 207, 279, 286, 297, 159, 90, 158, 292, 182, 257, 168, 141, 347, 212, 332, 310, 250, 381, 222, 179, 302, 343, 229, 338, 126, 313, 26, 378, 324, 246, 200, 272, 47, 194, 316, 61, 87, 195, 172, 175, 231, 2, 366, 48, 355, 181, 85, 82, 100, 312, 249, 169, 57, 89, 79, 190, 344, 233, 208, 230, 278, 247, 354, 10, 35, 20, 349, 372, 114, 117, 336, 365, 285, 136, 5, 13, 106, 202, 162, 341, 156, 204, 370, 217, 333, 331, 210, 299, 146, 227, 113, 362, 16, 147, 145, 51, 135, 206, 267, 110, 166, 281, 340, 289, 59, 65, 345, 258, 329, 69, 325, 17, 101, 180, 0, 363, 361, 139, 164, 104, 134, 49, 187, 124, 248, 326, 188, 46, 174, 327, 153, 149, 201, 335, 154, 138, 225, 300, 240, 199, 382, 88, 330, 64, 39, 37, 33, 53, 259, 11, 319, 205, 77, 185, 315, 228, 34, 262, 284, 350, 283, 45, 71, 178, 371, 22, 32, 1, 41, 269, 24, 152, 339, 54, 364, 52, 234, 165, 131, 74, 143, 121, 274, 215, 374, 220, 66, 306, 95, 14, 163, 323, 30, 98, 94, 170, 81, 28, 273, 125, 193, 253, 150, 76, 3, 337, 280, 266, 191, 119, 123, 151, 72, 7, 133, 27, 108, 224, 351, 254, 357, 155, 243, 314, 290, 252, 18, 196, 73, 255, 97, 216, 198, 177, 265, 213, 44, 9, 192, 176, 36, 321, 38, 303, 308, 197, 348, 311, 305, 245, 221, 270, 29, 167, 116, 160, 184, 261, 242, 309, 342, 360, 157, 122, 15, 80, 241, 78, 115, 219] + 38: [301, 52, 101, 191, 210, 46, 60, 185, 237, 251, 325, 380, 6, 121, 304, 312, 64, 98, 159, 161, 176, 339, 340, 38, 266, 125, 193, 142, 170, 250, 11, 231, 18, 232, 270, 346, 203, 169, 341, 224, 367, 84, 310, 375, 202, 281, 330, 318, 158, 373, 234, 95, 359, 260, 376, 104, 25, 163, 175, 243, 378, 32, 327, 355, 96, 228, 149, 200, 183, 24, 209, 133, 221, 220, 206, 90, 362, 368, 81, 296, 247, 364, 128, 370, 269, 309, 135, 361, 146, 349, 188, 353, 43, 295, 316, 127, 280, 194, 328, 106, 352, 136, 265, 181, 302, 102, 164, 160, 124, 248, 317, 225, 99, 308, 190, 356, 100, 311, 236, 155, 201, 274, 212, 16, 219, 156, 117, 26, 261, 204, 314, 291, 284, 115, 279, 97, 73, 255, 34, 139, 74, 61, 299, 1, 372, 226, 0, 338, 14, 307, 141, 134, 129, 80, 343, 140, 51, 322, 278, 53, 9, 150, 241, 28, 347, 215, 42, 20, 30, 285, 166, 268, 72, 36, 276, 272, 238, 13, 240, 198, 366, 354, 17, 54, 321, 211, 111, 235, 56, 252, 259, 132, 189, 263, 130, 137, 275, 165, 246, 258, 334, 35, 91, 303, 126, 86, 19, 108, 208, 45, 5, 180, 273, 44, 113, 22, 383, 8, 217, 119, 271, 58, 48, 40, 360, 55, 254, 218, 114, 292, 92, 344, 21, 262, 148, 41, 182, 351, 320, 315, 154, 332, 27, 267, 85, 323, 298, 78, 79, 205, 326, 365, 277, 123, 381, 253, 227, 57, 179, 152, 214, 186, 357, 162, 333, 94, 145, 300, 239, 144, 342, 174, 337, 290, 335, 377, 89, 118, 244, 77, 29, 87, 229, 2, 23, 71, 65, 153, 62, 66, 49, 319, 69, 88, 7, 15, 313, 283, 12, 83, 282, 245, 293, 286, 222, 264, 151, 242, 122, 168, 70, 110, 10, 345, 63, 288, 147, 184, 143, 103, 195, 374, 192, 196, 4, 348, 171, 233, 107, 256, 109, 305, 172, 294, 363, 249, 187, 178, 230, 329, 75, 216, 31, 76, 120, 82, 350, 207, 223, 369, 39, 331, 67, 105, 289, 33, 173, 138, 112, 116, 93, 50, 379, 199, 324, 68, 37, 131, 371, 297, 3, 336, 213, 287, 177, 257, 59, 358, 306, 197, 382, 47, 157, 167] + 39: [282, 39, 262, 261, 321, 304, 143, 149, 257, 22, 44, 209, 310, 302, 230, 120, 107, 339, 57, 201, 52, 9, 198, 361, 247, 138, 317, 106, 306, 245, 64, 342, 269, 380, 165, 89, 66, 248, 378, 281, 320, 4, 288, 175, 147, 356, 29, 334, 104, 81, 340, 358, 33, 275, 70, 20, 370, 308, 90, 21, 372, 208, 127, 101, 366, 45, 6, 335, 211, 351, 154, 11, 236, 163, 352, 190, 371, 95, 16, 287, 83, 67, 73, 115, 56, 374, 87, 54, 144, 167, 114, 216, 197, 219, 332, 148, 315, 333, 23, 48, 353, 134, 128, 60, 117, 359, 184, 186, 258, 174, 121, 146, 47, 177, 285, 183, 283, 10, 238, 344, 286, 119, 26, 7, 232, 126, 14, 357, 265, 159, 229, 316, 65, 267, 71, 51, 140, 367, 345, 354, 207, 291, 92, 284, 312, 242, 324, 365, 191, 152, 301, 97, 30, 346, 206, 125, 38, 279, 277, 369, 274, 69, 135, 88, 164, 19, 157, 237, 93, 325, 204, 76, 170, 74, 188, 185, 221, 41, 108, 124, 132, 360, 296, 178, 259, 34, 276, 193, 254, 111, 337, 318, 36, 13, 362, 49, 383, 137, 199, 133, 79, 103, 266, 12, 314, 218, 270, 3, 42, 239, 322, 289, 195, 303, 305, 294, 313, 139, 297, 77, 8, 225, 63, 61, 309, 168, 55, 105, 379, 215, 244, 40, 375, 31, 75, 373, 327, 343, 253, 233, 85, 113, 86, 141, 235, 278, 179, 182, 130, 196, 246, 256, 43, 151, 80, 172, 298, 336, 5, 293, 169, 91, 252, 136, 100, 156, 292, 250, 290, 260, 62, 203, 220, 192, 160, 158, 176, 153, 382, 330, 227, 323, 162, 72, 363, 27, 347, 205, 99, 25, 272, 319, 118, 122, 213, 212, 68, 58, 142, 349, 96, 1, 155, 24, 53, 299, 249, 102, 194, 171, 0, 210, 280, 368, 271, 341, 84, 112, 364, 311, 181, 166, 123, 326, 35, 348, 328, 300, 273, 28, 161, 228, 240, 109, 18, 129, 180, 223, 187, 15, 202, 251, 226, 355, 214, 376, 37, 255, 268, 82, 2, 98, 200, 331, 150, 241, 338, 189, 78, 224, 264, 234, 329, 17, 59, 131, 50, 173, 243, 222, 46, 32, 110, 116, 145, 295, 381, 263, 307, 94, 377, 217, 231, 350] + 40: [31, 249, 343, 376, 92, 105, 123, 149, 194, 214, 248, 300, 319, 308, 365, 29, 350, 0, 40, 252, 72, 312, 69, 170, 15, 334, 53, 155, 19, 116, 140, 325, 272, 240, 144, 353, 68, 296, 193, 187, 2, 171, 6, 186, 173, 154, 191, 331, 175, 258, 10, 382, 43, 329, 139, 23, 113, 255, 225, 344, 378, 198, 136, 38, 41, 120, 131, 288, 134, 39, 320, 5, 317, 208, 89, 145, 309, 165, 304, 118, 361, 34, 234, 220, 207, 251, 143, 30, 79, 7, 195, 147, 133, 104, 315, 20, 183, 81, 363, 211, 44, 306, 93, 238, 141, 285, 380, 127, 250, 64, 1, 253, 205, 290, 327, 36, 78, 111, 267, 83, 349, 164, 275, 289, 236, 150, 243, 301, 32, 374, 172, 335, 360, 232, 311, 162, 196, 110, 264, 261, 135, 66, 157, 126, 142, 26, 3, 271, 269, 65, 305, 314, 49, 16, 298, 146, 328, 160, 224, 168, 216, 263, 338, 293, 125, 203, 223, 197, 71, 297, 58, 129, 278, 316, 294, 273, 270, 50, 180, 24, 358, 247, 235, 336, 200, 99, 192, 37, 122, 368, 114, 42, 190, 52, 74, 284, 54, 56, 355, 48, 51, 268, 230, 245, 21, 337, 372, 362, 166, 307, 94, 318, 291, 115, 231, 313, 108, 265, 332, 9, 11, 137, 103, 167, 201, 84, 295, 45, 179, 283, 369, 303, 27, 259, 151, 371, 76, 212, 22, 352, 33, 62, 254, 159, 322, 370, 57, 323, 222, 184, 163, 381, 346, 101, 158, 210, 341, 302, 130, 287, 241, 14, 217, 87, 226, 98, 88, 28, 277, 221, 189, 333, 233, 266, 77, 109, 132, 75, 153, 347, 61, 199, 244, 356, 345, 102, 117, 86, 379, 367, 8, 91, 185, 12, 364, 237, 18, 310, 257, 176, 357, 373, 156, 59, 219, 46, 178, 281, 366, 124, 174, 112, 340, 67, 215, 17, 63, 148, 95, 330, 73, 276, 35, 96, 280, 55, 279, 228, 321, 229, 299, 206, 47, 383, 218, 152, 260, 204, 161, 97, 274, 262, 82, 282, 85, 348, 242, 342, 286, 359, 107, 90, 326, 239, 138, 177, 60, 13, 182, 292, 169, 213, 202, 100, 80, 188, 25, 339, 375, 121, 119, 354, 181, 70, 128, 351, 209, 227, 377, 324, 246, 256, 106, 4] + 41: [49, 64, 325, 181, 221, 148, 373, 341, 227, 216, 309, 357, 378, 222, 276, 339, 10, 214, 98, 137, 193, 135, 22, 278, 199, 326, 134, 271, 299, 208, 352, 141, 17, 343, 152, 85, 170, 280, 324, 254, 242, 315, 303, 147, 28, 16, 164, 178, 60, 360, 53, 116, 289, 338, 342, 13, 20, 2, 247, 132, 350, 161, 351, 345, 167, 186, 99, 105, 204, 58, 230, 296, 374, 314, 4, 301, 115, 290, 188, 72, 209, 304, 333, 31, 321, 358, 318, 26, 229, 63, 81, 1, 287, 110, 372, 263, 269, 34, 253, 264, 258, 144, 371, 123, 248, 265, 382, 228, 298, 187, 104, 163, 56, 66, 153, 75, 207, 375, 169, 118, 283, 7, 383, 353, 282, 380, 43, 15, 68, 47, 157, 120, 114, 365, 95, 202, 294, 250, 173, 37, 48, 14, 109, 52, 142, 200, 279, 35, 102, 117, 295, 272, 259, 29, 346, 233, 355, 168, 24, 3, 32, 340, 107, 128, 191, 175, 323, 40, 184, 84, 69, 127, 349, 322, 101, 18, 154, 234, 67, 361, 190, 273, 21, 45, 174, 235, 176, 19, 292, 297, 238, 362, 30, 86, 364, 108, 126, 54, 151, 284, 300, 38, 195, 291, 369, 57, 111, 44, 219, 55, 112, 244, 41, 149, 171, 145, 197, 150, 172, 9, 140, 256, 224, 61, 89, 310, 231, 240, 366, 255, 79, 80, 36, 46, 268, 277, 179, 320, 245, 319, 129, 249, 97, 334, 218, 125, 51, 246, 39, 381, 121, 356, 275, 90, 76, 302, 308, 379, 8, 130, 78, 0, 62, 88, 159, 212, 92, 347, 217, 225, 330, 213, 59, 158, 274, 177, 119, 307, 96, 206, 252, 313, 131, 6, 33, 65, 139, 189, 215, 27, 368, 306, 286, 156, 138, 74, 146, 73, 155, 203, 205, 376, 223, 82, 42, 237, 344, 220, 354, 113, 182, 281, 305, 251, 293, 262, 336, 70, 363, 91, 143, 77, 183, 211, 122, 11, 285, 311, 194, 93, 133, 160, 100, 165, 201, 270, 185, 71, 136, 198, 243, 241, 210, 261, 236, 180, 267, 162, 87, 166, 50, 348, 25, 83, 266, 94, 232, 257, 329, 312, 367, 103, 327, 337, 239, 260, 288, 5, 192, 316, 370, 377, 335, 196, 124, 12, 23, 226, 328, 317, 359, 332, 331, 106] + 42: [123, 193, 242, 294, 369, 376, 25, 71, 95, 254, 324, 339, 152, 313, 213, 285, 115, 13, 365, 24, 252, 289, 46, 27, 117, 232, 233, 175, 134, 331, 86, 361, 273, 352, 126, 121, 31, 259, 356, 160, 180, 7, 343, 292, 353, 15, 163, 40, 359, 209, 4, 103, 166, 371, 58, 346, 82, 136, 159, 66, 217, 255, 320, 161, 257, 311, 298, 364, 90, 304, 182, 74, 94, 208, 277, 287, 314, 192, 97, 378, 360, 186, 14, 91, 266, 263, 358, 230, 322, 137, 64, 53, 98, 330, 78, 240, 268, 41, 189, 178, 147, 362, 372, 205, 26, 229, 22, 190, 345, 179, 281, 341, 338, 227, 231, 310, 120, 261, 96, 276, 176, 150, 327, 221, 329, 39, 237, 258, 112, 199, 383, 44, 172, 264, 342, 29, 326, 76, 307, 34, 293, 267, 302, 12, 155, 282, 203, 30, 216, 149, 62, 299, 194, 272, 133, 35, 218, 72, 59, 354, 167, 38, 297, 226, 156, 197, 144, 54, 93, 63, 48, 248, 236, 316, 132, 246, 270, 321, 174, 47, 325, 215, 55, 379, 201, 335, 340, 139, 336, 104, 124, 382, 111, 151, 220, 235, 173, 36, 241, 283, 68, 102, 207, 249, 256, 279, 374, 265, 344, 234, 18, 260, 16, 363, 3, 177, 158, 350, 168, 21, 87, 92, 60, 315, 125, 290, 153, 11, 333, 116, 243, 373, 286, 223, 381, 183, 171, 56, 69, 70, 138, 212, 162, 122, 239, 366, 146, 323, 170, 284, 85, 57, 37, 247, 222, 127, 67, 280, 301, 9, 204, 73, 296, 0, 2, 211, 337, 319, 165, 181, 51, 89, 61, 185, 309, 100, 188, 305, 200, 348, 105, 308, 8, 219, 269, 318, 88, 110, 210, 184, 349, 191, 28, 143, 380, 157, 251, 145, 198, 271, 154, 303, 262, 275, 17, 77, 32, 244, 291, 250, 288, 43, 375, 106, 108, 131, 135, 33, 80, 49, 332, 368, 334, 142, 20, 357, 306, 370, 169, 45, 101, 238, 196, 312, 367, 79, 1, 119, 377, 274, 140, 118, 113, 130, 355, 10, 23, 317, 50, 228, 6, 141, 224, 52, 245, 109, 5, 128, 75, 81, 42, 114, 129, 328, 214, 164, 65, 99, 278, 206, 347, 83, 295, 148, 195, 351, 187, 202, 19, 225, 300, 84, 253, 107] + 43: [168, 374, 234, 293, 174, 116, 211, 317, 302, 195, 354, 3, 225, 264, 88, 306, 229, 325, 215, 258, 69, 104, 214, 245, 375, 322, 255, 200, 165, 226, 175, 260, 248, 176, 44, 348, 66, 186, 280, 327, 130, 107, 253, 163, 151, 7, 212, 95, 98, 377, 136, 32, 34, 345, 120, 65, 109, 41, 25, 182, 315, 363, 224, 73, 261, 273, 266, 193, 158, 106, 244, 331, 221, 97, 127, 238, 26, 242, 128, 294, 265, 124, 289, 373, 105, 254, 342, 368, 86, 24, 101, 207, 110, 285, 54, 169, 171, 20, 301, 250, 219, 185, 146, 329, 187, 311, 189, 192, 156, 13, 138, 85, 83, 241, 45, 292, 240, 162, 361, 206, 347, 349, 111, 379, 339, 196, 217, 170, 357, 276, 319, 382, 92, 39, 223, 321, 286, 28, 180, 288, 11, 290, 114, 139, 372, 216, 29, 89, 204, 67, 210, 18, 72, 300, 312, 376, 335, 37, 55, 121, 48, 157, 183, 370, 64, 119, 172, 350, 17, 364, 153, 78, 275, 160, 49, 16, 117, 118, 328, 91, 359, 59, 81, 147, 205, 115, 113, 256, 295, 179, 51, 310, 324, 112, 93, 198, 378, 123, 50, 80, 159, 316, 201, 33, 14, 35, 337, 178, 380, 277, 47, 343, 31, 209, 0, 333, 283, 197, 291, 108, 222, 140, 38, 309, 122, 320, 62, 58, 190, 129, 235, 307, 272, 36, 243, 284, 353, 134, 137, 133, 75, 371, 152, 305, 150, 61, 10, 46, 8, 173, 249, 15, 30, 142, 57, 360, 282, 257, 155, 56, 76, 287, 21, 383, 297, 218, 131, 227, 154, 84, 194, 167, 71, 233, 141, 42, 298, 68, 103, 1, 2, 149, 145, 213, 279, 330, 228, 208, 43, 278, 94, 338, 4, 365, 334, 74, 356, 355, 341, 166, 351, 308, 366, 269, 77, 79, 237, 70, 87, 381, 236, 270, 23, 332, 314, 318, 52, 367, 148, 125, 259, 188, 199, 184, 164, 177, 239, 323, 346, 296, 12, 143, 126, 247, 281, 340, 369, 161, 252, 202, 232, 135, 60, 9, 304, 336, 53, 96, 27, 6, 100, 274, 191, 5, 63, 251, 132, 19, 230, 231, 344, 181, 22, 203, 246, 271, 99, 263, 144, 268, 303, 352, 262, 82, 326, 313, 90, 299, 102, 220, 267, 358, 362, 40] + 44: [213, 259, 133, 209, 129, 332, 303, 12, 106, 116, 322, 374, 77, 121, 100, 87, 370, 78, 234, 27, 246, 31, 281, 206, 301, 358, 138, 315, 44, 282, 218, 62, 373, 127, 75, 339, 191, 306, 265, 287, 221, 185, 113, 321, 115, 73, 310, 13, 199, 63, 207, 364, 177, 334, 179, 256, 195, 361, 181, 189, 10, 264, 276, 192, 269, 107, 309, 357, 245, 176, 52, 308, 317, 187, 144, 349, 130, 196, 105, 18, 273, 260, 111, 161, 226, 98, 228, 173, 164, 15, 149, 345, 312, 268, 316, 295, 348, 313, 304, 219, 366, 126, 243, 53, 124, 247, 353, 112, 362, 146, 217, 254, 160, 162, 336, 314, 35, 194, 58, 371, 274, 93, 125, 296, 380, 46, 19, 263, 153, 283, 76, 16, 172, 34, 48, 266, 71, 134, 40, 169, 232, 50, 293, 197, 171, 288, 33, 244, 70, 381, 159, 211, 7, 251, 37, 208, 222, 29, 320, 275, 186, 338, 344, 59, 241, 375, 330, 88, 379, 82, 143, 102, 184, 352, 92, 205, 367, 255, 210, 148, 101, 376, 166, 365, 6, 285, 36, 74, 20, 137, 119, 297, 223, 257, 30, 198, 262, 39, 261, 90, 305, 183, 151, 356, 284, 84, 236, 329, 91, 56, 369, 165, 3, 49, 5, 152, 21, 267, 11, 142, 117, 201, 120, 83, 104, 51, 22, 335, 145, 215, 325, 38, 340, 150, 96, 41, 231, 57, 80, 302, 170, 188, 229, 249, 95, 290, 248, 354, 347, 72, 200, 307, 346, 227, 311, 118, 360, 163, 131, 108, 214, 4, 65, 99, 168, 180, 331, 182, 89, 85, 54, 327, 81, 203, 216, 47, 238, 190, 135, 383, 355, 24, 286, 26, 270, 204, 114, 240, 97, 60, 279, 212, 343, 55, 66, 136, 318, 230, 1, 298, 64, 271, 252, 294, 291, 122, 132, 278, 8, 250, 326, 363, 167, 155, 43, 289, 94, 350, 174, 128, 156, 193, 253, 123, 0, 351, 368, 86, 337, 154, 79, 147, 323, 319, 45, 372, 17, 110, 277, 333, 324, 61, 378, 220, 377, 9, 2, 28, 258, 382, 328, 242, 225, 299, 67, 157, 341, 109, 42, 69, 25, 300, 140, 280, 14, 224, 359, 292, 23, 237, 239, 139, 272, 178, 68, 141, 158, 175, 202, 342, 32, 235, 233, 103] + 45: [92, 189, 195, 117, 192, 64, 266, 300, 100, 174, 221, 347, 140, 16, 357, 365, 34, 273, 220, 337, 295, 240, 336, 138, 178, 380, 51, 41, 123, 87, 263, 268, 125, 13, 181, 83, 10, 272, 158, 80, 2, 116, 333, 15, 321, 305, 69, 355, 227, 124, 310, 112, 175, 344, 294, 381, 243, 56, 66, 185, 154, 212, 307, 313, 279, 330, 9, 151, 238, 62, 130, 72, 23, 165, 168, 328, 106, 35, 264, 374, 362, 270, 259, 299, 121, 298, 118, 250, 260, 43, 358, 79, 371, 193, 366, 361, 161, 160, 183, 155, 213, 4, 205, 255, 312, 132, 53, 244, 293, 369, 107, 229, 360, 302, 46, 231, 309, 102, 148, 137, 42, 375, 52, 36, 345, 1, 230, 241, 203, 90, 101, 17, 334, 247, 209, 115, 135, 96, 170, 379, 48, 323, 352, 14, 201, 339, 224, 326, 267, 24, 126, 211, 289, 332, 265, 40, 8, 356, 128, 45, 210, 236, 207, 322, 119, 111, 194, 235, 27, 141, 349, 122, 7, 208, 133, 282, 341, 214, 200, 49, 33, 176, 297, 196, 251, 67, 76, 37, 342, 377, 186, 75, 350, 171, 114, 162, 306, 109, 20, 5, 129, 71, 31, 184, 367, 120, 348, 55, 327, 315, 19, 283, 177, 3, 82, 99, 373, 274, 163, 223, 68, 26, 65, 98, 57, 136, 153, 149, 276, 197, 61, 239, 84, 32, 58, 296, 142, 78, 97, 311, 329, 182, 21, 187, 54, 292, 340, 29, 169, 225, 280, 60, 164, 346, 269, 198, 308, 94, 50, 150, 127, 93, 382, 325, 95, 199, 131, 359, 12, 179, 218, 320, 271, 202, 152, 172, 81, 139, 180, 254, 70, 217, 370, 38, 288, 113, 143, 6, 383, 25, 47, 228, 372, 44, 303, 291, 166, 147, 167, 77, 324, 284, 103, 331, 245, 156, 261, 59, 285, 73, 18, 237, 191, 317, 354, 275, 74, 108, 376, 319, 246, 368, 226, 287, 173, 110, 286, 351, 277, 242, 222, 145, 204, 190, 232, 335, 301, 316, 85, 39, 104, 86, 22, 290, 363, 91, 89, 11, 318, 144, 378, 258, 88, 281, 253, 256, 105, 364, 353, 28, 219, 134, 262, 234, 30, 249, 257, 252, 314, 215, 159, 216, 233, 188, 146, 248, 338, 157, 63, 304, 0, 343, 206, 278] + 46: [142, 95, 321, 290, 159, 15, 358, 14, 270, 252, 88, 356, 183, 155, 50, 292, 318, 7, 337, 330, 28, 323, 325, 302, 352, 115, 156, 268, 92, 364, 305, 87, 134, 61, 122, 175, 311, 55, 211, 353, 294, 32, 255, 265, 26, 366, 167, 365, 133, 254, 85, 232, 304, 336, 78, 241, 93, 196, 178, 82, 10, 60, 39, 326, 109, 57, 206, 291, 339, 288, 174, 213, 247, 124, 242, 83, 4, 377, 263, 264, 375, 229, 179, 27, 34, 209, 219, 143, 164, 205, 223, 277, 347, 11, 248, 36, 64, 6, 303, 8, 42, 59, 144, 333, 309, 269, 225, 147, 139, 324, 18, 62, 118, 272, 161, 243, 52, 21, 185, 379, 256, 332, 345, 108, 197, 273, 173, 56, 135, 94, 276, 5, 58, 221, 113, 131, 315, 172, 313, 322, 150, 75, 47, 220, 286, 274, 338, 280, 168, 128, 307, 382, 331, 381, 54, 231, 380, 246, 114, 228, 121, 53, 81, 346, 283, 98, 281, 123, 253, 239, 165, 49, 132, 162, 193, 289, 359, 361, 1, 182, 355, 191, 378, 351, 245, 23, 158, 328, 91, 285, 107, 349, 343, 238, 204, 30, 218, 41, 45, 35, 261, 171, 341, 0, 110, 202, 38, 282, 334, 362, 111, 373, 190, 234, 31, 181, 207, 51, 68, 215, 99, 298, 103, 299, 20, 86, 73, 251, 149, 96, 40, 3, 130, 63, 198, 89, 317, 138, 77, 84, 230, 16, 187, 102, 195, 176, 237, 297, 69, 67, 127, 100, 367, 25, 136, 350, 33, 129, 216, 119, 293, 208, 258, 151, 354, 170, 271, 266, 22, 316, 340, 169, 74, 257, 72, 306, 106, 249, 194, 374, 260, 192, 383, 141, 90, 184, 105, 212, 319, 46, 104, 320, 80, 259, 17, 24, 348, 65, 43, 70, 66, 240, 344, 76, 186, 295, 101, 308, 177, 262, 363, 279, 140, 327, 12, 2, 372, 369, 125, 97, 145, 370, 48, 148, 335, 224, 154, 44, 287, 180, 157, 360, 342, 244, 126, 296, 152, 368, 284, 79, 117, 13, 250, 189, 71, 112, 275, 200, 188, 314, 312, 199, 227, 137, 226, 329, 217, 278, 236, 29, 210, 19, 301, 203, 376, 37, 235, 160, 9, 163, 310, 357, 201, 233, 214, 146, 120, 166, 267, 222, 153, 371, 300, 116] + 47: [195, 146, 191, 373, 209, 369, 246, 293, 360, 56, 288, 326, 3, 252, 303, 32, 275, 247, 165, 261, 113, 7, 236, 62, 343, 50, 112, 335, 157, 282, 244, 232, 8, 128, 156, 239, 54, 255, 354, 211, 200, 190, 374, 361, 206, 272, 381, 214, 44, 98, 140, 25, 183, 37, 248, 380, 372, 217, 66, 169, 352, 176, 298, 148, 139, 185, 186, 216, 264, 6, 241, 311, 24, 114, 42, 73, 96, 60, 297, 92, 38, 256, 254, 318, 312, 159, 68, 197, 315, 292, 9, 266, 268, 167, 13, 83, 87, 238, 122, 65, 237, 378, 111, 97, 302, 99, 283, 182, 181, 129, 235, 135, 370, 45, 234, 276, 102, 304, 85, 125, 154, 175, 363, 219, 324, 101, 336, 337, 160, 325, 78, 161, 162, 127, 240, 265, 319, 205, 80, 147, 365, 86, 100, 295, 22, 286, 188, 351, 33, 17, 91, 213, 120, 339, 178, 221, 158, 314, 168, 251, 67, 364, 106, 126, 329, 10, 177, 299, 29, 309, 267, 63, 377, 199, 233, 23, 225, 19, 108, 220, 382, 345, 362, 342, 104, 166, 340, 366, 259, 48, 379, 142, 307, 227, 79, 349, 301, 41, 203, 121, 258, 130, 308, 116, 359, 333, 196, 131, 250, 231, 138, 201, 273, 152, 383, 11, 300, 193, 274, 218, 262, 103, 204, 71, 94, 253, 245, 88, 134, 280, 344, 367, 93, 89, 52, 164, 31, 132, 358, 270, 137, 61, 198, 153, 69, 224, 173, 210, 49, 20, 74, 202, 75, 34, 36, 229, 26, 151, 180, 27, 278, 334, 207, 47, 271, 1, 368, 110, 226, 172, 355, 15, 289, 189, 290, 76, 347, 109, 294, 287, 269, 228, 322, 321, 28, 149, 58, 105, 331, 170, 215, 118, 313, 323, 338, 356, 107, 260, 350, 145, 187, 279, 257, 376, 330, 284, 77, 348, 115, 310, 5, 222, 179, 119, 353, 40, 212, 184, 39, 53, 332, 81, 55, 82, 136, 375, 316, 21, 230, 263, 296, 243, 281, 150, 133, 317, 192, 35, 320, 327, 155, 285, 51, 18, 306, 144, 57, 64, 143, 371, 305, 346, 208, 223, 12, 70, 163, 242, 16, 30, 171, 43, 14, 123, 95, 249, 0, 141, 328, 46, 174, 277, 357, 291, 90, 59, 84, 124, 4, 117, 341, 2, 72, 194] + 48: [17, 41, 257, 24, 351, 16, 47, 278, 153, 258, 209, 250, 271, 354, 291, 111, 14, 40, 173, 335, 260, 336, 304, 308, 140, 139, 210, 329, 275, 367, 195, 29, 238, 333, 10, 196, 328, 5, 378, 321, 91, 211, 259, 373, 309, 355, 200, 227, 132, 6, 134, 192, 341, 180, 96, 178, 67, 225, 306, 114, 267, 287, 181, 179, 285, 70, 146, 216, 69, 148, 126, 129, 283, 203, 185, 371, 162, 218, 269, 169, 87, 149, 383, 334, 76, 74, 208, 175, 112, 202, 302, 39, 194, 60, 365, 314, 266, 121, 327, 293, 123, 231, 84, 292, 277, 154, 176, 245, 128, 343, 344, 319, 118, 79, 43, 206, 105, 244, 131, 1, 102, 236, 223, 155, 255, 360, 374, 346, 213, 350, 199, 30, 330, 44, 177, 232, 370, 168, 138, 151, 135, 110, 296, 136, 273, 234, 189, 305, 152, 357, 281, 127, 32, 37, 171, 220, 239, 19, 147, 380, 299, 221, 307, 253, 35, 143, 8, 23, 276, 99, 375, 252, 50, 212, 339, 63, 55, 294, 356, 348, 52, 226, 369, 184, 229, 89, 347, 12, 59, 163, 376, 167, 311, 116, 166, 115, 290, 31, 13, 81, 98, 61, 66, 103, 284, 215, 207, 316, 320, 170, 57, 36, 382, 3, 201, 337, 353, 358, 104, 186, 222, 340, 315, 18, 108, 93, 298, 54, 101, 109, 157, 325, 205, 160, 230, 352, 243, 326, 338, 68, 141, 150, 28, 34, 156, 73, 318, 349, 94, 165, 362, 122, 381, 301, 83, 240, 246, 120, 100, 49, 187, 310, 282, 183, 85, 279, 21, 235, 25, 233, 241, 204, 261, 313, 228, 265, 78, 145, 268, 359, 7, 247, 113, 72, 272, 92, 86, 248, 62, 117, 33, 48, 322, 174, 80, 119, 71, 217, 106, 46, 15, 65, 295, 363, 300, 0, 4, 53, 82, 214, 331, 251, 263, 77, 58, 264, 130, 324, 172, 280, 107, 27, 188, 364, 237, 270, 90, 342, 368, 242, 317, 97, 274, 224, 361, 197, 159, 26, 2, 372, 193, 190, 64, 158, 256, 366, 125, 124, 297, 249, 133, 38, 161, 75, 22, 289, 286, 95, 345, 332, 288, 20, 182, 254, 219, 88, 379, 11, 144, 323, 377, 262, 9, 51, 164, 142, 312, 45, 42, 198, 137, 56, 191, 303] + 49: [302, 135, 300, 322, 375, 38, 317, 337, 356, 64, 123, 191, 321, 227, 130, 22, 7, 266, 3, 347, 68, 188, 349, 92, 297, 344, 144, 362, 163, 295, 69, 2, 185, 236, 32, 370, 253, 60, 30, 336, 318, 139, 103, 106, 125, 363, 258, 373, 126, 325, 8, 254, 198, 55, 359, 46, 364, 372, 252, 24, 374, 262, 186, 250, 240, 20, 338, 61, 52, 167, 334, 26, 79, 44, 381, 366, 138, 376, 309, 0, 108, 87, 18, 129, 360, 39, 194, 328, 16, 224, 43, 339, 228, 343, 99, 288, 272, 275, 74, 195, 382, 296, 330, 14, 101, 104, 379, 369, 6, 201, 107, 5, 310, 41, 371, 71, 204, 324, 171, 247, 268, 270, 306, 131, 368, 323, 36, 244, 154, 51, 367, 50, 75, 212, 289, 94, 245, 34, 82, 358, 49, 182, 215, 263, 155, 105, 243, 333, 23, 128, 98, 345, 256, 147, 237, 219, 305, 78, 292, 100, 137, 4, 10, 280, 190, 168, 350, 203, 19, 66, 218, 264, 174, 170, 303, 119, 200, 232, 152, 291, 157, 353, 380, 175, 1, 365, 216, 142, 31, 326, 281, 383, 313, 116, 177, 45, 169, 277, 225, 351, 12, 354, 165, 11, 202, 213, 199, 255, 73, 196, 301, 156, 355, 145, 341, 230, 271, 293, 259, 331, 97, 140, 273, 307, 118, 299, 229, 217, 9, 115, 148, 329, 121, 28, 112, 173, 117, 151, 166, 231, 242, 181, 276, 269, 314, 86, 214, 134, 239, 113, 93, 159, 193, 158, 335, 286, 205, 279, 127, 179, 378, 206, 88, 265, 251, 150, 180, 91, 235, 80, 89, 327, 226, 124, 348, 249, 210, 136, 153, 260, 48, 340, 77, 238, 176, 283, 27, 287, 233, 234, 110, 132, 183, 311, 81, 308, 53, 192, 57, 162, 184, 59, 261, 220, 332, 70, 146, 178, 114, 361, 13, 42, 197, 65, 278, 120, 298, 56, 189, 33, 47, 285, 377, 102, 149, 209, 241, 312, 109, 221, 290, 62, 172, 143, 315, 96, 37, 257, 25, 187, 164, 54, 29, 67, 72, 282, 274, 141, 83, 304, 40, 246, 95, 223, 160, 267, 320, 284, 357, 207, 294, 15, 208, 133, 58, 63, 222, 352, 85, 84, 111, 21, 319, 346, 90, 76, 122, 17, 248, 35, 211, 342, 316, 161] + 50: [86, 116, 151, 28, 153, 330, 30, 95, 55, 230, 345, 365, 268, 154, 319, 294, 39, 123, 147, 83, 66, 222, 50, 93, 29, 0, 121, 244, 208, 378, 113, 34, 98, 173, 131, 241, 100, 312, 79, 195, 379, 357, 236, 212, 258, 31, 279, 374, 316, 146, 314, 375, 252, 105, 5, 308, 285, 207, 307, 257, 109, 373, 22, 117, 102, 255, 281, 344, 324, 176, 336, 264, 49, 125, 119, 6, 183, 85, 172, 288, 318, 104, 54, 111, 181, 110, 114, 363, 348, 382, 211, 289, 383, 60, 158, 136, 166, 273, 88, 133, 194, 371, 340, 4, 68, 89, 261, 364, 376, 35, 256, 132, 24, 250, 73, 185, 198, 323, 8, 120, 32, 218, 193, 298, 164, 94, 234, 178, 320, 112, 201, 44, 315, 262, 341, 353, 41, 269, 163, 243, 372, 179, 246, 84, 210, 361, 45, 16, 122, 310, 303, 223, 349, 286, 47, 144, 188, 370, 334, 271, 228, 189, 78, 127, 170, 346, 21, 161, 215, 233, 296, 187, 301, 267, 74, 356, 237, 150, 266, 182, 69, 352, 62, 155, 165, 359, 126, 191, 259, 40, 209, 283, 232, 284, 280, 184, 58, 219, 2, 347, 12, 293, 216, 309, 325, 367, 304, 343, 75, 168, 20, 202, 177, 77, 381, 332, 321, 38, 282, 253, 19, 134, 160, 292, 42, 291, 103, 167, 214, 14, 175, 51, 186, 43, 91, 369, 156, 36, 360, 23, 56, 333, 366, 331, 263, 206, 242, 337, 124, 82, 76, 221, 27, 97, 287, 295, 203, 290, 322, 204, 339, 11, 10, 225, 350, 229, 9, 224, 306, 169, 277, 17, 7, 171, 70, 227, 26, 67, 377, 130, 313, 139, 326, 251, 63, 138, 190, 260, 18, 140, 338, 272, 149, 53, 145, 362, 200, 162, 328, 302, 1, 137, 329, 270, 239, 247, 205, 297, 106, 87, 13, 317, 3, 311, 192, 213, 71, 342, 152, 37, 46, 354, 245, 52, 129, 299, 327, 249, 96, 197, 64, 300, 238, 90, 72, 115, 231, 226, 199, 335, 380, 174, 107, 143, 248, 148, 265, 118, 128, 235, 217, 159, 275, 180, 81, 278, 305, 65, 157, 254, 57, 25, 220, 15, 48, 240, 99, 135, 141, 355, 142, 368, 33, 80, 358, 274, 196, 276, 92, 108, 351, 101, 61, 59] + 51: [24, 219, 141, 157, 194, 336, 257, 266, 173, 202, 249, 261, 186, 81, 321, 100, 297, 15, 57, 150, 76, 121, 291, 326, 372, 314, 245, 209, 362, 258, 298, 68, 28, 22, 377, 151, 102, 95, 335, 271, 351, 361, 115, 319, 116, 132, 38, 21, 137, 60, 130, 183, 18, 119, 324, 30, 159, 154, 293, 348, 373, 104, 103, 285, 279, 79, 364, 80, 197, 174, 71, 290, 375, 270, 269, 292, 42, 59, 265, 288, 58, 201, 169, 156, 221, 152, 243, 85, 198, 248, 196, 315, 112, 272, 352, 226, 241, 67, 295, 299, 56, 34, 69, 54, 232, 14, 175, 2, 37, 64, 330, 259, 172, 10, 224, 307, 61, 277, 179, 138, 210, 176, 325, 8, 97, 33, 262, 223, 238, 88, 187, 139, 110, 122, 250, 247, 303, 192, 164, 27, 167, 7, 63, 155, 200, 356, 188, 5, 308, 310, 239, 338, 316, 158, 185, 327, 323, 127, 301, 92, 382, 280, 48, 365, 273, 1, 367, 255, 208, 180, 235, 26, 344, 190, 41, 334, 93, 148, 376, 267, 12, 228, 124, 263, 275, 181, 236, 349, 333, 214, 217, 231, 287, 182, 260, 134, 161, 222, 3, 312, 363, 66, 128, 36, 17, 52, 40, 11, 47, 366, 195, 332, 278, 43, 39, 318, 342, 284, 203, 49, 212, 329, 75, 94, 78, 378, 165, 302, 337, 193, 32, 346, 331, 360, 370, 289, 328, 53, 215, 29, 144, 113, 87, 253, 281, 142, 368, 311, 114, 339, 305, 274, 82, 123, 135, 246, 23, 254, 44, 234, 145, 340, 0, 70, 126, 357, 140, 168, 20, 359, 91, 171, 170, 240, 296, 117, 229, 99, 343, 282, 143, 237, 160, 317, 350, 25, 96, 131, 163, 16, 65, 153, 216, 107, 204, 84, 379, 149, 230, 256, 286, 111, 300, 304, 294, 162, 207, 313, 89, 125, 4, 225, 322, 98, 72, 146, 13, 345, 50, 306, 276, 353, 77, 355, 108, 206, 136, 177, 118, 341, 252, 369, 45, 184, 354, 244, 31, 178, 211, 129, 35, 380, 90, 320, 242, 55, 120, 133, 106, 6, 358, 283, 374, 86, 251, 383, 51, 218, 9, 147, 189, 101, 264, 105, 220, 74, 166, 205, 191, 109, 199, 227, 73, 371, 62, 233, 46, 268, 309, 213, 381, 19, 83, 347] + 52: [381, 45, 383, 310, 18, 203, 86, 264, 46, 38, 176, 180, 99, 39, 58, 51, 226, 36, 14, 272, 305, 247, 348, 375, 334, 223, 154, 308, 152, 278, 302, 338, 295, 360, 16, 175, 102, 127, 23, 319, 93, 194, 342, 147, 0, 376, 162, 212, 107, 64, 222, 248, 374, 242, 143, 126, 273, 240, 218, 370, 157, 60, 171, 221, 361, 213, 294, 303, 299, 11, 317, 106, 71, 33, 98, 359, 113, 324, 365, 356, 335, 12, 373, 204, 104, 62, 54, 74, 367, 22, 270, 139, 322, 158, 1, 94, 165, 346, 349, 256, 233, 115, 235, 189, 118, 363, 24, 164, 97, 172, 252, 285, 364, 351, 350, 207, 344, 132, 144, 199, 283, 90, 357, 312, 191, 300, 208, 148, 368, 232, 145, 254, 103, 174, 68, 95, 378, 29, 279, 352, 220, 111, 276, 241, 137, 321, 190, 32, 339, 160, 377, 225, 251, 197, 275, 142, 336, 271, 288, 87, 379, 167, 53, 268, 84, 141, 274, 366, 239, 195, 41, 266, 133, 30, 17, 161, 21, 380, 112, 122, 209, 290, 382, 355, 291, 281, 146, 27, 201, 198, 353, 159, 48, 323, 183, 210, 236, 124, 280, 109, 89, 100, 358, 202, 25, 289, 13, 128, 123, 5, 301, 287, 224, 136, 181, 261, 205, 214, 263, 196, 108, 286, 72, 69, 59, 230, 229, 329, 52, 85, 169, 56, 35, 105, 243, 40, 237, 28, 43, 19, 186, 73, 345, 269, 354, 315, 6, 91, 200, 101, 81, 65, 327, 362, 34, 219, 50, 178, 153, 66, 277, 156, 114, 265, 121, 250, 163, 44, 70, 341, 328, 333, 244, 150, 4, 185, 92, 246, 193, 259, 257, 227, 75, 119, 110, 249, 293, 187, 307, 151, 206, 168, 8, 173, 67, 125, 177, 347, 304, 255, 120, 63, 7, 10, 37, 83, 116, 326, 57, 129, 96, 314, 231, 130, 184, 245, 325, 166, 369, 253, 258, 188, 311, 42, 332, 182, 78, 260, 88, 117, 76, 170, 282, 179, 313, 79, 61, 31, 298, 140, 155, 234, 331, 316, 20, 318, 135, 297, 216, 267, 82, 80, 228, 262, 3, 55, 284, 49, 340, 15, 372, 371, 77, 26, 309, 296, 292, 337, 238, 47, 343, 131, 217, 320, 211, 192, 149, 215, 134, 306, 2, 330, 138, 9] + 53: [110, 18, 140, 153, 167, 312, 316, 185, 84, 114, 210, 272, 240, 231, 349, 196, 79, 337, 1, 190, 58, 377, 56, 90, 73, 319, 383, 89, 120, 117, 275, 310, 187, 191, 314, 252, 38, 103, 25, 176, 44, 19, 123, 289, 301, 192, 223, 265, 16, 162, 241, 77, 159, 105, 251, 331, 51, 212, 244, 22, 317, 302, 121, 168, 52, 102, 321, 101, 108, 163, 27, 344, 13, 92, 31, 219, 10, 76, 236, 133, 380, 17, 267, 307, 350, 42, 332, 325, 172, 67, 158, 220, 264, 147, 61, 71, 23, 227, 261, 5, 151, 146, 327, 232, 283, 193, 181, 99, 334, 155, 237, 127, 205, 126, 339, 255, 137, 366, 343, 245, 287, 179, 376, 95, 188, 72, 115, 333, 367, 34, 260, 290, 296, 379, 291, 171, 116, 15, 216, 136, 257, 39, 98, 318, 109, 224, 194, 253, 183, 281, 111, 28, 87, 82, 104, 11, 354, 345, 157, 24, 112, 47, 362, 371, 0, 234, 374, 263, 311, 363, 86, 20, 341, 149, 175, 370, 294, 305, 348, 124, 347, 214, 351, 326, 338, 150, 122, 278, 138, 9, 298, 304, 295, 273, 268, 269, 361, 259, 353, 30, 358, 249, 342, 355, 250, 365, 309, 284, 94, 49, 233, 166, 59, 14, 64, 356, 330, 46, 202, 107, 62, 37, 248, 2, 195, 154, 303, 165, 32, 322, 360, 130, 70, 300, 340, 375, 242, 80, 106, 161, 4, 292, 156, 144, 141, 378, 369, 169, 258, 7, 113, 148, 186, 280, 180, 271, 381, 78, 328, 57, 254, 246, 184, 211, 230, 142, 160, 119, 139, 239, 36, 178, 45, 21, 372, 213, 336, 173, 228, 50, 3, 197, 282, 96, 198, 29, 83, 286, 308, 164, 145, 225, 69, 335, 128, 206, 54, 201, 93, 135, 217, 320, 279, 221, 12, 152, 35, 174, 299, 100, 177, 33, 75, 129, 132, 26, 262, 235, 293, 207, 364, 199, 48, 288, 8, 238, 118, 368, 200, 277, 359, 306, 182, 43, 63, 204, 68, 247, 266, 134, 256, 313, 357, 189, 55, 53, 229, 41, 297, 270, 170, 125, 88, 373, 143, 276, 97, 352, 208, 382, 243, 66, 6, 285, 329, 324, 226, 65, 218, 323, 85, 274, 203, 222, 209, 81, 346, 60, 315, 215, 40, 131, 74, 91] + 54: [363, 13, 160, 205, 0, 36, 148, 232, 320, 336, 361, 378, 370, 383, 19, 200, 202, 358, 130, 212, 106, 123, 127, 354, 294, 125, 97, 326, 195, 229, 269, 292, 252, 193, 340, 344, 339, 156, 335, 239, 355, 293, 347, 243, 52, 270, 207, 78, 155, 10, 198, 94, 49, 298, 275, 74, 178, 77, 2, 342, 164, 352, 99, 225, 246, 61, 191, 145, 247, 138, 112, 349, 284, 57, 375, 50, 103, 105, 107, 33, 251, 89, 374, 297, 283, 290, 317, 113, 152, 365, 327, 18, 351, 46, 102, 188, 42, 174, 75, 197, 24, 184, 201, 62, 271, 108, 149, 222, 258, 53, 314, 196, 34, 55, 68, 244, 334, 343, 285, 241, 11, 8, 40, 76, 272, 368, 85, 151, 141, 362, 56, 256, 356, 172, 25, 41, 17, 1, 373, 9, 259, 206, 303, 37, 223, 20, 119, 143, 91, 177, 73, 175, 308, 261, 307, 126, 231, 98, 311, 132, 124, 381, 122, 167, 371, 359, 360, 165, 240, 250, 87, 332, 357, 382, 21, 136, 353, 12, 233, 51, 348, 30, 330, 27, 64, 54, 140, 242, 146, 15, 367, 291, 38, 133, 345, 101, 322, 182, 186, 137, 114, 238, 60, 203, 280, 214, 221, 58, 277, 142, 204, 304, 364, 110, 29, 65, 230, 369, 305, 181, 158, 92, 237, 22, 81, 63, 331, 104, 299, 80, 187, 121, 84, 217, 312, 111, 173, 139, 263, 199, 286, 260, 93, 26, 333, 321, 135, 366, 23, 219, 236, 210, 95, 82, 282, 180, 288, 323, 278, 144, 266, 224, 249, 4, 295, 150, 281, 16, 47, 45, 183, 48, 273, 234, 274, 28, 337, 257, 306, 372, 129, 159, 376, 209, 162, 194, 66, 226, 380, 216, 72, 329, 115, 220, 318, 69, 300, 346, 289, 350, 153, 267, 120, 309, 310, 325, 67, 171, 128, 6, 14, 379, 296, 287, 324, 86, 161, 109, 157, 313, 377, 245, 248, 227, 253, 39, 44, 117, 70, 265, 315, 163, 166, 192, 79, 35, 71, 90, 176, 3, 215, 279, 170, 179, 96, 88, 268, 116, 319, 83, 341, 228, 32, 185, 131, 301, 316, 235, 211, 134, 328, 147, 208, 118, 218, 190, 302, 255, 262, 213, 31, 154, 189, 338, 254, 7, 276, 5, 100, 264, 43, 59, 168, 169] + 55: [154, 254, 61, 201, 50, 369, 283, 186, 317, 215, 69, 305, 174, 27, 54, 63, 365, 261, 162, 45, 222, 245, 204, 34, 56, 301, 44, 5, 95, 286, 117, 367, 139, 324, 250, 96, 202, 227, 333, 233, 289, 322, 278, 97, 118, 75, 83, 285, 88, 307, 249, 312, 231, 257, 84, 219, 137, 181, 57, 109, 81, 111, 2, 366, 276, 79, 358, 160, 321, 248, 266, 196, 98, 238, 341, 216, 325, 240, 23, 212, 126, 338, 282, 32, 20, 16, 200, 300, 14, 360, 255, 1, 319, 129, 298, 351, 11, 252, 291, 221, 350, 42, 128, 237, 35, 38, 36, 100, 146, 311, 106, 213, 348, 47, 277, 264, 37, 66, 183, 205, 308, 39, 40, 164, 102, 343, 346, 31, 258, 145, 242, 9, 340, 165, 49, 368, 170, 361, 287, 153, 103, 275, 169, 167, 116, 339, 132, 138, 46, 246, 318, 82, 232, 354, 262, 158, 80, 89, 199, 290, 163, 378, 228, 22, 225, 147, 155, 108, 381, 182, 65, 349, 112, 105, 53, 294, 335, 58, 364, 229, 123, 357, 70, 272, 316, 265, 122, 85, 253, 209, 244, 230, 299, 370, 363, 12, 124, 217, 194, 296, 143, 107, 280, 25, 90, 218, 304, 60, 0, 380, 10, 171, 93, 336, 110, 345, 48, 62, 382, 71, 271, 236, 19, 157, 372, 315, 92, 24, 347, 7, 273, 99, 239, 371, 119, 207, 30, 297, 362, 59, 74, 234, 353, 114, 26, 224, 263, 52, 356, 168, 21, 142, 323, 4, 121, 293, 189, 91, 306, 198, 342, 120, 191, 131, 180, 309, 247, 134, 337, 375, 136, 87, 281, 177, 13, 310, 130, 151, 78, 223, 86, 377, 184, 193, 268, 55, 330, 195, 29, 355, 303, 8, 197, 284, 135, 314, 359, 243, 334, 288, 269, 235, 73, 3, 172, 190, 67, 302, 173, 125, 77, 292, 115, 256, 328, 179, 259, 274, 178, 214, 127, 374, 376, 379, 76, 270, 192, 331, 17, 352, 187, 313, 208, 150, 141, 148, 326, 43, 6, 320, 383, 251, 140, 94, 373, 161, 295, 18, 113, 327, 267, 279, 156, 210, 185, 101, 15, 104, 175, 220, 72, 28, 41, 176, 206, 329, 241, 33, 133, 64, 166, 144, 159, 152, 211, 226, 344, 260, 332, 149, 188, 68, 203, 51] + 56: [231, 142, 85, 224, 315, 146, 173, 202, 3, 90, 143, 208, 240, 65, 382, 217, 171, 318, 102, 229, 76, 109, 301, 74, 195, 324, 149, 286, 54, 375, 352, 203, 277, 237, 51, 61, 260, 197, 105, 280, 141, 201, 226, 93, 26, 366, 91, 235, 220, 77, 205, 135, 214, 368, 119, 37, 113, 108, 377, 165, 294, 327, 309, 182, 305, 248, 360, 232, 287, 234, 45, 247, 321, 306, 292, 101, 342, 198, 185, 62, 4, 43, 215, 194, 242, 104, 275, 168, 160, 206, 297, 96, 172, 16, 261, 365, 265, 363, 316, 223, 233, 12, 11, 145, 228, 29, 326, 380, 44, 263, 46, 344, 58, 268, 138, 314, 310, 67, 335, 25, 178, 9, 346, 246, 103, 269, 153, 262, 353, 78, 336, 111, 82, 156, 48, 238, 359, 22, 34, 190, 274, 24, 47, 216, 213, 371, 161, 154, 312, 136, 56, 271, 112, 350, 68, 114, 15, 322, 328, 80, 0, 92, 50, 42, 218, 289, 212, 174, 225, 351, 162, 188, 150, 285, 295, 209, 299, 200, 361, 358, 241, 267, 117, 19, 14, 278, 317, 151, 94, 28, 130, 325, 106, 303, 6, 71, 72, 73, 355, 133, 210, 124, 222, 256, 340, 158, 125, 302, 63, 137, 338, 227, 55, 290, 266, 7, 259, 184, 308, 140, 97, 27, 354, 127, 183, 293, 49, 349, 30, 296, 251, 39, 207, 40, 79, 270, 257, 307, 64, 370, 5, 98, 187, 99, 118, 69, 35, 164, 298, 323, 144, 52, 75, 163, 333, 41, 383, 376, 374, 243, 134, 347, 170, 59, 304, 88, 264, 284, 120, 334, 331, 1, 84, 341, 367, 32, 147, 155, 89, 66, 159, 128, 179, 33, 221, 276, 152, 131, 279, 53, 20, 83, 357, 193, 166, 13, 378, 57, 373, 272, 100, 186, 189, 329, 192, 38, 252, 282, 313, 139, 70, 239, 177, 116, 356, 332, 381, 253, 230, 17, 115, 319, 204, 157, 8, 107, 169, 18, 283, 281, 21, 364, 180, 132, 372, 36, 311, 273, 87, 2, 245, 250, 23, 255, 339, 291, 86, 176, 343, 81, 196, 362, 288, 10, 31, 123, 95, 330, 211, 258, 167, 199, 236, 219, 244, 148, 181, 129, 122, 379, 254, 337, 126, 191, 249, 320, 369, 300, 121, 110, 348, 175, 60, 345] + 57: [254, 56, 86, 234, 262, 275, 376, 293, 372, 301, 304, 343, 276, 167, 249, 12, 337, 175, 299, 375, 339, 280, 19, 285, 326, 127, 238, 214, 208, 351, 323, 307, 194, 218, 15, 233, 25, 174, 272, 228, 92, 71, 132, 122, 260, 239, 193, 143, 231, 6, 80, 243, 111, 210, 17, 303, 36, 67, 52, 367, 14, 124, 35, 176, 277, 55, 216, 363, 219, 78, 269, 202, 146, 359, 317, 99, 70, 47, 196, 379, 200, 361, 212, 34, 259, 108, 197, 91, 27, 203, 353, 18, 255, 222, 258, 201, 295, 223, 283, 58, 177, 120, 352, 41, 265, 48, 244, 229, 165, 257, 318, 338, 152, 38, 149, 334, 270, 242, 291, 105, 89, 128, 151, 347, 51, 289, 3, 29, 24, 256, 114, 69, 246, 101, 84, 144, 42, 284, 171, 162, 59, 374, 115, 77, 187, 65, 117, 319, 96, 358, 199, 371, 248, 192, 100, 181, 205, 235, 150, 182, 341, 147, 164, 186, 135, 66, 364, 221, 148, 366, 204, 354, 297, 183, 313, 215, 211, 263, 370, 327, 207, 104, 169, 251, 9, 54, 226, 311, 236, 383, 232, 294, 331, 140, 350, 250, 73, 116, 136, 98, 45, 50, 138, 340, 94, 198, 217, 62, 134, 102, 68, 330, 49, 87, 310, 245, 72, 7, 60, 75, 40, 377, 2, 362, 261, 333, 107, 278, 286, 156, 300, 163, 213, 184, 166, 28, 332, 224, 103, 189, 320, 63, 267, 123, 346, 349, 348, 81, 26, 126, 360, 37, 170, 368, 118, 44, 322, 21, 329, 23, 290, 188, 10, 292, 125, 57, 344, 88, 161, 298, 328, 369, 237, 4, 271, 336, 302, 85, 268, 79, 172, 113, 380, 178, 97, 168, 308, 131, 74, 220, 53, 306, 253, 382, 95, 190, 355, 109, 76, 82, 154, 121, 324, 309, 373, 46, 325, 155, 145, 357, 61, 30, 157, 314, 8, 160, 316, 312, 279, 282, 33, 32, 264, 129, 93, 133, 130, 64, 365, 356, 13, 209, 252, 0, 273, 153, 11, 158, 335, 345, 110, 173, 112, 321, 5, 378, 142, 195, 342, 16, 137, 185, 43, 83, 119, 296, 247, 241, 139, 227, 159, 381, 141, 22, 39, 191, 225, 90, 240, 281, 266, 206, 230, 31, 274, 180, 179, 20, 106, 288, 315, 1, 287, 305] + 58: [377, 280, 364, 201, 204, 340, 33, 38, 58, 134, 209, 308, 220, 71, 125, 319, 321, 335, 370, 10, 108, 150, 168, 184, 81, 144, 70, 170, 354, 349, 218, 233, 252, 292, 309, 3, 245, 104, 229, 244, 378, 161, 50, 221, 146, 137, 320, 279, 156, 191, 185, 381, 135, 253, 230, 268, 200, 216, 80, 147, 275, 196, 260, 236, 250, 347, 293, 100, 167, 301, 164, 158, 126, 45, 60, 48, 99, 37, 327, 173, 5, 57, 323, 23, 56, 376, 162, 192, 284, 339, 11, 113, 24, 109, 343, 165, 366, 36, 189, 73, 375, 300, 357, 180, 317, 51, 304, 166, 305, 116, 342, 106, 53, 382, 107, 179, 303, 290, 15, 313, 205, 257, 203, 198, 331, 241, 172, 64, 102, 117, 363, 212, 92, 72, 272, 328, 202, 302, 379, 338, 54, 337, 30, 291, 98, 289, 295, 255, 139, 350, 124, 278, 353, 163, 41, 17, 194, 69, 262, 111, 1, 181, 332, 148, 219, 128, 374, 127, 311, 22, 68, 132, 380, 258, 63, 13, 118, 213, 120, 91, 222, 9, 232, 130, 187, 25, 160, 334, 8, 264, 103, 52, 177, 49, 152, 315, 281, 62, 240, 288, 237, 330, 44, 153, 312, 235, 175, 97, 28, 251, 227, 318, 195, 276, 157, 78, 122, 299, 368, 6, 316, 238, 234, 345, 225, 359, 46, 159, 298, 206, 40, 85, 190, 129, 75, 154, 133, 261, 246, 361, 114, 93, 178, 141, 171, 55, 121, 176, 136, 296, 32, 188, 263, 322, 169, 367, 352, 266, 131, 186, 94, 243, 34, 267, 248, 373, 112, 47, 20, 215, 83, 228, 19, 138, 39, 365, 77, 101, 214, 31, 242, 59, 155, 21, 265, 142, 369, 89, 249, 271, 256, 18, 197, 82, 326, 2, 115, 360, 96, 74, 151, 14, 199, 341, 66, 239, 149, 231, 105, 294, 145, 174, 76, 211, 26, 324, 224, 314, 333, 371, 182, 29, 223, 217, 61, 210, 355, 16, 274, 325, 110, 208, 226, 95, 247, 310, 79, 344, 270, 119, 4, 306, 0, 362, 297, 372, 269, 140, 259, 348, 351, 12, 285, 383, 183, 65, 283, 282, 254, 88, 273, 27, 67, 277, 329, 143, 90, 43, 287, 87, 84, 193, 7, 356, 358, 42, 307, 207, 346, 123, 35, 86, 336, 286] + 59: [327, 51, 110, 93, 241, 56, 190, 308, 341, 215, 271, 189, 25, 223, 248, 69, 123, 246, 89, 230, 192, 375, 188, 154, 170, 175, 162, 258, 349, 149, 326, 11, 303, 351, 310, 79, 229, 262, 40, 373, 198, 61, 325, 261, 312, 335, 4, 268, 7, 178, 304, 252, 339, 2, 41, 145, 284, 270, 116, 235, 140, 265, 114, 62, 54, 101, 32, 207, 210, 305, 272, 90, 209, 172, 128, 297, 338, 289, 380, 343, 183, 301, 55, 111, 224, 221, 144, 35, 20, 180, 96, 129, 296, 239, 205, 117, 250, 294, 26, 57, 376, 269, 242, 34, 277, 358, 39, 15, 379, 371, 23, 161, 315, 120, 68, 135, 49, 103, 14, 139, 370, 263, 317, 109, 267, 214, 353, 299, 359, 3, 219, 293, 94, 204, 372, 112, 163, 356, 228, 48, 74, 44, 350, 77, 195, 247, 9, 133, 320, 222, 337, 71, 184, 357, 276, 347, 202, 298, 377, 290, 169, 47, 36, 227, 124, 148, 8, 345, 260, 374, 186, 342, 383, 179, 6, 0, 217, 118, 82, 99, 59, 311, 38, 45, 364, 158, 92, 64, 153, 88, 360, 171, 363, 381, 213, 176, 280, 10, 238, 63, 330, 24, 368, 323, 200, 251, 328, 91, 211, 236, 259, 43, 134, 191, 165, 72, 173, 70, 295, 278, 302, 355, 354, 193, 157, 275, 319, 122, 340, 95, 324, 212, 334, 329, 131, 194, 37, 336, 125, 244, 220, 5, 151, 164, 100, 286, 13, 206, 17, 249, 257, 83, 66, 233, 300, 86, 159, 306, 365, 333, 21, 106, 216, 52, 346, 226, 31, 196, 127, 245, 361, 27, 231, 119, 108, 104, 352, 147, 18, 255, 279, 19, 203, 138, 84, 321, 282, 87, 234, 197, 42, 58, 181, 142, 155, 287, 28, 132, 1, 264, 65, 97, 185, 309, 168, 29, 177, 156, 67, 115, 283, 167, 332, 313, 331, 243, 12, 143, 348, 107, 187, 307, 240, 121, 73, 256, 76, 281, 85, 22, 344, 362, 208, 102, 98, 378, 113, 46, 150, 266, 16, 218, 366, 314, 253, 53, 237, 78, 182, 75, 288, 232, 254, 174, 369, 199, 50, 166, 316, 126, 33, 285, 291, 382, 160, 60, 137, 292, 136, 201, 318, 152, 30, 105, 130, 80, 81, 225, 146, 274, 322, 273, 141, 367] + 60: [313, 317, 63, 154, 207, 348, 299, 288, 375, 22, 201, 355, 124, 218, 226, 242, 308, 89, 136, 210, 373, 62, 157, 214, 351, 262, 224, 135, 173, 4, 32, 101, 321, 10, 77, 87, 123, 8, 66, 380, 202, 50, 229, 163, 295, 352, 118, 109, 257, 140, 183, 274, 85, 79, 18, 258, 97, 164, 340, 44, 16, 235, 20, 162, 21, 95, 225, 198, 356, 71, 103, 73, 268, 234, 232, 336, 259, 304, 200, 219, 251, 379, 281, 275, 381, 76, 122, 152, 1, 116, 91, 144, 23, 374, 254, 177, 158, 2, 329, 27, 269, 110, 142, 30, 333, 320, 175, 227, 220, 153, 353, 231, 362, 188, 171, 100, 338, 178, 301, 289, 47, 312, 49, 117, 189, 344, 186, 190, 105, 45, 17, 292, 139, 120, 319, 272, 84, 11, 25, 322, 64, 19, 147, 138, 60, 247, 337, 184, 150, 240, 323, 370, 331, 137, 96, 367, 126, 230, 294, 131, 75, 318, 279, 293, 368, 67, 42, 68, 83, 167, 35, 245, 172, 6, 377, 70, 182, 325, 179, 111, 133, 194, 108, 206, 359, 51, 57, 233, 300, 363, 216, 298, 199, 78, 239, 278, 253, 217, 176, 346, 369, 383, 261, 291, 40, 48, 205, 15, 263, 36, 264, 31, 7, 168, 209, 72, 174, 222, 366, 213, 246, 132, 237, 273, 106, 371, 54, 378, 265, 212, 34, 98, 364, 221, 3, 107, 130, 148, 228, 86, 310, 316, 208, 311, 55, 37, 38, 360, 192, 315, 287, 159, 284, 339, 81, 166, 238, 114, 61, 53, 149, 270, 324, 165, 102, 113, 155, 193, 241, 93, 267, 187, 341, 125, 127, 43, 94, 39, 74, 354, 169, 255, 382, 285, 33, 376, 328, 191, 236, 372, 327, 309, 82, 358, 215, 52, 146, 307, 252, 249, 121, 347, 28, 345, 99, 283, 5, 90, 196, 282, 13, 306, 335, 244, 181, 119, 271, 248, 9, 134, 357, 197, 330, 326, 128, 343, 334, 290, 59, 58, 88, 56, 314, 12, 280, 29, 260, 303, 302, 112, 223, 69, 332, 26, 349, 14, 41, 160, 170, 129, 65, 180, 143, 277, 24, 305, 296, 286, 104, 204, 115, 266, 185, 365, 161, 151, 92, 141, 250, 156, 361, 350, 195, 243, 297, 46, 203, 0, 80, 342, 276, 256, 145, 211] + 61: [221, 44, 104, 37, 342, 246, 28, 217, 356, 155, 0, 58, 127, 366, 145, 306, 4, 54, 314, 226, 128, 321, 149, 319, 179, 159, 75, 308, 137, 174, 204, 262, 339, 376, 96, 94, 338, 286, 133, 260, 72, 365, 244, 354, 285, 175, 53, 208, 61, 294, 181, 43, 257, 186, 125, 359, 24, 265, 229, 171, 73, 68, 291, 87, 374, 259, 219, 274, 187, 60, 151, 287, 93, 380, 66, 32, 300, 147, 141, 144, 305, 373, 40, 116, 211, 1, 119, 227, 83, 97, 196, 176, 201, 95, 334, 343, 46, 264, 237, 309, 135, 52, 275, 124, 101, 256, 62, 189, 299, 184, 213, 169, 114, 328, 30, 71, 163, 272, 277, 350, 197, 209, 55, 243, 84, 245, 346, 317, 25, 123, 250, 183, 381, 59, 143, 156, 268, 23, 153, 81, 91, 99, 35, 210, 100, 241, 198, 282, 29, 206, 273, 132, 271, 234, 318, 382, 36, 270, 235, 160, 316, 368, 214, 26, 63, 18, 347, 134, 167, 19, 215, 15, 233, 69, 78, 295, 130, 301, 220, 355, 315, 10, 142, 31, 90, 3, 292, 340, 352, 364, 222, 39, 344, 203, 238, 349, 150, 336, 89, 375, 311, 255, 56, 254, 324, 117, 371, 177, 118, 113, 322, 341, 98, 232, 105, 41, 248, 178, 278, 112, 258, 304, 200, 240, 194, 13, 180, 357, 263, 323, 51, 192, 77, 166, 38, 8, 106, 312, 140, 267, 307, 122, 21, 188, 297, 27, 154, 280, 152, 49, 48, 320, 161, 205, 12, 57, 261, 2, 173, 22, 148, 251, 146, 367, 86, 165, 195, 42, 348, 252, 378, 79, 298, 7, 182, 266, 88, 351, 17, 129, 361, 379, 158, 80, 345, 310, 377, 289, 107, 247, 330, 168, 20, 313, 102, 269, 70, 358, 276, 281, 216, 326, 372, 327, 224, 190, 383, 9, 332, 185, 164, 296, 231, 170, 103, 115, 228, 47, 249, 283, 162, 74, 236, 50, 303, 157, 253, 369, 191, 11, 207, 67, 199, 293, 212, 16, 230, 120, 353, 34, 92, 109, 290, 85, 14, 111, 223, 76, 172, 360, 218, 110, 65, 82, 139, 242, 370, 6, 302, 279, 5, 108, 45, 288, 337, 329, 33, 126, 193, 239, 64, 131, 136, 335, 363, 333, 362, 121, 202, 225, 284, 331, 138, 325] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots416.yaml b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots416.yaml new file mode 100644 index 000000000000..cf48f10d4ae8 --- /dev/null +++ b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots416.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [259, 332, 218, 203, 349, 233, 337, 317, 99, 326, 334, 239, 201, 166, 45, 95, 278, 330, 265, 77, 213, 366, 252, 352, 23, 8, 166, 190, 157, 139, 360, 156, 300, 4, 178, 260, 340, 250, 118, 0, 45, 95, 63, 144, 124, 138, 137, 363, 350, 15, 167, 288, 256, 378, 80, 68, 290, 200, 44, 335, 192, 263, 175, 285, 120, 256, 346, 17, 293, 330, 361, 149, 214, 369, 229, 338, 226, 291, 150, 378, 29, 203, 126, 183, 123, 5, 25, 73, 2, 319, 61, 151, 378, 79, 333, 280, 90, 112, 46, 75, 210, 96, 287, 364, 74, 158, 62, 176, 290, 311, 7, 244, 211, 342, 380, 18, 88, 74, 241, 242, 37, 173, 297, 268, 377, 320, 127, 219, 220, 310, 71, 279, 17, 217, 3, 36, 31, 257, 215, 362, 306, 251, 13, 222, 193, 30, 16, 173, 129, 170, 227, 59, 328, 179, 301, 247, 222, 230, 104, 198, 209, 10, 94, 365, 275, 93, 113, 367, 303, 302, 279, 204, 318, 224, 258, 148, 187, 238, 98, 294, 329, 344, 351, 207, 79, 276, 224, 174, 248, 108, 116, 14, 315, 164, 370, 356, 264, 202, 232, 54, 105, 262, 91, 1, 6, 316, 121, 281, 356, 76, 153, 134, 284, 131, 38, 52, 195, 82, 53, 266, 58, 84, 135, 60, 321, 205, 57, 324, 331, 216, 67, 353, 125, 277, 159, 135, 21, 184, 78, 56, 114, 146, 246, 50, 212, 271, 130, 345, 235, 81, 134, 359, 354, 368, 240, 269, 20, 42, 142, 355, 322, 111, 60, 289, 307, 196, 89, 39, 106, 267, 152, 194, 373, 119, 111, 26, 327, 33, 11, 199, 85, 379, 382, 191, 231, 274, 236, 165, 122, 49, 181, 43, 208, 136, 19, 147, 69, 381, 185, 236, 254, 35, 48, 141, 283, 160, 32, 296, 286, 305, 374, 325, 245, 254, 347, 255, 143, 107, 234, 66, 292, 83, 12, 9, 171, 245, 111, 51, 343, 55, 313, 314, 308, 100, 154, 40, 298, 168, 371, 165, 163, 49, 72, 34, 309, 172, 180, 145, 115, 132, 323, 371, 165, 295, 184, 243, 117, 375, 133, 97, 102, 161, 304, 339, 383, 177, 312, 217, 272, 186, 28, 103, 372, 47, 64, 189, 128, 225, 162, 169, 376, 336, 341, 27, 87, 221, 197, 253, 24, 228, 225, 348, 188, 70, 22, 101, 358, 223, 270, 299, 206, 92, 273, 86, 155, 182, 65, 282, 109, 261, 249, 140, 237, 41, 357, 110] + 1: [134, 378, 73, 225, 59, 166, 202, 231, 154, 230, 314, 361, 157, 178, 254, 110, 225, 195, 187, 376, 60, 175, 284, 103, 66, 336, 380, 329, 41, 250, 273, 197, 264, 205, 165, 87, 220, 340, 255, 141, 127, 328, 3, 32, 281, 47, 140, 286, 181, 240, 64, 307, 251, 127, 377, 3, 173, 266, 214, 313, 318, 113, 54, 128, 159, 81, 163, 325, 182, 275, 285, 143, 259, 364, 8, 232, 374, 117, 200, 163, 67, 153, 224, 335, 239, 61, 115, 11, 130, 373, 15, 365, 291, 6, 198, 145, 99, 52, 79, 2, 26, 279, 183, 354, 365, 163, 67, 23, 138, 108, 358, 76, 296, 51, 369, 151, 257, 1, 167, 192, 120, 269, 212, 360, 102, 68, 48, 119, 155, 349, 235, 167, 5, 198, 37, 301, 53, 305, 161, 13, 262, 45, 144, 42, 177, 0, 105, 242, 125, 206, 348, 20, 97, 371, 253, 112, 42, 196, 70, 63, 89, 234, 203, 298, 19, 324, 309, 193, 327, 201, 169, 179, 244, 287, 355, 4, 170, 370, 268, 226, 322, 359, 201, 177, 270, 300, 211, 317, 219, 222, 65, 104, 246, 188, 199, 174, 72, 30, 120, 303, 146, 176, 121, 326, 55, 12, 208, 215, 174, 169, 40, 82, 29, 162, 88, 98, 33, 36, 331, 367, 368, 31, 72, 131, 248, 292, 218, 306, 237, 321, 186, 297, 160, 345, 22, 91, 319, 271, 46, 261, 38, 133, 136, 35, 356, 16, 258, 22, 91, 319, 351, 320, 142, 381, 362, 14, 227, 27, 132, 341, 308, 43, 21, 233, 243, 84, 93, 256, 57, 172, 83, 135, 363, 118, 263, 366, 299, 9, 74, 213, 204, 338, 94, 353, 191, 276, 137, 50, 168, 111, 272, 194, 289, 122, 17, 311, 372, 90, 267, 137, 152, 319, 82, 383, 109, 278, 236, 295, 350, 375, 207, 304, 18, 150, 241, 111, 272, 92, 228, 164, 107, 352, 129, 316, 100, 180, 126, 347, 185, 283, 379, 342, 147, 184, 293, 148, 217, 310, 180, 149, 347, 24, 25, 280, 171, 101, 156, 302, 49, 252, 339, 44, 382, 377, 182, 260, 78, 223, 238, 139, 96, 333, 343, 28, 158, 277, 123, 334, 247, 245, 282, 7, 332, 346, 344, 10, 312, 158, 382, 39, 56, 292, 288, 69, 95, 62, 290, 190, 209, 34, 106, 277, 210, 58, 71, 216, 116, 189, 229, 315, 357, 221, 265, 80, 85, 75, 294, 86, 274, 114, 330, 249, 323, 77, 124, 337] + 2: [53, 33, 322, 245, 22, 356, 126, 103, 38, 211, 98, 37, 375, 53, 75, 191, 70, 63, 209, 201, 41, 365, 231, 292, 260, 337, 43, 248, 124, 86, 26, 299, 327, 15, 305, 35, 183, 227, 190, 43, 197, 284, 44, 325, 87, 50, 348, 150, 141, 226, 315, 358, 180, 248, 97, 56, 83, 204, 361, 309, 142, 272, 251, 283, 199, 230, 248, 350, 280, 274, 294, 187, 241, 252, 52, 371, 177, 300, 96, 34, 185, 118, 122, 347, 55, 175, 257, 144, 291, 214, 282, 132, 212, 153, 86, 368, 224, 293, 193, 268, 233, 302, 369, 237, 132, 212, 110, 278, 200, 174, 58, 332, 2, 178, 373, 222, 363, 112, 143, 155, 73, 146, 61, 21, 10, 318, 261, 54, 310, 366, 112, 143, 84, 157, 24, 285, 76, 145, 39, 281, 74, 336, 381, 108, 247, 109, 286, 279, 196, 218, 65, 176, 376, 95, 117, 161, 108, 143, 19, 135, 122, 263, 27, 308, 29, 334, 167, 64, 228, 225, 125, 290, 220, 31, 344, 20, 265, 179, 207, 107, 378, 379, 341, 151, 19, 271, 296, 18, 156, 181, 312, 264, 288, 382, 258, 351, 100, 162, 105, 304, 90, 269, 250, 106, 239, 158, 345, 255, 313, 321, 168, 355, 85, 311, 236, 171, 49, 346, 205, 80, 326, 313, 189, 78, 172, 314, 170, 188, 129, 372, 275, 307, 147, 354, 357, 137, 111, 377, 328, 165, 67, 254, 136, 77, 164, 127, 119, 215, 295, 62, 91, 82, 6, 120, 349, 23, 319, 71, 182, 330, 215, 295, 62, 91, 11, 323, 297, 46, 219, 352, 329, 380, 32, 14, 221, 168, 66, 340, 342, 128, 360, 0, 367, 114, 266, 316, 89, 3, 101, 184, 51, 130, 92, 116, 9, 25, 8, 88, 287, 89, 3, 290, 123, 139, 140, 320, 160, 154, 47, 335, 13, 208, 343, 5, 149, 306, 324, 273, 68, 169, 12, 134, 267, 289, 301, 194, 81, 213, 133, 139, 206, 210, 186, 121, 238, 338, 353, 202, 194, 259, 232, 304, 314, 17, 203, 276, 364, 36, 59, 317, 246, 102, 1, 192, 42, 45, 7, 262, 69, 256, 195, 48, 72, 229, 93, 223, 232, 377, 217, 79, 383, 163, 94, 359, 216, 240, 339, 93, 298, 60, 131, 333, 159, 148, 277, 152, 4, 99, 234, 40, 115, 28, 242, 173, 235, 30, 138, 253, 362, 57, 243, 249, 270, 115, 223, 242, 166, 244, 303, 198, 16, 104, 331, 370, 113, 374] + 3: [39, 192, 97, 336, 36, 109, 94, 366, 115, 101, 116, 175, 226, 39, 3, 194, 37, 287, 99, 200, 214, 20, 362, 121, 95, 198, 22, 56, 263, 358, 177, 161, 166, 255, 67, 373, 360, 205, 38, 22, 208, 261, 342, 182, 271, 270, 160, 43, 93, 190, 347, 17, 66, 27, 333, 2, 316, 84, 55, 44, 361, 153, 88, 75, 52, 66, 27, 80, 168, 25, 277, 224, 105, 91, 259, 107, 241, 301, 66, 131, 194, 325, 221, 172, 235, 76, 249, 321, 140, 89, 9, 66, 163, 313, 227, 74, 78, 197, 237, 306, 285, 179, 308, 335, 66, 163, 313, 211, 212, 359, 256, 328, 340, 206, 183, 23, 193, 66, 13, 3, 10, 138, 289, 380, 268, 70, 282, 16, 127, 233, 71, 225, 157, 343, 64, 111, 345, 104, 187, 293, 251, 272, 257, 46, 317, 263, 58, 174, 279, 327, 196, 286, 269, 134, 315, 341, 34, 14, 213, 201, 382, 229, 204, 65, 367, 346, 191, 253, 370, 34, 171, 145, 164, 219, 136, 112, 276, 298, 309, 323, 352, 146, 34, 51, 150, 139, 12, 262, 274, 117, 7, 381, 369, 54, 292, 34, 31, 158, 186, 29, 149, 126, 135, 232, 61, 242, 90, 303, 34, 350, 348, 207, 223, 374, 98, 280, 291, 130, 284, 147, 59, 34, 19, 62, 63, 118, 154, 217, 144, 216, 258, 354, 152, 33, 79, 18, 49, 122, 304, 295, 103, 273, 15, 85, 210, 148, 199, 79, 18, 81, 267, 296, 8, 53, 30, 322, 254, 245, 320, 5, 310, 319, 252, 189, 77, 142, 236, 82, 353, 283, 378, 349, 371, 310, 319, 50, 250, 239, 162, 344, 125, 96, 86, 220, 4, 173, 310, 319, 180, 170, 11, 32, 108, 355, 376, 228, 372, 60, 326, 275, 40, 331, 26, 264, 312, 181, 48, 128, 123, 329, 368, 311, 260, 365, 42, 215, 240, 92, 155, 383, 184, 28, 379, 69, 167, 260, 365, 137, 337, 290, 297, 375, 307, 294, 356, 45, 141, 47, 260, 40, 6, 102, 202, 203, 218, 169, 83, 132, 338, 330, 278, 35, 365, 334, 178, 21, 114, 176, 113, 188, 72, 156, 124, 247, 377, 302, 363, 238, 357, 243, 300, 246, 305, 110, 234, 265, 332, 377, 302, 318, 339, 87, 351, 41, 129, 106, 68, 73, 1, 324, 377, 314, 151, 231, 195, 364, 57, 100, 248, 133, 0, 244, 185, 230, 209, 24, 120, 119, 288, 299, 266, 281, 165, 159, 143, 222] + 4: [188, 19, 273, 253, 68, 0, 185, 64, 157, 146, 255, 174, 182, 34, 376, 117, 36, 280, 116, 379, 125, 29, 122, 205, 89, 287, 340, 226, 194, 274, 333, 187, 359, 80, 292, 130, 81, 382, 351, 3, 88, 142, 377, 8, 38, 189, 232, 346, 304, 353, 245, 233, 3, 281, 194, 59, 312, 22, 323, 310, 78, 140, 311, 266, 104, 3, 226, 105, 155, 339, 82, 149, 322, 176, 147, 362, 102, 227, 18, 95, 254, 347, 160, 177, 93, 69, 138, 40, 378, 326, 336, 209, 239, 152, 61, 364, 244, 114, 46, 303, 327, 374, 373, 225, 209, 95, 254, 167, 83, 179, 252, 240, 184, 210, 48, 248, 208, 271, 228, 136, 329, 204, 7, 341, 328, 178, 355, 129, 212, 119, 271, 172, 302, 13, 344, 207, 360, 216, 263, 334, 279, 30, 242, 271, 9, 152, 200, 361, 264, 72, 269, 103, 295, 97, 293, 192, 37, 16, 241, 139, 235, 358, 330, 131, 349, 286, 115, 285, 17, 12, 15, 165, 246, 57, 141, 62, 132, 26, 170, 126, 270, 305, 375, 15, 143, 76, 27, 35, 14, 137, 52, 211, 55, 91, 79, 124, 23, 348, 325, 101, 159, 352, 100, 309, 156, 381, 363, 277, 199, 193, 148, 298, 49, 110, 75, 367, 50, 118, 316, 144, 128, 222, 23, 148, 56, 217, 153, 85, 158, 39, 71, 162, 265, 58, 258, 371, 257, 77, 365, 221, 166, 135, 66, 111, 247, 368, 63, 296, 214, 87, 4, 314, 278, 98, 261, 175, 291, 186, 299, 332, 296, 371, 45, 41, 43, 369, 60, 354, 230, 237, 112, 11, 337, 300, 214, 319, 372, 32, 283, 356, 74, 206, 181, 203, 301, 231, 343, 350, 117, 342, 198, 335, 168, 70, 92, 223, 191, 164, 67, 343, 259, 273, 86, 251, 42, 297, 229, 121, 249, 21, 190, 20, 343, 250, 87, 318, 357, 99, 307, 288, 84, 31, 380, 134, 272, 343, 320, 5, 275, 151, 276, 256, 108, 51, 180, 10, 195, 282, 343, 214, 319, 113, 44, 317, 169, 262, 267, 109, 173, 366, 321, 331, 24, 308, 106, 196, 243, 53, 313, 234, 289, 123, 224, 2, 331, 219, 6, 145, 218, 183, 133, 284, 370, 213, 96, 236, 107, 331, 315, 150, 197, 28, 260, 33, 383, 294, 268, 1, 238, 338, 331, 315, 171, 47, 73, 54, 290, 25, 306, 201, 163, 202, 90, 331, 315, 142, 345, 161, 324, 65, 94, 220, 127, 215, 154, 120] + 5: [172, 269, 64, 182, 327, 184, 19, 56, 271, 187, 297, 110, 179, 208, 169, 102, 364, 304, 143, 12, 20, 349, 199, 347, 226, 83, 61, 188, 41, 192, 66, 377, 195, 241, 13, 113, 197, 357, 23, 61, 305, 21, 89, 8, 218, 301, 289, 254, 298, 189, 27, 321, 61, 305, 21, 147, 155, 152, 104, 136, 119, 244, 290, 0, 166, 165, 193, 379, 198, 371, 38, 345, 341, 360, 221, 309, 123, 167, 154, 262, 259, 255, 238, 324, 79, 343, 376, 378, 111, 340, 196, 190, 105, 259, 130, 2, 3, 380, 382, 94, 283, 267, 282, 44, 190, 370, 211, 72, 369, 315, 228, 284, 330, 168, 125, 248, 118, 141, 105, 122, 132, 332, 45, 16, 325, 9, 129, 185, 294, 82, 213, 276, 159, 252, 163, 210, 51, 366, 229, 40, 224, 112, 303, 213, 100, 272, 246, 313, 314, 99, 317, 334, 270, 133, 234, 96, 213, 100, 372, 293, 131, 220, 33, 76, 383, 58, 148, 116, 157, 249, 223, 78, 43, 37, 207, 266, 230, 62, 263, 11, 146, 50, 249, 276, 231, 88, 171, 318, 93, 362, 77, 205, 73, 311, 354, 128, 346, 231, 200, 151, 251, 329, 257, 291, 374, 307, 36, 127, 128, 346, 159, 145, 174, 26, 46, 138, 149, 68, 373, 339, 177, 128, 265, 6, 7, 15, 140, 242, 201, 29, 235, 85, 328, 75, 306, 49, 272, 212, 268, 258, 28, 63, 48, 342, 245, 117, 278, 256, 60, 288, 331, 70, 206, 39, 353, 55, 103, 120, 158, 57, 84, 355, 6, 352, 375, 320, 115, 203, 275, 316, 32, 10, 142, 84, 178, 52, 279, 59, 153, 35, 150, 80, 299, 126, 202, 337, 296, 355, 381, 22, 124, 108, 183, 106, 191, 135, 95, 310, 176, 296, 355, 356, 25, 31, 69, 274, 300, 134, 363, 97, 243, 170, 308, 65, 30, 338, 233, 319, 253, 121, 344, 323, 139, 42, 164, 214, 1, 287, 91, 4, 204, 216, 239, 162, 107, 86, 273, 225, 214, 81, 356, 74, 209, 181, 227, 14, 232, 156, 261, 367, 368, 214, 47, 269, 335, 173, 67, 312, 250, 236, 359, 285, 350, 302, 5, 217, 288, 351, 358, 24, 186, 219, 87, 114, 101, 180, 326, 194, 98, 365, 17, 222, 175, 144, 240, 361, 71, 53, 336, 286, 194, 217, 52, 215, 264, 34, 54, 90, 281, 92, 322, 280, 277, 160, 260, 287, 247, 137, 333, 295, 161, 237, 18, 292, 109, 348] + 6: [322, 46, 350, 283, 176, 279, 69, 159, 299, 310, 78, 26, 90, 322, 99, 371, 255, 179, 158, 271, 225, 54, 73, 356, 127, 325, 322, 99, 9, 84, 113, 223, 237, 82, 215, 244, 250, 29, 161, 146, 11, 36, 56, 345, 37, 262, 284, 272, 364, 263, 92, 164, 76, 281, 375, 57, 277, 49, 43, 23, 16, 366, 238, 171, 257, 76, 281, 378, 368, 95, 311, 242, 372, 83, 251, 256, 72, 65, 76, 229, 166, 239, 128, 114, 10, 14, 168, 290, 266, 265, 153, 297, 240, 177, 34, 182, 221, 280, 330, 96, 123, 339, 192, 342, 305, 55, 180, 208, 278, 107, 332, 148, 198, 106, 51, 170, 135, 305, 240, 233, 118, 369, 77, 264, 81, 333, 328, 102, 20, 252, 97, 28, 205, 178, 41, 321, 210, 276, 309, 134, 131, 213, 373, 97, 341, 66, 184, 377, 44, 136, 145, 48, 360, 338, 191, 167, 97, 253, 209, 155, 352, 139, 296, 122, 105, 347, 162, 61, 75, 97, 253, 66, 291, 336, 327, 324, 316, 199, 85, 117, 2, 314, 212, 269, 233, 79, 236, 211, 143, 126, 346, 226, 17, 228, 132, 142, 174, 45, 189, 354, 30, 87, 60, 334, 74, 47, 273, 25, 142, 267, 173, 112, 270, 12, 361, 63, 381, 235, 121, 241, 35, 142, 258, 31, 319, 109, 282, 365, 86, 349, 357, 201, 91, 331, 142, 258, 67, 188, 185, 190, 22, 116, 70, 292, 374, 19, 320, 59, 174, 120, 196, 245, 169, 58, 362, 298, 39, 183, 1, 204, 295, 306, 175, 246, 165, 304, 110, 335, 125, 379, 302, 140, 247, 274, 343, 181, 293, 0, 68, 206, 355, 358, 62, 348, 200, 214, 274, 27, 177, 287, 230, 383, 286, 138, 18, 337, 303, 382, 186, 274, 249, 166, 129, 115, 64, 98, 194, 50, 152, 89, 141, 71, 5, 343, 13, 100, 234, 101, 137, 197, 268, 285, 313, 93, 275, 308, 32, 216, 367, 151, 231, 104, 4, 193, 108, 294, 353, 149, 308, 32, 318, 232, 300, 111, 6, 243, 42, 312, 38, 7, 40, 308, 53, 307, 187, 259, 202, 154, 329, 3, 203, 24, 340, 33, 308, 53, 163, 94, 301, 317, 207, 217, 124, 288, 195, 344, 52, 376, 103, 260, 119, 370, 144, 220, 254, 147, 80, 351, 218, 363, 376, 160, 359, 88, 156, 219, 15, 248, 224, 150, 21, 172, 315, 376, 32, 326, 289, 380, 133, 157, 261, 8, 130, 222, 227, 323] + 7: [163, 340, 328, 217, 31, 81, 5, 220, 351, 106, 254, 143, 154, 341, 340, 188, 100, 278, 372, 16, 134, 233, 295, 297, 147, 223, 327, 382, 212, 267, 248, 158, 314, 53, 161, 316, 124, 95, 110, 327, 382, 329, 215, 84, 21, 20, 335, 193, 184, 266, 198, 364, 327, 271, 97, 82, 125, 62, 204, 152, 68, 268, 63, 299, 338, 175, 343, 354, 307, 4, 330, 245, 121, 283, 1, 59, 347, 240, 175, 343, 15, 164, 55, 255, 180, 200, 93, 37, 315, 359, 50, 162, 342, 48, 113, 337, 377, 92, 44, 373, 72, 312, 54, 122, 157, 46, 378, 249, 289, 243, 259, 323, 225, 115, 228, 304, 202, 272, 265, 369, 47, 349, 70, 190, 273, 153, 79, 159, 195, 60, 182, 229, 118, 11, 168, 208, 186, 58, 40, 69, 317, 172, 279, 182, 265, 369, 294, 285, 210, 237, 380, 196, 194, 3, 332, 257, 7, 142, 298, 324, 17, 117, 178, 83, 101, 151, 277, 346, 42, 7, 142, 298, 183, 262, 264, 9, 155, 22, 320, 381, 57, 135, 7, 229, 362, 313, 29, 177, 130, 102, 65, 75, 85, 127, 376, 274, 28, 362, 14, 74, 191, 321, 353, 38, 126, 334, 45, 371, 274, 375, 71, 281, 23, 383, 107, 322, 366, 166, 367, 144, 76, 274, 375, 99, 73, 119, 221, 19, 137, 52, 269, 78, 242, 171, 238, 187, 253, 0, 284, 167, 32, 211, 120, 318, 128, 246, 357, 258, 176, 139, 244, 356, 206, 129, 293, 345, 209, 80, 352, 365, 275, 292, 25, 181, 148, 213, 358, 305, 116, 35, 43, 66, 276, 275, 260, 18, 77, 235, 214, 26, 51, 146, 286, 111, 12, 326, 336, 292, 201, 174, 374, 98, 24, 301, 87, 216, 252, 150, 160, 192, 49, 25, 131, 2, 224, 109, 141, 261, 287, 173, 165, 145, 192, 292, 103, 256, 360, 90, 140, 67, 309, 36, 96, 282, 34, 308, 49, 103, 94, 288, 219, 185, 86, 247, 112, 123, 138, 156, 108, 49, 105, 370, 325, 132, 311, 226, 8, 169, 231, 197, 236, 263, 41, 18, 104, 319, 300, 368, 310, 149, 363, 88, 199, 355, 263, 350, 280, 61, 296, 333, 189, 306, 291, 33, 10, 331, 91, 64, 133, 201, 27, 250, 241, 234, 232, 203, 222, 303, 230, 348, 64, 41, 105, 114, 339, 344, 89, 56, 227, 207, 39, 205, 30, 361, 239, 13, 302, 270, 170, 136, 6, 251, 218, 379, 290, 179] + 8: [273, 62, 122, 195, 22, 303, 125, 28, 126, 85, 220, 43, 289, 253, 10, 281, 374, 128, 183, 91, 117, 136, 35, 365, 274, 256, 253, 10, 281, 39, 166, 194, 358, 383, 266, 89, 287, 94, 288, 253, 243, 182, 364, 188, 112, 72, 217, 299, 9, 315, 301, 345, 104, 241, 74, 168, 69, 309, 60, 206, 382, 295, 323, 82, 222, 214, 321, 182, 377, 75, 167, 332, 98, 155, 235, 193, 363, 63, 5, 336, 276, 115, 107, 368, 61, 187, 143, 79, 357, 139, 124, 5, 321, 81, 156, 366, 278, 233, 71, 297, 46, 33, 203, 1, 250, 173, 36, 157, 226, 17, 120, 87, 327, 378, 324, 305, 49, 3, 312, 152, 131, 181, 198, 66, 263, 372, 159, 153, 53, 242, 279, 162, 229, 186, 119, 178, 320, 249, 47, 306, 277, 331, 176, 360, 12, 204, 20, 258, 40, 300, 90, 310, 29, 129, 329, 207, 360, 362, 371, 280, 346, 137, 114, 254, 37, 236, 24, 51, 163, 360, 362, 371, 86, 311, 196, 197, 174, 140, 105, 146, 142, 227, 360, 162, 152, 97, 172, 189, 8, 59, 68, 144, 158, 26, 96, 290, 13, 228, 223, 38, 78, 211, 191, 350, 54, 351, 138, 19, 290, 201, 337, 95, 34, 261, 318, 232, 151, 369, 208, 319, 234, 290, 201, 229, 252, 179, 202, 42, 45, 314, 339, 370, 354, 58, 246, 239, 204, 291, 231, 298, 4, 335, 190, 205, 15, 123, 102, 218, 239, 64, 148, 225, 50, 341, 116, 373, 322, 340, 245, 55, 292, 224, 221, 11, 83, 113, 177, 248, 149, 375, 77, 269, 170, 67, 264, 62, 251, 147, 169, 141, 237, 16, 200, 84, 160, 132, 67, 264, 221, 30, 240, 216, 48, 118, 56, 99, 326, 342, 127, 31, 7, 88, 349, 333, 92, 2, 212, 199, 265, 171, 70, 57, 31, 108, 154, 255, 283, 121, 164, 247, 184, 185, 271, 286, 0, 238, 367, 282, 380, 6, 330, 52, 230, 313, 325, 348, 343, 210, 135, 359, 353, 209, 23, 165, 267, 25, 32, 175, 244, 180, 100, 135, 161, 308, 328, 109, 65, 317, 293, 259, 14, 111, 80, 296, 338, 367, 381, 294, 150, 219, 192, 257, 352, 347, 103, 285, 213, 302, 359, 353, 316, 133, 275, 334, 73, 110, 93, 270, 268, 376, 106, 359, 308, 18, 356, 304, 284, 101, 41, 260, 76, 355, 262, 21, 21, 272, 44, 27, 215, 130, 379, 307, 145, 134, 344, 361] + 9: [212, 185, 287, 244, 158, 107, 323, 50, 60, 270, 167, 284, 25, 92, 47, 49, 95, 268, 228, 238, 339, 300, 375, 225, 240, 283, 92, 351, 159, 341, 169, 0, 181, 237, 2, 89, 347, 257, 250, 92, 381, 49, 95, 153, 231, 66, 274, 115, 221, 346, 40, 28, 92, 185, 262, 51, 120, 277, 141, 80, 261, 253, 326, 44, 61, 78, 47, 217, 10, 161, 87, 195, 109, 100, 162, 69, 73, 55, 304, 355, 210, 244, 360, 41, 186, 130, 379, 211, 178, 378, 265, 304, 355, 281, 64, 30, 321, 124, 39, 62, 84, 193, 93, 182, 304, 179, 329, 64, 144, 6, 230, 17, 45, 251, 383, 63, 369, 304, 184, 262, 147, 154, 306, 171, 5, 358, 305, 58, 118, 340, 166, 355, 380, 209, 187, 364, 36, 104, 79, 320, 368, 279, 348, 255, 327, 138, 67, 363, 183, 139, 276, 46, 330, 243, 266, 365, 255, 327, 370, 103, 135, 192, 207, 235, 117, 298, 86, 119, 24, 255, 327, 242, 271, 33, 334, 111, 288, 291, 229, 48, 113, 91, 349, 165, 316, 23, 3, 297, 332, 75, 213, 110, 290, 149, 27, 349, 222, 376, 147, 356, 128, 206, 246, 148, 134, 293, 278, 53, 349, 155, 146, 10, 137, 88, 218, 106, 114, 232, 227, 200, 157, 81, 74, 83, 56, 9, 129, 16, 319, 322, 286, 160, 328, 26, 143, 295, 127, 121, 302, 314, 123, 324, 190, 301, 68, 345, 173, 331, 219, 15, 177, 258, 99, 4, 248, 205, 76, 374, 273, 7, 220, 295, 316, 136, 289, 318, 189, 13, 197, 310, 170, 101, 236, 96, 350, 90, 214, 292, 54, 150, 308, 174, 12, 191, 249, 168, 315, 353, 313, 132, 82, 264, 72, 202, 112, 371, 140, 366, 234, 196, 357, 1, 14, 317, 180, 11, 359, 254, 224, 241, 85, 156, 31, 252, 29, 22, 98, 163, 35, 354, 373, 352, 43, 325, 57, 31, 299, 37, 102, 131, 145, 336, 208, 233, 71, 108, 176, 126, 280, 172, 198, 309, 367, 116, 142, 65, 52, 125, 188, 204, 122, 226, 263, 201, 282, 151, 97, 267, 175, 303, 21, 199, 294, 164, 226, 19, 344, 342, 285, 247, 70, 343, 203, 42, 34, 223, 77, 372, 263, 313, 32, 133, 259, 256, 8, 275, 333, 337, 296, 152, 260, 215, 90, 103, 377, 311, 105, 18, 382, 216, 59, 361, 338, 94, 94, 272, 239, 312, 20, 269, 307, 245, 38, 335, 194, 362] + 10: [294, 367, 2, 248, 293, 155, 129, 37, 193, 287, 217, 143, 160, 192, 126, 146, 47, 363, 231, 85, 328, 275, 226, 35, 347, 290, 255, 167, 331, 58, 67, 285, 21, 339, 261, 175, 16, 104, 366, 194, 279, 22, 182, 186, 249, 75, 51, 162, 173, 206, 150, 238, 194, 69, 31, 91, 76, 376, 81, 97, 247, 111, 372, 152, 110, 264, 286, 310, 241, 274, 280, 232, 357, 183, 298, 203, 94, 336, 264, 337, 102, 237, 316, 77, 314, 109, 340, 222, 127, 207, 176, 55, 18, 215, 177, 49, 359, 56, 329, 343, 370, 276, 118, 297, 55, 286, 57, 196, 208, 360, 27, 26, 317, 299, 165, 344, 82, 300, 365, 161, 117, 252, 199, 321, 20, 45, 169, 251, 83, 324, 300, 216, 54, 263, 136, 266, 105, 9, 95, 220, 133, 120, 29, 300, 216, 178, 171, 312, 0, 33, 267, 12, 375, 147, 156, 381, 300, 18, 73, 330, 377, 32, 214, 379, 174, 209, 53, 273, 68, 246, 365, 1, 114, 234, 229, 135, 378, 323, 364, 6, 334, 369, 180, 325, 146, 308, 353, 190, 335, 278, 42, 235, 187, 333, 23, 180, 365, 164, 201, 243, 90, 233, 258, 36, 80, 355, 277, 228, 93, 121, 225, 213, 115, 382, 341, 197, 123, 282, 245, 172, 358, 151, 289, 8, 66, 345, 271, 311, 25, 125, 100, 86, 352, 166, 151, 121, 92, 380, 7, 218, 288, 338, 239, 113, 224, 202, 70, 268, 262, 170, 40, 368, 61, 46, 256, 163, 244, 292, 63, 28, 265, 24, 195, 351, 348, 38, 219, 371, 318, 302, 119, 191, 236, 265, 24, 134, 153, 240, 132, 103, 15, 354, 62, 305, 227, 43, 137, 320, 179, 78, 130, 145, 72, 383, 184, 96, 205, 10, 242, 181, 320, 301, 138, 374, 212, 122, 230, 39, 131, 322, 30, 188, 319, 5, 310, 210, 284, 306, 106, 41, 254, 139, 373, 260, 270, 319, 5, 301, 19, 356, 223, 296, 327, 140, 326, 4, 157, 349, 319, 141, 144, 124, 342, 149, 3, 315, 13, 185, 34, 88, 283, 319, 320, 101, 304, 148, 98, 142, 74, 332, 159, 303, 14, 52, 319, 320, 79, 99, 204, 189, 60, 59, 50, 65, 17, 84, 89, 108, 5, 164, 269, 272, 198, 87, 346, 313, 257, 168, 361, 253, 48, 5, 309, 211, 259, 250, 107, 112, 362, 307, 158, 281, 295, 221, 5, 31, 154, 11, 128, 116, 71, 64, 350, 200, 44, 291] + 11: [156, 36, 236, 152, 367, 75, 101, 34, 324, 222, 201, 276, 137, 252, 11, 22, 247, 213, 127, 270, 279, 113, 46, 114, 180, 335, 98, 300, 144, 275, 45, 373, 208, 158, 164, 356, 240, 123, 281, 98, 11, 332, 192, 124, 2, 41, 295, 7, 74, 183, 60, 12, 77, 190, 218, 141, 319, 347, 255, 292, 142, 237, 277, 175, 105, 377, 233, 345, 331, 107, 262, 242, 63, 62, 8, 278, 16, 257, 377, 145, 0, 217, 283, 27, 186, 364, 383, 37, 4, 239, 321, 339, 121, 15, 284, 371, 93, 55, 312, 129, 103, 140, 202, 256, 339, 121, 15, 178, 66, 172, 289, 338, 67, 106, 322, 333, 273, 260, 171, 357, 204, 78, 166, 224, 193, 57, 184, 167, 51, 50, 299, 99, 100, 244, 320, 86, 341, 349, 64, 271, 272, 253, 308, 130, 235, 102, 165, 313, 151, 197, 211, 225, 358, 305, 352, 261, 130, 221, 357, 146, 154, 26, 198, 302, 120, 375, 132, 317, 191, 130, 221, 285, 330, 161, 56, 203, 311, 264, 5, 288, 76, 212, 88, 235, 126, 337, 169, 80, 254, 359, 293, 323, 287, 189, 232, 363, 38, 199, 39, 303, 10, 49, 366, 174, 19, 71, 216, 243, 87, 110, 162, 351, 170, 69, 81, 30, 43, 360, 68, 369, 134, 87, 326, 267, 52, 94, 251, 89, 265, 79, 274, 380, 111, 25, 227, 110, 157, 318, 207, 195, 96, 354, 291, 108, 214, 372, 72, 376, 110, 248, 131, 6, 374, 298, 59, 48, 266, 290, 28, 153, 90, 85, 44, 70, 196, 92, 259, 378, 187, 112, 310, 155, 329, 90, 85, 325, 286, 250, 350, 35, 336, 18, 138, 328, 344, 42, 314, 119, 116, 20, 13, 348, 301, 219, 263, 194, 268, 361, 9, 314, 150, 36, 228, 205, 370, 97, 147, 182, 143, 1, 238, 241, 314, 150, 116, 177, 17, 249, 327, 362, 188, 309, 379, 139, 307, 3, 149, 44, 73, 297, 160, 133, 91, 234, 210, 21, 206, 334, 83, 40, 285, 128, 47, 176, 14, 245, 33, 230, 229, 368, 269, 83, 65, 248, 95, 23, 340, 316, 179, 115, 200, 104, 181, 109, 215, 163, 29, 343, 346, 315, 365, 353, 125, 84, 173, 82, 220, 215, 163, 24, 54, 280, 226, 31, 135, 118, 306, 185, 122, 223, 296, 296, 126, 381, 136, 159, 209, 168, 258, 382, 342, 53, 61, 296, 163, 100, 282, 148, 117, 58, 294, 355, 231, 304, 246, 32] + 12: [156, 69, 98, 208, 288, 180, 52, 313, 257, 263, 212, 132, 353, 255, 289, 198, 260, 209, 107, 231, 232, 351, 285, 55, 94, 250, 255, 296, 151, 123, 371, 271, 315, 356, 40, 128, 237, 269, 366, 255, 378, 51, 259, 300, 104, 234, 321, 335, 30, 47, 145, 222, 255, 378, 361, 176, 101, 80, 376, 61, 334, 70, 316, 275, 359, 255, 69, 90, 75, 171, 129, 200, 138, 20, 207, 72, 196, 42, 204, 349, 369, 170, 39, 322, 0, 37, 235, 252, 8, 206, 21, 204, 177, 181, 340, 364, 164, 254, 67, 379, 85, 113, 248, 324, 204, 177, 306, 354, 133, 362, 114, 4, 121, 274, 131, 88, 92, 363, 82, 50, 295, 116, 197, 185, 262, 305, 279, 81, 143, 272, 49, 223, 290, 195, 160, 337, 253, 165, 5, 179, 125, 307, 302, 149, 223, 192, 3, 264, 261, 227, 29, 32, 365, 19, 7, 199, 149, 82, 287, 216, 89, 308, 77, 189, 183, 246, 236, 221, 99, 35, 124, 341, 6, 102, 294, 1, 62, 13, 312, 211, 111, 73, 48, 22, 203, 68, 10, 135, 188, 136, 154, 352, 18, 314, 141, 120, 109, 341, 205, 266, 23, 286, 319, 122, 146, 345, 311, 377, 120, 267, 46, 278, 2, 241, 350, 256, 65, 148, 117, 79, 225, 120, 292, 46, 281, 153, 273, 191, 280, 36, 217, 54, 118, 240, 381, 168, 172, 215, 147, 74, 155, 58, 11, 310, 93, 242, 373, 130, 127, 103, 332, 16, 326, 370, 45, 86, 186, 230, 184, 110, 130, 127, 203, 382, 27, 63, 9, 224, 293, 297, 320, 360, 43, 130, 175, 150, 330, 368, 318, 15, 187, 355, 44, 282, 163, 56, 245, 303, 97, 137, 277, 57, 41, 249, 159, 383, 347, 328, 331, 78, 167, 333, 276, 323, 157, 309, 139, 115, 106, 178, 342, 64, 78, 303, 126, 152, 218, 301, 142, 83, 336, 244, 367, 202, 66, 158, 327, 233, 134, 265, 380, 357, 119, 112, 284, 247, 26, 162, 158, 190, 150, 268, 229, 91, 201, 298, 140, 374, 299, 182, 372, 158, 190, 172, 348, 87, 59, 358, 325, 214, 251, 34, 210, 220, 219, 327, 333, 194, 25, 14, 144, 169, 38, 24, 96, 213, 95, 226, 344, 71, 346, 108, 31, 283, 173, 76, 105, 375, 228, 239, 226, 327, 243, 317, 60, 238, 193, 84, 12, 329, 258, 338, 53, 28, 17, 243, 339, 33, 291, 304, 161, 270, 174, 343, 100, 166] + 13: [80, 21, 310, 109, 40, 10, 344, 357, 127, 5, 85, 20, 314, 80, 135, 366, 171, 363, 2, 183, 76, 220, 119, 182, 156, 120, 9, 280, 186, 46, 266, 122, 327, 221, 345, 190, 205, 148, 367, 44, 172, 340, 162, 320, 178, 129, 380, 56, 104, 285, 200, 82, 44, 280, 12, 118, 16, 43, 121, 267, 133, 360, 191, 258, 204, 362, 282, 30, 307, 356, 211, 228, 337, 250, 70, 212, 264, 7, 362, 51, 12, 35, 202, 207, 283, 89, 325, 19, 1, 97, 324, 194, 41, 238, 176, 188, 339, 374, 105, 53, 270, 28, 278, 175, 329, 169, 110, 131, 246, 375, 239, 153, 257, 365, 167, 47, 351, 329, 24, 95, 227, 112, 321, 359, 331, 94, 13, 343, 272, 165, 137, 312, 274, 304, 143, 98, 253, 303, 311, 72, 215, 371, 350, 77, 312, 201, 198, 376, 79, 144, 326, 259, 61, 138, 302, 206, 170, 23, 30, 116, 289, 237, 225, 68, 348, 73, 101, 293, 27, 170, 39, 209, 323, 297, 240, 262, 248, 268, 313, 157, 317, 216, 170, 260, 342, 58, 75, 382, 29, 52, 305, 103, 233, 255, 322, 170, 260, 189, 295, 31, 355, 196, 288, 244, 38, 117, 332, 126, 170, 265, 231, 184, 54, 147, 107, 163, 308, 358, 63, 90, 140, 108, 328, 189, 111, 369, 141, 151, 37, 50, 115, 284, 349, 33, 108, 328, 100, 279, 273, 168, 25, 161, 381, 124, 372, 341, 301, 108, 23, 276, 49, 69, 179, 26, 286, 296, 235, 83, 354, 160, 57, 149, 238, 45, 364, 62, 361, 36, 59, 377, 60, 99, 155, 232, 87, 318, 335, 292, 373, 139, 222, 319, 249, 180, 181, 78, 232, 87, 318, 370, 158, 378, 177, 241, 174, 353, 185, 368, 193, 232, 149, 55, 64, 275, 219, 145, 379, 245, 234, 336, 132, 218, 81, 87, 330, 298, 334, 208, 146, 15, 84, 150, 226, 224, 192, 18, 195, 134, 88, 281, 6, 256, 125, 136, 214, 4, 290, 247, 18, 195, 271, 199, 91, 22, 74, 48, 113, 217, 203, 347, 32, 18, 195, 197, 11, 67, 299, 243, 164, 242, 287, 252, 251, 315, 3, 86, 102, 338, 294, 210, 93, 8, 71, 154, 128, 152, 230, 114, 14, 17, 142, 309, 65, 291, 106, 92, 130, 346, 236, 277, 114, 0, 274, 159, 261, 300, 333, 223, 263, 213, 383, 269, 66, 96, 14, 254, 306, 229, 42, 166, 34, 187, 123, 352, 173, 316] + 14: [147, 122, 85, 100, 297, 370, 377, 298, 273, 74, 47, 236, 331, 70, 34, 151, 52, 246, 319, 90, 276, 214, 78, 66, 247, 50, 70, 34, 141, 95, 123, 288, 221, 336, 10, 153, 185, 170, 98, 56, 186, 99, 124, 206, 68, 241, 0, 94, 278, 139, 11, 249, 160, 75, 254, 198, 25, 145, 286, 313, 248, 20, 263, 131, 29, 160, 239, 228, 32, 135, 326, 116, 359, 262, 365, 92, 144, 73, 160, 344, 23, 117, 158, 333, 172, 152, 224, 230, 31, 219, 318, 229, 209, 275, 97, 323, 324, 347, 330, 302, 127, 362, 188, 315, 229, 259, 235, 39, 149, 201, 373, 335, 43, 195, 57, 252, 83, 229, 259, 142, 342, 328, 291, 129, 86, 114, 338, 62, 283, 212, 229, 75, 155, 317, 33, 104, 109, 168, 310, 375, 120, 140, 289, 211, 2, 71, 231, 69, 184, 105, 284, 6, 103, 35, 320, 88, 211, 2, 71, 3, 163, 357, 237, 200, 106, 257, 84, 381, 165, 274, 209, 308, 213, 161, 193, 5, 316, 42, 36, 293, 307, 295, 143, 49, 275, 177, 343, 332, 380, 225, 285, 46, 55, 81, 121, 143, 268, 155, 371, 379, 159, 227, 300, 311, 309, 24, 181, 174, 261, 175, 378, 360, 125, 266, 222, 189, 350, 16, 299, 280, 38, 4, 30, 337, 271, 41, 40, 102, 148, 93, 14, 18, 51, 138, 4, 30, 118, 79, 208, 363, 101, 306, 173, 171, 367, 72, 196, 372, 112, 378, 269, 220, 382, 256, 191, 356, 58, 312, 303, 218, 207, 199, 146, 162, 59, 277, 82, 253, 28, 358, 180, 107, 205, 15, 110, 308, 64, 215, 369, 156, 176, 321, 304, 322, 340, 53, 15, 383, 355, 250, 216, 169, 128, 282, 240, 361, 45, 192, 272, 76, 17, 348, 154, 13, 301, 234, 115, 251, 136, 80, 91, 119, 76, 110, 355, 44, 26, 339, 226, 202, 376, 238, 132, 134, 292, 77, 255, 353, 111, 294, 346, 203, 264, 244, 19, 179, 27, 210, 164, 255, 126, 354, 166, 37, 178, 223, 63, 9, 258, 368, 232, 287, 255, 126, 61, 314, 167, 296, 89, 183, 190, 281, 54, 352, 204, 8, 327, 279, 270, 22, 48, 341, 329, 334, 87, 243, 351, 204, 60, 327, 374, 197, 108, 245, 242, 1, 290, 194, 65, 130, 345, 345, 150, 182, 157, 187, 325, 364, 96, 267, 366, 113, 67, 345, 137, 122, 21, 265, 133, 349, 12, 217, 305, 7, 233, 260] + 15: [68, 258, 128, 342, 357, 5, 229, 381, 383, 303, 293, 125, 260, 298, 258, 94, 168, 36, 372, 13, 377, 63, 116, 98, 299, 127, 0, 200, 104, 296, 100, 133, 77, 365, 224, 89, 126, 156, 249, 0, 200, 104, 113, 207, 49, 206, 294, 316, 54, 171, 304, 192, 0, 326, 266, 344, 335, 170, 10, 308, 37, 364, 292, 307, 368, 290, 231, 236, 353, 47, 78, 268, 61, 314, 188, 318, 287, 93, 15, 355, 175, 256, 338, 323, 363, 67, 211, 73, 140, 214, 27, 15, 355, 175, 311, 351, 209, 114, 137, 208, 129, 149, 297, 33, 15, 332, 217, 1, 352, 246, 43, 339, 84, 167, 181, 44, 46, 261, 155, 148, 173, 106, 238, 341, 30, 172, 190, 222, 183, 281, 340, 185, 252, 40, 164, 101, 233, 349, 337, 278, 330, 41, 245, 350, 155, 217, 174, 115, 212, 96, 53, 272, 153, 201, 336, 348, 76, 38, 119, 139, 242, 2, 62, 118, 92, 91, 380, 21, 25, 4, 189, 215, 23, 241, 105, 250, 107, 283, 210, 280, 243, 85, 88, 251, 90, 221, 109, 59, 69, 180, 26, 147, 370, 345, 186, 88, 324, 134, 83, 225, 28, 81, 141, 334, 3, 310, 286, 328, 88, 324, 134, 35, 235, 102, 282, 347, 64, 274, 327, 110, 356, 227, 300, 146, 151, 138, 142, 87, 99, 55, 184, 7, 70, 264, 227, 251, 203, 376, 143, 117, 331, 97, 204, 374, 58, 346, 144, 259, 160, 9, 329, 196, 279, 205, 226, 373, 65, 124, 17, 19, 295, 160, 121, 360, 18, 86, 343, 289, 216, 32, 14, 122, 6, 202, 160, 128, 135, 379, 247, 12, 271, 257, 273, 228, 178, 75, 367, 39, 94, 193, 319, 182, 150, 72, 123, 161, 312, 220, 132, 130, 57, 284, 16, 177, 305, 165, 313, 358, 239, 29, 302, 131, 130, 362, 74, 56, 31, 42, 253, 191, 95, 52, 361, 103, 198, 130, 362, 333, 48, 24, 166, 50, 306, 375, 79, 262, 219, 315, 130, 199, 159, 163, 285, 194, 254, 382, 213, 240, 195, 154, 152, 11, 359, 51, 232, 255, 263, 269, 354, 71, 378, 112, 120, 325, 11, 359, 237, 80, 111, 248, 34, 267, 45, 369, 136, 223, 366, 11, 371, 20, 157, 320, 82, 309, 234, 244, 158, 187, 288, 276, 11, 57, 148, 277, 162, 145, 8, 301, 265, 218, 66, 179, 322, 291, 359, 60, 317, 230, 275, 22, 321, 169, 197, 176, 108, 270] + 16: [11, 90, 171, 383, 97, 159, 357, 242, 244, 284, 138, 12, 1, 162, 129, 255, 265, 52, 41, 334, 358, 48, 381, 32, 6, 98, 100, 64, 250, 364, 314, 184, 137, 221, 207, 157, 176, 173, 362, 100, 64, 24, 287, 306, 219, 297, 252, 266, 166, 241, 192, 10, 140, 269, 78, 209, 307, 66, 122, 60, 253, 107, 230, 23, 239, 140, 49, 302, 283, 2, 281, 19, 225, 360, 14, 371, 278, 161, 134, 144, 205, 235, 365, 8, 276, 158, 337, 110, 216, 204, 361, 134, 147, 75, 212, 335, 282, 356, 214, 91, 172, 382, 79, 3, 134, 147, 259, 123, 82, 80, 322, 228, 55, 324, 169, 355, 257, 134, 269, 280, 89, 115, 155, 202, 336, 190, 128, 245, 350, 368, 15, 165, 348, 236, 321, 182, 372, 340, 272, 17, 116, 263, 191, 15, 165, 5, 46, 310, 34, 96, 290, 363, 88, 217, 291, 345, 15, 86, 280, 108, 71, 39, 229, 211, 83, 145, 243, 40, 320, 15, 273, 118, 199, 328, 31, 232, 95, 304, 113, 256, 326, 13, 68, 338, 271, 87, 316, 73, 160, 264, 30, 186, 28, 63, 148, 286, 7, 224, 189, 177, 231, 293, 104, 201, 127, 279, 327, 69, 286, 43, 374, 213, 65, 70, 106, 25, 323, 131, 359, 72, 296, 286, 43, 35, 187, 294, 227, 370, 132, 343, 168, 50, 299, 215, 286, 277, 271, 270, 99, 58, 174, 354, 193, 62, 251, 74, 94, 369, 312, 274, 298, 27, 167, 124, 208, 267, 218, 260, 130, 111, 369, 7, 183, 121, 84, 305, 9, 198, 240, 373, 332, 197, 53, 152, 92, 342, 85, 237, 311, 379, 313, 109, 163, 288, 135, 126, 152, 92, 203, 346, 38, 56, 42, 18, 206, 117, 308, 222, 133, 67, 262, 75, 347, 21, 175, 22, 151, 142, 233, 352, 254, 247, 330, 262, 289, 226, 377, 119, 339, 200, 93, 47, 210, 112, 344, 125, 275, 318, 180, 51, 0, 45, 54, 223, 20, 149, 329, 309, 331, 275, 118, 16, 248, 153, 220, 178, 319, 26, 170, 179, 376, 185, 275, 303, 367, 37, 103, 238, 29, 378, 268, 150, 341, 366, 114, 234, 303, 246, 292, 139, 301, 375, 196, 315, 33, 164, 156, 154, 234, 143, 146, 59, 380, 36, 102, 105, 261, 101, 349, 249, 120, 120, 44, 295, 181, 195, 188, 57, 333, 258, 77, 136, 325, 120, 300, 44, 141, 285, 76, 4, 317, 194, 81, 61, 351, 353] + 17: [374, 15, 155, 56, 22, 314, 304, 44, 379, 233, 65, 322, 54, 261, 78, 225, 8, 177, 135, 13, 338, 240, 364, 227, 334, 358, 261, 15, 383, 52, 180, 99, 19, 101, 249, 50, 355, 68, 156, 28, 357, 383, 104, 313, 298, 41, 216, 345, 106, 176, 242, 235, 212, 254, 117, 107, 243, 84, 20, 37, 94, 344, 83, 129, 152, 138, 219, 311, 157, 335, 9, 82, 95, 120, 372, 23, 285, 159, 131, 375, 100, 222, 46, 171, 61, 253, 238, 350, 352, 268, 34, 131, 375, 100, 290, 194, 18, 147, 221, 102, 369, 356, 368, 312, 131, 375, 26, 234, 273, 98, 336, 196, 195, 67, 182, 289, 127, 131, 91, 136, 339, 38, 139, 43, 365, 112, 128, 109, 184, 251, 220, 200, 26, 167, 206, 292, 47, 181, 179, 31, 32, 348, 69, 220, 200, 359, 172, 97, 165, 191, 342, 16, 64, 223, 349, 354, 119, 108, 185, 324, 231, 146, 340, 27, 300, 63, 53, 35, 123, 119, 108, 343, 2, 211, 256, 260, 115, 258, 25, 280, 51, 174, 119, 87, 362, 347, 381, 74, 332, 265, 161, 48, 40, 288, 202, 119, 12, 305, 328, 373, 10, 7, 204, 210, 55, 230, 11, 86, 307, 366, 279, 246, 296, 70, 36, 89, 110, 57, 163, 281, 209, 80, 126, 323, 113, 39, 245, 60, 341, 239, 326, 88, 306, 170, 105, 58, 114, 130, 96, 168, 116, 309, 360, 144, 164, 321, 252, 208, 337, 266, 1, 134, 142, 166, 271, 361, 270, 316, 17, 333, 250, 337, 294, 29, 218, 263, 331, 3, 85, 267, 224, 4, 237, 250, 337, 362, 125, 145, 33, 248, 21, 262, 150, 132, 77, 14, 140, 79, 294, 351, 148, 327, 173, 382, 287, 274, 201, 160, 72, 188, 79, 305, 62, 92, 143, 363, 81, 244, 76, 241, 353, 121, 188, 79, 232, 189, 24, 282, 264, 213, 310, 255, 315, 0, 301, 284, 319, 330, 6, 228, 371, 295, 49, 302, 318, 153, 247, 122, 284, 303, 286, 297, 380, 272, 42, 198, 299, 124, 378, 111, 293, 284, 303, 330, 133, 103, 367, 154, 118, 90, 278, 277, 291, 283, 284, 346, 193, 75, 207, 259, 215, 187, 93, 192, 141, 178, 203, 236, 226, 5, 276, 199, 214, 45, 205, 217, 151, 30, 229, 320, 236, 137, 197, 175, 73, 162, 59, 186, 183, 308, 66, 257, 325, 236, 190, 275, 370, 317, 376, 269, 377, 329, 158, 149, 71, 169] + 18: [208, 36, 106, 330, 181, 223, 122, 363, 383, 219, 364, 198, 379, 208, 304, 228, 99, 337, 307, 176, 264, 214, 224, 273, 24, 147, 6, 114, 228, 33, 103, 0, 288, 255, 275, 56, 2, 194, 91, 118, 119, 71, 378, 164, 225, 313, 335, 60, 338, 280, 345, 309, 356, 57, 71, 341, 318, 262, 232, 13, 166, 163, 172, 173, 52, 30, 245, 260, 190, 165, 131, 195, 244, 254, 197, 354, 206, 74, 30, 245, 260, 77, 70, 129, 157, 362, 234, 64, 370, 252, 80, 30, 65, 185, 326, 240, 135, 231, 145, 168, 332, 294, 18, 222, 30, 65, 368, 285, 274, 316, 302, 182, 175, 216, 161, 117, 227, 30, 215, 138, 246, 41, 331, 349, 205, 94, 123, 372, 221, 128, 30, 57, 138, 144, 346, 343, 350, 267, 108, 247, 180, 295, 186, 47, 300, 106, 3, 124, 22, 328, 132, 303, 259, 312, 310, 226, 47, 300, 185, 67, 204, 367, 235, 9, 287, 375, 37, 5, 344, 10, 28, 174, 297, 365, 79, 278, 329, 82, 170, 238, 160, 242, 10, 347, 110, 35, 89, 233, 201, 40, 48, 158, 299, 212, 268, 272, 113, 192, 266, 269, 104, 111, 38, 322, 381, 44, 100, 382, 43, 257, 116, 253, 19, 88, 361, 155, 92, 126, 95, 336, 97, 43, 154, 380, 105, 188, 296, 58, 63, 324, 229, 352, 277, 293, 271, 286, 26, 357, 171, 87, 7, 292, 143, 200, 281, 249, 32, 271, 320, 68, 291, 371, 283, 203, 279, 148, 17, 90, 305, 59, 271, 320, 321, 31, 20, 149, 289, 12, 243, 209, 115, 11, 183, 271, 207, 301, 376, 141, 334, 102, 81, 112, 130, 107, 62, 49, 271, 315, 25, 139, 23, 109, 39, 276, 179, 311, 258, 134, 55, 271, 217, 142, 151, 46, 121, 15, 54, 270, 133, 84, 21, 120, 271, 217, 236, 284, 366, 83, 152, 319, 256, 66, 75, 199, 156, 271, 257, 239, 282, 191, 177, 308, 125, 351, 76, 86, 72, 327, 193, 69, 167, 373, 73, 162, 50, 93, 127, 220, 29, 377, 340, 45, 251, 146, 360, 248, 241, 61, 96, 348, 4, 218, 323, 374, 53, 69, 355, 237, 342, 261, 159, 187, 213, 27, 42, 290, 314, 333, 251, 136, 34, 339, 263, 8, 150, 325, 317, 184, 14, 78, 16, 211, 189, 306, 202, 359, 140, 178, 153, 369, 85, 230, 1, 210, 251, 169, 101, 98, 137, 250, 265, 358, 51, 298, 196, 353] + 19: [187, 279, 300, 159, 53, 6, 306, 80, 242, 247, 201, 102, 196, 187, 180, 312, 35, 365, 167, 347, 3, 140, 37, 124, 282, 50, 187, 180, 8, 79, 46, 83, 249, 237, 168, 14, 290, 151, 250, 141, 126, 101, 58, 147, 182, 215, 10, 119, 63, 269, 43, 145, 243, 183, 207, 153, 135, 156, 345, 356, 200, 324, 224, 0, 316, 190, 313, 216, 354, 246, 172, 277, 331, 362, 59, 210, 360, 81, 190, 313, 373, 117, 335, 121, 260, 129, 82, 232, 51, 205, 355, 190, 322, 197, 275, 238, 76, 157, 65, 298, 28, 112, 327, 245, 211, 291, 307, 33, 1, 195, 325, 213, 32, 370, 136, 344, 375, 366, 330, 115, 334, 36, 16, 95, 108, 137, 181, 214, 163, 254, 366, 330, 115, 296, 116, 49, 244, 333, 343, 99, 150, 382, 263, 366, 161, 307, 38, 310, 283, 336, 380, 139, 105, 231, 55, 276, 24, 288, 363, 154, 134, 204, 194, 372, 326, 351, 252, 5, 378, 24, 288, 27, 218, 166, 221, 125, 148, 191, 361, 304, 56, 93, 24, 338, 184, 303, 278, 272, 273, 47, 309, 103, 348, 78, 109, 24, 338, 113, 164, 295, 170, 346, 23, 176, 174, 357, 104, 229, 339, 288, 328, 20, 352, 239, 287, 45, 341, 235, 106, 72, 305, 111, 188, 323, 193, 128, 257, 18, 19, 209, 208, 118, 199, 317, 179, 255, 66, 349, 222, 138, 84, 177, 22, 186, 367, 236, 350, 206, 70, 64, 77, 13, 29, 2, 26, 321, 94, 289, 69, 175, 48, 120, 312, 189, 171, 173, 4, 202, 212, 234, 381, 220, 12, 256, 91, 101, 264, 158, 270, 67, 75, 281, 294, 123, 248, 98, 256, 120, 219, 97, 285, 251, 40, 62, 383, 149, 162, 261, 107, 74, 52, 240, 89, 71, 96, 144, 160, 376, 315, 178, 308, 133, 7, 377, 114, 301, 122, 127, 143, 60, 225, 73, 267, 17, 142, 7, 377, 64, 227, 30, 368, 374, 131, 130, 340, 358, 169, 274, 7, 241, 240, 230, 15, 165, 110, 284, 87, 258, 100, 299, 369, 302, 364, 207, 271, 233, 353, 259, 262, 311, 332, 11, 226, 34, 265, 57, 318, 9, 292, 68, 297, 280, 329, 371, 21, 155, 319, 265, 57, 42, 152, 54, 359, 203, 293, 88, 286, 185, 379, 198, 90, 90, 192, 44, 41, 337, 31, 85, 320, 132, 39, 86, 253, 90, 61, 42, 217, 25, 228, 342, 92, 146, 314, 223, 266, 268] + 20: [116, 268, 138, 91, 342, 265, 178, 73, 191, 85, 136, 171, 280, 116, 268, 330, 10, 347, 343, 60, 12, 308, 199, 245, 31, 27, 116, 145, 185, 221, 104, 267, 346, 338, 139, 28, 16, 220, 331, 116, 145, 270, 141, 101, 219, 18, 106, 75, 289, 70, 251, 350, 334, 45, 93, 209, 172, 35, 274, 332, 94, 352, 147, 275, 262, 334, 45, 325, 158, 65, 218, 345, 5, 217, 21, 133, 301, 263, 334, 226, 188, 229, 132, 53, 208, 304, 84, 296, 234, 244, 167, 8, 112, 318, 228, 216, 41, 42, 127, 230, 79, 293, 1, 117, 122, 369, 162, 164, 49, 361, 215, 370, 11, 205, 211, 197, 51, 37, 369, 4, 286, 177, 359, 34, 294, 151, 355, 19, 23, 181, 278, 43, 339, 135, 13, 213, 114, 248, 349, 323, 354, 17, 198, 278, 43, 379, 160, 176, 367, 196, 310, 316, 40, 63, 103, 189, 174, 2, 32, 273, 77, 240, 169, 39, 374, 163, 47, 38, 241, 302, 87, 46, 264, 126, 194, 97, 33, 212, 102, 225, 269, 52, 302, 87, 282, 96, 90, 26, 238, 78, 337, 329, 305, 288, 54, 336, 287, 353, 311, 121, 348, 165, 377, 285, 271, 83, 152, 61, 336, 376, 335, 140, 190, 36, 300, 170, 24, 113, 157, 129, 200, 272, 99, 105, 125, 320, 146, 247, 231, 360, 356, 207, 186, 206, 14, 99, 4, 227, 57, 58, 22, 100, 154, 156, 295, 155, 76, 161, 193, 204, 128, 184, 0, 309, 257, 89, 7, 67, 44, 242, 161, 110, 46, 187, 322, 25, 150, 236, 351, 115, 373, 69, 249, 161, 123, 130, 362, 357, 375, 131, 29, 210, 333, 276, 203, 107, 161, 153, 307, 259, 239, 368, 233, 202, 327, 124, 48, 149, 299, 161, 153, 291, 50, 6, 314, 279, 365, 253, 321, 64, 192, 303, 161, 153, 335, 180, 74, 30, 119, 380, 254, 142, 183, 344, 281, 317, 193, 363, 95, 92, 235, 243, 15, 56, 120, 134, 237, 137, 55, 98, 224, 381, 358, 284, 340, 20, 80, 319, 261, 255, 168, 82, 98, 282, 9, 315, 175, 256, 232, 223, 88, 313, 298, 179, 82, 98, 195, 382, 182, 328, 166, 252, 159, 59, 108, 306, 266, 144, 81, 371, 324, 258, 118, 372, 3, 312, 250, 383, 109, 260, 144, 173, 62, 290, 71, 364, 246, 201, 326, 283, 68, 222, 214, 66, 66, 148, 111, 292, 378, 86, 297, 366, 341, 72, 143, 277] + 21: [152, 376, 345, 0, 121, 338, 60, 125, 258, 305, 114, 242, 35, 152, 376, 377, 150, 344, 18, 359, 65, 24, 83, 74, 32, 322, 152, 119, 226, 118, 194, 168, 304, 5, 146, 334, 76, 31, 127, 302, 124, 277, 178, 225, 331, 138, 155, 346, 134, 122, 59, 343, 316, 200, 92, 128, 23, 245, 28, 77, 203, 271, 130, 61, 101, 316, 124, 119, 147, 220, 33, 192, 275, 198, 181, 129, 142, 167, 20, 210, 140, 380, 135, 229, 98, 165, 126, 333, 375, 265, 58, 20, 82, 218, 349, 367, 373, 383, 193, 14, 96, 374, 116, 107, 289, 103, 117, 240, 219, 131, 62, 323, 186, 381, 217, 145, 283, 371, 103, 88, 238, 141, 288, 286, 341, 187, 315, 357, 161, 297, 371, 103, 169, 320, 209, 270, 21, 154, 274, 110, 201, 212, 311, 308, 50, 45, 182, 366, 63, 309, 133, 329, 195, 207, 222, 327, 308, 38, 232, 284, 8, 233, 97, 353, 79, 382, 55, 255, 321, 153, 80, 214, 252, 25, 73, 318, 290, 216, 273, 324, 285, 93, 355, 132, 348, 250, 196, 372, 223, 378, 335, 81, 37, 176, 264, 355, 53, 361, 149, 175, 352, 39, 237, 340, 319, 13, 326, 108, 355, 80, 199, 177, 299, 84, 139, 211, 276, 213, 56, 249, 306, 190, 158, 94, 279, 144, 325, 260, 115, 337, 241, 143, 166, 224, 254, 132, 113, 16, 293, 49, 362, 204, 102, 253, 156, 188, 272, 254, 132, 208, 22, 9, 263, 246, 354, 36, 215, 180, 328, 350, 256, 158, 257, 365, 221, 368, 291, 40, 136, 164, 70, 278, 332, 4, 29, 363, 51, 69, 336, 64, 317, 300, 157, 85, 296, 339, 4, 86, 140, 191, 52, 1, 259, 3, 170, 12, 239, 231, 11, 236, 360, 120, 91, 15, 174, 251, 30, 269, 67, 268, 282, 42, 46, 360, 113, 307, 43, 197, 262, 347, 261, 379, 54, 162, 123, 47, 205, 364, 184, 280, 358, 287, 295, 57, 27, 99, 202, 10, 47, 360, 185, 26, 111, 183, 356, 301, 104, 342, 41, 7, 248, 47, 360, 87, 151, 227, 267, 90, 370, 48, 244, 314, 228, 17, 247, 281, 112, 172, 189, 78, 303, 266, 292, 310, 330, 95, 235, 173, 206, 257, 351, 72, 234, 159, 100, 137, 294, 75, 298, 44, 173, 281, 71, 19, 230, 89, 66, 148, 34, 105, 2, 6, 179, 173, 281, 71, 109, 313, 243, 312, 171, 160, 163, 68, 106, 369] + 22: [371, 288, 112, 317, 161, 366, 233, 241, 279, 186, 150, 146, 360, 254, 337, 118, 58, 60, 361, 98, 155, 46, 182, 344, 167, 135, 254, 164, 169, 240, 174, 82, 131, 180, 262, 246, 5, 140, 143, 254, 242, 139, 130, 195, 40, 105, 256, 202, 30, 257, 231, 154, 254, 288, 230, 153, 264, 347, 380, 377, 96, 219, 162, 66, 292, 316, 337, 263, 55, 76, 191, 99, 335, 215, 334, 255, 277, 247, 281, 224, 177, 50, 67, 258, 47, 322, 265, 137, 291, 203, 2, 281, 224, 74, 352, 83, 156, 71, 45, 300, 134, 19, 243, 28, 33, 38, 110, 345, 222, 175, 133, 373, 138, 152, 296, 295, 250, 216, 115, 73, 188, 80, 63, 320, 269, 26, 274, 127, 284, 253, 216, 42, 183, 111, 100, 18, 248, 22, 356, 196, 104, 368, 79, 204, 207, 201, 218, 306, 374, 87, 65, 103, 35, 223, 84, 142, 51, 132, 183, 285, 8, 358, 341, 54, 364, 294, 168, 56, 313, 198, 147, 4, 252, 68, 48, 309, 81, 77, 85, 354, 114, 192, 9, 328, 144, 229, 106, 109, 41, 238, 271, 62, 171, 121, 325, 126, 328, 37, 348, 372, 340, 15, 0, 314, 129, 302, 166, 10, 343, 276, 94, 381, 116, 206, 72, 11, 97, 36, 69, 293, 95, 343, 328, 184, 338, 375, 220, 39, 236, 237, 187, 88, 304, 359, 332, 330, 94, 272, 17, 226, 225, 78, 211, 179, 308, 172, 205, 278, 261, 376, 7, 123, 53, 321, 382, 136, 266, 23, 21, 323, 278, 261, 110, 357, 259, 122, 59, 299, 13, 145, 128, 158, 157, 331, 261, 178, 70, 125, 57, 319, 199, 379, 282, 212, 163, 217, 209, 24, 178, 149, 365, 113, 1, 124, 273, 52, 342, 234, 267, 209, 24, 376, 351, 193, 349, 210, 327, 290, 190, 31, 197, 307, 181, 141, 93, 383, 214, 228, 12, 227, 353, 120, 16, 86, 280, 181, 148, 20, 251, 173, 25, 159, 301, 298, 151, 90, 107, 315, 181, 148, 20, 311, 336, 49, 305, 303, 208, 232, 326, 43, 101, 221, 141, 329, 333, 378, 3, 165, 32, 286, 91, 64, 194, 287, 221, 141, 29, 362, 312, 339, 324, 268, 108, 119, 370, 270, 289, 221, 141, 235, 117, 369, 170, 367, 283, 89, 160, 6, 213, 346, 34, 260, 44, 355, 185, 75, 239, 189, 275, 92, 350, 297, 310, 260, 260, 244, 61, 27, 14, 318, 200, 176, 102, 363, 245, 249] + 23: [26, 87, 84, 192, 91, 38, 52, 284, 243, 239, 360, 343, 168, 26, 269, 94, 374, 191, 224, 93, 258, 7, 373, 350, 83, 4, 26, 244, 156, 169, 249, 194, 1, 339, 68, 33, 266, 63, 75, 26, 244, 232, 241, 151, 342, 125, 250, 268, 25, 116, 22, 37, 126, 240, 176, 180, 271, 311, 53, 223, 99, 260, 62, 332, 234, 126, 306, 261, 11, 233, 272, 254, 58, 107, 46, 34, 145, 226, 328, 173, 27, 372, 220, 313, 333, 347, 285, 79, 310, 60, 166, 328, 24, 154, 55, 70, 152, 5, 162, 294, 205, 158, 112, 18, 281, 167, 47, 59, 213, 290, 329, 143, 245, 61, 88, 183, 216, 225, 167, 103, 127, 36, 237, 301, 330, 349, 214, 319, 217, 366, 264, 187, 177, 182, 312, 228, 365, 73, 335, 202, 230, 246, 15, 264, 82, 327, 86, 190, 157, 207, 303, 14, 336, 299, 118, 64, 124, 201, 287, 67, 149, 136, 98, 54, 21, 227, 238, 30, 291, 124, 331, 135, 2, 379, 356, 119, 344, 134, 355, 337, 278, 8, 124, 187, 324, 6, 340, 370, 178, 204, 155, 123, 181, 138, 265, 51, 295, 135, 351, 309, 222, 321, 352, 122, 97, 381, 141, 81, 175, 305, 85, 263, 184, 23, 150, 200, 28, 49, 383, 105, 3, 175, 305, 85, 283, 144, 215, 65, 341, 362, 308, 353, 293, 251, 69, 305, 252, 371, 102, 317, 137, 382, 236, 106, 357, 142, 186, 172, 48, 12, 257, 74, 95, 325, 316, 297, 369, 377, 42, 39, 172, 132, 323, 78, 219, 195, 96, 109, 16, 161, 253, 380, 13, 172, 132, 323, 130, 111, 221, 160, 363, 206, 248, 9, 262, 113, 314, 100, 197, 148, 131, 44, 153, 128, 302, 282, 322, 117, 189, 314, 100, 300, 218, 146, 129, 273, 90, 174, 367, 247, 318, 242, 314, 19, 72, 56, 35, 359, 17, 348, 163, 375, 259, 20, 235, 314, 77, 203, 41, 326, 210, 185, 280, 346, 188, 368, 198, 296, 275, 211, 358, 57, 378, 10, 32, 0, 108, 104, 276, 89, 29, 275, 147, 279, 270, 133, 361, 179, 193, 354, 139, 165, 289, 286, 275, 170, 300, 159, 209, 101, 307, 196, 121, 338, 171, 364, 288, 274, 376, 177, 304, 229, 40, 298, 256, 164, 140, 277, 199, 110, 120, 71, 72, 315, 80, 292, 231, 76, 114, 115, 334, 208, 43, 71, 71, 252, 92, 50, 255, 66, 45, 212, 267, 31, 345, 320] + 24: [0, 105, 104, 343, 356, 341, 280, 170, 271, 238, 248, 364, 368, 0, 70, 5, 135, 299, 338, 268, 203, 276, 317, 25, 371, 74, 290, 357, 128, 344, 274, 205, 295, 292, 126, 148, 143, 316, 298, 97, 357, 121, 192, 131, 345, 244, 30, 212, 67, 378, 340, 314, 52, 381, 189, 35, 355, 12, 313, 103, 196, 191, 43, 124, 353, 52, 28, 369, 306, 336, 376, 153, 258, 303, 288, 183, 322, 127, 87, 360, 324, 15, 96, 287, 233, 66, 284, 50, 101, 33, 152, 92, 139, 286, 277, 318, 307, 304, 194, 76, 99, 240, 17, 219, 92, 157, 361, 54, 177, 297, 13, 60, 151, 136, 250, 40, 358, 211, 185, 266, 204, 325, 293, 56, 37, 23, 291, 180, 69, 2, 110, 337, 85, 382, 39, 140, 178, 267, 348, 342, 362, 117, 45, 9, 223, 189, 144, 142, 310, 116, 181, 86, 373, 332, 100, 34, 9, 263, 85, 91, 149, 321, 235, 72, 38, 44, 109, 261, 374, 9, 263, 27, 190, 46, 58, 84, 218, 176, 217, 184, 200, 166, 246, 122, 128, 195, 260, 224, 1, 108, 173, 102, 18, 167, 81, 246, 223, 70, 366, 73, 114, 16, 215, 107, 42, 228, 214, 155, 246, 223, 369, 61, 351, 59, 147, 275, 150, 171, 367, 349, 232, 111, 210, 112, 354, 41, 245, 47, 312, 94, 281, 300, 82, 89, 154, 372, 220, 24, 93, 221, 168, 335, 14, 160, 229, 123, 129, 154, 372, 334, 26, 350, 279, 269, 365, 88, 163, 187, 141, 383, 154, 172, 294, 31, 226, 77, 222, 225, 161, 8, 174, 208, 36, 154, 259, 64, 164, 71, 326, 216, 333, 79, 209, 68, 199, 115, 265, 252, 230, 319, 130, 118, 242, 239, 188, 19, 201, 347, 379, 265, 7, 301, 137, 138, 83, 257, 282, 10, 57, 98, 285, 241, 265, 7, 255, 283, 113, 380, 11, 270, 175, 331, 162, 289, 264, 265, 7, 29, 231, 377, 182, 315, 134, 363, 193, 132, 305, 21, 95, 252, 207, 237, 49, 272, 197, 302, 249, 359, 247, 90, 133, 6, 252, 146, 53, 156, 48, 323, 309, 120, 55, 256, 243, 158, 6, 252, 186, 78, 308, 145, 51, 65, 32, 352, 75, 330, 179, 20, 22, 370, 213, 254, 206, 278, 339, 320, 262, 80, 119, 251, 20, 22, 375, 202, 106, 3, 311, 346, 253, 329, 125, 165, 327, 20, 328, 62, 159, 273, 63, 169, 4, 296, 234, 236, 227, 198] + 25: [315, 107, 284, 154, 197, 211, 316, 355, 17, 167, 322, 198, 194, 315, 107, 284, 136, 234, 225, 350, 116, 275, 183, 41, 217, 220, 315, 2, 29, 302, 306, 336, 132, 133, 110, 34, 155, 374, 263, 315, 25, 29, 120, 256, 218, 290, 134, 271, 157, 224, 26, 121, 315, 214, 309, 287, 373, 347, 295, 58, 248, 177, 119, 141, 267, 315, 214, 0, 18, 209, 348, 130, 370, 74, 320, 188, 285, 277, 269, 23, 376, 368, 153, 97, 357, 69, 223, 70, 64, 108, 43, 269, 23, 77, 230, 99, 279, 184, 365, 163, 22, 1, 360, 158, 269, 372, 327, 381, 341, 143, 187, 11, 238, 173, 293, 241, 106, 269, 372, 192, 207, 56, 261, 37, 328, 51, 292, 264, 363, 123, 269, 39, 52, 294, 252, 115, 304, 274, 243, 362, 142, 42, 195, 98, 61, 73, 318, 240, 321, 259, 201, 127, 86, 349, 90, 165, 139, 162, 232, 296, 89, 135, 330, 14, 193, 124, 174, 301, 239, 139, 332, 62, 342, 84, 359, 345, 24, 300, 208, 40, 47, 13, 65, 78, 333, 59, 76, 343, 15, 202, 109, 35, 48, 71, 129, 228, 276, 5, 102, 323, 196, 312, 31, 53, 96, 46, 148, 170, 228, 276, 190, 33, 54, 140, 68, 150, 159, 226, 175, 50, 6, 83, 375, 145, 380, 80, 227, 308, 257, 166, 250, 137, 12, 151, 258, 30, 7, 305, 366, 114, 221, 212, 200, 311, 371, 182, 206, 247, 180, 160, 8, 126, 291, 199, 339, 353, 281, 205, 57, 178, 247, 180, 273, 313, 367, 20, 319, 67, 251, 87, 16, 329, 255, 168, 282, 235, 66, 60, 369, 278, 331, 186, 36, 265, 88, 44, 268, 282, 280, 38, 138, 103, 55, 144, 249, 283, 118, 169, 303, 298, 282, 112, 378, 231, 161, 185, 28, 337, 93, 3, 335, 156, 379, 72, 113, 131, 215, 352, 203, 149, 340, 45, 128, 219, 382, 379, 72, 246, 100, 9, 191, 237, 310, 361, 210, 236, 272, 19, 379, 72, 62, 351, 253, 307, 213, 297, 111, 117, 63, 286, 326, 379, 101, 152, 346, 222, 242, 81, 254, 260, 75, 95, 172, 233, 379, 282, 192, 262, 377, 4, 383, 356, 79, 364, 289, 266, 270, 334, 317, 10, 32, 244, 176, 354, 105, 27, 21, 245, 288, 338, 82, 317, 171, 314, 229, 164, 324, 94, 179, 325, 344, 147, 91, 299, 85, 49, 216, 125, 146, 204, 181, 358, 122, 92, 104, 189] + 26: [80, 291, 307, 369, 133, 186, 127, 383, 25, 64, 324, 58, 168, 80, 253, 266, 135, 247, 294, 65, 107, 364, 347, 270, 46, 139, 150, 291, 276, 92, 148, 24, 20, 123, 12, 109, 313, 95, 203, 320, 338, 16, 88, 190, 367, 290, 256, 329, 112, 206, 155, 213, 136, 181, 254, 110, 67, 268, 357, 177, 362, 334, 258, 222, 246, 136, 181, 224, 146, 31, 221, 66, 232, 125, 225, 208, 345, 261, 136, 184, 160, 328, 198, 152, 113, 231, 157, 275, 314, 288, 132, 136, 263, 149, 332, 128, 122, 285, 19, 117, 244, 163, 28, 111, 136, 70, 253, 55, 360, 99, 161, 187, 264, 77, 199, 310, 86, 5, 371, 279, 245, 323, 356, 73, 350, 7, 346, 358, 248, 260, 6, 236, 278, 227, 262, 79, 215, 144, 325, 317, 243, 147, 169, 6, 21, 308, 223, 265, 57, 239, 93, 220, 8, 101, 164, 170, 233, 21, 38, 238, 26, 229, 45, 267, 47, 141, 42, 188, 175, 292, 342, 341, 293, 321, 44, 202, 252, 121, 335, 377, 286, 299, 292, 236, 130, 355, 178, 306, 98, 115, 340, 185, 284, 235, 209, 281, 63, 337, 249, 151, 173, 17, 143, 140, 40, 373, 322, 1, 89, 53, 344, 22, 179, 327, 87, 41, 78, 9, 205, 379, 54, 89, 53, 33, 315, 191, 97, 195, 50, 189, 303, 240, 207, 309, 89, 214, 183, 158, 319, 2, 35, 76, 197, 34, 274, 210, 52, 380, 90, 316, 226, 156, 201, 318, 153, 105, 165, 145, 283, 351, 380, 90, 137, 61, 176, 102, 159, 370, 49, 72, 56, 295, 374, 11, 172, 69, 30, 166, 353, 378, 114, 336, 39, 29, 85, 234, 11, 90, 230, 167, 180, 343, 277, 129, 219, 297, 216, 257, 218, 269, 18, 289, 376, 287, 241, 192, 312, 298, 74, 119, 84, 68, 269, 172, 339, 131, 259, 174, 212, 32, 301, 331, 326, 273, 60, 269, 172, 339, 62, 48, 27, 10, 37, 302, 361, 13, 330, 375, 349, 18, 116, 108, 359, 51, 365, 363, 280, 333, 193, 381, 228, 271, 217, 94, 36, 138, 142, 0, 196, 368, 100, 154, 211, 242, 124, 217, 304, 83, 272, 103, 104, 372, 134, 59, 91, 366, 106, 124, 217, 162, 82, 204, 118, 15, 75, 352, 354, 14, 282, 348, 237, 237, 120, 382, 200, 194, 71, 182, 300, 43, 296, 23, 96, 237, 237, 230, 126, 311, 251, 81, 171, 3, 255, 4, 250, 305] + 27: [34, 269, 80, 179, 32, 155, 215, 147, 367, 319, 142, 194, 310, 234, 69, 354, 143, 332, 261, 274, 362, 116, 184, 62, 160, 53, 234, 204, 349, 213, 82, 162, 379, 199, 167, 259, 86, 170, 60, 81, 69, 80, 369, 278, 254, 172, 129, 35, 344, 108, 40, 378, 103, 71, 354, 336, 294, 133, 304, 44, 365, 223, 110, 165, 342, 103, 73, 211, 228, 232, 191, 268, 375, 33, 14, 198, 134, 243, 103, 73, 211, 75, 258, 317, 136, 347, 3, 313, 257, 11, 212, 103, 252, 376, 97, 83, 15, 114, 67, 288, 185, 68, 52, 244, 103, 252, 227, 207, 308, 161, 122, 124, 256, 8, 157, 19, 285, 301, 164, 366, 335, 72, 187, 5, 111, 123, 132, 345, 206, 17, 37, 168, 377, 348, 325, 370, 6, 66, 121, 293, 364, 226, 338, 37, 233, 292, 183, 102, 98, 218, 43, 275, 264, 41, 331, 220, 104, 168, 24, 224, 23, 7, 296, 106, 107, 346, 382, 56, 163, 145, 265, 131, 222, 78, 328, 246, 249, 29, 359, 372, 245, 176, 177, 300, 202, 55, 343, 27, 214, 248, 361, 154, 158, 311, 137, 177, 265, 181, 182, 77, 241, 327, 18, 153, 208, 195, 314, 146, 117, 90, 358, 251, 216, 25, 125, 38, 47, 65, 302, 298, 235, 219, 90, 341, 230, 196, 322, 10, 267, 151, 231, 283, 371, 93, 59, 90, 341, 112, 238, 337, 360, 45, 79, 281, 334, 57, 209, 279, 374, 227, 237, 156, 305, 127, 287, 130, 306, 87, 277, 200, 58, 350, 320, 290, 99, 201, 175, 299, 0, 144, 192, 92, 36, 4, 312, 26, 323, 149, 140, 91, 2, 217, 50, 150, 138, 247, 4, 312, 135, 85, 242, 356, 316, 307, 101, 96, 166, 169, 51, 225, 61, 197, 340, 381, 141, 126, 120, 95, 113, 94, 255, 39, 225, 61, 357, 260, 380, 193, 326, 239, 205, 152, 273, 210, 100, 225, 312, 30, 303, 76, 373, 368, 318, 250, 74, 352, 355, 236, 286, 263, 128, 190, 115, 42, 174, 20, 84, 22, 229, 295, 203, 286, 263, 349, 188, 89, 88, 291, 171, 13, 280, 189, 363, 63, 286, 70, 315, 21, 383, 284, 309, 9, 353, 186, 330, 148, 270, 333, 31, 30, 178, 266, 109, 271, 28, 321, 253, 289, 1, 240, 351, 12, 49, 48, 339, 297, 329, 276, 105, 139, 64, 221, 180, 12, 12, 119, 272, 324, 118, 54, 46, 159, 282, 16, 173, 262] + 28: [4, 233, 253, 137, 285, 308, 121, 81, 366, 330, 267, 65, 11, 355, 172, 113, 320, 135, 216, 118, 208, 24, 331, 275, 189, 40, 161, 162, 272, 75, 240, 43, 206, 143, 74, 149, 111, 45, 173, 161, 243, 374, 236, 220, 315, 282, 82, 15, 351, 47, 181, 115, 161, 243, 357, 108, 32, 316, 184, 64, 34, 117, 101, 202, 150, 161, 155, 37, 166, 163, 193, 335, 322, 169, 213, 123, 94, 190, 161, 195, 224, 3, 67, 273, 244, 327, 382, 144, 232, 360, 298, 312, 162, 7, 219, 289, 247, 134, 21, 104, 246, 8, 87, 180, 0, 302, 307, 191, 167, 19, 290, 159, 211, 225, 17, 309, 336, 248, 302, 321, 227, 160, 20, 256, 171, 334, 284, 72, 313, 338, 201, 97, 214, 368, 361, 182, 365, 98, 84, 39, 85, 124, 207, 239, 120, 321, 223, 130, 217, 46, 295, 73, 372, 373, 304, 54, 310, 176, 286, 116, 63, 125, 265, 175, 230, 292, 231, 164, 215, 196, 306, 197, 51, 383, 340, 260, 314, 151, 31, 192, 187, 257, 168, 255, 99, 228, 235, 367, 78, 23, 332, 287, 277, 126, 10, 168, 306, 99, 59, 90, 221, 179, 148, 12, 131, 303, 212, 18, 77, 255, 25, 158, 352, 56, 183, 305, 342, 323, 29, 371, 60, 250, 185, 241, 165, 100, 271, 264, 106, 296, 345, 347, 319, 353, 250, 185, 241, 5, 261, 209, 112, 381, 297, 119, 76, 141, 86, 250, 152, 286, 204, 129, 349, 14, 358, 294, 218, 174, 350, 156, 250, 226, 25, 42, 157, 33, 91, 69, 270, 38, 234, 186, 359, 344, 317, 197, 71, 88, 376, 27, 49, 377, 109, 139, 83, 132, 344, 185, 93, 58, 200, 105, 245, 61, 36, 266, 140, 102, 379, 22, 317, 122, 325, 339, 13, 142, 249, 154, 79, 259, 145, 138, 324, 203, 52, 95, 28, 238, 194, 6, 136, 364, 35, 291, 279, 283, 70, 52, 378, 337, 278, 178, 177, 66, 110, 146, 198, 311, 26, 328, 288, 346, 370, 380, 333, 205, 210, 251, 229, 262, 329, 26, 188, 103, 50, 299, 16, 222, 274, 170, 293, 276, 62, 147, 26, 70, 107, 128, 41, 114, 1, 44, 363, 348, 55, 53, 301, 92, 153, 30, 369, 242, 48, 300, 362, 258, 80, 237, 318, 343, 92, 153, 107, 199, 269, 268, 341, 375, 133, 89, 252, 254, 2, 356, 356, 30, 263, 326, 127, 68, 57, 280, 96, 281, 354, 9] + 29: [383, 186, 89, 370, 382, 232, 310, 157, 79, 17, 296, 1, 206, 197, 262, 255, 204, 207, 257, 104, 312, 42, 69, 41, 100, 176, 197, 74, 160, 216, 102, 139, 353, 348, 70, 121, 223, 330, 240, 73, 67, 255, 68, 341, 106, 101, 230, 220, 179, 358, 80, 66, 73, 67, 346, 145, 82, 128, 20, 359, 13, 374, 297, 372, 367, 271, 110, 89, 237, 319, 273, 380, 315, 56, 227, 133, 331, 285, 283, 33, 148, 112, 298, 228, 167, 46, 111, 334, 92, 130, 39, 14, 265, 23, 134, 364, 222, 267, 31, 84, 62, 109, 226, 355, 196, 362, 311, 180, 314, 171, 340, 126, 191, 154, 201, 8, 26, 196, 362, 311, 243, 248, 280, 269, 224, 299, 113, 251, 165, 284, 88, 339, 173, 159, 132, 225, 125, 238, 338, 253, 254, 21, 322, 58, 129, 23, 235, 50, 136, 3, 22, 9, 194, 164, 286, 108, 195, 304, 309, 188, 193, 178, 32, 316, 161, 187, 141, 81, 305, 118, 295, 320, 163, 210, 12, 151, 10, 53, 0, 30, 231, 249, 118, 295, 44, 63, 293, 116, 292, 169, 153, 307, 325, 291, 177, 118, 304, 236, 190, 278, 349, 142, 47, 199, 15, 252, 268, 274, 124, 203, 236, 48, 61, 347, 294, 83, 288, 270, 234, 260, 6, 365, 203, 60, 24, 357, 156, 150, 28, 209, 71, 96, 5, 363, 244, 198, 44, 275, 123, 182, 264, 135, 213, 99, 301, 75, 212, 244, 198, 282, 175, 350, 18, 97, 302, 103, 149, 45, 335, 143, 244, 333, 60, 343, 38, 34, 105, 373, 261, 192, 290, 107, 98, 336, 323, 127, 140, 259, 300, 317, 229, 166, 239, 189, 2, 91, 336, 258, 173, 375, 4, 131, 345, 170, 287, 219, 263, 342, 289, 54, 323, 202, 87, 94, 329, 78, 93, 327, 162, 115, 138, 117, 318, 120, 127, 366, 324, 72, 313, 242, 7, 352, 279, 95, 185, 318, 354, 282, 332, 86, 200, 181, 52, 55, 306, 35, 221, 37, 51, 277, 208, 57, 172, 19, 205, 122, 250, 233, 155, 27, 29, 51, 277, 351, 119, 378, 59, 371, 369, 114, 168, 184, 241, 247, 25, 379, 16, 326, 215, 272, 344, 40, 381, 308, 211, 321, 214, 361, 76, 266, 174, 360, 146, 328, 36, 158, 356, 49, 245, 152, 361, 137, 11, 65, 85, 276, 77, 377, 368, 376, 337, 281, 256, 361, 137, 11, 183, 218, 90, 217, 147, 303, 43, 246, 64, 144] + 30: [140, 35, 10, 147, 100, 104, 359, 248, 263, 83, 102, 367, 322, 140, 35, 64, 41, 256, 23, 72, 79, 311, 81, 105, 379, 334, 140, 225, 28, 17, 210, 296, 280, 109, 139, 85, 352, 251, 123, 133, 378, 246, 164, 320, 160, 231, 360, 154, 82, 232, 252, 128, 158, 52, 321, 316, 151, 260, 197, 332, 32, 348, 80, 39, 134, 37, 214, 144, 33, 336, 353, 129, 241, 264, 99, 106, 148, 8, 37, 52, 234, 271, 58, 67, 329, 25, 337, 355, 113, 87, 21, 254, 214, 289, 315, 152, 325, 84, 297, 267, 323, 235, 42, 146, 124, 59, 62, 303, 177, 91, 294, 48, 350, 327, 265, 63, 185, 266, 135, 373, 49, 382, 344, 111, 349, 90, 203, 313, 171, 219, 266, 196, 351, 74, 195, 274, 319, 181, 220, 381, 0, 149, 237, 266, 78, 351, 312, 226, 356, 163, 44, 95, 117, 173, 305, 259, 184, 245, 143, 317, 275, 365, 114, 50, 377, 364, 178, 307, 268, 262, 245, 24, 12, 309, 174, 103, 374, 110, 370, 56, 328, 345, 262, 245, 143, 341, 193, 298, 142, 284, 230, 368, 16, 27, 331, 208, 53, 373, 240, 9, 258, 150, 290, 276, 3, 15, 354, 269, 208, 53, 156, 159, 141, 18, 172, 180, 304, 69, 255, 191, 118, 208, 242, 295, 224, 277, 292, 286, 375, 161, 57, 207, 182, 363, 208, 65, 46, 138, 339, 96, 38, 249, 183, 130, 120, 119, 94, 208, 65, 257, 221, 6, 343, 54, 282, 107, 287, 199, 372, 380, 208, 65, 168, 89, 215, 98, 335, 145, 194, 314, 308, 201, 101, 239, 13, 346, 68, 192, 60, 179, 288, 186, 291, 190, 30, 22, 239, 358, 223, 212, 204, 369, 306, 175, 342, 122, 169, 31, 36, 239, 358, 45, 283, 217, 281, 20, 211, 279, 116, 270, 26, 70, 239, 243, 188, 205, 299, 285, 162, 7, 121, 127, 233, 40, 47, 239, 366, 289, 302, 236, 157, 383, 318, 166, 153, 19, 131, 88, 239, 75, 206, 216, 357, 108, 51, 55, 324, 155, 338, 61, 176, 213, 187, 2, 340, 71, 200, 167, 73, 371, 5, 126, 228, 189, 333, 187, 156, 43, 136, 170, 376, 92, 202, 273, 330, 4, 227, 76, 137, 11, 362, 253, 278, 310, 244, 247, 125, 198, 229, 218, 222, 29, 361, 97, 77, 261, 66, 293, 326, 165, 300, 34, 238, 272, 86, 62, 250, 1, 132, 14, 115, 347, 209, 112, 93, 301] + 31: [248, 144, 227, 208, 189, 74, 312, 291, 371, 367, 216, 359, 341, 320, 144, 39, 378, 296, 54, 15, 237, 43, 178, 327, 230, 13, 125, 335, 328, 170, 106, 40, 97, 29, 1, 139, 280, 161, 151, 128, 278, 76, 171, 33, 23, 374, 110, 228, 309, 300, 199, 117, 128, 278, 96, 155, 236, 2, 46, 51, 177, 315, 181, 27, 154, 128, 326, 76, 60, 82, 366, 222, 273, 323, 340, 235, 174, 200, 331, 239, 227, 34, 290, 232, 307, 84, 91, 101, 64, 94, 165, 265, 239, 330, 253, 223, 231, 314, 8, 351, 298, 52, 357, 347, 201, 115, 87, 186, 244, 277, 383, 354, 204, 345, 127, 30, 104, 201, 115, 87, 206, 249, 169, 118, 45, 103, 47, 193, 18, 89, 68, 364, 317, 283, 269, 66, 190, 362, 83, 251, 113, 311, 120, 68, 133, 150, 209, 111, 182, 123, 285, 48, 42, 135, 86, 343, 32, 116, 268, 349, 138, 105, 217, 305, 352, 157, 234, 377, 148, 192, 143, 219, 146, 136, 176, 369, 271, 275, 129, 381, 163, 67, 221, 292, 339, 140, 254, 65, 310, 195, 224, 308, 7, 179, 71, 241, 143, 70, 188, 28, 38, 284, 49, 130, 240, 109, 218, 149, 100, 379, 261, 58, 380, 243, 318, 183, 14, 337, 372, 114, 246, 100, 121, 147, 72, 258, 245, 41, 196, 281, 375, 53, 301, 180, 0, 252, 185, 173, 93, 322, 336, 247, 6, 24, 44, 35, 172, 0, 252, 185, 325, 152, 137, 303, 11, 272, 168, 197, 313, 229, 0, 26, 70, 164, 334, 122, 288, 73, 194, 85, 99, 358, 10, 0, 26, 187, 75, 112, 63, 21, 16, 107, 226, 79, 184, 289, 0, 26, 95, 333, 78, 368, 77, 260, 264, 210, 321, 297, 5, 19, 158, 267, 22, 353, 205, 302, 20, 376, 262, 282, 124, 132, 365, 158, 69, 56, 348, 175, 256, 361, 198, 306, 382, 167, 212, 3, 324, 363, 62, 131, 57, 213, 141, 238, 263, 344, 370, 207, 159, 279, 215, 360, 286, 81, 31, 12, 266, 9, 156, 242, 90, 159, 166, 250, 355, 276, 134, 287, 4, 220, 259, 356, 257, 294, 159, 166, 25, 338, 214, 55, 316, 153, 346, 160, 191, 332, 203, 159, 324, 363, 299, 102, 98, 225, 142, 50, 80, 274, 255, 329, 36, 233, 261, 61, 145, 17, 202, 108, 59, 342, 126, 295, 350, 233, 233, 270, 119, 211, 88, 373, 92, 304, 162, 293, 319, 37] + 32: [94, 360, 40, 186, 227, 250, 84, 24, 210, 264, 118, 112, 316, 94, 122, 61, 348, 91, 324, 249, 337, 243, 73, 220, 192, 379, 63, 163, 275, 319, 298, 281, 183, 309, 311, 44, 52, 93, 162, 63, 51, 41, 75, 189, 25, 45, 158, 42, 153, 334, 272, 167, 63, 51, 166, 96, 369, 213, 382, 378, 102, 365, 23, 260, 312, 63, 8, 344, 184, 345, 26, 182, 193, 221, 356, 377, 159, 326, 63, 280, 344, 283, 306, 175, 237, 139, 110, 160, 149, 226, 333, 63, 66, 164, 211, 294, 111, 29, 315, 191, 197, 216, 98, 177, 218, 200, 166, 202, 266, 332, 59, 77, 284, 238, 46, 321, 373, 218, 163, 107, 126, 7, 50, 142, 198, 99, 341, 133, 242, 262, 288, 370, 38, 233, 308, 199, 286, 10, 154, 0, 196, 267, 114, 148, 130, 164, 279, 289, 231, 336, 22, 172, 67, 355, 328, 145, 148, 21, 38, 143, 252, 85, 39, 4, 152, 222, 137, 358, 265, 352, 229, 322, 65, 254, 269, 32, 340, 320, 305, 170, 338, 173, 352, 310, 322, 89, 297, 157, 230, 317, 34, 270, 188, 318, 90, 352, 271, 329, 225, 171, 64, 293, 62, 9, 248, 100, 215, 251, 155, 241, 16, 204, 72, 239, 115, 247, 43, 195, 180, 17, 383, 33, 190, 214, 359, 325, 178, 70, 363, 82, 330, 278, 295, 14, 33, 241, 275, 380, 212, 48, 3, 129, 299, 376, 76, 368, 375, 246, 190, 329, 174, 313, 28, 86, 349, 307, 105, 1, 95, 296, 234, 347, 342, 263, 168, 206, 253, 372, 282, 134, 343, 290, 57, 135, 347, 107, 151, 58, 71, 53, 381, 136, 374, 224, 74, 123, 268, 285, 219, 156, 19, 55, 81, 80, 124, 117, 31, 18, 244, 346, 285, 257, 140, 120, 354, 351, 366, 339, 119, 79, 194, 68, 367, 106, 274, 97, 357, 104, 350, 6, 132, 304, 259, 217, 181, 367, 92, 2, 78, 113, 36, 176, 302, 125, 128, 60, 161, 245, 367, 92, 12, 83, 301, 314, 150, 236, 144, 201, 203, 235, 54, 367, 323, 121, 258, 232, 303, 371, 205, 287, 20, 255, 207, 15, 367, 179, 122, 331, 273, 292, 240, 276, 103, 13, 364, 47, 300, 49, 187, 327, 116, 335, 69, 127, 37, 5, 35, 138, 11, 27, 108, 209, 327, 87, 228, 101, 256, 56, 361, 261, 208, 291, 147, 146, 362, 141, 131, 109, 185, 353, 30, 165, 223, 277, 88, 169] + 33: [86, 115, 267, 206, 208, 16, 45, 249, 70, 10, 242, 306, 277, 167, 107, 307, 89, 268, 354, 50, 114, 67, 76, 291, 302, 282, 91, 297, 179, 37, 227, 109, 216, 71, 350, 286, 312, 296, 189, 91, 343, 82, 361, 237, 138, 347, 364, 285, 128, 341, 374, 95, 91, 343, 342, 89, 133, 232, 157, 3, 336, 181, 118, 99, 357, 328, 81, 314, 164, 308, 351, 156, 15, 56, 238, 247, 269, 100, 51, 81, 73, 266, 192, 202, 256, 345, 366, 135, 315, 171, 102, 240, 96, 267, 5, 356, 187, 80, 279, 295, 225, 78, 46, 272, 140, 319, 94, 322, 311, 317, 262, 105, 53, 136, 363, 7, 143, 176, 319, 348, 72, 40, 355, 22, 369, 248, 11, 119, 2, 265, 75, 319, 68, 139, 289, 211, 382, 49, 52, 298, 175, 250, 142, 145, 379, 125, 352, 334, 18, 104, 371, 113, 346, 178, 274, 69, 320, 325, 94, 146, 9, 205, 196, 152, 261, 85, 188, 47, 0, 260, 148, 241, 84, 222, 144, 55, 122, 166, 182, 59, 97, 293, 260, 148, 358, 201, 217, 4, 123, 330, 281, 300, 98, 359, 239, 63, 231, 147, 108, 288, 116, 209, 43, 66, 74, 62, 309, 264, 259, 36, 307, 226, 131, 93, 246, 213, 271, 221, 165, 92, 287, 177, 36, 154, 270, 150, 174, 124, 13, 110, 207, 224, 284, 129, 177, 36, 154, 21, 87, 90, 383, 215, 24, 316, 326, 275, 220, 339, 380, 299, 233, 30, 303, 19, 137, 65, 23, 134, 44, 77, 339, 229, 301, 335, 151, 290, 197, 292, 28, 6, 244, 228, 223, 339, 229, 301, 255, 338, 254, 375, 58, 103, 20, 353, 313, 263, 331, 380, 39, 204, 218, 305, 370, 340, 324, 243, 332, 294, 83, 280, 64, 251, 54, 160, 126, 367, 212, 29, 130, 257, 172, 159, 191, 64, 251, 48, 117, 149, 79, 170, 26, 252, 199, 195, 333, 38, 31, 299, 273, 33, 235, 163, 180, 327, 27, 310, 304, 12, 38, 42, 283, 127, 155, 173, 276, 368, 101, 318, 378, 214, 372, 38, 158, 258, 376, 25, 14, 200, 88, 41, 168, 120, 321, 34, 38, 323, 258, 190, 210, 230, 198, 106, 349, 111, 365, 344, 32, 38, 121, 115, 141, 1, 61, 360, 186, 219, 381, 245, 57, 185, 377, 153, 283, 169, 193, 329, 162, 60, 112, 132, 184, 203, 236, 362, 362, 183, 17, 35, 8, 373, 234, 278, 194, 337, 253, 161] + 34: [372, 218, 25, 6, 184, 136, 50, 301, 374, 310, 90, 281, 189, 382, 218, 223, 300, 200, 217, 241, 298, 221, 265, 9, 351, 193, 366, 248, 321, 78, 210, 183, 367, 215, 219, 228, 171, 96, 341, 366, 248, 243, 11, 356, 166, 27, 42, 105, 380, 370, 350, 309, 108, 340, 178, 99, 373, 60, 102, 103, 64, 307, 26, 303, 273, 108, 340, 360, 379, 58, 199, 169, 154, 85, 331, 285, 94, 7, 108, 354, 40, 284, 225, 336, 346, 358, 23, 233, 212, 355, 202, 108, 304, 362, 271, 81, 368, 5, 230, 275, 86, 39, 188, 198, 29, 340, 31, 8, 144, 137, 204, 113, 98, 75, 173, 234, 121, 337, 15, 291, 79, 91, 133, 48, 318, 142, 28, 52, 130, 381, 337, 15, 167, 295, 134, 143, 227, 127, 261, 49, 57, 163, 14, 69, 158, 61, 2, 87, 67, 236, 283, 139, 269, 257, 43, 211, 69, 15, 203, 238, 153, 206, 73, 114, 383, 165, 205, 338, 213, 207, 239, 148, 83, 162, 70, 181, 371, 46, 282, 1, 71, 229, 47, 306, 61, 97, 124, 251, 150, 323, 147, 274, 342, 252, 377, 237, 311, 185, 267, 107, 197, 263, 308, 13, 305, 22, 54, 224, 151, 306, 190, 335, 84, 175, 170, 140, 192, 93, 260, 376, 32, 259, 82, 240, 191, 326, 313, 35, 232, 314, 161, 112, 120, 76, 259, 306, 293, 92, 53, 24, 89, 80, 296, 117, 176, 182, 177, 324, 334, 258, 246, 12, 123, 333, 180, 152, 255, 242, 276, 129, 299, 249, 315, 220, 110, 172, 359, 164, 268, 100, 17, 106, 66, 299, 334, 146, 347, 329, 37, 0, 63, 264, 286, 253, 216, 119, 357, 249, 138, 59, 21, 280, 231, 10, 149, 111, 141, 292, 365, 179, 16, 44, 168, 208, 195, 287, 322, 266, 34, 250, 369, 375, 179, 16, 44, 65, 95, 328, 270, 62, 56, 262, 20, 128, 132, 179, 16, 145, 41, 290, 187, 72, 19, 272, 109, 131, 344, 160, 179, 249, 3, 77, 125, 245, 30, 277, 51, 226, 297, 38, 343, 332, 16, 3, 4, 244, 33, 209, 157, 186, 319, 278, 320, 235, 126, 330, 196, 222, 174, 159, 116, 348, 68, 101, 55, 349, 279, 126, 330, 196, 254, 302, 256, 247, 361, 289, 36, 104, 156, 363, 18, 135, 190, 288, 345, 327, 364, 352, 88, 353, 339, 325, 122, 312, 74, 194, 115, 201, 317, 214, 294, 45, 118, 155, 378, 316] + 35: [48, 286, 3, 194, 266, 213, 337, 166, 376, 22, 232, 119, 251, 309, 133, 113, 326, 14, 215, 261, 145, 315, 317, 238, 351, 177, 363, 68, 281, 335, 162, 294, 350, 277, 378, 259, 354, 226, 341, 363, 133, 113, 349, 126, 93, 125, 204, 301, 207, 195, 242, 124, 80, 68, 281, 164, 331, 372, 57, 109, 6, 35, 101, 178, 200, 75, 318, 289, 307, 52, 104, 320, 273, 7, 29, 306, 280, 71, 75, 314, 322, 234, 191, 30, 302, 348, 17, 65, 139, 83, 13, 353, 295, 31, 355, 134, 5, 209, 216, 142, 151, 165, 225, 381, 220, 295, 276, 340, 360, 356, 235, 8, 338, 248, 255, 310, 284, 45, 67, 38, 239, 102, 61, 122, 82, 380, 370, 121, 252, 146, 45, 367, 203, 340, 160, 128, 12, 106, 73, 176, 183, 223, 304, 45, 367, 276, 346, 267, 325, 32, 369, 217, 198, 10, 271, 343, 169, 345, 72, 186, 163, 132, 170, 366, 84, 365, 167, 333, 228, 169, 345, 322, 224, 373, 36, 95, 34, 269, 149, 94, 51, 229, 260, 368, 72, 78, 275, 308, 90, 264, 196, 135, 89, 258, 59, 28, 297, 342, 231, 140, 244, 11, 202, 43, 103, 148, 197, 40, 28, 345, 21, 41, 37, 328, 42, 382, 285, 63, 77, 193, 85, 44, 297, 55, 265, 222, 190, 201, 339, 66, 105, 263, 233, 283, 246, 56, 138, 364, 136, 268, 137, 290, 99, 316, 26, 123, 100, 182, 1, 86, 154, 321, 9, 256, 91, 19, 161, 49, 188, 64, 205, 211, 324, 375, 159, 192, 143, 347, 175, 292, 358, 33, 199, 205, 131, 53, 50, 362, 334, 74, 214, 179, 377, 250, 336, 236, 240, 131, 53, 241, 361, 108, 92, 20, 374, 189, 262, 0, 120, 62, 1, 153, 352, 312, 171, 300, 117, 127, 156, 155, 208, 2, 237, 1, 129, 147, 157, 230, 287, 227, 25, 274, 107, 69, 293, 144, 254, 4, 15, 98, 81, 16, 305, 88, 79, 150, 270, 359, 130, 379, 86, 221, 168, 296, 96, 180, 279, 332, 97, 327, 60, 319, 299, 203, 206, 187, 272, 172, 111, 344, 87, 288, 219, 118, 319, 218, 141, 114, 27, 158, 152, 330, 70, 253, 23, 371, 291, 319, 218, 257, 311, 54, 298, 243, 383, 184, 245, 181, 210, 76, 319, 303, 47, 46, 329, 282, 313, 24, 174, 278, 249, 247, 212, 58, 299, 342, 116, 357, 18, 323, 110, 185, 115, 112, 39, 173] + 36: [53, 260, 301, 73, 191, 3, 171, 157, 244, 248, 35, 4, 160, 53, 230, 372, 287, 246, 122, 177, 187, 18, 44, 87, 26, 360, 131, 299, 12, 152, 55, 284, 195, 323, 58, 329, 126, 161, 375, 192, 272, 107, 352, 139, 68, 41, 143, 149, 173, 60, 181, 367, 252, 101, 223, 374, 349, 298, 147, 217, 29, 70, 324, 27, 145, 294, 272, 12, 133, 319, 91, 228, 344, 263, 198, 215, 222, 214, 22, 311, 380, 308, 166, 50, 179, 257, 119, 259, 226, 203, 25, 22, 311, 99, 150, 345, 289, 346, 206, 88, 169, 118, 136, 268, 22, 272, 225, 92, 67, 361, 102, 0, 318, 77, 316, 306, 377, 197, 328, 372, 5, 288, 183, 72, 335, 148, 265, 120, 353, 376, 370, 368, 373, 291, 94, 342, 140, 211, 135, 209, 8, 7, 302, 382, 368, 225, 47, 262, 103, 66, 310, 356, 295, 355, 285, 313, 74, 368, 56, 205, 111, 13, 364, 38, 351, 19, 337, 269, 43, 340, 325, 223, 97, 293, 172, 366, 305, 146, 338, 137, 82, 234, 340, 325, 107, 275, 282, 95, 100, 264, 159, 190, 115, 154, 336, 176, 371, 90, 207, 49, 164, 271, 81, 358, 249, 357, 286, 113, 341, 371, 369, 54, 162, 339, 280, 52, 220, 109, 61, 34, 270, 341, 371, 274, 332, 334, 63, 116, 104, 243, 281, 378, 96, 279, 184, 359, 40, 79, 231, 105, 11, 76, 23, 189, 254, 128, 290, 184, 59, 320, 31, 240, 15, 174, 255, 48, 6, 14, 227, 292, 184, 2, 178, 32, 112, 132, 202, 212, 80, 158, 110, 327, 16, 144, 10, 210, 45, 219, 69, 153, 36, 267, 251, 256, 106, 221, 144, 10, 258, 65, 9, 121, 237, 39, 78, 204, 312, 303, 37, 84, 62, 296, 42, 108, 167, 277, 180, 51, 247, 350, 1, 127, 57, 266, 193, 314, 33, 20, 168, 229, 365, 236, 21, 218, 182, 57, 185, 178, 261, 362, 297, 224, 30, 163, 315, 307, 89, 317, 250, 266, 326, 85, 175, 155, 117, 134, 239, 363, 235, 86, 123, 304, 238, 241, 233, 333, 276, 216, 71, 208, 24, 213, 151, 170, 304, 238, 193, 125, 309, 245, 343, 379, 188, 199, 130, 124, 186, 348, 232, 242, 93, 201, 322, 142, 28, 129, 383, 75, 330, 64, 232, 232, 98, 331, 194, 138, 46, 200, 283, 83, 253, 278, 273, 232, 232, 196, 156, 321, 300, 354, 17, 165, 381, 347, 141, 114] + 37: [109, 363, 72, 185, 131, 19, 285, 121, 274, 216, 308, 246, 358, 109, 366, 79, 253, 153, 176, 143, 100, 154, 138, 70, 180, 375, 188, 200, 79, 360, 32, 127, 135, 312, 94, 350, 63, 28, 226, 240, 200, 342, 302, 18, 370, 329, 337, 280, 260, 142, 119, 8, 268, 222, 193, 47, 330, 104, 278, 136, 172, 160, 214, 300, 120, 268, 199, 290, 245, 118, 258, 108, 262, 317, 215, 96, 227, 288, 268, 46, 305, 353, 194, 196, 141, 167, 249, 13, 107, 11, 251, 319, 222, 44, 7, 93, 270, 333, 49, 351, 266, 106, 281, 352, 301, 52, 190, 22, 117, 144, 338, 206, 42, 303, 146, 326, 383, 301, 205, 48, 88, 233, 21, 92, 83, 354, 60, 81, 381, 25, 71, 293, 272, 182, 181, 122, 134, 347, 97, 137, 283, 89, 238, 364, 293, 179, 252, 343, 208, 376, 80, 284, 175, 339, 186, 276, 66, 293, 362, 14, 204, 323, 82, 380, 24, 91, 184, 155, 75, 40, 314, 292, 344, 192, 27, 161, 247, 5, 110, 225, 2, 50, 40, 151, 372, 327, 328, 43, 39, 87, 298, 124, 57, 220, 189, 40, 125, 341, 165, 235, 1, 217, 244, 271, 115, 295, 367, 68, 40, 306, 289, 65, 147, 365, 304, 356, 26, 254, 130, 219, 99, 40, 178, 366, 318, 139, 203, 73, 55, 325, 286, 374, 275, 23, 294, 148, 349, 171, 133, 316, 3, 69, 4, 53, 357, 261, 236, 294, 148, 162, 114, 345, 76, 111, 373, 212, 291, 35, 20, 56, 294, 213, 162, 140, 164, 287, 15, 334, 105, 152, 62, 202, 239, 218, 158, 113, 59, 145, 173, 30, 331, 210, 256, 231, 369, 209, 218, 265, 113, 223, 336, 149, 36, 126, 187, 10, 166, 177, 112, 218, 348, 382, 0, 315, 64, 296, 207, 38, 17, 248, 197, 132, 218, 242, 95, 9, 163, 320, 34, 37, 33, 170, 191, 12, 282, 218, 309, 371, 355, 277, 85, 103, 255, 313, 183, 297, 159, 377, 218, 311, 234, 129, 257, 229, 29, 269, 102, 332, 198, 211, 346, 218, 148, 349, 156, 221, 228, 201, 98, 195, 169, 310, 90, 232, 273, 158, 174, 16, 359, 51, 264, 224, 267, 368, 324, 250, 67, 123, 243, 77, 361, 322, 6, 61, 128, 241, 378, 101, 45, 263, 379, 58, 340, 86, 157, 74, 41, 335, 116, 299, 237, 54, 31, 379, 58, 340, 0, 150, 168, 230, 321, 279, 78, 259, 307, 84] + 38: [282, 285, 39, 217, 275, 45, 349, 305, 131, 80, 299, 18, 60, 10, 354, 230, 92, 355, 36, 323, 353, 139, 294, 343, 232, 185, 4, 366, 293, 381, 352, 307, 88, 174, 15, 47, 70, 207, 38, 158, 263, 62, 63, 14, 87, 342, 376, 173, 273, 42, 198, 312, 158, 366, 62, 379, 147, 58, 151, 180, 337, 347, 112, 336, 243, 187, 30, 277, 119, 108, 208, 367, 84, 302, 186, 71, 209, 52, 6, 51, 177, 171, 146, 211, 276, 242, 81, 100, 326, 83, 330, 6, 8, 49, 130, 107, 68, 246, 26, 250, 261, 157, 1, 140, 6, 86, 333, 344, 286, 227, 229, 148, 195, 295, 357, 128, 339, 6, 30, 154, 267, 97, 85, 48, 31, 214, 283, 138, 153, 340, 223, 51, 89, 373, 361, 321, 368, 356, 238, 311, 236, 163, 237, 93, 292, 333, 169, 16, 72, 156, 76, 296, 120, 335, 127, 162, 301, 201, 89, 317, 137, 9, 136, 28, 200, 104, 65, 196, 133, 301, 284, 220, 166, 90, 358, 256, 181, 152, 252, 316, 365, 121, 301, 284, 32, 244, 99, 105, 57, 272, 2, 11, 320, 160, 304, 301, 372, 194, 77, 338, 67, 34, 289, 371, 44, 12, 20, 101, 301, 372, 309, 206, 329, 271, 188, 141, 122, 205, 290, 132, 191, 213, 201, 114, 135, 327, 95, 96, 306, 79, 297, 363, 175, 210, 287, 124, 154, 268, 331, 239, 111, 117, 172, 25, 164, 22, 270, 378, 280, 277, 54, 199, 73, 298, 260, 313, 351, 183, 116, 64, 378, 124, 114, 253, 234, 69, 179, 190, 360, 35, 247, 167, 98, 50, 280, 248, 257, 53, 21, 7, 5, 43, 375, 254, 377, 159, 369, 269, 115, 300, 319, 324, 143, 241, 182, 13, 55, 314, 251, 178, 269, 383, 328, 225, 29, 37, 228, 74, 82, 91, 281, 161, 348, 332, 126, 279, 233, 193, 216, 134, 310, 382, 24, 110, 370, 345, 123, 274, 278, 125, 224, 308, 41, 235, 374, 3, 259, 380, 221, 203, 189, 27, 75, 359, 264, 33, 40, 204, 240, 218, 318, 221, 203, 189, 145, 219, 262, 184, 149, 78, 215, 192, 315, 346, 266, 118, 126, 0, 106, 59, 142, 170, 334, 168, 102, 291, 46, 266, 94, 226, 17, 341, 150, 255, 258, 103, 61, 202, 350, 325, 266, 203, 322, 19, 222, 144, 109, 265, 129, 231, 113, 303, 176, 245, 66, 383, 212, 288, 362, 165, 197, 23, 56, 364, 155, 249] + 39: [43, 218, 2, 296, 118, 31, 5, 293, 309, 41, 347, 262, 356, 100, 13, 382, 297, 53, 251, 179, 61, 130, 288, 211, 67, 90, 176, 139, 379, 362, 170, 248, 301, 182, 254, 111, 55, 105, 11, 247, 312, 85, 329, 352, 191, 249, 302, 181, 166, 229, 219, 29, 247, 312, 292, 145, 274, 232, 226, 373, 240, 217, 0, 351, 44, 247, 215, 187, 277, 146, 336, 276, 320, 79, 189, 266, 73, 332, 99, 65, 86, 270, 383, 340, 213, 72, 220, 192, 18, 158, 148, 282, 315, 267, 172, 375, 239, 367, 357, 60, 120, 92, 339, 209, 282, 315, 119, 374, 84, 225, 199, 241, 243, 222, 157, 308, 57, 282, 315, 155, 333, 8, 127, 345, 212, 377, 70, 246, 124, 261, 282, 65, 313, 330, 87, 34, 353, 285, 214, 376, 359, 233, 321, 96, 113, 325, 300, 7, 101, 281, 88, 128, 335, 142, 349, 334, 280, 56, 244, 174, 42, 140, 162, 33, 207, 343, 238, 284, 21, 35, 56, 85, 80, 259, 48, 161, 134, 169, 171, 37, 147, 165, 180, 56, 328, 324, 200, 323, 307, 311, 363, 27, 20, 294, 361, 268, 151, 279, 250, 59, 138, 358, 193, 68, 109, 252, 350, 304, 264, 286, 313, 141, 81, 331, 150, 185, 230, 287, 160, 125, 9, 66, 153, 314, 17, 23, 47, 152, 354, 265, 91, 197, 129, 52, 66, 156, 314, 98, 298, 122, 366, 106, 221, 117, 46, 318, 186, 66, 286, 36, 295, 202, 278, 135, 289, 164, 159, 78, 224, 115, 110, 25, 132, 15, 77, 137, 50, 97, 338, 107, 337, 12, 154, 116, 1, 36, 40, 273, 112, 62, 133, 6, 103, 83, 205, 201, 258, 236, 272, 71, 227, 299, 75, 14, 327, 245, 10, 255, 22, 258, 368, 271, 26, 381, 28, 364, 45, 275, 346, 231, 256, 89, 234, 236, 242, 121, 235, 54, 63, 94, 195, 19, 305, 237, 269, 104, 372, 93, 178, 163, 131, 144, 355, 114, 283, 253, 326, 149, 104, 372, 360, 24, 3, 260, 317, 228, 183, 291, 39, 342, 198, 104, 236, 93, 369, 49, 190, 371, 167, 16, 303, 206, 32, 344, 82, 372, 38, 319, 365, 74, 188, 95, 4, 196, 175, 370, 257, 310, 310, 379, 204, 290, 263, 126, 203, 194, 216, 64, 210, 143, 310, 223, 132, 341, 51, 69, 177, 173, 30, 168, 108, 316, 184, 310, 348, 38, 76, 208, 378, 322, 102, 306, 58, 123, 136, 380] + 40: [31, 75, 164, 232, 321, 148, 82, 371, 6, 121, 285, 152, 42, 31, 98, 88, 254, 11, 193, 377, 327, 73, 256, 111, 234, 72, 31, 190, 226, 251, 136, 302, 290, 264, 131, 310, 203, 245, 249, 31, 190, 8, 81, 219, 262, 41, 323, 118, 116, 173, 354, 144, 31, 101, 156, 74, 271, 382, 110, 29, 293, 383, 230, 252, 343, 31, 62, 128, 90, 284, 259, 316, 237, 37, 13, 16, 24, 194, 132, 367, 153, 369, 211, 145, 263, 243, 350, 49, 368, 272, 92, 379, 373, 346, 58, 143, 239, 364, 329, 147, 270, 180, 154, 191, 317, 378, 202, 97, 296, 137, 53, 177, 85, 288, 231, 267, 225, 317, 67, 247, 228, 341, 188, 103, 294, 238, 186, 119, 352, 240, 357, 378, 107, 235, 326, 370, 299, 93, 60, 135, 218, 87, 20, 207, 55, 26, 198, 336, 46, 365, 195, 19, 124, 77, 176, 312, 207, 378, 33, 351, 362, 307, 309, 330, 301, 266, 242, 109, 127, 340, 175, 215, 3, 129, 229, 150, 282, 201, 14, 34, 342, 69, 280, 175, 33, 258, 10, 196, 189, 339, 36, 122, 325, 313, 331, 319, 358, 108, 28, 166, 30, 138, 333, 18, 184, 22, 114, 335, 319, 358, 108, 91, 253, 278, 130, 120, 47, 102, 295, 157, 353, 319, 175, 283, 9, 61, 308, 43, 167, 139, 361, 163, 45, 146, 183, 71, 21, 311, 80, 38, 306, 338, 314, 268, 40, 255, 105, 183, 71, 21, 275, 209, 205, 355, 155, 76, 133, 106, 4, 220, 260, 204, 265, 1, 162, 199, 2, 7, 241, 141, 134, 39, 83, 15, 328, 161, 372, 63, 12, 244, 324, 261, 276, 182, 298, 300, 15, 328, 226, 332, 277, 216, 151, 192, 51, 125, 217, 315, 126, 15, 359, 337, 347, 322, 236, 57, 206, 233, 23, 113, 292, 123, 250, 68, 297, 100, 303, 56, 79, 48, 304, 50, 117, 86, 214, 250, 68, 160, 224, 168, 227, 65, 318, 222, 84, 174, 112, 248, 286, 68, 52, 159, 185, 200, 99, 171, 273, 32, 140, 172, 376, 349, 142, 279, 59, 363, 44, 187, 281, 291, 78, 257, 381, 344, 349, 142, 346, 208, 89, 269, 95, 305, 375, 115, 35, 380, 5, 360, 70, 179, 210, 274, 221, 94, 165, 246, 212, 104, 96, 197, 360, 213, 179, 17, 289, 334, 25, 287, 345, 348, 66, 223, 170, 169, 181, 158, 64, 27, 54, 178, 356, 366, 0, 374, 320, 149] + 41: [205, 86, 119, 334, 318, 55, 340, 43, 173, 207, 175, 106, 281, 305, 30, 124, 172, 218, 345, 242, 66, 99, 65, 375, 341, 291, 122, 30, 368, 314, 219, 308, 210, 101, 123, 77, 14, 110, 120, 321, 150, 327, 12, 326, 93, 156, 130, 10, 68, 73, 177, 228, 321, 57, 119, 302, 239, 349, 266, 107, 75, 78, 231, 121, 152, 185, 255, 76, 136, 163, 70, 138, 153, 15, 143, 17, 62, 216, 49, 249, 92, 187, 243, 56, 117, 18, 192, 352, 47, 267, 309, 49, 190, 215, 358, 25, 134, 330, 290, 41, 381, 373, 88, 333, 49, 190, 97, 364, 36, 252, 51, 299, 33, 72, 182, 362, 382, 269, 184, 159, 347, 286, 64, 8, 181, 154, 100, 245, 323, 296, 269, 184, 159, 206, 35, 125, 258, 213, 244, 234, 195, 372, 118, 298, 374, 96, 7, 126, 102, 339, 295, 359, 38, 98, 109, 203, 298, 90, 215, 111, 194, 336, 328, 186, 59, 165, 29, 343, 31, 283, 374, 293, 34, 23, 26, 112, 246, 89, 179, 274, 2, 178, 283, 374, 285, 79, 353, 116, 288, 338, 232, 180, 67, 157, 247, 87, 212, 275, 279, 42, 32, 268, 131, 292, 105, 147, 346, 169, 367, 307, 275, 280, 53, 241, 237, 208, 146, 191, 329, 145, 263, 335, 350, 223, 161, 222, 54, 322, 5, 371, 248, 113, 211, 52, 378, 60, 50, 262, 45, 226, 167, 19, 214, 1, 238, 270, 227, 378, 350, 129, 21, 202, 174, 133, 176, 342, 13, 155, 162, 164, 378, 350, 129, 69, 264, 301, 380, 81, 303, 39, 171, 331, 357, 196, 60, 366, 306, 383, 276, 289, 91, 221, 332, 148, 312, 230, 103, 60, 197, 253, 80, 83, 115, 151, 300, 0, 265, 377, 135, 114, 355, 200, 108, 3, 225, 229, 160, 128, 48, 257, 356, 304, 114, 355, 365, 95, 9, 254, 344, 284, 37, 236, 20, 139, 278, 166, 170, 366, 82, 348, 256, 144, 63, 188, 259, 137, 189, 233, 71, 170, 84, 198, 351, 282, 294, 261, 220, 149, 316, 16, 58, 11, 170, 369, 24, 324, 313, 235, 94, 74, 297, 28, 183, 85, 199, 251, 197, 44, 4, 379, 271, 317, 61, 141, 209, 240, 22, 199, 376, 369, 311, 127, 46, 325, 277, 354, 287, 370, 319, 40, 199, 27, 273, 360, 140, 217, 224, 6, 315, 158, 310, 361, 193, 142, 142, 168, 337, 104, 260, 250, 363, 272, 320, 204, 201, 132] + 42: [2, 93, 111, 265, 341, 181, 381, 283, 53, 290, 207, 35, 242, 188, 350, 256, 203, 135, 167, 137, 238, 205, 73, 198, 159, 289, 210, 279, 150, 209, 29, 338, 241, 311, 171, 343, 134, 47, 294, 154, 218, 111, 191, 354, 143, 223, 232, 233, 56, 22, 253, 339, 375, 218, 215, 351, 320, 164, 76, 97, 156, 284, 75, 331, 369, 359, 212, 333, 189, 328, 366, 80, 246, 298, 360, 383, 57, 15, 359, 116, 93, 348, 221, 329, 101, 180, 5, 197, 124, 177, 74, 20, 247, 168, 377, 344, 335, 65, 315, 270, 318, 186, 44, 163, 1, 211, 337, 262, 287, 173, 132, 219, 336, 98, 285, 271, 361, 217, 155, 222, 200, 17, 201, 146, 225, 16, 82, 128, 43, 0, 217, 184, 349, 220, 230, 165, 301, 64, 34, 104, 309, 249, 376, 50, 155, 333, 319, 131, 234, 166, 99, 68, 84, 302, 69, 95, 42, 172, 158, 327, 370, 45, 362, 226, 278, 175, 288, 182, 352, 123, 345, 374, 129, 286, 322, 160, 139, 7, 145, 330, 107, 27, 123, 345, 243, 108, 214, 161, 51, 372, 378, 102, 185, 78, 66, 123, 303, 41, 263, 239, 36, 192, 49, 251, 206, 3, 54, 40, 123, 305, 37, 59, 236, 92, 149, 244, 291, 229, 23, 81, 252, 123, 176, 256, 342, 248, 280, 39, 109, 61, 250, 144, 110, 121, 123, 176, 158, 281, 274, 314, 227, 157, 196, 321, 174, 12, 193, 123, 172, 72, 373, 275, 316, 245, 89, 58, 363, 367, 70, 273, 94, 268, 138, 208, 140, 105, 19, 231, 300, 293, 153, 100, 86, 94, 268, 138, 255, 224, 87, 8, 62, 125, 194, 96, 115, 254, 266, 357, 195, 306, 187, 356, 113, 323, 258, 368, 347, 353, 324, 266, 106, 37, 48, 67, 216, 60, 307, 170, 120, 85, 190, 25, 295, 117, 162, 55, 28, 103, 340, 371, 269, 213, 88, 276, 71, 148, 117, 179, 6, 169, 308, 380, 260, 313, 10, 334, 14, 91, 114, 117, 264, 122, 4, 38, 52, 299, 346, 261, 136, 79, 46, 228, 117, 282, 127, 30, 77, 118, 310, 364, 272, 304, 83, 126, 152, 31, 325, 379, 141, 202, 257, 183, 332, 199, 292, 317, 240, 152, 31, 325, 235, 358, 33, 297, 130, 355, 267, 296, 382, 365, 152, 31, 63, 21, 259, 147, 32, 9, 204, 90, 312, 142, 13, 152, 119, 151, 277, 178, 326, 18, 237, 112, 26, 133, 11, 24] + 43: [164, 324, 351, 23, 73, 115, 261, 284, 215, 87, 300, 228, 189, 369, 324, 314, 363, 272, 219, 61, 46, 101, 173, 69, 236, 195, 27, 216, 43, 337, 255, 90, 196, 183, 226, 193, 258, 42, 184, 344, 37, 298, 93, 321, 246, 79, 217, 116, 259, 260, 208, 231, 315, 59, 131, 57, 2, 234, 130, 45, 288, 247, 285, 328, 95, 315, 364, 298, 153, 121, 346, 28, 47, 146, 233, 248, 281, 169, 262, 98, 351, 23, 222, 200, 365, 213, 220, 209, 199, 21, 376, 82, 98, 43, 291, 4, 282, 334, 113, 325, 151, 179, 383, 182, 181, 98, 297, 81, 280, 88, 36, 74, 125, 358, 230, 141, 54, 168, 112, 103, 235, 191, 286, 107, 49, 72, 279, 357, 333, 33, 168, 359, 297, 55, 368, 378, 148, 102, 329, 65, 265, 58, 289, 168, 359, 94, 186, 89, 154, 243, 293, 370, 19, 110, 270, 302, 6, 35, 366, 152, 77, 275, 237, 165, 309, 268, 276, 303, 192, 221, 66, 30, 178, 1, 135, 367, 120, 170, 163, 336, 15, 319, 221, 66, 100, 326, 150, 277, 67, 210, 175, 126, 290, 133, 350, 161, 66, 30, 29, 85, 232, 12, 242, 122, 117, 320, 96, 212, 105, 129, 14, 127, 307, 123, 5, 253, 295, 76, 330, 104, 25, 105, 197, 14, 111, 250, 238, 155, 229, 9, 134, 53, 0, 382, 171, 142, 13, 138, 147, 327, 63, 24, 207, 71, 355, 362, 206, 171, 371, 20, 342, 78, 380, 84, 167, 128, 109, 114, 62, 44, 156, 17, 39, 377, 379, 52, 241, 180, 263, 119, 124, 249, 331, 156, 17, 349, 22, 32, 86, 157, 143, 18, 132, 187, 340, 310, 177, 335, 283, 227, 108, 299, 296, 266, 159, 353, 188, 139, 244, 332, 335, 283, 323, 269, 339, 145, 99, 64, 294, 311, 137, 373, 347, 372, 254, 264, 305, 140, 34, 251, 292, 144, 240, 352, 361, 347, 372, 190, 301, 136, 204, 345, 60, 304, 8, 211, 91, 176, 225, 68, 252, 318, 224, 271, 149, 70, 31, 356, 162, 7, 214, 225, 218, 75, 203, 274, 83, 10, 38, 174, 118, 201, 41, 245, 225, 92, 190, 322, 202, 48, 194, 50, 80, 381, 341, 106, 348, 308, 92, 75, 223, 374, 205, 273, 185, 11, 267, 316, 40, 166, 375, 375, 239, 313, 198, 26, 160, 16, 256, 158, 312, 172, 354, 375, 278, 97, 360, 338, 306, 257, 56, 343, 287, 51, 317, 3] + 44: [362, 271, 325, 11, 14, 219, 286, 46, 211, 114, 273, 31, 133, 362, 60, 331, 187, 158, 296, 269, 90, 205, 53, 1, 302, 332, 274, 284, 311, 143, 121, 287, 131, 327, 179, 314, 326, 307, 303, 274, 284, 229, 288, 265, 70, 71, 165, 8, 367, 375, 47, 197, 168, 155, 135, 320, 207, 315, 66, 36, 237, 183, 45, 259, 295, 238, 223, 252, 198, 228, 192, 177, 196, 377, 9, 233, 123, 208, 97, 223, 346, 63, 33, 186, 67, 220, 169, 51, 312, 52, 310, 64, 101, 68, 329, 144, 184, 221, 41, 32, 4, 2, 37, 75, 172, 348, 140, 93, 340, 56, 352, 128, 3, 62, 245, 127, 281, 172, 101, 346, 306, 48, 24, 54, 15, 59, 109, 239, 139, 181, 171, 348, 279, 383, 324, 349, 380, 342, 113, 256, 200, 297, 209, 171, 348, 368, 146, 276, 100, 132, 83, 214, 321, 216, 356, 339, 167, 226, 170, 142, 102, 85, 261, 92, 318, 230, 119, 246, 353, 0, 226, 170, 118, 125, 244, 381, 108, 323, 250, 151, 247, 58, 17, 226, 145, 333, 291, 364, 130, 147, 156, 57, 373, 69, 189, 258, 317, 376, 141, 217, 122, 378, 26, 341, 203, 251, 372, 308, 213, 110, 145, 98, 355, 61, 174, 185, 270, 49, 124, 111, 13, 213, 351, 82, 313, 94, 201, 79, 136, 336, 319, 22, 176, 106, 213, 317, 84, 242, 275, 266, 120, 126, 309, 193, 5, 103, 12, 213, 317, 257, 280, 89, 175, 87, 157, 104, 137, 253, 152, 161, 25, 379, 43, 166, 117, 290, 202, 354, 81, 204, 42, 73, 371, 77, 10, 277, 30, 138, 163, 248, 107, 20, 7, 35, 335, 129, 77, 10, 328, 236, 91, 254, 359, 74, 149, 345, 153, 330, 374, 77, 379, 29, 34, 262, 350, 369, 278, 78, 235, 99, 283, 76, 272, 10, 227, 86, 343, 150, 366, 334, 231, 263, 210, 363, 148, 178, 382, 180, 264, 365, 154, 160, 292, 218, 232, 80, 293, 112, 300, 222, 267, 212, 337, 173, 338, 370, 347, 241, 50, 240, 316, 301, 199, 188, 182, 95, 55, 164, 40, 105, 357, 195, 260, 116, 301, 199, 215, 289, 360, 39, 285, 282, 243, 18, 27, 268, 322, 301, 222, 190, 38, 358, 299, 96, 162, 305, 65, 255, 361, 206, 191, 191, 21, 294, 304, 224, 44, 344, 23, 234, 194, 298, 88, 191, 199, 21, 249, 225, 6, 134, 159, 19, 72, 115, 28, 16] + 45: [154, 227, 269, 275, 223, 217, 26, 296, 282, 113, 111, 99, 69, 154, 227, 298, 107, 162, 260, 315, 59, 294, 268, 132, 278, 100, 324, 247, 329, 254, 359, 94, 32, 38, 37, 289, 31, 101, 117, 23, 141, 21, 331, 307, 157, 279, 252, 322, 190, 214, 77, 181, 23, 356, 164, 248, 196, 108, 67, 290, 263, 73, 91, 66, 195, 191, 339, 164, 234, 45, 86, 208, 358, 9, 48, 89, 265, 355, 173, 176, 61, 258, 310, 44, 292, 264, 283, 364, 189, 13, 60, 121, 120, 131, 351, 168, 281, 109, 87, 151, 362, 90, 259, 235, 121, 171, 329, 228, 239, 41, 96, 319, 333, 215, 321, 287, 185, 301, 27, 180, 39, 187, 156, 175, 256, 379, 377, 238, 172, 244, 318, 27, 47, 118, 198, 242, 365, 5, 98, 203, 18, 193, 64, 219, 274, 286, 163, 245, 210, 376, 20, 288, 341, 186, 352, 138, 140, 10, 276, 95, 30, 251, 253, 19, 177, 220, 152, 194, 336, 140, 8, 354, 165, 88, 313, 43, 344, 79, 166, 82, 206, 174, 140, 8, 276, 95, 104, 357, 261, 34, 57, 147, 102, 311, 49, 233, 10, 378, 81, 250, 328, 257, 230, 255, 202, 225, 325, 17, 188, 10, 262, 209, 158, 345, 123, 207, 374, 119, 382, 53, 14, 134, 197, 375, 114, 338, 112, 222, 126, 231, 71, 243, 237, 130, 144, 182, 227, 297, 372, 84, 340, 205, 241, 93, 136, 184, 299, 316, 346, 350, 380, 122, 249, 236, 291, 273, 368, 125, 6, 11, 92, 178, 146, 383, 74, 306, 22, 58, 314, 105, 143, 159, 300, 92, 178, 103, 348, 70, 54, 50, 133, 342, 15, 56, 148, 40, 92, 178, 85, 124, 155, 213, 2, 145, 29, 78, 332, 28, 149, 92, 201, 369, 183, 55, 179, 106, 320, 150, 312, 280, 295, 361, 92, 201, 367, 128, 36, 327, 63, 35, 129, 142, 167, 270, 221, 110, 334, 373, 212, 51, 360, 24, 304, 381, 371, 62, 366, 192, 161, 199, 160, 383, 277, 7, 303, 76, 271, 363, 226, 200, 83, 161, 334, 367, 224, 326, 135, 1, 46, 211, 343, 323, 335, 72, 317, 139, 350, 52, 16, 267, 302, 116, 169, 3, 337, 75, 240, 293, 42, 373, 81, 308, 12, 218, 65, 285, 0, 97, 353, 347, 293, 25, 33, 349, 229, 68, 370, 330, 204, 246, 232, 305, 266, 284, 42, 33, 272, 115, 80, 4, 170, 127, 309, 153, 216, 137] + 46: [152, 191, 140, 303, 280, 334, 17, 32, 71, 26, 347, 84, 200, 188, 239, 154, 165, 8, 295, 250, 81, 9, 339, 67, 105, 88, 142, 207, 12, 40, 117, 132, 237, 307, 159, 98, 150, 123, 357, 142, 202, 335, 254, 195, 353, 267, 255, 313, 264, 72, 107, 302, 142, 238, 66, 345, 131, 287, 158, 45, 20, 90, 258, 251, 5, 133, 51, 301, 338, 62, 232, 235, 383, 265, 244, 15, 306, 27, 133, 207, 312, 378, 130, 118, 292, 328, 56, 278, 112, 212, 270, 210, 96, 335, 60, 155, 186, 321, 308, 7, 93, 75, 196, 61, 10, 110, 344, 204, 143, 315, 103, 92, 189, 283, 281, 269, 122, 10, 25, 140, 2, 85, 42, 294, 329, 243, 309, 77, 1, 82, 201, 110, 66, 113, 211, 156, 273, 35, 346, 305, 135, 279, 323, 247, 170, 319, 194, 372, 192, 95, 101, 293, 190, 275, 296, 365, 247, 16, 354, 203, 39, 37, 41, 373, 317, 361, 127, 48, 147, 183, 311, 209, 271, 376, 166, 63, 263, 382, 371, 163, 47, 151, 183, 311, 319, 124, 227, 259, 272, 226, 74, 261, 31, 73, 36, 183, 249, 104, 214, 260, 4, 362, 169, 299, 370, 330, 179, 178, 34, 343, 106, 136, 83, 369, 23, 97, 173, 364, 236, 54, 70, 34, 311, 284, 55, 115, 99, 157, 111, 153, 262, 65, 381, 356, 352, 355, 324, 199, 30, 180, 53, 125, 342, 285, 21, 300, 175, 352, 355, 6, 114, 108, 326, 193, 268, 241, 91, 11, 134, 220, 352, 343, 367, 320, 266, 50, 164, 141, 348, 375, 208, 349, 14, 233, 253, 367, 3, 102, 197, 205, 78, 222, 86, 43, 87, 231, 19, 240, 274, 242, 215, 298, 176, 289, 318, 177, 94, 174, 252, 64, 46, 149, 68, 120, 80, 59, 336, 333, 217, 366, 181, 325, 64, 253, 149, 38, 33, 245, 340, 57, 277, 171, 337, 0, 100, 139, 380, 230, 44, 49, 218, 198, 290, 119, 145, 341, 29, 182, 139, 327, 221, 18, 228, 168, 216, 24, 297, 331, 310, 116, 148, 256, 224, 332, 219, 350, 137, 128, 377, 322, 234, 52, 248, 225, 256, 380, 246, 374, 351, 121, 172, 360, 223, 184, 358, 288, 379, 58, 286, 106, 187, 22, 316, 304, 206, 69, 257, 138, 185, 28, 58, 368, 354, 146, 79, 109, 162, 160, 89, 359, 126, 276, 167, 314, 286, 230, 76, 282, 13, 129, 144, 161, 291, 229, 363, 213] + 47: [195, 158, 196, 129, 159, 197, 164, 378, 186, 144, 100, 247, 56, 195, 61, 127, 175, 351, 199, 354, 321, 273, 325, 372, 88, 369, 195, 29, 353, 274, 69, 249, 12, 28, 332, 302, 174, 99, 318, 195, 29, 257, 36, 93, 285, 0, 126, 258, 152, 379, 295, 7, 75, 158, 368, 79, 250, 298, 138, 163, 37, 120, 105, 270, 191, 343, 87, 316, 362, 265, 370, 101, 335, 107, 242, 16, 64, 254, 343, 1, 14, 376, 219, 151, 52, 315, 132, 297, 277, 11, 8, 343, 34, 305, 240, 63, 203, 210, 15, 259, 266, 361, 47, 246, 54, 109, 345, 59, 98, 255, 117, 201, 328, 57, 304, 116, 156, 54, 87, 300, 153, 262, 166, 180, 23, 310, 46, 74, 232, 62, 54, 87, 333, 40, 176, 184, 106, 366, 2, 97, 133, 264, 146, 271, 170, 309, 21, 95, 41, 17, 71, 115, 102, 207, 6, 182, 347, 279, 286, 238, 251, 77, 252, 139, 32, 253, 317, 299, 373, 331, 119, 134, 188, 226, 65, 233, 356, 341, 149, 290, 125, 261, 187, 375, 134, 346, 114, 50, 27, 18, 60, 19, 350, 165, 272, 179, 35, 90, 168, 212, 73, 112, 141, 94, 275, 5, 161, 209, 136, 3, 294, 198, 230, 172, 322, 147, 306, 189, 358, 217, 66, 181, 43, 320, 267, 313, 140, 4, 96, 248, 334, 38, 85, 288, 181, 371, 215, 344, 330, 33, 205, 289, 111, 48, 177, 256, 83, 154, 3, 137, 110, 269, 323, 45, 336, 278, 282, 374, 167, 113, 154, 3, 137, 118, 155, 103, 355, 211, 185, 81, 380, 283, 128, 192, 3, 314, 287, 301, 223, 296, 213, 31, 365, 220, 76, 293, 143, 22, 300, 327, 377, 124, 39, 80, 281, 216, 222, 142, 381, 171, 22, 307, 123, 26, 228, 338, 121, 200, 160, 92, 78, 169, 162, 312, 193, 218, 68, 231, 237, 243, 337, 308, 86, 194, 13, 162, 312, 227, 349, 135, 89, 340, 183, 157, 130, 30, 145, 360, 44, 357, 359, 367, 67, 224, 173, 234, 20, 9, 383, 221, 241, 44, 291, 359, 229, 104, 324, 204, 225, 260, 10, 245, 206, 214, 44, 24, 382, 342, 208, 319, 25, 70, 292, 55, 339, 268, 236, 352, 24, 382, 363, 84, 263, 51, 303, 49, 150, 58, 244, 239, 352, 24, 307, 131, 122, 364, 53, 91, 276, 72, 178, 202, 326, 352, 312, 280, 235, 42, 284, 148, 348, 329, 190, 108, 82, 311] + 48: [237, 121, 83, 15, 156, 50, 230, 379, 259, 382, 149, 171, 314, 267, 236, 28, 166, 371, 174, 170, 36, 269, 310, 61, 86, 245, 267, 343, 78, 62, 300, 192, 275, 22, 35, 313, 27, 54, 303, 159, 44, 21, 62, 205, 112, 221, 307, 137, 216, 361, 46, 348, 297, 234, 85, 263, 312, 299, 347, 80, 346, 373, 224, 68, 306, 288, 19, 182, 235, 134, 285, 24, 180, 107, 173, 66, 8, 334, 9, 132, 2, 38, 186, 123, 118, 289, 16, 243, 225, 282, 209, 51, 132, 353, 240, 115, 322, 360, 113, 169, 194, 294, 296, 10, 283, 99, 284, 5, 305, 233, 357, 317, 274, 135, 105, 298, 196, 283, 132, 311, 372, 342, 7, 42, 96, 352, 69, 37, 332, 199, 76, 276, 52, 48, 316, 161, 231, 281, 55, 217, 278, 338, 227, 76, 276, 203, 264, 185, 247, 321, 351, 98, 366, 335, 362, 250, 20, 239, 311, 363, 34, 340, 329, 204, 280, 92, 87, 124, 167, 249, 239, 74, 254, 142, 0, 160, 49, 12, 39, 309, 365, 258, 266, 226, 284, 139, 354, 152, 315, 339, 195, 108, 191, 331, 304, 266, 116, 52, 359, 184, 130, 57, 241, 292, 214, 277, 183, 333, 17, 273, 101, 344, 380, 120, 242, 97, 43, 187, 350, 248, 355, 17, 273, 295, 147, 319, 179, 218, 14, 40, 82, 376, 122, 103, 17, 215, 353, 327, 181, 293, 172, 72, 18, 272, 356, 251, 131, 271, 330, 78, 15, 222, 370, 111, 202, 94, 261, 148, 154, 308, 271, 109, 381, 104, 378, 324, 255, 63, 374, 29, 165, 383, 265, 271, 358, 133, 207, 229, 73, 89, 11, 84, 93, 143, 262, 197, 128, 328, 85, 263, 210, 190, 31, 119, 53, 71, 201, 345, 114, 128, 330, 33, 6, 320, 257, 168, 158, 127, 256, 228, 377, 129, 26, 328, 268, 177, 41, 193, 13, 138, 151, 178, 188, 153, 30, 270, 328, 381, 157, 219, 25, 318, 64, 59, 286, 238, 60, 200, 102, 140, 141, 375, 155, 175, 91, 146, 349, 32, 110, 106, 336, 102, 140, 141, 189, 232, 246, 79, 302, 367, 47, 244, 125, 1, 77, 140, 90, 208, 325, 88, 341, 81, 198, 326, 323, 23, 364, 65, 279, 164, 369, 290, 45, 162, 253, 211, 3, 56, 126, 337, 117, 301, 58, 287, 368, 75, 212, 100, 144, 95, 213, 260, 136, 145, 150, 101, 223, 252, 291, 4, 70, 206, 67, 163, 176, 220] + 49: [374, 126, 201, 325, 255, 28, 29, 238, 85, 101, 88, 249, 369, 374, 126, 153, 199, 331, 80, 290, 308, 176, 190, 232, 141, 230, 79, 253, 9, 193, 381, 221, 170, 267, 72, 284, 27, 21, 50, 79, 213, 40, 251, 332, 70, 196, 62, 244, 52, 57, 231, 347, 210, 253, 278, 286, 121, 63, 112, 35, 47, 285, 147, 96, 92, 233, 253, 116, 260, 144, 169, 81, 225, 61, 49, 104, 171, 135, 360, 157, 202, 269, 264, 23, 277, 338, 345, 299, 377, 265, 370, 360, 293, 126, 91, 58, 336, 77, 273, 355, 361, 167, 237, 38, 302, 136, 242, 44, 186, 54, 128, 10, 53, 151, 93, 334, 373, 302, 150, 180, 148, 8, 174, 4, 173, 280, 192, 13, 102, 24, 302, 158, 105, 246, 164, 328, 127, 179, 117, 84, 185, 207, 252, 302, 181, 353, 74, 183, 368, 214, 327, 378, 200, 324, 215, 247, 302, 115, 202, 344, 95, 16, 318, 43, 114, 113, 229, 159, 300, 302, 157, 313, 187, 100, 311, 140, 227, 156, 154, 372, 287, 32, 184, 19, 335, 39, 45, 382, 163, 55, 130, 124, 168, 350, 322, 197, 19, 241, 243, 195, 340, 189, 330, 339, 283, 7, 363, 64, 321, 305, 275, 259, 312, 56, 240, 307, 351, 354, 315, 266, 68, 321, 305, 276, 177, 120, 223, 138, 309, 359, 145, 326, 217, 203, 321, 59, 242, 262, 329, 86, 198, 178, 239, 143, 341, 18, 258, 321, 234, 208, 251, 175, 137, 89, 352, 320, 342, 51, 42, 188, 272, 297, 271, 76, 122, 1, 34, 33, 14, 228, 316, 99, 317, 272, 65, 25, 133, 48, 254, 248, 139, 0, 204, 274, 152, 129, 149, 297, 78, 220, 131, 17, 160, 67, 211, 282, 343, 379, 337, 37, 297, 313, 306, 94, 366, 20, 301, 118, 206, 166, 161, 356, 83, 209, 66, 289, 5, 250, 146, 216, 119, 31, 357, 281, 3, 294, 257, 261, 218, 333, 97, 376, 303, 358, 2, 348, 263, 26, 319, 155, 90, 60, 30, 279, 41, 295, 226, 106, 182, 383, 349, 6, 75, 270, 292, 109, 205, 296, 82, 71, 46, 125, 291, 219, 6, 155, 271, 107, 235, 362, 222, 371, 69, 12, 364, 11, 288, 268, 304, 9, 193, 194, 245, 365, 36, 103, 256, 165, 162, 375, 268, 75, 212, 132, 298, 310, 224, 98, 142, 108, 87, 367, 123, 346, 15, 110, 380, 314, 73, 323, 134, 172, 22, 111, 236, 191] + 50: [57, 325, 129, 155, 271, 203, 310, 340, 204, 12, 381, 216, 131, 33, 325, 129, 22, 224, 195, 151, 145, 281, 156, 10, 8, 50, 80, 273, 26, 295, 130, 48, 92, 242, 39, 141, 13, 103, 55, 29, 232, 27, 256, 159, 290, 342, 251, 42, 200, 61, 190, 93, 29, 232, 280, 88, 213, 53, 252, 5, 259, 89, 82, 279, 307, 86, 69, 282, 314, 270, 263, 267, 105, 337, 238, 209, 265, 264, 86, 69, 26, 331, 15, 116, 180, 172, 289, 7, 323, 260, 254, 86, 35, 377, 0, 274, 228, 81, 163, 178, 318, 346, 158, 368, 25, 218, 287, 220, 133, 71, 378, 322, 202, 248, 21, 47, 230, 235, 262, 312, 199, 132, 380, 73, 126, 369, 148, 98, 144, 283, 268, 215, 175, 341, 298, 58, 326, 277, 243, 354, 112, 54, 84, 268, 215, 27, 327, 363, 306, 107, 294, 347, 90, 65, 120, 182, 268, 361, 56, 45, 6, 239, 64, 152, 37, 293, 285, 221, 317, 226, 370, 3, 334, 187, 165, 371, 78, 127, 212, 83, 328, 111, 100, 188, 350, 51, 184, 183, 250, 99, 4, 63, 11, 142, 336, 100, 188, 350, 79, 272, 154, 94, 211, 191, 68, 87, 261, 364, 299, 210, 18, 193, 249, 134, 206, 143, 170, 237, 40, 245, 332, 316, 233, 18, 114, 335, 379, 197, 223, 344, 305, 176, 360, 136, 316, 210, 9, 119, 121, 359, 174, 185, 339, 349, 147, 266, 44, 109, 284, 3, 343, 253, 375, 382, 2, 17, 355, 77, 31, 59, 109, 352, 1, 358, 16, 19, 74, 160, 362, 258, 179, 161, 23, 311, 315, 1, 217, 353, 194, 276, 357, 153, 34, 286, 309, 222, 137, 315, 56, 192, 24, 75, 139, 28, 135, 36, 138, 225, 52, 140, 32, 214, 62, 301, 186, 234, 255, 292, 46, 101, 104, 66, 49, 376, 214, 329, 196, 117, 85, 91, 297, 330, 30, 70, 241, 49, 32, 146, 231, 313, 219, 319, 113, 123, 308, 76, 157, 374, 181, 367, 321, 296, 122, 348, 208, 205, 288, 198, 291, 167, 302, 181, 376, 373, 304, 244, 168, 169, 108, 124, 320, 201, 173, 345, 67, 38, 321, 128, 164, 43, 247, 300, 106, 383, 60, 72, 257, 229, 14, 125, 128, 96, 240, 269, 278, 177, 372, 246, 207, 365, 166, 97, 366, 338, 41, 149, 303, 20, 236, 171, 162, 227, 115, 166, 333, 110, 231, 275, 102, 189, 356, 351, 150, 324, 95, 118] + 51: [322, 17, 62, 32, 143, 341, 90, 75, 30, 242, 199, 293, 255, 108, 17, 300, 325, 247, 331, 33, 164, 27, 363, 272, 128, 290, 31, 102, 140, 270, 374, 162, 219, 317, 258, 43, 125, 29, 225, 137, 52, 211, 188, 92, 47, 251, 306, 89, 353, 39, 305, 348, 137, 182, 126, 235, 259, 161, 34, 195, 328, 333, 215, 132, 156, 373, 102, 229, 95, 299, 382, 91, 361, 223, 145, 158, 217, 202, 373, 102, 163, 60, 20, 268, 18, 171, 334, 109, 4, 376, 249, 375, 287, 343, 203, 216, 86, 190, 205, 115, 378, 83, 71, 76, 375, 284, 65, 104, 294, 359, 280, 324, 312, 53, 184, 117, 244, 106, 287, 163, 40, 35, 237, 192, 239, 170, 93, 7, 277, 326, 221, 113, 82, 295, 11, 183, 160, 157, 365, 336, 15, 302, 377, 221, 193, 358, 146, 346, 212, 252, 265, 297, 339, 354, 367, 151, 189, 123, 126, 250, 5, 285, 368, 364, 349, 238, 218, 147, 150, 73, 12, 229, 260, 46, 74, 360, 369, 383, 25, 19, 63, 347, 241, 208, 82, 124, 13, 245, 321, 236, 54, 276, 165, 70, 327, 241, 12, 67, 286, 23, 380, 329, 84, 196, 167, 68, 274, 231, 371, 208, 144, 286, 220, 279, 42, 311, 315, 55, 9, 154, 261, 37, 99, 337, 129, 335, 254, 366, 313, 100, 112, 116, 266, 22, 37, 357, 136, 233, 263, 303, 204, 41, 379, 61, 214, 352, 36, 372, 186, 264, 135, 85, 107, 209, 69, 213, 240, 66, 187, 21, 372, 186, 72, 152, 281, 198, 119, 141, 288, 278, 88, 77, 138, 372, 186, 144, 87, 118, 56, 222, 262, 332, 232, 148, 185, 131, 210, 16, 337, 330, 26, 275, 370, 44, 80, 78, 14, 174, 291, 210, 186, 64, 105, 314, 344, 248, 10, 191, 298, 296, 355, 121, 101, 111, 176, 304, 134, 308, 207, 48, 350, 340, 28, 0, 2, 6, 24, 122, 283, 49, 271, 309, 50, 194, 319, 257, 120, 139, 110, 24, 127, 153, 130, 345, 351, 289, 94, 316, 1, 133, 173, 110, 24, 356, 282, 243, 142, 181, 320, 381, 159, 57, 227, 226, 200, 323, 180, 177, 103, 81, 310, 224, 45, 197, 96, 169, 155, 200, 24, 228, 253, 269, 97, 79, 3, 338, 273, 230, 179, 267, 178, 323, 342, 168, 246, 172, 59, 362, 114, 307, 201, 175, 256, 206, 98, 342, 301, 8, 292, 166, 234, 51, 58, 149, 318, 38] + 52: [52, 137, 184, 140, 149, 124, 367, 350, 242, 100, 293, 338, 14, 186, 137, 90, 68, 309, 74, 377, 268, 201, 380, 335, 185, 212, 327, 73, 25, 60, 257, 166, 3, 324, 282, 220, 363, 241, 1, 121, 97, 25, 346, 37, 339, 351, 161, 89, 101, 18, 36, 175, 92, 97, 48, 382, 123, 281, 248, 93, 383, 347, 229, 79, 162, 102, 165, 193, 205, 155, 178, 67, 208, 365, 322, 226, 65, 376, 102, 165, 48, 183, 108, 5, 153, 39, 194, 129, 343, 131, 218, 102, 97, 209, 190, 325, 285, 53, 27, 58, 356, 313, 9, 305, 307, 165, 345, 245, 236, 191, 113, 374, 251, 188, 353, 203, 106, 120, 104, 290, 56, 354, 378, 154, 45, 177, 156, 126, 145, 38, 231, 104, 239, 44, 95, 215, 134, 170, 84, 150, 138, 265, 94, 332, 104, 214, 269, 87, 221, 331, 40, 200, 179, 232, 15, 86, 61, 362, 85, 168, 210, 105, 301, 55, 148, 49, 4, 132, 247, 216, 250, 323, 219, 266, 234, 72, 69, 21, 51, 147, 273, 204, 107, 71, 34, 33, 291, 341, 369, 91, 258, 310, 299, 261, 176, 107, 103, 336, 128, 349, 30, 75, 326, 237, 28, 340, 159, 264, 107, 103, 336, 196, 32, 160, 364, 235, 110, 47, 302, 202, 272, 372, 71, 174, 260, 8, 359, 6, 333, 238, 136, 304, 366, 254, 283, 246, 321, 10, 252, 23, 233, 225, 57, 143, 96, 158, 348, 283, 71, 289, 192, 355, 167, 116, 306, 2, 330, 135, 187, 370, 217, 157, 271, 80, 50, 256, 319, 109, 270, 118, 344, 0, 24, 381, 334, 209, 78, 379, 222, 17, 253, 294, 197, 111, 19, 295, 381, 151, 206, 357, 98, 315, 243, 22, 207, 141, 43, 255, 16, 381, 157, 239, 259, 64, 173, 146, 316, 152, 230, 274, 122, 240, 381, 157, 195, 26, 54, 223, 280, 213, 20, 368, 112, 329, 180, 320, 334, 169, 7, 171, 83, 292, 287, 224, 303, 311, 297, 375, 371, 334, 263, 172, 35, 127, 115, 119, 352, 342, 114, 11, 164, 99, 182, 298, 205, 228, 227, 29, 308, 277, 198, 358, 142, 317, 99, 63, 82, 288, 88, 117, 361, 66, 244, 139, 276, 144, 373, 99, 130, 77, 13, 133, 286, 76, 125, 284, 278, 81, 314, 46, 99, 334, 211, 62, 70, 262, 328, 279, 189, 249, 181, 42, 360, 267, 31, 163, 41, 312, 296, 300, 337, 59, 318, 275, 12, 199] + 53: [230, 311, 132, 209, 88, 116, 76, 111, 182, 257, 106, 288, 90, 336, 365, 266, 145, 139, 361, 362, 216, 294, 87, 113, 98, 71, 308, 354, 302, 160, 25, 69, 67, 2, 28, 283, 51, 41, 290, 217, 354, 186, 291, 107, 378, 349, 371, 218, 234, 64, 104, 84, 75, 46, 227, 157, 241, 77, 89, 166, 215, 9, 100, 124, 99, 8, 322, 214, 221, 95, 151, 15, 105, 236, 249, 199, 165, 245, 68, 109, 363, 81, 326, 335, 315, 120, 251, 21, 74, 135, 114, 240, 109, 345, 26, 284, 49, 10, 370, 220, 195, 303, 204, 11, 240, 280, 347, 121, 368, 373, 149, 96, 358, 80, 63, 91, 307, 240, 292, 273, 119, 24, 300, 259, 189, 133, 154, 348, 263, 153, 110, 296, 228, 156, 329, 141, 146, 206, 79, 246, 310, 192, 377, 110, 296, 170, 180, 262, 70, 328, 169, 30, 305, 39, 260, 140, 110, 164, 92, 268, 168, 383, 102, 233, 97, 275, 184, 286, 344, 110, 16, 32, 86, 5, 231, 293, 340, 101, 337, 229, 33, 355, 110, 16, 118, 360, 94, 183, 175, 128, 254, 54, 34, 343, 265, 110, 16, 4, 279, 12, 381, 357, 174, 306, 323, 29, 211, 56, 110, 173, 155, 194, 253, 239, 159, 339, 255, 367, 147, 208, 185, 110, 142, 224, 225, 134, 52, 281, 321, 364, 14, 48, 148, 252, 297, 38, 186, 6, 62, 341, 143, 248, 59, 0, 163, 213, 318, 243, 38, 4, 351, 325, 205, 44, 369, 55, 201, 301, 382, 210, 203, 38, 179, 376, 20, 324, 47, 276, 138, 7, 298, 356, 167, 222, 320, 42, 202, 200, 188, 72, 375, 278, 43, 331, 212, 22, 66, 129, 330, 261, 125, 277, 60, 327, 40, 352, 17, 374, 272, 317, 287, 32, 309, 256, 338, 359, 115, 136, 131, 82, 27, 58, 317, 238, 330, 162, 31, 78, 176, 207, 198, 53, 342, 161, 18, 13, 334, 250, 332, 269, 37, 353, 35, 299, 137, 1, 93, 316, 13, 287, 347, 285, 171, 152, 126, 65, 19, 264, 83, 61, 223, 270, 334, 311, 103, 127, 172, 122, 36, 242, 258, 372, 274, 191, 73, 350, 379, 237, 319, 313, 150, 57, 123, 108, 193, 267, 312, 73, 23, 295, 130, 219, 112, 282, 196, 232, 45, 366, 187, 304, 73, 23, 295, 271, 3, 346, 226, 178, 117, 380, 85, 190, 314, 247, 350, 250, 144, 50, 197, 235, 158, 333, 289, 177, 181, 244] + 54: [295, 240, 240, 3, 129, 225, 135, 171, 128, 6, 31, 138, 344, 370, 260, 324, 120, 156, 217, 239, 304, 364, 12, 343, 379, 36, 370, 80, 306, 187, 41, 17, 131, 366, 23, 235, 206, 291, 383, 370, 369, 299, 26, 322, 272, 381, 179, 96, 139, 226, 35, 19, 370, 214, 133, 118, 197, 91, 302, 48, 273, 59, 367, 37, 126, 284, 164, 72, 221, 288, 5, 186, 68, 166, 259, 52, 51, 354, 284, 82, 95, 345, 70, 265, 204, 111, 173, 243, 362, 307, 378, 337, 164, 38, 228, 47, 357, 67, 326, 195, 213, 233, 169, 358, 380, 164, 348, 75, 310, 190, 184, 136, 81, 334, 102, 285, 13, 153, 257, 280, 314, 215, 198, 183, 246, 141, 110, 331, 77, 340, 296, 150, 30, 329, 276, 94, 376, 293, 347, 15, 261, 2, 349, 363, 155, 176, 93, 332, 182, 21, 107, 114, 194, 227, 116, 200, 363, 155, 208, 44, 181, 24, 312, 85, 18, 63, 359, 134, 202, 363, 155, 254, 282, 27, 142, 157, 327, 151, 88, 263, 210, 148, 363, 216, 281, 305, 101, 109, 55, 144, 266, 234, 269, 350, 127, 363, 231, 230, 40, 64, 45, 97, 301, 316, 178, 274, 222, 207, 363, 231, 230, 117, 32, 220, 355, 264, 43, 247, 289, 241, 297, 253, 339, 286, 87, 121, 279, 237, 163, 244, 308, 89, 65, 160, 294, 339, 286, 57, 58, 382, 368, 373, 9, 346, 108, 104, 205, 294, 223, 172, 25, 333, 335, 318, 278, 351, 74, 29, 112, 78, 294, 223, 8, 352, 99, 50, 100, 61, 191, 145, 236, 252, 193, 283, 287, 341, 290, 375, 34, 103, 209, 167, 251, 60, 154, 130, 283, 339, 299, 309, 158, 92, 125, 242, 146, 371, 203, 360, 0, 71, 356, 95, 330, 10, 152, 140, 122, 162, 245, 56, 270, 232, 319, 356, 53, 119, 76, 159, 54, 137, 377, 192, 66, 292, 320, 328, 11, 98, 7, 196, 325, 365, 298, 275, 14, 211, 256, 342, 189, 11, 38, 317, 143, 323, 315, 255, 262, 219, 79, 4, 212, 338, 267, 20, 180, 161, 277, 177, 201, 62, 238, 199, 374, 336, 147, 258, 174, 311, 115, 84, 1, 22, 33, 229, 248, 303, 361, 83, 258, 250, 86, 218, 321, 105, 69, 300, 168, 249, 188, 165, 90, 39, 280, 372, 113, 124, 49, 73, 175, 224, 268, 28, 106, 42, 42, 348, 16, 132, 185, 170, 313, 353, 271, 46, 149, 123] + 55: [56, 111, 323, 226, 60, 206, 228, 193, 294, 104, 58, 83, 131, 56, 307, 323, 71, 272, 233, 194, 277, 1, 97, 177, 334, 222, 326, 238, 74, 99, 114, 102, 240, 287, 61, 356, 248, 21, 205, 113, 16, 304, 223, 176, 348, 360, 257, 127, 173, 376, 175, 317, 72, 252, 74, 134, 3, 17, 135, 19, 133, 208, 141, 92, 196, 159, 80, 382, 293, 179, 5, 306, 185, 367, 93, 120, 220, 292, 202, 311, 347, 363, 271, 284, 27, 278, 63, 118, 13, 181, 245, 202, 80, 347, 180, 124, 350, 246, 171, 214, 38, 64, 244, 144, 152, 39, 78, 180, 156, 112, 67, 332, 143, 15, 162, 295, 215, 28, 165, 121, 269, 312, 371, 231, 23, 198, 201, 161, 100, 18, 88, 89, 48, 269, 316, 375, 265, 128, 212, 372, 324, 36, 51, 88, 339, 90, 106, 349, 172, 286, 322, 365, 219, 183, 298, 34, 327, 182, 211, 76, 164, 274, 378, 101, 314, 137, 315, 158, 351, 154, 357, 247, 29, 86, 190, 260, 33, 22, 243, 125, 142, 379, 154, 370, 273, 199, 138, 276, 251, 94, 87, 338, 354, 230, 24, 154, 116, 41, 40, 331, 259, 187, 263, 188, 373, 77, 191, 204, 154, 116, 353, 115, 216, 325, 343, 358, 82, 126, 145, 32, 148, 43, 340, 48, 115, 221, 14, 224, 84, 296, 129, 147, 345, 59, 270, 340, 73, 130, 26, 95, 236, 207, 264, 209, 68, 362, 250, 256, 218, 90, 130, 383, 163, 79, 119, 103, 225, 242, 169, 69, 81, 308, 123, 70, 366, 210, 380, 52, 117, 297, 261, 75, 45, 81, 62, 267, 65, 300, 105, 217, 85, 313, 55, 275, 288, 25, 235, 308, 381, 2, 8, 46, 42, 184, 258, 335, 280, 364, 305, 355, 146, 192, 341, 12, 344, 136, 318, 281, 253, 166, 282, 96, 151, 146, 227, 291, 333, 189, 53, 241, 37, 268, 203, 262, 109, 174, 234, 303, 76, 368, 289, 377, 157, 302, 168, 150, 195, 9, 174, 11, 299, 132, 290, 197, 178, 10, 232, 319, 374, 57, 167, 174, 11, 299, 49, 44, 239, 140, 122, 342, 35, 107, 108, 229, 174, 7, 381, 249, 279, 170, 47, 346, 160, 359, 369, 155, 285, 309, 20, 123, 301, 213, 352, 361, 255, 237, 30, 50, 110, 283, 4, 20, 6, 200, 320, 337, 54, 254, 153, 321, 66, 266, 310, 98, 98, 328, 29, 0, 91, 329, 149, 31, 336, 139, 330, 186] + 56: [279, 225, 9, 275, 66, 160, 31, 1, 218, 347, 157, 108, 76, 100, 225, 156, 48, 245, 0, 121, 123, 201, 274, 93, 203, 111, 177, 351, 75, 284, 159, 233, 22, 17, 183, 290, 24, 256, 146, 231, 184, 169, 161, 244, 137, 206, 248, 115, 139, 144, 45, 91, 231, 158, 30, 71, 288, 103, 317, 286, 190, 291, 170, 216, 165, 231, 303, 259, 140, 14, 344, 331, 34, 96, 293, 152, 272, 173, 231, 267, 259, 46, 283, 69, 35, 110, 145, 307, 204, 325, 202, 8, 296, 322, 188, 329, 38, 295, 33, 257, 323, 211, 358, 3, 195, 88, 340, 346, 63, 285, 79, 133, 314, 4, 70, 336, 208, 195, 163, 340, 320, 357, 376, 179, 127, 94, 232, 373, 52, 380, 195, 98, 106, 219, 19, 281, 136, 255, 180, 200, 43, 326, 224, 311, 155, 241, 251, 97, 359, 355, 185, 62, 330, 254, 215, 235, 321, 186, 106, 292, 327, 40, 338, 164, 298, 28, 350, 335, 90, 321, 53, 241, 2, 83, 128, 324, 42, 122, 37, 67, 217, 318, 343, 116, 5, 6, 207, 193, 92, 56, 313, 367, 49, 7, 109, 167, 107, 30, 162, 150, 250, 21, 243, 134, 310, 337, 68, 194, 242, 15, 371, 99, 80, 58, 230, 364, 339, 372, 64, 32, 301, 242, 15, 87, 316, 381, 197, 12, 151, 54, 112, 86, 36, 74, 265, 294, 304, 18, 332, 312, 252, 11, 119, 228, 258, 234, 315, 265, 273, 5, 117, 101, 278, 181, 129, 378, 16, 29, 366, 142, 44, 294, 75, 302, 362, 342, 198, 138, 262, 375, 345, 239, 237, 44, 294, 304, 41, 154, 73, 166, 13, 55, 132, 287, 349, 174, 260, 220, 263, 363, 192, 253, 135, 297, 221, 289, 266, 365, 102, 260, 213, 196, 125, 309, 27, 374, 209, 271, 379, 47, 382, 143, 260, 213, 20, 187, 223, 369, 149, 214, 353, 226, 361, 261, 51, 126, 220, 89, 333, 118, 148, 305, 270, 210, 319, 60, 176, 25, 191, 220, 147, 308, 383, 182, 354, 153, 84, 172, 131, 26, 229, 199, 81, 147, 39, 77, 300, 268, 227, 299, 175, 130, 59, 247, 178, 240, 249, 264, 72, 205, 280, 50, 341, 124, 222, 370, 171, 178, 240, 236, 104, 238, 246, 269, 282, 348, 65, 78, 277, 61, 82, 240, 189, 328, 168, 10, 105, 368, 95, 276, 113, 212, 114, 82, 240, 356, 306, 120, 334, 23, 141, 360, 57, 352, 377, 85] + 57: [157, 148, 125, 309, 163, 277, 377, 351, 41, 200, 48, 291, 234, 231, 128, 125, 22, 44, 27, 208, 337, 29, 4, 361, 292, 245, 231, 148, 118, 79, 123, 42, 203, 149, 162, 32, 222, 206, 105, 93, 101, 268, 283, 354, 173, 54, 209, 17, 188, 194, 189, 305, 11, 235, 8, 317, 75, 356, 349, 352, 26, 303, 308, 10, 293, 342, 65, 335, 197, 251, 298, 116, 321, 311, 168, 383, 193, 69, 159, 205, 344, 345, 80, 191, 382, 196, 261, 59, 239, 52, 280, 14, 25, 141, 267, 365, 73, 112, 166, 363, 357, 50, 114, 233, 14, 205, 324, 60, 43, 213, 328, 136, 371, 281, 256, 221, 15, 230, 25, 130, 302, 172, 110, 380, 296, 215, 255, 67, 212, 19, 276, 366, 170, 74, 58, 152, 38, 210, 145, 265, 287, 100, 285, 276, 104, 170, 302, 373, 346, 55, 282, 18, 87, 260, 30, 227, 276, 25, 320, 169, 167, 177, 47, 164, 379, 126, 36, 340, 153, 31, 326, 124, 84, 238, 243, 225, 226, 247, 24, 138, 129, 229, 381, 326, 72, 318, 272, 341, 106, 155, 98, 252, 61, 142, 301, 158, 326, 359, 350, 338, 70, 358, 71, 334, 241, 66, 278, 181, 16, 140, 331, 64, 306, 96, 92, 190, 323, 1, 224, 195, 37, 254, 187, 53, 217, 179, 176, 13, 68, 49, 0, 242, 269, 77, 254, 63, 137, 300, 161, 322, 279, 348, 355, 333, 109, 336, 131, 254, 156, 320, 74, 91, 113, 183, 362, 178, 122, 139, 360, 343, 254, 7, 94, 127, 347, 9, 12, 28, 81, 33, 273, 266, 376, 254, 198, 207, 88, 185, 20, 102, 3, 237, 236, 107, 244, 262, 133, 187, 286, 154, 253, 214, 120, 369, 132, 5, 78, 310, 304, 314, 368, 286, 154, 39, 51, 289, 313, 175, 270, 375, 294, 143, 146, 246, 207, 160, 35, 249, 147, 23, 330, 248, 370, 258, 56, 146, 246, 331, 180, 62, 83, 184, 199, 332, 299, 232, 339, 86, 259, 89, 257, 151, 250, 312, 21, 171, 240, 307, 374, 82, 201, 259, 89, 223, 117, 182, 46, 111, 90, 288, 135, 364, 115, 202, 295, 165, 72, 150, 319, 297, 325, 353, 290, 378, 192, 218, 367, 295, 57, 108, 204, 316, 228, 284, 216, 186, 45, 76, 271, 372, 121, 165, 94, 6, 99, 134, 95, 329, 315, 219, 263, 327, 34, 220, 85, 274, 174, 144, 40, 119, 2, 97, 211, 103, 264, 275] + 58: [263, 222, 122, 68, 285, 187, 110, 241, 170, 360, 246, 313, 158, 248, 222, 9, 262, 329, 229, 378, 65, 195, 354, 103, 88, 340, 156, 101, 93, 272, 171, 139, 0, 172, 64, 296, 304, 361, 3, 156, 373, 206, 31, 242, 251, 236, 228, 293, 138, 32, 89, 279, 220, 194, 299, 42, 197, 314, 215, 123, 283, 41, 336, 188, 33, 220, 194, 235, 119, 66, 375, 234, 161, 35, 100, 39, 267, 38, 220, 311, 177, 189, 190, 381, 83, 21, 113, 5, 34, 233, 292, 220, 311, 177, 16, 130, 284, 135, 13, 230, 157, 146, 193, 58, 275, 98, 312, 203, 224, 48, 90, 43, 265, 24, 145, 23, 166, 275, 271, 312, 367, 198, 192, 238, 347, 287, 102, 153, 291, 80, 77, 98, 322, 191, 185, 331, 160, 186, 94, 317, 369, 165, 252, 377, 27, 67, 342, 315, 258, 131, 253, 173, 268, 51, 52, 280, 377, 348, 289, 152, 28, 380, 346, 327, 237, 303, 84, 78, 364, 377, 344, 114, 341, 106, 383, 63, 300, 243, 142, 120, 212, 147, 377, 210, 114, 18, 260, 143, 326, 334, 179, 330, 269, 217, 150, 377, 211, 298, 324, 20, 207, 339, 2, 19, 213, 159, 320, 164, 377, 245, 122, 141, 4, 25, 382, 154, 133, 359, 349, 310, 370, 377, 245, 22, 45, 352, 55, 306, 11, 70, 294, 137, 363, 335, 377, 245, 298, 196, 274, 239, 124, 176, 136, 163, 290, 301, 308, 377, 14, 49, 47, 328, 1, 53, 371, 115, 261, 216, 91, 309, 249, 356, 214, 175, 132, 129, 240, 148, 338, 86, 44, 15, 168, 151, 7, 355, 232, 12, 350, 302, 144, 118, 29, 57, 140, 71, 76, 92, 358, 376, 73, 281, 155, 278, 50, 54, 167, 365, 134, 126, 205, 351, 85, 316, 244, 183, 288, 8, 221, 46, 17, 184, 126, 92, 69, 295, 307, 202, 250, 345, 182, 95, 109, 218, 125, 56, 305, 270, 36, 104, 325, 332, 208, 180, 128, 223, 74, 10, 56, 205, 72, 6, 60, 82, 75, 379, 226, 372, 117, 323, 319, 81, 366, 199, 368, 255, 59, 333, 318, 225, 254, 200, 286, 209, 81, 273, 26, 112, 111, 266, 149, 357, 219, 276, 337, 30, 201, 81, 305, 257, 178, 162, 62, 227, 107, 353, 282, 374, 127, 204, 61, 366, 256, 116, 97, 99, 37, 231, 105, 264, 247, 174, 108, 79, 259, 169, 40, 277, 181, 121, 362, 297, 87, 96, 343, 321] + 59: [332, 173, 76, 178, 112, 218, 10, 96, 43, 115, 148, 272, 235, 140, 200, 150, 317, 164, 267, 354, 289, 253, 207, 108, 282, 99, 140, 200, 185, 151, 342, 245, 69, 242, 34, 326, 98, 318, 104, 73, 363, 31, 372, 81, 143, 236, 32, 287, 275, 119, 56, 90, 209, 251, 18, 186, 160, 174, 54, 48, 380, 11, 273, 270, 188, 209, 363, 331, 23, 128, 286, 269, 51, 129, 89, 82, 360, 189, 113, 70, 288, 262, 86, 338, 344, 2, 187, 145, 368, 192, 154, 182, 95, 130, 9, 85, 225, 214, 199, 63, 132, 166, 87, 55, 327, 260, 346, 295, 16, 302, 238, 6, 84, 28, 8, 276, 116, 327, 260, 221, 65, 254, 373, 158, 292, 64, 24, 321, 264, 351, 327, 59, 263, 38, 57, 334, 177, 13, 217, 67, 319, 305, 341, 327, 59, 371, 328, 161, 62, 369, 228, 134, 17, 216, 219, 271, 33, 202, 381, 196, 278, 20, 356, 194, 71, 3, 312, 335, 268, 30, 195, 311, 309, 109, 181, 361, 337, 322, 343, 237, 141, 126, 105, 202, 220, 255, 176, 22, 306, 299, 138, 210, 296, 378, 139, 285, 195, 220, 58, 133, 19, 222, 314, 246, 44, 307, 78, 345, 75, 94, 66, 26, 114, 142, 339, 156, 41, 277, 103, 293, 308, 224, 7, 234, 42, 279, 364, 203, 155, 184, 153, 301, 240, 83, 224, 94, 298, 281, 40, 280, 366, 353, 124, 330, 165, 350, 215, 46, 7, 294, 324, 35, 252, 120, 68, 208, 93, 241, 77, 323, 25, 370, 66, 80, 290, 29, 47, 107, 27, 249, 53, 125, 79, 25, 370, 204, 377, 175, 100, 259, 36, 325, 261, 230, 205, 122, 25, 7, 247, 144, 91, 376, 179, 362, 193, 106, 118, 14, 15, 170, 229, 352, 65, 168, 163, 146, 101, 50, 359, 316, 239, 152, 170, 233, 352, 265, 127, 169, 180, 131, 136, 74, 257, 1, 190, 170, 5, 346, 213, 12, 248, 258, 274, 149, 88, 284, 347, 72, 250, 229, 173, 266, 223, 383, 348, 349, 231, 37, 283, 39, 310, 250, 229, 340, 382, 304, 315, 355, 0, 333, 191, 201, 171, 52, 256, 226, 340, 232, 45, 320, 137, 123, 110, 336, 358, 121, 111, 313, 147, 374, 243, 212, 60, 329, 206, 227, 183, 357, 244, 167, 97, 379, 291, 300, 159, 162, 198, 61, 135, 49, 102, 367, 4, 197, 379, 234, 172, 297, 211, 92, 365, 157, 21, 303, 375, 117] + 60: [284, 133, 174, 119, 184, 37, 162, 33, 192, 252, 261, 340, 118, 102, 133, 78, 180, 203, 93, 344, 250, 300, 330, 54, 173, 89, 94, 113, 40, 365, 59, 255, 88, 7, 369, 328, 258, 227, 226, 123, 126, 302, 2, 140, 150, 61, 273, 349, 127, 41, 191, 317, 123, 83, 222, 345, 27, 0, 377, 53, 296, 251, 145, 17, 136, 123, 83, 48, 166, 161, 114, 92, 30, 182, 361, 42, 86, 288, 236, 126, 334, 366, 82, 20, 304, 171, 225, 338, 378, 275, 109, 121, 39, 40, 112, 231, 358, 267, 149, 130, 376, 204, 249, 63, 313, 60, 208, 76, 213, 240, 342, 360, 256, 363, 298, 164, 218, 313, 343, 155, 35, 143, 132, 24, 85, 79, 262, 374, 72, 154, 313, 244, 65, 34, 55, 362, 56, 134, 331, 383, 291, 306, 207, 313, 347, 312, 239, 235, 221, 11, 196, 190, 228, 137, 71, 10, 313, 372, 310, 290, 117, 6, 110, 142, 368, 168, 68, 159, 375, 268, 139, 81, 327, 241, 277, 116, 323, 371, 67, 50, 175, 355, 268, 60, 181, 108, 131, 183, 215, 70, 144, 307, 356, 289, 242, 381, 303, 297, 74, 253, 152, 264, 187, 322, 178, 320, 32, 73, 381, 139, 185, 205, 172, 75, 382, 146, 100, 202, 224, 301, 201, 335, 129, 194, 49, 232, 217, 5, 90, 276, 156, 379, 43, 210, 351, 220, 199, 309, 364, 189, 57, 186, 64, 125, 97, 4, 352, 351, 220, 199, 15, 246, 58, 176, 21, 14, 105, 229, 135, 373, 351, 47, 339, 98, 169, 336, 3, 107, 219, 325, 111, 367, 308, 351, 47, 316, 329, 263, 283, 332, 285, 370, 209, 19, 163, 87, 124, 170, 153, 28, 359, 151, 274, 233, 286, 197, 147, 295, 101, 124, 115, 120, 294, 245, 318, 8, 346, 12, 315, 96, 281, 157, 124, 195, 310, 353, 223, 269, 188, 66, 341, 287, 179, 177, 321, 124, 243, 265, 311, 238, 69, 279, 52, 380, 270, 160, 165, 22, 124, 266, 212, 354, 99, 84, 38, 31, 314, 45, 280, 292, 214, 257, 16, 230, 319, 278, 1, 305, 141, 18, 333, 29, 138, 77, 257, 128, 247, 234, 206, 259, 25, 26, 148, 282, 211, 326, 348, 257, 260, 167, 337, 122, 51, 237, 293, 95, 23, 104, 350, 299, 158, 16, 174, 193, 271, 248, 80, 91, 357, 324, 216, 103, 44, 158, 16, 265, 46, 272, 36, 9, 200, 106, 13, 198, 254, 62] + 61: [231, 381, 235, 263, 261, 233, 245, 125, 4, 239, 138, 276, 116, 303, 381, 215, 213, 32, 268, 104, 30, 85, 358, 331, 289, 266, 230, 363, 238, 307, 309, 366, 69, 341, 144, 25, 95, 377, 319, 172, 45, 142, 248, 20, 362, 2, 79, 176, 271, 180, 226, 208, 61, 197, 294, 161, 156, 166, 97, 273, 225, 74, 49, 267, 320, 61, 197, 324, 307, 227, 29, 206, 37, 71, 54, 265, 302, 343, 73, 380, 371, 248, 191, 118, 329, 219, 106, 244, 164, 285, 149, 73, 1, 68, 143, 337, 77, 336, 365, 194, 63, 201, 279, 357, 5, 264, 17, 119, 92, 297, 257, 242, 332, 91, 293, 131, 334, 136, 184, 12, 66, 133, 84, 3, 9, 340, 364, 262, 183, 76, 335, 209, 100, 216, 31, 44, 23, 8, 67, 18, 232, 50, 367, 108, 59, 100, 107, 181, 82, 200, 78, 26, 354, 105, 256, 58, 360, 241, 103, 288, 110, 304, 290, 153, 283, 251, 7, 382, 16, 93, 203, 167, 86, 361, 224, 109, 80, 204, 274, 48, 222, 321, 93, 10, 253, 291, 159, 113, 102, 38, 14, 345, 212, 64, 356, 211, 19, 353, 129, 188, 374, 33, 186, 298, 162, 236, 220, 171, 211, 270, 36, 107, 349, 27, 145, 375, 81, 187, 28, 56, 96, 120, 117, 36, 115, 369, 11, 259, 322, 152, 370, 182, 60, 40, 157, 178, 218, 216, 286, 252, 202, 292, 124, 163, 146, 250, 0, 221, 299, 315, 327, 15, 135, 207, 214, 269, 123, 246, 39, 325, 221, 299, 21, 88, 160, 83, 147, 346, 196, 234, 13, 175, 296, 221, 122, 315, 192, 243, 300, 158, 137, 284, 130, 305, 151, 128, 221, 323, 167, 86, 87, 65, 308, 22, 132, 255, 314, 254, 350, 170, 205, 330, 112, 75, 316, 249, 89, 317, 6, 185, 355, 229, 179, 127, 344, 168, 34, 313, 378, 383, 98, 193, 373, 281, 287, 179, 127, 372, 88, 282, 190, 52, 306, 141, 101, 111, 210, 376, 179, 127, 51, 161, 258, 348, 47, 126, 342, 352, 272, 277, 189, 338, 127, 278, 55, 228, 150, 260, 174, 199, 310, 339, 347, 94, 338, 351, 344, 57, 177, 379, 173, 72, 148, 70, 140, 35, 155, 338, 165, 195, 333, 42, 121, 139, 154, 359, 311, 223, 318, 53, 46, 326, 198, 263, 43, 90, 328, 275, 240, 99, 24, 134, 217, 46, 247, 324, 237, 169, 114, 368, 280, 295, 301, 312, 62, 41] +layer_updates_per_iter: 0 +num_slots: 416 diff --git a/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots384.yaml b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots384.yaml new file mode 100644 index 000000000000..c55d3f2ebf7f --- /dev/null +++ b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 356, 245, 279, 17, 49, 224, 150, 151, 345, 86, 348, 207, 241, 332, 163, 347, 153, 104, 62, 232, 321, 376, 176, 278, 198, 280, 282, 55, 143, 205, 54, 11, 105, 313, 56, 354, 297, 129, 258, 324, 368, 7, 170, 149, 309, 314, 38, 52, 5, 91, 172, 227, 4, 108, 249, 363, 372, 275, 59, 215, 97, 106, 270, 299, 73, 267, 67, 260, 50, 252, 98, 294, 352, 316, 340, 380, 12, 219, 179, 381, 9, 357, 250, 298, 92, 287, 24, 228, 128, 344, 288, 118, 13, 110, 303, 111, 222, 225, 45, 79, 203, 217, 0, 302, 322, 155, 162, 264, 230, 312, 188, 35, 21, 202, 80, 218, 255, 70, 139, 293, 318, 3, 349, 336, 78, 72, 243, 117, 34, 283, 156, 10, 124, 265, 200, 261, 114, 248, 337, 123, 268, 89, 234, 66, 39, 133, 244, 335, 103, 146, 331, 116, 178, 216, 192, 296, 100, 379, 292, 83, 147, 326, 237, 342, 20, 210, 286, 338, 42, 212, 253, 53, 41, 113, 334, 194, 231, 285, 164, 18, 142, 304, 121, 281, 364, 273, 130, 370, 171, 201, 247, 378, 74, 371, 135, 95, 184, 173, 259, 351, 84, 119, 235, 193, 346, 182, 169, 51, 26, 204, 242, 327, 289, 65, 63, 37, 333, 144, 209, 360, 359, 307, 284, 131, 196, 341, 233, 109, 186, 361, 311, 112, 138, 44, 148, 31, 27, 199, 358, 87, 85, 308, 257, 213, 136, 240, 365, 140, 246, 19, 366, 180, 238, 320, 195, 82, 127, 328, 263, 362, 47, 102, 350, 161, 40, 64, 206, 315, 175, 2, 152, 301, 239, 220, 125, 167, 271, 329, 226, 291, 339, 8, 88, 310, 58, 373, 61, 166, 256, 236, 254, 60, 134, 330, 290, 71, 159, 383, 177, 76, 158, 190, 295, 122, 81, 30, 29, 157, 48, 343, 68, 276, 16, 272, 126, 22, 141, 181, 33, 101, 107, 183, 43, 57, 90, 36, 174, 94, 28, 300, 208, 77, 160, 262, 375, 223, 377, 187, 32, 214, 317, 46, 137, 75, 269, 99, 369, 211, 1, 25, 221, 197, 382, 93, 14, 145, 6, 154, 229, 305, 15, 96, 115, 306, 353, 69, 191, 319, 367, 266, 251, 132, 23, 189, 374, 325, 355, 323, 168, 277, 120, 274, 185] + 1: [163, 137, 382, 177, 67, 182, 225, 178, 380, 1, 308, 126, 149, 152, 254, 40, 6, 366, 70, 123, 73, 294, 299, 271, 185, 283, 29, 138, 269, 275, 86, 242, 125, 281, 74, 212, 197, 288, 261, 109, 114, 213, 176, 228, 289, 306, 376, 278, 236, 95, 205, 189, 164, 79, 348, 330, 65, 139, 286, 326, 19, 175, 33, 321, 290, 35, 230, 350, 352, 94, 323, 346, 344, 240, 333, 103, 83, 369, 331, 129, 316, 90, 151, 221, 132, 361, 16, 45, 258, 159, 28, 157, 312, 345, 265, 255, 319, 22, 277, 169, 347, 377, 82, 272, 81, 31, 18, 106, 50, 196, 378, 168, 39, 179, 75, 41, 105, 334, 250, 153, 351, 224, 46, 273, 9, 287, 71, 173, 216, 78, 285, 301, 218, 108, 92, 335, 143, 4, 53, 116, 381, 69, 360, 342, 61, 231, 237, 76, 121, 362, 305, 238, 338, 161, 318, 17, 156, 295, 165, 62, 48, 186, 293, 11, 97, 96, 8, 113, 54, 262, 49, 356, 357, 375, 372, 220, 340, 188, 373, 135, 66, 124, 155, 191, 144, 276, 359, 257, 199, 310, 15, 215, 365, 174, 180, 72, 127, 3, 111, 134, 141, 200, 118, 80, 263, 291, 329, 131, 30, 5, 325, 328, 110, 300, 244, 248, 233, 383, 243, 59, 303, 211, 260, 89, 234, 317, 355, 146, 166, 162, 142, 379, 214, 219, 47, 282, 358, 88, 38, 171, 101, 133, 102, 298, 122, 140, 170, 204, 364, 249, 332, 107, 115, 229, 154, 68, 87, 324, 315, 26, 104, 227, 181, 268, 226, 353, 77, 27, 148, 279, 371, 309, 193, 183, 253, 209, 208, 128, 322, 374, 307, 349, 339, 304, 267, 363, 327, 368, 42, 201, 158, 91, 167, 198, 120, 292, 251, 235, 44, 85, 150, 43, 21, 241, 192, 270, 0, 210, 63, 58, 56, 23, 24, 25, 320, 195, 37, 145, 32, 247, 245, 266, 274, 99, 187, 84, 194, 280, 223, 52, 206, 93, 203, 264, 202, 239, 147, 60, 98, 256, 7, 313, 222, 259, 370, 2, 20, 14, 296, 184, 136, 57, 172, 36, 284, 51, 311, 13, 302, 55, 12, 232, 119, 314, 190, 130, 246, 297, 160, 367, 207, 217, 10, 252, 343, 64, 117, 337, 341, 354, 112, 336, 34, 100] + 2: [143, 313, 215, 3, 242, 62, 86, 230, 96, 102, 28, 5, 189, 34, 33, 149, 101, 111, 191, 166, 66, 56, 70, 135, 133, 105, 45, 368, 146, 217, 82, 24, 323, 196, 174, 294, 130, 299, 273, 87, 27, 68, 203, 210, 361, 120, 76, 138, 332, 253, 16, 171, 175, 186, 254, 129, 372, 154, 39, 142, 318, 176, 179, 2, 195, 77, 281, 35, 178, 264, 261, 57, 243, 291, 107, 95, 183, 114, 267, 307, 378, 310, 37, 177, 369, 249, 315, 266, 316, 32, 337, 354, 300, 199, 381, 258, 248, 108, 89, 295, 232, 19, 314, 122, 351, 343, 298, 259, 321, 197, 162, 192, 185, 84, 350, 173, 355, 73, 278, 271, 42, 31, 51, 22, 274, 200, 11, 235, 30, 6, 342, 204, 140, 170, 18, 347, 50, 156, 188, 320, 187, 201, 148, 198, 349, 41, 65, 241, 193, 15, 160, 276, 364, 106, 312, 252, 29, 104, 362, 23, 211, 134, 334, 238, 376, 331, 319, 231, 292, 302, 288, 99, 251, 167, 59, 98, 182, 336, 227, 222, 13, 353, 380, 88, 287, 358, 339, 237, 246, 375, 161, 379, 53, 112, 194, 223, 290, 168, 304, 139, 341, 14, 81, 221, 100, 247, 75, 78, 153, 155, 97, 284, 131, 280, 245, 118, 184, 172, 296, 26, 83, 63, 340, 244, 303, 159, 209, 165, 344, 17, 356, 263, 55, 269, 67, 92, 20, 383, 218, 297, 46, 163, 265, 308, 309, 169, 103, 69, 256, 136, 12, 49, 257, 150, 94, 0, 25, 239, 346, 4, 272, 141, 144, 352, 329, 54, 71, 371, 335, 205, 48, 74, 113, 382, 283, 345, 214, 240, 127, 64, 228, 301, 208, 202, 363, 40, 229, 374, 43, 132, 93, 115, 212, 91, 377, 180, 225, 357, 1, 137, 151, 125, 60, 213, 110, 109, 322, 124, 157, 286, 44, 306, 123, 220, 324, 325, 328, 279, 333, 85, 285, 311, 61, 79, 206, 7, 90, 224, 293, 126, 262, 327, 21, 58, 128, 236, 360, 277, 10, 145, 348, 181, 250, 116, 9, 38, 305, 121, 268, 365, 152, 219, 36, 275, 367, 47, 52, 207, 233, 359, 216, 226, 370, 373, 338, 8, 164, 158, 117, 72, 234, 147, 317, 260, 289, 80, 326, 119, 270, 282, 255, 190, 366, 330] + 3: [66, 79, 27, 194, 71, 35, 51, 350, 13, 192, 157, 363, 158, 49, 213, 337, 26, 102, 178, 342, 164, 250, 325, 138, 287, 74, 177, 119, 290, 29, 202, 87, 364, 84, 142, 359, 229, 312, 92, 155, 181, 41, 274, 217, 235, 166, 94, 355, 273, 135, 48, 268, 125, 144, 366, 43, 216, 353, 20, 361, 281, 232, 7, 83, 381, 259, 110, 321, 86, 101, 283, 354, 183, 379, 251, 73, 156, 191, 329, 143, 241, 368, 75, 89, 124, 315, 60, 152, 33, 226, 332, 233, 324, 59, 311, 370, 34, 22, 18, 313, 275, 314, 31, 225, 208, 333, 348, 318, 334, 252, 6, 137, 139, 238, 343, 120, 358, 10, 189, 231, 221, 239, 316, 36, 64, 240, 264, 136, 78, 162, 289, 288, 32, 262, 374, 297, 345, 204, 108, 299, 55, 380, 197, 169, 105, 307, 113, 255, 383, 328, 100, 306, 248, 286, 187, 70, 67, 294, 15, 376, 61, 356, 68, 228, 153, 254, 133, 285, 245, 116, 220, 88, 234, 242, 45, 360, 253, 1, 95, 127, 265, 320, 69, 173, 167, 341, 335, 257, 198, 278, 47, 301, 310, 365, 319, 40, 3, 263, 209, 14, 19, 56, 97, 62, 151, 145, 50, 331, 42, 186, 201, 207, 336, 2, 227, 267, 212, 77, 296, 219, 182, 25, 21, 382, 271, 279, 161, 172, 99, 277, 351, 114, 176, 98, 375, 218, 103, 256, 53, 57, 327, 65, 117, 246, 44, 104, 214, 160, 196, 340, 96, 298, 188, 305, 367, 106, 165, 130, 85, 346, 362, 293, 269, 206, 190, 0, 134, 372, 140, 121, 338, 159, 369, 54, 90, 148, 272, 308, 349, 244, 326, 185, 5, 17, 52, 199, 292, 222, 260, 377, 39, 302, 163, 46, 230, 171, 317, 131, 261, 80, 24, 150, 81, 180, 215, 339, 63, 168, 170, 58, 211, 37, 122, 304, 195, 174, 11, 223, 12, 118, 357, 109, 8, 154, 295, 203, 149, 111, 243, 300, 126, 224, 200, 236, 344, 270, 112, 129, 266, 280, 76, 30, 82, 237, 276, 115, 322, 184, 249, 128, 291, 91, 123, 373, 132, 72, 282, 28, 309, 93, 258, 179, 378, 16, 323, 210, 107, 284, 141, 205, 147, 330, 352, 23, 175, 4, 347, 193, 371, 247, 9, 146, 303, 38] + 4: [343, 296, 15, 87, 142, 188, 37, 199, 24, 250, 172, 281, 5, 171, 143, 348, 342, 13, 197, 155, 139, 41, 77, 372, 361, 364, 218, 160, 333, 8, 73, 251, 110, 260, 159, 141, 54, 244, 243, 116, 307, 53, 330, 256, 185, 359, 149, 323, 269, 74, 232, 383, 135, 322, 288, 328, 138, 309, 294, 50, 184, 78, 51, 175, 291, 327, 237, 118, 71, 109, 286, 156, 126, 381, 115, 129, 123, 279, 173, 164, 368, 144, 102, 373, 326, 363, 299, 337, 192, 17, 79, 67, 90, 58, 104, 120, 331, 209, 95, 194, 148, 34, 12, 222, 219, 320, 228, 88, 45, 150, 165, 136, 253, 56, 329, 167, 145, 246, 318, 275, 357, 235, 83, 44, 49, 312, 161, 280, 35, 317, 38, 179, 183, 358, 221, 42, 133, 166, 356, 114, 379, 93, 60, 341, 64, 125, 131, 262, 100, 25, 284, 367, 178, 39, 267, 52, 230, 206, 29, 103, 170, 249, 210, 334, 111, 268, 355, 213, 186, 96, 378, 97, 112, 316, 1, 191, 366, 30, 245, 190, 285, 236, 195, 154, 2, 225, 242, 338, 63, 182, 277, 282, 3, 214, 371, 226, 273, 254, 340, 375, 258, 350, 193, 9, 376, 257, 6, 241, 86, 345, 59, 274, 325, 76, 61, 106, 151, 196, 314, 43, 27, 28, 344, 68, 7, 82, 187, 153, 278, 324, 276, 0, 85, 252, 168, 72, 62, 297, 33, 75, 261, 108, 216, 313, 80, 354, 158, 137, 370, 292, 92, 349, 263, 234, 121, 157, 122, 146, 140, 289, 147, 223, 40, 127, 255, 81, 311, 21, 374, 163, 362, 162, 55, 238, 134, 248, 293, 301, 266, 270, 174, 20, 208, 231, 321, 336, 305, 332, 271, 315, 23, 319, 117, 152, 18, 124, 300, 259, 16, 239, 19, 105, 308, 302, 36, 47, 377, 298, 347, 113, 4, 200, 32, 365, 57, 101, 339, 204, 217, 198, 22, 369, 177, 207, 283, 335, 99, 264, 290, 65, 189, 360, 98, 352, 169, 14, 132, 46, 229, 70, 69, 240, 94, 310, 84, 220, 66, 306, 176, 346, 303, 26, 295, 180, 181, 304, 211, 31, 201, 130, 10, 215, 205, 203, 48, 353, 380, 247, 91, 265, 202, 11, 224, 212, 89, 382, 227, 119, 233, 128, 272, 287, 107, 351] + 5: [61, 84, 346, 276, 269, 272, 21, 165, 308, 160, 81, 60, 370, 381, 78, 102, 215, 279, 252, 7, 246, 192, 338, 182, 327, 304, 131, 70, 151, 375, 332, 238, 181, 319, 251, 207, 324, 320, 333, 220, 33, 104, 46, 79, 266, 216, 329, 183, 230, 106, 239, 250, 341, 219, 76, 136, 281, 334, 48, 360, 134, 29, 271, 55, 205, 374, 299, 263, 107, 342, 187, 92, 224, 189, 347, 101, 133, 126, 267, 307, 311, 146, 340, 294, 117, 310, 339, 280, 166, 225, 167, 277, 286, 196, 303, 75, 213, 249, 194, 105, 52, 6, 159, 141, 256, 1, 65, 223, 188, 169, 122, 64, 130, 351, 17, 43, 293, 247, 335, 22, 124, 173, 4, 155, 37, 222, 358, 264, 69, 67, 318, 24, 210, 34, 153, 315, 54, 295, 99, 144, 51, 242, 345, 227, 14, 343, 362, 240, 63, 138, 203, 325, 254, 119, 149, 236, 344, 77, 191, 349, 359, 113, 323, 378, 199, 68, 168, 58, 148, 322, 32, 245, 185, 139, 285, 73, 243, 350, 357, 42, 180, 10, 116, 112, 348, 164, 82, 157, 337, 118, 127, 44, 128, 190, 296, 100, 356, 231, 172, 208, 5, 260, 47, 265, 262, 365, 372, 41, 72, 145, 352, 200, 89, 147, 198, 25, 31, 233, 66, 137, 171, 163, 59, 2, 108, 206, 204, 143, 26, 314, 152, 377, 301, 35, 195, 186, 380, 274, 93, 39, 20, 353, 201, 121, 257, 161, 317, 289, 275, 330, 80, 87, 361, 94, 229, 62, 235, 221, 71, 114, 18, 135, 40, 298, 261, 125, 95, 292, 11, 111, 197, 85, 328, 336, 158, 234, 0, 273, 27, 367, 142, 278, 96, 179, 23, 321, 368, 302, 214, 355, 217, 305, 287, 288, 259, 154, 306, 98, 178, 49, 193, 30, 211, 379, 255, 132, 212, 88, 331, 91, 364, 74, 209, 371, 8, 313, 15, 268, 369, 174, 184, 38, 140, 3, 175, 258, 45, 218, 16, 228, 28, 115, 12, 312, 253, 19, 56, 366, 382, 300, 241, 150, 284, 90, 383, 237, 9, 13, 291, 162, 232, 376, 363, 283, 103, 129, 270, 244, 156, 316, 297, 373, 86, 290, 53, 120, 309, 97, 282, 36, 202, 123, 109, 226, 248, 110, 83, 176, 326, 57, 50, 170, 177, 354] + 6: [97, 32, 343, 240, 66, 177, 5, 249, 269, 11, 326, 120, 350, 45, 163, 318, 232, 293, 189, 112, 319, 34, 178, 289, 156, 0, 270, 113, 179, 369, 182, 336, 317, 223, 30, 202, 282, 383, 144, 211, 143, 324, 154, 58, 210, 361, 104, 110, 276, 335, 330, 60, 355, 14, 145, 126, 18, 83, 124, 147, 349, 215, 299, 54, 73, 310, 39, 235, 80, 62, 251, 360, 266, 24, 195, 351, 89, 201, 263, 38, 7, 140, 91, 93, 273, 382, 228, 265, 252, 149, 25, 33, 320, 71, 40, 90, 142, 376, 174, 258, 233, 297, 59, 160, 341, 55, 205, 31, 9, 36, 307, 13, 88, 100, 368, 84, 255, 287, 291, 129, 259, 345, 277, 165, 185, 352, 245, 278, 311, 304, 37, 111, 321, 44, 64, 107, 264, 98, 220, 157, 271, 22, 206, 242, 225, 197, 23, 4, 362, 138, 316, 148, 48, 333, 198, 358, 298, 96, 268, 272, 134, 366, 285, 357, 106, 203, 347, 85, 17, 102, 303, 51, 313, 356, 302, 78, 26, 171, 19, 227, 344, 200, 61, 2, 314, 135, 167, 52, 275, 323, 161, 164, 308, 274, 53, 253, 99, 146, 295, 27, 267, 229, 209, 67, 173, 375, 359, 181, 187, 367, 57, 283, 118, 208, 155, 184, 370, 151, 301, 176, 128, 377, 380, 236, 49, 68, 12, 158, 133, 139, 114, 219, 15, 10, 286, 280, 6, 87, 207, 137, 116, 63, 243, 217, 194, 261, 122, 248, 199, 105, 193, 224, 8, 16, 381, 309, 364, 123, 312, 74, 150, 288, 328, 290, 117, 162, 338, 21, 121, 222, 339, 238, 29, 192, 127, 241, 340, 170, 218, 72, 65, 75, 214, 186, 315, 331, 247, 257, 322, 76, 305, 281, 166, 212, 103, 306, 28, 46, 180, 371, 175, 378, 260, 216, 94, 119, 56, 246, 188, 196, 79, 239, 300, 234, 95, 354, 109, 115, 41, 230, 231, 101, 279, 190, 169, 221, 77, 327, 332, 136, 296, 365, 262, 69, 43, 237, 159, 284, 82, 86, 254, 372, 329, 81, 168, 346, 3, 50, 334, 42, 70, 125, 379, 292, 244, 130, 152, 108, 226, 337, 256, 294, 348, 47, 183, 131, 250, 374, 92, 213, 141, 1, 172, 191, 20, 353, 153, 132, 363, 204, 35, 325, 342, 373] + 7: [327, 275, 41, 229, 340, 103, 163, 162, 258, 361, 260, 342, 139, 99, 97, 328, 244, 27, 181, 281, 11, 313, 164, 82, 248, 262, 168, 55, 84, 2, 319, 374, 383, 208, 177, 224, 90, 344, 333, 214, 24, 368, 314, 109, 358, 19, 32, 190, 211, 83, 335, 305, 67, 53, 353, 232, 251, 373, 65, 291, 87, 101, 52, 120, 37, 126, 268, 209, 363, 112, 69, 218, 85, 254, 111, 39, 252, 381, 78, 159, 195, 95, 45, 57, 352, 331, 143, 282, 34, 376, 236, 179, 135, 371, 338, 42, 7, 182, 263, 142, 382, 201, 298, 157, 308, 239, 176, 28, 13, 118, 48, 329, 77, 370, 256, 73, 14, 249, 113, 217, 289, 29, 349, 23, 270, 148, 250, 235, 117, 21, 221, 62, 330, 264, 98, 206, 9, 321, 311, 140, 185, 130, 129, 5, 220, 200, 102, 44, 141, 134, 306, 301, 283, 8, 227, 146, 116, 93, 193, 203, 79, 318, 35, 216, 33, 75, 36, 295, 88, 312, 173, 379, 297, 124, 63, 128, 198, 304, 230, 66, 290, 197, 127, 54, 122, 223, 145, 30, 365, 202, 171, 60, 274, 175, 64, 265, 105, 25, 369, 272, 336, 133, 187, 46, 280, 378, 354, 212, 104, 174, 114, 131, 183, 294, 307, 100, 125, 74, 119, 285, 337, 339, 296, 284, 70, 210, 170, 213, 158, 191, 243, 167, 234, 89, 26, 204, 136, 237, 180, 107, 322, 273, 58, 6, 51, 121, 56, 155, 161, 40, 309, 68, 149, 153, 225, 351, 184, 222, 151, 106, 286, 169, 194, 1, 10, 96, 367, 43, 317, 303, 228, 315, 242, 299, 332, 150, 205, 165, 138, 144, 76, 156, 279, 160, 91, 348, 357, 50, 49, 292, 192, 375, 343, 18, 362, 341, 238, 108, 350, 271, 253, 71, 15, 188, 0, 61, 94, 302, 324, 47, 215, 267, 278, 4, 17, 31, 288, 360, 325, 356, 81, 255, 219, 377, 372, 300, 241, 132, 259, 189, 16, 245, 92, 20, 186, 178, 137, 293, 380, 86, 152, 226, 310, 323, 233, 196, 247, 261, 22, 38, 345, 366, 115, 269, 320, 166, 287, 207, 316, 72, 59, 123, 80, 231, 3, 277, 334, 266, 246, 359, 346, 147, 172, 12, 347, 199, 355, 154, 326, 240, 276, 110, 257, 364] + 8: [360, 135, 239, 321, 308, 229, 273, 3, 292, 106, 224, 173, 272, 282, 36, 122, 252, 95, 316, 44, 364, 115, 131, 195, 128, 366, 75, 150, 109, 283, 119, 179, 330, 169, 165, 215, 304, 189, 183, 196, 130, 164, 233, 197, 2, 66, 4, 42, 101, 217, 212, 174, 379, 248, 257, 249, 184, 149, 47, 382, 266, 327, 68, 199, 79, 339, 35, 145, 306, 14, 369, 54, 365, 153, 351, 220, 84, 103, 244, 208, 26, 285, 70, 305, 142, 53, 139, 82, 288, 210, 242, 227, 376, 58, 96, 207, 253, 31, 264, 162, 221, 371, 281, 250, 218, 302, 108, 312, 243, 88, 64, 276, 20, 223, 380, 18, 328, 97, 156, 39, 166, 181, 69, 83, 23, 34, 6, 258, 92, 261, 65, 275, 303, 112, 198, 194, 317, 267, 120, 177, 341, 91, 114, 61, 59, 206, 254, 230, 237, 116, 232, 293, 151, 110, 259, 37, 190, 126, 16, 143, 144, 325, 378, 205, 295, 9, 29, 185, 323, 33, 277, 146, 15, 171, 370, 77, 329, 245, 354, 343, 43, 80, 331, 344, 256, 49, 57, 176, 163, 63, 102, 234, 290, 67, 367, 362, 62, 353, 204, 104, 279, 338, 7, 13, 241, 381, 337, 81, 291, 186, 255, 209, 11, 168, 280, 374, 225, 346, 172, 188, 133, 226, 38, 240, 298, 78, 216, 178, 219, 368, 167, 50, 320, 48, 141, 211, 125, 358, 72, 8, 45, 335, 383, 98, 28, 25, 247, 90, 297, 352, 310, 41, 307, 155, 313, 56, 99, 200, 85, 89, 236, 175, 93, 347, 134, 287, 326, 76, 193, 271, 129, 270, 269, 180, 160, 363, 51, 138, 342, 301, 222, 289, 0, 55, 296, 1, 132, 170, 359, 5, 21, 201, 10, 152, 182, 214, 246, 238, 161, 12, 336, 154, 228, 74, 30, 251, 157, 349, 294, 377, 86, 148, 311, 107, 22, 356, 27, 333, 147, 231, 202, 17, 121, 40, 278, 137, 309, 113, 300, 334, 192, 318, 332, 52, 60, 284, 187, 263, 117, 118, 87, 73, 191, 71, 350, 373, 32, 299, 136, 372, 314, 140, 105, 265, 235, 159, 260, 375, 322, 46, 315, 357, 111, 24, 324, 348, 158, 340, 319, 268, 123, 286, 355, 203, 94, 274, 124, 345, 361, 127, 19, 262, 213, 100] + 9: [92, 226, 47, 316, 244, 147, 81, 220, 280, 19, 357, 165, 184, 29, 15, 201, 138, 159, 281, 341, 136, 132, 177, 309, 9, 133, 82, 158, 363, 356, 120, 289, 264, 334, 231, 6, 364, 87, 116, 145, 186, 323, 207, 206, 124, 66, 189, 72, 106, 104, 343, 308, 208, 246, 50, 65, 52, 60, 275, 373, 174, 322, 254, 114, 371, 301, 251, 305, 224, 333, 211, 221, 347, 69, 188, 337, 290, 191, 368, 225, 345, 118, 328, 101, 93, 44, 194, 294, 91, 250, 61, 27, 236, 168, 338, 26, 304, 31, 295, 90, 64, 10, 166, 96, 372, 172, 353, 74, 179, 37, 272, 146, 217, 380, 210, 51, 67, 282, 56, 102, 98, 367, 317, 360, 135, 154, 3, 292, 297, 306, 54, 321, 314, 311, 259, 129, 269, 105, 4, 123, 230, 181, 238, 141, 237, 359, 80, 13, 276, 248, 354, 307, 379, 117, 205, 233, 300, 79, 203, 115, 42, 110, 12, 330, 298, 270, 162, 229, 346, 335, 241, 374, 293, 383, 243, 43, 325, 279, 361, 223, 149, 284, 273, 40, 362, 7, 122, 53, 77, 24, 365, 57, 255, 355, 263, 185, 49, 103, 78, 143, 315, 260, 299, 219, 155, 381, 198, 83, 127, 242, 329, 214, 121, 14, 32, 342, 131, 151, 285, 30, 377, 153, 268, 169, 88, 228, 0, 128, 183, 97, 20, 99, 142, 256, 139, 16, 70, 11, 171, 332, 288, 202, 39, 319, 339, 18, 175, 109, 100, 382, 45, 46, 190, 261, 358, 291, 84, 253, 71, 352, 310, 125, 38, 89, 48, 178, 59, 86, 34, 326, 68, 140, 176, 366, 63, 119, 278, 85, 378, 257, 164, 156, 265, 55, 152, 126, 340, 173, 349, 327, 94, 313, 262, 95, 212, 331, 196, 215, 252, 350, 222, 351, 344, 1, 376, 370, 287, 271, 209, 23, 239, 22, 258, 312, 161, 144, 187, 302, 137, 33, 277, 318, 180, 247, 192, 107, 41, 163, 35, 195, 267, 336, 36, 150, 111, 218, 274, 5, 75, 324, 17, 235, 8, 130, 303, 245, 148, 197, 62, 213, 112, 2, 320, 286, 375, 134, 76, 216, 21, 232, 199, 227, 167, 170, 193, 160, 58, 108, 240, 266, 249, 73, 296, 204, 200, 113, 28, 157, 25, 182, 369, 283, 348, 234] + 10: [319, 55, 24, 18, 310, 294, 268, 108, 289, 279, 331, 57, 102, 22, 134, 161, 117, 40, 138, 330, 124, 304, 308, 91, 67, 312, 234, 240, 348, 356, 377, 148, 77, 212, 376, 359, 382, 155, 250, 90, 103, 129, 107, 3, 56, 81, 85, 27, 315, 357, 59, 327, 41, 51, 371, 109, 340, 318, 184, 254, 317, 50, 362, 261, 298, 113, 257, 209, 326, 375, 80, 111, 127, 305, 147, 217, 322, 6, 16, 34, 94, 120, 44, 347, 30, 143, 202, 260, 270, 89, 160, 253, 291, 295, 29, 283, 5, 264, 265, 286, 146, 255, 137, 221, 325, 167, 309, 225, 73, 8, 170, 54, 263, 114, 66, 210, 99, 47, 248, 58, 115, 243, 272, 136, 368, 274, 342, 204, 360, 38, 306, 223, 149, 132, 199, 198, 116, 214, 135, 105, 288, 311, 106, 314, 329, 112, 378, 328, 74, 97, 383, 278, 42, 12, 354, 332, 45, 239, 123, 13, 370, 244, 307, 169, 131, 159, 302, 235, 187, 355, 205, 224, 4, 158, 133, 203, 118, 273, 157, 344, 361, 150, 63, 277, 28, 43, 238, 166, 176, 349, 68, 336, 300, 194, 151, 216, 164, 192, 93, 48, 262, 69, 367, 92, 79, 215, 1, 178, 78, 351, 241, 182, 237, 211, 269, 213, 76, 353, 130, 252, 363, 374, 186, 208, 190, 231, 271, 280, 285, 266, 145, 0, 33, 72, 46, 341, 296, 75, 122, 335, 339, 197, 20, 338, 37, 230, 15, 71, 162, 125, 39, 174, 140, 313, 163, 343, 175, 220, 65, 299, 350, 100, 62, 173, 206, 373, 303, 292, 35, 200, 245, 276, 334, 172, 83, 207, 352, 152, 227, 14, 52, 188, 110, 236, 82, 290, 358, 297, 320, 365, 180, 121, 31, 301, 246, 181, 141, 337, 126, 2, 101, 144, 195, 179, 171, 153, 19, 177, 196, 154, 201, 380, 7, 293, 11, 259, 345, 284, 49, 316, 98, 189, 249, 218, 61, 32, 128, 229, 233, 87, 321, 21, 232, 219, 60, 142, 9, 256, 26, 346, 379, 25, 267, 258, 36, 247, 64, 193, 275, 323, 95, 183, 185, 282, 226, 364, 96, 287, 139, 222, 372, 86, 119, 17, 168, 165, 251, 53, 88, 104, 281, 333, 191, 84, 156, 10, 242, 228, 381, 70, 23, 324, 366, 369] + 11: [130, 339, 215, 221, 116, 126, 156, 260, 363, 65, 326, 145, 325, 162, 332, 345, 331, 192, 343, 128, 39, 286, 318, 146, 154, 169, 23, 320, 94, 66, 280, 213, 348, 350, 80, 92, 93, 176, 56, 86, 341, 224, 14, 55, 208, 58, 209, 327, 362, 211, 30, 279, 338, 359, 179, 302, 164, 125, 120, 57, 129, 174, 355, 188, 309, 237, 231, 271, 266, 5, 106, 210, 21, 240, 104, 132, 272, 268, 114, 328, 180, 216, 139, 53, 253, 333, 122, 76, 212, 232, 137, 257, 61, 307, 105, 335, 314, 377, 83, 85, 11, 100, 357, 299, 88, 149, 119, 233, 199, 267, 24, 22, 275, 54, 284, 165, 73, 204, 282, 177, 17, 207, 250, 313, 161, 346, 124, 45, 249, 127, 159, 27, 195, 26, 251, 10, 198, 89, 298, 365, 186, 101, 242, 97, 147, 63, 59, 292, 353, 364, 193, 311, 187, 234, 264, 64, 43, 113, 62, 182, 143, 8, 19, 382, 200, 37, 323, 222, 288, 4, 140, 290, 278, 304, 183, 379, 60, 175, 246, 181, 111, 189, 276, 155, 329, 220, 321, 50, 308, 134, 243, 241, 296, 98, 87, 150, 36, 248, 15, 252, 376, 40, 38, 171, 300, 29, 0, 236, 247, 217, 141, 52, 131, 70, 337, 228, 205, 136, 78, 303, 6, 283, 371, 107, 370, 2, 75, 226, 315, 166, 374, 160, 133, 49, 197, 289, 259, 254, 270, 301, 295, 219, 168, 366, 354, 378, 158, 91, 324, 291, 115, 383, 48, 142, 263, 7, 74, 230, 358, 274, 103, 375, 84, 356, 173, 185, 322, 167, 380, 305, 277, 1, 238, 344, 352, 317, 372, 202, 82, 123, 153, 223, 25, 256, 109, 32, 42, 269, 163, 110, 90, 235, 121, 44, 285, 77, 227, 3, 99, 190, 157, 102, 218, 144, 152, 351, 178, 330, 244, 95, 381, 20, 13, 148, 196, 297, 47, 319, 170, 367, 262, 69, 117, 373, 172, 347, 151, 340, 316, 203, 255, 96, 31, 41, 81, 35, 336, 294, 245, 265, 312, 34, 135, 349, 118, 67, 293, 79, 258, 33, 225, 18, 138, 46, 194, 184, 360, 108, 306, 112, 310, 201, 287, 214, 68, 71, 342, 229, 368, 361, 369, 16, 51, 28, 239, 206, 334, 281, 72, 273, 191, 261, 9, 12] + 12: [255, 190, 127, 378, 172, 341, 245, 219, 109, 22, 296, 103, 50, 90, 198, 170, 354, 295, 134, 278, 216, 346, 268, 25, 147, 266, 364, 27, 2, 160, 300, 104, 107, 63, 135, 308, 238, 23, 301, 144, 358, 253, 357, 77, 1, 191, 114, 0, 45, 224, 313, 280, 232, 37, 321, 11, 214, 121, 159, 305, 379, 293, 112, 244, 329, 207, 217, 374, 106, 285, 44, 375, 345, 55, 96, 54, 117, 236, 258, 79, 182, 184, 118, 132, 163, 328, 275, 92, 377, 56, 66, 99, 250, 110, 272, 42, 204, 149, 78, 177, 150, 46, 49, 381, 28, 168, 267, 349, 71, 192, 51, 306, 208, 259, 3, 339, 382, 205, 276, 152, 317, 323, 16, 89, 153, 209, 265, 171, 264, 271, 164, 294, 80, 291, 31, 74, 286, 309, 304, 227, 188, 9, 52, 315, 29, 84, 298, 138, 136, 4, 187, 58, 319, 38, 355, 351, 20, 140, 334, 336, 365, 246, 297, 279, 274, 174, 70, 310, 34, 8, 93, 113, 212, 237, 320, 367, 143, 210, 26, 269, 360, 314, 100, 213, 166, 21, 64, 239, 373, 366, 225, 372, 120, 327, 226, 82, 243, 203, 363, 35, 17, 175, 292, 289, 97, 233, 151, 369, 260, 176, 123, 6, 281, 332, 348, 215, 218, 368, 60, 10, 39, 288, 101, 371, 322, 273, 180, 129, 261, 337, 59, 318, 155, 15, 193, 376, 200, 350, 231, 234, 83, 119, 67, 62, 262, 249, 173, 325, 76, 122, 115, 270, 13, 65, 12, 32, 5, 186, 312, 263, 24, 352, 146, 105, 30, 282, 178, 131, 81, 299, 125, 230, 19, 307, 94, 145, 221, 202, 342, 248, 359, 95, 141, 240, 43, 53, 199, 302, 130, 158, 303, 223, 69, 333, 156, 48, 344, 167, 124, 126, 287, 290, 361, 181, 98, 340, 75, 195, 68, 137, 194, 330, 229, 87, 108, 277, 133, 33, 102, 116, 362, 57, 197, 241, 326, 380, 157, 14, 91, 283, 201, 142, 370, 254, 185, 41, 165, 61, 189, 256, 161, 139, 356, 169, 335, 235, 40, 154, 36, 257, 183, 86, 179, 284, 128, 148, 85, 383, 252, 251, 47, 316, 343, 18, 347, 72, 211, 247, 196, 7, 111, 338, 242, 228, 206, 88, 311, 324, 331, 353, 222, 162, 73, 220] + 13: [170, 44, 14, 312, 274, 318, 3, 39, 265, 282, 201, 95, 110, 271, 366, 88, 338, 46, 306, 118, 279, 45, 227, 364, 16, 289, 273, 40, 143, 158, 363, 299, 373, 98, 210, 168, 79, 179, 43, 382, 145, 196, 107, 359, 151, 256, 146, 52, 221, 125, 303, 331, 161, 286, 326, 220, 345, 92, 259, 381, 50, 250, 187, 123, 70, 38, 73, 313, 150, 103, 119, 249, 185, 372, 383, 343, 284, 28, 63, 128, 99, 181, 20, 317, 269, 47, 156, 82, 351, 247, 301, 216, 350, 316, 7, 175, 108, 80, 362, 328, 280, 30, 194, 77, 0, 169, 51, 231, 100, 209, 134, 310, 11, 335, 162, 159, 307, 131, 111, 64, 75, 31, 69, 356, 54, 294, 67, 266, 334, 6, 378, 65, 147, 219, 240, 62, 144, 26, 225, 283, 374, 253, 327, 243, 15, 380, 37, 8, 163, 36, 337, 153, 319, 174, 257, 127, 263, 244, 308, 311, 360, 72, 115, 377, 235, 5, 365, 190, 138, 285, 85, 157, 226, 1, 60, 212, 90, 258, 224, 97, 332, 272, 354, 148, 255, 120, 277, 314, 126, 33, 155, 192, 232, 87, 329, 23, 189, 238, 137, 57, 96, 24, 21, 135, 55, 330, 102, 197, 186, 370, 171, 304, 35, 176, 295, 198, 369, 112, 297, 202, 246, 91, 281, 292, 178, 2, 300, 207, 10, 211, 237, 321, 25, 344, 262, 361, 93, 166, 183, 139, 164, 222, 223, 34, 48, 89, 68, 106, 357, 84, 242, 268, 59, 325, 113, 53, 94, 270, 217, 234, 213, 287, 104, 214, 233, 83, 167, 180, 252, 101, 203, 352, 152, 264, 347, 293, 349, 341, 368, 200, 322, 160, 66, 165, 324, 140, 78, 32, 18, 195, 114, 149, 260, 12, 9, 81, 86, 41, 172, 276, 342, 254, 340, 17, 199, 142, 298, 109, 116, 323, 58, 184, 49, 188, 275, 376, 309, 261, 229, 320, 208, 122, 42, 22, 375, 355, 339, 141, 228, 239, 291, 74, 121, 333, 177, 129, 29, 76, 241, 105, 267, 288, 248, 379, 136, 56, 305, 296, 245, 348, 133, 71, 154, 358, 13, 19, 124, 130, 61, 353, 182, 205, 4, 346, 336, 117, 215, 191, 278, 173, 251, 371, 132, 236, 290, 302, 367, 218, 193, 206, 27, 315, 230, 204] + 14: [229, 4, 110, 259, 327, 308, 147, 372, 137, 17, 268, 150, 337, 23, 254, 85, 52, 3, 64, 317, 360, 374, 44, 216, 166, 208, 13, 158, 149, 246, 343, 357, 319, 68, 370, 108, 167, 104, 40, 325, 286, 109, 296, 82, 116, 377, 129, 176, 276, 359, 316, 335, 89, 152, 313, 183, 356, 93, 376, 311, 10, 114, 302, 329, 309, 195, 36, 136, 20, 9, 19, 312, 179, 18, 132, 366, 62, 35, 84, 181, 236, 144, 219, 65, 192, 368, 131, 352, 232, 119, 196, 73, 212, 205, 88, 174, 160, 143, 204, 209, 122, 378, 275, 261, 164, 60, 112, 239, 353, 146, 228, 99, 100, 124, 97, 342, 61, 154, 182, 271, 157, 294, 220, 314, 161, 379, 215, 123, 69, 291, 326, 159, 37, 22, 339, 169, 102, 349, 178, 227, 5, 48, 241, 221, 341, 284, 86, 200, 202, 168, 264, 282, 350, 217, 240, 96, 285, 94, 214, 321, 153, 375, 338, 46, 238, 304, 14, 16, 230, 258, 45, 31, 66, 55, 367, 180, 11, 81, 247, 252, 188, 91, 134, 280, 29, 218, 138, 272, 318, 249, 260, 165, 345, 211, 15, 30, 34, 355, 71, 56, 77, 287, 383, 175, 186, 118, 142, 151, 95, 32, 177, 39, 162, 79, 279, 111, 41, 265, 125, 197, 59, 25, 323, 163, 184, 324, 369, 145, 277, 363, 333, 382, 128, 256, 172, 234, 222, 90, 156, 380, 330, 298, 0, 336, 300, 223, 191, 12, 364, 1, 63, 251, 106, 28, 6, 262, 278, 103, 171, 127, 74, 334, 290, 58, 263, 322, 80, 194, 362, 57, 92, 24, 120, 107, 320, 283, 113, 51, 243, 303, 38, 53, 292, 67, 315, 295, 289, 331, 255, 70, 76, 2, 75, 126, 155, 274, 207, 8, 199, 49, 344, 348, 235, 141, 198, 117, 231, 371, 213, 269, 21, 250, 354, 26, 270, 33, 135, 297, 206, 328, 288, 332, 201, 193, 301, 266, 133, 346, 187, 203, 245, 226, 373, 101, 347, 105, 237, 306, 225, 253, 189, 115, 242, 148, 224, 244, 248, 43, 42, 310, 173, 273, 358, 257, 365, 267, 78, 361, 305, 190, 299, 87, 281, 7, 47, 293, 185, 139, 140, 233, 381, 307, 170, 72, 27, 54, 340, 210, 351, 98, 130, 83, 121, 50] + 15: [130, 227, 57, 155, 94, 148, 298, 76, 202, 39, 38, 326, 333, 119, 121, 51, 232, 360, 139, 56, 296, 193, 256, 1, 242, 255, 230, 24, 352, 225, 338, 100, 170, 209, 305, 194, 145, 275, 59, 263, 269, 34, 268, 62, 50, 250, 12, 43, 301, 30, 313, 118, 321, 381, 267, 137, 244, 45, 272, 211, 373, 375, 358, 63, 116, 52, 65, 210, 378, 369, 374, 188, 29, 140, 176, 318, 58, 112, 222, 98, 299, 103, 223, 17, 243, 21, 302, 345, 27, 33, 245, 75, 198, 127, 368, 85, 11, 160, 251, 355, 217, 134, 290, 350, 367, 199, 185, 266, 74, 284, 146, 20, 221, 157, 23, 48, 353, 344, 311, 174, 177, 143, 18, 164, 379, 106, 351, 335, 133, 5, 49, 247, 101, 166, 117, 142, 343, 77, 331, 282, 69, 81, 254, 165, 377, 141, 191, 67, 180, 97, 365, 234, 314, 224, 123, 37, 92, 172, 84, 213, 167, 218, 239, 79, 91, 161, 89, 32, 262, 149, 187, 380, 124, 136, 330, 181, 44, 179, 304, 70, 286, 122, 219, 220, 132, 348, 25, 264, 325, 260, 356, 328, 0, 359, 362, 200, 128, 104, 340, 4, 291, 371, 332, 159, 236, 215, 60, 9, 317, 151, 376, 83, 113, 277, 342, 135, 241, 111, 109, 235, 31, 285, 357, 47, 78, 323, 28, 42, 102, 279, 86, 248, 87, 10, 150, 96, 363, 8, 206, 13, 271, 339, 349, 306, 53, 72, 308, 289, 55, 204, 169, 64, 26, 337, 316, 257, 240, 54, 278, 364, 73, 197, 153, 129, 370, 201, 293, 280, 327, 292, 66, 195, 154, 183, 307, 110, 108, 125, 156, 297, 186, 19, 46, 144, 322, 192, 93, 270, 15, 88, 324, 258, 175, 68, 261, 259, 295, 300, 189, 231, 252, 203, 90, 237, 80, 329, 16, 40, 35, 173, 168, 163, 138, 320, 196, 115, 162, 36, 207, 319, 182, 372, 238, 246, 105, 212, 2, 82, 309, 114, 229, 22, 205, 233, 341, 253, 294, 382, 107, 226, 347, 61, 354, 99, 208, 216, 383, 71, 283, 95, 334, 265, 273, 190, 3, 147, 274, 303, 184, 158, 14, 126, 312, 310, 7, 171, 361, 228, 178, 41, 120, 346, 288, 287, 214, 336, 249, 315, 152, 6, 131, 366, 281, 276] + 16: [134, 369, 234, 43, 303, 75, 68, 331, 300, 338, 49, 289, 259, 224, 250, 298, 283, 287, 213, 146, 246, 16, 226, 108, 27, 84, 306, 59, 97, 38, 37, 292, 73, 139, 0, 282, 41, 182, 34, 195, 232, 45, 42, 301, 137, 297, 4, 106, 290, 354, 228, 358, 208, 57, 214, 95, 223, 323, 319, 190, 244, 266, 240, 381, 163, 14, 88, 373, 218, 17, 113, 26, 216, 217, 50, 332, 230, 288, 169, 382, 79, 351, 130, 341, 173, 197, 327, 278, 361, 10, 191, 320, 309, 98, 3, 148, 15, 140, 92, 147, 64, 271, 162, 185, 312, 86, 129, 342, 348, 183, 78, 255, 383, 187, 141, 46, 367, 212, 346, 89, 294, 365, 52, 237, 2, 335, 316, 82, 119, 281, 159, 153, 380, 305, 76, 220, 229, 379, 372, 174, 9, 339, 370, 124, 104, 221, 151, 340, 200, 60, 211, 242, 333, 343, 83, 105, 91, 196, 193, 360, 186, 81, 127, 128, 47, 261, 258, 170, 116, 256, 77, 149, 61, 243, 352, 63, 164, 12, 23, 135, 350, 291, 179, 296, 362, 53, 345, 94, 111, 366, 156, 249, 286, 100, 262, 165, 44, 280, 67, 125, 154, 273, 90, 203, 5, 374, 274, 24, 265, 121, 364, 236, 87, 123, 180, 181, 285, 310, 321, 328, 307, 248, 99, 21, 80, 66, 155, 39, 31, 219, 8, 70, 356, 36, 334, 19, 160, 238, 96, 188, 198, 252, 375, 25, 132, 18, 102, 178, 337, 93, 272, 378, 109, 207, 201, 363, 62, 233, 157, 284, 107, 20, 131, 110, 308, 279, 176, 150, 138, 251, 33, 28, 355, 112, 40, 254, 299, 263, 72, 204, 325, 133, 13, 215, 126, 368, 257, 353, 120, 275, 152, 7, 269, 118, 11, 330, 114, 277, 144, 143, 318, 205, 35, 302, 171, 209, 189, 295, 235, 270, 347, 199, 85, 65, 314, 177, 71, 115, 51, 377, 175, 103, 58, 56, 311, 231, 184, 227, 167, 357, 122, 202, 293, 276, 22, 322, 317, 225, 264, 29, 336, 158, 54, 313, 206, 253, 267, 304, 55, 48, 142, 30, 194, 172, 315, 166, 324, 145, 268, 168, 117, 371, 359, 241, 245, 210, 260, 101, 32, 6, 349, 74, 329, 136, 192, 326, 222, 376, 161, 69, 239, 247, 1, 344] + 17: [131, 261, 250, 200, 305, 374, 138, 140, 190, 126, 91, 136, 155, 275, 185, 114, 1, 125, 189, 62, 75, 8, 56, 104, 207, 73, 218, 206, 148, 313, 211, 103, 143, 84, 272, 146, 282, 314, 162, 18, 332, 13, 166, 60, 45, 260, 215, 154, 118, 187, 221, 205, 101, 253, 21, 213, 360, 249, 195, 244, 299, 329, 16, 85, 64, 106, 278, 326, 364, 350, 63, 344, 53, 355, 316, 227, 23, 40, 277, 66, 257, 368, 11, 129, 77, 348, 111, 281, 209, 123, 34, 86, 69, 72, 121, 152, 119, 375, 303, 15, 26, 28, 80, 226, 346, 87, 219, 78, 225, 5, 343, 266, 130, 324, 133, 6, 175, 172, 52, 339, 96, 296, 335, 97, 381, 180, 380, 373, 10, 98, 33, 135, 292, 171, 245, 259, 340, 248, 47, 116, 59, 42, 269, 295, 49, 204, 377, 186, 341, 338, 342, 265, 94, 179, 262, 310, 90, 120, 161, 112, 267, 274, 369, 25, 192, 50, 150, 57, 163, 83, 153, 352, 149, 230, 356, 223, 349, 178, 242, 71, 247, 353, 268, 321, 252, 174, 54, 122, 159, 170, 202, 203, 284, 337, 188, 383, 330, 362, 307, 208, 319, 366, 254, 311, 359, 197, 232, 323, 246, 351, 29, 297, 370, 167, 290, 157, 134, 317, 22, 199, 145, 243, 38, 92, 367, 139, 298, 74, 165, 263, 70, 376, 191, 82, 61, 36, 331, 147, 7, 336, 81, 196, 115, 382, 37, 89, 181, 27, 110, 238, 302, 361, 102, 239, 210, 217, 128, 48, 308, 124, 67, 255, 318, 144, 164, 132, 241, 88, 280, 141, 201, 224, 4, 184, 160, 285, 289, 0, 35, 322, 301, 293, 169, 127, 235, 237, 251, 283, 236, 79, 220, 108, 100, 294, 212, 105, 137, 58, 12, 357, 117, 193, 286, 279, 113, 2, 347, 328, 276, 234, 222, 107, 39, 194, 46, 24, 231, 177, 228, 273, 371, 256, 99, 327, 214, 9, 142, 168, 264, 304, 19, 20, 173, 43, 41, 363, 216, 198, 365, 3, 309, 271, 95, 44, 379, 300, 93, 240, 258, 183, 345, 287, 151, 55, 158, 372, 76, 270, 31, 233, 32, 65, 315, 182, 378, 176, 109, 30, 229, 291, 288, 306, 334, 17, 51, 68, 156, 333, 14, 358, 312, 320, 325, 354] + 18: [271, 65, 106, 6, 356, 210, 315, 347, 119, 304, 239, 189, 167, 301, 139, 237, 284, 297, 144, 266, 33, 105, 240, 98, 165, 269, 20, 171, 202, 339, 121, 283, 149, 223, 233, 131, 135, 0, 22, 288, 313, 328, 61, 102, 111, 349, 205, 264, 319, 362, 132, 265, 155, 363, 270, 322, 325, 112, 275, 256, 60, 234, 332, 158, 27, 247, 126, 224, 369, 123, 76, 2, 107, 84, 370, 161, 29, 184, 294, 100, 305, 277, 72, 173, 62, 295, 198, 91, 327, 97, 222, 293, 1, 32, 59, 78, 30, 320, 245, 228, 118, 272, 16, 207, 154, 215, 110, 169, 236, 146, 68, 373, 306, 31, 151, 246, 99, 77, 67, 103, 204, 164, 89, 337, 346, 366, 248, 307, 83, 343, 79, 225, 88, 162, 137, 278, 350, 361, 58, 50, 289, 140, 203, 279, 38, 244, 93, 12, 182, 255, 187, 329, 153, 214, 166, 108, 48, 348, 143, 82, 216, 66, 219, 56, 163, 209, 133, 218, 180, 258, 86, 37, 85, 238, 273, 377, 230, 11, 117, 290, 336, 221, 196, 353, 128, 374, 120, 227, 314, 147, 309, 340, 251, 10, 69, 257, 57, 71, 185, 45, 333, 286, 113, 114, 380, 174, 116, 136, 25, 26, 34, 291, 253, 35, 330, 285, 326, 3, 188, 19, 318, 46, 41, 191, 141, 177, 104, 359, 261, 316, 263, 241, 296, 7, 232, 201, 231, 157, 39, 8, 308, 125, 178, 276, 96, 63, 40, 335, 292, 351, 148, 243, 254, 303, 127, 358, 168, 170, 338, 4, 51, 311, 197, 229, 375, 298, 312, 372, 172, 44, 90, 299, 280, 14, 345, 18, 160, 21, 206, 134, 310, 379, 186, 52, 74, 156, 242, 344, 208, 47, 43, 217, 300, 138, 260, 193, 53, 211, 28, 36, 368, 192, 142, 355, 321, 360, 376, 357, 282, 378, 341, 190, 101, 124, 70, 274, 181, 371, 365, 342, 23, 73, 331, 262, 87, 334, 129, 109, 367, 159, 122, 250, 302, 235, 152, 195, 15, 176, 54, 150, 81, 9, 145, 13, 267, 94, 287, 324, 92, 179, 213, 383, 175, 200, 64, 220, 259, 130, 17, 317, 381, 364, 75, 95, 42, 354, 115, 352, 281, 24, 5, 249, 212, 199, 252, 194, 323, 49, 55, 226, 80, 183, 268, 382] + 19: [24, 256, 120, 180, 312, 64, 211, 48, 302, 52, 291, 126, 318, 197, 66, 363, 218, 33, 296, 97, 230, 275, 189, 264, 158, 147, 246, 53, 285, 1, 278, 134, 6, 165, 167, 76, 353, 195, 257, 204, 2, 297, 4, 215, 95, 342, 336, 194, 85, 177, 23, 131, 60, 293, 280, 129, 225, 376, 362, 242, 298, 137, 209, 87, 186, 14, 208, 174, 28, 286, 371, 103, 112, 357, 267, 162, 185, 39, 224, 21, 155, 86, 56, 308, 226, 43, 199, 55, 142, 250, 378, 12, 34, 50, 81, 98, 187, 288, 265, 338, 42, 240, 141, 111, 364, 91, 161, 322, 192, 300, 113, 328, 154, 349, 227, 89, 301, 354, 35, 58, 365, 25, 335, 54, 71, 30, 222, 295, 127, 182, 68, 83, 156, 368, 138, 283, 157, 144, 259, 249, 374, 273, 84, 346, 19, 213, 45, 202, 75, 356, 3, 26, 191, 88, 200, 119, 130, 212, 22, 168, 232, 132, 234, 235, 63, 314, 37, 294, 150, 106, 11, 269, 201, 214, 252, 51, 205, 163, 236, 379, 282, 69, 220, 78, 145, 274, 93, 305, 107, 133, 198, 245, 190, 90, 57, 330, 207, 307, 243, 179, 61, 241, 255, 183, 219, 114, 373, 27, 20, 38, 303, 77, 159, 153, 217, 44, 41, 171, 238, 15, 13, 116, 128, 310, 228, 173, 96, 121, 251, 16, 272, 221, 260, 347, 277, 31, 306, 203, 325, 18, 333, 237, 10, 80, 331, 148, 47, 62, 309, 146, 326, 321, 176, 311, 32, 320, 258, 181, 332, 247, 94, 324, 59, 315, 231, 118, 124, 289, 290, 223, 304, 100, 299, 266, 248, 102, 151, 360, 327, 104, 175, 276, 109, 317, 253, 254, 263, 268, 366, 7, 377, 313, 101, 115, 339, 206, 74, 70, 188, 279, 8, 216, 184, 323, 193, 164, 334, 271, 152, 117, 79, 9, 292, 46, 135, 122, 233, 36, 352, 166, 359, 337, 270, 172, 29, 49, 170, 239, 40, 67, 345, 110, 244, 143, 287, 125, 284, 108, 380, 262, 160, 372, 92, 65, 140, 281, 329, 383, 341, 82, 139, 343, 370, 99, 105, 340, 149, 351, 73, 361, 348, 381, 210, 123, 178, 358, 367, 136, 344, 0, 5, 261, 169, 17, 72, 382, 196, 316, 229, 350, 319, 355, 369, 375] + 20: [161, 144, 87, 268, 46, 37, 317, 110, 376, 339, 130, 105, 185, 138, 162, 325, 91, 228, 273, 95, 96, 264, 227, 342, 49, 101, 172, 77, 182, 239, 176, 364, 348, 235, 378, 175, 265, 146, 58, 367, 279, 372, 22, 34, 238, 243, 165, 294, 29, 15, 5, 106, 380, 39, 3, 202, 327, 89, 84, 230, 154, 316, 366, 352, 289, 283, 319, 28, 40, 102, 205, 341, 136, 83, 16, 207, 234, 383, 295, 192, 103, 186, 306, 203, 298, 109, 269, 198, 249, 27, 51, 52, 54, 61, 76, 107, 116, 302, 66, 43, 145, 282, 174, 55, 287, 112, 291, 224, 32, 270, 371, 148, 324, 141, 128, 125, 382, 9, 286, 216, 292, 347, 92, 71, 126, 90, 74, 240, 194, 328, 26, 219, 53, 36, 359, 233, 309, 119, 18, 131, 166, 340, 345, 248, 297, 338, 232, 252, 332, 365, 201, 231, 360, 191, 223, 351, 217, 56, 210, 11, 59, 79, 21, 88, 113, 329, 271, 354, 211, 276, 19, 313, 261, 68, 108, 70, 220, 222, 344, 237, 1, 38, 143, 288, 260, 137, 167, 168, 179, 181, 189, 200, 334, 278, 82, 99, 45, 335, 122, 14, 173, 2, 379, 195, 363, 62, 188, 93, 209, 164, 111, 290, 381, 187, 180, 259, 135, 258, 184, 65, 104, 322, 357, 13, 57, 284, 35, 343, 41, 118, 368, 314, 247, 150, 300, 256, 114, 274, 346, 97, 169, 370, 310, 12, 33, 127, 73, 170, 312, 374, 337, 75, 94, 254, 80, 285, 151, 142, 115, 120, 124, 356, 333, 355, 47, 67, 225, 63, 48, 133, 293, 183, 147, 251, 275, 31, 152, 244, 44, 171, 17, 241, 277, 206, 214, 262, 263, 266, 98, 153, 336, 193, 369, 4, 8, 272, 81, 123, 226, 307, 353, 204, 318, 330, 158, 10, 229, 221, 311, 362, 50, 140, 160, 132, 320, 121, 358, 315, 190, 6, 177, 218, 267, 361, 375, 0, 25, 30, 213, 208, 178, 42, 196, 60, 215, 86, 246, 377, 20, 100, 78, 236, 304, 257, 326, 24, 212, 253, 159, 308, 139, 349, 156, 199, 250, 296, 321, 7, 85, 163, 323, 72, 305, 157, 64, 134, 373, 245, 149, 23, 129, 301, 155, 255, 69, 197, 117, 242, 280, 281, 299, 303, 331, 350] + 21: [360, 103, 371, 158, 124, 257, 256, 206, 53, 210, 200, 208, 88, 199, 214, 226, 178, 91, 177, 238, 22, 307, 147, 349, 15, 299, 293, 225, 43, 313, 141, 220, 368, 183, 243, 174, 73, 338, 18, 331, 287, 138, 60, 304, 97, 66, 383, 192, 370, 125, 347, 30, 295, 40, 100, 301, 136, 187, 79, 269, 57, 258, 126, 36, 215, 81, 83, 271, 319, 181, 342, 253, 156, 268, 74, 374, 217, 99, 37, 180, 176, 328, 6, 106, 326, 61, 212, 95, 235, 101, 127, 311, 44, 10, 17, 35, 152, 132, 308, 80, 119, 140, 190, 236, 29, 38, 218, 364, 94, 361, 232, 377, 150, 0, 240, 279, 184, 16, 365, 128, 175, 366, 227, 344, 52, 230, 111, 23, 78, 33, 233, 325, 263, 372, 49, 336, 362, 64, 223, 246, 62, 260, 286, 291, 211, 155, 378, 154, 65, 204, 171, 148, 137, 14, 102, 24, 274, 146, 216, 48, 244, 213, 134, 334, 12, 96, 315, 164, 70, 201, 330, 2, 68, 357, 56, 314, 249, 202, 242, 31, 161, 7, 116, 222, 248, 107, 167, 327, 108, 58, 93, 123, 355, 281, 20, 4, 71, 302, 289, 247, 205, 82, 92, 87, 169, 117, 120, 345, 252, 182, 149, 51, 380, 172, 26, 109, 196, 8, 194, 280, 135, 144, 367, 72, 245, 288, 270, 131, 84, 197, 267, 352, 359, 259, 262, 139, 98, 312, 303, 159, 317, 165, 5, 133, 77, 275, 266, 341, 337, 292, 198, 170, 340, 346, 261, 276, 273, 157, 27, 381, 67, 105, 382, 294, 75, 207, 13, 122, 76, 55, 114, 324, 255, 228, 265, 142, 278, 32, 166, 231, 322, 224, 283, 332, 179, 264, 306, 321, 47, 173, 316, 254, 376, 113, 153, 46, 86, 50, 112, 185, 363, 348, 277, 45, 118, 284, 250, 320, 191, 151, 351, 19, 121, 219, 25, 69, 9, 209, 221, 189, 234, 373, 89, 229, 168, 63, 1, 358, 39, 90, 309, 318, 251, 21, 356, 28, 354, 290, 323, 3, 237, 353, 115, 193, 34, 160, 104, 203, 329, 186, 335, 300, 379, 333, 305, 110, 195, 163, 310, 241, 143, 239, 41, 130, 129, 85, 375, 54, 162, 296, 285, 145, 298, 59, 188, 282, 343, 272, 297, 11, 42, 369, 339, 350] + 22: [254, 281, 209, 337, 178, 371, 51, 332, 276, 115, 242, 230, 177, 169, 29, 93, 58, 352, 50, 70, 153, 7, 311, 100, 80, 76, 8, 312, 60, 125, 185, 349, 170, 75, 113, 109, 48, 63, 258, 233, 309, 165, 12, 72, 367, 47, 305, 303, 200, 54, 189, 11, 78, 32, 0, 268, 211, 202, 89, 275, 46, 237, 262, 246, 102, 219, 151, 334, 30, 36, 152, 257, 64, 69, 171, 255, 150, 370, 5, 127, 43, 66, 277, 270, 166, 56, 194, 297, 10, 205, 289, 2, 28, 79, 95, 101, 141, 328, 278, 224, 20, 94, 204, 126, 330, 207, 42, 201, 4, 112, 329, 235, 383, 362, 333, 55, 381, 351, 285, 338, 173, 375, 195, 83, 106, 365, 116, 214, 206, 57, 228, 358, 156, 366, 191, 25, 324, 321, 41, 133, 239, 320, 248, 87, 98, 322, 269, 283, 256, 373, 45, 81, 286, 290, 215, 77, 273, 364, 265, 279, 274, 119, 266, 182, 52, 120, 62, 179, 342, 291, 90, 302, 296, 223, 6, 162, 21, 197, 146, 295, 293, 167, 86, 203, 313, 135, 249, 292, 143, 154, 157, 217, 181, 261, 216, 148, 288, 376, 316, 9, 331, 132, 38, 184, 37, 74, 139, 244, 355, 117, 240, 348, 357, 345, 188, 218, 111, 67, 161, 372, 123, 222, 17, 369, 122, 226, 175, 347, 53, 82, 220, 49, 131, 15, 99, 225, 1, 380, 341, 318, 65, 236, 199, 227, 327, 238, 299, 180, 300, 97, 271, 96, 353, 379, 298, 103, 108, 35, 160, 186, 92, 196, 129, 190, 350, 19, 104, 344, 308, 354, 88, 23, 140, 213, 158, 84, 172, 107, 243, 231, 315, 142, 280, 323, 247, 250, 253, 267, 221, 260, 343, 24, 183, 110, 33, 198, 34, 147, 164, 73, 144, 263, 118, 44, 61, 130, 317, 252, 229, 272, 149, 251, 336, 306, 68, 378, 174, 264, 193, 259, 27, 14, 361, 3, 340, 339, 40, 18, 374, 71, 59, 210, 105, 319, 39, 159, 301, 155, 22, 124, 377, 335, 382, 241, 13, 138, 314, 136, 356, 26, 176, 208, 232, 85, 187, 137, 282, 294, 145, 91, 134, 16, 168, 128, 212, 31, 363, 326, 284, 304, 163, 114, 245, 121, 368, 234, 325, 192, 287, 360, 307, 310, 346, 359] + 23: [26, 126, 100, 244, 72, 281, 274, 147, 48, 82, 306, 12, 327, 47, 176, 84, 241, 127, 372, 2, 270, 56, 130, 283, 209, 379, 144, 213, 378, 220, 191, 229, 152, 23, 342, 40, 255, 129, 359, 370, 150, 153, 254, 17, 273, 137, 178, 207, 162, 341, 196, 348, 90, 58, 344, 256, 107, 302, 108, 21, 114, 14, 268, 99, 123, 338, 227, 373, 308, 205, 49, 46, 310, 368, 230, 277, 383, 116, 247, 381, 138, 20, 208, 42, 83, 199, 112, 142, 166, 226, 113, 216, 189, 235, 81, 64, 314, 305, 175, 167, 85, 177, 69, 376, 77, 201, 173, 87, 279, 103, 261, 156, 67, 374, 169, 55, 6, 315, 78, 263, 133, 35, 326, 111, 219, 312, 91, 151, 317, 10, 228, 224, 136, 210, 356, 157, 32, 96, 160, 98, 53, 65, 301, 307, 382, 316, 284, 352, 258, 143, 0, 128, 245, 174, 206, 134, 121, 122, 297, 294, 239, 25, 139, 355, 106, 248, 140, 282, 158, 353, 34, 31, 322, 165, 334, 299, 343, 22, 289, 63, 183, 145, 332, 60, 186, 366, 234, 15, 291, 286, 242, 168, 124, 71, 264, 132, 252, 135, 51, 120, 170, 295, 240, 203, 197, 287, 154, 232, 180, 11, 57, 59, 159, 86, 257, 41, 340, 309, 146, 184, 74, 70, 36, 149, 95, 195, 38, 290, 215, 292, 44, 222, 298, 185, 329, 66, 231, 5, 325, 321, 223, 330, 193, 76, 45, 73, 109, 280, 28, 164, 7, 212, 163, 155, 349, 236, 336, 375, 33, 104, 260, 161, 188, 61, 253, 357, 88, 276, 337, 238, 259, 360, 364, 318, 30, 117, 246, 262, 217, 198, 296, 251, 29, 3, 39, 8, 320, 288, 172, 275, 328, 187, 323, 300, 225, 211, 19, 331, 24, 269, 324, 358, 27, 94, 192, 304, 182, 218, 351, 148, 371, 92, 190, 131, 80, 50, 102, 249, 233, 271, 313, 194, 237, 272, 221, 311, 101, 361, 1, 119, 93, 365, 52, 125, 333, 179, 347, 250, 204, 303, 54, 363, 339, 200, 335, 346, 16, 68, 354, 243, 362, 285, 97, 115, 369, 267, 214, 79, 367, 202, 9, 319, 62, 266, 350, 377, 171, 181, 118, 141, 345, 278, 105, 89, 293, 380, 13, 18, 110, 4, 75, 43, 37, 265] + 24: [154, 223, 52, 372, 189, 97, 111, 328, 139, 28, 27, 121, 266, 301, 29, 146, 231, 53, 78, 159, 354, 144, 91, 61, 273, 41, 106, 355, 350, 356, 149, 351, 341, 182, 307, 310, 287, 279, 376, 83, 257, 84, 13, 233, 197, 313, 1, 268, 66, 312, 108, 215, 346, 335, 267, 282, 271, 176, 88, 296, 107, 212, 188, 249, 136, 8, 148, 163, 262, 209, 288, 44, 109, 25, 18, 256, 187, 378, 240, 75, 141, 200, 316, 243, 371, 261, 364, 100, 2, 45, 21, 129, 232, 74, 81, 115, 265, 7, 6, 263, 369, 87, 211, 259, 122, 381, 324, 286, 104, 220, 375, 230, 319, 137, 202, 135, 277, 195, 15, 306, 93, 226, 131, 308, 142, 130, 96, 73, 338, 145, 12, 3, 206, 221, 140, 59, 147, 235, 269, 295, 311, 242, 51, 11, 292, 239, 60, 270, 37, 218, 170, 72, 276, 303, 120, 10, 363, 76, 284, 320, 281, 42, 67, 19, 55, 50, 234, 238, 132, 101, 68, 367, 247, 43, 174, 300, 90, 165, 227, 340, 33, 117, 305, 17, 152, 89, 127, 133, 251, 166, 198, 241, 252, 20, 92, 357, 70, 290, 110, 172, 185, 360, 361, 112, 294, 334, 62, 186, 283, 26, 31, 24, 164, 344, 382, 366, 71, 318, 254, 156, 138, 377, 39, 336, 224, 380, 245, 205, 272, 114, 321, 326, 178, 216, 168, 323, 116, 47, 244, 280, 339, 65, 225, 134, 103, 365, 333, 4, 38, 348, 150, 196, 173, 32, 151, 94, 352, 57, 331, 359, 329, 217, 291, 317, 183, 180, 184, 80, 201, 143, 228, 332, 119, 167, 285, 347, 69, 208, 322, 40, 219, 34, 155, 158, 327, 264, 298, 314, 9, 246, 0, 22, 85, 128, 95, 210, 337, 157, 105, 64, 5, 255, 370, 207, 237, 343, 213, 204, 192, 35, 190, 54, 325, 299, 274, 260, 46, 113, 49, 177, 293, 345, 118, 77, 48, 58, 297, 63, 169, 153, 16, 278, 222, 304, 315, 56, 302, 194, 30, 181, 309, 258, 203, 275, 23, 79, 14, 253, 86, 175, 161, 126, 99, 373, 191, 160, 171, 102, 193, 342, 248, 362, 236, 125, 229, 98, 162, 250, 124, 199, 289, 123, 349, 214, 330, 82, 358, 36, 374, 179, 368, 353, 379, 383] + 25: [315, 317, 107, 284, 83, 268, 85, 78, 39, 309, 152, 77, 73, 145, 235, 305, 33, 314, 318, 120, 207, 381, 346, 252, 222, 373, 56, 240, 256, 323, 367, 279, 164, 261, 348, 103, 114, 191, 115, 132, 295, 350, 68, 55, 354, 357, 15, 213, 365, 339, 201, 14, 144, 370, 31, 200, 53, 27, 361, 358, 166, 337, 271, 22, 35, 325, 250, 167, 21, 45, 87, 311, 371, 245, 174, 349, 224, 48, 1, 264, 141, 335, 42, 285, 219, 266, 148, 206, 194, 255, 338, 165, 189, 220, 123, 91, 269, 228, 276, 214, 62, 258, 334, 101, 162, 2, 52, 246, 171, 232, 190, 273, 8, 102, 380, 230, 18, 342, 154, 294, 306, 197, 377, 231, 229, 153, 138, 60, 227, 97, 307, 4, 196, 291, 225, 336, 304, 237, 199, 312, 345, 259, 184, 37, 328, 28, 116, 181, 69, 355, 105, 212, 238, 251, 340, 193, 275, 260, 111, 51, 177, 281, 34, 362, 320, 283, 226, 208, 40, 289, 64, 322, 137, 265, 63, 119, 147, 172, 71, 90, 301, 288, 50, 241, 13, 151, 43, 382, 239, 267, 158, 326, 379, 139, 180, 372, 192, 65, 298, 299, 375, 61, 0, 113, 112, 49, 5, 160, 313, 38, 32, 59, 368, 262, 287, 302, 215, 9, 84, 253, 125, 99, 244, 366, 343, 321, 218, 176, 359, 242, 20, 347, 352, 278, 316, 187, 383, 324, 330, 290, 297, 331, 58, 257, 94, 149, 24, 11, 356, 159, 249, 74, 243, 110, 353, 109, 163, 75, 36, 157, 183, 70, 210, 173, 16, 175, 236, 92, 41, 142, 95, 205, 286, 363, 374, 88, 104, 108, 272, 182, 329, 170, 195, 44, 19, 233, 263, 178, 282, 72, 247, 23, 29, 98, 168, 82, 30, 332, 25, 327, 376, 10, 333, 280, 7, 66, 296, 216, 378, 351, 100, 136, 131, 234, 341, 209, 76, 80, 89, 54, 126, 146, 161, 135, 143, 140, 211, 369, 203, 319, 81, 221, 130, 204, 308, 185, 202, 134, 254, 133, 274, 310, 67, 150, 79, 300, 223, 17, 179, 127, 248, 186, 292, 117, 93, 86, 124, 122, 96, 364, 293, 46, 128, 118, 188, 155, 3, 344, 57, 360, 26, 12, 217, 169, 198, 47, 6, 106, 270, 303, 121, 129, 156, 277] + 26: [136, 80, 18, 236, 253, 281, 63, 371, 304, 254, 116, 69, 33, 130, 38, 293, 238, 135, 22, 36, 92, 110, 332, 200, 180, 204, 198, 26, 319, 323, 151, 353, 306, 99, 327, 79, 186, 343, 194, 20, 285, 318, 277, 195, 10, 73, 378, 115, 171, 37, 256, 187, 123, 143, 372, 298, 25, 302, 264, 49, 300, 325, 362, 225, 255, 40, 334, 9, 275, 303, 39, 333, 326, 199, 101, 216, 373, 284, 14, 250, 330, 46, 164, 381, 95, 248, 207, 288, 203, 1, 54, 68, 86, 96, 106, 111, 237, 90, 380, 53, 291, 230, 5, 271, 263, 70, 160, 289, 94, 337, 308, 266, 227, 355, 315, 88, 30, 146, 328, 83, 55, 265, 148, 48, 259, 360, 321, 190, 133, 2, 356, 97, 103, 122, 201, 268, 66, 192, 15, 357, 81, 161, 35, 98, 0, 252, 196, 350, 93, 144, 232, 363, 19, 134, 336, 329, 220, 3, 140, 364, 317, 354, 72, 347, 34, 297, 331, 100, 29, 193, 13, 205, 296, 56, 313, 145, 282, 85, 222, 188, 23, 84, 286, 210, 309, 213, 139, 169, 170, 175, 209, 218, 89, 172, 292, 124, 21, 339, 233, 214, 342, 162, 149, 16, 279, 344, 137, 341, 369, 61, 158, 376, 245, 167, 131, 82, 67, 156, 31, 311, 138, 178, 191, 287, 367, 44, 27, 152, 173, 241, 118, 294, 127, 365, 17, 215, 65, 87, 45, 202, 107, 41, 370, 50, 114, 177, 182, 312, 117, 189, 368, 7, 78, 12, 105, 340, 47, 141, 109, 8, 64, 59, 77, 244, 154, 206, 91, 274, 324, 258, 377, 208, 283, 211, 345, 235, 366, 310, 322, 273, 52, 132, 260, 228, 234, 242, 246, 261, 269, 217, 6, 11, 181, 150, 320, 349, 184, 338, 224, 120, 276, 183, 316, 278, 307, 223, 226, 249, 108, 126, 382, 62, 247, 359, 128, 176, 272, 179, 166, 262, 142, 229, 174, 102, 251, 24, 221, 51, 57, 71, 104, 239, 159, 113, 212, 290, 267, 76, 32, 231, 129, 75, 153, 383, 280, 301, 157, 219, 197, 121, 125, 352, 165, 185, 335, 43, 346, 74, 112, 361, 163, 240, 119, 314, 358, 270, 42, 4, 243, 295, 28, 147, 257, 58, 155, 379, 60, 168, 299, 305, 348, 351, 374, 375] + 27: [103, 263, 265, 349, 80, 34, 104, 59, 351, 300, 269, 197, 119, 358, 131, 188, 348, 336, 237, 183, 251, 21, 303, 381, 266, 383, 216, 343, 83, 308, 32, 325, 373, 118, 337, 305, 15, 161, 162, 328, 309, 10, 246, 172, 136, 5, 127, 122, 45, 362, 239, 347, 2, 44, 318, 121, 353, 167, 367, 205, 361, 79, 151, 139, 132, 231, 184, 185, 359, 8, 74, 157, 352, 372, 68, 273, 62, 334, 87, 165, 226, 148, 170, 19, 295, 92, 371, 378, 63, 200, 236, 39, 53, 60, 137, 146, 225, 234, 37, 168, 30, 227, 354, 219, 58, 31, 374, 26, 357, 49, 366, 202, 213, 224, 143, 112, 335, 230, 290, 85, 260, 324, 196, 72, 238, 115, 332, 278, 322, 284, 356, 317, 187, 27, 261, 254, 91, 316, 326, 379, 114, 174, 218, 268, 287, 46, 18, 111, 147, 299, 120, 66, 101, 47, 365, 0, 84, 153, 116, 105, 33, 144, 281, 259, 282, 319, 22, 264, 229, 41, 110, 16, 158, 86, 166, 345, 52, 331, 56, 311, 255, 298, 169, 57, 206, 17, 93, 203, 247, 163, 176, 220, 286, 90, 4, 61, 252, 341, 301, 145, 333, 70, 233, 204, 135, 320, 181, 292, 369, 179, 97, 190, 228, 75, 178, 340, 242, 76, 99, 258, 102, 156, 78, 82, 297, 140, 141, 193, 98, 42, 7, 370, 125, 126, 54, 6, 291, 215, 296, 360, 375, 267, 67, 43, 248, 9, 199, 249, 307, 321, 217, 3, 275, 288, 159, 130, 14, 344, 113, 152, 313, 50, 223, 293, 302, 150, 289, 142, 195, 94, 189, 192, 1, 173, 245, 314, 210, 221, 355, 134, 209, 36, 100, 212, 270, 240, 244, 262, 12, 312, 177, 73, 69, 211, 81, 117, 279, 350, 164, 71, 315, 128, 376, 24, 377, 222, 207, 55, 182, 272, 48, 323, 380, 149, 339, 232, 77, 294, 23, 89, 201, 25, 109, 191, 241, 133, 155, 88, 368, 271, 329, 175, 214, 304, 274, 327, 124, 276, 129, 106, 20, 171, 28, 38, 29, 250, 95, 13, 107, 123, 35, 256, 306, 96, 280, 154, 208, 346, 65, 186, 253, 330, 364, 382, 257, 283, 198, 64, 108, 194, 138, 160, 11, 277, 40, 363, 243, 51, 180, 235, 285, 310, 338, 342] + 28: [161, 317, 302, 52, 286, 4, 201, 196, 328, 152, 97, 122, 93, 7, 199, 108, 236, 3, 320, 223, 59, 158, 58, 261, 235, 157, 285, 163, 337, 299, 32, 105, 340, 349, 308, 193, 247, 380, 316, 134, 178, 91, 14, 27, 256, 179, 264, 171, 249, 44, 314, 375, 205, 143, 61, 296, 66, 210, 334, 280, 34, 151, 382, 372, 218, 96, 266, 80, 351, 284, 292, 234, 229, 47, 192, 281, 267, 111, 140, 94, 202, 371, 83, 145, 313, 164, 186, 9, 132, 190, 10, 18, 60, 86, 115, 150, 250, 92, 70, 162, 107, 241, 355, 248, 77, 283, 120, 172, 233, 253, 224, 374, 75, 346, 378, 137, 368, 228, 165, 5, 100, 383, 361, 339, 28, 220, 41, 125, 209, 182, 216, 33, 367, 114, 127, 268, 206, 333, 282, 121, 194, 142, 46, 265, 208, 358, 57, 274, 305, 177, 362, 106, 74, 12, 270, 332, 342, 366, 154, 377, 331, 117, 89, 149, 323, 39, 293, 303, 17, 72, 276, 232, 29, 76, 275, 277, 102, 350, 53, 254, 181, 318, 198, 212, 11, 138, 298, 156, 180, 207, 215, 257, 26, 344, 153, 306, 30, 99, 321, 0, 310, 324, 188, 226, 195, 103, 214, 37, 272, 263, 50, 166, 191, 42, 227, 204, 200, 352, 88, 160, 135, 67, 289, 242, 269, 221, 56, 13, 19, 273, 315, 16, 341, 335, 222, 1, 290, 244, 365, 260, 49, 6, 81, 64, 69, 98, 381, 322, 175, 169, 104, 15, 170, 363, 24, 73, 136, 31, 330, 119, 287, 345, 110, 38, 131, 8, 85, 237, 139, 55, 101, 373, 291, 126, 65, 189, 354, 187, 319, 262, 301, 40, 147, 359, 279, 311, 329, 336, 185, 168, 356, 255, 243, 25, 197, 312, 239, 22, 203, 176, 155, 288, 113, 307, 357, 369, 128, 219, 95, 325, 71, 51, 116, 63, 129, 90, 130, 167, 370, 326, 240, 271, 217, 20, 238, 376, 278, 48, 43, 300, 68, 184, 78, 118, 183, 112, 245, 327, 23, 295, 82, 159, 21, 148, 36, 258, 230, 84, 211, 133, 297, 294, 109, 144, 79, 364, 213, 348, 251, 225, 246, 231, 146, 347, 259, 252, 174, 35, 123, 45, 141, 304, 360, 309, 87, 124, 62, 2, 54, 173, 379, 338, 343, 353] + 29: [118, 318, 323, 203, 67, 23, 255, 89, 58, 365, 354, 258, 262, 320, 148, 16, 65, 140, 174, 87, 175, 188, 216, 204, 207, 210, 38, 50, 94, 132, 248, 298, 12, 106, 178, 329, 18, 347, 272, 59, 97, 181, 380, 142, 20, 313, 371, 217, 316, 238, 242, 36, 126, 170, 10, 28, 191, 55, 381, 56, 42, 7, 166, 13, 374, 352, 227, 307, 168, 149, 17, 71, 45, 337, 30, 246, 109, 49, 41, 201, 2, 80, 5, 226, 64, 81, 221, 342, 247, 274, 367, 249, 39, 355, 240, 285, 244, 336, 137, 304, 60, 282, 127, 283, 14, 54, 379, 339, 265, 186, 202, 346, 112, 180, 57, 134, 190, 343, 275, 370, 382, 123, 293, 86, 364, 172, 259, 215, 34, 257, 232, 349, 225, 72, 280, 90, 150, 345, 353, 292, 105, 205, 317, 167, 47, 122, 224, 312, 315, 40, 369, 147, 209, 79, 103, 220, 70, 153, 250, 161, 15, 233, 69, 62, 356, 306, 334, 99, 115, 35, 251, 92, 164, 96, 223, 254, 8, 100, 27, 286, 1, 372, 335, 281, 98, 256, 206, 143, 177, 108, 91, 152, 361, 196, 277, 295, 236, 44, 311, 271, 195, 124, 120, 333, 110, 309, 208, 160, 326, 119, 48, 159, 332, 63, 163, 145, 82, 102, 357, 278, 4, 324, 314, 218, 128, 116, 273, 222, 136, 19, 146, 228, 264, 78, 310, 267, 125, 269, 328, 344, 373, 22, 229, 348, 93, 52, 302, 135, 213, 111, 287, 327, 84, 299, 338, 261, 187, 239, 121, 194, 113, 219, 308, 154, 184, 211, 141, 296, 234, 155, 358, 279, 21, 330, 75, 130, 291, 165, 268, 138, 305, 289, 214, 6, 185, 26, 363, 176, 197, 73, 51, 198, 362, 173, 11, 383, 88, 25, 76, 129, 33, 74, 351, 266, 183, 243, 366, 375, 235, 24, 237, 68, 341, 319, 193, 350, 61, 360, 378, 85, 182, 139, 156, 200, 131, 300, 171, 276, 151, 101, 104, 3, 32, 294, 340, 77, 359, 83, 169, 31, 157, 230, 46, 377, 53, 303, 114, 368, 158, 9, 288, 199, 192, 253, 179, 270, 162, 376, 0, 43, 263, 252, 290, 297, 325, 301, 133, 189, 331, 95, 245, 260, 231, 107, 321, 241, 117, 284, 37, 144, 212, 322, 29, 66] + 30: [208, 358, 35, 373, 133, 184, 76, 29, 242, 135, 225, 257, 188, 321, 10, 64, 147, 17, 271, 216, 49, 89, 240, 336, 357, 100, 275, 236, 151, 299, 217, 382, 296, 356, 67, 98, 174, 96, 132, 167, 54, 329, 114, 306, 20, 51, 172, 286, 109, 211, 181, 7, 297, 175, 318, 337, 154, 342, 202, 107, 220, 247, 304, 203, 57, 116, 327, 370, 117, 364, 127, 355, 173, 16, 232, 126, 300, 56, 235, 80, 39, 171, 182, 87, 252, 131, 238, 331, 8, 21, 22, 31, 36, 40, 47, 88, 239, 187, 52, 143, 158, 124, 222, 137, 75, 78, 46, 168, 11, 223, 45, 68, 164, 205, 315, 303, 340, 312, 221, 12, 253, 339, 192, 193, 9, 136, 195, 292, 170, 281, 258, 298, 365, 104, 278, 84, 38, 179, 335, 162, 376, 310, 280, 293, 241, 55, 244, 282, 50, 290, 288, 25, 186, 276, 121, 183, 263, 324, 90, 326, 83, 69, 155, 5, 130, 81, 153, 15, 169, 190, 330, 198, 112, 270, 265, 149, 42, 367, 228, 119, 4, 27, 176, 301, 354, 94, 101, 118, 123, 128, 134, 146, 140, 245, 262, 53, 156, 289, 213, 333, 86, 243, 196, 378, 295, 361, 28, 144, 346, 33, 212, 250, 362, 159, 224, 341, 77, 177, 152, 256, 215, 141, 210, 320, 277, 18, 274, 160, 60, 23, 200, 91, 108, 72, 383, 150, 359, 14, 197, 111, 66, 180, 332, 115, 248, 92, 79, 374, 194, 166, 230, 95, 267, 350, 139, 279, 348, 99, 85, 287, 165, 82, 368, 291, 105, 233, 308, 113, 255, 207, 338, 313, 328, 63, 148, 93, 34, 191, 30, 201, 305, 363, 185, 189, 218, 219, 237, 259, 266, 65, 37, 214, 351, 62, 254, 272, 13, 366, 59, 24, 2, 206, 234, 246, 41, 283, 316, 302, 97, 43, 74, 138, 317, 1, 6, 71, 58, 309, 204, 226, 344, 260, 285, 369, 157, 343, 325, 353, 261, 231, 103, 142, 163, 319, 294, 129, 349, 375, 48, 73, 249, 44, 284, 145, 360, 110, 311, 377, 347, 371, 32, 264, 161, 381, 125, 323, 209, 273, 3, 122, 314, 19, 178, 120, 352, 102, 106, 199, 0, 372, 26, 61, 229, 251, 70, 227, 307, 379, 268, 269, 322, 334, 345, 380] + 31: [0, 100, 252, 278, 185, 227, 32, 221, 3, 116, 335, 25, 267, 268, 317, 325, 349, 155, 60, 171, 283, 208, 338, 276, 348, 189, 223, 33, 82, 254, 138, 63, 23, 98, 368, 2, 175, 65, 105, 31, 77, 287, 374, 312, 21, 288, 217, 16, 92, 49, 285, 4, 153, 237, 20, 271, 177, 323, 91, 220, 228, 281, 107, 80, 162, 42, 309, 129, 160, 340, 337, 234, 300, 109, 191, 7, 156, 382, 372, 124, 370, 242, 301, 332, 161, 311, 358, 5, 104, 13, 341, 212, 289, 229, 132, 207, 159, 68, 158, 239, 70, 76, 125, 192, 365, 379, 364, 69, 250, 187, 150, 39, 146, 209, 186, 188, 61, 34, 72, 56, 131, 286, 258, 296, 78, 236, 244, 353, 231, 38, 54, 245, 134, 169, 182, 205, 176, 190, 316, 284, 225, 213, 310, 302, 45, 142, 73, 354, 196, 51, 362, 247, 14, 272, 275, 346, 371, 194, 198, 304, 47, 306, 240, 139, 101, 298, 9, 24, 342, 113, 135, 216, 53, 181, 44, 274, 126, 35, 18, 27, 319, 218, 163, 230, 10, 117, 165, 89, 71, 90, 294, 350, 128, 201, 324, 115, 363, 87, 320, 265, 241, 279, 121, 133, 147, 95, 219, 330, 173, 22, 119, 333, 378, 75, 360, 62, 355, 214, 112, 290, 145, 380, 111, 152, 74, 232, 55, 66, 81, 88, 122, 322, 46, 318, 41, 256, 314, 118, 123, 336, 141, 84, 8, 195, 260, 12, 183, 11, 59, 266, 43, 351, 1, 48, 264, 103, 238, 226, 85, 210, 259, 157, 251, 168, 99, 327, 356, 280, 293, 235, 197, 282, 255, 313, 167, 174, 357, 257, 94, 377, 151, 154, 343, 148, 180, 149, 347, 329, 233, 26, 166, 143, 144, 261, 248, 331, 19, 36, 292, 326, 215, 270, 339, 328, 96, 140, 206, 58, 170, 253, 164, 299, 334, 102, 269, 106, 28, 211, 249, 93, 136, 17, 40, 243, 366, 57, 277, 137, 222, 15, 202, 97, 307, 373, 383, 303, 369, 291, 29, 110, 361, 273, 108, 305, 376, 6, 83, 352, 204, 130, 224, 50, 263, 308, 375, 345, 367, 178, 315, 262, 321, 64, 381, 52, 127, 79, 193, 344, 295, 114, 184, 179, 30, 359, 86, 199, 297, 200, 67, 203, 37, 246, 172, 120] + 32: [63, 241, 327, 344, 275, 135, 108, 362, 179, 21, 8, 66, 274, 41, 16, 186, 89, 202, 263, 211, 143, 156, 233, 109, 232, 58, 289, 301, 171, 298, 325, 85, 354, 199, 50, 25, 28, 157, 250, 230, 293, 142, 336, 350, 249, 371, 70, 4, 276, 381, 22, 198, 349, 236, 372, 5, 110, 136, 361, 284, 299, 287, 125, 238, 248, 117, 20, 356, 330, 201, 67, 52, 149, 188, 220, 224, 100, 23, 180, 321, 242, 88, 112, 318, 290, 93, 328, 54, 90, 300, 147, 14, 123, 169, 177, 181, 367, 92, 347, 51, 164, 329, 155, 268, 146, 106, 310, 370, 121, 12, 141, 359, 319, 225, 75, 184, 279, 140, 87, 116, 273, 113, 357, 345, 189, 369, 91, 227, 104, 71, 292, 175, 314, 64, 281, 48, 86, 182, 382, 183, 351, 353, 115, 84, 30, 337, 24, 10, 56, 62, 158, 363, 311, 144, 210, 42, 102, 191, 165, 124, 304, 341, 128, 73, 374, 35, 160, 119, 79, 203, 255, 334, 364, 133, 46, 216, 368, 358, 98, 192, 207, 291, 194, 15, 27, 114, 316, 167, 244, 245, 251, 262, 352, 148, 285, 163, 38, 166, 288, 246, 346, 187, 229, 271, 200, 2, 257, 342, 214, 204, 348, 174, 283, 126, 97, 131, 331, 335, 19, 120, 7, 313, 254, 212, 72, 36, 111, 69, 101, 332, 213, 324, 45, 150, 237, 256, 286, 53, 39, 176, 80, 205, 129, 6, 309, 340, 77, 193, 154, 172, 320, 282, 221, 132, 43, 243, 195, 376, 365, 222, 153, 264, 261, 305, 196, 138, 170, 208, 377, 31, 137, 76, 259, 95, 215, 74, 47, 272, 338, 11, 235, 326, 145, 379, 173, 265, 296, 312, 94, 218, 33, 190, 122, 107, 322, 234, 49, 209, 323, 130, 280, 360, 219, 61, 40, 380, 65, 96, 83, 151, 78, 258, 308, 228, 252, 294, 306, 168, 266, 297, 55, 303, 185, 231, 26, 206, 269, 239, 178, 59, 253, 32, 127, 29, 240, 81, 3, 302, 247, 315, 366, 37, 378, 139, 317, 339, 34, 82, 9, 99, 307, 152, 103, 197, 105, 13, 223, 134, 270, 44, 0, 355, 278, 343, 118, 1, 60, 277, 260, 18, 161, 17, 217, 159, 295, 226, 267, 333, 162, 57, 68, 383, 373, 375] + 33: [38, 362, 148, 251, 299, 94, 240, 145, 331, 153, 158, 379, 107, 73, 125, 358, 146, 139, 5, 48, 190, 21, 54, 160, 1, 338, 155, 237, 40, 334, 9, 14, 126, 232, 144, 187, 205, 138, 4, 317, 347, 22, 246, 19, 382, 367, 370, 88, 114, 60, 213, 368, 15, 345, 71, 336, 281, 52, 65, 53, 278, 248, 295, 221, 135, 6, 298, 111, 130, 243, 225, 291, 332, 245, 62, 134, 175, 242, 337, 97, 250, 171, 313, 309, 57, 296, 47, 77, 357, 0, 159, 264, 95, 69, 142, 220, 91, 177, 229, 115, 283, 267, 167, 140, 320, 280, 31, 42, 96, 297, 179, 348, 147, 108, 164, 226, 37, 335, 233, 141, 35, 25, 87, 227, 117, 356, 289, 217, 311, 305, 290, 211, 202, 355, 109, 174, 123, 45, 124, 375, 156, 55, 104, 200, 180, 279, 340, 105, 170, 249, 292, 330, 24, 28, 349, 166, 271, 285, 261, 327, 252, 194, 20, 316, 181, 286, 132, 381, 78, 188, 257, 365, 247, 353, 315, 363, 195, 275, 274, 269, 359, 172, 304, 203, 100, 143, 282, 129, 161, 12, 277, 287, 339, 260, 64, 81, 301, 307, 89, 328, 176, 63, 377, 323, 325, 39, 82, 314, 322, 201, 352, 266, 127, 169, 273, 376, 17, 33, 150, 151, 208, 308, 133, 222, 61, 18, 116, 303, 329, 173, 16, 254, 216, 198, 80, 196, 162, 157, 209, 373, 186, 212, 369, 234, 58, 49, 43, 371, 350, 66, 56, 70, 26, 112, 29, 41, 67, 318, 182, 23, 346, 11, 207, 76, 27, 119, 310, 118, 98, 59, 326, 165, 378, 306, 253, 7, 44, 321, 2, 294, 185, 223, 272, 189, 83, 34, 293, 333, 36, 319, 380, 343, 258, 154, 86, 51, 75, 259, 191, 121, 231, 183, 342, 68, 241, 84, 72, 206, 361, 255, 270, 204, 218, 30, 210, 193, 192, 131, 268, 288, 8, 235, 354, 351, 93, 149, 90, 230, 383, 79, 276, 262, 256, 197, 50, 163, 360, 152, 13, 364, 122, 137, 106, 3, 215, 113, 366, 103, 101, 110, 324, 219, 136, 10, 128, 238, 74, 300, 85, 168, 184, 224, 120, 312, 178, 244, 341, 199, 92, 214, 374, 228, 99, 344, 284, 46, 302, 263, 239, 265, 236, 32, 102, 372] + 34: [108, 366, 299, 248, 196, 382, 237, 332, 135, 158, 223, 243, 31, 167, 178, 315, 379, 284, 2, 271, 254, 11, 6, 4, 125, 225, 91, 153, 329, 356, 302, 345, 175, 24, 136, 37, 143, 280, 195, 181, 236, 231, 72, 209, 204, 170, 35, 5, 308, 348, 352, 63, 19, 318, 277, 56, 23, 261, 45, 51, 296, 192, 314, 13, 305, 262, 101, 226, 34, 307, 380, 52, 176, 370, 26, 39, 297, 141, 90, 1, 54, 355, 163, 344, 378, 303, 234, 369, 235, 66, 377, 279, 14, 76, 119, 122, 179, 340, 259, 334, 44, 372, 47, 357, 312, 311, 145, 185, 240, 148, 138, 258, 347, 59, 79, 238, 222, 92, 300, 65, 95, 208, 124, 110, 107, 184, 81, 159, 123, 327, 197, 60, 133, 317, 336, 328, 346, 150, 27, 287, 333, 102, 247, 263, 361, 103, 154, 140, 157, 42, 10, 371, 221, 142, 186, 147, 64, 152, 374, 219, 233, 36, 310, 286, 100, 255, 28, 109, 57, 242, 250, 55, 342, 171, 212, 22, 128, 43, 292, 182, 188, 38, 276, 376, 309, 273, 177, 211, 316, 132, 160, 189, 16, 15, 69, 330, 3, 61, 207, 324, 74, 239, 304, 321, 362, 291, 194, 146, 220, 115, 97, 8, 335, 78, 267, 77, 87, 201, 134, 373, 144, 53, 12, 84, 256, 368, 187, 137, 172, 217, 33, 245, 30, 227, 48, 214, 364, 169, 367, 230, 301, 215, 113, 322, 114, 323, 62, 298, 139, 46, 272, 264, 85, 68, 268, 228, 269, 111, 265, 274, 93, 331, 49, 117, 131, 278, 260, 155, 253, 9, 112, 71, 281, 120, 96, 216, 338, 252, 320, 7, 121, 375, 198, 213, 341, 193, 202, 229, 249, 306, 337, 126, 218, 190, 29, 151, 18, 82, 354, 25, 293, 203, 40, 360, 246, 99, 295, 83, 288, 191, 41, 168, 244, 162, 21, 200, 58, 290, 210, 174, 326, 313, 183, 166, 199, 206, 251, 70, 67, 241, 270, 73, 0, 359, 89, 116, 50, 232, 80, 180, 164, 294, 127, 283, 358, 149, 105, 383, 266, 98, 88, 275, 289, 282, 161, 86, 118, 353, 75, 319, 165, 205, 173, 17, 285, 339, 104, 20, 257, 351, 156, 130, 325, 349, 106, 94, 350, 129, 32, 343, 224, 381, 363, 365] + 35: [319, 28, 131, 68, 342, 322, 48, 220, 182, 144, 379, 368, 286, 47, 21, 31, 221, 206, 224, 41, 46, 265, 194, 50, 362, 52, 222, 329, 275, 102, 187, 267, 325, 128, 9, 158, 192, 268, 213, 108, 42, 313, 337, 12, 32, 170, 287, 74, 20, 111, 305, 106, 117, 347, 330, 273, 7, 184, 19, 301, 285, 43, 344, 6, 35, 198, 105, 63, 135, 207, 377, 115, 112, 26, 155, 89, 288, 167, 10, 101, 178, 351, 225, 33, 219, 258, 247, 280, 251, 120, 236, 100, 118, 304, 359, 177, 45, 169, 218, 295, 86, 72, 340, 353, 246, 237, 303, 56, 318, 141, 138, 129, 3, 346, 239, 15, 375, 311, 307, 241, 361, 266, 54, 159, 312, 134, 160, 360, 296, 18, 132, 61, 5, 328, 215, 372, 125, 143, 95, 122, 302, 96, 235, 350, 277, 227, 180, 91, 264, 382, 383, 24, 174, 185, 127, 142, 175, 217, 25, 338, 103, 65, 292, 87, 189, 253, 149, 245, 94, 249, 195, 262, 165, 263, 139, 107, 271, 327, 310, 371, 252, 336, 0, 39, 341, 229, 2, 64, 146, 381, 60, 228, 1, 75, 299, 297, 53, 276, 281, 80, 44, 240, 130, 254, 314, 55, 324, 153, 231, 116, 355, 186, 352, 349, 326, 335, 162, 14, 126, 27, 98, 163, 321, 140, 244, 272, 308, 36, 282, 104, 190, 334, 209, 320, 201, 300, 11, 16, 172, 92, 109, 145, 202, 348, 82, 339, 216, 290, 99, 179, 70, 196, 279, 315, 84, 374, 79, 274, 332, 151, 259, 29, 316, 278, 181, 306, 250, 354, 255, 358, 150, 238, 69, 83, 333, 123, 193, 208, 51, 119, 71, 76, 124, 283, 291, 284, 40, 293, 345, 363, 205, 367, 133, 203, 113, 309, 260, 62, 58, 211, 67, 257, 4, 38, 289, 154, 78, 147, 364, 114, 234, 164, 191, 331, 136, 37, 157, 373, 357, 168, 356, 81, 230, 171, 93, 298, 294, 30, 137, 243, 152, 90, 256, 323, 261, 57, 8, 366, 110, 214, 369, 204, 34, 166, 376, 269, 380, 17, 66, 73, 88, 378, 176, 317, 365, 248, 161, 156, 370, 22, 232, 121, 23, 49, 77, 97, 183, 148, 197, 270, 223, 233, 188, 242, 210, 226, 173, 212, 59, 199, 85, 200, 13, 343] + 36: [232, 340, 304, 325, 225, 107, 294, 74, 62, 2, 210, 196, 241, 369, 99, 301, 291, 133, 47, 287, 314, 32, 156, 45, 201, 309, 246, 288, 319, 191, 139, 13, 63, 105, 68, 245, 69, 183, 276, 153, 354, 117, 147, 343, 237, 116, 46, 134, 211, 30, 71, 39, 257, 36, 206, 165, 351, 263, 119, 159, 208, 51, 58, 239, 24, 70, 295, 265, 315, 173, 251, 357, 60, 307, 75, 14, 115, 350, 126, 26, 151, 285, 330, 327, 89, 269, 181, 306, 234, 113, 376, 290, 64, 37, 123, 170, 22, 53, 57, 10, 193, 223, 252, 382, 176, 359, 328, 260, 98, 242, 40, 373, 97, 207, 150, 5, 125, 85, 31, 42, 33, 162, 334, 67, 219, 166, 293, 49, 349, 132, 284, 91, 50, 172, 342, 155, 72, 142, 280, 179, 41, 66, 195, 202, 102, 217, 180, 0, 52, 38, 143, 335, 244, 358, 267, 78, 318, 283, 18, 146, 329, 83, 190, 109, 259, 381, 198, 199, 337, 316, 347, 118, 110, 312, 355, 235, 161, 278, 34, 96, 27, 136, 286, 222, 367, 182, 317, 186, 292, 127, 273, 302, 184, 272, 144, 266, 178, 12, 192, 197, 250, 185, 59, 230, 296, 320, 56, 90, 275, 352, 152, 205, 92, 233, 93, 331, 175, 321, 262, 112, 111, 194, 94, 55, 361, 122, 297, 121, 298, 300, 289, 339, 171, 168, 216, 140, 271, 100, 177, 277, 310, 255, 187, 264, 104, 81, 344, 17, 157, 148, 129, 23, 220, 365, 29, 135, 281, 209, 204, 189, 169, 338, 77, 248, 130, 35, 215, 137, 61, 254, 213, 8, 378, 154, 303, 128, 124, 353, 4, 106, 214, 160, 114, 145, 270, 16, 313, 360, 371, 368, 341, 238, 311, 372, 131, 370, 84, 348, 101, 299, 258, 326, 274, 380, 374, 73, 308, 79, 332, 54, 261, 65, 333, 240, 108, 362, 9, 231, 345, 282, 138, 167, 103, 95, 164, 322, 3, 20, 15, 346, 366, 174, 11, 228, 224, 364, 323, 305, 229, 200, 76, 212, 28, 379, 88, 188, 149, 80, 243, 163, 48, 356, 44, 363, 247, 6, 158, 236, 383, 19, 249, 256, 21, 120, 226, 324, 253, 87, 7, 86, 1, 203, 227, 82, 141, 218, 377, 221, 43, 268, 279, 25, 336, 375] + 37: [218, 58, 366, 113, 240, 123, 242, 213, 178, 46, 362, 190, 342, 72, 382, 77, 59, 171, 47, 360, 182, 14, 65, 139, 93, 343, 330, 18, 345, 145, 157, 19, 229, 228, 270, 196, 74, 144, 203, 230, 41, 39, 30, 111, 134, 29, 87, 100, 335, 247, 37, 121, 128, 380, 195, 97, 102, 284, 105, 354, 267, 154, 249, 299, 256, 17, 215, 170, 332, 169, 115, 297, 339, 198, 166, 96, 184, 324, 202, 281, 155, 20, 11, 219, 186, 367, 275, 23, 68, 67, 99, 31, 8, 75, 112, 132, 40, 109, 158, 79, 340, 71, 66, 309, 151, 125, 205, 372, 179, 290, 234, 95, 86, 16, 355, 302, 253, 252, 22, 327, 117, 235, 163, 32, 133, 192, 359, 322, 127, 320, 258, 21, 27, 1, 149, 217, 82, 73, 143, 161, 333, 201, 285, 3, 356, 55, 206, 331, 207, 269, 244, 373, 212, 279, 24, 33, 325, 351, 172, 298, 110, 124, 91, 266, 216, 350, 291, 183, 310, 374, 308, 248, 106, 225, 107, 237, 28, 177, 180, 45, 369, 90, 381, 282, 50, 232, 84, 209, 189, 236, 239, 288, 268, 293, 379, 222, 349, 0, 364, 243, 348, 311, 52, 199, 341, 272, 305, 193, 174, 223, 156, 344, 129, 9, 353, 165, 318, 328, 194, 118, 153, 277, 221, 315, 370, 316, 287, 76, 51, 173, 168, 323, 43, 61, 338, 304, 92, 278, 103, 329, 15, 126, 167, 83, 69, 80, 262, 347, 312, 94, 241, 38, 274, 4, 5, 116, 26, 160, 368, 53, 303, 260, 78, 137, 378, 35, 259, 81, 63, 357, 57, 130, 214, 326, 197, 300, 54, 159, 307, 12, 211, 346, 25, 276, 251, 56, 120, 352, 294, 148, 301, 200, 162, 188, 319, 273, 265, 314, 306, 363, 289, 48, 44, 292, 371, 361, 114, 140, 185, 245, 7, 88, 147, 204, 233, 181, 131, 257, 164, 150, 336, 208, 85, 122, 104, 176, 64, 6, 365, 296, 36, 141, 264, 108, 135, 376, 34, 136, 49, 224, 334, 337, 321, 98, 255, 313, 271, 317, 280, 42, 210, 187, 138, 13, 254, 10, 152, 60, 286, 175, 295, 101, 283, 191, 142, 62, 146, 231, 70, 227, 119, 261, 220, 250, 89, 2, 246, 358, 238, 377, 263, 375, 226, 383] + 38: [301, 124, 51, 126, 277, 282, 223, 50, 345, 94, 8, 285, 115, 220, 293, 63, 217, 27, 278, 212, 169, 206, 268, 108, 331, 199, 137, 288, 219, 352, 146, 147, 85, 208, 9, 59, 150, 105, 144, 323, 184, 256, 96, 367, 48, 229, 342, 276, 289, 260, 170, 28, 376, 33, 181, 15, 131, 152, 23, 173, 238, 214, 371, 56, 261, 100, 25, 215, 61, 294, 205, 71, 102, 290, 357, 192, 91, 254, 65, 42, 350, 132, 209, 303, 365, 83, 110, 155, 133, 243, 121, 159, 340, 46, 60, 185, 6, 221, 201, 30, 383, 333, 187, 213, 369, 118, 123, 263, 322, 248, 177, 230, 130, 244, 17, 373, 317, 77, 119, 267, 319, 338, 233, 361, 90, 75, 275, 271, 67, 72, 21, 193, 362, 36, 45, 57, 88, 264, 156, 368, 298, 143, 188, 228, 141, 148, 84, 356, 180, 149, 272, 103, 139, 2, 250, 200, 334, 313, 43, 74, 382, 375, 82, 252, 80, 297, 186, 13, 138, 202, 363, 316, 113, 320, 112, 259, 18, 175, 128, 116, 22, 377, 127, 1, 249, 270, 304, 161, 38, 101, 237, 251, 266, 378, 280, 372, 189, 154, 62, 10, 93, 348, 245, 292, 86, 226, 194, 32, 379, 19, 92, 145, 0, 328, 54, 253, 257, 234, 329, 16, 225, 341, 99, 222, 324, 239, 69, 29, 224, 358, 307, 87, 58, 255, 165, 308, 179, 34, 111, 246, 5, 265, 190, 241, 76, 134, 41, 174, 305, 235, 81, 310, 296, 360, 122, 195, 273, 104, 120, 204, 44, 295, 168, 283, 35, 183, 299, 3, 12, 164, 326, 157, 163, 314, 291, 153, 167, 218, 160, 196, 140, 330, 318, 312, 176, 52, 191, 325, 203, 158, 269, 284, 366, 114, 89, 4, 287, 178, 66, 332, 354, 274, 309, 49, 39, 171, 381, 166, 279, 135, 344, 300, 53, 97, 286, 327, 106, 125, 355, 107, 14, 227, 73, 95, 321, 359, 262, 68, 211, 349, 216, 136, 142, 7, 37, 109, 151, 117, 31, 306, 197, 26, 242, 258, 353, 40, 78, 337, 302, 182, 79, 172, 129, 347, 11, 231, 47, 311, 374, 351, 55, 247, 343, 236, 70, 240, 335, 24, 364, 232, 20, 207, 281, 315, 198, 336, 162, 370, 346, 64, 98, 339, 210, 380] + 39: [282, 56, 65, 132, 379, 100, 280, 264, 82, 348, 153, 13, 279, 267, 382, 86, 80, 270, 324, 26, 98, 17, 15, 319, 77, 49, 23, 381, 235, 375, 170, 352, 248, 340, 251, 112, 336, 101, 34, 47, 353, 177, 150, 281, 317, 364, 162, 301, 302, 228, 102, 311, 95, 185, 133, 285, 230, 221, 30, 195, 363, 183, 327, 60, 166, 283, 58, 70, 196, 346, 189, 117, 46, 157, 206, 231, 142, 37, 10, 39, 284, 326, 351, 370, 294, 318, 237, 262, 269, 344, 201, 29, 321, 9, 149, 22, 310, 236, 286, 38, 314, 43, 96, 180, 116, 368, 113, 215, 360, 271, 272, 2, 145, 296, 204, 374, 40, 141, 341, 369, 3, 365, 59, 146, 298, 84, 8, 259, 191, 48, 323, 122, 278, 190, 137, 260, 126, 358, 135, 188, 144, 307, 161, 249, 14, 373, 355, 106, 167, 320, 173, 354, 275, 6, 79, 309, 68, 377, 194, 214, 376, 291, 109, 217, 288, 19, 303, 107, 337, 108, 20, 246, 0, 252, 253, 238, 233, 339, 136, 158, 67, 205, 12, 316, 356, 184, 11, 90, 334, 89, 143, 198, 247, 372, 258, 312, 36, 85, 99, 268, 234, 1, 156, 218, 325, 155, 328, 244, 174, 362, 71, 172, 295, 330, 24, 76, 208, 7, 81, 200, 227, 202, 53, 42, 140, 232, 28, 31, 263, 69, 74, 131, 152, 50, 276, 371, 5, 63, 75, 226, 134, 182, 61, 94, 289, 241, 193, 357, 4, 338, 243, 130, 220, 306, 169, 181, 120, 171, 343, 27, 335, 103, 222, 111, 55, 266, 83, 160, 211, 64, 197, 229, 342, 210, 129, 349, 125, 32, 105, 73, 332, 261, 115, 148, 21, 52, 257, 44, 66, 104, 315, 93, 313, 176, 35, 110, 223, 25, 151, 139, 242, 119, 187, 292, 329, 297, 277, 333, 121, 250, 300, 178, 163, 51, 87, 290, 273, 118, 383, 274, 127, 54, 299, 225, 239, 138, 331, 378, 367, 62, 199, 366, 322, 179, 213, 345, 33, 212, 72, 293, 45, 88, 97, 203, 128, 164, 16, 254, 240, 114, 207, 265, 91, 216, 245, 192, 287, 41, 168, 159, 78, 305, 347, 175, 18, 123, 92, 359, 219, 255, 147, 350, 256, 124, 224, 308, 57, 154, 186, 380, 304, 165, 361, 209] + 40: [31, 349, 71, 108, 132, 280, 213, 204, 98, 283, 52, 107, 279, 153, 254, 64, 251, 198, 258, 228, 17, 347, 362, 363, 136, 168, 321, 63, 322, 211, 27, 30, 216, 188, 221, 302, 269, 137, 56, 53, 365, 57, 138, 25, 377, 94, 355, 192, 338, 324, 167, 206, 177, 330, 273, 37, 47, 314, 85, 261, 6, 36, 118, 121, 141, 256, 276, 184, 268, 310, 140, 117, 113, 295, 35, 242, 111, 231, 114, 354, 342, 315, 176, 223, 292, 112, 127, 220, 225, 240, 344, 170, 343, 194, 214, 248, 319, 317, 183, 328, 358, 346, 379, 286, 169, 55, 373, 101, 297, 160, 202, 161, 88, 372, 208, 58, 3, 100, 28, 311, 9, 143, 89, 326, 274, 341, 185, 289, 54, 145, 308, 199, 382, 196, 229, 259, 187, 309, 263, 178, 82, 189, 290, 103, 2, 294, 287, 93, 318, 305, 281, 243, 304, 350, 60, 51, 366, 201, 76, 238, 147, 78, 186, 288, 348, 102, 124, 383, 368, 174, 66, 203, 77, 34, 217, 173, 352, 154, 313, 380, 381, 245, 298, 172, 272, 83, 197, 312, 5, 69, 376, 300, 15, 378, 250, 142, 21, 33, 226, 340, 70, 359, 75, 265, 337, 128, 156, 8, 232, 369, 74, 235, 90, 59, 91, 275, 1, 284, 336, 271, 219, 80, 277, 166, 11, 278, 38, 236, 370, 148, 262, 307, 44, 316, 364, 95, 43, 244, 110, 306, 165, 29, 120, 327, 371, 329, 323, 237, 195, 131, 293, 233, 345, 139, 241, 291, 270, 115, 133, 14, 0, 23, 135, 13, 50, 16, 182, 163, 257, 106, 104, 22, 180, 234, 96, 4, 87, 45, 252, 24, 320, 126, 42, 335, 144, 353, 92, 123, 68, 175, 207, 360, 190, 179, 357, 260, 181, 67, 367, 62, 247, 26, 215, 158, 164, 332, 81, 224, 97, 351, 159, 210, 303, 296, 129, 10, 209, 162, 61, 253, 193, 334, 200, 12, 205, 227, 46, 239, 79, 99, 41, 150, 65, 299, 130, 151, 7, 155, 264, 339, 333, 356, 282, 48, 171, 19, 18, 301, 73, 222, 246, 375, 116, 122, 212, 361, 84, 125, 266, 32, 49, 374, 40, 230, 218, 134, 285, 119, 325, 267, 152, 39, 109, 157, 255, 86, 20, 191, 72, 146, 331, 249, 105, 149] + 41: [49, 321, 114, 184, 197, 159, 305, 367, 103, 251, 90, 86, 273, 365, 293, 76, 7, 172, 24, 279, 206, 334, 198, 253, 80, 326, 127, 222, 286, 202, 353, 126, 252, 349, 301, 56, 313, 225, 93, 55, 237, 229, 340, 344, 242, 330, 288, 268, 299, 101, 181, 284, 19, 208, 160, 317, 61, 220, 89, 354, 37, 359, 154, 292, 1, 39, 191, 77, 72, 13, 248, 149, 316, 148, 155, 137, 29, 238, 370, 47, 183, 110, 323, 319, 270, 106, 341, 362, 382, 291, 230, 52, 263, 31, 120, 85, 378, 374, 283, 190, 366, 119, 122, 335, 166, 376, 212, 57, 84, 368, 124, 285, 136, 111, 358, 360, 347, 12, 95, 82, 383, 318, 351, 35, 239, 219, 194, 163, 308, 26, 64, 70, 54, 226, 256, 83, 294, 235, 325, 138, 339, 133, 210, 8, 63, 151, 295, 290, 81, 277, 261, 43, 173, 315, 146, 192, 15, 232, 300, 33, 143, 78, 287, 165, 259, 207, 38, 68, 98, 310, 182, 28, 265, 171, 245, 373, 201, 121, 343, 88, 177, 162, 109, 356, 304, 281, 164, 233, 22, 178, 152, 132, 199, 350, 298, 355, 369, 275, 205, 185, 71, 27, 307, 255, 200, 50, 97, 96, 187, 34, 69, 161, 280, 311, 108, 306, 348, 9, 4, 53, 42, 25, 23, 218, 336, 174, 116, 345, 241, 282, 46, 260, 258, 322, 117, 289, 250, 224, 380, 51, 131, 176, 186, 338, 91, 246, 94, 5, 371, 74, 128, 303, 59, 123, 342, 188, 236, 100, 381, 180, 297, 234, 105, 48, 257, 73, 175, 204, 375, 231, 274, 17, 62, 267, 2, 145, 189, 312, 346, 361, 169, 193, 118, 333, 40, 247, 216, 309, 170, 60, 269, 142, 30, 129, 215, 87, 196, 11, 249, 150, 168, 223, 327, 92, 79, 364, 21, 314, 302, 262, 44, 337, 104, 45, 3, 324, 264, 140, 36, 243, 217, 134, 32, 102, 125, 379, 254, 276, 156, 271, 167, 115, 144, 112, 266, 328, 363, 66, 107, 153, 18, 6, 130, 213, 244, 10, 221, 214, 41, 99, 75, 272, 352, 179, 0, 65, 141, 320, 332, 158, 147, 113, 195, 209, 14, 67, 329, 20, 139, 331, 157, 240, 211, 377, 372, 16, 58, 203, 135, 228, 227, 296, 278, 357] + 42: [123, 172, 111, 138, 210, 1, 114, 119, 184, 247, 63, 282, 168, 222, 262, 377, 129, 208, 203, 189, 127, 373, 277, 328, 320, 4, 259, 236, 187, 28, 230, 165, 326, 338, 36, 92, 143, 45, 33, 362, 227, 192, 380, 113, 51, 19, 8, 231, 225, 323, 283, 139, 244, 226, 34, 298, 196, 16, 355, 270, 278, 112, 346, 120, 56, 318, 267, 82, 197, 360, 128, 292, 23, 288, 330, 367, 22, 85, 133, 353, 249, 100, 79, 331, 78, 271, 13, 86, 66, 361, 46, 126, 25, 71, 95, 254, 152, 94, 268, 155, 256, 37, 188, 20, 295, 106, 303, 116, 151, 264, 41, 337, 162, 200, 6, 48, 281, 21, 191, 379, 275, 370, 274, 30, 140, 135, 286, 214, 167, 105, 161, 216, 335, 322, 280, 316, 118, 76, 257, 160, 32, 80, 241, 301, 311, 62, 372, 9, 89, 315, 183, 232, 299, 258, 7, 204, 378, 68, 125, 300, 293, 213, 84, 284, 206, 90, 321, 363, 73, 185, 124, 334, 144, 174, 383, 96, 153, 190, 12, 11, 83, 44, 159, 43, 317, 91, 74, 24, 273, 27, 294, 324, 117, 217, 345, 218, 325, 333, 154, 50, 148, 357, 211, 212, 215, 72, 374, 349, 220, 55, 263, 59, 342, 319, 265, 235, 358, 141, 354, 341, 221, 169, 239, 17, 287, 173, 329, 356, 181, 38, 164, 202, 52, 223, 149, 297, 245, 60, 18, 132, 137, 307, 180, 64, 238, 49, 109, 99, 58, 205, 5, 336, 291, 53, 170, 171, 364, 343, 175, 272, 229, 98, 145, 368, 312, 296, 75, 186, 136, 207, 302, 88, 253, 276, 35, 70, 107, 54, 182, 142, 382, 240, 163, 40, 289, 121, 369, 339, 31, 359, 266, 176, 93, 158, 2, 375, 42, 228, 305, 350, 279, 150, 179, 243, 195, 108, 306, 255, 209, 327, 122, 351, 348, 131, 248, 29, 178, 224, 67, 344, 87, 201, 147, 103, 308, 77, 314, 234, 366, 166, 39, 65, 381, 340, 101, 146, 237, 371, 219, 246, 260, 130, 157, 97, 310, 313, 61, 251, 332, 156, 269, 233, 26, 194, 261, 290, 199, 102, 250, 10, 104, 285, 3, 198, 304, 309, 347, 134, 69, 115, 47, 57, 110, 81, 14, 177, 0, 15, 365, 252, 352, 193, 242, 376] + 43: [168, 105, 92, 359, 190, 298, 43, 344, 6, 177, 68, 112, 216, 20, 252, 103, 314, 152, 291, 55, 138, 322, 264, 178, 1, 305, 147, 78, 250, 255, 272, 73, 232, 282, 88, 327, 339, 306, 115, 299, 367, 36, 84, 157, 237, 10, 63, 107, 49, 24, 113, 180, 217, 56, 251, 74, 125, 101, 174, 370, 215, 295, 329, 122, 259, 279, 76, 132, 247, 118, 134, 358, 211, 290, 240, 179, 260, 341, 276, 230, 106, 62, 249, 236, 383, 40, 15, 91, 176, 350, 302, 361, 169, 206, 214, 245, 225, 221, 156, 335, 283, 297, 351, 27, 82, 278, 218, 35, 37, 13, 239, 94, 57, 326, 337, 153, 29, 301, 203, 318, 108, 274, 269, 321, 85, 374, 150, 2, 238, 90, 346, 204, 86, 219, 200, 26, 149, 5, 365, 67, 196, 194, 345, 145, 99, 60, 50, 165, 45, 253, 120, 46, 263, 9, 80, 288, 175, 159, 207, 72, 144, 267, 353, 163, 19, 158, 173, 109, 320, 300, 110, 336, 285, 51, 69, 124, 7, 172, 141, 281, 42, 270, 133, 96, 212, 33, 166, 376, 192, 182, 382, 3, 375, 315, 347, 17, 14, 30, 23, 262, 161, 332, 142, 129, 59, 349, 97, 100, 131, 235, 377, 186, 111, 127, 323, 313, 224, 338, 202, 280, 379, 4, 77, 222, 378, 154, 380, 275, 286, 234, 123, 271, 160, 34, 61, 130, 241, 334, 273, 12, 242, 293, 38, 47, 210, 266, 284, 167, 128, 146, 256, 325, 170, 116, 304, 64, 294, 71, 117, 87, 151, 287, 233, 119, 330, 355, 187, 357, 311, 201, 53, 114, 317, 328, 21, 228, 362, 340, 333, 104, 25, 289, 184, 54, 189, 331, 44, 348, 66, 98, 171, 372, 324, 75, 164, 369, 181, 308, 371, 197, 364, 39, 254, 366, 22, 363, 93, 81, 223, 342, 360, 227, 198, 307, 136, 121, 89, 368, 191, 32, 205, 52, 246, 48, 83, 277, 140, 135, 296, 261, 257, 79, 28, 155, 243, 148, 102, 185, 229, 183, 143, 16, 70, 213, 220, 309, 343, 226, 18, 31, 292, 11, 356, 8, 65, 126, 193, 381, 268, 209, 188, 265, 316, 258, 162, 312, 248, 199, 41, 0, 137, 303, 208, 352, 58, 139, 244, 319, 231, 310, 195, 373, 95, 354] + 44: [213, 226, 172, 101, 145, 238, 0, 272, 110, 60, 188, 257, 43, 252, 135, 325, 288, 93, 98, 86, 333, 141, 63, 329, 48, 144, 125, 291, 355, 89, 121, 262, 244, 192, 61, 296, 154, 173, 364, 70, 285, 130, 248, 286, 54, 174, 369, 134, 157, 342, 41, 26, 196, 370, 165, 159, 214, 105, 323, 309, 156, 81, 305, 237, 234, 57, 250, 345, 367, 49, 51, 232, 273, 42, 45, 216, 239, 373, 115, 99, 246, 363, 176, 302, 47, 293, 73, 361, 58, 88, 75, 371, 209, 13, 332, 374, 77, 317, 274, 379, 223, 170, 97, 258, 178, 155, 190, 215, 376, 277, 279, 331, 11, 166, 146, 280, 289, 242, 294, 198, 33, 228, 225, 95, 324, 94, 207, 91, 6, 350, 254, 290, 163, 219, 287, 39, 71, 79, 338, 378, 67, 359, 120, 44, 327, 107, 46, 136, 15, 108, 185, 334, 270, 243, 78, 347, 341, 169, 3, 218, 241, 72, 193, 357, 4, 204, 62, 7, 124, 245, 80, 1, 27, 195, 151, 194, 247, 372, 260, 307, 123, 297, 127, 31, 316, 148, 181, 16, 161, 189, 303, 116, 301, 10, 348, 171, 21, 168, 167, 25, 382, 271, 227, 84, 29, 68, 140, 311, 236, 143, 320, 383, 182, 249, 264, 142, 365, 304, 275, 337, 343, 276, 265, 138, 266, 85, 175, 186, 122, 55, 100, 56, 352, 132, 366, 160, 202, 96, 261, 381, 128, 292, 220, 344, 92, 278, 74, 282, 149, 8, 231, 377, 23, 32, 20, 113, 263, 203, 319, 18, 53, 256, 137, 65, 35, 312, 200, 326, 153, 5, 251, 210, 283, 28, 69, 268, 139, 356, 335, 111, 76, 133, 208, 281, 295, 129, 206, 322, 191, 199, 362, 222, 284, 346, 64, 17, 300, 351, 180, 267, 82, 328, 368, 229, 30, 34, 313, 187, 212, 118, 38, 306, 358, 102, 217, 360, 158, 14, 340, 117, 184, 224, 24, 299, 349, 150, 201, 315, 66, 131, 164, 380, 269, 87, 221, 177, 36, 162, 354, 126, 147, 83, 90, 40, 205, 59, 318, 19, 336, 211, 104, 179, 235, 230, 9, 109, 314, 321, 183, 114, 255, 2, 253, 375, 233, 22, 119, 50, 240, 298, 103, 330, 259, 37, 152, 52, 353, 197, 310, 308, 112, 339, 12, 106] + 45: [92, 161, 334, 33, 276, 81, 219, 233, 316, 346, 274, 176, 369, 160, 85, 269, 298, 348, 114, 209, 234, 331, 39, 198, 239, 196, 45, 115, 104, 30, 338, 84, 112, 327, 80, 345, 249, 156, 44, 260, 279, 365, 24, 1, 22, 222, 218, 43, 330, 34, 116, 87, 256, 296, 320, 283, 29, 282, 211, 263, 342, 177, 285, 105, 142, 15, 48, 78, 3, 190, 377, 136, 225, 332, 132, 62, 89, 82, 97, 232, 193, 194, 366, 259, 6, 335, 184, 11, 149, 240, 14, 40, 137, 192, 100, 174, 140, 23, 42, 27, 373, 95, 191, 301, 134, 110, 199, 120, 356, 375, 262, 378, 131, 21, 124, 297, 52, 183, 258, 228, 359, 168, 162, 158, 36, 51, 245, 250, 12, 68, 251, 267, 135, 213, 157, 94, 292, 175, 67, 2, 302, 303, 253, 370, 58, 35, 19, 170, 304, 252, 145, 358, 98, 150, 322, 241, 151, 127, 288, 204, 79, 363, 289, 203, 371, 113, 215, 71, 73, 238, 321, 102, 153, 56, 152, 143, 353, 200, 295, 53, 216, 77, 99, 325, 130, 49, 69, 299, 244, 138, 83, 221, 178, 154, 293, 201, 350, 329, 383, 318, 144, 317, 139, 197, 141, 247, 146, 354, 47, 165, 163, 349, 224, 380, 275, 351, 187, 223, 55, 122, 229, 155, 88, 372, 54, 328, 306, 357, 41, 86, 108, 179, 32, 315, 208, 96, 123, 376, 63, 340, 76, 65, 133, 126, 205, 319, 291, 5, 37, 294, 333, 231, 255, 381, 374, 314, 166, 0, 119, 309, 268, 147, 343, 93, 91, 337, 323, 125, 280, 18, 382, 226, 206, 305, 352, 148, 278, 287, 172, 66, 60, 195, 181, 72, 355, 361, 266, 347, 10, 227, 121, 8, 367, 164, 324, 173, 188, 284, 25, 182, 171, 339, 103, 286, 180, 61, 212, 128, 272, 107, 248, 254, 118, 307, 310, 16, 326, 277, 74, 70, 308, 313, 210, 7, 360, 242, 281, 217, 106, 26, 109, 236, 4, 261, 257, 50, 20, 344, 207, 230, 46, 290, 59, 38, 264, 9, 273, 379, 57, 169, 129, 271, 246, 220, 341, 312, 364, 368, 362, 202, 243, 111, 214, 90, 189, 31, 186, 167, 159, 270, 101, 265, 13, 311, 28, 75, 237, 235, 185, 17, 117, 336, 64, 300] + 46: [142, 34, 58, 355, 149, 106, 335, 188, 368, 327, 16, 51, 274, 6, 209, 12, 214, 60, 38, 165, 113, 44, 76, 219, 350, 215, 211, 62, 115, 195, 22, 227, 80, 232, 121, 168, 180, 109, 4, 13, 129, 362, 53, 273, 172, 216, 157, 158, 45, 57, 226, 373, 328, 97, 360, 24, 297, 223, 346, 69, 9, 317, 322, 333, 145, 264, 190, 370, 184, 244, 339, 26, 347, 163, 65, 43, 77, 310, 281, 72, 87, 381, 48, 174, 212, 29, 185, 0, 100, 28, 148, 270, 220, 379, 61, 122, 183, 247, 139, 380, 207, 367, 140, 210, 314, 240, 170, 238, 191, 324, 104, 301, 271, 136, 204, 114, 345, 2, 374, 3, 102, 39, 351, 228, 130, 33, 155, 372, 287, 334, 218, 315, 259, 137, 37, 50, 176, 23, 162, 250, 198, 267, 235, 193, 125, 377, 32, 307, 141, 78, 144, 169, 299, 161, 318, 348, 159, 293, 265, 342, 261, 283, 217, 93, 361, 305, 234, 375, 337, 15, 112, 21, 127, 330, 341, 126, 288, 73, 196, 181, 279, 116, 276, 84, 151, 167, 178, 323, 231, 5, 36, 175, 352, 10, 256, 253, 110, 354, 66, 201, 19, 46, 25, 202, 246, 332, 154, 344, 194, 187, 254, 55, 18, 303, 146, 203, 266, 85, 30, 280, 8, 117, 120, 83, 326, 132, 245, 197, 295, 192, 316, 166, 63, 41, 95, 128, 292, 237, 340, 321, 336, 383, 81, 308, 206, 92, 289, 255, 313, 222, 173, 382, 7, 243, 277, 241, 359, 91, 291, 309, 285, 278, 98, 257, 150, 52, 135, 75, 258, 11, 366, 236, 363, 105, 269, 251, 296, 179, 47, 123, 70, 357, 225, 365, 252, 27, 88, 213, 311, 133, 64, 286, 343, 230, 319, 152, 233, 224, 249, 96, 239, 221, 284, 312, 124, 40, 68, 378, 338, 199, 242, 320, 79, 282, 49, 131, 108, 143, 376, 260, 186, 99, 353, 118, 156, 42, 369, 298, 164, 304, 294, 272, 205, 103, 17, 59, 268, 111, 101, 329, 263, 35, 160, 290, 89, 119, 189, 56, 71, 74, 153, 20, 364, 177, 90, 331, 86, 262, 171, 371, 208, 138, 67, 358, 275, 94, 229, 31, 300, 107, 1, 134, 306, 248, 54, 349, 200, 14, 182, 82, 147, 325, 302, 356] + 47: [195, 312, 162, 382, 134, 347, 136, 357, 35, 170, 61, 333, 286, 320, 294, 196, 362, 129, 376, 118, 59, 123, 342, 235, 114, 176, 330, 269, 95, 104, 135, 42, 166, 285, 263, 284, 172, 184, 50, 203, 0, 101, 338, 45, 52, 106, 117, 234, 23, 139, 201, 243, 71, 126, 289, 281, 276, 306, 111, 107, 329, 310, 132, 248, 334, 10, 150, 19, 48, 282, 57, 325, 74, 290, 108, 58, 16, 38, 220, 85, 145, 142, 194, 78, 116, 125, 217, 221, 182, 83, 381, 236, 191, 369, 56, 288, 3, 24, 154, 158, 307, 137, 331, 192, 291, 279, 34, 227, 345, 90, 215, 257, 36, 274, 240, 287, 188, 327, 367, 218, 251, 122, 351, 250, 84, 69, 219, 377, 124, 151, 231, 103, 33, 65, 255, 197, 12, 205, 138, 148, 340, 233, 112, 164, 53, 183, 356, 91, 321, 211, 303, 28, 225, 328, 332, 258, 115, 200, 157, 2, 297, 81, 9, 46, 242, 189, 308, 266, 383, 207, 350, 174, 133, 86, 178, 222, 202, 247, 11, 64, 268, 161, 165, 256, 241, 261, 113, 169, 7, 239, 246, 293, 343, 352, 181, 22, 300, 271, 179, 143, 371, 119, 109, 193, 309, 14, 368, 127, 344, 79, 168, 110, 40, 346, 153, 363, 159, 212, 226, 265, 208, 93, 301, 67, 224, 249, 228, 41, 323, 140, 364, 77, 204, 51, 355, 237, 180, 173, 252, 210, 96, 378, 18, 80, 15, 348, 163, 147, 259, 292, 278, 60, 49, 341, 186, 20, 97, 144, 160, 102, 365, 275, 55, 72, 177, 245, 5, 380, 339, 30, 105, 277, 47, 76, 270, 244, 82, 167, 232, 295, 8, 254, 128, 311, 156, 62, 373, 326, 54, 44, 87, 29, 359, 75, 187, 171, 43, 375, 1, 280, 314, 305, 316, 353, 198, 175, 267, 21, 238, 131, 229, 349, 63, 98, 313, 230, 155, 26, 262, 68, 89, 223, 324, 319, 298, 370, 199, 73, 296, 322, 17, 25, 39, 4, 354, 27, 315, 366, 141, 336, 70, 121, 335, 213, 273, 337, 94, 37, 31, 185, 32, 260, 130, 253, 190, 149, 120, 152, 302, 216, 379, 358, 374, 92, 304, 100, 317, 361, 372, 88, 299, 264, 99, 283, 6, 206, 66, 13, 272, 318, 214, 146, 209, 360] + 48: [17, 76, 102, 276, 353, 78, 62, 297, 26, 270, 279, 215, 99, 236, 182, 295, 33, 28, 166, 189, 177, 240, 38, 264, 34, 342, 181, 305, 320, 371, 229, 312, 322, 7, 25, 257, 370, 112, 190, 0, 160, 255, 321, 357, 315, 347, 79, 13, 379, 202, 113, 281, 72, 289, 97, 11, 84, 43, 349, 16, 151, 292, 94, 274, 261, 39, 187, 47, 217, 92, 67, 313, 27, 163, 294, 105, 110, 309, 188, 191, 8, 338, 60, 106, 296, 248, 23, 383, 103, 337, 199, 258, 30, 245, 196, 227, 271, 283, 330, 273, 284, 85, 15, 288, 20, 77, 145, 109, 234, 121, 74, 164, 268, 207, 369, 147, 344, 139, 48, 254, 142, 115, 134, 252, 155, 290, 210, 300, 161, 175, 192, 50, 120, 247, 88, 340, 329, 341, 212, 24, 168, 172, 42, 231, 307, 22, 80, 351, 119, 96, 302, 81, 198, 269, 367, 40, 206, 346, 107, 137, 108, 82, 272, 173, 366, 214, 69, 373, 224, 37, 87, 278, 244, 350, 361, 225, 298, 362, 282, 125, 46, 183, 262, 176, 220, 265, 355, 336, 136, 304, 114, 250, 140, 267, 266, 52, 311, 141, 237, 159, 249, 65, 301, 358, 19, 44, 2, 90, 21, 83, 104, 6, 5, 235, 359, 363, 186, 316, 184, 354, 319, 378, 222, 156, 130, 246, 299, 152, 285, 75, 45, 73, 89, 118, 242, 111, 4, 318, 91, 360, 180, 339, 241, 14, 138, 146, 36, 253, 211, 280, 12, 53, 55, 144, 35, 259, 243, 178, 71, 286, 95, 194, 93, 3, 56, 143, 277, 335, 66, 228, 356, 148, 154, 86, 345, 153, 124, 251, 54, 122, 303, 364, 131, 348, 10, 200, 129, 308, 328, 132, 128, 239, 101, 381, 263, 9, 51, 117, 150, 226, 116, 343, 203, 58, 133, 157, 287, 375, 223, 327, 208, 372, 325, 185, 380, 41, 232, 368, 205, 219, 193, 179, 293, 291, 233, 174, 123, 324, 162, 170, 31, 221, 230, 275, 218, 57, 317, 64, 100, 70, 49, 158, 204, 63, 374, 352, 18, 127, 98, 169, 59, 195, 382, 216, 256, 32, 135, 310, 326, 29, 165, 323, 213, 61, 201, 238, 376, 149, 171, 331, 377, 260, 365, 332, 68, 126, 306, 197, 333, 167, 1, 209, 334, 314] + 49: [302, 6, 305, 271, 193, 197, 346, 15, 234, 158, 116, 78, 25, 40, 153, 148, 259, 60, 380, 76, 289, 220, 187, 312, 255, 30, 332, 94, 109, 122, 58, 97, 169, 362, 382, 28, 221, 23, 311, 196, 198, 290, 189, 34, 41, 248, 20, 301, 295, 371, 238, 267, 330, 338, 227, 359, 239, 339, 320, 117, 378, 0, 61, 106, 49, 206, 342, 228, 143, 108, 145, 125, 88, 168, 316, 377, 51, 7, 341, 42, 159, 287, 161, 379, 141, 207, 291, 252, 129, 50, 24, 68, 300, 317, 64, 123, 321, 79, 268, 19, 202, 210, 37, 83, 304, 136, 181, 213, 105, 278, 110, 335, 180, 39, 325, 292, 107, 243, 306, 269, 381, 186, 175, 183, 298, 5, 195, 121, 250, 254, 174, 340, 80, 54, 86, 70, 138, 365, 29, 89, 160, 222, 296, 273, 225, 82, 10, 327, 352, 36, 134, 307, 103, 244, 14, 72, 53, 226, 119, 118, 200, 124, 84, 285, 283, 154, 22, 299, 182, 232, 274, 343, 93, 13, 87, 372, 96, 162, 334, 236, 171, 152, 217, 11, 247, 32, 3, 347, 188, 322, 337, 191, 297, 374, 272, 155, 313, 9, 251, 149, 294, 209, 65, 150, 353, 66, 241, 208, 201, 276, 262, 177, 286, 91, 260, 74, 120, 329, 45, 264, 95, 131, 333, 48, 205, 328, 73, 1, 63, 310, 16, 368, 240, 277, 318, 128, 224, 4, 127, 77, 146, 55, 216, 179, 33, 67, 43, 178, 156, 351, 172, 114, 85, 280, 69, 358, 355, 31, 2, 151, 282, 101, 361, 52, 354, 348, 326, 357, 166, 27, 147, 167, 165, 367, 266, 231, 265, 350, 281, 263, 230, 203, 288, 26, 370, 349, 375, 356, 253, 126, 360, 75, 157, 242, 233, 184, 319, 257, 59, 115, 293, 212, 261, 270, 90, 275, 44, 199, 218, 133, 246, 132, 344, 194, 331, 8, 100, 314, 164, 235, 144, 366, 336, 279, 17, 137, 245, 56, 223, 140, 170, 214, 112, 323, 163, 81, 376, 139, 303, 173, 308, 35, 98, 62, 309, 142, 345, 211, 47, 176, 71, 130, 12, 46, 204, 192, 284, 190, 113, 256, 364, 185, 324, 104, 111, 57, 315, 229, 363, 237, 102, 18, 21, 99, 249, 215, 383, 258, 369, 219, 373, 92, 135, 38] + 50: [86, 181, 315, 69, 214, 350, 129, 57, 226, 137, 333, 367, 361, 35, 146, 175, 282, 338, 327, 0, 193, 114, 22, 343, 253, 15, 159, 187, 132, 122, 164, 244, 306, 58, 117, 203, 194, 168, 219, 43, 189, 342, 180, 139, 382, 74, 92, 64, 277, 126, 78, 99, 340, 223, 113, 5, 236, 17, 39, 292, 4, 344, 123, 106, 349, 308, 156, 141, 237, 150, 90, 36, 65, 112, 82, 346, 30, 381, 176, 246, 207, 328, 265, 104, 95, 54, 309, 260, 115, 59, 336, 182, 222, 50, 111, 345, 268, 49, 32, 215, 321, 26, 3, 33, 235, 140, 97, 284, 370, 273, 373, 312, 377, 329, 45, 217, 358, 119, 295, 155, 224, 270, 213, 298, 133, 24, 301, 313, 239, 228, 195, 375, 348, 290, 240, 53, 206, 269, 276, 94, 371, 234, 73, 107, 294, 160, 2, 205, 211, 185, 108, 300, 178, 369, 127, 337, 289, 339, 177, 351, 7, 305, 12, 248, 330, 355, 354, 171, 98, 179, 11, 324, 101, 10, 83, 201, 173, 190, 72, 31, 266, 120, 167, 227, 52, 368, 307, 84, 23, 264, 136, 365, 29, 109, 376, 210, 232, 56, 18, 231, 25, 299, 67, 14, 352, 262, 110, 366, 9, 62, 296, 220, 334, 256, 314, 51, 184, 130, 121, 249, 274, 6, 41, 96, 71, 48, 19, 165, 154, 116, 359, 102, 247, 319, 174, 208, 267, 151, 378, 81, 152, 20, 242, 91, 357, 172, 278, 356, 135, 347, 297, 191, 204, 153, 42, 170, 34, 63, 320, 46, 212, 89, 258, 148, 138, 285, 286, 87, 40, 60, 13, 21, 47, 8, 360, 245, 261, 279, 161, 225, 317, 254, 131, 44, 283, 241, 55, 230, 100, 316, 166, 188, 325, 27, 1, 128, 80, 311, 229, 38, 233, 218, 125, 287, 280, 304, 192, 199, 341, 88, 331, 79, 272, 271, 335, 16, 353, 363, 196, 275, 380, 149, 186, 75, 263, 183, 134, 379, 252, 169, 197, 250, 310, 85, 326, 303, 163, 322, 255, 145, 105, 28, 251, 143, 37, 243, 362, 202, 281, 288, 124, 259, 198, 200, 372, 238, 68, 383, 318, 293, 162, 323, 291, 147, 77, 209, 61, 76, 157, 103, 70, 216, 142, 158, 144, 221, 302, 118, 66, 364, 332, 257, 93, 374] + 51: [24, 221, 110, 208, 342, 229, 108, 73, 178, 16, 113, 52, 127, 64, 358, 65, 87, 104, 260, 235, 330, 105, 153, 177, 269, 346, 134, 247, 13, 46, 118, 85, 331, 212, 344, 97, 172, 341, 56, 107, 18, 329, 190, 59, 90, 370, 251, 33, 141, 3, 361, 234, 157, 205, 262, 317, 213, 115, 312, 381, 379, 54, 328, 80, 307, 319, 273, 340, 197, 53, 238, 167, 184, 68, 66, 57, 4, 215, 14, 158, 154, 0, 132, 355, 175, 179, 266, 29, 22, 244, 267, 155, 327, 138, 156, 202, 186, 375, 210, 12, 337, 126, 322, 189, 6, 111, 123, 182, 356, 136, 67, 62, 152, 60, 40, 301, 325, 304, 146, 253, 103, 130, 263, 5, 259, 143, 20, 220, 183, 47, 382, 81, 74, 142, 359, 162, 360, 209, 351, 310, 204, 119, 309, 166, 50, 75, 48, 239, 320, 364, 171, 84, 196, 94, 349, 45, 51, 223, 338, 288, 145, 78, 363, 159, 58, 109, 112, 240, 218, 116, 148, 83, 354, 296, 88, 165, 174, 77, 217, 133, 293, 302, 352, 117, 36, 347, 377, 173, 121, 151, 249, 261, 372, 373, 37, 323, 17, 82, 286, 106, 101, 98, 357, 284, 180, 176, 72, 211, 343, 270, 32, 95, 250, 233, 283, 168, 246, 49, 314, 26, 299, 216, 374, 23, 285, 245, 275, 308, 345, 86, 268, 279, 42, 366, 280, 79, 34, 91, 160, 252, 10, 195, 236, 224, 306, 69, 289, 313, 100, 332, 334, 297, 350, 27, 114, 194, 276, 278, 333, 55, 61, 336, 316, 25, 125, 149, 1, 96, 9, 7, 257, 199, 277, 120, 376, 227, 71, 187, 63, 318, 225, 38, 131, 21, 226, 139, 291, 326, 102, 137, 241, 200, 287, 144, 163, 31, 371, 206, 99, 193, 228, 122, 264, 300, 140, 135, 203, 124, 188, 129, 295, 282, 243, 11, 335, 92, 8, 35, 294, 281, 292, 271, 161, 303, 254, 237, 198, 380, 219, 222, 181, 192, 207, 248, 321, 368, 44, 362, 41, 164, 265, 324, 311, 369, 383, 258, 365, 315, 89, 170, 191, 30, 242, 298, 93, 353, 339, 378, 43, 232, 19, 15, 214, 230, 28, 39, 272, 201, 305, 128, 367, 70, 169, 185, 274, 147, 150, 231, 256, 76, 255, 2, 290, 348] + 52: [381, 104, 283, 336, 25, 327, 120, 216, 371, 63, 362, 85, 290, 345, 169, 77, 90, 183, 172, 62, 357, 44, 259, 192, 266, 32, 8, 354, 257, 325, 312, 133, 339, 222, 234, 191, 178, 359, 30, 23, 364, 75, 243, 369, 113, 361, 53, 115, 22, 213, 324, 208, 125, 109, 161, 225, 270, 237, 200, 177, 294, 282, 152, 207, 220, 230, 136, 249, 143, 28, 310, 322, 226, 340, 4, 96, 353, 135, 232, 276, 297, 329, 314, 203, 159, 65, 122, 142, 317, 14, 295, 272, 204, 370, 180, 247, 99, 71, 103, 137, 48, 186, 307, 61, 320, 182, 250, 289, 195, 174, 263, 298, 7, 382, 190, 33, 260, 140, 196, 219, 87, 54, 95, 50, 291, 88, 228, 379, 127, 378, 262, 5, 83, 315, 215, 160, 233, 248, 6, 300, 153, 72, 331, 351, 279, 316, 55, 287, 253, 326, 268, 235, 352, 57, 242, 258, 365, 20, 21, 194, 51, 47, 330, 347, 303, 101, 380, 118, 344, 18, 126, 311, 358, 138, 274, 363, 9, 36, 185, 42, 273, 265, 338, 144, 373, 254, 376, 360, 162, 16, 305, 375, 334, 157, 165, 239, 52, 92, 332, 217, 31, 130, 73, 214, 321, 206, 193, 163, 184, 13, 346, 68, 245, 80, 128, 168, 210, 252, 64, 98, 70, 37, 123, 108, 256, 285, 117, 166, 296, 227, 124, 167, 29, 116, 280, 292, 3, 76, 17, 319, 39, 69, 333, 91, 45, 27, 308, 350, 58, 201, 383, 284, 148, 59, 84, 251, 318, 150, 368, 129, 188, 156, 141, 342, 43, 81, 299, 313, 181, 293, 112, 147, 132, 145, 187, 79, 11, 202, 366, 241, 1, 86, 94, 199, 218, 46, 175, 348, 102, 107, 97, 209, 205, 121, 231, 372, 267, 151, 246, 323, 271, 34, 211, 82, 78, 41, 288, 60, 10, 26, 56, 269, 355, 349, 149, 236, 309, 35, 155, 171, 74, 221, 286, 105, 341, 281, 173, 223, 367, 134, 146, 67, 301, 328, 154, 377, 374, 170, 66, 337, 40, 306, 119, 93, 89, 2, 110, 277, 224, 244, 238, 189, 139, 179, 100, 198, 49, 356, 278, 197, 111, 114, 335, 343, 229, 304, 302, 275, 158, 261, 255, 131, 12, 19, 15, 0, 24, 240, 164, 264, 106, 212, 38, 176] + 53: [110, 311, 330, 308, 68, 297, 66, 173, 322, 273, 363, 155, 92, 132, 156, 6, 351, 194, 261, 144, 130, 121, 368, 284, 319, 127, 219, 256, 62, 262, 69, 361, 116, 205, 37, 313, 49, 373, 149, 47, 349, 315, 293, 282, 67, 226, 340, 120, 339, 178, 375, 371, 327, 96, 19, 28, 255, 123, 299, 220, 294, 0, 195, 275, 201, 7, 137, 246, 9, 249, 82, 51, 310, 147, 348, 64, 229, 372, 135, 260, 356, 190, 27, 181, 286, 187, 355, 245, 290, 265, 252, 167, 316, 114, 210, 272, 240, 13, 334, 109, 347, 32, 230, 217, 222, 238, 129, 142, 46, 379, 179, 170, 302, 160, 360, 268, 291, 237, 81, 119, 221, 31, 20, 24, 95, 325, 200, 94, 346, 141, 341, 383, 77, 172, 378, 197, 277, 176, 281, 89, 76, 44, 102, 143, 328, 36, 248, 216, 169, 65, 166, 57, 207, 236, 117, 333, 133, 364, 232, 59, 55, 54, 337, 352, 53, 367, 131, 264, 43, 100, 177, 39, 298, 74, 106, 366, 93, 208, 382, 211, 41, 192, 343, 33, 204, 11, 99, 223, 314, 56, 18, 185, 73, 317, 350, 287, 354, 250, 336, 8, 243, 270, 320, 292, 214, 345, 227, 228, 266, 180, 209, 309, 376, 162, 225, 103, 285, 50, 269, 253, 171, 3, 88, 139, 12, 70, 338, 231, 188, 152, 324, 78, 52, 10, 150, 15, 159, 122, 175, 362, 357, 128, 35, 196, 206, 105, 276, 189, 306, 278, 79, 136, 40, 198, 218, 242, 97, 258, 257, 234, 108, 14, 289, 283, 323, 199, 48, 63, 342, 301, 113, 34, 163, 374, 263, 161, 91, 212, 98, 165, 304, 191, 344, 244, 377, 90, 140, 84, 38, 16, 23, 296, 295, 4, 186, 75, 203, 247, 164, 280, 365, 224, 42, 118, 145, 279, 202, 86, 157, 332, 26, 271, 134, 125, 107, 326, 241, 5, 168, 329, 25, 381, 112, 183, 151, 239, 335, 300, 235, 60, 259, 72, 126, 353, 359, 146, 233, 369, 111, 115, 2, 174, 321, 158, 370, 30, 215, 358, 138, 182, 251, 254, 101, 45, 87, 21, 154, 80, 380, 305, 331, 29, 83, 85, 1, 193, 17, 184, 303, 104, 267, 148, 288, 61, 124, 213, 274, 318, 71, 58, 307, 22, 153, 312] + 54: [363, 258, 231, 280, 95, 296, 328, 147, 287, 150, 214, 250, 8, 72, 176, 120, 345, 305, 352, 40, 7, 3, 317, 58, 10, 132, 76, 333, 196, 215, 158, 357, 225, 45, 5, 190, 272, 109, 279, 239, 184, 105, 186, 355, 131, 100, 246, 278, 264, 366, 85, 293, 137, 242, 304, 81, 167, 173, 347, 300, 275, 191, 238, 194, 251, 12, 88, 224, 235, 168, 211, 236, 289, 79, 268, 343, 362, 226, 270, 285, 116, 307, 28, 379, 2, 112, 37, 165, 349, 200, 358, 354, 0, 36, 148, 232, 370, 284, 11, 240, 230, 295, 153, 319, 83, 267, 82, 369, 98, 53, 281, 208, 44, 187, 87, 119, 309, 118, 75, 329, 181, 332, 276, 143, 47, 218, 129, 64, 335, 124, 323, 94, 265, 92, 159, 382, 381, 21, 157, 312, 55, 302, 183, 67, 144, 373, 301, 163, 327, 22, 209, 201, 146, 353, 195, 18, 244, 316, 175, 266, 245, 271, 63, 213, 15, 234, 74, 247, 249, 31, 261, 89, 359, 60, 46, 227, 360, 154, 51, 104, 134, 77, 252, 126, 207, 78, 344, 130, 106, 13, 320, 336, 294, 164, 42, 223, 38, 286, 337, 253, 189, 90, 216, 260, 30, 20, 306, 341, 282, 290, 330, 311, 26, 180, 314, 86, 27, 121, 375, 288, 41, 310, 113, 197, 277, 84, 17, 220, 321, 185, 34, 182, 54, 103, 140, 204, 97, 365, 315, 318, 1, 73, 298, 48, 255, 111, 107, 313, 62, 96, 33, 162, 151, 262, 23, 9, 351, 192, 229, 334, 259, 110, 346, 145, 14, 367, 206, 233, 269, 263, 66, 102, 56, 374, 292, 65, 210, 169, 138, 188, 241, 297, 193, 383, 212, 123, 160, 361, 339, 155, 283, 356, 348, 299, 380, 71, 338, 39, 257, 80, 133, 172, 174, 324, 254, 93, 221, 57, 25, 117, 372, 228, 16, 101, 156, 70, 99, 115, 32, 161, 322, 198, 50, 91, 325, 152, 217, 142, 24, 125, 170, 376, 237, 368, 49, 135, 177, 61, 171, 69, 68, 326, 122, 136, 179, 364, 114, 377, 166, 273, 43, 128, 141, 139, 243, 371, 308, 219, 6, 59, 178, 29, 274, 331, 52, 108, 248, 203, 199, 149, 35, 303, 4, 350, 291, 222, 256, 342, 340, 19, 202, 127, 205, 378] + 55: [154, 20, 340, 123, 347, 130, 113, 159, 256, 309, 7, 357, 339, 238, 111, 6, 211, 78, 382, 132, 49, 341, 293, 114, 179, 60, 333, 368, 216, 366, 383, 124, 233, 17, 46, 348, 284, 91, 274, 260, 378, 224, 246, 343, 79, 53, 228, 140, 19, 119, 122, 128, 171, 52, 94, 33, 314, 214, 264, 1, 342, 127, 365, 173, 15, 335, 372, 93, 201, 38, 55, 147, 161, 125, 66, 68, 324, 162, 183, 364, 295, 175, 334, 75, 110, 195, 100, 24, 351, 292, 45, 310, 222, 245, 196, 34, 174, 98, 308, 381, 48, 180, 115, 72, 327, 235, 4, 370, 39, 311, 192, 267, 353, 273, 70, 65, 106, 249, 134, 71, 3, 272, 44, 138, 221, 312, 26, 156, 239, 197, 163, 325, 276, 352, 289, 371, 27, 377, 231, 47, 306, 286, 187, 185, 263, 184, 318, 255, 157, 207, 101, 149, 313, 63, 31, 237, 232, 281, 367, 208, 219, 253, 97, 126, 336, 168, 243, 58, 354, 150, 145, 261, 13, 166, 77, 376, 298, 181, 282, 36, 266, 288, 158, 59, 131, 379, 205, 144, 250, 317, 69, 305, 56, 81, 146, 116, 90, 323, 29, 326, 152, 43, 151, 62, 218, 89, 252, 227, 328, 41, 247, 363, 301, 40, 2, 223, 226, 279, 12, 331, 164, 213, 316, 271, 206, 337, 189, 112, 102, 14, 95, 105, 210, 217, 236, 265, 240, 67, 178, 54, 332, 322, 257, 346, 23, 254, 193, 84, 87, 22, 117, 153, 212, 160, 103, 133, 37, 225, 268, 356, 319, 118, 209, 104, 338, 120, 275, 374, 139, 177, 203, 242, 141, 92, 262, 21, 244, 32, 108, 220, 345, 148, 167, 9, 51, 96, 109, 204, 202, 88, 11, 80, 299, 74, 269, 76, 28, 270, 355, 234, 182, 165, 16, 307, 303, 73, 121, 304, 199, 291, 200, 99, 176, 8, 320, 349, 290, 300, 0, 86, 344, 172, 259, 5, 170, 350, 190, 375, 329, 251, 380, 361, 360, 42, 135, 194, 136, 241, 10, 277, 287, 358, 278, 85, 296, 294, 302, 82, 198, 61, 258, 188, 143, 373, 359, 30, 35, 321, 297, 137, 129, 315, 107, 50, 64, 248, 369, 280, 169, 191, 155, 330, 362, 57, 142, 83, 230, 18, 229, 25, 283, 186, 285, 215] + 56: [231, 242, 82, 241, 30, 147, 8, 126, 273, 53, 98, 303, 371, 169, 356, 236, 333, 125, 117, 161, 71, 99, 292, 18, 332, 327, 154, 245, 283, 101, 383, 309, 197, 233, 334, 27, 285, 281, 137, 278, 92, 181, 252, 79, 354, 269, 12, 22, 11, 34, 153, 243, 141, 282, 364, 42, 378, 122, 201, 299, 298, 96, 145, 218, 228, 132, 307, 200, 330, 276, 65, 367, 78, 144, 352, 222, 373, 24, 211, 258, 26, 256, 215, 370, 326, 32, 377, 365, 114, 247, 235, 61, 76, 74, 146, 3, 240, 321, 178, 15, 340, 75, 100, 343, 199, 116, 163, 158, 351, 156, 20, 89, 264, 306, 162, 328, 316, 284, 275, 302, 19, 66, 159, 357, 244, 192, 72, 97, 344, 103, 300, 182, 69, 342, 193, 128, 324, 305, 31, 295, 374, 268, 317, 149, 214, 138, 13, 17, 127, 185, 56, 286, 115, 313, 360, 94, 55, 134, 262, 341, 310, 290, 323, 226, 379, 57, 139, 372, 266, 170, 43, 350, 113, 204, 131, 337, 108, 272, 68, 212, 59, 336, 45, 216, 380, 171, 237, 102, 109, 142, 173, 90, 195, 294, 44, 225, 106, 5, 279, 311, 191, 107, 155, 296, 267, 87, 189, 249, 104, 251, 308, 346, 320, 140, 219, 2, 83, 329, 288, 381, 168, 150, 120, 223, 369, 0, 58, 73, 38, 160, 250, 10, 105, 21, 136, 35, 179, 23, 355, 280, 50, 110, 151, 164, 123, 209, 129, 297, 190, 339, 62, 271, 183, 180, 84, 119, 172, 347, 28, 4, 232, 175, 291, 16, 29, 49, 70, 361, 93, 152, 254, 345, 234, 325, 239, 349, 261, 7, 36, 217, 25, 174, 91, 318, 301, 85, 315, 143, 260, 220, 265, 213, 259, 304, 177, 167, 81, 186, 88, 184, 322, 9, 196, 263, 363, 187, 6, 46, 48, 39, 41, 188, 362, 77, 80, 14, 63, 207, 118, 238, 246, 205, 359, 376, 40, 148, 312, 253, 135, 206, 198, 338, 166, 230, 121, 331, 1, 133, 255, 33, 270, 227, 248, 368, 221, 95, 348, 210, 257, 54, 314, 353, 375, 293, 124, 112, 319, 274, 37, 289, 67, 64, 86, 130, 60, 287, 47, 157, 382, 358, 176, 203, 52, 335, 366, 277, 111, 194, 165, 51, 229, 224, 202, 208] + 57: [254, 165, 205, 331, 170, 74, 11, 31, 314, 85, 198, 140, 257, 223, 124, 344, 118, 217, 150, 174, 117, 22, 88, 185, 44, 250, 238, 347, 338, 316, 373, 176, 134, 243, 51, 70, 42, 173, 83, 191, 102, 183, 203, 358, 12, 13, 119, 279, 166, 313, 352, 353, 209, 321, 155, 288, 186, 323, 162, 175, 237, 332, 315, 33, 303, 265, 200, 270, 32, 188, 219, 224, 50, 194, 256, 138, 364, 103, 139, 142, 10, 339, 114, 294, 327, 193, 292, 305, 37, 131, 201, 202, 56, 234, 293, 343, 276, 146, 89, 148, 94, 286, 154, 342, 158, 133, 220, 7, 366, 65, 274, 137, 335, 268, 300, 6, 127, 151, 345, 160, 309, 62, 75, 182, 144, 167, 365, 43, 322, 40, 73, 214, 177, 96, 20, 277, 382, 111, 54, 92, 120, 55, 380, 95, 348, 226, 71, 196, 351, 337, 369, 362, 330, 98, 17, 18, 379, 261, 26, 247, 252, 122, 236, 248, 307, 0, 87, 241, 109, 383, 242, 374, 48, 239, 287, 61, 310, 271, 115, 258, 82, 206, 269, 336, 69, 77, 153, 229, 19, 367, 376, 304, 326, 14, 259, 246, 207, 320, 157, 159, 381, 16, 368, 63, 104, 128, 359, 324, 141, 8, 60, 350, 84, 197, 180, 79, 35, 123, 251, 354, 99, 58, 39, 172, 179, 213, 297, 228, 152, 113, 298, 312, 21, 377, 147, 38, 289, 284, 325, 328, 68, 2, 136, 149, 3, 190, 296, 28, 178, 132, 311, 334, 41, 240, 97, 363, 5, 45, 211, 59, 255, 281, 168, 357, 66, 263, 36, 222, 67, 370, 107, 308, 360, 264, 244, 291, 129, 52, 340, 30, 195, 181, 280, 105, 233, 86, 262, 372, 25, 231, 295, 187, 72, 125, 302, 93, 230, 121, 57, 156, 235, 101, 108, 53, 130, 267, 169, 204, 318, 283, 317, 64, 306, 163, 80, 272, 319, 91, 253, 161, 110, 346, 9, 341, 27, 249, 46, 356, 349, 184, 116, 208, 47, 106, 112, 225, 329, 210, 164, 171, 199, 282, 90, 23, 216, 81, 215, 29, 371, 290, 145, 49, 355, 135, 24, 299, 378, 4, 126, 1, 333, 260, 361, 192, 375, 232, 76, 78, 273, 266, 100, 218, 212, 221, 189, 278, 227, 245, 15, 34, 143, 285, 275, 301] + 58: [377, 222, 298, 77, 61, 7, 344, 271, 299, 22, 358, 355, 270, 112, 175, 152, 376, 18, 85, 42, 31, 272, 315, 255, 106, 190, 197, 185, 285, 329, 48, 331, 251, 207, 143, 266, 325, 238, 90, 124, 160, 333, 37, 149, 110, 241, 13, 253, 176, 327, 334, 371, 21, 11, 287, 225, 353, 136, 105, 180, 297, 24, 213, 142, 372, 264, 86, 268, 282, 146, 290, 46, 223, 369, 120, 145, 216, 15, 78, 301, 174, 310, 363, 23, 165, 320, 279, 3, 125, 370, 364, 33, 38, 58, 134, 209, 220, 126, 305, 98, 177, 263, 151, 79, 27, 211, 373, 206, 49, 72, 214, 26, 169, 116, 232, 295, 342, 341, 367, 191, 12, 104, 111, 316, 20, 307, 224, 198, 229, 281, 202, 314, 244, 59, 62, 258, 236, 53, 155, 227, 183, 382, 0, 332, 144, 172, 357, 186, 107, 161, 345, 43, 70, 293, 195, 35, 8, 173, 338, 64, 243, 261, 360, 87, 317, 221, 303, 354, 200, 337, 51, 247, 96, 109, 32, 153, 17, 88, 30, 267, 140, 323, 361, 291, 80, 252, 71, 319, 10, 168, 201, 308, 81, 275, 366, 92, 311, 122, 248, 76, 273, 348, 14, 322, 235, 69, 351, 257, 256, 368, 36, 262, 196, 324, 47, 119, 189, 97, 132, 162, 4, 274, 66, 171, 28, 380, 192, 139, 381, 25, 82, 129, 99, 215, 306, 250, 234, 346, 240, 131, 63, 302, 362, 148, 231, 65, 123, 318, 347, 265, 115, 283, 50, 237, 219, 118, 226, 294, 296, 29, 95, 54, 330, 163, 41, 254, 374, 84, 269, 39, 57, 246, 159, 52, 193, 127, 365, 218, 343, 89, 212, 164, 147, 292, 321, 108, 184, 204, 245, 156, 56, 205, 194, 312, 114, 249, 259, 356, 210, 101, 93, 9, 289, 67, 199, 40, 178, 68, 45, 141, 6, 203, 16, 277, 130, 60, 260, 352, 242, 328, 73, 187, 1, 375, 55, 239, 383, 181, 284, 350, 83, 339, 326, 378, 75, 121, 135, 300, 208, 379, 154, 288, 278, 2, 228, 113, 19, 182, 179, 133, 94, 230, 170, 5, 102, 138, 100, 128, 359, 276, 157, 336, 103, 44, 167, 349, 117, 34, 137, 304, 286, 188, 74, 217, 91, 233, 313, 166, 158, 309, 335, 150, 280, 340] + 59: [327, 224, 94, 59, 340, 352, 73, 30, 46, 197, 5, 381, 247, 294, 331, 185, 172, 295, 9, 317, 262, 243, 309, 58, 133, 176, 278, 223, 57, 12, 16, 297, 364, 334, 383, 252, 143, 60, 245, 286, 214, 236, 222, 120, 137, 177, 238, 337, 365, 131, 123, 242, 199, 107, 48, 333, 138, 43, 41, 135, 187, 231, 134, 157, 28, 74, 89, 261, 44, 183, 191, 148, 273, 166, 230, 312, 303, 301, 108, 318, 282, 350, 14, 270, 272, 244, 347, 345, 139, 116, 52, 351, 111, 90, 79, 189, 25, 209, 379, 260, 173, 346, 65, 182, 105, 256, 233, 70, 311, 263, 221, 150, 42, 213, 186, 38, 178, 144, 266, 196, 86, 112, 212, 109, 161, 85, 254, 279, 181, 211, 163, 320, 376, 22, 174, 142, 329, 10, 355, 69, 198, 348, 92, 259, 155, 36, 228, 2, 61, 314, 32, 299, 287, 193, 63, 149, 129, 136, 184, 322, 67, 343, 359, 93, 49, 3, 24, 106, 11, 321, 296, 284, 358, 82, 165, 237, 239, 240, 360, 56, 335, 293, 264, 87, 78, 83, 72, 99, 235, 117, 341, 271, 170, 140, 370, 195, 363, 220, 332, 113, 75, 313, 147, 251, 374, 371, 130, 76, 31, 300, 324, 328, 23, 80, 382, 232, 114, 342, 91, 175, 40, 160, 168, 127, 100, 302, 169, 162, 373, 225, 218, 338, 54, 356, 179, 339, 269, 180, 366, 354, 203, 6, 353, 51, 68, 274, 362, 0, 206, 380, 64, 110, 325, 208, 84, 124, 21, 207, 153, 145, 277, 88, 249, 115, 118, 53, 119, 102, 241, 368, 319, 98, 141, 1, 192, 219, 121, 77, 171, 205, 126, 104, 122, 323, 308, 15, 188, 154, 229, 7, 250, 202, 200, 66, 234, 33, 285, 97, 226, 95, 298, 204, 291, 288, 18, 265, 372, 377, 151, 26, 281, 255, 128, 290, 45, 164, 304, 81, 35, 159, 267, 280, 62, 248, 315, 20, 29, 19, 306, 47, 158, 258, 344, 146, 369, 361, 289, 13, 101, 349, 156, 96, 292, 194, 227, 71, 27, 246, 34, 50, 253, 217, 17, 330, 37, 326, 336, 132, 210, 275, 201, 357, 307, 103, 216, 283, 257, 8, 316, 276, 305, 125, 39, 375, 367, 378, 152, 55, 190, 167, 4, 310, 268, 215] + 60: [313, 381, 139, 133, 265, 94, 128, 195, 303, 372, 316, 78, 247, 65, 297, 208, 205, 239, 76, 49, 365, 166, 345, 98, 245, 272, 143, 59, 241, 27, 169, 140, 84, 0, 362, 283, 269, 358, 183, 240, 80, 188, 382, 264, 215, 274, 304, 25, 141, 233, 107, 142, 285, 91, 149, 30, 300, 341, 95, 349, 380, 286, 225, 148, 325, 338, 156, 67, 251, 127, 209, 328, 179, 198, 145, 68, 204, 378, 298, 301, 159, 306, 43, 292, 164, 275, 295, 352, 109, 10, 226, 373, 63, 207, 22, 355, 124, 268, 47, 83, 174, 102, 335, 115, 129, 347, 339, 48, 167, 312, 334, 155, 28, 366, 294, 337, 180, 112, 193, 354, 278, 131, 231, 203, 271, 263, 99, 172, 75, 189, 58, 20, 93, 37, 217, 259, 11, 88, 267, 110, 38, 57, 279, 305, 323, 186, 70, 90, 273, 66, 200, 250, 182, 130, 357, 314, 296, 256, 14, 371, 197, 383, 282, 168, 202, 13, 252, 287, 19, 216, 280, 42, 374, 211, 29, 147, 72, 86, 249, 17, 281, 289, 71, 4, 32, 101, 77, 242, 62, 214, 348, 201, 351, 16, 220, 126, 40, 310, 236, 170, 243, 244, 39, 222, 230, 153, 302, 181, 74, 34, 108, 234, 353, 119, 329, 15, 213, 232, 184, 161, 235, 55, 246, 359, 336, 151, 277, 255, 114, 6, 36, 318, 92, 332, 162, 5, 3, 377, 8, 342, 134, 171, 346, 360, 187, 85, 33, 196, 190, 192, 106, 322, 276, 144, 18, 64, 315, 333, 105, 270, 262, 45, 330, 324, 137, 261, 41, 104, 379, 229, 97, 96, 291, 191, 326, 135, 173, 163, 138, 340, 44, 118, 87, 308, 210, 317, 299, 288, 123, 257, 158, 60, 199, 284, 121, 260, 266, 343, 113, 212, 194, 120, 185, 81, 2, 327, 35, 319, 290, 46, 311, 309, 206, 122, 117, 223, 238, 82, 364, 253, 152, 150, 69, 248, 221, 132, 51, 1, 344, 24, 9, 237, 61, 176, 116, 56, 26, 21, 52, 53, 31, 293, 7, 146, 100, 369, 79, 370, 368, 219, 12, 331, 125, 363, 307, 376, 23, 361, 178, 228, 320, 54, 258, 50, 224, 160, 356, 111, 175, 165, 350, 254, 227, 367, 177, 103, 73, 321, 218, 89, 136, 157, 154, 375] + 61: [221, 211, 381, 315, 263, 88, 303, 136, 120, 247, 122, 203, 241, 264, 363, 218, 12, 278, 238, 235, 237, 119, 327, 115, 34, 42, 31, 160, 227, 133, 87, 258, 224, 65, 268, 29, 300, 190, 313, 11, 109, 336, 69, 368, 207, 200, 102, 173, 72, 137, 346, 214, 275, 9, 365, 80, 269, 141, 340, 71, 124, 14, 342, 298, 162, 345, 63, 54, 354, 18, 364, 358, 312, 331, 246, 272, 95, 28, 48, 236, 250, 222, 60, 285, 254, 355, 382, 267, 376, 217, 266, 350, 116, 53, 94, 0, 127, 93, 197, 167, 248, 86, 231, 5, 360, 326, 351, 117, 270, 184, 45, 353, 330, 17, 294, 215, 213, 66, 291, 57, 177, 188, 15, 282, 32, 75, 92, 361, 362, 150, 233, 366, 83, 27, 113, 374, 2, 257, 245, 328, 52, 158, 290, 139, 186, 89, 292, 30, 375, 341, 22, 79, 283, 67, 295, 132, 199, 284, 204, 152, 370, 274, 352, 25, 310, 301, 101, 99, 49, 24, 13, 305, 180, 182, 146, 339, 318, 347, 64, 377, 279, 131, 210, 50, 367, 325, 334, 357, 343, 321, 149, 155, 179, 61, 299, 100, 344, 307, 107, 230, 335, 157, 165, 323, 19, 209, 380, 103, 21, 51, 68, 142, 143, 112, 129, 168, 110, 159, 349, 243, 169, 261, 228, 191, 348, 297, 316, 114, 135, 82, 252, 304, 47, 308, 206, 145, 104, 202, 33, 259, 322, 38, 78, 306, 8, 37, 125, 126, 85, 196, 26, 317, 176, 194, 359, 148, 70, 311, 91, 163, 187, 255, 234, 251, 7, 262, 185, 314, 164, 212, 223, 105, 35, 220, 56, 226, 151, 175, 134, 62, 41, 320, 16, 128, 356, 189, 208, 319, 338, 73, 46, 36, 324, 161, 216, 172, 108, 170, 205, 178, 10, 59, 1, 253, 195, 372, 371, 198, 55, 192, 288, 333, 369, 286, 181, 156, 309, 43, 337, 20, 118, 77, 90, 84, 44, 166, 121, 379, 329, 260, 3, 23, 97, 147, 249, 378, 154, 383, 153, 273, 280, 174, 242, 219, 225, 240, 81, 4, 144, 332, 98, 106, 193, 244, 130, 271, 239, 6, 123, 74, 232, 111, 201, 293, 373, 138, 265, 140, 256, 276, 302, 277, 281, 39, 289, 183, 76, 229, 296, 40, 171, 96, 287, 58] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots416.yaml b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots416.yaml new file mode 100644 index 000000000000..93595be1ffc3 --- /dev/null +++ b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep4_slots416.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [259, 151, 74, 356, 356, 322, 245, 225, 177, 165, 254, 135, 76, 193, 346, 332, 312, 295, 26, 60, 104, 242, 17, 95, 184, 232, 255, 376, 139, 68, 37, 333, 173, 290, 280, 282, 55, 143, 205, 54, 11, 105, 313, 56, 354, 297, 129, 258, 324, 368, 7, 170, 149, 309, 314, 38, 52, 5, 91, 172, 227, 4, 108, 249, 363, 372, 275, 59, 215, 97, 106, 270, 299, 73, 267, 67, 260, 50, 252, 98, 294, 352, 316, 340, 380, 12, 219, 179, 381, 9, 357, 250, 298, 92, 287, 24, 228, 128, 344, 288, 118, 13, 110, 303, 166, 256, 222, 302, 159, 236, 371, 383, 162, 165, 111, 235, 279, 241, 378, 45, 188, 35, 347, 153, 202, 30, 80, 95, 134, 49, 321, 70, 203, 176, 276, 16, 330, 290, 144, 209, 360, 359, 307, 284, 131, 196, 341, 233, 109, 186, 361, 311, 112, 138, 44, 148, 31, 27, 199, 358, 87, 85, 308, 257, 213, 136, 240, 365, 140, 246, 19, 366, 180, 238, 320, 195, 82, 127, 328, 263, 362, 47, 102, 350, 161, 40, 64, 206, 315, 175, 2, 152, 301, 239, 220, 125, 167, 271, 329, 226, 291, 339, 8, 88, 310, 58, 373, 61, 166, 150, 71, 222, 345, 236, 371, 225, 155, 165, 111, 135, 279, 158, 378, 45, 169, 51, 122, 81, 79, 204, 29, 218, 184, 327, 289, 65, 203, 217, 278, 198, 173, 224, 3, 349, 336, 78, 72, 243, 117, 34, 283, 156, 10, 124, 265, 200, 261, 114, 248, 337, 123, 268, 89, 234, 66, 39, 133, 244, 335, 103, 146, 331, 116, 178, 216, 192, 296, 100, 379, 292, 83, 147, 326, 237, 342, 20, 210, 286, 338, 42, 212, 253, 53, 41, 113, 334, 194, 231, 285, 164, 18, 142, 304, 121, 281, 364, 273, 130, 370, 171, 201, 247, 0, 256, 74, 351, 84, 119, 245, 86, 348, 254, 111, 264, 207, 230, 378, 190, 182, 163, 60, 21, 79, 62, 17, 157, 134, 49, 48, 343, 63, 217, 293, 318, 330, 224, 272, 126, 22, 141, 181, 33, 101, 107, 183, 43, 57, 90, 36, 174, 94, 28, 300, 208, 77, 160, 262, 375, 223, 377, 187, 32, 214, 317, 46, 137, 75, 269, 99, 369, 211, 1, 25, 221, 197, 382, 93, 14, 145, 6, 154, 229, 305, 15, 96, 115, 306, 353, 69, 191, 319, 367, 266, 251, 132, 23, 189, 374, 325, 355, 323, 168, 277, 120, 274, 185] + 1: [134, 365, 42, 174, 174, 137, 180, 158, 382, 382, 43, 72, 177, 167, 163, 329, 319, 319, 241, 39, 366, 0, 210, 73, 105, 182, 294, 120, 225, 111, 271, 82, 283, 272, 29, 195, 269, 275, 32, 247, 245, 266, 274, 99, 187, 84, 194, 280, 223, 52, 206, 93, 203, 264, 202, 239, 147, 60, 98, 256, 7, 313, 222, 259, 370, 2, 20, 14, 296, 184, 136, 57, 172, 36, 284, 51, 311, 13, 302, 55, 12, 232, 119, 314, 190, 130, 246, 297, 160, 367, 207, 217, 10, 252, 343, 64, 117, 337, 341, 354, 112, 336, 34, 100, 178, 81, 235, 201, 22, 118, 180, 158, 277, 150, 50, 91, 169, 167, 163, 254, 319, 347, 168, 6, 5, 70, 377, 41, 63, 58, 244, 198, 23, 248, 185, 82, 383, 272, 243, 292, 9, 145, 86, 242, 125, 281, 74, 212, 197, 288, 261, 109, 114, 213, 176, 228, 289, 306, 376, 278, 236, 95, 205, 189, 164, 79, 348, 330, 65, 139, 286, 326, 19, 175, 33, 321, 290, 35, 230, 350, 352, 94, 323, 346, 344, 240, 333, 103, 83, 369, 331, 129, 316, 90, 151, 221, 132, 361, 16, 45, 258, 159, 28, 157, 312, 345, 265, 255, 380, 200, 1, 42, 31, 137, 44, 80, 85, 126, 263, 72, 177, 291, 163, 127, 21, 347, 67, 192, 270, 75, 377, 110, 3, 3, 299, 198, 250, 111, 153, 24, 25, 320, 138, 273, 303, 211, 71, 173, 216, 78, 285, 301, 218, 108, 92, 335, 143, 4, 53, 116, 381, 69, 360, 342, 61, 231, 237, 76, 121, 362, 305, 238, 338, 161, 318, 17, 156, 295, 165, 62, 48, 186, 293, 11, 97, 96, 8, 113, 54, 262, 49, 356, 357, 375, 372, 220, 340, 188, 373, 135, 66, 124, 155, 191, 144, 276, 359, 257, 199, 310, 15, 215, 141, 251, 365, 201, 22, 308, 18, 106, 277, 149, 152, 91, 169, 196, 127, 378, 40, 131, 67, 30, 179, 325, 123, 328, 334, 182, 300, 56, 120, 225, 351, 233, 224, 46, 59, 292, 37, 287, 260, 89, 234, 317, 355, 146, 166, 162, 142, 379, 214, 219, 47, 282, 358, 88, 38, 171, 101, 133, 102, 298, 122, 140, 170, 204, 364, 249, 332, 107, 115, 229, 154, 68, 87, 324, 315, 26, 104, 227, 181, 268, 226, 353, 77, 27, 148, 279, 371, 309, 193, 183, 253, 209, 208, 128, 322, 374, 307, 349, 339, 304, 267, 363, 327, 368] + 2: [53, 96, 112, 108, 215, 89, 194, 115, 28, 81, 3, 295, 125, 143, 248, 197, 232, 149, 213, 185, 153, 111, 168, 322, 157, 91, 280, 286, 86, 135, 184, 172, 314, 139, 324, 22, 274, 200, 11, 235, 30, 6, 342, 204, 140, 170, 18, 347, 50, 156, 188, 320, 187, 201, 148, 198, 349, 41, 65, 241, 193, 15, 160, 276, 364, 106, 312, 252, 29, 104, 362, 23, 211, 134, 334, 238, 376, 331, 319, 231, 292, 302, 288, 99, 251, 167, 59, 98, 182, 336, 227, 222, 13, 353, 380, 88, 287, 358, 339, 237, 246, 375, 161, 379, 53, 132, 132, 225, 357, 14, 102, 115, 223, 1, 295, 321, 151, 212, 248, 33, 242, 232, 101, 19, 110, 62, 168, 191, 166, 131, 377, 278, 44, 271, 42, 304, 139, 51, 122, 26, 83, 63, 340, 244, 303, 159, 209, 165, 344, 17, 356, 263, 55, 269, 67, 92, 20, 383, 218, 297, 46, 163, 265, 308, 309, 169, 103, 69, 256, 136, 12, 49, 257, 150, 94, 0, 25, 239, 346, 4, 272, 141, 144, 352, 329, 54, 71, 371, 335, 205, 48, 74, 113, 382, 283, 345, 214, 240, 127, 64, 228, 301, 208, 202, 363, 40, 229, 374, 43, 180, 108, 351, 313, 215, 194, 93, 223, 5, 3, 137, 247, 143, 212, 248, 60, 192, 78, 290, 155, 62, 350, 124, 173, 91, 377, 70, 245, 306, 133, 105, 304, 45, 296, 325, 328, 279, 333, 85, 285, 311, 61, 79, 206, 7, 90, 224, 293, 126, 262, 327, 21, 58, 128, 236, 360, 277, 10, 145, 348, 181, 250, 116, 9, 38, 305, 121, 268, 365, 152, 219, 36, 275, 367, 47, 52, 207, 233, 359, 216, 226, 370, 373, 338, 8, 164, 158, 117, 72, 234, 147, 317, 260, 289, 80, 326, 119, 270, 282, 255, 190, 366, 330, 43, 230, 112, 341, 313, 89, 343, 93, 298, 259, 221, 189, 100, 143, 34, 75, 242, 162, 290, 19, 109, 84, 97, 284, 355, 66, 56, 73, 86, 118, 123, 220, 314, 31, 122, 368, 146, 217, 82, 24, 323, 196, 174, 294, 130, 299, 273, 87, 27, 68, 203, 210, 361, 120, 76, 138, 332, 253, 16, 171, 175, 186, 254, 129, 372, 154, 39, 142, 318, 176, 179, 2, 195, 77, 281, 35, 178, 264, 261, 57, 243, 291, 107, 95, 183, 114, 267, 307, 378, 310, 37, 177, 369, 249, 315, 266, 316, 32, 337, 354, 300, 199, 381, 258] + 3: [39, 66, 71, 34, 79, 275, 260, 230, 314, 40, 319, 171, 51, 27, 163, 192, 194, 97, 263, 24, 363, 145, 50, 42, 215, 26, 102, 120, 336, 227, 211, 325, 212, 304, 177, 316, 64, 21, 202, 264, 136, 162, 142, 154, 288, 262, 374, 243, 300, 181, 108, 218, 200, 55, 344, 112, 355, 135, 266, 246, 268, 82, 144, 160, 306, 322, 298, 361, 305, 291, 15, 83, 123, 61, 259, 228, 321, 309, 133, 206, 179, 183, 116, 16, 88, 121, 191, 45, 141, 253, 148, 89, 352, 315, 4, 69, 326, 341, 247, 335, 324, 278, 311, 301, 39, 66, 46, 34, 34, 310, 260, 377, 302, 365, 319, 14, 350, 225, 163, 208, 194, 263, 80, 348, 150, 334, 180, 331, 139, 238, 178, 170, 342, 250, 10, 267, 287, 74, 239, 36, 11, 290, 118, 382, 279, 8, 84, 289, 203, 229, 312, 114, 176, 98, 274, 103, 217, 380, 197, 327, 129, 65, 307, 44, 383, 214, 100, 237, 248, 340, 187, 20, 188, 281, 7, 106, 381, 85, 132, 68, 153, 293, 283, 285, 0, 245, 134, 251, 156, 338, 329, 369, 241, 147, 95, 75, 272, 175, 349, 152, 173, 226, 17, 332, 146, 199, 59, 38, 22, 66, 66, 34, 34, 310, 35, 377, 209, 365, 40, 18, 31, 27, 13, 3, 3, 313, 333, 151, 158, 49, 252, 137, 337, 339, 63, 343, 2, 358, 189, 231, 77, 296, 195, 182, 25, 223, 12, 357, 364, 78, 161, 359, 32, 277, 351, 297, 345, 375, 204, 224, 256, 166, 53, 57, 169, 117, 280, 255, 104, 30, 328, 196, 43, 115, 353, 249, 70, 232, 367, 376, 165, 130, 346, 282, 362, 101, 254, 93, 190, 378, 220, 323, 234, 107, 242, 159, 143, 54, 368, 127, 124, 308, 60, 244, 33, 185, 5, 52, 257, 292, 47, 222, 22, 66, 66, 34, 79, 310, 260, 377, 302, 365, 319, 18, 19, 317, 131, 56, 313, 261, 62, 157, 318, 81, 213, 6, 186, 201, 207, 168, 164, 58, 37, 122, 138, 221, 174, 219, 119, 240, 29, 87, 271, 109, 172, 99, 295, 149, 111, 92, 155, 41, 126, 299, 236, 235, 94, 270, 105, 273, 48, 113, 76, 125, 366, 276, 216, 286, 96, 184, 67, 128, 294, 91, 373, 356, 72, 110, 28, 86, 269, 258, 354, 379, 372, 73, 140, 210, 284, 360, 205, 90, 1, 330, 265, 23, 320, 347, 167, 193, 371, 9, 233, 198, 303, 370] + 4: [188, 209, 271, 124, 222, 343, 331, 331, 315, 350, 320, 193, 15, 172, 88, 19, 319, 257, 273, 150, 148, 152, 143, 241, 342, 345, 59, 155, 347, 246, 4, 372, 32, 365, 314, 44, 339, 8, 217, 251, 22, 82, 38, 207, 278, 335, 99, 264, 133, 166, 189, 72, 62, 352, 149, 75, 261, 125, 131, 313, 80, 354, 284, 310, 178, 39, 66, 306, 184, 346, 29, 157, 122, 180, 181, 304, 71, 223, 355, 156, 126, 81, 378, 21, 48, 353, 1, 247, 265, 134, 144, 224, 212, 89, 266, 299, 20, 192, 79, 242, 287, 58, 104, 120, 34, 18, 271, 375, 258, 300, 343, 331, 24, 259, 214, 23, 15, 9, 95, 376, 319, 87, 273, 171, 148, 152, 165, 136, 36, 56, 377, 167, 139, 113, 61, 275, 151, 196, 83, 101, 49, 28, 161, 280, 7, 369, 159, 141, 283, 244, 243, 42, 307, 53, 168, 114, 98, 93, 33, 323, 269, 74, 229, 383, 135, 322, 288, 137, 84, 220, 92, 349, 230, 78, 121, 175, 291, 249, 210, 334, 111, 31, 201, 213, 186, 381, 205, 129, 112, 163, 279, 191, 164, 366, 102, 248, 190, 301, 363, 154, 227, 225, 128, 67, 321, 63, 182, 277, 340, 3, 271, 37, 296, 296, 343, 331, 219, 250, 214, 371, 16, 228, 281, 226, 5, 105, 194, 117, 142, 254, 308, 348, 253, 47, 329, 298, 325, 41, 318, 200, 361, 364, 218, 160, 333, 312, 73, 198, 110, 260, 187, 179, 54, 324, 276, 116, 290, 252, 330, 256, 185, 297, 60, 14, 132, 108, 232, 70, 69, 240, 158, 328, 370, 309, 294, 50, 176, 234, 51, 26, 295, 327, 237, 118, 211, 268, 286, 130, 10, 215, 115, 203, 123, 362, 173, 162, 238, 30, 202, 293, 285, 236, 195, 270, 2, 119, 17, 231, 90, 336, 332, 351, 3, 3, 209, 12, 199, 343, 343, 331, 315, 315, 214, 371, 23, 239, 95, 226, 45, 87, 194, 117, 142, 254, 6, 302, 86, 13, 197, 274, 145, 76, 77, 106, 357, 235, 57, 43, 27, 204, 344, 68, 35, 317, 177, 153, 183, 358, 221, 0, 85, 65, 356, 360, 379, 359, 169, 341, 64, 46, 216, 262, 100, 25, 94, 367, 138, 292, 267, 52, 263, 206, 303, 103, 170, 146, 140, 289, 147, 109, 40, 127, 255, 96, 311, 97, 374, 316, 380, 55, 91, 368, 245, 11, 373, 326, 382, 174, 337, 208, 233, 272, 338, 107, 282, 305] + 5: [172, 190, 213, 213, 306, 296, 308, 5, 217, 47, 355, 60, 276, 100, 262, 305, 356, 287, 288, 272, 159, 259, 78, 21, 72, 279, 252, 7, 246, 91, 364, 74, 209, 371, 8, 137, 171, 163, 59, 2, 108, 206, 204, 143, 26, 314, 152, 377, 301, 35, 195, 186, 380, 274, 93, 39, 20, 353, 201, 121, 257, 161, 317, 289, 275, 330, 80, 87, 361, 94, 229, 62, 235, 221, 71, 114, 18, 135, 40, 298, 261, 125, 95, 292, 11, 111, 197, 85, 328, 336, 158, 234, 0, 273, 27, 367, 142, 278, 96, 179, 23, 321, 368, 302, 208, 154, 190, 249, 128, 296, 214, 194, 217, 65, 355, 49, 265, 105, 188, 305, 381, 365, 52, 6, 231, 159, 21, 41, 130, 351, 17, 43, 293, 147, 198, 25, 31, 233, 66, 313, 15, 268, 369, 174, 184, 38, 140, 3, 175, 258, 45, 218, 16, 228, 28, 115, 12, 312, 253, 19, 56, 366, 382, 300, 241, 150, 284, 90, 383, 237, 9, 13, 291, 162, 232, 376, 363, 283, 103, 129, 270, 244, 156, 316, 297, 373, 86, 290, 53, 120, 309, 97, 282, 36, 202, 123, 109, 226, 248, 110, 83, 176, 326, 57, 50, 170, 177, 354, 61, 61, 213, 128, 128, 84, 214, 194, 98, 81, 355, 346, 276, 100, 370, 169, 269, 356, 288, 272, 231, 211, 259, 379, 255, 132, 212, 88, 331, 192, 338, 182, 327, 304, 131, 70, 151, 375, 332, 238, 181, 319, 251, 207, 324, 320, 333, 220, 33, 104, 46, 79, 266, 216, 329, 183, 230, 106, 239, 250, 341, 219, 76, 136, 281, 334, 48, 360, 134, 29, 271, 55, 205, 374, 299, 263, 107, 342, 187, 92, 224, 189, 347, 101, 133, 126, 267, 307, 311, 146, 340, 294, 117, 310, 339, 280, 166, 225, 167, 277, 286, 196, 303, 75, 61, 165, 141, 249, 256, 84, 214, 160, 260, 1, 178, 346, 223, 105, 193, 269, 30, 287, 52, 6, 372, 122, 102, 64, 215, 145, 352, 200, 89, 247, 335, 22, 124, 173, 4, 155, 37, 222, 358, 264, 69, 67, 318, 24, 210, 34, 153, 315, 54, 295, 99, 144, 51, 242, 345, 227, 14, 343, 362, 240, 63, 138, 203, 325, 254, 119, 149, 236, 344, 77, 191, 349, 359, 113, 323, 378, 199, 68, 168, 58, 148, 322, 32, 245, 185, 139, 285, 73, 243, 350, 357, 42, 180, 10, 116, 112, 348, 164, 82, 157, 337, 118, 127, 44] + 6: [322, 76, 97, 212, 142, 274, 5, 376, 32, 32, 249, 267, 269, 55, 281, 99, 66, 166, 177, 31, 175, 378, 260, 216, 94, 119, 57, 283, 118, 208, 155, 184, 370, 151, 301, 176, 128, 377, 380, 236, 49, 68, 12, 158, 133, 139, 114, 219, 15, 10, 286, 280, 6, 87, 207, 137, 116, 63, 243, 217, 194, 261, 122, 248, 199, 105, 193, 224, 8, 16, 381, 309, 364, 123, 312, 74, 150, 288, 328, 290, 117, 162, 338, 21, 121, 222, 339, 238, 29, 192, 127, 241, 340, 170, 218, 72, 65, 75, 214, 186, 315, 331, 247, 257, 322, 76, 97, 142, 142, 295, 308, 376, 160, 53, 27, 174, 28, 253, 281, 46, 209, 233, 326, 371, 9, 36, 307, 13, 88, 367, 189, 112, 319, 34, 178, 289, 156, 0, 270, 113, 179, 115, 41, 230, 231, 101, 279, 190, 169, 221, 77, 327, 332, 136, 296, 365, 262, 69, 43, 237, 159, 284, 82, 86, 254, 372, 329, 81, 168, 346, 3, 50, 334, 42, 70, 125, 379, 292, 244, 130, 152, 108, 226, 337, 256, 294, 348, 47, 183, 131, 250, 374, 92, 213, 141, 1, 172, 191, 20, 353, 153, 132, 363, 204, 35, 325, 342, 373, 322, 297, 305, 97, 142, 274, 308, 376, 103, 53, 343, 258, 341, 253, 240, 11, 180, 233, 205, 67, 173, 45, 163, 318, 232, 293, 368, 84, 255, 287, 291, 129, 259, 345, 277, 165, 185, 352, 245, 278, 311, 304, 37, 111, 321, 44, 64, 107, 264, 98, 220, 157, 271, 22, 206, 242, 225, 197, 23, 4, 362, 138, 316, 148, 48, 333, 198, 358, 298, 96, 268, 272, 134, 366, 285, 357, 106, 203, 347, 85, 17, 102, 303, 51, 313, 356, 302, 78, 26, 171, 19, 227, 344, 200, 61, 2, 314, 135, 167, 52, 275, 323, 161, 164, 146, 76, 305, 97, 59, 274, 308, 308, 32, 343, 306, 174, 258, 240, 229, 99, 66, 166, 177, 120, 350, 375, 359, 181, 187, 100, 56, 246, 188, 196, 79, 239, 300, 234, 95, 354, 109, 369, 182, 336, 317, 223, 30, 202, 282, 383, 144, 211, 143, 324, 154, 58, 210, 361, 104, 110, 276, 335, 330, 60, 355, 14, 145, 126, 18, 83, 124, 147, 349, 215, 299, 54, 73, 310, 39, 235, 80, 62, 251, 360, 266, 24, 195, 351, 89, 201, 263, 38, 7, 140, 91, 93, 273, 382, 228, 265, 252, 149, 25, 33, 320, 71, 40, 90] + 7: [163, 162, 182, 7, 258, 275, 192, 263, 133, 49, 292, 375, 229, 46, 382, 382, 105, 103, 201, 253, 362, 118, 48, 329, 0, 61, 94, 131, 183, 47, 215, 267, 278, 4, 168, 55, 270, 148, 250, 235, 117, 21, 221, 62, 90, 344, 333, 214, 24, 368, 314, 109, 358, 19, 32, 190, 211, 83, 335, 305, 67, 53, 353, 232, 251, 373, 65, 291, 87, 101, 52, 120, 37, 126, 268, 209, 363, 112, 69, 218, 85, 254, 111, 39, 252, 381, 78, 159, 195, 95, 45, 57, 352, 331, 143, 282, 34, 376, 236, 179, 135, 371, 338, 42, 341, 175, 272, 7, 274, 336, 263, 64, 41, 49, 292, 28, 142, 229, 343, 340, 105, 18, 25, 25, 298, 378, 354, 188, 244, 27, 181, 281, 11, 313, 164, 82, 248, 262, 17, 31, 288, 360, 296, 284, 70, 210, 170, 213, 158, 191, 243, 167, 234, 89, 26, 204, 136, 237, 180, 107, 322, 273, 58, 6, 51, 121, 56, 155, 161, 40, 309, 68, 149, 153, 225, 351, 184, 222, 151, 106, 286, 169, 194, 1, 10, 96, 367, 43, 317, 303, 228, 315, 242, 299, 332, 150, 205, 165, 138, 144, 76, 156, 279, 160, 91, 348, 357, 50, 327, 327, 182, 274, 274, 192, 108, 64, 41, 350, 292, 187, 142, 265, 342, 340, 18, 103, 13, 99, 298, 369, 97, 212, 77, 370, 256, 73, 14, 249, 113, 217, 289, 29, 349, 23, 84, 2, 325, 356, 81, 255, 219, 377, 372, 300, 241, 132, 259, 189, 16, 245, 92, 20, 186, 178, 137, 293, 380, 86, 152, 226, 310, 323, 233, 196, 247, 261, 22, 38, 345, 366, 115, 269, 320, 166, 287, 207, 316, 72, 59, 123, 80, 231, 3, 277, 334, 266, 246, 359, 346, 147, 172, 12, 347, 199, 355, 154, 326, 240, 276, 110, 257, 364, 327, 175, 157, 7, 238, 275, 308, 361, 239, 49, 260, 176, 375, 265, 343, 271, 280, 139, 201, 71, 362, 369, 15, 328, 104, 174, 114, 302, 324, 294, 307, 100, 125, 74, 119, 285, 337, 339, 319, 374, 383, 208, 177, 224, 330, 264, 98, 206, 9, 321, 311, 140, 185, 130, 129, 5, 220, 200, 102, 44, 141, 134, 306, 301, 283, 8, 227, 146, 116, 93, 193, 203, 79, 318, 35, 216, 33, 75, 36, 295, 88, 312, 173, 379, 297, 124, 63, 128, 198, 304, 230, 66, 290, 197, 127, 54, 122, 223, 145, 30, 365, 202, 171, 60] + 8: [273, 5, 360, 290, 218, 31, 135, 302, 359, 7, 264, 13, 12, 162, 321, 243, 221, 308, 353, 64, 229, 229, 182, 281, 291, 223, 157, 349, 11, 377, 131, 195, 128, 107, 75, 150, 27, 333, 38, 240, 92, 78, 65, 275, 304, 189, 167, 194, 130, 267, 120, 177, 341, 52, 4, 42, 101, 335, 383, 174, 237, 248, 247, 90, 297, 352, 47, 299, 190, 126, 16, 56, 99, 265, 235, 205, 306, 175, 322, 46, 315, 33, 326, 220, 324, 103, 370, 208, 26, 180, 70, 286, 355, 80, 342, 82, 222, 289, 361, 127, 227, 1, 132, 207, 253, 104, 279, 360, 290, 292, 135, 21, 359, 367, 264, 239, 362, 162, 336, 10, 272, 381, 353, 152, 228, 204, 74, 281, 20, 186, 316, 18, 294, 97, 86, 39, 311, 366, 22, 188, 109, 226, 147, 179, 202, 169, 216, 178, 303, 112, 198, 50, 317, 334, 141, 211, 125, 66, 72, 8, 45, 206, 117, 230, 379, 25, 257, 249, 350, 110, 310, 382, 266, 155, 313, 140, 105, 325, 378, 145, 236, 375, 369, 54, 323, 153, 111, 24, 15, 171, 129, 270, 319, 245, 354, 305, 43, 138, 94, 344, 256, 49, 0, 55, 19, 63, 213, 234, 253, 214, 3, 360, 290, 67, 31, 338, 359, 161, 108, 201, 362, 173, 241, 62, 62, 308, 282, 152, 337, 204, 81, 276, 30, 251, 380, 44, 364, 168, 280, 374, 225, 346, 69, 83, 133, 283, 119, 231, 298, 261, 165, 40, 219, 368, 183, 113, 300, 48, 192, 318, 2, 91, 60, 61, 59, 263, 212, 98, 28, 73, 191, 71, 151, 373, 32, 37, 307, 327, 314, 199, 144, 339, 35, 89, 260, 14, 29, 185, 134, 357, 351, 146, 84, 271, 158, 340, 329, 285, 123, 343, 51, 203, 139, 274, 124, 345, 242, 296, 262, 102, 58, 170, 253, 5, 250, 360, 246, 67, 238, 106, 21, 367, 224, 239, 201, 312, 321, 10, 221, 88, 154, 371, 371, 36, 182, 122, 252, 95, 255, 209, 328, 115, 156, 148, 166, 181, 172, 356, 23, 34, 6, 258, 330, 17, 121, 215, 278, 137, 309, 196, 320, 164, 233, 197, 332, 358, 114, 284, 187, 217, 254, 118, 87, 116, 232, 293, 184, 149, 259, 41, 136, 372, 68, 143, 79, 200, 85, 159, 295, 9, 93, 347, 365, 287, 277, 76, 193, 348, 244, 77, 269, 268, 160, 363, 142, 53, 331, 301, 288, 57, 210, 176, 163, 376, 96, 100] + 9: [212, 304, 166, 349, 81, 96, 226, 260, 263, 19, 350, 219, 155, 355, 184, 185, 37, 1, 313, 376, 127, 242, 380, 281, 244, 95, 341, 209, 136, 282, 32, 102, 98, 312, 82, 158, 363, 356, 120, 289, 264, 334, 231, 6, 364, 87, 116, 145, 186, 323, 207, 206, 124, 66, 189, 72, 106, 104, 343, 308, 208, 246, 50, 65, 52, 60, 275, 373, 174, 322, 254, 114, 371, 301, 251, 305, 224, 333, 211, 221, 347, 69, 188, 337, 290, 191, 368, 225, 345, 118, 328, 101, 93, 44, 194, 294, 91, 250, 61, 27, 236, 168, 338, 26, 92, 92, 255, 255, 143, 220, 280, 94, 94, 252, 357, 165, 327, 355, 179, 185, 198, 15, 201, 316, 138, 262, 49, 210, 64, 271, 103, 147, 10, 23, 239, 22, 258, 151, 285, 144, 187, 302, 137, 33, 277, 318, 180, 247, 192, 107, 41, 163, 35, 195, 267, 336, 36, 150, 111, 218, 274, 5, 75, 324, 17, 235, 8, 130, 303, 245, 148, 197, 62, 213, 112, 2, 320, 286, 375, 134, 76, 216, 21, 232, 199, 227, 167, 170, 193, 160, 58, 108, 240, 266, 249, 73, 296, 204, 200, 113, 28, 157, 25, 182, 369, 283, 348, 234, 92, 78, 304, 349, 349, 196, 31, 226, 215, 299, 353, 74, 222, 327, 47, 351, 29, 83, 146, 316, 217, 262, 49, 287, 244, 95, 51, 147, 10, 14, 56, 309, 9, 133, 161, 360, 135, 154, 3, 292, 297, 306, 54, 321, 314, 311, 259, 129, 269, 105, 4, 123, 230, 181, 238, 141, 237, 359, 80, 13, 276, 248, 354, 307, 379, 117, 205, 233, 300, 79, 203, 115, 42, 110, 12, 330, 298, 270, 162, 229, 346, 335, 241, 374, 293, 383, 243, 43, 325, 279, 361, 223, 149, 284, 273, 40, 362, 7, 122, 53, 77, 24, 365, 57, 92, 304, 304, 255, 331, 315, 31, 372, 263, 172, 295, 295, 327, 355, 47, 381, 344, 272, 313, 90, 90, 370, 159, 329, 64, 214, 103, 121, 67, 132, 177, 342, 131, 367, 317, 30, 377, 153, 268, 169, 88, 228, 0, 128, 183, 97, 20, 99, 142, 256, 139, 16, 70, 11, 171, 332, 288, 202, 39, 319, 339, 18, 175, 109, 100, 382, 45, 46, 190, 261, 358, 291, 84, 253, 71, 352, 310, 125, 38, 89, 48, 178, 59, 86, 34, 326, 68, 140, 176, 366, 63, 119, 278, 85, 378, 257, 164, 156, 265, 55, 152, 126, 340, 173] + 10: [294, 55, 300, 180, 151, 265, 319, 108, 5, 141, 320, 289, 365, 18, 286, 167, 31, 146, 2, 79, 310, 310, 134, 54, 263, 114, 66, 182, 237, 154, 201, 380, 7, 293, 11, 259, 345, 284, 49, 316, 98, 189, 249, 218, 61, 32, 128, 229, 233, 87, 321, 21, 232, 219, 60, 142, 9, 256, 26, 346, 379, 25, 267, 258, 36, 247, 64, 193, 275, 323, 95, 183, 185, 282, 226, 364, 96, 287, 139, 222, 372, 86, 119, 17, 168, 165, 251, 53, 88, 104, 281, 333, 191, 84, 156, 10, 242, 228, 381, 70, 23, 324, 366, 369, 192, 264, 300, 180, 93, 265, 319, 319, 5, 320, 24, 121, 365, 216, 337, 126, 331, 146, 225, 73, 215, 301, 195, 178, 171, 153, 19, 177, 196, 211, 269, 213, 67, 243, 272, 136, 368, 274, 342, 204, 360, 38, 306, 223, 149, 132, 199, 198, 116, 214, 135, 105, 288, 311, 106, 314, 329, 112, 378, 328, 74, 97, 383, 278, 42, 12, 354, 332, 45, 239, 123, 13, 370, 244, 307, 169, 131, 159, 302, 235, 187, 355, 205, 224, 4, 158, 133, 203, 118, 273, 157, 344, 361, 150, 63, 277, 28, 43, 238, 166, 176, 349, 68, 336, 255, 264, 55, 246, 268, 137, 319, 221, 5, 320, 320, 121, 325, 216, 279, 367, 309, 164, 92, 102, 22, 301, 170, 179, 78, 351, 241, 210, 99, 47, 248, 58, 115, 312, 234, 240, 348, 356, 377, 148, 77, 212, 376, 359, 382, 155, 250, 90, 103, 129, 107, 3, 56, 81, 85, 27, 315, 357, 59, 327, 41, 51, 371, 109, 340, 318, 184, 254, 317, 50, 362, 261, 298, 113, 257, 209, 326, 375, 80, 111, 127, 305, 147, 217, 322, 6, 16, 34, 94, 120, 44, 347, 30, 143, 202, 260, 270, 89, 160, 253, 291, 295, 29, 283, 194, 194, 300, 300, 151, 181, 319, 48, 5, 5, 24, 262, 365, 18, 286, 69, 31, 164, 57, 101, 144, 8, 1, 161, 117, 40, 138, 330, 124, 304, 308, 91, 76, 353, 130, 252, 363, 374, 186, 208, 190, 231, 271, 280, 285, 266, 145, 0, 33, 72, 46, 341, 296, 75, 122, 335, 339, 197, 20, 338, 37, 230, 15, 71, 162, 125, 39, 174, 140, 313, 163, 343, 175, 220, 65, 299, 350, 100, 62, 173, 206, 373, 303, 292, 35, 200, 245, 276, 334, 172, 83, 207, 352, 152, 227, 14, 52, 188, 110, 236, 82, 290, 358, 297] + 11: [156, 339, 299, 130, 87, 314, 83, 296, 163, 40, 150, 326, 221, 171, 145, 11, 116, 157, 100, 357, 199, 29, 332, 144, 275, 54, 178, 52, 73, 204, 282, 177, 17, 207, 250, 313, 161, 346, 124, 45, 249, 127, 159, 27, 195, 26, 251, 10, 198, 89, 298, 365, 186, 101, 242, 97, 147, 63, 59, 292, 353, 364, 193, 311, 187, 234, 264, 64, 43, 113, 62, 182, 143, 8, 19, 382, 200, 37, 323, 222, 288, 4, 140, 290, 278, 304, 183, 379, 60, 175, 246, 181, 111, 189, 276, 155, 329, 220, 321, 50, 308, 134, 243, 241, 252, 377, 130, 130, 376, 90, 3, 215, 163, 85, 85, 110, 235, 221, 190, 11, 116, 248, 248, 285, 285, 15, 218, 22, 247, 217, 141, 165, 131, 70, 337, 228, 205, 136, 78, 303, 6, 283, 371, 107, 370, 2, 75, 226, 315, 166, 374, 160, 133, 49, 197, 289, 259, 254, 270, 301, 295, 219, 168, 366, 354, 378, 158, 91, 324, 291, 115, 383, 48, 142, 263, 7, 74, 230, 358, 274, 103, 375, 84, 356, 173, 185, 322, 167, 380, 305, 277, 1, 238, 344, 352, 317, 372, 202, 82, 123, 153, 223, 25, 256, 109, 32, 42, 269, 98, 77, 260, 88, 87, 314, 83, 296, 296, 65, 150, 110, 235, 99, 121, 300, 44, 44, 100, 357, 267, 15, 0, 345, 152, 351, 284, 128, 39, 286, 318, 146, 154, 169, 23, 320, 47, 319, 170, 367, 262, 69, 117, 373, 172, 347, 151, 340, 316, 203, 255, 96, 31, 41, 81, 35, 336, 294, 245, 265, 312, 34, 135, 349, 118, 67, 293, 79, 258, 33, 225, 18, 138, 46, 194, 184, 360, 108, 306, 112, 310, 201, 287, 214, 68, 71, 342, 229, 368, 361, 369, 16, 51, 28, 239, 206, 334, 281, 72, 273, 191, 261, 9, 12, 98, 377, 339, 363, 227, 90, 314, 215, 163, 149, 119, 110, 38, 121, 233, 36, 36, 325, 126, 126, 102, 162, 24, 236, 331, 192, 343, 330, 244, 95, 381, 20, 13, 148, 196, 297, 94, 66, 280, 213, 348, 350, 80, 92, 93, 176, 56, 86, 341, 224, 14, 55, 208, 58, 209, 327, 362, 211, 30, 279, 338, 359, 179, 302, 164, 125, 120, 57, 129, 174, 355, 188, 309, 237, 231, 271, 266, 5, 106, 210, 21, 240, 104, 132, 272, 268, 114, 328, 180, 216, 139, 53, 253, 333, 122, 76, 212, 232, 137, 257, 61, 307, 105, 335] + 12: [156, 204, 149, 35, 120, 78, 158, 226, 327, 190, 303, 267, 292, 223, 289, 378, 150, 172, 333, 46, 290, 192, 361, 369, 340, 354, 295, 339, 281, 216, 215, 317, 25, 87, 108, 133, 153, 2, 265, 264, 57, 271, 135, 294, 80, 380, 74, 91, 144, 309, 201, 227, 188, 52, 185, 41, 45, 61, 224, 262, 280, 356, 37, 169, 11, 38, 115, 36, 305, 334, 183, 336, 244, 246, 312, 374, 352, 285, 252, 251, 34, 316, 55, 131, 299, 237, 230, 258, 143, 307, 269, 118, 221, 202, 342, 311, 359, 324, 66, 239, 373, 162, 110, 372, 255, 255, 363, 48, 381, 130, 158, 226, 344, 190, 127, 168, 124, 82, 177, 69, 243, 126, 333, 203, 341, 50, 51, 198, 170, 259, 134, 195, 382, 332, 194, 152, 218, 323, 89, 10, 27, 160, 102, 300, 104, 273, 63, 129, 326, 31, 59, 14, 286, 304, 193, 142, 77, 9, 191, 114, 165, 189, 67, 256, 161, 232, 173, 321, 214, 235, 121, 270, 20, 379, 32, 86, 365, 297, 207, 279, 274, 174, 146, 105, 47, 8, 343, 347, 212, 211, 320, 19, 196, 111, 26, 145, 360, 328, 206, 88, 92, 21, 331, 99, 43, 366, 272, 302, 255, 255, 49, 149, 120, 245, 158, 28, 17, 327, 127, 109, 22, 82, 177, 378, 243, 172, 287, 46, 341, 233, 181, 306, 260, 176, 75, 278, 68, 205, 276, 330, 147, 368, 60, 364, 288, 33, 101, 171, 322, 180, 197, 308, 261, 238, 23, 318, 15, 283, 253, 376, 200, 1, 315, 0, 83, 84, 298, 313, 4, 139, 187, 319, 76, 122, 40, 159, 13, 257, 293, 5, 179, 284, 128, 263, 85, 106, 70, 44, 282, 345, 96, 18, 54, 72, 236, 367, 182, 7, 184, 132, 242, 314, 100, 248, 377, 64, 141, 353, 53, 250, 225, 220, 255, 204, 204, 120, 130, 130, 78, 219, 327, 167, 303, 175, 223, 349, 296, 69, 150, 71, 103, 203, 97, 90, 151, 98, 208, 123, 3, 6, 137, 346, 348, 268, 229, 266, 16, 277, 39, 209, 116, 371, 362, 107, 164, 241, 291, 337, 157, 301, 155, 358, 357, 370, 350, 254, 231, 234, 29, 119, 62, 138, 136, 249, 325, 58, 335, 355, 154, 351, 65, 140, 12, 112, 186, 329, 148, 217, 24, 383, 310, 30, 375, 93, 178, 113, 81, 117, 125, 247, 79, 210, 94, 338, 163, 228, 213, 275, 166, 95, 56, 240, 222, 199, 73, 42] + 13: [80, 362, 77, 170, 108, 232, 18, 114, 14, 195, 87, 23, 265, 169, 282, 280, 55, 95, 238, 30, 30, 318, 134, 310, 199, 335, 171, 159, 118, 131, 227, 49, 75, 69, 297, 246, 309, 91, 266, 363, 208, 373, 22, 65, 10, 219, 339, 382, 144, 291, 107, 283, 359, 177, 256, 139, 52, 241, 37, 8, 163, 337, 286, 153, 319, 242, 259, 296, 381, 50, 133, 187, 360, 358, 19, 73, 235, 150, 104, 214, 182, 285, 85, 346, 336, 284, 63, 352, 152, 99, 224, 97, 349, 290, 47, 200, 82, 160, 193, 314, 126, 350, 192, 204, 80, 194, 137, 170, 108, 232, 18, 96, 14, 87, 149, 328, 260, 312, 172, 135, 231, 189, 238, 12, 274, 318, 102, 186, 88, 370, 46, 109, 35, 295, 111, 64, 364, 188, 275, 273, 261, 294, 158, 334, 2, 6, 98, 207, 375, 355, 43, 62, 25, 239, 196, 361, 374, 333, 183, 243, 380, 222, 303, 331, 89, 288, 379, 326, 220, 56, 305, 59, 245, 348, 71, 311, 70, 72, 217, 213, 124, 287, 353, 249, 205, 83, 167, 226, 343, 60, 203, 191, 258, 173, 347, 293, 132, 269, 354, 302, 351, 218, 247, 324, 33, 315, 230, 7, 9, 362, 329, 170, 108, 232, 18, 114, 86, 195, 87, 23, 39, 24, 41, 280, 276, 342, 110, 209, 254, 330, 271, 366, 11, 162, 298, 116, 307, 279, 58, 198, 16, 112, 356, 376, 40, 143, 67, 292, 299, 300, 42, 210, 211, 179, 240, 141, 228, 26, 225, 74, 93, 253, 129, 146, 15, 76, 125, 34, 48, 36, 68, 357, 345, 174, 92, 127, 325, 113, 250, 53, 154, 115, 13, 377, 5, 130, 61, 119, 138, 185, 157, 383, 1, 101, 215, 128, 278, 181, 371, 317, 236, 272, 368, 255, 322, 120, 66, 206, 27, 140, 316, 175, 44, 44, 329, 170, 170, 57, 81, 3, 0, 195, 149, 328, 260, 312, 51, 21, 201, 189, 100, 12, 274, 340, 197, 17, 142, 338, 306, 304, 323, 176, 45, 184, 369, 31, 289, 202, 54, 229, 281, 320, 178, 122, 378, 147, 168, 79, 237, 321, 344, 145, 262, 121, 166, 151, 327, 29, 164, 221, 223, 105, 267, 161, 248, 106, 136, 84, 257, 268, 263, 244, 308, 94, 123, 270, 38, 234, 313, 365, 103, 190, 233, 4, 372, 180, 252, 117, 28, 212, 90, 264, 251, 20, 332, 341, 156, 148, 367, 277, 165, 301, 216, 155, 78, 32] + 14: [147, 229, 211, 274, 372, 76, 76, 345, 137, 255, 110, 175, 2, 209, 75, 34, 150, 355, 337, 146, 235, 142, 99, 254, 198, 117, 3, 371, 162, 374, 271, 111, 41, 265, 13, 197, 297, 149, 215, 343, 291, 319, 369, 277, 301, 266, 133, 382, 349, 203, 178, 226, 222, 241, 380, 237, 298, 284, 0, 316, 300, 152, 242, 148, 224, 244, 248, 96, 285, 114, 262, 321, 278, 257, 338, 267, 238, 361, 19, 16, 263, 87, 18, 31, 366, 367, 84, 139, 11, 107, 320, 307, 51, 72, 368, 303, 38, 210, 119, 272, 83, 249, 289, 50, 70, 160, 229, 143, 261, 15, 164, 204, 8, 255, 110, 30, 2, 209, 344, 34, 327, 126, 118, 378, 155, 71, 275, 85, 52, 124, 342, 213, 317, 269, 182, 250, 216, 166, 125, 314, 135, 379, 328, 123, 288, 324, 193, 370, 363, 104, 346, 40, 325, 286, 296, 373, 5, 101, 347, 129, 341, 306, 225, 336, 115, 168, 313, 364, 356, 63, 43, 251, 106, 10, 214, 329, 358, 309, 36, 46, 20, 304, 14, 58, 299, 179, 45, 194, 293, 55, 92, 180, 81, 233, 381, 65, 188, 134, 243, 340, 218, 232, 138, 67, 318, 295, 331, 88, 70, 160, 229, 143, 4, 207, 287, 345, 345, 255, 199, 112, 49, 259, 75, 186, 327, 355, 353, 308, 155, 71, 23, 141, 95, 32, 97, 64, 61, 360, 21, 44, 157, 270, 220, 158, 59, 206, 323, 69, 184, 332, 201, 159, 108, 167, 339, 187, 102, 256, 109, 227, 82, 116, 377, 221, 330, 359, 253, 200, 202, 223, 264, 282, 350, 1, 93, 42, 311, 94, 6, 273, 375, 171, 365, 74, 78, 9, 290, 190, 312, 258, 80, 7, 66, 57, 185, 120, 140, 236, 144, 252, 113, 192, 54, 131, 352, 292, 196, 98, 315, 121, 205, 165, 56, 160, 229, 211, 4, 15, 77, 204, 60, 17, 383, 30, 268, 259, 239, 122, 122, 126, 348, 378, 308, 275, 228, 151, 100, 231, 177, 39, 154, 79, 279, 354, 294, 26, 208, 33, 161, 25, 246, 163, 357, 68, 326, 145, 37, 22, 333, 169, 128, 172, 245, 234, 48, 90, 156, 105, 176, 276, 86, 189, 335, 89, 191, 12, 183, 217, 240, 376, 28, 310, 173, 302, 153, 103, 195, 127, 136, 334, 305, 230, 322, 281, 132, 47, 362, 62, 35, 24, 181, 247, 219, 283, 170, 91, 27, 280, 29, 53, 351, 130, 73, 212, 260, 174] + 15: [68, 15, 350, 88, 88, 367, 130, 11, 359, 57, 39, 160, 38, 355, 355, 200, 128, 159, 148, 134, 134, 203, 146, 9, 80, 329, 139, 56, 296, 193, 256, 1, 242, 255, 196, 115, 162, 36, 207, 319, 182, 372, 238, 246, 105, 212, 2, 82, 309, 114, 229, 22, 205, 233, 341, 253, 294, 382, 107, 226, 347, 61, 354, 99, 208, 216, 383, 71, 283, 95, 334, 265, 273, 190, 3, 147, 274, 303, 184, 158, 14, 126, 312, 310, 7, 171, 361, 228, 178, 41, 120, 346, 288, 287, 214, 336, 249, 315, 152, 6, 131, 366, 281, 276, 298, 15, 340, 4, 259, 295, 11, 291, 371, 362, 160, 251, 189, 155, 332, 258, 94, 217, 175, 104, 252, 119, 121, 237, 232, 360, 23, 48, 353, 344, 311, 174, 177, 143, 18, 235, 31, 285, 357, 47, 78, 323, 28, 42, 102, 279, 86, 248, 87, 10, 150, 96, 363, 8, 206, 13, 271, 339, 349, 306, 53, 72, 308, 289, 55, 204, 169, 64, 26, 337, 316, 257, 240, 54, 278, 364, 73, 197, 153, 129, 370, 201, 293, 280, 327, 292, 66, 195, 154, 183, 307, 110, 108, 125, 156, 297, 186, 19, 46, 144, 322, 192, 93, 270, 0, 0, 261, 76, 227, 130, 130, 11, 359, 57, 199, 251, 324, 155, 231, 326, 94, 217, 148, 236, 74, 284, 60, 20, 221, 151, 376, 83, 113, 277, 342, 135, 241, 111, 109, 164, 379, 106, 351, 335, 133, 5, 49, 247, 101, 166, 117, 142, 343, 77, 331, 282, 69, 81, 254, 165, 377, 141, 191, 67, 180, 97, 365, 234, 314, 224, 123, 37, 92, 172, 84, 213, 167, 218, 239, 79, 91, 161, 89, 32, 262, 149, 187, 380, 124, 136, 330, 181, 44, 179, 304, 70, 286, 122, 219, 220, 132, 348, 25, 264, 325, 260, 356, 328, 0, 290, 15, 88, 227, 202, 130, 11, 359, 362, 160, 300, 324, 185, 200, 258, 266, 128, 175, 104, 333, 215, 90, 51, 317, 157, 16, 40, 35, 173, 168, 163, 138, 320, 230, 24, 352, 225, 338, 100, 170, 209, 305, 194, 145, 275, 59, 263, 269, 34, 268, 62, 50, 250, 12, 43, 301, 30, 313, 118, 321, 381, 267, 137, 244, 45, 272, 211, 373, 375, 358, 63, 116, 52, 65, 210, 378, 369, 374, 188, 29, 140, 176, 318, 58, 112, 222, 98, 299, 103, 223, 17, 243, 21, 302, 345, 27, 33, 245, 75, 198, 127, 368, 85] + 16: [11, 134, 134, 15, 286, 67, 125, 120, 300, 275, 262, 7, 338, 86, 49, 64, 44, 118, 342, 75, 271, 35, 302, 255, 283, 265, 213, 235, 270, 87, 226, 85, 294, 285, 177, 71, 59, 248, 316, 292, 119, 73, 155, 282, 31, 182, 34, 70, 220, 45, 122, 372, 174, 339, 4, 370, 198, 104, 375, 358, 158, 208, 214, 211, 333, 267, 272, 105, 48, 109, 266, 194, 381, 315, 157, 284, 47, 218, 131, 110, 216, 217, 50, 241, 138, 243, 352, 32, 355, 12, 112, 329, 136, 173, 291, 179, 296, 362, 13, 69, 94, 309, 344, 353, 162, 134, 134, 15, 286, 152, 330, 120, 120, 275, 92, 7, 277, 147, 147, 64, 303, 303, 280, 259, 271, 183, 78, 24, 209, 287, 141, 236, 367, 123, 346, 89, 65, 310, 52, 321, 115, 335, 38, 21, 80, 139, 56, 159, 231, 41, 219, 195, 232, 379, 202, 293, 137, 9, 322, 188, 317, 221, 354, 340, 132, 54, 102, 95, 206, 343, 319, 55, 190, 196, 193, 360, 186, 81, 88, 373, 145, 20, 17, 258, 371, 279, 176, 149, 245, 260, 101, 382, 79, 349, 40, 135, 350, 263, 72, 204, 325, 53, 191, 239, 126, 368, 156, 249, 100, 140, 15, 68, 369, 152, 331, 154, 234, 275, 312, 43, 165, 273, 269, 90, 44, 118, 203, 318, 5, 205, 224, 171, 383, 121, 187, 295, 146, 347, 180, 108, 27, 365, 306, 328, 307, 51, 377, 82, 103, 58, 66, 153, 380, 305, 8, 227, 357, 36, 42, 301, 160, 297, 96, 106, 225, 252, 151, 336, 200, 57, 313, 178, 223, 253, 93, 83, 244, 207, 201, 363, 62, 233, 127, 128, 324, 268, 113, 26, 170, 116, 359, 150, 61, 230, 288, 28, 63, 351, 74, 254, 299, 192, 326, 222, 376, 133, 345, 320, 247, 1, 98, 257, 100, 140, 15, 286, 286, 369, 185, 114, 234, 262, 92, 43, 165, 144, 269, 129, 143, 289, 280, 75, 348, 374, 274, 250, 298, 189, 364, 46, 246, 212, 16, 199, 181, 314, 84, 237, 2, 97, 99, 37, 175, 281, 0, 39, 311, 184, 76, 167, 356, 229, 334, 19, 276, 238, 22, 124, 290, 264, 29, 228, 25, 18, 60, 242, 337, 323, 304, 378, 91, 142, 30, 240, 172, 163, 14, 166, 107, 261, 168, 117, 308, 256, 77, 332, 210, 251, 33, 169, 6, 164, 23, 130, 341, 197, 327, 278, 361, 10, 161, 215, 111, 366, 3, 148] + 17: [374, 131, 220, 119, 208, 250, 284, 236, 137, 303, 337, 366, 108, 200, 375, 15, 136, 100, 359, 5, 330, 305, 343, 323, 1, 2, 189, 6, 370, 167, 222, 107, 96, 296, 335, 97, 381, 180, 380, 373, 10, 98, 33, 135, 292, 171, 245, 259, 340, 248, 47, 116, 59, 42, 269, 295, 49, 204, 377, 186, 341, 338, 342, 265, 94, 179, 262, 310, 90, 120, 161, 112, 267, 274, 369, 25, 192, 50, 150, 57, 163, 83, 153, 352, 149, 230, 356, 223, 349, 178, 242, 71, 247, 353, 268, 321, 252, 174, 54, 122, 159, 170, 202, 203, 261, 138, 220, 119, 80, 188, 284, 284, 190, 346, 79, 337, 12, 200, 91, 78, 117, 100, 26, 197, 330, 232, 185, 266, 130, 125, 347, 297, 276, 172, 290, 339, 39, 194, 218, 199, 145, 243, 38, 92, 371, 256, 99, 327, 214, 9, 142, 168, 264, 304, 19, 20, 173, 43, 41, 363, 216, 198, 365, 3, 309, 271, 95, 44, 379, 300, 93, 240, 258, 183, 345, 287, 151, 55, 158, 372, 76, 270, 31, 233, 32, 65, 315, 182, 378, 176, 109, 30, 229, 291, 288, 306, 334, 17, 51, 68, 156, 333, 14, 358, 312, 320, 325, 354, 261, 131, 131, 119, 105, 140, 188, 236, 319, 79, 79, 126, 108, 375, 254, 15, 311, 155, 26, 193, 305, 294, 362, 114, 113, 351, 29, 328, 75, 234, 56, 157, 134, 317, 46, 206, 148, 313, 211, 103, 143, 84, 272, 146, 282, 314, 162, 18, 332, 13, 166, 60, 45, 260, 215, 154, 118, 187, 221, 205, 101, 253, 21, 213, 360, 249, 195, 244, 299, 329, 16, 85, 64, 106, 278, 326, 364, 350, 63, 344, 53, 355, 316, 227, 23, 40, 277, 66, 257, 368, 11, 129, 77, 348, 111, 281, 209, 123, 34, 86, 69, 72, 121, 152, 28, 212, 131, 119, 307, 250, 284, 236, 226, 303, 337, 58, 87, 375, 219, 357, 383, 383, 225, 275, 286, 294, 362, 279, 246, 324, 133, 62, 175, 8, 52, 104, 207, 73, 22, 24, 231, 177, 228, 273, 367, 139, 298, 74, 165, 263, 70, 376, 191, 82, 61, 36, 331, 147, 7, 336, 81, 196, 115, 382, 37, 89, 181, 27, 110, 238, 302, 361, 102, 239, 210, 217, 128, 48, 308, 124, 67, 255, 318, 144, 164, 132, 241, 88, 280, 141, 201, 224, 4, 184, 160, 285, 289, 0, 35, 322, 301, 293, 169, 127, 235, 237, 251, 283] + 18: [208, 30, 30, 272, 43, 271, 193, 53, 251, 69, 320, 154, 300, 300, 65, 304, 71, 106, 185, 380, 174, 189, 355, 26, 139, 237, 291, 297, 35, 99, 77, 326, 240, 70, 19, 269, 337, 171, 342, 73, 307, 359, 87, 233, 316, 131, 0, 137, 278, 201, 313, 58, 157, 39, 195, 176, 205, 38, 244, 96, 132, 265, 255, 292, 94, 322, 166, 254, 48, 127, 358, 168, 332, 220, 4, 219, 56, 197, 317, 76, 298, 75, 107, 86, 37, 90, 299, 281, 24, 100, 11, 160, 21, 336, 295, 198, 379, 226, 74, 97, 222, 293, 1, 32, 208, 30, 47, 10, 271, 271, 271, 210, 251, 320, 217, 217, 113, 245, 57, 114, 71, 228, 368, 110, 192, 136, 25, 301, 373, 306, 357, 282, 378, 341, 33, 101, 103, 98, 164, 318, 20, 346, 191, 248, 104, 283, 149, 261, 88, 263, 162, 367, 122, 232, 250, 302, 289, 8, 140, 308, 279, 150, 319, 9, 12, 182, 13, 363, 287, 148, 324, 92, 108, 213, 383, 82, 170, 64, 27, 311, 163, 209, 123, 133, 218, 180, 372, 354, 85, 161, 29, 273, 377, 249, 117, 277, 72, 206, 194, 323, 55, 128, 374, 156, 242, 344, 147, 309, 6, 30, 30, 10, 43, 271, 271, 16, 251, 286, 315, 257, 28, 65, 57, 36, 138, 106, 185, 169, 239, 236, 146, 68, 34, 376, 151, 144, 330, 266, 190, 3, 124, 165, 274, 46, 371, 365, 141, 339, 177, 331, 343, 79, 334, 109, 135, 296, 159, 288, 361, 50, 61, 102, 111, 15, 125, 178, 276, 362, 145, 335, 267, 329, 270, 153, 325, 179, 275, 348, 234, 175, 216, 338, 51, 247, 126, 224, 369, 375, 364, 95, 258, 172, 44, 115, 352, 294, 345, 230, 18, 212, 199, 62, 134, 310, 49, 91, 327, 120, 227, 314, 59, 78, 118, 356, 30, 47, 271, 271, 45, 333, 211, 69, 207, 257, 347, 245, 215, 119, 138, 228, 260, 260, 116, 142, 167, 321, 360, 31, 284, 253, 246, 285, 105, 67, 188, 204, 89, 181, 41, 366, 202, 23, 121, 83, 262, 223, 225, 129, 241, 22, 7, 350, 231, 328, 235, 152, 203, 349, 54, 264, 81, 93, 63, 40, 155, 187, 351, 243, 214, 112, 303, 256, 60, 143, 200, 158, 66, 259, 130, 17, 229, 381, 2, 312, 42, 84, 370, 238, 184, 280, 14, 5, 305, 290, 173, 252, 221, 196, 353, 186, 52, 80, 183, 268, 382, 340] + 19: [187, 190, 366, 24, 339, 74, 7, 90, 57, 241, 120, 255, 338, 330, 322, 279, 192, 219, 101, 197, 64, 66, 184, 323, 20, 38, 303, 89, 159, 275, 189, 58, 365, 147, 238, 15, 13, 30, 222, 295, 6, 165, 167, 76, 353, 195, 257, 204, 2, 297, 4, 215, 95, 342, 336, 194, 85, 177, 23, 131, 60, 293, 280, 129, 225, 376, 362, 242, 298, 137, 209, 87, 186, 14, 208, 174, 28, 286, 371, 103, 112, 357, 267, 162, 185, 39, 224, 21, 155, 86, 56, 308, 226, 43, 199, 55, 142, 250, 378, 12, 34, 50, 81, 98, 187, 190, 366, 24, 111, 256, 302, 90, 57, 364, 91, 288, 288, 291, 126, 183, 318, 207, 312, 240, 64, 373, 113, 27, 154, 349, 227, 77, 230, 153, 217, 44, 41, 171, 246, 53, 285, 36, 352, 166, 359, 337, 270, 172, 29, 49, 170, 239, 40, 67, 345, 110, 244, 143, 287, 125, 284, 108, 380, 262, 160, 372, 92, 65, 140, 281, 329, 383, 341, 82, 139, 343, 370, 99, 105, 340, 149, 351, 73, 361, 348, 381, 210, 123, 178, 358, 367, 136, 344, 0, 5, 261, 169, 17, 72, 382, 196, 316, 229, 350, 319, 355, 369, 375, 187, 190, 366, 24, 179, 256, 7, 265, 90, 377, 120, 70, 288, 161, 313, 180, 42, 207, 101, 300, 216, 307, 115, 328, 193, 164, 334, 97, 152, 117, 79, 9, 158, 25, 335, 54, 71, 116, 128, 310, 228, 173, 96, 121, 251, 368, 138, 283, 157, 144, 259, 249, 374, 273, 84, 346, 19, 213, 45, 202, 75, 356, 3, 26, 191, 88, 200, 119, 130, 212, 22, 168, 232, 132, 234, 235, 63, 314, 37, 294, 150, 106, 11, 269, 201, 214, 252, 51, 205, 163, 236, 379, 282, 69, 220, 78, 145, 274, 93, 305, 107, 133, 198, 245, 141, 243, 211, 24, 206, 48, 7, 265, 61, 377, 52, 188, 338, 330, 313, 180, 42, 312, 8, 240, 114, 307, 115, 363, 218, 33, 296, 271, 301, 354, 35, 264, 292, 46, 135, 122, 233, 1, 278, 134, 127, 182, 68, 83, 156, 16, 272, 221, 260, 347, 277, 31, 306, 203, 325, 18, 333, 237, 10, 80, 331, 148, 47, 62, 309, 146, 326, 321, 176, 311, 32, 320, 258, 181, 332, 247, 94, 324, 59, 315, 231, 118, 124, 289, 290, 223, 304, 100, 299, 266, 248, 102, 151, 360, 327, 104, 175, 276, 109, 317, 253, 254, 263, 268] + 20: [116, 334, 37, 302, 272, 161, 317, 144, 98, 98, 110, 153, 376, 43, 112, 268, 339, 335, 130, 353, 363, 188, 318, 93, 158, 111, 141, 125, 382, 187, 259, 227, 342, 320, 172, 71, 77, 315, 190, 176, 284, 218, 35, 361, 118, 0, 146, 367, 150, 208, 372, 114, 196, 60, 86, 345, 370, 20, 310, 252, 236, 304, 170, 201, 326, 24, 212, 230, 159, 217, 210, 11, 156, 79, 250, 88, 321, 102, 163, 271, 67, 211, 276, 313, 134, 373, 183, 70, 149, 23, 222, 344, 255, 44, 143, 288, 242, 280, 266, 281, 299, 303, 331, 350, 116, 334, 278, 174, 14, 161, 82, 66, 66, 193, 153, 287, 2, 369, 226, 145, 4, 307, 282, 282, 32, 185, 162, 325, 164, 324, 290, 95, 381, 9, 50, 160, 292, 132, 92, 65, 126, 322, 74, 177, 240, 235, 328, 375, 219, 36, 314, 359, 309, 279, 18, 22, 238, 215, 340, 169, 294, 15, 338, 33, 127, 73, 365, 231, 360, 191, 84, 253, 308, 56, 80, 349, 59, 199, 21, 296, 40, 7, 329, 323, 354, 305, 19, 157, 64, 261, 68, 245, 220, 129, 301, 152, 237, 1, 69, 197, 198, 260, 107, 137, 167, 168, 179, 181, 116, 334, 278, 302, 336, 161, 161, 144, 81, 193, 153, 99, 87, 369, 45, 145, 4, 335, 46, 195, 204, 270, 138, 371, 209, 228, 273, 128, 311, 362, 140, 286, 216, 101, 347, 121, 182, 357, 6, 13, 194, 348, 378, 41, 175, 53, 30, 58, 233, 300, 42, 256, 274, 243, 346, 246, 248, 377, 12, 100, 78, 332, 39, 3, 312, 374, 223, 75, 94, 154, 139, 366, 142, 289, 120, 319, 113, 85, 333, 341, 47, 136, 83, 207, 133, 293, 108, 295, 251, 186, 306, 155, 244, 38, 109, 269, 241, 277, 189, 200, 206, 214, 262, 263, 116, 8, 122, 336, 161, 161, 55, 82, 173, 98, 123, 99, 87, 43, 45, 268, 379, 291, 46, 224, 105, 62, 330, 148, 10, 91, 229, 221, 96, 264, 180, 135, 258, 49, 184, 358, 104, 90, 239, 57, 364, 267, 343, 26, 265, 25, 368, 213, 247, 178, 119, 34, 131, 166, 165, 97, 297, 29, 5, 232, 106, 380, 257, 202, 327, 89, 337, 351, 254, 316, 285, 151, 352, 115, 283, 124, 28, 356, 205, 355, 72, 225, 16, 63, 48, 234, 383, 147, 192, 103, 275, 31, 203, 298, 171, 17, 117, 249, 27, 51, 52, 54, 61, 76] + 21: [152, 316, 308, 355, 190, 256, 47, 173, 281, 360, 360, 132, 50, 103, 124, 124, 92, 257, 185, 364, 363, 199, 120, 226, 178, 91, 177, 238, 191, 151, 351, 19, 121, 219, 25, 69, 9, 209, 221, 189, 234, 373, 89, 229, 168, 63, 1, 358, 39, 90, 309, 318, 251, 21, 356, 28, 354, 290, 323, 3, 237, 353, 115, 193, 34, 160, 104, 203, 329, 186, 335, 300, 379, 333, 305, 110, 195, 163, 310, 241, 143, 239, 41, 130, 129, 85, 375, 54, 162, 296, 285, 145, 298, 59, 188, 282, 343, 272, 297, 11, 42, 369, 339, 350, 152, 20, 371, 153, 254, 4, 47, 247, 281, 360, 29, 132, 80, 103, 200, 376, 119, 218, 113, 140, 88, 361, 277, 345, 252, 284, 250, 320, 22, 307, 147, 349, 15, 299, 293, 225, 43, 313, 141, 220, 368, 183, 243, 174, 73, 338, 18, 331, 287, 138, 60, 304, 97, 66, 383, 192, 370, 125, 347, 30, 295, 40, 100, 301, 136, 187, 79, 269, 57, 258, 126, 36, 215, 81, 83, 271, 319, 181, 342, 253, 156, 268, 74, 374, 217, 99, 37, 180, 176, 328, 6, 106, 326, 61, 212, 95, 235, 101, 127, 311, 44, 10, 17, 35, 152, 20, 371, 355, 355, 4, 46, 173, 206, 205, 86, 132, 80, 103, 210, 119, 71, 112, 87, 208, 169, 117, 214, 45, 118, 182, 149, 51, 380, 172, 26, 109, 196, 8, 194, 280, 135, 144, 367, 23, 78, 33, 233, 325, 263, 372, 49, 336, 362, 64, 223, 246, 62, 260, 286, 291, 211, 155, 378, 154, 65, 204, 171, 148, 137, 14, 102, 24, 274, 146, 216, 48, 244, 213, 134, 334, 12, 96, 315, 164, 70, 201, 330, 2, 68, 357, 56, 314, 249, 202, 242, 31, 161, 7, 116, 222, 248, 107, 167, 327, 108, 58, 93, 123, 302, 316, 289, 308, 254, 236, 47, 173, 281, 360, 158, 158, 53, 38, 82, 376, 71, 257, 113, 140, 94, 348, 232, 377, 150, 0, 240, 279, 184, 16, 365, 128, 175, 366, 227, 344, 52, 230, 111, 72, 245, 288, 270, 131, 84, 197, 267, 352, 359, 259, 262, 139, 98, 312, 303, 159, 317, 165, 5, 133, 77, 275, 266, 341, 337, 292, 198, 170, 340, 346, 261, 276, 273, 157, 27, 381, 67, 105, 382, 294, 75, 207, 13, 122, 76, 55, 114, 324, 255, 228, 265, 142, 278, 32, 166, 231, 322, 224, 283, 332, 179, 264, 306, 321] + 22: [371, 281, 216, 198, 343, 331, 181, 260, 141, 148, 24, 261, 328, 207, 224, 242, 20, 184, 376, 230, 4, 74, 118, 244, 61, 317, 352, 229, 381, 153, 188, 251, 336, 375, 161, 8, 123, 60, 193, 214, 122, 57, 228, 3, 53, 339, 258, 374, 233, 71, 165, 12, 105, 320, 341, 98, 65, 22, 124, 189, 327, 335, 0, 180, 300, 271, 202, 136, 356, 26, 265, 103, 232, 102, 219, 182, 92, 294, 152, 134, 342, 19, 168, 212, 296, 354, 88, 326, 43, 213, 163, 114, 270, 167, 86, 231, 315, 192, 287, 360, 307, 310, 346, 359, 254, 254, 204, 51, 343, 209, 181, 34, 260, 148, 24, 330, 328, 115, 224, 288, 201, 20, 110, 94, 144, 169, 29, 44, 383, 362, 240, 50, 70, 345, 285, 311, 173, 80, 372, 378, 312, 125, 17, 369, 226, 170, 175, 358, 82, 191, 63, 49, 15, 309, 99, 225, 367, 39, 47, 305, 155, 200, 54, 227, 78, 238, 299, 241, 13, 211, 215, 96, 353, 379, 298, 108, 246, 119, 266, 282, 120, 30, 62, 179, 350, 104, 69, 302, 308, 363, 162, 5, 140, 304, 277, 172, 293, 243, 194, 234, 325, 205, 289, 2, 28, 79, 95, 101, 254, 316, 216, 126, 278, 278, 221, 221, 260, 141, 261, 276, 132, 42, 337, 164, 183, 73, 376, 94, 37, 263, 329, 235, 117, 58, 252, 348, 357, 351, 218, 338, 306, 67, 68, 83, 106, 264, 259, 27, 349, 361, 347, 113, 340, 366, 220, 18, 324, 59, 41, 210, 1, 380, 159, 87, 301, 236, 199, 283, 11, 373, 382, 268, 138, 290, 77, 89, 275, 46, 237, 262, 274, 160, 186, 137, 334, 36, 129, 190, 16, 64, 90, 344, 31, 150, 370, 127, 284, 66, 146, 295, 107, 121, 368, 297, 10, 135, 249, 292, 143, 154, 157, 217, 254, 281, 33, 9, 332, 209, 181, 221, 141, 141, 261, 328, 147, 38, 337, 288, 183, 178, 178, 110, 177, 112, 139, 93, 355, 130, 333, 55, 272, 149, 7, 111, 100, 195, 76, 174, 365, 222, 116, 185, 206, 14, 75, 109, 156, 48, 40, 25, 131, 321, 133, 72, 239, 319, 248, 318, 303, 322, 269, 377, 256, 32, 45, 81, 286, 97, 314, 273, 364, 176, 208, 279, 35, 85, 187, 151, 52, 196, 145, 91, 257, 291, 171, 128, 255, 223, 6, 23, 21, 197, 158, 84, 245, 166, 56, 203, 313, 142, 280, 323, 247, 250, 253, 267] + 23: [26, 126, 264, 124, 175, 314, 275, 120, 71, 170, 19, 305, 187, 82, 240, 244, 203, 85, 197, 324, 327, 47, 176, 156, 180, 127, 57, 55, 270, 56, 130, 41, 190, 35, 80, 50, 102, 249, 191, 229, 152, 10, 228, 224, 136, 210, 356, 157, 32, 96, 160, 98, 53, 65, 301, 307, 382, 316, 284, 352, 258, 143, 0, 128, 245, 174, 206, 134, 121, 122, 297, 294, 239, 25, 139, 355, 106, 248, 140, 282, 158, 353, 34, 31, 322, 165, 334, 299, 343, 22, 289, 63, 183, 145, 332, 60, 186, 366, 234, 15, 291, 286, 242, 168, 26, 126, 264, 124, 175, 314, 275, 71, 71, 147, 77, 305, 201, 167, 173, 269, 323, 252, 72, 135, 177, 177, 261, 232, 192, 304, 372, 59, 351, 315, 78, 263, 209, 309, 144, 184, 74, 70, 36, 149, 95, 195, 38, 290, 215, 311, 101, 361, 1, 119, 93, 365, 52, 125, 333, 179, 347, 250, 204, 303, 54, 363, 339, 200, 335, 346, 16, 68, 354, 243, 362, 285, 97, 115, 369, 267, 214, 79, 367, 202, 9, 319, 62, 266, 350, 377, 171, 181, 118, 141, 345, 278, 105, 89, 293, 380, 13, 18, 110, 4, 75, 43, 37, 265, 26, 328, 281, 51, 69, 172, 314, 274, 376, 100, 132, 132, 295, 167, 306, 244, 12, 252, 72, 135, 103, 358, 27, 94, 241, 374, 169, 218, 159, 148, 371, 92, 133, 131, 146, 111, 378, 220, 233, 271, 313, 194, 237, 272, 221, 292, 44, 222, 298, 185, 329, 66, 231, 5, 325, 321, 223, 330, 193, 76, 45, 73, 109, 280, 28, 164, 7, 212, 163, 155, 349, 236, 336, 375, 33, 104, 260, 161, 188, 61, 253, 357, 88, 276, 337, 238, 259, 360, 364, 318, 30, 117, 246, 262, 217, 198, 296, 251, 29, 3, 39, 8, 320, 288, 26, 328, 225, 124, 172, 172, 314, 275, 211, 100, 48, 305, 331, 187, 24, 87, 323, 85, 300, 300, 279, 287, 154, 84, 67, 11, 182, 2, 6, 86, 257, 283, 340, 379, 326, 213, 219, 312, 91, 151, 317, 23, 342, 40, 255, 129, 359, 370, 150, 153, 254, 17, 273, 137, 178, 207, 162, 341, 196, 348, 90, 58, 344, 256, 107, 302, 108, 21, 114, 14, 268, 99, 123, 338, 227, 373, 308, 205, 49, 46, 310, 368, 230, 277, 383, 116, 247, 381, 138, 20, 208, 42, 83, 199, 112, 142, 166, 226, 113, 216, 189, 235, 81, 64] + 24: [0, 87, 9, 9, 246, 154, 265, 6, 252, 7, 7, 172, 223, 185, 381, 357, 369, 27, 189, 286, 104, 220, 370, 186, 283, 53, 78, 159, 354, 144, 91, 61, 273, 41, 106, 355, 350, 356, 49, 177, 293, 345, 307, 310, 287, 279, 376, 83, 257, 84, 13, 233, 197, 313, 1, 268, 66, 312, 108, 215, 346, 335, 267, 282, 271, 176, 88, 296, 107, 212, 188, 249, 136, 8, 148, 163, 262, 209, 288, 44, 109, 25, 18, 256, 187, 378, 240, 75, 141, 200, 316, 243, 371, 261, 364, 100, 2, 45, 21, 129, 232, 74, 81, 115, 0, 92, 110, 9, 111, 154, 265, 20, 22, 252, 372, 259, 263, 139, 157, 357, 361, 324, 128, 112, 266, 334, 62, 207, 237, 343, 213, 24, 164, 344, 382, 366, 71, 318, 254, 156, 46, 113, 149, 351, 341, 182, 12, 3, 206, 221, 140, 59, 147, 235, 269, 295, 311, 242, 51, 11, 292, 239, 60, 270, 37, 218, 170, 72, 276, 303, 120, 10, 363, 76, 284, 320, 281, 42, 67, 19, 55, 50, 234, 238, 132, 101, 68, 367, 247, 43, 174, 300, 90, 165, 227, 340, 33, 117, 305, 17, 152, 89, 127, 133, 251, 166, 198, 241, 290, 52, 211, 246, 154, 265, 95, 20, 328, 252, 372, 210, 223, 337, 360, 70, 70, 85, 189, 64, 5, 255, 375, 230, 319, 137, 202, 204, 192, 35, 190, 54, 325, 299, 274, 260, 138, 377, 96, 73, 338, 145, 118, 77, 48, 58, 297, 63, 169, 153, 16, 278, 222, 304, 315, 56, 302, 194, 30, 181, 309, 258, 203, 275, 23, 79, 14, 253, 86, 175, 161, 126, 99, 373, 191, 160, 171, 102, 193, 342, 248, 362, 236, 125, 229, 98, 162, 250, 124, 199, 289, 123, 349, 214, 330, 82, 358, 36, 374, 179, 368, 353, 379, 383, 97, 52, 92, 246, 154, 265, 6, 20, 22, 252, 7, 122, 223, 263, 28, 105, 369, 85, 121, 128, 294, 301, 29, 146, 231, 26, 31, 135, 277, 195, 15, 306, 93, 226, 131, 308, 142, 130, 39, 336, 224, 380, 245, 205, 272, 114, 321, 326, 178, 216, 168, 323, 116, 47, 244, 280, 339, 65, 225, 134, 103, 365, 333, 4, 38, 348, 150, 196, 173, 32, 151, 94, 352, 57, 331, 359, 329, 217, 291, 317, 183, 180, 184, 80, 201, 143, 228, 332, 119, 167, 285, 347, 69, 208, 322, 40, 219, 34, 155, 158, 327, 264, 298, 314] + 25: [315, 315, 269, 228, 247, 247, 379, 82, 85, 101, 282, 375, 78, 61, 107, 25, 309, 284, 152, 62, 112, 333, 5, 273, 305, 38, 380, 59, 18, 351, 381, 346, 252, 9, 84, 231, 125, 153, 89, 367, 146, 164, 135, 4, 359, 20, 225, 352, 203, 237, 81, 68, 354, 308, 357, 213, 297, 28, 339, 181, 14, 355, 105, 356, 79, 53, 249, 74, 110, 127, 111, 271, 35, 75, 36, 86, 183, 283, 364, 208, 40, 175, 64, 92, 349, 3, 63, 344, 147, 363, 12, 42, 90, 169, 288, 329, 194, 170, 43, 44, 121, 156, 263, 91, 315, 315, 98, 65, 228, 298, 379, 379, 317, 72, 180, 276, 276, 23, 372, 214, 29, 52, 192, 376, 77, 232, 145, 7, 66, 33, 216, 230, 207, 262, 287, 302, 215, 373, 253, 209, 80, 256, 54, 366, 227, 161, 261, 348, 196, 291, 211, 336, 132, 319, 221, 383, 345, 259, 330, 37, 134, 331, 116, 257, 144, 310, 67, 212, 200, 27, 223, 17, 275, 353, 337, 163, 22, 117, 93, 124, 167, 70, 226, 173, 293, 289, 245, 41, 188, 265, 205, 119, 57, 141, 71, 217, 301, 198, 266, 241, 106, 13, 338, 382, 189, 239, 123, 178, 315, 269, 269, 139, 83, 268, 379, 299, 72, 282, 282, 180, 162, 372, 39, 214, 29, 327, 192, 246, 171, 49, 190, 160, 313, 102, 314, 318, 120, 342, 154, 131, 306, 197, 377, 56, 229, 244, 138, 60, 279, 321, 307, 103, 143, 114, 347, 115, 304, 316, 199, 187, 55, 204, 184, 290, 328, 58, 254, 201, 94, 370, 11, 31, 238, 159, 340, 193, 243, 166, 186, 109, 177, 281, 34, 250, 320, 45, 210, 311, 46, 371, 118, 174, 137, 155, 48, 264, 360, 172, 374, 285, 108, 219, 47, 50, 6, 151, 270, 19, 165, 220, 267, 158, 315, 269, 269, 139, 258, 168, 379, 334, 317, 72, 282, 30, 332, 23, 107, 2, 0, 284, 113, 62, 10, 73, 280, 235, 8, 296, 32, 378, 368, 100, 136, 294, 222, 234, 341, 240, 76, 99, 323, 126, 343, 97, 218, 176, 140, 242, 191, 369, 278, 295, 350, 312, 130, 324, 185, 15, 202, 365, 133, 274, 69, 149, 24, 150, 300, 251, 361, 179, 358, 260, 248, 51, 292, 325, 157, 362, 122, 21, 96, 87, 16, 128, 236, 322, 142, 224, 95, 1, 286, 335, 26, 88, 104, 272, 148, 182, 206, 195, 255, 303, 129, 233, 277, 326] + 26: [80, 136, 6, 292, 380, 269, 269, 237, 237, 172, 172, 53, 342, 371, 184, 291, 304, 254, 224, 116, 279, 33, 308, 307, 227, 223, 315, 376, 108, 146, 131, 82, 265, 204, 311, 176, 138, 319, 323, 133, 367, 306, 27, 97, 103, 221, 343, 57, 20, 192, 17, 277, 65, 113, 212, 0, 107, 76, 196, 50, 129, 123, 153, 363, 298, 189, 368, 78, 49, 3, 105, 352, 225, 354, 334, 43, 9, 297, 361, 333, 193, 13, 91, 274, 358, 270, 284, 208, 283, 211, 28, 164, 381, 155, 379, 288, 132, 213, 111, 139, 169, 170, 175, 209, 80, 136, 6, 281, 89, 11, 349, 237, 217, 217, 90, 214, 236, 181, 181, 338, 253, 339, 230, 16, 94, 183, 38, 266, 369, 135, 22, 30, 36, 92, 328, 55, 200, 359, 128, 48, 259, 191, 166, 262, 142, 2, 356, 251, 241, 186, 118, 294, 127, 71, 15, 239, 81, 161, 98, 378, 171, 252, 37, 350, 144, 75, 182, 383, 25, 134, 336, 7, 197, 140, 340, 47, 165, 255, 72, 8, 346, 77, 331, 244, 163, 326, 205, 216, 324, 258, 42, 145, 282, 85, 345, 257, 84, 310, 322, 273, 60, 203, 1, 54, 68, 86, 96, 106, 150, 136, 5, 292, 89, 11, 269, 124, 237, 18, 90, 53, 63, 21, 70, 291, 162, 160, 120, 289, 276, 316, 137, 341, 293, 61, 226, 88, 245, 382, 62, 332, 67, 156, 148, 198, 26, 360, 190, 151, 353, 174, 152, 327, 79, 24, 201, 268, 285, 104, 318, 195, 159, 10, 35, 202, 267, 370, 231, 114, 187, 143, 232, 19, 117, 301, 157, 264, 220, 12, 125, 362, 317, 109, 335, 64, 34, 59, 112, 39, 29, 206, 119, 314, 373, 313, 4, 14, 250, 330, 46, 188, 23, 58, 248, 207, 309, 260, 218, 228, 234, 242, 246, 261, 320, 136, 136, 233, 89, 380, 271, 124, 217, 18, 172, 90, 236, 21, 263, 253, 149, 339, 230, 69, 337, 344, 130, 278, 238, 355, 158, 249, 126, 167, 110, 83, 247, 180, 31, 272, 178, 179, 321, 287, 44, 229, 99, 102, 173, 122, 51, 194, 66, 365, 357, 215, 87, 73, 45, 290, 115, 41, 32, 256, 93, 177, 372, 312, 280, 302, 219, 329, 300, 121, 325, 364, 141, 185, 40, 347, 275, 74, 303, 100, 154, 240, 199, 101, 296, 56, 377, 243, 295, 222, 147, 235, 366, 95, 286, 210, 52, 168, 299, 305, 348, 351, 374, 375] + 27: [34, 103, 37, 177, 279, 58, 286, 333, 263, 61, 312, 90, 265, 168, 252, 269, 30, 197, 49, 80, 341, 358, 292, 131, 369, 207, 143, 183, 75, 251, 323, 85, 242, 76, 339, 258, 115, 156, 23, 32, 201, 141, 193, 187, 42, 27, 155, 162, 125, 126, 326, 214, 291, 215, 327, 268, 124, 267, 129, 248, 299, 171, 28, 318, 321, 365, 167, 0, 275, 35, 159, 130, 306, 96, 231, 154, 313, 223, 22, 293, 352, 110, 289, 382, 62, 86, 166, 192, 1, 194, 148, 314, 221, 295, 92, 57, 209, 51, 180, 235, 285, 310, 338, 342, 234, 103, 301, 177, 117, 4, 225, 351, 263, 31, 312, 374, 265, 233, 252, 69, 26, 315, 128, 80, 227, 211, 354, 24, 213, 224, 97, 55, 228, 272, 21, 303, 266, 149, 99, 232, 102, 83, 332, 325, 140, 284, 356, 317, 98, 133, 88, 254, 309, 54, 329, 6, 172, 136, 296, 122, 287, 276, 111, 106, 347, 9, 120, 249, 29, 250, 217, 84, 205, 123, 79, 151, 344, 280, 113, 259, 319, 50, 8, 264, 229, 150, 372, 158, 257, 334, 345, 108, 173, 331, 138, 170, 255, 355, 363, 206, 378, 93, 203, 247, 146, 163, 176, 220, 234, 103, 37, 145, 59, 225, 225, 12, 12, 61, 312, 90, 300, 164, 73, 69, 135, 320, 119, 376, 341, 366, 181, 377, 348, 222, 336, 190, 335, 48, 290, 340, 381, 324, 216, 72, 238, 294, 89, 82, 322, 373, 118, 337, 305, 161, 7, 370, 91, 271, 10, 175, 114, 304, 274, 127, 375, 362, 67, 239, 147, 2, 199, 307, 101, 47, 95, 367, 361, 153, 116, 105, 139, 144, 152, 184, 282, 346, 74, 253, 302, 330, 16, 142, 273, 94, 189, 64, 165, 56, 245, 311, 210, 277, 40, 371, 17, 36, 100, 212, 270, 240, 244, 262, 81, 103, 103, 104, 219, 4, 286, 286, 12, 70, 350, 90, 168, 71, 73, 204, 30, 357, 349, 349, 227, 211, 354, 202, 188, 179, 237, 112, 182, 230, 178, 260, 380, 383, 196, 77, 343, 308, 78, 278, 297, 25, 109, 191, 241, 15, 261, 328, 368, 316, 246, 379, 174, 5, 218, 360, 45, 46, 18, 43, 20, 44, 38, 66, 121, 353, 13, 3, 107, 288, 256, 33, 14, 132, 281, 208, 185, 359, 65, 186, 157, 41, 364, 68, 195, 283, 198, 87, 52, 226, 160, 11, 19, 298, 169, 134, 243, 63, 200, 236, 39, 53, 60, 137] + 28: [4, 312, 239, 310, 77, 344, 26, 26, 328, 203, 185, 152, 306, 302, 162, 195, 107, 103, 25, 286, 197, 113, 253, 7, 199, 346, 378, 137, 320, 71, 204, 58, 261, 352, 90, 361, 285, 220, 370, 32, 105, 182, 13, 349, 376, 114, 380, 43, 206, 178, 1, 290, 194, 183, 46, 245, 6, 249, 57, 274, 159, 21, 205, 175, 169, 230, 15, 84, 332, 297, 154, 377, 109, 31, 330, 119, 287, 351, 225, 131, 303, 146, 347, 192, 252, 101, 275, 140, 94, 141, 189, 254, 145, 313, 124, 62, 301, 138, 298, 150, 156, 180, 207, 215, 355, 161, 201, 196, 250, 250, 26, 92, 356, 70, 317, 226, 306, 97, 243, 172, 30, 288, 93, 286, 197, 214, 307, 374, 75, 50, 219, 3, 223, 59, 51, 5, 200, 383, 160, 130, 167, 163, 242, 299, 221, 56, 340, 238, 273, 247, 48, 16, 335, 222, 184, 78, 118, 244, 260, 264, 327, 358, 64, 69, 98, 381, 143, 322, 296, 12, 66, 170, 342, 34, 24, 294, 372, 218, 96, 266, 80, 39, 251, 246, 8, 17, 229, 139, 232, 174, 373, 277, 102, 202, 304, 354, 181, 318, 164, 186, 2, 40, 147, 359, 257, 279, 311, 329, 161, 161, 0, 168, 250, 22, 324, 356, 153, 188, 317, 255, 255, 120, 243, 233, 30, 52, 122, 99, 241, 321, 37, 357, 369, 108, 166, 95, 325, 42, 228, 116, 63, 235, 88, 135, 67, 289, 41, 240, 125, 217, 20, 33, 19, 193, 278, 316, 300, 333, 91, 14, 121, 142, 365, 265, 171, 208, 23, 295, 82, 375, 362, 61, 36, 258, 270, 334, 133, 280, 151, 382, 144, 79, 364, 149, 323, 345, 38, 284, 234, 85, 237, 47, 55, 76, 267, 123, 45, 126, 65, 53, 83, 309, 87, 262, 11, 132, 190, 10, 18, 60, 86, 115, 161, 161, 248, 168, 250, 344, 283, 92, 153, 70, 185, 185, 176, 302, 162, 155, 107, 52, 25, 99, 241, 321, 224, 272, 263, 128, 236, 191, 368, 227, 158, 165, 100, 129, 157, 339, 28, 337, 326, 269, 209, 271, 216, 308, 367, 315, 127, 268, 341, 134, 68, 282, 27, 256, 112, 179, 49, 81, 44, 314, 305, 177, 148, 106, 74, 104, 210, 211, 363, 366, 73, 136, 331, 117, 89, 213, 348, 110, 293, 292, 231, 72, 276, 259, 29, 281, 35, 111, 291, 350, 371, 360, 187, 319, 198, 212, 9, 54, 173, 379, 336, 338, 343, 353] + 29: [383, 14, 196, 195, 365, 244, 51, 25, 76, 354, 323, 198, 304, 362, 265, 74, 60, 44, 173, 173, 208, 11, 16, 255, 183, 243, 57, 134, 175, 188, 216, 145, 82, 319, 193, 350, 61, 360, 378, 85, 12, 106, 178, 329, 18, 347, 272, 59, 97, 181, 380, 142, 20, 313, 371, 217, 316, 238, 242, 36, 126, 170, 10, 28, 191, 55, 381, 56, 42, 7, 166, 13, 374, 352, 227, 307, 168, 149, 17, 71, 45, 337, 30, 246, 109, 49, 41, 201, 2, 80, 5, 226, 64, 81, 221, 342, 247, 274, 367, 249, 39, 355, 240, 285, 197, 271, 58, 118, 244, 336, 51, 361, 137, 277, 258, 203, 295, 339, 67, 262, 236, 309, 282, 23, 127, 148, 255, 89, 326, 180, 366, 375, 235, 24, 237, 68, 341, 123, 293, 86, 364, 172, 259, 218, 128, 116, 273, 222, 136, 19, 146, 228, 264, 78, 310, 267, 125, 269, 328, 344, 373, 22, 229, 348, 93, 52, 302, 135, 213, 111, 287, 327, 84, 299, 338, 261, 187, 239, 121, 194, 113, 219, 308, 154, 184, 211, 141, 296, 234, 155, 358, 279, 21, 330, 75, 130, 291, 165, 268, 138, 305, 289, 214, 6, 185, 26, 363, 176, 197, 283, 196, 118, 124, 336, 318, 361, 379, 277, 198, 333, 304, 129, 67, 186, 60, 44, 202, 23, 127, 11, 266, 346, 65, 140, 174, 87, 190, 343, 163, 370, 382, 210, 38, 50, 94, 132, 248, 298, 34, 257, 232, 349, 225, 72, 280, 90, 150, 345, 353, 292, 105, 205, 317, 167, 47, 122, 224, 312, 315, 40, 369, 147, 209, 79, 103, 220, 70, 153, 250, 161, 15, 233, 69, 62, 356, 306, 334, 99, 115, 35, 251, 92, 164, 96, 223, 254, 8, 100, 27, 286, 1, 372, 335, 281, 98, 256, 206, 143, 177, 108, 91, 152, 73, 73, 88, 118, 244, 54, 318, 361, 137, 120, 323, 203, 295, 362, 33, 110, 236, 320, 282, 311, 311, 351, 160, 89, 112, 119, 48, 159, 332, 63, 275, 204, 207, 102, 357, 278, 4, 324, 314, 215, 182, 139, 156, 200, 131, 300, 171, 276, 151, 101, 104, 3, 32, 294, 340, 77, 359, 83, 169, 31, 157, 230, 46, 377, 53, 303, 114, 368, 158, 9, 288, 199, 192, 253, 179, 270, 162, 376, 0, 43, 263, 252, 290, 297, 325, 301, 133, 189, 331, 95, 245, 260, 231, 107, 321, 241, 117, 284, 37, 144, 212, 322, 29, 66] + 30: [140, 37, 266, 262, 208, 208, 213, 333, 137, 187, 366, 65, 245, 196, 52, 35, 24, 351, 156, 156, 289, 321, 28, 346, 68, 17, 205, 303, 340, 49, 341, 221, 77, 253, 256, 58, 215, 141, 136, 226, 296, 274, 369, 258, 365, 325, 104, 353, 84, 231, 335, 306, 14, 319, 129, 111, 293, 55, 73, 181, 297, 290, 374, 318, 186, 154, 276, 107, 267, 371, 279, 161, 203, 99, 155, 5, 209, 364, 122, 314, 169, 178, 330, 120, 198, 106, 56, 265, 372, 367, 93, 229, 251, 30, 176, 301, 354, 88, 94, 101, 118, 123, 128, 134, 140, 37, 266, 262, 208, 239, 239, 76, 187, 13, 243, 65, 245, 78, 52, 35, 143, 295, 351, 2, 361, 234, 223, 64, 41, 212, 271, 315, 216, 224, 240, 317, 336, 1, 71, 193, 236, 151, 204, 277, 292, 281, 160, 67, 60, 23, 278, 261, 54, 72, 383, 162, 359, 20, 294, 66, 375, 211, 332, 7, 249, 50, 145, 288, 194, 311, 95, 121, 220, 350, 32, 304, 57, 83, 85, 323, 130, 117, 3, 153, 15, 19, 308, 352, 112, 207, 235, 0, 149, 63, 26, 182, 119, 4, 70, 227, 307, 379, 268, 269, 322, 334, 345, 380, 140, 254, 124, 208, 208, 239, 239, 272, 86, 358, 75, 242, 245, 135, 214, 378, 46, 257, 373, 62, 62, 188, 45, 144, 33, 164, 316, 362, 159, 43, 74, 12, 177, 152, 339, 192, 309, 299, 217, 195, 18, 170, 285, 98, 157, 343, 132, 108, 38, 329, 114, 142, 163, 197, 172, 280, 349, 241, 244, 248, 92, 44, 79, 360, 166, 110, 230, 202, 263, 247, 139, 326, 381, 116, 287, 165, 370, 368, 127, 355, 233, 16, 190, 255, 300, 270, 199, 313, 39, 42, 148, 34, 87, 252, 201, 305, 363, 146, 185, 189, 218, 219, 237, 259, 133, 158, 266, 184, 208, 239, 239, 222, 29, 358, 53, 53, 65, 59, 214, 225, 143, 168, 373, 206, 11, 289, 246, 10, 147, 283, 302, 250, 97, 312, 89, 138, 357, 100, 6, 275, 9, 210, 320, 382, 344, 260, 356, 298, 174, 200, 96, 91, 167, 179, 103, 150, 376, 51, 310, 286, 180, 109, 48, 115, 282, 284, 175, 25, 337, 342, 377, 183, 347, 324, 264, 90, 348, 69, 125, 327, 82, 273, 81, 291, 105, 173, 232, 113, 126, 102, 338, 80, 328, 171, 61, 228, 191, 27, 131, 238, 331, 8, 21, 22, 31, 36, 40, 47] + 31: [248, 265, 68, 221, 0, 19, 159, 36, 233, 324, 26, 26, 116, 115, 278, 326, 250, 70, 95, 261, 268, 87, 330, 227, 349, 209, 155, 58, 170, 283, 360, 338, 348, 102, 269, 106, 28, 236, 111, 152, 232, 40, 245, 368, 81, 57, 65, 176, 222, 190, 41, 284, 314, 288, 217, 303, 16, 84, 49, 285, 153, 12, 237, 20, 14, 177, 43, 91, 371, 194, 198, 107, 306, 226, 210, 139, 367, 251, 9, 168, 99, 113, 381, 127, 293, 193, 44, 344, 124, 35, 242, 319, 359, 257, 94, 199, 151, 13, 165, 148, 37, 149, 347, 350, 320, 331, 68, 241, 100, 0, 365, 159, 166, 324, 158, 26, 143, 133, 239, 144, 363, 147, 70, 185, 339, 87, 39, 96, 325, 22, 60, 61, 171, 75, 72, 355, 334, 258, 290, 33, 380, 249, 244, 138, 231, 38, 55, 243, 134, 88, 122, 137, 77, 202, 256, 374, 213, 118, 123, 302, 141, 8, 29, 110, 260, 273, 11, 305, 376, 266, 275, 351, 204, 264, 304, 238, 263, 375, 240, 101, 259, 157, 315, 337, 234, 327, 191, 216, 7, 235, 274, 282, 255, 114, 18, 179, 30, 161, 86, 230, 104, 154, 67, 341, 90, 289, 229, 329, 125, 128, 201, 192, 0, 0, 3, 159, 279, 166, 252, 379, 292, 115, 239, 335, 69, 215, 187, 185, 219, 150, 328, 227, 146, 186, 119, 188, 378, 253, 299, 56, 276, 286, 189, 296, 78, 82, 353, 136, 74, 17, 98, 2, 366, 169, 182, 105, 31, 318, 316, 225, 307, 373, 310, 336, 92, 142, 195, 361, 4, 51, 362, 271, 6, 272, 346, 352, 228, 48, 224, 50, 47, 162, 85, 309, 160, 178, 340, 342, 300, 64, 356, 52, 53, 79, 382, 126, 295, 184, 167, 301, 332, 218, 163, 358, 10, 200, 203, 89, 212, 294, 120, 132, 128, 128, 201, 32, 100, 0, 159, 233, 233, 158, 252, 121, 143, 364, 278, 144, 363, 25, 261, 270, 267, 317, 76, 76, 173, 140, 206, 333, 34, 208, 164, 62, 131, 214, 112, 223, 145, 211, 254, 93, 63, 23, 54, 66, 175, 277, 205, 322, 46, 15, 287, 97, 312, 21, 383, 369, 45, 291, 73, 354, 196, 183, 108, 247, 59, 323, 83, 220, 1, 130, 281, 103, 80, 308, 42, 345, 129, 298, 24, 262, 321, 135, 109, 280, 156, 181, 197, 372, 370, 313, 27, 174, 357, 311, 377, 297, 5, 117, 343, 71, 180, 246, 172, 207] + 32: [94, 63, 288, 352, 246, 234, 367, 146, 106, 92, 285, 190, 229, 370, 51, 360, 12, 257, 327, 107, 342, 166, 322, 16, 204, 319, 202, 263, 151, 211, 87, 331, 273, 113, 252, 289, 189, 168, 254, 212, 85, 354, 185, 50, 206, 28, 157, 239, 45, 230, 382, 286, 183, 240, 176, 3, 80, 276, 24, 309, 37, 378, 139, 236, 154, 144, 136, 282, 102, 191, 165, 103, 195, 376, 13, 222, 374, 330, 201, 0, 52, 138, 343, 377, 1, 31, 137, 216, 368, 242, 88, 47, 192, 295, 11, 226, 15, 54, 162, 57, 167, 181, 244, 245, 94, 63, 148, 148, 155, 346, 367, 367, 209, 92, 285, 241, 310, 200, 8, 66, 122, 327, 38, 164, 344, 329, 275, 275, 380, 65, 225, 75, 279, 97, 143, 233, 109, 232, 120, 58, 345, 369, 298, 227, 303, 111, 199, 101, 25, 332, 269, 48, 150, 237, 142, 127, 29, 350, 371, 84, 4, 205, 6, 10, 340, 77, 193, 372, 339, 110, 82, 221, 99, 307, 152, 287, 197, 105, 365, 73, 356, 261, 160, 119, 355, 170, 278, 334, 364, 133, 46, 180, 321, 215, 74, 217, 318, 291, 194, 267, 326, 114, 316, 147, 14, 123, 169, 177, 63, 63, 218, 352, 33, 135, 367, 108, 187, 179, 347, 241, 130, 21, 51, 280, 2, 274, 141, 107, 41, 166, 322, 40, 186, 348, 96, 83, 126, 78, 258, 116, 308, 19, 357, 306, 301, 266, 297, 72, 55, 71, 69, 231, 314, 64, 281, 250, 59, 253, 256, 32, 53, 249, 81, 115, 30, 337, 315, 366, 56, 62, 349, 363, 172, 320, 210, 361, 132, 43, 299, 125, 238, 248, 117, 223, 264, 270, 305, 67, 79, 149, 208, 220, 224, 60, 277, 259, 95, 18, 358, 112, 272, 290, 338, 328, 333, 145, 379, 173, 251, 262, 265, 296, 63, 63, 218, 352, 33, 268, 367, 49, 362, 323, 347, 190, 271, 163, 163, 122, 121, 219, 38, 164, 344, 329, 61, 214, 359, 89, 174, 184, 283, 140, 156, 131, 335, 228, 294, 7, 313, 171, 91, 325, 36, 104, 292, 26, 175, 213, 324, 178, 86, 182, 293, 336, 351, 39, 353, 70, 302, 129, 247, 381, 22, 198, 158, 317, 311, 5, 34, 42, 9, 284, 243, 124, 304, 341, 128, 20, 153, 134, 35, 44, 196, 203, 188, 255, 118, 100, 23, 76, 260, 161, 17, 98, 159, 207, 93, 235, 27, 90, 300, 68, 383, 312, 373, 375] + 33: [86, 240, 145, 63, 177, 339, 38, 362, 362, 121, 380, 36, 231, 319, 297, 343, 251, 301, 283, 154, 307, 68, 348, 358, 322, 89, 226, 37, 127, 335, 233, 17, 35, 25, 338, 117, 192, 40, 289, 217, 14, 126, 354, 202, 93, 355, 230, 4, 123, 198, 80, 246, 156, 50, 104, 200, 88, 279, 234, 364, 368, 345, 106, 71, 350, 349, 52, 70, 101, 110, 29, 219, 221, 135, 182, 238, 181, 11, 243, 27, 291, 310, 120, 62, 178, 353, 337, 199, 97, 275, 374, 269, 44, 344, 294, 203, 357, 282, 159, 189, 236, 102, 333, 372, 167, 51, 75, 260, 177, 331, 38, 377, 153, 64, 64, 229, 325, 379, 81, 107, 251, 258, 283, 299, 267, 179, 147, 241, 201, 89, 352, 266, 169, 190, 21, 54, 160, 87, 150, 227, 356, 334, 133, 9, 235, 116, 290, 351, 205, 173, 16, 254, 45, 347, 22, 262, 19, 157, 370, 163, 180, 152, 60, 58, 122, 249, 330, 371, 113, 28, 56, 26, 53, 285, 41, 327, 318, 128, 20, 23, 74, 130, 85, 381, 188, 224, 257, 118, 247, 175, 242, 363, 214, 171, 274, 359, 309, 172, 2, 46, 100, 223, 0, 265, 12, 34, 220, 277, 91, 91, 176, 320, 339, 339, 191, 38, 31, 323, 229, 36, 148, 319, 96, 343, 258, 183, 299, 82, 342, 267, 314, 94, 108, 72, 139, 5, 361, 270, 376, 141, 1, 33, 151, 193, 237, 131, 222, 311, 8, 305, 144, 211, 187, 149, 138, 317, 383, 276, 196, 375, 382, 55, 367, 373, 114, 369, 340, 213, 49, 137, 292, 215, 24, 281, 65, 166, 278, 324, 248, 295, 252, 194, 298, 346, 300, 207, 132, 225, 78, 119, 245, 365, 244, 59, 341, 378, 195, 306, 7, 228, 99, 57, 304, 302, 77, 143, 272, 264, 161, 32, 69, 142, 91, 328, 140, 260, 259, 280, 38, 38, 42, 158, 380, 36, 148, 319, 81, 115, 115, 39, 301, 154, 307, 73, 125, 94, 146, 84, 164, 206, 48, 255, 273, 204, 218, 30, 210, 155, 208, 308, 268, 288, 61, 18, 232, 303, 329, 90, 109, 174, 216, 79, 124, 162, 256, 197, 209, 360, 186, 212, 13, 105, 170, 15, 43, 3, 336, 66, 366, 103, 271, 112, 261, 67, 136, 10, 6, 316, 111, 286, 168, 76, 184, 332, 312, 98, 134, 326, 315, 165, 92, 250, 253, 313, 321, 296, 284, 47, 185, 263, 239, 129, 83, 95, 287, 293] + 34: [372, 108, 69, 207, 259, 357, 179, 126, 330, 16, 334, 82, 239, 15, 340, 218, 3, 145, 240, 190, 148, 61, 40, 315, 246, 284, 295, 271, 288, 11, 41, 4, 244, 162, 21, 373, 144, 53, 210, 345, 175, 183, 197, 37, 172, 280, 317, 328, 236, 30, 48, 214, 333, 364, 116, 50, 301, 215, 180, 164, 157, 42, 62, 283, 56, 46, 272, 51, 296, 152, 268, 289, 233, 36, 262, 226, 34, 331, 28, 49, 52, 205, 173, 26, 297, 104, 112, 257, 281, 43, 156, 325, 338, 252, 234, 350, 121, 375, 177, 198, 279, 14, 76, 119, 382, 108, 337, 47, 151, 299, 179, 18, 330, 16, 249, 306, 306, 340, 354, 248, 44, 25, 321, 190, 362, 167, 138, 258, 347, 59, 79, 83, 254, 92, 267, 65, 87, 225, 124, 153, 329, 356, 174, 302, 159, 123, 24, 199, 133, 251, 195, 245, 346, 227, 27, 209, 204, 102, 89, 5, 230, 361, 352, 63, 114, 318, 127, 358, 149, 105, 383, 264, 85, 88, 314, 374, 282, 161, 86, 101, 286, 100, 255, 109, 57, 242, 250, 39, 155, 9, 171, 22, 128, 355, 130, 216, 188, 38, 94, 320, 7, 309, 377, 224, 381, 229, 363, 365, 366, 108, 29, 237, 259, 299, 332, 312, 135, 16, 249, 306, 311, 15, 304, 248, 44, 223, 293, 196, 203, 61, 178, 360, 220, 115, 97, 238, 222, 191, 300, 168, 95, 208, 134, 110, 58, 184, 12, 326, 256, 327, 136, 137, 206, 217, 33, 67, 241, 150, 231, 0, 359, 169, 247, 35, 232, 348, 154, 140, 19, 294, 10, 298, 139, 23, 261, 45, 266, 98, 192, 219, 228, 305, 265, 118, 353, 307, 380, 165, 176, 370, 17, 285, 253, 339, 90, 1, 54, 351, 96, 163, 182, 378, 276, 376, 235, 66, 343, 213, 341, 189, 193, 202, 366, 108, 337, 69, 324, 179, 179, 126, 74, 16, 249, 334, 158, 15, 340, 218, 3, 185, 243, 196, 31, 291, 194, 146, 379, 99, 2, 8, 335, 78, 6, 77, 125, 201, 91, 200, 107, 290, 81, 84, 313, 368, 166, 187, 60, 143, 336, 70, 181, 270, 72, 73, 287, 170, 367, 263, 308, 80, 103, 113, 322, 323, 277, 371, 221, 142, 186, 147, 64, 68, 275, 13, 269, 111, 310, 274, 93, 75, 319, 117, 131, 278, 260, 55, 342, 141, 20, 212, 71, 120, 292, 344, 349, 106, 303, 369, 129, 32, 273, 211, 316, 122, 132, 160] + 35: [48, 353, 220, 169, 28, 240, 319, 319, 299, 379, 211, 297, 345, 367, 318, 68, 141, 257, 153, 86, 276, 31, 289, 281, 116, 206, 239, 15, 46, 114, 234, 335, 361, 266, 54, 159, 312, 134, 160, 168, 356, 81, 230, 171, 282, 104, 190, 334, 209, 320, 201, 300, 11, 16, 172, 92, 109, 145, 202, 348, 82, 339, 216, 290, 99, 179, 70, 196, 279, 315, 84, 374, 79, 274, 332, 151, 259, 29, 316, 278, 181, 306, 250, 354, 255, 358, 150, 238, 69, 83, 333, 123, 193, 208, 51, 119, 71, 76, 124, 283, 291, 284, 40, 293, 309, 75, 45, 260, 182, 205, 144, 319, 218, 303, 131, 56, 368, 367, 314, 133, 53, 138, 4, 342, 203, 72, 322, 281, 346, 340, 78, 186, 375, 265, 194, 164, 191, 331, 136, 37, 275, 102, 357, 360, 296, 18, 132, 61, 5, 328, 215, 372, 125, 143, 95, 122, 302, 96, 235, 350, 277, 227, 180, 91, 264, 382, 383, 24, 174, 185, 127, 142, 175, 217, 25, 338, 103, 65, 292, 87, 189, 253, 149, 245, 94, 249, 195, 262, 165, 263, 139, 107, 271, 327, 310, 371, 252, 336, 0, 39, 341, 229, 2, 64, 146, 381, 60, 228, 363, 80, 45, 28, 44, 205, 237, 58, 218, 1, 1, 131, 345, 67, 68, 133, 53, 324, 129, 86, 276, 38, 113, 113, 231, 340, 224, 147, 352, 349, 326, 241, 162, 14, 126, 27, 98, 163, 321, 140, 244, 272, 308, 36, 93, 298, 294, 30, 137, 243, 152, 90, 256, 323, 261, 57, 8, 366, 110, 214, 369, 204, 34, 166, 376, 269, 380, 17, 66, 73, 88, 378, 176, 317, 365, 248, 161, 156, 370, 22, 232, 121, 23, 49, 77, 97, 183, 148, 197, 270, 223, 233, 188, 242, 210, 226, 173, 212, 59, 199, 85, 200, 13, 343, 363, 75, 45, 169, 246, 62, 130, 319, 299, 254, 1, 297, 345, 295, 295, 286, 55, 47, 21, 342, 203, 72, 322, 3, 221, 154, 355, 41, 364, 311, 307, 50, 362, 52, 222, 329, 157, 373, 187, 267, 325, 128, 9, 158, 192, 268, 213, 108, 42, 313, 337, 12, 32, 170, 287, 74, 20, 111, 305, 106, 117, 347, 330, 273, 7, 184, 19, 301, 285, 43, 344, 6, 35, 198, 105, 63, 135, 207, 377, 115, 112, 26, 155, 89, 288, 167, 10, 101, 178, 351, 225, 33, 219, 258, 247, 280, 251, 120, 236, 100, 118, 304, 359, 177] + 36: [53, 22, 370, 340, 341, 144, 250, 304, 238, 266, 10, 2, 325, 368, 272, 230, 196, 296, 320, 40, 56, 12, 99, 107, 97, 73, 133, 205, 332, 233, 261, 65, 162, 240, 112, 111, 194, 231, 345, 55, 13, 122, 105, 95, 298, 69, 183, 339, 346, 142, 117, 174, 11, 100, 116, 364, 211, 255, 30, 200, 212, 81, 36, 17, 165, 358, 23, 80, 159, 208, 29, 356, 363, 209, 295, 158, 169, 236, 173, 19, 337, 35, 21, 14, 226, 312, 213, 126, 26, 154, 1, 128, 227, 89, 286, 306, 377, 182, 114, 270, 292, 313, 302, 360, 53, 22, 382, 74, 341, 184, 304, 232, 232, 185, 10, 371, 325, 328, 272, 101, 98, 178, 241, 369, 372, 372, 301, 107, 207, 352, 79, 5, 125, 31, 32, 331, 333, 321, 262, 362, 9, 319, 191, 349, 361, 297, 284, 121, 172, 322, 289, 20, 72, 168, 280, 179, 41, 237, 202, 102, 134, 305, 71, 0, 104, 143, 379, 206, 88, 188, 149, 119, 163, 283, 48, 58, 281, 70, 190, 109, 265, 315, 198, 249, 357, 60, 307, 120, 61, 324, 253, 235, 161, 303, 285, 203, 327, 141, 4, 218, 160, 221, 43, 376, 16, 25, 336, 375, 131, 294, 22, 176, 184, 84, 57, 348, 232, 266, 62, 371, 368, 368, 272, 299, 258, 242, 326, 274, 225, 380, 90, 223, 374, 150, 308, 47, 287, 314, 42, 45, 33, 201, 108, 246, 219, 166, 49, 139, 138, 63, 103, 245, 164, 3, 342, 15, 153, 354, 140, 271, 228, 195, 224, 277, 310, 217, 229, 39, 52, 257, 28, 335, 148, 351, 267, 220, 318, 51, 18, 135, 329, 83, 247, 6, 259, 383, 248, 199, 130, 316, 75, 118, 110, 350, 8, 87, 7, 278, 96, 27, 124, 82, 269, 222, 214, 234, 268, 186, 290, 127, 170, 273, 192, 252, 197, 340, 184, 144, 57, 232, 232, 238, 359, 59, 371, 311, 311, 260, 210, 178, 193, 193, 225, 12, 373, 223, 275, 291, 152, 92, 54, 85, 93, 156, 175, 309, 334, 67, 288, 94, 293, 282, 167, 132, 68, 91, 50, 300, 276, 155, 171, 216, 366, 147, 343, 66, 177, 46, 323, 180, 187, 264, 76, 38, 344, 157, 244, 129, 263, 78, 243, 365, 146, 239, 44, 24, 204, 189, 338, 381, 77, 251, 256, 215, 347, 137, 254, 115, 355, 378, 86, 151, 34, 330, 136, 353, 106, 181, 367, 317, 113, 145, 279, 64, 37, 123] + 37: [109, 268, 364, 40, 294, 218, 218, 379, 158, 158, 311, 314, 178, 205, 199, 363, 289, 48, 179, 72, 349, 234, 371, 340, 0, 156, 140, 129, 360, 7, 182, 318, 204, 117, 343, 32, 257, 277, 150, 315, 127, 316, 122, 21, 270, 1, 168, 43, 217, 338, 36, 161, 278, 103, 201, 34, 126, 136, 224, 334, 37, 262, 244, 255, 313, 102, 279, 274, 4, 5, 351, 26, 160, 254, 10, 53, 303, 332, 137, 378, 35, 101, 374, 142, 357, 184, 231, 214, 227, 119, 261, 54, 369, 89, 90, 246, 358, 25, 238, 99, 31, 120, 8, 75, 109, 319, 301, 66, 294, 218, 218, 123, 243, 348, 148, 148, 306, 52, 222, 366, 341, 190, 305, 290, 292, 162, 382, 77, 361, 0, 171, 355, 245, 252, 88, 65, 147, 235, 163, 131, 330, 345, 221, 157, 370, 229, 228, 51, 27, 64, 6, 323, 82, 73, 39, 304, 333, 135, 29, 15, 100, 167, 83, 247, 337, 269, 347, 98, 195, 212, 284, 280, 42, 210, 116, 249, 13, 110, 368, 266, 216, 350, 169, 115, 295, 339, 198, 166, 62, 106, 57, 107, 281, 155, 20, 300, 159, 250, 381, 275, 23, 50, 276, 209, 189, 226, 112, 132, 188, 268, 71, 40, 40, 218, 218, 379, 58, 242, 309, 151, 125, 293, 222, 200, 362, 372, 342, 79, 193, 95, 113, 113, 223, 59, 344, 47, 185, 353, 22, 14, 328, 233, 181, 118, 18, 164, 145, 322, 19, 320, 76, 258, 176, 196, 144, 365, 61, 41, 141, 92, 264, 134, 285, 376, 356, 49, 55, 331, 80, 207, 321, 312, 94, 241, 24, 317, 325, 267, 187, 298, 299, 124, 91, 152, 60, 260, 175, 183, 297, 259, 81, 308, 96, 225, 324, 70, 202, 28, 197, 11, 220, 186, 2, 12, 282, 68, 377, 251, 56, 375, 236, 239, 240, 268, 301, 40, 40, 294, 218, 273, 58, 265, 148, 213, 293, 293, 46, 200, 366, 272, 44, 79, 349, 162, 174, 340, 86, 16, 114, 302, 253, 9, 165, 327, 139, 194, 93, 153, 133, 192, 359, 336, 208, 85, 287, 104, 173, 74, 149, 203, 230, 296, 143, 30, 108, 111, 329, 3, 87, 335, 206, 69, 121, 128, 380, 373, 97, 271, 38, 33, 105, 354, 154, 172, 138, 256, 17, 215, 170, 286, 78, 291, 310, 283, 191, 63, 248, 146, 130, 237, 326, 177, 180, 45, 219, 307, 367, 211, 346, 232, 67, 84, 263, 288, 352, 383] + 38: [282, 6, 93, 301, 378, 369, 345, 266, 203, 203, 269, 124, 284, 51, 30, 366, 226, 383, 114, 248, 333, 333, 62, 230, 171, 244, 27, 166, 328, 344, 77, 257, 53, 338, 286, 288, 341, 355, 275, 146, 227, 67, 72, 95, 362, 359, 45, 211, 57, 349, 136, 156, 368, 111, 143, 151, 5, 289, 306, 28, 76, 134, 33, 174, 15, 235, 250, 337, 334, 182, 195, 43, 56, 347, 231, 204, 311, 61, 374, 13, 55, 247, 3, 357, 240, 113, 335, 42, 259, 175, 20, 116, 281, 377, 196, 155, 249, 243, 346, 159, 339, 101, 185, 237, 10, 6, 223, 301, 301, 178, 348, 245, 66, 332, 280, 201, 372, 8, 263, 366, 189, 126, 115, 277, 277, 220, 32, 39, 379, 217, 145, 0, 212, 206, 54, 267, 108, 199, 329, 233, 106, 90, 222, 107, 271, 85, 21, 9, 193, 262, 144, 87, 255, 256, 308, 367, 34, 298, 188, 342, 228, 148, 241, 84, 197, 41, 149, 258, 103, 40, 2, 173, 302, 214, 122, 74, 273, 11, 100, 252, 47, 297, 186, 283, 138, 290, 363, 70, 316, 320, 112, 24, 163, 18, 207, 128, 22, 315, 83, 336, 330, 270, 304, 64, 161, 38, 210, 380, 4, 187, 6, 301, 213, 378, 221, 266, 118, 123, 280, 124, 372, 51, 86, 285, 322, 274, 383, 154, 194, 89, 177, 62, 63, 92, 381, 278, 317, 135, 119, 253, 97, 319, 327, 361, 125, 219, 75, 147, 239, 73, 29, 59, 224, 105, 36, 68, 323, 264, 165, 96, 179, 37, 229, 276, 265, 31, 260, 356, 376, 242, 181, 305, 131, 78, 81, 200, 238, 360, 172, 371, 261, 104, 82, 25, 295, 80, 168, 351, 183, 202, 299, 236, 164, 91, 326, 364, 314, 232, 209, 153, 218, 365, 110, 127, 162, 133, 318, 98, 176, 52, 46, 60, 158, 158, 6, 301, 287, 50, 221, 266, 94, 203, 269, 201, 284, 292, 30, 354, 189, 126, 114, 154, 309, 89, 49, 293, 130, 19, 17, 373, 279, 169, 268, 300, 234, 331, 16, 137, 225, 99, 352, 14, 324, 69, 208, 321, 150, 358, 307, 58, 88, 184, 216, 142, 7, 48, 109, 246, 117, 141, 190, 170, 26, 180, 272, 353, 139, 152, 310, 23, 296, 79, 313, 129, 382, 120, 375, 215, 44, 294, 205, 35, 71, 102, 343, 12, 192, 254, 65, 157, 350, 291, 132, 167, 303, 160, 198, 1, 140, 370, 121, 312, 340, 191, 251, 325] + 39: [43, 282, 96, 268, 264, 110, 104, 82, 223, 236, 1, 153, 56, 113, 312, 215, 93, 360, 325, 242, 155, 328, 86, 244, 174, 362, 71, 172, 295, 330, 24, 319, 77, 49, 23, 381, 298, 84, 8, 259, 191, 48, 323, 122, 278, 190, 137, 260, 126, 358, 135, 188, 144, 307, 161, 249, 14, 373, 355, 106, 167, 320, 173, 354, 275, 6, 79, 309, 68, 377, 194, 214, 376, 291, 109, 217, 288, 19, 303, 107, 337, 108, 20, 246, 0, 252, 253, 238, 233, 339, 136, 158, 67, 205, 12, 316, 356, 184, 11, 90, 334, 89, 143, 198, 100, 99, 282, 35, 66, 258, 104, 310, 348, 372, 236, 156, 151, 315, 65, 139, 93, 132, 314, 313, 271, 272, 85, 85, 80, 270, 277, 374, 40, 17, 15, 76, 208, 7, 81, 200, 227, 202, 53, 42, 140, 232, 28, 31, 263, 69, 74, 131, 152, 50, 276, 371, 5, 63, 75, 226, 134, 182, 61, 94, 289, 241, 193, 357, 4, 338, 243, 130, 220, 306, 169, 181, 120, 171, 343, 27, 335, 103, 222, 111, 55, 266, 83, 160, 211, 64, 197, 229, 342, 210, 129, 349, 125, 32, 105, 73, 332, 261, 115, 148, 21, 52, 257, 44, 176, 247, 282, 180, 66, 116, 234, 310, 372, 372, 368, 286, 56, 315, 65, 13, 38, 36, 36, 313, 267, 187, 382, 292, 329, 297, 324, 26, 98, 141, 341, 369, 3, 365, 87, 290, 273, 118, 383, 274, 127, 54, 299, 225, 239, 138, 331, 378, 367, 62, 199, 366, 322, 179, 213, 345, 33, 212, 72, 293, 45, 88, 97, 203, 128, 164, 16, 254, 240, 114, 207, 265, 91, 216, 245, 192, 287, 41, 168, 159, 78, 305, 347, 175, 18, 123, 92, 359, 219, 255, 147, 350, 256, 124, 224, 308, 57, 154, 186, 380, 304, 165, 361, 209, 247, 247, 282, 280, 66, 258, 104, 310, 310, 236, 25, 286, 56, 315, 312, 218, 38, 132, 314, 279, 119, 379, 379, 2, 145, 296, 204, 333, 121, 250, 300, 178, 163, 51, 59, 146, 235, 375, 170, 352, 248, 340, 251, 112, 336, 101, 34, 47, 353, 177, 150, 281, 317, 364, 162, 301, 302, 228, 102, 311, 95, 185, 133, 285, 230, 221, 30, 195, 363, 183, 327, 60, 166, 283, 58, 70, 196, 346, 189, 117, 46, 157, 206, 231, 142, 37, 10, 39, 284, 326, 351, 370, 294, 318, 237, 262, 269, 344, 201, 29, 321, 9, 149, 22] + 40: [31, 31, 357, 319, 319, 260, 250, 360, 70, 213, 328, 358, 175, 67, 367, 190, 21, 179, 179, 26, 215, 346, 8, 158, 369, 74, 251, 224, 59, 228, 347, 275, 362, 143, 168, 219, 274, 322, 289, 166, 30, 308, 216, 221, 382, 229, 46, 307, 316, 41, 263, 138, 244, 377, 306, 355, 155, 192, 339, 167, 371, 177, 323, 195, 37, 293, 301, 345, 73, 261, 36, 147, 270, 186, 256, 288, 125, 135, 310, 32, 140, 174, 230, 77, 35, 106, 111, 173, 352, 313, 39, 380, 157, 245, 298, 320, 191, 197, 146, 240, 144, 376, 149, 300, 31, 31, 207, 280, 183, 15, 286, 360, 142, 68, 359, 71, 175, 378, 373, 190, 21, 108, 52, 160, 202, 156, 226, 88, 64, 372, 235, 3, 97, 28, 159, 1, 303, 129, 10, 271, 209, 277, 185, 11, 193, 200, 38, 205, 148, 302, 137, 259, 309, 365, 150, 178, 65, 25, 130, 103, 294, 338, 327, 318, 356, 282, 281, 171, 19, 131, 60, 233, 139, 76, 246, 118, 116, 121, 14, 84, 348, 124, 268, 13, 16, 117, 203, 163, 242, 104, 285, 325, 154, 267, 4, 315, 45, 223, 292, 272, 127, 126, 335, 344, 5, 343, 123, 248, 31, 132, 317, 340, 183, 15, 349, 349, 142, 68, 328, 358, 55, 378, 75, 101, 265, 108, 33, 33, 107, 279, 161, 153, 254, 81, 58, 258, 351, 100, 210, 311, 284, 296, 136, 326, 341, 80, 61, 27, 54, 334, 12, 188, 370, 262, 269, 44, 187, 99, 95, 57, 189, 299, 151, 2, 165, 120, 287, 93, 333, 329, 243, 237, 273, 350, 314, 366, 85, 201, 238, 375, 78, 115, 141, 361, 23, 102, 266, 50, 374, 40, 113, 257, 218, 217, 22, 119, 114, 234, 342, 109, 381, 255, 86, 172, 20, 83, 225, 331, 353, 249, 105, 214, 31, 379, 317, 207, 319, 15, 250, 169, 181, 68, 204, 71, 175, 378, 98, 62, 283, 337, 247, 297, 128, 346, 226, 164, 232, 332, 208, 198, 90, 17, 91, 9, 363, 89, 336, 321, 162, 63, 253, 211, 145, 278, 236, 199, 196, 227, 239, 56, 79, 53, 364, 82, 43, 290, 110, 94, 7, 29, 264, 324, 206, 305, 330, 48, 304, 18, 47, 51, 222, 241, 6, 291, 122, 133, 212, 276, 0, 184, 383, 49, 368, 182, 66, 295, 34, 134, 231, 180, 354, 96, 152, 87, 176, 252, 24, 112, 220, 42, 72, 312, 69, 170, 92, 194] + 41: [205, 49, 298, 298, 367, 114, 114, 142, 27, 170, 60, 350, 212, 90, 190, 86, 369, 366, 366, 223, 159, 124, 215, 285, 187, 111, 21, 360, 347, 311, 198, 337, 104, 45, 3, 53, 42, 25, 23, 163, 308, 26, 64, 70, 54, 226, 256, 83, 294, 235, 325, 138, 339, 133, 210, 8, 63, 151, 295, 290, 81, 277, 261, 43, 173, 315, 146, 192, 15, 232, 300, 33, 143, 78, 287, 165, 259, 207, 38, 68, 98, 310, 182, 28, 265, 171, 245, 373, 201, 121, 343, 88, 177, 162, 109, 356, 304, 281, 164, 233, 22, 178, 152, 132, 305, 49, 269, 283, 335, 103, 166, 199, 376, 170, 60, 350, 374, 184, 190, 30, 369, 168, 365, 129, 275, 119, 293, 96, 7, 34, 69, 161, 280, 12, 108, 306, 348, 9, 4, 324, 264, 140, 36, 243, 217, 134, 32, 102, 125, 379, 254, 276, 156, 271, 167, 115, 144, 112, 266, 328, 363, 66, 107, 153, 18, 6, 130, 213, 244, 10, 221, 214, 41, 99, 75, 272, 352, 179, 0, 65, 141, 320, 332, 158, 147, 113, 195, 209, 14, 67, 329, 20, 139, 331, 157, 240, 211, 377, 372, 16, 58, 203, 135, 228, 227, 296, 278, 357, 122, 185, 269, 283, 378, 196, 71, 199, 251, 355, 60, 307, 374, 184, 150, 57, 197, 84, 200, 368, 159, 327, 97, 92, 79, 172, 24, 279, 206, 334, 95, 82, 383, 318, 351, 35, 239, 219, 194, 218, 336, 174, 116, 345, 241, 282, 46, 260, 258, 322, 117, 289, 250, 224, 380, 51, 131, 176, 186, 338, 91, 246, 94, 5, 371, 74, 128, 303, 59, 123, 342, 188, 236, 100, 381, 180, 297, 234, 105, 48, 257, 73, 175, 204, 375, 231, 274, 17, 62, 267, 2, 145, 189, 312, 346, 361, 169, 193, 118, 333, 40, 247, 216, 309, 321, 321, 49, 87, 378, 378, 11, 199, 142, 170, 355, 350, 374, 249, 255, 30, 273, 197, 50, 129, 275, 119, 215, 76, 136, 364, 358, 314, 302, 262, 44, 253, 80, 326, 127, 222, 286, 202, 353, 126, 252, 349, 301, 56, 313, 225, 93, 55, 237, 229, 340, 344, 242, 330, 288, 268, 299, 101, 181, 284, 19, 208, 160, 317, 61, 220, 89, 354, 37, 359, 154, 292, 1, 39, 191, 77, 72, 13, 248, 149, 316, 148, 155, 137, 29, 238, 370, 47, 183, 110, 323, 319, 270, 106, 341, 362, 382, 291, 230, 52, 263, 31, 120, 85] + 42: [2, 20, 1, 123, 123, 94, 114, 152, 31, 117, 106, 303, 172, 155, 247, 93, 215, 282, 111, 158, 138, 37, 222, 349, 200, 377, 48, 209, 281, 319, 351, 277, 358, 248, 4, 341, 135, 239, 187, 287, 165, 201, 338, 36, 181, 38, 143, 366, 362, 76, 149, 381, 245, 18, 101, 137, 311, 372, 246, 89, 260, 130, 97, 226, 205, 7, 196, 332, 156, 170, 300, 346, 293, 84, 175, 284, 267, 321, 10, 363, 185, 292, 186, 136, 330, 302, 85, 133, 115, 276, 70, 79, 83, 81, 78, 142, 13, 240, 361, 289, 27, 121, 25, 71, 188, 359, 217, 123, 123, 94, 228, 152, 31, 117, 357, 305, 172, 211, 218, 350, 151, 63, 264, 256, 138, 179, 168, 337, 220, 129, 263, 255, 189, 21, 373, 379, 131, 370, 320, 178, 140, 236, 344, 230, 87, 161, 147, 216, 322, 314, 234, 45, 118, 39, 160, 192, 80, 60, 19, 146, 231, 371, 225, 238, 283, 139, 183, 310, 298, 258, 5, 291, 53, 270, 171, 233, 194, 343, 290, 229, 102, 98, 250, 73, 128, 296, 198, 304, 144, 383, 88, 153, 353, 12, 47, 57, 107, 159, 177, 317, 15, 365, 24, 40, 352, 294, 95, 254, 210, 359, 217, 123, 123, 266, 295, 152, 119, 117, 268, 345, 176, 184, 116, 218, 93, 325, 111, 158, 333, 41, 243, 195, 55, 108, 59, 203, 327, 122, 265, 348, 275, 141, 30, 29, 224, 169, 67, 214, 105, 173, 103, 308, 335, 77, 202, 316, 223, 257, 227, 380, 32, 241, 51, 8, 237, 180, 64, 9, 315, 109, 157, 299, 58, 313, 16, 204, 378, 125, 269, 112, 26, 56, 272, 206, 199, 197, 368, 104, 312, 3, 23, 334, 207, 367, 22, 134, 69, 190, 11, 110, 44, 54, 14, 43, 0, 91, 163, 273, 126, 242, 376, 339, 154, 375, 50, 42, 123, 266, 148, 152, 31, 117, 268, 345, 176, 155, 212, 279, 325, 72, 150, 256, 333, 37, 374, 162, 262, 306, 6, 208, 342, 127, 191, 235, 328, 274, 354, 221, 259, 286, 28, 17, 167, 329, 326, 356, 92, 164, 280, 33, 52, 166, 65, 297, 113, 340, 132, 301, 62, 307, 219, 323, 49, 244, 99, 232, 34, 61, 336, 251, 355, 68, 278, 364, 120, 213, 261, 318, 82, 90, 145, 360, 285, 124, 75, 288, 174, 309, 347, 96, 253, 35, 249, 100, 331, 182, 271, 382, 86, 66, 74, 252, 46, 193, 369, 324] + 43: [164, 82, 168, 221, 171, 156, 225, 375, 278, 92, 335, 197, 35, 98, 59, 324, 14, 20, 190, 239, 30, 103, 298, 351, 57, 326, 93, 81, 223, 301, 360, 227, 198, 307, 136, 121, 89, 368, 191, 32, 205, 52, 246, 48, 83, 277, 140, 135, 296, 261, 257, 79, 28, 155, 243, 148, 102, 185, 229, 183, 143, 16, 70, 213, 220, 309, 343, 226, 18, 31, 292, 11, 356, 8, 65, 126, 193, 381, 268, 209, 188, 265, 316, 258, 162, 312, 248, 199, 41, 0, 137, 303, 208, 352, 58, 139, 244, 319, 231, 310, 195, 373, 95, 354, 369, 262, 168, 161, 171, 332, 347, 225, 68, 372, 17, 142, 66, 112, 98, 37, 349, 283, 254, 252, 30, 94, 43, 351, 23, 235, 377, 186, 111, 127, 323, 313, 108, 274, 269, 321, 85, 374, 150, 2, 238, 90, 346, 204, 86, 219, 200, 26, 149, 5, 365, 67, 196, 194, 345, 145, 99, 60, 50, 165, 45, 253, 120, 46, 263, 9, 80, 288, 175, 159, 207, 72, 144, 267, 353, 163, 19, 158, 173, 109, 320, 300, 110, 336, 285, 51, 69, 124, 7, 172, 141, 281, 42, 270, 133, 96, 212, 33, 166, 376, 192, 182, 382, 3, 27, 315, 168, 221, 105, 156, 225, 375, 375, 372, 17, 371, 66, 359, 98, 216, 14, 13, 190, 75, 366, 297, 43, 131, 23, 152, 291, 55, 138, 342, 203, 318, 224, 338, 202, 280, 379, 4, 77, 222, 378, 154, 380, 275, 286, 234, 123, 271, 160, 34, 61, 130, 241, 334, 273, 12, 242, 293, 38, 47, 210, 266, 284, 167, 128, 146, 256, 325, 170, 116, 304, 64, 294, 71, 117, 87, 151, 287, 233, 119, 330, 355, 187, 357, 311, 201, 53, 114, 317, 328, 21, 228, 362, 340, 333, 104, 25, 289, 184, 54, 189, 331, 44, 348, 344, 315, 181, 6, 105, 177, 347, 308, 218, 92, 335, 129, 66, 359, 364, 324, 39, 283, 75, 97, 100, 297, 298, 314, 22, 363, 337, 153, 29, 322, 264, 178, 1, 305, 147, 78, 250, 255, 272, 73, 232, 282, 88, 327, 339, 306, 115, 299, 367, 36, 84, 157, 237, 10, 63, 107, 49, 24, 113, 180, 217, 56, 251, 74, 125, 101, 174, 370, 215, 295, 329, 122, 259, 279, 76, 132, 247, 118, 134, 358, 211, 290, 240, 179, 260, 341, 276, 230, 106, 62, 249, 236, 383, 40, 15, 91, 176, 350, 302, 361, 169, 206, 214, 245] + 44: [362, 97, 172, 0, 213, 77, 300, 191, 199, 10, 10, 351, 226, 348, 155, 60, 180, 215, 376, 43, 277, 346, 279, 325, 11, 166, 320, 86, 333, 242, 294, 198, 33, 228, 275, 337, 343, 276, 265, 138, 266, 85, 175, 186, 122, 55, 100, 56, 352, 132, 366, 160, 202, 96, 261, 381, 128, 292, 220, 344, 92, 278, 74, 282, 149, 8, 231, 377, 23, 32, 20, 113, 263, 203, 319, 18, 53, 256, 137, 65, 35, 312, 200, 326, 153, 5, 251, 210, 283, 28, 69, 268, 139, 356, 335, 111, 76, 133, 208, 281, 295, 129, 206, 322, 362, 64, 172, 167, 213, 77, 301, 191, 199, 382, 10, 110, 317, 348, 223, 284, 21, 188, 257, 29, 328, 346, 368, 229, 236, 93, 146, 383, 289, 249, 38, 306, 358, 102, 217, 95, 324, 89, 121, 262, 244, 192, 61, 296, 154, 173, 364, 70, 285, 130, 248, 286, 54, 174, 369, 134, 157, 342, 41, 26, 196, 370, 165, 159, 214, 105, 323, 309, 156, 81, 305, 237, 234, 57, 250, 345, 367, 49, 51, 232, 273, 42, 45, 216, 239, 373, 115, 99, 246, 363, 176, 302, 47, 293, 73, 361, 58, 88, 75, 371, 209, 13, 332, 374, 274, 168, 171, 258, 213, 77, 178, 301, 199, 222, 379, 317, 226, 101, 101, 284, 190, 267, 84, 145, 170, 252, 135, 311, 30, 143, 313, 280, 182, 118, 264, 142, 365, 304, 225, 360, 158, 14, 340, 117, 184, 224, 24, 299, 349, 150, 201, 315, 66, 131, 164, 380, 269, 87, 221, 177, 36, 162, 354, 126, 147, 83, 90, 40, 205, 59, 318, 19, 336, 211, 104, 179, 235, 230, 9, 109, 314, 321, 183, 114, 255, 2, 253, 375, 233, 22, 119, 50, 240, 298, 103, 330, 259, 37, 152, 52, 353, 197, 310, 308, 112, 339, 12, 106, 274, 238, 171, 17, 213, 25, 272, 301, 191, 222, 379, 317, 226, 348, 223, 271, 21, 227, 82, 145, 170, 68, 140, 331, 288, 34, 98, 187, 212, 141, 63, 329, 48, 144, 125, 291, 355, 94, 207, 91, 6, 350, 254, 290, 163, 219, 287, 39, 71, 79, 338, 378, 67, 359, 120, 44, 327, 107, 46, 136, 15, 108, 185, 334, 270, 243, 78, 347, 341, 169, 3, 218, 241, 72, 193, 357, 4, 204, 62, 7, 124, 245, 80, 1, 27, 195, 151, 194, 247, 372, 260, 307, 123, 297, 127, 31, 316, 148, 181, 16, 161, 189, 303, 116] + 45: [154, 173, 219, 140, 188, 92, 110, 293, 25, 334, 178, 182, 8, 27, 171, 247, 33, 369, 262, 103, 276, 180, 131, 21, 165, 124, 348, 349, 209, 183, 275, 228, 239, 168, 196, 16, 115, 30, 338, 250, 84, 313, 7, 267, 41, 242, 157, 260, 26, 315, 236, 24, 302, 253, 222, 370, 76, 65, 34, 170, 290, 252, 296, 264, 283, 273, 333, 231, 263, 177, 288, 314, 105, 0, 289, 312, 147, 3, 215, 93, 136, 238, 332, 189, 18, 89, 186, 232, 270, 194, 295, 148, 13, 6, 28, 184, 130, 49, 69, 72, 137, 336, 266, 221, 154, 121, 121, 140, 134, 92, 161, 293, 42, 178, 201, 10, 10, 27, 176, 227, 227, 350, 367, 373, 354, 47, 329, 61, 163, 212, 114, 383, 107, 248, 351, 118, 198, 223, 45, 122, 229, 104, 245, 308, 12, 112, 306, 357, 213, 249, 44, 179, 292, 67, 109, 2, 261, 303, 340, 43, 58, 330, 116, 126, 87, 319, 59, 5, 98, 294, 379, 255, 151, 342, 374, 79, 166, 363, 48, 203, 364, 113, 343, 202, 225, 337, 90, 102, 62, 31, 82, 167, 159, 193, 352, 216, 259, 77, 99, 325, 149, 240, 181, 17, 244, 361, 300, 347, 324, 191, 318, 140, 316, 92, 92, 284, 42, 334, 201, 346, 8, 120, 356, 227, 375, 350, 160, 373, 378, 329, 298, 164, 95, 81, 297, 128, 224, 234, 331, 39, 187, 307, 55, 158, 51, 155, 88, 70, 54, 327, 251, 80, 360, 108, 281, 217, 106, 175, 208, 96, 4, 22, 63, 218, 20, 35, 133, 230, 304, 145, 291, 38, 37, 29, 322, 241, 169, 381, 271, 204, 246, 341, 15, 309, 78, 368, 377, 71, 243, 214, 321, 125, 280, 56, 152, 226, 206, 305, 265, 53, 278, 335, 75, 237, 60, 235, 299, 40, 355, 192, 83, 174, 23, 23, 301, 233, 144, 92, 161, 317, 139, 199, 178, 197, 10, 274, 141, 339, 33, 367, 146, 85, 276, 286, 269, 164, 95, 81, 383, 272, 52, 380, 258, 254, 359, 310, 162, 326, 36, 277, 74, 372, 328, 68, 210, 135, 345, 86, 156, 94, 32, 279, 365, 1, 123, 376, 257, 50, 344, 19, 207, 205, 46, 256, 358, 320, 150, 9, 282, 211, 57, 127, 129, 285, 142, 220, 119, 268, 371, 190, 362, 73, 91, 111, 323, 132, 153, 382, 97, 143, 353, 200, 101, 366, 311, 287, 172, 66, 11, 195, 185, 14, 117, 138, 64, 100] + 46: [152, 210, 10, 183, 352, 233, 256, 314, 286, 327, 253, 343, 170, 110, 51, 191, 149, 230, 332, 106, 106, 140, 344, 312, 271, 136, 254, 55, 345, 2, 374, 3, 102, 39, 351, 228, 130, 195, 22, 227, 80, 232, 121, 168, 180, 109, 4, 13, 129, 362, 53, 273, 172, 216, 157, 158, 45, 57, 226, 373, 328, 97, 360, 24, 297, 223, 346, 69, 9, 317, 322, 333, 145, 264, 190, 370, 184, 244, 339, 26, 347, 163, 65, 43, 77, 310, 281, 72, 87, 381, 48, 174, 212, 29, 185, 0, 100, 28, 148, 270, 220, 379, 61, 122, 188, 133, 247, 247, 34, 352, 139, 58, 224, 380, 253, 343, 249, 110, 207, 202, 274, 367, 354, 284, 104, 66, 335, 335, 194, 187, 204, 114, 113, 44, 76, 219, 350, 215, 211, 62, 108, 143, 376, 260, 186, 99, 353, 197, 295, 192, 316, 166, 63, 41, 95, 128, 292, 237, 340, 321, 336, 383, 81, 308, 206, 92, 289, 255, 313, 222, 173, 382, 7, 243, 277, 241, 359, 91, 291, 309, 285, 278, 98, 257, 150, 52, 135, 75, 258, 11, 366, 236, 363, 105, 269, 251, 296, 179, 47, 123, 70, 357, 225, 365, 252, 27, 88, 213, 142, 142, 201, 183, 34, 19, 64, 58, 368, 240, 46, 355, 311, 16, 96, 239, 246, 230, 354, 324, 319, 319, 140, 301, 214, 40, 68, 378, 338, 199, 242, 320, 79, 282, 49, 131, 115, 33, 155, 372, 287, 334, 218, 315, 259, 137, 37, 50, 176, 23, 162, 250, 198, 267, 235, 193, 125, 377, 32, 307, 141, 78, 144, 169, 299, 161, 318, 348, 159, 293, 265, 342, 261, 283, 217, 93, 361, 305, 234, 375, 337, 15, 112, 21, 127, 330, 341, 126, 288, 73, 196, 181, 279, 116, 276, 84, 151, 167, 178, 323, 231, 5, 36, 175, 142, 133, 10, 183, 352, 64, 139, 256, 286, 380, 355, 311, 311, 25, 207, 238, 149, 221, 367, 6, 209, 66, 154, 12, 124, 60, 38, 165, 18, 303, 146, 203, 266, 85, 30, 280, 8, 117, 120, 83, 326, 132, 245, 118, 156, 42, 369, 298, 164, 304, 294, 272, 205, 103, 17, 59, 268, 111, 101, 329, 263, 35, 160, 290, 89, 119, 189, 56, 71, 74, 153, 20, 364, 177, 90, 331, 86, 262, 171, 371, 208, 138, 67, 358, 275, 94, 229, 31, 300, 107, 1, 134, 306, 248, 54, 349, 200, 14, 182, 82, 147, 325, 302, 356] + 47: [195, 75, 271, 179, 154, 154, 44, 352, 24, 312, 3, 3, 279, 109, 1, 61, 227, 307, 359, 134, 286, 320, 316, 353, 362, 129, 267, 287, 59, 123, 229, 218, 114, 212, 330, 230, 155, 26, 135, 377, 124, 223, 324, 41, 323, 184, 364, 203, 296, 51, 355, 25, 180, 173, 252, 53, 315, 378, 336, 91, 70, 121, 289, 213, 281, 292, 306, 111, 115, 185, 186, 132, 297, 160, 46, 242, 48, 152, 302, 72, 177, 74, 350, 304, 16, 100, 38, 222, 47, 145, 11, 194, 82, 167, 125, 256, 66, 261, 128, 318, 239, 373, 293, 56, 195, 343, 54, 347, 181, 171, 162, 352, 291, 312, 22, 43, 35, 87, 158, 158, 382, 382, 333, 345, 137, 305, 215, 127, 36, 175, 168, 21, 188, 327, 153, 349, 251, 176, 351, 250, 84, 69, 219, 68, 166, 151, 263, 284, 172, 65, 199, 73, 12, 322, 17, 148, 4, 354, 117, 27, 96, 139, 141, 80, 321, 126, 303, 147, 273, 94, 278, 60, 107, 32, 157, 260, 144, 81, 190, 150, 19, 189, 308, 325, 245, 358, 380, 174, 133, 30, 317, 372, 202, 270, 299, 268, 78, 161, 206, 221, 241, 182, 113, 311, 236, 191, 246, 360, 195, 343, 54, 331, 181, 143, 162, 44, 357, 24, 22, 371, 375, 170, 34, 29, 280, 359, 300, 309, 314, 14, 294, 196, 198, 274, 376, 110, 238, 131, 367, 235, 159, 122, 313, 265, 95, 93, 301, 42, 224, 285, 231, 103, 33, 370, 50, 197, 0, 205, 138, 237, 340, 233, 112, 164, 234, 183, 356, 15, 243, 71, 335, 225, 276, 328, 37, 258, 341, 200, 310, 248, 97, 253, 9, 102, 365, 282, 57, 216, 379, 207, 5, 92, 339, 105, 178, 277, 76, 247, 64, 244, 99, 283, 165, 295, 254, 272, 381, 7, 156, 62, 209, 288, 195, 343, 54, 187, 136, 192, 44, 352, 24, 312, 3, 3, 119, 87, 87, 29, 193, 307, 300, 134, 137, 90, 368, 257, 344, 79, 240, 118, 40, 346, 342, 363, 63, 98, 226, 269, 208, 104, 262, 67, 89, 249, 228, 319, 298, 140, 255, 77, 204, 101, 338, 45, 39, 52, 106, 210, 23, 366, 18, 201, 348, 211, 163, 28, 259, 337, 332, 31, 49, 329, 2, 20, 130, 334, 10, 149, 120, 275, 55, 266, 383, 290, 374, 108, 58, 86, 220, 361, 85, 88, 142, 264, 116, 232, 6, 217, 8, 13, 83, 169, 214, 146, 369, 326] + 48: [237, 51, 283, 266, 17, 271, 102, 145, 140, 328, 330, 273, 239, 99, 132, 343, 203, 284, 182, 101, 58, 141, 85, 78, 62, 15, 287, 6, 344, 235, 359, 363, 186, 342, 134, 252, 155, 290, 210, 300, 161, 175, 192, 50, 120, 247, 88, 340, 329, 341, 212, 24, 168, 172, 42, 231, 307, 22, 80, 351, 119, 96, 302, 81, 198, 269, 367, 40, 206, 346, 107, 137, 108, 82, 272, 173, 366, 214, 69, 373, 224, 37, 87, 278, 244, 350, 361, 225, 298, 362, 282, 125, 46, 183, 262, 176, 220, 265, 355, 336, 136, 304, 114, 250, 267, 288, 76, 20, 17, 271, 102, 117, 150, 328, 109, 358, 239, 276, 19, 236, 52, 311, 353, 295, 164, 381, 381, 78, 62, 207, 104, 375, 5, 139, 48, 254, 142, 115, 181, 305, 320, 371, 229, 312, 322, 7, 25, 257, 370, 112, 190, 0, 160, 255, 321, 357, 315, 347, 79, 13, 379, 202, 113, 281, 72, 289, 97, 11, 84, 43, 349, 16, 151, 292, 94, 274, 261, 39, 187, 47, 217, 92, 67, 313, 27, 163, 294, 105, 110, 309, 188, 191, 8, 338, 60, 106, 296, 248, 23, 383, 103, 337, 199, 258, 30, 245, 196, 227, 267, 9, 283, 249, 17, 128, 26, 77, 279, 140, 328, 215, 116, 276, 234, 44, 52, 311, 2, 101, 133, 141, 85, 28, 15, 263, 166, 189, 177, 240, 38, 264, 34, 316, 184, 354, 319, 378, 222, 156, 130, 246, 299, 152, 285, 75, 45, 73, 89, 118, 242, 111, 4, 318, 91, 360, 180, 339, 241, 14, 138, 146, 36, 253, 211, 280, 12, 53, 55, 144, 35, 259, 243, 178, 71, 286, 95, 194, 93, 3, 56, 143, 277, 335, 66, 228, 356, 148, 154, 86, 345, 153, 124, 251, 54, 122, 303, 364, 131, 348, 10, 200, 129, 308, 159, 297, 76, 266, 271, 128, 270, 65, 301, 140, 330, 273, 226, 132, 132, 121, 74, 284, 353, 90, 33, 268, 21, 83, 157, 263, 369, 147, 223, 327, 208, 372, 325, 185, 380, 41, 232, 368, 205, 219, 193, 179, 293, 291, 233, 174, 123, 324, 162, 170, 31, 221, 230, 275, 218, 57, 317, 64, 100, 70, 49, 158, 204, 63, 374, 352, 18, 127, 98, 169, 59, 195, 382, 216, 256, 32, 135, 310, 326, 29, 165, 323, 213, 61, 201, 238, 376, 149, 171, 331, 377, 260, 365, 332, 68, 126, 306, 197, 333, 167, 1, 209, 334, 314] + 49: [374, 360, 302, 302, 321, 272, 294, 346, 15, 209, 297, 305, 19, 158, 157, 126, 313, 66, 212, 278, 261, 201, 9, 276, 262, 193, 199, 218, 286, 289, 246, 269, 120, 186, 8, 264, 314, 94, 5, 48, 366, 73, 1, 362, 28, 80, 245, 70, 140, 318, 170, 112, 34, 4, 163, 146, 225, 216, 371, 238, 33, 35, 62, 309, 359, 244, 339, 47, 85, 378, 119, 61, 46, 2, 124, 285, 190, 361, 22, 52, 364, 324, 168, 111, 166, 51, 7, 229, 367, 102, 18, 231, 379, 281, 215, 383, 203, 258, 129, 347, 68, 135, 38, 64, 374, 360, 302, 302, 321, 272, 319, 6, 75, 155, 65, 234, 19, 181, 293, 126, 126, 353, 105, 25, 270, 153, 9, 275, 44, 177, 251, 380, 76, 260, 306, 187, 312, 331, 255, 183, 298, 109, 235, 58, 205, 254, 336, 340, 137, 54, 86, 223, 138, 198, 290, 224, 323, 222, 77, 81, 301, 303, 295, 327, 352, 36, 98, 338, 103, 345, 14, 211, 176, 69, 71, 0, 12, 31, 84, 192, 101, 283, 108, 299, 125, 185, 104, 357, 377, 13, 147, 165, 363, 162, 334, 161, 99, 152, 207, 291, 247, 288, 3, 50, 24, 92, 375, 191, 79, 210, 302, 197, 321, 37, 6, 268, 304, 257, 297, 59, 136, 157, 253, 253, 313, 202, 78, 271, 110, 90, 335, 242, 39, 259, 325, 292, 107, 91, 220, 344, 329, 45, 30, 100, 95, 164, 122, 144, 328, 169, 174, 63, 382, 16, 368, 311, 196, 277, 128, 29, 160, 127, 248, 376, 55, 139, 173, 308, 67, 43, 134, 227, 156, 142, 239, 320, 280, 226, 358, 118, 106, 204, 206, 282, 342, 113, 154, 354, 348, 326, 274, 316, 27, 315, 87, 341, 237, 159, 287, 265, 171, 350, 217, 11, 230, 369, 219, 373, 349, 322, 337, 123, 79, 233, 302, 184, 321, 149, 83, 268, 75, 155, 297, 305, 150, 115, 253, 213, 116, 202, 241, 271, 40, 208, 180, 242, 148, 193, 60, 251, 133, 243, 132, 74, 194, 381, 175, 332, 131, 333, 195, 121, 250, 97, 279, 17, 310, 221, 23, 56, 240, 365, 214, 89, 189, 41, 296, 20, 273, 82, 10, 179, 267, 330, 178, 307, 351, 172, 114, 72, 53, 117, 130, 355, 200, 49, 151, 284, 228, 143, 256, 145, 182, 88, 232, 343, 57, 93, 167, 372, 42, 96, 266, 21, 236, 249, 141, 263, 252, 32, 26, 370, 188, 300, 317, 356] + 50: [57, 86, 268, 100, 109, 140, 49, 67, 97, 32, 32, 233, 215, 215, 69, 273, 321, 146, 56, 56, 9, 26, 1, 3, 128, 128, 327, 0, 341, 114, 331, 155, 224, 270, 335, 16, 353, 122, 164, 313, 239, 48, 186, 375, 194, 116, 134, 379, 206, 319, 180, 250, 310, 74, 378, 81, 152, 20, 242, 205, 211, 28, 251, 300, 135, 369, 127, 202, 281, 344, 123, 106, 34, 200, 372, 141, 212, 355, 258, 36, 138, 112, 286, 324, 101, 10, 83, 201, 173, 190, 72, 31, 142, 279, 167, 221, 302, 118, 66, 364, 222, 257, 111, 345, 33, 86, 268, 100, 299, 137, 49, 229, 333, 376, 315, 210, 188, 262, 69, 325, 321, 214, 214, 27, 350, 282, 18, 129, 231, 62, 45, 220, 334, 256, 314, 343, 253, 15, 121, 187, 274, 24, 301, 244, 306, 228, 195, 165, 348, 183, 219, 102, 189, 169, 276, 208, 267, 85, 73, 107, 277, 126, 255, 99, 105, 172, 278, 5, 37, 243, 362, 337, 289, 153, 42, 170, 198, 305, 12, 46, 68, 383, 90, 148, 162, 179, 291, 346, 30, 381, 176, 246, 207, 328, 265, 104, 261, 54, 144, 225, 317, 59, 131, 44, 283, 264, 136, 365, 80, 86, 25, 226, 316, 109, 181, 166, 38, 376, 284, 352, 370, 218, 35, 325, 125, 366, 312, 175, 377, 350, 18, 129, 231, 338, 192, 199, 193, 88, 22, 51, 184, 271, 213, 298, 132, 363, 196, 275, 380, 58, 19, 75, 263, 290, 240, 43, 247, 269, 174, 94, 371, 151, 92, 64, 294, 160, 78, 145, 340, 185, 108, 143, 236, 17, 297, 292, 204, 339, 177, 351, 349, 308, 156, 238, 237, 89, 318, 171, 65, 285, 82, 147, 77, 209, 61, 76, 157, 103, 70, 216, 266, 120, 161, 227, 52, 254, 307, 84, 332, 50, 55, 230, 29, 29, 235, 268, 316, 311, 181, 166, 14, 367, 315, 210, 188, 361, 232, 232, 110, 373, 287, 27, 280, 26, 1, 3, 304, 329, 296, 217, 358, 119, 295, 79, 272, 130, 159, 249, 133, 6, 41, 96, 71, 149, 117, 203, 154, 168, 359, 53, 252, 342, 197, 139, 382, 234, 326, 303, 163, 322, 2, 91, 357, 223, 113, 356, 178, 347, 39, 191, 4, 288, 124, 259, 7, 63, 320, 248, 330, 150, 354, 293, 98, 323, 11, 87, 40, 60, 13, 21, 47, 8, 360, 245, 95, 158, 309, 260, 115, 368, 336, 182, 23, 241, 93, 374] + 51: [322, 375, 221, 189, 372, 210, 110, 178, 24, 111, 186, 357, 123, 113, 284, 17, 228, 176, 64, 72, 67, 211, 229, 65, 135, 270, 40, 124, 188, 105, 295, 168, 269, 130, 263, 5, 13, 46, 118, 85, 183, 47, 382, 81, 74, 142, 359, 162, 360, 209, 351, 310, 204, 119, 309, 166, 50, 75, 48, 239, 320, 364, 171, 84, 196, 94, 349, 45, 51, 223, 338, 288, 145, 78, 363, 159, 58, 109, 112, 240, 218, 116, 148, 83, 354, 296, 88, 165, 174, 77, 217, 133, 293, 302, 352, 117, 36, 347, 377, 173, 121, 151, 249, 261, 108, 373, 106, 241, 371, 372, 6, 200, 323, 24, 186, 208, 12, 193, 102, 52, 180, 122, 337, 264, 82, 126, 229, 343, 286, 286, 260, 235, 330, 129, 146, 177, 243, 11, 314, 26, 299, 216, 374, 23, 285, 245, 275, 308, 345, 86, 268, 279, 42, 366, 280, 79, 34, 91, 160, 252, 10, 195, 236, 224, 306, 69, 289, 313, 100, 332, 334, 297, 350, 27, 114, 194, 276, 278, 333, 55, 61, 336, 316, 25, 125, 149, 1, 96, 9, 7, 257, 199, 277, 120, 376, 227, 71, 187, 63, 318, 225, 38, 131, 21, 226, 139, 291, 326, 31, 137, 221, 73, 37, 210, 110, 206, 98, 24, 16, 186, 208, 287, 102, 17, 342, 127, 337, 144, 358, 126, 163, 62, 87, 104, 203, 301, 250, 233, 283, 253, 103, 49, 335, 92, 8, 35, 294, 281, 292, 271, 161, 303, 254, 237, 198, 380, 219, 222, 181, 192, 207, 248, 321, 368, 44, 362, 41, 164, 265, 324, 311, 369, 383, 258, 365, 315, 89, 170, 191, 30, 242, 298, 93, 353, 339, 378, 43, 232, 19, 15, 214, 230, 28, 39, 272, 201, 305, 128, 367, 70, 169, 185, 274, 147, 150, 231, 256, 76, 255, 2, 290, 348, 137, 373, 375, 241, 37, 372, 101, 200, 323, 24, 186, 99, 12, 287, 102, 182, 342, 356, 136, 144, 82, 300, 163, 140, 152, 60, 32, 95, 325, 304, 153, 282, 246, 346, 134, 247, 259, 143, 20, 220, 331, 212, 344, 97, 172, 341, 56, 107, 18, 329, 190, 59, 90, 370, 251, 33, 141, 3, 361, 234, 157, 205, 262, 317, 213, 115, 312, 381, 379, 54, 328, 80, 307, 319, 273, 340, 197, 53, 238, 167, 184, 68, 66, 57, 4, 215, 14, 158, 154, 0, 132, 355, 175, 179, 266, 29, 22, 244, 267, 155, 327, 138, 156, 202] + 52: [52, 102, 332, 107, 372, 381, 371, 99, 182, 334, 157, 71, 362, 104, 165, 97, 85, 336, 239, 34, 211, 48, 82, 184, 78, 205, 346, 62, 245, 140, 56, 269, 355, 349, 149, 354, 257, 325, 312, 133, 339, 222, 234, 191, 178, 359, 30, 23, 364, 75, 243, 369, 113, 361, 53, 115, 22, 213, 324, 208, 125, 109, 161, 225, 270, 237, 200, 177, 294, 282, 152, 207, 220, 230, 136, 249, 143, 28, 310, 322, 226, 340, 4, 96, 353, 135, 232, 276, 297, 329, 314, 203, 159, 65, 122, 142, 317, 14, 295, 272, 204, 370, 180, 247, 186, 102, 231, 107, 107, 381, 320, 99, 63, 334, 157, 71, 250, 104, 165, 73, 214, 239, 321, 174, 169, 193, 163, 90, 7, 205, 190, 33, 260, 80, 196, 192, 266, 54, 95, 50, 291, 88, 228, 379, 127, 378, 262, 5, 83, 315, 215, 160, 233, 248, 6, 300, 153, 72, 331, 351, 279, 316, 55, 287, 253, 326, 268, 235, 352, 57, 242, 258, 365, 20, 21, 194, 51, 47, 330, 347, 303, 101, 380, 118, 344, 18, 126, 311, 358, 138, 274, 363, 9, 36, 185, 42, 273, 265, 338, 144, 373, 254, 376, 360, 162, 16, 305, 375, 327, 102, 120, 61, 283, 381, 99, 99, 130, 334, 151, 71, 103, 104, 97, 137, 289, 290, 195, 206, 209, 209, 77, 25, 13, 41, 172, 68, 357, 44, 259, 219, 87, 32, 8, 236, 309, 35, 155, 171, 74, 221, 286, 105, 341, 281, 173, 223, 367, 134, 146, 67, 301, 328, 154, 377, 374, 170, 66, 337, 40, 306, 119, 93, 89, 2, 110, 277, 224, 244, 238, 189, 139, 179, 100, 198, 49, 356, 278, 197, 111, 114, 335, 343, 229, 304, 302, 275, 158, 261, 255, 131, 12, 19, 15, 0, 24, 240, 164, 264, 106, 212, 38, 176, 121, 92, 307, 216, 283, 217, 381, 267, 31, 334, 157, 246, 103, 165, 97, 137, 323, 336, 271, 345, 263, 48, 298, 25, 183, 382, 288, 60, 10, 26, 128, 168, 210, 252, 64, 98, 70, 37, 123, 108, 256, 285, 117, 166, 296, 227, 124, 167, 29, 116, 280, 292, 3, 76, 17, 319, 39, 69, 333, 91, 45, 27, 308, 350, 58, 201, 383, 284, 148, 59, 84, 251, 318, 150, 368, 129, 188, 156, 141, 342, 43, 81, 299, 313, 181, 293, 112, 147, 132, 145, 187, 79, 11, 202, 366, 241, 1, 86, 94, 199, 218, 46, 175, 348] + 53: [230, 240, 110, 110, 297, 222, 13, 73, 23, 287, 129, 296, 16, 142, 292, 365, 273, 295, 250, 330, 32, 4, 186, 132, 145, 279, 86, 291, 376, 225, 130, 221, 31, 50, 326, 95, 241, 168, 256, 262, 381, 341, 112, 77, 152, 335, 300, 52, 235, 47, 281, 159, 44, 293, 67, 328, 340, 35, 196, 178, 375, 276, 189, 370, 236, 215, 255, 358, 364, 220, 232, 97, 0, 54, 201, 108, 137, 131, 380, 323, 29, 82, 51, 39, 193, 64, 184, 163, 208, 267, 356, 91, 192, 181, 213, 33, 355, 99, 223, 307, 22, 18, 312, 210, 336, 68, 240, 110, 110, 203, 13, 73, 350, 334, 320, 296, 164, 109, 46, 354, 311, 347, 250, 330, 42, 118, 228, 170, 160, 6, 202, 157, 162, 332, 103, 119, 125, 284, 24, 127, 5, 3, 94, 12, 70, 361, 383, 205, 172, 313, 49, 277, 176, 259, 89, 72, 122, 175, 143, 146, 36, 128, 120, 2, 174, 166, 57, 327, 19, 30, 333, 133, 299, 251, 242, 55, 258, 337, 21, 352, 14, 246, 305, 43, 331, 83, 85, 342, 74, 17, 113, 34, 93, 382, 148, 211, 212, 98, 165, 187, 304, 71, 58, 290, 265, 153, 167, 114, 308, 8, 240, 110, 110, 66, 317, 247, 23, 287, 38, 38, 16, 109, 280, 311, 214, 295, 224, 179, 155, 32, 186, 266, 156, 209, 309, 194, 237, 26, 81, 285, 368, 20, 253, 319, 219, 200, 329, 139, 346, 338, 183, 188, 37, 239, 378, 197, 60, 149, 15, 126, 353, 282, 362, 357, 369, 248, 216, 206, 65, 105, 371, 96, 306, 117, 79, 40, 138, 182, 294, 101, 195, 87, 257, 7, 53, 80, 264, 9, 100, 177, 310, 1, 147, 301, 229, 303, 374, 260, 263, 288, 27, 343, 286, 204, 11, 344, 244, 314, 252, 140, 316, 272, 217, 75, 110, 110, 243, 317, 270, 73, 350, 334, 238, 38, 16, 173, 322, 354, 347, 363, 345, 379, 227, 4, 92, 302, 180, 360, 268, 351, 261, 144, 271, 121, 134, 107, 269, 171, 325, 88, 62, 25, 69, 141, 231, 116, 151, 324, 78, 373, 10, 150, 349, 76, 315, 102, 359, 226, 233, 111, 115, 339, 169, 321, 158, 207, 278, 28, 136, 123, 198, 218, 59, 254, 45, 275, 234, 154, 367, 289, 283, 249, 199, 48, 63, 298, 348, 106, 366, 372, 104, 135, 161, 41, 190, 61, 124, 274, 191, 318, 245, 377, 56, 90, 185, 84] + 54: [295, 337, 380, 363, 363, 71, 338, 147, 258, 356, 356, 155, 216, 164, 164, 80, 250, 38, 20, 299, 53, 95, 72, 176, 93, 221, 87, 119, 309, 314, 86, 16, 27, 121, 288, 41, 47, 218, 129, 322, 84, 124, 91, 5, 190, 272, 182, 24, 54, 381, 204, 237, 55, 368, 100, 67, 61, 144, 69, 68, 293, 111, 136, 201, 62, 81, 162, 18, 166, 300, 128, 266, 194, 229, 334, 213, 308, 234, 74, 178, 29, 249, 261, 52, 89, 263, 203, 56, 270, 116, 303, 210, 134, 169, 188, 222, 165, 349, 19, 200, 106, 354, 361, 378, 370, 370, 363, 363, 253, 294, 319, 42, 258, 11, 339, 339, 155, 257, 82, 214, 133, 348, 280, 230, 172, 306, 95, 208, 44, 345, 305, 25, 26, 118, 75, 329, 181, 332, 276, 76, 115, 196, 161, 215, 357, 50, 323, 94, 185, 152, 142, 109, 239, 184, 376, 105, 355, 131, 49, 1, 278, 264, 373, 85, 163, 122, 209, 304, 364, 353, 173, 195, 262, 23, 275, 141, 238, 251, 12, 63, 110, 346, 145, 14, 211, 289, 331, 268, 343, 248, 46, 226, 149, 360, 51, 65, 350, 138, 112, 256, 297, 193, 344, 358, 127, 160, 148, 232, 370, 284, 153, 363, 294, 283, 328, 90, 42, 11, 287, 223, 231, 231, 260, 240, 240, 348, 280, 230, 8, 286, 324, 341, 120, 290, 57, 352, 40, 180, 3, 228, 58, 156, 132, 143, 99, 310, 197, 158, 277, 225, 45, 325, 265, 92, 217, 382, 103, 140, 21, 312, 97, 315, 183, 318, 73, 171, 48, 301, 326, 107, 22, 242, 146, 167, 377, 151, 244, 316, 191, 351, 245, 243, 371, 259, 15, 224, 235, 247, 236, 274, 233, 79, 108, 66, 60, 199, 374, 154, 307, 4, 28, 77, 2, 241, 207, 78, 340, 202, 123, 13, 0, 36, 370, 284, 296, 363, 294, 283, 189, 83, 39, 267, 339, 223, 155, 150, 164, 369, 30, 38, 98, 299, 174, 286, 281, 254, 282, 187, 330, 311, 117, 7, 372, 317, 101, 10, 375, 70, 333, 32, 113, 64, 198, 335, 17, 220, 321, 34, 159, 279, 125, 170, 157, 186, 365, 302, 135, 177, 246, 298, 366, 255, 327, 137, 313, 179, 96, 33, 114, 347, 273, 43, 9, 175, 139, 192, 271, 88, 219, 6, 59, 168, 367, 31, 206, 269, 359, 362, 102, 227, 285, 35, 292, 104, 379, 291, 252, 37, 126, 342, 383, 130, 212, 205, 320, 336] + 55: [56, 202, 152, 154, 256, 81, 174, 174, 234, 146, 146, 218, 370, 39, 311, 16, 381, 123, 299, 90, 48, 78, 347, 74, 180, 29, 269, 76, 49, 291, 223, 134, 114, 272, 320, 349, 368, 312, 316, 156, 344, 259, 197, 46, 14, 276, 91, 375, 260, 27, 380, 47, 240, 286, 53, 194, 241, 119, 10, 346, 358, 207, 84, 94, 22, 314, 31, 153, 160, 342, 188, 37, 208, 253, 359, 319, 321, 38, 297, 129, 58, 120, 50, 374, 248, 166, 183, 376, 92, 295, 181, 36, 75, 32, 195, 230, 59, 148, 292, 51, 310, 285, 34, 69, 56, 202, 28, 154, 154, 355, 174, 4, 20, 11, 308, 357, 116, 165, 80, 238, 381, 267, 299, 90, 48, 121, 382, 74, 130, 29, 269, 301, 40, 106, 293, 71, 176, 12, 60, 44, 164, 300, 271, 26, 239, 189, 5, 163, 348, 95, 190, 371, 217, 251, 231, 361, 79, 178, 187, 140, 332, 184, 318, 287, 254, 171, 85, 33, 87, 63, 302, 212, 237, 258, 127, 365, 373, 335, 356, 126, 93, 336, 243, 338, 354, 275, 150, 68, 139, 324, 242, 280, 191, 298, 330, 21, 362, 108, 220, 158, 18, 351, 9, 45, 250, 245, 109, 204, 326, 159, 88, 154, 270, 81, 151, 309, 20, 11, 62, 340, 182, 339, 80, 111, 227, 328, 123, 73, 353, 273, 304, 323, 180, 70, 65, 76, 132, 200, 249, 226, 3, 279, 331, 290, 216, 221, 86, 124, 206, 172, 17, 170, 350, 284, 274, 210, 329, 236, 265, 246, 42, 306, 54, 228, 19, 263, 257, 23, 128, 157, 278, 149, 296, 117, 214, 82, 1, 232, 133, 367, 15, 219, 268, 372, 201, 118, 55, 104, 147, 107, 125, 145, 177, 13, 162, 77, 364, 155, 175, 244, 266, 142, 83, 100, 24, 229, 25, 205, 96, 186, 196, 215, 113, 72, 88, 327, 43, 235, 174, 98, 98, 7, 308, 340, 116, 89, 252, 307, 192, 6, 303, 41, 211, 247, 347, 323, 130, 363, 115, 115, 199, 341, 2, 99, 179, 8, 333, 138, 213, 366, 0, 383, 233, 337, 112, 102, 325, 352, 105, 289, 378, 224, 377, 343, 360, 67, 135, 136, 185, 322, 122, 277, 255, 52, 193, 101, 294, 313, 264, 198, 61, 103, 281, 143, 173, 225, 97, 30, 35, 209, 168, 137, 315, 161, 66, 64, 261, 203, 369, 141, 169, 262, 282, 334, 110, 57, 288, 345, 131, 167, 379, 283, 144, 222, 317, 305] + 56: [279, 8, 195, 167, 242, 44, 191, 178, 240, 220, 213, 15, 107, 98, 303, 267, 106, 106, 30, 5, 75, 9, 189, 263, 363, 125, 6, 346, 320, 140, 275, 2, 83, 77, 80, 14, 63, 207, 118, 238, 246, 205, 359, 376, 38, 160, 250, 10, 105, 21, 136, 35, 179, 23, 355, 280, 50, 110, 151, 164, 123, 209, 129, 297, 190, 339, 62, 271, 183, 180, 84, 119, 172, 347, 28, 4, 232, 175, 291, 16, 29, 49, 70, 361, 93, 152, 254, 345, 234, 325, 239, 349, 261, 7, 36, 217, 25, 174, 91, 318, 301, 85, 315, 143, 100, 231, 311, 321, 265, 260, 126, 82, 240, 220, 294, 273, 116, 163, 158, 351, 322, 259, 371, 87, 304, 147, 196, 236, 333, 187, 162, 161, 316, 99, 292, 18, 332, 329, 288, 381, 168, 192, 72, 97, 344, 233, 334, 27, 285, 281, 137, 278, 92, 181, 252, 79, 354, 269, 12, 22, 11, 34, 153, 243, 141, 282, 364, 42, 378, 122, 201, 299, 298, 96, 145, 218, 228, 132, 307, 200, 330, 276, 65, 367, 78, 144, 352, 222, 373, 24, 211, 258, 26, 256, 215, 370, 326, 32, 377, 365, 114, 247, 235, 61, 76, 74, 146, 3, 177, 231, 195, 343, 265, 260, 199, 178, 240, 81, 213, 15, 186, 155, 296, 225, 241, 340, 30, 5, 304, 156, 20, 249, 104, 251, 308, 46, 48, 39, 41, 188, 362, 327, 154, 245, 283, 101, 383, 309, 197, 103, 300, 182, 69, 342, 193, 128, 324, 305, 31, 295, 374, 268, 317, 149, 214, 138, 13, 17, 127, 185, 56, 286, 115, 313, 360, 94, 55, 134, 262, 341, 310, 290, 323, 226, 379, 57, 139, 372, 266, 170, 43, 350, 113, 204, 131, 337, 108, 272, 68, 212, 59, 336, 45, 216, 380, 171, 237, 102, 109, 142, 173, 90, 231, 231, 195, 321, 242, 44, 260, 82, 240, 220, 294, 294, 53, 88, 184, 225, 241, 340, 259, 169, 75, 147, 356, 89, 264, 306, 117, 328, 71, 284, 219, 302, 19, 66, 159, 357, 244, 150, 120, 223, 369, 0, 58, 73, 40, 148, 312, 253, 135, 206, 198, 338, 166, 230, 121, 331, 1, 133, 255, 33, 270, 227, 248, 368, 221, 95, 348, 210, 257, 54, 314, 353, 375, 293, 124, 112, 319, 274, 37, 289, 67, 64, 86, 130, 60, 287, 47, 157, 382, 358, 176, 203, 52, 335, 366, 277, 111, 194, 165, 51, 229, 224, 202, 208] + 57: [157, 14, 14, 276, 254, 314, 146, 295, 165, 246, 187, 156, 366, 25, 65, 101, 207, 223, 72, 286, 320, 324, 268, 8, 302, 300, 204, 174, 84, 197, 22, 64, 306, 123, 354, 182, 338, 167, 365, 373, 346, 134, 341, 214, 27, 113, 20, 277, 377, 183, 147, 289, 47, 12, 112, 328, 68, 2, 164, 196, 353, 209, 296, 216, 81, 186, 323, 371, 175, 237, 26, 363, 45, 122, 236, 299, 0, 4, 126, 219, 263, 383, 361, 375, 67, 232, 103, 308, 10, 271, 218, 212, 206, 327, 278, 195, 305, 131, 280, 233, 143, 56, 262, 301, 231, 342, 276, 158, 16, 133, 146, 295, 57, 89, 368, 7, 326, 25, 235, 148, 331, 108, 94, 274, 53, 170, 335, 118, 217, 60, 154, 150, 127, 283, 180, 309, 44, 163, 251, 272, 99, 58, 253, 172, 110, 213, 9, 228, 42, 173, 83, 191, 349, 102, 54, 203, 120, 55, 325, 279, 166, 210, 71, 171, 199, 321, 369, 155, 178, 98, 17, 29, 379, 332, 145, 315, 252, 135, 265, 255, 307, 32, 168, 241, 66, 50, 194, 374, 138, 364, 287, 273, 310, 266, 115, 258, 82, 52, 193, 336, 77, 245, 34, 229, 202, 86, 376, 343, 231, 159, 276, 31, 254, 254, 259, 220, 85, 246, 63, 140, 326, 104, 205, 148, 207, 359, 94, 124, 137, 170, 344, 125, 267, 74, 154, 6, 318, 151, 79, 88, 185, 75, 80, 347, 144, 91, 43, 161, 176, 40, 73, 70, 152, 96, 298, 312, 382, 111, 116, 38, 358, 106, 119, 95, 329, 226, 149, 3, 351, 282, 23, 362, 288, 132, 311, 162, 41, 290, 247, 355, 5, 303, 24, 248, 378, 281, 357, 1, 224, 36, 242, 192, 48, 370, 78, 139, 360, 244, 100, 291, 129, 221, 269, 30, 227, 37, 153, 105, 19, 285, 293, 372, 93, 11, 230, 381, 254, 254, 259, 121, 165, 89, 187, 198, 326, 25, 205, 128, 331, 257, 72, 286, 320, 130, 141, 125, 302, 74, 169, 350, 117, 317, 345, 160, 35, 62, 250, 238, 319, 39, 316, 179, 322, 243, 297, 51, 177, 249, 46, 356, 21, 184, 92, 208, 284, 380, 13, 225, 348, 136, 313, 352, 337, 190, 90, 28, 330, 215, 18, 334, 240, 261, 97, 49, 33, 211, 59, 200, 270, 87, 188, 333, 109, 260, 222, 256, 239, 76, 107, 61, 142, 264, 339, 114, 294, 340, 189, 292, 69, 181, 15, 201, 367, 234, 275, 304] + 58: [263, 220, 377, 377, 377, 249, 56, 79, 366, 205, 92, 344, 245, 98, 194, 101, 177, 299, 9, 22, 114, 358, 270, 256, 116, 178, 68, 196, 18, 47, 119, 191, 277, 132, 162, 20, 274, 185, 328, 28, 281, 192, 375, 55, 244, 62, 266, 99, 236, 83, 339, 333, 37, 240, 0, 63, 300, 253, 148, 231, 123, 334, 228, 21, 70, 293, 35, 50, 237, 230, 226, 170, 24, 261, 29, 372, 330, 268, 276, 41, 336, 44, 46, 223, 369, 246, 153, 304, 52, 286, 188, 174, 140, 343, 23, 212, 166, 279, 71, 321, 108, 280, 33, 38, 248, 220, 77, 377, 377, 126, 56, 81, 259, 305, 356, 210, 211, 14, 311, 373, 93, 235, 312, 298, 289, 67, 355, 169, 40, 36, 262, 324, 341, 6, 203, 272, 130, 111, 255, 316, 190, 307, 73, 198, 380, 1, 331, 381, 82, 143, 181, 284, 215, 53, 250, 326, 378, 121, 131, 332, 144, 13, 357, 154, 65, 161, 2, 43, 265, 115, 195, 179, 136, 219, 118, 297, 102, 138, 142, 100, 264, 221, 163, 354, 200, 374, 167, 349, 57, 117, 34, 137, 15, 88, 30, 365, 323, 91, 89, 291, 80, 147, 309, 370, 168, 201, 204, 209, 156, 220, 275, 377, 377, 151, 81, 61, 273, 205, 92, 27, 245, 271, 311, 222, 322, 49, 312, 298, 69, 351, 214, 199, 368, 175, 376, 295, 342, 85, 42, 189, 12, 315, 260, 106, 242, 197, 171, 329, 229, 139, 314, 251, 25, 383, 258, 350, 238, 90, 160, 234, 346, 75, 149, 110, 241, 362, 176, 186, 327, 318, 345, 11, 113, 19, 225, 8, 133, 173, 338, 243, 294, 296, 95, 87, 128, 86, 303, 157, 146, 337, 84, 269, 39, 109, 32, 159, 78, 301, 267, 217, 218, 363, 165, 313, 252, 292, 125, 335, 150, 364, 58, 134, 156, 220, 275, 377, 377, 76, 126, 81, 366, 305, 7, 348, 245, 98, 194, 222, 177, 206, 122, 122, 114, 72, 257, 26, 112, 232, 152, 45, 141, 367, 31, 16, 97, 104, 60, 4, 352, 66, 224, 285, 187, 48, 202, 239, 207, 59, 129, 325, 306, 155, 124, 227, 183, 382, 135, 302, 172, 208, 379, 288, 107, 278, 371, 347, 287, 182, 283, 353, 94, 105, 180, 64, 5, 213, 360, 317, 54, 359, 282, 254, 290, 103, 51, 247, 96, 120, 145, 216, 17, 193, 127, 74, 310, 361, 233, 320, 164, 158, 3, 319, 10, 184, 340, 308] + 59: [332, 182, 327, 33, 75, 25, 170, 197, 147, 229, 370, 94, 195, 260, 251, 173, 173, 220, 66, 204, 221, 352, 288, 18, 300, 65, 372, 377, 151, 26, 266, 196, 133, 112, 45, 109, 304, 81, 168, 279, 181, 302, 383, 252, 373, 225, 29, 19, 329, 236, 158, 69, 269, 348, 238, 354, 155, 13, 131, 51, 61, 199, 362, 299, 380, 27, 64, 110, 129, 50, 253, 134, 17, 207, 359, 93, 277, 132, 183, 11, 118, 53, 296, 103, 312, 368, 301, 237, 141, 282, 192, 125, 121, 293, 367, 78, 126, 104, 99, 52, 4, 111, 268, 271, 140, 209, 327, 285, 46, 25, 250, 256, 379, 233, 7, 94, 202, 59, 95, 200, 340, 311, 247, 371, 352, 234, 331, 31, 42, 213, 186, 328, 23, 80, 309, 232, 114, 342, 278, 223, 57, 12, 35, 159, 364, 334, 62, 162, 376, 60, 218, 338, 214, 356, 222, 258, 137, 146, 177, 259, 337, 353, 101, 123, 314, 274, 48, 206, 227, 71, 43, 135, 325, 184, 84, 124, 67, 74, 153, 49, 261, 44, 24, 191, 201, 357, 102, 230, 216, 303, 8, 108, 239, 240, 350, 14, 335, 77, 244, 87, 152, 139, 72, 323, 310, 90, 79, 154, 140, 113, 327, 105, 224, 170, 170, 97, 226, 229, 5, 7, 195, 260, 70, 200, 340, 381, 66, 263, 346, 291, 130, 185, 265, 65, 324, 317, 178, 243, 281, 58, 128, 176, 212, 164, 161, 160, 16, 127, 267, 280, 163, 320, 143, 22, 174, 142, 54, 47, 179, 339, 344, 180, 366, 203, 289, 365, 349, 68, 242, 292, 32, 194, 287, 193, 149, 246, 187, 136, 231, 217, 21, 343, 326, 89, 336, 210, 106, 115, 148, 273, 307, 241, 358, 165, 319, 316, 276, 305, 219, 270, 375, 171, 205, 347, 83, 55, 122, 167, 235, 117, 341, 189, 73, 209, 327, 30, 224, 25, 250, 313, 379, 229, 370, 7, 202, 59, 363, 363, 220, 298, 374, 294, 346, 234, 76, 150, 172, 295, 9, 38, 262, 144, 382, 255, 86, 290, 91, 175, 40, 85, 254, 297, 100, 211, 169, 248, 315, 20, 245, 286, 306, 10, 355, 120, 198, 369, 92, 361, 36, 6, 228, 2, 156, 96, 107, 0, 333, 138, 63, 41, 34, 208, 322, 157, 330, 28, 37, 145, 3, 88, 249, 275, 321, 119, 166, 284, 283, 82, 257, 98, 318, 1, 360, 56, 39, 272, 264, 378, 345, 190, 116, 351, 308, 15, 188, 215] + 60: [284, 121, 313, 313, 381, 124, 257, 158, 16, 115, 220, 47, 139, 244, 126, 133, 316, 212, 174, 310, 167, 312, 185, 81, 2, 239, 294, 319, 365, 46, 193, 309, 206, 232, 143, 223, 241, 55, 364, 359, 336, 0, 69, 248, 221, 37, 51, 1, 92, 24, 9, 237, 215, 274, 304, 305, 134, 186, 70, 53, 31, 66, 33, 30, 100, 192, 106, 370, 380, 256, 18, 64, 125, 363, 282, 376, 251, 45, 330, 328, 179, 54, 198, 50, 374, 211, 29, 96, 301, 191, 350, 281, 289, 164, 275, 340, 101, 321, 226, 136, 373, 154, 348, 355, 102, 236, 313, 381, 335, 124, 124, 158, 128, 195, 266, 47, 139, 343, 83, 83, 199, 222, 174, 310, 194, 120, 297, 208, 74, 366, 35, 337, 180, 112, 311, 354, 245, 131, 184, 203, 238, 263, 99, 253, 152, 150, 58, 20, 269, 358, 183, 240, 80, 332, 382, 264, 61, 176, 8, 56, 323, 21, 107, 142, 285, 85, 200, 250, 300, 130, 357, 349, 368, 219, 14, 148, 315, 338, 105, 270, 23, 13, 252, 324, 137, 261, 104, 42, 68, 204, 378, 291, 175, 249, 17, 254, 227, 163, 177, 295, 44, 118, 218, 89, 210, 63, 207, 201, 94, 123, 313, 268, 351, 351, 124, 257, 16, 170, 243, 303, 60, 372, 39, 133, 339, 40, 78, 265, 247, 65, 302, 181, 205, 34, 108, 49, 290, 166, 345, 98, 278, 122, 117, 161, 235, 27, 246, 140, 84, 151, 362, 283, 93, 132, 36, 318, 11, 88, 267, 110, 38, 57, 279, 25, 26, 233, 52, 360, 187, 293, 149, 196, 182, 369, 95, 314, 276, 144, 225, 371, 325, 333, 307, 67, 202, 361, 209, 287, 19, 216, 41, 145, 160, 356, 147, 298, 86, 165, 326, 135, 173, 367, 4, 103, 352, 10, 77, 242, 157, 317, 288, 375, 123, 123, 313, 268, 351, 351, 124, 257, 16, 260, 220, 129, 60, 347, 126, 113, 199, 48, 40, 265, 230, 153, 334, 155, 28, 327, 76, 234, 353, 119, 329, 15, 213, 272, 231, 59, 271, 82, 169, 172, 75, 189, 277, 255, 114, 6, 217, 259, 344, 188, 162, 5, 3, 377, 116, 342, 141, 171, 346, 90, 273, 91, 7, 146, 190, 341, 79, 322, 296, 286, 12, 331, 197, 383, 156, 168, 262, 127, 178, 228, 320, 258, 280, 224, 379, 229, 97, 111, 72, 159, 306, 43, 292, 71, 138, 32, 73, 109, 87, 308, 62, 214, 299, 22] + 61: [231, 73, 136, 93, 157, 170, 179, 46, 127, 127, 299, 323, 19, 184, 1, 45, 253, 218, 167, 21, 344, 371, 324, 198, 248, 263, 161, 119, 88, 107, 216, 168, 34, 159, 349, 243, 169, 261, 228, 191, 348, 297, 316, 114, 135, 82, 252, 304, 47, 308, 206, 145, 104, 202, 33, 259, 322, 38, 78, 306, 8, 37, 125, 126, 85, 196, 26, 317, 176, 194, 359, 148, 70, 311, 91, 163, 187, 255, 234, 251, 7, 262, 185, 314, 164, 212, 223, 105, 35, 220, 56, 226, 151, 175, 134, 62, 41, 320, 16, 128, 356, 189, 208, 319, 303, 73, 5, 211, 221, 221, 338, 46, 127, 127, 299, 178, 270, 264, 197, 363, 100, 36, 167, 195, 372, 278, 324, 142, 143, 213, 237, 66, 112, 216, 129, 115, 369, 42, 31, 160, 309, 43, 337, 20, 118, 77, 90, 84, 44, 166, 121, 379, 329, 260, 3, 23, 97, 147, 249, 378, 154, 383, 153, 273, 280, 174, 242, 219, 225, 240, 81, 4, 144, 332, 98, 106, 193, 244, 130, 271, 239, 6, 123, 74, 232, 111, 201, 293, 373, 138, 265, 140, 256, 276, 302, 277, 281, 39, 289, 183, 76, 229, 296, 40, 171, 96, 287, 58, 230, 61, 108, 360, 211, 221, 179, 338, 247, 165, 205, 117, 241, 59, 197, 381, 103, 36, 315, 330, 344, 68, 294, 215, 55, 263, 161, 86, 88, 107, 288, 333, 110, 286, 181, 156, 227, 133, 87, 258, 224, 65, 268, 29, 300, 190, 313, 11, 109, 336, 69, 368, 207, 200, 102, 173, 72, 137, 346, 214, 275, 9, 365, 80, 269, 141, 340, 71, 124, 14, 342, 298, 162, 345, 63, 54, 354, 18, 364, 358, 312, 331, 246, 272, 95, 28, 48, 236, 250, 222, 60, 285, 254, 355, 382, 267, 376, 217, 266, 350, 116, 53, 94, 0, 172, 61, 335, 93, 120, 221, 179, 338, 326, 351, 122, 203, 10, 209, 380, 381, 100, 353, 315, 12, 17, 51, 238, 235, 248, 307, 307, 86, 192, 327, 291, 57, 177, 188, 15, 282, 32, 75, 92, 361, 362, 150, 233, 366, 83, 27, 113, 374, 2, 257, 245, 328, 52, 158, 290, 139, 186, 89, 292, 30, 375, 341, 22, 79, 283, 67, 295, 132, 199, 284, 204, 152, 370, 274, 352, 25, 310, 301, 101, 99, 49, 24, 13, 305, 180, 182, 146, 339, 318, 347, 64, 377, 279, 131, 210, 50, 367, 325, 334, 357, 343, 321, 149, 155] +layer_updates_per_iter: 0 +num_slots: 416 diff --git a/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots384.yaml b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots384.yaml new file mode 100644 index 000000000000..4a5c707042a2 --- /dev/null +++ b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 45, 49, 259, 322, 235, 158, 163, 81, 29, 289, 70, 198, 144, 141, 181, 107, 43, 57, 361, 28, 248, 309, 262, 5, 91, 317, 137, 363, 192, 296, 270, 299, 326, 50, 252, 352, 316, 175, 2, 301, 125, 132, 304, 364, 273, 13, 373, 111, 279, 203, 150, 159, 348, 241, 295, 21, 80, 255, 376, 318, 272, 359, 72, 101, 233, 109, 129, 324, 208, 123, 314, 39, 133, 4, 108, 116, 216, 100, 379, 382, 263, 20, 210, 42, 212, 315, 334, 367, 251, 298, 329, 339, 8, 171, 274, 378, 135, 184, 0, 345, 177, 346, 188, 153, 242, 321, 65, 278, 209, 360, 284, 34, 313, 297, 311, 114, 44, 268, 234, 52, 308, 103, 146, 140, 19, 180, 320, 127, 14, 145, 350, 15, 64, 53, 69, 319, 164, 18, 226, 291, 88, 370, 185, 166, 254, 134, 217, 351, 155, 264, 332, 122, 62, 157, 68, 333, 280, 143, 205, 196, 283, 186, 200, 261, 148, 31, 358, 85, 187, 136, 240, 178, 99, 211, 195, 82, 328, 47, 102, 338, 340, 380, 179, 381, 250, 167, 374, 325, 323, 58, 201, 256, 225, 95, 330, 151, 383, 193, 182, 51, 104, 232, 63, 37, 126, 22, 307, 11, 341, 90, 258, 112, 300, 27, 89, 66, 32, 214, 249, 246, 275, 238, 292, 83, 93, 260, 229, 161, 253, 353, 152, 194, 357, 142, 24, 228, 130, 277, 61, 74, 371, 79, 290, 84, 119, 230, 169, 35, 202, 327, 139, 276, 349, 55, 33, 117, 156, 10, 36, 138, 7, 77, 375, 377, 172, 227, 331, 269, 366, 1, 25, 73, 67, 342, 98, 294, 206, 12, 219, 9, 285, 271, 121, 281, 168, 118, 247, 222, 245, 17, 173, 71, 86, 207, 312, 347, 30, 218, 176, 16, 3, 78, 243, 105, 183, 124, 174, 94, 337, 160, 199, 87, 244, 335, 365, 372, 369, 97, 106, 147, 237, 362, 286, 305, 115, 306, 191, 231, 220, 23, 189, 355, 288, 310, 303, 356, 236, 60, 224, 302, 162, 76, 190, 26, 204, 48, 343, 293, 282, 336, 54, 131, 56, 354, 265, 368, 170, 149, 38, 223, 257, 213, 46, 75, 59, 215, 221, 197, 267, 6, 154, 40, 96, 41, 113, 239, 266, 92, 287, 128, 344, 120, 110] + 1: [163, 177, 198, 292, 251, 18, 196, 378, 366, 328, 73, 153, 351, 195, 9, 32, 266, 285, 187, 261, 143, 282, 264, 202, 231, 189, 164, 238, 338, 326, 19, 33, 186, 51, 352, 346, 344, 77, 357, 220, 340, 217, 132, 16, 144, 28, 336, 265, 319, 169, 182, 225, 1, 80, 152, 40, 179, 41, 110, 271, 185, 273, 145, 173, 317, 355, 197, 194, 214, 176, 203, 239, 61, 98, 7, 305, 2, 20, 175, 136, 324, 11, 97, 8, 232, 314, 130, 246, 188, 373, 361, 45, 276, 339, 345, 15, 365, 72, 127, 82, 31, 106, 50, 21, 270, 123, 105, 248, 383, 243, 287, 71, 234, 212, 162, 379, 52, 53, 306, 376, 60, 205, 362, 204, 364, 17, 156, 68, 36, 284, 96, 181, 240, 103, 83, 297, 160, 207, 252, 343, 349, 341, 363, 327, 42, 91, 167, 178, 380, 118, 43, 241, 5, 75, 300, 244, 25, 46, 275, 86, 245, 301, 84, 92, 223, 116, 69, 360, 95, 76, 121, 259, 370, 107, 115, 62, 290, 26, 104, 113, 54, 119, 369, 331, 193, 209, 124, 155, 307, 354, 112, 368, 201, 382, 67, 272, 81, 44, 291, 329, 30, 210, 58, 56, 24, 59, 303, 247, 78, 274, 166, 142, 114, 93, 289, 278, 236, 298, 122, 330, 65, 286, 184, 154, 35, 230, 311, 268, 226, 333, 375, 372, 367, 221, 128, 322, 117, 257, 199, 255, 174, 277, 347, 134, 141, 308, 263, 131, 6, 325, 294, 299, 224, 320, 211, 260, 216, 99, 288, 109, 219, 228, 381, 342, 147, 237, 140, 170, 139, 318, 229, 165, 48, 293, 227, 55, 12, 49, 148, 309, 90, 151, 66, 374, 337, 157, 267, 100, 22, 158, 377, 111, 200, 126, 149, 254, 39, 0, 63, 250, 283, 29, 269, 242, 281, 74, 218, 280, 4, 47, 358, 171, 101, 102, 313, 222, 161, 332, 295, 321, 87, 315, 13, 302, 262, 356, 279, 371, 183, 253, 208, 64, 258, 304, 312, 215, 137, 180, 3, 120, 235, 85, 150, 168, 192, 70, 334, 23, 233, 138, 37, 89, 125, 146, 108, 335, 213, 206, 88, 38, 133, 256, 79, 348, 249, 14, 296, 57, 172, 350, 94, 323, 353, 27, 190, 129, 316, 135, 10, 191, 159, 359, 310, 34] + 2: [143, 295, 242, 180, 96, 137, 321, 78, 111, 173, 73, 245, 133, 325, 217, 235, 303, 165, 344, 273, 68, 67, 21, 138, 46, 171, 193, 129, 372, 252, 365, 219, 239, 47, 178, 261, 302, 288, 8, 164, 310, 37, 353, 88, 287, 337, 40, 374, 248, 3, 212, 230, 225, 28, 197, 33, 101, 284, 56, 70, 172, 368, 63, 333, 159, 209, 17, 87, 293, 92, 383, 148, 253, 265, 181, 103, 38, 312, 176, 179, 77, 238, 376, 319, 54, 71, 251, 98, 117, 227, 240, 289, 301, 202, 363, 161, 53, 223, 290, 314, 14, 5, 189, 213, 84, 166, 286, 44, 123, 45, 200, 82, 285, 61, 206, 224, 27, 262, 120, 218, 41, 65, 309, 69, 256, 12, 257, 0, 25, 281, 141, 352, 329, 99, 335, 158, 378, 283, 249, 315, 358, 282, 255, 366, 43, 115, 62, 86, 343, 298, 125, 162, 110, 322, 280, 278, 31, 51, 274, 24, 311, 79, 299, 263, 50, 203, 201, 76, 332, 241, 169, 250, 106, 39, 268, 2, 36, 334, 264, 359, 216, 370, 338, 307, 382, 177, 369, 64, 228, 354, 300, 381, 132, 93, 232, 139, 357, 259, 100, 192, 155, 124, 355, 271, 220, 22, 328, 85, 6, 342, 7, 90, 269, 210, 361, 297, 349, 308, 186, 254, 136, 49, 318, 23, 195, 346, 331, 57, 243, 371, 114, 267, 336, 222, 13, 380, 32, 339, 199, 330, 112, 194, 168, 304, 341, 221, 247, 60, 153, 97, 157, 135, 105, 296, 279, 340, 196, 174, 170, 347, 55, 188, 58, 128, 360, 145, 348, 276, 364, 121, 29, 362, 134, 4, 272, 231, 292, 373, 167, 59, 72, 234, 260, 127, 119, 270, 190, 379, 108, 89, 91, 377, 102, 1, 151, 149, 109, 191, 66, 118, 42, 26, 83, 244, 323, 204, 130, 356, 126, 327, 187, 198, 163, 16, 15, 160, 154, 142, 104, 94, 275, 35, 52, 144, 226, 95, 183, 48, 113, 345, 214, 266, 316, 208, 375, 258, 313, 215, 19, 122, 351, 81, 34, 75, 185, 350, 131, 306, 184, 324, 146, 11, 30, 294, 140, 18, 156, 320, 20, 236, 277, 10, 175, 116, 9, 305, 150, 152, 211, 367, 207, 233, 291, 107, 205, 74, 182, 147, 317, 80, 326, 237, 246, 229] + 3: [66, 171, 13, 62, 252, 331, 201, 120, 58, 231, 74, 316, 12, 87, 78, 288, 203, 374, 375, 299, 344, 197, 266, 280, 144, 366, 96, 249, 128, 7, 356, 321, 86, 133, 379, 140, 121, 329, 1, 95, 175, 60, 226, 233, 324, 47, 222, 370, 34, 14, 131, 80, 158, 180, 139, 178, 164, 37, 287, 174, 25, 118, 279, 142, 32, 111, 300, 274, 256, 57, 65, 246, 30, 160, 340, 70, 67, 15, 85, 346, 254, 206, 116, 220, 191, 45, 253, 352, 315, 69, 341, 257, 198, 311, 38, 301, 310, 27, 3, 275, 19, 192, 24, 150, 6, 102, 336, 2, 212, 239, 219, 240, 382, 161, 289, 262, 176, 345, 217, 53, 270, 117, 255, 125, 196, 115, 184, 232, 91, 132, 72, 101, 93, 179, 372, 242, 159, 54, 148, 320, 244, 5, 17, 278, 260, 18, 71, 314, 225, 333, 151, 50, 213, 63, 342, 358, 138, 221, 36, 290, 136, 109, 295, 277, 92, 98, 103, 55, 327, 135, 48, 328, 100, 353, 188, 305, 165, 381, 362, 293, 258, 183, 16, 210, 360, 90, 330, 349, 347, 371, 247, 292, 377, 40, 313, 230, 31, 97, 348, 49, 337, 339, 343, 211, 189, 304, 119, 21, 264, 8, 359, 312, 114, 41, 218, 235, 112, 105, 76, 214, 276, 286, 20, 281, 106, 259, 110, 269, 285, 245, 251, 338, 369, 147, 75, 124, 152, 33, 332, 303, 365, 302, 163, 46, 317, 261, 157, 81, 42, 238, 207, 10, 267, 296, 182, 223, 357, 162, 154, 149, 243, 126, 224, 380, 355, 129, 268, 82, 237, 322, 361, 291, 83, 61, 68, 309, 190, 0, 323, 107, 284, 368, 89, 265, 173, 167, 335, 59, 319, 79, 263, 209, 51, 208, 363, 334, 137, 26, 168, 227, 325, 195, 177, 29, 271, 364, 99, 351, 297, 204, 108, 236, 169, 273, 44, 104, 306, 248, 187, 294, 376, 130, 228, 153, 354, 378, 73, 156, 141, 205, 127, 308, 326, 185, 52, 199, 39, 22, 194, 35, 350, 56, 318, 145, 215, 186, 170, 250, 122, 77, 11, 64, 202, 84, 172, 229, 155, 181, 200, 166, 94, 307, 113, 383, 43, 216, 298, 367, 123, 373, 282, 28, 283, 134, 88, 234, 143, 241, 272, 23, 4, 193, 9, 146] + 4: [343, 148, 375, 259, 228, 257, 302, 345, 274, 76, 200, 361, 44, 312, 251, 35, 278, 276, 290, 133, 62, 379, 261, 64, 313, 94, 284, 39, 306, 121, 26, 180, 304, 201, 355, 96, 115, 353, 55, 368, 190, 285, 154, 2, 242, 120, 182, 277, 331, 142, 37, 24, 239, 105, 143, 253, 155, 41, 275, 357, 160, 8, 198, 187, 38, 335, 264, 189, 330, 169, 149, 229, 70, 240, 370, 220, 349, 303, 175, 249, 334, 40, 286, 126, 97, 163, 162, 366, 224, 301, 174, 20, 67, 336, 332, 351, 3, 319, 254, 12, 219, 88, 5, 136, 47, 329, 113, 77, 364, 101, 333, 22, 110, 283, 244, 0, 166, 72, 185, 341, 216, 232, 158, 288, 66, 52, 346, 122, 170, 118, 71, 213, 311, 203, 316, 247, 30, 245, 363, 227, 119, 272, 58, 282, 271, 226, 152, 124, 350, 172, 45, 241, 13, 197, 139, 318, 196, 27, 28, 280, 369, 179, 324, 243, 53, 356, 297, 323, 46, 262, 354, 84, 138, 267, 78, 103, 291, 147, 111, 255, 205, 378, 362, 191, 238, 293, 326, 337, 192, 321, 63, 305, 315, 95, 117, 18, 300, 16, 376, 308, 36, 377, 325, 61, 372, 57, 83, 217, 73, 260, 207, 358, 42, 168, 360, 352, 132, 269, 383, 25, 310, 92, 50, 230, 29, 327, 211, 223, 10, 186, 374, 279, 173, 134, 248, 89, 382, 17, 79, 90, 214, 15, 273, 340, 258, 193, 19, 6, 86, 59, 167, 246, 151, 235, 43, 49, 7, 82, 141, 183, 116, 307, 98, 93, 14, 108, 69, 100, 137, 292, 176, 234, 157, 140, 210, 31, 127, 215, 48, 112, 91, 202, 11, 212, 195, 225, 338, 107, 209, 23, 194, 34, 222, 320, 281, 171, 348, 56, 347, 4, 32, 314, 218, 344, 161, 177, 159, 99, 85, 65, 114, 359, 75, 125, 80, 135, 367, 309, 263, 184, 51, 181, 237, 109, 130, 81, 129, 380, 1, 265, 373, 236, 270, 208, 231, 104, 296, 371, 87, 188, 199, 250, 9, 150, 165, 342, 298, 145, 106, 365, 339, 204, 68, 317, 153, 54, 221, 252, 256, 33, 60, 74, 131, 322, 328, 178, 294, 206, 295, 146, 289, 268, 156, 381, 21, 123, 164, 144, 102, 266, 299, 233, 128, 287] + 5: [61, 305, 231, 141, 1, 223, 365, 64, 132, 200, 91, 182, 304, 70, 268, 238, 319, 24, 34, 220, 295, 186, 312, 19, 106, 250, 161, 90, 334, 236, 162, 55, 283, 378, 244, 58, 373, 111, 307, 282, 158, 294, 112, 348, 167, 57, 196, 302, 213, 105, 272, 154, 98, 265, 381, 41, 351, 7, 192, 74, 233, 313, 163, 264, 38, 207, 258, 377, 35, 144, 274, 227, 366, 300, 219, 289, 237, 360, 94, 376, 221, 263, 135, 92, 125, 101, 126, 85, 146, 42, 339, 142, 225, 50, 118, 354, 128, 100, 6, 165, 260, 49, 30, 379, 279, 88, 247, 22, 173, 137, 222, 174, 206, 3, 320, 315, 228, 79, 242, 183, 353, 240, 150, 325, 330, 13, 77, 62, 113, 129, 68, 316, 189, 290, 120, 73, 36, 123, 310, 176, 278, 286, 127, 75, 214, 276, 288, 208, 160, 60, 169, 78, 215, 352, 89, 335, 371, 155, 37, 369, 67, 143, 314, 218, 104, 115, 216, 39, 343, 121, 257, 203, 275, 149, 29, 191, 235, 71, 270, 298, 297, 347, 185, 309, 328, 357, 273, 10, 367, 157, 321, 177, 355, 346, 52, 172, 5, 178, 188, 102, 145, 43, 147, 25, 31, 8, 375, 2, 69, 204, 26, 45, 301, 99, 12, 253, 14, 239, 138, 136, 119, 9, 344, 232, 359, 103, 107, 168, 322, 245, 292, 267, 97, 336, 180, 27, 110, 82, 179, 303, 190, 217, 287, 259, 256, 65, 193, 211, 130, 17, 246, 338, 327, 66, 15, 358, 108, 318, 175, 333, 16, 195, 266, 329, 20, 382, 63, 284, 281, 80, 291, 271, 374, 299, 199, 40, 148, 32, 53, 285, 243, 340, 109, 116, 280, 96, 277, 368, 249, 194, 356, 21, 308, 81, 370, 372, 72, 212, 331, 364, 124, 131, 171, 59, 181, 140, 210, 153, 54, 28, 51, 345, 230, 362, 341, 76, 383, 87, 134, 229, 363, 114, 18, 156, 224, 95, 11, 139, 350, 234, 117, 248, 83, 164, 337, 44, 84, 296, 269, 159, 306, 47, 262, 122, 255, 252, 293, 198, 209, 4, 151, 332, 184, 251, 324, 152, 33, 46, 380, 93, 56, 201, 241, 317, 254, 48, 361, 349, 205, 323, 342, 187, 261, 86, 133, 197, 311, 202, 0, 226, 166, 326, 23, 170] + 6: [97, 281, 212, 306, 46, 371, 36, 318, 119, 246, 34, 239, 0, 113, 352, 236, 223, 202, 44, 211, 98, 280, 207, 116, 197, 261, 148, 199, 124, 349, 381, 366, 285, 152, 328, 117, 303, 183, 302, 29, 91, 344, 61, 65, 167, 35, 40, 90, 142, 240, 297, 249, 11, 67, 378, 216, 293, 84, 208, 129, 151, 176, 377, 278, 304, 111, 383, 219, 136, 365, 237, 242, 86, 217, 138, 81, 198, 224, 54, 309, 235, 130, 360, 162, 338, 313, 339, 92, 19, 93, 228, 153, 214, 320, 161, 164, 308, 253, 146, 27, 229, 31, 375, 181, 100, 112, 287, 289, 234, 165, 369, 336, 101, 190, 139, 327, 10, 157, 22, 110, 284, 23, 194, 316, 83, 105, 96, 16, 379, 357, 74, 337, 256, 348, 89, 356, 140, 127, 172, 353, 252, 363, 325, 247, 322, 258, 177, 160, 55, 120, 173, 163, 88, 368, 196, 184, 300, 301, 109, 245, 317, 37, 321, 77, 15, 154, 210, 69, 159, 243, 254, 329, 333, 346, 215, 42, 134, 292, 62, 108, 266, 351, 222, 250, 78, 141, 273, 200, 75, 149, 71, 323, 76, 174, 166, 103, 341, 205, 350, 359, 232, 189, 118, 291, 345, 277, 185, 380, 231, 279, 133, 64, 264, 58, 6, 104, 276, 82, 362, 122, 168, 358, 8, 70, 310, 312, 203, 288, 102, 294, 131, 201, 238, 227, 340, 218, 132, 135, 204, 257, 274, 343, 66, 295, 269, 180, 45, 307, 187, 57, 188, 178, 259, 95, 179, 41, 311, 158, 221, 107, 332, 286, 271, 87, 225, 330, 355, 145, 18, 193, 298, 268, 364, 244, 80, 347, 290, 51, 21, 374, 171, 241, 1, 20, 72, 25, 275, 342, 376, 53, 99, 59, 267, 209, 9, 260, 94, 56, 255, 79, 370, 270, 128, 182, 49, 12, 282, 144, 324, 220, 262, 206, 63, 335, 372, 126, 48, 3, 334, 299, 73, 123, 106, 226, 85, 24, 47, 38, 7, 192, 382, 2, 265, 33, 315, 373, 32, 305, 233, 5, 28, 326, 175, 13, 367, 283, 319, 155, 156, 354, 115, 230, 68, 30, 169, 114, 143, 296, 361, 43, 137, 60, 4, 14, 248, 50, 147, 272, 125, 39, 150, 251, 17, 195, 121, 263, 26, 213, 170, 191, 314, 52, 186, 331] + 7: [327, 265, 25, 272, 133, 271, 378, 328, 27, 73, 164, 248, 349, 288, 325, 208, 219, 191, 264, 9, 311, 204, 32, 178, 335, 121, 353, 373, 146, 68, 153, 79, 151, 287, 295, 1, 312, 3, 63, 198, 346, 12, 347, 154, 279, 110, 202, 364, 7, 229, 201, 157, 239, 46, 99, 329, 61, 281, 313, 267, 4, 31, 360, 284, 255, 62, 300, 214, 189, 245, 237, 107, 200, 86, 226, 323, 196, 291, 93, 203, 269, 106, 112, 72, 123, 39, 381, 159, 95, 66, 165, 138, 376, 236, 135, 257, 274, 142, 103, 162, 361, 342, 118, 212, 370, 302, 294, 82, 262, 23, 148, 235, 210, 377, 344, 167, 368, 109, 130, 5, 83, 44, 134, 301, 40, 261, 101, 351, 318, 166, 169, 218, 254, 379, 124, 315, 359, 150, 172, 127, 355, 30, 160, 50, 49, 375, 340, 369, 108, 28, 71, 188, 0, 94, 324, 100, 217, 119, 84, 250, 70, 170, 213, 132, 206, 26, 136, 190, 293, 102, 152, 310, 161, 8, 22, 345, 126, 320, 363, 69, 85, 111, 252, 228, 242, 304, 197, 143, 34, 326, 371, 42, 292, 41, 18, 341, 258, 187, 13, 97, 244, 131, 183, 47, 289, 285, 2, 319, 356, 21, 90, 241, 234, 140, 20, 180, 211, 58, 51, 53, 232, 227, 87, 52, 366, 35, 209, 194, 88, 173, 297, 334, 246, 45, 205, 54, 282, 240, 365, 60, 175, 64, 105, 163, 336, 176, 139, 15, 174, 181, 14, 215, 278, 17, 270, 296, 81, 224, 330, 333, 259, 16, 92, 19, 220, 380, 141, 56, 251, 309, 38, 193, 184, 216, 286, 316, 10, 367, 277, 78, 266, 57, 147, 331, 156, 223, 348, 171, 182, 263, 382, 298, 308, 260, 253, 48, 104, 114, 11, 113, 74, 168, 337, 374, 383, 177, 158, 98, 321, 89, 358, 129, 322, 6, 305, 306, 283, 247, 149, 120, 37, 222, 33, 207, 96, 231, 317, 128, 195, 230, 290, 144, 122, 179, 91, 357, 275, 192, 343, 362, 238, 350, 280, 354, 77, 256, 249, 307, 125, 29, 55, 339, 117, 221, 372, 243, 24, 314, 185, 186, 137, 273, 67, 155, 233, 65, 116, 225, 115, 268, 75, 36, 59, 80, 43, 303, 299, 332, 352, 199, 76, 145, 276, 338] + 8: [360, 308, 273, 238, 224, 272, 36, 252, 316, 364, 131, 128, 75, 109, 119, 330, 165, 219, 198, 113, 233, 52, 114, 101, 263, 379, 191, 90, 310, 136, 327, 200, 325, 236, 93, 323, 351, 324, 129, 340, 123, 305, 139, 344, 57, 262, 58, 234, 253, 162, 371, 279, 106, 336, 228, 276, 157, 294, 97, 311, 69, 23, 226, 179, 17, 303, 167, 196, 177, 125, 8, 284, 383, 25, 116, 249, 32, 382, 143, 140, 35, 260, 322, 134, 153, 15, 271, 26, 268, 80, 138, 288, 242, 127, 96, 170, 290, 362, 152, 214, 338, 12, 88, 81, 251, 209, 377, 374, 346, 83, 34, 240, 261, 178, 137, 50, 334, 211, 91, 61, 206, 98, 73, 293, 110, 266, 16, 199, 89, 295, 375, 365, 33, 146, 270, 269, 160, 355, 274, 301, 49, 163, 1, 207, 359, 201, 353, 104, 292, 13, 381, 74, 186, 349, 168, 148, 181, 188, 333, 258, 78, 275, 368, 183, 300, 141, 332, 60, 45, 117, 174, 71, 297, 47, 37, 126, 99, 339, 306, 14, 347, 24, 76, 158, 77, 245, 43, 203, 345, 176, 55, 102, 5, 239, 221, 281, 218, 108, 243, 337, 20, 255, 328, 280, 166, 22, 27, 283, 202, 169, 304, 194, 320, 192, 2, 358, 335, 217, 28, 232, 350, 299, 41, 155, 105, 85, 159, 369, 287, 357, 220, 244, 208, 285, 51, 53, 124, 0, 227, 132, 67, 264, 321, 204, 3, 302, 241, 64, 122, 380, 11, 86, 225, 172, 356, 147, 92, 65, 215, 112, 317, 120, 318, 4, 42, 254, 87, 247, 151, 373, 372, 313, 144, 235, 205, 185, 46, 277, 84, 348, 180, 354, 142, 331, 222, 210, 296, 100, 31, 367, 10, 182, 246, 161, 173, 282, 291, 95, 18, 115, 39, 107, 133, 6, 231, 121, 278, 309, 130, 48, 197, 66, 59, 212, 237, 257, 184, 149, 190, 314, 79, 265, 145, 29, 54, 326, 171, 103, 319, 70, 343, 342, 256, 361, 19, 376, 135, 21, 62, 229, 250, 7, 312, 154, 30, 223, 44, 156, 195, 366, 150, 38, 298, 216, 40, 189, 267, 164, 341, 72, 187, 230, 118, 248, 352, 259, 307, 68, 56, 378, 9, 175, 315, 111, 193, 370, 329, 286, 363, 94, 82, 289, 63, 213] + 9: [92, 244, 81, 372, 353, 184, 15, 138, 210, 67, 239, 258, 161, 135, 120, 297, 231, 192, 116, 186, 4, 230, 238, 237, 343, 235, 248, 382, 197, 261, 112, 291, 71, 298, 38, 229, 86, 34, 383, 266, 279, 44, 204, 157, 61, 236, 234, 26, 304, 64, 166, 280, 357, 179, 272, 217, 281, 136, 177, 9, 82, 377, 137, 88, 180, 314, 259, 35, 105, 36, 66, 141, 39, 324, 109, 307, 275, 233, 254, 110, 375, 310, 162, 347, 170, 193, 58, 240, 328, 284, 85, 164, 182, 77, 338, 57, 255, 313, 95, 143, 260, 219, 381, 83, 242, 271, 14, 342, 312, 360, 302, 289, 228, 6, 97, 99, 195, 123, 150, 218, 359, 104, 13, 354, 60, 45, 300, 2, 320, 251, 352, 232, 227, 69, 337, 140, 325, 73, 101, 378, 250, 27, 53, 340, 349, 185, 103, 331, 215, 350, 351, 1, 370, 214, 23, 309, 133, 30, 356, 292, 306, 247, 107, 129, 256, 336, 11, 332, 5, 308, 246, 8, 100, 148, 174, 358, 253, 305, 134, 89, 346, 167, 290, 368, 345, 249, 93, 200, 28, 122, 152, 348, 327, 47, 262, 212, 315, 252, 222, 344, 376, 287, 121, 56, 98, 317, 363, 268, 264, 0, 183, 41, 269, 139, 16, 189, 106, 75, 208, 50, 52, 245, 190, 203, 371, 330, 76, 211, 48, 188, 293, 243, 176, 118, 149, 273, 91, 265, 369, 365, 355, 295, 49, 78, 196, 299, 155, 198, 127, 329, 209, 32, 131, 285, 187, 3, 277, 54, 364, 20, 142, 267, 70, 181, 274, 80, 339, 65, 130, 117, 373, 114, 42, 286, 333, 21, 221, 241, 326, 68, 108, 361, 278, 194, 362, 25, 55, 173, 31, 263, 316, 10, 220, 19, 74, 29, 201, 159, 51, 132, 22, 367, 158, 154, 33, 334, 321, 311, 163, 323, 124, 171, 288, 319, 17, 18, 379, 46, 62, 322, 84, 301, 270, 125, 199, 59, 374, 43, 225, 63, 223, 113, 40, 7, 24, 168, 226, 94, 90, 147, 96, 172, 165, 37, 146, 380, 341, 282, 102, 151, 144, 153, 169, 318, 128, 87, 145, 207, 206, 111, 72, 202, 276, 175, 303, 205, 213, 79, 115, 12, 224, 216, 178, 335, 160, 191, 366, 119, 296, 294, 257, 156, 126, 283] + 10: [319, 310, 137, 325, 309, 73, 1, 263, 241, 196, 308, 91, 293, 259, 342, 360, 212, 223, 145, 90, 321, 21, 81, 329, 20, 338, 383, 36, 247, 193, 123, 343, 307, 350, 100, 173, 303, 168, 165, 334, 157, 344, 63, 14, 89, 253, 68, 369, 5, 146, 268, 289, 331, 102, 134, 179, 351, 330, 211, 67, 272, 363, 374, 148, 306, 149, 155, 103, 214, 288, 311, 339, 256, 37, 230, 162, 12, 174, 239, 185, 282, 96, 159, 222, 355, 4, 158, 94, 273, 30, 84, 277, 160, 166, 358, 336, 300, 164, 192, 48, 167, 225, 170, 161, 40, 182, 201, 213, 234, 368, 186, 208, 249, 285, 250, 0, 72, 296, 219, 9, 197, 41, 51, 71, 184, 317, 50, 183, 65, 299, 80, 187, 373, 35, 200, 118, 172, 352, 152, 52, 238, 176, 295, 283, 320, 121, 301, 181, 337, 2, 144, 54, 153, 210, 304, 58, 353, 252, 356, 204, 189, 280, 266, 233, 46, 3, 122, 335, 26, 346, 15, 258, 64, 275, 323, 261, 226, 364, 139, 206, 86, 17, 6, 34, 83, 207, 156, 242, 110, 236, 29, 297, 365, 24, 31, 294, 108, 279, 79, 195, 178, 114, 177, 154, 115, 130, 240, 377, 190, 376, 382, 132, 33, 129, 232, 85, 27, 357, 379, 25, 109, 318, 45, 313, 298, 113, 209, 62, 372, 119, 217, 16, 203, 281, 333, 202, 270, 381, 70, 366, 194, 265, 18, 246, 221, 126, 101, 22, 78, 66, 237, 269, 7, 243, 345, 274, 98, 38, 218, 128, 198, 107, 105, 106, 314, 112, 74, 97, 278, 354, 140, 362, 370, 244, 169, 302, 111, 147, 224, 133, 276, 44, 347, 227, 28, 43, 23, 324, 264, 151, 216, 93, 141, 367, 57, 215, 171, 138, 99, 248, 380, 312, 348, 284, 77, 231, 61, 32, 116, 87, 56, 75, 315, 378, 327, 267, 340, 39, 254, 163, 175, 220, 131, 287, 127, 305, 322, 245, 53, 104, 191, 143, 10, 188, 82, 349, 55, 180, 286, 255, 262, 69, 92, 8, 117, 19, 124, 47, 76, 11, 136, 49, 316, 271, 359, 199, 229, 135, 341, 60, 142, 59, 328, 371, 42, 125, 332, 95, 13, 257, 326, 375, 235, 205, 292, 251, 88, 120, 361, 150, 260, 228, 291, 290] + 11: [130, 221, 126, 299, 149, 190, 29, 218, 192, 141, 39, 20, 13, 250, 283, 107, 348, 92, 195, 86, 341, 255, 101, 327, 63, 30, 338, 179, 125, 264, 113, 62, 143, 184, 360, 222, 240, 132, 380, 229, 180, 181, 202, 123, 220, 321, 134, 243, 314, 235, 100, 77, 363, 145, 102, 22, 247, 330, 95, 381, 207, 320, 161, 45, 249, 226, 315, 340, 316, 14, 208, 301, 219, 59, 353, 193, 234, 383, 142, 263, 230, 271, 266, 112, 310, 167, 68, 183, 60, 53, 28, 239, 223, 25, 109, 12, 296, 150, 248, 156, 65, 119, 300, 236, 217, 343, 286, 318, 17, 297, 47, 280, 2, 373, 172, 151, 89, 298, 186, 270, 211, 168, 354, 349, 291, 293, 174, 182, 46, 194, 375, 210, 21, 214, 278, 277, 175, 369, 189, 276, 281, 72, 191, 335, 163, 85, 44, 252, 3, 99, 325, 345, 351, 284, 73, 177, 154, 313, 6, 124, 127, 159, 166, 374, 203, 96, 254, 242, 294, 245, 359, 91, 164, 57, 129, 188, 309, 274, 103, 106, 4, 140, 304, 328, 216, 139, 51, 206, 334, 50, 308, 269, 110, 215, 116, 260, 88, 233, 267, 332, 331, 128, 204, 282, 136, 23, 319, 170, 262, 75, 347, 10, 224, 289, 41, 81, 336, 279, 364, 302, 187, 64, 43, 7, 74, 231, 5, 323, 201, 322, 305, 379, 344, 246, 253, 76, 212, 137, 307, 241, 98, 83, 11, 357, 40, 326, 199, 24, 152, 165, 131, 228, 205, 303, 94, 367, 69, 117, 176, 56, 49, 197, 259, 35, 147, 292, 378, 158, 118, 115, 48, 18, 138, 358, 37, 84, 288, 290, 71, 342, 361, 352, 111, 82, 153, 256, 32, 42, 377, 90, 121, 15, 227, 38, 162, 0, 54, 178, 244, 146, 169, 196, 346, 213, 370, 80, 26, 251, 133, 365, 58, 209, 362, 366, 34, 135, 324, 79, 258, 355, 237, 382, 108, 306, 173, 287, 268, 114, 238, 317, 333, 155, 329, 273, 261, 9, 339, 87, 36, 285, 376, 171, 157, 144, 275, 52, 70, 337, 148, 78, 66, 371, 350, 27, 93, 160, 198, 55, 31, 97, 295, 265, 312, 311, 67, 120, 33, 225, 8, 19, 200, 356, 185, 104, 272, 1, 368, 16, 372, 122, 232, 257, 61, 105] + 12: [255, 35, 167, 22, 103, 306, 170, 75, 332, 152, 368, 10, 39, 101, 271, 129, 261, 318, 358, 357, 200, 231, 119, 256, 249, 325, 38, 121, 20, 293, 329, 148, 352, 44, 34, 343, 299, 247, 182, 269, 228, 88, 166, 66, 222, 220, 302, 42, 204, 177, 203, 381, 168, 296, 233, 369, 176, 195, 205, 348, 229, 277, 153, 102, 107, 135, 326, 14, 286, 142, 1, 315, 61, 62, 161, 321, 76, 154, 65, 32, 244, 217, 85, 252, 316, 113, 117, 320, 210, 145, 242, 100, 359, 239, 53, 225, 120, 82, 333, 48, 175, 289, 192, 361, 354, 339, 68, 194, 25, 60, 2, 300, 322, 294, 291, 157, 155, 376, 9, 191, 45, 138, 280, 58, 11, 40, 13, 183, 365, 263, 274, 310, 345, 131, 237, 125, 79, 26, 163, 206, 377, 240, 162, 272, 130, 223, 172, 363, 344, 349, 97, 181, 208, 295, 382, 330, 87, 16, 209, 371, 362, 63, 337, 23, 15, 201, 350, 41, 29, 298, 139, 187, 122, 159, 257, 86, 186, 128, 383, 105, 375, 55, 212, 258, 307, 338, 360, 213, 21, 331, 73, 110, 158, 127, 150, 49, 17, 124, 290, 51, 340, 134, 137, 215, 147, 89, 288, 116, 57, 164, 380, 74, 283, 253, 188, 114, 165, 67, 4, 173, 214, 270, 379, 112, 284, 207, 106, 146, 282, 96, 81, 230, 143, 94, 202, 248, 95, 56, 199, 372, 327, 303, 243, 156, 28, 109, 126, 50, 98, 259, 281, 216, 317, 108, 133, 160, 171, 180, 241, 59, 144, 193, 227, 52, 234, 224, 136, 232, 335, 115, 351, 12, 336, 246, 374, 70, 251, 178, 347, 236, 196, 111, 132, 328, 92, 324, 373, 250, 149, 190, 378, 46, 245, 267, 71, 90, 198, 123, 278, 346, 268, 323, 27, 33, 264, 197, 308, 31, 91, 304, 370, 185, 0, 84, 313, 37, 169, 235, 36, 140, 5, 297, 24, 174, 30, 8, 54, 72, 367, 184, 221, 342, 275, 64, 353, 366, 78, 226, 69, 341, 219, 292, 287, 151, 260, 3, 6, 276, 218, 266, 364, 265, 104, 273, 80, 238, 301, 309, 77, 254, 83, 189, 262, 356, 319, 355, 305, 334, 179, 312, 279, 285, 47, 93, 18, 211, 19, 7, 118, 314, 311, 141, 99, 43] + 13: [170, 274, 3, 41, 276, 330, 366, 370, 159, 176, 184, 31, 202, 143, 320, 6, 207, 355, 141, 145, 74, 333, 243, 221, 105, 161, 326, 84, 296, 50, 250, 72, 115, 150, 103, 138, 346, 101, 28, 278, 293, 236, 47, 351, 314, 126, 192, 175, 108, 23, 30, 77, 86, 135, 100, 310, 11, 306, 118, 58, 75, 273, 309, 67, 178, 300, 79, 179, 382, 361, 151, 177, 164, 303, 267, 379, 220, 305, 325, 308, 270, 217, 287, 104, 233, 383, 117, 63, 90, 181, 269, 354, 120, 277, 140, 155, 232, 328, 238, 57, 96, 231, 254, 197, 88, 171, 279, 295, 369, 356, 246, 292, 373, 378, 219, 237, 144, 283, 374, 129, 52, 8, 331, 153, 174, 257, 348, 71, 154, 234, 213, 214, 185, 167, 284, 212, 152, 224, 272, 368, 160, 66, 33, 230, 18, 149, 12, 194, 265, 282, 55, 102, 338, 162, 323, 227, 364, 275, 261, 158, 208, 22, 10, 43, 239, 225, 93, 183, 380, 34, 163, 106, 357, 268, 59, 113, 358, 13, 124, 119, 182, 372, 336, 352, 258, 347, 341, 156, 82, 193, 315, 316, 195, 14, 260, 81, 0, 201, 110, 17, 199, 46, 131, 45, 16, 289, 294, 281, 334, 65, 147, 62, 25, 262, 166, 327, 222, 125, 89, 68, 345, 92, 245, 187, 123, 19, 313, 190, 205, 85, 226, 60, 264, 20, 317, 148, 255, 247, 350, 78, 87, 114, 280, 9, 169, 21, 342, 271, 142, 298, 35, 198, 112, 297, 91, 229, 299, 210, 375, 321, 26, 291, 253, 256, 76, 241, 288, 248, 136, 259, 244, 133, 360, 73, 235, 353, 285, 157, 252, 203, 99, 251, 332, 200, 322, 165, 324, 7, 80, 329, 189, 318, 24, 172, 209, 134, 186, 109, 116, 111, 188, 69, 40, 363, 2, 42, 168, 240, 228, 196, 359, 139, 15, 223, 36, 337, 56, 242, 263, 311, 70, 38, 5, 365, 83, 4, 343, 191, 128, 97, 349, 302, 367, 206, 27, 32, 44, 362, 312, 137, 39, 51, 95, 340, 335, 304, 307, 64, 49, 376, 54, 266, 122, 98, 211, 339, 344, 107, 121, 146, 29, 37, 48, 286, 319, 127, 381, 53, 94, 377, 130, 61, 249, 180, 1, 215, 173, 371, 132, 290, 218, 301, 216, 204] + 14: [229, 355, 274, 8, 268, 348, 228, 95, 342, 154, 279, 41, 13, 158, 123, 319, 68, 37, 339, 286, 296, 90, 156, 237, 200, 189, 12, 183, 96, 285, 302, 329, 127, 74, 305, 179, 45, 366, 139, 140, 144, 65, 280, 53, 292, 249, 260, 174, 160, 209, 378, 261, 60, 239, 146, 141, 117, 371, 269, 44, 166, 33, 149, 163, 324, 145, 266, 169, 256, 226, 116, 221, 306, 316, 89, 282, 244, 376, 94, 321, 309, 267, 361, 190, 258, 7, 62, 180, 107, 219, 188, 303, 210, 351, 295, 289, 345, 2, 327, 147, 137, 17, 118, 142, 32, 97, 162, 250, 216, 197, 59, 246, 69, 370, 22, 333, 349, 227, 373, 105, 330, 225, 335, 148, 224, 251, 106, 214, 257, 365, 334, 19, 299, 80, 293, 185, 233, 252, 113, 54, 340, 67, 315, 165, 255, 30, 126, 56, 287, 383, 150, 99, 52, 39, 213, 111, 294, 208, 135, 206, 184, 159, 301, 133, 325, 234, 5, 377, 298, 0, 115, 191, 350, 240, 28, 6, 153, 195, 238, 16, 230, 132, 55, 367, 236, 283, 170, 131, 352, 98, 130, 331, 70, 110, 34, 71, 372, 49, 353, 23, 85, 177, 61, 182, 157, 270, 25, 328, 288, 326, 167, 104, 203, 245, 101, 347, 176, 253, 202, 242, 356, 248, 311, 173, 278, 338, 20, 9, 312, 31, 47, 35, 181, 307, 51, 243, 232, 119, 73, 50, 211, 204, 259, 308, 77, 112, 186, 254, 198, 231, 360, 21, 354, 220, 379, 343, 357, 193, 108, 346, 102, 109, 222, 129, 276, 359, 223, 152, 364, 43, 42, 262, 358, 36, 78, 58, 263, 281, 57, 92, 11, 247, 72, 27, 29, 138, 318, 88, 143, 76, 122, 275, 207, 175, 337, 235, 124, 64, 317, 271, 26, 125, 161, 215, 332, 369, 363, 382, 128, 178, 82, 380, 284, 86, 168, 264, 1, 63, 310, 273, 375, 46, 136, 14, 322, 194, 66, 84, 81, 320, 192, 368, 38, 196, 83, 121, 4, 15, 75, 155, 164, 199, 344, 151, 100, 3, 79, 374, 265, 314, 297, 323, 291, 201, 277, 40, 187, 172, 48, 241, 341, 336, 300, 313, 217, 93, 10, 114, 103, 171, 304, 290, 87, 18, 362, 24, 120, 381, 91, 134, 218, 272, 212, 205] + 15: [130, 258, 68, 259, 39, 236, 215, 80, 151, 56, 277, 174, 143, 164, 162, 78, 182, 247, 212, 86, 150, 282, 81, 43, 382, 347, 53, 137, 45, 37, 337, 116, 218, 91, 73, 129, 126, 380, 361, 299, 304, 110, 21, 336, 245, 144, 260, 270, 11, 200, 298, 295, 300, 159, 146, 237, 376, 83, 342, 177, 111, 115, 36, 47, 5, 194, 102, 263, 34, 331, 254, 165, 30, 226, 308, 208, 55, 64, 95, 213, 54, 79, 161, 158, 140, 280, 171, 44, 178, 243, 122, 220, 348, 322, 368, 276, 0, 355, 104, 350, 371, 326, 284, 60, 221, 16, 113, 311, 138, 24, 225, 335, 323, 305, 2, 248, 229, 50, 233, 253, 141, 306, 354, 244, 204, 71, 172, 257, 190, 147, 197, 188, 149, 310, 292, 154, 179, 286, 302, 345, 19, 6, 325, 85, 15, 155, 148, 76, 199, 332, 252, 20, 360, 48, 193, 135, 255, 379, 285, 319, 372, 42, 275, 142, 114, 69, 8, 12, 294, 67, 72, 234, 224, 211, 334, 265, 278, 210, 303, 262, 201, 327, 136, 98, 183, 108, 288, 249, 315, 75, 366, 328, 88, 324, 175, 340, 291, 231, 119, 9, 329, 40, 344, 163, 320, 235, 106, 357, 49, 145, 279, 309, 77, 96, 206, 13, 339, 118, 381, 289, 272, 92, 375, 167, 3, 239, 369, 370, 312, 7, 124, 181, 41, 307, 156, 214, 46, 25, 93, 356, 359, 251, 217, 261, 367, 185, 333, 90, 317, 23, 296, 256, 242, 196, 352, 351, 170, 238, 105, 117, 87, 268, 205, 341, 377, 349, 321, 267, 314, 169, 283, 316, 240, 65, 378, 184, 14, 187, 58, 112, 66, 103, 346, 219, 297, 152, 264, 281, 160, 362, 128, 290, 202, 38, 74, 121, 232, 139, 353, 168, 241, 109, 31, 338, 133, 28, 101, 59, 269, 10, 22, 250, 271, 313, 180, 61, 365, 123, 383, 358, 63, 273, 274, 89, 374, 176, 293, 222, 195, 120, 17, 125, 132, 27, 131, 127, 227, 57, 94, 134, 4, 189, 266, 203, 51, 157, 35, 173, 1, 18, 230, 207, 100, 209, 246, 166, 82, 343, 62, 363, 301, 191, 107, 97, 99, 216, 26, 373, 84, 52, 364, 153, 32, 29, 318, 330, 228, 223, 70, 287, 186, 33, 198, 192] + 16: [134, 118, 330, 185, 49, 342, 302, 383, 187, 235, 123, 294, 306, 2, 248, 73, 58, 153, 219, 356, 42, 301, 96, 188, 252, 200, 102, 333, 343, 109, 207, 363, 81, 373, 258, 371, 241, 61, 243, 63, 40, 254, 326, 361, 10, 320, 156, 249, 15, 303, 67, 331, 86, 318, 78, 283, 189, 270, 212, 65, 365, 307, 99, 21, 56, 380, 227, 70, 379, 9, 4, 106, 29, 336, 214, 337, 93, 105, 201, 62, 14, 128, 113, 110, 256, 138, 251, 355, 74, 329, 327, 362, 53, 94, 366, 257, 286, 44, 68, 125, 338, 289, 250, 298, 287, 87, 347, 285, 310, 97, 37, 103, 281, 311, 8, 167, 36, 160, 370, 124, 228, 358, 211, 242, 83, 378, 240, 163, 88, 284, 131, 117, 279, 210, 260, 6, 349, 136, 192, 204, 133, 111, 368, 353, 120, 7, 269, 11, 154, 129, 5, 374, 255, 213, 146, 346, 27, 314, 115, 38, 119, 139, 282, 34, 195, 45, 19, 339, 104, 264, 158, 18, 95, 272, 55, 193, 172, 127, 324, 168, 170, 359, 230, 288, 79, 351, 173, 197, 222, 161, 126, 1, 275, 92, 64, 271, 300, 143, 259, 224, 265, 141, 367, 16, 89, 321, 237, 51, 175, 155, 31, 76, 220, 122, 174, 238, 225, 354, 208, 54, 178, 304, 190, 266, 186, 157, 47, 261, 116, 50, 332, 169, 382, 130, 341, 72, 278, 191, 247, 98, 100, 262, 165, 280, 277, 144, 348, 183, 171, 364, 246, 180, 108, 177, 328, 377, 292, 159, 231, 182, 232, 334, 276, 22, 317, 151, 340, 313, 253, 267, 244, 30, 381, 166, 107, 20, 216, 77, 149, 352, 164, 12, 350, 179, 296, 345, 309, 344, 140, 234, 43, 162, 114, 90, 205, 35, 24, 295, 236, 199, 85, 84, 59, 316, 82, 0, 41, 184, 229, 202, 293, 322, 198, 221, 25, 57, 206, 319, 48, 142, 194, 233, 145, 268, 308, 176, 150, 33, 28, 112, 299, 263, 325, 13, 215, 3, 369, 152, 147, 75, 312, 273, 203, 274, 209, 121, 46, 226, 181, 52, 71, 335, 80, 66, 39, 305, 357, 372, 137, 297, 290, 375, 132, 60, 223, 323, 91, 196, 360, 315, 218, 17, 26, 217, 245, 101, 32, 23, 135, 291, 376, 69, 239, 148] + 17: [131, 383, 374, 226, 87, 311, 185, 113, 351, 6, 52, 96, 22, 97, 180, 98, 272, 165, 171, 332, 19, 116, 41, 363, 382, 89, 271, 379, 93, 244, 345, 48, 55, 326, 233, 163, 315, 378, 30, 66, 291, 334, 268, 301, 54, 69, 72, 121, 119, 15, 28, 137, 366, 136, 343, 114, 328, 234, 167, 339, 218, 145, 243, 10, 298, 282, 245, 191, 248, 173, 260, 204, 196, 186, 95, 360, 195, 310, 16, 112, 278, 364, 150, 164, 316, 23, 141, 257, 11, 285, 17, 281, 169, 152, 159, 170, 284, 200, 362, 208, 126, 78, 279, 266, 189, 175, 172, 39, 194, 381, 228, 103, 33, 214, 142, 259, 20, 45, 42, 49, 198, 341, 21, 94, 300, 102, 183, 217, 369, 76, 270, 32, 355, 149, 176, 349, 129, 306, 35, 321, 14, 122, 202, 203, 236, 108, 305, 138, 346, 219, 359, 5, 246, 347, 370, 222, 104, 199, 177, 313, 143, 84, 327, 168, 264, 13, 36, 336, 118, 221, 309, 181, 27, 240, 120, 329, 287, 372, 25, 318, 53, 83, 88, 201, 4, 368, 348, 322, 333, 86, 127, 235, 79, 303, 330, 307, 190, 254, 193, 197, 323, 133, 75, 8, 157, 73, 148, 211, 367, 139, 146, 162, 376, 166, 331, 147, 216, 377, 101, 338, 44, 238, 361, 85, 64, 308, 255, 350, 65, 132, 182, 109, 223, 178, 247, 51, 156, 123, 237, 251, 337, 188, 294, 80, 58, 91, 117, 232, 130, 29, 297, 56, 207, 335, 206, 273, 92, 135, 263, 70, 18, 61, 59, 269, 81, 187, 3, 342, 179, 249, 258, 161, 267, 158, 124, 31, 344, 352, 40, 230, 184, 242, 289, 111, 209, 174, 283, 312, 375, 250, 26, 105, 319, 357, 225, 275, 1, 324, 276, 290, 134, 296, 231, 380, 371, 256, 74, 9, 340, 47, 60, 7, 154, 365, 37, 253, 265, 262, 299, 210, 128, 106, 192, 63, 57, 153, 227, 277, 224, 160, 71, 353, 252, 293, 320, 325, 261, 220, 100, 212, 140, 12, 155, 286, 2, 125, 62, 107, 317, 46, 24, 38, 373, 99, 292, 314, 82, 304, 43, 215, 295, 115, 205, 213, 110, 302, 90, 239, 151, 274, 67, 50, 144, 241, 280, 356, 229, 288, 77, 0, 68, 34, 358, 354] + 18: [271, 169, 136, 26, 31, 144, 285, 3, 165, 46, 366, 73, 359, 233, 109, 22, 201, 50, 8, 176, 150, 9, 335, 329, 243, 179, 348, 82, 220, 311, 209, 76, 95, 354, 238, 294, 249, 290, 62, 198, 226, 222, 344, 1, 32, 59, 78, 147, 30, 286, 215, 110, 236, 68, 237, 297, 99, 326, 188, 274, 346, 339, 177, 149, 131, 0, 278, 58, 157, 349, 279, 81, 63, 267, 153, 325, 256, 175, 332, 259, 197, 375, 364, 86, 44, 352, 230, 18, 199, 323, 186, 80, 227, 382, 309, 340, 251, 65, 185, 193, 211, 347, 380, 189, 321, 306, 282, 266, 67, 70, 89, 41, 342, 83, 87, 334, 367, 232, 250, 235, 308, 205, 276, 132, 363, 148, 324, 127, 383, 158, 66, 224, 381, 2, 107, 85, 29, 377, 277, 336, 196, 353, 97, 242, 208, 245, 106, 356, 210, 154, 119, 116, 355, 376, 284, 330, 33, 124, 269, 371, 202, 331, 79, 225, 137, 7, 328, 289, 15, 125, 244, 12, 13, 351, 112, 108, 60, 216, 4, 56, 229, 312, 172, 90, 184, 14, 117, 72, 134, 379, 120, 268, 47, 300, 138, 260, 53, 28, 36, 174, 146, 139, 151, 35, 190, 98, 181, 20, 141, 283, 262, 88, 162, 288, 313, 102, 140, 178, 93, 145, 187, 94, 214, 213, 234, 338, 51, 126, 317, 75, 258, 115, 273, 24, 160, 173, 194, 49, 327, 293, 10, 257, 228, 272, 45, 315, 304, 142, 167, 34, 357, 341, 105, 103, 19, 365, 23, 307, 223, 263, 241, 350, 361, 39, 195, 38, 96, 182, 292, 270, 166, 303, 143, 170, 219, 130, 369, 180, 372, 161, 280, 345, 212, 206, 221, 55, 374, 314, 43, 217, 71, 118, 16, 207, 368, 192, 25, 360, 291, 378, 77, 204, 318, 337, 191, 104, 343, 129, 135, 159, 302, 152, 111, 264, 362, 40, 155, 287, 254, 48, 358, 64, 27, 17, 123, 218, 42, 370, 281, 5, 305, 252, 295, 91, 52, 156, 69, 320, 57, 6, 333, 113, 114, 239, 301, 373, 253, 246, 101, 240, 164, 171, 248, 121, 261, 316, 296, 122, 231, 61, 203, 54, 319, 265, 255, 322, 92, 275, 168, 200, 247, 163, 133, 298, 84, 37, 299, 100, 11, 21, 310, 128, 74, 183] + 19: [24, 207, 243, 302, 188, 219, 184, 218, 227, 159, 35, 158, 54, 71, 278, 173, 68, 251, 204, 2, 277, 273, 346, 237, 131, 75, 280, 146, 326, 212, 22, 186, 340, 351, 103, 112, 267, 39, 224, 86, 56, 226, 55, 142, 378, 355, 369, 375, 187, 313, 307, 111, 52, 183, 300, 323, 38, 97, 275, 58, 25, 335, 1, 134, 6, 76, 368, 257, 347, 215, 203, 18, 333, 202, 160, 3, 191, 383, 311, 32, 232, 174, 63, 37, 150, 269, 214, 252, 163, 236, 282, 78, 145, 93, 50, 81, 190, 330, 240, 206, 241, 322, 114, 363, 164, 271, 153, 264, 171, 238, 30, 295, 127, 172, 16, 272, 67, 345, 374, 125, 284, 262, 293, 26, 309, 242, 298, 87, 258, 235, 94, 59, 231, 289, 223, 304, 266, 248, 360, 104, 175, 109, 98, 107, 366, 338, 64, 179, 91, 126, 197, 328, 349, 77, 354, 44, 292, 122, 13, 222, 359, 83, 195, 138, 40, 31, 95, 287, 108, 380, 372, 62, 140, 329, 82, 139, 370, 247, 149, 73, 348, 123, 178, 367, 0, 5, 151, 72, 196, 229, 133, 198, 7, 120, 101, 339, 364, 291, 8, 113, 20, 303, 301, 217, 147, 135, 36, 166, 165, 96, 353, 221, 260, 249, 143, 84, 19, 23, 356, 65, 225, 362, 137, 209, 14, 208, 286, 294, 357, 162, 185, 21, 155, 261, 17, 199, 250, 305, 245, 253, 90, 377, 312, 211, 61, 161, 192, 66, 154, 296, 230, 189, 365, 15, 233, 352, 182, 167, 29, 283, 144, 4, 244, 325, 177, 45, 148, 47, 281, 119, 130, 168, 132, 234, 314, 315, 106, 11, 201, 51, 205, 308, 43, 220, 274, 317, 254, 263, 288, 57, 42, 141, 74, 255, 318, 373, 193, 334, 152, 79, 46, 246, 116, 310, 337, 270, 156, 239, 157, 110, 306, 336, 213, 10, 331, 129, 376, 321, 176, 320, 181, 332, 324, 361, 118, 124, 290, 100, 299, 379, 69, 327, 276, 350, 268, 319, 256, 265, 180, 115, 48, 70, 279, 216, 27, 33, 89, 117, 9, 41, 53, 285, 128, 228, 121, 49, 170, 297, 259, 342, 194, 85, 80, 60, 92, 88, 200, 341, 343, 99, 105, 28, 371, 381, 210, 358, 136, 344, 102, 169, 382, 316, 12, 34] + 20: [161, 174, 123, 307, 204, 330, 10, 273, 382, 140, 292, 92, 71, 322, 240, 35, 375, 314, 58, 18, 22, 86, 169, 15, 33, 73, 202, 89, 351, 210, 151, 115, 124, 329, 47, 136, 48, 234, 147, 103, 31, 298, 242, 27, 51, 52, 54, 61, 116, 4, 272, 287, 291, 32, 162, 164, 290, 96, 259, 258, 172, 126, 315, 194, 348, 26, 30, 359, 42, 256, 165, 246, 20, 78, 304, 231, 191, 94, 154, 156, 250, 321, 333, 67, 211, 64, 261, 149, 129, 152, 1, 197, 76, 107, 137, 167, 334, 87, 335, 14, 376, 130, 62, 371, 324, 128, 187, 342, 347, 65, 74, 284, 218, 361, 368, 213, 372, 238, 340, 294, 338, 106, 365, 312, 212, 159, 217, 352, 283, 28, 356, 72, 225, 133, 293, 192, 186, 155, 38, 269, 168, 179, 181, 189, 98, 99, 145, 37, 173, 112, 353, 188, 148, 141, 311, 362, 216, 101, 358, 357, 177, 328, 41, 36, 150, 208, 34, 166, 370, 310, 252, 39, 3, 374, 254, 316, 59, 199, 40, 7, 271, 16, 63, 373, 220, 222, 203, 44, 288, 200, 206, 214, 153, 193, 268, 122, 110, 226, 363, 185, 209, 229, 381, 180, 286, 49, 121, 6, 364, 235, 175, 146, 367, 119, 114, 60, 248, 377, 236, 257, 326, 337, 308, 56, 349, 21, 296, 163, 355, 19, 157, 383, 251, 275, 237, 69, 17, 249, 262, 263, 278, 66, 45, 8, 81, 2, 105, 138, 325, 111, 125, 264, 227, 132, 77, 90, 176, 343, 118, 0, 233, 300, 274, 346, 297, 29, 127, 170, 201, 24, 253, 285, 142, 289, 113, 85, 323, 305, 134, 68, 245, 301, 244, 109, 117, 260, 266, 281, 302, 144, 369, 282, 55, 339, 224, 270, 158, 91, 221, 50, 160, 184, 182, 239, 57, 267, 219, 25, 247, 178, 131, 215, 345, 12, 100, 332, 327, 84, 75, 80, 366, 120, 319, 205, 341, 83, 207, 108, 295, 306, 255, 143, 198, 277, 299, 303, 336, 82, 43, 46, 317, 379, 195, 318, 93, 228, 95, 9, 135, 320, 104, 190, 13, 378, 265, 53, 309, 279, 196, 243, 97, 5, 232, 380, 360, 223, 230, 139, 11, 79, 88, 102, 354, 276, 313, 183, 70, 23, 344, 171, 241, 280, 331, 350] + 21: [360, 80, 302, 206, 210, 364, 88, 345, 0, 240, 16, 26, 366, 69, 9, 189, 234, 325, 197, 336, 362, 318, 62, 286, 165, 30, 65, 301, 14, 102, 258, 335, 333, 305, 105, 342, 207, 130, 85, 37, 176, 6, 326, 95, 235, 311, 10, 17, 152, 158, 140, 247, 82, 208, 94, 377, 182, 320, 22, 349, 299, 280, 43, 23, 245, 131, 84, 18, 259, 262, 97, 303, 125, 378, 40, 100, 136, 104, 346, 48, 244, 334, 163, 241, 143, 41, 76, 56, 249, 106, 7, 116, 248, 327, 35, 58, 355, 4, 113, 256, 53, 185, 199, 232, 252, 250, 172, 351, 219, 227, 144, 220, 368, 229, 168, 1, 64, 223, 260, 159, 370, 5, 77, 193, 160, 198, 146, 276, 273, 381, 181, 253, 156, 74, 129, 180, 255, 31, 161, 166, 322, 332, 93, 123, 47, 254, 71, 236, 38, 218, 348, 214, 118, 149, 307, 147, 15, 194, 230, 111, 183, 243, 63, 49, 90, 309, 312, 28, 317, 323, 204, 171, 187, 170, 340, 300, 379, 110, 195, 164, 70, 330, 217, 314, 296, 142, 278, 188, 343, 11, 42, 264, 173, 308, 257, 190, 50, 112, 117, 277, 178, 238, 380, 19, 196, 225, 135, 221, 33, 174, 73, 267, 138, 60, 98, 192, 354, 133, 237, 148, 137, 203, 329, 216, 81, 83, 319, 294, 75, 13, 68, 114, 328, 145, 298, 212, 101, 127, 44, 306, 281, 371, 119, 46, 86, 87, 169, 226, 284, 51, 191, 128, 8, 25, 313, 141, 288, 270, 338, 331, 39, 304, 66, 383, 290, 154, 275, 266, 292, 269, 57, 261, 213, 134, 12, 315, 239, 374, 99, 324, 202, 242, 32, 222, 107, 167, 108, 321, 132, 20, 376, 153, 29, 92, 361, 120, 150, 279, 184, 109, 121, 293, 209, 367, 373, 89, 372, 358, 287, 246, 21, 356, 155, 3, 295, 341, 337, 24, 274, 126, 157, 27, 67, 382, 268, 2, 357, 54, 162, 265, 59, 231, 224, 283, 179, 339, 103, 316, 124, 289, 205, 200, 363, 45, 91, 177, 151, 365, 175, 344, 52, 72, 78, 233, 263, 352, 359, 139, 251, 291, 211, 347, 353, 115, 34, 79, 186, 36, 215, 271, 96, 310, 201, 122, 55, 375, 228, 285, 61, 282, 272, 297, 369, 350] + 22: [254, 376, 198, 34, 242, 112, 44, 355, 252, 70, 188, 111, 8, 222, 125, 170, 361, 366, 220, 233, 133, 367, 305, 200, 54, 256, 81, 300, 96, 353, 265, 246, 137, 282, 62, 257, 302, 150, 370, 304, 114, 270, 231, 10, 205, 2, 28, 79, 141, 224, 94, 126, 207, 184, 263, 244, 130, 348, 345, 311, 67, 83, 60, 185, 14, 3, 48, 25, 71, 12, 320, 318, 322, 283, 373, 241, 97, 136, 26, 103, 119, 182, 30, 91, 342, 344, 223, 6, 21, 146, 293, 234, 135, 249, 95, 101, 181, 24, 178, 204, 332, 73, 37, 329, 61, 55, 272, 218, 100, 378, 174, 193, 122, 228, 53, 258, 15, 99, 72, 47, 65, 189, 238, 45, 13, 271, 46, 208, 274, 151, 334, 129, 350, 128, 212, 88, 43, 158, 107, 243, 142, 280, 143, 154, 221, 148, 20, 33, 330, 115, 230, 235, 362, 317, 229, 149, 306, 161, 123, 214, 27, 113, 156, 49, 374, 210, 380, 87, 301, 199, 327, 299, 138, 314, 379, 262, 35, 160, 92, 145, 16, 168, 354, 162, 197, 163, 121, 368, 192, 287, 157, 217, 260, 209, 110, 9, 132, 164, 177, 139, 383, 240, 351, 338, 336, 68, 106, 116, 349, 358, 339, 191, 309, 165, 105, 248, 303, 269, 78, 32, 268, 202, 364, 279, 85, 187, 120, 152, 64, 69, 255, 23, 140, 277, 166, 56, 297, 289, 247, 250, 261, 278, 183, 371, 276, 38, 144, 29, 117, 352, 357, 7, 80, 372, 312, 369, 226, 175, 109, 18, 321, 59, 1, 39, 98, 227, 335, 382, 211, 215, 356, 298, 102, 219, 196, 179, 291, 90, 296, 326, 284, 295, 167, 86, 313, 292, 253, 267, 328, 343, 288, 316, 147, 42, 4, 169, 58, 333, 381, 285, 375, 195, 264, 17, 57, 75, 340, 63, 324, 225, 319, 341, 236, 124, 377, 180, 290, 77, 273, 176, 108, 266, 294, 190, 19, 104, 308, 5, 127, 66, 172, 194, 315, 323, 307, 310, 281, 216, 337, 51, 331, 201, 74, 118, 93, 50, 153, 251, 173, 76, 365, 259, 206, 347, 82, 40, 131, 41, 239, 159, 155, 22, 11, 0, 286, 89, 275, 237, 232, 186, 52, 36, 134, 171, 31, 363, 213, 84, 245, 203, 325, 360, 346, 359] + 23: [26, 135, 376, 19, 240, 324, 47, 374, 127, 148, 92, 340, 80, 70, 233, 23, 228, 292, 210, 298, 329, 231, 333, 223, 284, 258, 344, 164, 16, 14, 297, 239, 369, 214, 161, 9, 62, 165, 171, 299, 345, 89, 145, 186, 234, 189, 37, 64, 314, 167, 177, 211, 331, 203, 287, 94, 11, 218, 315, 263, 35, 213, 220, 149, 10, 290, 129, 157, 119, 66, 5, 307, 316, 76, 73, 200, 174, 68, 155, 236, 115, 104, 248, 61, 357, 31, 377, 343, 30, 117, 142, 380, 3, 216, 168, 288, 124, 187, 72, 274, 170, 24, 197, 154, 192, 182, 86, 41, 283, 326, 249, 191, 313, 224, 40, 222, 150, 96, 125, 178, 347, 352, 58, 363, 302, 206, 121, 349, 338, 139, 267, 46, 202, 34, 383, 181, 364, 42, 278, 198, 60, 366, 43, 81, 172, 132, 244, 281, 201, 269, 358, 176, 180, 59, 2, 56, 133, 146, 378, 91, 317, 237, 311, 356, 153, 160, 65, 301, 330, 250, 143, 109, 335, 212, 354, 362, 25, 375, 106, 188, 319, 88, 266, 116, 138, 208, 83, 217, 296, 4, 15, 242, 275, 100, 323, 120, 147, 82, 327, 261, 67, 372, 351, 130, 190, 144, 102, 312, 152, 272, 136, 361, 370, 17, 52, 325, 341, 204, 348, 280, 107, 7, 134, 99, 285, 355, 79, 367, 353, 230, 322, 247, 20, 141, 183, 293, 13, 251, 8, 235, 71, 175, 300, 225, 295, 87, 103, 156, 84, 55, 6, 371, 209, 50, 74, 36, 195, 342, 221, 44, 32, 365, 53, 137, 162, 193, 45, 0, 245, 108, 114, 243, 336, 373, 205, 49, 253, 276, 337, 334, 22, 318, 246, 112, 166, 29, 291, 265, 305, 264, 252, 69, 77, 173, 12, 232, 304, 57, 159, 78, 309, 111, 184, 229, 95, 194, 255, 101, 1, 93, 98, 207, 382, 196, 54, 128, 256, 21, 163, 268, 97, 33, 308, 140, 310, 277, 350, 259, 118, 289, 105, 262, 18, 110, 39, 286, 126, 328, 85, 51, 48, 306, 279, 27, 241, 169, 270, 257, 131, 379, 219, 151, 271, 38, 215, 359, 185, 254, 273, 179, 321, 303, 90, 339, 28, 346, 122, 294, 123, 227, 260, 282, 158, 368, 238, 360, 381, 63, 199, 332, 226, 113, 75, 320] + 24: [154, 357, 290, 172, 360, 112, 334, 186, 283, 159, 354, 91, 226, 274, 356, 49, 380, 3, 206, 63, 169, 295, 116, 11, 239, 60, 346, 282, 271, 296, 76, 161, 281, 160, 171, 291, 25, 18, 187, 240, 165, 316, 208, 364, 36, 374, 74, 81, 265, 263, 87, 259, 28, 286, 255, 230, 53, 135, 277, 382, 273, 260, 377, 39, 345, 118, 221, 321, 235, 323, 313, 280, 65, 225, 37, 170, 348, 10, 363, 320, 57, 331, 217, 317, 101, 68, 247, 250, 167, 340, 214, 17, 152, 133, 115, 166, 252, 372, 97, 210, 381, 64, 301, 207, 26, 204, 192, 15, 41, 106, 138, 336, 224, 307, 58, 297, 84, 168, 242, 315, 194, 30, 103, 4, 38, 14, 173, 249, 373, 163, 262, 342, 180, 184, 378, 300, 199, 347, 261, 40, 219, 158, 198, 241, 9, 22, 85, 328, 139, 361, 104, 375, 343, 78, 195, 306, 318, 254, 350, 96, 182, 12, 287, 83, 257, 233, 222, 56, 302, 134, 218, 203, 79, 88, 86, 188, 8, 148, 209, 44, 109, 236, 43, 174, 141, 123, 349, 82, 358, 179, 264, 298, 246, 6, 189, 111, 337, 324, 294, 146, 237, 24, 164, 190, 71, 355, 113, 149, 145, 245, 279, 376, 153, 269, 47, 51, 66, 181, 258, 267, 176, 120, 107, 284, 42, 67, 329, 238, 132, 256, 143, 228, 90, 227, 371, 100, 2, 21, 232, 314, 20, 92, 128, 110, 185, 27, 266, 62, 231, 213, 144, 61, 325, 308, 46, 351, 293, 77, 48, 59, 147, 278, 197, 1, 292, 215, 335, 333, 303, 150, 175, 94, 352, 359, 50, 234, 362, 367, 98, 162, 124, 285, 33, 305, 45, 127, 251, 353, 7, 52, 369, 95, 157, 105, 220, 370, 137, 202, 35, 54, 299, 131, 142, 177, 338, 205, 272, 140, 178, 13, 304, 244, 339, 270, 365, 72, 276, 196, 212, 126, 99, 191, 102, 193, 183, 80, 201, 332, 200, 243, 69, 322, 89, 129, 327, 379, 223, 0, 70, 211, 122, 121, 5, 29, 319, 31, 344, 366, 93, 156, 130, 73, 341, 310, 114, 326, 216, 16, 311, 268, 312, 108, 309, 275, 23, 253, 32, 151, 136, 19, 55, 288, 248, 125, 229, 75, 119, 289, 117, 330, 34, 155, 368, 383] + 25: [315, 30, 309, 77, 145, 305, 32, 120, 287, 215, 84, 240, 89, 343, 135, 359, 369, 304, 81, 130, 184, 202, 133, 14, 105, 300, 361, 127, 248, 22, 34, 122, 96, 293, 118, 349, 1, 286, 88, 104, 148, 106, 338, 233, 263, 123, 91, 158, 269, 168, 101, 2, 246, 232, 273, 102, 230, 342, 294, 197, 231, 153, 60, 97, 4, 291, 336, 237, 312, 259, 37, 28, 181, 355, 212, 251, 193, 260, 51, 281, 362, 283, 208, 289, 322, 265, 119, 172, 217, 272, 241, 194, 43, 19, 277, 326, 379, 192, 268, 375, 25, 376, 333, 7, 296, 378, 100, 131, 373, 76, 244, 146, 161, 143, 225, 319, 199, 345, 185, 328, 201, 94, 11, 356, 74, 166, 186, 177, 157, 21, 45, 40, 128, 137, 95, 57, 26, 301, 50, 329, 382, 239, 267, 178, 282, 372, 62, 298, 78, 0, 112, 5, 8, 318, 351, 381, 252, 377, 229, 54, 126, 218, 196, 347, 132, 221, 354, 290, 297, 257, 69, 370, 53, 27, 179, 271, 35, 325, 70, 210, 46, 236, 142, 63, 264, 71, 108, 198, 13, 270, 156, 220, 72, 23, 284, 258, 85, 39, 113, 49, 160, 38, 59, 262, 346, 9, 56, 256, 367, 321, 348, 242, 20, 203, 68, 55, 357, 331, 58, 310, 31, 200, 17, 358, 109, 163, 86, 124, 226, 175, 245, 155, 3, 147, 12, 285, 182, 6, 44, 121, 139, 276, 107, 98, 299, 61, 327, 10, 280, 66, 216, 18, 154, 222, 341, 80, 366, 279, 261, 140, 211, 295, 350, 204, 308, 134, 254, 144, 24, 79, 223, 243, 337, 75, 36, 167, 364, 371, 64, 41, 205, 360, 42, 90, 266, 170, 255, 165, 228, 180, 29, 83, 334, 162, 152, 171, 190, 33, 314, 368, 302, 234, 209, 99, 138, 227, 307, 114, 191, 278, 316, 324, 330, 365, 339, 149, 67, 238, 249, 275, 111, 292, 250, 183, 87, 311, 174, 224, 48, 141, 335, 169, 288, 151, 195, 129, 247, 317, 214, 65, 82, 332, 52, 73, 235, 313, 380, 207, 136, 306, 253, 125, 323, 164, 176, 103, 115, 352, 187, 383, 15, 213, 116, 274, 150, 159, 340, 110, 353, 117, 93, 320, 173, 16, 92, 188, 344, 363, 374, 219, 47, 206, 303, 189] + 26: [136, 150, 63, 162, 16, 183, 266, 223, 88, 167, 83, 247, 48, 138, 323, 353, 97, 103, 268, 365, 17, 113, 45, 0, 256, 75, 232, 301, 7, 49, 325, 225, 8, 64, 100, 29, 13, 216, 284, 330, 235, 310, 322, 132, 1, 54, 68, 86, 237, 53, 230, 214, 304, 289, 33, 137, 369, 249, 146, 328, 359, 148, 319, 190, 229, 174, 24, 221, 57, 318, 10, 73, 76, 32, 114, 182, 25, 219, 220, 340, 317, 43, 346, 244, 154, 91, 296, 4, 243, 222, 58, 155, 288, 96, 106, 111, 89, 18, 339, 233, 338, 116, 279, 38, 355, 158, 245, 82, 200, 311, 259, 321, 133, 102, 251, 294, 71, 104, 161, 35, 107, 350, 123, 143, 134, 264, 300, 352, 141, 109, 59, 77, 326, 101, 324, 14, 250, 345, 95, 248, 309, 139, 169, 170, 269, 124, 253, 271, 342, 120, 337, 130, 293, 315, 108, 382, 180, 31, 360, 262, 306, 99, 186, 343, 66, 215, 65, 98, 267, 50, 144, 372, 298, 336, 3, 140, 185, 335, 74, 112, 163, 119, 358, 145, 282, 28, 257, 210, 168, 175, 209, 218, 217, 11, 291, 320, 263, 254, 69, 308, 61, 135, 30, 62, 67, 176, 272, 151, 142, 152, 173, 194, 127, 239, 159, 202, 115, 231, 177, 312, 117, 368, 12, 105, 47, 40, 9, 303, 193, 199, 373, 208, 283, 46, 381, 273, 203, 228, 234, 242, 172, 380, 236, 281, 70, 224, 344, 316, 307, 376, 92, 131, 265, 198, 26, 191, 367, 27, 241, 118, 285, 277, 195, 212, 171, 37, 129, 383, 280, 157, 78, 362, 255, 72, 34, 331, 206, 205, 56, 313, 85, 147, 23, 379, 213, 246, 261, 305, 90, 292, 181, 5, 184, 149, 94, 341, 238, 226, 36, 110, 156, 204, 179, 287, 2, 327, 79, 51, 192, 15, 81, 290, 252, 196, 187, 363, 19, 302, 197, 364, 354, 334, 275, 361, 240, 274, 258, 377, 211, 164, 366, 207, 52, 260, 348, 351, 80, 6, 21, 349, 371, 160, 276, 278, 227, 22, 126, 332, 55, 128, 178, 166, 44, 356, 122, 201, 20, 357, 87, 378, 41, 370, 93, 153, 189, 329, 121, 125, 165, 347, 297, 39, 333, 314, 270, 42, 295, 188, 84, 286, 60, 299, 374, 375] + 27: [103, 354, 58, 164, 320, 181, 213, 143, 75, 303, 381, 339, 77, 89, 322, 118, 305, 7, 91, 54, 174, 218, 45, 276, 171, 199, 250, 365, 275, 105, 130, 259, 185, 65, 157, 289, 62, 283, 173, 138, 314, 57, 206, 200, 39, 53, 60, 137, 225, 73, 341, 301, 219, 204, 135, 358, 348, 237, 335, 290, 266, 99, 102, 23, 278, 317, 27, 88, 162, 329, 304, 296, 46, 362, 2, 9, 101, 95, 107, 159, 132, 96, 346, 74, 229, 364, 86, 334, 194, 148, 11, 92, 209, 203, 146, 163, 286, 168, 227, 81, 59, 71, 49, 24, 131, 222, 230, 48, 260, 196, 115, 83, 140, 141, 337, 155, 125, 10, 114, 215, 267, 18, 347, 28, 249, 367, 205, 35, 33, 184, 208, 264, 302, 372, 94, 166, 108, 160, 19, 169, 243, 212, 176, 220, 12, 265, 349, 34, 279, 269, 26, 128, 224, 112, 55, 178, 324, 216, 238, 78, 284, 356, 187, 161, 316, 379, 214, 274, 287, 43, 239, 318, 121, 167, 361, 79, 14, 280, 359, 50, 352, 16, 158, 345, 226, 170, 255, 363, 378, 235, 240, 244, 312, 61, 252, 117, 351, 300, 315, 366, 369, 190, 182, 21, 76, 149, 72, 32, 325, 191, 42, 254, 368, 326, 172, 5, 375, 106, 248, 66, 307, 0, 3, 256, 144, 231, 282, 223, 41, 330, 195, 192, 165, 311, 210, 298, 17, 63, 236, 262, 90, 263, 30, 104, 70, 374, 119, 376, 179, 336, 183, 323, 242, 383, 343, 82, 201, 193, 133, 370, 328, 271, 136, 268, 122, 67, 147, 44, 321, 217, 123, 288, 139, 281, 319, 8, 186, 142, 257, 64, 87, 331, 295, 371, 36, 93, 247, 310, 234, 4, 69, 145, 31, 350, 357, 202, 377, 97, 228, 85, 380, 232, 156, 308, 297, 109, 241, 261, 309, 6, 175, 327, 124, 129, 299, 120, 47, 353, 153, 116, 306, 113, 313, 293, 150, 110, 68, 189, 1, 56, 221, 355, 51, 100, 270, 338, 37, 177, 80, 211, 333, 233, 197, 292, 188, 207, 251, 272, 340, 258, 294, 332, 25, 373, 98, 15, 126, 246, 291, 127, 360, 111, 20, 38, 29, 13, 84, 151, 344, 152, 154, 22, 253, 382, 273, 198, 52, 245, 277, 40, 134, 180, 285, 342] + 28: [161, 0, 324, 120, 122, 7, 108, 191, 71, 58, 352, 339, 41, 240, 217, 367, 380, 43, 184, 78, 46, 245, 44, 98, 177, 36, 104, 363, 24, 294, 96, 80, 38, 284, 347, 232, 174, 140, 65, 53, 87, 262, 10, 18, 60, 86, 115, 150, 250, 25, 355, 77, 176, 288, 37, 75, 166, 223, 158, 63, 160, 28, 289, 105, 56, 19, 114, 341, 68, 282, 260, 264, 57, 159, 205, 296, 230, 133, 280, 331, 117, 213, 39, 131, 146, 29, 281, 277, 189, 254, 124, 62, 138, 156, 180, 207, 26, 302, 99, 312, 22, 226, 103, 307, 369, 378, 325, 204, 100, 157, 67, 337, 209, 271, 273, 127, 268, 134, 14, 183, 171, 208, 295, 375, 74, 66, 170, 73, 136, 89, 348, 225, 246, 229, 55, 76, 111, 126, 354, 164, 186, 147, 215, 257, 185, 306, 52, 4, 196, 152, 233, 224, 263, 219, 368, 51, 261, 90, 167, 163, 221, 182, 238, 278, 16, 1, 121, 365, 265, 64, 305, 148, 106, 210, 211, 151, 382, 364, 323, 110, 292, 276, 252, 101, 123, 141, 360, 198, 212, 173, 279, 311, 168, 255, 30, 197, 310, 203, 172, 113, 272, 50, 137, 227, 165, 235, 135, 220, 32, 340, 376, 48, 316, 91, 118, 142, 49, 81, 314, 143, 322, 12, 334, 154, 377, 218, 266, 251, 303, 237, 47, 373, 102, 304, 83, 309, 301, 190, 329, 336, 344, 317, 107, 321, 239, 188, 195, 253, 357, 128, 3, 228, 200, 88, 285, 326, 269, 13, 33, 193, 206, 178, 27, 244, 6, 249, 274, 362, 61, 258, 332, 297, 372, 330, 287, 293, 8, 17, 192, 35, 291, 371, 145, 313, 2, 40, 298, 338, 92, 70, 243, 241, 248, 328, 97, 93, 199, 236, 320, 42, 5, 129, 361, 242, 299, 216, 349, 315, 300, 333, 194, 256, 327, 358, 69, 381, 169, 270, 342, 34, 109, 31, 119, 345, 231, 72, 139, 267, 45, 202, 181, 318, 9, 54, 359, 343, 356, 153, 162, 286, 201, 283, 155, 214, 374, 346, 95, 59, 116, 383, 130, 370, 125, 20, 308, 247, 335, 222, 290, 112, 179, 23, 82, 21, 175, 15, 84, 366, 144, 79, 149, 351, 234, 85, 259, 275, 94, 350, 187, 319, 11, 132, 379, 353] + 29: [118, 304, 23, 283, 25, 129, 320, 346, 243, 87, 63, 204, 123, 50, 172, 218, 106, 329, 18, 272, 101, 353, 105, 217, 316, 242, 230, 28, 191, 56, 42, 166, 253, 270, 219, 308, 252, 30, 164, 189, 2, 130, 64, 138, 98, 206, 26, 363, 244, 295, 173, 271, 54, 339, 309, 160, 140, 134, 343, 145, 319, 278, 360, 85, 139, 222, 225, 59, 97, 3, 205, 317, 238, 31, 157, 135, 209, 368, 7, 250, 233, 62, 376, 71, 115, 246, 234, 201, 8, 5, 1, 268, 256, 367, 322, 29, 361, 203, 282, 383, 365, 333, 186, 266, 180, 375, 24, 68, 210, 350, 324, 215, 257, 349, 347, 228, 150, 292, 294, 328, 122, 224, 52, 46, 303, 220, 70, 338, 239, 194, 149, 17, 184, 296, 325, 254, 21, 27, 372, 342, 274, 37, 185, 240, 197, 198, 236, 255, 195, 379, 74, 16, 65, 48, 190, 216, 382, 38, 4, 378, 128, 273, 300, 146, 181, 380, 125, 340, 22, 229, 40, 10, 213, 327, 84, 161, 187, 121, 113, 99, 337, 297, 49, 41, 100, 226, 81, 221, 305, 6, 212, 285, 73, 323, 44, 89, 124, 76, 262, 148, 326, 57, 332, 163, 82, 86, 94, 259, 116, 232, 72, 171, 345, 104, 20, 371, 83, 169, 126, 369, 79, 103, 153, 261, 192, 179, 162, 334, 35, 251, 301, 279, 95, 245, 107, 321, 117, 144, 39, 91, 196, 277, 60, 11, 58, 120, 110, 351, 112, 366, 175, 237, 102, 357, 61, 298, 182, 178, 19, 280, 78, 310, 32, 167, 47, 36, 170, 302, 53, 287, 299, 13, 374, 227, 168, 0, 263, 92, 109, 223, 330, 286, 165, 335, 247, 214, 355, 152, 336, 137, 362, 127, 88, 354, 33, 202, 119, 174, 188, 275, 341, 293, 132, 248, 12, 200, 131, 90, 264, 142, 269, 344, 373, 312, 315, 147, 111, 114, 158, 199, 15, 69, 356, 154, 211, 141, 96, 358, 80, 75, 291, 281, 289, 249, 108, 66, 318, 51, 67, 311, 14, 258, 265, 208, 183, 159, 235, 370, 207, 193, 364, 314, 34, 156, 136, 276, 151, 267, 313, 77, 359, 348, 93, 377, 55, 381, 9, 288, 352, 307, 306, 43, 45, 290, 155, 133, 331, 260, 231, 241, 284, 143, 177, 176] + 30: [208, 76, 366, 225, 11, 144, 283, 250, 97, 341, 253, 256, 141, 136, 292, 98, 157, 278, 179, 335, 51, 310, 55, 48, 44, 374, 186, 121, 220, 139, 69, 85, 209, 314, 178, 352, 207, 265, 63, 61, 191, 238, 8, 21, 22, 31, 36, 40, 239, 333, 243, 59, 2, 45, 41, 315, 216, 240, 100, 58, 215, 226, 281, 298, 174, 91, 72, 114, 163, 286, 349, 181, 282, 360, 342, 377, 371, 161, 203, 327, 368, 122, 169, 330, 102, 338, 171, 93, 251, 301, 47, 88, 94, 101, 118, 123, 140, 35, 289, 213, 13, 378, 206, 28, 68, 303, 340, 74, 152, 275, 210, 204, 344, 369, 343, 261, 54, 383, 197, 129, 241, 73, 50, 145, 194, 183, 347, 326, 381, 5, 370, 355, 16, 113, 126, 313, 367, 182, 252, 305, 128, 134, 146, 185, 266, 52, 62, 184, 137, 78, 295, 223, 147, 302, 43, 221, 77, 192, 309, 382, 170, 160, 23, 353, 231, 142, 359, 111, 375, 248, 297, 25, 166, 202, 263, 32, 57, 125, 130, 291, 173, 120, 198, 0, 149, 228, 4, 70, 307, 189, 218, 219, 65, 214, 351, 133, 222, 242, 257, 188, 246, 17, 362, 159, 89, 1, 193, 236, 277, 260, 67, 60, 132, 329, 150, 376, 66, 293, 7, 92, 288, 337, 107, 267, 264, 83, 155, 82, 127, 233, 255, 300, 80, 328, 229, 27, 131, 331, 237, 259, 245, 53, 373, 254, 86, 75, 168, 321, 346, 212, 271, 49, 138, 336, 339, 299, 217, 18, 285, 200, 96, 167, 162, 14, 172, 180, 244, 284, 79, 110, 230, 247, 279, 99, 287, 273, 153, 19, 190, 106, 56, 372, 34, 87, 176, 354, 268, 269, 37, 358, 156, 124, 272, 135, 24, 234, 64, 33, 205, 312, 12, 357, 6, 9, 195, 274, 258, 365, 108, 84, 306, 319, 280, 109, 332, 290, 175, 154, 276, 324, 304, 116, 323, 117, 3, 15, 308, 270, 199, 39, 148, 30, 201, 363, 322, 334, 262, 187, 143, 158, 29, 196, 46, 361, 10, 164, 316, 224, 317, 177, 71, 151, 320, 296, 356, 325, 104, 38, 103, 20, 294, 211, 115, 249, 318, 311, 95, 350, 90, 348, 165, 364, 81, 105, 232, 112, 235, 42, 26, 119, 227, 379, 345, 380] + 31: [0, 185, 32, 36, 335, 95, 330, 209, 58, 253, 62, 102, 223, 82, 74, 17, 66, 122, 46, 15, 97, 383, 141, 49, 354, 237, 376, 266, 204, 224, 107, 210, 367, 315, 99, 113, 127, 197, 372, 27, 174, 257, 151, 165, 148, 229, 132, 207, 159, 239, 227, 241, 116, 25, 219, 39, 119, 171, 72, 355, 258, 145, 236, 63, 23, 243, 277, 31, 318, 225, 373, 16, 84, 361, 362, 6, 272, 228, 48, 50, 240, 129, 298, 321, 135, 7, 235, 370, 313, 359, 311, 104, 117, 71, 289, 329, 128, 115, 87, 265, 279, 326, 270, 328, 22, 333, 283, 338, 112, 296, 211, 136, 40, 366, 65, 322, 256, 374, 217, 92, 142, 153, 12, 305, 275, 351, 198, 103, 85, 309, 24, 234, 327, 156, 79, 126, 114, 357, 377, 230, 203, 180, 294, 350, 233, 143, 261, 331, 3, 133, 187, 317, 349, 60, 34, 299, 348, 290, 380, 353, 232, 245, 175, 205, 222, 202, 213, 21, 302, 73, 196, 183, 271, 346, 352, 304, 238, 375, 160, 157, 168, 381, 53, 193, 255, 184, 30, 161, 5, 154, 212, 347, 26, 252, 70, 125, 365, 364, 215, 150, 325, 155, 378, 164, 276, 189, 78, 111, 231, 55, 134, 182, 77, 287, 307, 118, 336, 195, 260, 273, 14, 177, 371, 281, 263, 42, 139, 340, 300, 64, 293, 382, 295, 18, 179, 86, 358, 343, 37, 149, 201, 158, 363, 320, 192, 379, 147, 267, 146, 186, 61, 360, 56, 269, 28, 249, 93, 38, 368, 57, 105, 316, 312, 288, 303, 291, 4, 51, 20, 323, 1, 130, 80, 308, 345, 9, 262, 109, 280, 274, 282, 167, 301, 163, 297, 200, 90, 246, 68, 324, 144, 248, 221, 292, 69, 339, 173, 140, 188, 75, 334, 214, 106, 254, 152, 54, 2, 88, 137, 41, 284, 123, 45, 8, 110, 108, 59, 43, 91, 194, 47, 162, 101, 178, 337, 356, 52, 44, 344, 242, 319, 94, 199, 67, 341, 120, 100, 166, 278, 76, 19, 121, 250, 268, 96, 206, 170, 208, 131, 286, 33, 244, 138, 98, 81, 169, 176, 190, 314, 310, 369, 29, 285, 11, 247, 83, 220, 264, 306, 226, 259, 251, 342, 191, 216, 181, 124, 35, 332, 218, 10, 13, 89, 172] + 32: [63, 146, 310, 66, 141, 204, 96, 126, 258, 109, 289, 171, 227, 104, 69, 64, 48, 237, 127, 240, 81, 337, 24, 349, 363, 311, 42, 102, 103, 197, 73, 374, 196, 203, 377, 31, 23, 321, 217, 338, 194, 162, 68, 14, 123, 169, 177, 181, 367, 166, 246, 209, 271, 2, 342, 319, 202, 211, 308, 232, 120, 189, 298, 71, 26, 25, 157, 150, 286, 53, 371, 129, 6, 340, 154, 172, 221, 132, 152, 304, 153, 264, 0, 355, 118, 224, 46, 368, 47, 207, 11, 27, 147, 244, 245, 251, 352, 51, 344, 234, 362, 130, 121, 41, 186, 283, 279, 116, 273, 345, 266, 91, 111, 50, 314, 281, 59, 142, 39, 3, 302, 10, 37, 193, 34, 82, 299, 124, 238, 222, 330, 201, 170, 208, 133, 259, 242, 112, 291, 226, 326, 167, 262, 265, 94, 163, 164, 322, 49, 229, 370, 12, 40, 65, 75, 78, 156, 252, 58, 301, 212, 354, 199, 213, 178, 230, 336, 29, 176, 247, 315, 56, 139, 339, 9, 99, 243, 365, 223, 35, 160, 52, 188, 60, 76, 95, 98, 295, 235, 333, 173, 296, 218, 241, 107, 275, 108, 179, 8, 257, 16, 89, 263, 140, 233, 335, 306, 254, 325, 36, 231, 28, 250, 293, 382, 353, 70, 4, 381, 77, 236, 5, 136, 284, 125, 376, 20, 356, 67, 79, 255, 100, 180, 18, 17, 272, 15, 54, 300, 312, 148, 190, 38, 288, 346, 106, 280, 274, 214, 348, 83, 97, 87, 113, 294, 313, 72, 85, 292, 269, 45, 253, 183, 351, 115, 276, 22, 198, 317, 110, 210, 307, 195, 105, 13, 134, 305, 278, 343, 137, 260, 358, 192, 290, 93, 90, 316, 373, 33, 347, 327, 155, 268, 323, 200, 219, 359, 225, 174, 143, 131, 228, 7, 369, 55, 303, 101, 332, 324, 182, 256, 249, 80, 30, 309, 378, 372, 144, 282, 43, 165, 248, 117, 261, 44, 138, 220, 364, 216, 161, 74, 159, 267, 114, 379, 375, 92, 285, 122, 329, 135, 187, 21, 360, 61, 380, 184, 151, 331, 19, 357, 168, 297, 185, 175, 206, 239, 86, 32, 350, 84, 205, 366, 62, 158, 320, 361, 191, 287, 341, 128, 270, 119, 149, 334, 1, 277, 215, 88, 318, 328, 145, 57, 383] + 33: [38, 307, 240, 331, 323, 39, 125, 146, 206, 335, 54, 33, 155, 40, 288, 126, 303, 173, 174, 347, 375, 157, 200, 369, 213, 43, 3, 28, 166, 248, 327, 20, 346, 132, 76, 332, 247, 353, 214, 171, 359, 2, 46, 0, 265, 293, 333, 372, 91, 115, 267, 140, 280, 121, 183, 348, 84, 266, 255, 204, 30, 227, 308, 217, 18, 351, 355, 254, 79, 262, 197, 163, 152, 170, 106, 71, 66, 26, 110, 67, 135, 74, 286, 291, 310, 118, 242, 363, 7, 228, 284, 47, 239, 129, 69, 142, 339, 343, 154, 51, 259, 158, 107, 314, 108, 5, 190, 141, 25, 193, 131, 9, 305, 211, 149, 4, 198, 246, 55, 373, 279, 364, 137, 371, 281, 70, 285, 219, 10, 316, 130, 381, 120, 62, 315, 165, 274, 44, 344, 357, 282, 12, 34, 220, 36, 81, 299, 328, 63, 31, 96, 179, 201, 352, 127, 376, 218, 151, 192, 289, 61, 354, 187, 109, 45, 80, 156, 370, 114, 60, 49, 330, 113, 65, 278, 41, 252, 182, 181, 85, 78, 257, 178, 59, 199, 253, 313, 294, 203, 159, 189, 277, 319, 148, 283, 167, 320, 42, 297, 68, 322, 226, 169, 273, 1, 210, 356, 222, 235, 144, 205, 230, 383, 196, 382, 367, 186, 234, 368, 292, 24, 52, 271, 261, 318, 298, 300, 168, 188, 312, 244, 175, 378, 374, 269, 57, 302, 272, 264, 287, 260, 229, 301, 86, 145, 153, 379, 73, 241, 164, 37, 270, 160, 150, 208, 268, 14, 290, 93, 138, 123, 276, 19, 50, 180, 13, 58, 249, 350, 366, 101, 324, 136, 194, 111, 11, 225, 245, 365, 337, 92, 250, 99, 172, 77, 143, 83, 95, 177, 380, 258, 89, 75, 377, 231, 342, 147, 139, 361, 233, 17, 338, 117, 334, 8, 116, 329, 16, 216, 22, 162, 104, 88, 212, 105, 15, 215, 349, 53, 112, 221, 128, 23, 207, 27, 119, 98, 326, 97, 275, 309, 296, 100, 223, 236, 102, 362, 64, 251, 94, 176, 191, 325, 82, 358, 72, 48, 21, 35, 87, 237, 133, 311, 232, 202, 90, 317, 124, 256, 209, 360, 340, 122, 345, 336, 56, 103, 29, 295, 6, 238, 243, 184, 224, 134, 341, 195, 306, 321, 304, 185, 263, 161, 32] + 34: [108, 44, 151, 135, 145, 240, 178, 246, 83, 271, 78, 162, 208, 290, 81, 256, 136, 133, 33, 181, 73, 0, 247, 263, 113, 140, 62, 298, 105, 266, 314, 13, 101, 226, 331, 205, 17, 285, 90, 1, 355, 378, 320, 7, 375, 14, 76, 119, 179, 334, 372, 357, 239, 321, 291, 258, 59, 288, 41, 95, 134, 373, 12, 159, 368, 206, 195, 241, 48, 287, 367, 230, 215, 294, 10, 358, 142, 296, 374, 219, 36, 286, 28, 109, 242, 342, 212, 22, 43, 182, 38, 235, 177, 211, 122, 132, 16, 330, 61, 324, 311, 185, 148, 146, 115, 238, 92, 65, 225, 153, 184, 326, 327, 199, 251, 67, 270, 209, 102, 308, 103, 164, 283, 371, 261, 64, 68, 228, 111, 353, 49, 117, 278, 253, 9, 54, 120, 188, 252, 309, 198, 213, 160, 189, 249, 126, 196, 382, 18, 304, 31, 360, 315, 295, 191, 6, 125, 201, 107, 356, 123, 197, 143, 217, 30, 227, 333, 169, 232, 154, 63, 127, 139, 272, 85, 88, 233, 161, 34, 255, 165, 26, 39, 104, 128, 156, 338, 106, 121, 32, 224, 193, 306, 299, 190, 29, 332, 354, 203, 194, 347, 79, 335, 267, 244, 21, 144, 174, 84, 183, 172, 336, 328, 231, 364, 170, 361, 180, 322, 277, 149, 383, 98, 192, 269, 262, 93, 319, 52, 370, 339, 141, 281, 292, 349, 303, 129, 66, 279, 202, 15, 259, 248, 207, 82, 158, 362, 167, 379, 2, 8, 300, 4, 124, 58, 53, 313, 187, 37, 280, 346, 72, 214, 35, 5, 80, 19, 318, 221, 186, 147, 268, 282, 310, 100, 380, 57, 173, 55, 171, 351, 96, 216, 94, 350, 273, 316, 229, 340, 69, 3, 237, 74, 25, 243, 138, 220, 97, 222, 11, 87, 91, 110, 302, 345, 24, 60, 70, 245, 150, 359, 116, 301, 348, 114, 323, 23, 46, 264, 275, 289, 265, 274, 75, 131, 250, 155, 112, 71, 130, 325, 234, 369, 343, 341, 363, 366, 337, 218, 47, 312, 223, 293, 40, 99, 284, 254, 168, 77, 200, 329, 210, 175, 166, 137, 317, 236, 27, 204, 89, 50, 352, 157, 42, 56, 45, 51, 152, 305, 86, 118, 307, 176, 260, 297, 20, 257, 163, 344, 276, 376, 377, 381, 365] + 35: [319, 68, 322, 220, 144, 141, 138, 31, 147, 114, 335, 362, 159, 275, 357, 296, 158, 5, 294, 137, 152, 11, 170, 74, 348, 264, 330, 99, 380, 73, 43, 176, 105, 189, 149, 112, 23, 354, 238, 83, 225, 219, 226, 39, 120, 304, 228, 177, 45, 295, 72, 353, 237, 56, 257, 38, 154, 15, 265, 50, 266, 27, 102, 168, 18, 61, 328, 108, 143, 90, 323, 57, 227, 91, 382, 273, 179, 196, 315, 378, 317, 151, 253, 22, 26, 262, 97, 101, 270, 252, 280, 119, 229, 381, 293, 343, 1, 367, 276, 48, 260, 211, 368, 129, 3, 224, 46, 307, 14, 222, 373, 160, 244, 308, 282, 372, 42, 320, 16, 172, 109, 214, 369, 383, 376, 19, 17, 88, 198, 292, 259, 245, 94, 121, 77, 10, 69, 310, 188, 210, 173, 236, 291, 359, 345, 297, 86, 281, 62, 254, 67, 153, 231, 78, 41, 326, 361, 136, 312, 267, 356, 36, 171, 30, 125, 95, 32, 350, 366, 202, 339, 24, 290, 142, 66, 344, 274, 332, 161, 370, 181, 155, 288, 107, 351, 233, 123, 258, 212, 199, 85, 284, 363, 131, 53, 80, 246, 303, 318, 4, 221, 239, 364, 349, 331, 54, 157, 321, 128, 81, 93, 213, 209, 337, 96, 287, 8, 305, 347, 166, 174, 269, 217, 25, 6, 365, 156, 29, 115, 49, 165, 167, 271, 33, 371, 336, 76, 2, 118, 200, 75, 218, 203, 340, 44, 130, 286, 21, 116, 206, 375, 194, 191, 126, 37, 360, 140, 132, 192, 215, 334, 201, 256, 261, 111, 145, 82, 204, 184, 127, 175, 84, 35, 63, 248, 278, 249, 195, 255, 139, 148, 327, 208, 0, 341, 64, 283, 60, 169, 299, 133, 309, 182, 58, 55, 47, 289, 186, 311, 234, 241, 329, 98, 134, 272, 9, 298, 190, 313, 300, 302, 235, 20, 180, 117, 216, 185, 301, 279, 338, 103, 87, 135, 377, 306, 250, 263, 150, 178, 333, 242, 51, 71, 124, 146, 13, 28, 205, 342, 113, 240, 379, 314, 324, 346, 355, 352, 164, 162, 52, 163, 187, 325, 230, 268, 104, 243, 122, 12, 92, 277, 110, 106, 34, 7, 70, 285, 374, 79, 65, 207, 316, 232, 89, 358, 183, 197, 223, 193, 247, 251, 59, 100, 40] + 36: [232, 372, 197, 348, 299, 296, 380, 374, 150, 314, 31, 45, 246, 231, 293, 13, 68, 245, 155, 168, 216, 228, 46, 310, 0, 52, 335, 244, 119, 208, 51, 209, 204, 338, 251, 357, 118, 110, 235, 151, 285, 89, 269, 160, 114, 64, 37, 123, 22, 10, 223, 382, 359, 210, 241, 90, 291, 47, 85, 65, 201, 67, 166, 55, 132, 91, 50, 339, 354, 174, 100, 177, 255, 187, 81, 344, 358, 78, 220, 58, 239, 189, 236, 383, 60, 120, 226, 87, 7, 34, 27, 106, 221, 43, 270, 170, 184, 266, 12, 370, 62, 230, 326, 373, 73, 5, 54, 32, 175, 362, 319, 49, 349, 95, 164, 20, 171, 179, 237, 224, 305, 229, 104, 17, 157, 129, 243, 135, 281, 6, 158, 199, 337, 21, 115, 350, 278, 96, 124, 218, 234, 113, 279, 273, 371, 238, 225, 131, 74, 328, 98, 56, 97, 308, 125, 261, 240, 334, 288, 282, 167, 105, 298, 342, 142, 117, 271, 277, 323, 264, 76, 28, 148, 263, 283, 18, 44, 247, 315, 173, 316, 137, 61, 355, 154, 303, 227, 181, 306, 145, 290, 302, 368, 304, 193, 107, 176, 2, 320, 99, 301, 205, 92, 331, 333, 108, 219, 139, 361, 121, 172, 289, 153, 140, 343, 364, 134, 71, 212, 36, 188, 267, 365, 29, 329, 295, 381, 198, 35, 307, 254, 213, 26, 128, 327, 222, 367, 186, 292, 360, 272, 57, 178, 192, 84, 59, 258, 274, 352, 79, 287, 156, 33, 112, 194, 94, 122, 284, 322, 183, 15, 280, 66, 195, 102, 200, 257, 379, 88, 149, 318, 356, 363, 109, 259, 19, 249, 215, 312, 253, 161, 203, 136, 286, 182, 317, 127, 375, 53, 144, 311, 252, 250, 260, 242, 369, 207, 133, 332, 42, 321, 309, 111, 345, 63, 297, 300, 276, 346, 147, 11, 116, 211, 30, 39, 206, 165, 23, 159, 48, 24, 70, 169, 77, 130, 347, 324, 8, 378, 330, 353, 4, 214, 268, 16, 313, 340, 341, 325, 294, 185, 101, 196, 40, 275, 152, 233, 93, 162, 262, 9, 191, 138, 103, 69, 3, 72, 366, 41, 202, 217, 180, 38, 143, 351, 80, 163, 146, 83, 190, 265, 248, 256, 75, 14, 126, 86, 1, 82, 141, 377, 376, 25, 336] + 37: [218, 213, 199, 48, 193, 223, 344, 253, 22, 139, 181, 18, 145, 19, 122, 27, 149, 82, 143, 108, 285, 356, 83, 337, 347, 313, 38, 4, 116, 160, 17, 170, 175, 35, 81, 62, 130, 119, 300, 89, 90, 68, 84, 31, 132, 236, 239, 288, 40, 188, 123, 314, 46, 272, 292, 361, 171, 245, 165, 204, 163, 133, 359, 208, 76, 173, 144, 61, 39, 333, 329, 126, 206, 121, 244, 94, 284, 42, 154, 13, 91, 216, 169, 297, 191, 96, 231, 155, 11, 220, 275, 232, 99, 189, 352, 383, 268, 366, 0, 273, 311, 205, 190, 72, 77, 140, 360, 88, 328, 343, 257, 150, 127, 228, 176, 6, 230, 141, 278, 29, 100, 224, 80, 380, 97, 24, 325, 187, 299, 10, 60, 137, 310, 198, 106, 324, 177, 45, 219, 211, 346, 209, 56, 75, 294, 200, 340, 66, 309, 52, 372, 290, 382, 114, 185, 182, 147, 93, 330, 221, 370, 287, 270, 168, 217, 36, 264, 201, 87, 55, 37, 321, 195, 279, 105, 351, 138, 368, 303, 78, 295, 374, 248, 57, 202, 197, 369, 246, 358, 251, 375, 112, 148, 222, 162, 364, 242, 306, 341, 179, 174, 86, 129, 7, 65, 117, 131, 192, 315, 229, 21, 196, 365, 41, 30, 134, 376, 49, 247, 128, 255, 241, 274, 5, 172, 124, 152, 286, 291, 283, 142, 146, 107, 326, 54, 159, 381, 25, 238, 120, 293, 158, 113, 71, 348, 125, 289, 342, 234, 16, 302, 353, 14, 233, 153, 164, 157, 316, 51, 74, 203, 296, 92, 111, 34, 335, 331, 269, 312, 271, 280, 210, 249, 254, 215, 332, 378, 339, 166, 184, 237, 28, 261, 186, 2, 23, 276, 226, 109, 58, 349, 319, 243, 178, 362, 305, 95, 156, 355, 252, 318, 235, 32, 277, 322, 320, 104, 64, 323, 73, 161, 103, 3, 136, 69, 262, 373, 102, 317, 267, 298, 256, 266, 350, 115, 101, 308, 225, 70, 227, 20, 250, 367, 50, 377, 8, 301, 379, 79, 240, 265, 151, 363, 44, 371, 59, 47, 9, 327, 194, 118, 345, 336, 85, 258, 1, 43, 338, 304, 135, 15, 167, 334, 207, 98, 212, 33, 354, 26, 110, 53, 260, 183, 259, 63, 357, 214, 281, 180, 307, 12, 282, 67, 263] + 38: [301, 282, 50, 94, 322, 32, 171, 145, 212, 54, 97, 199, 125, 352, 271, 208, 193, 262, 57, 184, 179, 143, 228, 31, 28, 242, 353, 2, 173, 214, 56, 11, 47, 186, 351, 363, 164, 254, 259, 207, 116, 365, 330, 304, 161, 46, 60, 185, 6, 154, 187, 348, 332, 274, 177, 19, 17, 328, 253, 319, 137, 341, 107, 239, 29, 358, 87, 349, 156, 48, 246, 117, 306, 134, 174, 139, 310, 334, 172, 382, 25, 204, 283, 13, 102, 240, 320, 163, 175, 22, 377, 370, 312, 176, 237, 251, 266, 51, 114, 4, 178, 292, 226, 49, 130, 166, 135, 119, 234, 327, 90, 222, 67, 95, 150, 144, 255, 136, 34, 188, 289, 170, 26, 272, 235, 337, 360, 371, 104, 100, 295, 297, 183, 3, 357, 157, 314, 291, 218, 160, 162, 270, 339, 325, 203, 372, 383, 10, 369, 123, 285, 220, 63, 27, 317, 268, 267, 286, 225, 355, 227, 69, 224, 105, 58, 256, 7, 229, 265, 260, 76, 33, 258, 40, 200, 313, 273, 375, 44, 374, 247, 236, 12, 326, 350, 20, 167, 198, 249, 318, 340, 380, 158, 284, 126, 62, 287, 118, 263, 309, 230, 381, 279, 344, 108, 338, 106, 99, 14, 73, 362, 359, 88, 264, 368, 109, 276, 190, 84, 149, 103, 81, 296, 122, 74, 82, 215, 168, 35, 290, 91, 113, 24, 209, 281, 83, 127, 346, 38, 52, 378, 201, 189, 89, 213, 245, 354, 248, 379, 244, 0, 77, 257, 16, 288, 75, 324, 72, 59, 36, 68, 165, 96, 298, 5, 141, 197, 180, 15, 78, 302, 195, 43, 347, 61, 80, 55, 202, 70, 335, 18, 232, 315, 110, 155, 121, 64, 101, 221, 124, 366, 333, 93, 66, 86, 194, 39, 92, 373, 206, 53, 329, 361, 275, 147, 21, 321, 45, 211, 216, 142, 37, 342, 241, 356, 181, 131, 250, 23, 182, 261, 120, 311, 294, 71, 299, 192, 65, 364, 128, 153, 196, 1, 133, 98, 191, 269, 280, 30, 277, 223, 345, 8, 115, 293, 217, 278, 169, 300, 331, 233, 219, 146, 85, 9, 307, 323, 308, 367, 111, 151, 148, 376, 41, 305, 152, 238, 79, 129, 231, 252, 205, 138, 343, 316, 112, 42, 132, 303, 336, 140, 243, 159, 210] + 39: [282, 36, 176, 116, 153, 218, 292, 329, 204, 98, 15, 163, 146, 381, 352, 127, 31, 278, 331, 358, 276, 5, 226, 33, 72, 289, 193, 4, 243, 377, 169, 120, 58, 335, 189, 266, 206, 0, 197, 210, 326, 124, 12, 308, 29, 148, 361, 209, 310, 93, 100, 264, 1, 13, 155, 244, 297, 374, 141, 369, 51, 200, 84, 274, 340, 112, 190, 131, 62, 366, 63, 301, 212, 311, 241, 354, 338, 309, 114, 265, 283, 192, 287, 222, 78, 246, 252, 10, 219, 129, 294, 73, 154, 11, 52, 143, 247, 38, 85, 180, 348, 215, 279, 187, 145, 277, 172, 76, 178, 87, 375, 8, 248, 251, 138, 47, 152, 199, 364, 249, 14, 102, 45, 133, 128, 6, 130, 214, 166, 109, 288, 168, 157, 160, 175, 229, 233, 147, 205, 105, 332, 90, 321, 22, 66, 312, 313, 35, 223, 139, 242, 272, 270, 71, 295, 319, 77, 23, 202, 118, 48, 122, 225, 69, 367, 150, 322, 213, 182, 228, 95, 173, 230, 16, 220, 194, 376, 217, 196, 303, 108, 305, 123, 92, 255, 351, 32, 318, 57, 115, 165, 149, 104, 65, 314, 280, 234, 151, 119, 382, 174, 333, 250, 330, 49, 7, 235, 170, 191, 299, 101, 74, 50, 177, 307, 161, 134, 355, 88, 320, 275, 30, 363, 181, 60, 291, 19, 111, 337, 231, 211, 359, 238, 136, 370, 316, 344, 186, 334, 44, 372, 286, 132, 99, 110, 25, 360, 271, 296, 324, 40, 341, 365, 81, 298, 259, 42, 323, 239, 34, 126, 135, 371, 162, 302, 94, 106, 203, 164, 254, 306, 183, 91, 70, 27, 103, 46, 347, 37, 64, 342, 158, 67, 224, 184, 261, 9, 89, 236, 258, 379, 96, 82, 156, 325, 328, 80, 362, 17, 24, 208, 290, 227, 53, 232, 28, 336, 137, 353, 188, 144, 75, 345, 293, 185, 97, 357, 79, 68, 207, 171, 343, 41, 159, 55, 20, 18, 253, 339, 349, 125, 237, 269, 201, 21, 198, 56, 315, 43, 268, 368, 113, 267, 86, 2, 26, 121, 300, 3, 59, 273, 383, 140, 54, 263, 260, 378, 281, 317, 179, 373, 61, 167, 285, 221, 195, 240, 327, 216, 245, 346, 117, 107, 83, 142, 39, 284, 350, 256, 262, 356, 380, 304, 257] + 40: [31, 359, 367, 52, 153, 64, 74, 3, 91, 363, 336, 341, 322, 278, 200, 188, 137, 44, 99, 189, 25, 155, 339, 333, 177, 304, 301, 139, 241, 122, 141, 125, 135, 16, 174, 106, 119, 180, 39, 87, 255, 191, 72, 5, 92, 149, 248, 300, 319, 71, 33, 260, 204, 283, 160, 128, 164, 58, 97, 28, 303, 136, 321, 289, 27, 236, 205, 259, 307, 150, 138, 94, 103, 29, 356, 281, 273, 293, 222, 246, 78, 121, 361, 13, 32, 182, 34, 111, 325, 315, 109, 292, 220, 225, 144, 105, 15, 328, 346, 280, 55, 101, 247, 8, 332, 372, 59, 228, 275, 326, 271, 166, 211, 12, 221, 239, 56, 57, 178, 377, 192, 120, 206, 329, 37, 18, 233, 291, 118, 256, 84, 266, 117, 40, 218, 231, 114, 313, 223, 298, 83, 126, 69, 343, 68, 142, 179, 357, 70, 373, 265, 161, 369, 81, 90, 17, 1, 168, 274, 185, 11, 199, 382, 46, 187, 263, 244, 290, 294, 324, 93, 243, 171, 314, 261, 6, 147, 212, 184, 268, 49, 113, 242, 22, 173, 96, 176, 24, 172, 335, 353, 376, 175, 360, 108, 379, 286, 62, 297, 215, 88, 235, 351, 210, 284, 143, 63, 80, 54, 38, 148, 227, 53, 364, 43, 110, 2, 338, 305, 282, 19, 131, 345, 238, 270, 14, 276, 310, 50, 66, 77, 217, 234, 152, 381, 86, 272, 146, 312, 194, 378, 349, 21, 226, 169, 75, 26, 279, 158, 198, 100, 311, 296, 89, 209, 61, 193, 216, 196, 262, 316, 95, 82, 130, 7, 264, 318, 237, 48, 47, 366, 36, 375, 288, 348, 383, 140, 203, 257, 104, 352, 342, 45, 245, 320, 240, 331, 214, 317, 250, 358, 340, 213, 67, 337, 156, 254, 251, 224, 159, 362, 129, 162, 277, 145, 334, 302, 269, 79, 365, 299, 306, 165, 287, 167, 323, 350, 60, 73, 201, 115, 186, 0, 102, 368, 230, 35, 134, 354, 4, 380, 112, 20, 197, 170, 249, 207, 183, 190, 132, 181, 98, 107, 202, 232, 208, 258, 347, 9, 10, 219, 253, 30, 308, 370, 229, 309, 41, 65, 151, 355, 327, 371, 330, 195, 51, 85, 76, 116, 133, 23, 124, 374, 295, 163, 285, 154, 267, 157, 252, 127, 42, 344, 123] + 41: [49, 190, 119, 335, 27, 86, 365, 293, 79, 24, 206, 306, 348, 127, 140, 126, 217, 301, 379, 260, 156, 340, 133, 288, 66, 107, 6, 130, 315, 303, 359, 154, 100, 165, 13, 248, 113, 155, 67, 20, 183, 145, 189, 362, 382, 233, 247, 216, 378, 184, 159, 367, 103, 255, 273, 92, 111, 358, 302, 82, 383, 3, 219, 194, 349, 102, 125, 93, 229, 325, 330, 380, 101, 181, 208, 160, 10, 192, 15, 300, 179, 180, 207, 38, 147, 182, 238, 47, 201, 240, 211, 356, 304, 230, 296, 278, 199, 355, 197, 122, 11, 249, 50, 97, 187, 161, 311, 108, 45, 4, 202, 243, 308, 64, 225, 256, 235, 117, 242, 8, 63, 290, 277, 261, 220, 214, 37, 342, 39, 65, 72, 332, 148, 137, 29, 274, 267, 319, 270, 16, 58, 118, 227, 309, 170, 142, 369, 215, 166, 57, 84, 76, 172, 279, 12, 95, 318, 222, 264, 163, 336, 70, 54, 276, 258, 167, 112, 210, 363, 153, 18, 43, 173, 89, 41, 188, 236, 381, 234, 105, 310, 28, 265, 329, 331, 88, 177, 372, 291, 52, 263, 357, 60, 283, 275, 185, 71, 150, 200, 285, 136, 314, 262, 44, 326, 351, 25, 23, 134, 116, 282, 55, 237, 289, 224, 266, 176, 186, 19, 213, 244, 146, 59, 272, 352, 0, 141, 68, 73, 175, 171, 370, 110, 323, 377, 109, 203, 333, 31, 120, 350, 114, 366, 305, 376, 307, 168, 96, 34, 69, 347, 337, 104, 53, 286, 218, 252, 56, 313, 254, 271, 115, 144, 328, 151, 295, 81, 317, 61, 128, 123, 292, 1, 191, 259, 48, 257, 204, 375, 373, 121, 343, 106, 341, 281, 228, 22, 85, 374, 298, 129, 87, 196, 90, 223, 124, 7, 360, 334, 198, 9, 324, 42, 353, 26, 32, 226, 83, 294, 138, 250, 268, 299, 284, 91, 94, 74, 221, 99, 75, 78, 77, 297, 149, 316, 209, 14, 245, 62, 2, 162, 346, 169, 135, 40, 132, 321, 269, 30, 205, 251, 212, 368, 327, 364, 21, 280, 253, 80, 35, 239, 36, 174, 345, 241, 46, 322, 344, 339, 51, 131, 338, 246, 5, 371, 354, 232, 33, 143, 287, 320, 158, 98, 195, 231, 17, 139, 157, 312, 361, 193, 164, 178, 152] + 42: [123, 184, 212, 41, 349, 55, 281, 327, 351, 370, 29, 135, 28, 329, 36, 92, 33, 362, 380, 241, 301, 225, 323, 183, 226, 61, 251, 156, 364, 84, 284, 82, 360, 23, 288, 347, 96, 353, 107, 14, 271, 163, 273, 242, 25, 71, 95, 254, 152, 93, 154, 50, 106, 151, 150, 337, 377, 208, 21, 235, 248, 30, 67, 17, 173, 216, 314, 316, 257, 192, 113, 311, 62, 89, 315, 58, 205, 378, 278, 26, 194, 267, 90, 363, 124, 334, 22, 85, 190, 110, 78, 43, 91, 289, 294, 324, 117, 218, 2, 1, 303, 116, 264, 262, 263, 209, 189, 373, 328, 221, 169, 344, 147, 181, 322, 52, 223, 245, 101, 132, 307, 238, 49, 310, 196, 16, 270, 293, 120, 206, 98, 73, 185, 136, 207, 133, 276, 79, 54, 142, 240, 352, 369, 339, 31, 155, 333, 375, 114, 211, 63, 374, 220, 6, 342, 191, 275, 320, 140, 187, 87, 326, 335, 280, 118, 149, 32, 60, 8, 371, 260, 130, 34, 298, 355, 269, 233, 175, 272, 197, 104, 285, 304, 309, 153, 12, 11, 81, 317, 365, 46, 126, 359, 172, 256, 210, 148, 228, 350, 72, 195, 306, 255, 122, 379, 131, 4, 239, 230, 201, 356, 77, 366, 39, 160, 340, 137, 372, 9, 139, 232, 258, 291, 68, 112, 213, 229, 321, 128, 292, 330, 367, 253, 35, 70, 159, 382, 66, 361, 376, 217, 176, 158, 188, 42, 357, 215, 179, 243, 200, 48, 265, 358, 354, 259, 224, 214, 161, 103, 164, 45, 76, 381, 18, 146, 180, 64, 244, 299, 313, 332, 125, 300, 261, 290, 145, 368, 75, 186, 383, 69, 115, 83, 182, 0, 15, 24, 27, 94, 345, 111, 37, 20, 305, 279, 282, 162, 129, 59, 127, 277, 274, 178, 236, 287, 165, 308, 38, 202, 227, 65, 80, 237, 219, 246, 109, 99, 5, 336, 53, 346, 343, 199, 102, 312, 296, 144, 174, 88, 47, 57, 331, 13, 86, 40, 121, 266, 268, 325, 138, 295, 119, 247, 168, 222, 108, 203, 319, 348, 141, 341, 286, 167, 105, 338, 143, 234, 166, 297, 51, 19, 231, 283, 157, 97, 7, 204, 170, 171, 56, 318, 250, 10, 3, 198, 302, 134, 249, 100, 44, 177, 74, 252, 193] + 43: [168, 359, 298, 262, 278, 112, 20, 103, 152, 55, 127, 313, 307, 321, 4, 222, 282, 204, 306, 26, 36, 67, 334, 148, 293, 180, 56, 74, 9, 370, 295, 329, 144, 353, 118, 134, 211, 357, 341, 69, 7, 141, 383, 104, 25, 166, 331, 44, 225, 17, 30, 344, 308, 35, 13, 94, 131, 186, 322, 227, 198, 121, 89, 191, 238, 275, 339, 271, 160, 365, 155, 243, 60, 38, 45, 251, 309, 343, 159, 304, 267, 76, 19, 173, 265, 316, 162, 114, 41, 228, 42, 133, 350, 54, 189, 354, 375, 335, 297, 27, 332, 197, 349, 366, 57, 337, 111, 264, 1, 147, 368, 150, 90, 346, 277, 299, 149, 84, 196, 107, 49, 229, 143, 120, 263, 288, 170, 122, 8, 65, 151, 233, 290, 240, 311, 124, 106, 236, 208, 333, 33, 310, 192, 95, 66, 372, 75, 369, 6, 142, 59, 254, 22, 81, 138, 323, 274, 269, 374, 2, 232, 327, 83, 123, 296, 130, 241, 145, 99, 47, 210, 284, 101, 174, 116, 207, 294, 132, 158, 268, 300, 110, 260, 230, 62, 249, 270, 58, 289, 361, 206, 214, 98, 92, 283, 23, 177, 68, 216, 252, 326, 377, 29, 360, 305, 202, 255, 272, 154, 246, 234, 140, 5, 61, 10, 63, 185, 165, 217, 46, 128, 325, 175, 72, 279, 87, 287, 358, 355, 187, 285, 248, 172, 281, 340, 139, 244, 184, 373, 245, 315, 347, 190, 164, 82, 129, 364, 239, 363, 93, 223, 203, 338, 136, 379, 32, 378, 48, 286, 135, 367, 257, 237, 345, 24, 50, 253, 167, 220, 226, 18, 64, 71, 117, 193, 109, 320, 179, 312, 199, 328, 21, 362, 91, 176, 231, 195, 3, 221, 156, 14, 351, 181, 371, 37, 97, 235, 291, 342, 318, 108, 78, 250, 73, 205, 88, 219, 200, 261, 157, 194, 273, 102, 113, 266, 213, 125, 80, 31, 292, 356, 163, 381, 119, 330, 258, 51, 276, 0, 137, 352, 96, 212, 302, 169, 348, 105, 171, 324, 43, 161, 218, 39, 100, 314, 153, 301, 178, 224, 280, 85, 77, 52, 380, 86, 115, 34, 79, 28, 12, 242, 183, 16, 70, 146, 256, 215, 11, 259, 126, 247, 209, 188, 336, 201, 53, 317, 303, 40, 15, 319, 376, 182, 382] + 44: [213, 101, 238, 272, 60, 257, 252, 325, 93, 86, 118, 329, 144, 337, 276, 138, 350, 290, 150, 70, 130, 380, 87, 134, 162, 136, 83, 159, 243, 309, 211, 237, 230, 345, 49, 114, 245, 375, 373, 99, 247, 176, 293, 31, 58, 75, 209, 106, 77, 379, 284, 167, 110, 227, 29, 135, 30, 280, 289, 63, 102, 217, 355, 117, 244, 299, 173, 201, 132, 286, 202, 369, 107, 46, 370, 40, 59, 318, 341, 113, 234, 109, 367, 65, 35, 80, 239, 210, 283, 260, 297, 52, 76, 181, 339, 116, 301, 222, 223, 97, 178, 180, 43, 140, 11, 187, 182, 306, 304, 275, 158, 265, 85, 175, 163, 364, 79, 338, 54, 120, 36, 126, 15, 185, 149, 231, 23, 20, 72, 193, 4, 51, 255, 200, 195, 119, 298, 268, 356, 111, 88, 371, 13, 332, 191, 171, 170, 0, 351, 188, 277, 279, 143, 98, 212, 198, 228, 125, 343, 91, 224, 61, 154, 39, 71, 378, 174, 177, 327, 344, 196, 74, 205, 347, 32, 305, 203, 319, 204, 62, 273, 216, 5, 115, 28, 330, 37, 152, 148, 16, 161, 322, 199, 172, 346, 17, 300, 84, 68, 331, 166, 146, 242, 294, 365, 360, 14, 340, 192, 24, 122, 287, 285, 160, 269, 381, 128, 220, 278, 334, 105, 323, 81, 3, 263, 357, 256, 137, 312, 1, 27, 151, 363, 302, 47, 73, 316, 281, 295, 374, 10, 274, 168, 64, 155, 190, 82, 311, 236, 383, 333, 142, 33, 95, 324, 262, 6, 186, 219, 56, 352, 366, 359, 44, 157, 41, 92, 90, 214, 377, 169, 104, 241, 9, 53, 183, 2, 253, 22, 251, 372, 307, 123, 127, 353, 308, 112, 303, 317, 362, 21, 25, 271, 215, 328, 368, 34, 320, 249, 264, 358, 225, 89, 121, 266, 296, 55, 100, 131, 248, 67, 221, 292, 354, 108, 165, 270, 19, 336, 179, 235, 18, 314, 232, 42, 45, 153, 50, 240, 69, 139, 335, 133, 208, 189, 12, 226, 348, 145, 258, 382, 267, 376, 229, 288, 313, 141, 38, 48, 291, 94, 207, 184, 254, 349, 315, 66, 164, 96, 261, 342, 26, 147, 282, 8, 78, 156, 218, 57, 250, 321, 7, 124, 326, 233, 194, 246, 103, 259, 361, 197, 310, 129, 206] + 45: [92, 95, 219, 284, 120, 375, 286, 165, 128, 183, 228, 223, 16, 104, 250, 112, 80, 213, 157, 315, 236, 4, 340, 65, 34, 205, 38, 264, 231, 255, 204, 105, 15, 147, 93, 136, 332, 31, 382, 200, 265, 259, 75, 237, 299, 361, 100, 174, 140, 27, 164, 318, 317, 274, 369, 354, 61, 349, 234, 254, 307, 158, 277, 70, 313, 267, 86, 108, 175, 67, 22, 253, 58, 116, 46, 358, 29, 322, 127, 288, 142, 289, 368, 202, 225, 90, 62, 143, 270, 366, 311, 184, 11, 240, 138, 64, 178, 8, 329, 301, 144, 25, 176, 378, 47, 348, 52, 275, 239, 162, 155, 74, 12, 327, 249, 217, 292, 96, 1, 370, 330, 126, 87, 5, 150, 333, 381, 374, 363, 48, 78, 343, 238, 321, 89, 82, 353, 53, 216, 325, 49, 72, 355, 221, 10, 201, 276, 173, 134, 139, 171, 103, 269, 124, 224, 39, 118, 310, 229, 245, 84, 7, 345, 44, 32, 2, 123, 218, 344, 170, 304, 320, 294, 379, 342, 177, 79, 203, 371, 362, 337, 323, 56, 152, 159, 194, 6, 335, 195, 17, 117, 83, 227, 334, 33, 383, 316, 182, 356, 262, 21, 163, 380, 258, 187, 196, 36, 88, 54, 357, 135, 281, 279, 109, 261, 257, 35, 19, 252, 145, 37, 211, 263, 314, 0, 119, 3, 377, 111, 214, 153, 167, 193, 295, 77, 28, 60, 69, 244, 347, 154, 42, 367, 191, 188, 346, 141, 146, 298, 212, 272, 331, 359, 45, 326, 372, 328, 251, 242, 94, 106, 208, 303, 222, 20, 207, 319, 291, 283, 241, 151, 285, 220, 341, 113, 71, 73, 102, 280, 97, 232, 352, 287, 99, 130, 181, 137, 266, 23, 293, 373, 81, 110, 197, 339, 85, 180, 297, 107, 248, 168, 122, 51, 30, 308, 306, 360, 179, 260, 24, 302, 50, 76, 133, 290, 59, 9, 273, 57, 129, 166, 309, 268, 190, 91, 125, 18, 186, 206, 101, 278, 66, 235, 14, 40, 192, 121, 161, 350, 324, 233, 199, 247, 160, 131, 114, 209, 351, 198, 55, 115, 338, 68, 210, 41, 156, 26, 365, 376, 63, 43, 230, 256, 296, 98, 282, 169, 271, 246, 312, 364, 215, 243, 132, 189, 226, 305, 148, 13, 172, 149, 185, 336, 300] + 46: [142, 355, 106, 201, 46, 238, 6, 301, 60, 114, 199, 3, 85, 62, 117, 260, 132, 118, 192, 298, 23, 250, 237, 193, 383, 373, 78, 297, 189, 9, 265, 145, 283, 262, 244, 371, 43, 358, 330, 126, 196, 296, 185, 100, 178, 365, 122, 175, 183, 253, 367, 188, 240, 51, 324, 312, 40, 165, 2, 320, 282, 280, 33, 372, 334, 315, 109, 13, 41, 272, 267, 158, 377, 329, 92, 24, 222, 56, 317, 241, 91, 370, 257, 52, 138, 275, 341, 363, 269, 29, 116, 151, 225, 82, 379, 356, 352, 380, 354, 210, 327, 202, 332, 12, 187, 55, 44, 219, 215, 131, 195, 227, 232, 197, 137, 50, 362, 273, 103, 59, 111, 308, 97, 169, 223, 69, 293, 342, 177, 93, 98, 150, 163, 310, 31, 236, 1, 174, 276, 70, 14, 147, 5, 213, 311, 286, 149, 335, 233, 96, 221, 344, 271, 38, 18, 146, 102, 30, 143, 120, 80, 218, 156, 316, 63, 95, 205, 157, 45, 101, 263, 289, 119, 318, 159, 277, 261, 217, 309, 171, 208, 15, 77, 366, 381, 48, 181, 47, 349, 270, 220, 27, 133, 58, 230, 66, 368, 170, 246, 284, 136, 68, 303, 203, 350, 351, 108, 155, 99, 353, 295, 166, 176, 128, 172, 340, 336, 307, 328, 360, 313, 173, 348, 20, 333, 90, 361, 234, 337, 112, 127, 229, 300, 134, 251, 54, 0, 28, 231, 36, 10, 256, 207, 152, 19, 25, 274, 154, 124, 254, 345, 76, 79, 211, 8, 376, 287, 121, 180, 37, 129, 294, 198, 17, 125, 226, 35, 144, 89, 161, 7, 74, 359, 190, 285, 339, 347, 135, 258, 281, 87, 73, 306, 123, 84, 167, 252, 88, 247, 139, 110, 319, 314, 16, 239, 104, 194, 204, 113, 242, 266, 228, 130, 22, 326, 168, 259, 369, 304, 53, 292, 235, 268, 32, 141, 290, 299, 346, 71, 322, 264, 291, 86, 278, 26, 67, 75, 94, 288, 105, 212, 179, 200, 182, 325, 302, 34, 64, 343, 140, 224, 249, 191, 209, 214, 378, 338, 374, 39, 49, 115, 83, 186, 245, 42, 4, 164, 162, 216, 321, 57, 81, 206, 160, 255, 382, 243, 153, 364, 331, 184, 305, 375, 65, 21, 11, 72, 107, 279, 248, 357, 148, 323, 61] + 47: [195, 307, 347, 357, 193, 309, 215, 196, 21, 40, 342, 63, 313, 95, 68, 166, 231, 65, 184, 51, 101, 173, 252, 183, 139, 211, 163, 281, 37, 107, 132, 297, 150, 275, 57, 325, 108, 339, 277, 88, 270, 167, 125, 254, 169, 7, 369, 326, 3, 29, 137, 171, 119, 61, 314, 353, 274, 287, 188, 363, 251, 351, 69, 104, 249, 324, 284, 199, 0, 138, 233, 112, 315, 80, 15, 289, 276, 306, 115, 20, 97, 160, 48, 282, 383, 174, 58, 220, 202, 244, 99, 165, 13, 272, 191, 246, 343, 158, 134, 179, 35, 170, 90, 294, 362, 240, 59, 153, 114, 330, 84, 219, 151, 228, 370, 197, 296, 148, 39, 210, 234, 141, 243, 28, 225, 60, 341, 260, 130, 190, 152, 302, 245, 5, 133, 222, 47, 299, 283, 206, 182, 83, 156, 293, 54, 22, 75, 192, 279, 280, 345, 316, 198, 168, 346, 367, 98, 226, 93, 301, 223, 285, 323, 255, 205, 17, 4, 354, 378, 336, 348, 147, 259, 111, 185, 32, 81, 9, 242, 216, 358, 374, 16, 105, 76, 142, 116, 217, 221, 311, 214, 360, 44, 154, 300, 136, 371, 34, 14, 257, 36, 110, 118, 218, 159, 176, 26, 377, 124, 263, 298, 77, 203, 25, 45, 27, 53, 91, 321, 213, 273, 258, 200, 157, 144, 10, 365, 266, 74, 290, 30, 317, 145, 11, 268, 256, 241, 318, 236, 56, 352, 162, 382, 331, 291, 227, 333, 320, 175, 129, 131, 123, 212, 250, 269, 67, 224, 103, 140, 364, 322, 355, 52, 106, 23, 356, 71, 126, 328, 332, 329, 186, 253, 46, 120, 72, 177, 380, 86, 38, 372, 264, 161, 295, 8, 113, 239, 288, 24, 181, 359, 187, 43, 1, 286, 127, 344, 267, 327, 229, 122, 230, 155, 135, 42, 41, 33, 50, 12, 338, 180, 117, 366, 18, 121, 303, 292, 278, 49, 310, 334, 102, 19, 308, 207, 350, 100, 178, 85, 64, 78, 232, 261, 128, 62, 373, 312, 87, 271, 143, 375, 109, 305, 368, 79, 376, 238, 349, 235, 265, 208, 262, 89, 319, 172, 73, 204, 237, 340, 164, 96, 201, 70, 335, 337, 94, 31, 2, 248, 149, 189, 55, 379, 92, 304, 361, 247, 194, 82, 6, 66, 381, 146, 209] + 48: [17, 52, 85, 288, 65, 358, 99, 90, 83, 369, 235, 48, 325, 252, 290, 219, 161, 25, 75, 88, 118, 31, 172, 218, 22, 100, 158, 63, 374, 40, 206, 259, 382, 272, 214, 69, 37, 87, 309, 148, 154, 153, 124, 68, 103, 199, 209, 196, 271, 276, 78, 9, 117, 226, 121, 164, 268, 375, 223, 264, 185, 380, 368, 312, 322, 192, 247, 73, 89, 212, 275, 231, 307, 70, 49, 36, 269, 367, 346, 107, 82, 71, 366, 29, 165, 335, 66, 356, 86, 345, 251, 122, 337, 348, 245, 227, 140, 239, 141, 159, 270, 150, 236, 58, 33, 6, 5, 372, 342, 134, 232, 222, 130, 291, 233, 45, 255, 24, 168, 57, 317, 281, 72, 204, 280, 12, 55, 274, 178, 256, 95, 93, 143, 277, 350, 149, 171, 260, 365, 126, 265, 355, 304, 114, 328, 273, 381, 297, 77, 109, 19, 74, 157, 189, 177, 254, 34, 305, 378, 300, 7, 293, 112, 324, 162, 221, 230, 79, 202, 113, 289, 81, 84, 18, 98, 137, 243, 286, 194, 326, 323, 213, 228, 191, 8, 377, 332, 383, 364, 131, 1, 250, 132, 330, 353, 263, 249, 301, 234, 295, 133, 287, 139, 363, 186, 354, 371, 210, 246, 299, 174, 123, 341, 111, 4, 13, 379, 351, 119, 97, 43, 349, 144, 35, 39, 173, 310, 313, 27, 294, 238, 225, 298, 60, 183, 262, 197, 333, 30, 308, 267, 102, 101, 237, 26, 145, 343, 182, 28, 104, 327, 208, 316, 181, 155, 229, 179, 50, 120, 190, 170, 357, 315, 42, 339, 14, 138, 302, 352, 53, 169, 59, 216, 32, 135, 373, 224, 61, 201, 376, 331, 282, 248, 23, 220, 167, 136, 314, 283, 128, 311, 15, 20, 279, 44, 2, 21, 166, 240, 38, 142, 41, 320, 156, 193, 257, 370, 0, 160, 242, 347, 360, 180, 241, 96, 11, 198, 127, 292, 94, 261, 47, 217, 3, 56, 105, 110, 188, 338, 106, 296, 176, 303, 258, 10, 129, 76, 266, 284, 62, 51, 215, 116, 203, 207, 147, 344, 359, 115, 184, 319, 205, 175, 152, 285, 340, 329, 321, 318, 91, 64, 80, 146, 253, 211, 16, 151, 195, 108, 187, 92, 67, 163, 278, 244, 361, 362, 125, 46, 54, 306, 336, 200, 334] + 49: [302, 37, 209, 115, 212, 208, 148, 60, 91, 74, 381, 264, 94, 48, 97, 362, 80, 56, 277, 29, 4, 81, 139, 179, 35, 338, 172, 85, 71, 355, 84, 192, 143, 145, 168, 316, 7, 341, 287, 21, 249, 230, 32, 68, 135, 356, 123, 191, 321, 157, 197, 304, 181, 78, 261, 201, 177, 133, 260, 194, 30, 332, 195, 366, 336, 310, 16, 138, 365, 34, 41, 273, 371, 352, 98, 227, 114, 176, 358, 12, 31, 190, 154, 364, 88, 343, 87, 372, 334, 161, 141, 263, 129, 188, 300, 64, 297, 305, 242, 251, 294, 59, 353, 25, 276, 325, 107, 132, 187, 45, 314, 333, 144, 174, 137, 221, 140, 170, 189, 296, 301, 82, 327, 134, 351, 244, 53, 117, 200, 151, 228, 22, 52, 232, 166, 51, 165, 162, 171, 350, 203, 258, 50, 322, 253, 155, 9, 193, 83, 65, 293, 241, 90, 44, 292, 220, 344, 8, 100, 122, 328, 254, 340, 368, 240, 128, 89, 222, 225, 303, 238, 178, 103, 345, 280, 378, 46, 2, 342, 256, 125, 324, 27, 315, 363, 102, 236, 207, 291, 3, 347, 375, 126, 75, 271, 210, 15, 234, 116, 110, 180, 262, 380, 289, 312, 255, 95, 5, 205, 73, 63, 245, 196, 198, 160, 163, 146, 10, 308, 62, 359, 239, 72, 119, 0, 204, 101, 283, 182, 326, 57, 93, 367, 266, 99, 152, 252, 26, 370, 38, 374, 268, 313, 233, 346, 136, 213, 40, 335, 39, 218, 246, 120, 331, 131, 109, 121, 1, 17, 23, 223, 214, 224, 248, 376, 173, 33, 43, 307, 339, 47, 226, 106, 49, 285, 108, 299, 104, 357, 167, 237, 159, 379, 281, 247, 288, 373, 317, 79, 6, 202, 184, 319, 150, 66, 278, 275, 199, 76, 243, 329, 175, 183, 235, 250, 169, 382, 86, 311, 290, 112, 127, 55, 216, 67, 330, 156, 142, 320, 130, 61, 124, 282, 361, 348, 185, 111, 147, 42, 96, 231, 215, 383, 219, 24, 92, 360, 272, 19, 149, 257, 158, 105, 270, 153, 259, 286, 306, 269, 186, 298, 164, 58, 279, 28, 54, 70, 318, 323, 77, 20, 295, 267, 36, 309, 14, 211, 69, 118, 206, 284, 113, 354, 274, 377, 13, 229, 18, 265, 217, 11, 369, 349, 337] + 50: [86, 69, 26, 33, 140, 370, 125, 304, 329, 256, 295, 184, 15, 133, 122, 71, 165, 375, 219, 189, 180, 310, 81, 64, 160, 105, 278, 347, 362, 153, 339, 349, 156, 141, 354, 162, 291, 209, 21, 201, 216, 31, 120, 52, 131, 23, 50, 345, 268, 215, 350, 57, 137, 367, 287, 280, 327, 334, 114, 130, 121, 274, 301, 239, 186, 194, 290, 53, 276, 267, 73, 152, 242, 340, 108, 5, 297, 4, 344, 7, 305, 212, 90, 36, 82, 40, 381, 103, 328, 158, 144, 115, 307, 222, 55, 365, 29, 188, 27, 128, 311, 229, 262, 373, 377, 296, 88, 79, 272, 270, 363, 196, 313, 19, 348, 359, 247, 208, 94, 92, 163, 255, 211, 223, 37, 39, 337, 259, 63, 308, 68, 150, 323, 147, 77, 10, 246, 245, 104, 161, 118, 336, 241, 93, 100, 210, 325, 129, 226, 333, 352, 146, 175, 220, 217, 314, 253, 213, 6, 24, 380, 75, 203, 134, 252, 250, 139, 303, 107, 126, 99, 113, 135, 127, 204, 177, 200, 320, 237, 355, 285, 11, 324, 13, 47, 360, 279, 309, 254, 364, 182, 111, 316, 315, 214, 3, 235, 38, 284, 366, 312, 199, 0, 22, 155, 249, 353, 96, 48, 228, 183, 102, 206, 197, 85, 234, 322, 145, 172, 251, 236, 191, 281, 123, 106, 46, 383, 148, 171, 87, 346, 61, 157, 70, 142, 167, 227, 368, 257, 136, 109, 32, 56, 1, 25, 67, 218, 273, 62, 338, 341, 119, 271, 159, 298, 275, 149, 117, 263, 240, 319, 342, 371, 74, 20, 91, 28, 185, 300, 202, 292, 170, 351, 238, 89, 258, 65, 112, 101, 76, 173, 72, 261, 221, 260, 59, 283, 230, 49, 376, 232, 18, 80, 97, 361, 35, 9, 45, 193, 331, 224, 335, 132, 41, 244, 58, 116, 168, 43, 174, 151, 326, 277, 78, 357, 143, 178, 369, 288, 42, 34, 372, 330, 318, 98, 179, 30, 83, 207, 265, 266, 225, 302, 66, 332, 374, 181, 166, 321, 231, 299, 14, 233, 110, 282, 192, 358, 51, 343, 16, 187, 164, 306, 195, 154, 379, 169, 269, 382, 378, 294, 2, 205, 356, 243, 17, 289, 124, 198, 12, 248, 293, 138, 286, 60, 176, 8, 190, 95, 54, 317, 44, 84, 264] + 51: [24, 12, 229, 73, 16, 182, 64, 65, 60, 235, 105, 253, 49, 247, 374, 23, 275, 74, 341, 366, 192, 34, 309, 195, 361, 320, 289, 332, 115, 89, 328, 278, 319, 339, 43, 240, 83, 354, 158, 277, 70, 133, 185, 244, 21, 226, 326, 348, 186, 208, 126, 189, 111, 52, 136, 62, 104, 301, 304, 168, 11, 5, 216, 47, 245, 142, 86, 329, 280, 91, 321, 44, 236, 265, 69, 383, 315, 381, 114, 242, 93, 109, 53, 25, 66, 9, 257, 305, 355, 227, 302, 347, 76, 255, 156, 202, 372, 323, 82, 106, 98, 284, 176, 211, 270, 124, 129, 177, 346, 92, 35, 85, 212, 308, 237, 107, 279, 351, 204, 251, 10, 362, 324, 311, 94, 258, 379, 191, 307, 273, 55, 167, 125, 148, 28, 272, 154, 132, 175, 274, 225, 267, 327, 249, 102, 200, 144, 286, 371, 113, 228, 72, 87, 32, 330, 146, 269, 314, 259, 20, 285, 344, 172, 268, 18, 181, 59, 160, 166, 41, 224, 171, 100, 312, 350, 338, 145, 363, 336, 378, 184, 1, 230, 165, 174, 217, 179, 266, 22, 38, 121, 139, 137, 110, 287, 31, 206, 193, 180, 264, 135, 203, 233, 153, 246, 263, 8, 294, 292, 382, 97, 162, 380, 310, 370, 119, 50, 48, 234, 317, 369, 365, 27, 54, 276, 333, 159, 112, 218, 57, 4, 14, 199, 376, 187, 352, 36, 231, 151, 290, 373, 210, 17, 108, 6, 99, 356, 300, 343, 95, 250, 283, 130, 26, 46, 220, 331, 161, 81, 198, 219, 190, 79, 252, 141, 3, 205, 262, 196, 349, 170, 223, 80, 340, 197, 232, 19, 116, 7, 215, 0, 120, 293, 29, 117, 377, 2, 138, 375, 37, 342, 322, 101, 357, 122, 67, 140, 260, 188, 295, 103, 335, 299, 118, 281, 303, 254, 56, 42, 209, 207, 33, 368, 164, 306, 84, 313, 297, 45, 194, 78, 298, 58, 238, 149, 214, 39, 296, 77, 128, 71, 63, 147, 131, 155, 261, 221, 241, 337, 163, 178, 123, 127, 358, 152, 40, 325, 282, 243, 134, 13, 143, 183, 271, 345, 359, 360, 222, 90, 248, 75, 239, 157, 364, 213, 334, 51, 30, 288, 353, 61, 316, 15, 68, 96, 88, 201, 367, 169, 318, 150, 256, 173, 291] + 52: [381, 209, 121, 372, 130, 289, 345, 82, 41, 33, 26, 269, 349, 50, 325, 379, 222, 105, 227, 23, 134, 146, 301, 331, 213, 333, 326, 268, 58, 110, 59, 152, 318, 249, 143, 342, 114, 4, 353, 276, 297, 203, 159, 142, 317, 199, 204, 176, 99, 239, 92, 217, 151, 323, 34, 298, 382, 60, 44, 168, 252, 98, 88, 133, 221, 191, 281, 160, 116, 280, 3, 154, 279, 208, 45, 225, 2, 277, 244, 21, 139, 347, 303, 197, 111, 96, 358, 363, 9, 185, 202, 144, 373, 264, 212, 180, 334, 336, 327, 216, 63, 214, 206, 77, 183, 68, 80, 192, 54, 354, 37, 171, 378, 5, 296, 30, 233, 292, 361, 319, 39, 287, 40, 93, 201, 177, 294, 189, 179, 100, 356, 322, 226, 311, 181, 112, 36, 187, 19, 15, 86, 272, 370, 247, 102, 137, 52, 332, 31, 73, 321, 193, 184, 190, 140, 196, 210, 236, 70, 312, 285, 286, 315, 124, 248, 369, 72, 17, 374, 337, 306, 308, 89, 242, 282, 84, 220, 129, 188, 118, 344, 313, 229, 302, 145, 255, 65, 122, 240, 360, 162, 348, 107, 103, 25, 231, 267, 362, 290, 211, 78, 346, 357, 219, 355, 95, 257, 123, 256, 234, 341, 223, 364, 300, 76, 53, 69, 66, 27, 161, 352, 383, 20, 238, 51, 136, 28, 310, 340, 343, 293, 147, 132, 314, 265, 338, 14, 295, 16, 305, 157, 283, 48, 307, 320, 246, 195, 263, 7, 172, 245, 259, 266, 8, 291, 108, 339, 166, 83, 173, 75, 243, 113, 377, 170, 324, 109, 119, 270, 200, 365, 207, 230, 330, 101, 380, 18, 126, 304, 275, 158, 42, 273, 241, 1, 376, 46, 175, 71, 97, 186, 61, 182, 85, 174, 163, 13, 62, 260, 56, 87, 149, 309, 228, 74, 117, 359, 167, 367, 6, 153, 351, 316, 55, 125, 235, 57, 258, 148, 194, 47, 198, 49, 141, 43, 299, 135, 232, 261, 79, 11, 366, 254, 94, 218, 375, 104, 165, 205, 120, 371, 250, 271, 169, 90, 288, 10, 128, 32, 64, 35, 155, 127, 262, 178, 215, 29, 67, 328, 115, 22, 91, 253, 350, 237, 284, 224, 251, 150, 368, 156, 278, 81, 335, 138, 274, 329, 131, 12, 0, 24, 164, 106, 38] + 53: [110, 363, 118, 160, 309, 237, 225, 134, 269, 127, 62, 12, 361, 77, 313, 197, 259, 159, 175, 146, 111, 339, 57, 370, 136, 358, 251, 101, 275, 7, 14, 43, 177, 1, 74, 34, 267, 212, 98, 286, 223, 307, 167, 185, 84, 114, 210, 272, 240, 311, 186, 68, 247, 280, 345, 92, 180, 268, 162, 130, 125, 24, 325, 94, 381, 383, 152, 300, 176, 89, 122, 362, 369, 216, 375, 189, 236, 333, 299, 59, 258, 21, 201, 289, 323, 48, 39, 64, 163, 260, 211, 165, 11, 191, 314, 312, 73, 354, 4, 8, 66, 173, 214, 155, 145, 360, 194, 103, 368, 284, 241, 168, 346, 341, 188, 324, 277, 47, 15, 143, 328, 120, 2, 166, 96, 215, 198, 294, 0, 45, 53, 80, 331, 83, 298, 106, 303, 263, 288, 213, 33, 99, 377, 316, 38, 109, 32, 217, 222, 142, 273, 227, 132, 6, 291, 26, 285, 326, 5, 200, 25, 338, 205, 378, 373, 281, 72, 282, 226, 128, 174, 65, 158, 278, 133, 220, 242, 257, 108, 380, 29, 199, 342, 113, 372, 356, 41, 27, 204, 245, 265, 18, 16, 296, 330, 75, 270, 164, 365, 42, 302, 202, 376, 81, 121, 20, 95, 256, 329, 112, 116, 37, 60, 10, 76, 353, 233, 248, 178, 276, 207, 28, 138, 232, 254, 234, 352, 264, 305, 85, 348, 184, 93, 104, 91, 61, 274, 58, 244, 140, 317, 287, 250, 230, 297, 129, 46, 224, 228, 156, 351, 144, 119, 31, 253, 3, 262, 141, 172, 239, 52, 150, 126, 359, 340, 36, 206, 321, 306, 117, 40, 364, 195, 54, 137, 246, 100, 82, 310, 229, 208, 135, 190, 181, 187, 318, 22, 56, 13, 334, 295, 336, 243, 320, 322, 379, 266, 209, 86, 332, 221, 50, 319, 88, 139, 183, 231, 335, 235, 149, 315, 293, 357, 196, 169, 105, 327, 79, 255, 182, 55, 337, 367, 131, 9, 51, 147, 301, 374, 382, 192, 343, 304, 344, 252, 90, 350, 23, 347, 308, 203, 238, 292, 179, 170, 279, 157, 261, 271, 107, 171, 219, 70, 69, 151, 49, 78, 349, 44, 102, 67, 35, 115, 371, 19, 30, 123, 218, 97, 87, 154, 283, 249, 63, 193, 17, 366, 161, 148, 124, 355, 71, 290, 153] + 54: [363, 71, 39, 80, 172, 341, 290, 311, 180, 228, 121, 70, 310, 357, 50, 325, 142, 109, 376, 186, 100, 67, 48, 85, 137, 62, 81, 262, 9, 175, 334, 110, 234, 367, 261, 89, 66, 270, 307, 134, 169, 297, 383, 106, 354, 336, 361, 378, 370, 299, 296, 83, 82, 250, 72, 254, 330, 119, 7, 329, 276, 333, 129, 64, 323, 190, 159, 279, 204, 237, 183, 318, 69, 163, 22, 242, 377, 273, 128, 266, 371, 219, 346, 29, 249, 108, 248, 374, 51, 65, 112, 256, 19, 200, 36, 148, 294, 231, 286, 253, 90, 260, 20, 324, 93, 57, 117, 86, 16, 132, 47, 32, 322, 91, 185, 272, 239, 184, 365, 302, 278, 264, 301, 136, 96, 33, 347, 191, 351, 12, 88, 145, 236, 274, 343, 362, 149, 35, 104, 252, 37, 342, 123, 13, 339, 223, 280, 380, 147, 257, 98, 306, 208, 305, 309, 3, 27, 156, 41, 196, 158, 17, 321, 182, 24, 21, 312, 49, 1, 366, 293, 107, 304, 173, 18, 300, 194, 229, 213, 6, 178, 233, 79, 60, 199, 116, 210, 188, 222, 349, 127, 160, 155, 356, 230, 153, 338, 150, 30, 281, 282, 221, 25, 314, 101, 332, 99, 218, 277, 225, 265, 92, 382, 157, 105, 135, 177, 373, 327, 122, 179, 162, 151, 316, 245, 243, 259, 235, 247, 331, 268, 46, 226, 154, 4, 77, 126, 78, 202, 232, 164, 11, 38, 337, 189, 216, 133, 174, 120, 187, 26, 118, 181, 375, 76, 197, 198, 335, 220, 152, 54, 381, 55, 368, 61, 144, 326, 111, 146, 167, 166, 23, 141, 251, 308, 59, 168, 289, 359, 102, 227, 292, 28, 138, 165, 193, 358, 320, 284, 258, 348, 295, 328, 287, 369, 53, 44, 345, 352, 372, 58, 10, 143, 113, 215, 45, 5, 217, 103, 140, 97, 315, 73, 171, 255, 313, 364, 353, 244, 43, 238, 271, 15, 224, 14, 31, 269, 203, 56, 360, 350, 291, 207, 340, 130, 205, 283, 42, 240, 95, 319, 267, 214, 8, 176, 87, 40, 75, 317, 288, 115, 161, 84, 124, 94, 34, 125, 170, 355, 131, 246, 298, 68, 209, 201, 114, 195, 275, 139, 192, 63, 74, 211, 206, 52, 263, 285, 303, 379, 2, 241, 344, 212, 0] + 55: [154, 323, 113, 256, 62, 39, 16, 41, 121, 106, 249, 71, 272, 300, 316, 233, 197, 276, 95, 260, 231, 343, 194, 136, 318, 157, 207, 313, 63, 160, 127, 173, 356, 321, 168, 315, 66, 68, 280, 169, 330, 362, 100, 345, 205, 222, 245, 109, 174, 90, 269, 43, 151, 89, 252, 353, 247, 40, 341, 176, 320, 164, 86, 239, 5, 325, 105, 378, 377, 47, 54, 241, 122, 254, 193, 22, 117, 232, 133, 208, 372, 336, 55, 147, 145, 261, 77, 376, 334, 75, 195, 351, 292, 144, 196, 215, 56, 80, 74, 326, 270, 309, 357, 307, 73, 78, 132, 223, 226, 60, 138, 26, 156, 112, 350, 190, 217, 236, 240, 228, 140, 346, 358, 278, 294, 302, 342, 188, 373, 359, 118, 38, 354, 374, 139, 203, 191, 155, 266, 158, 24, 25, 250, 204, 202, 116, 347, 115, 327, 235, 370, 227, 6, 70, 301, 134, 279, 44, 368, 271, 206, 163, 348, 375, 251, 380, 360, 187, 322, 257, 171, 52, 314, 214, 1, 365, 15, 335, 35, 243, 58, 107, 177, 369, 295, 175, 32, 108, 59, 379, 96, 317, 88, 340, 48, 76, 152, 4, 218, 192, 267, 363, 65, 99, 8, 331, 213, 0, 344, 46, 170, 289, 371, 246, 79, 53, 19, 119, 287, 101, 149, 82, 198, 103, 37, 30, 93, 209, 120, 125, 248, 162, 364, 21, 36, 288, 148, 167, 51, 34, 81, 308, 299, 29, 28, 355, 339, 311, 211, 273, 291, 200, 179, 349, 290, 383, 124, 17, 284, 91, 210, 265, 306, 286, 332, 277, 23, 84, 87, 264, 61, 367, 219, 253, 126, 104, 338, 150, 324, 183, 92, 262, 57, 142, 131, 9, 283, 69, 98, 146, 381, 130, 159, 7, 182, 111, 303, 382, 199, 293, 3, 333, 216, 366, 172, 259, 102, 274, 27, 224, 42, 135, 263, 184, 128, 85, 296, 31, 237, 281, 225, 268, 319, 137, 129, 275, 13, 166, 298, 181, 110, 220, 18, 229, 310, 305, 20, 11, 123, 180, 72, 234, 165, 238, 328, 304, 49, 2, 114, 12, 221, 312, 337, 189, 14, 352, 329, 361, 67, 178, 185, 10, 255, 94, 33, 153, 212, 258, 143, 97, 201, 297, 161, 50, 64, 242, 141, 282, 244, 83, 230, 45, 186, 285] + 56: [231, 340, 100, 199, 98, 371, 20, 333, 328, 316, 275, 66, 159, 150, 383, 0, 58, 40, 253, 135, 295, 166, 355, 110, 255, 270, 286, 115, 122, 134, 145, 347, 323, 274, 37, 16, 70, 93, 254, 272, 239, 261, 36, 25, 247, 142, 146, 3, 240, 225, 304, 167, 186, 158, 156, 89, 306, 46, 99, 18, 327, 245, 101, 223, 233, 376, 148, 10, 21, 79, 268, 280, 133, 164, 227, 368, 95, 94, 54, 341, 290, 200, 276, 372, 144, 130, 47, 337, 176, 349, 366, 111, 171, 301, 173, 90, 195, 15, 75, 343, 116, 184, 169, 236, 125, 161, 48, 2, 19, 288, 244, 97, 344, 334, 342, 278, 92, 252, 230, 121, 34, 153, 209, 297, 190, 348, 180, 314, 132, 293, 319, 65, 29, 86, 113, 131, 345, 203, 370, 377, 114, 237, 74, 85, 260, 213, 5, 311, 107, 296, 87, 196, 187, 6, 39, 219, 332, 14, 283, 309, 197, 73, 38, 137, 181, 31, 374, 331, 50, 33, 123, 282, 339, 299, 183, 262, 228, 28, 330, 289, 67, 352, 287, 157, 382, 212, 52, 7, 277, 91, 102, 143, 220, 82, 30, 8, 273, 88, 351, 356, 264, 162, 71, 292, 329, 381, 168, 120, 205, 27, 285, 250, 206, 198, 179, 12, 1, 17, 127, 56, 221, 201, 257, 84, 353, 124, 379, 367, 78, 43, 373, 258, 26, 325, 335, 216, 174, 165, 229, 208, 294, 178, 259, 177, 81, 163, 322, 249, 104, 117, 140, 302, 362, 154, 192, 72, 246, 300, 160, 193, 324, 338, 23, 22, 11, 13, 243, 364, 313, 360, 298, 119, 172, 307, 175, 57, 64, 350, 60, 152, 108, 358, 336, 217, 194, 235, 76, 224, 321, 44, 106, 279, 126, 53, 303, 9, 263, 308, 346, 41, 83, 77, 357, 118, 238, 103, 182, 312, 105, 305, 354, 317, 214, 151, 141, 129, 42, 62, 210, 218, 310, 4, 226, 139, 266, 361, 222, 204, 234, 215, 326, 45, 365, 318, 109, 315, 242, 265, 241, 147, 191, 155, 267, 189, 363, 251, 320, 284, 188, 80, 63, 207, 369, 359, 69, 281, 128, 136, 35, 269, 149, 138, 185, 248, 378, 271, 55, 96, 375, 112, 232, 291, 49, 170, 24, 211, 256, 68, 59, 32, 380, 51, 61, 202] + 57: [254, 125, 159, 121, 198, 257, 324, 267, 6, 283, 160, 163, 238, 91, 172, 213, 214, 113, 312, 111, 38, 55, 328, 226, 352, 209, 28, 132, 29, 237, 49, 211, 200, 168, 219, 256, 239, 78, 308, 258, 82, 221, 77, 153, 202, 262, 301, 304, 276, 148, 170, 342, 314, 7, 128, 53, 118, 150, 197, 88, 62, 272, 167, 373, 40, 51, 249, 356, 102, 203, 106, 95, 210, 196, 282, 155, 186, 162, 261, 315, 236, 255, 357, 333, 361, 370, 232, 10, 264, 206, 278, 227, 280, 367, 275, 343, 326, 205, 72, 93, 158, 63, 65, 124, 268, 350, 318, 79, 123, 182, 99, 253, 322, 228, 70, 83, 21, 183, 284, 13, 279, 171, 337, 90, 330, 18, 371, 97, 33, 59, 0, 32, 109, 194, 48, 364, 271, 266, 52, 327, 181, 15, 229, 372, 25, 187, 320, 11, 16, 368, 235, 137, 344, 169, 117, 64, 44, 354, 58, 39, 346, 73, 27, 298, 184, 147, 12, 380, 2, 164, 199, 362, 178, 215, 240, 145, 252, 24, 270, 126, 66, 36, 374, 139, 61, 115, 291, 292, 305, 34, 105, 376, 231, 246, 286, 157, 133, 156, 101, 274, 8, 204, 151, 22, 35, 251, 347, 161, 179, 297, 42, 46, 349, 92, 358, 112, 329, 313, 190, 321, 98, 334, 290, 247, 5, 265, 307, 281, 263, 50, 192, 76, 273, 218, 114, 340, 69, 245, 285, 56, 14, 89, 94, 154, 381, 57, 104, 359, 335, 300, 127, 317, 306, 80, 338, 316, 110, 9, 177, 277, 191, 54, 47, 119, 348, 71, 351, 369, 288, 17, 379, 363, 45, 303, 378, 87, 224, 242, 67, 103, 142, 244, 129, 189, 336, 201, 233, 293, 146, 165, 207, 302, 31, 220, 140, 223, 130, 60, 174, 345, 185, 75, 319, 43, 176, 341, 152, 20, 382, 116, 120, 225, 68, 136, 3, 296, 81, 311, 41, 332, 122, 135, 299, 241, 1, 222, 138, 107, 310, 100, 212, 193, 30, 37, 19, 86, 259, 295, 331, 74, 230, 85, 366, 108, 141, 217, 84, 180, 309, 250, 144, 365, 134, 243, 96, 173, 377, 208, 289, 325, 166, 149, 353, 23, 216, 323, 175, 26, 355, 248, 4, 188, 383, 260, 375, 287, 360, 339, 294, 269, 195, 131, 143, 234] + 58: [377, 169, 232, 324, 367, 132, 4, 190, 198, 229, 55, 239, 129, 236, 160, 183, 149, 362, 379, 334, 347, 113, 195, 8, 64, 243, 372, 330, 354, 374, 223, 109, 34, 15, 301, 91, 361, 279, 321, 168, 201, 340, 33, 38, 58, 134, 209, 308, 220, 114, 61, 27, 373, 9, 351, 256, 112, 376, 85, 42, 130, 162, 66, 185, 48, 202, 383, 284, 350, 326, 37, 0, 144, 148, 123, 345, 19, 35, 237, 170, 294, 87, 317, 41, 254, 247, 117, 145, 17, 30, 140, 89, 147, 292, 184, 204, 81, 194, 298, 76, 348, 101, 49, 67, 199, 368, 295, 342, 119, 104, 111, 307, 328, 380, 314, 207, 181, 238, 227, 346, 135, 172, 176, 288, 2, 287, 50, 133, 118, 5, 138, 221, 303, 103, 44, 349, 137, 52, 127, 218, 233, 158, 309, 280, 245, 98, 311, 263, 7, 344, 93, 69, 355, 116, 68, 6, 203, 315, 316, 20, 28, 281, 375, 82, 99, 215, 155, 382, 332, 13, 253, 161, 371, 293, 179, 136, 226, 24, 142, 359, 157, 336, 269, 120, 153, 78, 188, 323, 23, 320, 335, 364, 156, 92, 122, 248, 356, 210, 322, 22, 214, 178, 262, 47, 31, 12, 255, 197, 171, 1, 331, 143, 258, 306, 234, 378, 63, 302, 154, 107, 228, 265, 283, 94, 230, 296, 100, 54, 163, 146, 46, 369, 216, 88, 267, 343, 165, 80, 3, 370, 275, 205, 312, 249, 273, 211, 235, 72, 257, 36, 196, 141, 191, 97, 60, 242, 224, 192, 139, 59, 62, 53, 250, 240, 131, 208, 357, 278, 21, 11, 182, 105, 180, 102, 360, 86, 268, 200, 51, 96, 159, 193, 365, 363, 212, 164, 71, 10, 126, 305, 222, 77, 79, 271, 206, 358, 26, 40, 152, 341, 189, 277, 260, 352, 73, 187, 381, 244, 325, 83, 339, 75, 121, 300, 231, 65, 318, 115, 353, 219, 297, 213, 29, 128, 276, 290, 84, 39, 246, 286, 74, 217, 291, 166, 125, 108, 56, 366, 177, 151, 259, 14, 299, 289, 270, 175, 45, 18, 16, 272, 106, 274, 285, 329, 251, 25, 266, 90, 124, 333, 110, 241, 186, 327, 43, 70, 225, 173, 338, 261, 95, 264, 282, 337, 167, 57, 32, 304, 174, 310, 313, 252, 319, 150] + 59: [327, 220, 113, 105, 226, 374, 291, 172, 328, 151, 281, 290, 45, 40, 16, 181, 162, 315, 29, 47, 355, 348, 366, 6, 353, 199, 107, 333, 246, 34, 84, 330, 359, 44, 183, 53, 166, 303, 8, 141, 350, 375, 347, 345, 167, 4, 90, 189, 25, 260, 346, 182, 97, 251, 371, 150, 213, 295, 144, 196, 86, 176, 160, 279, 297, 248, 252, 245, 10, 236, 344, 238, 13, 228, 156, 48, 71, 27, 208, 134, 157, 93, 49, 24, 118, 296, 82, 165, 240, 14, 293, 87, 78, 72, 235, 154, 170, 202, 66, 33, 313, 70, 263, 130, 185, 324, 80, 255, 58, 212, 81, 35, 302, 163, 376, 338, 214, 158, 137, 92, 337, 123, 314, 32, 287, 41, 135, 184, 17, 326, 336, 275, 273, 307, 368, 319, 1, 56, 77, 171, 83, 99, 308, 215, 229, 195, 340, 73, 285, 5, 298, 288, 300, 186, 262, 382, 342, 109, 223, 254, 280, 334, 22, 60, 286, 258, 146, 369, 365, 2, 61, 194, 138, 43, 50, 124, 28, 74, 88, 249, 148, 102, 241, 237, 318, 360, 272, 244, 104, 122, 310, 271, 7, 94, 173, 332, 30, 147, 247, 331, 31, 372, 26, 266, 133, 175, 161, 12, 100, 169, 143, 19, 306, 69, 120, 354, 203, 51, 68, 299, 193, 63, 187, 217, 21, 89, 261, 115, 201, 230, 312, 108, 239, 192, 335, 378, 126, 323, 111, 268, 140, 370, 59, 234, 46, 95, 381, 294, 18, 9, 23, 243, 112, 91, 57, 127, 267, 383, 20, 218, 356, 179, 180, 177, 36, 101, 242, 0, 206, 64, 136, 253, 207, 153, 132, 210, 357, 103, 358, 98, 316, 305, 39, 264, 55, 190, 15, 341, 209, 379, 200, 65, 75, 233, 311, 76, 42, 38, 178, 309, 114, 278, 85, 159, 364, 320, 373, 142, 329, 339, 198, 361, 155, 131, 274, 362, 380, 110, 325, 231, 67, 145, 277, 191, 321, 284, 216, 257, 276, 219, 121, 367, 139, 116, 117, 79, 224, 250, 363, 352, 256, 197, 204, 221, 265, 377, 317, 232, 128, 164, 304, 168, 211, 62, 225, 174, 54, 222, 269, 259, 289, 349, 96, 292, 227, 149, 129, 322, 343, 37, 3, 106, 11, 119, 283, 301, 282, 125, 270, 205, 152, 52, 351, 188] + 60: [313, 94, 195, 372, 78, 65, 208, 239, 49, 166, 98, 272, 59, 27, 169, 151, 20, 6, 240, 332, 110, 215, 116, 186, 90, 85, 146, 182, 95, 219, 225, 338, 282, 361, 328, 258, 145, 356, 291, 72, 43, 289, 340, 10, 218, 214, 288, 201, 124, 174, 335, 129, 339, 230, 185, 74, 294, 180, 329, 278, 231, 238, 99, 336, 255, 269, 183, 11, 237, 274, 25, 134, 142, 293, 196, 300, 322, 256, 64, 197, 168, 13, 209, 179, 42, 204, 96, 86, 135, 367, 32, 44, 242, 154, 375, 355, 351, 83, 310, 170, 244, 212, 312, 81, 34, 319, 119, 15, 131, 161, 241, 253, 0, 277, 37, 1, 188, 264, 377, 304, 21, 107, 66, 7, 192, 370, 286, 18, 325, 270, 45, 228, 216, 50, 68, 97, 159, 254, 163, 4, 101, 308, 317, 22, 123, 60, 265, 128, 303, 316, 247, 297, 205, 76, 365, 345, 245, 143, 263, 359, 84, 283, 132, 318, 80, 267, 57, 342, 233, 70, 187, 30, 369, 314, 380, 148, 363, 376, 127, 287, 261, 224, 160, 147, 165, 281, 164, 103, 118, 89, 62, 299, 257, 139, 40, 121, 243, 113, 194, 334, 2, 108, 353, 193, 213, 117, 235, 246, 75, 58, 221, 36, 344, 9, 3, 279, 26, 346, 31, 200, 100, 357, 276, 12, 315, 307, 262, 330, 19, 198, 374, 378, 301, 306, 292, 177, 73, 77, 210, 207, 16, 47, 199, 236, 266, 39, 167, 302, 28, 35, 290, 311, 206, 184, 271, 364, 152, 69, 114, 51, 92, 162, 38, 8, 323, 52, 273, 33, 190, 106, 296, 14, 125, 105, 202, 252, 320, 280, 379, 29, 175, 326, 173, 138, 352, 87, 373, 348, 268, 220, 126, 284, 260, 343, 48, 153, 181, 327, 234, 112, 354, 232, 203, 55, 172, 189, 248, 93, 217, 88, 382, 176, 56, 141, 360, 91, 149, 341, 349, 144, 331, 383, 67, 23, 178, 137, 41, 211, 111, 191, 350, 71, 275, 321, 136, 157, 381, 158, 133, 102, 115, 347, 222, 120, 155, 366, 337, 46, 309, 122, 223, 82, 140, 150, 362, 358, 259, 24, 5, 61, 305, 171, 53, 285, 250, 130, 79, 368, 371, 333, 156, 251, 324, 54, 104, 229, 298, 249, 17, 227, 295, 109, 226, 63] + 61: [221, 248, 107, 360, 247, 241, 363, 330, 278, 235, 129, 115, 159, 32, 43, 348, 233, 44, 190, 47, 336, 207, 202, 139, 292, 8, 365, 225, 67, 71, 342, 193, 311, 163, 364, 312, 13, 314, 223, 236, 56, 281, 134, 41, 16, 356, 94, 58, 127, 167, 88, 120, 326, 270, 45, 12, 51, 237, 112, 110, 15, 169, 337, 224, 77, 83, 82, 109, 206, 145, 33, 72, 78, 275, 125, 269, 295, 124, 204, 298, 91, 239, 101, 74, 201, 272, 138, 250, 60, 226, 210, 50, 40, 128, 319, 0, 179, 100, 161, 230, 170, 323, 59, 253, 372, 142, 119, 333, 42, 156, 75, 258, 297, 29, 27, 379, 308, 368, 200, 259, 38, 273, 37, 126, 240, 317, 332, 152, 274, 25, 255, 358, 24, 293, 28, 339, 220, 285, 131, 267, 325, 296, 343, 208, 338, 36, 307, 172, 157, 178, 209, 103, 17, 215, 66, 57, 188, 243, 133, 361, 65, 84, 166, 11, 257, 23, 147, 173, 383, 214, 9, 79, 196, 132, 194, 106, 244, 271, 6, 99, 262, 305, 212, 105, 347, 377, 175, 183, 320, 229, 189, 53, 61, 381, 324, 86, 335, 205, 10, 184, 353, 238, 55, 327, 34, 349, 309, 228, 20, 316, 300, 113, 2, 260, 328, 290, 154, 137, 280, 242, 283, 340, 199, 359, 70, 130, 354, 123, 251, 185, 373, 265, 35, 222, 279, 289, 367, 334, 350, 155, 73, 197, 344, 231, 5, 351, 19, 1, 195, 371, 143, 288, 168, 286, 227, 261, 191, 268, 366, 313, 304, 3, 52, 102, 186, 153, 306, 341, 85, 141, 176, 14, 162, 352, 187, 18, 7, 111, 164, 48, 256, 64, 277, 382, 76, 217, 96, 321, 93, 46, 263, 216, 108, 165, 203, 264, 218, 294, 213, 192, 369, 31, 282, 87, 362, 90, 114, 252, 329, 69, 104, 249, 322, 89, 375, 22, 219, 81, 144, 284, 370, 345, 54, 301, 49, 331, 180, 146, 140, 276, 151, 355, 376, 266, 171, 287, 211, 299, 315, 303, 136, 122, 117, 380, 21, 68, 198, 291, 177, 181, 160, 92, 118, 150, 135, 121, 374, 245, 97, 158, 378, 346, 30, 174, 80, 26, 4, 98, 148, 63, 310, 234, 232, 246, 95, 182, 318, 302, 254, 39, 62, 357, 116, 149] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots416.yaml b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots416.yaml new file mode 100644 index 000000000000..434b41525a94 --- /dev/null +++ b/tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots416.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [259, 356, 356, 225, 177, 111, 230, 378, 51, 60, 204, 218, 232, 376, 203, 293, 224, 144, 141, 181, 107, 43, 57, 361, 28, 248, 309, 262, 5, 91, 317, 137, 363, 192, 296, 270, 299, 326, 50, 252, 352, 316, 175, 2, 301, 125, 132, 304, 364, 273, 13, 373, 166, 302, 236, 86, 348, 135, 193, 45, 169, 21, 79, 157, 184, 255, 217, 318, 330, 209, 22, 72, 101, 233, 109, 129, 324, 208, 123, 314, 39, 133, 4, 108, 116, 216, 100, 379, 382, 263, 20, 210, 42, 212, 315, 334, 367, 251, 298, 329, 339, 8, 171, 274, 166, 351, 322, 371, 165, 111, 207, 45, 188, 81, 79, 17, 327, 48, 217, 198, 330, 282, 55, 307, 11, 313, 297, 311, 114, 44, 268, 234, 52, 308, 103, 146, 140, 19, 180, 320, 127, 14, 145, 350, 15, 64, 53, 69, 319, 164, 18, 226, 291, 88, 370, 185, 0, 222, 236, 225, 162, 135, 279, 190, 182, 122, 62, 29, 134, 70, 139, 276, 290, 349, 336, 33, 117, 156, 10, 36, 261, 148, 31, 358, 85, 187, 136, 240, 178, 99, 211, 195, 82, 328, 47, 102, 338, 340, 380, 179, 381, 250, 167, 374, 325, 323, 58, 201, 256, 71, 159, 245, 254, 264, 158, 378, 163, 153, 202, 17, 49, 289, 176, 16, 173, 3, 78, 284, 34, 283, 186, 200, 112, 300, 27, 89, 66, 32, 214, 249, 246, 275, 238, 292, 83, 93, 260, 229, 161, 253, 353, 152, 194, 357, 142, 24, 228, 130, 277, 61, 256, 222, 345, 371, 165, 254, 76, 346, 35, 26, 104, 95, 49, 321, 68, 333, 173, 272, 359, 54, 131, 56, 354, 265, 368, 7, 77, 375, 377, 172, 227, 331, 269, 366, 1, 25, 73, 67, 342, 98, 294, 206, 12, 219, 9, 285, 271, 121, 281, 168, 118, 247, 150, 74, 119, 383, 155, 111, 279, 332, 312, 347, 242, 80, 134, 65, 63, 37, 290, 126, 360, 243, 105, 341, 90, 258, 138, 170, 149, 38, 87, 244, 335, 365, 372, 369, 97, 106, 147, 237, 362, 286, 305, 115, 306, 191, 231, 220, 23, 189, 355, 288, 310, 303, 151, 74, 84, 245, 165, 235, 241, 378, 295, 60, 30, 95, 184, 343, 203, 278, 224, 280, 143, 205, 196, 183, 124, 174, 94, 337, 160, 199, 223, 257, 213, 46, 75, 59, 215, 221, 197, 267, 6, 154, 40, 96, 41, 113, 239, 266, 92, 287, 128, 344, 120, 110] + 1: [134, 174, 118, 158, 150, 152, 169, 163, 21, 192, 325, 377, 63, 299, 111, 153, 224, 273, 287, 86, 216, 99, 108, 280, 4, 47, 358, 171, 101, 102, 313, 222, 161, 332, 295, 321, 87, 315, 13, 302, 262, 356, 279, 371, 183, 253, 208, 64, 258, 304, 312, 215, 178, 201, 137, 106, 382, 91, 167, 127, 319, 6, 5, 73, 334, 56, 120, 351, 272, 243, 37, 89, 125, 212, 162, 335, 213, 206, 88, 38, 133, 256, 79, 348, 249, 14, 296, 57, 172, 350, 94, 323, 353, 27, 190, 129, 316, 135, 10, 191, 159, 359, 310, 34, 380, 201, 137, 80, 277, 91, 167, 254, 131, 67, 270, 328, 58, 244, 225, 82, 383, 292, 275, 71, 234, 146, 288, 109, 219, 228, 289, 278, 236, 76, 121, 259, 370, 107, 115, 62, 290, 26, 104, 113, 54, 119, 369, 331, 193, 209, 124, 155, 307, 354, 112, 368, 141, 42, 308, 44, 126, 263, 177, 127, 347, 67, 70, 123, 105, 120, 250, 271, 320, 138, 303, 247, 317, 355, 197, 92, 223, 116, 69, 360, 95, 298, 122, 330, 65, 286, 184, 154, 35, 230, 311, 268, 226, 333, 375, 372, 367, 221, 128, 322, 117, 257, 199, 255, 251, 42, 22, 180, 149, 50, 169, 163, 40, 39, 179, 41, 3, 198, 23, 185, 46, 59, 9, 32, 266, 285, 187, 261, 143, 282, 306, 376, 60, 98, 7, 305, 2, 20, 175, 136, 324, 11, 97, 8, 232, 314, 130, 246, 188, 373, 361, 45, 276, 339, 345, 15, 81, 235, 31, 158, 85, 72, 177, 329, 347, 30, 366, 110, 182, 294, 111, 233, 272, 29, 269, 242, 281, 74, 218, 379, 52, 53, 264, 202, 231, 189, 164, 238, 338, 326, 19, 33, 186, 51, 352, 346, 344, 77, 357, 220, 340, 217, 132, 16, 144, 28, 336, 265, 200, 1, 174, 180, 382, 43, 196, 378, 319, 241, 0, 210, 3, 198, 225, 24, 283, 292, 211, 260, 245, 301, 84, 194, 214, 176, 203, 239, 61, 205, 362, 204, 364, 17, 156, 68, 36, 284, 96, 181, 240, 103, 83, 297, 160, 207, 252, 343, 349, 341, 363, 327, 365, 365, 22, 18, 277, 72, 291, 163, 319, 168, 75, 377, 182, 300, 248, 82, 25, 195, 145, 173, 78, 274, 166, 142, 114, 93, 381, 342, 147, 237, 140, 170, 139, 318, 229, 165, 48, 293, 227, 55, 12, 49, 148, 309, 90, 151, 66, 374, 337, 157, 267, 100] + 2: [53, 341, 215, 93, 223, 295, 125, 197, 232, 19, 109, 322, 131, 280, 86, 220, 31, 296, 200, 82, 285, 61, 206, 224, 27, 262, 120, 218, 41, 65, 309, 69, 256, 12, 257, 0, 25, 281, 141, 352, 329, 99, 335, 158, 378, 283, 249, 315, 358, 282, 255, 366, 53, 351, 313, 343, 259, 295, 143, 248, 149, 213, 62, 350, 91, 377, 44, 105, 304, 26, 83, 24, 311, 79, 299, 263, 50, 203, 201, 76, 332, 241, 169, 250, 106, 39, 268, 2, 36, 334, 264, 359, 216, 370, 338, 307, 382, 177, 369, 64, 228, 354, 300, 381, 43, 108, 89, 93, 298, 221, 143, 248, 242, 101, 155, 284, 66, 56, 245, 172, 314, 368, 279, 340, 196, 174, 170, 347, 55, 188, 58, 128, 349, 308, 186, 254, 136, 49, 318, 23, 195, 346, 331, 57, 243, 371, 114, 267, 336, 222, 13, 380, 32, 339, 199, 330, 43, 225, 89, 194, 81, 137, 212, 212, 242, 290, 84, 97, 355, 70, 118, 184, 51, 122, 328, 244, 323, 204, 130, 356, 126, 327, 187, 198, 163, 16, 15, 160, 154, 142, 104, 94, 275, 35, 52, 144, 226, 95, 183, 48, 113, 345, 214, 266, 316, 208, 375, 258, 180, 108, 14, 115, 28, 3, 151, 33, 232, 185, 153, 191, 91, 377, 86, 304, 314, 22, 274, 85, 6, 342, 7, 90, 269, 210, 361, 297, 360, 145, 348, 276, 364, 121, 29, 362, 134, 4, 272, 231, 292, 373, 167, 59, 72, 234, 260, 127, 119, 270, 190, 379, 230, 112, 215, 115, 223, 3, 100, 75, 192, 78, 62, 168, 166, 278, 271, 133, 139, 324, 146, 11, 30, 294, 140, 18, 156, 320, 20, 236, 277, 10, 175, 116, 9, 305, 150, 152, 211, 367, 207, 233, 291, 107, 205, 74, 182, 147, 317, 80, 326, 237, 246, 229, 96, 112, 313, 194, 5, 321, 143, 248, 162, 290, 111, 168, 173, 286, 135, 42, 45, 325, 217, 235, 303, 165, 344, 273, 68, 67, 21, 138, 46, 171, 193, 129, 372, 252, 365, 219, 239, 47, 178, 261, 302, 288, 8, 164, 310, 37, 353, 88, 287, 337, 40, 374, 132, 132, 357, 102, 1, 189, 247, 34, 60, 19, 110, 124, 157, 73, 306, 123, 139, 122, 63, 333, 159, 209, 17, 87, 293, 92, 383, 148, 253, 265, 181, 103, 38, 312, 176, 179, 77, 238, 376, 319, 54, 71, 251, 98, 117, 227, 240, 289, 301, 202, 363, 161] + 3: [39, 34, 310, 230, 314, 319, 350, 163, 313, 62, 334, 213, 26, 168, 358, 189, 304, 316, 64, 202, 364, 289, 277, 92, 155, 103, 217, 327, 135, 307, 82, 276, 216, 298, 291, 15, 165, 68, 153, 93, 190, 323, 107, 191, 141, 148, 23, 320, 193, 247, 324, 301, 39, 34, 275, 377, 302, 319, 31, 192, 97, 80, 145, 6, 186, 207, 2, 325, 221, 174, 290, 264, 8, 161, 288, 243, 345, 299, 380, 197, 105, 113, 76, 160, 306, 249, 70, 376, 130, 346, 101, 269, 379, 251, 121, 242, 54, 368, 315, 60, 33, 5, 278, 47, 22, 34, 34, 377, 302, 319, 51, 3, 194, 151, 318, 137, 215, 63, 342, 122, 77, 219, 25, 29, 279, 359, 262, 114, 176, 218, 256, 112, 65, 266, 125, 328, 286, 184, 67, 91, 373, 228, 321, 258, 354, 73, 88, 45, 143, 330, 175, 349, 226, 332, 198, 303, 22, 34, 79, 260, 365, 14, 27, 163, 263, 263, 81, 331, 139, 178, 164, 267, 296, 195, 21, 87, 109, 142, 32, 312, 375, 204, 236, 270, 117, 280, 214, 366, 322, 20, 188, 106, 85, 132, 293, 133, 0, 16, 140, 159, 205, 75, 272, 347, 341, 335, 199, 59, 66, 66, 310, 35, 365, 18, 317, 131, 194, 24, 363, 252, 201, 170, 250, 211, 212, 182, 11, 12, 271, 154, 229, 351, 41, 126, 55, 53, 355, 44, 104, 100, 340, 361, 305, 83, 381, 282, 362, 285, 179, 116, 156, 360, 241, 89, 352, 152, 173, 52, 257, 370, 66, 66, 310, 377, 209, 40, 19, 13, 313, 157, 158, 180, 339, 343, 227, 10, 287, 36, 119, 118, 136, 99, 149, 111, 300, 224, 235, 94, 169, 255, 383, 237, 248, 353, 232, 294, 61, 259, 309, 206, 183, 134, 234, 369, 253, 127, 124, 4, 326, 9, 233, 311, 66, 71, 79, 260, 40, 171, 225, 56, 261, 333, 49, 42, 337, 102, 336, 231, 74, 239, 240, 357, 162, 84, 295, 297, 181, 108, 200, 57, 273, 48, 30, 144, 115, 187, 281, 7, 123, 110, 86, 283, 378, 372, 338, 329, 147, 95, 308, 244, 185, 17, 146, 38, 66, 46, 34, 260, 365, 18, 27, 208, 3, 348, 150, 50, 238, 120, 58, 37, 138, 177, 223, 382, 78, 172, 203, 374, 98, 274, 166, 344, 129, 246, 268, 196, 43, 96, 128, 367, 356, 72, 28, 254, 245, 220, 210, 284, 90, 1, 265, 69, 167, 371, 292, 222] + 4: [188, 124, 258, 331, 315, 214, 15, 376, 319, 117, 254, 136, 36, 167, 246, 77, 196, 44, 49, 198, 317, 187, 358, 42, 166, 189, 352, 341, 64, 232, 354, 367, 178, 294, 346, 175, 170, 181, 211, 156, 126, 203, 353, 1, 238, 11, 373, 270, 2, 17, 90, 277, 34, 375, 296, 331, 315, 214, 23, 19, 87, 117, 148, 6, 13, 59, 347, 372, 32, 314, 27, 251, 22, 207, 183, 221, 85, 360, 185, 169, 125, 70, 69, 94, 309, 52, 263, 103, 295, 304, 109, 40, 215, 129, 112, 247, 366, 245, 326, 299, 192, 231, 58, 104, 340, 12, 199, 331, 259, 371, 172, 88, 105, 194, 142, 308, 47, 329, 76, 318, 365, 160, 333, 68, 35, 179, 324, 99, 290, 356, 379, 323, 132, 131, 135, 288, 220, 306, 78, 121, 249, 334, 268, 213, 186, 21, 48, 162, 265, 224, 190, 266, 20, 79, 287, 282, 3, 271, 300, 343, 315, 214, 15, 226, 87, 273, 142, 143, 345, 274, 145, 106, 357, 43, 339, 280, 7, 153, 278, 243, 252, 330, 359, 75, 269, 383, 100, 284, 138, 267, 234, 26, 122, 289, 223, 130, 10, 115, 123, 55, 91, 102, 301, 195, 337, 272, 338, 351, 3, 271, 343, 343, 24, 193, 9, 95, 257, 273, 148, 165, 56, 377, 139, 61, 364, 101, 28, 73, 110, 141, 54, 276, 65, 114, 98, 14, 261, 262, 80, 158, 39, 92, 230, 157, 146, 140, 147, 127, 255, 97, 374, 191, 164, 248, 285, 382, 208, 233, 107, 182, 3, 37, 222, 331, 250, 371, 16, 281, 5, 171, 152, 348, 86, 155, 41, 200, 361, 83, 204, 217, 260, 177, 335, 0, 307, 256, 297, 60, 46, 229, 322, 328, 370, 50, 176, 51, 180, 210, 71, 286, 81, 378, 163, 279, 30, 202, 236, 174, 225, 242, 63, 120, 18, 271, 296, 331, 350, 320, 228, 226, 45, 194, 254, 241, 342, 197, 325, 275, 151, 57, 312, 344, 369, 159, 244, 116, 133, 72, 62, 149, 108, 216, 25, 137, 84, 66, 184, 29, 291, 237, 111, 355, 381, 205, 316, 380, 134, 144, 89, 154, 227, 67, 321, 332, 209, 209, 343, 343, 219, 23, 239, 95, 319, 150, 152, 302, 253, 298, 113, 4, 235, 218, 8, 161, 82, 38, 283, 264, 53, 168, 93, 33, 74, 313, 240, 310, 292, 349, 206, 303, 327, 118, 31, 201, 96, 311, 362, 173, 368, 293, 212, 363, 119, 128, 336, 305] + 5: [172, 128, 296, 214, 217, 346, 265, 269, 269, 52, 231, 102, 279, 252, 247, 22, 124, 8, 375, 2, 108, 318, 258, 153, 228, 46, 216, 39, 20, 201, 138, 76, 275, 13, 361, 376, 363, 378, 199, 298, 189, 347, 126, 85, 311, 234, 0, 27, 348, 57, 23, 368, 208, 249, 296, 194, 81, 178, 100, 193, 287, 272, 211, 259, 215, 43, 192, 335, 173, 155, 15, 238, 181, 204, 175, 377, 35, 195, 242, 183, 230, 240, 63, 317, 334, 48, 94, 229, 374, 299, 135, 40, 224, 245, 111, 267, 328, 294, 117, 367, 225, 179, 321, 75, 61, 213, 84, 214, 1, 60, 276, 262, 287, 272, 122, 21, 351, 17, 331, 182, 327, 131, 268, 59, 67, 3, 210, 220, 33, 186, 380, 253, 353, 250, 257, 284, 237, 9, 344, 62, 235, 263, 107, 187, 322, 292, 11, 197, 146, 357, 310, 339, 164, 82, 337, 302, 61, 213, 84, 5, 217, 346, 276, 370, 365, 288, 159, 379, 132, 212, 147, 198, 371, 137, 37, 174, 184, 251, 320, 218, 54, 144, 51, 345, 106, 239, 161, 289, 254, 87, 77, 191, 359, 103, 68, 168, 373, 86, 53, 309, 336, 123, 109, 112, 176, 157, 286, 354, 61, 249, 256, 160, 47, 355, 100, 188, 356, 6, 259, 78, 130, 7, 293, 25, 31, 66, 163, 358, 38, 143, 26, 333, 295, 28, 12, 93, 343, 300, 241, 203, 330, 80, 134, 349, 113, 323, 244, 156, 297, 101, 185, 73, 282, 202, 226, 248, 278, 277, 196, 303, 165, 213, 306, 194, 65, 355, 105, 305, 356, 6, 231, 64, 145, 352, 91, 364, 304, 70, 151, 332, 319, 24, 34, 152, 104, 99, 274, 227, 14, 382, 219, 90, 383, 236, 291, 55, 205, 114, 18, 92, 261, 290, 120, 285, 350, 42, 180, 110, 83, 326, 118, 127, 154, 141, 128, 214, 98, 49, 223, 169, 30, 52, 372, 21, 72, 88, 89, 338, 233, 313, 171, 264, 69, 140, 314, 45, 16, 79, 266, 329, 366, 121, 341, 136, 281, 360, 29, 232, 221, 71, 342, 316, 125, 95, 139, 97, 243, 340, 10, 116, 142, 96, 50, 44, 190, 190, 128, 308, 260, 355, 105, 305, 381, 288, 159, 41, 255, 200, 246, 74, 209, 4, 222, 369, 206, 207, 324, 315, 301, 115, 312, 19, 56, 362, 150, 325, 119, 149, 162, 271, 283, 129, 270, 58, 148, 32, 133, 307, 36, 158, 273, 280, 166, 167, 170, 177] + 6: [322, 97, 274, 308, 53, 343, 240, 99, 233, 205, 36, 359, 187, 57, 196, 289, 151, 95, 179, 245, 317, 202, 321, 327, 324, 154, 22, 206, 197, 4, 355, 329, 346, 3, 96, 70, 366, 357, 106, 360, 294, 348, 201, 263, 92, 141, 170, 218, 149, 25, 35, 164, 322, 212, 274, 5, 32, 267, 341, 240, 209, 120, 350, 181, 119, 56, 34, 129, 156, 176, 377, 278, 304, 37, 282, 219, 15, 280, 6, 242, 284, 243, 138, 148, 199, 147, 349, 272, 310, 244, 288, 251, 117, 51, 89, 339, 192, 1, 172, 2, 65, 33, 71, 40, 322, 142, 142, 376, 160, 258, 269, 46, 180, 31, 173, 163, 293, 112, 319, 178, 300, 165, 369, 336, 101, 12, 133, 211, 143, 58, 210, 207, 335, 82, 372, 248, 83, 124, 16, 268, 123, 39, 62, 328, 256, 21, 121, 374, 213, 93, 273, 61, 153, 52, 275, 342, 146, 97, 274, 308, 32, 174, 28, 11, 66, 67, 175, 216, 100, 189, 287, 239, 370, 113, 352, 236, 223, 279, 139, 114, 10, 286, 69, 43, 225, 217, 362, 122, 18, 358, 215, 299, 364, 130, 152, 290, 24, 195, 313, 238, 140, 227, 191, 20, 314, 204, 320, 247, 76, 305, 295, 308, 53, 27, 55, 229, 66, 371, 9, 318, 367, 246, 255, 291, 234, 301, 109, 41, 311, 111, 383, 144, 98, 220, 271, 237, 116, 330, 261, 81, 168, 224, 8, 125, 379, 74, 150, 85, 266, 47, 183, 38, 7, 91, 200, 228, 252, 186, 315, 373, 76, 97, 142, 376, 32, 249, 253, 281, 166, 177, 45, 13, 232, 283, 118, 155, 345, 270, 128, 230, 68, 30, 44, 64, 136, 296, 87, 104, 63, 23, 14, 126, 48, 50, 334, 309, 73, 312, 108, 226, 17, 351, 356, 302, 171, 241, 340, 353, 135, 167, 325, 257, 76, 97, 59, 376, 103, 174, 258, 99, 166, 177, 375, 307, 94, 84, 188, 79, 259, 354, 115, 182, 49, 158, 169, 77, 264, 365, 262, 137, 159, 60, 194, 145, 333, 198, 298, 54, 134, 285, 203, 347, 102, 303, 222, 78, 26, 19, 344, 72, 75, 214, 323, 90, 297, 305, 142, 308, 343, 306, 253, 281, 233, 326, 378, 260, 88, 368, 208, 184, 0, 277, 185, 380, 231, 190, 221, 107, 332, 157, 361, 110, 276, 86, 254, 316, 105, 193, 42, 381, 292, 235, 80, 337, 162, 338, 131, 250, 29, 127, 382, 265, 132, 363, 331, 161] + 7: [163, 274, 336, 308, 41, 292, 265, 271, 280, 99, 362, 188, 0, 281, 11, 82, 248, 349, 288, 325, 21, 377, 372, 214, 368, 16, 92, 129, 137, 380, 134, 306, 40, 309, 153, 52, 37, 320, 169, 295, 88, 173, 124, 63, 195, 150, 205, 138, 154, 145, 365, 364, 341, 7, 275, 108, 239, 187, 142, 342, 103, 25, 369, 212, 174, 94, 324, 100, 289, 168, 270, 296, 255, 213, 264, 98, 89, 311, 185, 190, 200, 102, 51, 301, 283, 68, 116, 351, 115, 151, 112, 316, 96, 43, 252, 315, 95, 45, 12, 54, 122, 160, 276, 60, 327, 182, 192, 263, 133, 292, 46, 340, 103, 71, 362, 97, 104, 181, 249, 307, 74, 285, 337, 374, 81, 170, 158, 132, 321, 26, 358, 107, 83, 335, 53, 155, 373, 291, 101, 345, 222, 268, 207, 1, 10, 367, 381, 128, 359, 66, 290, 127, 376, 30, 91, 50, 327, 7, 258, 263, 41, 292, 265, 340, 139, 13, 378, 354, 61, 256, 313, 217, 125, 119, 2, 235, 383, 224, 90, 167, 259, 109, 237, 5, 211, 44, 152, 232, 251, 65, 22, 120, 79, 106, 286, 72, 59, 80, 317, 266, 246, 332, 197, 199, 355, 179, 371, 42, 327, 7, 275, 192, 350, 260, 229, 382, 105, 25, 298, 329, 244, 73, 14, 267, 278, 17, 84, 250, 210, 62, 300, 241, 234, 204, 19, 178, 273, 305, 226, 310, 196, 247, 93, 193, 269, 216, 75, 69, 254, 39, 297, 159, 242, 230, 165, 144, 76, 240, 135, 338, 175, 272, 238, 64, 49, 176, 375, 343, 18, 201, 369, 328, 77, 302, 294, 164, 262, 31, 339, 284, 117, 219, 330, 206, 24, 245, 130, 180, 293, 58, 67, 353, 8, 227, 87, 203, 184, 166, 287, 218, 85, 111, 3, 334, 304, 346, 331, 282, 34, 236, 348, 357, 175, 182, 274, 361, 49, 28, 142, 343, 18, 253, 298, 15, 370, 131, 183, 215, 4, 55, 360, 356, 208, 177, 191, 243, 189, 314, 136, 32, 220, 6, 141, 323, 161, 146, 149, 366, 126, 35, 33, 36, 123, 231, 303, 228, 198, 147, 172, 347, 156, 279, 202, 257, 162, 157, 274, 64, 49, 375, 229, 382, 105, 201, 118, 48, 27, 114, 47, 113, 29, 23, 148, 319, 70, 221, 344, 333, 9, 140, 20, 186, 322, 86, 121, 56, 233, 261, 38, 225, 318, 209, 363, 194, 312, 379, 277, 78, 299, 57, 352, 143, 223, 326, 110, 171] + 8: [273, 290, 31, 135, 359, 239, 362, 241, 308, 152, 228, 281, 251, 255, 377, 131, 181, 75, 283, 240, 92, 178, 303, 113, 300, 318, 66, 114, 217, 254, 73, 191, 352, 382, 136, 16, 200, 235, 375, 369, 153, 326, 271, 270, 268, 123, 142, 82, 124, 361, 1, 132, 253, 360, 292, 135, 359, 264, 362, 10, 272, 353, 204, 74, 20, 349, 97, 280, 311, 150, 27, 38, 78, 65, 368, 194, 48, 120, 125, 284, 206, 383, 248, 247, 373, 47, 266, 313, 325, 89, 295, 185, 365, 24, 193, 77, 269, 343, 203, 274, 288, 127, 163, 96, 253, 360, 67, 31, 359, 264, 201, 336, 308, 152, 229, 182, 252, 209, 168, 86, 128, 83, 333, 119, 261, 165, 137, 309, 164, 141, 332, 61, 59, 118, 237, 249, 350, 299, 372, 314, 339, 159, 306, 347, 287, 111, 324, 208, 319, 363, 53, 331, 49, 0, 102, 100, 253, 360, 67, 238, 21, 108, 173, 62, 62, 154, 371, 122, 95, 380, 115, 156, 166, 22, 133, 179, 202, 40, 278, 50, 317, 211, 91, 72, 263, 117, 25, 257, 149, 259, 155, 140, 105, 205, 9, 93, 357, 351, 15, 129, 329, 305, 51, 344, 345, 296, 227, 170, 104, 360, 290, 338, 367, 201, 312, 321, 381, 371, 229, 276, 186, 316, 328, 195, 107, 69, 226, 258, 17, 121, 189, 167, 130, 192, 2, 4, 187, 174, 379, 71, 151, 37, 190, 68, 144, 145, 236, 46, 315, 76, 84, 158, 26, 70, 355, 301, 222, 176, 262, 234, 214, 279, 290, 106, 161, 224, 12, 243, 221, 282, 204, 81, 291, 18, 11, 374, 366, 172, 109, 231, 298, 275, 112, 198, 267, 233, 52, 8, 45, 98, 28, 293, 184, 41, 307, 143, 79, 85, 175, 322, 33, 277, 171, 244, 180, 286, 43, 342, 57, 210, 376, 207, 5, 250, 218, 21, 367, 13, 162, 10, 221, 353, 36, 182, 30, 44, 364, 148, 225, 356, 23, 147, 169, 216, 304, 196, 320, 197, 358, 60, 101, 212, 116, 232, 110, 310, 327, 199, 265, 378, 14, 29, 323, 220, 348, 340, 245, 354, 80, 94, 289, 242, 63, 213, 5, 3, 246, 302, 7, 239, 162, 321, 88, 64, 337, 281, 223, 157, 294, 39, 346, 188, 34, 6, 330, 215, 219, 183, 334, 177, 341, 42, 335, 230, 87, 90, 297, 32, 126, 56, 99, 35, 260, 54, 134, 146, 103, 370, 285, 160, 138, 139, 256, 55, 19, 58] + 9: [212, 349, 81, 94, 94, 165, 155, 184, 1, 313, 262, 287, 244, 103, 10, 239, 133, 158, 302, 169, 88, 6, 364, 163, 35, 267, 36, 171, 5, 75, 17, 175, 245, 373, 174, 115, 42, 12, 270, 21, 178, 59, 193, 243, 266, 119, 278, 40, 362, 55, 152, 234, 92, 255, 331, 372, 172, 295, 327, 47, 272, 201, 242, 329, 64, 103, 67, 102, 312, 161, 377, 292, 306, 180, 314, 145, 142, 336, 66, 189, 104, 343, 339, 50, 100, 205, 322, 2, 320, 251, 224, 211, 69, 188, 34, 58, 118, 328, 204, 200, 250, 61, 24, 365, 92, 255, 220, 226, 19, 350, 327, 381, 15, 313, 262, 49, 95, 147, 14, 32, 131, 82, 135, 137, 297, 247, 192, 129, 269, 206, 150, 72, 274, 324, 18, 8, 382, 197, 213, 358, 301, 134, 76, 229, 199, 170, 160, 368, 279, 63, 149, 294, 164, 53, 77, 173, 92, 255, 96, 280, 252, 357, 355, 185, 29, 90, 90, 380, 271, 209, 282, 56, 9, 317, 153, 33, 264, 321, 107, 116, 323, 123, 124, 332, 359, 80, 276, 65, 52, 275, 79, 203, 286, 352, 333, 221, 347, 374, 383, 225, 240, 361, 85, 378, 156, 27, 236, 57, 92, 349, 349, 260, 263, 74, 222, 179, 83, 146, 217, 281, 244, 341, 136, 22, 258, 144, 187, 3, 334, 231, 183, 20, 195, 207, 181, 238, 202, 39, 208, 354, 117, 45, 190, 114, 371, 305, 298, 89, 346, 337, 191, 43, 325, 223, 44, 113, 91, 182, 126, 340, 78, 166, 143, 226, 299, 353, 355, 47, 37, 376, 127, 159, 214, 147, 132, 177, 98, 360, 356, 289, 228, 0, 87, 41, 105, 4, 230, 141, 288, 13, 235, 130, 379, 233, 300, 291, 84, 71, 125, 38, 227, 167, 290, 68, 345, 73, 296, 257, 28, 265, 168, 338, 304, 304, 315, 31, 263, 295, 355, 185, 344, 316, 370, 49, 95, 121, 23, 342, 151, 285, 154, 120, 277, 128, 97, 99, 186, 16, 70, 218, 237, 319, 248, 109, 303, 46, 62, 112, 253, 330, 310, 232, 48, 86, 326, 108, 176, 249, 284, 273, 7, 122, 283, 348, 304, 304, 196, 31, 215, 219, 327, 351, 198, 316, 138, 210, 64, 51, 10, 309, 367, 30, 363, 268, 318, 54, 311, 259, 256, 139, 11, 111, 106, 308, 246, 307, 60, 148, 261, 254, 110, 375, 216, 162, 335, 241, 293, 140, 366, 101, 93, 194, 157, 25, 369, 26] + 10: [294, 180, 151, 319, 141, 289, 365, 69, 31, 225, 8, 1, 153, 138, 154, 201, 312, 234, 356, 208, 190, 359, 382, 198, 214, 135, 311, 27, 315, 26, 379, 71, 125, 39, 275, 163, 282, 226, 96, 139, 86, 119, 322, 53, 94, 344, 361, 277, 188, 381, 324, 283, 192, 180, 93, 319, 5, 262, 216, 367, 31, 57, 310, 54, 263, 241, 304, 58, 353, 11, 348, 377, 212, 271, 149, 229, 87, 321, 219, 106, 357, 346, 51, 267, 162, 174, 45, 13, 113, 209, 326, 235, 187, 205, 165, 203, 334, 333, 191, 156, 228, 253, 291, 68, 255, 246, 181, 319, 5, 24, 18, 286, 164, 73, 144, 179, 171, 177, 237, 213, 115, 136, 368, 148, 77, 280, 285, 90, 233, 21, 232, 335, 339, 327, 41, 109, 340, 332, 50, 123, 175, 257, 375, 80, 355, 224, 158, 133, 104, 281, 152, 260, 270, 166, 176, 358, 194, 300, 265, 221, 5, 320, 216, 126, 331, 92, 301, 195, 351, 210, 99, 91, 293, 272, 345, 342, 189, 223, 266, 0, 33, 46, 288, 60, 9, 378, 97, 371, 247, 193, 140, 343, 244, 169, 131, 62, 206, 303, 6, 16, 88, 157, 84, 63, 28, 238, 295, 297, 194, 300, 137, 108, 5, 24, 365, 167, 309, 2, 310, 134, 114, 19, 211, 248, 7, 259, 374, 186, 231, 218, 61, 145, 72, 3, 56, 314, 329, 328, 74, 278, 42, 254, 239, 362, 220, 307, 100, 302, 373, 292, 200, 245, 120, 44, 30, 202, 89, 160, 290, 336, 264, 300, 268, 319, 320, 121, 365, 286, 146, 101, 301, 170, 40, 66, 47, 269, 243, 130, 284, 49, 38, 376, 132, 199, 116, 341, 81, 85, 256, 338, 230, 15, 36, 184, 323, 261, 370, 364, 287, 111, 127, 147, 35, 34, 118, 207, 352, 14, 242, 236, 82, 29, 264, 300, 265, 319, 320, 121, 325, 279, 146, 79, 22, 178, 117, 330, 124, 308, 76, 252, 274, 316, 98, 306, 32, 128, 103, 105, 75, 122, 112, 59, 25, 383, 318, 64, 317, 183, 298, 299, 159, 173, 305, 217, 4, 251, 172, 83, 150, 10, 52, 70, 23, 366, 55, 55, 151, 48, 5, 320, 18, 337, 164, 102, 215, 161, 78, 182, 196, 380, 67, 240, 363, 204, 360, 249, 155, 250, 129, 107, 296, 142, 197, 20, 37, 258, 12, 354, 313, 95, 185, 65, 350, 222, 372, 17, 168, 276, 273, 347, 143, 227, 43, 110, 349, 369] + 11: [156, 363, 314, 215, 65, 150, 235, 233, 44, 126, 102, 345, 152, 284, 286, 318, 154, 297, 47, 280, 2, 373, 172, 151, 89, 298, 186, 242, 294, 245, 359, 91, 164, 57, 129, 188, 309, 274, 103, 106, 4, 140, 304, 328, 216, 139, 51, 206, 334, 50, 308, 269, 252, 88, 376, 215, 85, 85, 235, 121, 116, 126, 162, 24, 192, 330, 244, 381, 169, 23, 319, 124, 127, 75, 347, 10, 224, 289, 41, 81, 336, 279, 364, 302, 187, 64, 43, 7, 74, 231, 5, 323, 201, 322, 305, 379, 344, 246, 253, 76, 212, 137, 307, 241, 98, 130, 90, 3, 40, 150, 221, 11, 116, 248, 29, 0, 351, 141, 204, 282, 136, 320, 161, 45, 249, 226, 315, 86, 341, 255, 101, 327, 63, 30, 338, 179, 125, 264, 113, 62, 143, 184, 360, 222, 240, 132, 380, 229, 180, 181, 202, 123, 220, 321, 134, 243, 98, 130, 90, 83, 163, 110, 99, 300, 325, 357, 267, 144, 275, 165, 131, 228, 205, 78, 66, 371, 350, 92, 195, 340, 316, 14, 208, 301, 219, 168, 354, 349, 291, 293, 174, 182, 46, 194, 375, 210, 21, 214, 278, 277, 175, 369, 189, 276, 281, 72, 191, 335, 77, 130, 87, 296, 296, 326, 221, 36, 44, 100, 15, 218, 54, 178, 95, 177, 17, 303, 94, 367, 69, 117, 26, 251, 198, 55, 31, 97, 295, 265, 312, 311, 67, 120, 33, 225, 8, 19, 200, 356, 185, 104, 272, 1, 368, 16, 372, 122, 232, 257, 61, 105, 377, 260, 314, 83, 163, 110, 171, 11, 36, 100, 15, 332, 217, 343, 70, 337, 148, 196, 283, 213, 370, 159, 166, 374, 203, 96, 254, 270, 211, 59, 353, 193, 234, 383, 142, 263, 230, 271, 266, 112, 310, 167, 68, 183, 60, 53, 28, 239, 223, 25, 109, 12, 377, 299, 87, 296, 163, 110, 121, 145, 248, 357, 199, 236, 331, 128, 39, 20, 13, 250, 346, 107, 348, 27, 93, 160, 133, 365, 58, 209, 362, 366, 34, 135, 324, 79, 258, 355, 237, 382, 108, 306, 173, 287, 268, 114, 238, 317, 333, 155, 329, 273, 261, 9, 339, 339, 227, 314, 149, 119, 38, 190, 157, 285, 285, 22, 247, 52, 73, 146, 207, 313, 6, 170, 262, 80, 176, 56, 49, 197, 259, 35, 147, 292, 378, 158, 118, 115, 48, 18, 138, 358, 37, 84, 288, 290, 71, 342, 361, 352, 111, 82, 153, 256, 32, 42] + 12: [156, 120, 130, 158, 327, 109, 22, 177, 150, 333, 97, 369, 354, 75, 216, 317, 266, 277, 27, 300, 322, 294, 337, 23, 155, 142, 9, 191, 61, 62, 161, 169, 11, 115, 334, 183, 246, 312, 285, 105, 375, 113, 81, 247, 307, 118, 221, 342, 359, 353, 162, 110, 255, 49, 381, 226, 344, 267, 292, 177, 243, 333, 192, 181, 170, 295, 332, 348, 147, 60, 33, 264, 362, 197, 238, 74, 15, 253, 200, 114, 165, 138, 280, 319, 76, 40, 305, 5, 179, 128, 106, 70, 316, 18, 299, 258, 7, 184, 328, 311, 64, 141, 366, 42, 255, 149, 130, 78, 167, 303, 82, 296, 150, 203, 233, 361, 123, 134, 137, 330, 87, 108, 160, 171, 104, 135, 326, 91, 286, 370, 52, 185, 119, 262, 136, 58, 335, 270, 379, 336, 244, 263, 85, 310, 93, 55, 54, 367, 111, 145, 360, 248, 324, 239, 222, 272, 255, 149, 130, 158, 327, 168, 124, 289, 243, 287, 341, 98, 260, 195, 205, 276, 25, 89, 209, 102, 271, 241, 291, 157, 304, 193, 77, 41, 45, 67, 139, 37, 235, 351, 140, 12, 329, 279, 274, 44, 282, 96, 212, 19, 210, 338, 163, 206, 166, 240, 53, 73, 255, 35, 120, 28, 17, 127, 349, 69, 172, 46, 90, 151, 176, 6, 382, 152, 368, 16, 2, 371, 57, 63, 380, 14, 144, 227, 254, 315, 189, 313, 249, 325, 122, 154, 65, 86, 186, 148, 383, 30, 47, 343, 72, 236, 143, 94, 228, 275, 21, 56, 250, 302, 255, 48, 120, 226, 190, 127, 223, 69, 71, 103, 290, 306, 340, 3, 346, 268, 323, 10, 153, 101, 273, 129, 261, 301, 309, 201, 188, 0, 83, 298, 232, 321, 214, 159, 257, 112, 284, 217, 24, 252, 8, 131, 211, 320, 182, 269, 314, 213, 95, 66, 43, 225, 204, 204, 78, 158, 190, 303, 82, 378, 172, 46, 50, 51, 259, 278, 281, 194, 218, 364, 288, 116, 107, 164, 31, 59, 283, 357, 350, 234, 29, 224, 356, 187, 355, 36, 13, 293, 297, 207, 174, 146, 345, 347, 117, 125, 196, 132, 242, 100, 92, 331, 199, 372, 204, 363, 245, 219, 327, 175, 223, 378, 126, 203, 341, 198, 208, 339, 68, 215, 229, 133, 39, 265, 180, 308, 80, 318, 358, 376, 1, 231, 84, 256, 4, 173, 38, 121, 20, 32, 365, 374, 352, 251, 34, 178, 237, 230, 79, 26, 202, 88, 377, 99, 373, 220] + 13: [80, 170, 232, 114, 195, 149, 260, 280, 189, 238, 318, 102, 370, 46, 279, 58, 112, 273, 309, 363, 2, 378, 179, 62, 228, 107, 374, 243, 241, 37, 163, 357, 84, 305, 113, 250, 70, 73, 5, 103, 205, 346, 343, 128, 278, 293, 132, 354, 218, 301, 33, 78, 80, 170, 232, 96, 14, 23, 312, 172, 95, 110, 274, 366, 88, 159, 118, 227, 31, 289, 91, 292, 373, 65, 375, 43, 239, 74, 359, 29, 222, 303, 337, 286, 220, 127, 348, 53, 154, 213, 313, 249, 4, 167, 117, 212, 264, 20, 332, 200, 82, 247, 155, 316, 9, 170, 232, 114, 14, 23, 265, 135, 276, 12, 30, 186, 199, 109, 35, 49, 75, 202, 40, 334, 6, 42, 219, 321, 26, 225, 333, 183, 76, 125, 48, 153, 345, 268, 245, 308, 115, 217, 287, 119, 138, 180, 284, 352, 99, 224, 341, 302, 351, 165, 27, 7, 44, 77, 108, 81, 0, 328, 312, 51, 55, 209, 254, 17, 162, 298, 295, 111, 188, 275, 229, 67, 178, 22, 211, 141, 344, 283, 93, 139, 380, 34, 288, 106, 136, 296, 381, 94, 72, 13, 130, 104, 285, 383, 1, 215, 181, 317, 236, 368, 277, 314, 315, 192, 44, 170, 170, 18, 86, 328, 39, 280, 189, 238, 318, 134, 11, 116, 307, 198, 16, 356, 261, 158, 300, 98, 355, 382, 144, 121, 166, 146, 15, 105, 36, 68, 242, 92, 325, 187, 270, 38, 150, 353, 83, 372, 101, 28, 152, 251, 269, 156, 120, 193, 350, 32, 362, 329, 57, 18, 195, 149, 260, 21, 201, 12, 330, 197, 338, 306, 131, 64, 364, 376, 294, 281, 122, 210, 79, 339, 291, 361, 253, 129, 164, 331, 161, 248, 319, 59, 263, 311, 358, 19, 124, 214, 185, 85, 252, 63, 258, 371, 272, 148, 367, 324, 140, 230, 362, 137, 108, 3, 195, 87, 24, 41, 231, 30, 274, 310, 335, 171, 176, 45, 69, 297, 143, 266, 299, 147, 10, 240, 145, 196, 151, 327, 221, 223, 89, 379, 174, 259, 50, 133, 360, 234, 235, 190, 182, 157, 336, 191, 90, 347, 290, 47, 160, 206, 126, 204, 194, 329, 108, 18, 87, 87, 169, 282, 342, 100, 340, 271, 142, 304, 323, 184, 369, 246, 54, 320, 208, 207, 168, 237, 25, 262, 177, 256, 52, 8, 267, 326, 56, 257, 244, 71, 123, 377, 365, 61, 233, 226, 60, 203, 173, 97, 349, 255, 322, 66, 216, 175] + 14: [147, 143, 15, 164, 60, 30, 268, 186, 126, 348, 235, 99, 32, 97, 374, 21, 294, 13, 297, 343, 332, 193, 363, 187, 325, 373, 5, 347, 341, 189, 335, 313, 244, 251, 28, 6, 278, 127, 136, 19, 87, 18, 55, 92, 236, 307, 134, 243, 218, 67, 318, 165, 70, 274, 76, 76, 17, 175, 2, 75, 122, 378, 142, 228, 124, 64, 154, 250, 41, 197, 379, 123, 288, 145, 37, 382, 286, 178, 101, 380, 359, 253, 115, 12, 356, 42, 285, 329, 103, 195, 334, 190, 312, 132, 367, 84, 233, 219, 170, 303, 210, 119, 83, 260, 70, 143, 4, 287, 255, 199, 209, 75, 327, 353, 275, 85, 100, 3, 79, 271, 216, 314, 161, 328, 357, 277, 108, 40, 102, 227, 82, 129, 298, 336, 223, 191, 183, 248, 10, 214, 309, 36, 20, 14, 179, 194, 293, 24, 181, 283, 72, 54, 352, 130, 73, 331, 56, 211, 372, 204, 255, 110, 259, 34, 355, 337, 71, 23, 117, 177, 269, 279, 166, 125, 25, 163, 291, 159, 301, 169, 128, 234, 48, 156, 176, 316, 202, 282, 217, 43, 106, 273, 171, 365, 9, 290, 258, 7, 66, 139, 11, 320, 192, 131, 29, 196, 249, 50, 160, 229, 207, 345, 137, 112, 2, 239, 150, 378, 71, 275, 231, 213, 162, 354, 270, 220, 206, 323, 68, 201, 22, 346, 203, 245, 241, 237, 284, 0, 89, 242, 224, 376, 310, 173, 358, 267, 238, 16, 263, 45, 57, 185, 247, 65, 188, 340, 292, 98, 315, 174, 160, 229, 15, 77, 255, 110, 259, 122, 126, 146, 155, 151, 52, 342, 360, 182, 157, 33, 135, 246, 319, 369, 104, 339, 172, 109, 90, 377, 306, 225, 300, 364, 63, 93, 114, 321, 375, 74, 78, 58, 299, 80, 62, 35, 107, 144, 113, 280, 53, 351, 121, 205, 160, 211, 4, 204, 8, 30, 49, 34, 355, 118, 155, 141, 198, 371, 317, 44, 265, 158, 59, 69, 184, 370, 167, 333, 256, 296, 116, 105, 330, 200, 168, 148, 1, 240, 94, 262, 257, 338, 361, 230, 322, 31, 366, 120, 140, 252, 51, 368, 232, 138, 212, 289, 229, 229, 261, 345, 345, 383, 209, 344, 327, 308, 308, 254, 95, 39, 61, 111, 26, 208, 149, 215, 324, 326, 266, 133, 349, 226, 222, 221, 276, 86, 152, 264, 350, 96, 311, 302, 153, 46, 304, 305, 281, 47, 362, 180, 81, 381, 91, 27, 38, 272, 295, 88] + 15: [68, 88, 295, 11, 371, 251, 155, 231, 128, 104, 252, 51, 360, 83, 353, 1, 241, 115, 162, 319, 170, 238, 105, 82, 309, 268, 81, 254, 294, 349, 381, 137, 244, 71, 283, 265, 54, 278, 369, 188, 201, 310, 7, 195, 154, 307, 122, 219, 348, 264, 366, 328, 298, 88, 259, 11, 362, 160, 185, 200, 217, 236, 74, 20, 151, 40, 344, 256, 111, 109, 225, 335, 182, 49, 166, 59, 114, 282, 69, 165, 382, 306, 72, 354, 224, 123, 375, 213, 167, 239, 161, 153, 126, 187, 124, 181, 44, 120, 125, 156, 186, 6, 131, 368, 0, 350, 367, 130, 359, 160, 38, 258, 266, 175, 284, 60, 232, 139, 173, 342, 177, 18, 36, 357, 5, 42, 102, 86, 87, 62, 50, 43, 339, 107, 321, 234, 314, 211, 373, 84, 116, 3, 197, 129, 14, 312, 292, 66, 178, 346, 288, 336, 19, 46, 127, 281, 0, 76, 227, 11, 362, 160, 155, 355, 128, 104, 333, 237, 329, 48, 296, 135, 242, 164, 379, 351, 209, 305, 212, 2, 10, 96, 363, 253, 30, 313, 97, 289, 55, 169, 337, 316, 240, 79, 91, 184, 140, 293, 112, 222, 179, 70, 286, 345, 249, 75, 198, 93, 0, 4, 88, 11, 57, 300, 324, 200, 217, 134, 134, 9, 157, 56, 113, 311, 320, 196, 285, 100, 133, 28, 145, 263, 269, 150, 233, 341, 141, 191, 61, 267, 204, 383, 95, 334, 273, 147, 274, 374, 149, 318, 136, 330, 183, 110, 108, 214, 27, 152, 192, 270, 290, 340, 202, 130, 359, 251, 324, 326, 94, 175, 215, 90, 221, 23, 193, 168, 143, 230, 106, 338, 323, 194, 101, 142, 34, 229, 250, 13, 377, 118, 347, 365, 216, 64, 26, 257, 218, 364, 73, 89, 370, 280, 327, 228, 103, 304, 21, 302, 315, 25, 325, 85, 15, 15, 130, 130, 359, 199, 355, 332, 159, 148, 203, 146, 80, 376, 277, 163, 255, 24, 352, 47, 78, 246, 279, 248, 343, 331, 8, 206, 271, 226, 53, 308, 45, 272, 172, 358, 52, 65, 303, 158, 29, 176, 58, 98, 299, 17, 243, 220, 132, 144, 322, 356, 15, 261, 227, 291, 57, 39, 189, 258, 94, 148, 119, 121, 317, 16, 35, 174, 138, 235, 31, 207, 372, 247, 275, 117, 77, 22, 205, 12, 301, 67, 180, 99, 208, 37, 92, 63, 190, 210, 378, 32, 262, 380, 171, 361, 41, 223, 287, 297, 33, 245, 260, 276] + 16: [11, 286, 369, 120, 300, 92, 338, 147, 303, 203, 5, 250, 383, 295, 246, 108, 181, 328, 335, 99, 103, 159, 380, 195, 229, 334, 160, 124, 225, 29, 25, 242, 337, 105, 48, 266, 233, 157, 261, 110, 216, 332, 251, 382, 79, 23, 341, 222, 53, 191, 309, 249, 162, 68, 152, 331, 262, 7, 165, 90, 118, 280, 35, 274, 298, 141, 212, 16, 285, 177, 307, 292, 119, 0, 182, 34, 379, 202, 276, 188, 317, 354, 57, 313, 343, 378, 91, 30, 172, 166, 20, 17, 308, 149, 245, 28, 63, 40, 350, 278, 133, 345, 366, 257, 100, 15, 152, 154, 234, 92, 165, 129, 118, 280, 183, 224, 121, 46, 367, 180, 365, 306, 248, 21, 80, 282, 31, 70, 220, 19, 9, 96, 221, 228, 200, 178, 223, 93, 109, 193, 186, 128, 324, 117, 116, 359, 260, 32, 355, 74, 299, 204, 376, 320, 1, 98, 100, 15, 67, 114, 234, 312, 273, 64, 44, 75, 348, 24, 209, 364, 347, 85, 27, 237, 115, 38, 281, 153, 311, 167, 356, 293, 297, 4, 290, 336, 158, 214, 253, 272, 142, 201, 62, 14, 47, 131, 217, 50, 138, 352, 12, 130, 136, 291, 361, 13, 94, 344, 140, 15, 286, 120, 120, 262, 277, 269, 143, 342, 374, 302, 283, 213, 270, 346, 314, 84, 51, 316, 58, 66, 41, 8, 232, 372, 238, 22, 252, 358, 54, 102, 323, 83, 244, 363, 81, 127, 218, 26, 170, 241, 243, 169, 6, 112, 197, 72, 296, 215, 247, 156, 140, 15, 369, 185, 275, 7, 86, 64, 303, 75, 271, 255, 287, 236, 87, 226, 310, 321, 97, 37, 175, 39, 184, 76, 36, 42, 137, 322, 264, 340, 132, 95, 206, 55, 190, 360, 315, 373, 268, 113, 371, 150, 61, 288, 351, 329, 263, 179, 325, 239, 368, 353, 134, 134, 286, 330, 275, 43, 147, 49, 44, 259, 271, 171, 189, 235, 146, 89, 65, 71, 59, 82, 73, 155, 305, 219, 45, 122, 174, 370, 104, 151, 208, 211, 333, 304, 196, 240, 163, 88, 107, 168, 256, 77, 210, 33, 164, 135, 173, 326, 362, 69, 111, 3, 134, 134, 286, 125, 275, 43, 144, 269, 289, 318, 205, 78, 265, 187, 123, 199, 294, 52, 2, 377, 139, 56, 231, 227, 357, 301, 339, 106, 198, 375, 18, 60, 267, 319, 207, 194, 381, 284, 145, 258, 279, 176, 230, 101, 349, 254, 192, 327, 10, 161, 126, 148] + 17: [374, 119, 188, 236, 303, 58, 87, 15, 383, 193, 305, 343, 351, 62, 8, 56, 207, 24, 231, 38, 98, 146, 282, 259, 82, 61, 42, 269, 196, 377, 338, 181, 249, 195, 299, 287, 274, 326, 364, 63, 53, 316, 141, 201, 291, 11, 247, 111, 123, 34, 69, 72, 261, 119, 250, 236, 346, 79, 200, 254, 117, 225, 305, 323, 1, 189, 276, 157, 96, 199, 145, 228, 84, 298, 171, 245, 13, 47, 59, 295, 49, 115, 37, 21, 179, 240, 90, 345, 55, 158, 255, 233, 65, 352, 176, 109, 229, 129, 289, 268, 174, 54, 121, 152, 261, 119, 140, 236, 226, 337, 375, 375, 383, 197, 330, 362, 2, 297, 234, 107, 317, 97, 381, 211, 139, 99, 263, 18, 264, 36, 260, 7, 204, 205, 101, 213, 360, 361, 329, 210, 48, 308, 50, 318, 132, 241, 40, 277, 4, 160, 353, 321, 301, 127, 312, 320, 28, 119, 250, 284, 190, 337, 200, 219, 136, 275, 330, 362, 125, 6, 75, 52, 194, 335, 180, 92, 367, 327, 9, 142, 191, 20, 331, 154, 118, 382, 253, 342, 238, 262, 239, 112, 128, 278, 270, 57, 163, 182, 23, 30, 257, 306, 348, 281, 209, 86, 235, 237, 212, 220, 208, 188, 79, 79, 375, 91, 311, 5, 286, 185, 324, 328, 172, 290, 39, 46, 313, 103, 143, 135, 214, 168, 248, 166, 43, 41, 81, 3, 309, 44, 379, 302, 183, 161, 151, 369, 350, 144, 164, 227, 149, 66, 349, 71, 77, 35, 252, 122, 251, 283, 138, 220, 307, 284, 319, 126, 108, 15, 155, 26, 232, 114, 130, 29, 370, 104, 73, 22, 243, 380, 256, 33, 292, 376, 332, 116, 147, 215, 198, 365, 89, 95, 300, 244, 102, 16, 64, 25, 192, 344, 355, 88, 280, 223, 368, 285, 334, 322, 293, 169, 159, 170, 131, 131, 80, 284, 303, 366, 108, 357, 100, 26, 294, 279, 246, 133, 167, 222, 134, 206, 148, 273, 371, 74, 165, 70, 304, 19, 173, 363, 216, 186, 341, 265, 110, 310, 120, 85, 267, 124, 67, 150, 83, 153, 378, 224, 178, 242, 17, 68, 156, 14, 202, 203, 131, 131, 105, 284, 137, 337, 12, 78, 100, 359, 294, 266, 113, 347, 175, 339, 296, 218, 177, 373, 10, 272, 314, 162, 340, 60, 45, 336, 187, 221, 271, 27, 94, 93, 258, 217, 106, 372, 76, 31, 32, 315, 230, 356, 184, 288, 0, 51, 333, 358, 325, 354] + 18: [208, 10, 271, 210, 251, 315, 300, 57, 138, 380, 174, 68, 306, 284, 378, 67, 103, 46, 371, 342, 83, 87, 88, 241, 201, 313, 152, 140, 264, 362, 265, 363, 243, 325, 348, 168, 332, 311, 224, 76, 218, 372, 238, 281, 5, 305, 336, 323, 226, 97, 242, 59, 208, 272, 271, 16, 251, 320, 65, 114, 106, 185, 236, 25, 360, 297, 285, 190, 165, 181, 41, 141, 359, 79, 334, 22, 278, 50, 61, 349, 54, 319, 335, 187, 351, 179, 303, 175, 216, 66, 17, 229, 75, 42, 370, 294, 100, 11, 21, 196, 353, 120, 268, 78, 6, 10, 271, 333, 251, 207, 245, 119, 138, 185, 189, 167, 376, 151, 330, 101, 204, 89, 171, 339, 104, 149, 109, 296, 122, 361, 8, 195, 150, 9, 63, 155, 287, 92, 275, 82, 170, 219, 130, 123, 298, 172, 85, 29, 377, 290, 62, 194, 55, 374, 344, 340, 118, 47, 271, 53, 69, 217, 347, 57, 71, 260, 239, 321, 237, 291, 246, 3, 240, 19, 366, 248, 177, 343, 129, 135, 288, 231, 289, 176, 205, 276, 132, 255, 322, 112, 108, 60, 338, 247, 163, 133, 364, 354, 90, 184, 14, 160, 173, 221, 91, 52, 183, 382, 356, 47, 271, 45, 69, 217, 113, 36, 228, 260, 142, 146, 34, 253, 99, 33, 188, 164, 365, 73, 307, 233, 263, 137, 159, 328, 235, 15, 125, 244, 12, 13, 270, 254, 127, 383, 220, 51, 126, 369, 312, 86, 44, 352, 249, 117, 72, 134, 49, 327, 314, 309, 30, 30, 43, 271, 320, 257, 245, 215, 228, 110, 192, 26, 31, 357, 35, 326, 124, 269, 337, 191, 283, 262, 131, 162, 350, 302, 102, 203, 178, 93, 182, 329, 153, 214, 213, 234, 64, 27, 209, 375, 2, 107, 115, 273, 24, 212, 206, 295, 128, 74, 222, 1, 30, 30, 43, 193, 211, 257, 28, 304, 71, 368, 116, 301, 139, 282, 266, 77, 70, 318, 20, 202, 331, 261, 225, 0, 7, 58, 157, 308, 279, 81, 145, 292, 94, 166, 48, 358, 158, 4, 56, 317, 95, 84, 37, 299, 230, 18, 199, 198, 379, 156, 293, 147, 30, 30, 271, 271, 286, 154, 300, 65, 106, 169, 136, 355, 373, 144, 341, 105, 98, 274, 346, 23, 121, 223, 316, 367, 232, 250, 39, 111, 38, 96, 40, 267, 148, 324, 256, 143, 200, 259, 197, 381, 180, 258, 161, 280, 345, 277, 252, 310, 186, 80, 227, 32] + 19: [187, 24, 48, 302, 61, 91, 161, 313, 312, 300, 64, 113, 33, 303, 117, 44, 292, 53, 285, 222, 359, 76, 29, 283, 144, 4, 244, 325, 177, 45, 148, 47, 281, 119, 130, 168, 132, 234, 314, 315, 106, 11, 201, 51, 205, 308, 43, 220, 274, 317, 263, 268, 187, 24, 256, 7, 364, 255, 288, 180, 219, 8, 307, 115, 164, 77, 159, 79, 46, 54, 71, 278, 337, 96, 251, 204, 2, 215, 203, 18, 333, 202, 160, 3, 191, 383, 311, 32, 232, 174, 63, 37, 150, 269, 214, 252, 163, 236, 282, 78, 145, 93, 81, 98, 187, 24, 256, 7, 377, 120, 288, 180, 207, 312, 64, 323, 193, 296, 153, 58, 25, 335, 1, 166, 127, 172, 156, 239, 157, 110, 306, 336, 213, 10, 331, 129, 376, 321, 176, 320, 181, 332, 324, 361, 118, 124, 290, 100, 299, 379, 69, 327, 276, 350, 319, 355, 141, 366, 74, 7, 57, 70, 338, 279, 42, 101, 373, 27, 218, 97, 230, 217, 147, 135, 36, 134, 6, 83, 195, 138, 40, 31, 95, 287, 108, 380, 372, 62, 140, 329, 82, 139, 370, 247, 149, 73, 348, 123, 178, 367, 0, 5, 151, 72, 196, 229, 198, 245, 243, 24, 339, 90, 377, 288, 338, 183, 318, 197, 307, 115, 38, 271, 301, 189, 171, 246, 116, 310, 182, 167, 49, 170, 297, 259, 342, 194, 85, 80, 60, 92, 88, 200, 341, 343, 99, 105, 28, 371, 381, 210, 358, 136, 344, 102, 169, 382, 316, 12, 34, 50, 190, 211, 206, 265, 90, 120, 330, 313, 207, 101, 66, 184, 349, 227, 275, 264, 158, 122, 13, 128, 228, 121, 16, 272, 67, 345, 143, 84, 19, 23, 356, 65, 225, 362, 137, 209, 14, 208, 286, 294, 357, 162, 185, 21, 155, 261, 17, 199, 250, 305, 253, 254, 190, 366, 111, 90, 241, 188, 330, 126, 42, 240, 114, 328, 20, 334, 354, 9, 41, 15, 233, 352, 173, 68, 353, 221, 260, 249, 374, 125, 284, 262, 293, 26, 309, 242, 298, 87, 258, 235, 94, 59, 231, 289, 223, 304, 266, 248, 360, 104, 175, 109, 107, 133, 190, 366, 179, 265, 57, 52, 291, 322, 192, 240, 216, 363, 154, 89, 152, 35, 365, 238, 30, 295, 165, 270, 368, 257, 347, 277, 273, 346, 237, 131, 75, 280, 146, 326, 212, 22, 186, 340, 351, 103, 112, 267, 39, 224, 86, 56, 226, 55, 142, 378, 369, 375] + 20: [116, 174, 161, 144, 193, 110, 2, 226, 379, 224, 105, 93, 228, 273, 362, 259, 132, 347, 90, 239, 218, 343, 265, 314, 208, 18, 166, 246, 248, 12, 236, 257, 191, 253, 217, 210, 199, 296, 356, 323, 354, 157, 134, 183, 23, 222, 255, 288, 280, 303, 331, 350, 116, 302, 272, 82, 98, 153, 376, 268, 335, 130, 270, 318, 209, 128, 382, 227, 292, 172, 315, 13, 194, 26, 25, 30, 233, 34, 131, 340, 377, 100, 332, 3, 312, 212, 316, 285, 289, 319, 7, 329, 225, 207, 234, 245, 103, 275, 298, 17, 117, 27, 51, 52, 116, 302, 14, 161, 66, 287, 369, 145, 291, 282, 32, 371, 324, 221, 9, 135, 258, 92, 322, 6, 235, 35, 175, 367, 300, 22, 238, 345, 15, 338, 304, 231, 360, 230, 159, 366, 115, 283, 102, 163, 47, 83, 293, 70, 149, 306, 38, 109, 241, 107, 137, 167, 116, 336, 336, 317, 81, 99, 369, 45, 4, 353, 363, 325, 111, 141, 187, 160, 216, 121, 182, 177, 240, 375, 36, 58, 247, 114, 196, 346, 20, 310, 73, 201, 326, 337, 56, 80, 156, 88, 321, 341, 67, 276, 373, 147, 192, 344, 44, 143, 242, 168, 179, 181, 334, 37, 161, 144, 98, 153, 87, 268, 335, 46, 62, 162, 10, 95, 311, 140, 49, 184, 357, 57, 364, 41, 0, 146, 178, 372, 243, 169, 294, 252, 127, 170, 24, 84, 308, 151, 59, 21, 40, 271, 136, 19, 64, 68, 220, 152, 237, 197, 260, 214, 262, 263, 334, 278, 161, 55, 98, 123, 87, 145, 307, 46, 185, 330, 164, 290, 264, 180, 320, 71, 77, 190, 284, 361, 118, 359, 150, 119, 215, 86, 370, 33, 78, 365, 89, 223, 154, 139, 142, 120, 113, 355, 211, 63, 133, 108, 251, 155, 244, 269, 277, 266, 281, 299, 334, 278, 161, 82, 173, 153, 43, 112, 339, 195, 204, 148, 91, 229, 96, 50, 101, 65, 126, 74, 267, 328, 219, 213, 309, 256, 274, 97, 297, 232, 106, 39, 374, 351, 254, 11, 352, 124, 28, 205, 72, 16, 48, 383, 186, 31, 203, 171, 249, 189, 200, 206, 8, 122, 161, 66, 193, 99, 43, 45, 4, 282, 188, 138, 158, 125, 381, 286, 342, 358, 104, 176, 348, 378, 53, 368, 279, 42, 60, 165, 29, 5, 380, 202, 327, 75, 94, 349, 79, 250, 85, 333, 305, 313, 261, 295, 129, 301, 1, 69, 198, 54, 61, 76] + 21: [152, 153, 4, 173, 205, 132, 53, 376, 71, 113, 361, 232, 178, 177, 172, 351, 8, 69, 52, 72, 245, 131, 168, 1, 64, 309, 312, 28, 317, 323, 204, 171, 187, 170, 340, 300, 379, 110, 195, 164, 70, 330, 217, 314, 296, 142, 278, 188, 343, 11, 42, 264, 152, 355, 256, 173, 281, 132, 80, 124, 257, 185, 348, 214, 118, 149, 380, 109, 121, 194, 230, 141, 373, 89, 372, 358, 259, 262, 97, 303, 125, 378, 40, 100, 136, 104, 346, 48, 244, 334, 163, 241, 143, 41, 76, 56, 249, 106, 7, 116, 248, 327, 35, 58, 152, 355, 4, 247, 281, 158, 50, 200, 92, 364, 94, 345, 0, 238, 191, 19, 196, 25, 313, 111, 183, 243, 63, 49, 90, 223, 260, 159, 370, 5, 77, 193, 160, 198, 146, 276, 273, 381, 181, 253, 156, 74, 129, 180, 255, 31, 161, 166, 322, 332, 93, 123, 302, 308, 236, 173, 281, 132, 38, 210, 112, 87, 117, 120, 150, 279, 184, 128, 219, 293, 209, 221, 33, 174, 73, 267, 138, 60, 98, 192, 354, 133, 237, 148, 137, 203, 329, 216, 81, 83, 319, 294, 75, 13, 68, 114, 328, 145, 298, 212, 101, 127, 44, 306, 316, 371, 254, 47, 360, 158, 80, 376, 71, 140, 88, 377, 91, 51, 307, 147, 15, 227, 144, 367, 288, 233, 263, 352, 359, 246, 21, 356, 155, 3, 295, 341, 337, 24, 274, 126, 157, 27, 67, 382, 268, 2, 357, 54, 162, 265, 59, 231, 224, 283, 179, 339, 316, 308, 254, 47, 360, 360, 103, 119, 218, 113, 199, 277, 252, 250, 22, 349, 299, 225, 9, 189, 234, 325, 197, 336, 287, 139, 251, 291, 211, 347, 353, 115, 34, 79, 186, 36, 215, 271, 96, 310, 201, 122, 55, 375, 228, 285, 61, 282, 272, 297, 369, 350, 20, 289, 190, 46, 206, 86, 103, 82, 257, 208, 363, 45, 284, 240, 151, 365, 175, 344, 43, 23, 78, 270, 338, 331, 39, 304, 66, 383, 290, 154, 275, 266, 292, 269, 57, 261, 213, 134, 12, 315, 239, 374, 99, 324, 202, 242, 32, 222, 107, 167, 108, 321, 20, 371, 355, 47, 360, 29, 103, 124, 119, 140, 169, 226, 182, 320, 16, 26, 366, 280, 135, 220, 368, 229, 84, 18, 362, 318, 62, 286, 165, 30, 65, 301, 14, 102, 258, 335, 333, 305, 105, 342, 207, 130, 85, 37, 176, 6, 326, 95, 235, 311, 10, 17] + 22: [371, 126, 332, 221, 260, 330, 42, 224, 184, 376, 177, 44, 61, 229, 153, 111, 100, 174, 106, 214, 57, 113, 340, 25, 131, 210, 320, 98, 65, 377, 373, 299, 138, 215, 379, 298, 160, 182, 92, 129, 350, 212, 308, 127, 284, 277, 167, 194, 325, 2, 28, 79, 254, 204, 331, 181, 148, 24, 328, 164, 183, 230, 74, 29, 117, 55, 351, 338, 80, 76, 125, 17, 122, 75, 191, 63, 15, 133, 380, 87, 301, 227, 78, 382, 290, 136, 364, 103, 232, 266, 36, 179, 19, 168, 354, 162, 304, 172, 293, 297, 192, 289, 247, 250, 254, 51, 278, 221, 260, 261, 38, 224, 183, 94, 4, 235, 130, 333, 70, 7, 195, 372, 222, 193, 349, 109, 156, 18, 324, 72, 39, 318, 303, 283, 11, 241, 97, 314, 176, 108, 246, 137, 334, 91, 257, 344, 363, 326, 43, 158, 121, 56, 313, 323, 307, 310, 254, 198, 343, 221, 141, 261, 132, 288, 201, 376, 169, 139, 362, 240, 272, 285, 67, 68, 60, 116, 206, 3, 53, 258, 71, 165, 1, 47, 200, 269, 32, 45, 286, 77, 275, 208, 274, 282, 120, 145, 16, 90, 31, 370, 66, 295, 243, 203, 10, 249, 95, 101, 254, 9, 343, 181, 141, 276, 207, 337, 73, 178, 37, 244, 383, 50, 149, 251, 375, 161, 264, 259, 226, 175, 82, 374, 233, 12, 319, 305, 322, 189, 327, 180, 211, 96, 356, 279, 85, 219, 30, 152, 291, 302, 223, 88, 213, 114, 270, 234, 142, 287, 157, 217, 316, 216, 209, 260, 141, 328, 147, 288, 178, 110, 263, 118, 58, 252, 357, 188, 306, 8, 312, 27, 361, 347, 339, 49, 309, 225, 239, 341, 22, 54, 238, 0, 300, 273, 46, 237, 102, 186, 196, 190, 104, 69, 255, 23, 21, 146, 107, 368, 205, 292, 253, 267, 281, 33, 209, 34, 141, 261, 115, 337, 20, 110, 112, 329, 355, 348, 345, 311, 173, 378, 365, 369, 14, 228, 48, 40, 59, 99, 105, 248, 236, 124, 256, 268, 271, 89, 353, 262, 35, 187, 294, 134, 64, 171, 150, 6, 197, 84, 245, 231, 135, 280, 143, 154, 281, 216, 278, 181, 148, 24, 328, 242, 20, 94, 144, 93, 317, 352, 381, 218, 336, 83, 123, 185, 170, 358, 366, 220, 321, 41, 367, 159, 155, 199, 335, 81, 13, 202, 26, 265, 119, 151, 52, 62, 342, 128, 296, 5, 140, 163, 166, 86, 315, 360, 346, 359] + 23: [26, 124, 172, 274, 376, 305, 167, 173, 203, 135, 103, 156, 192, 182, 56, 283, 209, 144, 312, 149, 317, 40, 221, 361, 153, 254, 52, 179, 347, 348, 73, 109, 302, 21, 114, 99, 338, 33, 106, 61, 158, 88, 350, 381, 118, 42, 246, 60, 166, 3, 39, 265, 26, 124, 172, 120, 71, 48, 187, 306, 252, 197, 177, 232, 67, 218, 159, 371, 131, 146, 378, 91, 195, 237, 210, 356, 119, 160, 137, 207, 382, 76, 258, 200, 335, 134, 121, 349, 115, 227, 214, 282, 319, 34, 238, 259, 22, 345, 145, 332, 251, 29, 8, 320, 26, 124, 314, 275, 100, 77, 187, 240, 85, 72, 287, 27, 127, 57, 148, 92, 190, 213, 102, 191, 10, 228, 292, 44, 150, 365, 125, 178, 162, 352, 363, 339, 174, 7, 354, 285, 97, 104, 248, 140, 9, 62, 116, 334, 318, 117, 199, 112, 296, 15, 291, 288, 26, 51, 175, 275, 147, 305, 331, 87, 323, 324, 177, 261, 374, 372, 86, 130, 379, 326, 70, 233, 194, 342, 311, 101, 298, 17, 273, 321, 341, 196, 58, 344, 346, 16, 155, 362, 375, 369, 308, 46, 310, 277, 377, 171, 138, 30, 83, 198, 13, 110, 43, 37, 126, 264, 175, 275, 100, 132, 295, 244, 85, 135, 358, 176, 241, 169, 315, 263, 133, 50, 220, 151, 313, 290, 129, 359, 96, 93, 231, 307, 223, 303, 143, 0, 164, 108, 14, 268, 123, 355, 79, 188, 357, 31, 322, 181, 20, 208, 183, 380, 18, 113, 235, 81, 126, 264, 69, 314, 170, 19, 167, 269, 323, 300, 279, 94, 304, 55, 270, 41, 340, 184, 74, 36, 23, 38, 136, 370, 32, 98, 5, 301, 316, 284, 54, 256, 107, 212, 122, 297, 239, 373, 205, 367, 368, 266, 337, 360, 364, 278, 262, 217, 226, 234, 286, 242, 328, 281, 314, 71, 71, 132, 201, 244, 12, 300, 327, 84, 180, 2, 6, 257, 35, 80, 219, 271, 95, 272, 215, 222, 185, 329, 65, 333, 330, 193, 90, 280, 28, 206, 243, 236, 336, 267, 161, 49, 353, 276, 165, 247, 141, 63, 89, 293, 366, 216, 189, 168, 328, 225, 172, 314, 211, 305, 82, 24, 252, 72, 47, 154, 11, 59, 351, 78, 309, 111, 249, 229, 152, 224, 255, 157, 1, 66, 53, 325, 250, 204, 45, 128, 245, 68, 163, 294, 25, 139, 260, 202, 253, 230, 383, 299, 343, 289, 105, 142, 186, 4, 75, 64] + 24: [0, 246, 265, 20, 328, 210, 223, 360, 85, 121, 294, 186, 283, 24, 164, 382, 299, 131, 142, 177, 338, 205, 272, 321, 235, 323, 116, 11, 239, 60, 346, 282, 271, 296, 76, 161, 281, 160, 171, 291, 25, 18, 187, 240, 165, 316, 208, 364, 36, 374, 74, 81, 0, 246, 265, 20, 252, 7, 139, 381, 85, 189, 255, 375, 53, 78, 195, 306, 93, 156, 130, 96, 380, 3, 206, 63, 169, 295, 313, 280, 194, 30, 103, 4, 38, 14, 173, 249, 373, 163, 262, 342, 180, 184, 378, 300, 199, 347, 261, 40, 219, 158, 198, 241, 290, 9, 154, 6, 252, 259, 263, 28, 361, 128, 301, 29, 137, 204, 192, 54, 318, 254, 350, 73, 341, 310, 114, 326, 178, 13, 304, 244, 339, 270, 365, 72, 276, 196, 212, 126, 99, 191, 102, 193, 183, 80, 201, 332, 200, 243, 69, 322, 89, 129, 327, 379, 97, 9, 154, 95, 252, 122, 223, 157, 27, 189, 334, 62, 26, 202, 35, 15, 226, 106, 138, 351, 293, 77, 48, 59, 147, 278, 197, 1, 292, 215, 335, 333, 303, 150, 175, 94, 352, 359, 50, 234, 362, 367, 98, 162, 124, 285, 33, 305, 45, 127, 251, 353, 52, 110, 111, 265, 252, 172, 263, 70, 369, 286, 266, 207, 237, 31, 144, 61, 325, 308, 46, 336, 224, 307, 58, 297, 153, 269, 47, 51, 302, 134, 218, 203, 79, 88, 86, 188, 8, 148, 209, 44, 109, 236, 43, 174, 141, 123, 349, 82, 358, 179, 264, 298, 52, 9, 246, 265, 22, 372, 337, 105, 369, 64, 5, 230, 319, 213, 344, 366, 71, 355, 113, 149, 145, 245, 279, 376, 84, 168, 242, 315, 65, 225, 37, 170, 348, 10, 363, 320, 57, 331, 217, 317, 101, 68, 247, 250, 167, 340, 214, 17, 152, 133, 115, 166, 87, 211, 154, 20, 22, 372, 185, 357, 324, 128, 220, 370, 343, 159, 354, 91, 41, 274, 356, 49, 182, 12, 287, 83, 257, 233, 222, 56, 66, 181, 258, 267, 176, 120, 107, 284, 42, 67, 329, 238, 132, 256, 143, 228, 90, 227, 371, 100, 2, 21, 232, 314, 92, 92, 154, 6, 7, 7, 223, 357, 70, 112, 104, 146, 231, 135, 277, 190, 273, 260, 377, 39, 345, 118, 221, 140, 216, 16, 311, 268, 312, 108, 309, 275, 23, 253, 32, 151, 136, 19, 55, 288, 248, 125, 229, 75, 119, 289, 117, 330, 34, 155, 368, 383] + 25: [315, 269, 298, 379, 85, 282, 61, 25, 0, 192, 333, 5, 38, 380, 342, 287, 252, 253, 80, 367, 227, 4, 359, 352, 203, 383, 345, 37, 328, 274, 69, 24, 159, 361, 353, 337, 325, 86, 183, 208, 175, 92, 188, 119, 363, 88, 301, 329, 194, 303, 189, 277, 315, 98, 268, 379, 72, 180, 23, 39, 327, 152, 49, 235, 66, 318, 207, 346, 9, 84, 153, 138, 97, 261, 291, 225, 316, 221, 308, 357, 58, 254, 144, 67, 251, 17, 275, 163, 22, 250, 320, 311, 371, 174, 349, 264, 172, 374, 198, 266, 151, 270, 220, 123, 315, 139, 258, 379, 72, 282, 332, 214, 29, 192, 232, 145, 296, 314, 262, 381, 234, 377, 256, 89, 321, 348, 143, 369, 278, 350, 259, 330, 28, 116, 94, 356, 79, 340, 260, 248, 281, 157, 70, 226, 46, 118, 265, 205, 147, 42, 104, 241, 106, 44, 129, 267, 315, 139, 247, 379, 317, 276, 276, 2, 284, 113, 73, 160, 8, 59, 368, 131, 373, 240, 125, 60, 343, 307, 20, 191, 319, 68, 354, 213, 297, 133, 149, 105, 27, 223, 110, 111, 35, 36, 283, 364, 289, 245, 3, 48, 335, 26, 108, 288, 195, 255, 233, 178, 315, 65, 83, 82, 101, 375, 78, 107, 29, 62, 77, 273, 313, 230, 18, 302, 222, 341, 244, 366, 164, 176, 196, 336, 132, 187, 55, 290, 331, 339, 370, 150, 53, 249, 166, 109, 177, 362, 21, 96, 293, 322, 155, 95, 141, 71, 272, 50, 6, 382, 156, 263, 315, 228, 228, 299, 282, 282, 162, 214, 284, 376, 171, 280, 33, 378, 120, 294, 197, 231, 99, 323, 161, 103, 140, 115, 304, 312, 130, 15, 202, 181, 14, 212, 200, 179, 358, 51, 292, 124, 167, 173, 40, 64, 224, 63, 57, 12, 90, 182, 206, 19, 165, 326, 269, 269, 247, 334, 317, 30, 372, 372, 52, 62, 112, 7, 305, 216, 351, 136, 215, 56, 76, 54, 146, 135, 242, 211, 237, 199, 324, 185, 365, 257, 355, 31, 300, 193, 243, 271, 117, 34, 122, 87, 16, 41, 142, 1, 286, 285, 219, 148, 170, 43, 121, 158, 269, 269, 168, 379, 72, 180, 23, 107, 309, 246, 10, 190, 102, 32, 100, 154, 306, 209, 229, 126, 279, 218, 114, 347, 295, 81, 204, 184, 134, 201, 310, 11, 238, 74, 127, 186, 75, 93, 45, 210, 128, 236, 137, 344, 360, 217, 169, 47, 13, 338, 239, 91] + 26: [80, 292, 269, 124, 217, 90, 21, 70, 339, 230, 316, 137, 227, 88, 245, 55, 156, 31, 191, 151, 306, 251, 79, 294, 104, 17, 161, 202, 171, 231, 144, 312, 298, 219, 49, 340, 185, 72, 297, 244, 163, 205, 258, 42, 85, 345, 155, 322, 132, 1, 54, 68, 80, 281, 11, 237, 217, 90, 342, 253, 160, 120, 344, 38, 238, 315, 146, 131, 359, 176, 178, 321, 44, 356, 221, 343, 127, 277, 65, 35, 115, 350, 143, 232, 302, 78, 121, 325, 141, 40, 275, 100, 154, 199, 56, 377, 222, 257, 95, 286, 60, 299, 374, 375, 150, 292, 11, 269, 18, 90, 236, 338, 339, 116, 337, 341, 223, 158, 36, 83, 200, 48, 138, 166, 353, 97, 173, 194, 192, 318, 73, 45, 107, 50, 75, 182, 189, 329, 140, 362, 354, 347, 74, 303, 206, 314, 313, 4, 250, 46, 23, 210, 168, 139, 169, 170, 320, 233, 380, 237, 237, 214, 181, 181, 253, 230, 33, 308, 293, 376, 167, 328, 204, 198, 259, 190, 142, 27, 122, 118, 365, 357, 87, 0, 41, 32, 129, 363, 280, 264, 300, 364, 109, 335, 77, 331, 240, 101, 296, 14, 282, 147, 366, 273, 203, 175, 209, 218, 136, 136, 380, 124, 237, 53, 236, 263, 254, 224, 94, 266, 135, 226, 108, 332, 67, 148, 179, 287, 174, 102, 103, 57, 71, 15, 113, 290, 76, 37, 187, 19, 117, 368, 3, 105, 165, 8, 346, 333, 326, 216, 324, 145, 283, 164, 84, 379, 213, 228, 234, 242, 136, 5, 89, 271, 217, 53, 371, 184, 304, 69, 183, 130, 369, 30, 382, 110, 180, 128, 360, 133, 2, 327, 241, 268, 285, 195, 159, 212, 252, 256, 177, 383, 25, 336, 220, 352, 255, 334, 59, 112, 193, 274, 270, 284, 211, 28, 381, 288, 86, 96, 106, 111, 136, 6, 89, 269, 172, 172, 63, 291, 162, 16, 279, 278, 61, 22, 126, 82, 247, 311, 319, 262, 367, 99, 24, 201, 20, 215, 10, 98, 267, 114, 123, 153, 301, 7, 12, 125, 225, 43, 9, 361, 13, 91, 358, 208, 295, 188, 310, 207, 309, 246, 261, 305, 136, 6, 89, 349, 18, 172, 21, 291, 149, 289, 276, 307, 355, 249, 92, 62, 265, 272, 26, 323, 229, 152, 186, 51, 66, 239, 81, 378, 370, 196, 93, 372, 134, 157, 197, 47, 317, 64, 34, 39, 29, 119, 373, 243, 330, 235, 58, 248, 52, 260, 348, 351] + 27: [34, 177, 225, 286, 12, 350, 168, 204, 30, 349, 227, 354, 179, 336, 230, 290, 380, 77, 308, 78, 140, 356, 241, 162, 368, 326, 215, 268, 124, 67, 9, 120, 353, 3, 361, 105, 14, 152, 282, 264, 330, 16, 195, 87, 1, 148, 210, 40, 17, 100, 270, 338, 234, 145, 4, 225, 263, 90, 71, 73, 357, 80, 341, 202, 369, 55, 182, 260, 324, 72, 115, 32, 201, 187, 161, 155, 54, 10, 136, 274, 46, 43, 20, 307, 101, 217, 35, 79, 139, 259, 346, 186, 302, 68, 273, 64, 165, 311, 298, 371, 93, 212, 146, 163, 234, 177, 58, 286, 12, 90, 168, 269, 30, 49, 358, 377, 213, 183, 335, 340, 76, 196, 156, 23, 322, 337, 15, 261, 271, 6, 172, 327, 267, 106, 347, 249, 29, 13, 275, 33, 306, 154, 313, 253, 352, 158, 62, 166, 194, 314, 355, 57, 209, 235, 176, 220, 81, 104, 279, 286, 31, 312, 265, 252, 197, 349, 341, 24, 188, 237, 251, 21, 242, 339, 294, 89, 297, 191, 42, 88, 309, 175, 114, 218, 276, 239, 147, 318, 250, 367, 107, 130, 344, 208, 185, 65, 41, 364, 86, 189, 226, 11, 19, 134, 243, 236, 240, 244, 103, 103, 4, 225, 263, 374, 164, 73, 320, 128, 227, 354, 207, 143, 48, 323, 381, 99, 343, 325, 141, 317, 133, 254, 126, 379, 174, 127, 375, 248, 299, 66, 121, 0, 205, 151, 132, 113, 359, 74, 150, 372, 94, 198, 331, 138, 255, 206, 378, 247, 262, 310, 103, 301, 59, 333, 61, 312, 265, 252, 315, 119, 366, 131, 348, 112, 228, 85, 149, 258, 83, 332, 284, 109, 98, 370, 91, 246, 304, 360, 45, 18, 2, 199, 365, 95, 123, 116, 144, 184, 319, 293, 110, 289, 257, 108, 52, 160, 221, 92, 36, 180, 285, 342, 103, 37, 117, 12, 70, 90, 233, 69, 26, 80, 211, 181, 222, 190, 75, 303, 383, 232, 102, 278, 373, 193, 27, 7, 316, 214, 291, 122, 287, 129, 44, 28, 321, 84, 153, 159, 96, 231, 50, 8, 157, 142, 283, 345, 56, 245, 277, 169, 51, 200, 39, 53, 103, 37, 219, 351, 61, 312, 300, 69, 135, 376, 211, 292, 224, 97, 272, 178, 266, 216, 238, 82, 25, 118, 305, 328, 125, 329, 5, 296, 362, 111, 171, 38, 47, 167, 288, 256, 280, 281, 223, 22, 229, 382, 334, 192, 173, 170, 295, 363, 63, 203, 60, 137] + 28: [4, 168, 250, 324, 203, 255, 176, 155, 52, 122, 321, 37, 108, 95, 325, 228, 235, 135, 285, 32, 209, 216, 315, 48, 341, 290, 183, 260, 249, 64, 305, 322, 169, 270, 297, 154, 117, 119, 345, 225, 17, 229, 101, 123, 94, 189, 313, 62, 301, 86, 115, 150, 355, 168, 77, 26, 188, 185, 302, 162, 30, 286, 197, 7, 346, 236, 71, 204, 129, 339, 67, 370, 217, 20, 376, 268, 335, 282, 256, 264, 171, 23, 82, 148, 104, 211, 34, 24, 372, 266, 110, 293, 146, 47, 55, 373, 350, 354, 181, 164, 54, 359, 336, 338, 161, 248, 344, 356, 153, 317, 120, 243, 52, 93, 321, 224, 128, 378, 59, 51, 383, 90, 28, 242, 221, 349, 193, 278, 206, 91, 244, 179, 6, 314, 177, 106, 296, 170, 363, 382, 144, 364, 323, 131, 303, 259, 29, 267, 141, 53, 83, 87, 11, 190, 257, 279, 161, 201, 344, 92, 153, 317, 306, 233, 107, 25, 113, 253, 199, 137, 320, 58, 200, 157, 337, 269, 105, 308, 367, 43, 178, 78, 118, 245, 327, 274, 21, 205, 230, 334, 366, 377, 109, 330, 287, 246, 8, 347, 76, 275, 202, 254, 145, 124, 40, 298, 311, 329, 161, 239, 250, 26, 70, 152, 97, 243, 103, 99, 241, 357, 75, 3, 223, 116, 63, 88, 220, 240, 125, 238, 247, 380, 134, 68, 27, 46, 208, 57, 159, 143, 74, 210, 280, 151, 79, 96, 80, 38, 234, 139, 232, 111, 45, 65, 309, 262, 138, 10, 18, 60, 161, 310, 250, 26, 328, 185, 302, 162, 30, 286, 241, 272, 263, 191, 368, 165, 100, 361, 163, 326, 271, 33, 19, 16, 333, 14, 121, 265, 358, 295, 375, 61, 36, 15, 332, 294, 218, 149, 351, 284, 72, 276, 174, 140, 102, 304, 318, 186, 2, 147, 156, 180, 161, 196, 250, 283, 70, 226, 255, 172, 107, 25, 214, 307, 369, 166, 227, 158, 352, 130, 167, 299, 182, 13, 273, 316, 222, 1, 194, 112, 49, 69, 381, 175, 258, 84, 342, 136, 331, 213, 39, 251, 85, 237, 281, 277, 126, 371, 319, 212, 9, 173, 207, 215, 312, 0, 22, 92, 356, 185, 306, 195, 288, 99, 197, 374, 50, 219, 42, 5, 261, 160, 289, 41, 56, 340, 114, 127, 300, 184, 142, 365, 81, 44, 98, 362, 12, 66, 133, 73, 31, 89, 348, 292, 231, 192, 252, 35, 291, 360, 187, 198, 132, 379, 343, 353] + 29: [383, 118, 336, 361, 137, 333, 295, 186, 60, 311, 311, 255, 119, 48, 188, 216, 82, 278, 4, 298, 34, 178, 72, 228, 150, 292, 294, 328, 122, 224, 52, 46, 303, 368, 7, 166, 253, 270, 219, 308, 252, 30, 164, 189, 2, 130, 64, 138, 98, 206, 26, 363, 197, 118, 244, 318, 354, 323, 304, 262, 320, 282, 148, 160, 180, 57, 235, 68, 102, 357, 360, 85, 182, 200, 136, 276, 101, 353, 105, 217, 316, 31, 157, 135, 209, 220, 70, 338, 239, 194, 149, 17, 184, 296, 325, 254, 21, 27, 372, 342, 274, 37, 185, 240, 197, 118, 244, 51, 379, 203, 295, 74, 236, 202, 351, 255, 243, 366, 175, 204, 210, 293, 364, 215, 12, 329, 18, 146, 181, 380, 125, 340, 22, 229, 40, 10, 213, 327, 84, 161, 187, 121, 113, 99, 337, 297, 49, 41, 100, 226, 81, 221, 305, 6, 212, 285, 73, 88, 54, 361, 137, 198, 362, 33, 236, 282, 11, 266, 183, 134, 343, 275, 341, 350, 172, 378, 106, 273, 300, 59, 97, 3, 205, 317, 238, 242, 230, 28, 191, 56, 42, 288, 352, 307, 306, 43, 45, 290, 155, 133, 331, 260, 231, 241, 284, 143, 177, 176, 73, 58, 336, 361, 277, 198, 129, 67, 44, 173, 208, 89, 112, 159, 63, 163, 207, 50, 61, 218, 128, 232, 19, 280, 78, 310, 32, 77, 359, 348, 93, 377, 55, 381, 9, 250, 233, 62, 376, 71, 115, 246, 234, 201, 8, 5, 1, 268, 256, 367, 322, 29, 271, 195, 365, 25, 76, 203, 304, 110, 44, 23, 127, 346, 140, 174, 190, 145, 319, 193, 324, 259, 139, 349, 225, 171, 345, 104, 20, 371, 83, 169, 126, 369, 79, 103, 153, 261, 192, 179, 162, 334, 35, 251, 301, 279, 95, 245, 107, 321, 117, 144, 39, 91, 283, 196, 244, 51, 277, 258, 339, 67, 60, 23, 11, 16, 65, 87, 24, 237, 382, 86, 132, 314, 257, 156, 347, 272, 151, 267, 313, 344, 373, 312, 315, 147, 111, 114, 158, 199, 15, 69, 356, 154, 211, 141, 96, 358, 80, 75, 291, 281, 289, 249, 108, 66, 14, 196, 124, 318, 120, 323, 362, 265, 309, 173, 127, 89, 326, 375, 332, 370, 123, 38, 94, 248, 116, 222, 131, 90, 264, 142, 269, 167, 47, 36, 170, 302, 53, 287, 299, 13, 374, 227, 168, 0, 263, 92, 109, 223, 330, 286, 165, 335, 247, 214, 355, 152] + 30: [140, 262, 208, 213, 187, 366, 78, 225, 168, 373, 321, 10, 147, 316, 224, 240, 100, 58, 141, 136, 344, 369, 343, 261, 54, 383, 197, 129, 241, 73, 50, 145, 194, 183, 347, 161, 203, 327, 368, 122, 169, 330, 102, 338, 171, 93, 251, 301, 36, 40, 47, 88, 140, 262, 208, 333, 187, 243, 59, 35, 351, 351, 234, 223, 212, 250, 159, 89, 1, 193, 151, 204, 18, 285, 200, 96, 167, 162, 163, 286, 349, 181, 282, 360, 342, 377, 371, 326, 381, 5, 370, 355, 16, 113, 126, 313, 367, 182, 252, 305, 94, 101, 118, 123, 140, 208, 208, 239, 13, 65, 245, 214, 143, 206, 62, 45, 283, 362, 216, 317, 336, 339, 210, 320, 296, 160, 23, 91, 72, 114, 359, 111, 375, 248, 297, 25, 166, 202, 263, 32, 57, 125, 130, 291, 173, 120, 198, 0, 149, 228, 4, 70, 307, 128, 134, 146, 133, 184, 208, 76, 137, 75, 196, 35, 295, 373, 188, 28, 17, 315, 97, 12, 77, 256, 309, 382, 170, 356, 325, 104, 38, 103, 20, 294, 211, 115, 249, 318, 311, 95, 350, 90, 348, 165, 364, 81, 105, 232, 112, 235, 42, 26, 119, 227, 379, 334, 345, 380, 158, 266, 239, 239, 358, 65, 65, 378, 257, 156, 289, 64, 41, 302, 43, 74, 152, 275, 299, 217, 292, 98, 157, 278, 179, 335, 51, 310, 55, 48, 44, 374, 186, 121, 220, 139, 69, 85, 209, 314, 178, 352, 207, 265, 63, 61, 191, 238, 8, 21, 22, 31, 37, 266, 208, 272, 86, 53, 245, 52, 143, 2, 11, 246, 164, 303, 340, 341, 253, 192, 9, 195, 274, 258, 174, 353, 231, 142, 376, 66, 293, 7, 92, 288, 337, 107, 267, 264, 83, 155, 82, 127, 233, 255, 300, 80, 328, 229, 27, 131, 331, 185, 189, 218, 37, 266, 239, 239, 358, 242, 245, 214, 24, 62, 361, 144, 33, 205, 312, 221, 177, 6, 215, 226, 281, 298, 365, 108, 84, 150, 14, 172, 180, 244, 284, 79, 110, 230, 247, 279, 99, 287, 273, 153, 19, 190, 106, 56, 372, 34, 87, 176, 354, 219, 237, 259, 254, 124, 239, 222, 29, 53, 135, 52, 46, 156, 289, 346, 68, 271, 49, 138, 357, 71, 236, 277, 260, 67, 60, 132, 329, 306, 319, 280, 109, 332, 290, 175, 154, 276, 324, 304, 116, 323, 117, 3, 15, 308, 270, 199, 39, 148, 30, 201, 363, 268, 269, 322] + 31: [248, 241, 0, 159, 279, 26, 116, 326, 215, 261, 219, 39, 140, 206, 34, 164, 348, 296, 236, 254, 40, 368, 81, 65, 202, 225, 307, 310, 141, 361, 51, 108, 266, 275, 1, 107, 375, 240, 251, 9, 99, 52, 7, 274, 295, 319, 311, 199, 151, 148, 37, 347, 320, 221, 0, 36, 233, 379, 115, 278, 147, 261, 268, 96, 325, 333, 75, 72, 258, 189, 28, 152, 232, 66, 175, 137, 222, 256, 373, 336, 8, 29, 196, 271, 272, 351, 130, 281, 308, 42, 157, 315, 327, 280, 181, 44, 313, 27, 357, 297, 10, 341, 90, 229, 125, 192, 100, 159, 166, 26, 364, 144, 363, 187, 87, 330, 186, 119, 171, 355, 334, 33, 249, 136, 74, 54, 169, 105, 46, 41, 21, 302, 291, 73, 4, 20, 14, 91, 371, 238, 80, 345, 298, 262, 300, 216, 156, 382, 184, 242, 257, 94, 117, 203, 246, 207, 128, 68, 0, 159, 324, 158, 292, 278, 70, 95, 267, 227, 173, 188, 378, 338, 102, 269, 380, 353, 23, 245, 88, 322, 77, 316, 288, 369, 84, 49, 260, 247, 6, 346, 228, 50, 263, 101, 129, 24, 321, 127, 53, 372, 35, 167, 161, 86, 200, 71, 180, 172, 128, 68, 0, 159, 324, 252, 143, 335, 69, 270, 339, 227, 146, 61, 253, 56, 276, 223, 82, 111, 17, 55, 57, 205, 318, 284, 312, 383, 92, 195, 273, 11, 59, 220, 204, 103, 306, 139, 259, 337, 113, 381, 193, 282, 124, 174, 332, 230, 104, 165, 289, 329, 128, 32, 100, 365, 158, 252, 143, 144, 363, 70, 317, 76, 349, 58, 208, 299, 286, 290, 78, 93, 63, 98, 277, 176, 31, 287, 118, 303, 142, 110, 153, 305, 376, 352, 264, 304, 162, 85, 367, 342, 135, 109, 235, 344, 370, 179, 359, 358, 5, 343, 294, 350, 331, 201, 19, 233, 166, 26, 133, 239, 25, 185, 150, 76, 22, 60, 283, 62, 131, 106, 211, 138, 231, 2, 366, 122, 190, 97, 213, 123, 16, 285, 183, 237, 177, 43, 48, 224, 226, 210, 160, 168, 234, 191, 293, 126, 255, 301, 218, 163, 13, 67, 149, 132, 265, 201, 0, 3, 233, 121, 115, 239, 250, 185, 87, 328, 209, 155, 170, 360, 214, 112, 145, 244, 38, 243, 134, 182, 15, 374, 314, 217, 45, 354, 12, 362, 323, 83, 194, 198, 47, 309, 178, 340, 64, 356, 79, 197, 114, 18, 30, 377, 154, 89, 212, 120] + 32: [94, 352, 234, 146, 187, 241, 163, 163, 257, 141, 329, 16, 186, 263, 97, 258, 19, 357, 189, 212, 85, 231, 314, 324, 182, 293, 39, 115, 30, 309, 340, 236, 311, 210, 132, 287, 105, 365, 134, 160, 79, 208, 60, 180, 161, 17, 272, 328, 145, 57, 173, 373, 94, 352, 135, 367, 92, 347, 229, 360, 327, 107, 344, 40, 359, 184, 78, 131, 109, 289, 313, 227, 36, 185, 64, 250, 237, 142, 240, 84, 4, 6, 62, 317, 339, 361, 307, 124, 197, 222, 270, 305, 149, 255, 31, 277, 95, 74, 318, 235, 326, 147, 14, 123, 63, 218, 268, 367, 209, 241, 370, 51, 2, 38, 166, 275, 319, 174, 126, 233, 308, 58, 168, 254, 71, 292, 213, 48, 59, 382, 249, 3, 80, 24, 378, 372, 5, 34, 43, 299, 341, 73, 356, 119, 203, 220, 118, 76, 18, 358, 207, 93, 333, 167, 169, 177, 63, 218, 346, 367, 362, 190, 200, 8, 121, 164, 342, 322, 380, 75, 140, 331, 273, 306, 266, 297, 111, 69, 332, 239, 86, 256, 350, 70, 302, 381, 56, 158, 110, 82, 191, 243, 248, 223, 330, 35, 196, 334, 224, 23, 260, 88, 291, 194, 54, 162, 181, 244, 63, 288, 246, 49, 92, 285, 130, 280, 12, 327, 329, 214, 204, 83, 211, 87, 232, 252, 345, 325, 55, 26, 175, 178, 45, 286, 183, 176, 337, 315, 77, 363, 172, 221, 99, 165, 376, 117, 264, 0, 52, 343, 133, 259, 242, 47, 192, 267, 114, 379, 296, 312, 63, 148, 33, 367, 323, 190, 21, 51, 122, 107, 41, 322, 348, 202, 279, 156, 228, 7, 171, 91, 104, 101, 25, 269, 230, 32, 53, 371, 276, 10, 22, 193, 144, 136, 102, 125, 304, 20, 153, 44, 355, 377, 364, 46, 321, 98, 295, 11, 15, 300, 245, 251, 63, 148, 33, 367, 179, 347, 310, 122, 274, 38, 166, 61, 89, 96, 151, 143, 113, 120, 301, 72, 303, 50, 206, 157, 150, 336, 351, 81, 205, 366, 37, 349, 320, 282, 284, 103, 195, 128, 261, 201, 138, 188, 1, 137, 368, 112, 159, 226, 27, 68, 383, 375, 63, 352, 155, 108, 106, 285, 271, 66, 219, 164, 344, 275, 65, 225, 283, 116, 335, 294, 369, 298, 354, 199, 28, 281, 253, 127, 29, 353, 129, 247, 198, 139, 154, 42, 9, 152, 238, 13, 374, 67, 170, 278, 100, 216, 215, 217, 290, 338, 90, 316, 262, 265] + 33: [86, 63, 280, 191, 362, 36, 325, 343, 183, 154, 307, 147, 84, 352, 335, 233, 33, 210, 40, 288, 126, 354, 90, 138, 276, 375, 157, 370, 152, 340, 345, 330, 281, 26, 101, 327, 10, 182, 286, 168, 119, 120, 353, 199, 97, 313, 344, 203, 357, 129, 83, 333, 167, 260, 339, 38, 158, 380, 319, 96, 39, 299, 73, 241, 146, 206, 361, 17, 35, 117, 237, 268, 235, 351, 187, 4, 123, 80, 156, 360, 186, 364, 368, 71, 350, 103, 271, 41, 221, 23, 181, 225, 78, 62, 178, 337, 275, 374, 172, 294, 263, 159, 161, 293, 91, 145, 177, 362, 323, 229, 319, 81, 258, 283, 68, 125, 89, 5, 48, 204, 30, 151, 131, 222, 61, 202, 329, 317, 45, 22, 197, 209, 369, 60, 249, 215, 24, 366, 110, 29, 135, 298, 11, 243, 310, 245, 59, 341, 171, 274, 296, 302, 77, 272, 102, 287, 91, 320, 177, 377, 121, 64, 148, 343, 301, 283, 267, 94, 322, 37, 270, 54, 1, 193, 356, 9, 8, 211, 173, 109, 347, 262, 19, 163, 180, 213, 137, 371, 113, 52, 285, 67, 136, 6, 74, 381, 224, 257, 175, 242, 195, 269, 44, 304, 282, 264, 236, 372, 91, 260, 259, 38, 64, 380, 148, 107, 251, 301, 267, 94, 201, 266, 255, 376, 25, 338, 334, 217, 116, 290, 355, 230, 198, 124, 50, 104, 212, 13, 15, 43, 66, 65, 112, 248, 194, 316, 207, 132, 332, 312, 326, 315, 214, 359, 99, 2, 143, 0, 32, 69, 328, 75, 339, 38, 31, 36, 231, 115, 115, 154, 307, 348, 72, 139, 190, 21, 87, 150, 308, 311, 18, 232, 205, 174, 216, 196, 382, 373, 114, 58, 49, 106, 349, 166, 278, 219, 252, 238, 300, 27, 184, 365, 244, 363, 306, 7, 309, 46, 100, 189, 12, 220, 51, 176, 331, 38, 153, 36, 379, 297, 251, 82, 342, 314, 89, 164, 169, 273, 160, 227, 192, 133, 14, 303, 93, 254, 383, 162, 256, 200, 88, 234, 170, 3, 336, 70, 53, 261, 128, 20, 111, 76, 291, 98, 134, 378, 250, 253, 57, 284, 223, 239, 95, 142, 240, 140, 339, 38, 42, 229, 319, 81, 258, 299, 179, 358, 108, 226, 127, 141, 218, 155, 208, 289, 305, 144, 149, 16, 79, 246, 55, 367, 279, 105, 122, 292, 28, 56, 324, 295, 318, 346, 130, 85, 188, 118, 247, 165, 92, 228, 321, 47, 185, 265, 34, 277] + 34: [372, 237, 259, 179, 135, 306, 311, 218, 3, 190, 148, 360, 246, 271, 288, 4, 244, 21, 53, 210, 368, 187, 143, 317, 150, 73, 359, 50, 301, 113, 157, 358, 221, 45, 266, 268, 161, 274, 307, 49, 57, 250, 9, 171, 43, 156, 252, 234, 121, 177, 279, 202, 382, 47, 151, 179, 330, 82, 239, 248, 3, 196, 167, 40, 59, 79, 335, 77, 95, 124, 290, 302, 256, 199, 206, 195, 227, 209, 204, 5, 230, 180, 294, 371, 46, 272, 98, 314, 282, 101, 353, 117, 176, 17, 155, 257, 120, 96, 182, 276, 235, 211, 341, 363, 366, 69, 299, 18, 16, 249, 15, 304, 44, 190, 61, 194, 115, 97, 222, 168, 87, 91, 184, 81, 183, 197, 280, 67, 181, 214, 333, 247, 361, 63, 322, 62, 56, 51, 296, 374, 305, 118, 34, 255, 242, 285, 141, 20, 71, 130, 188, 320, 309, 213, 381, 365, 366, 207, 259, 126, 16, 249, 15, 248, 44, 196, 203, 146, 220, 238, 11, 41, 201, 200, 107, 12, 313, 37, 60, 328, 241, 48, 364, 35, 308, 164, 323, 298, 149, 186, 64, 192, 111, 226, 100, 380, 370, 55, 342, 22, 281, 216, 349, 369, 32, 273, 119, 122, 108, 29, 179, 179, 74, 334, 340, 340, 25, 321, 31, 258, 379, 83, 78, 6, 208, 373, 58, 84, 175, 137, 172, 336, 30, 231, 170, 367, 215, 140, 318, 283, 139, 383, 152, 13, 269, 265, 93, 319, 278, 39, 297, 1, 128, 163, 338, 350, 375, 224, 14, 76, 108, 337, 299, 312, 16, 249, 158, 340, 185, 243, 61, 178, 347, 8, 92, 267, 225, 110, 329, 345, 123, 24, 251, 245, 346, 72, 287, 263, 232, 352, 42, 127, 105, 147, 68, 219, 228, 262, 331, 165, 205, 26, 253, 90, 351, 325, 378, 376, 66, 343, 132, 160, 108, 337, 357, 332, 330, 306, 306, 218, 145, 240, 291, 138, 99, 2, 191, 300, 162, 153, 144, 326, 327, 136, 217, 70, 236, 27, 169, 116, 348, 154, 114, 10, 23, 261, 85, 275, 36, 86, 286, 109, 131, 260, 104, 112, 54, 292, 106, 303, 129, 198, 316, 229, 108, 69, 324, 126, 16, 334, 15, 354, 223, 293, 362, 315, 284, 295, 254, 65, 125, 134, 356, 174, 159, 166, 133, 33, 270, 0, 102, 89, 80, 103, 19, 277, 142, 264, 88, 289, 233, 310, 75, 28, 52, 173, 339, 212, 355, 344, 38, 94, 7, 377, 189, 193] + 35: [48, 28, 44, 130, 218, 211, 367, 314, 53, 342, 276, 113, 221, 224, 114, 164, 14, 126, 102, 267, 325, 158, 192, 372, 313, 300, 302, 350, 366, 110, 82, 216, 269, 142, 175, 338, 176, 365, 29, 316, 112, 262, 263, 107, 197, 371, 252, 39, 173, 199, 304, 177, 309, 260, 62, 319, 218, 131, 367, 295, 257, 86, 203, 322, 340, 15, 364, 335, 361, 329, 163, 187, 272, 61, 5, 108, 42, 95, 11, 287, 227, 180, 382, 273, 7, 70, 315, 378, 274, 87, 189, 22, 94, 250, 97, 139, 69, 310, 258, 119, 251, 283, 146, 40, 363, 169, 205, 319, 303, 56, 345, 286, 138, 4, 38, 3, 231, 147, 46, 326, 191, 159, 373, 168, 296, 36, 93, 294, 243, 122, 256, 57, 277, 106, 264, 24, 174, 301, 217, 84, 79, 332, 253, 149, 232, 354, 167, 10, 178, 33, 219, 280, 71, 124, 284, 293, 363, 169, 205, 319, 379, 131, 368, 133, 141, 86, 203, 322, 206, 186, 311, 194, 331, 27, 98, 321, 81, 308, 298, 215, 209, 201, 323, 92, 109, 91, 369, 290, 184, 127, 73, 25, 103, 248, 207, 377, 26, 155, 255, 183, 327, 225, 242, 226, 341, 59, 291, 228, 80, 45, 240, 58, 299, 297, 295, 318, 53, 342, 276, 289, 340, 41, 352, 241, 266, 54, 312, 140, 356, 171, 282, 334, 125, 152, 96, 261, 20, 348, 339, 330, 179, 380, 43, 344, 65, 105, 161, 245, 249, 49, 288, 148, 83, 333, 336, 210, 76, 236, 200, 60, 75, 45, 246, 319, 299, 297, 345, 68, 324, 21, 72, 281, 154, 239, 265, 307, 52, 222, 275, 160, 128, 9, 268, 190, 320, 12, 16, 235, 8, 214, 117, 34, 185, 17, 66, 88, 317, 151, 259, 115, 181, 23, 77, 101, 351, 233, 188, 51, 229, 100, 85, 359, 75, 45, 182, 144, 254, 1, 345, 133, 47, 129, 31, 281, 346, 78, 375, 50, 362, 37, 157, 357, 18, 230, 104, 213, 143, 90, 32, 74, 145, 202, 347, 383, 99, 19, 285, 6, 35, 292, 156, 370, 306, 89, 165, 150, 270, 223, 208, 247, 212, 2, 381, 343, 353, 220, 28, 237, 1, 1, 67, 68, 55, 153, 72, 113, 116, 355, 349, 234, 162, 136, 134, 360, 244, 132, 328, 30, 137, 337, 170, 172, 111, 305, 204, 166, 376, 196, 279, 374, 198, 63, 135, 278, 121, 195, 358, 238, 271, 123, 193, 0, 120, 64, 118, 13] + 36: [53, 340, 184, 232, 232, 371, 325, 260, 178, 320, 380, 107, 150, 205, 125, 42, 201, 334, 219, 191, 297, 105, 172, 339, 153, 117, 11, 46, 255, 30, 212, 335, 148, 263, 283, 48, 209, 295, 383, 248, 35, 75, 226, 235, 151, 96, 353, 218, 234, 376, 64, 123, 53, 176, 184, 304, 185, 10, 368, 101, 296, 193, 56, 99, 291, 152, 31, 45, 175, 111, 94, 55, 13, 121, 3, 289, 354, 271, 195, 224, 323, 264, 76, 206, 88, 149, 208, 29, 83, 247, 381, 77, 316, 14, 110, 378, 303, 285, 136, 106, 221, 268, 25, 37, 131, 340, 341, 348, 266, 62, 311, 272, 210, 274, 12, 90, 97, 5, 54, 65, 162, 112, 194, 293, 63, 68, 50, 155, 216, 366, 343, 102, 217, 229, 81, 36, 358, 220, 318, 239, 329, 189, 236, 173, 60, 120, 324, 126, 161, 27, 124, 181, 214, 145, 290, 360, 192, 382, 184, 304, 266, 10, 328, 311, 98, 369, 12, 373, 352, 79, 314, 261, 321, 108, 288, 139, 361, 245, 300, 276, 72, 147, 228, 364, 211, 71, 257, 17, 165, 267, 51, 18, 281, 158, 265, 199, 130, 21, 350, 8, 7, 203, 89, 286, 317, 43, 279, 273, 252, 74, 341, 232, 238, 59, 371, 230, 196, 193, 225, 301, 207, 308, 85, 93, 240, 246, 231, 49, 167, 91, 164, 15, 168, 280, 237, 202, 180, 187, 143, 379, 188, 23, 159, 135, 363, 6, 259, 249, 357, 347, 115, 355, 154, 1, 227, 222, 377, 186, 292, 375, 294, 370, 144, 57, 232, 2, 325, 299, 242, 241, 225, 107, 73, 133, 233, 32, 309, 262, 9, 345, 122, 103, 322, 342, 142, 140, 41, 277, 134, 39, 52, 344, 129, 119, 365, 146, 70, 190, 169, 251, 256, 118, 61, 253, 278, 128, 82, 269, 182, 114, 127, 170, 22, 22, 84, 250, 238, 359, 368, 272, 178, 326, 372, 223, 275, 47, 287, 156, 33, 67, 166, 349, 138, 284, 69, 183, 346, 174, 100, 116, 305, 200, 38, 28, 351, 78, 243, 58, 44, 109, 338, 19, 337, 307, 254, 87, 26, 330, 141, 4, 367, 270, 313, 302, 22, 197, 144, 57, 232, 371, 368, 272, 258, 40, 372, 223, 374, 92, 332, 331, 333, 362, 319, 282, 132, 95, 298, 20, 171, 179, 66, 177, 310, 0, 104, 157, 244, 80, 163, 356, 24, 204, 315, 198, 215, 137, 312, 213, 86, 34, 327, 306, 160, 113, 16, 336] + 37: [109, 40, 294, 218, 242, 311, 293, 200, 289, 79, 349, 371, 156, 344, 7, 22, 117, 163, 133, 150, 208, 122, 74, 149, 73, 141, 134, 29, 167, 83, 121, 312, 97, 38, 354, 154, 160, 215, 286, 78, 259, 191, 106, 107, 326, 54, 369, 246, 358, 251, 375, 239, 109, 40, 294, 273, 58, 151, 125, 222, 48, 342, 349, 113, 361, 302, 253, 182, 194, 181, 277, 315, 19, 228, 64, 144, 338, 39, 264, 34, 49, 55, 337, 373, 212, 317, 5, 351, 256, 17, 60, 137, 283, 198, 146, 237, 28, 261, 186, 381, 25, 238, 120, 112, 188, 40, 40, 218, 348, 309, 293, 363, 366, 72, 162, 382, 59, 114, 353, 327, 328, 131, 257, 322, 85, 76, 196, 323, 82, 92, 278, 376, 136, 224, 262, 98, 195, 279, 210, 298, 110, 368, 350, 183, 310, 142, 357, 231, 155, 300, 307, 12, 282, 67, 263, 383, 240, 66, 294, 218, 265, 148, 178, 199, 190, 290, 292, 174, 16, 140, 252, 65, 147, 32, 330, 336, 229, 104, 173, 365, 61, 30, 111, 3, 87, 334, 37, 255, 94, 284, 267, 26, 138, 266, 332, 291, 35, 374, 225, 70, 202, 197, 89, 90, 68, 84, 31, 288, 268, 301, 218, 379, 243, 213, 205, 366, 362, 79, 193, 77, 223, 129, 360, 14, 204, 118, 192, 359, 316, 258, 176, 6, 296, 161, 333, 15, 100, 247, 128, 321, 271, 280, 4, 116, 124, 91, 260, 378, 101, 81, 96, 324, 177, 11, 220, 275, 232, 99, 189, 352, 268, 71, 218, 123, 158, 148, 306, 46, 341, 44, 234, 340, 86, 171, 9, 88, 233, 93, 345, 221, 370, 51, 27, 203, 217, 304, 103, 201, 335, 331, 269, 380, 102, 24, 42, 187, 254, 10, 303, 115, 295, 166, 62, 57, 281, 45, 219, 211, 346, 209, 56, 236, 268, 364, 218, 379, 58, 314, 52, 200, 372, 305, 162, 113, 0, 47, 185, 165, 235, 343, 18, 157, 127, 287, 1, 168, 41, 36, 135, 329, 126, 206, 207, 347, 313, 274, 325, 249, 299, 152, 216, 169, 297, 63, 248, 130, 119, 180, 250, 367, 50, 377, 8, 75, 319, 301, 40, 218, 158, 148, 293, 222, 272, 179, 95, 340, 0, 355, 245, 318, 139, 153, 164, 145, 320, 21, 270, 43, 230, 143, 108, 285, 356, 69, 80, 244, 241, 33, 105, 172, 13, 53, 170, 175, 339, 308, 184, 214, 227, 20, 159, 2, 23, 276, 226, 132] + 38: [282, 301, 178, 266, 203, 269, 51, 86, 126, 154, 333, 62, 217, 145, 317, 253, 108, 233, 90, 107, 239, 208, 359, 36, 349, 367, 7, 229, 31, 190, 134, 258, 131, 23, 296, 74, 273, 252, 47, 35, 183, 12, 192, 364, 314, 153, 218, 110, 370, 159, 339, 251, 10, 301, 369, 266, 118, 201, 292, 30, 383, 114, 333, 62, 92, 27, 279, 77, 338, 329, 219, 75, 69, 59, 262, 45, 184, 165, 298, 276, 228, 356, 376, 305, 139, 173, 302, 129, 261, 25, 295, 283, 247, 70, 316, 42, 259, 116, 281, 196, 133, 346, 191, 325, 4, 301, 50, 266, 203, 269, 372, 366, 226, 114, 89, 49, 171, 373, 135, 257, 331, 286, 99, 352, 73, 321, 105, 307, 264, 156, 48, 109, 117, 170, 26, 181, 78, 250, 334, 371, 104, 82, 61, 374, 202, 299, 240, 24, 163, 128, 22, 198, 270, 64, 161, 60, 158, 93, 378, 221, 203, 280, 284, 354, 189, 154, 309, 293, 244, 381, 212, 119, 53, 288, 355, 275, 85, 9, 193, 211, 256, 308, 111, 151, 5, 241, 180, 353, 235, 200, 238, 172, 382, 204, 311, 205, 290, 363, 320, 112, 18, 207, 315, 127, 162, 304, 38, 210, 158, 301, 378, 221, 94, 124, 51, 30, 126, 115, 89, 177, 379, 0, 169, 300, 97, 137, 341, 14, 324, 72, 224, 58, 255, 96, 368, 246, 289, 84, 197, 272, 152, 310, 360, 43, 56, 215, 44, 13, 71, 236, 91, 326, 175, 20, 377, 1, 140, 318, 101, 185, 187, 223, 301, 245, 66, 201, 8, 263, 274, 248, 220, 32, 130, 166, 206, 54, 199, 16, 225, 146, 271, 95, 362, 68, 57, 216, 37, 342, 265, 28, 76, 33, 15, 2, 79, 313, 11, 100, 297, 351, 102, 343, 254, 65, 350, 167, 303, 336, 243, 121, 340, 380, 6, 6, 213, 345, 123, 280, 372, 366, 189, 277, 277, 39, 63, 278, 344, 268, 319, 327, 125, 147, 227, 21, 150, 144, 88, 136, 34, 143, 148, 306, 41, 149, 40, 81, 214, 122, 120, 375, 294, 186, 55, 3, 164, 157, 232, 209, 365, 155, 249, 312, 52, 46, 6, 6, 287, 348, 332, 124, 284, 285, 322, 383, 194, 230, 19, 17, 328, 267, 234, 361, 106, 222, 67, 29, 358, 87, 323, 142, 179, 188, 141, 260, 242, 174, 103, 337, 182, 195, 347, 231, 80, 168, 138, 357, 113, 335, 291, 132, 160, 83, 330, 98, 176, 237] + 39: [43, 268, 258, 310, 223, 286, 56, 139, 360, 325, 271, 85, 270, 277, 121, 319, 365, 59, 227, 53, 54, 225, 336, 34, 126, 188, 364, 226, 182, 61, 167, 357, 6, 309, 183, 327, 291, 217, 287, 168, 108, 206, 123, 92, 210, 147, 124, 224, 356, 90, 334, 209, 100, 180, 110, 104, 372, 368, 315, 312, 93, 313, 328, 86, 145, 374, 295, 178, 208, 381, 298, 352, 191, 28, 138, 47, 358, 150, 317, 249, 302, 106, 45, 203, 164, 30, 240, 181, 166, 192, 335, 222, 157, 231, 142, 359, 233, 158, 125, 316, 261, 115, 89, 143, 176, 35, 116, 234, 236, 156, 56, 218, 38, 279, 272, 85, 297, 324, 250, 341, 51, 81, 84, 259, 140, 251, 190, 378, 353, 281, 307, 161, 134, 311, 241, 193, 4, 16, 363, 60, 120, 70, 288, 189, 266, 246, 37, 197, 255, 351, 318, 237, 332, 148, 21, 361, 247, 282, 258, 104, 372, 25, 56, 13, 38, 242, 379, 379, 296, 204, 141, 15, 77, 146, 235, 8, 232, 31, 263, 260, 152, 366, 322, 301, 33, 355, 320, 97, 128, 79, 377, 214, 376, 27, 196, 303, 305, 347, 64, 10, 342, 349, 256, 73, 154, 186, 304, 22, 247, 96, 66, 310, 372, 236, 315, 65, 132, 314, 119, 2, 329, 26, 40, 369, 3, 200, 273, 383, 340, 323, 101, 331, 177, 135, 144, 345, 212, 72, 289, 354, 338, 130, 68, 194, 216, 245, 41, 117, 78, 160, 18, 229, 339, 136, 67, 262, 184, 11, 9, 149, 247, 280, 66, 82, 310, 286, 315, 312, 36, 36, 267, 292, 362, 333, 98, 76, 163, 290, 375, 42, 127, 112, 278, 137, 62, 199, 63, 75, 14, 293, 185, 173, 275, 195, 220, 265, 171, 343, 103, 107, 55, 83, 252, 39, 219, 129, 32, 105, 269, 29, 321, 44, 99, 282, 66, 310, 348, 1, 113, 65, 93, 313, 187, 382, 174, 172, 330, 24, 49, 87, 202, 274, 248, 122, 239, 74, 50, 276, 179, 213, 228, 94, 95, 285, 221, 243, 306, 169, 91, 58, 19, 111, 337, 175, 211, 238, 284, 350, 294, 308, 344, 201, 52, 198, 282, 282, 264, 104, 236, 153, 151, 215, 132, 314, 155, 244, 80, 71, 17, 300, 7, 23, 118, 170, 48, 299, 69, 131, 367, 371, 5, 162, 373, 102, 88, 133, 230, 254, 114, 207, 283, 109, 346, 159, 46, 20, 0, 253, 326, 370, 205, 12, 57, 380, 165, 257] + 40: [31, 357, 15, 360, 68, 328, 378, 101, 21, 26, 107, 226, 81, 208, 90, 1, 303, 321, 277, 185, 334, 199, 227, 239, 41, 82, 43, 355, 155, 324, 282, 323, 18, 233, 139, 375, 121, 256, 23, 310, 174, 257, 218, 111, 352, 87, 176, 172, 20, 146, 144, 300, 31, 207, 15, 349, 70, 204, 175, 98, 179, 179, 156, 88, 64, 258, 17, 159, 129, 336, 322, 289, 200, 38, 262, 269, 365, 138, 65, 151, 192, 339, 177, 281, 293, 345, 73, 238, 186, 288, 125, 383, 182, 66, 106, 119, 354, 109, 381, 272, 127, 240, 69, 194, 31, 207, 15, 360, 213, 328, 378, 62, 283, 160, 128, 8, 332, 235, 100, 347, 143, 10, 209, 11, 193, 205, 302, 307, 309, 150, 25, 110, 29, 264, 206, 330, 131, 301, 76, 246, 133, 141, 124, 49, 368, 295, 34, 231, 267, 39, 157, 298, 42, 72, 249, 105, 31, 340, 260, 349, 142, 358, 55, 75, 337, 52, 346, 164, 232, 198, 351, 311, 296, 168, 63, 253, 278, 188, 148, 137, 53, 95, 299, 2, 7, 93, 333, 48, 304, 366, 85, 36, 116, 361, 184, 13, 16, 203, 134, 173, 154, 342, 255, 320, 191, 312, 353, 214, 31, 280, 183, 286, 68, 359, 175, 190, 265, 297, 215, 226, 372, 58, 59, 275, 284, 326, 274, 27, 54, 12, 370, 44, 187, 364, 377, 306, 338, 327, 329, 237, 19, 47, 261, 147, 270, 276, 0, 268, 117, 77, 35, 285, 96, 152, 252, 112, 126, 335, 376, 149, 31, 319, 319, 250, 142, 71, 378, 373, 108, 33, 202, 153, 254, 3, 28, 210, 363, 219, 162, 166, 30, 221, 382, 56, 79, 263, 290, 103, 294, 167, 371, 195, 37, 314, 241, 6, 78, 212, 135, 50, 374, 163, 242, 22, 114, 315, 223, 24, 220, 344, 5, 248, 132, 317, 183, 169, 181, 358, 67, 190, 21, 33, 279, 161, 369, 224, 228, 91, 89, 136, 341, 211, 145, 216, 229, 46, 99, 178, 189, 130, 120, 287, 305, 243, 350, 51, 222, 118, 115, 14, 102, 266, 40, 113, 104, 325, 313, 380, 245, 86, 83, 225, 343, 123, 379, 317, 319, 250, 68, 71, 175, 367, 108, 247, 346, 158, 74, 251, 97, 9, 362, 271, 80, 61, 308, 236, 196, 259, 316, 57, 244, 94, 165, 318, 356, 171, 273, 60, 201, 291, 122, 84, 348, 32, 140, 230, 217, 180, 234, 4, 45, 292, 197, 331, 170, 92] + 41: [205, 87, 103, 114, 355, 350, 184, 30, 273, 366, 124, 215, 34, 69, 347, 306, 348, 127, 140, 353, 26, 32, 226, 83, 294, 138, 250, 268, 299, 284, 91, 94, 74, 221, 99, 75, 78, 77, 297, 149, 316, 209, 14, 245, 62, 2, 162, 346, 169, 135, 40, 132, 305, 283, 114, 199, 376, 60, 374, 190, 366, 223, 119, 285, 79, 24, 311, 95, 326, 35, 239, 36, 174, 345, 241, 46, 322, 344, 339, 51, 131, 338, 246, 5, 371, 354, 232, 33, 143, 287, 320, 158, 98, 195, 231, 17, 139, 157, 312, 361, 193, 164, 178, 152, 122, 283, 196, 199, 142, 307, 184, 57, 197, 50, 119, 92, 136, 161, 206, 82, 383, 4, 202, 218, 252, 56, 313, 254, 271, 115, 144, 328, 151, 295, 81, 317, 61, 128, 123, 292, 1, 191, 259, 48, 257, 204, 375, 373, 121, 343, 106, 341, 281, 228, 22, 85, 321, 298, 335, 142, 170, 355, 374, 255, 168, 368, 159, 97, 364, 358, 302, 253, 80, 351, 25, 23, 134, 116, 282, 55, 237, 289, 224, 266, 176, 186, 19, 213, 244, 146, 59, 272, 352, 0, 141, 68, 73, 175, 171, 370, 110, 323, 377, 109, 203, 333, 31, 120, 321, 298, 378, 71, 170, 350, 90, 30, 369, 365, 159, 76, 7, 314, 262, 337, 104, 53, 286, 243, 308, 64, 225, 256, 235, 117, 242, 8, 63, 290, 277, 261, 220, 214, 37, 342, 39, 65, 72, 332, 148, 137, 29, 274, 267, 319, 270, 16, 58, 118, 227, 309, 185, 269, 378, 11, 251, 60, 374, 86, 369, 200, 327, 96, 187, 360, 334, 44, 318, 222, 264, 163, 336, 70, 54, 276, 258, 167, 112, 210, 363, 153, 18, 43, 173, 89, 41, 188, 236, 381, 234, 105, 310, 28, 265, 329, 331, 88, 177, 372, 291, 52, 263, 357, 49, 49, 378, 199, 27, 60, 212, 190, 84, 129, 275, 293, 172, 279, 12, 108, 45, 3, 219, 194, 349, 102, 125, 93, 229, 340, 133, 288, 66, 107, 6, 130, 315, 303, 359, 154, 100, 165, 13, 248, 113, 155, 67, 20, 183, 145, 189, 362, 382, 233, 247, 216, 49, 269, 367, 166, 170, 350, 249, 150, 197, 129, 275, 215, 111, 21, 280, 198, 9, 324, 42, 126, 217, 301, 379, 260, 156, 325, 330, 380, 101, 181, 208, 160, 10, 192, 15, 300, 179, 180, 207, 38, 147, 182, 238, 47, 201, 240, 211, 356, 304, 230, 296, 278] + 42: [2, 123, 123, 152, 119, 357, 211, 93, 63, 256, 333, 195, 108, 59, 319, 277, 141, 30, 236, 287, 87, 103, 38, 202, 223, 297, 241, 8, 311, 89, 244, 99, 313, 16, 355, 112, 26, 206, 267, 104, 312, 136, 330, 153, 35, 249, 182, 142, 13, 361, 294, 25, 188, 123, 123, 152, 31, 268, 155, 247, 282, 158, 41, 168, 377, 48, 122, 379, 248, 4, 169, 214, 105, 147, 92, 316, 257, 149, 245, 137, 307, 246, 49, 97, 298, 204, 156, 171, 194, 318, 90, 250, 124, 186, 207, 85, 115, 100, 44, 14, 91, 252, 126, 324, 210, 123, 123, 152, 31, 345, 176, 279, 151, 256, 333, 162, 55, 208, 342, 351, 275, 341, 135, 28, 173, 338, 314, 280, 76, 160, 80, 101, 371, 323, 130, 226, 258, 196, 170, 364, 120, 261, 98, 368, 3, 23, 309, 88, 353, 110, 54, 78, 240, 273, 27, 71, 154, 42, 266, 295, 117, 305, 155, 116, 325, 264, 37, 222, 262, 209, 281, 373, 358, 221, 259, 344, 329, 356, 322, 366, 39, 227, 113, 19, 237, 238, 109, 157, 7, 5, 125, 269, 56, 272, 197, 10, 285, 198, 367, 134, 190, 11, 81, 177, 365, 289, 121, 95, 375, 50, 266, 148, 117, 303, 176, 350, 215, 111, 37, 243, 306, 6, 21, 348, 274, 320, 239, 17, 165, 326, 77, 234, 52, 381, 18, 132, 231, 9, 283, 299, 58, 251, 378, 346, 293, 229, 102, 363, 185, 75, 144, 133, 12, 47, 107, 317, 15, 24, 352, 339, 359, 217, 94, 114, 117, 268, 172, 218, 93, 150, 138, 349, 200, 255, 327, 191, 328, 178, 140, 187, 161, 216, 164, 143, 362, 192, 60, 51, 62, 64, 315, 310, 205, 291, 53, 300, 343, 284, 82, 145, 296, 304, 174, 22, 69, 57, 331, 382, 86, 40, 46, 254, 359, 217, 94, 228, 117, 106, 184, 218, 325, 111, 138, 337, 220, 203, 189, 265, 131, 354, 286, 230, 167, 308, 335, 33, 118, 380, 340, 146, 372, 225, 139, 183, 61, 336, 270, 233, 213, 290, 321, 360, 292, 288, 383, 96, 253, 79, 83, 271, 66, 74, 193, 369, 20, 1, 123, 152, 31, 345, 172, 212, 72, 158, 179, 374, 129, 263, 127, 235, 370, 29, 224, 67, 201, 36, 181, 45, 166, 65, 32, 301, 180, 219, 260, 232, 34, 332, 68, 278, 84, 175, 199, 73, 128, 334, 302, 347, 276, 70, 159, 43, 0, 163, 242, 376] + 43: [164, 161, 171, 375, 375, 197, 359, 37, 349, 190, 366, 131, 326, 377, 29, 227, 108, 121, 379, 73, 232, 48, 286, 299, 367, 61, 155, 345, 60, 50, 45, 284, 146, 343, 159, 304, 267, 76, 151, 233, 290, 240, 311, 124, 106, 236, 208, 333, 33, 310, 192, 95, 369, 221, 171, 375, 278, 129, 35, 216, 14, 97, 100, 314, 57, 153, 342, 360, 338, 136, 374, 222, 378, 88, 234, 123, 5, 79, 28, 107, 49, 165, 143, 46, 263, 370, 175, 72, 279, 87, 158, 268, 300, 110, 260, 230, 62, 249, 270, 58, 289, 361, 206, 214, 27, 221, 105, 225, 92, 142, 66, 59, 20, 75, 297, 43, 23, 55, 127, 264, 305, 147, 4, 272, 154, 275, 83, 140, 34, 130, 241, 148, 185, 47, 217, 74, 125, 174, 116, 207, 356, 163, 381, 119, 330, 258, 51, 276, 0, 137, 352, 96, 212, 302, 169, 348, 344, 6, 105, 225, 372, 335, 359, 324, 13, 190, 297, 43, 22, 81, 111, 318, 224, 280, 85, 77, 282, 204, 339, 26, 160, 84, 196, 145, 99, 183, 253, 167, 128, 325, 215, 11, 294, 132, 287, 358, 355, 187, 285, 248, 172, 281, 340, 139, 244, 184, 373, 245, 315, 168, 177, 347, 92, 17, 66, 98, 283, 75, 94, 298, 363, 337, 301, 323, 307, 202, 368, 150, 52, 346, 306, 135, 296, 257, 237, 12, 242, 180, 16, 70, 309, 288, 170, 122, 8, 65, 247, 209, 188, 336, 201, 53, 317, 303, 40, 15, 319, 376, 182, 382, 315, 168, 332, 347, 218, 17, 66, 98, 283, 252, 30, 351, 235, 93, 223, 178, 1, 78, 89, 32, 205, 327, 86, 115, 261, 157, 194, 63, 24, 113, 56, 120, 101, 256, 295, 329, 144, 353, 118, 173, 265, 316, 162, 114, 41, 228, 42, 133, 350, 54, 189, 354, 262, 168, 156, 308, 372, 335, 98, 324, 39, 239, 30, 351, 152, 291, 138, 203, 274, 269, 255, 2, 238, 246, 277, 271, 149, 365, 10, 243, 293, 38, 210, 251, 9, 80, 31, 292, 259, 126, 19, 134, 211, 357, 341, 69, 7, 141, 383, 104, 25, 166, 331, 44, 82, 181, 156, 225, 68, 371, 112, 364, 14, 254, 103, 298, 23, 186, 322, 313, 198, 321, 250, 191, 90, 380, 219, 200, 36, 67, 334, 273, 102, 229, 266, 213, 220, 226, 18, 64, 71, 117, 193, 109, 320, 179, 312, 199, 328, 21, 362, 91, 176, 231, 195, 3] + 44: [362, 17, 213, 178, 199, 379, 101, 101, 190, 43, 277, 279, 93, 86, 212, 329, 228, 275, 276, 121, 184, 296, 163, 287, 66, 164, 87, 221, 342, 126, 147, 165, 270, 309, 32, 305, 230, 9, 367, 114, 255, 216, 22, 119, 246, 103, 259, 361, 197, 310, 129, 206, 362, 258, 213, 272, 191, 379, 348, 284, 227, 82, 170, 135, 288, 187, 182, 306, 144, 125, 324, 91, 6, 290, 150, 56, 352, 338, 359, 44, 36, 136, 92, 74, 149, 231, 341, 179, 235, 345, 314, 7, 124, 375, 195, 151, 363, 302, 47, 73, 316, 281, 295, 374, 274, 167, 77, 191, 199, 317, 317, 223, 188, 257, 346, 311, 236, 280, 141, 63, 48, 95, 355, 117, 266, 186, 122, 39, 71, 160, 54, 261, 107, 46, 83, 40, 8, 377, 81, 3, 241, 109, 4, 51, 273, 1, 27, 50, 247, 176, 293, 31, 58, 75, 209, 106, 274, 0, 25, 300, 382, 10, 348, 271, 21, 145, 346, 229, 30, 146, 118, 38, 365, 360, 94, 265, 350, 24, 154, 315, 131, 366, 96, 369, 162, 220, 370, 159, 59, 318, 169, 104, 57, 250, 49, 62, 312, 326, 153, 194, 283, 260, 297, 52, 76, 181, 339, 116, 168, 171, 77, 191, 10, 10, 348, 155, 21, 145, 68, 331, 34, 98, 333, 198, 102, 337, 343, 138, 224, 254, 55, 100, 132, 378, 67, 120, 128, 41, 108, 282, 105, 323, 211, 218, 72, 193, 204, 137, 2, 253, 373, 115, 28, 268, 356, 111, 88, 371, 13, 332, 238, 171, 77, 301, 222, 110, 226, 60, 180, 29, 328, 368, 143, 383, 289, 142, 304, 291, 89, 207, 85, 175, 219, 364, 130, 286, 269, 134, 327, 344, 15, 185, 205, 347, 23, 20, 203, 319, 321, 65, 35, 200, 5, 251, 372, 307, 123, 127, 353, 308, 112, 12, 97, 172, 213, 301, 222, 351, 226, 284, 215, 376, 252, 140, 166, 313, 249, 264, 358, 225, 14, 340, 192, 61, 349, 70, 285, 380, 174, 177, 157, 354, 278, 334, 243, 78, 336, 237, 234, 357, 256, 183, 42, 45, 233, 210, 298, 330, 37, 152, 148, 16, 161, 322, 64, 172, 213, 301, 199, 317, 226, 223, 267, 84, 170, 325, 11, 320, 242, 294, 33, 217, 158, 262, 244, 299, 173, 201, 79, 248, 202, 381, 292, 26, 196, 90, 214, 19, 156, 113, 263, 18, 53, 232, 245, 80, 239, 99, 240, 69, 139, 335, 133, 208, 189, 303] + 45: [154, 140, 92, 161, 42, 197, 8, 176, 350, 373, 378, 61, 81, 297, 234, 228, 187, 158, 155, 372, 54, 267, 41, 157, 67, 109, 302, 370, 330, 116, 256, 358, 283, 231, 263, 79, 166, 203, 147, 362, 214, 321, 89, 143, 159, 352, 335, 184, 130, 72, 137, 64, 154, 233, 316, 92, 334, 346, 8, 356, 367, 146, 47, 164, 95, 128, 209, 258, 168, 45, 36, 250, 112, 306, 249, 179, 32, 24, 4, 218, 76, 126, 87, 38, 98, 333, 342, 374, 220, 119, 190, 73, 91, 132, 153, 232, 194, 366, 77, 99, 235, 40, 355, 300, 324, 140, 134, 110, 139, 178, 120, 227, 375, 367, 329, 131, 212, 114, 107, 254, 239, 122, 104, 88, 12, 135, 345, 156, 292, 1, 123, 43, 20, 170, 252, 296, 29, 322, 381, 271, 0, 289, 113, 202, 337, 90, 62, 226, 305, 216, 6, 28, 195, 14, 117, 266, 23, 318, 92, 284, 25, 178, 27, 227, 227, 262, 286, 164, 95, 272, 224, 331, 307, 55, 277, 74, 84, 7, 86, 44, 315, 2, 303, 340, 19, 207, 145, 291, 294, 255, 151, 314, 142, 309, 371, 377, 225, 125, 56, 186, 200, 101, 259, 325, 149, 17, 244, 83, 23, 219, 144, 161, 42, 10, 10, 141, 33, 85, 276, 269, 81, 383, 183, 39, 359, 326, 115, 308, 327, 251, 108, 281, 279, 365, 22, 63, 35, 133, 319, 59, 37, 211, 57, 285, 341, 15, 3, 71, 243, 189, 280, 167, 193, 295, 287, 172, 240, 185, 138, 100, 191, 140, 188, 293, 334, 201, 274, 247, 33, 160, 180, 329, 124, 383, 380, 118, 198, 16, 229, 70, 68, 357, 213, 217, 106, 96, 261, 50, 344, 230, 304, 264, 150, 379, 177, 288, 246, 268, 364, 343, 111, 323, 31, 152, 270, 53, 278, 237, 49, 181, 361, 347, 173, 301, 92, 293, 178, 182, 10, 171, 350, 103, 276, 21, 165, 349, 52, 275, 223, 196, 51, 338, 313, 80, 360, 260, 26, 236, 253, 222, 65, 34, 290, 320, 273, 241, 169, 204, 105, 48, 368, 93, 136, 102, 18, 97, 353, 148, 13, 75, 60, 299, 192, 174, 121, 121, 92, 317, 199, 201, 27, 339, 369, 373, 354, 298, 163, 348, 248, 351, 310, 162, 30, 245, 328, 210, 242, 94, 175, 208, 376, 257, 58, 205, 46, 5, 9, 282, 127, 129, 363, 312, 78, 215, 238, 332, 382, 82, 206, 265, 311, 66, 11, 69, 336, 221] + 46: [152, 183, 352, 256, 286, 343, 110, 207, 221, 354, 66, 335, 194, 165, 199, 3, 102, 30, 108, 227, 186, 168, 42, 4, 362, 250, 103, 17, 111, 101, 35, 160, 313, 56, 71, 333, 145, 90, 361, 375, 15, 75, 127, 229, 300, 134, 279, 349, 14, 182, 5, 36, 188, 183, 34, 58, 368, 343, 16, 51, 367, 106, 106, 301, 40, 68, 338, 76, 215, 62, 130, 83, 132, 245, 192, 37, 41, 53, 292, 193, 268, 329, 97, 360, 89, 189, 293, 265, 283, 190, 285, 26, 52, 135, 94, 126, 105, 174, 296, 0, 100, 365, 27, 88, 142, 247, 352, 139, 224, 355, 25, 191, 149, 367, 66, 335, 214, 55, 18, 203, 39, 211, 117, 120, 80, 118, 156, 166, 164, 95, 216, 321, 377, 307, 263, 290, 223, 382, 243, 322, 177, 291, 278, 98, 347, 21, 77, 366, 363, 1, 306, 84, 200, 270, 325, 213, 142, 247, 352, 139, 380, 253, 311, 239, 149, 354, 319, 154, 136, 204, 303, 219, 350, 351, 195, 155, 232, 121, 180, 13, 129, 162, 237, 340, 383, 32, 141, 255, 161, 173, 159, 241, 359, 93, 262, 234, 208, 43, 258, 281, 87, 196, 179, 47, 151, 82, 231, 122, 142, 183, 34, 64, 240, 46, 311, 96, 230, 284, 104, 12, 124, 114, 44, 146, 79, 228, 33, 22, 326, 197, 295, 298, 176, 273, 198, 235, 57, 81, 206, 169, 222, 346, 7, 277, 91, 217, 244, 339, 337, 358, 275, 341, 288, 269, 248, 54, 70, 225, 252, 175, 133, 10, 64, 58, 380, 253, 249, 238, 246, 332, 140, 140, 60, 38, 113, 374, 282, 131, 115, 260, 287, 315, 137, 316, 304, 128, 205, 158, 45, 226, 92, 289, 119, 318, 74, 153, 261, 331, 86, 257, 138, 67, 330, 236, 107, 181, 29, 185, 28, 148, 147, 302, 133, 201, 19, 256, 286, 311, 110, 202, 274, 6, 209, 312, 187, 254, 345, 320, 266, 280, 8, 372, 334, 218, 259, 50, 63, 294, 267, 59, 125, 308, 328, 24, 299, 348, 317, 20, 364, 309, 305, 171, 150, 112, 11, 72, 73, 48, 212, 123, 357, 323, 379, 356, 210, 10, 233, 314, 327, 355, 170, 207, 230, 324, 319, 344, 271, 378, 2, 242, 85, 49, 143, 376, 99, 353, 109, 369, 23, 272, 172, 157, 336, 373, 78, 144, 297, 69, 9, 342, 264, 370, 184, 371, 163, 65, 310, 31, 381, 251, 116, 276, 167, 178, 220, 61] + 47: [195, 347, 171, 162, 291, 3, 109, 34, 382, 345, 314, 353, 129, 267, 327, 153, 122, 250, 95, 377, 151, 263, 172, 197, 0, 138, 354, 53, 234, 91, 70, 28, 259, 258, 32, 260, 334, 150, 189, 55, 383, 304, 86, 220, 76, 64, 99, 256, 254, 7, 156, 209, 195, 331, 143, 162, 357, 3, 170, 158, 193, 134, 320, 316, 362, 21, 59, 349, 159, 351, 104, 42, 224, 284, 184, 50, 51, 17, 173, 210, 378, 141, 243, 213, 276, 60, 185, 310, 160, 10, 120, 266, 358, 108, 58, 222, 88, 142, 283, 165, 261, 381, 373, 360, 195, 187, 192, 352, 312, 371, 119, 158, 307, 300, 14, 294, 175, 168, 40, 218, 251, 330, 208, 67, 89, 103, 65, 364, 101, 148, 4, 112, 23, 336, 71, 335, 94, 278, 49, 186, 81, 190, 275, 57, 177, 374, 30, 317, 145, 194, 116, 295, 13, 83, 62, 369, 195, 179, 154, 352, 312, 43, 35, 29, 227, 134, 137, 127, 36, 118, 123, 229, 98, 269, 84, 262, 249, 228, 323, 77, 204, 45, 39, 117, 366, 80, 321, 147, 273, 31, 200, 132, 297, 242, 282, 216, 245, 380, 100, 38, 85, 244, 161, 221, 241, 113, 146, 246, 75, 271, 154, 352, 312, 3, 279, 61, 307, 359, 90, 215, 274, 110, 238, 363, 212, 313, 69, 219, 223, 324, 33, 199, 205, 355, 233, 27, 96, 15, 348, 225, 292, 306, 107, 20, 144, 102, 48, 302, 74, 350, 105, 361, 202, 268, 78, 206, 272, 311, 236, 288, 343, 54, 136, 44, 24, 22, 87, 87, 359, 300, 305, 368, 79, 287, 346, 342, 114, 226, 93, 135, 124, 41, 370, 203, 296, 237, 340, 252, 139, 356, 126, 303, 328, 37, 341, 2, 130, 46, 19, 72, 379, 174, 133, 277, 270, 299, 232, 6, 8, 169, 214, 56, 343, 54, 181, 44, 24, 22, 87, 1, 382, 333, 137, 257, 344, 376, 188, 235, 63, 265, 26, 301, 285, 231, 298, 255, 322, 25, 180, 106, 183, 18, 211, 163, 281, 111, 329, 248, 97, 9, 365, 325, 290, 92, 339, 372, 247, 264, 167, 125, 182, 128, 191, 293, 343, 54, 181, 44, 24, 3, 375, 29, 280, 309, 286, 196, 198, 240, 131, 367, 176, 230, 155, 68, 166, 319, 140, 73, 12, 338, 52, 164, 315, 201, 121, 289, 337, 332, 115, 157, 253, 149, 152, 308, 207, 5, 16, 178, 47, 11, 82, 217, 66, 318, 239, 326] + 48: [237, 266, 128, 77, 301, 273, 116, 132, 311, 295, 58, 83, 157, 287, 139, 254, 142, 41, 320, 156, 322, 25, 75, 88, 118, 31, 172, 218, 22, 100, 158, 63, 374, 40, 206, 259, 382, 272, 214, 69, 37, 87, 309, 148, 154, 153, 124, 68, 103, 199, 209, 196, 267, 249, 128, 102, 328, 273, 239, 19, 52, 353, 268, 28, 15, 369, 235, 363, 186, 305, 371, 222, 130, 291, 233, 123, 341, 111, 4, 13, 379, 351, 119, 97, 43, 349, 98, 137, 243, 286, 194, 326, 323, 213, 228, 191, 8, 377, 332, 383, 364, 131, 1, 250, 267, 266, 271, 145, 328, 215, 226, 234, 52, 2, 141, 78, 15, 6, 5, 48, 34, 354, 378, 300, 7, 192, 247, 73, 89, 212, 275, 231, 307, 70, 49, 36, 269, 367, 346, 107, 82, 71, 366, 29, 165, 335, 66, 356, 86, 345, 251, 122, 337, 348, 245, 227, 159, 20, 271, 117, 140, 358, 239, 132, 284, 353, 133, 85, 263, 147, 344, 208, 316, 181, 319, 205, 175, 50, 120, 190, 170, 221, 230, 79, 202, 113, 289, 81, 84, 18, 144, 35, 39, 173, 310, 313, 27, 294, 238, 225, 298, 60, 183, 262, 197, 333, 30, 308, 297, 76, 271, 65, 140, 330, 276, 121, 74, 101, 33, 85, 263, 189, 177, 372, 342, 134, 155, 229, 179, 299, 174, 45, 255, 24, 168, 57, 317, 281, 72, 204, 280, 12, 55, 274, 178, 256, 95, 93, 143, 277, 350, 149, 171, 260, 365, 126, 265, 355, 304, 114, 288, 76, 17, 26, 279, 330, 99, 236, 284, 182, 141, 21, 62, 166, 327, 264, 185, 380, 368, 312, 193, 257, 285, 340, 329, 321, 318, 91, 64, 80, 146, 253, 211, 16, 151, 195, 108, 187, 92, 67, 163, 278, 244, 361, 362, 125, 46, 54, 306, 336, 200, 334, 9, 283, 17, 102, 150, 109, 276, 44, 203, 90, 381, 381, 207, 375, 223, 359, 115, 184, 232, 210, 246, 152, 370, 0, 160, 242, 347, 360, 180, 241, 96, 11, 198, 53, 169, 59, 216, 32, 135, 373, 224, 61, 201, 376, 331, 282, 248, 23, 220, 167, 136, 314, 51, 283, 17, 270, 140, 328, 132, 343, 311, 101, 164, 78, 62, 104, 240, 38, 325, 252, 290, 219, 161, 293, 112, 324, 162, 357, 315, 42, 339, 14, 138, 302, 352, 127, 292, 94, 261, 47, 217, 3, 56, 105, 110, 188, 338, 106, 296, 176, 303, 258, 10, 129] + 49: [374, 184, 272, 268, 155, 65, 181, 293, 116, 25, 270, 276, 193, 218, 260, 306, 45, 255, 164, 235, 328, 279, 221, 23, 240, 112, 189, 376, 303, 371, 267, 178, 345, 320, 53, 130, 2, 151, 190, 22, 125, 357, 51, 147, 237, 18, 350, 291, 369, 50, 24, 38, 374, 197, 321, 346, 257, 297, 19, 213, 353, 212, 153, 9, 44, 199, 133, 187, 312, 332, 109, 5, 205, 340, 137, 223, 198, 290, 41, 163, 301, 238, 43, 62, 142, 211, 280, 61, 31, 285, 361, 354, 185, 232, 377, 341, 159, 231, 379, 383, 32, 129, 135, 337, 79, 302, 149, 6, 209, 297, 19, 253, 313, 105, 90, 335, 148, 251, 91, 220, 331, 30, 298, 144, 73, 336, 54, 86, 140, 224, 34, 248, 216, 10, 67, 134, 244, 72, 176, 355, 49, 84, 101, 108, 324, 104, 166, 165, 162, 161, 99, 263, 288, 219, 92, 375, 79, 302, 37, 6, 15, 234, 150, 126, 202, 271, 110, 275, 193, 251, 107, 74, 186, 8, 131, 58, 169, 174, 80, 368, 196, 170, 222, 146, 139, 179, 35, 309, 359, 339, 378, 71, 46, 206, 143, 256, 88, 274, 315, 87, 367, 21, 236, 11, 258, 26, 349, 123, 210, 302, 272, 268, 75, 305, 115, 157, 66, 241, 208, 180, 39, 325, 76, 344, 381, 175, 94, 122, 366, 17, 382, 70, 365, 29, 160, 81, 225, 327, 330, 338, 172, 47, 85, 118, 204, 282, 342, 145, 364, 316, 93, 167, 42, 266, 249, 141, 230, 370, 300, 356, 233, 302, 321, 319, 75, 305, 158, 253, 313, 78, 201, 9, 262, 60, 286, 269, 329, 100, 333, 195, 97, 1, 28, 56, 138, 89, 323, 20, 55, 173, 352, 227, 351, 114, 226, 119, 12, 124, 113, 154, 326, 343, 13, 7, 363, 265, 171, 217, 203, 3, 188, 64, 360, 302, 321, 83, 155, 297, 136, 126, 202, 271, 40, 242, 259, 380, 243, 132, 194, 264, 314, 48, 250, 362, 16, 245, 277, 128, 4, 296, 82, 295, 33, 98, 156, 239, 69, 358, 106, 192, 228, 52, 182, 111, 27, 372, 102, 287, 152, 207, 252, 373, 322, 191, 360, 302, 321, 294, 304, 59, 157, 253, 126, 278, 261, 242, 177, 292, 289, 246, 120, 183, 95, 121, 254, 63, 310, 311, 318, 214, 127, 77, 273, 308, 36, 307, 103, 14, 117, 0, 200, 284, 283, 299, 348, 168, 57, 229, 96, 334, 281, 215, 247, 347, 68, 317] + 50: [57, 100, 140, 49, 14, 352, 370, 325, 214, 312, 282, 1, 128, 220, 256, 331, 15, 249, 133, 301, 228, 195, 194, 240, 319, 276, 267, 107, 322, 91, 211, 143, 243, 39, 289, 123, 34, 46, 68, 354, 323, 346, 60, 61, 207, 31, 142, 309, 59, 336, 50, 93, 33, 100, 137, 49, 376, 284, 215, 35, 146, 175, 350, 1, 304, 199, 358, 51, 184, 298, 132, 196, 239, 165, 263, 53, 342, 197, 74, 326, 152, 242, 105, 356, 17, 362, 344, 259, 308, 141, 212, 36, 138, 11, 40, 176, 190, 72, 279, 227, 115, 131, 257, 374, 80, 226, 311, 181, 376, 233, 188, 325, 366, 56, 280, 129, 128, 327, 88, 79, 272, 187, 274, 244, 48, 186, 168, 359, 252, 139, 85, 73, 20, 78, 185, 108, 135, 292, 204, 106, 63, 372, 355, 258, 112, 87, 77, 21, 47, 265, 54, 144, 254, 84, 332, 345, 29, 268, 109, 166, 97, 210, 215, 232, 125, 287, 26, 3, 231, 217, 334, 295, 271, 335, 24, 164, 306, 75, 154, 43, 269, 174, 151, 303, 294, 99, 357, 5, 37, 202, 4, 170, 7, 320, 150, 293, 285, 82, 209, 83, 103, 70, 158, 225, 52, 66, 241, 230, 29, 268, 109, 166, 333, 210, 188, 273, 321, 56, 350, 18, 329, 296, 119, 155, 224, 213, 122, 41, 149, 117, 116, 219, 206, 208, 310, 64, 126, 2, 340, 300, 178, 297, 339, 177, 305, 156, 89, 318, 179, 324, 30, 13, 173, 245, 95, 161, 368, 182, 23, 55, 86, 25, 316, 67, 367, 315, 218, 69, 321, 27, 377, 18, 62, 45, 341, 343, 253, 16, 353, 313, 58, 19, 290, 134, 247, 94, 382, 81, 277, 145, 28, 251, 347, 127, 153, 42, 198, 238, 237, 90, 65, 286, 381, 246, 328, 104, 261, 167, 302, 364, 222, 111, 86, 235, 316, 181, 32, 32, 361, 232, 110, 214, 26, 129, 231, 0, 193, 22, 130, 121, 6, 275, 380, 375, 183, 379, 169, 180, 371, 92, 160, 205, 172, 278, 369, 191, 288, 124, 349, 248, 330, 148, 98, 291, 10, 201, 8, 360, 120, 221, 317, 44, 283, 136, 86, 268, 299, 229, 38, 315, 262, 69, 373, 27, 9, 3, 338, 192, 114, 314, 270, 159, 363, 96, 71, 203, 348, 102, 189, 250, 234, 378, 163, 255, 223, 113, 236, 337, 281, 351, 200, 12, 383, 171, 162, 147, 101, 76, 157, 216, 266, 260, 118, 307, 264, 365] + 51: [322, 241, 37, 101, 323, 186, 113, 102, 127, 72, 67, 229, 286, 260, 304, 177, 243, 5, 8, 294, 47, 382, 172, 279, 42, 351, 90, 33, 10, 239, 324, 171, 332, 315, 89, 30, 242, 55, 378, 43, 184, 96, 215, 14, 305, 227, 71, 266, 347, 173, 2, 290, 108, 241, 371, 6, 323, 186, 12, 52, 180, 144, 82, 163, 152, 301, 250, 283, 49, 92, 259, 118, 292, 97, 86, 198, 222, 192, 204, 368, 44, 224, 306, 369, 383, 45, 51, 288, 145, 340, 61, 167, 218, 230, 296, 88, 174, 355, 175, 274, 231, 377, 139, 202, 31, 221, 372, 206, 98, 16, 208, 17, 342, 264, 358, 163, 286, 124, 325, 168, 246, 247, 13, 20, 331, 308, 237, 56, 329, 310, 91, 252, 75, 48, 320, 313, 100, 312, 170, 191, 78, 93, 109, 112, 149, 57, 28, 165, 77, 217, 185, 63, 225, 76, 255, 291, 137, 189, 372, 200, 24, 208, 12, 182, 228, 136, 126, 343, 104, 203, 105, 153, 130, 26, 46, 23, 285, 81, 74, 380, 18, 190, 119, 309, 3, 164, 205, 311, 115, 349, 223, 338, 319, 333, 336, 316, 68, 66, 4, 199, 277, 133, 293, 117, 36, 256, 151, 261, 137, 73, 372, 110, 111, 186, 193, 102, 122, 64, 126, 62, 60, 32, 188, 253, 269, 263, 35, 85, 212, 344, 142, 107, 219, 280, 248, 321, 362, 236, 364, 317, 213, 381, 379, 80, 307, 159, 58, 232, 116, 148, 354, 201, 154, 70, 169, 352, 38, 267, 138, 156, 373, 106, 210, 200, 24, 357, 287, 102, 356, 144, 82, 229, 87, 235, 330, 146, 346, 134, 143, 374, 271, 161, 254, 268, 209, 181, 370, 251, 195, 361, 69, 289, 258, 297, 350, 194, 298, 363, 53, 238, 15, 214, 39, 272, 128, 367, 179, 29, 22, 131, 121, 249, 373, 221, 37, 110, 24, 186, 123, 17, 342, 337, 211, 65, 270, 40, 129, 295, 11, 314, 299, 281, 183, 303, 341, 359, 366, 59, 207, 166, 141, 234, 157, 262, 94, 365, 54, 328, 278, 273, 197, 25, 125, 83, 7, 257, 120, 376, 302, 318, 244, 155, 327, 326, 375, 375, 210, 178, 24, 99, 287, 284, 176, 337, 300, 140, 135, 95, 233, 282, 103, 335, 216, 220, 245, 275, 345, 162, 360, 79, 34, 160, 50, 41, 265, 84, 196, 334, 27, 114, 276, 353, 339, 240, 19, 1, 9, 158, 0, 132, 187, 147, 150, 21, 226, 348] + 52: [52, 107, 381, 267, 31, 71, 104, 97, 323, 174, 169, 298, 78, 68, 80, 259, 349, 236, 88, 312, 222, 234, 341, 160, 233, 6, 113, 351, 374, 337, 306, 308, 89, 242, 282, 84, 220, 129, 188, 118, 344, 313, 229, 302, 145, 255, 65, 122, 240, 360, 162, 348, 186, 107, 283, 99, 130, 246, 104, 165, 289, 195, 48, 82, 13, 60, 10, 128, 54, 354, 35, 123, 285, 262, 296, 223, 367, 300, 72, 17, 170, 324, 109, 119, 270, 200, 365, 207, 230, 330, 101, 380, 18, 126, 304, 275, 158, 42, 273, 241, 1, 376, 46, 175, 327, 216, 381, 99, 334, 151, 103, 137, 239, 206, 209, 163, 205, 172, 357, 269, 87, 95, 291, 379, 127, 166, 315, 215, 116, 280, 3, 115, 22, 91, 125, 235, 57, 258, 148, 194, 47, 198, 49, 141, 43, 299, 135, 232, 261, 79, 11, 366, 254, 94, 218, 375, 121, 61, 381, 99, 334, 157, 362, 165, 85, 239, 48, 25, 382, 346, 260, 219, 266, 64, 257, 133, 339, 105, 178, 23, 364, 67, 328, 154, 279, 208, 45, 225, 2, 277, 244, 21, 139, 347, 303, 197, 111, 96, 358, 363, 9, 185, 202, 144, 373, 264, 212, 180, 92, 332, 107, 99, 334, 157, 103, 137, 290, 345, 263, 90, 183, 62, 245, 192, 210, 149, 70, 171, 74, 191, 227, 124, 29, 369, 76, 53, 213, 333, 326, 268, 58, 110, 59, 152, 318, 249, 143, 342, 114, 4, 353, 276, 297, 203, 159, 142, 317, 199, 204, 176, 102, 307, 217, 320, 182, 71, 165, 73, 336, 271, 193, 25, 7, 33, 140, 196, 32, 50, 37, 228, 221, 286, 281, 173, 134, 146, 301, 331, 316, 55, 253, 350, 237, 284, 224, 251, 150, 368, 156, 278, 81, 335, 138, 274, 329, 131, 12, 0, 24, 164, 106, 38, 102, 120, 372, 371, 63, 71, 104, 97, 214, 34, 211, 184, 41, 190, 26, 168, 355, 8, 325, 155, 378, 5, 83, 30, 248, 292, 361, 319, 39, 287, 40, 93, 201, 177, 294, 189, 179, 100, 356, 322, 226, 311, 181, 112, 36, 187, 19, 15, 86, 272, 370, 247, 102, 231, 283, 381, 334, 157, 250, 97, 336, 321, 209, 77, 205, 288, 44, 56, 252, 98, 309, 108, 256, 117, 359, 167, 75, 243, 153, 377, 69, 66, 27, 161, 352, 383, 20, 238, 51, 136, 28, 310, 340, 343, 293, 147, 132, 314, 265, 338, 14, 295, 16, 305] + 53: [230, 110, 317, 247, 287, 38, 16, 292, 295, 179, 32, 186, 360, 291, 376, 119, 125, 24, 200, 62, 341, 183, 313, 49, 10, 72, 353, 357, 128, 339, 174, 57, 30, 333, 220, 232, 275, 21, 14, 283, 177, 298, 74, 34, 93, 288, 181, 204, 11, 307, 90, 84, 336, 110, 66, 73, 23, 296, 109, 46, 214, 345, 118, 266, 180, 309, 26, 121, 134, 326, 168, 329, 69, 231, 37, 197, 235, 15, 122, 146, 35, 120, 65, 371, 28, 136, 251, 55, 258, 7, 53, 380, 82, 310, 106, 366, 267, 356, 98, 286, 344, 290, 56, 185, 308, 110, 222, 73, 334, 320, 164, 365, 273, 250, 227, 302, 160, 268, 225, 285, 368, 319, 88, 12, 346, 77, 152, 277, 60, 159, 44, 359, 111, 216, 276, 327, 236, 40, 138, 59, 45, 108, 137, 249, 199, 1, 147, 184, 104, 161, 343, 274, 191, 223, 22, 114, 217, 110, 203, 73, 23, 296, 109, 354, 347, 250, 42, 228, 279, 351, 332, 130, 284, 171, 325, 94, 361, 383, 335, 300, 259, 89, 102, 143, 340, 206, 321, 370, 278, 123, 198, 242, 54, 154, 367, 43, 331, 85, 64, 229, 135, 148, 212, 187, 304, 314, 140, 316, 75, 110, 243, 270, 287, 238, 16, 322, 295, 330, 4, 186, 6, 157, 162, 81, 107, 253, 219, 262, 141, 116, 151, 52, 149, 126, 293, 67, 248, 196, 105, 189, 215, 255, 364, 101, 195, 201, 80, 305, 48, 63, 301, 163, 208, 41, 190, 124, 71, 245, 18, 312, 8, 110, 297, 317, 334, 129, 16, 280, 363, 330, 4, 92, 209, 194, 237, 221, 31, 269, 3, 25, 381, 188, 172, 373, 150, 349, 282, 226, 233, 178, 166, 158, 19, 358, 182, 294, 337, 352, 246, 9, 100, 342, 17, 113, 374, 91, 192, 165, 99, 58, 265, 272, 68, 240, 110, 13, 350, 38, 173, 311, 311, 379, 32, 170, 156, 86, 261, 103, 50, 127, 5, 256, 338, 112, 324, 78, 176, 76, 315, 362, 369, 2, 375, 96, 306, 133, 218, 254, 87, 257, 289, 323, 29, 51, 193, 303, 382, 211, 27, 213, 318, 244, 252, 210, 240, 240, 110, 13, 350, 38, 142, 354, 347, 224, 155, 132, 145, 202, 144, 271, 20, 95, 241, 139, 70, 205, 239, 378, 47, 281, 175, 328, 36, 115, 169, 207, 117, 79, 299, 97, 0, 234, 131, 264, 83, 39, 348, 372, 260, 263, 61, 33, 355, 377, 153, 167] + 54: [295, 363, 71, 319, 356, 223, 231, 260, 38, 299, 172, 254, 282, 330, 118, 3, 10, 375, 99, 197, 225, 91, 190, 382, 103, 157, 131, 135, 298, 301, 107, 22, 146, 18, 166, 9, 192, 271, 234, 247, 31, 206, 248, 102, 285, 65, 379, 37, 126, 383, 205, 148, 370, 363, 253, 328, 11, 339, 216, 369, 38, 299, 8, 208, 44, 311, 314, 75, 121, 70, 32, 64, 357, 325, 92, 142, 381, 312, 97, 1, 278, 48, 326, 179, 96, 347, 316, 141, 238, 213, 308, 59, 211, 269, 359, 199, 374, 4, 28, 241, 78, 340, 13, 0, 370, 363, 294, 189, 11, 287, 150, 82, 250, 98, 95, 72, 187, 352, 40, 317, 58, 41, 47, 161, 198, 323, 152, 279, 125, 204, 302, 100, 171, 255, 327, 201, 62, 377, 43, 128, 229, 334, 346, 14, 29, 261, 362, 203, 360, 104, 134, 188, 165, 19, 106, 232, 370, 363, 294, 338, 267, 339, 231, 80, 348, 20, 306, 95, 221, 87, 117, 228, 27, 76, 196, 158, 277, 94, 272, 109, 239, 376, 368, 177, 246, 69, 111, 136, 167, 162, 23, 275, 251, 259, 15, 235, 367, 52, 89, 46, 149, 51, 350, 256, 193, 344, 160, 36, 370, 363, 294, 147, 258, 155, 257, 164, 348, 230, 53, 176, 290, 25, 26, 329, 101, 288, 218, 322, 84, 5, 34, 182, 170, 186, 365, 318, 144, 85, 163, 304, 81, 173, 262, 175, 139, 12, 110, 145, 249, 331, 66, 60, 116, 303, 77, 2, 297, 202, 127, 336, 284, 153, 283, 83, 39, 223, 155, 240, 240, 280, 286, 281, 345, 57, 7, 372, 156, 132, 115, 113, 50, 220, 265, 217, 184, 237, 55, 49, 264, 373, 137, 313, 353, 195, 273, 266, 194, 63, 219, 168, 274, 79, 108, 227, 154, 307, 138, 112, 342, 358, 354, 361, 284, 296, 283, 90, 42, 339, 155, 214, 133, 280, 286, 324, 120, 305, 180, 86, 332, 276, 333, 215, 335, 45, 321, 159, 140, 105, 355, 183, 73, 68, 293, 242, 364, 151, 244, 191, 243, 371, 6, 74, 236, 233, 263, 226, 35, 292, 169, 222, 207, 200, 123, 320, 337, 380, 363, 42, 258, 356, 164, 164, 30, 230, 174, 341, 93, 119, 309, 16, 181, 143, 310, 129, 124, 17, 185, 24, 54, 21, 315, 67, 61, 366, 122, 209, 33, 114, 300, 351, 245, 88, 224, 178, 289, 268, 343, 56, 270, 210, 291, 252, 349, 130, 212, 378] + 55: [56, 154, 235, 174, 146, 146, 370, 307, 381, 90, 121, 382, 29, 65, 291, 249, 8, 333, 312, 316, 189, 5, 348, 210, 217, 231, 79, 228, 263, 346, 358, 101, 314, 82, 1, 281, 219, 359, 118, 55, 120, 374, 13, 162, 298, 21, 362, 345, 18, 25, 250, 69, 56, 154, 355, 174, 11, 62, 165, 80, 6, 303, 78, 347, 180, 301, 106, 134, 3, 44, 221, 271, 337, 112, 14, 375, 260, 343, 240, 140, 332, 23, 254, 94, 22, 31, 232, 365, 335, 372, 201, 104, 58, 50, 177, 280, 295, 334, 32, 195, 167, 379, 285, 196, 326, 154, 81, 174, 20, 340, 89, 252, 267, 299, 48, 74, 363, 115, 199, 71, 12, 320, 300, 26, 239, 102, 276, 190, 251, 265, 306, 54, 19, 257, 52, 193, 313, 264, 103, 367, 253, 97, 35, 129, 147, 64, 261, 141, 169, 36, 75, 100, 351, 45, 96, 215, 113, 327, 256, 174, 7, 308, 116, 238, 381, 90, 247, 347, 70, 269, 341, 2, 114, 290, 213, 383, 233, 163, 352, 105, 329, 47, 42, 194, 241, 287, 255, 149, 296, 212, 237, 133, 208, 319, 209, 168, 161, 68, 139, 242, 191, 330, 266, 230, 24, 283, 222, 317, 72, 88, 81, 151, 234, 218, 39, 311, 227, 41, 211, 323, 180, 76, 49, 99, 176, 138, 368, 156, 344, 170, 325, 371, 236, 377, 178, 187, 184, 122, 157, 278, 117, 214, 342, 143, 373, 30, 321, 137, 315, 66, 248, 77, 364, 244, 57, 288, 229, 9, 310, 305, 159, 88, 270, 309, 11, 308, 339, 80, 328, 123, 353, 304, 29, 269, 200, 293, 279, 331, 366, 86, 259, 197, 284, 274, 27, 380, 286, 53, 322, 277, 128, 33, 87, 198, 61, 127, 173, 126, 38, 297, 107, 125, 166, 183, 262, 282, 110, 158, 148, 292, 186, 204, 202, 152, 43, 98, 20, 340, 116, 111, 192, 73, 273, 323, 130, 115, 40, 223, 179, 349, 216, 0, 172, 17, 95, 91, 378, 246, 360, 136, 185, 318, 207, 84, 294, 153, 160, 188, 15, 268, 336, 338, 354, 145, 324, 376, 92, 181, 108, 220, 131, 51, 144, 34, 202, 28, 154, 4, 98, 357, 182, 16, 123, 299, 48, 74, 130, 76, 132, 226, 272, 60, 164, 124, 206, 46, 350, 289, 224, 361, 67, 135, 119, 10, 171, 85, 63, 302, 258, 37, 225, 356, 93, 243, 275, 150, 203, 369, 155, 175, 142, 83, 59, 205, 245, 109] + 56: [279, 167, 260, 82, 240, 15, 116, 225, 106, 371, 304, 249, 333, 162, 39, 41, 77, 80, 192, 72, 344, 73, 40, 253, 92, 31, 374, 280, 50, 33, 123, 364, 122, 201, 180, 218, 310, 200, 330, 372, 144, 352, 204, 258, 272, 203, 59, 377, 171, 237, 224, 146, 100, 343, 260, 82, 220, 213, 163, 303, 241, 5, 147, 189, 251, 161, 316, 292, 327, 381, 244, 97, 197, 334, 160, 250, 21, 136, 269, 121, 110, 151, 227, 42, 378, 210, 183, 119, 228, 112, 319, 367, 49, 70, 24, 131, 234, 239, 7, 36, 194, 51, 229, 208, 177, 321, 265, 178, 240, 15, 107, 225, 340, 259, 9, 196, 125, 117, 284, 219, 66, 159, 101, 383, 205, 376, 38, 278, 181, 198, 166, 12, 11, 17, 270, 248, 190, 62, 257, 341, 375, 226, 175, 291, 170, 350, 287, 345, 358, 349, 326, 216, 25, 91, 301, 143, 231, 195, 44, 199, 81, 294, 155, 296, 340, 30, 147, 356, 306, 6, 99, 275, 329, 154, 207, 118, 103, 27, 285, 10, 105, 79, 179, 331, 1, 164, 141, 129, 115, 360, 54, 314, 293, 28, 379, 16, 29, 43, 113, 337, 325, 212, 261, 45, 380, 102, 109, 90, 231, 195, 44, 178, 240, 294, 88, 184, 259, 30, 156, 20, 187, 328, 71, 302, 362, 288, 168, 309, 0, 300, 148, 312, 305, 252, 230, 22, 34, 153, 282, 286, 221, 271, 298, 353, 347, 124, 274, 37, 67, 361, 93, 47, 382, 370, 335, 217, 111, 165, 74, 173, 231, 311, 265, 260, 220, 213, 98, 351, 241, 5, 304, 236, 264, 46, 320, 188, 332, 245, 63, 238, 246, 359, 281, 193, 135, 338, 268, 317, 138, 13, 127, 56, 95, 348, 96, 145, 290, 323, 57, 139, 64, 86, 152, 157, 256, 176, 52, 366, 247, 61, 76, 3, 231, 321, 242, 126, 220, 294, 53, 267, 106, 87, 75, 263, 363, 308, 140, 18, 19, 14, 283, 223, 369, 182, 69, 137, 206, 295, 23, 355, 133, 255, 209, 368, 339, 94, 55, 84, 172, 4, 232, 289, 266, 222, 373, 254, 108, 68, 336, 365, 174, 318, 85, 315, 8, 195, 242, 191, 240, 273, 186, 158, 322, 169, 75, 89, 104, 346, 48, 2, 83, 357, 150, 120, 233, 58, 342, 128, 324, 35, 354, 149, 214, 243, 185, 297, 313, 299, 134, 262, 132, 307, 276, 65, 78, 130, 60, 211, 26, 215, 32, 277, 114, 235, 142, 202] + 57: [157, 158, 133, 259, 165, 198, 326, 235, 359, 72, 320, 125, 267, 150, 127, 64, 35, 182, 338, 161, 322, 341, 113, 83, 183, 147, 380, 225, 2, 164, 199, 216, 288, 29, 41, 247, 135, 236, 0, 126, 260, 192, 48, 78, 360, 212, 52, 269, 181, 201, 56, 275, 231, 31, 254, 295, 246, 187, 104, 148, 331, 274, 130, 344, 74, 154, 151, 79, 123, 238, 319, 43, 134, 214, 27, 20, 111, 116, 55, 119, 166, 171, 190, 296, 215, 371, 261, 97, 122, 59, 307, 1, 109, 374, 138, 61, 142, 291, 340, 193, 69, 15, 367, 343, 231, 381, 254, 146, 165, 140, 326, 205, 108, 94, 320, 125, 217, 154, 197, 22, 306, 347, 144, 316, 243, 70, 152, 298, 377, 289, 47, 325, 136, 71, 351, 23, 81, 17, 175, 363, 211, 265, 4, 188, 36, 222, 239, 273, 10, 218, 221, 278, 245, 229, 202, 304, 93, 276, 314, 220, 89, 156, 25, 65, 257, 94, 137, 141, 74, 204, 317, 345, 163, 251, 91, 179, 176, 9, 42, 191, 382, 38, 358, 13, 329, 352, 321, 155, 178, 334, 332, 145, 303, 248, 378, 219, 66, 361, 76, 103, 310, 114, 206, 195, 77, 34, 285, 301, 11, 276, 254, 259, 246, 187, 25, 101, 207, 286, 170, 335, 60, 174, 84, 88, 44, 354, 58, 365, 213, 51, 177, 46, 184, 92, 12, 328, 226, 149, 353, 362, 98, 311, 240, 49, 33, 255, 87, 168, 383, 256, 370, 139, 271, 258, 327, 292, 37, 153, 86, 376, 342, 276, 254, 295, 89, 63, 25, 128, 207, 286, 170, 8, 302, 6, 318, 160, 185, 272, 99, 253, 40, 228, 249, 356, 349, 203, 120, 112, 68, 196, 282, 369, 186, 18, 379, 355, 45, 24, 270, 333, 224, 375, 67, 308, 244, 100, 82, 189, 305, 233, 143, 372, 159, 230, 254, 121, 57, 7, 326, 148, 331, 124, 324, 118, 302, 169, 283, 180, 75, 80, 39, 373, 346, 73, 173, 277, 102, 54, 106, 279, 210, 313, 337, 90, 132, 323, 290, 315, 252, 200, 32, 357, 50, 194, 232, 107, 264, 339, 129, 336, 131, 280, 234, 262, 14, 14, 16, 146, 85, 368, 366, 205, 223, 72, 53, 268, 300, 350, 117, 309, 62, 250, 167, 172, 110, 297, 96, 312, 21, 208, 284, 95, 348, 3, 209, 28, 330, 162, 237, 26, 5, 299, 281, 241, 263, 242, 364, 287, 266, 115, 294, 30, 227, 105, 19, 293] + 58: [263, 377, 126, 56, 259, 348, 245, 373, 322, 298, 358, 270, 36, 152, 47, 119, 315, 60, 66, 171, 139, 55, 207, 99, 238, 124, 183, 63, 241, 154, 65, 345, 182, 8, 237, 243, 294, 95, 359, 254, 374, 269, 120, 159, 301, 217, 218, 165, 252, 370, 364, 308, 248, 377, 76, 126, 366, 27, 98, 194, 299, 9, 114, 169, 40, 262, 342, 189, 130, 4, 352, 285, 187, 239, 25, 129, 306, 333, 121, 131, 300, 288, 327, 43, 265, 50, 136, 64, 5, 142, 221, 163, 146, 84, 39, 32, 78, 267, 91, 89, 279, 125, 150, 38, 156, 377, 377, 61, 205, 92, 245, 222, 49, 312, 351, 256, 116, 68, 141, 272, 12, 316, 274, 329, 229, 381, 82, 258, 83, 234, 37, 135, 362, 176, 107, 228, 293, 353, 105, 338, 138, 360, 268, 282, 290, 223, 109, 145, 193, 74, 310, 320, 164, 321, 184, 134, 156, 377, 377, 79, 305, 356, 271, 311, 93, 298, 72, 214, 175, 196, 85, 42, 132, 162, 185, 328, 192, 202, 143, 325, 90, 326, 382, 332, 253, 357, 334, 11, 70, 225, 230, 170, 24, 29, 330, 157, 337, 247, 117, 153, 286, 174, 140, 23, 166, 335, 280, 209, 220, 275, 249, 81, 205, 92, 14, 222, 177, 22, 289, 355, 232, 324, 367, 16, 97, 260, 197, 224, 48, 331, 59, 181, 53, 227, 378, 302, 208, 148, 278, 347, 287, 179, 94, 180, 296, 317, 54, 276, 200, 167, 57, 34, 88, 365, 361, 233, 292, 319, 168, 58, 220, 275, 151, 81, 273, 210, 211, 194, 235, 122, 114, 199, 368, 376, 6, 203, 111, 20, 190, 28, 1, 375, 62, 266, 155, 160, 346, 110, 144, 186, 123, 2, 19, 195, 173, 118, 102, 372, 86, 303, 103, 51, 96, 137, 15, 30, 323, 212, 147, 309, 201, 204, 220, 77, 377, 81, 305, 7, 98, 311, 206, 312, 67, 26, 112, 295, 18, 191, 277, 106, 242, 198, 281, 314, 244, 350, 236, 339, 75, 149, 13, 379, 318, 21, 113, 35, 133, 297, 261, 100, 264, 41, 336, 349, 246, 304, 52, 188, 363, 291, 80, 71, 108, 33, 220, 377, 377, 56, 366, 344, 245, 101, 177, 122, 69, 257, 178, 45, 341, 31, 104, 255, 307, 73, 380, 251, 383, 284, 215, 250, 240, 0, 172, 231, 161, 371, 115, 283, 219, 226, 213, 87, 128, 354, 44, 46, 369, 216, 17, 127, 343, 313, 158, 3, 10, 340] + 59: [332, 285, 170, 170, 229, 7, 260, 173, 173, 66, 352, 31, 172, 328, 23, 266, 290, 278, 12, 35, 302, 383, 60, 218, 47, 179, 180, 92, 13, 131, 96, 107, 71, 43, 136, 322, 74, 145, 3, 115, 148, 284, 303, 98, 318, 219, 272, 367, 190, 116, 90, 154, 140, 30, 25, 250, 229, 370, 195, 95, 220, 371, 221, 150, 300, 9, 26, 232, 133, 109, 304, 159, 334, 62, 225, 29, 54, 69, 344, 354, 289, 51, 61, 0, 287, 110, 129, 84, 67, 49, 261, 249, 273, 307, 165, 319, 305, 270, 171, 378, 345, 351, 308, 341, 140, 105, 25, 256, 226, 370, 195, 70, 220, 294, 346, 331, 295, 38, 262, 309, 342, 175, 161, 16, 181, 252, 373, 142, 214, 222, 137, 259, 155, 2, 242, 206, 380, 135, 325, 253, 330, 89, 336, 11, 357, 102, 216, 8, 276, 14, 335, 78, 126, 323, 15, 79, 73, 33, 224, 197, 147, 7, 202, 363, 381, 66, 234, 130, 65, 317, 151, 281, 176, 212, 160, 297, 267, 162, 376, 286, 306, 339, 369, 177, 365, 349, 199, 362, 227, 246, 187, 217, 21, 93, 277, 275, 321, 241, 358, 316, 239, 360, 77, 244, 104, 99, 111, 215, 209, 327, 46, 313, 379, 94, 59, 200, 340, 204, 352, 76, 65, 377, 178, 382, 112, 45, 85, 279, 100, 320, 143, 338, 10, 355, 348, 238, 337, 68, 156, 299, 27, 64, 184, 124, 207, 37, 88, 191, 201, 103, 312, 237, 141, 350, 293, 205, 83, 52, 4, 188, 209, 327, 25, 250, 229, 5, 202, 251, 311, 247, 234, 185, 265, 324, 243, 255, 86, 91, 81, 127, 211, 169, 22, 174, 356, 258, 146, 366, 353, 101, 274, 48, 138, 63, 50, 157, 28, 153, 210, 106, 118, 230, 283, 108, 282, 56, 39, 264, 139, 72, 310, 189, 113, 327, 224, 97, 379, 94, 59, 200, 340, 263, 346, 18, 42, 372, 144, 196, 114, 223, 57, 168, 364, 248, 315, 19, 329, 120, 269, 361, 6, 228, 292, 32, 333, 41, 34, 231, 343, 326, 44, 24, 53, 296, 368, 301, 240, 192, 375, 87, 152, 167, 235, 268, 182, 327, 75, 170, 233, 7, 260, 363, 298, 374, 291, 288, 213, 186, 80, 58, 128, 164, 40, 254, 280, 163, 20, 245, 236, 158, 198, 203, 36, 123, 314, 194, 193, 149, 208, 134, 17, 359, 132, 183, 119, 166, 82, 257, 1, 125, 121, 347, 55, 122, 117, 271] + 60: [284, 381, 335, 158, 128, 129, 139, 133, 199, 265, 153, 302, 2, 49, 119, 354, 272, 184, 82, 359, 152, 20, 6, 1, 332, 382, 274, 304, 171, 107, 85, 200, 341, 370, 256, 18, 315, 67, 202, 330, 54, 198, 229, 298, 159, 43, 173, 103, 44, 218, 317, 355, 102, 268, 351, 124, 115, 266, 244, 39, 40, 40, 230, 181, 327, 35, 112, 311, 245, 223, 263, 169, 189, 58, 269, 240, 80, 267, 176, 116, 141, 70, 285, 146, 100, 106, 219, 14, 338, 105, 13, 252, 261, 50, 374, 29, 86, 17, 292, 32, 352, 136, 62, 207, 94, 268, 351, 124, 260, 303, 139, 83, 199, 310, 194, 208, 205, 234, 180, 193, 232, 231, 27, 246, 0, 283, 93, 36, 344, 110, 38, 342, 186, 360, 293, 33, 192, 79, 296, 148, 125, 282, 45, 178, 320, 224, 379, 97, 165, 135, 227, 177, 101, 226, 63, 348, 123, 313, 351, 257, 16, 47, 60, 133, 339, 310, 247, 81, 366, 76, 353, 15, 213, 59, 238, 364, 151, 255, 114, 217, 11, 237, 57, 8, 21, 52, 187, 196, 182, 95, 380, 371, 383, 376, 361, 287, 179, 145, 211, 111, 72, 326, 367, 4, 73, 89, 210, 22, 123, 313, 124, 257, 16, 220, 347, 126, 222, 174, 120, 297, 34, 294, 46, 329, 131, 143, 271, 140, 75, 277, 37, 183, 24, 9, 215, 56, 323, 90, 66, 7, 369, 314, 276, 64, 197, 156, 262, 328, 258, 280, 356, 291, 175, 281, 71, 295, 118, 242, 154, 299, 123, 313, 124, 257, 16, 47, 60, 113, 212, 174, 312, 185, 28, 337, 166, 309, 278, 161, 235, 172, 336, 248, 221, 51, 92, 264, 61, 305, 134, 53, 273, 30, 190, 349, 368, 331, 325, 168, 23, 209, 19, 41, 160, 147, 191, 306, 289, 275, 321, 308, 157, 288, 236, 313, 351, 124, 195, 220, 343, 83, 316, 265, 65, 334, 239, 108, 365, 345, 122, 117, 55, 253, 84, 362, 358, 318, 188, 162, 377, 25, 26, 142, 91, 149, 130, 357, 286, 12, 363, 270, 251, 228, 137, 104, 204, 96, 301, 254, 164, 340, 109, 87, 214, 375, 121, 313, 381, 158, 170, 243, 372, 126, 48, 78, 167, 155, 74, 319, 290, 98, 206, 203, 241, 99, 150, 69, 132, 259, 88, 5, 3, 279, 233, 346, 31, 250, 300, 322, 144, 225, 333, 307, 127, 324, 216, 42, 68, 378, 249, 350, 163, 138, 10, 77, 373, 201] + 61: [231, 211, 221, 179, 326, 203, 270, 381, 218, 167, 51, 238, 263, 86, 107, 288, 177, 160, 169, 191, 348, 114, 135, 379, 2, 3, 200, 290, 89, 292, 341, 80, 141, 340, 332, 98, 70, 54, 18, 364, 232, 314, 180, 140, 256, 277, 279, 183, 320, 296, 189, 149, 303, 93, 120, 338, 127, 299, 19, 381, 36, 21, 371, 324, 143, 161, 88, 57, 369, 282, 32, 87, 65, 29, 300, 304, 109, 328, 52, 378, 383, 273, 280, 126, 85, 132, 14, 342, 274, 91, 187, 99, 24, 185, 182, 223, 347, 285, 131, 210, 325, 334, 356, 319, 230, 93, 211, 338, 127, 323, 241, 45, 36, 195, 278, 324, 55, 237, 88, 115, 188, 156, 227, 361, 362, 366, 83, 252, 260, 206, 147, 173, 72, 306, 8, 365, 240, 81, 194, 359, 244, 63, 354, 251, 7, 13, 28, 146, 318, 64, 254, 50, 367, 350, 116, 58, 172, 360, 221, 179, 247, 178, 184, 380, 103, 315, 68, 294, 248, 307, 112, 168, 286, 181, 43, 337, 77, 90, 27, 11, 47, 245, 202, 249, 137, 346, 37, 79, 283, 71, 199, 298, 193, 352, 6, 123, 262, 305, 95, 105, 35, 377, 39, 289, 41, 128, 343, 0, 61, 335, 221, 338, 127, 122, 59, 197, 253, 330, 344, 198, 213, 119, 327, 129, 34, 31, 261, 228, 150, 233, 190, 374, 329, 23, 97, 33, 186, 30, 275, 125, 225, 317, 284, 204, 162, 163, 301, 101, 49, 201, 164, 339, 220, 56, 175, 134, 229, 16, 321, 94, 61, 108, 157, 179, 127, 299, 10, 363, 353, 12, 17, 142, 248, 66, 192, 333, 42, 15, 133, 258, 118, 84, 44, 113, 336, 368, 207, 259, 322, 214, 375, 242, 196, 26, 144, 152, 311, 271, 239, 358, 111, 293, 212, 48, 222, 60, 281, 62, 76, 357, 96, 208, 73, 5, 170, 46, 351, 117, 209, 1, 100, 167, 344, 235, 307, 161, 216, 291, 110, 243, 75, 92, 297, 316, 82, 121, 308, 145, 104, 139, 154, 78, 174, 219, 67, 295, 124, 148, 370, 25, 310, 74, 312, 272, 373, 236, 250, 226, 355, 382, 376, 40, 171, 155, 73, 136, 221, 46, 165, 205, 264, 197, 100, 315, 372, 215, 263, 86, 107, 216, 159, 349, 309, 20, 224, 268, 166, 313, 257, 69, 158, 102, 38, 153, 9, 22, 269, 4, 176, 106, 345, 130, 255, 234, 331, 246, 138, 265, 276, 302, 151, 267, 217, 266, 53, 287] +layer_updates_per_iter: 0 +num_slots: 416 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index f369b6569b57..441176cb262f 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 @@ -101,6 +102,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 4ecc2c109ed4..7a03e3dc8f06 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -72,6 +72,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -99,5 +100,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 5fd232627cf8..c641ccfed360 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -100,5 +101,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL.yaml index fd2677ca1579..0f3d964382a6 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL.yaml @@ -71,6 +71,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON stream_interval: 20 num_postprocess_workers: 4 allreduce_strategy: MNNVL @@ -92,5 +93,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON moe_config: backend: TRTLLM diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index e57ea6d424e8..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,103 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k - # Mirror of the Dynamo wide-EP DSR1 FP4 GB200 deployment recipe: - # https://github.com/ai-dynamo/dynamo/blob/main/recipes/deepseek-r1/trtllm/disagg/wide_ep/gb200/deploy.yaml -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: "--gres=gpu:4" - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1024' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - # Env vars taken from the Dynamo deploy.yaml (NCCL_MNNVL/CUMEM, MoE all-to-all - # without all-gather, PDL) so that the perf test exercises the same code paths - # as the production deployment. - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1 TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER=1" - server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" -profiling: - nsys_on: false -worker_config: - gen: - print_iter_log: true - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - - 768 - - 1024 - - 2048 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - dtype: fp8 - moe_config: - backend: CUTEDSL - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - ctx: - print_iter_log: true - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 1227 - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml deleted file mode 100644 index 5d2b230f8195..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,105 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1024' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 32 - max_num_tokens: 128 - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index c3ff2770bcb5..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,92 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k - config_index: -1 -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: "--gres=gpu:4" - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 8 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1024' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" - server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" -profiling: - nsys_on: false -worker_config: - gen: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - context_parallel_size: 1 - max_batch_size: 768 - max_num_tokens: 768 - max_seq_len: 2068 - cuda_graph_config: - enable_padding: true - max_batch_size: 768 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTLASS - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - stream_interval: 100 - num_postprocess_workers: 4 - ctx: - max_batch_size: 16 - max_num_tokens: 16896 - max_seq_len: 2044 - tensor_parallel_size: 4 - context_parallel_size: 1 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml deleted file mode 100644 index e3b3ef2506d5..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,104 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 128 - max_num_tokens: 512 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: TRTLLM - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 - stream_interval: 20 - allreduce_strategy: MNNVL - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml deleted file mode 100644 index 9165da2fec27..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,108 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 768 - max_num_tokens: 1536 - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 768 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml deleted file mode 100644 index 1e27a99a6798..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ /dev/null @@ -1,105 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '3072' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 768 - max_num_tokens: 1536 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 768 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml deleted file mode 100644 index 657333376548..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ /dev/null @@ -1,105 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '3072' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 768 - max_num_tokens: 1536 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 768 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - max_draft_len: 1 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_con1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_con1_ccb-NIXL.yaml deleted file mode 100644 index dfe64bdcfc64..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_con1_ccb-NIXL.yaml +++ /dev/null @@ -1,103 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 8 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 4 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - enable_attention_dp: false - pipeline_parallel_size: 1 - max_batch_size: 32 - max_num_tokens: 128 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: TRTLLM - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - allreduce_strategy: MNNVL - ctx: - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml deleted file mode 100644 index 481d5978511f..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml +++ /dev/null @@ -1,103 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 8 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - MIMALLOC_PURGE_DELAY=0 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index f2f793df5219..da114fd46a5c 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 7c2e9f2d770c..a65b43a27794 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -72,6 +72,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -100,5 +101,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 4709cb8b6d59..67f1cd34b71d 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml index ee18edcd2fce..071dacdd6e52 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON stream_interval: 20 num_postprocess_workers: 4 allreduce_strategy: MNNVL @@ -95,3 +96,4 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml deleted file mode 100644 index b089131feecf..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,116 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1024' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 32 - max_num_tokens: 128 - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 256 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml deleted file mode 100644 index 9da91a2d426d..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,112 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 1 - max_num_tokens: 512 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: - enable_padding: true - max_batch_size: 1 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: TRTLLM - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - allreduce_strategy: MNNVL - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml deleted file mode 100644 index 45ab3d59c928..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ /dev/null @@ -1,113 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 1024 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml deleted file mode 100644 index 81796ae2fd04..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ /dev/null @@ -1,113 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 1024 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - max_draft_len: 1 - num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index f72d1b107939..ab9555369acf 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -76,6 +76,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -113,5 +114,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index 94340dd398ef..c23bb51e6007 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -80,6 +80,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 @@ -117,6 +118,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 40f1d141ab89..8f110a2e5479 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -77,6 +77,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -113,5 +114,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 63f2ef1ae8f9..983b2148a3dd 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -40,7 +40,7 @@ environment: # Env vars taken from the Dynamo deploy.yaml (NCCL MNNVL/CUMEM/NVLS, UCX MNNVL # IPC, MoE all-to-all without all-gather, PDL, GC disable) so the perf test # exercises the same code paths as the production deployment. - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=1 NVIDIA_GDRCOPY=1 UCX_CUDA_IPC_ENABLE_MNNVL=1 NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER=1 TRTLLM_FORCE_COMM_METHOD=NVLINK_TWO_SIDED ENABLE_CONFIGURABLE_MOE=1" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=1 NVIDIA_GDRCOPY=1 UCX_CUDA_IPC_ENABLE_MNNVL=1 NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER=1 TRTLLM_FORCE_COMM_METHOD=NVLINK_TWO_SIDED" server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false @@ -70,6 +70,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + transceiver_runtime: PYTHON num_postprocess_workers: 8 stream_interval: 10 ctx: @@ -97,3 +98,4 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + transceiver_runtime: PYTHON diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index 78973f5ecc75..1373758fc1ca 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -78,6 +78,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -112,5 +113,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 4fda7f7549f2..8bb7d7329cb3 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -73,6 +73,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -108,5 +109,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml index 0257c507e4d1..804cb98a48b2 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml @@ -78,6 +78,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -109,4 +110,5 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml deleted file mode 100644 index 899270e7d29f..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml +++ /dev/null @@ -1,108 +0,0 @@ -metadata: - model_name: glm_5_nvfp4 - precision: fp4 - model_dir_name: GLM-5-NVFP4 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '4096' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 1024 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 256 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml deleted file mode 100644 index 00dced799780..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml +++ /dev/null @@ -1,108 +0,0 @@ -metadata: - model_name: glm_5_nvfp4 - precision: fp4 - model_dir_name: GLM-5-NVFP4 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '4096' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 1024 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 256 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml deleted file mode 100644 index c46d573cf25d..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml +++ /dev/null @@ -1,102 +0,0 @@ -metadata: - model_name: gpt_oss_120b_fp4 - precision: fp4 - model_dir_name: GPT-OSS-120B-FP4 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 1536 - max_num_tokens: 20000 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 - cuda_graph_config: - enable_padding: true - max_batch_size: 1536 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: TRTLLM - cache_transceiver_config: - max_tokens_in_buffer: 1024 - backend: UCX - disable_overlap_scheduler: true - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 32 - max_num_tokens: 20000 - tensor_parallel_size: 1 - moe_expert_parallel_size: 1 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 1024 - backend: UCX - disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index d1e80f7a9229..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,106 +0,0 @@ -metadata: - model_name: gpt_oss_120b_fp4 - precision: fp4 - model_dir_name: GPT-OSS-120B-FP4 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '512' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 1536 - max_num_tokens: 20000 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 - cuda_graph_config: - enable_padding: true - max_batch_size: 1536 - kv_cache_config: - use_kv_cache_manager_v2: false - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: TRTLLM - cache_transceiver_config: - max_tokens_in_buffer: 1024 - backend: NIXL - transceiver_runtime: PYTHON - disable_overlap_scheduler: true - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 32 - max_num_tokens: 20000 - tensor_parallel_size: 1 - moe_expert_parallel_size: 1 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: null - kv_cache_config: - use_kv_cache_manager_v2: false - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 1024 - backend: NIXL - transceiver_runtime: PYTHON - disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 9f4b7086060d..ffa8cdfa4522 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -75,6 +75,7 @@ worker_config: max_tokens_in_buffer: 9216 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: false num_postprocess_workers: 4 stream_interval: 20 @@ -103,6 +104,7 @@ worker_config: max_tokens_in_buffer: 9216 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 9ed015cd6cda..afa141b6ae82 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -99,4 +100,5 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 5476fae34c0e..c8dffadebfa7 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -99,4 +100,5 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index 5853a2570802..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,103 +0,0 @@ -metadata: - model_name: k25_thinking_fp4 - precision: fp4 - model_dir_name: Kimi-K2.5-NVFP4 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 64 - max_num_tokens: 128 - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 8192 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index 0c94eb253749..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,103 +0,0 @@ -metadata: - model_name: k25_thinking_fp4 - precision: fp4 - model_dir_name: Kimi-K2.5-NVFP4 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '4096' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 512 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 8192 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml deleted file mode 100644 index a4c68609cb29..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml +++ /dev/null @@ -1,103 +0,0 @@ -metadata: - model_name: k25_thinking_fp4 - precision: fp4 - model_dir_name: Kimi-K2.5-NVFP4 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '4096' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 512 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - trust_remote_code: true - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 8192 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - trust_remote_code: true diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index 268db9e6f545..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,101 +0,0 @@ -metadata: - model_name: k25_thinking_fp4 - precision: fp4 - model_dir_name: Kimi-K2.5-NVFP4 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '4' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 4 - max_num_tokens: 128 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: TRTLLM - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 8192 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true diff --git a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml deleted file mode 100644 index 2e3783298ba9..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL.yaml +++ /dev/null @@ -1,97 +0,0 @@ -metadata: - model_name: qwen3_235b_a22b_fp4 - precision: fp4 - model_dir_name: Qwen3-235B-A22B-FP4 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 8 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 4 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - enable_attention_dp: false - pipeline_parallel_size: 1 - max_batch_size: 32 - max_num_tokens: 128 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: TRTLLM - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - allreduce_strategy: MNNVL - ctx: - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL diff --git a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml deleted file mode 100644 index 9b97a86243ca..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml +++ /dev/null @@ -1,106 +0,0 @@ -metadata: - model_name: qwen3_235b_a22b_fp4 - precision: fp4 - model_dir_name: Qwen3-235B-A22B-FP4 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 8 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - dtype: fp8 - moe_config: - backend: CUTEDSL - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: Eagle - max_draft_len: 3 - eagle3_one_model: true - speculative_model: Qwen3/qwen3-235B-eagle3 - ctx: - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - speculative_config: - decoding_type: Eagle - max_draft_len: 3 - eagle3_one_model: true - speculative_model: Qwen3/qwen3-235B-eagle3 diff --git a/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml index 0701596a0bad..9e9dd0877c90 100644 --- a/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml @@ -80,6 +80,7 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: Eagle @@ -110,5 +111,6 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml deleted file mode 100644 index ad6b5d9a8c47..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,125 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k - accuracy: - datasets: - - dataset_name: gsm8k_local - expected_value: 0.9454 - threshold_type: hypothesis_test - filter_type: flexible-extract -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 03:00:00 - job_name: unified-benchmark - extra_args: "--gres=gpu:4" - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: true - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: "local-completions" - model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - enable_layerwise_nvtx_marker: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - enable_layerwise_nvtx_marker: true - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml deleted file mode 100644 index c455cc0644d4..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,126 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k - accuracy: - datasets: - - dataset_name: gpqa_diamond_local - expected_value: 0.65 - threshold_type: hypothesis_test - filter_type: strict-match -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 03:00:00 - job_name: unified-benchmark - extra_args: "--gres=gpu:4" - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: true - env_var: - HF_HOME: - tasks: - gpqa_diamond_local: - model: "local-chat-completions" - model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" - extra_kwargs: - apply_chat_template: true - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gpqa_diamond_local.yaml -worker_config: - gen: - enable_layerwise_nvtx_marker: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 16384 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 8448 - backend: NIXL - stream_interval: 100 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - enable_layerwise_nvtx_marker: true - max_batch_size: 4 - max_num_tokens: 8448 - max_seq_len: 8448 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 8448 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml deleted file mode 100644 index fa8fbbf1027e..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml +++ /dev/null @@ -1,102 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1024' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - enable_layerwise_nvtx_marker: true - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - ctx: - enable_layerwise_nvtx_marker: true - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse.yaml deleted file mode 100644 index 81d204655861..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse.yaml +++ /dev/null @@ -1,102 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1024' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - enable_layerwise_nvtx_marker: true - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: true - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - ctx: - enable_layerwise_nvtx_marker: true - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: true - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml deleted file mode 100644 index e0546168bf38..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml +++ /dev/null @@ -1,108 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - enable_layerwise_nvtx_marker: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - enable_layerwise_nvtx_marker: true - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml deleted file mode 100644 index 0e84cd8498ec..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml +++ /dev/null @@ -1,108 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - enable_layerwise_nvtx_marker: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: UCX - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - enable_layerwise_nvtx_marker: true - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: UCX - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml deleted file mode 100644 index 6061cd6f79cf..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml +++ /dev/null @@ -1,110 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k - config_index: 7 -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '12288' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - enable_layerwise_nvtx_marker: true - tensor_parallel_size: 48 - moe_expert_parallel_size: 48 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 1024 - max_num_tokens: 1024 - max_seq_len: 2176 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - dtype: fp8 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 8320 - backend: NIXL - stream_interval: 20 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - enable_layerwise_nvtx_marker: true - max_batch_size: 4 - max_num_tokens: 4480 - max_seq_len: 2176 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - cache_transceiver_config: - max_tokens_in_buffer: 8320 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml index 68f84616e696..95dd7805d24c 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml @@ -80,6 +80,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON stream_interval: 20 num_postprocess_workers: 4 speculative_config: @@ -104,6 +105,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON speculative_config: decoding_type: MTP num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml deleted file mode 100644 index f70a01669734..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL.yaml +++ /dev/null @@ -1,110 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k - config_index: 1 -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 04:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1024' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - ctx: - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - moe_config: - backend: CUTEDSL - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml deleted file mode 100644 index dce80cbd3c11..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml +++ /dev/null @@ -1,116 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k - config_index: 0 -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 04:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - moe_config: - backend: CUTEDSL - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml deleted file mode 100644 index 892ab01d13bf..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL.yaml +++ /dev/null @@ -1,115 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k - config_index: 7 -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 04:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '12288' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - tensor_parallel_size: 48 - moe_expert_parallel_size: 48 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 1024 - max_num_tokens: 1024 - max_seq_len: 2176 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8320 - backend: NIXL - stream_interval: 20 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - max_batch_size: 4 - max_num_tokens: 4480 - max_seq_len: 2176 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - moe_config: - backend: CUTEDSL - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 8320 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml index 1463bc140873..7cffd086d00f 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml @@ -87,6 +87,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON stream_interval: 20 num_postprocess_workers: 4 speculative_config: @@ -112,6 +113,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON speculative_config: decoding_type: MTP num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml deleted file mode 100644 index 06f5edadd981..000000000000 --- a/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,125 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k - accuracy: - datasets: - - dataset_name: gsm8k_local - expected_value: 0.9454 - threshold_type: hypothesis_test - filter_type: flexible-extract -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 03:00:00 - job_name: unified-benchmark - extra_args: "--gres=gpu:4" - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 20 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: true - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: "local-completions" - model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - enable_layerwise_nvtx_marker: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - enable_layerwise_nvtx_marker: true - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-NIXL.yaml index 4c78d385ab66..7054e26225e9 100644 --- a/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_wideep_stress-deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-NIXL.yaml @@ -97,6 +97,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON stream_interval: 20 num_postprocess_workers: 4 speculative_config: @@ -121,6 +122,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON speculative_config: decoding_type: MTP num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index a4f0a8a690d4..ae4d45435096 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con1_ctx1_pp4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -73,6 +73,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -100,5 +101,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index b967b8c64a3d..f3d9e4d5d8b7 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -17,7 +17,7 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 5 + multi_round: 3 benchmark_ratio: 0.0 streaming: true concurrency_list: '256' diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml index e905b6b9759e..8f3db7438f47 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml @@ -17,7 +17,7 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 5 + multi_round: 3 benchmark_ratio: 0.0 streaming: true concurrency_list: '256' diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 9c57f7c60524..322ea37dec46 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con64_ctx1_pp4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -75,6 +75,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml deleted file mode 100644 index 5e8a2dc5bfb1..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ /dev/null @@ -1,105 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '3072' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 768 - max_num_tokens: 1536 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 768 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml deleted file mode 100644 index 16bb6e78b923..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ /dev/null @@ -1,105 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '3072' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 768 - max_num_tokens: 1536 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 768 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index c26f1c2c5f20..a1d00838ed91 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 3bb488a84f18..85c61995b19e 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -72,6 +72,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -100,5 +101,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml deleted file mode 100644 index c437058dc4b7..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,116 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1024' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 32 - max_num_tokens: 128 - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 256 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml deleted file mode 100644 index bbe406f38964..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL.yaml +++ /dev/null @@ -1,113 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 1024 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml deleted file mode 100644 index b2bb6ab6fed2..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ /dev/null @@ -1,113 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 1024 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 785583ae10c6..ef25f632da2c 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -76,6 +76,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -113,5 +114,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index 3f9961739b88..9d55248aa617 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -80,6 +80,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 @@ -117,6 +118,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 80bd1609ab66..c0f31040f515 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -77,6 +77,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -113,5 +114,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index f5fd13315d4f..06d0ba1ff9d3 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -40,7 +40,7 @@ environment: # Env vars taken from the Dynamo deploy.yaml (NCCL MNNVL/CUMEM/NVLS, UCX MNNVL # IPC, MoE all-to-all without all-gather, PDL, GC disable) so the perf test # exercises the same code paths as the production deployment. - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=1 NVIDIA_GDRCOPY=1 UCX_CUDA_IPC_ENABLE_MNNVL=1 NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER=1 TRTLLM_FORCE_COMM_METHOD=NVLINK_TWO_SIDED ENABLE_CONFIGURABLE_MOE=1" + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=1 NVIDIA_GDRCOPY=1 UCX_CUDA_IPC_ENABLE_MNNVL=1 NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_MOE_ENABLE_ALLTOALL_WITHOUT_ALLGATHER=1 TRTLLM_FORCE_COMM_METHOD=NVLINK_TWO_SIDED" server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" profiling: nsys_on: false @@ -70,6 +70,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 num_postprocess_workers: 8 stream_interval: 10 @@ -98,4 +99,5 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + transceiver_runtime: PYTHON kv_transfer_timeout_ms: 600000 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index b152141dd661..63ccfef8aaf6 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -78,6 +78,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -112,5 +113,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 01726b0b4c3e..8924b184630e 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -73,6 +73,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -108,5 +109,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml index 872efcf4f66f..69e86d25e59f 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL.yaml @@ -78,6 +78,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -109,4 +110,5 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con1229_ctx7_dep4_gen1_dep8_eplb384_mtp3_ccb-NIXL.yaml similarity index 54% rename from tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con1229_ctx7_dep4_gen1_dep8_eplb384_mtp3_ccb-NIXL.yaml index 75b20315942c..f385d09babe4 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con1229_ctx7_dep4_gen1_dep8_eplb384_mtp3_ccb-NIXL.yaml @@ -1,11 +1,11 @@ metadata: - model_name: deepseek_r1_0528_fp4_v2 + model_name: deepseek_v4_pro_fp4 precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 + model_dir_name: DeepSeek-V4-Pro supported_gpus: - GB300 script_file: disaggr_torch.slurm - benchmark_type: 1k1k + benchmark_type: 8k1k slurm: script_file: disaggr_torch.slurm partition: @@ -20,13 +20,15 @@ benchmark: multi_round: 10 benchmark_ratio: 0.0 streaming: true - concurrency_list: '1' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json + concurrency_list: '1229' + input_length: 7392 + output_length: 922 + # Random synthetic dataset (no captured dataset file yet for DSv4-Pro); the + # placeholder falls through to test_perf_sanity.py's random-dataset path. + dataset_file: hardware: gpus_per_node: 4 - num_ctx_servers: 1 + num_ctx_servers: 7 num_gen_servers: 1 environment: container_mount: @@ -35,70 +37,70 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 "PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core" TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true max_batch_size: 128 max_num_tokens: 512 + max_seq_len: 9256 tensor_parallel_size: 8 moe_expert_parallel_size: 8 pipeline_parallel_size: 1 context_parallel_size: 1 - enable_attention_dp: false + enable_attention_dp: true + enable_lm_head_tp_in_adp: true cuda_graph_config: enable_padding: true - max_batch_size: 128 + batch_sizes: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128] kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.9 + free_gpu_memory_fraction: 0.8 dtype: fp8 + tokens_per_block: 128 moe_config: - backend: TRTLLM + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + load_balancer: tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep8_slots384.yaml cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL - disable_overlap_scheduler: true + transceiver_runtime: PYTHON + disable_overlap_scheduler: false speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 - stream_interval: 20 - allreduce_strategy: MNNVL + stream_interval: 100 ctx: print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 tensor_parallel_size: 4 moe_expert_parallel_size: 4 pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: true - enable_lm_head_tp_in_adp: true + enable_lm_head_tp_in_adp: false cuda_graph_config: null kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.5 dtype: fp8 + tokens_per_block: 128 moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true + backend: TRTLLM + load_balancer: tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con2_ctx1_dep4_gen5_tep4_eplb0_mtp3_ccb-NIXL.yaml similarity index 58% rename from tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con2_ctx1_dep4_gen5_tep4_eplb0_mtp3_ccb-NIXL.yaml index 4ed531013bec..20875695182d 100644 --- a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con2_ctx1_dep4_gen5_tep4_eplb0_mtp3_ccb-NIXL.yaml @@ -1,11 +1,11 @@ metadata: - model_name: glm_5_nvfp4 + model_name: deepseek_v4_pro_fp4 precision: fp4 - model_dir_name: GLM-5-NVFP4 + model_dir_name: DeepSeek-V4-Pro supported_gpus: - - GB200 + - GB300 script_file: disaggr_torch.slurm - benchmark_type: 1k1k + benchmark_type: 8k1k slurm: script_file: disaggr_torch.slurm partition: @@ -20,14 +20,16 @@ benchmark: multi_round: 10 benchmark_ratio: 0.0 streaming: true - concurrency_list: '1' - input_length: 1024 - output_length: 1024 + concurrency_list: '2' + input_length: 7456 + output_length: 923 + # Random synthetic dataset (no captured dataset file yet for DSv4-Pro); the + # placeholder falls through to test_perf_sanity.py's random-dataset path. dataset_file: hardware: gpus_per_node: 4 num_ctx_servers: 1 - num_gen_servers: 1 + num_gen_servers: 5 environment: container_mount: container_image: @@ -35,70 +37,68 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 "PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core" TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true - max_batch_size: 128 - max_num_tokens: 512 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 tensor_parallel_size: 4 moe_expert_parallel_size: 4 pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: false + enable_lm_head_tp_in_adp: false cuda_graph_config: enable_padding: true - max_batch_size: 128 + batch_sizes: [1, 2, 4] kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.85 + free_gpu_memory_fraction: 0.9 dtype: fp8 + tokens_per_block: 128 moe_config: backend: TRTLLM + use_low_precision_moe_combine: true cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL - disable_overlap_scheduler: true + transceiver_runtime: PYTHON + disable_overlap_scheduler: false speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 - stream_interval: 20 - allreduce_strategy: MNNVL + stream_interval: 100 ctx: print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 tensor_parallel_size: 4 moe_expert_parallel_size: 4 pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: true - enable_lm_head_tp_in_adp: true + enable_lm_head_tp_in_adp: false cuda_graph_config: null kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.5 dtype: fp8 + tokens_per_block: 128 moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true + backend: TRTLLM cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx8_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL.yaml similarity index 56% rename from tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx8_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL.yaml index 916584e71cbb..d51f201b7b02 100644 --- a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx8_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL.yaml @@ -1,11 +1,11 @@ metadata: - model_name: glm_5_nvfp4 + model_name: deepseek_v4_pro_fp4 precision: fp4 - model_dir_name: GLM-5-NVFP4 + model_dir_name: DeepSeek-V4-Pro supported_gpus: - - GB200 + - GB300 script_file: disaggr_torch.slurm - benchmark_type: 1k1k + benchmark_type: 8k1k slurm: script_file: disaggr_torch.slurm partition: @@ -20,13 +20,15 @@ benchmark: multi_round: 10 benchmark_ratio: 0.0 streaming: true - concurrency_list: '512' - input_length: 1024 - output_length: 1024 + concurrency_list: '666' + input_length: 7408 + output_length: 922 + # Random synthetic dataset (no captured dataset file yet for DSv4-Pro); the + # placeholder falls through to test_perf_sanity.py's random-dataset path. dataset_file: hardware: gpus_per_node: 4 - num_ctx_servers: 1 + num_ctx_servers: 8 num_gen_servers: 1 environment: container_mount: @@ -35,26 +37,17 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 "PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core" TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml worker_config: gen: print_iter_log: true max_batch_size: 16 max_num_tokens: 64 + max_seq_len: 9256 tensor_parallel_size: 32 moe_expert_parallel_size: 32 pipeline_parallel_size: 1 @@ -63,43 +56,51 @@ worker_config: enable_lm_head_tp_in_adp: true cuda_graph_config: enable_padding: true - max_batch_size: 16 + batch_sizes: [1, 2, 4, 8, 16] kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.7 dtype: fp8 + tokens_per_block: 128 moe_config: - backend: CUTEDSL + backend: MEGAMOE_DEEPGEMM use_low_precision_moe_combine: true + load_balancer: tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_gen_ep32_slots384.yaml cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL - disable_overlap_scheduler: true + transceiver_runtime: PYTHON + disable_overlap_scheduler: false speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 - stream_interval: 20 + stream_interval: 100 ctx: print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 tensor_parallel_size: 4 moe_expert_parallel_size: 4 pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: true - enable_lm_head_tp_in_adp: true + enable_lm_head_tp_in_adp: false cuda_graph_config: null kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.5 dtype: fp8 + tokens_per_block: 128 moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true + backend: TRTLLM + load_balancer: tests/scripts/perf/disaggregated/deepseek-v4-pro-eplb/moe_load_balancer_ctx_ep4_384.yaml cache_transceiver_config: - max_tokens_in_buffer: 16384 + max_tokens_in_buffer: 8192 + kv_transfer_timeout_ms: 600000 backend: NIXL + transceiver_runtime: PYTHON disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml deleted file mode 100644 index 58bc0b8bfa6c..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,104 +0,0 @@ -metadata: - model_name: glm_5_nvfp4 - precision: fp4 - model_dir_name: GLM-5-NVFP4 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '1' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 128 - max_num_tokens: 512 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: TRTLLM - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 - stream_interval: 20 - allreduce_strategy: MNNVL - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml deleted file mode 100644 index e5785b456ecb..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml +++ /dev/null @@ -1,108 +0,0 @@ -metadata: - model_name: glm_5_nvfp4 - precision: fp4 - model_dir_name: GLM-5-NVFP4 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '4096' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 1024 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 256 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml deleted file mode 100644 index 17c05eb03d04..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml +++ /dev/null @@ -1,108 +0,0 @@ -metadata: - model_name: glm_5_nvfp4 - precision: fp4 - model_dir_name: GLM-5-NVFP4 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '4096' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 1024 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 256 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml deleted file mode 100644 index e041b5b9b538..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,105 +0,0 @@ -metadata: - model_name: glm_5_nvfp4 - precision: fp4 - model_dir_name: GLM-5-NVFP4 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '512' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 64 - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 16384 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index 5dfd6bcc60b9..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,103 +0,0 @@ -metadata: - model_name: k25_thinking_fp4 - precision: fp4 - model_dir_name: Kimi-K2.5-NVFP4 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 64 - max_num_tokens: 128 - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 8192 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index 474f526110bd..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,103 +0,0 @@ -metadata: - model_name: k25_thinking_fp4 - precision: fp4 - model_dir_name: Kimi-K2.5-NVFP4 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '4096' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 512 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 8192 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml deleted file mode 100644 index 35413b5ad7b5..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml +++ /dev/null @@ -1,103 +0,0 @@ -metadata: - model_name: k25_thinking_fp4 - precision: fp4 - model_dir_name: Kimi-K2.5-NVFP4 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 5 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '4096' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 512 - max_num_tokens: 512 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - trust_remote_code: true - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 8192 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: UCX - disable_overlap_scheduler: true - trust_remote_code: true diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml deleted file mode 100644 index 20303670ffba..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL.yaml +++ /dev/null @@ -1,101 +0,0 @@ -metadata: - model_name: k25_thinking_fp4 - precision: fp4 - model_dir_name: Kimi-K2.5-NVFP4 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '4' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -accuracy: - enable_accuracy_test: false - env_var: - HF_HOME: - tasks: - gsm8k_local: - model: local-completions - model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 - extra_kwargs: - trust_remote_code: true - custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml -worker_config: - gen: - print_iter_log: true - max_batch_size: 4 - max_num_tokens: 128 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - moe_config: - backend: TRTLLM - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 16 - max_num_tokens: 8192 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 16384 - backend: NIXL - disable_overlap_scheduler: true - trust_remote_code: true diff --git a/tests/scripts/perf/disaggregated/gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml deleted file mode 100644 index e077a304f857..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL.yaml +++ /dev/null @@ -1,106 +0,0 @@ -metadata: - model_name: qwen3_235b_a22b_fp4 - precision: fp4 - model_dir_name: Qwen3-235B-A22B-FP4 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 8 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - dtype: fp8 - moe_config: - backend: CUTEDSL - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: Eagle - max_draft_len: 3 - eagle3_one_model: true - speculative_model: Qwen3/qwen3-235B-eagle3 - ctx: - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - speculative_config: - decoding_type: Eagle - max_draft_len: 3 - eagle3_one_model: true - speculative_model: Qwen3/qwen3-235B-eagle3 diff --git a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml deleted file mode 100644 index 885e3918f290..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml +++ /dev/null @@ -1,108 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - enable_layerwise_nvtx_marker: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - enable_layerwise_nvtx_marker: true - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml deleted file mode 100644 index 320fab5661eb..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX.yaml +++ /dev/null @@ -1,108 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - enable_layerwise_nvtx_marker: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: UCX - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - enable_layerwise_nvtx_marker: true - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: UCX - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml index 5d972ec794e0..a4cb411bd487 100644 --- a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml @@ -80,6 +80,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON stream_interval: 20 num_postprocess_workers: 4 speculative_config: @@ -104,6 +105,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON speculative_config: decoding_type: MTP num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml deleted file mode 100644 index 28b3d8f13c11..000000000000 --- a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL.yaml +++ /dev/null @@ -1,116 +0,0 @@ -metadata: - model_name: deepseek_v32_fp4 - precision: fp4 - model_dir_name: DeepSeek-V3.2-FP4-v2 - supported_gpus: - - GB300 - script_file: disaggr_torch.slurm - benchmark_type: 1k1k - config_index: 0 -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 04:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: gen_only - use_nv_sa_benchmark: false - multi_round: 1 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '2048' - input_length: 1024 - output_length: 1024 - dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 2 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 MIMALLOC_PURGE_DELAY=0 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 2251 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - print_iter_log: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - ctx: - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 2251 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - cuda_graph_config: null - disable_overlap_scheduler: true - moe_config: - backend: CUTEDSL - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: NIXL - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml index fca868eceb88..851d9f5ebb03 100644 --- a/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL.yaml @@ -87,6 +87,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON stream_interval: 20 num_postprocess_workers: 4 speculative_config: @@ -112,6 +113,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 8448 backend: NIXL + transceiver_runtime: PYTHON speculative_config: decoding_type: MTP num_nextn_predict_layers: 3 diff --git a/tests/test_common/_session_utils.py b/tests/test_common/_session_utils.py new file mode 100644 index 000000000000..1c5f0cd76d09 --- /dev/null +++ b/tests/test_common/_session_utils.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Helpers shared by the MPI session reuse and session prefetch layers. + +``session_reuse.py`` (keeps pools alive across tests) and +``session_prefetcher.py`` (spawns the next pool in the background) manage the +same object — a live ``MpiPoolSession`` handed to a test that did not spawn +it — so they share the same invariant: the worker-visible state a pool +freezes at spawn. Keeping it in one place means a fix applies to both layers. + +Policy stays in the layers: what to DO with a snapshot mismatch (discard vs +proceed) is each layer's own decision. +""" + +import os +import sys + +# Workers freeze the parent environment AND sys.path at spawn time, so a +# pool spawned earlier must not be handed to a test that changed either +# (silently stale env / unimportable monkeypatched modules). Process +# bookkeeping that legitimately drifts between tests is ignored; a false +# mismatch only costs one synchronous rebuild. +_ENV_IGNORE = frozenset( + { + "PYTEST_CURRENT_TEST", # changes every test phase by design + "COLUMNS", + "LINES", + "PWD", + "OLDPWD", + "SHLVL", + "_", + } +) + + +def _spawn_snapshot(): + """The worker-visible state a pool freezes at spawn: env + sys.path.""" + return ( + {k: v for k, v in os.environ.items() if k not in _ENV_IGNORE}, + list(sys.path), + ) + + +def _isinstance_transparent_shim(real_cls, factory): + """A seam replacement that intercepts construction but stays a real type. + + The pool-creation seams used to hold a plain FUNCTION in place of + ``MpiPoolSession``. Library code that does ``isinstance(x, + MpiPoolSession)`` against the patched module attribute then raises + ``TypeError: isinstance() arg 2 must be a type`` — proxy.py's + killed-worker detection added exactly such a check and every bare + ``LLM()`` creation failed until it was worked around with an + exclusion-based match. This shim removes the hazard for good: a real + class whose metaclass routes construction to ``factory`` and + instance/subclass checks to ``real_cls``, so both usage patterns keep + working — including consumers added after this layer was written. + """ + + class _SeamMeta(type): + def __call__(cls, *args, **kwargs): + return factory(*args, **kwargs) + + def __instancecheck__(cls, obj): + return isinstance(obj, real_cls) + + def __subclasscheck__(cls, sub): + return issubclass(sub, real_cls) + + def __repr__(cls): + return f"" + + return _SeamMeta("MpiPoolSession", (), {}) diff --git a/tests/test_common/error_utils.py b/tests/test_common/error_utils.py index 9cb57aaf88f7..74ead0bc31b7 100644 --- a/tests/test_common/error_utils.py +++ b/tests/test_common/error_utils.py @@ -12,6 +12,22 @@ "PMI2_Init failed to initialize", "OSError", ] +# Autotuner warmup intentionally probes tactics that can OOM and logs e.g. +# "[Autotuner] Single-pair run failed ... CUDA out of memory ...". Only that +# specific marker+OOM combination is benign: an autotuner-prefixed line with +# any other error keyword (e.g. "[Autotuner] RuntimeError: ...") is a real +# failure and must still be reported. +AUTOTUNER_MARKER = "[Autotuner]" +AUTOTUNER_BENIGN_TEXTS = [ + "out of memory", +] + + +def is_benign_line(line: str) -> bool: + """True for lines expected during a HEALTHY run despite an error keyword.""" + return AUTOTUNER_MARKER in line and any(text in line for text in AUTOTUNER_BENIGN_TEXTS) + + SLURM_LOG_TAIL_LINES = 200 # Number of lines to print from slurm job logs ERROR_CONTEXT_LINES = 100 # Number of lines to print before and after error line @@ -23,6 +39,8 @@ def check_error(file_path: str) -> list[tuple[int, str]]: error_lines = [] with open(file_path, "r", errors="replace") as f: for line_idx, line in enumerate(f, start=1): + if is_benign_line(line): + continue for keyword in ERROR_KEYWORDS: if keyword in line: error_lines.append((line_idx, line.strip())) @@ -45,17 +63,22 @@ def report_error( for log_file in log_files: if not os.path.exists(log_file): messages.append(f"Failed to read {log_file}: Path doesn't exist") + continue all_lines = None error_lines = [] try: with open(log_file, "r", errors="replace") as f: all_lines = f.readlines() - for line_idx, line in enumerate(f, start=1): - for keyword in ERROR_KEYWORDS: - if keyword in line: - error_lines.append((line_idx, line.strip())) - break + # Scan the buffered lines (iterating the exhausted file handle + # after readlines() would yield nothing). + for line_idx, line in enumerate(all_lines, start=1): + if is_benign_line(line): + continue + for keyword in ERROR_KEYWORDS: + if keyword in line: + error_lines.append((line_idx, line.strip())) + break except Exception as e: all_lines = None error_lines = [] @@ -74,7 +97,10 @@ def report_error( end_idx = min(len(all_lines), first_idx + ERROR_CONTEXT_LINES) context_lines = all_lines[start_idx:end_idx] messages.append("".join(context_lines)) - else: + if all_lines is not None: + # ALWAYS include the end of the log, even when a keyword matched + # above: the first keyword hit may be benign noise while the + # actual fatal error sits at the end of the file. tail_content = "".join(all_lines[-tail_lines:]) if all_lines else "(empty)" messages.append(f"--- {log_file} [last {tail_lines} lines] ---") messages.append(tail_content) diff --git a/tests/test_common/http_utils.py b/tests/test_common/http_utils.py index 9628dff1809f..9ed7953ab202 100644 --- a/tests/test_common/http_utils.py +++ b/tests/test_common/http_utils.py @@ -1,9 +1,47 @@ +import os import subprocess import time import requests -from test_common.error_utils import check_error +from test_common.error_utils import check_error, report_error + + +def _fail_with_server_logs(message: str, check_files: list[str] | None): + """Raise RuntimeError(message), appending the tail of each server log. + + A server that never becomes healthy often prints its story only to its own + log file; without dumping it here the CI log shows nothing but the client + poll loop, making the failure unclassifiable. + """ + if check_files: + # report_error raises RuntimeError(message + error context / log tails). + report_error(message, check_files) + raise RuntimeError(message) + + +def fail_if_proc_died( + proc: subprocess.Popen | None, + what: str, + check_files: list[str] | None = None, +): + """Event-driven fail-fast: raise (with server-log tails) if ``proc`` exited. + + Process death is an event, not a timeout: a babysitter that checks its + child converts a dead server into an immediate, diagnosable failure + instead of burning GPUs until a timeout. The resulting nonzero rank exit + ends that rank's SLURM step (``srun --kill-on-bad-exit=1`` in the perf + scripts); the remaining steps then fail fast on the dead endpoint via the + bounded ready-wait. + """ + if proc is None: + return + exit_code = proc.poll() + if exit_code is not None: + _fail_with_server_logs( + f"{what} exited unexpectedly with code {exit_code} while it was still needed.", + check_files, + ) def wait_for_endpoint_ready( @@ -11,28 +49,44 @@ def wait_for_endpoint_ready( timeout: int = 300, check_files: list[str] | None = None, server_proc: subprocess.Popen | None = None, + check_interval: float = 30.0, ): + """Poll ``url`` until it returns 200, failing fast and loudly otherwise. + + Fail-fast paths (all of which dump the tails of ``check_files`` so the + server-side story lands in the CI log): + - ``server_proc`` exited -> no point polling a dead server for the + remaining timeout; + - an error keyword appears in a ``check_files`` log (scanned every + ``check_interval`` seconds; time-based rather than iteration-based); + - ``timeout`` elapses without the endpoint becoming ready. + """ start = time.monotonic() - iteration = 0 + next_file_check = start + check_interval + missing_warned: set[str] = set() while time.monotonic() - start < timeout: # Check server_proc if provided (singular) - if server_proc is not None: - exit_code = server_proc.poll() - if exit_code is not None: - raise RuntimeError( - f"Server process exited with code {exit_code} before becoming ready." - ) - - iteration += 1 - if check_files and iteration % 300 == 0: + fail_if_proc_died(server_proc, "Server process (before becoming ready)", check_files) + + if check_files and time.monotonic() >= next_file_check: + next_file_check = time.monotonic() + check_interval for check_file in check_files: + if not os.path.exists(check_file): + if check_file not in missing_warned: + missing_warned.add(check_file) + print( + f"[WARNING] server log {check_file} does not exist " + "(yet?); cannot scan it for errors" + ) + continue error_lines = check_error(check_file) if error_lines: error_lines_str = ", ".join( [f"line {line_idx}: {line_str}" for line_idx, line_str in error_lines] ) - raise RuntimeError( - f"Found error in server file {check_file}: {error_lines_str}" + _fail_with_server_logs( + f"Found error in server file {check_file}: {error_lines_str}", + check_files, ) try: time.sleep(1) @@ -41,7 +95,9 @@ def wait_for_endpoint_ready( return except Exception as err: print(f"endpoint {url} is not ready, with exception: {err}") - raise RuntimeError(f"Endpoint {url} did not become ready within {timeout} seconds") + _fail_with_server_logs( + f"Endpoint {url} did not become ready within {timeout} seconds", check_files + ) def wait_for_endpoint_down(url: str, timeout: int = 300): diff --git a/tests/test_common/perf_metrics_utils.py b/tests/test_common/perf_metrics_utils.py index c63faa6d81f8..7ed5269d5c35 100644 --- a/tests/test_common/perf_metrics_utils.py +++ b/tests/test_common/perf_metrics_utils.py @@ -1,19 +1,81 @@ +import json +import time +from pathlib import Path + import requests -def get_timing_metrics(server_url: str): - response = requests.get(f"{server_url}/perf_metrics", timeout=10) - assert response.status_code == 200 - perf_metrics = response.json() - assert len(perf_metrics) > 0 - return perf_metrics[0] +def _read_perf_metrics_jsonl(output_dir): + records = [] + for path in Path(output_dir).glob("perf_metrics-*.jsonl"): + for line in path.read_text().splitlines(): + if not line: + continue + try: + records.append((path, json.loads(line))) + except json.JSONDecodeError: + continue + return records + + +def read_perf_metrics_jsonl(output_dir): + return [record for _, record in _read_perf_metrics_jsonl(output_dir)] + + +def wait_for_perf_metrics_jsonl(output_dir, expected_count: int, timeout: float = 10): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + records = read_perf_metrics_jsonl(output_dir) + if len(records) >= expected_count: + return records + time.sleep(0.1) + raise AssertionError( + f"Timed out waiting for {expected_count} performance metrics JSONL records" + ) + + +def get_timing_metrics(output_dir, timeout: float = 10): + """Read and join one completed disaggregated request from JSONL files.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + records = _read_perf_metrics_jsonl(output_dir) + by_request = {} + for path, record in records: + disagg_request_id = record.get("disagg_request_id") + server_kind = path.name.removeprefix("perf_metrics-").split("-", 1)[0] + if disagg_request_id is not None and server_kind: + by_request.setdefault(disagg_request_id, {})[server_kind] = record + + for records_by_kind in by_request.values(): + if not {"context", "generation", "disagg"} <= records_by_kind.keys(): + continue + ctx_record = records_by_kind["context"] + gen_record = records_by_kind["generation"] + disagg_record = records_by_kind["disagg"] + ctx_metrics = { + key: value + for key, value in ctx_record.items() + if key not in ("disagg_request_id", "status") + } + gen_metrics = { + key: value + for key, value in gen_record.items() + if key not in ("disagg_request_id", "status") + } + return { + **disagg_record, + "ctx_perf_metrics": ctx_metrics, + "gen_perf_metrics": gen_metrics, + } + time.sleep(0.1) + raise AssertionError("Timed out waiting for joined performance metrics JSONL records") def validate_timing_metrics(perf_metrics_item, request_context="", time_tolerance_seconds=0.005): """Helper function to validate timing metrics relationships. Args: - perf_metrics_item: A single performance metrics item from the /perf_metrics endpoint + perf_metrics_item: Joined context, generation, and disagg JSONL metrics request_context: String context for error messages (e.g., "request 1", "streaming") """ # Validate basic structure diff --git a/tests/test_common/s3_output.py b/tests/test_common/s3_output.py index 77ee959d0bd4..4f3f8f272770 100644 --- a/tests/test_common/s3_output.py +++ b/tests/test_common/s3_output.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import argparse +import io import logging import os import re @@ -20,6 +21,7 @@ import threading import time from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass from datetime import datetime import pytest @@ -41,6 +43,197 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) +@dataclass(frozen=True) +class FileSlice: + path: str + offset: int + size: int + + +class FileSliceReader(io.RawIOBase): + def __init__(self, file_slice): + self._slice = file_slice + self._file = open(file_slice.path, "rb", buffering=0) + self._position = 0 + self._file.seek(file_slice.offset) + + def readable(self): + return True + + def seekable(self): + return True + + def tell(self): + return self._position + + def seek(self, offset, whence=os.SEEK_SET): + if whence == os.SEEK_SET: + position = offset + elif whence == os.SEEK_CUR: + position = self._position + offset + elif whence == os.SEEK_END: + position = self._slice.size + offset + else: + raise ValueError(f"Unsupported whence: {whence}") + if position < 0: + raise ValueError("Negative seek position") + self._file.seek(self._slice.offset + position) + self._position = position + return position + + def readinto(self, buffer): + remaining = self._slice.size - self._position + if remaining <= 0: + return 0 + view = memoryview(buffer)[: min(len(buffer), remaining)] + read_size = self._file.readinto(view) + if read_size is None: + return 0 + self._position += read_size + return read_size + + def close(self): + if not self.closed: + self._file.close() + super().close() + + +class SessionFDSpool: + def __init__(self, target_fd, path): + self.target_fd = target_fd + self.path = path + self._saved_fd = None + self._spool_fd = None + self._attached = False + + def _flush_target_stream(self): + stream = sys.stdout if self.target_fd == 1 else sys.stderr + try: + stream.flush() + except (OSError, ValueError): + pass + + def start(self): + self._flush_target_stream() + self._saved_fd = os.dup(self.target_fd) + try: + self._spool_fd = os.open( + self.path, + os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_APPEND, + 0o600, + ) + os.dup2(self._spool_fd, self.target_fd, inheritable=True) + self._attached = True + except OSError: + if self._spool_fd is not None: + os.close(self._spool_fd) + self._spool_fd = None + os.close(self._saved_fd) + self._saved_fd = None + raise + + def suspend_parent(self): + if not self._attached: + return + self._flush_target_stream() + os.dup2(self._saved_fd, self.target_fd, inheritable=True) + self._attached = False + + def resume_parent(self): + if self._attached or self._spool_fd is None: + return + self._flush_target_stream() + os.dup2(self._spool_fd, self.target_fd, inheritable=True) + self._attached = True + + def snapshot(self): + self._flush_target_stream() + return os.fstat(self._spool_fd).st_size + + def stop(self): + self.suspend_parent() + if self._spool_fd is not None: + os.close(self._spool_fd) + self._spool_fd = None + if self._saved_fd is not None: + os.close(self._saved_fd) + self._saved_fd = None + + +class SessionCapture: + def __init__(self, output_path): + spool_dir = os.path.join(output_path, ".s3-spool") + os.makedirs(spool_dir, exist_ok=True) + suffix = f"{os.getpid()}-{time.time_ns()}" + self._spools = { + "stdout.log": SessionFDSpool(1, os.path.join(spool_dir, f"stdout-{suffix}.log")), + "stderr.log": SessionFDSpool(2, os.path.join(spool_dir, f"stderr-{suffix}.log")), + } + self._suspend_depth = 0 + self._started = False + + def start(self): + started = [] + try: + for spool in self._spools.values(): + spool.start() + started.append(spool) + except Exception: + for spool in reversed(started): + spool.stop() + raise + self._started = True + + def snapshot(self): + return {filename: spool.snapshot() for filename, spool in self._spools.items()} + + def slices_since(self, offsets): + current = self.snapshot() + return { + filename: FileSlice( + path=spool.path, + offset=offsets[filename], + size=max(0, current[filename] - offsets[filename]), + ) + for filename, spool in self._spools.items() + } + + def suspend_parent(self): + if not self._started: + return + self._suspend_depth += 1 + if self._suspend_depth == 1: + for spool in self._spools.values(): + spool.suspend_parent() + + def resume_parent(self): + if not self._started or self._suspend_depth == 0: + return + self._suspend_depth -= 1 + if self._suspend_depth == 0: + for spool in self._spools.values(): + spool.resume_parent() + + def stop(self): + if not self._started: + return + self._suspend_depth = 0 + for spool in self._spools.values(): + spool.stop() + self._started = False + + def remove_files(self): + for spool in self._spools.values(): + try: + os.remove(spool.path) + except FileNotFoundError: + pass + try: + os.rmdir(os.path.dirname(next(iter(self._spools.values())).path)) + except OSError: + pass + + class FDRedirector: def __init__( self, @@ -232,10 +425,11 @@ def __init__( output_path, echo_to_stdout=False, skip_upload=False, - capture_mode="timestamped", + capture_mode="session", upload_mode="sync", upload_workers=8, inline_output_max_bytes=256, + session_capture=None, ): self.upload_path = upload_path self.output_path = output_path @@ -250,8 +444,10 @@ def __init__( self.inline_output_max_bytes = inline_output_max_bytes if self.inline_output_max_bytes < 0: raise ValueError("--s3-inline-output-max-bytes must be >= 0") - if self.capture_mode == "direct" and self.echo_to_stdout: - raise ValueError("--s3-capture-mode=direct cannot be used with --s3-echo-stdout") + if self.capture_mode in ("session", "direct") and self.echo_to_stdout: + raise ValueError( + f"--s3-capture-mode={self.capture_mode} cannot be used with --s3-echo-stdout" + ) self.s3 = None if not self.skip_upload: try: @@ -266,15 +462,14 @@ def __init__( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, ) - # nodeid -> dict of open capture context managers + handler. - # Capture must span setup + call (so fixture output is recorded) but - # not teardown — closing the capture before teardown lets us add - # upload metadata inside pytest_runtest_logreport(teardown), where - # `report.sections.append(...)` is still seen by the junitxml plugin - # (pluggy LIFO order makes our handler fire before junitxml's). + # nodeid -> dict of open capture state + logging handler. Session mode + # spans setup, call, and teardown; legacy modes close after the call. self._active_capture: dict = {} self._test_names: dict = {} self._deferred_uploads = [] + self._captured_slices = {} + self._session_capture = session_capture + self._upload_failed = False def normalize_test_name(self, nodeid): import hashlib @@ -288,7 +483,7 @@ def normalize_test_name(self, nodeid): return f"{test_name}-{suffix}-{timestamp}" def _open_capture(self, item): - """Open FDRedirector + logging capture for ``item``. + """Open stdout/stderr and logging capture for ``item``. Returns a state dict on success, or ``None`` if setup failed (in which case the test runs uncaptured). Never propagates exceptions. @@ -300,8 +495,6 @@ def _open_capture(self, item): output_path = os.path.join(self.output_path, test_name) os.makedirs(output_path, exist_ok=True) - stdout_file = os.path.join(output_path, "stdout.log") - stderr_file = os.path.join(output_path, "stderr.log") log_file = os.path.join(output_path, "logging.log") log_date_format = item.config.getini("log_date_format") @@ -322,26 +515,33 @@ def _open_capture(self, item): handler.setFormatter(formatter) state["handler"] = handler - fd_kwargs = {} - if log_date_format: - fd_kwargs["date_format"] = log_date_format - if timestamp_format: - fd_kwargs["timestamp_format"] = timestamp_format - - if self.capture_mode == "direct": - state["stdout_redir"] = DirectFDRedirector(1, stdout_file) + if self.capture_mode == "session": + if self._session_capture is None: + raise RuntimeError("Session capture has not started") + state["session_offsets"] = self._session_capture.snapshot() else: - state["stdout_redir"] = FDRedirector( - 1, stdout_file, echo_to_original=self.echo_to_stdout, **fd_kwargs - ) - state["stdout_redir"].__enter__() - if self.capture_mode == "direct": - state["stderr_redir"] = DirectFDRedirector(2, stderr_file) - else: - state["stderr_redir"] = FDRedirector( - 2, stderr_file, echo_to_original=self.echo_to_stdout, **fd_kwargs - ) - state["stderr_redir"].__enter__() + stdout_file = os.path.join(output_path, "stdout.log") + stderr_file = os.path.join(output_path, "stderr.log") + fd_kwargs = {} + if log_date_format: + fd_kwargs["date_format"] = log_date_format + if timestamp_format: + fd_kwargs["timestamp_format"] = timestamp_format + + if self.capture_mode == "direct": + state["stdout_redir"] = DirectFDRedirector(1, stdout_file) + else: + state["stdout_redir"] = FDRedirector( + 1, stdout_file, echo_to_original=self.echo_to_stdout, **fd_kwargs + ) + state["stdout_redir"].__enter__() + if self.capture_mode == "direct": + state["stderr_redir"] = DirectFDRedirector(2, stderr_file) + else: + state["stderr_redir"] = FDRedirector( + 2, stderr_file, echo_to_original=self.echo_to_stdout, **fd_kwargs + ) + state["stderr_redir"].__enter__() state["log_cm"] = catching_logs(handler) state["log_cm"].__enter__() return state @@ -356,6 +556,14 @@ def _close_capture(self, state): """Close capture state opened by ``_open_capture``. Never raises.""" if not state: return + session_offsets = state.get("session_offsets") + if session_offsets is not None and self._session_capture is not None: + try: + slices = self._session_capture.slices_since(session_offsets) + for filename, file_slice in slices.items(): + self._captured_slices[(state["test_name"], filename)] = file_slice + except (OSError, ValueError) as e: + logger.warning("Error finalizing session capture: %s", e) # Close in reverse order so fd 1/2 are restored before logging stops. for key in ("log_cm", "stderr_redir", "stdout_redir"): cm = state.get(key) @@ -380,19 +588,101 @@ def pytest_runtest_setup(self, item): self._test_names[item.nodeid] = state["test_name"] yield + @pytest.hookimpl(wrapper=True) + def pytest_sessionstart(self, session): + result = yield + if self.capture_mode == "session": + if self._session_capture is None: + self._session_capture = SessionCapture(self.output_path) + self._session_capture.start() + else: + self._session_capture.resume_parent() + return result + + @pytest.hookimpl(wrapper=True) + def pytest_runtest_makereport(self, item, call): + report = yield + if self.capture_mode != "session" and ( + report.when == "call" or (report.when == "setup" and report.outcome != "passed") + ): + self._close_capture(self._active_capture.pop(item.nodeid, None)) + return report + @pytest.hookimpl(wrapper=True) def pytest_runtest_teardown(self, item, nextitem): - # Stop capturing BEFORE teardown runs, so the captured log files are - # final by the time we upload in pytest_runtest_logreport(teardown). - self._close_capture(self._active_capture.pop(item.nodeid, None)) + if self.capture_mode == "session": + try: + return (yield) + finally: + self._close_capture(self._active_capture.pop(item.nodeid, None)) + + # Fallback for custom or interrupted runtest protocols that did not + # produce a call report. Normal execution closes capture earlier. + state = self._active_capture.pop(item.nodeid, None) + self._close_capture(state) + if state is not None: + logger.warning( + "S3 capture for %r remained active until teardown; " + "pytest result/progress output may have been captured", + item.nodeid, + ) yield + def _suspend_session_capture(self): + if self._session_capture is not None: + self._session_capture.suspend_parent() + + def _resume_session_capture(self): + if self._session_capture is not None: + self._session_capture.resume_parent() + + @pytest.hookimpl(wrapper=True) + def pytest_runtest_logstart(self, nodeid, location): + self._suspend_session_capture() + try: + return (yield) + finally: + self._resume_session_capture() + + @pytest.hookimpl(wrapper=True) + def pytest_runtest_logfinish(self, nodeid, location): + self._suspend_session_capture() + try: + return (yield) + finally: + self._resume_session_capture() + def get_file_size(self, path): try: return os.path.getsize(path) except FileNotFoundError: return None + def _capture_source(self, test_name, filename): + file_slice = self._captured_slices.pop((test_name, filename), None) + if file_slice is not None: + return file_slice + return os.path.join(self.output_path, test_name, filename) + + def _source_exists(self, source): + if isinstance(source, FileSlice): + return os.path.exists(source.path) + return os.path.exists(source) + + def _source_size(self, source): + if isinstance(source, FileSlice): + return source.size + return os.path.getsize(source) + + def _open_source(self, source): + if isinstance(source, FileSlice): + return io.BufferedReader(FileSliceReader(source)) + return open(source, "rb") + + def _remove_source(self, source): + if not isinstance(source, FileSlice): + self._remove_local_log_file(source) + def _object_key(self, test_name, filename): return os.path.join(self.upload_path, test_name, filename) @@ -404,20 +694,30 @@ def _file_url(self, object_key): object_key, ) - def _upload_file(self, filepath, object_key): - self.s3.upload_file( - filepath, - self.bucket, - object_key, - ExtraArgs={"ContentType": "text/plain"}, - ) + def _upload_source(self, source, object_key): + extra_args = {"ContentType": "text/plain"} + if isinstance(source, FileSlice): + with self._open_source(source) as source_file: + self.s3.upload_fileobj( + source_file, + self.bucket, + object_key, + ExtraArgs=extra_args, + ) + else: + self.s3.upload_file( + source, + self.bucket, + object_key, + ExtraArgs=extra_args, + ) - def _append_upload_failed(self, report, section_name, filepath, filesize, error): - with open(filepath, "r", encoding="utf-8") as f: + def _append_upload_failed(self, report, section_name, source, filesize, error): + with self._open_source(source) as source_file: limit = 65536 # Limit content to 64k (65536 bytes) trail_content = "... [truncated]" - content = f.read(limit + 1) + content = source_file.read(limit + 1).decode("utf-8", errors="replace") if len(content) > limit: content = content[: limit - len(trail_content)] + trail_content report.sections.append( @@ -430,9 +730,9 @@ def _append_upload_failed(self, report, section_name, filepath, filesize, error) def _should_inline_output(self, filename, filesize): return filename in ("stdout.log", "stderr.log") and filesize < self.inline_output_max_bytes - def _append_inline_output(self, report, section_name, filepath): - with open(filepath, "rb") as f: - content = f.read().decode("utf-8", errors="replace") + def _append_inline_output(self, report, section_name, source): + with self._open_source(source) as source_file: + content = source_file.read().decode("utf-8", errors="replace") report.sections.append((section_name, content)) def _remove_local_log_file(self, filepath): @@ -454,18 +754,18 @@ def _remove_local_log_file(self, filepath): parent = os.path.dirname(parent) def upload_and_report(self, report, test_name, filename, section_name): - filepath = os.path.join(self.output_path, test_name, filename) - if not os.path.exists(filepath): + source = self._capture_source(test_name, filename) + if not self._source_exists(source): report.sections.append((section_name, "")) return - filesize = os.path.getsize(filepath) + filesize = self._source_size(source) if filesize == 0: report.sections.append((section_name, "")) - self._remove_local_log_file(filepath) + self._remove_source(source) return if self._should_inline_output(filename, filesize): - self._append_inline_output(report, section_name, filepath) - self._remove_local_log_file(filepath) + self._append_inline_output(report, section_name, source) + self._remove_source(source) return object_key = self._object_key(test_name, filename) fileurl = self._file_url(object_key) @@ -480,7 +780,7 @@ def upload_and_report(self, report, test_name, filename, section_name): ) return if self.upload_mode == "deferred": - self._deferred_uploads.append((filepath, object_key, test_name, filename)) + self._deferred_uploads.append((source, object_key, test_name, filename)) report.sections.append( ( section_name, @@ -489,67 +789,89 @@ def upload_and_report(self, report, test_name, filename, section_name): ) return try: - self._upload_file(filepath, object_key) + self._upload_source(source, object_key) report.sections.append( ( section_name, f"{filesize} bytes uploaded to {fileurl}", ) ) - self._remove_local_log_file(filepath) + self._remove_source(source) except Exception as e: + self._upload_failed = True logger.warning( f"Upload failed. test_name: {test_name}, filename: {filename}, error: {e}" ) - self._append_upload_failed(report, section_name, filepath, filesize, e) + self._append_upload_failed(report, section_name, source, filesize, e) + @pytest.hookimpl(wrapper=True) def pytest_runtest_logreport(self, report): - if report.when == "teardown": - test_name = self._test_names.pop(report.nodeid, None) - if test_name is None: - test_name = self.normalize_test_name(report.nodeid) - # Add S3 report sections here so they are visible to the junitxml - # plugin's logreport handler. Pluggy hook order is LIFO; this - # plugin is registered after junitxml, so our handler runs first. - self.upload_and_report(report, test_name, "stdout.log", "Captured stdout") - self.upload_and_report(report, test_name, "stderr.log", "Captured stderr") - self.upload_and_report(report, test_name, "logging.log", "Captured log") + self._suspend_session_capture() + try: + if report.when == "teardown": + test_name = self._test_names.pop(report.nodeid, None) + if test_name is None: + test_name = self.normalize_test_name(report.nodeid) + # Add S3 report sections before the junitxml plugin handles + # this report, while pytest's terminal output bypasses capture. + self.upload_and_report(report, test_name, "stdout.log", "Captured stdout") + self.upload_and_report(report, test_name, "stderr.log", "Captured stderr") + self.upload_and_report(report, test_name, "logging.log", "Captured log") + return (yield) + finally: + self._resume_session_capture() + @pytest.hookimpl(tryfirst=True) def pytest_sessionfinish(self, session, exitstatus): - if self.skip_upload or not self._deferred_uploads: - return - workers = min(self.upload_workers, len(self._deferred_uploads)) - logger.info( - "Uploading %d deferred S3 test log files with %d workers", - len(self._deferred_uploads), - workers, - ) - with ThreadPoolExecutor(max_workers=workers) as executor: - futures = { - executor.submit(self._upload_file, filepath, object_key): ( - filepath, - object_key, - test_name, - filename, - ) - for filepath, object_key, test_name, filename in self._deferred_uploads - } - for future in as_completed(futures): - filepath, object_key, test_name, filename = futures[future] - try: - future.result() - self._remove_local_log_file(filepath) - except Exception as e: - logger.warning( - "Deferred upload failed. test_name: %s, filename: %s, " - "object_key: %s, filepath: %s, error: %s", + for state in self._active_capture.values(): + self._close_capture(state) + self._active_capture.clear() + + if self._session_capture is not None: + self._session_capture.stop() + + if not self.skip_upload and self._deferred_uploads: + workers = min(self.upload_workers, len(self._deferred_uploads)) + logger.info( + "Uploading %d deferred S3 test log files with %d workers", + len(self._deferred_uploads), + workers, + ) + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = { + executor.submit(self._upload_source, source, object_key): ( + source, + object_key, test_name, filename, - object_key, - filepath, - e, ) - self._deferred_uploads.clear() + for source, object_key, test_name, filename in self._deferred_uploads + } + for future in as_completed(futures): + source, object_key, test_name, filename = futures[future] + try: + future.result() + self._remove_source(source) + except Exception as e: + self._upload_failed = True + logger.warning( + "Deferred upload failed. test_name: %s, filename: %s, " + "object_key: %s, source: %s, error: %s", + test_name, + filename, + object_key, + source, + e, + ) + self._deferred_uploads.clear() + + if ( + self._session_capture is not None + and not self.skip_upload + and not self._upload_failed + and not self._captured_slices + ): + self._session_capture.remove_files() def add_options(parser): @@ -594,8 +916,8 @@ def add_options(parser): action="store_true", default=False, help="Besides capturing stdout/stderr to per-test log files, also echo " - "them through to the original stdout/stderr (e.g. so progress stays " - "visible in the CI console). Should be set on the outer pytest " + "them through to the original stdout/stderr for live debugging. " + "This requires --s3-capture-mode=timestamped and should be set on the outer pytest " "invocation; nested pytest invocations spawned by individual tests " "should NOT set this, to avoid duplicating their output back through " "the outer pipe.", @@ -612,12 +934,11 @@ def add_options(parser): parser.addoption( "--s3-capture-mode", action="store", - choices=("timestamped", "direct"), - default="timestamped", - help="Capture stdout/stderr through the timestamped pipe reader, or " - "redirect file descriptors directly to files. Direct mode is intended " - "for nested pytest runs with many short cases and does not support " - "--s3-echo-stdout.", + choices=("session", "timestamped", "direct"), + default="session", + help="Capture each stream through a session-scoped append-only file and " + "split it into per-test ranges. Timestamped and direct retain the legacy " + "per-test FD redirection modes.", ) parser.addoption( "--s3-upload-mode", @@ -647,14 +968,14 @@ def add_options(parser): ) -def register_plugin(config): +def register_plugin(config, session_capture=None): """Register UploadLogPlugin if --s3-upload-path and --output-dir are both set.""" s3_upload_path = config.getoption("--s3-upload-path", default=None) output_dir = config.getoption("--output-dir", default=None) if not (s3_upload_path and output_dir): - return - capture_mode = config.getoption("capture", default="no") - if capture_mode != "no": + return None + pytest_capture_mode = config.getoption("capture", default="no") + if pytest_capture_mode != "no": raise ValueError("capture mode must be 'no' when upload path is specified") s3_secret_key = config.getoption("--s3-secret-key") skip_upload = config.getoption("--s3-skip-upload", default=False) @@ -662,6 +983,7 @@ def register_plugin(config): raise ValueError( "--s3-secret-key (or S3_SECRET_KEY env var) is required when --s3-upload-path is set" ) + s3_capture_mode = config.getoption("--s3-capture-mode", default="session") plugin = UploadLogPlugin( endpoint_url=config.getoption("--s3-endpoint"), aws_access_key_id=config.getoption("--s3-username"), @@ -671,9 +993,11 @@ def register_plugin(config): output_path=output_dir, echo_to_stdout=config.getoption("--s3-echo-stdout", default=False), skip_upload=skip_upload, - capture_mode=config.getoption("--s3-capture-mode", default="timestamped"), + capture_mode=s3_capture_mode, upload_mode=config.getoption("--s3-upload-mode", default="sync"), upload_workers=config.getoption("--s3-upload-workers", default=8), inline_output_max_bytes=config.getoption("--s3-inline-output-max-bytes", default=256), + session_capture=session_capture if s3_capture_mode == "session" else None, ) config.pluginmanager.register(plugin, "upload_log_plugin") + return plugin diff --git a/tests/test_common/s3_output_hooks.py b/tests/test_common/s3_output_hooks.py new file mode 100644 index 000000000000..ccaf34f6ea02 --- /dev/null +++ b/tests/test_common/s3_output_hooks.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Start S3 FD capture before pytest imports the initial conftests.""" + +import argparse +import os +from collections.abc import Generator +from dataclasses import dataclass +from typing import cast + +import pytest + +from test_common import s3_output + +_CAPTURE_STATE_ATTRIBUTE = "_s3_output_early_capture_state" + + +@dataclass +class _EarlyCaptureState: + capture: s3_output.SessionCapture + claimed: bool = False + + def cleanup(self) -> None: + self.capture.stop() + if not self.claimed: + self.capture.remove_files() + + +def _is_xdist_worker(config: pytest.Config) -> bool: + return bool(os.environ.get("PYTEST_XDIST_WORKER")) or hasattr(config, "workerinput") + + +def _is_xdist_controller(config: pytest.Config) -> bool: + if _is_xdist_worker(config): + return False + namespace = getattr(config, "known_args_namespace", config.option) + num_processes = getattr(namespace, "numprocesses", None) + if num_processes in (None, 0, "0"): + num_processes = getattr(config.option, "numprocesses", None) + return num_processes not in (None, 0, "0") + + +def _parse_early_capture_options( + early_config: pytest.Config, args: list[str] +) -> tuple[str | None, str | None, str, str]: + parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False) + parser.add_argument("--output-dir", "-O") + parser.add_argument("--s3-upload-path") + parser.add_argument("--s3-capture-mode") + parsed, _ = parser.parse_known_args(args) + + namespace = early_config.known_args_namespace + output_path = cast( + str | None, + getattr(namespace, "output_dir", None) or parsed.output_dir, + ) + upload_path = cast( + str | None, + getattr(namespace, "s3_upload_path", None) + or parsed.s3_upload_path + or os.environ.get("S3_UPLOAD_PATH"), + ) + capture_mode = cast( + str, + getattr(namespace, "s3_capture_mode", None) or parsed.s3_capture_mode or "session", + ) + pytest_capture = cast(str, getattr(namespace, "capture", "fd")) + return output_path, upload_path, capture_mode, pytest_capture + + +def _capture_state(config: pytest.Config) -> _EarlyCaptureState | None: + return cast( + _EarlyCaptureState | None, + getattr(config, _CAPTURE_STATE_ATTRIBUTE, None), + ) + + +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_load_initial_conftests( + early_config: pytest.Config, args: list[str] +) -> Generator[None, object, object]: + output_path, upload_path, capture_mode, pytest_capture = _parse_early_capture_options( + early_config, args + ) + if ( + _capture_state(early_config) is not None + or _is_xdist_controller(early_config) + or not output_path + or not upload_path + or capture_mode != "session" + or pytest_capture != "no" + ): + return (yield) + + capture = s3_output.SessionCapture(output_path) + capture.start() + state = _EarlyCaptureState(capture) + setattr(early_config, _CAPTURE_STATE_ATTRIBUTE, state) + early_config.add_cleanup(state.cleanup) + try: + return (yield) + finally: + # MPI runtimes initialized while conftests load retain the spool FD. + # Restore the parent so pytest configuration output stays visible. + capture.suspend_parent() + + +@pytest.hookimpl(trylast=True) +def pytest_configure(config: pytest.Config) -> None: + if _is_xdist_controller(config): + return + state = _capture_state(config) + session_capture = state.capture if state is not None else None + plugin = s3_output.register_plugin(config, session_capture=session_capture) + if plugin is not None and state is not None: + state.claimed = True diff --git a/tests/test_common/session_prefetcher.py b/tests/test_common/session_prefetcher.py new file mode 100644 index 000000000000..188a6aa70cc3 --- /dev/null +++ b/tests/test_common/session_prefetcher.py @@ -0,0 +1,609 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Background prefetch of the NEXT test's MPI session — zero test changes. + +Multi-GPU LLM-API tests pay ~50-65s per bare ``LLM(...)`` to spawn an MPI +pool whose workers import ``tensorrt_llm``. That is pure CPU/IO work, so +while the CURRENT test runs on the GPUs a spare pool for the next test can +be spawned in a background thread — hiding the spawn cost behind the +previous test's runtime. Prefetched workers run no kernels and allocate +nothing before handover; depending on the library version, importing +tensorrt_llm may leave an idle CUDA context (~a few hundred MB), which +safely coexists with the running test. + +Mechanism (wired by ``tests/test_common/session_prefetcher_hooks.py``, +loaded from each test tree's top-level conftest): ``pytest_runtest_setup`` +lazily patches the library seams that construct ``MpiPoolSession`` for a +bare ``LLM(...)`` with a factory that (a) hands over the prefetched pool +when its size and spawn-time env/sys.path still match, and (b) re-arms a +spare pool of the same size for the next test. A miss falls back to the +normal synchronous spawn, so a wrong prefetch can only cost time, never +correctness. + +Weight page-cache warming: when the NEXT test's model differs from the +current one, its weight files are read in a background thread so the kernel +page cache is hot by the time that test loads weights. The next model is +discovered automatically from the accuracy-harness ``MODEL_PATH`` class +attribute or a ``model_folder``-style test parameter (modeling unit tests), +or declared explicitly with +``@pytest.mark.prefetch_model_dir("/path/to/model")``. This complements +pool prefetch (pool reuse does not cover model IO). Page cache is +reclaimable memory, so warming cannot OOM the host; a wasted warm (test +skipped or reordered) costs only IO bandwidth. + +Coexistence with MPI session reuse: when ``test_common/session_reuse.py`` is +wired and enabled it owns the same pool-creation seams and eliminates the +respawn outright, so the prefetcher automatically stays off the seams (see +``_reuse_layer_active``). Weight warming stays active, and reuse consumes +this layer's shadow pools on its cache misses (first pool of a size, +post-drain rebuild, post-retire replacement) via ``take``/``schedule_shadow`` +— the two layers compose: reuse covers the steady state, prefetch covers the +misses. + +Enabled by default; ``TRTLLM_TEST_PREFETCH_SESSION=0`` disables BOTH pool +prefetch and weight warming (one kill switch for the whole plugin). Suites +that never import tensorrt_llm's executor modules pay nothing — not even +the tensorrt_llm import. +""" + +import glob +import math +import os +import sys +import threading +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from typing import NamedTuple + +# The spawn snapshot is shared with the session-reuse layer (both hand a +# live pool to a test that did not spawn it — same invariant: workers freeze +# the FULL env + sys.path at spawn). +from test_common._session_utils import _isinstance_transparent_shim, _spawn_snapshot + +# The only places in the library that construct MpiPoolSession for a bare +# LLM(...); tests passing their own _mpi_session never reach these lines. +# test_patch_targets_cover_all_library_construction_sites keeps this list +# honest against new construction sites appearing in the library. +_PATCH_TARGETS = ( + "tensorrt_llm.executor.proxy", + "tensorrt_llm.executor.rpc_proxy", + "tensorrt_llm.llmapi.llm", +) + +# Identity collection is followed by one lightweight diagnostic submitted to +# the workers. Keep enough room for that hand-off after the lower-level +# bootstrap deadline expires. +_SHADOW_BUILD_FINISH_GRACE = 30.0 +_FALLBACK_IDENTITY_TIMEOUT = 300.0 + + +def _fallback_identity_timeout() -> float: + """Mirror the lower-level env contract while its module is still loading.""" + raw = os.environ.get("TRTLLM_MPI_IDENTITY_TIMEOUT") + if raw: + try: + value = float(raw) + if math.isfinite(value) and value > 0: + return value + except ValueError: + pass + return _FALLBACK_IDENTITY_TIMEOUT + + +def _shadow_build_wait_timeout() -> float: + """Upper-level wait budget derived from the MPI bootstrap deadline. + + Do not import TensorRT-LLM here: this plugin must stay usable by pure-logic + tests and suites without built bindings. A real shadow build imports + ``mpi_session`` before it can construct a pool, so the live lower-level + setting is present by the time ``take()`` waits on that build. + """ + mpi_session = sys.modules.get("tensorrt_llm.llmapi.mpi_session") + timeout_fn = getattr(mpi_session, "_identity_barrier_timeout", None) + identity_timeout = timeout_fn() if timeout_fn is not None else _fallback_identity_timeout() + return identity_timeout + _SHADOW_BUILD_FINISH_GRACE + + +def _reuse_layer_active() -> bool: + """True when the MPI session-reuse layer owns the pool-creation seams. + + ``test_common.session_reuse`` keeps pools alive across tests at the SAME + seams this module would patch, and saves the whole respawn rather than + just hiding it — strictly better where it applies. When it is wired and + enabled, the prefetcher must stay off the seams so the two factories + don't fight over them (whoever patches first would silently disable the + other). Weight page-cache warming is orthogonal and stays on either way. + """ + mod = sys.modules.get("test_common.session_reuse") + if mod is None: + return False # not wired into this suite: seams are ours to patch + try: + return bool(mod.REUSE.is_active()) + except Exception: + return True # loaded but unreadable: err on staying out of the way + + +_READ_CHUNK = 64 << 20 # 64MB + + +def _weight_files(model_dir: str): + """The weight files the loader will actually read, in loader order. + + Mirrors HfWeightLoader.load_weights' selection: safetensors first — + minus "consolidated" copies, which the loader deliberately skips (they + duplicate the shards and can be enormous) — else *.bin, else *.pth. + Warming anything else is pure wasted IO. + """ + files = [ + f + for f in glob.glob(os.path.join(model_dir, "*.safetensors")) + if "consolidated" not in os.path.basename(f) + ] + for fallback in ("*.bin", "*.pth"): + if files: + break + files = glob.glob(os.path.join(model_dir, fallback)) + return sorted(files) + + +def _available_host_memory(): + """MemAvailable from /proc/meminfo in bytes, or None when unreadable.""" + try: + with open("/proc/meminfo") as fh: + for line in fh: + if line.startswith("MemAvailable:"): + return int(line.split()[1]) * 1024 + except (OSError, ValueError, IndexError): + pass + return None + + +# Parametrized-test convention: the model lives in a parameter with one of +# these names (e.g. test_modeling_* files), holding either an absolute path +# or a directory name under LLM_MODELS_ROOT. +_MODEL_PARAM_NAMES = ("model_folder", "model_dir", "model_path") + + +def _models_root(): + """The models root the tests themselves resolve against, or None.""" + root = os.environ.get("LLM_MODELS_ROOT") + if root: + return root + try: # same fallback the test suites use (CI default scratch path) + from test_common.llm_data import llm_models_root + + root = llm_models_root() + return str(root) if root else None + except Exception: + return None + + +def _model_dir_of(item): + """A test item's model dir: marker, else class or parameter convention. + + Discovery order: the explicit ``prefetch_model_dir`` marker; the accuracy + harness's ``MODEL_PATH`` class attribute (120+ classes across + tests/integration/defs/accuracy); a ``model_folder``-style test parameter + (modeling unit tests), resolved under LLM_MODELS_ROOT unless absolute. + All of it is guess-tolerant: a value that is not a real directory of + weight files (e.g. an HF model id) makes ``warm_page_cache`` a silent + no-op, so a wrong guess costs nothing. + """ + marker = item.get_closest_marker("prefetch_model_dir") + if marker is not None and marker.args: + return marker.args[0] + model_path = getattr(getattr(item, "cls", None), "MODEL_PATH", None) + if isinstance(model_path, str): + return model_path + params = getattr(getattr(item, "callspec", None), "params", None) or {} + for name in _MODEL_PARAM_NAMES: + value = params.get(name) + if isinstance(value, str) and value: + if os.path.isabs(value): + return value + root = _models_root() + if root: + return os.path.join(root, value) + return None + + +def warm_page_cache(model_dir: str) -> float: + """Read ``model_dir``'s weight files to keep them in the OS page cache. + + The next LLM create then loads the weights from RAM, not disk. Pure file + IO — never touches CUDA, safe to run while another test owns the GPUs. + Returns the number of GiB read. + """ + files = _weight_files(model_dir) + if not files: + return 0.0 # not a local weight dir (e.g. an HF model id): nothing to warm + total_bytes = sum(os.stat(f).st_size for f in files) + available = _available_host_memory() + if available is not None and total_bytes > available: + # Larger than RAM: pages would be evicted before the test loads them — + # pure filer traffic with zero benefit (e.g. multi-hundred-GB models). + print( + f"[session-prefetch] skipping warm of {model_dir}: {total_bytes >> 30} GiB " + f"exceeds available host memory ({available >> 30} GiB)", + flush=True, + ) + return 0.0 + t0 = time.monotonic() + + def _read(path): + n = 0 + with open(path, "rb") as fh: + while True: + chunk = fh.read(_READ_CHUNK) + if not chunk: + return n + n += len(chunk) + + # thread_name_prefix keeps the IO workers inside the pytest.ini + # threadleak_exclude pattern (session-prefetch-\w+): a large warm can + # legitimately still be reading during the next test's threadleak check. + with ThreadPoolExecutor(max_workers=4, thread_name_prefix="session-prefetch-io") as ex: + total = sum(ex.map(_read, files)) + gib = total / (1 << 30) + print( + f"[session-prefetch] warmed page cache: {gib:.1f} GiB from " + f"{model_dir} in {time.monotonic() - t0:.1f}s", + flush=True, + ) + return gib + + +def _worker_import_report_cuda() -> bool: + """Import tensorrt_llm (the expensive part) and report CUDA state. + + Some library versions initialize a CUDA context at import time; that + idle context (~a few hundred MB, no kernels/allocations) is acceptable + and coexists with the running test, so it is reported, not asserted. + """ + import torch + + import tensorrt_llm # noqa: F401 + + return torch.cuda.is_initialized() + + +class _Built(NamedTuple): + """A finished background build: everything published (and consumed) together.""" + + spec: int + session: object + snapshot: object + + +class SessionPrefetcher: + def __init__(self): + self._lock = threading.Lock() + self._drain_lock = threading.Lock() + self._thread = None + self._building_spec = None # spec of the in-flight build, while _thread is set + self._build_timed_out = False + self._build_gen = 0 # bumped when a pending build is abandoned + self._built = None # Optional[_Built], set only by _publish() + self._patched = set() + self._next_model = None # item -> next model dir; built lazily + self._warmed_dirs = set() + self._disposed = False + # Activity counters, reported once per session by dispose(). pytest + # captures per-test stdout (swallowing the per-event prints for + # passing tests), but pytest_sessionfinish runs OUTSIDE capture, so + # the summary is the one line guaranteed to reach the CI console. + self.stats = Counter() + self._warmed_gib = 0.0 + + @property + def enabled(self) -> bool: + # Under pytest-xdist every worker sees the FULL collection but runs a + # scheduler-assigned subset, and N workers would each hold a live + # pool plus a spare. Disable in xdist workers. + if os.environ.get("PYTEST_XDIST_WORKER"): + return False + return os.environ.get("TRTLLM_TEST_PREFETCH_SESSION", "1").lower() in ( + "1", + "true", + "yes", + "on", + ) + + @staticmethod + def _next_model_map(items): + """Per item, the model dir of the NEXT test declaring one. + + One reverse pass, one lookup per item — O(n) once; ``on_test_setup`` + then costs a single dict lookup per test. (The naive alternative — + scanning the remaining collection at every test setup — is O(n^2) + lookups per session, seconds to minutes on large suites.) + """ + next_model, mapping = None, {} + for item in reversed(items): + mapping[item] = next_model + model = _model_dir_of(item) + if model is not None: + next_model = model + return mapping + + def on_test_setup(self, item) -> None: + """Warm the NEXT test's model weights while this test runs. + + Fires when the next model differs from the current one — including + between tests that share a pool (pool prefetch does not cover model + IO). The next-model map is built lazily on the FIRST test setup, from + ``session.items``: by then every reordering/deselecting plugin + (pytest-split runs trylast, --test-list filtering, -k/-m) has produced + the final run order, which a ``pytest_collection_modifyitems`` hook + could not guarantee. + """ + if self._next_model is None: + items = getattr(item.session, "items", None) or [item] + self._next_model = self._next_model_map(items) if self.enabled else {} + nxt = self._next_model.get(item) + if not nxt or nxt == _model_dir_of(item): + return + # Main-pytest-thread only (pytest_runtest_setup): no lock needed. + if nxt in self._warmed_dirs: + return # already warmed (or being warmed) this session + self._warmed_dirs.add(nxt) + threading.Thread( + target=self._warm, args=(nxt,), daemon=True, name="session-prefetch-warm" + ).start() + + def _warm(self, model_dir: str) -> None: + try: + gib = warm_page_cache(model_dir) + with self._lock: + if gib > 0: + self.stats["warms"] += 1 + self._warmed_gib += gib + else: + self.stats["warm_noops"] += 1 # no local weights / RAM guard + except Exception as e: # warming must never break the tests + print(f"[session-prefetch] page-cache warm failed (harmless): {e}", flush=True) + + def schedule_shadow(self, spec: int, env_overlay=None) -> None: + """Start building a spare ``spec``-worker pool in the background. + + Heuristic: the next test most likely needs a pool of the same size as + the current one. A mismatched in-flight build is drained before a + synchronous miss to preserve allocation-wide single-flight. This can + add latency when the size prediction is wrong, but avoids two MPI + bootstraps contending on the same allocation. + + ``env_overlay``: extra env vars to freeze into the WORKERS at spawn + (session_reuse restocks shadows with its worker-side weight cache + on). Passed through the library's worker-env channel, so the parent + process environment — and therefore the take()-time snapshot + comparison — is never touched. + """ + if not self.enabled or spec < 1: + return + with self._lock: + if self._thread is not None or (self._built is not None and self._built.spec == spec): + return # already building / built + self._building_spec = spec + self._build_timed_out = False + self._thread = threading.Thread( + target=self._build, + args=(spec, self._build_gen, env_overlay), + daemon=True, + name="session-prefetch-build", + ) + self._thread.start() + + def _build(self, spec: int, gen: int, env_overlay=None) -> None: + try: + from tensorrt_llm._utils import mpi_disabled + + if mpi_disabled(): + return + from tensorrt_llm.llmapi.mpi_session import MpiPoolSession + + snapshot = _spawn_snapshot() # workers freeze env+sys.path at spawn + # wait_shutdown: see _make_factory — every pool this layer hands + # out blocks its shutdown on actual worker exit. + session = MpiPoolSession(n_workers=spec, wait_shutdown=True, env_overrides=env_overlay) + if any(session.submit_sync(_worker_import_report_cuda)): + print( + "[session-prefetch] note: tensorrt_llm import initialized an idle " + "CUDA context in the prefetched workers (library version behavior)", + flush=True, + ) + self._publish(spec, session, snapshot, gen) + except Exception as e: # prefetch must never break the tests + print( + f"[session-prefetch] background build failed (falling back to synchronous): {e}", + flush=True, + ) + + def _publish(self, spec, session, snapshot, gen: int) -> None: + """Publish a finished background build, unless it was abandoned. + + ``_drain()`` bumps ``_build_gen`` when a build outlives its join + timeout; such a late build must shut its pool down instead of + publishing (a late publish would overwrite — and leak — a newer pool, + or hand a stale pool to a future test). The empty-slot check likewise + prevents overwriting an unconsumed pool. + """ + with self._lock: + if gen == self._build_gen and self._built is None: + self._built = _Built(spec, session, snapshot) + self.stats["pools_built"] += 1 + return + self.stats["pools_discarded_superseded"] += 1 + print("[session-prefetch] discarding superseded background build", flush=True) + session.shutdown() + + def _drain(self, timeout: float | None = None) -> _Built | None: + """Join a pending build and pop the completed shadow slot. + + A live build is kept registered and marked terminal on timeout. + Callers fail closed instead of starting a second pool while the first + bootstrap is still running; later calls fail immediately until the + thread exits, then clear the terminal state. + """ + # Serialize drains so concurrent LLM construction cannot make multiple + # callers wait through the full deadline before one records the + # terminal timeout. + with self._drain_lock: + # Read _thread under the lock: schedule_shadow() assigns-then-starts + # inside its critical section, and an unlocked read here can observe + # the assigned-but-not-yet-started thread ("cannot join thread before + # it is started" when a test creates LLMs concurrently). + with self._lock: + thread = self._thread + build_timed_out = self._build_timed_out + if thread is not None: + if build_timed_out and thread.is_alive(): + raise TimeoutError( + "session-prefetch shadow build previously timed out and " + "is still running; refusing to start a concurrent MPI pool" + ) + if not build_timed_out: + if timeout is None: + timeout = _shadow_build_wait_timeout() + thread.join(timeout=timeout) + with self._lock: + if thread is not None and thread.is_alive(): + # Invalidate a late publish but retain _thread so + # schedule_shadow() cannot start another build alongside it. + if not self._build_timed_out: + self._build_timed_out = True + self._build_gen += 1 + self.stats["pool_build_timeouts"] += 1 + raise TimeoutError( + "session-prefetch shadow build did not finish within " + f"{timeout}s; refusing to start a concurrent MPI pool" + ) + self._thread = None + self._building_spec = None + self._build_timed_out = False + built, self._built = self._built, None + return built + + def take(self, spec: int) -> object | None: + """Return a prefetched session for ``spec``, or None to build sync.""" + if not self.enabled: + return None + # The upper-level deadline is derived from the identity barrier's + # bootstrap deadline plus a small post-bootstrap diagnostic grace. + # It must never expire while the lower layer still considers the + # in-flight pool healthy. A wrong-size build is also drained before + # returning a miss: starting the requested size alongside it would + # recreate the same concurrent-bootstrap contention. + built = self._drain() + if built is None: + return None + if built.spec == spec and built.snapshot == _spawn_snapshot(): + # An instant handover is safe against the previous worker's GPU + # memory: every pool these layers hand out is built with + # wait_shutdown=True, so its shutdown() blocked until the workers + # actually exited (and released their memory). + self.stats["pools_handed_over"] += 1 + print(f"[session-prefetch] handing over prefetched {spec}-worker pool", flush=True) + return built.session + # Spec/env/sys.path mismatch (test skipped, reordered, or changed + # state the frozen workers would not see): discard. + self.stats["pools_discarded_stale"] += 1 + threading.Thread( + target=built.session.shutdown, daemon=True, name="session-prefetch-discard" + ).start() + return None + + def _make_factory(self, real_cls): + """A drop-in for ``MpiPoolSession`` that consumes and re-arms the shadow.""" + + def factory(n_workers, *args, **kwargs): + if args or kwargs: + return real_cls(n_workers, *args, **kwargs) + # n_workers == 1 included: the default single-GPU path also spawns + # a 1-worker pool (executor.py -> proxy.py) costing ~50s of + # spawn+import, the same as multi-GPU pools. + # wait_shutdown: this pool's shutdown must not return until its + # workers exited (and released GPU memory) — the NEXT pool is + # handed over instantly, without the ~50s sync spawn that used to + # hide the release window. Such spawns fail closed when identity + # collection cannot complete. Do not immediately retry or degrade + # to a plain pool: unidentified workers may still be exiting, and + # wait_shutdown=False cannot protect the next handover. + session = self.take(n_workers) + if session is None: + session = real_cls(n_workers=n_workers, wait_shutdown=True) + self.schedule_shadow(n_workers) # re-arm for the NEXT test + return session + + return factory + + def install_pool_factory_if_loaded(self) -> None: + """Lazily patch the pool-creation seams for zero-test-change prefetch. + + Only patches target modules ALREADY imported by the test suite, so + suites that never touch tensorrt_llm pay nothing (not even the + import). Idempotent — called from ``pytest_runtest_setup``. + Only the ``mpi_session is None`` branches construct ``MpiPoolSession`` + directly, so tests passing their own session (shared/grouped pools) + are never intercepted. + """ + if not self.enabled: + return + if len(self._patched) == len(_PATCH_TARGETS): + return # everything already patched: per-test fast path + if _reuse_layer_active(): + # session_reuse owns the seams: skip MPI-pool prefetch entirely + # (reuse eliminates the respawn; prefetch could only hide it). + self.stats["mpi_yielded_to_reuse"] = 1 + return + pending = [n for n in _PATCH_TARGETS if n in sys.modules and n not in self._patched] + if not pending: + return + from tensorrt_llm.llmapi.mpi_session import MpiPoolSession as real_cls + + factory = self._make_factory(real_cls) + for name in pending: + mod = sys.modules[name] + if getattr(mod, "MpiPoolSession", None) is real_cls: + # A real type, not a bare function: library code may run + # isinstance(x, MpiPoolSession) against the patched attribute + # (proxy.py's killed-worker detection did) — a function there + # raises TypeError and kills every LLM creation. + mod.MpiPoolSession = _isinstance_transparent_shim(real_cls, factory) + self._patched.add(name) + + def dispose(self) -> None: + """Shut down any unconsumed shadow pool (end-of-session cleanup). + + Uses the same coordinated deadline as ``take()``. Ending the pytest + session is not permission to abandon a bootstrap while the lower + layer still considers it healthy. Idempotent: a repository-root run + dispatches sessionfinish from both the repo-root and the subtree + conftest. + """ + if self._disposed: + return + self._disposed = True + try: + built = self._drain() + except TimeoutError as e: + # Do not turn pytest_sessionfinish into an internal error. The + # build remains registered, preventing another shadow from being + # launched; the daemon thread is bounded by the lower-level + # identity timeout unless the MPI runtime itself is wedged. + print(f"[session-prefetch] cleanup timed out: {e}", flush=True) + built = None + if built is not None: + built.session.shutdown() + # One line per session, emitted OUTSIDE pytest's per-test capture + # (pytest_sessionfinish) so it reaches the CI console: the per-event + # prints above are swallowed for passing tests. Silent when the + # prefetcher never did anything (non-LLM suites). + if self.stats: + parts = ", ".join(f"{k}={v}" for k, v in sorted(self.stats.items())) + if self._warmed_gib: + parts += f", warmed_gib={self._warmed_gib:.1f}" + print(f"[session-prefetch] session summary: {parts}", flush=True) + + +PREFETCHER = SessionPrefetcher() diff --git a/tests/test_common/session_prefetcher_hooks.py b/tests/test_common/session_prefetcher_hooks.py new file mode 100644 index 000000000000..26c5d7201472 --- /dev/null +++ b/tests/test_common/session_prefetcher_hooks.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pytest plugin wiring for the session prefetcher (repo-wide, demand-driven). + +Loaded via ``pytest_plugins`` in each test tree's top-level conftest. Factory +installation is LAZY: nothing is patched until the test suite itself imports +tensorrt_llm's executor modules (which only happens for tests that create MPI +pools), so suites that never create pools pay nothing — not even the +tensorrt_llm import. +""" + +import os + +from test_common.session_prefetcher import PREFETCHER + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "prefetch_model_dir(path): model dir the test loads, for page-cache warming", + ) + + +def pytest_runtest_setup(item): + # Last-line fail-open: prefetch is an optimization wired into EVERY test's + # setup, so an unexpected error here must degrade to baseline speed via + # the kill switch — never error the suite. + try: + PREFETCHER.install_pool_factory_if_loaded() + PREFETCHER.on_test_setup(item) + except Exception as e: + os.environ["TRTLLM_TEST_PREFETCH_SESSION"] = "0" + print(f"[session-prefetch] disabled by unexpected error: {e}", flush=True) + + +def pytest_sessionfinish(session, exitstatus): + try: + PREFETCHER.dispose() + except Exception as e: # never fail the session over cleanup + print(f"[session-prefetch] dispose failed: {e}", flush=True) diff --git a/tests/test_common/session_reuse.py b/tests/test_common/session_reuse.py index 145ee5cf2248..ea1b6272a5c5 100644 --- a/tests/test_common/session_reuse.py +++ b/tests/test_common/session_reuse.py @@ -23,10 +23,15 @@ bounding worker state accumulation Between handouts every worker runs a torch.compile/Dynamo reset (exactly once -per worker, barrier-pinned: ``grouped_test_utils.submit_sync_per_worker``) and -the handover waits for -the previous worker's GPU memory to actually be released (NVML settle barrier) -— both failure modes were observed in validation, not hypothetical. +per worker, barrier-pinned: ``grouped_test_utils.submit_sync_per_worker``). +Handover cannot race the previous worker's GPU-memory release: every pool +these layers build is constructed with ``wait_shutdown=True``, so its +shutdown blocks until the workers actually exited. + +Cache misses (first pool of a size, post-drain rebuild, post-retire +replacement) take a shadow pool pre-spawned by the session-prefetch layer +when it is wired (``_prefetcher``), hiding the ~50s spawn; each miss restocks +one shadow for the next. Enable/disable with ``TRTLLM_TEST_REUSE_SESSION`` (default on; ``0`` disables). Disabled under pytest-xdist workers (parallel tests would multiply live pools). @@ -35,8 +40,10 @@ import os import sys import threading -import time +# The spawn snapshot is shared with the session-prefetch layer (both hand a +# live pool to a test that did not spawn it — same invariant). +from test_common._session_utils import _isinstance_transparent_shim, _spawn_snapshot from test_common.grouped_test_utils import reset_worker_torch_compile_state, submit_sync_per_worker # The only places in the library that construct MpiPoolSession for a bare @@ -59,148 +66,45 @@ "TRTLLM_HF_WEIGHT_CACHE_MAX_ENTRIES": "1", } -# Workers freeze the parent environment AND sys.path at spawn time, so a -# cached pool must not be handed to a test that changed either (silently -# stale env / unimportable monkeypatched modules). Process bookkeeping that -# legitimately drifts between tests is ignored; a false mismatch only costs -# one synchronous rebuild. -_ENV_IGNORE = frozenset( - { - "PYTEST_CURRENT_TEST", - "COLUMNS", - "LINES", - "PWD", - "OLDPWD", - "SHLVL", - "_", - } -) - - -def _spawn_snapshot(): - """The worker-visible state a pool freezes at spawn: env + sys.path.""" - return ( - {k: v for k, v in os.environ.items() if k not in _ENV_IGNORE}, - list(sys.path), - ) - -# GPU-memory settle barrier at handover: a reused live pool skips the ~50s -# synchronous spawn that used to give the previous LLM's worker time to exit; -# its CUDA memory is only released when the process actually exits. Building -# the next model into that race fails with "insufficient GPU memory". -_SETTLE_MIN_FREE_FRAC = 0.85 -_SETTLE_POLL_S = 0.5 -_SETTLE_FLAT_POLLS = 3 -_SETTLE_EPSILON = 256 << 20 -_SETTLE_TIMEOUT_S = 30.0 - - -def _visible_gpu_indices(count: int): - visible = os.environ.get("CUDA_VISIBLE_DEVICES") - if not visible: - return list(range(count)) - indices = [] - for token in visible.split(","): - token = token.strip() - if not token.isdigit() or int(token) >= count: - return list(range(count)) # UUID/MIG form: fall back to all GPUs - indices.append(int(token)) - return indices or list(range(count)) +_RETIRE_THREADS: list = [] +_RETIRE_LOCK = threading.Lock() -def wait_gpu_memory_settle() -> None: - """Wait until visible GPUs are mostly free or free memory stops rising. +def _reap_retires(timeout: float = 60.0) -> None: + """Join in-flight retire threads (bounded); no-op when none are running. - Never raises: on any NVML problem the handover proceeds as before. + A retired pool's workers hold their (full-model) GPU memory until they + exit; the retire thread blocks on that exit (``wait_shutdown=True``), but + it is a BACKGROUND thread — the test hot path never waits on it. Before + an instant cached-pool handover, joining in-flight retires is what makes + the handover safe against a corpse still releasing (e.g. the duplicate + retired by ``_release`` moments earlier); every other path spawns fresh + (~50s), which outlasts the release naturally. Also called at drain + rendezvous points so disposals cannot leak past the session. """ - try: - import pynvml - - pynvml.nvmlInit() - except Exception: - return - try: - handles = [ - pynvml.nvmlDeviceGetHandleByIndex(i) - for i in _visible_gpu_indices(pynvml.nvmlDeviceGetCount()) - ] - - def _free_total(): - infos = [pynvml.nvmlDeviceGetMemoryInfo(h) for h in handles] - return [i.free for i in infos], [i.total for i in infos] - - t0 = time.monotonic() - flat, prev = 0, None - while True: - free, total = _free_total() - if all(f >= _SETTLE_MIN_FREE_FRAC * t for f, t in zip(free, total)): - break - if prev is not None and all(f - p < _SETTLE_EPSILON for f, p in zip(free, prev)): - flat += 1 - if flat >= _SETTLE_FLAT_POLLS: - break # not increasing: that memory is legitimately in use - else: - flat = 0 - if time.monotonic() - t0 >= _SETTLE_TIMEOUT_S: - break - prev = free - time.sleep(_SETTLE_POLL_S) - waited = time.monotonic() - t0 - if waited >= _SETTLE_POLL_S: + with _RETIRE_LOCK: + in_flight, _RETIRE_THREADS[:] = list(_RETIRE_THREADS), [] + for t in in_flight: + t.join(timeout=timeout) + if t.is_alive(): print( - f"[session-reuse] waited {waited:.1f}s before handover for GPU memory release", + "[session-reuse] WARNING: pool retirement did not finish within 60s", flush=True, ) - except Exception: - pass - finally: - try: - pynvml.nvmlShutdown() - except Exception: - pass - - -_RETIRE_THREADS: list = [] -_RETIRE_LOCK = threading.Lock() -def _proc_start_time(pid: int): - """Kernel start time (jiffies since boot) of ``pid``, or None if gone. +def _prefetcher(): + """The session-prefetch singleton when that layer is wired, else None. - PIDs are recycled by the OS, but the (pid, start_time) pair is unique: - verifying it right before SIGKILL prevents killing an unrelated process - (e.g. a replacement pool's worker) that inherited a dead worker's PID. + Mirror of the prefetcher's own reuse probe: coordination goes through + sys.modules so neither layer imports the other at module load (a suite + wired with only one layer pays nothing for the other). The prefetcher + yields the pool SEAMS to reuse; reuse in turn consumes prefetched + shadows on its cache misses — the two layers compose, not compete. """ - try: - with open(f"/proc/{pid}/stat", "rb") as f: - stat = f.read() - # Field 2 (comm) may contain spaces/parens; parse after the last ')'. - return stat.rsplit(b")", 1)[1].split()[19] # field 22 overall - except OSError: - return None - - -def _get_worker_pid() -> tuple: - """Runs inside a worker; module-level so it is picklable.""" - pid = os.getpid() - return (pid, _proc_start_time(pid)) - - -def _collect_worker_pids(real) -> tuple: - """Record the worker PIDs of a freshly spawned pool. - - ``_retire`` uses them to SIGKILL wedged workers: a graceful shutdown - blocks forever on a broken pool and ``shutdown_abort`` would MPI_Abort - the parent test process too. Records (pid, start_time) pairs so the kill - can verify the PID was not recycled. ``submit_sync_per_worker`` runs the - collection exactly once per worker. Best effort — if it fails, the pool - just falls back to graceful shutdown. - """ - try: - return tuple(sorted(submit_sync_per_worker(real, _get_worker_pid))) - except Exception: - return () + mod = sys.modules.get("test_common.session_prefetcher") + return getattr(mod, "PREFETCHER", None) def _describe_mismatch(spawn_snap, now_snap, uses, max_uses): @@ -273,6 +177,17 @@ def enabled(self) -> bool: "on", ) + def is_active(self) -> bool: + """Public probe for sibling layers: does reuse own the pool seams? + + The session prefetcher yields the ``MpiPoolSession`` seams when this + returns True (reuse eliminates the respawn outright; prefetch could + only hide it). Deliberately ignores ``_suspended``: a per-test + cache bypass (``private_mpi_session``) does not change seam + ownership. + """ + return self.enabled + @property def max_uses(self) -> int: return int(os.environ.get("TRTLLM_TEST_REUSE_MAX_USES", "16")) @@ -299,10 +214,13 @@ def _retire(real, broken: bool = False): def _dispose(): import signal + # Lazy: only runs when a pool exists, so tensorrt_llm is loaded. + from tensorrt_llm.llmapi.mpi_session import _process_start_time + for pid, start_time in pids: # Guard against PID recycling: only kill if the process at # this PID is still the worker we recorded at spawn. - if start_time is None or _proc_start_time(pid) != start_time: + if start_time is None or _process_start_time(pid) != start_time: continue try: os.kill(pid, signal.SIGKILL) @@ -356,23 +274,32 @@ def rpc_factory(n_workers, *args, **kwargs): # Fires exactly when an RPC executor is constructed, whatever the # test is named — no name heuristics. cache.drain() - return real_cls(n_workers, *args, **kwargs) + if args or kwargs: + return real_cls(n_workers, *args, **kwargs) + # wait_shutdown: the private pool dies at LLM shutdown; block + # there until its workers exited so the next pool (often handed + # over instantly from the cache) cannot race the GPU release. + return real_cls(n_workers=n_workers, wait_shutdown=True) for name in pending: mod = sys.modules[name] if getattr(mod, "MpiPoolSession", None) is real_cls: - mod.MpiPoolSession = rpc_factory if name == _RPC_PATCH_TARGET else factory + mod.MpiPoolSession = _isinstance_transparent_shim( + real_cls, rpc_factory if name == _RPC_PATCH_TARGET else factory + ) self._patched.add(name) # ---- cache operations ---- def acquire(self, real_cls, n_workers): - """Hand out a cached same-size pool (reset + settled) or build one.""" + """Hand out a cached same-size pool (workers reset) or build one.""" if self._suspended or not self.enabled: # Opt-out test (private_mpi_session) or the kill switch flipped # after the seams were patched: untracked fresh pool that the LLM - # owns and destroys normally. - return real_cls(n_workers=n_workers) + # owns and destroys normally (wait_shutdown: its shutdown blocks + # until the workers exited, so the next handover cannot race the + # GPU-memory release). + return real_cls(n_workers=n_workers, wait_shutdown=True) with self._lock: real = self._pools.pop(n_workers, None) if real is not None: @@ -391,8 +318,15 @@ def acquire(self, real_cls, n_workers): self._retire(real) # stale worker state or lifetime cap else: try: + # An instant handover must not race a corpse still + # releasing its GPU memory. Retire threads block on the + # workers' exit (wait_shutdown=True) but run in the + # BACKGROUND, so join any in flight (a duplicate retired + # by _release moments ago held full model memory). No-op + # on the common path; every non-cached path spawns fresh + # (~50s), which outlasts the release naturally. + _reap_retires() submit_sync_per_worker(real, reset_worker_torch_compile_state) - wait_gpu_memory_settle() print( f"[session-reuse] reusing {n_workers}-worker pool " f"(use #{real._reuse_uses + 1})", @@ -408,27 +342,54 @@ def acquire(self, real_cls, n_workers): return _ReusableSession(self._spawn_fresh(real_cls, n_workers), self) def _spawn_fresh(self, real_cls, n_workers): - """Spawn a cache-managed pool with the worker-side HF weight cache on. - - The cache env vars must be visible at spawn (workers freeze the env) - and are removed right after, so non-managed pools (private/RPC) and - the rest of the suite keep the production default. The spawn snapshot - is taken BEFORE adding them so later acquire-time comparisons (which - see the restored env) still match. An explicit user setting of either - var is respected and left untouched. + """Obtain a cache-managed pool: prefetched if one is armed, else spawn. + + Every cache miss lands here (first pool of a size, post-drain + rebuild, post-retire replacement). When the session-prefetch layer is + wired, a shadow pool armed at the PREVIOUS miss is taken instantly — + hiding the ~50s spawn the miss would otherwise pay — and a + replacement shadow is armed for the next miss of this size. Without + the prefetch layer (or on a shadow miss) the synchronous spawn is + unchanged. + + The worker-side HF weight cache env is frozen into the workers via + the library's ``env_overrides`` channel (parent env untouched, so + acquire-time snapshot comparisons still match); an explicit user + setting of either var is respected and left untouched. Prefetched + shadows were armed with the same overlay. wait_shutdown: shutdown of + this pool blocks until its workers exited, so a successor cannot + race the GPU-memory release. """ snapshot = _spawn_snapshot() - added = [k for k in _WEIGHT_CACHE_ENV if k not in os.environ] - for k in added: - os.environ[k] = _WEIGHT_CACHE_ENV[k] - try: - real = real_cls(n_workers=n_workers) - finally: - for k in added: - os.environ.pop(k, None) + overrides = {k: v for k, v in _WEIGHT_CACHE_ENV.items() if k not in os.environ} + real = None + prefetcher = _prefetcher() + if prefetcher is not None: + # A timeout must fail closed: starting a synchronous replacement + # would create two MPI pools concurrently on the same allocation. + # Unexpected prefetcher errors also propagate instead of silently + # hiding lifecycle bugs behind a synchronous fallback. + real = prefetcher.take(n_workers) + if real is None: + # One attempt gets the full worker-bootstrap deadline. If identity + # collection still fails, unidentified workers may remain alive; + # an immediate retry would overlap another MPI bootstrap with + # them, so propagate the fail-closed error. + real = real_cls(n_workers=n_workers, wait_shutdown=True, env_overrides=overrides) + if prefetcher is not None: + try: + # Restock only after the current pool is ready. On a shadow + # miss, scheduling before the synchronous spawn would make + # two MPI pools bootstrap concurrently on the same GPUs. + prefetcher.schedule_shadow(n_workers, env_overlay=overrides) + except Exception: + pass real._reuse_uses = 0 real._reuse_spawn_snapshot = snapshot - real._reuse_worker_pids = _collect_worker_pids(real) + # (pid, start_time) per worker, recorded by the library at spawn + # (wait_shutdown=True above). _retire uses them to SIGKILL wedged + # workers; best effort — empty means graceful shutdown only. + real._reuse_worker_pids = getattr(real, "_worker_identities", ()) return real def _release(self, real): @@ -457,15 +418,7 @@ def drain(self) -> None: disposals from leaking past the session without ever blocking the per-test hot path. The join is bounded for the same reason as below. """ - with _RETIRE_LOCK: - in_flight, _RETIRE_THREADS[:] = list(_RETIRE_THREADS), [] - for t in in_flight: - t.join(timeout=60) - if t.is_alive(): - print( - "[session-reuse] WARNING: pool retirement did not finish within 60s", - flush=True, - ) + _reap_retires() with self._lock: pools, self._pools = list(self._pools.values()), {} if not pools: diff --git a/tests/unittest/_torch/attention/model_attn_config.py b/tests/unittest/_torch/attention/model_attn_config.py index 29f292a89c12..b6f32e2fbbe3 100644 --- a/tests/unittest/_torch/attention/model_attn_config.py +++ b/tests/unittest/_torch/attention/model_attn_config.py @@ -52,7 +52,9 @@ """ from dataclasses import dataclass -from typing import List, Optional +from typing import Literal, Optional + +AttentionPhase = Literal["ctx", "gen"] @dataclass(frozen=True) @@ -68,11 +70,11 @@ class ModelAttnConfig: no_cache: bool = False # bidirectional DiT/encoder (no KV cache) is_cross: bool = False # encoder-decoder cross attention # MLA (DeepSeek-style latent attention). For absorbed generation num_kv_heads - # is 1 (single latent head); for the up-projected context pass (mla_context) - # it is MHA (num_kv_heads == num_heads) with asymmetric K/V (K head_dim = - # qk_nope + qk_rope, V head_dim = v_head_dim). + # is 1 (single latent head); for the up-projected context pass it is MHA + # (num_kv_heads == num_heads) with asymmetric K/V (K head_dim = qk_nope + + # qk_rope, V head_dim = v_head_dim). is_mla: bool = False - mla_context: bool = False # up-projected MHA context pass (vs absorbed gen) + phases: tuple[AttentionPhase, ...] | None = None kv_lora_rank: Optional[int] = None q_lora_rank: Optional[int] = None qk_nope_head_dim: Optional[int] = None @@ -576,6 +578,7 @@ class ModelAttnConfig: num_heads=128, num_kv_heads=1, head_dim=192, # qk_nope+qk_rope + phases=("gen",), is_mla=True, kv_lora_rank=512, q_lora_rank=1536, @@ -590,6 +593,7 @@ class ModelAttnConfig: num_heads=32, num_kv_heads=1, head_dim=192, + phases=("gen",), is_mla=True, kv_lora_rank=512, q_lora_rank=1536, @@ -604,6 +608,7 @@ class ModelAttnConfig: num_heads=64, num_kv_heads=1, head_dim=192, + phases=("gen",), is_mla=True, kv_lora_rank=512, q_lora_rank=1536, @@ -620,7 +625,7 @@ class ModelAttnConfig: num_heads=128, num_kv_heads=128, head_dim=192, - mla_context=True, + phases=("ctx",), is_mla=True, kv_lora_rank=512, q_lora_rank=1536, @@ -634,7 +639,7 @@ class ModelAttnConfig: num_heads=32, num_kv_heads=32, head_dim=192, - mla_context=True, + phases=("ctx",), is_mla=True, kv_lora_rank=512, q_lora_rank=1536, @@ -648,7 +653,7 @@ class ModelAttnConfig: num_heads=64, num_kv_heads=64, head_dim=192, - mla_context=True, + phases=("ctx",), is_mla=True, kv_lora_rank=512, q_lora_rank=1536, @@ -813,4 +818,4 @@ class ModelAttnConfig: ), ] -MODEL_CONFIGS: List[ModelAttnConfig] = _STANDARD + _MLA + _CROSS + _NO_CACHE +MODEL_CONFIGS: list[ModelAttnConfig] = _STANDARD + _MLA + _CROSS + _NO_CACHE diff --git a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_compressor_module.py b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_compressor_module.py index 0784d7256264..3ba703006935 100644 --- a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_compressor_module.py +++ b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_compressor_module.py @@ -449,6 +449,50 @@ def test_mixed_context_generation_position_ids_follow_compact_output(): assert actual_position_ids == [0, 4, 4, 4] +@pytest.mark.parametrize( + "compress_ratio,cached_tokens,kv_lens", + [ + pytest.param(1, [0], [4096], id="cr1_single"), + pytest.param(4, [0, 75, 4000], [4096, 4171, 8096], id="cr4_multi_boundary"), + pytest.param(128, [0], [64], id="cr128_empty_output"), + pytest.param(128, [973632, 8064], [990016, 8192], id="cr128_chunked_long_ctx"), + ], +) +def test_ctx_position_ids_host_sizes_match_device_scalar_fallback( + compress_ratio, cached_tokens, kv_lens +): + """Host-int ctx_output_sizes must reproduce the device-scalar fallback exactly. + + prepare() threads host-computed ctx compressed-token counts into + _compute_ctx_compressed_position_ids so the arange size and slice bound + are Python ints (no implicit D2H + stream sync). Both paths must produce + identical position IDs, including the untouched padding tail. + """ + num_contexts = len(kv_lens) + cached = torch.tensor(cached_tokens, dtype=torch.int32, device=DEVICE) + kv = torch.tensor(kv_lens, dtype=torch.int32, device=DEVICE) + past = (cached // compress_ratio).to(torch.int32) + new_comp = (kv // compress_ratio).to(torch.int32) - past + cu = F.pad(torch.cumsum(new_comp, dim=0), (1, 0)).to(torch.int32) + total = int(cu[num_contexts].item()) + + def _run(ctx_output_sizes): + out = torch.full((total + 8,), -1, dtype=torch.int32, device=DEVICE) + DeepseekV4TrtllmAttentionMetadata._compute_ctx_compressed_position_ids( + {compress_ratio: past}, + {compress_ratio: cu}, + {compress_ratio: out}, + num_contexts, + [compress_ratio], + ctx_output_sizes, + ) + return out + + golden = _run(None) + fast = _run({compress_ratio: total}) + assert torch.equal(golden, fast) + + def precompute_freqs_cis( dim, seqlen, original_seq_len, base, factor, beta_fast, beta_slow ) -> torch.Tensor: diff --git a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_cache_manager.py b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_cache_manager.py index 460fdde0ab82..8161ac5f3525 100644 --- a/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_cache_manager.py +++ b/tests/unittest/_torch/attention/sparse/deepseek_v4/test_deepseek_v4_cache_manager.py @@ -34,7 +34,6 @@ ) from tensorrt_llm._torch.disaggregation.resource.page import MapperKind from tensorrt_llm._torch.pyexecutor._util import CacheCost -from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import BlockReusePolicy from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, LlmRequestState from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._utils import binding_to_torch_dtype @@ -42,11 +41,7 @@ from tensorrt_llm.bindings.internal.batch_manager import CacheType as CacheTypeCpp from tensorrt_llm.llmapi.llm_args import DeepSeekV4SparseAttentionConfig, KvCacheConfig from tensorrt_llm.mapping import Mapping -from tensorrt_llm.runtime.kv_cache_manager_v2 import ( - GpuCacheTierConfig, - KVCacheManagerConfig, - PageIndexMode, -) +from tensorrt_llm.runtime.kv_cache_manager_v2 import BatchDesc, KVCacheDesc, PageIndexMode from tensorrt_llm.runtime.kv_cache_manager_v2._common import BAD_PAGE_INDEX _RequestCache = Dict[ @@ -55,6 +50,16 @@ ] +@pytest.mark.parametrize(("avg_seq_len", "expected"), [(None, 1024), (256, 256)]) +def test_typical_seq_len_preserves_deepseek_v4_fallback( + avg_seq_len: int | None, expected: int +) -> None: + manager = object.__new__(DeepseekV4CacheManager) + manager.max_seq_len = 1024 + + assert manager._get_typical_seq_len(KvCacheConfig(avg_seq_len=avg_seq_len)) == expected + + def test_cache_size_estimation_uses_model_attention_layer_count(): class FakeModelConfig: sparse_attention_config = SimpleNamespace( @@ -187,117 +192,6 @@ def _view_fp8_as_uint8(buffer: torch.Tensor) -> torch.Tensor: return buffer -def _build_deepseek_v4_cache_config_for_test( - kv_cache_config: KvCacheConfig, - *, - max_batch_size: int = 4, - max_seq_len: int = 1024, - max_num_tokens: int | None = 2048, - max_draft_len: int = 0, - is_draft: bool = False, -) -> KVCacheManagerConfig: - cache_manager = object.__new__(DeepseekV4CacheManager) - cache_manager.pp_layers = [0, 1, 2] - cache_manager._compress_ratios = [1, 4, 128] - cache_manager._swa_window_size = 128 - cache_manager._max_draft_len = max_draft_len - cache_manager._max_num_tokens = max_num_tokens - cache_manager.compressed_block_sizes = [128, 32, 1] - cache_manager.index_head_dim = 128 - cache_manager.head_dim = 512 - cache_manager.tokens_per_block = 128 - cache_manager.dtype = DataType.BF16 - cache_manager._indexer_k_dtype = "fp8" - cache_manager.max_batch_size = max_batch_size - cache_manager.max_seq_len = max_seq_len - cache_manager.enable_stats = False - cache_manager.enable_swa_scratch_reuse = False - cache_manager.num_extra_kv_tokens = 0 - cache_manager.block_reuse_policy = BlockReusePolicy(kv_cache_config.block_reuse_policy) - cache_manager.is_draft = is_draft - - return cache_manager._build_cache_config( - kv_cache_config, - tokens_per_block=128, - vocab_size=129280, - cache_tiers=[GpuCacheTierConfig(quota=1 << 30)], - ) - - -def test_deepseek_v4_pool_ratio_overrides_typical_step_and_constraints(): - config = _build_deepseek_v4_cache_config_for_test( - KvCacheConfig(pool_ratio=[0.2, 0.3, 0.5], avg_seq_len=256) - ) - - assert config.initial_pool_ratio == pytest.approx([0.2, 0.3, 0.5]) - assert config.typical_step is None - assert config.constraints == [] - - -def test_deepseek_v4_avg_seq_len_updates_typical_step(): - config = _build_deepseek_v4_cache_config_for_test( - KvCacheConfig(avg_seq_len=256), - max_batch_size=3, - max_seq_len=1024, - max_num_tokens=2048, - max_draft_len=2, - ) - - assert config.initial_pool_ratio is None - assert config.typical_step is not None - assert config.typical_step.kv_caches[0].capacity == 2048 - assert config.typical_step.kv_caches[0].history_length == 0 - assert [kv.capacity for kv in config.typical_step.kv_caches[1:]] == [256, 256] - assert [kv.history_length for kv in config.typical_step.kv_caches[1:]] == [253, 253] - assert config.constraints[0].kv_caches[0].capacity == 1024 - assert config.constraints[0].kv_caches[0].history_length == 1023 - - -def test_deepseek_v4_avg_seq_len_must_not_exceed_max_seq_len(): - with pytest.raises(ValueError, match="avg_seq_len"): - _build_deepseek_v4_cache_config_for_test( - KvCacheConfig(avg_seq_len=2048), - max_seq_len=1024, - ) - - -@pytest.mark.parametrize( - ("enable_block_reuse", "block_reuse_policy", "is_draft", "commit_min_snapshot"), - [ - (True, "all_reusable", False, False), - (True, "per_request", False, True), - (False, "per_request", False, False), - (True, "per_request", True, True), - ], -) -def test_deepseek_v4_commit_min_snapshot_follows_block_reuse_policy( - enable_block_reuse: bool, - block_reuse_policy: str, - is_draft: bool, - commit_min_snapshot: bool, -) -> None: - config = _build_deepseek_v4_cache_config_for_test( - KvCacheConfig( - enable_block_reuse=enable_block_reuse, - block_reuse_policy=block_reuse_policy, - enable_partial_reuse=True, - ), - is_draft=is_draft, - ) - - assert config.commit_min_snapshot is commit_min_snapshot - assert config.enable_partial_reuse - - -@pytest.mark.parametrize("enable_partial_reuse", [False, True]) -def test_deepseek_v4_propagates_partial_reuse_config(enable_partial_reuse: bool) -> None: - config = _build_deepseek_v4_cache_config_for_test( - KvCacheConfig(enable_partial_reuse=enable_partial_reuse) - ) - - assert config.enable_partial_reuse is enable_partial_reuse - - @pytest.fixture(params=[False, True], ids=["scratch_reuse_disabled", "scratch_reuse_enabled"]) def scratch_reuse_enabled(request) -> bool: return request.param @@ -1099,6 +993,28 @@ def _assert_cache_equal( msg=f"Mismatch for layer {layer_idx}, attention type {attn_type.name} (scales)", ) + def test_max_num_tokens_is_used_by_base_config(self): + max_batch_size = 2 + max_seq_len = 1024 + max_input_len = 127 + max_num_tokens = max_batch_size * (max_input_len + 1) + cache_manager, _ = self._create_deepseek_v4_cache_manager( + tokens_per_block=self.tokens_per_block, + max_batch_size=max_batch_size, + max_seq_len=max_seq_len, + max_input_len=max_input_len, + compress_ratios=[1, 4], + dtype=DataType.BF16, + compressor_dtype=DataType.FLOAT, + ) + + assert cache_manager.kv_cache_manager_py_config.typical_step == BatchDesc( + [ + KVCacheDesc(capacity=max_num_tokens, history_length=0), + KVCacheDesc(capacity=max_seq_len, history_length=max_seq_len - 1), + ] + ) + def test_indexer_cache_layout_default(self): """DeepSeek-V4 defaults to FP4 indexer K cache on Blackwell+.""" cache_manager, _ = self._create_deepseek_v4_cache_manager( diff --git a/tests/unittest/_torch/attention/sparse/dsa/test_dsa_indexer.py b/tests/unittest/_torch/attention/sparse/dsa/test_dsa_indexer.py index d6929e0506d9..cf4e9af27de4 100644 --- a/tests/unittest/_torch/attention/sparse/dsa/test_dsa_indexer.py +++ b/tests/unittest/_torch/attention/sparse/dsa/test_dsa_indexer.py @@ -3165,6 +3165,91 @@ def test_prepare_swaps_and_restore_recovers(self): torch.testing.assert_close(meta.host_kv_cache_block_offsets, original_host_offsets) +@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available") +@skip_pre_blackwell +def test_cutedsl_mqa_logits_output_buffer_persistent(): + """Regression: the CuteDSL paged-MQA-logits output must have a STABLE address + across calls, not be a per-forward ``torch.empty``. + + At long context the ``[B*next_n, kv_len]`` output is large; as a churning + transient it goes stale under CUDA-graph replay when another subsystem + co-captured in the same graph (e.g. the MTP / one-model spec sampler) + perturbs the shared pool. It must instead be drawn from the reserved + ``get_memory_buffers`` arena (like the CuteDSL topk runner), so its address is + identical across calls. + + Fails before the fix (two distinct ``torch.empty`` addresses while the first + output is kept alive); passes after (same reserved-arena address). + """ + from tensorrt_llm._torch.memory_buffer_utils import get_memory_buffers + + batch_size, next_n = 4, 1 + head_dim, block_size, index_topk = 128, 64, 2048 + heads = 32 + kv_len = 4096 + + cache_manager, sparse_attn_config = create_dsa_cache_manager( + batch_size=batch_size, + head_dim=head_dim, + tokens_per_block=block_size, + max_seq_len=kv_len, + num_layers=1, + index_topk=index_topk, + ) + create_indexer(sparse_attn_config, layer_idx=0) + + request_ids = list(range(batch_size)) + kv_lens = torch.full((batch_size,), kv_len, dtype=torch.int32) + cache_manager.add_dummy_requests( + request_ids=request_ids, token_nums=kv_lens.tolist(), is_gen=False, prepare_resource=True + ) + + metadata = _create_mock_metadata( + request_ids, + batch_size, + num_contexts=0, + num_generations=batch_size, + seq_lens=torch.full((batch_size,), next_n, dtype=torch.int32), + kv_lens=kv_lens.clone(), + num_cached_tokens=[kv_len - next_n] * batch_size, + cache_manager=cache_manager, + num_ctx_tokens=0, + num_tokens=batch_size * next_n, + max_draft_tokens=next_n - 1, + index_topk=index_topk, + use_cute_dsl_paged_mqa_logits=True, + ) + Indexer.prepare(metadata) + + kv_cache = cache_manager.get_indexer_k_cache_buffers(0) + q = torch.randn((batch_size, next_n, heads, head_dim), device="cuda", dtype=torch.bfloat16).to( + torch.float8_e4m3fn + ) + weights = torch.randn((batch_size * next_n, heads), device="cuda", dtype=torch.float32) + context_lens = metadata.gen_indexer_kv_lens_cuda_runtime + block_table = metadata.indexer_k_cache_block_offsets[0:batch_size] + sched = metadata.scheduler_metadata_buffer + + def _mqa(): + return torch.ops.trtllm.cute_dsl_fp8_paged_mqa_logits( + q, kv_cache, weights, context_lens, block_table, sched, kv_len + ) + + out1 = _mqa() + ptr1 = out1.data_ptr() + out2 = _mqa() # out1 kept alive: a fresh torch.empty would land elsewhere + ptr2 = out2.data_ptr() + + assert ptr1 == ptr2, ( + f"CuteDSL mqa-logits output address changed across calls " + f"({ptr1:#x} -> {ptr2:#x}); it must be a persistent reserved-arena buffer " + f"to avoid stale-pointer IMA under CUDA-graph replay" + ) + assert "cute_dsl_mqa_logits" in get_memory_buffers().buffers, ( + "CuteDSL mqa-logits output must be drawn from the get_memory_buffers arena" + ) + + @pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available") @skip_pre_hopper def test_topk_indices_buffer_cuda_graph(): @@ -3245,3 +3330,55 @@ def _run_indexer(): assert "indexer_topk_out_buffer" in metadata.cuda_graph_buffers.buffers, ( "indexer topk-output buffer must be drawn from the cuda_graph_buffers arena" ) + + +def test_kv_lens_row_reorder_threshold(): + """_compute_kv_lens_row_reorder engages iff num_generations * next_n >= 2 * num_sms, + and produces a descending argsort of gen_kv_lens when active.""" + num_sms = 16 # small synthetic value; threshold = 2 * 16 = 32 rows + next_n = 2 # max_draft_tokens=1 → next_n = 1 + 1 = 2 + + def make_mock(num_generations, kv_lens_list): + kv_cuda = torch.tensor(kv_lens_list, dtype=torch.int32, device="cuda") + buf = torch.zeros(64, dtype=torch.int32, device="cuda") + ns = SimpleNamespace( + enable_heuristic_topk=True, + use_cute_dsl_topk=True, + num_generations=num_generations, + num_sms=num_sms, + max_draft_tokens=next_n - 1, + num_contexts=0, + num_seqs=num_generations, + kv_lens_cuda=kv_cuda, + kv_lens_row_reorder_buffer=buf, + kv_lens_row_reorder=None, + ) + ns._compute_kv_lens_row_reorder = ( + lambda: DSAtrtllmAttentionMetadata._compute_kv_lens_row_reorder(ns) + ) + return ns + + # Fixed unsorted sequence for deterministic sort verification (len == num_sms) + kv_vals = [4, 1, 8, 2, 16, 3, 12, 6, 7, 9, 5, 11, 13, 10, 14, 15] + + # Below threshold: 1 * 2 = 2 < 32 → None + md_below = make_mock(1, [1000]) + md_below._compute_kv_lens_row_reorder() + assert md_below.kv_lens_row_reorder is None + + # At threshold: num_sms * 2 = 32 → engages, verify descending argsort + md_at = make_mock(num_sms, kv_vals) + md_at._compute_kv_lens_row_reorder() + assert md_at.kv_lens_row_reorder is not None + reorder = md_at.kv_lens_row_reorder.cpu().tolist() + assert [kv_vals[i] for i in reorder] == sorted(kv_vals, reverse=True), ( + "order_row must be a descending argsort of gen_kv_lens" + ) + + # Above threshold: (num_sms + 1) * 2 = 34 > 32 → also engages with correct sort + kv_vals2 = kv_vals + [100] + md_above = make_mock(num_sms + 1, kv_vals2) + md_above._compute_kv_lens_row_reorder() + assert md_above.kv_lens_row_reorder is not None + reorder2 = md_above.kv_lens_row_reorder.cpu().tolist() + assert [kv_vals2[i] for i in reorder2] == sorted(kv_vals2, reverse=True) diff --git a/tests/unittest/_torch/attention/sparse/test_cpp_custom_ops.py b/tests/unittest/_torch/attention/sparse/test_cpp_custom_ops.py index 20cfaa24805c..09f16a5e7666 100644 --- a/tests/unittest/_torch/attention/sparse/test_cpp_custom_ops.py +++ b/tests/unittest/_torch/attention/sparse/test_cpp_custom_ops.py @@ -17,6 +17,7 @@ - ``torch.ops.trtllm.indexer_k_cache_gather_op`` - ``torch.ops.trtllm.convert_req_index_to_global`` - ``torch.ops.trtllm.fused_cat_fp4`` +- ``torch.ops.trtllm.cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell`` """ import pytest @@ -488,7 +489,106 @@ def test_convert_req_index_to_global_block_table_padding(): # =================================================================== -# Test 3: fused_cat_fp4 — bit-exact vs DeepGEMM per_token_cast_to_fp4 +# Test 3: native CuTe DSL Indexer-Q projection + RoPE + FP4 fusion +# =================================================================== + + +@skip_pre_blackwell +@pytest.mark.parametrize("num_tokens", [1, 4, 5, 8, 16, 32, 64, 128]) +def test_cute_dsl_fp8_indexer_q_gemm_rope_fp4_matches_unfused(num_tokens): + """The CuTe DSL fusion must match the production chain bit for bit.""" + from _torch.helpers import per_block_cast_to_fp8_e8m0 + + from tensorrt_llm._torch.autotuner import DistributedTuningStrategy, OptimizationProfile + from tensorrt_llm._torch.custom_ops.cute_dsl_custom_ops import CuteDSLIndexerQBlackwellRunner + from tensorrt_llm.quantization.utils import fp8_utils + + torch.manual_seed(2026) + n_heads = 64 + hidden_size = 1536 + output_size = n_heads * HEAD_DIM + max_position = num_tokens * 2 + + qr = torch.randn(num_tokens, hidden_size, device="cuda", dtype=torch.bfloat16) + weight = torch.randn(output_size, hidden_size, device="cuda", dtype=torch.bfloat16) + weight_fp8, weight_scale = per_block_cast_to_fp8_e8m0(weight) + + weight_scale_deepgemm = fp8_utils.transform_sf_into_required_layout( + weight_scale, + mn=output_size, + k=hidden_size, + recipe=(1, 128, 128), + is_sfa=False, + ) + weight_scale_cutedsl = weight_scale.repeat_interleave(128, dim=0)[:output_size] + weight_scale_cutedsl = weight_scale_cutedsl.repeat_interleave(4, dim=1) + weight_scale_cutedsl = torch.ops.trtllm.block_scale_interleave( + weight_scale_cutedsl.to(torch.float8_e8m0fnu).view(torch.uint8) + ) + + angles = torch.randn(max_position, HEAD_DIM // 4, device="cuda", dtype=torch.float32) + cos_sin_cache = torch.stack((angles.cos(), angles.sin()), dim=1).contiguous() + position_ids = torch.arange(num_tokens, device="cuda", dtype=torch.int32) * 2 + alpha = torch.ones(1, device="cuda", dtype=torch.float32) + + runner = CuteDSLIndexerQBlackwellRunner(use_tvm_ffi=False) + assert runner.tuning_config.exclude_from_cache + assert runner.tuning_config.distributed_tuning_strategy == DistributedTuningStrategy.INDEPENDENT + tactics = runner.get_valid_tactics( + [ + qr, + weight_fp8, + weight_scale_cutedsl, + position_ids, + cos_sin_cache.view(max_position, HEAD_DIM // 2), + alpha, + ], + OptimizationProfile(), + ) + if num_tokens <= 16: + assert all(tactic in tactics for tactic in runner._small_m_tactics) + else: + assert all(tactic[0] != "swap_ab" for tactic in tactics) + assert ("native", (256, 128), (2, 1), False, 0) in tactics + + packed, scale = torch.ops.trtllm.cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell( + qr, + weight_fp8, + weight_scale_cutedsl, + position_ids, + cos_sin_cache.view(max_position, HEAD_DIM // 2), + alpha, + use_tvm_ffi=False, + ) + + q_ref = torch.ops.trtllm.fp8_swap_ab_gemm( + qr, + weight_fp8, + weight_scale_deepgemm, + disable_ue8m0_cast=True, + ).view(num_tokens, n_heads, HEAD_DIM) + torch.ops.trtllm.mla_rope_inplace( + q_ref, + position_ids, + cos_sin_cache, + n_heads, + HEAD_DIM // 2, + HEAD_DIM // 2, + False, + False, + ) + packed_ref, scale_ref = torch.ops.trtllm.fused_cat_fp4( + q_ref[..., : HEAD_DIM // 2], q_ref[..., HEAD_DIM // 2 :] + ) + + assert packed.shape == (num_tokens, output_size // 2) + assert scale.shape == (num_tokens, n_heads) + assert torch.equal(packed.view_as(packed_ref), packed_ref) + assert torch.equal(scale.view_as(scale_ref), scale_ref) + + +# =================================================================== +# Test 4: fused_cat_fp4 — bit-exact vs DeepGEMM per_token_cast_to_fp4 # =================================================================== diff --git a/tests/unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py b/tests/unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py index c29fd0bb232f..f8f7215eb6ee 100644 --- a/tests/unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py +++ b/tests/unittest/_torch/attention/sparse/test_cute_dsl_gvr_topk_decode.py @@ -15,10 +15,16 @@ from typing import Optional +import cutlass +import cutlass.cute as cute import pytest import torch +from cutlass.cute import runtime as _crt import tensorrt_llm._torch.custom_ops.cute_dsl_custom_ops # noqa: F401 +from tensorrt_llm._torch.cute_dsl_kernels.blackwell.top_k.gvr_topk_decode import ( + GvrTopKKernel as _GvrTopKKernel, +) from tensorrt_llm._utils import get_sm_version skip_not_sm100 = pytest.mark.skipif( @@ -27,7 +33,7 @@ ) -def _make_inputs( +def _make_inputs_impl( num_rows: int, N: int, top_k: int, @@ -96,8 +102,7 @@ def _make_inputs( if preidx_hit_rate <= 0.0: # Worst-case: only slot 0 is meaningful, rest are junk arange. - for j in range(1, top_k): - pre_idx[:, j] = j + pre_idx[:, 1:] = torch.arange(1, top_k, dtype=torch.int32, device="cuda") else: # Realistic: mix ``preidx_hit_rate`` real torch.topk indices with # random in-range fillers. Tests the Guess-phase short-circuit @@ -114,6 +119,66 @@ def _make_inputs( return logits, pre_idx, seq_lens +# Module-level input memoization. ``_make_inputs_impl`` is fully deterministic +# (seed-keyed RNG), and the parametrized sweeps below request the same +# (shape, dtype, hit-rate, ...) combination once per cluster_size / dispatch +# variant — regenerating logits + the reference topk dominated suite +# wall-clock, not the kernel under test. Cached tensors are returned WITHOUT +# cloning under a strict read-only convention: the op writes only +# ``out_indices`` (allocated fresh by every test), never its inputs. +_inputs_cache: dict = {} +# Reference top-K values memoized per cached-inputs identity (see +# ``_tie_aware_check``). Keyed on object ids, which is safe only because the +# keying tensors are pinned for the process lifetime by ``_inputs_cache``. +_ref_vals_cache: dict = {} + + +def _make_inputs( + num_rows: int, + N: int, + top_k: int, + dtype: torch.dtype, + next_n: int, + seed: int, + compress_ratio: int = 1, + preidx_hit_rate: float = 0.0, + varlen: bool = False, + seq_lens: Optional[torch.Tensor] = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Memoizing wrapper around ``_make_inputs_impl`` (same signature). + + Caller-provided ``seq_lens`` bypasses the cache (the tensor identity is + not part of a hashable key). + """ + if seq_lens is not None: + return _make_inputs_impl( + num_rows, + N, + top_k, + dtype, + next_n, + seed, + compress_ratio=compress_ratio, + preidx_hit_rate=preidx_hit_rate, + varlen=varlen, + seq_lens=seq_lens, + ) + key = (num_rows, N, top_k, dtype, next_n, seed, compress_ratio, preidx_hit_rate, varlen) + if key not in _inputs_cache: + _inputs_cache[key] = _make_inputs_impl( + num_rows, + N, + top_k, + dtype, + next_n, + seed, + compress_ratio=compress_ratio, + preidx_hit_rate=preidx_hit_rate, + varlen=varlen, + ) + return _inputs_cache[key] + + def _tie_aware_check( out_indices: torch.Tensor, logits: torch.Tensor, @@ -154,13 +219,23 @@ def _tie_aware_check( actual_kv_len = seq_lens_per_row - next_n + ofs + 1 N_eff = actual_kv_len // compress_ratio # [num_rows] - # Mask logits beyond per-row N_eff to -inf so torch.topk ignores tails. - col_idx = torch.arange(N, device=device) - in_range_mask = col_idx[None, :] < N_eff[:, None] # [num_rows, N] - masked_logits = torch.where(in_range_mask, logits_f32, float("-inf")) - - # Reference per-row top-K, sorted descending. - ref_vals, _ = torch.topk(masked_logits, k=top_k, largest=True, sorted=True, dim=-1) + # Reference per-row top-K, sorted descending, over logits masked beyond + # per-row N_eff. Memoized when (logits, seq_lens) come from the pinned + # ``_inputs_cache`` (identity match), since the reference only depends on + # (logits, seq_lens, top_k, next_n, compress_ratio) — not on the launch + # variant (cluster_size / order_row / ...) the test is exercising. + ref_key = None + if any(logits is v[0] and seq_lens is v[2] for v in _inputs_cache.values()): + ref_key = (id(logits), id(seq_lens), top_k, next_n, compress_ratio) + if ref_key is not None and ref_key in _ref_vals_cache: + ref_vals = _ref_vals_cache[ref_key] + else: + col_idx = torch.arange(N, device=device) + in_range_mask = col_idx[None, :] < N_eff[:, None] # [num_rows, N] + masked_logits = torch.where(in_range_mask, logits_f32, float("-inf")) + ref_vals, _ = torch.topk(masked_logits, k=top_k, largest=True, sorted=True, dim=-1) + if ref_key is not None: + _ref_vals_cache[ref_key] = ref_vals # ---- 1. Out-of-range / -1 placeholder check (single fused mask) ---- out_of_range = (out_indices < 0) | (out_indices >= N_eff[:, None]) @@ -227,7 +302,6 @@ def _tie_aware_check( @pytest.mark.parametrize("compress_ratio", [1, 4]) @pytest.mark.parametrize("preidx_hit_rate", [0.0, 0.5]) @pytest.mark.parametrize("cluster_size", [1, 4]) -@pytest.mark.parametrize("seqlen_sorted", [False, True]) def test_cute_dsl_gvr_topk_decode( dtype, top_k, @@ -238,7 +312,6 @@ def test_cute_dsl_gvr_topk_decode( compress_ratio, preidx_hit_rate, cluster_size, - seqlen_sorted, ): """Compare custom op output against torch.topk reference (tie-aware). @@ -250,14 +323,9 @@ def test_cute_dsl_gvr_topk_decode( ``varlen=False`` uses uniform seq_lens=N*cr across the batch; ``varlen=True`` draws per-row seq_lens uniformly in [N/2, N]*cr. - ``seqlen_sorted=True`` exercises the LJF host-side dispatch order: - we build ``order_row`` as a descending argsort over ``seq_lens`` and - pass it through the custom op. The - kernel must produce the same per-row top-K (rows are still written - back at their original positions, since the kernel uses - ``row_idx = order_row[req] * next_n + nn`` for both reads and - writes). The reference comparison is unchanged — it asserts that - each row's output is a valid top-K of that row's masked logits. + The LJF host-side dispatch order (``order_row``) is covered by the + dedicated ``test_cute_dsl_gvr_topk_decode_seqlen_sorted`` below on + representative cells instead of doubling this whole sweep. """ if N - next_n + 1 < top_k: pytest.skip(f"N_eff < top_k ({N - next_n + 1} < {top_k}) is a degenerate path") @@ -279,12 +347,66 @@ def test_cute_dsl_gvr_topk_decode( out_indices = torch.empty(num_rows, top_k, dtype=torch.int32, device="cuda") - # LJF dispatch order — request-level descending argsort of seq_lens. - order_row = ( - torch.argsort(seq_lens, descending=True, stable=False).to(torch.int32) - if seqlen_sorted - else None + torch.ops.trtllm.cute_dsl_gvr_topk_decode( + logits, + pre_idx, + seq_lens, + out_indices, + top_k=top_k, + next_n=next_n, + compress_ratio=compress_ratio, + cluster_size=cluster_size, ) + torch.cuda.synchronize() + + _tie_aware_check(out_indices, logits, seq_lens, top_k, next_n, compress_ratio=compress_ratio) + + +@skip_not_sm100 +@pytest.mark.parametrize( + "dtype,top_k,N,batch_size,varlen,next_n,compress_ratio,cluster_size", + [ + # Representative cells for the LJF host-side dispatch order + # (previously a full extra dimension on the sweep above): varlen + # batches so the argsort is a real permutation, both SMEM layout + # endpoints (bf16/K=512, fp32/K=2048), next_n=2 for the + # ``order_row[req] * next_n + nn`` row expansion, cr=4 for the + # order_row + compressed seq_lens interaction, cluster and + # single-CTA paths, and a batch_size=1 trivial-permutation smoke. + (torch.bfloat16, 512, 65536, 32, True, 1, 1, 4), + (torch.float32, 2048, 65536, 32, True, 2, 1, 1), + (torch.bfloat16, 1024, 4096, 32, True, 2, 4, 1), + (torch.float16, 1024, 65536, 1, False, 1, 1, 1), + ], +) +def test_cute_dsl_gvr_topk_decode_seqlen_sorted( + dtype, top_k, N, batch_size, varlen, next_n, compress_ratio, cluster_size +): + """LJF host-side dispatch order: ``order_row`` = descending argsort of + ``seq_lens`` passed through the custom op. + + The kernel must produce the same per-row top-K as the unsorted launch + (rows are still written back at their original positions, since the + kernel uses ``row_idx = order_row[req] * next_n + nn`` for both reads + and writes). The reference comparison is unchanged — it asserts that + each row's output is a valid top-K of that row's masked logits. + """ + num_rows = batch_size * next_n + logits, pre_idx, seq_lens = _make_inputs( + num_rows, + N, + top_k, + dtype, + next_n, + seed=42, + compress_ratio=compress_ratio, + preidx_hit_rate=0.5, + varlen=varlen, + ) + out_indices = torch.empty(num_rows, top_k, dtype=torch.int32, device="cuda") + + # LJF dispatch order — request-level descending argsort of seq_lens. + order_row = torch.argsort(seq_lens, descending=True, stable=False).to(torch.int32) torch.ops.trtllm.cute_dsl_gvr_topk_decode( logits, @@ -588,9 +710,11 @@ def test_lb_main_branches(dtype, top_k, scenario, N, seq_lens_mode, batch_size, @pytest.mark.parametrize( "dtype,top_k", [ + # SMEM-layout endpoints only. LB dispatch (prepare partition + + # long/short branch selection) is dtype-insensitive; the full + # dtype x K production map stays covered by the main sweep above + # and by test_cute_dsl_gvr_topk_decode_r0_equivalence. (torch.bfloat16, 512), - (torch.bfloat16, 1024), - (torch.float16, 1024), (torch.float32, 2048), ], ) @@ -663,3 +787,351 @@ def test_lb_vs_reference( next_n, compress_ratio=compress_ratio, ) + + +# =========================================================================== +# R0 histogram-ladder admission equivalence tests. +# +# ``enable_r0=True`` (the GvrTopKKernel default) replaces the Phase-2 secant +# threshold search with a single-pass multi-threshold "rung ladder" admission +# seeded by a 256-bin histogram over the prev-topK gathered values. This must +# select the SAME top-K as the retained secant baseline (``enable_r0=False``). +# +# top-K is order-independent, so correctness is checked by INDEX SET (not +# position): for continuous fp32 logits (tie-free with probability 1) the R0 +# and base index sets must be identical; for bf16/fp16 boundary value-ties can +# make two equally-valid selections differ in index, so there the guarantee is +# value-set (multiset) equality against the tie-aware torch.topk reference. +# +# The custom op does not plumb ``enable_r0`` (activation / dispatch land in a +# follow-up PR), so these tests drive ``GvrTopKKernel`` directly. This is also +# the only remaining coverage of the secant fallback path, since every op-level +# test above now inherits the ``enable_r0=True`` default. +# =========================================================================== + +_R0_DT = { + torch.float32: cutlass.Float32, + torch.bfloat16: cutlass.BFloat16, + torch.float16: cutlass.Float16, +} +# Compiled-kernel cache keyed on (enable_r0, dtype, top_k, cluster_size, T, +# min_blocks_per_mp). Shapes (num_rows / N / batch) are symbolic, so one +# compile covers every N and batch_size within a bucket (mirrors the runner). +_r0_kernel_cache: dict = {} + + +def _compile_gvr_direct(kernel): + """Compile a ``GvrTopKKernel`` with symbolic shapes, mirroring the + production runner's fake-tensor construction (128-bit loads, no + ``order_row`` / ``output_values``).""" + n_rows, n_cols, n_batch = cute.sym_int(), cute.sym_int(), cute.sym_int() + in_f = _crt.make_fake_compact_tensor( + kernel.dtype, (n_rows, n_cols), stride_order=(1, 0), assumed_align=16 + ) + pi_f = _crt.make_fake_compact_tensor( + cutlass.Int32, (n_batch, kernel.top_k), stride_order=(1, 0), assumed_align=16 + ) + sl_f = _crt.make_fake_compact_tensor(cutlass.Int32, (n_batch,), stride_order=(0,)) + oi_f = _crt.make_fake_compact_tensor( + cutlass.Int32, (n_rows, kernel.top_k), stride_order=(1, 0), assumed_align=16 + ) + fs = _crt.make_fake_stream(use_tvm_ffi_env_stream=True) + # __call__(input, pre_idx, seq_lens, output_values, output_indices, order_row, stream) + return cute.compile( + kernel, in_f, pi_f, sl_f, None, oi_f, None, stream=fs, options="--enable-tvm-ffi" + ) + + +def _run_gvr_direct(logits, pre_idx, seq_lens, top_k, enable_r0, cluster_size): + """Drive ``GvrTopKKernel`` directly (bypassing the custom op, which does + not expose ``enable_r0``). Fixed at ``next_n=1``, ``compress_ratio=1``, + 128-bit loads. When ``enable_r0=True`` the ctor auto-derives the shipped + R0 config (r0_qfracs=M2D, cs-aware p1b_cache, K512 kC-diet, P4 + rank-scatter) — i.e. the exact default arm. Returns int32 + ``[num_rows, top_k]`` indices.""" + num_rows, N = logits.shape + num_sms = torch.cuda.get_device_properties(0).multi_processor_count + num_threads = 1024 if (num_rows <= num_sms and N >= 65536) else 512 + min_blocks_per_mp = 1 if num_rows <= num_sms else 3 + key = (enable_r0, logits.dtype, top_k, cluster_size, num_threads, min_blocks_per_mp) + if key not in _r0_kernel_cache: + kernel = _GvrTopKKernel( + dtype=_R0_DT[logits.dtype], + top_k=top_k, + next_n=1, + num_threads=num_threads, + compress_ratio=1, + use_256bit_load=False, + min_blocks_per_mp=min_blocks_per_mp, + cluster_size=cluster_size, + return_output_values=False, + enable_r0=enable_r0, + ) + _r0_kernel_cache[key] = _compile_gvr_direct(kernel) + out = torch.empty(num_rows, top_k, dtype=torch.int32, device="cuda") + _r0_kernel_cache[key](logits, pre_idx, seq_lens, None, out, None) + torch.cuda.synchronize() + return out + + +def _assert_index_sets_equal_tie_aware(out_base, out_r0, logits): + """Assert two arms' top-K index sets match, modulo boundary value-ties. + + fp32 randn logits DO collide bit-exactly at these sample counts; when the + duplicated value sits on the top-K boundary, each arm may legitimately + keep a different member of the tie class. Indices in the symmetric + difference must all carry the row's boundary (minimum kept) value — + anything else is a genuine divergence. + """ + base_sorted, _ = out_base.sort(dim=-1) + r0_sorted, _ = out_r0.sort(dim=-1) + mismatch = (base_sorted != r0_sorted).any(dim=-1) + for bad in mismatch.nonzero().flatten().tolist(): + base_set = set(out_base[bad].tolist()) + r0_set = set(out_r0[bad].tolist()) + diff = sorted(base_set.symmetric_difference(r0_set)) + row_vals = logits[bad].float() + kth = row_vals[out_base[bad].long()].min() + diff_vals = row_vals[torch.tensor(diff, device=logits.device, dtype=torch.long)] + if not bool((diff_vals == kth).all().item()): + raise AssertionError( + f"row={bad}: R0 index set != secant-base index set beyond a " + f"boundary value-tie (kth={kth.item()}, " + f"diff={[(i, row_vals[i].item()) for i in diff]}, " + f"base={sorted(base_set)}, r0={sorted(r0_set)})" + ) + + +def _make_r0_pre_idx(logits, top_k, hint, seed): + """Build ``pre_idx`` in the kernel's native cr=1 convention: the kernel + reads ``logits[pre_idx + 1]``, so store ``true_index - 1``. + + ``hint='real'`` seeds a warm hint (near-topK indices) so R0's admission + ladder hits on the first pass; ``hint='rand'`` seeds a cold hint (random + in-range indices) that misses admission and forces the R0-miss inline + log-falsi (R1) + fb_fix fallback.""" + num_rows, N = logits.shape + g = torch.Generator(device="cuda").manual_seed(seed) + if hint == "real": + noised = logits.float() + 0.15 * torch.randn(num_rows, N, generator=g, device="cuda") + pre = noised.topk(top_k, dim=1).indices.int() + else: + pre = torch.randint(0, N, (num_rows, top_k), generator=g, device="cuda").int() + return (pre - 1).contiguous() + + +@skip_not_sm100 +@pytest.mark.parametrize( + "dtype,top_k", + [ + (torch.bfloat16, 512), + (torch.bfloat16, 1024), + (torch.float16, 1024), + (torch.float32, 2048), + ], +) +@pytest.mark.parametrize("N", [8192, 65536]) +@pytest.mark.parametrize("batch_size", [1, 16]) +@pytest.mark.parametrize("hint", ["real", "rand"]) +@pytest.mark.parametrize("cluster_size", [1, 4, 8]) +def test_cute_dsl_gvr_topk_decode_r0_equivalence(dtype, top_k, N, batch_size, hint, cluster_size): + """R0 admission (``enable_r0=True``, the new default) selects the same + top-K as the secant baseline (``enable_r0=False``), by index set. + + ``hint='real'`` exercises the R0 admission-hit fast path; ``hint='rand'`` + forces the R0-miss log-falsi (R1) + fb_fix fallback. ``cluster_size=4`` + confirms R0 gates to single-CTA and the ``None`` R0 buffers propagate + cleanly through the cluster path; ``cluster_size=8`` covers the runner's + tiny-grid large-N pick (BS<=4, N>=128K -> cs=8): 7-peer DSMEM + aggregation, large-N only (per-CTA slice too short below 64K). + """ + if N < top_k * 2: + pytest.skip(f"N ({N}) < 2*top_k ({2 * top_k}): GVR histogram bucket too coarse") + if cluster_size == 8 and N < 65536: + pytest.skip("cs=8 is a large-N production config (runner picks it only at N >= 131072)") + + num_rows = batch_size # next_n = 1 + torch.manual_seed(0) + torch.cuda.manual_seed(0) + logits = (torch.randn(num_rows, N, device="cuda") * 2.0).to(dtype).contiguous() + seq_lens = torch.full((num_rows,), N, dtype=torch.int32, device="cuda") + pre_idx = _make_r0_pre_idx(logits, top_k, hint, seed=1) + + out_base = _run_gvr_direct( + logits, pre_idx, seq_lens, top_k, enable_r0=False, cluster_size=cluster_size + ) + out_r0 = _run_gvr_direct( + logits, pre_idx, seq_lens, top_k, enable_r0=True, cluster_size=cluster_size + ) + + # 1. Both arms independently produce a valid top-K (tie-aware value set). + _tie_aware_check(out_base, logits, seq_lens, top_k, next_n=1, compress_ratio=1) + _tie_aware_check(out_r0, logits, seq_lens, top_k, next_n=1, compress_ratio=1) + + # 2. Equivalence. fp32 logits are ALMOST tie-free, so R0 and base must + # return the identical index set (order-independent) — but randn + # quantized to fp32 does collide (bs=16, N=8192, seed 0: + # logits[3,2956] == logits[3,4949] bit-exactly, straddling the K=2048 + # boundary), and a boundary value-tie makes the arms' distinct index + # picks equally valid. Where the sets differ, require every differing + # index to carry the row's boundary (k-th) value; anything else is a + # real divergence. bf16/fp16 boundary ties are common, so equivalence + # there is the value-set equality already established in step 1 + # (both == torch.topk reference). + if dtype == torch.float32: + _assert_index_sets_equal_tie_aware(out_base, out_r0, logits) + + +@skip_not_sm100 +@pytest.mark.parametrize( + "dtype,top_k,N,batch_size,cluster_size", + [ + # Multi-wave single-CTA grids (batch_size > num_sms): exercises the + # occupancy regime where rows alone oversubscribe the device. + (torch.bfloat16, 512, 16384, 256, 1), + (torch.float32, 2048, 65536, 256, 1), + # Multi-wave cluster grid (batch_size * cs > num_sms): DSMEM handoff + # correctness across wave boundaries. + (torch.bfloat16, 512, 65536, 64, 4), + ], +) +@pytest.mark.parametrize("hint", ["real", "rand"]) +def test_cute_dsl_gvr_topk_decode_r0_equivalence_bigbs( + dtype, top_k, N, batch_size, hint, cluster_size +): + """Big-batch R0-vs-secant equivalence: multi-wave grids only. + + The main equivalence grid tops out at batch_size=16 (single wave). + These cells lock R0 + cluster correctness when the grid spans several + waves — the throughput-bound regime of the BS-scaling study.""" + num_rows = batch_size + torch.manual_seed(0) + torch.cuda.manual_seed(0) + logits = (torch.randn(num_rows, N, device="cuda") * 2.0).to(dtype).contiguous() + seq_lens = torch.full((num_rows,), N, dtype=torch.int32, device="cuda") + pre_idx = _make_r0_pre_idx(logits, top_k, hint, seed=1) + + out_base = _run_gvr_direct( + logits, pre_idx, seq_lens, top_k, enable_r0=False, cluster_size=cluster_size + ) + out_r0 = _run_gvr_direct( + logits, pre_idx, seq_lens, top_k, enable_r0=True, cluster_size=cluster_size + ) + _tie_aware_check(out_base, logits, seq_lens, top_k, next_n=1, compress_ratio=1) + _tie_aware_check(out_r0, logits, seq_lens, top_k, next_n=1, compress_ratio=1) + if dtype == torch.float32: + _assert_index_sets_equal_tie_aware(out_base, out_r0, logits) + + +@skip_not_sm100 +@pytest.mark.parametrize( + "top_k,N,cluster_size", + [(512, 16384, 1), (1024, 131072, 4), (2048, 131072, 4)], +) +@pytest.mark.parametrize("band", ["sub_resolution", "one_ulp"]) +def test_cute_dsl_gvr_topk_decode_p4_exact_tail_ties(top_k, N, cluster_size, band): + """fp32 near-tie adversarial exactness (``p4_exact_tail``). + + The P4 rank-scatter fine recursion resolves candidate values to + range/(kNumBins*256); distinct values spaced below that which straddle + the top-K boundary land in ONE fine bin and were previously kept in + arrival order (observed on real DSv4-Pro 512k-ISL captures as |miss|=1 + with dv ~ 3e-6). ``sub_resolution`` plants ~2.4k distinct values spaced + 5e-8 around the boundary; ``one_ulp`` plants a two-value bitwise plateau + (``nextafter`` pairs). Both stay within the kC candidate budget (tie + sets wider than kC are outside the kernel's contract). The default fp32 + kernel must return the exact top-K value set; natural random data never + triggers this, so the adversarial construction is the only regression + coverage.""" + torch.manual_seed(3) + torch.cuda.manual_seed(3) + logits = (torch.randn(1, N, device="cuda") * 2.0).float().contiguous() + boundary = torch.topk(logits[0], top_k).values[top_k - 1].item() + n_tie = 2400 if band == "sub_resolution" else 2000 + plant = torch.randperm(N)[:n_tie] + if band == "sub_resolution": + tie_vals = ( + boundary + (torch.arange(n_tie, dtype=torch.float32, device="cuda") - n_tie // 2) * 5e-8 + ) + else: + tie_vals = torch.full((n_tie,), boundary, device="cuda") + tie_vals[::2] = torch.nextafter(tie_vals[::2], torch.tensor(float("inf"), device="cuda")) + logits[0, plant] = tie_vals + seq_lens = torch.full((1,), N, dtype=torch.int32, device="cuda") + pre_idx = _make_r0_pre_idx(logits, top_k, "real", seed=4) + + out = _run_gvr_direct( + logits, pre_idx, seq_lens, top_k, enable_r0=True, cluster_size=cluster_size + ) + + # Value-multiset exactness (the boundary index set is not unique under + # bitwise plateaus, so indices are compared through their values). + sel = logits[0][out[0].long()].sort().values + ref = torch.topk(logits[0], top_k).values.sort().values + torch.testing.assert_close(sel, ref, rtol=0.0, atol=0.0) + + +@skip_not_sm100 +def test_cute_dsl_gvr_topk_decode_pick_config_policy(): + """``pick_config`` returns the runner-equivalent launch shapes. + + Locks the (BS, N) -> cluster_size map and the BS-aware occupancy knobs + (the 2026-07-15 big-BS triage: a config frozen at the BS=1 optimum is + geomean 2.27x slower than these picks at BS in {64, 256, 1024}).""" + sms = 148 # policy is expressed against a fixed SM count for determinism + pc = _GvrTopKKernel.pick_config + + # cluster_size policy: N<64K -> 1; tiny grid large-N -> 8; single-wave + # -> 4/2; multi-wave -> 1. + assert pc(torch.float32, 1, 32768, num_sms=sms)["cluster_size"] == 1 + assert pc(torch.float32, 2, 131072, num_sms=sms)["cluster_size"] == 8 + assert pc(torch.float32, 16, 65536, num_sms=sms)["cluster_size"] == 4 + assert pc(torch.float32, 64, 65536, num_sms=sms)["cluster_size"] == 2 + assert pc(torch.float32, 256, 65536, num_sms=sms)["cluster_size"] == 1 + + # Occupancy knobs at multi-wave BS: T=512 + mbpm>=2 (NOT the BS=1 + # frozen T=1024/mbpm=1 that loses 2.3-6x at big BS). + big = pc(torch.float32, 1024, 65536, num_sms=sms) + assert big["num_threads"] == 512 and big["min_blocks_per_mp"] == 2 + big16 = pc(torch.bfloat16, 1024, 65536, num_sms=sms) + assert big16["num_threads"] == 512 and big16["min_blocks_per_mp"] == 3 + + # Graph-capture contract: max_seq_len (peak N) overrides the capture N. + cap = pc(torch.bfloat16, 1, 8192, max_seq_len=131072, num_sms=sms) + assert cap["cluster_size"] == 8 # picked for the replay shape + + +@skip_not_sm100 +@pytest.mark.parametrize( + "dtype,top_k,N,batch_size", + [ + (torch.float32, 2048, 32768, 1), # cs=1 small-N + (torch.bfloat16, 512, 65536, 16), # cs=4 single-wave + (torch.float32, 1024, 131072, 2), # cs=8 tiny grid large-N + (torch.bfloat16, 1024, 65536, 256), # cs=1 multi-wave big-BS + ], +) +def test_cute_dsl_gvr_topk_decode_launch_autoconfig(dtype, top_k, N, batch_size): + """``GvrTopKKernel.launch`` (pick_config + variant cache) produces a + valid top-K at every launch-shape regime the policy can pick, including + cluster_size=8. Direct-drive users get production-equivalent shapes.""" + num_rows = batch_size + torch.manual_seed(0) + torch.cuda.manual_seed(0) + logits = (torch.randn(num_rows, N, device="cuda") * 2.0).to(dtype).contiguous() + seq_lens = torch.full((num_rows,), N, dtype=torch.int32, device="cuda") + pre_idx = _make_r0_pre_idx(logits, top_k, "real", seed=1) + out = torch.empty(num_rows, top_k, dtype=torch.int32, device="cuda") + + _GvrTopKKernel.launch(logits, pre_idx, seq_lens, out, top_k) + torch.cuda.synchronize() + _tie_aware_check(out, logits, seq_lens, top_k, next_n=1, compress_ratio=1) + + # Override path: forcing the secant arm through launch() must also be a + # valid top-K and (fp32, tie-free) the identical index set. + out_sec = torch.empty(num_rows, top_k, dtype=torch.int32, device="cuda") + _GvrTopKKernel.launch(logits, pre_idx, seq_lens, out_sec, top_k, enable_r0=False) + torch.cuda.synchronize() + _tie_aware_check(out_sec, logits, seq_lens, top_k, next_n=1, compress_ratio=1) + if dtype == torch.float32: + assert torch.equal(out.sort(dim=-1).values, out_sec.sort(dim=-1).values) diff --git a/tests/unittest/_torch/attention/sparse/test_minimax_m3_msa_backend.py b/tests/unittest/_torch/attention/sparse/test_minimax_m3_msa_backend.py new file mode 100644 index 000000000000..2e44c27e9bd3 --- /dev/null +++ b/tests/unittest/_torch/attention/sparse/test_minimax_m3_msa_backend.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Structural tests for the MiniMax-M3 MSA sparse attention backend. + +These validate backend selection and decode scratch-buffer sizing without +launching kernels. Numerical parity against the Triton reference is covered +by the SM100 integration accuracy test. +""" + +import pytest +import torch + +from tensorrt_llm._torch.attention_backend.sparse.minimax_m3 import MiniMaxM3MsaSparseAttention +from tensorrt_llm._torch.attention_backend.sparse.utils import _resolve_minimax_m3_backend_cls +from tensorrt_llm.llmapi.llm_args import MiniMaxM3SparseAttentionConfig + + +def test_resolver_selects_msa_backend_when_available(monkeypatch): + import tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_availability as avail + + monkeypatch.setattr(avail, "ensure_msa_available", lambda: None) + params = MiniMaxM3SparseAttentionConfig(implementation="msa").to_sparse_params() + assert _resolve_minimax_m3_backend_cls(params) is MiniMaxM3MsaSparseAttention + + +def test_msa_requires_block_size_128(): + # The MSA implementation is fixed to a 128-token page size; a mismatched + # sparse_block_size must fail loudly at config construction rather than being + # silently overridden at runtime. + with pytest.raises(ValueError, match=r"sparse_block_size == 128"): + MiniMaxM3SparseAttentionConfig(implementation="msa", sparse_block_size=64) + + # The Triton reference is unaffected by the constraint. + cfg = MiniMaxM3SparseAttentionConfig(implementation="triton", sparse_block_size=64) + assert cfg.sparse_block_size == 64 + + +def test_msa_metadata_rejects_undersized_max_score_buffer(): + metadata_cls = MiniMaxM3MsaSparseAttention.Metadata + metadata = metadata_cls.__new__(metadata_cls) + # Flat backing store sized for 4 heads * 8 k-tiles * 2 batch = 64 elements, + # too small for the plan's required 4 * 16 * 2 = 128. + metadata.msa_max_score = torch.zeros(4 * 8 * 2) + metadata.kv_cache_manager = None + + with pytest.raises(ValueError, match=r"msa_max_score backing store"): + metadata._ensure_msa_decode_scratch_buffers( + num_index_heads=4, + max_batch=2, + capture_graph=False, + required_max_k_tiles=16, + ) + + +def test_msa_proxy_max_score_view_is_contiguous_over_stable_store(): + """The proxy view fed to fmha_sm100 must be contiguous in the exact + [num_index_heads, plan_max_k_tiles, num_tokens] shape the kernel writes, + backed by a stable store so its data_ptr survives CUDA graph replay. + """ + metadata_cls = MiniMaxM3MsaSparseAttention.Metadata + metadata = metadata_cls.__new__(metadata_cls) + # Worst-case store: 4 heads * 16 k-tiles * 8 batch. + num_index_heads, worst_k, max_batch = 4, 16, 8 + metadata.msa_max_score = torch.zeros(num_index_heads * worst_k * max_batch) + store_ptr = metadata.msa_max_score.data_ptr() + + # A smaller live step still yields a contiguous view sized to that step, + # which is what the kernel's stride-agnostic write requires. + view = metadata.msa_proxy_max_score_view(num_index_heads, 5, 3) + assert view.shape == (num_index_heads, 5, 3) + assert view.is_contiguous() + assert view.data_ptr() == store_ptr + + # Oversized requests are rejected rather than silently corrupting memory. + with pytest.raises(ValueError, match=r"msa_max_score backing store"): + metadata.msa_proxy_max_score_view(num_index_heads, worst_k, max_batch + 1) + + +def test_msa_index_k_uses_hnd_cache_view_and_writer(): + metadata_cls = MiniMaxM3MsaSparseAttention.Metadata + metadata = metadata_cls.__new__(metadata_cls) + num_pages, coalescing_scale, page_size, head_dim = 2, 7, 8, 16 + pool = torch.zeros( + num_pages, + coalescing_scale, + 1, + page_size, + head_dim, + dtype=torch.bfloat16, + ) + hnd_cache = pool[:, 0] + + class FakeCacheManager: + def __init__(self): + self.calls = [] + + def get_index_k_buffer(self, layer_idx, kv_layout="NHD"): + self.calls.append((layer_idx, kv_layout)) + return hnd_cache + + manager = FakeCacheManager() + metadata.kv_cache_manager = manager + metadata.msa_out_cache_loc = torch.tensor([2, page_size + 5], dtype=torch.int32) + values = torch.arange(2 * head_dim, dtype=torch.float32).reshape(2, 1, head_dim) + + returned = metadata.msa_idx_k_cache(3) + metadata.msa_write_idx_k(3, values) + + assert returned.data_ptr() == hnd_cache.data_ptr() + assert not returned.is_contiguous() + assert manager.calls == [(3, "HND"), (3, "HND")] + torch.testing.assert_close(hnd_cache[0, 0, 2], values[0, 0].to(torch.bfloat16)) + torch.testing.assert_close(hnd_cache[1, 0, 5], values[1, 0].to(torch.bfloat16)) + + +def test_msa_indexer_preserves_strided_hnd_index_k(monkeypatch): + import tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_indexer as indexer_module + from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.common import MiniMaxM3SparseConfig + + config = MiniMaxM3SparseConfig( + num_q_heads=4, + num_kv_heads=1, + head_dim=128, + num_index_heads=4, + sparse_index_dim=128, + block_size=128, + topk=16, + ) + indexer = indexer_module.MsaIndexer(config) + pool = torch.randn(2, 7, 1, 128, 128, dtype=torch.bfloat16) + idx_k_paged = pool[:, 0] + captured = {} + + def fake_proxy_max_score(idx_q, passed_idx_k, **kwargs): + del kwargs + captured["idx_k"] = passed_idx_k + return torch.zeros(4, 2, idx_q.shape[0]) + + expected = torch.zeros(1, 1, 16, dtype=torch.int32) + + def fake_select_blocks_from_maxscore(*args, **kwargs): + del args, kwargs + return expected + + monkeypatch.setattr(indexer_module, "_proxy_max_score", fake_proxy_max_score) + monkeypatch.setattr( + indexer_module, + "select_blocks_from_maxscore", + fake_select_blocks_from_maxscore, + ) + + result = indexer.select_blocks( + torch.zeros(1, 4, 128, dtype=torch.bfloat16), + idx_k_paged, + idx_sm_scale=128**-0.5, + kv_indices=torch.arange(2, dtype=torch.int32), + qo_lens_cpu=torch.tensor([1], dtype=torch.int32), + kv_lens_cpu=torch.tensor([256], dtype=torch.int32), + qo_offset_cpu=torch.tensor([255], dtype=torch.int32), + ) + + assert captured["idx_k"] is idx_k_paged + assert captured["idx_k"].data_ptr() == idx_k_paged.data_ptr() + assert not captured["idx_k"].is_contiguous() + assert result is expected + + +def test_msa_proxy_max_score_strided_index_k_matches_packed(): + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + if torch.cuda.get_device_capability()[0] != 10: + pytest.skip("SM100 (Blackwell) required") + + from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_indexer import _proxy_max_score + from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_utils import ( + msa_package_available, + ) + + if not msa_package_available(): + pytest.skip("fmha_sm100 (MSA) not importable") + + page_size = head_dim = 128 + num_index_heads = 4 + coalescing_scale = 57 + kv_lens_cpu = torch.tensor([1, 130, 257, 128, 511, 1024, 33, 900], dtype=torch.int32) + pages_per_sequence = (kv_lens_cpu + page_size - 1) // page_size + num_pages = int(pages_per_sequence.sum().item()) + + generator = torch.Generator(device="cuda").manual_seed(0) + index_k_pool = torch.randn( + num_pages, + coalescing_scale, + 1, + page_size, + head_dim, + generator=generator, + device="cuda", + dtype=torch.bfloat16, + ) + index_k_strided = index_k_pool[:, 0] + index_k_packed = index_k_strided.contiguous() + index_q = torch.randn( + kv_lens_cpu.numel(), + num_index_heads, + head_dim, + generator=generator, + device="cuda", + dtype=torch.bfloat16, + ) + kwargs = { + "qo_lens_cpu": torch.ones_like(kv_lens_cpu), + "kv_lens_cpu": kv_lens_cpu, + "qo_offset_cpu": kv_lens_cpu - 1, + "kv_indices": torch.arange(num_pages, device="cuda", dtype=torch.int32), + "sm_scale": head_dim**-0.5, + "causal": True, + } + + strided_scores = _proxy_max_score(index_q, index_k_strided, **kwargs) + packed_scores = _proxy_max_score(index_q, index_k_packed, **kwargs) + torch.cuda.synchronize() + + assert not index_k_strided.is_contiguous() + assert index_k_strided.stride(0) == coalescing_scale * page_size * head_dim + assert torch.equal(strided_scores, packed_scores) diff --git a/tests/unittest/_torch/attention/test_attention_backends.py b/tests/unittest/_torch/attention/test_attention_backends.py index 8bae3401ba7a..ca00dcd21125 100644 --- a/tests/unittest/_torch/attention/test_attention_backends.py +++ b/tests/unittest/_torch/attention/test_attention_backends.py @@ -9,7 +9,7 @@ configurations actually used by the supported models (``model_attn_config.py``), so each case maps to a real workload. The orthogonal dimensions are bounded: -* default cross on every cacheable config: phase {ctx, dec, mix} x +* default cross on every cacheable config: phase {ctx, gen, mix} x precision {bf16, fp8-KV} x KV-manager {v1, v2}, at page_size=32, layout=HND. * the non-default dimension values (page_size=64, layout=NHD, dtype=fp16) are exercised on a small representative set (GQA / MHA / MQA) to avoid a full @@ -85,6 +85,10 @@ def _phases_for(cfg: ModelAttnConfig) -> dict: return _phases_from_window(cfg.sliding_window) +def _phases_to_run(cfg: ModelAttnConfig, available_phases: dict) -> tuple[str, ...]: + return tuple(available_phases) if cfg.phases is None else cfg.phases + + def _rope_dict(cfg: ModelAttnConfig): if cfg.rope is None: return None @@ -193,13 +197,12 @@ def _expand(cfg: ModelAttnConfig, precisions, kv_layouts, page_sizes): **base, ), ) - elif cfg.is_mla and cfg.mla_context: - yield f"{cfg.id}-ctx-{tag}", BackendCase(**phases["ctx"], **base) - elif cfg.is_mla: - yield f"{cfg.id}-gen-{tag}", BackendCase(**phases["gen"], **base) else: - for phase_name, phase in phases.items(): - yield f"{cfg.id}-{phase_name}-{tag}", BackendCase(**phase, **base) + for phase_name in _phases_to_run(cfg, phases): + yield ( + f"{cfg.id}-{phase_name}-{tag}", + BackendCase(**phases[phase_name], **base), + ) def _model_cases(): diff --git a/tests/unittest/_torch/compilation/test_remove_copy_pass.py b/tests/unittest/_torch/compilation/test_remove_copy_pass.py new file mode 100644 index 000000000000..ef78351e9453 --- /dev/null +++ b/tests/unittest/_torch/compilation/test_remove_copy_pass.py @@ -0,0 +1,175 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable +from operator import getitem + +import pytest +import torch +from torch._higher_order_ops.auto_functionalize import auto_functionalized_v2 +from torch.fx import Graph + +import tensorrt_llm._torch.compilation.remove_copy_pass as remove_copy_pass + + +def test_remove_copy_for_mutates_args_auto_functionalized_v2( + monkeypatch: pytest.MonkeyPatch, +) -> None: + graph = Graph() + output = graph.placeholder("output") + src = graph.placeholder("src") + functionalized = graph.call_function( + auto_functionalized_v2, + args=(torch.ops.aten.copy_.default,), + kwargs={ + "_all_bases": (output,), + "_self_base_index": 0, + "src": src, + }, + ) + mutated_output = graph.call_function(getitem, args=(functionalized, 1)) + clone = graph.call_function(torch.ops.aten.clone.default, args=(mutated_output,)) + graph.output(clone) + + monkeypatch.setattr( + remove_copy_pass, + "inplace_info", + lambda: {torch.ops.aten.copy_.default: {1: "self"}}, + ) + + remove_copy_pass.remove_copy_for_mutates_args(graph) + + inplace_nodes = [node for node in graph.nodes if node.target == torch.ops.aten.copy_.default] + assert len(inplace_nodes) == 1 + assert inplace_nodes[0].kwargs == {"src": src, "self": output} + assert clone.args[0] is output + assert all(node.target != auto_functionalized_v2 for node in graph.nodes) + graph.lint() + + +@pytest.mark.parametrize( + "inplace_func", + [ + torch.ops.trtllm.pp_recv_tensors.default, + torch.ops.trtllm.pp_send_tensors.default, + ], +) +def test_remove_copy_for_mutates_tensor_list( + inplace_func: Callable[..., object], +) -> None: + graph = Graph() + tensor_0 = graph.placeholder("tensor_0") + tensor_1 = graph.placeholder("tensor_1") + functionalized = graph.call_function( + auto_functionalized_v2, + args=(inplace_func,), + kwargs={ + "_all_bases": (tensor_0, tensor_1), + "_tensors_length": 2, + "_tensors_0_base_index": 0, + "_tensors_1_base_index": 1, + }, + ) + mutated_0 = graph.call_function(getitem, args=(functionalized, 1)) + mutated_1 = graph.call_function(getitem, args=(functionalized, 2)) + clone_0 = graph.call_function(torch.ops.aten.clone.default, args=(mutated_0,)) + clone_1 = graph.call_function(torch.ops.aten.clone.default, args=(mutated_1,)) + graph.output((clone_0, clone_1)) + + remove_copy_pass.remove_copy_for_mutates_args(graph) + + inplace_nodes = [node for node in graph.nodes if node.target == inplace_func] + assert len(inplace_nodes) == 1 + assert inplace_nodes[0].kwargs == {"tensors": [tensor_0, tensor_1]} + assert clone_0.args[0] is tensor_0 + assert clone_1.args[0] is tensor_1 + assert all(node.target != auto_functionalized_v2 for node in graph.nodes) + graph.lint() + + +def test_remove_copy_for_mutates_args_restores_optional_none() -> None: + graph = Graph() + hidden_states = graph.placeholder("hidden_states") + output = graph.placeholder("output") + inplace_func = torch.ops.trtllm.mla_custom_op_inplace.default + functionalized = graph.call_function( + auto_functionalized_v2, + args=(inplace_func,), + kwargs={ + "hidden_states": hidden_states, + "position_ids": None, + "layer_idx": "0", + "latent_cache_gen": None, + "enable_dsv4_epilogue_fusion": False, + "_all_bases": (output,), + "_output_base_index": 0, + "_dsv4_output_base_index": None, + "_dsv4_output_sf_base_index": None, + }, + ) + mutated_output = graph.call_function(getitem, args=(functionalized, 1)) + clone = graph.call_function(torch.ops.aten.clone.default, args=(mutated_output,)) + graph.output(clone) + + remove_copy_pass.remove_copy_for_mutates_args(graph) + + inplace_nodes = [node for node in graph.nodes if node.target == inplace_func] + assert len(inplace_nodes) == 1 + assert inplace_nodes[0].kwargs["output"] is output + assert inplace_nodes[0].kwargs["dsv4_output"] is None + assert inplace_nodes[0].kwargs["dsv4_output_sf"] is None + assert clone.args[0] is output + graph.lint() + + +def test_remove_copy_for_mutates_args_rejects_getitem_for_optional_none( + monkeypatch: pytest.MonkeyPatch, +) -> None: + graph = Graph() + hidden_states = graph.placeholder("hidden_states") + output = graph.placeholder("output") + inplace_func = torch.ops.trtllm.mla_custom_op_inplace.default + functionalized = graph.call_function( + auto_functionalized_v2, + args=(inplace_func,), + kwargs={ + "hidden_states": hidden_states, + "position_ids": None, + "layer_idx": "0", + "latent_cache_gen": None, + "enable_dsv4_epilogue_fusion": False, + "_all_bases": (output,), + "_output_base_index": 0, + "_dsv4_output_base_index": None, + "_dsv4_output_sf_base_index": None, + }, + ) + optional_output = graph.call_function(getitem, args=(functionalized, 2)) + clone = graph.call_function(torch.ops.aten.clone.default, args=(optional_output,)) + graph.output(clone) + + monkeypatch.setattr( + remove_copy_pass, + "inplace_info", + lambda: {inplace_func: {1: "output", 2: "dsv4_output"}}, + ) + + with pytest.raises( + AssertionError, + match=( + "getitem user for optional output 'dsv4_output' has no " + "base tensor -- graph is malformed" + ), + ): + remove_copy_pass.remove_copy_for_mutates_args(graph) diff --git a/tests/unittest/_torch/executor/test_adp_router.py b/tests/unittest/_torch/executor/test_adp_router.py index 4c3726578e0d..f59a3c3b233a 100644 --- a/tests/unittest/_torch/executor/test_adp_router.py +++ b/tests/unittest/_torch/executor/test_adp_router.py @@ -1207,11 +1207,23 @@ def test_lru_eviction_bounds_map(self): assert len(router._conv_to_rank) == 2 assert set(router._conv_to_rank) == {"c3", "c4"} - def test_explicit_target_dp_rank_respected(self): + def test_explicit_target_dp_rank_establishes_binding(self): router = self._router(tp_size=4) - item = _make_conv_request_item(1, "A", target_dp_rank=2, attention_dp_relax=False) - pos = self._route(router, self._states(4), [item]) - assert pos[1] == 2 + first = _make_conv_request_item(1, "A", target_dp_rank=2, attention_dp_relax=False) + assert self._route(router, self._states(4), [first])[1] == 2 + assert router._conv_to_rank["A"] == 2 + + second = _make_conv_request_item(2, "A") + assert self._route(router, self._states(4), [second])[2] == 2 + + def test_existing_binding_wins_over_later_explicit_target(self): + router = self._router(tp_size=4) + first = _make_conv_request_item(1, "A", target_dp_rank=2, attention_dp_relax=False) + self._route(router, self._states(4), [first]) + + second = _make_conv_request_item(2, "A", target_dp_rank=1, attention_dp_relax=False) + assert self._route(router, self._states(4), [second])[2] == 2 + assert router._conv_to_rank["A"] == 2 def test_sticky_overflow_keeps_mapping(self): """When the home rank is saturated at the HARD cap (max_num_active_requests), @@ -1247,6 +1259,62 @@ def test_factory_selects_conversation_router(self): router = ADPRouter.create(dist=_mock_dist(), kv_cache_manager=None, attention_dp_config=cfg) assert isinstance(router, ConversationAwareADPRouter) assert router._max_sessions == 8 + # A mocked (non-string) placement value must fall back to round_robin. + assert router._new_conv_placement == "round_robin" + + @staticmethod + def _lq_router(tp_size=4): + return ConversationAwareADPRouter( + dist=_mock_dist(tp_size=tp_size), new_conv_placement="least_queued" + ) + + def test_least_queued_places_new_conversation_on_least_loaded_rank(self): + router = self._lq_router() + pos = self._route( + router, self._states(4, active=[5, 1, 3, 4]), [_make_conv_request_item(1, "A")] + ) + assert pos[1] == 1 + assert router._conv_to_rank["A"] == 1 + + def test_least_queued_fills_valleys_within_one_batch(self): + """The shared count accumulator spreads a burst valley-first: + [3, 0, 2, 3] + 4 new conversations ends level at [3, 3, 3, 3].""" + items = [_make_conv_request_item(i, f"c{i}") for i in range(4)] + pos = self._route(self._lq_router(), self._states(4, active=[3, 0, 2, 3]), items) + placed = [sum(1 for v in pos.values() if v == r) for r in range(4)] + assert placed == [0, 3, 1, 0] + + def test_least_queued_sticky_returns_unaffected(self): + """A later turn returns to its pinned rank even when it is the busiest.""" + router = self._lq_router() + home = self._route( + router, self._states(4, active=[2, 0, 1, 1]), [_make_conv_request_item(1, "A")] + )[1] + assert home == 1 + pos = self._route( + router, self._states(4, active=[0, 9, 0, 0]), [_make_conv_request_item(2, "A")], cap=100 + ) + assert pos[2] == home + + def test_least_queued_routing_is_deterministic_across_ranks(self): + convs = ["A", "B", None, "A", "C", None, "B"] + + def run(): + items = [_make_conv_request_item(i, convs[i]) for i in range(len(convs))] + return self._route(self._lq_router(), self._states(4, active=[2, 5, 0, 1]), items) + + assert run() == run() + + def test_new_conv_placement_config(self): + """Factory forwards the knob; unknown values fall back to round_robin.""" + cfg = MagicMock() + cfg.kv_cache_routing_conversation_affinity = True + cfg.kv_cache_routing_max_sessions = 8 + cfg.kv_cache_routing_new_conv_placement = "least_queued" + router = ADPRouter.create(dist=_mock_dist(), kv_cache_manager=None, attention_dp_config=cfg) + assert router._new_conv_placement == "least_queued" + bad = ConversationAwareADPRouter(dist=_mock_dist(tp_size=4), new_conv_placement="banana") + assert bad._new_conv_placement == "round_robin" def test_factory_default_when_disabled(self): cfg = MagicMock() diff --git a/tests/unittest/_torch/executor/test_disagg_inflight_cancel_gate.py b/tests/unittest/_torch/executor/test_disagg_inflight_cancel_gate.py index 3d9a3f83dcc3..08a6f4701761 100644 --- a/tests/unittest/_torch/executor/test_disagg_inflight_cancel_gate.py +++ b/tests/unittest/_torch/executor/test_disagg_inflight_cancel_gate.py @@ -23,6 +23,10 @@ from tensorrt_llm._torch.pyexecutor import py_executor as executor_module from tensorrt_llm._torch.pyexecutor.kv_cache_transceiver import BindKvCacheTransceiver from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequestState +from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import ( + CppMambaHybridCacheManager, + MambaHybridCacheManagerV2, +) from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor from tensorrt_llm.llmapi.llm_args import CacheTransceiverConfig @@ -519,6 +523,59 @@ def test_flag_unset_preserves_python_transceiver(monkeypatch): constructor.assert_called_once() +def test_python_nixl_transceiver_accepts_v2_mamba_manager(monkeypatch): + config = CacheTransceiverConfig(backend="NIXL", transceiver_runtime="PYTHON") + expected = object() + constructor = Mock(return_value=expected) + fake_module = SimpleNamespace(KvCacheTransceiverV2=constructor) + monkeypatch.setitem(sys.modules, "tensorrt_llm._torch.disaggregation.transceiver", fake_module) + manager = object.__new__(MambaHybridCacheManagerV2) + + result = transceiver_module.create_kv_cache_transceiver( + Mock(), Mock(), manager, Mock(), config, manager + ) + + assert result is expected + constructor.assert_called_once() + + +@pytest.mark.parametrize("runtime", [None, "CPP", "auto"]) +def test_cpp_runtime_rejects_v2_mamba_manager(runtime): + config = CacheTransceiverConfig(backend="NIXL", transceiver_runtime=runtime) + manager = object.__new__(MambaHybridCacheManagerV2) + + with pytest.raises(ValueError, match="requires transceiver_runtime='PYTHON'"): + transceiver_module.create_kv_cache_transceiver( + Mock(), Mock(), manager, Mock(), config, manager + ) + + +def test_python_runtime_rejects_cpp_mamba_manager(): + config = CacheTransceiverConfig(backend="NIXL", transceiver_runtime="PYTHON") + manager = object.__new__(CppMambaHybridCacheManager) + + with pytest.raises(ValueError, match="cannot drive CppMambaHybridCacheManager"): + transceiver_module.create_kv_cache_transceiver( + Mock(), Mock(), manager, Mock(), config, manager + ) + + +@pytest.mark.parametrize("runtime", [None, "CPP"]) +def test_cpp_runtime_keeps_cpp_mamba_manager(monkeypatch, runtime): + config = CacheTransceiverConfig(backend="NIXL", transceiver_runtime=runtime) + manager = object.__new__(CppMambaHybridCacheManager) + expected = object() + constructor = Mock(return_value=expected) + monkeypatch.setattr(transceiver_module, "BindKvCacheTransceiver", constructor) + + result = transceiver_module.create_kv_cache_transceiver( + Mock(), Mock(), manager, Mock(), config, manager + ) + + assert result is expected + constructor.assert_called_once() + + def test_flag_unset_preserves_libfabric_selection(monkeypatch): monkeypatch.setenv(transceiver_module._NIXL_KVCACHE_BACKEND_ENV, "LIBFABRIC") config = CacheTransceiverConfig(backend="NIXL") diff --git a/tests/unittest/_torch/executor/test_hang_detector_kill.py b/tests/unittest/_torch/executor/test_hang_detector_kill.py index b4f122bb8ad3..0962df441cc6 100644 --- a/tests/unittest/_torch/executor/test_hang_detector_kill.py +++ b/tests/unittest/_torch/executor/test_hang_detector_kill.py @@ -14,12 +14,14 @@ # limitations under the License. """HangDetector timer behavior and the hard-kill propagation mechanism (no GPU).""" +import asyncio import os import signal import subprocess import sys import time +from tensorrt_llm._torch.pyexecutor import hang_detector as hang_detector_module from tensorrt_llm._torch.pyexecutor.hang_detector import HangDetector @@ -64,6 +66,39 @@ def test_pause_suppresses_detection(): assert hd.detected() is False +def test_status_provider_errors_are_logged(monkeypatch): + events = [] + + async def no_sleep(_timeout): + pass + + def failing_provider(): + raise RuntimeError("provider failed") + + monkeypatch.setattr(hang_detector_module.asyncio, "sleep", no_sleep) + monkeypatch.setattr( + hang_detector_module, + "_best_effort_log_error", + lambda message: events.append(("log", message)), + ) + monkeypatch.setattr( + hang_detector_module, + "print_all_stacks", + lambda: events.append(("stacks", None)), + ) + + detector = HangDetector(timeout=1, on_detected=lambda: events.append(("detected", None))) + detector.register_status_provider(failing_provider) + detector.register_status_provider(lambda: "transceiver status") + + asyncio.run(detector._detect_hang()) + + messages = "\n".join(message for kind, message in events if kind == "log") + assert "provider failed" in messages + assert "transceiver status" in messages + assert events[-2:] == [("stacks", None), ("detected", None)] + + def test_propagate_hard_kill_self_sigkills_without_mpi(): """With MPI disabled, propagate_hard_kill self-SIGKILLs the process. diff --git a/tests/unittest/_torch/executor/test_iter_stats_populate.py b/tests/unittest/_torch/executor/test_iter_stats_populate.py index 051047c4442b..1aad42de9499 100644 --- a/tests/unittest/_torch/executor/test_iter_stats_populate.py +++ b/tests/unittest/_torch/executor/test_iter_stats_populate.py @@ -47,7 +47,7 @@ ADPIterStatsBuffer, ) from tensorrt_llm._torch.pyexecutor.scheduler.adp_router import RankIterStatsPayload, RankState -from tensorrt_llm.bindings.executor import InflightBatchingStats, IterationStats +from tensorrt_llm.bindings.executor import InflightBatchingStats, IterationStats, SpecDecodingStats class _StubRequest: @@ -73,6 +73,9 @@ def __init__( is_attention_dp_dummy: bool = False, is_cuda_graph_dummy: bool = False, is_dummy_request: bool = False, + num_draft_tokens: int = 0, + py_draft_tokens=None, + py_num_accepted_draft_tokens=None, ): self.context_current_position = context_current_position # py_last_context_chunk = (begin_compute, end_compute). For a fresh @@ -88,6 +91,14 @@ def __init__( self.is_attention_dp_dummy = is_attention_dp_dummy self.is_cuda_graph_dummy = is_cuda_graph_dummy self.is_dummy_request = is_dummy_request + # Speculative-decoding accessors read by the specdec-stats aggregation + # branch of ``_update_iter_stats``. Defaults match a non-drafted request + # (draft_len resolves to 0, so it is skipped by the ``draft_len > 0`` + # gate). A dummy gen request mirrors resource_manager.add_dummy_requests' + # is_gen branch: py_draft_tokens = [1] * k plus an accepted count. + self.num_draft_tokens = num_draft_tokens + self.py_draft_tokens = py_draft_tokens + self.py_num_accepted_draft_tokens = py_num_accepted_draft_tokens @property def is_dummy(self) -> bool: @@ -189,6 +200,7 @@ def _invoke_update_iter_stats( num_ctx_tokens, enable_attention_dp=False, scheduled_batch_stats=None, + specdec=False, ): """Call real ``PyExecutor._update_iter_stats`` unbound; return the stats. @@ -207,6 +219,11 @@ def _invoke_update_iter_stats( Explicit scheduled-batch counters passed to ``_update_iter_stats``. When None, defaults to the same partial ``num_ctx_tokens`` payload used by older tests. + specdec : bool + Allocate ``stats.specdec_stats`` so the speculative-decoding + aggregation branch runs. On a fresh ``IterationStats`` this member is + None (production sets it in ``_get_init_iter_stats`` only when spec + decode is enabled), and the branch is skipped otherwise. """ from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor, ScheduledBatchStats @@ -226,6 +243,8 @@ def _invoke_update_iter_stats( # The method reads ``stats.inflight_batching_stats.*`` unconditionally; # the default on a fresh IterationStats is None, so we allocate one. stats.inflight_batching_stats = InflightBatchingStats() + if specdec: + stats.specdec_stats = SpecDecodingStats() with patch( "tensorrt_llm._torch.pyexecutor.py_executor.torch.cuda.mem_get_info", @@ -587,6 +606,96 @@ def test_num_gen_kv_tokens_uses_scheduled_batch_stats(): assert ifb.num_gen_kv_tokens == 1024 +# --------------------------------------------------------------------------- +# Speculative-decoding aggregation tests: the specdec branch of +# ``_update_iter_stats`` sums draft/accepted tokens across generation +# requests and derives the acceptance length. Attention-DP / CUDA-graph +# padding adds *generation* dummies carrying synthetic +# ``py_draft_tokens = [1] * k`` and a spec-sampler accepted count +# (resource_manager.add_dummy_requests is_gen branch); they must not enter +# the aggregate or they depress the reported acceptance rate/length. This +# path is distinct from the KV-token dummy filtering above and is only +# reached when ``stats.specdec_stats`` is allocated (spec decode enabled). +# --------------------------------------------------------------------------- + + +def test_specdec_excludes_dummy_generation_requests(): + """Regression guard: dummy gen requests must not pollute specdec AR/AL. + + One real drafted request (k=4, 3 accepted) alongside an attention-DP + dummy and a CUDA-graph dummy, each mirroring the is_gen dummy branch + (``num_draft_tokens == 0`` with ``py_draft_tokens = [1] * 4``). The + aggregate must reflect only the real request: 4 draft tokens, 3 + accepted, one drafted request, and acceptance_length = (3 + 1) / 1. + + Without the ``is_dummy`` filter each dummy would contribute draft_len=4 + (counted from the non-zero ``py_draft_tokens``) and 0 accepted, giving + num_requests_with_draft=3 and a depressed acceptance_length of + (3 + 3) / 3 = 2.0 instead of 4.0. + """ + real = _StubRequest(num_draft_tokens=4, py_num_accepted_draft_tokens=3) + adp_dummy = _StubRequest( + py_draft_tokens=[1, 1, 1, 1], + py_num_accepted_draft_tokens=0, + is_attention_dp_dummy=True, + ) + cuda_graph_dummy = _StubRequest( + py_draft_tokens=[1, 1, 1, 1], + py_num_accepted_draft_tokens=0, + is_cuda_graph_dummy=True, + ) + stats = _invoke_update_iter_stats( + _StubScheduledBatch(gen_reqs=[real, adp_dummy, cuda_graph_dummy]), + [], + num_ctx_tokens=0, + specdec=True, + ) + sd = stats.specdec_stats + assert sd.num_draft_tokens == 4 + assert sd.num_accepted_tokens == 3 + assert sd.num_requests_with_draft_tokens == 1 + assert sd.acceptance_length == 4.0 + + +def test_specdec_all_dummy_generation_requests_yield_zero(): + """An iteration of only dummy gen requests reports no drafted work.""" + dummies = [ + _StubRequest(py_draft_tokens=[1, 1, 1, 1], is_attention_dp_dummy=True), + _StubRequest(py_draft_tokens=[1, 1, 1, 1], is_cuda_graph_dummy=True), + ] + stats = _invoke_update_iter_stats( + _StubScheduledBatch(gen_reqs=dummies), + [], + num_ctx_tokens=0, + specdec=True, + ) + sd = stats.specdec_stats + assert sd.num_draft_tokens == 0 + assert sd.num_accepted_tokens == 0 + assert sd.num_requests_with_draft_tokens == 0 + assert sd.acceptance_length == 0.0 + + +def test_specdec_aggregates_multiple_real_requests(): + """Baseline: with no dummies, every real drafted request is aggregated.""" + reqs = [ + _StubRequest(num_draft_tokens=4, py_num_accepted_draft_tokens=3), + _StubRequest(num_draft_tokens=4, py_num_accepted_draft_tokens=1), + ] + stats = _invoke_update_iter_stats( + _StubScheduledBatch(gen_reqs=reqs), + [], + num_ctx_tokens=0, + specdec=True, + ) + sd = stats.specdec_stats + assert sd.num_draft_tokens == 8 + assert sd.num_accepted_tokens == 4 + assert sd.num_requests_with_draft_tokens == 2 + # (total_accepted + num_requests_with_draft) / num_requests_with_draft + assert sd.acceptance_length == (4 + 2) / 2 + + # --------------------------------------------------------------------------- # Attention-DP fanout tests: completed rank-local payloads are carried by the # next ADP allgather, then rank 0 appends one row per ADP rank. diff --git a/tests/unittest/_torch/executor/test_kv_block_offset_overlap_race.py b/tests/unittest/_torch/executor/test_kv_block_offset_overlap_race.py index 92aba91ead1b..ff5743430938 100644 --- a/tests/unittest/_torch/executor/test_kv_block_offset_overlap_race.py +++ b/tests/unittest/_torch/executor/test_kv_block_offset_overlap_race.py @@ -102,6 +102,53 @@ def _reference_offsets(mgr, ids): return dst[:, : len(ids)].clone() +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a CUDA device") +def test_copy_batch_block_offsets_max_blocks_staging_width(): + """``max_blocks`` bounds the staged H2D width; ``None`` stages the full + allocated width. + + The bounded copy must leave destination columns beyond the cap untouched + (callers only pass a cap when nothing reads past it), and the unbounded + copy must ship every allocated block: speculative decoding allocates + blocks past the host kv_lens snapshot and its kernels dereference those + columns, so capping by a host-derived block count corrupted the device + block table (EAGLE3 warmup illegal memory access). + """ + mgr = _build_manager() + + ids = list(range(1, 1 + _BATCH)) + toks = [_TOKENS_PER_SEQ] * _BATCH # 5 allocated blocks per sequence + mgr.add_dummy_requests(request_ids=ids, token_nums=toks, prepare_resource=True) + allocated_blocks = _TOKENS_PER_SEQ // _TOKENS_PER_BLOCK + + ref = _reference_offsets(mgr, ids) + + sentinel = -12345 + capped_width = 2 + assert capped_width < allocated_blocks <= mgr.max_blocks_per_seq + + dst = torch.full( + (mgr.num_pools, 2 * _BATCH, 2, mgr.max_blocks_per_seq), + sentinel, + dtype=torch.int32, + device="cuda", + ) + mgr.copy_batch_block_offsets(dst, ids, 1, _BATCH, _BATCH, max_blocks=capped_width) + torch.cuda.synchronize() + assert torch.equal(dst[:, :_BATCH, :, :capped_width], ref[..., :capped_width]) + assert (dst[:, :_BATCH, :, capped_width:] == sentinel).all(), ( + "bounded staging must not write past the requested block width" + ) + + dst.fill_(sentinel) + mgr.copy_batch_block_offsets(dst, ids, 1, _BATCH, _BATCH, max_blocks=None) + torch.cuda.synchronize() + assert torch.equal(dst[:, :_BATCH, :, :allocated_blocks], ref[..., :allocated_blocks]), ( + "max_blocks=None must stage every allocated block, including blocks " + "past the batch's current kv length (speculative decoding reads them)" + ) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a CUDA device") def test_copy_batch_block_offsets_survives_overlap_overwrite(): mgr = _build_manager() diff --git a/tests/unittest/_torch/executor/test_kv_cache_compression_manager.py b/tests/unittest/_torch/executor/test_kv_cache_compression_manager.py index 13cc7fd9d6d2..7c11357c3210 100644 --- a/tests/unittest/_torch/executor/test_kv_cache_compression_manager.py +++ b/tests/unittest/_torch/executor/test_kv_cache_compression_manager.py @@ -2,18 +2,18 @@ # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. """Unit tests for the KV-cache compression manager framework -(``BaseKVCacheCompressionManager`` in ``resource_manager.py``) — the +(``KVCacheCompressionManager`` in ``resource_manager.py``) — the ``BaseResourceManager``-based single-manager design. Covers: -- :class:`BaseKVCacheCompressionManager` contract: the four lifecycle hooks +- :class:`KVCacheCompressionManager` contract: the four lifecycle hooks default to no-op, zero resource counts, and it inherits :class:`BaseResourceManager` (so PyExecutor auto-drives it once registered). - The resource-manager API -> lifecycle-hook translation, gated on PyExecutor's own signals: ``prepare_resources`` fires ``on_request_init`` on each request's first prefill chunk (``is_first_context_chunk``); - ``update_resources`` fires ``on_context_step_end`` for each request in - ``context_requests_last_chunk`` + one ``on_generation_step_end`` per + ``update_resources`` fires ``on_context_step_end`` once with the + ``context_requests_last_chunk`` list + one ``on_generation_step_end`` per iteration; ``free_resources`` fires ``on_request_finish``. - :func:`create_kv_cache_compression_manager` factory. @@ -22,6 +22,8 @@ lives in ``_util.py`` next to ``_create_kv_cache_manager``. """ +from types import SimpleNamespace +from typing import ClassVar from unittest.mock import MagicMock, patch import pytest @@ -29,8 +31,10 @@ from tensorrt_llm._torch.pyexecutor import _util as util_mod from tensorrt_llm._torch.pyexecutor._util import create_kv_cache_compression_manager from tensorrt_llm._torch.pyexecutor.resource_manager import ( - BaseKVCacheCompressionManager, BaseResourceManager, + KVCacheCompressionManager, + ResourceManager, + ResourceManagerType, ) # ---------------------------------------------------------------------- # @@ -51,29 +55,41 @@ def _record(self, hook_name: str): self._record_list.append(f"{self._name}:{hook_name}") -class _MockCompressionManager(_RecordingMixin, BaseKVCacheCompressionManager): +class _MockCompressionManager(_RecordingMixin, KVCacheCompressionManager): """Mock manager that records the four lifecycle hooks.""" def on_request_init(self, request): self._record("on_request_init") - def on_context_step_end(self, request, metadata): - self._record("on_context_step_end") + def on_context_step_end(self, requests): + self._record(f"on_context_step_end[{len(requests)}]") - def on_generation_step_end(self, scheduled_batch, attn_metadata): + def on_generation_step_end(self, scheduled_batch): self._record("on_generation_step_end") def on_request_finish(self, request): self._record("on_request_finish") +class _LengthAdjustingCompressionManager(KVCacheCompressionManager): + adjusts_generation_kv_length: ClassVar[bool] = True + + +def _v2_manager(*, is_draft: bool): + from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2 + + manager = KVCacheManagerV2.__new__(KVCacheManagerV2) + manager.enable_block_reuse = False + manager.kv_compression_manages_history = False + manager.is_draft = is_draft + return manager + + @pytest.fixture def fake_kv_cache_manager(): """A stand-in KVCacheManagerV2. The framework reads enable_block_reuse off it in __init__; default it to False, like a normal run with reuse off.""" - m = MagicMock(name="fake_KVCacheManagerV2") - m.enable_block_reuse = False - return m + return _v2_manager(is_draft=False) def _req(rid, first_chunk=True): @@ -92,37 +108,73 @@ def _batch(context=(), generation=(), last_chunk=()): # ---------------------------------------------------------------------- # -# 1. BaseKVCacheCompressionManager contract # +# 1. KVCacheCompressionManager contract # # ---------------------------------------------------------------------- # class TestBaseABC: def test_inherits_base_resource_manager(self): # So PyExecutor's main loop auto-invokes prepare/update/free_resources. - assert issubclass(BaseKVCacheCompressionManager, BaseResourceManager) + assert issubclass(KVCacheCompressionManager, BaseResourceManager) def test_four_hooks_default_noop(self, fake_kv_cache_manager): - m = BaseKVCacheCompressionManager(fake_kv_cache_manager) - meta = MagicMock() + m = KVCacheCompressionManager(fake_kv_cache_manager) assert m.on_request_init(MagicMock()) is None - assert m.on_context_step_end(MagicMock(), meta) is None - assert m.on_generation_step_end(MagicMock(), meta) is None + assert m.on_context_step_end([MagicMock()]) is None + assert m.on_generation_step_begin(MagicMock()) is None + assert m.on_generation_step_end(MagicMock()) is None assert m.on_request_finish(MagicMock()) is None def test_hooks_accept_extra_kwargs(self, fake_kv_cache_manager): # **kwargs lets the framework pass new args later without breaking # existing overrides. - m = BaseKVCacheCompressionManager(fake_kv_cache_manager) + m = KVCacheCompressionManager(fake_kv_cache_manager) assert m.on_request_init(MagicMock(), future_arg=1) is None - assert m.on_generation_step_end(MagicMock(), MagicMock(), future_arg=1) is None + assert m.on_generation_step_end(MagicMock(), future_arg=1) is None def test_resource_counts_are_zero(self, fake_kv_cache_manager): - m = BaseKVCacheCompressionManager(fake_kv_cache_manager) + m = KVCacheCompressionManager(fake_kv_cache_manager) # The manager owns no physical resources (the V2 cache manager does), # so it must not gate the scheduler. assert m.get_max_resource_count() == 0 assert m.get_needed_resource_to_completion(MagicMock()) == 0 + def test_length_adjustment_marks_target_and_draft_v2(self): + # The draft cache is compacted together with the target, so both + # managers diverge from the logical length in the same way. + target = _v2_manager(is_draft=False) + draft = _v2_manager(is_draft=True) + + manager = _LengthAdjustingCompressionManager(target, draft) + + assert manager.kv_cache_manager is target + assert manager.draft_kv_cache_manager is draft + assert manager.has_independent_draft_kv_cache + assert target.kv_compression_manages_history is True + assert draft.kv_compression_manages_history is True + + def test_rejects_non_v2_ownership(self): + with pytest.raises(TypeError, match="requires KVCacheManagerV2"): + KVCacheCompressionManager(MagicMock()) + with pytest.raises(TypeError, match="requires KVCacheManagerV2"): + KVCacheCompressionManager(_v2_manager(is_draft=False), MagicMock()) + + def test_request_field_defaults_to_zero(self): + """LlmRequest carries the compression count (the manager's only + channel to the runtime); a fresh request must default to 0 so runs + without a compression manager are unchanged.""" + from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest + from tensorrt_llm.bindings import SamplingConfig + + request = LlmRequest( + request_id=1, + max_new_tokens=8, + input_tokens=[1, 2, 3], + sampling_config=SamplingConfig(), + is_streaming=False, + ) + assert request.py_num_compressed_tokens == 0 + # ---------------------------------------------------------------------- # # 2. Resource-manager API -> lifecycle-hook translation # @@ -131,6 +183,46 @@ def test_resource_counts_are_zero(self, fake_kv_cache_manager): class TestResourceManagerAPI: + def test_target_update_receives_metadata_before_final_compression(self): + calls = [] + metadata = MagicMock(name="attention_metadata") + draft = MagicMock(name="draft_kv_cache_manager") + target = MagicMock(name="target_kv_cache_manager") + compression = MagicMock(name="compression_manager") + draft.update_resources.side_effect = lambda *args: calls.append(("draft", args)) + target.update_resources.side_effect = lambda *args: calls.append(("target", args)) + compression.update_resources.side_effect = lambda *args: calls.append(("compression", args)) + manager = ResourceManager( + { + ResourceManagerType.DRAFT_KV_CACHE_MANAGER: draft, + ResourceManagerType.KV_CACHE_MANAGER: target, + ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER: compression, + } + ) + batch = _batch(generation=[_req(1)]) + + manager.update_resources(batch, metadata, 2.0) + + assert calls == [ + ("draft", (batch,)), + ("target", (batch, metadata, 2.0)), + ("compression", (batch,)), + ] + + def test_real_v2_target_receives_relocation_metadata(self): + from tensorrt_llm._torch.pyexecutor import kv_cache_manager_v2 as kv_cache_v2_module + + target = _v2_manager(is_draft=False) + target.kv_cache_map = {} + batch = _batch(generation=[_req(1)]) + metadata = MagicMock(name="attention_metadata") + manager = ResourceManager({ResourceManagerType.KV_CACHE_MANAGER: target}) + + with patch.object(kv_cache_v2_module, "_update_kv_cache_draft_token_location") as relocate: + manager.update_resources(batch, metadata, 2.0) + + relocate.assert_called_once_with(target, batch, metadata, 2.0) + def test_prepare_fires_init_on_first_chunk_only(self, fake_kv_cache_manager): rec = [] m = _MockCompressionManager(fake_kv_cache_manager, rec, "s") @@ -144,9 +236,10 @@ def test_update_fires_context_end_on_last_chunk(self, fake_kv_cache_manager): rec = [] m = _MockCompressionManager(fake_kv_cache_manager, rec, "s") req = _req(1) - # Request's final prefill chunk this iteration -> context_step_end fires. - m.update_resources(_batch(generation=[req], last_chunk=[req]), attn_metadata=MagicMock()) - assert "s:on_context_step_end" in rec + # Final prefill chunks this iteration -> one batched context_step_end. + req2 = _req(2) + m.update_resources(_batch(generation=[req], last_chunk=[req, req2])) + assert "s:on_context_step_end[2]" in rec assert rec[-1] == "s:on_generation_step_end" # Subsequent decode iteration (not in last_chunk) -> no context_step_end. rec.clear() @@ -185,6 +278,42 @@ def test_warns_for_unregistered_algorithm(self, fake_kv_cache_manager): create_kv_cache_compression_manager(cfg, fake_kv_cache_manager) mock_logger.warning.assert_called_once() + def test_factory_accepts_independent_draft_manager(self): + cfg = MagicMock() + cfg.algorithm = "made_up_method" + target = _v2_manager(is_draft=False) + draft = _v2_manager(is_draft=True) + + assert ( + create_kv_cache_compression_manager( + cfg, + target, + draft_kv_cache_manager=draft, + ) + is None + ) + + def test_eviction_method_predicate_defaults_false(self): + # Non-evicting methods (e.g. offloading) are never restricted by the + # speculative mode: the call-site gate reads this config predicate. + from tensorrt_llm.llmapi.llm_args import KvCacheCompressionConfig + + config = KvCacheCompressionConfig(algorithm="offload") + assert config.kv_cache_compression_mode.is_eviction_method() is False + m = KVCacheCompressionManager(_v2_manager(is_draft=False)) + assert not hasattr(m, "spec_config") + + def test_spec_gate_only_restricts_eviction_methods(self): + from tensorrt_llm._torch.pyexecutor._util import validate_kv_cache_compression_with_spec + from tensorrt_llm._torch.speculative.interface import SpeculativeDecodingMode + from tensorrt_llm.llmapi.llm_args import KvCacheCompressionConfig + + # Non-evicting methods pass with any speculative mode; no exception. + config = KvCacheCompressionConfig(algorithm="offload") + spec_config = SimpleNamespace(spec_dec_mode=SpeculativeDecodingMode.DFLASH) + validate_kv_cache_compression_with_spec(config, spec_config, None) + validate_kv_cache_compression_with_spec(config, None, None) + # ---------------------------------------------------------------------- # # 4. Canonical names live in resource_manager, not in the sparse module # @@ -197,7 +326,7 @@ def test_names_importable_from_canonical_modules(self): # Base class stays in resource_manager (it IS a resource manager); the # factory lives in _util next to _create_kv_cache_manager. - assert hasattr(resource_manager, "BaseKVCacheCompressionManager") + assert hasattr(resource_manager, "KVCacheCompressionManager") assert hasattr(_util, "create_kv_cache_compression_manager") def test_names_not_in_sparse_module(self): @@ -205,7 +334,7 @@ def test_names_not_in_sparse_module(self): # sparse-attention backend); the sparse package no longer exports it. from tensorrt_llm._torch.attention_backend import sparse - assert not hasattr(sparse, "BaseKVCacheCompressionManager") + assert not hasattr(sparse, "KVCacheCompressionManager") assert not hasattr(sparse, "create_kv_cache_compression_manager") @@ -219,13 +348,13 @@ class TestBlockReuseGuard: and values, the same check RocketKVCacheManager makes.""" def _mgr(self, enable_block_reuse): - m = MagicMock(name="KVCacheManagerV2") + m = _v2_manager(is_draft=False) m.enable_block_reuse = enable_block_reuse return m def test_raises_when_reuse_on(self): with pytest.raises(ValueError, match="block reuse"): - BaseKVCacheCompressionManager(self._mgr(enable_block_reuse=True)) + KVCacheCompressionManager(self._mgr(enable_block_reuse=True)) def test_ok_when_reuse_off(self): - BaseKVCacheCompressionManager(self._mgr(enable_block_reuse=False)) # no raise + KVCacheCompressionManager(self._mgr(enable_block_reuse=False)) # no raise diff --git a/tests/unittest/_torch/executor/test_kv_cache_estimation.py b/tests/unittest/_torch/executor/test_kv_cache_estimation.py index 8a5be984f312..6a30bf974d68 100644 --- a/tests/unittest/_torch/executor/test_kv_cache_estimation.py +++ b/tests/unittest/_torch/executor/test_kv_cache_estimation.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Tests for KV cache token estimation in KvCacheCreator._get_token_num_for_estimation. Guards the ADP (Attention Data Parallelism) cache-block reduction: when @@ -601,6 +604,9 @@ def test_estimation_temporarily_uses_inferred_pool_sizing() -> None: ) model_engine = Mock() model_engine.model.model_config.attn_backend = "TRTLLM" + # Explicit False: try_prepare_estimation skips estimation for + # encoder-decoder models, and a bare Mock attribute is truthy. + model_engine.model.model_config.is_encoder_decoder = False llm_args = Mock(cache_transceiver_config=None) with patch.object( @@ -649,3 +655,60 @@ def test_estimation_temporarily_uses_inferred_pool_sizing() -> None: assert kv_cache_config.max_tokens == user_max_tokens assert kv_cache_config.pool_ratio == pool_ratio assert kv_cache_config.avg_seq_len == avg_seq_len + + +@pytest.mark.parametrize( + ("estimating_kv_cache", "expected_avg_seq_len"), + [(True, 2045), (False, 2055)], +) +def test_manager_estimation_clamps_only_temporary_avg_seq_len( + estimating_kv_cache, + expected_avg_seq_len, +) -> None: + import torch + + from tensorrt_llm._torch.pyexecutor._util import _create_kv_cache_manager + + captured_configs = [] + + class _RecordingKVCacheManagerV2(KVCacheManagerV2): + def __init__(self, kv_cache_config, _kv_cache_type, **kwargs) -> None: + captured_configs.append(kv_cache_config) + self.max_seq_len = kwargs["max_seq_len"] + + pretrained = SimpleNamespace( + hidden_size=1024, + num_attention_heads=8, + num_key_value_heads=8, + num_hidden_layers=2, + vocab_size=32000, + ) + model_config = Mock() + model_config.pretrained_config = pretrained + model_config.quant_config = None + kv_cache_config = KvCacheConfig( + max_tokens=2048, + avg_seq_len=2055, + ) + + _create_kv_cache_manager( + model_engine=None, + kv_cache_manager_cls=_RecordingKVCacheManagerV2, + mapping=Mock(), + kv_cache_config=kv_cache_config, + tokens_per_block=32, + max_seq_len=2045, + max_batch_size=4, + spec_config=None, + sparse_attention_config=None, + max_num_tokens=2048, + max_beam_width=1, + kv_connector_manager=None, + estimating_kv_cache=estimating_kv_cache, + model_config=model_config, + dtype=torch.bfloat16, + is_draft=False, + ) + + assert captured_configs[0].avg_seq_len == expected_avg_seq_len + assert kv_cache_config.avg_seq_len == 2055 diff --git a/tests/unittest/_torch/executor/test_kv_cache_manager_v2.py b/tests/unittest/_torch/executor/test_kv_cache_manager_v2.py index be1f2936abf8..b625cc974dee 100644 --- a/tests/unittest/_torch/executor/test_kv_cache_manager_v2.py +++ b/tests/unittest/_torch/executor/test_kv_cache_manager_v2.py @@ -1,14 +1,43 @@ -# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass, field from types import SimpleNamespace +from unittest.mock import patch import pytest +import torch from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import BlockReusePolicy, KVCacheManagerV2 -from tensorrt_llm.bindings.internal.batch_manager import CacheType as CacheTypeCpp +from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests +from tensorrt_llm.bindings import DataType +from tensorrt_llm.bindings.internal.batch_manager import CacheType +from tensorrt_llm.conversation_params import ConversationParams from tensorrt_llm.llmapi.llm_args import KvCacheConfig -from tensorrt_llm.runtime.kv_cache_manager_v2 import GpuCacheTierConfig, KVCacheManagerConfig +from tensorrt_llm.mapping import Mapping +from tensorrt_llm.runtime.kv_cache_manager_v2 import ( + DEFAULT_BEAM_INDEX, + BatchDesc, + GpuCacheTierConfig, + KVCacheDesc, + KVCacheManagerConfig, +) +from tensorrt_llm.runtime.kv_cache_manager_v2._utils import init_cuda_once + +TOKENS_PER_BLOCK = 4 +MAX_SEQ_LEN = 16 class _FakeKVCache: @@ -25,26 +54,37 @@ def stop_committing(self) -> None: self.stopped_committing = True -def _build_cache_config_for_test( - kv_cache_config: KvCacheConfig, *, is_draft: bool = False +def _make_cache_config_for_test( + kv_cache_config: KvCacheConfig, + *, + is_draft: bool = False, + max_batch_size: int = 1, + max_seq_len: int = 1024, + max_num_tokens: int | None = None, + max_draft_len: int = 0, + num_extra_kv_tokens: int = 0, ) -> KVCacheManagerConfig: cache_manager = object.__new__(KVCacheManagerV2) - cache_manager.kv_cache_type = CacheTypeCpp.SELFKONLY + cache_manager.kv_cache_type = CacheType.SELFKONLY + cache_manager.dtype = DataType.HALF cache_manager.head_dim_per_layer = [128] cache_manager.enable_swa_scratch_reuse = False - cache_manager.num_extra_kv_tokens = 0 + cache_manager.num_extra_kv_tokens = num_extra_kv_tokens cache_manager.enable_stats = False cache_manager.block_reuse_policy = BlockReusePolicy(kv_cache_config.block_reuse_policy) cache_manager.is_draft = is_draft cache_manager.num_local_layers = 1 cache_manager.pp_layers = [0] cache_manager.max_attention_window_vec = [None] + cache_manager.max_seq_len = max_seq_len + cache_manager.max_batch_size = max_batch_size + cache_manager.max_num_tokens = max_num_tokens + cache_manager.max_draft_len = max_draft_len cache_manager.get_layer_bytes_per_token = lambda **_: 128 - return cache_manager._build_cache_config( + return cache_manager._build_base_config( kv_cache_config, tokens_per_block=128, - vocab_size=129280, cache_tiers=[GpuCacheTierConfig(quota=1 << 30)], ) @@ -64,7 +104,7 @@ def test_commit_min_snapshot_follows_block_reuse_policy( is_draft: bool, commit_min_snapshot: bool, ) -> None: - config = _build_cache_config_for_test( + config = _make_cache_config_for_test( KvCacheConfig( enable_block_reuse=enable_block_reuse, block_reuse_policy=block_reuse_policy, @@ -79,11 +119,99 @@ def test_commit_min_snapshot_follows_block_reuse_policy( @pytest.mark.parametrize("enable_partial_reuse", [False, True]) def test_propagates_partial_reuse_config(enable_partial_reuse: bool) -> None: - config = _build_cache_config_for_test(KvCacheConfig(enable_partial_reuse=enable_partial_reuse)) + config = _make_cache_config_for_test(KvCacheConfig(enable_partial_reuse=enable_partial_reuse)) assert config.enable_partial_reuse is enable_partial_reuse +def test_pool_ratio_overrides_constraints() -> None: + config = _make_cache_config_for_test( + KvCacheConfig(pool_ratio=[1.0], avg_seq_len=256, host_cache_size=0), + max_batch_size=3, + max_num_tokens=2048, + ) + + assert config.initial_pool_ratio == pytest.approx([1.0]) + assert config.typical_step is None + assert config.constraints == [] + + +def test_default_uses_allocator_fallback() -> None: + config = _make_cache_config_for_test( + KvCacheConfig(host_cache_size=0), + max_batch_size=3, + max_seq_len=1024, + max_num_tokens=2048, + max_draft_len=2, + ) + + assert config.initial_pool_ratio is None + assert config.typical_step is None + assert config.constraints == [] + + +def test_avg_seq_len_builds_warmup_constraints() -> None: + config = _make_cache_config_for_test( + KvCacheConfig(host_cache_size=0, avg_seq_len=1024), + max_batch_size=3, + max_seq_len=1024, + max_num_tokens=2048, + max_draft_len=2, + ) + + assert config.typical_step == BatchDesc( + [KVCacheDesc(capacity=2048, history_length=0)] + + [KVCacheDesc(capacity=1024, history_length=1021)] * 2 + ) + assert config.constraints == [ + BatchDesc( + [ + KVCacheDesc(capacity=1024, history_length=1023), + KVCacheDesc(capacity=3, history_length=0), + KVCacheDesc(capacity=3, history_length=0), + ] + ), + BatchDesc([KVCacheDesc(capacity=2048, history_length=0)]), + ] + + +def test_avg_seq_len_updates_typical_step() -> None: + config = _make_cache_config_for_test( + KvCacheConfig(avg_seq_len=256), + max_batch_size=3, + max_seq_len=1024, + max_num_tokens=2048, + max_draft_len=2, + ) + + assert config.typical_step == BatchDesc( + [KVCacheDesc(capacity=2048, history_length=0)] + + [KVCacheDesc(capacity=256, history_length=253)] * 2 + ) + + +def test_avg_seq_len_must_not_exceed_max_seq_len() -> None: + with pytest.raises(ValueError, match="avg_seq_len"): + _make_cache_config_for_test( + KvCacheConfig(avg_seq_len=2048), + max_seq_len=1024, + ) + + +def test_extra_tokens_are_in_context_capacity() -> None: + config = _make_cache_config_for_test( + KvCacheConfig(avg_seq_len=264), + max_batch_size=1, + max_seq_len=264, + max_num_tokens=256, + max_draft_len=3, + num_extra_kv_tokens=2, + ) + + assert config.typical_step == BatchDesc([KVCacheDesc(capacity=258, history_length=0)]) + assert config.constraints[1] == BatchDesc([KVCacheDesc(capacity=258, history_length=0)]) + + def test_try_commit_blocks_commits_partial_block_at_context_end() -> None: request = SimpleNamespace( py_request_id=1, @@ -104,3 +232,341 @@ def test_try_commit_blocks_commits_partial_block_at_context_end() -> None: assert kv_cache.committed_tokens == [4, 5, 6, 7, 8, 9] assert kv_cache.num_committed_tokens == 10 assert kv_cache.stopped_committing + + +@dataclass +class _ContextRequest: + request_id: int + tokens: list[int] + context_remaining_length: int + conversation_id: str + py_request_id: int = field(init=False) + py_conversation_params: ConversationParams | None = field(init=False) + use_conversation_params: bool = True + lora_task_id: int | None = None + cache_salt: str | None = None + is_first_context_chunk: bool = True + is_last_context_chunk: bool = True + is_disagg_generation_init_state: bool = False + is_dummy_request: bool = False + context_current_position: int = 0 + prepopulated_prompt: tuple[int, int] | None = None + multimodal_hashes: None = None + multimodal_positions: None = None + multimodal_lengths: None = None + + def __post_init__(self) -> None: + self.py_request_id = self.request_id + if not self.use_conversation_params: + self.py_conversation_params = None + return + self.py_conversation_params = ConversationParams(conversation_id=self.conversation_id) + + @property + def prompt_len(self) -> int: + return len(self.tokens) + + @property + def is_dummy(self) -> bool: + return self.is_dummy_request + + @property + def prepopulated_prompt_len(self) -> int: + if self.prepopulated_prompt is None: + return 0 + return self.prepopulated_prompt[0] + + def get_tokens(self, beam_id: int = DEFAULT_BEAM_INDEX) -> list[int]: + assert beam_id == DEFAULT_BEAM_INDEX + return self.tokens + + def set_prepopulated_prompt_len(self, length: int, tokens_per_block: int) -> None: + self.prepopulated_prompt = (length, tokens_per_block) + + +@pytest.fixture +def manager() -> KVCacheManagerV2: + if not torch.cuda.is_available(): + pytest.skip("requires CUDA") + init_cuda_once() + manager = KVCacheManagerV2( + KvCacheConfig( + enable_block_reuse=True, + enable_partial_reuse=True, + max_gpu_total_bytes=16 << 20, + max_attention_window=[MAX_SEQ_LEN, TOKENS_PER_BLOCK], + max_util_for_resume=1.0, + block_reuse_policy="per_conversation", + ), + CacheType.SELF, + num_layers=2, + num_kv_heads=128, + head_dim=1024, + tokens_per_block=TOKENS_PER_BLOCK, + max_seq_len=MAX_SEQ_LEN, + max_batch_size=2, + mapping=Mapping(world_size=1, rank=0, tp_size=1, pp_size=1), + dtype=DataType.HALF, + vocab_size=4096, + enable_stats=False, + ) + try: + yield manager + finally: + manager.shutdown() + + +def _context_batch(*requests: _ContextRequest) -> ScheduledRequests: + batch = ScheduledRequests() + for request in requests: + batch.append_context_request(request) + return batch + + +def _prepare_context_resources( + manager: KVCacheManagerV2, + *requests: _ContextRequest, +) -> ScheduledRequests: + batch = _context_batch(*requests) + manager.prepare_resources(batch) + return batch + + +def _update_context_resources( + manager: KVCacheManagerV2, + batch: ScheduledRequests, +) -> None: + manager.update_context_resources(batch) + + +def _free_if_active( + manager: KVCacheManagerV2, + request: _ContextRequest, +) -> None: + manager.free_resources(request) + + +def _run_context( + manager: KVCacheManagerV2, + request: _ContextRequest, +) -> None: + batch = _prepare_context_resources(manager, request) + assert manager.prepare_context(request) + request.context_remaining_length = request.prompt_len - request.context_current_position + assert manager.resize_context(request, num_tokens=request.context_remaining_length) + request.context_current_position = request.prompt_len + request.context_remaining_length = 0 + _update_context_resources(manager, batch) + + +def test_per_conversation_policy_delays_commit_until_last_context_chunk( + manager: KVCacheManagerV2, +) -> None: + request = _ContextRequest(1, list(range(8)), 8, "conv-1") + + try: + batch = _prepare_context_resources(manager, request) + assert manager.prepare_context(request) + assert manager.resize_context(request, num_tokens=4) + request.context_current_position = 4 + request.context_remaining_length = 4 + _update_context_resources(manager, batch) + + kv_cache = manager.kv_cache_map[request.py_request_id] + assert kv_cache.num_committed_tokens == 0 + assert kv_cache.history_length == 4 + + request.is_first_context_chunk = False + batch = _prepare_context_resources(manager, request) + assert manager.prepare_context(request) + assert manager.resize_context(request, num_tokens=4) + request.context_current_position = 8 + request.context_remaining_length = 0 + _update_context_resources(manager, batch) + + assert kv_cache.num_committed_tokens == 8 + assert kv_cache.history_length == 8 + finally: + _free_if_active(manager, request) + + +def test_per_conversation_policy_without_params_uses_per_request_commit( + manager: KVCacheManagerV2, +) -> None: + request = _ContextRequest( + 1, + list(range(8)), + 8, + "conv-1", + use_conversation_params=False, + ) + batch = _context_batch(request) + + try: + assert manager.prepare_context(request) + assert manager.resize_context(request, num_tokens=4) + request.context_current_position = 4 + request.context_remaining_length = 4 + _update_context_resources(manager, batch) + + kv_cache = manager.kv_cache_map[request.py_request_id] + assert kv_cache.num_committed_tokens == 0 + assert kv_cache.history_length == 4 + finally: + if request.py_request_id in manager.kv_cache_map: + manager.free_resources(request) + + +def test_per_conversation_policy_releases_cancelled_request( + manager: KVCacheManagerV2, +) -> None: + request_a = _ContextRequest(1, list(range(8)), 8, "conv-1") + request_b = _ContextRequest(2, list(range(8)), 8, "conv-1") + + try: + batch_a = _prepare_context_resources(manager, request_a) + assert manager.prepare_context(request_a) + assert manager.resize_context(request_a, num_tokens=4) + request_a.context_current_position = 4 + request_a.context_remaining_length = 4 + _update_context_resources(manager, batch_a) + _free_if_active(manager, request_a) + + batch_b = _prepare_context_resources(manager, request_b) + with patch( + "tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2.logger.warning" + ) as mock_warning: + assert manager.prepare_context(request_b) + mock_warning.assert_not_called() + assert manager.resize_context(request_b, num_tokens=request_b.prompt_len) + request_b.context_current_position = request_b.prompt_len + request_b.context_remaining_length = 0 + _update_context_resources(manager, batch_b) + finally: + _free_if_active(manager, request_b) + _free_if_active(manager, request_a) + + +def test_per_conversation_policy_drops_previous_divergent_blocks( + manager: KVCacheManagerV2, +) -> None: + request_a = _ContextRequest(1, list(range(8)), 8, "conv-1") + request_b = _ContextRequest( + 2, + [*range(8), 100, 101, 102, 103], + 12, + "conv-1", + ) + request_old_prompt = _ContextRequest(3, list(range(8)), 8, "conv-2") + try: + _run_context(manager, request_a) + _free_if_active(manager, request_a) + + _run_context(manager, request_b) + assert request_b.prepopulated_prompt_len == 8 + _free_if_active(manager, request_b) + + assert manager.prepare_context(request_old_prompt) + assert request_old_prompt.prepopulated_prompt_len == 0 + finally: + _free_if_active(manager, request_old_prompt) + _free_if_active(manager, request_b) + _free_if_active(manager, request_a) + + +def test_per_conversation_policy_ignores_overlapping_request( + manager: KVCacheManagerV2, +) -> None: + request_a = _ContextRequest(1, list(range(8)), 8, "conv-1") + request_b = _ContextRequest(2, [0, 1, 2, 3, 100, 101, 102, 103], 8, "conv-1") + request_old_prompt = _ContextRequest(3, list(range(8)), 8, "conv-2") + conversation_params = request_b.py_conversation_params + + try: + batch_a = _prepare_context_resources(manager, request_a) + assert manager.prepare_context(request_a) + assert manager.resize_context(request_a, num_tokens=4) + request_a.context_current_position = 4 + request_a.context_remaining_length = 4 + _update_context_resources(manager, batch_a) + + batch_b = _prepare_context_resources(manager, request_b) + with patch( + "tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2.logger.warning" + ) as mock_warning: + assert manager.prepare_context(request_b) + mock_warning.assert_called_once_with( + "Conversation conv-1 already has current request 1. " + "Request 2 will ignore conversation params." + ) + assert request_b.py_conversation_params is conversation_params + assert manager.resize_context(request_b, num_tokens=request_b.prompt_len) + request_b.context_current_position = request_b.prompt_len + request_b.context_remaining_length = 0 + _update_context_resources(manager, batch_b) + _free_if_active(manager, request_b) + + request_a.is_first_context_chunk = False + batch_a = _prepare_context_resources(manager, request_a) + assert manager.prepare_context(request_a) + assert manager.resize_context(request_a, num_tokens=4) + request_a.context_current_position = 8 + request_a.context_remaining_length = 0 + _update_context_resources(manager, batch_a) + _free_if_active(manager, request_a) + + assert manager.prepare_context(request_old_prompt) + assert request_old_prompt.prepopulated_prompt_len == request_old_prompt.prompt_len - 1 + finally: + _free_if_active(manager, request_old_prompt) + _free_if_active(manager, request_b) + _free_if_active(manager, request_a) + + +def test_iteration_stats_reports_physical_pool_groups_without_window_metadata() -> None: + manager = object.__new__(KVCacheManagerV2) + manager.enable_stats = True + snapshot_delta = SimpleNamespace( + iter_snapshot_lookups=2, + iter_snapshot_hits=1, + iter_snapshot_misses=1, + iter_reused_tokens=32, + iter_unreused_tokens=16, + iter_aligned_snapshot_hits=1, + iter_unaligned_snapshot_hits=0, + ) + manager.impl = SimpleNamespace( + cache_tier_list=[object()], + get_and_reset_iteration_stats=lambda: {}, + get_and_reset_ssm_snapshot_iteration_stats=lambda: {3: snapshot_delta}, + ) + manager._stats_life_cycle_metadata = lambda: {3: (1, None, "ssm")} + manager._storage_pool_groups_by_window = lambda: {} + manager._get_and_reset_iteration_peak_block_stats = lambda _level: [None, None] + manager._get_storage_statistics = lambda _level: [object(), object()] + manager._build_pool_group_iteration_stats = lambda pool_group_id, *_args: pool_group_id + + stats = manager.get_iteration_stats() + + assert stats.by_pool_group == {0: 0, 1: 1} + ssm_stats = stats.by_life_cycle[3] + assert ssm_stats.kind == "ssm" + assert ssm_stats.pool_group_id == 1 + assert ssm_stats.snapshot_stats.iter_snapshot_hit_rate == 0.5 + assert ssm_stats.snapshot_stats.iter_reused_tokens == 32 + + +def test_disagg_role_mapper_kinds_default_to_indexed(): + from tensorrt_llm._torch.disaggregation.resource.page import MapperKind + from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import Role + + manager = object.__new__(KVCacheManagerV2) + + # K/V default to the TRTLLM head-major layout; the index-key side cache + # defaults to REPLICATED (every shipped index-K — DSA V1, MiniMax M3 — + # is TP-replicated). The INDEX_KEY entry is inert unless a subclass + # registers such buffers. + assert manager.get_disagg_role_mapper_kinds() == { + Role.ALL: MapperKind.INDEXED, + Role.INDEX_KEY: MapperKind.REPLICATED, + } diff --git a/tests/unittest/_torch/executor/test_kv_cache_v2_capacity_only.py b/tests/unittest/_torch/executor/test_kv_cache_v2_capacity_only.py new file mode 100644 index 000000000000..a37288a80895 --- /dev/null +++ b/tests/unittest/_torch/executor/test_kv_cache_v2_capacity_only.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +import tensorrt_llm +import tensorrt_llm.bindings +from tensorrt_llm._torch.pyexecutor import kv_cache_manager_v2 as kv_cache_v2_module +from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2 +from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, LlmRequestState, SamplingConfig + +DataType = tensorrt_llm.bindings.DataType +CacheType = tensorrt_llm.bindings.internal.batch_manager.CacheType + + +def _manager( + *, + is_draft: bool, + kv_compression_manages_history: bool = False, + kv_reserve_draft_tokens: int = 0, +) -> KVCacheManagerV2: + manager = KVCacheManagerV2.__new__(KVCacheManagerV2) + manager.is_draft = is_draft + manager.kv_compression_manages_history = kv_compression_manages_history + manager._kv_reserve_draft_tokens = kv_reserve_draft_tokens + manager.kv_cache_map = {} + return manager + + +def _request( + request_id: int, + *, + rewind: int = 0, + accepted_draft_tokens: int = 0, + complete: bool = False, +) -> SimpleNamespace: + return SimpleNamespace( + py_request_id=request_id, + py_rewind_len=rewind, + py_num_accepted_draft_tokens=accepted_draft_tokens, + max_beam_num_tokens=201, + state=LlmRequestState.GENERATION_COMPLETE + if complete + else LlmRequestState.GENERATION_IN_PROGRESS, + ) + + +def _cache(*, capacity: int = 256, active: bool = True) -> MagicMock: + cache = MagicMock() + cache.capacity = capacity + cache.is_active = active + cache.resize.return_value = True + return cache + + +@pytest.fixture(autouse=True) +def _disable_draft_token_relocation(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(kv_cache_v2_module, "_update_kv_cache_draft_token_location", MagicMock()) + + +def test_manager_initializes_capacity_only_policy_to_false() -> None: + class StopInitialization(RuntimeError): + pass + + class StopAfterPolicyConfig: + @property + def enable_swa_scratch_reuse(self): + raise StopInitialization + + manager = KVCacheManagerV2.__new__(KVCacheManagerV2) + mapping = SimpleNamespace(cp_config={}) + + with ( + patch.object(kv_cache_v2_module, "get_pp_layers", return_value=([0], 1)), + pytest.raises(StopInitialization), + ): + manager.__init__( + StopAfterPolicyConfig(), + kv_cache_v2_module.CacheTypeCpp.SELF, + num_layers=1, + num_kv_heads=1, + head_dim=128, + tokens_per_block=64, + max_seq_len=256, + max_batch_size=1, + mapping=mapping, + ) + + assert manager.kv_compression_manages_history is False + + +def test_default_generation_resize_updates_capacity_and_history() -> None: + manager = _manager(is_draft=False) + request = _request(1, rewind=3) + cache = _cache() + manager.kv_cache_map[request.py_request_id] = cache + + manager.update_resources(SimpleNamespace(generation_requests=[request])) + + cache.resize.assert_called_once_with(253, 200) + + +def test_capacity_only_is_scoped_to_target_manager() -> None: + request = _request(1, rewind=3) + batch = SimpleNamespace(generation_requests=[request]) + target = _manager(is_draft=False, kv_compression_manages_history=True) + draft = _manager(is_draft=True) + target_cache = _cache() + draft_cache = _cache() + target.kv_cache_map[request.py_request_id] = target_cache + draft.kv_cache_map[request.py_request_id] = draft_cache + + draft.update_resources(batch) + target.update_resources(batch) + + draft_cache.resize.assert_called_once_with(253, 200) + target_cache.resize.assert_called_once_with(253, None) + + +@pytest.mark.parametrize( + ("is_draft", "expected_capacity"), + [(True, 201), (False, 230)], + ids=["draft-reclaims-reserve", "target-has-no-reserve"], +) +def test_dynamic_tree_reserved_capacity(is_draft: bool, expected_capacity: int) -> None: + manager = _manager(is_draft=is_draft, kv_reserve_draft_tokens=60) + # The runtime tree used 31 draft positions: 26 rejected and 5 accepted. + request = _request(1, rewind=26, accepted_draft_tokens=5) + cache = _cache() + manager.kv_cache_map[request.py_request_id] = cache + + manager.update_resources(SimpleNamespace(generation_requests=[request])) + + cache.resize.assert_called_once_with(expected_capacity, 200) + + +def test_capacity_only_completion_preserves_history() -> None: + manager = _manager(is_draft=False, kv_compression_manages_history=True) + request = _request(1, complete=True) + cache = _cache() + manager.kv_cache_map[request.py_request_id] = cache + + manager.update_resources(SimpleNamespace(generation_requests=[request])) + + cache.resize.assert_called_once_with(None, None) + + +def test_capacity_only_skips_suspended_cache() -> None: + manager = _manager(is_draft=False, kv_compression_manages_history=True) + request = _request(1, rewind=3) + cache = _cache(active=False) + manager.kv_cache_map[request.py_request_id] = cache + + manager.update_resources(SimpleNamespace(generation_requests=[request])) + + cache.resize.assert_not_called() + + +def test_generation_update_has_no_request_compaction_marker() -> None: + manager = _manager(is_draft=False, kv_compression_manages_history=True) + request = _request(1, rewind=3) + cache = _cache() + manager.kv_cache_map[request.py_request_id] = cache + + manager.update_resources(SimpleNamespace(generation_requests=[request])) + + assert "py_kv_cache_kv_compression_manages_history" not in vars(request) + assert "py_kv_cache_compaction" not in vars(request) + + +def test_llm_request_has_no_compression_consumer_marker() -> None: + request = LlmRequest( + request_id=1, + max_new_tokens=1, + input_tokens=[1], + sampling_config=SamplingConfig(1), + is_streaming=False, + ) + + assert "py_kv_cache_kv_compression_manages_history" not in vars(request) + assert "py_kv_cache_compaction" not in vars(request) diff --git a/tests/unittest/_torch/executor/test_kv_cache_v2_extra_buffers.py b/tests/unittest/_torch/executor/test_kv_cache_v2_extra_buffers.py index d23a677dad47..073190eacf71 100644 --- a/tests/unittest/_torch/executor/test_kv_cache_v2_extra_buffers.py +++ b/tests/unittest/_torch/executor/test_kv_cache_v2_extra_buffers.py @@ -75,6 +75,15 @@ def _extra_buffers_per_layer(self, *, tokens_per_block): } +class _CoalescedIndexKeyV2(_IndexKeyOnSparseLayersV2): + """Mirror M3's pool mapping when INDEX_KEY coalesces with K/V.""" + + def _kv_pool_mapping_offset(self, layer_id, layer_group_id, key_base_addr): + del key_base_addr + layers_in_group = list(self.impl.layer_grouping[int(layer_group_id)]) + return layers_in_group.index(int(layer_id)) + + class _DuplicateRoleV2(KVCacheManagerV2): """Negative-control subclass: register Role.KEY as an "extra" so the standard buffer + extra duplicate. Must raise.""" @@ -238,11 +247,10 @@ def test_duplicate_role_against_standard_buffer_asserts(self): class TestIndexKeyBufferAccessor(unittest.TestCase): """CUDA/GPU regressions for :meth:`KVCacheManagerV2.get_index_k_buffer`. - The accessor returns a paged torch view shaped - ``[num_pages, tokens_per_block, num_heads, head_dim]`` over the - managed ``Role.INDEX_KEY`` pool, returns ``None`` for dense layers, - rejects wiring mismatches against the V2-reported page stride, and - is zero-copy (writes propagate; ``data_ptr`` stable across calls). + The accessor returns NHD or HND paged torch views over the managed + ``Role.INDEX_KEY`` pool, returns ``None`` for dense layers, rejects + wiring mismatches against the V2-reported page stride, and is zero-copy + (writes propagate; ``data_ptr`` stable across calls). """ NUM_HEADS = 1 @@ -353,6 +361,62 @@ def test_accessor_view_is_zero_copy_over_pool(self): mgr.shutdown() del mgr + def test_accessor_hnd_and_nhd_alias_coalesced_pool(self): + # Match MiniMax-M3's one-head Index-K byte size to one main K/V + # head so V2 coalesces all roles. Both layout views must retain the + # larger physical page stride while addressing the same bytes. + mgr = _CoalescedIndexKeyV2( + sparse_layer_ids=[0, 1, 2, 3], + **_make_kwargs( + num_layers=4, + num_kv_heads=1, + dtype=DataType.BF16, + ), + ) + try: + layer_idx = 3 + nhd = mgr.get_index_k_buffer( + layer_idx, + num_heads=self.NUM_HEADS, + head_dim=self.HEAD_DIM, + dtype=torch.bfloat16, + ) + hnd = mgr.get_index_k_buffer( + layer_idx, + num_heads=self.NUM_HEADS, + head_dim=self.HEAD_DIM, + dtype=torch.bfloat16, + kv_layout="HND", + ) + self.assertIsNotNone(nhd) + self.assertIsNotNone(hnd) + + converter = mgr.impl.get_page_index_converter( + mgr.layer_offsets[layer_idx], Role.INDEX_KEY + ) + self.assertGreater(int(converter.scale), 1) + self.assertEqual( + nhd.shape, + (nhd.shape[0], mgr.tokens_per_block, self.NUM_HEADS, self.HEAD_DIM), + ) + self.assertEqual( + hnd.shape, + (hnd.shape[0], self.NUM_HEADS, mgr.tokens_per_block, self.HEAD_DIM), + ) + self.assertEqual(nhd.data_ptr(), hnd.data_ptr()) + self.assertEqual(nhd.stride(0), hnd.stride(0)) + self.assertEqual(hnd.stride(2), self.HEAD_DIM) + self.assertEqual(hnd.stride(3), 1) + self.assertFalse(hnd.is_contiguous()) + + sentinel = torch.tensor(37.0, dtype=torch.bfloat16, device="cuda") + hnd[0, 0, 3, 7] = sentinel + torch.cuda.synchronize() + self.assertEqual(nhd[0, 3, 0, 7].item(), float(sentinel)) + finally: + mgr.shutdown() + del mgr + def test_accessor_pointer_matches_v2_pool_base(self): # Verify the wrapper points at the exact V2-managed pool base # for INDEX_KEY, so the view participates in the same lifecycle @@ -416,6 +480,21 @@ def test_accessor_rejects_wrong_dtype(self): mgr.shutdown() del mgr + def test_accessor_rejects_unknown_layout(self): + mgr = self._make_sparse_mgr(sparse_layer_ids=(3,)) + try: + with self.assertRaisesRegex(ValueError, "Unsupported kv_layout"): + mgr.get_index_k_buffer( + 3, + num_heads=self.NUM_HEADS, + head_dim=self.HEAD_DIM, + dtype=torch.bfloat16, + kv_layout="HDN", + ) + finally: + mgr.shutdown() + del mgr + def test_accessor_returns_none_on_base_v2_default_manager(self): # The base KVCacheManagerV2 (no extra buffers) must report None # for every local layer when queried for INDEX_KEY, because the diff --git a/tests/unittest/_torch/executor/test_kv_cache_v2_scheduler.py b/tests/unittest/_torch/executor/test_kv_cache_v2_scheduler.py index 135740961927..0a7c67e9e702 100644 --- a/tests/unittest/_torch/executor/test_kv_cache_v2_scheduler.py +++ b/tests/unittest/_torch/executor/test_kv_cache_v2_scheduler.py @@ -23,7 +23,7 @@ import pytest from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequestState -from tensorrt_llm.llmapi.llm_args import CapacitySchedulerPolicy +from tensorrt_llm.llmapi.llm_args import CapacitySchedulerPolicy, ContextChunkingPolicy # --------------------------------------------------------------------------- # State value constants @@ -75,6 +75,7 @@ def make_ctx_request( req.context_remaining_length = context_remaining_length req.prompt_len = prompt_len or context_remaining_length req.context_current_position = 0 + req.expect_snapshot_points = [] req.num_draft_tokens = num_draft_tokens req.has_draft_tokens = num_draft_tokens > 0 req.py_draft_tokens = [0] * num_draft_tokens if num_draft_tokens > 0 else [] @@ -1456,6 +1457,106 @@ def test_has_chunking_flag_set(self): # We verify indirectly: the request should be scheduled assert len(out.context_requests) == 1 + def test_force_chunk_uses_expected_snapshot_points(self): + mgr = make_kv_cache_manager(tokens_per_block=32) + sched = make_scheduler( + mgr, + max_num_tokens=8192, + ctx_chunk_config=(ContextChunkingPolicy.FORCE_CHUNK, 256), + ) + req = make_ctx_request(0, context_remaining_length=1176) + req.expect_snapshot_points = [256, 512, 768, 1024, 1176] + + out = sched.schedule_request([req], set()) + + assert ids(out.context_requests) == [0] + assert req.context_chunk_size == 256 + + def test_force_chunk_defaults_to_remaining_context_without_snapshot_points(self): + mgr = make_kv_cache_manager(tokens_per_block=32) + sched = make_scheduler( + mgr, + max_num_tokens=8192, + ctx_chunk_config=(ContextChunkingPolicy.FORCE_CHUNK, 256), + ) + req = make_ctx_request(0, context_remaining_length=1176) + + out = sched.schedule_request([req], set()) + + assert ids(out.context_requests) == [0] + assert req.context_chunk_size == 1176 + + def test_force_chunk_uses_next_expected_snapshot_point(self): + mgr = make_kv_cache_manager(tokens_per_block=32) + sched = make_scheduler( + mgr, + max_num_tokens=8192, + ctx_chunk_config=(ContextChunkingPolicy.FORCE_CHUNK, 256), + ) + req = make_ctx_request( + 0, + context_remaining_length=920, + prompt_len=1176, + is_first_context_chunk=False, + is_last_context_chunk=False, + ) + req.context_current_position = 256 + req.expect_snapshot_points = [256, 512, 768, 1024, 1176] + + out = sched.schedule_request([req], set()) + + assert ids(out.context_requests) == [0] + assert req.context_chunk_size == 256 + + def test_force_chunk_allows_prompt_end_shorter_than_unit_size(self): + mgr = make_kv_cache_manager(tokens_per_block=32) + sched = make_scheduler( + mgr, + max_num_tokens=8192, + ctx_chunk_config=(ContextChunkingPolicy.FORCE_CHUNK, 256), + ) + req = make_ctx_request(0, context_remaining_length=150) + req.expect_snapshot_points = [150] + + out = sched.schedule_request([req], set()) + + assert ids(out.context_requests) == [0] + assert req.context_chunk_size == 150 + + def test_force_chunk_allows_snapshot_point_shorter_than_unit_size(self): + mgr = make_kv_cache_manager(tokens_per_block=32) + sched = make_scheduler( + mgr, + max_num_tokens=8192, + ctx_chunk_config=(ContextChunkingPolicy.FORCE_CHUNK, 256), + ) + req = make_ctx_request( + 0, + context_remaining_length=1176, + is_last_context_chunk=False, + ) + req.expect_snapshot_points = [150] + + out = sched.schedule_request([req], set()) + + assert ids(out.context_requests) == [0] + assert req.context_chunk_size == 150 + + def test_force_chunk_makes_progress_before_budget_limited_snapshot(self): + mgr = make_kv_cache_manager(tokens_per_block=32) + sched = make_scheduler( + mgr, + max_num_tokens=128, + ctx_chunk_config=(ContextChunkingPolicy.FORCE_CHUNK, 32), + ) + req = make_ctx_request(0, context_remaining_length=512) + req.expect_snapshot_points = [256, 512] + + out = sched.schedule_request([req], set()) + + assert ids(out.context_requests) == [0] + assert req.context_chunk_size == 128 + def test_multiple_ctx_share_budget(self): """Two ctx requests share the budget.""" mgr = make_kv_cache_manager(tokens_per_block=64) diff --git a/tests/unittest/_torch/executor/test_kv_pool_rebalance.py b/tests/unittest/_torch/executor/test_kv_pool_rebalance.py index 59f4f3eb39ea..ed0cc5636e7b 100644 --- a/tests/unittest/_torch/executor/test_kv_pool_rebalance.py +++ b/tests/unittest/_torch/executor/test_kv_pool_rebalance.py @@ -32,7 +32,7 @@ import pytest from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor -from tensorrt_llm.runtime.kv_cache_manager_v2._exceptions import OutOfPagesError +from tensorrt_llm.runtime.kv_cache_manager_v2 import OutOfPagesError # --------------------------------------------------------------------------- # # Helpers diff --git a/tests/unittest/_torch/executor/test_kvv2_host_tier_sizing.py b/tests/unittest/_torch/executor/test_kvv2_host_tier_sizing.py new file mode 100644 index 000000000000..f4c3c237592d --- /dev/null +++ b/tests/unittest/_torch/executor/test_kvv2_host_tier_sizing.py @@ -0,0 +1,93 @@ +"""Tests for KVCacheManagerV2 rank-aware auto host tier sizing. + +The auto-provisioned host tier is computed per rank but drawn from a +node-level memory budget, so it must be divided by the number of ranks +co-located on the same physical node to avoid host OOM. +""" + +import pytest + +from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import _compute_auto_host_tier_quota + +GiB = 1 << 30 + + +class TestComputeAutoHostTierQuota: + def test_single_rank_uses_device_quota_when_memory_is_ample(self): + # 1 rank, 440 GiB available: cap = 220 GiB > quota -> quota wins. + assert ( + _compute_auto_host_tier_quota( + quota=173 * GiB, + local_ranks=1, + mem_available=float(440 * GiB), + memlock_limit=float("inf"), + ) + == 173 * GiB + ) + + def test_colocated_ranks_divide_node_memory_budget(self): + # 4 co-located ranks, 440 GiB available: each gets 440/4*0.5 = 55 GiB + # instead of the full device quota (4 x 173 GiB would exceed node RAM). + assert _compute_auto_host_tier_quota( + quota=173 * GiB, + local_ranks=4, + mem_available=float(440 * GiB), + memlock_limit=float("inf"), + ) == int(440 * GiB / 4 * 0.5) + + def test_aggregate_across_ranks_stays_within_available_memory(self): + local_ranks = 4 + mem_available = float(440 * GiB) + per_rank = _compute_auto_host_tier_quota( + quota=173 * GiB, + local_ranks=local_ranks, + mem_available=mem_available, + memlock_limit=float("inf"), + ) + assert per_rank * local_ranks <= mem_available + + def test_memlock_limit_caps_quota(self): + assert _compute_auto_host_tier_quota( + quota=173 * GiB, + local_ranks=1, + mem_available=float("inf"), + memlock_limit=float(10 * GiB), + ) == int(10 * GiB * 0.8) + + def test_unknown_limits_fall_back_to_device_quota(self): + assert ( + _compute_auto_host_tier_quota( + quota=173 * GiB, + local_ranks=8, + mem_available=float("inf"), + memlock_limit=float("inf"), + ) + == 173 * GiB + ) + + @pytest.mark.parametrize("memlock_limit", [0.0, float(1)]) + def test_non_positive_result_falls_back_to_device_quota(self, memlock_limit): + # RLIMIT_MEMLOCK of 0 (common in restricted containers) would yield a + # zero quota; a zero host tier would deadlock the MAX_UTILIZATION + # scheduler's suspend/resume path, so fall back to the device quota. + assert ( + _compute_auto_host_tier_quota( + quota=173 * GiB, + local_ranks=4, + mem_available=float(440 * GiB), + memlock_limit=memlock_limit, + ) + == 173 * GiB + ) + + def test_result_is_always_positive(self): + # Exhausted node memory reading must not produce a non-positive tier. + assert ( + _compute_auto_host_tier_quota( + quota=173 * GiB, + local_ranks=4, + mem_available=0.0, + memlock_limit=float("inf"), + ) + > 0 + ) diff --git a/tests/unittest/_torch/executor/test_mamba_cache_manager.py b/tests/unittest/_torch/executor/test_mamba_cache_manager.py index 0e9e56feaf52..dda9c28b93c9 100644 --- a/tests/unittest/_torch/executor/test_mamba_cache_manager.py +++ b/tests/unittest/_torch/executor/test_mamba_cache_manager.py @@ -1,32 +1,618 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Regression tests for MambaCacheManager padding-slot behavior and -CppMambaHybridCacheManager PP-sharding edge cases.""" +"""Regression tests for Python, Cpp, and V2 Mamba cache managers.""" import os from types import SimpleNamespace +from unittest.mock import MagicMock import pytest import torch +from tensorrt_llm._torch.disaggregation.resource.kv_extractor import build_page_table_from_manager +from tensorrt_llm._torch.disaggregation.resource.page import AttentionLayerGroup, MambaLayerGroup +from tensorrt_llm._torch.disaggregation.transceiver import KvCacheTransceiverV2 +from tensorrt_llm._torch.modules.mamba.mamba2_metadata import Mamba2Metadata +from tensorrt_llm._torch.pyexecutor._util import ( + KvCacheCreator, + _create_kv_cache_manager, + get_kv_cache_manager_cls, +) +from tensorrt_llm._torch.pyexecutor.config_utils import ( + MambaKVCacheParams, + extract_mamba_kv_cache_params, +) from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import CUDA_GRAPH_DUMMY_REQUEST_ID -from tensorrt_llm._torch.pyexecutor.llm_request import ATTENTION_DP_DUMMY_REQUEST_ID +from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import BlockReusePolicy, KVCacheManagerV2 +from tensorrt_llm._torch.pyexecutor.llm_request import ( + ATTENTION_DP_DUMMY_REQUEST_ID, + LlmRequest, + SamplingConfig, +) from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import ( MIN_REPLAY_HISTORY_SIZE, CppMambaHybridCacheManager, + MambaHybridCacheManagerV2, + MambaRole, + MixedMambaHybridCacheManager, PythonMambaCacheManager, + ReplayStateUpdateMetadata, + _advance_replay_state, + _get_local_mamba_cache_layout, _get_mamba_hybrid_pool_size, + _get_num_cuda_graph_padding_dummy_slots, + _mamba_snapshot_rule_counts, +) +from tensorrt_llm._torch.pyexecutor.resource_manager import ( + CacheTypeCpp, + DataType, + KVCacheManager, + get_pp_layers, ) -from tensorrt_llm._torch.pyexecutor.resource_manager import CacheTypeCpp, DataType from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._utils import torch_dtype_to_binding from tensorrt_llm.bindings.internal.batch_manager import LinearCacheType -from tensorrt_llm.llmapi.llm_args import KvCacheConfig, MTPDecodingConfig +from tensorrt_llm.llmapi.llm_args import ( + CacheTransceiverConfig, + KvCacheConfig, + MambaStateConfig, + MTPDecodingConfig, + TorchLlmArgs, +) +from tensorrt_llm.llmapi.llm_utils import ( + _resolve_kv_cache_manager_v2_auto, + _resolve_transceiver_runtime_auto, + apply_model_defaults_to_llm_args, +) from tensorrt_llm.mapping import Mapping +from tensorrt_llm.runtime.kv_cache_manager_v2 import ( + AttentionLayerConfig, + BatchDesc, + BufferConfig, + GpuCacheTierConfig, + KVCacheDesc, + KVCacheManagerConfig, + LayerId, + SsmLayerConfig, + _introspection, +) +from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheManager as RuntimeKVCacheManager skip_no_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_advance_replay_state_uses_checkpoint_predicate_and_skips_dummies(): + metadata = ReplayStateUpdateMetadata( + prev_num_accepted_tokens=torch.tensor([11, 12, 13], dtype=torch.int32), + cache_buf_idx=torch.tensor([0, 1, 1], dtype=torch.int32), + replay_step_width=5, + replay_history_size=16, + ) + + _advance_replay_state( + metadata, + state_indices=torch.tensor([0, 1, 2], dtype=torch.int32), + accepted_tokens=torch.tensor([2, 3, 4], dtype=torch.int32), + is_dummy_request=torch.tensor([False, False, True]), + ) + + # Equality does not write a checkpoint; overflow does. Dummy slots do not + # advance either piece of replay bookkeeping. + assert metadata.prev_num_accepted_tokens.tolist() == [13, 3, 13] + assert metadata.cache_buf_idx.tolist() == [0, 0, 1] + + +def test_cuda_graph_padding_dummy_slot_count_tracks_reachable_draft_lengths(): + assert _get_num_cuda_graph_padding_dummy_slots(None, 64) == 1 + + static = MTPDecodingConfig(max_draft_len=4) + assert _get_num_cuda_graph_padding_dummy_slots(static, 64) == 1 + + gated = MTPDecodingConfig( + max_draft_len=4, + acceptance_rate_window_size=8, + acceptance_rate_threshold=0.5, + ) + assert _get_num_cuda_graph_padding_dummy_slots(gated, 64) == 2 + + dynamic = MTPDecodingConfig( + max_draft_len=4, + draft_len_schedule={4: 4, 8: 2, 32: 1}, + ) + assert _get_num_cuda_graph_padding_dummy_slots(dynamic, 5) == 2 + assert _get_num_cuda_graph_padding_dummy_slots(dynamic, 32) == 3 + assert _get_num_cuda_graph_padding_dummy_slots(dynamic, 64) == 4 + + repeated = MTPDecodingConfig( + max_draft_len=4, + draft_len_schedule={4: 4, 8: 4, 32: 2}, + ) + assert _get_num_cuda_graph_padding_dummy_slots(repeated, 64) == 3 + + +def _hybrid_model_config(): + config = SimpleNamespace( + architectures=["Qwen3_5MoeForCausalLM"], + num_hidden_layers=2, + layer_types=["linear_attention", "full_attention"], + ) + return SimpleNamespace( + pretrained_config=config, + sparse_attention_config=None, + get_num_mamba_layers=lambda: 1, + ) + + +def _hybrid_cache_sizing_model_config(layer_types): + config = SimpleNamespace( + architectures=["Qwen3_5ForCausalLM"], + num_hidden_layers=len(layer_types), + layer_types=layer_types, + linear_key_head_dim=8, + linear_conv_kernel_dim=4, + linear_num_value_heads=4, + linear_num_key_heads=1, + linear_value_head_dim=8, + num_key_value_heads=2, + num_attention_heads=4, + hidden_size=32, + torch_dtype=torch.float16, + ) + return SimpleNamespace(pretrained_config=config, quant_config=None) + + +def test_mamba_kv_cache_params_separate_target_and_draft_masks(): + model_config = _hybrid_cache_sizing_model_config( + [ + "linear_attention", + "full_attention", + "linear_attention", + "full_attention", + ] + ) + params = extract_mamba_kv_cache_params( + model_config.pretrained_config, + spec_config=MTPDecodingConfig(max_draft_len=1), + ) + + assert params.mamba_layer_mask == [True, False, True, False] + assert params.target_full_attention_layer_mask == [False, True, False, True] + assert params.num_draft_layers == 1 + + assert params.get_layer_masks() == ( + [True, False, True, False, False], + [False, True, False, True, True], + ) + assert params.get_layer_masks(use_separate_draft_kv_cache=True) == ( + [True, False, True, False], + [False, True, False, True], + ) + assert params.get_layer_masks(is_draft=True) == ( + [False, False, False, False, False], + [False, False, False, False, True], + ) + + +@pytest.mark.parametrize( + ("use_v2", "enable_block_reuse", "expected"), + [ + (True, False, MambaHybridCacheManagerV2), + (True, True, MambaHybridCacheManagerV2), + (False, True, CppMambaHybridCacheManager), + ("auto", True, CppMambaHybridCacheManager), + ], +) +def test_hybrid_cache_manager_factory_honors_v2_setting( + monkeypatch, use_v2, enable_block_reuse, expected +): + monkeypatch.delenv("TRTLLM_USE_PY_MAMBA", raising=False) + monkeypatch.delenv("TLLM_MAMBA_MANAGER_PREFERENCE", raising=False) + kv_cache_config = KvCacheConfig( + enable_block_reuse=enable_block_reuse, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), + use_kv_cache_manager_v2=use_v2, + ) + + assert get_kv_cache_manager_cls(_hybrid_model_config(), kv_cache_config) is expected + + +def test_qwen3_gdn_replay_falls_back_for_v2_manager(monkeypatch): + """GDN's all-layer replay commit requires the contiguous C++ state view.""" + captured_cpp = {} + captured_v2 = {} + + class RecordingCppManager(CppMambaHybridCacheManager): + def __init__(self, *args, **kwargs): + captured_cpp.update(kwargs) + + class RecordingV2Manager(MambaHybridCacheManagerV2): + def __init__(self, *args, **kwargs): + captured_v2.update(kwargs) + + pretrained_config = SimpleNamespace( + architectures=["Qwen3_5MoeForCausalLM"], + hidden_size=32, + num_attention_heads=4, + num_key_value_heads=2, + num_hidden_layers=2, + layer_types=["linear_attention", "full_attention"], + ) + model_config = SimpleNamespace( + pretrained_config=pretrained_config, + quant_config=None, + ) + mamba_params = MambaKVCacheParams( + state_size=8, + conv_kernel=4, + num_heads=4, + n_groups=1, + head_dim=8, + mamba_layer_mask=[True, False], + target_full_attention_layer_mask=[False, True], + num_mamba_layers=1, + num_draft_layers=1, + dtype=torch.bfloat16, + mamba_ssm_cache_dtype=torch.bfloat16, + ) + monkeypatch.setenv("TRTLLM_USE_GDN_REPLAY", "1") + monkeypatch.setattr("tensorrt_llm._torch.pyexecutor._util.get_sm_version", lambda: 90) + monkeypatch.setattr( + "tensorrt_llm._torch.pyexecutor._util.extract_mamba_kv_cache_params", + lambda *args, **kwargs: mamba_params, + ) + + common_kwargs = dict( + model_engine=None, + mapping=Mapping(world_size=1, tp_size=1, pp_size=1), + tokens_per_block=32, + max_seq_len=2048, + # Cover the batch size at which upstream selects partitioned replay. + max_batch_size=16, + spec_config=MTPDecodingConfig(max_draft_len=3), + sparse_attention_config=None, + max_num_tokens=256, + max_beam_width=1, + kv_connector_manager=None, + model_config=model_config, + dtype=torch.bfloat16, + is_draft=False, + ) + _create_kv_cache_manager( + kv_cache_manager_cls=RecordingCppManager, + kv_cache_config=KvCacheConfig(use_kv_cache_manager_v2=False), + **common_kwargs, + ) + _create_kv_cache_manager( + kv_cache_manager_cls=RecordingV2Manager, + kv_cache_config=KvCacheConfig(use_kv_cache_manager_v2=True), + **common_kwargs, + ) + + assert captured_cpp["use_replay_state_update"] is True + assert captured_cpp["model_type"] == "qwen3_next" + assert captured_v2["use_replay_state_update"] is False + assert "model_type" not in captured_v2 + assert captured_v2["conv_state_layout"] == "q_k_v" + + +def test_hybrid_cache_manager_factory_rejects_cpp_preference_with_explicit_v2( + monkeypatch, +): + monkeypatch.delenv("TRTLLM_USE_PY_MAMBA", raising=False) + monkeypatch.setenv("TLLM_MAMBA_MANAGER_PREFERENCE", "CPP") + + with pytest.raises(ValueError, match="conflicts with explicit"): + get_kv_cache_manager_cls( + _hybrid_model_config(), + KvCacheConfig( + enable_block_reuse=True, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), + use_kv_cache_manager_v2=True, + ), + ) + + +def test_hybrid_cache_manager_factory_v2_preference_does_not_select_v2( + monkeypatch, +): + monkeypatch.delenv("TRTLLM_USE_PY_MAMBA", raising=False) + monkeypatch.setenv("TLLM_MAMBA_MANAGER_PREFERENCE", "V2") + + assert ( + get_kv_cache_manager_cls( + _hybrid_model_config(), + KvCacheConfig( + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), + use_kv_cache_manager_v2=False, + ), + ) + is CppMambaHybridCacheManager + ) + + +@pytest.mark.parametrize( + ("field", "offsets"), + [ + ("additional_snapshot_offsets_from_start", [128]), + ("additional_snapshot_offsets_from_end", [0]), + ], +) +def test_hybrid_cache_manager_factory_requires_v2_for_explicit_snapshots( + monkeypatch, + field, + offsets, +): + monkeypatch.delenv("TRTLLM_USE_PY_MAMBA", raising=False) + monkeypatch.delenv("TLLM_MAMBA_MANAGER_PREFERENCE", raising=False) + + kv_cache_config = KvCacheConfig( + enable_block_reuse=False, + mamba_state_config=MambaStateConfig(**{field: offsets}), + use_kv_cache_manager_v2="auto", + ) + llm_args = TorchLlmArgs( + model="/tmp/dummy_model", + kv_cache_config=kv_cache_config, + ) + + assert _resolve_kv_cache_manager_v2_auto(llm_args, {}) is False + assert llm_args.kv_cache_config.use_kv_cache_manager_v2 is False + with pytest.raises(ValueError, match="use_kv_cache_manager_v2=True"): + get_kv_cache_manager_cls( + _hybrid_model_config(), + llm_args.kv_cache_config, + ) + + +@pytest.mark.parametrize("backend", ["NIXL", "DEFAULT"]) +def test_hybrid_cache_manager_factory_routes_explicit_v2_disagg(monkeypatch, backend): + monkeypatch.delenv("TRTLLM_USE_PY_MAMBA", raising=False) + monkeypatch.delenv("TLLM_MAMBA_MANAGER_PREFERENCE", raising=False) + for env_var in ( + "TRTLLM_USE_NIXL_KVCACHE", + "TRTLLM_USE_UCX_KVCACHE", + "TRTLLM_USE_MOONCAKE_KVCACHE", + "TRTLLM_USE_MPI_KVCACHE", + ): + monkeypatch.delenv(env_var, raising=False) + assert ( + get_kv_cache_manager_cls( + _hybrid_model_config(), + KvCacheConfig( + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), + use_kv_cache_manager_v2=True, + ), + is_disagg=True, + cache_transceiver_config=CacheTransceiverConfig( + backend=backend, transceiver_runtime="PYTHON" + ), + ) + is MambaHybridCacheManagerV2 + ) + + +def test_hybrid_cache_manager_factory_rejects_python_v1_disagg_reuse(monkeypatch): + monkeypatch.delenv("TRTLLM_USE_PY_MAMBA", raising=False) + monkeypatch.delenv("TLLM_MAMBA_MANAGER_PREFERENCE", raising=False) + + with pytest.raises(ValueError, match="requires use_kv_cache_manager_v2=True"): + get_kv_cache_manager_cls( + _hybrid_model_config(), + KvCacheConfig( + enable_block_reuse=True, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), + use_kv_cache_manager_v2=False, + ), + is_disagg=True, + cache_transceiver_config=CacheTransceiverConfig( + backend="NIXL", + transceiver_runtime="PYTHON", + ), + ) + + +@pytest.mark.parametrize( + ("backend", "runtime", "backend_env"), + [ + ("DEFAULT", "PYTHON", "TRTLLM_USE_UCX_KVCACHE"), + ("UCX", "PYTHON", None), + ("NIXL", "auto", None), + ("NIXL", None, None), + ("NIXL", "CPP", None), + ("UCX", None, None), + ], +) +def test_hybrid_cache_manager_factory_rejects_unsupported_v2_disagg_route( + monkeypatch, backend, runtime, backend_env +): + monkeypatch.delenv("TRTLLM_USE_PY_MAMBA", raising=False) + monkeypatch.delenv("TLLM_MAMBA_MANAGER_PREFERENCE", raising=False) + for env_var in ( + "TRTLLM_USE_NIXL_KVCACHE", + "TRTLLM_USE_UCX_KVCACHE", + "TRTLLM_USE_MOONCAKE_KVCACHE", + "TRTLLM_USE_MPI_KVCACHE", + ): + monkeypatch.delenv(env_var, raising=False) + if backend_env is not None: + monkeypatch.setenv(backend_env, "1") + + with pytest.raises(ValueError, match="requires transceiver_runtime='PYTHON'"): + get_kv_cache_manager_cls( + _hybrid_model_config(), + KvCacheConfig( + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), + use_kv_cache_manager_v2=True, + ), + is_disagg=True, + cache_transceiver_config=CacheTransceiverConfig( + backend=backend, transceiver_runtime=runtime + ), + ) + + +@pytest.mark.parametrize( + ("env_name", "env_value", "expected_error"), + [ + ( + "TLLM_MAMBA_MANAGER_PREFERENCE", + "MIXED", + "does not support block reuse", + ), + ( + "TRTLLM_USE_PY_MAMBA", + "1", + "does not support block reuse", + ), + ], +) +def test_hybrid_cache_manager_factory_rejects_mixed_override_with_reuse( + monkeypatch, env_name, env_value, expected_error +): + monkeypatch.delenv("TRTLLM_USE_PY_MAMBA", raising=False) + monkeypatch.delenv("TLLM_MAMBA_MANAGER_PREFERENCE", raising=False) + monkeypatch.setenv(env_name, env_value) + + with pytest.raises(ValueError, match=expected_error): + get_kv_cache_manager_cls( + _hybrid_model_config(), + KvCacheConfig( + enable_block_reuse=True, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), + ), + ) + + +@pytest.mark.parametrize("use_v2", [False, "auto"]) +def test_hybrid_cache_manager_factory_keeps_v1_disagg_route(monkeypatch, use_v2): + monkeypatch.delenv("TRTLLM_USE_PY_MAMBA", raising=False) + monkeypatch.delenv("TLLM_MAMBA_MANAGER_PREFERENCE", raising=False) + + assert ( + get_kv_cache_manager_cls( + _hybrid_model_config(), + KvCacheConfig( + enable_block_reuse=False, + use_kv_cache_manager_v2=use_v2, + ), + is_disagg=True, + ) + is CppMambaHybridCacheManager + ) + assert ( + get_kv_cache_manager_cls( + _hybrid_model_config(), + KvCacheConfig( + enable_block_reuse=False, + use_kv_cache_manager_v2=use_v2, + ), + is_disagg=True, + cache_transceiver_config=CacheTransceiverConfig( + backend="NIXL", transceiver_runtime="PYTHON" + ), + ) + is MixedMambaHybridCacheManager + ) + + +def test_hybrid_models_default_to_v2_and_python_transceiver(monkeypatch): + from tensorrt_llm._torch.models.modeling_nemotron_h import NemotronHForCausalLM + from tensorrt_llm._torch.models.modeling_qwen3_5 import Qwen3_5VLModel + from tensorrt_llm._torch.models.modeling_qwen3_next import Qwen3NextForCausalLM + + for env_var in ( + "TRTLLM_USE_NIXL_KVCACHE", + "TRTLLM_USE_UCX_KVCACHE", + "TRTLLM_USE_MOONCAKE_KVCACHE", + "TRTLLM_USE_MPI_KVCACHE", + ): + monkeypatch.delenv(env_var, raising=False) + + for model_cls in (NemotronHForCausalLM, Qwen3NextForCausalLM, Qwen3_5VLModel): + llm_args = TorchLlmArgs( + model="/tmp/dummy_model", + cache_transceiver_config=CacheTransceiverConfig(backend="DEFAULT"), + ) + model_defaults = model_cls.get_model_defaults(llm_args) + apply_model_defaults_to_llm_args(llm_args, model_defaults) + _resolve_transceiver_runtime_auto(llm_args, model_cls) + _resolve_kv_cache_manager_v2_auto(llm_args, model_defaults, original_setting="auto") + assert llm_args.kv_cache_config.use_kv_cache_manager_v2 is True + assert llm_args.kv_cache_config.enable_block_reuse is False + assert llm_args.cache_transceiver_config.transceiver_runtime == "PYTHON" + + +def test_v2_disagg_slice_skips_state_index_on_mamba_free_pp_rank(): + manager = object.__new__(MambaHybridCacheManagerV2) + manager.local_num_mamba_layers = 0 + transceiver = object.__new__(KvCacheTransceiverV2) + transceiver._kv_cache_manager = manager + transceiver._reuse_adapter = SimpleNamespace(tokens_per_block=32) + transceiver._page_table = SimpleNamespace(layer_groups=[]) + request = SimpleNamespace( + is_generation_only_request=lambda: False, + prompt_len=0, + py_request_id=123, + ) + + kv_slice = transceiver._create_kv_slice(request) + + assert kv_slice.mamba_state_index is None + + +def test_v2_disagg_slice_reads_state_index_without_refreshing_batch_mask(): + manager = object.__new__(MambaHybridCacheManagerV2) + manager.local_num_mamba_layers = 1 + manager._request_id_to_state_index = {123: 7} + manager.get_state_indices = MagicMock( + side_effect=AssertionError("state-index lookup must not refresh the dummy mask") + ) + transceiver = object.__new__(KvCacheTransceiverV2) + transceiver._kv_cache_manager = manager + transceiver._reuse_adapter = SimpleNamespace(tokens_per_block=32) + transceiver._page_table = SimpleNamespace(layer_groups=[]) + request = SimpleNamespace( + is_generation_only_request=lambda: False, + prompt_len=0, + py_request_id=123, + ) + + kv_slice = transceiver._create_kv_slice(request) + + assert kv_slice.mamba_state_index == 7 + manager.get_state_indices.assert_not_called() + + +@pytest.mark.parametrize( + "max_beam_width, has_connector, expected", + [ + (2, False, "max_beam_width > 1"), + (1, True, "kv_connector_manager"), + (2, True, "kv_connector_manager, max_beam_width > 1"), + ], +) +def test_v2_hybrid_incompatibility_fails_without_cpp_fallback( + max_beam_width, has_connector, expected +): + config = SimpleNamespace( + architectures=["Qwen3_5MoeForCausalLM"], + num_hidden_layers=2, + layer_types=["linear_attention", "full_attention"], + ) + model_config = SimpleNamespace( + pretrained_config=config, + sparse_attention_config=None, + ) + creator = object.__new__(KvCacheCreator) + creator._kv_connector_manager = object() if has_connector else None + creator._max_beam_width = max_beam_width + + with pytest.raises(NotImplementedError, match=expected): + creator._fallback_if_unsupported_kv_cache_manager_v2( + MambaHybridCacheManagerV2, model_config, KvCacheConfig() + ) + + def _make_mgr( max_batch_size=4, max_draft_len=2, enable_attention_dp=False, use_replay_state_update=False ): @@ -342,16 +928,689 @@ def test_non_mtp_pytorch_prepare_and_get_state_indices_flow(): ] +def test_v2_hybrid_prepare_expect_snapshot_points(): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.enable_block_reuse = True + mgr.kv_cache_config = KvCacheConfig( + enable_block_reuse=True, + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=64, + additional_snapshot_offsets_from_start=[128, 999], + additional_snapshot_offsets_from_end=[0, 22, 999], + ), + ) + requests = [ + SimpleNamespace(prompt_len=150, expect_snapshot_points=[999]), + SimpleNamespace(prompt_len=128, expect_snapshot_points=[]), + SimpleNamespace(prompt_len=32, expect_snapshot_points=[]), + ] + + mgr.prepare_expect_snapshot_points(requests) + + assert [request.expect_snapshot_points for request in requests] == [ + [64, 128, 150], + [64, 106, 128], + [10, 32], + ] + + +def test_v2_hybrid_prepare_expect_snapshot_points_without_periodic_snapshots(): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.enable_block_reuse = True + mgr.kv_cache_config = KvCacheConfig( + enable_block_reuse=True, + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=0, + additional_snapshot_offsets_from_start=[128], + additional_snapshot_offsets_from_end=[0, 13], + ), + ) + request = SimpleNamespace(prompt_len=150, expect_snapshot_points=[]) + + mgr.prepare_expect_snapshot_points([request]) + + assert request.expect_snapshot_points == [128, 137, 150] + + +def test_mamba_snapshot_rule_count_deduplicates_and_filters_unreachable_points(): + config = KvCacheConfig( + enable_block_reuse=True, + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=0, + additional_snapshot_offsets_from_start=[64, 65, 64], + additional_snapshot_offsets_from_end=[0, 32, 4096], + ), + ) + + assert _mamba_snapshot_rule_counts(config, 128, 32) == (4, 3) + + +def test_v2_hybrid_snapshot_sizing_scales_with_pp_and_explicit_rules(): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.max_batch_size = 4 + mgr.mapping = Mapping(world_size=2, rank=0, tp_size=1, pp_size=2) + mgr.max_seq_len = 128 + mgr.tokens_per_block = 32 + mgr._num_reserved_dummy_slots = 0 + config = KvCacheConfig( + enable_block_reuse=True, + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=0, + additional_snapshot_offsets_from_start=[64], + additional_snapshot_offsets_from_end=[0, 32], + ), + ) + + assert mgr._num_ssm_snapshots_for_capacity(512, config) == 24 + assert mgr._num_ssm_states_per_typical_request(128, config) == 4 + assert [desc.capacity for desc in mgr._typical_request_descs(128, config)] == [ + 32, + 32, + 32, + 32, + ] + + periodic_config = KvCacheConfig( + enable_block_reuse=True, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=48), + ) + assert mgr._num_ssm_states_per_typical_request(47, periodic_config) == 1 + assert mgr._num_ssm_states_per_typical_request(48, periodic_config) == 1 + + +def test_cpp_mamba_estimator_handles_disabled_snapshots_without_attention(): + manager = object.__new__(KVCacheManager) + manager._primary_pool_memory_bytes = 4096 + manager._secondary_pool_memory_bytes = 0 + manager.linear_attention_metadata = SimpleNamespace( + all_recurrent_states_bytes=64, + states_snapshot_interval=0, + ) + manager.max_attention_window_vec = [LinearCacheType.RECURRENT_STATES.value] + manager.get_cache_bytes_per_token = lambda: 0 + manager.mapping = SimpleNamespace(pp_size=1) + manager.max_batch_size = 4 + manager.max_seq_len = 128 + manager.tokens_per_block = 32 + manager.spec_config = None + + blocks = manager._calculate_max_num_blocks_for_linear_attention( + KvCacheConfig( + max_tokens=512, + enable_block_reuse=False, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=0), + ) + ) + + assert blocks[128] == (16, 0) + assert blocks[LinearCacheType.RECURRENT_STATES.value] == (5, 0) + + +def test_hybrid_mtp_layout_honors_explicit_base_partition(): + model_config = _hybrid_cache_sizing_model_config( + [ + "linear_attention", + "full_attention", + "linear_attention", + "full_attention", + ] + ) + spec_config = MTPDecodingConfig(max_draft_len=1) + expected_pp_layers = ([0, 1], [2, 3, 4]) + + for rank in range(2): + mapping = Mapping( + world_size=2, + rank=rank, + tp_size=1, + pp_size=2, + pp_partition=[2, 2], + ) + pp_layers, total_layers = get_pp_layers( + 5, + mapping, + spec_config=spec_config, + layer_mask=[True] * 5, + ) + assert total_layers == 5 + assert pp_layers == expected_pp_layers[rank] + + _, local_mamba_layers, local_attention_layers = _get_local_mamba_cache_layout( + model_config, + mapping, + spec_config=spec_config, + ) + assert local_mamba_layers == 1 + assert local_attention_layers == rank + 1 + + for manager_cls in ( + CppMambaHybridCacheManager, + MambaHybridCacheManagerV2, + ): + cache_cost = manager_cls.get_cache_size_per_token( + model_config, + mapping, + max_batch_size=1, + kv_cache_config=KvCacheConfig(enable_block_reuse=False), + spec_config=spec_config, + ) + assert cache_cost == (64 * (rank + 1), 2400) + + +def test_hybrid_separate_mtp_draft_estimator_has_no_mamba_state(): + model_config = _hybrid_cache_sizing_model_config( + [ + "linear_attention", + "full_attention", + "linear_attention", + "full_attention", + ] + ) + spec_config = MTPDecodingConfig(max_draft_len=1) + + for rank in range(2): + mapping = Mapping(world_size=2, rank=rank, tp_size=1, pp_size=2) + + target_cost = MambaHybridCacheManagerV2.get_cache_size_per_token( + model_config, + mapping, + max_batch_size=1, + kv_cache_config=KvCacheConfig(enable_block_reuse=False), + spec_config=spec_config, + use_separate_draft_kv_cache=True, + ) + assert target_cost == (64, 2400) + + _, local_mamba_layers, local_attention_layers = _get_local_mamba_cache_layout( + model_config, + mapping, + spec_config=spec_config, + is_draft=True, + ) + assert local_mamba_layers == 0 + assert local_attention_layers == rank + + draft_cost = MambaHybridCacheManagerV2.get_cache_size_per_token( + model_config, + mapping, + max_batch_size=1, + kv_cache_config=KvCacheConfig(enable_block_reuse=False), + num_layers=1, + spec_config=spec_config, + is_draft=True, + ) + assert draft_cost == (64 * rank, 0) + + +@pytest.mark.parametrize( + ("spec_config", "enable_attention_dp", "expected_intercept"), + [ + (None, False, 320), + (MTPDecodingConfig(max_draft_len=4), True, 384), + ( + MTPDecodingConfig( + max_draft_len=4, + draft_len_schedule={1: 4, 2: 2, 3: 1}, + ), + True, + 576, + ), + ], +) +def test_v2_hybrid_estimator_counts_dummy_states_without_attention_capacity( + monkeypatch, spec_config, enable_attention_dp, expected_intercept +): + monkeypatch.setattr( + KVCacheManager, + "get_cache_size_per_token", + lambda *args, **kwargs: 11, + ) + monkeypatch.setattr( + "tensorrt_llm._torch.pyexecutor.mamba_cache_manager._get_local_mamba_cache_layout", + lambda *args, **kwargs: ( + SimpleNamespace(get_states_bytes_per_layer=lambda mapping: 64), + 1, + 1, + ), + ) + mapping = Mapping( + world_size=1, + tp_size=1, + pp_size=1, + enable_attention_dp=enable_attention_dp, + ) + + assert MambaHybridCacheManagerV2.get_cache_size_per_token( + object(), + mapping, + max_batch_size=4, + kv_cache_config=KvCacheConfig(), + spec_config=spec_config, + ) == (11, expected_intercept) + + +def test_v2_hybrid_attention_bound_is_snapshot_alignment_agnostic(): + model_config = _hybrid_cache_sizing_model_config(["linear_attention", "full_attention"]) + mapping = Mapping(world_size=1, rank=0, tp_size=1, pp_size=1) + + def estimate(interval): + return MambaHybridCacheManagerV2.get_cache_size_per_token( + model_config, + mapping, + max_batch_size=2, + kv_cache_config=KvCacheConfig( + enable_block_reuse=True, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=interval), + ), + tokens_per_block=32, + max_seq_len=128, + ) + + aligned = estimate(32) + unaligned = estimate(48) + + assert aligned[1] == unaligned[1] + + +def _base_attention_layer_configs(num_layers): + return [ + AttentionLayerConfig( + layer_id=LayerId(layer_idx), + buffers=[BufferConfig(role="key", size=256)], + ) + for layer_idx in range(num_layers) + ] + + +def test_v2_hybrid_typical_batch_splits_capacity_across_ssm_states_and_dummies(): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.kv_cache_type = CacheTypeCpp.SELF + mgr.head_dim_per_layer = [64, 64] + mgr.pp_layers = [0, 1] + mgr._mamba_layer_mask = [True, False] + mgr.ssm_bytes = 64 + mgr.conv_bytes = 32 + mgr.max_attention_window_vec = [128, 128] + mgr.max_batch_size = 2 + mgr.mapping = Mapping(world_size=1, rank=0, tp_size=1, pp_size=1) + mgr.max_seq_len = 128 + mgr.max_num_tokens = 128 + mgr.tokens_per_block = 32 + mgr.num_local_layers = 2 + mgr.local_num_mamba_layers = 1 + mgr._num_reserved_dummy_slots = 1 + mgr.dtype = DataType.HALF + mgr.enable_swa_scratch_reuse = False + mgr.enable_stats = False + mgr.num_extra_kv_tokens = 0 + mgr.get_layer_bytes_per_token = lambda **kwargs: 8 + mgr._minimum_live_gpu_quota = lambda: 0 + kv_cache_config = KvCacheConfig( + enable_partial_reuse=True, + avg_seq_len=96, + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=48, + additional_snapshot_offsets_from_start=[32], + additional_snapshot_offsets_from_end=[0], + ), + ) + mgr.kv_cache_config = kv_cache_config + constraints = [BatchDesc([KVCacheDesc(capacity=64, history_length=0)])] + base_layers = _base_attention_layer_configs(2) + base_config = KVCacheManagerConfig( + tokens_per_block=32, + cache_tiers=[GpuCacheTierConfig(quota=1 << 20)], + layers=base_layers, + constraints=constraints, + ) + + config = mgr._build_cache_config(base_config) + + assert isinstance(config.layers[0], SsmLayerConfig) + assert isinstance(config.layers[1], AttentionLayerConfig) + assert int(config.layers[1].layer_id) == int(base_layers[1].layer_id) + assert config.typical_step == BatchDesc( + [KVCacheDesc(capacity=32, history_length=31)] * 6 + + [KVCacheDesc(capacity=0, history_length=0)] + ) + # The caller-provided constraint keeps its dummy-slot padding. + assert ( + BatchDesc( + [ + KVCacheDesc(capacity=64, history_length=0), + KVCacheDesc(capacity=0, history_length=0), + ] + ) + in config.constraints + ) + # An explicit SSM floor constraint is always emitted so the recurrent pool + # can hold every live + reserved-dummy state slot even when the caller + # supplies no constraints (e.g. avg_seq_len unset). It is built from + # zero-capacity requests, so it costs no attention pages. + required_ssm_slots = mgr._max_resident_sequences() + mgr._num_reserved_dummy_slots + assert any( + all(kv.capacity == 0 for kv in batch.kv_caches) + and len(batch.kv_caches) >= required_ssm_slots + for batch in config.constraints + ) + assert sum(kv.capacity for kv in config.typical_step.kv_caches) == 2 * 96 + assert not hasattr(config.typical_step.kv_caches[0], "num_ssm_slots") + + +def test_v2_hybrid_warns_when_avg_seq_len_is_missing(monkeypatch): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.max_seq_len = 4096 + warnings_seen = [] + monkeypatch.setattr( + "tensorrt_llm._torch.pyexecutor.mamba_cache_manager.logger.warning", + lambda message: warnings_seen.append(message), + ) + + capacity = mgr._get_typical_request_capacity(KvCacheConfig()) + + assert capacity == 2048 + assert len(warnings_seen) == 1 + assert "kv_cache_config.avg_seq_len" in warnings_seen[0] + assert "max_seq_len / 2=2048" in warnings_seen[0] + assert "workload's average total sequence length" in warnings_seen[0] + + +def test_v2_hybrid_rejects_quota_below_live_state_floor(): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.max_batch_size = 2 + mgr.mapping = Mapping(world_size=1, rank=0, tp_size=1, pp_size=1) + mgr.local_num_mamba_layers = 1 + mgr.ssm_bytes = 64 + mgr.conv_bytes = 32 + mgr._num_reserved_dummy_slots = 1 + mgr.tokens_per_block = 32 + mgr.num_local_layers = 2 + mgr.pp_layers = [0, 1] + mgr.max_attention_window_vec = [128, 128] + mgr.max_num_tokens = 128 + mgr.enable_swa_scratch_reuse = False + mgr.get_layer_bytes_per_token = lambda **kwargs: 8 + mgr._attention_cache_bytes_per_token = lambda: 16 + mgr.kv_cache_config = KvCacheConfig(enable_partial_reuse=False) + minimum_quota = mgr._minimum_live_gpu_quota() + + base_config = KVCacheManagerConfig( + tokens_per_block=32, + cache_tiers=[GpuCacheTierConfig(quota=minimum_quota - 1)], + layers=[], + ) + + with pytest.raises(ValueError, match="too small for live recurrent states"): + mgr._build_cache_config(base_config) + + +def test_v2_hybrid_pure_mamba_rank_does_not_reserve_attention_page(): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.max_batch_size = 2 + mgr.mapping = Mapping(world_size=1, rank=0, tp_size=1, pp_size=1) + mgr.local_num_mamba_layers = 1 + mgr.ssm_bytes = 64 + mgr.conv_bytes = 32 + mgr._num_reserved_dummy_slots = 1 + mgr.tokens_per_block = 32 + mgr.num_local_layers = 1 + mgr.pp_layers = [0] + mgr.max_attention_window_vec = [128] + mgr.max_num_tokens = 128 + mgr.enable_swa_scratch_reuse = False + mgr.get_layer_bytes_per_token = lambda **kwargs: 0 + mgr._attention_cache_bytes_per_token = lambda: 0 + mgr.kv_cache_config = KvCacheConfig(enable_block_reuse=False) + + assert mgr._minimum_live_gpu_quota() == 3 * (64 + 32) + + +def test_cpp_hybrid_prepare_expect_snapshot_points(): + mgr = object.__new__(CppMambaHybridCacheManager) + mgr.enable_block_reuse = True + mgr.kv_cache_config = KvCacheConfig( + enable_block_reuse=True, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=64), + ) + mgr.linear_attention_metadata = SimpleNamespace(states_snapshot_interval=64) + requests = [ + SimpleNamespace(prompt_len=150, expect_snapshot_points=[999]), + SimpleNamespace(prompt_len=128, expect_snapshot_points=[]), + SimpleNamespace(prompt_len=32, expect_snapshot_points=[]), + ] + + mgr.prepare_expect_snapshot_points(requests) + + assert [request.expect_snapshot_points for request in requests] == [ + [64, 128], + [64, 128], + [], + ] + + +@pytest.mark.parametrize( + ("allocated_offsets", "context_current_position", "expected_offset"), + [ + ({9: 25}, 0, 25), + ({7: 17, 9: 25}, 0, 17), + ({7: 17, 9: 25}, 256, 25), + ], +) +def test_cpp_hybrid_state_indices_skip_context_placeholders( + allocated_offsets, context_current_position, expected_offset +): + """Capacity-limited chunks use the next real snapshot/final block.""" + null_index = torch.iinfo(torch.int32).max + block_offsets = torch.full((1, 1, 2, 10), null_index, dtype=torch.int32) + for logical_index, pool_offset in allocated_offsets.items(): + block_offsets[0, 0, 0, logical_index] = pool_offset + + request = SimpleNamespace( + py_request_id=0, + prompt_len=314, + is_context_finished=False, + context_current_position=context_current_position, + context_chunk_size=32, + prepopulated_prompt_len=0, + is_dummy=False, + ) + mgr = object.__new__(CppMambaHybridCacheManager) + mgr.local_num_mamba_layers = 1 + mgr.requests = [request] + mgr.tokens_per_block = 32 + mgr.kv_cache_config = SimpleNamespace(enable_block_reuse=True) + mgr.impl = SimpleNamespace( + copy_batch_block_offsets=lambda *args: None, + get_cache_block_ids=lambda *args: [], + ) + mgr.host_block_offsets = block_offsets + mgr.recurrent_states_pool_index = 0 + mgr.blocks_per_window = {LinearCacheType.RECURRENT_STATES.value: (1264, 0)} + mgr._host_state_indices = torch.zeros(1, dtype=torch.int32) + mgr.cuda_state_indices = torch.zeros(1, dtype=torch.int32) + mgr._row_indices = torch.arange(1, dtype=torch.long) + mgr._request_id_to_state_index = {} + mgr._request_id_to_is_dummy = {} + mgr._dummy_request_mask = None + + mgr._setup_state_indices() + + assert mgr._host_state_indices.tolist() == [expected_offset] + assert mgr.cuda_state_indices.tolist() == [expected_offset] + assert mgr.get_state_indices([request.py_request_id], [False]) == [expected_offset] + + +def test_v2_block_reuse_commit_saves_ssm_snapshot_at_snapshot_point(): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.enable_block_reuse = True + mgr.is_draft = False + mgr._augment_tokens_for_block_reuse = lambda tokens, request, start, end: tokens[start:end] + mgr._mark_context_position_as_history = MagicMock() + + token_ids = list(range(150)) + request = SimpleNamespace( + prompt_len=150, + context_current_position=137, + context_remaining_length=13, + expect_snapshot_points=[137], + is_dummy_request=False, + is_dummy=False, + py_request_id=0, + get_tokens=lambda beam_idx: token_ids, + ) + kv_cache = SimpleNamespace( + num_committed_tokens=0, + commit=MagicMock(), + stop_committing=MagicMock(), + ) + + mgr.try_commit_blocks(request, kv_cache) + + kv_cache.commit.assert_called_once_with(token_ids[:137]) + kv_cache.stop_committing.assert_not_called() + mgr._mark_context_position_as_history.assert_called_once_with(request, kv_cache) + + # The remaining suffix advances request history but must not publish a + # second attention/SSM snapshot beyond the configured boundary. + kv_cache.num_committed_tokens = 137 + request.context_current_position = 150 + request.context_remaining_length = 0 + mgr.try_commit_blocks(request, kv_cache) + + kv_cache.commit.assert_called_once_with(token_ids[:137]) + kv_cache.stop_committing.assert_called_once_with() + + +def test_v2_hybrid_add_dummy_requests_forwards_encoder_output_lens(mocker): + mgr = object.__new__(MambaHybridCacheManagerV2) + base_add_dummy_requests = mocker.patch.object( + KVCacheManagerV2, "add_dummy_requests", return_value=[] + ) + + mgr.add_dummy_requests([123], encoder_output_lens=[17]) + + assert base_add_dummy_requests.call_args.kwargs["encoder_output_lens"] == [17] + + +@pytest.mark.parametrize( + "manager_cls", + [ + MambaHybridCacheManagerV2, + CppMambaHybridCacheManager, + MixedMambaHybridCacheManager, + ], + ids=["v2", "cpp", "mixed"], +) +def test_hybrid_prepare_expect_snapshot_points_clears_when_reuse_disabled(manager_cls): + mgr = object.__new__(manager_cls) + mgr.enable_block_reuse = False + mgr.kv_cache_config = KvCacheConfig(enable_block_reuse=False) + request = SimpleNamespace(prompt_len=64, expect_snapshot_points=[64]) + + mgr.prepare_expect_snapshot_points([request]) + + assert request.expect_snapshot_points == [] + + +def test_cpp_hybrid_prepare_expect_snapshot_points_clears_for_disabled_interval(): + mgr = object.__new__(CppMambaHybridCacheManager) + mgr.enable_block_reuse = True + mgr.kv_cache_config = SimpleNamespace( + enable_block_reuse=True, + mamba_state_config=SimpleNamespace(periodic_snapshot_interval=0), + ) + request = SimpleNamespace(prompt_len=150, expect_snapshot_points=[64]) + + mgr.prepare_expect_snapshot_points([request]) + + assert request.expect_snapshot_points == [] + + +def test_expect_snapshot_points_binding_round_trip(): + request = LlmRequest( + request_id=1, + max_new_tokens=1, + input_tokens=[1, 2, 3], + sampling_config=SamplingConfig(), + is_streaming=False, + ) + + assert request.expect_snapshot_points == [] + request.expect_snapshot_points = [64, 128] + assert request.expect_snapshot_points == [64, 128] + + +@skip_no_cuda +def test_v2_hybrid_pool_ratio_controls_allocated_memory(): + def allocated_memory(pool_ratio): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.kv_cache_type = CacheTypeCpp.SELF + mgr.head_dim_per_layer = [64, 64] + mgr.pp_layers = [0, 1] + mgr._mamba_layer_mask = [True, False] + mgr.ssm_bytes = 64 + mgr.conv_bytes = 32 + mgr.max_attention_window_vec = [128, 128] + mgr.max_batch_size = 2 + mgr.mapping = Mapping(world_size=1, rank=0, tp_size=1, pp_size=1) + mgr.max_seq_len = 128 + mgr.max_num_tokens = 128 + mgr.tokens_per_block = 32 + mgr.num_local_layers = 2 + mgr.local_num_mamba_layers = 1 + mgr._num_reserved_dummy_slots = 1 + mgr.dtype = DataType.HALF + mgr.enable_swa_scratch_reuse = False + mgr.enable_stats = False + mgr.num_extra_kv_tokens = 0 + mgr.get_layer_bytes_per_token = lambda **kwargs: 8 + + kv_cache_config = KvCacheConfig( + pool_ratio=pool_ratio, + enable_partial_reuse=False, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=64), + ) + mgr.kv_cache_config = kv_cache_config + base_config = KVCacheManagerConfig( + tokens_per_block=32, + cache_tiers=[GpuCacheTierConfig(quota=64 << 20)], + layers=_base_attention_layer_configs(2), + initial_pool_ratio=pool_ratio, + ) + config = mgr._build_cache_config(base_config) + runtime_manager = RuntimeKVCacheManager(config) + try: + statistics = _introspection.storage_statistics(runtime_manager) + + def _slot_sizes(stat): + # cpp binding exposes `slot_sizes`; the Python backend `slot_size`. + return stat.slot_sizes if hasattr(stat, "slot_sizes") else stat.slot_size + + allocated_bytes = [ + int(stats.total) * sum(int(size) for size in _slot_sizes(stats)) + for stats in statistics + ] + return allocated_bytes, _introspection.current_gpu_ratio(runtime_manager) + finally: + runtime_manager.shutdown() + + low_mamba_allocation, low_actual_ratio = allocated_memory([0.25, 0.75]) + high_mamba_allocation, high_actual_ratio = allocated_memory([0.75, 0.25]) + + assert low_actual_ratio == pytest.approx([0.25, 0.75]) + assert high_actual_ratio == pytest.approx([0.75, 0.25]) + assert high_mamba_allocation[0] > low_mamba_allocation[0] + assert high_mamba_allocation[1] < low_mamba_allocation[1] + + # --------------------------------------------------------------------------- -# CppMambaHybridCacheManager: recurrent-state snapshot pool sizing +# Cpp/V2 Mamba hybrid managers: recurrent-state allocation and reuse # -# Sized in KVCacheManager._calculate_max_num_blocks_for_linear_attention. -# Mirrors the MixedMambaCacheManager fix where each kind of padding sentinel -# (CUDA-graph dummy, plus one per draft length under spec decoding) must not -# evict live recurrent state. Wanli's fix made all sentinels share one slot -# in the Python manager (#13489); the C++ hybrid path instead reserves a -# dedicated slot per sentinel kind in the underlying pool — same invariant, -# different mechanism. These tests guard the pool sizing. +# The Cpp pool is sized in +# KVCacheManager._calculate_max_num_blocks_for_linear_attention. It reserves a +# dedicated slot for each padding sentinel kind so dummy requests cannot evict +# live recurrent state. The V2 tests cover unified-pool state views, slot +# bookkeeping, replay, and snapshot reuse. # --------------------------------------------------------------------------- @@ -359,12 +1618,13 @@ def _build_hybrid_with_mamba_layer( spec_config=None, max_batch_size=4, enable_block_reuse=False, - mamba_state_cache_interval=256, + periodic_snapshot_interval=256, is_estimating_kv_cache=False, dtype=DataType.HALF, mamba_layer_mask=None, attention_layer_mask=None, mamba_ssm_cache_dtype=torch.float16, + use_replay_state_update=False, ): """Construct a real CppMambaHybridCacheManager with one mamba layer + one full-attention layer so the parent KVCacheManager goes through the @@ -377,7 +1637,7 @@ def _build_hybrid_with_mamba_layer( kv_cache_config = KvCacheConfig( max_tokens=512, enable_block_reuse=enable_block_reuse, - mamba_state_cache_interval=mamba_state_cache_interval, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=periodic_snapshot_interval), ) return CppMambaHybridCacheManager( mamba_d_state=8, @@ -402,6 +1662,681 @@ def _build_hybrid_with_mamba_layer( layer_mask=attn_mask, is_estimating_kv_cache=is_estimating_kv_cache, dtype=dtype, + use_replay_state_update=use_replay_state_update, + ) + + +def _build_v2_hybrid_with_mamba_layer( + max_batch_size=4, + num_mamba_layers=1, + num_attention_layers=1, + num_kv_heads=4, + mapping=None, + spec_config=None, + use_replay_state_update=False, + enable_block_reuse=False, + enable_partial_reuse=True, + block_reuse_policy="all_reusable", + periodic_snapshot_interval=0, + additional_snapshot_offsets_from_end=None, + enable_attention_dp=False, + enable_swa_scratch_reuse=False, + dtype=DataType.HALF, + conv_state_layout="x_b_c", +): + """Construct a real MambaHybridCacheManagerV2.""" + mamba_mask = [True] * num_mamba_layers + [False] * num_attention_layers + attn_mask = [False] * num_mamba_layers + [True] * num_attention_layers + if mapping is None: + mapping = Mapping( + world_size=1, + rank=0, + tp_size=1, + pp_size=1, + enable_attention_dp=enable_attention_dp, + ) + kv_cache_config = KvCacheConfig( + max_tokens=512, + enable_block_reuse=enable_block_reuse, + enable_partial_reuse=enable_partial_reuse, + block_reuse_policy=block_reuse_policy, + enable_swa_scratch_reuse=enable_swa_scratch_reuse, + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=periodic_snapshot_interval, + additional_snapshot_offsets_from_end=list(additional_snapshot_offsets_from_end or []), + ), + dtype="nvfp4" if dtype == DataType.NVFP4 else "auto", + ) + return MambaHybridCacheManagerV2( + mamba_d_state=8, + mamba_d_conv=4, + mamba_num_heads=4, + mamba_n_groups=1, + mamba_head_dim=8, + mamba_num_layers=num_mamba_layers, + mamba_layer_mask=mamba_mask, + mamba_cache_dtype=torch.float16, + mamba_ssm_cache_dtype=torch.float16, + kv_cache_config=kv_cache_config, + kv_cache_type=CacheTypeCpp.SELF, + num_layers=num_attention_layers, + num_kv_heads=num_kv_heads, + head_dim=64, + tokens_per_block=32, + max_seq_len=128, + max_batch_size=max_batch_size, + mapping=mapping, + spec_config=spec_config, + layer_mask=attn_mask, + vocab_size=1024, + use_replay_state_update=use_replay_state_update, + dtype=dtype, + conv_state_layout=conv_state_layout, + ) + + +def _make_wide_spec_config(max_draft_len=2, tokens_per_gen_step=5): + """Spec config whose per-step token width is wider than draft depth. + + This mirrors parallel-draft style metadata closely enough for cache-manager + sizing without constructing the full speculative worker stack. + """ + return SimpleNamespace( + max_draft_len=max_draft_len, + max_total_draft_tokens=tokens_per_gen_step - 1, + tokens_per_gen_step=tokens_per_gen_step, + spec_dec_mode=SimpleNamespace(use_one_engine=lambda: False), + ) + + +def _assert_replay_layer_cache_uses_history_size(layer_cache, history_size): + assert layer_cache.old_x is not None + assert layer_cache.old_B is not None + assert layer_cache.old_dt is not None + assert layer_cache.old_dA_cumsum is not None + assert layer_cache.cache_buf_idx is not None + assert layer_cache.prev_num_accepted_tokens is not None + assert layer_cache.old_x.dim() == 5 + cache_size = layer_cache.temporal.shape[0] + assert layer_cache.old_x.shape[0] == cache_size + assert layer_cache.old_B.shape[0] == cache_size + assert layer_cache.old_dt.shape[0] == cache_size + assert layer_cache.old_dA_cumsum.shape[0] == cache_size + assert layer_cache.cache_buf_idx.shape[0] == cache_size + assert layer_cache.prev_num_accepted_tokens.shape[0] == cache_size + assert layer_cache.old_x.shape[1] == 2 + assert layer_cache.old_B.shape[1] == 2 + assert layer_cache.old_dt.shape[1] == 2 + assert layer_cache.old_dA_cumsum.shape[1] == 2 + assert layer_cache.old_x.shape[2] == history_size + assert layer_cache.old_B.shape[2] == history_size + assert layer_cache.old_dt.shape[-1] == history_size + assert layer_cache.old_dA_cumsum.shape[-1] == history_size + + +@skip_no_cuda +def test_v2_hybrid_allocates_mamba_state_and_dummy_indices(): + mgr = _build_v2_hybrid_with_mamba_layer(max_batch_size=4) + try: + assert mgr.local_num_mamba_layers == 1 + assert len(mgr.all_ssm_states) == 1 + assert len(mgr.all_conv_states) == 1 + assert mgr.all_ssm_states[0].shape[1:] == torch.Size([4, 8, 8]) + assert mgr.all_conv_states[0].shape[1:] == torch.Size([48, 3]) + assert mgr.get_max_resource_count() == 4 + assert mgr.blocks_in_primary_pool > 0 + assert isinstance(mgr.check_invalid_values_in_kv_cache(), bool) + + requests = mgr.add_dummy_requests([123], token_nums=[8], is_gen=False) + + assert len(requests) == 1 + indices = mgr.get_state_indices([123], [False]) + assert len(indices) == 1 + assert indices[0] >= 0 + assert mgr._request_id_to_is_dummy[123] + assert mgr.cuda_state_indices[0].item() == indices[0] + assert mgr.get_ssm_states(0).data_ptr() == mgr.all_ssm_states[0].data_ptr() + assert mgr.get_conv_states(0).data_ptr() == mgr.all_conv_states[0].data_ptr() + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_nvfp4_page_table_omits_ssm_block_scales(): + mgr = _build_v2_hybrid_with_mamba_layer(dtype=DataType.NVFP4) + try: + ssm_pool_id = mgr.impl.get_layer_group_id(LayerId(0)) + attention_pool_id = mgr.impl.get_layer_group_id(LayerId(1)) + + assert torch.count_nonzero(mgr.kv_cache_pool_pointers[ssm_pool_id, :, 1]) == 0 + assert torch.count_nonzero(mgr.kv_cache_pool_pointers[attention_pool_id, :, 1]) > 0 + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_supports_pure_mamba_pp_rank(): + mgr = _build_v2_hybrid_with_mamba_layer( + max_batch_size=2, + num_attention_layers=0, + ) + try: + assert mgr.local_num_mamba_layers == 1 + assert mgr.blocks_in_primary_pool == 0 + assert mgr._attention_cache_bytes_per_token() == 0 + assert len(mgr.all_ssm_states) == 1 + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_metadata_supports_attention_only_pp_rank(): + mapping = Mapping(world_size=2, rank=1, tp_size=1, pp_size=2) + mgr = _build_v2_hybrid_with_mamba_layer( + max_batch_size=1, + mapping=mapping, + ) + try: + assert mgr.local_num_mamba_layers == 0 + mgr.add_dummy_requests([123], token_nums=[8], is_gen=False) + + metadata = Mamba2Metadata(max_batch_size=1, chunk_size=8) + seq_lens = torch.tensor([8], dtype=torch.int32) + metadata.prepare( + SimpleNamespace( + seq_lens=seq_lens, + seq_lens_cuda=seq_lens.cuda(), + num_contexts=1, + num_ctx_tokens=8, + kv_cache_manager=mgr, + request_ids=[123], + kv_cache_params=SimpleNamespace( + num_cached_tokens_per_seq=torch.tensor([0], dtype=torch.int32) + ), + ) + ) + + assert metadata.state_indices[0].item() == 0 + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_invalid_check_scans_distinct_attention_pools(): + mgr = _build_v2_hybrid_with_mamba_layer( + num_attention_layers=2, + num_kv_heads=[1, 2, 4], + ) + try: + mgr.check_invalid_values_in_kv_cache(fill_with_zero=True) + first_attention_buffer = mgr.get_buffers(1) + second_attention_buffer = mgr.get_buffers(2) + assert first_attention_buffer.data_ptr() != second_attention_buffer.data_ptr() + + for buffer in ( + second_attention_buffer, + mgr.all_ssm_states[0], + mgr.all_conv_states[0], + ): + buffer.flatten()[0] = torch.nan + assert mgr.check_invalid_values_in_kv_cache() + assert mgr.check_invalid_values_in_kv_cache(fill_with_zero=True) + assert not torch.isnan(buffer).any() + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_dummy_indices_keep_cuda_buffer_address(): + max_batch_size = 1 + mgr = _build_v2_hybrid_with_mamba_layer(max_batch_size=max_batch_size, enable_attention_dp=True) + try: + request_ids = list(range(100, 100 + max_batch_size)) + mgr.add_dummy_requests( + request_ids, + token_nums=[8] * max_batch_size, + is_gen=False, + ) + state_indices_ptr = mgr.cuda_state_indices.data_ptr() + + new_requests = mgr.add_dummy_requests( + [ATTENTION_DP_DUMMY_REQUEST_ID], token_nums=[8], is_gen=False + ) + + assert len(new_requests) == 1 + expected_capacity = max_batch_size + mgr._num_reserved_dummy_slots + assert mgr.cuda_state_indices.shape[0] == expected_capacity + assert mgr._host_state_indices.shape[0] == expected_capacity + assert mgr.cuda_state_indices.data_ptr() == state_indices_ptr + assert ( + mgr.cuda_state_indices[0].item() + == mgr.get_state_indices([ATTENTION_DP_DUMMY_REQUEST_ID], [False])[0] + ) + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_reserves_every_persistent_dummy_slot(): + spec_config = MTPDecodingConfig( + max_draft_len=4, + draft_len_schedule={1: 4, 2: 2, 3: 1}, + ) + mgr = _build_v2_hybrid_with_mamba_layer( + max_batch_size=4, + spec_config=spec_config, + enable_attention_dp=True, + ) + try: + runtime_draft_lengths = [4, 2, 1, 0] + cuda_graph_dummy_ids = [ + CUDA_GRAPH_DUMMY_REQUEST_ID - draft_len for draft_len in runtime_draft_lengths + ] + request_ids = [101, 102, 103, 104] + + assert mgr._num_reserved_dummy_slots == 5 + assert mgr.index_mapper.num_free_slots() == len(request_ids) + 5 + + assert ( + mgr.add_dummy_requests(request_ids, token_nums=[1] * len(request_ids), is_gen=False) + is not None + ) + for request_id, draft_len in zip(cuda_graph_dummy_ids, runtime_draft_lengths): + assert ( + mgr.add_dummy_requests( + [request_id], + is_gen=True, + max_num_draft_tokens=draft_len, + ) + is not None + ) + assert ( + mgr.add_dummy_requests([ATTENTION_DP_DUMMY_REQUEST_ID], token_nums=[1], is_gen=False) + is not None + ) + + all_request_ids = request_ids + cuda_graph_dummy_ids + [ATTENTION_DP_DUMMY_REQUEST_ID] + state_indices = mgr.get_state_indices(all_request_ids, [False] * len(all_request_ids)) + assert len(set(state_indices)) == len(all_request_ids) + assert mgr.index_mapper.num_free_slots() == 0 + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_free_resources_drops_stale_state_index_mapping(): + mgr = _build_v2_hybrid_with_mamba_layer() + try: + request = mgr.add_dummy_requests([123], token_nums=[8], is_gen=False)[0] + request_id = request.py_request_id + assert request_id in mgr._request_id_to_state_index + assert request_id in mgr._request_id_to_is_dummy + + # Move state-index preparation to another request before freeing the + # older one, as happens when an asynchronous transfer finishes late. + mgr.add_dummy_requests([456], token_nums=[8], is_gen=False) + mgr.free_resources(request) + + assert request_id not in mgr._request_id_to_state_index + assert request_id not in mgr._request_id_to_is_dummy + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_uses_upstream_min_snapshot_policy(): + mgr = _build_v2_hybrid_with_mamba_layer( + enable_block_reuse=True, + enable_partial_reuse=True, + ) + try: + assert mgr.block_reuse_policy is BlockReusePolicy.PER_REQUEST + assert mgr.kv_cache_config.enable_partial_reuse + assert mgr.kv_cache_manager_py_config.commit_min_snapshot + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_preserves_per_conversation_and_disables_periodic_snapshots(): + mgr = _build_v2_hybrid_with_mamba_layer( + enable_block_reuse=True, + block_reuse_policy="per_conversation", + periodic_snapshot_interval=64, + additional_snapshot_offsets_from_end=[0], + ) + try: + assert mgr.block_reuse_policy is BlockReusePolicy.PER_CONVERSATION + assert mgr.conversation_manager is not None + assert mgr.kv_cache_config.mamba_state_config.periodic_snapshot_interval == 0 + assert mgr.kv_cache_manager_py_config.commit_min_snapshot + request = SimpleNamespace(prompt_len=150, expect_snapshot_points=[]) + mgr.prepare_expect_snapshot_points([request]) + assert request.expect_snapshot_points == [150] + finally: + mgr.shutdown() + + +def test_v2_hybrid_saves_conversation_plan_only_after_final_context_chunk(): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.enable_block_reuse = True + mgr.is_draft = False + mgr.block_reuse_policy = BlockReusePolicy.PER_CONVERSATION + events = [] + mgr._augment_tokens_for_block_reuse = lambda tokens, request, start, end: tokens[start:end] + mgr._mark_context_position_as_history = MagicMock() + mgr.conversation_manager = MagicMock() + mgr.conversation_manager.save_drop_plan.side_effect = lambda request, kv_cache: events.append( + "save" + ) + + request = SimpleNamespace( + py_request_id=7, + is_dummy_request=False, + context_current_position=128, + context_remaining_length=0, + expect_snapshot_points=[128], + prompt_len=128, + is_last_context_chunk=True, + get_tokens=lambda beam_idx: list(range(128)), + ) + kv_cache = SimpleNamespace( + is_active=True, + num_committed_tokens=0, + resize=MagicMock(return_value=True), + enable_swa_scratch_reuse=True, + ) + + def commit(tokens): + events.append("commit") + kv_cache.num_committed_tokens += len(tokens) + + kv_cache.commit = MagicMock(side_effect=commit) + kv_cache.stop_committing = MagicMock(side_effect=lambda: events.append("stop")) + mgr.kv_cache_map = {request.py_request_id: kv_cache} + batch = ScheduledRequests() + batch.append_context_request(request) + + mgr.update_context_resources(batch) + + kv_cache.commit.assert_called_once_with(list(range(128))) + kv_cache.stop_committing.assert_called_once_with() + mgr.conversation_manager.save_drop_plan.assert_called_once_with(request, kv_cache) + assert events == ["commit", "stop", "save"] + assert not kv_cache.enable_swa_scratch_reuse + + +@skip_no_cuda +def test_v2_hybrid_mamba_state_views_use_logical_slots(): + mgr = _build_v2_hybrid_with_mamba_layer(max_batch_size=4, num_mamba_layers=2) + try: + assert len(mgr.all_ssm_states) == 2 + assert len(mgr.all_conv_states) == 2 + + ssm_slots = mgr.all_ssm_states[0].shape[0] + conv_slots = mgr.all_conv_states[0].shape[0] + assert all(t.shape[0] == ssm_slots for t in mgr.all_ssm_states) + assert all(t.shape[0] == conv_slots for t in mgr.all_conv_states) + assert ssm_slots == conv_slots + + local_layer_ids = [mgr.layer_offsets[layer_id] for layer_id in mgr.mamba_pp_layers] + for local_layer_idx, ssm_state, conv_state in zip( + local_layer_ids, mgr.all_ssm_states, mgr.all_conv_states + ): + layer_id = LayerId(local_layer_idx) + ssm_scale = mgr.impl.get_page_index_scale(layer_id, MambaRole.SSM_STATE) + conv_scale = mgr.impl.get_page_index_scale(layer_id, MambaRole.CONV_STATE) + assert ssm_state.stride(0) == ssm_state[0].numel() * ssm_scale + assert conv_state.stride(0) == conv_state[0].numel() * conv_scale + assert ( + ssm_state.shape[0] + == ( + mgr.impl.get_page_index_upper_bound(layer_id, MambaRole.SSM_STATE) + + ssm_scale + - 1 + ) + // ssm_scale + ) + assert ( + conv_state.shape[0] + == ( + mgr.impl.get_page_index_upper_bound(layer_id, MambaRole.CONV_STATE) + + conv_scale + - 1 + ) + // conv_scale + ) + + mgr.add_dummy_requests([123, 456], token_nums=[8, 8], is_gen=False) + indices = mgr.get_state_indices([123, 456], [False, False]) + assert all(0 <= index < ssm_slots for index in indices) + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_swa_scratch_keeps_ssm_placeholder_rows(): + mgr = _build_v2_hybrid_with_mamba_layer( + max_batch_size=4, + enable_swa_scratch_reuse=True, + ) + try: + request_id = 123 + mgr.add_dummy_requests([request_id], token_nums=[8], is_gen=False) + block_offsets = torch.zeros( + mgr.num_attention_op_pools, + 1, + 2, + mgr.max_blocks_per_seq, + dtype=torch.int32, + device="cuda", + ) + + mgr.copy_batch_block_offsets( + block_offsets, + [request_id], + beam_width=1, + num_contexts=1, + num_seqs=1, + ) + torch.cuda.synchronize() + + assert mgr.num_attention_op_pools == mgr.num_local_layers + assert mgr.kv_cache_pool_mapping.shape[0] == mgr.num_local_layers + assert mgr.kv_cache_pool_pointers[0, 0].item() != 0 + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_disagg_page_table_preserves_lifecycle_indices(): + mgr = _build_v2_hybrid_with_mamba_layer(max_batch_size=4, num_mamba_layers=2) + try: + page_table = build_page_table_from_manager(mgr) + + assert len(page_table.layer_groups) == len(mgr.impl.layer_grouping) + assert isinstance(page_table.layer_groups[0], MambaLayerGroup) + assert isinstance(page_table.layer_groups[1], AttentionLayerGroup) + + requests = mgr.add_dummy_requests([123], token_nums=[64], is_gen=False) + assert len(requests) == 1 + attention_blocks = list( + mgr.kv_cache_map[123].get_aggregated_page_indices(1, valid_only=True) + ) + assert attention_blocks + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_disagg_page_table_uses_qwen3_next_conv_sections(): + mgr = _build_v2_hybrid_with_mamba_layer( + max_batch_size=4, + conv_state_layout="q_k_v", + ) + try: + page_table = build_page_table_from_manager(mgr) + mamba_group = page_table.layer_groups[0] + + assert isinstance(mamba_group, MambaLayerGroup) + d_conv_m1 = mgr.conv_state_shape[1] + conv_elem_size = mgr.all_conv_states[0].element_size() + assert mamba_group.conv_section_bytes == [ + dim * d_conv_m1 * conv_elem_size for dim in mgr.conv_section_dims + ] + assert mgr.conv_section_dims == [8, 8, 32] + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_intermediate_states_size_by_tokens_per_gen_step(): + spec_config = _make_wide_spec_config(max_draft_len=2, tokens_per_gen_step=5) + mgr = _build_v2_hybrid_with_mamba_layer( + max_batch_size=4, + spec_config=spec_config, + ) + try: + assert mgr.intermediate_ssm_states.shape[2] == 5 + assert mgr.intermediate_conv_states.shape[2] == 5 + layer_cache = mgr.mamba_layer_cache(0) + assert layer_cache.intermediate_ssm.data_ptr() == ( + mgr.intermediate_ssm_states[0].data_ptr() + ) + finally: + mgr.shutdown() + + +@skip_no_cuda +def test_v2_hybrid_static_dynamic_tree_capacity(): + spec_config = MTPDecodingConfig( + max_draft_len=6, + max_total_draft_tokens=31, + use_dynamic_tree=True, + dynamic_tree_max_topK=10, + ) + mgr = _build_v2_hybrid_with_mamba_layer( + max_batch_size=4, + spec_config=spec_config, + ) + try: + assert mgr.intermediate_ssm_states.shape[2] == 32 + assert mgr.intermediate_conv_states.shape[2] == 32 + assert mgr._kv_reserve_draft_tokens == 31 + assert mgr._num_reserved_dummy_slots == 1 + assert not mgr.use_replay_state_update + finally: + mgr.shutdown() + + +@skip_no_cuda +@pytest.mark.parametrize( + "builder", + [_build_hybrid_with_mamba_layer, _build_v2_hybrid_with_mamba_layer], + ids=["cpp", "v2"], +) +def test_hybrid_replay_buffers_size_by_tokens_per_gen_step(builder): + spec_config = _make_wide_spec_config(max_draft_len=2, tokens_per_gen_step=5) + mgr = builder( + max_batch_size=4, + spec_config=spec_config, + use_replay_state_update=True, + ) + try: + replay_metadata = mgr.get_replay_state_update_metadata() + assert mgr.use_replay_state_update is True + assert replay_metadata is not None + assert replay_metadata.replay_step_width == spec_config.tokens_per_gen_step + assert replay_metadata.replay_history_size == max( + MIN_REPLAY_HISTORY_SIZE, spec_config.tokens_per_gen_step + ) + layer_cache = mgr.mamba_layer_cache(0) + _assert_replay_layer_cache_uses_history_size( + layer_cache, replay_metadata.replay_history_size + ) + finally: + mgr.shutdown() + + +def test_v2_hybrid_replay_update_skips_dummy_and_padding_rows(monkeypatch): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.local_num_mamba_layers = 1 + mgr._request_id_to_state_index = { + 100: 0, + 101: 1, + 102: 2, + 103: 3, + } + mgr._request_id_to_is_dummy = { + 100: False, + 101: False, + 102: True, + 103: False, + } + mgr._dummy_request_mask = torch.zeros(4, dtype=torch.bool) + mgr._dummy_request_mask_host = torch.zeros(4, dtype=torch.bool) + mgr._use_replay_state_update = True + mgr.replay_step_width = 5 + mgr.replay_history_size = 16 + mgr.prev_num_accepted_tokens = torch.full((4,), 13, dtype=torch.int32) + mgr.cache_buf_idx = torch.ones(4, dtype=torch.int32) + mgr.intermediate_state_indices = torch.arange(4, dtype=torch.int32) + mgr.all_ssm_states = [] + mgr.all_conv_states = [torch.empty(0)] + mgr.intermediate_conv_states = torch.empty(0) + monkeypatch.setattr( + "tensorrt_llm._torch.pyexecutor.mamba_cache_manager._promote_mamba_state_triton", + lambda *args, **kwargs: None, + ) + + request_ids = [100, 101, 102, 103] + state_indices = torch.tensor( + mgr.get_state_indices(request_ids, [False, False, False, True]), + dtype=torch.int32, + ) + assert mgr._dummy_request_mask.tolist() == [False, False, True, True] + + mgr.update_mamba_states( + SimpleNamespace(num_seqs=4, num_contexts=1), + torch.tensor([1, 3, 3, 3], dtype=torch.int32), + state_indices=state_indices, + ) + + assert mgr.prev_num_accepted_tokens.tolist() == [13, 3, 13, 13] + assert mgr.cache_buf_idx.tolist() == [1, 0, 1, 1] + + +def test_v2_hybrid_dynamic_tree_promotes_accepted_leaf_state(monkeypatch): + mgr = object.__new__(MambaHybridCacheManagerV2) + mgr.local_num_mamba_layers = 1 + mgr._use_replay_state_update = False + mgr.intermediate_state_indices = torch.arange(2, dtype=torch.int32) + mgr.all_ssm_states = [torch.empty(0)] + mgr.all_conv_states = [torch.empty(0)] + mgr.intermediate_ssm_states = torch.empty((1, 2, 8)) + mgr.intermediate_conv_states = torch.empty((1, 2, 8)) + + promoted_positions = [] + + def capture_promoted_position(_dst, _src, _src_indices, positions, _dst_indices): + promoted_positions.append(positions.clone()) + + monkeypatch.setattr( + "tensorrt_llm._torch.pyexecutor.mamba_cache_manager._promote_mamba_state_triton", + capture_promoted_position, + ) + + mgr.update_mamba_states( + SimpleNamespace(num_seqs=3, num_contexts=1), + torch.tensor([1, 2, 3], dtype=torch.int32), + state_indices=torch.tensor([9, 10, 11], dtype=torch.int32), + accepted_leaf_positions=torch.tensor([4, 7], dtype=torch.int64), + ) + + assert len(promoted_positions) == 2 + assert all( + torch.equal(positions, torch.tensor([4, 7], dtype=torch.int32)) + for positions in promoted_positions ) @@ -577,7 +2512,7 @@ def test_cpp_hybrid_recurrent_pool_floor_with_block_reuse(): """With block reuse enabled, the block-reuse branch must not drop the live-state + CUDA-graph-padding floor. - With max_batch_size=4, mamba_state_cache_interval=256, max_tokens=512: + With max_batch_size=4, periodic_snapshot_interval=256, max_tokens=512: naive: max_snapshots = 512 // 256 = 2 (drops live-state floor!) fixed: max_snapshots = max(2, 4 + 1) = 5 """ @@ -586,7 +2521,7 @@ def test_cpp_hybrid_recurrent_pool_floor_with_block_reuse(): spec_config=None, max_batch_size=max_batch_size, enable_block_reuse=True, - mamba_state_cache_interval=256, + periodic_snapshot_interval=256, ) recurrent_primary, _ = mgr.blocks_per_window[LinearCacheType.RECURRENT_STATES.value] assert recurrent_primary >= max_batch_size + 1, ( @@ -609,7 +2544,7 @@ def test_cpp_hybrid_dry_run_recurrent_pool_additive_with_block_reuse(): spec_config=None, max_batch_size=max_batch_size, enable_block_reuse=True, - mamba_state_cache_interval=256, + periodic_snapshot_interval=256, is_estimating_kv_cache=True, ) recurrent_primary, _ = mgr.blocks_per_window[LinearCacheType.RECURRENT_STATES.value] @@ -631,9 +2566,8 @@ def test_cpp_hybrid_dry_run_recurrent_pool_additive_with_block_reuse(): # - call the real parent KVCacheManager with the union layer_mask and # num_layers=num_layers (not mamba_num_layers + num_layers), # - skip allocating any mamba-only state, and -# - leave self.requests = [] so the guards on prepare_resources / -# update_mamba_states / _setup_state_indices can no-op without touching -# uninitialized state. +# - leave self.requests = [] so Mamba-only hooks and metadata preparation +# can no-op without touching uninitialized state. # # We exercise the same Python branch with world_size=1 (so the real C++ # KVCacheManager init doesn't need MPI) and a layer mask that contains zero @@ -641,7 +2575,10 @@ def test_cpp_hybrid_dry_run_recurrent_pool_additive_with_block_reuse(): # --------------------------------------------------------------------------- -def _build_zero_mamba_hybrid(): +def _build_zero_mamba_hybrid( + enable_block_reuse=False, + periodic_snapshot_interval=256, +): """Construct a real CppMambaHybridCacheManager whose this-rank slice has no mamba layers. world_size=1 / pp_size=1 keeps the real parent KVCacheManager off the MPI path.""" @@ -654,7 +2591,11 @@ def _build_zero_mamba_hybrid(): mapping = Mapping(world_size=1, rank=0, tp_size=1, pp_size=1) # Cap KV pool size so the real C++ allocator only takes a tiny slice of # GPU memory; we don't actually use the cache. - kv_cache_config = KvCacheConfig(max_tokens=128) + kv_cache_config = KvCacheConfig( + max_tokens=128, + enable_block_reuse=enable_block_reuse, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=periodic_snapshot_interval), + ) mgr = CppMambaHybridCacheManager( # mamba cache parameters — values are unused on the early-exit path @@ -687,9 +2628,12 @@ def _build_zero_mamba_hybrid(): @skip_no_cuda def test_cpp_hybrid_zero_local_mamba_layers(): """End-to-end: real parent KVCacheManager + real early-exit. Verifies - early-exit invariants on the manager state AND that the three guarded - methods no-op without raising on uninitialized mamba-only state.""" - mgr = _build_zero_mamba_hybrid() + early-exit invariants on the manager state and that Mamba hooks and + metadata preparation do not touch uninitialized Mamba-only state.""" + mgr = _build_zero_mamba_hybrid( + enable_block_reuse=True, + periodic_snapshot_interval=64, + ) # Early-exit indicators. assert mgr.local_num_mamba_layers == 0 @@ -727,9 +2671,40 @@ def test_cpp_hybrid_zero_local_mamba_layers(): # Parent must not have been told to treat this as linear attention. assert mgr.is_linear_attention is False + # The scheduler hook is still advertised on ranks without local Mamba + # layers, so its inputs must be initialized with the same interval as + # Mamba-owning PP ranks to keep their scheduling decisions aligned. + assert mgr.kv_cache_config.enable_block_reuse is True + assert mgr.linear_attention_metadata.states_snapshot_interval == 64 + request = SimpleNamespace(prompt_len=150, expect_snapshot_points=[]) + mgr.prepare_expect_snapshot_points([request]) + assert request.expect_snapshot_points == [64, 128] + # The shared interval must not make this attention-only rank consult its + # nonexistent recurrent-state pool and report zero KV capacity. + attention_capacity = KVCacheManager.get_num_available_tokens(mgr, 128) + assert attention_capacity > 0 + assert mgr.get_num_available_tokens(128) == attention_capacity + # Guards on the three mamba-only methods must turn them into no-ops # instead of crashing on the missing state above. empty_batch = ScheduledRequests() mgr.prepare_resources(empty_batch) # super() runs, then guard returns mgr.update_mamba_states(attn_metadata=None, num_accepted_tokens=None, state_indices=None) mgr._setup_state_indices() + + metadata = Mamba2Metadata(max_batch_size=1, chunk_size=8) + seq_lens = torch.tensor([8], dtype=torch.int32) + metadata.prepare( + SimpleNamespace( + seq_lens=seq_lens, + seq_lens_cuda=seq_lens.cuda(), + num_contexts=1, + num_ctx_tokens=8, + kv_cache_manager=mgr, + request_ids=[123], + kv_cache_params=SimpleNamespace( + num_cached_tokens_per_seq=torch.tensor([0], dtype=torch.int32) + ), + ) + ) + assert metadata.state_indices[0].item() == 0 diff --git a/tests/unittest/_torch/executor/test_model_loader_gms.py b/tests/unittest/_torch/executor/test_model_loader_gms.py index 34dd8223a6a5..b56935da6378 100644 --- a/tests/unittest/_torch/executor/test_model_loader_gms.py +++ b/tests/unittest/_torch/executor/test_model_loader_gms.py @@ -14,6 +14,9 @@ from tensorrt_llm._torch.pyexecutor import model_loader as model_loader_mod from tensorrt_llm._torch.pyexecutor.model_loader import ModelLoader from tensorrt_llm._torch.weight_sharing import ( + ARTIFACT_IDENTITY_FORMAT_VERSION, + SOURCE_IDENTITY_FORMAT_VERSION, + ArtifactIdentity, PostTransformProfile, PostTransformProfileRegistry, PostTransformTransferScope, @@ -21,7 +24,12 @@ from tensorrt_llm.llmapi.llm_args import LoadFormat _SOURCE_IDENTITY = model_loader_mod.SourceIdentity( - format_version=1, + format_version=SOURCE_IDENTITY_FORMAT_VERSION, + artifact_identity=ArtifactIdentity( + format_version=ARTIFACT_IDENTITY_FORMAT_VERSION, + scheme="checkpoint_manifest_sha256", + digest="0" * 64, + ), model_fingerprint="model", quant_fingerprint="quant", backend_fingerprint="backend", @@ -100,10 +108,25 @@ def _make_loader(monkeypatch, *, events, spec_config=None): monkeypatch.setattr(model_loader_mod, "MetaInitMode", lambda: nullcontext()) # These tests stub ModelConfig, while SourceIdentity has dedicated # coverage. Keep this file focused on ModelLoader GMS branch behavior. + + def _build_artifact_identity(_cls, checkpoint_dir): + assert checkpoint_dir == "/ckpt" + return _SOURCE_IDENTITY.artifact_identity + + monkeypatch.setattr( + model_loader_mod.ArtifactIdentity, + "from_checkpoint", + classmethod(_build_artifact_identity), + ) + + def _build_source_identity(_cls, *_args, **kwargs): + assert kwargs["artifact_identity"] is _SOURCE_IDENTITY.artifact_identity + return _SOURCE_IDENTITY + monkeypatch.setattr( model_loader_mod.SourceIdentity, "from_model_config", - classmethod(lambda cls, *_args, **_kwargs: _SOURCE_IDENTITY), + classmethod(_build_source_identity), ) monkeypatch.setattr( model_loader_mod.AutoModelForCausalLM, @@ -181,6 +204,31 @@ def _tiny_profile_registry() -> PostTransformProfileRegistry: ) +def test_gms_artifact_identity_failure_remains_fatal(monkeypatch): + loader = _make_loader(monkeypatch, events=[]) + artifact_error = ValueError( + "Checkpoint manifests do not support nested symlinked directories: /ckpt/shards" + ) + monkeypatch.setattr( + model_loader_mod.ArtifactIdentity, + "from_checkpoint", + MagicMock(side_effect=artifact_error), + ) + source_identity_factory = MagicMock() + monkeypatch.setattr( + model_loader_mod.SourceIdentity, + "from_model_config", + source_identity_factory, + ) + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "MX" + + with pytest.raises(ValueError, match="nested symlinked directories"): + loader.load("/ckpt", checkpoint_loader) + + source_identity_factory.assert_not_called() + + @pytest.mark.parametrize( "is_rw, expected_events", [ diff --git a/tests/unittest/_torch/executor/test_model_loader_mx.py b/tests/unittest/_torch/executor/test_model_loader_mx.py index 7cbfaa4f9941..ccdf960ea786 100644 --- a/tests/unittest/_torch/executor/test_model_loader_mx.py +++ b/tests/unittest/_torch/executor/test_model_loader_mx.py @@ -17,12 +17,16 @@ from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models import modeling_llama as modeling_llama_mod +from tensorrt_llm._torch.models.checkpoints.mx.checkpoint_loader import MXCheckpointLoader from tensorrt_llm._torch.modules import mla as mla_mod from tensorrt_llm._torch.modules.linear import Linear from tensorrt_llm._torch.modules.mla import MLA from tensorrt_llm._torch.pyexecutor import model_loader as model_loader_mod from tensorrt_llm._torch.pyexecutor.model_loader import ModelLoader from tensorrt_llm._torch.weight_sharing import ( + ARTIFACT_IDENTITY_FORMAT_VERSION, + SOURCE_IDENTITY_FORMAT_VERSION, + ArtifactIdentity, PostTransformFeature, PostTransformProfile, PostTransformProfileRegistry, @@ -32,7 +36,12 @@ from tensorrt_llm.llmapi.llm_args import LoadFormat _SOURCE_IDENTITY = model_loader_mod.SourceIdentity( - format_version=1, + format_version=SOURCE_IDENTITY_FORMAT_VERSION, + artifact_identity=ArtifactIdentity( + format_version=ARTIFACT_IDENTITY_FORMAT_VERSION, + scheme="checkpoint_manifest_sha256", + digest="0" * 64, + ), model_fingerprint="model", quant_fingerprint="quant", backend_fingerprint="backend", @@ -187,10 +196,25 @@ def _make_loader(monkeypatch, *, events, spec_config=None): monkeypatch.setattr(model_loader_mod, "MetaInitMode", lambda: nullcontext()) # These tests stub ModelConfig, while SourceIdentity has dedicated # coverage. Keep this file focused on ModelLoader MX branch behavior. + + def _build_artifact_identity(_cls, checkpoint_dir): + assert checkpoint_dir == "/ckpt" + return _SOURCE_IDENTITY.artifact_identity + + monkeypatch.setattr( + model_loader_mod.ArtifactIdentity, + "from_checkpoint", + classmethod(_build_artifact_identity), + ) + + def _build_source_identity(_cls, *_args, **kwargs): + assert kwargs["artifact_identity"] is _SOURCE_IDENTITY.artifact_identity + return _SOURCE_IDENTITY + monkeypatch.setattr( model_loader_mod.SourceIdentity, "from_model_config", - classmethod(lambda cls, *_args, **_kwargs: _SOURCE_IDENTITY), + classmethod(_build_source_identity), ) monkeypatch.setattr( model_loader_mod.AutoModelForCausalLM, @@ -206,6 +230,26 @@ def _make_loader(monkeypatch, *, events, spec_config=None): return loader +def test_construct_checkpoint_loader_passes_mx_config(): + mx_config = SimpleNamespace( + server_url="http://mx:8001", + server_query_timeout_s=17, + ) + + checkpoint_loader = model_loader_mod._construct_checkpoint_loader( + "pytorch", + None, + "MX", + mx_config=mx_config, + mx_model_name="Qwen/Qwen2.5-7B-Instruct", + ) + + assert isinstance(checkpoint_loader, MXCheckpointLoader) + assert checkpoint_loader.mx_server_url == "http://mx:8001" + assert checkpoint_loader.query_timeout_s == 17 + assert checkpoint_loader.model_name == "Qwen/Qwen2.5-7B-Instruct" + + def test_mx_success_initializes_mapper_skips_weight_mapping_and_reload_works(monkeypatch): events = [] loader = _make_loader(monkeypatch, events=events) @@ -463,6 +507,53 @@ def test_mx_fallback_runs_standard_weight_mapping(monkeypatch): ) +def test_mx_artifact_identity_failure_falls_back_to_disk(monkeypatch): + events = [] + loader = _make_loader(monkeypatch, events=events) + monkeypatch.setattr( + ModelLoader, + "_POST_TRANSFORM_PROFILE_REGISTRY", + _tiny_profile_registry(), + ) + artifact_error = ValueError( + "Checkpoint manifests do not support nested symlinked directories: /ckpt/shards" + ) + monkeypatch.setattr( + model_loader_mod.ArtifactIdentity, + "from_checkpoint", + MagicMock(side_effect=artifact_error), + ) + source_identity_factory = MagicMock() + monkeypatch.setattr( + model_loader_mod.SourceIdentity, + "from_model_config", + source_identity_factory, + ) + warning = MagicMock() + monkeypatch.setattr(model_loader_mod.logger, "warning", warning) + + checkpoint_loader = MagicMock(name="checkpoint_loader") + checkpoint_loader.checkpoint_format = "MX" + checkpoint_loader.is_weights_preloaded.return_value = False + checkpoint_loader.load_weights.return_value = {"weight": MagicMock()} + checkpoint_loader.get_initialized_weight_mapper.return_value = MagicMock() + + model, _ = loader.load("/ckpt", checkpoint_loader) + + assert loader._source_identity is None + source_identity_factory.assert_not_called() + warning.assert_called_once() + assert "falling back to regular checkpoint loading" in warning.call_args.args[0] + _args, kwargs = checkpoint_loader.load_weights.call_args + assert kwargs["source_identity"] is None + checkpoint_loader.post_load_publish.assert_called_once_with( + model, + checkpoint_dir="/ckpt", + weights_preloaded=False, + source_identity=None, + ) + + class _HookRecorder(nn.Module): def __init__( self, diff --git a/tests/unittest/_torch/executor/test_py_executor.py b/tests/unittest/_torch/executor/test_py_executor.py index a0d77293b616..101edffbe155 100644 --- a/tests/unittest/_torch/executor/test_py_executor.py +++ b/tests/unittest/_torch/executor/test_py_executor.py @@ -16,7 +16,7 @@ import threading import time import types -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock import pytest @@ -893,6 +893,90 @@ def test_pp_schedule_restores_propagated_gate_decision(self): assert wait_for_progress +def test_nonzero_pp_rank_prepares_snapshot_points_before_local_schedule( + monkeypatch, +): + class StopLocalSchedule(RuntimeError): + pass + + executor = object.__new__(PyExecutor) + executor.dist = Mock(pp_rank=1, rank=1) + executor.device_id = 0 + profiler = MagicMock() + profiler.__enter__.return_value = Mock() + executor._profiler = Mock(return_value=profiler) + executor.hang_detector = MagicMock() + executor.enable_iter_perf_stats = False + executor._handle_disagg_cache_errors_synced = Mock() + executor._fetch_and_activate_new_requests = Mock(return_value=[]) + executor.is_shutdown = False + executor._handle_control_request = Mock() + executor.kv_cache_transceiver = None + executor._pad_attention_dp_dummy_request = Mock() + scheduled_batch = Mock() + executor._pp_schedule_and_propagate = Mock(return_value=(scheduled_batch, [], 0, False)) + executor._pp_retry_until_can_schedule = Mock() + request = Mock() + executor.active_requests = [request] + executor.inflight_req_ids = set() + executor.kv_cache_manager = Mock() + executor.scheduler = Mock() + + calls = [] + executor.kv_cache_manager.prepare_expect_snapshot_points.side_effect = ( + lambda requests: calls.append(("prepare", requests)) + ) + + def stop_after_schedule(requests, inflight_req_ids): + calls.append(("schedule", requests, inflight_req_ids)) + raise StopLocalSchedule + + executor.scheduler.schedule_request.side_effect = stop_after_schedule + + monkeypatch.setattr("tensorrt_llm._torch.pyexecutor.py_executor.torch.cuda.set_device", Mock()) + monkeypatch.setattr("tensorrt_llm._torch.pyexecutor.py_executor.cudart.cudaSetDevice", Mock()) + monkeypatch.setattr("tensorrt_llm._torch.pyexecutor.py_executor.CUASSERT", Mock()) + + with pytest.raises(StopLocalSchedule): + PyExecutor._executor_loop_pp(executor) + + assert calls == [ + ("prepare", executor.active_requests), + ("schedule", executor.active_requests, executor.inflight_req_ids), + ] + + +def test_schedule_prepares_snapshot_points_before_scheduling(): + class StopSchedule(RuntimeError): + pass + + executor = object.__new__(PyExecutor) + request = Mock() + executor.active_requests = [request] + executor.inflight_req_ids = set() + executor.kv_cache_manager = Mock() + executor.scheduler = Mock() + + calls = [] + executor.kv_cache_manager.prepare_expect_snapshot_points.side_effect = ( + lambda requests: calls.append(("prepare", requests)) + ) + + def stop_after_schedule(requests, inflight_req_ids): + calls.append(("schedule", requests, inflight_req_ids)) + raise StopSchedule + + executor.scheduler.schedule_request.side_effect = stop_after_schedule + + with pytest.raises(StopSchedule): + PyExecutor._schedule(executor) + + assert calls == [ + ("prepare", executor.active_requests), + ("schedule", executor.active_requests, executor.inflight_req_ids), + ] + + class TestComputeScheduledTokens: """Tests for PyExecutor._compute_scheduled_tokens. diff --git a/tests/unittest/_torch/executor/test_py_executor_creator_mla_cache_reuse_sync.py b/tests/unittest/_torch/executor/test_py_executor_creator_mla_cache_reuse_sync.py index f55827f3e775..d878a50c3c24 100644 --- a/tests/unittest/_torch/executor/test_py_executor_creator_mla_cache_reuse_sync.py +++ b/tests/unittest/_torch/executor/test_py_executor_creator_mla_cache_reuse_sync.py @@ -15,8 +15,15 @@ from types import SimpleNamespace +import pytest + from tensorrt_llm._torch.pyexecutor import py_executor_creator +from tensorrt_llm._torch.pyexecutor.py_executor_creator import ( + _MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS, + _MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS, +) from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManagerType +from tensorrt_llm.llmapi.llm_args import CacheTransceiverConfig, ContextChunkingPolicy from tensorrt_llm.quantization import QuantAlgo @@ -112,21 +119,30 @@ def build_managers(self, resources, estimating_kv_cache): class _DummyModelEngine: """Mock model engine that exposes attention runtime features and model configuration.""" - def __init__(self, *, attn_runtime_features, kv_cache_quant_algo): + def __init__( + self, + *, + attn_runtime_features, + kv_cache_quant_algo, + enable_flash_mla=False, + max_seq_len=128, + ): """Initialize with runtime features and quantization algorithm. Args: attn_runtime_features: AttentionRuntimeFeatures instance. kv_cache_quant_algo: Quantization algorithm for KV cache. + enable_flash_mla: Whether to emulate the FlashMLA block-size override. + max_seq_len: Effective sequence length reported by the model engine. """ self.attn_runtime_features = attn_runtime_features - self.max_seq_len = 128 + self.max_seq_len = max_seq_len self.max_num_tokens = 128 self.sparse_attention_config = None self.attn_metadata = None self.model = SimpleNamespace( model_config=SimpleNamespace( - enable_flash_mla=False, + enable_flash_mla=enable_flash_mla, is_generation=True, pretrained_config=SimpleNamespace(), quant_config=SimpleNamespace(kv_cache_quant_algo=kv_cache_quant_algo), @@ -146,7 +162,11 @@ def _make_llm_args(): enable_partial_reuse=False, tokens_per_block=32, max_attention_window=None, - mamba_state_cache_interval=1, + mamba_state_config=SimpleNamespace( + periodic_snapshot_interval=256, + additional_snapshot_offsets_from_start=[], + additional_snapshot_offsets_from_end=[], + ), ) scheduler_config = SimpleNamespace( context_chunking_policy=None, @@ -166,6 +186,7 @@ def _make_llm_args(): trust_remote_code=False, mm_encoder_only=False, enable_chunked_prefill=False, + sparse_attention_config=None, attn_backend="TRTLLM", speculative_config=None, disable_overlap_scheduler=True, @@ -184,8 +205,19 @@ def _make_llm_args(): ) -def _run_create_py_executor(monkeypatch, *, sm_version, kv_cache_quant_algo): - """Execute create_py_executor with mocked dependencies and return cache reuse flags. +def _run_create_py_executor( + monkeypatch, + *, + sm_version, + kv_cache_quant_algo, + cache_transceiver_config=None, + enable_flash_mla=False, + model_max_seq_len=128, + enable_chunked_prefill=False, + is_hybrid_linear_model=False, + ctx_chunk_configs=None, +): + """Execute create_py_executor with mocked dependencies and return MLA runtime flags. Mocks all external dependencies (model engine, resource managers, etc.) to isolate executor creation logic and verify that KV cache reuse configuration is synchronized @@ -195,11 +227,20 @@ def _run_create_py_executor(monkeypatch, *, sm_version, kv_cache_quant_algo): monkeypatch: pytest fixture for mocking. sm_version: CUDA SM version to simulate (e.g., 89, 90). kv_cache_quant_algo: Quantization algorithm to use (e.g., NO_QUANT, INT8). + cache_transceiver_config: Optional transceiver configuration to mutate. + enable_flash_mla: Whether to emulate the FlashMLA block-size override. + model_max_seq_len: Effective sequence length reported by the model engine. + enable_chunked_prefill: Whether to request MLA chunked prefill support. + is_hybrid_linear_model: Whether to emulate a hybrid linear model. + ctx_chunk_configs: Optional list that receives the executor chunk config. Returns: - Tuple of (kv_cache_reuse_flag, runtime_cache_reuse_flag) from created executor. + Tuple of (kv_cache_reuse_flag, runtime_cache_reuse_flag, + runtime_chunked_prefill_flag) from created executor. """ llm_args = _make_llm_args() + llm_args.cache_transceiver_config = cache_transceiver_config + llm_args.enable_chunked_prefill = enable_chunked_prefill fake_mapping = SimpleNamespace( rank=0, tp_size=1, @@ -230,7 +271,11 @@ def _run_create_py_executor(monkeypatch, *, sm_version, kv_cache_quant_algo): monkeypatch.setattr(py_executor_creator, "_adjust_torch_mem_fraction", lambda: None) monkeypatch.setattr(py_executor_creator, "log_memory_usage", lambda *args, **kwargs: None) monkeypatch.setattr(py_executor_creator, "is_mla", lambda _: True) - monkeypatch.setattr(py_executor_creator, "is_hybrid_linear", lambda _: False) + monkeypatch.setattr( + py_executor_creator, + "is_hybrid_linear", + lambda _: is_hybrid_linear_model, + ) monkeypatch.setattr(py_executor_creator, "get_sm_version", lambda: sm_version) monkeypatch.setattr(py_executor_creator, "KvCacheCreator", _DummyKvCacheCreator) @@ -248,11 +293,15 @@ def _create_model_engine(**kwargs): return _DummyModelEngine( attn_runtime_features=kwargs["attn_runtime_features"], kv_cache_quant_algo=kv_cache_quant_algo, + enable_flash_mla=enable_flash_mla, + max_seq_len=model_max_seq_len, ) monkeypatch.setattr(py_executor_creator, "PyTorchModelEngine", _create_model_engine) def _create_py_executor_instance(**kwargs): + if ctx_chunk_configs is not None: + ctx_chunk_configs.append(kwargs["ctx_chunk_config"]) return _DummyPyExecutor( resources=kwargs["resources"], model_engine=kwargs["model_engine"], @@ -275,6 +324,7 @@ def _create_py_executor_instance(**kwargs): return ( kv_cache_manager.enable_block_reuse, py_executor.model_engine.attn_runtime_features.cache_reuse, + py_executor.model_engine.attn_runtime_features.chunked_prefill, ) @@ -287,7 +337,7 @@ def test_mla_unsupported_sm_fallback_syncs_cache_reuse(monkeypatch): This test ensures invariant synchronization is maintained across the fallback. """ - kv_cache_reuse, runtime_cache_reuse = _run_create_py_executor( + kv_cache_reuse, runtime_cache_reuse, _ = _run_create_py_executor( monkeypatch, sm_version=89, kv_cache_quant_algo=QuantAlgo.NO_QUANT, @@ -307,7 +357,7 @@ def test_mla_unsupported_kv_quant_fallback_syncs_cache_reuse(monkeypatch): This test ensures invariant synchronization is maintained across the fallback. """ - kv_cache_reuse, runtime_cache_reuse = _run_create_py_executor( + kv_cache_reuse, runtime_cache_reuse, _ = _run_create_py_executor( monkeypatch, sm_version=90, kv_cache_quant_algo=QuantAlgo.INT8, @@ -317,21 +367,106 @@ def test_mla_unsupported_kv_quant_fallback_syncs_cache_reuse(monkeypatch): assert runtime_cache_reuse is False -def test_mla_supported_configuration_preserves_cache_reuse(monkeypatch): - """Verify MLA supported configuration preserves cache reuse in both config and runtime. +@pytest.mark.parametrize("sm_version", _MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS) +def test_mla_supported_configuration_preserves_cache_reuse(monkeypatch, sm_version): + """Verify every supported MLA SM preserves cache reuse in both config and runtime. - When both SM version (90) and KV quantization (NO_QUANT) are supported for MLA, - no fallback occurs and: + When the SM version is in the MLA allowlist and KV quantization is + NO_QUANT, no unsupported-SM fallback should occur and: - kv_cache_config.enable_block_reuse remains True - model_engine.attn_runtime_features.cache_reuse remains True - - This positive test ensures the default path does not regress. """ - kv_cache_reuse, runtime_cache_reuse = _run_create_py_executor( + kv_cache_reuse, runtime_cache_reuse, _ = _run_create_py_executor( + monkeypatch, + sm_version=sm_version, + kv_cache_quant_algo=QuantAlgo.NO_QUANT, + ) + + assert kv_cache_reuse is True + assert runtime_cache_reuse is True + + +def test_default_transceiver_buffer_rounds_up_to_tokens_per_block(monkeypatch): + config = CacheTransceiverConfig() + + _run_create_py_executor( + monkeypatch, + sm_version=90, + kv_cache_quant_algo=QuantAlgo.NO_QUANT, + cache_transceiver_config=config, + model_max_seq_len=130, + ) + + assert config.max_tokens_in_buffer == 160 + + +def test_default_transceiver_buffer_uses_flash_mla_block_size(monkeypatch): + config = CacheTransceiverConfig() + + _run_create_py_executor( + monkeypatch, + sm_version=90, + kv_cache_quant_algo=QuantAlgo.NO_QUANT, + cache_transceiver_config=config, + enable_flash_mla=True, + model_max_seq_len=130, + ) + + assert config.max_tokens_in_buffer == 192 + + +def test_explicit_transceiver_buffer_size_is_preserved(monkeypatch): + config = CacheTransceiverConfig(max_tokens_in_buffer=256) + + _run_create_py_executor( + monkeypatch, + sm_version=90, + kv_cache_quant_algo=QuantAlgo.NO_QUANT, + cache_transceiver_config=config, + enable_flash_mla=True, + model_max_seq_len=130, + ) + + assert config.max_tokens_in_buffer == 256 + + +def test_hybrid_force_chunk_uses_block_alignment_unit(monkeypatch): + ctx_chunk_configs = [] + _run_create_py_executor( monkeypatch, sm_version=90, kv_cache_quant_algo=QuantAlgo.NO_QUANT, + is_hybrid_linear_model=True, + ctx_chunk_configs=ctx_chunk_configs, + ) + + assert ctx_chunk_configs == [(ContextChunkingPolicy.FORCE_CHUNK, 32)] + + +@pytest.mark.parametrize("sm_version", _MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS) +def test_mla_supported_configuration_preserves_chunked_prefill(monkeypatch, sm_version): + """Verify every supported MLA SM preserves chunked prefill when requested.""" + _, _, runtime_chunked_prefill = _run_create_py_executor( + monkeypatch, + sm_version=sm_version, + kv_cache_quant_algo=QuantAlgo.NO_QUANT, + enable_chunked_prefill=True, + ) + + assert runtime_chunked_prefill is True + + +def test_mla_sm121_fallback_preserves_cache_reuse_and_disables_chunked_prefill( + monkeypatch, +): + """Verify SM121 keeps cache reuse while disabling unsupported chunked prefill.""" + kv_cache_reuse, runtime_cache_reuse, runtime_chunked_prefill = _run_create_py_executor( + monkeypatch, + sm_version=121, + kv_cache_quant_algo=QuantAlgo.NO_QUANT, + enable_chunked_prefill=True, ) assert kv_cache_reuse is True assert runtime_cache_reuse is True + assert runtime_chunked_prefill is False diff --git a/tests/unittest/_torch/executor/test_py_scheduler.py b/tests/unittest/_torch/executor/test_py_scheduler.py index 8776295bbda9..81a813a43796 100644 --- a/tests/unittest/_torch/executor/test_py_scheduler.py +++ b/tests/unittest/_torch/executor/test_py_scheduler.py @@ -1478,9 +1478,9 @@ def test_draft_tokens_no_chunking_discard_none(self): class TestForceChunkPolicy: """ Tests for FORCE_CHUNK chunking policy in PyMicroBatchScheduler. - FORCE_CHUNK always chunks every context request to at most chunk_unit_size - tokens per scheduling step, regardless of whether the full context would fit - in the budget. + FORCE_CHUNK advances context requests to expected snapshot points. Without + a remaining snapshot point, it avoids artificial boundaries and consumes + the full remaining context unless the scheduling budget requires chunking. Aligned with C++ ForceChunkTest in microBatchSchedulerTest.cpp. """ @@ -1513,12 +1513,12 @@ def _expect_positions(requests, expected, label=""): ) # --- Direct _set_ctx_requests_chunk_size tests --- - # C++ ref: ForceChunkTest::Basic through CapacityAcrossIterations + # C++ ref: ForceChunkTest direct chunk-size tests. - def test_basic(self): + def test_without_snapshot_points_uses_remaining_context(self): """ - A single request with prompt_len > chunk_unit_size is chunked to unit_size. - C++ ref: ForceChunkTest.Basic + A request without snapshot points is not split at chunk_unit_size. + C++ ref: ForceChunkTest.NoSnapshotPointsUsesRemainingContext """ config = ContextChunkingConfig(ChunkingPolicy.FORCE_CHUNK, chunk_unit_size=10) scheduler = PyMicroBatchScheduler( @@ -1526,11 +1526,11 @@ def test_basic(self): ) reqs = [make_context_request(0, prompt_len=30)] scheduler._set_ctx_requests_chunk_size(reqs, None) - assert reqs[0].context_chunk_size == 10 + assert reqs[0].context_chunk_size == 30 def test_prompt_smaller_than_unit(self): """ - When prompt_len < chunk_unit_size, chunk_size = prompt_len (min). + Without snapshot points, a short prompt is consumed in full. C++ ref: ForceChunkTest.PromptSmallerThanUnit """ config = ContextChunkingConfig(ChunkingPolicy.FORCE_CHUNK, chunk_unit_size=20) @@ -1543,7 +1543,7 @@ def test_prompt_smaller_than_unit(self): def test_exact_unit_size(self): """ - When prompt_len == chunk_unit_size, chunk_size = prompt_len. + Without snapshot points, an exact-unit prompt is consumed in full. C++ ref: ForceChunkTest.ExactUnitSize """ config = ContextChunkingConfig(ChunkingPolicy.FORCE_CHUNK, chunk_unit_size=10) @@ -1556,7 +1556,8 @@ def test_exact_unit_size(self): def test_multiple_requests(self): """ - Each request independently gets min(remaining, unit_size). + Requests without snapshot points independently consume their remaining + contexts. C++ ref: ForceChunkTest.MultipleRequests """ config = ContextChunkingConfig(ChunkingPolicy.FORCE_CHUNK, chunk_unit_size=10) @@ -1569,13 +1570,14 @@ def test_multiple_requests(self): make_context_request(2, prompt_len=5), ] scheduler._set_ctx_requests_chunk_size(reqs, None) - assert reqs[0].context_chunk_size == 10 - assert reqs[1].context_chunk_size == 10 - assert reqs[2].context_chunk_size == 5 # min(5, 10) + assert reqs[0].context_chunk_size == 25 + assert reqs[1].context_chunk_size == 15 + assert reqs[2].context_chunk_size == 5 def test_capacity_limits(self): """ - When capacity is limited, later requests get chunk_size=0. + Budget truncation is aligned to chunk_unit_size; later requests with + less than one unit available are delayed. C++ ref: ForceChunkTest.CapacityLimits """ config = ContextChunkingConfig(ChunkingPolicy.FORCE_CHUNK, chunk_unit_size=10) @@ -1587,13 +1589,13 @@ def test_capacity_limits(self): make_context_request(1, prompt_len=30), ] scheduler._set_ctx_requests_chunk_size(reqs, capacity=15) - # req0 gets 10, req1 would push total to 20 > 15 → 0 + # req0 is budget-truncated to 10; only 5 remain, so req1 gets 0. assert reqs[0].context_chunk_size == 10 assert reqs[1].context_chunk_size == 0 def test_capacity_exact_fit(self): """ - When capacity exactly accommodates all chunks. + Capacity can exactly accommodate two requested snapshot chunks. C++ ref: ForceChunkTest.CapacityExactFit """ config = ContextChunkingConfig(ChunkingPolicy.FORCE_CHUNK, chunk_unit_size=10) @@ -1604,17 +1606,53 @@ def test_capacity_exact_fit(self): make_context_request(0, prompt_len=30), make_context_request(1, prompt_len=30), ] + for req in reqs: + req.expect_snapshot_points = [10] scheduler._set_ctx_requests_chunk_size(reqs, capacity=20) assert reqs[0].context_chunk_size == 10 assert reqs[1].context_chunk_size == 10 + def test_expected_snapshot_points(self): + """ + Expected snapshot points are absolute context positions. + C++ ref: ForceChunkTest.ExpectedChunkingPoints + """ + reqs = [make_context_request(0, prompt_len=30)] + reqs[0].expect_snapshot_points = [12, 25] + + self._chunk_iteration(reqs, 10) + self._expect_positions(reqs, [12], "iter 1") + + self._chunk_iteration(reqs, 10) + self._expect_positions(reqs, [25], "iter 2") + + self._chunk_iteration(reqs, 10) + self._expect_positions(reqs, [30], "iter 3") + + def test_capacity_rounds_expected_snapshot_down_to_unit(self): + """ + Capacity truncation rounds expected chunks down to chunk_unit_size. + C++ ref: ForceChunkTest.CapacityRoundsExpectedChunkDownToUnit + """ + config = ContextChunkingConfig(ChunkingPolicy.FORCE_CHUNK, chunk_unit_size=10) + scheduler = PyMicroBatchScheduler( + max_batch_size=64, max_num_tokens=1000, ctx_chunk_config=config + ) + reqs = [make_context_request(0, prompt_len=50)] + reqs[0].expect_snapshot_points = [30] + + scheduler._set_ctx_requests_chunk_size(reqs, capacity=25) + + assert reqs[0].context_chunk_size == 20 + def test_multi_iteration(self): """ - A request with prompt_len=25 and chunk_unit_size=10 processes in 3 - iterations: chunk 1: 10, chunk 2: 10, chunk 3: 5. + Snapshot points at 10 and 20 split a 25-token prompt into three + iterations. C++ ref: ForceChunkTest.MultiIteration """ reqs = [make_context_request(0, prompt_len=25)] + reqs[0].expect_snapshot_points = [10, 20] # Iteration 1 self._chunk_iteration(reqs, 10) @@ -1630,14 +1668,16 @@ def test_multi_iteration(self): def test_multi_request_multi_iteration(self): """ - Two requests with different lengths processed over multiple iterations. - prompt_len={25, 12}, chunk_unit_size=10. + Two requests with different snapshot boundaries are processed over + multiple iterations. C++ ref: ForceChunkTest.MultiRequestMultiIteration """ reqs = [ make_context_request(0, prompt_len=25), make_context_request(1, prompt_len=12), ] + reqs[0].expect_snapshot_points = [10, 20] + reqs[1].expect_snapshot_points = [10] # Iteration 1: both get 10 self._chunk_iteration(reqs, 10) @@ -1653,14 +1693,16 @@ def test_multi_request_multi_iteration(self): def test_capacity_across_iterations(self): """ - With limited capacity, some requests may be delayed to later iterations. - prompt_len={25, 25}, chunk_unit_size=10, capacity=15. + With limited capacity, requests sharing snapshot points may be delayed + to later iterations. C++ ref: ForceChunkTest.CapacityAcrossIterations """ reqs = [ make_context_request(0, prompt_len=25), make_context_request(1, prompt_len=25), ] + for req in reqs: + req.expect_snapshot_points = [10, 20] # Iteration 1: req0=10, req1=0 (10+10=20 > 15) self._chunk_iteration(reqs, 10, capacity=15) @@ -1685,11 +1727,11 @@ def test_capacity_across_iterations(self): # --- Full scheduler.schedule() tests --- # C++ ref: ForceChunkTest::FullSchedulerPath through FullSchedulerWithGeneration - def test_full_scheduler_path(self): + def test_full_scheduler_without_snapshot_points_avoids_chunking(self): """ - FORCE_CHUNK always re-chunks even when all contexts fit within the - token budget. Test via the full schedule() path. - C++ ref: ForceChunkTest.FullSchedulerPath + FORCE_CHUNK does not introduce a boundary when no snapshot is needed + and the full context fits. Test via the full schedule() path. + C++ ref: ForceChunkTest.FullSchedulerWithoutSnapshotPoints """ config = ContextChunkingConfig(ChunkingPolicy.FORCE_CHUNK, chunk_unit_size=10) scheduler = PyMicroBatchScheduler( @@ -1697,9 +1739,8 @@ def test_full_scheduler_path(self): ) req = make_context_request(0, prompt_len=30) _enc, ctx, gen = scheduler.schedule([req], set()) - # Despite budget=100 >> prompt=30, FORCE_CHUNK limits chunk to unit_size=10. assert len(ctx) == 1 - assert ctx[0].context_chunk_size == 10 + assert ctx[0].context_chunk_size == 30 assert len(gen) == 0 def test_full_scheduler_multiple_requests(self): @@ -1720,8 +1761,8 @@ def test_full_scheduler_multiple_requests(self): assert len(ctx) == 3 # Find by request_id since sorting may reorder. chunks = {r.request_id: r.context_chunk_size for r in ctx} - assert chunks[0] == 10 - assert chunks[1] == 10 + assert chunks[0] == 25 + assert chunks[1] == 15 assert chunks[2] == 5 def test_full_scheduler_with_generation(self): @@ -1741,7 +1782,8 @@ def test_full_scheduler_with_generation(self): _enc, ctx, gen = scheduler.schedule(requests, set()) assert len(gen) == 1 assert len(ctx) == 1 - # Budget remaining = 15 - 1 (gen) = 14; chunk = min(30, 10) = 10 + # Budget remaining = 14, so the 30-token context is rounded down to + # one 10-token chunk-unit boundary. assert ctx[0].context_chunk_size == 10 diff --git a/tests/unittest/_torch/executor/test_pytorch_model_engine.py b/tests/unittest/_torch/executor/test_pytorch_model_engine.py index a1583b33b756..4df9e6705748 100644 --- a/tests/unittest/_torch/executor/test_pytorch_model_engine.py +++ b/tests/unittest/_torch/executor/test_pytorch_model_engine.py @@ -9,8 +9,12 @@ import tensorrt_llm from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.models.modeling_multimodal_mixin import \ + MultimodalModelMixin from tensorrt_llm._torch.pyexecutor.connectors.kv_cache_connector import \ KvCacheConnectorWorker +from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import ( + _restore_spec_decode_capture_state, _save_spec_decode_capture_state) from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest from tensorrt_llm._torch.pyexecutor.model_engine import ( PyTorchModelEngine, _build_request_multimodal_input) @@ -72,6 +76,30 @@ def forward(self, *args, **kwargs) -> torch.Tensor: return {"logits": torch.randn((batch_size, 10), device='cuda')} +class DummyMultimodalIndexModel(torch.nn.Module): + + class Config: + vocab_size = 100 + + config = Config() + + @property + def multimodal_token_ids(self) -> torch.Tensor: + return torch.tensor([90, 91], dtype=torch.int32) + + +class DummyLegacyMultimodalIndexModel(MultimodalModelMixin, torch.nn.Module): + + class Config: + vocab_size = 100 + + config = Config() + + @property + def mm_token_ids(self) -> torch.Tensor: + return torch.tensor([90, 91], dtype=torch.int32) + + class DummyModelEngine(PyTorchModelEngine): def __init__(self, llm_args: TorchLlmArgs, dtype: torch.dtype) -> None: @@ -148,6 +176,26 @@ def create_model_engine_and_kvcache(llm_args: TorchLlmArgs = None, class PyTorchModelEngineTestCase(unittest.TestCase): + def test_prepare_multimodal_indices_uses_mixin_token_ids(self) -> None: + engine = object.__new__(PyTorchModelEngine) + engine.model = DummyMultimodalIndexModel() + + text_indices, multimodal_indices = engine._prepare_multimodal_indices( + [1, 90, 2, 91, 3]) + + torch.testing.assert_close(text_indices, torch.tensor([0, 2, 4])) + torch.testing.assert_close(multimodal_indices, torch.tensor([1, 3])) + + def test_prepare_multimodal_indices_uses_legacy_token_ids(self) -> None: + engine = object.__new__(PyTorchModelEngine) + engine.model = DummyLegacyMultimodalIndexModel() + + text_indices, multimodal_indices = engine._prepare_multimodal_indices( + [1, 90, 2, 91, 3]) + + torch.testing.assert_close(text_indices, torch.tensor([0, 2, 4])) + torch.testing.assert_close(multimodal_indices, torch.tensor([1, 3])) + def test_build_request_multimodal_input_skips_when_cache_disabled( self) -> None: request = LlmRequest( @@ -167,6 +215,25 @@ def test_build_request_multimodal_input_skips_when_cache_disabled( self.assertIsNone( _build_request_multimodal_input(request, cache_enabled=False)) + def test_spec_decode_capture_restores_kv_lens_between_warmups(self) -> None: + attn_metadata = Mock() + attn_metadata.num_seqs = 1 + attn_metadata.kv_lens_cuda = torch.tensor([4095], dtype=torch.int32) + + saved_kv_lens_cuda = _save_spec_decode_capture_state( + attn_metadata, enable_spec_decode=True) + + # CUDA graph capture performs two eager warmup forwards. A speculative + # draft loop may advance the static attention metadata during each + # forward, but the next warmup must start from the original input. + for _ in range(2): + attn_metadata.kv_lens_cuda.add_(1) + _restore_spec_decode_capture_state(attn_metadata, + saved_kv_lens_cuda) + self.assertEqual(attn_metadata.kv_lens_cuda.tolist(), [4095]) + + self.assertEqual(attn_metadata.on_update_kv_lens.call_count, 2) + def test_pad_generation_requests(self) -> None: model_engine, kv_cache_manager = create_model_engine_and_kvcache() resource_manager = ResourceManager( @@ -665,12 +732,156 @@ def test_prepare_tp_inputs_with_partial_mrope_segments(self) -> None: torch.testing.assert_close(position_ids, expected, atol=0, rtol=0) self.assertEqual(result["mrope_delta_write_seq_slots"].cpu().tolist(), [0]) + # Read slots are dense w.r.t. the generation batch: the padded dummy + # has no MRoPE metadata, so it resolves to the reserved zero slot + # (max_num_tokens * pp_size) rather than being dropped, which would + # shift every later request onto another request's delta. self.assertEqual(result["mrope_delta_read_seq_slots"].cpu().tolist(), - [0]) + [0, 32]) self.assertNotIn("multimodal_embedding", multimodal_request.py_multimodal_data) kv_cache_manager.shutdown() + def _setup_mrope_engine(self, max_num_tokens: int = 32): + """Build a DummyModelEngine that takes the MRoPE path, plus its KV cache + manager and attention metadata.""" + llm_args = TorchLlmArgs(model="dummy") + model_engine = DummyModelEngine(llm_args, dtype=torch.half) + model_engine.model.model_config.pretrained_config.rope_scaling = { + "type": "mrope" + } + + mapping = Mapping(world_size=1, tp_size=1, rank=0) + kv_cache_config = KvCacheConfig(max_tokens=max_num_tokens) + kv_cache_manager = KVCacheManager( + kv_cache_config, + tensorrt_llm.bindings.internal.batch_manager.CacheType.SELF, + num_layers=1, + num_kv_heads=16, + head_dim=16, + tokens_per_block=1, + max_seq_len=max_num_tokens, + max_batch_size=4, + mapping=mapping, + dtype=tensorrt_llm.bindings.DataType.HALF, + ) + attn_metadata = AttentionMetadata(max_num_requests=4, + max_num_tokens=max_num_tokens, + kv_cache_manager=kv_cache_manager) + attn_metadata.is_cuda_graph = False + + model_engine.max_num_tokens = max_num_tokens + model_engine.input_ids_cuda = torch.zeros(max_num_tokens, + dtype=torch.int32, + device='cuda') + model_engine.position_ids_cuda = torch.zeros(max_num_tokens, + dtype=torch.int32, + device='cuda') + model_engine.mrope_position_ids_cuda = torch.zeros( + (3, 1, max_num_tokens), dtype=torch.int32, device='cuda') + model_engine.previous_batch_indices_cuda = torch.zeros( + max_num_tokens, dtype=torch.int32, device='cuda') + return model_engine, kv_cache_manager, attn_metadata + + @staticmethod + def _make_mrope_gen_request(num_tokens: int, req_id: int, seq_slot: int, + delta): + """Generation request carrying an MRoPE delta when `delta` is not None, + and no multimodal data at all otherwise (a text-only prompt).""" + request = _create_request(num_tokens, req_id) + request.py_prompt_len = num_tokens + request.py_batch_idx = None + request.py_seq_slot = seq_slot + request.sampling_config.beam_width = 1 + if delta is None: + request.py_multimodal_data = {} + else: + request.py_multimodal_data = { + "mrope_config": { + "mrope_position_deltas": + torch.tensor([[delta]], dtype=torch.int32) + }, + } + return request + + def test_prepare_tp_inputs_mixed_text_only_keeps_mrope_deltas_dense( + self) -> None: + """A text-only request between two multimodal ones must not compact the + MRoPE delta read slots. + + The attention kernel indexes `mrope_position_deltas` by generation batch + index, so a list that skips the text-only request would hand request 2's + delta to the text-only request and read out of bounds for request 2. + """ + model_engine, kv_cache_manager, attn_metadata = self._setup_mrope_engine( + ) + + # (num_tokens, seq_slot, delta); the middle request is text-only. + requests = [ + self._make_mrope_gen_request(4, 1, 0, 10), + self._make_mrope_gen_request(5, 2, 1, None), + self._make_mrope_gen_request(6, 3, 2, 20), + ] + + scheduled_requests = ScheduledRequests() + scheduled_requests.context_requests_last_chunk = [] + scheduled_requests.generation_requests = requests + + result, _ = model_engine._prepare_tp_inputs( + scheduled_requests=scheduled_requests, + kv_cache_manager=kv_cache_manager, + attn_metadata=attn_metadata) + + # One entry per generation request, in batch order. Slot 32 is the + # reserved zero slot (max_num_tokens * pp_size) standing in for the + # text-only request's zero delta. + self.assertEqual(result["mrope_delta_read_seq_slots"].cpu().tolist(), + [0, 32, 2]) + # Only the two multimodal requests seed the seq-slot delta cache. + self.assertEqual(result["mrope_delta_write_seq_slots"].cpu().tolist(), + [0, 2]) + + # past_seen_token_num is num_tokens - 1, offset by the request's delta; + # the text-only request keeps the plain scalar position on all 3 axes. + position_ids = result["position_ids"] + self.assertEqual(tuple(position_ids.shape), (3, 1, 3)) + expected = torch.tensor([[[13, 4, 25]]] * 3, + dtype=torch.int32, + device='cuda') + torch.testing.assert_close(position_ids, expected, atol=0, rtol=0) + kv_cache_manager.shutdown() + + def test_prepare_tp_inputs_all_text_only_drops_mrope_deltas(self) -> None: + """A generation batch with no MRoPE metadata at all emits no delta + tensors, so the steady-state generation fast path stays reachable.""" + model_engine, kv_cache_manager, attn_metadata = self._setup_mrope_engine( + ) + + scheduled_requests = ScheduledRequests() + scheduled_requests.context_requests_last_chunk = [] + scheduled_requests.generation_requests = [ + self._make_mrope_gen_request(4, 1, 0, None), + self._make_mrope_gen_request(6, 2, 1, None), + ] + + result, _ = model_engine._prepare_tp_inputs( + scheduled_requests=scheduled_requests, + kv_cache_manager=kv_cache_manager, + attn_metadata=attn_metadata) + + # An all-zero delta vector is identical to passing no deltas at all. + self.assertNotIn("mrope_delta_read_seq_slots", result) + self.assertNotIn("mrope_delta_write_seq_slots", result) + + # MRoPE models keep the (3,1,N) layout even with no mrope work. + position_ids = result["position_ids"] + self.assertEqual(tuple(position_ids.shape), (3, 1, 2)) + expected = torch.tensor([[[3, 5]]] * 3, + dtype=torch.int32, + device='cuda') + torch.testing.assert_close(position_ids, expected, atol=0, rtol=0) + kv_cache_manager.shutdown() + def test_kv_cache_manager_with_execution_stream(self): """Test that KVCacheManager uses the provided execution_stream. """ diff --git a/tests/unittest/_torch/executor/test_request_utils.py b/tests/unittest/_torch/executor/test_request_utils.py index f42375c5e892..f2d6d6bc605b 100644 --- a/tests/unittest/_torch/executor/test_request_utils.py +++ b/tests/unittest/_torch/executor/test_request_utils.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + """Tests for request_utils.py functions. This module tests: @@ -17,6 +20,7 @@ attach_py_objects_to_requests, can_process_attention_dp_request, derive_attention_dp_per_rank_request_cap, + executor_request_to_llm_request, get_from_waiting_queue, merge_helix_requests, merge_requests, @@ -95,6 +99,40 @@ def test_request_broadcaster_requires_conversation_params_attr(): RequestBroadcaster._collect_py_objects(None, source_items) +def test_executor_request_to_llm_request_adopts_context_phase_draft_tokens() -> None: + request_id = 42 + first_gen_tokens = [100] + draft_tokens = [101, 102, 103] + context_phase_params = trtllm.ContextPhaseParams( + first_gen_tokens, + request_id, + None, + draft_tokens, + None, + None, + ) + executor_request = trtllm.Request( + input_token_ids=[1, 2, 3], + max_tokens=10, + type=trtllm.RequestType.REQUEST_TYPE_GENERATION_ONLY, + context_phase_params=context_phase_params, + ) + + llm_request = executor_request_to_llm_request( + request_id, + executor_request, + child_req_ids=[], + exclude_last_generation_logits=False, + ) + + assert llm_request.is_generation_only_request() + assert llm_request.has_draft_tokens() + assert llm_request.num_draft_tokens == len(draft_tokens) + assert llm_request.draft_tokens == draft_tokens + assert llm_request.py_draft_tokens == draft_tokens + assert llm_request.context_phase_params.draft_tokens == draft_tokens + + def test_merge_helix_requests_with_padding(): """Test merge_helix_requests with basic valid input.""" diff --git a/tests/unittest/_torch/kv_cache_compression/conftest.py b/tests/unittest/_torch/kv_cache_compression/conftest.py new file mode 100644 index 000000000000..4e475f35a674 --- /dev/null +++ b/tests/unittest/_torch/kv_cache_compression/conftest.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared harness for the KV-cache compaction tests.""" + +import torch + + +def encode_block_offsets(page_ids: torch.Tensor) -> torch.Tensor: + """Native V2 [pool, request, K/V, block] layout: K = 2*page, V = K+1.""" + if page_ids.ndim == 2: + page_ids = page_ids.unsqueeze(0) + encoded = torch.empty( + page_ids.shape[0], + page_ids.shape[1], + 2, + page_ids.shape[2], + dtype=torch.int32, + device=page_ids.device, + ) + encoded[:, :, 0] = page_ids.to(torch.int32) * 2 + encoded[:, :, 1] = encoded[:, :, 0] + 1 + return encoded + + +def _write_move_offsets(compaction, offsets, moves_per_request): + cumulative = [0] + for count in moves_per_request: + cumulative.append(cumulative[-1] + count) + # Rows past the cohort are padding and contribute no moves. + cumulative.extend(cumulative[-1:] * (compaction["request_count"] - len(moves_per_request))) + offsets.copy_(torch.tensor(cumulative, dtype=torch.int32), non_blocking=True) + + +def set_protected_tails(compaction, tail_lengths, draft_tail_lengths=None): + """Load per-request protected tails into the caller-owned move offsets.""" + if len(tail_lengths) > compaction["request_count"]: + raise ValueError("the cohort exceeds the compaction request capacity") + if any(tail < 0 or tail > compaction["protected_tail_capacity"] for tail in tail_lengths): + raise ValueError("a protected tail exceeds the configured capacity") + _write_move_offsets( + compaction, + compaction["dense_move_offsets"], + [compaction["decode_keep_count"] + int(tail) for tail in tail_lengths], + ) + if compaction["has_swa"]: + _write_move_offsets( + compaction, + compaction["swa_move_offsets"], + [compaction["swa_window"] + int(tail) for tail in tail_lengths], + ) + if compaction["draft_move_offsets"] is not None: + if draft_tail_lengths is None: + draft_tail_lengths = [0] * len(tail_lengths) + if len(draft_tail_lengths) != len(tail_lengths): + raise ValueError("draft protected tails must match the cohort") + if any( + tail < 0 or tail > compaction["draft_protected_tail_capacity"] + for tail in draft_tail_lengths + ): + raise ValueError("a draft protected tail exceeds the configured capacity") + _write_move_offsets( + compaction, + compaction["draft_move_offsets"], + [compaction["decode_keep_count"] + int(tail) for tail in draft_tail_lengths], + ) + + +def make_ramp_pools( + count, + *, + num_kv_heads=2, + pages=6, + tokens_per_block=32, + head_dim=64, + layer_stride=37, + base=0, + device=None, +): + """bf16 pools with a shifted ``arange % 251`` ramp: every wrong move + lands on a different byte pattern (supported geometry defaults).""" + return [ + ( + ( + torch.arange( + pages * 2 * num_kv_heads * tokens_per_block * head_dim, + dtype=torch.int32, + device=device, + ) + + base + + layer * layer_stride + ) + % 251 + ) + .view(pages, 2, num_kv_heads, tokens_per_block, head_dim) + .to(torch.bfloat16) + for layer in range(count) + ] + + +def build_compaction(**overrides): + """``build_compaction_params`` with the suite's 2-layer defaults: + allocates the caller-owned move-offset rows (capacity cumsum) and SWA + destination bases, and hands the test's pre-settled + ``kept_token_ordinals`` in as the decision rows. Returns the opaque + ``params`` plus a test-side mirror of the caller-owned inputs.""" + from tensorrt_llm._torch.kv_cache_compression.compaction import build_compaction_params + + args = dict( + eviction_mode="union", + dense_layers=[0, 1], + swa_layers=[], + layer_group_representative={0: 0, 1: 1}, + layer_pool_ids=[0, 0], + request_count=2, + decode_keep_count=4, + swa_window=None, + ) + args.update(overrides) + args.pop("eviction_mode") + kept = args.pop("kept_token_ordinals") + request_count = args["request_count"] + keep_count = args["decode_keep_count"] + tail = int(args.get("protected_tail_capacity", 0)) + draft_tail = int(args.get("draft_protected_tail_capacity") or 0) + has_draft = bool(args.get("draft_layers")) + has_swa = bool(args["swa_layers"]) + device = args["layer_pools"][args["dense_layers"][0]].device + swa_window = int(args["swa_window"] or 0) if has_swa else 0 + swa_destination_bases = torch.empty_like(args["prompt_offsets"]) if has_swa else None + + def capacity_offsets(count): + return torch.arange(0, (request_count + 1) * count, count, dtype=torch.int32, device=device) + + args.setdefault("dense_move_offsets", capacity_offsets(keep_count + tail)) + args.setdefault("swa_move_offsets", capacity_offsets(swa_window + tail) if has_swa else None) + if has_draft: + args.setdefault("draft_move_offsets", capacity_offsets(keep_count + draft_tail)) + params_list = [ + build_compaction_params( + dict( + layer_pools=args["layer_pools"], + dense_layers=args["dense_layers"], + swa_layers=args["swa_layers"], + swa_window=args["swa_window"], + layer_pool_ids=args["layer_pool_ids"], + ), + block_offsets=args["kv_block_offsets"], + kept_ordinals=kept.reshape(-1, keep_count), + source_lengths=args["valid_sequence_lengths"], + dense_destination_bases=args["prompt_offsets"], + dense_move_offsets=args["dense_move_offsets"], + protected_tail_capacity=tail, + swa_move_offsets=args["swa_move_offsets"], + swa_destination_bases=swa_destination_bases, + ) + ] + if has_draft: + params_list.append( + build_compaction_params( + dict( + layer_pools=args["draft_layer_pools"], + dense_layers=args["draft_layers"], + swa_layers=[], + layer_pool_ids=args["draft_layer_pool_ids"], + ), + block_offsets=args["draft_kv_block_offsets"], + kept_ordinals=kept.reshape(-1, keep_count), + source_lengths=args["valid_sequence_lengths"], + dense_destination_bases=args["prompt_offsets"], + dense_move_offsets=args["draft_move_offsets"], + protected_tail_capacity=draft_tail, + ) + ) + params = tuple(params_list) + # Opaque plans plus a test-side mirror of the caller-owned construction + # inputs (production binds the same values as manager attributes); the + # standalone helpers here need the move-offset rows and SWA staging back. + return dict( + params=params, + prompt_offsets=args["prompt_offsets"], + request_count=request_count, + decode_keep_count=keep_count, + protected_tail_capacity=tail, + draft_protected_tail_capacity=draft_tail if has_draft else 0, + dense_move_offsets=args["dense_move_offsets"], + swa_move_offsets=args["swa_move_offsets"], + draft_move_offsets=args["draft_move_offsets"] if has_draft else None, + has_swa=has_swa, + swa_window=swa_window, + swa_destination_bases=swa_destination_bases, + swa_rebase_delta=keep_count - swa_window, + ) + + +def run_compaction(compaction): + """Replica of the round's move stage in production order: SWA + destination rebase, then ``compact`` loops the opaque params (each packs + its decision rows into move sources and fires its native moves).""" + from tensorrt_llm._torch.kv_cache_compression.compaction import compact + + if compaction["swa_destination_bases"] is not None: + torch.add( + compaction["prompt_offsets"], + compaction["swa_rebase_delta"], + out=compaction["swa_destination_bases"], + ) + compact(compaction["params"], compaction["request_count"]) diff --git a/tests/unittest/_torch/kv_cache_compression/test_compaction.py b/tests/unittest/_torch/kv_cache_compression/test_compaction.py new file mode 100644 index 000000000000..1e7913016851 --- /dev/null +++ b/tests/unittest/_torch/kv_cache_compression/test_compaction.py @@ -0,0 +1,324 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Physical KV-cache compaction: packed moves, protected tails, SWA windows, +and draft co-compaction, checked byte-exactly against torch oracles.""" + +from types import SimpleNamespace + +import pytest +import torch +from conftest import build_compaction as _build_compaction +from conftest import encode_block_offsets as _encode_block_offsets +from conftest import make_ramp_pools as _make_ramp_pools +from conftest import run_compaction as _run_compaction +from conftest import set_protected_tails as _set_protected_tails + +requires_sm100 = pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0), + reason="KV-cache compaction kernels require SM100", +) + + +def _logical_view(pool: torch.Tensor, pages: torch.Tensor) -> torch.Tensor: + """Gather one request's pages into [K/V, head, token, dim] order.""" + num_kv_heads = int(pool.shape[2]) + head_dim = int(pool.shape[4]) + return pool.index_select(0, pages).permute(1, 2, 0, 3, 4).reshape(2, num_kv_heads, -1, head_dim) + + +@pytest.mark.parametrize("eviction_mode", ["union", "per_head", "per_layer_perhead"]) +def test_eager_compaction_preserves_exact_selected_bytes_and_tail(eviction_mode): + # Supported bf16 geometry; kept ordinals span all three pages so moves + # cross page boundaries. + device = torch.device("cuda", torch.cuda.current_device()) + request_count = 2 + num_layers = 2 + num_kv_heads = 2 + # Mixed prompt lengths prove per-request destination rebasing. + prompt_lens = [2, 5] + decode_keep_count = 4 + seq_len = 80 + tokens_per_block = 32 + pages_per_request = 3 + head_dim = 64 + protected_tails = [2, 1] + page_tables = torch.tensor([[0, 1, 2], [3, 4, 5]], dtype=torch.int32, device=device) + initial_pools = _make_ramp_pools(num_layers, device=device) + pools = [pool.clone() for pool in initial_pools] + + # Decode-only kept ordinals holding absolute positions. + union_decode = torch.tensor( + [[16, 32, 56, 72], [24, 40, 48, 64]], dtype=torch.int64, device=device + ) + if eviction_mode == "union": + keep = union_decode + selection_rows = 1 + else: + selection_rows = num_kv_heads if eviction_mode == "per_head" else num_layers * num_kv_heads + keep = torch.empty( + request_count, + selection_rows, + decode_keep_count, + dtype=torch.int64, + device=device, + ) + for request in range(request_count): + for row in range(selection_rows): + keep[request, row] = torch.tensor( + sorted( + { + prompt_lens[request] + ((request + row + offset * 2) % 8) * 8 + for offset in range(decode_keep_count) + } + ), + dtype=torch.int64, + device=device, + ) + + compaction = _build_compaction( + eviction_mode=eviction_mode, + layer_pools=pools, + kept_token_ordinals=keep.to(torch.int32), + valid_sequence_lengths=torch.tensor([seq_len, seq_len], dtype=torch.int32, device=device), + kv_block_offsets=_encode_block_offsets(page_tables.unsqueeze(0)), + prompt_offsets=torch.tensor(prompt_lens, dtype=torch.int32, device=device), + protected_tail_capacity=max(protected_tails), + ) + _set_protected_tails(compaction, protected_tails) + # Production settles the kept ordinals into the contract's decision + # rows; with pre-settled ordinals the pack launch inside compact() is + # its exact analog. + _run_compaction(compaction) + torch.cuda.synchronize(device) + + for layer, (before_pool, after_pool) in enumerate(zip(initial_pools, pools)): + for request in range(request_count): + prompt_len = prompt_lens[request] + pages = page_tables[request].to(torch.long) + before = ( + before_pool[pages] + .permute(1, 2, 0, 3, 4) + .reshape(2, num_kv_heads, pages_per_request * tokens_per_block, head_dim) + ) + after = after_pool[pages].permute(1, 2, 0, 3, 4).reshape_as(before) + assert torch.equal(after[:, :, :prompt_len], before[:, :, :prompt_len]) + for head in range(num_kv_heads): + if eviction_mode == "union": + selected = keep[request] + elif eviction_mode == "per_head": + selected = keep[request, head] + else: + selected = keep[request, layer * num_kv_heads + head] + tail = torch.arange( + seq_len, + seq_len + protected_tails[request], + dtype=torch.int64, + device=device, + ) + source = torch.cat((selected, tail)) + destination = torch.arange( + prompt_len, + prompt_len + source.numel(), + dtype=torch.int64, + device=device, + ) + assert torch.equal( + after[:, head].index_select(1, destination), + before[:, head].index_select(1, source), + ) + + +@requires_sm100 +def test_eager_compaction_rebases_masked_swa_window_and_tail(): + # Supported bf16 geometry; dense and SWA moves stay page-crossing. + device = torch.device("cuda", torch.cuda.current_device()) + dense_tables = torch.tensor([[2, 0, 1], [5, 3, 4]], dtype=torch.int32, device=device) + swa_tables = torch.tensor([[1, 2, 0], [4, 5, 3]], dtype=torch.int32, device=device) + initial_pools = _make_ramp_pools(2, num_kv_heads=1, device=device) + pools = [pool.clone() for pool in initial_pools] + # Decode-only kept ordinals holding absolute positions past the prompt. + keep = torch.tensor( + [[16, 32, 40, 56], [16, 24, 40, 48]], + dtype=torch.int64, + device=device, + ) + valid_seq_lens = torch.tensor([64, 56], dtype=torch.int32, device=device) + protected_tails = [2, 1] + compaction = _build_compaction( + layer_pools=pools, + dense_layers=[0], + swa_layers=[1], + layer_group_representative={0: 0}, + # Dense layer 0 stages in plane 0, the SWA layer in its own plane 1. + layer_pool_ids=[0, 1], + kept_token_ordinals=keep.to(torch.int32), + valid_sequence_lengths=valid_seq_lens, + kv_block_offsets=_encode_block_offsets(torch.stack((dense_tables, swa_tables))), + prompt_offsets=torch.tensor([2, 2], dtype=torch.int32, device=device), + swa_window=2, + protected_tail_capacity=max(protected_tails), + ) + _set_protected_tails(compaction, protected_tails) + _run_compaction(compaction) + torch.cuda.synchronize(device) + + for request, (valid_seq_len, tail_length) in enumerate( + zip(valid_seq_lens.tolist(), protected_tails) + ): + dense_pages = dense_tables[request].to(torch.long) + swa_pages = swa_tables[request].to(torch.long) + dense_before = initial_pools[0][dense_pages].permute(1, 2, 0, 3, 4).reshape(2, 1, -1, 64) + dense_after = pools[0][dense_pages].permute(1, 2, 0, 3, 4).reshape_as(dense_before) + swa_before = initial_pools[1][swa_pages].permute(1, 2, 0, 3, 4).reshape(2, 1, -1, 64) + swa_after = pools[1][swa_pages].permute(1, 2, 0, 3, 4).reshape_as(swa_before) + tail = torch.arange( + valid_seq_len, + valid_seq_len + tail_length, + dtype=torch.int64, + device=device, + ) + dense_source = torch.cat((keep[request], tail)) + dense_destination = torch.arange( + 2, 2 + dense_source.numel(), dtype=torch.int64, device=device + ) + swa_source = torch.arange( + valid_seq_len - 2, + valid_seq_len + tail_length, + dtype=torch.int64, + device=device, + ) + swa_destination = torch.arange(4, 4 + swa_source.numel(), dtype=torch.int64, device=device) + assert torch.equal(dense_after[:, :, :2], dense_before[:, :, :2]) + assert torch.equal(swa_after[:, :, :2], swa_before[:, :, :2]) + assert torch.equal( + dense_after.index_select(2, dense_destination), + dense_before.index_select(2, dense_source), + ) + assert torch.equal( + swa_after.index_select(2, swa_destination), + swa_before.index_select(2, swa_source), + ) + + +def _launched_draft_compaction(draft_protected_tails): + """Target and draft pools with distinct head counts (supported bf16 + geometry, mod-251 ramp payload), compacted in one round.""" + device = torch.device("cuda", torch.cuda.current_device()) + request_count = 2 + prompt_len = 2 + target_protected_tails = [2, 1] + valid_seq_lens = [10, 9] + + target_tables = torch.tensor([[0, 1, 2], [3, 4, 5]], dtype=torch.int32, device=device) + draft_tables = torch.tensor([[1, 0, 2], [5, 4, 3]], dtype=torch.int32, device=device) + target_pools = _make_ramp_pools(2, num_kv_heads=2, device=device) + draft_pool = _make_ramp_pools(1, num_kv_heads=4, base=149, device=device)[0] + assert target_pools[0].shape[2] != draft_pool.shape[2] + initial_target = [pool.clone() for pool in target_pools] + initial_draft = draft_pool.clone() + + keep = torch.tensor([[2, 4, 7, 9], [3, 5, 6, 8]], dtype=torch.int64, device=device) + + compaction = _build_compaction( + layer_pools=target_pools, + layer_pool_ids=[0, 0], + kept_token_ordinals=keep.to(torch.int32), + valid_sequence_lengths=torch.tensor(valid_seq_lens, dtype=torch.int32, device=device), + kv_block_offsets=_encode_block_offsets(target_tables), + prompt_offsets=torch.full((request_count,), prompt_len, dtype=torch.int32, device=device), + protected_tail_capacity=max(target_protected_tails), + draft_layer_pools=[draft_pool], + draft_layers=[0], + draft_layer_group_representative={0: 0}, + draft_layer_pool_ids=[0], + draft_protected_tail_capacity=max(draft_protected_tails), + draft_kv_block_offsets=_encode_block_offsets(draft_tables), + ) + _set_protected_tails(compaction, target_protected_tails, draft_protected_tails) + _run_compaction(compaction) + torch.cuda.synchronize(device) + + return SimpleNamespace( + device=device, + request_count=request_count, + prompt_len=prompt_len, + keep=keep, + valid_seq_lens=valid_seq_lens, + target_protected_tails=target_protected_tails, + draft_protected_tails=draft_protected_tails, + target_tables=target_tables, + draft_tables=draft_tables, + target_pools=target_pools, + draft_pool=draft_pool, + initial_target=initial_target, + initial_draft=initial_draft, + compaction=compaction, + ) + + +def test_draft_moves_and_pack_match_keep_broadcast_and_tail_oracle(): + # Ragged draft tails [1, 2] against target tails [2, 1]: one request's + # draft tail below and one above its target, subsuming the uniform row. + built = _launched_draft_compaction(draft_protected_tails=[1, 2]) + device = built.device + prompt_len = built.prompt_len + + expected_offsets = [0] + for request in range(built.request_count): + valid = built.valid_seq_lens[request] + # Target dense layers compact the union keep set plus the target tail. + target_pages = built.target_tables[request].to(torch.long) + target_tail = torch.arange( + valid, + valid + built.target_protected_tails[request], + dtype=torch.int64, + device=device, + ) + target_source = torch.cat((built.keep[request], target_tail)) + target_destination = torch.arange( + prompt_len, + prompt_len + target_source.numel(), + dtype=torch.int64, + device=device, + ) + for before_pool, after_pool in zip(built.initial_target, built.target_pools): + before = _logical_view(before_pool, target_pages) + after = _logical_view(after_pool, target_pages) + assert torch.equal(after[:, :, :prompt_len], before[:, :, :prompt_len]) + assert torch.equal( + after.index_select(2, target_destination), + before.index_select(2, target_source), + ) + + # Same kept ordinals through the draft's OWN table/heads/tail. + draft_pages = built.draft_tables[request].to(torch.long) + draft_tail = torch.arange( + valid, + valid + built.draft_protected_tails[request], + dtype=torch.int64, + device=device, + ) + draft_source = torch.cat((built.keep[request], draft_tail)) + draft_destination = torch.arange( + prompt_len, + prompt_len + draft_source.numel(), + dtype=torch.int64, + device=device, + ) + before = _logical_view(built.initial_draft, draft_pages) + after = _logical_view(built.draft_pool, draft_pages) + assert torch.equal(after[:, :, :prompt_len], before[:, :, :prompt_len]) + for head in range(int(built.draft_pool.shape[2])): + assert torch.equal( + after[:, head].index_select(1, draft_destination), + before[:, head].index_select(1, draft_source), + ) + + expected_offsets.append(expected_offsets[-1] + int(draft_source.numel())) + + # The test-owned draft move-offset row must match the broadcast-plus-tail + # oracle; the packed move sources themselves are covered byte-exactly by + # the pool assertions above (the ramp payload makes every wrong move land + # on different bytes) and by the pack-kernel oracle suite. + assert built.compaction["draft_move_offsets"].cpu().tolist() == expected_offsets diff --git a/tests/unittest/_torch/lora/test_moe_lora_grouped_gemm.py b/tests/unittest/_torch/lora/test_moe_lora_grouped_gemm.py index 97037f7a68be..69663873bf75 100644 --- a/tests/unittest/_torch/lora/test_moe_lora_grouped_gemm.py +++ b/tests/unittest/_torch/lora/test_moe_lora_grouped_gemm.py @@ -19,14 +19,22 @@ on-device fed by captured H2D copies of the stable pinned slot tables, so reassigning a slot's adapter in place is reflected on replay WITHOUT re-capture (mirroring attention LoRA and the normal decode loop). + 5. CUDA-graph workspace reservation for both base-weight modes MoE LoRA + supports: unquantized bf16 and per-tensor FP8 (qdq). They require a CUDA GPU and the built `trtllm::fused_moe` op. """ +from types import SimpleNamespace + import pytest import torch +from tensorrt_llm._torch.modules.fused_moe.fused_moe_cutlass import CutlassFusedMoE from tensorrt_llm._torch.peft.lora.moe_layout import make_per_expert_lora, reference_swiglu_moe_lora +from tensorrt_llm._torch.utils import ActivationType +from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig +from tests.unittest.utils.util import skip_pre_ada _TRTLLM_AVAILABLE = hasattr(torch.ops, "trtllm") and hasattr(torch.ops.trtllm, "fused_moe") @@ -146,7 +154,9 @@ def _ptrs(key): ) -def _call_fused_moe(x, w3_w1, w2, topk_ids, topk_scores, output_dtype, lora_kwargs): +def _call_fused_moe( + x, w3_w1, w2, topk_ids, topk_scores, output_dtype, lora_kwargs, quant_scales=None +): common = dict( input=x, token_selected_experts=topk_ids, @@ -156,7 +166,7 @@ def _call_fused_moe(x, w3_w1, w2, topk_ids, topk_scores, output_dtype, lora_kwar fc2_expert_weights=w2, fc2_expert_biases=None, output_dtype=output_dtype, - quant_scales=[], + quant_scales=quant_scales if quant_scales is not None else [], ) common.update(lora_kwargs) return torch.ops.trtllm.fused_moe(**common)[0] @@ -323,7 +333,6 @@ def test_reserve_prevents_growth_across_captures(): (TRTLLM-12507). """ from tensorrt_llm._torch.custom_ops.torch_custom_ops import MoERunner - from tensorrt_llm._torch.utils import ActivationType device = torch.device("cuda") dtype = torch.bfloat16 @@ -406,6 +415,245 @@ def test_reserve_prevents_growth_across_captures(): torch.testing.assert_close(out2, ref2, rtol=_RTOL, atol=_ATOL) +# -- Per-tensor FP8 (qdq) base weights --------------------------------------- +# +# MoE LoRA also runs on per-tensor FP8 base weights: the kernel dequantizes the +# FP8 activations to the bf16/fp16 LoRA compute type before the LoRA GEMM. The +# runtime op then sees FP8 activations and FP8 weights with a bf16 output, which +# is a different FusedMoeRunner cache key than the unquantized bf16 path. + +_FP8_E4M3_MAX = 448.0 + + +def _quant_per_tensor_fp8(t): + """Per-tensor symmetric FP8 (e4m3) quantization. Returns (t_fp8, dequant) + with t ~= t_fp8.float() * dequant.""" + amax = t.detach().abs().max().float().clamp(min=1e-6) + dequant = amax / _FP8_E4M3_MAX + t_fp8 = (t.float() / dequant).clamp(-_FP8_E4M3_MAX, _FP8_E4M3_MAX).to(torch.float8_e4m3fn) + return t_fp8, dequant + + +def _build_fp8_moe_inputs(x, w3_w1, w2): + """Quantize the bf16 base MoE inputs to per-tensor FP8 (qdq). + + Returns (x_fp8, w3_w1_fp8, w2_fp8, quant_scales), where quant_scales holds + the four entries moeOp.cpp::getQuantParams consumes, in order: fc1_dequant + (per-expert), fc2_quant (scalar), fc2_dequant (per-expert), fc1_input_dequant + (scalar). fc2_quant is 1.0 because at these shapes the SwiGLU intermediate + stays inside the e4m3 range, so no calibrated activation scale is needed. + """ + + def _per_expert(w): + w_fp8 = torch.empty_like(w, dtype=torch.float8_e4m3fn) + scales = torch.empty(w.shape[0], dtype=torch.float32, device=w.device) + for e in range(w.shape[0]): + w_fp8[e], scales[e] = _quant_per_tensor_fp8(w[e]) + return w_fp8, scales + + x_fp8, input_dequant = _quant_per_tensor_fp8(x) + w3_w1_fp8, w3_w1_scale = _per_expert(w3_w1) + w2_fp8, w2_scale = _per_expert(w2) + quant_scales = [ + (w3_w1_scale * input_dequant).to(torch.float32), + torch.tensor(1.0, dtype=torch.float32, device=x.device), + w2_scale.to(torch.float32), + input_dequant.to(torch.float32).reshape(()), + ] + return x_fp8, w3_w1_fp8, w2_fp8, quant_scales + + +def _make_fp8_qdq_moe_layer(w3_w1_fp8, top_k, dtype=torch.bfloat16): + """A CutlassFusedMoE carrying just the state + reserve_moe_lora_cuda_graph_workspace reads, with a real FP8-qdq QuantConfig + so has_fp8_qdq and has_any_quant report what they do on a loaded layer. The + tests drive the real method rather than restating its key derivation. + """ + layer = CutlassFusedMoE.__new__(CutlassFusedMoE) + layer._moe_lora_enabled = True + layer._weights_created = True # gates the has_* quant properties + layer.quant_config = QuantConfig(quant_algo=QuantAlgo.FP8) + layer.dtype = dtype + layer.w3_w1_weight = w3_w1_fp8 + layer.routing_method = SimpleNamespace(experts_per_token=top_k) + layer.tp_size = 1 + layer.tp_rank = 0 + layer.ep_size = 1 + layer.ep_rank = 0 + layer.cluster_size = 1 + layer.cluster_rank = 0 + layer.use_fused_finalize = True + layer.activation_type = int(ActivationType.Swiglu) + layer.is_gated_activation = True + return layer + + +@requires_cuda_and_op +@skip_pre_ada +def test_fp8_qdq_reserve_resolves_to_runtime_runner(): + """An FP8-qdq layer must reserve on the same cached C++ FusedMoeRunner the + runtime op resolves to. + + The runner cache is keyed by (x_dtype, weight_dtype, output_dtype) plus the + quant flags. On the FP8-qdq path the op sees FP8 activations, because the + layer quantizes x to e4m3 before the call, with a bf16 output, so the + reservation must key on the FP8 activation dtype rather than self.dtype. + Keying on self.dtype leaves both calls succeeding on two different cached + runners, so the assertions inspect the cache instead of the output. + """ + from tensorrt_llm._torch.custom_ops.torch_custom_ops import MoERunner + + device = torch.device("cuda") + dtype = torch.bfloat16 + num_tokens, hidden_size, inter_size = 16, 128, 256 + num_experts, top_k, rank = 4, 2, 8 + max_lora_size = 2 + + x, w3_w1, w2, topk_ids, topk_scores = _build_base_inputs( + num_tokens, hidden_size, inter_size, num_experts, top_k, dtype, device + ) + x_fp8, w3_w1_fp8, w2_fp8, quant_scales = _build_fp8_moe_inputs(x, w3_w1, w2) + + layer = _make_fp8_qdq_moe_layer(w3_w1_fp8, top_k, dtype=dtype) + CutlassFusedMoE.reserve_moe_lora_cuda_graph_workspace(layer, num_tokens, rank, max_lora_size) + + assert len(MoERunner.runner_dict) == 1, ( + f"reservation must build exactly one cached runner; got {list(MoERunner.runner_dict)}" + ) + ((reserved_key, reserved_runner),) = MoERunner.runner_dict.items() + assert reserved_key[:3] == (torch.float8_e4m3fn, torch.float8_e4m3fn, dtype), ( + f"FP8-qdq reservation keyed on {reserved_key[:3]}; expected FP8 activations " + f"and weights with a {dtype} output, matching what the runtime op sees." + ) + + # The runtime op derives its own key from the tensors it is handed, so a + # mismatch caches a second runner. + adapters = _make_adapter_set( + num_experts, rank, hidden_size, inter_size, dtype, device, base_seed=1100 + ) + token_to_slot = torch.zeros(num_tokens, dtype=torch.int32) + slot_kwargs = _slot_kwargs(token_to_slot, [adapters], rank) + out = _call_fused_moe( + x_fp8, + w3_w1_fp8, + w2_fp8, + topk_ids, + topk_scores, + dtype, + dict(slot_kwargs), + quant_scales=quant_scales, + ) + + assert torch.isfinite(out).all() + assert len(MoERunner.runner_dict) == 1, ( + "the FP8-qdq runtime call cached a second FusedMoeRunner, so the " + "reservation pre-sized LoRA scratch on a runner capture never uses: " + f"reserved {reserved_key}, cache now holds {list(MoERunner.runner_dict)}" + ) + assert MoERunner.runner_dict[reserved_key] is reserved_runner, ( + "the cached C++ runner was rebuilt instead of reused, discarding the reserved LoRA scratch" + ) + + +@requires_cuda_and_op +@skip_pre_ada +def test_fp8_qdq_reserve_prevents_growth_across_captures(): + """FP8-qdq counterpart of test_reserve_prevents_growth_across_captures, going + through the layer's own reserve method. + + Reserving the worst case up front, then capturing two graphs at growing slot + counts (1 then 2) on the same cached runner, must not reallocate the LoRA + scratch. Once a capture has been observed the C++ op rejects any growth, so + a reservation that missed this runner fails the second capture outright + instead of corrupting replay. + """ + from tensorrt_llm._torch.custom_ops.torch_custom_ops import MoERunner + + device = torch.device("cuda") + dtype = torch.bfloat16 + num_tokens, hidden_size, inter_size = 16, 128, 256 + num_experts, top_k, rank = 4, 2, 8 + max_lora_size = 2 + + x, w3_w1, w2, topk_ids, topk_scores = _build_base_inputs( + num_tokens, hidden_size, inter_size, num_experts, top_k, dtype, device + ) + x_fp8, w3_w1_fp8, w2_fp8, quant_scales = _build_fp8_moe_inputs(x, w3_w1, w2) + + layer = _make_fp8_qdq_moe_layer(w3_w1_fp8, top_k, dtype=dtype) + CutlassFusedMoE.reserve_moe_lora_cuda_graph_workspace(layer, num_tokens, rank, max_lora_size) + + baseline = _call_fused_moe( + x_fp8, w3_w1_fp8, w2_fp8, topk_ids, topk_scores, dtype, {}, quant_scales=quant_scales + ).clone() + + adapter_a = _make_adapter_set( + num_experts, rank, hidden_size, inter_size, dtype, device, base_seed=1200 + ) + adapter_b = _make_adapter_set( + num_experts, rank, hidden_size, inter_size, dtype, device, base_seed=1300 + ) + + # Capture #1: one active slot. + tts1 = torch.zeros(num_tokens, dtype=torch.int32) + sk1 = _slot_kwargs(tts1, [adapter_a], rank) + graph1, captured1 = _warmup_and_capture( + lambda: _call_fused_moe( + x_fp8, + w3_w1_fp8, + w2_fp8, + topk_ids, + topk_scores, + dtype, + dict(sk1), + quant_scales=quant_scales, + ) + ) + + # Capture #2: two active slots on the same cached runner, with no cache clear + # in between. + tts2 = (torch.arange(num_tokens) % 2).to(torch.int32) + sk2 = _slot_kwargs(tts2, [adapter_a, adapter_b], rank) + graph2, captured2 = _warmup_and_capture( + lambda: _call_fused_moe( + x_fp8, + w3_w1_fp8, + w2_fp8, + topk_ids, + topk_scores, + dtype, + dict(sk2), + quant_scales=quant_scales, + ) + ) + + assert len(MoERunner.runner_dict) == 1, ( + "the FP8-qdq LoRA captures must all share the reserved runner; cache holds " + f"{list(MoERunner.runner_dict)}" + ) + + graph1.replay() + graph2.replay() + torch.cuda.synchronize() + + out1 = captured1.clone() + out2 = captured2.clone() + assert torch.isfinite(out1).all() + assert torch.isfinite(out2).all() + + # Numerics for the FP8 + LoRA math are covered in test_moe_lora_op.py; here + # it is enough that each replayed graph applied its own LoRA routing. + def _mean_abs(p, q): + return (p.float() - q.float()).abs().mean().item() + + assert _mean_abs(out1, baseline) > 1e-3, "captured FP8 graph applied no LoRA delta" + assert _mean_abs(out2, baseline) > 1e-3, "captured FP8 graph applied no LoRA delta" + assert _mean_abs(out1, out2) > 1e-3, ( + "the two captures routed different adapter slots but produced the same " + "output, so the slot tables were not honored on replay" + ) + + def _set_slot_ptrs_inplace(slot_kwargs, adapters, slot_index=0): """Overwrite one slot's (A, B) pointer rows for all three modules in place, preserving the pinned-tensor storage (and thus the data_ptr the captured diff --git a/tests/unittest/_torch/lora/test_moe_lora_model_path.py b/tests/unittest/_torch/lora/test_moe_lora_model_path.py index 852b719d45f3..e69abd2c278f 100644 --- a/tests/unittest/_torch/lora/test_moe_lora_model_path.py +++ b/tests/unittest/_torch/lora/test_moe_lora_model_path.py @@ -11,8 +11,7 @@ that a non-empty lora_params is forwarded: 1. QwenMoE.forward to the routed self.experts call (legacy wrapper). - 2. ConfigurableMoE.forward_impl to scheduler.forward (the default - ENABLE_CONFIGURABLE_MOE=1 path). + 2. ConfigurableMoE.forward_impl to scheduler.forward. 3. ExternalCommMoEScheduler._get_backend_kwargs to the CutlassFusedMoE run_moe kwargs, and not to backends that cannot carry LoRA. """ @@ -74,8 +73,7 @@ def test_qwen_moe_forward_passes_lora_params_to_routed_experts(): def test_configurable_moe_forward_impl_forwards_lora_params_to_scheduler(): """ConfigurableMoE.forward_impl must forward lora_params to the scheduler - so routed-expert MoE LoRA is not dropped on the default - ENABLE_CONFIGURABLE_MOE=1 path.""" + so routed-expert MoE LoRA is not dropped.""" x = torch.randn(4, 8) router_logits = torch.randn(4, 2) diff --git a/tests/unittest/_torch/misc/test_autotuner.py b/tests/unittest/_torch/misc/test_autotuner.py index 898f01415de0..c31e9d04e9c8 100644 --- a/tests/unittest/_torch/misc/test_autotuner.py +++ b/tests/unittest/_torch/misc/test_autotuner.py @@ -1,3 +1,4 @@ +import enum import itertools import json import math @@ -524,6 +525,63 @@ def test_autotuner_tuning_configs(): runner_0([x, w], tactic=deserialized_tactic) +def test_load_cache_skips_non_literal_tactic(): + """Regression: a non-literal tactic repr must be skipped on load, not crash it. + + ``_deserialize_cache_data`` reconstructs tactics with ``ast.literal_eval``, + which raises ``SyntaxError`` on non-literal reprs (e.g. enum tactic reprs, + until #16782 serializes enums by value). It must skip such entries -- once + ``SyntaxError`` was uncaught and had no ``continue``, crashing the load. + """ + import ast + + class _NonLiteralTactic: + + def __repr__(self): + return "<_NonLiteralTactic object nvfp4>" + + poisoned_repr = repr(_NonLiteralTactic()) # non-literal object repr + # Precondition: confirm this repr really does raise SyntaxError. + with pytest.raises(SyntaxError): + ast.literal_eval(poisoned_repr) + + cache = AutoTuner.get().profiling_cache + cache.clear() + good_key = "('op_good', 'R', '0', ((1, 128),))" + bad_key = "('op_bad', 'R', '0', ((2, 128),))" + doc = { + "metadata": cache._serialize_metadata(), + "shared": {}, + "rank_0": { + good_key: { + "runner_id": 0, + "tactic": "7", + "min_time": 0.001 + }, + bad_key: { + "runner_id": 1, + "tactic": poisoned_repr, + "min_time": 0.002 + }, + }, + } + temp_dir = tempfile.TemporaryDirectory() + cache_path = os.path.join(temp_dir.name, "poisoned_cache.json") + with open(cache_path, "w") as f: + json.dump(doc, f) + + # Must not raise (previously raised SyntaxError out of load_cache). + cache.load_cache(cache_path, rank=0) + + # The literal-safe entry survived with its exact tactic ... + good = ("op_good", "R", "0", ((1, 128), )) + assert good in cache.cache + assert cache.cache[good][1] == 7 + # ... and the non-literal entry was skipped, not silently mis-decoded. + bad = ("op_bad", "R", "0", ((2, 128), )) + assert bad not in cache.cache + + def test_kernel_testing_single_context(): """Test kernel testing with a single choose_one context""" x, w = torch.randn(16, 64), torch.randn(64, 128) @@ -1265,3 +1323,222 @@ def _cublas_call(): f"CuteDSL heuristic kernel ({heuristic_us:.2f} us) is " f">{cublas_tolerance:.2f}x slower than cuBLAS NVFP4 " f"({cublas_us:.2f} us) for M={m}, N={n}, K={k}") + + +@pytest.mark.parametrize("distribution", ["random", "balanced"]) +def test_trtllm_gen_moe_dummy_topk_local_experts_less_than_topk( + distribution, monkeypatch): + """NVBugs 6457853: autotuner warmup must not fail on EP shards where + local_num_experts < top_k (e.g. gpt-oss-120b: 128 experts, top_k=4, + EP64 -> 2 local experts per rank, attention-DP => use_dp=True). + Dummy rows keep the production shape: top_k distinct ids per row, all + local experts present, remaining slots padded with out-of-shard ids.""" + from tensorrt_llm._torch.custom_ops.trtllm_gen_custom_ops import \ + prepare_dummy_topk_and_hook + + monkeypatch.setenv("TRTLLM_GEN_MOE_AUTOTUNE_DUMMY_DISTRIBUTION", + distribution) + num_tokens, top_k = 8, 4 + num_experts, local_num_experts, local_expert_offset = 128, 2, 6 + hidden_states = torch.randn(num_tokens, + 64, + dtype=torch.bfloat16, + device="cuda") + topk_ids = torch.randint(0, + num_experts, (num_tokens, top_k), + dtype=torch.int32, + device="cuda") + topk_weights = torch.ones(num_tokens, + top_k, + dtype=torch.bfloat16, + device="cuda") + + with autotune(): + _, dummy_weights, dummy_ids, _ = prepare_dummy_topk_and_hook( + topk_weights, + topk_ids, + hidden_states, + None, + 1, + TuningConfig(), + top_k, + num_experts, + local_num_experts, + None, + None, + None, + local_expert_offset=local_expert_offset, + use_dp=True) + + assert dummy_ids.shape == (num_tokens, top_k) + assert dummy_ids.dtype == torch.int32 + assert dummy_weights.shape == (num_tokens, top_k) + shard = range(local_expert_offset, local_expert_offset + local_num_experts) + for row in dummy_ids.tolist(): + assert len(set(row)) == top_k, f"duplicate ids in row {row}" + assert sum(x in shard for x in row) == local_num_experts, ( + f"expected all {local_num_experts} local experts in row {row}") + assert all(0 <= x < num_experts for x in row), row + + +def test_post_tune_merge_tactics_min_time_and_subset_kept(): + tuner = autotuner.AutoTuner() + tuner.mapping = Mapping(world_size=2, rank=0, tp_size=2) + + r0 = { + ("gemm", "X"): (0, ("cutlass", 10), 1.0), + ("q", "A"): (0, ("trtllm", ), 1.0) + } + r1 = { + ("gemm", "X"): (0, ("cublaslt", 0), 0.5), + ("vae", "B"): (0, ("cutlass", 2), 1.0) + } + + class _FakeDist: + + def tp_cp_allgather(self, obj): + return [r0, r1] + + tuner._dist = _FakeDist() + tuner.profiling_cache.cache = dict(r0) + tuner.post_tune_merge_tactics() + + cache = tuner.profiling_cache.cache + assert cache[("gemm", "X")] == (0, ("cublaslt", 0), 0.5) + assert cache[("q", "A")] == r0[("q", "A")] + assert cache[("vae", "B")] == r1[("vae", "B")] + + +def test_post_tune_merge_tactics_single_rank_noop(): + tuner = autotuner.AutoTuner() + tuner.mapping = Mapping(world_size=1, rank=0, tp_size=1) + tuner._dist = None + original = {("gemm", "X"): (0, ("cutlass", 10), 1.0)} + tuner.profiling_cache.cache = dict(original) + tuner.post_tune_merge_tactics() + assert tuner.profiling_cache.cache == original + + +def test_profiling_cache_enum_tactic_roundtrip(tmp_path): + # An enum tactic must survive save -> load: repr("") isn't + # ast.literal_eval-parsable, so the cache serializes tactic.value instead. + class _Tac(enum.IntEnum): + TRTLLM = -1 + + key = ("op::x", "Runner", "(1,)") + src = autotuner.AutoTuner().profiling_cache + src.cache[key] = (0, _Tac.TRTLLM, 1.0) + path = str(tmp_path / "cache.json") + src.save_cache(path, rank=0) # must not raise + + dst = autotuner.AutoTuner().profiling_cache + dst.load_cache(path, rank=0) + assert dst.cache[key][1] == _Tac.TRTLLM # IntEnum compares equal to -1 + + +def test_autotune_post_tune_merge_before_save(tmp_path): + # autotune(post_tune_merge_dist=...) must merge across ranks and persist the + # merged winner, then restore the singleton's prior distributed state. + tuner = AutoTuner.get() + tuner.clear_cache() + r0 = {("gemm", "X"): (0, ("cutlass", 10), 1.0)} + r1 = {("gemm", "X"): (0, ("cublaslt", 0), 0.5)} + tuner.profiling_cache.cache = dict(r0) + + class _FakeDist: + mapping = Mapping(world_size=2, rank=0, tp_size=2) + + def tp_cp_allgather(self, obj): + return [r0, r1] + + prev_mapping, prev_dist = tuner.mapping, tuner._dist + path = str(tmp_path / "cache.json") + with autotune(cache_path=path, post_tune_merge_dist=_FakeDist()): + pass + + # Merged in memory (fastest tactic won) and persisted before the context + # exits (the saved file carries the merged winner). + assert tuner.profiling_cache.cache[("gemm", "X")] == (0, ("cublaslt", 0), + 0.5) + persisted = autotuner.AutoTuner().profiling_cache + persisted.load_cache(path, rank=0) + assert persisted.cache[("gemm", "X")][1] == ("cublaslt", 0) + # The temporary full-world distributed state was restored. + assert tuner.mapping is prev_mapping + assert tuner._dist is prev_dist + + +def _post_tune_merge_worker(world_size): + """Run on each MPI rank: seed a distinct cache, then merge for real.""" + rank = tensorrt_llm.mpi_rank() + mapping = Mapping(world_size=world_size, + rank=rank, + tp_size=world_size, + pp_size=1) + tuner = AutoTuner.get() + tuner.clear_cache() + tuner.setup_distributed_state(mapping) + + # Shared key: rank 1's tactic is faster (should win). Plus a rank-unique key. + shared = ("gemm", "X") + tuner.profiling_cache.cache = { + shared: (0, ("cutlass", 10), 1.0) if rank == 0 else + (0, ("cublaslt", 0), 0.5), + (f"only_rank{rank}", "K"): (0, ("trtllm", ), 1.0), + } + tuner.post_tune_merge_tactics() + + cache = tuner.profiling_cache.cache + return { + "shared": cache[shared], + "has_r0": ("only_rank0", "K") in cache, + "has_r1": ("only_rank1", "K") in cache, + } + + +@pytest.mark.parametrize("mpi_pool_executor", [2], indirect=True) +def test_post_tune_merge_tactics_multi_rank(mpi_pool_executor): + # Real 2-rank merge over an actual process group (not a FakeDist): every + # rank must converge to the same cache — the min-time winner for the shared + # key and both ranks' unique keys kept. + world_size = 2 + results = list( + mpi_pool_executor.map(_post_tune_merge_worker, + *zip(*[(world_size, )] * world_size))) + assert len(results) == world_size + for r in results: + assert r["shared"] == (0, ("cublaslt", 0), 0.5) + assert r["has_r0"] and r["has_r1"] + + +def _autotune_dist_allgather_worker(world_size): + import torch.distributed as dist + + from tensorrt_llm._torch.visual_gen.mapping import _VisualGenAutotuneDist + rank = tensorrt_llm.mpi_rank() + os.environ["RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29557") + if not dist.is_initialized(): + dist.init_process_group(backend="gloo", + world_size=world_size, + rank=rank) + # VG's real mesh has tp_size=1; the communicator must gather over the whole + # world regardless, so the mapping's tp axis is irrelevant to the gather. + d = _VisualGenAutotuneDist( + Mapping(world_size=world_size, rank=rank, tp_size=world_size)) + return d.tp_cp_allgather(f"rank{rank}") + + +@pytest.mark.parametrize("mpi_pool_executor", [2], indirect=True) +def test_visual_gen_autotune_dist_world_allgather(mpi_pool_executor): + # _VisualGenAutotuneDist.tp_cp_allgather must gather every rank's object over + # the default world group (not the single-rank tp subgroup), and construct + # without running TorchDist.__init__. + world_size = 2 + results = list( + mpi_pool_executor.map(_autotune_dist_allgather_worker, + *zip(*[(world_size, )] * world_size))) + for got in results: + assert got == ["rank0", "rank1"] diff --git a/tests/unittest/_torch/modeling/test_gemma4_multimodal.py b/tests/unittest/_torch/modeling/test_gemma4_multimodal.py index 1424a5e840a4..c51b328951ff 100644 --- a/tests/unittest/_torch/modeling/test_gemma4_multimodal.py +++ b/tests/unittest/_torch/modeling/test_gemma4_multimodal.py @@ -62,11 +62,17 @@ Gemma4ForConditionalGeneration, Gemma4MultimodalEmbedder, ) +from tensorrt_llm._torch.models.modeling_multimodal_mixin import MultimodalModelMixin # noqa: E402 from tensorrt_llm._torch.models.modeling_multimodal_utils import ( # noqa: E402 find_input_mm_embeds, get_multimodal_embeddings, ) -from tensorrt_llm.inputs.multimodal import MultimodalParams, MultimodalRuntimeData # noqa: E402 +from tensorrt_llm.inputs.multimodal import ( # noqa: E402 + MultimodalInput, + MultimodalParams, + MultimodalRuntimeData, +) +from tensorrt_llm.llmapi.llm_args import MultimodalConfig # noqa: E402 from tensorrt_llm.mapping import Mapping # noqa: E402 # --------------------------------------------------------------------------- @@ -125,6 +131,59 @@ } +class _Gemma4EncoderCacheHarness(MultimodalModelMixin): + """Lightweight Gemma4 encoder-cache harness without model weights.""" + + supports_encoder_cache = True + encode_multimodal_inputs = Gemma4ForConditionalGeneration.encode_multimodal_inputs + + def __init__(self, embedding_dim: int = 12) -> None: + self.model_config = ModelConfig( + multimodal_config=MultimodalConfig(encoder_cache_max_bytes=4096) + ) + self._embedding_dim = embedding_dim + self.encoder_calls = 0 + self.audio_tower = None + + @property + def embedding_dim(self) -> int: + return self._embedding_dim + + @property + def embedding_dtype(self) -> torch.dtype: + return torch.float32 + + def _get_image_features(self, pixel_values: torch.Tensor, **kwargs) -> torch.Tensor: + del kwargs + self.encoder_calls += 1 + return torch.full( + (pixel_values.shape[0] * 2, self.embedding_dim), + float(self.encoder_calls), + dtype=self.embedding_dtype, + ) + + +def _make_keyed_image_param() -> MultimodalParams: + embedding_lengths = [2] + return MultimodalParams( + multimodal_input=MultimodalInput( + multimodal_hashes=[[1, 2, 3, 4, 5, 6, 7, 8]], + multimodal_positions=[0], + multimodal_lengths=embedding_lengths, + ), + multimodal_data={ + "image": {"pixel_values": torch.empty(1, 1, 1)}, + "multimodal_embedding_lengths": embedding_lengths, + "mm_processor_kwargs_hash": "kwargs-a", + }, + multimodal_runtime=MultimodalRuntimeData( + embed_mask_cumsum=torch.arange(1, 3, dtype=torch.int64), + past_seen_token_num=0, + chunk_end_pos=2, + ), + ) + + # Mirror the engine's encoder runtime sizes (``get_encoder_runtime_sizes`` -> # ``encoder_max_batch_size`` / ``encoder_max_num_tokens``, defaulting to # ``max_batch_size`` / ``max_num_tokens``). The encoder ``AttentionMetadata`` is @@ -702,6 +761,19 @@ def test_instantiation_with_vision(self): # TRT-LLM class, not ``transformers.AutoModel`` output. self.assertIsInstance(model.vision_tower, Gemma4VisionModel) + def test_encoder_cache_reuses_image_embedding_across_requests(self): + """Persistent cache reuse applies to the shared dense/MoE Gemma4 wrapper.""" + self.assertTrue(issubclass(Gemma4ForConditionalGeneration, MultimodalModelMixin)) + self.assertTrue(Gemma4ForConditionalGeneration.supports_encoder_cache) + + model = _Gemma4EncoderCacheHarness() + first = model._get_or_encode_multimodal_embeddings([_make_keyed_image_param()]) + second = model._get_or_encode_multimodal_embeddings([_make_keyed_image_param()]) + + self.assertEqual(model.encoder_calls, 1) + torch.testing.assert_close(second, first) + self.assertEqual(len(model._multimodal_encoder_cache), 1) + def test_chunked_prefill_reuses_cached_vision_embeddings(self): """Later active chunks slice cached features without rerunning vision.""" model = self._make_model() @@ -724,7 +796,7 @@ def test_chunked_prefill_reuses_cached_vision_embeddings(self): model, "_get_image_features", return_value=expected_embeddings ) as image_encoder: all_embeddings = get_multimodal_embeddings( - model._forward_multimodal_encoder, [multimodal_param] + model.encode_multimodal_inputs, [multimodal_param] ) first_chunk = find_input_mm_embeds(all_embeddings, [multimodal_param]) @@ -734,7 +806,7 @@ def test_chunked_prefill_reuses_cached_vision_embeddings(self): embed_mask_cumsum=embed_mask_cumsum, ) all_embeddings = get_multimodal_embeddings( - model._forward_multimodal_encoder, [multimodal_param] + model.encode_multimodal_inputs, [multimodal_param] ) second_chunk = find_input_mm_embeds(all_embeddings, [multimodal_param]) @@ -752,8 +824,9 @@ def test_chunk_without_multimodal_tokens_is_inactive(self): embed_mask_cumsum=torch.tensor([0, 0, 1, 2], dtype=torch.int64), ), ) - self.assertFalse( - Gemma4ForConditionalGeneration._has_active_multimodal_tokens(multimodal_param) + self.assertEqual( + Gemma4ForConditionalGeneration.select_multimodal_params(None, [multimodal_param], 1), + [], ) def test_mixed_modality_batch_preserves_request_order(self): @@ -791,7 +864,7 @@ def test_mixed_modality_batch_preserves_request_order(self): "_get_image_features", side_effect=lambda pixel_values, **_: pixel_values[:, 0], ): - embeddings = get_multimodal_embeddings(model._forward_multimodal_encoder, params) + embeddings = get_multimodal_embeddings(model.encode_multimodal_inputs, params) expected = torch.tensor([[2.0], [1.0], [3.0]]) torch.testing.assert_close(embeddings[0], expected) @@ -800,8 +873,8 @@ def test_mixed_modality_batch_preserves_request_order(self): param.multimodal_data["multimodal_embedding"], expected_embedding.unsqueeze(0) ) - def test_single_request_with_multiple_modalities_is_allowed(self): - """The request-order fix does not reject an existing mixed request.""" + def test_single_request_with_multiple_modalities_raises(self): + """Mixed-modality requests are rejected until prompt-order handling is supported.""" model = self._make_model() multimodal_param = MultimodalParams( multimodal_data={ @@ -810,14 +883,11 @@ def test_single_request_with_multiple_modalities_is_allowed(self): } ) - with unittest.mock.patch.object( - model, - "_get_image_features", - side_effect=lambda pixel_values, **_: pixel_values[:, 0], + with self.assertRaisesRegex( + ValueError, + "requests containing multiple multimodal input types are not supported", ): - embeddings = model._forward_multimodal_encoder([multimodal_param]) - - torch.testing.assert_close(embeddings, torch.tensor([[1.0], [2.0]])) + model.encode_multimodal_inputs([multimodal_param]) def test_instantiation_without_vision(self): """VLM wrapper works text-only when vision_config is None.""" diff --git a/tests/unittest/_torch/modeling/test_modeling_deepseekv4.py b/tests/unittest/_torch/modeling/test_modeling_deepseekv4.py index 106ca0e5c59d..b46ed3e477b9 100644 --- a/tests/unittest/_torch/modeling/test_modeling_deepseekv4.py +++ b/tests/unittest/_torch/modeling/test_modeling_deepseekv4.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import ast import inspect import json @@ -33,6 +36,7 @@ DeepseekV4MTP, _copy_deepseek_v4_fused_a_weight_scale, _deepseek_v4_pos_embd_params, + _normalize_deepseek_v4_nvfp4_mixed_precision_config, _remap_deepseek_v4_checkpoint_keys, _resolve_enable_fused_hc, ) @@ -442,6 +446,48 @@ def test_deepseek_v4_moe_auto_backend_on_blackwell(monkeypatch): assert ModelConfig.resolve_moe_backend("AUTO", "DeepseekV4ForCausalLM") == "TRTLLM" +def test_deepseek_v4_nvfp4_mixed_precision_config(): + config = DeepseekV4Config() + config.quantization_config = { + "quant_method": "fp8", + "weight_block_size": [128, 128], + "modules_to_not_convert": ["lm_head"], + } + mixed_quant_config = QuantConfig( + quant_algo=QuantAlgo.MIXED_PRECISION, + group_size=16, + exclude_modules=["*.attn.*", "*.ffn.shared_experts.*", "head", "mtp.*"], + ) + mixed_quant_config.mamba_ssm_cache_dtype = torch.bfloat16 + assert not mixed_quant_config.layer_quant_mode.has_fp8_block_scales() + experts_quant_config = QuantConfig(quant_algo=QuantAlgo.NVFP4, group_size=16) + model_config = ModelConfig( + pretrained_config=config, + quant_config=mixed_quant_config, + quant_config_dict={"model.layers.0.mlp.experts": experts_quant_config}, + ) + model_config._frozen = True + + normalized_config = _normalize_deepseek_v4_nvfp4_mixed_precision_config(model_config) + + assert normalized_config is model_config + assert mixed_quant_config.quant_algo == QuantAlgo.MIXED_PRECISION + assert normalized_config.quant_config.quant_algo == QuantAlgo.FP8_BLOCK_SCALES + assert normalized_config.quant_config.layer_quant_mode.has_fp8_block_scales() + assert normalized_config.quant_config.group_size == 128 + assert normalized_config.quant_config.mamba_ssm_cache_dtype == torch.bfloat16 + assert normalized_config.quant_config.exclude_modules == [ + "lm_head", + "*kv_b_proj*", + "*k_b_proj*", + "*eh_proj*", + ] + assert ( + normalized_config.quant_config_dict["model.layers.0.mlp.experts"].quant_algo + == QuantAlgo.NVFP4 + ) + + def test_deepseek_v4_routed_moe_quant_config_from_mxfp4_header(tmp_path, monkeypatch): monkeypatch.setattr("tensorrt_llm._torch.model_config.get_sm_version", lambda: 100) tensor_name = "layers.0.ffn.experts.0.w1.weight" diff --git a/tests/unittest/_torch/modeling/test_modeling_gemma4.py b/tests/unittest/_torch/modeling/test_modeling_gemma4.py index f752703b0009..b66eea54cc95 100644 --- a/tests/unittest/_torch/modeling/test_modeling_gemma4.py +++ b/tests/unittest/_torch/modeling/test_modeling_gemma4.py @@ -22,10 +22,14 @@ import unittest import unittest.mock from copy import deepcopy +from types import SimpleNamespace +from typing import TYPE_CHECKING import torch from transformers import Gemma4Config, Gemma4TextConfig +from tensorrt_llm._torch.attention_backend import FlashInferAttention, FlashInferAttentionMetadata +from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.checkpoints.hf.gemma4_weight_mapper import Gemma4HfWeightMapper from tensorrt_llm._torch.models.modeling_gemma4 import ( @@ -39,6 +43,12 @@ ) from tensorrt_llm.mapping import Mapping +if TYPE_CHECKING: + from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2 + +_FLASHINFER_WORKSPACE_BYTES = 320 * 1024 * 1024 +_TRTLLM_GEN_TOKENS_PER_BLOCK = 32 + # --------------------------------------------------------------------------- # Small test configs # --------------------------------------------------------------------------- @@ -361,6 +371,91 @@ def test_num_kv_heads_per_layer_type(self): class TestGemma4HfWeightMapper(unittest.TestCase): """Tests for Gemma4-specific checkpoint key transformations.""" + def test_duplicate_full_attention_kv_projection_tensors(self): + """Full K=V layers should duplicate all missing k_proj tensors to v_proj.""" + fields = ("weight", "weight_scale", "input_scale", "weight_scale_2", "pre_quant_scale") + for is_vlm in (False, True): + with self.subTest(is_vlm=is_vlm): + mapper = Gemma4HfWeightMapper() + config = SimpleNamespace( + attention_k_eq_v=True, + layer_types=["sliding_attention", "full_attention"], + ) + layer_scalar_buffers = [unittest.mock.Mock(), unittest.mock.Mock()] + layers = [ + SimpleNamespace(layer_scalar=layer_scalar_buffer) + for layer_scalar_buffer in layer_scalar_buffers + ] + if is_vlm: + mapper._model = SimpleNamespace( + config=config, + vision_tower=object(), + llm=SimpleNamespace(model=SimpleNamespace(layers=layers)), + ) + model_prefix = "language_model.model" + else: + mapper._model = SimpleNamespace( + config=config, + model=SimpleNamespace(layers=layers), + ) + model_prefix = "model" + + weights = {} + if is_vlm: + weights["model.vision_tower.marker"] = object() + raw_prefix = "model.language_model" + full_k_prefix = f"{model_prefix}.layers.1.self_attn.k_proj." + full_v_prefix = f"{model_prefix}.layers.1.self_attn.v_proj." + sliding_v_prefix = f"{model_prefix}.layers.0.self_attn.v_proj." + for field in fields: + weights[f"{raw_prefix}.layers.1.self_attn.k_proj.{field}"] = object() + weights[f"{raw_prefix}.layers.0.self_attn.k_proj.{field}"] = object() + + explicit_v_input_scale = object() + weights[f"{raw_prefix}.layers.1.self_attn.v_proj.input_scale"] = ( + explicit_v_input_scale + ) + explicit_v_scale = object() + weights[f"{raw_prefix}.layers.1.self_attn.k_proj.k_scale"] = object() + weights[f"{raw_prefix}.layers.1.self_attn.k_proj.k_bias"] = object() + weights[f"{raw_prefix}.layers.1.self_attn.v_proj.v_scale"] = explicit_v_scale + weights[f"{raw_prefix}.layers.1.self_attn.k_norm.weight"] = object() + layer_scalar_value = object() + weights[f"{raw_prefix}.layers.1.layer_scalar"] = layer_scalar_value + + result = mapper.preprocess_weights(weights) + + self.assertIs( + result[f"{full_v_prefix}input_scale"], + explicit_v_input_scale, + ) + for field in fields: + if field == "input_scale": + continue + self.assertIs( + result[f"{full_v_prefix}{field}"], + result[f"{full_k_prefix}{field}"], + ) + self.assertIs(result[f"{full_v_prefix}v_scale"], explicit_v_scale) + self.assertIs( + result[f"{full_v_prefix}v_bias"], + result[f"{full_k_prefix}k_bias"], + ) + self.assertNotIn(f"{full_v_prefix}k_scale", result) + self.assertNotIn(f"{full_v_prefix}k_bias", result) + for field in fields: + self.assertNotIn(f"{sliding_v_prefix}{field}", result) + self.assertNotIn( + f"{model_prefix}.layers.1.self_attn.v_norm.weight", + result, + ) + layer_scalar_buffers[1].copy_.assert_called_once_with(layer_scalar_value) + self.assertNotIn(f"{model_prefix}.layers.1.layer_scalar", result) + + second_result = mapper.preprocess_weights(result) + for key, value in result.items(): + self.assertIs(second_result[key], value) + def test_remap_modelopt_nvfp4_per_expert_weights(self): """ModelOpt's split expert tensors should map to the VANILLA MoE layout.""" mapper = Gemma4HfWeightMapper() @@ -2225,6 +2320,328 @@ class TestGemma4CUDAGraph(unittest.TestCase): _get_kv_cache_manager = staticmethod(_build_gemma4_kv_cache_manager) + def _make_trtllm_gen_decode_case( + self, + initial_page_counts: list[int], + *, + reserved_page_counts: list[int] | None = None, + max_pages: int = 64, + manager_batch_size: int | None = None, + ) -> tuple[ + "KVCacheManagerV2", + list["FlashInferAttention"], + "FlashInferAttentionMetadata", + list[torch.Tensor], + list[torch.Tensor], + list[torch.Tensor], + ]: + """Build a two-pool Gemma4 trtllm-gen CUDA-graph decode case.""" + batch_size = len(initial_page_counts) + if reserved_page_counts is None: + reserved_page_counts = initial_page_counts + if manager_batch_size is None: + manager_batch_size = batch_size + + config = Gemma4TextConfig(**deepcopy(GEMMA4_E2B_REAL_DIMS_CONFIG)) + kv_cache_manager = self._get_kv_cache_manager( + config, + num_blocks=max_pages, + tokens_per_block=_TRTLLM_GEN_TOKENS_PER_BLOCK, + batch_size=manager_batch_size, + ) + self.addCleanup(kv_cache_manager.shutdown) + self.assertTrue(kv_cache_manager.is_vswa, "Expected VSWA manager") + + request_ids = list(range(batch_size)) + requests = kv_cache_manager.add_dummy_requests( + request_ids, + token_nums=[ + (count - 1) * _TRTLLM_GEN_TOKENS_PER_BLOCK + 1 for count in reserved_page_counts + ], + ) + if requests is None: + self.fail("Failed to allocate dummy requests for the test") + + layer_indices = [ + config.layer_types.index("sliding_attention"), + config.layer_types.index("full_attention"), + ] + layers = [] + queries = [] + keys = [] + values = [] + for layer_idx in layer_indices: + is_sliding = config.layer_types[layer_idx] == "sliding_attention" + head_dim = config.head_dim if is_sliding else config.global_head_dim + layers.append( + FlashInferAttention( + layer_idx=layer_idx, + num_heads=config.num_attention_heads, + head_dim=head_dim, + num_kv_heads=config.num_key_value_heads, + flashinfer_backend="trtllm-gen", + ) + ) + queries.append( + torch.randn( + batch_size, + config.num_attention_heads * head_dim, + dtype=config.torch_dtype, + device="cuda", + ) + ) + keys.append( + torch.randn( + batch_size, + config.num_key_value_heads * head_dim, + dtype=config.torch_dtype, + device="cuda", + ) + ) + values.append( + torch.randn( + batch_size, + config.num_key_value_heads * head_dim, + dtype=config.torch_dtype, + device="cuda", + ) + ) + + kv_buffer = kv_cache_manager.get_buffers(layer_idx) + self.assertIsNotNone(kv_buffer) + torch.nn.init.normal_(kv_buffer) + + seq_lens = torch.ones(batch_size, dtype=torch.int) + metadata = FlashInferAttentionMetadata( + seq_lens=seq_lens, + num_contexts=0, + is_cuda_graph=True, + kv_cache_params=KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=[ + (count - 1) * _TRTLLM_GEN_TOKENS_PER_BLOCK for count in initial_page_counts + ], + ), + workspace_buffer=torch.empty( + _FLASHINFER_WORKSPACE_BYTES, dtype=torch.uint8, device="cuda" + ), + max_num_requests=batch_size, + max_num_tokens=batch_size, + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + ) + metadata.prepare() + for layer, query, key, value in zip(layers, queries, keys, values, strict=True): + layer.forward(query, key, value, metadata) + + return kv_cache_manager, layers, metadata, queries, keys, values + + def _prepare_decode_page_counts( + self, + metadata: "FlashInferAttentionMetadata", + request_ids: list[int], + page_counts: list[int], + *, + preserve_cuda_graph_shape: bool = False, + ) -> None: + """Refresh decode metadata with one query token per request.""" + self.assertTrue(all(count > 0 for count in page_counts)) + seq_lens = torch.ones(len(page_counts), dtype=torch.int) + if preserve_cuda_graph_shape: + # Keep the captured device buffer at maximum batch size while shrinking the active + # host metadata, matching the stale-row condition handled by FlashInfer prepare(). + metadata._seq_lens = seq_lens + metadata.on_update() + else: + metadata.seq_lens = seq_lens + metadata.request_ids = request_ids + metadata.kv_cache_params = KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=[ + (count - 1) * _TRTLLM_GEN_TOKENS_PER_BLOCK for count in page_counts + ], + ) + metadata.prepare() + + def _expected_decode_block_table( + self, + metadata: "FlashInferAttentionMetadata", + head_dim: int, + page_counts: list[int], + *, + rows: int, + width: int, + ) -> torch.Tensor: + """Build the expected compact table from one VSWA pool's host indices.""" + pool_id = metadata._vswa_head_dim_to_pool[head_dim] + pool_indices = metadata._host_pool_indices[pool_id].numpy() + expected = torch.zeros((rows, width), dtype=torch.int32) + source_offset = metadata.num_context_blocks + for row, page_count in enumerate(page_counts): + copy_width = min(page_count, width) + expected[row, :copy_width] = torch.from_numpy( + pool_indices[source_offset : source_offset + copy_width].copy() + ) + source_offset += page_count + return expected + + @torch.no_grad() + @unittest.mock.patch( + "tensorrt_llm.runtime.kv_cache_manager_v2._utils.assert_critical", lambda *a, **kw: None + ) + def test_cuda_graph_trtllm_gen_block_table_transitions(self) -> None: + """Shrinking the active rectangle clears stale rows and columns.""" + initial_page_counts = [8, 5, 3, 2] + _, _, metadata, _, _, _ = self._make_trtllm_gen_decode_case(initial_page_counts) + + new_page_counts = [2, 1] + self._prepare_decode_page_counts( + metadata, + request_ids=[0, 1], + page_counts=new_page_counts, + preserve_cuda_graph_shape=True, + ) + torch.cuda.synchronize() + + for plan_params, wrappers in metadata._plan_params_to_wrappers.items(): + with self.subTest(head_dim=plan_params.head_dim): + block_tables = wrappers.decode_wrapper._block_tables + expected = self._expected_decode_block_table( + metadata, + plan_params.head_dim, + new_page_counts, + rows=len(initial_page_counts), + width=max(initial_page_counts), + ) + torch.testing.assert_close( + block_tables[: len(initial_page_counts), : max(initial_page_counts)].cpu(), + expected, + atol=0, + rtol=0, + ) + self.assertEqual(wrappers.decode_block_table_active_rows, len(new_page_counts)) + self.assertEqual(wrappers.decode_block_table_active_width, max(new_page_counts)) + + @torch.no_grad() + @unittest.mock.patch( + "tensorrt_llm.runtime.kv_cache_manager_v2._utils.assert_critical", lambda *a, **kw: None + ) + def test_cuda_graph_trtllm_gen_host_table_growth_keeps_device_pointer(self) -> None: + """Crossing 64 pages grows host staging without moving the graph buffer.""" + initial_page_counts = [63, 2] + _, _, metadata, _, _, _ = self._make_trtllm_gen_decode_case( + initial_page_counts, + reserved_page_counts=[65, 2], + max_pages=512, + ) + + initial_state = {} + for plan_params, wrappers in metadata._plan_params_to_wrappers.items(): + block_tables = wrappers.decode_wrapper._block_tables + self.assertGreaterEqual(block_tables.size(1), 65) + self.assertEqual(block_tables.size(1), metadata.kv_cache_manager.max_blocks_per_seq) + self.assertEqual(wrappers.host_decode_block_tables.size(1), 64) + initial_state[plan_params.head_dim] = ( + block_tables.data_ptr(), + wrappers.host_decode_block_tables.data_ptr(), + ) + + new_page_counts = [65, 2] + self._prepare_decode_page_counts(metadata, [0, 1], new_page_counts) + torch.cuda.synchronize() + + for plan_params, wrappers in metadata._plan_params_to_wrappers.items(): + with self.subTest(head_dim=plan_params.head_dim): + block_tables = wrappers.decode_wrapper._block_tables + old_device_ptr, old_host_ptr = initial_state[plan_params.head_dim] + self.assertEqual(block_tables.data_ptr(), old_device_ptr) + self.assertNotEqual(wrappers.host_decode_block_tables.data_ptr(), old_host_ptr) + self.assertGreaterEqual(wrappers.host_decode_block_tables.size(1), 65) + expected = self._expected_decode_block_table( + metadata, + plan_params.head_dim, + new_page_counts, + rows=len(new_page_counts), + width=max(new_page_counts), + ) + torch.testing.assert_close( + block_tables[: len(new_page_counts), : max(new_page_counts)].cpu(), + expected, + atol=0, + rtol=0, + ) + + @torch.no_grad() + @unittest.mock.patch( + "tensorrt_llm.runtime.kv_cache_manager_v2._utils.assert_critical", lambda *a, **kw: None + ) + def test_cuda_graph_trtllm_gen_request_turnover_matches_eager(self) -> None: + """A captured graph remains correct when long requests are replaced by short ones.""" + initial_page_counts = [8, 4] + ( + kv_cache_manager, + layers, + metadata, + queries, + keys, + values, + ) = self._make_trtllm_gen_decode_case( + initial_page_counts, + max_pages=64, + manager_batch_size=4, + ) + + new_request_ids = [100, 101] + new_page_counts = [2, 1] + new_requests = kv_cache_manager.add_dummy_requests( + new_request_ids, + token_nums=[ + (count - 1) * _TRTLLM_GEN_TOKENS_PER_BLOCK + 1 for count in new_page_counts + ], + ) + self.assertIsNotNone(new_requests) + + for layer, query, key, value in zip(layers, queries, keys, values, strict=True): + layer.forward(query, key, value, metadata) + + graph_outputs = [] + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for layer, query, key, value in zip(layers, queries, keys, values, strict=True): + graph_outputs.append(layer.forward(query, key, value, metadata)) + torch.cuda.synchronize() + + self._prepare_decode_page_counts(metadata, new_request_ids, new_page_counts) + graph.replay() + torch.cuda.synchronize() + + reference_metadata = FlashInferAttentionMetadata( + seq_lens=torch.ones(len(new_request_ids), dtype=torch.int), + num_contexts=0, + kv_cache_params=KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=[ + (count - 1) * _TRTLLM_GEN_TOKENS_PER_BLOCK for count in new_page_counts + ], + ), + max_num_requests=len(new_request_ids), + max_num_tokens=len(new_request_ids), + kv_cache_manager=kv_cache_manager, + request_ids=new_request_ids, + ) + reference_metadata.prepare() + for layer_idx, (layer, query, key, value) in enumerate( + zip(layers, queries, keys, values, strict=True) + ): + reference_output = layer.forward(query, key, value, reference_metadata) + torch.testing.assert_close( + graph_outputs[layer_idx], + reference_output, + atol=1e-2, + rtol=0, + msg=f"Layer {layer.layer_idx}: request-turnover graph output diverges from eager", + ) + @torch.no_grad() @unittest.mock.patch( "tensorrt_llm.runtime.kv_cache_manager_v2._utils.assert_critical", lambda *a, **kw: None diff --git a/tests/unittest/_torch/modeling/test_modeling_gemma4_unified.py b/tests/unittest/_torch/modeling/test_modeling_gemma4_unified.py index da04acdb8644..d4f80943c6db 100644 --- a/tests/unittest/_torch/modeling/test_modeling_gemma4_unified.py +++ b/tests/unittest/_torch/modeling/test_modeling_gemma4_unified.py @@ -26,6 +26,7 @@ from types import SimpleNamespace import numpy as np +import pytest import torch from tensorrt_llm._torch.configs import ( @@ -43,6 +44,7 @@ _GEMMA4_ARCHITECTURES, MODEL_CLASS_MAPPER_MAPPING, MODEL_CLASS_MAPPING, + ModelConfig, get_model_architecture, ) @@ -205,6 +207,14 @@ def test_get_model_architecture_resolves_wrapper(): assert arch == _UNIFIED_ARCH +def test_wrapper_rejects_missing_image_token_id(): + config = Gemma4UnifiedConfig(image_token_id=None) + model_config = ModelConfig(pretrained_config=config) + + with pytest.raises(ValueError, match="requires config.image_token_id"): + Gemma4UnifiedForConditionalGeneration(model_config) + + def test_weight_mapper_registered(): # The Gemma4 HF weight mapper must claim the unified arch too, otherwise the # generic mapper would mishandle the per-layer head_dim / k_eq_v / layer_scalar diff --git a/tests/unittest/_torch/modeling/test_modeling_gpt_oss.py b/tests/unittest/_torch/modeling/test_modeling_gpt_oss.py index f61b8c9a48a7..410c50189192 100644 --- a/tests/unittest/_torch/modeling/test_modeling_gpt_oss.py +++ b/tests/unittest/_torch/modeling/test_modeling_gpt_oss.py @@ -47,6 +47,10 @@ """ +def test_gpt_oss_prefers_python_transceiver() -> None: + assert GptOssForCausalLM.get_preferred_transceiver_runtime() == "PYTHON" + + def dump_config_json(dst_dir): if os.path.exists(dst_dir): shutil.rmtree(dst_dir) diff --git a/tests/unittest/_torch/modeling/test_modeling_kimi_k25.py b/tests/unittest/_torch/modeling/test_modeling_kimi_k25.py index af2dc5e16814..232893dfca3e 100644 --- a/tests/unittest/_torch/modeling/test_modeling_kimi_k25.py +++ b/tests/unittest/_torch/modeling/test_modeling_kimi_k25.py @@ -605,6 +605,12 @@ def test_auto_model_registered(self): self.assertIsNotNone(cls, "KimiK25ForConditionalGeneration not in MODEL_CLASS_MAPPING") self.assertIs(cls, KimiK25ForConditionalGeneration) + def test_prefers_python_transceiver(self): + """Kimi-K2.5 defaults to the Python KV-cache transceiver in disagg.""" + self.assertEqual( + KimiK25ForConditionalGeneration.get_preferred_transceiver_runtime(), "PYTHON" + ) + # --------------------------------------------------------------------------- # E2E Smoke Test — requires GPU + model checkpoint diff --git a/tests/unittest/_torch/modeling/test_modeling_minicpmv4_6.py b/tests/unittest/_torch/modeling/test_modeling_minicpmv4_6.py new file mode 100644 index 000000000000..ab41a41fb8e8 --- /dev/null +++ b/tests/unittest/_torch/modeling/test_modeling_minicpmv4_6.py @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for MiniCPM-V 4.6 config loading on the PyTorch backend. + +These tests are intentionally CPU-only and do not require model weights or a +native ``minicpmv4_6`` transformers release, so they run (and protect the fix) +on the repo's currently-pinned ``transformers==5.5.4`` CI as well as on +``>=5.7.0``: + +* ``_resolve_composite_torch_dtype`` dtype resolution. +* ``_build_minicpmv4_6_config`` / ``load_pretrained_config`` composite-config + construction, including the regression guard for the checkpoint that declares + **no** ``torch_dtype`` (which used to crash the hybrid mamba KV-cache paths + with ``None.itemsize``). +* The self-contained ``MiniCPMV4_6VisionConfig`` window helpers. +* The runtime ``transformers>=5.7.0`` guard used by the input processor. + +A single ``transformers>=5.7.0``-gated test asserts the native config is present +once the pin is bumped (at which point the local shim can be removed). +""" + +import copy +import json + +import pytest +import torch +import transformers +from packaging.version import Version + +from tensorrt_llm._torch.configs.minicpmv4_6 import MiniCPMV4_6Config, MiniCPMV4_6VisionConfig +from tensorrt_llm._torch.pyexecutor.config_utils import ( + _build_minicpmv4_6_config, + _resolve_composite_torch_dtype, + load_pretrained_config, +) + +_MIN_TRANSFORMERS = "5.7.0" + +requires_native_minicpmv4_6 = pytest.mark.skipif( + Version(transformers.__version__) < Version(_MIN_TRANSFORMERS), + reason=f"native minicpmv4_6 requires transformers>={_MIN_TRANSFORMERS}", +) + + +def _minicpmv4_6_config_dict() -> dict: + """A faithful, minimal ``config.json`` for openbmb/MiniCPM-V-4.6. + + Mirrors the real checkpoint: composite ``minicpmv4_6`` with a SigLIP2 vision + tower and a Qwen3.5 dense hybrid text tower, and crucially **no** + ``torch_dtype``/``dtype`` at any level. + """ + return copy.deepcopy( + { + "architectures": ["MiniCPMV4_6ForConditionalGeneration"], + "model_type": "minicpmv4_6", + "drop_vision_last_layer": False, + "image_size": 1120, + "insert_layer_id": 6, + "image_token_id": 248056, + "video_token_id": 248057, + "tie_word_embeddings": True, + "vision_config": { + "model_type": "minicpmv4_6_vision", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 980, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + }, + "text_config": { + "model_type": "qwen3_5_text", + "attention_bias": False, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 1024, + "intermediate_size": 3584, + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 16, + "linear_value_head_dim": 128, + "max_position_embeddings": 262144, + "num_attention_heads": 8, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default", + }, + "vocab_size": 248094, + "tie_word_embeddings": True, + }, + } + ) + + +# --------------------------------------------------------------------------- +# _resolve_composite_torch_dtype +# --------------------------------------------------------------------------- +class TestResolveCompositeTorchDtype: + def test_defaults_to_bfloat16_when_absent(self): + assert _resolve_composite_torch_dtype({}, {}) is torch.bfloat16 + + def test_respects_explicit_string(self): + assert _resolve_composite_torch_dtype({"torch_dtype": "float16"}) is torch.float16 + + def test_respects_dtype_alias_key(self): + assert _resolve_composite_torch_dtype({"dtype": "float32"}) is torch.float32 + + def test_respects_torch_dtype_object(self): + assert _resolve_composite_torch_dtype({"torch_dtype": torch.float16}) is torch.float16 + + def test_auto_and_none_are_skipped(self): + assert ( + _resolve_composite_torch_dtype( + {"torch_dtype": "auto"}, + {"dtype": None}, + {"torch_dtype": "float16"}, + ) + is torch.float16 + ) + + def test_first_declaration_wins(self): + assert ( + _resolve_composite_torch_dtype( + {"torch_dtype": "bfloat16"}, + {"torch_dtype": "float16"}, + ) + is torch.bfloat16 + ) + + +# --------------------------------------------------------------------------- +# _build_minicpmv4_6_config +# --------------------------------------------------------------------------- +class TestBuildMiniCPMV46Config: + def test_pins_bfloat16_when_checkpoint_declares_none(self): + # Regression: the real checkpoint declares no torch_dtype anywhere, which + # crashed the hybrid mamba KV-cache byte sizing (None.itemsize). Both the + # composite config and the inner text tower must carry a concrete dtype. + cfg = _build_minicpmv4_6_config(_minicpmv4_6_config_dict()) + assert cfg.torch_dtype is torch.bfloat16 + assert cfg.text_config.torch_dtype is torch.bfloat16 + + def test_respects_explicit_dtype(self): + raw = _minicpmv4_6_config_dict() + raw["torch_dtype"] = "float16" + cfg = _build_minicpmv4_6_config(raw) + assert cfg.torch_dtype is torch.float16 + assert cfg.text_config.torch_dtype is torch.float16 + + def test_keeps_top_level_model_type(self): + # Needed for MULTIMODAL_PLACEHOLDER_REGISTRY lookup; otherwise requests + # fail with "Unknown modality". + cfg = _build_minicpmv4_6_config(_minicpmv4_6_config_dict()) + assert cfg.model_type == "minicpmv4_6" + + def test_normalizes_text_tower_to_qwen3next(self): + cfg = _build_minicpmv4_6_config(_minicpmv4_6_config_dict()) + assert isinstance(cfg.text_config, transformers.Qwen3NextConfig) + assert cfg.text_config.architectures == ["Qwen3_5ForCausalLM"] + assert cfg.text_config.num_hidden_layers == 24 + + def test_preserves_multimodal_token_ids(self): + cfg = _build_minicpmv4_6_config(_minicpmv4_6_config_dict()) + assert cfg.image_token_id == 248056 + assert cfg.video_token_id == 248057 + + def test_propagates_insert_layer_id_to_vision(self): + cfg = _build_minicpmv4_6_config(_minicpmv4_6_config_dict()) + assert cfg.insert_layer_id == 6 + assert cfg.vision_config.insert_layer_id == 6 + assert cfg.vision_config.hidden_size == 1152 + assert cfg.vision_config.num_hidden_layers == 27 + + +# --------------------------------------------------------------------------- +# load_pretrained_config routing (offline, via a temp config.json) +# --------------------------------------------------------------------------- +class TestLoadPretrainedConfigRouting: + def _write_config(self, tmp_path) -> str: + (tmp_path / "config.json").write_text(json.dumps(_minicpmv4_6_config_dict())) + return str(tmp_path) + + def test_routes_minicpmv4_6_by_model_type(self, tmp_path): + cfg = load_pretrained_config(self._write_config(tmp_path)) + assert isinstance(cfg, MiniCPMV4_6Config) + assert cfg.model_type == "minicpmv4_6" + assert cfg.torch_dtype is torch.bfloat16 + + def test_routes_when_only_architecture_present(self, tmp_path): + raw = _minicpmv4_6_config_dict() + raw.pop("model_type") + (tmp_path / "config.json").write_text(json.dumps(raw)) + cfg = load_pretrained_config(str(tmp_path)) + assert isinstance(cfg, MiniCPMV4_6Config) + + +# --------------------------------------------------------------------------- +# MiniCPMV4_6VisionConfig helpers +# --------------------------------------------------------------------------- +class TestVisionConfig: + def test_window_helpers_scale_by_kernel(self): + vc = MiniCPMV4_6VisionConfig( + hidden_size=1152, intermediate_size=4304, window_kernel_size=(2, 2) + ) + assert vc.window_kernel_size == (2, 2) + assert vc.window_hidden_size == 1152 * 2 * 2 + assert vc.window_intermediate_size == 4304 * 2 * 2 + + def test_window_kernel_size_is_tuple(self): + # Accept list from JSON but expose a tuple. + vc = MiniCPMV4_6VisionConfig(window_kernel_size=[2, 2]) + assert vc.window_kernel_size == (2, 2) + + +# --------------------------------------------------------------------------- +# transformers>=5.7.0 runtime guard (used by the input processor) +# --------------------------------------------------------------------------- +class TestTransformersGuard: + @pytest.mark.parametrize("version", ["5.5.4", "5.6.9"]) + def test_raises_on_old_transformers(self, monkeypatch, version): + from tensorrt_llm._torch.models import modeling_minicpmv4_6 as mod + + monkeypatch.setattr(transformers, "__version__", version) + with pytest.raises(RuntimeError, match="transformers>="): + mod._ensure_transformers_supports_minicpmv4_6() + + @pytest.mark.parametrize("version", ["5.7.0", "5.8.1", "5.12.1"]) + def test_passes_on_supported_transformers(self, monkeypatch, version): + from tensorrt_llm._torch.models import modeling_minicpmv4_6 as mod + + monkeypatch.setattr(transformers, "__version__", version) + mod._ensure_transformers_supports_minicpmv4_6() + + +# --------------------------------------------------------------------------- +# Native transformers config (only once the pin is bumped to >=5.7.0) +# --------------------------------------------------------------------------- +@requires_native_minicpmv4_6 +def test_native_minicpmv4_6_config_available(): + # Once transformers>=5.7.0 is pinned, the local MiniCPMV4_6Config shim can + # be dropped in favor of this native config. + from transformers import MiniCPMV4_6Config as HFMiniCPMV4_6Config + + assert HFMiniCPMV4_6Config.model_type == "minicpmv4_6" diff --git a/tests/unittest/_torch/modeling/test_modeling_mistral.py b/tests/unittest/_torch/modeling/test_modeling_mistral.py index 793b70455057..6cb29057a8bd 100644 --- a/tests/unittest/_torch/modeling/test_modeling_mistral.py +++ b/tests/unittest/_torch/modeling/test_modeling_mistral.py @@ -19,7 +19,7 @@ from tensorrt_llm._torch import model_config as model_config_lib from tensorrt_llm._torch.attention_backend import utils as attention_utils from tensorrt_llm._torch.models import modeling_mistral -from tensorrt_llm._torch.models.modeling_mistral import Mistral3InputProcessor +from tensorrt_llm._torch.models.modeling_mistral import MistralHFInputProcessor from tensorrt_llm._torch.models.modeling_utils import MetaInitMode from tensorrt_llm._torch.pyexecutor import resource_manager from tensorrt_llm.bindings import executor as executor_lib @@ -540,7 +540,7 @@ def test_processor_get_num_tokens_per_image( with mock.patch( "tensorrt_llm._torch.models.modeling_mistral.AutoProcessor" ) as mocked_auto_processor: - input_processor = modeling_mistral.Mistral3InputProcessor( + input_processor = modeling_mistral.MistralHFInputProcessor( model_path=str(tmp_path), config=mistral_3_config, tokenizer=mock.MagicMock(), @@ -637,7 +637,7 @@ def test_mistral_attention_swa_layer_types(): # Deterministic dummy-input sizing (Mistral3 / Pixtral input processor). # # CPU-only unit tests for the encoder-profiling dummy contract: reach into -# Mistral3InputProcessor directly (no model load) and stub the geometry the +# MistralHFInputProcessor directly (no model load) and stub the geometry the # dummy math reads. The ViT token unit is the pre-merge patch count # ``(h//patch)*(w//patch)`` -- deliberately *not* the hashing path's LLM-side # Pixtral count with framing tokens. @@ -649,7 +649,7 @@ def _make_dummy_processor(*, patch_size=14, spatial_merge_size=2, image_size=154 ``_processor`` forces ``_vision_geometry`` to fall back to ``vision_config`` (the HF ``mistral3`` path). """ - instance = Mistral3InputProcessor.__new__(Mistral3InputProcessor) + instance = MistralHFInputProcessor.__new__(MistralHFInputProcessor) instance._config = SimpleNamespace( vision_config=SimpleNamespace( patch_size=patch_size, image_size=image_size, num_channels=num_channels diff --git a/tests/unittest/_torch/modeling/test_modeling_multimodal.py b/tests/unittest/_torch/modeling/test_modeling_multimodal.py index 112b665dfb8c..7a57ba500eca 100644 --- a/tests/unittest/_torch/modeling/test_modeling_multimodal.py +++ b/tests/unittest/_torch/modeling/test_modeling_multimodal.py @@ -628,6 +628,7 @@ def get_hybrid_kv_cache_manager( } mamba_params = extract_mamba_kv_cache_params(text_config) + mamba_layer_mask, full_attention_layer_mask = mamba_params.get_layer_masks() if mamba_params.dtype not in dtype_map: raise ValueError( f"Unsupported dtype for hybrid cache manager: " @@ -640,7 +641,7 @@ def get_hybrid_kv_cache_manager( head_dim = text_config.hidden_size // text_config.num_attention_heads # CppMambaHybridCacheManager reads Pydantic-only fields - # (mamba_state_cache_interval, enable_block_reuse) so we have to + # (mamba_state_config, enable_block_reuse) so we have to # construct the llmapi.llm_args.KvCacheConfig here, not the C++ # bindings KvCacheConfig that the standard KVCacheManager path uses. kv_cache_config = PyKvCacheConfig(max_tokens=num_blocks * tokens_per_block) @@ -654,15 +655,15 @@ def get_hybrid_kv_cache_manager( mamba_params.n_groups, mamba_params.head_dim, mamba_params.num_mamba_layers, - mamba_params.mamba_layer_mask, + mamba_layer_mask, mamba_params.dtype, mamba_params.mamba_ssm_cache_dtype, # kv cache parameters (positional) kv_cache_config, tensorrt_llm.bindings.internal.batch_manager.CacheType.SELF, # kw-only - num_layers=mamba_params.num_full_attention_layers, - layer_mask=mamba_params.full_attention_layer_mask, + num_layers=sum(full_attention_layer_mask), + layer_mask=full_attention_layer_mask, num_kv_heads=text_config.num_key_value_heads, head_dim=head_dim, tokens_per_block=tokens_per_block, diff --git a/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py b/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py index b01d961655b6..a7633100f34f 100644 --- a/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py +++ b/tests/unittest/_torch/modeling/test_modeling_qwen3vl.py @@ -351,6 +351,104 @@ def setup_scenario(self, scenario: TestQwen3VLScenario): ) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_qwen3vl_deepstack_fusion_reuses_registered_buffer(): + hidden_size = 4 + num_tokens = 5 + mm_token_id = 7 + config_dict = copy.deepcopy(QWEN3_VL_8B_CONFIG) + config_dict.update( + image_token_id=mm_token_id, + video_token_id=8, + vision_start_token_id=9, + vision_end_token_id=10, + ) + config_dict["text_config"].update( + hidden_size=hidden_size, + intermediate_size=16, + num_hidden_layers=1, + num_attention_heads=1, + num_key_value_heads=1, + head_dim=hidden_size, + vocab_size=32, + ) + config_dict["vision_config"].update( + deepstack_visual_indexes=[0, 1], + depth=2, + hidden_size=32, + intermediate_size=64, + num_heads=1, + out_hidden_size=hidden_size, + ) + model_config = ModelConfig( + pretrained_config=Qwen3VLConfig.from_dict(config_dict), + disable_mm_encoder=True, + max_num_tokens=num_tokens, + ) + model = Qwen3VLModel(model_config, disable_fuse_rope=True).to("cuda") + + input_ids = torch.tensor([1, mm_token_id, 2, mm_token_id, 3], device="cuda") + text_token_indices = torch.tensor([0, 2, 4], device="cuda") + mm_token_indices = torch.tensor([1, 3], device="cuda") + primary_embeds = [ + torch.full( + (2, hidden_size), + 10.0, + dtype=model.embedding_dtype, + device="cuda", + ) + ] + deepstack_embeds = [ + torch.full( + (2, hidden_size), + value, + dtype=model.embedding_dtype, + device="cuda", + ) + for value in (20.0, 30.0) + ] + + _, _, fused_deepstack = model._fuse_multimodal_embeddings( + input_ids=input_ids, + multimodal_embeddings=primary_embeds, + mm_token_ids=model.multimodal_token_ids, + embedding_layer=model.text_embedding_layer, + extra_embeds=deepstack_embeds, + text_token_indices=text_token_indices, + mm_token_indices=mm_token_indices, + ) + + assert len(fused_deepstack) == 2 + # The returned per-level tensors must be views into the model-owned scratch + # buffer, not newly allocated full-sequence tensors. + assert ( + fused_deepstack[0].untyped_storage().data_ptr() + == model.deepstack_input_embeds.untyped_storage().data_ptr() + ) + # Sharing storage alone does not prove that fusion scattered each level to + # the right token rows, so verify the multimodal positions independently. + torch.testing.assert_close( + fused_deepstack[0][mm_token_indices], + deepstack_embeds[0], + ) + torch.testing.assert_close( + fused_deepstack[1][mm_token_indices], + deepstack_embeds[1], + ) + # The scratch buffer is reused across forwards; non-multimodal positions + # must be cleared before every scatter to avoid leaking stale features. + torch.testing.assert_close( + model.deepstack_input_embeds[:, text_token_indices], + torch.zeros( + 2, + len(text_token_indices), + hidden_size, + dtype=model.embedding_dtype, + device="cuda", + ), + ) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") def test_qwen3vl_init_preserves_caller_quant_config(): """Building Qwen3VLModel must not mutate the caller's quant_config.""" diff --git a/tests/unittest/_torch/modeling/test_modeling_speculative.py b/tests/unittest/_torch/modeling/test_modeling_speculative.py index 59ddd3cee128..87ba48e8ac5b 100644 --- a/tests/unittest/_torch/modeling/test_modeling_speculative.py +++ b/tests/unittest/_torch/modeling/test_modeling_speculative.py @@ -13,13 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for Eagle3ForCausalLM.apply_eagle3_fc fc_norm branch.""" +"""Unit tests for speculative modeling classes.""" + +from unittest.mock import MagicMock, patch import pytest import torch from torch import nn +from transformers import PretrainedConfig -from tensorrt_llm._torch.models.modeling_speculative import Eagle3ForCausalLM +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.models.modeling_speculative import ( + Eagle3ForCausalLM, + SpecDecOneEngineForCausalLM, +) from tensorrt_llm._torch.modules.rms_norm import RMSNorm @@ -144,3 +151,73 @@ def test_apply_eagle3_fc_with_fc_norm(num_capture_layers): "fc_norm should apply per-chunk normalization which differs from " "whole-tensor normalization" ) + + +# --------------------------------------------------------------------------- +# SpecDecOneEngineForCausalLM: optional hidden_size / vocab_size +# --------------------------------------------------------------------------- + +_BASE_CLS = "tensorrt_llm._torch.models.modeling_utils.DecoderModelForCausalLM" + + +def _init_specdec_with_mocked_base(model_config, **kwargs): + """Instantiate SpecDecOneEngineForCausalLM with the base class stubbed out. + + DecoderModelForCausalLM is built on the PostInitCaller metaclass, which + invokes __post_init__/__pp_init__ right after __init__ returns. Those + hooks must be stubbed too: the mocked __init__ never sets the attributes + (model_config, lm_head, ...) they rely on. + + Returns the kwargs captured by the mocked base __init__. + """ + with ( + patch(f"{_BASE_CLS}.__init__", return_value=None) as mock_init, + patch(f"{_BASE_CLS}.__post_init__"), + patch(f"{_BASE_CLS}.__pp_init__"), + ): + SpecDecOneEngineForCausalLM(MagicMock(), model_config, **kwargs) + _, captured_kwargs = mock_init.call_args + return captured_kwargs + + +def test_specdec_one_engine_reads_from_pretrained_config() -> None: + """Default path: hidden_size/vocab_size come from pretrained_config.""" + hidden_size = 4096 + vocab_size = 32000 + model_config = ModelConfig( + pretrained_config=PretrainedConfig(hidden_size=hidden_size, vocab_size=vocab_size) + ) + + kwargs = _init_specdec_with_mocked_base(model_config) + assert kwargs["hidden_size"] == hidden_size + assert kwargs["vocab_size"] == vocab_size + + +def test_specdec_one_engine_accepts_explicit_sizes() -> None: + """Composite configs (e.g. VL wrappers) can pass sizes explicitly.""" + hidden_size = 8192 + vocab_size = 128256 + # Bare PretrainedConfig lacks hidden_size/vocab_size; the caller + # supplies them instead. + model_config = ModelConfig(pretrained_config=PretrainedConfig()) + + kwargs = _init_specdec_with_mocked_base( + model_config, hidden_size=hidden_size, vocab_size=vocab_size + ) + assert kwargs["hidden_size"] == hidden_size + assert kwargs["vocab_size"] == vocab_size + + +def test_specdec_one_engine_explicit_overrides_pretrained_config() -> None: + """Explicit args take precedence over pretrained_config when both present.""" + hidden_size = 2048 + vocab_size = 64000 + model_config = ModelConfig( + pretrained_config=PretrainedConfig(hidden_size=4096, vocab_size=32000) + ) + + kwargs = _init_specdec_with_mocked_base( + model_config, hidden_size=hidden_size, vocab_size=vocab_size + ) + assert kwargs["hidden_size"] == hidden_size + assert kwargs["vocab_size"] == vocab_size diff --git a/tests/unittest/_torch/modeling/test_modeling_whisper.py b/tests/unittest/_torch/modeling/test_modeling_whisper.py new file mode 100644 index 000000000000..55c823482a22 --- /dev/null +++ b/tests/unittest/_torch/modeling/test_modeling_whisper.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for modeling_whisper.py. + +CPU-only parity tests pinning ``WhisperLogMelFrontend`` (the engine-side GPU +log-mel front-end) to the HF reference (``_torch_extract_fbank_features``); +drift here silently corrupts transcripts. +""" + +import numpy as np +import pytest +import torch +from transformers import WhisperConfig, WhisperFeatureExtractor + +from tensorrt_llm._torch.models.modeling_whisper import WhisperLogMelFrontend + + +def _synthetic_waveform_batch(n_samples: int, seed: int = 1234) -> np.ndarray: + """[3, n_samples] fp32 batch: full window, ~1/3 window, and near-silence, + zero-padded to a common length (the request contract).""" + rng = np.random.default_rng(seed) + time = np.arange(n_samples, dtype=np.float32) + batch = np.zeros((3, n_samples), dtype=np.float32) + + full = 0.4 * np.sin(2 * np.pi * 440.0 / 16000.0 * time) + full += 0.2 * np.sin(2 * np.pi * 1333.0 / 16000.0 * time) + full += 0.05 * rng.standard_normal(n_samples) + batch[0] = full.astype(np.float32) + + short = n_samples // 3 + batch[1, :short] = (0.3 * rng.standard_normal(short)).astype(np.float32) + + batch[2, :1600] = 1e-4 # hard-zero tail exercises the log floor + return batch + + +def _reference_log_mel(extractor: WhisperFeatureExtractor, batch: np.ndarray) -> torch.Tensor: + return torch.from_numpy(extractor._torch_extract_fbank_features(batch, device="cpu")) + + +@pytest.mark.parametrize("num_mel_bins", [80, 128]) +def test_log_mel_frontend_matches_hf(num_mel_bins): + """Default-parameter parity (no preprocessor config: whisper-tiny=80, + large-v3=128 mel bins).""" + config = WhisperConfig(num_mel_bins=num_mel_bins) + frontend = WhisperLogMelFrontend(config) + extractor = WhisperFeatureExtractor(feature_size=num_mel_bins) + + batch = _synthetic_waveform_batch(extractor.n_samples) + ours = frontend(torch.from_numpy(batch)) + reference = _reference_log_mel(extractor, batch) + + assert ours.shape == (3, num_mel_bins, extractor.nb_max_frames) + torch.testing.assert_close(ours, reference, atol=1e-4, rtol=1e-4) + + +def test_log_mel_frontend_reads_preprocessor_config(tmp_path): + """Non-default STFT parameters (hop_length) and dither must be read from + the checkpoint's preprocessor_config.json, not assumed.""" + extractor = WhisperFeatureExtractor(feature_size=80, hop_length=320, dither=0.02) + extractor.save_pretrained(tmp_path) + + config = WhisperConfig(num_mel_bins=80) + config._name_or_path = str(tmp_path) + frontend = WhisperLogMelFrontend(config) + + assert frontend.hop_length == 320 + assert frontend.dither == pytest.approx(0.02) + + # Both implementations draw the dither noise from torch's default + # generator with an identical shape, so seeding both sides identically + # makes the (random) dither path exactly comparable. + batch = _synthetic_waveform_batch(extractor.n_samples) + torch.manual_seed(0) + ours = frontend(torch.from_numpy(batch)) + torch.manual_seed(0) + reference = _reference_log_mel(extractor, batch) + + torch.testing.assert_close(ours, reference, atol=1e-4, rtol=1e-4) + + +def test_log_mel_frontend_feature_size_mismatch_falls_back(tmp_path): + """A preprocessor config contradicting config.num_mel_bins (broken + checkpoint) must not win over the model config: the conv stem's input + channel count comes from num_mel_bins.""" + WhisperFeatureExtractor(feature_size=80).save_pretrained(tmp_path) + + config = WhisperConfig(num_mel_bins=128) + config._name_or_path = str(tmp_path) + frontend = WhisperLogMelFrontend(config) + + assert frontend._mel_filters_np.shape[1] == 128 + batch = _synthetic_waveform_batch(WhisperFeatureExtractor().n_samples) + assert frontend(torch.from_numpy(batch)).shape[1] == 128 + + +def test_log_mel_frontend_does_not_mutate_input(): + """The waveform buffer belongs to the request; dither must not be added + in place.""" + extractor = WhisperFeatureExtractor(feature_size=80) + config = WhisperConfig(num_mel_bins=80) + frontend = WhisperLogMelFrontend(config) + frontend.dither = 0.02 + + batch = torch.from_numpy(_synthetic_waveform_batch(extractor.n_samples)) + snapshot = batch.clone() + frontend(batch) + torch.testing.assert_close(batch, snapshot, atol=0.0, rtol=0.0) diff --git a/tests/unittest/_torch/models/checkpoints/hf/test_minimaxm3_weight_mapper.py b/tests/unittest/_torch/models/checkpoints/hf/test_minimaxm3_weight_mapper.py new file mode 100644 index 000000000000..896539beaa71 --- /dev/null +++ b/tests/unittest/_torch/models/checkpoints/hf/test_minimaxm3_weight_mapper.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.models.checkpoints.auto_mapper import AutoCheckpointMapper +from tensorrt_llm._torch.models.checkpoints.hf.minimaxm3_weight_mapper import ( + MiniMaxM3HfWeightMapper, +) +from tensorrt_llm._torch.models.checkpoints.hf.weight_mapper import HfWeightMapper +from tensorrt_llm._torch.models.modeling_minimaxm3 import ( + MiniMaxM3ForCausalLM, + MiniMaxM3VLForConditionalGeneration, +) +from tensorrt_llm._torch.models.modeling_utils import DecoderModelForCausalLM +from tensorrt_llm.mapping import Mapping +from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig + +_NUM_KV_HEADS = 4 +_ROWS_PER_HEAD = 2 + + +def _make_mapper(tp_size: int = 8, num_kv_heads: int = _NUM_KV_HEADS) -> MiniMaxM3HfWeightMapper: + config = SimpleNamespace(num_key_value_heads=num_kv_heads, num_attention_heads=8) + model_config = ModelConfig( + pretrained_config=config, + mapping=Mapping(world_size=tp_size, rank=0, tp_size=tp_size), + ) + model = SimpleNamespace(model_config=model_config, config=config) + mapper = MiniMaxM3HfWeightMapper() + mapper.init_model_and_config(model, model_config) + return mapper + + +def _duplicate_heads(tensor: torch.Tensor, repetitions: int) -> torch.Tensor: + return ( + tensor.reshape(_NUM_KV_HEADS, _ROWS_PER_HEAD, -1) + .repeat_interleave(repetitions, dim=0) + .reshape(_NUM_KV_HEADS * repetitions * _ROWS_PER_HEAD, -1) + ) + + +@pytest.mark.parametrize( + "architecture", + [ + "MiniMaxM3SparseForCausalLM", + "MiniMaxM3SparseForConditionalGeneration", + ], +) +def test_mapper_registration_and_mx_fallback(architecture: str) -> None: + assert isinstance(AutoCheckpointMapper.get("HF", architecture), MiniMaxM3HfWeightMapper) + assert isinstance(AutoCheckpointMapper.get("MX", architecture), MiniMaxM3HfWeightMapper) + + +@pytest.mark.parametrize( + "model_class", + [MiniMaxM3ForCausalLM, MiniMaxM3VLForConditionalGeneration], +) +def test_load_weights_exposes_weight_mapper(model_class: type) -> None: + assert "weight_mapper" in inspect.signature(model_class.load_weights).parameters + + +@pytest.mark.parametrize("scale_name", ["weight_scale_inv", "weight_scale"]) +def test_tp8_mxfp8_duplicates_kv_weight_and_scale_via_callbacks(scale_name: str) -> None: + mapper = _make_mapper() + module = SimpleNamespace(quant_config=QuantConfig(quant_algo=QuantAlgo.MXFP8)) + prefix = "model.layers.0.self_attn" + + weights = {} + sources = {} + for offset, projection in enumerate(("k_proj", "v_proj")): + weight = torch.arange(24, dtype=torch.float32).reshape(8, 3) + offset * 100 + scale = torch.arange(16, dtype=torch.uint8).reshape(8, 2) + offset * 32 + weights[f"{prefix}.{projection}.weight"] = weight + weights[f"{prefix}.{projection}.{scale_name}"] = scale + sources[projection] = {"weight": weight, scale_name: scale} + + mapped = mapper.apply_callbacks(module, "qkv_proj", prefix.split("."), weights) + + assert mapped[0] == {} + for projection, projected in zip(("k_proj", "v_proj"), mapped[1:]): + for name, source in sources[projection].items(): + torch.testing.assert_close(projected[name], _duplicate_heads(source, repetitions=2)) + + +def test_nvfp4_scale_behavior_is_preserved() -> None: + mapper = _make_mapper() + module = SimpleNamespace(quant_config=QuantConfig(quant_algo=QuantAlgo.NVFP4)) + weight = torch.arange(24, dtype=torch.float32).reshape(8, 3) + weight_scale = torch.arange(16, dtype=torch.float32).reshape(8, 2) + scale_inv = torch.arange(16, dtype=torch.uint8).reshape(8, 2) + + mapped = mapper._duplicate_kv_weights( + module, + "k_proj", + { + "weight": weight, + "weight_scale": weight_scale, + "weight_scale_inv": scale_inv, + }, + ) + + torch.testing.assert_close(mapped["weight"], _duplicate_heads(weight, repetitions=2)) + torch.testing.assert_close( + mapped["weight_scale"], _duplicate_heads(weight_scale, repetitions=2) + ) + assert mapped["weight_scale_inv"] is scale_inv + + +def test_quant_config_none_is_guarded() -> None: + mapper = _make_mapper() + module = SimpleNamespace(quant_config=None) + weight = torch.arange(24, dtype=torch.float32).reshape(8, 3) + scale_inv = torch.arange(16, dtype=torch.uint8).reshape(8, 2) + + mapped = mapper._duplicate_kv_weights( + module, + "k_proj", + { + "weight": weight, + "weight_scale_inv": scale_inv, + }, + ) + + torch.testing.assert_close(mapped["weight"], _duplicate_heads(weight, repetitions=2)) + assert mapped["weight_scale_inv"] is scale_inv + + +def test_kv_scale_is_not_expanded_when_kv_heads_cover_tp() -> None: + mapper = _make_mapper(tp_size=2) + module = SimpleNamespace(quant_config=QuantConfig(quant_algo=QuantAlgo.MXFP8)) + scale_inv = torch.arange(16, dtype=torch.uint8).reshape(8, 2) + + mapped = mapper._duplicate_kv_weights( + module, + "v_proj", + {"weight_scale_inv": scale_inv}, + ) + + torch.testing.assert_close(mapped["weight_scale_inv"], scale_inv) + + +def test_gate_bias_params_map() -> None: + mapper = MiniMaxM3HfWeightMapper() + source_name = "model.layers.3.block_sparse_moe.e_score_correction_bias" + target_name = "model.layers.3.block_sparse_moe.gate.e_score_correction_bias" + bias = torch.arange(4, dtype=torch.float32) + gate_weight = torch.ones(4, 4) + + renamed = mapper.rename_by_params_map( + mapper.params_map, + { + source_name: bias, + "model.layers.3.block_sparse_moe.gate.weight": gate_weight, + }, + ) + + assert source_name not in renamed + assert renamed[target_name] is bias + assert renamed["model.layers.3.block_sparse_moe.gate.weight"] is gate_weight + + +def test_load_weights_accepts_base_mapper_without_params_map() -> None: + config = SimpleNamespace(num_key_value_heads=_NUM_KV_HEADS, num_attention_heads=8) + model_config = ModelConfig(pretrained_config=config, mapping=Mapping()) + model = object.__new__(MiniMaxM3ForCausalLM) + torch.nn.Module.__init__(model) + model.model_config = model_config + mapper = HfWeightMapper() + source_name = "model.layers.3.block_sparse_moe.e_score_correction_bias" + target_name = "model.layers.3.block_sparse_moe.gate.e_score_correction_bias" + bias = torch.arange(4, dtype=torch.float32) + + with patch.object(DecoderModelForCausalLM, "load_weights") as base_load_weights: + model.load_weights({source_name: bias}, weight_mapper=mapper) + + call_kwargs = base_load_weights.call_args.kwargs + assert call_kwargs["weight_mapper"] is mapper + renamed = mapper.rename_by_params_map(call_kwargs["params_map"], {source_name: bias}) + assert renamed[target_name] is bias diff --git a/tests/unittest/_torch/models/checkpoints/mx/test_mx_checkpoint_loader.py b/tests/unittest/_torch/models/checkpoints/mx/test_mx_checkpoint_loader.py index 9c7610e6652f..e80026d31114 100644 --- a/tests/unittest/_torch/models/checkpoints/mx/test_mx_checkpoint_loader.py +++ b/tests/unittest/_torch/models/checkpoints/mx/test_mx_checkpoint_loader.py @@ -10,9 +10,10 @@ These tests intentionally do not exercise the upstream modelexpress library. The import-failure path blocks modelexpress symbols from sys.modules so the -assertion is about our fallback behavior, not the upstream API. +assertion is about our dependency handling, not the upstream API. """ +import json import os import sys from contextlib import ExitStack @@ -28,6 +29,7 @@ Qwen3NextHfWeightMapper, ) from tensorrt_llm._torch.models.checkpoints.hf.weight_mapper import HfWeightMapper +from tensorrt_llm._torch.models.checkpoints.mx import checkpoint_loader as mx_checkpoint_loader from tensorrt_llm._torch.models.checkpoints.mx.checkpoint_loader import ( _MX_SOURCE_IDENTITY_METADATA_KEY, _MX_STAGED_TRANSFORM_PROTOCOL_VERSION, @@ -38,30 +40,53 @@ _build_mx_source_metadata, _normalize_model_identity, _resolve_mx_model_name, + _serialize_source_identity, +) +from tensorrt_llm._torch.weight_sharing import ( + ARTIFACT_IDENTITY_FORMAT_VERSION, + SOURCE_IDENTITY_FORMAT_VERSION, + ArtifactIdentity, + SourceIdentity, ) -from tensorrt_llm._torch.weight_sharing import SourceIdentity _MISSING = object() -def _identity(rank: int = 0) -> SourceIdentity: +def _identity(rank: int = 0, suffix: str = "same") -> SourceIdentity: return SourceIdentity( - format_version=1, - model_fingerprint="model", - quant_fingerprint="quant", - backend_fingerprint="backend", - parallel_fingerprint="parallel", + format_version=SOURCE_IDENTITY_FORMAT_VERSION, + artifact_identity=ArtifactIdentity( + format_version=ARTIFACT_IDENTITY_FORMAT_VERSION, + scheme="checkpoint_manifest_sha256", + digest="0" * 64, + ), + model_fingerprint=f"model-{suffix}", + quant_fingerprint=f"quant-{suffix}", + backend_fingerprint=f"backend-{suffix}", + parallel_fingerprint=f"parallel-{suffix}", rank=rank, - shard_fingerprint=f"shard-{rank}", + shard_fingerprint=f"shard-{rank}-{suffix}", model_name="TinyLlama/TinyLlama-1.1B-Chat-v1.0", ) -def _source_instance(identity: SourceIdentity, *, post_transform: bool = True): - metadata = _build_mx_source_metadata(identity) - if not post_transform: - metadata[_MX_WEIGHT_LAYOUT_METADATA_KEY] = "pre_transform" - return SimpleNamespace(metadata=metadata) +def _source_identity(rank=0, suffix="same"): + return _identity(rank=rank, suffix=suffix) + + +def _source_instance( + identity: SourceIdentity | None = None, *, post_transform: bool = True, rank=0 +): + if identity is not None: + metadata = _build_mx_source_metadata(identity) + if not post_transform: + metadata[_MX_WEIGHT_LAYOUT_METADATA_KEY] = "pre_transform" + return SimpleNamespace(metadata=metadata, worker_rank=identity.rank) + return SimpleNamespace( + mx_source_id=f"source-{rank}", + worker_id=f"worker-{rank}", + worker_rank=rank, + ) # --------------------------------------------------------------------------- @@ -160,7 +185,7 @@ def test_unknown_arch_uses_default_mx_mapper(self): class TestLoadWeightsFallback: """Disk-fallback paths that should not touch the upstream MX library. - All four fallback triggers share the same observable contract: + All fallback triggers share the same observable contract: is_weights_preloaded() stays False, HfCheckpointLoader.load_weights is invoked exactly once, and its return value is propagated unchanged. """ @@ -174,11 +199,6 @@ def _no_url(stack): # noqa: ARG004 - stack unused for this trigger. def _no_model(stack): # noqa: ARG004 return MXCheckpointLoader(mx_server_url="http://mx:8001"), {} - @staticmethod - def _modelexpress_unavailable(stack): - stack.enter_context(_block_modelexpress()) - return (MXCheckpointLoader(mx_server_url="http://mx:8001"), {"model": MagicMock()}) - @staticmethod def _upstream_raises(stack): identity = _identity() @@ -190,18 +210,29 @@ def _upstream_raises(stack): stack.enter_context(_install_fake_modelexpress(fake_mx)) return (loader, {"model": MagicMock(), "source_identity": identity}) + @staticmethod + def _source_probe_raises(stack): + identity = _identity() + loader = MXCheckpointLoader(mx_server_url="http://mx:8001") + fake_mx = _build_fake_modelexpress() + fake_mx.trtllm_live_transfer.MxClient.return_value.list_sources.side_effect = RuntimeError( + "server unavailable" + ) + stack.enter_context(_install_fake_modelexpress(fake_mx)) + return (loader, {"model": MagicMock(), "source_identity": identity}) + @pytest.mark.parametrize( "trigger_id, setup", [ ("no_mx_server_url", _no_url), ("no_model_kwarg", _no_model), - ("modelexpress_not_installed", _modelexpress_unavailable), + ("source_probe_raises", _source_probe_raises), ("upstream_raises", _upstream_raises), ], ids=[ "no-mx-server-url", "no-model-kwarg", - "modelexpress-not-installed", + "source-probe-raises", "upstream-raises", ], ) @@ -224,6 +255,25 @@ def test_falls_back_to_disk(self, trigger_id, setup): ) mock_super_load.assert_called_once() + def test_missing_modelexpress_client_fails_with_install_hint(self): + loader = MXCheckpointLoader(mx_server_url="http://mx:8001") + with ( + _block_modelexpress(), + patch.object(HfCheckpointLoader, "load_weights") as mock_super_load, + pytest.raises(ImportError) as exc_info, + ): + loader.load_weights( + "/nonexistent", + mapping=MagicMock(), + model=MagicMock(), + ) + + message = str(exc_info.value) + assert 'pip install "tensorrt-llm[mx]"' in message + assert "select a different `checkpoint_format`" in message + assert loader.is_weights_preloaded() is False + mock_super_load.assert_not_called() + # --------------------------------------------------------------------------- # load_weights: MX-success and mixed-success paths with mocked upstream. @@ -580,6 +630,24 @@ def test_publish_called_with_model(self): ) assert _MX_SOURCE_IDENTITY_METADATA_KEY in metadata + def test_publish_synchronizes_cuda_before_exposing_source(self, monkeypatch): + events = [] + monkeypatch.setattr( + mx_checkpoint_loader, + "_synchronize_cuda_for_mx_publish", + lambda: events.append("synchronize"), + ) + + def _publish(*_args, **_kwargs): + events.append("publish") + + loader = MXCheckpointLoader(mx_server_url="http://mx:8001") + fake_mx = _build_fake_modelexpress(publish_side_effect=_publish) + with _install_fake_modelexpress(fake_mx): + loader.publish_as_source(MagicMock(), source_identity=_identity()) + + assert events == ["synchronize", "publish"] + def test_source_identity_required_for_post_transform_publish(self): loader = MXCheckpointLoader(mx_server_url="http://mx:8001") fake_mx = _build_fake_modelexpress() @@ -589,18 +657,32 @@ def test_source_identity_required_for_post_transform_publish(self): fake_mx.trtllm_live_transfer.publish_model_params.assert_not_called() - def test_publish_skipped_when_metadata_unsupported(self): + def test_publish_without_metadata_kwarg_uses_identity_metadata(self): loader = MXCheckpointLoader(mx_server_url="http://mx:8001") calls = [] + captured = {} def _publish_without_metadata(model): calls.append(model) + captured["identity"] = fake_mx.trtllm_live_transfer._build_trtllm_identity( + model_name="local-model" + ) fake_mx = _build_fake_modelexpress(publish_model_params=_publish_without_metadata) + source_identity = _identity() with _install_fake_modelexpress(fake_mx): - loader.publish_as_source(MagicMock(), source_identity=_identity()) - - assert calls == [] + model = MagicMock() + loader.publish_as_source(model, source_identity=source_identity) + + assert calls == [model] + metadata = captured["identity"].extra_parameters + serialized_identity = json.loads(metadata[_MX_SOURCE_IDENTITY_METADATA_KEY]) + assert "model_name" not in serialized_identity + assert SourceIdentity.from_dict(serialized_identity).matches(source_identity).matched + assert metadata[_MX_WEIGHT_LAYOUT_METADATA_KEY] == _MX_WEIGHT_LAYOUT_POST_TRANSFORM + assert metadata[_MX_TRANSFORM_PROTOCOL_VERSION_METADATA_KEY] == str( + _MX_STAGED_TRANSFORM_PROTOCOL_VERSION + ) def test_env_var_set_during_publish_then_restored(self): loader = MXCheckpointLoader(mx_server_url="http://mx-instance:9999") @@ -648,6 +730,48 @@ def test_publish_exception_swallowed(self): with _install_fake_modelexpress(fake_mx): loader.publish_as_source(MagicMock(), source_identity=_identity()) # must not raise + def test_publish_attaches_trtllm_source_identity_to_mx_identity(self): + source_identity = _source_identity() + loader = MXCheckpointLoader(mx_server_url="http://mx:8001") + loader._local_source_identity = source_identity + captured = {} + + def _publish_side_effect(model, **_kwargs): + identity = fake_mx.trtllm_live_transfer._build_trtllm_identity(model_name="local-model") + captured["identity"] = identity + + fake_mx = _build_fake_modelexpress(publish_side_effect=_publish_side_effect) + + with _install_fake_modelexpress(fake_mx): + loader.publish_as_source( + MagicMock(), + checkpoint_dir="/scratch/local-model", + source_identity=source_identity, + ) + + serialized = captured["identity"].extra_parameters["trtllm_source_identity"] + published_identity = SourceIdentity.from_dict(json.loads(serialized)) + assert published_identity.model_name is None + assert published_identity.matches(source_identity).matched + assert ( + captured["identity"].extra_parameters[_MX_WEIGHT_LAYOUT_METADATA_KEY] + == _MX_WEIGHT_LAYOUT_POST_TRANSFORM + ) + assert captured["identity"].extra_parameters[ + _MX_TRANSFORM_PROTOCOL_VERSION_METADATA_KEY + ] == str(_MX_STAGED_TRANSFORM_PROTOCOL_VERSION) + + def test_serialized_identity_ignores_local_checkpoint_path(self): + donor_identity = _identity() + receiver_payload = donor_identity.to_dict() + receiver_payload["model_name"] = "/tmp/no-shards/TinyLlama" + receiver_identity = SourceIdentity.from_dict(receiver_payload) + + assert donor_identity.model_name != receiver_identity.model_name + assert _serialize_source_identity(donor_identity) == _serialize_source_identity( + receiver_identity + ) + # --------------------------------------------------------------------------- # Helpers — fake modelexpress modules and import blockers @@ -711,7 +835,9 @@ def _build_fake_modelexpress( if source_metadata is not None: client_instance.get_source_metadata.return_value = source_metadata fake_trtllm_live.MxClient = MagicMock(return_value=client_instance) - fake_trtllm_live._build_trtllm_identity = MagicMock(return_value=MagicMock()) + fake_trtllm_live._build_trtllm_identity = MagicMock( + return_value=SimpleNamespace(extra_parameters={}) + ) # publish_model_params(model) if publish_model_params is not None: @@ -767,8 +893,6 @@ def _isolated_env(self, monkeypatch): def test_no_registered_source_gets_short_default_during_load(self): identity = _identity() - source_metadata = _build_mx_source_metadata(identity) - source_metadata[_MX_WEIGHT_LAYOUT_METADATA_KEY] = "pre_transform" def _assert_timeout(*args, **kwargs): assert os.environ.get("MX_SOURCE_QUERY_TIMEOUT") == "30" @@ -777,7 +901,6 @@ def _assert_timeout(*args, **kwargs): loader = MXCheckpointLoader(mx_server_url="http://mx:8001") fake_mx = _build_fake_modelexpress( load_weights_side_effect=_assert_timeout, - source_metadata=source_metadata, ) with _install_fake_modelexpress(fake_mx): loader.load_weights( @@ -785,7 +908,33 @@ def _assert_timeout(*args, **kwargs): mapping=MagicMock(), model=MagicMock(), source_identity=identity, + allow_post_transform_weights=True, + prepare_post_transform_receiver=lambda _model: None, ) + mx_loader = fake_mx.trtllm_live_transfer.MxLiveWeightLoader.return_value + mx_loader.load_weights.assert_called_once() + assert "MX_SOURCE_QUERY_TIMEOUT" not in os.environ + + def test_no_registered_source_honors_configured_timeout(self): + identity = _identity() + + def _assert_timeout(*args, **kwargs): + assert os.environ.get("MX_SOURCE_QUERY_TIMEOUT") == "900" + return {} + + loader = MXCheckpointLoader(mx_server_url="http://mx:8001", query_timeout_s=900) + fake_mx = _build_fake_modelexpress(load_weights_side_effect=_assert_timeout) + with _install_fake_modelexpress(fake_mx): + loader.load_weights( + "/nonexistent", + mapping=MagicMock(), + model=MagicMock(), + source_identity=identity, + allow_post_transform_weights=True, + prepare_post_transform_receiver=lambda _model: None, + ) + mx_loader = fake_mx.trtllm_live_transfer.MxLiveWeightLoader.return_value + mx_loader.load_weights.assert_called_once() assert "MX_SOURCE_QUERY_TIMEOUT" not in os.environ def test_existing_source_keeps_upstream_default_when_unset(self): @@ -965,6 +1114,39 @@ def test_explicit_arg_normalized_too(self): assert _resolve_mx_model_name("/scratch/explicit-path", "/cache/ignored") == "explicit-path" +class TestLoadWeightsModelName: + def test_uses_resolved_model_name_during_load_and_restores_env(self, monkeypatch): + monkeypatch.setenv("MODEL_NAME", "prior-model") + identity = _identity() + snapshot = "/cache/hub/models--Other--Model/snapshots/abc123" + + def _assert_model_name(*args, **kwargs): + assert os.environ.get("MODEL_NAME") == "Qwen/Qwen2.5-72B-Instruct" + return {} + + loader = MXCheckpointLoader( + mx_server_url="http://mx:8001", + model_name="Qwen/Qwen2.5-72B-Instruct", + ) + fake_mx = _build_fake_modelexpress( + load_weights_side_effect=_assert_model_name, + source_instances=[_source_instance(identity, post_transform=False)], + ) + + with _install_fake_modelexpress(fake_mx): + loader.load_weights( + snapshot, + mapping=MagicMock(), + model=MagicMock(), + source_identity=identity, + ) + + assert os.environ.get("MODEL_NAME") == "prior-model" + fake_mx.trtllm_live_transfer._build_trtllm_identity.assert_called_with( + model_name="Qwen/Qwen2.5-72B-Instruct" + ) + + class TestPublishAsSourceModelName: """publish_as_source sets MODEL_NAME for upstream publish_model_params. diff --git a/tests/unittest/_torch/models/test_minimax_m3.py b/tests/unittest/_torch/models/test_minimax_m3.py index ca2ed046e57c..ecf64099d77d 100644 --- a/tests/unittest/_torch/models/test_minimax_m3.py +++ b/tests/unittest/_torch/models/test_minimax_m3.py @@ -32,9 +32,13 @@ from utils.llm_data import llm_models_root from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.models.checkpoints.hf.minimaxm3_weight_mapper import ( + MiniMaxM3HfWeightMapper, +) from tensorrt_llm._torch.models.modeling_minimaxm3 import ( MiniMaxM3Attention, _build_swiglu_oai_dense_mlp, + _minimax_m3_swiglu_oai, _strip_language_model_prefix, _wrap_dict_as_config, get_moe_layer_ids, @@ -43,7 +47,7 @@ get_text_config, is_minimax_m3_vl_config, ) -from tensorrt_llm._torch.models.modeling_utils import _load_weights_impl +from tensorrt_llm._torch.models.modeling_utils import _load_weights_impl_v2 from tensorrt_llm._torch.modules.fused_moe.routing import ( MiniMaxM2MoeRoutingMethod, MiniMaxM3MoeRoutingMethod, @@ -276,17 +280,11 @@ def test_get_moe_layer_ids_length_mismatch_raises(): # ``use_gemma=True`` and ``hidden_size=head_dim``; the # :meth:`apply_qk_norm` reshape matches an independent hand-written # reference. -# * Sparse index branch: ``index_q_proj`` is column-parallel and -# projects to ``num_index_heads * sparse_index_dim``; -# ``index_k_proj`` is **replicated** (tp_mode is None) and projects -# to **only** ``sparse_index_dim`` (single K per token, broadcast -# across all index heads for block-selection scoring) — this is the -# SGLang reference contract, confirmed by the M3 checkpoint shape -# ``(sparse_index_dim, hidden_size)``. +# * Sparse index branch: fused replicated (tp_mode None) index_qk_proj with +# output [idx_q | idx_k] = num_index_heads * sparse_index_dim + sparse_index_dim +# (idx_k is one K per token). # * Dense layers do not expose any index branch attributes (negative # control). -# * Real M3 checkpoint shape for ``index_k_proj.weight`` is -# ``(sparse_index_dim, hidden_size)`` = ``(128, 6144)``. def _make_attention_test_config(): @@ -401,6 +399,7 @@ def test_minimax_m3_attention_dense_construction_matches_config(): for name in ( "index_q_proj", "index_k_proj", + "index_qk_proj", "index_q_norm", "index_k_norm", ): @@ -478,18 +477,13 @@ def test_minimax_m3_attention_apply_qk_norm_matches_reference(): @pytest.mark.gpu @pytest.mark.skipif(not _has_cuda(), reason="MiniMax-M3 attention construction needs CUDA") def test_minimax_m3_attention_sparse_construction_matches_config(): - """Sparse layer adds index branch with the SGLang-correct shapes. - - Verifies the **bug fix** from the iter-4 work: - * ``index_q_proj`` is column-parallel and projects to - ``num_index_heads * sparse_index_dim``. - * ``index_k_proj`` is replicated (``tp_mode is None``) and projects - to **only** ``sparse_index_dim`` — a single replicated K per - token, *not* per-head. This matches SGLang's ``ReplicatedLinear`` - and the M3 checkpoint's ``index_k_proj.weight`` shape - ``(sparse_index_dim, hidden_size)``. - * ``index_q_norm`` / ``index_k_norm`` are per-head Gemma RMSNorm - of width ``sparse_index_dim``. + """Sparse layer adds the index branch with the fused index projection. + + * index_qk_proj is replicated (tp_mode None), out = + num_index_heads * sparse_index_dim (idx_q) + sparse_index_dim (idx_k), + where idx_k is one K per token (SGLang ReplicatedLinear contract). + * index_q_norm / index_k_norm are per-head Gemma RMSNorm of width + sparse_index_dim. """ text_cfg, model_cfg = _make_attention_test_config() sparse_cfg = text_cfg.sparse_attention_config @@ -506,33 +500,19 @@ def test_minimax_m3_attention_sparse_construction_matches_config(): assert attn.is_sparse_attention_layer is True assert attn.disable_index_value is True - # index_q_proj: per-head Q for the index branch. As of iter-15 this - # is **replicated** (tp_mode=None) across TP ranks, not - # column-parallel: the sparse forward consumes ``idx_q`` reshaped to - # ``[num_tokens, num_index_heads, sparse_index_dim]`` and a - # column-parallel split would slice the head dimension (breaking the - # reshape at any ``tp_size > num_index_heads`` geometry, including - # the TP=8 configuration the real-checkpoint smoke test now uses). - # The replicated weight is small (~3 MiB BF16) so the per-rank - # memory cost is negligible. - assert attn.index_q_proj.in_features == hidden - assert attn.index_q_proj.out_features == num_index_heads * sparse_index_dim - assert attn.index_q_proj.tp_mode is None, ( - f"index_q_proj must be replicated (tp_mode=None) so the sparse " - f"forward's `idx_q.view(num_tokens, num_index_heads, sparse_index_dim)` " - f"reshape is well-defined at any TP geometry, got " - f"{attn.index_q_proj.tp_mode!r}" - ) - - # index_k_proj: REPLICATED, only sparse_index_dim outputs. - assert attn.index_k_proj.in_features == hidden - assert attn.index_k_proj.out_features == sparse_index_dim, ( - f"index_k_proj.out_features must be sparse_index_dim={sparse_index_dim}, " - f"got {attn.index_k_proj.out_features} (regression of the iter-4 fix)" - ) - assert attn.index_k_proj.tp_mode is None, ( - f"index_k_proj must be replicated (tp_mode=None), got {attn.index_k_proj.tp_mode!r}" + # Replication keeps the idx_q -> [num_tokens, num_index_heads, + # sparse_index_dim] reshape valid at any TP geometry, whereas a + # column-parallel split would slice the head dimension. + assert attn.index_q_size == num_index_heads * sparse_index_dim + assert attn.index_k_size == sparse_index_dim + assert attn.index_qk_proj.in_features == hidden + assert attn.index_qk_proj.out_features == num_index_heads * sparse_index_dim + sparse_index_dim + assert attn.index_qk_proj.tp_mode is None, ( + f"index_qk_proj must be replicated, got {attn.index_qk_proj.tp_mode!r}" ) + # Only the fused projection exists. + assert not hasattr(attn, "index_q_proj") + assert not hasattr(attn, "index_k_proj") # Per-head Gemma RMSNorm of width sparse_index_dim. assert attn.index_q_norm.use_gemma is True @@ -625,17 +605,10 @@ def test_minimax_m3_attention_dense_apply_index_qk_norm_raises(): def test_minimax_m3_attention_real_config_index_branch_shapes(): """Real M3 config → sparse-layer index branch has the checkpoint's shapes. - Asserts the iter-4 fix in numbers: - * ``index_q_proj.out_features == 512`` (= 4 * 128 - = ``num_index_heads * sparse_index_dim``). - * ``index_k_proj.out_features == 128`` (= ``sparse_index_dim``) - and ``tp_mode is None`` (replicated). The real - ``index_k_proj.weight`` in the checkpoint has shape - ``(128, 6144)``; a regression to the old - ``num_index_heads * sparse_index_dim`` (512) would break weight - loading at runtime. - * ``index_q_norm.weight.shape == (128,)`` and - ``index_k_norm.weight.shape == (128,)``. + Asserts the fused index projection in numbers: + * index_qk_proj.out_features == 640 (4 * 128 + 128), replicated. Source + weights (512, 6144) + (128, 6144) merge into (640, 6144) at load time. + * index_q_norm / index_k_norm weights have shape (128,). """ pytest.importorskip("transformers") cfg = AutoConfig.from_pretrained(_checkpoint_path(), trust_remote_code=True) @@ -668,22 +641,13 @@ def test_minimax_m3_attention_real_config_index_branch_shapes(): assert num_index_heads == 4 assert sparse_index_dim == 128 - # index_q_proj: 4 * 128 = 512 out, replicated (tp_mode=None) as of - # iter-15. The downstream sparse forward reshapes ``idx_q`` to - # ``[num_tokens, num_index_heads, sparse_index_dim]``; a - # column-parallel split would slice the head dimension and break - # that reshape at any ``tp_size > num_index_heads`` geometry - # (including TP=8 used by the real-checkpoint smoke test). The - # replicated weight is ~3 MiB BF16 — the per-rank memory cost is - # negligible. - assert attn.index_q_proj.in_features == int(text_cfg.hidden_size) - assert attn.index_q_proj.out_features == num_index_heads * sparse_index_dim - assert attn.index_q_proj.tp_mode is None - - # index_k_proj: 128 out (NOT 512), replicated. - assert attn.index_k_proj.in_features == int(text_cfg.hidden_size) - assert attn.index_k_proj.out_features == sparse_index_dim - assert attn.index_k_proj.tp_mode is None + assert attn.index_q_size == num_index_heads * sparse_index_dim + assert attn.index_k_size == sparse_index_dim + assert attn.index_qk_proj.in_features == int(text_cfg.hidden_size) + assert attn.index_qk_proj.out_features == num_index_heads * sparse_index_dim + sparse_index_dim + assert attn.index_qk_proj.tp_mode is None + assert not hasattr(attn, "index_q_proj") + assert not hasattr(attn, "index_k_proj") # Per-head Gemma index norms: width sparse_index_dim. assert tuple(attn.index_q_norm.weight.shape) == (sparse_index_dim,) @@ -761,10 +725,89 @@ def test_minimax_m3_routing_method_default_scale_is_identity(): torch.testing.assert_close(same_weights, base_weights, rtol=0, atol=0) +@pytest.mark.gpu +@pytest.mark.skipif(not _has_cuda(), reason="fused MiniMax-M3 routing requires CUDA") +@pytest.mark.parametrize("num_tokens", [1, 64, 8192]) +def test_minimax_m3_fused_routing_matches_reference( + num_tokens: int, monkeypatch: pytest.MonkeyPatch +) -> None: + num_experts = 128 + top_k = 4 + routed_scaling_factor = 2.0 + logits = torch.full((num_tokens, num_experts), -4.0, device="cuda", dtype=torch.float32) + token_offsets = torch.arange(num_tokens, device="cuda", dtype=torch.int64).unsqueeze(1) + expert_offsets = torch.arange(top_k, device="cuda", dtype=torch.int64).unsqueeze(0) + selected_experts = (token_offsets + expert_offsets) % num_experts + selected_logits = torch.tensor([4.0, 3.0, 2.0, 1.0], device="cuda").expand(num_tokens, -1) + logits.scatter_(1, selected_experts, selected_logits) + bias = torch.linspace(-0.01, 0.01, num_experts, device="cuda", dtype=torch.float32) + + fused = MiniMaxM3MoeRoutingMethod( + top_k=top_k, + num_experts=num_experts, + callable_e_score_correction_bias=lambda: bias, + routed_scaling_factor=routed_scaling_factor, + ) + + scores = torch.sigmoid(logits) + _, reference_idx = torch.topk(scores + bias, k=top_k, dim=-1, sorted=False) + reference_weights = scores.gather(1, reference_idx) + reference_idx = reference_idx.to(torch.int32) + reference_weights = ( + reference_weights / (reference_weights.sum(dim=-1, keepdim=True) + 1e-20) + ) * routed_scaling_factor + + monkeypatch.setattr( + MiniMaxM2MoeRoutingMethod, + "apply", + lambda *_args, **_kwargs: pytest.fail("production FP32 routing used the PyTorch fallback"), + ) + fused_idx, fused_weights = fused.apply(logits) + + reference_order = reference_idx.argsort(dim=-1) + fused_order = fused_idx.argsort(dim=-1) + reference_idx = reference_idx.gather(1, reference_order) + reference_weights = reference_weights.gather(1, reference_order) + fused_idx = fused_idx.gather(1, fused_order) + fused_weights = fused_weights.gather(1, fused_order) + + torch.testing.assert_close(fused_idx, reference_idx, rtol=0, atol=0) + torch.testing.assert_close(fused_weights, reference_weights, rtol=1e-5, atol=1e-6) + + +@pytest.mark.gpu +@pytest.mark.skipif(not _has_cuda(), reason="fused MiniMax-M3 routing requires CUDA") +def test_minimax_m3_fused_routing_cuda_graph_replay_tracks_inputs() -> None: + num_tokens = 64 + num_experts = 128 + torch.manual_seed(1) + logits = torch.empty(num_tokens, num_experts, device="cuda", dtype=torch.float32) + bias = torch.randn(num_experts, device="cuda", dtype=torch.float32) * 0.1 + routing = MiniMaxM3MoeRoutingMethod( + top_k=4, + num_experts=num_experts, + callable_e_score_correction_bias=lambda: bias, + routed_scaling_factor=2.0, + ) + + logits.normal_() + routing.apply(logits) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + graph_idx, graph_weights = routing.apply(logits) + + logits.normal_(mean=1.0, std=2.0) + reference_idx, reference_weights = routing.apply(logits.clone()) + graph.replay() + + torch.testing.assert_close(graph_idx, reference_idx, rtol=0, atol=0) + torch.testing.assert_close(graph_weights, reference_weights, rtol=0, atol=0) + + @pytest.mark.gpu @pytest.mark.skipif(not _has_cuda(), reason="MiniMax-M3 needs CUDA") -def test_text_norm_weights_real_loader_smoke(): - """real ``_load_weights_impl`` populates norm parameters. +def test_text_norm_weights_real_loader_smoke(monkeypatch: pytest.MonkeyPatch): + """real ``_load_weights_impl_v2`` populates norm parameters. Constructs a memory-safe stub containing the top-level ``model.norm`` and the first decoder layer's ``input_layernorm`` and @@ -772,7 +815,7 @@ def test_text_norm_weights_real_loader_smoke(): :class:`RMSNorm`, ~12 KB on CUDA), reads the corresponding tensors from the real checkpoint via ``safetensors``, strips the ``language_model.`` prefix exactly as the M3 VL wrapper does, and - invokes :func:`_load_weights_impl` end-to-end. The test fails if any + invokes :func:`_load_weights_impl_v2` end-to-end. The test fails if any target parameter remains at its zero-initialisation, proving the canonical loader walks the module tree and copies the correct source keys for these BF16 parameters. @@ -871,17 +914,21 @@ def __init__(self) -> None: "model.layers.0.post_attention_layernorm.weight", } - # Invoke the canonical loader. `_load_weights_impl` walks the stub's + # Invoke the canonical loader. `_load_weights_impl_v2` walks the stub's # module tree and uses the generic per-parameter copy fallback because # RMSNorm does not define ``load_weights``. Disable the parallel # executor so a failure surfaces immediately rather than as a thread # traceback (the parallel path is exercised in production; for this # tiny 3-module slice the serial walk is what the test should observe). - os.environ["TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL"] = "True" - try: - _load_weights_impl(stub, text_weights, allow_partial_loading=True) - finally: - os.environ.pop("TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL", None) + monkeypatch.setenv("TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL", "True") + weight_mapper = MiniMaxM3HfWeightMapper() + weight_mapper.init_model_and_config(stub, stub.model_config) + _load_weights_impl_v2( + stub, + text_weights, + weight_mapper, + allow_partial_loading=True, + ) # The three norms should now hold the source tensors' values. torch.testing.assert_close( @@ -971,3 +1018,44 @@ def test_minimax_m3_swiglu_oai_dense_mlp_under_adp_is_replicated(): "ADP down_proj must skip the cross-rank all-reduce; otherwise it " "mixes outputs across independent rank-local token sets" ) + + +# --------------------------------------------------------------------------- +# Fused SwiGLU-OAI numeric equivalence +# --------------------------------------------------------------------------- +# +# The dense MLP and MoE shared expert express swigluoai as plain SwiGLU with +# (alpha, beta, limit) so it routes through the fused silu_and_mul kernel. + + +@pytest.mark.gpu +@pytest.mark.skipif(not _has_cuda(), reason="fused silu_and_mul Triton kernel requires CUDA") +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) +def test_minimax_m3_swiglu_oai_fused_matches_reference(dtype): + """Fused silu_and_mul with (alpha, beta, limit) matches the eager reference. + + Also checks that alpha=1 and beta=0 recover plain SwiGLU. + """ + from tensorrt_llm._torch.modules.swiglu import swiglu + + torch.manual_seed(0) + alpha, limit = 1.702, 7.0 + # Wide range so both clamp branches (gate upper, up symmetric) are hit. + gate_up = torch.randn(64, 2 * 128, device="cuda", dtype=dtype) * 6.0 + + ref = _minimax_m3_swiglu_oai(gate_up, alpha=alpha, limit=limit) + fused = swiglu(gate_up, swiglu_alpha=alpha, swiglu_beta=1.0, swiglu_limit=limit) + + assert fused.shape == ref.shape + assert fused.dtype == gate_up.dtype + # fp32 accumulation inside the kernel; loose tol for bf16 rounding. + atol = 2e-2 if dtype == torch.bfloat16 else 1e-4 + torch.testing.assert_close(fused.float(), ref.float(), atol=atol, rtol=1e-2) + + # alpha=1 / beta=0 reduces to plain SwiGLU: silu(gate_clamped) * up_clamped. + gate, up = gate_up.chunk(2, dim=-1) + gate_c = gate.clamp(max=limit) + up_c = up.clamp(min=-limit, max=limit) + plain_ref = torch.nn.functional.silu(gate_c) * up_c + plain = swiglu(gate_up, swiglu_limit=limit) + torch.testing.assert_close(plain.float(), plain_ref.float(), atol=atol, rtol=1e-2) diff --git a/tests/unittest/_torch/models/test_qwen3_next_eager_fusion.py b/tests/unittest/_torch/models/test_qwen3_next_eager_fusion.py new file mode 100644 index 000000000000..e0a73d9c1f05 --- /dev/null +++ b/tests/unittest/_torch/models/test_qwen3_next_eager_fusion.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import torch +from torch import nn + +from tensorrt_llm._torch.distributed import AllReduceFusionOp +from tensorrt_llm._torch.models.modeling_qwen3_next import ( + Qwen3NextForCausalLM, + Qwen3NextLinearDecoderLayer, + _eager_fusion_enabled, +) +from tensorrt_llm._torch.modules.rms_norm import RMSNorm + + +def _new_causal_lm() -> Qwen3NextForCausalLM: + model = Qwen3NextForCausalLM.__new__(Qwen3NextForCausalLM) + nn.Module.__init__(model) + return model + + +@torch.no_grad() +def test_setup_aliases_does_not_read_meta_weights() -> None: + model = _new_causal_lm() + model.model_config = SimpleNamespace(pretrained_config=SimpleNamespace(num_hidden_layers=2)) + model.model = nn.Module() + model.model.layers = nn.ModuleList([nn.Module(), nn.Module()]) + for layer in model.model.layers: + layer.input_layernorm = RMSNorm( + hidden_size=4, + eps=1e-6, + dtype=torch.bfloat16, + device=torch.device("meta"), + use_gemma=True, + ) + layer.next_layer_layernorm = None + model.model.norm = RMSNorm( + hidden_size=4, eps=1e-6, dtype=torch.bfloat16, device=torch.device("meta"), use_gemma=True + ) + + model.setup_aliases() + + assert model.model.layers[0].next_layer_layernorm is model.model.layers[1].input_layernorm + assert model.model.layers[1].next_layer_layernorm is model.model.norm + assert not hasattr(model.model.norm, "_fused_norm_weight") + + +@torch.no_grad() +def test_cache_derived_state_refreshes_gemma_norm_weight() -> None: + model = _new_causal_lm() + model.gemma_norm = RMSNorm(hidden_size=4, eps=1e-6, dtype=torch.bfloat16, use_gemma=True) + model.standard_norm = RMSNorm(hidden_size=4, eps=1e-6, dtype=torch.bfloat16) + model.gemma_norm.weight.copy_(torch.tensor([-0.5, 0.0, 0.5, 1.0], dtype=torch.bfloat16)) + + model.cache_derived_state() + + expected = (model.gemma_norm.weight.float() + 1.0).to(torch.bfloat16) + # Exact: cache_derived_state bakes (1+weight) with the same fp32-add-then- + # cast recomputed here, so the result must be bitwise-identical. + torch.testing.assert_close(model.gemma_norm._fused_norm_weight, expected, atol=0.0, rtol=0.0) + assert not hasattr(model.standard_norm, "_fused_norm_weight") + + model.gemma_norm.weight.add_(1.0) + model.cache_derived_state() + expected = (model.gemma_norm.weight.float() + 1.0).to(torch.bfloat16) + # Exact: cache_derived_state bakes (1+weight) with the same fp32-add-then- + # cast recomputed here, so the result must be bitwise-identical. + torch.testing.assert_close(model.gemma_norm._fused_norm_weight, expected, atol=0.0, rtol=0.0) + + +@torch.no_grad() +def test_eager_fusion_is_enabled_for_gdn_by_default(monkeypatch) -> None: + monkeypatch.delenv("TRTLLM_QWEN3_EAGER_FUSION_DISABLED", raising=False) + assert _eager_fusion_enabled(enable_attention_dp=False) + assert not _eager_fusion_enabled(enable_attention_dp=True) + + monkeypatch.setenv("TRTLLM_QWEN3_EAGER_FUSION_DISABLED", "1") + assert not _eager_fusion_enabled(enable_attention_dp=False) + + +@torch.no_grad() +def test_gdn_fusion_has_single_allreduce_owner() -> None: + hidden_states = torch.randn(2, 4, dtype=torch.bfloat16) + residual = torch.randn_like(hidden_states) + # Use real RMSNorm modules (lightweight) rather than mocks: their + # weight / use_gemma / variance_epsilon are exactly what the fused-norm + # path reads. The linear_attn/allreduce/mlp below stay mocks because the + # test asserts on *how they are called* (call_count / call_args). + post_attention_norm = RMSNorm(hidden_size=4, eps=1e-6, dtype=torch.bfloat16, use_gemma=True) + next_layer_norm = RMSNorm(hidden_size=4, eps=1e-6, dtype=torch.bfloat16, use_gemma=True) + post_attention_norm.weight.copy_(torch.tensor([-0.5, 0.0, 0.5, 1.0], dtype=torch.bfloat16)) + next_layer_norm.weight.copy_(torch.tensor([0.0, 0.25, 0.5, 0.75], dtype=torch.bfloat16)) + linear_attn = MagicMock(side_effect=lambda hidden_states, *args, **kwargs: hidden_states) + allreduce = MagicMock( + side_effect=lambda hidden_states, *, all_reduce_params: ( + hidden_states, + all_reduce_params.residual, + ) + ) + mlp = MagicMock(side_effect=lambda hidden_states, *args, **kwargs: hidden_states) + layer = SimpleNamespace( + layer_idx=0, + input_layernorm=MagicMock(), + linear_attn=linear_attn, + post_attention_layernorm=post_attention_norm, + next_layer_layernorm=next_layer_norm, + fusion_config=SimpleNamespace(PRE_MOE_FUSION=True, POST_MOE_FUSION=True), + disable_attn_allreduce=True, + allreduce=allreduce, + mlp=mlp, + mapping=SimpleNamespace(tp_size=2), + moe_allreduce=MagicMock(), + ) + + Qwen3NextLinearDecoderLayer.forward( + layer, + position_ids=torch.arange(2), + hidden_states=hidden_states, + attn_metadata=SimpleNamespace(), + residual=residual, + ) + + internal_ar_params = linear_attn.call_args.kwargs["all_reduce_params"] + assert not internal_ar_params.enable_allreduce + + # Two module-level allreduces at the layer boundary: pre-MoE and post-MoE. + # The GDN linear_attn's own allreduce is disabled (single-owner, asserted + # just above), so it does not add a third. + assert allreduce.call_count == 2 + pre_ar_params = allreduce.call_args_list[0].kwargs["all_reduce_params"] + post_ar_params = allreduce.call_args_list[1].kwargs["all_reduce_params"] + assert pre_ar_params.enable_allreduce + assert post_ar_params.enable_allreduce + assert pre_ar_params.fusion_op == AllReduceFusionOp.RESIDUAL_RMS_NORM + assert post_ar_params.fusion_op == AllReduceFusionOp.RESIDUAL_RMS_NORM + # Exact: norm_weight is the same baked (1+weight) recomputed here. + torch.testing.assert_close( + pre_ar_params.norm_weight, + (post_attention_norm.weight.float() + 1.0).to(torch.bfloat16), + atol=0.0, + rtol=0.0, + ) + torch.testing.assert_close( + post_ar_params.norm_weight, + (next_layer_norm.weight.float() + 1.0).to(torch.bfloat16), + atol=0.0, + rtol=0.0, + ) + + mlp_ar_params = mlp.call_args.kwargs["all_reduce_params"] + assert not mlp_ar_params.enable_allreduce + assert mlp.call_args.kwargs["do_finalize"] diff --git a/tests/unittest/_torch/models/test_qwen3_next_moe_quant.py b/tests/unittest/_torch/models/test_qwen3_next_moe_quant.py new file mode 100644 index 000000000000..fb4e190f1dd0 --- /dev/null +++ b/tests/unittest/_torch/models/test_qwen3_next_moe_quant.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unquantized-MoE probe for the Qwen3Next / Qwen3.5 MTP layer.""" + +from types import SimpleNamespace + +import pytest + +from tensorrt_llm._torch.models.modeling_qwen3_5 import _normalize_qwen35_exclude_modules +from tensorrt_llm._torch.models.modeling_qwen3_next import _experts_excluded_from_quant +from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo + +NUM_HIDDEN_LAYERS = 40 +# MTP layers are appended after the regular ones (MTPForCausalLM start_layer_idx). +MTP_LAYER_IDX = NUM_HIDDEN_LAYERS + + +def _model_config(exclude_modules, quant_algo=QuantAlgo.NVFP4): + return SimpleNamespace( + quant_config=QuantConfig( + quant_algo=quant_algo, + kv_cache_quant_algo=QuantAlgo.FP8, + exclude_modules=exclude_modules, + ), + pretrained_config=SimpleNamespace( + num_hidden_layers=NUM_HIDDEN_LAYERS, + num_nextn_predict_layers=1, + ), + ) + + +# All of these cover model.layers.40.mlp.experts under QuantConfig's matcher -- exact name, +# ancestor walk (a parent entry excludes everything below it), trailing ".*", "re:" regex -- in +# either the translated TRT-LLM namespace or the raw HF one, so every one must be recognized. +@pytest.mark.parametrize( + "pattern", + [ + f"model.layers.{MTP_LAYER_IDX}", + f"model.layers.{MTP_LAYER_IDX}*", + f"model.layers.{MTP_LAYER_IDX}.*", + f"model.layers.{MTP_LAYER_IDX}.mlp", + f"model.layers.{MTP_LAYER_IDX}.mlp*", + f"model.layers.{MTP_LAYER_IDX}.mlp.*", + f"model.layers.{MTP_LAYER_IDX}.mlp.experts", + f"model.layers.{MTP_LAYER_IDX}.mlp.experts*", + rf"re:model\.layers\.{MTP_LAYER_IDX}\..*", + r"re:.*\.mlp\.experts$", + "mtp.layers.0", + "mtp.layers.0*", + "mtp.layers.0.mlp", + "mtp.layers.0.mlp*", + "mtp.layers.0.mlp.experts*", + ], +) +def test_experts_covered_by_exclusion(pattern): + model_config = _model_config([pattern, "lm_head"]) + + assert _experts_excluded_from_quant(model_config, MTP_LAYER_IDX) + + +# Leaf-level exclusions around the experts (what FP8 block-scale checkpoints emit) leave the +# experts quantized: forcing the CUTLASS fp8_blockscale GEMM there aborts on Blackwell. +@pytest.mark.parametrize( + "pattern", + [ + "mtp.fc", + "mtp.norm", + "mtp.layers.0.mlp.gate", + "mtp.layers.0.mlp.shared_expert_gate", + "mtp.layers.0.mlp.shared_expert.down_proj", + "mtp.layers.0.self_attn.q_proj", + f"model.layers.{MTP_LAYER_IDX}.mlp.gate", + f"model.layers.{MTP_LAYER_IDX}.mlp.shared_expert.down_proj", + f"model.layers.{MTP_LAYER_IDX - 1}*", + "*kv_b_proj*", + "*linear_attn.conv1d", + ], +) +def test_experts_not_covered_by_exclusion(pattern): + model_config = _model_config([pattern, "lm_head"]) + + assert not _experts_excluded_from_quant(model_config, MTP_LAYER_IDX) + + +# The exclude_modules shapes checkpoints are known to emit, run through the same normalization +# the Qwen3.5 entry points apply. ``expected`` says whether the routed experts end up bf16. +@pytest.mark.parametrize( + "shape,exclude_modules,quant_algo,expected", + [ + ("whole MTP layer wildcard", ["mtp.layers.0*", "lm_head"], QuantAlgo.NVFP4, True), + ( + "bare mtp subtree plus layer wildcard", + ["mtp*", "mtp.layers.0*"], + QuantAlgo.MIXED_PRECISION, + True, + ), + ( + "leaf-level entries only", + ["mtp.fc", "mtp.layers.0.mlp.gate", "mtp.layers.0.mlp.shared_expert_gate"], + QuantAlgo.FP8_BLOCK_SCALES, + False, + ), + ( + "leaf-level entries incl. top-level norms", + [ + "mtp.fc", + "mtp.norm", + "mtp.pre_fc_norm_embedding", + "mtp.pre_fc_norm_hidden", + "mtp.layers.0.input_layernorm", + "mtp.layers.0.mlp.gate", + ], + QuantAlgo.FP8_BLOCK_SCALES, + False, + ), + ], +) +def test_known_exclude_module_shapes(shape, exclude_modules, quant_algo, expected): + model_config = _model_config(exclude_modules, quant_algo=quant_algo) + _normalize_qwen35_exclude_modules(model_config) + + assert _experts_excluded_from_quant(model_config, MTP_LAYER_IDX) is expected, shape + + +def test_no_exclusions_needs_no_fallback(): + assert not _experts_excluded_from_quant(_model_config(None), MTP_LAYER_IDX) + + +@pytest.mark.parametrize( + "pattern,layer_idx,expected", + [ + # A regular layer's experts get the same treatment as the MTP layer's: + # create_moe must see the unquantized config, or it picks a backend the + # experts will not have once the exclusion pass runs. + ("model.layers.5.mlp.experts", 5, True), + ("model.layers.5.mlp", 5, True), + ("model.layers.5*", 5, True), + ("model.layers.5.mlp.experts", 6, False), + # The raw HF mtp.* spelling only applies to layers past the last + # regular one, so it must not leak onto a regular layer. + ("mtp.layers.0.mlp.experts", 5, False), + ("mtp.layers.0.mlp.experts", MTP_LAYER_IDX, True), + ], +) +def test_regular_layers_are_covered_too(pattern, layer_idx, expected): + model_config = _model_config([pattern, "lm_head"]) + + assert _experts_excluded_from_quant(model_config, layer_idx) is expected + + +def test_missing_layer_idx_is_a_noop(): + assert not _experts_excluded_from_quant(_model_config(["model.layers.5*"]), None) diff --git a/tests/unittest/_torch/modules/fused_ops/test_fused_qk_norm_rope_gate.py b/tests/unittest/_torch/modules/fused_ops/test_fused_qk_norm_rope_gate.py new file mode 100644 index 000000000000..f1557dbc23fd --- /dev/null +++ b/tests/unittest/_torch/modules/fused_ops/test_fused_qk_norm_rope_gate.py @@ -0,0 +1,294 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import torch + +from tensorrt_llm._torch.modules.attention import Attention +from tensorrt_llm._torch.modules.fused_ops.fused_qk_norm_rope_gate import ( + fused_qkv_gemma_rmsnorm_rope_gate, + fused_sigmoid_mul, +) + + +def _make_cos_sin(max_positions: int, rotary_dim: int, theta: float = 1_000_000.0): + inverse_frequency = 1.0 / ( + theta ** (torch.arange(0, rotary_dim, 2, dtype=torch.float32, device="cuda") / rotary_dim) + ) + positions = torch.arange(max_positions, dtype=torch.float32, device="cuda") + frequency = torch.outer(positions, inverse_frequency) + return torch.stack((frequency.cos(), frequency.sin()), dim=1).contiguous() + + +def _reference( + qkv, + q_weight, + k_weight, + cos_sin, + positions, + eps, + num_q_heads, + num_kv_heads, + head_dim, + rotary_dim, + mrope_section=None, +): + num_tokens = qkv.shape[0] + q_size = num_q_heads * head_dim + kv_size = num_kv_heads * head_dim + q_gate, k, v = qkv.split((2 * q_size, kv_size, kv_size), dim=-1) + q_gate = q_gate.view(num_tokens, num_q_heads, 2 * head_dim) + q = q_gate[..., :head_dim] + gate = q_gate[..., head_dim:] + k = k.view(num_tokens, num_kv_heads, head_dim) + + def _gemma_norm(x, weight): + x_float = x.float() + inverse_rms = torch.rsqrt(x_float.square().mean(dim=-1, keepdim=True) + eps) + return (x_float * inverse_rms * (weight.float() + 1.0)).to(qkv.dtype) + + q = _gemma_norm(q, q_weight) + k = _gemma_norm(k, k_weight) + half_rotary = rotary_dim // 2 + if mrope_section is None: + selected = cos_sin[positions.long()] + cos = selected[:, 0].unsqueeze(1) + sin = selected[:, 1].unsqueeze(1) + else: + positions = positions.reshape(3, num_tokens).long() + rotary_offsets = torch.arange(half_rotary, device=qkv.device) + sections = torch.zeros_like(rotary_offsets) + sections[(rotary_offsets % 3 == 1) & (rotary_offsets < mrope_section[1] * 3)] = 1 + sections[(rotary_offsets % 3 == 2) & (rotary_offsets < mrope_section[2] * 3)] = 2 + token_offsets = torch.arange(num_tokens, device=qkv.device).unsqueeze(1) + selected_positions = positions[sections.unsqueeze(0), token_offsets] + cos = cos_sin[selected_positions, 0, rotary_offsets].unsqueeze(1) + sin = cos_sin[selected_positions, 1, rotary_offsets].unsqueeze(1) + + def _rope(x): + x_first = x[..., :half_rotary].float() + x_second = x[..., half_rotary:rotary_dim].float() + first = x_first * cos - x_second * sin + second = x_second * cos + x_first * sin + return torch.cat((first, second, x[..., rotary_dim:].float()), dim=-1).to(qkv.dtype) + + q = _rope(q).reshape(num_tokens, q_size) + k = _rope(k).reshape(num_tokens, kv_size) + return torch.cat((q, k, v), dim=-1), gate + + +@pytest.mark.parametrize("dtype", (torch.bfloat16, torch.float16)) +@pytest.mark.parametrize( + "num_tokens,num_q_heads,num_kv_heads,head_dim,rotary_dim", + ( + (1, 8, 2, 128, 128), + (7, 8, 2, 256, 128), + (65, 16, 4, 256, 256), + (333, 16, 4, 256, 256), + ), +) +def test_fused_qkv_gemma_rmsnorm_rope_gate_matches_reference( + dtype, num_tokens, num_q_heads, num_kv_heads, head_dim, rotary_dim +): + torch.manual_seed(1234) + q_size = num_q_heads * head_dim + kv_size = num_kv_heads * head_dim + width = 2 * q_size + 2 * kv_size + storage = torch.randn((num_tokens, width + 37), dtype=dtype, device="cuda") + qkv = storage[:, :width] + q_weight = torch.randn((head_dim,), dtype=dtype, device="cuda") * 0.1 + k_weight = torch.randn((head_dim,), dtype=dtype, device="cuda") * 0.1 + cos_sin = _make_cos_sin(2048, rotary_dim) + positions = torch.randint(0, 2048, (num_tokens,), dtype=torch.int32, device="cuda") + eps = 1e-6 + + actual_qkv, actual_gate = fused_qkv_gemma_rmsnorm_rope_gate( + qkv, + q_weight, + k_weight, + cos_sin, + positions, + eps, + num_q_heads, + num_kv_heads, + head_dim, + rotary_dim, + ) + expected_qkv, expected_gate = _reference( + qkv, + q_weight, + k_weight, + cos_sin, + positions, + eps, + num_q_heads, + num_kv_heads, + head_dim, + rotary_dim, + ) + + torch.testing.assert_close(actual_qkv, expected_qkv, atol=0.02, rtol=0.02) + torch.testing.assert_close(actual_gate, expected_gate, atol=0, rtol=0) + torch.testing.assert_close(actual_qkv[:, q_size + kv_size :], qkv[:, 2 * q_size + kv_size :]) + + +@pytest.mark.parametrize("inplace", (False, True)) +def test_fused_sigmoid_mul_supports_strided_gate(inplace): + torch.manual_seed(7) + num_tokens, num_heads, head_dim = 17, 8, 128 + attention = torch.randn((num_tokens, num_heads * head_dim), dtype=torch.bfloat16, device="cuda") + gate_storage = torch.randn( + (num_tokens, num_heads, 2 * head_dim), dtype=torch.bfloat16, device="cuda" + ) + gate = gate_storage[..., :head_dim] + expected = attention.float() * torch.sigmoid(gate.reshape(num_tokens, -1).float()) + actual = fused_sigmoid_mul(attention.clone(), gate, inplace=inplace) + torch.testing.assert_close(actual.float(), expected, atol=0.02, rtol=0.02) + + +@pytest.mark.parametrize("dtype", (torch.bfloat16, torch.float16)) +def test_fused_qkv_gemma_rmsnorm_rope_gate_supports_interleaved_mrope(dtype): + torch.manual_seed(2026) + num_tokens, num_q_heads, num_kv_heads = 37, 16, 2 + head_dim, rotary_dim = 256, 64 + mrope_section = (11, 11, 10) + q_size = num_q_heads * head_dim + kv_size = num_kv_heads * head_dim + qkv = torch.randn((num_tokens, 2 * q_size + 2 * kv_size), dtype=dtype, device="cuda") + q_weight = torch.randn((head_dim,), dtype=dtype, device="cuda") * 0.1 + k_weight = torch.randn((head_dim,), dtype=dtype, device="cuda") * 0.1 + cos_sin = _make_cos_sin(4096, rotary_dim, theta=10_000_000.0) + base_positions = torch.randint(0, 4000, (num_tokens,), dtype=torch.int32, device="cuda") + positions = torch.stack( + (base_positions, base_positions + 3, base_positions + 7), dim=0 + ).unsqueeze(1) + eps = 1e-6 + + actual_qkv, actual_gate = fused_qkv_gemma_rmsnorm_rope_gate( + qkv, + q_weight, + k_weight, + cos_sin, + positions, + eps, + num_q_heads, + num_kv_heads, + head_dim, + rotary_dim, + mrope_section, + ) + expected_qkv, expected_gate = _reference( + qkv, + q_weight, + k_weight, + cos_sin, + positions, + eps, + num_q_heads, + num_kv_heads, + head_dim, + rotary_dim, + mrope_section, + ) + + torch.testing.assert_close(actual_qkv, expected_qkv, atol=0.02, rtol=0.02) + torch.testing.assert_close(actual_gate, expected_gate, atol=0, rtol=0) + + +@pytest.mark.parametrize("num_tokens", (1, 37, 333)) +def test_fused_qkv_gemma_rmsnorm_rope_gate_matches_production_thop(num_tokens): + torch.manual_seed(9027) + num_q_heads, num_kv_heads = 16, 2 + head_dim, rotary_dim = 256, 64 + mrope_section = (11, 11, 10) + q_size = num_q_heads * head_dim + kv_size = num_kv_heads * head_dim + qkv = torch.randn( + (num_tokens, 2 * q_size + 2 * kv_size), + dtype=torch.bfloat16, + device="cuda", + ) + q_weight = torch.randn((head_dim,), dtype=torch.bfloat16, device="cuda") * 0.1 + k_weight = torch.randn((head_dim,), dtype=torch.bfloat16, device="cuda") * 0.1 + cos_sin = _make_cos_sin(4096, rotary_dim, theta=10_000_000.0) + base_positions = torch.randint(0, 4000, (num_tokens,), dtype=torch.int32, device="cuda") + positions = torch.stack( + (base_positions, base_positions + 3, base_positions + 7), dim=0 + ).contiguous() + + q_gate, k, v = qkv.split((2 * q_size, kv_size, kv_size), dim=-1) + q, expected_gate = [ + value.reshape(num_tokens, -1) + for value in torch.chunk(q_gate.view(num_tokens, num_q_heads, 2 * head_dim), 2, dim=-1) + ] + expected_qkv = torch.cat((q, k, v), dim=-1) + torch.ops.trtllm.fused_qk_norm_rope( + expected_qkv, + num_q_heads, + num_kv_heads, + num_kv_heads, + head_dim, + rotary_dim, + 1e-6, + q_weight, + k_weight, + 10_000_000.0, + True, + positions, + 1.0, + 0.0, + 0.0, + 1.0, + True, + True, + True, + mrope_section[1], + mrope_section[2], + ) + + actual_qkv, actual_gate = fused_qkv_gemma_rmsnorm_rope_gate( + qkv, + q_weight, + k_weight, + cos_sin, + positions, + 1e-6, + num_q_heads, + num_kv_heads, + head_dim, + rotary_dim, + mrope_section, + ) + + torch.testing.assert_close(actual_qkv, expected_qkv, atol=0.02, rtol=0.02) + torch.testing.assert_close(actual_gate.reshape(num_tokens, -1), expected_gate, atol=0, rtol=0) + + +def test_output_gate_fallback_flattens_fused_gate(): + torch.manual_seed(17) + attention = torch.randn((5, 256), dtype=torch.bfloat16, device="cuda") + gate = torch.randn((5, 2, 128), dtype=torch.bfloat16, device="cuda") + expected = attention * torch.sigmoid(gate.reshape_as(attention)) + actual = Attention.apply_output_gate(None, attention, gate) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + +def test_fused_sigmoid_mul_supports_flat_gate(): + torch.manual_seed(11) + attention = torch.randn((9, 512), dtype=torch.float16, device="cuda") + gate = torch.randn_like(attention) + expected = attention.float() * torch.sigmoid(gate.float()) + actual = fused_sigmoid_mul(attention, gate) + torch.testing.assert_close(actual.float(), expected, atol=0.005, rtol=0.005) diff --git a/tests/unittest/_torch/modules/mamba/test_gdn_kernel_optimizations.py b/tests/unittest/_torch/modules/mamba/test_gdn_kernel_optimizations.py index 71fabfcfc3dd..33bc67aeea6a 100644 --- a/tests/unittest/_torch/modules/mamba/test_gdn_kernel_optimizations.py +++ b/tests/unittest/_torch/modules/mamba/test_gdn_kernel_optimizations.py @@ -28,6 +28,21 @@ ) +@pytest.mark.parametrize( + "value,expected", + [(None, False), ("", False), ("0", False), ("false", False), ("typo", False), ("1", True)], +) +def test_gdn_replay_env_requires_one(monkeypatch, value, expected): + from tensorrt_llm._torch.utils import is_gdn_replay_enabled + + if value is None: + monkeypatch.delenv("TRTLLM_USE_GDN_REPLAY", raising=False) + else: + monkeypatch.setenv("TRTLLM_USE_GDN_REPLAY", value) + + assert is_gdn_replay_enabled() is expected + + # ---- Reference implementations ---- @@ -466,6 +481,40 @@ def test_pack_gdn_decode_qkv( # ---- Tests for the GDN compile boundary and derived state ---- +@skip_no_cuda +def test_reset_gdn_states_preserves_initialized_and_invalid_slots(): + from tensorrt_llm._torch.modules.mamba.gdn_mixer import _reset_gdn_states + + num_slots = 4 + ssm_state_size = 6 + conv_state_size = 8 + state_pool = ( + torch.arange( + num_slots * (ssm_state_size + conv_state_size), + device="cuda", + dtype=torch.float32, + ) + .to(torch.bfloat16) + .reshape(num_slots, -1) + ) + state_pool_before = state_pool.clone() + ssm_states = state_pool[:, :ssm_state_size].view(num_slots, 2, 3) + conv_states = state_pool[:, ssm_state_size:].view(num_slots, 2, 4) + state_indices = torch.tensor([1, 3, -1], device="cuda", dtype=torch.int32) + has_initial_states = torch.tensor([False, True, False], device="cuda", dtype=torch.bool) + + _reset_gdn_states( + ssm_states, + conv_states, + state_indices, + has_initial_states, + ) + + torch.testing.assert_close(state_pool[1], torch.zeros_like(state_pool[1])) + for state_idx in (0, 2, 3): + torch.testing.assert_close(state_pool[state_idx], state_pool_before[state_idx]) + + def test_gdn_custom_op_forwards_split_inputs(monkeypatch): """The compile boundary consumes split inputs and mutates its output.""" import tensorrt_llm._torch.modules.mamba.gdn_mixer as gdn_mixer diff --git a/tests/unittest/_torch/modules/mamba/test_gdn_replay_recurrent.py b/tests/unittest/_torch/modules/mamba/test_gdn_replay_recurrent.py new file mode 100644 index 000000000000..dab201710b67 --- /dev/null +++ b/tests/unittest/_torch/modules/mamba/test_gdn_replay_recurrent.py @@ -0,0 +1,516 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Equivalence test for the GDN MTP replay verify kernel. + +Simulates several target-verify iterations with randomized acceptance and +checks cached replay plus PNAT/double-buffer bookkeeping mirroring +update_mamba_states against: + 1. the legacy path (fused_recurrent_gated_delta_rule_update with + intermediate_states_buffer + accepted-state copy), and + 2. a sequential fp32 reference. +""" + +import pytest +import torch + +from tensorrt_llm._torch.modules.fla.cached_replay import ( + fused_recurrent_gated_delta_rule_cached_replay_update, +) +from tensorrt_llm._torch.modules.fla.fused_recurrent import fused_recurrent_gated_delta_rule_update + + +def _seq_ref_step(S, q, k, v, g, beta, scale): + """Sequential fp32 reference over T steps for one request. + + S: [HV, V, K] fp32 (modified out-of-place). Returns (o [T, HV, V], + per-step states list). + """ + T = q.shape[0] + HV = S.shape[0] + H = k.shape[1] + outs, states = [], [] + S = S.clone() + for t in range(T): + o_t = torch.empty(*S.shape[:1], S.shape[1], dtype=torch.float32, device=S.device) + for hv in range(HV): + h = hv // (HV // H) + qt = q[t, h].float() + kt = k[t, h].float() + vt = v[t, hv].float() + qt = qt / (qt.norm() + 1e-6) * scale + kt = kt / (kt.norm() + 1e-6) + St = S[hv] * torch.exp(g[t, hv].float()) + vt = (vt - (St * kt[None, :]).sum(-1)) * beta[t, hv].float() + St = St + kt[None, :] * vt[:, None] + o_t[hv] = (St * qt[None, :]).sum(-1) + S[hv] = St + outs.append(o_t) + states.append(S.clone()) + return torch.stack(outs), states + + +@pytest.mark.parametrize("fused_gating", [False, True], ids=["pre_gated", "fused_gating"]) +@pytest.mark.parametrize( + "pool_dtype", [torch.bfloat16, torch.float32], ids=["bf16_pool", "fp32_pool"] +) +@pytest.mark.parametrize( + "T,HIST,iters", + [(2, 14, 16), (4, 12, 10), (5, 11, 12), (4, 16, 10)], + ids=["T2_H14", "T4_H12", "T5_H11", "T4_H16"], +) +@pytest.mark.parametrize( + "H,HV,K,V", + [(4, 8, 128, 128), (2, 4, 64, 64), (4, 16, 128, 128)], + ids=["qwen3_like", "small", "qwen3_5_like"], +) +def test_gdn_replay_vs_legacy_and_ref(H, HV, K, V, T, HIST, iters, pool_dtype, fused_gating): + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + torch.manual_seed(1234) + device = "cuda" + dtype = torch.bfloat16 + N = 4 + scale = K**-0.5 + slots = N + 2 # exercise non-trivial slot indices + + # Fused-gating mode: the replay kernel receives RAW a/b plus + # A_log/dt_bias and applies g = -exp(A_log)*softplus(a + dt_bias), + # beta = sigmoid(b) in-kernel; legacy + reference use the same gating + # computed on the host. + A_log_t = (torch.randn(HV, device=device, dtype=torch.float32) * 0.5) if fused_gating else None + dt_bias_t = ( + (torch.randn(HV, device=device, dtype=torch.float32) * 0.5) if fused_gating else None + ) + + pool_init = torch.randn(slots, HV, V, K, device=device, dtype=torch.float32) * 0.5 + # Exercise a pool-backed (non-contiguous, padded slot stride) layout: + # the kernel must commit through the real strides — a contiguous copy + # would silently discard checkpoint commits (Qwen3.5 looping root + # cause). Odd T uses the strided layout, even T stays dense. + strided_pool = T % 2 == 1 + # .clone() everywhere: for fp32 pools .to(pool_dtype)/.float() are no-ops + # returning the same tensor, which would alias pool/reference storage. + if strided_pool: + _pad = 1024 + _backing = torch.zeros(slots, HV * V * K + _pad, device=device, dtype=pool_dtype) + pool_replay = _backing[:, : HV * V * K].view(slots, HV, V, K) + pool_replay.copy_(pool_init.to(pool_dtype)) + assert not pool_replay.is_contiguous() + else: + pool_replay = pool_init.to(pool_dtype).clone() + pool_legacy = pool_init.to(pool_dtype).clone() + ref_S = pool_init.to(pool_dtype).float().clone() # follow quantized init + + # Replay buffers + bookkeeping (per-layer slice shapes) + old_v = torch.zeros(slots, 2, HIST, HV, V, device=device, dtype=dtype) + old_k = torch.zeros(slots, 2, HIST, H, K, device=device, dtype=dtype) + old_g = torch.zeros(slots, 2, HV, HIST, device=device, dtype=torch.float32) + old_beta = torch.zeros(slots, 2, HV, HIST, device=device, dtype=torch.float32) + buf_idx = torch.zeros(slots, dtype=torch.int32, device=device) + pnat = torch.zeros(slots, dtype=torch.int32, device=device) + # Non-identity slot mapping + state_indices = torch.arange(1, N + 1, dtype=torch.int32, device=device) + + # Legacy intermediate buffer, indexed by [0..N) + intermediate_ssm = torch.zeros(N, T, HV, V, K, device=device, dtype=pool_dtype) + arange_n = torch.arange(N, dtype=torch.int32, device=device) + + n_checkpoints = 0 + prev_ref_S = ref_S.clone() + for it in range(iters): + q = torch.randn(N, T, H, K, device=device, dtype=dtype) + k = torch.randn(N, T, H, K, device=device, dtype=dtype) + v = torch.randn(N, T, HV, V, device=device, dtype=dtype) * 0.5 + packed_qkv = None + if T == 4 and HIST == 16: + packed_qkv = torch.cat((q.flatten(2), k.flatten(2), v.flatten(2)), dim=-1).reshape( + N * T, -1 + ) + packed_qkv_3d = packed_qkv.view(N, T, -1) + key_width = H * K + q = packed_qkv_3d[..., :key_width].view(N, T, H, K) + k = packed_qkv_3d[..., key_width : 2 * key_width].view(N, T, H, K) + v = packed_qkv_3d[..., 2 * key_width :].view(N, T, HV, V) + assert packed_qkv.is_contiguous() + assert not q.is_contiguous() and not k.is_contiguous() and not v.is_contiguous() + if fused_gating: + a_raw = torch.randn(N, T, HV, device=device, dtype=dtype) * 0.5 + b_raw = torch.randn(N, T, HV, device=device, dtype=dtype) + # Host reference of the in-kernel gating (fp32, same inputs). + g = -(A_log_t.exp() * torch.nn.functional.softplus(a_raw.float() + dt_bias_t)) + beta = torch.sigmoid(b_raw.float()) + else: + g = -torch.rand(N, T, HV, device=device, dtype=torch.float32) * 2.0 + beta = torch.rand(N, T, HV, device=device, dtype=torch.float32) + num_accepted = torch.randint(1, T + 1, (N,)) + + # --- replay path --- + replay_kwargs = {} + # Piggyback PDL coverage on the fused-gating arm: a PDL launch + # without a producer signal is legal (waits resolve at predecessor + # completion), so this exercises the gdc_wait code path. + replay_kwargs["launch_with_pdl"] = fused_gating + if packed_qkv is not None: + replay_kwargs["packed_qkv"] = packed_qkv + replay_output = torch.empty((N, T, HV, V), device=device, dtype=dtype) + replay_kwargs["output"] = replay_output + o_replay = fused_recurrent_gated_delta_rule_cached_replay_update( + q, + k, + v, + a_raw if fused_gating else g, + b_raw if fused_gating else beta, + pool_replay, + state_indices, + old_v, + old_k, + old_g, + old_beta, + buf_idx, + pnat, + history_size=HIST, + scale=scale, + use_qk_l2norm_in_kernel=True, + A_log=A_log_t, + dt_bias=dt_bias_t, + **replay_kwargs, + ) + assert o_replay is replay_output + + # --- legacy path --- + src = pool_legacy[state_indices] + o_legacy = fused_recurrent_gated_delta_rule_update( + q=q, + k=k, + v=v, + g=g.to(dtype), + beta=beta.to(dtype), + scale=scale, + initial_state_source=src, + initial_state_indices=arange_n, + use_qk_l2norm_in_kernel=True, + disable_state_update=True, + intermediate_states_buffer=intermediate_ssm, + cache_steps=T, + ) + + # --- reference outputs + acceptance --- + for n in range(N): + o_ref, states = _seq_ref_step( + ref_S[state_indices[n]], q[n], k[n], v[n], g[n], beta[n], scale + ) + torch.testing.assert_close(o_replay[n].float(), o_ref, atol=6e-3, rtol=2e-2) + torch.testing.assert_close(o_legacy[n].float(), o_ref, atol=6e-3, rtol=2e-2) + ref_S[state_indices[n]] = states[num_accepted[n] - 1] + + # replay vs legacy outputs directly (both bf16 outputs, tighter) + torch.testing.assert_close(o_replay.float(), o_legacy.float(), atol=4e-3, rtol=2e-2) + + # --- legacy acceptance: copy accepted state --- + pool_legacy[state_indices] = intermediate_ssm[ + arange_n.long(), (num_accepted - 1).to(device).long() + ] + + # --- replay bookkeeping (mirrors update_mamba_states) --- + pnat_h = pnat.cpu() + buf_h = buf_idx.cpu() + for n in range(N): + slot = state_indices[n].item() + if (pnat_h[slot] + T) > HIST: + # Kernel committed the pre-iteration state this launch + # (prev_ref_S = reference state before this iteration's + # acceptance advance). + err = (pool_replay[slot].float() - prev_ref_S[slot]).abs().max().item() + assert err < 6e-3, f"checkpoint state err {err}" + n_checkpoints += 1 + pnat_h[slot] = num_accepted[n] + buf_h[slot] = 1 - buf_h[slot] + else: + pnat_h[slot] = pnat_h[slot] + num_accepted[n] + pnat.copy_(pnat_h.to(device)) + buf_idx.copy_(buf_h.to(device)) + prev_ref_S = ref_S.clone() + + # Ensure the test exercised checkpoint commits and buffer flips. + assert n_checkpoints >= N, f"only {n_checkpoints} checkpoints hit; increase iters" + + +def test_gdn_replay_negative_slot_skipped(): + """Padding requests (slot < 0) must not touch pool or history.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + torch.manual_seed(0) + device = "cuda" + dtype = torch.bfloat16 + N, T, H, HV, K, V, HIST = 2, 4, 2, 4, 64, 64, 16 + + pool = torch.randn(N, HV, V, K, device=device, dtype=dtype) + pool_orig = pool.clone() + old_v = torch.zeros(N, 2, HIST, HV, V, device=device, dtype=dtype) + old_k = torch.zeros(N, 2, HIST, H, K, device=device, dtype=dtype) + old_g = torch.zeros(N, 2, HV, HIST, device=device, dtype=torch.float32) + old_beta = torch.zeros(N, 2, HV, HIST, device=device, dtype=torch.float32) + buf_idx = torch.zeros(N, dtype=torch.int32, device=device) + pnat = torch.full((N,), HIST, dtype=torch.int32, device=device) # force write + state_indices = torch.tensor([-1, -1], dtype=torch.int32, device=device) + + q = torch.randn(N, T, H, K, device=device, dtype=dtype) + k = torch.randn(N, T, H, K, device=device, dtype=dtype) + v = torch.randn(N, T, HV, V, device=device, dtype=dtype) + g = -torch.rand(N, T, HV, device=device, dtype=torch.float32) + beta = torch.rand(N, T, HV, device=device, dtype=torch.float32) + + fused_recurrent_gated_delta_rule_cached_replay_update( + q, + k, + v, + g, + beta, + pool, + state_indices, + old_v, + old_k, + old_g, + old_beta, + buf_idx, + pnat, + history_size=HIST, + use_qk_l2norm_in_kernel=True, + ) + + torch.testing.assert_close(pool, pool_orig) + assert old_v.abs().max().item() == 0 + assert old_k.abs().max().item() == 0 + + +def _cached_replay_commit_reference(initial_states, old_u, old_k, old_G, work_items, n_writes): + reference = initial_states.clone() + num_layers, _, HV, _, _ = reference.shape + H = old_k.shape[-2] + rows = work_items[: int(n_writes.item())].cpu().tolist() + for _, slot, pnat, active_buffer in rows: + for layer in range(num_layers): + for hv in range(HV): + h = hv // (HV // H) + history_k = old_k[layer, slot, active_buffer, :pnat, h] + history_u = old_u[layer, slot, active_buffer, :pnat, hv] + history_G = old_G[layer, slot, active_buffer, hv, :pnat] + g_start = history_G[-1] + commit_decay = torch.exp(g_start - history_G) + scaled_u = history_u.float() * commit_decay[:, None] + scaled_u_hi = scaled_u.to(history_u.dtype) + scaled_u_lo = (scaled_u - scaled_u_hi.float()).to(history_u.dtype) + delta = history_k.float().T @ scaled_u_hi.float() + delta += history_k.float().T @ scaled_u_lo.float() + committed = reference[layer, slot, hv].float() * torch.exp(g_start) + reference[layer, slot, hv] = (committed + delta.T).to(reference.dtype) + return reference + + +def test_gdn_cached_replay_all_layer_commit_matches_reference(): + """The all-layer commit must match its direct mathematical reference.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + + from tensorrt_llm._torch.modules.fla.cached_replay import ( + commit_gdn_cached_replay_history_layers, + ) + + torch.manual_seed(1234) + device = "cuda" + dtype = torch.bfloat16 + num_layers, N, num_slots = 3, 16, 18 + H, HV, K, V, HIST = 2, 8, 128, 128, 16 + initial_states = torch.randn(num_layers, num_slots, HV, V, K, device=device, dtype=dtype) + old_u = torch.randn(num_layers, num_slots, 2, HIST, HV, V, device=device, dtype=dtype) + old_k = torch.randn(num_layers, num_slots, 2, HIST, H, K, device=device, dtype=dtype) + old_G = -torch.cumsum( + torch.rand( + num_layers, + num_slots, + 2, + HV, + HIST, + device=device, + dtype=torch.float32, + ) + * 0.1, + dim=-1, + ) + + positions = torch.arange(N, device=device, dtype=torch.int32) + cache_slots = positions + 1 + pnat = positions.remainder(HIST - 1) + 1 + active_buffers = positions.remainder(2) + work_items = torch.stack((positions, cache_slots, pnat, active_buffers), dim=1) + n_writes = torch.tensor([N // 2], device=device, dtype=torch.int32) + + expected_states = _cached_replay_commit_reference( + initial_states, + old_u, + old_k, + old_G, + work_items, + n_writes, + ) + actual_states = initial_states.clone() + commit_gdn_cached_replay_history_layers( + ssm_states=actual_states, + old_u=old_u, + old_k=old_k, + old_G=old_G, + replay_work_items=work_items, + n_writes=n_writes, + history_size=HIST, + ) + + torch.testing.assert_close(actual_states.float(), expected_states.float(), rtol=2e-2, atol=2e-2) + + +@pytest.mark.parametrize("batch_size", [8, 16], ids=["small_fused", "large_all_layer"]) +def test_gdn_cached_replay_dispatch_cuda_graph_matches_eager(batch_size): + """Both sides of the BS16 dispatch must be safe under CUDA graphs.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + + from tensorrt_llm._torch.modules.fla.cached_replay import ( + CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, + commit_gdn_cached_replay_history_layers, + ) + + torch.manual_seed(1234) + device = "cuda" + dtype = torch.bfloat16 + num_layers, T, HIST = 2, 4, 16 + H, HV, K, V = 2, 4, 64, 64 + num_slots = batch_size + 1 + use_all_layer_commit = batch_size >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE + + initial_states = torch.randn(num_layers, num_slots, HV, V, K, device=device, dtype=dtype) + initial_u = torch.randn(num_layers, num_slots, 2, HIST, HV, V, device=device, dtype=dtype) + initial_k = torch.randn(num_layers, num_slots, 2, HIST, H, K, device=device, dtype=dtype) + initial_G = -torch.cumsum( + torch.rand( + num_layers, + num_slots, + 2, + HV, + HIST, + device=device, + dtype=torch.float32, + ), + dim=-1, + ) + old_beta = torch.empty_like(initial_G) + state_indices = torch.arange(1, batch_size + 1, device=device, dtype=torch.int32) + cache_buf_idx = torch.zeros(num_slots, device=device, dtype=torch.int32) + pnat = torch.full((num_slots,), HIST, device=device, dtype=torch.int32) + q = torch.randn(num_layers, batch_size, T, H, K, device=device, dtype=dtype) + k = torch.randn_like(q) + v = torch.randn(num_layers, batch_size, T, HV, V, device=device, dtype=dtype) + g = -torch.rand(num_layers, batch_size, T, HV, device=device, dtype=torch.float32) + beta = torch.rand_like(g) + + replay_work_items = None + n_writes = None + if use_all_layer_commit: + positions = torch.arange(batch_size, device=device, dtype=torch.int32) + replay_work_items = torch.stack( + ( + positions, + state_indices, + torch.full_like(positions, HIST), + torch.zeros_like(positions), + ), + dim=1, + ) + n_writes = torch.tensor([batch_size], device=device, dtype=torch.int32) + + def run_replay(states, old_u, old_k, old_G): + outputs = [] + for layer in range(num_layers): + outputs.append( + fused_recurrent_gated_delta_rule_cached_replay_update( + q[layer], + k[layer], + v[layer], + g[layer], + beta[layer], + states[layer], + state_indices, + old_u[layer], + old_k[layer], + old_G[layer], + old_beta[layer], + cache_buf_idx, + pnat, + history_size=HIST, + use_qk_l2norm_in_kernel=True, + replay_work_items=replay_work_items, + n_writes=n_writes, + use_all_layer_commit=use_all_layer_commit, + ) + ) + return outputs + + def run_all_layer_commit(states, old_u, old_k, old_G): + if use_all_layer_commit: + commit_gdn_cached_replay_history_layers( + ssm_states=states, + old_u=old_u, + old_k=old_k, + old_G=old_G, + replay_work_items=replay_work_items, + n_writes=n_writes, + history_size=HIST, + ) + + warmup_states = initial_states.clone() + warmup_u = initial_u.clone() + warmup_k = initial_k.clone() + warmup_G = initial_G.clone() + run_replay(warmup_states, warmup_u, warmup_k, warmup_G) + run_all_layer_commit(warmup_states, warmup_u, warmup_k, warmup_G) + torch.cuda.synchronize() + + eager_states = initial_states.clone() + eager_u = initial_u.clone() + eager_k = initial_k.clone() + eager_G = initial_G.clone() + eager_outputs = run_replay(eager_states, eager_u, eager_k, eager_G) + run_all_layer_commit(eager_states, eager_u, eager_k, eager_G) + + graph_states = initial_states.clone() + graph_u = initial_u.clone() + graph_k = initial_k.clone() + graph_G = initial_G.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + graph_outputs = run_replay(graph_states, graph_u, graph_k, graph_G) + graph_states.copy_(initial_states) + graph_u.copy_(initial_u) + graph_k.copy_(initial_k) + graph_G.copy_(initial_G) + torch.cuda.synchronize() + graph.replay() + run_all_layer_commit(graph_states, graph_u, graph_k, graph_G) + torch.cuda.synchronize() + + for eager_output, graph_output in zip(eager_outputs, graph_outputs): + torch.testing.assert_close(graph_output, eager_output, rtol=0, atol=0) + torch.testing.assert_close(graph_states, eager_states, rtol=0, atol=0) + torch.testing.assert_close(graph_u, eager_u, rtol=0, atol=0) + torch.testing.assert_close(graph_k, eager_k, rtol=0, atol=0) + torch.testing.assert_close(graph_G, eager_G, rtol=0, atol=0) diff --git a/tests/unittest/_torch/modules/moe/test_communication_factory.py b/tests/unittest/_torch/modules/moe/test_communication_factory.py new file mode 100644 index 000000000000..841f0e9157cc --- /dev/null +++ b/tests/unittest/_torch/modules/moe/test_communication_factory.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import sys +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.modules.fused_moe import nccl_ep_utils +from tensorrt_llm._torch.modules.fused_moe.communication import communication_factory +from tensorrt_llm._torch.modules.fused_moe.communication.allgather_reducescatter import ( + AllGatherReduceScatter, +) +from tensorrt_llm._torch.modules.fused_moe.communication.nccl_ep import NcclEP + + +def _make_model_config( + act_dtype: torch.dtype = torch.bfloat16, + moe_max_num_tokens: int | None = 1024, +): + mapping = SimpleNamespace( + enable_attention_dp=True, + dp_size=2, + moe_tp_size=1, + moe_ep_size=2, + moe_ep_rank=0, + ) + return SimpleNamespace( + mapping=mapping, + pretrained_config=SimpleNamespace(hidden_size=4096), + torch_dtype=act_dtype, + quant_config=None, + max_num_tokens=1024, + moe_max_num_tokens=moe_max_num_tokens, + use_cuda_graph=False, + use_low_precision_moe_combine=False, + moe_load_balancer=None, + ) + + +def _strategy_unavailable(*args, **kwargs): + raise RuntimeError("strategy unavailable") + + +def _install_failing_nccl_module(monkeypatch: pytest.MonkeyPatch, error: BaseException): + def fail_get_version(): + raise error + + monkeypatch.setattr(nccl_ep_utils, "_nccl_ep_installed", None) + monkeypatch.setitem(sys.modules, "nccl", SimpleNamespace(get_version=fail_get_version)) + monkeypatch.delitem(sys.modules, "nccl.ep", raising=False) + + +def test_nccl_ep_installed_handles_runtime_probe_failure(monkeypatch: pytest.MonkeyPatch): + _install_failing_nccl_module(monkeypatch, RuntimeError("missing libnccl_ep")) + + assert nccl_ep_utils.is_nccl_ep_installed() is False + assert nccl_ep_utils._nccl_ep_installed is False + + +class _FakeNcclEP: + def __init__( + self, + mapping, + num_slots, + hidden_size, + max_num_tokens, + moe_max_num_tokens, + top_k=8, + ): + self.mapping = mapping + self.num_slots = num_slots + self.hidden_size = hidden_size + self.max_num_tokens = max_num_tokens + self.moe_max_num_tokens = moe_max_num_tokens + self.top_k = top_k + + +@pytest.mark.parametrize( + ("act_dtype", "moe_max_num_tokens", "match"), + [ + (torch.float16, 1024, "act_dtype=torch.bfloat16"), + ], +) +def test_forced_nccl_ep_validates_preconditions( + act_dtype: torch.dtype, + moe_max_num_tokens: int | None, + match: str, +): + model_config = _make_model_config(act_dtype, moe_max_num_tokens) + + with pytest.raises(ValueError, match=match): + communication_factory.CommunicationFactory._create_forced_method( + "NCCL_EP", + model_config, + num_experts=32, + num_slots=32, + top_k=8, + expert_size_per_partition=16, + payload_in_workspace=False, + alltoall_result_do_sum=True, + use_flashinfer=False, + hidden_size=4096, + ) + + +def test_forced_nccl_ep_allows_missing_moe_max_num_tokens( + monkeypatch: pytest.MonkeyPatch, +): + model_config = _make_model_config(torch.bfloat16, None) + monkeypatch.setattr(communication_factory, "NcclEP", _FakeNcclEP) + + strategy = communication_factory.CommunicationFactory._create_forced_method( + "NCCL_EP", + model_config, + num_experts=32, + num_slots=32, + top_k=8, + expert_size_per_partition=16, + payload_in_workspace=False, + alltoall_result_do_sum=True, + use_flashinfer=False, + hidden_size=4096, + ) + + assert isinstance(strategy, _FakeNcclEP) + assert strategy.max_num_tokens == model_config.max_num_tokens + assert strategy.moe_max_num_tokens is None + + +def test_auto_selection_uses_nccl_ep_with_missing_moe_max_num_tokens( + monkeypatch: pytest.MonkeyPatch, +): + model_config = _make_model_config(torch.bfloat16, None) + + monkeypatch.setattr(communication_factory, "NVLinkOneSided", _strategy_unavailable) + monkeypatch.setattr(communication_factory, "NVLinkTwoSided", _strategy_unavailable) + monkeypatch.setenv("TRTLLM_CAN_USE_DEEP_EP", "0") + monkeypatch.setattr(communication_factory, "NcclEP", _FakeNcclEP) + + strategy = communication_factory.CommunicationFactory.create_strategy( + model_config, + num_experts=32, + num_slots=32, + top_k=8, + expert_size_per_partition=16, + hidden_size=4096, + ) + + assert isinstance(strategy, _FakeNcclEP) + assert strategy.max_num_tokens == model_config.max_num_tokens + assert strategy.moe_max_num_tokens is None + + +@pytest.mark.parametrize( + ("act_dtype", "moe_max_num_tokens"), + [ + (torch.float16, 1024), + ], +) +def test_auto_selection_skips_nccl_ep_when_preconditions_fail( + monkeypatch: pytest.MonkeyPatch, + act_dtype: torch.dtype, + moe_max_num_tokens: int | None, +): + model_config = _make_model_config(act_dtype, moe_max_num_tokens) + + monkeypatch.setattr(communication_factory, "NVLinkOneSided", _strategy_unavailable) + monkeypatch.setattr(communication_factory, "NVLinkTwoSided", _strategy_unavailable) + monkeypatch.setenv("TRTLLM_CAN_USE_DEEP_EP", "0") + + def fail_if_called(*args, **kwargs): + raise AssertionError("NcclEP should not be constructed") + + monkeypatch.setattr(communication_factory, "NcclEP", fail_if_called) + + strategy = communication_factory.CommunicationFactory.create_strategy( + model_config, + num_experts=32, + num_slots=32, + top_k=8, + expert_size_per_partition=16, + hidden_size=4096, + ) + + assert isinstance(strategy, AllGatherReduceScatter) + + +def test_auto_selection_skips_nccl_ep_for_quantized_moe( + monkeypatch: pytest.MonkeyPatch, +): + model_config = _make_model_config() + model_config.quant_config = SimpleNamespace( + layer_quant_mode=SimpleNamespace(has_any_quant=lambda **_: True) + ) + monkeypatch.setattr(communication_factory, "NVLinkOneSided", _strategy_unavailable) + monkeypatch.setattr(communication_factory, "NVLinkTwoSided", _strategy_unavailable) + monkeypatch.setenv("TRTLLM_CAN_USE_DEEP_EP", "0") + monkeypatch.setattr( + communication_factory, + "NcclEP", + lambda *args, **kwargs: pytest.fail("NcclEP should not be constructed for quantized MoE"), + ) + + strategy = communication_factory.CommunicationFactory.create_strategy( + model_config, + num_experts=32, + num_slots=32, + top_k=8, + expert_size_per_partition=16, + hidden_size=4096, + ) + + assert isinstance(strategy, AllGatherReduceScatter) + + +def test_auto_selection_falls_back_when_nccl_probe_runtime_fails( + monkeypatch: pytest.MonkeyPatch, +): + model_config = _make_model_config(torch.bfloat16, None) + monkeypatch.setattr(communication_factory, "NVLinkOneSided", _strategy_unavailable) + monkeypatch.setattr(communication_factory, "NVLinkTwoSided", _strategy_unavailable) + monkeypatch.setenv("TRTLLM_CAN_USE_DEEP_EP", "0") + _install_failing_nccl_module(monkeypatch, OSError("missing native NCCL EP library")) + + strategy = communication_factory.CommunicationFactory.create_strategy( + model_config, + num_experts=32, + num_slots=32, + top_k=8, + expert_size_per_partition=16, + hidden_size=4096, + ) + + assert isinstance(strategy, AllGatherReduceScatter) + + +def test_nccl_ep_context_init_rejects_cuda_graph_capture( + monkeypatch: pytest.MonkeyPatch, +): + strategy = object.__new__(NcclEP) + strategy._ctx = None + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True) + + with pytest.raises(RuntimeError, match="context must be initialized before CUDA graph capture"): + strategy._get_context() + + +def test_nccl_ep_handle_init_rejects_cuda_graph_capture( + monkeypatch: pytest.MonkeyPatch, +): + strategy = object.__new__(NcclEP) + strategy._handle = None + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True) + + def fail_create_handle(*args, **kwargs): + raise AssertionError("create_handle should not run during CUDA graph capture") + + ctx = SimpleNamespace( + ep_group=SimpleNamespace(create_handle=fail_create_handle), + layout=object(), + ) + + with pytest.raises( + RuntimeError, match="dispatch handle must be initialized before CUDA graph capture" + ): + strategy._setup_handle(ctx, object(), 0) diff --git a/tests/unittest/_torch/modules/moe/test_megamoe_streaming_load.py b/tests/unittest/_torch/modules/moe/test_megamoe_streaming_load.py new file mode 100644 index 000000000000..b61dc8ce204a --- /dev/null +++ b/tests/unittest/_torch/modules/moe/test_megamoe_streaming_load.py @@ -0,0 +1,199 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for MegaMoE-CuteDSL NVFP4 streaming source weights.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +torch = pytest.importorskip("torch") + +from torch import nn # noqa: E402 + +if TYPE_CHECKING: + from tensorrt_llm._torch.modules.fused_moe.interface import MoEWeightLoadingMode + from tensorrt_llm._torch.modules.fused_moe.quantization import NVFP4MegaMoECuteDslMethod + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available(), + reason="needs 1 CUDA GPU (initial-load sources are rematerialized on cuda)", +) + +NUM_EXPERTS = 4 +HIDDEN_SIZE = 256 +INTERMEDIATE_SIZE = 64 + +_STREAMED_PARAMS = ("w3_w1_weight", "w3_w1_weight_scale", "w2_weight", "w2_weight_scale") + + +def _load_classes() -> tuple[type[MoEWeightLoadingMode], type[NVFP4MegaMoECuteDslMethod]]: + from tensorrt_llm._torch.modules.fused_moe.interface import MoEWeightLoadingMode + from tensorrt_llm._torch.modules.fused_moe.quantization import NVFP4MegaMoECuteDslMethod + + return MoEWeightLoadingMode, NVFP4MegaMoECuteDslMethod + + +class _StreamingMoEModule(nn.Module): + """Minimal single-rank module for the quant-method load path.""" + + def __init__(self, weight_loading_mode: MoEWeightLoadingMode) -> None: + super().__init__() + self.num_experts = NUM_EXPERTS + self.hidden_size = HIDDEN_SIZE + self.intermediate_size_per_partition = INTERMEDIATE_SIZE + self.expand_intermediate_size_per_partition = 2 * INTERMEDIATE_SIZE + self.expert_size_per_partition = NUM_EXPERTS + self.initial_local_expert_ids = list(range(NUM_EXPERTS)) + self.tp_size = 1 + self.tp_rank = 0 + self.ep_size = 1 + self.ep_rank = 0 + self.dtype = torch.bfloat16 + self.bias = False + self.weight_loading_mode = weight_loading_mode + # No EPLB in this test: need_load_shared_weights() must be False. + self.layer_load_balancer = None + + def _add_raw_shared_weights_for_unmap(self, weight_tensors: list[torch.Tensor]) -> None: + # Only forwards to the dynamic load balancer in production; no-op here. + del weight_tensors + + +def _w13_input_scale(expert_id: int) -> float: + return 0.5 + 0.125 * expert_id + + +def _w2_input_scale(expert_id: int) -> float: + return 0.25 + 0.0625 * expert_id + + +def _make_vanilla_weights(seed: int = 20260708) -> dict[str, torch.Tensor]: + gen = torch.Generator(device="cuda").manual_seed(seed) + + def rand_u8(*shape: int) -> torch.Tensor: + return torch.randint(0, 256, shape, dtype=torch.uint8, device="cuda", generator=gen) + + def rand_fp8(*shape: int) -> torch.Tensor: + # Any byte payload works (pure moves/reinterprets); stay clear of the + # 0x7f/0xff NaN encodings for hygiene. + raw = torch.randint(1, 120, shape, dtype=torch.uint8, device="cuda", generator=gen) + return raw.view(torch.float8_e4m3fn) + + weights = {} + for e in range(NUM_EXPERTS): + weights[f"{e}.w1.weight"] = rand_u8(INTERMEDIATE_SIZE, HIDDEN_SIZE // 2) + weights[f"{e}.w3.weight"] = rand_u8(INTERMEDIATE_SIZE, HIDDEN_SIZE // 2) + weights[f"{e}.w2.weight"] = rand_u8(HIDDEN_SIZE, INTERMEDIATE_SIZE // 2) + weights[f"{e}.w1.weight_scale"] = rand_fp8(INTERMEDIATE_SIZE, HIDDEN_SIZE // 16) + weights[f"{e}.w3.weight_scale"] = rand_fp8(INTERMEDIATE_SIZE, HIDDEN_SIZE // 16) + weights[f"{e}.w2.weight_scale"] = rand_fp8(HIDDEN_SIZE, INTERMEDIATE_SIZE // 16) + # w1/w3 input scales must match per expert (parent PWAL asserts). + weights[f"{e}.w1.input_scale"] = torch.tensor(_w13_input_scale(e), dtype=torch.float32) + weights[f"{e}.w3.input_scale"] = torch.tensor(_w13_input_scale(e), dtype=torch.float32) + weights[f"{e}.w2.input_scale"] = torch.tensor(_w2_input_scale(e), dtype=torch.float32) + # w1/w3 weight_scale_2 must match per expert (reconcile warns/maxes). + ws13 = torch.tensor(0.01 * (e + 1), dtype=torch.float32) + weights[f"{e}.w1.weight_scale_2"] = ws13 + weights[f"{e}.w3.weight_scale_2"] = ws13.clone() + weights[f"{e}.w2.weight_scale_2"] = torch.tensor(0.02 * (e + 1), dtype=torch.float32) + return weights + + +def _fresh( + seed: int = 20260708, +) -> tuple[ + NVFP4MegaMoECuteDslMethod, + _StreamingMoEModule, + dict[str, torch.Tensor], +]: + mode_cls, method_cls = _load_classes() + module = _StreamingMoEModule(mode_cls.VANILLA) + method = method_cls() + with torch.device("cuda"): + method.create_weights(module) + return method, module, _make_vanilla_weights(seed) + + +def _load( + method: NVFP4MegaMoECuteDslMethod, + module: _StreamingMoEModule, + bucket: dict[str, torch.Tensor], + allow_partial_loading: bool, +) -> None: + mode_cls, _ = _load_classes() + method.load_weights( + module, bucket, mode_cls.VANILLA, allow_partial_loading=allow_partial_loading + ) + + +def _streamed_numels(module: _StreamingMoEModule) -> dict[str, int]: + return {name: getattr(module, name).data.numel() for name in _STREAMED_PARAMS} + + +def _assert_sources_freed(module: _StreamingMoEModule, context: str = "") -> None: + numels = _streamed_numels(module) + assert all(n == 0 for n in numels.values()), ( + f"streamed source params should be 0-element placeholders {context}: {numels}" + ) + + +def _expected_mega_fc2(weights: dict[str, torch.Tensor]) -> torch.Tensor: + return torch.stack([weights[f"{e}.w2.weight"] for e in range(NUM_EXPERTS)]) + + +def _expected_mega_fc1(weights: dict[str, torch.Tensor]) -> torch.Tensor: + per_slot = [] + for e in range(NUM_EXPERTS): + gate = weights[f"{e}.w1.weight"].view(INTERMEDIATE_SIZE // 16, 16, HIDDEN_SIZE // 2) + up = weights[f"{e}.w3.weight"].view(INTERMEDIATE_SIZE // 16, 16, HIDDEN_SIZE // 2) + per_slot.append( + torch.stack([gate, up], dim=1).reshape(2 * INTERMEDIATE_SIZE, HIDDEN_SIZE // 2) + ) + return torch.stack(per_slot) + + +def _expected_fc1_norm_const() -> torch.Tensor: + return torch.tensor( + [1.0 / _w2_input_scale(e) for e in range(NUM_EXPERTS)], + dtype=torch.float32, + device="cuda", + ) + + +def test_initial_streaming_load_layer_atomic() -> None: + method, module, weights = _fresh() + + # create_weights replaces streamed sources with empty placeholders. + _assert_sources_freed(module, "right after create_weights") + + _load(method, module, weights, allow_partial_loading=False) + _assert_sources_freed(module, "after the initial eager load") + + assert torch.equal(module.mega_fc2_weight.data, _expected_mega_fc2(weights)) + assert torch.equal(module.mega_fc1_weight.data, _expected_mega_fc1(weights)) + assert torch.allclose(module.fc1_norm_const.data, _expected_fc1_norm_const()) + + +def test_partial_load_rejected_before_source_materialization() -> None: + method, module, weights = _fresh() + _assert_sources_freed(module, "before partial load") + + with pytest.raises(NotImplementedError, match="only supports full initial weight loading"): + _load(method, module, weights, allow_partial_loading=True) + + _assert_sources_freed(module, "after rejected partial load") diff --git a/tests/unittest/_torch/modules/moe/test_moe_backend.py b/tests/unittest/_torch/modules/moe/test_moe_backend.py index a6ca12ada111..8223e08d67bb 100644 --- a/tests/unittest/_torch/modules/moe/test_moe_backend.py +++ b/tests/unittest/_torch/modules/moe/test_moe_backend.py @@ -57,7 +57,9 @@ DeepSeekV3MoeRoutingMethod, RenormalizeMoeRoutingMethod, ) -from tensorrt_llm._torch.modules.fused_moe.create_moe import create_moe_backend +from tensorrt_llm._torch.modules.fused_moe.create_moe import create_moe_backend, get_moe_cls +from tensorrt_llm._torch.modules.fused_moe.fused_moe_cutlass import CutlassFusedMoE +from tensorrt_llm._torch.modules.fused_moe.fused_moe_marlin import MarlinFusedMoE from tensorrt_llm._torch.modules.fused_moe.interface import MoE, MoEWeightLoadingMode from tensorrt_llm._torch.modules.fused_moe.mega_moe import MegaMoECuteDsl, MegaMoEDeepGemm from tensorrt_llm._torch.modules.fused_moe.quantization import ( @@ -69,7 +71,7 @@ from tensorrt_llm._torch.utils import ActivationType, is_gated_activation from tensorrt_llm._utils import mpi_rank from tensorrt_llm.mapping import Mapping -from tensorrt_llm.models.modeling_utils import QuantAlgo +from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig logger = logging.getLogger(__name__) @@ -330,6 +332,41 @@ def test_marlin_moe_repack_is_transform_stage(): assert NVFP4MarlinFusedMoEMethod.post_load_weights is FusedMoEMethodBase.post_load_weights +def _marlin_model_config(quant_algo=QuantAlgo.NVFP4): + cfg = ModelConfig() + cfg.moe_backend = "MARLIN" + cfg.quant_config = QuantConfig(quant_algo=quant_algo) if quant_algo else None + return cfg + + +def test_get_moe_cls_marlin_selects_marlin_for_nvfp4(): + assert get_moe_cls(_marlin_model_config()) is MarlinFusedMoE + + +@pytest.mark.parametrize( + "quant_algo", + [ + pytest.param(None, id="unquantized"), + pytest.param(QuantAlgo.FP8, id="fp8"), + ], +) +def test_get_moe_cls_marlin_falls_back_to_cutlass_on_non_nvfp4(quant_algo): + """MARLIN + non-NVFP4 layers (e.g. unquantized MTP draft layers in + MIXED_PRECISION checkpoints) fall back to CutlassFusedMoE instead of + raising, matching CUTEDSL/DENSEGEMM fallback behavior.""" + assert get_moe_cls(_marlin_model_config(quant_algo)) is CutlassFusedMoE + + +def test_get_moe_cls_marlin_override_quant_config_per_layer(): + """Per-layer override (the MTP draft-layer path): an unquantized per-layer + override falls back to Cutlass even though the global config is NVFP4.""" + cfg = _marlin_model_config() + assert ( + get_moe_cls(cfg, override_quant_config=QuantConfig(quant_algo=None), layer_idx=52) + is CutlassFusedMoE + ) + + def test_megamoe_cutedsl_post_load_weights_uses_staged_hooks(): moe = MegaMoECuteDsl.__new__(MegaMoECuteDsl) torch.nn.Module.__init__(moe) @@ -477,6 +514,63 @@ class DummyModule: method.post_load_weights(DummyModule()) +def _make_megamoe_cutedsl_for_ctor_test() -> MegaMoECuteDsl: + model_config = ModelConfig( + mapping=Mapping(world_size=1, rank=0, tp_size=1, moe_tp_size=1, moe_ep_size=1), + moe_backend=MoeBackendType.MEGAMOE_CUTEDSL.value, + skip_create_weights_in_init=True, + ) + return MegaMoECuteDsl( + routing_method=RenormalizeMoeRoutingMethod(top_k=2), + num_experts=8, + hidden_size=512, + intermediate_size=512, + dtype=torch.bfloat16, + model_config=model_config, + init_load_balancer=False, + ) + + +def test_megamoe_cutedsl_tuning_mode_forces_top_maxt_bucket( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Profiling scratch is sized for the largest adaptive bucket. + monkeypatch.setenv("MEGAMOE_TACTIC_AUTOTUNE", "1") + moe = _make_megamoe_cutedsl_for_ctor_test() + buckets = moe._maxt_buckets + assert len(buckets) >= 2, f"expected a multi-bucket ladder, got {buckets}" + small_hint = buckets[0] + assert moe._select_launch_max_tokens(small_hint) == buckets[0] + monkeypatch.setattr(AutoTuner.get(), "is_tuning_mode", True) + assert moe._select_launch_max_tokens(small_hint) == buckets[-1] + + +def test_megamoe_cutedsl_tactic_autotune_defaults_off( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Standard serving must not pay for the 36-tactic sweep by default. + monkeypatch.delenv("MEGAMOE_TACTIC_AUTOTUNE", raising=False) + moe = _make_megamoe_cutedsl_for_ctor_test() + assert moe.tactic_autotune is False + + +def test_enumerate_megamoe_candidate_tactics_curated_space() -> None: + from tensorrt_llm._torch.custom_ops import cute_dsl_megamoe_custom_op as megamoe_op + + decode = megamoe_op.enumerate_megamoe_candidate_tactics(1024) + prefill = megamoe_op.enumerate_megamoe_candidate_tactics(16384) + assert len(decode) == len(prefill) == 36 + assert {t[-1] for t in decode} == {(1, 1)} + assert {t[-1] for t in prefill} == {(2, 4)} + # The deterministic fallback stays inside the curated axes. + for num_tokens in (64, 4096, 16384): + megamoe_op.validate_megamoe_tactic(megamoe_op.default_megamoe_tactic(num_tokens)) + invalid_tactic = list(megamoe_op.default_megamoe_tactic(64)) + invalid_tactic[2] = 511 + with pytest.raises(ValueError, match=r"group_hint must be an int >= 512"): + megamoe_op.validate_megamoe_tactic(tuple(invalid_tactic)) + + def run_backend_moe( backend: MoE, backend_type: MoeBackendType, diff --git a/tests/unittest/_torch/modules/moe/test_moe_comm.py b/tests/unittest/_torch/modules/moe/test_moe_comm.py index fe8f6cc2723b..e988723317a2 100644 --- a/tests/unittest/_torch/modules/moe/test_moe_comm.py +++ b/tests/unittest/_torch/modules/moe/test_moe_comm.py @@ -68,6 +68,7 @@ ) from tensorrt_llm._torch.modules.fused_moe.communication.deep_ep import DeepEP from tensorrt_llm._torch.modules.fused_moe.communication.deep_ep_low_latency import DeepEPLowLatency +from tensorrt_llm._torch.modules.fused_moe.communication.nccl_ep import NcclEP from tensorrt_llm._torch.modules.fused_moe.communication.nvlink_one_sided import NVLinkOneSided from tensorrt_llm._torch.modules.fused_moe.communication.nvlink_two_sided import NVLinkTwoSided from tensorrt_llm._torch.modules.fused_moe.communication.nvlink_two_sided_flashinfer import ( @@ -75,6 +76,7 @@ ) from tensorrt_llm._torch.modules.fused_moe.deep_ep_utils import deep_ep_installed from tensorrt_llm._torch.modules.fused_moe.ep_group_health import EPGroupHealth +from tensorrt_llm._torch.modules.fused_moe.nccl_ep_utils import is_nccl_ep_installed from tensorrt_llm.deep_ep.buffer import Buffer from tensorrt_llm.mapping import Mapping @@ -95,6 +97,7 @@ COMM_NVLINK_ONE_SIDED = "NVLinkOneSided" COMM_NVLINK_TWO_SIDED = "NVLinkTwoSided" COMM_NVLINK_TWO_SIDED_FLASHINFER = "NVLinkTwoSidedFlashinfer" +COMM_NCCL_EP = "NcclEP" ALL_COMM_TYPES = [ COMM_ALLGATHER_RS, @@ -103,6 +106,7 @@ COMM_NVLINK_ONE_SIDED, COMM_NVLINK_TWO_SIDED, COMM_NVLINK_TWO_SIDED_FLASHINFER, + COMM_NCCL_EP, ] # Must be in DeepEPLowLatency.SUPPORTED_HIDDEN_SIZES @@ -602,6 +606,16 @@ def create_comm_object( alltoall_result_do_sum=True, ) + elif comm_type == COMM_NCCL_EP: + return NcclEP( + mapping=mapping, + num_slots=num_slots, + hidden_size=config.hidden_size, + max_num_tokens=max_num_tokens, + moe_max_num_tokens=max_num_tokens, + top_k=config.top_k, + ) + else: raise ValueError(f"Unknown comm type: {comm_type}") @@ -740,6 +754,11 @@ def check_platform_support(comm_type: str) -> Optional[str]: if comm_type == COMM_NVLINK_TWO_SIDED_FLASHINFER: return _check_flashinfer_mnnvl_support() + if comm_type == COMM_NCCL_EP: + if not is_nccl_ep_installed(): + return "NCCL EP not available (install the nccl4py wheel)" + return None + return f"Unknown comm type: {comm_type}" @@ -790,6 +809,10 @@ def check_feasibility(comm_type: str, config: CommTestConfig) -> Optional[str]: if config.top_k > NVLinkOneSided.MAX_TOP_K: return f"NVLinkOneSided MAX_TOP_K={NVLinkOneSided.MAX_TOP_K}, got top_k={config.top_k}" + if comm_type == COMM_NCCL_EP: + if config.quant_mode != "none": + return f"NcclEP does not support quant_mode={config.quant_mode}" + if comm_type == COMM_NVLINK_TWO_SIDED_FLASHINFER: # FlashInfer alltoallv requires every 2D payload row to be 16-byte aligned. # This test dispatches both int32 slots [N, top_k] and bf16 scales @@ -1175,6 +1198,109 @@ def _worker_full_pipeline(config: CommTestConfig) -> dict: raise +def _nccl_ep_replay_slots( + *, + target_rank: int, + num_tokens: int, + experts_per_rank: int, +) -> torch.Tensor: + """Route every local token to one EP rank, using distinct local experts.""" + local_experts = torch.arange(num_tokens, device="cuda", dtype=torch.int32) + local_experts %= experts_per_rank + return (target_rank * experts_per_rank + local_experts).view(num_tokens, 1) + + +def _worker_nccl_ep_cuda_graph_replay(config: CommTestConfig) -> dict: + """Capture LL dispatch, change routing, and verify the replay sees the change. + + ``NcclEP.dispatch`` converts the stable input routing tensor to the dtype + expected by nccl-ep inside the graph. The captured handle therefore must + consume the updated device buffer on each replay, rather than reusing the + routes present while the graph was captured. + """ + rank = tllm.mpi_rank() + torch.cuda.set_device(rank) + comm = None + try: + mapping = Mapping( + rank=rank, + tp_size=config.ep_size, + moe_ep_size=config.ep_size, + world_size=config.ep_size, + ) + comm = create_comm_object(COMM_NCCL_EP, mapping, config) + num_tokens = config.all_num_tokens[rank] + experts_per_rank = config.num_experts // config.ep_size + all_rank_num_tokens = config.all_num_tokens + + # A rank-tagged payload makes a routing change visible without relying + # on private nccl-ep state: local routing receives rank + 1, while the + # second replay must receive the peer rank tag. + hidden_states = torch.full( + (num_tokens, config.hidden_size), + float(rank + 1), + dtype=torch.bfloat16, + device="cuda", + ) + weights = torch.ones(num_tokens, 1, dtype=torch.float32, device="cuda") + local_routes = _nccl_ep_replay_slots( + target_rank=rank, + num_tokens=num_tokens, + experts_per_rank=experts_per_rank, + ) + peer_rank = (rank + 1) % config.ep_size + peer_routes = _nccl_ep_replay_slots( + target_rank=peer_rank, + num_tokens=num_tokens, + experts_per_rank=experts_per_rank, + ) + + # Initialize the context and handle eagerly. Capture is intentionally + # rejected before this point, so this mirrors production graph setup. + comm.dispatch( + hidden_states, + None, + local_routes, + weights, + all_rank_num_tokens, + ) + torch.cuda.synchronize() + + static_routes = local_routes.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + recv_hs, _, recv_slots, _ = comm.dispatch( + hidden_states, + None, + static_routes, + weights, + all_rank_num_tokens, + ) + torch.cuda.synchronize() + + def replay_and_check(expected_sender: int) -> dict: + graph.replay() + torch.cuda.synchronize() + valid = recv_slots[:, 0] >= 0 + received = recv_hs[valid, 0].to(torch.float32).round().to(torch.int64) + return { + "valid_count": int(valid.sum().item()), + "sender_matches": bool(torch.all(received == expected_sender + 1).item()), + } + + static_routes.copy_(local_routes) + local_result = replay_and_check(rank) + static_routes.copy_(peer_routes) + peer_result = replay_and_check(peer_rank) + return {"rank": rank, "local": local_result, "peer": peer_result} + except Exception: + traceback.print_exc() + raise + finally: + if comm is not None: + comm.destroy() + + def _make_rank_mask_config( ep_size: int, local_num_tokens: int, @@ -2296,6 +2422,39 @@ def _run_full_test_group(mpi_pool_executor, group: CommTestGroup): _verify_full_test_results(all_results, pending.config) +def _run_nccl_ep_cuda_graph_replay_test(mpi_pool_executor) -> None: + """Verify graph replay observes routing changed between replays.""" + ep_size = mpi_pool_executor.num_workers + config = CommTestConfig( + comm_type=COMM_NCCL_EP, + ep_size=ep_size, + num_experts=FIXED_NUM_EXPERTS, + top_k=1, + hidden_size=DEFAULT_HIDDEN_SIZE, + all_num_tokens=[16] * ep_size, + ) + skip_reason = _get_skip_reason(config) + if skip_reason: + pytest.skip(skip_reason) + + futures = [ + mpi_pool_executor.submit(_worker_nccl_ep_cuda_graph_replay, config) + for _ in range(config.ep_size) + ] + results = sorted((future.result() for future in futures), key=lambda result: result["rank"]) + for result in results: + rank = result["rank"] + for replay_name in ("local", "peer"): + replay = result[replay_name] + valid_count = replay["valid_count"] + assert valid_count == 16, ( + f"rank {rank}: {replay_name} replay received {valid_count} valid rows, expected 16" + ) + assert replay["sender_matches"], ( + f"rank {rank}: {replay_name} replay did not observe the expected routing buffer" + ) + + def _skip_if_rank_mask_config_unsupported(config: CommTestConfig) -> None: """Skip active-rank-mask tests when NVLinkOneSided cannot run locally.""" skip_reason = check_platform_support(config.comm_type) @@ -2453,6 +2612,12 @@ def test_moe_comm_non_divisible_ep(self, mpi_pool_executor, group: CommTestGroup """Verify NVLinkOneSided with non-divisible EP (num_experts % ep_size != 0).""" _run_full_test_group(mpi_pool_executor, group) + @pytest.mark.threadleak(enabled=False) + @pytest.mark.parametrize("mpi_pool_executor", [2], indirect=True) + def test_nccl_ep_cuda_graph_replay_uses_updated_routing(self, mpi_pool_executor) -> None: + """Verify LL CUDA graph replay reads routing written after capture.""" + _run_nccl_ep_cuda_graph_replay_test(mpi_pool_executor) + @pytest.mark.threadleak(enabled=False) @pytest.mark.parametrize( "mpi_pool_executor,local_num_tokens,top_k", diff --git a/tests/unittest/_torch/modules/moe/test_moe_module.py b/tests/unittest/_torch/modules/moe/test_moe_module.py index f9fcaec58a9c..d9ba6045e1a7 100644 --- a/tests/unittest/_torch/modules/moe/test_moe_module.py +++ b/tests/unittest/_torch/modules/moe/test_moe_module.py @@ -78,6 +78,7 @@ DefaultMoeRoutingMethod, Llama4RenormalizeMoeRoutingMethod, MiniMaxM2MoeRoutingMethod, + MiniMaxM3MoeRoutingMethod, RenormalizeMoeRoutingMethod, RenormalizeNaiveMoeRoutingMethod, SigmoidRenormMoeRoutingMethod, @@ -411,15 +412,18 @@ def _create_routing_method(routing_method_cls, top_k, num_experts, dtype, model_ is_fused=False, # Use PyTorch implementation for testing ) - # MiniMaxM2 routing method requires special parameters - if routing_method_cls == MiniMaxM2MoeRoutingMethod: + # MiniMax routing methods require the correction bias and expert count. + if routing_method_cls in (MiniMaxM2MoeRoutingMethod, MiniMaxM3MoeRoutingMethod): # Create e_score_correction_bias as a zero tensor (no bias correction in test) e_score_correction_bias = torch.zeros(num_experts, dtype=dtype, device="cuda") - return routing_method_cls( + kwargs = dict( top_k=top_k, num_experts=num_experts, callable_e_score_correction_bias=lambda: e_score_correction_bias, ) + if routing_method_cls == MiniMaxM3MoeRoutingMethod: + kwargs["routed_scaling_factor"] = 2.0 + return routing_method_cls(**kwargs) # SigmoidRenorm routing method requires num_experts if routing_method_cls == SigmoidRenormMoeRoutingMethod: @@ -1343,24 +1347,33 @@ def generate_multi_gpu_test_params( return params -def _generate_megamoe_multi_gpu_test_params( +def _generate_focused_multi_gpu_test_params( *, backend_type, quant_algo, - should_skip_fn, + parallel_modes, + comm_methods, + should_skip_fn=None, ) -> List: - """Generate focused MegaMoE module multi-GPU coverage for one backend. - - Both MegaMoE backends share the same multi-GPU matrix shape; only the - backend/quant enum and capability skip hook differ between DeepGemm - (W4A8_MXFP4_MXFP8) and CuteDsl (NVFP4). The comm method is hardcoded to - the ``IGNORE`` sentinel because the fused kernel owns dispatch/combine - (the worker pops ``TRTLLM_FORCE_COMM_METHOD`` and takes the fused path). + """Generate focused module multi-GPU coverage for one backend. + + Shared by the backends that opt out of the full ``COMM_METHODS`` matrix: + + - MegaMoE (DeepGemm / CuteDsl): the fused kernel owns dispatch/combine, + so the comm method is the ``IGNORE`` sentinel (the worker pops + ``TRTLLM_FORCE_COMM_METHOD`` and takes the fused path) and each backend + passes its capability skip hook via ``should_skip_fn``. + - Marlin: an EXTERNAL_COMM backend whose ``run_moe`` can route internally, + but under attention-DP the scheduler precomputes routing and dispatches + plain BF16 activations (W4A16 — no activation scales). Coverage is + pinned to ALLGATHER, which is available on every SM90 box; the NVLink + a2a strategies in ``COMM_METHODS`` require MNNVL fabric that Hopper CI + nodes lack. """ params: List = [] seq_lens = [8] if IS_CI_MODE else SEQ_LENS - for parallel_mode, comm_method in product(MEGAMOE_PARALLEL_MODES, [MEGAMOE_IGNORE_COMM_METHOD]): + for parallel_mode, comm_method in product(parallel_modes, comm_methods): for ( swiglu_alpha, swiglu_beta, @@ -1382,7 +1395,7 @@ def _generate_megamoe_multi_gpu_test_params( [quant_algo], MULTI_GPU_ROUTING_METHODS, ): - if not skip_reason: + if not skip_reason and should_skip_fn is not None: skip_reason = should_skip_fn( parallel_mode, comm_method, @@ -1562,9 +1575,9 @@ def test_configurable_moe_single_gpu( # ============================================================================ # FP32 Routing Bias Tests # ============================================================================ -# MiniMax-M2 and DeepSeek models can have fp32 routing_bias with bf16 model dtype. -# These tests verify that the trtllmGen MoE backend correctly handles fp32 bias -# across all quantization paths (fp4, fp8, mxfp4, fp8_per_tensor). +# MiniMax-M2/M3 and DeepSeek models can have fp32 routing_bias with bf16 model +# dtype. These tests verify that the trtllmGen MoE backend correctly handles +# fp32 bias across all quantization paths (fp4, fp8, mxfp4, fp8_per_tensor). def _create_routing_method_with_bias(routing_method_cls, top_k, num_experts, bias_tensor): @@ -1594,6 +1607,7 @@ def _create_routing_method_with_bias(routing_method_cls, top_k, num_experts, bia "routing_method_cls,moe_model_config", [ (MiniMaxM2MoeRoutingMethod, MoeModelConfig(256, 6, 2048, 1408)), + (MiniMaxM3MoeRoutingMethod, MoeModelConfig(128, 4, 512, 512)), (DeepSeekV3MoeRoutingMethod, MoeModelConfig(256, 8, 7168, 2048, n_group=8, topk_group=4)), ], ) @@ -1611,10 +1625,12 @@ def test_trtllm_gen_fp32_routing_bias(routing_method_cls, moe_model_config, quan """ Test that trtllmGen MoE backend correctly handles fp32 routing_bias. - MiniMax-M2 and DeepSeek models emit fp32 routing_bias from trust_remote_code - model definitions. This test verifies that the fp32 bias is correctly plumbed - through the thop boundary (TORCH_CHECK), Runner::run() (dtypeRoutingBias), - and routing kernels (mDtypeBias) without silent corruption (reading fp32 as bf16). + MiniMax-M2/M3 and DeepSeek models emit fp32 routing_bias from + trust_remote_code model definitions. This test verifies that the fp32 bias + is correctly plumbed through the thop boundary (TORCH_CHECK), Runner::run() + (dtypeRoutingBias), and routing kernels (mDtypeBias) without silent + corruption (reading fp32 as bf16). The MiniMax-M3 case also verifies its + routed scaling factor through the unified ConfigurableMoE output reference. Compares fused trtllmGen output against the PyTorch reference module. """ @@ -1645,9 +1661,9 @@ def test_trtllm_gen_fp32_routing_bias(routing_method_cls, moe_model_config, quan ) dtype_routing_logits = None - if ( - moe_backend == MoeBackendType.TRTLLM.value - and routing_method_cls == DeepSeekV3MoeRoutingMethod + if moe_backend == MoeBackendType.TRTLLM.value and routing_method_cls in ( + DeepSeekV3MoeRoutingMethod, + MiniMaxM3MoeRoutingMethod, ): dtype_routing_logits = torch.float32 @@ -1676,19 +1692,34 @@ def test_trtllm_gen_fp32_routing_bias(routing_method_cls, moe_model_config, quan seq_lens=[8] if IS_CI_MODE else SEQ_LENS, dtypes=DTYPES, backend_types=[ - b for b in BACKEND_TYPES if b != MoeBackendType.MARLIN - ], # Marlin doesn't support fused routing + # Marlin gets focused ALLGATHER coverage below; the NVLink a2a + # strategies in COMM_METHODS require MNNVL fabric unavailable on + # the Hopper (SM90) nodes Marlin runs on. + b + for b in BACKEND_TYPES + if b != MoeBackendType.MARLIN + ], quant_algos=QUANT_ALGOS, routing_methods=MULTI_GPU_ROUTING_METHODS, ) -MULTI_GPU_TEST_PARAMS += _generate_megamoe_multi_gpu_test_params( +MULTI_GPU_TEST_PARAMS += _generate_focused_multi_gpu_test_params( + backend_type=MoeBackendType.MARLIN, + quant_algo=QuantAlgo.NVFP4, + parallel_modes=["DEP"] if IS_CI_MODE else ["DEP", "TEP"], + comm_methods=["ALLGATHER"], +) +MULTI_GPU_TEST_PARAMS += _generate_focused_multi_gpu_test_params( backend_type=MoeBackendType.MEGAMOE_DEEPGEMM, quant_algo=QuantAlgo.W4A8_MXFP4_MXFP8, + parallel_modes=MEGAMOE_PARALLEL_MODES, + comm_methods=[MEGAMOE_IGNORE_COMM_METHOD], should_skip_fn=should_skip_MegaMoEDeepGemm, ) -MULTI_GPU_TEST_PARAMS += _generate_megamoe_multi_gpu_test_params( +MULTI_GPU_TEST_PARAMS += _generate_focused_multi_gpu_test_params( backend_type=MoeBackendType.MEGAMOE_CUTEDSL, quant_algo=QuantAlgo.NVFP4, + parallel_modes=MEGAMOE_PARALLEL_MODES, + comm_methods=[MEGAMOE_IGNORE_COMM_METHOD], should_skip_fn=should_skip_MegaMoECuteDsl, ) diff --git a/tests/unittest/_torch/modules/test_linear_uneven_tp.py b/tests/unittest/_torch/modules/test_linear_uneven_tp.py new file mode 100644 index 000000000000..f2010940750c --- /dev/null +++ b/tests/unittest/_torch/modules/test_linear_uneven_tp.py @@ -0,0 +1,1371 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import pytest +import torch + +from tensorrt_llm._torch.modules.linear import ( + Linear, + TensorParallelMode, + WeightMode, + WeightsLoadingConfig, +) +from tensorrt_llm._utils import get_sm_version, is_sm_100f +from tensorrt_llm.mapping import Mapping +from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo + + +class FakeMapping(Mapping): + def __new__(cls, *args, **kwargs): + return object.__new__(cls) + + def __init__(self, world_size, rank): + super().__init__( + world_size=world_size, + rank=rank, + tp_size=world_size, + ) + self.tp_rank = rank + + +@pytest.fixture(autouse=True) +def seed(): + torch.manual_seed(42) + + +def build_weights(in_features, out_features, quant_algo, bias=True): + if quant_algo == QuantAlgo.NO_QUANT: + w = { + "weight": torch.randn(out_features, in_features) + * torch.rsqrt(torch.tensor(float(in_features))) + } + if bias: + w["bias"] = torch.randn(out_features) + return [w] + elif quant_algo == QuantAlgo.FP8: + fp32_weight = torch.randn(out_features, in_features) * torch.rsqrt( + torch.tensor(float(in_features)) + ) + max_fp8 = torch.finfo(torch.float8_e4m3fn).max + weight_scale = fp32_weight.abs().max() / max_fp8 + fp8_weight = (fp32_weight / weight_scale).to(torch.float8_e4m3fn) + w = { + "weight": fp8_weight, + "weight_scale": weight_scale, + "input_scale": torch.tensor(1.0, dtype=torch.float32), + } + if bias: + w["bias"] = torch.randn(out_features) + return [w] + elif quant_algo == QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN: + fp32_weight = torch.randn(out_features, in_features) * torch.rsqrt( + torch.tensor(float(in_features)) + ) + max_fp8 = torch.finfo(torch.float8_e4m3fn).max + # Per-row scale: one scale per output row + row_max = fp32_weight.abs().amax(dim=1) + weight_scale = row_max / max_fp8 + fp8_weight = (fp32_weight / weight_scale.unsqueeze(1)).to(torch.float8_e4m3fn) + w = { + "weight": fp8_weight, + "weight_scale": weight_scale, + } + if bias: + w["bias"] = torch.randn(out_features) + return [w] + elif quant_algo == QuantAlgo.FP8_BLOCK_SCALES: + fp32_weight = torch.randn(out_features, in_features) * torch.rsqrt( + torch.tensor(float(in_features)) + ) + max_fp8 = torch.finfo(torch.float8_e4m3fn).max + # Per 128-element block scales + scale_rows = math.ceil(out_features / 128) + scale_cols = math.ceil(in_features / 128) + weight_scale = torch.empty(scale_rows, scale_cols, dtype=torch.float32) + fp8_weight = torch.empty(out_features, in_features, dtype=torch.float8_e4m3fn) + for r in range(scale_rows): + for c in range(scale_cols): + r_start, r_end = r * 128, min((r + 1) * 128, out_features) + c_start, c_end = c * 128, min((c + 1) * 128, in_features) + block = fp32_weight[r_start:r_end, c_start:c_end] + block_max = block.abs().max() + s = block_max / max_fp8 + weight_scale[r, c] = s + fp8_weight[r_start:r_end, c_start:c_end] = (block / s).to(torch.float8_e4m3fn) + w = { + "weight": fp8_weight, + "weight_scale": weight_scale, + } + if bias: + w["bias"] = torch.randn(out_features) + return [w] + elif quant_algo == QuantAlgo.NVFP4: + FP8_MAX, E2M1_MAX = 448.0, 6.0 + scaling_vector_size = 16 + fp32_weight = torch.randn( + out_features, in_features, device="cuda", dtype=torch.bfloat16 + ) * torch.rsqrt(torch.tensor(float(in_features))) + weight_amax = fp32_weight.abs().max().float() + weight_scale_2 = weight_amax / (FP8_MAX * E2M1_MAX) + input_scale = torch.tensor(FP8_MAX * E2M1_MAX, dtype=torch.float32) + # Quantize weight to FP4 using the TRTLLM op (NVFP4: sfVecSize=16, UE8M0=False) + global_scale = torch.tensor(FP8_MAX * E2M1_MAX / weight_amax, device="cuda") + fp4_weight, fp4_weight_scale = torch.ops.trtllm.fp4_quantize( + fp32_weight, + global_scale, + scaling_vector_size, + sfUseUE8M0=False, + isSfSwizzledLayout=False, + ) + fp4_weight_scale = fp4_weight_scale.reshape( + out_features, in_features // scaling_vector_size + ) + w = { + "weight": fp4_weight.cpu(), + "weight_scale": fp4_weight_scale.cpu(), + "input_scale": input_scale, + "weight_scale_2": weight_scale_2, + } + if bias: + w["bias"] = torch.randn(out_features) + return [w] + elif quant_algo == QuantAlgo.W4A8_NVFP4_FP8: + scaling_vector_size = 32 + import tensorrt_llm.quantization.utils.fp4_utils as fp4_utils + + # FP4 E2M1: 1.0 = 0b0010. Super-diagonal: M[i, i+1] = 1. + # Packed pairs: element i+1 sits in byte (i+1)//2, nibble (i+1)%2. + # This is an easy way to generate synthetic data that will not cause + # overflows but still requires cross-gpu communication (ie not block diagonal) + packed_cols = in_features // 2 + raw = torch.zeros(out_features, packed_cols, dtype=torch.uint8) + for i in range(min(out_features, in_features - 1)): + j = i + 1 + byte_idx = j // 2 + if j % 2 == 0: + raw[i, byte_idx] = 0x02 # low nibble + else: + raw[i, byte_idx] = 0x20 # high nibble + fp4_weight = raw.view(fp4_utils.float4_e2m1x2) + scale_shape = (out_features, in_features // scaling_vector_size) + fp4_weight_scale = torch.ones(scale_shape, dtype=torch.float32).to(torch.float8_e4m3fn) + input_scale = torch.tensor(1.0, dtype=torch.float32) + weight_scale_2 = torch.tensor(1.0, dtype=torch.float32) + w = { + "weight": fp4_weight, + "weight_scale": fp4_weight_scale, + "input_scale": input_scale, + "weight_scale_2": weight_scale_2, + } + if bias: + w["bias"] = torch.zeros(out_features) + return [w] + elif quant_algo in (QuantAlgo.W4A8_MXFP4_FP8, QuantAlgo.W4A8_MXFP4_MXFP8): + scaling_vector_size = 32 + fp32_weight = torch.randn( + out_features, in_features, device="cuda", dtype=torch.bfloat16 + ) * torch.rsqrt(torch.tensor(float(in_features))) + # MXFP4: sfVecSize=32, UE8M0=True, no globalScale needed + fp4_weight, fp4_weight_scale = torch.ops.trtllm.fp4_quantize( + fp32_weight, None, scaling_vector_size, sfUseUE8M0=True, isSfSwizzledLayout=False + ) + fp4_weight_scale = fp4_weight_scale.reshape( + out_features, in_features // scaling_vector_size + ) + w = { + "weight": fp4_weight.cpu(), + "weight_scale": fp4_weight_scale.cpu(), + } + if bias: + w["bias"] = torch.randn(out_features) + return [w] + elif quant_algo == QuantAlgo.W8A16: + # Match the existing THOP weight-only linear test: quantize a logical + # (in_features, out_features) matrix and store checkpoint weight as + # (out_features, in_features). + fp32_weight = torch.randn(in_features, out_features) * torch.rsqrt( + torch.tensor(float(in_features)) + ) + quant_weight, _, weight_scale = ( + torch.ops.trtllm._symmetric_quantize_last_axis_of_batched_matrix( + fp32_weight.cpu(), torch.int8 + ) + ) + w = { + "weight": quant_weight.T.contiguous(), + "weight_scale": weight_scale, + } + if bias: + w["bias"] = torch.randn(out_features) + return [w] + elif quant_algo == QuantAlgo.W4A16: + # INT4 weight-only checkpoint stores the output dimension packed 2:1: + # quant_weight is (in_features, out_features // 2), so transposed + # checkpoint weight is (out_features // 2, in_features). + fp32_weight = torch.randn(in_features, out_features) * torch.rsqrt( + torch.tensor(float(in_features)) + ) + quant_weight, _, weight_scale = ( + torch.ops.trtllm._symmetric_quantize_last_axis_of_batched_matrix( + fp32_weight.cpu(), torch.quint4x2 + ) + ) + w = { + "weight": quant_weight.T.contiguous(), + "weight_scale": weight_scale, + } + if bias: + w["bias"] = torch.randn(out_features) + return [w] + elif quant_algo in (QuantAlgo.W4A16_AWQ, QuantAlgo.W4A8_AWQ): + group_size = 128 + dtype = torch.float16 if quant_algo == QuantAlgo.W4A16_AWQ else torch.bfloat16 + # Checkpoint weight is packed along output dim. Use a sparse synthetic + # super-diagonal so sharded and full GEMMs compare stably. + raw_weight = torch.zeros(in_features, out_features // 2, dtype=torch.uint8, device="cuda") + for i in range(min(in_features, out_features - 1)): + j = i + 1 + byte_idx = j // 2 + if j % 2 == 0: + raw_weight[i, byte_idx] = 0x01 # low nibble + else: + raw_weight[i, byte_idx] = 0x10 # high nibble + pre_quant_scale = torch.ones(in_features, dtype=dtype, device="cuda") + scale_dtype = torch.float32 if quant_algo == QuantAlgo.W4A16_AWQ else torch.float16 + weight_scale = torch.ones( + in_features // group_size, out_features, dtype=scale_dtype, device="cuda" + ) + w = { + "weight": raw_weight.T.contiguous(), + "weight_scale": weight_scale.T.contiguous(), + "pre_quant_scale": pre_quant_scale, + } + if quant_algo == QuantAlgo.W4A8_AWQ: + w["input_scale"] = torch.tensor(1.0, dtype=torch.float32) + w["weight_scale_2"] = torch.tensor(1.0, dtype=torch.float32) + if bias: + w["bias"] = torch.zeros(out_features) + return [w] + else: + raise NotImplementedError(f"Test does not support QuantAlgo {quant_algo}") + + +DEFAULT_DTYPES = { + QuantAlgo.NO_QUANT: torch.float32, + QuantAlgo.FP8: torch.float32, + QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN: torch.bfloat16, + QuantAlgo.FP8_BLOCK_SCALES: torch.bfloat16, + QuantAlgo.NVFP4: torch.bfloat16, + QuantAlgo.W4A8_NVFP4_FP8: torch.bfloat16, + QuantAlgo.W4A8_MXFP4_FP8: torch.bfloat16, + QuantAlgo.W4A8_MXFP4_MXFP8: torch.bfloat16, + QuantAlgo.W8A16: torch.float16, + QuantAlgo.W4A16: torch.float16, + QuantAlgo.W4A16_AWQ: torch.float16, + QuantAlgo.W4A8_AWQ: torch.bfloat16, +} + + +def build_linears( + in_features, + out_features, + world_size, + quant_algo, + bias=True, + dtype=None, + overrides=None, + **kwargs, +): + """Build one Linear per rank, load shared weights. + + Args: + overrides: Optional list of per-rank override_tp_sharding tuples. + Length must equal world_size. Overrides auto tp_sharding. + """ + weights = build_weights(in_features, out_features, quant_algo, bias=bias) + if dtype is None: + dtype = DEFAULT_DTYPES[quant_algo] + if overrides is not None: + assert len(overrides) == world_size + linears = [] + for rank in range(world_size): + mapping = FakeMapping(world_size, rank) + if quant_algo in (QuantAlgo.W4A16_AWQ, QuantAlgo.W4A8_AWQ): + quant_config = QuantConfig(quant_algo=quant_algo, group_size=128, has_zero_point=False) + else: + quant_config = QuantConfig(quant_algo=quant_algo) + override = overrides[rank] if overrides is not None else None + linear = Linear( + in_features, + out_features, + bias=bias, + dtype=dtype, + mapping=mapping, + quant_config=quant_config, + reduce_output=False, + override_tp_sharding=override, + **kwargs, + ) + linear.load_weights(weights) + linear.post_load_weights() + linear.cuda() + linears.append(linear) + return linears, weights + + +def _fused_shard_indices_mapping(shard_keys, ranges): + mapping = {} + offset = 0 + for key in shard_keys: + start, end = ranges[key] + size = end - start + mapping[key] = (offset, size) + offset += size + return mapping + + +def _prepare_fused_weights_for_loading(weights, quant_algo): + if quant_algo == QuantAlgo.NVFP4: + shared_weight_scale_2 = weights[0]["weight_scale_2"].clone() + for weight in weights: + weight["weight_scale"] = weight["weight_scale"].view(torch.float8_e4m3fn) + weight["weight_scale_2"] = shared_weight_scale_2.clone() + + +def build_fused_linears( + in_features, + sub_out_features, + world_size, + quant_algo, + weight_mode, + shard_keys, + overrides=None, + allow_partial_loading=False, +): + weights = [ + build_weights(in_features, sub_out_features, quant_algo, bias=True)[0] for _ in shard_keys + ] + _prepare_fused_weights_for_loading(weights, quant_algo) + dtype = DEFAULT_DTYPES[quant_algo] + if overrides is not None: + assert len(overrides) == world_size + linears = [] + for rank in range(world_size): + mapping = FakeMapping(world_size, rank) + quant_config = QuantConfig(quant_algo=quant_algo) + override = overrides[rank] if overrides is not None else None + shard_indices_mapping = ( + _fused_shard_indices_mapping(shard_keys, override) + if override is not None and (allow_partial_loading or quant_algo == QuantAlgo.NVFP4) + else None + ) + linear = Linear( + in_features, + sub_out_features * len(shard_keys), + bias=True, + dtype=dtype, + mapping=mapping, + quant_config=quant_config, + weights_loading_config=WeightsLoadingConfig(weight_mode=weight_mode), + reduce_output=False, + override_tp_sharding=override, + fused_weight_shard_indices_mapping=shard_indices_mapping, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + linear.load_weights(weights, allow_partial_loading=allow_partial_loading) + if allow_partial_loading: + linear.process_weights_after_loading() + linear.post_load_weights() + linear.cuda() + linears.append(linear) + return linears, weights + + +def build_fused_reference( + in_features, sub_out_features, quant_algo, weight_mode, shard_keys, weights +): + dtype = DEFAULT_DTYPES[quant_algo] + mapping = FakeMapping(1, 0) + quant_config = QuantConfig(quant_algo=quant_algo) + shard_indices_mapping = _fused_shard_indices_mapping( + shard_keys, {key: (0, sub_out_features) for key in shard_keys} + ) + ref = Linear( + in_features, + sub_out_features * len(shard_keys), + bias=True, + dtype=dtype, + mapping=mapping, + quant_config=quant_config, + weights_loading_config=WeightsLoadingConfig(weight_mode=weight_mode), + reduce_output=False, + fused_weight_shard_indices_mapping=shard_indices_mapping, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + ref.load_weights(weights) + ref.post_load_weights() + ref.cuda() + return ref + + +def build_reference(in_features, out_features, quant_algo, weights, bias=True, **kwargs): + """Build a single tp_size=1 linear loaded with the given weights.""" + dtype = DEFAULT_DTYPES.get(quant_algo, torch.float32) + mapping = FakeMapping(1, 0) + quant_config = QuantConfig(quant_algo=quant_algo) + ref = Linear( + in_features, + out_features, + bias=bias, + dtype=dtype, + mapping=mapping, + quant_config=quant_config, + reduce_output=False, + **kwargs, + ) + ref.load_weights(weights) + ref.post_load_weights() + ref.cuda() + return ref + + +def _legacy_even_slice(total, tp_size, rank): + shard = total // tp_size + return rank * shard, (rank + 1) * shard + + +def _assert_same_storage(actual, expected): + torch.testing.assert_close( + actual.detach().cpu().view(torch.uint8), + expected.detach().cpu().view(torch.uint8), + rtol=0, + atol=0, + ) + + +def _check_fused_weight_reconstruction(linears, weights, shard_keys, per_rank_ranges): + for rank, linear in enumerate(linears): + expected_weights = [] + expected_biases = [] + for key in shard_keys: + start, end = per_rank_ranges[rank][key] + expected_weights.append(weights[shard_keys.index(key)]["weight"][start:end]) + expected_biases.append(weights[shard_keys.index(key)]["bias"][start:end]) + + expected_weight = torch.cat(expected_weights, dim=0) + expected_bias = torch.cat(expected_biases, dim=0).to(linear.bias.dtype) + _assert_same_storage(linear.weight, expected_weight) + torch.testing.assert_close(linear.bias.detach().cpu(), expected_bias) + + +def _assemble_fused_outputs(outputs, shard_keys, per_rank_ranges): + per_key_outputs = {key: [] for key in shard_keys} + for output, ranges in zip(outputs, per_rank_ranges): + offset = 0 + for key in shard_keys: + start, end = ranges[key] + size = end - start + per_key_outputs[key].append(output[..., offset : offset + size]) + offset += size + + return torch.cat([torch.cat(per_key_outputs[key], dim=-1) for key in shard_keys], dim=-1) + + +def _check_fused_forward(linears, ref, shard_keys, per_rank_ranges, quant_algo): + x = torch.randn(2, ref.in_features, device="cuda", dtype=DEFAULT_DTYPES[quant_algo]) + outputs = [linear(x) for linear in linears] + result = _assemble_fused_outputs(outputs, shard_keys, per_rank_ranges) + expected = ref(x) + if quant_algo == QuantAlgo.NO_QUANT: + torch.testing.assert_close(result, expected, rtol=1e-4, atol=1e-4) + else: + torch.testing.assert_close(result, expected, rtol=0.2, atol=0.2) + + +# ── Test parametrizations ── + +# Pipeline: unified input → column(in→hidden) → row(hidden→out) → sum +# (in_features, hidden, out_features, tp_size) +PIPELINE_CASES = [ + # even + (32, 32, 32, 2), + (32, 64, 32, 4), + # uneven hidden (column out and row in both split unevenly) + (32, 10, 32, 3), + (16, 7, 16, 2), + (16, 13, 16, 4), + (8, 5, 8, 3), +] + + +class TestMLP: + """Unified input → ColumnParallel → RowParallel(no allreduce) → sum.""" + + @pytest.mark.parametrize("in_features,hidden,out_features,tp_size", PIPELINE_CASES) + def test_pipeline(self, in_features, hidden, out_features, tp_size): + col_linears, col_weights = build_linears( + in_features, + hidden, + tp_size, + QuantAlgo.NO_QUANT, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_linears, row_weights = build_linears( + hidden, + out_features, + tp_size, + QuantAlgo.NO_QUANT, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + + x = torch.randn(2, in_features, device="cuda") + + partial_outputs = [] + for rank in range(tp_size): + col_out = col_linears[rank](x) + row_out = row_linears[rank](col_out) + partial_outputs.append(row_out) + + result = sum(partial_outputs) + + w_col = col_weights[0]["weight"].cuda() + b_col = col_weights[0]["bias"].cuda() + w_row = row_weights[0]["weight"].cuda() + b_row = row_weights[0]["bias"].cuda() + expected = (x @ w_col.t() + b_col) @ w_row.t() + b_row + torch.testing.assert_close(result, expected, rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("in_features,hidden,out_features,tp_size", PIPELINE_CASES) + def test_pipeline_no_bias(self, in_features, hidden, out_features, tp_size): + col_linears, col_weights = build_linears( + in_features, + hidden, + tp_size, + QuantAlgo.NO_QUANT, + bias=False, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_linears, row_weights = build_linears( + hidden, + out_features, + tp_size, + QuantAlgo.NO_QUANT, + bias=False, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + + x = torch.randn(2, in_features, device="cuda") + + partial_outputs = [] + for rank in range(tp_size): + col_out = col_linears[rank](x) + row_out = row_linears[rank](col_out) + partial_outputs.append(row_out) + + result = sum(partial_outputs) + + w_col = col_weights[0]["weight"].cuda() + w_row = row_weights[0]["weight"].cuda() + expected = (x @ w_col.t()) @ w_row.t() + torch.testing.assert_close(result, expected, rtol=1e-4, atol=1e-4) + + +@pytest.mark.skipif( + get_sm_version() < 89, + reason="FP8 per-tensor is supported on SM 89+ GPUs", +) +class TestFP8QDQMLP: + """FP8QDQ: unified input → ColumnParallel → RowParallel → sum.""" + + @pytest.mark.parametrize( + "in_features,hidden,out_features,tp_size", + [ + (32, 32, 32, 2), + (32, 64, 32, 4), + (64, 48, 64, 3), + ], + ) + def test_pipeline(self, in_features, hidden, out_features, tp_size): + col_linears, col_weights = build_linears( + in_features, + hidden, + tp_size, + QuantAlgo.FP8, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_linears, row_weights = build_linears( + hidden, + out_features, + tp_size, + QuantAlgo.FP8, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + col_ref = build_reference( + in_features, + hidden, + QuantAlgo.FP8, + weights=col_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_ref = build_reference( + hidden, + out_features, + QuantAlgo.FP8, + weights=row_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + + x = torch.randn(2, in_features, device="cuda") + + partial_outputs = [] + for rank in range(tp_size): + col_out = col_linears[rank](x) + row_out = row_linears[rank](col_out) + partial_outputs.append(row_out) + result = sum(partial_outputs) + + expected = row_ref(col_ref(x)) + torch.testing.assert_close(result, expected, rtol=1e-3, atol=1e-3) + + +FP8R = QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN + + +@pytest.mark.skipif( + get_sm_version() != 90, + reason="FP8 rowwise is supported on Hopper GPUs", +) +class TestFP8RowwiseMLP: + """FP8 Rowwise: unified input → ColumnParallel → RowParallel → sum.""" + + @pytest.mark.parametrize( + "in_features,hidden,out_features,tp_size", + [ + (32, 32, 32, 2), + (32, 64, 32, 4), + (64, 48, 64, 3), + ], + ) + def test_pipeline(self, in_features, hidden, out_features, tp_size): + col_linears, col_weights = build_linears( + in_features, + hidden, + tp_size, + FP8R, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_linears, row_weights = build_linears( + hidden, + out_features, + tp_size, + FP8R, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + col_ref = build_reference( + in_features, + hidden, + FP8R, + weights=col_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_ref = build_reference( + hidden, + out_features, + FP8R, + weights=row_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + + # fp8 input bypasses dynamic per-token quantization in both column + # and row linears. Scale keeps values in fp8 normal range. + x = (torch.randn(2, in_features, device="cuda") * 0.1).to(torch.float8_e4m3fn) + + # Column outputs bf16; cast to fp8 before row to avoid requantization + partial_outputs = [] + for rank in range(tp_size): + col_out = col_linears[rank](x).to(torch.float8_e4m3fn) + row_out = row_linears[rank](col_out) + partial_outputs.append(row_out) + result = sum(partial_outputs) + + col_ref_out = col_ref(x).to(torch.float8_e4m3fn) + expected = row_ref(col_ref_out) + # atol accounts for bf16 accumulation order differences between + # sharded and full GEMM (max observed diff ~0.008) + torch.testing.assert_close(result, expected, rtol=1e-2, atol=1e-2) + + +FP8BS = QuantAlgo.FP8_BLOCK_SCALES + +# All shard boundaries must be 128-aligned (scale_span=128 assertion in +# load_shard). The tp_size=3 case distributes five 128-blocks as 2,2,1. +FP8BS_PIPELINE_CASES = [ + (256, 256, 256, 2), + (512, 512, 512, 4), + (640, 640, 640, 3), +] + + +@pytest.mark.skipif( + not (get_sm_version() == 90 or is_sm_100f()), + reason="FP8 block scales are supported on Hopper and SM 100 family GPUs", +) +class TestFP8BlockScalesMLP: + """FP8 Block Scales: column → row pipeline.""" + + @pytest.mark.parametrize("in_features,hidden,out_features,tp_size", FP8BS_PIPELINE_CASES) + def test_pipeline(self, in_features, hidden, out_features, tp_size): + col_linears, col_weights = build_linears( + in_features, + hidden, + tp_size, + FP8BS, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_linears, row_weights = build_linears( + hidden, + out_features, + tp_size, + FP8BS, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + col_ref = build_reference( + in_features, + hidden, + FP8BS, + weights=col_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_ref = build_reference( + hidden, + out_features, + FP8BS, + weights=row_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + + x = torch.randn(2, in_features, device="cuda", dtype=torch.bfloat16) + + partial_outputs = [] + for rank in range(tp_size): + col_out = col_linears[rank](x) + row_out = row_linears[rank](col_out) + partial_outputs.append(row_out) + result = sum(partial_outputs) + + expected = row_ref(col_ref(x)) + torch.testing.assert_close(result, expected, rtol=1e-2, atol=1e-2) + + +@pytest.mark.skipif( + not is_sm_100f(), + reason="This test is only supported on SM 100 family GPUs", +) +class TestNVFP4MLP: + """NVFP4: column → row pipeline. ROW requires 16-aligned shard boundaries.""" + + @pytest.mark.parametrize( + "in_features,hidden,out_features,tp_size", + [ + (256, 256, 256, 2), # even + (256, 256, 256, 3), # uneven: ROW shards 16 blocks → 6,5,5 + (256, 256, 256, 4), # even + ], + ) + def test_pipeline(self, in_features, hidden, out_features, tp_size): + col_linears, col_weights = build_linears( + in_features, + hidden, + tp_size, + QuantAlgo.NVFP4, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_linears, row_weights = build_linears( + hidden, + out_features, + tp_size, + QuantAlgo.NVFP4, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + col_ref = build_reference( + in_features, + hidden, + QuantAlgo.NVFP4, + weights=col_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_ref = build_reference( + hidden, + out_features, + QuantAlgo.NVFP4, + weights=row_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + + x = torch.randn(2, in_features, device="cuda", dtype=torch.bfloat16) + + partial_outputs = [] + for rank in range(tp_size): + col_out = col_linears[rank](x) + row_out = row_linears[rank](col_out) + partial_outputs.append(row_out) + result = sum(partial_outputs) + + expected = row_ref(col_ref(x)) + torch.testing.assert_close(result, expected, rtol=1e-2, atol=1e-2) + + +@pytest.mark.skipif( + not is_sm_100f(), + reason="This test is only supported on SM 100 family GPUs", +) +class TestW4A8MXFP4FP8MLP: + """W4A8 MXFP4/FP8: column → row pipeline. + + CUTLASS MXFP8xMXFP4 kernel requires shard dims divisible by 128. + Uneven test uses explicit overrides with 128-aligned splits. + """ + + def test_pipeline_even(self): + self._run_pipeline(256, 2) + + def test_pipeline_uneven(self): + overrides = [(0, 256), (256, 512), (512, 640)] + self._run_pipeline(640, 3, overrides=overrides) + + def _run_pipeline(self, dim, tp_size, overrides=None): + col_linears, col_weights = build_linears( + dim, + dim, + tp_size, + QuantAlgo.W4A8_MXFP4_FP8, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + overrides=overrides, + ) + row_linears, row_weights = build_linears( + dim, + dim, + tp_size, + QuantAlgo.W4A8_MXFP4_FP8, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + overrides=overrides, + ) + col_ref = build_reference( + dim, + dim, + QuantAlgo.W4A8_MXFP4_FP8, + weights=col_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_ref = build_reference( + dim, + dim, + QuantAlgo.W4A8_MXFP4_FP8, + weights=row_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + x = torch.randn(2, dim, device="cuda", dtype=torch.bfloat16) + partial_outputs = [] + for rank in range(tp_size): + partial_outputs.append(row_linears[rank](col_linears[rank](x))) + result = sum(partial_outputs) + expected = row_ref(col_ref(x)) + torch.testing.assert_close(result, expected, rtol=0.2, atol=0.2) + + +@pytest.mark.skipif( + not is_sm_100f(), + reason="This test is only supported on SM 100 family GPUs", +) +class TestW4A8NVFP4FP8MLP: + """W4A8 NVFP4/FP8: column → row pipeline. + + Uses synthetic weights with reinterpreted scale dtype. + Same 128-aligned override requirement as MXFP4. + """ + + def test_pipeline_even(self): + self._run_pipeline(256, 2) + + def test_pipeline_uneven(self): + overrides = [(0, 256), (256, 512), (512, 640)] + self._run_pipeline(640, 3, overrides=overrides) + + def _run_pipeline(self, dim, tp_size, overrides=None): + col_linears, col_weights = build_linears( + dim, + dim, + tp_size, + QuantAlgo.W4A8_NVFP4_FP8, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + overrides=overrides, + ) + row_linears, row_weights = build_linears( + dim, + dim, + tp_size, + QuantAlgo.W4A8_NVFP4_FP8, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + overrides=overrides, + ) + col_ref = build_reference( + dim, + dim, + QuantAlgo.W4A8_NVFP4_FP8, + weights=col_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_ref = build_reference( + dim, + dim, + QuantAlgo.W4A8_NVFP4_FP8, + weights=row_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + x = torch.randn(2, dim, device="cuda", dtype=torch.bfloat16) + partial_outputs = [] + for rank in range(tp_size): + partial_outputs.append(row_linears[rank](col_linears[rank](x))) + result = sum(partial_outputs) + expected = row_ref(col_ref(x)) + torch.testing.assert_close(result, expected, rtol=0.2, atol=0.2) + + +@pytest.mark.skipif( + not is_sm_100f(), + reason="This test is only supported on SM 100 family GPUs", +) +class TestW4A8MXFP4MXFP8MLP: + """W4A8 MXFP4/MXFP8: inherits W4A8MXFP4FP8, uses mxfp8_quantize for activation.""" + + def test_pipeline_even(self): + self._run_pipeline(256, 2) + + def test_pipeline_uneven(self): + overrides = [(0, 256), (256, 512), (512, 640)] + self._run_pipeline(640, 3, overrides=overrides) + + def _run_pipeline(self, dim, tp_size, overrides=None): + algo = QuantAlgo.W4A8_MXFP4_MXFP8 + col_linears, col_weights = build_linears( + dim, + dim, + tp_size, + algo, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + overrides=overrides, + ) + row_linears, row_weights = build_linears( + dim, + dim, + tp_size, + algo, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + overrides=overrides, + ) + col_ref = build_reference( + dim, + dim, + algo, + weights=col_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_ref = build_reference( + dim, + dim, + algo, + weights=row_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + x = torch.randn(2, dim, device="cuda", dtype=torch.bfloat16) + partial_outputs = [] + for rank in range(tp_size): + partial_outputs.append(row_linears[rank](col_linears[rank](x))) + result = sum(partial_outputs) + expected = row_ref(col_ref(x)) + torch.testing.assert_close(result, expected, rtol=0.2, atol=0.2) + + +@pytest.mark.skipif( + get_sm_version() < 80, + reason="Weight-only INT8/INT4 is supported on Ampere+ GPUs", +) +class TestWeightOnlyQuantMLP: + """Weight-only INT8 and INT4 quantization.""" + + @pytest.mark.parametrize("quant_algo", [QuantAlgo.W8A16, QuantAlgo.W4A16]) + @pytest.mark.parametrize( + "in_features,hidden,out_features,tp_size", + [ + (256, 256, 256, 2), # even + (256, 256, 256, 3), # uneven + ], + ) + def test_pipeline(self, in_features, hidden, out_features, tp_size, quant_algo): + col_linears, col_weights = build_linears( + in_features, + hidden, + tp_size, + quant_algo, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_linears, row_weights = build_linears( + hidden, + out_features, + tp_size, + quant_algo, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + col_ref = build_reference( + in_features, + hidden, + quant_algo, + weights=col_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_ref = build_reference( + hidden, + out_features, + quant_algo, + weights=row_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + + x = torch.randn(2, in_features, device="cuda", dtype=torch.float16) + + partial_outputs = [] + for rank in range(tp_size): + col_out = col_linears[rank](x) + row_out = row_linears[rank](col_out) + partial_outputs.append(row_out) + result = sum(partial_outputs) + + expected = row_ref(col_ref(x)) + torch.testing.assert_close(result, expected, rtol=1e-2, atol=1e-2) + + +class _AWQMLPMixin: + quant_algo = None + + @pytest.mark.parametrize( + "dim,tp_size", + [ + (256, 2), # even + (640, 3), # uneven: 128-group shards -> 256,256,128 + ], + ) + def test_pipeline(self, dim, tp_size): + quant_algo = self.quant_algo + col_linears, col_weights = build_linears( + dim, + dim, + tp_size, + quant_algo, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_linears, row_weights = build_linears( + dim, + dim, + tp_size, + quant_algo, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + col_ref = build_reference( + dim, + dim, + quant_algo, + weights=col_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.COLUMN, + ) + row_ref = build_reference( + dim, + dim, + quant_algo, + weights=row_weights, + bias=True, + tensor_parallel_mode=TensorParallelMode.ROW, + ) + + x = torch.randn(2, dim, device="cuda", dtype=DEFAULT_DTYPES[quant_algo]) + partial_outputs = [] + for rank in range(tp_size): + partial_outputs.append(row_linears[rank](col_linears[rank](x))) + result = sum(partial_outputs) + expected = row_ref(col_ref(x)) + torch.testing.assert_close(result, expected, rtol=1e-2, atol=1e-2) + + +@pytest.mark.skipif( + get_sm_version() < 80, + reason="W4A16 AWQ is supported on Ampere+ GPUs", +) +class TestW4A16AWQMLP(_AWQMLPMixin): + """W4A16 AWQ with grouped scales.""" + + quant_algo = QuantAlgo.W4A16_AWQ + + +@pytest.mark.skipif( + not (get_sm_version() in (89, 90) or is_sm_100f()), + reason="W4A8 AWQ is supported on Ada, Hopper, and SM 100 family GPUs", +) +class TestW4A8AWQMLP(_AWQMLPMixin): + """W4A8 AWQ with grouped scales.""" + + quant_algo = QuantAlgo.W4A8_AWQ + + +FUSED_QUANT_ALGO = QuantAlgo.W4A8_MXFP4_FP8 + + +class TestFusedLinearLoading: + """Fused QKV and Gate/Up loading for legacy even and override uneven TP.""" + + @pytest.mark.parametrize( + "weight_mode,quant_algo,sub_out,tp_size", + [ + (WeightMode.FUSED_QKV_LINEAR, QuantAlgo.NO_QUANT, 96, 3), + (WeightMode.FUSED_GATE_UP_LINEAR, QuantAlgo.NO_QUANT, 96, 3), + ], + ) + def test_even_no_override(self, weight_mode, quant_algo, sub_out, tp_size): + shard_keys = weight_mode.shard_keys + linears, weights = build_fused_linears( + 256, + sub_out, + tp_size, + quant_algo, + weight_mode, + shard_keys, + ) + ranges = [] + for rank in range(tp_size): + start, end = _legacy_even_slice(sub_out, tp_size, rank) + ranges.append({key: (start, end) for key in shard_keys}) + + ref = build_fused_reference(256, sub_out, quant_algo, weight_mode, shard_keys, weights) + _check_fused_weight_reconstruction(linears, weights, shard_keys, ranges) + _check_fused_forward(linears, ref, shard_keys, ranges, quant_algo) + if quant_algo != QuantAlgo.NO_QUANT: + for linear in linears: + assert linear.weight_scale.numel() > 0 + + @pytest.mark.parametrize("quant_algo", [QuantAlgo.NO_QUANT]) + @pytest.mark.parametrize( + "weight_mode", + [ + WeightMode.FUSED_QKV_LINEAR, + WeightMode.FUSED_GATE_UP_LINEAR, + ], + ) + def test_uneven_override(self, weight_mode, quant_algo): + shard_keys = weight_mode.shard_keys + sub_out = 640 + tp_size = 3 + boundaries = [(0, 256), (256, 512), (512, 640)] + overrides = [{key: boundary for key in shard_keys} for boundary in boundaries] + linears, weights = build_fused_linears( + 256, + sub_out, + tp_size, + quant_algo, + weight_mode, + shard_keys, + overrides=overrides, + ) + ranges = [{key: boundary for key in shard_keys} for boundary in boundaries] + + ref = build_fused_reference(256, sub_out, quant_algo, weight_mode, shard_keys, weights) + _check_fused_weight_reconstruction(linears, weights, shard_keys, ranges) + _check_fused_forward(linears, ref, shard_keys, ranges, quant_algo) + if quant_algo != QuantAlgo.NO_QUANT: + for linear in linears: + assert linear.weight_scale.numel() > 0 + + @pytest.mark.parametrize("quant_algo", [QuantAlgo.NO_QUANT]) + @pytest.mark.parametrize( + "weight_mode", + [ + WeightMode.FUSED_QKV_LINEAR, + WeightMode.FUSED_GATE_UP_LINEAR, + ], + ) + def test_uneven_override_partial_loading(self, weight_mode, quant_algo): + shard_keys = weight_mode.shard_keys + sub_out = 640 + tp_size = 3 + boundaries = [(0, 256), (256, 512), (512, 640)] + overrides = [{key: boundary for key in shard_keys} for boundary in boundaries] + linears, weights = build_fused_linears( + 256, + sub_out, + tp_size, + quant_algo, + weight_mode, + shard_keys, + overrides=overrides, + allow_partial_loading=True, + ) + ranges = [{key: boundary for key in shard_keys} for boundary in boundaries] + + ref = build_fused_reference(256, sub_out, quant_algo, weight_mode, shard_keys, weights) + _check_fused_weight_reconstruction(linears, weights, shard_keys, ranges) + _check_fused_forward(linears, ref, shard_keys, ranges, quant_algo) + if quant_algo != QuantAlgo.NO_QUANT: + for linear in linears: + assert linear.weight_scale.numel() > 0 + + +@pytest.mark.skipif( + not is_sm_100f(), + reason="Fused FP4/NVFP4 loading is supported on SM 100 family GPUs", +) +class TestFusedQuantizedLinearLoading: + """Quantized fused QKV and Gate/Up loading for override uneven TP.""" + + @pytest.mark.parametrize( + "weight_mode,quant_algo,sub_out,tp_size", + [ + (WeightMode.FUSED_QKV_LINEAR, FUSED_QUANT_ALGO, 256, 2), + (WeightMode.FUSED_GATE_UP_LINEAR, FUSED_QUANT_ALGO, 256, 2), + ], + ) + def test_even_no_override(self, weight_mode, quant_algo, sub_out, tp_size): + shard_keys = weight_mode.shard_keys + linears, weights = build_fused_linears( + 256, + sub_out, + tp_size, + quant_algo, + weight_mode, + shard_keys, + ) + ranges = [] + for rank in range(tp_size): + start, end = _legacy_even_slice(sub_out, tp_size, rank) + ranges.append({key: (start, end) for key in shard_keys}) + + ref = build_fused_reference(256, sub_out, quant_algo, weight_mode, shard_keys, weights) + _check_fused_weight_reconstruction(linears, weights, shard_keys, ranges) + _check_fused_forward(linears, ref, shard_keys, ranges, quant_algo) + for linear in linears: + assert linear.weight_scale.numel() > 0 + + @pytest.mark.parametrize("quant_algo", [FUSED_QUANT_ALGO]) + @pytest.mark.parametrize( + "weight_mode", + [ + WeightMode.FUSED_QKV_LINEAR, + WeightMode.FUSED_GATE_UP_LINEAR, + ], + ) + def test_uneven_override(self, weight_mode, quant_algo): + shard_keys = weight_mode.shard_keys + sub_out = 640 + tp_size = 3 + boundaries = [(0, 256), (256, 512), (512, 640)] + overrides = [{key: boundary for key in shard_keys} for boundary in boundaries] + linears, weights = build_fused_linears( + 256, + sub_out, + tp_size, + quant_algo, + weight_mode, + shard_keys, + overrides=overrides, + ) + ranges = [{key: boundary for key in shard_keys} for boundary in boundaries] + + ref = build_fused_reference(256, sub_out, quant_algo, weight_mode, shard_keys, weights) + _check_fused_weight_reconstruction(linears, weights, shard_keys, ranges) + _check_fused_forward(linears, ref, shard_keys, ranges, quant_algo) + for linear in linears: + assert linear.weight_scale.numel() > 0 + + @pytest.mark.parametrize("quant_algo", [QuantAlgo.NVFP4]) + @pytest.mark.parametrize( + "weight_mode", + [ + WeightMode.FUSED_QKV_LINEAR, + WeightMode.FUSED_GATE_UP_LINEAR, + ], + ) + def test_uneven_override_partial_loading(self, weight_mode, quant_algo): + shard_keys = weight_mode.shard_keys + sub_out = 640 + tp_size = 3 + boundaries = [(0, 256), (256, 512), (512, 640)] + overrides = [{key: boundary for key in shard_keys} for boundary in boundaries] + linears, weights = build_fused_linears( + 256, + sub_out, + tp_size, + quant_algo, + weight_mode, + shard_keys, + overrides=overrides, + allow_partial_loading=True, + ) + ranges = [{key: boundary for key in shard_keys} for boundary in boundaries] + + ref = build_fused_reference(256, sub_out, quant_algo, weight_mode, shard_keys, weights) + _check_fused_weight_reconstruction(linears, weights, shard_keys, ranges) + _check_fused_forward(linears, ref, shard_keys, ranges, quant_algo) + for linear in linears: + assert linear.weight_scale.numel() > 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unittest/_torch/modules/test_mla_registry.py b/tests/unittest/_torch/modules/test_mla_registry.py new file mode 100644 index 000000000000..908069a02560 --- /dev/null +++ b/tests/unittest/_torch/modules/test_mla_registry.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch + +import torch +from torch import nn + +from tensorrt_llm._torch.attention_backend.interface import PositionalEmbeddingParams, RopeParams +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.modules.mla import MLA +from tensorrt_llm.functional import PositionEmbeddingType + + +class _FakeAttention(nn.Module): + def support_fused_rope(self) -> bool: + return True + + def update_quant_config(self, _quant_config: object) -> None: + pass + + +def _make_mla(config: ModelConfig) -> MLA: + position_embedding = PositionalEmbeddingParams( + type=PositionEmbeddingType.rope_gpt_neox, + rope=RopeParams(dim=2, max_positions=8), + ) + return MLA( + hidden_size=8, + num_attention_heads=2, + num_key_value_heads=1, + qk_nope_head_dim=2, + qk_rope_head_dim=2, + v_head_dim=2, + q_lora_rank=4, + kv_lora_rank=4, + predicted_tokens_per_seq=1, + max_position_embeddings=8, + bias=False, + pos_embd_params=position_embedding, + layer_idx=0, + dtype=torch.bfloat16, + config=config, + o_lora_rank=2, + ) + + +def test_duplicate_layer_ids_preserve_all_mla_registrations() -> None: + target_config = ModelConfig(skip_create_weights_in_init=True) + draft_config = ModelConfig(skip_create_weights_in_init=True) + next_config = ModelConfig(skip_create_weights_in_init=True) + draft_config.extra_attrs = target_config.extra_attrs + next_config.extra_attrs = target_config.extra_attrs + + with patch( + "tensorrt_llm._torch.modules.mla.create_attention", + side_effect=lambda *args, **kwargs: _FakeAttention(), + ): + target_mla = _make_mla(target_config) + draft_mla = _make_mla(draft_config) + next_mla = _make_mla(next_config) + + assert target_mla.layer_idx == draft_mla.layer_idx == next_mla.layer_idx == 0 + assert target_mla.layer_idx_str == "0" + assert draft_mla.layer_idx_str == "0_0" + assert next_mla.layer_idx_str == "0_1" + registry = target_config.extra_attrs["mla_layers"] + assert registry["0"]() is target_mla + assert registry["0_0"]() is draft_mla + assert registry["0_1"]() is next_mla diff --git a/tests/unittest/_torch/modules/test_rotary_embedding.py b/tests/unittest/_torch/modules/test_rotary_embedding.py index cc20dee61a45..13a861c0a494 100644 --- a/tests/unittest/_torch/modules/test_rotary_embedding.py +++ b/tests/unittest/_torch/modules/test_rotary_embedding.py @@ -245,3 +245,83 @@ def test_with_qk_rope_head_dim(self): rp = RopeParams.from_config(self._make_config(qk_rope_head_dim=64)) assert rp.duplicate_data is True assert rp.dim == 64 + + +class TestUnfusedRopeOwnership: + """With rope_fusion=False the Python rotary module owns RoPE; the backend + must receive no position-embedding params. yarn is not listed in + PositionEmbeddingType.is_rope(), which used to leak the params through and + made the attention kernel rotate a second time (double RoPE).""" + + def test_unfused_yarn_rope_is_applied_exactly_once(self): + from tensorrt_llm._torch.attention_backend.interface import \ + PositionalEmbeddingParams + from tensorrt_llm._torch.model_config import ModelConfig + from tensorrt_llm._torch.modules.attention import Attention + from tensorrt_llm.functional import PositionEmbeddingType + + yarn_params = PositionalEmbeddingParams( + type=PositionEmbeddingType.yarn, + rope=RopeParams( + dim=32, + theta=150000, + scale_type=RotaryScalingType.yarn, + scale=32.0, + max_positions=1024, + original_max_positions=256, + beta_fast=32, + beta_slow=1, + duplicate_data=False, + ), + is_neox=True, + ) + attn = Attention( + hidden_size=256, + num_attention_heads=8, + num_key_value_heads=8, + max_position_embeddings=1024, + bias=False, + pos_embd_params=yarn_params, + rope_fusion=False, + layer_idx=0, + dtype=torch.bfloat16, + config=ModelConfig(), + ) + + assert attn.rotary_emb is not None + # 0 means the kernel side received no position embedding. + assert attn.attn.position_embedding_type == 0 + + def test_unfused_yarn_rotation_matches_fused_kernel_convention(self): + """The unfused (Python) yarn rotation must match the NeoX rotate-half + convention the fused kernel applies with the same cos/sin table.""" + head_dim = 64 + num_pos, num_heads = 64, 4 + rope_params = RopeParams( + dim=head_dim, + theta=150000, + scale_type=RotaryScalingType.yarn, + scale=32.0, + max_positions=1024, + original_max_positions=256, + beta_fast=32, + beta_slow=1, + duplicate_data=False, + ) + emb = RotaryEmbedding(rope_params, head_dim=head_dim, is_neox=True) + torch.manual_seed(0) + q = torch.randn(num_pos, num_heads * head_dim) + positions = torch.arange(num_pos).cuda() + # Single-target call takes the pure-torch path. + q_unfused = emb(positions, [q.cuda()])[0].cpu() + + # Reference: NeoX rotate-half with the exact table the kernel reads. + table = emb.rotary_cos_sin.cpu() # (max_pos, 2, head_dim/2) + cos = table[:num_pos, 0, :].unsqueeze(1) + sin = table[:num_pos, 1, :].unsqueeze(1) + qh = q.view(num_pos, num_heads, head_dim) + q1, q2 = qh[..., :head_dim // 2], qh[..., head_dim // 2:] + q_ref = torch.cat((q1 * cos - q2 * sin, q2 * cos + q1 * sin), + dim=-1).reshape(num_pos, -1) + + torch.testing.assert_close(q_unfused, q_ref, rtol=1e-5, atol=1e-5) diff --git a/tests/unittest/_torch/multi_gpu/test_linear.py b/tests/unittest/_torch/multi_gpu/test_linear.py index 8452baa19e7c..4db1c6b053ce 100644 --- a/tests/unittest/_torch/multi_gpu/test_linear.py +++ b/tests/unittest/_torch/multi_gpu/test_linear.py @@ -286,13 +286,8 @@ def test_column_linear(hidden_size, mpi_pool_executor): run_single_rank, *zip(*[(tensor_parallel_size, column_linear_forward, x, [l0_weight], hidden_size, dtype)] * 2)) - if hidden_size % 2 != 0: - with pytest.raises(AssertionError): - for r in results: - assert r is True - else: - for r in results: - assert r is True + for r in results: + assert r is True @pytest.mark.skipif(torch.cuda.device_count() < 2, @@ -311,13 +306,8 @@ def test_row_linear(hidden_size, mpi_pool_executor): run_single_rank, *zip(*[(tensor_parallel_size, row_linear_forward, x, [l0_weight], hidden_size, dtype)] * 2)) - if hidden_size % 2 != 0: - with pytest.raises(AssertionError): - for r in results: - assert r is True - else: - for r in results: - assert r is True + for r in results: + assert r is True @pytest.mark.skipif(torch.cuda.device_count() < 2, diff --git a/tests/unittest/_torch/multimodal/test_encoder_group.py b/tests/unittest/_torch/multimodal/test_encoder_group.py new file mode 100644 index 000000000000..01a60e4f4ca4 --- /dev/null +++ b/tests/unittest/_torch/multimodal/test_encoder_group.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for the generic encoder-group primitives. + +Covers the pure helpers behind ``encode_multimodal_by_groups``: + +* ``_lengths_by_modality`` — invert prompt-ordered ``multimodal_embedding_lengths`` + into per-modality per-item lists using ``mm_item_order``. +* ``_synthesize_single_modality_manifest`` — trivial manifest for single-modality + requests that don't carry an explicit one. +* ``_reorder_embeds_by_manifest`` — slice per-modality tensors and concat in + each request's prompt-order manifest. +""" + +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.models.modeling_multimodal_mixin import ( + _lengths_by_modality, + _reorder_embeds_by_manifest, + _synthesize_single_modality_manifest, +) + + +def _mp(*, mm_item_order=None, embedding_lengths=None, buckets=None): + """Minimal MultimodalParams stand-in — the helpers only touch + ``multimodal_data`` and ``mm_item_order``.""" + data = dict(buckets or {}) + if embedding_lengths is not None: + data["multimodal_embedding_lengths"] = embedding_lengths + return SimpleNamespace(multimodal_data=data, mm_item_order=mm_item_order) + + +class TestLengthsByModality: + def test_mixed_manifest_image_video_image(self): + # Manifest order: image#0 (4 rows), video#0 (6), image#1 (2). + mp = _mp( + mm_item_order=[ + {"modality": "image", "index": 0}, + {"modality": "video", "index": 0}, + {"modality": "image", "index": 1}, + ], + embedding_lengths=[4, 6, 2], + buckets={"image": {}, "video": {}}, + ) + assert _lengths_by_modality([mp], ("image", "video")) == { + "image": [4, 2], + "video": [6], + } + + def test_single_modality_no_manifest(self): + # No mm_item_order; flat list is already in per-modality order. + mp = _mp(embedding_lengths=[64, 64], buckets={"image": {}}) + assert _lengths_by_modality([mp], ("image", "video")) == { + "image": [64, 64], + "video": [], + } + + def test_length_mismatch_raises(self): + # ``strict=True`` inside the helper catches manifest/length divergence. + mp = _mp( + mm_item_order=[ + {"modality": "image", "index": 0}, + {"modality": "video", "index": 0}, + ], + embedding_lengths=[4], # one short + buckets={"image": {}, "video": {}}, + ) + with pytest.raises(ValueError): + _lengths_by_modality([mp], ("image", "video")) + + +class TestSynthesizeSingleModalityManifest: + def test_image_only(self): + mp = _mp(embedding_lengths=[10, 20, 30], buckets={"image": {}}) + assert _synthesize_single_modality_manifest(mp, ("image", "video")) == [ + {"modality": "image", "index": 0}, + {"modality": "image", "index": 1}, + {"modality": "image", "index": 2}, + ] + + def test_no_group_modality_returns_empty(self): + # Request has audio but the group only covers image/video. + mp = _mp(embedding_lengths=[10], buckets={"audio": {}}) + assert _synthesize_single_modality_manifest(mp, ("image", "video")) == [] + + +class TestReorderEmbedsByManifest: + @staticmethod + def _marker_tensor(marker, rows, hidden=2): + # Rows all share a marker so the reorder can be asserted by column-0. + return torch.full((rows, hidden), float(marker)) + + def test_mixed_image_video_image_in_one_request(self): + # Prompt: image#0 (4 rows, marker 10), video#0 (6, 20), image#1 (2, 30). + mp = _mp( + mm_item_order=[ + {"modality": "image", "index": 0}, + {"modality": "video", "index": 0}, + {"modality": "image", "index": 1}, + ], + buckets={"image": {}, "video": {}}, + ) + # Per-modality tensors are cat'd in encounter order — image items 0 + # and 1 concatenated, then video item 0. + per_modality_embeds = { + "image": torch.cat([self._marker_tensor(10, 4), self._marker_tensor(30, 2)], dim=0), + "video": self._marker_tensor(20, 6), + } + per_modality_lengths = {"image": [4, 2], "video": [6]} + out = _reorder_embeds_by_manifest([mp], per_modality_embeds, per_modality_lengths) + # Expected column-0: 10 (×4), 20 (×6), 30 (×2) in prompt order. + expected = torch.tensor([10.0] * 4 + [20.0] * 6 + [30.0] * 2) + assert torch.equal(out[:, 0], expected) + + def test_cross_request_cursors_advance_per_modality(self): + # Request A: [image#0 (2)]; Request B: [video#0 (3), image#0 (1)]. + # Global per-modality indexing: image=[A#0, B#0], video=[B#0]. + # Manifests use per-request indices — cursor must translate them. + mp_a = _mp( + mm_item_order=[{"modality": "image", "index": 0}], + buckets={"image": {}}, + ) + mp_b = _mp( + mm_item_order=[ + {"modality": "video", "index": 0}, + {"modality": "image", "index": 0}, + ], + buckets={"image": {}, "video": {}}, + ) + per_modality_embeds = { + "image": torch.cat([self._marker_tensor(1, 2), self._marker_tensor(3, 1)], dim=0), + "video": self._marker_tensor(2, 3), + } + per_modality_lengths = {"image": [2, 1], "video": [3]} + out = _reorder_embeds_by_manifest([mp_a, mp_b], per_modality_embeds, per_modality_lengths) + # A: image#0 → marker 1 (×2). B: video#0 → 2 (×3), then image#0 → 3 (×1). + expected = torch.tensor([1.0] * 2 + [2.0] * 3 + [3.0] * 1) + assert torch.equal(out[:, 0], expected) + + def test_single_modality_falls_back_to_synthesized_manifest(self): + # Two image items, no explicit manifest — reorder still works by + # synthesizing a trivial per-modality manifest from the request's + # multimodal_embedding_lengths. + mp = _mp(embedding_lengths=[2, 3], buckets={"image": {}}) + per_modality_embeds = { + "image": torch.cat([self._marker_tensor(7, 2), self._marker_tensor(9, 3)], dim=0), + } + per_modality_lengths = {"image": [2, 3]} + out = _reorder_embeds_by_manifest([mp], per_modality_embeds, per_modality_lengths) + expected = torch.tensor([7.0] * 2 + [9.0] * 3) + assert torch.equal(out[:, 0], expected) + + def test_empty_bookkeeping_returns_typed_empty(self): + # Mirrors the executor KV-cache profiling pass: the dummy batch runs the + # encoder but carries no ``multimodal_embedding_lengths``, so per-modality + # lengths are empty and the sliced embeds are zero-row. Reorder must + # return a correctly-typed empty tensor, not crash on ``torch.cat([])``. + mp = _mp(buckets={"image": {}}) # modality present, no manifest, no lengths + hidden = 5 + per_modality_embeds = {"image": torch.empty((0, hidden), dtype=torch.float16)} + per_modality_lengths = {"image": []} + out = _reorder_embeds_by_manifest([mp], per_modality_embeds, per_modality_lengths) + assert out.shape == (0, hidden) + assert out.dtype == torch.float16 + + def test_no_embeds_returns_empty(self): + # Defensive: no group produced embeddings at all — return an empty + # tensor rather than crashing. + mp = _mp(buckets={}) + out = _reorder_embeds_by_manifest([mp], {}, {}) + assert out.numel() == 0 diff --git a/tests/unittest/_torch/multimodal/test_mm_encoder_cross_iter_prefetch.py b/tests/unittest/_torch/multimodal/test_mm_encoder_cross_iter_prefetch.py index 01e538d16092..8ae672f430fe 100644 --- a/tests/unittest/_torch/multimodal/test_mm_encoder_cross_iter_prefetch.py +++ b/tests/unittest/_torch/multimodal/test_mm_encoder_cross_iter_prefetch.py @@ -23,13 +23,19 @@ import pytest import torch +from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models import modeling_multimodal_mixin as mm_mixin from tensorrt_llm._torch.models.modeling_multimodal_mixin import ( MultimodalModelMixin, _get_mm_aux_stream, maybe_prefetch_mm_encoder_for_next_iter, ) +from tensorrt_llm._torch.models.modeling_multimodal_utils import get_multimodal_embeddings from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, SamplingConfig +from tensorrt_llm.inputs.multimodal import MultimodalParams, MultimodalRuntimeData +from tensorrt_llm.llmapi.llm_args import MultimodalConfig + +requires_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") class _StubModel(MultimodalModelMixin): @@ -38,10 +44,12 @@ def __init__(self, hidden_size: int, tokens_per_image: int): self._tokens_per_image = tokens_per_image self.encoder_call_count = 0 self.encoder_call_stream_id = None + self.encoder_mm_item_order = None def encode_multimodal_inputs(self, multimodal_params, **kwargs) -> torch.Tensor: self.encoder_call_count += 1 self.encoder_call_stream_id = torch.cuda.current_stream().cuda_stream + self.encoder_mm_item_order = multimodal_params[0].mm_item_order pv = multimodal_params[0].multimodal_data["image"]["pixel_values"] assert pv.device.type == "cuda", ( "pixel_values should be on CUDA after to_device in the helper." @@ -54,6 +62,47 @@ def encode_multimodal_inputs(self, multimodal_params, **kwargs) -> torch.Tensor: return embeddings +class _CacheStubModel(_StubModel): + supports_encoder_cache = True + + def __init__(self, hidden_size: int, tokens_per_image: int): + super().__init__(hidden_size, tokens_per_image) + self.model_config = ModelConfig( + multimodal_config=MultimodalConfig( + encoder_cache_max_bytes=4096, + encoder_side_stream_max_ahead=2, + ) + ) + self.last_encoder_batch_size = 0 + + @property + def embedding_dim(self) -> int: + return self._hidden_size + + @property + def embedding_dtype(self) -> torch.dtype: + return torch.float32 + + def encode_multimodal_inputs(self, multimodal_params, **kwargs) -> torch.Tensor: + self.encoder_call_count += 1 + self.last_encoder_batch_size = len(multimodal_params) + self.encoder_call_stream_id = torch.cuda.current_stream().cuda_stream + total_rows = sum( + sum( + param.multimodal_data.get( + "multimodal_embedding_lengths", + [self._tokens_per_image], + ) + ) + for param in multimodal_params + ) + return torch.full( + (total_rows, self._hidden_size), + float(self.encoder_call_count), + device="cuda", + ) + + def _make_request(request_id: int, num_tokens: int) -> LlmRequest: pixel_values = torch.randn(3, 32, 32) # CPU tensor, simulating unscheduled cumsum = torch.arange(1, num_tokens + 1, dtype=torch.int64) @@ -72,6 +121,61 @@ def _make_request(request_id: int, num_tokens: int) -> LlmRequest: ) +def _make_cacheable_request( + request_id: int, + num_tokens: int, + *, + item_hash: list[int] | None = None, +) -> LlmRequest: + if item_hash is None: + item_hash = [1, 2, 3, 4, 5, 6, 7, 8] + return LlmRequest( + request_id=request_id, + max_new_tokens=1, + input_tokens=[0] * num_tokens, + sampling_config=SamplingConfig(beam_width=1), + is_streaming=False, + py_multimodal_data={ + "image": {"pixel_values": torch.randn(3, 32, 32)}, + "multimodal_embed_mask_cumsum": torch.arange(1, num_tokens + 1, dtype=torch.int64), + "multimodal_embedding_lengths": [num_tokens], + "mm_processor_kwargs_hash": "kwargs-a", + }, + multimodal_hashes=[item_hash], + multimodal_positions=[0], + multimodal_lengths=[num_tokens], + multimodal_uuids=[None], + ) + + +def _make_two_item_cacheable_request( + request_id: int, + item_hashes: list[list[int]], +) -> LlmRequest: + tokens_per_item = 4 + num_tokens = len(item_hashes) * tokens_per_item + return LlmRequest( + request_id=request_id, + max_new_tokens=1, + input_tokens=[0] * num_tokens, + sampling_config=SamplingConfig(beam_width=1), + is_streaming=False, + py_multimodal_data={ + "image": { + "pixel_values": torch.randn(len(item_hashes), 3, 32, 32), + "image_sizes": [[32, 32]] * len(item_hashes), + }, + "multimodal_embed_mask_cumsum": torch.arange(1, num_tokens + 1, dtype=torch.int64), + "multimodal_embedding_lengths": [tokens_per_item] * len(item_hashes), + "mm_processor_kwargs_hash": "kwargs-a", + }, + multimodal_hashes=item_hashes, + multimodal_positions=list(range(0, num_tokens, tokens_per_item)), + multimodal_lengths=[tokens_per_item] * len(item_hashes), + multimodal_uuids=[None] * len(item_hashes), + ) + + def _make_metadata_only_request(request_id: int, num_tokens: int) -> LlmRequest: cumsum = torch.arange(1, num_tokens + 1, dtype=torch.int64) return LlmRequest( @@ -222,6 +326,7 @@ def test_cross_iter_prefetch_no_op_when_config_off(): assert req.py_mm_encoder_event is None +@requires_cuda def test_cross_iter_prefetch_materializes_on_side_stream(monkeypatch): max_prefetch_ahead = 1 aux_stream = _get_mm_aux_stream(max_prefetch_ahead) @@ -229,12 +334,18 @@ def test_cross_iter_prefetch_materializes_on_side_stream(monkeypatch): model = _StubModel(hidden_size=8, tokens_per_image=4) req = _make_request(request_id=0, num_tokens=4) + expected_item_order = [ + {"modality": "image", "index": 0}, + {"modality": "video", "index": 0}, + ] + req.py_mm_item_order = expected_item_order n = maybe_prefetch_mm_encoder_for_next_iter(model, [req], max_prefetch_ahead=max_prefetch_ahead) assert n == 1 assert model.encoder_call_count == 1 assert model.encoder_call_stream_id == aux_stream.cuda_stream + assert model.encoder_mm_item_order == expected_item_order embedding = req.py_multimodal_data.get("multimodal_embedding") assert isinstance(embedding, torch.Tensor) assert embedding.shape == (4, 8) @@ -242,6 +353,141 @@ def test_cross_iter_prefetch_materializes_on_side_stream(monkeypatch): req.py_mm_encoder_event.synchronize() +@requires_cuda +def test_cross_iter_prefetch_populates_and_reuses_persistent_cache(): + max_prefetch_ahead = 2 + aux_stream = _get_mm_aux_stream(max_prefetch_ahead) + assert aux_stream is not None + model = _CacheStubModel(hidden_size=8, tokens_per_image=4) + + first = _make_cacheable_request(request_id=0, num_tokens=4) + assert ( + maybe_prefetch_mm_encoder_for_next_iter( + model, [first], max_prefetch_ahead=max_prefetch_ahead + ) + == 1 + ) + first.py_mm_encoder_event.synchronize() + first_embedding = first.py_multimodal_data["multimodal_embedding"] + + cache = model._multimodal_encoder_cache + assert cache is not None + assert len(cache) == 1 + assert model.encoder_call_count == 1 + assert model.encoder_call_stream_id == aux_stream.cuda_stream + + second = _make_cacheable_request(request_id=1, num_tokens=4) + assert ( + maybe_prefetch_mm_encoder_for_next_iter( + model, [second], max_prefetch_ahead=max_prefetch_ahead + ) + == 1 + ) + second.py_mm_encoder_event.synchronize() + + assert model.encoder_call_count == 1 + torch.testing.assert_close(second.py_multimodal_data["multimodal_embedding"], first_embedding) + + +@requires_cuda +def test_cross_iter_prefetch_mixed_cache_hit_and_miss_encodes_only_miss(): + max_prefetch_ahead = 2 + model = _CacheStubModel(hidden_size=8, tokens_per_image=4) + first = _make_cacheable_request(request_id=0, num_tokens=4) + maybe_prefetch_mm_encoder_for_next_iter(model, [first], max_prefetch_ahead=max_prefetch_ahead) + first.py_mm_encoder_event.synchronize() + + hit = _make_cacheable_request(request_id=1, num_tokens=4) + miss = _make_cacheable_request(request_id=2, num_tokens=4, item_hash=[9] * 8) + assert ( + maybe_prefetch_mm_encoder_for_next_iter( + model, + [hit, miss], + max_prefetch=2, + max_prefetch_ahead=max_prefetch_ahead, + ) + == 2 + ) + hit.py_mm_encoder_event.synchronize() + + assert model.encoder_call_count == 2 + assert model.last_encoder_batch_size == 1 + assert hit.py_multimodal_data["image"]["pixel_values"].device.type == "cpu" + assert miss.py_multimodal_data["image"]["pixel_values"].device.type == "cuda" + torch.testing.assert_close( + hit.py_multimodal_data["multimodal_embedding"], + torch.ones((4, 8), device="cuda"), + ) + torch.testing.assert_close( + miss.py_multimodal_data["multimodal_embedding"], + torch.full((4, 8), 2.0, device="cuda"), + ) + + +@requires_cuda +def test_cross_iter_prefetch_partial_hit_encodes_only_missing_item(): + max_prefetch_ahead = 2 + model = _CacheStubModel(hidden_size=8, tokens_per_image=4) + hit_hash = [1, 2, 3, 4, 5, 6, 7, 8] + miss_hash = [9] * 8 + + first = _make_cacheable_request(request_id=0, num_tokens=4, item_hash=hit_hash) + maybe_prefetch_mm_encoder_for_next_iter(model, [first], max_prefetch_ahead=max_prefetch_ahead) + first.py_mm_encoder_event.synchronize() + + partial = _make_two_item_cacheable_request(request_id=1, item_hashes=[hit_hash, miss_hash]) + assert ( + maybe_prefetch_mm_encoder_for_next_iter( + model, + [partial], + max_prefetch_ahead=max_prefetch_ahead, + ) + == 1 + ) + partial.py_mm_encoder_event.synchronize() + + assert model.encoder_call_count == 2 + assert model.last_encoder_batch_size == 1 + torch.testing.assert_close( + partial.py_multimodal_data["multimodal_embedding"][:4], + torch.ones((4, 8), device="cuda"), + ) + torch.testing.assert_close( + partial.py_multimodal_data["multimodal_embedding"][4:], + torch.full((4, 8), 2.0, device="cuda"), + ) + cache = model._multimodal_encoder_cache + assert cache is not None + assert len(cache) == 2 + + +@requires_cuda +def test_cross_iter_prefetch_cache_model_preserves_uncacheable_fallbacks(): + max_prefetch_ahead = 2 + model = _CacheStubModel(hidden_size=8, tokens_per_image=4) + unkeyable = _make_request(request_id=0, num_tokens=4) + mixed_modality = _make_cacheable_request(request_id=1, num_tokens=4) + mixed_modality.py_multimodal_data["audio"] = {"input_features": torch.empty(1)} + + assert ( + maybe_prefetch_mm_encoder_for_next_iter( + model, + [unkeyable, mixed_modality], + max_prefetch=2, + max_prefetch_ahead=max_prefetch_ahead, + ) + == 2 + ) + unkeyable.py_mm_encoder_event.synchronize() + + assert model.encoder_call_count == 1 + assert model.last_encoder_batch_size == 2 + cache = model._multimodal_encoder_cache + assert cache is not None + assert len(cache) == 0 + + +@requires_cuda def test_cross_iter_prefetch_skips_in_flight_and_cached(monkeypatch): model = _StubModel(hidden_size=8, tokens_per_image=4) in_flight_req = _make_request(request_id=0, num_tokens=4) @@ -259,3 +505,103 @@ def test_cross_iter_prefetch_skips_in_flight_and_cached(monkeypatch): assert n == 1 assert "multimodal_embedding" not in in_flight_req.py_multimodal_data assert fresh_req.py_multimodal_data.get("multimodal_embedding") is not None + + +@requires_cuda +def test_cross_iter_prefetch_does_not_synchronize_main_stream(mocker): + """Routing through the cache must not block the calling (main) stream. + + The prefetch path may only enqueue aux-stream work and cross-stream waits; a + blocking host/stream synchronization would defeat the overlap the side stream + exists for. Spy on the synchronization entry points and assert none fire while + the routed dispatch runs. + """ + model = _CacheStubModel(hidden_size=8, tokens_per_image=4) + req = _make_cacheable_request(request_id=0, num_tokens=4) + device_synchronize = mocker.spy(torch.cuda, "synchronize") + stream_synchronize = mocker.spy(torch.cuda.Stream, "synchronize") + event_synchronize = mocker.spy(torch.cuda.Event, "synchronize") + + n = maybe_prefetch_mm_encoder_for_next_iter(model, [req], max_prefetch_ahead=2) + + assert n == 1 + device_synchronize.assert_not_called() + stream_synchronize.assert_not_called() + event_synchronize.assert_not_called() + # Drain the queued aux-stream work before teardown. + req.py_mm_encoder_event.synchronize() + + +@requires_cuda +def test_cross_iter_prefetch_does_not_rewrite_request_local_embedding(): + """A present request-local embedding skips both the cache lookup and the write. + + Reproduces the next-iteration in-iter consume of a request whose embedding was + already produced by a prefetch miss: the encoder must not run again and the + persistent cache must not be rewritten. + """ + model = _CacheStubModel(hidden_size=8, tokens_per_image=4) + req = _make_cacheable_request(request_id=0, num_tokens=4) + assert maybe_prefetch_mm_encoder_for_next_iter(model, [req], max_prefetch_ahead=2) == 1 + req.py_mm_encoder_event.synchronize() + + cache = model._multimodal_encoder_cache + stats_before = cache.stats() + assert stats_before.insertions == 1 + + cumsum = req.py_multimodal_data["multimodal_embed_mask_cumsum"] + param = MultimodalParams( + multimodal_input=mm_mixin._build_request_multimodal_input(req, cache_enabled=True), + multimodal_data=req.py_multimodal_data, + multimodal_runtime=MultimodalRuntimeData( + past_seen_token_num=0, + chunk_end_pos=cumsum.numel(), + embed_mask_cumsum=cumsum, + ), + ) + model._get_or_encode_multimodal_embeddings([param]) + + assert model.encoder_call_count == 1 # no re-encode + stats_after = cache.stats() + assert stats_after.insertions == stats_before.insertions + assert stats_after.replacements == stats_before.replacements + + +@requires_cuda +@pytest.mark.parametrize("embedding_count", [1, 2]) +def test_prefetched_embedding_records_main_consumer_stream(monkeypatch, embedding_count): + """The request may release an aux-produced embedding before its main-stream gather completes.""" + aux_stream = torch.cuda.Stream() + consumer_stream = torch.cuda.Stream() + producer_event = torch.cuda.Event() + with torch.cuda.stream(aux_stream): + embeddings = [torch.ones(4, device="cuda") for _ in range(embedding_count)] + producer_event.record(aux_stream) + + record_stream_calls = [] + original_record_stream = torch.Tensor.record_stream + + def record_stream(tensor, stream): + record_stream_calls.append((tensor.data_ptr(), stream.cuda_stream)) + return original_record_stream(tensor, stream) + + monkeypatch.setattr(torch.Tensor, "record_stream", record_stream) + param = MultimodalParams( + multimodal_data={ + "multimodal_embedding": embeddings[0] if embedding_count == 1 else embeddings + }, + encoder_event=producer_event, + ) + + with torch.cuda.stream(consumer_stream): + gathered = get_multimodal_embeddings( + encoder_forward_fn=lambda _: pytest.fail("attached embeddings must skip the encoder"), + multimodal_params=[param], + ) + + consumer_stream.synchronize() + expected_calls = { + (embedding.data_ptr(), consumer_stream.cuda_stream) for embedding in embeddings + } + assert expected_calls.issubset(record_stream_calls) + torch.testing.assert_close(gathered[0], torch.ones(4 * embedding_count, device="cuda")) diff --git a/tests/unittest/_torch/multimodal/test_mm_encoder_standalone.py b/tests/unittest/_torch/multimodal/test_mm_encoder_standalone.py index afedd960f9c3..426a6e659af8 100644 --- a/tests/unittest/_torch/multimodal/test_mm_encoder_standalone.py +++ b/tests/unittest/_torch/multimodal/test_mm_encoder_standalone.py @@ -1034,9 +1034,10 @@ def test_multi_request_batch_chat( inputs_with_embeddings): assert isinstance(input, dict) assert isinstance(input_with_embedding, dict) - assert list( - set(input.keys()) - ^ set(input_with_embedding.keys())) == ["multi_modal_data"] + assert (set(input.keys()) + ^ set(input_with_embedding.keys())) == { + "multi_modal_data", "mm_item_order" + } assert set(input_with_embedding.keys()) == set( ["prompt", "multi_modal_embeddings"]) assert input["prompt"] == input_with_embedding["prompt"] diff --git a/tests/unittest/_torch/multimodal/test_multimodal_mixin.py b/tests/unittest/_torch/multimodal/test_multimodal_mixin.py index bc525e20d805..71aa44539766 100644 --- a/tests/unittest/_torch/multimodal/test_multimodal_mixin.py +++ b/tests/unittest/_torch/multimodal/test_multimodal_mixin.py @@ -76,6 +76,8 @@ def encode_multimodal_inputs(self, multimodal_params, **encoder_kwargs) -> torch class NoEmbeddingMetadataMultimodalModel(DummyMultimodalModel): + supports_encoder_cache = True + @property def embedding_dim(self) -> int: raise NotImplementedError @@ -86,6 +88,8 @@ def embedding_dtype(self) -> torch.dtype: class CountingEncoderMultimodalModel(DummyMultimodalModel): + supports_encoder_cache = True + def __init__( self, embedding: Embedding, @@ -101,9 +105,14 @@ def __init__( def encode_multimodal_inputs(self, multimodal_params, **encoder_kwargs) -> torch.Tensor: self.encode_calls += 1 - total_rows = sum( - param.multimodal_runtime.total_embeds_in_request for param in multimodal_params - ) + total_rows = 0 + for param in multimodal_params: + # Residuals built by the partial-cache path carry `multimodal_embedding_lengths` + # but no `multimodal_runtime`; fall through to the metadata in that case. + if param.multimodal_runtime is not None: + total_rows += param.multimodal_runtime.total_embeds_in_request + else: + total_rows += sum(param.multimodal_data["multimodal_embedding_lengths"]) return torch.full( (total_rows, self.embedding.embedding_dim), float(self.encode_calls), @@ -138,9 +147,18 @@ def make_keyed_multimodal_param( item_hashes = [[1, 2, 3, 4, 5, 6, 7, 8]] if embedding_lengths is None: embedding_lengths = [2] + n_items = len(embedding_lengths) + # Pattern-A image data so the mixin's default `build_multimodal_encoder_input` can + # slice this param (dim-0 `pixel_values [B, C, H, W]` parallel to a per-item + # `image_sizes` list). mm_data = { - "image": {"pixel_values": torch.empty(1)}, + "image": { + "pixel_values": torch.arange(n_items * 3 * 2 * 2, dtype=torch.float32).reshape( + n_items, 3, 2, 2 + ), + "image_sizes": [[2, 2]] * n_items, + }, "multimodal_embedding_lengths": embedding_lengths, "mm_processor_kwargs_hash": kwargs_hash, } @@ -255,6 +273,15 @@ def test_encoder_cache_first_request_writes_per_item_entries(): assert len(model._multimodal_encoder_cache) == 2 +def test_encoder_cache_requires_model_opt_in(): + model = DummyMultimodalModel(make_embedding(hidden_size=4), torch.tensor([7])) + model.model_config = ModelConfig( + multimodal_config=MultimodalConfig(encoder_cache_max_bytes=4096) + ) + + assert not model.encoder_cache_active + + def test_encoder_cache_creation_logs_embedding_row_capacity(): model = CountingEncoderMultimodalModel( make_embedding(hidden_size=4), @@ -372,7 +399,7 @@ def test_encoder_cache_mixed_attached_and_uncached_requests(): assert len(cache) == 1 -def test_encoder_cache_partial_hit_logs_and_uses_encoder(): +def test_encoder_cache_partial_hit_encodes_miss_and_interleaves(): model = CountingEncoderMultimodalModel( make_embedding(hidden_size=4), torch.tensor([7]), @@ -391,13 +418,45 @@ def test_encoder_cache_partial_hit_logs_and_uses_encoder(): with patch("tensorrt_llm._torch.models.modeling_multimodal_mixin.logger.debug") as debug: embeddings = model._get_or_encode_multimodal_embeddings([partial]) - messages = [" ".join(map(str, call.args)) for call in debug.call_args_list] + # Encoder ran twice: once for the initial miss item, once for the residual containing + # only the second request's novel item. Assembled tensor puts the cached hit before + # the freshly encoded miss in item-index order. assert model.encode_calls == 2 - assert embeddings.shape == (4, 4) + torch.testing.assert_close(embeddings[:2], torch.full((2, 4), 1.0)) + torch.testing.assert_close(embeddings[2:], torch.full((2, 4), 2.0)) + assert len(model._multimodal_encoder_cache) == 2 + messages = [" ".join(map(str, call.args)) for call in debug.call_args_list] assert any( - "mm_encoder_cache: cache miss; hit_items=1, total_items=2" in msg for msg in messages + "mm_encoder_cache: partial-hit encode total_items=2 hit_items=1 encoded_items=1" in msg + for msg in messages + ) + + +def test_encoder_cache_partial_hit_batches_encoder_across_partial_params(): + # Two partial-hit params in the same batch must share a single encoder call so + # launch overhead scales with iterations, not with partial-hit count. + model = CountingEncoderMultimodalModel( + make_embedding(hidden_size=4), + torch.tensor([7]), + encoder_cache_max_bytes=4096, + ) + shared = [1, 1, 1, 1, 1, 1, 1, 1] + seed = make_keyed_multimodal_param(item_hashes=[shared, [2] * 8], embedding_lengths=[2, 2]) + partial_a = make_keyed_multimodal_param(item_hashes=[shared, [3] * 8], embedding_lengths=[2, 2]) + partial_b = make_keyed_multimodal_param( + item_hashes=[[2] * 8, [4] * 8], embedding_lengths=[2, 2] ) + model._get_or_encode_multimodal_embeddings([seed]) + encode_calls_before = model.encode_calls + model._get_or_encode_multimodal_embeddings([partial_a, partial_b]) + + # Single batched encoder call for both partial residuals. + assert model.encode_calls == encode_calls_before + 1 + # Both new miss items (`[3]*8` and `[4]*8`) written to cache alongside the two + # already-cached seed items. + assert len(model._multimodal_encoder_cache) == 4 + def test_encoder_cache_logs_rejected_oversized_write(): model = CountingEncoderMultimodalModel( @@ -495,3 +554,171 @@ def test_request_local_multimodal_embedding_wins_over_encoder_cache(): torch.testing.assert_close(embeddings, local_embedding) put.assert_not_called() assert cache.stats().replacements == 0 + + +def test_partition_encoder_cache_dispatches_by_hit_outcome(): + model = CountingEncoderMultimodalModel( + make_embedding(hidden_size=4), + torch.tensor([7]), + encoder_cache_max_bytes=4096, + ) + seeded = make_keyed_multimodal_param(item_hashes=[[1] * 8, [2] * 8], embedding_lengths=[2, 2]) + model._get_or_encode_multimodal_embeddings([seeded]) + cache = model._multimodal_encoder_cache + + full_hit = make_keyed_multimodal_param(item_hashes=[[1] * 8, [2] * 8], embedding_lengths=[2, 2]) + part = model.partition_encoder_cache(full_hit, cache) + assert part.is_full_hit and not part.is_full_miss and part.miss_indices == [] + + full_miss = make_keyed_multimodal_param( + item_hashes=[[8] * 8, [9] * 8], embedding_lengths=[2, 2] + ) + part = model.partition_encoder_cache(full_miss, cache) + assert part.is_full_miss and not part.is_full_hit and part.hits == {} + + partial = make_keyed_multimodal_param(item_hashes=[[1] * 8, [3] * 8], embedding_lengths=[2, 2]) + part = model.partition_encoder_cache(partial, cache) + assert not part.is_full_hit and not part.is_full_miss + assert list(part.hits) == [0] and part.miss_indices == [1] + + +def test_assemble_full_embedding_preserves_item_order(): + per_item = { + 0: torch.tensor([[0.0]]), + 1: torch.tensor([[1.0], [1.5]]), + 2: torch.tensor([[2.0]]), + } + torch.testing.assert_close( + MultimodalModelMixin.assemble_full_embedding(per_item, 3), + torch.tensor([[0.0], [1.0], [1.5], [2.0]]), + ) + # Single-item fast path returns the item tensor without an extra copy. + single = per_item[1] + assert MultimodalModelMixin.assemble_full_embedding({0: single}, 1) is single + + +def test_build_multimodal_encoder_input_slices_packed_grid_thw(): + # Qwen-VL-style layout: `pixel_values` is a single packed tensor sized by the + # cumulative patch counts declared in `image_grid_thw`. `second_per_grid_ts` + # stands in for any per-item sibling field (e.g. Qwen2.5-VL video timing) that + # must stay in sync with the sliced items; `per_request_scalar` stands in for + # non-per-item siblings that must pass through unchanged. + grids = torch.tensor([[1, 1, 2], [1, 1, 3], [1, 1, 1]]) # 2 + 3 + 1 patches + pixels = torch.arange(12, dtype=torch.float32).reshape(6, 2) + per_item_meta = torch.tensor([0.1, 0.2, 0.3]) + param = MultimodalParams( + multimodal_input=MultimodalInput( + multimodal_hashes=[[i] * 8 for i in range(3)], + multimodal_positions=[0, 0, 0], + multimodal_lengths=[2, 3, 1], + ), + multimodal_data={ + "image": { + "pixel_values": pixels, + "image_grid_thw": grids, + "second_per_grid_ts": per_item_meta, + "per_item_list": ["a", "b", "c"], + "per_request_scalar": torch.tensor(42.0), + }, + "multimodal_embedding_lengths": [2, 3, 1], + "mm_processor_kwargs_hash": "kw", + }, + ) + model = DummyMultimodalModel(make_embedding(hidden_size=1), torch.tensor([0])) + + residual = model.build_multimodal_encoder_input(param, [2, 0]) + + # Item 2 spans rows [5], item 0 spans rows [0, 1]; residual concatenates them in + # the requested item order and slices every parallel sibling the same way. + residual_image = residual.multimodal_data["image"] + torch.testing.assert_close( + residual_image["pixel_values"], torch.cat([pixels[5:6], pixels[0:2]], dim=0) + ) + torch.testing.assert_close(residual_image["image_grid_thw"], grids[[2, 0]]) + torch.testing.assert_close(residual_image["second_per_grid_ts"], per_item_meta[[2, 0]]) + assert residual_image["per_item_list"] == ["c", "a"] + torch.testing.assert_close(residual_image["per_request_scalar"], torch.tensor(42.0)) + + +def test_build_multimodal_encoder_input_stacked_crops_padding_to_miss_max_size(): + # `pixel_values` is padded to request-wide 5x5 but item 0's true size is (3, 4) + # and item 1's is (5, 5). Slicing to just item 0 must crop `pixel_values` down to + # (3, 4) so Mistral 3's `batch_pixel_values` (which re-pads to + # `max(image_sizes)`) doesn't apply a negative pad amount. + pixels = torch.arange(2 * 3 * 5 * 5, dtype=torch.float32).reshape(2, 3, 5, 5) + param = MultimodalParams( + multimodal_input=MultimodalInput( + multimodal_hashes=[[i] * 8 for i in range(2)], + multimodal_positions=[0, 0], + multimodal_lengths=[1, 1], + ), + multimodal_data={ + "image": { + "pixel_values": pixels, + "image_sizes": [[3, 4], [5, 5]], + }, + "multimodal_embedding_lengths": [1, 1], + "mm_processor_kwargs_hash": "kw", + }, + ) + model = DummyMultimodalModel(make_embedding(hidden_size=1), torch.tensor([0])) + + residual = model.build_multimodal_encoder_input(param, [0]) + + residual_image = residual.multimodal_data["image"] + assert residual_image["image_sizes"] == [[3, 4]] + assert residual_image["pixel_values"].shape == (1, 3, 3, 4) + # Cropped tensor preserves item 0's top-left (H=0..3, W=0..4) window. + torch.testing.assert_close(residual_image["pixel_values"], pixels[0:1, :, :3, :4]) + + +def test_build_multimodal_encoder_input_slices_audio_input_features(): + # Whisper / Qwen2-Audio / Gemma4-audio layout: `input_features [B, mel, T]` + # stacked on dim 0, with an optional per-item mask that sibling-slices + # automatically. Two clips: item 0 and item 1 -- slice to [1, 0] to also + # confirm item order is preserved. + features = torch.arange(2 * 4 * 3, dtype=torch.float32).reshape(2, 4, 3) + mask = torch.tensor([[1, 1, 0], [1, 1, 1]]) + param = MultimodalParams( + multimodal_input=MultimodalInput( + multimodal_hashes=[[i] * 8 for i in range(2)], + multimodal_positions=[0, 0], + multimodal_lengths=[1, 1], + ), + multimodal_data={ + "audio": { + "input_features": features, + "input_features_mask": mask, + }, + "multimodal_embedding_lengths": [1, 1], + "mm_processor_kwargs_hash": "kw", + }, + ) + model = DummyMultimodalModel(make_embedding(hidden_size=1), torch.tensor([0])) + + residual = model.build_multimodal_encoder_input(param, [1, 0]) + + residual_audio = residual.multimodal_data["audio"] + torch.testing.assert_close(residual_audio["input_features"], features[[1, 0]]) + # Per-item mask is caught by the generic sibling-slice pass. + torch.testing.assert_close(residual_audio["input_features_mask"], mask[[1, 0]]) + + +@pytest.mark.parametrize( + "mm_data, expected_match", + [ + # `_encoder_cache_modality` returns None -> single-modality guard fires. + ({}, "only supports single-modality"), + # Modality present but layout is neither pattern A (image_sizes) nor pattern B + # (grid_thw); default has nothing to dispatch on. + ({"image": {"pixel_values": torch.zeros(2)}}, "cannot slice image layout"), + # Audio modality but no `input_features`; default falls through. + ({"audio": {"nonsense": torch.zeros(2)}}, "cannot slice audio layout"), + ], + ids=["no_modality", "unhandled_image_layout", "unhandled_audio_layout"], +) +def test_build_multimodal_encoder_input_unhandled_layout_raises(mm_data, expected_match): + param = MultimodalParams(multimodal_data=mm_data) + model = DummyMultimodalModel(make_embedding(hidden_size=1), torch.tensor([0])) + with pytest.raises(NotImplementedError, match=expected_match): + model.build_multimodal_encoder_input(param, [0]) diff --git a/tests/unittest/_torch/multimodal/test_qwen2vl_text_only_prompt.py b/tests/unittest/_torch/multimodal/test_qwen2vl_text_only_prompt.py new file mode 100644 index 000000000000..515ed58b7ba7 --- /dev/null +++ b/tests/unittest/_torch/multimodal/test_qwen2vl_text_only_prompt.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Text-only prompts on the Qwen VL input processors carry no MRoPE metadata. + +Without vision spans the M-RoPE coordinates degenerate to the scalar token +positions and the position delta is zero, so the processor emits no +`multimodal_data` at all and the model engine falls back to broadcasting the +scalar positions. Synthesizing an (3, 1, N) `mrope_position_ids` tensor per +request instead costs an O(seq_len) device allocation, and in disaggregated +serving the prefill worker re-registers it as a CUDA IPC handle no one reads. +""" + +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.models.modeling_qwen2vl import ( + Qwen2_5VLInputProcessorBase, + Qwen2VLInputProcessorBase, +) +from tensorrt_llm._torch.models.modeling_qwen3vl import Qwen3VLInputProcessorBase + +_TOKEN_IDS = [151644, 872, 198, 9707, 151645] + +PROCESSOR_CLASSES = [ + Qwen2VLInputProcessorBase, + Qwen2_5VLInputProcessorBase, + Qwen3VLInputProcessorBase, +] + + +class _FakeTokenizer: + def __call__(self, prompt, return_tensors=None): + assert prompt == "text prompt" + assert return_tensors == "pt" + return SimpleNamespace(input_ids=torch.tensor([_TOKEN_IDS])) + + +def _make_processor(processor_cls): + processor = object.__new__(processor_cls) + processor._tokenizer = _FakeTokenizer() + + def _fail(*args, **kwargs): + raise AssertionError("get_mrope_config must not run for a text-only prompt") + + processor.get_mrope_config = _fail + return processor + + +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) +@pytest.mark.parametrize("mm_data", [None, {}], ids=["mm_data_none", "mm_data_empty"]) +def test_text_only_prompt_emits_no_multimodal_data(processor_cls, mm_data): + processor = _make_processor(processor_cls) + + token_ids, extra = processor.call_with_text_prompt( + {"prompt": "text prompt", "multi_modal_data": mm_data}, + sampling_params=None, + ) + + assert token_ids == _TOKEN_IDS + assert extra is None + + +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) +def test_text_only_prompt_without_multi_modal_data_key(processor_cls): + """`multi_modal_data` absent entirely takes the same path.""" + processor = _make_processor(processor_cls) + + token_ids, extra = processor.call_with_text_prompt( + {"prompt": "text prompt"}, + sampling_params=None, + ) + + assert token_ids == _TOKEN_IDS + assert extra is None diff --git a/tests/unittest/_torch/multimodal/test_qwen3vl_mixed_modality_encoder.py b/tests/unittest/_torch/multimodal/test_qwen3vl_mixed_modality_encoder.py new file mode 100644 index 000000000000..11ea35f3c6b7 --- /dev/null +++ b/tests/unittest/_torch/multimodal/test_qwen3vl_mixed_modality_encoder.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Encoder-level tests for Qwen3-VL through the generic mm-encoder-group path. + +Wires an `EncoderGroup` around a stub `encode_batched` marker function and +drives the shared `encode_multimodal_by_groups` helper directly. Asserts that +mixed image+video requests, heterogeneous single-modality batches, and +cross-request batching all produce prompt-order embeddings via one ViT call. +The same helper is what `Qwen3VisionModelBase.forward` (the mm-encoder-only +disagg entrypoint) delegates to, so this suite covers both paths. +""" + +from types import SimpleNamespace +from typing import Optional + +import pytest +import torch + +from tensorrt_llm._torch.models.modeling_multimodal_mixin import ( + EncoderGroup, + encode_multimodal_by_groups, +) +from tensorrt_llm._torch.models.modeling_qwen3vl import _qwen3vl_build_batched_input + + +def _encode_batched(pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + """Marker encoder: one output row per input patch, marker propagates via + column 0. Emulates a modality-blind ViT with no compression.""" + return pixel_values.clone().repeat(1, 2) # widen to hidden=marker_dim*2 + + +_QWEN3VL_ENCODER_GROUPS = ( + EncoderGroup( + modalities=("image", "video"), + encoder_fn=_encode_batched, + build_batched_input=_qwen3vl_build_batched_input, + ), +) + + +def _param( + image: Optional[dict] = None, + video: Optional[dict] = None, + order: Optional[list] = None, + lengths: Optional[list] = None, +) -> SimpleNamespace: + data: dict = {} + if image is not None: + data["image"] = image + if video is not None: + data["video"] = video + if lengths is not None: + data["multimodal_embedding_lengths"] = lengths + return SimpleNamespace(multimodal_data=data, mm_item_order=order) + + +def _patches(marker: int, n: int, dim: int = 2) -> torch.Tensor: + t = torch.zeros((n, dim), dtype=torch.float32) + t[:, 0] = float(marker) + return t + + +def _encode(params: list) -> torch.Tensor: + return encode_multimodal_by_groups(_QWEN3VL_ENCODER_GROUPS, params) + + +def test_image_only_request() -> None: + out = _encode( + [ + _param( + image={ + "pixel_values": _patches(10, 4), + "image_grid_thw": torch.tensor([[1, 2, 2]]), + }, + lengths=[4], + ), + ] + ) + assert torch.equal(out[:, 0], torch.full((4,), 10.0)) + + +def test_mixed_image_video_image_prompt_order() -> None: + # Prompt: image#0 (4 rows), video#0 (6), image#1 (2). + out = _encode( + [ + _param( + image={ + "pixel_values": torch.cat([_patches(10, 4), _patches(30, 2)], dim=0), + "image_grid_thw": torch.tensor([[1, 2, 2], [1, 1, 2]]), + }, + video={ + "pixel_values_videos": _patches(20, 6), + "video_grid_thw": torch.tensor([[2, 1, 3]]), + }, + order=[ + {"modality": "image", "index": 0}, + {"modality": "video", "index": 0}, + {"modality": "image", "index": 1}, + ], + lengths=[4, 6, 2], + ), + ] + ) + expected = torch.tensor([10.0] * 4 + [20.0] * 6 + [30.0] * 2) + assert torch.equal(out[:, 0], expected) + + +def test_mixed_without_manifest_raises() -> None: + """Mixed request must carry mm_item_order; the shared helper enforces this + once for every model that registers a multi-modality EncoderGroup.""" + with pytest.raises(ValueError, match="mm_item_order"): + _encode( + [ + _param( + image={ + "pixel_values": _patches(1, 1), + "image_grid_thw": torch.tensor([[1, 1, 1]]), + }, + video={ + "pixel_values_videos": _patches(2, 1), + "video_grid_thw": torch.tensor([[1, 1, 1]]), + }, + lengths=[1, 1], + # order deliberately omitted + ) + ] + ) + + +def test_batch_of_two_image_only_requests() -> None: + out = _encode( + [ + _param( + image={ + "pixel_values": _patches(1, 2), + "image_grid_thw": torch.tensor([[1, 1, 2]]), + }, + lengths=[2], + ), + _param( + image={ + "pixel_values": _patches(2, 3), + "image_grid_thw": torch.tensor([[1, 1, 3]]), + }, + lengths=[3], + ), + ] + ) + expected = torch.tensor([1.0] * 2 + [2.0] * 3) + assert torch.equal(out[:, 0], expected) + + +def test_heterogeneous_batch_image_only_and_video_only() -> None: + """One image-only request + one video-only request go through the same + modality-blind ViT via the generic path — no per-request manifest needed + because neither individual request is mixed.""" + out = _encode( + [ + _param( + image={ + "pixel_values": _patches(1, 2), + "image_grid_thw": torch.tensor([[1, 1, 2]]), + }, + lengths=[2], + ), + _param( + video={ + "pixel_values_videos": _patches(2, 3), + "video_grid_thw": torch.tensor([[1, 1, 3]]), + }, + lengths=[3], + ), + ] + ) + expected = torch.tensor([1.0] * 2 + [2.0] * 3) + assert torch.equal(out[:, 0], expected) + + +def test_heterogeneous_batch_video_first_then_image() -> None: + """Reviewer's mm-encoder-only regression case: `[video-only req0, + image-only req1]`. `_qwen3vl_build_batched_input` concatenates images + across requests first, then videos, so without prompt-order reordering + the encoder output would be `[img(req1), vid(req0)]` — request-ordered + downstream splits would then hand each request the other's rows. + Assert the shared helper reorders rows into request order.""" + out = _encode( + [ + _param( + video={ + "pixel_values_videos": _patches(20, 8), + "video_grid_thw": torch.tensor([[1, 1, 8]]), + }, + lengths=[8], + ), + _param( + image={ + "pixel_values": _patches(10, 5), + "image_grid_thw": torch.tensor([[1, 1, 5]]), + }, + lengths=[5], + ), + ] + ) + expected = torch.tensor([20.0] * 8 + [10.0] * 5) + assert torch.equal(out[:, 0], expected) + + +def test_raw_inputs_route_through_encode_multimodal_by_groups() -> None: + """Regression: ``Qwen3VLModelBase.forward``'s raw image/video branch + routes through ``encode_multimodal_by_groups`` (via + ``get_multimodal_embeddings``). Confirms the same callable signature + used there returns the expected prompt-order embedding tensor.""" + params = [ + _param( + image={ + "pixel_values": _patches(11, 3), + "image_grid_thw": torch.tensor([[1, 1, 3]]), + }, + video={ + "pixel_values_videos": _patches(22, 5), + "video_grid_thw": torch.tensor([[1, 1, 5]]), + }, + order=[ + {"modality": "video", "index": 0}, + {"modality": "image", "index": 0}, + ], + lengths=[5, 3], + ), + ] + out = encode_multimodal_by_groups(_QWEN3VL_ENCODER_GROUPS, params) + expected = torch.tensor([22.0] * 5 + [11.0] * 3) + assert torch.equal(out[:, 0], expected) + + +def test_mixed_request_alongside_ordered_one_still_rejects_unordered() -> None: + """Even when a sibling request carries a valid manifest, a mixed request + without its own manifest must still raise — validation is per-request.""" + with pytest.raises(ValueError, match="mm_item_order"): + _encode( + [ + _param( + image={ + "pixel_values": _patches(10, 2), + "image_grid_thw": torch.tensor([[1, 1, 2]]), + }, + video={ + "pixel_values_videos": _patches(20, 2), + "video_grid_thw": torch.tensor([[1, 1, 2]]), + }, + order=[ + {"modality": "image", "index": 0}, + {"modality": "video", "index": 0}, + ], + lengths=[2, 2], + ), + _param( + image={ + "pixel_values": _patches(30, 1), + "image_grid_thw": torch.tensor([[1, 1, 1]]), + }, + video={ + "pixel_values_videos": _patches(40, 1), + "video_grid_thw": torch.tensor([[1, 1, 1]]), + }, + lengths=[1, 1], + # No manifest on this request even though it has both modalities. + ), + ] + ) diff --git a/tests/unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py b/tests/unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py index 6ca685efadd5..36da87eca93c 100644 --- a/tests/unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py +++ b/tests/unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py @@ -1,4 +1,5 @@ import base64 +import gc import importlib.util import multiprocessing import pickle @@ -36,6 +37,19 @@ pytestmark = pytest.mark.threadleak(enabled=False) +@pytest.fixture(autouse=True) +def release_shared_cuda_memory(): + """Reclaim producer-side CUDA IPC memory between parametrize IDs.""" + yield + # Break reference cycles so the test-local hf_model actually dies now. + gc.collect() + # Free sent IPC storages whose consumers have already closed them. + torch.cuda.ipc_collect() + # Return freed cached segments to the driver so other processes + # (the next test's Ray workers) can allocate them. + torch.cuda.empty_cache() + + @pytest.mark.part0 @skip_pre_blackwell @pytest.mark.parametrize( @@ -53,6 +67,11 @@ def test_llm_update_weights_fp8(model_dir, fp8_model_dir): additional_kwargs["moe_config"] = { "backend": "DEEPGEMM", } + # moe_intermediate_size is 768, and FP8 block scaling needs each + # MoE TP shard to stay a multiple of the 128 block size, so MoE TP + # is capped at 2 (768 / 2 = 384) and EP covers the rest of tp=4. + additional_kwargs["moe_tensor_parallel_size"] = 2 + additional_kwargs["moe_expert_parallel_size"] = 2 num_hidden_layers = 1 hf_model = RefHFModelWithIPCHandles(fp8_model_dir, num_hidden_layers=num_hidden_layers) tokenizer = AutoTokenizer.from_pretrained(fp8_model_dir) @@ -60,7 +79,7 @@ def test_llm_update_weights_fp8(model_dir, fp8_model_dir): with LLM( model=model_dir, ray_worker_extension_cls="tensorrt_llm.llmapi.rlhf_utils.WorkerExtension", - tensor_parallel_size=2, + tensor_parallel_size=4, load_format="dummy", pipeline_parallel_size=1, kv_cache_config=kv_cache_config, @@ -88,7 +107,7 @@ def test_llm_update_weights_fp8(model_dir, fp8_model_dir): temperature=0, return_generation_logits=True, max_tokens=1024 ) - ipc_handles = hf_model.get_weight_ipc_handles_serialized([0, 1]) + ipc_handles = hf_model.get_weight_ipc_handles_serialized([0, 1, 2, 3]) llm._collective_rpc("update_weights", (ipc_handles,)) # Finalize the update weights @@ -97,6 +116,8 @@ def test_llm_update_weights_fp8(model_dir, fp8_model_dir): llm_logits, ref_logits = run_generate(llm, hf_model, prompts, sampling_params) compare_logits(llm_logits, ref_logits) + del hf_model + @pytest.mark.part1 @skip_pre_blackwell @@ -115,6 +136,11 @@ def test_llm_partial_update_weights_fp8(model_dir, fp8_model_dir): additional_kwargs["moe_config"] = { "backend": "DEEPGEMM", } + # moe_intermediate_size is 768, and FP8 block scaling needs each + # MoE TP shard to stay a multiple of the 128 block size, so MoE TP + # is capped at 2 (768 / 2 = 384) and EP covers the rest of tp=4. + additional_kwargs["moe_tensor_parallel_size"] = 2 + additional_kwargs["moe_expert_parallel_size"] = 2 num_hidden_layers = 1 hf_model = RefHFModelWithIPCHandles(fp8_model_dir, num_hidden_layers=num_hidden_layers) tokenizer = AutoTokenizer.from_pretrained(fp8_model_dir) @@ -122,7 +148,7 @@ def test_llm_partial_update_weights_fp8(model_dir, fp8_model_dir): with LLM( model=model_dir, ray_worker_extension_cls="tensorrt_llm.llmapi.rlhf_utils.WorkerExtension", - tensor_parallel_size=2, + tensor_parallel_size=4, load_format="dummy", pipeline_parallel_size=1, kv_cache_config=kv_cache_config, @@ -169,7 +195,7 @@ def filter_fn(name: str) -> bool: for filter_name in filter_list: weight_filter = common_filter(filter_name=filter_name) ipc_handles = hf_model.get_weight_ipc_handles_serialized( - [0, 1], weight_filter=weight_filter + [0, 1, 2, 3], weight_filter=weight_filter ) llm._collective_rpc("update_weights", (ipc_handles,)) # Finalize the update weights @@ -178,6 +204,8 @@ def filter_fn(name: str) -> bool: llm_logits, ref_logits = run_generate(llm, hf_model, prompts, sampling_params) compare_logits(llm_logits, ref_logits) + del hf_model + class RefNVFP4ModelWithIPCHandles(RefHFModel): """Reference model that loads bf16 weights from HuggingFace, quantizes @@ -330,10 +358,11 @@ def _quantize_and_replicate_weights(self): assert not fusion_buffer, f"Incomplete fusion groups: {list(fusion_buffer.keys())}" + # Only populate the owning device. Extra replicas are materialized + # lazily by ``get_weight_ipc_handles_serialized`` so that GPUs never + # asked for via IPC don't hold NVFP4 quantized tensors that persist + # across parametrize IDs. self.all_weights[self.device_id] = model_weights - for i in range(torch.cuda.device_count()): - if i != self.device_id: - self.all_weights[i] = [(n, p.to(f"cuda:{i}")) for n, p in model_weights] with torch.no_grad(): param_dict = dict(self.model.named_parameters()) @@ -435,6 +464,10 @@ def get_weight_ipc_handles_serialized( device_list = list(range(torch.cuda.device_count())) if device_ids is None else device_ids for device in device_list: + if device not in self.all_weights: + src = self.all_weights[self.device_id] + self.all_weights[device] = [(n, p.to(f"cuda:{device}")) for n, p in src] + all_handles = [] for item in self.all_weights[device]: name, p = item @@ -476,7 +509,7 @@ def test_llm_update_weights_nvfp4(model_dir, kv_cache_dtype): with LLM( model=model_dir, ray_worker_extension_cls="tensorrt_llm.llmapi.rlhf_utils.WorkerExtension", - tensor_parallel_size=2, + tensor_parallel_size=4, load_format="dummy", pipeline_parallel_size=1, kv_cache_config=kv_cache_config, @@ -501,7 +534,7 @@ def test_llm_update_weights_nvfp4(model_dir, kv_cache_dtype): temperature=0, return_generation_logits=True, max_tokens=1024 ) - ipc_handles = hf_model.get_weight_ipc_handles_serialized([0, 1]) + ipc_handles = hf_model.get_weight_ipc_handles_serialized([0, 1, 2, 3]) llm._collective_rpc("update_weights", (ipc_handles,)) llm._collective_rpc("update_weights", (None,)) @@ -509,6 +542,8 @@ def test_llm_update_weights_nvfp4(model_dir, kv_cache_dtype): # Use a looser threshold because NVFP4 logits are compared against a BF16 reference. compare_logits(llm_logits, ref_logits, threshold=0.8) + del hf_model + @pytest.mark.part3 @skip_pre_blackwell @@ -537,7 +572,7 @@ def test_llm_partial_update_weights_nvfp4(model_dir, kv_cache_dtype): with LLM( model=model_dir, ray_worker_extension_cls="tensorrt_llm.llmapi.rlhf_utils.WorkerExtension", - tensor_parallel_size=2, + tensor_parallel_size=4, load_format="dummy", pipeline_parallel_size=1, kv_cache_config=kv_cache_config, @@ -580,7 +615,7 @@ def filter_fn(name: str) -> bool: for filter_name in filter_list: weight_filter = common_filter(filter_name=filter_name) ipc_handles = hf_model.get_weight_ipc_handles_serialized( - [0, 1], weight_filter=weight_filter + [0, 1, 2, 3], weight_filter=weight_filter ) llm._collective_rpc("update_weights", (ipc_handles,)) llm._collective_rpc("update_weights", (None,)) @@ -589,6 +624,8 @@ def filter_fn(name: str) -> bool: # Use a looser threshold because NVFP4 logits are compared against a BF16 reference. compare_logits(llm_logits, ref_logits, threshold=0.8) + del hf_model + @pytest.fixture def mamba_deps(): @@ -726,6 +763,8 @@ def filter_fn(name: str) -> bool: llm_logits, ref_logits = run_generate(llm, hf_model, prompts, sampling_params) compare_logits(llm_logits, ref_logits) + del hf_model + def _nemotron_h_subprocess_entry(result_queue): try: diff --git a/tests/unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py b/tests/unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py index 09703d40cf52..df9322b10921 100644 --- a/tests/unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py +++ b/tests/unittest/_torch/ray_orchestrator/single_gpu/test_llm_update_weights.py @@ -1,4 +1,5 @@ import base64 +import gc import pickle import re from typing import Callable, List, Optional, Tuple @@ -26,6 +27,19 @@ pytestmark = pytest.mark.threadleak(enabled=False) +@pytest.fixture(autouse=True) +def release_shared_cuda_memory(): + """Reclaim producer-side CUDA IPC memory between parametrize IDs.""" + yield + # Break reference cycles so the test-local hf_model actually dies now. + gc.collect() + # Free sent IPC storages whose consumers have already closed them. + torch.cuda.ipc_collect() + # Return freed cached segments to the driver so other processes + # (the next test's Ray workers) can allocate them. + torch.cuda.empty_cache() + + class RefHFModelWithIPCHandles(RefHFModel): def __init__(self, model_dir: str, device_id: int = 0, num_hidden_layers: int = 4): self.device_id = device_id @@ -43,13 +57,12 @@ def _replicate_weights(self): for n, p in self.model.named_parameters(): model_weights.append((n, p.detach().clone())) + # Only populate the owning device. Extra replicas are materialized + # lazily by ``get_weight_ipc_handles_serialized`` so that GPUs never + # asked for via IPC (e.g. cuda:2/3 on a 4-GPU runner when a TP=2 + # test only requests device_ids=[0, 1]) don't hold weight copies + # that persist across parametrize IDs. self.all_weights[self.device_id] = model_weights - for i in range(torch.cuda.device_count()): - if i != self.device_id: - cur_weights = [] - for n, p in self.all_weights[self.device_id]: - cur_weights.append((n, p.to("cuda:" + str(i)))) - self.all_weights[i] = cur_weights def get_weight_ipc_handles_serialized( self, @@ -70,6 +83,10 @@ def get_weight_ipc_handles_serialized( device_list = list(range(torch.cuda.device_count())) if device_ids is None else device_ids for device in device_list: + if device not in self.all_weights: + src = self.all_weights[self.device_id] + self.all_weights[device] = [(n, p.to(f"cuda:{device}")) for n, p in src] + all_handles = [] for item in self.all_weights[device]: name, p = item @@ -182,6 +199,8 @@ def test_llm_update_weights(model_dir): llm_logits, ref_logits = run_generate(llm, hf_model, prompts, sampling_params) compare_logits(llm_logits, ref_logits) + del hf_model + @skip_pre_hopper @pytest.mark.parametrize( @@ -254,6 +273,8 @@ def filter_fn(name: str) -> bool: llm_logits, ref_logits = run_generate(llm, hf_model, prompts, sampling_params) compare_logits(llm_logits, ref_logits) + del hf_model + @skip_pre_hopper @pytest.mark.parametrize( @@ -314,3 +335,5 @@ def test_llm_update_weights_with_quant_config(model_dir, fp8_model_dir, kv_cache llm_logits, ref_logits = run_generate(llm, hf_model, prompts, sampling_params) compare_logits(llm_logits, ref_logits) + + del hf_model diff --git a/tests/unittest/_torch/sampler/test_beam_search.py b/tests/unittest/_torch/sampler/test_beam_search.py index 9b4d1365a19e..82f879f2debb 100644 --- a/tests/unittest/_torch/sampler/test_beam_search.py +++ b/tests/unittest/_torch/sampler/test_beam_search.py @@ -36,7 +36,9 @@ from tensorrt_llm._torch.pyexecutor.sampler import (BeamHistory, SampleStateTorch, TorchSampler) -from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import ( +from tensorrt_llm._torch.pyexecutor.sampler.logprobs import \ + convert_logprobs_tensor_to_list +from tensorrt_llm._torch.pyexecutor.sampler.sampler_strategy import ( BEAM_SEARCH_PAD_TOKEN, BeamSearchMetadata, beam_search_sampling_batch) from tensorrt_llm.bindings.executor import FinishReason from tensorrt_llm.executor import RequestError @@ -1094,7 +1096,7 @@ def _uut_provider( ) > 0, "Original log prob indices must not only contain zeros. Otherwise change the seed." # set the logprobs in the request: - token_logprobs = sampler._convert_logprobs_tensor_to_list( + token_logprobs = convert_logprobs_tensor_to_list( original_logprob_indices[:beam_width, :num_generated_tokens - 1], original_logprobs[:beam_width, :num_generated_tokens - 1], ) diff --git a/tests/unittest/_torch/sampler/test_logits_logprobs.py b/tests/unittest/_torch/sampler/test_logits_logprobs.py index ad07f70a3e5f..5e9238c9691c 100644 --- a/tests/unittest/_torch/sampler/test_logits_logprobs.py +++ b/tests/unittest/_torch/sampler/test_logits_logprobs.py @@ -7,7 +7,7 @@ from utils.util import force_ampere from tensorrt_llm import LLM, SamplingParams -from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import _StrategyImpls +from tensorrt_llm._torch.pyexecutor.sampler.sampler_strategy import _StrategyImpls from tensorrt_llm.executor.result import TokenLogprobs from tensorrt_llm.llmapi.llm_utils import KvCacheConfig @@ -778,6 +778,8 @@ def test_processed_logprobs_e2e(logprobs_k: int, simple_llm: LLM): group_logit_indices=None, top_k=torch.tensor([topk], dtype=torch.int32, device="cuda"), top_p=torch.tensor([topp], dtype=torch.float32, device="cuda"), + # None disables the min-p stage; no request here sets min_p. + min_p=None, temperature=torch.tensor([temperature], dtype=torch.float32, device="cuda"), generator=None, ) diff --git a/tests/unittest/_torch/sampler/test_penalties.py b/tests/unittest/_torch/sampler/test_penalties.py new file mode 100644 index 000000000000..6b9826425ddd --- /dev/null +++ b/tests/unittest/_torch/sampler/test_penalties.py @@ -0,0 +1,515 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import Fusions +from tensorrt_llm._torch.pyexecutor.sampler.penalties import PenaltyHandler + +apply_batched_occurrence_penalties = Fusions.apply_batched_occurrence_penalties +update_occurrence_workspace = Fusions.update_occurrence_workspace + + +@pytest.fixture(autouse=True) +def _dynamo_recompile_headroom(): + """Recompile headroom for the fullgraph=True penalty op. + + These cases sweep tensor shapes/dtypes, so the op legitimately builds one graph per shape + -- more than the default recompile_limit (8). A served model has fixed shapes; raising the + limit only here avoids tripping fullgraph's hard-fail without touching production. + """ + import torch._dynamo + + with torch._dynamo.config.patch(recompile_limit=128): + yield + + +def _col(values: list[float]) -> torch.Tensor: + return torch.tensor(values, dtype=torch.float32, device="cuda").view(-1, 1) + + +def _dense_penalty_reference( + logits: torch.Tensor, + counts: torch.Tensor, + presence: torch.Tensor | None, + rep: torch.Tensor, + pre: torch.Tensor, + freq: torch.Tensor, + temp: torch.Tensor, +) -> torch.Tensor: + """Dense post-temperature reference for ``apply_batched_occurrence_penalties``. + + Follows the TorchSampler order: repetition where the token is present anywhere + (``counts > 0`` or the prefix mask), then presence + frequency where counted + (``counts > 0``), followed by temperature division in the sampling strategy. + ``rep/pre/freq/temp`` are per-row ``[A, 1]`` tensors. + """ + penalized = logits.float() + present = counts > 0 + if presence is not None: + present = present | (presence > 0) + penalized = torch.where( + present, + torch.where(penalized < 0, penalized * rep, penalized / rep), + penalized, + ) + counts_f = counts.to(torch.float32) + sub = torch.where(counts > 0, pre + freq * counts_f, penalized.new_zeros(())) + return (penalized - sub) / temp + + +def _dense_presence_prefix(counts: torch.Tensor, presence: torch.Tensor) -> torch.Tensor: + prefix = torch.zeros( + presence.size(0), + presence.size(1), + dtype=torch.bool, + device=presence.device, + ) + prefix_slots, prefix_tokens = torch.nonzero(presence, as_tuple=True) + empty = torch.empty(0, dtype=torch.int64, device=presence.device) + update_occurrence_workspace( + counts, + prefix, + empty, + empty, + prefix_slots, + prefix_tokens, + ) + return prefix + + +@pytest.mark.parametrize( + "name,rep,pre,freq,temp,use_prefix", + [ + # repetition only, exercises the sign branch (>1, <1) at temp=1 + ("repetition", [1.3, 2.0, 0.7], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 1.0], False), + # presence only + ("presence", [1.0, 1.0], [0.5, 1.5], [0.0, 0.0], [1.0, 1.0], False), + # frequency only (counts > 1 -> proportional) + ("frequency", [1.0, 1.0], [0.0, 0.0], [0.4, 0.9], [1.0, 1.0], False), + # combined with temperature != 1 (exercises penalty-before-temperature order) + ( + "combined_temp", + [1.2, 0.8, 1.5], + [0.3, 0.0, 0.7], + [0.2, 0.5, 0.0], + [0.7, 1.3, 2.0], + False, + ), + # ignored-prompt-prefix mask affects repetition only, not presence/frequency + ("prefix", [1.4, 1.1, 0.9], [0.4, 0.6, 0.2], [0.3, 0.1, 0.5], [1.0, 0.8, 1.6], True), + ], +) +@pytest.mark.parametrize("num_steps", [1, 3], ids=["regular", "speculative"]) +def test_penalties_match_dense_logits_reference( + name: str, + rep: list[float], + pre: list[float], + freq: list[float], + temp: list[float], + use_prefix: bool, + num_steps: int, +) -> None: + # vocab=5000 is deliberately not a round power of two. + A, V = len(rep), 5000 + gen = torch.Generator(device="cuda").manual_seed(sum(name.encode()) + num_steps) + logits = torch.randn(A * num_steps, V, device="cuda", generator=gen) * 5.0 + counts = torch.randint(0, 4, (A, V), dtype=torch.int32, device="cuda", generator=gen) + presence = ( + torch.randint(0, 2, (A, V), dtype=torch.int32, device="cuda", generator=gen) + if use_prefix + else None + ) + presence_prefix = _dense_presence_prefix(counts, presence) if presence is not None else None + rep_t, pre_t, freq_t, temp_t = _col(rep), _col(pre), _col(freq), _col(temp) + slots = torch.arange(A, dtype=torch.int64, device="cuda") + row_slots = slots.repeat_interleave(num_steps) + + got = logits.clone() + apply_batched_occurrence_penalties( + got, + counts, + presence_prefix, + torch.ones(A, dtype=torch.bool, device="cuda"), + torch.zeros(A, dtype=torch.bool, device="cuda"), + torch.zeros(1, A, 1, dtype=torch.int32, device="cuda"), + slots, + torch.arange(0, A * num_steps, num_steps, dtype=torch.int32, device="cuda"), + torch.full((A,), num_steps, dtype=torch.int32, device="cuda"), + rep_t.squeeze(1), + pre_t.squeeze(1), + freq_t.squeeze(1), + ) + row_presence = presence[row_slots] if presence is not None else None + ref = _dense_penalty_reference( + logits, + counts[row_slots], + row_presence, + rep_t[row_slots], + pre_t[row_slots], + freq_t[row_slots], + temp_t[row_slots], + ) + # the kernel is pre-temperature-division; divide by temp to compare to the final value. + torch.testing.assert_close(got / temp_t[row_slots], ref, rtol=1e-4, atol=1e-4) + + +def test_penalties_indirect_indexing_bf16() -> None: + # Permuted request offsets and sequence slots penalize a subset of logits rows, with + # repeated slot mappings. Other rows must stay untouched. bfloat16 also covers the + # fp32-compute -> bf16-store cast path. + gen = torch.Generator(device="cuda").manual_seed(3) + num_slots, num_rows, vocab = 5, 10, 3000 + logits = (torch.randn(num_rows, vocab, device="cuda", generator=gen) * 3).to(torch.bfloat16) + orig = logits.clone() + counts = torch.randint( + 0, 4, (num_slots, vocab), dtype=torch.int32, device="cuda", generator=gen + ) + rep = torch.empty(num_slots, device="cuda").uniform_(0.7, 1.6, generator=gen) + pre = torch.empty(num_slots, device="cuda").uniform_(0.0, 0.6, generator=gen) + freq = torch.empty(num_slots, device="cuda").uniform_(0.0, 0.4, generator=gen) + temp = torch.empty(num_slots, device="cuda").uniform_(0.6, 1.4, generator=gen) + # Explicitly exercise permuted rows and repeated slot mappings. + active_rows = torch.tensor([8, 1, 6, 3, 9, 0, 5], dtype=torch.int64, device="cuda") + row_slots = torch.tensor([4, 1, 4, 0, 2, 1, 3], dtype=torch.int64, device="cuda") + + active = torch.ones(num_slots, dtype=torch.bool, device="cuda") + active[1] = False + apply_batched_occurrence_penalties( + logits, + counts, + None, + active, + torch.zeros(num_slots, dtype=torch.bool, device="cuda"), + torch.zeros(1, num_slots, 1, dtype=torch.int32, device="cuda"), + row_slots, + active_rows.to(torch.int32), + torch.ones(active_rows.numel(), dtype=torch.int32, device="cuda"), + rep, + pre, + freq, + ) + + active_row_mask = active[row_slots] + active_slots = row_slots[active_row_mask] + ref = _dense_penalty_reference( + orig[active_rows[active_row_mask]], + counts[active_slots], + None, + rep[active_slots].view(-1, 1), + pre[active_slots].view(-1, 1), + freq[active_slots].view(-1, 1), + temp[active_slots].view(-1, 1), + ) + expected = orig[active_rows].clone() + active_temperature = temp[active_slots].view(-1, 1) + # Recover the pre-temperature op output, then match its fp32-compute -> bf16-store + # boundary. This keeps the tolerance about the op's fp32 math, not bf16 rounding. + expected[active_row_mask] = (ref * active_temperature).to(torch.bfloat16) + torch.testing.assert_close(logits[active_rows], expected, rtol=5e-3, atol=5e-3) + torch.testing.assert_close( + logits[active_rows[~active_row_mask]], + orig[active_rows[~active_row_mask]], + rtol=0, + atol=0, + ) + untouched = torch.ones(num_rows, dtype=torch.bool, device="cuda") + untouched[active_rows] = False + torch.testing.assert_close(logits[untouched], orig[untouched], rtol=0, atol=0) + + +def test_prefix_marking_matches_dense_logits_reference() -> None: + vocab = 70 + counts = torch.zeros(1, vocab, dtype=torch.int32, device="cuda") + presence_prefix = torch.zeros(1, vocab, dtype=torch.bool, device="cuda") + + counted_tokens = torch.tensor([31, 31, 45], dtype=torch.int64, device="cuda") + prefix_tokens = torch.tensor([0, 31, 31, 32, 63, 69], dtype=torch.int64, device="cuda") + counted_slots = torch.zeros_like(counted_tokens) + prefix_slots = torch.zeros_like(prefix_tokens) + update_occurrence_workspace( + counts, + presence_prefix, + counted_slots, + counted_tokens, + prefix_slots, + prefix_tokens, + ) + + logits = torch.linspace(-7.0, 7.0, vocab, device="cuda").view(1, -1) + original = logits.clone() + apply_batched_occurrence_penalties( + logits, + counts, + presence_prefix, + torch.ones(1, dtype=torch.bool, device="cuda"), + torch.zeros(1, dtype=torch.bool, device="cuda"), + torch.zeros(1, 1, 1, dtype=torch.int32, device="cuda"), + torch.zeros(1, dtype=torch.int64, device="cuda"), + torch.zeros(1, dtype=torch.int32, device="cuda"), + torch.ones(1, dtype=torch.int32, device="cuda"), + torch.tensor([1.2], device="cuda"), + torch.tensor([0.4], device="cuda"), + torch.tensor([0.3], device="cuda"), + ) + + dense_prefix = torch.zeros_like(counts) + dense_prefix[0, torch.unique(prefix_tokens)] = 1 + expected = _dense_penalty_reference( + original, + counts, + dense_prefix, + torch.tensor([[1.2]], device="cuda"), + torch.tensor([[0.4]], device="cuda"), + torch.tensor([[0.3]], device="cuda"), + torch.ones(1, 1, device="cuda"), + ) + assert presence_prefix.shape == (1, vocab) + torch.testing.assert_close(logits, expected, rtol=1e-4, atol=1e-4) + + +def test_penalty_op_does_not_latch_pending_token() -> None: + """The penalty op must not write ``has_previous_token``. + + The op reads the flag to decide whether to fold the pending ``new_tokens`` token + into ``counts``; it must never write it (the host re-arms the flag after the op). + Here the flag is False with a stale token far up the vocab: nothing may be folded, + the flag must stay False, and the logits must be untouched. + """ + vocab = 3000 + stale_token = 2500 # a stale pending token far up the vocab + has_previous_token = torch.zeros(1, dtype=torch.bool, device="cuda") + new_tokens = torch.zeros(1, 1, 1, dtype=torch.int32, device="cuda") + new_tokens[0, 0, 0] = stale_token + counts = torch.zeros(1, vocab, dtype=torch.int32, device="cuda") + logits = torch.linspace(-4.0, 4.0, steps=vocab, device="cuda").view(1, vocab) + original = logits.clone() + + apply_batched_occurrence_penalties( + logits, + counts, + None, + torch.ones(1, dtype=torch.bool, device="cuda"), + has_previous_token, + new_tokens, + torch.zeros(1, dtype=torch.int64, device="cuda"), + torch.zeros(1, dtype=torch.int32, device="cuda"), + torch.ones(1, dtype=torch.int32, device="cuda"), + torch.tensor([1.5], device="cuda"), + torch.tensor([0.5], device="cuda"), + torch.tensor([0.4], device="cuda"), + ) + + # Deterministic: the penalty op must leave the latch untouched (host re-arms it). + assert not bool(has_previous_token.item()) + # With has_previous_token False and counts all zero, no penalty may be applied; the + # stale token in particular must not be folded (would perturb logits[2500]). + torch.testing.assert_close(logits, original, rtol=0, atol=0) + + +def _make_handler_request( + *, + slot: int, + tokens: list[int], + prompt_ignore_length: int = 0, + beam_width: int = 1, +) -> SimpleNamespace: + return SimpleNamespace( + sampling_config=SimpleNamespace( + repetition_penalty=[1.2], + presence_penalty=[0.4], + frequency_penalty=[0.3], + temperature=[1.0], + prompt_ignore_length=[prompt_ignore_length], + beam_width=beam_width, + beam_width_array=None, + ), + py_orig_prompt_len=len(tokens), + py_seq_slot=slot, + py_return_log_probs=False, + get_tokens=lambda _beam_idx: tokens, + py_is_draft=False, + ) + + +def _admit(handler: PenaltyHandler, request: SimpleNamespace, slot: int) -> None: + """Admit one request, mirroring TorchSampler.setup_sampler_step. + + ``prepare_for_new_request`` only accumulates on the host; the device buffers are + written by the batched ``update_for_new_requests`` flush at the end of the step. + """ + handler.prepare_for_new_request(request, slot=slot) + handler.update_for_new_requests( + new_seq_slots_cuda_long=torch.tensor([slot], dtype=torch.int64, device="cuda") + ) + + +def _apply_handler( + handler: PenaltyHandler, + request: SimpleNamespace, + logits: torch.Tensor, + num_steps: int, + new_tokens: torch.Tensor, +) -> None: + handler.apply( + logits, + [request], + new_tokens=new_tokens, + seq_slots=torch.tensor([request.py_seq_slot], dtype=torch.int64, device="cuda"), + request_offsets=torch.zeros(1, dtype=torch.int32), + request_num_steps=torch.tensor([num_steps], dtype=torch.int32), + ) + + +def test_handler_tracks_overlap_and_commits_speculative_tail() -> None: + vocab = 16 + slot = 2 + handler = PenaltyHandler( + max_num_sequences=3, + device="cuda", + ) + history = [3] + request = _make_handler_request(slot=slot, tokens=history) + _admit(handler, request, slot) + new_tokens = torch.zeros(3, 3, 1, dtype=torch.int32, device="cuda") + + # The first apply initializes the prompt and marks the first sampled token as + # pending. The request's host history need not be updated before the next apply. + _apply_handler(handler, request, torch.zeros(1, vocab, device="cuda"), 1, new_tokens) + new_tokens[0, slot, 0] = 5 + overlap_logits = torch.linspace(-2.0, 2.0, vocab, device="cuda").view(1, vocab) + overlap_original = overlap_logits.clone() + _apply_handler(handler, request, overlap_logits, 1, new_tokens) + overlap_counts = torch.bincount(torch.tensor([3, 5], device="cuda"), minlength=vocab).to( + torch.int32 + )[None] + overlap_expected = _dense_penalty_reference( + overlap_original, + overlap_counts, + None, + torch.full((1, 1), 1.2, device="cuda"), + torch.full((1, 1), 0.4, device="cuda"), + torch.full((1, 1), 0.3, device="cuda"), + torch.ones(1, 1, device="cuda"), + ) + torch.testing.assert_close(overlap_logits, overlap_expected, rtol=1e-4, atol=1e-4) + + # The next invocation is speculative. All rows use the same confirmed history; + # the current draft window remains tentative until acceptance is resolved. + history.extend([5, 6]) + new_tokens[0, slot, 0] = 6 + spec_logits = torch.linspace(-3.0, 3.0, steps=3 * vocab, device="cuda").view(3, vocab) + spec_original = spec_logits.clone() + _apply_handler(handler, request, spec_logits, 3, new_tokens) + spec_counts = torch.bincount(torch.tensor(history, device="cuda"), minlength=vocab).to( + torch.int32 + )[None] + spec_expected = _dense_penalty_reference( + spec_original, + spec_counts.expand(3, -1), + None, + torch.full((3, 1), 1.2, device="cuda"), + torch.full((3, 1), 0.4, device="cuda"), + torch.full((3, 1), 0.3, device="cuda"), + torch.ones(3, 1, device="cuda"), + ) + torch.testing.assert_close(spec_logits, spec_expected, rtol=1e-4, atol=1e-4) + + # Sampler-side acceptance commits the complete finalized sequence. Deliberately + # leave a different raw target token in the device buffer, as rejection sampling + # can do; clearing the pending flag must prevent it from entering the workspace. + history.extend([7, 8, 7]) + new_tokens[0, slot, 0] = 4 + handler.update_token_counts([(slot, [7, 8, 7])]) + logits = torch.linspace(-4.0, 4.0, steps=3 * vocab, device="cuda").view(3, vocab) + original = logits.clone() + _apply_handler(handler, request, logits, 3, new_tokens) + + expected_counts = torch.bincount(torch.tensor(history, device="cuda"), minlength=vocab).to( + torch.int32 + )[None] + expected = _dense_penalty_reference( + original, + expected_counts.expand(3, -1), + None, + torch.full((3, 1), 1.2, device="cuda"), + torch.full((3, 1), 0.4, device="cuda"), + torch.full((3, 1), 0.3, device="cuda"), + torch.ones(3, 1, device="cuda"), + ) + torch.testing.assert_close(logits, expected, rtol=1e-4, atol=1e-4) + + +def test_regular_handler_slot_reuse_does_not_leak_penalties() -> None: + vocab = 16 + handler = PenaltyHandler( + max_num_sequences=1, + device="cuda", + ) + new_tokens = torch.zeros(1, 1, 1, dtype=torch.int32, device="cuda") + + first = _make_handler_request(slot=0, tokens=[3, 3], prompt_ignore_length=1) + _admit(handler, first, 0) + _apply_handler(handler, first, torch.zeros(1, vocab, device="cuda"), 1, new_tokens) + + second = _make_handler_request(slot=0, tokens=[5]) + _admit(handler, second, 0) + logits = torch.linspace(-2.0, 2.0, steps=vocab, device="cuda").view(1, vocab) + original = logits.clone() + _apply_handler(handler, second, logits, 1, new_tokens) + + expected_counts = torch.zeros(1, vocab, dtype=torch.int32, device="cuda") + expected_counts[0, 5] = 1 + expected = _dense_penalty_reference( + original, + expected_counts, + None, + torch.full((1, 1), 1.2, device="cuda"), + torch.full((1, 1), 0.4, device="cuda"), + torch.full((1, 1), 0.3, device="cuda"), + torch.ones(1, 1, device="cuda"), + ) + torch.testing.assert_close(logits, expected, rtol=1e-4, atol=1e-4) + + +def test_handler_ignores_occurrence_penalties_with_beam_search() -> None: + """Beam-search requests never become penalty-active. + + ``PenaltyHandler.validate_request`` rejects this combination at admission, so the + handler should only ever see beam_width == 1 requests. It stays defensive anyway: + a beam-search request leaves its slot inactive, and ``apply`` is then a no-op. + """ + vocab = 16 + handler = PenaltyHandler(max_num_sequences=1, device="cuda") + new_tokens = torch.zeros(1, 1, 1, dtype=torch.int32, device="cuda") + + request = _make_handler_request(slot=0, tokens=[3, 3], beam_width=2) + _admit(handler, request, 0) + + logits = torch.linspace(-2.0, 2.0, steps=vocab, device="cuda").view(1, vocab) + original = logits.clone() + _apply_handler(handler, request, logits, 1, new_tokens) + + assert not bool(handler.store.active_cuda[0].item()) + torch.testing.assert_close(logits, original, rtol=0, atol=0) + + +def test_validate_request_rejects_penalties_with_beam_search() -> None: + """The admission-time check that keeps the combination above from arriving.""" + PenaltyHandler.validate_request(_make_handler_request(slot=0, tokens=[3], beam_width=1)) + with pytest.raises(ValueError, match="penalties with beam search"): + PenaltyHandler.validate_request(_make_handler_request(slot=0, tokens=[3], beam_width=2)) diff --git a/tests/unittest/_torch/sampler/test_penalties_e2e.py b/tests/unittest/_torch/sampler/test_penalties_e2e.py new file mode 100644 index 000000000000..332f8a675c14 --- /dev/null +++ b/tests/unittest/_torch/sampler/test_penalties_e2e.py @@ -0,0 +1,329 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from pathlib import Path + +import pytest +import torch +from utils.llm_data import llm_models_root + +from tensorrt_llm import LLM, SamplingParams +from tensorrt_llm.executor.result import CompletionOutput, GenerationResult +from tensorrt_llm.llmapi import CudaGraphConfig, NGramDecodingConfig +from tensorrt_llm.llmapi import KvCacheConfig as TRT_KvCacheConfig + + +@pytest.fixture(scope="module") +def model_path() -> Path: + return llm_models_root() / "llama-models-v2/TinyLlama-1.1B-Chat-v1.0" + + +@dataclass(frozen=True) +class _PenaltyE2ECase: + name: str + prompt: str + sampling_params: SamplingParams + + +def _penalty_sampling_params( + max_tokens: int = 1, + logprobs: int = 1, + **penalties: float | int, +) -> SamplingParams: + return SamplingParams( + max_tokens=max_tokens, + temperature=1.3, + seed=12345, + ignore_eos=True, + logprobs=logprobs, + logprobs_mode="processed", + return_generation_logits=True, + **penalties, + ) + + +def _make_penalty_e2e_cases() -> list[_PenaltyE2ECase]: + repeated_answer_prompt = "The capital of France is Paris. The capital of France is" + capital_prompt = "The capital of France is" + repeated_token_prompt = "cat cat cat cat The capital of France is" + + return [ + _PenaltyE2ECase( + "repetition_discourage", + repeated_answer_prompt, + _penalty_sampling_params(repetition_penalty=100.0), + ), + _PenaltyE2ECase( + "repetition_encourage", + capital_prompt, + _penalty_sampling_params(repetition_penalty=0.01), + ), + _PenaltyE2ECase( + "additive_reward", + repeated_token_prompt, + _penalty_sampling_params(presence_penalty=-10.0, frequency_penalty=-2.0), + ), + _PenaltyE2ECase( + "frequency_count", + repeated_token_prompt, + _penalty_sampling_params(frequency_penalty=5.0), + ), + _PenaltyE2ECase( + "additive_prompt_ignored", + capital_prompt, + _penalty_sampling_params( + presence_penalty=100.0, + frequency_penalty=100.0, + prompt_ignore_length=10_000, + ), + ), + _PenaltyE2ECase( + "combined_penalties", + repeated_answer_prompt, + _penalty_sampling_params( + max_tokens=6, + logprobs=5, + repetition_penalty=1.7, + presence_penalty=2.0, + frequency_penalty=0.75, + prompt_ignore_length=2, + ), + ), + ] + + +def _create_torch_llm( + model_dir: Path, + max_batch_size: int | None = None, + speculative_config: NGramDecodingConfig | None = None, + enable_iter_perf_stats: bool = False, +) -> LLM: + llm_kwargs: dict[str, object] = {} + if max_batch_size is not None: + llm_kwargs["max_batch_size"] = max_batch_size + if speculative_config is not None: + llm_kwargs["speculative_config"] = speculative_config + + return LLM( + model=str(model_dir), + tensor_parallel_size=1, + trust_remote_code=True, + enable_chunked_prefill=True, + cuda_graph_config=CudaGraphConfig(), + sampler_type="TorchSampler", + kv_cache_config=TRT_KvCacheConfig(enable_block_reuse=False), + max_num_tokens=128, + enable_iter_perf_stats=enable_iter_perf_stats, + **llm_kwargs, + ) + + +def _run_penalty_e2e_cases( + model_dir: Path, + cases: list[_PenaltyE2ECase], +) -> tuple[dict[str, GenerationResult], dict[str, tuple[int, ...]]]: + with _create_torch_llm(model_dir) as llm: + outputs = llm.generate( + [case.prompt for case in cases], + sampling_params=[case.sampling_params for case in cases], + use_tqdm=False, + ) + + results = dict(zip((case.name for case in cases), outputs, strict=True)) + prompt_token_ids = { + case.name: tuple(int(token_id) for token_id in output.prompt_token_ids) + for case, output in zip(cases, outputs, strict=True) + } + return results, prompt_token_ids + + +def _reference_penalized_logits( + raw_logits: torch.Tensor, + token_history: list[int], + prompt_length: int, + sampling_params: SamplingParams, +) -> torch.Tensor: + """Apply the documented penalties independently of TorchSampler.""" + vocab_size = raw_logits.numel() + history = torch.tensor(token_history, dtype=torch.int64) + valid_history = history[(history >= 0) & (history < vocab_size)] + adjusted_logits = raw_logits.float() + + repetition_penalty = sampling_params.repetition_penalty or 1.0 + if repetition_penalty != 1.0 and valid_history.numel() > 0: + repetition_mask = torch.bincount(valid_history, minlength=vocab_size).bool() + repetition_scaled_logits = torch.where( + adjusted_logits < 0, + adjusted_logits * repetition_penalty, + adjusted_logits / repetition_penalty, + ) + adjusted_logits = torch.where(repetition_mask, repetition_scaled_logits, adjusted_logits) + + prompt_ignore_length = sampling_params.prompt_ignore_length or 0 + occurrence_start = max(0, min(prompt_ignore_length, prompt_length)) + occurrence_history = history[occurrence_start:] + valid_occurrences = occurrence_history[ + (occurrence_history >= 0) & (occurrence_history < vocab_size) + ] + occurrence_counts = torch.bincount(valid_occurrences, minlength=vocab_size).float() + + presence_penalty = sampling_params.presence_penalty or 0.0 + frequency_penalty = sampling_params.frequency_penalty or 0.0 + adjusted_logits -= presence_penalty * (occurrence_counts > 0) + adjusted_logits -= frequency_penalty * occurrence_counts + + dtype_limit = torch.finfo(raw_logits.dtype).max + return adjusted_logits.clamp(min=-dtype_limit, max=dtype_limit).to(raw_logits.dtype) + + +def _reference_processed_logprobs( + raw_logits: torch.Tensor, + token_history: list[int], + prompt_length: int, + sampling_params: SamplingParams, +) -> torch.Tensor: + penalized_logits = _reference_penalized_logits( + raw_logits, + token_history, + prompt_length, + sampling_params, + ) + temperature = sampling_params.temperature + if temperature is not None and temperature != 0.0: + penalized_logits = penalized_logits / max(temperature, 1e-5) + processed_logits = penalized_logits.float() + sampling_probs = torch.softmax(processed_logits, dim=-1) + processed_logits = processed_logits.masked_fill(sampling_probs == 0, float("-inf")) + return torch.log_softmax(processed_logits, dim=-1) + + +def _assert_completion_penalty_logprobs( + case: _PenaltyE2ECase, + completion: CompletionOutput, + prompt_token_ids: tuple[int, ...], +) -> None: + assert completion.token_ids is not None, case.name + assert completion.generation_logits is not None, case.name + assert completion.logprobs is not None, case.name + + token_history = list(prompt_token_ids) + expected_cumulative_logprob = 0.0 + for step, (token_id, raw_logits, actual_logprobs) in enumerate( + zip(completion.token_ids, completion.generation_logits, completion.logprobs, strict=True) + ): + location = f"{case.name}/step_{step}" + assert token_id in actual_logprobs, location + expected_logprobs = _reference_processed_logprobs( + raw_logits, + token_history, + len(prompt_token_ids), + case.sampling_params, + ) + + for returned_token_id, actual in actual_logprobs.items(): + assert actual.logprob == pytest.approx( + float(expected_logprobs[returned_token_id]), + rel=2e-5, + abs=2e-4, + ), location + + num_logprobs = case.sampling_params.logprobs + if num_logprobs: + ranked_logprobs = { + actual.rank: actual.logprob + for actual in actual_logprobs.values() + if actual.rank is not None and actual.rank <= num_logprobs + } + assert set(ranked_logprobs) == set(range(1, num_logprobs + 1)), location + expected_top_logprobs = torch.topk(expected_logprobs, k=num_logprobs).values + for rank, expected in enumerate(expected_top_logprobs, start=1): + assert ranked_logprobs[rank] == pytest.approx( + float(expected), rel=2e-5, abs=2e-4 + ), location + + expected_cumulative_logprob += float(expected_logprobs[token_id]) + token_history.append(token_id) + + assert completion.cumulative_logprob == pytest.approx( + expected_cumulative_logprob, rel=2e-5, abs=2e-4 + ), case.name + + +@pytest.mark.high_cuda_memory +def test_torch_sampler_penalty_logits_e2e(model_path: Path) -> None: + """Validate TorchSampler's processed logits against the penalty formulas.""" + cases = _make_penalty_e2e_cases() + results, prompt_token_ids = _run_penalty_e2e_cases(model_path, cases) + + for case in cases: + for completion in results[case.name].outputs: + _assert_completion_penalty_logprobs( + case, + completion, + prompt_token_ids[case.name], + ) + + +@pytest.mark.high_cuda_memory +def test_torch_sampler_speculative_penalty_e2e(model_path: Path) -> None: + """Validate the speculative (NGram) path's penalized logprobs against the formula. + + A positive temperature keeps the processed logprobs a real distribution the penalty + formula can be checked against (greedy would collapse them to one-hot). At this + temperature the penalties push the target away from NGram's repetition drafts, so drafts + are proposed but not accepted -- every emitted token is target-sampled and its processed + logprobs match the formula. Accepted draft tokens report a one-hot logprob and cannot be + formula-checked; the accepted-token confirmed-history commit path is covered at the logit + level by ``test_handler_tracks_overlap_and_commits_speculative_tail``. + """ + case = _PenaltyE2ECase( + "ngram_speculative_penalties", + "red blue red blue red blue red blue red blue", + _penalty_sampling_params( + max_tokens=8, + logprobs=5, + repetition_penalty=1.5, + presence_penalty=1.0, + frequency_penalty=1.0, + prompt_ignore_length=2, + ), + ) + speculative_config = NGramDecodingConfig( + max_draft_len=3, + max_matching_ngram_size=2, + is_keep_all=True, + is_use_oldest=True, + is_public_pool=False, + ) + with _create_torch_llm( + model_path, + max_batch_size=1, + speculative_config=speculative_config, + enable_iter_perf_stats=True, + ) as llm: + speculative_outputs = llm.generate( + [case.prompt], sampling_params=[case.sampling_params], use_tqdm=False + ) + stats = llm.get_stats(timeout=5) + + assert any(stat.get("specDecodingStats", {}).get("numDraftTokens", 0) > 0 for stat in stats), ( + "NGram must produce draft tokens in this test" + ) + + speculative_prompt_token_ids = tuple( + int(token_id) for token_id in speculative_outputs[0].prompt_token_ids + ) + for completion in speculative_outputs[0].outputs: + _assert_completion_penalty_logprobs(case, completion, speculative_prompt_token_ids) diff --git a/tests/unittest/_torch/sampler/test_token_ban.py b/tests/unittest/_torch/sampler/test_token_ban.py new file mode 100644 index 000000000000..c0c0905d7c2a --- /dev/null +++ b/tests/unittest/_torch/sampler/test_token_ban.py @@ -0,0 +1,569 @@ +# Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Unit tests for the token-ban handlers (bad words, no-repeat ngram, +min-length EOS suppression) in +tensorrt_llm/_torch/pyexecutor/sampler/token_ban.py.""" + +from typing import cast + +import pytest +import torch + +from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest +from tensorrt_llm._torch.pyexecutor.sampler.token_ban import ( + OverlappedTokenBanHandler, + SynchronousTokenBanHandler, +) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +class TestApplyBadWords: + """Unit tests for the bad-words path of TokenBanHandler. + + Single-token words are banned unconditionally; multi-token words ban their + final token only when the token suffix (prompt + generated) matches the + word prefix. + """ + + VOCAB = 16 + + class MockLlmRequest: + """Minimal stub exposing the attributes the bad-words path reads.""" + + def __init__(self, tokens, *, bad_words=None, prompt_len=0, seq_slot=None): + # get_tokens(beam) returns the full token sequence (prompt + + # generated); py_orig_prompt_len marks where generation starts. + self.py_orig_prompt_len = prompt_len + self._tokens = list(tokens) + self.py_bad_words = bad_words + self.py_seq_slot = seq_slot + + def get_tokens(self, beam_idx): + return self._tokens + + def _run(self, requests, num_steps, num_beams): + total_rows = sum(s * b for s, b in zip(num_steps, num_beams)) + logits = torch.zeros(total_rows, self.VOCAB, device="cuda") + ngram_sizes: list[int | None] = [None] * len(requests) + handler = SynchronousTokenBanHandler() + bans = handler.generate_ban_list( + cast(list[LlmRequest], requests), num_steps, num_beams, ngram_sizes + ) + handler.apply_ban_list(logits, bans) + return logits + + @staticmethod + def _banned_cols(logits_row): + return set(torch.nonzero(torch.isinf(logits_row)).flatten().tolist()) + + def test_single_token_unconditional(self): + req = self.MockLlmRequest(tokens=[3, 4], bad_words=[[7]]) + logits = self._run([req], num_steps=[1], num_beams=[1]) + assert self._banned_cols(logits[0]) == {7} + + def test_multi_token_prefix_hit(self): + # tokens end with [9]; word [9, 2] -> ban token 2. + req = self.MockLlmRequest(tokens=[1, 9], bad_words=[[9, 2]]) + logits = self._run([req], num_steps=[1], num_beams=[1]) + assert self._banned_cols(logits[0]) == {2} + + def test_multi_token_prefix_miss(self): + # tokens end with [3]; word [9, 2] prefix [9] does not match. + req = self.MockLlmRequest(tokens=[1, 3], bad_words=[[9, 2]]) + logits = self._run([req], num_steps=[1], num_beams=[1]) + assert self._banned_cols(logits[0]) == set() + + def test_multi_token_prefix_in_prompt(self): + # The prefix [9] lies in the prompt (nothing generated yet); the word + # [9, 2] must still ban token 2. + req = self.MockLlmRequest(tokens=[1, 9], bad_words=[[9, 2]], prompt_len=2) + logits = self._run([req], num_steps=[1], num_beams=[1]) + assert self._banned_cols(logits[0]) == {2} + + # --- Overlap-scheduler (stale-host) path ------------------------------- + # + # With the overlap scheduler the host token list lags the device state by + # one token; the newest token is read from new_tokens_cuda[0, seq_slot, 0] + # on the GPU. These tests drive the overlap handler with stale_by_one set. + + NUM_SLOTS = 4 + + def _new_tokens_cuda(self, slot_tokens): + buf = torch.full((1, self.NUM_SLOTS, 1), -1, dtype=torch.int32, device="cuda") + for slot, tok in slot_tokens.items(): + buf[0, slot, 0] = tok + return buf + + def _run_stale(self, requests, stale_by_one, slot_tokens): + total_rows = len(requests) + logits = torch.zeros(total_rows, self.VOCAB, device="cuda") + ngram_sizes: list[int | None] = [None] * len(requests) + num_steps = [1] * len(requests) + num_beams = [1] * len(requests) + handler = OverlappedTokenBanHandler() + bans = handler.generate_ban_list( + cast(list[LlmRequest], requests), + num_steps, + num_beams, + ngram_sizes, + stale_by_one=stale_by_one, + ) + handler.apply_ban_list(logits, bans, new_tokens_cuda=self._new_tokens_cuda(slot_tokens)) + return logits + + def test_stale_two_token_device_hit(self): + # Host context [1]; device holds the pending token 9; word [9, 2] + # completes its prefix on the device side -> ban token 2. + req = self.MockLlmRequest(tokens=[1], bad_words=[[9, 2]], seq_slot=0) + logits = self._run_stale([req], [True], {0: 9}) + assert self._banned_cols(logits[0]) == {2} + + def test_stale_two_token_device_miss(self): + # Device token is 3, not the required prefix 9 -> nothing banned. + req = self.MockLlmRequest(tokens=[1], bad_words=[[9, 2]], seq_slot=0) + logits = self._run_stale([req], [True], {0: 3}) + assert self._banned_cols(logits[0]) == set() + + def test_stale_three_token_host_and_device_hit(self): + # Word [5, 9, 2]: host suffix must be [5], device token must be 9. + req = self.MockLlmRequest(tokens=[1, 5], bad_words=[[5, 9, 2]], seq_slot=1) + logits = self._run_stale([req], [True], {1: 9}) + assert self._banned_cols(logits[0]) == {2} + + def test_stale_three_token_host_miss(self): + # Host suffix [3] does not match the word prefix [5]; the device token + # matching is irrelevant -> nothing banned. + req = self.MockLlmRequest(tokens=[1, 3], bad_words=[[5, 9, 2]], seq_slot=1) + logits = self._run_stale([req], [True], {1: 9}) + assert self._banned_cols(logits[0]) == set() + + def test_stale_single_token_unconditional(self): + # Single-token words are banned regardless of the device token. + req = self.MockLlmRequest(tokens=[1], bad_words=[[7]], seq_slot=0) + logits = self._run_stale([req], [True], {0: 3}) + assert self._banned_cols(logits[0]) == {7} + + def test_stale_and_fresh_requests_mixed(self): + # Request 0 (stale) matches via the device token; request 1 (fresh) + # matches via the host context, as on the regular path. + stale_req = self.MockLlmRequest(tokens=[1], bad_words=[[9, 2]], seq_slot=0) + fresh_req = self.MockLlmRequest(tokens=[1, 9], bad_words=[[9, 4]], seq_slot=1) + logits = self._run_stale([stale_req, fresh_req], [True, False], {0: 9}) + assert self._banned_cols(logits[0]) == {2} + assert self._banned_cols(logits[1]) == {4} + + def test_stale_conditional_and_unconditional_same_cell(self): + # An unconditional single-token ban and a device-conditional ban on the + # same logit must combine to -inf (no NaN from -inf + -inf). + req = self.MockLlmRequest(tokens=[1], bad_words=[[2], [9, 2]], seq_slot=0) + logits = self._run_stale([req], [True], {0: 9}) + assert self._banned_cols(logits[0]) == {2} + assert not torch.isnan(logits).any() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +class TestApplyNoRepeatNgram: + """Unit tests for the no-repeat-ngram path of TokenBanHandler. + + With ngram size n, the last n-1 tokens of the sequence form the current + prefix; the final token of every existing n-gram whose first n-1 tokens + match that prefix is banned, so no n-gram is generated twice. + """ + + VOCAB = 16 + + class MockLlmRequest: + """Minimal stub exposing the attributes the no-repeat-ngram path reads.""" + + def __init__(self, tokens, *, seq_slot=None): + self._tokens = list(tokens) + self.py_seq_slot = seq_slot + + def get_tokens(self, beam_idx): + # Copy, like the real binding: the sampler's mirror must not alias. + return list(self._tokens) + + def get_num_tokens(self, beam_idx): + return len(self._tokens) + + def get_last_tokens(self, beam_idx): + return self._tokens[-1] + + class MultiBeamMockLlmRequest: + """Stub with a distinct token history per beam for multi-beam tests.""" + + def __init__(self, beam_tokens, *, seq_slot=None): + self._beam_tokens = [list(t) for t in beam_tokens] + self.py_seq_slot = seq_slot + + def get_tokens(self, beam_idx): + return list(self._beam_tokens[beam_idx]) + + def get_num_tokens(self, beam_idx): + return len(self._beam_tokens[beam_idx]) + + def get_last_tokens(self, beam_idx): + return self._beam_tokens[beam_idx][-1] + + @staticmethod + def _distinct_logits(total_rows, vocab): + # Distinct non-zero values per cell, so a comparison against the input + # can detect any logit the sampler touched but should not have. + return (torch.arange(total_rows * vocab, device="cuda", dtype=torch.float) + 1.0).reshape( + total_rows, vocab + ) + + def _run(self, requests, ngram_sizes, num_steps=None, num_beams=None, handler=None): + num_steps = num_steps or [1] * len(requests) + num_beams = num_beams or [1] * len(requests) + total_rows = sum(s * b for s, b in zip(num_steps, num_beams, strict=True)) + logits = self._distinct_logits(total_rows, self.VOCAB) + before = logits.clone() + handler = handler or SynchronousTokenBanHandler() + bans = handler.generate_ban_list( + cast(list[LlmRequest], requests), num_steps, num_beams, ngram_sizes + ) + handler.apply_ban_list(logits, bans) + self._assert_only_banned_changed(logits, before) + return logits + + @staticmethod + def _banned_cols(logits_row): + return set(torch.nonzero(torch.isinf(logits_row)).flatten().tolist()) + + @classmethod + def _assert_only_banned_changed(cls, logits, before): + # Every non-banned cell must still hold its original (non-zero) value; + # banned cells become -inf. + banned = torch.isinf(logits) + assert torch.equal(logits[~banned], before[~banned]), ( + "a logit outside the banned set was modified" + ) + + def test_bigram_repeat_banned(self): + # Sequence [1, 2, 3, 1, 2] with n=2: prefix [2]; existing bigram + # (2, 3) -> ban 3. The trailing (1, 2) is the prefix itself. + req = self.MockLlmRequest(tokens=[1, 2, 3, 1, 2]) + logits = self._run([req], [2]) + assert self._banned_cols(logits[0]) == {3} + + def test_trigram_repeat_banned(self): + # Sequence [1, 2, 3, 4, 1, 2] with n=3: prefix [1, 2]; existing + # trigram (1, 2, 3) -> ban 3. + req = self.MockLlmRequest(tokens=[1, 2, 3, 4, 1, 2]) + logits = self._run([req], [3]) + assert self._banned_cols(logits[0]) == {3} + + def test_multiple_matches_all_banned(self): + # Prefix [2] occurs twice with different continuations -> ban both. + req = self.MockLlmRequest(tokens=[2, 3, 2, 5, 2]) + logits = self._run([req], [2]) + assert self._banned_cols(logits[0]) == {3, 5} + + def test_no_match_nothing_banned(self): + req = self.MockLlmRequest(tokens=[1, 2, 3, 4]) + logits = self._run([req], [3]) + assert self._banned_cols(logits[0]) == set() + + def test_sequence_shorter_than_ngram(self): + req = self.MockLlmRequest(tokens=[1, 2]) + logits = self._run([req], [3]) + assert self._banned_cols(logits[0]) == set() + + def test_unigram_bans_all_seen_tokens(self): + req = self.MockLlmRequest(tokens=[4, 7, 4]) + logits = self._run([req], [1]) + assert self._banned_cols(logits[0]) == {4, 7} + + @pytest.mark.parametrize("disabled_size", [None, 0]) + def test_disabled_request_untouched(self, disabled_size): + # Both None and 0 disable the restriction for that request. + active = self.MockLlmRequest(tokens=[1, 2, 3, 1, 2]) + disabled = self.MockLlmRequest(tokens=[1, 2, 3, 1, 2]) + logits = self._run([active, disabled], [2, disabled_size]) + assert self._banned_cols(logits[0]) == {3} + assert self._banned_cols(logits[1]) == set() + + def test_incremental_cache_updates_with_growing_context(self): + # The per-request index must extend as the token history grows. + # Reuse one sampler so the incremental cache carries across calls. + handler = SynchronousTokenBanHandler() + req = self.MockLlmRequest(tokens=[1, 2, 3]) + logits = self._run([req], [2], handler=handler) + assert self._banned_cols(logits[0]) == set() + req._tokens.extend([1, 2]) # now [1, 2, 3, 1, 2]: bigram (2, 3) -> ban 3 + logits = self._run([req], [2], handler=handler) + assert self._banned_cols(logits[0]) == {3} + + def test_cache_rebuilt_on_history_rollback(self): + # A shrunken history (speculative rollback) must invalidate the cache. + # Reuse one sampler so the stale cache would be hit if not rebuilt. + handler = SynchronousTokenBanHandler() + req = self.MockLlmRequest(tokens=[1, 2, 3, 1, 2]) + logits = self._run([req], [2], handler=handler) + assert self._banned_cols(logits[0]) == {3} + req._tokens[:] = [4, 5] + logits = self._run([req], [2], handler=handler) + assert self._banned_cols(logits[0]) == set() + + def test_multi_step_rows_all_banned(self): + # Speculative steps share the (host-approximated) banned set. + req = self.MockLlmRequest(tokens=[1, 2, 3, 1, 2]) + logits = self._run([req], [2], num_steps=[2]) + assert self._banned_cols(logits[0]) == {3} + assert self._banned_cols(logits[1]) == {3} + + def test_multi_beam_uses_per_beam_history(self): + # Beam histories diverge, so each beam's row is banned from its own + # n-grams; the shared per-request cache must not be used here. + req = self.MultiBeamMockLlmRequest( + beam_tokens=[ + [1, 2, 3, 1, 2], # beam 0: bigram (2, 3) -> ban 3 + [4, 5, 6, 4, 5], # beam 1: bigram (5, 6) -> ban 6 + ] + ) + # A beam-major / step-minor row layout with num_beams=2, num_steps=1 + # yields one row per beam. + logits = self._run([req], [2], num_beams=[2]) + assert self._banned_cols(logits[0]) == {3} + assert self._banned_cols(logits[1]) == {6} + + # --- Overlap-scheduler (stale-host) path ------------------------------- + # + # The host token list lags the device state by one token, read from + # new_tokens_cuda[0, seq_slot, 0] on the GPU. + + NUM_SLOTS = 4 + + def _new_tokens_cuda(self, slot_tokens): + buf = torch.full((1, self.NUM_SLOTS, 1), -1, dtype=torch.int32, device="cuda") + for slot, tok in slot_tokens.items(): + buf[0, slot, 0] = tok + return buf + + def _run_stale(self, requests, ngram_sizes, stale_by_one, slot_tokens, handler=None): + total_rows = len(requests) + logits = self._distinct_logits(total_rows, self.VOCAB) + before = logits.clone() + handler = handler or OverlappedTokenBanHandler() + num_steps = [1] * len(requests) + num_beams = [1] * len(requests) + bans = handler.generate_ban_list( + cast(list[LlmRequest], requests), + num_steps, + num_beams, + ngram_sizes, + stale_by_one=stale_by_one, + ) + handler.apply_ban_list(logits, bans, new_tokens_cuda=self._new_tokens_cuda(slot_tokens)) + self._assert_only_banned_changed(logits, before) + return logits + + def test_stale_bigram_device_hit(self): + # True sequence [1, 2, 3, 1] + device 2 == [1, 2, 3, 1, 2], n=2: + # window (2, 3) matches the prefix [d=2] -> ban 3. + req = self.MockLlmRequest(tokens=[1, 2, 3, 1], seq_slot=0) + logits = self._run_stale([req], [2], [True], {0: 2}) + assert self._banned_cols(logits[0]) == {3} + + def test_stale_bigram_device_miss(self): + # Device token 5 never occurred before -> nothing banned. + req = self.MockLlmRequest(tokens=[1, 2, 3, 1], seq_slot=0) + logits = self._run_stale([req], [2], [True], {0: 5}) + assert self._banned_cols(logits[0]) == set() + + def test_stale_trigram_host_and_device_hit(self): + # True sequence [1, 2, 3, 4, 1] + device 2, n=3: prefix [1, 2]; + # window (1, 2, 3) -> ban 3. + req = self.MockLlmRequest(tokens=[1, 2, 3, 4, 1], seq_slot=1) + logits = self._run_stale([req], [3], [True], {1: 2}) + assert self._banned_cols(logits[0]) == {3} + + def test_stale_trigram_host_miss(self): + # Prefix head is [4]; no earlier window starts with 4 followed by the + # device token -> nothing banned. + req = self.MockLlmRequest(tokens=[1, 2, 3, 4], seq_slot=1) + logits = self._run_stale([req], [3], [True], {1: 5}) + assert self._banned_cols(logits[0]) == set() + + def test_stale_window_ending_at_device_token(self): + # True sequence [7, 7] + device 7, n=2: window (7, 7) ends at the + # device token; the match forces d == context[-1] -> ban 7. + req = self.MockLlmRequest(tokens=[7, 7], seq_slot=0) + logits = self._run_stale([req], [2], [True], {0: 7}) + assert self._banned_cols(logits[0]) == {7} + + def test_stale_unigram_bans_host_and_device_tokens(self): + req = self.MockLlmRequest(tokens=[4, 7], seq_slot=2) + logits = self._run_stale([req], [1], [True], {2: 9}) + assert self._banned_cols(logits[0]) == {4, 7, 9} + + def test_stale_and_fresh_requests_mixed(self): + stale_req = self.MockLlmRequest(tokens=[1, 2, 3, 1], seq_slot=0) + fresh_req = self.MockLlmRequest(tokens=[1, 2, 3, 1, 2], seq_slot=1) + logits = self._run_stale([stale_req, fresh_req], [2, 2], [True, False], {0: 2}) + assert self._banned_cols(logits[0]) == {3} + assert self._banned_cols(logits[1]) == {3} + + def test_stale_no_nan_on_duplicate_bans(self): + # Duplicate bans on one cell must combine to -inf, not NaN: bigrams + # starting with 2 continue with 3 (twice) and with 2 (window at d). + req = self.MockLlmRequest(tokens=[2, 3, 2, 3, 2], seq_slot=0) + logits = self._run_stale([req], [2], [True], {0: 2}) + assert self._banned_cols(logits[0]) == {2, 3} + assert not torch.isnan(logits).any() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +class TestAddMinLengthBans: + """Unit tests for the min-length EOS-suppression path of TokenBanHandler. + + EOS (the request's original end_id) is banned on every step whose generated + length is still below py_min_length; it is an unconditional ban with no + stale-host *suffix* variant, but the length it compares against must include + tokens still pending write-back under the overlap scheduler. + """ + + VOCAB = 16 + END_ID = 5 + + class MockLlmRequest: + """Minimal stub exposing the attributes the min-length path reads.""" + + def __init__(self, num_tokens, *, min_length=None, prompt_len=0, end_id=5): + self._num_tokens = num_tokens # total tokens (prompt + generated) + self.py_min_length = [min_length] if min_length is not None else None + self.py_orig_prompt_len = prompt_len + self.py_original_end_id = end_id + self.py_end_id = end_id + + def get_num_tokens(self, beam_idx): + return self._num_tokens + + @staticmethod + def _distinct_logits(total_rows, vocab): + return (torch.arange(total_rows * vocab, device="cuda", dtype=torch.float) + 1.0).reshape( + total_rows, vocab + ) + + @staticmethod + def _banned_cols(logits_row): + return set(torch.nonzero(torch.isinf(logits_row)).flatten().tolist()) + + def _run(self, requests, num_steps=None, num_beams=None): + num_steps = num_steps or [1] * len(requests) + num_beams = num_beams or [1] * len(requests) + total_rows = sum(s * b for s, b in zip(num_steps, num_beams, strict=True)) + logits = self._distinct_logits(total_rows, self.VOCAB) + before = logits.clone() + ngram_sizes: list[int | None] = [None] * len(requests) + handler = SynchronousTokenBanHandler() + bans = handler.generate_ban_list( + cast(list[LlmRequest], requests), num_steps, num_beams, ngram_sizes + ) + handler.apply_ban_list(logits, bans) + # Only banned cells (-inf) changed; everything else keeps its value. + banned = torch.isinf(logits) + assert torch.equal(logits[~banned], before[~banned]) + return logits + + def test_below_min_length_bans_eos(self): + # 2 generated tokens (num_tokens 2, prompt 0), min_length 5 -> ban EOS. + req = self.MockLlmRequest(num_tokens=2, min_length=5) + logits = self._run([req]) + assert self._banned_cols(logits[0]) == {self.END_ID} + + def test_at_min_length_no_ban(self): + # 5 generated tokens, min_length 5 -> already satisfied, nothing banned. + req = self.MockLlmRequest(num_tokens=5, min_length=5) + logits = self._run([req]) + assert self._banned_cols(logits[0]) == set() + + def test_generated_length_excludes_prompt(self): + # num_tokens 7 with prompt_len 4 -> only 3 generated < min_length 5 -> ban. + req = self.MockLlmRequest(num_tokens=7, min_length=5, prompt_len=4) + logits = self._run([req]) + assert self._banned_cols(logits[0]) == {self.END_ID} + + def test_no_min_length_untouched(self): + active = self.MockLlmRequest(num_tokens=1, min_length=5) + disabled = self.MockLlmRequest(num_tokens=1, min_length=None) + logits = self._run([active, disabled]) + assert self._banned_cols(logits[0]) == {self.END_ID} + assert self._banned_cols(logits[1]) == set() + + def test_invalid_end_id_skipped(self): + # end_id <= -1 (e.g. ignore_eos) -> nothing to suppress. + req = self.MockLlmRequest(num_tokens=1, min_length=5, end_id=-1) + logits = self._run([req]) + assert self._banned_cols(logits[0]) == set() + + def test_multi_step_stops_at_min_length(self): + # 3 generated, min_length 5, 3 speculative steps: gen+step < 5 for + # steps 0,1 (3,4) but not step 2 (5) -> ban only rows 0 and 1. + req = self.MockLlmRequest(num_tokens=3, min_length=5) + logits = self._run([req], num_steps=[3]) + assert self._banned_cols(logits[0]) == {self.END_ID} + assert self._banned_cols(logits[1]) == {self.END_ID} + assert self._banned_cols(logits[2]) == set() + + # -- overlap scheduler: get_num_tokens() lags by the pending-step count --- + + def _run_overlapped(self, requests, pending_steps, num_steps=None, num_beams=None): + num_steps = num_steps or [1] * len(requests) + num_beams = num_beams or [1] * len(requests) + total_rows = sum(s * b for s, b in zip(num_steps, num_beams, strict=True)) + logits = self._distinct_logits(total_rows, self.VOCAB) + ngram_sizes: list[int | None] = [None] * len(requests) + handler = OverlappedTokenBanHandler() + bans = handler.generate_ban_list( + cast(list[LlmRequest], requests), + num_steps, + num_beams, + ngram_sizes, + pending_steps=pending_steps, + ) + handler.apply_ban_list(logits, bans) + return logits + + def test_pending_step_reaches_min_length(self): + # Host sees 4 generated tokens but one more was already sampled and is + # awaiting write-back, so the true length is 5 == min_length. Without + # the correction EOS would stay banned and overshoot min_length. + req = self.MockLlmRequest(num_tokens=4, min_length=5) + logits = self._run_overlapped([req], pending_steps=[1]) + assert self._banned_cols(logits[0]) == set() + + def test_pending_step_still_below_min_length(self): + # Host sees 3, one pending -> true length 4 < 5, so EOS stays banned. + req = self.MockLlmRequest(num_tokens=3, min_length=5) + logits = self._run_overlapped([req], pending_steps=[1]) + assert self._banned_cols(logits[0]) == {self.END_ID} + + def test_no_pending_steps_matches_synchronous(self): + # pending_steps=None -> no correction; same verdict as the sync handler. + req = self.MockLlmRequest(num_tokens=4, min_length=5) + logits = self._run_overlapped([req], pending_steps=None) + assert self._banned_cols(logits[0]) == {self.END_ID} + + def test_pending_steps_are_per_request(self): + # Only the second request has a pending token; the correction must not + # leak across requests. + lagging = self.MockLlmRequest(num_tokens=4, min_length=5) + fresh = self.MockLlmRequest(num_tokens=4, min_length=5) + logits = self._run_overlapped([lagging, fresh], pending_steps=[0, 1]) + assert self._banned_cols(logits[0]) == {self.END_ID} + assert self._banned_cols(logits[1]) == set() diff --git a/tests/unittest/_torch/sampler/test_torch_sampler.py b/tests/unittest/_torch/sampler/test_torch_sampler.py index c16da7a2c70e..3fcea1270c52 100644 --- a/tests/unittest/_torch/sampler/test_torch_sampler.py +++ b/tests/unittest/_torch/sampler/test_torch_sampler.py @@ -47,18 +47,24 @@ _request_get_sampling_params, _request_strategy, ) -from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import ( +from tensorrt_llm._torch.pyexecutor.sampler.finish_reasons import FinishReasonsHandler +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import min_p_renorm_probs +from tensorrt_llm._torch.pyexecutor.sampler.sampler_common import UtilsSamplingParams +from tensorrt_llm._torch.pyexecutor.sampler.sampler_strategy import ( GREEDY, BeamSearch, FlashInferGroupedStrategySampler, Greedy, + MinP, Strategy, StrategyMetadata, TemperatureOnly, TopK, TopKTopP, TopP, - UtilsSamplingParams, + TopPDecayMetadata, + resolve_sampling_strategy, + sample, ) from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm.bindings import SamplingConfig @@ -349,6 +355,88 @@ def test_top_k_top_p_with_temperature(self): assert strat[2] == pytest.approx(0.7) assert strat[3] == pytest.approx(0.9) + # --- min_p --- + # A min_p strategy is ("min_p", top_k, top_p, min_p, temperature). When + # unset, top_k carries the disabled sentinel 0 ("keep all"; sanitized to + # vocab_size downstream) and top_p carries 1.0, so min_p composes with any + # subset of temperature/top_k/top_p. + + @pytest.mark.parametrize( + "trivial_temperature, trivial_top_p, trivial_top_k", + [ + pytest.param(temperature, top_p, top_k) + for (temperature, top_k, top_p) in product( + TEMPERATURE_NEUTRAL_VALS, TOP_K_NEUTRAL_VALS, TOP_P_NEUTRAL_VALS + ) + ], + ) + def test_min_p_only( + self, + trivial_temperature: Optional[float], + trivial_top_p: Optional[float], + trivial_top_k: Optional[int], + ): + params = SamplingParams( + min_p=0.1, temperature=trivial_temperature, top_p=trivial_top_p, top_k=trivial_top_k + ) + self._check_params(params) + request = self._build_mock_llm_request(params) + strat = _request_strategy(request, vocab_size=self.VOCAB_SIZE) + assert len(strat) == 5 + assert strat[0] == "min_p" + assert strat[1] == 0 # top_k disabled sentinel (0 == "keep all") + assert strat[2] == pytest.approx(1.0) # top_p disabled sentinel + assert strat[3] == pytest.approx(0.1) + assert strat[4] == pytest.approx(1.0) # temperature default + + def test_min_p_with_temperature(self): + params = SamplingParams(min_p=0.1, temperature=0.8) + request = self._build_mock_llm_request(params) + strat = _request_strategy(request, vocab_size=self.VOCAB_SIZE) + assert strat[0] == "min_p" + assert strat[3] == pytest.approx(0.1) + assert strat[4] == pytest.approx(0.8) + + def test_min_p_with_top_k_top_p(self): + params = SamplingParams(min_p=0.1, top_k=42, top_p=0.7, temperature=0.8) + request = self._build_mock_llm_request(params) + strat = _request_strategy(request, vocab_size=self.VOCAB_SIZE) + assert len(strat) == 5 + assert strat[0] == "min_p" + assert strat[1] == 42 + assert strat[2] == pytest.approx(0.7) + assert strat[3] == pytest.approx(0.1) + assert strat[4] == pytest.approx(0.8) + + def test_min_p_0_not_selected(self): + # min_p == 0 disables min_p; a plain temperature strategy is chosen. + params = SamplingParams(min_p=0.0, temperature=0.7) + request = self._build_mock_llm_request(params) + strat = _request_strategy(request, vocab_size=self.VOCAB_SIZE) + assert strat[0] == "temperature" + + def test_min_p_1_is_greedy(self): + # min_p == 1 keeps only the row max, i.e. an explicit greedy control + # (like top_p == 0), so it must not reach the min_p sampling path. + params = SamplingParams(min_p=1.0, temperature=0.7) + self._check_params(params) + request = self._build_mock_llm_request(params) + assert _request_strategy(request, vocab_size=self.VOCAB_SIZE) is GREEDY + + @pytest.mark.parametrize( + "greedy_kwargs", + [ + pytest.param({"top_k": 1}, id="top_k_1"), + pytest.param({"temperature": 0}, id="temperature_0"), + ], + ) + def test_min_p_greedy_triggers_win(self, greedy_kwargs: dict[str, Any]): + # An explicit greedy trigger collapses to a single token even with min_p. + params = SamplingParams(min_p=0.1, **greedy_kwargs) + self._check_params(params) + request = self._build_mock_llm_request(params) + assert _request_strategy(request, vocab_size=self.VOCAB_SIZE) is GREEDY + def test_param_validation(self): with pytest.raises(ValueError, match="require temperature >= 0, got temperature=-1"): SamplingParams(temperature=-1) @@ -362,6 +450,12 @@ def test_param_validation(self): with pytest.raises(ValueError, match="require top_k >= 0, got top_k=-1"): SamplingParams(top_k=-1) + with pytest.raises(ValueError, match="require 0 <= min_p <= 1, got min_p=-1"): + SamplingParams(min_p=-1) + + with pytest.raises(ValueError, match="require 0 <= min_p <= 1, got min_p=2"): + SamplingParams(min_p=2) + @pytest.mark.parametrize( "top_k, top_p", [ @@ -875,11 +969,9 @@ def stop_words_that_raises(*args, **kwargs): @contextmanager def raising_stop_words_ctx(expect_raise: bool) -> Generator[None, None, None]: with monkeypatch.context() as patch_ctx: + patch_ctx.setattr(FinishReasonsHandler, "_are_stop_words", stop_words_that_raises) patch_ctx.setattr( - TorchSampler.FinishReasonsHandler, "_are_stop_words", stop_words_that_raises - ) - patch_ctx.setattr( - TorchSampler.FinishReasonsHandler, + FinishReasonsHandler, "_are_stop_words_single_token", stop_words_that_raises, ) @@ -931,7 +1023,7 @@ def stop_words_that_raises(*args, **kwargs): def raising_single_token_stop_words_ctx(expect_raise: bool) -> Generator[None, None, None]: with monkeypatch.context() as patch_ctx: patch_ctx.setattr( - TorchSampler.FinishReasonsHandler, + FinishReasonsHandler, "_are_stop_words_single_token", stop_words_that_raises, ) @@ -1112,6 +1204,69 @@ def setup_sampler_step_with_size_check(self, scheduled_requests: ScheduledReques run_test_with_warmup(uut_provider_with_resize_on_demand, max_sync_s=None) +@pytest.mark.parametrize("min_p", [0.0, 0.1, 0.5, 0.9]) +def test_min_p_renorm_probs(min_p: float): + """min_p_renorm_probs keeps tokens with p >= min_p * max and renormalizes.""" + torch.manual_seed(0) + probs = torch.softmax(torch.randn(4, 16), dim=-1) + + got = min_p_renorm_probs(probs.clone(), min_p) + + max_probs = probs.max(dim=-1, keepdim=True).values + kept = probs >= (min_p * max_probs) + expected = torch.where(kept, probs, torch.zeros_like(probs)) + expected = expected / expected.sum(dim=-1, keepdim=True) + + torch.testing.assert_close(got, expected) + # every row still sums to 1 and the argmax token always survives + torch.testing.assert_close(got.sum(dim=-1), torch.ones(probs.size(0))) + assert (got.gather(1, probs.argmax(dim=-1, keepdim=True)) > 0).all() + + +def test_min_p_renorm_probs_per_request_tensor(): + """A per-request min_p tensor applies a distinct threshold per row.""" + torch.manual_seed(1) + probs = torch.softmax(torch.randn(3, 16), dim=-1) + min_p = torch.tensor([0.0, 0.3, 0.95]) + + got = min_p_renorm_probs(probs.clone(), min_p) + + max_probs = probs.max(dim=-1, keepdim=True).values + kept = probs >= (min_p.reshape(-1, 1) * max_probs) + expected = torch.where(kept, probs, torch.zeros_like(probs)) + expected = expected / expected.sum(dim=-1, keepdim=True) + torch.testing.assert_close(got, expected) + # row 0 (min_p=0) keeps everything; row 2 (min_p=0.95) prunes more aggressively + assert (got[0] > 0).all() + assert (got[2] > 0).sum() <= (got[0] > 0).sum() + + +def test_min_p_sample_top_k_disabled_sentinel(): + """min_p + unset top_k must survive the standalone sample() dispatch. + + Draft-model rejection sampling resolves strategies with vocab_size=2**31 + (the greedy probe), so a min_p request with an unset top_k carries the + disabled-top_k sentinel 0. That 0 flows straight into sample() -> + top_k_top_p_sampling_batch without sanitize_top_k, so the vanilla path must + treat it as "keep all" instead of tripping ``assert top_k > 1``. Regression + test for min_p under speculative decoding with rejection sampling. + """ + min_p = 0.5 + # ("min_p", top_k, top_p, min_p, temperature) with the top_k=0 sentinel. + strategy: MinP = ("min_p", 0, 1.0, min_p, 1.0) + + torch.manual_seed(0) + logits = torch.randn(4, 32) + # Must not raise (top_k=0 previously hit ``assert top_k > 1``). + tokens, _, _ = sample(strategy, logits.clone()) + + assert tokens.shape == (4,) + # min_p filtering was applied: every sampled token clears the min_p mask. + probs = torch.softmax(logits, dim=-1) + kept = probs >= (min_p * probs.max(dim=-1, keepdim=True).values) + assert kept.gather(1, tokens.unsqueeze(-1)).all() + + class TestBatchedSampling: """Validate batched/mixed sampling. @@ -1142,6 +1297,7 @@ def _build_test_cases( TopP: SamplingParams(top_p=0.42, temperature=0.2), TopK: SamplingParams(top_k=27, temperature=0.5), TopKTopP: SamplingParams(top_k=27, top_p=0.6, temperature=0.5), + MinP: SamplingParams(min_p=0.02, top_k=40, top_p=0.9, temperature=1.0), } # Check that all relevant strategies are covered @@ -1256,9 +1412,13 @@ def _perturb_params(param: SamplingParams): temperature = param.temperature if temperature is not None: temperature *= max(rng.random(), 1e-6) + min_p = param.min_p + if min_p is not None: + min_p *= max(rng.random(), 1e-6) return SamplingParams( top_p=top_p, top_k=top_k, + min_p=min_p, temperature=temperature, ) @@ -1553,6 +1713,7 @@ def _uut_provider(is_warmup: bool) -> Generator[Callable[[], None], None, None]: TopP, TopK, TopKTopP, + MinP, ] } @@ -1616,7 +1777,8 @@ def _uut(): torch.testing.assert_close(probs, expected_probs_after_temperature) else: if strategy[0] not in [ - strategy_tags[strategy_type] for strategy_type in [TopP, TopK, TopKTopP] + strategy_tags[strategy_type] + for strategy_type in [TopP, TopK, TopKTopP, MinP] ]: raise ValueError(f"Unknown strategy: {strategy}") @@ -1682,6 +1844,14 @@ def _uut(): cast(float, top_p), ).all() + if strategy[0] == strategy_tags[MinP]: + # Renorm preserves the ratio, so every kept token satisfies + # prob >= min_p * max (holds with top_k/top_p also applied). + min_p_val = cast(float, strategy[3]) + kept = probs != 0.0 + ratio = probs / probs.amax(dim=-1, keepdim=True) + assert torch.all((ratio >= min_p_val - 1e-6)[kept]) + # All indices not selected must have logits less or equal # to the smallest selected logit. probs_selected_min = torch.where( @@ -1862,6 +2032,7 @@ class _TorchUtilsSamplingParams: temperature: Optional[torch.Tensor] top_p: Optional[torch.Tensor] top_k: Optional[torch.Tensor] + min_p: Optional[torch.Tensor] = None @dataclass(frozen=True, kw_only=True) class _MockSamplingLogEntry: @@ -1925,6 +2096,52 @@ def _mock_flashinfer_top_k_top_p( _mock_flashinfer_top_k_top_p, ) + def _mock_flashinfer_top_k_top_p_from_probs( + probs: torch.Tensor, + *, + top_k: torch.Tensor, + top_p: torch.Tensor, + filter_apply_order: str, + deterministic: bool, + check_nan: bool, + generator: torch.Generator, + seed: Optional[Union[int, torch.Tensor]] = None, + offset: Optional[Union[int, torch.Tensor]] = None, + ) -> torch.Tensor: + # The min_p strategy terminates its renorm chain here, so the probs + # recorded below already have min_p applied; min_p itself never + # reaches a flashinfer kernel and thus cannot be captured as a param. + # Patching this is not optional: unpatched, the real flashinfer + # implementation delegates to the *patched* top_p_sampling_from_probs + # with kwargs its mock does not accept. + assert filter_apply_order == "top_k_first" + assert deterministic + assert not check_nan, "check_nan syncs" + assert generator is sampler.get_generator(probs.device) + nonlocal mock_sampling_log + new_entries = [ + TestBatchedSampling._MockSamplingLogEntry( + probs=probs[row_idx], + sampling_params=TestBatchedSampling._TorchUtilsSamplingParams( + top_k=top_k[row_idx], + top_p=top_p[row_idx], + temperature=None, + ), + ) + for row_idx in range(probs.size(0)) + ] + mock_tokens = torch.arange( + len(mock_sampling_log), len(mock_sampling_log) + len(new_entries) + ) + mock_sampling_log += new_entries + return mock_tokens + + patch_ctx.setattr( + flashinfer.sampling, + "top_k_top_p_sampling_from_probs", + _mock_flashinfer_top_k_top_p_from_probs, + ) + def _mock_flashinfer_from_logits( logits: torch.Tensor, *, @@ -2021,6 +2238,40 @@ def _mock_flashinfer_top_p( patch_ctx.setattr(flashinfer.sampling, "top_p_sampling_from_probs", _mock_flashinfer_top_p) + def _mock_flashinfer_min_p( + probs: torch.Tensor, + min_p: torch.Tensor, + *, + deterministic: bool, + check_nan: bool, + generator: torch.Generator, + seed: Optional[Union[int, torch.Tensor]] = None, + offset: Optional[Union[int, torch.Tensor]] = None, + ) -> torch.Tensor: + assert deterministic + assert not check_nan, "check_nan syncs" + assert generator is sampler.get_generator(probs.device) + nonlocal mock_sampling_log + new_entries = [ + TestBatchedSampling._MockSamplingLogEntry( + probs=probs[row_idx], + sampling_params=TestBatchedSampling._TorchUtilsSamplingParams( + top_k=None, + top_p=None, + temperature=None, + min_p=min_p[row_idx], + ), + ) + for row_idx in range(probs.size(0)) + ] + mock_tokens = torch.arange( + len(mock_sampling_log), len(mock_sampling_log) + len(new_entries) + ) + mock_sampling_log += new_entries + return mock_tokens + + patch_ctx.setattr(flashinfer.sampling, "min_p_sampling_from_probs", _mock_flashinfer_min_p) + def _mock_flashinfer_from_probs( probs: torch.Tensor, *, @@ -2105,6 +2356,10 @@ def _validate_intercepted_probs( log_entry.sampling_params.top_k is not None and log_entry.sampling_params.top_k.item() != vocab_size ) + req_has_min_p = ( + log_entry.sampling_params.min_p is not None + and log_entry.sampling_params.min_p.item() != 0 + ) if req_has_top_k: assert req_params.top_k is not None assert log_entry.sampling_params.top_k is not None @@ -2113,7 +2368,12 @@ def _validate_intercepted_probs( assert req_params.top_p is not None assert log_entry.sampling_params.top_p is not None assert np.allclose(req_params.top_p, log_entry.sampling_params.top_p.item()) - if req_has_top_k or req_has_top_p: + if req_has_min_p: + assert req_params.min_p is not None + assert log_entry.sampling_params.min_p is not None + assert np.allclose(req_params.min_p, log_entry.sampling_params.min_p.item()) + # min_p also filters to a top-prefix subset, so it reuses the validation below. + if req_has_top_k or req_has_top_p or req_has_min_p: # for top-k and/or top-p _sampling_, probs contains only the top probs, # whereas log_entry.probs contains all probs passed to the sampling code. @@ -2577,6 +2837,11 @@ class UutResultWrapper: result: Optional[UutResult] = None res = UutResultWrapper() + # Precomputed outside the no-sync region (mirrors the production + # resident device copy of seq_slots). + seq_slots_tensor_cuda = ( + seq_slots_tensor.to(torch.int64).pin_memory().to("cuda", non_blocking=True) + ) def _uut(res=res): new_tokens_host = sampler._unbatch_sampling_results( @@ -2584,6 +2849,7 @@ def _uut(res=res): new_tokens_cuda=new_tokens_cuda, req_num_generated_tokens=req_num_steps, seq_slots=seq_slots_tensor, + seq_slots_cuda=seq_slots_tensor_cuda, ) res.result = UutResult(new_tokens_host=new_tokens_host) @@ -2618,3 +2884,202 @@ def _uut(res=res): input_offset += steps run_test_with_warmup(_uut_provider, max_sync_s=0.2) + + +class TestTopPDecay: + """Minimal functional guards for Top-P Decay in TorchSampler. + + Covers strategy routing, the post-sample runtime update (parity with the + C++ computeToppDecay recurrence; cases ported from + topPSamplingLayerTest.cpp), and per-request rejection of unsupported + combinations. + """ + + VOCAB_SIZE = 1000 + + @staticmethod + def _params(**kw) -> UtilsSamplingParams: + base = dict(temperature=None, top_p=None, top_k=None, use_beam_search=False) + base.update(kw) + return UtilsSamplingParams(**base) + + @staticmethod + def _make_sampler(*, max_draft_len=0): + return TorchSampler( + TorchSampler.Args( + max_seq_len=128, + max_draft_len=max_draft_len, + max_num_sequences=8, + max_beam_width=1, + max_total_draft_tokens=max_draft_len, + disable_overlap_scheduler=True, + ) + ) + + def test_strategy_routing(self): + # Active decay (set and < 1.0) forces a top-p-capable strategy even for + # an otherwise-greedy request (initial top-p defaults to 1.0), so the + # decayed runtime value can take effect on later steps. + s = resolve_sampling_strategy(self._params(top_p_decay=0.5), vocab_size=self.VOCAB_SIZE) + assert s[0] == "top_p" and s[1] == pytest.approx(1.0) + s = resolve_sampling_strategy( + self._params(top_k=50, top_p=0.9, top_p_decay=0.8), vocab_size=self.VOCAB_SIZE + ) + assert s[0] == "top_k_top_p" + # decay == 1.0 (the C++ default) is a no-op and does not activate... + s = resolve_sampling_strategy(self._params(top_p_decay=1.0), vocab_size=self.VOCAB_SIZE) + assert s is GREEDY + # ...and an explicit greedy control wins over an active decay. + s = resolve_sampling_strategy( + self._params(top_p_decay=0.5, top_k=1), vocab_size=self.VOCAB_SIZE + ) + assert s is GREEDY + # min_p wins the strategy choice, but the request keeps carrying top_p, + # so decay stays applicable (see test_decay_metadata_dispatch). + s = resolve_sampling_strategy( + self._params(min_p=0.1, top_p=0.9, top_p_decay=0.8), vocab_size=self.VOCAB_SIZE + ) + assert s[0] == "min_p" + + # Every strategy a decay-active request can resolve to carries a per-row + # top-p, so all of them must be offered the decay metadata. A strategy + # missing from the dispatch silently drops decay: the request is still + # admitted and its runtime top-p still decays, but sampling keeps reading + # the static initial value. + @pytest.mark.parametrize( + "params", + [ + pytest.param(dict(top_p_decay=0.8), id="top_p"), + pytest.param(dict(top_k=50, top_p=0.9, top_p_decay=0.8), id="top_k_top_p"), + pytest.param(dict(min_p=0.1, top_p=0.9, top_p_decay=0.8), id="min_p"), + ], + ) + def test_decay_metadata_dispatch(self, params): + strategy = resolve_sampling_strategy(self._params(**params), vocab_size=self.VOCAB_SIZE) + group_key = FlashInferGroupedStrategySampler.strategy_grouping_key(strategy) + assert ( + FlashInferGroupedStrategySampler.get_metadata_type_for_group(group_key) + is TopPDecayMetadata + ) + + # Companion to the dispatch test: the metadata must not just be handed over + # but actually override the per-row top-p. Logits are chosen so the static + # top_p=1.0 leaves every token samplable (min_p=0.1 keeps them all too), + # while the decayed runtime top-p of 0.3 is below the argmax's own + # probability and collapses the nucleus onto it. + @pytest.mark.parametrize( + "strategy", + [ + pytest.param(("top_p", 1.0, 1.0), id="top_p"), + pytest.param(("top_k_top_p", 5, 1.0, 1.0), id="top_k_top_p"), + pytest.param(("min_p", 0, 1.0, 0.1, 1.0), id="min_p"), + ], + ) + @pytest.mark.parametrize("return_probs", [True, False], ids=["with_probs", "sample_only"]) + def test_decay_override_reaches_sampling(self, strategy, return_probs): + num_rows, vocab, decayed_top_p = 64, 5, 0.3 + logits = torch.zeros(num_rows, vocab, device="cuda") + logits[:, 0] = 1.0 + argmax = 0 + + def run(is_decay_slot: bool) -> set[int]: + metadata = TopPDecayMetadata( + # All rows share slot 0, so a single store entry gates them all. + slots=torch.zeros(num_rows, dtype=torch.int64, device="cuda"), + runtime_top_p=torch.tensor([decayed_top_p], dtype=torch.float32, device="cuda"), + is_decay_slot=torch.tensor([is_decay_slot], dtype=torch.bool, device="cuda"), + ) + tokens, _, _ = FlashInferGroupedStrategySampler.sample_grouped_strategies( + FlashInferGroupedStrategySampler.strategy_grouping_key(strategy), + [cast(Strategy, strategy)] * num_rows, + logits, + generator=torch.Generator(device="cuda").manual_seed(0), + return_probs=return_probs, + group_metadata=metadata, + ) + return set(tokens.flatten().tolist()) + + # Gate off: the static top-p applies and sampling spreads over the vocab. + assert len(run(is_decay_slot=False)) > 1 + # Gate on: the decayed runtime top-p replaces it and only the argmax survives. + assert run(is_decay_slot=True) == {argmax} + + def test_runtime_update_parity(self): + # Post-sample update parity with the C++ computeToppDecay recurrence + # (a negative reset_id never matches, since token ids are non-negative): + # runtime = initial if token == reset_id + # = max(runtime * decay, min) otherwise + sampler = self._make_sampler() + store = sampler._top_p_decay.store + configs = [ + dict(initial=0.8, decay=0.3, top_p_min=0.5, reset_id=2), # decay, then reset + dict(initial=0.2, decay=0.9, top_p_min=0.1, reset_id=-1), # plain decay, floored + dict(initial=0.3, decay=0.5, top_p_min=0.6, reset_id=-1), # min > initial: rises + ] + token_steps = [[1, 2, 3], [9, 9, 9], [9, 9, 9]] + slots = list(range(len(configs))) + for slot, cfg in zip(slots, configs): + sampler._top_p_decay._slots.add(slot) + store.runtime_top_p_decay_cuda[slot] = cfg["initial"] + store.initial_top_p_decay_cuda[slot] = cfg["initial"] + store.top_p_decay_cuda[slot] = cfg["decay"] + store.top_p_decay_min_cuda[slot] = cfg["top_p_min"] + store.top_p_decay_reset_ids_cuda[slot] = cfg["reset_id"] + store.is_top_p_decay_slot_cuda[slot] = True + + runtime = [cfg["initial"] for cfg in configs] + slots_cuda = torch.tensor(slots, dtype=torch.int64, device="cuda") + for step in range(3): + for slot in slots: + sampler.store.new_tokens[0, slot, 0] = token_steps[slot][step] + sampler._top_p_decay.update_after_sample( + step_tokens=sampler.store.new_tokens[0, :, 0], sampled_slots_cuda=slots_cuda + ) + got = store.runtime_top_p_decay_cuda.cpu() + for slot, cfg in zip(slots, configs): + tok = token_steps[slot][step] + if tok == cfg["reset_id"]: + runtime[slot] = cfg["initial"] + else: + runtime[slot] = max(runtime[slot] * cfg["decay"], cfg["top_p_min"]) + assert got[slot].item() == pytest.approx(runtime[slot], abs=1e-6), (step, slot) + + @staticmethod + def _mock_request(params: SamplingParams, *, draft_tokens=None): + params._validate() + req = SimpleNamespace( + sampling_config=SamplingConfig(params._get_sampling_config()), + is_context_init_state=False, + py_sampling_strategy=None, + py_draft_tokens=draft_tokens, + ) + req.get_beam_width_by_iter = lambda for_next_iteration=False: 1 + return cast(LlmRequest, req) + + @pytest.mark.parametrize( + "bad_kwargs", + [ + {"top_p_decay": 1.5}, + {"top_p_decay": -0.5}, + {"top_p_decay": 0.0}, + {"top_p_min": 0.0}, + {"top_p_min": 1.5}, + {"top_p_reset_ids": -1}, + ], + ) + def test_out_of_range_decay_params_rejected(self, bad_kwargs): + # Out-of-range decay params raise (mirroring the executor::SamplingConfig + # constructor's hard checks) instead of the former warn-and-default. + with pytest.raises(ValueError): + SamplingParams(**bad_kwargs) + + def test_reject_speculative_draft_tokens(self): + # Decay + draft tokens through TorchSampler is rejected per-request at + # admission (validate_request), so only the offending request fails. + sampler = self._make_sampler(max_draft_len=4) + with pytest.raises(ValueError, match="speculative"): + sampler.validate_request( + self._mock_request(SamplingParams(top_p=0.9, top_p_decay=0.5), draft_tokens=[1, 2]) + ) + # Same request without decay is accepted. + sampler.validate_request(self._mock_request(SamplingParams(top_p=0.9), draft_tokens=[1, 2])) diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_advanced_sampling_mode.py b/tests/unittest/_torch/speculative/hw_agnostic/test_advanced_sampling_mode.py new file mode 100644 index 000000000000..9c760526ff79 --- /dev/null +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_advanced_sampling_mode.py @@ -0,0 +1,158 @@ +# Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for one-model ``advanced_sampling_mode``. + +Covers the config contract (enum + skip properties + the use_rejection_sampling +requirement for the top-p-disabling modes), ``resolve_advanced_sampling_filters`` +mode resolution, a CUDA check that NO_TOPK yields the same distribution as FULL +when top_k is disabled, and native greedy handling (greedy rows return argmax). +""" + +import pytest +import torch + +from tensorrt_llm._torch.pyexecutor.sampler.ops import flashinfer as su +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import GREEDY_TEMPERATURE_THRESHOLD +from tensorrt_llm.llmapi.llm_args import AdvancedSamplingMode, DecodingBaseConfig, MTPDecodingConfig + + +def test_enum_skip_properties(): + """Enum members + the skip properties (single source of truth for filter skipping).""" + M = AdvancedSamplingMode + assert [m.value for m in M] == ["full", "no_topk", "no_topp", "no_topk_no_topp"] + assert (M.FULL.skips_top_k, M.FULL.skips_top_p) == (False, False) + assert (M.NO_TOPK.skips_top_k, M.NO_TOPK.skips_top_p) == (True, False) + assert (M.NO_TOPP.skips_top_k, M.NO_TOPP.skips_top_p) == (False, True) + assert (M.NO_TOPK_NO_TOPP.skips_top_k, M.NO_TOPK_NO_TOPP.skips_top_p) == (True, True) + + +def test_advanced_sampling_mode_on_base_config(): + """The field lives on DecodingBaseConfig (not MTP-specific) and defaults to FULL.""" + assert "advanced_sampling_mode" in DecodingBaseConfig.model_fields + assert MTPDecodingConfig(max_draft_len=1).advanced_sampling_mode == AdvancedSamplingMode.FULL + + +def test_all_modes_construct_regardless_of_rejection(): + """Every mode constructs with or without rejection sampling (no config gating).""" + for mode in ("full", "no_topk", "no_topp", "no_topk_no_topp"): + for rej in (False, True): + cfg = MTPDecodingConfig( + max_draft_len=1, advanced_sampling_mode=mode, use_rejection_sampling=rej + ) + assert cfg.advanced_sampling_mode.value == mode + + +@pytest.mark.parametrize( + "mode,expect_top_k_none,expect_top_p_none", + [ + ("full", False, False), + ("no_topk", True, False), + ("no_topp", False, True), + ("no_topk_no_topp", True, True), + ], +) +def test_resolve_advanced_sampling_filters(mode, expect_top_k_none, expect_top_p_none): + """Mode resolution None-ifies disabled filters (so the op skips that kernel) + and passes kept filters through unchanged.""" + top_k = torch.zeros(2, dtype=torch.int32) + top_p = torch.ones(2) + eff_top_k, eff_top_p = su.resolve_advanced_sampling_filters( + AdvancedSamplingMode(mode), top_k, top_p + ) + assert (eff_top_k is None) is expect_top_k_none + assert (eff_top_p is None) is expect_top_p_none + if not expect_top_k_none: + assert eff_top_k is top_k + if not expect_top_p_none: + assert eff_top_p is top_p + + +@pytest.mark.skipif( + not torch.cuda.is_available(), reason="requires CUDA + flashinfer sampling kernels" +) +@pytest.mark.parametrize("top_p_val", [1.0, 0.9]) +def test_no_topk_matches_full(top_p_val): + """With top_k disabled, NO_TOPK skips the top_k mask kernel (a no-op at k=vocab) + and yields the same sampling distribution as FULL. We compare the resulting + probability distributions rather than the sampled tokens: the flashinfer top_k mask + at k=vocab injects ~1e-8 fp noise that leaves the distribution unchanged but can flip + an individual sampled token across GPU archs, so exact-token equality is not portable. + A real (non-no-op) filter would move mass by orders of magnitude, far above atol.""" + dev = "cuda" + torch.manual_seed(0) + batch, vocab = 64, 32000 + logits = torch.randn(batch, vocab, device=dev, dtype=torch.float32) * 2.0 + temperatures = torch.full((batch,), 0.7, device=dev, dtype=torch.float32) + top_k = torch.zeros(batch, device=dev, dtype=torch.int32) # disabled + top_p = torch.full((batch,), top_p_val, device=dev, dtype=torch.float32) + + ek_full, ep_full = su.resolve_advanced_sampling_filters( + AdvancedSamplingMode.FULL, top_k.clone(), top_p + ) + ek_nt, ep_nt = su.resolve_advanced_sampling_filters( + AdvancedSamplingMode.NO_TOPK, top_k.clone(), top_p + ) + probs_full = su.compute_probs_from_logits(logits.clone(), temperatures, ek_full, ep_full) + probs_no_topk = su.compute_probs_from_logits(logits.clone(), temperatures, ek_nt, ep_nt) + assert torch.allclose(probs_full, probs_no_topk, atol=1e-5, rtol=0) + + +@pytest.mark.skipif( + not torch.cuda.is_available(), reason="requires CUDA + flashinfer sampling kernels" +) +@pytest.mark.parametrize("mode", ["no_topk", "no_topk_no_topp"]) +def test_greedy_row_returns_argmax_natively(mode): + """Greedy rows carry the sentinel temperature, so the sampler returns their + argmax token even in a mixed batch -- this is why no mixed-batch guard is needed.""" + dev = "cuda" + torch.manual_seed(0) + batch, vocab = 8, 4096 + logits = torch.randn(batch, vocab, device=dev, dtype=torch.float32) * 3.0 + disable = GREEDY_TEMPERATURE_THRESHOLD / 10 # sentinel for greedy rows + temperatures = torch.full((batch,), 0.7, device=dev, dtype=torch.float32) + temperatures[0] = disable # greedy rows mixed with sampled rows + temperatures[1] = disable + top_k = torch.zeros(batch, device=dev, dtype=torch.int32) + top_p = torch.ones(batch, device=dev, dtype=torch.float32) + seed = torch.tensor([7], dtype=torch.int64, device=dev) + offset = torch.tensor([0], dtype=torch.int64, device=dev) + + eff_top_k, eff_top_p = su.resolve_advanced_sampling_filters( + AdvancedSamplingMode(mode), top_k, top_p + ) + tokens = su.sample_from_logits_op( + logits, temperatures, eff_top_k, eff_top_p, seed=seed, offset=offset + ) + argmax = logits.argmax(dim=-1) + assert tokens[0].item() == argmax[0].item() + assert tokens[1].item() == argmax[1].item() + + +def test_advanced_mode_accepted_on_all_spec_paths(): + """The MTP-one-model-only gate was removed (the field is on the base config), + so non-FULL modes construct on any spec path instead of raising at config time.""" + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + + args = TorchLlmArgs( + model="/tmp/dummy_model", + skip_tokenizer_init=True, + speculative_config=MTPDecodingConfig( + max_draft_len=1, use_mtp_vanilla=True, advanced_sampling_mode="no_topk" + ), + ) + assert args.speculative_config.advanced_sampling_mode == AdvancedSamplingMode.NO_TOPK + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash.py index bb920e27bfae..3258e9065b56 100644 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_dflash.py +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dflash.py @@ -85,7 +85,8 @@ def test_dflash_qwen3_8b(disable_overlap_scheduler: bool): dflash_model_dir=f"{models_path}/Qwen3-8B-DFlash-b16", disable_overlap_scheduler=disable_overlap_scheduler, ) - _run_and_check(llm_config, min_avg_accepted=1.0) + # Expected acceptance is 1.77, use 1.5 to leave 15% margin + _run_and_check(llm_config, min_avg_accepted=1.5) @pytest.mark.parametrize("disable_overlap_scheduler", [True, False]) @@ -101,7 +102,8 @@ def test_dflash_qwen3_5_4b(disable_overlap_scheduler: bool): dflash_model_dir=f"{models_path}/Qwen3.5-4B-DFlash", disable_overlap_scheduler=disable_overlap_scheduler, ) - _run_and_check(llm_config, min_avg_accepted=1.0) + # Expected acceptance is 1.77, use 1.5 to leave 15% margin + _run_and_check(llm_config, min_avg_accepted=1.5) @pytest.mark.high_cuda_memory diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_attention.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_attention.py new file mode 100644 index 000000000000..6ed4f1cba050 --- /dev/null +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_attention.py @@ -0,0 +1,454 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the DSpark captured-context attention primitives (CPU). + +The two primitives are validated against fully independent computations: +``dspark_sparse_attn`` vs (a) a per-element loop reference of the kernel formula +and (b) ``torch.nn.functional.scaled_dot_product_attention`` for the no-sink, +all-valid case; ``get_dspark_topk_idxs`` vs the reference index formula. +""" + +import types +from unittest.mock import Mock + +import pytest +import torch +import torch.nn.functional as F + +import tensorrt_llm._torch.models.modeling_dspark as modeling_dspark +from tensorrt_llm._torch.models.dspark.attention import ( + apply_dspark_rotary, + dspark_attention_forward, + dspark_sparse_attn, + get_dspark_topk_idxs, + precompute_dspark_freqs_cis, +) +from tensorrt_llm._torch.models.modeling_dspark import DSparkDraftModel + + +def test_rope_table_is_cached_once_per_device(): + model = types.SimpleNamespace( + _attn_params={"rope_head_dim": 16}, + _freqs_cap=64, + _rope_theta=10000.0, + _freqs_table_cache={}, + ) + + first = DSparkDraftModel._dspark_freqs_table(model, torch.device("cpu")) + second = DSparkDraftModel._dspark_freqs_table(model, torch.device("cpu")) + + assert first.data_ptr() == second.data_ptr() + assert len(model._freqs_table_cache) == 1 + positions = torch.tensor([1, 17, 63]) + expected = precompute_dspark_freqs_cis(16, 64, rope_theta=10000.0) + torch.testing.assert_close(first[positions], expected[positions]) + + +def test_dspark_block_uses_stage_id_as_attention_layer_idx(monkeypatch): + captured = {} + + def fake_decoder_layer_init( + self, + model_config, + layer_idx, + aux_stream_dict, + attention_layer_idx=None, + mapping_with_cp=None, + disable_post_moe_fusion=False, + ): + torch.nn.Module.__init__(self) + self.model_config = model_config + self.config = model_config.pretrained_config + self.layer_idx = layer_idx + captured.update( + layer_idx=layer_idx, + attention_layer_idx=attention_layer_idx, + aux_stream_dict=aux_stream_dict, + mapping_with_cp=mapping_with_cp, + disable_post_moe_fusion=disable_post_moe_fusion, + ) + + monkeypatch.setattr( + modeling_dspark.DeepseekV4DecoderLayer, + "__init__", + fake_decoder_layer_init, + ) + model_config = types.SimpleNamespace( + pretrained_config=types.SimpleNamespace(vocab_size=128, hc_mult=2), + spec_config=None, + ) + + block = modeling_dspark.DSparkBlock( + model_config, + layer_idx=10, + aux_stream_dict={}, + stage_id=1, + num_stages=3, + num_capture_layers=0, + ) + + assert block.layer_idx == captured["layer_idx"] == 10 + assert captured["attention_layer_idx"] == block.stage_id == 1 + assert captured["disable_post_moe_fusion"] is True + + +@pytest.mark.parametrize("enable_fused_hc", [True, False]) +def test_forward_stage_honors_enable_fused_hc(monkeypatch, enable_fused_hc): + """The draft stage must use the inherited fused-HC rollback setting.""" + torch.manual_seed(71) + num_requests, block_size, hc_mult, hidden_size = 1, 2, 2, 3 + h = torch.randn(num_requests, block_size, hc_mult, hidden_size) + attention_input = torch.randn(num_requests, block_size, hidden_size) + attention_output = torch.randn_like(attention_input) + mid_residual = torch.randn_like(h) + attention_post_mix = torch.randn(num_requests, block_size, hc_mult, 1) + attention_comb_mix = torch.randn(num_requests, block_size, hc_mult, hc_mult) + ffn_post_mix = torch.randn_like(attention_post_mix) + ffn_comb_mix = torch.randn_like(attention_comb_mix) + raw_ffn_input = torch.randn_like(attention_input) + normed_ffn_input = torch.randn_like(attention_input) + moe_output = torch.randn(num_requests * block_size, hidden_size) + final_h = torch.randn_like(h) + events = [] + + def record(name, result): + def call(*args, **kwargs): + events.append(name) + return result + + return call + + monkeypatch.setattr( + modeling_dspark, + "dspark_attention_forward", + Mock(return_value=attention_output), + ) + + hc_attn = types.SimpleNamespace( + pre_mapping=Mock(return_value=(attention_post_mix, attention_comb_mix, attention_input)), + post_mapping=Mock(side_effect=record("attention_post", mid_residual)), + ) + hc_ffn = types.SimpleNamespace( + fused_hc=Mock( + side_effect=record( + "fused", + (mid_residual, ffn_post_mix, ffn_comb_mix, normed_ffn_input), + ) + ), + pre_mapping=Mock( + side_effect=record("ffn_pre", (ffn_post_mix, ffn_comb_mix, raw_ffn_input)) + ), + post_mapping=Mock(side_effect=record("ffn_post", final_h)), + ) + post_attention_layernorm = Mock(side_effect=record("ffn_norm", normed_ffn_input)) + post_attention_layernorm.weight = torch.ones(hidden_size) + post_attention_layernorm.variance_epsilon = 1e-6 + stage = types.SimpleNamespace( + enable_fused_hc=enable_fused_hc, + hc_attn=hc_attn, + hc_ffn=hc_ffn, + input_layernorm=Mock(side_effect=lambda tensor: tensor), + post_attention_layernorm=post_attention_layernorm, + mlp=Mock(return_value=moe_output), + _dspark_attn={}, + ) + model = types.SimpleNamespace( + use_real_mla=False, + _attn_params={"window_size": 2, "head_dim": 1}, + model_config=types.SimpleNamespace( + mapping=types.SimpleNamespace(enable_attention_dp=False, tp_size=8) + ), + ) + + actual = DSparkDraftModel._forward_stage( + model, + stage, + h, + torch.randn(num_requests, hidden_size), + 1, + torch.empty(0), + torch.zeros(num_requests, block_size, dtype=torch.long), + ) + + assert actual is final_h + torch.testing.assert_close( + stage.mlp.call_args.args[0], + normed_ffn_input.reshape(num_requests * block_size, hidden_size), + ) + # Non-attention-DP multi-GPU (enable_attention_dp=False, tp_size>1): the draft + # MoE must all-reduce its TP-sharded output, mirroring the target MoE. The + # attention-DP and single-GPU paths keep it disabled. + assert stage.mlp.call_args.kwargs["final_all_reduce_params"].enable_allreduce is True + if enable_fused_hc: + assert events == ["fused", "ffn_post"] + hc_ffn.fused_hc.assert_called_once() + hc_attn.post_mapping.assert_not_called() + hc_ffn.pre_mapping.assert_not_called() + post_attention_layernorm.assert_not_called() + fused_kwargs = hc_ffn.fused_hc.call_args.kwargs + assert fused_kwargs["norm_weight"] is post_attention_layernorm.weight + assert fused_kwargs["norm_eps"] == post_attention_layernorm.variance_epsilon + else: + assert events == ["attention_post", "ffn_pre", "ffn_norm", "ffn_post"] + hc_ffn.fused_hc.assert_not_called() + hc_attn.post_mapping.assert_called_once() + hc_ffn.pre_mapping.assert_called_once_with(mid_residual) + post_attention_layernorm.assert_called_once_with(raw_ffn_input) + + +def _ref_precompute_freqs_cis(dim, seqlen, base): + """DeepSpec precompute_freqs_cis with original_seq_len == 0 (no YaRN).""" + freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + t = torch.arange(seqlen) + freqs = torch.outer(t, freqs) + return torch.polar(torch.ones_like(freqs), freqs) + + +def _ref_apply_rotary_emb(x, freqs_cis, inverse=False): + """DeepSpec apply_rotary_emb (returns a fresh tensor instead of in-place).""" + xc = torch.view_as_complex(x.float().unflatten(-1, (-1, 2))) + if inverse: + freqs_cis = freqs_cis.conj() + if xc.ndim == 3: + fc = freqs_cis.view(1, xc.size(1), xc.size(-1)) + else: + fc = freqs_cis.view(1, xc.size(1), 1, xc.size(-1)) + return torch.view_as_real(xc * fc).flatten(-2).to(x.dtype) + + +def _loop_reference(q, kv, attn_sink, topk_idxs, scale): + """Obvious, slow per-(b,m,h) reference of the exact kernel math.""" + b, m, h, d = q.shape + out = torch.zeros(b, m, h, d, dtype=torch.float32) + qf, kvf, sink = q.float(), kv.float(), attn_sink.float() + for bi in range(b): + for mi in range(m): + idxs = topk_idxs[bi, mi].tolist() + for hi in range(h): + scores, vecs = [], [] + for j in idxs: + if j < 0: + continue + k = kvf[bi, j] + scores.append(torch.dot(qf[bi, mi, hi], k) * scale) + vecs.append(k) + if not scores: + continue + s = torch.stack(scores) + smax = s.max() + p = torch.exp(s - smax) + denom = p.sum() + torch.exp(sink[hi] - smax) + num = (p.unsqueeze(-1) * torch.stack(vecs)).sum(0) + out[bi, mi, hi] = num / denom + return out + + +@pytest.mark.parametrize("seed", [0, 1, 2]) +def test_sparse_attn_matches_loop_reference(seed): + torch.manual_seed(seed) + b, m, h, d, n, topk = 2, 5, 3, 16, 40, 12 + q = torch.randn(b, m, h, d) + kv = torch.randn(b, n, d) + attn_sink = torch.randn(h) + idx = torch.stack( + [torch.stack([torch.randperm(n)[:topk] for _ in range(m)]) for _ in range(b)] + ).int() + scale = d**-0.5 + got = dspark_sparse_attn(q, kv, attn_sink, idx, scale).float() + ref = _loop_reference(q, kv, attn_sink, idx, scale) + torch.testing.assert_close(got, ref, rtol=1e-4, atol=1e-4) + + +def test_sparse_attn_no_sink_matches_sdpa(): + """With a -inf sink (no sink mass) and all-valid contiguous indices, the + primitive must equal standard scaled-dot-product attention over the gathered + KV — an independent implementation.""" + torch.manual_seed(0) + b, m, h, d, topk = 2, 4, 5, 16, 9 + q = torch.randn(b, m, h, d) + kv = torch.randn(b, topk, d) # n == topk, attend to all + attn_sink = torch.full((h,), float("-inf")) + idx = torch.arange(topk).view(1, 1, -1).expand(b, m, topk).int() + scale = d**-0.5 + got = dspark_sparse_attn(q, kv, attn_sink, idx, scale).float() + + # SDPA: q [b,h,m,d], k/v [b,h,topk,d] (broadcast the shared KV over heads). + qh = q.permute(0, 2, 1, 3) + kvh = kv.unsqueeze(1).expand(b, h, topk, d) + ref = F.scaled_dot_product_attention(qh, kvh, kvh, scale=scale).permute(0, 2, 1, 3) + torch.testing.assert_close(got, ref, rtol=1e-4, atol=1e-4) + + +def test_sparse_attn_sink_reduces_mass(): + """A finite sink must strictly shrink the attention output magnitude vs an + infinitely-negative (disabled) sink, because it adds denominator mass only.""" + torch.manual_seed(0) + b, m, h, d, topk = 1, 2, 2, 16, 6 + q = torch.randn(b, m, h, d) + kv = torch.randn(b, topk, d) + idx = torch.arange(topk).view(1, 1, -1).expand(b, m, topk).int() + scale = d**-0.5 + no_sink = dspark_sparse_attn(q, kv, torch.full((h,), float("-inf")), idx, scale) + with_sink = dspark_sparse_attn(q, kv, torch.zeros(h), idx, scale) + assert with_sink.abs().sum() < no_sink.abs().sum() + + +def test_sparse_attn_masked_indices_excluded(): + """An index of -1 must be excluded exactly (equiv. to dropping that column).""" + torch.manual_seed(0) + b, m, h, d = 1, 1, 2, 16 + q = torch.randn(b, m, h, d) + kv = torch.randn(b, 5, d) + sink = torch.full((h,), float("-inf")) + scale = d**-0.5 + full = torch.tensor([[[0, 1, 2, 3]]]).int() + masked = torch.tensor([[[0, 1, 2, -1]]]).int() + drop3 = torch.tensor([[[0, 1, 2]]]).int() + got_masked = dspark_sparse_attn(q, kv, sink, masked, scale) + got_drop = dspark_sparse_attn(q, kv, sink, drop3, scale) + torch.testing.assert_close(got_masked, got_drop, rtol=1e-5, atol=1e-5) + # And masking genuinely changes the result vs attending to position 3. + got_full = dspark_sparse_attn(q, kv, sink, full, scale) + assert not torch.allclose(got_full, got_masked, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize( + "start_pos,window,block", [(1, 128, 5), (3, 128, 5), (10, 4, 5), (200, 128, 6)] +) +def test_get_dspark_topk_idxs_matches_reference(start_pos, window, block): + bsz = 3 + got = get_dspark_topk_idxs(window, bsz, block, start_pos) + # Reference formula (DeepSpec get_dspark_topk_idxs). + ctx = torch.arange(min(window, start_pos + 1)) + blk = window + torch.arange(block) + ref_row = torch.cat([ctx, blk]).int() + assert got.shape == (bsz, block, ref_row.numel()) + for bi in range(bsz): + for mi in range(block): + torch.testing.assert_close(got[bi, mi], ref_row) + + +def test_get_dspark_topk_idxs_requires_generation(): + with pytest.raises(AssertionError): + get_dspark_topk_idxs(128, 1, 5, 0) + + +@pytest.mark.parametrize("rope_head_dim,seqlen", [(64, 16), (64, 1), (128, 8)]) +def test_precompute_freqs_cis_matches_reference(rope_head_dim, seqlen): + got = precompute_dspark_freqs_cis(rope_head_dim, seqlen, rope_theta=10000.0) + ref = _ref_precompute_freqs_cis(rope_head_dim, seqlen, 10000.0) + torch.testing.assert_close(got, ref) + + +@pytest.mark.parametrize("ndim", [3, 4]) +def test_apply_rotary_matches_reference(ndim): + torch.manual_seed(0) + b, s, h, rd = 2, 5, 4, 64 + x = torch.randn(b, s, h, rd) if ndim == 4 else torch.randn(b, s, rd) + fc = precompute_dspark_freqs_cis(rd, s) + got = apply_dspark_rotary(x, fc) + ref = _ref_apply_rotary_emb(x, fc) + torch.testing.assert_close(got, ref) + + +@pytest.mark.parametrize("ndim", [3, 4]) +def test_apply_rotary_inverse_roundtrip(ndim): + """De-rotation (inverse) must undo the forward rotation (property test).""" + torch.manual_seed(1) + b, s, h, rd = 2, 6, 3, 64 + x = torch.randn(b, s, h, rd) if ndim == 4 else torch.randn(b, s, rd) + fc = precompute_dspark_freqs_cis(rd, s) + roundtrip = apply_dspark_rotary(apply_dspark_rotary(x, fc), fc, inverse=True) + torch.testing.assert_close(roundtrip, x, rtol=1e-5, atol=1e-5) + + +def _make_attn_inputs(seed=0): + """Small synthetic DSpark attention inputs/weights (CPU bf16).""" + torch.manual_seed(seed) + dim, n_heads, head_dim, rd = 12, 4, 8, 4 + q_lora, o_lora, n_groups = 6, 5, 2 + window, block, start_pos = 8, 3, 5 + b = 2 + g = dict( + dim=dim, + n_heads=n_heads, + head_dim=head_dim, + rope_head_dim=rd, + q_lora=q_lora, + o_lora=o_lora, + n_groups=n_groups, + window=window, + block=block, + start_pos=start_pos, + b=b, + eps=1e-6, + softmax_scale=head_dim**-0.5, + ) + bf = torch.bfloat16 + g["x"] = torch.randn(b, block, dim, dtype=bf) + g["main_x"] = torch.randn(b, 1, dim, dtype=bf) + g["kv_cache0"] = torch.randn(b, window, head_dim, dtype=bf) + g["wq_a"] = torch.randn(q_lora, dim, dtype=bf) * 0.1 + g["wq_b"] = torch.randn(n_heads * head_dim, q_lora, dtype=bf) * 0.1 + g["wkv"] = torch.randn(head_dim, dim, dtype=bf) * 0.1 + g["wo_a"] = torch.randn(n_groups * o_lora, n_heads * head_dim // n_groups, dtype=bf) * 0.1 + g["wo_b"] = torch.randn(dim, n_groups * o_lora, dtype=bf) * 0.1 + g["q_norm"] = torch.ones(q_lora) + g["kv_norm"] = torch.ones(head_dim) + g["attn_sink"] = torch.randn(n_heads) + g["freqs"] = precompute_dspark_freqs_cis(rd, start_pos + 1 + block + 2) + return g + + +def _run(g): + return dspark_attention_forward( + g["x"], + g["main_x"], + g["start_pos"], + g["kv_cache0"], + wq_a=g["wq_a"], + q_norm_w=g["q_norm"], + wq_b=g["wq_b"], + wkv=g["wkv"], + kv_norm_w=g["kv_norm"], + wo_a=g["wo_a"], + wo_b=g["wo_b"], + attn_sink=g["attn_sink"], + n_heads=g["n_heads"], + head_dim=g["head_dim"], + rope_head_dim=g["rope_head_dim"], + n_groups=g["n_groups"], + o_lora_rank=g["o_lora"], + window_size=g["window"], + eps=g["eps"], + softmax_scale=g["softmax_scale"], + freqs_cis=g["freqs"], + ) + + +def test_attention_forward_shape_and_determinism(): + g = _make_attn_inputs() + o = _run(g) + assert tuple(o.shape) == (g["b"], g["block"], g["dim"]) + assert torch.isfinite(o.float()).all() + torch.testing.assert_close(o, _run(g)) # deterministic + + +def test_attention_forward_does_not_mutate_kv_cache(): + """The rolling window write must be functional (cache cloned, not mutated).""" + g = _make_attn_inputs() + before = g["kv_cache0"].clone() + _run(g) + torch.testing.assert_close(g["kv_cache0"], before) diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_cuda_graph.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_cuda_graph.py new file mode 100644 index 000000000000..f66a37834fab --- /dev/null +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_cuda_graph.py @@ -0,0 +1,252 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CUDA-graph-safety unit tests for the DSpark batched draft-attention path. + +The load-bearing invariant for the DSpark batched draft path (the default, used +whenever ``cuda_graph_config`` is set, since the one-engine drafter is captured in +the target's graph) is that the batched, sync-free primitives are **numerically +identical, per request**, to the validated scalar path — only the host-int +``start_pos`` and the per-request window indexing are tensorized. These tests assert +that equivalence on CPU (so they run in pre-merge CI without a GPU), plus a +GPU-gated capture+replay smoke test that proves the batched attention is actually +graph-capturable. +""" + +import pytest +import torch + +from tensorrt_llm._torch.models.dspark.attention import ( + apply_dspark_rotary, + apply_dspark_rotary_batched, + dspark_attention_forward, + dspark_attention_forward_batched, + get_dspark_topk_idxs, + get_dspark_topk_idxs_batched, + precompute_dspark_freqs_cis, +) + + +def _make_batched_inputs(seed=0, start_positions=(1, 3, 20)): + """Per-request DSpark attention inputs/weights (shared weights, distinct pos). + + Mirrors ``test_dspark_attention._make_attn_inputs`` but builds ``G`` requests + each with its own ``start_pos`` (small => partial context, large => full + rolling window) and its own pre-seeded window, so the batched-vs-scalar + comparison exercises the per-request RoPE gather + windowed context read. + """ + torch.manual_seed(seed) + dim, n_heads, head_dim, rd = 12, 4, 8, 4 + q_lora, o_lora, n_groups = 6, 5, 2 + window, block = 8, 3 + G = len(start_positions) + bf = torch.bfloat16 + # A single fixed RoPE table covering every request's positions (both paths + # index/gather the same values, so freqs are identical across paths). + maxlen = max(start_positions) + 1 + block + 4 + g = dict( + dim=dim, + n_heads=n_heads, + head_dim=head_dim, + rope_head_dim=rd, + q_lora=q_lora, + o_lora=o_lora, + n_groups=n_groups, + window=window, + block=block, + G=G, + start_positions=list(start_positions), + eps=1e-6, + softmax_scale=head_dim**-0.5, + x=torch.randn(G, block, dim, dtype=bf), + main_x=torch.randn(G, 1, dim, dtype=bf), + # Distinct, non-zero seeded window per request to exercise context reads. + kv_cache=torch.randn(G, window, head_dim, dtype=bf), + wq_a=torch.randn(q_lora, dim, dtype=bf) * 0.1, + wq_b=torch.randn(n_heads * head_dim, q_lora, dtype=bf) * 0.1, + wkv=torch.randn(head_dim, dim, dtype=bf) * 0.1, + wo_a=torch.randn(n_groups * o_lora, n_heads * head_dim // n_groups, dtype=bf) * 0.1, + wo_b=torch.randn(dim, n_groups * o_lora, dtype=bf) * 0.1, + q_norm=torch.ones(q_lora), + kv_norm=torch.ones(head_dim), + attn_sink=torch.randn(n_heads), + freqs=precompute_dspark_freqs_cis(rd, maxlen), + ) + return g + + +def _attn_kwargs(g): + return dict( + wq_a=g["wq_a"], + q_norm_w=g["q_norm"], + wq_b=g["wq_b"], + wkv=g["wkv"], + kv_norm_w=g["kv_norm"], + wo_a=g["wo_a"], + wo_b=g["wo_b"], + attn_sink=g["attn_sink"], + n_heads=g["n_heads"], + head_dim=g["head_dim"], + rope_head_dim=g["rope_head_dim"], + n_groups=g["n_groups"], + o_lora_rank=g["o_lora"], + window_size=g["window"], + eps=g["eps"], + softmax_scale=g["softmax_scale"], + freqs_cis=g["freqs"], + ) + + +def _scalar_reference(g): + """Run the validated scalar attention once per request and stack the outputs.""" + outs = [] + for i, sp in enumerate(g["start_positions"]): + out_i = dspark_attention_forward( + g["x"][i : i + 1], + g["main_x"][i : i + 1], + int(sp), + g["kv_cache"][i : i + 1].clone(), + **_attn_kwargs(g), + ) + outs.append(out_i) + return torch.cat(outs, dim=0) + + +def _batched(g, persist=False): + G = g["G"] + start_pos = torch.tensor(g["start_positions"], dtype=torch.long) + slots = torch.arange(G, dtype=torch.long) + return dspark_attention_forward_batched( + g["x"], + g["main_x"], + start_pos, + g["kv_cache"].clone(), + slots, + persist=persist, + **_attn_kwargs(g), + ) + + +@pytest.mark.parametrize("seed", [0, 1, 2]) +def test_batched_attention_matches_scalar_per_request(seed): + """Batched attention == per-request scalar attention at distinct start_pos. + + This is the invariant that lets the batched path replace the per-request loop + under CUDA graphs without changing draft quality / greedy parity. + """ + g = _make_batched_inputs(seed=seed) + ref = _scalar_reference(g) + got = _batched(g) + assert tuple(got.shape) == (g["G"], g["block"], g["dim"]) + torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-2) + + +def test_batched_attention_persist_writes_through_window(): + """persist=True writes main_kv into the shared window at start_pos%window.""" + g = _make_batched_inputs(seed=3) + G, win = g["G"], g["window"] + start_pos = torch.tensor(g["start_positions"], dtype=torch.long) + slots = torch.arange(G, dtype=torch.long) + cache = g["kv_cache"].clone() + before = cache.clone() + dspark_attention_forward_batched( + g["x"], g["main_x"], start_pos, cache, slots, persist=True, **_attn_kwargs(g) + ) + # Exactly the start_pos%win row of each request changed. + for i, sp in enumerate(g["start_positions"]): + changed = (cache[i] != before[i]).any(dim=-1) + expected = torch.zeros(win, dtype=torch.bool) + expected[sp % win] = True + assert torch.equal(changed, expected), f"req {i}: wrong window row written" + + +def test_batched_attention_no_persist_keeps_window(): + """persist=False must not mutate the caller's window (functional).""" + g = _make_batched_inputs(seed=4) + before = g["kv_cache"].clone() + _batched(g, persist=False) + torch.testing.assert_close(g["kv_cache"], before) + + +@pytest.mark.parametrize("start_positions", [(1, 3, 20), (5, 5, 5), (2, 7, 200)]) +def test_batched_topk_matches_scalar(start_positions): + """Fixed-size masked batched topk == scalar topk per request (valid slots).""" + window, block = 8, 3 + start_pos = torch.tensor(start_positions, dtype=torch.long) + batched = get_dspark_topk_idxs_batched(window, block, start_pos) + assert tuple(batched.shape) == (len(start_positions), block, window + block) + for i, sp in enumerate(start_positions): + scalar = get_dspark_topk_idxs(window, 1, block, int(sp))[0] # [block, topk_i] + # The batched row drops the -1-masked context slots to recover the scalar + # (variable-width) index list; both must then be identical. + for m in range(block): + valid = batched[i, m][batched[i, m] >= 0] + torch.testing.assert_close(valid, scalar[m].to(valid.dtype)) + + +@pytest.mark.parametrize("ndim", [3, 4]) +def test_batched_rotary_matches_scalar_per_row(ndim): + """apply_dspark_rotary_batched (per-row freqs) == scalar applied row by row.""" + torch.manual_seed(0) + G, s, h, rd = 3, 4, 2, 8 + x = torch.randn(G, s, h, rd) if ndim == 4 else torch.randn(G, s, rd) + table = precompute_dspark_freqs_cis(rd, 64) + # Per-row absolute start positions -> per-row freq windows. + starts = [1, 9, 30] + per_row = torch.stack([table[sp : sp + s] for sp in starts], dim=0) # [G, s, rd/2] + got = apply_dspark_rotary_batched(x, per_row) + for i, sp in enumerate(starts): + ref_i = apply_dspark_rotary(x[i : i + 1], table[sp : sp + s]) + torch.testing.assert_close(got[i : i + 1], ref_i) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA graph capture needs a GPU") +def test_batched_attention_cuda_graph_capture_replay(): + """The batched attention captures + replays and matches eager output. + + Proves the path is free of capture-illegal ops (host syncs, dynamic shapes). + """ + g = _make_batched_inputs(seed=0) + dev = "cuda" + G = g["G"] + start_pos = torch.tensor(g["start_positions"], dtype=torch.long, device=dev) + slots = torch.arange(G, dtype=torch.long, device=dev) + # Static input tensors the graph reads/writes. + x = g["x"].to(dev) + main_x = g["main_x"].to(dev) + cache = g["kv_cache"].to(dev) + kw = {k: (v.to(dev) if torch.is_tensor(v) else v) for k, v in _attn_kwargs(g).items()} + + def run(persist): + return dspark_attention_forward_batched( + x, main_x, start_pos, cache, slots, persist=persist, **kw + ) + + eager = run(persist=False) + + # Warmup (PyTorch CUDA-graph semantics) then capture on a non-persist call so + # the comparison isn't perturbed by the window write-through. + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(s): + for _ in range(3): + run(persist=False) + torch.cuda.current_stream().wait_stream(s) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = run(persist=False) + graph.replay() + torch.cuda.synchronize() + torch.testing.assert_close(out, eager, rtol=2e-2, atol=2e-2) diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_draft.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_draft.py new file mode 100644 index 000000000000..fed0da3a2803 --- /dev/null +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_draft.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the DSpark draft I/O proposal stage.""" + +import torch + +from tensorrt_llm._torch.models.dspark.draft import build_draft_input_ids, dspark_propose +from tensorrt_llm._torch.models.dspark.heads import DSparkConfidenceHead, build_markov_head + +VOCAB, HID, RANK, B, BLK = 257, 32, 16, 2, 5 +NOISE_ID = 199 + + +def test_build_draft_input_ids(): + bonus = torch.tensor([7, 9]) + ids = build_draft_input_ids(bonus, block_size=BLK, noise_token_id=NOISE_ID) + assert ids.shape == (B, BLK) + assert torch.equal(ids[:, 0], bonus) + assert torch.all(ids[:, 1:] == NOISE_ID) + + +def test_dspark_propose_full_block_no_confidence(): + torch.manual_seed(0) + markov = build_markov_head( + markov_head_type="rnn", vocab_size=VOCAB, markov_rank=RANK, hidden_size=HID + ).eval() + base = torch.randn(B, BLK, VOCAB) + bonus = torch.randint(0, VOCAB, (B,)) + hid = torch.randn(B, BLK, HID) + with torch.no_grad(): + tokens, num = dspark_propose( + base, + bonus_token_ids=bonus, + block_hidden=hid, + markov_head=markov, + confidence_head=None, + block_size=BLK, + ) + assert tokens.shape == (B, BLK) + # No confidence head -> propose the full block. + assert torch.all(num == BLK) + # Tokens match the markov head's own greedy block sampling. + ref_tokens, _ = markov.sample_block_tokens( + base, first_prev_token_ids=bonus, hidden_states=hid, temperature=0.0 + ) + assert torch.equal(tokens, ref_tokens) + + +def test_dspark_propose_confidence_truncates(): + torch.manual_seed(1) + markov = build_markov_head( + markov_head_type="vanilla", vocab_size=VOCAB, markov_rank=RANK, hidden_size=HID + ).eval() + conf = DSparkConfidenceHead(hidden_size=HID).eval() + # The confidence proj is bias-free, so drive the logit via a constant weight + # against a constant hidden: logit = weight_val * HID per position. + base = torch.randn(1, BLK, VOCAB) + bonus = torch.randint(0, VOCAB, (1,)) + hid = torch.ones(1, BLK, HID) + with torch.no_grad(): + conf.proj.weight.fill_(5.0 / HID) # logit ~ 5 -> sigmoid ~ 0.993, all confident + with torch.no_grad(): + _, num = dspark_propose( + base, + bonus_token_ids=bonus, + block_hidden=hid, + markov_head=markov, + confidence_head=conf, + block_size=BLK, + confidence_threshold=0.5, + ) + # All-confident -> full block proposed. + assert int(num[0]) == BLK + # Now make the head output low confidence everywhere -> truncate to 0... but + # confident_prefix_length returns first sub-threshold index (0 here). + with torch.no_grad(): + conf.proj.weight.fill_(-5.0 / HID) # logit ~ -5 -> sigmoid ~ 0.0067 < 0.5 + _, num2 = dspark_propose( + base, + bonus_token_ids=bonus, + block_hidden=hid, + markov_head=markov, + confidence_head=conf, + block_size=BLK, + confidence_threshold=0.5, + ) + assert int(num2[0]) == 0 diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_eplb_config.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_eplb_config.py new file mode 100644 index 000000000000..c59c2c5909ae --- /dev/null +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_eplb_config.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""EPLB wiring for the one-model DSpark drafter (CPU-only, no weights). + +Covers the three P0 guarantees: + 1. the target's ``moe_load_balancer`` reaches the DSpark draft config -- and + ONLY DSpark's (PARD / DFlash / draft-target must be untouched); + 2. an ``initial_global_assignments`` map that predates DSpark fails early with + every missing stage index listed, not a bare ``KeyError``; + 3. online EPLB is rejected at config time instead of deadlocking at runtime. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from tensorrt_llm._torch.models import modeling_dspark +from tensorrt_llm._torch.models.modeling_dspark import ( + validate_dspark_eplb_layer_base, + validate_dspark_eplb_stage_layers, +) +from tensorrt_llm._torch.models.modeling_speculative import external_drafter_config_kwargs +from tensorrt_llm._torch.speculative.interface import SpeculativeDecodingMode +from tensorrt_llm.llmapi.llm_args import MoeLoadBalancerConfig + +NUM_HIDDEN_LAYERS = 61 +NUM_STAGES = 3 +NUM_EXPERTS = 384 +# DSpark stages register as EPLB layers 61, 62, 63. +DSPARK_LAYERS = list(range(NUM_HIDDEN_LAYERS, NUM_HIDDEN_LAYERS + NUM_STAGES)) + + +def _assignments(layer_ids): + """A structurally valid placement (one permutation of the experts) per layer.""" + return {layer_id: list(range(NUM_EXPERTS)) for layer_id in layer_ids} + + +def _lb_config(layer_ids=None, layer_updates_per_iter=0): + return MoeLoadBalancerConfig( + num_slots=NUM_EXPERTS, + initial_global_assignments=(_assignments(layer_ids) if layer_ids is not None else None), + layer_updates_per_iter=layer_updates_per_iter, + ) + + +def _model_config(lb_config=None, num_hidden_layers=NUM_HIDDEN_LAYERS): + return SimpleNamespace( + moe_load_balancer=lb_config, + attn_backend="TRTLLM", + moe_backend="CUTLASS", + mapping=object(), + max_num_tokens=8192, + moe_max_num_tokens=8192, + pretrained_config=SimpleNamespace(num_hidden_layers=num_hidden_layers), + ) + + +def _spec_config(mode): + return SimpleNamespace(spec_dec_mode=mode) + + +@pytest.fixture +def eplb_active(): + """Pretend an engine-wide MoeLoadBalancer is live during model construction.""" + with patch.object(modeling_dspark, "_active_moe_load_balancer", return_value=object()): + yield + + +# -------------------------------------------------------------------------- +# 1. config propagation -- DSpark only +# -------------------------------------------------------------------------- + + +def test_dspark_draft_config_inherits_load_balancer(): + lb_config = _lb_config(DSPARK_LAYERS) + kwargs = external_drafter_config_kwargs( + _model_config(lb_config), _spec_config(SpeculativeDecodingMode.DSPARK) + ) + # The very same object, so MoeLoadBalancerConfig.setup() done on the target + # side is already visible to the draft. + assert kwargs["moe_load_balancer"] is lb_config + + +def test_dspark_draft_config_does_not_recurse_into_spec_dec(): + # Regression guard: propagating moe_load_balancer must not tempt anyone into + # also forwarding spec_config, which would recursively build a drafter. + kwargs = external_drafter_config_kwargs( + _model_config(_lb_config(DSPARK_LAYERS)), _spec_config(SpeculativeDecodingMode.DSPARK) + ) + assert kwargs["spec_config"] is None + + +@pytest.mark.parametrize( + "mode", + [ + SpeculativeDecodingMode.PARD, + SpeculativeDecodingMode.DFLASH, + SpeculativeDecodingMode.DRAFT_TARGET_ONE_MODEL, + ], + # SpeculativeDecodingMode is an IntEnum, so the default ids would be the raw + # numbers -- name the cases so a CI failure says which drafter regressed. + ids=["pard", "dflash", "draft_target_one_model"], +) +def test_non_dspark_external_drafters_do_not_inherit_load_balancer(mode): + # These are independent checkpoints whose expert topology and layer-index + # namespace need not match the target's. Keep the DSpark fix from being + # silently generalized by a future refactor. + kwargs = external_drafter_config_kwargs( + _model_config(_lb_config(DSPARK_LAYERS)), _spec_config(mode) + ) + assert "moe_load_balancer" not in kwargs + + +def test_external_drafter_kwargs_are_stable_across_modes(): + common = external_drafter_config_kwargs( + _model_config(_lb_config(DSPARK_LAYERS)), _spec_config(SpeculativeDecodingMode.PARD) + ) + dspark = external_drafter_config_kwargs( + _model_config(_lb_config(DSPARK_LAYERS)), _spec_config(SpeculativeDecodingMode.DSPARK) + ) + assert set(dspark) - set(common) == {"moe_load_balancer"} + + +# -------------------------------------------------------------------------- +# 2. stage-layer placement coverage +# -------------------------------------------------------------------------- + + +def test_complete_assignments_accepted(eplb_active): + lb_config = _lb_config(list(range(NUM_HIDDEN_LAYERS)) + DSPARK_LAYERS) + validate_dspark_eplb_stage_layers(_model_config(lb_config), NUM_HIDDEN_LAYERS, NUM_STAGES) + + +def test_missing_stage_layers_listed_in_one_error(eplb_active): + # A config generated back when the drafter was 1-layer MTP: covers layer 61 + # (the old MTP layer) but not the two extra DSpark stages. + lb_config = _lb_config(list(range(NUM_HIDDEN_LAYERS + 1))) + with pytest.raises(ValueError) as excinfo: + validate_dspark_eplb_stage_layers(_model_config(lb_config), NUM_HIDDEN_LAYERS, NUM_STAGES) + message = str(excinfo.value) + assert "[62, 63]" in message + assert "61" not in message.split("missing DSpark layer(s)")[1].split("]")[0] + + +def test_auto_placement_needs_no_assignments(eplb_active): + # initial_global_assignments omitted -> auto-generated placement covers every + # registered layer, including the DSpark stages. + validate_dspark_eplb_stage_layers( + _model_config(_lb_config(None)), NUM_HIDDEN_LAYERS, NUM_STAGES + ) + + +def test_no_validation_when_eplb_inactive(): + # Without an active balancer the config is never consumed, so an incomplete + # (or online) one must not break the non-EPLB DSpark path. + lb_config = _lb_config(list(range(NUM_HIDDEN_LAYERS)), layer_updates_per_iter=4) + with patch.object(modeling_dspark, "_active_moe_load_balancer", return_value=None): + validate_dspark_eplb_stage_layers(_model_config(lb_config), NUM_HIDDEN_LAYERS, NUM_STAGES) + + +# -------------------------------------------------------------------------- +# 3. online EPLB rejected at config time +# -------------------------------------------------------------------------- + + +def test_online_eplb_rejected(eplb_active): + lb_config = _lb_config(list(range(NUM_HIDDEN_LAYERS)) + DSPARK_LAYERS, layer_updates_per_iter=2) + with pytest.raises(ValueError, match="static EPLB only"): + validate_dspark_eplb_stage_layers(_model_config(lb_config), NUM_HIDDEN_LAYERS, NUM_STAGES) + + +# -------------------------------------------------------------------------- +# 4. draft/target layer namespace must line up +# -------------------------------------------------------------------------- + + +def test_layer_base_mismatch_rejected(eplb_active): + with pytest.raises(ValueError, match="num_hidden_layers"): + validate_dspark_eplb_layer_base( + _model_config(_lb_config(DSPARK_LAYERS)), + _model_config(_lb_config(DSPARK_LAYERS), num_hidden_layers=3), + ) + + +def test_layer_base_match_accepted(eplb_active): + validate_dspark_eplb_layer_base( + _model_config(_lb_config(DSPARK_LAYERS)), _model_config(_lb_config(DSPARK_LAYERS)) + ) + + +def test_layer_base_not_checked_without_eplb(): + # A draft-only checkpoint config with its own depth stays valid when EPLB is + # off -- the layer namespace only has to line up for EPLB placement keys. + with patch.object(modeling_dspark, "_active_moe_load_balancer", return_value=None): + validate_dspark_eplb_layer_base( + _model_config(None), _model_config(None, num_hidden_layers=3) + ) diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_heads.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_heads.py new file mode 100644 index 000000000000..16b6a01acd17 --- /dev/null +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_heads.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the DSpark draft-network heads (hardware-agnostic, CPU).""" + +import pytest +import torch + +from tensorrt_llm._torch.models.dspark.heads import ( + DSparkConfidenceHead, + RNNHead, + VanillaMarkov, + build_markov_head, + confident_prefix_length, +) + +VOCAB, RANK, HID, B, BLK = 257, 16, 32, 3, 5 + + +@pytest.mark.parametrize("head_type", ["vanilla", "gated", "rnn"]) +def test_markov_block_sampling_shapes_and_determinism(head_type): + torch.manual_seed(0) + head = build_markov_head( + markov_head_type=head_type, vocab_size=VOCAB, markov_rank=RANK, hidden_size=HID + ).eval() + base = torch.randn(B, BLK, VOCAB) + first = torch.randint(0, VOCAB, (B,)) + hid = torch.randn(B, BLK, HID) + with torch.no_grad(): + tok, logits = head.sample_block_tokens( + base, first_prev_token_ids=first, hidden_states=hid, temperature=0.0 + ) + tok2, _ = head.sample_block_tokens( + base, first_prev_token_ids=first, hidden_states=hid, temperature=0.0 + ) + assert tok.shape == (B, BLK) + assert logits.shape == (B, BLK, VOCAB) + # Greedy is deterministic. + assert torch.equal(tok, tok2) + # Each sampled token is the argmax of its (bias-corrected) step logits. + assert torch.equal(tok, logits.argmax(dim=-1)) + + +def test_markov_bias_is_additive_low_rank(): + # bias = W2(W1[token]); the corrected first-step logits == base + bias. + torch.manual_seed(1) + head = VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK).eval() + base = torch.randn(B, BLK, VOCAB) + first = torch.randint(0, VOCAB, (B,)) + with torch.no_grad(): + _, corrected = head.sample_block_tokens( + base, first_prev_token_ids=first, hidden_states=None, temperature=0.0 + ) + expected0 = base[:, 0] + head.markov_w2(head.markov_w1(first)) + assert torch.allclose(corrected[:, 0], expected0, atol=1e-5) + + +def test_rnn_state_carries_across_positions(): + torch.manual_seed(2) + head = RNNHead(vocab_size=VOCAB, markov_rank=RANK, hidden_size=HID).eval() + initial_state = torch.zeros(1, RANK) + prev_embedding = head.get_prev_embeddings(torch.zeros(1, dtype=torch.long)) + prefix_hidden = torch.randn(1, HID) + current_hidden = torch.randn(1, HID) + + with torch.no_grad(): + state_a, _ = head._rnn_step(initial_state, prev_embedding, prefix_hidden) + state_b, _ = head._rnn_step(initial_state, prev_embedding, -prefix_hidden) + _, bias_a = head._rnn_step(state_a, prev_embedding, current_hidden) + _, bias_b = head._rnn_step(state_b, prev_embedding, current_hidden) + + assert not torch.allclose(state_a, state_b) + assert not torch.allclose(bias_a, bias_b) + + +def test_build_markov_head_rank_zero_returns_none(): + assert ( + build_markov_head( + markov_head_type="vanilla", vocab_size=VOCAB, markov_rank=0, hidden_size=HID + ) + is None + ) + + +def test_confidence_head_and_prefix_truncation(): + head = DSparkConfidenceHead(hidden_size=HID) + conf = head(torch.randn(B, BLK, HID)) + assert conf.shape == (B, BLK) + # threshold 0 disables truncation. + assert confident_prefix_length(conf, block_size=BLK, threshold=0.0) == BLK + # First sub-threshold position truncates the prefix. + logits = torch.tensor([[10.0, 10.0, -10.0, 10.0, 10.0]]) + assert confident_prefix_length(logits, block_size=BLK, threshold=0.5) == 2 + # All-confident -> full block. + logits_hi = torch.full((1, BLK), 10.0) + assert confident_prefix_length(logits_hi, block_size=BLK, threshold=0.5) == BLK + + +def test_confidence_head_with_markov_concat_dim(): + head = DSparkConfidenceHead(hidden_size=HID, markov_rank=RANK, with_markov=True) + hid = torch.randn(B, BLK, HID) + prev_emb = torch.randn(B, BLK, RANK) + out = head(hid, prev_embeddings=prev_emb) + assert out.shape == (B, BLK) + with pytest.raises(AssertionError): + head(hid) # with_markov requires prev_embeddings diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_worker.py b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_worker.py new file mode 100644 index 000000000000..4625d1b0115a --- /dev/null +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_dspark_worker.py @@ -0,0 +1,456 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GPU unit tests for the DSpark worker and metadata plumbing. + +Covers the framework-side logic that does NOT need the full draft model: +``DSparkSpecMetadata`` hidden-state capture (incl. the mHC hc-mean reduction) +and ``DSparkWorker`` slot / rolling-KV-window management. The end-to-end block +draft and acceptance path is covered by the DSpark test in +``integration/defs/accuracy/test_llm_api_pytorch.py``. +""" + +import types + +import pytest +import torch + +from tensorrt_llm._torch.speculative.dspark import DSparkSpecMetadata, DSparkWorker +from tensorrt_llm._torch.speculative.interface import SpeculativeDecodingMode + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available(), reason="DSpark metadata/worker allocate CUDA buffers" +) + +HIDDEN = 128 +NCAP = 3 +HC_MULT = 4 + + +def _make_metadata(max_num_requests=8, max_num_tokens=64, layers=(58, 59, 60)): + return DSparkSpecMetadata( + max_draft_len=5, + max_total_draft_tokens=5, + spec_dec_mode=SpeculativeDecodingMode.DSPARK, + max_num_requests=max_num_requests, + layers_to_capture=list(layers), + hidden_size=HIDDEN, + max_num_tokens=max_num_tokens, + dtype=torch.bfloat16, + ) + + +def test_metadata_buffer_and_layer_lookup(): + meta = _make_metadata() + assert meta.num_capture_layers == NCAP + assert meta.captured_hidden_states.shape == (64, HIDDEN * NCAP) + # sorted, O(1) membership + assert meta.is_layer_capture(58) and meta.is_layer_capture(60) + assert not meta.is_layer_capture(0) and not meta.is_layer_capture(61) + + +def test_metadata_capture_plain_hidden(): + """A [num_tokens, hidden] capture is stored at the layer's slice as-is.""" + meta = _make_metadata() + hs = torch.randn(4, HIDDEN, device="cuda", dtype=torch.bfloat16) + meta.maybe_capture_hidden_states(59, hs) # layer 59 -> capture index 1 + got = meta.get_hidden_states(4) + assert torch.equal(got[:, HIDDEN : 2 * HIDDEN], hs) + + +def test_metadata_capture_hc_mean_reduction(): + """A flattened mHC residual [N, hc_mult*hidden] is reduced by mean over hc.""" + meta = _make_metadata() + mhc = torch.randn(4, HC_MULT * HIDDEN, device="cuda", dtype=torch.bfloat16) + meta.maybe_capture_hidden_states(58, mhc) # layer 58 -> capture index 0 + expected = mhc.reshape(4, HC_MULT, HIDDEN).mean(dim=1) + got = meta.get_hidden_states(4) + assert torch.equal(got[:, 0:HIDDEN], expected) + + +def test_metadata_no_capture_for_unlisted_layer(): + meta = _make_metadata() + meta.captured_hidden_states.zero_() + meta.maybe_capture_hidden_states(10, torch.randn(4, HIDDEN, device="cuda")) + assert torch.count_nonzero(meta.get_hidden_states(4)) == 0 + + +def test_metadata_prepare_batch_indices(): + meta = _make_metadata() + meta.request_ids = [7, 3, 5] + meta.prepare() + assert meta.batch_indices_cuda[:3].tolist() == [0, 1, 2] + + +def _make_worker(): + cfg = types.SimpleNamespace( + max_draft_len=5, + spec_dec_mode=SpeculativeDecodingMode.DSPARK, + confidence_threshold=0.5, + ) + from tensorrt_llm.mapping import Mapping + + return DSparkWorker(cfg, Mapping()) + + +def _fake_draft_model(num_stages=3, window_size=128, head_dim=64): + return types.SimpleNamespace( + num_stages=num_stages, + block_size=5, + _attn_params={"window_size": window_size, "head_dim": head_dim}, + ) + + +def test_worker_lazy_init_window_buffers(): + worker = _make_worker() + dm = _fake_draft_model(num_stages=3, window_size=128, head_dim=64) + meta = _make_metadata(max_num_requests=8) + worker._lazy_init(dm, meta) + # max_batch (8) request slots + 1 scratch row for padded / unknown IDs. + assert worker._kv_windows.shape == (9, 3, 128, 64) + assert worker._ctx_len.shape == (9,) + assert worker._scratch_slot == 8 + # Dummy-id floor separates real request ids from CUDA-graph padding ids. + assert worker._graph_dummy_id_floor == (1 << 64) - 1 - worker.max_draft_len + # The scratch row is never handed out through the free pool. + assert list(worker._free_slots) == list(range(8)) + assert worker._batch_to_slot is not None + assert worker._batch_to_slot.shape == (8,) + assert worker._batch_to_slot.device.type == "cuda" + # idempotent + buf_id = id(worker._kv_windows) + worker._lazy_init(dm, meta) + assert id(worker._kv_windows) == buf_id + + +def test_worker_rejects_mismatched_block_size(): + worker = _make_worker() + draft_model = _fake_draft_model() + draft_model.block_size = 4 + + with pytest.raises(ValueError, match="block_size must equal worker max_draft_len"): + worker._lazy_init(draft_model, _make_metadata()) + + +def test_worker_slot_assignment_and_reset(): + worker = _make_worker() + worker._lazy_init(_fake_draft_model(), _make_metadata(max_num_requests=4)) + + s0 = worker._assign_slot(100, reset=False) + s1 = worker._assign_slot(101, reset=False) + assert s0 != s1 + # same request id -> same slot (no reset) + assert worker._assign_slot(100, reset=False) == s0 + + # mark a position, then reset -> slot freed + window/pos cleared + worker._ctx_len[s0] = 42 + worker._kv_windows[s0].fill_(1.0) + s0b = worker._assign_slot(100, reset=True) + assert int(worker._ctx_len[s0b]) == 0 + assert float(worker._kv_windows[s0b].abs().sum()) == 0.0 + + +def test_worker_slot_exhaustion_preserves_live_request(): + worker = _make_worker() + worker._lazy_init(_fake_draft_model(), _make_metadata(max_num_requests=1)) + + slot = worker._assign_slot(100, reset=False) + worker._ctx_len[slot] = 42 + worker._kv_windows[slot].fill_(1.0) + + with pytest.raises(RuntimeError, match="no free rolling-window slots"): + worker._assign_slot(101, reset=False) + + assert worker._req_to_slot == {100: slot} + assert int(worker._ctx_len[slot]) == 42 + assert torch.all(worker._kv_windows[slot] == 1.0) + + +def test_seed_context_windows_preserves_state_across_prefill_chunks(): + class DraftModel: + num_stages = 1 + block_size = 5 + _attn_params = {"window_size": 8, "head_dim": 4} + + def __init__(self): + self.written_positions = [] + + def write_context_windows(self, hidden, positions, windows): + self.written_positions.append(positions.clone()) + windows.add_(1) + + worker = _make_worker() + draft_model = DraftModel() + metadata = types.SimpleNamespace( + max_num_requests=1, + request_ids=[100], + get_hidden_states=lambda _num_tokens: torch.zeros( + 3, HIDDEN * NCAP, device="cuda", dtype=torch.bfloat16 + ), + ) + worker._lazy_init(draft_model, metadata) + + first_chunk = types.SimpleNamespace(num_contexts=1, _seq_lens=[3]) + worker._seed_context_windows( + draft_model, metadata, first_chunk, torch.tensor([[0, 1, 2]], device="cuda"), 3 + ) + slot = worker._req_to_slot[100] + assert int(worker._ctx_len[slot]) == 3 + + metadata.get_hidden_states = lambda _num_tokens: torch.zeros( + 2, HIDDEN * NCAP, device="cuda", dtype=torch.bfloat16 + ) + second_chunk = types.SimpleNamespace(num_contexts=1, _seq_lens=[2]) + worker._seed_context_windows( + draft_model, metadata, second_chunk, torch.tensor([[3, 4]], device="cuda"), 2 + ) + + assert int(worker._ctx_len[slot]) == 5 + assert [positions.tolist() for positions in draft_model.written_positions] == [ + [1, 2, 3], + [4, 5], + ] + assert torch.all(worker._kv_windows[slot] == 2.0) + + +def test_prepare_builds_batch_to_slot_on_batched_path(): + """prepare() mirrors the host slot map into _batch_to_slot (default batched path).""" + worker = _make_worker() + meta = _make_metadata(max_num_requests=4) + worker._lazy_init(_fake_draft_model(), meta) # batched is the default + meta._dspark_worker = worker + + # Assign slots for two requests (as the prefill path would). + sa = worker._assign_slot(100, reset=True) + sb = worker._assign_slot(101, reset=True) + + meta.request_ids = [101, 100] + meta.prepare() + # Mirror reflects request-order -> slot. + assert worker._batch_to_slot[:2].tolist() == [sb, sa] + + +def test_prepare_frees_stale_slots_on_batched_path(): + """A request that drops out of the batch returns its slot to the free pool.""" + worker = _make_worker() + meta = _make_metadata(max_num_requests=4) + worker._lazy_init(_fake_draft_model(), meta) + meta._dspark_worker = worker + + sa = worker._assign_slot(100, reset=True) + worker._assign_slot(101, reset=True) + worker._ctx_len[sa] = 17 + + # Only request 101 survives; 100's slot must be freed + cleared. + meta.request_ids = [101] + meta.prepare() + assert 100 not in worker._req_to_slot + assert sa in worker._free_slots + assert int(worker._ctx_len[sa]) == 0 + + +def test_prepare_maps_unknown_request_to_scratch_row_not_slot_zero(): + """Padded / unknown request IDs route to the scratch row, never a live slot. + + Regression for the rolling-window aliasing bug (GitHub #16767): an unknown + request id (CUDA-graph padding, ADP idle request, or a disagg seed forward + without a real id) must not overwrite the request that owns slot 0. + """ + worker = _make_worker() + meta = _make_metadata(max_num_requests=4) + worker._lazy_init(_fake_draft_model(), meta) + meta._dspark_worker = worker + + # A live request takes the first free slot (0) and populates its window. + s_real = worker._assign_slot(100, reset=True) + assert s_real == 0 + worker._ctx_len[s_real] = 17 + worker._kv_windows[s_real].fill_(1.0) + + # Batch contains the live request plus an unknown id (e.g. graph padding). + meta.request_ids = [100, 999] + meta.prepare() + + # The unknown id maps to the scratch row, not to slot 0. + assert worker._batch_to_slot[:2].tolist() == [s_real, worker._scratch_slot] + assert worker._scratch_slot != s_real + # It is not silently registered as a real request and did not consume a slot. + assert 999 not in worker._req_to_slot + assert list(worker._free_slots) == [1, 2, 3] + # The live request's rolling window and position are untouched. + assert int(worker._ctx_len[s_real]) == 17 + assert torch.all(worker._kv_windows[s_real] == 1.0) + + +def test_prepare_assigns_slots_to_disagg_generation_requests(): + """Disagg gen requests (never seeded here) get distinct slots, not one shared row. + + Regression for GitHub #16767: on the disaggregated generation server the + prompt is prefilled (and the DSpark window seeded) on the *context* server, + so ``_seed_context_windows`` never runs here and ``_req_to_slot`` stays empty. + ``prepare()`` must therefore assign each real generation request its own + rolling-window slot instead of collapsing them all onto the shared scratch + row (which corrupts drafts and collapses accept length at batch size > 1). + """ + worker = _make_worker() + meta = _make_metadata(max_num_requests=4) + worker._lazy_init(_fake_draft_model(), meta) + meta._dspark_worker = worker + + # All-generation batch (num_contexts == 0), no prior seeding. + meta.request_ids = [1000, 1001] + meta.num_generations = 2 + meta.prepare() + + s0 = worker._req_to_slot[1000] + s1 = worker._req_to_slot[1001] + assert s0 != s1 + assert s0 != worker._scratch_slot and s1 != worker._scratch_slot + assert worker._batch_to_slot[:2].tolist() == [s0, s1] + + # Stable across steps: the same ids keep their slots (no churn / reassignment). + meta.prepare() + assert worker._req_to_slot[1000] == s0 + assert worker._req_to_slot[1001] == s1 + assert worker._batch_to_slot[:2].tolist() == [s0, s1] + + +def test_prepare_keeps_dummy_generation_requests_on_scratch_row(): + """ADP-idle (id 0) and CUDA-graph padding dummies never consume a real slot.""" + from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import CUDA_GRAPH_DUMMY_REQUEST_ID + from tensorrt_llm._torch.pyexecutor.llm_request import ATTENTION_DP_DUMMY_REQUEST_ID + + worker = _make_worker() + meta = _make_metadata(max_num_requests=4) + worker._lazy_init(_fake_draft_model(), meta) + meta._dspark_worker = worker + + graph_dummy = CUDA_GRAPH_DUMMY_REQUEST_ID - worker.max_draft_len + meta.request_ids = [1000, ATTENTION_DP_DUMMY_REQUEST_ID, graph_dummy] + meta.num_generations = 3 + meta.prepare() + + s_real = worker._req_to_slot[1000] + assert s_real != worker._scratch_slot + # Dummies are neither registered nor given a real slot; they map to scratch. + assert ATTENTION_DP_DUMMY_REQUEST_ID not in worker._req_to_slot + assert graph_dummy not in worker._req_to_slot + assert worker._batch_to_slot[:3].tolist() == [ + s_real, + worker._scratch_slot, + worker._scratch_slot, + ] + # Exactly one real slot consumed. + assert list(worker._free_slots) == [1, 2, 3] + + +def test_forward_mixed_batch_routes_through_base_entries(monkeypatch): + """Mixed (context + gen) batch: ``forward`` must route acceptance and + production through the unified ``SpecWorkerBase`` entries, one-hot-fill the + context requests' draft-prob rows, and assemble + ``next_draft_tokens = [ctx zeros ; gen argmax]``. + + Spies replace the base sampling entries and the heavy sub-calls (context + seeding, per-request draft backbone) so this exercises the worker's + context/gen orchestration — the exact surface the #15775 refactor changed — + without a real draft model or MPI. + """ + worker = _make_worker() + worker.guided_decoder = None + dm = _fake_draft_model(num_stages=3, window_size=128, head_dim=64) + + K = worker.max_draft_len + vocab = 16 + num_contexts, num_gens = 2, 3 + batch_size = num_contexts + num_gens + + meta = _make_metadata(max_num_requests=8) + meta.request_ids = [10, 11, 20, 21, 22] # 2 context + 3 gen + meta.prepare() + + attn_metadata = types.SimpleNamespace( + num_seqs=batch_size, + num_contexts=num_contexts, + num_ctx_tokens=0, + num_tokens=batch_size, + ) + + # Acceptance: return a fixed verified prefix (one accepted token per request). + accepted = torch.arange(batch_size * (K + 1), dtype=torch.int32, device="cuda").reshape( + batch_size, K + 1 + ) + num_accepted = torch.ones(batch_size, dtype=torch.int32, device="cuda") + accept_calls = {} + + def fake_accept(logits, am, sm): + accept_calls["args"] = (am, sm) + return accepted, num_accepted + + monkeypatch.setattr(worker, "sample_and_accept_draft_tokens", fake_accept) + # Context-window seeding is covered by its own test; stub it out here. + monkeypatch.setattr(worker, "_seed_context_windows", lambda *a, **k: None) + + # The gen-block helper now returns the corrected block logits [num_gens,K,vocab]. + gen_logits = torch.randn(num_gens, K, vocab, device="cuda") + monkeypatch.setattr(worker, "_draft_gen_block_batched", lambda *a, **k: gen_logits) + + sdt_calls = {} + # The gen scatter publishes the FULL (post-TP-gather) vocab width, which is + # wider than the sharded gen_logits width (`vocab`). The worker must pass this + # published width (draft_probs_last_dim) to write_context_onehot_draft_probs, + # NOT gen_logits.shape[-1]. + FULL_VOCAB = 97 + + def fake_sample_draft_tokens(gl, sm, bs, *, num_contexts): + sdt_calls["logits"] = gl + sdt_calls["batch_size"] = bs + sdt_calls["num_contexts"] = num_contexts + sm.draft_probs_last_dim = FULL_VOCAB # simulate the full-vocab scatter + return gl.argmax(dim=-1).to(torch.int32) + + monkeypatch.setattr(worker, "sample_draft_tokens", fake_sample_draft_tokens) + + onehot_calls = {} + monkeypatch.setattr( + worker, + "write_context_onehot_draft_probs", + lambda sm, nc, ng, k, gv: onehot_calls.update(nc=nc, ng=ng, k=k, gv=gv), + ) + + input_ids = torch.zeros(batch_size, dtype=torch.long, device="cuda") + position_ids = torch.zeros(batch_size, dtype=torch.long, device="cuda") + hidden = torch.zeros(batch_size, HIDDEN, device="cuda", dtype=torch.bfloat16) + logits = torch.zeros(batch_size, vocab, device="cuda") + + out = worker.forward(input_ids, position_ids, hidden, logits, attn_metadata, meta, dm) + + # Acceptance went through the unified entry with the right metadata objects. + assert accept_calls["args"] == (attn_metadata, meta) + # Production fed the [num_gens, K, vocab] block logits to the base sampler, + # with num_contexts so it slices the gen segment. + assert sdt_calls["num_contexts"] == num_contexts + assert sdt_calls["logits"].shape == (num_gens, K, vocab) + # Context rows one-hot-filled with the *scatter* width (draft_probs_last_dim, + # FULL_VOCAB), not the sharded gen_logits width (vocab). + assert onehot_calls == {"nc": num_contexts, "ng": num_gens, "k": K, "gv": FULL_VOCAB} + + # next_draft_tokens = [context zeros ; gen argmax]; gen subset is not polluted + # by the context rows. + nd = out["next_draft_tokens"] + assert nd.shape == (batch_size, K) + assert torch.all(nd[:num_contexts] == 0) + assert torch.equal(nd[num_contexts:], gen_logits.argmax(dim=-1).to(torch.int32)) + # Verified tokens are surfaced unchanged. + assert torch.equal(out["new_tokens"], accepted) + assert torch.equal(out["new_tokens_lens"], num_accepted) diff --git a/tests/unittest/_torch/speculative/test_capture_override_leak.py b/tests/unittest/_torch/speculative/test_capture_override_leak.py new file mode 100644 index 000000000000..81e1331d1edd --- /dev/null +++ b/tests/unittest/_torch/speculative/test_capture_override_leak.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU unit tests for the lifetime of the capture-only sampling override. + +The advanced-sampling CUDA graph capture pass sets +``_force_non_greedy_for_capture=True`` on the live ``SpecMetadata`` so that +parameter-less warmup requests scan as non-greedy and the advanced-sampling +branch is the one recorded into the graph. + +``create_cuda_graph_metadata`` shallow-copies the live metadata, so every graph +captured during that pass caches a copy that inherited the flag, and those +copies are reseated as the live spec_metadata on every later replay. Clearing +the flag on the base object alone therefore leaves it set forever on the copies, +and ``_scan_one_model_sampling`` then rewrites EVERY serving request's sampling +params to the synthetic capture values. + +These tests use a real (base) ``SpecMetadata`` -- its ``__post_init__`` is a +no-op and none of the fields exercised here are tensors -- plus an unbound call +of ``CUDAGraphRunner.clear_capture_only_spec_state`` on a stand-in holding only +``graph_metadata``, mirroring test_group_all_greedy_sync.py. No GPU, no runner +construction, and no model forward is needed. +""" + +import types + +import torch + +from tensorrt_llm._torch.pyexecutor.cuda_graph_runner import CUDAGraphRunner +from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequestState +from tensorrt_llm._torch.speculative.interface import SpecMetadata + +# The synthetic params the capture override substitutes for the request's own, +# and the sentinel that means "top-k disabled" (see _scan_one_model_sampling). +CAPTURE_TEMPERATURE, CAPTURE_TOP_K, CAPTURE_TOP_P = 0.7, 50, 0.9 +DISABLE_TOPK_VAL = torch.iinfo(torch.int32).max + + +def _base_meta(): + """A live (non-graph) SpecMetadata, as the model engine holds it.""" + return SpecMetadata(max_num_requests=8, max_draft_len=1, max_total_draft_tokens=1) + + +def _graph_copy(meta, batch_size=8): + """The shallow copy maybe_get_cuda_graph caches for one captured graph.""" + graph_meta = meta.create_cuda_graph_metadata(batch_size) + assert graph_meta is not meta + return graph_meta + + +def _clear(graph_metadata): + """CUDAGraphRunner.clear_capture_only_spec_state, called unbound.""" + return CUDAGraphRunner.clear_capture_only_spec_state( + types.SimpleNamespace(graph_metadata=graph_metadata) + ) + + +def _request(temperature=None, top_k=None, top_p=None, slot=0): + return types.SimpleNamespace( + sampling_config=types.SimpleNamespace( + temperature=[temperature] if temperature is not None else None, + top_k=[top_k] if top_k is not None else None, + top_p=[top_p] if top_p is not None else None, + ), + state=LlmRequestState.GENERATION_IN_PROGRESS, + py_seq_slot=slot, + ) + + +def _scan(meta, requests): + normalized, _ = SpecMetadata._scan_one_model_sampling(meta, requests) + # Drop the trailing num_tokens; only the sampling params matter here. + return [entry[:3] for entry in normalized] + + +def test_graph_copy_inherits_flag_and_base_teardown_does_not_reach_it(): + # The mechanism the bug rests on: copy.copy carries the flag over, and the + # copies are independent objects, so clearing the base misses them. + meta = _base_meta() + meta._force_non_greedy_for_capture = True + copies = [_graph_copy(meta, bs) for bs in (1, 2, 4)] + assert all(copy._force_non_greedy_for_capture for copy in copies) + + meta._force_non_greedy_for_capture = False + assert all(copy._force_non_greedy_for_capture for copy in copies) + + +def test_clear_capture_only_spec_state_clears_every_cached_copy(): + meta = _base_meta() + meta._force_non_greedy_for_capture = True + advanced = [_graph_copy(meta, bs) for bs in (1, 2, 4)] + # Graphs captured by the greedy pass never had the flag, and non-spec + # graphs cache no spec_metadata at all; both must be left alone. + meta._force_non_greedy_for_capture = False + greedy = _graph_copy(meta, 8) + + graph_metadata = {("greedy", 8): {"spec_metadata": greedy}} + graph_metadata[("no_spec", 1)] = {"spec_metadata": None} + for i, copy in enumerate(advanced): + graph_metadata[("advanced", i)] = {"spec_metadata": copy} + + assert _clear(graph_metadata) == len(advanced) + assert not any(copy._force_non_greedy_for_capture for copy in advanced) + assert greedy._force_non_greedy_for_capture is False + # Idempotent: a second teardown finds nothing left to clear. + assert _clear(graph_metadata) == 0 + + +def test_serving_scan_honors_client_params_after_capture_teardown(): + # End-to-end property of the fix, and the case that fails without it: with + # only the base-object teardown this scan returns (0.7, 50, 0.9). + meta = _base_meta() + meta._force_non_greedy_for_capture = True + graph_meta = _graph_copy(meta) + + meta._force_non_greedy_for_capture = False # base-object teardown + _clear({("advanced", 8): {"spec_metadata": graph_meta}}) # the fix + + # Replay reseats the cached copy as the live spec_metadata, so the serving + # scan runs on it, not on the base object. + assert _scan(graph_meta, [_request(temperature=1.0, top_p=1.0)]) == [ + (1.0, DISABLE_TOPK_VAL, 1.0) + ] + assert graph_meta.is_all_greedy_sample is False + + +def test_override_stays_live_while_the_flag_is_set(): + # Anti-regression for the rejected "clear at copy time" fix: the flag is + # load-bearing *during* capture. Cleared any earlier, the pass-2 populate + # would scan these parameter-less warmup requests as greedy and bake the + # argmax fast path -- with no top-k/top-p kernels -- into the graph keyed + # as the advanced-sampling variant. + meta = _base_meta() + meta._force_non_greedy_for_capture = True + graph_meta = _graph_copy(meta) + + warmup_requests = [_request(slot=None), _request(slot=None)] + assert _scan(graph_meta, warmup_requests) == [ + (CAPTURE_TEMPERATURE, CAPTURE_TOP_K, CAPTURE_TOP_P) + ] * len(warmup_requests) + assert graph_meta.is_all_greedy_sample is False + assert not graph_meta.skip_temperature + assert not graph_meta.skip_top_k + assert not graph_meta.skip_top_p diff --git a/tests/unittest/_torch/speculative/test_eagle3.py b/tests/unittest/_torch/speculative/test_eagle3.py index 6298565205eb..33a3667d0db4 100644 --- a/tests/unittest/_torch/speculative/test_eagle3.py +++ b/tests/unittest/_torch/speculative/test_eagle3.py @@ -11,7 +11,8 @@ import torch from test_common.llm_data import with_mocked_hf_download_for_single_gpu from utils.llm_data import llm_models_root -from utils.util import skip_blackwell, skip_num_gpus_less_than +from utils.util import (skip_blackwell, skip_num_gpus_less_than, + skip_pre_blackwell) from tensorrt_llm import LLM, SamplingParams from tensorrt_llm._torch.attention_backend.trtllm import TrtllmAttentionMetadata @@ -21,14 +22,87 @@ from tensorrt_llm._torch.pyexecutor.py_executor_creator import \ _extend_full_attention_windows_for_spec_decode from tensorrt_llm._torch.speculative.eagle3 import Eagle3OneModelSpecMetadata +from tensorrt_llm._torch.speculative.mtp_dynamic_tree import \ + MTPEagleDynamicTreeWorker from tensorrt_llm.executor.request import LoRARequest from tensorrt_llm.llmapi import (CudaGraphConfig, Eagle3DecodingConfig, - KvCacheConfig) + KvCacheConfig, MoeConfig, MTPDecodingConfig) from tensorrt_llm.lora_helper import LoraConfig sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +def test_dynamic_tree_metadata_forces_target_mask_prepare_each_step() -> None: + metadata = TrtllmAttentionMetadata( + seq_lens=None, + seq_lens_kv=None, + num_contexts=0, + max_num_requests=1, + max_num_tokens=1, + max_seq_len=1, + ) + common_kwargs = dict( + batch_size=0, + is_spec_decoding_enabled=False, + is_spec_dec_tree=True, + max_draft_len=1, + max_total_draft_tokens=1, + ) + + metadata.update_spec_dec_param(is_spec_dec_dynamic_tree=True, + **common_kwargs) + assert metadata.force_prepare_spec_dec_tree_mask + + metadata.update_spec_dec_param(is_spec_dec_dynamic_tree=False, + **common_kwargs) + assert not metadata.force_prepare_spec_dec_tree_mask + + +def test_mtp_dynamic_tree_relocation_uses_full_attention_window( + monkeypatch: pytest.MonkeyPatch) -> None: + worker = object.__new__(MTPEagleDynamicTreeWorker) + worker._kv_head_dim_bytes = 256 + worker._accepted_draft_indices_tensor = torch.tensor([[0, 1], [2, -1]], + dtype=torch.int32) + worker._num_accepted_tokens_buf = torch.tensor([2, 1], dtype=torch.int32) + + attention_pool_pointers = object() + attention_block_offsets = object() + cache_manager = SimpleNamespace( + num_kv_heads_per_layer=[0, 8, 0, 8], + kv_cache_pool_mapping=[[0, 0], [2, 0], [1, 0], [2, 1]], + kv_cache_pool_pointers=[object(), + object(), attention_pool_pointers], + max_attention_window_vec=[None], + max_seq_len=8192, + max_total_draft_tokens=31, + max_blocks_per_seq=256, + tokens_per_block=32, + ) + attention_metadata = SimpleNamespace( + kv_cache_manager=cache_manager, + kv_lens_cuda=torch.tensor([128, 256], dtype=torch.int32), + kv_cache_block_offsets=[object(), + object(), attention_block_offsets], + ) + update_op = MagicMock() + monkeypatch.setattr( + torch.ops.tensorrt_llm, + "update_kv_cache_draft_token_location_2d", + update_op, + ) + + worker._relocate_kv_eagerly(attention_metadata, batch_size=2) + + update_op.assert_called_once() + args = update_op.call_args.args + assert args[4] == 2 + assert args[5] == 8 + assert args[8] == cache_manager.max_seq_len + assert args[9] is attention_pool_pointers + assert args[10] is attention_block_offsets + + def test_eagle3_draft_kv_cache_uses_full_window_when_draft_has_no_swa() -> None: kv_cache_config = KvCacheConfig(max_attention_window=[128, 131072]) draft_pretrained_config = SimpleNamespace(num_hidden_layers=3) @@ -207,6 +281,81 @@ def test_kv_lens_runtime_with_eagle3_one_model(): f"kv_lens should be {expected_kv_lens_with_extra.tolist()}, but got {kv_lens_internal.tolist()}" +def _make_mock_kv_cache_manager(num_seqs: int) -> MagicMock: + mock_kv_cache_manager = MagicMock() + mock_kv_cache_manager.tokens_per_block = 32 + mock_kv_cache_manager.num_pools = 1 + mock_kv_cache_manager.num_attention_op_pools = 1 + mock_kv_cache_manager.max_blocks_per_seq = 16 + mock_kv_cache_manager.max_batch_size = num_seqs + mock_kv_cache_manager.max_seq_len = 512 + mock_kv_cache_manager.copy_batch_block_offsets = MagicMock() + return mock_kv_cache_manager + + +@pytest.mark.parametrize("spec_signal", [ + None, "num_extra_kv_tokens", "is_spec_decoding_enabled", + "has_speculative_draft_tokens", "draft_kv_cache_manager" +]) +def test_block_offsets_staging_width_spec_gate(spec_signal): + """prepare() caps the staged block-table width by the batch's max KV + length only on the non-speculative path. + + Any speculative-decoding signal must disable the cap (max_blocks=None): + spec kernels address block columns past the host kv_lens snapshot + (device-side kv_lens advances in draft/tree sub-steps, draft-token blocks + are allocated ahead), so a host-derived cap leaves columns they + dereference unstaged. Regression test for the EAGLE3 warmup illegal + memory access. + """ + num_seqs = 3 + prompt_lens = [50, 100, 75] + seq_lens_q = [1, 1, 1] + num_cached_tokens_per_seq = [ + prompt_lens[i] - seq_lens_q[i] for i in range(num_seqs) + ] + + mock_kv_cache_manager = _make_mock_kv_cache_manager(num_seqs) + metadata_kwargs = dict( + max_num_requests=num_seqs, + max_num_tokens=sum(seq_lens_q), + kv_cache_manager=mock_kv_cache_manager, + ) + mock_draft_manager = None + if spec_signal == "draft_kv_cache_manager": + mock_draft_manager = _make_mock_kv_cache_manager(num_seqs) + metadata_kwargs["draft_kv_cache_manager"] = mock_draft_manager + + attn_metadata = TrtllmAttentionMetadata(**metadata_kwargs) + if spec_signal == "is_spec_decoding_enabled": + attn_metadata.is_spec_decoding_enabled = True + elif spec_signal == "has_speculative_draft_tokens": + attn_metadata.runtime_features.has_speculative_draft_tokens = True + + attn_metadata.request_ids = list(range(1, num_seqs + 1)) + attn_metadata.prompt_lens = prompt_lens + attn_metadata._seq_lens = torch.tensor(seq_lens_q, dtype=torch.int32) + attn_metadata._seq_lens_kv = torch.tensor(seq_lens_q, dtype=torch.int32) + attn_metadata.kv_cache_params = KVCacheParams( + use_cache=True, + num_cached_tokens_per_seq=num_cached_tokens_per_seq, + num_extra_kv_tokens=(7 if spec_signal == "num_extra_kv_tokens" else 0)) + + attn_metadata.prepare() + + if spec_signal is None: + # Non-speculative: capped at ceil(max kv len / tokens_per_block). + expected_max_blocks = -(-max(prompt_lens) // + mock_kv_cache_manager.tokens_per_block) + else: + expected_max_blocks = None + call_kwargs = mock_kv_cache_manager.copy_batch_block_offsets.call_args.kwargs + assert call_kwargs["max_blocks"] == expected_max_blocks + if mock_draft_manager is not None: + draft_kwargs = mock_draft_manager.copy_batch_block_offsets.call_args.kwargs + assert draft_kwargs["max_blocks"] is None + + @pytest.mark.parametrize( "use_cuda_graph,attn_backend,disable_overlap_scheduler,enable_block_reuse,use_one_model,enable_chunked_prefill,use_chain_drafter,multi_batch,attention_dp,use_hf_speculative_model", [ @@ -1002,6 +1151,84 @@ def test_llama_eagle3_rejection_sampling_modes(use_dynamic_tree: bool, assert len(results[0].outputs[0].token_ids) > 0 +@pytest.mark.parametrize("disable_overlap_scheduler", [False, True]) +@pytest.mark.parametrize("use_cuda_graph", [False, True]) +@pytest.mark.high_cuda_memory +@skip_pre_blackwell +def test_nemotron_super_mtp_dynamic_tree_dl6_k10_dt31( + use_cuda_graph: bool, disable_overlap_scheduler: bool): + if torch.cuda.device_count() < 8: + pytest.skip("Nemotron Super dynamic-tree MTP test requires 8 GPUs") + + models_path = llm_models_root() + model_path = f"{models_path}/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" + + max_batch_size = 1 + max_draft_len = 6 + kv_cache_config = KvCacheConfig(enable_block_reuse=False, + mamba_ssm_cache_dtype="float16", + free_gpu_memory_fraction=0.8) + cuda_graph_config = CudaGraphConfig( + batch_sizes=[1]) if use_cuda_graph else None + + llm_common_config = dict( + model=model_path, + tensor_parallel_size=8, + moe_expert_parallel_size=8, + pipeline_parallel_size=1, + moe_config=MoeConfig(backend="TRTLLM"), + disable_overlap_scheduler=disable_overlap_scheduler, + cuda_graph_config=cuda_graph_config, + max_batch_size=max_batch_size, + kv_cache_config=kv_cache_config, + max_seq_len=8192, + ) + spec_config = MTPDecodingConfig(max_draft_len=max_draft_len, + mtp_eagle_one_model=True, + use_dynamic_tree=True, + dynamic_tree_max_topK=10, + max_total_draft_tokens=31) + + llm_spec = LLM(**llm_common_config, speculative_config=spec_config) + prompt = llm_spec.tokenizer.apply_chat_template( + [{ + "role": "user", + "content": "The future of AI is" + }], + tokenize=False, + add_generation_prompt=True, + ) + tok_ids = llm_spec.tokenizer.encode(prompt) + + sampling_params = SamplingParams(max_tokens=128, temperature=0) + num_tokens = 0 + num_drafted = 0 + num_accepted = 0 + for output in llm_spec.generate_async(tok_ids, + sampling_params, + streaming=True): + new_tokens = output.outputs[0].token_ids + num_drafted += max_draft_len + num_accepted += len(new_tokens) - num_tokens - 1 + num_tokens = len(new_tokens) + + accept_rate = num_accepted / num_drafted + assert accept_rate > 0.20 + + sampling_params = SamplingParams(max_tokens=10, temperature=0) + results_spec = llm_spec.generate([prompt], sampling_params) + generated_text_spec = [result.outputs[0].text for result in results_spec] + llm_spec.shutdown() + + llm_ref = LLM(**llm_common_config) + results_ref = llm_ref.generate([prompt], sampling_params) + generated_text_ref = [result.outputs[0].text for result in results_ref] + llm_ref.shutdown() + + for text_spec, text_ref in zip(generated_text_spec, generated_text_ref): + assert text_spec == text_ref + + @pytest.mark.parametrize("use_cuda_graph", [True, False]) def test_eagle3_lora(use_cuda_graph: bool): """Test LoRA with 3 requests and max_batch_size=4. diff --git a/tests/unittest/_torch/speculative/test_force_accepted_tokens.py b/tests/unittest/_torch/speculative/test_force_accepted_tokens.py index ac8228562bf8..fbfaca610386 100644 --- a/tests/unittest/_torch/speculative/test_force_accepted_tokens.py +++ b/tests/unittest/_torch/speculative/test_force_accepted_tokens.py @@ -35,6 +35,7 @@ import pytest import torch +from tensorrt_llm._torch.attention_backend.interface import AttentionMetadata from tensorrt_llm._torch.speculative.interface import ( FORCE_NUM_ACCEPTED_TOKENS_ENV_VAR, SpecWorkerBase, @@ -44,7 +45,7 @@ class _StubSpecWorker(SpecWorkerBase): - """Concrete ``SpecWorkerBase`` that stubs out the only abstract API. + """Concrete ``SpecWorkerBase`` that stubs out the abstract API. Used purely to drive ``_apply_force_accepted_tokens`` in isolation. """ @@ -53,6 +54,9 @@ class _StubSpecWorker(SpecWorkerBase): def max_draft_len(self) -> int: return 8 + def _forward_impl(self, *args: object, **kwargs: object) -> None: + raise NotImplementedError + def _make_worker(value: Optional[float] = None) -> _StubSpecWorker: worker = _StubSpecWorker() @@ -363,3 +367,28 @@ def test_cuda_graph_replay_advances_rng_state(): if __name__ == "__main__": pytest.main([__file__, "-v"]) + + +class _FailingSpecWorker(_StubSpecWorker): + """Stub whose forward saves spec-dec metadata state and then fails.""" + + def _forward_impl(self, *args: object, **kwargs: object) -> None: + attn_metadata = kwargs["attn_metadata"] + attn_metadata.prepare_for_spec_dec("_seq_lens", "_seq_lens_cuda") + raise RuntimeError("simulated draft failure") + + +def test_forward_restores_spec_dec_state_on_failure() -> None: + """A failure between prepare_for_spec_dec and restore must not leak saved + attn-metadata state; SpecWorkerBase.forward restores it in its cleanup + (https://nvbugs/6442074).""" + _require_cuda() + attn_metadata = AttentionMetadata(max_num_requests=2, max_num_tokens=16) + attn_metadata.seq_lens = torch.ones(2, dtype=torch.int32) + worker = _FailingSpecWorker() + for _ in range(2): + # The second iteration would trip the pairing assert inside + # prepare_for_spec_dec if the first failure had leaked saved state. + with pytest.raises(RuntimeError, match="simulated draft failure"): + worker(attn_metadata=attn_metadata, spec_metadata=None) + assert not attn_metadata.has_spec_dec_saved_state diff --git a/tests/unittest/_torch/speculative/test_group_all_greedy_sync.py b/tests/unittest/_torch/speculative/test_group_all_greedy_sync.py new file mode 100644 index 000000000000..bdcc3fd77992 --- /dev/null +++ b/tests/unittest/_torch/speculative/test_group_all_greedy_sync.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU unit tests for the group-synchronized ``is_all_greedy_sample`` override. + +Under ADP + LM-head TP with rejection sampling, the greedy-vs-advanced path +choice gates group collectives, so the model engine all-gathers the per-rank +flags and stores the group AND in ``SpecMetadata.group_all_greedy_sample``; +``_scan_one_model_sampling`` must then re-apply it on every rescan (populate +runs after the CUDA graph key is built and would otherwise resurrect the +rank-local value). + +These tests call ``_scan_one_model_sampling`` unbound on a SimpleNamespace +stand-in, mirroring test_rejection_buffers_guard.py, so no GPU or full +SpecMetadata construction is needed. +""" + +import types + +from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequestState +from tensorrt_llm._torch.speculative.interface import SpecMetadata + + +def _fake_request(temperature=None, top_k=None, top_p=None, slot=0): + return types.SimpleNamespace( + sampling_config=types.SimpleNamespace( + temperature=[temperature] if temperature is not None else None, + top_k=[top_k] if top_k is not None else None, + top_p=[top_p] if top_p is not None else None, + ), + state=LlmRequestState.GENERATION_IN_PROGRESS, + py_seq_slot=slot, + ) + + +def _fake_meta(group_all_greedy_sample=None, force_capture=False): + return types.SimpleNamespace( + runtime_draft_len=2, + dummy_slot_row=0, + group_all_greedy_sample=group_all_greedy_sample, + _force_non_greedy_for_capture=force_capture, + ) + + +def _scan(meta, requests): + return SpecMetadata._scan_one_model_sampling(meta, requests) + + +def test_local_value_used_when_no_group_sync(): + meta = _fake_meta(group_all_greedy_sample=None) + _scan(meta, [_fake_request(), _fake_request()]) + assert meta.is_all_greedy_sample is True + + _scan(meta, [_fake_request(), _fake_request(temperature=0.8)]) + assert meta.is_all_greedy_sample is False + + +def test_group_override_pulls_greedy_rank_onto_advanced_path(): + # This rank's batch is all-greedy, but another rank in the LM-head-TP + # group has a sampling request: the group AND (False) must win so the + # whole group takes the advanced path together. + meta = _fake_meta(group_all_greedy_sample=False) + _scan(meta, [_fake_request(), _fake_request()]) + assert meta.is_all_greedy_sample is False + + +def test_group_override_survives_rescan(): + # populate_sampling_params_for_one_model rescans after the CUDA graph key + # is built; the override must keep applying so the key, the buffers, and + # the worker branches all agree. + meta = _fake_meta(group_all_greedy_sample=False) + for _ in range(3): + _scan(meta, [_fake_request()]) + assert meta.is_all_greedy_sample is False + + +def test_group_override_true_keeps_greedy(): + meta = _fake_meta(group_all_greedy_sample=True) + _scan(meta, [_fake_request()]) + assert meta.is_all_greedy_sample is True + + +def test_capture_override_composes_with_group_sync(): + # Warmup forces the advanced variant to capture its CUDA graph; the group + # value is derived from capture-forced locals (all False), so the final + # flag stays False regardless of composition order. + meta = _fake_meta(group_all_greedy_sample=False, force_capture=True) + _scan(meta, [_fake_request()]) + assert meta.is_all_greedy_sample is False diff --git a/tests/unittest/_torch/speculative/test_rejection_buffers_guard.py b/tests/unittest/_torch/speculative/test_rejection_buffers_guard.py index e2cbe8752d7a..a71b9f1e6640 100644 --- a/tests/unittest/_torch/speculative/test_rejection_buffers_guard.py +++ b/tests/unittest/_torch/speculative/test_rejection_buffers_guard.py @@ -201,6 +201,9 @@ class _Worker(SpecWorkerBase): def max_draft_len(self) -> int: return K + def _forward_impl(self, *args: object, **kwargs: object) -> None: + raise NotImplementedError + def _dispatch_meta(**over): base = dict( diff --git a/tests/unittest/_torch/speculative/test_sa_hybrid_state_promotion.py b/tests/unittest/_torch/speculative/test_sa_hybrid_state_promotion.py new file mode 100644 index 000000000000..17e964d45fbf --- /dev/null +++ b/tests/unittest/_torch/speculative/test_sa_hybrid_state_promotion.py @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Regression tests for SAWorker promoting accepted recurrent states on +hybrid (SSM) models. + +Verification writes per-step recurrent states to the cache manager's +speculative scratch buffers, never the live pools; the worker must promote +the accepted step via ``update_mamba_states`` after acceptance (mirroring +the dflash/eagle3 one-engine workers), or hybrid models silently corrupt +their recurrent state under standalone SA speculative decoding. +""" + +from types import SimpleNamespace +from unittest import mock + +import pytest + +from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import MixedMambaHybridCacheManager +from tensorrt_llm._torch.speculative.sa_worker import SAWorker + + +def _make_worker() -> SAWorker: + spec_config = SimpleNamespace(max_draft_len=4, max_matching_ngram_size=2) + worker = SAWorker(spec_config) + # Stub out everything around the state-promotion call site; the mocked + # sampler return values flow into update_mamba_states unchanged. + worker._execute_guided_decoder_if_present = mock.MagicMock() + worker._sample_and_accept_draft_tokens = mock.MagicMock( + return_value=(mock.sentinel.accepted_tokens, mock.sentinel.num_accepted_tokens) + ) + worker._generate_draft_tokens = mock.MagicMock(return_value=mock.sentinel.next_draft_tokens) + worker._prepare_next_new_tokens = mock.MagicMock(return_value=mock.sentinel.next_new_tokens) + return worker + + +def _make_metadata(kv_cache_manager, num_seqs: int, num_contexts: int): + attn_metadata = SimpleNamespace( + num_seqs=num_seqs, + num_contexts=num_contexts, + kv_cache_manager=kv_cache_manager, + mamba_metadata=SimpleNamespace(state_indices=mock.sentinel.state_indices), + ) + spec_metadata = SimpleNamespace( + runtime_draft_len=4, + batch_indices_cuda=mock.sentinel.batch_indices_cuda, + ) + return attn_metadata, spec_metadata + + +def _run_forward(worker, attn_metadata, spec_metadata): + return worker._forward_impl( + input_ids=mock.sentinel.input_ids, + position_ids=mock.sentinel.position_ids, + hidden_states=mock.sentinel.hidden_states, + logits=mock.sentinel.logits, + attn_metadata=attn_metadata, + spec_metadata=spec_metadata, + ) + + +def test_hybrid_manager_promotes_accepted_states(): + worker = _make_worker() + manager = mock.MagicMock(spec=MixedMambaHybridCacheManager) + attn_metadata, spec_metadata = _make_metadata(manager, num_seqs=2, num_contexts=0) + + result = _run_forward(worker, attn_metadata, spec_metadata) + + manager.update_mamba_states.assert_called_once_with( + attn_metadata=attn_metadata, + num_accepted_tokens=mock.sentinel.num_accepted_tokens, + state_indices=mock.sentinel.state_indices, + ) + assert result["new_tokens"] is mock.sentinel.accepted_tokens + + +def test_hybrid_manager_context_only_batch_skips_promotion(): + worker = _make_worker() + manager = mock.MagicMock(spec=MixedMambaHybridCacheManager) + attn_metadata, spec_metadata = _make_metadata(manager, num_seqs=2, num_contexts=2) + + _run_forward(worker, attn_metadata, spec_metadata) + + manager.update_mamba_states.assert_not_called() + + +def test_pure_attention_manager_skips_promotion(): + worker = _make_worker() + manager = mock.MagicMock() # not a MambaHybridCacheManager + attn_metadata, spec_metadata = _make_metadata(manager, num_seqs=2, num_contexts=0) + + _run_forward(worker, attn_metadata, spec_metadata) + + manager.update_mamba_states.assert_not_called() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/unittest/_torch/test_model_config.py b/tests/unittest/_torch/test_model_config.py index 486c150f2040..28b2b0f9a96a 100644 --- a/tests/unittest/_torch/test_model_config.py +++ b/tests/unittest/_torch/test_model_config.py @@ -134,6 +134,36 @@ def test_validate_and_set_kv_cache_quant_rejects_invalid_dtype(): validate_and_set_kv_cache_quant(model_config, "invalid_dtype") +def _make_mixed_precision_model_config(): + """MIXED_PRECISION checkpoint shape: global config plus per-layer entries + whose kv_cache_quant_algo comes only from hf_quant_config.json (None here).""" + return ModelConfig( + quant_config=QuantConfig(quant_algo=QuantAlgo.MIXED_PRECISION, kv_cache_quant_algo=None), + quant_config_dict={ + "model.layers.0.attention": QuantConfig(quant_algo=QuantAlgo.FP8), + "model.layers.0.mixer.experts": QuantConfig(quant_algo=QuantAlgo.NVFP4), + }, + ) + + +def test_validate_and_set_kv_cache_quant_propagates_to_quant_config_dict(): + """Explicit kv_cache_config.dtype must override the per-layer QuantConfigs + too, otherwise the KV pool (sized from the global config) and attention + modules (built from per-layer configs) disagree on KV element size.""" + model_config = _make_mixed_precision_model_config() + validate_and_set_kv_cache_quant(model_config, "fp8") + assert model_config.quant_config.kv_cache_quant_algo == QuantAlgo.FP8 + for layer_quant_config in model_config.quant_config_dict.values(): + assert layer_quant_config.kv_cache_quant_algo == QuantAlgo.FP8 + + +def test_validate_and_set_kv_cache_quant_auto_keeps_quant_config_dict(): + model_config = _make_mixed_precision_model_config() + validate_and_set_kv_cache_quant(model_config, "auto") + for layer_quant_config in model_config.quant_config_dict.values(): + assert layer_quant_config.kv_cache_quant_algo is None + + def _write_safetensors_header(checkpoint_dir, tensor_dtype, tensor_shape): shard_name = "model-00001-of-00001.safetensors" header = { @@ -170,6 +200,34 @@ def test_deepseek_v4_base_checkpoint_detection( assert ModelConfig._is_deepseek_v4_base_checkpoint(str(tmp_path)) is expected_is_base +def test_deepseek_v4_missing_compress_ratios_raises(tmp_path, monkeypatch): + """DeepSeek-V4 load must fail fast with a clear error when neither the + checkpoint config nor a user ``sparse_attention_config`` provides + ``compress_ratios``. + + Regression test: previously ``compress_ratios`` could stay ``None`` and the + internal normalization comprehension raised an opaque + ``TypeError: 'NoneType' object is not iterable`` mid-load. + """ + from tensorrt_llm._torch import model_config as model_config_module + from tensorrt_llm._torch.configs.deepseekv4 import DeepseekV4Config + + pretrained_config = DeepseekV4Config( + architectures=["DeepseekV4ForCausalLM"], + compress_ratios=None, + num_hidden_layers=4, + ) + + # Avoid touching the filesystem for the HF config load; the empty tmp_path + # makes the real ``_is_deepseek_v4_base_checkpoint`` probe return False. + monkeypatch.setattr( + model_config_module, "load_pretrained_config", lambda *args, **kwargs: pretrained_config + ) + + with pytest.raises(ValueError, match="compress_ratios"): + ModelConfig.from_pretrained(str(tmp_path)) + + def test_model_config_sets_is_encoder_decoder_from_pretrained_config(): model_config = ModelConfig( pretrained_config=make_pretrained_config( diff --git a/tests/unittest/_torch/test_tensor_lru_cache.py b/tests/unittest/_torch/test_tensor_lru_cache.py index 389991e7ab6e..cb65559394f3 100644 --- a/tests/unittest/_torch/test_tensor_lru_cache.py +++ b/tests/unittest/_torch/test_tensor_lru_cache.py @@ -180,3 +180,86 @@ def write_and_read(index: int) -> None: hit = cache.get(index) if hit is not None: assert hit.numel() * hit.element_size() == 8 + + +def test_stream_aware_mode_leaves_cpu_cache_behavior_unchanged() -> None: + cache = TensorLRUCache[str](max_bytes=16, cuda_stream_aware=True) + source = torch.arange(4, dtype=torch.float32) + + assert cache.put("key", source) + cached = cache.get("key") + + assert cached is not None + assert cached.device.type == "cpu" + torch.testing.assert_close(cached, source) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_cuda_stream_aware_cache_orders_cross_stream_producer_and_consumer() -> None: + cache = TensorLRUCache[str](max_bytes=16, cuda_stream_aware=True) + producer_stream = torch.cuda.Stream() + consumer_stream = torch.cuda.Stream() + source = torch.zeros(4, device="cuda") + + with torch.cuda.stream(producer_stream): + torch.cuda._sleep(10_000_000) + source.fill_(7) + assert cache.put("key", source) + + with torch.cuda.stream(consumer_stream): + cached = cache.get("key") + assert cached is not None + observed = cached.clone() + + consumer_stream.synchronize() + torch.testing.assert_close(observed, torch.full_like(observed, 7)) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_cuda_stream_aware_cache_pop_orders_cross_stream_consumer() -> None: + cache = TensorLRUCache[str](max_bytes=16, cuda_stream_aware=True) + producer_stream = torch.cuda.Stream() + consumer_stream = torch.cuda.Stream() + source = torch.zeros(4, device="cuda") + + with torch.cuda.stream(producer_stream): + torch.cuda._sleep(10_000_000) + source.fill_(5) + assert cache.put("key", source) + + with torch.cuda.stream(consumer_stream): + popped = cache.pop("key") + assert popped is not None + observed = popped.clone() + + consumer_stream.synchronize() + assert len(cache) == 0 + torch.testing.assert_close(observed, torch.full_like(observed, 5)) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_cuda_stream_aware_cache_preserves_evicted_consumer_storage() -> None: + cache = TensorLRUCache[str](max_bytes=4 * 1024, cuda_stream_aware=True) + producer_stream = torch.cuda.Stream() + consumer_stream = torch.cuda.Stream() + source = torch.full((1024,), 3.0, device="cuda") + + with torch.cuda.stream(producer_stream): + assert cache.put("key", source) + + with torch.cuda.stream(consumer_stream): + cached = cache.get("key") + assert cached is not None + torch.cuda._sleep(10_000_000) + observed = cached.clone() + del cached + + with torch.cuda.stream(producer_stream): + assert cache.put("other", torch.full_like(source, 9.0)) + + stats = cache.stats() + assert stats.evictions == 1 + assert stats.replacements == 0 + + consumer_stream.synchronize() + torch.testing.assert_close(observed, torch.full_like(observed, 3.0)) diff --git a/tests/unittest/_torch/thop/parallel/test_fp8_quantize.py b/tests/unittest/_torch/thop/parallel/test_fp8_quantize.py index 110fedc494dc..10a8dae82131 100644 --- a/tests/unittest/_torch/thop/parallel/test_fp8_quantize.py +++ b/tests/unittest/_torch/thop/parallel/test_fp8_quantize.py @@ -397,6 +397,7 @@ def _decode_packed_int32_ue8m0(packed_int32): (16, 7168), (127, 4096), (1024, 7168), + pytest.param(262141, 128, id="grid-y-overflow"), ]) def test_fp8_quantize_1x128_packed_ue8m0_matches_legacy(m, k): """The fused packed op should produce the same FP8 output and UE8M0 diff --git a/tests/unittest/_torch/thop/parallel/test_fused_dit_layernorm_shift_scale.py b/tests/unittest/_torch/thop/parallel/test_fused_dit_layernorm_shift_scale.py new file mode 100644 index 000000000000..10ca4522f26e --- /dev/null +++ b/tests/unittest/_torch/thop/parallel/test_fused_dit_layernorm_shift_scale.py @@ -0,0 +1,644 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Tests for trtllm::fused_adaptive_layernorm[_quant]. + +Covers all 6 compile-time instantiations: + (has_ln_affine=F, has_modulation=F, has_quant=F) -- plain LN -> bf16 + (has_ln_affine=F, has_modulation=F, has_quant=T) -- plain LN -> FP4 + (has_ln_affine=T, has_modulation=F, has_quant=F) -- LN+affine -> bf16 (norm2) + (has_ln_affine=T, has_modulation=F, has_quant=T) -- LN+affine -> FP4 (norm2) + (has_ln_affine=F, has_modulation=T, has_quant=F) -- LN+AdaLN -> bf16 (norm1, norm3) + (has_ln_affine=F, has_modulation=T, has_quant=T) -- LN+AdaLN -> FP4 (norm1, norm3) +""" + +import pytest +import torch +import torch.nn.functional as F +from utils.util import skip_pre_blackwell + +D = 5120 +EPS = 1e-6 + + +# --------------------------------------------------------------------------- +# Float32 reference implementations +# --------------------------------------------------------------------------- + + +@torch.inference_mode() +def _ref_layernorm_adaln(x, scale_msa, shift_msa, seq_len_per_batch, eps): + """y = (1 + scale_msa) * x_hat + shift_msa, broadcast over seq.""" + M, _ = x.shape + xf = x.float() + mean = xf.mean(dim=-1, keepdim=True) + var = xf.var(dim=-1, keepdim=True, unbiased=False) + x_hat = (xf - mean) / (var + eps).sqrt() + batch_idx = torch.arange(M, device=x.device) // seq_len_per_batch + return (1.0 + scale_msa.float()[batch_idx]) * x_hat + shift_msa.float()[batch_idx] + + +@torch.inference_mode() +def _ref_layernorm_affine(x, ln_weight, ln_bias, eps): + """y = ln_weight * x_hat + ln_bias.""" + xf = x.float() + mean = xf.mean(dim=-1, keepdim=True) + var = xf.var(dim=-1, keepdim=True, unbiased=False) + x_hat = (xf - mean) / (var + eps).sqrt() + return ln_weight.float() * x_hat + ln_bias.float() + + +@torch.inference_mode() +def _ref_layernorm_plain(x, eps): + xf = x.float() + mean = xf.mean(dim=-1, keepdim=True) + var = xf.var(dim=-1, keepdim=True, unbiased=False) + return (xf - mean) / (var + eps).sqrt() + + +def _make_inputs(M, B, has_ln_affine, has_modulation, seed=42): + torch.manual_seed(seed) + device = torch.device("cuda") + x = torch.randn(M, D, device=device).to(torch.bfloat16) + ln_weight = ln_bias = scale_msa = shift_msa = None + seq_len_per_batch = M # default (B=1 case) + if has_ln_affine: + ln_weight = (torch.ones(D, device=device) + torch.randn(D, device=device) * 0.1).to( + torch.bfloat16 + ) + ln_bias = (torch.randn(D, device=device) * 0.1).to(torch.bfloat16) + ref = _ref_layernorm_affine(x, ln_weight, ln_bias, EPS) + elif has_modulation: + seq_len_per_batch = M // B + scale_msa = (torch.randn(B, D, device=device) * 0.2).to(torch.bfloat16) + shift_msa = (torch.randn(B, D, device=device) * 0.2).to(torch.bfloat16) + ref = _ref_layernorm_adaln(x, scale_msa, shift_msa, seq_len_per_batch, EPS) + else: + ref = _ref_layernorm_plain(x, EPS) + return x, ln_weight, ln_bias, scale_msa, shift_msa, seq_len_per_batch, ref + + +# --------------------------------------------------------------------------- +# BF16 correctness: all modes × several (M, B) shapes +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "M,B", + [ + (1, 1), # single row edge case + (4, 4), # tpb=1: single token per batch element + (32, 2), # non-multiple of warp; B=2 + (128, 4), # nominal Wan batch + (512, 1), # large M, B=1 + ], +) +@pytest.mark.parametrize( + "has_ln_affine,has_modulation", + [ + (False, False), + (True, False), + (False, True), + ], +) +def test_bf16_correctness(M, B, has_ln_affine, has_modulation): + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + if has_modulation and M % B != 0: + pytest.skip("M not divisible by B") + + x, ln_w, ln_b, scale_msa, shift_msa, seq_len, ref = _make_inputs( + M, B, has_ln_affine, has_modulation + ) + out = torch.ops.trtllm.fused_adaptive_layernorm( + x, ln_w, ln_b, scale_msa, shift_msa, seq_len, EPS + ) + + assert out.shape == (M, D) + assert out.dtype == torch.bfloat16 + + # bf16 accumulation gives ~1e-3 max absolute error vs fp32 reference. + torch.testing.assert_close(out.float(), ref.to(torch.bfloat16).float(), rtol=2e-2, atol=2e-2) + + +# --------------------------------------------------------------------------- +# Batch modulation correctness: each batch element gets its own scale/shift. +# With very different modulators the output rows must differ significantly. +# --------------------------------------------------------------------------- + + +def test_adaln_batch_modulation_correctness(): + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + + M, B = 128, 4 + S = M // B + device = torch.device("cuda") + torch.manual_seed(0) + + x = torch.randn(M, D, device=device).to(torch.bfloat16) + # Very distinct scale/shift per batch element. + scale_msa = torch.stack([torch.ones(D, device=device) * (i * 0.5) for i in range(B)]).to( + torch.bfloat16 + ) + shift_msa = torch.stack([torch.ones(D, device=device) * (i * 1.0) for i in range(B)]).to( + torch.bfloat16 + ) + + out = torch.ops.trtllm.fused_adaptive_layernorm(x, None, None, scale_msa, shift_msa, S, EPS) + ref = _ref_layernorm_adaln(x, scale_msa, shift_msa, S, EPS).to(torch.bfloat16) + + # Cross-batch: rows from different batch elements should be measurably different. + out_b0 = out[:S].float().mean() + out_b1 = out[S : 2 * S].float().mean() + assert abs(out_b0.item() - out_b1.item()) > 0.1, ( + "Batch elements have indistinguishable outputs — modulation broadcast may be broken" + ) + + # Still matches reference row-by-row. + torch.testing.assert_close(out.float(), ref.float(), rtol=2e-2, atol=2e-2) + + +# --------------------------------------------------------------------------- +# Affine fp32->bf16 downcast: model keeps norm2 weight/bias in float32; the op +# reads them as bf16. Verify the downcast matches the float32-weight reference. +# --------------------------------------------------------------------------- + + +def test_affine_fp32_weight_downcast_matches_eager(): + """fp32 weight/bias downcast to bf16 matches the float32-weight reference.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + torch.manual_seed(0) + M = 128 + + x = torch.randn(M, D, device=device).to(torch.bfloat16) + ln_weight = torch.ones(D, device=device) + torch.randn(D, device=device) * 0.1 + ln_bias = torch.randn(D, device=device) * 0.1 + + # The op reads weight/bias as bf16 (kernel ABI); mimic the model's downcast. + out = torch.ops.trtllm.fused_adaptive_layernorm( + x, ln_weight.to(torch.bfloat16), ln_bias.to(torch.bfloat16), None, None, M, EPS + ) + ref_fp32 = _ref_layernorm_affine(x, ln_weight, ln_bias, EPS) + torch.testing.assert_close( + out.float(), ref_fp32.to(torch.bfloat16).float(), rtol=2e-2, atol=2e-2 + ) + + # Isolated fp32->bf16 weight-downcast error stays well below the bf16 output tol. + ref_bf16 = _ref_layernorm_affine( + x, ln_weight.to(torch.bfloat16), ln_bias.to(torch.bfloat16), EPS + ) + assert (ref_fp32 - ref_bf16).norm() / ref_fp32.norm() < 5e-3 + + +# --------------------------------------------------------------------------- +# FP4 correctness: feed quant output through nvfp4_gemm, cosine_sim > 0.98. +# Pattern from Yiyun's LTX-2 test_fused_dit_gate_resid_norm_shift_scale.py. +# --------------------------------------------------------------------------- + + +@skip_pre_blackwell +@pytest.mark.parametrize( + "M,B", + [ + (1, 1), # M=1: maximum SF over-allocation (1 row padded to 128) + (128, 4), + (256, 1), + ], +) +@pytest.mark.parametrize( + "has_ln_affine,has_modulation", + [ + (False, False), + (True, False), + (False, True), + ], +) +def test_fp4_quant_gemm_correctness(M, B, has_ln_affine, has_modulation): + if has_modulation and M % B != 0: + pytest.skip("M not divisible by B") + + device = torch.device("cuda") + out_dim = 1024 + x, ln_w, ln_b, scale_msa, shift_msa, seq_len, _ = _make_inputs( + M, B, has_ln_affine, has_modulation, seed=7 + ) + # Scale x to moderate magnitude so FP4 clipping is minimal. + x = (x * 0.5).to(torch.bfloat16) + + W = torch.randn(out_dim, D, dtype=torch.bfloat16, device=device) * 0.05 + + # bf16 reference: unfused LN + F.linear. + if has_ln_affine: + out_bf16 = torch.ops.trtllm.fused_adaptive_layernorm( + x, ln_w, ln_b, None, None, seq_len, EPS + ) + elif has_modulation: + out_bf16 = torch.ops.trtllm.fused_adaptive_layernorm( + x, None, None, scale_msa, shift_msa, seq_len, EPS + ) + else: + out_bf16 = torch.ops.trtllm.fused_adaptive_layernorm( + x, None, None, None, None, seq_len, EPS + ) + D_ref = F.linear(out_bf16, W) + + sf_scale_x = (448.0 * 6.0) / out_bf16.abs().max().float() + sf_scale_w = (448.0 * 6.0) / W.abs().max().float() + + y_fp4, y_sf = torch.ops.trtllm.fused_adaptive_layernorm_quant( + x, ln_w, ln_b, scale_msa, shift_msa, sf_scale_x, seq_len, EPS + ) + + assert y_fp4.shape == (M, D // 2) + assert y_fp4.dtype == torch.uint8 + assert y_sf.dtype == torch.uint8 + + W_fp4, W_sf = torch.ops.trtllm.fp4_quantize(W, sf_scale_w, 16) + alpha = 1.0 / (sf_scale_x * sf_scale_w).float() + C = torch.ops.trtllm.nvfp4_gemm(y_fp4, W_fp4, y_sf, W_sf, alpha, torch.bfloat16) + + cos_sim = F.cosine_similarity(C.flatten().float(), D_ref.flatten().float(), dim=0).item() + assert cos_sim > 0.98, ( + f"FP4 GEMM cosine similarity {cos_sim:.4f} < 0.98 " + f"(M={M} B={B} has_ln_affine={has_ln_affine} has_modulation={has_modulation})" + ) + + +# --------------------------------------------------------------------------- +# Input validation: the op must reject invalid argument combinations. +# --------------------------------------------------------------------------- + + +def test_validation_mutual_exclusivity(): + """Both ln_weight and scale_msa provided → TORCH_CHECK error.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + x = torch.randn(32, D, device=device).to(torch.bfloat16) + ln_w = torch.ones(D, device=device).to(torch.bfloat16) + ln_b = torch.zeros(D, device=device).to(torch.bfloat16) + scale = torch.zeros(1, D, device=device).to(torch.bfloat16) + shift = torch.zeros(1, D, device=device).to(torch.bfloat16) + with pytest.raises(RuntimeError, match="mutually exclusive"): + torch.ops.trtllm.fused_adaptive_layernorm(x, ln_w, ln_b, scale, shift, 32, EPS) + + +def test_validation_wrong_hidden_dim(): + """D != 5120 must raise a descriptive error.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + x = torch.randn(32, 4096, device=device).to(torch.bfloat16) + with pytest.raises(RuntimeError): + torch.ops.trtllm.fused_adaptive_layernorm(x, None, None, None, None, 32, EPS) + + +def test_validation_wrong_dtype(): + """float32 input must raise a descriptive error.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + x = torch.randn(32, D, device=device) # float32 + with pytest.raises(RuntimeError): + torch.ops.trtllm.fused_adaptive_layernorm(x, None, None, None, None, 32, EPS) + + +def test_validation_non_divisible_seq_len(): + """M not divisible by seq_len_per_batch must raise for AdaLN path.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + x = torch.randn(33, D, device=device).to(torch.bfloat16) + scale = torch.zeros(3, D, device=device).to(torch.bfloat16) + shift = torch.zeros(3, D, device=device).to(torch.bfloat16) + # seq_len_per_batch=11 → B=3, 3*11=33 ✓; seq_len_per_batch=10 → 33%10≠0 ✗ + with pytest.raises(RuntimeError): + torch.ops.trtllm.fused_adaptive_layernorm(x, None, None, scale, shift, 10, EPS) + + +def test_validation_non_contiguous_x(): + """Non-contiguous x must raise a RuntimeError mentioning 'contiguous'.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + # Create a strided (non-contiguous) view by slicing a wider tensor. + base = torch.randn(32, D * 2, device=device).to(torch.bfloat16) + x_strided = base[:, :D] + assert not x_strided.is_contiguous() + with pytest.raises(RuntimeError, match=r"contiguous"): + torch.ops.trtllm.fused_adaptive_layernorm(x_strided, None, None, None, None, 32, EPS) + + +def test_validation_ln_weight_without_ln_bias(): + """ln_weight provided without ln_bias must raise (partial affine pair).""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + x = torch.randn(32, D, device=device).to(torch.bfloat16) + ln_w = torch.ones(D, device=device).to(torch.bfloat16) + with pytest.raises(RuntimeError, match="ln_weight and ln_bias must both be provided together"): + torch.ops.trtllm.fused_adaptive_layernorm(x, ln_w, None, None, None, 32, EPS) + + +def test_validation_scale_msa_without_shift_msa(): + """scale_msa provided without shift_msa must raise (partial modulation pair).""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + x = torch.randn(32, D, device=device).to(torch.bfloat16) + scale = torch.zeros(1, D, device=device).to(torch.bfloat16) + with pytest.raises( + RuntimeError, match="scale_msa and shift_msa must both be provided together" + ): + torch.ops.trtllm.fused_adaptive_layernorm(x, None, None, scale, None, 32, EPS) + + +def test_validation_non_positive_seq_len(): + """seq_len_per_batch <= 0 on the AdaLN path must raise.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + x = torch.randn(32, D, device=device).to(torch.bfloat16) + scale = torch.zeros(1, D, device=device).to(torch.bfloat16) + shift = torch.zeros(1, D, device=device).to(torch.bfloat16) + with pytest.raises(RuntimeError, match="seq_len_per_batch must be positive"): + torch.ops.trtllm.fused_adaptive_layernorm(x, None, None, scale, shift, 0, EPS) + + +@skip_pre_blackwell +def test_validation_sf_scale_wrong_dtype(): + """sf_scale with a non-float32 dtype must raise a descriptive error.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + x = torch.randn(32, D, device=device).to(torch.bfloat16) + sf_scale = torch.ones(1, device=device).to(torch.bfloat16) # wrong dtype + with pytest.raises(RuntimeError, match="dtype"): + torch.ops.trtllm.fused_adaptive_layernorm_quant( + x, None, None, None, None, sf_scale, 32, EPS + ) + + +@skip_pre_blackwell +def test_validation_sf_scale_non_scalar(): + """sf_scale with more than one element must raise a descriptive error.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + device = torch.device("cuda") + x = torch.randn(32, D, device=device).to(torch.bfloat16) + sf_scale = torch.ones(2, device=device) # non-scalar + with pytest.raises(RuntimeError, match="scalar tensor"): + torch.ops.trtllm.fused_adaptive_layernorm_quant( + x, None, None, None, None, sf_scale, 32, EPS + ) + + +# --------------------------------------------------------------------------- +# Production shapes: Wan 2.2 T2V-A14B at both default resolutions. +# +# Model: 14B, hidden_size=5120, patch_size=[1,2,2], vae_spatial=8, vae_temporal=4. +# Default resolutions (pipeline_wan.py): 480×832 and 720×1280, 81 frames. +# +# Token counts (latent → patchify): +# 480p / 81-frame: latent=(21, 60, 104) → 21×30×52 = 32760 tokens ← real default +# 720p / 81-frame: latent=(21, 90, 160) → 21×45×80 = 75600 tokens ← real default +# 480p / 1-frame: latent=( 1, 60, 104) → 1×30×52 = 1560 tokens ← CI-speed subset +# +# Three modes match the three norm sites in WanTransformerBlock: +# plain LN — standalone norm (no modulation) +# AdaLN (no affine) — norm1 (pre-attn) and norm3 (pre-cross-attn) +# affine LN — norm2 (pre-FFN, learned weight/bias) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "M,B,has_ln_affine,has_modulation", + [ + # 720p 81-frame — all three kernel modes + (75600, 1, False, False), # plain LN + (75600, 1, False, True), # AdaLN (norm1 / norm3 sites) + (75600, 1, True, False), # affine (norm2 site) + # 480p 81-frame (real default) — the two modulated modes + (32760, 1, False, True), # AdaLN 480p full + (32760, 1, True, False), # affine 480p full + # 480p 1-frame (CI-speed subset) + (1560, 1, False, True), # AdaLN 480p 1-frame + (1560, 1, True, False), # affine 480p 1-frame + # 480p 1-frame, B=2 — batch inference + (3120, 2, False, True), # AdaLN B=2 + ], +) +def test_wan14b_production_shape_bf16(M, B, has_ln_affine, has_modulation): + """BF16 correctness at Wan 14B production token counts across all norm-site modes.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + + x, ln_w, ln_b, scale_msa, shift_msa, seq_len, ref = _make_inputs( + M, B, has_ln_affine, has_modulation, seed=99 + ) + out = torch.ops.trtllm.fused_adaptive_layernorm( + x, ln_w, ln_b, scale_msa, shift_msa, seq_len, EPS + ) + + assert out.shape == (M, D) + assert out.dtype == torch.bfloat16 + torch.testing.assert_close(out.float(), ref.to(torch.bfloat16).float(), rtol=2e-2, atol=2e-2) + + +@skip_pre_blackwell +@pytest.mark.parametrize( + "M,B,has_ln_affine,has_modulation", + [ + # 480p 1-frame subset: avoids a slow M=32760 GEMM reference in CI while + # still exercising real patchified token counts. + (1560, 1, False, True), # AdaLN (norm1/norm3) — the dominant quantized site + (1560, 1, True, False), # affine (norm2) + (3120, 2, False, True), # AdaLN B=2 + ], +) +def test_fp4_quant_production_scale(M, B, has_ln_affine, has_modulation): + """FP4 GEMM quality at 480p/1-frame token count. + + Mirrors Yiyun's KA/KC-quant approach: feed fused-kernel fp4 output through + nvfp4_gemm and verify cosine_similarity > 0.98 vs a bf16 F.linear reference. + Full 480p (32760 tokens) and 720p shape/SF checks are in + test_fp4_quant_720p_smoke and test_fp4_sf_allocation_boundaries. + """ + device = torch.device("cuda") + out_dim = 1024 + x, ln_w, ln_b, scale_msa, shift_msa, seq_len, _ = _make_inputs( + M, B, has_ln_affine, has_modulation, seed=7 + ) + x = (x * 0.5).to(torch.bfloat16) + W = torch.randn(out_dim, D, dtype=torch.bfloat16, device=device) * 0.05 + + out_bf16 = torch.ops.trtllm.fused_adaptive_layernorm( + x, ln_w, ln_b, scale_msa, shift_msa, seq_len, EPS + ) + D_ref = F.linear(out_bf16, W) + + sf_scale_x = (448.0 * 6.0) / out_bf16.abs().max().float() + sf_scale_w = (448.0 * 6.0) / W.abs().max().float() + + y_fp4, y_sf = torch.ops.trtllm.fused_adaptive_layernorm_quant( + x, ln_w, ln_b, scale_msa, shift_msa, sf_scale_x, seq_len, EPS + ) + + W_fp4, W_sf = torch.ops.trtllm.fp4_quantize(W, sf_scale_w, 16) + alpha = 1.0 / (sf_scale_x * sf_scale_w).float() + C = torch.ops.trtllm.nvfp4_gemm(y_fp4, W_fp4, y_sf, W_sf, alpha, torch.bfloat16) + + cos_sim = F.cosine_similarity(C.flatten().float(), D_ref.flatten().float(), dim=0).item() + assert cos_sim > 0.98, ( + f"FP4 GEMM cosine {cos_sim:.4f} < 0.98 " + f"(M={M} B={B} has_ln_affine={has_ln_affine} has_modulation={has_modulation})" + ) + + +# --------------------------------------------------------------------------- +# Production modulator pattern: Wan extracts scale/shift via +# .chunk(6, dim=2) → .squeeze(2) → .reshape(B, -1) +# The reshape on a non-contiguous chunk output returns a contiguous copy, +# so scale_msa/shift_msa are always contiguous by the time they reach our op. +# This test verifies the full extraction pipeline produces correct results +# and that the op rejects genuinely non-contiguous modulators. +# --------------------------------------------------------------------------- + + +def test_adaln_production_modulator_extraction(): + """AdaLN with scale/shift extracted via the Wan chunk→squeeze→reshape pipeline.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + + B, T, S = 2, 1, 64 + M = B * S + K = 6 # Wan uses 6 modulation slots + device = torch.device("cuda") + torch.manual_seed(5) + + x = torch.randn(M, D, device=device).to(torch.bfloat16) + + # Mimic Wan WanBlock: temb [B, T, K, D] → chunk → squeeze → reshape + temb = torch.randn(B, T, K, D, device=device).to(torch.bfloat16) + chunks = temb.chunk(K, dim=2) # K tensors of [B, T, 1, D] + shift_msa_raw = chunks[0].squeeze(2) # [B, T, D], non-contiguous strides + scale_msa_raw = chunks[1].squeeze(2) + # reshape may return a non-contiguous view; .contiguous() mirrors the wrapper behavior + scale_msa = scale_msa_raw.reshape(B, -1).contiguous() + shift_msa = shift_msa_raw.reshape(B, -1).contiguous() + assert scale_msa.is_contiguous() + assert scale_msa.shape == (B, D) # T=1 so T*D == D + + out = torch.ops.trtllm.fused_adaptive_layernorm(x, None, None, scale_msa, shift_msa, S, EPS) + ref = _ref_layernorm_adaln(x, scale_msa, shift_msa, S, EPS).to(torch.bfloat16) + + assert out.shape == (M, D) + torch.testing.assert_close(out.float(), ref.float(), rtol=2e-2, atol=2e-2) + + +def test_adaln_rejects_non_contiguous_modulators(): + """Op must raise when scale_msa/shift_msa are non-contiguous (before reshape).""" + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + + B, S = 4, 32 + M = B * S + device = torch.device("cuda") + x = torch.randn(M, D, device=device).to(torch.bfloat16) + + # chunk produces non-contiguous views + temb = torch.randn(B, 6, D, device=device).to(torch.bfloat16) + chunks = temb.chunk(6, dim=1) + scale_nc = chunks[0].squeeze(1) # [B, D], non-contiguous + shift_nc = chunks[1].squeeze(1) + assert not scale_nc.is_contiguous() + + with pytest.raises(RuntimeError, match=r"contiguous"): + torch.ops.trtllm.fused_adaptive_layernorm(x, None, None, scale_nc, shift_nc, S, EPS) + + +# --------------------------------------------------------------------------- +# FP4 smoke tests at full production shapes — shape / SF-size checks only. +# The [M, D] × [out_dim, D]^T GEMM reference is prohibitively slow at M=75600, +# so these tests verify the kernel runs without error and emits the correct +# output shapes; cosine correctness is covered by test_fp4_quant_production_scale. +# --------------------------------------------------------------------------- + + +@skip_pre_blackwell +@pytest.mark.parametrize( + "M,B,has_ln_affine,has_modulation", + [ + # 720p 81-frame — dominant FP4 quantized sites + (75600, 1, False, True), # AdaLN (norm1/norm3) + (75600, 1, True, False), # affine (norm2) + # 480p 81-frame — full default resolution + (32760, 1, False, True), # AdaLN 480p full + (32760, 1, True, False), # affine 480p full + ], +) +def test_fp4_quant_production_shape_smoke(M, B, has_ln_affine, has_modulation): + """FP4 kernel output shapes are correct at full Wan 2.2 production token counts.""" + device = torch.device("cuda") + x, ln_w, ln_b, scale_msa, shift_msa, seq_len, _ = _make_inputs( + M, B, has_ln_affine, has_modulation, seed=11 + ) + x = (x * 0.5).to(torch.bfloat16) + + sf_scale = torch.tensor(1.0, dtype=torch.float32, device=device) + y_fp4, y_sf = torch.ops.trtllm.fused_adaptive_layernorm_quant( + x, ln_w, ln_b, scale_msa, shift_msa, sf_scale, seq_len, EPS + ) + + assert y_fp4.shape == (M, D // 2), f"y_fp4 shape mismatch: {y_fp4.shape}" + assert y_fp4.dtype == torch.uint8 + + sf_cols = D // 16 # 320 for D=5120 + expected_sf_numel = (M + 127) // 128 * 128 * sf_cols + assert y_sf.numel() == expected_sf_numel, ( + f"y_sf numel {y_sf.numel()} != expected {expected_sf_numel} (M={M})" + ) + + +# --------------------------------------------------------------------------- +# SF allocation boundary test: verifies the (M+127)/128*128 * (D/16) formula +# at the 128-row tile boundary. M=128 is the only case with no padding; +# M=127 and M=129 straddle it on either side. +# --------------------------------------------------------------------------- + + +@skip_pre_blackwell +@pytest.mark.parametrize("M", [1, 127, 128, 129, 255, 256]) +def test_fp4_sf_allocation_boundaries(M): + """SF tensor size matches the inline formula at every 128-row tile boundary.""" + device = torch.device("cuda") + torch.manual_seed(M) + x = (torch.randn(M, D, device=device) * 0.5).to(torch.bfloat16) + sf_scale = torch.tensor(1.0, dtype=torch.float32, device=device) + + y_fp4, y_sf = torch.ops.trtllm.fused_adaptive_layernorm_quant( + x, None, None, None, None, sf_scale, M, EPS + ) + + assert y_fp4.shape == (M, D // 2) + + sf_cols = D // 16 # 320 for D=5120; already divisible by 4 + expected_sf_numel = (M + 127) // 128 * 128 * sf_cols + assert y_sf.numel() == expected_sf_numel, ( + f"M={M}: y_sf.numel()={y_sf.numel()} expected={expected_sf_numel}" + ) diff --git a/tests/unittest/_torch/thop/parallel/test_indexer_topk.py b/tests/unittest/_torch/thop/parallel/test_indexer_topk.py index ee0e2bc2129c..aa9a327d2f89 100644 --- a/tests/unittest/_torch/thop/parallel/test_indexer_topk.py +++ b/tests/unittest/_torch/thop/parallel/test_indexer_topk.py @@ -45,6 +45,10 @@ import tensorrt_llm # noqa: F401 from tensorrt_llm._torch.custom_ops import cute_dsl_custom_ops +from tensorrt_llm._torch.custom_ops.cute_dsl_custom_ops import CuteDSLTopKPrefillSingleCTARunner +from tensorrt_llm._torch.cute_dsl_kernels.blackwell.top_k.filtered_top_k_decode_varlen import ( + cute_dsl_radix_filter_topk_wrapper, +) from tensorrt_llm._torch.cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE if not torch.cuda.is_available(): @@ -150,62 +154,89 @@ def compare_top_k_results( Returns: True if results match within tolerance, False otherwise """ - num_rows = cuda_indices.shape[0] - - # Calculate valid lengths for each row (vectorized) - row_lengths = row_ends - row_starts - - for row_idx in range(num_rows): - row_len = row_lengths[row_idx].item() - expected_valid = min(row_len, top_k) - - cuda_row = cuda_indices[row_idx] - torch_row = torch_indices[row_idx] - - cuda_valid_mask = cuda_row != -1 - torch_valid_mask = torch_row != -1 - - cuda_valid = cuda_row[cuda_valid_mask] - torch_valid = torch_row[torch_valid_mask] - - if cuda_valid.shape[0] != torch_valid.shape[0]: - print( - f"Row {row_idx}: Different number of valid indices - " - f"CUDA: {cuda_valid.shape[0]}, PyTorch: {torch_valid.shape[0]}" - ) - return False - - if cuda_valid.shape[0] != expected_valid: - print( - f"Row {row_idx}: Expected {expected_valid} valid indices, got {cuda_valid.shape[0]}" + # --- vectorized implementation (no per-row .item() syncs) --- + row_lengths = row_ends - row_starts # [num_rows] + expected_valid = torch.minimum(row_lengths, torch.full_like(row_lengths, top_k)) + + cuda_valid_counts = (cuda_indices != -1).sum(dim=1) + torch_valid_counts = (torch_indices != -1).sum(dim=1) + + # Count mismatch between cuda and torch + mismatch = (cuda_valid_counts != torch_valid_counts).nonzero(as_tuple=True)[0] + if mismatch.numel() > 0: + row_idx = mismatch[0].item() + print( + f"Row {row_idx}: Different number of valid indices - " + f"CUDA: {cuda_valid_counts[row_idx].item()}, " + f"PyTorch: {torch_valid_counts[row_idx].item()}" + ) + return False + + # Count mismatch vs expected + wrong = (cuda_valid_counts != expected_valid).nonzero(as_tuple=True)[0] + if wrong.numel() > 0: + row_idx = wrong[0].item() + print( + f"Row {row_idx}: Expected {expected_valid[row_idx].item()} valid indices, " + f"got {cuda_valid_counts[row_idx].item()}" + ) + return False + + # Value check: safe gather (-1 → 0), add row_start, gather logits, mask invalid → -inf, sort, compare. + # cuda_indices and torch_indices may have different column counts (e.g. decode path where + # torch_indices is built with min(top_k, max_row_len) columns), so use separate masks. + cuda_invalid = cuda_indices == -1 + torch_invalid = torch_indices == -1 + + cuda_safe = cuda_indices.clone() + torch_safe = torch_indices.clone() + cuda_safe[cuda_invalid] = 0 + torch_safe[torch_invalid] = 0 + + cuda_abs = cuda_safe.long() + row_starts.unsqueeze(1) + torch_abs = torch_safe.long() + row_starts.unsqueeze(1) + + cuda_vals = logits.gather(1, cuda_abs) + torch_vals = logits.gather(1, torch_abs) + cuda_vals[cuda_invalid] = float("-inf") + torch_vals[torch_invalid] = float("-inf") + + cuda_sorted = cuda_vals.sort(dim=1, descending=True).values + torch_sorted = torch_vals.sort(dim=1, descending=True).values + + # Build per-tensor position masks (first expected_valid[i] positions per row). + # Flattening both gives the same element count because counts were verified equal above. + cuda_k = cuda_indices.shape[1] + torch_k = torch_indices.shape[1] + cuda_pos_mask = torch.arange(cuda_k, device=cuda_indices.device).unsqueeze( + 0 + ) < expected_valid.unsqueeze(1) + torch_pos_mask = torch.arange(torch_k, device=torch_indices.device).unsqueeze( + 0 + ) < expected_valid.unsqueeze(1) + + if not torch.allclose( + cuda_sorted[cuda_pos_mask], torch_sorted[torch_pos_mask], rtol=tolerance, atol=tolerance + ): + bad = (cuda_k == torch_k) and ( + ( + ~torch.isclose(cuda_sorted, torch_sorted, rtol=tolerance, atol=tolerance) + & cuda_pos_mask ) - return False - - if cuda_valid.shape[0] == 0: - continue - - row_start = row_starts[row_idx].item() - logits_row = logits[row_idx] - - cuda_abs_indices = cuda_valid + row_start - torch_abs_indices = torch_valid + row_start - - cuda_values = logits_row[cuda_abs_indices] - torch_values = logits_row[torch_abs_indices] - - cuda_values_sorted, _ = torch.sort(cuda_values, descending=True) - torch_values_sorted, _ = torch.sort(torch_values, descending=True) - - if not torch.allclose( - cuda_values_sorted, torch_values_sorted, rtol=tolerance, atol=tolerance - ): + .any(dim=1) + .nonzero(as_tuple=True)[0] + ) + if bad is not False and bad.numel() > 0: + row_idx = bad[0].item() + cuda_valid = cuda_indices[row_idx][cuda_indices[row_idx] != -1] + torch_valid = torch_indices[row_idx][torch_indices[row_idx] != -1] cuda_set = set(cuda_valid.cpu().tolist()) torch_set = set(torch_valid.cpu().tolist()) if cuda_set != torch_set: print(" Different indices selected:") print(f" Only in CUDA: {cuda_set - torch_set}") print(f" Only in Torch: {torch_set - cuda_set}") - return False + return False return True @@ -720,123 +751,122 @@ def _run_cute_dsl_topk_test(batch_size, next_n, index_topk, num_tokens, dtype, r @pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") @skip_pre_blackwell -@pytest.mark.parametrize("batch_size", [1, 4, 64, 256]) -@pytest.mark.parametrize("next_n", [1, 3]) -@pytest.mark.parametrize("index_topk", [2048]) -@pytest.mark.parametrize("num_tokens", [4096, 8192]) -@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("load_balance", [False, True]) -def test_cute_dsl_topk_decode_single_cta( - batch_size, next_n, index_topk, num_tokens, dtype, load_balance -): - """Correctness test for CuTE DSL single-CTA TopK decode on Blackwell.""" - _run_cute_dsl_topk_test( - batch_size, - next_n, - index_topk, - num_tokens, - dtype, - lambda logits, seq_lens: torch.ops.trtllm.cute_dsl_topk_decode_blackwell( +@pytest.mark.parametrize("batch_size", [1, 8]) +@pytest.mark.parametrize("index_topk", [1023, 2047, 4095, 4096, 8192, 16384]) +@pytest.mark.parametrize("num_tokens", [32768, 131072]) +def test_cute_dsl_topk_decode_high_and_odd_k(batch_size, index_topk, num_tokens): + """Large (>4096) and odd top_k stay bit-exact on the decode entry points. + + The indexer entry point is the path KV-cache eviction uses with large or + odd keep budgets, so it is checked for every top_k across all three dtypes + (the scalar output write at vecsize_out=1 is dtype dependent). The + single-CTA and multi-CTA wrappers, whose raised guard this test backs, only + support even top_k, so they are checked on the even values in fp32. + """ + + def run_single_cta(logits, seq_lens): + return torch.ops.trtllm.cute_dsl_topk_decode_blackwell( input_values=logits, seq_lens=seq_lens, top_k=index_topk, - next_n=next_n, + next_n=1, num_copy_bits=256, - load_balance=load_balance, - ), - ) - + ) -@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") -@skip_pre_blackwell -@pytest.mark.parametrize("batch_size", [1, 4, 64]) -@pytest.mark.parametrize("next_n", [1, 3]) -@pytest.mark.parametrize("index_topk", [2048]) -@pytest.mark.parametrize("num_tokens", [32768, 65536]) -@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("chunk_size_per_cta", [16384]) -@pytest.mark.parametrize("dynamic", [False, True]) -def test_cute_dsl_topk_decode_multi_cta( - batch_size, next_n, index_topk, num_tokens, dtype, chunk_size_per_cta, dynamic -): - """Correctness test for CuTE DSL multi-CTA TopK decode on Blackwell.""" - _run_cute_dsl_topk_test( - batch_size, - next_n, - index_topk, - num_tokens, - dtype, - lambda logits, seq_lens: torch.ops.trtllm.cute_dsl_topk_decode_multi_cta_blackwell( + def run_multi_cta(logits, seq_lens): + return torch.ops.trtllm.cute_dsl_topk_decode_multi_cta_blackwell( input_values=logits, seq_lens=seq_lens, top_k=index_topk, - next_n=next_n, + next_n=1, num_copy_bits=256, - chunk_size_per_cta=chunk_size_per_cta, - dynamic=dynamic, - ), - ) - - -@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") -@skip_pre_blackwell -@pytest.mark.parametrize("batch_size", [1, 4, 64, 128]) -@pytest.mark.parametrize("next_n", [1, 2]) -@pytest.mark.parametrize("index_topk", [2048]) -@pytest.mark.parametrize("num_tokens", [4096, 8192, 65536, 131072]) -def test_cute_dsl_indexer_topk_decode(batch_size, next_n, index_topk, num_tokens): - """Correctness test for CuTE DSL indexer TopK decode with in-place output.""" - num_gen_tokens = batch_size * next_n + chunk_size_per_cta=16384, + dynamic=False, + ) - def run_fn(logits, seq_lens): - """Run CuTE DSL indexer TopK decode and return output indices.""" - output_indices = torch.empty(num_gen_tokens, index_topk, dtype=torch.int32, device="cuda") + def run_indexer(logits, seq_lens): + output_indices = torch.empty(batch_size, index_topk, dtype=torch.int32, device="cuda") torch.ops.trtllm.cute_dsl_indexer_topk_decode( input_values=logits, seq_lens=seq_lens, output_indices=output_indices, top_k=index_topk, - next_n=next_n, + next_n=1, num_copy_bits=256, ) return output_indices - _run_cute_dsl_topk_test( - batch_size, - next_n, - index_topk, - num_tokens, - torch.float32, - run_fn, - ) + if index_topk % 2 == 0: + # Even top_k: all three entry points are defined; the raised guard this + # test backs is exercised on the wrappers in fp32. + dtype_runs = [(torch.float32, (run_single_cta, run_multi_cta, run_indexer))] + else: + # Odd top_k: only the indexer path supports it, checked across dtypes. + dtype_runs = [ + (torch.float32, (run_indexer,)), + (torch.float16, (run_indexer,)), + (torch.bfloat16, (run_indexer,)), + ] + + for dtype, run_fns in dtype_runs: + for run_fn in run_fns: + _run_cute_dsl_topk_test( + batch_size, + 1, + index_topk, + num_tokens, + dtype, + run_fn, + ) + + # Regression (effective row length <= top_k in a radix-filter cluster): + # fp32 num_tokens=32768, top_k=16384 routes the indexer to a 4-CTA cluster + # (chunk=8192). A row whose effective length spans >1 chunk but is <= top_k + # (eff=12000) has a merged histogram total (== eff) that never exceeds + # top_k, so the cluster radix threshold search cannot fire; the kernel must + # fall back to the solo trivial path (select all valid columns, pad -1). + if batch_size == 1 and index_topk == 16384 and num_tokens == 32768: + eff = 12000 # chunk (8192) < eff <= top_k + logits = torch.full((1, num_tokens), float("-inf"), dtype=torch.float32, device="cuda") + logits[0, :eff] = torch.randn(eff, dtype=torch.float32, device="cuda") + seq_lens = torch.tensor([eff], dtype=torch.int32, device="cuda") + got = run_indexer(logits, seq_lens)[0].cpu() + valid = got[got != -1] + assert valid.numel() == eff, f"expected {eff} valid, got {valid.numel()}" + assert set(valid.tolist()) == set(range(eff)), "selected set != all valid columns" + assert (got == -1).sum().item() == index_topk - eff, "wrong -1 pad count" @pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") @skip_pre_blackwell -@pytest.mark.parametrize("batch_size", [1, 4, 8, 16, 256]) -@pytest.mark.parametrize("next_n", [1, 2, 3]) -@pytest.mark.parametrize("index_topk", [2048]) -@pytest.mark.parametrize("num_tokens", [32768, 65536, 131072, 262144]) -@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) -def test_cute_dsl_topk_decode_single_pass_multi_cta( - batch_size, next_n, index_topk, num_tokens, dtype -): - """Correctness test for CuTE DSL single-pass multi-CTA TopK on Blackwell.""" - _run_cute_dsl_topk_test( - batch_size, - next_n, - index_topk, - num_tokens, +@pytest.mark.parametrize("top_k", [1023, 2047, 2048]) +@pytest.mark.parametrize("dtype_name", ["float32", "float16", "bfloat16"]) +def test_filtered_topk_varlen_odd_k(top_k, dtype_name): + """The filtered varlen kernel supports odd top_k (scalar output tail). + + The reference check inside the runner asserts bitwise agreement with a + torch reference; 2048 is the even control. The scalar tail write is dtype + dependent, so all three supported dtypes are exercised. + """ + import cutlass + + from tensorrt_llm._torch.cute_dsl_kernels.blackwell.top_k.filtered_top_k_decode_varlen import ( + run_topk_decode, + ) + + dtype = { + "float32": cutlass.Float32, + "float16": cutlass.Float16, + "bfloat16": cutlass.BFloat16, + }[dtype_name] + + run_topk_decode( dtype, - lambda logits, - seq_lens: cute_dsl_custom_ops.CuteDSLTopKDecodeSinglePassMultiCTARunner.forward( - input_values=logits, - seq_lens=seq_lens, - top_k=index_topk, - next_n=next_n, - return_val=False, - num_copy_bits=256, - )[0], + batch_size=16, + max_num_cols=4096, + top_k=top_k, + next_n=3, + do_benchmark=False, ) @@ -1294,17 +1324,15 @@ def generate_pre_idx_v4( return pre_idx +# radix filter single-cta test. @pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") @skip_pre_blackwell @pytest.mark.parametrize("batch_size", [1, 4, 64, 256]) @pytest.mark.parametrize("next_n", [1, 3]) -@pytest.mark.parametrize("index_topk", [2048]) +@pytest.mark.parametrize("index_topk", [512, 1024, 2048]) @pytest.mark.parametrize("num_tokens", [4096, 8192]) @pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("load_balance", [False, True]) -def test_cute_dsl_topk_decode_single_cta( # noqa: F811 - batch_size, next_n, index_topk, num_tokens, dtype, load_balance -): +def test_cute_dsl_radix_topk_decode_single_cta(batch_size, next_n, index_topk, num_tokens, dtype): """Correctness test for CuTE DSL single-CTA TopK decode on Blackwell.""" _run_cute_dsl_topk_test( batch_size, @@ -1318,21 +1346,21 @@ def test_cute_dsl_topk_decode_single_cta( # noqa: F811 top_k=index_topk, next_n=next_n, num_copy_bits=256, - load_balance=load_balance, ), ) +# radix filter 2-pass multi-cta test. @pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") @skip_pre_blackwell @pytest.mark.parametrize("batch_size", [1, 4, 64]) @pytest.mark.parametrize("next_n", [1, 3]) -@pytest.mark.parametrize("index_topk", [2048]) +@pytest.mark.parametrize("index_topk", [512, 1024, 2048]) @pytest.mark.parametrize("num_tokens", [32768, 65536]) @pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) @pytest.mark.parametrize("chunk_size_per_cta", [16384]) @pytest.mark.parametrize("dynamic", [False, True]) -def test_cute_dsl_topk_decode_multi_cta( # noqa: F811 +def test_cute_dsl_radix_topk_decode_multi_cta( batch_size, next_n, index_topk, num_tokens, dtype, chunk_size_per_cta, dynamic ): """Correctness test for CuTE DSL multi-CTA TopK decode on Blackwell.""" @@ -1354,13 +1382,14 @@ def test_cute_dsl_topk_decode_multi_cta( # noqa: F811 ) +# cute dsl top-k public interface test, tuned between single-cta and multi-cta. @pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") @skip_pre_blackwell @pytest.mark.parametrize("batch_size", [1, 4, 64, 128]) @pytest.mark.parametrize("next_n", [1, 2]) -@pytest.mark.parametrize("index_topk", [2048]) +@pytest.mark.parametrize("index_topk", [512, 1024, 2048]) @pytest.mark.parametrize("num_tokens", [4096, 8192, 65536, 131072]) -def test_cute_dsl_indexer_topk_decode(batch_size, next_n, index_topk, num_tokens): # noqa: F811 +def test_cute_dsl_indexer_radix_topk_decode(batch_size, next_n, index_topk, num_tokens): """Correctness test for CuTE DSL indexer TopK decode with in-place output.""" num_gen_tokens = batch_size * next_n @@ -1387,14 +1416,15 @@ def run_fn(logits, seq_lens): ) +# radix select multi-cta (atomicAdd gmem for cta communication) test. @pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") @skip_pre_blackwell @pytest.mark.parametrize("batch_size", [1, 16, 256]) @pytest.mark.parametrize("next_n", [1, 3]) -@pytest.mark.parametrize("index_topk", [2048]) +@pytest.mark.parametrize("index_topk", [512, 1024, 2048]) @pytest.mark.parametrize("num_tokens", [32768, 131072]) @pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) -def test_cute_dsl_topk_decode_single_pass_multi_cta( # noqa: F811 +def test_cute_dsl_radix_topk_decode_single_pass_multi_cta( batch_size, next_n, index_topk, num_tokens, dtype ): """Correctness test for CuTE DSL single-pass multi-CTA TopK on Blackwell.""" @@ -1416,14 +1446,15 @@ def test_cute_dsl_topk_decode_single_pass_multi_cta( # noqa: F811 ) +# radix select multi-cta cluster test. @pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") @skip_pre_blackwell @pytest.mark.parametrize("batch_size", [1, 16, 256]) @pytest.mark.parametrize("next_n", [1, 3]) -@pytest.mark.parametrize("index_topk", [2048]) +@pytest.mark.parametrize("index_topk", [512, 1024, 2048]) @pytest.mark.parametrize("num_tokens", [32768, 131072]) @pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) -def test_cute_dsl_topk_decode_single_pass_multi_cta_cluster( +def test_cute_dsl_radix_topk_decode_single_pass_multi_cta_cluster( batch_size, next_n, index_topk, num_tokens, dtype ): """Correctness test for CuTE DSL single-pass multi-CTA cluster TopK on Blackwell.""" @@ -1452,6 +1483,166 @@ def run_fn(logits, seq_lens): ) +# radix filter single-pass multi-cta test. +@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") +@skip_pre_blackwell +@pytest.mark.parametrize("batch_size", [1, 16]) +@pytest.mark.parametrize("next_n", [1, 3]) +@pytest.mark.parametrize("index_topk", [512, 1024, 2048]) +@pytest.mark.parametrize("num_tokens", [32768, 131072]) +# fp32 (4 refine rounds -> 5 histogram merges) MUST be covered; the +# merge-placement path only fully exercises on fp32. +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("cluster_size", [2, 4]) +def test_cute_dsl_radix_filter_topk_decode_single_pass_multi_cta( + batch_size, next_n, index_topk, num_tokens, dtype, cluster_size +): + """Correctness test for the radix-FILTER single-pass multi-CTA (cluster) + decode top-k. num_tokens is large enough that ceil(seq/chunk) >= 2 for the + longer rows, exercising the DSMEM histogram-merge + prefix-scan collection. + """ + _run_cute_dsl_topk_test( + batch_size, + next_n, + index_topk, + num_tokens, + dtype, + lambda logits, + seq_lens: cute_dsl_custom_ops.CuteDSLTopKDecodeRadixFilterSPMultiCTARunner.forward( + input_values=logits, + seq_lens=seq_lens, + top_k=index_topk, + next_n=next_n, + cluster_size=cluster_size, + return_val=False, + num_copy_bits=256, + )[0], + ) + + +def _rf_sp_check(logits, seq_lens, top_k, next_n, cluster_size): + """Value-based correctness check for radix-filter SP multi-CTA on custom + inputs (controls the value distribution for edge cases). + """ + idx, _ = cute_dsl_custom_ops.CuteDSLTopKDecodeRadixFilterSPMultiCTARunner.forward( + input_values=logits, + seq_lens=seq_lens, + top_k=top_k, + next_n=next_n, + cluster_size=cluster_size, + return_val=False, + num_copy_bits=256, + ) + torch.cuda.synchronize() + num_rows = logits.shape[0] + for r in range(num_rows): + b = r // next_n + off = r % next_n + eff = int(seq_lens[b].item()) - next_n + off + 1 + k = min(top_k, eff) + row = logits[r, :eff] + ref = row.topk(k)[0].sort(descending=True)[0].float() + gi = idx[r, :k].long() + gi = gi[(gi >= 0) & (gi < eff)] + got = row[gi].sort(descending=True)[0].float() + assert got.numel() == ref.numel() and torch.allclose(got, ref, atol=1e-3), ( + f"row {r}: mismatch (got {got.numel()} vs ref {ref.numel()})" + ) + + +# radix filter single-pass multi-cta group-2-heavy test. +@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") +@skip_pre_blackwell +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("cluster_size", [2, 4]) +@pytest.mark.parametrize("index_topk", [512, 1024]) +def test_cute_dsl_radix_filter_sp_multi_cta_group2_heavy(dtype, cluster_size, index_topk): + """Group-2-heavy: quantized logits create many exact ties at the threshold + bin, spread across CTAs -> exercises per-CTA s_last_remain, exclusive_offset_2 + and the pos heavy ties. + logits = ((torch.randn(batch, num_tokens, device="cuda") * 4).round() / 4).to(dtype) + _rf_sp_check(logits, seq_lens, index_topk, 1, cluster_size) + + +# radix filter single-pass multi-cta solo and degrade test. +@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") +@skip_pre_blackwell +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("cluster_size", [2, 4]) +def test_cute_dsl_radix_filter_sp_multi_cta_solo_and_degrade(dtype, cluster_size): + """Short rows: needed_ctas==1 (solo fast path, extra CTAs exit -> no cluster + sync, no deadlock) and 2<=needed2048) and odd top_k stay bit-exact on the single-pass multi-CTA + and cluster decode paths. + + The raised wrapper guard forwards these values to both dispatch paths, so + this covers the odd-K scalar output write on routes the other tests only + exercise at top_k=2048. + """ + num_tokens = 131072 + next_n = 1 + + if use_cluster: + runner = cute_dsl_custom_ops.CuteDSLTopKDecodeSinglePassMultiCTAClusterRunner + else: + runner = cute_dsl_custom_ops.CuteDSLTopKDecodeSinglePassMultiCTARunner + + def run_fn(logits, seq_lens): + result = runner.forward( + input_values=logits, + seq_lens=seq_lens, + top_k=index_topk, + next_n=next_n, + return_val=False, + num_copy_bits=256, + ) + # The cluster runner returns None when the problem exceeds its capacity. + if result[0] is None: + pytest.skip("Problem size exceeds cluster kernel capacity") + return result[0] + + _run_cute_dsl_topk_test( + batch_size, + next_n, + index_topk, + num_tokens, + dtype, + run_fn, + ) + + # ============================================================================ # Heuristic Decode Distribution-Parameterised Tests # ============================================================================ @@ -1727,3 +1918,435 @@ def test_indexer_topk_decode_dist_v4_cr4( _run_indexer_topk_decode_v4_gvr_check( batch_size, next_n, index_topk, num_tokens, dtype, dist_cfg, success_ratio ) + + +# ============================================================================ +# CuTE DSL Prefill Top-K Tests +# ============================================================================ + + +def _run_cute_dsl_topk_prefill_test(batch_size, index_topk, num_tokens, dtype, row_start_offset=0): + """Common test logic for CuTE DSL prefill top-k kernel. + + Tests that the DSL kernel outputs LOCAL indices (0-indexed within each + row's valid range) matching torch.topk. + + Args: + row_start_offset: Fixed offset added to all row_starts. 0 = standard + zero-start case; nonzero exercises the subtract_row_start_on_output path. + """ + torch.manual_seed(77) + torch.cuda.manual_seed(77) + + seq_lens = generate_seq_lens(batch_size, index_topk, num_tokens) + num_rows = int(seq_lens.sum().item()) + + # Build per-row token counts from seq_lens (same logic as CUDA prefill tests). + row_indices = torch.arange(1, seq_lens.max() + 1, dtype=torch.int32, device="cuda") + row_lengths = row_indices.expand(seq_lens.size(0), -1)[ + row_indices.expand(seq_lens.size(0), -1) <= seq_lens.unsqueeze(1) + ].contiguous() # shape: (num_rows,) + + # Apply fixed offset: row_start = offset, row_end = offset + row_length. + row_starts = torch.full((num_rows,), row_start_offset, dtype=torch.int32, device="cuda") + row_ends = (row_starts + row_lengths).contiguous() + + logits = create_random_logits(row_starts, row_ends, dtype, 77) + + # Run DSL kernel — writes LOCAL indices (0-indexed within [row_start, row_end)) + # into the caller-provided output_indices buffer. + output_indices = torch.empty((num_rows, index_topk), dtype=torch.int32, device="cuda") + torch.ops.trtllm.cute_dsl_indexer_topk_prefill_blackwell( + logits, row_starts, row_ends, output_indices, index_topk, 256 + ) + torch.cuda.synchronize() + + # Reference: torch.topk returns ABSOLUTE indices; convert to LOCAL. + # logits has -inf outside [row_start, row_end), so topk selects valid cols. + max_row_end = int(row_ends.max().item()) + torch_abs = logits.topk(min(index_topk, max_row_end), dim=-1)[1].to(torch.int32) + torch_local = torch_abs - row_starts.unsqueeze(1) + valid = (torch_abs >= row_starts.unsqueeze(1)) & (torch_abs < row_ends.unsqueeze(1)) + torch_local = torch_local.masked_fill(~valid, -1) + + # compare_top_k_results expects LOCAL indices from both sides; it converts + # to absolute internally (+ row_start) for logit-value comparison. + assert compare_top_k_results( + logits, output_indices, torch_local, row_starts, row_ends, index_topk + ), "CuTE DSL prefill top-k results don't match torch.topk" + + +@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") +@skip_pre_blackwell +@pytest.mark.parametrize("batch_size", [1, 4, 32]) +@pytest.mark.parametrize("index_topk", [512, 1024, 2048]) +@pytest.mark.parametrize("num_tokens", [4096, 8192]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_cute_dsl_radix_topk_prefill_zero_row_starts(batch_size, index_topk, num_tokens, dtype): + """Correctness test for CuTE DSL prefill top-k with row_start=0 (all rows). + + Note: batch_size here is the number of sequences; num_rows = sum(seq_lens) + can be much larger. Keep batch_size small to avoid OOM on large num_tokens. + """ + _run_cute_dsl_topk_prefill_test(batch_size, index_topk, num_tokens, dtype, row_start_offset=0) + + +@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") +@skip_pre_blackwell +@pytest.mark.parametrize("batch_size", [1, 4, 32]) +@pytest.mark.parametrize("index_topk", [512, 1024, 2048]) +@pytest.mark.parametrize("num_tokens", [4096, 8192]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_cute_dsl_radix_topk_prefill_nonzero_row_starts(batch_size, index_topk, num_tokens, dtype): + """Correctness test for CuTE DSL prefill top-k with nonzero row_starts. + + Uses a fixed offset = index_topk // 4 to verify that output indices are + properly localised (subtract_row_start_on_output path). + """ + _run_cute_dsl_topk_prefill_test( + batch_size, index_topk, num_tokens, dtype, row_start_offset=index_topk // 4 + ) + + +# =========================================================================== +# Overflow-policy correctness tests (moved from +# test_cute_dsl_topk_overflow_policy.py). Reuse the generate_seq_lens / +# compare_top_k_results / create_random_logits helpers defined above. +# +# Overflow policies: +# GMEM_SPILL - spill threshold-bucket overflow to a pre-allocated GMEM buffer (exact) +# TRUNCATE - discard overflow elements; histogram stays consistent (non-exact) +# REREAD_ALWAYS - first pass builds histogram only; second GMEM scan fills s_input_idx (exact) +# REREAD - optimistic SMEM collection; falls back to GMEM re-scan on overflow (exact) +# All policies are identical when num_cols <= smem_input_size; these cover the +# overflow scenarios. SMEM overflow thresholds (large_occupancy, B200): bf16/fp16 +# = 16384, fp32 = 8192; overflow occurs when num_cols > smem_input_size. +# =========================================================================== + + +def _build_torch_ref(logits, row_starts, row_ends, top_k): + """Exact-copy of the reference computation in existing tests.""" + max_row_len = int(row_ends.max().item()) + torch_indices = logits.topk(min(top_k, max_row_len), dim=-1)[1] + mask = (torch_indices >= 0) & ((torch_indices - (row_ends - row_starts)[:, None]) < 0) + torch_indices = torch_indices.masked_fill(~mask, -1) + return torch_indices + + +def _compare_truncate_result( + logits: torch.Tensor, + cuda_indices: torch.Tensor, + row_starts: torch.Tensor, + row_ends: torch.Tensor, + top_k: int, +) -> bool: + """TRUNCATE correctness check. + + TRUNCATE is non-exact: it returns top-K of the first smem_size elements + (memory order) in the threshold-coarse bin, NOT the global top-K. With + adversarial data layout the returned values can legitimately be below the + global k-th largest, so comparing against torch.topk would be incorrect. + + We verify instead: + 1. Each row returns exactly min(top_k, row_len) valid (non -1) elements. + 2. All returned indices are within [row_start, row_end). + 3. No duplicate indices within a row. + + cuda_indices must contain absolute indices into logits (not row-local). + """ + invalid_mask = cuda_indices == -1 # [num_rows, K] + valid_counts = (~invalid_mask).sum(dim=1) # [num_rows] + + # 1. count check + row_lens = row_ends - row_starts + expected = torch.minimum(row_lens, row_lens.new_full((), top_k)) + bad_rows = (valid_counts != expected).nonzero(as_tuple=True)[0] + if bad_rows.numel() > 0: + r = bad_rows[0].item() + print( + f"TRUNCATE Row {r}: expected {expected[r].item()} valid elements, got {valid_counts[r].item()}" + ) + return False + + # 2. bounds check: safe-clamp -1 slots to 0 before comparison, then mask them out + safe = cuda_indices.clone() + safe[invalid_mask] = 0 + out_of_bounds = ( + (safe < row_starts.unsqueeze(1)) | (safe >= row_ends.unsqueeze(1)) + ) & ~invalid_mask + bad_rows = out_of_bounds.any(dim=1).nonzero(as_tuple=True)[0] + if bad_rows.numel() > 0: + r = bad_rows[0].item() + bad_vals = safe[r][out_of_bounds[r]] + print( + f"TRUNCATE Row {r}: index out of [row_start={row_starts[r].item()}, " + f"row_end={row_ends[r].item()}): {bad_vals[:4].tolist()}" + ) + return False + + # 3. no-duplicate check: replace invalid with INT_MAX so they sort to the end, + # then compare adjacent entries that are not sentinel. + sentinel = torch.iinfo(torch.int32).max + safe_for_dup = cuda_indices.clone() + safe_for_dup[invalid_mask] = sentinel + sorted_dup = safe_for_dup.sort(dim=1).values # [num_rows, K] + dups = (sorted_dup[:, 1:] == sorted_dup[:, :-1]) & (sorted_dup[:, 1:] != sentinel) + bad_rows = dups.any(dim=1).nonzero(as_tuple=True)[0] + if bad_rows.numel() > 0: + r = bad_rows[0].item() + print(f"TRUNCATE Row {r}: duplicate indices found") + return False + + return True + + +# ── decode kernel run via wrapper ───────────────────────────────────────────── + + +def _run_decode_kernel(logits, seq_lens, top_k, next_n, overflow_policy): + """Run decode kernel via cute_dsl_radix_filter_topk_wrapper with given overflow_policy.""" + indices, _ = cute_dsl_radix_filter_topk_wrapper( + logits, + seq_lens, + top_k, + next_n, + return_val=False, + overflow_policy=overflow_policy, + ) + return indices + + +# ── shared test bodies ──────────────────────────────────────────────────────── + + +def _run_decode_policy_test(policy, batch_size, next_n, top_k, num_tokens, dtype, seed=42): + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + + num_gen_tokens = batch_size * next_n + row_starts = torch.zeros(num_gen_tokens, dtype=torch.int32, device="cuda") + row_indices = torch.arange(num_gen_tokens, device="cuda") // next_n + next_n_offset = torch.arange(num_gen_tokens, device="cuda") % next_n + + seq_lens = generate_seq_lens(batch_size, top_k, num_tokens) + seq_lens = seq_lens.clamp(min=next_n) + row_ends = seq_lens[row_indices] - next_n + next_n_offset + 1 + + logits = create_random_logits(row_starts, row_ends, dtype, seed) + + cuda_indices = _run_decode_kernel(logits, seq_lens, top_k, next_n, policy) + torch.cuda.synchronize() + + cuda_indices = cuda_indices.to(torch.int32) + + if policy == "TRUNCATE": + assert _compare_truncate_result(logits, cuda_indices, row_starts, row_ends, top_k), ( + f"TRUNCATE decode: invalid results (policy={policy})" + ) + else: + torch_indices = _build_torch_ref(logits, row_starts, row_ends, top_k) + assert compare_top_k_results( + logits, cuda_indices, torch_indices, row_starts, row_ends, top_k + ), f"Decode results mismatch vs torch.topk (policy={policy})" + + +def _run_prefill_policy_test( + policy, batch_size, top_k, num_tokens, dtype, row_start_offset=0, seed=77 +): + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + + seq_lens = generate_seq_lens(batch_size, top_k, num_tokens) + num_rows = int(seq_lens.sum().item()) + + row_indices = torch.arange(1, seq_lens.max() + 1, dtype=torch.int32, device="cuda") + row_lengths = row_indices.expand(seq_lens.size(0), -1)[ + row_indices.expand(seq_lens.size(0), -1) <= seq_lens.unsqueeze(1) + ].contiguous() + + row_starts = torch.full((num_rows,), row_start_offset, dtype=torch.int32, device="cuda") + row_ends = (row_starts + row_lengths).contiguous() + + logits = create_random_logits(row_starts, row_ends, dtype, seed) + + cuda_local, _ = CuteDSLTopKPrefillSingleCTARunner.forward( + logits, + row_starts, + row_ends, + top_k, + return_val=False, + overflow_policy=policy, + ) + + if policy == "TRUNCATE": + # TRUNCATE returns LOCAL indices; convert reference to LOCAL too. + max_row_end = int(row_ends.max().item()) + torch_abs = logits.topk(min(top_k, max_row_end), dim=-1)[1].to(torch.int32) + torch_local = torch_abs - row_starts.unsqueeze(1) + valid = (torch_abs >= row_starts.unsqueeze(1)) & (torch_abs < row_ends.unsqueeze(1)) + torch_local = torch_local.masked_fill(~valid, -1) + + # For TRUNCATE with local indices: build absolute cuda_indices for comparison. + cuda_abs = cuda_local.clone() + not_neg1 = cuda_abs != -1 + cuda_abs[not_neg1] = ( + cuda_abs[not_neg1] + row_starts.unsqueeze(1).expand_as(cuda_abs)[not_neg1] + ) + # compare_truncate expects absolute indices in logits space and row_starts=0 for absolute check. + # Easier: rebuild logits for absolute-index comparison using row_starts. + assert _compare_truncate_result(logits, cuda_abs, row_starts, row_ends, top_k), ( + f"TRUNCATE prefill: invalid results (policy={policy})" + ) + else: + # Exact policy: compare LOCAL indices using compare_top_k_results. + max_row_end = int(row_ends.max().item()) + torch_abs = logits.topk(min(top_k, max_row_end), dim=-1)[1].to(torch.int32) + torch_local = torch_abs - row_starts.unsqueeze(1) + valid = (torch_abs >= row_starts.unsqueeze(1)) & (torch_abs < row_ends.unsqueeze(1)) + torch_local = torch_local.masked_fill(~valid, -1) + assert compare_top_k_results( + logits, cuda_local, torch_local, row_starts, row_ends, top_k + ), f"Prefill results mismatch vs torch.topk (policy={policy})" + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Parametrized tests +# ═══════════════════════════════════════════════════════════════════════════════ + +_ALL_POLICIES = ["GMEM_SPILL", "TRUNCATE", "REREAD_ALWAYS", "REREAD"] + +# ---------------------------------------------------------------------------- +# Decode single-CTA — overflow (num_tokens > smem_input_size) +# bf16: smem_size=16384 → overflow at 32768 +# fp32: smem_size=8192 → overflow at 16384 +# Use large batch_size (256) to trigger large_occupancy path. +# ---------------------------------------------------------------------------- + + +@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") +@skip_pre_blackwell +@pytest.mark.parametrize("overflow_policy", _ALL_POLICIES) +@pytest.mark.parametrize("batch_size", [256]) +@pytest.mark.parametrize("next_n", [1, 3]) +@pytest.mark.parametrize("top_k", [512, 1024, 2048]) +@pytest.mark.parametrize( + "num_tokens, dtype", + [ + (32768, torch.bfloat16), # bf16: 32768 > smem_size 16384 → overflow + (65536, torch.bfloat16), # bf16: 65536 >> smem_size + (16384, torch.float32), # fp32: 16384 > smem_size 8192 → overflow + (32768, torch.float32), # fp32: 32768 >> smem_size + ], +) +def test_decode_overflow_policy_overflow( + overflow_policy, batch_size, next_n, top_k, num_tokens, dtype +): + """Decode single-CTA: all policies, large num_tokens (SMEM overflow triggered).""" + _run_decode_policy_test(overflow_policy, batch_size, next_n, top_k, num_tokens, dtype) + + +# ---------------------------------------------------------------------------- +# Decode — REREAD policy: enable_reread=True but did_overflow=False at runtime +# +# REREAD has two compile-time conditions: +# enable_reread=True requires num_tokens > filtered_topk_smem_input_size +# (large_occ bf16: 16384, fp32: 8192) +# did_overflow=False at runtime: threshold-bin count fits in SMEM +# +# With uniform random data and top_k=512, the threshold bin holds +# ~num_tokens / 256 ≈ 128 elements (bf16@32768) or 64 elements (fp32@16384), +# both far below smem_input_size → did_overflow=False → SMEM-refinement path. +# +# This test exercises the REREAD path that test_decode_overflow_policy_no_overflow +# cannot cover (those use num_tokens ≤ smem_size → enable_reread=False). +# ---------------------------------------------------------------------------- + + +@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") +@skip_pre_blackwell +@pytest.mark.parametrize("batch_size", [1, 64, 256]) +@pytest.mark.parametrize("next_n", [1, 3]) +@pytest.mark.parametrize("top_k", [512, 2048]) +@pytest.mark.parametrize( + "num_tokens, dtype", + [ + (32768, torch.bfloat16), # large_occ bf16: enable_reread=True, threshold-bin ~128 << 16384 + (16384, torch.float32), # large_occ fp32: enable_reread=True, threshold-bin ~64 << 8192 + ], +) +def test_decode_reread_no_smem_overflow(batch_size, next_n, top_k, num_tokens, dtype): + """REREAD: enable_reread=True (num_tokens > smem_size) but runtime did_overflow=False. + + Exercises the SMEM-based refinement path inside the REREAD policy. + """ + _run_decode_policy_test("REREAD", batch_size, next_n, top_k, num_tokens, dtype) + + +# ---------------------------------------------------------------------------- +# Decode — small batch (no large_occupancy), overflow path uses base-class smem_size +# Base-class: bf16 Uint16 num_buffer=1 → max_smem=64K → smem_size=min(64K, max_cols) +# Only overflows at very large num_tokens (> 65536); use 131072. +# ---------------------------------------------------------------------------- + + +@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") +@skip_pre_blackwell +@pytest.mark.parametrize("overflow_policy", _ALL_POLICIES) +@pytest.mark.parametrize("batch_size", [1, 4]) +@pytest.mark.parametrize("next_n", [1]) +@pytest.mark.parametrize("top_k", [512, 2048]) +@pytest.mark.parametrize( + "num_tokens, dtype", + [ + (131072, torch.bfloat16), + (131072, torch.float32), + ], +) +def test_decode_overflow_policy_small_batch_overflow( + overflow_policy, batch_size, next_n, top_k, num_tokens, dtype +): + """Decode small batch (no large_occupancy): overflow at very large num_tokens.""" + _run_decode_policy_test(overflow_policy, batch_size, next_n, top_k, num_tokens, dtype) + + +# ---------------------------------------------------------------------------- +# Prefill — overflow (num_tokens > smem_input_size) +# Prefill large_occupancy smem_input_size: bf16 Uint16 num_buffer=1 → 16384 +# fp32 Uint16 num_buffer=2 → 8192 +# ---------------------------------------------------------------------------- + + +@pytest.mark.skipif(not IS_CUTLASS_DSL_AVAILABLE, reason="CuTE DSL not available") +@skip_pre_blackwell +@pytest.mark.parametrize("overflow_policy", _ALL_POLICIES) +@pytest.mark.parametrize("batch_size", [1]) +@pytest.mark.parametrize("top_k", [512, 1024, 2048]) +@pytest.mark.parametrize( + "num_tokens, dtype", + [ + (20480, torch.bfloat16), # bf16: 20480 > smem_size 16384 → overflow + (12288, torch.float32), # fp32: 12288 > smem_size 8192 → overflow + ], +) +@pytest.mark.parametrize("row_start_offset", [256]) +def test_prefill_overflow_policy_overflow( + overflow_policy, batch_size, top_k, num_tokens, dtype, row_start_offset +): + """Prefill: all policies, large num_tokens (SMEM overflow triggered).""" + # Skip combinations where the logits tensor would exceed ~80 GB. + # logits shape = (sum_seq_lens, num_tokens); sum_seq_lens ≈ batch_size * num_tokens / 2. + dtype_bytes = 2 if dtype == torch.bfloat16 else 4 + est_gb = batch_size * (num_tokens // 2) * num_tokens * dtype_bytes / (1024**3) + if est_gb > 80: + pytest.skip( + f"Estimated logits tensor ~{est_gb:.0f} GB exceeds memory budget; " + f"use smaller batch_size or num_tokens" + ) + _run_prefill_policy_test( + overflow_policy, + batch_size, + top_k, + num_tokens, + dtype, + row_start_offset=row_start_offset, + ) diff --git a/tests/unittest/_torch/thop/serial/test_sparse_kv_cache_compact.py b/tests/unittest/_torch/thop/serial/test_sparse_kv_cache_compact.py new file mode 100644 index 000000000000..ca40c09667b9 --- /dev/null +++ b/tests/unittest/_torch/thop/serial/test_sparse_kv_cache_compact.py @@ -0,0 +1,442 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the layered V2 sparse-KV compaction op (pipelined bf16 kernels).""" + +from typing import NamedTuple, Optional + +import pytest +import torch + +import tensorrt_llm # noqa: F401 # Register torch.ops.trtllm operators. + +_TOKENS_PER_BLOCK = 32 +_NUM_KV_HEADS = 2 +_BATCH_SIZE = 2 +_PAGE_INDEX_DIVISOR = 2 + +# Profiler probe names: the pipelined bf16 kernels are the only shipped +# path; the retired register-staging kernel must never appear. +_FAST_KERNEL_NAME = "sparseKvCacheCompactV2Bf16PipelineKernel" +_RETIRED_KERNEL_NAME = "updateSparseKvCacheAfterFmha" + + +def _encode_k_block_offsets( + page_table: torch.Tensor, page_index_scale: int = _PAGE_INDEX_DIVISOR +) -> torch.Tensor: + encoded = torch.empty( + page_table.shape[0], + 2, + page_table.shape[1], + dtype=torch.int32, + device=page_table.device, + ) + encoded[:, 0] = page_table * page_index_scale + encoded[:, 1] = encoded[:, 0] + 1 + return encoded[:, 0] + + +class _DeviceArguments(NamedTuple): + pool_pointers: torch.Tensor + source_indices: torch.Tensor + source_offsets: torch.Tensor + source_layer_indices: Optional[torch.Tensor] + + +def _make_pools( + num_layers: int, + dtype: torch.dtype, + head_dim: int, + page_index_scale: int = _PAGE_INDEX_DIVISOR, +) -> tuple[list[torch.Tensor], list[torch.Tensor], list[torch.Tensor]]: + pages_per_seq = 3 + num_pages = _BATCH_SIZE * pages_per_seq * page_index_scale // _PAGE_INDEX_DIVISOR + shape = ( + num_pages, + 2, + _NUM_KV_HEADS, + _TOKENS_PER_BLOCK, + head_dim, + ) + numel = torch.Size(shape).numel() + pools_cpu = [ + ((torch.arange(numel, dtype=torch.int32) + layer * 37) % 251).reshape(shape).to(dtype) + for layer in range(num_layers) + ] + pools = [pool.cuda() for pool in pools_cpu] + raw_pages = [[4, 1, 5], [2, 0, 3]] + assert set(raw_pages[0]).isdisjoint(raw_pages[1]) + raw_page_table = torch.tensor(raw_pages, dtype=torch.int32, device="cuda") + page_table = _encode_k_block_offsets(raw_page_table, page_index_scale) + page_tables = [page_table] * num_layers + assert page_tables[0].stride(0) == 2 * page_tables[0].shape[1] + return pools_cpu, pools, page_tables + + +def _device_arguments( + pools: list[torch.Tensor], + source_indices: torch.Tensor, + source_offsets: torch.Tensor, + source_layer_indices: Optional[torch.Tensor] = None, +) -> _DeviceArguments: + device = pools[0].device + return _DeviceArguments( + pool_pointers=torch.tensor( + [pool.data_ptr() for pool in pools], dtype=torch.int64, device=device + ), + source_indices=source_indices.to(device), + source_offsets=source_offsets.to(device), + source_layer_indices=( + None if source_layer_indices is None else source_layer_indices.to(device) + ), + ) + + +def _reference_compact( + pools: list[torch.Tensor], + page_tables: list[torch.Tensor], + source_indices: torch.Tensor, + source_offsets: torch.Tensor, + destination_base: "int | list[int]", + source_layer_indices: Optional[torch.Tensor] = None, + tokens_per_block: int = _TOKENS_PER_BLOCK, + batch_size: int = _BATCH_SIZE, +) -> list[torch.Tensor]: + original = [pool.clone() for pool in pools] + expected = [pool.clone() for pool in pools] + for group_layer, (source_pool, destination_pool, page_table) in enumerate( + zip(original, expected, page_tables) + ): + # The kernel decodes K offsets as offset // 2 regardless of the + # encoder's scale; the reference mirrors the kernel. + raw_page_table = page_table // _PAGE_INDEX_DIVISOR + if source_indices.ndim == 2: + layer_sources = source_indices + else: + assert source_layer_indices is not None + layer_sources = source_indices[int(source_layer_indices[group_layer])] + for request in range(batch_size): + begin = int(source_offsets[request]) + end = int(source_offsets[request + 1]) + request_base = ( + destination_base[request] + if isinstance(destination_base, (list, tuple)) + else destination_base + ) + for head in range(layer_sources.shape[0]): + for request_move, global_move in enumerate(range(begin, end)): + source_token = int(layer_sources[head, global_move]) + destination_token = request_base + request_move + source_page = int(raw_page_table[request, source_token // tokens_per_block]) + destination_page = int( + raw_page_table[request, destination_token // tokens_per_block] + ) + destination_pool[ + destination_page, + :, + head, + destination_token % tokens_per_block, + :, + ] = source_pool[ + source_page, + :, + head, + source_token % tokens_per_block, + :, + ] + return expected + + +def _compact( + pools: list[torch.Tensor], + page_tables: list[torch.Tensor], + arguments: _DeviceArguments, + destination_base: "int | list[int]", + batch_size: int = _BATCH_SIZE, +) -> None: + # The op takes per-request destination bases; scalar test parameters are + # broadcast to the batch here. torch.full stays CUDA-graph-capturable. + if isinstance(destination_base, int): + destination_bases = torch.full( + (batch_size,), destination_base, dtype=torch.int32, device="cuda" + ) + else: + destination_bases = torch.tensor(destination_base, dtype=torch.int32, device="cuda") + torch.ops.trtllm.sparse_kv_cache_compact_layers( + pools, + arguments.pool_pointers, + page_tables[0], + arguments.source_indices, + arguments.source_offsets, + destination_bases, + arguments.source_layer_indices, + ) + + +_SMALL_ROW = [2, 5, 8, 3, 7, 10] +# The production-shaped fast-geometry matrix below is the byte-exact anchor +# (both head dims, both page sizes, per-request destination bases, 3-D +# per-layer routing, multi-tile ragged moves). These two rows keep the +# op-level contracts it does not pin: the destination-base-0 (prompt 0) +# boundary and the fixed //2 K-offset decode against a scale-4 encoder. +_LAYER_CASES = [ + pytest.param(dict(head_dim=64, dest=0, scale=2), id="bf16_h64_dest0_scale2"), + pytest.param(dict(head_dim=64, dest=2, scale=4), id="bf16_h64_dest2_scale4"), +] + + +@pytest.mark.parametrize("case", _LAYER_CASES) +def test_sparse_kv_cache_compact_layers(case): + pools_cpu, pools, page_tables = _make_pools(3, torch.bfloat16, case["head_dim"], case["scale"]) + page_tables_cpu = [page_table.cpu() for page_table in page_tables] + source_offsets = torch.tensor((0, 3, 6), dtype=torch.int32) + source_row = torch.tensor(_SMALL_ROW, dtype=torch.int32) + source_indices = source_row.view(1, -1).expand(_NUM_KV_HEADS, -1).contiguous() + destination_base = case["dest"] + expected = _reference_compact( + pools_cpu, + page_tables_cpu, + source_indices, + source_offsets, + destination_base, + ) + arguments = _device_arguments(pools, source_indices, source_offsets) + + _compact(pools, page_tables, arguments, destination_base) + torch.cuda.synchronize() + + for actual, reference in zip(pools, expected): + assert torch.equal(actual.cpu(), reference) + + +def test_sparse_kv_cache_compact_layers_cuda_graph_replay(): + """Check operation-level capture safety inside an externally captured graph.""" + pools_cpu, pools, page_tables = _make_pools(3, torch.bfloat16, 64) + page_tables_cpu = [page_table.cpu() for page_table in page_tables] + source_offsets = torch.tensor([0, 3, 6], dtype=torch.int32) + source_row = torch.tensor(_SMALL_ROW, dtype=torch.int32) + source_indices = source_row.view(1, -1).expand(_NUM_KV_HEADS, -1).contiguous() + replay_row = torch.tensor([3, 6, 9, 2, 5, 8], dtype=torch.int32) + replay_indices = replay_row.view(1, -1).expand(_NUM_KV_HEADS, -1).contiguous() + destination_base = 2 + expected = _reference_compact( + pools_cpu, + page_tables_cpu, + replay_indices, + source_offsets, + destination_base, + ) + arguments = _device_arguments(pools, source_indices, source_offsets) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + _compact(pools, page_tables, arguments, destination_base) + + for pool, initial in zip(pools, pools_cpu): + pool.copy_(initial) + arguments.source_indices.copy_(replay_indices) + graph.replay() + torch.cuda.synchronize() + + for actual, reference in zip(pools, expected): + assert torch.equal(actual.cpu(), reference) + + +# --- Production-shaped geometry for the pipelined bf16 fast path ---------- + +_FAST_BATCH_SIZE = 3 +# Ragged+full tiles, an empty request, and a prologue/epilogue-only tile. +_FAST_MOVE_COUNTS = (71, 0, 29) +# Mixed prompt lengths: none tile- or page-aligned. +_FAST_DESTINATION_BASES = [3, 9, 17] +# Allocation-wide index buffers: padding past the round's total move count +# makes a device-derived stride read padding and fail the byte-compare. +_FAST_SOURCE_PAD = 37 +_FAST_IDENTITY_MOVES = 5 + + +class _FastGeometryCase(NamedTuple): + pools_cpu: list[torch.Tensor] + pools: list[torch.Tensor] + page_tables_cpu: list[torch.Tensor] + page_tables: list[torch.Tensor] + source_indices: torch.Tensor + source_offsets: torch.Tensor + source_layer_indices: Optional[torch.Tensor] + destination_bases: list[int] + tokens_per_block: int + batch_size: int + + +def _fast_sources_row( + base: int, count: int, limit: int, generator: torch.Generator +) -> torch.Tensor: + """Distinct sorted source tokens >= base, so src(i) >= base + i (the op's + in-place-safety contract). The first few moves are identities (src == dst), + which the kernel skips storing.""" + if count == 0: + return torch.empty(0, dtype=torch.int32) + identity = min(_FAST_IDENTITY_MOVES, count) + candidates = torch.arange(base + identity, limit, dtype=torch.int32) + picks = torch.randperm(candidates.numel(), generator=generator)[: count - identity] + tail = candidates[picks].sort().values + return torch.cat((torch.arange(base, base + identity, dtype=torch.int32), tail)) + + +def _make_fast_geometry_case( + head_dim: int, + tokens_per_block: int, + dtype: torch.dtype = torch.bfloat16, + num_layers: int = 2, + per_layer_sources: bool = False, +) -> _FastGeometryCase: + batch_size = _FAST_BATCH_SIZE + pages_per_seq = max(2, 128 // tokens_per_block) + tokens_per_seq = pages_per_seq * tokens_per_block + num_pages = batch_size * pages_per_seq + shape = (num_pages, 2, _NUM_KV_HEADS, tokens_per_block, head_dim) + numel = torch.Size(shape).numel() + pools_cpu = [ + ((torch.arange(numel, dtype=torch.int32) + layer * 37) % 251).reshape(shape).to(dtype) + for layer in range(num_layers) + ] + pools = [pool.cuda() for pool in pools_cpu] + + # Deterministic per-geometry inputs so any failure reproduces exactly. + generator = torch.Generator().manual_seed(20260720 + head_dim * 1000 + tokens_per_block) + raw_page_table = ( + torch.randperm(num_pages, generator=generator) + .to(torch.int32) + .reshape(batch_size, pages_per_seq) + .cuda() + ) + page_table = _encode_k_block_offsets(raw_page_table) + page_tables = [page_table] * num_layers + assert page_tables[0].stride(0) == 2 * pages_per_seq + page_tables_cpu = [table.cpu() for table in page_tables] + + offsets = [0] + for count in _FAST_MOVE_COUNTS: + offsets.append(offsets[-1] + count) + source_offsets = torch.tensor(offsets, dtype=torch.int32) + width = offsets[-1] + _FAST_SOURCE_PAD + source_layers = 3 if per_layer_sources else 1 + # Padding is a valid token id so a stride bug corrupts output (caught by + # the byte-compare) instead of faulting. + rows = torch.full((source_layers, _NUM_KV_HEADS, width), tokens_per_seq - 1, dtype=torch.int32) + for layer in range(source_layers): + for head in range(_NUM_KV_HEADS): + cursor = 0 + for request, count in enumerate(_FAST_MOVE_COUNTS): + rows[layer, head, cursor : cursor + count] = _fast_sources_row( + _FAST_DESTINATION_BASES[request], count, tokens_per_seq, generator + ) + cursor += count + if per_layer_sources: + source_indices = rows.contiguous() + source_layer_indices = torch.tensor([2, 0], dtype=torch.int32) + else: + source_indices = rows[0].contiguous() + source_layer_indices = None + + return _FastGeometryCase( + pools_cpu=pools_cpu, + pools=pools, + page_tables_cpu=page_tables_cpu, + page_tables=page_tables, + source_indices=source_indices, + source_offsets=source_offsets, + source_layer_indices=source_layer_indices, + destination_bases=list(_FAST_DESTINATION_BASES), + tokens_per_block=tokens_per_block, + batch_size=batch_size, + ) + + +def _run_fast_geometry_case(case: _FastGeometryCase) -> list[torch.Tensor]: + expected = _reference_compact( + case.pools_cpu, + case.page_tables_cpu, + case.source_indices, + case.source_offsets, + case.destination_bases, + case.source_layer_indices, + tokens_per_block=case.tokens_per_block, + batch_size=case.batch_size, + ) + arguments = _device_arguments( + case.pools, case.source_indices, case.source_offsets, case.source_layer_indices + ) + _compact( + case.pools, + case.page_tables, + arguments, + case.destination_bases, + batch_size=case.batch_size, + ) + torch.cuda.synchronize() + return expected + + +# The full fast-path gate matrix; the per-layer-source row keeps the 3-D +# routing path runnable through the fast kernel. +_FAST_GEOMETRY_MATRIX = [(64, 32), (128, 32), (64, 128), (128, 128)] + + +@pytest.mark.parametrize( + "head_dim,tokens_per_block,per_layer", + [(h, t, False) for h, t in _FAST_GEOMETRY_MATRIX] + [(64, 32, True)], +) +def test_sparse_kv_cache_compact_layers_fast_geometry(head_dim, tokens_per_block, per_layer): + # Byte-compare against the CPU reference, plus the dispatch probe: the + # pipelined kernel must actually run (and the retired one must not) -- + # every byte-equality here would still pass on a silent fallback. + case = _make_fast_geometry_case(head_dim, tokens_per_block, per_layer_sources=per_layer) + with torch.profiler.profile(activities=[torch.profiler.ProfilerActivity.CUDA]) as profiler: + expected = _run_fast_geometry_case(case) + names = [event.name for event in profiler.events()] + assert any(_FAST_KERNEL_NAME in name for name in names) + assert not any(_RETIRED_KERNEL_NAME in name for name in names) + for actual, reference in zip(case.pools, expected): + assert torch.equal(actual.cpu(), reference) + + +# One representative per reject family; no fallback kernel exists, so every +# reject must fail loudly and leave the pools untouched. +@pytest.mark.parametrize( + "dtype,head_dim,tokens_per_block,flat_with_layer_indices,match", + [ + pytest.param(torch.float16, 64, 32, False, "bf16|BF16", id="dtype_outside_gate"), + pytest.param(torch.bfloat16, 256, 32, False, "bf16|BF16", id="head_dim_outside_gate"), + pytest.param(torch.bfloat16, 64, 16, False, "bf16|BF16", id="page_size_outside_gate"), + pytest.param( + torch.bfloat16, + 64, + 32, + True, + "require 3-D per-layer source_indices", + id="flat_source_with_layer_indices", + ), + ], +) +def test_sparse_kv_cache_compact_layers_rejects_invalid_launch( + dtype, head_dim, tokens_per_block, flat_with_layer_indices, match +): + case = _make_fast_geometry_case(head_dim, tokens_per_block, dtype=dtype) + source_layer_indices = case.source_layer_indices + if flat_with_layer_indices: + source_layer_indices = torch.tensor([0, 0], dtype=torch.int32) + arguments = _device_arguments( + case.pools, case.source_indices, case.source_offsets, source_layer_indices + ) + with pytest.raises((RuntimeError, ValueError), match=match): + _compact( + case.pools, + case.page_tables, + arguments, + case.destination_bases, + batch_size=case.batch_size, + ) + torch.cuda.synchronize() + for actual, reference in zip(case.pools, case.pools_cpu): + assert torch.equal(actual.cpu(), reference) diff --git a/tests/unittest/_torch/visual_gen/conftest.py b/tests/unittest/_torch/visual_gen/conftest.py new file mode 100644 index 000000000000..f6d6187b2e21 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/conftest.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared pytest configuration for the VisualGen unit tests.""" + +from collections.abc import Iterator + +import pytest + + +@pytest.fixture(scope="module") +def disable_cosmos3_guardrails() -> Iterator[None]: + """Disable Cosmos3 guardrails for the requesting module, leak-free. + + Patches both the environment variable (re-read by + ``load_standard_components`` on every call) and the pipeline module's + derived global (assigned by that same function), so teardown restores + both. Opt in per module via ``pytest.mark.usefixtures``. + """ + import tensorrt_llm._torch.visual_gen.models.cosmos3.pipeline_cosmos3 as pipe_mod + + patcher = pytest.MonkeyPatch() + patcher.setenv("TRTLLM_DISABLE_COSMOS3_GUARDRAILS", "1") + patcher.setattr(pipe_mod, "TRTLLM_DISABLE_COSMOS3_GUARDRAILS", True) + yield + patcher.undo() diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_flux_tp.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_flux_tp.py index 376469e0eb9e..88bea80e3e09 100644 --- a/tests/unittest/_torch/visual_gen/multi_gpu/test_flux_tp.py +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_flux_tp.py @@ -57,6 +57,13 @@ from tensorrt_llm.models.modeling_utils import QuantConfig + from .tp_shard_utils import ( + copy_tp_parameter, + shard_dim1, + shard_fused_gate_up, + shard_fused_qkv_by_heads, + ) + MODULES_AVAILABLE = True except ImportError: MODULES_AVAILABLE = False @@ -161,6 +168,18 @@ def run_test_in_distributed(world_size: int, test_fn: Callable, use_cuda: bool = timestep_guidance_channels=256, ) +# TP=3 uneven configs: 8 heads already gives an uneven 3+3+2 attention split. +_FLUX1_UNEVEN_TP3_CONFIG = { + **_FLUX1_TEST_CONFIG, + # FLUX.1 FFN intermediate is 512 * 4 = 2048, also uneven over TP=3. +} + +_FLUX2_UNEVEN_TP3_CONFIG = { + **_FLUX2_TEST_CONFIG, + "mlp_ratio": 3.5, + # Keep heads unchanged; make FLUX.2 MLP hidden dim 512 * 3.5 = 1792, uneven over TP=3. +} + def _make_model_config(pretrained_dict, tp_size=1, ulysses_size=1, backend="VANILLA"): """Create DiffusionModelConfig for testing with TP and/or Ulysses.""" @@ -206,72 +225,30 @@ def _stabilize_model_weights(model): # ============================================================================= -# TP weight sharding helpers +# TP weight sharding helpers (see tp_shard_utils.py) # ============================================================================= -def _shard_dim0(tensor, tp_rank, tp_size): - """Shard a tensor along dim 0.""" - chunk = tensor.shape[0] // tp_size - return tensor[tp_rank * chunk : (tp_rank + 1) * chunk].contiguous() - - -def _shard_dim1(tensor, tp_rank, tp_size): - """Shard a tensor along dim 1.""" - chunk = tensor.shape[1] // tp_size - return tensor[:, tp_rank * chunk : (tp_rank + 1) * chunk].contiguous() - - -def _shard_fused_qkv(tensor, tp_rank, tp_size, q_dim, kv_dim): - """Shard a fused QKV weight [q_dim + 2*kv_dim, ...] preserving Q/K/V structure.""" - q, k, v = tensor.split([q_dim, kv_dim, kv_dim], dim=0) - return torch.cat( - [ - _shard_dim0(q, tp_rank, tp_size), - _shard_dim0(k, tp_rank, tp_size), - _shard_dim0(v, tp_rank, tp_size), - ], - dim=0, - ) - - -def _shard_fused_gate_up(tensor, tp_rank, tp_size): - """Shard a fused gate_up weight [2*intermediate, ...] preserving gate/up structure.""" - half = tensor.shape[0] // 2 - gate, up = tensor.split([half, half], dim=0) - return torch.cat( - [ - _shard_dim0(gate, tp_rank, tp_size), - _shard_dim0(up, tp_rank, tp_size), - ], - dim=0, - ) - - -def _copy_ref_weights_to_tp(ref_model, tp_model, tp_rank, tp_size): - """Copy weights from a TP=1 reference model into a TP model with correct sharding. - - Handles column-parallel (QKV, MLP up/gate), row-parallel (output projs), - fused QKV/gate_up weights, and wrapper projectors (FluxJointAttnMLPProj, - FluxJointQKVMLPProj) that have different sub-module structure at TP>1. - """ +def _copy_ref_weights_to_tp(ref_model, tp_model, tp_rank, tp_size, config_dict): + """Copy weights from a TP=1 reference model into a TP model with correct sharding.""" ref_params = dict(ref_model.named_parameters()) - - # First handle wrapper projectors whose sub-module names differ between TP=1 and TP>1. - # At TP=1: single .proj Linear. At TP>1: split into sub-Linears. + num_heads = config_dict["num_attention_heads"] + head_dim = config_dict["attention_head_dim"] + vgm = getattr(tp_model.model_config, "visual_gen_mapping", None) + ulysses_size = vgm.ulysses_size if vgm is not None else 1 handled_tp_params = set() for tp_name, tp_module in tp_model.named_modules(): if isinstance(tp_module, FluxJointAttnMLPProj) and tp_module.tp_size > 1: - # TP model has .attn_proj + .mlp_proj; ref has .proj - ref_w = ref_params[f"{tp_name}.proj.weight"] # [out, attn_dim + mlp_dim] + ref_w = ref_params[f"{tp_name}.proj.weight"] w_attn = ref_w[:, : tp_module.attn_dim] w_mlp = ref_w[:, tp_module.attn_dim :] + attn_start, attn_end = tp_module.attn_shard tp_model.get_parameter(f"{tp_name}.attn_proj.weight").data.copy_( - _shard_dim1(w_attn, tp_rank, tp_size) + w_attn[:, attn_start:attn_end].contiguous() ) tp_model.get_parameter(f"{tp_name}.mlp_proj.weight").data.copy_( - _shard_dim1(w_mlp, tp_rank, tp_size) + shard_dim1(w_mlp, tp_rank, tp_size) ) handled_tp_params.update( [ @@ -285,20 +262,25 @@ def _copy_ref_weights_to_tp(ref_model, tp_model, tp_rank, tp_size): handled_tp_params.add(f"{tp_name}.bias") elif isinstance(tp_module, FluxJointQKVMLPProj) and tp_module.tp_size > 1: - # TP model has .qkv_proj + .mlp_proj; ref has .proj - ref_w = ref_params[f"{tp_name}.proj.weight"] # [qkv+mlp, hidden] + ref_w = ref_params[f"{tp_name}.proj.weight"] w_qkv = ref_w[: tp_module.full_qkv_dim] w_mlp = ref_w[tp_module.full_qkv_dim :] - # QKV: split into Q/K/V, shard each, re-fuse tp_model.get_parameter(f"{tp_name}.qkv_proj.weight").data.copy_( - _shard_fused_qkv( - w_qkv, tp_rank, tp_size, tp_module.full_q_dim, tp_module.full_kv_dim + shard_fused_qkv_by_heads( + w_qkv, + tp_rank, + tp_size, + num_heads, + num_heads, + head_dim, + tp_module.full_q_dim, + tp_module.full_kv_dim, + ulysses_size, ) ) - # MLP: split gate/up, shard each, re-fuse tp_model.get_parameter(f"{tp_name}.mlp_proj.weight").data.copy_( - _shard_fused_gate_up(w_mlp, tp_rank, tp_size) + shard_fused_gate_up(w_mlp, tp_rank, tp_size) ) handled_tp_params.update( [ @@ -306,18 +288,25 @@ def _copy_ref_weights_to_tp(ref_model, tp_model, tp_rank, tp_size): f"{tp_name}.mlp_proj.weight", ] ) - # Handle bias if present if f"{tp_name}.proj.bias" in ref_params: ref_b = ref_params[f"{tp_name}.proj.bias"] b_qkv = ref_b[: tp_module.full_qkv_dim] b_mlp = ref_b[tp_module.full_qkv_dim :] tp_model.get_parameter(f"{tp_name}.qkv_proj.bias").data.copy_( - _shard_fused_qkv( - b_qkv, tp_rank, tp_size, tp_module.full_q_dim, tp_module.full_kv_dim + shard_fused_qkv_by_heads( + b_qkv, + tp_rank, + tp_size, + num_heads, + num_heads, + head_dim, + tp_module.full_q_dim, + tp_module.full_kv_dim, + ulysses_size, ) ) tp_model.get_parameter(f"{tp_name}.mlp_proj.bias").data.copy_( - _shard_fused_gate_up(b_mlp, tp_rank, tp_size) + shard_fused_gate_up(b_mlp, tp_rank, tp_size) ) handled_tp_params.update( [ @@ -326,49 +315,20 @@ def _copy_ref_weights_to_tp(ref_model, tp_model, tp_rank, tp_size): ] ) - # Now handle all remaining parameters by shape comparison. with torch.no_grad(): for tp_name, tp_param in tp_model.named_parameters(): - if tp_name in handled_tp_params: - continue - if tp_name not in ref_params: + if tp_name in handled_tp_params or tp_name not in ref_params: continue - - ref_param = ref_params[tp_name] - - if tp_param.shape == ref_param.shape: - # Replicated parameter (norms, embeddings, etc.) - tp_param.data.copy_(ref_param.data) - elif tp_param.ndim >= 2 and tp_param.shape[1] == ref_param.shape[1]: - # Column parallel: dim 0 is smaller (output dim sharded) - if "qkv_proj" in tp_name or "add_qkv_proj" in tp_name: - # Fused QKV: figure out q_dim from total (q=k=v for FLUX) - q_dim = ref_param.shape[0] // 3 - tp_param.data.copy_( - _shard_fused_qkv(ref_param.data, tp_rank, tp_size, q_dim, q_dim) - ) - elif "gate_up_proj" in tp_name: - tp_param.data.copy_(_shard_fused_gate_up(ref_param.data, tp_rank, tp_size)) - else: - tp_param.data.copy_(_shard_dim0(ref_param.data, tp_rank, tp_size)) - elif tp_param.ndim >= 2 and tp_param.shape[0] == ref_param.shape[0]: - # Row parallel: dim 1 is smaller (input dim sharded) - tp_param.data.copy_(_shard_dim1(ref_param.data, tp_rank, tp_size)) - elif tp_param.ndim == 1 and tp_param.shape[0] < ref_param.shape[0]: - # 1D bias for column parallel - if "qkv_proj" in tp_name or "add_qkv_proj" in tp_name: - q_dim = ref_param.shape[0] // 3 - tp_param.data.copy_( - _shard_fused_qkv(ref_param.data, tp_rank, tp_size, q_dim, q_dim) - ) - elif "gate_up_proj" in tp_name: - tp_param.data.copy_(_shard_fused_gate_up(ref_param.data, tp_rank, tp_size)) - else: - tp_param.data.copy_(_shard_dim0(ref_param.data, tp_rank, tp_size)) - else: - raise ValueError( - f"Cannot shard {tp_name}: ref={ref_param.shape}, tp={tp_param.shape}" - ) + copy_tp_parameter( + tp_name, + ref_params[tp_name], + tp_param, + tp_rank, + tp_size, + num_heads, + head_dim, + ulysses_size=ulysses_size, + ) # ============================================================================= @@ -421,6 +381,16 @@ def _logic_flux1_tp_forward(rank, world_size): def _logic_flux1_tp_vs_single_gpu(rank, world_size): """FLUX.1: TP 2-GPU output matches single-GPU reference.""" + _logic_flux1_tp_vs_single_gpu_with_config(rank, world_size, _FLUX1_TEST_CONFIG) + + +def _logic_flux1_tp3_uneven_vs_single_gpu(rank, world_size): + """FLUX.1: TP=3 with uneven head/MLP dims matches single-GPU reference.""" + _logic_flux1_tp_vs_single_gpu_with_config(rank, world_size, _FLUX1_UNEVEN_TP3_CONFIG) + + +def _logic_flux1_tp_vs_single_gpu_with_config(rank, world_size, config_dict): + """FLUX.1: TP output matches single-GPU reference.""" from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux import FluxTransformer2DModel device = torch.device(f"cuda:{rank}") @@ -429,22 +399,25 @@ def _logic_flux1_tp_vs_single_gpu(rank, world_size): batch = 1 img_seq = 16 txt_seq = 8 + in_channels = 64 # Create single-GPU reference model torch.manual_seed(123) - ref_config = _make_model_config(_FLUX1_TEST_CONFIG, tp_size=1) + ref_config = _make_model_config(config_dict, tp_size=1) ref_model = FluxTransformer2DModel(ref_config).to(device).to(compute_dtype) _stabilize_model_weights(ref_model) # Create TP model and copy sharded weights from ref torch.manual_seed(123) - tp_config = _make_model_config(_FLUX1_TEST_CONFIG, tp_size=world_size) + tp_config = _make_model_config(config_dict, tp_size=world_size) tp_model = FluxTransformer2DModel(tp_config).to(device).to(compute_dtype) - _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size) + _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size, config_dict) # Same inputs on all ranks torch.manual_seed(456) - hidden_states = torch.randn(batch, img_seq, 64, device=device, dtype=compute_dtype) * 0.1 + hidden_states = ( + torch.randn(batch, img_seq, in_channels, device=device, dtype=compute_dtype) * 0.1 + ) encoder_hidden_states = ( torch.randn(batch, txt_seq, 256, device=device, dtype=compute_dtype) * 0.1 ) @@ -528,6 +501,16 @@ def _logic_flux2_tp_forward(rank, world_size): def _logic_flux2_tp_vs_single_gpu(rank, world_size): """FLUX.2: TP 2-GPU output matches single-GPU reference.""" + _logic_flux2_tp_vs_single_gpu_with_config(rank, world_size, _FLUX2_TEST_CONFIG) + + +def _logic_flux2_tp3_uneven_vs_single_gpu(rank, world_size): + """FLUX.2: TP=3 with uneven head/MLP dims matches single-GPU reference.""" + _logic_flux2_tp_vs_single_gpu_with_config(rank, world_size, _FLUX2_UNEVEN_TP3_CONFIG) + + +def _logic_flux2_tp_vs_single_gpu_with_config(rank, world_size, config_dict): + """FLUX.2: TP output matches single-GPU reference.""" from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2 import Flux2Transformer2DModel device = torch.device(f"cuda:{rank}") @@ -536,22 +519,25 @@ def _logic_flux2_tp_vs_single_gpu(rank, world_size): batch = 1 img_seq = 16 txt_seq = 8 + in_channels = 128 # Create single-GPU reference model torch.manual_seed(123) - ref_config = _make_model_config(_FLUX2_TEST_CONFIG, tp_size=1) + ref_config = _make_model_config(config_dict, tp_size=1) ref_model = Flux2Transformer2DModel(ref_config).to(device).to(compute_dtype) _stabilize_model_weights(ref_model) # Create TP model and copy sharded weights from ref torch.manual_seed(123) - tp_config = _make_model_config(_FLUX2_TEST_CONFIG, tp_size=world_size) + tp_config = _make_model_config(config_dict, tp_size=world_size) tp_model = Flux2Transformer2DModel(tp_config).to(device).to(compute_dtype) - _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size) + _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size, config_dict) # Same inputs on all ranks torch.manual_seed(456) - hidden_states = torch.randn(batch, img_seq, 128, device=device, dtype=compute_dtype) * 0.1 + hidden_states = ( + torch.randn(batch, img_seq, in_channels, device=device, dtype=compute_dtype) * 0.1 + ) encoder_hidden_states = ( torch.randn(batch, txt_seq, 256, device=device, dtype=compute_dtype) * 0.1 ) @@ -619,7 +605,7 @@ def _logic_flux2_tp_ulysses_vs_single_gpu(rank, world_size): ) combined_model = Flux2Transformer2DModel(combined_config).to(device).to(compute_dtype) vgm = combined_config.visual_gen_mapping - _copy_ref_weights_to_tp(ref_model, combined_model, vgm.tp_rank, tp_size) + _copy_ref_weights_to_tp(ref_model, combined_model, vgm.tp_rank, tp_size, _FLUX2_TEST_CONFIG) # Same inputs on all ranks (Ulysses shards at runtime) torch.manual_seed(456) @@ -693,5 +679,17 @@ def test_flux2_tp_ulysses_vs_single_gpu(self): run_test_in_distributed(world_size=4, test_fn=_logic_flux2_tp_ulysses_vs_single_gpu) +class TestFluxUnevenTP3: + """TP=3 tests where head count and MLP dims are not divisible by tp_size.""" + + def test_flux1_tp3_uneven_vs_single_gpu(self): + """FLUX.1 TP=3 (8 heads, uneven MLP) matches single-GPU reference.""" + run_test_in_distributed(world_size=3, test_fn=_logic_flux1_tp3_uneven_vs_single_gpu) + + def test_flux2_tp3_uneven_vs_single_gpu(self): + """FLUX.2 TP=3 (8 heads, uneven MLP) matches single-GPU reference.""" + run_test_in_distributed(world_size=3, test_fn=_logic_flux2_tp3_uneven_vs_single_gpu) + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_async_ulysses.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_async_ulysses.py index e6e269565197..fe21222a254e 100644 --- a/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_async_ulysses.py +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_async_ulysses.py @@ -298,10 +298,10 @@ def _build_av_model( ): """Build LTXModel (AudioVideo) with deterministic weights via shared seed. - ``configure_audio_ulysses(audio_seq_len)`` gates audio_attn1's Ulysses - activity by divisibility: not divisible → ``set_ulysses_active(False)`` - swaps the audio backend to plain (no ``forward_async``), forcing async - self-attn to fall through the ``hasattr`` guard in ``LTX2Attention.forward``. + audio_attn1's attention TYPE is fixed at construction from the AudioShardMode + env constant: CONDITIONAL (default) builds a plain backend (no + ``forward_async``), so async self-attn falls through the ``hasattr`` guard in + ``LTX2Attention.forward``; legacy FULL builds the Ulysses wrapper. """ from tensorrt_llm._torch.visual_gen.models.ltx2.transformer_ltx2 import LTXModel, LTXModelType diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_ulysses.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_ulysses.py index d54076374bec..9ace40540dd0 100644 --- a/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_ulysses.py +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_ltx2_ulysses.py @@ -330,6 +330,173 @@ def _logic_ltx2_av_ulysses_vs_single_gpu(rank, world_size, backend, audio_seq_le ) +def _make_model_config_cfg( + cfg_size: int, + ulysses_size: int, + backend: str = "VANILLA", +) -> "DiffusionModelConfig": + """DiffusionModelConfig with CFG x Ulysses parallelism (dist must be up).""" + ws = dist.get_world_size() + rk = dist.get_rank() + vgm = VisualGenMapping(world_size=ws, rank=rk, cfg_size=cfg_size, ulysses_size=ulysses_size) + config = DiffusionModelConfig( + pretrained_config=SimpleNamespace(), + quant_config=QuantConfig(), + torch_compile=TorchCompileConfig(enable=False), + attention=AttentionConfig(backend=backend), + visual_gen_mapping=vgm, + cache=None, + attention_metadata_state=( + create_attention_metadata_state() if backend.upper() == "TRTLLM" else None + ), + parallel=ParallelConfig(cfg_size=cfg_size, ulysses_size=ulysses_size), + skip_create_weights_in_init=False, + ) + config.mapping = vgm.to_llm_mapping() + return config + + +def _build_stage2_groups_for_test(vgm): + """Mirror of LTX2TwoStagesPipeline._build_stage2_dit_groups.""" + from tensorrt_llm._torch.visual_gen.models.ltx2.transformer_ltx2 import Stage2Groups + + rank = dist.get_rank() + fold = vgm.flatten_cfg_ranks() + uly_group = None + for ranks in fold: + g = dist.new_group(ranks, use_local_synchronization=False) + if rank in ranks: + uly_group = g + flat = [r for ranks in fold for r in ranks] + if len(fold) == 1: + seq_group, gather_index = uly_group, None + else: + seq_group = dist.new_group(sorted(flat), use_local_synchronization=False) + gather_index = flat + return Stage2Groups( + ulysses_group=uly_group, + seq_group=seq_group, + seq_rank=flat.index(rank), + seq_size=len(flat), + gather_index=gather_index, + ) + + +def _logic_ltx2_dual_topology(rank, world_size, backend, audio_seq_len): + """{default, stage2} switch: stack alternation, shard/gather round-trip, + and full-forward numerical equivalence vs the single-GPU reference. + + cfg2 x u(world/2): the default topology shards each cfg branch's sequence + over its own fiber; stage2 folds cfg into one world-spanning ulysses group. + Stage 2 has no CFG, so a forward on identical inputs must match the + reference in BOTH topologies, across repeated back-and-forth switches. + """ + from tensorrt_llm._torch.visual_gen.models.ltx2.transformer_ltx2 import LTXModel, LTXModelType + + device = torch.device(f"cuda:{rank}") + dtype = torch.bfloat16 + batch = 1 + v_dims = (1, 4, 4) + v_patches = v_dims[0] * v_dims[1] * v_dims[2] # 16: divisible by u2 and u4 + + torch.manual_seed(123) + ref_cfg = _make_model_config(ulysses_size=1, backend=backend) + ref_model = ( + LTXModel(model_type=LTXModelType.AudioVideo, model_config=ref_cfg, **_AV_CONFIG) + .to(device, dtype=dtype) + .eval() + ) + _init_all_weights(ref_model) + ref_model.configure_audio_ulysses(audio_seq_len) + ref_state = ref_model.state_dict() + + torch.manual_seed(123) + d_cfg = _make_model_config_cfg(cfg_size=2, ulysses_size=world_size // 2, backend=backend) + s2 = _build_stage2_groups_for_test(d_cfg.visual_gen_mapping) + d_model = ( + LTXModel( + model_type=LTXModelType.AudioVideo, + model_config=d_cfg, + stage2_groups=s2, + **_AV_CONFIG, + ) + .to(device, dtype=dtype) + .eval() + ) + d_model.load_state_dict(ref_state) + d_model.configure_audio_ulysses(audio_seq_len) + + assert d_model._has_stage2 + assert d_model._sharder.size == world_size // 2 + assert d_model._sharder_s2.size == world_size + + video, audio, v_ctx, a_ctx, v_pos, a_pos = _build_inputs( + batch, v_patches, v_dims, audio_seq_len, dtype, device + ) + ref_cache = ref_model.prepare_text_cache( + video_context=v_ctx, + video_positions=v_pos, + audio_context=a_ctx, + audio_positions=a_pos, + dtype=dtype, + ) + with torch.no_grad(): + ref_v, ref_a = ref_model(video=video, audio=audio, text_cache=ref_cache) + + x = torch.randn(1, 32, 8, device=device, dtype=dtype) + for is_stage2 in (False, True, False, True): + d_model.set_ulysses_topology(is_stage2=is_stage2) + + blk = d_model.transformer_blocks[0] + expect = blk.attn1._attn_stage2 if is_stage2 else blk.attn1._attn_default + assert blk.attn1.attn is expect, f"Rank {rank}: attn stack not switched" + # is_ulysses must track the ACTIVE stack (the pair can differ in type). + from tensorrt_llm._torch.visual_gen.attention_backend.parallel import UlyssesAttention + + for name in ("attn1", "video_to_audio_attn", "audio_attn1"): + mod = getattr(blk, name, None) + if mod is not None: + assert mod.is_ulysses == isinstance(mod.attn, UlyssesAttention), ( + f"Rank {rank}: {name}.is_ulysses stale (is_stage2={is_stage2})" + ) + # CONDITIONAL audio mode keeps audio self-attn plain even at ulysses>1. + assert not isinstance(blk.audio_attn1.attn, UlyssesAttention), ( + f"Rank {rank}: audio_attn1 wrapped despite CONDITIONAL mode (is_stage2={is_stage2})" + ) + sh = d_model._active_sharder + assert sh.size == (world_size if is_stage2 else world_size // 2) + assert torch.equal(sh.gather(sh.shard(x, dim=1), dim=1), x), ( + f"Rank {rank}: shard/gather round-trip broken (is_stage2={is_stage2})" + ) + + # prepare_text_cache is topology-dependent — always AFTER the switch. + d_cache = d_model.prepare_text_cache( + video_context=v_ctx, + video_positions=v_pos, + audio_context=a_ctx, + audio_positions=a_pos, + dtype=dtype, + ) + video_i, audio_i, *_ = _build_inputs(batch, v_patches, v_dims, audio_seq_len, dtype, device) + with torch.no_grad(): + d_v, d_a = d_model(video=video_i, audio=audio_i, text_cache=d_cache) + + torch.testing.assert_close( + d_v, + ref_v, + rtol=5e-2, + atol=5e-2, + msg=f"Rank {rank}: video mismatch (is_stage2={is_stage2})", + ) + torch.testing.assert_close( + d_a, + ref_a, + rtol=5e-2, + atol=5e-2, + msg=f"Rank {rank}: audio mismatch (is_stage2={is_stage2})", + ) + + # ============================================================================= # Test classes # ============================================================================= @@ -357,3 +524,122 @@ def test_av_ulysses_audio_pad(self, backend): if __name__ == "__main__": pytest.main([__file__, "-v"]) + + +def _logic_ltx2_full_audio_construction(rank, world_size, backend, audio_seq_len): + """Legacy FULL mode at ulysses>1: audio_attn1 is CONSTRUCTED as ulysses (env + constant is the only selector) and, given stage-2 groups under cfg2, gets + its own distinct {default, stage2} stack pair like attn1/v2a.""" + import tensorrt_llm._torch.visual_gen.models.ltx2.transformer_ltx2 as tl + from tensorrt_llm._torch.visual_gen.models.ltx2.transformer_ltx2 import LTXModel, LTXModelType + + device = torch.device(f"cuda:{rank}") + dtype = torch.bfloat16 + + orig = tl._LTX2_AUDIO_CONDITIONAL_SHARD + tl._LTX2_AUDIO_CONDITIONAL_SHARD = False + try: + torch.manual_seed(123) + d_cfg = _make_model_config_cfg(cfg_size=2, ulysses_size=world_size // 2, backend=backend) + s2 = _build_stage2_groups_for_test(d_cfg.visual_gen_mapping) + model = ( + LTXModel( + model_type=LTXModelType.AudioVideo, + model_config=d_cfg, + stage2_groups=s2, + **_AV_CONFIG, + ) + .to(device, dtype=dtype) + .eval() + ) + _init_all_weights(model) + model.configure_audio_ulysses(audio_seq_len) + blk = model.transformer_blocks[0] + assert blk.audio_attn1.is_ulysses, "FULL mode must construct ulysses audio_attn1" + assert blk.audio_attn1._attn_stage2 is not blk.audio_attn1._attn_default, ( + "FULL audio_attn1 must get a distinct stage-2 stack under cfg2" + ) + # Both topologies forward without error (audio sharded across the active group). + video, audio, v_ctx, a_ctx, v_pos, a_pos = _build_inputs( + 1, 16, (1, 4, 4), audio_seq_len, dtype, device + ) + for is_stage2 in (False, True, False): + model.set_ulysses_topology(is_stage2=is_stage2) + cache = model.prepare_text_cache( + video_context=v_ctx, + video_positions=v_pos, + audio_context=a_ctx, + audio_positions=a_pos, + dtype=dtype, + ) + video_i, audio_i, *_ = _build_inputs(1, 16, (1, 4, 4), audio_seq_len, dtype, device) + with torch.no_grad(): + v, a = model(video=video_i, audio=audio_i, text_cache=cache) + assert v.shape[1] == 16 and a.shape[1] == audio_seq_len + finally: + tl._LTX2_AUDIO_CONDITIONAL_SHARD = orig + + +def _logic_ltx2_stage2_head_divisibility_raises(rank, world_size, backend, _unused): + """Construction fails fast when the head count divides the stage-1 ulysses + size (u = ws/2) but not the stage-2 group size (cfg*u = ws).""" + from tensorrt_llm._torch.visual_gen.models.ltx2.transformer_ltx2 import LTXModel, LTXModelType + + d_cfg = _make_model_config_cfg(cfg_size=2, ulysses_size=world_size // 2, backend=backend) + s2 = _build_stage2_groups_for_test(d_cfg.visual_gen_mapping) + bad = dict( + _AV_CONFIG, + num_attention_heads=6, + cross_attention_dim=6 * _AV_CONFIG["attention_head_dim"], + ) + assert 6 % (world_size // 2) == 0 and 6 % world_size != 0 + try: + LTXModel( + model_type=LTXModelType.AudioVideo, + model_config=d_cfg, + stage2_groups=s2, + **bad, + ) + raise AssertionError(f"Rank {rank}: indivisible stage-2 head count was not rejected") + except ValueError as e: + assert "stage-2 ulysses requires" in str(e), f"Rank {rank}: {e}" + + +def _logic_ltx2_trtllm_audio_downgrade_with_stage2(rank, world_size, backend, _unused): + """cfg2 x u1 stage-2 fold under a TRTLLM backend: audio may be padded to the + stage-2 multiple, whose key_padding_mask TRTLLM drops — audio_attn1 must + downgrade to VANILLA at construction even though ulysses_size == 1.""" + from tensorrt_llm._torch.visual_gen.models.ltx2.transformer_ltx2 import LTXModel, LTXModelType + + d_cfg = _make_model_config_cfg(cfg_size=2, ulysses_size=1, backend="TRTLLM") + s2 = _build_stage2_groups_for_test(d_cfg.visual_gen_mapping) + model = LTXModel( + model_type=LTXModelType.AudioVideo, + model_config=d_cfg, + stage2_groups=s2, + **_AV_CONFIG, + ) + blk = model.transformer_blocks[0] + assert blk.audio_attn1.attn_backend == "VANILLA", ( + f"Rank {rank}: audio_attn1 backend {blk.audio_attn1.attn_backend}, expected VANILLA" + ) + assert blk.attn1.attn_backend == "TRTLLM", f"Rank {rank}: video attn1 must keep TRTLLM" + + +class TestLTX2TopologySwitch: + """{default, stage2} dual-topology switch on 4 GPUs (cfg2 x u2 -> u4): + pointer alternation, shard/gather round-trip, and whole-dataflow numerical + equivalence vs the single-GPU reference in both topologies.""" + + @pytest.mark.parametrize("audio_seq_len", [64, 62], ids=["no_pad", "pad2"]) + def test_dual_topology_alternation_matches_reference(self, audio_seq_len): + run_test_in_distributed(4, _logic_ltx2_dual_topology, "VANILLA", audio_seq_len) + + def test_full_audio_mode_construction_and_switch(self): + run_test_in_distributed(4, _logic_ltx2_full_audio_construction, "VANILLA", 64) + + def test_stage2_head_divisibility_raises(self): + run_test_in_distributed(4, _logic_ltx2_stage2_head_divisibility_raises, "VANILLA", 0) + + def test_trtllm_audio_downgrade_with_stage2_fold(self): + run_test_in_distributed(2, _logic_ltx2_trtllm_audio_downgrade_with_stage2, "TRTLLM", 0) diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_qwen_image_attention_parallel.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_qwen_image_attention_parallel.py new file mode 100644 index 000000000000..3a4967e7072d --- /dev/null +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_qwen_image_attention_parallel.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Synthetic multi-GPU topology tests for Qwen Image joint attention.""" + +import os +from typing import Callable + +os.environ["TLLM_DISABLE_MPI"] = "1" + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +try: + import sys + from pathlib import Path + + from tensorrt_llm._torch.visual_gen.attention_backend.parallel import ( + Attention2DAttention, + RingAttention, + UlyssesAttention, + ) + from tensorrt_llm._torch.visual_gen.config import AttentionConfig, DiffusionModelConfig + from tensorrt_llm._torch.visual_gen.mapping import VisualGenMapping + from tensorrt_llm._torch.visual_gen.models.qwen_image import QwenJointAttention + from tensorrt_llm.visual_gen.args import ParallelConfig + + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from _visual_gen_dist_utils import spawn_with_retry + + MODULES_AVAILABLE = True +except ImportError: + MODULES_AVAILABLE = False + +try: + from tensorrt_llm._torch.visual_gen.attention_backend.flash_attn4 import ( + _flash_attn_fwd as _fa4_fwd, + ) + from tensorrt_llm._torch.visual_gen.attention_backend.parallel import ( + _flash_attn_combine as _fa_combine, + ) + + _FLASH_ATTN4_AVAILABLE = _fa4_fwd is not None + _ATTN2D_AVAILABLE = _fa4_fwd is not None and _fa_combine is not None +except (ImportError, OSError): + _FLASH_ATTN4_AVAILABLE = False + _ATTN2D_AVAILABLE = False + + +@pytest.fixture(autouse=True, scope="module") +def _cleanup_mpi_env(): + yield + os.environ.pop("TLLM_DISABLE_MPI", None) + + +def _init_distributed_worker( + rank: int, world_size: int, backend: str = "nccl", port: int = 29500 +) -> None: + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(port) + os.environ["RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + torch.cuda.set_device(rank % torch.cuda.device_count()) + dist.init_process_group(backend=backend, rank=rank, world_size=world_size) + + +def _cleanup_distributed() -> None: + if dist.is_initialized(): + dist.destroy_process_group() + + +def _distributed_worker(rank, world_size, backend, test_fn, port, kwargs): + try: + _init_distributed_worker(rank, world_size, backend, port) + test_fn(rank, world_size, **kwargs) + finally: + _cleanup_distributed() + + +def _run_test_in_distributed(world_size: int, test_fn: Callable, **kwargs) -> None: + if not MODULES_AVAILABLE: + pytest.skip("Required modules not available") + if torch.cuda.device_count() < world_size: + pytest.skip(f"Test requires {world_size} GPUs, only {torch.cuda.device_count()} available") + spawn_with_retry( + lambda port: mp.spawn( + _distributed_worker, + args=(world_size, "nccl", test_fn, port, kwargs), + nprocs=world_size, + join=True, + ) + ) + + +def _test_qwen_image_attention_parallel_topology( + rank: int, + world_size: int, + *, + parallel: dict, + backend: str, + topology: str, +) -> None: + parallel_config = ParallelConfig(**parallel) + parallel_config.validate_world_size(world_size) + attn2d_row_size, attn2d_col_size = parallel_config.attn2d_size + visual_gen_mapping = VisualGenMapping( + world_size=world_size, + rank=rank, + tp_size=parallel_config.tp_size, + ring_size=parallel_config.ring_size, + ulysses_size=parallel_config.ulysses_size, + attn2d_row_size=attn2d_row_size, + attn2d_col_size=attn2d_col_size, + ) + config = DiffusionModelConfig( + mapping=visual_gen_mapping.to_llm_mapping(), + visual_gen_mapping=visual_gen_mapping, + attention=AttentionConfig(backend=backend), + parallel=parallel_config, + ) + attention = QwenJointAttention( + dim=256, + num_attention_heads=4, + attention_head_dim=64, + config=config, + ).cuda() + + torch.manual_seed(11) + with torch.no_grad(): + for name, parameter in attention.named_parameters(): + if name.endswith("bias"): + parameter.zero_() + elif "norm" in name and name.endswith("weight"): + parameter.fill_(1) + else: + parameter.normal_(mean=0.0, std=0.02) + + if topology == "tp": + assert not isinstance(attention.attn, UlyssesAttention) + else: + assert isinstance(attention.attn, UlyssesAttention) + if topology == "ring": + assert isinstance(attention.attn.inner_backend, RingAttention) + else: + assert isinstance(attention.attn.inner_backend, Attention2DAttention) + + hidden_states = torch.randn(1, 8, 256, device="cuda", dtype=torch.bfloat16) + encoder_hidden_states = torch.randn(1, 4, 256, device="cuda", dtype=torch.bfloat16) + image_output, text_output = attention( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + ) + + assert image_output.shape == hidden_states.shape + assert text_output.shape == encoder_hidden_states.shape + assert torch.isfinite(image_output).all() + assert torch.isfinite(text_output).all() + dist.barrier() + + +@pytest.mark.parametrize( + "world_size,parallel,backend,topology", + [ + pytest.param(2, {"tp_size": 2}, "VANILLA", "tp", marks=pytest.mark.gpu2, id="tp2"), + pytest.param( + 4, + {"ring_size": 2, "ulysses_size": 2}, + "FA4", + "ring", + marks=pytest.mark.gpu4, + id="ring2_ulysses2", + ), + pytest.param( + 4, + {"attn2d_size": (2, 1), "ulysses_size": 2}, + "FA4", + "attn2d", + marks=pytest.mark.gpu4, + id="attn2d_2x1_ulysses2", + ), + ], +) +def test_qwen_image_attention_parallel_topologies(world_size, parallel, backend, topology) -> None: + if topology == "ring" and not _FLASH_ATTN4_AVAILABLE: + pytest.skip("FlashAttn4 JIT kernels not available") + if topology == "attn2d" and not _ATTN2D_AVAILABLE: + pytest.skip("FA4 / flash_attn_combine JIT kernels not available") + _run_test_in_distributed( + world_size, + _test_qwen_image_attention_parallel_topology, + parallel=parallel, + backend=backend, + topology=topology, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "-s"]) diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_tp_attention.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_tp_attention.py index c77204d2d1ba..f090cf7522d0 100644 --- a/tests/unittest/_torch/visual_gen/multi_gpu/test_tp_attention.py +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_tp_attention.py @@ -167,70 +167,75 @@ def _shard_tp_weights(ref_attn, tp_attn, tp_rank, tp_size, qkv_mode=QKVMode.FUSE RMSNorm (TP-enabled): split weight """ with torch.no_grad(): - if qkv_mode == QKVMode.FUSE_QKV: - # Fused QKV: weight is [q_dim + 2*kv_dim, hidden_size] - full_w = ref_attn.qkv_proj.weight.data - q_dim = ref_attn.q_dim - kv_dim = ref_attn.kv_dim - q_w, k_w, v_w = full_w.split([q_dim, kv_dim, kv_dim], dim=0) + q_start, q_end = tp_attn.local_q_dim_start, tp_attn.local_q_dim_end + kv_start, kv_end = tp_attn.local_kv_dim_start, tp_attn.local_kv_dim_end - q_shard = _shard_dim0(q_w, tp_rank, tp_size) - k_shard = _shard_dim0(k_w, tp_rank, tp_size) - v_shard = _shard_dim0(v_w, tp_rank, tp_size) - tp_attn.qkv_proj.weight.data.copy_(torch.cat([q_shard, k_shard, v_shard], dim=0)) + if qkv_mode == QKVMode.FUSE_QKV: + q_w, k_w, v_w = ref_attn.qkv_proj.weight.data.split( + [ref_attn.q_dim, ref_attn.kv_dim, ref_attn.kv_dim], dim=0 + ) + tp_attn.qkv_proj.weight.data.copy_( + torch.cat( + [ + q_w[q_start:q_end], + k_w[kv_start:kv_end], + v_w[kv_start:kv_end], + ], + dim=0, + ).contiguous() + ) if ref_attn.qkv_proj.bias is not None: - full_b = ref_attn.qkv_proj.bias.data - q_b, k_b, v_b = full_b.split([q_dim, kv_dim, kv_dim], dim=0) + q_b, k_b, v_b = ref_attn.qkv_proj.bias.data.split( + [ref_attn.q_dim, ref_attn.kv_dim, ref_attn.kv_dim], dim=0 + ) tp_attn.qkv_proj.bias.data.copy_( torch.cat( [ - _shard_dim0(q_b, tp_rank, tp_size), - _shard_dim0(k_b, tp_rank, tp_size), - _shard_dim0(v_b, tp_rank, tp_size), + q_b[q_start:q_end], + k_b[kv_start:kv_end], + v_b[kv_start:kv_end], ], dim=0, - ) + ).contiguous() ) else: - for name in ("to_q", "to_k", "to_v"): + for name, bounds in ( + ("to_q", (q_start, q_end)), + ("to_k", (kv_start, kv_end)), + ("to_v", (kv_start, kv_end)), + ): ref_proj = getattr(ref_attn, name) tp_proj = getattr(tp_attn, name) - tp_proj.weight.data.copy_(_shard_dim0(ref_proj.weight.data, tp_rank, tp_size)) + start, end = bounds + tp_proj.weight.data.copy_(ref_proj.weight.data[start:end].contiguous()) if ref_proj.bias is not None: - tp_proj.bias.data.copy_(_shard_dim0(ref_proj.bias.data, tp_rank, tp_size)) + tp_proj.bias.data.copy_(ref_proj.bias.data[start:end].contiguous()) - # Output projection: row-parallel (split input dim = dim 1) + # Output projection: row-parallel (split input dim = dim 1, head-aligned) ref_out = ref_attn.to_out[0] tp_out = tp_attn.to_out[0] - shard_size = math.ceil(ref_out.weight.shape[1] / tp_size) - start = tp_rank * shard_size - end = min(start + shard_size, ref_out.weight.shape[1]) - tp_out.weight.data.copy_(ref_out.weight.data[:, start:end].contiguous()) + q_start, q_end = tp_attn.local_q_dim_start, tp_attn.local_q_dim_end + tp_out.weight.data.copy_(ref_out.weight.data[:, q_start:q_end].contiguous()) if ref_out.bias is not None: tp_out.bias.data.copy_(ref_out.bias.data) - # QK norm weights (if TP-enabled, they're sharded) + # QK norm weights (if TP-enabled, use Attention head-based shard bounds) if hasattr(ref_attn, "norm_q") and hasattr(tp_attn, "norm_q"): if tp_attn.norm_q.enable_tp: - shard_size = ref_attn.norm_q.weight.shape[0] // tp_size - start = tp_rank * shard_size - end = start + shard_size - tp_attn.norm_q.weight.data.copy_(ref_attn.norm_q.weight.data[start:end]) - tp_attn.norm_k.weight.data.copy_(ref_attn.norm_k.weight.data[start:end]) + tp_attn.norm_q.weight.data.copy_( + ref_attn.norm_q.weight.data[tp_attn.local_q_dim_start : tp_attn.local_q_dim_end] + ) + tp_attn.norm_k.weight.data.copy_( + ref_attn.norm_k.weight.data[ + tp_attn.local_kv_dim_start : tp_attn.local_kv_dim_end + ] + ) else: tp_attn.norm_q.weight.data.copy_(ref_attn.norm_q.weight.data) tp_attn.norm_k.weight.data.copy_(ref_attn.norm_k.weight.data) -def _shard_dim0(tensor, tp_rank, tp_size): - """Shard a tensor along dim 0 (works for both 1D bias and 2D weight).""" - shard_size = math.ceil(tensor.shape[0] / tp_size) - start = tp_rank * shard_size - end = min(start + shard_size, tensor.shape[0]) - return tensor[start:end].contiguous() - - # ============================================================================= # Manual F.sdpa reference # ============================================================================= @@ -410,16 +415,18 @@ def _logic_tp_hidden_512(rank, world_size): _run_tp_with_params(rank, world_size, batch=2, seq=16, hidden_size=512, num_heads=4) -def _logic_tp_heads_not_divisible(rank, world_size): - """TP when num_heads % tp_size != 0. Expected to fail until uneven sharding is implemented.""" - _run_tp_with_params(rank, world_size, batch=2, seq=16, hidden_size=320, num_heads=5) +def _logic_tp_size_3_uneven_heads(rank, world_size): + """TP=3 when num_heads and hidden_size are not divisible by tp_size.""" + _run_tp_with_params(rank, world_size, batch=2, seq=16, hidden_size=512, num_heads=8) def _logic_tp_world_size_4(rank, world_size): _run_tp_with_params(rank, world_size, batch=2, seq=16, hidden_size=512, num_heads=16) -def _logic_tp_ulysses_combined(rank, world_size, ulysses_size, tp_size): +def _logic_tp_ulysses_combined( + rank, world_size, ulysses_size, tp_size, hidden_size=512, num_heads=16 +): """TP + Ulysses combined matches F.sdpa reference on the full sequence. 4 GPUs: tp_size=2, ulysses_size=2. @@ -428,8 +435,6 @@ def _logic_tp_ulysses_combined(rank, world_size, ulysses_size, tp_size): assert tp_size * ulysses_size == world_size device = torch.device(f"cuda:{rank}") - hidden_size = 512 - num_heads = 16 head_dim = hidden_size // num_heads batch = 2 seq_per_rank = 8 @@ -471,6 +476,18 @@ def _logic_tp_ulysses_combined(rank, world_size, ulysses_size, tp_size): torch.testing.assert_close(combined_out, expected_shard, rtol=1e-2, atol=1e-2) +def _logic_tp3_ulysses_uneven_combined(rank, world_size): + """TP=3 + Ulysses=2 with 8 attention heads (4+2+2 TP split).""" + _logic_tp_ulysses_combined( + rank, + world_size, + ulysses_size=2, + tp_size=3, + hidden_size=512, + num_heads=8, + ) + + # ============================================================================= # Test classes # ============================================================================= @@ -501,9 +518,9 @@ def test_hidden_128(self): def test_hidden_512(self): _run(2, _logic_tp_hidden_512) - @pytest.mark.xfail(reason="Uneven head sharding not yet implemented", raises=Exception) - def test_tp_heads_not_divisible(self): - _run(2, _logic_tp_heads_not_divisible) + def test_tp_size_3_uneven_heads(self): + """TP=3 with 8 heads (4+2+2 split) matches F.sdpa reference.""" + _run(3, _logic_tp_size_3_uneven_heads) def test_tp_world_size_4(self): _run(4, _logic_tp_world_size_4) @@ -530,6 +547,9 @@ def test_tp_4_ulysses_2(self): world = ulysses_size * tp_size _run(world, _logic_tp_ulysses_combined, ulysses_size, tp_size) + def test_tp_3_ulysses_2_uneven(self): + _run(6, _logic_tp3_ulysses_uneven_combined) + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_visual_gen_mapping.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_visual_gen_mapping.py index a870d46ed045..c73762ae3c7a 100644 --- a/tests/unittest/_torch/visual_gen/multi_gpu/test_visual_gen_mapping.py +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_visual_gen_mapping.py @@ -162,6 +162,23 @@ def test_ulysses_and_attn2d_constructible(self): assert vgm.ulysses_size == 2 assert vgm.seq_size == 4 + def test_tp_and_attn2d_constructible(self): + """TP + Attention2D is allowed (mapping guard removed in commit 084755212d). + + Prior behavior raised NotImplementedError; ensure the mesh now composes. + """ + vgm = VisualGenMapping( + world_size=4, + rank=0, + tp_size=2, + attn2d_row_size=2, + attn2d_col_size=1, + ) + assert vgm.tp_size == 2 + assert vgm.cp_size == 2 + assert vgm.attn2d_row_size == 2 + assert vgm.attn2d_col_size == 1 + def test_ring_and_attn2d_raises(self): """Combining ring and Attention2D raises ValueError (both shard the sequence axis).""" with pytest.raises(ValueError, match="mutually exclusive"): @@ -174,6 +191,32 @@ def test_ring_and_attn2d_raises(self): ) +class TestFlattenCfgRanks: + """flatten_cfg_ranks is pure layout arithmetic: one rank list per combined + coordinate of every non-(cfg, ulysses) mesh dim, cfg outermost / ulysses + innermost within each list.""" + + def test_cfg2_u4_is_one_world_list(self): + vgm = VisualGenMapping(world_size=8, rank=0, cfg_size=2, ulysses_size=4) + assert vgm.flatten_cfg_ranks() == [[0, 1, 2, 3, 4, 5, 6, 7]] + + def test_cfg2_cp2_u2_preserves_ring_fibers(self): + vgm = VisualGenMapping(world_size=8, rank=0, cfg_size=2, ring_size=2, ulysses_size=2) + lists = vgm.flatten_cfg_ranks() + assert lists == [[0, 1, 4, 5], [2, 3, 6, 7]] + # Every list holds a single cp coordinate, so stage-1 ring pairs + # ({0,2},{1,3},{4,6},{5,7}) are never split across lists. + for a, b in ((0, 2), (1, 3), (4, 6), (5, 7)): + assert sum(a in grp for grp in lists) == 1 + assert not any(a in grp and b in grp for grp in lists) + + def test_cfg2_attn2d_2x2_u1(self): + vgm = VisualGenMapping( + world_size=8, rank=0, cfg_size=2, attn2d_row_size=2, attn2d_col_size=2 + ) + assert vgm.flatten_cfg_ranks() == [[0, 4], [1, 5], [2, 6], [3, 7]] + + class TestSingleGPURanksAndGroups: def test_ranks_are_zero(self): vgm = VisualGenMapping(world_size=1, rank=0) @@ -507,6 +550,52 @@ def _logic_attn2d_seq_rank_matches_global_rank(rank, world_size): ) +def _logic_tp2_attn2d_2x1_groups(rank, world_size): + """Groups compose under tp=2 × attn2d(2×1) on 4 GPUs (guard removed). + + Mesh cfg-tp-cp_row-cp_col-ulysses with tp=2, attn2d_row=2, attn2d_col=1: + tp_rank = (rank // 2) % 2 + cp_rank = rank % 2 (row-major over cp_row × cp_col) + TP groups: {0,2} and {1,3}. attn2d_col groups: {0,1} and {2,3}. + """ + from tensorrt_llm._torch.device_mesh import DeviceMeshTopologyImpl + + DeviceMeshTopologyImpl.device_mesh = None + + vgm = VisualGenMapping( + world_size=world_size, + rank=rank, + tp_size=2, + attn2d_row_size=2, + attn2d_col_size=1, + ) + + assert vgm.tp_size == 2 + assert vgm.cp_size == 2 + assert vgm.tp_rank == (rank // 2) % 2 + assert vgm.cp_rank == rank % 2 + + assert vgm.tp_group_pg is not None + assert vgm.attn2d_row_group is not None + assert vgm.attn2d_col_group is not None + assert vgm.attn2d_mesh_group is not None + assert dist.get_world_size(vgm.tp_group_pg) == 2 + assert dist.get_world_size(vgm.attn2d_row_group) == 1 + assert dist.get_world_size(vgm.attn2d_col_group) == 2 + assert dist.get_world_size(vgm.attn2d_mesh_group) == 2 + + device = torch.device(f"cuda:{rank}") + one = torch.ones(1, device=device) + + x = one.clone() + dist.all_reduce(x, group=vgm.tp_group_pg) + assert x.item() == 2.0, f"Rank {rank}: tp all_reduce expected 2, got {x.item()}" + + x = one.clone() + dist.all_reduce(x, group=vgm.attn2d_col_group) + assert x.item() == 2.0, f"Rank {rank}: attn2d_col all_reduce expected 2, got {x.item()}" + + @pytest.mark.skipif(not MODULES_AVAILABLE, reason="Modules not available") class TestMultiGPU: def test_default_order_cfg2_ulysses2(self): @@ -531,3 +620,7 @@ def test_cfg2_ring2_ulysses2(self): def test_attn2d_ulysses_seq_rank_matches_global_rank(self): """Row-major mesh: seq_rank == global rank when cfg=tp=1 (8-way Attn2D+Ulysses).""" _run_multi_gpu(8, _logic_attn2d_seq_rank_matches_global_rank) + + def test_tp2_attn2d_2x1_groups(self): + """TP + Attn2D groups coexist and support collectives on 4 GPUs.""" + _run_multi_gpu(4, _logic_tp2_attn2d_2x1_groups) diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_tp.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_tp.py index a7fa1e38c31d..7c9ce3d2ccbb 100644 --- a/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_tp.py +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_tp.py @@ -53,6 +53,8 @@ from tensorrt_llm.models.modeling_utils import QuantConfig + from .tp_shard_utils import copy_tp_parameter + MODULES_AVAILABLE = True except ImportError: MODULES_AVAILABLE = False @@ -154,6 +156,11 @@ def run_test_in_distributed(world_size: int, test_fn: Callable, use_cuda: bool = added_kv_proj_dim=256, # add_k_proj input dim = hidden_size (image embeds projected to hidden_size before blocks) ) +# Existing WAN configs are already uneven with TP=3: +# 4 attention heads split as 2+1+1, and ffn_dim=512 is not divisible by 3. +_WAN_UNEVEN_TP3_CONFIG = dict(_WAN_T2V_TEST_CONFIG) +_WAN_I2V_UNEVEN_TP3_CONFIG = dict(_WAN_I2V_TEST_CONFIG) + # ============================================================================= # Model config + weight helpers @@ -204,95 +211,32 @@ def _stabilize_model_weights(model): # ============================================================================= -# TP weight sharding helpers +# TP weight sharding helpers (see tp_shard_utils.py) # ============================================================================= -def _shard_dim0(tensor, tp_rank, tp_size): - """Shard a tensor along dim 0.""" - chunk = tensor.shape[0] // tp_size - return tensor[tp_rank * chunk : (tp_rank + 1) * chunk].contiguous() - - -def _shard_dim1(tensor, tp_rank, tp_size): - """Shard a tensor along dim 1.""" - chunk = tensor.shape[1] // tp_size - return tensor[:, tp_rank * chunk : (tp_rank + 1) * chunk].contiguous() - - -def _shard_fused_qkv(tensor, tp_rank, tp_size, q_dim, kv_dim): - """Shard a fused QKV weight [q_dim + 2*kv_dim, ...] preserving Q/K/V structure.""" - q, k, v = tensor.split([q_dim, kv_dim, kv_dim], dim=0) - return torch.cat( - [ - _shard_dim0(q, tp_rank, tp_size), - _shard_dim0(k, tp_rank, tp_size), - _shard_dim0(v, tp_rank, tp_size), - ], - dim=0, - ) - - -def _shard_fused_gate_up(tensor, tp_rank, tp_size): - """Shard a fused gate_up weight [2*intermediate, ...] preserving gate/up structure.""" - half = tensor.shape[0] // 2 - gate, up = tensor.split([half, half], dim=0) - return torch.cat( - [ - _shard_dim0(gate, tp_rank, tp_size), - _shard_dim0(up, tp_rank, tp_size), - ], - dim=0, - ) - - -def _copy_ref_weights_to_tp(ref_model, tp_model, tp_rank, tp_size): - """Copy weights from a TP=1 reference model into a TP model with correct sharding. - - Handles column-parallel (QKV, MLP up/gate), row-parallel (output projs), - fused QKV/gate_up weights, and replicated parameters (norms, embeddings). - """ +def _copy_ref_weights_to_tp(ref_model, tp_model, tp_rank, tp_size, config_dict): + """Copy weights from a TP=1 reference model into a TP model with correct sharding.""" ref_params = dict(ref_model.named_parameters()) + num_heads = config_dict["num_attention_heads"] + head_dim = config_dict["attention_head_dim"] + vgm = getattr(tp_model.model_config, "visual_gen_mapping", None) + ulysses_size = vgm.ulysses_size if vgm is not None else 1 with torch.no_grad(): for tp_name, tp_param in tp_model.named_parameters(): if tp_name not in ref_params: continue - - ref_param = ref_params[tp_name] - - if tp_param.shape == ref_param.shape: - # Replicated parameter (norms, embeddings, etc.) - tp_param.data.copy_(ref_param.data) - elif tp_param.ndim >= 2 and tp_param.shape[1] == ref_param.shape[1]: - # Column parallel: dim 0 is smaller (output dim sharded) - if "qkv_proj" in tp_name or "add_qkv_proj" in tp_name: - q_dim = ref_param.shape[0] // 3 - tp_param.data.copy_( - _shard_fused_qkv(ref_param.data, tp_rank, tp_size, q_dim, q_dim) - ) - elif "gate_up_proj" in tp_name: - tp_param.data.copy_(_shard_fused_gate_up(ref_param.data, tp_rank, tp_size)) - else: - tp_param.data.copy_(_shard_dim0(ref_param.data, tp_rank, tp_size)) - elif tp_param.ndim >= 2 and tp_param.shape[0] == ref_param.shape[0]: - # Row parallel: dim 1 is smaller (input dim sharded) - tp_param.data.copy_(_shard_dim1(ref_param.data, tp_rank, tp_size)) - elif tp_param.ndim == 1 and tp_param.shape[0] < ref_param.shape[0]: - # 1D bias for column parallel - if "qkv_proj" in tp_name or "add_qkv_proj" in tp_name: - q_dim = ref_param.shape[0] // 3 - tp_param.data.copy_( - _shard_fused_qkv(ref_param.data, tp_rank, tp_size, q_dim, q_dim) - ) - elif "gate_up_proj" in tp_name: - tp_param.data.copy_(_shard_fused_gate_up(ref_param.data, tp_rank, tp_size)) - else: - tp_param.data.copy_(_shard_dim0(ref_param.data, tp_rank, tp_size)) - else: - raise ValueError( - f"Cannot shard {tp_name}: ref={ref_param.shape}, tp={tp_param.shape}" - ) + copy_tp_parameter( + tp_name, + ref_params[tp_name], + tp_param, + tp_rank, + tp_size, + num_heads, + head_dim, + ulysses_size=ulysses_size, + ) # ============================================================================= @@ -341,6 +285,16 @@ def _logic_wan_t2v_tp_forward(rank, world_size): def _logic_wan_t2v_tp_vs_single_gpu(rank, world_size): """WAN T2V: TP 2-GPU output matches single-GPU reference.""" + _logic_wan_t2v_tp_vs_single_gpu_with_config(rank, world_size, _WAN_T2V_TEST_CONFIG) + + +def _logic_wan_t2v_tp3_uneven_vs_single_gpu(rank, world_size): + """WAN T2V: TP=3 with uneven head/FFN dims matches single-GPU reference.""" + _logic_wan_t2v_tp_vs_single_gpu_with_config(rank, world_size, _WAN_UNEVEN_TP3_CONFIG) + + +def _logic_wan_t2v_tp_vs_single_gpu_with_config(rank, world_size, config_dict): + """WAN T2V: TP output matches single-GPU reference.""" from tensorrt_llm._torch.visual_gen.models.wan.transformer_wan import WanTransformer3DModel device = torch.device(f"cuda:{rank}") @@ -353,15 +307,15 @@ def _logic_wan_t2v_tp_vs_single_gpu(rank, world_size): # Create single-GPU reference model torch.manual_seed(123) - ref_config = _make_model_config(_WAN_T2V_TEST_CONFIG, tp_size=1) + ref_config = _make_model_config(config_dict, tp_size=1) ref_model = WanTransformer3DModel(ref_config).to(device).to(compute_dtype) _stabilize_model_weights(ref_model) # Create TP model and copy sharded weights from ref torch.manual_seed(123) - tp_config = _make_model_config(_WAN_T2V_TEST_CONFIG, tp_size=world_size) + tp_config = _make_model_config(config_dict, tp_size=world_size) tp_model = WanTransformer3DModel(tp_config).to(device).to(compute_dtype) - _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size) + _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size, config_dict) # Same inputs on all ranks torch.manual_seed(456) @@ -421,7 +375,7 @@ def _logic_wan_t2v_tp_ulysses_vs_single_gpu(rank, world_size): ) combined_model = WanTransformer3DModel(combined_config).to(device).to(compute_dtype) vgm = combined_config.visual_gen_mapping - _copy_ref_weights_to_tp(ref_model, combined_model, vgm.tp_rank, tp_size) + _copy_ref_weights_to_tp(ref_model, combined_model, vgm.tp_rank, tp_size, _WAN_T2V_TEST_CONFIG) # Same inputs on all ranks (Ulysses shards at runtime) torch.manual_seed(456) @@ -505,6 +459,16 @@ def _logic_wan_i2v_tp_forward(rank, world_size): def _logic_wan_i2v_tp_vs_single_gpu(rank, world_size): """WAN I2V: TP 2-GPU output matches single-GPU reference.""" + _logic_wan_i2v_tp_vs_single_gpu_with_config(rank, world_size, _WAN_I2V_TEST_CONFIG) + + +def _logic_wan_i2v_tp3_uneven_vs_single_gpu(rank, world_size): + """WAN I2V: TP=3 with uneven head/FFN dims matches single-GPU reference.""" + _logic_wan_i2v_tp_vs_single_gpu_with_config(rank, world_size, _WAN_I2V_UNEVEN_TP3_CONFIG) + + +def _logic_wan_i2v_tp_vs_single_gpu_with_config(rank, world_size, config_dict): + """WAN I2V: TP output matches single-GPU reference.""" from tensorrt_llm._torch.visual_gen.models.wan.transformer_wan import WanTransformer3DModel device = torch.device(f"cuda:{rank}") @@ -518,15 +482,15 @@ def _logic_wan_i2v_tp_vs_single_gpu(rank, world_size): # Create single-GPU reference model torch.manual_seed(123) - ref_config = _make_model_config(_WAN_I2V_TEST_CONFIG, tp_size=1) + ref_config = _make_model_config(config_dict, tp_size=1) ref_model = WanTransformer3DModel(ref_config).to(device).to(compute_dtype) _stabilize_model_weights(ref_model) # Create TP model and copy sharded weights from ref torch.manual_seed(123) - tp_config = _make_model_config(_WAN_I2V_TEST_CONFIG, tp_size=world_size) + tp_config = _make_model_config(config_dict, tp_size=world_size) tp_model = WanTransformer3DModel(tp_config).to(device).to(compute_dtype) - _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size) + _copy_ref_weights_to_tp(ref_model, tp_model, rank, world_size, config_dict) # Same inputs on all ranks torch.manual_seed(456) @@ -537,7 +501,8 @@ def _logic_wan_i2v_tp_vs_single_gpu(rank, world_size): torch.randn(batch, txt_seq, 128, device=device, dtype=compute_dtype) * 0.1 ) encoder_hidden_states_image = ( - torch.randn(batch, img_seq, 64, device=device, dtype=compute_dtype) * 0.1 + torch.randn(batch, img_seq, config_dict["image_dim"], device=device, dtype=compute_dtype) + * 0.1 ) timestep = torch.tensor([0.5], device=device, dtype=compute_dtype) @@ -601,5 +566,17 @@ def test_wan_i2v_tp_vs_single_gpu(self): run_test_in_distributed(world_size=2, test_fn=_logic_wan_i2v_tp_vs_single_gpu) +class TestWanUnevenTP3: + """TP=3 tests where head count and FFN dims are not divisible by tp_size.""" + + def test_wan_t2v_tp3_uneven_vs_single_gpu(self): + """WAN T2V TP=3 (4 heads, uneven FFN) matches single-GPU reference.""" + run_test_in_distributed(world_size=3, test_fn=_logic_wan_t2v_tp3_uneven_vs_single_gpu) + + def test_wan_i2v_tp3_uneven_vs_single_gpu(self): + """WAN I2V TP=3 (4 heads, uneven FFN) matches single-GPU reference.""" + run_test_in_distributed(world_size=3, test_fn=_logic_wan_i2v_tp3_uneven_vs_single_gpu) + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_transformer_parallel.py b/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_transformer_parallel.py index 07bedbb5a68a..9da4942eddaf 100644 --- a/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_transformer_parallel.py +++ b/tests/unittest/_torch/visual_gen/multi_gpu/test_wan_transformer_parallel.py @@ -51,6 +51,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from _visual_gen_dist_utils import spawn_with_retry + from .tp_shard_utils import copy_tp_parameter + MODULES_AVAILABLE = True except ImportError: MODULES_AVAILABLE = False @@ -158,8 +160,8 @@ def run_test_in_distributed(world_size: int, test_fn: Callable, use_cuda: bool = ATOL = 1e-2 RTOL = 1e-3 -# All valid 8-GPU combinations of (ulysses, ring, attn2d): -# world_size = (ring or attn2d_row*attn2d_col or 1) * ulysses = 8 +# All valid 8-GPU combinations of (tp, ulysses, ring, attn2d): +# world_size = tp * (ring or attn2d_row*attn2d_col or 1) * ulysses = 8 _WAN_8GPU_PARALLEL_COMBINATIONS = [ # Ulysses-only family (no ring / no attn2d) ("ulysses_only_ul8", dict(dit_ulysses_size=8)), @@ -170,6 +172,15 @@ def run_test_in_distributed(world_size: int, test_fn: Callable, use_cuda: bool = # Attention2D/Ulysses family ("attn2d_1x8_ul1", dict(dit_attn2d_row_size=1, dit_attn2d_col_size=8, dit_ulysses_size=1)), ("attn2d_2x4_ul1", dict(dit_attn2d_row_size=2, dit_attn2d_col_size=4, dit_ulysses_size=1)), + # TP + Attention2D (guard removed in mapping.py — commit 084755212d) + ( + "tp2_attn2d_2x1_ul2", + dict(dit_tp_size=2, dit_attn2d_row_size=2, dit_attn2d_col_size=1, dit_ulysses_size=2), + ), + ( + "tp2_attn2d_2x2_ul1", + dict(dit_tp_size=2, dit_attn2d_row_size=2, dit_attn2d_col_size=2), + ), ] @@ -194,6 +205,7 @@ def _make_model_config( pretrained_dict, *, cfg_size=1, + tp_size=1, ulysses_size=1, ring_size=1, attn2d_row_size=1, @@ -204,6 +216,7 @@ def _make_model_config( # Accept both shorthand names (cfg_size, ...) and VisualGen-style names # (dit_cfg_size, ...), since tests pass the latter. cfg_size = parallel_kwargs.pop("dit_cfg_size", cfg_size) + tp_size = parallel_kwargs.pop("dit_tp_size", tp_size) ulysses_size = parallel_kwargs.pop("dit_ulysses_size", ulysses_size) ring_size = parallel_kwargs.pop("dit_ring_size", ring_size) attn2d_row_size = parallel_kwargs.pop("dit_attn2d_row_size", attn2d_row_size) @@ -213,7 +226,11 @@ def _make_model_config( pretrained_config = SimpleNamespace(**pretrained_dict) use_dist = ( - cfg_size > 1 or ulysses_size > 1 or ring_size > 1 or attn2d_row_size * attn2d_col_size > 1 + cfg_size > 1 + or tp_size > 1 + or ulysses_size > 1 + or ring_size > 1 + or attn2d_row_size * attn2d_col_size > 1 ) and dist.is_initialized() if use_dist: ws = dist.get_world_size() @@ -225,6 +242,7 @@ def _make_model_config( world_size=ws, rank=rk, cfg_size=cfg_size, + tp_size=tp_size, ulysses_size=ulysses_size, ring_size=ring_size, attn2d_row_size=attn2d_row_size, @@ -248,6 +266,27 @@ def _free(*objs) -> None: torch.cuda.empty_cache() +def _copy_ref_weights_to_tp(ref_model, tp_model, pretrained_config) -> None: + """Copy reference weights into a TP model using its local parameter layouts.""" + ref_params = dict(ref_model.named_parameters()) + vgm = tp_model.model_config.visual_gen_mapping + num_heads = int(pretrained_config["num_attention_heads"]) + head_dim = int(pretrained_config["attention_head_dim"]) + + with torch.no_grad(): + for tp_name, tp_param in tp_model.named_parameters(): + copy_tp_parameter( + tp_name, + ref_params[tp_name], + tp_param, + vgm.tp_rank, + vgm.tp_size, + num_heads, + head_dim, + ulysses_size=vgm.ulysses_size, + ) + + # ============================================================================= # Core logic # ============================================================================= @@ -282,7 +321,10 @@ def _logic_wan_transformer_parallel_vs_single_gpu( except (ImportError, ValueError, NotImplementedError) as e: pytest.skip(f"[{label}] Parallel backend unavailable: {e}") - dist_model.load_state_dict(ref_state) + if dist_config.visual_gen_mapping.tp_size > 1: + _copy_ref_weights_to_tp(ref_model, dist_model, pretrained_cfg) + else: + dist_model.load_state_dict(ref_state) torch.manual_seed(SEED_INPUT) hidden_states = torch.randn((B, C, T, H, W), device=device, dtype=dtype) * 0.1 diff --git a/tests/unittest/_torch/visual_gen/multi_gpu/tp_shard_utils.py b/tests/unittest/_torch/visual_gen/multi_gpu/tp_shard_utils.py new file mode 100644 index 000000000000..7abbb121214e --- /dev/null +++ b/tests/unittest/_torch/visual_gen/multi_gpu/tp_shard_utils.py @@ -0,0 +1,297 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TP weight-sharding helpers for VisualGen multi-GPU unit tests. + +Mirrors ``Linear._calc_shard`` for MLP dims and ``Attention.shard_start`` for +head-aligned Q/K/V shards so reference weights match TP module layouts. +""" + +from __future__ import annotations + +import torch + + +def calc_shard(total: int, tp_size: int, rank: int) -> int: + """Start index for *rank* when splitting *total* elements across *tp_size* ranks.""" + return (total // tp_size) * rank + min(total % tp_size, rank) + + +def qkv_head_bounds( + tp_rank: int, + tp_size: int, + num_attention_heads: int, + num_key_value_heads: int, + head_dim: int, + ulysses_size: int = 1, +) -> tuple[int, int, int, int]: + """Return (q_start, q_end, kv_start, kv_end) feature bounds for one TP rank.""" + gqa_ratio = num_attention_heads // num_key_value_heads + kv_heads_per_ulysses = num_key_value_heads // ulysses_size + kv_head_start = calc_shard(kv_heads_per_ulysses, tp_size, tp_rank) * ulysses_size + kv_head_end = calc_shard(kv_heads_per_ulysses, tp_size, tp_rank + 1) * ulysses_size + attn_head_start = kv_head_start * gqa_ratio + attn_head_end = kv_head_end * gqa_ratio + q_start = attn_head_start * head_dim + q_end = attn_head_end * head_dim + kv_start = kv_head_start * head_dim + kv_end = kv_head_end * head_dim + return q_start, q_end, kv_start, kv_end + + +def shard_dim0(tensor: torch.Tensor, tp_rank: int, tp_size: int) -> torch.Tensor: + """Shard a tensor along dim 0 (column-parallel output / 1D bias).""" + start = calc_shard(tensor.shape[0], tp_size, tp_rank) + end = calc_shard(tensor.shape[0], tp_size, tp_rank + 1) + return tensor[start:end].contiguous() + + +def shard_dim1(tensor: torch.Tensor, tp_rank: int, tp_size: int) -> torch.Tensor: + """Shard a tensor along dim 1 (row-parallel input).""" + start = calc_shard(tensor.shape[1], tp_size, tp_rank) + end = calc_shard(tensor.shape[1], tp_size, tp_rank + 1) + return tensor[:, start:end].contiguous() + + +def shard_fused_qkv_by_heads( + tensor: torch.Tensor, + tp_rank: int, + tp_size: int, + num_attention_heads: int, + num_key_value_heads: int, + head_dim: int, + q_dim: int, + kv_dim: int, + ulysses_size: int = 1, +) -> torch.Tensor: + """Shard fused QKV using Attention head boundaries, not flat row splits.""" + q, k, v = tensor.split([q_dim, kv_dim, kv_dim], dim=0) + q_start, q_end, kv_start, kv_end = qkv_head_bounds( + tp_rank, tp_size, num_attention_heads, num_key_value_heads, head_dim, ulysses_size + ) + return torch.cat([q[q_start:q_end], k[kv_start:kv_end], v[kv_start:kv_end]], dim=0).contiguous() + + +def shard_kv_dim0( + tensor: torch.Tensor, + tp_rank: int, + tp_size: int, + num_key_value_heads: int, + head_dim: int, + ulysses_size: int = 1, +) -> torch.Tensor: + """Shard column-parallel K/V (or KV-norm) weights along head boundaries.""" + _, _, kv_start, kv_end = qkv_head_bounds( + tp_rank, tp_size, num_key_value_heads, num_key_value_heads, head_dim, ulysses_size + ) + return tensor[kv_start:kv_end].contiguous() + + +def shard_q_dim0( + tensor: torch.Tensor, + tp_rank: int, + tp_size: int, + num_attention_heads: int, + head_dim: int, + num_key_value_heads: int | None = None, + ulysses_size: int = 1, +) -> torch.Tensor: + """Shard column-parallel Q weights along head boundaries.""" + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + q_start, q_end, _, _ = qkv_head_bounds( + tp_rank, tp_size, num_attention_heads, num_key_value_heads, head_dim, ulysses_size + ) + return tensor[q_start:q_end].contiguous() + + +def shard_q_dim1( + tensor: torch.Tensor, + tp_rank: int, + tp_size: int, + num_attention_heads: int, + head_dim: int, + num_key_value_heads: int | None = None, + ulysses_size: int = 1, +) -> torch.Tensor: + """Shard row-parallel output-proj weights along Q head boundaries.""" + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + q_start, q_end, _, _ = qkv_head_bounds( + tp_rank, tp_size, num_attention_heads, num_key_value_heads, head_dim, ulysses_size + ) + return tensor[:, q_start:q_end].contiguous() + + +def shard_kv_dim1( + tensor: torch.Tensor, + tp_rank: int, + tp_size: int, + num_key_value_heads: int, + head_dim: int, + ulysses_size: int = 1, +) -> torch.Tensor: + """Shard row-parallel K/V output weights along head boundaries.""" + _, _, kv_start, kv_end = qkv_head_bounds( + tp_rank, tp_size, num_key_value_heads, num_key_value_heads, head_dim, ulysses_size + ) + return tensor[:, kv_start:kv_end].contiguous() + + +def shard_fused_gate_up(tensor: torch.Tensor, tp_rank: int, tp_size: int) -> torch.Tensor: + """Shard fused gate_up weight [2*intermediate, ...] preserving gate/up structure.""" + half = tensor.shape[0] // 2 + gate, up = tensor.split([half, half], dim=0) + return torch.cat( + [ + shard_dim0(gate, tp_rank, tp_size), + shard_dim0(up, tp_rank, tp_size), + ], + dim=0, + ) + + +def copy_tp_parameter( + tp_name: str, + ref_param: torch.Tensor, + tp_param: torch.Tensor, + tp_rank: int, + tp_size: int, + num_attention_heads: int, + head_dim: int, + num_key_value_heads: int | None = None, + ulysses_size: int = 1, +) -> None: + """Copy one reference parameter into its TP-sharded counterpart.""" + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + if tp_param.shape == ref_param.shape: + tp_param.data.copy_(ref_param.data) + return + + if tp_param.ndim >= 2 and tp_param.shape[1] == ref_param.shape[1]: + if "qkv_proj" in tp_name or "add_qkv_proj" in tp_name: + q_dim = num_attention_heads * head_dim + kv_dim = num_key_value_heads * head_dim + tp_param.data.copy_( + shard_fused_qkv_by_heads( + ref_param.data, + tp_rank, + tp_size, + num_attention_heads, + num_key_value_heads, + head_dim, + q_dim, + kv_dim, + ulysses_size, + ) + ) + elif "to_q" in tp_name: + tp_param.data.copy_( + shard_q_dim0( + ref_param.data, + tp_rank, + tp_size, + num_attention_heads, + head_dim, + num_key_value_heads, + ulysses_size, + ) + ) + elif ( + "to_k" in tp_name + or "to_v" in tp_name + or "add_k_proj" in tp_name + or "add_v_proj" in tp_name + ): + tp_param.data.copy_( + shard_kv_dim0( + ref_param.data, tp_rank, tp_size, num_key_value_heads, head_dim, ulysses_size + ) + ) + elif "gate_up_proj" in tp_name: + tp_param.data.copy_(shard_fused_gate_up(ref_param.data, tp_rank, tp_size)) + else: + tp_param.data.copy_(shard_dim0(ref_param.data, tp_rank, tp_size)) + elif tp_param.ndim >= 2 and tp_param.shape[0] == ref_param.shape[0]: + if "to_add_out" in tp_name: + tp_param.data.copy_( + shard_kv_dim1( + ref_param.data, tp_rank, tp_size, num_key_value_heads, head_dim, ulysses_size + ) + ) + elif "to_out" in tp_name: + tp_param.data.copy_( + shard_q_dim1( + ref_param.data, + tp_rank, + tp_size, + num_attention_heads, + head_dim, + num_key_value_heads, + ulysses_size, + ) + ) + else: + tp_param.data.copy_(shard_dim1(ref_param.data, tp_rank, tp_size)) + elif tp_param.ndim == 1 and tp_param.shape[0] < ref_param.shape[0]: + if "qkv_proj" in tp_name or "add_qkv_proj" in tp_name: + q_dim = num_attention_heads * head_dim + kv_dim = num_key_value_heads * head_dim + tp_param.data.copy_( + shard_fused_qkv_by_heads( + ref_param.data, + tp_rank, + tp_size, + num_attention_heads, + num_key_value_heads, + head_dim, + q_dim, + kv_dim, + ulysses_size, + ) + ) + elif "to_q" in tp_name or "norm_q" in tp_name: + tp_param.data.copy_( + shard_q_dim0( + ref_param.data, + tp_rank, + tp_size, + num_attention_heads, + head_dim, + num_key_value_heads, + ulysses_size, + ) + ) + elif ( + "to_k" in tp_name + or "to_v" in tp_name + or "add_k_proj" in tp_name + or "add_v_proj" in tp_name + or "norm_added_k" in tp_name + or "norm_k" in tp_name + ): + tp_param.data.copy_( + shard_kv_dim0( + ref_param.data, tp_rank, tp_size, num_key_value_heads, head_dim, ulysses_size + ) + ) + elif "gate_up_proj" in tp_name: + tp_param.data.copy_(shard_fused_gate_up(ref_param.data, tp_rank, tp_size)) + else: + tp_param.data.copy_(shard_dim0(ref_param.data, tp_rank, tp_size)) + else: + raise ValueError(f"Cannot shard {tp_name}: ref={ref_param.shape}, tp={tp_param.shape}") diff --git a/tests/unittest/_torch/visual_gen/test_attention_cute_dsl.py b/tests/unittest/_torch/visual_gen/test_attention_cute_dsl.py index eb6a6ec1093a..e122a84fc104 100644 --- a/tests/unittest/_torch/visual_gen/test_attention_cute_dsl.py +++ b/tests/unittest/_torch/visual_gen/test_attention_cute_dsl.py @@ -13,36 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import cutlass import pytest import torch import torch.nn.functional as F -from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention import cute_dsl_fmha_fwd -from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention.fmha import ( - get_cute_dsl_fmha_cubin, +from tensorrt_llm._torch.visual_gen.attention_backend.cute_dsl.fmha import ( + _COMPILE_CACHE, + _quantize_blockscaled_one, + _quantize_fp8_v, + clear_cute_dsl_fmha_cache, + cute_dsl_fmha_fwd, ) -def test_cute_dsl_cubin_kernel_can_import_and_load() -> None: - gpu_arch = _require_supported_gpu_arch() - try: - kernel = get_cute_dsl_fmha_cubin( - torch.bfloat16, - torch.bfloat16, - torch.bfloat16, - 128, - is_causal=False, - is_persistent=False, - varlen=True, - enable_tvm_ffi=True, - gpu_arch=gpu_arch, - ) - except ImportError as exc: - pytest.skip(str(exc)) - - assert kernel is not None - - def _require_supported_gpu_arch() -> str: if not torch.cuda.is_available(): pytest.skip("CUDA is required for CuTe DSL FMHA kernels.") @@ -55,19 +39,31 @@ def _require_supported_gpu_arch() -> str: return gpu_arch -def _make_indptr(lens: list[int], device: torch.device) -> torch.Tensor: - lens_tensor = torch.tensor(lens, dtype=torch.int32, device=device) - return torch.cat( - [ - torch.zeros(1, dtype=torch.int32, device=device), - lens_tensor.cumsum(0).int(), - ] +def test_cute_dsl_jit_compile_smoke() -> None: + """Compile (or fetch from cache) the kernel once and verify the cache grew.""" + _require_supported_gpu_arch() + device = torch.device("cuda:0") + batch_size, seq_len, num_heads, head_dim = 1, 128, 1, 128 + sm_scale = head_dim**-0.5 + + torch.manual_seed(0) + q = ( + torch.randn(batch_size, seq_len, num_heads, head_dim, dtype=torch.bfloat16, device=device) + * 0.4 ) + k = torch.randn_like(q) * 0.4 + v = torch.randn_like(q) * 0.4 + out = torch.empty_like(q) + + clear_cute_dsl_fmha_cache() + cute_dsl_fmha_fwd(q, k, v, out, is_causal=False, sm_scale=sm_scale) + torch.cuda.synchronize() + assert len(_COMPILE_CACHE) == 1 def _make_tensor( - max_len: int, - total_len: int, + batch_size: int, + seq_len: int, num_heads: int, head_dim: int, dtype: torch.dtype, @@ -75,63 +71,54 @@ def _make_tensor( scale: float, device: torch.device, ) -> tuple[torch.Tensor, torch.Tensor]: - tensor_f32 = torch.randn( - max_len + total_len, - num_heads, - head_dim, - dtype=torch.float32, - device=device, + """Build a (B, S, H, D) tensor in `dtype` plus its dequantized reference.""" + tensor_f32 = ( + torch.randn( + batch_size, + seq_len, + num_heads, + head_dim, + dtype=torch.float32, + device=device, + ) + * 0.4 ) - tensor_f32 *= 0.1 if dtype == torch.float8_e4m3fn: tensor = (tensor_f32 / scale).to(dtype) ref = (tensor.float() * scale).to(ref_dtype) else: tensor = tensor_f32.to(dtype) ref = tensor.to(ref_dtype) - return tensor[max_len:], ref[max_len:] + return tensor, ref def _sdpa_ref( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, - qo_indptr: torch.Tensor, - kv_indptr: torch.Tensor, is_causal: bool, sm_scale: float, ) -> torch.Tensor: - out = [] - for batch_idx in range(qo_indptr.numel() - 1): - q_start = int(qo_indptr[batch_idx].item()) - q_end = int(qo_indptr[batch_idx + 1].item()) - kv_start = int(kv_indptr[batch_idx].item()) - kv_end = int(kv_indptr[batch_idx + 1].item()) - - q_i = q[q_start:q_end].transpose(0, 1).unsqueeze(0) - k_i = k[kv_start:kv_end].transpose(0, 1).unsqueeze(0) - v_i = v[kv_start:kv_end].transpose(0, 1).unsqueeze(0) - if q_i.shape[1] != k_i.shape[1]: - repeat_factor = q_i.shape[1] // k_i.shape[1] - k_i = k_i.repeat_interleave(repeat_factor, dim=1) - v_i = v_i.repeat_interleave(repeat_factor, dim=1) - out_i = F.scaled_dot_product_attention( - q_i, - k_i, - v_i, - is_causal=is_causal, - scale=sm_scale, - ) - out.append(out_i.squeeze(0).transpose(0, 1)) - return torch.cat(out, dim=0) + """Reference: SDPA on (B, S, H, D) tensors, with GQA broadcast if needed.""" + q_bhsd = q.transpose(1, 2) + k_bhsd = k.transpose(1, 2) + v_bhsd = v.transpose(1, 2) + if q_bhsd.shape[1] != k_bhsd.shape[1]: + repeat = q_bhsd.shape[1] // k_bhsd.shape[1] + k_bhsd = k_bhsd.repeat_interleave(repeat, dim=1) + v_bhsd = v_bhsd.repeat_interleave(repeat, dim=1) + out = F.scaled_dot_product_attention( + q_bhsd, k_bhsd, v_bhsd, is_causal=is_causal, scale=sm_scale + ) + return out.transpose(1, 2) @pytest.mark.parametrize( - ("q_lens", "kv_lens", "is_causal"), + ("batch_size", "seq_len_q", "seq_len_kv", "is_causal"), [ - pytest.param([256], [256], False, id="single_nocausal"), - pytest.param([512], [512], True, id="single_causal"), - pytest.param([64, 128], [128, 512], False, id="varlen_nocausal"), + pytest.param(1, 256, 256, False, id="b1_s256_nocausal"), + pytest.param(1, 512, 512, True, id="b1_s512_causal"), + pytest.param(2, 256, 256, False, id="b2_s256_nocausal"), ], ) @pytest.mark.parametrize( @@ -164,9 +151,10 @@ def _sdpa_ref( ], ) @pytest.mark.parametrize("head_dim", [128]) -def test_cute_dsl_fmha_context_forward_cubin_smoke( - q_lens: list[int], - kv_lens: list[int], +def test_cute_dsl_fmha_context_forward( + batch_size: int, + seq_len_q: int, + seq_len_kv: int, is_causal: bool, qk_dtype: torch.dtype, pv_dtype: torch.dtype, @@ -177,72 +165,166 @@ def test_cute_dsl_fmha_context_forward_cubin_smoke( atol: float, rtol: float, ) -> None: - gpu_arch = _require_supported_gpu_arch() + _require_supported_gpu_arch() device = torch.device("cuda:0") sm_scale = head_dim**-0.5 scale_v = 0.06 if pv_dtype == torch.float8_e4m3fn else 1.0 torch.manual_seed(42) torch.cuda.manual_seed_all(42) - qo_indptr = _make_indptr(q_lens, device) - kv_indptr = _make_indptr(kv_lens, device) - total_q = int(qo_indptr[-1].item()) - total_kv = int(kv_indptr[-1].item()) - max_qo_len = max(q_lens) - max_kv_len = max(kv_lens) q, q_ref = _make_tensor( - max_qo_len, total_q, num_heads, head_dim, qk_dtype, out_dtype, 1.0, device + batch_size, seq_len_q, num_heads, head_dim, qk_dtype, out_dtype, 1.0, device ) k, k_ref = _make_tensor( - max_kv_len, total_kv, num_heads_kv, head_dim, qk_dtype, out_dtype, 1.0, device + batch_size, seq_len_kv, num_heads_kv, head_dim, qk_dtype, out_dtype, 1.0, device ) v, v_ref = _make_tensor( - max_kv_len, total_kv, num_heads_kv, head_dim, pv_dtype, out_dtype, scale_v, device - ) - out_storage = torch.empty( - max_qo_len + total_q, - num_heads, - head_dim, - dtype=out_dtype, - device=device, - ) - out = out_storage[max_qo_len:] - kernel_fn = get_cute_dsl_fmha_cubin( - qk_dtype, - pv_dtype, - out_dtype, - head_dim, - is_causal, - is_persistent=False, - varlen=True, - enable_tvm_ffi=True, - gpu_arch=gpu_arch, + batch_size, seq_len_kv, num_heads_kv, head_dim, pv_dtype, out_dtype, scale_v, device ) + out = torch.empty(batch_size, seq_len_q, num_heads, head_dim, dtype=out_dtype, device=device) + lse = torch.empty(batch_size, seq_len_q, num_heads, dtype=torch.float32, device=device) cute_dsl_fmha_fwd( q, k, v, out, - qo_indptr, - kv_indptr, is_causal=is_causal, sm_scale=sm_scale, scale_v=scale_v, - max_qo_len=max_qo_len, - max_kv_len=max_kv_len, - kernel_fn=kernel_fn, + lse=lse, ) torch.cuda.synchronize() - out_ref = _sdpa_ref( - q_ref, - k_ref, - v_ref, - qo_indptr, - kv_indptr, - is_causal, - sm_scale, + out_ref = _sdpa_ref(q_ref, k_ref, v_ref, is_causal, sm_scale) + torch.testing.assert_close(out, out_ref, atol=atol, rtol=rtol) + + +@pytest.mark.parametrize( + ("qk_sf_vec", "qk_cutlass_dtype_name", "atol", "rtol"), + [ + pytest.param(32, "Float8E4M3FN", 8e-2, 8e-2, id="mxfp8"), + pytest.param(16, "Float4E2M1FN", 2e-1, 2e-1, id="nvfp4"), + ], +) +@pytest.mark.parametrize( + ("batch_size", "seq_len_q", "seq_len_kv", "is_causal"), + [ + pytest.param(1, 256, 256, False, id="b1_s256_nocausal"), + pytest.param(1, 512, 512, True, id="b1_s512_causal"), + ], +) +@pytest.mark.parametrize( + ("num_heads", "num_heads_kv"), + [ + pytest.param(4, 4, id="mha"), + pytest.param(4, 2, id="gqa"), + ], +) +@pytest.mark.parametrize("v_block_size", [0, 1]) +def test_cute_dsl_fmha_blockscaled_forward( + batch_size: int, + seq_len_q: int, + seq_len_kv: int, + v_block_size: int, + is_causal: bool, + qk_sf_vec: int, + qk_cutlass_dtype_name: str, + num_heads: int, + num_heads_kv: int, + atol: float, + rtol: float, +) -> None: + """End-to-end MXFP8 / NVFP4 block-scaled Q@K path through cute_dsl_fmha_fwd. + + Drives the block-scaled kernel with TRT-LLM-quantized Q/K and FP8 V using either + one tensor scale or per-head-per-channel scales. + """ + _require_supported_gpu_arch() + + device = torch.device("cuda:0") + head_dim = 128 # kernel-imposed for block-scaled MXFP8 / NVFP4 + sm_scale = head_dim**-0.5 + + torch.manual_seed(42) + torch.cuda.manual_seed_all(42) + + q_bf16 = ( + torch.randn( + batch_size, + seq_len_q, + num_heads, + head_dim, + dtype=torch.bfloat16, + device=device, + ) + * 0.5 + ) + k_bf16 = ( + torch.randn( + batch_size, + seq_len_kv, + num_heads_kv, + head_dim, + dtype=torch.bfloat16, + device=device, + ) + * 0.5 + ) + v_bf16 = ( + torch.randn( + batch_size, + seq_len_kv, + num_heads_kv, + head_dim, + dtype=torch.bfloat16, + device=device, + ) + * 0.5 ) + + # Exercise both V modes: one tensor scale (0) and an (H, D) scale tensor (1). + q_q, q_sf, scale_q = _quantize_blockscaled_one(q_bf16, qk_sf_vec) + k_q, k_sf, scale_k = _quantize_blockscaled_one(k_bf16, qk_sf_vec) + v_q, scale_v, scale_v_channels = _quantize_fp8_v(v_bf16, per_head_channel=v_block_size == 1) + qk_cutlass_dtype = getattr(cutlass, qk_cutlass_dtype_name) + + out = torch.empty( + batch_size, + seq_len_q, + num_heads, + head_dim, + dtype=torch.bfloat16, + device=device, + ) + lse = torch.empty( + batch_size, + seq_len_q, + num_heads, + dtype=torch.float32, + device=device, + ) + + cute_dsl_fmha_fwd( + q_q, + k_q, + v_q, + out, + is_causal=is_causal, + scale_v=scale_v, + scale_v_channels=scale_v_channels, + sm_scale=sm_scale, + lse=lse, + scale_q=scale_q, + scale_k=scale_k, + qk_sf_vec=qk_sf_vec, + q_sf=q_sf, + k_sf=k_sf, + qk_cutlass_dtype=qk_cutlass_dtype, + ) + torch.cuda.synchronize() + + out_ref = _sdpa_ref(q_bf16, k_bf16, v_bf16, is_causal, sm_scale) + assert torch.isfinite(out).all(), "Block-scaled FMHA produced NaN / Inf" torch.testing.assert_close(out, out_ref, atol=atol, rtol=rtol) diff --git a/tests/unittest/_torch/visual_gen/test_attention_cute_dsl_vsa.py b/tests/unittest/_torch/visual_gen/test_attention_cute_dsl_vsa.py index cdad7f6f8b72..63684cd0519a 100644 --- a/tests/unittest/_torch/visual_gen/test_attention_cute_dsl_vsa.py +++ b/tests/unittest/_torch/visual_gen/test_attention_cute_dsl_vsa.py @@ -24,13 +24,49 @@ import torch import torch.nn.functional as F -from tensorrt_llm._torch.visual_gen.attention_backend import VSAMetadataBuilder +from tensorrt_llm._torch.visual_gen.attention_backend import ( + CuTeDSLAttention, + VSAAttention, + VSAMetadataBuilder, +) +from tensorrt_llm._torch.visual_gen.attention_backend.utils import create_attention from tensorrt_llm._torch.visual_gen.config import ( DiffusionModelConfig, create_attention_metadata_state, ) from tensorrt_llm._torch.visual_gen.modules.attention import Attention, QKVMode -from tensorrt_llm.visual_gen.args import AttentionConfig, VideoSparseAttentionConfig +from tensorrt_llm.visual_gen.args import ( + AttentionConfig, + QuantAttentionConfig, + VideoSparseAttentionConfig, +) + + +def test_cute_dsl_factory_dispatches_quantized_fmha_and_vsa() -> None: + quant_config = QuantAttentionConfig(qk_dtype="mxfp8", v_dtype="fp8", v_block_size=1) + dense_config = AttentionConfig(backend="CUTEDSL", quant_attention_config=quant_config) + dense_attention = create_attention( + backend="CUTEDSL", + layer_idx=0, + num_heads=8, + head_dim=128, + attention_config=dense_config, + ) + + sparse_config = VideoSparseAttentionConfig(vsa_sparsity=0.9) + vsa_config = AttentionConfig(backend="CUTEDSL", sparse_attention_config=sparse_config) + vsa_attention = create_attention( + backend="CUTEDSL", + layer_idx=0, + num_heads=8, + head_dim=128, + attention_config=vsa_config, + ) + + assert isinstance(dense_attention, CuTeDSLAttention) + assert dense_attention.quant_attention_config is quant_config + assert isinstance(vsa_attention, VSAAttention) + assert vsa_attention.sparse_attention_config is sparse_config def _make_config( diff --git a/tests/unittest/_torch/visual_gen/test_attention_integration.py b/tests/unittest/_torch/visual_gen/test_attention_integration.py index b9aa954c37de..a8babe53afb8 100644 --- a/tests/unittest/_torch/visual_gen/test_attention_integration.py +++ b/tests/unittest/_torch/visual_gen/test_attention_integration.py @@ -7,7 +7,6 @@ """ from types import SimpleNamespace -from typing import Optional import pytest import torch @@ -167,7 +166,7 @@ def create_model_config( return config -def _require_attention_backend(attn_backend: str, head_dim: Optional[int] = None) -> None: +def _require_attention_backend(attn_backend: str) -> None: if attn_backend == "FA4" and not _flash_attn4_available: pytest.fail("FlashAttention 4 backend is required for FA4 attention test") if attn_backend == "CUTEDSL" and not _cute_dsl_available: @@ -177,8 +176,6 @@ def _require_attention_backend(attn_backend: str, head_dim: Optional[int] = None gpu_arch = f"sm_{compute_capability[0]}{compute_capability[1]}a" if gpu_arch not in ("sm_100a", "sm_103a"): pytest.skip("CUTEDSL attention test requires a supported Blackwell-class GPU") - if head_dim is not None and head_dim != 128: - pytest.skip("CUTEDSL attention test requires head_dim=128") def _make_cross_attention_with_mapping( @@ -357,7 +354,7 @@ def test_self_attention_equivalence( head_dim: int, attn_backend: str, quant_attention_config: "QuantAttentionConfig | None" ): """Test that integrated self-attention produces same output as naive.""" - _require_attention_backend(attn_backend, head_dim) + _require_attention_backend(attn_backend) print("\n" + "=" * 60) print("Testing Self-Attention Equivalence") @@ -550,7 +547,7 @@ def test_cross_attention_equivalence( head_dim: int, attn_backend: str, quant_attention_config: "QuantAttentionConfig | None" ): """Test that integrated cross-attention produces same output as naive.""" - _require_attention_backend(attn_backend, head_dim) + _require_attention_backend(attn_backend) print("\n" + "=" * 60) print("Testing Cross-Attention Equivalence") @@ -651,7 +648,7 @@ def test_fast_cross_attention_wan_shapes( quant_attention_config: "QuantAttentionConfig | None", ): """Test fast cross-attention correctness at Wan-realistic shapes.""" - _require_attention_backend(attn_backend, head_dim) + _require_attention_backend(attn_backend) hidden_size = num_heads * head_dim device = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/tests/unittest/_torch/visual_gen/test_attention_perf.py b/tests/unittest/_torch/visual_gen/test_attention_perf.py index 763309f3f96d..a662b788cf0d 100644 --- a/tests/unittest/_torch/visual_gen/test_attention_perf.py +++ b/tests/unittest/_torch/visual_gen/test_attention_perf.py @@ -64,7 +64,7 @@ _cute_dsl_available = _cute_dsl_import_error is None -def _require_attention_backend(backend: str, head_dim: Optional[int] = None) -> None: +def _require_attention_backend(backend: str) -> None: if backend == "FA4" and not _flash_attn4_available: pytest.fail( "FlashAttention 4 backend is required for FA4 attention perf test" @@ -82,8 +82,6 @@ def _require_attention_backend(backend: str, head_dim: Optional[int] = None) -> gpu_arch = f"sm_{compute_capability[0]}{compute_capability[1]}a" if gpu_arch not in ("sm_100a", "sm_103a"): pytest.skip("CUTEDSL attention perf test requires a supported Blackwell-class GPU") - if head_dim is not None and head_dim != 128: - pytest.skip("CUTEDSL attention perf test requires head_dim=128") # NVTX support for profiling @@ -808,7 +806,7 @@ def test_self_attention_perf( quant_attention_config: "QuantAttentionConfig | None", ): """Test that attention backend runs without errors.""" - _require_attention_backend(backend, head_dim) + _require_attention_backend(backend) batch_size, num_heads, seq_len = 1, 24, 1024 diff --git a/tests/unittest/_torch/visual_gen/test_cache_dit.py b/tests/unittest/_torch/visual_gen/test_cache_dit.py index 9a345ba4a404..09cbaa55d35c 100644 --- a/tests/unittest/_torch/visual_gen/test_cache_dit.py +++ b/tests/unittest/_torch/visual_gen/test_cache_dit.py @@ -15,6 +15,7 @@ import logging import os from types import SimpleNamespace +from unittest.mock import patch import pytest import torch @@ -157,6 +158,40 @@ def _total_accumulated_cached_steps(stats: dict) -> int: return total +def _cache_dit_captured_blocks(transformer: torch.nn.Module) -> list[torch.nn.Module]: + """Block modules cache_dit captured at enable time. + + cache_dit replaces ``transformer.forward`` with a ``functools.partial`` + whose closure holds the UnifiedBlocks mapping; each cached-blocks module + keeps the block list it captured on ``.transformer_blocks``. Walk the + closure graph to collect those blocks so tests can assert on the exact + module references cache_dit drives at denoise time. + """ + fwd = transformer.forward + stack = [getattr(fwd, "func", fwd)] + seen: set[int] = set() + captured: list[torch.nn.Module] = [] + while stack: + obj = stack.pop() + if id(obj) in seen: + continue + seen.add(id(obj)) + closure = getattr(obj, "__closure__", None) + if closure: + for cell in closure: + try: + stack.append(cell.cell_contents) + except ValueError: + pass # empty cell + elif isinstance(obj, dict): + stack.extend(obj.values()) + elif isinstance(obj, torch.nn.ModuleList): + stack.extend(obj) + elif isinstance(obj, torch.nn.Module) and hasattr(obj, "transformer_blocks"): + captured.extend(obj.transformer_blocks) + return captured + + # --------------------------------------------------------------------------- # 1) Wan 2.2 high/low step split (no mocks) # --------------------------------------------------------------------------- @@ -239,7 +274,12 @@ def _teardown_cache_dit(pipeline: object) -> None: pipeline.cache_accelerator = None @staticmethod - def _load_visual_gen_pipeline(checkpoint_dir: str, *, text_encoder_path: str = ""): + def _load_visual_gen_pipeline( + checkpoint_dir: str, + *, + text_encoder_path: str = "", + enable_torch_compile: bool = False, + ): from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader from tensorrt_llm.visual_gen.args import ( CacheDiTConfig, @@ -267,7 +307,7 @@ def _load_visual_gen_pipeline(checkpoint_dir: str, *, text_encoder_path: str = " residual_diff_threshold=0.25, ), torch_compile_config=TorchCompileConfig( - enable=False, + enable=enable_torch_compile, enable_autotune=False, ), compilation_config=CompilationConfig(skip_warmup=True), @@ -314,6 +354,66 @@ def test_wan_cache_dit_skips_blocks_after_forward(self): if pipeline is not None: self._teardown_cache_dit(pipeline) + def test_wan_cache_dit_enabled_after_torch_compile(self): + """Locks the enable-after-compile ordering from PipelineLoader.load. + + Cache-DiT captures block-module references at enable time. If it were + enabled before torch_compile() (which replaces the block lists with + compiled copies), it would keep driving the stale eager blocks and + torch.compile would contribute nothing — a perf-only regression that + is invisible to the correctness assertions of the compile-off tests. + """ + ckpt = _resolve_wan_checkpoint() + if ckpt is None: + pytest.skip( + "Wan 2.1 1.3B not found: set TRTLLM_CACHE_DIT_WAN_CHECKPOINT, " + f"install under {_DEFAULT_WAN_CHECKPOINT}, " + f"or under $LLM_MODELS_ROOT/{_WAN_SUBPATH}" + ) + + pipeline = None + with _suppress_stdlib_logging_for_cache_dit(): + try: + pipeline = self._load_visual_gen_pipeline(ckpt, enable_torch_compile=True) + assert pipeline.cache_accelerator is not None + assert pipeline.cache_accelerator.is_enabled() + + # The blocks cache_dit holds must be the torch.compile wrappers, + # and exactly the ones currently installed on the transformer — + # not the eager blocks that torch_compile() replaced. + captured = _cache_dit_captured_blocks(pipeline.transformer) + assert captured, "no cache_dit-captured blocks found on the patched forward" + assert all(isinstance(b, torch._dynamo.OptimizedModule) for b in captured), ( + "cache_dit captured eager blocks; it was enabled before torch.compile" + ) + assert {id(b) for b in captured} == {id(b) for b in pipeline.transformer.blocks} + + # OptimizedModule blocks can still silently fall back to eager + # (LTX-2 does today): require dynamo to report compiled frames + # after a real forward. + counters = torch._dynamo.utils.counters + counters.clear() + with torch.inference_mode(): + pipeline.forward( + prompt="cache dit compile ordering validation", + negative_prompt="", + height=480, + width=832, + num_frames=33, + num_inference_steps=8, + guidance_scale=5.0, + seed=0, + max_sequence_length=256, + ) + assert counters["frames"]["ok"] > 0, ( + "torch.compile produced no compiled frames during forward; " + "compiled blocks fell back to eager" + ) + finally: + if pipeline is not None: + self._teardown_cache_dit(pipeline) + torch._dynamo.reset() + def test_flux_cache_dit_skips_blocks_after_forward(self): ckpt = _resolve_flux_checkpoint() if ckpt is None: @@ -410,3 +510,90 @@ def test_ltx2_cache_dit_skips_blocks_after_forward(self): finally: if pipeline is not None: self._teardown_cache_dit(pipeline) + + +# --------------------------------------------------------------------------- +# 3) FLUX enabler check flags for compiled vs. eager blocks (CPU) +# --------------------------------------------------------------------------- + + +class _IdentityBlock(torch.nn.Module): + def forward(self, hidden_states, encoder_hidden_states): + return hidden_states, encoder_hidden_states + + +@requires_cache_dit +class TestFluxEnablerCompiledBlockCheckFlags: + """enable_cache_dit_for_flux must disable cache_dit's inspect-based + forward-pattern checks exactly when blocks are torch.compile wrappers. + + Compiled blocks expose a ``(*args, **kwargs)`` forward, so cache_dit's + checks would assert at enable time (server startup) if left on; eager + blocks must keep them on. torch.compile here is lazy — no compilation + happens, so this runs on CPU. + """ + + @staticmethod + def _make_flux_like_pipeline(*, compiled: bool) -> SimpleNamespace: + def _blocks() -> torch.nn.ModuleList: + blocks = [_IdentityBlock(), _IdentityBlock()] + if compiled: + blocks = [torch.compile(b) for b in blocks] + return torch.nn.ModuleList(blocks) + + return SimpleNamespace( + transformer=SimpleNamespace( + transformer_blocks=_blocks(), + single_transformer_blocks=_blocks(), + ) + ) + + def _block_adapter_kwargs(self, *, compiled: bool, is_flux2: bool) -> dict: + from tensorrt_llm._torch.visual_gen.cache import cache_dit_enablers + from tensorrt_llm.visual_gen.args import CacheDiTConfig + + pipeline = self._make_flux_like_pipeline(compiled=compiled) + with ( + patch.object(cache_dit_enablers, "BlockAdapter") as adapter_cls, + patch.object(cache_dit_enablers, "cache_dit"), + ): + cache_dit_enablers.enable_cache_dit_for_flux( + pipeline, CacheDiTConfig(), is_flux2=is_flux2 + ) + adapter_cls.assert_called_once() + return adapter_cls.call_args.kwargs + + @pytest.mark.parametrize("is_flux2", [False, True]) + def test_compiled_blocks_disable_pattern_checks(self, is_flux2): + kwargs = self._block_adapter_kwargs(compiled=True, is_flux2=is_flux2) + assert kwargs["check_forward_pattern"] is False + assert kwargs["check_num_outputs"] is False + + @pytest.mark.parametrize("is_flux2", [False, True]) + def test_eager_blocks_keep_pattern_checks(self, is_flux2): + kwargs = self._block_adapter_kwargs(compiled=False, is_flux2=is_flux2) + assert kwargs["check_forward_pattern"] is True + assert kwargs["check_num_outputs"] is True + + +# --------------------------------------------------------------------------- +# 4) Enabler registry declaration consistency (CPU) +# --------------------------------------------------------------------------- + + +@requires_cache_dit +class TestCacheDiTEnablerRegistry: + def test_enabler_keys_name_registered_pipeline_classes(self): + """Every CUSTOM_CACHE_DIT_ENABLERS key must name a real registered + pipeline class; a typo or rename would otherwise turn every enable + attempt for that pipeline into the 'no enabler registered' error.""" + import tensorrt_llm._torch.visual_gen.models # noqa: F401 (populates the registry) + from tensorrt_llm._torch.visual_gen.cache.cache_dit_enablers import ( + CUSTOM_CACHE_DIT_ENABLERS, + ) + from tensorrt_llm._torch.visual_gen.pipeline_registry import PIPELINE_REGISTRY + + unknown = sorted(set(CUSTOM_CACHE_DIT_ENABLERS) - set(PIPELINE_REGISTRY)) + assert not unknown, ( + f"CUSTOM_CACHE_DIT_ENABLERS keys not in the pipeline registry: {unknown}" + ) diff --git a/tests/unittest/_torch/visual_gen/test_cosmos3_distilled.py b/tests/unittest/_torch/visual_gen/test_cosmos3_distilled.py new file mode 100644 index 000000000000..e6664dea569e --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_cosmos3_distilled.py @@ -0,0 +1,944 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the Cosmos3 sampling policy (base vs distilled checkpoints) +and its pipeline wiring: scheduler loading, recipe validation, generation +defaults, mode resolution, and the guidance-1.0 denoise-loop contract.""" + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +from diffusers import FlowMatchEulerDiscreteScheduler, UniPCMultistepScheduler + +from tensorrt_llm._torch.visual_gen.models.cosmos3.defaults import ( + COSMOS3_720P_PARAMS, + COSMOS3_T2I_PARAMS, +) +from tensorrt_llm._torch.visual_gen.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniMoTPipeline +from tensorrt_llm._torch.visual_gen.models.cosmos3.sampling import ( + DISTILLED_GUIDANCE_SCALE, + Cosmos3SamplingPolicy, + load_scheduler, +) +from tensorrt_llm._torch.visual_gen.pipeline_registry import PIPELINE_REGISTRY, AutoPipeline + +pytestmark = [pytest.mark.cosmos3, pytest.mark.usefixtures("disable_cosmos3_guardrails")] + +# The relevant subset of the 4-Step checkpoint's scheduler config +# (values verbatim; keys unrelated to distilled detection omitted). +DISTILLED_SIGMAS = (1.0, 0.9375, 0.8333333333333334, 0.625) +DISTILLED_SCHEDULER_CONFIG = { + "_class_name": "FlowMatchEulerDiscreteScheduler", + "num_train_timesteps": 1000, + "shift": 1.0, + "stochastic_sampling": True, + "use_karras_sigmas": False, + "fixed_step_requires_explicit_sigmas": True, + "fixed_step_sampler_config": { + "sample_type": "sde", + "t_list": list(DISTILLED_SIGMAS), + }, +} + +UNIPC_SCHEDULER_CONFIG = { + "_class_name": "UniPCMultistepScheduler", + "num_train_timesteps": 1000, + "flow_shift": 1.0, + "prediction_type": "flow_prediction", + "use_flow_sigmas": True, + "solver_order": 2, +} + +SKIP_NON_SCHEDULER = ["text_tokenizer", "tokenizer", "vae", "sound_tokenizer"] + + +def _write_scheduler_config(checkpoint_dir: Path, config: dict) -> None: + scheduler_dir = checkpoint_dir / "scheduler" + scheduler_dir.mkdir(parents=True, exist_ok=True) + with open(scheduler_dir / "scheduler_config.json", "w") as f: + json.dump(config, f) + + +def _distilled_policy() -> Cosmos3SamplingPolicy: + scheduler = FlowMatchEulerDiscreteScheduler.from_config(DISTILLED_SCHEDULER_CONFIG) + return Cosmos3SamplingPolicy.from_scheduler(scheduler) + + +def _base_scheduler() -> UniPCMultistepScheduler: + return UniPCMultistepScheduler.from_config(UNIPC_SCHEDULER_CONFIG) + + +def _base_policy() -> Cosmos3SamplingPolicy: + return Cosmos3SamplingPolicy.from_scheduler(_base_scheduler()) + + +def _bare_pipeline(**attrs) -> Cosmos3OmniMoTPipeline: + """A pipeline instance without heavyweight __init__; ``rank``/``dtype``/ + ``device`` are BasePipeline properties and must not be set here.""" + pipeline = object.__new__(Cosmos3OmniMoTPipeline) + defaults = dict( + audio_gen=False, + action_gen=False, + sampling=Cosmos3SamplingPolicy(), + default_use_system_prompt=False, + ) + defaults.update(attrs) + for key, value in defaults.items(): + setattr(pipeline, key, value) + return pipeline + + +def _fake_request(output_type: str = "video", **param_overrides) -> SimpleNamespace: + """A DiffusionRequest look-alike with executor-merged (None = unset) params.""" + params = SimpleNamespace( + height=None, + width=None, + num_inference_steps=None, + guidance_scale=None, + num_frames=COSMOS3_720P_PARAMS["num_frames"], + max_sequence_length=COSMOS3_720P_PARAMS["max_sequence_length"], + frame_rate=COSMOS3_720P_PARAMS["frame_rate"], + seed=0, + negative_prompt=None, + image=None, + extra_params={"output_type": output_type}, + ) + for key, value in param_overrides.items(): + setattr(params, key, value) + return SimpleNamespace(prompt="x", params=params) + + +class TestSchedulerLoading: + def test_flow_match_declared(self, tmp_path): + _write_scheduler_config(tmp_path, DISTILLED_SCHEDULER_CONFIG) + assert isinstance(load_scheduler(str(tmp_path)), FlowMatchEulerDiscreteScheduler) + + def test_unipc_declared(self, tmp_path): + _write_scheduler_config(tmp_path, UNIPC_SCHEDULER_CONFIG) + assert isinstance(load_scheduler(str(tmp_path)), UniPCMultistepScheduler) + + def test_missing_class_name_defaults_to_unipc(self, tmp_path): + config = {k: v for k, v in UNIPC_SCHEDULER_CONFIG.items() if k != "_class_name"} + _write_scheduler_config(tmp_path, config) + assert isinstance(load_scheduler(str(tmp_path)), UniPCMultistepScheduler) + + def test_unknown_class_name_raises(self, tmp_path): + """Silently substituting UniPC for an unknown declared scheduler would + sample the checkpoint with the wrong integrator.""" + _write_scheduler_config(tmp_path, {**UNIPC_SCHEDULER_CONFIG, "_class_name": "DDIM"}) + with pytest.raises(ValueError, match="DDIM"): + load_scheduler(str(tmp_path)) + + +class TestPolicyFacts: + def test_is_distilled(self): + assert _distilled_policy().is_distilled + assert not _base_policy().is_distilled + assert not Cosmos3SamplingPolicy().is_distilled + + def test_diffusers_retains_unexpected_config_keys(self): + """Canary: diffusers must keep the unexpected fixed_step_sampler_config + key in scheduler.config; if an upgrade drops it, distilled detection + silently breaks.""" + scheduler = FlowMatchEulerDiscreteScheduler.from_config(DISTILLED_SCHEDULER_CONFIG) + policy = Cosmos3SamplingPolicy.from_scheduler(scheduler) + assert policy.fixed_sigmas == DISTILLED_SIGMAS + assert scheduler.config.stochastic_sampling is True + + def test_sigma_values_coerced_to_floats(self): + config = { + **DISTILLED_SCHEDULER_CONFIG, + "fixed_step_sampler_config": {"t_list": [1, "0.5"]}, + } + scheduler = FlowMatchEulerDiscreteScheduler.from_config(config) + assert Cosmos3SamplingPolicy.from_scheduler(scheduler).fixed_sigmas == (1.0, 0.5) + + def test_generation_default_overrides(self): + assert _distilled_policy().generation_default_overrides() == { + "num_inference_steps": 4, + "guidance_scale": DISTILLED_GUIDANCE_SCALE, + } + assert _base_policy().generation_default_overrides() == {} + + def test_num_steps(self): + assert _distilled_policy().num_steps(2) == 4 + assert _base_policy().num_steps(2) == 2 + + def test_checkpoint_flow_shift(self): + assert _base_policy().checkpoint_flow_shift == 1.0 + assert _distilled_policy().checkpoint_flow_shift == 1.0 # no UniPC config + + def test_scheduler_step_kwargs(self): + generator = torch.Generator().manual_seed(7) + assert _distilled_policy().scheduler_step_kwargs(generator) == {"generator": generator} + assert _base_policy().scheduler_step_kwargs(generator) == {} + + +class TestMalformedRecipeValidation: + """Only two recipes are valid; everything else must fail at load.""" + + @pytest.mark.parametrize("broken_fixed_step", [None, {}, {"t_list": []}]) + def test_required_sigmas_missing_raises(self, broken_fixed_step): + config = {k: v for k, v in DISTILLED_SCHEDULER_CONFIG.items()} + config.pop("fixed_step_sampler_config") + if broken_fixed_step is not None: + config["fixed_step_sampler_config"] = broken_fixed_step + scheduler = FlowMatchEulerDiscreteScheduler.from_config(config) + + with pytest.raises(ValueError, match="fixed_step_requires_explicit_sigmas"): + Cosmos3SamplingPolicy.from_scheduler(scheduler) + + def test_t_list_on_unipc_raises(self): + """UniPC cannot honor the distilled policy (no seeded step noise, + no baked-in guidance) even though its set_timesteps accepts sigmas.""" + config = { + **UNIPC_SCHEDULER_CONFIG, + "fixed_step_sampler_config": DISTILLED_SCHEDULER_CONFIG["fixed_step_sampler_config"], + } + scheduler = UniPCMultistepScheduler.from_config(config) + + with pytest.raises(ValueError, match="Unsupported Cosmos3 sampling recipe"): + Cosmos3SamplingPolicy.from_scheduler(scheduler) + + def test_unipc_with_declared_requirement_but_no_sigmas_raises(self): + config = {**UNIPC_SCHEDULER_CONFIG, "fixed_step_requires_explicit_sigmas": True} + scheduler = UniPCMultistepScheduler.from_config(config) + + with pytest.raises(ValueError, match="fixed_step_requires_explicit_sigmas"): + Cosmos3SamplingPolicy.from_scheduler(scheduler) + + def test_unipc_with_flag_and_sigmas_gets_unsupported_error(self): + config = { + **UNIPC_SCHEDULER_CONFIG, + "fixed_step_requires_explicit_sigmas": True, + "fixed_step_sampler_config": DISTILLED_SCHEDULER_CONFIG["fixed_step_sampler_config"], + } + scheduler = UniPCMultistepScheduler.from_config(config) + + with pytest.raises(ValueError, match="Unsupported Cosmos3 sampling recipe"): + Cosmos3SamplingPolicy.from_scheduler(scheduler) + + def test_flow_match_without_sigmas_raises(self): + config = {k: v for k, v in DISTILLED_SCHEDULER_CONFIG.items()} + config.pop("fixed_step_sampler_config") + config.pop("fixed_step_requires_explicit_sigmas") + scheduler = FlowMatchEulerDiscreteScheduler.from_config(config) + + with pytest.raises(ValueError, match="Unsupported Cosmos3 sampling recipe"): + Cosmos3SamplingPolicy.from_scheduler(scheduler) + + @pytest.mark.parametrize("stochastic", [False, "absent"]) + def test_non_stochastic_fixed_schedule_raises(self, stochastic): + """The distilled policy assumes SDE noise every step (seeded generator); + an ODE fixed-step recipe must not silently load as distilled.""" + config = dict(DISTILLED_SCHEDULER_CONFIG) + if stochastic == "absent": + config.pop("stochastic_sampling") # diffusers defaults it to False + else: + config["stochastic_sampling"] = stochastic + scheduler = FlowMatchEulerDiscreteScheduler.from_config(config) + + with pytest.raises(ValueError, match="stochastic_sampling"): + Cosmos3SamplingPolicy.from_scheduler(scheduler) + + def test_declared_non_sde_sample_type_raises(self): + config = { + **DISTILLED_SCHEDULER_CONFIG, + "fixed_step_sampler_config": { + "sample_type": "ode", + "t_list": list(DISTILLED_SIGMAS), + }, + } + scheduler = FlowMatchEulerDiscreteScheduler.from_config(config) + + with pytest.raises(ValueError, match="sample_type"): + Cosmos3SamplingPolicy.from_scheduler(scheduler) + + +class TestValidateRequest: + @pytest.mark.parametrize("steps", [None, 4]) + @pytest.mark.parametrize("guidance", [None, 1, 1.0]) + def test_valid_values_pass(self, steps, guidance): + _distilled_policy().validate_request(steps, guidance) + + @pytest.mark.parametrize("bad_steps", [1, 10, 35, 50, 100]) + def test_explicit_steps_mismatch_raises(self, bad_steps): + with pytest.raises(ValueError, match="distilled"): + _distilled_policy().validate_request(bad_steps, None) + + @pytest.mark.parametrize("bad_guidance", [0.5, 3.5, 6.0, 7.0]) + def test_explicit_guidance_mismatch_raises(self, bad_guidance): + with pytest.raises(ValueError, match="distilled"): + _distilled_policy().validate_request(None, bad_guidance) + + def test_base_policy_accepts_anything(self): + _base_policy().validate_request(17, 5.5) + _base_policy().validate_request(None, None) + + +class TestFlowShift: + def test_unipc_rebuilds_on_change(self): + policy = _base_policy() + scheduler = _base_scheduler() + + rebuilt = policy.set_flow_shift(scheduler, 3.0) + assert rebuilt is not scheduler + assert isinstance(rebuilt, UniPCMultistepScheduler) + assert float(rebuilt.config.flow_shift) == 3.0 + + def test_current_shift_read_from_scheduler_config(self): + """No separate shift-tracking state: a second call with the same target + on the rebuilt instance is a no-op; restoring rebuilds again.""" + policy = _base_policy() + rebuilt = policy.set_flow_shift(_base_scheduler(), 3.0) + assert policy.set_flow_shift(rebuilt, 3.0) is rebuilt + + restored = policy.set_flow_shift(rebuilt, 1.0) + assert restored is not rebuilt + assert float(restored.config.flow_shift) == 1.0 + + def test_distilled_is_structural_noop(self): + policy = _distilled_policy() + scheduler = FlowMatchEulerDiscreteScheduler.from_config(DISTILLED_SCHEDULER_CONFIG) + assert policy.set_flow_shift(scheduler, 3.0) is scheduler + + +class TestSetTimesteps: + def test_distilled_programs_fixed_sigmas(self): + policy = _distilled_policy() + scheduler = FlowMatchEulerDiscreteScheduler.from_config(DISTILLED_SCHEDULER_CONFIG) + policy.set_timesteps(scheduler, num_inference_steps=4, device="cpu") + expected = [s * 1000.0 for s in DISTILLED_SIGMAS] + assert torch.allclose(scheduler.timesteps.float(), torch.tensor(expected), atol=1e-3) + + def test_base_programs_step_count(self): + policy = _base_policy() + scheduler = _base_scheduler() + policy.set_timesteps(scheduler, num_inference_steps=7, device="cpu") + assert len(scheduler.timesteps) == 7 + + +class TestStochasticStepDeterminism: + """The seeded generator must fully determine the SDE noise trajectory.""" + + def _run_steps(self, seed): + policy = _distilled_policy() + scheduler = FlowMatchEulerDiscreteScheduler.from_config(DISTILLED_SCHEDULER_CONFIG) + policy.set_timesteps(scheduler, num_inference_steps=4, device="cpu") + generator = torch.Generator().manual_seed(seed) + kwargs = policy.scheduler_step_kwargs(generator) + + latents = torch.zeros(1, 4, 1, 2, 2) + velocity = torch.full_like(latents, 0.5) + for t in scheduler.timesteps: + latents = scheduler.step(velocity, t, latents, return_dict=False, **kwargs)[0] + return latents + + def test_same_seed_reproduces_sde_trajectory(self): + assert torch.equal(self._run_steps(seed=123), self._run_steps(seed=123)) + + def test_different_seeds_diverge(self): + assert not torch.equal(self._run_steps(seed=123), self._run_steps(seed=456)) + + +class TestGenerationDefaults: + def test_distilled_defaults_report_checkpoint_truth(self): + params = _bare_pipeline(sampling=_distilled_policy()).default_generation_params + assert params["num_inference_steps"] == 4 + assert params["guidance_scale"] == DISTILLED_GUIDANCE_SCALE + assert params["height"] is None # mode-dependent, resolved in infer() + assert params["num_frames"] == COSMOS3_720P_PARAMS["num_frames"] + + def test_base_defaults_leave_mode_dependent_fields_unset(self): + params = _bare_pipeline().default_generation_params + for field in ("height", "width", "num_inference_steps", "guidance_scale"): + assert params[field] is None + assert params["num_frames"] == COSMOS3_720P_PARAMS["num_frames"] + assert params["max_sequence_length"] == COSMOS3_720P_PARAMS["max_sequence_length"] + + +class TestInferModeResolution: + def _captured_forward_kwargs(self, pipeline, req): + captured = {} + pipeline.forward = lambda **kwargs: captured.update(kwargs) + pipeline.infer(req) + return captured + + def test_video_unset_resolves_to_video_table(self): + got = self._captured_forward_kwargs(_bare_pipeline(), _fake_request("video")) + assert got["height"] == COSMOS3_720P_PARAMS["height"] + assert got["width"] == COSMOS3_720P_PARAMS["width"] + assert got["num_inference_steps"] == COSMOS3_720P_PARAMS["num_inference_steps"] + assert got["guidance_scale"] == COSMOS3_720P_PARAMS["guidance_scale"] + + def test_t2i_unset_resolves_to_t2i_table(self): + got = self._captured_forward_kwargs(_bare_pipeline(), _fake_request("image")) + assert got["height"] == COSMOS3_T2I_PARAMS["height"] + assert got["width"] == COSMOS3_T2I_PARAMS["width"] + assert got["num_inference_steps"] == COSMOS3_T2I_PARAMS["num_inference_steps"] + assert got["guidance_scale"] == COSMOS3_T2I_PARAMS["guidance_scale"] + + def test_explicit_values_pass_through(self): + req = _fake_request("image", height=512, num_inference_steps=20) + got = self._captured_forward_kwargs(_bare_pipeline(), req) + assert got["height"] == 512 + assert got["num_inference_steps"] == 20 + assert got["width"] == COSMOS3_T2I_PARAMS["width"] + + def test_distilled_merged_defaults_pass_through(self): + req = _fake_request("image", num_inference_steps=4, guidance_scale=1.0) + got = self._captured_forward_kwargs(_bare_pipeline(sampling=_distilled_policy()), req) + assert got["num_inference_steps"] == 4 + assert got["guidance_scale"] == DISTILLED_GUIDANCE_SCALE + assert got["height"] == COSMOS3_T2I_PARAMS["height"] + + +class TestPipelineSchedulerLoading: + def test_distilled_checkpoint_loads_flow_match(self, tmp_path): + _write_scheduler_config(tmp_path, DISTILLED_SCHEDULER_CONFIG) + pipeline = _bare_pipeline() + + pipeline.load_standard_components( + str(tmp_path), torch.device("cpu"), skip_components=SKIP_NON_SCHEDULER + ) + + assert isinstance(pipeline.scheduler, FlowMatchEulerDiscreteScheduler) + assert pipeline.sampling.is_distilled + assert pipeline.sampling.fixed_sigmas == DISTILLED_SIGMAS + + def test_base_checkpoint_loads_unipc(self, tmp_path): + _write_scheduler_config(tmp_path, UNIPC_SCHEDULER_CONFIG) + pipeline = _bare_pipeline() + + pipeline.load_standard_components( + str(tmp_path), torch.device("cpu"), skip_components=SKIP_NON_SCHEDULER + ) + + assert isinstance(pipeline.scheduler, UniPCMultistepScheduler) + assert not pipeline.sampling.is_distilled + assert pipeline.sampling.checkpoint_flow_shift == 1.0 + + def test_audio_scheduler_is_separate_same_class_instance(self, tmp_path): + _write_scheduler_config(tmp_path, DISTILLED_SCHEDULER_CONFIG) + pipeline = _bare_pipeline(audio_gen=True) + + pipeline.load_standard_components( + str(tmp_path), torch.device("cpu"), skip_components=SKIP_NON_SCHEDULER + ) + + assert isinstance(pipeline.audio_scheduler, FlowMatchEulerDiscreteScheduler) + assert pipeline.audio_scheduler is not pipeline.scheduler + + +class TestWarmupAndForwardValidation: + def test_warmup_steps_follow_distilled_schedule(self): + assert _bare_pipeline(sampling=_distilled_policy()).default_warmup_steps == 4 + + def test_warmup_steps_base_default(self): + assert _bare_pipeline().default_warmup_steps == 2 # BasePipeline default + + @pytest.mark.parametrize( + "policy_factory, expected_guidance", + [(_distilled_policy, DISTILLED_GUIDANCE_SCALE), (_base_policy, 6.0)], + ) + def test_warmup_guidance_uses_pipeline_defaults(self, policy_factory, expected_guidance): + pipeline = _bare_pipeline(sampling=policy_factory()) + captured = {} + pipeline.forward = lambda **kwargs: captured.update(kwargs) + + pipeline._run_warmup(height=720, width=1280, num_frames=9, steps=4) + + assert captured["guidance_scale"] == expected_guidance + + @pytest.mark.parametrize( + "bad_kwargs", + [ + {"num_inference_steps": 10, "guidance_scale": 1.0}, + {"num_inference_steps": 4, "guidance_scale": 3.5}, + ], + ) + def test_forward_rejects_explicit_mismatch(self, bad_kwargs): + pipeline = _bare_pipeline(sampling=_distilled_policy()) + with pytest.raises(ValueError, match="distilled"): + pipeline.forward(prompt="x", seed=0, use_guardrails=False, **bad_kwargs) + + @pytest.mark.parametrize( + "policy_factory, sampling_kwargs", + [ + (_base_policy, {}), + ( + _distilled_policy, + {"num_inference_steps": 4, "guidance_scale": DISTILLED_GUIDANCE_SCALE}, + ), + ], + ) + def test_image_conditioning_passes_validation(self, policy_factory, sampling_kwargs): + """No sampling policy rejects image conditioning by checkpoint kind. + + Distilled checkpoints keep the conditioned frame clean by re-anchoring + it after every scheduler step (see TestDistilledConditioningAnchor) + rather than by refusing the request. forward() therefore gets past + validation for both policies and only fails later on the bare double. + """ + pipeline = _bare_pipeline(sampling=policy_factory()) + with pytest.raises(AttributeError): + pipeline.forward( + prompt="x", + seed=0, + use_guardrails=False, + image="frame.png", + **sampling_kwargs, + ) + + @pytest.mark.parametrize("bad_output_type", ["imgae", "png", "", "both"]) + def test_invalid_output_type_raises(self, bad_output_type): + pipeline = _bare_pipeline() + with pytest.raises(ValueError, match="output_type"): + pipeline.forward(prompt="x", seed=0, use_guardrails=False, output_type=bad_output_type) + + +class _AdditiveScheduler: + """step(v, t, x) = x + v, so final latents are the exact sum of all + predictions. The strict signature also pins that the loop passes no silent + extra step kwargs unless the caller supplies them.""" + + def __init__(self, timesteps): + self.timesteps = timesteps + + def step(self, model_output, timestep, sample, return_dict=False): + assert return_dict is False + return (sample + model_output,) + + +class _GeneratorRecordingScheduler(_AdditiveScheduler): + def __init__(self, timesteps): + super().__init__(timesteps) + self.generators = [] + + def step(self, model_output, timestep, sample, return_dict=False, generator=None): + self.generators.append(generator) + return super().step(model_output, timestep, sample, return_dict) + + +def _denoise_ready_pipeline() -> Cosmos3OmniMoTPipeline: + return _bare_pipeline( + pipeline_config=SimpleNamespace(visual_gen_mapping=None), + cache_accelerator=None, + _predenoise_pending=False, + _postdenoise_pending=False, + _is_warmup=False, + _profile_range=None, + ) + + +class TestDistilledDenoiseLoop: + POS_IDS = torch.arange(8).unsqueeze(0) + NEG_IDS = torch.arange(8).unsqueeze(0) + 100 + POS_MASK = torch.ones(1, 8, dtype=torch.long) + NEG_MASK = torch.zeros(1, 8, dtype=torch.long) + + def _run(self, scheduler=None, scheduler_step_kwargs=None): + pipeline = _denoise_ready_pipeline() + timesteps = torch.tensor([s * 1000.0 for s in DISTILLED_SIGMAS]) + scheduler = scheduler if scheduler is not None else _AdditiveScheduler(timesteps) + calls = [] + + def forward_fn(latent_input, extra_streams, step_index, timestep, embeds, extras): + calls.append( + { + "batch": latent_input.shape[0], + "timestep": float(timestep[0]), + "text_ids": extras["text_ids"], + } + ) + return torch.full_like(latent_input, 0.5) + + latents = torch.zeros(1, 4, 3, 2, 2) + result = pipeline.denoise( + latents=latents, + scheduler=scheduler, + prompt_embeds=self.POS_IDS, + neg_prompt_embeds=self.NEG_IDS, + guidance_scale=DISTILLED_GUIDANCE_SCALE, + forward_fn=forward_fn, + extra_cfg_tensors={ + "text_ids": (self.POS_IDS, self.NEG_IDS), + "text_mask": (self.POS_MASK, self.NEG_MASK), + }, + scheduler_step_kwargs=scheduler_step_kwargs, + ) + return result, calls + + def test_guidance_one_single_forward_per_step(self): + result, calls = self._run() + + assert len(calls) == 4, "one forward per distilled step, no CFG branch" + assert all(c["batch"] == 1 for c in calls), "no CFG batch duplication" + assert all(c["text_ids"] is self.POS_IDS for c in calls), "positive prompt only" + assert [c["timestep"] for c in calls] == pytest.approx( + [s * 1000.0 for s in DISTILLED_SIGMAS], abs=1e-3 + ) + assert torch.all(result == 2.0) # 4 additive steps of +0.5 from 0 + + def test_scheduler_step_kwargs_reach_every_step(self): + generator = torch.Generator().manual_seed(7) + timesteps = torch.tensor([s * 1000.0 for s in DISTILLED_SIGMAS]) + scheduler = _GeneratorRecordingScheduler(timesteps) + + result, calls = self._run( + scheduler=scheduler, + scheduler_step_kwargs=_distilled_policy().scheduler_step_kwargs(generator), + ) + + assert len(calls) == 4 + assert scheduler.generators == [generator] * 4 + assert torch.all(result == 2.0) + + +class _PerturbingScheduler(_AdditiveScheduler): + """step(v, t, x) = x + v + 1.0 — every position moves every step even + where the velocity is zero, emulating the distilled SDE step's + re-noising. A conditioned frame stays clean only if something re-anchors + it after each step.""" + + def step(self, model_output, timestep, sample, return_dict=False): + assert return_dict is False + return (sample + model_output + 1.0,) + + +class TestDistilledConditioningAnchor: + """Per-step re-anchoring of image-conditioned frames under SDE sampling.""" + + CLEAN = 7.0 # conditioned-frame latent value; drift is detected against it + + def _clean_frame(self): + return torch.full((1, 4, 1, 2, 2), self.CLEAN) + + def test_anchor_gating(self): + image_latent = self._clean_frame() + distilled = _bare_pipeline(sampling=_distilled_policy()) + assert callable(distilled._conditioning_anchor_post_step(image_latent)) + assert distilled._conditioning_anchor_post_step(None) is None + assert ( + _bare_pipeline(sampling=_base_policy())._conditioning_anchor_post_step(image_latent) + is None + ) + assert _bare_pipeline()._conditioning_anchor_post_step(image_latent) is None + + def test_anchor_writes_only_frame_zero_in_place(self): + pipeline = _bare_pipeline(sampling=_distilled_policy()) + post_step_fn = pipeline._conditioning_anchor_post_step(self._clean_frame()) + + latents = torch.arange(48, dtype=torch.float32).reshape(1, 4, 3, 2, 2) + untouched = latents[:, :, 1:].clone() + returned = post_step_fn(latents) + + assert returned is latents, "must write in place, not copy" + assert torch.all(latents[:, :, 0:1] == self.CLEAN) + assert torch.equal(latents[:, :, 1:], untouched) + + def _run_denoise(self, with_anchor: bool): + """Run the real BasePipeline.denoise loop with a perturbing scheduler, + recording what the transformer receives at every step.""" + pipeline = _denoise_ready_pipeline() + pipeline.sampling = _distilled_policy() + timesteps = torch.tensor([s * 1000.0 for s in DISTILLED_SIGMAS]) + scheduler = _PerturbingScheduler(timesteps) + + seen = [] + + def forward_fn(latent_input, extra_streams, step_index, timestep, embeds, extras): + seen.append(latent_input.clone()) + return torch.full_like(latent_input, 0.5) + + latents = torch.zeros(1, 4, 3, 2, 2) + latents[:, :, 0:1] = self.CLEAN # frame 0 pinned clean, rest noise-like + image_latent = self._clean_frame() + + post_step_fn = ( + pipeline._conditioning_anchor_post_step(image_latent) if with_anchor else None + ) + result = pipeline.denoise( + latents=latents, + scheduler=scheduler, + prompt_embeds=torch.arange(8).unsqueeze(0), + neg_prompt_embeds=torch.arange(8).unsqueeze(0) + 100, + guidance_scale=DISTILLED_GUIDANCE_SCALE, + forward_fn=forward_fn, + extra_cfg_tensors={}, + post_step_fn=post_step_fn, + ) + return result, seen + + def test_every_forward_sees_clean_conditioned_frame(self): + result, seen = self._run_denoise(with_anchor=True) + + assert len(seen) == 4 + for step, latent_input in enumerate(seen): + assert torch.all(latent_input[:, :, 0:1] == self.CLEAN), ( + f"transformer input at step {step} lost the clean conditioning frame" + ) + # The perturbing step really moved everything else: unconditioned + # frames accumulate (velocity 0.5 + drift 1.0) per completed step. + for step, latent_input in enumerate(seen): + assert torch.all(latent_input[:, :, 1:] == step * 1.5) + assert torch.all(result[:, :, 0:1] == self.CLEAN) + assert torch.all(result[:, :, 1:] == 4 * 1.5) + + def test_without_anchor_the_conditioned_frame_drifts(self): + """Control: the same loop without the anchor corrupts frame 0 from the + second forward on — the exact failure mode the anchor exists for.""" + _, seen = self._run_denoise(with_anchor=False) + + assert torch.all(seen[0][:, :, 0:1] == self.CLEAN) + for step, latent_input in enumerate(seen[1:], start=1): + assert torch.all(latent_input[:, :, 0:1] == self.CLEAN + step * 1.5) + + +def _forward_ready_pipeline(**attrs) -> Cosmos3OmniMoTPipeline: + """A pipeline stubbed just enough for forward() to run end to end.""" + defaults = dict( + sampling=_distilled_policy(), + pipeline_config=SimpleNamespace(torch_dtype=torch.float32, visual_gen_mapping=None), + transformer=SimpleNamespace( + latent_channel_size=4, + reset_cache=lambda: None, + device=torch.device("cpu"), + ), + vae_scale_factor_temporal=4, + vae_scale_factor_spatial=16, + scheduler=SimpleNamespace( + set_timesteps=lambda *args, **kwargs: None, + config=SimpleNamespace(num_train_timesteps=1000), + ), + ) + defaults.update(attrs) + pipeline = _bare_pipeline(**defaults) + pipeline._tokenize_prompt = lambda *args, **kwargs: ( + torch.ones(1, 4, dtype=torch.long), + torch.ones(1, 4, dtype=torch.long), + ) + return pipeline + + +class TestForwardConditioningWiring: + """forward() must hand the denoise loop the anchor exactly when the + checkpoint is distilled and the request carries image conditioning.""" + + T_LAT, H_LAT, W_LAT = 2, 2, 2 # from num_frames=5, 32x32, scale 4/16 + CLEAN = 7.0 + + def _wiring_pipeline(self): + pipeline = _forward_ready_pipeline() + pipeline._encode_conditioning_video = lambda *args, **kwargs: torch.full( + (1, 4, self.T_LAT, self.H_LAT, self.W_LAT), self.CLEAN + ) + pipeline.decode_latents = lambda latents, decode_fn: torch.zeros(1, 5, 32, 32, 3) + + captured = {} + + def denoise(**kwargs): + captured.update(kwargs) + return kwargs["latents"] + + pipeline.denoise = denoise + return pipeline, captured + + def _forward(self, pipeline, image): + return pipeline.forward( + prompt="x", + seed=0, + image=image, + height=32, + width=32, + num_frames=5, + num_inference_steps=4, + guidance_scale=DISTILLED_GUIDANCE_SCALE, + use_guardrails=False, + enable_audio=False, + ) + + def test_i2v_request_wires_anchor_and_seeded_steps(self): + pipeline, captured = self._wiring_pipeline() + self._forward(pipeline, image=torch.zeros(3, 32, 32)) + + post_step_fn = captured["post_step_fn"] + assert post_step_fn is not None + latents = torch.zeros(1, 4, self.T_LAT, self.H_LAT, self.W_LAT) + post_step_fn(latents) + assert torch.all(latents[:, :, 0:1] == self.CLEAN) + assert torch.all(latents[:, :, 1:] == 0.0) + + assert isinstance(captured["scheduler_step_kwargs"]["generator"], torch.Generator) + # Initial latents enter the loop with the clean frame already pinned. + assert torch.all(captured["latents"][:, :, 0:1] == self.CLEAN) + + def test_t2v_request_wires_no_anchor(self): + pipeline, captured = self._wiring_pipeline() + self._forward(pipeline, image=None) + assert captured["post_step_fn"] is None + + +class TestSystemPromptDefault: + """use_system_prompt defaults are checkpoint-declared via model_index.json.""" + + def _write_model_index(self, checkpoint_dir: Path, content: dict) -> None: + with open(checkpoint_dir / "model_index.json", "w") as f: + json.dump(content, f) + + def _loaded_pipeline(self, tmp_path) -> Cosmos3OmniMoTPipeline: + _write_scheduler_config(tmp_path, DISTILLED_SCHEDULER_CONFIG) + pipeline = _bare_pipeline() + pipeline.load_standard_components( + str(tmp_path), torch.device("cpu"), skip_components=SKIP_NON_SCHEDULER + ) + return pipeline + + def test_checkpoint_declared_true(self, tmp_path): + self._write_model_index(tmp_path, {"default_use_system_prompt": True}) + pipeline = self._loaded_pipeline(tmp_path) + + assert pipeline.default_use_system_prompt is True + assert pipeline.extra_param_specs["use_system_prompt"].default is True + # The shared spec table must stay untouched (model_copy, not mutation). + from tensorrt_llm._torch.visual_gen.models.cosmos3.defaults import COSMOS3_EXTRA_SPECS + + assert COSMOS3_EXTRA_SPECS["use_system_prompt"].default is False + + def test_missing_model_index_keeps_false(self, tmp_path): + pipeline = self._loaded_pipeline(tmp_path) + assert pipeline.default_use_system_prompt is False + assert pipeline.extra_param_specs["use_system_prompt"].default is False + + def test_model_index_without_field_keeps_false(self, tmp_path): + self._write_model_index(tmp_path, {"_class_name": "Cosmos3OmniPipeline"}) + pipeline = self._loaded_pipeline(tmp_path) + assert pipeline.default_use_system_prompt is False + + def _captured_use_system_prompt(self, pipeline, extra_params): + captured = {} + pipeline.forward = lambda **kwargs: captured.update(kwargs) + pipeline.infer(_fake_request("video", extra_params=extra_params)) + return captured["use_system_prompt"] + + def test_infer_passes_unset_key_through_as_none(self): + """forward() owns the resolution, so infer() must forward "unset" + rather than pre-resolving it — otherwise the two entry points can + drift apart again.""" + pipeline = _bare_pipeline(default_use_system_prompt=True) + assert self._captured_use_system_prompt(pipeline, {"output_type": "video"}) is None + + def test_infer_passes_explicit_false_through(self): + pipeline = _bare_pipeline(default_use_system_prompt=True) + got = self._captured_use_system_prompt( + pipeline, {"output_type": "video", "use_system_prompt": False} + ) + assert got is False + + def _tokenized_use_system_prompt(self, pipeline, **forward_kwargs): + """Run forward() far enough to observe what it hands the tokenizer.""" + seen = {} + + class _Stop(Exception): + pass + + def fake_tokenize(prompt, max_sequence_length, use_system_prompt, system_prompt=None): + seen["value"] = use_system_prompt + raise _Stop + + pipeline._tokenize_prompt = fake_tokenize + with pytest.raises(_Stop): + pipeline.forward( + prompt="x", + seed=0, + use_guardrails=False, + num_inference_steps=4, + guidance_scale=DISTILLED_GUIDANCE_SCALE, + **forward_kwargs, + ) + return seen["value"] + + def test_forward_unset_resolves_to_checkpoint_default(self): + """Direct forward() callers (warmup included) must build the same + prompt as served requests.""" + pipeline = _forward_ready_pipeline(default_use_system_prompt=True) + assert self._tokenized_use_system_prompt(pipeline) is True + + def test_forward_explicit_false_overrides_checkpoint_default(self): + pipeline = _forward_ready_pipeline(default_use_system_prompt=True) + assert self._tokenized_use_system_prompt(pipeline, use_system_prompt=False) is False + + def test_forward_default_false_checkpoint_unchanged(self): + pipeline = _forward_ready_pipeline(default_use_system_prompt=False) + assert self._tokenized_use_system_prompt(pipeline) is False + + def test_warmup_leaves_system_prompt_unset(self): + """_run_warmup must not pin the historical False: leaving it unset is + what lets forward() resolve the checkpoint default.""" + pipeline = _bare_pipeline(sampling=_distilled_policy(), default_use_system_prompt=True) + captured = {} + pipeline.forward = lambda **kwargs: captured.update(kwargs) + + pipeline._run_warmup(height=720, width=1280, num_frames=9, steps=4) + + assert captured.get("use_system_prompt") is None + + +class TestAudioWeightPresenceGuard: + """enable_audio=True must fail loudly when the checkpoint ships no audio + tower — a weight-presence guard, not a workflow restriction.""" + + def _pipeline(self, **attrs): + return _bare_pipeline(sampling=_distilled_policy(), scheduler=None, **attrs) + + def test_explicit_audio_on_audioless_checkpoint_raises(self): + with pytest.raises(ValueError, match="audio tower"): + self._pipeline().forward( + prompt="x", + seed=0, + use_guardrails=False, + enable_audio=True, + num_inference_steps=4, + guidance_scale=DISTILLED_GUIDANCE_SCALE, + ) + + def test_t2i_disables_audio_before_the_guard(self): + """T2I force-disables audio for every checkpoint (existing semantics); + the guard must not fire for it. The batch error proves forward got + past the guard.""" + with pytest.raises(ValueError, match="Batch generation"): + self._pipeline().forward( + prompt=["a", "b"], + seed=0, + use_guardrails=False, + enable_audio=True, + output_type="image", + num_inference_steps=4, + guidance_scale=DISTILLED_GUIDANCE_SCALE, + ) + + def test_audio_capable_checkpoint_passes_the_guard(self): + with pytest.raises(ValueError, match="Batch generation"): + self._pipeline(audio_gen=True).forward( + prompt=["a", "b"], + seed=0, + use_guardrails=False, + enable_audio=True, + num_inference_steps=4, + guidance_scale=DISTILLED_GUIDANCE_SCALE, + ) + + +class TestRegistryDispatch: + def test_model_index_class_name_dispatches(self, tmp_path): + with open(tmp_path / "model_index.json", "w") as f: + json.dump({"_class_name": "Cosmos3OmniPipeline"}, f) + assert AutoPipeline._detect_from_checkpoint(str(tmp_path)) == "Cosmos3OmniMoTPipeline" + + def test_hf_id_registered(self): + entry = PIPELINE_REGISTRY["Cosmos3OmniMoTPipeline"] + assert "nvidia/Cosmos3-Super-Text2Image-4Step" in entry.hf_ids + assert "nvidia/Cosmos3-Super-Image2Video-4Step" in entry.hf_ids diff --git a/tests/unittest/_torch/visual_gen/test_cosmos3_pipeline.py b/tests/unittest/_torch/visual_gen/test_cosmos3_pipeline.py index c0329b71a060..d1993549615b 100644 --- a/tests/unittest/_torch/visual_gen/test_cosmos3_pipeline.py +++ b/tests/unittest/_torch/visual_gen/test_cosmos3_pipeline.py @@ -30,7 +30,6 @@ from pathlib import Path os.environ["TLLM_DISABLE_MPI"] = "1" -os.environ["TRTLLM_DISABLE_COSMOS3_GUARDRAILS"] = "1" import PIL.Image import pytest @@ -46,7 +45,7 @@ from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader from tensorrt_llm.visual_gen.args import TorchCompileConfig, VisualGenArgs -pytestmark = pytest.mark.cosmos3 +pytestmark = [pytest.mark.cosmos3, pytest.mark.usefixtures("disable_cosmos3_guardrails")] @pytest.fixture(autouse=True, scope="module") diff --git a/tests/unittest/_torch/visual_gen/test_cosmos3_transformer.py b/tests/unittest/_torch/visual_gen/test_cosmos3_transformer.py index 76e6c1a01a8c..9293c371f459 100644 --- a/tests/unittest/_torch/visual_gen/test_cosmos3_transformer.py +++ b/tests/unittest/_torch/visual_gen/test_cosmos3_transformer.py @@ -20,20 +20,24 @@ import gc import os from pathlib import Path +from types import SimpleNamespace os.environ["TLLM_DISABLE_MPI"] = "1" -os.environ["TRTLLM_DISABLE_COSMOS3_GUARDRAILS"] = "1" import pytest import torch from tensorrt_llm._torch.modules.linear import Linear from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig, DiffusionPipelineConfig -from tensorrt_llm._torch.visual_gen.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer +from tensorrt_llm._torch.visual_gen.models.cosmos3.transformer_cosmos3 import ( + PRETRAINED_CONFIG_COMPAT_DEFAULTS, + Cosmos3VFMTransformer, + apply_pretrained_config_compat_defaults, +) from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineComponent, PipelineLoader from tensorrt_llm.visual_gen.args import TorchCompileConfig, VisualGenArgs -pytestmark = pytest.mark.cosmos3 +pytestmark = [pytest.mark.cosmos3, pytest.mark.usefixtures("disable_cosmos3_guardrails")] @pytest.fixture(autouse=True, scope="module") @@ -468,3 +472,93 @@ def test_load_fp8_quantization(self, quant_algo: str): del pipeline gc.collect() torch.cuda.empty_cache() + + +# --- CPU-only coverage: checkpoint config schema compatibility --- + + +class TestConfigCompatDefaults: + """Newer diffusers conversions omit fields older ones carried explicitly.""" + + def test_new_schema_gets_defaults(self): + config = SimpleNamespace(hidden_size=64, rope_axes_dim=[4, 2, 2]) + apply_pretrained_config_compat_defaults(config) + for key, value in PRETRAINED_CONFIG_COMPAT_DEFAULTS.items(): + assert getattr(config, key) == value + + def test_old_schema_untouched(self): + # Every field deliberately differs from its compat default, so an + # overwrite of any one of them fails its assertion. + config = SimpleNamespace( + position_embedding_type="rope_3d", + max_position_embeddings=12345, + temporal_compression_factor_sound=7, + ) + apply_pretrained_config_compat_defaults(config) + assert config.position_embedding_type == "rope_3d" + assert config.max_position_embeddings == 12345 + assert config.temporal_compression_factor_sound == 7 + + def test_idempotent(self): + config = SimpleNamespace(hidden_size=64) + apply_pretrained_config_compat_defaults(config) + snapshot = vars(config).copy() + apply_pretrained_config_compat_defaults(config) + assert vars(config) == snapshot + + +class TestI2V4StepConfigShape: + """The Image2Video-4Step conversion drops the audio/action towers + (``sound_dim: null``, no ``action_*`` keys) and carries newer schema + fields (``qk_norm_for_text``, ``hidden_act``, nested ``rope_theta``). + The transformer must construct from that exact key set. CPU-only with + shrunk dimensions; the real 64B shape is covered by the checkpoint + integration test.""" + + def _reduced_i2v_config(self) -> SimpleNamespace: + # Key set mirrors the checkpoint's transformer/config.json verbatim; + # only the sizes are reduced (head_dim 8 -> mrope_section sums to 4). + return SimpleNamespace( + attention_bias=False, + attention_dropout=0.0, + base_fps=16, + enable_fps_modulation=True, + head_dim=8, + hidden_act="silu", + hidden_size=32, + intermediate_size=64, + latent_channel=4, + latent_patch_size=2, + num_attention_heads=4, + num_hidden_layers=2, + num_key_value_heads=2, + patch_latent_dim=16, + qk_norm_for_text=True, + rms_norm_eps=1e-6, + rope_axes_dim=[2, 1, 1], + rope_scaling={ + "mrope_interleaved": True, + "mrope_section": [2, 1, 1], + "rope_theta": 5000000, + "rope_type": "default", + }, + rope_theta=5000000, + sound_dim=None, + sound_gen=False, + sound_latent_fps=25, + timestep_scale=0.001, + unified_3d_mrope_reset_spatial_ids=True, + unified_3d_mrope_temporal_modality_margin=15000, + vocab_size=64, + ) + + def test_constructs_without_audio_or_action_towers(self): + model_config = DiffusionModelConfig(pretrained_config=self._reduced_i2v_config()) + model = Cosmos3VFMTransformer(model_config) + + assert model.audio_gen is False + assert model.action_gen is False + assert not hasattr(model, "audio2llm") + assert not hasattr(model, "audio_modality_embed") + assert model.base_fps == 16 + assert len(model.gen_layers) == 2 diff --git a/tests/unittest/_torch/visual_gen/test_flux2_image_conditioning.py b/tests/unittest/_torch/visual_gen/test_flux2_image_conditioning.py new file mode 100644 index 000000000000..ac4a5ee84b14 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_flux2_image_conditioning.py @@ -0,0 +1,302 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for FLUX.2 reference-image conditioning.""" + +import io +from collections.abc import Iterator +from types import SimpleNamespace +from unittest.mock import MagicMock + +import PIL.Image +import pytest +import torch +from diffusers.pipelines.flux2.image_processor import Flux2ImageProcessor + +from tensorrt_llm._torch.visual_gen.models.flux.pipeline_flux2 import Flux2Pipeline + + +def _png_bytes() -> bytes: + buffer = io.BytesIO() + PIL.Image.new("RGB", (64, 64), color=(10, 20, 30)).save(buffer, format="PNG") + return buffer.getvalue() + + +def test_load_reference_images_accepts_pil_path_and_bytes(tmp_path) -> None: + pil_image = PIL.Image.new("L", (64, 64), color=128) + image_path = tmp_path / "reference.png" + pil_image.save(image_path) + + images = Flux2Pipeline._load_reference_images([pil_image, str(image_path), _png_bytes()]) + + assert len(images) == 3 + assert all(image.mode == "RGB" for image in images) + assert [image.size for image in images] == [(64, 64), (64, 64), (64, 64)] + + +@pytest.mark.parametrize("image", [[], [object()]]) +def test_load_reference_images_rejects_invalid_inputs(image: list[object]) -> None: + with pytest.raises(ValueError): + Flux2Pipeline._load_reference_images(image) + + +def test_preprocess_reference_images_caps_area_and_aligns_to_16() -> None: + pipeline = Flux2Pipeline.__new__(Flux2Pipeline) + pipeline.vae_scale_factor = 8 + pipeline.image_processor = Flux2ImageProcessor(vae_scale_factor=16) + images = [PIL.Image.new("RGB", (81, 65)), PIL.Image.new("RGB", (2048, 1024))] + + processed = pipeline._preprocess_reference_images(images) + + assert processed[0].shape == (1, 3, 64, 80) + for tensor in processed: + height, width = tensor.shape[-2:] + assert height % 16 == 0 + assert width % 16 == 0 + assert height * width <= 1024 * 1024 + + +def test_target_dimensions_default_to_first_processed_reference() -> None: + condition_images = [torch.zeros(1, 3, 64, 80), torch.zeros(1, 3, 96, 112)] + + assert Flux2Pipeline._resolve_target_dimensions(None, None, condition_images) == (64, 80) + assert Flux2Pipeline._resolve_target_dimensions(128, None, condition_images) == (128, 80) + assert Flux2Pipeline._resolve_target_dimensions(None, None, None) == (1024, 1024) + + +def test_prepare_request_resolves_dimensions_and_infer_reuses_images() -> None: + pipeline = Flux2Pipeline.__new__(Flux2Pipeline) + pipeline.vae_scale_factor = 8 + pipeline.image_processor = Flux2ImageProcessor(vae_scale_factor=16) + req = SimpleNamespace( + prompt=["edit this image"], + params=SimpleNamespace( + image=_png_bytes(), + height=None, + width=None, + num_inference_steps=1, + guidance_scale=3.5, + seed=0, + max_sequence_length=512, + num_images_per_prompt=1, + num_frames=None, + ), + prepared_inputs={}, + ) + + pipeline.prepare_request(req) + + condition_images = req.prepared_inputs["condition_images"] + assert req.params.height == 64 + assert req.params.width == 64 + assert len(condition_images) == 1 + assert condition_images[0].shape == (1, 3, 64, 64) + assert pipeline.request_warmup_cache_key(req) == (64, 64, 1, ((64, 64),)) + + pipeline.forward = MagicMock(return_value=object()) + pipeline.infer(req) + + assert pipeline.forward.call_args.kwargs["_condition_images"] is condition_images + + +def test_text_only_request_preserves_existing_warmup_cache_key() -> None: + pipeline = Flux2Pipeline.__new__(Flux2Pipeline) + req = SimpleNamespace( + params=SimpleNamespace(height=1024, width=1024, num_frames=None), + prepared_inputs={}, + ) + + assert pipeline.request_warmup_cache_key(req) == (1024, 1024) + + +def test_reference_warmup_cache_key_preserves_count_and_ordered_shapes() -> None: + pipeline = Flux2Pipeline.__new__(Flux2Pipeline) + req = SimpleNamespace( + params=SimpleNamespace(height=128, width=256, num_frames=None), + prepared_inputs={ + "condition_images": [ + torch.zeros(1, 3, 32, 48), + torch.zeros(1, 3, 64, 80), + ] + }, + ) + + assert pipeline.request_warmup_cache_key(req) == ( + 128, + 256, + 2, + ((32, 48), (64, 80)), + ) + + +@pytest.mark.parametrize("cache_backend", ["teacache", "cache_dit"]) +@pytest.mark.parametrize("reference_count", [1, 3]) +def test_reference_images_run_with_cache_acceleration( + cache_backend: str, + reference_count: int, +) -> None: + pipeline = Flux2Pipeline.__new__(Flux2Pipeline) + pipeline.pipeline_config = SimpleNamespace(cache_backend=cache_backend) + pipeline._encode_prompt = lambda _prompt, _max_length: ( + torch.zeros(1, 2, 8), + torch.zeros(2, 4), + ) + pipeline._preprocess_reference_images = lambda images: [ + torch.zeros(1, 3, 16, 16) for _ in images + ] + pipeline._prepare_latents = lambda _batch_size, _height, _width, _generator: ( + torch.zeros(1, 4, 8), + torch.zeros(4, 4), + ) + pipeline._prepare_image_latents = lambda images, batch_size: ( + torch.zeros(batch_size, 4 * len(images), 8), + torch.zeros(4 * len(images), 4), + ) + + class Transformer: + guidance_embeds = False + sharder = SimpleNamespace(is_active=False) + + def __init__(self) -> None: + self.sequence_lengths: list[int] = [] + + def parameters(self) -> Iterator[torch.Tensor]: + return iter([torch.empty(0)]) + + def __call__(self, hidden_states: torch.Tensor, **_kwargs) -> tuple[torch.Tensor]: + self.sequence_lengths.append(hidden_states.shape[1]) + return (hidden_states + 1,) + + class Scheduler: + config = SimpleNamespace(use_flow_sigmas=False) + + def set_timesteps(self, *_args, **_kwargs) -> None: + self.timesteps = torch.tensor([1000.0]) + + def set_begin_index(self, _index: int) -> None: + return None + + transformer = Transformer() + pipeline.transformer = transformer + pipeline.scheduler = Scheduler() + graph_runner = SimpleNamespace(enabled=True) + pipeline._cuda_graph_runners = {"transformer": graph_runner} + + denoised_latents: list[torch.Tensor] = [] + graph_states_during_denoise: list[bool] = [] + + def denoise(**kwargs) -> torch.Tensor: + graph_states_during_denoise.append(graph_runner.enabled) + result = kwargs["forward_fn"]( + kwargs["latents"], + {}, + 0, + kwargs["timesteps"][0], + kwargs["prompt_embeds"], + {}, + ) + denoised_latents.append(result) + return result + + pipeline.denoise = denoise + pipeline.decode_latents = lambda _latents, _decode_fn: torch.zeros(1, 16, 16, 3) + + references = [_png_bytes() for _ in range(reference_count)] + result = pipeline.forward( + prompt="edit this image", + seed=0, + image=references[0] if reference_count == 1 else references, + num_inference_steps=1, + ) + + assert transformer.sequence_lengths == [4 + 4 * reference_count] + assert graph_states_during_denoise == [False] + assert graph_runner.enabled is True + assert denoised_latents[0].shape == (1, 4, 8) + assert result.image.shape == (1, 16, 16, 3) + + +def test_cuda_graph_bypass_preserves_text_only_state_and_restores_after_failure() -> None: + pipeline = Flux2Pipeline.__new__(Flux2Pipeline) + graph_runner = SimpleNamespace(enabled=True) + pipeline._cuda_graph_runners = {"transformer": graph_runner} + + with pipeline._temporarily_disable_cuda_graphs(disable=False): + assert graph_runner.enabled is True + + with pytest.raises(RuntimeError, match="denoise failed"): + with pipeline._temporarily_disable_cuda_graphs(disable=True): + assert graph_runner.enabled is False + raise RuntimeError("denoise failed") + + assert graph_runner.enabled is True + + +def test_prepare_image_ids_assigns_distinct_time_offsets() -> None: + first = torch.zeros(1, 128, 2, 3) + second = torch.zeros(1, 128, 1, 2) + + image_ids = Flux2Pipeline._prepare_image_ids([first, second]) + + assert image_ids.shape == (8, 4) + torch.testing.assert_close(image_ids[:6, 0], torch.full((6,), 10.0)) + torch.testing.assert_close(image_ids[6:, 0], torch.full((2,), 20.0)) + assert torch.count_nonzero(image_ids[:, 3]) == 0 + + +def test_reference_sequence_length_rejects_nondivisible_parallel_shape() -> None: + sharder = SimpleNamespace(is_active=True, size=4) + + with pytest.raises(ValueError, match="not divisible by the configured sequence-parallel"): + Flux2Pipeline._validate_reference_sequence_length( + target_seq_len=16, + reference_seq_len=6, + sharder=sharder, + ) + + +def test_reference_sequence_length_accepts_divisible_parallel_shape() -> None: + sharder = SimpleNamespace(is_active=True, size=4) + + Flux2Pipeline._validate_reference_sequence_length( + target_seq_len=16, + reference_seq_len=8, + sharder=sharder, + ) + + +def test_patchify_and_pack_reference_latents() -> None: + latents = torch.arange(1 * 2 * 4 * 6).reshape(1, 2, 4, 6) + pipeline = Flux2Pipeline.__new__(Flux2Pipeline) + + patchified = Flux2Pipeline._patchify_latents(latents) + packed = pipeline._pack_latents(patchified) + + assert patchified.shape == (1, 8, 2, 3) + assert packed.shape == (1, 6, 8) + torch.testing.assert_close(packed[0, 0], latents[0, :, :2, :2].reshape(-1)) + + +def test_encode_vae_image_uses_mode_patchify_and_batch_norm() -> None: + source_latents = torch.arange(1 * 2 * 4 * 6, dtype=torch.float32).reshape(1, 2, 4, 6) + + class LatentDistribution: + mode_called = False + + def mode(self) -> torch.Tensor: + self.mode_called = True + return source_latents + + latent_distribution = LatentDistribution() + vae = SimpleNamespace( + encode=lambda _image: SimpleNamespace(latent_dist=latent_distribution), + bn=SimpleNamespace(running_mean=torch.zeros(8), running_var=torch.ones(8)), + config=SimpleNamespace(batch_norm_eps=0.0), + ) + pipeline = Flux2Pipeline.__new__(Flux2Pipeline) + pipeline.vae = vae + + encoded = pipeline._encode_vae_image(torch.zeros(1, 3, 32, 48)) + + assert latent_distribution.mode_called + torch.testing.assert_close(encoded, Flux2Pipeline._patchify_latents(source_latents)) diff --git a/tests/unittest/_torch/visual_gen/test_flux_infer.py b/tests/unittest/_torch/visual_gen/test_flux_infer.py index 7b28ae2b42e0..87dd0f6a3d99 100644 --- a/tests/unittest/_torch/visual_gen/test_flux_infer.py +++ b/tests/unittest/_torch/visual_gen/test_flux_infer.py @@ -19,6 +19,7 @@ def test_infer_forwards_num_images_per_prompt( pipeline.forward = Mock(return_value="image") request = SimpleNamespace( prompt="a cat", + prepared_inputs={}, params=SimpleNamespace( height=256, width=256, @@ -27,6 +28,7 @@ def test_infer_forwards_num_images_per_prompt( seed=42, max_sequence_length=512, num_images_per_prompt=2, + image=[b"reference"], ), ) @@ -35,3 +37,7 @@ def test_infer_forwards_num_images_per_prompt( assert result == "image" pipeline.forward.assert_called_once() assert pipeline.forward.call_args.kwargs["num_images_per_prompt"] == 2 + if pipeline_cls is Flux2Pipeline: + assert pipeline.forward.call_args.kwargs["image"] == [b"reference"] + else: + assert "image" not in pipeline.forward.call_args.kwargs diff --git a/tests/unittest/_torch/visual_gen/test_flux_pipeline.py b/tests/unittest/_torch/visual_gen/test_flux_pipeline.py index 0455e25faff7..4e33fee0670f 100644 --- a/tests/unittest/_torch/visual_gen/test_flux_pipeline.py +++ b/tests/unittest/_torch/visual_gen/test_flux_pipeline.py @@ -18,6 +18,7 @@ from pathlib import Path import numpy as np +import PIL.Image import pytest import torch import torch.distributed as dist @@ -772,6 +773,76 @@ def test_flux2_e2e_vs_hf(self, flux2_checkpoint_exists): gc.collect() torch.cuda.empty_cache() + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_flux2_reference_image_e2e_vs_hf(self, flux2_checkpoint_exists): + """FLUX.2 reference-image generation matches the diffusers pipeline.""" + from diffusers import Flux2Pipeline as HFFlux2Pipeline + + reference_array = np.zeros((256, 256, 3), dtype=np.uint8) + reference_array[..., 0] = np.arange(256, dtype=np.uint8)[None, :] + reference_array[..., 1] = np.arange(256, dtype=np.uint8)[:, None] + reference_array[..., 2] = 127 + reference_image = PIL.Image.fromarray(reference_array) + cases = [ + { + "image": reference_image, + "prompt": "turn the reference into a detailed watercolor painting", + "height": 256, + "width": 256, + "num_images_per_prompt": 1, + }, + { + "image": [ + reference_image.resize((160, 128)), + PIL.Image.new("RGB", (112, 96), color=(30, 90, 180)), + PIL.Image.new("RGB", (80, 64), color=(180, 90, 30)), + ], + "prompt": [ + "combine the references into a watercolor scene", + "combine the references into a pencil illustration", + ], + "height": None, + "width": None, + "num_images_per_prompt": 2, + }, + ] + + hf_pipe = HFFlux2Pipeline.from_pretrained( + FLUX2_CHECKPOINT_PATH, torch_dtype=torch.bfloat16 + ).to("cuda") + hf_images = [] + for case in cases: + hf_result = hf_pipe( + **case, + num_inference_steps=4, + guidance_scale=4.0, + generator=torch.Generator("cuda").manual_seed(42), + ) + hf_images.append(np.stack([np.array(image) for image in hf_result.images])) + del hf_pipe + gc.collect() + torch.cuda.empty_cache() + + pipeline = PipelineLoader(VisualGenArgs(model=FLUX2_CHECKPOINT_PATH)).load() + for case, hf_image_batch in zip(cases, hf_images): + result = pipeline.forward( + **case, + num_inference_steps=4, + guidance_scale=4.0, + seed=42, + ) + native_image_batch = result.image.cpu().numpy() + + assert native_image_batch.shape == hf_image_batch.shape + for hf_image, native_image in zip(hf_image_batch, native_image_batch): + mse = ((hf_image.astype(float) - native_image.astype(float)) ** 2).mean() + psnr = 10 * np.log10(255**2 / mse) if mse > 0 else float("inf") + assert psnr > 20.0, f"PSNR too low: {psnr:.2f} dB (expected >20 dB)" + + del pipeline + gc.collect() + torch.cuda.empty_cache() + class TestFluxBatchGeneration: """Batch generation tests for FLUX pipelines. diff --git a/tests/unittest/_torch/visual_gen/test_flux_transformer.py b/tests/unittest/_torch/visual_gen/test_flux_transformer.py index ec0b10c01343..f77662e76cb5 100644 --- a/tests/unittest/_torch/visual_gen/test_flux_transformer.py +++ b/tests/unittest/_torch/visual_gen/test_flux_transformer.py @@ -12,14 +12,18 @@ import unittest from copy import deepcopy from types import SimpleNamespace +from unittest import mock import pytest import torch import torch.nn.functional as F from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig +from tensorrt_llm._torch.visual_gen.quantization.loader import DynamicLinearWeightLoader +from tensorrt_llm._utils import get_sm_version from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo from tensorrt_llm.visual_gen.args import AttentionConfig # FLUX.1 dev config (12B params) @@ -64,6 +68,320 @@ def reduce_flux_config(mem_for_full_model: int, config_dict: dict): config_dict["num_single_layers"] = min(num_single_layers, 4) +def _make_fake_flux2_parallel_attn(): + from tensorrt_llm._torch.visual_gen.models.flux.attention import Flux2ParallelSelfAttention + + attn = Flux2ParallelSelfAttention.__new__(Flux2ParallelSelfAttention) + gate_up_proj = SimpleNamespace( + _weights_created=True, + has_nvfp4=True, + has_bias=False, + out_features=256, + use_cute_dsl_blockscaling_mm=True, + input_scale=torch.ones(1), + pre_quant_scale=None, + force_dynamic_quantization=False, + ) + down_proj = SimpleNamespace( + _weights_created=True, + has_nvfp4=True, + in_features=128, + input_scale=torch.ones(1), + pre_quant_scale=None, + force_dynamic_quantization=False, + ) + attn.to_qkv_mlp_proj = SimpleNamespace( + tp_size=2, + qkv_proj=object(), + mlp_proj=gate_up_proj, + ) + attn.to_out = SimpleNamespace(mlp_proj=down_proj) + return attn, gate_up_proj, down_proj + + +def test_flux2_single_stream_fp4out_guard_requires_compatible_packed_width(monkeypatch): + from tensorrt_llm._torch.visual_gen.models.flux import attention as flux_attention + + monkeypatch.setattr(flux_attention.torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(flux_attention, "is_sm_100f", lambda: True) + attn, gate_up_proj, down_proj = _make_fake_flux2_parallel_attn() + + hidden_states = torch.empty(1, 128, 16) + assert attn._can_project_hidden_mlp_with_fp4out(hidden_states) + + down_proj.in_features = 64 + assert not attn._can_project_hidden_mlp_with_fp4out(hidden_states) + + gate_up_proj.out_features = 258 + down_proj.in_features = 129 + assert not attn._can_project_hidden_mlp_with_fp4out(hidden_states) + + +def test_flux2_single_stream_fp4out_guard_requires_static_nvfp4(monkeypatch): + from tensorrt_llm._torch.visual_gen.models.flux import attention as flux_attention + + monkeypatch.setattr(flux_attention.torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(flux_attention, "is_sm_100f", lambda: True) + attn, gate_up_proj, down_proj = _make_fake_flux2_parallel_attn() + hidden_states = torch.empty(1, 128, 16) + + gate_up_proj.has_nvfp4 = False + assert not attn._can_project_hidden_mlp_with_fp4out(hidden_states) + + gate_up_proj.has_nvfp4 = True + gate_up_proj.force_dynamic_quantization = True + assert not attn._can_project_hidden_mlp_with_fp4out(hidden_states) + + gate_up_proj.force_dynamic_quantization = False + gate_up_proj.pre_quant_scale = torch.ones(16) + assert not attn._can_project_hidden_mlp_with_fp4out(hidden_states) + + gate_up_proj.pre_quant_scale = None + gate_up_proj.input_scale = None + assert not attn._can_project_hidden_mlp_with_fp4out(hidden_states) + + gate_up_proj.input_scale = torch.ones(1) + down_proj.force_dynamic_quantization = True + assert not attn._can_project_hidden_mlp_with_fp4out(hidden_states) + + down_proj.force_dynamic_quantization = False + down_proj.input_scale = None + assert not attn._can_project_hidden_mlp_with_fp4out(hidden_states) + + +def test_flux2_single_stream_cute_dsl_guard_requires_interleaved_weights(monkeypatch): + from tensorrt_llm._torch.visual_gen.models.flux import attention as flux_attention + + monkeypatch.setattr(flux_attention.torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(flux_attention, "is_sm_100f", lambda: True) + attn, gate_up_proj, _ = _make_fake_flux2_parallel_attn() + + assert attn._can_project_hidden_mlp_with_cute_dsl() + + gate_up_proj.use_cute_dsl_blockscaling_mm = False + assert not attn._can_project_hidden_mlp_with_cute_dsl() + + +def test_flux2_single_stream_cute_dsl_layout_requires_tensor_parallelism(): + from tensorrt_llm._torch.visual_gen.models.flux.attention import Flux2ParallelSelfAttention + + assert not Flux2ParallelSelfAttention._is_cute_dsl_swiglu_layout_compatible( + tp_size=1, + gate_up_out_features=256, + down_in_features=128, + ) + assert Flux2ParallelSelfAttention._is_cute_dsl_swiglu_layout_compatible( + tp_size=2, + gate_up_out_features=256, + down_in_features=128, + ) + + +def test_flux2_joint_qkv_mlp_enables_cutedsl_for_mlp_only(): + from tensorrt_llm._torch.visual_gen.models.flux.joint_proj import FluxJointQKVMLPProj + + proj = FluxJointQKVMLPProj( + in_dim=256, + q_dim=256, + kv_dim=256, + mlp_dim=512, + quant_config=QuantConfig(), + skip_create_weights_in_init=True, + use_cute_dsl_blockscaling_mm=True, + mapping=Mapping(world_size=2, rank=0, tp_size=2), + override_qkv_sharding={ + "q": (0, 128), + "k": (0, 128), + "v": (0, 128), + }, + ) + + assert not proj.qkv_proj.use_cute_dsl_blockscaling_mm + assert proj.mlp_proj.use_cute_dsl_blockscaling_mm + + +def test_flux2_moe_swiglu_reorders_gate_up(monkeypatch): + from tensorrt_llm._torch.visual_gen.models.flux.attention import Flux2ParallelSelfAttention + + attn = Flux2ParallelSelfAttention.__new__(Flux2ParallelSelfAttention) + captured = {} + + def fake_quantize(x, *args): + captured["input"] = x + return torch.empty(x.shape[0], x.shape[1] // 4), torch.empty(1) + + class FakeProj: + input_scale = torch.ones(1) + + def __call__(self, x): + return torch.empty(1, 2, 4) + + attn.to_out = SimpleNamespace(mlp_proj=FakeProj()) + attn._combine_split_projection = lambda attn_out, mlp_out: mlp_out + monkeypatch.setattr(torch.ops.trtllm, "moe_swiglu_nvfp4_quantize", fake_quantize) + + gate = torch.full((1, 2, 4), 1.0) + up = torch.full((1, 2, 4), 2.0) + attn._project_split_output_with_fp4_mlp(torch.empty(1), torch.cat((gate, up), dim=-1)) + + expected = torch.cat((up, gate), dim=-1).reshape(2, 8) + torch.testing.assert_close(captured["input"][:2], expected) + + +def _make_flux2_nvfp4_swiglu_test_modules(hidden_states): + from tensorrt_llm._torch.modules.linear import ( + Linear, + TensorParallelMode, + WeightMode, + WeightsLoadingConfig, + ) + from tensorrt_llm._torch.visual_gen.models.flux.attention import Flux2ParallelSelfAttention + + mapping = Mapping(world_size=2, rank=0, tp_size=2) + quant_config = QuantConfig(quant_algo=QuantAlgo.NVFP4) + config = DiffusionModelConfig( + mapping=mapping, + quant_config=quant_config, + dynamic_weight_quant=True, + attention=AttentionConfig(backend="VANILLA"), + ) + attention = Flux2ParallelSelfAttention( + hidden_size=256, + num_attention_heads=2, + head_dim=128, + mlp_ratio=2.0, + bias=False, + config=config, + ).cuda() + gate_up_proj = attention.to_qkv_mlp_proj.mlp_proj + down_proj = attention.to_out.mlp_proj + + reference_gate_up_proj = Linear( + 256, + 1024, + bias=False, + dtype=torch.bfloat16, + quant_config=quant_config, + mapping=mapping, + tensor_parallel_mode=TensorParallelMode.COLUMN, + reduce_output=False, + weights_loading_config=WeightsLoadingConfig(weight_mode=WeightMode.FUSED_GATE_UP_LINEAR), + fused_weight_shard_indices_mapping={ + "gate": (0, 256), + "up": (256, 256), + }, + override_tp_sharding={ + "gate": (0, 256), + "up": (0, 256), + }, + ).cuda() + + torch.manual_seed(7) + gate_weight = torch.randn(512, 256, device="cuda", dtype=torch.bfloat16) * 0.1 + up_weight = torch.randn(512, 256, device="cuda", dtype=torch.bfloat16) * 0.1 + down_weight = torch.randn(256, 512, device="cuda", dtype=torch.bfloat16) * 0.1 + + input_scale = hidden_states.abs().max().float() / (448 * 6) + local_gate = F.linear(hidden_states, gate_weight[:256]) + local_up = F.linear(hidden_states, up_weight[:256]) + down_input = F.silu(local_gate.float()) * local_up.float() + down_input_scale = down_input.abs().max() / (448 * 6) + + loader = DynamicLinearWeightLoader(config) + gate_up_weights = [ + {"weight": gate_weight, "input_scale": input_scale}, + {"weight": up_weight, "input_scale": input_scale}, + ] + loader.load_linear_weights(gate_up_proj, "gate_up_proj", gate_up_weights) + loader.load_linear_weights(reference_gate_up_proj, "gate_up_proj", gate_up_weights) + loader.load_linear_weights( + down_proj, + "down_proj", + [{"weight": down_weight, "input_scale": down_input_scale}], + ) + for projection in (gate_up_proj, reference_gate_up_proj, down_proj): + projection.post_load_weights() + + return attention, reference_gate_up_proj, down_proj + + +@pytest.mark.skipif( + get_sm_version() not in (100, 103), + reason="Real NVFP4 SwiGLU kernels require SM100 or SM103", +) +@pytest.mark.parametrize( + "path,num_tokens", + [ + ("cute_bf16_output", 64), + ("cute_fp4_output", 128), + ("moe_split_fallback", 128), + ], +) +def test_flux2_nvfp4_swiglu_paths_match_eager(path, num_tokens): + torch.manual_seed(5) + hidden_states = torch.randn(1, num_tokens, 256, device="cuda", dtype=torch.bfloat16) + attention, reference_gate_up_proj, down_proj = _make_flux2_nvfp4_swiglu_test_modules( + hidden_states + ) + + gate_up_hidden = reference_gate_up_proj(hidden_states) + gate, up = gate_up_hidden.chunk(2, dim=-1) + eager_mlp = (F.silu(gate.float()) * up.float()).to(torch.bfloat16) + expected = down_proj(eager_mlp) + + if path == "moe_split_fallback": + with mock.patch.object( + attention, + "_combine_split_projection", + side_effect=lambda _attn, mlp: mlp, + ): + actual = attention._project_split_output_with_fp4_mlp( + torch.empty(0, device="cuda"), gate_up_hidden + ) + else: + expect_fp4_output = path == "cute_fp4_output" + assert attention._can_project_hidden_mlp_with_fp4out(hidden_states) is expect_fp4_output + actual = attention._project_hidden_mlp_with_cute_dsl(hidden_states) + + cosine = F.cosine_similarity(actual.float().flatten(), expected.float().flatten(), dim=0) + relative_error = (actual.float() - expected.float()).norm() / expected.float().norm() + assert cosine > 0.995 + assert relative_error < 0.12 + + +def test_flux2_mlp_fp4_guard_requires_blackwell(monkeypatch): + from tensorrt_llm._torch.visual_gen.models.flux import attention as flux_attention + + monkeypatch.setattr(flux_attention.torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(flux_attention, "is_sm_100f", lambda: True) + attn, _, _ = _make_fake_flux2_parallel_attn() + assert attn._can_project_mlp_out_from_fp4() + + monkeypatch.setattr(flux_attention, "is_sm_100f", lambda: False) + assert not attn._can_project_mlp_out_from_fp4() + + +def test_flux2_mlp_fp4_guard_requires_static_output_quant(monkeypatch): + from tensorrt_llm._torch.visual_gen.models.flux import attention as flux_attention + + monkeypatch.setattr(flux_attention.torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(flux_attention, "is_sm_100f", lambda: True) + attn, _, down_proj = _make_fake_flux2_parallel_attn() + + assert attn._can_project_mlp_out_from_fp4() + + down_proj.input_scale = None + assert not attn._can_project_mlp_out_from_fp4() + + down_proj.input_scale = torch.ones(1) + down_proj.pre_quant_scale = torch.ones(128) + assert not attn._can_project_mlp_out_from_fp4() + + down_proj.pre_quant_scale = None + down_proj.force_dynamic_quantization = True + assert not attn._can_project_mlp_out_from_fp4() + + class TestFluxTransformer(unittest.TestCase): """Unit tests for FLUX transformer models.""" @@ -127,6 +445,92 @@ def test_flux2_model_structure(self): self.assertEqual(len(model.transformer_blocks), 1) self.assertEqual(len(model.single_transformer_blocks), 1) + def test_flux2_dual_stream_ffn_enables_cutedsl_blockscaling(self): + """FLUX.2 dual-stream FFNs should reach the fused NVFP4 SwiGLU path.""" + from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2 import ( + Flux2TransformerBlock, + ) + + with ( + mock.patch.object(torch.cuda, "is_available", return_value=True), + mock.patch( + "tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2.is_sm_100f", + return_value=True, + ), + ): + model_config = self._create_model_config(FLUX2_CONFIG) + model_config.quant_config = QuantConfig(quant_algo=QuantAlgo.NVFP4) + model_config.skip_create_weights_in_init = True + block = Flux2TransformerBlock( + dim=16, + num_attention_heads=2, + attention_head_dim=8, + mlp_ratio=2.0, + config=model_config, + ) + + self.assertTrue(block.ff.use_cute_dsl_blockscaling_mm) + self.assertTrue(block.ff_context.use_cute_dsl_blockscaling_mm) + + def test_flux2_dual_stream_ffn_disables_cutedsl_blockscaling_off_sm100f(self): + from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2 import ( + Flux2TransformerBlock, + ) + + with ( + mock.patch.object(torch.cuda, "is_available", return_value=True), + mock.patch( + "tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2.is_sm_100f", + return_value=False, + ), + ): + model_config = self._create_model_config(FLUX2_CONFIG) + model_config.quant_config = QuantConfig(quant_algo=QuantAlgo.NVFP4) + model_config.skip_create_weights_in_init = True + block = Flux2TransformerBlock( + dim=16, + num_attention_heads=2, + attention_head_dim=8, + mlp_ratio=2.0, + config=model_config, + ) + + self.assertFalse(block.ff.use_cute_dsl_blockscaling_mm) + self.assertFalse(block.ff_context.use_cute_dsl_blockscaling_mm) + + def test_flux2_dual_stream_ffn_handles_missing_config(self): + from tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2 import ( + Flux2TransformerBlock, + ) + + with ( + mock.patch.object(torch.cuda, "is_available", return_value=True), + mock.patch( + "tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2.is_sm_100f", + return_value=True, + ), + mock.patch( + "tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2.FluxJointAttention", + return_value=torch.nn.Identity(), + ), + mock.patch( + "tensorrt_llm._torch.visual_gen.models.flux.transformer_flux2.GatedMLP", + side_effect=lambda **_: torch.nn.Identity(), + ) as gated_mlp, + ): + Flux2TransformerBlock( + dim=16, + num_attention_heads=2, + attention_head_dim=8, + mlp_ratio=2.0, + config=None, + skip_create_weights=True, + ) + + self.assertEqual(gated_mlp.call_count, 2) + for call in gated_mlp.call_args_list: + self.assertFalse(call.kwargs["use_cute_dsl_blockscaling_mm"]) + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") def test_flux1_forward_sanity(self): """Test FLUX.1 forward pass produces valid output.""" diff --git a/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py b/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py index d2bbf46d32a5..282fc81cf8c0 100644 --- a/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py +++ b/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py @@ -1239,20 +1239,96 @@ def fake_get_free_gpu_memory_gib(device=None): assert queried_devices == ["cuda:1"] def test_two_stage_cuda_graph_key_includes_lora_state(self): - """Original and merged LoRA bindings must not share CUDA graph keys.""" - lora_state = {"value": "original"} + """No (lora_state, topology) combination may share a CUDA graph key. + + Red/green matrix over the states production visits — (original, + default) for stage 1 and (merged, stage2) for stage 2 — plus the + off-diagonal combinations, proving both key dimensions isolate + independently. + """ + state = {"lora": "original", "topology": "default"} runner = ltx2_two_stages._LTX2TwoStageCUDAGraphRunner( ltx2_two_stages.CUDAGraphRunnerConfig(use_cuda_graph=True), - lambda: lora_state["value"], + lambda: state["lora"], + lambda: state["topology"], ) - original_key = runner.get_graph_key(torch.empty(1, 2)) - lora_state["value"] = "merged" - merged_key = runner.get_graph_key(torch.empty(1, 2)) + keys = {} + for lora in ("original", "merged"): + for topology in ("default", "stage2"): + state["lora"], state["topology"] = lora, topology + keys[(lora, topology)] = runner.get_graph_key(torch.empty(1, 2)) + + assert len(set(keys.values())) == 4, "graph keys shared across states" + for (lora, topology), key in keys.items(): + assert key[-2] == ("ltx2_two_stage_lora_state", lora) + assert key[-1] == ("ltx2_two_stage_topology", topology) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_run_warmup_precaptures_both_topologies(self): + """The production warmup entrypoint (_run_warmup) pre-captures one + graph per topology; a repeat warmup-shaped request performs zero new + captures. forward is stubbed to model the two-stage boundary, but the + driver is the real inherited _run_warmup.""" + + class TinyTransformer: + def __init__(self): + self.lin = torch.nn.Linear(8, 8, device="cuda") + self.active_topology = "default" + self.device = "cuda" + + def forward(self, x): + return self.lin(x) + + pipeline = object.__new__(ltx2_two_stages.LTX2TwoStagesPipeline) + pipeline.pipeline_config = DiffusionPipelineConfig( + cuda_graph=CudaGraphConfig(enable=True), + torch_compile=TorchCompileConfig(enable=False), + ) + pipeline.transformer = TinyTransformer() + pipeline._cuda_graph_runners = {} + pipeline._setup_cuda_graphs() + runner = pipeline._cuda_graph_runners["transformer"] + + x = torch.randn(2, 8, device="cuda") + + def fake_forward(*args, **kwargs): + for topology in ("default", "stage2"): + pipeline.transformer.active_topology = topology + pipeline.transformer.forward(x) + + pipeline.forward = fake_forward + + ltx2_two_stages.LTX2TwoStagesPipeline._run_warmup(pipeline, 512, 768, 121, 2) + assert sorted(key[-1] for key in runner.graphs) == [ + ("ltx2_two_stage_topology", "default"), + ("ltx2_two_stage_topology", "stage2"), + ] + + captures = [] + original_capture = runner.capture + runner.capture = lambda *a, **k: (captures.append(a), original_capture(*a, **k)) + ltx2_two_stages.LTX2TwoStagesPipeline._run_warmup(pipeline, 512, 768, 121, 2) + assert not captures, "post-warmup request re-captured a graph" + assert len(runner.graphs) == 2 + + def test_two_stage_forward_rejects_indivisible_stage2_tokens(self): + """forward() fails fast at request entry when the full-resolution token + count does not divide the stage-2 seq-plane size.""" + lat = ltx2_two_stages.VideoLatentShape.from_pixel_shape( + ltx2_two_stages.VideoPixelShape(batch=1, frames=121, height=512, width=768, fps=24.0) + ) + tokens = lat.frames * lat.height * lat.width + bad_size = next(s for s in (7, 11, 13) if tokens % s != 0) + + pipeline = object.__new__(ltx2_two_stages.LTX2TwoStagesPipeline) + pipeline.transformer = SimpleNamespace( + _has_stage2=True, + _sharder_s2=SimpleNamespace(size=bad_size), + ) - assert original_key != merged_key - assert original_key[-1] == ("ltx2_two_stage_lora_state", "original") - assert merged_key[-1] == ("ltx2_two_stage_lora_state", "merged") + with pytest.raises(ValueError, match="not divisible by the stage-2"): + pipeline.forward(prompt="x", seed=0, height=512, width=768, num_frames=121) def test_two_stage_cuda_graph_setup_uses_pipeline_config(self): """CUDA graph setup runs before the two-stage model_config is assigned.""" diff --git a/tests/unittest/_torch/visual_gen/test_qwen_image_infer.py b/tests/unittest/_torch/visual_gen/test_qwen_image_infer.py index 5b731904f24e..1112e178a502 100644 --- a/tests/unittest/_torch/visual_gen/test_qwen_image_infer.py +++ b/tests/unittest/_torch/visual_gen/test_qwen_image_infer.py @@ -83,7 +83,7 @@ def test_infer_forwards_generation_params(): assert captured["height"] == 768 assert captured["width"] == 1024 assert captured["num_inference_steps"] == 7 - assert captured["true_cfg_scale"] == 3.25 + assert captured["negative_prompt_cfg_scale"] == 3.25 assert captured["seed"] == 123 assert captured["max_sequence_length"] == 256 diff --git a/tests/unittest/_torch/visual_gen/test_qwen_image_layered_parity.py b/tests/unittest/_torch/visual_gen/test_qwen_image_layered_parity.py new file mode 100644 index 000000000000..caa286dd1887 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_qwen_image_layered_parity.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Per-module parity tests for Qwen-Image-Layered.""" + +import pytest +import torch + + +def _cosine(a: torch.Tensor, b: torch.Tensor) -> float: + a = a.detach().float().flatten() + b = b.detach().float().flatten() + return torch.nn.functional.cosine_similarity(a, b, dim=0).item() + + +requires_cuda = pytest.mark.skipif( + not torch.cuda.is_available(), + reason="CUDA is required for BF16 parity tests.", +) + + +@requires_cuda +def test_qwen_layered_timestep_proj_embedding_additional_cond_parity(): + transformer_qwenimage = pytest.importorskip( + "diffusers.models.transformers.transformer_qwenimage" + ) + RefTimestep = getattr(transformer_qwenimage, "QwenTimestepProjEmbeddings", None) + if RefTimestep is None: + pytest.skip("diffusers QwenTimestepProjEmbeddings is unavailable") + + from tensorrt_llm._torch.visual_gen.models.qwen_image import ( + QwenTimestepProjEmbeddings as OurTimestep, + ) + + device = torch.device("cuda") + dtype = torch.bfloat16 + torch.manual_seed(0) + + try: + ref = RefTimestep(embedding_dim=32, use_additional_t_cond=True).to(dtype).to(device).eval() + except TypeError: + pytest.skip("diffusers QwenTimestepProjEmbeddings lacks use_additional_t_cond") + + our = OurTimestep(embedding_dim=32, use_additional_t_cond=True).to(dtype).to(device).eval() + our.load_state_dict(ref.state_dict(), strict=True) + + timestep = torch.tensor([0.25, 0.75], dtype=dtype, device=device) + hidden_states = torch.zeros(2, 1, 32, dtype=dtype, device=device) + additional_t_cond = torch.tensor([0, 1], dtype=torch.long, device=device) + + with torch.inference_mode(): + ref_out = ref(timestep, hidden_states, additional_t_cond) + our_out = our(timestep, hidden_states, additional_t_cond) + + sim = _cosine(ref_out, our_out) + assert sim > 0.999, f"cosine={sim}" + + +@requires_cuda +@pytest.mark.parametrize( + "layer_fhws", + [ + [(1, 4, 4), (1, 4, 4), (1, 4, 4)], + [(1, 4, 6), (1, 2, 6), (1, 4, 6)], + ], +) +@pytest.mark.parametrize("max_txt_seq_len", [16, 64]) +def test_qwen_embed_layer3d_rope_parity(layer_fhws, max_txt_seq_len): + transformer_qwenimage = pytest.importorskip( + "diffusers.models.transformers.transformer_qwenimage" + ) + RefLayerRope = getattr(transformer_qwenimage, "QwenEmbedLayer3DRope", None) + if RefLayerRope is None: + pytest.skip("diffusers QwenEmbedLayer3DRope is unavailable") + + from tensorrt_llm._torch.visual_gen.models.qwen_image_layered.transformer_qwen_image_layered import ( + QwenEmbedLayer3DRope, + ) + + device = torch.device("cuda") + ref = RefLayerRope(theta=10000, axes_dim=[16, 56, 56], scale_rope=True).to(device) + our = QwenEmbedLayer3DRope(theta=10000, axes_dim=[16, 56, 56], scale_rope=True).to(device) + + ref_vid, ref_txt = ref([layer_fhws], max_txt_seq_len=max_txt_seq_len, device=device) + our_vid, our_txt = our([layer_fhws], max_txt_seq_len=max_txt_seq_len, device=device) + + assert torch.equal(ref_vid, our_vid), "layered video freqs differ" + assert torch.equal(ref_txt, our_txt), "layered text freqs differ" diff --git a/tests/unittest/_torch/visual_gen/test_qwen_image_layered_pipeline_config.py b/tests/unittest/_torch/visual_gen/test_qwen_image_layered_pipeline_config.py new file mode 100644 index 000000000000..74bdc2d622a8 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_qwen_image_layered_pipeline_config.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pipeline-level configuration tests for Qwen-Image-Layered.""" + +import json + +import pytest + +# Importing the models package applies the Qwen-Image-Layered registration side effect. +from tensorrt_llm._torch.visual_gen import models # noqa: F401 +from tensorrt_llm._torch.visual_gen.config import DiffusionPipelineConfig +from tensorrt_llm._torch.visual_gen.models.qwen_image_layered import QwenImageLayeredPipeline +from tensorrt_llm._torch.visual_gen.models.qwen_image_layered.transformer_qwen_image_layered import ( + QwenEmbedLayer3DRope, + QwenImageLayeredTransformer2DModel, +) +from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader +from tensorrt_llm.visual_gen.args import VisualGenArgs + + +def _tiny_qwen_layered_transformer_config(**overrides): + config = { + "_class_name": "QwenImageTransformer2DModel", + "patch_size": 1, + "in_channels": 4, + "out_channels": 4, + "num_layers": 1, + "attention_head_dim": 8, + "num_attention_heads": 2, + "joint_attention_dim": 12, + "axes_dims_rope": [2, 2, 4], + } + config.update(overrides) + return config + + +def _write_minimal_qwen_layered_checkpoint(tmp_path, transformer_config=None): + """Create the minimum diffusers layout needed by PipelineLoader config code.""" + (tmp_path / "model_index.json").write_text( + json.dumps( + { + "_class_name": "QwenImageLayeredPipeline", + "transformer": ["diffusers", "QwenImageTransformer2DModel"], + } + ) + ) + transformer_dir = tmp_path / "transformer" + transformer_dir.mkdir() + transformer_config = transformer_config or {"_class_name": "QwenImageTransformer2DModel"} + (transformer_dir / "config.json").write_text(json.dumps(transformer_config)) + return tmp_path + + +def test_qwen_layered_pipeline_config_defaults_to_empty_dict(tmp_path): + checkpoint_dir = _write_minimal_qwen_layered_checkpoint(tmp_path) + + args = VisualGenArgs(model=str(checkpoint_dir)) + resolved = PipelineLoader(args)._resolve_pipeline_config(str(checkpoint_dir)) + + assert resolved == {} + + +def test_qwen_layered_pipeline_config_rejects_unknown_keys(tmp_path): + checkpoint_dir = _write_minimal_qwen_layered_checkpoint(tmp_path) + + args = VisualGenArgs( + model=str(checkpoint_dir), + pipeline_config={"text_encoder_path": "/tmp/not-a-qwen-layered-knob"}, + ) + with pytest.raises( + ValueError, match="Unknown pipeline_config keys for QwenImageLayeredPipeline" + ): + PipelineLoader(args)._resolve_pipeline_config(str(checkpoint_dir)) + + +def test_qwen_layered_config_initializes_layered_transformer(tmp_path): + checkpoint_dir = _write_minimal_qwen_layered_checkpoint( + tmp_path, + transformer_config=_tiny_qwen_layered_transformer_config( + use_additional_t_cond=True, + use_layer3d_rope=True, + ), + ) + args = VisualGenArgs(model=str(checkpoint_dir)) + + config = DiffusionPipelineConfig.from_pretrained(str(checkpoint_dir), args=args) + pipeline = QwenImageLayeredPipeline(config) + + assert pipeline.guidance_embeds is False + assert pipeline.zero_cond_t is False + assert isinstance(pipeline.transformer, QwenImageLayeredTransformer2DModel) + assert pipeline.transformer.time_text_embed.use_additional_t_cond is True + assert isinstance(pipeline.transformer.pos_embed, QwenEmbedLayer3DRope) diff --git a/tests/unittest/_torch/visual_gen/test_qwen_image_layered_registry.py b/tests/unittest/_torch/visual_gen/test_qwen_image_layered_registry.py new file mode 100644 index 000000000000..79e61bb65067 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_qwen_image_layered_registry.py @@ -0,0 +1,286 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Registry and transformer smoke tests for Qwen-Image-Layered.""" + +import json +import sys +from types import SimpleNamespace + +import pytest +import torch + +# Importing the models package side-effects the ``@register_pipeline`` +# decorator on ``QwenImageLayeredPipeline`` being applied. +from tensorrt_llm._torch.visual_gen import models # noqa: F401 +from tensorrt_llm._torch.visual_gen.config import ( + AttentionConfig, + DiffusionModelConfig, + DiffusionPipelineConfig, +) +from tensorrt_llm._torch.visual_gen.cuda_graph_runner import CUDAGraphRunner, CUDAGraphRunnerConfig +from tensorrt_llm._torch.visual_gen.models.qwen_image import QwenImagePipeline +from tensorrt_llm._torch.visual_gen.models.qwen_image_layered import QwenImageLayeredPipeline +from tensorrt_llm._torch.visual_gen.models.qwen_image_layered.transformer_qwen_image_layered import ( + QwenEmbedLayer3DRope, + QwenImageLayeredTransformer2DModel, +) +from tensorrt_llm._torch.visual_gen.pipeline import BasePipeline +from tensorrt_llm._torch.visual_gen.pipeline_registry import ( + PIPELINE_REGISTRY, + AutoPipeline, + PipelineComponent, +) + + +def _pipeline_config(model_config=None): + return DiffusionPipelineConfig( + model_configs={"transformer": model_config or DiffusionModelConfig()} + ) + + +def test_qwen_image_layered_pipeline_is_registered(): + """@register_pipeline("QwenImageLayeredPipeline") must have been applied.""" + assert "QwenImageLayeredPipeline" in PIPELINE_REGISTRY + entry = PIPELINE_REGISTRY["QwenImageLayeredPipeline"] + assert entry.pipeline_cls is QwenImageLayeredPipeline + assert entry.hf_ids == ["Qwen/Qwen-Image-Layered"] + + +def test_qwen_image_layered_pipeline_has_separate_boundary(): + """Layered is registered as its own pipeline variant, not a QwenImagePipeline subclass.""" + assert issubclass(QwenImageLayeredPipeline, BasePipeline) + assert not issubclass(QwenImageLayeredPipeline, QwenImagePipeline) + + +def test_auto_pipeline_detects_qwen_image_layered_class_name(tmp_path): + """Qwen-Image-Layered checkpoints must resolve to their layered pipeline.""" + (tmp_path / "model_index.json").write_text( + json.dumps({"_class_name": "QwenImageLayeredPipeline"}) + ) + assert AutoPipeline._detect_from_checkpoint(str(tmp_path)) == "QwenImageLayeredPipeline" + + +def test_auto_pipeline_routes_qwen_image_layered_variants(tmp_path): + """Unknown layered variants stay on the layered pipeline, not the base Qwen pipeline.""" + (tmp_path / "model_index.json").write_text( + json.dumps({"_class_name": "QwenImageLayeredEditPipeline"}) + ) + assert AutoPipeline._detect_from_checkpoint(str(tmp_path)) == "QwenImageLayeredPipeline" + + +def test_qwen_image_layered_processor_loads_from_subfolder(monkeypatch, tmp_path): + """Layered checkpoints store the VL processor in the processor/ subfolder.""" + calls = [] + + class FakeProcessor: + @classmethod + def from_pretrained(cls, checkpoint_dir, **kwargs): + calls.append((checkpoint_dir, kwargs)) + return cls() + + monkeypatch.setitem( + sys.modules, + "transformers", + SimpleNamespace(Qwen2VLProcessor=FakeProcessor), + ) + pipeline = QwenImageLayeredPipeline(_pipeline_config()) + pipeline.load_standard_components( + str(tmp_path), + torch.device("cpu"), + skip_components=[ + PipelineComponent.TOKENIZER, + PipelineComponent.TEXT_ENCODER, + PipelineComponent.VAE, + PipelineComponent.SCHEDULER, + PipelineComponent.IMAGE_PROCESSOR, + ], + ) + + assert calls == [ + (str(tmp_path), {"subfolder": "processor"}), + ] + assert isinstance(pipeline.processor, FakeProcessor) + + +def test_qwen_image_layered_default_params_match_runtime_inputs(): + """Layered derives size from image+resolution unless height/width are explicitly set.""" + pipeline = QwenImageLayeredPipeline(_pipeline_config()) + assert pipeline.default_generation_params["height"] is None + assert pipeline.default_generation_params["width"] is None + assert pipeline.extra_param_specs["resolution"].range is None + assert pipeline.default_warmup_num_frames == [1] + assert pipeline.warmup_cache_key(None, None, num_frames=1) == (640, 640) + assert pipeline.warmup_cache_key(512, 768, num_frames=1) == (512, 768) + + +def test_qwen_image_layered_rejects_multi_frame_latent_input(): + """Layered latent input accepts one conditioning frame before packing.""" + pipeline = QwenImageLayeredPipeline(_pipeline_config()) + image = torch.zeros(1, pipeline.latent_channels, 2, 4, 4) + + assert pipeline._is_layered_latent_image(image) + with pytest.raises(ValueError, match="exactly one conditioning frame"): + pipeline._validate_single_conditioning_frame(image) + + +def test_qwen_image_layered_repeats_conditioning_in_prompt_order(): + """Conditioning images follow prompt expansion order for num_images_per_prompt > 1.""" + image_latents = torch.arange(2).view(2, 1, 1, 1, 1) + + repeated = QwenImageLayeredPipeline._repeat_conditioning_batch(image_latents, 4, "image") + + assert repeated[:, 0, 0, 0, 0].tolist() == [0, 0, 1, 1] + + +def test_qwen_image_layered_aligns_captions_to_expanded_prompts(): + """Per-image captions are repeated positionally for expanded prompt batches.""" + prompts = ["", "", "", ""] + captions = ["caption 0", "caption 1"] + + aligned_prompts = QwenImageLayeredPipeline._align_prompts_to_image_batch(prompts, 2) + aligned_captions = QwenImageLayeredPipeline._expand_values_to_batch( + captions, + len(aligned_prompts), + "caption", + ) + + assert aligned_prompts == prompts + assert aligned_captions == ["caption 0", "caption 0", "caption 1", "caption 1"] + + +def test_qwen_image_layered_layer_stack_to_image_grid(): + """Layer stacks are converted to a saveable image grid instead of video output.""" + layer_stack = torch.arange(12, dtype=torch.uint8).view(1, 3, 2, 2, 1) + + grid = QwenImageLayeredPipeline._layer_stack_to_image_grid(layer_stack) + + assert grid.shape == (1, 4, 4, 1) + assert grid[0, :, :, 0].tolist() == [ + [0, 1, 4, 5], + [2, 3, 6, 7], + [8, 9, 0, 0], + [10, 11, 0, 0], + ] + + +def test_transformer_constructs_with_layered_config(): + """Layered config fields select layer-aware RoPE and additional time conditioning.""" + model = QwenImageLayeredTransformer2DModel( + model_config=None, + num_layers=2, + use_additional_t_cond=True, + use_layer3d_rope=True, + ) + assert isinstance(model.pos_embed, QwenEmbedLayer3DRope) + assert model.time_text_embed.use_additional_t_cond is True + assert hasattr(model.time_text_embed, "addition_t_embedding") + + +def test_transformer_from_config_dict_preserves_layered_fields(): + """from_config_dict should not drop Qwen-Image-Layered transformer fields.""" + model = QwenImageLayeredTransformer2DModel.from_config_dict( + { + "num_layers": 2, + "use_additional_t_cond": True, + "use_layer3d_rope": True, + } + ) + assert isinstance(model.pos_embed, QwenEmbedLayer3DRope) + assert model.time_text_embed.use_additional_t_cond is True + + +def test_layered_transformer_cuda_graph_key_includes_img_shapes(): + """Layered RoPE shape metadata must be part of the CUDA graph key.""" + runner = CUDAGraphRunner(CUDAGraphRunnerConfig(use_cuda_graph=True)) + model = QwenImageLayeredTransformer2DModel( + model_config=None, + patch_size=1, + in_channels=4, + out_channels=4, + num_layers=1, + attention_head_dim=8, + num_attention_heads=2, + joint_attention_dim=12, + axes_dims_rope=(2, 2, 4), + use_layer3d_rope=True, + ) + model.register_cuda_graph_extra_key_fns(runner) + + tensor_kwargs = { + "hidden_states": torch.empty(1, 12, 4), + "encoder_hidden_states": torch.empty(1, 5, 12), + "timestep": torch.empty(1), + } + key_a = runner.get_graph_key( + **tensor_kwargs, + img_shapes=[[(1, 2, 3), (1, 2, 3)]], + ) + key_b = runner.get_graph_key( + **tensor_kwargs, + img_shapes=[[(1, 3, 2), (1, 3, 2)]], + ) + + assert ("img_shapes", (((1, 2, 3), (1, 2, 3)),)) in key_a + assert ("img_shapes", (((1, 3, 2), (1, 3, 2)),)) in key_b + assert key_a != key_b + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +@pytest.mark.parametrize("with_text_mask", [False, True]) +def test_layered_transformer_forward_sanity(with_text_mask): + """A tiny Qwen-Image-Layered transformer runs layer-aware RoPE and t-cond paths.""" + torch.manual_seed(0) + device = torch.device("cuda") + dtype = torch.bfloat16 + model_config = DiffusionModelConfig( + attention=AttentionConfig(backend="VANILLA"), + skip_create_weights_in_init=False, + ) + model = ( + QwenImageLayeredTransformer2DModel( + model_config=model_config, + patch_size=1, + in_channels=4, + out_channels=4, + num_layers=1, + attention_head_dim=8, + num_attention_heads=2, + joint_attention_dim=12, + axes_dims_rope=(2, 2, 4), + use_additional_t_cond=True, + use_layer3d_rope=True, + ) + .to(device, dtype=dtype) + .eval() + ) + + hidden_states = torch.randn(1, 12, 4, device=device, dtype=dtype) + encoder_hidden_states = torch.randn(1, 5, 12, device=device, dtype=dtype) + encoder_hidden_states_mask = None + if with_text_mask: + encoder_hidden_states_mask = torch.tensor([[True, True, True, False, False]], device=device) + + with torch.inference_mode(): + output = model( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + encoder_hidden_states_mask=encoder_hidden_states_mask, + timestep=torch.tensor([0.5], device=device, dtype=dtype), + additional_t_cond=torch.zeros(1, device=device, dtype=torch.long), + img_shapes=[[(1, 2, 2), (1, 2, 2), (1, 2, 2)]], + ) + + assert isinstance(output, tuple) + assert output[0].shape == (1, 12, 4) diff --git a/tests/unittest/_torch/visual_gen/test_qwen_image_pipeline.py b/tests/unittest/_torch/visual_gen/test_qwen_image_pipeline.py index d2d3f08bbd87..4c2c5b7115cc 100644 --- a/tests/unittest/_torch/visual_gen/test_qwen_image_pipeline.py +++ b/tests/unittest/_torch/visual_gen/test_qwen_image_pipeline.py @@ -10,6 +10,8 @@ combination) and the non-CFG path, on CPU. """ +from types import SimpleNamespace + import torch from tensorrt_llm._torch.visual_gen.models.qwen_image import QwenImagePipeline @@ -81,6 +83,9 @@ def _pipeline_with_test_doubles(): pipe.vae_scale_factor = 8 pipe.transformer = _RecordingTransformer() pipe.scheduler = _RecordingScheduler() + pipe.pipeline_config = SimpleNamespace( + cuda_graph=SimpleNamespace(enable=False), visual_gen_mapping=None + ) captured = {"encoded_prompts": []} def _encode_prompt(prompt, device, max_sequence_length): @@ -129,7 +134,7 @@ def _expanded_noise(values, *, batch_size, seq_len, dtype=torch.float32): return pattern.view(1, 1, -1).expand(batch_size, seq_len, -1) -def test_forward_runs_without_true_cfg(): +def test_forward_runs_without_negative_prompt_cfg(): pipe, captured = _pipeline_with_test_doubles() output = pipe.forward( @@ -138,7 +143,7 @@ def test_forward_runs_without_true_cfg(): height=32, width=48, num_inference_steps=3, - true_cfg_scale=4.0, + negative_prompt_cfg_scale=1.0, seed=123, max_sequence_length=16, sigmas=[1.0, 0.5, 0.25], @@ -160,7 +165,37 @@ def test_forward_runs_without_true_cfg(): ) -def test_forward_runs_true_cfg_pipeline(): +def test_forward_defaults_missing_negative_prompt_to_empty_string_for_cfg(): + pipe, captured = _pipeline_with_test_doubles() + + output = pipe.forward( + prompt=["a cat"], + negative_prompt=None, + height=32, + width=48, + num_inference_steps=2, + negative_prompt_cfg_scale=4.0, + seed=123, + max_sequence_length=16, + sigmas=[1.0, 0.5], + ) + + assert output.image.shape == (1, 32, 48, 3) + assert captured["encoded_prompts"] == [ + (["a cat"], torch.device("cpu"), 16), + ([""], torch.device("cpu"), 16), + ] + assert len(pipe.transformer.calls) == 4 + assert torch.all(pipe.transformer.calls[0]["encoder_hidden_states"] > 0) + assert torch.all(pipe.transformer.calls[1]["encoder_hidden_states"] > 0) + assert len(pipe.scheduler.step_calls) == 2 + assert torch.allclose( + pipe.scheduler.step_calls[0]["noise_pred"], + _expanded_noise([1, 2, 3, 4], batch_size=1, seq_len=6), + ) + + +def test_forward_runs_negative_prompt_cfg_pipeline(): pipe, captured = _pipeline_with_test_doubles() output = pipe.forward( @@ -169,7 +204,7 @@ def test_forward_runs_true_cfg_pipeline(): height=32, width=48, num_inference_steps=2, - true_cfg_scale=3.0, + negative_prompt_cfg_scale=3.0, seed=123, max_sequence_length=16, sigmas=[1.0, 0.5], diff --git a/tests/unittest/_torch/visual_gen/test_qwen_image_pipeline_config.py b/tests/unittest/_torch/visual_gen/test_qwen_image_pipeline_config.py index 0a8ef1900479..db0f9a6db398 100644 --- a/tests/unittest/_torch/visual_gen/test_qwen_image_pipeline_config.py +++ b/tests/unittest/_torch/visual_gen/test_qwen_image_pipeline_config.py @@ -4,16 +4,45 @@ """Pipeline-level configuration tests for Qwen-Image.""" import json +from pathlib import Path from types import SimpleNamespace +from typing import Any import pytest +import torch +import torch.nn.functional as F + +from tensorrt_llm._torch.modules.linear import TensorParallelMode +from tensorrt_llm._torch.utils import gelu_tanh # Importing the models package applies the Qwen-Image registration side effect. from tensorrt_llm._torch.visual_gen import models # noqa: F401 +from tensorrt_llm._torch.visual_gen import modules as visual_gen_modules +from tensorrt_llm._torch.visual_gen.attention_backend.parallel import ( + Attention2DAttention, + RingAttention, + UlyssesAttention, +) from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig, DiffusionPipelineConfig -from tensorrt_llm._torch.visual_gen.models.qwen_image import QwenJointAttention -from tensorrt_llm._torch.visual_gen.modules.attention import QKVMode +from tensorrt_llm._torch.visual_gen.models.qwen_image import ( + QwenImagePipeline, + QwenImageTransformerBlock, + QwenJointAttention, + apply_rotary_emb_qwen, +) +from tensorrt_llm._torch.visual_gen.models.qwen_image.transformer_qwen_image import ( + FeedForward, + QwenImageTransformer2DModel, + _build_joint_attention_mask, + _get_feedforward_activation, + _is_qwen_sequence_parallel_attention, + _supports_qwen_key_padding_mask, + qwen_complex_freqs_to_cos_sin, + qwen_joint_freqs_to_cos_sin, +) +from tensorrt_llm._torch.visual_gen.modules.attention import QKVMode, apply_rotary_emb from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader +from tensorrt_llm.mapping import Mapping from tensorrt_llm.quantization.mode import QuantAlgo from tensorrt_llm.visual_gen.args import ( AttentionConfig, @@ -24,7 +53,7 @@ ) -def _write_minimal_qwen_checkpoint(tmp_path): +def _write_minimal_qwen_checkpoint(tmp_path: Path) -> Path: """Create the minimum diffusers layout needed by PipelineLoader config code.""" (tmp_path / "model_index.json").write_text( json.dumps( @@ -187,31 +216,42 @@ def test_qwen_pipeline_quant_config_parses_from_args( @pytest.mark.parametrize( - ("visual_gen_mapping", "not_wrapped_as"), + ("visual_gen_mapping", "expected_use_ulysses"), [ pytest.param( SimpleNamespace( - attn2d_row_size=2, - attn2d_col_size=2, + cp_size=2, ring_size=1, - ulysses_size=1, + ulysses_size=2, ), - "Attention2DAttention", - id="attention2d", + True, + id="hybrid-context-parallel-ulysses", ), pytest.param( SimpleNamespace( - attn2d_row_size=1, - attn2d_col_size=1, + cp_size=2, ring_size=2, ulysses_size=1, ), - "RingAttention", + False, id="ring", ), ], ) -def test_qwen_joint_attention_keeps_separate_qkv_path_unwrapped(visual_gen_mapping, not_wrapped_as): +def test_qwen_joint_attention_wraps_separate_qkv_self_attention( + monkeypatch, visual_gen_mapping, expected_use_ulysses +): + captured = {} + + def capture_parallel_attention(attn, **kwargs): + captured.update(kwargs) + return attn + + monkeypatch.setattr( + visual_gen_modules.attention, + "wrap_parallel_attention", + capture_parallel_attention, + ) config = DiffusionModelConfig( attention=AttentionConfig(backend="VANILLA"), visual_gen_mapping=visual_gen_mapping, @@ -225,4 +265,544 @@ def test_qwen_joint_attention_keeps_separate_qkv_path_unwrapped(visual_gen_mappi ) assert attention.qkv_mode == QKVMode.SEPARATE_QKV - assert attention.attn.__class__.__name__ != not_wrapped_as + assert captured["enable_sequence_parallel"] + assert captured["use_ulysses"] is expected_use_ulysses + + +class _FakeTokenBatch: + def __init__(self, attention_mask): + self.attention_mask = attention_mask + self.input_ids = torch.arange(attention_mask.numel()).view_as(attention_mask) + + def to(self, device): + self.attention_mask = self.attention_mask.to(device) + self.input_ids = self.input_ids.to(device) + return self + + +class _FakeTokenizer: + def __init__(self, attention_mask): + self.attention_mask = attention_mask + + def __call__(self, *args, **kwargs): + return _FakeTokenBatch(self.attention_mask.clone()) + + +class _FakeTextEncoder: + def __call__(self, input_ids, **kwargs): + hidden = torch.arange( + input_ids.numel() * 4, + dtype=torch.float32, + device=input_ids.device, + ).view(*input_ids.shape, 4) + return SimpleNamespace(hidden_states=[hidden]) + + +@pytest.mark.parametrize( + ("attention_mask", "expect_none"), + [ + pytest.param(torch.ones(2, 36, dtype=torch.long), True, id="all-valid"), + pytest.param( + torch.tensor([[1] * 36, [1] * 35 + [0]], dtype=torch.long), + False, + id="has-padding", + ), + ], +) +def test_qwen_encode_prompt_returns_none_for_all_valid_masks(attention_mask, expect_none): + pipeline_config = DiffusionPipelineConfig( + model_configs={ + "transformer": DiffusionModelConfig(pretrained_config=SimpleNamespace()), + } + ) + pipeline = QwenImagePipeline(pipeline_config) + pipeline.tokenizer = _FakeTokenizer(attention_mask) + pipeline.text_encoder = _FakeTextEncoder() + + prompt_embeds, prompt_embeds_mask = pipeline._encode_prompt( + ["one", "two"], + torch.device("cpu"), + max_sequence_length=8, + ) + + assert prompt_embeds.shape == (2, 2, 4) + if expect_none: + assert prompt_embeds_mask is None + else: + assert prompt_embeds_mask is not None + assert prompt_embeds_mask.shape == (2, 2) + assert not prompt_embeds_mask.bool().all() + + +def test_qwen_joint_attention_passes_padding_mask_to_backend(monkeypatch): + attention = QwenJointAttention( + dim=16, + num_attention_heads=2, + attention_head_dim=8, + config=DiffusionModelConfig(attention=AttentionConfig(backend="VANILLA")), + ) + captured = {} + + def fake_attn_impl(q, k, v, **kwargs): + captured["key_padding_mask"] = kwargs.get("key_padding_mask") + return q.new_zeros(q.shape) + + monkeypatch.setattr(attention, "_attn_impl", fake_attn_impl) + + hidden_states = torch.randn(1, 4, 16, dtype=torch.bfloat16) + encoder_hidden_states = torch.randn(1, 3, 16, dtype=torch.bfloat16) + attention_mask = torch.tensor([[True, False, True, True, True, True, True]]) + + attention( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + ) + + assert captured["key_padding_mask"] is attention_mask + + +def test_qwen_joint_attention_rejects_unsupported_masked_sequence_parallel(monkeypatch): + attention = QwenJointAttention( + dim=16, + num_attention_heads=2, + attention_head_dim=8, + config=DiffusionModelConfig(attention=AttentionConfig(backend="VANILLA")), + ) + attention.attn_backend = "FA4" + attention._supports_key_padding_mask = False + attention._uses_sequence_parallel_attention = True + monkeypatch.setattr( + attention, + "_prepare_qkv", + lambda *args, **kwargs: ( + torch.empty(1, 7, 16), + torch.empty(1, 7, 16), + torch.empty(1, 7, 16), + ), + ) + + with pytest.raises(NotImplementedError, match="Padded Qwen-Image prompts"): + attention( + hidden_states=torch.empty(1, 4, 16), + encoder_hidden_states=torch.empty(1, 3, 16), + attention_mask=torch.tensor([[True, False, True, True, True, True, True]]), + ) + + +def test_qwen_joint_attention_tp2_shards_both_streams(): + attention = QwenJointAttention( + dim=16, + num_attention_heads=2, + attention_head_dim=8, + config=DiffusionModelConfig( + mapping=Mapping(world_size=2, rank=0, tp_size=2), + attention=AttentionConfig(backend="VANILLA"), + ), + ) + + joint_q, joint_k, joint_v = attention._prepare_qkv_unfused( + hidden_states=torch.randn(1, 4, 16, dtype=torch.bfloat16), + encoder_hidden_states=torch.randn(1, 3, 16, dtype=torch.bfloat16), + image_rotary_emb=None, + ) + + assert not attention.fuse_qk_norm_rope + assert attention.add_q_proj.tp_mode == TensorParallelMode.COLUMN + assert attention.add_k_proj.tp_mode == TensorParallelMode.COLUMN + assert attention.add_v_proj.tp_mode == TensorParallelMode.COLUMN + assert attention.to_add_out.tp_mode == TensorParallelMode.ROW + assert joint_q.shape == (1, 7, 8) + assert joint_k.shape == (1, 7, 8) + assert joint_v.shape == (1, 7, 8) + + +def test_qwen_complex_freqs_convert_to_shared_rope_format(): + torch.manual_seed(0) + seq_len = 8 + head_dim = 16 + x = torch.randn(2, seq_len, 3, head_dim) + phases = torch.randn(seq_len, head_dim // 2) + freqs_cis = torch.polar(torch.ones_like(phases), phases) + + freqs_cos, freqs_sin = qwen_complex_freqs_to_cos_sin(freqs_cis) + + ref = apply_rotary_emb_qwen(x, freqs_cis) + out = apply_rotary_emb(x, freqs_cos, freqs_sin) + torch.testing.assert_close(out, ref, rtol=1e-6, atol=1e-6) + + +def test_qwen_joint_attention_fused_rope_passes_2d_freqs_to_kernel(monkeypatch): + torch.manual_seed(0) + txt_seq = 5 + img_seq = 7 + batch_size = 2 + head_dim = 8 + attention = QwenJointAttention( + dim=16, + num_attention_heads=2, + attention_head_dim=head_dim, + config=DiffusionModelConfig(), + ) + captured = {} + + def fake_apply_packed_qk_norm_rope(qkv, freqs_cos, freqs_sin, **kwargs): + captured["cos_shape"] = tuple(freqs_cos.shape) + captured["sin_shape"] = tuple(freqs_sin.shape) + + monkeypatch.setattr(attention, "apply_packed_qk_norm_rope", fake_apply_packed_qk_norm_rope) + + hidden_states = torch.randn(batch_size, img_seq, 16, dtype=torch.bfloat16) + encoder_hidden_states = torch.randn(batch_size, txt_seq, 16, dtype=torch.bfloat16) + img_phases = torch.randn(img_seq, head_dim // 2) + txt_phases = torch.randn(txt_seq, head_dim // 2) + image_rotary_emb = ( + torch.polar(torch.ones_like(img_phases), img_phases), + torch.polar(torch.ones_like(txt_phases), txt_phases), + ) + + attention._prepare_qkv_fused(hidden_states, encoder_hidden_states, image_rotary_emb) + + assert captured == { + "cos_shape": (txt_seq + img_seq, head_dim), + "sin_shape": (txt_seq + img_seq, head_dim), + } + + +def test_qwen_joint_attention_reuses_precomputed_fused_rope(monkeypatch): + from tensorrt_llm._torch.visual_gen.models.qwen_image import ( + transformer_qwen_image as qwen_transformer, + ) + + attention = QwenJointAttention( + dim=16, + num_attention_heads=2, + attention_head_dim=8, + config=DiffusionModelConfig(), + ) + hidden_states = torch.randn(2, 7, 16, dtype=torch.bfloat16) + encoder_hidden_states = torch.randn(2, 5, 16, dtype=torch.bfloat16) + image_rotary_emb = ( + torch.polar(torch.ones(7, 4), torch.randn(7, 4)), + torch.polar(torch.ones(5, 4), torch.randn(5, 4)), + ) + fused_rotary_emb = qwen_joint_freqs_to_cos_sin(image_rotary_emb) + captured = {} + + def fake_apply_packed_qk_norm_rope(qkv, freqs_cos, freqs_sin, **kwargs): + captured["freqs_cos"] = freqs_cos + captured["freqs_sin"] = freqs_sin + + def fail_recompute(*args, **kwargs): + raise AssertionError("precomputed RoPE should bypass complex-to-real conversion") + + monkeypatch.setattr(attention, "apply_packed_qk_norm_rope", fake_apply_packed_qk_norm_rope) + monkeypatch.setattr(qwen_transformer, "qwen_joint_freqs_to_cos_sin", fail_recompute) + + attention._prepare_qkv_fused( + hidden_states, + encoder_hidden_states, + image_rotary_emb, + fused_rotary_emb, + ) + + assert captured["freqs_cos"] is fused_rotary_emb[0] + assert captured["freqs_sin"] is fused_rotary_emb[1] + + +def test_qwen_joint_attention_fused_rope_requires_qk_norm(): + attention = QwenJointAttention( + dim=128, + num_attention_heads=2, + attention_head_dim=64, + config=DiffusionModelConfig(), + ) + hidden_states = SimpleNamespace(is_cuda=True, dtype=torch.bfloat16) + image_rotary_emb = (object(), object()) + + assert attention._use_fused_qk_norm_rope(hidden_states, image_rotary_emb) + + attention.qk_norm = False + assert not attention._use_fused_qk_norm_rope(hidden_states, image_rotary_emb) + + +def test_qwen_key_padding_mask_support_matrix(): + vanilla = object() + attention_2d = Attention2DAttention.__new__(Attention2DAttention) + ring = RingAttention.__new__(RingAttention) + ulysses = UlyssesAttention.__new__(UlyssesAttention) + + assert _supports_qwen_key_padding_mask("VANILLA", vanilla) + assert _supports_qwen_key_padding_mask("VANILLA", ulysses) + # Qwen's joint mask is [valid text | padded text | valid image], while + # FA4's key_padding_mask path assumes valid tokens form one prefix. + assert not _supports_qwen_key_padding_mask("FA4", vanilla) + assert not _supports_qwen_key_padding_mask("FA4", ulysses) + assert not _supports_qwen_key_padding_mask("TRTLLM", vanilla) + assert not _supports_qwen_key_padding_mask("VANILLA", attention_2d) + assert not _supports_qwen_key_padding_mask("VANILLA", ring) + assert _is_qwen_sequence_parallel_attention(attention_2d) + assert _is_qwen_sequence_parallel_attention(ring) + assert _is_qwen_sequence_parallel_attention(ulysses) + assert not _is_qwen_sequence_parallel_attention(vanilla) + + +def test_qwen_build_joint_attention_mask_appends_valid_image_tokens(): + hidden_states = torch.empty(2, 4, 8) + text_mask = torch.tensor([[1, 0, 1], [1, 1, 0]], dtype=torch.int64) + + joint_mask = _build_joint_attention_mask(text_mask, hidden_states) + + expected = torch.tensor( + [ + [True, False, True, True, True, True, True], + [True, True, False, True, True, True, True], + ], + dtype=torch.bool, + ) + torch.testing.assert_close(joint_mask, expected) + + +def test_qwen_build_joint_attention_mask_none_stays_none(): + assert _build_joint_attention_mask(None, torch.empty(2, 4, 8)) is None + + +def test_qwen_sequence_sharding_pads_streams_and_interleaves_rank_masks(): + class RankZeroSharder: + is_active = True + size = 2 + + def __init__(self, rank_one_mask): + self.rank_one_mask = rank_one_mask + + def shard(self, tensor, dim=1, pad_to_multiple=False): + if pad_to_multiple and tensor.shape[dim] % self.size: + pad_shape = list(tensor.shape) + pad_shape[dim] = self.size - tensor.shape[dim] % self.size + tensor = torch.cat([tensor, tensor.new_zeros(pad_shape)], dim=dim) + return tensor.narrow(dim, 0, tensor.shape[dim] // self.size).contiguous() + + def gather(self, tensor, dim=1, unpad_to=None): + assert unpad_to is None + return torch.cat([tensor, self.rank_one_mask], dim=dim) + + model = QwenImageTransformer2DModel( + model_config=DiffusionModelConfig(), + patch_size=1, + in_channels=4, + out_channels=4, + num_layers=1, + attention_head_dim=8, + num_attention_heads=2, + joint_attention_dim=16, + axes_dims_rope=(2, 2, 4), + ) + rank_one_mask = torch.tensor([[True, False, False, True, True, True, False]]) + model.sharder = RankZeroSharder(rank_one_mask) + hidden_states = torch.randn(1, 7, 16) + encoder_hidden_states = torch.randn(1, 5, 16) + encoder_mask = torch.tensor([[True, False, True, True, False]]) + image_rotary_emb = ( + torch.polar(torch.ones(7, 4), torch.randn(7, 4)), + torch.polar(torch.ones(5, 4), torch.randn(5, 4)), + ) + + hidden_shard, text_shard, rope_shard, joint_mask, image_seq_len = model._shard_sequences( + hidden_states, + encoder_hidden_states, + encoder_mask, + image_rotary_emb, + ) + + assert hidden_shard.shape == (1, 4, 16) + assert text_shard.shape == (1, 3, 16) + assert rope_shard[0].shape == (4, 4) + assert rope_shard[1].shape == (3, 4) + assert image_seq_len == 7 + torch.testing.assert_close( + joint_mask, + torch.tensor( + [ + [ + True, + False, + True, + True, + True, + True, + True, + True, + False, + False, + True, + True, + True, + False, + ] + ] + ), + ) + + +def test_qwen_sequence_sharding_rejects_padding_without_key_mask_support(): + class CachedBlocksWrapper(torch.nn.Module): + def __init__(self, transformer_blocks): + super().__init__() + self.transformer_blocks = transformer_blocks + + model = QwenImageTransformer2DModel( + model_config=DiffusionModelConfig(), + patch_size=1, + in_channels=4, + out_channels=4, + num_layers=1, + attention_head_dim=8, + num_attention_heads=2, + joint_attention_dim=16, + axes_dims_rope=(2, 2, 4), + ) + model.sharder = SimpleNamespace(is_active=True, size=2) + first_block_attn = model.transformer_blocks[0].attn + first_block_attn._supports_key_padding_mask = False + model.transformer_blocks = torch.nn.ModuleList([CachedBlocksWrapper(model.transformer_blocks)]) + + assert model._first_block_attn() is first_block_attn + + with pytest.raises(NotImplementedError, match="requires VANILLA Ulysses"): + model._shard_sequences( + torch.randn(1, 7, 16), + torch.randn(1, 4, 16), + None, + ( + torch.polar(torch.ones(7, 4), torch.randn(7, 4)), + torch.polar(torch.ones(4, 4), torch.randn(4, 4)), + ), + ) + + +def test_qwen_joint_attention_cpu_fallback_uses_unfused_qk_norm_rope(): + torch.manual_seed(0) + attention = ( + QwenJointAttention( + dim=16, + attention_head_dim=8, + num_attention_heads=2, + ) + .to(torch.bfloat16) + .eval() + ) + hidden_states = torch.randn(1, 4, 16, dtype=torch.bfloat16) + encoder_hidden_states = torch.randn(1, 5, 16, dtype=torch.bfloat16) + img_phases = torch.randn(4, 4) + txt_phases = torch.randn(5, 4) + image_rotary_emb = ( + torch.polar(torch.ones_like(img_phases), img_phases), + torch.polar(torch.ones_like(txt_phases), txt_phases), + ) + + assert attention.fuse_qk_norm_rope + assert not attention._use_fused_qk_norm_rope(hidden_states, image_rotary_emb) + q, k, v = attention._prepare_qkv( + hidden_states, + encoder_hidden_states, + image_rotary_emb, + ) + + assert q.shape == k.shape == v.shape == (1, 9, 16) + + +def test_qwen_transformer_block_modulation_helpers(): + x = torch.tensor([[[1.0, -2.0], [3.0, -4.0]]]) + mod_params = torch.tensor([[0.5, -1.0, 2.0, -0.5, 0.25, 0.75]]) + + modulated, gate = QwenImageTransformerBlock._modulate(x, mod_params) + + expected_modulated = x * torch.tensor([[[3.0, 0.5]]]) + torch.tensor([[[0.5, -1.0]]]) + expected_gate = torch.tensor([[[0.25, 0.75]]]) + torch.testing.assert_close(modulated, expected_modulated) + torch.testing.assert_close(gate, expected_gate) + + +def test_qwen_transformer_block_gate_residual_helper(): + hidden_states = torch.tensor([[[1.0, 2.0], [3.0, 4.0]]]) + gate = torch.tensor([[[0.25, 0.5]]]) + residual = torch.tensor([[[8.0, 6.0], [4.0, 2.0]]]) + + output = QwenImageTransformerBlock._apply_gate_residual(hidden_states, gate, residual) + + torch.testing.assert_close(output, hidden_states + gate * residual) + + +def test_qwen_feedforward_uses_shared_gelu_tanh(): + assert _get_feedforward_activation("gelu-approximate") is gelu_tanh + assert _get_feedforward_activation("gelu") is F.gelu + + with pytest.raises(ValueError, match="Unsupported activation_fn=relu"): + _get_feedforward_activation("relu") + + +def test_qwen_feedforward_tp2_reduces_row_parallel_output(): + feedforward = FeedForward( + dim=16, + config=DiffusionModelConfig(mapping=Mapping(world_size=2, rank=0, tp_size=2)), + ) + + assert feedforward.down_proj.tp_mode == TensorParallelMode.ROW + assert feedforward.down_proj.reduce_output + + +def _qwen_image_edit_pipeline_for_cfg_compile_test(cfg_size: int) -> Any: + from tensorrt_llm._torch.visual_gen.models.qwen_image import QwenImageEditPlusPipeline + + pipeline = QwenImageEditPlusPipeline.__new__(QwenImageEditPlusPipeline) + pipeline.pipeline_config = SimpleNamespace( + visual_gen_mapping=SimpleNamespace(cfg_size=cfg_size), + ) + return pipeline + + +def test_qwen_image_edit_cfg_parallel_keeps_torch_compile(monkeypatch: pytest.MonkeyPatch) -> None: + from tensorrt_llm._torch.visual_gen.pipeline import BasePipeline + + called = False + + def record_call(self: object) -> None: + nonlocal called + called = True + + monkeypatch.setattr(BasePipeline, "torch_compile", record_call) + pipeline = _qwen_image_edit_pipeline_for_cfg_compile_test(cfg_size=2) + + pipeline.torch_compile() + + assert called + + +def test_qwen_image_edit_cfg1_keeps_torch_compile(monkeypatch: pytest.MonkeyPatch) -> None: + from tensorrt_llm._torch.visual_gen.pipeline import BasePipeline + + called = False + + def record_call(self: object) -> None: + nonlocal called + called = True + + monkeypatch.setattr(BasePipeline, "torch_compile", record_call) + pipeline = _qwen_image_edit_pipeline_for_cfg_compile_test(cfg_size=1) + + pipeline.torch_compile() + + assert called + + +def test_qwen_image_edit_rejects_multiple_images_per_prompt() -> None: + from tensorrt_llm._torch.visual_gen.models.qwen_image import QwenImageEditPlusPipeline + + pipeline = QwenImageEditPlusPipeline.__new__(QwenImageEditPlusPipeline) + req = SimpleNamespace( + prompt="edit prompt", + params=SimpleNamespace(num_images_per_prompt=2), + ) + + with pytest.raises(ValueError, match="num_images_per_prompt=1 only"): + pipeline.infer(req) diff --git a/tests/unittest/_torch/visual_gen/test_qwen_image_registry.py b/tests/unittest/_torch/visual_gen/test_qwen_image_registry.py index 93c85c8ecc08..7e23ecb93aa3 100644 --- a/tests/unittest/_torch/visual_gen/test_qwen_image_registry.py +++ b/tests/unittest/_torch/visual_gen/test_qwen_image_registry.py @@ -9,6 +9,8 @@ """ import json +from pathlib import Path +from types import SimpleNamespace import pytest import torch @@ -26,6 +28,7 @@ from tensorrt_llm._torch.visual_gen import models # noqa: F401 from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig from tensorrt_llm._torch.visual_gen.models.qwen_image import ( + QwenImageEditPlusPipeline, QwenImagePipeline, QwenImageTransformer2DModel, ) @@ -80,6 +83,21 @@ def test_auto_pipeline_detects_qwen_image_class_name(tmp_path): assert AutoPipeline._detect_from_checkpoint(str(tmp_path)) == "QwenImagePipeline" +def test_qwen_image_edit_plus_pipeline_is_registered() -> None: + """Qwen-Image-Edit checkpoints should route to the edit-plus pipeline.""" + assert "QwenImageEditPlusPipeline" in PIPELINE_REGISTRY + assert PIPELINE_REGISTRY["QwenImageEditPlusPipeline"].pipeline_cls is QwenImageEditPlusPipeline + assert "Qwen/Qwen-Image-Edit-2511" in PIPELINE_REGISTRY["QwenImageEditPlusPipeline"].hf_ids + + +def test_auto_pipeline_detects_qwen_image_edit_plus_class_name(tmp_path: Path) -> None: + """model_index.json with _class_name=QwenImageEditPlusPipeline resolves.""" + (tmp_path / "model_index.json").write_text( + json.dumps({"_class_name": "QwenImageEditPlusPipeline"}) + ) + assert AutoPipeline._detect_from_checkpoint(str(tmp_path)) == "QwenImageEditPlusPipeline" + + def test_transformer_constructs_with_defaults(): """The full transformer instantiates with the documented defaults. @@ -107,6 +125,74 @@ def test_transformer_load_weights_detects_mismatch(): model.load_weights({}) +def test_qwen_image_defaults_missing_negative_prompt_to_empty_string(): + assert QwenImagePipeline._normalize_negative_prompt(None, 2) == ["", ""] + assert QwenImagePipeline._normalize_negative_prompt("low quality", 2) == [ + "low quality", + "low quality", + ] + assert QwenImagePipeline._normalize_negative_prompt(["blur"], 2) == ["blur", "blur"] + assert QwenImagePipeline._normalize_negative_prompt(["a", "b"], 2) == ["a", "b"] + with pytest.raises(ValueError, match="negative_prompt"): + QwenImagePipeline._normalize_negative_prompt(["a", "b", "c"], 2) + + +def test_qwen_image_cfg_parallel_state(monkeypatch): + pipeline = object.__new__(QwenImagePipeline) + pipeline.pipeline_config = SimpleNamespace( + visual_gen_mapping=SimpleNamespace(cfg_size=2, cfg_rank=1, cfg_group="cfg_pg") + ) + monkeypatch.setattr( + "tensorrt_llm._torch.visual_gen.models.qwen_image.pipeline_qwen_image.dist.is_initialized", + lambda: True, + ) + + assert pipeline._cfg_parallel_state(True) == (True, 2, 1, "cfg_pg") + assert pipeline._cfg_parallel_state(False) == (False, 2, 1, "cfg_pg") + + pipeline.pipeline_config = SimpleNamespace(visual_gen_mapping=None) + assert pipeline._cfg_parallel_state(True) == (False, 1, 0, None) + + pipeline.pipeline_config = SimpleNamespace( + visual_gen_mapping=SimpleNamespace(cfg_size=3, cfg_rank=0, cfg_group="cfg_pg") + ) + with pytest.raises(ValueError, match="cfg_size=3"): + pipeline._cfg_parallel_state(True) + + pipeline.pipeline_config = SimpleNamespace( + visual_gen_mapping=SimpleNamespace(cfg_size=2, cfg_rank=0, cfg_group="cfg_pg") + ) + monkeypatch.setattr( + "tensorrt_llm._torch.visual_gen.models.qwen_image.pipeline_qwen_image.dist.is_initialized", + lambda: False, + ) + with pytest.raises(RuntimeError, match="torch.distributed"): + pipeline._cfg_parallel_state(True) + + +def test_qwen_image_selects_cfg_inputs_by_rank(): + prompt = torch.tensor([1]) + prompt_mask = torch.tensor([2]) + negative = torch.tensor([3]) + negative_mask = torch.tensor([4]) + + selected = QwenImagePipeline._select_cfg_inputs(0, prompt, prompt_mask, negative, negative_mask) + assert selected[0] is prompt + assert selected[1] is prompt_mask + + selected = QwenImagePipeline._select_cfg_inputs(1, prompt, prompt_mask, negative, negative_mask) + assert selected[0] is negative + assert selected[1] is negative_mask + + +def test_qwen_image_negative_prompt_cfg_combination_keeps_reference_formula(): + noise_pred = torch.tensor([[[2.0, 0.0]]]) + neg_noise_pred = torch.tensor([[[1.0, 0.0]]]) + combined = QwenImagePipeline._combine_negative_prompt_cfg(noise_pred, neg_noise_pred, 4.0) + assert combined.shape == noise_pred.shape + assert torch.allclose(combined, torch.tensor([[[2.0, 0.0]]])) + + def test_transformer_applies_quant_config_ignore_list() -> None: """Qwen-Image should honor selective dynamic quantization exclusions.""" model_config = DiffusionModelConfig( diff --git a/tests/unittest/_torch/visual_gen/test_teacache.py b/tests/unittest/_torch/visual_gen/test_teacache.py index c53f77aaa135..aeb36cbb5377 100644 --- a/tests/unittest/_torch/visual_gen/test_teacache.py +++ b/tests/unittest/_torch/visual_gen/test_teacache.py @@ -19,8 +19,13 @@ from unittest.mock import MagicMock, patch import pytest +import torch -from tensorrt_llm._torch.visual_gen.cache.teacache import TeaCacheBackend +from tensorrt_llm._torch.visual_gen.cache.teacache import ( + ExtractorConfig, + TeaCacheBackend, + register_extractor_from_config, +) from tensorrt_llm._torch.visual_gen.config import DiffusionModelConfig from tensorrt_llm._torch.visual_gen.pipeline import BasePipeline from tensorrt_llm.visual_gen.args import TeaCacheConfig @@ -301,6 +306,107 @@ def test_no_backends_is_noop(self): acc.refresh(10) # should not raise +class _TupleTransformer(torch.nn.Module): + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.Tensor, + return_dict: bool = False, + ) -> tuple[torch.Tensor]: + assert not return_dict + return (hidden_states + 1,) + + +def _identity_timestep_embedding( + _module: torch.nn.Module, + timestep: torch.Tensor, + **_kwargs, +) -> torch.Tensor: + return timestep + + +def test_teacache_preserves_tuple_output_on_cache_miss_and_hit() -> None: + transformer = _TupleTransformer() + register_extractor_from_config( + ExtractorConfig( + model_class_name=transformer.__class__.__name__, + timestep_embed_fn=_identity_timestep_embedding, + forward_params=["hidden_states", "timestep", "return_dict"], + return_dict_default=False, + return_tuple_when_return_dict_false=True, + ) + ) + backend = TeaCacheBackend( + TeaCacheConfig( + coefficients=[0.0, 0.0], + teacache_thresh=0.2, + use_ret_steps=False, + ) + ) + backend.enable(transformer) + backend.refresh(num_inference_steps=4) + + hidden_states = torch.zeros(1, 8, 4) + timestep = torch.ones(1, 4) + try: + cache_miss = transformer(hidden_states, timestep, return_dict=False) + cache_hit = transformer(hidden_states, timestep, return_dict=False) + cache_stats = backend.get_stats() + finally: + backend.disable(transformer) + + assert isinstance(cache_miss, tuple) + assert isinstance(cache_hit, tuple) + torch.testing.assert_close(cache_miss[0], hidden_states + 1) + torch.testing.assert_close(cache_hit[0], hidden_states + 1) + assert cache_stats["cached"] == 1 + + +class _TensorTransformer(torch.nn.Module): + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + return hidden_states + timestep[:, :1].unsqueeze(-1) + + +def test_teacache_preserves_tensor_output_on_cache_miss_and_hit() -> None: + transformer = _TensorTransformer() + register_extractor_from_config( + ExtractorConfig( + model_class_name=transformer.__class__.__name__, + timestep_embed_fn=_identity_timestep_embedding, + forward_params=["hidden_states", "timestep"], + return_dict_default=False, + ) + ) + backend = TeaCacheBackend( + TeaCacheConfig( + coefficients=[0.0, 0.0], + teacache_thresh=0.2, + use_ret_steps=False, + ) + ) + backend.enable(transformer) + backend.refresh(num_inference_steps=4) + + hidden_states = torch.zeros(1, 8, 4) + timestep = torch.ones(1, 4) + try: + cache_miss = transformer(hidden_states, timestep) + cache_hit = transformer(hidden_states, timestep) + cache_stats = backend.get_stats() + finally: + backend.disable(transformer) + + assert isinstance(cache_miss, torch.Tensor) + assert isinstance(cache_hit, torch.Tensor) + torch.testing.assert_close(cache_miss, hidden_states + 1) + torch.testing.assert_close(cache_hit, hidden_states + 1) + assert cache_stats["cached"] == 1 + + class TestFlux2TeacacheTable: """FLUX.2 built-in coefficient table (dev variant).""" @@ -362,6 +468,7 @@ def test_ltx2_succeeds_with_explicit_coefficients(self): ): with patch.object(TeaCacheBackend, "enable"): LTX2Pipeline.post_load_weights(pipe) + BasePipeline._setup_cache_acceleration(pipe) assert pipe.cache_accelerator is not None def test_wan22_raises_when_teacache_enabled_without_both_coefficient_lists(self): @@ -449,6 +556,7 @@ def test_wan22_t2v_installs_two_teacache_backends_when_coefficients_provided(sel ) as TB: TB.side_effect = [backend_a, backend_b] WanPipeline.post_load_weights(pipe) + BasePipeline._setup_cache_acceleration(pipe) assert TB.call_count == 2 assert mock_enable.call_count == 2 assert pipe.cache_accelerator is not None @@ -480,6 +588,7 @@ def test_wan22_t2v_transformer_gets_coefficients_and_transformer_2_gets_coeffici ) as TB: TB.return_value = MagicMock() WanPipeline.post_load_weights(pipe) + BasePipeline._setup_cache_acceleration(pipe) assert TB.call_count == 2 cfg_high = TB.call_args_list[0][0][0] @@ -515,6 +624,7 @@ def test_wan22_i2v_transformer_gets_coefficients_and_transformer_2_gets_coeffici ) as TB: TB.return_value = MagicMock() WanImageToVideoPipeline.post_load_weights(pipe) + BasePipeline._setup_cache_acceleration(pipe) assert TB.call_count == 2 cfg_high = TB.call_args_list[0][0][0] diff --git a/tests/unittest/_torch/visual_gen/test_utils.py b/tests/unittest/_torch/visual_gen/test_utils.py index bffe0d5ad9a3..384196e43e59 100644 --- a/tests/unittest/_torch/visual_gen/test_utils.py +++ b/tests/unittest/_torch/visual_gen/test_utils.py @@ -62,16 +62,6 @@ def test_shard_rope_passthrough_when_inactive(self): rope = (cos, cos) assert s.shard_rope(rope, seq_len=4, seq_dim=1) is rope - def test_disable_enable_no_collectives(self): - s = SequenceSharder(size=4, rank=0, group=None) - assert s.is_active - s.disable() - assert not s.is_active - x = torch.randn(1, 8, 2) - assert s.shard(x, dim=1) is x - s.enable() - assert s.is_active - class TestSequenceSharderShardSlices: """Active sharder: block slice math without ``gather``.""" diff --git a/tests/unittest/_torch/visual_gen/test_visual_gen_args.py b/tests/unittest/_torch/visual_gen/test_visual_gen_args.py index 5ee5c004aa9d..13532eaa3458 100644 --- a/tests/unittest/_torch/visual_gen/test_visual_gen_args.py +++ b/tests/unittest/_torch/visual_gen/test_visual_gen_args.py @@ -120,6 +120,48 @@ def test_supported_quant_config_cute(self): assert attention.quant_attention_config is not None + @pytest.mark.parametrize("v_block_size", [0, 1]) + def test_supported_quant_config_cute_mxfp8(self, v_block_size): + attention = AttentionConfig( + backend="CUTEDSL", + quant_attention_config=QuantAttentionConfig( + qk_dtype="mxfp8", + v_dtype="fp8", + v_block_size=v_block_size, + ), + ) + + assert attention.quant_attention_config is not None + assert attention.quant_attention_config.v_block_size == v_block_size + + @pytest.mark.parametrize("v_block_size", [0, 1]) + def test_supported_quant_config_cute_nvfp4(self, v_block_size): + attention = AttentionConfig( + backend="CUTEDSL", + quant_attention_config=QuantAttentionConfig( + qk_dtype="nvfp4", + v_dtype="fp8", + v_block_size=v_block_size, + ), + ) + + assert attention.quant_attention_config is not None + assert attention.quant_attention_config.v_block_size == v_block_size + + def test_blockscaled_qk_dtype_rejected_on_trtllm(self): + with pytest.raises(ValidationError, match="Unsupported quant_attention_config"): + AttentionConfig( + backend="TRTLLM", + quant_attention_config=QuantAttentionConfig(qk_dtype="nvfp4"), + ) + + def test_sage_qk_block_size_rejected_on_cute(self): + with pytest.raises(ValidationError, match="Unsupported quant_attention_config"): + AttentionConfig( + backend="CUTEDSL", + quant_attention_config=QuantAttentionConfig(qk_dtype="mxfp8", q_block_size=1), + ) + class TestPipelineRegistryUnique: """Guard against duplicate HF IDs across PIPELINE_REGISTRY entries. diff --git a/tests/unittest/_torch/visual_gen/test_visual_gen_params.py b/tests/unittest/_torch/visual_gen/test_visual_gen_params.py index e69c81d21f1b..75930e827a7f 100644 --- a/tests/unittest/_torch/visual_gen/test_visual_gen_params.py +++ b/tests/unittest/_torch/visual_gen/test_visual_gen_params.py @@ -375,6 +375,31 @@ def test_user_values_not_overwritten(self): assert req.params.width == 1920 assert req.params.num_inference_steps == 50 # Default filled + def test_flux2_reference_dimensions_remain_unset_for_pipeline_resolution(self): + from tensorrt_llm._torch.visual_gen.models.flux.pipeline_flux2 import Flux2Pipeline + + executor = self._make_mock_executor(Flux2Pipeline) + executor.pipeline.derive_output_size_from_reference = True + req = self._make_request(image=b"encoded image") + + self._merge(executor, req) + + assert req.params.height is None + assert req.params.width is None + assert req.params.num_inference_steps == 50 + + def test_flux2_reference_dimensions_preserve_explicit_values(self): + from tensorrt_llm._torch.visual_gen.models.flux.pipeline_flux2 import Flux2Pipeline + + executor = self._make_mock_executor(Flux2Pipeline) + executor.pipeline.derive_output_size_from_reference = True + req = self._make_request(image=b"encoded image", height=768, width=512) + + self._merge(executor, req) + + assert req.params.height == 768 + assert req.params.width == 512 + def test_extra_params_defaults_merged(self): from tensorrt_llm._torch.visual_gen.models.ltx2.pipeline_ltx2 import LTX2Pipeline @@ -1160,7 +1185,7 @@ def test_runtime_error_carried_on_response(self): executor = self._make_executor(FluxPipeline) executor._merge_defaults = lambda req: DiffusionExecutor._merge_defaults(executor, req) - executor.pipeline.warmup_cache_key = MagicMock(return_value=(1024, 1024, None)) + executor.pipeline.request_warmup_cache_key = MagicMock(return_value=(1024, 1024, None)) executor.pipeline._warmed_up_shapes = None executor.pipeline.infer = MagicMock(side_effect=RuntimeError("oops")) @@ -1176,3 +1201,41 @@ def test_runtime_error_carried_on_response(self): resp = executor.response_queue.put.call_args[0][0] assert isinstance(resp, DiffusionResponse) assert resp.error_msg == "oops" + + def test_reference_size_is_prepared_before_warmup_lookup(self): + from tensorrt_llm._torch.visual_gen.executor import DiffusionExecutor, DiffusionRequest + from tensorrt_llm._torch.visual_gen.models.flux.pipeline_flux2 import Flux2Pipeline + from tensorrt_llm.visual_gen.params import VisualGenParams + + events = [] + executor = self._make_executor(Flux2Pipeline) + executor.rank = 1 + executor._merge_defaults = lambda req: DiffusionExecutor._merge_defaults(executor, req) + executor.pipeline.derive_output_size_from_reference = True + + def prepare_request(req): + events.append("prepare") + req.params.height = 64 + req.params.width = 80 + + def request_warmup_cache_key(req): + events.append("warmup_cache_key") + return (req.params.height, req.params.width) + + executor.pipeline.prepare_request = MagicMock(side_effect=prepare_request) + executor.pipeline.request_warmup_cache_key = MagicMock(side_effect=request_warmup_cache_key) + executor.pipeline._warmed_up_shapes = {(1024, 1024)} + executor.pipeline.infer = MagicMock( + side_effect=lambda _req: events.append("infer") or MagicMock() + ) + req = DiffusionRequest( + request_id=8, + prompt=["test"], + params=VisualGenParams(image=b"encoded image"), + ) + + DiffusionExecutor.process_request(executor, req) + + assert events == ["prepare", "warmup_cache_key", "infer"] + executor.pipeline.request_warmup_cache_key.assert_called_once_with(req) + executor.pipeline.infer.assert_called_once_with(req) diff --git a/tests/unittest/_torch/visual_gen/test_warmup.py b/tests/unittest/_torch/visual_gen/test_warmup.py index c882da486f70..ddf0c84092bf 100644 --- a/tests/unittest/_torch/visual_gen/test_warmup.py +++ b/tests/unittest/_torch/visual_gen/test_warmup.py @@ -3,6 +3,7 @@ """Tests for VisualGen warmup configuration, plan resolution, and shape validation.""" +import contextlib from unittest.mock import MagicMock import pytest @@ -104,6 +105,8 @@ def __init__(self, warmup_cfg): self._warmed_up_shapes = set() self.pipeline_config = MagicMock() self.pipeline_config.compilation = warmup_cfg or CompilationConfig() + # warmup() branches on this; keep the stub on the plain (no-autotune) path. + self.pipeline_config.torch_compile.enable_autotune = False def forward(self, *args, **kwargs): pass @@ -287,6 +290,66 @@ def test_warmup_user_shapes_recorded(self): assert pipe._warmed_up_shapes == {(480, 832, 33)} +class TestWarmupPhasing: + """warmup() tune/merge/capture phasing: number of warmup passes per config. + + Fully mocked (autotune, communicator, dist, the pass itself) so it exercises + the branch logic without a GPU or a real process group. + """ + + def _count_passes(self, monkeypatch, *, enable_autotune, world_size, cuda_graph_enable): + import tensorrt_llm._torch.visual_gen.pipeline as pmod + + pipe = _make_stub_pipeline(CompilationConfig(resolutions=[(480, 832)], num_frames=[33])) + pipe.pipeline_config.torch_compile.enable_autotune = enable_autotune + pipe.pipeline_config.cuda_graph.enable = cuda_graph_enable + + passes = {"n": 0} + monkeypatch.setattr( + pipe, "_run_warmup_pass", lambda shapes, steps: passes.__setitem__("n", passes["n"] + 1) + ) + monkeypatch.setattr(pipe, "disallow_cuda_graph_capture", lambda: contextlib.nullcontext()) + monkeypatch.setattr(pmod, "autotune", lambda **kw: contextlib.nullcontext()) + monkeypatch.setattr(pmod, "_VisualGenAutotuneDist", lambda amap: object()) + monkeypatch.setattr(pmod.dist, "is_initialized", lambda: world_size > 1) + monkeypatch.setattr(pmod.dist, "get_world_size", lambda: world_size) + + pipe.warmup() + return passes["n"] + + def test_no_autotune_one_pass(self, monkeypatch): + assert ( + self._count_passes( + monkeypatch, enable_autotune=False, world_size=1, cuda_graph_enable=True + ) + == 1 + ) + + def test_single_rank_one_pass(self, monkeypatch): + assert ( + self._count_passes( + monkeypatch, enable_autotune=True, world_size=1, cuda_graph_enable=True + ) + == 1 + ) + + def test_multi_rank_two_pass(self, monkeypatch): + assert ( + self._count_passes( + monkeypatch, enable_autotune=True, world_size=2, cuda_graph_enable=True + ) + == 2 + ) + + def test_multi_rank_graph_off_one_pass(self, monkeypatch): + assert ( + self._count_passes( + monkeypatch, enable_autotune=True, world_size=2, cuda_graph_enable=False + ) + == 1 + ) + + class TestRequestValidation: """Request-level validation: validate_resolution() + _warmed_up_shapes check.""" diff --git a/tests/unittest/_torch/weight_sharing/_source_identity_fakes.py b/tests/unittest/_torch/weight_sharing/_source_identity_fakes.py index c68b10506b7a..ff567b742cec 100644 --- a/tests/unittest/_torch/weight_sharing/_source_identity_fakes.py +++ b/tests/unittest/_torch/weight_sharing/_source_identity_fakes.py @@ -19,9 +19,14 @@ `test_source_identity.py` and `test_mx_source_identity_gate.py`. """ +import hashlib from typing import Optional, Sequence -from tensorrt_llm._torch.weight_sharing import SourceIdentity +from tensorrt_llm._torch.weight_sharing import ( + ARTIFACT_IDENTITY_FORMAT_VERSION, + ArtifactIdentity, + SourceIdentity, +) _UNSET = object() @@ -204,16 +209,37 @@ def named_buffers(self): return list(self._buffers.items()) -def identity_from(config: FakeModelConfig, *, model_name: Optional[str] = None) -> SourceIdentity: +def make_artifact_identity(key: str = "same") -> ArtifactIdentity: + """Build a deterministic local-checkpoint identity for tests.""" + return ArtifactIdentity( + format_version=ARTIFACT_IDENTITY_FORMAT_VERSION, + scheme="checkpoint_manifest_sha256", + digest=hashlib.sha256(key.encode("utf-8")).hexdigest(), + ) + + +def identity_from( + config: FakeModelConfig, + *, + model_name: Optional[str] = None, + artifact_key: str = "same", +) -> SourceIdentity: """Build a :class:`SourceIdentity` from a fake config and derived model.""" return SourceIdentity.from_model_config( - config, FakeModel(config.pretrained_config), model_name=model_name + config, + FakeModel(config.pretrained_config), + artifact_identity=make_artifact_identity(artifact_key), + model_name=model_name, ) def make_identity( - *, attn_backend: str = "TRTLLM", rank: int = 0, model_name: str = "m" + *, + attn_backend: str = "TRTLLM", + rank: int = 0, + model_name: str = "m", + artifact_key: str = "same", ) -> SourceIdentity: """Build a :class:`SourceIdentity` from a fake config for `rank`.""" cfg = FakeModelConfig(mapping=FakeMapping(rank=rank, tp_rank=rank), attn_backend=attn_backend) - return identity_from(cfg, model_name=model_name) + return identity_from(cfg, model_name=model_name, artifact_key=artifact_key) diff --git a/tests/unittest/_torch/weight_sharing/test_artifact_identity.py b/tests/unittest/_torch/weight_sharing/test_artifact_identity.py new file mode 100644 index 000000000000..30526ce77c66 --- /dev/null +++ b/tests/unittest/_torch/weight_sharing/test_artifact_identity.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for immutable checkpoint artifact identities.""" + +from pathlib import Path + +import pytest + +from tensorrt_llm._torch.weight_sharing import ArtifactIdentity + + +def _write_checkpoint(path: Path, weights: bytes = b"weights") -> None: + path.mkdir(parents=True) + (path / "config.json").write_text('{"architectures":["LlamaForCausalLM"]}') + (path / "model.safetensors").write_bytes(weights) + + +def test_local_checkpoint_identity_is_path_independent(tmp_path: Path) -> None: + left = tmp_path / "left" / "checkpoint" + right = tmp_path / "right" / "checkpoint" + _write_checkpoint(left) + _write_checkpoint(right) + + assert ArtifactIdentity.from_checkpoint(left) == ArtifactIdentity.from_checkpoint(right) + + +def test_local_checkpoint_identity_binds_file_contents(tmp_path: Path) -> None: + left = tmp_path / "left" + right = tmp_path / "right" + _write_checkpoint(left, weights=b"fine-tune-a") + _write_checkpoint(right, weights=b"fine-tune-b") + + assert ArtifactIdentity.from_checkpoint(left) != ArtifactIdentity.from_checkpoint(right) + + +def test_local_checkpoint_identity_ignores_cache_and_scm_metadata(tmp_path: Path) -> None: + left = tmp_path / "left" + right = tmp_path / "right" + _write_checkpoint(left) + _write_checkpoint(right) + (left / ".cache").mkdir() + (left / ".cache" / "download.lock").write_text("transient") + (right / ".git").mkdir() + (right / ".git" / "HEAD").write_text("ref: refs/heads/main") + + assert ArtifactIdentity.from_checkpoint(left) == ArtifactIdentity.from_checkpoint(right) + + +def test_local_checkpoint_identity_rejects_nested_directory_symlink(tmp_path: Path) -> None: + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + external_weights = tmp_path / "external-weights" + external_weights.mkdir() + (external_weights / "model.safetensors").write_bytes(b"weights") + (checkpoint / "weights").symlink_to(external_weights, target_is_directory=True) + + with pytest.raises(ValueError, match="nested symlinked directories"): + ArtifactIdentity.from_checkpoint(checkpoint) + + +def test_hf_snapshot_identity_binds_revision_across_cache_roots(tmp_path: Path) -> None: + revision = "a" * 40 + left = tmp_path / "cache-a" / "models--org--model" / "snapshots" / revision + right = tmp_path / "cache-b" / "models--org--model" / "snapshots" / revision + left.mkdir(parents=True) + right.mkdir(parents=True) + + left_identity = ArtifactIdentity.from_checkpoint(left) + right_identity = ArtifactIdentity.from_checkpoint(right) + assert left_identity == right_identity + assert left_identity.scheme == "hf_snapshot_revision" + + +def test_hf_snapshot_identity_binds_revision_and_subpath(tmp_path: Path) -> None: + snapshot = tmp_path / "models--org--model" / "snapshots" + revision_a = snapshot / ("a" * 40) + revision_b = snapshot / ("b" * 40) + (revision_a / "variant-a").mkdir(parents=True) + (revision_a / "variant-b").mkdir() + revision_b.mkdir(parents=True) + + root_identity = ArtifactIdentity.from_checkpoint(revision_a) + assert root_identity != ArtifactIdentity.from_checkpoint(revision_b) + assert root_identity != ArtifactIdentity.from_checkpoint(revision_a / "variant-a") + assert ArtifactIdentity.from_checkpoint( + revision_a / "variant-a" + ) != ArtifactIdentity.from_checkpoint(revision_a / "variant-b") + + +def test_serialization_roundtrip(tmp_path: Path) -> None: + checkpoint = tmp_path / "checkpoint" + _write_checkpoint(checkpoint) + identity = ArtifactIdentity.from_checkpoint(checkpoint) + + assert ArtifactIdentity.from_dict(identity.to_dict()) == identity + + +def test_rejects_unknown_format_version(tmp_path: Path) -> None: + checkpoint = tmp_path / "checkpoint" + _write_checkpoint(checkpoint) + payload = ArtifactIdentity.from_checkpoint(checkpoint).to_dict() + payload["format_version"] += 1 + + with pytest.raises(ValueError, match="Unsupported ArtifactIdentity format version"): + ArtifactIdentity.from_dict(payload) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("scheme", "unknown", "Unsupported ArtifactIdentity scheme"), + ("scheme", [], "scheme must be a string"), + ("digest", "not-a-digest", "64-character hex value"), + ("digest", 1, "digest must be a string"), + ], +) +def test_rejects_invalid_serialized_fields( + tmp_path: Path, field: str, value: object, message: str +) -> None: + checkpoint = tmp_path / "checkpoint" + _write_checkpoint(checkpoint) + payload = ArtifactIdentity.from_checkpoint(checkpoint).to_dict() + payload[field] = value + + with pytest.raises(ValueError, match=message): + ArtifactIdentity.from_dict(payload) + + +@pytest.mark.parametrize("version", [True, "1"]) +def test_rejects_non_integer_format_version(tmp_path: Path, version: object) -> None: + checkpoint = tmp_path / "checkpoint" + _write_checkpoint(checkpoint) + payload = ArtifactIdentity.from_checkpoint(checkpoint).to_dict() + payload["format_version"] = version + + with pytest.raises(ValueError, match="format version must be an integer"): + ArtifactIdentity.from_dict(payload) + + +def test_rejects_missing_or_empty_checkpoint(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + ArtifactIdentity.from_checkpoint(tmp_path / "missing") + + empty = tmp_path / "empty" + empty.mkdir() + with pytest.raises(ValueError, match="contains no files"): + ArtifactIdentity.from_checkpoint(empty) diff --git a/tests/unittest/_torch/weight_sharing/test_gms_source_identity_gate.py b/tests/unittest/_torch/weight_sharing/test_gms_source_identity_gate.py index f6689bfd0caa..99f21b4bd9b9 100644 --- a/tests/unittest/_torch/weight_sharing/test_gms_source_identity_gate.py +++ b/tests/unittest/_torch/weight_sharing/test_gms_source_identity_gate.py @@ -79,6 +79,14 @@ def test_gate_raises_on_mismatch(): loader._check_gms_source_identity(_FakeGMSBackend(writer)) +def test_gate_raises_on_checkpoint_artifact_mismatch(): + local = _identity(artifact_key="fine-tune-a") + writer = _identity(artifact_key="fine-tune-b") + loader = _new_loader(local) + with pytest.raises(SourceIdentityMismatchError): + loader._check_gms_source_identity(_FakeGMSBackend(writer)) + + def test_gate_raises_when_writer_identity_unavailable(): # Publisher metadata not wired yet (get_source_identity returns None); # GMS has no disk fallback, so unverified sharing must raise. diff --git a/tests/unittest/_torch/weight_sharing/test_mx_source_identity_gate.py b/tests/unittest/_torch/weight_sharing/test_mx_source_identity_gate.py index 4eb1b566df56..e2658ca86d21 100644 --- a/tests/unittest/_torch/weight_sharing/test_mx_source_identity_gate.py +++ b/tests/unittest/_torch/weight_sharing/test_mx_source_identity_gate.py @@ -14,12 +14,9 @@ # limitations under the License. """Exercise the real MX checkpoint loader pre-transfer SourceIdentity gate. -These tests drive `MXCheckpointLoader._source_identity_compatible` directly — -the single decision point the MX `load_weights` path consults before starting -a P2P transfer. Upstream `modelexpress` is never imported: the discovery -client / identity builder are passed as stubs, and the publisher-identity fetch -seam is patched, so the gate logic runs against real `SourceIdentity` objects -without any model, GPU, or RDMA. +Upstream `modelexpress` is never imported. The tests run the compatibility +decision against real `SourceIdentity` objects and use a small discovery +client stub for the pinned ModelExpress 0.4.1 API shape. """ from types import SimpleNamespace @@ -33,50 +30,51 @@ ) -def _new_loader(local_identity, source_identity, fetched=True): - """Construct a loader bypassing heavy base __init__, wire the seams.""" +def _new_loader(local_identity): + """Construct a loader while bypassing the heavy base initializer.""" loader = MXCheckpointLoader.__new__(MXCheckpointLoader) loader._local_source_identity = local_identity - # Patch the single fetch seam to return the publisher identity (or None). - loader._fetch_source_identity = lambda *a, **k: source_identity if fetched else None return loader -# MxClient / build_identity are only forwarded to the (patched) fetch seam. -_STUB_CLIENT = object() -_STUB_BUILD = object() - - def test_gate_proceeds_on_matching_identity(): local = _identity(attn_backend="TRTLLM") source = _identity(attn_backend="TRTLLM") - loader = _new_loader(local, source) - assert loader._source_identity_compatible("ckpt", _STUB_CLIENT, _STUB_BUILD) is True + loader = _new_loader(local) + assert loader._source_metadata_identity_compatible(_build_mx_source_metadata(source)) is True def test_gate_falls_back_on_mismatch(): local = _identity(attn_backend="TRTLLM") source = _identity(attn_backend="FLASHINFER") - loader = _new_loader(local, source) - assert loader._source_identity_compatible("ckpt", _STUB_CLIENT, _STUB_BUILD) is False + loader = _new_loader(local) + assert loader._source_metadata_identity_compatible(_build_mx_source_metadata(source)) is False + + +def test_gate_falls_back_on_checkpoint_artifact_mismatch(): + local = _identity(artifact_key="fine-tune-a") + source = _identity(artifact_key="fine-tune-b") + loader = _new_loader(local) + assert loader._source_metadata_identity_compatible(_build_mx_source_metadata(source)) is False def test_gate_falls_back_when_no_local_identity(): # MX must not consume shared weights unless the receiver identity exists. - loader = _new_loader(None, _identity()) - assert loader._source_identity_compatible("ckpt", _STUB_CLIENT, _STUB_BUILD) is False + loader = _new_loader(None) + assert ( + loader._source_metadata_identity_compatible(_build_mx_source_metadata(_identity())) is False + ) def test_gate_falls_back_when_source_identity_unavailable(): - # Publisher identity not yet fetchable (upstream metadata channel pending); - # reject P2P and fall back to disk rather than sharing unverified weights. - local = _identity() - loader = _new_loader(local, None, fetched=False) - assert loader._source_identity_compatible("ckpt", _STUB_CLIENT, _STUB_BUILD) is False + loader = _new_loader(_identity()) + assert loader._source_metadata_identity_compatible(None) is False -def test_fetch_source_identity_returns_none_when_metadata_unavailable(): +def test_fetch_source_metadata_supports_modelexpress_0_4_1_client_shape_and_close_failure(): + local = _identity() loader = MXCheckpointLoader.__new__(MXCheckpointLoader) + loader._local_source_identity = local loader._mx_server_url = "http://mx:8001" loader._model_name = None @@ -84,27 +82,24 @@ class _Client: def __init__(self, *, server_url): self.server_url = server_url - def list_sources(self, *, identity): - return SimpleNamespace(instances=[]) - - assert loader._fetch_source_identity("ckpt", _Client, lambda **_kw: object()) is None - + def get_metadata(self, mx_source_id, worker_id): + raise AssertionError("ID-based metadata lookup should not be used for identity queries") -def test_fetch_source_identity_from_source_metadata(): - source = _identity() - loader = MXCheckpointLoader.__new__(MXCheckpointLoader) - loader._mx_server_url = "http://mx:8001" - loader._model_name = None + def list_sources(self, *, identity): + return SimpleNamespace( + instances=[SimpleNamespace(mx_source_id="source", worker_id="worker")] + ) - class _Client: - def __init__(self, *, server_url): - self.server_url = server_url + def close(self): + raise RuntimeError("close failed") - def list_sources(self, *, identity): - instance = SimpleNamespace(metadata=_build_mx_source_metadata(source)) - return SimpleNamespace(instances=[instance]) + metadata = loader._fetch_source_metadata( + "ckpt", + _Client, + lambda **_kw: SimpleNamespace(extra_parameters={}), + ) - assert loader._fetch_source_identity("ckpt", _Client, lambda **_kw: object()) == source + assert metadata == _build_mx_source_metadata(local) def test_load_weights_pops_source_identity_kwarg(): diff --git a/tests/unittest/_torch/weight_sharing/test_source_identity.py b/tests/unittest/_torch/weight_sharing/test_source_identity.py index a02995c4f520..779dc8c355bd 100644 --- a/tests/unittest/_torch/weight_sharing/test_source_identity.py +++ b/tests/unittest/_torch/weight_sharing/test_source_identity.py @@ -19,6 +19,7 @@ """ import copy +from pathlib import Path import pytest from _source_identity_fakes import ( @@ -29,6 +30,7 @@ FakeQuantConfig, FakeQuantConfigWithPythonOnlyField, identity_from, + make_artifact_identity, ) from tensorrt_llm._torch.weight_sharing import ( @@ -48,6 +50,24 @@ def test_identical_configs_match(): assert bool(result) is True +def test_from_model_config_derives_artifact_identity(tmp_path: Path) -> None: + (tmp_path / "model.safetensors").write_bytes(b"checkpoint") + identity = SourceIdentity.from_model_config(FakeModelConfig(), checkpoint_dir=str(tmp_path)) + assert identity.artifact_identity.scheme == "checkpoint_manifest_sha256" + + +def test_from_model_config_requires_one_artifact_source() -> None: + config = FakeModelConfig() + with pytest.raises(ValueError, match="Exactly one"): + SourceIdentity.from_model_config(config) + with pytest.raises(ValueError, match="Exactly one"): + SourceIdentity.from_model_config( + config, + checkpoint_dir="/checkpoint", + artifact_identity=make_artifact_identity(), + ) + + def test_rank_defaults_from_mapping(): cfg = FakeModelConfig(mapping=FakeMapping(rank=3, tp_rank=3)) identity = identity_from(cfg) @@ -83,10 +103,14 @@ def test_param_dtype_override_flags_shard(): # the realized-layout fingerprint catches it. cfg = FakeModelConfig() a = SourceIdentity.from_model_config( - cfg, FakeModel(cfg.pretrained_config, dtype="torch.bfloat16") + cfg, + FakeModel(cfg.pretrained_config, dtype="torch.bfloat16"), + artifact_identity=make_artifact_identity(), ) b = SourceIdentity.from_model_config( - cfg, FakeModel(cfg.pretrained_config, dtype="torch.float16") + cfg, + FakeModel(cfg.pretrained_config, dtype="torch.float16"), + artifact_identity=make_artifact_identity(), ) result = a.matches(b) assert not result.matched @@ -109,11 +133,24 @@ def test_cross_architecture_same_shapes_flags_global(): assert "model_fingerprint" in result.mismatched_fields +def test_different_checkpoint_artifacts_flag_global(): + a = identity_from(FakeModelConfig(), artifact_key="fine-tune-a") + b = identity_from(FakeModelConfig(), artifact_key="fine-tune-b") + result = a.matches(b) + assert not result.matched + assert result.mismatched_fields == ["artifact_identity"] + assert a.global_fingerprint != b.global_fingerprint + + def test_no_model_degrades_to_architecture_only(): - # Without a module, the fingerprint still builds (architecture-only) and - # two identical configs still match. - a = SourceIdentity.from_model_config(FakeModelConfig(), None) - b = SourceIdentity.from_model_config(FakeModelConfig(), None) + # Without a module, the shard fingerprint has no realized tensor layout, + # while matching artifacts and configurations remain comparable. + a = SourceIdentity.from_model_config( + FakeModelConfig(), None, artifact_identity=make_artifact_identity() + ) + b = SourceIdentity.from_model_config( + FakeModelConfig(), None, artifact_identity=make_artifact_identity() + ) assert a.matches(b).matched @@ -168,6 +205,29 @@ def test_serialization_roundtrip(): restored = SourceIdentity.from_dict(a.to_dict()) assert restored == a assert a.matches(restored).matched + assert restored.artifact_identity == a.artifact_identity + + +def test_deserialization_rejects_missing_artifact_identity(): + payload = identity_from(FakeModelConfig()).to_dict() + payload.pop("artifact_identity") + with pytest.raises(KeyError): + SourceIdentity.from_dict(payload) + + +def test_deserialization_rejects_unknown_format_version(): + payload = identity_from(FakeModelConfig()).to_dict() + payload["format_version"] += 1 + with pytest.raises(ValueError, match="Unsupported SourceIdentity format version"): + SourceIdentity.from_dict(payload) + + +def test_deserialization_rejects_v1_identity_without_artifact_binding(): + payload = identity_from(FakeModelConfig()).to_dict() + payload["format_version"] = 1 + payload.pop("artifact_identity") + with pytest.raises(ValueError, match="Unsupported SourceIdentity format version"): + SourceIdentity.from_dict(payload) def test_check_warn_fallback_on_mismatch(): @@ -219,6 +279,7 @@ def test_format_version_mismatch_never_matches(): if hasattr(copy, "replace") else SourceIdentity( format_version=a.format_version + 1, + artifact_identity=a.artifact_identity, model_fingerprint=a.model_fingerprint, quant_fingerprint=a.quant_fingerprint, backend_fingerprint=a.backend_fingerprint, diff --git a/tests/unittest/api_stability/api_stability_core.py b/tests/unittest/api_stability/api_stability_core.py index 2a82d11fe3ee..c9b9a42388a7 100644 --- a/tests/unittest/api_stability/api_stability_core.py +++ b/tests/unittest/api_stability/api_stability_core.py @@ -157,9 +157,18 @@ def from_inspect(cls, method: MethodType): return_annotation = eval(return_annotation) return cls(parameters, return_annotation) + @classmethod + def _strip_api_status_tag(cls, docstring: str) -> str: + """Strip the :tag:`...` prefix added by @set_api_status decorator.""" + if docstring and docstring.startswith(":tag:"): + import re + docstring = re.sub(r'^:tag:`[^`]*`\s*', '', docstring) + return docstring + @classmethod def from_docstring(cls, method: MethodType): - doc = docstring_parser.parse(method.__doc__) + docstring = cls._strip_api_status_tag(method.__doc__) + doc = docstring_parser.parse(docstring) parameters = {} for param in doc.params: if param.args[0] == 'param': diff --git a/tests/unittest/api_stability/references/llm.yaml b/tests/unittest/api_stability/references/llm.yaml index 7ff208c41ce3..e30864532cb3 100644 --- a/tests/unittest/api_stability/references/llm.yaml +++ b/tests/unittest/api_stability/references/llm.yaml @@ -35,6 +35,10 @@ methods: annotation: bool default: False status: prototype + perf_metrics_output_dir: + annotation: Optional[str] + default: null + status: prototype # Bindings and mirrored configs peft_cache_config: annotation: Optional[tensorrt_llm.llmapi.llm_args.PeftCacheConfig] @@ -60,6 +64,10 @@ methods: annotation: Optional[str] default: null status: prototype + num_serve_frontends: + annotation: int + default: 1 + status: prototype custom_tokenizer: annotation: Optional[str] default: null @@ -198,7 +206,7 @@ methods: perf_metrics_max_requests: annotation: int default: 0 - status: prototype + status: deprecated prometheus_metrics_config: annotation: Optional[tensorrt_llm.llmapi.llm_args.PrometheusMetricsConfig] default: null @@ -437,6 +445,10 @@ methods: default: 2 return_annotation: tensorrt_llm.executor.result.IterationResult status: beta + get_data_transceiver_state: + parameters: {} + return_annotation: bytes + status: prototype shutdown: parameters: {} return_annotation: None diff --git a/tests/unittest/api_stability/references/trtllm_serve_api.yaml b/tests/unittest/api_stability/references/trtllm_serve_api.yaml index 471491aaa0b9..99d263914f05 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_api.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_api.yaml @@ -301,6 +301,12 @@ models: default: null status: prototype required: false + priority: + kind: extension + type: Optional[float] + default: null + status: prototype + required: false CompletionResponseChoice: fields: @@ -802,6 +808,12 @@ models: default: null status: beta required: false + priority: + kind: extension + type: Optional[float] + default: null + status: prototype + required: false ChatCompletionResponseChoice: fields: diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml index 5d681cdeb9d8..c3430712d780 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml @@ -303,6 +303,15 @@ commands: is_flag: false flags: - "--num_postprocess_workers" + num_serve_frontends: + type: int + default: 1 + status: prototype + required: false + multiple: false + is_flag: false + flags: + - "--num_serve_frontends" num_input_processor_workers: type: int default: 8 diff --git a/tests/unittest/api_stability/references_committed/llm.yaml b/tests/unittest/api_stability/references_committed/llm.yaml index cf6f04373a90..8f36fe11823e 100644 --- a/tests/unittest/api_stability/references_committed/llm.yaml +++ b/tests/unittest/api_stability/references_committed/llm.yaml @@ -59,7 +59,7 @@ methods: default: null # Speculative decoding speculative_config: - annotation: Union[tensorrt_llm.llmapi.llm_args.DraftTargetDecodingConfig, tensorrt_llm.llmapi.llm_args.EagleDecodingConfig, tensorrt_llm.llmapi.llm_args.Eagle3DecodingConfig, tensorrt_llm.llmapi.llm_args.LookaheadDecodingConfig, tensorrt_llm.llmapi.llm_args.MedusaDecodingConfig, tensorrt_llm.llmapi.llm_args.MTPDecodingConfig, tensorrt_llm.llmapi.llm_args.NGramDecodingConfig, tensorrt_llm.llmapi.llm_args.SADecodingConfig, tensorrt_llm.llmapi.llm_args.UserProvidedDecodingConfig, tensorrt_llm.llmapi.llm_args.SaveHiddenStatesDecodingConfig, tensorrt_llm.llmapi.llm_args.PARDDecodingConfig, tensorrt_llm.llmapi.llm_args.DFlashDecodingConfig, tensorrt_llm.llmapi.llm_args.AutoDecodingConfig, NoneType] + annotation: Union[tensorrt_llm.llmapi.llm_args.DraftTargetDecodingConfig, tensorrt_llm.llmapi.llm_args.EagleDecodingConfig, tensorrt_llm.llmapi.llm_args.Eagle3DecodingConfig, tensorrt_llm.llmapi.llm_args.LookaheadDecodingConfig, tensorrt_llm.llmapi.llm_args.MedusaDecodingConfig, tensorrt_llm.llmapi.llm_args.MTPDecodingConfig, tensorrt_llm.llmapi.llm_args.NGramDecodingConfig, tensorrt_llm.llmapi.llm_args.SADecodingConfig, tensorrt_llm.llmapi.llm_args.UserProvidedDecodingConfig, tensorrt_llm.llmapi.llm_args.SaveHiddenStatesDecodingConfig, tensorrt_llm.llmapi.llm_args.PARDDecodingConfig, tensorrt_llm.llmapi.llm_args.DFlashDecodingConfig, tensorrt_llm.llmapi.llm_args.DSparkDecodingConfig, tensorrt_llm.llmapi.llm_args.AutoDecodingConfig, NoneType] default: null # generation constraints max_batch_size: diff --git a/tests/unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py b/tests/unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py index 3b4957d08124..b9ae83d8f020 100644 --- a/tests/unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py +++ b/tests/unittest/auto_deploy/multigpu/custom_ops/test_ad_dist_strategies.py @@ -12,18 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import signal -import subprocess -import tempfile -from contextlib import contextmanager -from pathlib import Path - import pytest import torch import torch.nn as nn -import yaml -from _model_test_utils import get_small_model_config -from click.testing import CliRunner from utils.cpp_paths import llm_root # noqa: F401 from tensorrt_llm._torch.auto_deploy.export import torch_export_to_gm @@ -37,279 +28,12 @@ from tensorrt_llm._torch.auto_deploy.transform.library.sharding_ir import SplitDimension from tensorrt_llm._torch.auto_deploy.utils._graph import recompile from tensorrt_llm._torch.auto_deploy.utils.node_utils import is_op -from tensorrt_llm.commands.bench import main from tensorrt_llm.functional import AllReduceStrategy # needed since LLM API uses MPI executor pool internally for TP>1, which leaks a thread on shutdown pytestmark = pytest.mark.threadleak(enabled=False) -class TimeoutError(Exception): - """Exception raised when a test times out.""" - - pass - - -@contextmanager -def timeout(seconds): - """Context manager that raises TimeoutError if code block exceeds time limit. - - Args: - seconds: Maximum time in seconds to allow the code block to run - - Raises: - TimeoutError: If the code block execution exceeds the time limit - """ - - def timeout_handler(signum, frame): - raise TimeoutError(f"Test execution exceeded {seconds} seconds timeout") - - # Set the signal handler and alarm - old_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(seconds) - try: - yield - finally: - # Restore the old signal handler and cancel the alarm - signal.alarm(0) - signal.signal(signal.SIGALRM, old_handler) - - -@pytest.fixture(scope="module", autouse=True) -def prewarm_flashinfer_jit(): - """Pre-warm FlashInfer JIT kernels before multi-GPU tests. - - This prevents a race condition where multiple MPI ranks try to JIT-compile - FlashInfer kernels simultaneously to the same cache directory, causing - Ninja build failures like: "ninja: error: opening build log: No such file or directory" - - By triggering the compilation in the main process first, the kernels are - cached and available for all worker ranks. - """ - try: - import flashinfer - import flashinfer.page - import flashinfer.sampling - - if torch.cuda.is_available(): - # Prevent concurrent JIT warmup across multiple pytest processes (e.g., xdist). - try: - import fcntl # Linux-only - except ImportError: - fcntl = None - - lock_f = None - if fcntl is not None: - import pathlib - import tempfile - - lock_path = pathlib.Path(tempfile.gettempdir()) / "flashinfer_jit_prewarm.lock" - lock_f = open(lock_path, "w") - fcntl.flock(lock_f.fileno(), fcntl.LOCK_EX) - # Create dummy tensors to trigger kernel JIT compilation - with torch.no_grad(): - device = torch.device("cuda:0") - - # Trigger page kernel compilation - try: - # Force module loading (this triggers JIT compilation) - _ = flashinfer.page.gen_page_module() - except Exception as exc: # noqa: BLE001 - import warnings - - warnings.warn(f"FlashInfer page-kernel prewarm failed: {exc!r}", RuntimeWarning) - - # Trigger sampling kernel compilation - try: - dummy_probs = torch.softmax(torch.randn(1, 100, device=device), dim=-1) - _ = flashinfer.sampling.sampling_from_probs(dummy_probs, deterministic=True) - except Exception as exc: # noqa: BLE001 - import warnings - - warnings.warn( - f"FlashInfer sampling-kernel prewarm failed: {exc!r}", RuntimeWarning - ) - - torch.cuda.empty_cache() - if lock_f is not None: - lock_f.close() - - except ImportError: - pass # FlashInfer not available - - yield - - -@pytest.fixture(scope="module") -def shared_dataset(llm_root): # noqa: F811 - """Prepare dataset once for all tests in this module.""" - model_name = "meta-llama/Meta-Llama-3.1-8B-Instruct" - config = get_small_model_config(model_name) - with tempfile.TemporaryDirectory() as temp_dir: - dataset_path = _prepare_dataset( - llm_root, temp_dir, config["args"]["model"], num_requests=10 - ) - # Read dataset content to return it (temp_dir will be deleted) - with open(dataset_path, "r") as f: - dataset_content = f.read() - yield dataset_content - - -def _prepare_dataset(root_dir: str, temp_dir: str, model_path_or_name: str, num_requests: int = 10): - """Prepare a synthetic dataset for benchmarking.""" - _DATASET_NAME = "synthetic_128_128.txt" - dataset_path = Path(temp_dir, _DATASET_NAME) - dataset_tool = Path(root_dir, "benchmarks", "cpp", "prepare_dataset.py") - script_dir = Path(root_dir, "benchmarks", "cpp") - - # Generate a small dataset to run a test - matching workload configuration - command = [ - "python3", - f"{dataset_tool}", - "--stdout", - "--tokenizer", - model_path_or_name, - "token-norm-dist", - "--input-mean", - "128", - "--output-mean", - "128", - "--input-stdev", - "0", - "--output-stdev", - "0", - "--num-requests", - str(num_requests), - ] - print(f"Running command: {' '.join(command)}") - result = subprocess.run( - command, cwd=str(script_dir), capture_output=True, text=True, timeout=300 - ) - if result.returncode != 0: - raise RuntimeError(f"Failed to prepare dataset: {result.stderr}") - # Grab the stdout and write it to a dataset file for passing to suite. - with open(dataset_path, "w") as dataset: - dataset.write(result.stdout) - return dataset_path - - -@pytest.mark.parametrize( - "allreduce_strategy", - [ - "AUTO", - "ONESHOT", - "TWOSHOT", - "MIN_LATENCY", - "NCCL", - "SYMM_MEM", - ], -) -def test_allreduce_strategies(llm_root, shared_dataset, allreduce_strategy): # noqa: F811 - """Test different allreduce strategies with multi-GPU configuration making sure that there are no crashes or hangs. - - Configuration: - The allreduce_strategy is set in the transforms config: - ```yaml - transforms: - detect_sharding: - allreduce_strategy: "ONESHOT" # or AUTO, NCCL, TWOSHOT, etc. - ``` - - Test configuration: - - Model: Llama-3.1-8B with TP=2 - - Dataset: 10 synthetic requests (128 input, 128 output tokens) - - Timeout: 300 seconds to catch hangs - - Skipped if fewer than 2 GPUs available - - Args: - llm_root: Root directory fixture - shared_dataset: Shared dataset fixture (prepared once for all test runs) - allreduce_strategy: Strategy to test (AUTO, ONESHOT, TWOSHOT, MIN_LATENCY, NCCL) - """ - # Fixed timeout for all strategies (5 minutes should be enough) - TEST_TIMEOUT_SECONDS = 300 - - model_name = "meta-llama/Meta-Llama-3.1-8B-Instruct" - config = get_small_model_config(model_name) - tp_size = 2 - max_batch_size = 256 - max_num_tokens = 8192 - - if not torch.cuda.is_available() or torch.cuda.device_count() < tp_size: - pytest.skip(f"Allreduce strategy test requires at least {tp_size} GPUs, skipping") - - with tempfile.TemporaryDirectory() as temp_dir: - # Write shared dataset to temp location - dataset_path = Path(temp_dir, "synthetic_128_128.txt") - with open(dataset_path, "w") as f: - f.write(shared_dataset) - - # Create configuration with specified allreduce strategy in transforms - extra_llm_api_options_path = f"{temp_dir}/extra_llm_api_options.yaml" - with open(extra_llm_api_options_path, "w") as f: - yaml.dump( - { - **config["args"], - "max_batch_size": max_batch_size, - "max_num_tokens": max_num_tokens, - "max_seq_len": 256, - "transforms": { - "detect_sharding": { - "stage": "sharding", - "allreduce_strategy": allreduce_strategy, - }, - "compile_model": { - "stage": "compile", - "backend": "torch-cudagraph", - "cuda_graph_batch_sizes": [1, 2, 4, 8, 16, 32, 64, 128, 256], - }, - }, - }, - f, - ) - - # Run benchmark with specified allreduce strategy with timeout protection - runner = CliRunner() - args = [ - "--model", - model_name, - ] - - # Only pass --model_path if it's a local filesystem path - # Note: --model_path must come BEFORE the subcommand (throughput) - if str(config["args"]["model"]).startswith("/"): - args.extend(["--model_path", str(config["args"]["model"])]) - - # Add the subcommand and its options - args.extend( - [ - "throughput", - "--backend", - "_autodeploy", - "--dataset", - str(dataset_path), - "--extra_llm_api_options", - extra_llm_api_options_path, - "--tp", - str(tp_size), - "--max_batch_size", - str(max_batch_size), - "--max_num_tokens", - str(max_num_tokens), - ] - ) - - try: - with timeout(TEST_TIMEOUT_SECONDS): - result = runner.invoke(main, args, catch_exceptions=False) - assert result.exit_code == 0, f"Benchmark failed with output: {result.output}" - except TimeoutError as e: - pytest.fail( - f"Test timed out after {TEST_TIMEOUT_SECONDS}s for strategy {allreduce_strategy}. " - f"This might indicate a hang (e.g., TWOSHOT without C++ fix). Error: {e}" - ) - - @pytest.mark.parametrize( "strategy", [ diff --git a/tests/unittest/auto_deploy/multigpu/custom_ops/test_multi_stream_moe_trailing_allreduce.py b/tests/unittest/auto_deploy/multigpu/custom_ops/test_multi_stream_moe_trailing_allreduce.py index 139d76cd5a97..a08d633a5381 100644 --- a/tests/unittest/auto_deploy/multigpu/custom_ops/test_multi_stream_moe_trailing_allreduce.py +++ b/tests/unittest/auto_deploy/multigpu/custom_ops/test_multi_stream_moe_trailing_allreduce.py @@ -476,7 +476,9 @@ def _run_with_retries(worker_fn, world_size, **kwargs): max_retries = 5 last_exc = None for _ in range(max_retries): - pool = MpiPoolSession(n_workers=world_size) + # wait_shutdown: block shutdown until the workers exited, so a test + # handed a live pool right after this one cannot race the GPU release. + pool = MpiPoolSession(n_workers=world_size, wait_shutdown=True) try: return pool.submit_sync(worker_fn, port=None, world_size=world_size, **kwargs) except DistNetworkError as e: diff --git a/tests/unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py b/tests/unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py new file mode 100644 index 000000000000..2517196708e2 --- /dev/null +++ b/tests/unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py @@ -0,0 +1,307 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import signal +import subprocess +import tempfile +from contextlib import contextmanager +from pathlib import Path + +import pytest +import torch +import yaml +from _model_test_utils import get_small_model_config +from click.testing import CliRunner +from utils.cpp_paths import llm_root # noqa: F401 + +from tensorrt_llm.commands.bench import main + +# needed since LLM API uses MPI executor pool internally for TP>1, which leaks a thread on shutdown +pytestmark = pytest.mark.threadleak(enabled=False) + + +class TimeoutError(Exception): + """Exception raised when a test times out.""" + + pass + + +@contextmanager +def timeout(seconds): + """Context manager that raises TimeoutError if code block exceeds time limit. + + Args: + seconds: Maximum time in seconds to allow the code block to run + + Raises: + TimeoutError: If the code block execution exceeds the time limit + """ + + def timeout_handler(signum, frame): + raise TimeoutError(f"Test execution exceeded {seconds} seconds timeout") + + # Set the signal handler and alarm + old_handler = signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(seconds) + try: + yield + finally: + # Restore the old signal handler and cancel the alarm + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + + +@pytest.fixture(scope="module", autouse=True) +def prewarm_flashinfer_jit(): + """Pre-warm FlashInfer JIT kernels before multi-GPU tests. + + This prevents a race condition where multiple MPI ranks try to JIT-compile + FlashInfer kernels simultaneously to the same cache directory, causing + Ninja build failures like: "ninja: error: opening build log: No such file or directory" + + By triggering the compilation in the main process first, the kernels are + cached and available for all worker ranks. + """ + try: + import flashinfer + import flashinfer.page + import flashinfer.sampling + + if torch.cuda.is_available(): + # Prevent concurrent JIT warmup across multiple pytest processes (e.g., xdist). + try: + import fcntl # Linux-only + except ImportError: + fcntl = None + + lock_f = None + if fcntl is not None: + import pathlib + import tempfile + + lock_path = pathlib.Path(tempfile.gettempdir()) / "flashinfer_jit_prewarm.lock" + lock_f = open(lock_path, "w") + fcntl.flock(lock_f.fileno(), fcntl.LOCK_EX) + # Create dummy tensors to trigger kernel JIT compilation + with torch.no_grad(): + device = torch.device("cuda:0") + + # Trigger page kernel compilation + try: + # Force module loading (this triggers JIT compilation) + _ = flashinfer.page.gen_page_module() + except Exception as exc: # noqa: BLE001 + import warnings + + warnings.warn(f"FlashInfer page-kernel prewarm failed: {exc!r}", RuntimeWarning) + + # Trigger sampling kernel compilation + try: + dummy_probs = torch.softmax(torch.randn(1, 100, device=device), dim=-1) + _ = flashinfer.sampling.sampling_from_probs(dummy_probs, deterministic=True) + except Exception as exc: # noqa: BLE001 + import warnings + + warnings.warn( + f"FlashInfer sampling-kernel prewarm failed: {exc!r}", RuntimeWarning + ) + + torch.cuda.empty_cache() + if lock_f is not None: + lock_f.close() + + except ImportError: + pass # FlashInfer not available + + yield + + +@pytest.fixture(scope="module") +def shared_dataset(llm_root): # noqa: F811 + """Prepare dataset once for all tests in this module.""" + model_name = "meta-llama/Meta-Llama-3.1-8B-Instruct" + config = get_small_model_config(model_name) + with tempfile.TemporaryDirectory() as temp_dir: + dataset_path = _prepare_dataset( + llm_root, temp_dir, config["args"]["model"], num_requests=10 + ) + # Read dataset content to return it (temp_dir will be deleted) + with open(dataset_path, "r") as f: + dataset_content = f.read() + yield dataset_content + + +def _prepare_dataset(root_dir: str, temp_dir: str, model_path_or_name: str, num_requests: int = 10): + """Prepare a synthetic dataset for benchmarking.""" + _DATASET_NAME = "synthetic_128_128.txt" + dataset_path = Path(temp_dir, _DATASET_NAME) + + # Generate a small dataset to run a test - matching workload configuration. + # Write straight to a file via --output (the documented usage) instead of + # scraping --stdout: trtllm-bench prints an import-time banner on stdout that + # would otherwise corrupt the captured JSONL. + command = [ + "trtllm-bench", + "--model", + model_path_or_name, + "prepare-dataset", + "--output", + str(dataset_path), + "token-norm-dist", + "--input-mean", + "128", + "--output-mean", + "128", + "--input-stdev", + "0", + "--output-stdev", + "0", + "--num-requests", + str(num_requests), + ] + print(f"Running command: {' '.join(command)}") + result = subprocess.run(command, cwd=str(temp_dir), capture_output=True, text=True, timeout=300) + if result.returncode != 0: + raise RuntimeError(f"Failed to prepare dataset: {result.stderr}") + return dataset_path + + +@pytest.mark.parametrize( + "allreduce_strategy", + [ + "AUTO", + "ONESHOT", + "TWOSHOT", + "MIN_LATENCY", + "NCCL", + "SYMM_MEM", + ], +) +def test_allreduce_strategies(llm_root, shared_dataset, allreduce_strategy): # noqa: F811 + """Test different allreduce strategies with multi-GPU configuration making sure that there are no crashes or hangs. + + Configuration: + The allreduce_strategy is set in the transforms config: + ```yaml + transforms: + detect_sharding: + allreduce_strategy: "ONESHOT" # or AUTO, NCCL, TWOSHOT, etc. + ``` + + Test configuration: + - Model: Llama-3.1-8B with TP=2 + - Dataset: 10 synthetic requests (128 input, 128 output tokens) + - Timeout: 300 seconds to catch hangs + - Skipped if fewer than 2 GPUs available + + Args: + llm_root: Root directory fixture + shared_dataset: Shared dataset fixture (prepared once for all test runs) + allreduce_strategy: Strategy to test (AUTO, ONESHOT, TWOSHOT, MIN_LATENCY, NCCL) + """ + # Fixed timeout for all strategies (5 minutes should be enough) + TEST_TIMEOUT_SECONDS = 300 + + model_name = "meta-llama/Meta-Llama-3.1-8B-Instruct" + # Override hidden_size to a multiple of one warp's worth of 128-bit accesses (32 threads * + # 8 fp16 elements/access = 256) so the fused all-reduce/RMSNorm kernel never launches a + # partial-warp block. This isolates whether CI failures are specific to the partial-warp + # code path or are unrelated infra flakiness that also affects the full-warp path. + config = get_small_model_config( + model_name, + model_kwargs={ + "num_hidden_layers": 1, + "hidden_size": 256, + "intermediate_size": 256, + "num_attention_heads": 2, + "num_key_value_heads": 1, + }, + ) + tp_size = 2 + max_batch_size = 256 + max_num_tokens = 8192 + + if not torch.cuda.is_available() or torch.cuda.device_count() < tp_size: + pytest.skip(f"Allreduce strategy test requires at least {tp_size} GPUs, skipping") + + with tempfile.TemporaryDirectory() as temp_dir: + # Write shared dataset to temp location + dataset_path = Path(temp_dir, "synthetic_128_128.txt") + with open(dataset_path, "w") as f: + f.write(shared_dataset) + + # Create configuration with specified allreduce strategy in transforms + extra_llm_api_options_path = f"{temp_dir}/extra_llm_api_options.yaml" + with open(extra_llm_api_options_path, "w") as f: + yaml.dump( + { + **config["args"], + "max_batch_size": max_batch_size, + "max_num_tokens": max_num_tokens, + "max_seq_len": 256, + "transforms": { + "detect_sharding": { + "stage": "sharding", + "allreduce_strategy": allreduce_strategy, + }, + "compile_model": { + "stage": "compile", + "backend": "torch-cudagraph", + "cuda_graph_batch_sizes": [1, 2, 4, 8, 16, 32, 64, 128, 256], + }, + }, + }, + f, + ) + + # Run benchmark with specified allreduce strategy with timeout protection + runner = CliRunner() + args = [ + "--model", + model_name, + ] + + # Only pass --model_path if it's a local filesystem path + # Note: --model_path must come BEFORE the subcommand (throughput) + if str(config["args"]["model"]).startswith("/"): + args.extend(["--model_path", str(config["args"]["model"])]) + + # Add the subcommand and its options + args.extend( + [ + "throughput", + "--backend", + "_autodeploy", + "--dataset", + str(dataset_path), + "--extra_llm_api_options", + extra_llm_api_options_path, + "--tp", + str(tp_size), + "--max_batch_size", + str(max_batch_size), + "--max_num_tokens", + str(max_num_tokens), + ] + ) + + try: + with timeout(TEST_TIMEOUT_SECONDS): + result = runner.invoke(main, args, catch_exceptions=False) + assert result.exit_code == 0, f"Benchmark failed with output: {result.output}" + except TimeoutError as e: + pytest.fail( + f"Test timed out after {TEST_TIMEOUT_SECONDS}s for strategy {allreduce_strategy}. " + f"This might indicate a hang (e.g., TWOSHOT without C++ fix). Error: {e}" + ) diff --git a/tests/unittest/auto_deploy/multigpu/transformations/library/test_allreduce_residual_rmsnorm_fusion.py b/tests/unittest/auto_deploy/multigpu/transformations/library/test_allreduce_residual_rmsnorm_fusion.py index f9602d84b8e9..94a2f47c649f 100644 --- a/tests/unittest/auto_deploy/multigpu/transformations/library/test_allreduce_residual_rmsnorm_fusion.py +++ b/tests/unittest/auto_deploy/multigpu/transformations/library/test_allreduce_residual_rmsnorm_fusion.py @@ -203,7 +203,9 @@ def test_allreduce_fusion(device_count, ModuleCls, strategy, rmsnorm_op): max_retries = 5 last_exc: Exception | None = None for _ in range(max_retries): - mpi_pool = MpiPoolSession(n_workers=n_workers) + # wait_shutdown: block shutdown until the workers exited, so a test + # handed a live pool right after this one cannot race the GPU release. + mpi_pool = MpiPoolSession(n_workers=n_workers, wait_shutdown=True) try: mpi_pool.submit_sync( _test_allreduce_fusion, diff --git a/tests/unittest/auto_deploy/singlegpu/smoke/test_ad_trtllm_bench.py b/tests/unittest/auto_deploy/singlegpu/smoke/test_ad_trtllm_bench.py index e227aae954a9..6ba35a0aa44d 100644 --- a/tests/unittest/auto_deploy/singlegpu/smoke/test_ad_trtllm_bench.py +++ b/tests/unittest/auto_deploy/singlegpu/smoke/test_ad_trtllm_bench.py @@ -65,7 +65,6 @@ def run_benchmark( def prepare_dataset(root_dir: str, temp_dir: str, model_path_or_name: str): _DATASET_NAME = "synthetic_128_128.txt" dataset_path = Path(temp_dir, _DATASET_NAME) - script_dir = Path(root_dir, "benchmarks", "cpp") # Generate a small dataset to run a test - matching workload configuration command = [ @@ -88,9 +87,7 @@ def prepare_dataset(root_dir: str, temp_dir: str, model_path_or_name: str): "10", ] print(f"Running command: {' '.join(command)}") - result = subprocess.run( - command, cwd=str(script_dir), capture_output=True, text=True, timeout=300 - ) + result = subprocess.run(command, cwd=str(temp_dir), capture_output=True, text=True, timeout=300) if result.returncode != 0: raise RuntimeError(f"Failed to prepare dataset: {result.stderr}") diff --git a/tests/unittest/auto_deploy/standalone/test_standalone_package.py b/tests/unittest/auto_deploy/standalone/test_standalone_package.py index 8074058b26d8..7cd06acd075c 100644 --- a/tests/unittest/auto_deploy/standalone/test_standalone_package.py +++ b/tests/unittest/auto_deploy/standalone/test_standalone_package.py @@ -20,7 +20,7 @@ (including source, tests, and pyproject.toml) 2. Creates a venv and installs the package with dev deps 3. Verifies TRTLLM_AVAILABLE is False and core subsystems work -4. Runs the copied unit tests from the standalone package's own tests/ dir +4. Collects the complete copied test tree and runs its standalone unit tests The venv Python is run with `-I` (isolated mode) to prevent the host env's editable TRT-LLM install from leaking in. @@ -217,11 +217,12 @@ def test_compat_types(self, standalone_package): assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}" def test_run_unit_tests(self, standalone_package): - """Run the copied unit tests from the standalone package's tests/ dir. + """Collect every copied test and run the standalone single-GPU tests. Tests have been import-rewritten to use `paragraf` instead of `tensorrt_llm._torch.auto_deploy`, so they run directly against the - standalone package. + standalone package. Optional TensorRT-LLM tests must skip cleanly when + its wheel is not installed. """ python = standalone_package["python"] pkg_dir = standalone_package["pkg_dir"] @@ -286,6 +287,21 @@ def test_run_unit_tests(self, standalone_package): f"stdout:\n{isolation_probe.stdout}\nstderr:\n{isolation_probe.stderr}" ) + collection_result = subprocess.run( + [python, "-I", "-m", "pytest", tests_dir, "--collect-only", "-q"], + capture_output=True, + text=True, + timeout=300, + cwd=pkg_dir, + env=standalone_env, + ) + assert collection_result.returncode == 0, ( + "The complete standalone test tree must collect without " + "TensorRT-LLM installed. Optional tests should skip at module " + f"collection.\nstdout:\n{collection_result.stdout[-5000:]}\n" + f"stderr:\n{collection_result.stderr[-5000:]}" + ) + cmd = [ python, "-I", @@ -330,12 +346,12 @@ def test_run_unit_tests(self, standalone_package): f"Stderr:\n{result.stderr[-3000:]}" ) - # Strict: zero test failures — if a test can't pass standalone, it must - # be in the EXCLUDE list in create_standalone_package.py + # Strict: zero test failures. Tests that require the optional + # TensorRT-LLM wheel must use the generated module-level guard. assert num_failed == 0, ( f"{num_failed} test(s) failed in standalone mode!\n" f"Summary: {summary_str}\n" - f"These tests should be added to EXCLUDE_TEST_FILES in " + f"TensorRT-LLM-dependent tests should be guarded in " f"create_standalone_package.py.\n" f"Failed tests:\n" + "\n".join(lin for lin in lines if lin.startswith("FAILED")) diff --git a/tests/unittest/auto_deploy/standalone/test_standalone_test_export.py b/tests/unittest/auto_deploy/standalone/test_standalone_test_export.py index 706a22854287..49525ff30cfd 100644 --- a/tests/unittest/auto_deploy/standalone/test_standalone_test_export.py +++ b/tests/unittest/auto_deploy/standalone/test_standalone_test_export.py @@ -28,111 +28,65 @@ LEGACY_CREATE_SCRIPT = ( REPO_ROOT / "examples" / "auto_deploy" / "llmc" / "create_standalone_package.py" ) -AD_SINGLEGPU_TESTS = REPO_ROOT / "tests" / "unittest" / "auto_deploy" / "singlegpu" -AD_MULTIGPU_TESTS = REPO_ROOT / "tests" / "unittest" / "auto_deploy" / "multigpu" -AD_TORCH_TESTS = REPO_ROOT / "tests" / "unittest" / "_torch" / "auto_deploy" -PARAGRAF_TRTLLM_SINGLEGPU_SMOKE_TEST = Path("smoke/test_ad_build_small_single.py") -PARAGRAF_TRTLLM_MULTIGPU_SMOKE_TEST = Path("smoke/test_ad_build_small_multi.py") -PARAGRAF_TRTLLM_RUNNER_TESTS = { - PARAGRAF_TRTLLM_SINGLEGPU_SMOKE_TEST, - Path("smoke/test_ad_guided_decoding_regex.py"), - Path("smoke/test_ad_speculative_decoding.py"), - Path("smoke/test_ad_trtllm_sampler.py"), - PARAGRAF_TRTLLM_MULTIGPU_SMOKE_TEST, +AUTODEPLOY = re.compile(r"auto_?deploy|_ad_", re.IGNORECASE) +TEST_FILE = re.compile(r"(?:^test_.*|.*_test)\.py$") +CANONICAL_IMPORT = "tensorrt_llm._torch.auto_deploy" +OPTIONAL_TRTLLM_GUARD = "Paragraf optional TRT-LLM tests require" +GENERATED_TEST_RENAMES = { + "test_llm_api_autodeploy.py": "test_llm_api_paragraf_trtllm.py", + "test_ad_disagg.py": "test_paragraf_trtllm_disagg.py", + "test_ad_disagg_trtllm_serve.py": "test_paragraf_trtllm_disagg_serve.py", + "test_ad_guided_decoding.py": "test_paragraf_trtllm_guided_decoding.py", + "test_ad_speculative_decoding.py": "test_paragraf_trtllm_speculative_decoding.py", + "test_ad_dist_strategies.py": "test_paragraf_trtllm_dist_strategies.py", + "test_ad_allreduce_strategies.py": "test_paragraf_trtllm_allreduce_strategies.py", + "test_ad_build_small_multi.py": "test_paragraf_trtllm_build_small_multi.py", + "test_ad_moe_op.py": "test_paragraf_trtllm_moe_op.py", + "test_ad_executor_swa_eviction.py": "test_paragraf_trtllm_executor_swa_eviction.py", + "test_create_ad_executor.py": "test_create_paragraf_trtllm_executor.py", + "test_ad_build_small_single.py": "test_paragraf_trtllm_build_small_single.py", + "test_ad_guided_decoding_regex.py": "test_paragraf_trtllm_guided_decoding_regex.py", + "test_ad_trtllm_bench.py": "test_paragraf_trtllm_bench.py", + "test_ad_trtllm_sampler.py": "test_paragraf_trtllm_sampler.py", + "test_ad_trtllm_serve.py": "test_paragraf_trtllm_serve.py", } -EXPECTED_SINGLEGPU_SMOKE_FILES = { - path for path in PARAGRAF_TRTLLM_RUNNER_TESTS if path != PARAGRAF_TRTLLM_MULTIGPU_SMOKE_TEST -} | { - Path("smoke/test_ad_trtllm_bench.py"), - Path("smoke/test_ad_trtllm_serve.py"), - Path("smoke/test_disagg.py"), -} -EXPECTED_MULTIGPU_FILES = { - Path("custom_ops/test_dist.py"), - Path("custom_ops/test_sharded_rmsnorm.py"), - PARAGRAF_TRTLLM_MULTIGPU_SMOKE_TEST, - Path("transformations/library/conftest.py"), - Path("transformations/library/test_apply_sharding_hints.py"), - Path("transformations/library/test_bmm_sharding.py"), - Path("transformations/library/test_ep_sharding.py"), - Path("transformations/library/test_rmsnorm_sharding.py"), - Path("transformations/library/test_sharding_num_correctness.py"), - Path("transformations/library/test_step3p7_sharding_ir.py"), - Path("transformations/library/test_tp_sharding.py"), -} -EXPECTED_TORCH_UNIT_FILES = { - Path("unit/singlegpu/models/test_gpt_oss_modeling.py"), -} -CANONICAL_IMPORT = "tensorrt_llm._torch.auto_deploy" -STANDALONE_IMPORT = "paragraf" -BUILD_AND_RUN_AD_IMPORT = "from build_and_run_ad import ExperimentConfig, main" -TRTLLM_IMPORT_RE = re.compile( - r"(?m)^(?:from|import) " - r"(?:tensorrt_llm(?:\.|\b)|paragraf\.models\.custom\.modeling_gpt_oss(?:\.|\b))" -) -PARAGRAF_OPTIONAL_TRTLLM_GUARD = """ -_trtllm_redirect_value = os.environ.get("TRTLLM_REDIRECT_AD_TO_PARAGRAF") -if _trtllm_redirect_value is None: - _trtllm_redirect_value = os.environ.get("TRTLLM_REDIRECT_AD_TO_LLMC", "") -_trtllm_redirect_value = _trtllm_redirect_value.lower() -if _trtllm_redirect_value not in {"1", "true", "yes", "on"}: - pytest.skip( - "Paragraf optional TRT-LLM tests require TRTLLM_REDIRECT_AD_TO_PARAGRAF=true", - allow_module_level=True, + +def _tracked_autodeploy_tests() -> set[Path]: + result = subprocess.run( + ["git", "ls-files", "tests"], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, ) -pytest.importorskip("tensorrt_llm")""" -PARAGRAF_TRTLLM_RUNNER_IMPORT = ( - "from runners.trtllm.build_and_run_paragraf_trtllm import ExperimentConfig, main" -) + return { + Path(path) + for path in result.stdout.splitlines() + if TEST_FILE.fullmatch(Path(path).name) + and AUTODEPLOY.search(path) + and "/standalone/" not in path + } -def _expected_exported_test(source: str, relative_path: Path) -> str: - expected = source.replace(CANONICAL_IMPORT, STANDALONE_IMPORT) - - def ensure_imports(before_pos: int, *imports: str) -> None: - nonlocal expected - prefix = expected[:before_pos] - missing_imports = [ - import_name for import_name in imports if f"import {import_name}\n" not in prefix - ] - if not missing_imports: - return - first_import = re.search(r"(?m)^(?:import|from) ", expected) - assert first_import is not None - expected = ( - expected[: first_import.start()] - + "\n".join(f"import {import_name}" for import_name in missing_imports) - + "\n" - + expected[first_import.start() :] - ) - - def insert_optional_trtllm_guard() -> None: - nonlocal expected - pytest_import = re.search(r"(?m)^import pytest\n", expected) - assert pytest_import is not None - expected = ( - expected[: pytest_import.end()] - + PARAGRAF_OPTIONAL_TRTLLM_GUARD - + "\n" - + expected[pytest_import.end() :] - ) - - if relative_path in PARAGRAF_TRTLLM_RUNNER_TESTS: - build_import_pos = expected.index(BUILD_AND_RUN_AD_IMPORT) - ensure_imports(build_import_pos, "os", "pytest") - insert_optional_trtllm_guard() - expected = expected.replace(BUILD_AND_RUN_AD_IMPORT, PARAGRAF_TRTLLM_RUNNER_IMPORT) +def _generated_test_path(source_path: Path) -> Path: + path = source_path.as_posix() + legacy_prefix = "tests/unittest/auto_deploy/" + torch_prefix = "tests/unittest/_torch/auto_deploy/" + integration_prefix = "tests/integration/" + if path.startswith(legacy_prefix): + generated_path = Path(path.removeprefix(legacy_prefix)) + elif path.startswith(torch_prefix): + generated_path = Path(path.removeprefix(torch_prefix)) + if generated_path.parts[0] == "unit": + generated_path = Path(*generated_path.parts[1:]) + elif path.startswith(integration_prefix): + generated_path = Path("integration") / path.removeprefix(integration_prefix) else: - trtllm_import = TRTLLM_IMPORT_RE.search(expected) - if trtllm_import is not None: - ensure_imports(trtllm_import.start(), "os", "pytest") - insert_optional_trtllm_guard() - expected = expected.replace( - ' script_dir = Path(root_dir, "benchmarks", "cpp")\n', - " script_dir = Path(temp_dir)\n", + raise AssertionError(f"Unhandled AutoDeploy test path: {source_path}") + return generated_path.with_name( + GENERATED_TEST_RENAMES.get(generated_path.name, generated_path.name) ) - return expected @pytest.fixture(scope="module") @@ -162,30 +116,13 @@ def generated_package(tmp_path_factory: pytest.TempPathFactory) -> Path: return output_dir -@pytest.fixture(scope="module") -def generated_singlegpu_tests(generated_package: Path) -> Path: - """Return the generated single-GPU test directory.""" - return generated_package / "tests" / "singlegpu" - - -@pytest.fixture(scope="module") -def generated_multigpu_tests(generated_package: Path) -> Path: - """Return the generated multi-GPU test directory.""" - return generated_package / "tests" / "multigpu" - - -@pytest.fixture(scope="module") -def generated_torch_unit_tests(generated_package: Path) -> Path: - """Return the generated _torch/auto_deploy unit test directory.""" - return generated_package / "tests" / "_torch" / "auto_deploy" - - def test_generates_paragraf_package_identity(generated_package: Path) -> None: assert (generated_package / "paragraf" / "__init__.py").is_file() assert (generated_package / "llmc").is_symlink() pyproject = (generated_package / "pyproject.toml").read_text() assert 'name = "nvidia-llmc"' in pyproject assert 'include = ["paragraf*", "llmc"]' in pyproject + assert 'trtllm = ["tensorrt-llm"]' in pyproject def test_legacy_generator_entrypoint(tmp_path: Path) -> None: @@ -201,37 +138,27 @@ def test_legacy_generator_entrypoint(tmp_path: Path) -> None: assert (output_dir / "llmc").is_symlink() -def test_exports_exact_multigpu_allowlist(generated_multigpu_tests: Path) -> None: - exported_files = { - path.relative_to(generated_multigpu_tests) - for path in generated_multigpu_tests.rglob("*.py") +def test_exports_every_tracked_autodeploy_test(generated_package: Path) -> None: + generated_tests = generated_package / "tests" + missing = { + source_path + for source_path in _tracked_autodeploy_tests() + if not (generated_tests / _generated_test_path(source_path)).is_file() } - assert exported_files == EXPECTED_MULTIGPU_FILES - + assert not missing -def test_exports_exact_singlegpu_smoke_allowlist(generated_singlegpu_tests: Path) -> None: - exported_smoke_files = { - path.relative_to(generated_singlegpu_tests) - for path in (generated_singlegpu_tests / "smoke").rglob("*.py") - } - assert exported_smoke_files == EXPECTED_SINGLEGPU_SMOKE_FILES - -def test_exports_exact_torch_unit_allowlist(generated_torch_unit_tests: Path) -> None: - exported_files = { - path.relative_to(generated_torch_unit_tests) - for path in generated_torch_unit_tests.rglob("test*.py") - } - assert exported_files == EXPECTED_TORCH_UNIT_FILES +def test_does_not_export_generator_meta_tests(generated_package: Path) -> None: + assert not (generated_package / "tests" / "standalone").exists() -def test_registers_inert_threadleak_marker(generated_multigpu_tests: Path) -> None: - pyproject = generated_multigpu_tests.parents[1] / "pyproject.toml" +def test_registers_inert_threadleak_marker(generated_package: Path) -> None: + pyproject = generated_package / "pyproject.toml" assert '"threadleak(enabled): configure thread-leak checks' in pyproject.read_text() -def test_conftest_allows_explicit_redirect_mode(generated_multigpu_tests: Path) -> None: - conftest = generated_multigpu_tests.parent / "conftest.py" +def test_conftest_allows_explicit_redirect_mode(generated_package: Path) -> None: + conftest = generated_package / "tests" / "conftest.py" content = conftest.read_text() assert "TRTLLM_REDIRECT_AD_TO_PARAGRAF" in content assert "TRTLLM_REDIRECT_AD_TO_LLMC" in content @@ -245,7 +172,95 @@ def test_creates_minimal_utils_util_stub(generated_package: Path) -> None: util_stub = generated_package / "tests" / "utils" / "util.py" content = util_stub.read_text() assert "skip_pre_hopper" in content + assert "skip_no_hopper" in content + assert "skip_pre_blackwell" in content assert "torch.cuda.get_device_capability" in content + assert (generated_package / "tests" / "utils" / "cpp_paths.py").is_file() + assert (generated_package / "tests" / "_torch" / "helpers.py").is_file() + + +def test_copies_focused_integration_support(generated_package: Path) -> None: + integration_defs = generated_package / "tests" / "integration" / "defs" + assert (integration_defs / "conftest.py").is_file() + assert (integration_defs / "accuracy" / "accuracy_core.py").is_file() + assert (integration_defs / "disaggregated" / "disagg_test_utils.py").is_file() + assert (generated_package / "runners" / "trtllm" / "model_registry" / "models.yaml").is_file() + + +def test_uses_paragraf_test_layout(generated_package: Path) -> None: + generated_tests = generated_package / "tests" + assert not (generated_tests / "_torch" / "auto_deploy").exists() + assert not (generated_package / "examples" / "auto_deploy").exists() + assert not { + path.relative_to(generated_tests) + for path in generated_tests.rglob("*") + if "auto_deploy" in path.relative_to(generated_tests).parts + } + assert ( + generated_tests / "integration" / "defs" / "accuracy" / "test_llm_api_paragraf_trtllm.py" + ).is_file() + assert (generated_tests / "singlegpu" / "models" / "test_gpt_oss_modeling.py").is_file() + stale_names = { + path.name for path in generated_tests.rglob("test*.py") if AUTODEPLOY.search(path.name) + } + assert not stale_names + + +def test_keeps_layout_rewrites_out_of_source_tests() -> None: + llm_api_source = ( + REPO_ROOT / "tests" / "integration" / "defs" / "accuracy" / "test_llm_api_autodeploy.py" + ).read_text() + assert "'examples' / 'auto_deploy'" in llm_api_source + assert '"runners" / "trtllm"' not in llm_api_source + + mrope_source = ( + REPO_ROOT + / "tests" + / "unittest" + / "auto_deploy" + / "singlegpu" + / "transformations" + / "library" + / "test_mrope_delta_cache.py" + ).read_text() + assert "Path(__file__).resolve().parents[6]" in mrope_source + assert '_repo_root() / "tensorrt_llm" / "_torch" / "auto_deploy"' in mrope_source + + example_configs_source = ( + REPO_ROOT + / "tests" + / "unittest" + / "auto_deploy" + / "singlegpu" + / "utils" + / "test_example_configs.py" + ).read_text() + assert '_AD_EXAMPLES_DIR = _REPO_ROOT / "examples" / "auto_deploy"' in example_configs_source + + +def test_rewrites_generated_test_resource_layout(generated_package: Path) -> None: + generated_tests = generated_package / "tests" + llm_api = ( + generated_tests / "integration" / "defs" / "accuracy" / "test_llm_api_paragraf_trtllm.py" + ).read_text() + runner_registry = '"runners" / "trtllm" / "model_registry"' + assert runner_registry in llm_api + assert f'{runner_registry} / "configs" / "nano_v3.yaml"' in llm_api + assert f'{runner_registry} / "configs" / "super_v3.yaml"' in llm_api + + mrope = ( + generated_tests / "singlegpu" / "transformations" / "library" / "test_mrope_delta_cache.py" + ).read_text() + assert "Path(__file__).resolve().parents[4]" in mrope + assert '_repo_root() / "paragraf" / "config" / "default.yaml"' in mrope + assert runner_registry in mrope + + example_configs = ( + generated_tests / "singlegpu" / "utils" / "test_example_configs.py" + ).read_text() + assert ( + '_AD_EXAMPLES_DIR = _REPO_ROOT / "runners" / "trtllm" / "model_registry"' in example_configs + ) def test_does_not_add_test_guards_to_runners(generated_package: Path) -> None: @@ -262,48 +277,55 @@ def test_generates_legacy_runner_wrapper(generated_package: Path) -> None: assert "main()" in content -@pytest.mark.parametrize("relative_path", sorted(EXPECTED_SINGLEGPU_SMOKE_FILES, key=str), ids=str) -def test_rewrites_singlegpu_smoke_auto_deploy_imports( - generated_singlegpu_tests: Path, relative_path: Path -) -> None: - source = (AD_SINGLEGPU_TESTS / relative_path).read_text() - exported = (generated_singlegpu_tests / relative_path).read_text() - - assert exported == _expected_exported_test(source, relative_path) - assert CANONICAL_IMPORT not in exported - if relative_path in PARAGRAF_TRTLLM_RUNNER_TESTS: - assert BUILD_AND_RUN_AD_IMPORT not in exported - assert "build_and_run_paragraf_trtllm" in exported - else: - assert ( - "Paragraf optional TRT-LLM tests require TRTLLM_REDIRECT_AD_TO_PARAGRAF=true" - in exported - ) - assert "TRTLLM_REDIRECT_AD_TO_PARAGRAF" in exported - - -@pytest.mark.parametrize("relative_path", sorted(EXPECTED_MULTIGPU_FILES, key=str), ids=str) -def test_rewrites_multigpu_auto_deploy_imports( - generated_multigpu_tests: Path, relative_path: Path -) -> None: - source = (AD_MULTIGPU_TESTS / relative_path).read_text() - exported = (generated_multigpu_tests / relative_path).read_text() - - assert exported == _expected_exported_test(source, relative_path) - assert CANONICAL_IMPORT not in exported - if relative_path == PARAGRAF_TRTLLM_MULTIGPU_SMOKE_TEST: - assert BUILD_AND_RUN_AD_IMPORT not in exported - assert "build_and_run_paragraf_trtllm" in exported - assert "TRTLLM_REDIRECT_AD_TO_PARAGRAF" in exported - - -@pytest.mark.parametrize("relative_path", sorted(EXPECTED_TORCH_UNIT_FILES, key=str), ids=str) -def test_rewrites_torch_unit_auto_deploy_imports( - generated_torch_unit_tests: Path, relative_path: Path +def test_rewrites_all_canonical_auto_deploy_imports(generated_package: Path) -> None: + offenders = { + path.relative_to(generated_package) + for path in (generated_package / "tests").rglob("*.py") + if CANONICAL_IMPORT in path.read_text() + } + assert not offenders + + +@pytest.mark.parametrize( + "relative_path", + [ + Path("singlegpu/custom_ops/attention/test_trtllm_attention_op.py"), + Path("singlegpu/shim/test_engine.py"), + Path("multigpu/compile/test_bypass_captured_graphs.py"), + Path("singlegpu/models/test_gpt_oss_modeling.py"), + Path("integration/defs/examples/test_paragraf_trtllm_guided_decoding.py"), + ], + ids=str, +) +def test_guards_optional_trtllm_tests(generated_package: Path, relative_path: Path) -> None: + content = (generated_package / "tests" / relative_path).read_text() + assert OPTIONAL_TRTLLM_GUARD in content + assert 'pytest.importorskip("tensorrt_llm")' in content + + +@pytest.mark.parametrize( + "relative_path", + [ + Path("singlegpu/test_pattern_matcher.py"), + Path("singlegpu/transformations/library/test_mrope_delta_cache.py"), + Path("multigpu/custom_ops/test_dist.py"), + ], + ids=str, +) +def test_keeps_pure_standalone_tests_unguarded( + generated_package: Path, relative_path: Path ) -> None: - source = (AD_TORCH_TESTS / relative_path).read_text() - exported = (generated_torch_unit_tests / relative_path).read_text() - - assert exported == _expected_exported_test(source, relative_path) - assert CANONICAL_IMPORT not in exported - assert "Paragraf optional TRT-LLM tests require TRTLLM_REDIRECT_AD_TO_PARAGRAF=true" in exported + content = (generated_package / "tests" / relative_path).read_text() + assert OPTIONAL_TRTLLM_GUARD not in content + + +def test_rewrites_runner_imports_in_optional_tests(generated_package: Path) -> None: + content = ( + generated_package + / "tests" + / "singlegpu" + / "smoke" + / "test_paragraf_trtllm_build_small_single.py" + ).read_text() + assert "from build_and_run_ad import" not in content + assert "from runners.trtllm.build_and_run_paragraf_trtllm import" in content diff --git a/tests/unittest/bindings/binding_test_utils.py b/tests/unittest/bindings/binding_test_utils.py deleted file mode 100644 index e34316005e57..000000000000 --- a/tests/unittest/bindings/binding_test_utils.py +++ /dev/null @@ -1,58 +0,0 @@ -import logging as _log -import os as _os -import pathlib as _pl -import subprocess as _sp -import sys as _sys -import typing as _tp - -import numpy as _np - - -def run_command(command: _tp.Sequence[str], - cwd: _pl.Path, - *, - shell=False, - env=None) -> None: - _log.info("Running: cd %s && %s", str(cwd), " ".join(command)) - _sp.check_call(command, cwd=cwd, shell=shell, env=env) - - -def prepare_model_tests( - llm_root: _pl.Path, - resource_path: _pl.Path, - model_name: str, - model_cache_arg=[], -): - scripts_dir = resource_path / "scripts" - python_exe = _sys.executable - model_env = {**_os.environ, "PYTHONPATH": f"examples/{model_name}"} - build_engines = [ - python_exe, - str(scripts_dir / f"build_{model_name}_engines.py") - ] + model_cache_arg - run_command(build_engines, cwd=llm_root, env=model_env) - - model_env["PYTHONPATH"] = "examples" - generate_expected_output = [ - python_exe, - str(scripts_dir / f"generate_expected_{model_name}_output.py") - ] - run_command(generate_expected_output, cwd=llm_root, env=model_env) - - -def prepare_lora_configs(llm_root: _pl.Path, resource_path: _pl.Path, - lora_config_path: _pl.Path): - python_exe = _sys.executable - generate_lora_data_args_tp1 = [ - python_exe, - str(resource_path / "scripts" / "generate_test_lora_weights.py"), - f"--out-dir={str(lora_config_path)}", "--tp-size=1", - "--hidden-size=768", "--num-layers=12", "--config-ids-filter=0", - "--no-generate-cache-pages" - ] - run_command(generate_lora_data_args_tp1, cwd=llm_root) - - -def sequence_lengths(sequences: _np.ndarray, pad_id: int) -> _np.ndarray: - return _np.apply_along_axis(lambda x: _np.searchsorted(x, True), 1, - sequences == pad_id).astype("int32") diff --git a/tests/unittest/bindings/test_bindings_ut.py b/tests/unittest/bindings/test_bindings_ut.py index 210ea3378d3e..961748c25aad 100644 --- a/tests/unittest/bindings/test_bindings_ut.py +++ b/tests/unittest/bindings/test_bindings_ut.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + import json import pickle import tempfile @@ -417,6 +420,55 @@ def test_llm_request(): assert torch.equal(llm_request.draft_logits, logits) +def test_generation_only_llm_request_adopts_draft_tokens() -> None: + request_id = 42 + first_gen_tokens = [100] + draft_tokens = [101, 102, 103] + llm_request_type = _tb.internal.batch_manager.LlmRequestType + context_phase_params = _tb.executor.ContextPhaseParams( + first_gen_tokens, + request_id, + None, + draft_tokens, + None, + None, + ) + + llm_request = _tb.internal.batch_manager.LlmRequest( + request_id=request_id, + max_new_tokens=10, + sampling_config=_tb.SamplingConfig(1), + input_tokens=[1, 2, 3], + is_streaming=False, + llm_request_type=llm_request_type.LLMREQUEST_TYPE_GENERATION_ONLY, + context_phase_params=context_phase_params, + ) + + assert llm_request.is_generation_only_request + assert llm_request.has_draft_tokens() + assert llm_request.num_draft_tokens == len(draft_tokens) + assert llm_request.draft_tokens == draft_tokens + assert llm_request.context_phase_params.draft_tokens == draft_tokens + + late_llm_request = _tb.internal.batch_manager.LlmRequest( + request_id=request_id, + max_new_tokens=10, + sampling_config=_tb.SamplingConfig(1), + input_tokens=[1, 2, 3], + is_streaming=False, + llm_request_type=llm_request_type.LLMREQUEST_TYPE_GENERATION_ONLY, + ) + + assert late_llm_request.draft_tokens is None + assert late_llm_request.num_draft_tokens == 0 + + late_llm_request.context_phase_params = context_phase_params + + assert late_llm_request.has_draft_tokens() + assert late_llm_request.num_draft_tokens == len(draft_tokens) + assert late_llm_request.draft_tokens == draft_tokens + + def test_llm_request_kv_cache_transfer_metric_bindings(): request = _tb.internal.batch_manager.LlmRequest( request_id=0, diff --git a/tests/unittest/bindings/test_datatype_parity.py b/tests/unittest/bindings/test_datatype_parity.py new file mode 100644 index 000000000000..d3ca39f7d962 --- /dev/null +++ b/tests/unittest/bindings/test_datatype_parity.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Serialization/format-compatibility guard for the internal data-type enum. + +``tensorrt_llm.bindings.DataType`` is backed by ``tensorrt_llm::DataType`` +(``common/tllmDataType.h``), currently an alias of ``nvinfer1::DataType`` and +slated to become a standalone enum when the TensorRT-engine execution path is +removed. Its enumerator *integer values* must continue to match the legacy +``nvinfer1::DataType`` values so that previously-serialized executor configs +and KV-cache metadata remain byte-compatible, and so the public Python member +set is unchanged. +""" + +import pytest + +bindings = pytest.importorskip("tensorrt_llm.bindings") + +# Legacy nvinfer1::DataType integer values. These MUST remain stable. +LEGACY_DATATYPE_VALUES = { + "FLOAT": 0, + "HALF": 1, + "INT8": 2, + "INT32": 3, + "BOOL": 4, + "UINT8": 5, + "FP8": 6, + "BF16": 7, + "INT64": 8, + "NVFP4": 10, +} + + +def test_bindings_datatype_values_match_legacy(): + """Each exposed DataType enumerator keeps its legacy numeric value.""" + dt = bindings.DataType + for name, value in LEGACY_DATATYPE_VALUES.items(): + assert hasattr(dt, name), f"tensorrt_llm.bindings.DataType is missing '{name}'" + assert getattr(dt, name).value == value, ( + f"DataType.{name} numeric value changed to {getattr(dt, name).value}; " + f"expected {value} (breaks serialization compatibility)" + ) + + +def test_bindings_datatype_member_set_unchanged(): + """The public DataType member set still contains all legacy members.""" + members = {m for m in dir(bindings.DataType) if not m.startswith("_")} + missing = set(LEGACY_DATATYPE_VALUES) - members + assert not missing, f"tensorrt_llm.bindings.DataType lost members: {sorted(missing)}" diff --git a/tests/unittest/bindings/test_executor_bindings.py b/tests/unittest/bindings/test_executor_bindings.py index 438101bba3da..324126d02b84 100644 --- a/tests/unittest/bindings/test_executor_bindings.py +++ b/tests/unittest/bindings/test_executor_bindings.py @@ -1,854 +1,19 @@ import datetime +import inspect import json import os as _os import pickle -import random -import sys as _sys -import time import typing as tp -from pathlib import Path import numpy as np import pytest import torch -from binding_test_utils import * from pydantic import BaseModel import tensorrt_llm.bindings.executor as trtllm import tensorrt_llm.version as trtllm_version -from tensorrt_llm._utils import torch_to_numpy from tensorrt_llm.models.modeling_utils import PretrainedConfig -_sys.path.append(_os.path.join(_os.path.dirname(__file__), '..')) -import inspect - -from utils.cpp_paths import * -from utils.llm_data import llm_models_root -from utils.util import skip_pre_hopper - - -@pytest.fixture -def model_files(llm_root: Path, resource_path: Path, results_data_path: Path): - # Model engines and expected outputs need to be generated. - if not results_data_path.exists(): - model_cache = llm_models_root() - model_cache_arg = ["--model_cache", str(model_cache) - ] if model_cache is not None else [] - prepare_model_tests(llm_root, resource_path, "gpt", model_cache_arg) - - -@pytest.fixture -def lora_config_paths(llm_root: Path, resource_path: Path, - lora_config_path: Path): - if not lora_config_path.exists(): - prepare_lora_configs(llm_root, resource_path, lora_config_path) - return (lora_config_path / "source.npy", lora_config_path / "config.npy") - - -def get_expected_num_tokens(prompt_len, max_tokens, streaming, - exclude_input_from_output): - if not streaming and not exclude_input_from_output: - return prompt_len + max_tokens - return max_tokens - - -def test_executor_valid_ctor(model_files, model_path): - executor_config = trtllm.ExecutorConfig( - 1, kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - -def test_executor_from_memory(model_files, model_path): - executor_config = trtllm.ExecutorConfig( - 1, kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - engine_buffer = open(model_path / "rank0.engine", mode="rb").read() - json_config_str = open(model_path / "config.json", 'r').read() - executor = trtllm.Executor(engine_buffer, json_config_str, - trtllm.ModelType.DECODER_ONLY, executor_config) - - -def test_executor_with_managed_weights(model_files, model_path): - """Test executor constructor with standard dtypes in managed weights.""" - - executor_config = trtllm.ExecutorConfig( - 1, kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - engine_buffer = open(model_path / "rank0.engine", mode="rb").read() - json_config_str = open(model_path / "config.json", 'r').read() - - managed_weights = { - "weight_float32": - np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32), - "weight_int32": - np.array([[1, 2], [3, 4]], dtype=np.int32), - "weight_int64": - np.array([[1, 2], [3, 4]], dtype=np.int64), - "weight_int8": - np.array([[1, 2], [3, 4]], dtype=np.int8), - "weight_fp16": - np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float16), - "weight_bf16": - torch_to_numpy( - torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.bfloat16)), - "weight_fp8": - torch_to_numpy( - torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float8_e4m3fn)), - } - - executor = trtllm.Executor(engine_buffer, json_config_str, - trtllm.ModelType.DECODER_ONLY, executor_config, - managed_weights) - - assert executor.can_enqueue_requests() == True - - -def test_executor_invalid_ctor(): - executor_config = trtllm.ExecutorConfig( - 1, kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - invalid_path = "Bla" - try: - executor = trtllm.Executor(invalid_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - assert False, "Expected an error" - except Exception as e: - assert "File does not exist" in str(e) - - -def test_shutdown(model_files, model_path): - - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - # Create the request - max_tokens = 5 - input_tokens = [1, 2, 3, 4] - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - streaming=False, - sampling_config=trtllm.SamplingConfig()) - - # Enqueue the request - assert executor.can_enqueue_requests() == True - req_id = executor.enqueue_request(request) - - executor.shutdown() - assert executor.can_enqueue_requests() == False - - with pytest.raises(Exception): - executor.enqueue_request(request) - with pytest.raises(Exception): - executor.await_responses() - with pytest.raises(Exception): - executor.get_latest_iteration_stats() - with pytest.raises(Exception): - executor.get_latest_request_stats() - with pytest.raises(Exception): - executor.get_latest_debug_tensors() - with pytest.raises(Exception): - executor.cancel_request(req_id) - with pytest.raises(Exception): - executor.get_num_responses_ready(req_id) - - -def test_embedding_bias(model_files, model_path): - streaming = False - exclude_input_from_output = False - output_config = trtllm.OutputConfig() - output_config.exclude_input_from_output = exclude_input_from_output - - # Create executor - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - # Create the request - max_tokens = 5 - input_tokens = [1, 2, 3, 4] - # Set embedding bias so "biased_output" is always picked - biased_output = 10 - vocab_size_padded = 50257 - embedding_bias = torch.zeros(vocab_size_padded) - embedding_bias[biased_output] = torch.finfo(torch.float32).max - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - streaming=streaming, - sampling_config=trtllm.SamplingConfig(), - output_config=output_config, - embedding_bias=embedding_bias) - - # Enqueue the request - request_id = executor.enqueue_request(request) - - # Get the new tokens - tokens = [] - done = False - i = 0 - max_wait_ms = 10000 - while not done and i < max_wait_ms: - wait_time = datetime.timedelta(milliseconds=1) - responses = executor.await_responses(request_id, wait_time) - for response in responses: - assert not response.has_error( - ), f"Request id {request_id} failed with err {response.error_msg}" - result = response.result - done = result.is_final - new_tokens = result.output_token_ids[beam_width - 1] - tokens.extend(new_tokens) - i += 1 - assert i < max_wait_ms - assert len(tokens) == get_expected_num_tokens( - len(input_tokens), max_tokens, streaming, - exclude_input_from_output), f"{request_id}" - # All generated tokens should equal biased_output - assert tokens[-max_tokens:] == [biased_output] * max_tokens - - -@pytest.mark.parametrize("streaming", [False, True]) -@pytest.mark.parametrize("exclude_input_from_output", [False]) -def test_single_request(streaming: bool, exclude_input_from_output: bool, - model_files, model_path): - output_config = trtllm.OutputConfig() - output_config.exclude_input_from_output = exclude_input_from_output - - # Create executor - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - # Create the request - max_tokens = 5 - input_tokens = [1, 2, 3, 4] - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - streaming=streaming, - sampling_config=trtllm.SamplingConfig(), - output_config=output_config) - - # Enqueue the request - request_id = executor.enqueue_request(request) - - # Get the new tokens - tokens = [] - done = False - i = 0 - max_wait_ms = 10000 - while not done and i < max_wait_ms: - wait_time = datetime.timedelta(milliseconds=1) - responses = executor.await_responses(request_id, wait_time) - for response in responses: - assert not response.has_error( - ), f"Request id {request_id} failed with err {response.error_msg}" - result = response.result - done = result.is_final - new_tokens = result.output_token_ids[beam_width - 1] - tokens.extend(new_tokens) - i += 1 - assert i < max_wait_ms - assert len(tokens) == get_expected_num_tokens( - len(input_tokens), max_tokens, streaming, - exclude_input_from_output), f"{request_id}" - - executor.get_latest_iteration_stats() - executor.get_latest_request_stats() - executor.get_latest_debug_tensors() - - -def test_single_request_lora(model_files, model_path_lora, lora_config_paths): - streaming = False - exclude_input_from_output = False - output_config = trtllm.OutputConfig() - output_config.exclude_input_from_output = exclude_input_from_output - - # Create executor - beam_width = 1 - - peft_cache_config = trtllm.PeftCacheConfig(num_put_workers=4, - num_ensure_workers=4) - executor_config = trtllm.ExecutorConfig( - 1, - peft_cache_config=peft_cache_config, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path_lora, trtllm.ModelType.DECODER_ONLY, - executor_config) - - # Create the request - max_tokens = 5 - input_tokens = [1, 2, 3, 4] - lora_weights = torch.tensor(np.load(lora_config_paths[0])).half() - lora_config = torch.tensor(np.load(lora_config_paths[1])) - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - streaming=streaming, - sampling_config=trtllm.SamplingConfig(), - output_config=output_config, - lora_config=trtllm.LoraConfig( - 0, lora_weights, lora_config)) - - # Enqueue the request - request_id = executor.enqueue_request(request) - - # Get the new tokens - tokens = [] - done = False - i = 0 - max_wait_ms = 10000 - while not done and i < max_wait_ms: - wait_time = datetime.timedelta(milliseconds=1) - responses = executor.await_responses(request_id, wait_time) - for response in responses: - assert not response.has_error( - ), f"Request id {request_id} failed with err {response.error_msg}" - result = response.result - done = result.is_final - new_tokens = result.output_token_ids[beam_width - 1] - tokens.extend(new_tokens) - i += 1 - assert i < max_wait_ms - assert len(tokens) == get_expected_num_tokens( - len(input_tokens), max_tokens, streaming, - exclude_input_from_output), f"{request_id}" - - -@pytest.mark.parametrize("streaming", [False, True]) -@pytest.mark.parametrize("exclude_input_from_output", [False]) -def test_multi_request(streaming: bool, exclude_input_from_output: bool, - model_files, model_path): - output_config = trtllm.OutputConfig() - output_config.exclude_input_from_output = exclude_input_from_output - - # Create executor - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - num_requests = 20 - max_prompt_len = 20 - max_max_tokens = 20 - end_id = -1 - - # Enqueue the requests - tokens = {} - expected_num_tokens = {} - for i in range(num_requests): - prompt_len = random.randint(1, max_prompt_len) - max_tokens = random.randint(1, max_max_tokens) - input_tokens = [1] * prompt_len - - # Some requests has num_return_sequences > 1. - num_return_sequences = 2 if i % 5 == 1 else 1 - - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - streaming=streaming, - sampling_config=trtllm.SamplingConfig( - num_return_sequences=num_return_sequences), - output_config=output_config, - end_id=end_id) - request_id = executor.enqueue_request(request) - tokens[request_id] = [ - [] for _ in range(request.sampling_config.num_return_sequences) - ] - expected_num_tokens[request_id] = get_expected_num_tokens( - prompt_len, max_tokens, streaming, exclude_input_from_output) - - # Get the new tokens for each request - num_finished = 0 - i = 0 - num_responses = 0 - max_wait_ms = 10000 - while num_finished < num_requests and i < max_wait_ms: - wait_time = datetime.timedelta(milliseconds=1) - responses = executor.await_responses(wait_time) - for response in responses: - num_responses += 1 - assert not response.has_error( - ), f"Request id {response.request_id} failed with err {response.error_msg}" - result = response.result - num_finished += result.is_final - new_tokens = result.output_token_ids[beam_width - 1] - tokens[response.request_id][result.sequence_index].extend( - new_tokens) - i += 1 - assert i < max_wait_ms - - for request_id in expected_num_tokens: - for actual_tokens in tokens[request_id]: - assert len(actual_tokens) == expected_num_tokens[request_id] - - -@pytest.mark.parametrize("streaming", [False, True]) -@pytest.mark.parametrize("exclude_input_from_output", [False]) -def test_multi_request_with_ids(streaming: bool, - exclude_input_from_output: bool, model_files, - model_path): - output_config = trtllm.OutputConfig() - output_config.exclude_input_from_output = exclude_input_from_output - - # Create executor - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - num_requests = 20 - max_prompt_len = 20 - max_max_tokens = 20 - end_id = -1 - - # Enqueue the requests - tokens = {} - expected_num_tokens = {} - for i in range(num_requests): - prompt_len = random.randint(1, max_prompt_len) - max_tokens = random.randint(1, max_max_tokens) - input_tokens = [1] * prompt_len - num_return_sequences = 2 if i % 5 == 1 else 1 - - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - streaming=streaming, - sampling_config=trtllm.SamplingConfig( - num_return_sequences=num_return_sequences), - output_config=output_config, - end_id=end_id) - request_id = executor.enqueue_request(request) - tokens[request_id] = [ - [] for _ in range(request.sampling_config.num_return_sequences) - ] - expected_num_tokens[request_id] = get_expected_num_tokens( - prompt_len, max_tokens, streaming, exclude_input_from_output) - - # Get the new tokens for each request - num_finished = 0 - i = 0 - num_responses = 0 - max_wait_ms = 10000 - while num_finished < num_requests and i < max_wait_ms: - wait_time = datetime.timedelta(milliseconds=1) - id_responses = executor.await_responses(list(tokens.keys()), wait_time) - for responses in id_responses: - for response in responses: - num_responses += 1 - # Allow response with error only if await_response processed a terminated request id - if response.has_error(): - terminated_request_error = "ReqId " + str( - response.request_id - ) + " has already been processed and was terminated." - assert response.error_msg == terminated_request_error, ( - f"Request id {response.request_id} failed with err " - f"{response.error_msg}") - else: - result = response.result - num_finished += result.is_final - new_tokens = result.output_token_ids[beam_width - 1] - tokens[response.request_id][result.sequence_index].extend( - new_tokens) - i += 1 - assert i < max_wait_ms - - for request_id in expected_num_tokens: - for seq_idx, actual_tokens in enumerate(tokens[request_id]): - assert len(actual_tokens) == expected_num_tokens[request_id] - - -@pytest.mark.parametrize("streaming", [False, True]) -@pytest.mark.parametrize("exclude_input_from_output", [False]) -def test_get_num_responses_ready(streaming: bool, - exclude_input_from_output: bool, model_files, - model_path): - output_config = trtllm.OutputConfig() - output_config.exclude_input_from_output = exclude_input_from_output - - # Create executor - executor_config = trtllm.ExecutorConfig( - 1, kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - max_prompt_len = 20 - max_max_tokens = 20 - - # Enqueue the requests - num_requests = random.randint(1, 50) - num_expected_responses = 0 - req_num_expected_responses = {} - for i in range(num_requests): - prompt_len = random.randint(1, max_prompt_len) - max_tokens = random.randint(1, max_max_tokens) - num_return_sequences = 2 if i % 5 == 1 else 1 - - request = trtllm.Request([1] * prompt_len, - max_tokens=max_tokens, - streaming=streaming, - sampling_config=trtllm.SamplingConfig( - num_return_sequences=num_return_sequences), - output_config=output_config) - request_id = executor.enqueue_request(request) - req_num_expected_responses[request_id] = ( - (max_tokens if streaming else 1) * num_return_sequences) - num_expected_responses += req_num_expected_responses[request_id] - - i = 0 - num_ready = 0 - max_wait_ms = 10000 - while num_ready < num_expected_responses and i < max_wait_ms: - num_ready = 0 - for request_id in req_num_expected_responses: - num_ready += executor.get_num_responses_ready(request_id) - time.sleep(0.001) - i += 1 - assert i < max_wait_ms - - for request_id in req_num_expected_responses: - num_ready = executor.get_num_responses_ready(request_id) - assert num_ready == req_num_expected_responses[request_id] - assert executor.get_num_responses_ready() == num_expected_responses - - -@pytest.mark.parametrize("batching_type", [trtllm.BatchingType.INFLIGHT]) -@pytest.mark.parametrize("streaming", [False, True]) -@pytest.mark.parametrize("beam_width", [1]) -@pytest.mark.parametrize("compute_log_probs", [False, True]) -@pytest.mark.parametrize("exclude_input_from_output", [False]) -@pytest.mark.parametrize("return_context_logits", [False, True]) -@pytest.mark.parametrize("return_generation_logits", [False, True]) -def test_token_comparison(batching_type: trtllm.BatchingType, streaming: bool, - beam_width: int, compute_log_probs: bool, - exclude_input_from_output: bool, - return_context_logits: bool, - return_generation_logits: bool, model_files, - model_path, model_path_return_logits, input_data_path, - results_data_path, results_data_path_beam_width_2): - if streaming and beam_width > 1: - pytest.skip("Test does not support streaming with beam search") - - vocab_size_padded = 50257 - pad_id = 50256 - remove_input = not exclude_input_from_output and not streaming - - def load_test_data(input_path, results_path): - # Inputs - assert input_path.is_file() - given_input = np.load(input_path).astype("int32") - input_shape = given_input.shape - assert len(input_shape) == 2 - max_input_length = input_shape[1] - given_input_lengths = sequence_lengths(given_input, pad_id) - assert np.all(given_input_lengths <= max_input_length) - # Expected results - assert results_path.is_file() - expected_outputs = np.load(results_path).astype("int32") - output_shape = expected_outputs.shape - assert len(output_shape) == 2 - assert input_shape[0] * beam_width == output_shape[0] - max_seq_length = output_shape[1] - max_tokens = max_seq_length - max_input_length - - end_ids = [pad_id for _ in range(len(given_input_lengths))] - expected_lengths = [] - for i in range(len(given_input_lengths)): - expected_lengths.append([ - given_input_lengths[i] + max_tokens for _ in range(beam_width) - ]) - - test_data = { - "expected_output_ids": expected_outputs, - "expected_output_lengths": expected_lengths, - "max_seq_length": max_seq_length, - "end_ids": end_ids - } - return given_input, given_input_lengths, max_input_length, test_data - - def validate_results_shapes(result, input_length, max_output_len, - beam_tokens): - if compute_log_probs: - assert result.cum_log_probs is not None - assert result.log_probs is not None - assert len(result.cum_log_probs) == beam_width - assert len(result.log_probs) == beam_width - for beam in range(beam_width): - expected_len = len( - beam_tokens[beam]) - (input_length if remove_input else 0) - assert len(result.log_probs[beam]) == expected_len - else: - assert result.cum_log_probs is None - assert result.log_probs is None - if return_context_logits: - assert result.context_logits is not None - assert len(result.context_logits.shape) == 2 - assert list(result.context_logits.shape) == [ - input_length, vocab_size_padded - ] - else: - assert result.context_logits is None - if return_generation_logits: - assert len(result.generation_logits.shape) == 3 - if streaming: - assert list(result.generation_logits.shape) == [ - max_output_len, beam_width, vocab_size_padded - ] or list(result.generation_logits.shape) == [ - 1, beam_width, vocab_size_padded - ] - else: - assert list(result.generation_logits.shape) == [ - beam_width, max_output_len, vocab_size_padded - ] - - def verify_output(beam_tokens, test_data, given_input_lengths): - - for batch_id, seq_tokens in beam_tokens.items(): - input_length = given_input_lengths[batch_id] - end_id = test_data["end_ids"][batch_id] - for tokens in seq_tokens: - for beam in range(beam_width): - - predicted_tokens = tokens[beam] - if remove_input: - predicted_tokens = predicted_tokens[input_length:] - expected_length = test_data["expected_output_lengths"][ - batch_id][beam] - input_length - assert len(predicted_tokens) == expected_length - - expected_tokens = test_data["expected_output_ids"][ - batch_id * beam_width + beam][input_length:] - - # From experiments find out when set return_context_logits - # or return_generation_logits, the predicted_tokens cannot match with expected_tokens - # Fixed by comparing partial output tokens like in c++ test - compare_length = 2 if ( - return_context_logits - or return_generation_logits) else len(predicted_tokens) - - for i in range(compare_length): - if expected_tokens[i] == end_id: - break - # Predicted: [21221, 290, 373, 257, 2888, 286, 262, 4141] - # Expected: [21221, 290, 257, 4255, 379, 262, 1957, 7072] - # generation logits are almost same at token ids 257 and 373, - # which causes unstable generation results. - assert predicted_tokens[i] == expected_tokens[i], \ - f"Predicted: {predicted_tokens} vs Expected: {expected_tokens}" - - output_config = trtllm.OutputConfig() - output_config.exclude_input_from_output = exclude_input_from_output - output_config.return_log_probs = compute_log_probs - output_config.return_generation_logits = return_generation_logits - output_config.return_context_logits = return_context_logits - # Change free_gpu_memory_fraction to solve OOM error - kv_cache_config = trtllm.KvCacheConfig(False, free_gpu_memory_fraction=0.3) - executor_config = trtllm.ExecutorConfig(beam_width) - executor_config.batching_type = batching_type - executor_config.kv_cache_config = kv_cache_config - if return_generation_logits: - executor_config.gather_generation_logits = True - - if return_context_logits or return_generation_logits: - model_path = model_path_return_logits - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - # Load test data - results_path = results_data_path if beam_width == 1 else results_data_path_beam_width_2 - given_input, given_input_lengths, max_input_length, test_data = load_test_data( - input_data_path, results_path) - - # Create requests from input data - num_requests = len(given_input_lengths) - requests = [] - req_max_tokens = [] - - for i in range(num_requests): - input_len = given_input_lengths[i] - max_tokens = test_data["max_seq_length"] - max_input_length - req_max_tokens.append(max_tokens) - req_tokens = given_input[i][:input_len] - num_return_sequences = 2 if i % 5 == 1 else 1 - request = trtllm.Request(req_tokens, - max_tokens=max_tokens, - streaming=streaming, - sampling_config=trtllm.SamplingConfig( - beam_width, - num_return_sequences=num_return_sequences), - output_config=output_config, - end_id=-1) - requests.append(request) - - req_ids = executor.enqueue_requests(requests) - - req_to_batch_id = {req_ids[i]: i for i in range(len(requests))} - tokens = { - i: [[[] for _ in range(beam_width)] - for _ in range(req.sampling_config.num_return_sequences)] - for i, req in enumerate(requests) - } - - num_finished = 0 - i = 0 - num_responses = 0 - max_wait_ms = 10000 - while num_finished < num_requests and i < max_wait_ms: - wait_time = datetime.timedelta(milliseconds=1) - responses = executor.await_responses(wait_time) - for response in responses: - num_responses += 1 - assert not response.has_error( - ), f"Request id {response.request_id} failed with err {response.error_msg}" - result = response.result - num_finished += result.is_final - - batch_id = req_to_batch_id[response.request_id] - for beam in range(beam_width): - new_tokens = result.output_token_ids[beam] - tokens[batch_id][result.sequence_index][beam] += new_tokens - - validate_results_shapes(result, given_input_lengths[batch_id], - req_max_tokens[batch_id], - tokens[batch_id][result.sequence_index]) - i += 1 - assert i < max_wait_ms - verify_output(tokens, test_data, given_input_lengths) - - -@pytest.mark.parametrize("streaming", [False, True]) -@pytest.mark.parametrize("beam_width", [1]) -def test_finish_reason(streaming: bool, beam_width: int, model_files, - model_path): - if streaming and beam_width > 1: - pytest.skip("Test does not support streaming with beam search") - executor = trtllm.Executor( - model_path, trtllm.ModelType.DECODER_ONLY, - trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5))) - requests = [ - # Finish due to length. - trtllm.Request([1, 2, 3, 4], - max_tokens=5, - streaming=streaming, - sampling_config=trtllm.SamplingConfig(beam_width)), - # Finish due to end id. - trtllm.Request([1, 2, 3, 4], - max_tokens=5, - streaming=streaming, - sampling_config=trtllm.SamplingConfig(beam_width), - end_id=4), - # Finish due to stop word. - trtllm.Request([1, 2, 3, 4], - max_tokens=5, - streaming=streaming, - sampling_config=trtllm.SamplingConfig(beam_width), - stop_words=[[4, 2]]), - ] - req_ids = executor.enqueue_requests(requests) - req_to_batch_id = {req_ids[i]: i for i in range(len(requests))} - - num_finished = 0 - i = 0 - num_responses = 0 - max_wait_ms = 10000 - while num_finished < len(requests) and i < max_wait_ms: - wait_time = datetime.timedelta(milliseconds=1) - responses = executor.await_responses(wait_time) - for response in responses: - num_responses += 1 - assert not response.has_error( - ), f"Request id {response.request_id} failed with err {response.error_msg}" - result = response.result - num_finished += result.is_final - batch_id = req_to_batch_id[response.request_id] - - # Non final results should have "NOT_FINISHED". Revise this when streaming + beam_width > 1 is enabled. - if not result.is_final: - assert all([ - r == trtllm.FinishReason.NOT_FINISHED - for r in result.finish_reasons - ]) - # Check if finish reason is correct. - elif batch_id == 0: - assert all([ - r == trtllm.FinishReason.LENGTH - for r in result.finish_reasons - ]) - elif batch_id == 1: - assert all([ - r == trtllm.FinishReason.END_ID - for r in result.finish_reasons - ]) - elif batch_id == 2: - assert all([ - r == trtllm.FinishReason.STOP_WORDS - for r in result.finish_reasons - ]) - i += 1 - assert i < max_wait_ms - - -def test_gpt_executor_timed_out(model_files, model_path): - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - # No requests enqueued, expect no responses - num_responses_ready = executor.get_num_responses_ready() - assert num_responses_ready == 0 - - wait_time = datetime.timedelta(milliseconds=10) - responses = executor.await_responses(wait_time) - assert len(responses) == 0 - - -def test_single_request_invalid_inputs(model_files, model_path): - streaming = True - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - max_tokens = 5 - input_tokens = [1, 2, 3, 4] - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - streaming=streaming) - # Invalid embedding bias shape - embedding_bias = torch.ones(1) - request.embedding_bias = embedding_bias - expected_error_msg = "embedding bias shape is not as expected" - - request_id = executor.enqueue_request(request) - - done = False - i = 0 - max_wait_ms = 10000 - while not done and i < max_wait_ms: - wait_time = datetime.timedelta(milliseconds=1) - responses = executor.await_responses(request_id, wait_time) - for response in responses: - assert response.has_error(), "Expected an error" - assert expected_error_msg in response.error_msg - done = True - i += 1 - assert done - def test_sampling_config(): beam_width = 1 @@ -1149,36 +314,6 @@ def test_lora_config(): assert (lora_config.config == config).all() -def test_wakeup(model_files, model_path): - import threading - - def resp_thread(stop_signal: threading.Event, executor: trtllm.Executor): - while not stop_signal.is_set(): - timeout = None - responses = executor.await_responses(timeout=timeout) - if stop_signal.is_set(): - return - for response in responses: - response.result.output_token_ids - - executor = trtllm.Executor( - model_path, trtllm.ModelType.DECODER_ONLY, - trtllm.ExecutorConfig(kv_cache_config=trtllm.KvCacheConfig( - free_gpu_memory_fraction=0.5))) - stop_signal = threading.Event() - thread = threading.Thread(target=resp_thread, args=(stop_signal, executor)) - thread.start() - request = trtllm.Request(input_token_ids=[1, 2, 3, 4], - max_tokens=5, - streaming=True) - executor.enqueue_request(request) - time.sleep(2) - stop_signal.set() - executor.shutdown() - thread.join() - assert not thread.is_alive() - - def test_guided_decoding_params(): guided_decoding_params = trtllm.GuidedDecodingParams( trtllm.GuidedDecodingParams.GuideType.JSON) @@ -1698,7 +833,7 @@ def test_decoding_mode(): assert mode.isEagle() -def test_speculative_decoding_config(): +def test_decoding_config(): config = trtllm.DecodingConfig() assert config.decoding_mode is None assert config.lookahead_decoding_config is None @@ -1975,427 +1110,6 @@ def test_peft_cache_config(): assert peft_cache_config.lora_prefetch_dir == lora_prefetch_dir -def test_logits_post_processor(model_files, model_path): - - # Define the logits post-processor callback - def logits_post_processor(req_id: int, logits: torch.Tensor, - ids: tp.List[tp.List[int]], stream_ptr: int, - client_id: tp.Optional[int]): - assert client_id == 123 - with torch.cuda.stream(torch.cuda.ExternalStream(stream_ptr)): - logits[:] = float("-inf") - logits[..., 42] = 0 - - # Create executor - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor_config.logits_post_processor_config = trtllm.LogitsPostProcessorConfig( - {"my_logits_pp": logits_post_processor}) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - # Create the request - max_tokens = 5 - input_tokens = [1, 2, 3, 4] - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - streaming=False, - client_id=123) - request.logits_post_processor_name = "my_logits_pp" - - # Enqueue the request - request_id = executor.enqueue_request(request) - - # Get the new tokens - tokens = [] - done = False - i = 0 - max_wait_ms = 10000 - while not done and i < max_wait_ms: - wait_time = datetime.timedelta(milliseconds=1) - responses = executor.await_responses(request_id, wait_time) - for response in responses: - assert not response.has_error( - ), f"Request id {request_id} failed with err {response.error_msg}" - result = response.result - done = result.is_final - new_tokens = result.output_token_ids[beam_width - 1] - tokens.extend(new_tokens) - i += 1 - assert i < max_wait_ms - assert len(tokens) == get_expected_num_tokens(len(input_tokens), max_tokens, - False, False), f"{request_id}" - - # check that all output tokens are 42 - print(tokens) - assert tokens[-max_tokens:] == [42] * max_tokens - - -def test_logits_post_processor_batched(model_files, model_path): - - # Define the logits post-processor callback - def logits_post_processor_batched( - req_id_batch: tp.List[int], logits_batch: tp.List[torch.Tensor], - ids_batch: tp.List[tp.List[tp.List[int]]], stream_ptr: int, - client_id_batch: tp.List[tp.Optional[int]]): - for client_id in client_id_batch: - assert client_id == 123 - with torch.cuda.stream(torch.cuda.ExternalStream(stream_ptr)): - for logits in logits_batch: - logits[:] = float("-inf") - logits[..., 42] = 0 - - # Create executor - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor_config.logits_post_processor_config = trtllm.LogitsPostProcessorConfig( - None, logits_post_processor_batched) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - # Create the request - max_tokens = 5 - input_tokens = [1, 2, 3, 4] - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - streaming=False, - client_id=123) - request.logits_post_processor_name = request.BATCHED_POST_PROCESSOR_NAME - - batch_size = 4 - # Enqueue the requests - request_ids = [] - for _ in range(batch_size): - request_id = executor.enqueue_request(request) - request_ids.append(request_id) - - # Get the new tokens - tokens = {req_id: [] for req_id in request_ids} - num_finished = 0 - i = 0 - max_wait_ms = 10000 - while num_finished < len(request_ids) and i < max_wait_ms: - responses = executor.await_responses(datetime.timedelta(milliseconds=1)) - for response in responses: - req_id = response.request_id - assert not response.has_error( - ), f"Request id {req_id} failed with err {response.error_msg}" - result = response.result - num_finished += 1 if result.is_final else 0 - new_tokens = result.output_token_ids[beam_width - 1] - tokens[req_id].extend(new_tokens) - assert i < max_wait_ms - - expected_num_tokens = get_expected_num_tokens(len(input_tokens), max_tokens, - False, False) - for req_id in request_ids: - assert len(tokens[req_id]) == expected_num_tokens, f"{req_id}" - - -@pytest.mark.skip("https://nvbugs/5082576") -def test_kv_event_stream(model_path): - - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(True, - 4 * 64, - event_buffer_max_size=1024, - host_cache_size=3000000, - free_gpu_memory_fraction=0.5)) - - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - cache_manager = executor.get_kv_cache_event_manager() - - events = cache_manager.get_latest_events() - - assert len(events) == 1 - assert isinstance(events[0], trtllm.kv_cache.KVCacheEvent) - assert events[0].event_id == 0 - assert isinstance(events[0].data, trtllm.kv_cache.KVCacheCreatedData) - - for req in range(2): - input_tokens = list(range(req, req + 127)) - request = trtllm.Request(input_tokens, - max_tokens=5, - streaming=False, - sampling_config=trtllm.SamplingConfig()) - - id = executor.enqueue_request(request) - - responses = executor.await_responses(id) - - for response in responses: - assert not response.has_error() - if response.result.is_final: - time.sleep(0.1) - events = cache_manager.get_latest_events( - datetime.timedelta(milliseconds=100)) - - if req == 0: - assert events[0].event_id == 1 - assert isinstance(events[0].data, - trtllm.kv_cache.KVCacheStoredData) - assert events[0].data.parent_hash is None - assert len(events[0].data.blocks) == 1 - - assert events[1].data.parent_hash == events[0].data.blocks[ - 0].block_hash - assert len(events[1].data.blocks) == 2 - else: - # Swap a block to secondary - assert isinstance(events[0].data, - trtllm.kv_cache.KVCacheUpdatedData) - assert events[0].data.cache_level.old_value == 0 - assert events[0].data.cache_level.new_value == 1 - # Store the filled context block - assert isinstance(events[1].data, - trtllm.kv_cache.KVCacheStoredData) - assert len(events[1].data.blocks) == 1 - assert events[1].data.parent_hash is None - # Swap another block to secondary - assert isinstance(events[2].data, - trtllm.kv_cache.KVCacheUpdatedData) - assert events[2].data.cache_level.old_value == 0 - assert events[2].data.cache_level.new_value == 1 - assert isinstance(events[2].data.cache_level, - trtllm.kv_cache.KVCacheEventDiffInt) - # Remove the first block in secondary - assert isinstance(events[3].data, - trtllm.kv_cache.KVCacheRemovedData) - assert len(events[3].data.block_hashes) == 1 - assert events[3].data.block_hashes[0] == events[ - 0].data.block_hash - # Store the second context block and the decode block - assert isinstance(events[4].data, - trtllm.kv_cache.KVCacheStoredData) - assert len(events[4].data.blocks) == 2 - assert events[4].data.parent_hash == events[1].data.blocks[ - 0].block_hash - - -@pytest.mark.parametrize("streaming", [False, True]) -def test_request_perf_metrics(streaming: bool, model_path): - - # Create executor - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - # Create request - max_tokens = 5 - input_tokens = [1, 2, 3, 4] - output_config = trtllm.OutputConfig(return_perf_metrics=True) - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - streaming=streaming, - output_config=output_config) - - # Enqueue the request - request_id = executor.enqueue_request(request) - - def check_perf_metrics(perf_metrics, done, response_id): - assert perf_metrics is not None - - timing_metrics = perf_metrics.timing_metrics - assert timing_metrics.arrival_time < timing_metrics.first_scheduled_time - assert timing_metrics.first_scheduled_time < timing_metrics.first_token_time - if done: - assert timing_metrics.first_token_time < timing_metrics.last_token_time - else: - assert timing_metrics.last_token_time == datetime.timedelta(0) - - kv_cache_metrics = perf_metrics.kv_cache_metrics - assert kv_cache_metrics.num_total_allocated_blocks == 1 - assert kv_cache_metrics.num_new_allocated_blocks == 1 - assert kv_cache_metrics.num_reused_blocks == 0 - assert kv_cache_metrics.num_missed_blocks == 1 - assert kv_cache_metrics.kv_cache_hit_rate == 0 - - assert perf_metrics.first_iter == 0 - if done: - assert perf_metrics.iter == (max_tokens - 1) - assert perf_metrics.last_iter == max_tokens - 1 - else: - assert perf_metrics.iter == response_id - assert perf_metrics.last_iter is None - - # Get the new tokens - tokens = [] - done = False - i = 0 - max_wait_ms = 10000 - response_id = 0 - while not done and i < max_wait_ms: - wait_time = datetime.timedelta(milliseconds=1) - responses = executor.await_responses(request_id, wait_time) - for response in responses: - assert not response.has_error( - ), f"Request id {request_id} failed with err {response.error_msg}" - result = response.result - done = result.is_final - check_perf_metrics(result.request_perf_metrics, done, response_id) - new_tokens = result.output_token_ids[beam_width - 1] - tokens.extend(new_tokens) - response_id += 1 - i += 1 - assert i < max_wait_ms - assert len(tokens) == get_expected_num_tokens( - len(input_tokens), - max_tokens, - streaming=streaming, - exclude_input_from_output=False), f"{request_id}" - - -def test_request_perf_metrics_kv_cache(model_path): - - # Create executor - beam_width = 1 - executor_config = trtllm.ExecutorConfig(beam_width) - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - # Create request: model uses 32 tokens per block, so it will fill a full block - max_tokens = 32 - input_tokens = [1, 2, 3, 4] - request = trtllm.Request(input_tokens, max_tokens=max_tokens) - - # Enqueue the request - request_id = executor.enqueue_request(request) - # Store two blocks with a total of 35 reusable tokens (4 input + 32 output, but last token is not stored) - - # Get the response - responses = executor.await_responses(request_id) - assert not responses[0].has_error() - result = responses[0].result - assert result.is_final - - # Prepare second request using the first output - input_tokens = result.output_token_ids[beam_width - 1] + [1, 2, 3, 4] - output_config = trtllm.OutputConfig(return_perf_metrics=True) - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - output_config=output_config) - - # Enqueue the request - # New query has 36 tokens that match input and output of first request plus 4 additional tokens. - # First block reused completely (32 tokens), from second block we'll partially reuse 3 tokens. - request_id = executor.enqueue_request(request) - - # Get the response - responses = executor.await_responses(request_id) - assert not responses[0].has_error() - result = responses[0].result - assert result.is_final - - # Check KV cache metric: Two blocks will be reused, although second block is partially reused. - # Cache hit rate is 100% since granularity is blocks, not tokens. - kv_cache_metrics = result.request_perf_metrics.kv_cache_metrics - assert kv_cache_metrics.num_total_allocated_blocks == 0 - assert kv_cache_metrics.num_new_allocated_blocks == 0 - assert kv_cache_metrics.num_reused_blocks == 2 - assert kv_cache_metrics.num_missed_blocks == 0 - assert kv_cache_metrics.kv_cache_hit_rate == 1.0 - - -# Skip test for pre-Hopper: https://nvbugs/5404000 -@skip_pre_hopper -@pytest.mark.parametrize("exclude_input_from_output", [False, True]) -def test_request_perf_metrics_draft(model_path_draft_tokens_external, - exclude_input_from_output: bool): - - # Create executor - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(free_gpu_memory_fraction=0.5)) - executor = trtllm.Executor(model_path_draft_tokens_external, - trtllm.ModelType.DECODER_ONLY, executor_config) - - # Create request - max_tokens = 5 - input_tokens = [1, 2, 3, 4] - - # Only first two tokens will be accepted -> 50% acceptance rate - draft_config = trtllm.ExternalDraftTokensConfig([2, 4, 9, 10]) - output_config = trtllm.OutputConfig( - exclude_input_from_output=exclude_input_from_output, - return_perf_metrics=True) - request = trtllm.Request(input_tokens, - max_tokens=max_tokens, - output_config=output_config, - external_draft_tokens_config=draft_config) - - # Enqueue the request - request_id = executor.enqueue_request(request) - - # Get the response - responses = executor.await_responses(request_id) - assert not responses[0].has_error() - result = responses[0].result - assert result.is_final - - # check the new tokens - new_tokens = result.output_token_ids[beam_width - 1] - if exclude_input_from_output: - assert new_tokens == [2, 4, 2] - else: - assert new_tokens == [1, 2, 3, 4, 2, 4, 2] - - # Check the perf metrics - perf_metrics = result.request_perf_metrics - assert perf_metrics is not None - - timing_metrics = perf_metrics.timing_metrics - assert timing_metrics.arrival_time < timing_metrics.first_scheduled_time - assert timing_metrics.first_scheduled_time < timing_metrics.first_token_time - assert timing_metrics.first_token_time <= timing_metrics.last_token_time - - assert perf_metrics.first_iter == 0 - assert perf_metrics.iter == 0 - assert perf_metrics.last_iter == 0 - - spec_dec_metrics = perf_metrics.speculative_decoding - assert spec_dec_metrics.acceptance_rate == 0.5 - assert spec_dec_metrics.total_accepted_draft_tokens == 2 - assert spec_dec_metrics.total_draft_tokens == 4 - - -def test_kv_event_stream_timeout(model_path): - - beam_width = 1 - executor_config = trtllm.ExecutorConfig( - beam_width, - kv_cache_config=trtllm.KvCacheConfig(True, - 4 * 64, - event_buffer_max_size=1024, - free_gpu_memory_fraction=0.5)) - - executor = trtllm.Executor(model_path, trtllm.ModelType.DECODER_ONLY, - executor_config) - - cache_manager = executor.get_kv_cache_event_manager() - - events = cache_manager.get_latest_events() - assert len(events) == 1 - - start = datetime.datetime.now() - events = cache_manager.get_latest_events(1000) - end = datetime.datetime.now() - # Make sure that it actually waited - assert abs(end - start) > datetime.timedelta(milliseconds=900) - assert len(events) == 0 - - def test_request_perf_metrics_pickle(): metrics = trtllm.RequestPerfMetrics() random_delta = datetime.timedelta(seconds=42, milliseconds=123) diff --git a/tests/unittest/conftest.py b/tests/unittest/conftest.py index e1d5943a0a57..4d3d24efb824 100644 --- a/tests/unittest/conftest.py +++ b/tests/unittest/conftest.py @@ -39,7 +39,15 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from integration.defs import test_list_parser +# Dispatched explicitly (not via pytest_plugins, which pytest forbids in a +# non-top-level conftest: a repo-root invocation like `pytest tests` loads +# this file as a NESTED conftest and would fail collection; and not via "-p" +# in pytest.ini addopts, which imports at preparse, before the ini pythonpath +# entries are usable). The wrappers below forward to the plugin; hooks are +# idempotent, so a repo-root run that also dispatches from tests/conftest.py +# is harmless. from test_common import s3_output +from test_common import session_prefetcher_hooks as _prefetch_hooks def dump_threads(signum, frame): @@ -47,6 +55,7 @@ def dump_threads(signum, frame): def pytest_configure(config): + _prefetch_hooks.pytest_configure(config) os.environ.setdefault("TRTLLM_NO_USAGE_STATS", "1") # avoid thread leak of tqdm's TMonitor @@ -93,8 +102,6 @@ def pytest_configure(config): print_info(f" XML path: {periodic_junit_xmlpath}") print_info(f" Batch size: {periodic_batch_size}") - s3_output.register_plugin(config) - @pytest.hookimpl(wrapper=True) def pytest_runtest_protocol(item, nextitem): @@ -510,3 +517,11 @@ def setup_ray_cluster() -> Generator[int, None, None]: finally: if ray.is_initialized(): ray.shutdown() + + +def pytest_runtest_setup(item): + _prefetch_hooks.pytest_runtest_setup(item) + + +def pytest_sessionfinish(session, exitstatus): + _prefetch_hooks.pytest_sessionfinish(session, exitstatus) diff --git a/tests/unittest/disaggregated/kv_transfer_harness.py b/tests/unittest/disaggregated/kv_transfer_harness.py new file mode 100644 index 000000000000..d03a1cfeab6a --- /dev/null +++ b/tests/unittest/disaggregated/kv_transfer_harness.py @@ -0,0 +1,521 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Threaded single-process NIXL harness for V2 disaggregated KV transfer tests. + +Creates one ``KvCacheTransceiverV2`` per rank inside a single process using +threads plus a Barrier-based ``Distributed`` mock, then drives a full +ctx-send / gen-receive transfer and hands verification back to the caller. +Model specifics (cache-manager construction, pool initialization, post-transfer +verification) are injected as hooks; see ``test_deepseek_v4_kv_transfer.py`` +and ``test_minimax_m3_kv_transfer.py`` for the two current users. +""" + +import os +import threading +import uuid +from typing import Dict, List, Optional, Protocol, Sequence, TypeVar + +import tensorrt_llm +import tensorrt_llm.bindings +import tensorrt_llm.tensorrt_llm_transfer_agent_binding # noqa: F401 +from tensorrt_llm import DisaggregatedParams, Mapping, SamplingParams +from tensorrt_llm._torch.disaggregation.transceiver import KvCacheTransceiverV2 +from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2 +from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, LlmRequestType +from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests +from tensorrt_llm.llmapi.llm_args import CacheTransceiverConfig + +# This harness builds real NIXL transfer agents, so force a known-good +# transport config *unconditionally* rather than deferring to inherited env: +# a poisoned developer/CI environment must not be able to destabilize the +# transport or oversubscribe CPUs. +# - One NIXL worker thread per agent: the default (8) causes heavy contention +# when many agents are created on a single GPU in the same process. +# - ``^ib,gdr_copy`` disables InfiniBand and GDR copy, which are unavailable +# (and flaky) in the single-process threaded harness. +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" +os.environ["UCX_TLS"] = "^ib,gdr_copy" + + +# --------------------------------------------------------------------------- +# Harness-scale constants shared by all model-specific manager factories +# --------------------------------------------------------------------------- +TOKENS_PER_BLOCK = 128 +MAX_SEQ_LEN = 512 +MAX_BATCH_SIZE = 16 +VOCAB_SIZE = 129280 + +_CacheManagerT = TypeVar("_CacheManagerT", bound=KVCacheManagerV2) +_CacheManagerT_co = TypeVar("_CacheManagerT_co", bound=KVCacheManagerV2, covariant=True) +_CacheManagerT_contra = TypeVar("_CacheManagerT_contra", bound=KVCacheManagerV2, contravariant=True) + + +class ManagerFactory(Protocol[_CacheManagerT_co]): + """Build one cache manager per rank of a (tp x pp) instance. + + Model-specific knobs (dtype, compress ratios, sparse layers, ...) are + closed over by the caller rather than threaded through the harness. + """ + + def __call__( + self, + tp: int, + pp: int, + enable_dp: bool, + /, + ) -> Sequence[_CacheManagerT_co]: ... + + +class CacheInitializer(Protocol[_CacheManagerT_contra]): + def __call__( + self, + managers: Sequence[_CacheManagerT_contra], + tp: int, + /, + *, + seed_base: int = 0, + fill_random: bool = True, + ) -> None: ... + + +class CacheVerifier(Protocol[_CacheManagerT_contra]): + def __call__( + self, + *, + request_lengths: List[int], + ctx_managers: Sequence[_CacheManagerT_contra], + gen_managers: Sequence[_CacheManagerT_contra], + ctx_tp: int, + ctx_pp: int, + gen_tp: int, + gen_pp: int, + ctx_enable_dp: bool, + gen_enable_dp: bool, + ctx_request_ids: List[int], + gen_request_ids: List[int], + ) -> None: ... + + +# --------------------------------------------------------------------------- +# ThreadSafeDistributed: threading.Barrier-based Distributed mock +# --------------------------------------------------------------------------- +class ThreadSafeDistributed: + """Distributed mock using threading.Barrier for single-process multi-rank testing. + + Provides the same interface as TorchDistributedWrapper from test_py_cache_transceiver_mp.py + but uses Barrier + Lock + shared dict instead of torch.distributed. + """ + + def __init__( + self, + local_rank: int, + world_size: int, + tp_size: int, + pp_size: int, + tp_rank: int, + pp_rank: int, + shared: dict, + ): + self.rank = local_rank + self._world_size = world_size + self._tp_size = tp_size + self._pp_size = pp_size + self._tp_rank = tp_rank + self._pp_rank = pp_rank + self._s = shared + self._bcast_idx = 0 + self._ag_idx = 0 + self._pp_ag_idx = 0 + self._tp_ag_idx = 0 + + @property + def tp_size(self): + return self._tp_size + + @property + def pp_size(self): + return self._pp_size + + @property + def world_size(self): + return self._world_size + + def broadcast(self, obj, root=0): + idx = self._bcast_idx + self._bcast_idx += 1 + key = f"bcast_{idx}" + if self.rank == root: + self._s[key] = obj + self._s["barrier"].wait() + result = self._s[key] + self._s["barrier"].wait() + return result + + def allgather(self, obj): + idx = self._ag_idx + self._ag_idx += 1 + key = f"ag_{idx}" + with self._s["lock"]: + if key not in self._s: + self._s[key] = [None] * self._world_size + self._s[key][self.rank] = obj + self._s["barrier"].wait() + result = list(self._s[key]) + self._s["barrier"].wait() + return result + + def pp_allgather(self, obj): + idx = self._pp_ag_idx + self._pp_ag_idx += 1 + key = f"pp_ag_{idx}_tp{self._tp_rank}" + with self._s["lock"]: + if key not in self._s: + self._s[key] = [None] * self._pp_size + self._s[key][self._pp_rank] = obj + self._s["barrier"].wait() + result = list(self._s[key]) + self._s["barrier"].wait() + return result + + def tp_allgather(self, obj): + idx = self._tp_ag_idx + self._tp_ag_idx += 1 + key = f"tp_ag_{idx}_pp{self._pp_rank}" + with self._s["lock"]: + if key not in self._s: + self._s[key] = [None] * self._tp_size + self._s[key][self._tp_rank] = obj + self._s["barrier"].wait() + result = list(self._s[key]) + self._s["barrier"].wait() + return result + + +# --------------------------------------------------------------------------- +# Threading helpers +# --------------------------------------------------------------------------- +def run_concurrent(items, fn): + """Run fn(item) for each item concurrently in threads and propagate errors.""" + errors = [None] * len(items) + results = [None] * len(items) + + def _worker(idx, item): + try: + results[idx] = fn(item) + except Exception as e: + errors[idx] = e + + threads = [threading.Thread(target=_worker, args=(i, item)) for i, item in enumerate(items)] + for t in threads: + t.start() + for t in threads: + t.join() + for i, err in enumerate(errors): + if err is not None: + raise err + return results + + +def _create_transceiver_in_thread(rank, mapping, cache_manager, dist_mock, config, results, errors): + """Thread target: create one KvCacheTransceiverV2.""" + try: + tc = KvCacheTransceiverV2( + mapping=mapping, + dist=dist_mock, + kv_cache_manager=cache_manager, + cache_transceiver_config=config, + ) + results[rank] = tc + except Exception as e: + errors[rank] = e + + +def create_instance_transceivers( + tp: int, + pp: int, + enable_dp: bool, + cache_managers: Sequence[KVCacheManagerV2], + config: CacheTransceiverConfig, +) -> List[KvCacheTransceiverV2]: + """Create KvCacheTransceiverV2 for all ranks via threaded init.""" + world_size = tp * pp + shared = {"barrier": threading.Barrier(world_size), "lock": threading.Lock()} + results = [None] * world_size + errors = [None] * world_size + threads = [] + + for rank in range(world_size): + pp_rank = rank // tp + tp_rank = rank % tp + mapping = Mapping( + world_size=world_size, + rank=rank, + tp_size=tp, + pp_size=pp, + enable_attention_dp=enable_dp, + ) + dist_mock = ThreadSafeDistributed(rank, world_size, tp, pp, tp_rank, pp_rank, shared) + t = threading.Thread( + target=_create_transceiver_in_thread, + args=( + rank, + mapping, + cache_managers[rank], + dist_mock, + config, + results, + errors, + ), + ) + threads.append(t) + + for t in threads: + t.start() + for t in threads: + t.join() + + for rank, err in enumerate(errors): + if err is not None: + raise err + + return results + + +def get_ctx_info_endpoint(tc: KvCacheTransceiverV2) -> Optional[str]: + """Extract the context_info_endpoint from a transceiver's disaggregated params.""" + endpoints = tc.get_disaggregated_params().get("ctx_info_endpoint") or [] + return endpoints[0] if endpoints else None + + +def get_layers_per_pp(num_layers: int, pp_size: int) -> List[int]: + """Return a list of layer counts per PP rank (mirrors C++ getLayerNumPPRank). + + When num_layers is not evenly divisible by pp_size, the first + (num_layers % pp_size) ranks get one extra layer. + Matches Mapping.pp_layers / torch.tensor_split behaviour. + """ + base = num_layers // pp_size + extra = num_layers % pp_size + return [base + (1 if r < extra else 0) for r in range(pp_size)] + + +def run_kv_transfer_test( + ctx_tp: int, + ctx_pp: int, + gen_tp: int, + gen_pp: int, + ctx_enable_dp: bool, + gen_enable_dp: bool, + update_before_transfer: bool = True, + *, + manager_factory: ManagerFactory[_CacheManagerT], + init_fn: CacheInitializer[_CacheManagerT], + verify_fn: CacheVerifier[_CacheManagerT], +) -> None: + """Run one ctx->gen KV transfer with injectable model-specific cache hooks.""" + ctx_world = ctx_tp * ctx_pp + gen_world = gen_tp * gen_pp + + # Mix of block-aligned and non-aligned lengths for boundary testing. + # TOKENS_PER_BLOCK=128: 65=half+1, 256=2x exact, 129=1x+1, 383=3x-1 + request_lengths = [65, 256, 129, 383] + + # ===== 1. Create cache managers ===== + ctx_managers = list(manager_factory(ctx_tp, ctx_pp, ctx_enable_dp)) + gen_managers = list(manager_factory(gen_tp, gen_pp, gen_enable_dp)) + + # ===== 2. Initialize data ===== + # ctx: random data, seed=pp_rank (same across TP, different across PP) + init_fn(ctx_managers, ctx_tp, seed_base=1000, fill_random=True) + # gen: zeros + init_fn(gen_managers, gen_tp, fill_random=False) + + # ===== 3. Create KvCacheTransceiverV2 instances (threaded init) ===== + config = CacheTransceiverConfig( + backend="NIXL", + transceiver_runtime="PYTHON", + max_tokens_in_buffer=512, + ) + ctx_tcs = create_instance_transceivers(ctx_tp, ctx_pp, ctx_enable_dp, ctx_managers, config) + gen_tcs = create_instance_transceivers(gen_tp, gen_pp, gen_enable_dp, gen_managers, config) + + try: + ctx_info_endpoint = get_ctx_info_endpoint(ctx_tcs[0]) + + # ===== 4. Create requests and determine handle map ===== + # handle_map: rank -> [(req_idx, ctx_request, gen_request)] + ctx_handle_map: Dict[int, List] = {r: [] for r in range(ctx_world)} + gen_handle_map: Dict[int, List] = {r: [] for r in range(gen_world)} + ctx_request_ids: List[int] = [] + gen_request_ids: List[int] = [] + + sampling_params = SamplingParams() + + for req_idx, req_len in enumerate(request_lengths): + unique_rid = uuid.uuid4().int & 0x7FFFFFFFFFFFFFFF + ctx_rid = req_idx * 2 + gen_rid = req_idx * 2 + 1 + ctx_request_ids.append(ctx_rid) + gen_request_ids.append(gen_rid) + + ctx_dp_rank = req_idx % ctx_tp if ctx_enable_dp else 0 + + ctx_request = LlmRequest( + request_id=ctx_rid, + max_new_tokens=1, + input_tokens=list(range(req_len)), + sampling_config=tensorrt_llm.bindings.SamplingConfig( + sampling_params._get_sampling_config() + ), + is_streaming=False, + llm_request_type=LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY, + ) + ctx_request.py_disaggregated_params = DisaggregatedParams(disagg_request_id=unique_rid) + + gen_request = LlmRequest( + request_id=gen_rid, + max_new_tokens=1, + input_tokens=list(range(req_len)), + sampling_config=tensorrt_llm.bindings.SamplingConfig( + sampling_params._get_sampling_config() + ), + is_streaming=False, + llm_request_type=LlmRequestType.LLMREQUEST_TYPE_GENERATION_ONLY, + ) + gen_request.py_disaggregated_params = DisaggregatedParams( + ctx_request_id=ctx_rid, + ctx_dp_rank=ctx_dp_rank, + ctx_info_endpoint=ctx_info_endpoint, + disagg_request_id=unique_rid, + ) + + for rank in range(ctx_world): + tp_rank = rank % ctx_tp + should_handle = (not ctx_enable_dp) or (req_idx % ctx_tp == tp_rank) + if should_handle: + ctx_handle_map[rank].append((req_idx, ctx_request)) + + for rank in range(gen_world): + tp_rank = rank % gen_tp + should_handle = (not gen_enable_dp) or (req_idx % gen_tp == tp_rank) + if should_handle: + gen_handle_map[rank].append((req_idx, gen_request)) + + # ===== 5. Allocate KV cache for all ranks ===== + # prepare_resources is a no-op for non-draft KVCacheManagerV2. + # All ranks must allocate BEFORE mutating shared request objects + # (add_new_token changes is_first_context_chunk). + # + # Gen ranks take the disagg-gen-init path: prepare_disagg_gen_init + # sizes the cache for the full prompt and pre-declares + # history_length=prompt_len, matching what the V2 scheduler's + # _try_schedule_disagg_gen_init does in production so the + # transceiver's TRANS_COMPLETE contract check is satisfied. + # Ctx ranks take the regular prefill path (prepare_context + + # resize_context). + gen_batches: Dict[int, ScheduledRequests] = {} + for rank in range(gen_world): + reqs = [req for _, req in gen_handle_map[rank]] + if reqs: + batch = ScheduledRequests() + batch.context_requests_last_chunk = reqs + for req in reqs: + gen_managers[rank].prepare_disagg_gen_init(req) + gen_batches[rank] = batch + + ctx_batches: Dict[int, ScheduledRequests] = {} + for rank in range(ctx_world): + reqs = [req for _, req in ctx_handle_map[rank]] + if reqs: + batch = ScheduledRequests() + batch.context_requests_last_chunk = reqs + for req in reqs: + ctx_managers[rank].prepare_context(req) + ctx_managers[rank].resize_context(req, req.context_chunk_size) + ctx_batches[rank] = batch + + # ===== 5.5. context_current_position + add_new_token ===== + # Set position on each unique request once (needed for transfer metadata). + seen: set = set() + for rank in range(ctx_world): + for _, req in ctx_handle_map[rank]: + if req.py_request_id not in seen: + req.context_current_position = req.prompt_len + req.add_new_token(req.prompt_len, 0) + seen.add(req.py_request_id) + + seen = set() + for rank in range(gen_world): + for _, req in gen_handle_map[rank]: + if req.py_request_id not in seen: + req.context_current_position = req.prompt_len + req.add_new_token(req.prompt_len, 0) + seen.add(req.py_request_id) + + # ===== 5.6. update_resources BEFORE transfer (mode: update_before) ===== + if update_before_transfer: + for rank, batch in ctx_batches.items(): + ctx_managers[rank].update_resources(batch) + for rank, batch in gen_batches.items(): + gen_managers[rank].update_resources(batch) + + # ===== 6. gen receive + ctx send ===== + for rank in range(gen_world): + for _, req in gen_handle_map[rank]: + gen_tcs[rank].request_and_receive_async(req) + for rank in range(ctx_world): + for _, req in ctx_handle_map[rank]: + ctx_tcs[rank].respond_and_send_async(req) + + # ===== 7. Wait for completion (threaded, dist calls inside) ===== + run_concurrent( + ctx_tcs, lambda tc: tc.check_context_transfer_status(None, mark_complete=True) + ) + run_concurrent(gen_tcs, lambda tc: tc.check_gen_transfer_status(None)) + + # ===== 7.5. update_resources AFTER transfer (mode: update_after) ===== + if not update_before_transfer: + for rank, batch in ctx_batches.items(): + ctx_managers[rank].update_resources(batch) + for rank, batch in gen_batches.items(): + gen_managers[rank].update_resources(batch) + + # ===== 8. Verify ===== + verify_fn( + request_lengths=request_lengths, + ctx_managers=ctx_managers, + gen_managers=gen_managers, + ctx_tp=ctx_tp, + ctx_pp=ctx_pp, + gen_tp=gen_tp, + gen_pp=gen_pp, + ctx_enable_dp=ctx_enable_dp, + gen_enable_dp=gen_enable_dp, + ctx_request_ids=ctx_request_ids, + gen_request_ids=gen_request_ids, + ) + + finally: + for tc in ctx_tcs + gen_tcs: + try: + tc.shutdown() + except Exception: + pass + for mgr in ctx_managers + gen_managers: + try: + mgr.shutdown() + except Exception: + pass diff --git a/tests/unittest/disaggregated/region/test_block.py b/tests/unittest/disaggregated/region/test_block.py index cc6cc3323608..796683dc8e14 100644 --- a/tests/unittest/disaggregated/region/test_block.py +++ b/tests/unittest/disaggregated/region/test_block.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np from tensorrt_llm._torch.disaggregation.base.region import ( @@ -6,9 +21,8 @@ SpecRegionPair, ) from tensorrt_llm._torch.disaggregation.native.mixers.attention.peer import ( - HeadMatchMapper, - HeadMismatchMapper, - IdentityMapper, + HNDHeadMismatchMapper, + IntactMapper, ) from tensorrt_llm._torch.disaggregation.native.mixers.attention.spec import AttentionInfo from tensorrt_llm._torch.disaggregation.native.rank_info import RankInfo @@ -81,12 +95,13 @@ def test_spec_region_and_spec_region_pair(): assert pair.dst.spec == "spec_dst" -def test_identity_mapper(): +def test_intact_mapper_identity_degenerate(): + """Full contiguous overlap degrades to a whole-region pass-through copy.""" src_group = MemRegionGroup(ptrs=np.array([100, 200], dtype=np.int64), bytes_per_region=32) dst_group = MemRegionGroup(ptrs=np.array([300, 400], dtype=np.int64), bytes_per_region=32) src_spec = SpecRegion(memory=src_group, spec="a") dst_spec = SpecRegion(memory=dst_group, spec="b") - mapper = IdentityMapper() + mapper = IntactMapper([0, 16], [0, 16], 16, 16) result = mapper.map(src_spec, dst_spec) assert isinstance(result, SpecRegionPair) np.testing.assert_array_equal(result.src.memory.ptrs, [100, 200]) @@ -95,14 +110,11 @@ def test_identity_mapper(): assert result.dst.memory.bytes_per_region == 32 -def test_head_match_mapper(): +def test_intact_mapper_partial_layers(): + """Selecting a contiguous layer subset yields one shifted fragment.""" self_ri = make_rankinfo(kv_heads_per_rank=2) - peer_ri = make_rankinfo(kv_heads_per_rank=2) - transfer_layers = 2 - src_layer_off = 1 - dst_layer_off = 1 - # slot_size_per_layer = kv_factor * kv_heads * tokens_per_block * dims_per_head * element_bytes - slot_size_per_layer = ( + # bytes_per_layer = kv_factor * kv_heads * tokens_per_block * dims_per_head * element_bytes + bytes_per_layer = ( self_ri.attention.kv_factor * self_ri.attention.kv_heads_per_rank * self_ri.attention.tokens_per_block @@ -113,46 +125,81 @@ def test_head_match_mapper(): dst_group = MemRegionGroup(ptrs=np.array([30, 40], dtype=np.int64), bytes_per_region=1) src_spec = SpecRegion(memory=src_group, spec="srcspec") dst_spec = SpecRegion(memory=dst_group, spec="dstspec") - mapper = HeadMatchMapper( - transfer_layers, - src_layer_off, - dst_layer_off, - self_ri, - peer_ri, - slot_size_per_layer=slot_size_per_layer, - ) + # Self selects layers 1..2 of its slot; the peer stores the same class at + # layers 2..3. Distinct src/dst layer offsets make the test fail if the + # mapper ever applies one side's offset to the other. + src_offsets = [bytes_per_layer, 2 * bytes_per_layer] + dst_offsets = [2 * bytes_per_layer, 3 * bytes_per_layer] + mapper = IntactMapper(src_offsets, dst_offsets, bytes_per_layer, bytes_per_layer) result = mapper.map(src_spec, dst_spec) - expected_off = transfer_layers * slot_size_per_layer + # Both sides are internally contiguous, so the two layers merge into a + # single fragment shifted by each side's own first-layer offset. + expected_bytes = 2 * bytes_per_layer np.testing.assert_array_equal( - result.src.memory.ptrs, [10 + mapper._src_block_off, 20 + mapper._src_block_off] + result.src.memory.ptrs, [10 + bytes_per_layer, 20 + bytes_per_layer] ) np.testing.assert_array_equal( - result.dst.memory.ptrs, [30 + mapper._dst_block_off, 40 + mapper._dst_block_off] + result.dst.memory.ptrs, [30 + 2 * bytes_per_layer, 40 + 2 * bytes_per_layer] ) - assert result.src.memory.bytes_per_region == expected_off - assert result.dst.memory.bytes_per_region == expected_off + assert result.src.memory.bytes_per_region == expected_bytes + assert result.dst.memory.bytes_per_region == expected_bytes def test_head_mismatch_mapper(): + # Self holds 2 KV heads at TP=2; the peer holds 4 KV heads at TP=4. Using + # peer tp_rank=1 (not 2) forces a *nonzero* source-side head offset, so an + # incorrect head-offset computation cannot pass silently. self_ri = make_rankinfo(kv_heads_per_rank=2, tp_size=2, tp_rank=1) - peer_ri = make_rankinfo(kv_heads_per_rank=4, tp_size=4, tp_rank=2) - transfer_layers = 1 - src_layer_off = 0 - peer_layer_off = 1 - src_group = MemRegionGroup(ptrs=np.array([111], dtype=np.int64), bytes_per_region=32) - dst_group = MemRegionGroup(ptrs=np.array([222], dtype=np.int64), bytes_per_region=32) + peer_ri = make_rankinfo(kv_heads_per_rank=4, tp_size=4, tp_rank=1) + buffers_per_layer = 2 # K and V + # buffer bytes = heads * tokens_per_block * dims_per_head * element_bytes + self_buffer_bytes = 2 * 4 * 2 * 1 + peer_buffer_bytes = 4 * 4 * 2 * 1 + self_bytes_per_layer = buffers_per_layer * self_buffer_bytes # kv_factor x K-buffer bytes + peer_bytes_per_layer = buffers_per_layer * peer_buffer_bytes + bytes_per_head = self_buffer_bytes // self_ri.attention.kv_heads_per_rank # equals peer side + src_base, dst_base = 111, 222 + src_group = MemRegionGroup(ptrs=np.array([src_base], dtype=np.int64), bytes_per_region=32) + dst_group = MemRegionGroup(ptrs=np.array([dst_base], dtype=np.int64), bytes_per_region=32) src_spec = SpecRegion(memory=src_group, spec="srcspec") dst_spec = SpecRegion(memory=dst_group, spec="dstspec") - mapper = HeadMismatchMapper(transfer_layers, src_layer_off, peer_layer_off, self_ri, peer_ri) + peer_layer_off = peer_bytes_per_layer # copy targets layer 1 on the peer side + mapper = HNDHeadMismatchMapper( + src_layer_offsets=[0], + dst_layer_offsets=[peer_layer_off], + self_ri=self_ri, + peer_ri=peer_ri, + self_bytes_per_layer=self_bytes_per_layer, + peer_bytes_per_layer=peer_bytes_per_layer, + self_buffers_per_layer=buffers_per_layer, + peer_buffers_per_layer=buffers_per_layer, + ) result = mapper.map(src_spec, dst_spec) - expected_frag_count = self_ri.attention.kv_factor * transfer_layers assert isinstance(result, SpecRegionPair) - assert len(result.src.memory.ptrs) == expected_frag_count - assert len(result.dst.memory.ptrs) == expected_frag_count assert isinstance(result.src.memory.ptrs, np.ndarray) assert isinstance(result.dst.memory.ptrs, np.ndarray) - assert result.src.memory.bytes_per_region == mapper._bytes_cont_heads - assert result.dst.memory.bytes_per_region == mapper._bytes_cont_heads + # Source head offset for self rank 1 slicing into the peer's 4-head layout: + # (peer_tp_rank * self_kv_heads * self_tp) // peer_tp % self_kv_heads + # = (1 * 2 * 2) // 4 % 2 = 1 head -> 1 * bytes_per_head. The peer side is + # the coarser layout, so its head offset is 0. + src_head_off = 1 * bytes_per_head + # One fragment per (layer, K/V buffer), buffers back-to-back within a layer. + # Source layer offset is 0; the peer layer offset is nonzero. + expected_src_ptrs = [ + src_base + src_head_off + buf * self_buffer_bytes for buf in range(buffers_per_layer) + ] + expected_dst_ptrs = [ + dst_base + peer_layer_off + buf * peer_buffer_bytes for buf in range(buffers_per_layer) + ] + np.testing.assert_array_equal(result.src.memory.ptrs, expected_src_ptrs) + np.testing.assert_array_equal(result.dst.memory.ptrs, expected_dst_ptrs) + # Each fragment copies min(self_heads, peer_heads) contiguous heads. + cont_heads_bytes = ( + min(self_ri.attention.kv_heads_per_rank, peer_ri.attention.kv_heads_per_rank) + * bytes_per_head + ) + assert result.src.memory.bytes_per_region == cont_heads_bytes + assert result.dst.memory.bytes_per_region == cont_heads_bytes def test_rankinfo_kv_factor(): diff --git a/tests/unittest/disaggregated/region/test_page.py b/tests/unittest/disaggregated/region/test_page.py index 14042299fa3b..2c93bb5f7fea 100644 --- a/tests/unittest/disaggregated/region/test_page.py +++ b/tests/unittest/disaggregated/region/test_page.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np from tensorrt_llm._torch.disaggregation.resource.page import ( @@ -6,6 +21,7 @@ KVCachePageTable, LayerGroup, LocalLayer, + MapperKind, PhysicalPool, PhysicalPoolGroup, PoolView, @@ -27,15 +43,25 @@ def test_physical_pool_construction(): assert pool.base_address == 0x10000 assert pool.slot_bytes == 256 assert pool.num_slots == 4 + assert pool.slot_stride_bytes == 256 + assert pool.layer_stride_bytes == 1024 def test_physical_pool_roundtrip(): - pool = PhysicalPool(base_address=0x10000, slot_bytes=256, num_slots=4) + pool = PhysicalPool( + base_address=0x10000, + slot_bytes=256, + num_slots=4, + slot_stride_bytes=2048, + layer_stride_bytes=512, + ) d = pool.to_dict() restored = PhysicalPool.from_dict(d) assert restored.base_address == pool.base_address assert restored.slot_bytes == pool.slot_bytes assert restored.num_slots == pool.num_slots + assert restored.slot_stride_bytes == pool.slot_stride_bytes + assert restored.layer_stride_bytes == pool.layer_stride_bytes def test_pool_view_roundtrip(): @@ -49,6 +75,24 @@ def test_pool_view_roundtrip(): assert restored.buffer_entries[1]["offset"] == 128 +def test_pool_view_kind_roundtrip(): + """REPLICATED / NHD kinds survive serialization; default stays INDEXED.""" + entries = _make_buffer_entries() + for kind in (MapperKind.REPLICATED, MapperKind.NHD): + view = PoolView( + pool_idx=2, + buffer_entries=entries, + pool_role=frozenset({"index_key"}), + mapper_kind=kind, + ) + restored = PoolView.from_dict(view.to_dict()) + assert restored.mapper_kind == kind + assert restored.pool_role == frozenset({"index_key"}) + + legacy = PoolView(pool_idx=0, buffer_entries=entries).to_dict() + assert PoolView.from_dict(legacy).mapper_kind == MapperKind.INDEXED + + def test_local_layer_roundtrip(): ll = LocalLayer(local_layer_id=0, global_layer_id=5) d = ll.to_dict() @@ -100,3 +144,94 @@ def test_kv_cache_page_table_roundtrip(): assert len(restored.layer_groups) == 1 assert len(restored.pool_groups) == 1 assert restored.pool_groups[0].pools[0].base_address == 0x10000 + + +# --------------------------------------------------------------------------- +# bytes_per_layer serialization and get_layer_byte_ranges geometry +# --------------------------------------------------------------------------- + + +def test_pool_view_bytes_per_layer_roundtrip(): + entries = _make_buffer_entries() + view = PoolView( + pool_idx=1, + buffer_entries=entries, + pool_role=frozenset({"key", "value"}), + mapper_kind=MapperKind.NHD, + bytes_per_layer=256, + ) + restored = PoolView.from_dict(view.to_dict()) + assert restored.bytes_per_layer == 256 + + # None (INDEXED views) survives the roundtrip too. + legacyless = PoolView(pool_idx=0, buffer_entries=entries) + assert PoolView.from_dict(legacyless.to_dict()).bytes_per_layer is None + + +def _view(entries, bytes_per_layer=None): + return PoolView( + pool_idx=0, + buffer_entries=np.array(entries, dtype=BUFFER_ENTRY_DTYPE), + pool_role=frozenset({"key", "value"}), + mapper_kind=MapperKind.NHD, + bytes_per_layer=bytes_per_layer, + ) + + +def test_layer_byte_ranges_uniform_stride(): + from tensorrt_llm._torch.disaggregation.resource.utils import get_layer_byte_ranges + + # Dedicated K/V pool: dense layers, uniform stride. + starts, bytes_per_layer = get_layer_byte_ranges( + _view([(0, 0, 128), (0, 128, 128), (1, 256, 128), (1, 384, 128)]) + ) + assert starts == {0: 0, 1: 256} + assert bytes_per_layer == 256 + + +def test_layer_byte_ranges_non_uniform_stride(): + from tensorrt_llm._torch.disaggregation.resource.utils import get_layer_byte_ranges + + # Coalesced slot: another role class interleaves 64B after layer 0, + # so the layer stride is non-uniform while sizes stay uniform. + starts, bytes_per_layer = get_layer_byte_ranges( + _view([(0, 0, 128), (0, 128, 128), (1, 320, 128), (1, 448, 128)]) + ) + assert starts == {0: 0, 1: 320} + assert bytes_per_layer == 256 + + +def test_layer_byte_ranges_noncontiguous_layer_raises(): + import pytest + + from tensorrt_llm._torch.disaggregation.resource.utils import get_layer_byte_ranges + + with pytest.raises(ValueError, match="not contiguous"): + get_layer_byte_ranges(_view([(0, 0, 128), (0, 192, 128)])) + + +def test_layer_byte_ranges_nonuniform_size_raises(): + import pytest + + from tensorrt_llm._torch.disaggregation.resource.utils import get_layer_byte_ranges + + with pytest.raises(ValueError, match="not uniform"): + get_layer_byte_ranges(_view([(0, 0, 128), (1, 128, 64)])) + + +def test_layer_byte_ranges_declared_size_mismatch_raises(): + import pytest + + from tensorrt_llm._torch.disaggregation.resource.utils import get_layer_byte_ranges + + with pytest.raises(ValueError, match="declares bytes_per_layer"): + get_layer_byte_ranges(_view([(0, 0, 128)], bytes_per_layer=256)) + + +def test_layer_byte_ranges_empty_view_raises(): + import pytest + + from tensorrt_llm._torch.disaggregation.resource.utils import get_layer_byte_ranges + + with pytest.raises(ValueError, match="no buffer entries"): + get_layer_byte_ranges(_view([])) diff --git a/tests/unittest/disaggregated/test_agent.py b/tests/unittest/disaggregated/test_agent.py index f2212a6c5f82..c8425ab49e18 100644 --- a/tests/unittest/disaggregated/test_agent.py +++ b/tests/unittest/disaggregated/test_agent.py @@ -6,8 +6,10 @@ import pytest import torch -# Exclude IB (no fabric) and gdr_copy (UCX rcache SIGABRT at teardown). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" from tensorrt_llm import logger from tensorrt_llm._torch.disaggregation.base.agent import ( diff --git a/tests/unittest/disaggregated/test_agent_multi_backends.py b/tests/unittest/disaggregated/test_agent_multi_backends.py index d983af152542..d82815c99a79 100644 --- a/tests/unittest/disaggregated/test_agent_multi_backends.py +++ b/tests/unittest/disaggregated/test_agent_multi_backends.py @@ -3,8 +3,11 @@ import pytest -# Exclude IB (no fabric) and gdr_copy (UCX rcache SIGABRT at teardown). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see test_kv_transfer.py for the full rationale. The subprocesses +# spawned below inherit these via os.environ.copy(). +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" def test_load_agent_missing_module(): diff --git a/tests/unittest/disaggregated/test_bounce.py b/tests/unittest/disaggregated/test_bounce.py index 27bd78b5caec..38d72164b4d3 100644 --- a/tests/unittest/disaggregated/test_bounce.py +++ b/tests/unittest/disaggregated/test_bounce.py @@ -194,14 +194,16 @@ def test_encode_tail_handles_unset_base(self): def test_kv_result_prefix_roundtrip(): """The KV_AGENT_RESULT binary prefix (transfer.py) must round-trip exactly.""" tfr = pytest.importorskip("tensorrt_llm._torch.disaggregation.native.transfer") - for rank, rid, sl, last, status in [ - (7, 6925227277844486, 42, True, tfr.AgentResult.SUCCESS), - (0, 1, 0, False, tfr.AgentResult.FAILED), - (31, 2**62, 9999, True, tfr.AgentResult.SUCCESS), + for rank, rid, sl, last, status, size in [ + (7, 6925227277844486, 42, True, tfr.AgentResult.SUCCESS, 4096), + (0, 1, 0, False, tfr.AgentResult.FAILED, 0), + (31, 2**62, 9999, True, tfr.AgentResult.SUCCESS, 2**40), ]: - packed = tfr._KV_RESULT_PREFIX.pack(rank, rid, sl, last, tfr._AGENT_RESULT_CODE[status]) - r, i, s, last_out, c = tfr._KV_RESULT_PREFIX.unpack(packed) - assert (r, i, s, last_out) == (rank, rid, sl, last) + packed = tfr._KV_RESULT_PREFIX.pack( + rank, rid, sl, last, tfr._AGENT_RESULT_CODE[status], size + ) + r, i, s, last_out, c, sz = tfr._KV_RESULT_PREFIX.unpack(packed) + assert (r, i, s, last_out, sz) == (rank, rid, sl, last, size) assert tfr._AGENT_RESULT_BY_CODE[c] is status @@ -210,11 +212,11 @@ def test_make_kv_result_msg_uses_binary_frame(result_name): """Every KV result (success and failure) uses the binary frame so the receiver can decode it.""" tfr = pytest.importorskip("tensorrt_llm._torch.disaggregation.native.transfer") result = getattr(tfr.AgentResult, result_name) - msg = tfr._make_kv_result_msg(3, 12345, 7, True, result) + msg = tfr._make_kv_result_msg(3, 12345, 7, True, result, transfer_size=8192) assert msg[0] == tfr.MessageType.KV_AGENT_RESULT assert len(msg) == 2 # prefix only; no bounce tail when none is passed - r, rid, sl, last, code = tfr._KV_RESULT_PREFIX.unpack(msg[1]) - assert (r, rid, sl, last) == (3, 12345, 7, True) + r, rid, sl, last, code, size = tfr._KV_RESULT_PREFIX.unpack(msg[1]) + assert (r, rid, sl, last, size) == (3, 12345, 7, True, 8192) assert tfr._AGENT_RESULT_BY_CODE[code] is result @@ -229,13 +231,19 @@ def test_fanin_bounce_safe_gate(): expected_transfers) must fall back to the per-fragment path. """ tfr = pytest.importorskip("tensorrt_llm._torch.disaggregation.native.transfer") + from tensorrt_llm._torch.disaggregation.resource.page import MapperKind + safe = tfr.Receiver._fanin_bounce_safe - def ov(dup, pp): - return SimpleNamespace(duplicate_head_factor=dup, overlap_pp_size=pp) + def ov(dup, pp, ranks=(0,)): + return SimpleNamespace(duplicate_head_factor=dup, overlap_pp_size=pp, ranks=list(ranks)) + + def ri(lpp, page_table=None): + return SimpleNamespace(layer_num_per_pp=lpp, page_table=page_table) - def ri(lpp): - return SimpleNamespace(layer_num_per_pp=lpp) + def pt(mapper_kind): + view = SimpleNamespace(mapper_kind=mapper_kind) + return SimpleNamespace(layer_groups=[SimpleNamespace(pool_views=[view])]) # single PP stage (overlap_pp_size <= 1): only duplicate_head_factor matters assert safe(ov(1, 1), ri([24])) is True @@ -249,6 +257,12 @@ def ri(lpp): assert safe(ov(1, 4), ri([20])) is False # duplicate heads blocks even an otherwise-even PP split assert safe(ov(2, 4), ri([20, 20, 20, 20])) is False + # replicated views (one elected sender per destination) make multi-writer + # contributions unequal -> fall back; single-writer overlap stays safe, + # and sharded-only view schemes are unaffected + assert safe(ov(1, 1, ranks=(0, 1)), ri([24], pt(MapperKind.REPLICATED))) is False + assert safe(ov(1, 1, ranks=(0,)), ri([24], pt(MapperKind.REPLICATED))) is True + assert safe(ov(1, 1, ranks=(0, 1)), ri([24], pt(MapperKind.NHD))) is True # --------------------------------------------------------------------------- # @@ -609,13 +623,18 @@ def test_orphan_reservation_quarantines_and_is_idempotent(self, monkeypatch): _K3_KDA_LAYERS = 69 _K3_CONV_SLOT_BYTES = 294_912 # [3*H*hd, W] bf16 per layer _K3_SSM_SLOT_BYTES = 6_291_456 # [H, hd, hd] fp32 per layer (95.5% of the state) -_K3_KDA_PAYLOAD_BYTES = 454_459_392 # 69 x (conv + delta) per request per rank, from the geometry above +_K3_KDA_PAYLOAD_BYTES = ( + 454_459_392 # 69 x (conv + delta) per request per rank, from the geometry above +) def _k3_page_table() -> KVCachePageTable: - """A K3-shaped page table exactly as the builders produce it: attention layer group(s) - first and the mamba layer group appended LAST (kv_extractor.py, both ``build_page_table`` - and ``_build_page_table_v2``, append it after every attention group).""" + """A K3-shaped page table exactly as the builders produce it. + + Attention layer group(s) first and the mamba layer group appended LAST + (kv_extractor.py, both ``build_page_table`` and ``_build_page_table_v2``, + append it after every attention group). + """ attn = AttentionLayerGroup( pool_group_idx=0, kv_head_num_per_rank=1, diff --git a/tests/unittest/disaggregated/test_cache_reuse_adapter.py b/tests/unittest/disaggregated/test_cache_reuse_adapter.py index c6828bac8684..c131183461be 100644 --- a/tests/unittest/disaggregated/test_cache_reuse_adapter.py +++ b/tests/unittest/disaggregated/test_cache_reuse_adapter.py @@ -117,20 +117,27 @@ class _FakeMgr: def __init__(self): self.beam_width = None + self.pool_indices_window = None def get_batch_cache_indices(self, request_ids, layer_idx=None, beam_width=1): self.beam_width = beam_width return [[10, 11, 12, 13]] + def get_memory_pool_block_indices(self, block_ids, window_size): + # Identity translation: nothing offloaded, block_id == pool slot. + self.pool_indices_window = window_size + return block_ids + req = _FakeReq(prompt_len=7) req.py_request_id = 1 req.py_beam_width = 4 req.sampling_config = _FakeSamplingConfig(beam_width=1) mgr = _FakeMgr() - block_ids = _CacheReuseAdapterV1(mgr).get_block_ids(req, 0, _lg()) + block_ids = _CacheReuseAdapterV1(mgr).get_block_ids(req, 0, _lg(window=512)) assert mgr.beam_width == 4 + assert mgr.pool_indices_window == 512 np.testing.assert_array_equal(block_ids, [10, 11, 12, 13]) def test_pack_beam_cache_indices_single_block_prompt_keeps_all_beams(self): diff --git a/tests/unittest/disaggregated/test_cache_transceiver_harness.py b/tests/unittest/disaggregated/test_cache_transceiver_harness.py index baaa3f421f35..cb37e89b4660 100644 --- a/tests/unittest/disaggregated/test_cache_transceiver_harness.py +++ b/tests/unittest/disaggregated/test_cache_transceiver_harness.py @@ -353,6 +353,24 @@ def test_single_node_transfer(tmp_path): f"ctx log tail:\n{_log_tail(ctx_log)}\n" f"gen log tail:\n{_log_tail(gen_log)}" ) + # Bandwidth must actually be parsed from the perf CSVs, not just + # the transfer verified. The PYTHON+NIXL combination writes its + # perf CSVs via PerfLogManager into TRTLLM_KVCACHE_TIME_OUTPUT_PATH + # (set by the driver) as "_.csv"; a naming or + # glob mismatch between perf_logger.py and report.py leaves + # per_gpu_BW_GBps None while status stays PASS, so assert on it + # explicitly. + ctx_csv_dir = os.path.join(work_dir, "csv", "0", "ctx") + csv_listing = os.listdir(ctx_csv_dir) if os.path.isdir(ctx_csv_dir) else [] + assert sweep["per_gpu_BW_GBps"] is not None and sweep["per_gpu_BW_GBps"] > 0, ( + f"per_gpu_BW_GBps missing for {combo['combination']} " + f"sweep={sweep['sweep']} — perf CSVs were not parsed " + f"(ctx csv dir contents: {csv_listing})" + ) + assert sweep["num_samples"] > 0, ( + f"num_samples is 0 for {combo['combination']} sweep={sweep['sweep']} " + f"(ctx csv dir contents: {csv_listing})" + ) best_path = os.path.splitext(results_path)[0] + ".best.json" assert os.path.exists(best_path), "results.best.json was not created" diff --git a/tests/unittest/disaggregated/test_cache_transceiver_harness_report.py b/tests/unittest/disaggregated/test_cache_transceiver_harness_report.py index 24e3b2d72cfb..ecd8744cea1a 100644 --- a/tests/unittest/disaggregated/test_cache_transceiver_harness_report.py +++ b/tests/unittest/disaggregated/test_cache_transceiver_harness_report.py @@ -186,7 +186,8 @@ def test_emit_ucx_env_second_sweep(self, sample_cfg, capfd): # --------------------------------------------------------------------------- class TestParseCppRecvCsvs: def test_basic(self, tmp_path): - csv_path = tmp_path / "rank_0_recv.csv" + # C++ names files "__recv.csv" (instanceId is a UUID). + csv_path = tmp_path / "3c9f0e2a-1111-2222-3333-444455556666_0_recv.csv" with open(csv_path, "w", newline="") as f: w = csv.writer(f) w.writerow(["RequestID", "Bandwidth(Gbps)", "Bandwidth(Gbps)"]) @@ -199,6 +200,7 @@ def test_basic(self, tmp_path): assert abs(bws[0] - 15.0) < 0.01 def test_renamed_pattern(self, tmp_path): + # Legacy "rank_*" prefix must still parse (backward compatibility). csv_path = tmp_path / "rank_0_recv__c0.csv" with open(csv_path, "w", newline="") as f: w = csv.writer(f) @@ -225,7 +227,7 @@ def test_basic(self, tmp_path): { "unique_rid": "0", "throughput_mbs": "1048.576", - "task_type": "Send", + "task_type": "KVSendTask", "other": "", } ) @@ -234,7 +236,17 @@ def test_basic(self, tmp_path): { "unique_rid": "0", "throughput_mbs": "2000.0", - "task_type": "Recv", + "task_type": "KVRecvTask", + "other": "", + } + ) + # Aux sends are tiny metadata transfers; they must not drag the + # KV bandwidth stats down. + w.writerow( + { + "unique_rid": "0", + "throughput_mbs": "1.0", + "task_type": "AuxSendTask", "other": "", } ) @@ -244,6 +256,47 @@ def test_basic(self, tmp_path): assert len(result[0]) == 1 assert abs(result[0][0] - 1048.576 * 1024 * 1024 / 1e9) < 0.001 + def test_cpp_naming_without_py_prefix(self, tmp_path): + """PerfLogManager prefers the C++ output-path env var for naming. + + With TRTLLM_KVCACHE_TIME_OUTPUT_PATH set, Python task CSVs are named + "_.csv" (no py_ prefix); the parser must find them + by header columns instead of file name. + """ + csv_path = tmp_path / "cd93dae6-1111-2222-3333-444455556666_3.csv" + with open(csv_path, "w", newline="") as f: + w = csv.DictWriter( + f, + fieldnames=["unique_rid", "throughput_mbs", "task_type"], + ) + w.writeheader() + w.writerow({"unique_rid": "0", "throughput_mbs": "1048.576", "task_type": "KVSendTask"}) + result = _parse_python_csvs(str(tmp_path)) + assert 0 in result + assert abs(result[0][0] - 1048.576 * 1024 * 1024 / 1e9) < 0.001 + + def test_ignores_cpp_csvs_in_same_dir(self, tmp_path): + """C++ CSVs sharing the directory must not contribute samples. + + C++ send/recv and gen-summary CSVs lack the unique_rid/throughput_mbs + columns, so the header check skips them. + """ + with open( + tmp_path / "3c9f0e2a-aaaa-bbbb-cccc-ddddeeeeffff_0_recv.csv", "w", newline="" + ) as f: + w = csv.writer(f) + w.writerow(["RequestID", "Bandwidth(Gbps)"]) + w.writerow([0, 10.0]) + with open( + tmp_path / "3c9f0e2a-aaaa-bbbb-cccc-ddddeeeeffff_0_gen_transfer_summary.csv", + "w", + newline="", + ) as f: + w = csv.writer(f) + w.writerow(["timestamp", "RequestID", "gen_side_transfer_time(ms)", "kv_cache_size"]) + w.writerow(["2026-01-01 00:00:00.000", 0, 1.0, 1024]) + assert _parse_python_csvs(str(tmp_path)) == {} + def test_empty_dir(self, tmp_path): assert _parse_python_csvs(str(tmp_path)) == {} @@ -466,7 +519,9 @@ def _setup_work_dir(self, tmp_path, sample_cfg): # Create CSV for gen side (C++ recv) gen_csv = work / "csv" / "0" / "gen" gen_csv.mkdir(parents=True) - with open(gen_csv / "rank_0_recv.csv", "w", newline="") as f: + with open( + gen_csv / "5d7b1f80-aaaa-bbbb-cccc-ddddeeeeffff_0_recv.csv", "w", newline="" + ) as f: w = csv.writer(f) w.writerow(["RequestID", "Bandwidth(Gbps)"]) # warmup rid (r=0) should be excluded diff --git a/tests/unittest/disaggregated/test_cache_transceiver_precheck_e2e.py b/tests/unittest/disaggregated/test_cache_transceiver_precheck_e2e.py new file mode 100644 index 000000000000..e8a92f99c99c --- /dev/null +++ b/tests/unittest/disaggregated/test_cache_transceiver_precheck_e2e.py @@ -0,0 +1,339 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""End-to-end test for the disagg perf-sanity cache-transceiver PRECHECK driver. + +Drives ``run_precheck.py`` (tests/scripts/perf-sanity/cache_transceiver_precheck) +exactly like the SLURM gate does — one ``mpirun`` world per ctx/gen server +instance, a shared ``--work-dir`` for rendezvous/status, the disagg yaml as +the single config source — but on ONE node with every process sharing one +physical GPU. This is the CI net for the precheck's NON-network failure +modes: internal TRT-LLM API drift, KV pool construction, transceiver +setup/transfer/verification, multi-instance pairing, and failure verdicts. + +Requires: 1 GPU, mpirun, mpi4py, tensorrt_llm. Modeled on +tests/unittest/disaggregated/test_cache_transceiver_harness.py (same +single-node NIXL/PYTHON transceiver combination that stage already runs). +""" + +import json +import os +import shutil +import signal +import subprocess +import sys +import time + +import pytest +import yaml + +# Same single-GPU environment pinning as kv_transfer_harness.py: +# - One NIXL worker thread per agent: the default (8) causes heavy +# contention when many agents share a single GPU. +# - ``^ib,gdr_copy`` disables InfiniBand and GDR copy, which are +# unavailable (and flaky) on single-node loopback. +# Set here (not just in the child env) so any in-process tensorrt_llm +# import sees them too; the mpirun children inherit via os.environ.copy(). +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" +os.environ["UCX_TLS"] = "^ib,gdr_copy" + +PRECHECK_DIR = os.path.normpath( + os.path.join( + os.path.dirname(__file__), + os.pardir, + os.pardir, + os.pardir, + "tests", + "scripts", + "perf-sanity", + "cache_transceiver_precheck", + ) +) +DRIVER_SCRIPT = os.path.join(PRECHECK_DIR, "run_precheck.py") +_LOG_TAIL_CHARS = 16 * 1024 +_PROCESS_TIMEOUT_SECONDS = 240 +_TERMINATE_GRACE_SECONDS = 5 + +TINY_MODEL_CONFIG = { + # Llama-shaped so model_kv_shape() exercises the real config.json path + # (2 layers x 1 kv head x 64 dim: a few hundred KB of KV per process). + "architectures": ["LlamaForCausalLM"], + "num_hidden_layers": 2, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 64, + "hidden_size": 256, + "vocab_size": 32000, +} + + +def _find_mpirun(): + path = shutil.which("mpirun") + if path is None: + pytest.skip("mpirun not found on PATH") + return path + + +def _log_tail(path): + try: + with open(path, errors="replace") as f: + log = f.read() + except OSError: + return f"(no log at {path})" + if len(log) <= _LOG_TAIL_CHARS: + return log + return ( + f"... {len(log) - _LOG_TAIL_CHARS} earlier characters omitted ...\n{log[-_LOG_TAIL_CHARS:]}" + ) + + +def _terminate_process_groups(processes): + for sig in (signal.SIGTERM, signal.SIGKILL): + # ALWAYS signal the group, even when the mpirun leader already + # exited: its ranks stay in the group and would otherwise leak GPU + # memory into the following tests (same contract as the harness's + # test_terminate_process_groups_signals_group_after_leader_exit). + for proc in processes: + try: + os.killpg(proc.pid, sig) + except ProcessLookupError: + pass + deadline = time.monotonic() + _TERMINATE_GRACE_SECONDS + for proc in processes: + if proc.poll() is None: + try: + proc.wait(timeout=max(0.0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + pass + + +def _disagg_yaml(num_ctx, num_gen, ctx_tp, gen_tp, request_lengths=(64,)): + """Minimal disagg perf-sanity yaml shaped like the checked-in configs.""" + tokens_per_block = 32 + + def side(tp): + return { + "tensor_parallel_size": tp, + "pipeline_parallel_size": 1, + "kv_cache_config": { + "dtype": "bf16", + "tokens_per_block": tokens_per_block, + # Same combination the merged single-node harness test runs: + # NIXL + PYTHON transceiver requires the V2 manager. + "use_kv_cache_manager_v2": True, + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 512, + }, + } + + return { + "metadata": {"model_dir_name": "tiny-llama"}, + "benchmark": {"mode": "e2e", "input_length": 64, "output_length": 8}, + "hardware": { + "gpus_per_node": 1, + "num_ctx_servers": num_ctx, + "num_gen_servers": num_gen, + }, + "worker_config": {"ctx": side(ctx_tp), "gen": side(gen_tp)}, + "cache_transceiver_precheck": { + "request_lengths": list(request_lengths), + "num_requests": 1, + "warmup_requests": 1, + "wave_timeout_s": 60, + "wireup_timeout_s": 30, + "rendezvous_timeout_s": 90, + }, + } + + +def _write_inputs(tmp_path, cfg, name="precheck"): + models_root = tmp_path / "models" + model_dir = models_root / "tiny-llama" + model_dir.mkdir(parents=True, exist_ok=True) + (model_dir / "config.json").write_text(json.dumps(TINY_MODEL_CONFIG)) + config_path = tmp_path / f"{name}.yaml" + config_path.write_text(yaml.safe_dump(cfg)) + return str(config_path), str(models_root) + + +def _launch_instances(tmp_path, jobs, models_root): + """One mpirun per (role, idx, world, config) job; returns [(name, proc, log)].""" + mpirun = _find_mpirun() + work_dir = str(tmp_path / "work") + log_dir = tmp_path / "logs" + log_dir.mkdir(exist_ok=True) + + env = os.environ.copy() # carries the module-level UCX/NIXL pinning + env["CUDA_VISIBLE_DEVICES"] = env.get("CUDA_VISIBLE_DEVICES", "0").split(",")[0] + # run_precheck resolves metadata.model_dir_name under LLM_MODELS_ROOT. + env["LLM_MODELS_ROOT"] = models_root + + launched = [] + try: + for role, idx, world, config_path in jobs: + _launch_one(launched, mpirun, role, idx, world, config_path, work_dir, log_dir, env) + except BaseException: + # A failed Popen mid-list must not leak the instances already + # started: they never reach _wait_all's cleanup. + _terminate_process_groups([p for _, p, _ in launched]) + raise + return work_dir, launched + + +def _launch_one(launched, mpirun, role, idx, world, config_path, work_dir, log_dir, env): + name = f"{role}_{idx}" + log_path = str(log_dir / f"{name}.log") + cmd = [ + mpirun, + "--allow-run-as-root", + "--oversubscribe", + "-np", + str(world), + sys.executable, + DRIVER_SCRIPT, + "--role", + role, + "--server-idx", + str(idx), + "--config", + config_path, + "--work-dir", + work_dir, + ] + with open(log_path, "wb") as log_file: + proc = subprocess.Popen( + cmd, + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + launched.append((name, proc, log_path)) + + +def _wait_all(launched): + procs = [p for _, p, _ in launched] + deadline = time.monotonic() + _PROCESS_TIMEOUT_SECONDS + try: + for name, proc, log_path in launched: + try: + proc.wait(timeout=max(0.0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + tails = "\n".join(f"----- {n} -----\n{_log_tail(lp)}" for n, _, lp in launched) + pytest.fail(f"precheck instance {name} timed out\n{tails}") + finally: + _terminate_process_groups(procs) + + +def _read_status(work_dir, name): + text_path = os.path.join(work_dir, "status", f"{name}.status") + json_path = os.path.join(work_dir, "status", f"{name}.json") + with open(text_path) as f: + text = f.read() + with open(json_path) as f: + doc = json.load(f) + return text, doc + + +def _assert_all_passed(work_dir, launched): + failures = [] + for name, proc, log_path in launched: + if proc.returncode != 0: + failures.append(f"{name} exited rc={proc.returncode}:\n{_log_tail(log_path)}") + if failures: + pytest.fail("\n".join(failures)) + for name, _, log_path in launched: + text, doc = _read_status(work_dir, name) + assert text.startswith(f"PASS {name}"), ( + f"{name} status not PASS: {text}\n{_log_tail(log_path)}" + ) + assert doc["overall"] == "PASS" + assert doc["transceiver_runtime"] == "PYTHON" + assert doc["kv_cache_manager"] == "V2" + # The Python transceiver records bandwidth on the ctx (sender) leader + # via PerfLogManager CSVs in TRTLLM_KVCACHE_TIME_OUTPUT_PATH. Asserting + # it here ties the whole chain (perf_logger naming -> CSV -> parser) + # into the e2e check, so a writer/parser naming mismatch fails loudly + # instead of silently dropping the bandwidth field. + if name.startswith("ctx_"): + assert doc.get("per_gpu_bw_gbps", 0) > 0, ( + f"{name} verdict has no per_gpu_bw_gbps — Python perf CSVs were " + f"not written or not parsed\n{_log_tail(log_path)}" + ) + + +def _jobs(cfg, config_path): + ctx_world = cfg["worker_config"]["ctx"]["tensor_parallel_size"] + gen_world = cfg["worker_config"]["gen"]["tensor_parallel_size"] + jobs = [] + for i in range(cfg["hardware"]["num_ctx_servers"]): + jobs.append(("ctx", i, ctx_world, config_path)) + for i in range(cfg["hardware"]["num_gen_servers"]): + jobs.append(("gen", i, gen_world, config_path)) + return jobs + + +@pytest.mark.timeout(300) +@pytest.mark.parametrize( + "num_ctx,num_gen,ctx_tp,gen_tp", + [ + pytest.param(1, 1, 1, 1, id="symmetric_1x1"), + pytest.param(2, 2, 1, 1, id="multi_instance_2x2"), + pytest.param(1, 1, 2, 1, id="asymmetric_tp2_to_tp1"), + ], +) +def test_precheck_passes(tmp_path, num_ctx, num_gen, ctx_tp, gen_tp): + """All (ctx, gen) pairs transfer + verify on one GPU; every verdict PASS.""" + pytest.importorskip("mpi4py") + cfg = _disagg_yaml(num_ctx, num_gen, ctx_tp, gen_tp) + config_path, models_root = _write_inputs(tmp_path, cfg) + work_dir, launched = _launch_instances(tmp_path, _jobs(cfg, config_path), models_root) + _wait_all(launched) + _assert_all_passed(work_dir, launched) + # Every gen instance must have exercised every ctx peer. + for gj in range(num_gen): + _, doc = _read_status(work_dir, f"gen_{gj}") + peers = {c["peer"] for c in doc["cases"] if c["status"] == "PASS"} + assert peers == {f"ctx_{ci}" for ci in range(num_ctx)} + + +@pytest.mark.timeout(300) +def test_precheck_fails_fast_on_fingerprint_mismatch(tmp_path): + """Mismatched ctx/gen yamls must produce FAIL verdicts, not a hang. + + This is the failure-attribution path the SLURM gate consumes: non-zero + exit codes plus .status files whose first line names the root cause. + """ + pytest.importorskip("mpi4py") + ctx_cfg = _disagg_yaml(1, 1, 1, 1, request_lengths=(64,)) + gen_cfg = _disagg_yaml(1, 1, 1, 1, request_lengths=(32, 64)) # different fingerprint + ctx_path, models_root = _write_inputs(tmp_path, ctx_cfg, name="ctx") + gen_path, _ = _write_inputs(tmp_path, gen_cfg, name="gen") + jobs = [("ctx", 0, 1, ctx_path), ("gen", 0, 1, gen_path)] + work_dir, launched = _launch_instances(tmp_path, jobs, models_root) + _wait_all(launched) + + for name, proc, log_path in launched: + assert proc.returncode != 0, ( + f"{name} unexpectedly passed with mismatched yamls\n{_log_tail(log_path)}" + ) + text, doc = _read_status(work_dir, name) + assert text.startswith(f"FAIL {name}"), f"{name} status: {text}" + assert doc["overall"] == "FAIL" + # The gen driver saw the ctx abort its handshake: the reason must name it. + text, _ = _read_status(work_dir, "gen_0") + assert "fingerprint" in text or "abort" in text.lower() diff --git a/tests/unittest/disaggregated/test_cache_transceiver_single_process.py b/tests/unittest/disaggregated/test_cache_transceiver_single_process.py index 1e9e9a4e3899..26b9e2e9199d 100644 --- a/tests/unittest/disaggregated/test_cache_transceiver_single_process.py +++ b/tests/unittest/disaggregated/test_cache_transceiver_single_process.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Single-process test for KVCacheManager (V1/V2) + KvCacheTransceiverV2. Uses threading + ThreadSafeDistributed to create KvCacheTransceiverV2 instances @@ -13,7 +28,10 @@ # Exclude UCX IB transport (avoid NIXL setup hangs without IB) and gdr_copy # (avoid SIGSEGV at process exit from UCX rcache cleanup; gdr_copy disabled # falls back to cuda_ipc / cuda_copy without affecting correctness). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" from dataclasses import dataclass from typing import Dict, List, Optional @@ -40,13 +58,11 @@ AttentionTypeCpp = tensorrt_llm.bindings.internal.batch_manager.AttentionType -# Reduce NIXL threads for unit test: default 8 threads per agent causes heavy -# contention when creating multiple agents on a single GPU in the same process. -os.environ.setdefault("TRTLLM_NIXL_NUM_THREADS", "0") # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- NUM_LAYERS = 4 +INDEXER_HEAD_DIM = 128 NUM_KV_HEADS = 4 # 1 for MLA HEAD_DIM = 128 TOKENS_PER_BLOCK = 8 @@ -309,8 +325,10 @@ def _create_cache_manager( max_attention_window_vec: Optional[List[int]] = None, num_layers: int = NUM_LAYERS, max_batch_size: int = MAX_BATCH_SIZE, + enable_indexer_k_cache: bool = False, ) -> "KVCacheManager | KVCacheManagerV2": """Create a KVCacheManager (V1) or KVCacheManagerV2 for the given mapping.""" + assert not (enable_indexer_k_cache and use_v2), "DSA indexer K cache is V1-only" num_kv_heads = 1 if is_mla else NUM_KV_HEADS cache_type = CacheTypeCpp.SELFKONLY if is_mla else CacheTypeCpp.SELF @@ -377,6 +395,9 @@ def _create_cache_manager( mapping=mapping, dtype=DataType.FLOAT, model_config=model_config, + enable_indexer_k_cache=enable_indexer_k_cache, + indexer_k_cache_quant_block_size=128, + indexer_k_cache_index_head_dim=INDEXER_HEAD_DIM if enable_indexer_k_cache else 0, ) @@ -389,6 +410,7 @@ def _create_managers_for_instance( max_attention_window_vec: Optional[List[int]] = None, num_layers: int = NUM_LAYERS, max_batch_size: int = MAX_BATCH_SIZE, + enable_indexer_k_cache: bool = False, ) -> List: """Create cache managers for all ranks in an instance.""" managers = [] @@ -404,7 +426,13 @@ def _create_managers_for_instance( ) managers.append( _create_cache_manager( - mapping, is_mla, use_v2, max_attention_window_vec, num_layers, max_batch_size + mapping, + is_mla, + use_v2, + max_attention_window_vec, + num_layers, + max_batch_size, + enable_indexer_k_cache, ) ) return managers @@ -439,6 +467,27 @@ def _init_pool_data_v1( else: pool_tensor.zero_() + if getattr(mgr, "enable_indexer_k_cache", False): + # DSA indexer K is TP-replicated: seed by PP stage only so every + # TP rank of a stage holds identical bytes. + indexer_pool = mgr.impl.get_indexer_k_cache_pool().view(torch.uint8) + if fill_random: + generator = torch.Generator(device=indexer_pool.device).manual_seed( + seed_base + 7000 + pp_rank + ) + indexer_pool.copy_( + torch.randint( + 0, + 256, + indexer_pool.shape, + dtype=torch.uint8, + device=indexer_pool.device, + generator=generator, + ) + ) + else: + indexer_pool.zero_() + def _init_pool_data_v2( managers: List[KVCacheManagerV2], @@ -816,6 +865,77 @@ def verify_all_requests( ) +def _get_indexer_block_data(mgr, request_id, layer_idx, num_layers, pp, tp, enable_dp, req_idx): + """Per-request indexer-K bytes for one global layer on the owning rank. + + Indexer K is TP-replicated, so any TP rank of the layer's PP stage works; + with attention DP only the request's DP group holds the request. + """ + pp_rank = _pp_rank_of_layer(layer_idx, num_layers, pp) + tp_rank = req_idx % tp if enable_dp else 0 + owner = mgr[pp_rank * tp + tp_rank] + block_indices = owner.get_batch_cache_indices([request_id], layer_idx)[0] + valid = [idx for idx in block_indices if idx >= 0] + if not valid: + return None + local_layer = layer_idx - _pp_layer_start(pp_rank, num_layers, pp) + # Pool shape: (numBlocks, numLayers, kvFactor, blockSize), dtype uint8. + pool = owner.impl.get_indexer_k_cache_pool().view(torch.uint8) + return pool[valid, local_layer] + + +def _verify_indexer_k_all_requests( + request_lengths: List[int], + ctx_managers: List, + gen_managers: List, + ctx_tp: int, + ctx_pp: int, + gen_tp: int, + gen_pp: int, + ctx_enable_dp: bool, + gen_enable_dp: bool, + ctx_request_ids: List[int], + gen_request_ids: List[int], + num_layers: int, +): + """Compare the transferred DSA indexer K bytes for every request/layer.""" + for req_idx, _req_len in enumerate(request_lengths): + for layer_idx in range(num_layers): + ctx_data = _get_indexer_block_data( + ctx_managers, + ctx_request_ids[req_idx], + layer_idx, + num_layers, + ctx_pp, + ctx_tp, + ctx_enable_dp, + req_idx, + ) + gen_data = _get_indexer_block_data( + gen_managers, + gen_request_ids[req_idx], + layer_idx, + num_layers, + gen_pp, + gen_tp, + gen_enable_dp, + req_idx, + ) + if ctx_data is None or gen_data is None: + continue + assert ctx_data.shape == gen_data.shape, ( + f"Indexer shape mismatch at req={req_idx} layer={layer_idx}: " + f"ctx={ctx_data.shape} gen={gen_data.shape}" + ) + torch.testing.assert_close( + gen_data, + ctx_data, + rtol=0, + atol=0, + msg=lambda m: (f"Indexer data mismatch at req={req_idx} layer={layer_idx}: {m}"), + ) + + # --------------------------------------------------------------------------- # Main test orchestrator # --------------------------------------------------------------------------- @@ -831,6 +951,7 @@ def run_transfer_test( max_attention_window_vec: Optional[List[int]] = None, num_layers: int = NUM_LAYERS, request_lengths: Optional[List[int]] = None, + enable_indexer_k_cache: bool = False, ): """Run a full KV transfer test using KvCacheTransceiverV2.""" if request_lengths is None: @@ -849,6 +970,7 @@ def run_transfer_test( max_attention_window_vec, num_layers, max_batch_size, + enable_indexer_k_cache, ) gen_managers = _create_managers_for_instance( gen_tp, @@ -859,6 +981,7 @@ def run_transfer_test( max_attention_window_vec, num_layers, max_batch_size, + enable_indexer_k_cache, ) # 2. Initialize data: random for ctx, zeros for gen @@ -987,6 +1110,21 @@ def run_transfer_test( max_attention_window_vec=max_attention_window_vec, num_layers=num_layers, ) + if enable_indexer_k_cache: + _verify_indexer_k_all_requests( + request_lengths=request_lengths, + ctx_managers=ctx_managers, + gen_managers=gen_managers, + ctx_tp=ctx_tp, + ctx_pp=ctx_pp, + gen_tp=gen_tp, + gen_pp=gen_pp, + ctx_enable_dp=ctx_enable_dp, + gen_enable_dp=gen_enable_dp, + ctx_request_ids=ctx_request_ids, + gen_request_ids=gen_request_ids, + num_layers=num_layers, + ) # 9. Cleanup if use_v2: @@ -1311,6 +1449,50 @@ def test_cache_transceiver_boundary_lengths( print("PASSED") +# DSA (DeepSeek V3.2) indexer K cache: V1-only, MLA, TP-replicated single +# index head. Covers the REPLICATED pool view end to end through the real +# python transceiver: fan-in owner election, fan-out, and PP layer subsets +# (layer-strided ReplicatedMapper offsets). +DSA_INDEXER_CONFIGS = [ + # (ctx_tp, ctx_pp, gen_tp, gen_pp, ctx_dp, gen_dp, test_id) + (1, 1, 1, 1, False, False, "tp1_to_tp1"), + (2, 1, 1, 1, False, False, "tp2_to_tp1_fanin"), + (1, 1, 2, 1, False, False, "tp1_to_tp2_fanout"), + (1, 2, 1, 1, False, False, "pp2_to_pp1"), + (1, 1, 1, 2, False, False, "pp1_to_pp2"), + (2, 2, 1, 1, False, False, "tp2pp2_to_tp1"), + (2, 1, 2, 1, False, True, "tp2_to_dep2"), +] + + +@pytest.mark.timeout(180) +@pytest.mark.parametrize( + "ctx_tp,ctx_pp,gen_tp,gen_pp,ctx_enable_dp,gen_enable_dp", + [c[:6] for c in DSA_INDEXER_CONFIGS], + ids=[c[6] for c in DSA_INDEXER_CONFIGS], +) +def test_cache_transceiver_v1_dsa_indexer( + ctx_tp, + ctx_pp, + gen_tp, + gen_pp, + ctx_enable_dp, + gen_enable_dp, +): + """V1 KVCacheManager + DSA indexer K cache through KvCacheTransceiverV2.""" + run_transfer_test( + ctx_tp=ctx_tp, + ctx_pp=ctx_pp, + gen_tp=gen_tp, + gen_pp=gen_pp, + ctx_enable_dp=ctx_enable_dp, + gen_enable_dp=gen_enable_dp, + is_mla=True, + use_v2=False, + enable_indexer_k_cache=True, + ) + + if __name__ == "__main__": # Quick smoke test run_transfer_test(1, 1, 1, 1, False, False, False, False) diff --git a/tests/unittest/disaggregated/test_coordinator_e2e.py b/tests/unittest/disaggregated/test_coordinator_e2e.py new file mode 100644 index 000000000000..e3907818b2ba --- /dev/null +++ b/tests/unittest/disaggregated/test_coordinator_e2e.py @@ -0,0 +1,481 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""End-to-end coordinator/worker disagg serving with mocked ctx/gen workers. + +CPU-only, MPI-free, single-process (three uvicorn threads): + + * mocked ctx + gen HTTP workers serve ``/health`` + ``/v1/completions`` + (ctx returns a context_only response with disaggregated_params so the disagg + server proceeds to gen; gen returns the final completion text), + * a real ``CoordinatorServer`` (wrapping a ``DisaggCoordinatorService``) runs on + an internal port -- the gen router is a *stateful* conversation router, so gen + placement is delegated to it via ``/select``; the ctx router is round-robin + (placed locally in the disagg server), + * a real ``OpenAIDisaggServer`` in worker mode (``coordinator_url`` set, so it + holds a ``CoordinatorClient``) serves the public ``/v1/completions``. + +A real HTTP completion is sent to the disagg server and must round-trip +ctx -> (coordinator /select) -> gen, returning the gen worker's text. This +exercises the whole chain including the coordinator HTTP hop. +""" + +import asyncio +import os +import subprocess +import sys +import tempfile +import threading +import time +from pathlib import Path + +import aiohttp +import pytest +import uvicorn +import yaml +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from prometheus_client.parser import text_string_to_metric_families + +from tensorrt_llm.llmapi.disagg_utils import ( + CtxGenServerConfig, + DisaggServerConfig, + RouterConfig, + ServerRole, +) +from tensorrt_llm.logger import logger +from tensorrt_llm.serve.coordinator_server import CoordinatorServer +from tensorrt_llm.serve.disagg_coordinator import DisaggCoordinatorService +from tensorrt_llm.serve.openai_client import OpenAIHttpClient +from tensorrt_llm.serve.openai_disagg_server import OpenAIDisaggServer + +GEN_TEXT = "HELLO_FROM_GEN" + +# The uvicorn worker threads / CLI-output pump thread are background threads that +# outlive a strict thread snapshot; exempt this module (same as the other e2e). +pytestmark = pytest.mark.threadleak(enabled=False) + + +@pytest.fixture(autouse=True) +def _reset_prometheus_registry(): + """Reset role-prefixed Prometheus counters. + + Counters use the global default registry, so clear it between tests to avoid + duplicate-timeseries errors when another server is built in this process. + """ + from prometheus_client import REGISTRY + + yield + for collector in list(REGISTRY._collector_to_names): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + +def _free_port(): + import socket + + s = socket.socket() + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +class _UvicornThread: + """Run a FastAPI app in a background uvicorn server thread.""" + + def __init__(self, app, port): + self.port = port + self._server = uvicorn.Server( + uvicorn.Config(app, host="127.0.0.1", port=port, log_level="warning") + ) + self._thread = threading.Thread(target=self._server.run, daemon=True) + + def __enter__(self): + self._thread.start() + for _ in range(100): + if self._server.started: + break + time.sleep(0.1) + return self + + def __exit__(self, *a): + self._server.should_exit = True + self._thread.join(timeout=10) + + +def _mock_worker_app(role: str) -> FastAPI: + """A ctx or gen worker: /health + /server_info + /v1/completions.""" + app = FastAPI() + + @app.get("/health") + async def health(): + return Response(status_code=200) + + @app.get("/server_info") + async def server_info(): + return JSONResponse({"kv_cache_hash_algo": "v1"}) + + @app.post("/v1/completions") + async def completions(raw: Request): + body = await raw.json() + dp = body.get("disaggregated_params") or {} + model = body.get("model", "m") + if dp.get("request_type") == "context_only": + # Context phase: return disagg params so the disagg server proceeds + # to the gen worker (finish_reason "length" => needs generation). + rid = dp.get("disagg_request_id") + return JSONResponse( + { + "id": "cmpl-ctx", + "object": "text_completion", + "created": 0, + "model": model, + "prompt_token_ids": [1, 2, 3], + "choices": [ + { + "index": 0, + "text": "", + "finish_reason": "length", + "disaggregated_params": { + "request_type": "context_only", + "ctx_request_id": rid, + "disagg_request_id": rid, + }, + } + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 0, "total_tokens": 3}, + } + ) + # Generation phase: final answer. + return JSONResponse( + { + "id": "cmpl-gen", + "object": "text_completion", + "created": 0, + "model": model, + "choices": [{"index": 0, "text": GEN_TEXT, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + ) + + return app + + +class _ReadinessClient: + """Minimal client the coordinator uses only for server readiness probing. + + In coordinator/worker mode the coordinator never sends completions (the disagg + servers do), so its readiness client needs no metrics -- reusing the real + ``check_ready_for_servers`` keeps the probe faithful while avoiding a second + set of role-prefixed Prometheus counters in this single-process test. + """ + + def __init__(self, router): + self._router = router + self._session = aiohttp.ClientSession() + + async def check_ready(self): + ready, unready = await OpenAIHttpClient.check_ready_for_servers( + self._session, self._router.servers + ) + if ready: + await self._router.prepare_servers(ready) + return ready, unready + + async def shutdown(self): + await self._session.close() + + +def _make_config(ctx_url, gen_url, public_port): + def host_port(url): + return url.split(":")[0], int(url.split(":")[1]) + + ctx_host, ctx_port = host_port(ctx_url) + gen_host, gen_port = host_port(gen_url) + return DisaggServerConfig( + server_configs=[ + CtxGenServerConfig(type="ctx", hostname=ctx_host, port=ctx_port), + CtxGenServerConfig(type="gen", hostname=gen_host, port=gen_port), + ], + hostname="127.0.0.1", + port=public_port, + # ctx: stateless (placed locally in the disagg server); + # gen: stateful conversation router (placement delegated to coordinator). + ctx_router_config=RouterConfig(type="round_robin", server_role=ServerRole.CONTEXT), + gen_router_config=RouterConfig(type="conversation", server_role=ServerRole.GENERATION), + ) + + +class _CoordinatorThread: + """Run a CoordinatorServer (DisaggCoordinatorService) in a uvicorn thread.""" + + def __init__(self, config): + self.port = _free_port() + self.url = f"http://127.0.0.1:{self.port}" + # The coordinator builds its own owner routers from config. + self._coordinator = DisaggCoordinatorService( + config, client_factory=lambda router, role, mr=1: _ReadinessClient(router) + ) + self._impl = _UvicornThread(CoordinatorServer(self._coordinator).app, self.port) + + def __enter__(self): + self._impl.__enter__() + return self + + def __exit__(self, *a): + self._impl.__exit__(*a) + + +async def _wait_healthy(url, timeout_s=30.0): + deadline = time.time() + timeout_s + async with aiohttp.ClientSession() as sess: + while time.time() < deadline: + try: + async with sess.get(f"{url}/health", timeout=1) as r: + if r.status == 200: + return True + except Exception: + pass + await asyncio.sleep(0.2) + return False + + +def test_disagg_completion_e2e_through_coordinator( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path)) + with ( + _UvicornThread(_mock_worker_app("ctx"), _free_port()) as ctx, + _UvicornThread(_mock_worker_app("gen"), _free_port()) as gen, + ): + ctx_url = f"127.0.0.1:{ctx.port}" + gen_url = f"127.0.0.1:{gen.port}" + public_port = _free_port() + config = _make_config(ctx_url, gen_url, public_port) + + with _CoordinatorThread(config) as coord: + assert asyncio.run(_wait_healthy(coord.url)), "coordinator never became healthy" + + disagg = OpenAIDisaggServer(config=config, coordinator_url=coord.url) + with _UvicornThread(disagg.app, public_port) as server: + base = f"http://127.0.0.1:{server.port}" + assert asyncio.run(_wait_healthy(base)), "disagg server never became healthy" + + async def drive(): + async with aiohttp.ClientSession() as sess: + payload = {"model": "m", "prompt": "hello", "max_tokens": 8} + # X-Session-ID -> conversation_id, so the gen router + # (conversation) delegates placement to the coordinator. + headers = {"X-Session-ID": "conv-e2e"} + async with sess.post( + f"{base}/v1/completions", json=payload, headers=headers, timeout=30 + ) as r: + assert r.status == 200, await r.text() + return await r.json() + + body = asyncio.run(drive()) + + # The full ctx -> coordinator/select -> gen chain returned the gen text. + assert body["choices"][0]["text"] == GEN_TEXT, body + assert body["choices"][0]["finish_reason"] == "stop" + + +def _write_config(path, ctx_url, gen_url, public_port, num_workers=1): + """A disagg config YAML: round-robin ctx, conversation gen (delegated).""" + cfg = { + "hostname": "127.0.0.1", + "port": public_port, + "num_workers": num_workers, + "context_servers": { + "num_instances": 1, + "urls": [ctx_url], + "router": {"type": "round_robin"}, + }, + "generation_servers": { + "num_instances": 1, + "urls": [gen_url], + "router": {"type": "conversation"}, + }, + } + with open(path, "w") as f: + yaml.safe_dump(cfg, f) + + +def _request_counter_values(metrics_text: str) -> dict[str, float]: + requested_samples = { + "total_requests_total", + "nonstream_requests_total", + "total_responses_total", + } + return { + sample.name: sample.value + for family in text_string_to_metric_families(metrics_text) + for sample in family.samples + if sample.name in requested_samples + } + + +def test_disagg_completion_e2e_web_concurrency_4() -> None: + """Exercise requests and Prometheus scrapes through a real CLI fleet. + + ``num_workers=4`` starts a coordinator and four disaggregated servers on the + public port. Completions traverse fleet workers, the coordinator, and mock + context/generation workers. Independent scrape connections can land on any + fleet worker, and must all return the same deployment-wide request totals. + """ + logger.set_level("info") # trtllm logger defaults to "error"; show progress + WORKERS = 4 + with ( + _UvicornThread(_mock_worker_app("ctx"), _free_port()) as ctx, + _UvicornThread(_mock_worker_app("gen"), _free_port()) as gen, + ): + ctx_url = f"127.0.0.1:{ctx.port}" + gen_url = f"127.0.0.1:{gen.port}" + # port-1 is the coordinator, so pick a public port with room below it. + public_port = _free_port() + coord_port = public_port - 1 + + with tempfile.TemporaryDirectory() as td: + cfg_path = os.path.join(td, "disagg.yaml") + _write_config(cfg_path, ctx_url, gen_url, public_port, WORKERS) + + env = dict(os.environ) + # Unbuffered so the child's launch logs stream out live (else stdout + # to a pipe is block-buffered and nothing shows until it exits). + env["PYTHONUNBUFFERED"] = "1" + # trtllm logger defaults to "error"; raise it so the coordinator/fleet + # launch logs are visible in the streamed [cli] output. + env["TLLM_LOG_LEVEL"] = "info" + # The CLI parent must create a fresh directory for this deployment + # and pass it to all frontend workers. + env.pop("PROMETHEUS_MULTIPROC_DIR", None) + # A plain HTTP fleet, never an MPI rank -- strip any launcher env so + # the CLI's own strip is not even relied upon. + for k in list(env): + if k.startswith( + ("SLURM_", "PMIX_", "PMI_", "OMPI_", "UCX_", "I_MPI_", "HYDRA_", "MPI_") + ): + env.pop(k) + + logger.info( + f"mock ctx={ctx_url} gen={gen_url}; launching " + f"`trtllm-serve disaggregated` num_workers={WORKERS}, " + f"public={public_port} coordinator={coord_port}" + ) + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "tensorrt_llm.commands.serve", + "disaggregated", + "-c", + cfg_path, + ], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + start_new_session=True, + ) + + # Stream the CLI child's stdout live (prefixed) so coordinator/fleet + # startup is visible in real time instead of only at teardown. + def _pump(): + for line in proc.stdout: + logger.info(f"[cli] {line.rstrip()}") + + pump = threading.Thread(target=_pump, daemon=True) + pump.start() + try: + base = f"http://127.0.0.1:{public_port}" + coord = f"http://127.0.0.1:{coord_port}" + + async def _wait_all(): + # Both the coordinator (port-1) and the public fleet must be + # up before the fleet reports ready (fleet is_ready proxies + # the coordinator). + logger.info("waiting for coordinator health...") + assert await _wait_healthy(coord, 120.0), "coordinator never became healthy" + logger.info("coordinator healthy; waiting for fleet health...") + assert await _wait_healthy(base, 120.0), "disagg fleet never became healthy" + logger.info("fleet healthy") + + asyncio.run(_wait_all()) + + async def drive(): + # Separate concurrent connections allow SO_REUSEPORT to + # distribute requests across the four frontend processes. + async def complete(i): + connector = aiohttp.TCPConnector(force_close=True) + async with aiohttp.ClientSession(connector=connector) as sess: + payload = {"model": "m", "prompt": f"hello-{i}", "max_tokens": 8} + headers = {"X-Session-ID": f"conv-{i}"} + async with sess.post( + f"{base}/v1/completions", json=payload, headers=headers, timeout=30 + ) as r: + assert r.status == 200, await r.text() + return (await r.json())["choices"][0]["text"] + + return await asyncio.gather(*(complete(i) for i in range(8))) + + texts = asyncio.run(drive()) + assert all(t == GEN_TEXT for t in texts), texts + logger.info(f"all {len(texts)} requests round-tripped to GEN_TEXT") + + async def scrape_metrics(): + async def scrape(): + connector = aiohttp.TCPConnector(force_close=True) + async with aiohttp.ClientSession(connector=connector) as sess: + async with sess.get( + f"{base}/prometheus/metrics", + headers={"Connection": "close"}, + timeout=30, + ) as r: + assert r.status == 200, await r.text() + return _request_counter_values(await r.text()) + + # Each scrape uses a separate connection and can be handled + # by a different SO_REUSEPORT frontend process. + return await asyncio.gather(*(scrape() for _ in range(16))) + + scrape_values = asyncio.run(scrape_metrics()) + expected = { + "total_requests_total": len(texts), + "nonstream_requests_total": len(texts), + "total_responses_total": len(texts), + } + assert scrape_values == [expected] * len(scrape_values) + finally: + # Kill the whole process group: the CLI parent + coordinator + + # all uvicorn workers. Terminating only proc leaves the workers + # holding the stdout pipe open, so _pump never sees EOF. + logger.info("terminating CLI process group") + import signal + + pgid = os.getpgid(proc.pid) + os.killpg(pgid, signal.SIGTERM) + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + os.killpg(pgid, signal.SIGKILL) + proc.wait() + pump.join(timeout=10) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v", "-s"])) diff --git a/tests/unittest/disaggregated/test_coordinator_worker.py b/tests/unittest/disaggregated/test_coordinator_worker.py new file mode 100644 index 000000000000..39ae11bc33a4 --- /dev/null +++ b/tests/unittest/disaggregated/test_coordinator_worker.py @@ -0,0 +1,604 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Coordinator/worker disagg routing: cross-process placement contract. + +CPU-only, MPI-free. Wires the real coordinator surface to a real worker-side +coordinator: + + * fake ctx/gen HTTP workers answer ``/health`` (readiness only), + * a real ``CoordinatorServer`` (wrapping a ``DisaggCoordinatorService`` over the + configured routers) runs in a uvicorn thread on an internal port, + * a ``CoordinatorClient`` (what a worker holds) wraps only *stateful* routers + in a ``CoordinatorDelegatingRouter`` whose ``get_next_server`` computes the + routing key locally and POSTs it to the coordinator's ``/select``; + ``finish_request`` releases coordinator-side state via ``/finish`` and the + returned handle. *Stateless* routers (round_robin) are used as-is and place + locally in the worker. + +This proves the routing split: stateful routers (conversation, kv_cache_aware) +delegate to the coordinator via ``routing_key`` + ``get_next_server_by_key``, +while stateless routers never touch the coordinator. +""" + +import asyncio +import json +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import AsyncMock + +import aiohttp +import pytest +import uvicorn + +from tensorrt_llm.llmapi.disagg_utils import ( + CtxGenServerConfig, + DisaggClusterConfig, + DisaggServerConfig, + RouterConfig, + ServerRole, +) +from tensorrt_llm.serve.coordinator_server import CoordinatorServer +from tensorrt_llm.serve.disagg_coordinator import ( + COORDINATOR_RESERVATION_TIMEOUT_ENV, + CoordinatorClient, + DisaggCoordinatorService, + coordinator_reservation_timeout, +) +from tensorrt_llm.serve.openai_protocol import ( + ChatCompletionRequest, + CompletionRequest, + DisaggregatedParams, +) +from tensorrt_llm.serve.router import ( + KV_CACHE_HASH_ALGO_V1, + KV_CACHE_HASH_ALGO_V2, + CoordinatorDelegatingRouter, + KvCacheAwareRouter, +) +from tensorrt_llm.serve.router_utils import BlockHashMixin as SharedBlockHashMixin + + +@pytest.fixture(autouse=True) +def _reset_prometheus_registry(): + """Reset role-prefixed Prometheus counters. + + Tests create multiple coordinators in one process, so clear their shared + default registry between tests to avoid duplicate-timeseries errors. + """ + from prometheus_client import REGISTRY + + yield + for collector in list(REGISTRY._collector_to_names): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + +def _free_port(): + import socket + + s = socket.socket() + # SO_REUSEADDR so a port left in TIME_WAIT by a sibling server in the same + # suite can be rebound immediately (closes the alloc->bind race window). + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +class _FakeWorker: + """Minimal HTTP worker exposing health and routing metadata.""" + + def __init__(self, server_info=None): + self.port = _free_port() + server_info_body = json.dumps(server_info or {}).encode() + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def do_GET(self): + if self.path == "/health": + self.send_response(200) + elif self.path == "/server_info": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(server_info_body))) + else: + self.send_response(404) + self.end_headers() + if self.path == "/server_info": + self.wfile.write(server_info_body) + + self._httpd = ThreadingHTTPServer(("127.0.0.1", self.port), Handler) + self._thread = threading.Thread(target=self._httpd.serve_forever, daemon=True) + + @property + def url(self): + return f"127.0.0.1:{self.port}" + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *a): + self._httpd.shutdown() + + +def _make_config(ctx_urls, gen_urls, ctx_router_type, gen_router_type): + server_configs = [ + CtxGenServerConfig(type="ctx", hostname=u.split(":")[0], port=int(u.split(":")[1])) + for u in ctx_urls + ] + [ + CtxGenServerConfig(type="gen", hostname=u.split(":")[0], port=int(u.split(":")[1])) + for u in gen_urls + ] + return DisaggServerConfig( + server_configs=server_configs, + ctx_router_config=RouterConfig(type=ctx_router_type, server_role=ServerRole.CONTEXT), + gen_router_config=RouterConfig(type=gen_router_type, server_role=ServerRole.GENERATION), + ) + + +def _client_factory(router, role, max_retries=1): + from tensorrt_llm.serve.openai_client import OpenAIHttpClient + + return OpenAIHttpClient(router, role, 30, max_retries) + + +class _CoordinatorThread: + """Run a CoordinatorServer (DisaggCoordinatorService) in a background thread.""" + + def __init__(self, config): + self.port = _free_port() + self.url = f"http://127.0.0.1:{self.port}" + # The coordinator builds its own owner routers from config. + self._cluster = DisaggCoordinatorService(config, _client_factory) + self._server = uvicorn.Server( + uvicorn.Config( + CoordinatorServer(self._cluster).app, + host="127.0.0.1", + port=self.port, + log_level="warning", + ) + ) + self._thread = threading.Thread(target=self._server.run, daemon=True) + + def __enter__(self): + self._thread.start() + for _ in range(100): + if self._server.started: + break + time.sleep(0.1) + return self + + def __exit__(self, *a): + self._server.should_exit = True + self._thread.join(timeout=10) + + +async def _wait_coord_ready(url, timeout_s=30.0): + deadline = time.time() + timeout_s + async with aiohttp.ClientSession() as sess: + while time.time() < deadline: + try: + async with sess.get(f"{url}/health", timeout=1) as r: + if r.status == 200: + return True + except Exception: + pass + await asyncio.sleep(0.2) + return False + + +def test_coordinator_rejects_unknown_role(): + config = _make_config([], [], "round_robin", "round_robin") + coordinator = DisaggCoordinatorService(config, _client_factory) + + with pytest.raises(ValueError, match="Unsupported coordinator role"): + coordinator._router_for_role("typo") + + +def test_kv_router_rejects_mixed_prepared_hash_algorithms(): + router = KvCacheAwareRouter(server_role=ServerRole.CONTEXT, servers=["server-a", "server-b"]) + router._prepared_ready_servers.update(router.servers) + router._server_state["server-a"].set_hash_algo(KV_CACHE_HASH_ALGO_V1) + router._server_state["server-b"].set_hash_algo(KV_CACHE_HASH_ALGO_V2) + + with pytest.raises(RuntimeError, match="one hash algorithm per role"): + router.routing_key_config() + + +@pytest.mark.asyncio +async def test_coordinator_exposes_role_hash_algorithm(): + ctx_server = "127.0.0.1:1234" + gen_server = "127.0.0.1:1235" + config = _make_config([ctx_server], [gen_server], "kv_cache_aware", "kv_cache_aware") + coordinator = DisaggCoordinatorService(config, _client_factory) + ctx_router = coordinator.ctx_router + gen_router = coordinator.gen_router + assert isinstance(ctx_router, KvCacheAwareRouter) + assert isinstance(gen_router, KvCacheAwareRouter) + ctx_router._prepared_ready_servers.add(ctx_server) + gen_router._prepared_ready_servers.add(gen_server) + ctx_router._server_state[ctx_server].set_hash_algo(KV_CACHE_HASH_ALGO_V2) + gen_router._server_state[gen_server].set_hash_algo(KV_CACHE_HASH_ALGO_V1) + + info = await coordinator.cluster_info() + + assert info["routing_key_configs"]["context"] == { + "tokens_per_block": 32, + "kv_cache_hash_algo": KV_CACHE_HASH_ALGO_V2, + } + assert info["routing_key_configs"]["generation"] == { + "tokens_per_block": 32, + "kv_cache_hash_algo": KV_CACHE_HASH_ALGO_V1, + } + + +@pytest.mark.asyncio +async def test_coordinator_expires_stale_reservation(): + config = _make_config([], ["gen:8000"], "round_robin", "conversation") + coordinator = DisaggCoordinatorService( + config, + _client_factory, + reservation_timeout_secs=0.01, + ) + + await coordinator.select("generation", "conversation", 123, None) + assert coordinator.gen_router._server_content_load["gen:8000"] == 1 + + await asyncio.sleep(0.02) + + assert coordinator.gen_router._server_content_load["gen:8000"] == 0 + assert coordinator._reservation_tasks == {} + + +def test_coordinator_compacts_route_info(): + compact = DisaggCoordinatorService._compact_route_info( + { + "block_hashes": [["large-hash"]], + "hash_algo": KV_CACHE_HASH_ALGO_V2, + "matches": [64, 32], + "match_length": 64, + "num_tokens": 128, + "server_info": { + "tokens_per_block": 32, + "disaggregated_params": {"ctx_info_endpoint": "tcp://ctx"}, + }, + } + ) + + assert compact == { + "match_length": 64, + "num_tokens": 128, + "server_info": {"disaggregated_params": {"ctx_info_endpoint": "tcp://ctx"}}, + } + + +def test_coordinator_reservation_timeout_env(monkeypatch): + monkeypatch.delenv(COORDINATOR_RESERVATION_TIMEOUT_ENV, raising=False) + assert coordinator_reservation_timeout() == 180 + + monkeypatch.setenv(COORDINATOR_RESERVATION_TIMEOUT_ENV, "60") + assert coordinator_reservation_timeout() == 60 + + +def test_coordinator_client_configures_empty_delegating_kv_router(): + config = _make_config([], [], "kv_cache_aware", "round_robin") + client = CoordinatorClient("http://coordinator", config) + assert isinstance(client.ctx_router, CoordinatorDelegatingRouter) + local = client.ctx_router._local + assert isinstance(local, KvCacheAwareRouter) + assert local.servers == [] + + client._sync_delegating_router_configs( + { + "routing_key_configs": { + "context": { + "tokens_per_block": 64, + "kv_cache_hash_algo": KV_CACHE_HASH_ALGO_V2, + } + } + } + ) + + request = CompletionRequest(model="m", prompt=[1, 2, 3]) + routing_key = local.routing_key(request) + assert local.servers == [] + assert local._tokens_per_block == 64 + assert set(routing_key["block_hashes_by_algo"]) == {KV_CACHE_HASH_ALGO_V2} + assert routing_key["num_tokens"] == 3 + assert "token_lists" not in routing_key + + +@pytest.mark.asyncio +async def test_coordinator_client_readiness_is_cached(): + config = _make_config([], [], "round_robin", "round_robin") + client = CoordinatorClient("http://coordinator", config) + client._is_ready = True + + assert await client.is_ready() is True + assert client._session is None + await client.stop() + + +@pytest.mark.asyncio +async def test_coordinator_state_sync_starts_in_background(): + config = _make_config([], [], "round_robin", "round_robin") + client = CoordinatorClient("http://coordinator", config) + client._state_sync_interval_s = 10 + client._await_coordinator = AsyncMock( + return_value={ + "is_ready": True, + "server_lists": {"context": [], "generation": []}, + } + ) + client._sync_coordinator_state = AsyncMock() + + await client.start() + + assert client._is_ready is True + client._sync_coordinator_state.assert_called_once_with(10) + await client.stop() + + +def test_service_discovery_sets_coordinator_state_sync_interval(): + config = _make_config([], [], "round_robin", "round_robin") + config.disagg_cluster_config = DisaggClusterConfig( + cluster_uri="http://cluster-storage", + heartbeat_interval_sec=7, + ) + + client = CoordinatorClient("http://coordinator", config) + + assert client._state_sync_interval_s == 7 + + +@pytest.mark.asyncio +async def test_cluster_info_updates_readiness_and_stateless_servers(): + config = _make_config(["ctx-old:8001"], [], "round_robin", "round_robin") + client = CoordinatorClient("http://coordinator", config) + client.ctx_router.remove_server = AsyncMock(wraps=client.ctx_router.remove_server) + client.ctx_router.add_server = AsyncMock(wraps=client.ctx_router.add_server) + client.ctx_router.prepare_servers = AsyncMock() + client.ctx_router._fetch_server_info = AsyncMock(return_value={}) + + await client._apply_cluster_info( + { + "is_ready": True, + "server_lists": { + "context": ["ctx-new:8001"], + "generation": [], + }, + } + ) + + assert await client.is_ready() is True + assert client.ctx_router.servers == ["ctx-new:8001"] + client.ctx_router.remove_server.assert_awaited_once_with("ctx-old:8001") + client.ctx_router.add_server.assert_awaited_once_with("ctx-new:8001") + client.ctx_router.prepare_servers.assert_awaited_once() + await client.stop() + + +def test_content_affinity_key_uses_fixed_seed(): + request = ChatCompletionRequest(model="m", messages=[{"role": "user", "content": "hello"}]) + + assert KvCacheAwareRouter._content_affinity_key(request) == 7306401829117098140 + + +def test_prefix_token_cache_retokenizes_extended_text(): + class BoundarySensitiveTokenizer: + def __init__(self): + self.calls = [] + + def encode(self, text, add_special_tokens=False): + assert add_special_tokens is False + self.calls.append(text) + return {"ab": [1], "abc": [2], "c": [3]}[text] + + tokenizer = BoundarySensitiveTokenizer() + block_hashing = SharedBlockHashMixin() + block_hashing._init_block_hashing() + + assert block_hashing._encode_with_prefix_cache("ab", 1, tokenizer) == [1] + assert block_hashing._encode_with_prefix_cache("abc", 1, tokenizer) == [2] + assert block_hashing._encode_with_prefix_cache("abc", 1, tokenizer) == [2] + assert tokenizer.calls == ["ab", "abc"] + + +def test_stateless_router_places_locally_in_worker(): + """Verify stateless round-robin placement remains local. + + The worker uses the real router without calling the coordinator. + """ + from tensorrt_llm.serve.router import CoordinatorDelegatingRouter, RoundRobinRouter + + with _FakeWorker() as ctx0, _FakeWorker() as gen0, _FakeWorker() as gen1: + config = _make_config([ctx0.url], [gen0.url, gen1.url], "round_robin", "round_robin") + with _CoordinatorThread(config) as coord: + assert asyncio.run(_wait_coord_ready(coord.url)), "coordinator never became healthy" + + async def drive(): + remote = CoordinatorClient(coord.url, config) + # Stateless -> real local router, not a delegating proxy. + assert isinstance(remote.gen_router, RoundRobinRouter) + assert not isinstance(remote.gen_router, CoordinatorDelegatingRouter) + picks = [] + for _ in range(4): + req = CompletionRequest(model="m", prompt="hello") + server, _info = await remote.gen_router.get_next_server(req) + picks.append(server) + await remote.gen_router.finish_request(req) + await remote.stop() + return picks + + picks = asyncio.run(drive()) + assert set(picks) == {gen0.url, gen1.url}, ( + f"local round-robin should hit both gen workers, got {picks}" + ) + + +def test_static_stateless_router_prepares_generation_first_server_info(): + """Fleet startup prepares static local routers for generation-first.""" + from tensorrt_llm.serve.router import RoundRobinRouter + + ctx_info_endpoint = "tcp://127.0.0.1:12345" + ctx_server_info = {"disaggregated_params": {"ctx_info_endpoint": ctx_info_endpoint}} + with _FakeWorker(ctx_server_info) as ctx0, _FakeWorker() as gen0: + config = _make_config([ctx0.url], [gen0.url], "round_robin", "round_robin") + config.schedule_style = "generation_first" + with _CoordinatorThread(config) as coord: + assert asyncio.run(_wait_coord_ready(coord.url)) + + async def drive(): + remote = CoordinatorClient(coord.url, config) + await remote.start() + assert isinstance(remote.ctx_router, RoundRobinRouter) + request = CompletionRequest(model="m", prompt="hello") + server, info = await remote.ctx_router.get_next_server(request) + await remote.stop() + return server, info + + server, info = asyncio.run(drive()) + assert server == ctx0.url + assert ( + info["server_info"]["disaggregated_params"]["ctx_info_endpoint"] + == ctx_info_endpoint + ) + + +@pytest.mark.asyncio +async def test_stateless_router_syncs_coordinator_server_add_remove(): + """Coordinator server lists propagate metadata topology changes.""" + config = _make_config(["ctx-old:8000"], [], "round_robin", "round_robin") + client = CoordinatorClient("http://coordinator", config) + client.ctx_router._fetch_server_info = AsyncMock(return_value={}) + + await client._sync_stateless_routers( + {"server_lists": {"context": ["ctx-new:8001"], "generation": []}} + ) + + assert client.ctx_router.servers == ["ctx-new:8001"] + client.ctx_router._fetch_server_info.assert_awaited_once_with("ctx-new:8001", None) + await client.stop() + + +@pytest.mark.asyncio +async def test_stateless_router_keeps_old_server_when_replacement_is_unprepared(): + config = _make_config(["ctx-old:8000"], [], "round_robin", "round_robin") + client = CoordinatorClient("http://coordinator", config) + client.ctx_router._prepared_ready_servers.add("ctx-old:8000") + client.ctx_router._fetch_server_info = AsyncMock( + side_effect=RuntimeError("server info unavailable") + ) + + with pytest.raises(RuntimeError, match="Failed to prepare ctx-new:8001"): + await client._sync_stateless_routers( + {"server_lists": {"context": ["ctx-new:8001"], "generation": []}} + ) + + assert client.ctx_router.servers == ["ctx-old:8000"] + await client.stop() + + +def test_conversation_coordinator_sticky_by_conv_id(): + """Verify conversation IDs remain sticky through delegated routing. + + The stateful generation router delegates placement to coordinator + ``/select``. + """ + from tensorrt_llm.serve.router import CoordinatorDelegatingRouter + + with _FakeWorker() as ctx0, _FakeWorker() as gen0, _FakeWorker() as gen1: + config = _make_config([ctx0.url], [gen0.url, gen1.url], "round_robin", "conversation") + with _CoordinatorThread(config) as coord: + assert asyncio.run(_wait_coord_ready(coord.url)) + + def _req(conv_id, request_id): + return CompletionRequest( + model="m", + prompt="hi", + disaggregated_params=DisaggregatedParams( + request_type="generation_only", + ctx_request_id=request_id, + conversation_id=conv_id, + ), + ) + + async def drive(): + remote = CoordinatorClient(coord.url, config) + await remote.start() + # Stateful -> wrapped in a coordinator-delegating router. + assert isinstance(remote.gen_router, CoordinatorDelegatingRouter) + assert await remote.is_ready() is True + first_request = _req("conv-A", 1) + first, _ = await remote.gen_router.get_next_server(first_request) + await remote.gen_router.finish_request(first_request) + # Repeated conv-A requests must land on the same worker. + repeats = [] + for request_id in range(2, 5): + request = _req("conv-A", request_id) + s, _ = await remote.gen_router.get_next_server(request) + repeats.append(s) + await remote.gen_router.finish_request(request) + await remote.stop() + return first, repeats + + first, repeats = asyncio.run(drive()) + assert all(s == first for s in repeats), ( + f"conv-A must be sticky, got first={first} repeats={repeats}" + ) + + +def test_worker_generates_disagg_request_id_before_generation_routing(): + """Generation routing uses the ID generated by the coordinator client.""" + from tensorrt_llm.serve.router import CoordinatorDelegatingRouter + + with _FakeWorker() as ctx0, _FakeWorker() as gen0: + config = _make_config([ctx0.url], [gen0.url], "round_robin", "conversation") + with _CoordinatorThread(config) as coord: + assert asyncio.run(_wait_coord_ready(coord.url)) + + async def drive(): + remote = CoordinatorClient(coord.url, config) + assert isinstance(remote.gen_router, CoordinatorDelegatingRouter) + assigned_id = await remote.get_disagg_request_id() + request = CompletionRequest( + model="m", + prompt="hello", + disaggregated_params=DisaggregatedParams( + request_type="generation_only", + ctx_request_id=assigned_id, + disagg_request_id=None, + conversation_id="conv-A", + ), + ) + await remote.gen_router.get_next_server(request) + assert request.disaggregated_params.disagg_request_id is None + assert request.disaggregated_params.ctx_request_id == assigned_id + await remote.gen_router.finish_request(request) + await remote.stop() + return assigned_id + + assigned_id = asyncio.run(drive()) + assert assigned_id > 0 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v", "-s"])) diff --git a/tests/unittest/disaggregated/test_deepseek_v4_kv_transfer.py b/tests/unittest/disaggregated/test_deepseek_v4_kv_transfer.py index 0fbe520ee067..cd055f17b1f3 100644 --- a/tests/unittest/disaggregated/test_deepseek_v4_kv_transfer.py +++ b/tests/unittest/disaggregated/test_deepseek_v4_kv_transfer.py @@ -1,22 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Test KV Transfer for DeepseekV4CacheManager with KvCacheTransceiverV2. -Uses threading + ThreadSafeDistributed to create KvCacheTransceiverV2 instances -in a single process. Validates all DeepseekV4AttentionType cache transfers across -different TP/PP/DP configurations. +Drives the shared threaded single-process harness (``kv_transfer_harness``) +with DeepSeek-V4 cache managers. Validates all DeepseekV4AttentionType cache +transfers across different TP/PP/DP configurations. """ -import os -import threading -import uuid -from typing import Dict, List, Optional, Tuple +import functools +from typing import Dict, List, Optional, Sequence, Tuple import pytest import torch +from kv_transfer_harness import ( + MAX_BATCH_SIZE, + MAX_SEQ_LEN, + TOKENS_PER_BLOCK, + VOCAB_SIZE, + get_layers_per_pp, + run_kv_transfer_test, +) -import tensorrt_llm -import tensorrt_llm.bindings -import tensorrt_llm.tensorrt_llm_transfer_agent_binding # noqa: F401 -from tensorrt_llm import DisaggregatedParams, Mapping, SamplingParams +from tensorrt_llm import Mapping from tensorrt_llm._torch.attention_backend.sparse.deepseek_v4 import DeepseekV4CacheManager from tensorrt_llm._torch.attention_backend.sparse.deepseek_v4.deepseek_v4 import ( DEEPSEEK_V4_OVERLAP_COMPRESSOR_RATIO, @@ -25,22 +43,10 @@ ) from tensorrt_llm._torch.disaggregation.resource.kv_extractor import KVRegionExtractorV1 from tensorrt_llm._torch.disaggregation.resource.utils import get_physical_pool, get_pool_bytes -from tensorrt_llm._torch.disaggregation.transceiver import KvCacheTransceiverV2 -from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest, LlmRequestType -from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm._utils import TensorWrapper, convert_to_torch_tensor from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.internal.batch_manager import CacheType as CacheTypeCpp -from tensorrt_llm.llmapi.llm_args import ( - CacheTransceiverConfig, - DeepSeekV4SparseAttentionConfig, - KvCacheConfig, -) - -# Reduce NIXL threads for unit test: default 8 threads per agent causes heavy -# contention when creating multiple agents on a single GPU in the same process. -os.environ.setdefault("TRTLLM_NIXL_NUM_THREADS", "0") - +from tensorrt_llm.llmapi.llm_args import DeepSeekV4SparseAttentionConfig, KvCacheConfig # --------------------------------------------------------------------------- # Constants matching DeepseekV4CacheManager defaults @@ -48,200 +54,13 @@ HEAD_DIM = 256 # Reduced from 512: test validates transfer, not attention correctness INDEX_HEAD_DIM = 128 WINDOW_SIZE = 128 -TOKENS_PER_BLOCK = 128 -MAX_SEQ_LEN = 512 -MAX_BATCH_SIZE = 16 -VOCAB_SIZE = 129280 NUM_KV_HEADS = 1 -INDEXER_QUANT_BLOCK_SIZE = 128 - # DeepSeek-V4 specific ratios (mirrors module constants) SPARSE_RATIO = DEEPSEEK_V4_SPARSE_RATIO OVERLAP_COMPRESSOR_RATIO = DEEPSEEK_V4_OVERLAP_COMPRESSOR_RATIO -# --------------------------------------------------------------------------- -# ThreadSafeDistributed: threading.Barrier-based Distributed mock -# --------------------------------------------------------------------------- -class ThreadSafeDistributed: - """Distributed mock using threading.Barrier for single-process multi-rank testing. - - Provides the same interface as TorchDistributedWrapper from test_py_cache_transceiver_mp.py - but uses Barrier + Lock + shared dict instead of torch.distributed. - """ - - def __init__( - self, - local_rank: int, - world_size: int, - tp_size: int, - pp_size: int, - tp_rank: int, - pp_rank: int, - shared: dict, - ): - self.rank = local_rank - self._world_size = world_size - self._tp_size = tp_size - self._pp_size = pp_size - self._tp_rank = tp_rank - self._pp_rank = pp_rank - self._s = shared - self._bcast_idx = 0 - self._ag_idx = 0 - self._pp_ag_idx = 0 - self._tp_ag_idx = 0 - - @property - def tp_size(self): - return self._tp_size - - @property - def pp_size(self): - return self._pp_size - - @property - def world_size(self): - return self._world_size - - def broadcast(self, obj, root=0): - idx = self._bcast_idx - self._bcast_idx += 1 - key = f"bcast_{idx}" - if self.rank == root: - self._s[key] = obj - self._s["barrier"].wait() - result = self._s[key] - self._s["barrier"].wait() - return result - - def allgather(self, obj): - idx = self._ag_idx - self._ag_idx += 1 - key = f"ag_{idx}" - with self._s["lock"]: - if key not in self._s: - self._s[key] = [None] * self._world_size - self._s[key][self.rank] = obj - self._s["barrier"].wait() - result = list(self._s[key]) - self._s["barrier"].wait() - return result - - def pp_allgather(self, obj): - idx = self._pp_ag_idx - self._pp_ag_idx += 1 - key = f"pp_ag_{idx}_tp{self._tp_rank}" - with self._s["lock"]: - if key not in self._s: - self._s[key] = [None] * self._pp_size - self._s[key][self._pp_rank] = obj - self._s["barrier"].wait() - result = list(self._s[key]) - self._s["barrier"].wait() - return result - - def tp_allgather(self, obj): - idx = self._tp_ag_idx - self._tp_ag_idx += 1 - key = f"tp_ag_{idx}_pp{self._pp_rank}" - with self._s["lock"]: - if key not in self._s: - self._s[key] = [None] * self._tp_size - self._s[key][self._tp_rank] = obj - self._s["barrier"].wait() - result = list(self._s[key]) - self._s["barrier"].wait() - return result - - -# --------------------------------------------------------------------------- -# Threading helpers -# --------------------------------------------------------------------------- -def run_concurrent(items, fn): - """Run fn(item) for each item concurrently in threads and propagate errors.""" - errors = [None] * len(items) - results = [None] * len(items) - - def _worker(idx, item): - try: - results[idx] = fn(item) - except Exception as e: - errors[idx] = e - - threads = [threading.Thread(target=_worker, args=(i, item)) for i, item in enumerate(items)] - for t in threads: - t.start() - for t in threads: - t.join() - for i, err in enumerate(errors): - if err is not None: - raise err - return results - - -def _create_transceiver_in_thread(rank, mapping, cache_manager, dist_mock, config, results, errors): - """Thread target: create one KvCacheTransceiverV2.""" - try: - tc = KvCacheTransceiverV2( - mapping=mapping, - dist=dist_mock, - kv_cache_manager=cache_manager, - cache_transceiver_config=config, - ) - results[rank] = tc - except Exception as e: - errors[rank] = e - - -def create_instance_transceivers( - tp: int, pp: int, enable_dp: bool, cache_managers: List, config: CacheTransceiverConfig -) -> List[KvCacheTransceiverV2]: - """Create KvCacheTransceiverV2 for all ranks via threaded init.""" - world_size = tp * pp - shared = {"barrier": threading.Barrier(world_size), "lock": threading.Lock()} - results = [None] * world_size - errors = [None] * world_size - threads = [] - - for rank in range(world_size): - pp_rank = rank // tp - tp_rank = rank % tp - mapping = Mapping( - world_size=world_size, - rank=rank, - tp_size=tp, - pp_size=pp, - enable_attention_dp=enable_dp, - ) - dist_mock = ThreadSafeDistributed(rank, world_size, tp, pp, tp_rank, pp_rank, shared) - t = threading.Thread( - target=_create_transceiver_in_thread, - args=( - rank, - mapping, - cache_managers[rank], - dist_mock, - config, - results, - errors, - ), - ) - threads.append(t) - - for t in threads: - t.start() - for t in threads: - t.join() - - for rank, err in enumerate(errors): - if err is not None: - raise err - - return results - - # --------------------------------------------------------------------------- # DeepseekV4CacheManager creation helpers # --------------------------------------------------------------------------- @@ -302,7 +121,12 @@ def _create_managers_for_instance( return managers -def _init_pool_data(managers: List, tp: int, seed_base: int = 0, fill_random: bool = True): +def _init_pool_data( + managers: Sequence[DeepseekV4CacheManager], + tp: int, + seed_base: int = 0, + fill_random: bool = True, +) -> None: """Initialize pool data for all managers. Uses half-precision view of pool memory for initialization. @@ -407,32 +231,31 @@ def _expected_valid_blocks( def _split_blockwise_buffer( buffer: torch.Tensor, - index_head_dim: int = INDEX_HEAD_DIM, - quant_block_size: int = INDEXER_QUANT_BLOCK_SIZE, + data_size: int, + scale_size: int, + scale_dtype: torch.dtype, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Split a blockwise FP8 quantized buffer into value and scale buffers. + """Split a blockwise quantized buffer into value and scale buffers. Args: buffer: shape [num_blocks, tokens_per_block, bytes_per_token] Returns: - (values_buffer, scales_buffer) where values are uint8 and scales are float32 + (values_buffer, scales_buffer), preserving the manager's scale dtype. """ num_blocks, tokens_per_block, bytes_per_token = buffer.shape bytes_per_block = bytes_per_token * tokens_per_block # Value buffer - value_shape = (num_blocks, tokens_per_block, index_head_dim) - value_stride = (bytes_per_block, index_head_dim, 1) + value_shape = (num_blocks, tokens_per_block, data_size) + value_stride = (bytes_per_block, data_size, 1) value_buffer = buffer.as_strided(value_shape, value_stride, 0).view(torch.uint8) # Scale buffer - scale_dim = index_head_dim // quant_block_size - scale_bytes = scale_dim * 4 # float32 = 4 bytes - scale_shape = (num_blocks, tokens_per_block, scale_bytes) - scale_stride = (bytes_per_block, scale_bytes, 1) - scale_offset = index_head_dim * tokens_per_block - scale_buffer = buffer.as_strided(scale_shape, scale_stride, scale_offset).view(torch.float32) + scale_shape = (num_blocks, tokens_per_block, scale_size) + scale_stride = (bytes_per_block, scale_size, 1) + scale_offset = data_size * tokens_per_block + scale_buffer = buffer.as_strided(scale_shape, scale_stride, scale_offset).view(scale_dtype) return value_buffer, scale_buffer @@ -458,7 +281,13 @@ def _read_cache_data( return torch.tensor([]), None if attn_type == DeepseekV4AttentionType.INDEXER_COMPRESS: - values_buf, scales_buf = _split_blockwise_buffer(buffer) + scale_dtype = torch.float32 if mgr._indexer_k_dtype == "fp8" else torch.uint8 + values_buf, scales_buf = _split_blockwise_buffer( + buffer, + data_size=mgr._indexer_data_size, + scale_size=mgr._indexer_scale_size, + scale_dtype=scale_dtype, + ) return values_buf[indices], scales_buf[indices] return buffer[indices], None @@ -466,7 +295,7 @@ def _read_cache_data( def _find_ctx_rank_for_layer( layer_idx: int, - ctx_managers: List[DeepseekV4CacheManager], + ctx_managers: Sequence[DeepseekV4CacheManager], ctx_tp: int, ctx_enable_dp: bool, req_idx: int, @@ -491,8 +320,8 @@ def _find_ctx_rank_for_layer( def verify_all_requests( request_lengths: List[int], compress_ratios: List[int], - ctx_managers: List[DeepseekV4CacheManager], - gen_managers: List[DeepseekV4CacheManager], + ctx_managers: Sequence[DeepseekV4CacheManager], + gen_managers: Sequence[DeepseekV4CacheManager], ctx_tp: int, ctx_pp: int, gen_tp: int, @@ -581,12 +410,6 @@ def verify_all_requests( # --------------------------------------------------------------------------- # Main test function # --------------------------------------------------------------------------- -def _get_ctx_info_endpoint(tc: KvCacheTransceiverV2) -> Optional[str]: - """Extract the context_info_endpoint from a transceiver's disaggregated params.""" - endpoints = tc.get_disaggregated_params().get("ctx_info_endpoint") or [] - return endpoints[0] if endpoints else None - - def run_deepseek_v4_transfer_test( ctx_tp: int, ctx_pp: int, @@ -596,202 +419,22 @@ def run_deepseek_v4_transfer_test( gen_enable_dp: bool, compress_ratios: List[int], update_before_transfer: bool = True, -): - """Run a full DeepSeek-V4 KV transfer test.""" - ctx_world = ctx_tp * ctx_pp - gen_world = gen_tp * gen_pp - - # Mix of block-aligned and non-aligned lengths for boundary testing. - # TOKENS_PER_BLOCK=128: 65=half+1, 256=2x exact, 129=1x+1, 383=3x-1 - request_lengths = [65, 256, 129, 383] - - # ===== 1. Create DeepseekV4CacheManagers ===== - ctx_managers = _create_managers_for_instance(ctx_tp, ctx_pp, ctx_enable_dp, compress_ratios) - gen_managers = _create_managers_for_instance(gen_tp, gen_pp, gen_enable_dp, compress_ratios) - - # ===== 2. Initialize data ===== - # ctx: random data, seed=pp_rank (same across TP, different across PP) - _init_pool_data(ctx_managers, ctx_tp, seed_base=1000, fill_random=True) - # gen: zeros - _init_pool_data(gen_managers, gen_tp, fill_random=False) - - # ===== 3. Create KvCacheTransceiverV2 instances (threaded init) ===== - config = CacheTransceiverConfig( - backend="NIXL", - transceiver_runtime="PYTHON", - max_tokens_in_buffer=512, +) -> None: + """Run the shared transfer harness with DeepSeek-V4 cache hooks.""" + run_kv_transfer_test( + ctx_tp=ctx_tp, + ctx_pp=ctx_pp, + gen_tp=gen_tp, + gen_pp=gen_pp, + ctx_enable_dp=ctx_enable_dp, + gen_enable_dp=gen_enable_dp, + update_before_transfer=update_before_transfer, + manager_factory=lambda tp, pp, enable_dp: _create_managers_for_instance( + tp, pp, enable_dp, compress_ratios + ), + init_fn=_init_pool_data, + verify_fn=functools.partial(verify_all_requests, compress_ratios=compress_ratios), ) - ctx_tcs = create_instance_transceivers(ctx_tp, ctx_pp, ctx_enable_dp, ctx_managers, config) - gen_tcs = create_instance_transceivers(gen_tp, gen_pp, gen_enable_dp, gen_managers, config) - - try: - ctx_info_endpoint = _get_ctx_info_endpoint(ctx_tcs[0]) - - # ===== 4. Create requests and determine handle map ===== - # handle_map: rank -> [(req_idx, ctx_request, gen_request)] - ctx_handle_map: Dict[int, List] = {r: [] for r in range(ctx_world)} - gen_handle_map: Dict[int, List] = {r: [] for r in range(gen_world)} - ctx_request_ids: List[int] = [] - gen_request_ids: List[int] = [] - - sampling_params = SamplingParams() - - for req_idx, req_len in enumerate(request_lengths): - unique_rid = uuid.uuid4().int & 0x7FFFFFFFFFFFFFFF - ctx_rid = req_idx * 2 - gen_rid = req_idx * 2 + 1 - ctx_request_ids.append(ctx_rid) - gen_request_ids.append(gen_rid) - - ctx_dp_rank = req_idx % ctx_tp if ctx_enable_dp else 0 - - ctx_request = LlmRequest( - request_id=ctx_rid, - max_new_tokens=1, - input_tokens=list(range(req_len)), - sampling_config=tensorrt_llm.bindings.SamplingConfig( - sampling_params._get_sampling_config() - ), - is_streaming=False, - llm_request_type=LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY, - ) - ctx_request.py_disaggregated_params = DisaggregatedParams(disagg_request_id=unique_rid) - - gen_request = LlmRequest( - request_id=gen_rid, - max_new_tokens=1, - input_tokens=list(range(req_len)), - sampling_config=tensorrt_llm.bindings.SamplingConfig( - sampling_params._get_sampling_config() - ), - is_streaming=False, - llm_request_type=LlmRequestType.LLMREQUEST_TYPE_GENERATION_ONLY, - ) - gen_request.py_disaggregated_params = DisaggregatedParams( - ctx_request_id=ctx_rid, - ctx_dp_rank=ctx_dp_rank, - ctx_info_endpoint=ctx_info_endpoint, - disagg_request_id=unique_rid, - ) - - for rank in range(ctx_world): - tp_rank = rank % ctx_tp - should_handle = (not ctx_enable_dp) or (req_idx % ctx_tp == tp_rank) - if should_handle: - ctx_handle_map[rank].append((req_idx, ctx_request)) - - for rank in range(gen_world): - tp_rank = rank % gen_tp - should_handle = (not gen_enable_dp) or (req_idx % gen_tp == tp_rank) - if should_handle: - gen_handle_map[rank].append((req_idx, gen_request)) - - # ===== 5. Allocate KV cache for all ranks ===== - # prepare_resources is a no-op for non-draft KVCacheManagerV2. - # All ranks must allocate BEFORE mutating shared request objects - # (add_new_token changes is_first_context_chunk). - # - # Gen ranks take the disagg-gen-init path: prepare_disagg_gen_init - # sizes the cache for the full prompt and pre-declares - # history_length=prompt_len, matching what the V2 scheduler's - # _try_schedule_disagg_gen_init does in production so the - # transceiver's TRANS_COMPLETE contract check is satisfied. - # Ctx ranks take the regular prefill path (prepare_context + - # resize_context). - gen_batches: Dict[int, ScheduledRequests] = {} - for rank in range(gen_world): - reqs = [req for _, req in gen_handle_map[rank]] - if reqs: - batch = ScheduledRequests() - batch.context_requests_last_chunk = reqs - for req in reqs: - gen_managers[rank].prepare_disagg_gen_init(req) - gen_batches[rank] = batch - - ctx_batches: Dict[int, ScheduledRequests] = {} - for rank in range(ctx_world): - reqs = [req for _, req in ctx_handle_map[rank]] - if reqs: - batch = ScheduledRequests() - batch.context_requests_last_chunk = reqs - for req in reqs: - ctx_managers[rank].prepare_context(req) - ctx_managers[rank].resize_context(req, req.context_chunk_size) - ctx_batches[rank] = batch - - # ===== 5.5. context_current_position + add_new_token ===== - # Set position on each unique request once (needed for transfer metadata). - seen: set = set() - for rank in range(ctx_world): - for _, req in ctx_handle_map[rank]: - if req.py_request_id not in seen: - req.context_current_position = req.prompt_len - req.add_new_token(req.prompt_len, 0) - seen.add(req.py_request_id) - - seen = set() - for rank in range(gen_world): - for _, req in gen_handle_map[rank]: - if req.py_request_id not in seen: - req.context_current_position = req.prompt_len - req.add_new_token(req.prompt_len, 0) - seen.add(req.py_request_id) - - # ===== 5.6. update_resources BEFORE transfer (mode: update_before) ===== - if update_before_transfer: - for rank, batch in ctx_batches.items(): - ctx_managers[rank].update_resources(batch) - for rank, batch in gen_batches.items(): - gen_managers[rank].update_resources(batch) - - # ===== 6. gen receive + ctx send ===== - for rank in range(gen_world): - for _, req in gen_handle_map[rank]: - gen_tcs[rank].request_and_receive_async(req) - for rank in range(ctx_world): - for _, req in ctx_handle_map[rank]: - ctx_tcs[rank].respond_and_send_async(req) - - # ===== 7. Wait for completion (threaded, dist calls inside) ===== - run_concurrent( - ctx_tcs, lambda tc: tc.check_context_transfer_status(None, mark_complete=True) - ) - run_concurrent(gen_tcs, lambda tc: tc.check_gen_transfer_status(None)) - - # ===== 7.5. update_resources AFTER transfer (mode: update_after) ===== - if not update_before_transfer: - for rank, batch in ctx_batches.items(): - ctx_managers[rank].update_resources(batch) - for rank, batch in gen_batches.items(): - gen_managers[rank].update_resources(batch) - - # ===== 8. Verify ===== - verify_all_requests( - request_lengths=request_lengths, - compress_ratios=compress_ratios, - ctx_managers=ctx_managers, - gen_managers=gen_managers, - ctx_tp=ctx_tp, - ctx_pp=ctx_pp, - gen_tp=gen_tp, - gen_pp=gen_pp, - ctx_enable_dp=ctx_enable_dp, - gen_enable_dp=gen_enable_dp, - ctx_request_ids=ctx_request_ids, - gen_request_ids=gen_request_ids, - ) - - finally: - for tc in ctx_tcs + gen_tcs: - try: - tc.shutdown() - except Exception: - pass - for mgr in ctx_managers + gen_managers: - try: - mgr.shutdown() - except Exception: - pass # --------------------------------------------------------------------------- @@ -862,21 +505,6 @@ def test_deepseek_v4_kv_transfer( print("PASSED") -# --------------------------------------------------------------------------- -# PP layer distribution helpers (mirrors C++ getLayerNumPPRank) -# --------------------------------------------------------------------------- -def _get_layers_per_pp(num_layers: int, pp_size: int) -> List[int]: - """Return a list of layer counts per PP rank. - - When num_layers is not evenly divisible by pp_size, the first - (num_layers % pp_size) ranks get one extra layer. - Matches Mapping.pp_layers / torch.tensor_split behaviour. - """ - base = num_layers // pp_size - extra = num_layers % pp_size - return [base + (1 if r < extra else 0) for r in range(pp_size)] - - # --------------------------------------------------------------------------- # Uneven PP layer test configurations # --------------------------------------------------------------------------- @@ -945,8 +573,8 @@ def test_deepseek_v4_kv_transfer_uneven_pp( f"ctx_tp={ctx_tp} ctx_pp={ctx_pp} gen_tp={gen_tp} gen_pp={gen_pp} " f"ctx_dp={ctx_enable_dp} gen_dp={gen_enable_dp} " f"num_layers={num_layers} compress_ratios={compress_ratios} " - f"layers_per_pp(ctx)={_get_layers_per_pp(num_layers, ctx_pp)} " - f"layers_per_pp(gen)={_get_layers_per_pp(num_layers, gen_pp)}" + f"layers_per_pp(ctx)={get_layers_per_pp(num_layers, ctx_pp)} " + f"layers_per_pp(gen)={get_layers_per_pp(num_layers, gen_pp)}" ) run_deepseek_v4_transfer_test( diff --git a/tests/unittest/disaggregated/test_disagg_openai_client.py b/tests/unittest/disaggregated/test_disagg_openai_client.py index edc09809732b..9b8144fd9a2e 100644 --- a/tests/unittest/disaggregated/test_disagg_openai_client.py +++ b/tests/unittest/disaggregated/test_disagg_openai_client.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. +# Copyright (c) 2025-2026, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,6 +26,8 @@ DisaggregatedParams, UsageInfo, ) +from tensorrt_llm.serve.perf_metrics import _PERF_METRICS_HEADER_BUDGET_BYTES, SSE_METRICS_EVENT +from tensorrt_llm.serve.responses_utils import ResponseHooks from tensorrt_llm.serve.router import Router @@ -116,6 +118,16 @@ def test_initialization(self, mock_router, mock_session): assert client._session == mock_session assert client._max_retries == 5 + @pytest.mark.asyncio + async def test_internal_client_accepts_perf_metrics_header_size(self, mock_router): + with ( + patch("tensorrt_llm.serve.openai_client.ClientMetricsCollector"), + patch("tensorrt_llm.serve.openai_client.aiohttp.ClientSession") as session, + ): + OpenAIHttpClient(router=mock_router, role=ServerRole.GENERATION) + + assert session.call_args.kwargs["max_field_size"] == _PERF_METRICS_HEADER_BUDGET_BYTES + @pytest.mark.asyncio async def test_non_streaming_completion_request( self, openai_client, completion_request, mock_session, mock_router @@ -189,6 +201,45 @@ async def mock_iter_any(): streaming_completion_request, mock_session, success=True ) + @pytest.mark.asyncio + async def test_malformed_streaming_metrics_do_not_fail_request( + self, openai_client, streaming_completion_request, mock_session, mock_router + ): + openai_client._request_perf_metrics = True + mock_http_response = AsyncMock() + mock_http_response.status = 200 + mock_http_response.headers = {"Content-Type": "text/event-stream"} + + response_data = b'data: "Hello"\n\ndata: [DONE]\n\n' + metrics_data = f"event: {SSE_METRICS_EVENT}\ndata: not-json\n\n".encode() + + async def mock_iter_any(): + yield b"" + yield response_data + yield metrics_data + + mock_http_response.content = AsyncMock() + mock_http_response.content.iter_any = mock_iter_any + mock_http_response.__aenter__ = AsyncMock(return_value=mock_http_response) + mock_http_response.__aexit__ = AsyncMock() + mock_session.post.return_value = mock_http_response + hooks = MagicMock(spec=ResponseHooks) + + response_generator = await openai_client.send_request( + streaming_completion_request, hooks=hooks + ) + chunks = [chunk async for chunk in response_generator] + + assert b"".join(chunks) == response_data + hooks.on_first_token.assert_called_once_with("localhost:8000", streaming_completion_request) + hooks.on_perf_metrics.assert_not_called() + hooks.on_resp_done.assert_called_once_with( + "localhost:8000", streaming_completion_request, None + ) + mock_router.finish_request.assert_called_once_with( + streaming_completion_request, mock_session, success=True + ) + @pytest.mark.asyncio async def test_request_with_custom_server( self, openai_client, completion_request, mock_session, mock_router @@ -376,7 +427,11 @@ def _make_client(self, session, **kwargs): async def test_retry_regenerates_disagg_id(self): session = AsyncMock(spec=aiohttp.ClientSession) ids = iter(range(1000, 2000)) - client = self._make_client(session, disagg_id_generator=lambda: next(ids)) + + async def next_id(): + return next(ids) + + client = self._make_client(session, disagg_id_generator=next_id) session.post.side_effect = [ aiohttp.ClientError("transient"), diff --git a/tests/unittest/disaggregated/test_disagg_utils.py b/tests/unittest/disaggregated/test_disagg_utils.py index a086108187ad..732c06103d6e 100644 --- a/tests/unittest/disaggregated/test_disagg_utils.py +++ b/tests/unittest/disaggregated/test_disagg_utils.py @@ -6,9 +6,10 @@ # isort: off from tensorrt_llm.llmapi.disagg_utils import ( - MIN_GLOBAL_ID, CtxGenServerConfig, DisaggServerConfig, extract_ctx_gen_cfgs, - extract_router_config, extract_disagg_cfg, get_global_disagg_request_id, - get_local_request_id, get_server_configs_dict, parse_disagg_config_file) + MIN_GLOBAL_ID, CtxGenServerConfig, DisaggServerConfig, + disagg_process_id_space, extract_ctx_gen_cfgs, extract_router_config, + extract_disagg_cfg, get_global_disagg_request_id, get_local_request_id, + get_server_configs_dict, parse_disagg_config_file, worker_local_process_id) # isort: on @@ -110,9 +111,42 @@ def test_parse_disagg_config_file(sample_yaml_file, sample_yaml_config): @pytest.mark.parametrize("sample_yaml_config", ["disagg_cluster", ""], indirect=True) def test_extract_disagg_cfg(sample_yaml_config): + sample_yaml_config.update({ + "gen_tokids_ctxbytes": + True, + "num_workers": + 4, + "disagg_coordinator_url": + "http://coordinator:7999", + }) config = extract_disagg_cfg(**sample_yaml_config) assert isinstance(config, DisaggServerConfig) verify_disagg_config(config, sample_yaml_config) + assert config.gen_tokids_ctxbytes is True + assert config.num_workers == 4 + assert config.disagg_coordinator_url == "http://coordinator:7999" + + +def test_extract_disagg_metrics_controls(): + yaml_config = get_yaml_config() + yaml_config["context_servers"]["return_perf_metrics"] = False + yaml_config["generation_servers"]["return_perf_metrics"] = False + config = extract_disagg_cfg( + **yaml_config, + return_perf_metrics=True, + perf_metrics_output_dir="/tmp/perf", + ) + + assert config.return_perf_metrics is True + assert config.perf_metrics_output_dir == "/tmp/perf" + assert all("perf_metrics_output_dir" not in server.other_args + for server in config.server_configs) + + +@pytest.mark.parametrize("node_id", [-1, 256]) +def test_extract_disagg_cfg_rejects_out_of_range_node_id(node_id): + with pytest.raises(ValueError, match="node_id must be in range"): + extract_disagg_cfg(node_id=node_id) def test_extract_ctx_gen_cfgs(): @@ -177,6 +211,17 @@ def test_extract_router_config_propagates_tokens_per_block(): }).args +def test_extract_router_config_propagates_kv_model_path() -> None: + cfg = { + "model": "/models/gpt-oss-checkpoint", + "router": { + "type": "kv_cache_aware" + }, + } + router_config = extract_router_config(cfg) + assert router_config.args["model_path"] == "/models/gpt-oss-checkpoint" + + def test_get_server_configs_dict(): server_configs = [ CtxGenServerConfig(type="ctx", @@ -200,31 +245,59 @@ def test_get_server_configs_dict(): ids=["multithread", "singlethread"]) def test_get_global_disagg_request_id(multithread): iter = 10000 - node_ids = list(range(10)) - thread_num = len(node_ids) - - def get_ids(node_ids): - all_node_ids = [[] for _ in range(len(node_ids))] + # (node_id, process_id) pairs — the pair uniquely identifies a fleet worker. + # Mix of distinct nodes and distinct processes on the same node. + worker_ids = [(0, 0), (0, 1), (1, 0), (1, 1), (2, 0), (3, 5), (7, 2), + (10, 0), (10, 3), (255, 63)] + thread_num = len(worker_ids) + + def get_ids(worker_ids): + all_ids = [[] for _ in range(len(worker_ids))] for i in range(iter): if i % (4000 // thread_num) == 0: time.sleep(0.001) - for i, node_id in enumerate(node_ids): - all_node_ids[i].append(get_global_disagg_request_id(node_id)) - return all_node_ids + for j, (node_id, process_id) in enumerate(worker_ids): + all_ids[j].append( + get_global_disagg_request_id(node_id, process_id)) + return all_ids if multithread: - with ThreadPoolExecutor(max_workers=len(node_ids)) as executor: - all_node_ids = [ - ids[0] for ids in executor.map(get_ids, [[i] for i in node_ids]) + with ThreadPoolExecutor(max_workers=len(worker_ids)) as executor: + all_worker_ids = [ + ids[0] + for ids in executor.map(get_ids, [[w] for w in worker_ids]) ] else: - all_node_ids = get_ids(node_ids) + all_worker_ids = get_ids(worker_ids) - all_ids = set(i for ids in all_node_ids for i in ids) - assert len(all_ids) == iter * len(node_ids) + all_ids = set(i for ids in all_worker_ids for i in ids) + # Each (node_id, process_id) worker's ids must be globally unique across all. + assert len(all_ids) == iter * len(worker_ids) assert all(id >= MIN_GLOBAL_ID and id < ((1 << 63) - 1) for id in all_ids) +def test_get_global_disagg_request_id_range_validation(): + # node_id: 8 bits [0,256); process_id: 6 bits [0,64). + with pytest.raises(ValueError): + get_global_disagg_request_id(256, 0) + with pytest.raises(ValueError): + get_global_disagg_request_id(0, 64) + # valid extremes + assert get_global_disagg_request_id(255, 63) >= MIN_GLOBAL_ID + assert get_global_disagg_request_id(0, 0) >= MIN_GLOBAL_ID + + +def test_worker_local_process_id_range_validation(monkeypatch): + process_id_space = disagg_process_id_space() + monkeypatch.setenv("TRTLLM_DISAGG_WORKER_PROCESS_ID", + str(process_id_space - 1)) + assert worker_local_process_id() == process_id_space - 1 + + monkeypatch.setenv("TRTLLM_DISAGG_WORKER_PROCESS_ID", str(process_id_space)) + with pytest.raises(ValueError): + worker_local_process_id() + + def test_get_local_request_id(): last_id = MIN_GLOBAL_ID - 100 ids = set() @@ -236,5 +309,4 @@ def test_get_local_request_id(): assert len(ids) == 1000 assert min(ids) == 0 assert max(ids) == MIN_GLOBAL_ID - 1 - assert max(ids) - min(ids) > ( - 1 << 40) # ensure there is enough space for local ids + assert max(ids) - min(ids) == MIN_GLOBAL_ID - 1 diff --git a/tests/unittest/disaggregated/test_extractor.py b/tests/unittest/disaggregated/test_extractor.py index 6fc2f68fef68..69fb84b26887 100644 --- a/tests/unittest/disaggregated/test_extractor.py +++ b/tests/unittest/disaggregated/test_extractor.py @@ -1,20 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import pytest +import torch from tensorrt_llm._torch.disaggregation.base.region import MemRegionGroup, SpecRegion from tensorrt_llm._torch.disaggregation.resource.kv_extractor import ( KVRegionExtractorV1, + _build_v2_mamba_state_pool, build_page_table, + build_page_table_from_manager, ) from tensorrt_llm._torch.disaggregation.resource.page import MapperKind from tensorrt_llm._torch.disaggregation.resource.utils import ( get_global_layer_ids, + get_layer_byte_ranges, get_layer_to_layer_group, get_num_layer_groups, get_num_layers, get_physical_pool, + get_slot_address, get_unique_layers, ) +from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import Role from tensorrt_llm._torch.pyexecutor.resource_manager import ( CacheTypeCpp, DataType, @@ -187,6 +208,142 @@ def test_build_page_table(): manager.shutdown() +def _make_v1_dsa_manager(pp_size: int = 1, pp_rank: int = 0) -> KVCacheManager: + """V1 KVCacheManager with the DSA indexer K cache enabled (MLA-style).""" + return KVCacheManager( + kv_cache_config=KvCacheConfig( + max_tokens=512, + enable_block_reuse=False, + event_buffer_max_size=0, + ), + kv_cache_type=CacheTypeCpp.SELFKONLY, + num_layers=4, + num_kv_heads=1, + head_dim=64, + tokens_per_block=32, + max_seq_len=256, + max_batch_size=2, + mapping=Mapping(world_size=pp_size, rank=pp_rank, tp_size=1, pp_size=pp_size), + dtype=DataType.HALF, + enable_indexer_k_cache=True, + indexer_k_cache_quant_block_size=128, + indexer_k_cache_index_head_dim=128, + ) + + +def _byte_view(ptr: int, nbytes: int): + from tensorrt_llm._utils import TensorWrapper, convert_to_torch_tensor + + return convert_to_torch_tensor(TensorWrapper(int(ptr), DataType.INT8, [int(nbytes)])) + + +@pytest.mark.cuda +def test_v1_dsa_indexer_page_table_is_replicated_with_per_layer_entries(): + manager = _make_v1_dsa_manager() + try: + page_table = build_page_table(manager) + lg = page_table.layer_groups[0] + assert len(lg.pool_views) == 2 + + kv_view, idx_view = lg.pool_views + assert kv_view.mapper_kind == MapperKind.INDEXED + assert idx_view.mapper_kind == MapperKind.REPLICATED + assert idx_view.pool_role == frozenset({"indexer_k"}) + + # One synthesized entry per LG layer, equal-sized, contiguous from 0. + assert get_unique_layers(idx_view) == get_unique_layers(kv_view) + idx_pool = get_physical_pool(page_table, 0, idx_view.pool_idx) + sizes = {int(entry["size"]) for entry in idx_view.buffer_entries} + assert len(sizes) == 1 + per_layer = sizes.pop() + assert per_layer * len(idx_view.buffer_entries) == idx_pool.slot_bytes + offsets = sorted(int(entry["offset"]) for entry in idx_view.buffer_entries) + assert offsets == [i * per_layer for i in range(len(offsets))] + finally: + manager.shutdown() + + +@pytest.mark.cuda +def test_v1_dsa_indexer_replicated_transfer_across_pp(): + """PP1 ctx sends the DSA indexer K cache into two PP2 gen ranks. + + Exercises the full python path on real V1 managers: page-table build, + role-set matching, ReplicatedMapper layer-strided offsets, and a + byte-level copy that must land each gen rank's layer subset at the + right offsets. + """ + import torch + + from tensorrt_llm._torch.disaggregation.native.mixers.attention.peer import ReplicatedMapper + from tensorrt_llm._torch.disaggregation.native.peer import PeerRegistrar + from tensorrt_llm._torch.disaggregation.native.rank_info import RankInfo + + ctx = _make_v1_dsa_manager() + gens = [_make_v1_dsa_manager(pp_size=2, pp_rank=r) for r in range(2)] + try: + ctx_extractor = KVRegionExtractorV1(ctx) + ctx_ri = RankInfo.from_kv_cache_manager("ctx", ctx, device_id=0) + registrar = PeerRegistrar(ctx_ri, ctx_extractor) + + # Fill ctx indexer pool with position-dependent bytes; zero gen pools. + ctx_pool_tensor = ctx.impl.get_indexer_k_cache_pool() + flat = ctx_pool_tensor.view(torch.uint8).flatten() + flat.copy_(torch.arange(flat.numel(), dtype=torch.int64, device=flat.device) % 251) + for gen in gens: + gen.impl.get_indexer_k_cache_pool().view(torch.uint8).zero_() + + block_ids = np.array([0, 2, 3], dtype=np.int64) + ctx_pt = ctx_extractor.page_table + idx_pool = get_physical_pool(ctx_pt, 0, 1) + layers_per_gen = 2 + per_layer = idx_pool.slot_bytes // 4 + + for gen_pp_rank, gen in enumerate(gens): + gen_ri = RankInfo.from_kv_cache_manager("gen", gen, device_id=0) + registrar.register("gen", gen_ri.instance_rank, gen_ri) + mapping = registrar.get_pool_mapping(gen_ri) + # KV pool and indexer pool each match 1:1. + assert mapping == {(0, 0): (0, 0), (0, 1): (0, 1)} + + mapper = registrar.get_kv_map(gen_ri, (0, 1), (0, 1)) + assert isinstance(mapper, ReplicatedMapper) + + gen_extractor = registrar.peer_extractor("gen", gen_ri.instance_rank) + pair = mapper.map( + ctx_extractor.extract(block_ids, layer_group_id=0, pool_idx=1), + gen_extractor.extract(block_ids, layer_group_id=0, pool_idx=1), + ) + # This gen rank holds 2 of the 4 layers; the fragment is the + # contiguous 2-layer range at this PP stage's offset within + # the ctx slot. + assert pair.src.memory.bytes_per_region == layers_per_gen * per_layer + expected_src_off = gen_pp_rank * layers_per_gen * per_layer + base_ptrs = idx_pool.base_address + block_ids * idx_pool.slot_bytes + np.testing.assert_array_equal(pair.src.memory.ptrs, base_ptrs + expected_src_off) + + # Emulate the transfer with raw byte copies, then verify content. + for src_ptr, dst_ptr in zip(pair.src.memory.ptrs, pair.dst.memory.ptrs): + _byte_view(dst_ptr, pair.dst.memory.bytes_per_region).copy_( + _byte_view(src_ptr, pair.src.memory.bytes_per_region) + ) + + gen_pool_tensor = gen.impl.get_indexer_k_cache_pool() + gen_bytes = gen_pool_tensor.view(torch.uint8).reshape(gen_pool_tensor.shape[0], -1) + ctx_bytes = ctx_pool_tensor.view(torch.uint8).reshape(ctx_pool_tensor.shape[0], -1) + src_lo = gen_pp_rank * layers_per_gen * per_layer + src_hi = src_lo + layers_per_gen * per_layer + torch.testing.assert_close( + gen_bytes[block_ids], + ctx_bytes[block_ids, src_lo:src_hi], + rtol=0, + atol=0, + ) + finally: + ctx.shutdown() + for gen in gens: + gen.shutdown() + + def test_layer_group_meta_serialization(): import numpy as np @@ -232,11 +389,215 @@ def test_layer_group_meta_serialization(): assert len(restored_lg.pool_views[0].buffer_entries) == 2 +def _make_fake_v2_manager(attrs, role_mapper_kinds, *, num_pools=1, slot_bytes_list=(640,)): + """Build a minimal duck-typed KVCacheManagerV2 for _build_page_table_v2. + + ``attrs`` keeps the storage-layer shape ``{(layer_id, role): + (life_cycle_id, pool_index, offset, size)}``; the fake synthesizes the + public ``pool_group_descs`` view from it. Buffer order within a + coalesced buffer follows ascending offset, mirroring how the real + storage config assigns offsets in ``buffer_ids`` order. + """ + from types import SimpleNamespace + + base = 0x1000 + + per_pool: dict[int, list] = {} + for (layer_id, role), attr in attrs.items(): + per_pool.setdefault(int(attr.pool_index), []).append( + (int(attr.offset), layer_id, role, int(attr.size)) + ) + + coalesced_buffers = [] + for pool_idx in range(num_pools): + buffers = sorted(per_pool.get(pool_idx, [])) + sizes = {size for _, _, _, size in buffers} + assert len(sizes) == 1, "buffers in a coalesced buffer are uniform-size by construction" + coalesced_buffers.append( + SimpleNamespace( + single_buffer_size=sizes.pop(), + buffer_ids=[ + SimpleNamespace(layer_id=layer_id, role=role) + for _, layer_id, role, _ in buffers + ], + ) + ) + + pg_desc = SimpleNamespace( + pool_group_index=0, + num_slots=4, + pools=[ + SimpleNamespace(pool_index=pi, base_address=base, slot_bytes=sb) + for pi, sb in enumerate(slot_bytes_list) + ], + slot_desc=SimpleNamespace( + variants=[SimpleNamespace(layer_group_id=0, coalesced_buffers=coalesced_buffers)] + ), + ) + + impl = SimpleNamespace( + layer_grouping=((0, 1),), + init_config=SimpleNamespace( + tokens_per_block=16, + layers=[SimpleNamespace(window_size=None), SimpleNamespace(window_size=None)], + ), + pool_group_descs=[pg_desc], + ) + return SimpleNamespace( + impl=impl, + pp_layers=[0, 1], + num_kv_heads_per_layer=[1, 1], + get_disagg_role_mapper_kinds=lambda: role_mapper_kinds, + ) + + +def test_v2_builder_stamps_homogeneous_pool_kinds(): + """Split KV / INDEX_KEY pools get NHD and REPLICATED views respectively.""" + from types import SimpleNamespace + + attrs = { + (0, Role.KEY): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=0, size=128), + (0, Role.VALUE): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=128, size=128), + (1, Role.KEY): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=256, size=128), + (1, Role.VALUE): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=384, size=128), + (1, Role.INDEX_KEY): SimpleNamespace(life_cycle_id=0, pool_index=1, offset=0, size=128), + } + manager = _make_fake_v2_manager( + attrs, + {Role.ALL: MapperKind.NHD, Role.INDEX_KEY: MapperKind.REPLICATED}, + num_pools=2, + slot_bytes_list=(512, 128), + ) + + views = build_page_table_from_manager(manager).layer_groups[0].pool_views + assert len(views) == 2 + assert views[0].pool_role == frozenset({"key", "value"}) + assert views[0].mapper_kind == MapperKind.NHD + assert get_unique_layers(views[0]) == {0, 1} + assert views[1].pool_role == frozenset({"index_key"}) + assert views[1].mapper_kind == MapperKind.REPLICATED + assert get_unique_layers(views[1]) == {1} + + +def test_v2_builder_splits_mixed_kind_pool_into_per_class_views(): + """A pool coalescing several role classes yields one view per class. + + Miniature of the MiniMax M3 layout at TP degrees where + K == V == INDEX_KEY bytes per block: V2 storage coalesces all three + into one pool and the slot interleaves the sparse layer's INDEX_KEY + between the layers' K/V regions. The builder must split the pool into + a NHD K/V view (non-uniform layer offsets, uniform per-layer size) + and a REPLICATED INDEX_KEY view covering only the sparse layer. + """ + from types import SimpleNamespace + + attrs = { + (0, Role.KEY): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=0, size=128), + (0, Role.VALUE): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=128, size=128), + (0, Role.INDEX_KEY): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=256, size=128), + (1, Role.KEY): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=384, size=128), + (1, Role.VALUE): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=512, size=128), + } + manager = _make_fake_v2_manager( + attrs, + {Role.ALL: MapperKind.NHD, Role.INDEX_KEY: MapperKind.REPLICATED}, + ) + + views = build_page_table_from_manager(manager).layer_groups[0].pool_views + assert len(views) == 2 + kv_view, idx_view = views + # Both views address the same physical pool. + assert kv_view.pool_idx == idx_view.pool_idx == 0 + + assert kv_view.pool_role == frozenset({"key", "value"}) + assert kv_view.mapper_kind == MapperKind.NHD + assert get_unique_layers(kv_view) == {0, 1} + assert kv_view.bytes_per_layer == 256 + kv_starts, kv_bytes_per_layer = get_layer_byte_ranges(kv_view) + # Layer stride is non-uniform: INDEX_KEY interleaves after layer 0. + assert kv_starts == {0: 0, 1: 384} + assert kv_bytes_per_layer == 256 + + assert idx_view.pool_role == frozenset({"index_key"}) + assert idx_view.mapper_kind == MapperKind.REPLICATED + assert get_unique_layers(idx_view) == {0} + assert idx_view.bytes_per_layer == 128 + idx_starts, _ = get_layer_byte_ranges(idx_view) + assert idx_starts == {0: 256} + + +def test_v2_builder_validates_role_mapper_declaration(): + from types import SimpleNamespace + + attrs = { + (0, Role.KEY): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=0, size=128), + (0, Role.VALUE): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=128, size=128), + (1, Role.KEY): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=256, size=128), + (1, Role.VALUE): SimpleNamespace(life_cycle_id=0, pool_index=0, offset=384, size=128), + } + + # Default INDEXED declaration keeps the whole-slot HND view. + views = ( + build_page_table_from_manager(_make_fake_v2_manager(attrs, {Role.ALL: MapperKind.INDEXED})) + .layer_groups[0] + .pool_views + ) + assert len(views) == 1 + assert views[0].mapper_kind == MapperKind.INDEXED + + # V2 managers must expose the capability method explicitly. + missing_capability = _make_fake_v2_manager(attrs, {Role.ALL: MapperKind.INDEXED}) + del missing_capability.get_disagg_role_mapper_kinds + with pytest.raises(AttributeError, match="get_disagg_role_mapper_kinds"): + build_page_table_from_manager(missing_capability) + + with pytest.raises(ValueError, match="must define Role.ALL"): + build_page_table_from_manager( + _make_fake_v2_manager(attrs, {Role.INDEX_KEY: MapperKind.REPLICATED}) + ) + + with pytest.raises(ValueError, match="Invalid disaggregation mapper kind 'HND'"): + build_page_table_from_manager(_make_fake_v2_manager(attrs, {Role.ALL: "HND"})) + + with pytest.raises(ValueError, match="INDEXED is only valid as the Role.ALL mapping"): + build_page_table_from_manager( + _make_fake_v2_manager(attrs, {Role.ALL: MapperKind.NHD, Role.KEY: MapperKind.INDEXED}) + ) + + # INDEXED as the Role.ALL fallback coexists with side-cache roles that + # declare their own kind (the base-manager default for INDEX_KEY); with + # no INDEX_KEY buffers registered the declaration is inert. + views = ( + build_page_table_from_manager( + _make_fake_v2_manager( + attrs, + {Role.ALL: MapperKind.INDEXED, Role.INDEX_KEY: MapperKind.REPLICATED}, + ) + ) + .layer_groups[0] + .pool_views + ) + assert len(views) == 1 + assert views[0].mapper_kind == MapperKind.INDEXED + + def test_mamba_layer_group_serialization(): from tensorrt_llm._torch.disaggregation.resource.page import MambaLayerGroup, PhysicalPool - conv_pool = PhysicalPool(base_address=1000, slot_bytes=128, num_slots=10) - ssm_pool = PhysicalPool(base_address=8000, slot_bytes=256, num_slots=8) + conv_pool = PhysicalPool( + base_address=1000, + slot_bytes=128, + num_slots=10, + slot_stride_bytes=512, + layer_stride_bytes=256, + ) + ssm_pool = PhysicalPool( + base_address=8000, + slot_bytes=256, + num_slots=8, + slot_stride_bytes=1024, + layer_stride_bytes=512, + ) mlg = MambaLayerGroup( pool_group_idx=1, mamba_layer_offsets={10: 0, 11: 1, 12: 2}, @@ -256,14 +617,128 @@ def test_mamba_layer_group_serialization(): assert isinstance(restored, MambaLayerGroup) assert restored.mamba_layer_offsets == {10: 0, 11: 1, 12: 2} assert restored.conv_states.base_address == 1000 - assert restored.conv_states.slot_bytes == 128 - assert restored.conv_states.num_slots == 10 + assert restored.conv_states.slot_stride_bytes == 512 + assert get_slot_address(restored.conv_states, 3) == 1000 + 3 * 512 assert restored.ssm_states.base_address == 8000 - assert restored.ssm_states.slot_bytes == 256 - assert restored.ssm_states.num_slots == 8 + assert restored.ssm_states.slot_stride_bytes == 1024 assert restored.conv_section_bytes == [512, 256, 256] assert restored.ssm_bytes_per_head == 128 + legacy_pool = PhysicalPool.from_dict({"base_address": 1000, "slot_bytes": 128, "num_slots": 10}) + assert legacy_pool.slot_stride_bytes == legacy_pool.slot_bytes + assert legacy_pool.layer_stride_bytes == legacy_pool.num_slots * legacy_pool.slot_bytes + + +def test_v2_mamba_state_pool_uses_affine_layer_and_slot_strides(): + num_slots = 3 + state_bytes = 64 + storage = torch.empty((num_slots, 4, state_bytes), dtype=torch.uint8) + states = [storage[:, 0, :], storage[:, 2, :]] + + pool = _build_v2_mamba_state_pool(states) + + assert pool.base_address == states[0].data_ptr() + assert pool.slot_bytes == state_bytes + assert pool.num_slots == num_slots + assert pool.slot_stride_bytes == 4 * state_bytes + assert pool.layer_stride_bytes == 2 * state_bytes + + +def test_v2_mamba_single_layer_pool_preserves_shared_role_footprint(): + state_bytes = 64 + storage = torch.empty((3, 2, state_bytes), dtype=torch.uint8) + + pool = _build_v2_mamba_state_pool([storage[:, 1, :]]) + + assert pool.slot_bytes == state_bytes + assert pool.slot_stride_bytes == 2 * state_bytes + assert pool.layer_stride_bytes == 2 * state_bytes + + +def test_v2_mamba_state_pool_rejects_non_affine_layer_offsets(): + storage = torch.empty((3, 6, 64), dtype=torch.uint8) + states = [storage[:, 0, :], storage[:, 2, :], storage[:, 5, :]] + + with pytest.raises(ValueError, match="uniform layer stride"): + _build_v2_mamba_state_pool(states) + + +def test_v2_mamba_registration_uses_coalesced_physical_pool(): + from tensorrt_llm._torch.disaggregation.resource.page import ( + KVCachePageTable, + MambaLayerGroup, + PhysicalPool, + PhysicalPoolGroup, + ) + from tensorrt_llm._torch.disaggregation.resource.utils import get_unique_pool_memory_descs + + state_bytes = 64 + num_layers = 2 + num_slots = 8 + # Equal-sized SSM and convolution states share one interleaved V2 pool. + physical_slot_bytes = state_bytes * num_layers * 2 + physical_pool = PhysicalPool( + base_address=1000, + slot_bytes=physical_slot_bytes, + num_slots=num_slots, + ) + mamba_group = MambaLayerGroup( + pool_group_idx=0, + mamba_layer_offsets={1: 0, 2: 1}, + conv_states=PhysicalPool( + base_address=1000 + state_bytes, + slot_bytes=state_bytes, + num_slots=num_slots, + slot_stride_bytes=physical_slot_bytes, + layer_stride_bytes=2 * state_bytes, + ), + ssm_states=PhysicalPool( + base_address=1000, + slot_bytes=state_bytes, + num_slots=num_slots, + slot_stride_bytes=physical_slot_bytes, + layer_stride_bytes=2 * state_bytes, + ), + ) + page_table = KVCachePageTable( + tokens_per_block=16, + layer_groups=[mamba_group], + pool_groups=[PhysicalPoolGroup(pools=[physical_pool])], + ) + + assert get_unique_pool_memory_descs(page_table, device_id=3) == [ + (1000, physical_slot_bytes * num_slots, 3, "kv_cache_memory_pool0") + ] + + +def test_legacy_mamba_registration_uses_layer_major_pools(): + from tensorrt_llm._torch.disaggregation.resource.page import ( + KVCachePageTable, + MambaLayerGroup, + PhysicalPool, + ) + from tensorrt_llm._torch.disaggregation.resource.utils import get_unique_pool_memory_descs + + num_layers = 3 + conv_pool = PhysicalPool(base_address=1000, slot_bytes=128, num_slots=10) + ssm_pool = PhysicalPool(base_address=8000, slot_bytes=256, num_slots=8) + mamba_group = MambaLayerGroup( + pool_group_idx=0, + mamba_layer_offsets={10: 0, 11: 1, 12: 2}, + conv_states=conv_pool, + ssm_states=ssm_pool, + ) + page_table = KVCachePageTable( + tokens_per_block=16, + layer_groups=[mamba_group], + pool_groups=[], + ) + + assert get_unique_pool_memory_descs(page_table, device_id=3) == [ + (1000, num_layers * conv_pool.num_slots * conv_pool.slot_bytes, 3, "kv_cache_memory_pool0"), + (8000, num_layers * ssm_pool.num_slots * ssm_pool.slot_bytes, 3, "kv_cache_memory_pool1"), + ] + def test_mixed_page_table_serialization(): import numpy as np diff --git a/tests/unittest/disaggregated/test_kv_transfer.py b/tests/unittest/disaggregated/test_kv_transfer.py index 76a0d8db25b3..c2cfc1382e37 100644 --- a/tests/unittest/disaggregated/test_kv_transfer.py +++ b/tests/unittest/disaggregated/test_kv_transfer.py @@ -5,8 +5,19 @@ import time import uuid -# Exclude IB (no fabric) and gdr_copy (UCX rcache SIGABRT at teardown). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX config regardless of what the cluster/CI injects +# (the CI agent bootstrap exports UCX_TLS=tcp,cuda_copy,cuda_ipc before pytest +# starts, which a setdefault would leave in place): exclude IB (no fabric +# assumed) and gdr_copy (UCX rcache SIGABRT at teardown). +os.environ["UCX_TLS"] = "^ib,gdr_copy" +# Each NIXL agent spawns TRTLLM_NIXL_NUM_THREADS (default 8) busy-polling +# progress threads, and a single case builds up to 8 TransferWorkers (one per +# rank). On CI nodes shared with other single-GPU jobs the resulting CPU +# oversubscription inflates agent construction from ~3s to ~30s each, blowing +# the 120s per-test timeout intermittently (https://nvbugs/6426834). One +# progress thread is enough here: these tests verify transfer logic, not +# transfer-engine threading. +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" from dataclasses import dataclass from typing import List, Optional diff --git a/tests/unittest/disaggregated/test_kv_transfer_mp.py b/tests/unittest/disaggregated/test_kv_transfer_mp.py index 76e5f4aa7573..a52d95bde7a4 100644 --- a/tests/unittest/disaggregated/test_kv_transfer_mp.py +++ b/tests/unittest/disaggregated/test_kv_transfer_mp.py @@ -8,7 +8,13 @@ import torch.multiprocessing as mp # Exclude IB (no fabric) and gdr_copy (UCX rcache SIGABRT at teardown). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX config regardless of what the cluster/CI injects; +# see test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +# Limit NIXL busy-polling progress threads; see test_kv_transfer.py for the +# full rationale (intermittent 120s timeouts on shared CI nodes, +# https://nvbugs/6426834). +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" import tensorrt_llm import tensorrt_llm.bindings diff --git a/tests/unittest/disaggregated/test_mamba_transfer.py b/tests/unittest/disaggregated/test_mamba_transfer.py index 0cf0b3209899..dd701a0bc5ba 100644 --- a/tests/unittest/disaggregated/test_mamba_transfer.py +++ b/tests/unittest/disaggregated/test_mamba_transfer.py @@ -12,24 +12,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import threading import uuid from typing import Dict, List +import numpy as np import pytest import torch +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" + import tensorrt_llm import tensorrt_llm.bindings import tensorrt_llm.tensorrt_llm_transfer_agent_binding # noqa: F401 from tensorrt_llm import DisaggregatedParams, Mapping, SamplingParams +from tensorrt_llm._torch.disaggregation.native.mixers.ssm import peer +from tensorrt_llm._torch.disaggregation.resource import page from tensorrt_llm._torch.disaggregation.transceiver import KvCacheTransceiverV2 from tensorrt_llm._torch.pyexecutor.llm_request import ( ATTENTION_DP_DUMMY_REQUEST_ID, LlmRequest, LlmRequestType, ) -from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import MixedMambaHybridCacheManager +from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import ( + MambaHybridCacheManagerV2, + MixedMambaHybridCacheManager, +) from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.internal.batch_manager import CacheType as CacheTypeCpp @@ -183,8 +195,14 @@ def _init(rank): return results -def _create_managers(tp, max_batch_size=MAX_BATCH_SIZE, enable_attention_dp=False): - """Create MixedMambaHybridCacheManagers for all TP ranks (PP=1). +def _create_managers( + tp, + max_batch_size=MAX_BATCH_SIZE, + enable_attention_dp=False, + use_v2=False, + conv_state_layout="x_b_c", +): + """Create Mamba hybrid cache managers for all TP ranks (PP=1). Layer 0 is a dummy attention layer required by page table infrastructure. Layers 1..NUM_MAMBA_LAYERS are mamba layers under test. @@ -194,7 +212,16 @@ def _create_managers(tp, max_batch_size=MAX_BATCH_SIZE, enable_attention_dp=Fals mapping = Mapping( world_size=tp, rank=rank, tp_size=tp, pp_size=1, enable_attention_dp=enable_attention_dp ) - mgr = MixedMambaHybridCacheManager( + manager_cls = MambaHybridCacheManagerV2 if use_v2 else MixedMambaHybridCacheManager + manager_kwargs = ( + { + "is_disagg": True, + "conv_state_layout": conv_state_layout, + } + if use_v2 + else {} + ) + mgr = manager_cls( mamba_d_state=MAMBA_D_STATE, mamba_d_conv=MAMBA_D_CONV, mamba_num_heads=MAMBA_NUM_HEADS, @@ -220,19 +247,83 @@ def _create_managers(tp, max_batch_size=MAX_BATCH_SIZE, enable_attention_dp=Fals max_batch_size=max_batch_size, mapping=mapping, dtype=DataType.FLOAT, + **manager_kwargs, ) managers.append(mgr) return managers +def _mamba_layer_ids(manager): + if isinstance(manager, MambaHybridCacheManagerV2): + return manager.mamba_layer_offsets + return manager._impl.mamba_layer_offsets + + +def _mamba_state_slot(manager, request_id): + if isinstance(manager, MambaHybridCacheManagerV2): + return manager._request_id_to_state_index[request_id] + return manager.mamba_cache_index[request_id] + + +def _zero_mamba_states(manager): + for layer_idx in _mamba_layer_ids(manager): + manager.get_conv_states(layer_idx).zero_() + manager.get_ssm_states(layer_idx).zero_() + + +def test_mamba_policy_layer_major_v1_ptrs(): + pool = page.PhysicalPool(base_address=100, slot_bytes=10, num_slots=8) + + ptrs = peer.MambaPolicy._build_layer_ptrs( + pool=pool, + layer_offsets={1: 0, 2: 1}, + overlapping_layers=[1, 2], + slot=3, + ) + + np.testing.assert_array_equal(ptrs, [130, 210]) + + +def test_mamba_policy_slot_major_interleaved_role_ptrs(): + """Layer/role offsets remain inside each coalesced V2 physical slot.""" + state_bytes = 64 + physical_slot_bytes = 4 * state_bytes + conv_pool = page.PhysicalPool( + base_address=1000 + state_bytes, + slot_bytes=state_bytes, + num_slots=8, + slot_stride_bytes=physical_slot_bytes, + layer_stride_bytes=2 * state_bytes, + ) + + ptrs = peer.MambaPolicy._build_layer_ptrs( + pool=conv_pool, + layer_offsets={1: 0, 2: 1}, + overlapping_layers=[1, 2], + slot=3, + ) + + np.testing.assert_array_equal( + ptrs, + [ + 1000 + state_bytes + 3 * physical_slot_bytes, + 1000 + 3 * state_bytes + 3 * physical_slot_bytes, + ], + ) + + # --------------------------------------------------------------------------- # Ground truth: generate, shard, write, compute expected, read actual # --------------------------------------------------------------------------- -def _full_conv_section_dims() -> List[int]: - """Full (unsharded) first-dim sizes: [x(d_inner) | B(ng*ds) | C(ng*ds)].""" +def _full_conv_section_dims(conv_state_layout="x_b_c") -> List[int]: + """Full first-dimension sizes in the model's convolution-state order.""" d_inner = MAMBA_HEAD_DIM * MAMBA_NUM_HEADS ng_ds = MAMBA_N_GROUPS * MAMBA_D_STATE - return [d_inner, ng_ds, ng_ds] + if conv_state_layout == "x_b_c": + return [d_inner, ng_ds, ng_ds] + if conv_state_layout == "q_k_v": + return [ng_ds, ng_ds, d_inner] + raise ValueError(f"Unsupported convolution state layout: {conv_state_layout!r}") def _generate_ground_truth(num_requests: int, seed: int = 12345): @@ -270,29 +361,48 @@ def _shard_ssm(full_ssm: torch.Tensor, tp: int, tp_rank: int) -> torch.Tensor: return full_ssm[tp_rank * n : (tp_rank + 1) * n].clone() -def _shard_conv(full_conv: torch.Tensor, tp: int, tp_rank: int) -> torch.Tensor: - """Shard conv per-section along dim 0: [x | B | C] each independently.""" +def _shard_conv( + full_conv: torch.Tensor, + tp: int, + tp_rank: int, + conv_state_layout="x_b_c", +) -> torch.Tensor: + """Shard each semantic convolution-state section independently.""" parts = [] offset = 0 - for sec_dim in _full_conv_section_dims(): + for sec_dim in _full_conv_section_dims(conv_state_layout): n = sec_dim // tp parts.append(full_conv[offset + tp_rank * n : offset + (tp_rank + 1) * n]) offset += sec_dim return torch.cat(parts, dim=0).clone() -def _write_ground_truth_to_ctx(managers, tp, ground_truth, request_ids): +def _write_ground_truth_to_ctx( + managers, + tp, + ground_truth, + request_ids, + conv_state_layout="x_b_c", +): """Write sharded ground truth into ctx managers' allocated mamba slots.""" for rank, mgr in enumerate(managers): for req_idx, rid in enumerate(request_ids): - slot = mgr.mamba_cache_index[rid] - for layer_idx in mgr._impl.mamba_layer_offsets: + slot = _mamba_state_slot(mgr, rid) + for layer_idx in _mamba_layer_ids(mgr): full = ground_truth[req_idx][layer_idx] mgr.get_ssm_states(layer_idx)[slot] = _shard_ssm(full["ssm"], tp, rank) - mgr.get_conv_states(layer_idx)[slot] = _shard_conv(full["conv"], tp, rank) - - -def _compute_expected(ground_truth, gen_managers, gen_tp, gen_request_ids) -> Dict: + mgr.get_conv_states(layer_idx)[slot] = _shard_conv( + full["conv"], tp, rank, conv_state_layout + ) + + +def _compute_expected( + ground_truth, + gen_managers, + gen_tp, + gen_request_ids, + conv_state_layout="x_b_c", +) -> Dict: """Compute expected mamba states BEFORE transfer. Returns: {(gen_rank, req_idx, layer_idx): {"conv": Tensor, "ssm": Tensor}} @@ -300,11 +410,11 @@ def _compute_expected(ground_truth, gen_managers, gen_tp, gen_request_ids) -> Di expected = {} for gen_rank, mgr in enumerate(gen_managers): for req_idx in range(len(gen_request_ids)): - for layer_idx in mgr._impl.mamba_layer_offsets: + for layer_idx in _mamba_layer_ids(mgr): full = ground_truth[req_idx][layer_idx] expected[(gen_rank, req_idx, layer_idx)] = { "ssm": _shard_ssm(full["ssm"], gen_tp, gen_rank), - "conv": _shard_conv(full["conv"], gen_tp, gen_rank), + "conv": _shard_conv(full["conv"], gen_tp, gen_rank, conv_state_layout), } return expected @@ -317,8 +427,8 @@ def _read_actual(gen_managers, gen_request_ids) -> Dict: actual = {} for gen_rank, mgr in enumerate(gen_managers): for req_idx, rid in enumerate(gen_request_ids): - slot = mgr.mamba_cache_index[rid] - for layer_idx in mgr._impl.mamba_layer_offsets: + slot = _mamba_state_slot(mgr, rid) + for layer_idx in _mamba_layer_ids(mgr): actual[(gen_rank, req_idx, layer_idx)] = { "conv": mgr.get_conv_states(layer_idx)[slot].cpu().clone(), "ssm": mgr.get_ssm_states(layer_idx)[slot].cpu().clone(), @@ -358,14 +468,26 @@ def test_mamba_disagg_attention_dp_dummy_with_batch_size_one(): mgr.shutdown() -def run_mamba_transfer_test(ctx_tp: int, gen_tp: int): +def run_mamba_transfer_test( + ctx_tp: int, + gen_tp: int, + use_v2: bool = False, + conv_state_layout: str = "x_b_c", +): """Test mamba transfer: ctx_tp -> gen_tp (PP=1, no DP).""" # -- 1. Create managers, zero mamba caches -- - ctx_mgrs = _create_managers(ctx_tp) - gen_mgrs = _create_managers(gen_tp) + ctx_mgrs = _create_managers( + ctx_tp, + use_v2=use_v2, + conv_state_layout=conv_state_layout, + ) + gen_mgrs = _create_managers( + gen_tp, + use_v2=use_v2, + conv_state_layout=conv_state_layout, + ) for mgr in ctx_mgrs + gen_mgrs: - mgr._impl.mamba_cache.conv.zero_() - mgr._impl.mamba_cache.temporal.zero_() + _zero_mamba_states(mgr) # -- 2. Create transceivers -- config = CacheTransceiverConfig( @@ -419,14 +541,29 @@ def run_mamba_transfer_test(ctx_tp: int, gen_tp: int): # -- 4. Allocate slots -- ctx_batch = ScheduledRequests() - ctx_batch.reset_context_requests(ctx_reqs) - for mgr in ctx_mgrs: - mgr.prepare_resources(ctx_batch) + if use_v2: + ctx_batch.context_requests_last_chunk = ctx_reqs + for mgr in ctx_mgrs: + for req in ctx_reqs: + assert mgr.prepare_context(req) + assert mgr.resize_context(req, req.context_chunk_size) + mgr.prepare_resources(ctx_batch) + else: + ctx_batch.reset_context_requests(ctx_reqs) + for mgr in ctx_mgrs: + mgr.prepare_resources(ctx_batch) gen_batch = ScheduledRequests() - gen_batch.reset_context_requests(gen_reqs) - for mgr in gen_mgrs: - mgr.prepare_resources(gen_batch) + if use_v2: + gen_batch.context_requests_last_chunk = gen_reqs + for mgr in gen_mgrs: + for req in gen_reqs: + assert mgr.prepare_disagg_gen_init(req) + mgr.prepare_resources(gen_batch) + else: + gen_batch.reset_context_requests(gen_reqs) + for mgr in gen_mgrs: + mgr.prepare_resources(gen_batch) for req in ctx_reqs + gen_reqs: req.context_current_position = req.prompt_len @@ -438,10 +575,22 @@ def run_mamba_transfer_test(ctx_tp: int, gen_tp: int): # -- 5. Ground truth -> shard -> write to ctx -- ground_truth = _generate_ground_truth(len(REQUEST_LENGTHS)) - _write_ground_truth_to_ctx(ctx_mgrs, ctx_tp, ground_truth, ctx_rids) + _write_ground_truth_to_ctx( + ctx_mgrs, + ctx_tp, + ground_truth, + ctx_rids, + conv_state_layout, + ) # -- 6. Compute expected BEFORE transfer -- - expected = _compute_expected(ground_truth, gen_mgrs, gen_tp, gen_rids) + expected = _compute_expected( + ground_truth, + gen_mgrs, + gen_tp, + gen_rids, + conv_state_layout, + ) # -- 7. Transfer -- for rank in range(gen_tp): @@ -499,3 +648,23 @@ def test_mamba_transfer(ctx_tp, gen_tp): print(f"\nMamba transfer test: ctx_tp={ctx_tp} -> gen_tp={gen_tp}") run_mamba_transfer_test(ctx_tp, gen_tp) print("PASSED") + + +@pytest.mark.timeout(180) +@pytest.mark.parametrize( + "ctx_tp,gen_tp,conv_state_layout", + [ + (2, 2, "x_b_c"), + (2, 4, "q_k_v"), + (4, 2, "x_b_c"), + ], + ids=["same_tp_xbc", "expand_tp_qkv", "contract_tp_xbc"], +) +def test_v2_mamba_transfer(ctx_tp, gen_tp, conv_state_layout): + """Transfer slot-major V2 Mamba states through Python/NIXL.""" + run_mamba_transfer_test( + ctx_tp, + gen_tp, + use_v2=True, + conv_state_layout=conv_state_layout, + ) diff --git a/tests/unittest/disaggregated/test_minimax_m3_kv_transfer.py b/tests/unittest/disaggregated/test_minimax_m3_kv_transfer.py new file mode 100644 index 000000000000..a05b49a0781d --- /dev/null +++ b/tests/unittest/disaggregated/test_minimax_m3_kv_transfer.py @@ -0,0 +1,647 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MiniMax M3 heterogeneous-topology KV transfer tests. + +This drives the shared threaded NIXL harness (``kv_transfer_harness``) with +MiniMax M3 cache managers and validates both ordinary K/V and the +replicated INDEX_KEY cache. V2 storage coalesces buffers purely by (life +cycle, size), so at TP degrees where K == V == INDEX_KEY bytes per block the +index-K cache shares the K/V pool and the slot interleaves per layer; at +other degrees it gets its own pool. The disagg page-table builder splits each +pool into per-mapper-kind views, so both layouts (and transfers between them) +are exercised by the topology matrix below. +""" + +from collections.abc import Sequence + +import kv_transfer_harness as transfer_harness +import pytest +import torch + +from tensorrt_llm import Mapping +from tensorrt_llm._torch.attention_backend.sparse.minimax_m3 import MiniMaxM3KVCacheManagerV2 +from tensorrt_llm._torch.disaggregation.resource.kv_extractor import KVRegionExtractorV1 +from tensorrt_llm._torch.disaggregation.resource.page import MapperKind +from tensorrt_llm._torch.disaggregation.resource.utils import get_physical_pool, get_pool_bytes +from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2, Role +from tensorrt_llm._utils import TensorWrapper, convert_to_torch_tensor +from tensorrt_llm.bindings import DataType +from tensorrt_llm.bindings.internal.batch_manager import CacheType as CacheTypeCpp +from tensorrt_llm.llmapi.llm_args import KvCacheConfig + +NUM_LAYERS = 4 +NUM_KV_HEADS = 2 +HEAD_DIM = 128 +INDEX_DIM = 128 +TOKENS_PER_BLOCK = transfer_harness.TOKENS_PER_BLOCK +SPARSE_LAYERS = [3] + + +class _FakeKVCache: + num_blocks = 3 + + @staticmethod + def get_base_page_indices(_pool_id): + return [4, 5, 6, -1] + + +class _ShortFakeKVCache: + num_blocks = 2 + + @staticmethod + def get_base_page_indices(_pool_id): + return [7, 8, -1, -1] + + +def test_minimax_cache_indices_support_block_count_limit() -> None: + manager = object.__new__(MiniMaxM3KVCacheManagerV2) + manager.kv_cache_map = {7: _FakeKVCache(), 8: _ShortFakeKVCache()} + + assert manager._get_batch_cache_indices_by_pool_id([7]) == [[4, 5, 6, -1]] + assert manager._get_batch_cache_indices_by_pool_id([7], num_blocks_per_seq=[2]) == [[4, 5]] + assert manager._get_batch_cache_indices_by_pool_id([7], num_blocks_per_seq=[99]) == [[4, 5, 6]] + assert manager._get_batch_cache_indices_by_pool_id([7, 8]) == [ + [4, 5, 6, -1], + [7, 8, -1, -1], + ] + assert manager.get_block_ids_per_seq([7]).tolist() == [[4, 5, 6, 0]] + assert manager.get_block_ids_per_seq([7, 8]).tolist() == [ + [4, 5, 6, 0], + [7, 8, 0, 0], + ] + + +def test_v2_disagg_role_mapper_kind_defaults() -> None: + manager = object.__new__(KVCacheManagerV2) + + assert manager.get_disagg_role_mapper_kinds() == { + Role.ALL: MapperKind.INDEXED, + Role.INDEX_KEY: MapperKind.REPLICATED, + } + + +def test_minimax_disagg_role_mapper_kinds() -> None: + manager = object.__new__(MiniMaxM3KVCacheManagerV2) + + role_mapper_kinds = manager.get_disagg_role_mapper_kinds() + + assert role_mapper_kinds == { + Role.ALL: MapperKind.NHD, + Role.INDEX_KEY: MapperKind.REPLICATED, + } + + +def test_minimax_disagg_rejects_unmanaged_index_value(monkeypatch) -> None: + def fake_base_init(self, *args, **kwargs): + self.is_disagg = kwargs.get("is_disagg", False) + self.layer_offsets = {layer_id: layer_id for layer_id in range(kwargs["num_layers"])} + + monkeypatch.setattr(KVCacheManagerV2, "__init__", fake_base_init) + + with pytest.raises(ValueError, match="requires disable_index_value=True"): + MiniMaxM3KVCacheManagerV2( + num_layers=4, + sparse_layer_ids=[3], + disable_index_value_layer_ids=[], + sparse_index_dim=INDEX_DIM, + is_disagg=True, + ) + + +class _StubGroupingImpl: + """Stub ``manager.impl`` exposing only what ``_kv_pool_mapping_offset`` reads.""" + + def __init__(self, grouping, key_addrs): + self.layer_grouping = grouping + self._key_addrs = key_addrs + + def get_mem_pool_base_address(self, layer_id, role, index_mode=None): + assert role == Role.KEY + return self._key_addrs[int(layer_id)] + + +def test_minimax_kv_pool_mapping_offset_ignores_layer_grouping_order() -> None: + """Offsets must rank layers by physical K address, not grouping order. + + ``layer_grouping``'s iteration order is not a V2 API contract; here it + deliberately disagrees with the physical slot layout (addresses put the + layers in order 0, 2, 1) and the offsets must follow the addresses. + """ + manager = object.__new__(MiniMaxM3KVCacheManagerV2) + manager.impl = _StubGroupingImpl( + grouping=((2, 0, 1),), + key_addrs={0: 0x1000, 1: 0x3000, 2: 0x2000}, + ) + + offsets = {layer_id: manager._kv_pool_mapping_offset(layer_id, 0, 0) for layer_id in (0, 1, 2)} + + assert offsets == {0: 0, 2: 1, 1: 2} + + +def _create_manager( + mapping: Mapping, dtype: DataType, sparse_layers: list[int] | None = None +) -> MiniMaxM3KVCacheManagerV2: + max_num_tokens = 2048 + kv_cache_dtype = { + DataType.FP8: "fp8", + DataType.NVFP4: "nvfp4", + }.get(dtype, "auto") + return MiniMaxM3KVCacheManagerV2( + kv_cache_config=KvCacheConfig( + enable_block_reuse=False, + max_tokens=max_num_tokens, + event_buffer_max_size=0, + dtype=kv_cache_dtype, + ), + kv_cache_type=CacheTypeCpp.SELF, + num_layers=NUM_LAYERS, + num_kv_heads=NUM_KV_HEADS, + head_dim=HEAD_DIM, + tokens_per_block=TOKENS_PER_BLOCK, + max_seq_len=transfer_harness.MAX_SEQ_LEN, + max_batch_size=transfer_harness.MAX_BATCH_SIZE, + mapping=mapping, + dtype=dtype, + vocab_size=transfer_harness.VOCAB_SIZE, + max_num_tokens=max_num_tokens, + sparse_layer_ids=sparse_layers if sparse_layers is not None else SPARSE_LAYERS, + disable_index_value_layer_ids=sparse_layers if sparse_layers is not None else SPARSE_LAYERS, + sparse_index_dim=INDEX_DIM, + ) + + +def test_minimax_m3_pool_view_scheme_coalesced_vs_separate() -> None: + """Pool composition varies with TP; the view scheme must not. + + TP=2 makes kv_heads_per_rank == 1, so K == V == INDEX_KEY bytes per + block and V2 coalesces all three into one pool whose slot interleaves + the sparse layer's index-K between K/V regions (non-uniform layer + stride). TP=1 doubles K/V, so index-K gets its own pool and strides are + uniform. Both layouts must yield the same two per-class views. + """ + from tensorrt_llm._torch.disaggregation.resource.kv_extractor import ( + build_page_table_from_manager, + ) + from tensorrt_llm._torch.disaggregation.resource.utils import get_layer_byte_ranges + + # --- TP=2: coalesced, interleaved slot --- + manager = _create_manager( + Mapping(world_size=2, rank=0, tp_size=2, pp_size=1), DataType.BF16, sparse_layers=[1] + ) + try: + lg = build_page_table_from_manager(manager).layer_groups[0] + assert len(lg.pool_views) == 2 + kv_view = next(pv for pv in lg.pool_views if pv.mapper_kind == MapperKind.NHD) + idx_view = next(pv for pv in lg.pool_views if pv.mapper_kind == MapperKind.REPLICATED) + assert kv_view.pool_role == frozenset({str(Role.KEY), str(Role.VALUE)}) + assert idx_view.pool_role == frozenset({str(Role.INDEX_KEY)}) + # Coalesced: both views address the same physical pool. + assert kv_view.pool_idx == idx_view.pool_idx + starts, bytes_per_layer = get_layer_byte_ranges(kv_view) + unit = bytes_per_layer // 2 # one K or V buffer per block + # Slot: L0K L0V L1K L1V L1IDX L2K L2V L3K L3V — the sparse layer's + # index-K makes the K/V layer stride non-uniform. + assert starts == {0: 0, 1: 2 * unit, 2: 5 * unit, 3: 7 * unit} + idx_starts, idx_bytes_per_layer = get_layer_byte_ranges(idx_view) + assert idx_starts == {1: 4 * unit} + assert idx_bytes_per_layer == unit + finally: + manager.shutdown() + + # --- TP=1: separate pools, uniform strides --- + manager = _create_manager( + Mapping(world_size=1, rank=0, tp_size=1, pp_size=1), DataType.BF16, sparse_layers=[1] + ) + try: + lg = build_page_table_from_manager(manager).layer_groups[0] + assert len(lg.pool_views) == 2 + kv_view = next(pv for pv in lg.pool_views if pv.mapper_kind == MapperKind.NHD) + idx_view = next(pv for pv in lg.pool_views if pv.mapper_kind == MapperKind.REPLICATED) + assert kv_view.pool_idx != idx_view.pool_idx + starts, bytes_per_layer = get_layer_byte_ranges(kv_view) + assert starts == {i: i * bytes_per_layer for i in range(NUM_LAYERS)} + finally: + manager.shutdown() + + +def _create_managers( + tp: int, + pp: int, + enable_dp: bool, + dtype: DataType = DataType.BF16, + sparse_layers: list[int] | None = None, +) -> list[MiniMaxM3KVCacheManagerV2]: + return [ + _create_manager( + Mapping( + world_size=tp * pp, + rank=rank, + tp_size=tp, + pp_size=pp, + enable_attention_dp=enable_dp, + ), + dtype, + sparse_layers, + ) + for rank in range(tp * pp) + ] + + +def _zero_physical_pools(manager: MiniMaxM3KVCacheManagerV2) -> None: + page_table = KVRegionExtractorV1(manager).page_table + unique_pools = {} + for pool_group in page_table.pool_groups: + for pool in pool_group.pools: + unique_pools[pool.base_address] = max( + unique_pools.get(pool.base_address, 0), get_pool_bytes(pool) + ) + for base_address, size in unique_pools.items(): + tensor = convert_to_torch_tensor(TensorWrapper(base_address, DataType.INT8, [size])) + tensor.zero_() + + +def _get_nvfp4_scale_view( + manager: MiniMaxM3KVCacheManagerV2, layer_idx: int +) -> torch.Tensor | None: + if manager.dtype != DataType.NVFP4: + return None + page_table = KVRegionExtractorV1(manager).page_table + local_layer_id = manager.layer_offsets[layer_idx] + scale_roles = frozenset({"key_block_scale", "value_block_scale"}) + for layer_group_id, layer_group in enumerate(page_table.layer_groups): + for pool_view in getattr(layer_group, "pool_views", ()): + # Scale buffers land in their own pool (smaller size class), so + # selecting the pool by role set suffices; every entry for the + # layer inside that pool is a scale buffer. + if not (pool_view.pool_role & scale_roles): + continue + matching_entries = [ + entry + for entry in pool_view.buffer_entries + if int(entry["local_layer_id"]) == local_layer_id + ] + if not matching_entries: + continue + pool = get_physical_pool(page_table, layer_group_id, pool_view.pool_idx) + raw_slots = convert_to_torch_tensor( + TensorWrapper( + pool.base_address, + DataType.INT8, + [pool.num_slots, pool.slot_bytes], + ) + ) + start = min(int(entry["offset"]) for entry in matching_entries) + end = max(int(entry["offset"] + entry["size"]) for entry in matching_entries) + return raw_slots[:, start:end] + raise AssertionError(f"missing NVFP4 scale view for layer {layer_idx}") + + +def _fill_position_dependent( + tensor: torch.Tensor, + *, + layer_idx: int, + first_global_head: int, +) -> None: + """Fill ``[block, role, token, head, dim]`` with exact small integers.""" + block = torch.arange(tensor.shape[0], device=tensor.device)[:, None, None, None, None] + role = torch.arange(tensor.shape[1], device=tensor.device)[None, :, None, None, None] + token = torch.arange(tensor.shape[2], device=tensor.device)[None, None, :, None, None] + head = (first_global_head + torch.arange(tensor.shape[3], device=tensor.device))[ + None, None, None, :, None + ] + dim = torch.arange(tensor.shape[4], device=tensor.device)[None, None, None, None, :] + values = (layer_idx * 17 + block * 11 + role * 13 + token * 3 + head * 19 + dim) % 97 + tensor.copy_(values.to(tensor.dtype)) + + +def _as_nvfp4_scale_tensor( + manager: MiniMaxM3KVCacheManagerV2, + layer_idx: int, +) -> torch.Tensor | None: + scale_view = _get_nvfp4_scale_view(manager, layer_idx) + if scale_view is None: + return None + local_layer_id = manager.layer_offsets[layer_idx] + local_heads = manager.num_kv_heads_per_layer[local_layer_id] + bytes_per_token_head = HEAD_DIM // 16 + return scale_view.view( + scale_view.shape[0], + 2, + TOKENS_PER_BLOCK, + local_heads, + bytes_per_token_head, + ) + + +def _valid_indices( + manager: MiniMaxM3KVCacheManagerV2, + request_id: int, + layer_idx: int, +) -> list[int]: + return [ + index for index in manager.get_batch_cache_indices([request_id], layer_idx)[0] if index >= 0 + ] + + +def _first_global_head(manager: MiniMaxM3KVCacheManagerV2) -> int: + """Map a TP rank to its first logical head, including duplicated heads.""" + if manager.mapping.enable_attention_dp: + return 0 + return manager.mapping.tp_rank * NUM_KV_HEADS // manager.mapping.tp_size + + +def _find_ctx_source( + ctx_managers: list[MiniMaxM3KVCacheManagerV2], + *, + ctx_tp: int, + ctx_enable_dp: bool, + request_idx: int, + layer_idx: int, + global_head: int, +) -> tuple[MiniMaxM3KVCacheManagerV2, int]: + """Return the context manager/local head that owns one logical KV head.""" + for manager in ctx_managers: + if layer_idx not in manager.pp_layers: + continue + tp_rank = manager.mapping.tp_rank + local_layer_id = manager.layer_offsets[layer_idx] + local_heads = manager.num_kv_heads_per_layer[local_layer_id] + if ctx_enable_dp: + if tp_rank == request_idx % ctx_tp: + return manager, global_head + continue + first_global_head = _first_global_head(manager) + if first_global_head <= global_head < first_global_head + local_heads: + return manager, global_head - first_global_head + raise AssertionError( + f"missing context source for request={request_idx}, layer={layer_idx}, " + f"global_head={global_head}" + ) + + +def _initialize_cache( + managers: Sequence[MiniMaxM3KVCacheManagerV2], + _tp: int, + seed_base: int = 0, + fill_random: bool = True, +) -> None: + del seed_base + for manager in managers: + _zero_physical_pools(manager) + if not fill_random: + continue + + for layer_idx in manager.pp_layers: + kv = manager.get_buffers(layer_idx, kv_layout="NHD") + first_global_head = _first_global_head(manager) + _fill_position_dependent( + kv, + layer_idx=layer_idx, + first_global_head=first_global_head, + ) + + index_key = manager.get_index_k_buffer(layer_idx) + if index_key is not None: + index_tensor = index_key.unsqueeze(1) + _fill_position_dependent( + index_tensor, + layer_idx=layer_idx, + first_global_head=0, + ) + + scale_tensor = _as_nvfp4_scale_tensor(manager, layer_idx) + if scale_tensor is not None: + _fill_position_dependent( + scale_tensor, + layer_idx=layer_idx, + first_global_head=first_global_head, + ) + + +def _verify_cache( + request_lengths: list[int], + ctx_managers: Sequence[MiniMaxM3KVCacheManagerV2], + gen_managers: Sequence[MiniMaxM3KVCacheManagerV2], + ctx_tp: int, + ctx_pp: int, + gen_tp: int, + gen_pp: int, + ctx_enable_dp: bool, + gen_enable_dp: bool, + ctx_request_ids: list[int], + gen_request_ids: list[int], +) -> None: + del ctx_pp + + for req_idx, _request_length in enumerate(request_lengths): + gen_request_id = gen_request_ids[req_idx] + for gen_rank, manager in enumerate(gen_managers): + tp_rank = gen_rank % gen_tp + if gen_enable_dp and req_idx % gen_tp != tp_rank: + continue + + for layer_idx in manager.pp_layers: + gen_indices = _valid_indices(manager, gen_request_id, layer_idx) + assert gen_indices + + gen_kv = manager.get_buffers(layer_idx, kv_layout="NHD")[gen_indices] + local_heads = gen_kv.shape[3] + first_global_head = _first_global_head(manager) + for kv_idx in range(2): + for local_head in range(local_heads): + global_head = first_global_head + local_head + ctx_manager, ctx_local_head = _find_ctx_source( + ctx_managers, + ctx_tp=ctx_tp, + ctx_enable_dp=ctx_enable_dp, + request_idx=req_idx, + layer_idx=layer_idx, + global_head=global_head, + ) + ctx_indices = _valid_indices( + ctx_manager, ctx_request_ids[req_idx], layer_idx + ) + ctx_kv = ctx_manager.get_buffers(layer_idx, kv_layout="NHD") + torch.testing.assert_close( + gen_kv[:, kv_idx, :, local_head, :], + ctx_kv[ctx_indices, kv_idx, :, ctx_local_head, :], + rtol=0, + atol=0, + ) + + index_key = manager.get_index_k_buffer(layer_idx) + if index_key is not None: + ctx_manager, _ = _find_ctx_source( + ctx_managers, + ctx_tp=ctx_tp, + ctx_enable_dp=ctx_enable_dp, + request_idx=req_idx, + layer_idx=layer_idx, + global_head=0, + ) + ctx_indices = _valid_indices(ctx_manager, ctx_request_ids[req_idx], layer_idx) + ctx_index_key = ctx_manager.get_index_k_buffer(layer_idx) + assert ctx_index_key is not None + torch.testing.assert_close( + index_key[gen_indices], + ctx_index_key[ctx_indices], + rtol=0, + atol=0, + ) + + gen_scales = _as_nvfp4_scale_tensor(manager, layer_idx) + if gen_scales is not None: + for local_head in range(local_heads): + global_head = first_global_head + local_head + ctx_manager, ctx_local_head = _find_ctx_source( + ctx_managers, + ctx_tp=ctx_tp, + ctx_enable_dp=ctx_enable_dp, + request_idx=req_idx, + layer_idx=layer_idx, + global_head=global_head, + ) + ctx_indices = _valid_indices( + ctx_manager, ctx_request_ids[req_idx], layer_idx + ) + ctx_scales = _as_nvfp4_scale_tensor(ctx_manager, layer_idx) + assert ctx_scales is not None + torch.testing.assert_close( + gen_scales[gen_indices, :, :, local_head, :], + ctx_scales[ctx_indices, :, :, ctx_local_head, :], + rtol=0, + atol=0, + ) + + +# Production is expected to use TEP/DEP context and DEP generation. Bias the +# committed matrix toward head-matched layouts while retaining representative +# head-mismatch, degree-8 duplication, fan-in/fan-out, and PP2 coverage. +HEAD_MATCHED_BF16_TOPOLOGIES = [ + (1, 1, True, 1, 1, True, "dep1_to_dep1"), + (2, 1, True, 2, 1, True, "dep2_to_dep2"), + (4, 1, True, 4, 1, True, "dep4_to_dep4"), + (8, 1, True, 8, 1, True, "dep8_to_dep8"), + (1, 1, True, 8, 1, True, "dep1_to_dep8"), + (8, 1, True, 1, 1, True, "dep8_to_dep1"), + (1, 1, False, 4, 1, True, "tep1_to_dep4"), + (1, 1, False, 8, 1, True, "tep1_to_dep8"), + (1, 2, False, 2, 1, True, "tp1pp2_to_dep2"), +] + +HEAD_MISMATCHED_BF16_TOPOLOGIES = [ + (2, 1, False, 2, 1, True, "tep2_to_dep2"), + (4, 1, False, 4, 1, True, "tep4_to_dep4"), + (8, 1, False, 8, 1, True, "tep8_to_dep8"), + (8, 1, False, 1, 1, True, "tep8_to_dep1"), +] + +BF16_TOPOLOGIES = HEAD_MATCHED_BF16_TOPOLOGIES + HEAD_MISMATCHED_BF16_TOPOLOGIES + +# Smaller quantized-cache sets cover packed K/V geometry across the same +# production directions without repeating the BF16 topology matrix. NVFP4 +# additionally exercises its block-scale pools. +QUANTIZED_TOPOLOGIES = [ + (4, 1, True, 4, 1, True, "dep4_to_dep4"), + (8, 1, True, 1, 1, True, "dep8_to_dep1"), + (1, 1, False, 8, 1, True, "tep1_to_dep8"), + (4, 1, False, 4, 1, True, "tep4_to_dep4"), + (8, 1, False, 1, 1, True, "tep8_to_dep1"), +] + +CACHE_CASES = [(*topology[:6], DataType.BF16, topology[6]) for topology in BF16_TOPOLOGIES] + [ + (*topology[:6], dtype, f"{prefix}_{topology[6]}") + for dtype, prefix in ((DataType.FP8, "fp8"), (DataType.NVFP4, "nvfp4")) + for topology in QUANTIZED_TOPOLOGIES +] + + +@pytest.mark.cuda +@pytest.mark.timeout(180) +@pytest.mark.parametrize( + "ctx_tp,ctx_pp,ctx_enable_dp,gen_tp,gen_pp,gen_enable_dp,cache_dtype", + [case[:7] for case in CACHE_CASES], + ids=[case[7] for case in CACHE_CASES], +) +@pytest.mark.parametrize( + "update_before_transfer", + [True, False], + ids=["update_before", "update_after"], +) +def test_minimax_m3_kv_transfer( + ctx_tp, + ctx_pp, + ctx_enable_dp, + gen_tp, + gen_pp, + gen_enable_dp, + cache_dtype, + update_before_transfer, +) -> None: + transfer_harness.run_kv_transfer_test( + ctx_tp=ctx_tp, + ctx_pp=ctx_pp, + gen_tp=gen_tp, + gen_pp=gen_pp, + ctx_enable_dp=ctx_enable_dp, + gen_enable_dp=gen_enable_dp, + update_before_transfer=update_before_transfer, + manager_factory=lambda tp, pp, enable_dp: _create_managers(tp, pp, enable_dp, cache_dtype), + init_fn=_initialize_cache, + verify_fn=_verify_cache, + ) + + +# Multiple sparse layers spread across PP stages: the replicated index-key +# pool overlaps only partially between peers, exercising the layer-strided +# ReplicatedMapper offsets (a single-sparse-layer model always fully +# overlaps and would degenerate to a whole-slot copy). +MULTI_SPARSE_LAYERS = [1, 2, 3] + + +@pytest.mark.cuda +@pytest.mark.timeout(180) +@pytest.mark.parametrize( + "ctx_tp,ctx_pp,ctx_enable_dp,gen_tp,gen_pp,gen_enable_dp", + [ + (1, 1, True, 1, 2, False), # ctx full index pool -> per-PP-stage subsets + (1, 2, False, 1, 1, True), # per-PP-stage subsets -> full index pool + (2, 1, False, 1, 2, False), # TEP fan-in + PP subset on the same transfer + ], + ids=["dep1_to_tp1pp2", "tp1pp2_to_dep1", "tep2_to_tp1pp2"], +) +def test_minimax_m3_multi_sparse_layer_pp_transfer( + ctx_tp, + ctx_pp, + ctx_enable_dp, + gen_tp, + gen_pp, + gen_enable_dp, +) -> None: + transfer_harness.run_kv_transfer_test( + ctx_tp=ctx_tp, + ctx_pp=ctx_pp, + gen_tp=gen_tp, + gen_pp=gen_pp, + ctx_enable_dp=ctx_enable_dp, + gen_enable_dp=gen_enable_dp, + update_before_transfer=True, + manager_factory=lambda tp, pp, enable_dp: _create_managers( + tp, pp, enable_dp, DataType.BF16, sparse_layers=MULTI_SPARSE_LAYERS + ), + init_fn=_initialize_cache, + verify_fn=_verify_cache, + ) diff --git a/tests/unittest/disaggregated/test_openai_disagg_server.py b/tests/unittest/disaggregated/test_openai_disagg_server.py index beff804f9ed8..5b865027c2bf 100644 --- a/tests/unittest/disaggregated/test_openai_disagg_server.py +++ b/tests/unittest/disaggregated/test_openai_disagg_server.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest +from fastapi import Request from starlette.datastructures import Headers from tensorrt_llm.llmapi.disagg_utils import extract_disagg_cfg @@ -29,6 +31,39 @@ def _raw_request(headers: dict[str, str]): return SimpleNamespace(headers=Headers(headers=headers)) +@pytest.mark.asyncio +async def test_http_cluster_storage_request_is_proxied_to_coordinator(): + payload = b'{"key":"worker","value":"ready"}' + + async def receive(): + return {"type": "http.request", "body": payload, "more_body": False} + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/set", + "query_string": b"source=worker", + "headers": [(b"content-type", b"application/json")], + }, + receive, + ) + server = OpenAIDisaggServer.__new__(OpenAIDisaggServer) + server._coordinator = SimpleNamespace( + proxy_cluster_storage_request=AsyncMock( + return_value=(b'{"result":true}', 200, "application/json") + ) + ) + + response = await server._proxy_cluster_storage_request(request) + + server._coordinator.proxy_cluster_storage_request.assert_awaited_once_with( + "POST", "/set", [("source", "worker")], payload, "application/json" + ) + assert response.status_code == 200 + assert response.body == b'{"result":true}' + + def test_extract_conversation_id_from_headers(): cases = [ ({"X-Session-ID": "session-id"}, "session-id"), diff --git a/tests/unittest/disaggregated/test_openai_disagg_service.py b/tests/unittest/disaggregated/test_openai_disagg_service.py index 7cc7fe4a649e..f19d36739dca 100644 --- a/tests/unittest/disaggregated/test_openai_disagg_service.py +++ b/tests/unittest/disaggregated/test_openai_disagg_service.py @@ -23,12 +23,14 @@ from tensorrt_llm.disaggregated_params import DisaggregatedParams as LlmDisaggregatedParams from tensorrt_llm.executor.result import Logprob from tensorrt_llm.llmapi.disagg_utils import ( + ConditionalDisaggConfig, DisaggClusterConfig, DisaggServerConfig, MinimalInstances, ServerRole, ) from tensorrt_llm.serve.disagg_auto_scaling import DisaggClusterManager, WorkerInfo +from tensorrt_llm.serve.disagg_coordinator import DisaggCoordinatorService from tensorrt_llm.serve.openai_disagg_service import OpenAIDisaggregatedService from tensorrt_llm.serve.openai_protocol import ( ChatCompletionRequest, @@ -54,7 +56,7 @@ chat_response_post_processor, completion_response_post_processor, ) -from tensorrt_llm.serve.router import Router +from tensorrt_llm.serve.router import KvCacheAwareRouter, Router def _client_factory(*_args, **_kwargs): @@ -63,11 +65,41 @@ def _client_factory(*_args, **_kwargs): def _make_service(schedule_style: str) -> OpenAIDisaggregatedService: config = DisaggServerConfig(server_configs=[], schedule_style=schedule_style) + # The coordinator builds its own (empty) routers from config; override them + # with mocks so tests can stub placement / readiness directly. + cluster = DisaggCoordinatorService(config, client_factory=_client_factory) ctx_router = AsyncMock(spec=Router) gen_router = AsyncMock(spec=Router) - return OpenAIDisaggregatedService( - config, ctx_router, gen_router, client_factory=_client_factory + cluster._ctx_router = ctx_router + cluster._gen_router = gen_router + service = OpenAIDisaggregatedService(config, cluster, client_factory=_client_factory) + # Convenience handles for tests that stub placement / readiness directly. + service._ctx_router = ctx_router + service._gen_router = gen_router + return service + + +@pytest.mark.asyncio +async def test_conditional_disagg_uses_selected_server_match_length(): + service = _make_service("context_first") + service._config.conditional_disagg_config = ConditionalDisaggConfig(max_local_prefill_length=32) + router = KvCacheAwareRouter(server_role=ServerRole.GENERATION, servers=[]) + router.get_next_server = AsyncMock( + return_value=( + "gen:8000", + { + "match_length": 64, + "num_tokens": 96, + }, + ) ) + service._gen_router = router + request = CompletionRequest(model="model", prompt=[1] * 96) + + server, need_context = await service._check_conditional_disagg(request, 123) + + assert server == "gen:8000" + assert need_context is False def _make_completion_response( @@ -213,16 +245,18 @@ async def test_is_ready_waits_for_router_preparation(): ), AsyncMock(), ) - service._disagg_cluster_manager = cluster_manager + # Readiness now lives on the DisaggCoordinatorService the service holds. + local = service._coordinator + local._disagg_cluster_manager = cluster_manager cluster_manager._current_ctx_workers["ctx"] = WorkerInfo( worker_id="ctx", role=ServerRole.CONTEXT ) - service._ctx_router = SimpleNamespace(num_prepared_servers=0) - service._gen_router = SimpleNamespace(num_prepared_servers=1) + local._ctx_router = SimpleNamespace(num_prepared_servers=0) + local._gen_router = SimpleNamespace(num_prepared_servers=1) assert await service.is_ready() is False - service._ctx_router.num_prepared_servers = 1 + local._ctx_router.num_prepared_servers = 1 assert await service.is_ready() is False cluster_manager._current_gen_workers["gen"] = WorkerInfo( @@ -348,6 +382,31 @@ async def _delayed_gen_response(*_args, **_kwargs): ) +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_context_only_response_finishes_hooks(stream): + service = _make_service("context_first") + service._ctx_client = AsyncMock() + service._gen_client = AsyncMock() + service._coordinator.get_disagg_request_id = AsyncMock(return_value=42) + service._check_conditional_disagg = AsyncMock(return_value=(None, True)) + service._check_gen_only_disagg = AsyncMock(return_value=False) + service._ctx_router.get_next_server = AsyncMock(return_value=("ctx:9000", {"server_info": {}})) + ctx_response = _make_completion_response("done", finish_reason="stop", context_only=True) + service._ctx_client.send_request = AsyncMock(return_value=ctx_response) + hooks = mock.Mock() + request = CompletionRequest(model="test-model", prompt="hello", stream=stream) + + result = await service._send_disagg_request(request, hooks) + if stream: + assert [chunk async for chunk in result] == [b"data: [DONE]\n\n"] + else: + assert result is ctx_response + + hooks.on_resp_done.assert_called_once_with("", request, ctx_response) + service._gen_client.send_request.assert_not_awaited() + + @pytest.mark.asyncio @pytest.mark.parametrize("schedule_style", ["context_first", "generation_first"]) async def test_send_disagg_request_leaves_streaming_usage_to_gen_server(schedule_style): @@ -398,6 +457,37 @@ async def _gen_response(*_args, **_kwargs): ) +@pytest.mark.asyncio +async def test_context_retry_preserves_generation_reservation_id(): + service = _make_service("context_first") + service._ctx_client = AsyncMock() + service._gen_client = AsyncMock() + service._coordinator.get_disagg_request_id = AsyncMock(return_value=101) + service._check_conditional_disagg = AsyncMock(return_value=("gen:9001", True)) + service._check_gen_only_disagg = AsyncMock(return_value=False) + service._ctx_router.get_next_server = AsyncMock(return_value=("ctx:9000", {"server_info": {}})) + + async def _ctx_response(request, *_args, **_kwargs): + # OpenAIHttpClient regenerates this field before a successful retry. + request.disaggregated_params.disagg_request_id = 202 + return _make_completion_response("", finish_reason="length", disagg_request_id=202) + + service._ctx_client.send_request = AsyncMock(side_effect=_ctx_response) + service._gen_client.send_request = AsyncMock( + return_value=_make_completion_response( + "done", finish_reason="stop", disagg_request_id=202, context_only=False + ) + ) + + request = CompletionRequest(model="test-model", prompt="hello") + await service._send_disagg_request(request) + + service._gen_router.get_next_server.assert_not_awaited() + gen_call = service._gen_client.send_request.call_args + assert gen_call.kwargs["req_id"] == 101 + assert gen_call.args[0].disaggregated_params.ctx_request_id == 202 + + def test_generation_postprocessor_rewrites_usage_from_disaggregated_params(): ctx_usage = UsageInfo( prompt_tokens=128, diff --git a/tests/unittest/disaggregated/test_peer.py b/tests/unittest/disaggregated/test_peer.py index d2329d74e1aa..9b52f754dbdb 100644 --- a/tests/unittest/disaggregated/test_peer.py +++ b/tests/unittest/disaggregated/test_peer.py @@ -1,10 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import pytest +import tensorrt_llm._torch.disaggregation.native.peer as peer_module +from tensorrt_llm._torch.disaggregation.base.region import MemRegionGroup, SpecRegion from tensorrt_llm._torch.disaggregation.native.mixers.attention.peer import ( - HeadMatchMapper, - HeadMismatchMapper, - IdentityMapper, + HNDHeadMismatchMapper, + IntactMapper, + NHDHeadMismatchMapper, + ReplicatedMapper, ) from tensorrt_llm._torch.disaggregation.native.mixers.attention.spec import AttentionInfo from tensorrt_llm._torch.disaggregation.native.peer import PeerOverlap, PeerRegistrar @@ -16,6 +34,7 @@ KVCachePageTable, LocalLayer, MambaLayerGroup, + MapperKind, PhysicalPool, PhysicalPoolGroup, PoolView, @@ -31,21 +50,22 @@ def make_page_table(pool_ptrs=None, block_bytes=None, global_layer_ids=None): if global_layer_ids is None: global_layer_ids = [0, 1] - # Build buffer entries: K + V per local layer - buffer_size = 256 # bytes per buffer entry (arbitrary for tests) - entries = [] - for i in range(len(global_layer_ids)): - base_offset = i * buffer_size * 2 - entries.append((i, base_offset, buffer_size)) - entries.append((i, base_offset + buffer_size, buffer_size)) - buffer_entries = np.array(entries, dtype=BUFFER_ENTRY_DTYPE) - local_layers = [ LocalLayer(local_layer_id=i, global_layer_id=gid) for i, gid in enumerate(global_layer_ids) ] - pool_views = [ - PoolView(pool_idx=pi, buffer_entries=buffer_entries) for pi in range(len(pool_ptrs)) - ] + # Build buffer entries: K + V per local layer, sized so the layers fill + # the pool slot (keeps entry geometry consistent with slot_bytes). + pool_views = [] + for pi, bs in enumerate(block_bytes): + buffer_size = bs // (len(global_layer_ids) * 2) + entries = [] + for i in range(len(global_layer_ids)): + base_offset = i * buffer_size * 2 + entries.append((i, base_offset, buffer_size)) + entries.append((i, base_offset + buffer_size, buffer_size)) + pool_views.append( + PoolView(pool_idx=pi, buffer_entries=np.array(entries, dtype=BUFFER_ENTRY_DTYPE)) + ) physical_pools = [ PhysicalPool(base_address=ptr, slot_bytes=bs, num_slots=128) for ptr, bs in zip(pool_ptrs, block_bytes) @@ -210,6 +230,37 @@ def test_no_overlap(): assert overlap.ranks == [] +@pytest.mark.parametrize( + ("self_heads", "peer_heads"), + [(0, 0), (0, 2), (2, 0)], +) +def test_attention_free_stage_has_no_head_duplication( + self_heads: int, + peer_heads: int, +) -> None: + self_ri = make_rankinfo( + "self", + tp_size=2, + tp_rank=0, + kv_heads_per_rank=self_heads, + layer_num_per_pp=[2], + ) + peer_ri = make_rankinfo( + "peer", + tp_size=2, + tp_rank=0, + kv_heads_per_rank=peer_heads, + layer_num_per_pp=[2], + ) + reg, peer_ri = _make_peer_registrar_and_peer_ri(self_ri, peer_ri) + + overlap = reg.get_peer_overlap(peer_ri, peer_dp_rank=0) + + assert overlap.duplicate_head_factor == 1 + assert overlap.peer_duplicate_head_factor == 1 + assert overlap.ranks == [0] + + def test_pp_ratio_peer_smaller(): self_ri = make_rankinfo( "self", @@ -359,7 +410,15 @@ def test_peer_registrar_get_kv_map_identity(): ) reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) mapper = reg.get_kv_map(peer_ri, (0, 0), (0, 0)) - assert isinstance(mapper, IdentityMapper) + # Full contiguous overlap: one whole-region fragment per block. + assert isinstance(mapper, IntactMapper) + pair = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000]), bytes_per_region=1024)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000]), bytes_per_region=1024)), + ) + assert not isinstance(pair, list) + assert pair.src.memory.ptrs.tolist() == [1000] + assert pair.src.memory.bytes_per_region == 1024 def test_peer_registrar_get_kv_map_head_match(): @@ -380,12 +439,23 @@ def test_peer_registrar_get_kv_map_head_match(): ) reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) mapper = reg.get_kv_map(peer_ri, (0, 0), (0, 0)) - assert isinstance(mapper, HeadMatchMapper) + # Partial layer overlap with matching heads: a single shifted fragment. + assert isinstance(mapper, IntactMapper) + pair = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000]), bytes_per_region=1024)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000]), bytes_per_region=512)), + ) + # Overlap is global layer 1: self slot holds layers [0, 1] (512B each), + # peer slot holds only layer 1. + assert pair.src.memory.ptrs.tolist() == [1512] + assert pair.dst.memory.ptrs.tolist() == [2000] + assert pair.src.memory.bytes_per_region == 512 def test_peer_registrar_get_kv_map_head_mismatch(): self_rankinfo = make_rankinfo(instance_name="local", page_table=make_page_table()) reg = _make_peer_registrar(self_rankinfo) + # Twice the KV heads per rank -> twice the slot bytes on the peer side. peer_ri = make_rankinfo( instance_name="peer", instance_rank=3, @@ -397,11 +467,257 @@ def test_peer_registrar_get_kv_map_head_mismatch(): tokens_per_block=16, dims_per_head=8, layer_num_per_pp=[2], - page_table=make_page_table(), + page_table=make_page_table(block_bytes=[2048]), + ) + reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) + mapper = reg.get_kv_map(peer_ri, (0, 0), (0, 0)) + assert isinstance(mapper, HNDHeadMismatchMapper) + + +def test_peer_registrar_get_kv_map_uses_physical_offset_order(): + """The layer byte offset must follow physical buffer order, not sorted global id. + + ``self`` lays out its two global layers in physical order ``[5, 3]`` (layer 3 + sits at physical slot 1); ``peer`` holds only layer 3. The transfer must copy + ``self``'s slot at byte offset 512 -- a sort-by-global-id would wrongly pick + offset 0 (layer 5's bytes). The entries-driven mapper resolves each layer's + offset from the view's buffer entries, so no ordering convention is involved. + """ + self_pt = make_page_table(global_layer_ids=[5, 3]) + peer_pt = make_page_table(global_layer_ids=[3], block_bytes=[512]) + + self_rankinfo = make_rankinfo(instance_name="local", page_table=self_pt) + reg = _make_peer_registrar(self_rankinfo) + peer_ri = make_rankinfo( + instance_name="peer", + instance_rank=2, + layer_num_per_pp=[1], + page_table=peer_pt, ) reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) mapper = reg.get_kv_map(peer_ri, (0, 0), (0, 0)) - assert isinstance(mapper, HeadMismatchMapper) + assert isinstance(mapper, IntactMapper) + pair = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000]), bytes_per_region=1024)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000]), bytes_per_region=512)), + ) + # Layer 3 (512B) sits at physical slot 1 on self and slot 0 on peer. + assert pair.src.memory.ptrs.tolist() == [1512] + assert pair.dst.memory.ptrs.tolist() == [2000] + assert pair.src.memory.bytes_per_region == 512 + + +def test_peer_registrar_get_kv_map_non_contiguous_overlap_splits_fragments(): + """Shared layers that are not a contiguous slot run transfer as separate fragments. + + ``self`` holds layers ``[0, 1, 2]`` and ``peer`` holds ``[0, 2]``: the overlap + ``{0, 2}`` is not contiguous within ``self``'s slot, so a single contiguous + fragment transfer would corrupt layer 1's bytes. The entries-driven mapper + selects layers by explicit per-layer byte offsets, so it must emit one + correctly-shifted fragment per contiguous run instead of rejecting (or + silently corrupting) the transfer. + """ + self_pt = make_page_table(global_layer_ids=[0, 1, 2], block_bytes=[1536]) + peer_pt = make_page_table(global_layer_ids=[0, 2]) + + self_rankinfo = make_rankinfo(instance_name="local", page_table=self_pt) + reg = _make_peer_registrar(self_rankinfo) + peer_ri = make_rankinfo( + instance_name="peer", + instance_rank=2, + layer_num_per_pp=[2], + page_table=peer_pt, + ) + reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) + mapper = reg.get_kv_map(peer_ri, (0, 0), (0, 0)) + assert isinstance(mapper, IntactMapper) + pairs = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000]), bytes_per_region=1536)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000]), bytes_per_region=1024)), + ) + # Layer 0: self offset 0 -> peer offset 0; layer 2: self offset 1024 -> + # peer offset 512. Non-adjacent on self, so two fragments. + assert [p.src.memory.ptrs.tolist() for p in pairs] == [[1000], [2024]] + assert [p.dst.memory.ptrs.tolist() for p in pairs] == [[2000], [2512]] + assert [p.src.memory.bytes_per_region for p in pairs] == [512, 512] + + +def test_peer_registrar_get_kv_map_merges_non_monotonic_contiguous_layout(): + """Physically contiguous layers merge into one fragment despite global-id order. + + Both peers lay out global layers in physical order ``[5, 3]``: byte-contiguous + on both sides, just not monotonic in global id. Iterating the overlap in + self's physical order keeps the offset arrays adjacent, so the mapper merges + the two layers into a single 1024B fragment; a sorted-by-global-id iteration + would visit offsets ``[512, 0]`` and split the copy into two fragments. + """ + self_pt = make_page_table(global_layer_ids=[5, 3]) + peer_pt = make_page_table(global_layer_ids=[5, 3]) + + self_rankinfo = make_rankinfo(instance_name="local", page_table=self_pt) + reg = _make_peer_registrar(self_rankinfo) + peer_ri = make_rankinfo( + instance_name="peer", + instance_rank=2, + layer_num_per_pp=[2], + page_table=peer_pt, + ) + reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) + mapper = reg.get_kv_map(peer_ri, (0, 0), (0, 0)) + assert isinstance(mapper, IntactMapper) + pair = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000]), bytes_per_region=1024)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000]), bytes_per_region=1024)), + ) + assert not isinstance(pair, list) + assert pair.src.memory.ptrs.tolist() == [1000] + assert pair.dst.memory.ptrs.tolist() == [2000] + assert pair.src.memory.bytes_per_region == 1024 + + +def test_nhd_head_mismatch_mapper_slices_each_token(): + self_ri = make_rankinfo( + instance_name="local", + tp_size=2, + tp_rank=0, + dp_size=2, + dp_rank=0, + kv_heads_per_rank=2, + tokens_per_block=2, + dims_per_head=2, + element_bytes=2, + enable_attention_dp=True, + ) + peer_ri = make_rankinfo( + instance_name="peer", + tp_size=2, + tp_rank=0, + kv_heads_per_rank=1, + tokens_per_block=2, + dims_per_head=2, + element_bytes=2, + ) + mapper = NHDHeadMismatchMapper( + src_layer_offsets=[0], + dst_layer_offsets=[0], + self_ri=self_ri, + peer_ri=peer_ri, + self_bytes_per_layer=32, + peer_bytes_per_layer=16, + self_buffers_per_layer=2, + peer_buffers_per_layer=2, + ) + + pair = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000]), bytes_per_region=32)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000]), bytes_per_region=16)), + ) + + # Source NHD has two heads per token: select head 0 at offsets 0 and 8 + # for K, then 16 and 24 for V. Destination has one head per token. + assert pair.src.memory.ptrs.tolist() == [1000, 1008, 1016, 1024] + assert pair.dst.memory.ptrs.tolist() == [2000, 2004, 2008, 2012] + assert pair.src.memory.bytes_per_region == 4 + assert pair.dst.memory.bytes_per_region == 4 + + +def test_nhd_head_mismatch_mapper_uses_scale_pool_geometry(): + self_ri = make_rankinfo( + tp_size=2, + kv_heads_per_rank=2, + tokens_per_block=2, + dims_per_head=128, + element_bytes=0.5, + ) + peer_ri = make_rankinfo( + instance_name="peer", + tp_size=1, + kv_heads_per_rank=1, + tokens_per_block=2, + dims_per_head=128, + element_bytes=0.5, + ) + mapper = NHDHeadMismatchMapper( + src_layer_offsets=[0], + dst_layer_offsets=[0], + self_ri=self_ri, + peer_ri=peer_ri, + self_bytes_per_layer=8, + peer_bytes_per_layer=4, + self_buffers_per_layer=2, + peer_buffers_per_layer=2, + ) + + pair = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000]), bytes_per_region=8)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000]), bytes_per_region=4)), + ) + + assert pair.src.memory.ptrs.tolist() == [1000, 1002, 1004, 1006] + assert pair.dst.memory.ptrs.tolist() == [2000, 2001, 2002, 2003] + assert pair.src.memory.bytes_per_region == 1 + assert pair.dst.memory.bytes_per_region == 1 + + +def test_replicated_mapper_ignores_kv_head_mismatch(): + self_pt = make_page_table(global_layer_ids=[0]) + peer_pt = make_page_table(global_layer_ids=[0]) + for page_table in (self_pt, peer_pt): + view = page_table.layer_groups[0].pool_views[0] + view.pool_role = frozenset({"index_key"}) + view.mapper_kind = MapperKind.REPLICATED + + self_rankinfo = make_rankinfo(instance_name="local", kv_heads_per_rank=1, page_table=self_pt) + reg = _make_peer_registrar(self_rankinfo) + peer_ri = make_rankinfo( + instance_name="peer", + instance_rank=3, + tp_size=1, + kv_heads_per_rank=8, + layer_num_per_pp=[1], + page_table=peer_pt, + ) + mapper = reg.get_kv_map(peer_ri, (0, 0), (0, 0)) + assert isinstance(mapper, ReplicatedMapper) + + +@pytest.mark.parametrize("peer_dp_rank", [0, 1, 3, 7]) +def test_replicated_pool_owner_rotates_by_destination_dp_rank(peer_dp_rank): + """Exactly one owner per fan-in group, rotated by destination DP rank. + + Mirrors the C++ MLACacheFormatter::needSendCache pairing so the + replicated traffic spreads across local ranks for multi-DP generation. + """ + page_table = make_page_table(global_layer_ids=[0]) + view = page_table.layer_groups[0].pool_views[0] + view.mapper_kind = MapperKind.REPLICATED + + peer_ri = make_rankinfo( + instance_name="peer", + tp_size=8, + dp_size=8, + dp_rank=peer_dp_rank, + enable_attention_dp=True, + page_table=page_table, + layer_num_per_pp=[1], + ) + overlap = PeerOverlap() + ownership = [] + for tp_rank in range(8): + self_ri = make_rankinfo( + instance_name="local", + tp_size=8, + tp_rank=tp_rank, + page_table=page_table, + layer_num_per_pp=[1], + ) + reg = _make_peer_registrar(self_ri) + ownership.append(reg.should_send_pool(overlap, peer_ri, 0, 0)) + + # ratio = self_tp(8) / peer_tp_per_dp(1) = 8: exactly one owner, at the + # slot selected by the destination dp rank. + assert sum(ownership) == 1 + assert ownership.index(True) == peer_dp_rank % 8 def test_peer_registrar_tpb_divisible_warns_but_compatible(): @@ -432,3 +748,379 @@ def test_peer_registrar_tpb_not_divisible_raises(): ) with pytest.raises(ValueError): reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) + + +@pytest.mark.parametrize("mapper_kind", [MapperKind.NHD, MapperKind.REPLICATED]) +def test_peer_registrar_exact_tpb_mapper_rejects_divisible_mismatch(mapper_kind): + self_pt = make_page_table() + peer_pt = make_page_table() + self_pt.layer_groups[0].pool_views[0].mapper_kind = mapper_kind + peer_pt.layer_groups[0].pool_views[0].mapper_kind = mapper_kind + self_ri = make_rankinfo(page_table=self_pt, tokens_per_block=16) + peer_ri = make_rankinfo( + instance_name="peer", + instance_rank=7, + page_table=peer_pt, + tokens_per_block=32, + ) + reg = _make_peer_registrar(self_ri) + + with pytest.raises(ValueError): + reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) + + +def test_intact_mapper_region_size_mismatch_raises(): + with pytest.raises(ValueError, match="cache region size mismatch"): + IntactMapper([0], [0], 256, 128) + + +def test_replicated_mapper_per_layer_size_mismatch_raises(): + with pytest.raises(ValueError, match="Replicated cache region size mismatch"): + ReplicatedMapper( + src_layer_offsets=[0, 128], + dst_layer_offsets=[0, 64], + self_bytes_per_layer=128, + peer_bytes_per_layer=64, + ) + + +def test_replicated_mapper_selects_partial_layer_range(): + """PP mismatch selects the overlap layers via explicit offsets. + + The peer slot holds a superset of layers; only the overlap moves, and + contiguous layers on both sides merge into one fragment. + """ + mapper = ReplicatedMapper( + src_layer_offsets=[0, 128], # self slot: 2 layers x 128B + dst_layer_offsets=[128, 256], # peer slot: 3 layers x 128B, overlap at 1..2 + self_bytes_per_layer=128, + peer_bytes_per_layer=128, + ) + + pair = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000]), bytes_per_region=256)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000]), bytes_per_region=384)), + ) + + assert pair.src.memory.ptrs.tolist() == [1000] + assert pair.dst.memory.ptrs.tolist() == [2128] + assert pair.src.memory.bytes_per_region == 256 + assert pair.dst.memory.bytes_per_region == 256 + + +def test_intact_mapper_splits_non_contiguous_runs(): + """Interleaved slots (another role class between layers) split runs. + + Source layers sit at non-uniform strides (something interleaves after + layer 0); destination is densely packed. Runs must break where either + side is discontiguous, and each fragment must carry the run's bytes. + """ + mapper = IntactMapper( + src_layer_offsets=[0, 192, 320], # gap after layer 0 (64B interleaved) + dst_layer_offsets=[0, 128, 256], # dense + self_bytes_per_layer=128, + peer_bytes_per_layer=128, + ) + + pairs = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000]), bytes_per_region=448)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000]), bytes_per_region=384)), + ) + + assert isinstance(pairs, list) and len(pairs) == 2 + # Run 1: layer 0 alone (src gap breaks the run). + assert pairs[0].src.memory.ptrs.tolist() == [1000] + assert pairs[0].dst.memory.ptrs.tolist() == [2000] + assert pairs[0].src.memory.bytes_per_region == 128 + # Run 2: layers 1-2 contiguous on both sides -> merged. + assert pairs[1].src.memory.ptrs.tolist() == [1192] + assert pairs[1].dst.memory.ptrs.tolist() == [2128] + assert pairs[1].src.memory.bytes_per_region == 256 + + +def test_nhd_mapper_rejects_non_divisible_region_geometry(): + self_ri = make_rankinfo(kv_heads_per_rank=2, tokens_per_block=2) + peer_ri = make_rankinfo( + instance_name="peer", + tp_size=1, + kv_heads_per_rank=4, + tokens_per_block=2, + ) + + with pytest.raises(ValueError, match="not evenly divisible"): + NHDHeadMismatchMapper( + src_layer_offsets=[0], + dst_layer_offsets=[0], + self_ri=self_ri, + peer_ri=peer_ri, + self_bytes_per_layer=17, + peer_bytes_per_layer=32, + self_buffers_per_layer=2, + peer_buffers_per_layer=2, + ) + + +def test_nhd_mapper_rejects_tokens_per_block_mismatch(): + self_ri = make_rankinfo(tokens_per_block=2) + peer_ri = make_rankinfo(instance_name="peer", tokens_per_block=4) + + with pytest.raises(ValueError, match="requires equal tokens_per_block"): + NHDHeadMismatchMapper( + src_layer_offsets=[0], + dst_layer_offsets=[0], + self_ri=self_ri, + peer_ri=peer_ri, + self_bytes_per_layer=32, + peer_bytes_per_layer=64, + self_buffers_per_layer=2, + peer_buffers_per_layer=2, + ) + + +def test_get_buffers_per_layer_rejects_non_uniform_nhd_entries(): + pool_view = PoolView( + pool_idx=0, + buffer_entries=np.array( + [(0, 0, 16), (0, 16, 16), (1, 32, 16)], + dtype=BUFFER_ENTRY_DTYPE, + ), + mapper_kind=MapperKind.NHD, + ) + + with pytest.raises(ValueError, match="layer_group=3, pool=4"): + PeerRegistrar._get_buffers_per_layer( + pool_view, + layer_group_id=3, + pool_idx=4, + ) + + +def test_non_uniform_view_geometry_rejected_for_all_kinds(): + """Entries-driven views require uniform per-layer regions for every kind. + + Under the unified contract INDEXED views are no longer exempt: a view + whose layers have different region sizes cannot be addressed per layer + and must fail loudly instead of transferring garbage. + """ + entries = np.array( + [(0, 0, 16), (0, 16, 16), (1, 32, 16)], + dtype=BUFFER_ENTRY_DTYPE, + ) + self_pt = make_page_table() + peer_pt = make_page_table() + self_pt.layer_groups[0].pool_views[0].buffer_entries = entries + peer_pt.layer_groups[0].pool_views[0].buffer_entries = entries.copy() + self_ri = make_rankinfo(page_table=self_pt) + peer_ri = make_rankinfo(instance_name="peer", page_table=peer_pt) + reg = _make_peer_registrar(self_ri) + reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) + + with pytest.raises(ValueError, match="not uniform"): + reg.get_kv_map(peer_ri, (0, 0), (0, 0)) + + +def test_peer_registrar_allows_byte_aligned_subbyte_head_mismatch(): + """Byte-aligned sub-byte head slicing is allowed on the HND path. + + Per-head bytes are integral here: tpb=16 x dims=8 x 0.5B = 64B. + """ + self_ri = make_rankinfo( + element_bytes=0.5, + kv_heads_per_rank=2, + tp_size=2, + page_table=make_page_table(), + ) + peer_ri = make_rankinfo( + instance_name="peer", + element_bytes=0.5, + kv_heads_per_rank=4, + tp_size=1, + page_table=make_page_table(block_bytes=[2048]), + ) + reg = _make_peer_registrar(self_ri) + reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) + + mapper = reg.get_kv_map(peer_ri, (0, 0), (0, 0)) + + assert isinstance(mapper, HNDHeadMismatchMapper) + + +def test_peer_registrar_rejects_misaligned_subbyte_head_mismatch(): + """Head slicing that lands mid-byte must fail at registration. + + tpb=1 x dims=1 x 0.5B = 0.5B per head is not byte-aligned. + """ + self_ri = make_rankinfo( + element_bytes=0.5, + kv_heads_per_rank=2, + tokens_per_block=1, + dims_per_head=1, + tp_size=2, + page_table=make_page_table(), + ) + peer_ri = make_rankinfo( + instance_name="peer", + element_bytes=0.5, + kv_heads_per_rank=4, + tokens_per_block=1, + dims_per_head=1, + tp_size=1, + page_table=make_page_table(block_bytes=[2048]), + ) + reg = _make_peer_registrar(self_ri) + + with pytest.raises(ValueError): + reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) + + +def test_peer_registrar_dispatches_nhd_mapper(): + self_pt = make_page_table() + peer_pt = make_page_table(block_bytes=[2048]) + self_pt.layer_groups[0].pool_views[0].mapper_kind = MapperKind.NHD + peer_pt.layer_groups[0].pool_views[0].mapper_kind = MapperKind.NHD + self_ri = make_rankinfo( + kv_heads_per_rank=2, + tp_size=2, + page_table=self_pt, + ) + peer_ri = make_rankinfo( + instance_name="peer", + kv_heads_per_rank=4, + tp_size=1, + page_table=peer_pt, + ) + reg = _make_peer_registrar(self_ri) + reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) + + mapper = reg.get_kv_map(peer_ri, (0, 0), (0, 0)) + + assert isinstance(mapper, NHDHeadMismatchMapper) + + +def test_peer_registrar_warns_for_nhd_head_mismatch(monkeypatch): + self_pt = make_page_table() + peer_pt = make_page_table(block_bytes=[2048]) + self_pt.layer_groups[0].pool_views[0].mapper_kind = MapperKind.NHD + peer_pt.layer_groups[0].pool_views[0].mapper_kind = MapperKind.NHD + self_ri = make_rankinfo(kv_heads_per_rank=2, tp_size=2, page_table=self_pt) + peer_ri = make_rankinfo( + instance_name="peer", + kv_heads_per_rank=4, + tp_size=1, + page_table=peer_pt, + ) + warnings = [] + monkeypatch.setattr( + peer_module.logger, + "warning_once", + lambda *message, key: warnings.append((" ".join(map(str, message)), key)), + ) + + _make_peer_registrar(self_ri).register( + peer_ri.instance_name, + peer_ri.instance_rank, + peer_ri, + ) + + assert len(warnings) == 1 + message, key = warnings[0] + assert "4 NIXL descriptors per transferred token per peer" in message + assert "local_kv_heads=2, peer_kv_heads=4" in message + assert key == "native-nhd-head-mismatch-2-4-4" + + +def test_peer_registrar_nhd_head_match_uses_intact_mapper(): + """NHD + equal heads takes the merged-run fast path, not per-token slicing. + + With a dedicated (non-interleaved) K/V pool the run merge collapses the + whole class region into a single fragment per block — the fragment-count + budget for separate layouts. + """ + self_pt = make_page_table() + peer_pt = make_page_table() + self_pt.layer_groups[0].pool_views[0].mapper_kind = MapperKind.NHD + peer_pt.layer_groups[0].pool_views[0].mapper_kind = MapperKind.NHD + self_ri = make_rankinfo(page_table=self_pt) + peer_ri = make_rankinfo(instance_name="peer", instance_rank=9, page_table=peer_pt) + reg = _make_peer_registrar(self_ri) + reg.register(peer_ri.instance_name, peer_ri.instance_rank, peer_ri) + + mapper = reg.get_kv_map(peer_ri, (0, 0), (0, 0)) + assert isinstance(mapper, IntactMapper) + assert not isinstance(mapper, NHDHeadMismatchMapper) + + pair = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000, 5000]), bytes_per_region=1024)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000, 6000]), bytes_per_region=1024)), + ) + # Fully contiguous on both sides -> exactly one fragment per block. + assert not isinstance(pair, list) + assert pair.src.memory.ptrs.tolist() == [1000, 5000] + assert pair.dst.memory.ptrs.tolist() == [2000, 6000] + assert pair.src.memory.bytes_per_region == 1024 + + +def test_indexed_head_mismatch_subbyte_geometry_is_entries_derived(): + """HND head-mismatch byte math comes from slot bytes, never element_bytes. + + Emulates an NVFP4-like cache: element_bytes is fractional (0.5), but the + slot size registered by storage is whole bytes, so every derived offset + and fragment size must be an exact integer. + """ + # self: 2 kv heads/rank, per-head bytes = 16 tokens x 8 dims x 0.5B = 64. + self_ri = make_rankinfo( + tp_size=1, + tp_rank=0, + kv_heads_per_rank=2, + tokens_per_block=16, + dims_per_head=8, + element_bytes=0.5, + ) + # peer: twice the TP -> 1 kv head/rank, half the slot bytes. + peer_ri = make_rankinfo( + instance_name="peer", + tp_size=2, + tp_rank=1, + kv_heads_per_rank=1, + tokens_per_block=16, + dims_per_head=8, + element_bytes=0.5, + ) + mapper = HNDHeadMismatchMapper( + src_layer_offsets=[0, 256], # 2 layers x 256B/layer (kv_factor 2 x 128B) + dst_layer_offsets=[0, 128], # 2 layers x 128B/layer (kv_factor 2 x 64B) + self_ri=self_ri, + peer_ri=peer_ri, + self_bytes_per_layer=256, + peer_bytes_per_layer=128, + self_buffers_per_layer=2, + peer_buffers_per_layer=2, + ) + + pair = mapper.map( + SpecRegion(memory=MemRegionGroup(ptrs=np.array([1000]), bytes_per_region=512)), + SpecRegion(memory=MemRegionGroup(ptrs=np.array([2000]), bytes_per_region=256)), + ) + # peer tp_rank=1 selects head 1 inside self's per-rank pair of heads: + # src_head_off = 1 x 64B; fragments = (layer, k/v) x 2 layers. + assert pair.src.memory.ptrs.tolist() == [1064, 1192, 1320, 1448] + assert pair.dst.memory.ptrs.tolist() == [2000, 2064, 2128, 2192] + assert pair.src.memory.bytes_per_region == 64 + assert pair.src.memory.ptrs.dtype == np.int64 + assert pair.dst.memory.ptrs.dtype == np.int64 + + +def test_indexed_head_mismatch_inconsistent_slot_geometry_raises(): + self_ri = make_rankinfo(tp_size=1, kv_heads_per_rank=2) + peer_ri = make_rankinfo(instance_name="peer", tp_size=2, kv_heads_per_rank=1) + with pytest.raises(ValueError, match="HND bytes per head mismatch"): + HNDHeadMismatchMapper( + src_layer_offsets=[0, 256], + dst_layer_offsets=[0, 256], + self_ri=self_ri, + peer_ri=peer_ri, + self_bytes_per_layer=256, + peer_bytes_per_layer=256, # should be 128 for half the heads + self_buffers_per_layer=2, + peer_buffers_per_layer=2, + ) diff --git a/tests/unittest/disaggregated/test_perf_logger.py b/tests/unittest/disaggregated/test_perf_logger.py index 682a69a13c15..73723a3e9b15 100644 --- a/tests/unittest/disaggregated/test_perf_logger.py +++ b/tests/unittest/disaggregated/test_perf_logger.py @@ -1,3 +1,5 @@ +import os +import tempfile import time from unittest.mock import patch @@ -45,9 +47,20 @@ def test_unrecorded_peer_returns_zero(self): # PerfLogManager tests # --------------------------------------------------------------------------- def _reset_singleton(): - """Reset PerfLogManager singleton so each test gets a fresh instance.""" + """Reset PerfLogManager singleton so each test gets a fresh instance. + + Closes any file handlers the previous instance attached to the global + logging registry so temporary directories can be cleaned up and no file + descriptors leak across tests. + """ from tensorrt_llm._torch.disaggregation.native import perf_logger + instance = perf_logger.PerfLogManager._instance + if instance is not None: + for file_logger in getattr(instance, "_file_loggers", {}).values(): + for handler in list(file_logger.handlers): + file_logger.removeHandler(handler) + handler.close() perf_logger.PerfLogManager._instance = None @@ -77,6 +90,78 @@ def test_disabled_when_env_unset(self): # log() should be a no-op, not crash mgr.log("inst", 0, "csv_line", "info_msg") + def test_log_gen_transfer_summary_writes_csv(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + try: + with patch.dict( + "os.environ", + {"TRTLLM_KVCACHE_TIME_OUTPUT_PATH": tmpdir}, + clear=False, + ): + from tensorrt_llm._torch.disaggregation.native.perf_logger import PerfLogManager + + mgr = PerfLogManager() + mgr.log_gen_transfer_summary( + unique_rid=42, + instance_name="test_inst", + instance_rank=0, + gen_side_transfer_time_ms=12.345, + kv_cache_size=1024, + ) + csv_path = os.path.join(tmpdir, "test_inst_0_gen_transfer_summary.csv") + assert os.path.exists(csv_path), f"CSV file not created at {csv_path}" + with open(csv_path) as f: + lines = f.readlines() + assert len(lines) >= 2 # header + at least 1 data row + assert "gen_side_transfer_time(ms)" in lines[0] + assert "42" in lines[1] + assert "12.345" in lines[1] + assert "1024" in lines[1] + finally: + # Close file handlers before the temporary directory exits. + _reset_singleton() + + def test_cpp_env_takes_priority_over_legacy_for_log(self) -> None: + """With both C++ and legacy env vars set, log() uses the C++ naming.""" + with tempfile.TemporaryDirectory() as tmpdir: + try: + legacy_base = os.path.join(tmpdir, "legacy") + with patch.dict( + "os.environ", + { + "TRTLLM_KVCACHE_TIME_OUTPUT_PATH": tmpdir, + "TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO": "1", + "TLLM_KV_TRANSFER_PERF_LOG_FILE": legacy_base, + }, + clear=False, + ): + from tensorrt_llm._torch.disaggregation.native.perf_logger import PerfLogManager + + mgr = PerfLogManager() + mgr.log("test_inst", 0, "csv_line", "info_msg") + cpp_path = os.path.join(tmpdir, "test_inst_0.csv") + legacy_path = f"{legacy_base}_test_inst_0.csv" + assert os.path.exists(cpp_path), f"C++-style CSV not created at {cpp_path}" + assert not os.path.exists(legacy_path), "legacy-style CSV should not be used" + finally: + # Close file handlers before the temporary directory exits. + _reset_singleton() + + @patch.dict("os.environ", {}, clear=False) + def test_log_gen_transfer_summary_disabled_without_env(self) -> None: + os.environ.pop("TRTLLM_KVCACHE_TIME_OUTPUT_PATH", None) + from tensorrt_llm._torch.disaggregation.native.perf_logger import PerfLogManager + + mgr = PerfLogManager() + # Should be a no-op, not crash + mgr.log_gen_transfer_summary( + unique_rid=1, + instance_name="test", + instance_rank=0, + gen_side_transfer_time_ms=0.0, + kv_cache_size=0, + ) + @patch.dict("os.environ", {"TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO": "1"}, clear=False) def test_log_task_perf_does_not_crash(self): from tensorrt_llm._torch.disaggregation.native.perf_logger import PerfLogManager, PerfTimer diff --git a/tests/unittest/disaggregated/test_pool_matching.py b/tests/unittest/disaggregated/test_pool_matching.py index 884852c3a539..ba1aeea4d90c 100644 --- a/tests/unittest/disaggregated/test_pool_matching.py +++ b/tests/unittest/disaggregated/test_pool_matching.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Golden tests for ``PeerRegistrar.get_pool_mapping``. Locks the current (pre-refactor) behavior of disagg pool matching across the @@ -6,7 +21,7 @@ * single KV pool, full layer overlap (basic MHA) * MLA (kv_factor=1, KEY-only pool) * KV + block-scale pools coexisting in one LG - * KV + FLAT pools coexisting in one LG (FLAT has empty buffer_entries) + * KV + REPLICATED indexer pools coexisting in one LG * PP partial layer overlap (Step-1 LG match by global_layer_id) * Two pools with same role but different layer sets within a peer LG (DSv4 virtual-layer scenario, exercises ``best_overlap``) @@ -33,6 +48,7 @@ PhysicalPoolGroup, PoolView, ) +from tensorrt_llm._torch.disaggregation.resource.utils import get_layer_to_layer_group # --------------------------------------------------------------------------- # Builders @@ -70,13 +86,13 @@ def _pool_view(pool_idx, layer_role_pairs, *, pool_role=None, mapper_kind=Mapper ) -def _empty_pool_view(pool_idx, *, pool_role=frozenset({"indexer_k"})): - """Pool view with no buffer entries — FLAT pool convention.""" - return PoolView( - pool_idx=pool_idx, - buffer_entries=np.array([], dtype=BUFFER_ENTRY_DTYPE), +def _indexer_pool_view(pool_idx, local_layer_ids, *, pool_role=frozenset({"indexer_k"})): + """Replicated indexer pool view: one synthesized entry per local layer.""" + return _pool_view( + pool_idx, + [(lid, "indexer_k") for lid in local_layer_ids], pool_role=pool_role, - mapper_kind=MapperKind.FLAT, + mapper_kind=MapperKind.REPLICATED, ) @@ -229,16 +245,16 @@ def _bq_pool(pool_idx, lids): def test_kv_and_indexer_in_same_lg(): - """KV pool + FLAT pool. FLAT has empty buffer_entries; matches by role.""" + """KV pool + replicated indexer pool match independently by role.""" self_lg = _attn_lg( 0, [(0, 0), (1, 1)], - [_kv_pool_view(0, [0, 1]), _empty_pool_view(1)], + [_kv_pool_view(0, [0, 1]), _indexer_pool_view(1, [0, 1])], ) peer_lg = _attn_lg( 0, [(0, 0), (1, 1)], - [_kv_pool_view(0, [0, 1]), _empty_pool_view(1)], + [_kv_pool_view(0, [0, 1]), _indexer_pool_view(1, [0, 1])], ) self_pt = _page_table([self_lg], pool_specs={0: [(1024, 64, 0x1000), (512, 64, 0x2000)]}) @@ -251,6 +267,58 @@ def test_kv_and_indexer_in_same_lg(): assert mapping == {(0, 0): (0, 0), (0, 1): (0, 1)} +def test_minimax_split_kv_and_replicated_index_pools_match(): + """MiniMax M3 shape: NHD KV pool + REPLICATED index-key pool. + + coalescing_group derivation keeps INDEX_KEY in its own pool on every + topology, so both sides always present the same two role sets and + role-set matching pairs them 1:1 with kinds intact. + """ + + def _lg(): + return _attn_lg( + 0, + [(0, 0), (1, 1)], + [ + _pool_view( + 0, + [(lid, role) for lid in (0, 1) for role in ("key", "value")], + mapper_kind=MapperKind.NHD, + ), + _pool_view( + 1, [(0, "index_key"), (1, "index_key")], mapper_kind=MapperKind.REPLICATED + ), + ], + ) + + self_pt = _page_table([_lg()], pool_specs={0: [(512, 64, 0x1000), (256, 64, 0x2000)]}) + peer_pt = _page_table([_lg()], pool_specs={0: [(512, 64, 0x3000), (256, 64, 0x4000)]}) + + reg = _registrar(self_pt) + peer_ri = _rank_info(name="peer", rank=1, page_table=peer_pt) + + mapping = reg.get_pool_mapping(peer_ri) + assert mapping == {(0, 0): (0, 0), (0, 1): (0, 1)} + + +def test_mismatched_view_kinds_raise(): + """Kind disagreement on one role set must fail loudly. + + Same role set but different kinds on the two sides is a version or + configuration inconsistency. + """ + self_lg = _attn_lg( + 0, [(0, 0)], [_pool_view(0, [(0, "index_key")], mapper_kind=MapperKind.REPLICATED)] + ) + peer_lg = _attn_lg(0, [(0, 0)], [_pool_view(0, [(0, "index_key")], mapper_kind=MapperKind.NHD)]) + + reg = _registrar(_page_table([self_lg])) + peer_ri = _rank_info(name="peer", rank=1, page_table=_page_table([peer_lg])) + + with pytest.raises(ValueError, match="incompatible mapper"): + reg.get_pool_mapping(peer_ri) + + def test_pp_partial_layer_overlap(): """Self covers global layers {10,11}, peer covers {11,12}. Match via overlap on layer 11.""" self_lg = _attn_lg(0, [(0, 10), (1, 11)], [_kv_pool_view(0, [0, 1])]) @@ -265,6 +333,47 @@ def test_pp_partial_layer_overlap(): assert mapping == {(0, 0): (0, 0)} +def test_pool_view_spanning_multiple_peer_lgs_raises(): + """A self pool view spanning two peer LGs must fail loudly. + + Mismatched layer grouping between peers is an unsupported topology; + silently transferring only the first LG's overlap would drop layers. + """ + self_lg = _attn_lg(0, [(0, 10), (1, 11)], [_kv_pool_view(0, [0, 1])]) + # Peer holds the same global layers but split across two layer groups. + peer_lg0 = _attn_lg(0, [(0, 10)], [_kv_pool_view(0, [0])]) + peer_lg1 = _attn_lg(1, [(0, 11)], [_kv_pool_view(0, [0])]) + + reg = _registrar(_page_table([self_lg])) + peer_ri = _rank_info(name="peer", rank=1, page_table=_page_table([peer_lg0, peer_lg1])) + + with pytest.raises(ValueError, match="multiple peer layer groups"): + reg.get_pool_mapping(peer_ri) + + +def test_skewed_buffer_entries_per_layer_raises(): + """A skewed per-layer entry distribution must raise. + + 1 + 3 entries over two layers passes ``total % layers == 0`` (4 % 2) + but is not a uniform per-layer layout — must raise, not return 2. + """ + view = _pool_view(0, [(0, "key"), (1, "key"), (1, "key"), (1, "key")]) + with pytest.raises(ValueError, match="not evenly distributed"): + PeerRegistrar._get_buffers_per_layer(view, layer_group_id=0, pool_idx=0) + + +def test_duplicate_global_layer_id_across_lgs_raises(): + """Layer groups must partition a rank's attention layers. + + The same global_layer_id in two LGs would silently corrupt peer matching. + """ + lg0 = _attn_lg(0, [(0, 10)], [_kv_pool_view(0, [0])]) + lg1 = _attn_lg(1, [(0, 10)], [_kv_pool_view(0, [0])]) + + with pytest.raises(ValueError, match="layer groups must partition"): + get_layer_to_layer_group(_page_table([lg0, lg1])) + + def test_two_pools_distinct_roles_in_same_lg(): """Two pools with distinct pool_role in one LG match by role, not by layer overlap. @@ -274,11 +383,11 @@ def test_two_pools_distinct_roles_in_same_lg(): pool with the same pool_role. """ self_kv = _kv_pool_view(0, [0, 1]) - self_indexer = _empty_pool_view(1) + self_indexer = _indexer_pool_view(1, [0, 1]) self_lg = _attn_lg(0, [(0, 10), (1, 11)], [self_kv, self_indexer]) peer_kv = _kv_pool_view(0, [0, 1]) - peer_indexer = _empty_pool_view(1) + peer_indexer = _indexer_pool_view(1, [0, 1]) peer_lg = _attn_lg(0, [(0, 10), (1, 11)], [peer_kv, peer_indexer]) self_pt = _page_table([self_lg], pool_specs={0: [(1024, 64, 0x1000), (512, 64, 0x2000)]}) @@ -323,20 +432,19 @@ def test_same_role_pools_disambiguated_by_layer_overlap(): @pytest.mark.parametrize( ("self_mapper_kind", "peer_mapper_kind"), [ - (MapperKind.FLAT, MapperKind.INDEXED), - (MapperKind.INDEXED, MapperKind.FLAT), + (MapperKind.REPLICATED, MapperKind.INDEXED), + (MapperKind.INDEXED, MapperKind.REPLICATED), ], ) def test_mixed_mapper_kinds_are_rejected(self_mapper_kind, peer_mapper_kind): """Pool layouts must use the same mapper kind on both peers.""" def _indexer_view(mapper_kind): - if mapper_kind == MapperKind.FLAT: - return _empty_pool_view(0) return _pool_view( 0, [(0, "indexer_k"), (1, "indexer_k")], pool_role=frozenset({"indexer_k"}), + mapper_kind=mapper_kind, ) self_lg = _attn_lg(0, [(0, 10), (1, 11)], [_indexer_view(self_mapper_kind)]) diff --git a/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py b/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py index a81e05026cbb..d414d18fb48b 100644 --- a/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py +++ b/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py @@ -15,8 +15,10 @@ import torch.distributed as dist import torch.multiprocessing as mp -# Exclude IB (no fabric) and gdr_copy (UCX rcache SIGABRT at teardown). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" import tensorrt_llm import tensorrt_llm.bindings diff --git a/tests/unittest/disaggregated/test_rank_info.py b/tests/unittest/disaggregated/test_rank_info.py index 5c934152efed..e209e3d5cd50 100644 --- a/tests/unittest/disaggregated/test_rank_info.py +++ b/tests/unittest/disaggregated/test_rank_info.py @@ -1,6 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from types import SimpleNamespace + import numpy as np +import pytest +from tensorrt_llm import bindings +from tensorrt_llm._torch.disaggregation.native import rank_info as rank_info_module from tensorrt_llm._torch.disaggregation.native.auxiliary import AuxBufferMeta +from tensorrt_llm._torch.disaggregation.native.mixers.ssm.peer import MambaPolicy from tensorrt_llm._torch.disaggregation.native.rank_info import RankInfo @@ -75,3 +96,102 @@ def test_rank_info_roundtrip_with_aux_meta(): np.testing.assert_array_equal(restored.aux_meta.size, [1024, 2048]) np.testing.assert_array_equal(restored.aux_meta.item_sizes, [64, 128]) assert restored.aux_meta.device == "cpu" + + +def test_from_kv_cache_manager_uses_first_nonzero_kv_head_count(monkeypatch) -> None: + monkeypatch.setattr(rank_info_module, "build_page_table_from_manager", lambda _: None) + mapping = SimpleNamespace( + rank=0, + tp_size=1, + tp_rank=0, + pp_size=1, + pp_rank=0, + dp_size=1, + cp_size=1, + cp_rank=0, + enable_attention_dp=False, + ) + manager = SimpleNamespace( + mapping=mapping, + num_kv_heads_per_layer=[0, 8, 0], + pp_layers=[0, 1, 2], + tokens_per_block=32, + head_dim=128, + dtype=bindings.DataType.HALF, + kv_factor=2, + ) + + info = RankInfo.from_kv_cache_manager("ctx", manager, device_id=0) + + assert info.attention.kv_heads_per_rank == 8 + + +def test_from_kv_cache_manager_preserves_attention_dp_on_attention_free_stage( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(rank_info_module, "build_page_table_from_manager", lambda _: None) + mapping = SimpleNamespace( + rank=1, + tp_size=2, + tp_rank=1, + pp_size=1, + pp_rank=0, + dp_size=1, + cp_size=1, + cp_rank=0, + enable_attention_dp=True, + ) + manager = SimpleNamespace( + mapping=mapping, + num_kv_heads_per_layer=[0, 0], + pp_layers=[0, 1], + tokens_per_block=32, + head_dim=128, + dtype=bindings.DataType.HALF, + kv_factor=2, + ) + + info = RankInfo.from_kv_cache_manager("ctx", manager, device_id=0) + + assert info.attention is not None + assert info.attention.kv_heads_per_rank == 0 + assert info.attention.enable_attention_dp + assert MambaPolicy._mamba_tp(info) == (1, 0) + + +@pytest.mark.parametrize( + ("dtype", "expected_element_bytes", "expected_type"), + [(bindings.DataType.NVFP4, 0.5, float), (bindings.DataType.HALF, 2, int)], +) +def test_rank_info_represents_cache_element_bytes( + monkeypatch, dtype, expected_element_bytes, expected_type +): + monkeypatch.setattr(rank_info_module, "build_page_table_from_manager", lambda _manager: None) + manager = SimpleNamespace( + mapping=SimpleNamespace( + rank=0, + tp_size=2, + tp_rank=0, + pp_size=1, + pp_rank=0, + dp_size=1, + cp_size=1, + cp_rank=0, + enable_attention_dp=False, + ), + pp_layers=[0], + num_kv_heads_per_layer=[4], + tokens_per_block=64, + head_dim=128, + dtype=dtype, + kv_factor=2, + ) + + rank_info = RankInfo.from_kv_cache_manager("ctx", manager, device_id=0) + + assert rank_info.attention.element_bytes == expected_element_bytes + assert isinstance(rank_info.attention.element_bytes, expected_type) + + restored = RankInfo.from_bytes(rank_info.to_bytes()) + assert restored.attention.element_bytes == expected_element_bytes + assert isinstance(restored.attention.element_bytes, expected_type) diff --git a/tests/unittest/disaggregated/test_request_id.py b/tests/unittest/disaggregated/test_request_id.py index 70ea8a36fcad..3b5deac0b1b2 100644 --- a/tests/unittest/disaggregated/test_request_id.py +++ b/tests/unittest/disaggregated/test_request_id.py @@ -1,10 +1,14 @@ -"""Tests for _get_request_id in ExecutorRequestQueue. +"""Tests for disaggregated request-id handling. -Demonstrates the known bug: disagg_request_id=0 is falsy and gets skipped. +Covers ExecutorRequestQueue._get_request_id and the Receiver's sender_req_id +fallback in the native KV transceiver (nvbugs/6482576). """ +from types import SimpleNamespace from unittest.mock import MagicMock +import pytest + def _make_queue(max_batch_size=128): """Create an ExecutorRequestQueue with mocked Distributed.""" @@ -56,3 +60,48 @@ def test_get_request_id_zero_bug(): # BUG: should return 0, but returns auto-incremented id instead assert rid != 0, "If this fails, the bug has been fixed — update this test" assert rid == 128 # falls through to auto-increment + + +# --------------------------------------------------------------------------- # +# Receiver._build_recv_req_info: sender_req_id fallback (nvbugs/6482576) +# --------------------------------------------------------------------------- # +def _make_recv_task(ctx_request_id, disagg_request_id, unique_rid=123): + """Minimal stand-in for KVRecvTask with only the fields the method reads.""" + return SimpleNamespace( + _params=SimpleNamespace(ctx_request_id=ctx_request_id, disagg_request_id=disagg_request_id), + _unique_rid=unique_rid, + _kv_slice=SimpleNamespace(block_ids_per_layer_groups=[], mamba_state_index=None), + _aux_slot=None, + slice_id=0, + ) + + +def _build_recv_req_info(tfr, task): + """Call the unbound Receiver method against a mocked registrar.""" + recv_self = SimpleNamespace( + _registrar=SimpleNamespace( + self_rank_info=SimpleNamespace(instance_name="gen-0", instance_rank=0) + ) + ) + return tfr.Receiver._build_recv_req_info(recv_self, task) + + +def test_build_recv_req_info_prefers_ctx_request_id(): + """Normal disagg flow: ctx_request_id keys the sender's TxSession.""" + tfr = pytest.importorskip("tensorrt_llm._torch.disaggregation.native.transfer") + info = _build_recv_req_info(tfr, _make_recv_task(ctx_request_id=7, disagg_request_id=99)) + assert info.sender_req_id == 7 + + +def test_build_recv_req_info_falls_back_to_disagg_request_id(): + """nvbugs/6482576: fall back to disagg_request_id when ctx_request_id is None.""" + tfr = pytest.importorskip("tensorrt_llm._torch.disaggregation.native.transfer") + info = _build_recv_req_info(tfr, _make_recv_task(ctx_request_id=None, disagg_request_id=99)) + assert info.sender_req_id == 99 + + +def test_build_recv_req_info_both_ids_none_raises(): + """Raise when neither request id is available (survives python -O).""" + tfr = pytest.importorskip("tensorrt_llm._torch.disaggregation.native.transfer") + with pytest.raises(ValueError): + _build_recv_req_info(tfr, _make_recv_task(ctx_request_id=None, disagg_request_id=None)) diff --git a/tests/unittest/disaggregated/test_router.py b/tests/unittest/disaggregated/test_router.py index b8d5980e087b..dd0b6d5facb3 100644 --- a/tests/unittest/disaggregated/test_router.py +++ b/tests/unittest/disaggregated/test_router.py @@ -2,16 +2,19 @@ import copy import random import threading +from pathlib import Path +from types import SimpleNamespace from unittest import mock import aiohttp +import msgpack import pytest from tensorrt_llm.llmapi.disagg_utils import RouterConfig from tensorrt_llm.runtime.kv_cache_hash import (get_cache_salt_id, hash_v1_block_key, truncate_sha256_hash_to_int64) -from tensorrt_llm.runtime.kv_cache_manager_v2._block_radix_tree import ( +from tensorrt_llm.runtime.kv_cache_manager_v2 import ( ReuseScope, sequence_to_blockchain_keys) # yapf: disable from tensorrt_llm.serve.openai_protocol import (ChatCompletionRequest, @@ -24,6 +27,7 @@ KV_CACHE_HASH_ALGO_V2, KV_CACHE_HASH_ALGO_V2_SHA256_64, BlockHashMixin, ConversationRouter, + CoordinatorDelegatingRouter, KvCacheAwareRouter, KvCacheAwareServerState, LoadBalancingRouter, RoundRobinRouter, @@ -74,6 +78,57 @@ def _make_mock_aiohttp_session(return_value=None): return mock_session +@pytest.mark.asyncio +async def test_coordinator_finish_retry_is_bounded(): + local_router = RoundRobinRouter(server_role=None, servers=["server1"]) + router = CoordinatorDelegatingRouter("http://coordinator", local_router, + "generation") + + def _response(status, body): + response = mock.AsyncMock() + response.status = status + response.read = mock.AsyncMock( + return_value=msgpack.packb(body, use_bin_type=True)) + context = mock.AsyncMock() + context.__aenter__ = mock.AsyncMock(return_value=response) + context.__aexit__ = mock.AsyncMock(return_value=False) + return context + + session = mock.MagicMock() + session.post = mock.MagicMock(side_effect=[ + _response(503, {"error": "temporarily unavailable"}), + _response(503, {"error": "temporarily unavailable"}), + _response(503, {"error": "temporarily unavailable"}), + ]) + session.close = mock.AsyncMock() + router._session = session + + with mock.patch("tensorrt_llm.serve.router.asyncio.sleep", + new_callable=mock.AsyncMock) as sleep: + await router._finish_async(123, True) + + assert session.post.call_count == 3 + assert sleep.await_count == 2 + assert all(call.kwargs["timeout"] == 5 + for call in session.post.call_args_list) + + +@pytest.mark.asyncio +async def test_coordinator_finish_queue_is_bounded(): + local_router = RoundRobinRouter(server_role=None, servers=["server1"]) + router = CoordinatorDelegatingRouter("http://coordinator", local_router, + "generation") + router._finish_queue = asyncio.Queue(maxsize=1) + router._ensure_finish_workers = mock.Mock() + + request = mock.Mock() + await router.finish_request(request, req_id=1) + await router.finish_request(request, req_id=2) + + assert router._finish_queue.qsize() == 1 + assert router._dropped_finishes == 1 + + @pytest.fixture(autouse=True) def mock_aiohttp_session(request): """Auto-mock aiohttp.ClientSession so poll_events doesn't make real HTTP calls.""" @@ -517,7 +572,7 @@ def __init__(self, prompt): use_tokens=False, max_batch_size=32, tokens_per_block=tokens_per_block) - monkeypatch.setattr("tensorrt_llm.serve.router.get_cache_salt_id", + monkeypatch.setattr("tensorrt_llm.serve.router_utils.get_cache_salt_id", lambda cache_salt: cache_salt_id) for server in servers: router._server_info[server] = { @@ -786,33 +841,8 @@ def test_kv_cache_aware_server_state_remove_blocks_silent_on_missing(): @pytest.mark.asyncio -async def test_kv_cache_aware_router_tracks_routed_blocks_at_routing(servers): - tokens_per_block = 4 - token_lists = [[1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008]] - router = KvCacheAwareRouter(server_role=None, - servers=servers, - use_tokens=False, - max_batch_size=32, - tokens_per_block=tokens_per_block, - track_routed_blocks=True) - - request = CompletionRequest(model="TinyLlama", - prompt=copy.deepcopy(token_lists)) - server, info = await router.get_next_server(request) - - assert id(request) in router._pending_routed_blocks - stashed, stashed_algo = router._pending_routed_blocks[id(request)] - expected_flat = [h for hl in info["block_hashes"] for h in hl] - assert stashed == expected_flat - assert stashed and not isinstance(stashed[0], list) - assert stashed_algo == info["hash_algo"] - - await router.finish_request(request) - assert id(request) not in router._pending_routed_blocks - - -@pytest.mark.asyncio -async def test_kv_cache_aware_router_inserts_routed_blocks_on_finish(servers): +async def test_kv_cache_aware_router_applies_blocks_after_successful_finish( + servers): tokens_per_block = 4 token_lists = [[2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008]] router = KvCacheAwareRouter(server_role=None, @@ -855,8 +885,6 @@ async def test_kv_cache_aware_router_routed_blocks_disabled_skips_pending( prompt=copy.deepcopy(token_lists)) server, info = await router.get_next_server(request) - assert router._pending_routed_blocks == {} - await router.finish_request(request) assert await router._server_state[server].matched_tokens( @@ -864,7 +892,7 @@ async def test_kv_cache_aware_router_routed_blocks_disabled_skips_pending( @pytest.mark.asyncio -async def test_kv_cache_aware_router_drops_routed_blocks_on_failure(servers): +async def test_kv_cache_aware_router_discards_routed_blocks_on_failure(servers): tokens_per_block = 4 token_lists = [[4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, 4008]] router = KvCacheAwareRouter(server_role=None, @@ -877,11 +905,11 @@ async def test_kv_cache_aware_router_drops_routed_blocks_on_failure(servers): request = CompletionRequest(model="TinyLlama", prompt=copy.deepcopy(token_lists)) server, info = await router.get_next_server(request) - assert id(request) in router._pending_routed_blocks + assert await router._server_state[server].matched_tokens( + info["block_hashes"], hash_algo=info["hash_algo"]) == 0 await router.finish_request(request, success=False) - assert id(request) not in router._pending_routed_blocks assert await router._server_state[server].matched_tokens( info["block_hashes"], hash_algo=info["hash_algo"]) == 0 @@ -1740,7 +1768,9 @@ def test_block_hash_mixin_routes_through_transformers_tokenizer(): DeepSeek-V3 Metaspace, ``_fallback_to_fast_tokenizer`` for DeepSeek-V3.2 on transformers >= 5.x). Without this routing, ``trtllm-serve`` would tokenize prompts differently from the rest of TRT-LLM when computing - block hashes for cache hits. + block hashes for cache hits. The request's model is client-controlled, so + it must not enable remote tokenizer code unless an operator configured a + fixed tokenizer directory. """ class _Probe(BlockHashMixin): @@ -1756,12 +1786,26 @@ class _Probe(BlockHashMixin): return_value=wrapper) as routed: out = probe._get_tokenizer("dummy/model") - routed.assert_called_once_with("dummy/model", trust_remote_code=True) + routed.assert_called_once_with("dummy/model", trust_remote_code=False) # The cached tokenizer must be the raw HF tokenizer used by _tokenize, # not the TransformersTokenizer wrapper. assert out is inner assert probe._tokenizers["dummy/model"] is inner + fixed_probe = _Probe() + fixed_probe._init_block_hashing(tokenizer_dir="/trusted/tokenizer") + + fixed_inner = mock.MagicMock() + fixed_wrapper = mock.MagicMock(tokenizer=fixed_inner) + with mock.patch( + "tensorrt_llm.tokenizer.TransformersTokenizer.from_pretrained", + return_value=fixed_wrapper) as routed: + fixed_out = fixed_probe._get_tokenizer("dummy/model") + + routed.assert_called_once_with("/trusted/tokenizer", trust_remote_code=True) + assert fixed_out is fixed_inner + assert fixed_probe._tokenizers["dummy/model"] is fixed_inner + @pytest.mark.asyncio async def test_finish_request_forwards_explicit_session(servers): @@ -1835,6 +1879,169 @@ def _mock_tokenizer(token_ids=None): return tok +def test_router_model_type_uses_checkpoint_config(tmp_path: Path) -> None: + (tmp_path / "config.json").write_text('{"model_type": "gpt_oss"}', + encoding="utf-8") + router = KvCacheAwareRouter(server_role=None, + servers=["server1"], + model_path=str(tmp_path)) + + assert router._get_model_type() == "gpt_oss" + + +@pytest.mark.asyncio +async def test_gpt_oss_router_tokens_match_chat_harmony_server_input() -> None: + """KV-cache routing must hash the same Harmony tokens used by the server.""" + from tensorrt_llm.serve.openai_server import OpenAIServer + + router = KvCacheAwareRouter(server_role=None, + servers=["server1"], + use_tokens=False, + max_batch_size=32, + tokens_per_block=32, + model_path="/models/gpt-oss-checkpoint") + router_tokenizer = _mock_tokenizer(token_ids=[900, 901, 902]) + harmony_tokens = [100, 101, 102, 103] + harmony_adapter = mock.MagicMock() + harmony_adapter.openai_to_harmony_tokens.return_value = harmony_tokens + harmony_adapter.get_stop_tokens.return_value = [42] + promise = mock.MagicMock() + promise.prompt_token_ids = [] + + request = ChatCompletionRequest( + model="my-model", + messages=[{ + "role": "developer", + "content": "Use tools when useful." + }, { + "role": "user", + "content": "weather in Paris?" + }], + tools=[_get_weather_tool()], + tool_choice="auto", + reasoning_effort="medium", + stream=True, + max_completion_tokens=1, + ) + router_request = copy.deepcopy(request) + server_request = copy.deepcopy(request) + + server = OpenAIServer.__new__(OpenAIServer) + server.allow_request_chat_template = False + server.await_disconnected = mock.AsyncMock() + server.generator = SimpleNamespace( + args=SimpleNamespace(num_postprocess_workers=0), + generate_async=mock.MagicMock(return_value=promise), + ) + server.harmony_adapter = harmony_adapter + server.model_config = SimpleNamespace(vocab_size=1000) + server.tokenizer = SimpleNamespace(tokenizer=SimpleNamespace( + vocab_size=1000)) + + with mock.patch.object( + router, "_get_tokenizer", + return_value=router_tokenizer), mock.patch( + "tensorrt_llm.serve.harmony_adapter." + "get_harmony_adapter", + return_value=harmony_adapter), mock.patch( + "tensorrt_llm.serve.router_utils." + "resolve_model_type_from_config", + return_value="gpt_oss") as resolve_model_type: + router_token_ids = router._tokenize(router_request)[0] + await server.chat_harmony(server_request, raw_request=None) + + server_token_ids = server.generator.generate_async.call_args.kwargs[ + "inputs"] + assert router_token_ids == server_token_ids + assert router_request.prompt_token_ids == harmony_tokens + first_call, second_call = harmony_adapter.openai_to_harmony_tokens.call_args_list + assert first_call.args == second_call.args + assert first_call.kwargs == second_call.kwargs + resolve_model_type.assert_called_once_with("/models/gpt-oss-checkpoint") + router_tokenizer.apply_chat_template.assert_not_called() + + +def test_gpt_oss_router_respects_disable_harmony_adapter( + monkeypatch: pytest.MonkeyPatch) -> None: + """Router follows the same DISABLE_HARMONY_ADAPTER gate as the server.""" + monkeypatch.setenv("DISABLE_HARMONY_ADAPTER", "1") + router = KvCacheAwareRouter(server_role=None, + servers=["server1"], + use_tokens=False, + max_batch_size=32, + tokens_per_block=32, + model_path="/models/gpt-oss-checkpoint") + router_tokenizer = _mock_tokenizer(token_ids=[900, 901, 902]) + harmony_adapter = mock.MagicMock() + + request = ChatCompletionRequest( + model="openai/gpt-oss-20b", + messages=[{ + "role": "user", + "content": "weather in Paris?" + }], + tools=[_get_weather_tool()], + ) + + with mock.patch.object( + router, "_get_tokenizer", + return_value=router_tokenizer), mock.patch( + "tensorrt_llm.serve.harmony_adapter." + "get_harmony_adapter", + return_value=harmony_adapter), mock.patch( + "tensorrt_llm.serve.router_utils." + "resolve_model_type_from_config", + side_effect=AssertionError( + "disabled Harmony must not load model config")): + assert router._tokenize(request) == [[900, 901, 902]] + + harmony_adapter.openai_to_harmony_tokens.assert_not_called() + router_tokenizer.apply_chat_template.assert_called_once() + + +@pytest.mark.asyncio +async def test_chat_harmony_preserves_original_tool_conversion_error() -> None: + """Harmony diagnostics must not rerun the conversion that already failed.""" + from tensorrt_llm.serve.openai_server import OpenAIServer + + original_error = RuntimeError("original tool conversion failure") + diagnostic_error = RuntimeError("diagnostic tool conversion failure") + + class FailingTool: + + def __init__(self) -> None: + self.calls = 0 + + def model_dump(self) -> dict[str, object]: + self.calls += 1 + if self.calls == 1: + raise original_error + raise diagnostic_error + + failing_tool = FailingTool() + request = ChatCompletionRequest( + model="my-model", + messages=[{ + "role": "user", + "content": "weather in Paris?" + }], + ) + object.__setattr__(request, "tools", [failing_tool]) + + server = OpenAIServer.__new__(OpenAIServer) + server.allow_request_chat_template = False + server.harmony_adapter = mock.MagicMock() + server.create_error_response = mock.MagicMock( + return_value=str(original_error)) + + response = await server.chat_harmony(request, raw_request=None) + + assert response == str(original_error) + assert failing_tool.calls == 1 + server.create_error_response.assert_called_once_with( + message=str(original_error), err_type="internal_error") + + @pytest.mark.parametrize("router_class", [KvCacheAwareRouter, ConversationRouter]) def test_tokenize_forwards_tools_and_chat_template_kwargs(router_class): @@ -1854,6 +2061,8 @@ def test_tokenize_forwards_tools_and_chat_template_kwargs(router_class): tokens_per_block=32) tok = _mock_tokenizer() + documents = [{"title": "Paris", "text": "Paris is in France."}] + chat_template = "{% for message in messages %}{{ message.content }}{% endfor %}" with mock.patch.object(router, "_get_tokenizer", return_value=tok): req = ChatCompletionRequest( model="TinyLlama", @@ -1862,6 +2071,8 @@ def test_tokenize_forwards_tools_and_chat_template_kwargs(router_class): "content": "what's the weather in Paris?" }], tools=[_get_weather_tool()], + documents=documents, + chat_template=chat_template, chat_template_kwargs={"thinking": True}, ) router._tokenize(req) @@ -1878,6 +2089,8 @@ def test_tokenize_forwards_tools_and_chat_template_kwargs(router_class): assert "parameters" in tool_dict["function"] # chat_template_kwargs must be forwarded as **kwargs (not nested). assert kwargs.get("thinking") is True + assert kwargs["documents"] == documents + assert kwargs["chat_template"] == chat_template @pytest.mark.parametrize("router_class", @@ -2051,7 +2264,7 @@ def test_prefix_cache_tokenize_matches_full_encode(servers): assert req.prompt_token_ids == reference -def test_prefix_cache_tokenize_encodes_only_delta(servers): +def test_prefix_cache_tokenize_uses_canonical_encoding(servers): router = KvCacheAwareRouter(server_role=None, servers=servers, tokens_per_block=4) @@ -2065,7 +2278,7 @@ def _recording_encode(text, add_special_tokens=False): tok.encode = _recording_encode with mock.patch.object(router, "_get_tokenizer", return_value=tok): - for index, convo in enumerate(_grow_conversation()): + for convo in _grow_conversation(): encoded_lengths.clear() req = ChatCompletionRequest(model="mock", messages=convo) rendered = tok.apply_chat_template( @@ -2074,8 +2287,7 @@ def _recording_encode(text, add_special_tokens=False): tokenize=False) router._tokenize(req) assert encoded_lengths - if index >= 1: - assert min(encoded_lengths) < len(rendered) + assert encoded_lengths == [len(rendered)] def test_prefix_cache_tokenize_falls_back_on_divergent_prefix(servers): diff --git a/tests/unittest/disaggregated/test_sync_transfer_timing.py b/tests/unittest/disaggregated/test_sync_transfer_timing.py new file mode 100644 index 000000000000..51834429dd1d --- /dev/null +++ b/tests/unittest/disaggregated/test_sync_transfer_timing.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for KvCacheTransceiverV2._sync_transfer_timing.""" + +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock, patch + +if TYPE_CHECKING: + from tensorrt_llm._torch.disaggregation.transceiver import KvCacheTransceiverV2 + + +class _FakeRequest: + """Minimal stand-in for LlmRequest with timing getters/setters.""" + + def __init__(self, rid: int, start: int, end: int, kv_size: int) -> None: + self.request_id = rid + self.py_disaggregated_params = None # forces get_unique_rid -> request_id + self._start = start + self._end = end + self._kv_cache_size = kv_size + + def get_kv_cache_transfer_start(self) -> int: + return self._start + + def get_kv_cache_transfer_end(self) -> int: + return self._end + + @property + def kv_cache_size(self) -> int: + return self._kv_cache_size + + def set_kv_cache_transfer_start(self, v: int) -> None: + self._start = v + + def set_kv_cache_transfer_end(self, v: int) -> None: + self._end = v + + def set_kv_cache_size(self, v: int) -> None: + self._kv_cache_size = v + + +def _make_transceiver(**overrides: Any) -> "KvCacheTransceiverV2": + """Create a minimal mock of KvCacheTransceiverV2 with _sync_transfer_timing.""" + from tensorrt_llm._torch.disaggregation.transceiver import KvCacheTransceiverV2 + + tc = object.__new__(KvCacheTransceiverV2) + tc._gen_need_sync = overrides.get("gen_need_sync", True) + tc._gen_allgather = overrides.get("gen_allgather", lambda x: [x]) + return tc + + +class TestSyncTransferTiming: + @patch.dict("os.environ", {"TRTLLM_KVCACHE_TIME_OUTPUT_PATH": "/tmp/test"}, clear=False) + def test_merges_correctly_two_ranks(self) -> None: + """Two simulated ranks — verify min(start), max(end), sum(size).""" + req = _FakeRequest(rid=1, start=10, end=20, kv_size=100) + + def fake_allgather(local_data): + # Simulate rank 0 = local_data, rank 1 = different timing + rank1_data = {1: (5, 25, 200)} + return [local_data, rank1_data] + + tc = _make_transceiver(gen_need_sync=True, gen_allgather=fake_allgather) + tc._sync_transfer_timing([req]) + + assert req._start == 5 # min(10, 5) + assert req._end == 25 # max(20, 25) + assert req._kv_cache_size == 300 # 100 + 200 + + @patch.dict("os.environ", {"TRTLLM_KVCACHE_TIME_OUTPUT_PATH": "/tmp/test"}, clear=False) + def test_multiple_requests_batched(self) -> None: + """Multiple requests in one allgather call.""" + req_a = _FakeRequest(rid=1, start=10, end=20, kv_size=100) + req_b = _FakeRequest(rid=2, start=30, end=40, kv_size=200) + + def fake_allgather(local_data): + rank1_data = { + 1: (8, 22, 150), + 2: (28, 45, 250), + } + return [local_data, rank1_data] + + tc = _make_transceiver(gen_need_sync=True, gen_allgather=fake_allgather) + tc._sync_transfer_timing([req_a, req_b]) + + assert req_a._start == 8 + assert req_a._end == 22 + assert req_a._kv_cache_size == 250 + + assert req_b._start == 28 + assert req_b._end == 45 + assert req_b._kv_cache_size == 450 + + @patch.dict("os.environ", {"TRTLLM_KVCACHE_TIME_OUTPUT_PATH": "/tmp/test"}, clear=False) + def test_all_ranks_updated(self) -> None: + """Every request object should be updated, not just rank-0.""" + req = _FakeRequest(rid=1, start=10, end=20, kv_size=100) + + def fake_allgather(local_data): + return [local_data, {1: (5, 25, 200)}] + + tc = _make_transceiver(gen_need_sync=True, gen_allgather=fake_allgather) + tc._sync_transfer_timing([req]) + + # All fields should reflect the merged values + assert req._start == 5 + assert req._end == 25 + assert req._kv_cache_size == 300 + + @patch.dict("os.environ", {}, clear=False) + def test_skips_when_no_env(self) -> None: + """Without TRTLLM_KVCACHE_TIME_OUTPUT_PATH, allgather is not called.""" + import os + + os.environ.pop("TRTLLM_KVCACHE_TIME_OUTPUT_PATH", None) + + allgather_mock = MagicMock() + tc = _make_transceiver(gen_need_sync=True, gen_allgather=allgather_mock) + + req = _FakeRequest(rid=1, start=10, end=20, kv_size=100) + tc._sync_transfer_timing([req]) + + allgather_mock.assert_not_called() + assert req._start == 10 # unchanged + + @patch.dict("os.environ", {"TRTLLM_KVCACHE_TIME_OUTPUT_PATH": "/tmp/test"}, clear=False) + def test_skips_when_single_rank(self) -> None: + """When _gen_need_sync is False, allgather is not called.""" + allgather_mock = MagicMock() + tc = _make_transceiver(gen_need_sync=False, gen_allgather=allgather_mock) + + req = _FakeRequest(rid=1, start=10, end=20, kv_size=100) + tc._sync_transfer_timing([req]) + + allgather_mock.assert_not_called() + assert req._start == 10 # unchanged + + @patch.dict("os.environ", {"TRTLLM_KVCACHE_TIME_OUTPUT_PATH": "/tmp/test"}, clear=False) + def test_empty_list(self) -> None: + """Empty request list should return immediately.""" + allgather_mock = MagicMock() + tc = _make_transceiver(gen_need_sync=True, gen_allgather=allgather_mock) + + tc._sync_transfer_timing([]) + + allgather_mock.assert_not_called() diff --git a/tests/unittest/disaggregated/test_transceiver_bounded_polling.py b/tests/unittest/disaggregated/test_transceiver_bounded_polling.py index cfad765968e2..76eff6adc439 100644 --- a/tests/unittest/disaggregated/test_transceiver_bounded_polling.py +++ b/tests/unittest/disaggregated/test_transceiver_bounded_polling.py @@ -20,6 +20,8 @@ from typing import Optional from unittest.mock import Mock +import pytest + from tensorrt_llm._torch.disaggregation.base.transfer import SessionStatus, WaitResult from tensorrt_llm._torch.disaggregation.native.transfer import TaskStatus, TxSession from tensorrt_llm._torch.disaggregation.transceiver import KvCacheTransceiverV2 @@ -135,9 +137,20 @@ def _make_tx_session( return session -def test_context_transfer_status_bounded_poll_keeps_not_ready_session_queued() -> None: +def test_context_transfer_status_bounded_poll_keeps_not_ready_session_queued( + monkeypatch, +) -> None: session = _FakeSession(rid=11, wait_result=None) transceiver = _make_transceiver({11: session}) + monotonic = Mock(side_effect=[0.0, 0.0, 0.123]) + monkeypatch.setattr( + "tensorrt_llm._torch.disaggregation.transceiver.time.monotonic", + monotonic, + ) + monkeypatch.setattr( + "tensorrt_llm._torch.disaggregation.transceiver.time.sleep", + Mock(), + ) completed, failed = transceiver.check_context_transfer_status(at_least_request_num=1) @@ -150,6 +163,38 @@ def test_context_transfer_status_bounded_poll_keeps_not_ready_session_queued() - assert transceiver._transfer_worker.sweep_count == 1 +def test_context_transfer_status_bounded_poll_reaps_completion(monkeypatch) -> None: + session = _FakeSession(rid=14, wait_result=WaitResult.COMPLETED) + req = _FakeRequest() + transceiver = _make_transceiver({14: session}, {14: req}) + + def complete_on_poll(blocking: bool = True) -> WaitResult: + session.blocking_calls.append(blocking) + session._is_completed = True + return WaitResult.COMPLETED + + session.wait_complete = complete_on_poll + sleep = Mock() + monkeypatch.setattr( + "tensorrt_llm._torch.disaggregation.transceiver.time.monotonic", + Mock(return_value=0.0), + ) + monkeypatch.setattr( + "tensorrt_llm._torch.disaggregation.transceiver.time.sleep", + sleep, + ) + + completed, failed = transceiver.check_context_transfer_status(at_least_request_num=1) + + assert completed == [14] + assert failed == [] + assert session.blocking_calls == [False, False] + sleep.assert_called_once_with(0.001) + assert session.closed + assert 14 not in transceiver._send_sessions + assert 14 not in transceiver._send_reqs + + def test_context_transfer_status_block_all_uses_blocking_wait() -> None: session = _FakeSession(rid=12, wait_result=WaitResult.COMPLETED) req = _FakeRequest() @@ -269,6 +314,48 @@ def fake_allgather(payload): assert new_completed == [7] # intersection only (8 is completed on the peer only) +@pytest.mark.skip( + reason="ctx idle fast-path was dropped from this branch. TODO: when the " + "fast-path is reintroduced, its terminal-count reduction must mirror " + "_ctx_consensus()'s communicator scope (TP group, then PP group; TP " + "skipped under attention DP) — a WORLD-scoped allreduce hangs under " + "ADP+PP because independent attention-DP lanes poll on their own " + "schedules. Re-enable this test and add scoped mock coverage for the " + "TP+PP and ADP+PP configurations plus real-collective MP tests." +) +def test_ctx_consensus_fastpath_skips_when_idle(monkeypatch) -> None: + # With the fast-path enabled, an all-zero terminal count (one fixed-size + # allreduce) makes every rank skip the variable-length consensus; a non-zero + # count falls through to the normal consensus path. + monkeypatch.setattr( + "tensorrt_llm._torch.disaggregation.transceiver._CTX_CONSENSUS_FASTPATH", True + ) + transceiver = object.__new__(KvCacheTransceiverV2) + transceiver._ever_had_send_session = True + transceiver._ctx_need_tp_sync = True + transceiver._ctx_need_pp_sync = False + transceiver._send_sessions = {} + transceiver._send_reqs = {} + transceiver._dist = Mock() + transceiver._dist.allreduce = Mock(return_value=0) + transceiver._ctx_consensus = Mock(return_value=[]) + transceiver._build_to_process = Mock(return_value=[]) + transceiver._ctx_consensus_outcome = Mock(return_value=([], [], [], [])) + transceiver._transfer_worker = _FakeTransferWorker() + transceiver._close_failed_sessions = Mock() + + completed, failed = transceiver.check_context_transfer_status(at_least_request_num=0) + + assert completed == [] and failed == [] + transceiver._dist.allreduce.assert_called_once() + transceiver._ctx_consensus.assert_not_called() # idle fast-path skipped the consensus + + # Non-zero global terminal count => fast-path does not skip; consensus runs. + transceiver._dist.allreduce = Mock(return_value=2) + transceiver.check_context_transfer_status(at_least_request_num=0) + transceiver._ctx_consensus.assert_called_once() + + def test_tx_session_wait_complete_defaults_to_blocking() -> None: task = _FakeTask(TaskStatus.INIT, wait_result=False) session = _make_tx_session([task]) diff --git a/tests/unittest/dynamo/test_imports.py b/tests/unittest/dynamo/test_imports.py index 981bf90f9df5..8e29ff34127a 100644 --- a/tests/unittest/dynamo/test_imports.py +++ b/tests/unittest/dynamo/test_imports.py @@ -51,6 +51,7 @@ ("tensorrt_llm.tokenizer", "TOKENIZER_ALIASES"), ("tensorrt_llm.llmapi.llm_args", "KvCacheConnectorConfig"), ("tensorrt_llm.llmapi.llm_args", "TorchLlmArgs"), + ("tensorrt_llm.llmapi.llm_args", "BaseLlmArgs"), ("tensorrt_llm.llmapi.llm_utils", "update_llm_args_with_extra_options"), ("tensorrt_llm.llmapi.tokenizer", "tokenizer_factory"), ("tensorrt_llm.llmapi.disagg_utils", "get_global_disagg_request_id"), diff --git a/tests/unittest/executor/test_base_worker.py b/tests/unittest/executor/test_base_worker.py index 3c5b54b96569..7bee69ef87f9 100644 --- a/tests/unittest/executor/test_base_worker.py +++ b/tests/unittest/executor/test_base_worker.py @@ -35,6 +35,9 @@ def raise_load_error(lora_request): raise RuntimeError("bad adapter") worker = object.__new__(BaseWorker) + # GC-time __del__ -> shutdown() reads this; __init__ is bypassed here, so + # seed it to keep teardown a clean no-op. + worker.doing_shutdown = False worker._lora_manager = LoraManager() worker._load_lora_adapter = raise_load_error request = type( @@ -176,7 +179,9 @@ def setup_method(self): self.session = self.create_worker_session() def create_worker_session(self): - session = MpiPoolSession(n_workers=2) + # wait_shutdown: block shutdown until the workers exited, so a test + # handed a live pool right after this one cannot race the GPU release. + session = MpiPoolSession(n_workers=2, wait_shutdown=True) return session @pytest.mark.gpu2 diff --git a/tests/unittest/executor/test_event_loop_error_broadcast.py b/tests/unittest/executor/test_event_loop_error_broadcast.py index 04851b8e651b..df8db031cd9e 100644 --- a/tests/unittest/executor/test_event_loop_error_broadcast.py +++ b/tests/unittest/executor/test_event_loop_error_broadcast.py @@ -13,9 +13,16 @@ import datetime import queue as _stdlib_queue +import pytest + from tensorrt_llm.executor.base_worker import AwaitResponseHelper from tensorrt_llm.executor.utils import ErrorResponse +# CI's CPU stages select tests with ``-m "cpu_only and not disabled"``; without +# this marker the whole file is deselected and pytest exits 5 (no tests ran), +# which the runner reports as a failure. These are pure stub-based unit tests. +pytestmark = pytest.mark.cpu_only + class _EngineStub: """Stub for self.worker.engine: returns whatever the test plugged in.""" @@ -56,6 +63,9 @@ def __init__(self, engine, num_pending: int = 1): self.popped = [] self.result_queue = None self.postproc_queues = None + # responses_handler() reads this unguarded (base_worker.py); BaseWorker + # sets it in __init__, which this stub bypasses, so seed it to None. + self.frontend_result_queues = None # Echoed straight back so __call__'s filter is a no-op. def _engine_response_callback(r): diff --git a/tests/unittest/executor/test_multi_frontend_routing.py b/tests/unittest/executor/test_multi_frontend_routing.py new file mode 100644 index 000000000000..d34aa62d8dde --- /dev/null +++ b/tests/unittest/executor/test_multi_frontend_routing.py @@ -0,0 +1,302 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CPU-only tests for classic-path multi-frontend serving. + +Multi-frontend serving (num_serve_frontends) on the classic IPC +executor path. Covers client-id namespacing, worker-side response-lane routing, and the +attached-frontend proxy lifecycle over real ipc:// sockets. +""" + +import os +import tempfile +import time +from types import SimpleNamespace + +import pytest + +from tensorrt_llm.executor.utils import ( + FRONTEND_COUNTER_MASK, + MAX_NUM_FRONTENDS, + bucket_responses_by_frontend, + frontend_lane_index, + get_frontend_id, + namespace_client_id, +) + +# The CI CPU stages collect with -m "cpu_only and not disabled" and skip +# files that don't mention pytest.mark.cpu_only (see unittest/conftest.py). +pytestmark = pytest.mark.cpu_only + + +class TestClientIdNamespacing: + def test_frontend_zero_keeps_legacy_ids(self): + for client_id in (1, 42, FRONTEND_COUNTER_MASK): + assert namespace_client_id(0, client_id) == client_id + + def test_roundtrip(self): + for frontend_id in (0, 1, 7, MAX_NUM_FRONTENDS - 1): + for counter in (1, 12345, FRONTEND_COUNTER_MASK): + client_id = namespace_client_id(frontend_id, counter) + assert get_frontend_id(client_id) == frontend_id + assert client_id & FRONTEND_COUNTER_MASK == counter + # Bit 63 stays clear: ids remain positive as signed int64. + assert 0 < client_id < (1 << 63) + + def test_knob_cap_matches_encoding(self): + from tensorrt_llm.llmapi.llm_args import BaseLlmArgs + + meta = BaseLlmArgs.model_fields["num_serve_frontends"].metadata + assert any(getattr(m, "le", None) == MAX_NUM_FRONTENDS for m in meta) + + def test_counter_wraparound_stays_in_namespace(self): + # A counter overflowing its field must not leak into the frontend bits. + client_id = namespace_client_id(3, FRONTEND_COUNTER_MASK + 5) + assert get_frontend_id(client_id) == 3 + assert client_id & FRONTEND_COUNTER_MASK == 4 + + def test_non_int_client_id_routes_to_launcher(self): + assert get_frontend_id(None) == 0 + + def test_lane_index_clamps_to_launcher(self): + assert frontend_lane_index(namespace_client_id(2, 7), 3) == 2 + assert frontend_lane_index(namespace_client_id(7, 1), 2) == 0 + assert frontend_lane_index(None, 4) == 0 + + +def _response(client_id): + return SimpleNamespace(client_id=client_id) + + +class TestClassicResponseBucketing: + """The classic IPC path routes batches with bucket_responses_by_frontend.""" + + def test_buckets_by_namespace(self): + responses = [ + _response(namespace_client_id(0, 1)), + _response(namespace_client_id(1, 1)), + _response(namespace_client_id(1, 2)), + _response(namespace_client_id(2, 1)), + ] + buckets = bucket_responses_by_frontend(responses, 3) + assert [len(b) for b in buckets] == [1, 2, 1] + assert all(get_frontend_id(r.client_id) == 1 for r in buckets[1]) + + def test_none_client_id_routes_to_launcher(self): + # ADP dummy responses carry client_id=None: they must land in the + # launcher's lane (which silently discards them), not raise. + buckets = bucket_responses_by_frontend([_response(None)], 4) + assert len(buckets[0]) == 1 + assert all(not b for b in buckets[1:]) + + def test_out_of_range_frontend_routes_to_launcher(self): + buckets = bucket_responses_by_frontend([_response(namespace_client_id(7, 1))], 2) + assert len(buckets[0]) == 1 and not buckets[1] + + +class _LaneStub: + def __init__(self): + self.items = [] + + def put(self, obj): + self.items.append(obj) + + +class TestClassicSendRspLaneRouting: + """_send_rsp selects the origin frontend's lane on the non-postproc path.""" + + @staticmethod + def _fake_worker(num_lanes): + pops = [] + return SimpleNamespace( + result_queue=None, + postproc_queues=None, + frontend_result_queues=[_LaneStub() for _ in range(num_lanes)], + _pop_result=pops.append, + ), pops + + def test_error_response_routes_to_origin_lane(self): + from tensorrt_llm.executor.base_worker import _send_rsp + from tensorrt_llm.executor.utils import ErrorResponse + + worker, pops = self._fake_worker(3) + client_id = namespace_client_id(2, 7) + _send_rsp(worker, ErrorResponse(client_id, "boom", 1)) + assert [len(q.items) for q in worker.frontend_result_queues] == [0, 0, 1] + assert pops == [client_id] + + def test_none_client_id_routes_to_launcher_lane(self): + from tensorrt_llm.executor.base_worker import _send_rsp + from tensorrt_llm.executor.utils import ErrorResponse + + worker, _ = self._fake_worker(2) + _send_rsp(worker, ErrorResponse(None, "adp dummy", 1)) + assert len(worker.frontend_result_queues[0].items) == 1 + assert not worker.frontend_result_queues[1].items + + def test_rsp_batch_defers_lane_selection(self): + from tensorrt_llm.executor.base_worker import _send_rsp + from tensorrt_llm.executor.utils import ErrorResponse + + worker, _ = self._fake_worker(2) + rsp_batch = [] + _send_rsp(worker, ErrorResponse(namespace_client_id(1, 3), "x", 1), rsp_batch=rsp_batch) + assert len(rsp_batch) == 1 + assert all(not q.items for q in worker.frontend_result_queues) + + +class TestClassicFrontendProxyEndToEnd: + """GenerationExecutorFrontendProxy against a fake rank0 worker. + + Real ipc:// sockets: namespaced submit, cancel-on-shutdown, and -- + critically -- that an attached frontend NEVER emits the None + engine-shutdown sentinel. + """ + + @staticmethod + def _make_proxy_and_fake_worker(tmpdir, frontend_id=1, num_frontends=2): + import zmq + + from tensorrt_llm.executor.ipc import IpcQueue + from tensorrt_llm.executor.proxy import GenerationExecutorFrontendProxy + + hmac_key = os.urandom(32) + request_addr = f"ipc://{os.path.join(tmpdir, 'request.sock')}" + result_addrs = [ + f"ipc://{os.path.join(tmpdir, f'result_{i}.sock')}" for i in range(num_frontends) + ] + # The fake rank0 worker binds the request ingress (PULL), exactly + # like worker_main does in multi-frontend mode. + worker_ingress = IpcQueue( + (request_addr, hmac_key), + is_server=True, + socket_type=zmq.PULL, + name="fake_worker_request_queue", + ) + proxy = GenerationExecutorFrontendProxy( + { + "mode": "classic", + "request_addr": request_addr, + "result_addrs": result_addrs, + "hmac_key": hmac_key.hex(), + }, + frontend_id=frontend_id, + ) + return proxy, worker_ingress, hmac_key, result_addrs + + @staticmethod + def _stop_dispatch_thread(proxy, result_addrs, hmac_key): + """End the dispatch thread the way the worker does at engine teardown. + + The worker fans a per-lane None sentinel to every frontend (see + notify_proxy_threads_to_quit). Frontend shutdown deliberately leaves + the thread to process teardown, which the threadleak checker would + report as a leak. + """ + import zmq + + from tensorrt_llm.executor.ipc import FusedIpcQueue + + worker_lane = FusedIpcQueue( + (result_addrs[proxy._frontend_id], hmac_key), + is_server=False, + fuse_message=False, + socket_type=zmq.PUSH, + name="fake_worker_shutdown_lane", + ) + worker_lane.put(None) + proxy.dispatch_result_thread.join(timeout=10) + assert not proxy.dispatch_result_thread.is_alive() + + def test_submit_namespaces_and_shutdown_never_sends_sentinel(self): + from tensorrt_llm.executor.request import CancellingRequest, GenerationRequest + from tensorrt_llm.sampling_params import SamplingParams + + with tempfile.TemporaryDirectory() as tmpdir: + proxy, worker_ingress, hmac_key, result_addrs = self._make_proxy_and_fake_worker(tmpdir) + + # Attributes read by OpenAIServer at init must exist (a missing + # _resource_governor_queue crashed all siblings in the first e2e). + assert proxy.resource_governor_queue is None + + result = proxy.submit(GenerationRequest([1, 2, 3], SamplingParams())) + assert get_frontend_id(result.request_id) == 1 + assert worker_ingress.poll(5) + received = worker_ingress.get() + assert isinstance(received, GenerationRequest) + assert received.id == result.request_id + + # Frontend shutdown aborts its in-flight requests (cancel) but + # must NOT emit the None engine-shutdown sentinel. + proxy.shutdown() + assert worker_ingress.poll(5) + cancel = worker_ingress.get() + assert isinstance(cancel, CancellingRequest) + assert cancel.id == result.request_id + assert not worker_ingress.poll(1), ( + "an attached frontend must never send the engine-shutdown sentinel" + ) + self._stop_dispatch_thread(proxy, result_addrs, hmac_key) + + def test_check_health_and_submit_reflect_fatal_error(self): + from tensorrt_llm.executor.request import GenerationRequest + from tensorrt_llm.executor.utils import EngineDeadError + from tensorrt_llm.sampling_params import SamplingParams + + with tempfile.TemporaryDirectory() as tmpdir: + proxy, _, _, _ = self._make_proxy_and_fake_worker(tmpdir) + + # Regression: this state lives in GenerationExecutorProxy.__init__, + # which an attached frontend deliberately skips, so it must be + # initialized explicitly (a missing _engine_dead broke the first + # submit(); a missing _worker_process_monitor broke /health). + assert proxy.check_health() + assert proxy.model_world_size == 1 + + proxy._set_fatal_error(RuntimeError("rank0 worker died")) + assert not proxy.check_health() + with pytest.raises(EngineDeadError): + proxy.submit(GenerationRequest([1], SamplingParams())) + + def test_dispatch_routes_own_lane_responses(self): + import zmq + + from tensorrt_llm.executor.ipc import FusedIpcQueue + from tensorrt_llm.executor.request import GenerationRequest + from tensorrt_llm.executor.utils import ErrorResponse + from tensorrt_llm.sampling_params import SamplingParams + + with tempfile.TemporaryDirectory() as tmpdir: + proxy, _, hmac_key, result_addrs = self._make_proxy_and_fake_worker(tmpdir) + result = proxy.submit(GenerationRequest([1, 2, 3], SamplingParams())) + client_id = result.request_id + assert client_id in proxy._results + + # The fake worker pushes an ErrorResponse down this frontend's + # result lane; the dispatcher must deliver it and retire the + # request. + worker_lane = FusedIpcQueue( + (result_addrs[1], hmac_key), + is_server=False, + fuse_message=False, + socket_type=zmq.PUSH, + name="fake_worker_result_lane", + ) + worker_lane.put(ErrorResponse(client_id, "boom", 1)) + deadline = time.time() + 5 + while client_id in proxy._results and time.time() < deadline: + time.sleep(0.01) + assert client_id not in proxy._results + + self._stop_dispatch_thread(proxy, result_addrs, hmac_key) diff --git a/tests/unittest/executor/test_proxy_fast_death.py b/tests/unittest/executor/test_proxy_fast_death.py index ed0e62ae063a..c72ed8c21b1a 100644 --- a/tests/unittest/executor/test_proxy_fast_death.py +++ b/tests/unittest/executor/test_proxy_fast_death.py @@ -16,10 +16,15 @@ import asyncio import queue as _queue +import time as _time +from concurrent.futures import Future as _Future +from unittest.mock import Mock +from unittest.mock import Mock as _Mock import pytest from tensorrt_llm.executor import EngineDeadError +from tensorrt_llm.executor import proxy as proxy_module from tensorrt_llm.executor.proxy import GenerationExecutorProxy from tensorrt_llm.executor.result import GenerationResult @@ -48,6 +53,7 @@ def _bare_proxy(): proxy._results = {} # Set so the __del__ -> shutdown() path is a clean no-op at GC time. proxy.workers_started = False + proxy._multi_frontend_ipc_dir = None return proxy @@ -97,6 +103,20 @@ def test_handle_worker_death_broadcasts_event_driven(): assert proxy._error_queue.get_nowait() is cause +def test_register_worker_processes_with_session_reuse_factory(monkeypatch): + """Session reuse replaces proxy.MpiPoolSession with a factory function.""" + pool_session = object() + monkeypatch.setattr(proxy_module, "MpiPoolSession", lambda n_workers: pool_session) + proxy = _bare_proxy() + proxy.mpi_session = proxy_module.MpiPoolSession(1) + proxy._worker_process_monitor = Mock() + identities = [object()] + + proxy._register_worker_processes((proxy.READY_SIGNAL, None, identities)) + + proxy._worker_process_monitor.register.assert_called_once_with(identities) + + def test_result_step_raises_on_engine_dead(): res = GenerationResult.__new__(GenerationResult) res.queue = _queue.Queue() @@ -322,3 +342,199 @@ def test_proxy_check_remote_worker_death_marks_engine_dead(): proxy2 = _bare_proxy() proxy2.mpi_session = object() assert proxy2._check_remote_worker_death() is False + + +# --- Non-blocking teardown on a dead engine --- +# An abruptly-killed worker world never completes its mpi4py futures and +# never sends the result-queue shutdown sentinel, so an unbounded shutdown() +# blocks forever on f.result(), the dispatcher join, and the pool join. +# These tests pin the bounded-teardown behavior. + + +def _teardown_proxy(engine_dead): + proxy = _bare_proxy() + proxy.workers_started = True + proxy.doing_shutdown = True # skip pre_shutdown(); teardown path only + proxy._engine_dead = engine_dead + proxy._fatal_error = RuntimeError("worker died") if engine_dead else None + proxy.dispatch_result_thread = None + proxy.rpc_client = None + proxy.request_queue = _Mock() + proxy.worker_init_status_queue = _Mock() + proxy.result_queue = _Mock() + proxy._resource_governor_queue = None + proxy._owns_mpi_session = True + proxy.mpi_session = _Mock() + proxy._handle_background_error = lambda *a, **k: None + return proxy + + +def test_shutdown_does_not_block_on_dead_engine(): + """With the engine dead, never-completing futures must not hang teardown.""" + proxy = _teardown_proxy(engine_dead=True) + pending = _Future() # never completes: abrupt worker death + done = _Future() + done.set_exception(RuntimeError("captured by mpi_done_callback already")) + proxy.mpi_futures = [pending, done] + + dispatcher = _Mock() + dispatcher.is_alive.return_value = True + proxy.dispatch_result_thread = dispatcher + + start = _time.monotonic() + proxy.shutdown() + elapsed = _time.monotonic() - start + + # One collective 5 s grace, not an unbounded f.result() per future. + assert elapsed < 30 + assert not pending.done() + # The dispatcher join is bounded (daemon thread is leaked, not awaited). + dispatcher.stop.assert_called_once() + dispatcher.join.assert_called_once_with(timeout=5.0) + # The dead pool is not joined; the session is abandoned instead. + proxy.mpi_session.abandon.assert_called_once_with() + proxy.mpi_session.shutdown.assert_not_called() + assert proxy.workers_started is False + + +def test_shutdown_keeps_blocking_semantics_when_engine_alive(): + """Orderly shutdown is unchanged: futures reaped, session joined.""" + proxy = _teardown_proxy(engine_dead=False) + done = _Future() + done.set_result(None) + proxy.mpi_futures = [done] + + dispatcher = _Mock() + dispatcher.is_alive.return_value = True + proxy.dispatch_result_thread = dispatcher + + proxy.shutdown() + + dispatcher.join.assert_called_once_with(timeout=None) + proxy.mpi_session.shutdown.assert_called_once_with() + proxy.mpi_session.abandon.assert_not_called() + assert proxy.workers_started is False + + +def test_shutdown_does_not_shut_down_external_session(): + """An externally owned session must stay alive even on a dead engine.""" + proxy = _teardown_proxy(engine_dead=True) + proxy._owns_mpi_session = False + proxy.mpi_futures = [] + + proxy.shutdown() + + proxy.mpi_session.shutdown.assert_not_called() + proxy.mpi_session.abandon.assert_not_called() + + +def test_abandon_mpi_pool_threads_unblocks_interpreter_exit(): + """Both exit-join mechanisms release a wedged pool manager thread. + + The thread is deregistered from mpi4py's THREADS_QUEUES and CPython's + _shutdown_locks, so process exit can proceed without joining it. + """ + import sys as _sys + import threading as _threading + import types as _types + + from tensorrt_llm.llmapi.mpi_session import _abandon_mpi_pool_threads + + release = _threading.Event() + wedged = _threading.Thread(target=release.wait, name="fake_manager") + wedged.daemon = False + wedged.start() + try: + # Fake mpi4py registry module, as mpi4py would have registered it. + fake_mod = _types.ModuleType("mpi4py.futures._lib") + fake_mod.THREADS_QUEUES = {wedged: object()} + prev = _sys.modules.get("mpi4py.futures._lib") + _sys.modules["mpi4py.futures._lib"] = fake_mod + try: + fake_pool = _Mock() + fake_pool._pool.thread = wedged + + _abandon_mpi_pool_threads(fake_pool) + + assert wedged not in fake_mod.THREADS_QUEUES + shutdown_locks = getattr(_threading, "_shutdown_locks", None) + if shutdown_locks is not None: # CPython 3.9-3.12 + assert wedged._tstate_lock not in shutdown_locks + finally: + if prev is None: + del _sys.modules["mpi4py.futures._lib"] + else: + _sys.modules["mpi4py.futures._lib"] = prev + finally: + release.set() + wedged.join(timeout=5) + + +def test_abandon_mpi_pool_threads_tolerates_missing_pool(): + from tensorrt_llm.llmapi.mpi_session import _abandon_mpi_pool_threads + + _abandon_mpi_pool_threads(None) + _abandon_mpi_pool_threads(object()) # no _pool attribute + + +def test_mark_engine_dead_releases_exit_joins_immediately(): + """Exit-join release must happen at detection time, not at teardown. + + CPython's exit sequence joins non-daemon threads before atexit/GC can + run shutdown(), so a wedged pool manager thread must be deregistered + the moment the engine is marked dead. + """ + proxy = _bare_proxy() + proxy.mpi_session = _Mock() + + proxy._mark_engine_dead(RuntimeError("worker died")) + proxy.mpi_session.release_exit_joins.assert_called_once_with() + + # Sticky: a second death report must not re-release. + proxy._mark_engine_dead(RuntimeError("again")) + proxy.mpi_session.release_exit_joins.assert_called_once_with() + + +def test_mark_engine_dead_releases_external_sessions_too(): + """Ownership must not gate the exit-join release. + + The LLM API creates the session and passes it in, so the proxy does + not own it; the release is non-destructive bookkeeping and must still + happen. + """ + proxy = _bare_proxy() + proxy._owns_mpi_session = False + proxy.mpi_session = _Mock() + + proxy._mark_engine_dead(RuntimeError("worker died")) + proxy.mpi_session.release_exit_joins.assert_called_once_with() + # But destructive teardown is still reserved for the owner. + proxy.mpi_session.abandon.assert_not_called() + proxy.mpi_session.shutdown.assert_not_called() + + +def test_pool_session_shutdown_never_blocks_after_release(): + """A released session never blocks, even on an explicit shutdown(). + + After release_exit_joins(), a blocking shutdown() from the session + owner must not join the dead pool. + """ + from tensorrt_llm.llmapi.mpi_session import MpiPoolSession + + session = MpiPoolSession.__new__(MpiPoolSession) + pool = _Mock() + pool._pool.thread = None # no real manager thread to deregister + session.mpi_pool = pool + # __init__ is bypassed above; seed the attributes that shutdown() reads + # so the test exercises the release/shutdown contract, not attribute + # lookup on a raw-constructed session. Keep them minimal: + # * n_workers is only used by shutdown()'s log line. + # * _wait_shutdown gates the post-shutdown worker-exit barrier; + # the test targets the blocking-join guard, so leave it off. + session.n_workers = 1 + session._wait_shutdown = False + + session.release_exit_joins() + session.shutdown() # owner asks for the default blocking shutdown + + pool.shutdown.assert_called_once_with(wait=False) diff --git a/tests/unittest/executor/test_proxy_postproc_terminate.py b/tests/unittest/executor/test_proxy_postproc_terminate.py index 00b6883c8734..0043360a7a32 100644 --- a/tests/unittest/executor/test_proxy_postproc_terminate.py +++ b/tests/unittest/executor/test_proxy_postproc_terminate.py @@ -38,7 +38,12 @@ def put(self, res): def _make_proxy(): # Avoid GenerationExecutorProxy.__init__ (it spawns workers); we only # exercise the pure dispatch logic with hand-set attributes. - return object.__new__(GenerationExecutorProxy) + proxy = object.__new__(GenerationExecutorProxy) + # GC-time __del__ -> shutdown() reads these; __init__ is bypassed here, so + # seed them to keep teardown a clean no-op. + proxy.workers_started = False + proxy._multi_frontend_ipc_dir = None + return proxy def test_late_response_after_terminate_is_dropped_without_keyerror(): diff --git a/tests/unittest/executor/test_stats_serializer.py b/tests/unittest/executor/test_stats_serializer.py index bb061e4ec80d..b051947966ae 100644 --- a/tests/unittest/executor/test_stats_serializer.py +++ b/tests/unittest/executor/test_stats_serializer.py @@ -24,6 +24,8 @@ KVCacheV2IterationStatsReport, KVCacheV2LifeCycleIterationStats, KVCacheV2PoolGroupIterationStats, + KVCacheV2SsmLifeCycleIterationStats, + KVCacheV2SsmSnapshotIterationStats, ) from tensorrt_llm.executor.base_worker import BaseWorker @@ -387,7 +389,20 @@ def test_serializer_with_v2_pool_group_stats(self): window_size=16, kind="attention", stats=life_cycle_stats, - ) + ), + 4: KVCacheV2SsmLifeCycleIterationStats( + life_cycle_id=4, + pool_group_id=8, + snapshot_stats=KVCacheV2SsmSnapshotIterationStats( + iter_snapshot_lookups=4, + iter_snapshot_hits=3, + iter_snapshot_misses=1, + iter_reused_tokens=96, + iter_unreused_tokens=32, + iter_aligned_snapshot_hits=2, + iter_unaligned_snapshot_hits=1, + ), + ), }, ) @@ -425,6 +440,23 @@ def test_serializer_with_v2_pool_group_stats(self): assert life_cycle["iterReusedBlocks"] == 5 assert life_cycle["iterMissedBlocks"] == 3 assert "iterGenAllocBlocks" not in life_cycle + ssm_life_cycle = d["kvCacheIterationStatsByLifecycle"]["4"] + assert ssm_life_cycle == { + "lifeCycleId": 4, + "poolGroupId": 8, + "windowSize": None, + "kind": "ssm", + "snapshotStats": { + "iterSnapshotLookups": 4, + "iterSnapshotHits": 3, + "iterSnapshotMisses": 1, + "iterSnapshotHitRate": 0.75, + "iterReusedTokens": 96, + "iterUnreusedTokens": 32, + "iterAlignedSnapshotHits": 2, + "iterUnalignedSnapshotHits": 1, + }, + } def test_v2_peak_block_stats_reset_tracks_interval_peak(self): """Peak block stats should cover the interval since the previous reset.""" diff --git a/tests/unittest/inputs/test_chat_template_dispatch.py b/tests/unittest/inputs/test_chat_template_dispatch.py index 3a20271d2f19..16115ac336ef 100644 --- a/tests/unittest/inputs/test_chat_template_dispatch.py +++ b/tests/unittest/inputs/test_chat_template_dispatch.py @@ -480,7 +480,7 @@ async def fake_mm_coroutine(): monkeypatch.setattr( rg, "parse_chat_messages_coroutines", - lambda messages, model_config, _: ([], fake_mm_coroutine(), [{}]), + lambda messages, model_config, _: ([], fake_mm_coroutine(), [{}], None), ) # Must resolve the top-level model type, matching the serving call # sites (not the raw model_config.model_type). @@ -526,7 +526,7 @@ async def fake_mm_coroutine(): monkeypatch.setattr( ru, "parse_chat_messages_coroutines", - lambda messages, model_config: ([], fake_mm_coroutine(), [{}]), + lambda messages, model_config: ([], fake_mm_coroutine(), [{}], None), ) monkeypatch.setattr(ru, "resolve_top_level_model_type", lambda cfg: "resolved-model-type") monkeypatch.setattr(ru, "_get_chat_completion_function_tools", lambda tools: []) diff --git a/tests/unittest/inputs/test_video_decode.py b/tests/unittest/inputs/test_video_decode.py new file mode 100644 index 000000000000..fc900f53f380 --- /dev/null +++ b/tests/unittest/inputs/test_video_decode.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Contract tests for `_load_video_by_cv2` return shapes and the HF passthrough.""" + +from __future__ import annotations + +import numpy as np +import pytest +import torch +from PIL import Image +from transformers.video_utils import make_batched_videos + +pytest.importorskip("cv2") +import cv2 # noqa: E402 + +from tensorrt_llm.inputs.media_io import _load_video_by_cv2 # noqa: E402 + + +@pytest.fixture(scope="module") +def sample_video_path(tmp_path_factory: pytest.TempPathFactory) -> str: + """Encode a tiny mp4 with distinguishable per-frame pixel values.""" + width, height, num_frames = 64, 64, 20 + path = tmp_path_factory.mktemp("video_decode") / "sample.mp4" + writer = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), 30, (width, height)) + for i in range(num_frames): + writer.write(np.full((height, width, 3), (i * 10) % 256, dtype=np.uint8)) + writer.release() + return str(path) + + +def test_np_format_returns_stacked_uint8_ndarray(sample_video_path: str) -> None: + video = _load_video_by_cv2(sample_video_path, num_frames=10, fps=-1, format="np") + assert isinstance(video.frames, np.ndarray) + assert video.frames.shape == (10, 64, 64, 3) + assert video.frames.dtype == np.uint8 + assert video.frames.flags["C_CONTIGUOUS"] + + +def test_pt_format_returns_list_of_chw_tensors(sample_video_path: str) -> None: + video = _load_video_by_cv2(sample_video_path, num_frames=10, fps=-1, format="pt") + assert isinstance(video.frames, list) + assert len(video.frames) == 10 + for frame in video.frames: + assert isinstance(frame, torch.Tensor) + assert frame.shape == (3, 64, 64) + assert frame.dtype == torch.float32 + assert 0.0 <= frame.min().item() and frame.max().item() <= 1.0 + + +def test_pil_format_returns_list_of_pil_images(sample_video_path: str) -> None: + video = _load_video_by_cv2(sample_video_path, num_frames=10, fps=-1, format="pil") + assert isinstance(video.frames, list) + assert len(video.frames) == 10 + for frame in video.frames: + assert isinstance(frame, Image.Image) + assert frame.size == (64, 64) + + +def test_np_format_hits_hf_video_processor_fast_path(sample_video_path: str) -> None: + """HF `make_batched_videos` returns a 4D ndarray input without copying.""" + video = _load_video_by_cv2(sample_video_path, num_frames=10, fps=-1, format="np") + batched = make_batched_videos([video.frames]) + + assert len(batched) == 1 + assert np.shares_memory(video.frames, batched[0]) diff --git a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_event_manager.py b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_event_manager.py index db5921867760..c3d88dc0433b 100644 --- a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_event_manager.py +++ b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_event_manager.py @@ -15,6 +15,7 @@ import gc import os +import pickle import threading import time from importlib.util import find_spec @@ -28,10 +29,28 @@ KV_CACHE_HASH_ALGO_V2_SHA256_64, truncate_sha256_hash_to_int64, ) +from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheCreatedData as NativeKVCacheCreatedData +from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheEvent as NativeKVCacheEvent +from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheEventDiff as NativeKVCacheEventDiff +from tensorrt_llm.runtime.kv_cache_manager_v2 import ( + KVCacheEventManager as NativeKVCacheEventManager, +) +from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheRemovedData as NativeKVCacheRemovedData +from tensorrt_llm.runtime.kv_cache_manager_v2 import ( + KVCacheStoredBlockData as NativeKVCacheStoredBlockData, +) +from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheStoredData as NativeKVCacheStoredData +from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheUpdatedData as NativeKVCacheUpdatedData +from tensorrt_llm.runtime.kv_cache_manager_v2 import UniqueToken as NativeUniqueToken from tensorrt_llm.runtime.kv_cache_manager_v2._event_manager import ( + KVCacheCreatedData, + KVCacheEvent, KVCacheEventDiff, KVCacheEventManager, + KVCacheRemovedData, KVCacheStoredBlockData, + KVCacheStoredData, + KVCacheUpdatedData, UniqueToken, ) @@ -64,6 +83,7 @@ _DEFAULT_CACHE_LEVEL = CacheLevel(0) +_USING_CPP_BACKEND = os.environ.get("TLLM_KV_CACHE_MANAGER_V2_BACKEND", "cpp").lower() != "python" class _FakePage: @@ -138,6 +158,13 @@ def _flush_serialized_events(event_manager): return KVCacheEventSerializer.serialize(event_manager.get_latest_events(0)) +def _flush_contract_events(event_manager): + events = _flush_serialized_events(event_manager) + for event in events: + event["hash_algo"] = "" + return events + + def _stored_events(events): return [event for event in events if event["data"]["type"] == "stored"] @@ -169,6 +196,251 @@ def _commit_and_close(manager, stream, tokens, *, input_tokens=None, reuse_scope gc.collect() +@pytest.mark.skipif(not _USING_CPP_BACKEND, reason="requires the native C++ event manager") +def test_native_event_manager_queue_and_stored_coalescing(): + event_manager = NativeKVCacheEventManager( + max_kv_event_entries=8, + window_size=128, + hash_algo="v2_sha256", + ) + event_manager.add_stored_event( + None, + [ + NativeKVCacheStoredBlockData( + "block0", + [NativeUniqueToken(1), NativeUniqueToken(2)], + cache_level=0, + priority=35, + ) + ], + ) + event_manager.add_stored_event( + "block0", + [ + NativeKVCacheStoredBlockData( + "block1", + [NativeUniqueToken(3), NativeUniqueToken(4)], + cache_level=0, + priority=35, + ) + ], + ) + + event_manager.flush_iteration_events() + event_objects = pickle.loads(pickle.dumps(event_manager.get_latest_events(0))) + events = KVCacheEventSerializer.serialize(event_objects) + + assert len(events) == 1 + assert events[0]["hash_algo"] == "v2_sha256" + assert [block["block_hash"] for block in events[0]["data"]["blocks"]] == [ + "block0", + "block1", + ] + + +def test_native_event_manager_v1_hash_matches_legacy_cpp_hasher(): + _tb = pytest.importorskip("tensorrt_llm.bindings") + block_key = _tb.internal.batch_manager.BlockKey + block_key_hasher = _tb.internal.batch_manager.BlockKeyHasher + parent_hash = block_key_hasher.hash(block_key([1, 2, 3, 4])) + + assert NativeKVCacheEventManager._hash_block_key([1, 2, 3, 4], 0, None, None) == parent_hash + assert NativeKVCacheEventManager._hash_block_key( + [5, 6], parent_hash, None, None + ) == block_key_hasher.hash(block_key([5, 6]), parent_hash) + + +@pytest.mark.skipif(not _USING_CPP_BACKEND, reason="requires the native C++ event manager") +def test_native_event_manager_attention_dp_gather_callback(): + gathered_events = [] + + def gather(local_events): + gathered_events.extend(local_events) + return [pickle.loads(pickle.dumps(local_events))] + + event_manager = NativeKVCacheEventManager( + max_kv_event_entries=2, + window_size=128, + attention_dp_rank=0, + attention_dp_gather=gather, + ) + event_manager.add_created_event([4]) + + events = _flush_serialized_events(event_manager) + + assert len(gathered_events) == 1 + assert events[0]["attention_dp_rank"] == 0 + assert events[0]["data"]["num_blocks_per_cache_level"] == [4] + + +@pytest.mark.skipif(not _USING_CPP_BACKEND, reason="compares native and Python event managers") +def test_native_event_data_value_semantics_match_python_reference(): + native_token = NativeUniqueToken("token", 7) + native_block = NativeKVCacheStoredBlockData( + "block", + [native_token], + cache_level=1, + priority=35, + mm_keys=[(b"short-mm-key", 3), (b"another-key", 5, "uuid")], + cache_salt="salt", + ) + native_diff = NativeKVCacheEventDiff(0, 1) + native_objects = [ + native_token, + NativeKVCacheCreatedData([2, 3]), + native_block, + NativeKVCacheStoredData(None, [native_block]), + NativeKVCacheRemovedData(["block"]), + native_diff, + NativeKVCacheUpdatedData("block", native_diff, None), + ] + native_event = NativeKVCacheEvent( + 4, + native_objects[3], + 128, + "same-hash-label", + 1, + 2, + ) + native_objects.append(native_event) + + python_token = UniqueToken("token", 7) + python_block = KVCacheStoredBlockData( + "block", + [python_token], + cache_level=1, + priority=35, + mm_keys=[(b"short-mm-key", 3), (b"another-key", 5, "uuid")], + cache_salt="salt", + ) + python_diff = KVCacheEventDiff(0, 1) + python_objects = [ + python_token, + KVCacheCreatedData([2, 3]), + python_block, + KVCacheStoredData(None, [python_block]), + KVCacheRemovedData(["block"]), + python_diff, + KVCacheUpdatedData("block", python_diff, None), + ] + python_objects.append( + KVCacheEvent( + 4, + python_objects[3], + 128, + "same-hash-label", + 1, + 2, + ) + ) + + for value, python_value in zip(native_objects, python_objects): + assert pickle.loads(pickle.dumps(value)) == value + assert repr(value) == repr(python_value) + + python_event = KVCacheEventManager(max_kv_event_entries=1) + native_manager = NativeKVCacheEventManager(max_kv_event_entries=1) + python_event.add_stored_event(None, [python_block], layer_group_id=2) + native_manager.add_stored_event(None, [native_block], layer_group_id=2) + assert _flush_contract_events(native_manager) == _flush_contract_events(python_event) + + +@pytest.mark.skipif(not _USING_CPP_BACKEND, reason="compares native and Python event managers") +def test_native_event_manager_public_methods_match_python_reference(): + python_manager = KVCacheEventManager( + max_kv_event_entries=16, + window_size=128, + window_size_by_layer_group=None, + ) + native_manager = NativeKVCacheEventManager( + max_kv_event_entries=16, + window_size=128, + window_size_by_layer_group=None, + ) + python_manager.set_layer_group_window_sizes({0: 64, 1: 96}) + native_manager.set_layer_group_window_sizes({0: 64, 1: 96}) + + for manager, block_type, token_type, diff_type in ( + (python_manager, KVCacheStoredBlockData, UniqueToken, KVCacheEventDiff), + ( + native_manager, + NativeKVCacheStoredBlockData, + NativeUniqueToken, + NativeKVCacheEventDiff, + ), + ): + manager.add_created_event([2, 3], layer_group_ids=[0, 1]) + manager.add_stored_event( + None, + [block_type("block-0", [token_type(1)], cache_level=0, priority=35)], + layer_group_id=0, + ) + manager.add_stored_event( + "block-0", + [block_type("block-1", [token_type(2)], cache_level=0, priority=35)], + layer_group_id=0, + ) + manager.add_removed_event(block_hash for block_hash in ("removed-0", "removed-1")) + manager.add_updated_event( + "updated", + cache_level=diff_type(0, 1), + priority=diff_type(35, 50), + layer_group_id=1, + ) + + assert _flush_contract_events(native_manager) == _flush_contract_events(python_manager) + assert native_manager.get_latest_events(0) == python_manager.get_latest_events(0) == [] + assert native_manager.get_latest_events(-1) == python_manager.get_latest_events(-1) == [] + + +@pytest.mark.skipif(not _USING_CPP_BACKEND, reason="compares native and Python event managers") +def test_native_event_manager_unknown_raw_keys_match_python_noop_behavior(): + python_manager = KVCacheEventManager(max_kv_event_entries=4) + native_manager = NativeKVCacheEventManager(max_kv_event_entries=4) + + for manager, diff_type in ( + (python_manager, KVCacheEventDiff), + (native_manager, NativeKVCacheEventDiff), + ): + manager.add_removed_event([b"short", b"still-not-a-radix-key"]) + manager.add_removed_life_cycle_event(b"short", 0) + manager.add_updated_event(b"short") + manager.add_updated_event(b"short", cache_level=diff_type(0, 1)) + + assert _flush_contract_events(native_manager) == _flush_contract_events(python_manager) == [] + + +@pytest.mark.skipif(not _USING_CPP_BACKEND, reason="compares native and Python event managers") +def test_native_event_manager_disabled_queue_and_blocking_read_match_python_reference(): + for manager_type in (KVCacheEventManager, NativeKVCacheEventManager): + disabled = manager_type(max_kv_event_entries=0) + disabled.add_created_event([1]) + disabled.flush_iteration_events() + assert disabled.get_latest_events(0) == [] + + manager = manager_type(max_kv_event_entries=1) + result = [] + reader = threading.Thread(target=lambda: result.extend(manager.get_latest_events())) + reader.start() + time.sleep(0.05) + assert reader.is_alive() + manager.add_created_event([1]) + manager.flush_iteration_events() + reader.join(timeout=1) + assert not reader.is_alive() + assert len(result) == 1 + + +@pytest.mark.skipif(not _USING_CPP_BACKEND, reason="compares native and Python event managers") +def test_native_event_manager_constructor_errors_match_python_reference(): + for manager_type in (KVCacheEventManager, NativeKVCacheEventManager): + with pytest.raises(ValueError, match="Unsupported V2 KV cache event hash algorithm"): + manager_type(max_kv_event_entries=1, hash_algo="unsupported") + + with pytest.raises(TypeError): + NativeKVCacheUpdatedData("block") + + def test_v2_kv_cache_event_manager_serialization(): event_manager = KVCacheEventManager(max_kv_event_entries=4, window_size=128) event_manager.add_created_event([2, 3]) @@ -587,7 +859,7 @@ def test_v1_and_v2_managers_emit_same_v1_hash_stored_events(): manager_v1.flush_iteration_events() v1_events = KVCacheEventSerializer.serialize(manager_v1.get_latest_events(10)) - event_manager_v2 = KVCacheEventManager( + event_manager_v2 = NativeKVCacheEventManager( max_kv_event_entries=event_buffer_max_size, window_size=max_seq_len, hash_algo=KV_CACHE_HASH_ALGO_V1, @@ -907,7 +1179,7 @@ def test_v2_stored_events_match_block_hash_chain(): gc.collect() gc.disable() - event_manager = KVCacheEventManager(max_kv_event_entries=16, window_size=128) + event_manager = NativeKVCacheEventManager(max_kv_event_entries=16, window_size=128) manager = None try: tokens_per_block = 4 @@ -922,12 +1194,16 @@ def test_v2_stored_events_match_block_hash_chain(): stored_events = _stored_events(events) assert len(stored_events) == 1 + # Both the Python and C++ backends hash blocks with SHA-256 over the same + # little-endian token encoding, so the block-key chain matches the + # pure-Python Block.make_key implementation on either backend. root_key = RootBlock.make_key(ReuseScope()) block0_key = Block.make_key(root_key, tokens[:tokens_per_block]) block1_key = Block.make_key(block0_key, tokens[tokens_per_block:]) expected_hashes = [block0_key.hex(), block1_key.hex()] assert _stored_block_hashes(stored_events) == expected_hashes + assert stored_events[0]["hash_algo"] == "v2_sha256" assert stored_events[0]["data"]["parent_hash"] is None assert [ block["block_hash"] for block in stored_events[0]["data"]["blocks"] @@ -938,6 +1214,15 @@ def test_v2_stored_events_match_block_hash_chain(): assert [ token["token_id"] for token in stored_events[0]["data"]["blocks"][1]["tokens"] ] == list(range(tokens_per_block, 2 * tokens_per_block)) + + if _USING_CPP_BACKEND: + event_manager.add_updated_event( + bytes.fromhex(expected_hashes[0]), + cache_level=NativeKVCacheEventDiff(old_value=0, new_value=1), + layer_group_id=0, + ) + updated_events = _flush_serialized_events(event_manager) + assert updated_events[0]["data"]["block_hash"] == expected_hashes[0] finally: gc.enable() if manager is not None: @@ -950,7 +1235,7 @@ def test_v2_v1_hash_events_include_cache_salt_from_kv_cache(): gc.collect() gc.disable() - event_manager = KVCacheEventManager( + event_manager = NativeKVCacheEventManager( max_kv_event_entries=16, window_size=128, hash_algo=KV_CACHE_HASH_ALGO_V1, @@ -986,7 +1271,7 @@ def test_v2_reused_prefix_does_not_emit_duplicate_stored_events(): gc.collect() gc.disable() - event_manager = KVCacheEventManager(max_kv_event_entries=16, window_size=128) + event_manager = NativeKVCacheEventManager(max_kv_event_entries=16, window_size=128) manager = None try: tokens_per_block = 4 @@ -1010,12 +1295,14 @@ def test_v2_reused_prefix_does_not_emit_duplicate_stored_events(): reuse_events = _flush_serialized_events(event_manager) reused_hashes = _stored_block_hashes(reuse_events) + # SHA-256 block-key chain is identical across both backends. root_key = RootBlock.make_key(ReuseScope()) block0_key = Block.make_key(root_key, prefix_tokens[:tokens_per_block]) block1_key = Block.make_key(block0_key, prefix_tokens[tokens_per_block:]) block2_key = Block.make_key(block1_key, new_tokens) + prefix_keys = [block0_key, block1_key] - assert prefix_hashes == [block0_key.hex(), block1_key.hex()] + assert prefix_hashes == [block_key.hex() for block_key in prefix_keys] assert reused_hashes == [block2_key.hex()] assert not (set(prefix_hashes) & set(reused_hashes)) finally: @@ -1030,7 +1317,7 @@ def test_v2_removed_events_match_stored_hashes(): gc.collect() gc.disable() - event_manager = KVCacheEventManager(max_kv_event_entries=16, window_size=128) + event_manager = NativeKVCacheEventManager(max_kv_event_entries=16, window_size=128) manager = None try: tokens_per_block = 4 @@ -1075,7 +1362,7 @@ def test_v2_removed_event_emitted_when_last_level_page_is_dropped(): tokens_per_block = 8 window_size = 8 num_blocks = 4 - event_manager = KVCacheEventManager(max_kv_event_entries=16, window_size=window_size) + event_manager = NativeKVCacheEventManager(max_kv_event_entries=16, window_size=window_size) manager = None try: manager = _create_test_manager( @@ -1133,17 +1420,23 @@ def test_v2_kv_cache_event_manager_emits_updated_on_level_migration(): # a small window is used: it keeps the min_slots floor low enough that the # GPU level can be shrunk below the committed block count, forcing surplus # reusable pages to migrate to host. + # + # More committed blocks and a deeper shrink than the removed-event test are + # used here on purpose: the GPU eviction/migration walks the two attention + # pool groups in insertion order, so the surplus must exceed the first pool + # group's block count for the second pool group to also migrate — only then + # do both layer groups emit an "updated" (GPU -> host) event. tokens_per_block = 8 window_size = 8 - num_blocks = 4 - event_manager = KVCacheEventManager(max_kv_event_entries=16, window_size=window_size) + num_blocks = 8 + event_manager = NativeKVCacheEventManager(max_kv_event_entries=64, window_size=window_size) manager = None try: manager = _create_test_manager( event_manager, tokens_per_block=tokens_per_block, - gpu_quota=16 << 20, - host_quota=8 << 20, + gpu_quota=32 << 20, + host_quota=32 << 20, window_size=window_size, kv_buf_size=1 << 20, ) @@ -1163,7 +1456,7 @@ def test_v2_kv_cache_event_manager_emits_updated_on_level_migration(): del kv_cache gc.collect() - assert manager.resize(CacheLevel(0), 8 << 20) + assert manager.resize(CacheLevel(0), 12 << 20) event_manager.flush_iteration_events() events = KVCacheEventSerializer.serialize(event_manager.get_latest_events()) diff --git a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py index 9b8bda967ca3..e5b17c6bece3 100755 --- a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py +++ b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_manager_v2.py @@ -46,6 +46,7 @@ KVCacheManagerConfig, LayerGroupId, LayerId, + PlannedDropHandle, ReuseScope, SsmLayerConfig, SwaScratchReuseConfig, @@ -79,7 +80,6 @@ round_up, temporary_sys_path, typed_range, - unwrap_rawref, ) else: from tensorrt_llm.runtime.kv_cache_manager_v2 import ( @@ -99,6 +99,7 @@ KVCacheManagerConfig, LayerGroupId, LayerId, + PlannedDropHandle, ReuseScope, SsmLayerConfig, SwaScratchReuseConfig, @@ -136,7 +137,6 @@ round_up, temporary_sys_path, typed_range, - unwrap_rawref, ) from copy import deepcopy @@ -149,6 +149,19 @@ def get_cached_cuda_event_type(): + backend = os.environ.get("TLLM_KV_CACHE_MANAGER_V2_BACKEND", "cpp").lower() + if backend == "cpp": + try: + from bindings.internal.batch_manager.kv_cache_manager_v2 import CachedCudaEvent + + return CachedCudaEvent + except ImportError: + from tensorrt_llm.bindings.internal.batch_manager.kv_cache_manager_v2 import ( + CachedCudaEvent, + ) + + return CachedCudaEvent + if find_spec("kv_cache_manager_v2") is not None: from kv_cache_manager_v2._utils import CachedCudaEvent @@ -671,21 +684,18 @@ def test_commit_min_snapshot_reuses_swa_post_commit_prefix(self) -> None: kv1.close() stream_holder.take_finish_event().synchronize() - match = self.manager._radix_tree.match(ReuseScope(), prompt) - self.assertEqual(match.num_tokens, len(prompt)) - self.assertEqual(len(match.blocks), 4) - - swa_lc_id = next( - lc_id - for lc_id, lc in self.manager._life_cycles.attention_life_cycles() - if lc.window_size is not None + swa_lc_id = _introspection.swa_life_cycle_ids(self.manager)[0] + num_tokens, pages = _introspection.reuse_match_pages( + self.manager, ReuseScope(), prompt, swa_lc_id ) + self.assertEqual(num_tokens, len(prompt)) + self.assertEqual(len(pages), 4) # The committed snapshot is reusable at the post-commit token count, but # old SWA blocks outside that window should not keep reusable pages. - self.assertIsNone(match.blocks[0].storage[swa_lc_id]) - self.assertIsNone(match.blocks[1].storage[swa_lc_id]) - self.assertIsNotNone(match.blocks[2].storage[swa_lc_id]) - self.assertIsNotNone(match.blocks[3].storage[swa_lc_id]) + self.assertIsNone(pages[0]) + self.assertIsNone(pages[1]) + self.assertIsNotNone(pages[2]) + self.assertIsNotNone(pages[3]) self.assertEqual( self.manager.probe_reuse(input_tokens=prompt[: tokens_per_block * 3]), 0, @@ -695,6 +705,46 @@ def test_commit_min_snapshot_reuses_swa_post_commit_prefix(self) -> None: self.assertEqual(kv2.num_committed_tokens, len(prompt)) kv2.close() + def test_planned_drop_handle(self) -> None: + window_size = 8 + self.prepare(16 << 20, 0, 0, 2, window_size, 0, tokens_per_block=8) + long_tokens = [self.next_token() for _ in range(24)] + short_tokens = long_tokens[:8] + + def plan_drop(tokens: list[TokenIdExt]) -> PlannedDropHandle: + kv_cache = self.manager.create_kv_cache(None, tokens) + with TemporaryCudaStream([]) as stream_holder: + stream = cast(CudaStream, stream_holder.handle) + self.assertTrue(kv_cache.resume(stream)) + self.assertTrue(kv_cache.resize(len(tokens))) + uncommitted = tokens[kv_cache.num_committed_tokens :] + if uncommitted: + kv_cache.commit(uncommitted) + kv_cache.stop_committing() + drop_handle = kv_cache.plan_committed_block_drop() + self.assertIsNotNone(drop_handle) + self.assertIsInstance(drop_handle, PlannedDropHandle) + _ = stream_holder.take_finish_event() + kv_cache.close() + assert drop_handle is not None + return drop_handle + + long_handle = plan_drop(long_tokens) + short_handle = plan_drop(short_tokens) + self.assertEqual(self.manager.probe_reuse(None, short_tokens), len(short_tokens)) + + short_handle.drop() + self.assertEqual(self.manager.probe_reuse(None, short_tokens), 0) + self.assertEqual(self.manager.probe_reuse(None, long_tokens), len(long_tokens)) + + long_handle.drop() + # The SWA window is dropped, while older full-attention blocks remain reusable. + self.assertEqual( + self.manager.probe_reuse(None, long_tokens), len(long_tokens) - window_size + ) + with self.assertRaisesRegex(ValueError, "already been dropped"): + long_handle.drop() + def test_reuse_scope_isolates_reuse(self) -> None: self.prepare(16 << 20, 0, 0, 2, None, 0, tokens_per_block=8) tokens = [TokenId(i) for i in range(64)] @@ -1947,6 +1997,89 @@ def test_no_reuse_with_ssm(self) -> None: kv_cache.resume(stream) kv_cache.close() + @parameterized.expand( + [ + ("miss", None, 48, False, (1, 0, 1, 0, 48, 0, 0)), + ("aligned_hit", 32, 48, False, (1, 1, 0, 32, 16, 1, 0)), + ("unaligned_hit", 48, 64, True, (1, 1, 0, 48, 16, 0, 1)), + ] + ) + def test_ssm_snapshot_iteration_stats( + self, + _name: str, + snapshot_length: int | None, + lookup_length: int, + enable_partial_reuse: bool, + expected: tuple[int, int, int, int, int, int, int], + ) -> None: + tokens_per_block = 32 + cfg = self._make_ssm_config( + tokens_per_block=tokens_per_block, + enable_partial_reuse=enable_partial_reuse, + ) + self.manager = KVCacheManager(cfg) + stream_holder = CachedCudaStream() + stream = cast(CudaStream, stream_holder.handle) + prompt = [self.next_token() for _ in range(lookup_length)] + + if snapshot_length is not None: + seed = self.manager.create_kv_cache() + seed.resume(stream) + seed.capacity = snapshot_length + seed.history_length = snapshot_length + seed.commit(prompt[:snapshot_length], is_end=True) + seed.close() + + reused = self.manager.create_kv_cache( + input_tokens=prompt, + id=101, + # This is only a sizing hint; lookup telemetry must use the + # actual input_tokens length. + expected_prompt_length=lookup_length + 17, + ) + self.assertEqual(reused.num_committed_tokens, expected[3]) + self.assertEqual(self.manager.get_dirty_stats_kv_cache_ids(), {101}) + reused.commit_pending_stats() + self.assertEqual(self.manager.get_dirty_stats_kv_cache_ids(), set()) + + ssm_life_cycle_id = _introspection.ssm_life_cycle_id(self.manager) + assert ssm_life_cycle_id is not None + snapshot_stats = self.manager.get_and_reset_ssm_snapshot_iteration_stats() + self.assertEqual(set(snapshot_stats), {ssm_life_cycle_id}) + stats = snapshot_stats[ssm_life_cycle_id] + self.assertEqual( + ( + stats.iter_snapshot_lookups, + stats.iter_snapshot_hits, + stats.iter_snapshot_misses, + stats.iter_reused_tokens, + stats.iter_unreused_tokens, + stats.iter_aligned_snapshot_hits, + stats.iter_unaligned_snapshot_hits, + ), + expected, + ) + self.assertEqual(stats.iter_snapshot_hit_rate, expected[1] / expected[0]) + self.assertEqual(self.manager.get_and_reset_ssm_snapshot_iteration_stats(), {}) + + reused.resume(stream) + reused.close() + + def test_discard_ssm_snapshot_stats_clears_dirty_state(self) -> None: + cfg = self._make_ssm_config() + self.manager = KVCacheManager(cfg) + tokens = [self.next_token() for _ in range(16)] + + kv_cache = self.manager.create_kv_cache(input_tokens=tokens, id=101) + self.assertEqual(self.manager.get_dirty_stats_kv_cache_ids(), {101}) + kv_cache.discard_pending_stats() + + self.assertEqual(self.manager.get_dirty_stats_kv_cache_ids(), set()) + self.assertEqual(self.manager.get_and_reset_ssm_snapshot_iteration_stats(), {}) + stream_holder = CachedCudaStream() + kv_cache.resume(cast(CudaStream, stream_holder.handle)) + kv_cache.close() + def test_ssm(self) -> None: """Inference with SSM layer: prefill 63 tokens, decode 52 tokens.""" cfg = self._make_ssm_config() @@ -2098,6 +2231,180 @@ def test_ssm_reuse_keeps_snapshots_from_multiple_commits(self) -> None: kv4.resume(stream) kv4.close() + def test_ssm_planned_drop_targets_latest_snapshot_with_shared_plans(self) -> None: + """Shared plans drop only their conversation endpoint snapshot.""" + cfg = self._make_ssm_config(tokens_per_block=32) + self.manager = KVCacheManager(cfg) + stream_holder = CachedCudaStream() + stream = cast(CudaStream, stream_holder.handle) + prompt = [self.next_token() for _ in range(64)] + + kv_cache = self.manager.create_kv_cache() + kv_cache.resume(stream) + kv_cache.capacity = 32 + kv_cache.commit(prompt[:32]) + kv_cache.capacity = 64 + kv_cache.commit(prompt[32:]) + kv_cache.stop_committing() + first_handle = kv_cache.plan_committed_block_drop() + second_handle = kv_cache.plan_committed_block_drop() + self.assertIsNotNone(first_handle) + self.assertIsNotNone(second_handle) + kv_cache.close() + + self.assertEqual(self.manager.probe_reuse(input_tokens=prompt), 64) + assert first_handle is not None + first_handle.drop() + self.assertEqual(self.manager.probe_reuse(input_tokens=prompt), 64) + assert second_handle is not None + second_handle.drop() + self.assertEqual(self.manager.probe_reuse(input_tokens=prompt), 32) + + empty_cache = self.manager.create_kv_cache() + empty_cache.resume(stream) + empty_cache.stop_committing() + self.assertIsNone(empty_cache.plan_committed_block_drop()) + empty_cache.close() + + def test_ssm_planned_drop_includes_partial_swa_window(self) -> None: + """Hybrid plans include SSM and every partial SWA-window page.""" + cfg = self._make_ssm_config( + tokens_per_block=32, + num_attn_layers=1, + num_ssm_layers=1, + window_size=32, + ) + self.manager = KVCacheManager(cfg) + stream_holder = CachedCudaStream() + stream = cast(CudaStream, stream_holder.handle) + prompt = [self.next_token() for _ in range(48)] + + kv_cache = self.manager.create_kv_cache() + kv_cache.resume(stream) + kv_cache.capacity = len(prompt) + kv_cache.commit(prompt) + kv_cache.stop_committing() + drop_handle = kv_cache.plan_committed_block_drop() + self.assertIsNotNone(drop_handle) + + attn_lc_id = _introspection.attention_life_cycle_ids(self.manager)[0] + ssm_lc_id = _introspection.ssm_life_cycle_id(self.manager) + assert ssm_lc_id is not None + num_tokens, attn_counts = _introspection.reuse_match_planned_drop_counts( + self.manager, ReuseScope(), prompt, attn_lc_id, self.manager.enable_partial_match + ) + self.assertEqual(num_tokens, len(prompt)) + # Every partial SWA-window attention page is planned for drop exactly once. + self.assertTrue(attn_counts and all(count == 1 for count in attn_counts)) + _, ssm_counts = _introspection.reuse_match_planned_drop_counts( + self.manager, ReuseScope(), prompt, ssm_lc_id, self.manager.enable_partial_match + ) + # The SSM snapshot on the last committed block is planned for drop. + self.assertEqual(ssm_counts[-1], 1) + + kv_cache.close() + assert drop_handle is not None + drop_handle.drop() + self.assertEqual(self.manager.probe_reuse(input_tokens=prompt), 0) + + def test_ssm_same_block_snapshots_support_monotonic_multi_turn_reuse(self) -> None: + cfg = self._make_ssm_config(tokens_per_block=32, enable_partial_reuse=True) + self.manager = KVCacheManager(cfg) + engine = FakeEngine(cfg) + stream_holder = CachedCudaStream() + stream = cast(CudaStream, stream_holder.handle) + + prompt = [self.next_token() for _ in range(64)] + + for snapshot_length, expected_reuse in ((10, 0), (20, 10), (25, 20)): + kv_cache = self.manager.create_kv_cache(input_tokens=prompt[:snapshot_length]) + self.assertEqual(kv_cache.num_committed_tokens, expected_reuse) + kv_cache.resume(stream) + kv_cache.capacity = snapshot_length + engine.execute( + [ + Step( + kv_cache, + prompt[expected_reuse:snapshot_length], + prompt[:expected_reuse], + ) + ], + stream, + ) + kv_cache.history_length = snapshot_length + kv_cache.commit(prompt[expected_reuse:snapshot_length]) + kv_cache.close() + + exact = self.manager.create_kv_cache(input_tokens=prompt[:25]) + self.assertEqual(exact.num_committed_tokens, 25) + exact.resume(stream) + exact.capacity = 25 + exact.history_length = 25 + engine.execute([Step(exact, [], prompt[:25])], stream) + exact.close() + + def test_ssm_same_block_forks_only_reuse_safe_snapshots(self) -> None: + cfg = self._make_ssm_config(tokens_per_block=32, enable_partial_reuse=True) + self.manager = KVCacheManager(cfg) + engine = FakeEngine(cfg) + stream_holder = CachedCudaStream() + stream = cast(CudaStream, stream_holder.handle) + prompt = [self.next_token() for _ in range(64)] + + source = self.manager.create_kv_cache() + source.resume(stream) + commit_start = 0 + for commit_end in (10, 20): + source.capacity = commit_end + chunk = prompt[commit_start:commit_end] + engine.execute([Step(source, chunk, prompt[:commit_start])], stream) + source.history_length = commit_end + source.commit(chunk) + commit_start = commit_end + source.close() + + # The retained 20-token state is in the future of a fork at token 15. + # Falling back to zero reuse is safe; reusing that state would corrupt + # the fork's SSM history. + early_fork = prompt[:15] + [self.next_token() for _ in range(25)] + early = self.manager.create_kv_cache(input_tokens=early_fork) + self.assertEqual(early.num_committed_tokens, 0) + early.resume(stream) + early.capacity = len(early_fork) + engine.execute([Step(early, early_fork, [])], stream) + early.history_length = len(early_fork) + engine.execute([Step(early, [], early_fork)], stream) + early.close() + + later_fork = prompt[:25] + [self.next_token() for _ in range(15)] + later = self.manager.create_kv_cache(input_tokens=later_fork) + self.assertEqual(later.num_committed_tokens, 20) + later.resume(stream) + later.capacity = len(later_fork) + engine.execute([Step(later, later_fork[20:], later_fork[:20])], stream) + later.history_length = len(later_fork) + engine.execute([Step(later, [], later_fork)], stream) + later.close() + + aligned = self.manager.create_kv_cache(input_tokens=prompt[:32]) + self.assertEqual(aligned.num_committed_tokens, 20) + aligned.resume(stream) + aligned.capacity = 32 + engine.execute([Step(aligned, prompt[20:32], prompt[:20])], stream) + aligned.history_length = 32 + aligned.commit(prompt[20:32]) + aligned.close() + + aligned_fork = prompt[:40] + [self.next_token() for _ in range(8)] + reused = self.manager.create_kv_cache(input_tokens=aligned_fork) + self.assertEqual(reused.num_committed_tokens, 32) + reused.resume(stream) + reused.capacity = len(aligned_fork) + engine.execute([Step(reused, aligned_fork[32:], aligned_fork[:32])], stream) + reused.history_length = len(aligned_fork) + engine.execute([Step(reused, [], aligned_fork)], stream) + reused.close() + def test_ssm_partial_snapshot_respects_partial_reuse_setting(self) -> None: """Partial SSM snapshots are created, but partial prompt reuse remains optional.""" tokens_per_block = 32 @@ -2131,17 +2438,21 @@ def test_ssm_partial_snapshot_respects_partial_reuse_setting(self) -> None: exact.resume(stream) exact.close() - match = self.manager._radix_tree.match( - ReuseScope(), prompt[:48], self.manager.enable_partial_match + ssm_lc_id = _introspection.ssm_life_cycle_id(self.manager) + assert ssm_lc_id is not None + num_tokens, pages = _introspection.reuse_match_pages( + self.manager, + ReuseScope(), + prompt[:48], + ssm_lc_id, + self.manager.enable_partial_match, ) - self.assertEqual(match.num_tokens, 48) - assert self.manager._life_cycles.ssm_life_cycle_id is not None - page_ref = match.blocks[-1].storage[self.manager._life_cycles.ssm_life_cycle_id] - assert page_ref is not None - page = unwrap_rawref(page_ref) - self.assertEqual(page.num_tokens_in_block, 16) - - del exact, kv1, longer, match, page, page_ref + self.assertEqual(num_tokens, 48) + last_page = pages[-1] + assert last_page is not None + self.assertEqual(last_page[1], 16) + + del exact, kv1, longer gc.collect() stream_holder.synchronize() self.manager.shutdown() @@ -2161,30 +2472,30 @@ def test_commit_is_end_moves_partial_attention_and_ssm_pages(self) -> None: kv_cache.capacity = len(prompt) kv_cache.history_length = len(prompt) - attn_lc_id = next(iter(self.manager._life_cycles.attention_life_cycles()))[0] - assert self.manager._life_cycles.ssm_life_cycle_id is not None - ssm_lc_id = self.manager._life_cycles.ssm_life_cycle_id + attn_lc_id = _introspection.attention_life_cycle_ids(self.manager)[0] + ssm_lc_id = _introspection.ssm_life_cycle_id(self.manager) + assert ssm_lc_id is not None attn_tail_slot = kv_cache.get_base_page_indices(LayerGroupId(attn_lc_id))[1] ssm_slot = kv_cache.get_ssm_block_base_index(LayerGroupId(ssm_lc_id)) kv_cache.commit(prompt, is_end=True) kv_cache.close() - match = self.manager._radix_tree.match( - ReuseScope(), prompt, self.manager.enable_partial_match + _, attn_pages = _introspection.reuse_match_pages( + self.manager, ReuseScope(), prompt, attn_lc_id, self.manager.enable_partial_match + ) + num_tokens, ssm_pages = _introspection.reuse_match_pages( + self.manager, ReuseScope(), prompt, ssm_lc_id, self.manager.enable_partial_match ) - self.assertEqual(match.num_tokens, len(prompt)) - tree_block = match.blocks[-1] + self.assertEqual(num_tokens, len(prompt)) - attn_ref = tree_block.storage[attn_lc_id] - ssm_ref = tree_block.storage[ssm_lc_id] - assert attn_ref is not None - assert ssm_ref is not None - attn_page = unwrap_rawref(attn_ref) - ssm_page = unwrap_rawref(ssm_ref) - self.assertEqual(attn_page.slot_id, attn_tail_slot) - self.assertEqual(ssm_page.slot_id, ssm_slot) - self.assertEqual(ssm_page.num_tokens_in_block, 16) + attn_page = attn_pages[-1] + ssm_page = ssm_pages[-1] + assert attn_page is not None + assert ssm_page is not None + self.assertEqual(attn_page[0], attn_tail_slot) + self.assertEqual(ssm_page[0], ssm_slot) + self.assertEqual(ssm_page[1], 16) def test_commit_min_snapshot_requires_history_alignment(self) -> None: """commit_min_snapshot requires commit() to start or end at history length.""" @@ -2335,6 +2646,9 @@ def tearDown(self) -> None: # Non-power-of-2 sizes so granularity rounding is non-trivial. PG0_SLOT_SIZE = 786432 # 768KB (windowed) PG1_SLOT_SIZE = 1310720 # 1280KB (non-windowed) + SSM_STATE_SLOT_SIZE = 23592960 + SSM_CONV_SLOT_SIZE = 829440 + ATTN_SLOT_SIZE = 245760 def _make_config( self, @@ -2389,6 +2703,38 @@ def _make_config( swa_scratch_reuse=(SwaScratchReuseConfig() if enable_swa_scratch_reuse else None), ) + def _make_hybrid_config(self, gpu_quota: int = 128 << 20) -> KVCacheManagerConfig: + return KVCacheManagerConfig( + tokens_per_block=self.TOKENS_PER_BLOCK, + cache_tiers=[GpuCacheTierConfig(quota=gpu_quota)], + layers=[ + SsmLayerConfig( + layer_id=LayerId(0), + buffers=[ + BufferConfig( + role=DataRole("ssm_state"), + size=self.SSM_STATE_SLOT_SIZE, + ), + BufferConfig( + role=DataRole("conv_state"), + size=self.SSM_CONV_SLOT_SIZE, + ), + ], + ), + AttentionLayerConfig( + layer_id=LayerId(1), + buffers=[ + BufferConfig( + role=DataRole("key"), + size=self.ATTN_SLOT_SIZE, + ), + ], + ), + ], + enable_partial_reuse=False, + commit_min_snapshot=True, + ) + def test_default_init_ratio(self): """Without typical_step or constraints, uses hardcoded fallback.""" cfg = self._make_config() @@ -2426,6 +2772,25 @@ def test_typical_step_long_sequences(self): self.assertLess(ratio[0], 0.15) manager.shutdown() + def test_zero_capacity_request_reserves_only_an_ssm_slot(self): + """Every request reserves one SSM slot, including a zero-token dummy.""" + manager = KVCacheManager(self._make_hybrid_config()) + ssm_lc = _introspection.ssm_life_cycle_id(manager) + assert ssm_lc is not None + ssm_pg = _introspection.pool_group_index(manager, ssm_lc) + attn_pg = 1 - ssm_pg + + batch = BatchDesc( + kv_caches=[ + KVCacheDesc(capacity=64, history_length=63), + KVCacheDesc(capacity=0, history_length=0), + ] + ) + slots = _introspection.compute_slots_for_batch(manager, batch, self.TOKENS_PER_BLOCK, None) + self.assertEqual(slots[ssm_pg], 2) + self.assertEqual(slots[attn_pg], 2) + manager.shutdown() + def test_constraints_floor_typical_step(self): """Constraints clamp the typical_step ratio from below.""" typical = BatchDesc(kv_caches=[KVCacheDesc(capacity=4096, history_length=4000)] * 32) @@ -2443,8 +2808,41 @@ def test_constraints_floor_typical_step(self): mgr_unconstrained.shutdown() mgr_constrained.shutdown() - def test_initial_pool_ratio_overrides_typical_step_and_constraints(self): - """Explicit initial_pool_ratio takes precedence over inferred sizing inputs.""" + def test_constraint_reserves_resume_headroom(self): + """A full constraint batch must stay below the resume utilization gate.""" + num_requests = 32 + constraint = BatchDesc(kv_caches=[KVCacheDesc(capacity=1, history_length=0)] * num_requests) + granularity = 2 << 20 + gpu_quota = round_up(num_requests * self.PG0_SLOT_SIZE, granularity) + round_up( + num_requests * self.PG1_SLOT_SIZE, granularity + ) + cfg = self._make_config(gpu_quota=gpu_quota, constraints=[constraint]) + cfg.max_util_for_resume = 0.95 + manager = KVCacheManager(cfg) + stream_holder = CachedCudaStream() + stream = cast(CudaStream, stream_holder.handle) + + kv_caches = [] + for _ in range(num_requests): + kv_cache = manager.create_kv_cache() + self.assertTrue(kv_cache.resume(stream)) + kv_cache.capacity = 1 + kv_caches.append(kv_cache) + + for kv_cache in kv_caches: + kv_cache.close() + manager.shutdown() + + def test_constraint_floor_overrides_infeasible_initial_pool_ratio(self): + """A constraint's feasibility floor overrides an infeasible initial_pool_ratio. + + initial_pool_ratio is the target split and still overrides typical_step, but + constraints stay feasibility floors (mirrors PR #16269): if a declared batch + needs more slots than its target share can hold, that pool group's share is + clamped up so the batch can be resumed, rather than starving it during warmup. + Here pool group 1's 0.2 target cannot satisfy the 256-request constraint, so + its share is clamped above 0.2 and pool group 0 gives up the remainder. + """ typical = BatchDesc(kv_caches=[KVCacheDesc(capacity=4096, history_length=4000)] * 32) constraint = BatchDesc(kv_caches=[KVCacheDesc(capacity=256, history_length=128)] * 256) cfg = self._make_config( @@ -2455,7 +2853,8 @@ def test_initial_pool_ratio_overrides_typical_step_and_constraints(self): manager = KVCacheManager(cfg) ratio = _introspection.current_gpu_ratio(manager) - self.assertGreater(ratio[0], ratio[1]) + self.assertGreater(ratio[1], 0.2) + self.assertLess(ratio[0], 0.8) self.assertAlmostEqual(sum(ratio), 1.0, places=6) manager.shutdown() @@ -3323,11 +3722,7 @@ def test_reuse_across_prefill_turns_keeps_only_window_minus_one(self) -> None: tokens_per_block=tokens_per_block, gpu_quota=64 << 20, ) - swa_lc_id = next( - lc_id - for lc_id, lc in self.manager._life_cycles.attention_life_cycles() - if lc.window_size is not None - ) + swa_lc_id = _introspection.swa_life_cycle_ids(self.manager)[0] prompt1 = [TokenId(i) for i in range(2 * window_size - 1)] # 127 tokens prompt2 = [TokenId(i) for i in range(200)] # first 127 tokens identical to prompt1 @@ -3350,11 +3745,12 @@ def test_reuse_across_prefill_turns_keeps_only_window_minus_one(self) -> None: # match() is documented volatile, so read what we need and drop the reference before # mutating the tree again (holding live Block refs across a later clear would leave the # eviction accounting inconsistent). - match1 = self.manager._radix_tree.match(ReuseScope(), prompt1) - self.assertEqual(match1.num_tokens, len(prompt1)) - self.assertEqual(len(match1.blocks), 4) - has_page = [b.storage[swa_lc_id] is not None for b in match1.blocks] - del match1 + num_tokens1, pages1 = _introspection.reuse_match_pages( + self.manager, ReuseScope(), prompt1, swa_lc_id + ) + self.assertEqual(num_tokens1, len(prompt1)) + self.assertEqual(len(pages1), 4) + has_page = [p is not None for p in pages1] self.assertEqual( has_page, [False, False, True, True], # positions 0..63 out of window; 64..126 in window diff --git a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_salting.py b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_salting.py index c6d1d4e76a43..4c2dc9615686 100644 --- a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_salting.py +++ b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_salting.py @@ -20,19 +20,15 @@ from typing import TYPE_CHECKING, cast if not TYPE_CHECKING and find_spec("kv_cache_manager_v2") is not None: - from kv_cache_manager_v2 import ReuseScope, TokenId - from kv_cache_manager_v2._block_radix_tree import ( - Block, - BlockRadixTree, - sequence_to_blockchain_keys, - ) + from kv_cache_manager_v2 import TokenId, sequence_to_blockchain_keys + from kv_cache_manager_v2._block_radix_tree import Block, BlockRadixTree, ReuseScope from kv_cache_manager_v2._life_cycle_registry import LifeCycleRegistry else: - from tensorrt_llm.runtime.kv_cache_manager_v2 import ReuseScope, TokenId + from tensorrt_llm.runtime.kv_cache_manager_v2 import TokenId, sequence_to_blockchain_keys from tensorrt_llm.runtime.kv_cache_manager_v2._block_radix_tree import ( Block, BlockRadixTree, - sequence_to_blockchain_keys, + ReuseScope, ) from tensorrt_llm.runtime.kv_cache_manager_v2._life_cycle_registry import LifeCycleRegistry @@ -49,7 +45,11 @@ def attention_life_cycles(self) -> Iterator[tuple[object, object]]: class TestReuseScope(unittest.TestCase): - def test_to_bytes_distinguishes_scope_fields(self) -> None: + def test_reuse_scope_seeds_distinct_keys(self) -> None: + # Distinct reuse scopes -- including the None-vs-0 cases for each field -- + # must seed distinct radix-tree keys. The first blockchain key is the + # root (the reuse-scope digest), so hashing the same tokens under each + # scope isolates the scope's contribution to the key. scopes = [ ReuseScope(), ReuseScope(lora_id=0), @@ -57,11 +57,15 @@ def test_to_bytes_distinguishes_scope_fields(self) -> None: ReuseScope(lora_id=0, salt=0), ReuseScope(lora_id=7, salt=11), ] + tokens = [TokenId(1), TokenId(2)] + + def root_key(scope: "ReuseScope") -> bytes: + return next(iter(sequence_to_blockchain_keys(2, scope, tokens)))[1] - serialized = [scope.to_bytes() for scope in scopes] - self.assertEqual(len(set(serialized)), len(scopes)) - self.assertEqual(serialized[0], b"\x00") - self.assertEqual(serialized, [scope.to_bytes() for scope in scopes]) + roots = [root_key(scope) for scope in scopes] + self.assertEqual(len(set(roots)), len(scopes)) + # Deterministic across repeated derivation. + self.assertEqual(roots, [root_key(scope) for scope in scopes]) def test_blockchain_keys_are_seeded_by_reuse_scope(self) -> None: tokens = [TokenId(1), TokenId(2), TokenId(3), TokenId(4)] diff --git a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_stats_api.py b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_stats_api.py index fa43bf2d0992..a74723c95e6e 100644 --- a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_stats_api.py +++ b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_stats_api.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import pytest import torch @@ -26,11 +27,18 @@ KVCacheManager, KVCacheManagerConfig, KVCacheStatsDelta, + PoolGroupPeakBlockStats, + SsmSnapshotIterationStatsDelta, ) pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.fixture(scope="module", autouse=True) +def initialize_cuda_context() -> None: + torch.empty(1, device="cuda") + + def _make_config(*, enable_stats: bool = True) -> KVCacheManagerConfig: return KVCacheManagerConfig( tokens_per_block=4, @@ -63,6 +71,59 @@ def test_stats_delta_arithmetic() -> None: assert iteration.empty assert iteration.iter_cache_hit_rate == 0.0 + snapshot = SsmSnapshotIterationStatsDelta( + iter_snapshot_lookups=4, + iter_snapshot_hits=3, + iter_snapshot_misses=1, + iter_reused_tokens=96, + iter_unreused_tokens=32, + iter_aligned_snapshot_hits=2, + iter_unaligned_snapshot_hits=1, + ) + assert snapshot.iter_snapshot_hit_rate == 0.75 + snapshot.clear() + assert snapshot.empty + assert snapshot.iter_snapshot_hit_rate == 0.0 + + +def test_cpp_stats_types_are_native() -> None: + if os.environ.get("TLLM_KV_CACHE_MANAGER_V2_BACKEND", "cpp").lower() != "cpp": + pytest.skip("C++ backend only") + + from tensorrt_llm.bindings.internal.batch_manager import kv_cache_manager_v2 as cpp + + assert KVCacheStatsDelta is cpp.KVCacheStatsDelta + assert KVCacheIterationStatsDelta is cpp.KVCacheIterationStatsDelta + assert PoolGroupPeakBlockStats is cpp.PoolGroupPeakBlockStats + + stats = KVCacheStatsDelta(alloc_total_blocks=1, reused_blocks=2) + assert repr(stats) == ( + "KVCacheStatsDelta(alloc_total_blocks=1, alloc_new_blocks=0, " + "reused_blocks=2, missed_blocks=0)" + ) + peak = PoolGroupPeakBlockStats(available=3, unavailable=4, evictable=5) + assert peak == PoolGroupPeakBlockStats(3, 4, 5) + with pytest.raises(AttributeError): + peak.available = 6 + + +def test_manager_accepts_uint64_max_request_id() -> None: + manager = KVCacheManager(_make_config()) + cache = None + cuda_graph_dummy_request_id = (1 << 64) - 1 + try: + cache = manager.create_kv_cache(id=cuda_graph_dummy_request_id) + assert cache.id == cuda_graph_dummy_request_id + manager.mark_stats_dirty(cuda_graph_dummy_request_id) + assert manager.get_dirty_stats_kv_cache_ids() == {cuda_graph_dummy_request_id} + manager.mark_stats_excluded(cuda_graph_dummy_request_id) + assert manager.is_stats_excluded(cuda_graph_dummy_request_id) + assert manager.get_dirty_stats_kv_cache_ids() == set() + finally: + if cache is not None: + cache.close() + manager.shutdown() + @pytest.mark.parametrize("enable_stats", [False, True]) def test_manager_stats_config_and_api(enable_stats: bool) -> None: @@ -72,6 +133,7 @@ def test_manager_stats_config_and_api(enable_stats: bool) -> None: assert manager.init_config.enable_stats is enable_stats assert manager.get_committed_stats() == KVCacheStatsDelta() assert manager.get_and_reset_iteration_stats() == {} + assert manager.get_and_reset_ssm_snapshot_iteration_stats() == {} peak_stats = manager.get_and_reset_iteration_peak_block_stats(GPU_LEVEL) assert len(peak_stats) == 1 assert peak_stats[0].available >= 0 diff --git a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_stats_behavior.py b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_stats_behavior.py index a43818a18090..9c130f12977e 100644 --- a/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_stats_behavior.py +++ b/tests/unittest/kv_cache_manager_v2_tests/test_kv_cache_stats_behavior.py @@ -60,6 +60,7 @@ class _StatsRequest: state: LlmRequestState = LlmRequestState.GENERATION_IN_PROGRESS context_current_position: int = 0 context_chunk_size: int = 0 + expect_snapshot_points: list[int] = field(default_factory=list) prepopulated_prompt: tuple[int, int] | None = None kv_cache_perf_metric_calls: list[dict[str, int]] = field(default_factory=list) multimodal_hashes: None = None diff --git a/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py b/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py index 20f7ef29eea6..02b2b4421b1b 100644 --- a/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py +++ b/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py @@ -7,8 +7,6 @@ import pytest from test_common.http_utils import (wait_for_endpoint_down, wait_for_endpoint_ready) -from test_common.perf_metrics_utils import (get_timing_metrics, - validate_timing_metrics) from tensorrt_llm._utils import get_sm_version @@ -121,8 +119,6 @@ def worker(model_name: str, ctx_tp_pp_size: tuple, gen_tp_pp_size: tuple): "enable_block_reuse": False, }, "disable_overlap_scheduler": True, - "perf_metrics_max_requests": 1000, - "return_perf_metrics": True, } if is_ctx_node(): print(f"starting ctx_server for rank {RANK} node rank {NODE_RANK}") @@ -203,11 +199,6 @@ def test_completion(client: openai.OpenAI, message = completion.choices[0].text assert message.startswith('2.') - perf_metrics = get_timing_metrics(disagg_server.url_root) - # allow 5ms leniency when comparing the time points from disagg and ctx/gen servers - validate_timing_metrics(perf_metrics, - "multinode test_completion", - time_tolerance_seconds=0.005) # sleep 10 seconds to ensure a successful wait_for_endpoint_ready on rank1 time.sleep(10) disagg_server.terminate() diff --git a/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes_service_discovery.py b/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes_service_discovery.py index 0e6c997a41a4..780844f2a77d 100644 --- a/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes_service_discovery.py +++ b/tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes_service_discovery.py @@ -6,7 +6,6 @@ import openai import pytest -from test_common.perf_metrics_utils import get_timing_metrics, validate_timing_metrics from tensorrt_llm._utils import get_free_port from tensorrt_llm.llmapi.disagg_utils import ServerRole @@ -112,8 +111,6 @@ def worker(model_name: str, disagg_cluster_config: dict): "enable_block_reuse": False, }, "disable_overlap_scheduler": True, - "return_perf_metrics": True, - "perf_metrics_max_requests": 1000, } # start workers on 0.0.0.0:, then the workers should be able to # report their correct hostname:port to the disagg server @@ -165,7 +162,6 @@ def disagg_server(disagg_cluster_config: dict): "disagg_cluster": disagg_cluster_config, "port": DISAGG_SERVER_PORT, "hostname": "0.0.0.0", - "perf_metrics_max_requests": 1000, } print(f"starting disagg_server for rank {RANK} node rank {NODE_RANK}") # ctx/gen servers are unnecessary for service discovery test @@ -217,9 +213,6 @@ def test_completion( message = completion.choices[0].text assert message.startswith("2.") - perf_metrics = get_timing_metrics(disagg_server.url_root) - validate_timing_metrics(perf_metrics, "multinode test_completion") - disagg_server.terminate() elif is_gen_node(): diff --git a/tests/unittest/llmapi/apps/_test_openai_chat_harmony_perf_metrics.py b/tests/unittest/llmapi/apps/_test_openai_chat_harmony_perf_metrics.py index 85cd2a073b1a..7bd3cfa17019 100644 --- a/tests/unittest/llmapi/apps/_test_openai_chat_harmony_perf_metrics.py +++ b/tests/unittest/llmapi/apps/_test_openai_chat_harmony_perf_metrics.py @@ -1,25 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for /perf_metrics population on the Harmony (GPT-OSS) chat path. - -Asserts two behaviors of the Harmony chat endpoint: - * The streaming generator stamps `server_first_token_time` and calls - `_extract_metrics(...)` after `data: [DONE]`, so the `/perf_metrics` - deque is populated for streaming chat requests. - * When `TRTLLM_KVCACHE_TIME_OUTPUT_PATH` is set, the per-request - `sampling_params.return_perf_metrics` flag is enabled so the engine - emits metrics for each request. Without it, the deque stays empty - even with `return_perf_metrics: True` at the LLM-args level. -""" - -import json +"""Tests JSONL performance metrics for the Harmony chat path.""" + import os import tempfile -from urllib.request import urlopen import openai import pytest import yaml +from test_common.perf_metrics_utils import read_perf_metrics_jsonl, wait_for_perf_metrics_jsonl from utils.llm_data import llm_datasets_root from ..test_llm import get_model_path @@ -46,15 +35,14 @@ def kv_cache_time_output_dir(tmp_path_factory): @pytest.fixture(scope="module") -def extra_llm_api_options_file(): +def extra_llm_api_options_file(kv_cache_time_output_dir: str): fd, path = tempfile.mkstemp(suffix=".yaml", prefix="extra_llm_api_options_") os.close(fd) try: with open(path, "w") as f: yaml.dump( { - "return_perf_metrics": True, - "perf_metrics_max_requests": 16, + "perf_metrics_output_dir": kv_cache_time_output_dir, }, f, ) @@ -92,17 +80,10 @@ def async_client(server: RemoteOpenAIServer): return server.get_async_client() -def _drain_perf_metrics(server: RemoteOpenAIServer): - response = urlopen(f"{server.url_root}/perf_metrics") - assert response.status == 200 - return json.loads(response.read()) - - def _assert_perf_metrics_entry_well_formed(entry: dict): assert "request_id" in entry - assert "perf_metrics" in entry + assert entry["status"] == "complete" pm = entry["perf_metrics"] - assert "first_iter" in pm and "last_iter" in pm assert pm["first_iter"] <= pm["last_iter"] tm = pm["timing_metrics"] @@ -115,10 +96,12 @@ def _assert_perf_metrics_entry_well_formed(entry: dict): @pytest.mark.asyncio(loop_scope="module") async def test_non_streaming_perf_metrics( - async_client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model: str + async_client: openai.AsyncOpenAI, + server: RemoteOpenAIServer, + model: str, + kv_cache_time_output_dir: str, ): - # Drain anything from prior tests in this module session. - _drain_perf_metrics(server) + previous_count = len(read_perf_metrics_jsonl(kv_cache_time_output_dir)) response = await async_client.chat.completions.create( model=model, messages=[{"role": "user", "content": "Reply with exactly the single word: PONG."}], @@ -126,20 +109,19 @@ async def test_non_streaming_perf_metrics( ) assert response.choices[0].message.content is not None - entries = _drain_perf_metrics(server) - assert len(entries) == 1, ( - "Expected exactly one /perf_metrics entry after a single non-streaming " - f"harmony chat completion, got {len(entries)}: {entries}" - ) - _assert_perf_metrics_entry_well_formed(entries[0]) + records = wait_for_perf_metrics_jsonl(kv_cache_time_output_dir, previous_count + 1) + entry = records[-1] + _assert_perf_metrics_entry_well_formed(entry) @pytest.mark.asyncio(loop_scope="module") async def test_streaming_perf_metrics( - async_client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model: str + async_client: openai.AsyncOpenAI, + server: RemoteOpenAIServer, + model: str, + kv_cache_time_output_dir: str, ): - # Drain anything from prior tests in this module session. - _drain_perf_metrics(server) + previous_count = len(read_perf_metrics_jsonl(kv_cache_time_output_dir)) stream = await async_client.chat.completions.create( model=model, messages=[{"role": "user", "content": "Explain transformers in one sentence."}], @@ -153,10 +135,6 @@ async def test_streaming_perf_metrics( saw_done = True assert saw_done, "Streaming chat returned no chunks" - entries = _drain_perf_metrics(server) - assert len(entries) == 1, ( - "Expected exactly one /perf_metrics entry after a single streaming " - f"harmony chat completion, got {len(entries)}: {entries}. " - "This usually means _extract_metrics did not run after [DONE]." - ) - _assert_perf_metrics_entry_well_formed(entries[0]) + records = wait_for_perf_metrics_jsonl(kv_cache_time_output_dir, previous_count + 1) + entry = records[-1] + _assert_perf_metrics_entry_well_formed(entry) diff --git a/tests/unittest/llmapi/apps/_test_openai_metrics.py b/tests/unittest/llmapi/apps/_test_openai_metrics.py index 12ce84f63646..957297f5a5ca 100644 --- a/tests/unittest/llmapi/apps/_test_openai_metrics.py +++ b/tests/unittest/llmapi/apps/_test_openai_metrics.py @@ -1,5 +1,6 @@ """Test the metrics endpoint when using OpenAI API to send requests""" +import time from unittest.mock import patch import pytest @@ -51,6 +52,34 @@ def test_version(client): assert response.status_code == 200 +def test_metrics_available_before_first_request(client): + """Verify that KV cache config stats are available at startup, + before any inference request is sent. This is critical for external + metric scrapers (e.g. the Kubernetes Inference Gateway EPP) that need + cache_config_info immediately to make routing decisions.""" + # Poll until the background stats collector processes the initial stats + deadline = time.time() + 5.0 + stats = [] + while time.time() < deadline: + response = client.get("/metrics") + assert response.status_code == 200 + stats = response.json() + if stats: + break + time.sleep(0.1) + assert stats, "Expected initial stats before first request" + response_dict = stats[0] + assert "kvCacheStats" in response_dict, \ + "kvCacheStats should be present before first request" + kv_stats = response_dict["kvCacheStats"] + assert "maxNumBlocks" in kv_stats + assert "tokensPerBlock" in kv_stats + assert kv_stats["maxNumBlocks"] > 0, \ + "maxNumBlocks should be positive at startup" + assert kv_stats["tokensPerBlock"] > 0, \ + "tokensPerBlock should be positive at startup" + + def test_metrics(client): response = client.post("/v1/completions", json={ diff --git a/tests/unittest/llmapi/apps/_test_openai_perf_metrics.py b/tests/unittest/llmapi/apps/_test_openai_perf_metrics.py index c1049939953b..b285f2190e3f 100644 --- a/tests/unittest/llmapi/apps/_test_openai_perf_metrics.py +++ b/tests/unittest/llmapi/apps/_test_openai_perf_metrics.py @@ -1,11 +1,13 @@ -import json import logging import os import tempfile -from urllib.request import urlopen import pytest +import requests import yaml +from test_common.perf_metrics_utils import wait_for_perf_metrics_jsonl + +from tensorrt_llm.serve import perf_metrics from ..test_llm import get_model_path from .openai_server import RemoteOpenAIServer @@ -21,13 +23,18 @@ def model_name(): @pytest.fixture(scope="module") -def temp_extra_llm_api_options_file(request): +def perf_metrics_output_dir(tmp_path_factory): + return tmp_path_factory.mktemp("perf_metrics") + + +@pytest.fixture(scope="module") +def temp_extra_llm_api_options_file(perf_metrics_output_dir): temp_dir = tempfile.gettempdir() temp_file_path = os.path.join(temp_dir, "extra_llm_api_options.yaml") try: extra_llm_api_options_dict = { "return_perf_metrics": True, - "perf_metrics_max_requests": 10 + "perf_metrics_output_dir": str(perf_metrics_output_dir), } with open(temp_file_path, 'w') as f: @@ -51,34 +58,43 @@ def server(model_name: str, logger.info("Tests completed, shutting down server") -def test_metrics_endpoint(server: RemoteOpenAIServer): - - client = server.get_client() - client.completions.create( - model="Server", - prompt="Hello, my name is", - max_tokens=25, - stream=False, +def test_return_perf_metrics_and_jsonl_dump(server: RemoteOpenAIServer, + perf_metrics_output_dir): + response = requests.post( + f"{server.url_root}/v1/completions", + headers={perf_metrics.RETURN_METRICS_HEADER: "1"}, + json={ + "model": "Server", + "prompt": "Hello, my name is", + "max_tokens": 2, + }, + timeout=120, ) - - response = urlopen(f'{server.url_root}/perf_metrics') - assert response.status is 200 - - data_list = json.loads(response.read()) - assert len(data_list) == 1 - assert "perf_metrics" in data_list[0] - assert "request_id" in data_list[0] - - data = data_list[0]["perf_metrics"] - assert "first_iter" in data - assert "last_iter" in data - assert data["first_iter"] <= data["last_iter"] - - timing_metrics = data["timing_metrics"] - assert "arrival_time" in timing_metrics - assert "first_scheduled_time" in timing_metrics - assert "first_token_time" in timing_metrics - assert "last_token_time" in timing_metrics + assert response.status_code == 200 + + for header in ( + perf_metrics.SERVER_TIMING_HEADER, + perf_metrics.START_END_TIME_HEADER, + perf_metrics.STEP_METRICS_HEADER, + perf_metrics.CTX_CHUNK_METRICS_HEADER, + ): + assert response.headers.get(header) + + records = wait_for_perf_metrics_jsonl(perf_metrics_output_dir, + expected_count=1) + data = records[-1] + assert data["status"] == "complete" + assert set(data) == { + "request_id", + "perf_metrics", + "time_breakdown_metrics", + "status", + } + + request_metrics = data["perf_metrics"] + assert request_metrics["first_iter"] <= request_metrics["last_iter"] + + timing_metrics = request_metrics["timing_metrics"] assert timing_metrics["arrival_time"] < timing_metrics[ "first_scheduled_time"] assert timing_metrics["first_scheduled_time"] < timing_metrics[ @@ -86,16 +102,35 @@ def test_metrics_endpoint(server: RemoteOpenAIServer): assert timing_metrics["first_token_time"] <= timing_metrics[ "last_token_time"] - kv_cache_metrics = data["kv_cache_metrics"] - assert "num_total_allocated_blocks" in kv_cache_metrics - assert "num_new_allocated_blocks" in kv_cache_metrics - assert "num_reused_blocks" in kv_cache_metrics - assert "num_missed_blocks" in kv_cache_metrics + kv_cache_metrics = request_metrics["kv_cache_metrics"] assert kv_cache_metrics["num_new_allocated_blocks"] <= kv_cache_metrics[ "num_total_allocated_blocks"] - # exclude disagg specific metrics - assert "ctx_request_id" not in data_list[0] - assert "kv_cache_size" not in timing_metrics + assert "ctx_request_id" not in data assert "kv_cache_transfer_start" not in timing_metrics assert "kv_cache_transfer_end" not in timing_metrics + + +def test_streaming_metrics_require_request_opt_in(server: RemoteOpenAIServer): + payload = { + "model": "Server", + "prompt": "Hello, my name is", + "max_tokens": 2, + "stream": True, + } + response = requests.post(f"{server.url_root}/v1/completions", + json=payload, + timeout=120) + assert response.status_code == 200 + assert "data: [DONE]" in response.text + assert f"event: {perf_metrics.SSE_METRICS_EVENT}" not in response.text + + response = requests.post( + f"{server.url_root}/v1/completions", + headers={perf_metrics.RETURN_METRICS_HEADER: "1"}, + json=payload, + timeout=120, + ) + assert response.status_code == 200 + assert "data: [DONE]" in response.text + assert f"event: {perf_metrics.SSE_METRICS_EVENT}" in response.text diff --git a/tests/unittest/llmapi/apps/_test_openai_prometheus.py b/tests/unittest/llmapi/apps/_test_openai_prometheus.py index 48abe35263aa..a113274ad0c2 100644 --- a/tests/unittest/llmapi/apps/_test_openai_prometheus.py +++ b/tests/unittest/llmapi/apps/_test_openai_prometheus.py @@ -127,6 +127,31 @@ def _parse_all_kv_metrics(data: str, prefix: str) -> Dict[str, float | None]: return {name: _parse_prometheus_sample(data, name) for name in names} +def test_kv_cache_metrics_available_before_first_request( + server: RemoteOpenAIServer): + """Verify that KV cache metrics are available at startup, before any + inference request. External scrapers (e.g. the Kubernetes Inference + Gateway EPP) rely on these metrics for routing decisions.""" + metric_prefix = "trtllm_" + max_wait_time = 10.0 + poll_interval = 0.5 + start_time = time.time() + metrics_found = False + + while time.time() - start_time < max_wait_time: + response = urlopen(f'{server.url_root}/prometheus/metrics') + assert response.status == 200 + data = response.read().decode("utf-8") + if metric_prefix + "kv_cache_utilization" in data: + metrics_found = True + break + time.sleep(poll_interval) + + assert metrics_found, \ + (f"{metric_prefix}kv_cache_utilization not found in /prometheus/metrics " + f"after {max_wait_time}s — it should be available before any request") + + def test_metrics_endpoint(server: RemoteOpenAIServer): """Verify that Prometheus metrics are correctly exposed after serving requests. diff --git a/tests/unittest/llmapi/apps/test_chat_utils.py b/tests/unittest/llmapi/apps/test_chat_utils.py index 5c44c41770c1..bed19c4575c9 100644 --- a/tests/unittest/llmapi/apps/test_chat_utils.py +++ b/tests/unittest/llmapi/apps/test_chat_utils.py @@ -6,6 +6,7 @@ from tensorrt_llm.inputs.media_io import AudioMediaIO from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY +from tensorrt_llm.inputs.utils import retrieve_multimodal_placeholder from tensorrt_llm.serve import chat_utils as _chat_utils from tensorrt_llm.serve.chat_utils import ( _make_media_io, @@ -419,6 +420,20 @@ def test_jinjalike_literal(self): ) +def _expected_item(modality: str, index: int) -> dict: + """Build an expected ``item_order`` entry, mirroring ``add_data``. + + ``add_data`` records the placeholder produced for each item, formatting the + modality template with a 1-based running count that equals ``index + 1`` on + the data (non-embedding) path exercised by these tests. + """ + return { + "modality": modality, + "index": index, + "placeholder": retrieve_multimodal_placeholder(_MM_MODEL_TYPE, modality, index + 1), + } + + class TestMultimodalPlaceholderCounts: """Verify per-message multimodal placeholder counts. @@ -529,11 +544,78 @@ class _StubConfig: mock_config = _StubConfig() - _, _, mm_placeholder_counts = parse_chat_messages_coroutines(messages, mock_config, None) + _, _, mm_placeholder_counts, _ = parse_chat_messages_coroutines(messages, mock_config, None) assert mm_placeholder_counts == expected_mm_placeholder_counts +class TestMmItemOrderReturn: + """Tests for the ``mm_item_order`` return element. + + The 4th tuple element from ``parse_chat_messages_coroutines`` is the + ``MultimodalDataTracker.item_order()`` manifest, indexed within modality + in content-parts order. Each entry carries ``modality``, ``index`` and the + ``placeholder`` string recorded by ``add_data``. + """ + + @pytest.mark.parametrize( + "messages, expected", + [ + # Mixed image+video+image within one message: proves both + # content-parts order preservation (image, video, image) and + # per-modality index advance (image indices 0, 1; video 0). + ( + [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "a"}}, + {"type": "video_url", "video_url": {"url": "b"}}, + {"type": "image_url", "image_url": {"url": "c"}}, + ], + } + ], + [ + _expected_item("image", 0), + _expected_item("video", 0), + _expected_item("image", 1), + ], + ), + # Items spanning multiple messages: indices accumulate across + # messages, not per-message. + ( + [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "a"}}, + ], + }, + {"role": "assistant", "content": "ok"}, + { + "role": "user", + "content": [ + {"type": "video_url", "video_url": {"url": "b"}}, + {"type": "image_url", "image_url": {"url": "c"}}, + ], + }, + ], + [ + _expected_item("image", 0), + _expected_item("video", 0), + _expected_item("image", 1), + ], + ), + ], + ) + def test_item_order(self, messages: list, expected: list) -> None: + class _StubConfig: + model_type = _MM_MODEL_TYPE + + _, _, _, item_order = parse_chat_messages_coroutines(messages, _StubConfig(), None) + assert item_order == expected + + class TestParseChatMessageContentPart: """Unit tests for parse_chat_message_content_part.""" diff --git a/tests/unittest/llmapi/apps/test_chat_utils_validator_iterator.py b/tests/unittest/llmapi/apps/test_chat_utils_validator_iterator.py index 6b575ef849cc..1e248df5197f 100644 --- a/tests/unittest/llmapi/apps/test_chat_utils_validator_iterator.py +++ b/tests/unittest/llmapi/apps/test_chat_utils_validator_iterator.py @@ -151,7 +151,7 @@ def test_list_tool_calls(self): {"role": "assistant", "content": None, "tool_calls": [TOOL_CALL]}, {"role": "tool", "content": "72F", "tool_call_id": "call_1"}, ] - conv, _, _ = parse_chat_messages_coroutines(messages, self._mock_config(), None) + conv, _, _, _ = parse_chat_messages_coroutines(messages, self._mock_config(), None) assert len(conv) == 3 assert conv[1]["tool_calls"][0]["function"]["arguments"] == PARSED_ARGS @@ -160,7 +160,7 @@ def test_iterator_tool_calls(self): {"role": "user", "content": "hi"}, {"role": "assistant", "content": None, "tool_calls": SingleUseIterator([TOOL_CALL])}, ] - conv, _, _ = parse_chat_messages_coroutines(messages, self._mock_config(), None) + conv, _, _, _ = parse_chat_messages_coroutines(messages, self._mock_config(), None) assert conv[1]["tool_calls"][0]["function"]["arguments"] == PARSED_ARGS def test_extra_fields_raw_dict(self): @@ -169,7 +169,7 @@ def test_extra_fields_raw_dict(self): {"role": "user", "content": "hi"}, {"role": "assistant", "content": None, "tool_calls": [tc]}, ] - conv, _, _ = parse_chat_messages_coroutines(messages, self._mock_config(), None) + conv, _, _, _ = parse_chat_messages_coroutines(messages, self._mock_config(), None) assert conv[1]["tool_calls"][0]["name"] == "get_weather" diff --git a/tests/unittest/llmapi/apps/test_disagg_perf_metrics_collector.py b/tests/unittest/llmapi/apps/test_disagg_perf_metrics_collector.py deleted file mode 100644 index 70ab2432d98f..000000000000 --- a/tests/unittest/llmapi/apps/test_disagg_perf_metrics_collector.py +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import asyncio - -import pytest - -from tensorrt_llm.serve import perf_metrics - - -class _DummyMetric: - def inc(self): - pass - - def observe(self, _value): - pass - - -class _BlockingClient: - def __init__(self): - self.entered = asyncio.Event() - self.release = asyncio.Event() - self.active_collectors = 0 - self.max_active_collectors = 0 - self.calls = 0 - - async def collect_metrics(self): - self.calls += 1 - self.active_collectors += 1 - self.max_active_collectors = max(self.max_active_collectors, self.active_collectors) - self.entered.set() - await self.release.wait() - self.active_collectors -= 1 - return {} - - -@pytest.mark.asyncio -async def test_disagg_perf_metrics_collection_is_serialized(monkeypatch): - monkeypatch.setattr(perf_metrics, "instance_metric", lambda _definition: _DummyMetric()) - collector = perf_metrics.DisaggPerfMetricsCollector(max_requests=8) - client = _BlockingClient() - collector.add_client(client) - - first_task = asyncio.create_task(collector.get_perf_metrics()) - await client.entered.wait() - - second_task = asyncio.create_task(collector.get_perf_metrics()) - await asyncio.sleep(0) - - assert client.calls == 1 - assert client.max_active_collectors == 1 - - client.release.set() - assert await first_task == [] - assert await second_task == [] - assert client.calls == 2 - assert client.max_active_collectors == 1 diff --git a/tests/unittest/llmapi/apps/test_disagg_serving_perf_metrics.py b/tests/unittest/llmapi/apps/test_disagg_serving_perf_metrics.py index 9b18e3025d73..fc48349d3dfa 100644 --- a/tests/unittest/llmapi/apps/test_disagg_serving_perf_metrics.py +++ b/tests/unittest/llmapi/apps/test_disagg_serving_perf_metrics.py @@ -3,15 +3,25 @@ import openai import pytest +import requests from test_common.http_utils import wait_for_endpoint_ready from test_common.perf_metrics_utils import ( get_prometheus_metrics, get_timing_metrics, validate_timing_metrics, + wait_for_perf_metrics_jsonl, ) +from utils.llm_data import llm_models_root from tensorrt_llm._utils import get_free_ports -from tests.unittest.utils.llm_data import llm_models_root +from tensorrt_llm.serve.perf_metrics import ( + CTX_CHUNK_METRICS_HEADER, + RETURN_METRICS_HEADER, + SERVER_TIMING_HEADER, + SSE_METRICS_EVENT, + START_END_TIME_HEADER, + STEP_METRICS_HEADER, +) from ..test_llm import get_model_path from .openai_server import RemoteDisaggOpenAIServer, RemoteOpenAIServer @@ -44,6 +54,11 @@ def model_name(): return model_path +@pytest.fixture +def perf_metrics_output_dir(tmp_path): + return tmp_path / "perf_metrics" + + @pytest.fixture def disagg_cluster_config(disagg_port: int): return { @@ -52,7 +67,7 @@ def disagg_cluster_config(disagg_port: int): } -def worker_config(model_name: str, disagg_cluster_config: dict): +def worker_config(model_name: str, disagg_cluster_config: dict, perf_metrics_output_dir): return { "model": model_name, "disagg_cluster": disagg_cluster_config, @@ -66,14 +81,21 @@ def worker_config(model_name: str, disagg_cluster_config: dict): "disable_overlap_scheduler": True, "cuda_graph_config": None, "return_perf_metrics": True, + "perf_metrics_output_dir": str(perf_metrics_output_dir), "perf_metrics_max_requests": 1000, } @pytest.fixture -def workers(model_name: str, disagg_cluster_config: dict, ctx_port: int, gen_port: int): +def workers( + model_name: str, + disagg_cluster_config: dict, + ctx_port: int, + gen_port: int, + perf_metrics_output_dir, +): model_path = get_model_path(model_name) - extra_config = worker_config(model_name, disagg_cluster_config) + extra_config = worker_config(model_name, disagg_cluster_config, perf_metrics_output_dir) def worker(server_role: str, port: int): return RemoteOpenAIServer( @@ -92,12 +114,14 @@ def worker(server_role: str, port: int): @pytest.fixture -def disagg_server(disagg_cluster_config: dict, workers, disagg_port: int): +def disagg_server(disagg_cluster_config: dict, workers, disagg_port: int, perf_metrics_output_dir): disagg_config = { "hostname": "localhost", "port": disagg_port, "disagg_cluster": disagg_cluster_config, "perf_metrics_max_requests": 1000, + "return_perf_metrics": True, + "perf_metrics_output_dir": str(perf_metrics_output_dir), } with RemoteDisaggOpenAIServer( ctx_servers=[], @@ -149,6 +173,62 @@ def check_historgram(metrics_dict: dict, count: int, range: tuple[float, float]) assert mean > range[0] and mean < range[1] +@pytest.mark.timeout(300) +def test_return_perf_metrics_and_jsonl_dump( + workers: Tuple[RemoteOpenAIServer, RemoteOpenAIServer], + disagg_server: RemoteDisaggOpenAIServer, + model_name: str, + perf_metrics_output_dir, +): + assert len(workers) == 2 + for worker in workers: + worker.wait_for_server(timeout=120) + wait_for_endpoint_ready(disagg_server.url_root + "/health") + + payload = { + "model": model_name, + "prompt": "Reply with one token.", + "max_tokens": 1, + "temperature": 0.0, + } + response = requests.post( + f"{disagg_server.url_root}/v1/completions", + headers={RETURN_METRICS_HEADER: "1"}, + json=payload, + timeout=120, + ) + assert response.status_code == 200 + assert response.json()["id"] is not None + for header in ( + SERVER_TIMING_HEADER, + START_END_TIME_HEADER, + STEP_METRICS_HEADER, + CTX_CHUNK_METRICS_HEADER, + ): + assert response.headers.get(header) + + timing_metrics = get_timing_metrics(perf_metrics_output_dir) + validate_timing_metrics(timing_metrics, "test_return_perf_metrics_and_jsonl_dump") + records = wait_for_perf_metrics_jsonl(perf_metrics_output_dir, expected_count=3) + disagg_record = next(record for record in records if "ctx_server" in record) + disagg_request_id = disagg_record["disagg_request_id"] + for record in records: + assert record["status"] == "complete" + assert record["disagg_request_id"] == disagg_request_id + + assert "ctx_perf_metrics" in disagg_record + assert "gen_perf_metrics" in disagg_record + + response = requests.post( + f"{disagg_server.url_root}/v1/completions", + json={**payload, "stream": True}, + timeout=120, + ) + assert response.status_code == 200 + assert "data: [DONE]" in response.text + assert f"event: {SSE_METRICS_EVENT}" not in response.text + + @pytest.mark.asyncio @pytest.mark.timeout(300) async def test_completion_metrics( @@ -171,9 +251,6 @@ async def test_completion_metrics( max_token=max_token, model_name=model_name, ) - timing_metrics = get_timing_metrics(disagg_server.url_root) - validate_timing_metrics(timing_metrics, "test_completion_metrics") - metrics = get_prometheus_metrics(disagg_server.url_root) print(metrics) @@ -183,7 +260,11 @@ async def test_completion_metrics( assert metrics[f"{role}_error_requests"] == 0 assert f"{role}_retry_requests" in metrics - check_historgram(metrics["gen_first_token_latency_seconds"], total_requests, (0.0, 0.3)) + check_historgram( + metrics["gen_first_token_latency_seconds"], + total_requests, + (0.0, 0.3), + ) check_historgram(metrics["gen_complete_latency_seconds"], total_requests, (0.0, 0.6)) assert metrics["total_requests"] == total_requests @@ -219,5 +300,9 @@ async def test_completion_metrics( assert metrics["http_exceptions"] == 0 assert metrics["internal_errors"] == 0 - check_historgram(metrics["gen_complete_latency_seconds"], total_requests * 2, (0.0, 0.6)) + check_historgram( + metrics["gen_complete_latency_seconds"], + total_requests * 2, + (0.0, 0.6), + ) check_historgram(metrics["queue_latency_seconds"], total_requests * 2, (0.0, 0.03)) diff --git a/tests/unittest/llmapi/apps/test_request_metrics.py b/tests/unittest/llmapi/apps/test_request_metrics.py new file mode 100644 index 000000000000..7429e50aaf47 --- /dev/null +++ b/tests/unittest/llmapi/apps/test_request_metrics.py @@ -0,0 +1,400 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import pytest + +from tensorrt_llm.serve.perf_metrics import ( + CTX_CHUNK_METRICS_HEADER, + RETURN_METRICS_HEADER, + SERVER_TIMING_HEADER, + SSE_METRICS_EVENT, + START_END_TIME_HEADER, + STEP_METRICS_HEADER, + PerfMetricsJsonlWriter, + PerfMetricsMiddleware, + _jsonl_record, + build_metrics_headers, + build_metrics_record_from_headers, + combine_disagg_metrics, +) +from tensorrt_llm.serve.scripts.time_breakdown import RequestDataParser + + +def _record(status="complete"): + return { + "schema_version": 1, + "request_id": "42", + "status": status, + "phases": { + "server": { + "timing_metrics": { + "arrival_time": 1.0, + "first_scheduled_time": 1.01, + "first_token_time": 1.02, + "last_token_time": 1.05, + "kv_cache_transfer_start": None, + "kv_cache_transfer_end": None, + }, + "time_breakdown_metrics": { + "step_metrics": [ + { + "iter": 3, + "forward_start_time": 2.0, + "forward_end_time": 2.002, + "sample_start_time": 2.002, + "sample_end_time": 2.003, + "gpu_forward_time": 1.5, + "gpu_sample_time": 0.5, + } + ], + "ctx_chunk_metrics": [ + { + "forward_start_time": 1.0, + "forward_end_time": 1.004, + "sample_start_time": 1.004, + "sample_end_time": 1.005, + "gpu_forward_time": 3.0, + "gpu_sample_time": 0.25, + } + ], + }, + } + }, + } + + +def test_metrics_headers_use_metric_list_syntax(): + headers = build_metrics_headers([_record()]) + + assert "server_queue;dur=10.000000" in headers[SERVER_TIMING_HEADER] + assert "server_ttft;dur=20.000000" in headers[SERVER_TIMING_HEADER] + assert "server_e2e;dur=50.000000" in headers[SERVER_TIMING_HEADER] + assert "server-start;ts=1.000000000" in headers[START_END_TIME_HEADER] + assert "server-end;ts=1.050000000" in headers[START_END_TIME_HEADER] + assert "server-step-3-forward;dur=2.000000" in headers[STEP_METRICS_HEADER] + assert "server-step-3-gpu-sample;dur=0.500000" in headers[STEP_METRICS_HEADER] + assert "server-ctx-chunk-0-forward;dur=4.000000" in headers[CTX_CHUNK_METRICS_HEADER] + + +def test_combine_disagg_metrics_is_request_local(): + ctx = { + "request_id": "ctx-7", + "ctx_request_id": 7, + "metrics_headers": {CTX_CHUNK_METRICS_HEADER: "ctx-ctx-chunk-0-forward;dur=4.000000"}, + "phases": {"server": {"timing_metrics": {"arrival_time": 1.0}}}, + } + gen = { + "request_id": "gen-7", + "ctx_request_id": 7, + "metrics_headers": {STEP_METRICS_HEADER: "gen-step-3-forward;dur=2.000000"}, + "phases": {"server": {"timing_metrics": {"arrival_time": 2.0}}}, + } + + record = combine_disagg_metrics( + "7", + {"timing_metrics": {}}, + ctx, + gen, + disagg_request_id=7, + ) + + assert set(record["phases"]) == {"disagg", "ctx", "gen"} + assert record["disagg_request_id"] == 7 + assert record["phases"]["ctx"]["request_id"] == "ctx-7" + assert record["phases"]["gen"]["ctx_request_id"] == 7 + headers = build_metrics_headers([record]) + assert "ctx-ctx-chunk-0-forward;dur=4.000000" in headers[CTX_CHUNK_METRICS_HEADER] + assert "gen-step-3-forward;dur=2.000000" in headers[STEP_METRICS_HEADER] + + +def test_time_breakdown_parser_accepts_header_derived_disagg_record(): + headers = build_metrics_headers([_record()]) + ctx = build_metrics_record_from_headers(headers, "ctx", request_id="42") + gen = build_metrics_record_from_headers(headers, "gen", request_id="42") + record = combine_disagg_metrics( + "42", + { + "ctx_server": "ctx:8000", + "gen_server": "gen:8000", + "timing_metrics": { + "server_arrival_time": 0.99, + "ctx_dispatch_time": 1.0, + "server_first_token_time": 1.03, + }, + }, + ctx, + gen, + disagg_request_id=42, + ) + + parsed = RequestDataParser().parse_request(_jsonl_record(record), 0) + combined_headers = build_metrics_headers([record]) + + assert parsed["ctx_arrival_time"] == pytest.approx(1.0) + assert parsed["ctx_first_scheduled_time"] == pytest.approx(1.01) + assert parsed["ctx_first_token_time"] == pytest.approx(1.02) + assert parsed["ctx_server_arrival_time"] == pytest.approx(1.0) + assert parsed["ctx_server_first_token_time"] == pytest.approx(1.05) + assert parsed["gen_arrival_time"] == pytest.approx(1.0) + assert parsed["gen_first_scheduled_time"] == pytest.approx(1.01) + assert parsed["gen_first_token_time"] == pytest.approx(1.02) + assert parsed["gen_server_arrival_time"] == pytest.approx(1.0) + assert parsed["gen_server_first_token_time"] == pytest.approx(1.05) + assert parsed["disagg_server_arrival_time"] == pytest.approx(0.99) + assert combined_headers[START_END_TIME_HEADER].count("ctx-start;") == 1 + assert combined_headers[SERVER_TIMING_HEADER].count("ctx_queue;") == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("expose_headers", "request_metrics", "expected"), + [ + (False, True, False), + (True, False, False), + (True, True, True), + ], +) +async def test_middleware_controls_public_headers(expose_headers, request_metrics, expected): + sent = [] + + async def app(scope, receive, send): + scope["state"]["perf_metrics_records"].extend([_record(), _record()]) + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"application/json")], + } + ) + await send( + { + "type": "http.response.body", + "body": b"{}", + "more_body": False, + } + ) + + middleware = PerfMetricsMiddleware(app, expose_headers=expose_headers) + headers = [(RETURN_METRICS_HEADER.encode(), b"1")] if request_metrics else [] + scope = {"type": "http", "headers": headers, "state": {}} + + async def capture(message): + sent.append(message) + + await middleware(scope, None, capture) + + header_names = {key.lower() for key, _ in sent[0]["headers"]} + assert (SERVER_TIMING_HEADER.lower().encode() in header_names) is expected + assert (STEP_METRICS_HEADER.lower().encode() in header_names) is expected + assert (CTX_CHUNK_METRICS_HEADER.lower().encode() in header_names) is expected + if expected: + headers = dict(sent[0]["headers"]) + assert headers[SERVER_TIMING_HEADER.encode()].count(b"server_queue;") == 2 + + +@pytest.mark.asyncio +async def test_middleware_limits_non_streaming_metrics_headers(): + sent = [] + + async def app(scope, receive, send): + record = _record() + breakdown = record["phases"]["server"]["time_breakdown_metrics"] + breakdown["step_metrics"] *= 2000 + breakdown["ctx_chunk_metrics"] *= 2000 + scope["state"]["perf_metrics_records"].append(record) + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"application/json")], + } + ) + await send( + { + "type": "http.response.body", + "body": b"{}", + "more_body": False, + } + ) + + async def capture(message): + sent.append(message) + + middleware = PerfMetricsMiddleware(app, expose_headers=True) + await middleware( + { + "type": "http", + "headers": [(RETURN_METRICS_HEADER.encode(), b"1")], + "state": {}, + }, + None, + capture, + ) + + header_names = {key.lower() for key, _ in sent[0]["headers"]} + assert SERVER_TIMING_HEADER.lower().encode() in header_names + assert START_END_TIME_HEADER.lower().encode() in header_names + assert STEP_METRICS_HEADER.lower().encode() not in header_names + assert CTX_CHUNK_METRICS_HEADER.lower().encode() not in header_names + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("expose_headers", "request_metrics", "expected"), + [ + (False, True, False), + (True, False, False), + (True, True, True), + ], +) +async def test_stream_metrics_follow_done(expose_headers, request_metrics, expected): + sent = [] + + async def app(scope, receive, send): + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"text/event-stream")], + } + ) + await send( + { + "type": "http.response.body", + "body": b"data: [DONE]\n\n", + "more_body": True, + } + ) + scope["state"]["perf_metrics_records"].append(_record()) + await send( + { + "type": "http.response.body", + "body": b"", + "more_body": False, + } + ) + + async def capture(message): + sent.append(message) + + middleware = PerfMetricsMiddleware(app, expose_headers=expose_headers) + headers = [(RETURN_METRICS_HEADER.encode(), b"1")] if request_metrics else [] + await middleware({"type": "http", "headers": headers, "state": {}}, None, capture) + + assert sent[-2]["body"] == b"data: [DONE]\n\n" + has_metrics_event = f"event: {SSE_METRICS_EVENT}".encode() in sent[-1]["body"] + assert has_metrics_event is expected + + +@pytest.mark.asyncio +async def test_disconnect_after_done_is_ignored(): + async def app(scope, receive, send): + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"text/event-stream")], + } + ) + await send( + { + "type": "http.response.body", + "body": b"data: [DONE]\n\n", + "more_body": True, + } + ) + scope["state"]["perf_metrics_records"].append(_record()) + await send( + { + "type": "http.response.body", + "body": b"", + "more_body": False, + } + ) + + async def disconnect(message): + if message["type"] == "http.response.body" and not message.get("more_body", False): + raise OSError("client disconnected") + + middleware = PerfMetricsMiddleware(app, expose_headers=True) + await middleware( + { + "type": "http", + "headers": [(RETURN_METRICS_HEADER.encode(), b"1")], + "state": {}, + }, + None, + disconnect, + ) + + +@pytest.mark.asyncio +async def test_file_middleware_intercepts_detail_headers(tmp_path): + writer = PerfMetricsJsonlWriter(str(tmp_path), "test") + await writer.start() + + async def app(scope, receive, send): + records = [_record(), _record()] + records[0]["disagg_request_id"] = 17 + records[1]["disagg_request_id"] = 18 + scope["state"]["perf_metrics_records"].extend(records) + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"application/json")], + } + ) + await send( + { + "type": "http.response.body", + "body": b"{}", + "more_body": False, + } + ) + + sent = [] + + async def capture(message): + sent.append(message) + + middleware = PerfMetricsMiddleware(app, expose_headers=False, writer=writer) + await middleware({"type": "http", "headers": [], "state": {}}, None, capture) + await writer.close() + + header_names = {key.lower() for key, _ in sent[0]["headers"]} + assert STEP_METRICS_HEADER.lower().encode() not in header_names + assert CTX_CHUNK_METRICS_HEADER.lower().encode() not in header_names + + output_file = next(tmp_path.glob("perf_metrics-test-*.jsonl")) + saved = [json.loads(line) for line in output_file.read_text().splitlines()] + assert [record["disagg_request_id"] for record in saved] == [17, 18] + assert saved[0]["time_breakdown_metrics"]["step_metrics"] + assert saved[0]["time_breakdown_metrics"]["ctx_chunk_metrics"] + + +@pytest.mark.asyncio +async def test_jsonl_writer_drops_only_malformed_record(tmp_path): + writer = PerfMetricsJsonlWriter(str(tmp_path), "test") + await writer.start() + writer.submit({"phases": {}}) + writer.submit(_record()) + await writer.close() + + output_file = next(tmp_path.glob("perf_metrics-test-*.jsonl")) + records = [json.loads(line) for line in output_file.read_text().splitlines()] + assert writer.dropped_records == 1 + assert records[0]["request_id"] == 42 diff --git a/tests/unittest/llmapi/apps/test_tool_parsers.py b/tests/unittest/llmapi/apps/test_tool_parsers.py index 2805e3f4e190..f24bc9b61862 100644 --- a/tests/unittest/llmapi/apps/test_tool_parsers.py +++ b/tests/unittest/llmapi/apps/test_tool_parsers.py @@ -799,6 +799,221 @@ def test_qwen3_format_compliance(self, sample_tools, parser): assert result.calls[0].name == "get_weather" assert json.loads(result.calls[0].parameters) == {"location": "Tokyo"} + # ------------------------------------------------------------------ + # NVBug 6240584: bare-JSON fallback in detect_and_parse + # + # Some Qwen3 chat templates (notably Qwen3.6 FP8 with + # `--reasoning_parser qwen3_5 --tool_parser qwen3`) emit tool calls + # as bare JSON, without a `...` wrapper, once + # the reasoning parser strips the `` block. The parser must + # recover those before dropping the text into `normal_text`. + # ------------------------------------------------------------------ + + def test_detect_and_parse_bare_json_dict(self, sample_tools, parser): + """Bare JSON dict without wrapper is parsed as a tool call.""" + text = '{"name":"get_weather","arguments":{"location":"Paris"}}' + + result = parser.detect_and_parse(text, sample_tools) + + assert result.normal_text == "" + assert len(result.calls) == 1 + assert result.calls[0].name == "get_weather" + assert json.loads(result.calls[0].parameters) == {"location": "Paris"} + + def test_detect_and_parse_bare_json_list(self, sample_tools, parser): + """Bare JSON list of tool calls without wrapper is parsed.""" + text = '[{"name":"get_weather","arguments":{"location":"Paris"}}]' + + result = parser.detect_and_parse(text, sample_tools) + + assert result.normal_text == "" + assert len(result.calls) == 1 + assert result.calls[0].name == "get_weather" + assert json.loads(result.calls[0].parameters) == {"location": "Paris"} + + def test_detect_and_parse_bare_json_parameters_key(self, sample_tools, + parser): + """Bare JSON with `parameters` (instead of `arguments`) is still parsed.""" + text = '{"name":"get_weather","parameters":{"location":"Paris"}}' + + result = parser.detect_and_parse(text, sample_tools) + + assert len(result.calls) == 1 + assert result.calls[0].name == "get_weather" + assert json.loads(result.calls[0].parameters) == {"location": "Paris"} + + def test_detect_and_parse_non_json_text_falls_through( + self, sample_tools, parser): + """Plain non-JSON text passes through as normal_text with no calls.""" + text = "Hello world" + + result = parser.detect_and_parse(text, sample_tools) + + assert result.normal_text == "Hello world" + assert result.calls == [] + + def test_detect_and_parse_bare_json_scalar_falls_through( + self, sample_tools, parser): + """A JSON scalar (e.g. `"42"`) must fall through cleanly, not crash. + + This exercises the explicit `isinstance(parsed, (dict, list))` guard — + `parse_base_json` would raise `AttributeError` on a bare int, so the + guard prevents relying on exception catching for scalar JSON. + """ + text = "42" + + result = parser.detect_and_parse(text, sample_tools) + + assert result.calls == [] + # No crash is the important part. + + def test_detect_and_parse_malformed_bare_json_falls_through( + self, sample_tools, parser): + """Malformed JSON without wrapper falls through cleanly.""" + text = '{"name": "get_weather", "arguments": MALFORMED}' + + result = parser.detect_and_parse(text, sample_tools) + + assert result.calls == [] + assert result.normal_text == text + + def test_detect_and_parse_bare_json_with_trailing_content( + self, sample_tools, parser): + """Bare JSON followed by trailing non-whitespace text is still parsed. + + NVBug 6240584 review follow-up: `json.loads(text.strip())` raises + `json.JSONDecodeError: Extra data` on `'{...} trailing text'`, which + used to drop the valid tool call into `normal_text`. The parser now + uses `raw_decode` to consume only the leading JSON value and must + recover the tool call regardless of what follows. + """ + text = ('{"name":"get_weather","arguments":{"city":"Paris"}}\n' + 'Extra text after the tool call.') + + result = parser.detect_and_parse(text, sample_tools) + + assert len(result.calls) == 1 + assert result.calls[0].name == "get_weather" + assert json.loads(result.calls[0].parameters) == {"city": "Paris"} + + # ------------------------------------------------------------------ + # NVBug 6240584: bare-JSON fallback in parse_streaming_increment + # + # The streaming path must also recover bare-JSON tool calls when the + # `` wrapper never appears. Without this, streaming clients + # receive the JSON as `delta.content` with `finish_reason="stop"`. + # ------------------------------------------------------------------ + + def test_streaming_bare_json_one_chunk(self, sample_tools, parser): + """A complete bare-JSON tool call arriving in a single chunk emits calls.""" + result = parser.parse_streaming_increment( + '{"name":"get_weather","arguments":{"city":"Paris"}}', sample_tools) + + names = [c.name for c in result.calls if c.name] + assert "get_weather" in names + params = "".join(c.parameters for c in result.calls if c.parameters) + assert "Paris" in params + assert result.normal_text == "" + + def test_streaming_bare_json_split_across_chunks(self, sample_tools, + parser): + """Bare-JSON tool call split across multiple chunks parses on completion.""" + r1 = parser.parse_streaming_increment('{"name":"get_', sample_tools) + r2 = parser.parse_streaming_increment('weather","arguments":', + sample_tools) + r3 = parser.parse_streaming_increment('{"city":"Paris"}}', sample_tools) + + all_calls = list(r1.calls) + list(r2.calls) + list(r3.calls) + names = [c.name for c in all_calls if c.name] + assert "get_weather" in names + params = "".join(c.parameters for c in all_calls if c.parameters) + assert "Paris" in params + + def test_streaming_bare_json_does_not_leak_content(self, sample_tools, + parser): + """After a bare-JSON tool call is emitted, trailing text is not leaked. + + This must be the case even for subsequent empty/whitespace chunks: + leaking any normal_text would flip `finish_reason` back to `stop`. + """ + r1 = parser.parse_streaming_increment( + '{"name":"get_weather","arguments":{"city":"Paris"}}', sample_tools) + # Any subsequent chunks must not emit normal_text either. + r2 = parser.parse_streaming_increment("", sample_tools) + + assert r1.normal_text == "" + assert r2.normal_text == "" + + def test_streaming_non_json_text_flushed_as_normal(self, sample_tools, + parser): + """Non-JSON text without a wrapper is flushed to normal_text.""" + result = parser.parse_streaming_increment("Hello world", sample_tools) + + assert result.normal_text == "Hello world" + assert result.calls == [] + + def test_streaming_bare_json_with_trailing_content(self, sample_tools, + parser): + """Bare-JSON tool call plus trailing text: emit calls, don't buffer. + + NVBug 6240584 review follow-up: previously the streaming path called + `json.loads(stripped)`, which fails with `Extra data` when the + buffered content is `'{...} trailing text'`. The parser would then + keep buffering forever and never emit the tool call. With + `raw_decode`, the tool call must be emitted at the JSON boundary + and the trailing text must be dropped (bare-JSON mode already + suppresses subsequent chunks). + """ + result = parser.parse_streaming_increment( + '{"name":"get_weather","arguments":{"city":"Paris"}}\n' + 'Extra text after the tool call.', sample_tools) + + names = [c.name for c in result.calls if c.name] + assert "get_weather" in names + params = "".join(c.parameters for c in result.calls if c.parameters) + assert "Paris" in params + # Trailing text must NOT be surfaced as normal_text — it would flip + # finish_reason back to "stop". + assert result.normal_text == "" + + def test_streaming_bare_json_trailing_content_split_chunk( + self, sample_tools, parser): + """Same as above but the trailing text arrives in a later chunk. + + This exercises the state machine: chunk 1 completes the JSON (parser + must emit calls now, not wait for more input), chunk 2 arrives after + the parser is already in `_STREAM_MODE_BARE_JSON` and must be + suppressed. + """ + r1 = parser.parse_streaming_increment( + '{"name":"get_weather","arguments":{"city":"Paris"}}', sample_tools) + r2 = parser.parse_streaming_increment('\nExtra text.', sample_tools) + + names = [c.name for c in r1.calls if c.name] + assert "get_weather" in names + assert r1.normal_text == "" + # Trailing chunk is fully suppressed. + assert r2.calls == [] + assert r2.normal_text == "" + + def test_streaming_wrapped_form_unregressed(self, sample_tools, parser): + """The pre-existing wrapped-form streaming path continues to work.""" + # Send bot token. + parser.parse_streaming_increment("\n", sample_tools) + + # Partial JSON with name -> emits name with empty params. + r_name = parser.parse_streaming_increment('{"name":"get_weather"', + sample_tools) + assert len(r_name.calls) == 1 + assert r_name.calls[0].name == "get_weather" + assert r_name.calls[0].parameters == "" + + # Complete the JSON and the wrapper. + r_args = parser.parse_streaming_increment( + ',"arguments":{"location":"SF"}}\n', sample_tools) + assert len(r_args.calls) == 1 + assert json.loads(r_args.calls[0].parameters) == {"location": "SF"} + class TestQwen3CoderToolParser(BaseToolParserTestClass): """Test suite for Qwen3CoderToolParser class.""" @@ -2693,6 +2908,153 @@ def test_auto_detect_laguna(self, tmp_path): class TestToolParserIntegration: """Integration tests for tool parsers.""" + def test_qwen3_5_reasoning_plus_qwen3_tool_parser_bare_json_pipeline( + self, sample_tools): + r"""NVBug 6240584: end-to-end reasoning + tool parser pipeline. + + Reproduces the exact scenario from the bug report: the Qwen3.6 FP8 + chat template pre-injects `\n` into the assistant prompt + prefix, so the model output starts *inside* the reasoning block + with no opening `` tag. Content up to `` is the + reasoning, and what follows is a bare JSON tool call (no + `` wrapper). The `qwen3_5` reasoning parser (registered + with `reasoning_at_start=True`) strips the thinking block, then + the `qwen3` tool parser must recover the tool call so + `args.has_tool_call[0]` is True and downstream logic sets + `finish_reason="tool_calls"` (see chat_response_post_processor). + """ + from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest + from tensorrt_llm.serve.postprocess_handlers import ( + ChatPostprocArgs, apply_reasoning_parser, apply_tool_parser) + + # Bug-report input: reasoning content (no leading `` — the + # chat template already injected it into the prompt prefix) followed + # by a bare JSON tool call. + text = ('Reasoning here.\n' + '{"name":"get_weather","arguments":{"city":"Paris"}}') + + # Build a minimal request so we can construct ChatPostprocArgs. + req = ChatCompletionRequest( + model="Qwen/Qwen3.6-27B-FP8", + messages=[{ + "role": "user", + "content": "What is the weather in Paris?" + }], + tools=sample_tools, + ) + args = ChatPostprocArgs.from_request(req) + args.reasoning_parser = "qwen3_5" + args.tool_parser = "qwen3" + + # Non-streaming path. + content, reasoning_content = apply_reasoning_parser(args, + output_index=0, + text=text, + streaming=False) + assert reasoning_content == "Reasoning here." + # The reasoning parser strips `...` — the remaining + # content is the bare JSON, possibly with a leading newline. + assert '"name":"get_weather"' in content + + normal_text, calls = apply_tool_parser(args, + output_index=0, + text=content, + streaming=False) + + assert len(calls) == 1 + assert calls[0].name == "get_weather" + assert json.loads(calls[0].parameters) == {"city": "Paris"} + # Downstream (chat_response_post_processor) checks this flag to flip + # finish_reason from "stop" to "tool_calls". + assert args.has_tool_call.get(0) is True + # And no bare JSON leaks into the visible content. + assert normal_text == "" + + def test_qwen3_5_reasoning_plus_qwen3_tool_parser_bare_json_streaming( + self, sample_tools): + r"""NVBug 6240584: streaming variant of reasoning+tool parser pipeline. + + The bug most commonly reproduces on streamed chat completions — + the model emits tokens one at a time and the OpenAI server relies + on the tool parser to flip `finish_reason` to `tool_calls` before + the stream ends. Feed the same reasoning + bare-JSON payload + through `apply_reasoning_parser` / `apply_tool_parser` with + `streaming=True` in small chunks and assert: + - `args.has_tool_call[0]` is True at end-of-stream, + - the accumulated tool-call name/arguments are correct, + - the bare JSON never leaks into visible content. + """ + from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest + from tensorrt_llm.serve.postprocess_handlers import ( + ChatPostprocArgs, apply_reasoning_parser, apply_tool_parser) + + text = ('Reasoning here.\n' + '{"name":"get_weather","arguments":{"city":"Paris"}}') + + # Chunk the input to force the streaming state machines to buffer + # across boundaries. The split intentionally lands inside both the + # `` tag and the JSON payload. + chunks = [ + 'Reasoning ', + 'here.\n{"name":"get_', + 'weather","arguments":', + '{"city":"Pa', + 'ris"}}', + ] + + req = ChatCompletionRequest( + model="Qwen/Qwen3.6-27B-FP8", + messages=[{ + "role": "user", + "content": "What is the weather in Paris?" + }], + tools=sample_tools, + ) + args = ChatPostprocArgs.from_request(req) + args.reasoning_parser = "qwen3_5" + args.tool_parser = "qwen3" + + accumulated_content = "" + accumulated_normal_text = "" + collected_calls = [] + + for chunk in chunks: + content, _reasoning = apply_reasoning_parser(args, + output_index=0, + text=chunk, + streaming=True) + accumulated_content += content + if not content: + continue + normal_text, calls = apply_tool_parser(args, + output_index=0, + text=content, + streaming=True) + if normal_text: + accumulated_normal_text += normal_text + collected_calls.extend(calls) + + # The reasoning parser must have stripped everything through the + # `` tag; the bare JSON survives into content. + assert '"name":"get_weather"' in accumulated_content + + # The tool parser must have flipped `has_tool_call` before the + # stream ended — this is the exact condition + # `chat_response_post_processor` uses to set + # `finish_reason="tool_calls"`. + assert args.has_tool_call.get(0) is True + + # We must have received the tool name and its arguments (potentially + # across multiple streaming increments). + names = [c.name for c in collected_calls if c.name] + assert names == ["get_weather"] + params = "".join(c.parameters for c in collected_calls if c.parameters) + assert json.loads(params) == {"city": "Paris"} + + # And no visible content is leaked from the bare-JSON payload. + assert accumulated_normal_text == "" + def test_end_to_end_single_tool(self, sample_tools): """Test end-to-end parsing of a single tool call.""" parser = Qwen3ToolParser() diff --git a/tests/unittest/llmapi/test_disagg_telemetry_launcher.py b/tests/unittest/llmapi/test_disagg_telemetry_launcher.py index d67a3f84faae..5d8530c53a1e 100644 --- a/tests/unittest/llmapi/test_disagg_telemetry_launcher.py +++ b/tests/unittest/llmapi/test_disagg_telemetry_launcher.py @@ -49,7 +49,13 @@ def test_disaggregated_command_sets_shared_deployment_id(monkeypatch) -> None: raising=False, ) - disagg_config = SimpleNamespace(hostname="127.0.0.1", port=0, schedule_style=None) + disagg_config = SimpleNamespace( + hostname="127.0.0.1", + port=0, + schedule_style=None, + num_workers=1, + disagg_coordinator_url=None, + ) fake_socket = mock.MagicMock() fake_socket.__enter__.return_value = fake_socket deployment_id = SimpleNamespace(hex="deploy123") @@ -60,7 +66,7 @@ def test_disaggregated_command_sets_shared_deployment_id(monkeypatch) -> None: mock.patch.object(serve.socket, "socket", return_value=fake_socket), mock.patch.object(serve, "parse_metadata_server_config_file", return_value=None), mock.patch.object(serve, "OpenAIDisaggServer"), - mock.patch.object(serve.asyncio, "run"), + mock.patch.object(serve.uvloop, "run"), ): serve.disaggregated.callback( config_file="disagg.yaml", @@ -76,6 +82,80 @@ def test_disaggregated_command_sets_shared_deployment_id(monkeypatch) -> None: fake_socket.bind.assert_called_once_with(("127.0.0.1", 0)) +@pytest.mark.parametrize("schedule_style", ["context_first", "generation_first"]) +def test_launch_disagg_fleet_propagates_resolved_schedule_style( + monkeypatch, + schedule_style, +) -> None: + """Fleet children receive the resolved CLI-or-config schedule style.""" + observed_envs = [] + + class _FakePopen: + pid = 12345 + + def __init__(self, _command, **kwargs): + observed_envs.append(kwargs["env"]) + + def poll(self): + return None + + disagg_config = SimpleNamespace( + hostname="127.0.0.1", + port=8000, + schedule_style=schedule_style, + ) + monkeypatch.setattr(serve.subprocess, "Popen", _FakePopen) + monkeypatch.setattr(serve.atexit, "register", lambda *_: None) + monkeypatch.setattr(serve.signal, "signal", lambda *_: None) + + serve._launch_disagg_fleet( + disagg_config, + "disagg.yaml", + None, + 180, + 180, + 1, + "http://coordinator:8001", + ) + + assert observed_envs[0][serve.DisaggWorkerEnvs.TLLM_DISAGG_SCHEDULE_STYLE] == schedule_style + + +@pytest.mark.parametrize( + ("config_style", "resolved_style"), + [ + ("context_first", "generation_first"), + ("generation_first", "context_first"), + ], +) +def test_build_fleet_worker_applies_resolved_schedule_style( + monkeypatch, + config_style, + resolved_style, +) -> None: + """A fleet worker overrides its reparsed YAML with the parent resolution.""" + disagg_config = SimpleNamespace(schedule_style=config_style) + monkeypatch.setenv(serve.DisaggWorkerEnvs.TLLM_DISAGG_CONFIG_FILE, "disagg.yaml") + monkeypatch.setenv( + serve.DisaggWorkerEnvs.TLLM_DISAGG_COORDINATOR_URL, + "http://coordinator:8001", + ) + monkeypatch.setenv( + serve.DisaggWorkerEnvs.TLLM_DISAGG_SCHEDULE_STYLE, + resolved_style, + ) + + with ( + mock.patch.object(serve, "parse_disagg_config_file", return_value=disagg_config), + mock.patch.object(serve, "parse_metadata_server_config_file", return_value=None), + mock.patch.object(serve, "OpenAIDisaggServer") as mock_server, + ): + serve._build_disagg_server_from_env() + + assert disagg_config.schedule_style == resolved_style + assert mock_server.call_args.kwargs["config"] is disagg_config + + def test_launch_disaggregated_leader_propagates_deployment_id(monkeypatch) -> None: """Leader subprocess env keeps the shared telemetry deployment id.""" observed = {} @@ -153,7 +233,10 @@ def test_launch_disaggregated_server_sets_worker_role( llm_args = {"model": "dummy/model"} server_config = SimpleNamespace(type=server_type, hostname="127.0.0.1", port=8000) - disagg_config = SimpleNamespace(server_configs=[server_config]) + disagg_config = SimpleNamespace( + server_configs=[server_config], + allow_request_chat_template=False, + ) with ( mock.patch.object(serve, "parse_disagg_config_file", return_value=disagg_config), @@ -167,4 +250,5 @@ def test_launch_disaggregated_server_sets_worker_role( host="127.0.0.1", port=8000, llm_args=llm_args, + allow_request_chat_template=False, ) diff --git a/tests/unittest/llmapi/test_executor.py b/tests/unittest/llmapi/test_executor.py index 1b11587f7e15..dce923c6211c 100644 --- a/tests/unittest/llmapi/test_executor.py +++ b/tests/unittest/llmapi/test_executor.py @@ -2,8 +2,10 @@ import datetime import tempfile import threading +import time from concurrent.futures import ProcessPoolExecutor from pathlib import Path +from queue import Empty import pytest import torch @@ -17,6 +19,7 @@ GenerationResultBase, PostprocWorker) from tensorrt_llm.executor.ipc import FusedIpcQueue, ZeroMqQueue from tensorrt_llm.llmapi.tokenizer import TransformersTokenizer +from tensorrt_llm.llmapi.utils import AsyncQueue from tensorrt_llm.sampling_params import SamplingParams # isort: off @@ -115,6 +118,74 @@ def test_GenerationResult(): assert result._done +def test_result_timeout_raises(): + request = GenerationRequest(prompt_token_ids=[12, 23, 34], + sampling_params=SamplingParams(max_tokens=4)) + result = GenerationResult(request) + + # Queue stays empty (no worker pushing responses) -> must time out fast, not block indefinitely. + start = time.monotonic() + with pytest.raises(TimeoutError): + result.result(timeout=0.1) + elapsed = time.monotonic() - start + assert elapsed < 2.0, f"result() did not honor timeout (took {elapsed:.2f}s)" + assert not result._done + + +def test_result_timeout_budget_across_steps(): + request = GenerationRequest(prompt_token_ids=[12, 23, 34], + sampling_params=SamplingParams(max_tokens=4)) + result = GenerationResult(request) + + # A single non-final response is available, then the queue goes empty and the request never + # completes. + result.queue.put(create_rsp(33, finished=False)) + + start = time.monotonic() + with pytest.raises(TimeoutError): + result.result(timeout=0.1) + elapsed = time.monotonic() - start + assert elapsed < 2.0, f"result() did not honor timeout (took {elapsed:.2f}s)" + assert not result._done + + +def test_result_zero_timeout_completes_with_queued_responses(): + request = GenerationRequest(prompt_token_ids=[12, 23, 34], + sampling_params=SamplingParams(max_tokens=4)) + result = GenerationResult(request) + + result.queue.put(create_rsp(33, finished=False)) + result.queue.put(create_rsp(44, finished=True)) + + assert result.result(timeout=0) is result + assert result._done + assert len(result.outputs[0].token_ids) == 2 + + +def test_sync_queue_zero_timeout_checks_for_queued_item(): + queue = AsyncQueue() + queue.put("ready") + + with pytest.warns(UserWarning): + assert queue.sync_q.get(timeout=0) == "ready" + with pytest.warns(UserWarning), pytest.raises(Empty): + queue.sync_q.get(timeout=0) + + +def test_result_completes_within_timeout(): + request = GenerationRequest(prompt_token_ids=[12, 23, 34], + sampling_params=SamplingParams(max_tokens=4)) + result = GenerationResult(request) + + result.queue.put(create_rsp(33, finished=False)) + result.queue.put(create_rsp(44, finished=True)) + + ret = result.result(timeout=30.0) + assert ret is result + assert result._done + assert len(result.outputs[0].token_ids) == 2 + + def test_DetokenizedGenerationResultBase(): sampling_params = SamplingParams(max_tokens=4) model_path = llm_models_root() / "llama-models-v2/TinyLlama-1.1B-Chat-v1.0" @@ -318,7 +389,7 @@ def ResponsePostprocessWorker_worker_task(pull_pipe_addr, push_pipe_addr, tokenizer_dir): worker = PostprocWorker( pull_pipe_addr=pull_pipe_addr, - push_pipe_addr=push_pipe_addr, + push_pipe_addrs=[push_pipe_addr], tokenizer_dir=tokenizer_dir, record_creator=ResponsePostprocessWorker_record_creator) worker.start() diff --git a/tests/unittest/llmapi/test_llm.py b/tests/unittest/llmapi/test_llm.py index 85d9eaf95797..e010a23809bd 100644 --- a/tests/unittest/llmapi/test_llm.py +++ b/tests/unittest/llmapi/test_llm.py @@ -17,7 +17,8 @@ from tensorrt_llm.bindings import executor as tllm from tensorrt_llm.executor import GenerationResultBase, RequestError from tensorrt_llm.llmapi import (KvCacheConfig, KvCacheRetentionConfig, - LookaheadDecodingConfig, RequestOutput) + LookaheadDecodingConfig, RequestOutput, + SADecodingConfig) from tensorrt_llm.llmapi.llm_args import DynamicBatchConfig, SchedulerConfig from tensorrt_llm.llmapi.llm_utils import _ParallelConfig from tensorrt_llm.llmapi.tokenizer import (TokenizerBase, TransformersTokenizer, @@ -1106,6 +1107,38 @@ def _test_llm_capture_request_error(pytorch_backend: bool, tp_size: int = 1): llm.generate(prompt) +def test_oversized_prompt_rejected_with_inflated_max_seq_len(): + """Oversized prompt is rejected gracefully when max_seq_len is inflated internally. + + With speculative decoding, KV cache max_seq_len is inflated to reserve space + for draft tokens. Admission control must use the net (user-configured) limit, + not the inflated value. Otherwise oversized prompts crash the event loop. + """ + max_seq_len = 256 + + llm = LLM( + model=llama_model_path, + max_seq_len=max_seq_len, + max_batch_size=1, + max_num_tokens=(max_seq_len + 32), + kv_cache_config=KvCacheConfig(enable_block_reuse=False), + speculative_config=SADecodingConfig(max_draft_len=4), + disable_overlap_scheduler=True, + ) + + # Prompt that fills the entire net budget -> 0 tokens left for output -> rejected + oversized_prompt_ids = [1] * (max_seq_len + 1) + with pytest.raises(RequestError): + llm.generate([oversized_prompt_ids], + sampling_params=SamplingParams(max_tokens=1)) + + # Executor must still be alive + result = llm.generate(["Hello"], + sampling_params=SamplingParams(max_tokens=5)) + assert len(result) == 1 + assert len(result[0].outputs[0].text) > 0 + + def test_llm_shutdown_executor(): llm = LLM( model=llama_model_path, @@ -1206,6 +1239,7 @@ class _FakeCompletionGeneratorArgs: gather_generation_logits = False num_postprocess_workers = 0 return_perf_metrics = False + reasoning_parser = None class _FakeModelConfig: @@ -1304,6 +1338,7 @@ async def run_request(): server.tokenizer = None server.metrics_collector = None server.perf_metrics = None + server._input_proc_executor = None request = CompletionRequest(model="test-model", prompt=["A", "B"], diff --git a/tests/unittest/llmapi/test_llm_args.py b/tests/unittest/llmapi/test_llm_args.py index c77fa26d7fd9..0edebc20c181 100644 --- a/tests/unittest/llmapi/test_llm_args.py +++ b/tests/unittest/llmapi/test_llm_args.py @@ -18,6 +18,7 @@ from utils.util import force_ampere import tensorrt_llm.bindings.executor as tle +import tensorrt_llm.llmapi as public_llmapi import tensorrt_llm.llmapi.llm_args as llm_args_mod from tensorrt_llm import LLM as TorchLLM from tensorrt_llm._torch.auto_deploy.llm_args import \ @@ -34,6 +35,7 @@ DecodeCudaGraphConfig, DecodingBaseConfig, DeepSeekV4SparseAttentionConfig, + DSparkDecodingConfig, DynamicBatchConfig, Eagle3DecodingConfig, EagleDecodingConfig, @@ -41,7 +43,8 @@ ExecutorMemoryType, ExtendedRuntimePerfKnobConfig, KvCacheConfig, - LookaheadDecodingConfig, MoeConfig, + LookaheadDecodingConfig, + MambaStateConfig, MoeConfig, MTPDecodingConfig, MultimodalConfig, MultimodalEncoderCudaGraphConfig, PeftCacheConfig, PybindMirror, @@ -51,7 +54,8 @@ StrictBaseModel, TorchCompileConfig, TorchLlmArgs, UserProvidedDecodingConfig, - update_llm_args_with_extra_dict) + update_llm_args_with_extra_dict, + update_llm_args_with_extra_options) # fmt: on from tensorrt_llm.llmapi.llm_utils import (_resolve_kv_cache_manager_v2_auto, _resolve_transceiver_runtime_auto, @@ -106,6 +110,53 @@ def test_MTPDecodingConfig_default_draft_len_is_not_user_set(): assert "max_draft_len" in explicit_config.model_fields_set +def test_rejection_sampling_allows_attention_dp(monkeypatch): + """ADP (incl. ADP+LM-head-TP) supports rejection sampling. + + The draft path bypasses LM-head-TP for advanced sampling and the greedy + flag is group-synchronized, so the former attention-DP gate is lifted. + """ + import tensorrt_llm._torch.flashinfer_utils as fi_utils + monkeypatch.setattr(fi_utils, "IS_FLASHINFER_AVAILABLE", True) + + # Vanilla MTP is one of the "newly wired" methods the parallel gate used + # to cover: lifting the ADP gate is the actual behavior change here. + for lm_head_tp in (False, True): + spec_cfg = MTPDecodingConfig(max_draft_len=2, + use_rejection_sampling=True, + use_mtp_vanilla=True) + args = TorchLlmArgs(model=llama_model_path, + enable_attention_dp=True, + enable_lm_head_tp_in_adp=lm_head_tp, + speculative_config=spec_cfg) + assert args.speculative_config.use_rejection_sampling is True + + # MTP-Eagle (default) was never parallel-gated; keep it as a regression + # guard that the gate rework did not accidentally start rejecting it. + spec_cfg = MTPDecodingConfig(max_draft_len=2, use_rejection_sampling=True) + args = TorchLlmArgs(model=llama_model_path, + enable_attention_dp=True, + enable_lm_head_tp_in_adp=True, + speculative_config=spec_cfg) + assert args.speculative_config.use_rejection_sampling is True + + +def test_rejection_sampling_still_gated_on_context_parallel(): + """Context parallelism remains an unsupported rejection combination. + + Explicit opt-in raises; default-inherited silently disables. The parallel + gate applies to the newly wired methods (vanilla MTP, PARD, DFlash, + DraftTarget one-model), so use vanilla MTP here. + """ + spec_cfg = MTPDecodingConfig(max_draft_len=2, + use_rejection_sampling=True, + use_mtp_vanilla=True) + with pytest.raises(ValueError, match="context parallelism"): + TorchLlmArgs(model=llama_model_path, + context_parallel_size=2, + speculative_config=spec_cfg) + + class TestYaml: def _yaml_to_dict(self, yaml_content: str) -> dict: @@ -132,6 +183,37 @@ def test_llm_args_with_kvcache_config(self): 1024, 1024, 1024 ] + def test_from_yaml_migrates_legacy_mamba_interval(self, tmp_path): + yaml_path = tmp_path / "legacy_mamba_interval.yaml" + yaml_path.write_text( + yaml.safe_dump({ + "model": str(llama_model_path), + "kv_cache_config": { + "mamba_state_cache_interval": 64, + }, + }), + encoding="utf-8", + ) + + llm_args = TorchLlmArgs.from_yaml(yaml_path) + + assert llm_args.kv_cache_config.mamba_state_config.periodic_snapshot_interval == 64 + + def test_from_yaml_empty_file_reports_missing_model(self, tmp_path): + yaml_path = tmp_path / "empty.yaml" + yaml_path.write_text("", encoding="utf-8") + + with pytest.raises(ValidationError, match="model"): + TorchLlmArgs.from_yaml(yaml_path) + + def test_from_yaml_rejects_non_mapping_root(self, tmp_path): + yaml_path = tmp_path / "list.yaml" + yaml_path.write_text("[]", encoding="utf-8") + + with pytest.raises(ValueError, + match="Configuration file root must be a mapping"): + TorchLlmArgs.from_yaml(yaml_path) + def test_llm_args_with_pydantic_options(self): yaml_content = """ max_batch_size: 16 @@ -241,6 +323,145 @@ def _capture_warning(msg, *args, **kwargs): for m in warnings_seen) +def test_dspark_block_size_resolved_from_checkpoint(tmp_path): + (tmp_path / "config.json").write_text('{"dspark_block_size": 5}') + spec_cfg = DSparkDecodingConfig(max_draft_len=5, + speculative_model=str(tmp_path)) + + args = TorchLlmArgs( + model="/tmp/dummy_model", + skip_tokenizer_init=True, + speculative_config=spec_cfg, + ) + + assert args.speculative_config.block_size == 5 + + +def test_dspark_block_size_must_match_max_draft_len(tmp_path): + (tmp_path / "config.json").write_text('{"dspark_block_size": 4}') + spec_cfg = DSparkDecodingConfig(max_draft_len=5, + speculative_model=str(tmp_path)) + + with pytest.raises(ValueError, match="block_size must equal max_draft_len"): + TorchLlmArgs( + model="/tmp/dummy_model", + skip_tokenizer_init=True, + speculative_config=spec_cfg, + ) + + +def test_dspark_target_layer_ids_resolved_from_checkpoint(tmp_path): + # When the user leaves target_layer_ids unset, the checkpoint's ordered + # dspark_target_layer_ids must be adopted verbatim. + (tmp_path / "config.json").write_text( + '{"dspark_block_size": 5, "dspark_target_layer_ids": [3, 1, 2]}') + spec_cfg = DSparkDecodingConfig(max_draft_len=5, + speculative_model=str(tmp_path)) + + args = TorchLlmArgs( + model="/tmp/dummy_model", + skip_tokenizer_init=True, + speculative_config=spec_cfg, + ) + + # Order is preserved (projection columns are order-dependent). + assert args.speculative_config.target_layer_ids == [3, 1, 2] + + +def test_dspark_target_layer_ids_matching_override_accepted(tmp_path): + # An explicit override that matches the checkpoint list exactly is fine. + (tmp_path / "config.json").write_text( + '{"dspark_block_size": 5, "dspark_target_layer_ids": [1, 2, 3]}') + spec_cfg = DSparkDecodingConfig(max_draft_len=5, + speculative_model=str(tmp_path), + target_layer_ids=[1, 2, 3]) + + args = TorchLlmArgs( + model="/tmp/dummy_model", + skip_tokenizer_init=True, + speculative_config=spec_cfg, + ) + + assert args.speculative_config.target_layer_ids == [1, 2, 3] + + +def test_dspark_target_layer_ids_mismatched_count_rejected(tmp_path): + # A different number of layers would mismatch main_proj.in_features. + (tmp_path / "config.json").write_text( + '{"dspark_block_size": 5, "dspark_target_layer_ids": [1, 2, 3]}') + spec_cfg = DSparkDecodingConfig(max_draft_len=5, + speculative_model=str(tmp_path), + target_layer_ids=[1, 2]) + + with pytest.raises(ValueError, match="must match the checkpoint"): + TorchLlmArgs( + model="/tmp/dummy_model", + skip_tokenizer_init=True, + speculative_config=spec_cfg, + ) + + +def test_dspark_target_layer_ids_same_count_different_layers_rejected(tmp_path): + # Same count but different layers: shapes line up, but the draft would see + # hidden states it was not trained on, so this must be rejected too. + (tmp_path / "config.json").write_text( + '{"dspark_block_size": 5, "dspark_target_layer_ids": [1, 2, 3]}') + spec_cfg = DSparkDecodingConfig(max_draft_len=5, + speculative_model=str(tmp_path), + target_layer_ids=[1, 2, 4]) + + with pytest.raises(ValueError, match="must match the checkpoint"): + TorchLlmArgs( + model="/tmp/dummy_model", + skip_tokenizer_init=True, + speculative_config=spec_cfg, + ) + + +def test_dspark_target_layer_ids_order_mismatch_rejected(tmp_path): + # Same set but different order: projection columns are order-dependent, so a + # reordered override must be rejected rather than silently accepted. + (tmp_path / "config.json").write_text( + '{"dspark_block_size": 5, "dspark_target_layer_ids": [1, 2, 3]}') + spec_cfg = DSparkDecodingConfig(max_draft_len=5, + speculative_model=str(tmp_path), + target_layer_ids=[3, 2, 1]) + + with pytest.raises(ValueError, match="must match the checkpoint"): + TorchLlmArgs( + model="/tmp/dummy_model", + skip_tokenizer_init=True, + speculative_config=spec_cfg, + ) + + +def test_dspark_requires_speculative_model(): + # The DSpark draft weights live in the checkpoint's mtp.* namespace, so an + # unset speculative_model must fail fast at config validation instead of + # raising an opaque TypeError deep inside engine construction. + spec_cfg = DSparkDecodingConfig(max_draft_len=5) + + with pytest.raises(ValueError, + match="requires speculative_config.speculative_model"): + TorchLlmArgs( + model="/tmp/dummy_model", + skip_tokenizer_init=True, + speculative_config=spec_cfg, + ) + + +def test_dspark_requires_positive_max_draft_len(tmp_path): + (tmp_path / "config.json").write_text('{"dspark_block_size": 5}') + spec_cfg = DSparkDecodingConfig(speculative_model=str(tmp_path)) + + with pytest.raises(ValueError, match="max_draft_len must be > 0"): + TorchLlmArgs( + model="/tmp/dummy_model", + skip_tokenizer_init=True, + speculative_config=spec_cfg, + ) + + def test_post_processor_hook_rejected_with_skip_tokenizer_init(): """post_processor_hook + skip_tokenizer_init must fail fast. @@ -273,7 +494,9 @@ def test_kv_cache_manager_v2_auto_uses_model_default(self, explicit_auto): model_defaults = {"kv_cache_config": {"use_kv_cache_manager_v2": True}} apply_model_defaults_to_llm_args(llm_args, model_defaults) - _resolve_kv_cache_manager_v2_auto(llm_args, model_defaults) + _resolve_kv_cache_manager_v2_auto(llm_args, + model_defaults, + original_setting="auto") assert llm_args.kv_cache_config.use_kv_cache_manager_v2 is True @@ -284,6 +507,45 @@ def test_kv_cache_manager_v2_auto_falls_back_to_false(self): assert llm_args.kv_cache_config.use_kv_cache_manager_v2 is False + @pytest.mark.parametrize( + ("backend", "runtime"), + [ + ("NIXL", "CPP"), + ("UCX", None), + ("MPI", None), + ], + ) + def test_kv_cache_manager_v2_auto_falls_back_for_incompatible_disagg( + self, backend, runtime): + llm_args = TorchLlmArgs( + model="/tmp/dummy_model", + cache_transceiver_config=CacheTransceiverConfig( + backend=backend, transceiver_runtime=runtime), + ) + model_defaults = {"kv_cache_config": {"use_kv_cache_manager_v2": True}} + + apply_model_defaults_to_llm_args(llm_args, model_defaults) + _resolve_kv_cache_manager_v2_auto(llm_args, + model_defaults, + original_setting="auto") + + assert llm_args.kv_cache_config.use_kv_cache_manager_v2 is False + + def test_kv_cache_manager_v2_auto_keeps_python_nixl_model_default(self): + llm_args = TorchLlmArgs( + model="/tmp/dummy_model", + cache_transceiver_config=CacheTransceiverConfig( + backend="NIXL", transceiver_runtime="PYTHON"), + ) + model_defaults = {"kv_cache_config": {"use_kv_cache_manager_v2": True}} + + apply_model_defaults_to_llm_args(llm_args, model_defaults) + _resolve_kv_cache_manager_v2_auto(llm_args, + model_defaults, + original_setting="auto") + + assert llm_args.kv_cache_config.use_kv_cache_manager_v2 is True + @pytest.mark.parametrize("user_setting", [False, True]) def test_kv_cache_manager_v2_explicit_value_overrides_model_default( self, user_setting): @@ -399,6 +661,8 @@ def get_model_defaults(cls, llm_args): def test_KvCacheConfig_declaration(): + assert KvCacheConfig().mamba_state_cache_interval is None + assert KvCacheConfig().mamba_state_config.periodic_snapshot_interval == 0 assert KvCacheConfig().kv_cache_event_hash_algo == "auto" assert KvCacheConfig().block_reuse_policy == "all_reusable" assert KvCacheConfig().enable_swa_scratch_reuse is False @@ -409,6 +673,8 @@ def test_KvCacheConfig_declaration(): use_kv_cache_manager_v2=False).use_kv_cache_manager_v2 is False with pytest.raises(ValidationError, match="use_kv_cache_manager_v2"): KvCacheConfig(use_kv_cache_manager_v2="invalid") + with pytest.raises(ValidationError, match="max_util_for_resume"): + KvCacheConfig(max_util_for_resume=0) config = KvCacheConfig(enable_block_reuse=True, max_tokens=1024, @@ -424,6 +690,11 @@ def test_KvCacheConfig_declaration(): enable_swa_scratch_reuse=True, enable_partial_reuse=True, copy_on_partial_reuse=True, + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=0, + additional_snapshot_offsets_from_start=[128], + additional_snapshot_offsets_from_end=[0, 32], + ), pool_ratio=[0.25, 0.75], avg_seq_len=2048, block_reuse_policy="per_request", @@ -446,6 +717,13 @@ def test_KvCacheConfig_declaration(): assert config.pool_ratio == [0.25, 0.75] assert config.avg_seq_len == 2048 assert config.block_reuse_policy == "per_request" + assert config.mamba_state_config.periodic_snapshot_interval == 0 + assert config.mamba_state_config.additional_snapshot_offsets_from_start == [ + 128 + ] + assert config.mamba_state_config.additional_snapshot_offsets_from_end == [ + 0, 32 + ] assert not hasattr(pybind_config, "pool_ratio") assert not hasattr(pybind_config, "avg_seq_len") assert not hasattr(pybind_config, "block_reuse_policy") @@ -461,10 +739,155 @@ def test_KvCacheConfig_declaration(): assert pybind_config.enable_partial_reuse == True assert pybind_config.copy_on_partial_reuse == True assert pybind_config.attention_dp_events_gather_period_ms == 10 + assert (KvCacheConfig(block_reuse_policy="per_conversation"). + block_reuse_policy == "per_conversation") with pytest.raises(ValidationError): KvCacheConfig(block_reuse_policy="invalid") +def test_MambaStateConfig_defaults_use_independent_lists(): + first = MambaStateConfig() + second = MambaStateConfig() + + assert first.periodic_snapshot_interval == 0 + first.additional_snapshot_offsets_from_start.append(128) + first.additional_snapshot_offsets_from_end.append(0) + + assert second.additional_snapshot_offsets_from_start == [] + assert second.additional_snapshot_offsets_from_end == [] + assert public_llmapi.MambaStateConfig is MambaStateConfig + + +def test_MambaStateConfig_rejects_unknown_fields(): + with pytest.raises(ValidationError, match="extra_forbidden"): + MambaStateConfig(unknown_snapshot_policy=1) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("periodic_snapshot_interval", -1), + ("additional_snapshot_offsets_from_start", [0]), + ("additional_snapshot_offsets_from_start", [True]), + ("additional_snapshot_offsets_from_end", [-1]), + ("additional_snapshot_offsets_from_end", [1.5]), + ], +) +def test_MambaStateConfig_rejects_invalid_snapshot_offsets(field, value): + with pytest.raises(ValidationError, match=field): + MambaStateConfig(**{field: value}) + + +@pytest.mark.parametrize( + ("field", "offsets"), + [ + ("additional_snapshot_offsets_from_start", [128]), + ("additional_snapshot_offsets_from_end", [0]), + ], +) +def test_KvCacheConfig_requires_v2_for_additional_snapshot_offsets( + field, offsets): + state_config = MambaStateConfig(**{field: offsets}) + + with pytest.raises(ValidationError, match="use_kv_cache_manager_v2=True"): + KvCacheConfig( + mamba_state_config=state_config, + use_kv_cache_manager_v2=False, + ) + + config = KvCacheConfig( + mamba_state_config=state_config, + use_kv_cache_manager_v2=True, + ) + assert getattr(config.mamba_state_config, field) == offsets + + +def test_KvCacheConfig_migrates_deprecated_mamba_interval(monkeypatch): + warnings_seen = [] + monkeypatch.setattr(llm_args_mod.logger, "warning", + lambda message: warnings_seen.append(message)) + + config = KvCacheConfig(mamba_state_cache_interval=64) + + assert config.mamba_state_cache_interval == 64 + assert config.mamba_state_config.periodic_snapshot_interval == 64 + assert any("mamba_state_cache_interval' is deprecated" in message + for message in warnings_seen) + assert "mamba_state_cache_interval" not in config.model_dump() + + +def test_KvCacheConfig_warns_when_disabling_periodic_conversation_snapshots( + monkeypatch): + warnings_seen = [] + monkeypatch.setattr(llm_args_mod.logger, "warning", + lambda message: warnings_seen.append(message)) + + config = KvCacheConfig( + block_reuse_policy="per_conversation", + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=64, + additional_snapshot_offsets_from_end=[0], + ), + ) + + assert config.mamba_state_config.periodic_snapshot_interval == 0 + assert config.mamba_state_config.additional_snapshot_offsets_from_end == [0] + assert len(warnings_seen) == 1 + assert "periodic_snapshot_interval=64" in warnings_seen[0] + assert "block_reuse_policy=per_conversation" in warnings_seen[0] + assert "setting it to 0" in warnings_seen[0] + + warnings_seen.clear() + KvCacheConfig(block_reuse_policy="per_conversation") + assert warnings_seen == [] + + +def test_update_llm_args_with_empty_options_file(tmp_path): + yaml_path = tmp_path / "empty.yaml" + yaml_path.write_text("", encoding="utf-8") + llm_args = {"model": "dummy"} + + assert update_llm_args_with_extra_options(llm_args, + str(yaml_path)) == llm_args + + +def test_config_file_merge_migrates_legacy_mamba_interval_without_mutating_input( +): + yaml_dict = { + "kv_cache_config": { + "mamba_state_cache_interval": 64, + "mamba_state_config": { + "additional_snapshot_offsets_from_end": [0], + }, + }, + } + + merged = update_llm_args_with_extra_dict({"model": "dummy"}, yaml_dict) + + assert merged[ + "kv_cache_config"].mamba_state_config.periodic_snapshot_interval == 64 + assert merged[ + "kv_cache_config"].mamba_state_config.additional_snapshot_offsets_from_end == [ + 0 + ] + assert yaml_dict["kv_cache_config"]["mamba_state_cache_interval"] == 64 + + +def test_config_file_merge_rejects_legacy_and_new_mamba_intervals(): + with pytest.raises(ValueError, match="Cannot set both"): + update_llm_args_with_extra_dict( + {"model": "dummy"}, + { + "kv_cache_config": { + "mamba_state_cache_interval": 64, + "mamba_state_config": { + "periodic_snapshot_interval": 64, + }, + }, + }, + ) + + def test_KvCacheConfig_disk_cache_validation(tmp_path): config = KvCacheConfig(disk_cache_size=2048, disk_cache_path=str(tmp_path)) @@ -558,10 +981,9 @@ def test_encoder_cache_max_bytes_rejects_invalid_values(self, value): def test_torch_llm_args_with_encoder_side_stream_max_ahead(self): args = TorchLlmArgs(model=llama_model_path, multimodal_config=MultimodalConfig( - encoder_side_stream_max_ahead=2, - encoder_cache_max_bytes=0, - )) + encoder_side_stream_max_ahead=2, )) assert args.multimodal_config.encoder_side_stream_max_ahead == 2 + assert args.multimodal_config.encoder_cache_max_bytes == 128 * 1024**2 def test_torch_llm_args_with_multimodal_video_pruning_rate(self): args = TorchLlmArgs( @@ -622,12 +1044,14 @@ def test_encoder_cuda_graph_and_side_stream_max_ahead_are_exclusive(self): }, ) - def test_encoder_cache_and_side_stream_max_ahead_are_exclusive(self): - with pytest.raises(ValidationError, match="mutually exclusive"): - MultimodalConfig( - encoder_cache_max_bytes="1MiB", - encoder_side_stream_max_ahead=1, - ) + def test_encoder_cache_and_side_stream_max_ahead_can_be_combined(self): + config = MultimodalConfig( + encoder_cache_max_bytes="1MiB", + encoder_side_stream_max_ahead=1, + ) + + assert config.encoder_cache_max_bytes == 1024**2 + assert config.encoder_side_stream_max_ahead == 1 @pytest.mark.parametrize("kwargs", [ @@ -1368,18 +1792,19 @@ class TestPiecewiseCudaGraphCaptureDefaults: powers-of-2 + 256-stride list when `enable_piecewise_cuda_graph` is True (and stays `None` otherwise). The fixed list keeps the capture set small to bound startup time and CUDA graph memory; - the model-engine filter (invariants 2 and 3) ensures the largest - reachable size is always captured even when it is not in this - default list. + the model-engine filter (invariants 2 and 3) clamps out-of-range + entries to the reachable ceiling and never invents sizes beyond + this list. 2. `_filter_piecewise_capture_num_tokens` caps the candidate list at `max_batch_size * (max_seq_len - 1 - num_extra_decoding_steps)` -- the largest forward-pass `num_tokens` the warmup builder can construct, since every in-flight request must leave room for at least one decode token. - 3. The reachable ceiling itself is always present in the returned - capture set (when positive), so runtime ISLs in the gap between - the next-largest candidate and the ceiling get a graph rather - than falling back to eager. + 3. Candidates above the reachable ceiling are clamped down to the + ceiling (a requested 128 becomes 127), and no size beyond the + user's list is ever invented (an appended far ceiling would make + runtime padding execute the full ceiling shape for every + iteration in the gap). """ _EXPECTED_DEFAULT_CAPTURE_NUM_TOKENS = [2**i for i in range(8)] + list( @@ -1436,14 +1861,50 @@ def test_torch_llm_args_capture_num_tokens_default_when_piecewise_enabled( ) assert args.torch_compile_config.capture_num_tokens == self._EXPECTED_DEFAULT_CAPTURE_NUM_TOKENS - def test_piecewise_filter_drops_entries_above_reachable_ceiling(self): - """Drop candidates above `max_batch_size * (max_seq_len - 1)`. + def test_piecewise_filter_never_invents_far_ceiling(self): + """A ceiling far above the largest candidate is NOT added. - Without the cap, the warmup loop would silently skip these entries - and the outer padding logic would pad to a target with no captured - graph. They must be removed from `kept` and surfaced in - `unrecordable` so the warning fires. The ceiling itself is then - appended so ISLs in the gap still get a graph. + Runtime padding rounds each iteration up to the nearest captured + size, so an invented far ceiling (e.g. 65536 over a list topping + out at 13914) would make every iteration in the gap execute the + full ceiling shape. The filter must never invent sizes the user + did not request. + """ + from tensorrt_llm._torch.pyexecutor.model_engine import \ + _filter_piecewise_capture_num_tokens + + candidates = [512, 1024, 2048, 4096, 8192, 13914] + kept, unrecordable = _filter_piecewise_capture_num_tokens( + candidates, + max_num_tokens=65536, + max_batch_size=896, + max_seq_len=32768, + ) + assert kept == candidates + assert unrecordable == [] + + def test_piecewise_filter_clamps_multiple_oversized_candidates(self): + """All above-ceiling candidates collapse to one ceiling entry.""" + from tensorrt_llm._torch.pyexecutor.model_engine import \ + _filter_piecewise_capture_num_tokens + + kept, unrecordable = _filter_piecewise_capture_num_tokens( + [64, 120, 128, 200, 256], + max_num_tokens=256, + max_batch_size=1, + max_seq_len=128, + ) + # Ceiling: 1 * (128 - 1) = 127; 128/200/256 clamp to 127, deduped. + assert kept == [64, 120, 127] + assert unrecordable == [128, 200, 256] + + def test_piecewise_filter_clamps_entries_above_reachable_ceiling(self): + """Clamp candidates above `max_batch_size * (max_seq_len - 1)`. + + Entries above the ceiling cannot be recorded by the warmup loop; + they are clamped down to the ceiling and surfaced in + `unrecordable` so the warning fires. ISLs in the gap still get a + graph at the nearest recordable size. """ from tensorrt_llm._torch.pyexecutor.model_engine import \ _filter_piecewise_capture_num_tokens @@ -1501,9 +1962,8 @@ def test_piecewise_filter_subtracts_extra_decoding_steps(self): Drafting loops consume extra decode steps; the filter must mirror the `max_seq_len - 1 - num_extra_decoding_steps` constraint - applied when warmup requests are built. The ceiling is appended - whenever it is strictly greater than the largest surviving - candidate. + applied when warmup requests are built. Candidates above the + reduced ceiling are clamped down to it; nothing is appended. """ from tensorrt_llm._torch.pyexecutor.model_engine import \ _filter_piecewise_capture_num_tokens @@ -1517,7 +1977,7 @@ def test_piecewise_filter_subtracts_extra_decoding_steps(self): max_seq_len=128, num_extra_decoding_steps=5, ) - assert kept[-1] == 122 + assert kept[-1] == 120 # nothing above the 122 ceiling to clamp assert 120 in kept assert unrecordable == [] # Same setup with 9 extra decoding steps -> ceiling 118; 120 drops. @@ -1565,9 +2025,12 @@ def test_piecewise_filter_returns_empty_when_ceiling_is_zero(self): assert kept == [] assert unrecordable == [1, 2, 4] - def test_piecewise_filter_appends_ceiling_when_only_smaller_candidates( - self): - """No candidate near the ceiling -> ceiling still appended.""" + def test_piecewise_filter_keeps_small_candidates_unchanged(self): + """No candidate above the ceiling -> the list is used as-is. + + The ceiling (1016 here) is not appended; iterations above the + largest candidate run eagerly at their true size. + """ from tensorrt_llm._torch.pyexecutor.model_engine import \ _filter_piecewise_capture_num_tokens @@ -1577,8 +2040,8 @@ def test_piecewise_filter_appends_ceiling_when_only_smaller_candidates( max_batch_size=8, max_seq_len=128, ) - # Ceiling: 8 * (128 - 1) = 1016. - assert kept == [1, 2, 4, 8, 1016] + # Ceiling: 8 * (128 - 1) = 1016 -- far above max candidate 8. + assert kept == [1, 2, 4, 8] class TestTorchLlmArgs: @@ -2893,6 +3356,126 @@ def get_preferred_transceiver_runtime(cls, pretrained_config=None): return None +class _NoModelDefaults: + + @classmethod + def get_model_defaults(cls, llm_args): + return {} + + +class TestMambaSnapshotConfigResolution: + + @staticmethod + def _load_config(monkeypatch, args, architecture): + from unittest.mock import MagicMock + + from tensorrt_llm._torch.pyexecutor import \ + model_loader as model_loader_mod + + monkeypatch.setattr( + model_loader_mod.AutoModelForCausalLM, + "_resolve_class", + staticmethod(lambda config: _NoModelDefaults), + ) + fake_loader = MagicMock() + fake_config = MagicMock() + fake_config.pretrained_config.architectures = [architecture] + fake_config.pretrained_config.hybrid_override_pattern = None + fake_loader.load_config.return_value = fake_config + return model_loader_mod.ModelLoader.load_config_and_apply_defaults( + "/tmp/dummy_model", args, fake_loader) + + @staticmethod + def _capture_warnings(monkeypatch): + from tensorrt_llm._torch.pyexecutor import \ + model_loader as model_loader_mod + + messages = [] + monkeypatch.setattr( + model_loader_mod.logger, "warning", + lambda message, *args: messages.append(message % args + if args else message)) + return messages + + @pytest.mark.parametrize( + ("kv_cache_config", "expected_reuse", "expected_warning"), + [ + (KvCacheConfig(), False, True), + ( + KvCacheConfig( + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=64), + use_kv_cache_manager_v2=False, + ), + True, + False, + ), + ( + KvCacheConfig( + mamba_state_config=MambaStateConfig( + additional_snapshot_offsets_from_end=[0]), + use_kv_cache_manager_v2=True, + ), + True, + False, + ), + ( + KvCacheConfig( + block_reuse_policy="per_conversation", + mamba_state_config=MambaStateConfig( + periodic_snapshot_interval=64), + use_kv_cache_manager_v2=True, + ), + False, + True, + ), + ], + ids=["none", "periodic", "fixed", "per-conversation-no-fixed"], + ) + def test_hybrid_snapshot_policy_controls_block_reuse( + self, + monkeypatch, + kv_cache_config, + expected_reuse, + expected_warning, + ): + args = TorchLlmArgs(model="/tmp/dummy_model", + kv_cache_config=kv_cache_config) + warnings = self._capture_warnings(monkeypatch) + + self._load_config(monkeypatch, args, "Qwen3NextForCausalLM") + + assert args.kv_cache_config.enable_block_reuse is expected_reuse + assert bool(warnings) is expected_warning + if expected_warning: + assert "no Mamba state snapshot policy" in warnings[0] + + def test_hybrid_fixed_snapshot_rejects_auto_resolved_v1(self, monkeypatch): + args = TorchLlmArgs( + model="/tmp/dummy_model", + kv_cache_config=KvCacheConfig( + mamba_state_config=MambaStateConfig( + additional_snapshot_offsets_from_start=[128]), + use_kv_cache_manager_v2="auto", + ), + ) + + with pytest.raises( + ValueError, + match="use_kv_cache_manager_v2=True after resolving"): + self._load_config(monkeypatch, args, "Qwen3NextForCausalLM") + + def test_non_hybrid_without_snapshot_policy_preserves_block_reuse( + self, monkeypatch): + args = TorchLlmArgs(model="/tmp/dummy_model") + warnings = self._capture_warnings(monkeypatch) + + self._load_config(monkeypatch, args, "LlamaForCausalLM") + + assert args.kv_cache_config.enable_block_reuse is True + assert warnings == [] + + class TestTransceiverRuntimeAutoResolution: """Tests for the transceiver_runtime 'auto' selection mechanism.""" @@ -2951,13 +3534,32 @@ def test_default_backend_resolves_to_nixl_and_adopts_preference( # creation time. assert args.cache_transceiver_config.backend == "DEFAULT" - def test_default_backend_env_override_falls_back_to_cpp(self, monkeypatch): - """DEFAULT + TRTLLM_USE_UCX_KVCACHE=1 means effective UCX -> C++.""" - monkeypatch.delenv("TRTLLM_USE_NIXL_KVCACHE", raising=False) - monkeypatch.setenv("TRTLLM_USE_UCX_KVCACHE", "1") + @pytest.mark.parametrize( + "backend_env", + ["TRTLLM_USE_UCX_KVCACHE", "TRTLLM_USE_MPI_KVCACHE"], + ) + def test_default_backend_env_override_falls_back_to_v1_cpp( + self, monkeypatch, backend_env): + """An incompatible DEFAULT route falls back to the V1 C++ path.""" + for env_var in ( + "TRTLLM_USE_NIXL_KVCACHE", + "TRTLLM_USE_UCX_KVCACHE", + "TRTLLM_USE_MOONCAKE_KVCACHE", + "TRTLLM_USE_MPI_KVCACHE", + ): + monkeypatch.delenv(env_var, raising=False) + monkeypatch.setenv(backend_env, "1") args = self._disagg_args(backend="DEFAULT") + model_defaults = {"kv_cache_config": {"use_kv_cache_manager_v2": True}} + apply_model_defaults_to_llm_args(args, model_defaults) + _resolve_transceiver_runtime_auto(args, _PreferPythonTransceiverModel) + _resolve_kv_cache_manager_v2_auto(args, + model_defaults, + original_setting="auto") + assert args.cache_transceiver_config.transceiver_runtime is None + assert args.kv_cache_config.use_kv_cache_manager_v2 is False def test_disagg_disabled_is_noop(self): """Resolver never creates a config when cache_transceiver_config is None.""" @@ -3157,3 +3759,60 @@ def test_resolve_default_backend_env_priority(self, monkeypatch): # An explicit backend bypasses the env vars entirely. assert CacheTransceiverConfig( backend="UCX")._resolve_default_backend() == ("UCX", None) + + +class TestGlm5TransceiverPreference: + """GLM-5 defaults to the Python KV-cache transceiver in disagg. + + DeepseekV3ForCausalLM is shared by DeepSeek-V3/V3.2 and GLM-5 + (GlmMoeDsaForCausalLM); the preference must apply to GLM checkpoints + only. + """ + + @staticmethod + def _pretrained_config(architectures, model_type): + from transformers import PretrainedConfig + cfg = PretrainedConfig(architectures=architectures) + cfg.model_type = model_type + return cfg + + @pytest.mark.parametrize( + "architectures,model_type,expected", + [ + (["GlmMoeDsaForCausalLM"], "glm_moe_dsa", "PYTHON"), + (["DeepseekV3ForCausalLM"], "deepseek_v3", None), + (["DeepseekV32ForCausalLM"], "deepseek_v32", None), + # Each predicate in isolation: the architecture match and the + # model_type fallback must each suffice on their own. + (["GlmMoeDsaForCausalLM"], "deepseek_v32", "PYTHON"), + (["DeepseekV32ForCausalLM"], "glm_moe_dsa", "PYTHON"), + ]) + def test_preference_per_architecture(self, architectures, model_type, + expected): + from tensorrt_llm._torch.models.modeling_deepseekv3 import \ + DeepseekV3ForCausalLM + cfg = self._pretrained_config(architectures, model_type) + assert DeepseekV3ForCausalLM.get_preferred_transceiver_runtime( + cfg) == expected + + def test_no_config_defers_to_cpp(self): + """Without a pretrained config the class defers to the C++ default.""" + from tensorrt_llm._torch.models.modeling_deepseekv3 import \ + DeepseekV3ForCausalLM + assert DeepseekV3ForCausalLM.get_preferred_transceiver_runtime() is None + + def test_glm5_resolves_auto_to_python_on_nixl(self): + """GLM-5 on NIXL adopts the Python transceiver from 'auto'. + + End-to-end through _resolve_transceiver_runtime_auto with the real + model class and a GLM pretrained config. + """ + from tensorrt_llm._torch.models.modeling_deepseekv3 import \ + DeepseekV3ForCausalLM + args = TorchLlmArgs( + model="/tmp/dummy_model", + cache_transceiver_config=CacheTransceiverConfig(backend="NIXL"), + ) + cfg = self._pretrained_config(["GlmMoeDsaForCausalLM"], "glm_moe_dsa") + _resolve_transceiver_runtime_auto(args, DeepseekV3ForCausalLM, cfg) + assert args.cache_transceiver_config.transceiver_runtime == "PYTHON" diff --git a/tests/unittest/llmapi/test_llm_kv_cache_events.py b/tests/unittest/llmapi/test_llm_kv_cache_events.py index b170cd01219c..d2447c1aea2f 100644 --- a/tests/unittest/llmapi/test_llm_kv_cache_events.py +++ b/tests/unittest/llmapi/test_llm_kv_cache_events.py @@ -276,8 +276,8 @@ def test_apply_mm_hashes_with_uuids(): 0] # UUID changes hash # Second hash should be content-only (same as without UUID) assert hashes_partial["image"][1] == hashes_no_uuid["image"][1] - # UUIDs list should have the UUID and None - assert uuids_partial == ["sku-1234-a", None] + # UUIDs are returned per-modality, parallel to mm_hashes. + assert uuids_partial == {"image": ["sku-1234-a", None]} # Test with all UUIDs mm_uuids_all = {"image": ["sku-1234-a", "sku-1234-b"]} @@ -290,7 +290,7 @@ def test_apply_mm_hashes_with_uuids(): assert hashes_all["image"][1] != hashes_no_uuid["image"][1] # Different UUIDs with different content should produce different hashes assert hashes_all["image"][0] != hashes_all["image"][1] - assert uuids_all == ["sku-1234-a", "sku-1234-b"] + assert uuids_all == {"image": ["sku-1234-a", "sku-1234-b"]} def test_apply_mm_hashes_uuid_content_combined(): @@ -781,8 +781,9 @@ def test_apply_mm_hashes_multiple_modalities(): assert hashes["image"][0] != hashes_no_uuid["image"][0] assert hashes["video"][0] != hashes_no_uuid["video"][0] - # Check flattened UUID list (order may vary based on dict iteration) - assert set(uuids_list) == {"img-uuid-001", "vid-uuid-001"} + # UUIDs are returned per-modality; flatten across modalities to compare. + flat_uuids = [u for lst in uuids_list.values() for u in lst] + assert set(flat_uuids) == {"img-uuid-001", "vid-uuid-001"} def test_mm_keys_in_stored_events(): diff --git a/tests/unittest/llmapi/test_llm_pytorch.py b/tests/unittest/llmapi/test_llm_pytorch.py index f1fb240e1395..bbfbc3a89885 100644 --- a/tests/unittest/llmapi/test_llm_pytorch.py +++ b/tests/unittest/llmapi/test_llm_pytorch.py @@ -1187,7 +1187,7 @@ def test_min_tokens(use_speculative: bool): def test_min_tokens_long_prompt(): """Check min_tokens is respected when prompt is longer than min_tokens. - Regression test for NVBug 5823135: _apply_min_length_penalty compared + Regression test for NVBug 5823135: the min-length EOS suppression compared total token count (prompt + generated) against the raw min_tokens value instead of comparing generated token count only. When prompt_len >= min_tokens the EOS suppression was never activated, allowing early @@ -1546,18 +1546,22 @@ def test_llm_context_only_timed_out(transceiver_runtime): disaggregated_params=disaggregated_params): print(output) + # Wait until the context-only request has allocated KV cache blocks max_retries = 10 + all_results = [] for _ in range(max_retries): results = llm.get_stats(2) - if len(results) == 1: + all_results.extend(results) + if all_results and all_results[-1]["kvCacheStats"]["usedNumBlocks"] > 0: break time.sleep(1) else: pytest.fail( - f"Failed to get stats with len==1 after {max_retries} retries") + f"Context-only KV cache blocks not allocated after {max_retries} retries" + ) + results = all_results - assert len(results) == 1 - context_only_used_num_blocks = results[0]["kvCacheStats"]["usedNumBlocks"] + context_only_used_num_blocks = results[-1]["kvCacheStats"]["usedNumBlocks"] print(f"Context only used num blocks: {context_only_used_num_blocks}") # Sleep 5 seconds to allow context only request to time out @@ -1567,11 +1571,20 @@ def test_llm_context_only_timed_out(transceiver_runtime): for output in llm.generate(prompts0, sampling_params=sampling_params): print(output) - # Get number of allocated blocks - results = llm.get_stats(2) - assert len(results) == 1 - final_used_num_blocks = results[0]["kvCacheStats"]["usedNumBlocks"] + # Wait until KV cache blocks are released (usedNumBlocks == 0) + max_retries = 10 + all_results = [] + for _ in range(max_retries): + results = llm.get_stats(2) + all_results.extend(results) + if all_results and all_results[-1]["kvCacheStats"]["usedNumBlocks"] == 0: + break + time.sleep(1) + else: + pytest.fail(f"KV cache blocks not released after {max_retries} retries") + results = all_results + final_used_num_blocks = results[-1]["kvCacheStats"]["usedNumBlocks"] assert final_used_num_blocks == 0 diff --git a/tests/unittest/llmapi/test_mpi_session.py b/tests/unittest/llmapi/test_mpi_session.py index 781947fe65bf..f0db0d2ba99e 100644 --- a/tests/unittest/llmapi/test_mpi_session.py +++ b/tests/unittest/llmapi/test_mpi_session.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import os import subprocess # nosec B404 import sys @@ -10,8 +13,10 @@ import pytest from tensorrt_llm.bindings.BuildInfo import ENABLE_MULTI_DEVICE -from tensorrt_llm.llmapi.mpi_session import (MPINodeState, MpiPoolSession, +from tensorrt_llm.llmapi.mpi_session import (_DEFAULT_IDENTITY_TIMEOUT, + MPINodeState, MpiPoolSession, RemoteMpiCommSessionClient, + _identity_barrier_timeout, split_mpi_env) # isort: off @@ -166,3 +171,163 @@ def read_stream(stream, output_stream): if return_code != 0: raise subprocess.CalledProcessError(return_code, command) + + +# ---- wait_shutdown: shutdown blocks until worker processes actually exit ---- + + +def _wait_workers_exit(identities, timeout: float) -> None: + """Call the unbound method on an inert stand-in (no MPI spawn). + + ``_wait_workers_exit`` only reads ``self._worker_identities``; a real + ``MpiPoolSession`` shell would trigger the base class's abort machinery + at garbage collection. + """ + import types + + stand_in = types.SimpleNamespace(_worker_identities=identities) + MpiPoolSession._wait_workers_exit(stand_in, timeout=timeout) + + +def test_process_start_time_live_and_gone(): + from tensorrt_llm.llmapi.mpi_session import _process_start_time + + assert _process_start_time(os.getpid()) is not None + child = Popen(["true"]) # nosec B603, B607 + child.wait() + assert _process_start_time(child.pid) is None # reaped: /proc entry gone + + +def test_wait_workers_exit_returns_once_workers_are_gone(): + from tensorrt_llm.llmapi.mpi_session import _process_start_time + + child = Popen(["true"]) # nosec B603, B607 + identity = (child.pid, _process_start_time(child.pid)) + child.wait() + # Dead worker -> returns immediately; a None start_time is skipped + # (identity collection failed for that worker: nothing to wait on). + _wait_workers_exit((identity, (os.getpid(), None)), timeout=5.0) + + +def test_wait_workers_exit_bounded_by_timeout_on_live_worker(): + import time as _time + + from tensorrt_llm.llmapi.mpi_session import _process_start_time + + me = (os.getpid(), _process_start_time(os.getpid())) + t0 = _time.monotonic() + _wait_workers_exit((me, ), timeout=0.2) # this process will not exit + waited = _time.monotonic() - t0 + assert 0.2 <= waited < 2.0 # bounded: a wedged worker cannot hang teardown + + +def _collect_identities(monkeypatch, + results, + pending=0, + n_workers=2, + observed_timeouts=None): + """Drive _collect_worker_identities on an inert stand-in (no MPI spawn).""" + import types + from concurrent.futures import Future + + futs = [] + for r in results: + f = Future() + f.set_result(r) + futs.append(f) + never = [Future() for _ in range(pending)] # never resolve + + from tensorrt_llm.llmapi import mpi_session as m + + def _fake_wait(fs, timeout): + if observed_timeouts is not None: + observed_timeouts.append(timeout) + return futs, never + + monkeypatch.setattr(m, "futures_wait", _fake_wait) + killed = [] + monkeypatch.setattr(os, "kill", lambda pid, sig: killed.append(pid)) + it = iter(futs + never) + stand_in = types.SimpleNamespace( + n_workers=n_workers, + mpi_pool=types.SimpleNamespace(submit=lambda fn: next(it), + shutdown=lambda wait=True: None), + _teardown_unidentified_pool=lambda ids: MpiPoolSession. + _teardown_unidentified_pool(stand_in, ids), + ) + result = MpiPoolSession._collect_worker_identities(stand_in) + return result, killed + + +def test_identity_collection_complete_returns_identities(monkeypatch): + from tensorrt_llm.llmapi.mpi_session import _process_start_time + + me = (os.getpid(), _process_start_time(os.getpid())) + other = (1, b"1") # pid 1: exists but start_time won't match -> unique pid + ids, killed = _collect_identities(monkeypatch, [me, other]) + assert set(ids) == {me, other} and not killed + + +def test_identity_collection_fails_closed_on_timeout(monkeypatch): + # A pending barrier task means the pool cannot honor wait_shutdown: + # the session must be torn down and rejected, NOT handed out with the + # contract silently downgraded (review requirement). + import pytest as _pytest + + from tensorrt_llm.llmapi.mpi_session import _process_start_time + + me = (os.getpid(), _process_start_time(os.getpid())) + with _pytest.raises(RuntimeError, match="incomplete"): + _collect_identities(monkeypatch, [me], pending=1) + + +def test_identity_collection_fails_closed_on_duplicate_pids(monkeypatch): + import pytest as _pytest + + from tensorrt_llm.llmapi.mpi_session import _process_start_time + + me = (os.getpid(), _process_start_time(os.getpid())) + with _pytest.raises(RuntimeError, match="incomplete"): + _collect_identities(monkeypatch, [me, me]) # one worker answered twice + + +def test_identity_collection_uses_configured_timeout(monkeypatch): + from tensorrt_llm.llmapi.mpi_session import _process_start_time + + monkeypatch.setenv("TRTLLM_MPI_IDENTITY_TIMEOUT", "123.5") + observed_timeouts = [] + me = (os.getpid(), _process_start_time(os.getpid())) + _collect_identities(monkeypatch, [me], + n_workers=1, + observed_timeouts=observed_timeouts) + assert observed_timeouts == [123.5] + + +def test_identity_timeout_covers_worker_bootstrap(monkeypatch): + # The deadline bounds spawn + `import tensorrt_llm`, not barrier latency: + # it must exceed the slowest bootstrap the repo measures (~117s busy node). + monkeypatch.delenv("TRTLLM_MPI_IDENTITY_TIMEOUT", raising=False) + assert _identity_barrier_timeout() > 117.0 + + +# Invalid values (unparsable, non-positive) fall back to the default rather +# than turning the barrier into a busy-wait or an unbounded block. +@pytest.mark.parametrize("raw, expected", + [("90", 90.0), ("0.5", 0.5), + ("", _DEFAULT_IDENTITY_TIMEOUT), + ("0", _DEFAULT_IDENTITY_TIMEOUT), + ("-1", _DEFAULT_IDENTITY_TIMEOUT), + ("abc", _DEFAULT_IDENTITY_TIMEOUT), + ("nan", _DEFAULT_IDENTITY_TIMEOUT), + ("inf", _DEFAULT_IDENTITY_TIMEOUT), + ("-inf", _DEFAULT_IDENTITY_TIMEOUT), + ("1e309", _DEFAULT_IDENTITY_TIMEOUT)]) +def test_identity_timeout_env_override(monkeypatch, raw, expected): + monkeypatch.setenv("TRTLLM_MPI_IDENTITY_TIMEOUT", raw) + assert _identity_barrier_timeout() == expected + + +def test_prefetch_fallback_identity_timeout_matches_mpi_default(): + from test_common.session_prefetcher import _FALLBACK_IDENTITY_TIMEOUT + + assert _FALLBACK_IDENTITY_TIMEOUT == _DEFAULT_IDENTITY_TIMEOUT diff --git a/tests/unittest/llmapi/test_sampling_params.py b/tests/unittest/llmapi/test_sampling_params.py index 8988cc137b93..f9e4ad99e5e6 100644 --- a/tests/unittest/llmapi/test_sampling_params.py +++ b/tests/unittest/llmapi/test_sampling_params.py @@ -201,6 +201,66 @@ def test_thinking_budget_logits_processor_ignores_closed_reasoning_block(): assert torch.equal(logits, torch.zeros(1, 1, 8)) +def _run(processor, token_ids): + """Call the processor on a fresh logits row and return it.""" + logits = torch.zeros(1, 1, 8) + processor(0, logits, token_ids, None, None) + return logits + + +def test_thinking_budget_logits_processor_does_not_force_end_twice_on_stale_view(): + """Regression: a stale token view must not re-force the end tag. + + With the overlap scheduler, logits processors run one step + behind the sampled tokens, so the call right after forcing the end token + sees token_ids WITHOUT it. A stateless processor forces the end token a + second time, producing e.g. `` — the reasoning parser + consumes the first tag and the second leaks into message.content. + """ + processor = ThinkingBudgetLogitsProcessor( + thinking_token_budget=2, + reasoning_start_token_ids=[1], + reasoning_end_token_ids=[2], + ) + stale = [[1, 5, 6]] # budget spent, forced end not visible yet + + logits = _run(processor, stale) + assert logits[0, 0, 2] == 0 and torch.isneginf(logits[0, 0, 3]) + + # Next step: the forced end token is still not visible (stale view). + assert torch.equal(_run(processor, stale), torch.zeros(1, 1, 8)) + + # Once the view catches up (end tag present), progress state resets so a + # later reasoning block is budgeted again. + assert torch.equal(_run(processor, [[1, 5, 6, 2, 7]]), torch.zeros(1, 1, 8)) + logits = _run(processor, [[1, 5, 6, 2, 7, 1, 8, 9]]) + assert logits[0, 0, 2] == 0 and torch.isneginf(logits[0, 0, 3]) + + +def test_thinking_budget_logits_processor_continues_end_sequence_on_stale_view(): + """Multi-token end sequence must continue, not restart, on a stale view. + + Under the overlap scheduler the processor must continue from its + recorded progress rather than re-derive from token_ids. + """ + processor = ThinkingBudgetLogitsProcessor( + thinking_token_budget=2, + reasoning_start_token_ids=[1], + reasoning_end_token_ids=[2, 3], + ) + stale = [[1, 5, 6]] + + assert _run(processor, stale)[0, 0, 2] == 0 + + # Stale view: forced first end token not visible; must force the SECOND + # end token, not the first again. + logits = _run(processor, stale) + assert logits[0, 0, 3] == 0 and torch.isneginf(logits[0, 0, 2]) + + # Sequence complete: no further forcing even while the view is stale. + assert torch.equal(_run(processor, stale), torch.zeros(1, 1, 8)) + + def test_add_thinking_budget_logits_processor_uses_reasoning_parser_tokens(): class FakeTokenizer: def encode(self, text, add_special_tokens=False): diff --git a/tests/unittest/llmapi/test_session_prefetcher.py b/tests/unittest/llmapi/test_session_prefetcher.py new file mode 100644 index 000000000000..b91869f24aab --- /dev/null +++ b/tests/unittest/llmapi/test_session_prefetcher.py @@ -0,0 +1,621 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pure-logic tests for the session prefetcher — no MPI, no GPU.""" + +import re +import sys +import threading +import time +import types +from pathlib import Path + +import pytest +from test_common import session_prefetcher +from test_common.session_prefetcher import SessionPrefetcher, warm_page_cache + + +class _FakeMarker: + def __init__(self, *args): + self.args = args + + +class _FakeItem: + def __init__(self, model_dir=None, cls=None, params=None): + self._marker = _FakeMarker(model_dir) if model_dir else None + self.cls = cls + if params is not None: + self.callspec = types.SimpleNamespace(params=params) + + def get_closest_marker(self, name): + return self._marker if name == "prefetch_model_dir" else None + + +def _as_session(*items): + """Link fake items into a fake pytest session (final run order).""" + session = types.SimpleNamespace(items=list(items)) + for item in items: + item.session = session + return list(items) + + +def _wait_for(cond, timeout=5.0): + """Poll ``cond`` (fire-and-forget background work) until true or timeout.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if cond(): + return True + time.sleep(0.05) + return cond() + + +@pytest.fixture +def prefetcher(monkeypatch): + monkeypatch.setenv("TRTLLM_TEST_PREFETCH_SESSION", "1") + # These pure-logic tests may themselves run under xdist; pin the worker + # marker off so the prefetcher's xdist guard does not disable it here. + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + p = SessionPrefetcher() + # Record build/warm triggers instead of spawning MPI pools/reading weights. + built, warmed, overlays = [], [], [] + + def _fake_build(self, spec, gen, env_overlay=None): + built.append(spec) + overlays.append(env_overlay) + + monkeypatch.setattr(SessionPrefetcher, "_build", _fake_build) + monkeypatch.setattr(SessionPrefetcher, "_warm", lambda self, d: warmed.append(d)) + p.built, p.warmed, p.overlays = built, warmed, overlays + return p + + +class _FakePool: + def __init__(self, n_workers, wait_shutdown=False): + self.n_workers = n_workers + self.wait_shutdown = wait_shutdown + self.shut = False + + def shutdown(self): + self.shut = True + + +def _arm(prefetcher, pool, spec=4): + """Publish ``pool`` into the shadow slot through the real API.""" + prefetcher._publish(spec, pool, session_prefetcher._spawn_snapshot(), prefetcher._build_gen) + + +@pytest.mark.parametrize( + "stage_name", + [ + None, + "A10-PyTorch-2", + "DGX_H100-4_GPUs-PyTorch-DeepSeek-1", + "DGX_B200-PyTorch-4", + "Any-Future-Stage", + ], +) +def test_enabled_by_default_in_all_stages(monkeypatch, stage_name): + monkeypatch.delenv("TRTLLM_TEST_PREFETCH_SESSION", raising=False) + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + if stage_name is None: + monkeypatch.delenv("stageName", raising=False) + else: + monkeypatch.setenv("stageName", stage_name) + assert SessionPrefetcher().enabled + + +def test_explicit_env_overrides_default(monkeypatch): + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + monkeypatch.setenv("TRTLLM_TEST_PREFETCH_SESSION", "0") + assert not SessionPrefetcher().enabled + monkeypatch.setenv("TRTLLM_TEST_PREFETCH_SESSION", "1") + assert SessionPrefetcher().enabled + + +def test_disabled_is_noop(monkeypatch): + monkeypatch.setenv("TRTLLM_TEST_PREFETCH_SESSION", "0") + p = SessionPrefetcher() + p.schedule_shadow(2) + assert p._thread is None + assert p.take(2) is None + + +def test_disabled_in_xdist_worker(monkeypatch): + # Under xdist each worker runs a scheduler-assigned subset; N workers + # would each hold a live pool plus a spare. + monkeypatch.setenv("TRTLLM_TEST_PREFETCH_SESSION", "1") + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw0") + assert not SessionPrefetcher().enabled + + +def test_factory_miss_builds_sync_and_arms_shadow(prefetcher): + # _build is stubbed by the fixture to record specs instead of spawning. + factory = prefetcher._make_factory(_FakePool) + session = factory(4) # nothing prefetched yet -> sync build + assert isinstance(session, _FakePool) and session.n_workers == 4 + # Every pool this layer hands out must block its shutdown on worker + # exit: the NEXT pool is handed over instantly, without the ~50s sync + # spawn that used to hide the GPU-memory release window. + assert session.wait_shutdown + prefetcher._thread.join(timeout=10) + assert prefetcher.built == [4] # shadow armed for the next test + + +def test_factory_single_worker_also_prefetches(prefetcher): + # The default single-GPU path spawns a 1-worker pool too (executor.py -> + # proxy.py), paying the same ~50s spawn+import: it must benefit as well. + factory = prefetcher._make_factory(_FakePool) + session = factory(1) + assert isinstance(session, _FakePool) + prefetcher._thread.join(timeout=10) + assert prefetcher.built == [1] # shadow armed for the next 1-GPU test + + +def test_schedule_shadow_passes_env_overlay_to_build(prefetcher): + # session_reuse restocks shadows with its worker-side weight-cache env; + # the overlay must reach the build (frozen into workers via the library's + # env_overrides channel — the parent env is never touched). + prefetcher.schedule_shadow(2, env_overlay={"TRTLLM_HF_WEIGHT_CACHE": "1"}) + prefetcher._thread.join(timeout=10) + assert prefetcher.built == [2] + assert prefetcher.overlays == [{"TRTLLM_HF_WEIGHT_CACHE": "1"}] + + +def test_take_wrong_size_in_flight_waits_before_miss(prefetcher, monkeypatch): + # A wrong-size shadow is a miss, but the caller must not start its sync + # pool until that in-flight bootstrap finishes. Otherwise a group-size + # transition can make two MPI pools compete on the same allocation. + release = threading.Event() + + def _slow_build(self, spec, gen, env_overlay=None): + release.wait() + self._publish(spec, _FakePool(spec), session_prefetcher._spawn_snapshot(), gen) + + monkeypatch.setattr(SessionPrefetcher, "_build", _slow_build) + prefetcher.schedule_shadow(2) + result = [] + taker = threading.Thread(target=lambda: result.append(prefetcher.take(4))) + taker.start() + time.sleep(0.1) + assert taker.is_alive() # blocked behind the only in-flight bootstrap + release.set() + taker.join(timeout=10) + assert result == [None] # wrong-size pool was drained, then rejected + + +def test_shadow_wait_budget_tracks_identity_timeout(monkeypatch): + fake_mpi_session = types.SimpleNamespace(_identity_barrier_timeout=lambda: 125.0) + monkeypatch.setitem(sys.modules, "tensorrt_llm.llmapi.mpi_session", fake_mpi_session) + assert ( + session_prefetcher._shadow_build_wait_timeout() + == 125 + session_prefetcher._SHADOW_BUILD_FINISH_GRACE + ) + + +def test_shadow_wait_budget_handles_partially_loaded_mpi_module(monkeypatch): + monkeypatch.setitem(sys.modules, "tensorrt_llm.llmapi.mpi_session", types.SimpleNamespace()) + monkeypatch.setenv("TRTLLM_MPI_IDENTITY_TIMEOUT", "625") + assert ( + session_prefetcher._shadow_build_wait_timeout() + == 625 + session_prefetcher._SHADOW_BUILD_FINISH_GRACE + ) + + +def test_take_timeout_is_terminal_until_build_exits(prefetcher, monkeypatch): + join_timeouts = [] + + class _HungBuild: + alive = True + + def join(self, timeout): + join_timeouts.append(timeout) + + def is_alive(self): + return self.alive + + thread = _HungBuild() + prefetcher._thread = thread + prefetcher._building_spec = 2 + monkeypatch.setattr(session_prefetcher, "_shadow_build_wait_timeout", lambda: 321.0) + + with pytest.raises(TimeoutError, match="refusing to start a concurrent MPI pool"): + prefetcher.take(2) + with pytest.raises(TimeoutError, match="previously timed out"): + prefetcher.take(2) + + # Only the first call spends the full wait budget and records a timeout. + # Later calls fail fast until the abandoned build actually exits. + assert join_timeouts == [321.0] + assert prefetcher._thread is thread + assert prefetcher._build_gen == 1 + assert prefetcher.stats["pool_build_timeouts"] == 1 + + thread.alive = False + assert prefetcher.take(2) is None + assert prefetcher._thread is None + assert not prefetcher._build_timed_out + + +def test_concurrent_take_timeout_waits_only_once(prefetcher, monkeypatch): + join_started = threading.Event() + release_join = threading.Event() + join_timeouts = [] + errors = [] + + class _HungBuild: + def join(self, timeout): + join_timeouts.append(timeout) + join_started.set() + release_join.wait() + + def is_alive(self): + return True + + prefetcher._thread = _HungBuild() + prefetcher._building_spec = 2 + monkeypatch.setattr(session_prefetcher, "_shadow_build_wait_timeout", lambda: 321.0) + + def _take(): + try: + prefetcher.take(2) + except TimeoutError as e: + errors.append(e) + + first = threading.Thread(target=_take) + second = threading.Thread(target=_take) + first.start() + assert join_started.wait(timeout=5) + second.start() + release_join.set() + first.join(timeout=5) + second.join(timeout=5) + + assert len(errors) == 2 + assert join_timeouts == [321.0] + assert prefetcher._build_gen == 1 + assert prefetcher.stats["pool_build_timeouts"] == 1 + + +def test_factory_spawn_failure_propagates_without_retry(prefetcher): + # Once the full identity deadline expires, an unidentified worker may + # still be alive. Retrying or downgrading to wait_shutdown=False could + # overlap it with another pool, so the failure must propagate. + calls = [] + + class _FailingWaitPool(_FakePool): + def __init__(self, n_workers, wait_shutdown=False): + calls.append(wait_shutdown) + raise RuntimeError("identity collection incomplete") + + factory = prefetcher._make_factory(_FailingWaitPool) + with pytest.raises(RuntimeError, match="identity collection incomplete"): + factory(2) + assert calls == [True] + assert prefetcher._thread is None # failure did not restock a shadow + + +def test_factory_hit_hands_over_shadow(prefetcher): + pool = _FakePool(4) + _arm(prefetcher, pool, spec=4) + factory = prefetcher._make_factory(_FakePool) + assert factory(4) is pool # prefetched pool handed over + + +def test_take_spec_mismatch_returns_none(prefetcher): + pool = _FakePool(4) + _arm(prefetcher, pool, spec=4) + assert prefetcher.take(2) is None # wrong size: sync fallback + + +def test_take_discards_on_env_mismatch(prefetcher, monkeypatch): + pool = _FakePool(4) + _arm(prefetcher, pool, spec=4) + monkeypatch.setenv("TLLM_TEST_ONLY_FLAG", "changed-after-spawn") + assert prefetcher.take(4) is None # frozen workers would miss the new env + assert _wait_for(lambda: pool.shut) # stale shadow torn down in background + + +def test_take_discards_on_nonprefixed_env_mismatch(prefetcher, monkeypatch): + # Workers inherit the WHOLE parent env at spawn; test knobs outside any + # TRTLLM*/TLLM* prefix (e.g. OVERRIDE_QUANT_ALGO, read inside workers by + # model_config.py) must also invalidate a prefetched pool, else workers + # silently run with stale env (review finding on the prefix allowlist). + pool = _FakePool(4) + _arm(prefetcher, pool, spec=4) + monkeypatch.setenv("OVERRIDE_QUANT_ALGO", "W4A16_MXFP4") + assert prefetcher.take(4) is None + + +def test_pytest_current_test_drift_does_not_discard(prefetcher, monkeypatch): + # PYTEST_CURRENT_TEST changes every test phase by design; it must not + # invalidate the snapshot or no prefetched pool would ever be handed over. + monkeypatch.setenv("PYTEST_CURRENT_TEST", "test_a (call)") + pool = _FakePool(4) + _arm(prefetcher, pool, spec=4) + monkeypatch.setenv("PYTEST_CURRENT_TEST", "test_b (setup)") + assert prefetcher.take(4) is pool + + +def test_take_discards_on_syspath_mismatch(prefetcher, monkeypatch): + # test_modeling_out_of_tree monkeypatches sys.path before + # LLM(); pool workers freeze sys.path at spawn (MPIPoolExecutor(path=...)), + # so a pool spawned earlier can't import the out-of-tree module and dies + # during initialization. sys.path must be part of the handover guard. + pool = _FakePool(4) + _arm(prefetcher, pool, spec=4) + monkeypatch.syspath_prepend("/oot/example/path") + assert prefetcher.take(4) is None # frozen workers would miss the new path + + +def test_take_does_not_join_unstarted_shadow_thread(prefetcher, monkeypatch): + # A test creating LLMs concurrently (ThreadPoolExecutor) + # raced take()'s unlocked read of _thread against schedule_shadow()'s + # assign-then-start critical section, joining a thread that had not been + # started yet ("cannot join thread before it is started"). A slow start() + # widens the assign->start window deterministically. + class _SlowStartThread(threading.Thread): + def start(self): + time.sleep(0.3) # hold the assigned-but-unstarted state visible + super().start() + + monkeypatch.setattr(session_prefetcher.threading, "Thread", _SlowStartThread) + errors = [] + + def _taker(): + time.sleep(0.1) # let schedule_shadow enter its critical section first + try: + prefetcher.take(1) + except RuntimeError as e: # pre-fix: "cannot join thread before it is started" + errors.append(e) + + taker = threading.Thread(target=_taker) + taker.start() + prefetcher.schedule_shadow(1) + taker.join(timeout=10) + assert errors == [] + + +def test_abandoned_build_publish_discards_pool(prefetcher): + # A build that outlives _drain()'s join timeout is abandoned (generation + # bump); its late _publish() must shut the pool down, not land it — + # landing would overwrite (and leak) a newer pool or hand stale state + # to a future test. + pool = _FakePool(4) + gen = prefetcher._build_gen + prefetcher._build_gen += 1 # what _drain() does on abandonment + prefetcher._publish(4, pool, session_prefetcher._spawn_snapshot(), gen) + assert pool.shut + assert prefetcher._built is None + + +def test_publish_never_overwrites_unconsumed_pool(prefetcher): + first, second = _FakePool(4), _FakePool(4) + _arm(prefetcher, first, spec=4) + _arm(prefetcher, second, spec=4) + assert prefetcher._built.session is first # slot kept + assert second.shut and not first.shut # newcomer discarded, not the slot + + +def test_session_summary_counters_and_emission(prefetcher, capfd): + # Handover / stale-discard / superseded each count once; + # dispose() emits ONE summary line (outside pytest capture in real runs, + # the only guaranteed console-visible record of prefetch activity). + hit = _FakePool(4) + _arm(prefetcher, hit, spec=4) + assert prefetcher.take(4) is hit + stale = _FakePool(4) + _arm(prefetcher, stale, spec=4) + assert prefetcher.take(2) is None # spec mismatch -> stale discard + late = _FakePool(4) + prefetcher._publish(4, late, session_prefetcher._spawn_snapshot(), prefetcher._build_gen - 1) + assert prefetcher.stats["pools_handed_over"] == 1 + assert prefetcher.stats["pools_discarded_stale"] == 1 + assert prefetcher.stats["pools_discarded_superseded"] == 1 + prefetcher.dispose() + out = capfd.readouterr().out + assert "[session-prefetch] session summary:" in out + assert "pools_handed_over=1" in out + + +def test_no_summary_when_prefetch_never_fired(prefetcher, capfd): + prefetcher.dispose() + assert "session summary" not in capfd.readouterr().out + + +def test_warm_counters_track_gib(tmp_path, monkeypatch): + monkeypatch.setenv("TRTLLM_TEST_PREFETCH_SESSION", "1") + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + p = SessionPrefetcher() # real _warm (fixture would stub it) + (tmp_path / "model-00001.safetensors").write_bytes(b"x" * (1 << 20)) + p._warm(str(tmp_path)) + assert p.stats["warms"] == 1 and p._warmed_gib > 0 + p._warm("not/a/real/dir") + assert p.stats["warm_noops"] == 1 + + +def test_model_switch_triggers_warm_and_dedups(prefetcher): + items = _as_session(_FakeItem("/models/a"), _FakeItem("/models/b")) + prefetcher.on_test_setup(items[0]) + # Warm threads are fire-and-forget; poll briefly for the recorded call. + assert _wait_for(lambda: prefetcher.warmed) + assert prefetcher.warmed == ["/models/b"] + # Same next-model again: deduplicated. + prefetcher.on_test_setup(items[0]) + time.sleep(0.2) + assert prefetcher.warmed == ["/models/b"] + + +def test_same_next_model_does_not_warm(prefetcher): + # Consecutive tests on the same model: its weights are already hot. + items = _as_session(_FakeItem("/models/a"), _FakeItem("/models/a")) + prefetcher.on_test_setup(items[0]) + time.sleep(0.2) + assert prefetcher.warmed == [] + + +def test_no_marker_suite_never_warms(prefetcher): + items = _as_session(_FakeItem(), _FakeItem(), _FakeItem()) + prefetcher.on_test_setup(items[0]) + time.sleep(0.2) + assert prefetcher.warmed == [] + + +def test_auto_model_dir_from_accuracy_class_attr(prefetcher): + # Accuracy-harness classes declare MODEL_PATH; warming must pick it up + # automatically, with no marker on the test. + class _TestLlama: + MODEL_PATH = "/models/llama" + + class _TestQwen: + MODEL_PATH = "/models/qwen" + + items = _as_session(_FakeItem(cls=_TestLlama), _FakeItem(cls=_TestQwen)) + prefetcher.on_test_setup(items[0]) + assert _wait_for(lambda: prefetcher.warmed) + assert prefetcher.warmed == ["/models/qwen"] + + +def test_model_param_discovery(monkeypatch): + # Modeling unit tests carry the model as a `model_folder`-style parameter + # (a name under LLM_MODELS_ROOT, or an absolute path) — discovered + # without any test-file changes. + monkeypatch.setenv("LLM_MODELS_ROOT", "/models-root") + item = _FakeItem(params={"model_folder": "Nemotron-H-8B-Base-8K"}) + assert session_prefetcher._model_dir_of(item) == "/models-root/Nemotron-H-8B-Base-8K" + item = _FakeItem(params={"model_dir": "/abs/path/model"}) + assert session_prefetcher._model_dir_of(item) == "/abs/path/model" + # Non-model params never produce a guess. + assert session_prefetcher._model_dir_of(_FakeItem(params={"dtype": "fp8"})) is None + + +def test_marker_overrides_class_model_path(): + class _TestCls: + MODEL_PATH = "/models/from-class" + + item = _FakeItem(model_dir="/models/from-marker", cls=_TestCls) + assert session_prefetcher._model_dir_of(item) == "/models/from-marker" + + +def test_accuracy_harness_still_declares_model_path(): + # _model_dir_of auto-discovers models via the MODEL_PATH class attribute of + # the accuracy harnesses (accuracy_core.py); renaming that attribute would + # silently kill warming repo-wide. Textual check — importing accuracy_core + # would drag integration-only dependencies into this unit test. + core = Path(__file__).parents[2] / "integration" / "defs" / "accuracy" / "accuracy_core.py" + assert re.search(r"^\s+MODEL_PATH\s*=", core.read_text(), re.MULTILINE), ( + "accuracy_core.py no longer declares MODEL_PATH — update " + "session_prefetcher._model_dir_of to the harness's new convention" + ) + + +def test_warm_selects_files_like_the_weight_loader(tmp_path): + # Selection must mirror HfWeightLoader.load_weights: safetensors first + # (minus huge "consolidated" copies the loader skips), so the .bin copy + # and the consolidated file must NOT be read here. + payload = b"x" * (1 << 20) + (tmp_path / "model-00001.safetensors").write_bytes(payload) + (tmp_path / "consolidated.safetensors").write_bytes(payload * 4) + (tmp_path / "pytorch_model.bin").write_bytes(payload) + (tmp_path / "config.json").write_bytes(b"{}") # not a weight file + assert warm_page_cache(str(tmp_path)) == pytest.approx(1 / 1024, rel=1e-3) + + +def test_warm_falls_back_to_bin_then_pth(tmp_path): + payload = b"x" * (1 << 20) + bin_dir, pth_dir = tmp_path / "bin", tmp_path / "pth" + bin_dir.mkdir(), pth_dir.mkdir() + (bin_dir / "pytorch_model.bin").write_bytes(payload) + (pth_dir / "model.pth").write_bytes(payload) + assert warm_page_cache(str(bin_dir)) == pytest.approx(1 / 1024, rel=1e-3) + assert warm_page_cache(str(pth_dir)) == pytest.approx(1 / 1024, rel=1e-3) + + +def test_warm_page_cache_ignores_non_weight_dirs(tmp_path): + # MODEL_PATH may be an HF model id or a dir without local weights: no-op. + assert warm_page_cache(str(tmp_path)) == 0.0 + assert warm_page_cache("not/a/real/dir") == 0.0 + + +def test_warm_skips_models_larger_than_host_memory(tmp_path, monkeypatch): + # Warming a model bigger than free RAM is pure filer traffic: the pages + # would be evicted before the test loads them (DeepSeek-R1-class dirs). + (tmp_path / "model-00001.safetensors").write_bytes(b"x" * (1 << 20)) + monkeypatch.setattr(session_prefetcher, "_available_host_memory", lambda: 1 << 10) + assert warm_page_cache(str(tmp_path)) == 0.0 + + +def test_warm_io_thread_names_covered_by_threadleak_exclude(): + # Both pytest.ini threadleak_exclude lists contain r"session-prefetch-\w+"; + # the warm executor's thread_name_prefix must keep its IO workers inside + # that pattern (a large warm can outlive the test that started it). + assert re.fullmatch(r"session-prefetch-\w+", "session-prefetch-io_0") + + +def _fake_reuse_module(enabled): + return types.SimpleNamespace(REUSE=types.SimpleNamespace(is_active=lambda: enabled)) + + +def test_yields_mpi_seams_to_active_session_reuse(monkeypatch): + # session_reuse owns the same seams and saves the whole respawn; when it + # is enabled the prefetcher must not install its factory (whoever patched + # first would silently disable the other layer). + monkeypatch.setenv("TRTLLM_TEST_PREFETCH_SESSION", "1") + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + monkeypatch.setitem(sys.modules, "test_common.session_reuse", _fake_reuse_module(enabled=True)) + p = SessionPrefetcher() + p.install_pool_factory_if_loaded() + assert not p._patched + assert p.stats["mpi_yielded_to_reuse"] == 1 + + +def test_reuse_layer_inactive_or_absent_does_not_block_prefetch(monkeypatch): + monkeypatch.setitem(sys.modules, "test_common.session_reuse", _fake_reuse_module(enabled=False)) + assert not session_prefetcher._reuse_layer_active() + monkeypatch.delitem(sys.modules, "test_common.session_reuse") + assert not session_prefetcher._reuse_layer_active() + + +def test_unreadable_reuse_module_errs_on_yielding(monkeypatch): + # Module present but attribute layout changed: stay out of the way. + monkeypatch.setitem(sys.modules, "test_common.session_reuse", types.SimpleNamespace()) + assert session_prefetcher._reuse_layer_active() + + +def test_install_wraps_seam_in_isinstance_transparent_shim(prefetcher, monkeypatch): + # The patched seam must stay a real TYPE: proxy.py's killed-worker + # detection runs isinstance(x, MpiPoolSession) against this attribute, + # and a bare function there raises TypeError (the #16338 breakage class). + mpi_mod = pytest.importorskip("tensorrt_llm.llmapi.mpi_session") + fake = types.ModuleType("fake_seam_mod") + fake.MpiPoolSession = mpi_mod.MpiPoolSession + monkeypatch.setitem(sys.modules, "fake_seam_mod", fake) + monkeypatch.setattr(session_prefetcher, "_PATCH_TARGETS", ("fake_seam_mod",)) + monkeypatch.setattr(session_prefetcher, "_reuse_layer_active", lambda: False) + prefetcher.install_pool_factory_if_loaded() + assert fake.MpiPoolSession is not mpi_mod.MpiPoolSession # patched + # isinstance must not raise, and must answer for the real class. + assert isinstance(object(), fake.MpiPoolSession) is False + assert issubclass(mpi_mod.MpiPoolSession, fake.MpiPoolSession) + + +def test_patch_targets_cover_all_library_construction_sites(): + # The factory only intercepts the modules listed in _PATCH_TARGETS. If the + # library grows another MpiPoolSession(...) construction site, prefetch + # would silently stop covering it (armed spare pools would idle next to + # directly-constructed ones) — turn that drift into a red test. + import tensorrt_llm + + root = Path(tensorrt_llm.__file__).parent + # mpi_session.py defines the class (and the MGMN server path, which + # legitimately builds its own pool outside the bare-LLM() seams). + exempt = {"tensorrt_llm.llmapi.mpi_session"} + offenders = [] + for py in root.rglob("*.py"): + if re.search(r"(? "helloworld" -- on + transformers >= 5.x). Without this routing the bench would see a + different tokenizer than the rest of TRT-LLM uses. + """ + inner = mock.MagicMock() + inner.pad_token_id = 0 # already set => add_special_tokens not invoked + wrapper = mock.MagicMock(tokenizer=inner) + + with mock.patch( + "tensorrt_llm.bench.utils.data.TransformersTokenizer.from_pretrained", return_value=wrapper + ) as routed: + out = initialize_tokenizer("dummy/model") + + routed.assert_called_once_with("dummy/model", padding_side="left", trust_remote_code=True) + # Bench code uses the raw HF tokenizer (calls __call__, encode, + # add_special_tokens on it), so the wrapper must be peeled off. + assert out is inner + + +def test_int8_not_offered_as_bench_quant_choice() -> None: + # INT8 is unsupported by the build path, so it must not be advertised as a + # trtllm-bench --quantization choice (issue #7091). + assert f"{QuantAlgo.INT8}" not in get_args(VALID_QUANT_ALGOS) @pytest.mark.parametrize( diff --git a/tests/unittest/others/test_bench_data.py b/tests/unittest/others/test_bench_data.py deleted file mode 100644 index 0f1e69fedc64..000000000000 --- a/tests/unittest/others/test_bench_data.py +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import io -from unittest import mock - -import pytest - -from tensorrt_llm.bench.utils.data import ( - DatasetFormatError, - create_dataset_from_stream, - initialize_tokenizer, -) - - -class _FakeTokenizer: - """Minimal tokenizer stub for testing create_dataset_from_stream.""" - - def __call__(self, text, **kwargs): - ids = list(range(len(text.split()))) - return {"input_ids": ids} - - def encode(self, text, **kwargs): - return list(range(len(text.split()))) - - -def test_empty_stream_raises_dataset_format_error(): - tokenizer = _FakeTokenizer() - empty_stream = io.StringIO("") - - with pytest.raises(DatasetFormatError, match="No data was read from the dataset stream"): - create_dataset_from_stream(tokenizer, empty_stream) - - -def test_initialize_tokenizer_routes_through_transformers_tokenizer(): - """``initialize_tokenizer`` must call ``TransformersTokenizer.from_pretrained``. - - Routing through ``TransformersTokenizer`` is what lets ``trtllm-bench`` - inherit the post-load fixes (e.g. ``maybe_fix_byte_level_tokenizer``, - which prevents DeepSeek-V3 from loading with a Metaspace pre-tokenizer - that silently strips spaces -- "hello world" -> "helloworld" -- on - transformers >= 5.x). Without this routing the bench would see a - different tokenizer than the rest of TRT-LLM uses. - """ - inner = mock.MagicMock() - inner.pad_token_id = 0 # already set => add_special_tokens not invoked - wrapper = mock.MagicMock(tokenizer=inner) - - with mock.patch( - "tensorrt_llm.bench.utils.data.TransformersTokenizer.from_pretrained", return_value=wrapper - ) as routed: - out = initialize_tokenizer("dummy/model") - - routed.assert_called_once_with("dummy/model", padding_side="left", trust_remote_code=True) - # Bench code uses the raw HF tokenizer (calls __call__, encode, - # add_special_tokens on it), so the wrapper must be peeled off. - assert out is inner diff --git a/tests/unittest/others/test_cache_transceiver_precheck_config.py b/tests/unittest/others/test_cache_transceiver_precheck_config.py new file mode 100644 index 000000000000..0bcff41adb48 --- /dev/null +++ b/tests/unittest/others/test_cache_transceiver_precheck_config.py @@ -0,0 +1,773 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CPU-only tests for the disagg cache-transceiver precheck config resolution. + +Target: tests/scripts/perf-sanity/cache_transceiver_precheck/precheck_config.py +""" + +import json +import os +import sys + +import pytest + +_PRECHECK_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "..", + "..", + "tests", + "scripts", + "perf-sanity", + "cache_transceiver_precheck", +) +sys.path.insert(0, os.path.abspath(_PRECHECK_DIR)) + +import precheck_config as pcfg # noqa: E402 +import run_precheck as rp # noqa: E402 (stdlib-only at import time) + + +def _disagg_yaml(ctx_extra=None, gen_extra=None, **overrides): + """Minimal disagg perf-sanity yaml shaped like the checked-in configs.""" + ctx = { + "tensor_parallel_size": 4, + "pipeline_parallel_size": 1, + "context_parallel_size": 1, + "enable_attention_dp": True, + "kv_cache_config": {"dtype": "fp8"}, + "cache_transceiver_config": {"max_tokens_in_buffer": 16384, "backend": "NIXL"}, + "speculative_config": {"decoding_type": "MTP", "num_nextn_predict_layers": 1}, + } + gen = { + "tensor_parallel_size": 16, + "pipeline_parallel_size": 1, + "context_parallel_size": 1, + "enable_attention_dp": True, + "kv_cache_config": {"dtype": "fp8"}, + "cache_transceiver_config": {"max_tokens_in_buffer": 16384, "backend": "NIXL"}, + "speculative_config": {"decoding_type": "MTP", "num_nextn_predict_layers": 1}, + } + ctx.update(ctx_extra or {}) + gen.update(gen_extra or {}) + cfg = { + "metadata": {"model_name": "deepseek_r1_0528_fp4_v2"}, + "benchmark": {"mode": "e2e", "input_length": 8192, "output_length": 1024}, + "hardware": {"gpus_per_node": 4, "num_ctx_servers": 1, "num_gen_servers": 1}, + "worker_config": {"ctx": ctx, "gen": gen}, + } + cfg.update(overrides) + return cfg + + +def test_resolve_plan_adp_asymmetric(): + plan = pcfg.resolve_plan(_disagg_yaml()) + assert not plan["skip"] + assert plan["ctx"] == { + "tp": 4, + "pp": 1, + "cp": 1, + "enable_attention_dp": True, + "world_size": 4, + "dp_size": 4, + } + assert plan["gen"]["world_size"] == 16 and plan["gen"]["dp_size"] == 16 + # Cover every gen dp rank. + assert plan["n_pairs"] == 16 + assert plan["wave_size"] == 8 + assert plan["request_lengths"] == [1024, 8192] + assert plan["ctx_num_nextn_predict_layers"] == 1 + assert plan["ctx_cache_transceiver_config"]["backend"] == "NIXL" + + +def test_spec_nextn_max_draft_len_fallback(): + # Checked-in yamls spell the MTP depth either way (num_nextn_predict_layers + # is MTPDecodingConfig's deprecated alias of max_draft_len); both must + # resolve to the same spec-layer count. + cfg = _disagg_yaml( + ctx_extra={"speculative_config": {"decoding_type": "MTP", "max_draft_len": 3}}, + gen_extra={ + "speculative_config": { + "decoding_type": "MTP", + "num_nextn_predict_layers": 3, + "max_draft_len": 3, + } + }, + ) + plan = pcfg.resolve_plan(cfg) + assert plan["ctx_num_nextn_predict_layers"] == 3 + assert plan["gen_num_nextn_predict_layers"] == 3 + + # Non-MTP speculation never contributes MTP KV layers, even with a + # max_draft_len present. + cfg = _disagg_yaml( + ctx_extra={"speculative_config": {"decoding_type": "Eagle", "max_draft_len": 3}}, + gen_extra={"speculative_config": {"decoding_type": "Eagle", "max_draft_len": 3}}, + ) + plan = pcfg.resolve_plan(cfg) + assert plan["ctx_num_nextn_predict_layers"] == 0 + assert plan["gen_num_nextn_predict_layers"] == 0 + + +def test_request_lengths_clamped_by_buffer_and_cap(): + cfg = _disagg_yaml(benchmark={"mode": "e2e", "input_length": 131072}) + cfg["worker_config"]["ctx"]["cache_transceiver_config"]["max_tokens_in_buffer"] = 131104 + cfg["worker_config"]["gen"]["cache_transceiver_config"]["max_tokens_in_buffer"] = 131104 + plan = pcfg.resolve_plan(cfg) + # Derived ISL is capped by max_request_length (default 32768). + assert plan["request_lengths"] == [1024, 32768] + + cfg["worker_config"]["ctx"]["cache_transceiver_config"]["max_tokens_in_buffer"] = 4096 + cfg["worker_config"]["gen"]["cache_transceiver_config"]["max_tokens_in_buffer"] = 4096 + plan = pcfg.resolve_plan(cfg) + assert plan["request_lengths"] == [1024, 4096] + + # Explicit yaml override is used as-is (not capped). + cfg["cache_transceiver_precheck"] = {"request_lengths": [64000]} + cfg["worker_config"]["ctx"]["cache_transceiver_config"]["max_tokens_in_buffer"] = 131104 + cfg["worker_config"]["gen"]["cache_transceiver_config"]["max_tokens_in_buffer"] = 131104 + plan = pcfg.resolve_plan(cfg) + assert plan["request_lengths"] == [64000] + + +def test_gen_only_no_context_skips(): + cfg = _disagg_yaml(benchmark={"mode": "gen_only_no_context", "input_length": 1024}) + plan = pcfg.resolve_plan(cfg, benchmark_mode="gen_only") + assert plan["skip"] + # e2e over the same yaml still runs (ctx servers are launched there). + assert not pcfg.resolve_plan(cfg, benchmark_mode="e2e")["skip"] + + +def test_backend_mismatch_raises(): + cfg = _disagg_yaml( + gen_extra={"cache_transceiver_config": {"backend": "UCX", "max_tokens_in_buffer": 16384}} + ) + with pytest.raises(ValueError, match="backend mismatch"): + pcfg.resolve_plan(cfg) + + +def test_pair_participation_and_waves(): + plan = pcfg.resolve_plan(_disagg_yaml()) + # ADP ctx (dp4): pair k belongs to tp_rank k % 4. + assert pcfg.pair_participates(plan, "ctx", 1, 5) + assert not pcfg.pair_participates(plan, "ctx", 0, 5) + # ADP gen (dp16): 1:1. + assert pcfg.pair_participates(plan, "gen", 5, 5) + assert not pcfg.pair_participates(plan, "gen", 4, 5) + assert pcfg.waves(plan) == [list(range(8)), list(range(8, 16))] + # ctx rank owns 2 pairs per wave of 8; gen rank owns at most 1. + assert pcfg.max_owned_per_wave(plan, "ctx") == 2 + assert pcfg.max_owned_per_wave(plan, "gen") == 1 + + # Non-ADP side participates everywhere and owns the whole wave. + plan_pp = pcfg.resolve_plan( + _disagg_yaml( + ctx_extra={ + "tensor_parallel_size": 1, + "pipeline_parallel_size": 8, + "enable_attention_dp": False, + } + ) + ) + assert plan_pp["ctx"]["dp_size"] == 1 and plan_pp["n_pairs"] == 16 + assert pcfg.pair_participates(plan_pp, "ctx", 0, 11) + assert pcfg.max_owned_per_wave(plan_pp, "ctx") == plan_pp["wave_size"] + + +def test_fingerprint_role_agnostic(): + plan_a = pcfg.resolve_plan(_disagg_yaml()) + plan_b = pcfg.resolve_plan(_disagg_yaml()) + assert plan_a["fingerprint"] == plan_b["fingerprint"] + changed = _disagg_yaml() + changed["worker_config"]["gen"]["tensor_parallel_size"] = 8 + assert pcfg.resolve_plan(changed)["fingerprint"] != plan_a["fingerprint"] + + +def test_model_kv_shape_mla_and_gqa(tmp_path): + mla = tmp_path / "mla" + mla.mkdir() + (mla / "config.json").write_text( + json.dumps( + { + "num_hidden_layers": 61, + "kv_lora_rank": 512, + "qk_rope_head_dim": 64, + "num_attention_heads": 128, + } + ) + ) + shape = pcfg.model_kv_shape(str(mla)) + assert shape == { + "num_layers": 61, + "num_kv_heads": 1, + "head_dim": 576, + "is_mla": True, + "vocab_size": None, + "source": "config.json (MLA)", + } + + gqa = tmp_path / "gqa" + gqa.mkdir() + (gqa / "config.json").write_text( + json.dumps( + { + "num_hidden_layers": 32, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "hidden_size": 4096, + } + ) + ) + shape = pcfg.model_kv_shape(str(gqa)) + assert shape["num_kv_heads"] == 8 and shape["head_dim"] == 128 and not shape["is_mla"] + + # Unresolvable model dir -> synthetic fallback (precheck still runs). + assert pcfg.model_kv_shape(None)["source"] == "fallback" + assert pcfg.model_kv_shape(str(tmp_path / "missing"))["source"] == "fallback" + + +def test_side_plan_views(): + plan = pcfg.resolve_plan(_disagg_yaml()) + ctx_view = pcfg.side_plan(plan, "ctx") + gen_view = pcfg.side_plan(plan, "gen") + assert ctx_view["parallel"]["world_size"] == 4 + assert ctx_view["num_peers"] == 1 and gen_view["num_peers"] == 1 + assert gen_view["cache_transceiver_config"]["max_tokens_in_buffer"] == 16384 + + +class TestControlWireFormat: + """run_precheck's HMAC-JSON control frames (importable without torch).""" + + def test_roundtrip(self): + key = b"\x01" * 32 + msg = ["go", {"li": 0, "rep": 1, "wave": 2}] + assert rp.unpack_msg(rp.pack_msg(msg, key), key) == msg + + def test_tampered_frame_rejected(self): + key = b"\x01" * 32 + raw = rp.pack_msg(["hello", {}], key) + bad = raw[:-1] + bytes([raw[-1] ^ 0xFF]) + with pytest.raises(rp._TransferError): + rp.unpack_msg(bad, key) + + def test_wrong_key_rejected(self): + raw = rp.pack_msg(["hello", {}], b"\x01" * 32) + with pytest.raises(rp._TransferError): + rp.unpack_msg(raw, b"\x02" * 32) + + def test_short_frame_rejected(self): + with pytest.raises(rp._TransferError): + rp.unpack_msg(b"tiny", b"\x01" * 32) + + def test_addr_file_owner_only(self, tmp_path): + path = str(tmp_path / "rendezvous" / "ctx0_gen0.addr") + rp.write_addr(path, {"host": "h", "port": 1, "key": "aa"}) + assert (os.stat(path).st_mode & 0o777) == 0o600 + with open(path) as f: + assert json.load(f)["key"] == "aa" + + +def test_use_kv_cache_manager_v2_flags(): + # Absent -> "auto" (the driver resolves it against the model's + # get_model_defaults at runtime, like serving). + plan = pcfg.resolve_plan(_disagg_yaml()) + assert plan["ctx_use_kv_cache_manager_v2"] == "auto" + assert plan["gen_use_kv_cache_manager_v2"] == "auto" + assert pcfg.side_plan(plan, "ctx")["use_kv_cache_manager_v2"] == "auto" + + # Explicit yaml values win, per side. + plan = pcfg.resolve_plan( + _disagg_yaml( + ctx_extra={"kv_cache_config": {"dtype": "fp8", "use_kv_cache_manager_v2": False}}, + gen_extra={"kv_cache_config": {"dtype": "fp8", "use_kv_cache_manager_v2": True}}, + ) + ) + assert plan["ctx_use_kv_cache_manager_v2"] is False + assert plan["gen_use_kv_cache_manager_v2"] is True + assert pcfg.side_plan(plan, "gen")["use_kv_cache_manager_v2"] is True + + +def test_model_kv_shape_vocab_size(tmp_path): + model_dir = tmp_path / "m" + model_dir.mkdir() + (model_dir / "config.json").write_text( + json.dumps( + { + "num_hidden_layers": 2, + "num_attention_heads": 8, + "num_key_value_heads": 8, + "head_dim": 128, + "vocab_size": 129280, + } + ) + ) + assert pcfg.model_kv_shape(str(model_dir))["vocab_size"] == 129280 + + +class TestRendezvousStaleness: + """wait_for_addr must skip addr files stamped by a previous run.""" + + def test_same_job_accepted(self, tmp_path, monkeypatch): + monkeypatch.setenv("SLURM_JOB_ID", "12345") + p = str(tmp_path / "rendezvous" / "ctx0_gen0.addr") + rp.write_addr(p, {"host": "h", "port": 1, "key": "aa"}) + got = rp.wait_for_addr(p, timeout_s=2) + assert got["job"] == "12345" and got["port"] == 1 + + def test_stale_job_skipped_until_timeout(self, tmp_path, monkeypatch): + monkeypatch.setenv("SLURM_JOB_ID", "11111") + p = str(tmp_path / "rendezvous" / "ctx0_gen0.addr") + rp.write_addr(p, {"host": "h", "port": 1, "key": "aa"}) # stamped 11111 + monkeypatch.setenv("SLURM_JOB_ID", "22222") # new run + with pytest.raises(rp._Timeout): + rp.wait_for_addr(p, timeout_s=2) + + def test_no_job_id_accepts_any(self, tmp_path, monkeypatch): + monkeypatch.setenv("SLURM_JOB_ID", "11111") + p = str(tmp_path / "rendezvous" / "ctx0_gen0.addr") + rp.write_addr(p, {"host": "h", "port": 1, "key": "aa"}) + monkeypatch.delenv("SLURM_JOB_ID") # manual non-slurm run + assert rp.wait_for_addr(p, timeout_s=2)["port"] == 1 + + def test_write_addr_replaces_stale_file(self, tmp_path, monkeypatch): + monkeypatch.setenv("SLURM_JOB_ID", "11111") + p = str(tmp_path / "rendezvous" / "ctx0_gen0.addr") + rp.write_addr(p, {"host": "old", "port": 1, "key": "aa"}) + monkeypatch.setenv("SLURM_JOB_ID", "22222") + rp.write_addr(p, {"host": "new", "port": 2, "key": "bb"}) + got = rp.wait_for_addr(p, timeout_s=2) + assert got["host"] == "new" and got["job"] == "22222" + + +def test_wireup_timeout_derivation(): + plan = pcfg.resolve_plan(_disagg_yaml()) # ctx dep4 -> gen dep16 + assert plan["wireup_timeout_s"] == min(1800, 150 * 16) + plan = pcfg.resolve_plan(_disagg_yaml(gen_extra={"tensor_parallel_size": 4})) + assert plan["wireup_timeout_s"] == 600 + plan = pcfg.resolve_plan(_disagg_yaml(cache_transceiver_precheck={"wireup_timeout_s": 42})) + assert plan["wireup_timeout_s"] == 42 + + +def _enabled_line(cfg): + lines = pcfg.precheck_prefix_lines(cfg, "e2e", "$c", "unset &&", max_world=8) + return next(x for x in lines if x.startswith("export ctPrecheckEnabled")) + + +def test_precheck_env_kill_switch_truthy(monkeypatch): + """The TRTLLM_DISAGG_CT_PRECHECK kill switch parses the usual boolean spellings. + + So a force-enable like =true is not silently read as "off", and anything + ambiguous is rejected instead of guessed at. + """ + cfg = {"cache_transceiver_precheck": {"enabled": True}} + monkeypatch.delenv("TRTLLM_DISAGG_CT_PRECHECK", raising=False) + assert _enabled_line(cfg).endswith("=1") # yaml default + for v in ("1", "true", "on", "YES", " True "): + monkeypatch.setenv("TRTLLM_DISAGG_CT_PRECHECK", v) + assert _enabled_line(cfg).endswith("=1"), v + for v in ("0", "false", "off", "no"): + monkeypatch.setenv("TRTLLM_DISAGG_CT_PRECHECK", v) + assert _enabled_line(cfg).endswith("=0"), v + # env overrides yaml either way (kill switch): yaml opt-out but env force-on + monkeypatch.setenv("TRTLLM_DISAGG_CT_PRECHECK", "true") + assert _enabled_line({"cache_transceiver_precheck": {"enabled": False}}).endswith("=1") + monkeypatch.setenv("TRTLLM_DISAGG_CT_PRECHECK", "maybe") + with pytest.raises(ValueError): + _enabled_line(cfg) + + +def test_gate_library_content(tmp_path): + """The gate library loads from next to the draft, with an in-repo fallback. + + It falls back to the in-repo copy for an external draft, strips blank + lines, and errors if truly absent. + """ + dd = tmp_path / "disaggregated" + dd.mkdir() + (dd / "slurm_ct_precheck_gate.sh").write_text( + "run_cache_transceiver_precheck() { :; }\n\n\nrun_cache_transceiver_precheck\n" + ) + draft = str(dd / "slurm_launch_draft.sh") + got = pcfg.gate_library_content(draft, str(tmp_path)) + assert "run_cache_transceiver_precheck()" in got + assert got.endswith("\n") and "\n\n" not in got # blank lines stripped + + # external draft -> fall back to /jenkins/.../slurm_ct_precheck_gate.sh + repo = tmp_path / "repo" + gate2 = repo / "jenkins" / "scripts" / "perf" / "disaggregated" / "slurm_ct_precheck_gate.sh" + gate2.parent.mkdir(parents=True) + gate2.write_text("echo hi\n") + assert pcfg.gate_library_content("/nowhere/draft.sh", str(repo)) == "echo hi\n" + + with pytest.raises(FileNotFoundError): + pcfg.gate_library_content("/nowhere/draft.sh", str(tmp_path / "empty")) + + +def test_rid_tags_dense_within_session(): + """Rids must be dense within a (ctx, gen) session. + + The C++ notification tag is rid & 0xFFF, so dense rids keep tags from + aliasing across reps/lengths. + """ + plan = pcfg.resolve_plan(_disagg_yaml()) # n_pairs=16 + total_reps = plan["warmup_requests"] + plan["num_requests"] + n_pairs = plan["n_pairs"] + + def session_rids(ctx_idx, gen_idx): + out = [] + for li in range(2): + for rep in range(total_reps): + for pair in range(n_pairs): + seq = (li * total_reps + rep) * n_pairs + pair + out.append(rp.make_rid(ctx_idx, gen_idx, 2, seq)) + return out + + a = session_rids(0, 0) + b = session_rids(1, 0) + assert len(set(a)) == len(a) and len(set(b)) == len(b) + assert not (set(a) & set(b)) # globally unique across sessions + tags = [r & 0xFFF for r in a] + assert len(set(tags)) == len(tags) # no tag aliasing within a session + + +class TestMultiPeerOrchestration: + """CPU-only end-to-end run of the 2-ctx x 1-gen session protocol. + + Exercises the exact multi-instance logic of the hardware "B" topology: + real ZMQ sockets + HMAC frames + StatusRecorder + rendezvous files via + the real PrecheckRunner/_serve_gen_peers/_drive_ctx_peers, with only the + GPU transfer methods stubbed out. + """ + + class _FakeComm: + def Get_rank(self): + return 0 + + def Get_size(self): + return 1 + + def bcast(self, obj, root=0): + return obj + + def gather(self, obj, root=0): + return [obj] + + def allgather(self, obj): + return [obj] + + class _FakeParams: + first_gen_tokens = [0] + req_id = 1 + opaque_state = b"op" + draft_tokens = None + ctx_dp_rank = 0 + disagg_info_endpoint = None + + def _mk_runner(self, role, server_idx, plan, work_dir, monkeypatch, fail_ctx=False): + import sys + import types + + # PrecheckRunner.__init__ imports mpi4py only to ensure MPI init. + monkeypatch.setitem(sys.modules, "mpi4py", types.SimpleNamespace(MPI=None)) + # The gen side converts wire params through tensorrt_llm bindings; + # identity is fine here (params_to_wire is covered separately). + monkeypatch.setattr(rp, "params_from_wire", lambda d: d) + + args = types.SimpleNamespace(server_idx=server_idx, work_dir=work_dir) + side = pcfg.side_plan(plan, role) + runner = rp.PrecheckRunner(args, plan, side, self._FakeComm()) + + calls = {"waves": 0} + + def ctx_run_wave(peer_idx, li, req_len, rep, wave): + if fail_ctx: + raise rp._TransferError("injected ctx failure") + calls["waves"] += 1 + return {p: self._FakeParams() for p in wave}, {} + + runner.ctx_run_wave = ctx_run_wave + runner.ctx_finish_wave = lambda reqs: None + runner.gen_run_wave = lambda peer_idx, li, req_len, rep, wave, params: (True, "") + runner._calls = calls + return runner + + def _run(self, tmp_path, monkeypatch, fail_ctx_idx=None): + import threading + + monkeypatch.setenv("SLURM_JOB_ID", "777") + # Publish loopback in the addr files: the real node hostname may not + # resolve in sandboxed/CI environments, and everything is one process. + monkeypatch.setenv("SLURMD_NODENAME", "127.0.0.1") + cfg = _disagg_yaml( + hardware={"gpus_per_node": 4, "num_ctx_servers": 2, "num_gen_servers": 1}, + cache_transceiver_precheck={ + "request_lengths": [32], + "num_requests": 1, + "warmup_requests": 1, + "rendezvous_timeout_s": 30, + "wave_timeout_s": 30, + "wireup_timeout_s": 0, + }, + ) + plan = pcfg.resolve_plan(cfg) + work = str(tmp_path) + noop = lambda *a, **k: None # noqa: E731 - signal.alarm needs main thread + + gen = self._mk_runner("gen", 0, plan, work, monkeypatch) + ctxs = [ + self._mk_runner( + "ctx", + i, + plan, + work, + monkeypatch, + fail_ctx=(fail_ctx_idx is not None and i == fail_ctx_idx), + ) + for i in range(2) + ] + + failures = [] + + def rec(peer, exc): + failures.append((peer, type(exc).__name__)) + + threads = [ + threading.Thread( + target=rp._serve_gen_peers, args=(c, plan, noop, noop, rec), daemon=True + ) + for c in ctxs + ] + for t in threads: + t.start() + rp._drive_ctx_peers( + gen, noop, noop, rp._make_peer_failure_recorder(gen, noop, {"what": "test"}) + ) + for t in threads: + t.join(timeout=60) + assert not t.is_alive(), "ctx serve thread wedged" + return plan, gen, ctxs, failures + + def test_two_ctx_full_pass(self, tmp_path, monkeypatch): + plan, gen, ctxs, failures = self._run(tmp_path, monkeypatch) + assert not failures + # gen recorded a PASS per (peer, req_len) + assert {(c["peer"], c["status"]) for c in gen.recorder.cases} == { + ("ctx_0", "PASS"), + ("ctx_1", "PASS"), + } + # every ctx served the full schedule (reps x waves) and got its + # deferred done (PASS recorded only after done/bye completes) + total_waves = len(pcfg.waves(plan)) * (plan["warmup_requests"] + plan["num_requests"]) + for c in ctxs: + assert c._calls["waves"] == total_waves + assert [x["status"] for x in c.recorder.cases] == ["PASS"] + + def test_ctx_failure_last_peer(self, tmp_path, monkeypatch): + # The failing pair is driven LAST: the earlier healthy peer already + # completed, so there is nothing left to fail-fast/skip. + plan, gen, ctxs, failures = self._run(tmp_path, monkeypatch, fail_ctx_idx=1) + # gen side: healthy peer unaffected, failing peer gets a clear verdict + by_peer = {c["peer"]: c["status"] for c in gen.recorder.cases} + assert by_peer == {"ctx_0": "PASS", "ctx_1": "TRANSFER_ERROR"} + # ctx_1's own serve loop surfaced the failure (its peer is gen_0) + assert ("gen_0", "_TransferError") in failures + # ctx_0 served its full schedule and got the deferred done + assert [c["status"] for c in ctxs[0].recorder.cases] == ["PASS"] + + def test_fail_fast_skips_remaining(self, tmp_path, monkeypatch): + # The FIRST-driven pair fails: the remaining pair must be skipped + # (not tested against a fabric already known bad), and told to abort + # so it tears down promptly instead of waiting out its handshake alarm. + plan, gen, ctxs, failures = self._run(tmp_path, monkeypatch, fail_ctx_idx=0) + by_peer = {c["peer"]: c["status"] for c in gen.recorder.cases} + assert by_peer == {"ctx_0": "TRANSFER_ERROR", "ctx_1": "SKIP"} + # ctx_1 never ran a single transfer wave: fail-fast reached it first. + assert ctxs[1]._calls["waves"] == 0 + # ctx_1 recorded a non-failing SKIP (its driver aborted the session). + assert [c["status"] for c in ctxs[1].recorder.cases] == ["SKIP"] + # the shared, job-stamped abort flag was dropped. + assert rp.abort_flag_reason(str(tmp_path)) is not None + # SKIP does not count toward the overall verdict; only ctx_0 failed. + assert [c["peer"] for c in gen.recorder.failed_cases()] == ["ctx_0"] + + def test_abort_flag_stale_job_ignored(self, tmp_path, monkeypatch): + # A flag left by a previous run (different SLURM_JOB_ID) in a reused + # work dir must not fail-fast a fresh run -- same staleness rule as + # addr files. + monkeypatch.setenv("SLURM_JOB_ID", "111") + rp.raise_abort_flag(str(tmp_path), "old run failure") + assert rp.abort_flag_reason(str(tmp_path)) == "old run failure" + monkeypatch.setenv("SLURM_JOB_ID", "222") + assert rp.abort_flag_reason(str(tmp_path)) is None + + +def test_ctx_run_wave_missing_params_broadcast(tmp_path, monkeypatch): + """#4 regression: the "missing context_phase_params" verdict must be broadcast. + + It is computed only on the instance leader (only it holds the gathered + params) -- without the broadcast, a NON-leader rank keeps reason=None, + returns, and enters the next collective while the leader raises, + deadlocking the step until the watchdog SIGKILLs it (misreported as + TIMEOUT). + + Run the real ctx_run_wave on a non-leader rank: with the leader's verdict + delivered via bcast the rank must raise; with a clean (None) broadcast it + must return normally. + """ + import sys + import types + + monkeypatch.setitem(sys.modules, "mpi4py", types.SimpleNamespace(MPI=None)) + # ctx_run_wave imports tensorrt_llm only for logger.info, never reached with + # no owned pairs; a stub keeps the test pure-CPU. + monkeypatch.setitem( + sys.modules, + "tensorrt_llm", + types.SimpleNamespace(logger=types.SimpleNamespace(info=lambda *a, **k: None)), + ) + + class _NonLeaderComm: + def __init__(self, bcast_ret): + self._bcast_ret = bcast_ret + + def Get_rank(self): + return 1 # non-leader (leader is rank 0) + + def Get_size(self): + return 2 + + def allgather(self, obj): + return ["", ""] # no local send error on any rank + + def gather(self, obj, root=0): + return None # only the leader receives the gathered params + + def bcast(self, obj, root=0): + return self._bcast_ret # the leader's verdict reaching this rank + + plan = pcfg.resolve_plan( + _disagg_yaml( + cache_transceiver_precheck={ + "request_lengths": [32], + "num_requests": 1, + "warmup_requests": 1, + "wireup_timeout_s": 0, + } + ) + ) + side = pcfg.side_plan(plan, "ctx") + args = types.SimpleNamespace(server_idx=0, work_dir=str(tmp_path)) + + def _mk(bcast_ret): + r = rp.PrecheckRunner(args, plan, side, _NonLeaderComm(bcast_ret)) + r.mapping = types.SimpleNamespace(pp_rank=1, tp_rank=1) # unused non-leader + r._owned = lambda wave: [] # skip the GPU send path; verdict arrives via bcast + return r + + # leader broadcast a missing-params verdict -> the non-leader raises it too + with pytest.raises(rp._TransferError, match="missing context_phase_params"): + _mk("missing context_phase_params for pairs [0]").ctx_run_wave( + peer_idx=0, li=0, req_len=32, rep=0, wave=[0] + ) + + # leader broadcast None (all good) -> the non-leader returns cleanly + params, reqs = _mk(None).ctx_run_wave(peer_idx=0, li=0, req_len=32, rep=0, wave=[0]) + assert params == {} and reqs == {} + + +def test_status_env_snapshot_excludes_nixl(tmp_path, monkeypatch): + """NIXL_* is not captured. + + The only such variable seen in practice is NIXL_VERSION, a stale + NGC-base-image marker that misstates the version of the actually-linked + library. + """ + monkeypatch.setenv("NIXL_VERSION", "1.0.0") + monkeypatch.setenv("NIXL_PLUGIN_DIR", "/opt/x") + monkeypatch.setenv("UCX_TLS", "rc,cuda_copy") + rec = rp.StatusRecorder(str(tmp_path), "gen", 0, is_leader=True) + assert not any(k.startswith("NIXL_") for k in rec.env) + assert rec.env["UCX_TLS"] == "rc,cuda_copy" # behavioral vars still captured + + +def test_sparse_attention_model_uses_simplified_mla_pool(tmp_path): + """DeepSeek V4 / DSA can't be modeled as a single KV pool. + + The precheck is a NETWORK check, so it falls back to a simple MLA-flavored + stand-in pool (real layer count, one latent head, is_mla=True) and still + runs -- never skips. + """ + d = tmp_path / "v4" + d.mkdir() + (d / "config.json").write_text( + json.dumps( + { + "architectures": ["DeepseekV4ForCausalLM"], + "num_hidden_layers": 43, + "num_attention_heads": 64, + "num_key_value_heads": 1, + "head_dim": 512, + "index_head_dim": 128, + "index_n_heads": 64, + "index_topk": 512, + "sliding_window": 128, + "vocab_size": 129280, + } + ) + ) + shape = pcfg.model_kv_shape(str(d)) + assert shape.get("simplified") and "sparse" in shape["simplified"] + assert shape["is_mla"] is True and shape["num_kv_heads"] == 1 + assert shape["num_layers"] == 43 # real layer count preserved + assert shape["head_dim"] == 512 and shape["vocab_size"] == 129280 + # a plain MLA model is modeled normally (no simplified marker) + m = tmp_path / "mla" + m.mkdir() + (m / "config.json").write_text( + json.dumps({"num_hidden_layers": 4, "kv_lora_rank": 512, "qk_rope_head_dim": 64}) + ) + assert not pcfg.model_kv_shape(str(m)).get("simplified") + + +def test_python_transceiver_bandwidth_csv(tmp_path): + """Bandwidth from the Python transceiver's perf_logger CSVs. + + PerfLogManager names them "_.csv" (it gives + TRTLLM_KVCACHE_TIME_OUTPUT_PATH top priority); the parser identifies them + by header columns. Median over KVSendTask throughput_mbs (MiB/s -> GB/s), + receiver rows ignored. + """ + header = ( + "timestamp,task_type,unique_rid,peer_rank,transfer_size_bytes," + "avg_segment_size_bytes,transfer_entry_count,prepare_args_latency_ms," + "queue_latency_ms,transfer_latency_ms,task_latency_ms,throughput_mbs" + ) + # two ctx ranks, each its own perf file; KVSendTask rows carry throughput + (tmp_path / "cd93dae6-9d75-4b0e-8a89-2c9e2f0f1a2b_0.csv").write_text( + header + "\n" + "t,KVSendTask,1,0,1000,,,0,0,0,0,102400.00\n" + "t,AuxSendTask,1,0,10,,,0,0,0,0,1.00\n" # tiny metadata, ignored + ) + (tmp_path / "cd93dae6-9d75-4b0e-8a89-2c9e2f0f1a2b_1.csv").write_text( + header + "\nt,KVSendTask,2,1,1000,,,0,0,0,0,204800.00\n" + ) + # a receiver file (no throughput) must not contribute + (tmp_path / "5d7b1f80-aaaa-bbbb-cccc-ddddeeeeffff_8.csv").write_text( + header + "\nt,KVRecvTask,3,0,,,,,,,5.0,\n" + ) + bw = rp.parse_python_bandwidth_gbps(str(tmp_path)) + # median(102400, 204800) = 153600 MiB/s -> GB/s (*1024^2/1e9) + assert abs(bw - 153600 * 1024 * 1024 / 1e9) < 1e-9 + # no perf files -> None + assert rp.parse_python_bandwidth_gbps(str(tmp_path / "empty")) is None diff --git a/tests/unittest/others/test_cache_transceiver_precheck_run.py b/tests/unittest/others/test_cache_transceiver_precheck_run.py new file mode 100644 index 000000000000..ed060d1e9053 --- /dev/null +++ b/tests/unittest/others/test_cache_transceiver_precheck_run.py @@ -0,0 +1,467 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the disagg cache-transceiver precheck DRIVER. + +Target: tests/scripts/perf-sanity/cache_transceiver_precheck/run_precheck.py + +Two halves: + +- Pure-logic tests (no torch / tensorrt_llm / MPI): wire format, rid/seed + scheme, rendezvous + abort-flag files, StatusRecorder, bandwidth CSV + parsing, schedule/timeout derivation. + +- Internal-API contract tests (import tensorrt_llm, no GPU work): the + precheck drives TRT-LLM internals directly (_torch.pyexecutor.*, + bindings.internal.*, private llm_utils resolvers), which carry no stability + promise. run_precheck.load_internal_apis() is the single owner of those + imports; these tests exercise it plus the constructor/signature shapes the + driver relies on, so an upstream refactor fails HERE in pre-merge CI + instead of aborting the SLURM disagg perf pipeline at runtime. +""" + +import base64 +import json +import os +import sys +import types + +import pytest + +_PRECHECK_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "..", + "..", + "tests", + "scripts", + "perf-sanity", + "cache_transceiver_precheck", +) +sys.path.insert(0, os.path.abspath(_PRECHECK_DIR)) + +import run_precheck as rp # noqa: E402 (stdlib-only at import time) + +KEY = b"k" * 32 + + +# --------------------------------------------------------------------------- # +# rid / seed scheme +# --------------------------------------------------------------------------- # +def test_make_rid_unique_and_dense_within_session(): + num_ctx, num_gen, seqs = 3, 2, 50 + rids = set() + for gj in range(num_gen): + for ci in range(num_ctx): + session = [rp.make_rid(ci, gj, num_ctx, s) for s in range(seqs)] + # Dense in-session sequence: consecutive rids -> unique low-12-bit + # tags among any 4096 consecutive requests (tagFromRequestId). + assert session == list(range(session[0], session[0] + seqs)) + rids.update(session) + assert len(rids) == num_ctx * num_gen * seqs + assert all(r >= 1 for r in rids) + + +def test_seed_for_deterministic_and_distinct(): + assert rp.seed_for(7, 3) == rp.seed_for(7, 3) # rank-independent by construction + seeds = {rp.seed_for(rid, layer) for rid in (1, 2, 3) for layer in (0, 1, 2)} + assert len(seeds) == 9 + assert all(0 <= s <= 0x7FFFFFFF for s in seeds) + + +# --------------------------------------------------------------------------- # +# HMAC control-channel wire format +# --------------------------------------------------------------------------- # +def test_pack_unpack_roundtrip(): + obj = ["go", {"li": 0, "rep": 1, "wave": [0, 1]}] + assert rp.unpack_msg(rp.pack_msg(obj, KEY), KEY) == obj + + +def test_unpack_rejects_tampered_frame(): + raw = bytearray(rp.pack_msg(["hello", {}], KEY)) + raw[0] ^= 0xFF + with pytest.raises(rp._TransferError, match="HMAC"): + rp.unpack_msg(bytes(raw), KEY) + + +def test_unpack_rejects_wrong_key(): + raw = rp.pack_msg(["hello", {}], KEY) + with pytest.raises(rp._TransferError, match="HMAC"): + rp.unpack_msg(raw, b"x" * 32) + + +def test_unpack_rejects_short_frame(): + with pytest.raises(rp._TransferError, match="too short"): + rp.unpack_msg(b"tiny", KEY) + + +def test_params_to_wire_is_json_safe(): + p = types.SimpleNamespace( + first_gen_tokens=[1, 2], + req_id=42, + opaque_state=b"\x00\x01binary", + draft_tokens=None, + ctx_dp_rank=3, + disagg_info_endpoint="tcp://h:1", + ) + wire = rp.params_to_wire(p) + decoded = json.loads(json.dumps(wire)) # must survive the ZMQ JSON hop + assert base64.b64decode(decoded["opaque_state"]) == p.opaque_state + assert decoded["req_id"] == 42 + assert decoded["ctx_dp_rank"] == 3 + assert decoded["ctx_info_endpoint"] == "tcp://h:1" + + +# --------------------------------------------------------------------------- # +# Rendezvous + abort-flag files +# --------------------------------------------------------------------------- # +def test_addr_roundtrip(tmp_path, monkeypatch): + monkeypatch.setenv("SLURM_JOB_ID", "123") + path = rp.addr_path(str(tmp_path), 0, 1) + rp.write_addr(path, {"host": "h", "port": 5, "key": KEY.hex()}) + assert os.stat(path).st_mode & 0o777 == 0o600 # carries the HMAC key + payload = rp.wait_for_addr(path, timeout_s=5) + assert (payload["host"], payload["port"], payload["job"]) == ("h", 5, "123") + + +def test_wait_for_addr_rejects_stale_job(tmp_path, monkeypatch): + monkeypatch.setenv("SLURM_JOB_ID", "old-run") + path = rp.addr_path(str(tmp_path), 0, 0) + rp.write_addr(path, {"host": "h", "port": 5, "key": KEY.hex()}) + monkeypatch.setenv("SLURM_JOB_ID", "new-run") # requeued job, reused work dir + with pytest.raises(rp._Timeout): + rp.wait_for_addr(path, timeout_s=1.5) + + +def test_wait_for_addr_times_out_on_missing_file(tmp_path): + with pytest.raises(rp._Timeout): + rp.wait_for_addr(rp.addr_path(str(tmp_path), 0, 0), timeout_s=0) + + +def test_abort_flag_roundtrip_and_write_once(tmp_path, monkeypatch): + monkeypatch.setenv("SLURM_JOB_ID", "1") + work = str(tmp_path) + assert rp.abort_flag_reason(work) is None + rp.raise_abort_flag(work, "first failure\nsecond line ignored") + assert rp.abort_flag_reason(work) == "first failure" + rp.raise_abort_flag(work, "later failure") # write-once: first reason wins + assert rp.abort_flag_reason(work) == "first failure" + + +def test_abort_flag_stale_job_ignored(tmp_path, monkeypatch): + monkeypatch.setenv("SLURM_JOB_ID", "old") + rp.raise_abort_flag(str(tmp_path), "stale") + monkeypatch.setenv("SLURM_JOB_ID", "new") + assert rp.abort_flag_reason(str(tmp_path)) is None + + +# --------------------------------------------------------------------------- # +# StatusRecorder +# --------------------------------------------------------------------------- # +def _read_status(tmp_path, name): + with open(os.path.join(str(tmp_path), "status", f"{name}.status")) as f: + text = f.read() + with open(os.path.join(str(tmp_path), "status", f"{name}.json")) as f: + doc = json.load(f) + return text, doc + + +def test_recorder_pass(tmp_path): + rec = rp.StatusRecorder(str(tmp_path), "gen", 0, is_leader=True) + rec.record("ctx_0", 1024, "PASS") + text, doc = _read_status(tmp_path, "gen_0") + assert text.startswith("RUNNING") # not final yet: a SIGKILL must not read as PASS + assert doc["overall"] == "RUNNING" + rec.finalize(extra={"transceiver_runtime": "CPP"}) + text, doc = _read_status(tmp_path, "gen_0") + assert text.startswith("PASS gen_0") + assert doc["overall"] == "PASS" + assert doc["transceiver_runtime"] == "CPP" + + +def test_recorder_failure_summary_first_line_only(tmp_path): + rec = rp.StatusRecorder(str(tmp_path), "ctx", 1, is_leader=True) + rec.record("gen_0", 1024, "PASS") + rec.record("gen_1", 2048, "TRANSFER_ERROR", "boom\ntraceback line\nmore") + rec.finalize() + text, doc = _read_status(tmp_path, "ctx_1") + assert text.startswith("FAIL ctx_1") + assert "boom | traceback line" in text + assert "more" not in text # full reason only in the json + assert doc["overall"] == "FAIL" + assert doc["cases"][1]["reason"].endswith("more") + + +def test_recorder_skip_is_not_a_failure(tmp_path): + rec = rp.StatusRecorder(str(tmp_path), "gen", 0, is_leader=True) + rec.record("ctx_0", 0, "SKIP", "fail-fast") + assert rec.failed_cases() == [] + rec.finalize() + text, _ = _read_status(tmp_path, "gen_0") + assert text.startswith("PASS") + + +def test_recorder_non_leader_writes_nothing(tmp_path): + rec = rp.StatusRecorder(str(tmp_path), "gen", 0, is_leader=False) + rec.record("ctx_0", 0, "TRANSFER_ERROR", "x") + rec.finalize() + assert not os.path.exists(os.path.join(str(tmp_path), "status")) + + +# --------------------------------------------------------------------------- # +# Bandwidth CSV parsing +# --------------------------------------------------------------------------- # +def test_parse_bandwidth_gbps_median(tmp_path): + # C++ names timing CSVs "__.csv" (instanceId is a + # runtime UUID), so the parser must suffix-match, not expect "rank_*". + path = tmp_path / "3c9f0e2a-1111-2222-3333-444455556666_2_recv.csv" + path.write_text("RequestID,Bandwidth(Gbps),Delay(ms)\n1,80,0\n2,160,0\n3,240,0\n") + # Gbps -> GB/s (/8); median of [10, 20, 30] + assert rp.parse_bandwidth_gbps(str(tmp_path), 2) == 20.0 + + +def test_parse_bandwidth_gbps_duplicate_columns_mean(tmp_path): + # C++ repeats the Bandwidth(Gbps) column once per transmission; the parser + # must average them per request (DictReader would keep only the last). + (tmp_path / "uuid_0_recv.csv").write_text( + "RequestID,Bandwidth(Gbps),Bandwidth(Gbps)\n1,80,240\n" + ) + # mean(80, 240) = 160 Gbps -> /8 = 20 GB/s + assert rp.parse_bandwidth_gbps(str(tmp_path), 0) == 20.0 + + +def test_parse_bandwidth_gbps_rank_suffix_no_cross_match(tmp_path): + # Rank 1 must not pick up rank 11's file (the suffix's leading "_"). + (tmp_path / "uuid_11_recv.csv").write_text("RequestID,Bandwidth(Gbps)\n1,80\n") + assert rp.parse_bandwidth_gbps(str(tmp_path), 1) is None + assert rp.parse_bandwidth_gbps(str(tmp_path), 11) == 10.0 + + +def test_parse_bandwidth_gbps_missing_or_malformed(tmp_path): + assert rp.parse_bandwidth_gbps(str(tmp_path), 0) is None + (tmp_path / "uuid_0_recv.csv").write_text("RequestID,Delay(ms)\n1,0\n") + assert rp.parse_bandwidth_gbps(str(tmp_path), 0) is None + + +def test_parse_python_bandwidth_gbps(tmp_path): + # PerfLogManager gives TRTLLM_KVCACHE_TIME_OUTPUT_PATH top priority and + # names task CSVs "_.csv" (no fixed prefix); the + # parser identifies them by header columns, not name. + (tmp_path / "cd93dae6-9d75-4b0e-8a89-2c9e2f0f1a2b_0.csv").write_text( + "task_type,throughput_mbs\nKVSendTask,1024\nKVRecvTask,\n" + ) + (tmp_path / "cd93dae6-9d75-4b0e-8a89-2c9e2f0f1a2b_1.csv").write_text( + "task_type,throughput_mbs\nKVSendTask,3072\n" + ) + # MiB/s -> GB/s (*1024^2/1e9); median of [1024, 3072] MiB/s = 2048 MiB/s + expected = 2048 * 1024 * 1024 / 1e9 + assert abs(rp.parse_python_bandwidth_gbps(str(tmp_path)) - expected) < 1e-9 + assert rp.parse_python_bandwidth_gbps(str(tmp_path / "nowhere")) is None + + +def test_parse_python_bandwidth_gbps_ignores_cpp_csvs(tmp_path): + # C++ send/recv and gen-summary CSVs share csv_dir; they lack the + # task_type/throughput_mbs columns and must not contribute samples. + (tmp_path / "uuid_0_recv.csv").write_text("RequestID,Bandwidth(Gbps)\n1,80\n") + (tmp_path / "uuid_0_gen_transfer_summary.csv").write_text( + "RequestID,gen_side_transfer_time(ms),kv_cache_size\n1,1.0,1024\n" + ) + assert rp.parse_python_bandwidth_gbps(str(tmp_path)) is None + + +def test_parse_bandwidth_gbps_ignores_gen_summary(tmp_path): + # "__gen_transfer_summary.csv" must not match the + # "__recv.csv" suffix. + (tmp_path / "uuid_0_gen_transfer_summary.csv").write_text( + "RequestID,gen_side_transfer_time(ms),kv_cache_size\n1,1.0,1024\n" + ) + assert rp.parse_bandwidth_gbps(str(tmp_path), 0) is None + + +# --------------------------------------------------------------------------- # +# Schedule / timeout derivation +# --------------------------------------------------------------------------- # +def _plan(**overrides): + plan = { + "request_lengths": [64, 128], + "warmup_requests": 1, + "num_requests": 2, + "n_pairs": 3, + "wave_size": 2, + "rendezvous_timeout_s": 600, + "wireup_timeout_s": 300, + "wave_timeout_s": 180, + } + plan.update(overrides) + return plan + + +def test_schedule_covers_all_cells_in_lockstep_order(): + plan = _plan() + sched = rp._schedule(plan) + # 2 lengths x (1 warmup + 2 measured) reps x 2 waves ([0,1] and [2]) + assert len(sched) == 2 * 3 * 2 + assert sched[0] == (0, 64, 0, [0, 1]) + assert sched[1] == (0, 64, 0, [2]) + assert sched[-1] == (1, 128, 2, [2]) + + +def test_timeout_budgets(): + plan = _plan() + # Handshakes serialize across peers: rendezvous + per-peer slack. + assert rp.hello_timeout_s(plan, 2) == 600 + 2 * (300 + 300) + # Only the schedule's FIRST rep pays the NIXL wire-up allowance. + assert rp.wave_timeout_s(plan, 0, 0) == 180 + 300 + assert rp.wave_timeout_s(plan, 0, 1) == 180 + assert rp.wave_timeout_s(plan, 1, 0) == 180 + + +# --------------------------------------------------------------------------- # +# Internal-API contract (imports tensorrt_llm; no GPU work) +# --------------------------------------------------------------------------- # +class TestInternalApiContract: + @pytest.fixture(scope="class") + def api(self): + pytest.importorskip("tensorrt_llm") + return rp.load_internal_apis() + + def test_loader_caches(self, api): + assert rp.load_internal_apis() is api + + def test_create_kv_cache_transceiver_signature(self, api): + import inspect + + params = inspect.signature(api.create_kv_cache_transceiver).parameters + # Exactly the positional call shape PrecheckRunner.setup uses. + assert list(params)[:5] == [ + "mapping", + "dist", + "kv_cache_manager", + "attention_type", + "cache_transceiver_config", + ] + + def test_transceiver_interface_methods(self, api): + import importlib + + mod = importlib.import_module(api.create_kv_cache_transceiver.__module__) + base = mod.KvCacheTransceiver + for meth in ( + "respond_and_send_async", + "request_and_receive_async", + "check_context_transfer_status", + "check_gen_transfer_status", + ): + assert hasattr(base, meth), f"KvCacheTransceiver lost {meth}" + + @pytest.mark.parametrize("manager_attr", ["KVCacheManager", "KVCacheManagerV2"]) + def test_kv_cache_manager_ctor_kwargs(self, api, manager_attr): + import inspect + + params = inspect.signature(getattr(api, manager_attr).__init__).parameters + needed = { + "num_layers", + "num_kv_heads", + "head_dim", + "tokens_per_block", + "max_seq_len", + "max_batch_size", + "mapping", + "dtype", + "spec_config", + } + if manager_attr == "KVCacheManagerV2": + needed |= {"vocab_size", "is_disagg"} + missing = needed - set(params) + assert not missing, f"{manager_attr} ctor lost kwargs: {sorted(missing)}" + + def test_serving_resolvers(self, api): + import inspect + + # The driver calls resolve_kv_cache_manager_v2_auto(shim, defaults): + # the first two params are fixed, anything added later must default. + v2 = inspect.signature(api.resolve_kv_cache_manager_v2_auto).parameters + assert list(v2)[:2] == ["llm_args", "model_defaults_dict"] + assert all(p.default is not inspect.Parameter.empty for p in list(v2.values())[2:]) + rt = inspect.signature(api.resolve_transceiver_runtime_auto).parameters + assert list(rt)[:1] == ["llm_args"] and len(rt) >= 3 + + def test_enum_members(self, api): + for enum, members in ( + (api.DataType, ("FP8", "HALF", "BF16")), + (api.CacheTypeCpp, ("SELF", "SELFKONLY")), + (api.AttentionTypeCpp, ("DEFAULT", "MLA")), + (api.LlmRequestState, ("DISAGG_GENERATION_TRANS_COMPLETE", "DISAGG_TRANS_ERROR")), + ( + api.LlmRequestType, + ("LLMREQUEST_TYPE_CONTEXT_ONLY", "LLMREQUEST_TYPE_GENERATION_ONLY"), + ), + ): + for m in members: + assert hasattr(enum, m), f"{enum} lost member {m}" + + def test_hang_detector_surface(self, api): + import inspect + + params = inspect.signature(api.HangDetector.__init__).parameters + assert {"timeout", "on_detected"} <= set(params) + for meth in ("start", "checkpoint", "cancel_task", "stop"): + assert hasattr(api.HangDetector, meth), f"HangDetector lost {meth}" + + def test_config_constructors(self, api): + cache_cfg = api.CacheTransceiverConfig(backend="UCX", max_tokens_in_buffer=1024) + assert hasattr(cache_cfg, "transceiver_runtime") + api.KvCacheConfigCpp(max_tokens=64, enable_block_reuse=False) + api.MTPDecodingConfig(num_nextn_predict_layers=1) + api.Mapping( + world_size=1, + rank=0, + gpus_per_node=1, + tp_size=1, + pp_size=1, + cp_size=1, + enable_attention_dp=False, + ) + assert hasattr(api.Distributed, "get") + + def test_params_wire_roundtrip_through_real_bindings(self, api): + # opaque_state must DESERIALIZE in the ContextPhaseParams ctor + # (arbitrary bytes -> std::bad_alloc), so use the serialized empty + # state: b"" is re-encoded by the bindings into its canonical form. + source = api.DisaggregatedParams( + ctx_request_id=42, + first_gen_tokens=[7, 8], + opaque_state=b"", + draft_tokens=[9], + ctx_dp_rank=1, + ctx_info_endpoint="tcp://host:1234", + ).get_context_phase_params() + restored = rp.params_from_wire(rp.params_to_wire(source)) + assert rp.params_to_wire(restored) == rp.params_to_wire(source) + + def test_make_request_shapes(self, api): + ctx_req = rp.make_request(True, rid=11, req_len=8, runtime="CPP") + assert ctx_req.py_request_id == 11 + py_ctx = rp.make_request(True, rid=12, req_len=8, runtime="PYTHON") + assert py_ctx.py_disaggregated_params.request_type == "context_only" + ctx_params = api.DisaggregatedParams( + ctx_request_id=13, + first_gen_tokens=[1], + opaque_state=b"", + ctx_dp_rank=0, + ).get_context_phase_params() + gen_req = rp.make_request(False, rid=13, req_len=8, runtime="CPP", ctx_params=ctx_params) + assert gen_req.py_request_id == 13 + py_gen = rp.make_request(False, rid=14, req_len=8, runtime="PYTHON", ctx_params=ctx_params) + assert py_gen.py_disaggregated_params.request_type == "generation_only" diff --git a/tests/unittest/others/test_http_utils_fail_fast.py b/tests/unittest/others/test_http_utils_fail_fast.py new file mode 100644 index 000000000000..f11996525eab --- /dev/null +++ b/tests/unittest/others/test_http_utils_fail_fast.py @@ -0,0 +1,163 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fail-fast + log-tail-dump behavior of wait_for_endpoint_ready (no GPU, no server).""" + +import subprocess + +import pytest +from test_common.error_utils import check_error, report_error +from test_common.http_utils import fail_if_proc_died, wait_for_endpoint_ready + +# The CPU-Generic CI stages select tests with `-m cpu_only`; without this +# marker every test here is deselected and pytest exits with code 5. +pytestmark = pytest.mark.cpu_only + +# Nothing listens here; connections are refused immediately. +DEAD_URL = "http://127.0.0.1:9/health" + + +@pytest.fixture(autouse=True) +def _no_proxy(monkeypatch): + """A corporate HTTP proxy answering 200 would make DEAD_URL look ready.""" + monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost") + monkeypatch.setenv("no_proxy", "127.0.0.1,localhost") + monkeypatch.delenv("HTTP_PROXY", raising=False) + monkeypatch.delenv("http_proxy", raising=False) + + +def _server_log(tmp_path, content): + p = tmp_path / "server.log" + p.write_text(content) + return str(p) + + +def test_dead_server_proc_fails_fast_and_dumps_log(tmp_path): + log = _server_log(tmp_path, "loading weights...\nlast server words\n") + proc = subprocess.Popen(["true"]) + proc.wait() + with pytest.raises(RuntimeError) as e: + wait_for_endpoint_ready(DEAD_URL, timeout=30, check_files=[log], server_proc=proc) + msg = str(e.value) + assert "exited unexpectedly with code" in msg + # The server-side story must land in the failure message (CI log). + assert "last server words" in msg + + +def test_error_keyword_in_server_log_fails_fast(tmp_path): + log = _server_log(tmp_path, "starting\nRuntimeError: engine exploded\n") + with pytest.raises(RuntimeError) as e: + wait_for_endpoint_ready(DEAD_URL, timeout=30, check_files=[log], check_interval=0.0) + msg = str(e.value) + assert "Found error in server file" in msg + assert "engine exploded" in msg + + +def test_ready_timeout_dumps_log_tail(tmp_path): + log = _server_log(tmp_path, "clean init so far, no error keywords\n") + with pytest.raises(RuntimeError) as e: + # check_interval > timeout so the error-scan never trips; we want the + # timeout path itself. + wait_for_endpoint_ready(DEAD_URL, timeout=2, check_files=[log], check_interval=60.0) + msg = str(e.value) + assert "did not become ready within" in msg + assert "clean init so far" in msg + + +def test_missing_check_file_does_not_crash_the_wait(tmp_path): + missing = str(tmp_path / "not-written-yet.log") + with pytest.raises(RuntimeError) as e: + wait_for_endpoint_ready(DEAD_URL, timeout=2, check_files=[missing], check_interval=0.0) + # Reaches the timeout path (not a FileNotFoundError) and reports the + # missing log in the dump. + msg = str(e.value) + assert "did not become ready within" in msg + assert "Path doesn't exist" in msg + + +def test_fail_if_proc_died_raises_with_log_tail(tmp_path): + """Event-driven babysitter check: dead child -> immediate raise + log tail.""" + log = _server_log(tmp_path, "gen server init...\nfinal gen words\n") + proc = subprocess.Popen(["true"]) + proc.wait() + with pytest.raises(RuntimeError) as e: + fail_if_proc_died(proc, "GEN_0 server", [log]) + msg = str(e.value) + assert "GEN_0 server exited unexpectedly with code 0" in msg + assert "final gen words" in msg + + +def test_fail_if_proc_died_noop_when_alive_or_none(tmp_path): + fail_if_proc_died(None, "no server") # no-op + proc = subprocess.Popen(["sleep", "5"]) + try: + fail_if_proc_died(proc, "alive server") # no raise + finally: + proc.kill() + proc.wait() + + +def test_report_error_keyword_scan_fires(tmp_path): + """Regression: report_error scanned an exhausted handle; keywords never matched.""" + log = _server_log( + tmp_path, + "line one\nRuntimeError: boom\nline three\n", + ) + with pytest.raises(RuntimeError) as e: + report_error("wrapper message", [log]) + msg = str(e.value) + assert "wrapper message" in msg + assert "Error line 2" in msg + assert "RuntimeError: boom" in msg + + +def test_check_error_skips_benign_autotuner_lines(tmp_path): + """Autotuner warmup probe-OOMs are expected on healthy startups.""" + log = _server_log( + tmp_path, + "[Autotuner] Single-pair run failed: CUDA out of memory. Tried to allocate...\n" + "normal progress line\n", + ) + assert check_error(log) == [] + # A real error outside the benign marker still trips the scan. + log2 = _server_log(tmp_path, "RuntimeError: engine exploded\n") + assert len(check_error(log2)) == 1 + + +def test_autotuner_marker_does_not_hide_real_errors(tmp_path): + """Only the marker+OOM combination is benign; other autotuner errors are real.""" + log = _server_log( + tmp_path, + "[Autotuner] Single-pair run failed: CUDA out of memory. Tried to allocate...\n" + "[Autotuner] RuntimeError: invalid tactic configuration\n", + ) + hits = check_error(log) + assert [(idx, "RuntimeError" in line) for idx, line in hits] == [(2, True)] + # report_error's scanner applies the same boundary. + with pytest.raises(RuntimeError) as e: + report_error("wrapper", [log]) + assert "Error line 2" in str(e.value) + + +def test_report_error_always_appends_tail_even_on_keyword_hit(tmp_path): + """The first keyword hit may be noise; the fatal error can sit at EOF.""" + lines = ["ValueError: early benign-looking hit\n"] + lines += [f"filler {i}\n" for i in range(300)] + lines += ["the actual fatal last words\n"] + log = _server_log(tmp_path, "".join(lines)) + with pytest.raises(RuntimeError) as e: + report_error("wrapper", [log]) + msg = str(e.value) + assert "Error line 1" in msg # keyword context present + assert "the actual fatal last words" in msg # AND the tail diff --git a/tests/unittest/others/test_kv_cache_transceiver.py b/tests/unittest/others/test_kv_cache_transceiver.py index cd84403c6104..241a4aee0d09 100644 --- a/tests/unittest/others/test_kv_cache_transceiver.py +++ b/tests/unittest/others/test_kv_cache_transceiver.py @@ -1,5 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import gc import multiprocessing +import os import sys import time import uuid @@ -8,6 +12,11 @@ import pytest import torch +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see disaggregated/test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" + import tensorrt_llm import tensorrt_llm.bindings import tensorrt_llm.bindings.executor as trtllm @@ -33,6 +42,23 @@ KV_TRANSFER_COMPLETION_MARGIN_S = 10.0 +@pytest.mark.parametrize("transceiver_runtime", ["CPP", "auto"]) +def test_cpp_transceiver_rejects_mixed_mamba_manager(transceiver_runtime): + config = CacheTransceiverConfig(backend="NIXL", + transceiver_runtime=transceiver_runtime) + mixed_manager = object.__new__(MixedMambaHybridCacheManager) + + with pytest.raises( + ValueError, + match="MixedMambaHybridCacheManager requires the Python"): + create_kv_cache_transceiver(mapping=None, + dist=None, + kv_cache_manager=None, + attention_type=AttentionTypeCpp.DEFAULT, + cache_transceiver_config=config, + mamba_cache_manager=mixed_manager) + + def create_kv_cache_manager(mapping, dtype, max_tokens=256, @@ -716,7 +742,7 @@ def create_hybrid_cache_manager(mapping, dtype, mamba_conv_dtype=torch.float16, mamba_ssm_dtype=torch.float16): - """Create a MixedMambaHybridCacheManager for testing hybrid models. + """Create a mixed hybrid manager for Python transceiver tests. This manager handles both KV cache (attention layers) and Mamba cache (RNN layers). @@ -790,7 +816,7 @@ def hybrid_dtypes(request): Returns (kv_dtype, mamba_conv_dtype, mamba_ssm_dtype) based on the parametrized string. KV dtype: fp8, bf16 - Conv dtype: fp8, bf16, fp32 + Conv dtype: bf16, fp32 SSM dtype: bf16, fp32 """ kv_dtype_str, conv_dtype_str, ssm_dtype_str = request.param @@ -813,7 +839,6 @@ def hybrid_dtypes(request): @pytest.mark.timeout(120) -@pytest.mark.parametrize("backend", ["NIXL", "UCX"], ids=["NIXL", "UCX"]) @pytest.mark.parametrize( "hybrid_dtypes", [ @@ -839,18 +864,20 @@ def hybrid_dtypes(request): ], indirect=["hybrid_dtypes"], ) -def test_hybrid_cache_transceiver_single_process(backend, hybrid_dtypes): +def test_hybrid_cache_transceiver_single_process(hybrid_dtypes, request): mapping = Mapping(world_size=1, rank=0) kv_dtype, mamba_conv_dtype, mamba_ssm_dtype = hybrid_dtypes # Create hybrid cache managers (combines KV + Mamba) for context and generation hybrid_cache_manager_ctx = create_hybrid_cache_manager( mapping, kv_dtype, mamba_conv_dtype, mamba_ssm_dtype) + request.addfinalizer(hybrid_cache_manager_ctx.shutdown) hybrid_cache_manager_gen = create_hybrid_cache_manager( mapping, kv_dtype, mamba_conv_dtype, mamba_ssm_dtype) + request.addfinalizer(hybrid_cache_manager_gen.shutdown) - cache_transceiver_config = CacheTransceiverConfig(backend=backend, - max_tokens_in_buffer=512) + cache_transceiver_config = CacheTransceiverConfig( + backend="NIXL", transceiver_runtime="PYTHON", max_tokens_in_buffer=512) dist = Distributed.get(mapping) # Create transceivers - the hybrid manager serves as both kv_cache_manager and mamba_cache_manager @@ -861,6 +888,7 @@ def test_hybrid_cache_transceiver_single_process(backend, hybrid_dtypes): AttentionTypeCpp.DEFAULT, cache_transceiver_config, mamba_cache_manager=hybrid_cache_manager_ctx) + request.addfinalizer(cache_transceiver_ctx.shutdown) cache_transceiver_gen = create_kv_cache_transceiver( mapping, @@ -869,6 +897,7 @@ def test_hybrid_cache_transceiver_single_process(backend, hybrid_dtypes): AttentionTypeCpp.DEFAULT, cache_transceiver_config, mamba_cache_manager=hybrid_cache_manager_gen) + request.addfinalizer(cache_transceiver_gen.shutdown) # Fill both KV and Mamba cache buffers with random data fill_hybrid_cache_buffers(hybrid_cache_manager_ctx) @@ -883,6 +912,9 @@ def test_hybrid_cache_transceiver_single_process(backend, hybrid_dtypes): sampling_params._get_sampling_config()), is_streaming=False, llm_request_type=LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY) + ctx_request.py_disaggregated_params = tensorrt_llm.DisaggregatedParams( + request_type="context_only", + disagg_request_id=uuid.uuid4().int & 0x7FFFFFFFFFFFFFFF) # Prepare resources for hybrid manager (handles both KV and Mamba) scheduled_ctx = ScheduledRequests() @@ -902,6 +934,14 @@ def test_hybrid_cache_transceiver_single_process(backend, hybrid_dtypes): is_streaming=False, llm_request_type=LlmRequestType.LLMREQUEST_TYPE_GENERATION_ONLY, context_phase_params=ctx_request.context_phase_params) + gen_request.py_disaggregated_params = tensorrt_llm.DisaggregatedParams( + request_type="generation_only", + disagg_request_id=ctx_request.py_disaggregated_params.disagg_request_id, + ctx_request_id=ctx_request.request_id, + ctx_dp_rank=ctx_request.context_phase_params.ctx_dp_rank, + ctx_info_endpoint=ctx_request.context_phase_params.disagg_info_endpoint, + first_gen_tokens=ctx_request.context_phase_params.first_gen_tokens, + draft_tokens=ctx_request.context_phase_params.draft_tokens) # Prepare resources for hybrid manager on gen side scheduled_gen = ScheduledRequests() @@ -911,6 +951,7 @@ def test_hybrid_cache_transceiver_single_process(backend, hybrid_dtypes): cache_transceiver_gen.request_and_receive_async(gen_request) completed_ctx_ids = set() + expected_ctx_id = get_context_completed_request_id(ctx_request, "PYTHON") def poll_transfers(): completed, failed = cache_transceiver_ctx.check_context_transfer_status( @@ -920,8 +961,7 @@ def poll_transfers(): cache_transceiver_gen.check_gen_transfer_status(1) def transfers_done(): - return (ctx_request.py_request_id in completed_ctx_ids - and gen_request.state + return (expected_ctx_id in completed_ctx_ids and gen_request.state == LlmRequestState.DISAGG_GENERATION_TRANS_COMPLETE) wait_for_transfer_completion(poll_transfers, transfers_done) @@ -934,10 +974,10 @@ def transfers_done(): # independently-allocated slots on each side, so we check the # request's own slot instead of the full state buffer (which has # extra padding-dummy slots that only the ctx side touched). - slot_ctx = hybrid_cache_manager_ctx._impl.mamba_impl.get_cache_index( - ctx_request.py_request_id) - slot_gen = hybrid_cache_manager_gen._impl.mamba_impl.get_cache_index( - gen_request.py_request_id) + slot_ctx = hybrid_cache_manager_ctx.mamba_cache_index[ + ctx_request.py_request_id] + slot_gen = hybrid_cache_manager_gen.mamba_cache_index[ + gen_request.py_request_id] assert torch.equal( hybrid_cache_manager_gen.get_conv_states(1)[slot_gen], hybrid_cache_manager_ctx.get_conv_states(1)[slot_ctx]), ( @@ -947,20 +987,22 @@ def transfers_done(): hybrid_cache_manager_gen.get_ssm_states(1)[slot_gen], hybrid_cache_manager_ctx.get_ssm_states(1)[slot_ctx]), ( "different mamba ssm states") + shutdown_transceivers(cache_transceiver_gen, cache_transceiver_ctx) @pytest.mark.timeout(120) -@pytest.mark.parametrize("backend", ["NIXL", "UCX"], ids=["NIXL", "UCX"]) -def test_hybrid_cache_transceiver_cancel_request(backend): +def test_hybrid_cache_transceiver_cancel_request(request): mapping = Mapping(world_size=1, rank=0) dtype = DataType.HALF hybrid_cache_manager_ctx = create_hybrid_cache_manager(mapping, dtype) + request.addfinalizer(hybrid_cache_manager_ctx.shutdown) hybrid_cache_manager_gen = create_hybrid_cache_manager(mapping, dtype) + request.addfinalizer(hybrid_cache_manager_gen.shutdown) - cache_transceiver_config = CacheTransceiverConfig(backend=backend, - max_tokens_in_buffer=512) + cache_transceiver_config = CacheTransceiverConfig( + backend="NIXL", transceiver_runtime="PYTHON", max_tokens_in_buffer=512) dist = Distributed.get(mapping) cache_transceiver_ctx = create_kv_cache_transceiver( @@ -970,6 +1012,7 @@ def test_hybrid_cache_transceiver_cancel_request(backend): AttentionTypeCpp.DEFAULT, cache_transceiver_config, mamba_cache_manager=hybrid_cache_manager_ctx) + request.addfinalizer(cache_transceiver_ctx.shutdown) cache_transceiver_gen = create_kv_cache_transceiver( mapping, @@ -978,6 +1021,7 @@ def test_hybrid_cache_transceiver_cancel_request(backend): AttentionTypeCpp.DEFAULT, cache_transceiver_config, mamba_cache_manager=hybrid_cache_manager_gen) + request.addfinalizer(cache_transceiver_gen.shutdown) fill_hybrid_cache_buffers(hybrid_cache_manager_ctx) @@ -991,6 +1035,9 @@ def test_hybrid_cache_transceiver_cancel_request(backend): sampling_params._get_sampling_config()), is_streaming=False, llm_request_type=LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY) + ctx_request.py_disaggregated_params = tensorrt_llm.DisaggregatedParams( + request_type="context_only", + disagg_request_id=uuid.uuid4().int & 0x7FFFFFFFFFFFFFFF) scheduled_ctx = ScheduledRequests() scheduled_ctx.context_requests_last_chunk = [ctx_request] @@ -999,13 +1046,6 @@ def test_hybrid_cache_transceiver_cancel_request(backend): # Send ctx request cache_transceiver_ctx.respond_and_send_async(ctx_request) - # Wait for ctx request to be sent - time.sleep(2) - - # Cancel ctx request - is_cancelled = cache_transceiver_ctx.cancel_request(ctx_request) - assert is_cancelled - # Init gen request gen_request = LlmRequest( request_id=0, @@ -1016,6 +1056,14 @@ def test_hybrid_cache_transceiver_cancel_request(backend): is_streaming=False, llm_request_type=LlmRequestType.LLMREQUEST_TYPE_GENERATION_ONLY, context_phase_params=ctx_request.context_phase_params) + gen_request.py_disaggregated_params = tensorrt_llm.DisaggregatedParams( + request_type="generation_only", + disagg_request_id=ctx_request.py_disaggregated_params.disagg_request_id, + ctx_request_id=ctx_request.request_id, + ctx_dp_rank=ctx_request.context_phase_params.ctx_dp_rank, + ctx_info_endpoint=ctx_request.context_phase_params.disagg_info_endpoint, + first_gen_tokens=ctx_request.context_phase_params.first_gen_tokens, + draft_tokens=ctx_request.context_phase_params.draft_tokens) scheduled_gen = ScheduledRequests() scheduled_gen.context_requests_last_chunk = [gen_request] @@ -1024,7 +1072,21 @@ def test_hybrid_cache_transceiver_cancel_request(backend): # Try to receive gen request cache_transceiver_gen.request_and_receive_async(gen_request) - # Block the main thread due to the async operation - time.sleep(2) - cache_transceiver_gen.check_gen_transfer_status(0) - assert gen_request.state == LlmRequestState.DISAGG_TRANS_ERROR + generation_cancelled = [False] + + def cancel_generation_transfer(): + generation_cancelled[0] = cache_transceiver_gen.cancel_request( + gen_request) + + wait_for_transfer_completion(cancel_generation_transfer, + lambda: generation_cancelled[0]) + assert cache_transceiver_gen.check_gen_transfer_complete() + + context_cancelled = [False] + + def cancel_context_transfer(): + context_cancelled[0] = cache_transceiver_ctx.cancel_request(ctx_request) + + wait_for_transfer_completion(cancel_context_transfer, + lambda: context_cancelled[0]) + shutdown_transceivers(cache_transceiver_gen, cache_transceiver_ctx) diff --git a/tests/unittest/others/test_leak.py b/tests/unittest/others/test_leak.py deleted file mode 100644 index 819744fb74b9..000000000000 --- a/tests/unittest/others/test_leak.py +++ /dev/null @@ -1,88 +0,0 @@ -import unittest - -import tensorrt_llm as tllm -import tensorrt_llm.profiler as profiler - -import psutil # isort:skip - - -def create_model(): - ''' Lots of parameters are created here, and thus memory increases - ''' - profiler.print_memory_usage('Before creating Module') - - config = { - 'architecture': "LlamaForCausalLM", - 'dtype': 'float32', - 'num_hidden_layers': 2, - 'num_attention_heads': 80, - 'hidden_size': 12800, - 'num_key_value_heads': 80, - 'vocab_size': 50000, - 'position_embedding_type': 'rope_gpt_neox', - 'max_position_embeddings': 2048, - 'hidden_act': 'silu' - } - config = tllm.models.PretrainedConfig.from_dict(config) - - # About 24GiB model size, big enough to detect leak and avoid noise and false positive - # and small enough to make sure CI single-gpu machine can run it. - model = tllm.models.LLaMAForCausalLM.from_config(config) - profiler.print_memory_usage('After creating Module') - return model - - -def create_optimize_network(): - builder = tllm.Builder() - model = create_model() - network = builder.create_network() - network.plugin_config.gpt_attention_plugin = 'float16' - network.plugin_config.paged_kv_cache = True - network.plugin_config.use_fp8_context_fmha = False - profiler.print_memory_usage('Before creating Network') - with tllm.net_guard(network): - # Forward - inputs = model.prepare_inputs(max_batch_size=1, - max_input_len=1024, - max_seq_len=1024 + 32, - max_num_tokens=1024, - use_cache=True, - max_beam_width=1) - model(**inputs) - profiler.print_memory_usage('After creating Network') - - # When the Network has gpt attention plugin layer, graph rewriting pattern matching is triggered, - # thus the Network._get_graph_impl will be called, and a lru_cache will be created to cache this Network object - # and thus these registered ndarrays inside the Network, these objects are destroyed only when the cache is full or the - # program ends - tllm.graph_rewriting.optimize(network) - - -def run(): - # Create a TRT builder to warm up the memory, and avoid the noise of leak detection. - # Builder creation will create global objects like kernels. - _ = tllm.Builder() - - used, _, _ = profiler.host_memory_info() - - for i in range(5): - # Ideally the memory used inside create_optimize_network will all be released after the function returns - profiler.print_memory_usage(f'create_optimize_network {i} started') - create_optimize_network() - profiler.print_memory_usage(f'create_optimize_network {i} returned') - - used_after, _, _ = profiler.host_memory_info() - mem_increase_in_gb = (used_after - used) / (1024**3) - # The model has more than 10GB, so if there is leak, it will be absolutely bigger than 1GB - assert mem_increase_in_gb < 1, f"Memory increased {mem_increase_in_gb} GB" - - -class TestHostMemLeak(unittest.TestCase): - - def test_host_mem_leak(self): - tllm.logger.set_level('info') - run() - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unittest/others/test_lm_eval.py b/tests/unittest/others/test_lm_eval.py index a3a0be55dac5..a7d20a97a8f1 100644 --- a/tests/unittest/others/test_lm_eval.py +++ b/tests/unittest/others/test_lm_eval.py @@ -21,17 +21,32 @@ * ``tensorrt_llm.evaluate.lm_eval_tasks.mmmu_pro.utils`` — ``parse_multi_choice_response`` reverse-scan and the ``MMMU_PRO_PROMPT_MODE`` env switch. +* ``LmEvalWrapper._log_spec_stats`` — the ``TLLM_EVAL_SPEC_STATS``-gated + speculative-decoding acceptance-length (AL) corpus summary, + iteration-weighted to match ``bench/dataclasses/reporting.py``. +* ``LmEvalWrapper._generate_until_windowed`` — the + ``TLLM_EVAL_MAX_IN_FLIGHT`` submission window: in-flight cap, + submission-order results under out-of-order completion, and fail-fast + propagation of request errors. +* End-to-end: lm-eval's real ``evaluate()`` loop (real ``ConfigurableTask``, + filters, aggregation) driven through ``LmEvalWrapper`` over a mocked LLM, + for both the final score and the partial-score running estimates. """ from __future__ import annotations import importlib import os +import threading +import time from unittest.mock import MagicMock, patch +import pytest + from tensorrt_llm.evaluate.covost2 import CoVoST2 from tensorrt_llm.evaluate.lm_eval import ( LM_EVAL_DEFAULT_IMAGE_PLACEHOLDER, + MAX_IN_FLIGHT_ENV_VAR, LmEvalWrapper, MultimodalLmEvalWrapper, ) @@ -43,6 +58,7 @@ strip_string, ) from tensorrt_llm.inputs.content_format import ContentFormat +from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY from tensorrt_llm.sampling_params import SamplingParams # =========================================================================== @@ -286,14 +302,30 @@ def test_sampling_override_no_cli_falls_back_to_yaml(): # correctly-ordered OpenAI content list. -# Uses ``gemma3`` by default because it is always registered regardless of -# transformers version; the wrapper's interleave logic itself is generic. -def _make_multimodal_wrapper(model_type: str = "gemma3") -> MultimodalLmEvalWrapper: +# Interleaving is opt-in per model: the wrapper reads +# ``MULTIMODAL_PLACEHOLDER_REGISTRY.get_interleave_placeholders(model_type)`` +# at construction, and models that don't opt in keep the historical +# strip-and-bulk-insert behaviour. These tests drive that flag directly +# instead of naming an opted-in model, because which models are registered +# varies with the installed transformers version — keying on a real model +# name would make the tests environment-dependent. ``interleave=False`` +# (the default here) matches an unregistered model such as ``gemma3``. +def _make_multimodal_wrapper( + model_type: str = "gemma3", + interleave: bool = False, +) -> MultimodalLmEvalWrapper: fake_llm = MagicMock() fake_llm.tokenizer = MagicMock() fake_llm.input_processor = MagicMock() fake_llm.input_processor.processor = MagicMock() - with patch.object(MultimodalLmEvalWrapper, "_get_model_type", return_value=model_type): + with ( + patch.object(MultimodalLmEvalWrapper, "_get_model_type", return_value=model_type), + patch.object( + MULTIMODAL_PLACEHOLDER_REGISTRY, + "get_interleave_placeholders", + return_value=interleave, + ), + ): return MultimodalLmEvalWrapper( fake_llm, sampling_params=None, @@ -335,8 +367,8 @@ def _fake_trtllm_apply(**kwargs): return convs[0] -def test_single_image_does_not_interleave(): - """Single-image prompts never need interleaving. +def test_not_opted_in_model_does_not_interleave(): + """A model that does not opt in keeps the historical bulk-insert path. content_parts stays absent so the existing BEFORE_TEXT default keeps working. """ @@ -352,7 +384,7 @@ def test_multi_image_openai_builds_content_parts(): ``_build_openai_content`` then emits media entries at the correct positions. """ - wrapper = _make_multimodal_wrapper() + wrapper = _make_multimodal_wrapper(interleave=True) ph = LM_EVAL_DEFAULT_IMAGE_PLACEHOLDER text = f"Consider {ph}. What does {ph} show?" conv = _call_apply(wrapper, text, content_format=ContentFormat.OPENAI) @@ -368,8 +400,8 @@ def test_multi_image_openai_builds_content_parts(): assert [p["media_index"] for p in media_parts] == [0, 1] -def test_multi_image_string_format_skips_interleave(): - """STRING-format chat templates skip the interleaving path. +def test_multi_image_string_format_not_opted_in_uses_placeholders(): + """STRING-format templates on a non-opted-in model use flat placeholders. Placeholders are inserted into the flat text via ``add_multimodal_placeholders`` instead, so ``content_parts`` stays absent. @@ -391,7 +423,7 @@ def test_trailing_text_after_last_image_preserved(): Otherwise the question suffix ('Answer:') is dropped before it reaches the model. """ - wrapper = _make_multimodal_wrapper() + wrapper = _make_multimodal_wrapper(interleave=True) ph = LM_EVAL_DEFAULT_IMAGE_PLACEHOLDER text = f"Compare {ph} with {ph}. Answer with a letter." conv = _call_apply(wrapper, text, content_format=ContentFormat.OPENAI) @@ -406,7 +438,7 @@ def test_leading_image_no_empty_text_segment(): content_parts must begin with the image entry itself. """ - wrapper = _make_multimodal_wrapper() + wrapper = _make_multimodal_wrapper(interleave=True) ph = LM_EVAL_DEFAULT_IMAGE_PLACEHOLDER text = f"{ph} {ph} Answer?" conv = _call_apply(wrapper, text, content_format=ContentFormat.OPENAI) @@ -745,7 +777,6 @@ def apply(self, instances): class _FakeTask: - def __init__(self): self._filters = [_FakeEnsemble("strict-match")] @@ -756,7 +787,6 @@ def process_results(self, doc, results): class _FakeInstance: - def __init__(self, task_name, doc): self.task_name = task_name self.doc = doc @@ -766,6 +796,7 @@ def __init__(self, task_name, doc): def _make_tracker(interval=2): from tensorrt_llm.evaluate.lm_eval import _RunningScoreTracker + return _RunningScoreTracker({"fake_task": _FakeTask()}, interval) @@ -777,7 +808,7 @@ def test_running_score_tracker_aggregates_mean(): for doc, text in zip(docs, responses): tracker.update(_FakeInstance("fake_task", doc), text) assert not tracker.disabled - key = "exact_match,strict-match" + key = "fake_task,exact_match,strict-match" assert tracker.metric_counts[key] == 3 assert tracker.metric_sums[key] == 2.0 @@ -802,7 +833,7 @@ def test_running_score_tracker_unknown_task_disables(): tracker.maybe_log(10, 100) -def test_running_score_tracker_logs_on_interval(caplog): +def test_running_score_tracker_logs_on_interval(): """maybe_log emits at every `interval` responses and at completion.""" tracker = _make_tracker(interval=2) with patch("tensorrt_llm.evaluate.lm_eval.logger") as mock_logger: @@ -818,7 +849,7 @@ def test_running_score_tracker_logs_on_interval(caplog): message = mock_logger.info.call_args[0][0] assert "2/3" not in message # latest call reports 3/3 assert "3/3" in message - assert "exact_match,strict-match" in message + assert "fake_task,exact_match,strict-match" in message # 2 of 3 correct -> ~66.67 on the 0~100 scale. assert "66.67" in message @@ -839,7 +870,7 @@ def test_running_score_tracker_process_results_list_convention(): doc = {"answer": "42"} tracker.update(_FakeInstance("fake_task", doc), "42") assert not tracker.disabled - key = "exact_match,strict-match" + key = "fake_task,exact_match,strict-match" assert tracker.metric_sums[key] == 1.0, ( "multi-digit answer scored as miss — process_results likely received " "a bare string so results[0] returned only the first character" @@ -849,7 +880,665 @@ def test_running_score_tracker_process_results_list_convention(): def test_running_score_tracker_task_groups_flattened(): """Nested task_dict groups resolve to their leaf tasks.""" from tensorrt_llm.evaluate.lm_eval import _RunningScoreTracker + tracker = _RunningScoreTracker({"group": {"fake_task": _FakeTask()}}, 1) tracker.update(_FakeInstance("fake_task", {"answer": "42"}), "42") assert not tracker.disabled - assert tracker.metric_counts["exact_match,strict-match"] == 1 + assert tracker.metric_counts["fake_task,exact_match,strict-match"] == 1 + + +def test_running_score_tracker_separate_keys_per_task(): + """Two tasks with the same metric/filter don't mix their running estimates.""" + from tensorrt_llm.evaluate.lm_eval import _RunningScoreTracker + + task_a = _FakeTask() + task_b = _FakeTask() + tracker = _RunningScoreTracker({"task_a": task_a, "task_b": task_b}, 999) + tracker.update(_FakeInstance("task_a", {"answer": "x"}), "x") # correct + tracker.update(_FakeInstance("task_b", {"answer": "x"}), "y") # wrong + assert not tracker.disabled + assert tracker.metric_sums["task_a,exact_match,strict-match"] == 1.0 + assert tracker.metric_sums["task_b,exact_match,strict-match"] == 0.0 + + +# =========================================================================== +# _parse_partial_scores_env — env-var parsing +# =========================================================================== + + +def test_parse_partial_scores_env_positive(monkeypatch): + """A positive integer returns that interval.""" + from tensorrt_llm.evaluate.lm_eval import PARTIAL_SCORES_ENV_VAR, _parse_partial_scores_env + + monkeypatch.setenv(PARTIAL_SCORES_ENV_VAR, "100") + assert _parse_partial_scores_env() == 100 + + +def test_parse_partial_scores_env_zero_disables(monkeypatch): + """Zero disables partial scoring (returns None).""" + from tensorrt_llm.evaluate.lm_eval import PARTIAL_SCORES_ENV_VAR, _parse_partial_scores_env + + monkeypatch.setenv(PARTIAL_SCORES_ENV_VAR, "0") + assert _parse_partial_scores_env() is None + + +def test_parse_partial_scores_env_negative_disables(monkeypatch): + """Negative values disable partial scoring (returns None).""" + from tensorrt_llm.evaluate.lm_eval import PARTIAL_SCORES_ENV_VAR, _parse_partial_scores_env + + monkeypatch.setenv(PARTIAL_SCORES_ENV_VAR, "-5") + assert _parse_partial_scores_env() is None + + +def test_parse_partial_scores_env_invalid_raises(monkeypatch): + """A non-integer value raises ValueError.""" + from tensorrt_llm.evaluate.lm_eval import PARTIAL_SCORES_ENV_VAR, _parse_partial_scores_env + + monkeypatch.setenv(PARTIAL_SCORES_ENV_VAR, "abc") + with pytest.raises(ValueError, match=PARTIAL_SCORES_ENV_VAR): + _parse_partial_scores_env() + + +def test_parse_partial_scores_env_unset_returns_none(monkeypatch): + """Unset env var returns None.""" + from tensorrt_llm.evaluate.lm_eval import PARTIAL_SCORES_ENV_VAR, _parse_partial_scores_env + + monkeypatch.delenv(PARTIAL_SCORES_ENV_VAR, raising=False) + assert _parse_partial_scores_env() is None + + +# =========================================================================== +# LmEvalWrapper.generate_until — partial scorer invocation +# =========================================================================== + + +def test_generate_until_invokes_partial_scorer(): + """generate_until calls scorer.update and scorer.maybe_log for each response.""" + from tensorrt_llm.evaluate.lm_eval import LmEvalWrapper, _RunningScoreTracker + + fake_output = MagicMock() + fake_output.result.return_value.outputs = [MagicMock(text="42")] + fake_llm = MagicMock() + fake_llm.generate_async.return_value = fake_output + + wrapper = LmEvalWrapper( + llm=fake_llm, + partial_scores_every=1, + partial_scoring_task_dict={"fake_task": _FakeTask()}, + ) + + fake_request = MagicMock() + fake_request.args = ("hello world", {}) + fake_request.task_name = "fake_task" + fake_request.doc = {"answer": "42"} + + with ( + patch.object(_RunningScoreTracker, "update") as mock_update, + patch.object(_RunningScoreTracker, "maybe_log") as mock_log, + ): + wrapper.generate_until([fake_request], disable_tqdm=True) + + mock_update.assert_called_once_with(fake_request, "42") + mock_log.assert_called_once_with(1, 1) + + +# =========================================================================== +# TLLM_EVAL_SPEC_STATS — speculative-decoding AL stats +# =========================================================================== +# +# Only AL (acceptance length) is reported for now; AR needs the +# request_perf_metrics.speculative_decoding counters, which the TorchSampler +# used by one-engine spec-dec does not populate (see _log_spec_stats). +# AL is iteration-weighted (total decoded tokens / total decode iterations) +# to agree with the repo's canonical definition in +# ``bench/dataclasses/reporting.py``. + + +def _make_spec_output( + tokens_per_iter: float | None = None, + decoding_iter: int | None = 1, +) -> MagicMock: + """Fake RequestOutput with an optional per-request AL sample. + + ``tokens_per_iter`` as None models a request without speculative + metrics (non-spec-dec run, or a response that never reported them). + ``decoding_iter`` is the AL aggregation weight (decode iterations the + request ran); None models a result that never populated it. + """ + output = MagicMock() + output.avg_decoded_tokens_per_iter = tokens_per_iter + output.decoding_iter = decoding_iter + output.outputs = [MagicMock()] + return output + + +def test_spec_stats_env_unset_disables(monkeypatch): + """Unset env leaves the feature off.""" + from tensorrt_llm.evaluate.lm_eval import SPEC_STATS_ENV_VAR + + monkeypatch.delenv(SPEC_STATS_ENV_VAR, raising=False) + wrapper = _make_lm_eval_wrapper() + assert wrapper.spec_stats is False + + +def test_spec_stats_env_enabled(monkeypatch): + """TLLM_EVAL_SPEC_STATS=1 turns the feature on.""" + from tensorrt_llm.evaluate.lm_eval import SPEC_STATS_ENV_VAR + + monkeypatch.setenv(SPEC_STATS_ENV_VAR, "1") + wrapper = _make_lm_eval_wrapper() + assert wrapper.spec_stats is True + + +@pytest.mark.parametrize("value", ["0", "true", "yes", ""]) +def test_spec_stats_env_non_one_values_disable(monkeypatch, value): + """Only the literal "1" enables the feature.""" + from tensorrt_llm.evaluate.lm_eval import SPEC_STATS_ENV_VAR + + monkeypatch.setenv(SPEC_STATS_ENV_VAR, value) + wrapper = _make_lm_eval_wrapper() + assert wrapper.spec_stats is False + + +def test_log_spec_stats_reports_al_mean_min_max(): + """AL over equal-weight requests equals the plain mean; min/max/n present.""" + wrapper = _make_lm_eval_wrapper() + outputs = [ + _make_spec_output(tokens_per_iter=2.0, decoding_iter=5), + _make_spec_output(tokens_per_iter=4.0, decoding_iter=5), + ] + with patch("tensorrt_llm.evaluate.lm_eval.logger") as mock_logger: + wrapper._log_spec_stats(outputs) + assert mock_logger.info.call_count == 1 + al_message = mock_logger.info.call_args[0][0] + assert "AL" in al_message + assert "3.000" in al_message # equal weights -> mean of 2.0 and 4.0 + assert "min 2.000" in al_message + assert "max 4.000" in al_message + assert "n=2" in al_message + + +def test_log_spec_stats_weights_by_decode_iterations(): + """AL matches reporting.py's token-level mean: weighted by decode iterations. + + (2.0 tok/iter over 1 iter) + (4.0 tok/iter over 3 iters) = 14 decoded + tokens over 4 iterations = 3.5 — NOT the unweighted mean 3.0, which + would bias the result toward short requests (see the explicit rationale + in ``bench/dataclasses/reporting.py``). + """ + wrapper = _make_lm_eval_wrapper() + outputs = [ + _make_spec_output(tokens_per_iter=2.0, decoding_iter=1), + _make_spec_output(tokens_per_iter=4.0, decoding_iter=3), + ] + with patch("tensorrt_llm.evaluate.lm_eval.logger") as mock_logger: + wrapper._log_spec_stats(outputs) + message = mock_logger.info.call_args[0][0] + assert "3.500" in message + # min/max stay per-request values, unweighted. + assert "min 2.000" in message + assert "max 4.000" in message + + +def test_log_spec_stats_missing_decoding_iter_falls_back_to_weight_one(): + """Requests without a usable decoding_iter contribute with weight 1.""" + wrapper = _make_lm_eval_wrapper() + outputs = [ + _make_spec_output(tokens_per_iter=2.0, decoding_iter=None), + _make_spec_output(tokens_per_iter=4.0, decoding_iter=0), + ] + with patch("tensorrt_llm.evaluate.lm_eval.logger") as mock_logger: + wrapper._log_spec_stats(outputs) + message = mock_logger.info.call_args[0][0] + assert "3.000" in message # both fall back to weight 1 -> plain mean + + +def test_log_spec_stats_skips_requests_without_metrics(): + """Requests lacking spec metrics are excluded, not counted as zero.""" + wrapper = _make_lm_eval_wrapper() + outputs = [ + _make_spec_output(tokens_per_iter=3.0), + _make_spec_output(), # no metrics (e.g. dropped by the engine) + ] + with patch("tensorrt_llm.evaluate.lm_eval.logger") as mock_logger: + wrapper._log_spec_stats(outputs) + assert mock_logger.info.call_count == 1 + message = mock_logger.info.call_args[0][0] + assert "3.000" in message + assert "n=1" in message + + +def test_log_spec_stats_silent_on_non_spec_run(): + """A run with no speculative metrics at all logs nothing.""" + wrapper = _make_lm_eval_wrapper() + outputs = [_make_spec_output(), _make_spec_output()] + with patch("tensorrt_llm.evaluate.lm_eval.logger") as mock_logger: + wrapper._log_spec_stats(outputs) + mock_logger.info.assert_not_called() + + +def test_generate_until_logs_spec_stats_when_enabled(monkeypatch): + """generate_until forwards the collected outputs to _log_spec_stats.""" + from tensorrt_llm.evaluate.lm_eval import SPEC_STATS_ENV_VAR, LmEvalWrapper + + monkeypatch.setenv(SPEC_STATS_ENV_VAR, "1") + fake_result = MagicMock() + fake_result.outputs = [MagicMock(text="42")] + fake_output = MagicMock() + fake_output.result.return_value = fake_result + fake_llm = MagicMock() + fake_llm.generate_async.return_value = fake_output + + wrapper = LmEvalWrapper(llm=fake_llm) + fake_request = MagicMock() + fake_request.args = ("hello world", {}) + + with patch.object(LmEvalWrapper, "_log_spec_stats") as mock_stats: + wrapper.generate_until([fake_request], disable_tqdm=True) + + mock_stats.assert_called_once_with([fake_result]) + + +def test_generate_until_skips_spec_stats_when_disabled(monkeypatch): + """Without the env var, generate_until never touches _log_spec_stats.""" + from tensorrt_llm.evaluate.lm_eval import SPEC_STATS_ENV_VAR, LmEvalWrapper + + monkeypatch.delenv(SPEC_STATS_ENV_VAR, raising=False) + fake_output = MagicMock() + fake_output.result.return_value.outputs = [MagicMock(text="42")] + fake_llm = MagicMock() + fake_llm.generate_async.return_value = fake_output + + wrapper = LmEvalWrapper(llm=fake_llm) + fake_request = MagicMock() + fake_request.args = ("hello world", {}) + + with patch.object(LmEvalWrapper, "_log_spec_stats") as mock_stats: + wrapper.generate_until([fake_request], disable_tqdm=True) + + mock_stats.assert_not_called() + + +# =========================================================================== +# TLLM_EVAL_MAX_IN_FLIGHT — windowed generate_until +# =========================================================================== +# +# The windowed path caps concurrently in-flight requests at W, tops the +# window up as responses complete, and collects outputs into an +# index-addressed list. The correctness property the whole design exists to +# preserve is SUBMISSION-ORDER RESULTS under arbitrary completion order; +# the liveness property is that a failed request propagates promptly +# instead of deadlocking behind other in-flight waiters. + + +class _FakeAsyncOutput: + """Async handle whose blocking .result() is supplied by the test.""" + + def __init__(self, result_fn): + self._result_fn = result_fn + + def result(self): + return self._result_fn() + + +def _text_result(text: str) -> MagicMock: + result = MagicMock() + result.outputs = [MagicMock(text=text)] + return result + + +def _make_windowed_llm(events, error_idx=None): + """LLM whose request i blocks until events[i] is set, then yields resp-i. + + Returns the fake llm and the (mutated) list of submitted request indices, + so tests can observe how far submission has progressed. + """ + submitted = [] + llm = MagicMock() + llm.tokenizer = MagicMock() + + def generate_async(prompt, sampling_params=None, streaming=False): + idx = len(submitted) + submitted.append(idx) + + def _result(): + assert events[idx].wait(timeout=30), f"request {idx} never released" + if error_idx is not None and idx == error_idx: + raise RuntimeError(f"request {idx} failed") + return _text_result(f"resp-{idx}") + + return _FakeAsyncOutput(_result) + + llm.generate_async = generate_async + return llm, submitted + + +def _make_requests(n: int) -> list: + requests = [] + for i in range(n): + request = MagicMock() + request.args = (f"prompt-{i}", {}) + requests.append(request) + return requests + + +def _wait_until(predicate, timeout: float = 10.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + +def test_windowed_caps_in_flight_and_preserves_order(monkeypatch): + """At most W requests in flight, and results follow submission order. + + Request 1 is completed before request 0 (out-of-order completion); the + window tops up with request 2 only after that completion, and the + returned list is still resp-0..resp-4 in submission order. + """ + monkeypatch.setenv(MAX_IN_FLIGHT_ENV_VAR, "2") + total = 5 + events = [threading.Event() for _ in range(total)] + llm, submitted = _make_windowed_llm(events) + wrapper = LmEvalWrapper(llm=llm) + assert wrapper.max_in_flight == 2 + + returned = [] + worker = threading.Thread( + target=lambda: returned.append( + wrapper.generate_until(_make_requests(total), disable_tqdm=True) + ) + ) + worker.start() + try: + # Only the first W requests are submitted while none have completed. + assert _wait_until(lambda: len(submitted) == 2) + time.sleep(0.05) + assert len(submitted) == 2, "window overshot max_in_flight" + # Completing request 1 (out of order) tops the window up by one. + events[1].set() + assert _wait_until(lambda: len(submitted) == 3) + time.sleep(0.05) + assert len(submitted) == 3 + finally: + for event in events: + event.set() + worker.join(timeout=30) + assert not worker.is_alive() + assert returned and returned[0] == [f"resp-{i}" for i in range(total)] + + +def test_windowed_failed_request_raises_without_waiting(monkeypatch): + """A failed request propagates while another request is still in flight. + + Regression guard for the deadlock the review called out: a blocking + pool shutdown would join every other outstanding waiter with no + cancellation or timeout, so if any of them never resolved the exception + could not escape and the eval hung instead of failing. + """ + monkeypatch.setenv(MAX_IN_FLIGHT_ENV_VAR, "2") + events = [threading.Event() for _ in range(2)] + llm, _ = _make_windowed_llm(events, error_idx=0) + wrapper = LmEvalWrapper(llm=llm) + events[0].set() # request 0 fails immediately; request 1 stays blocked + try: + with pytest.raises(RuntimeError, match="request 0 failed"): + wrapper.generate_until(_make_requests(2), disable_tqdm=True) + # The exception escaped while request 1 had not resolved. + assert not events[1].is_set() + finally: + events[1].set() # release the lingering waiter thread + + +def test_windowed_window_larger_than_request_count(monkeypatch): + """W >= len(requests) submits each request exactly once and stays ordered.""" + monkeypatch.setenv(MAX_IN_FLIGHT_ENV_VAR, "64") + total = 3 + events = [threading.Event() for _ in range(total)] + for event in events: + event.set() + llm, submitted = _make_windowed_llm(events) + wrapper = LmEvalWrapper(llm=llm) + result = wrapper.generate_until(_make_requests(total), disable_tqdm=True) + assert result == [f"resp-{i}" for i in range(total)] + assert submitted == list(range(total)) + + +def test_windowed_empty_request_list(monkeypatch): + """Zero requests short-circuit without creating a thread pool.""" + monkeypatch.setenv(MAX_IN_FLIGHT_ENV_VAR, "2") + llm, _ = _make_windowed_llm([]) + wrapper = LmEvalWrapper(llm=llm) + assert wrapper.generate_until([], disable_tqdm=True) == [] + + +def test_windowed_invokes_partial_scorer(monkeypatch): + """The windowed path feeds every completion to the partial scorer.""" + from tensorrt_llm.evaluate.lm_eval import _RunningScoreTracker + + monkeypatch.setenv(MAX_IN_FLIGHT_ENV_VAR, "2") + total = 3 + events = [threading.Event() for _ in range(total)] + for event in events: + event.set() + llm, _ = _make_windowed_llm(events) + wrapper = LmEvalWrapper( + llm=llm, + partial_scores_every=1, + partial_scoring_task_dict={"fake_task": _FakeTask()}, + ) + with ( + patch.object(_RunningScoreTracker, "update") as mock_update, + patch.object(_RunningScoreTracker, "maybe_log") as mock_log, + ): + wrapper.generate_until(_make_requests(total), disable_tqdm=True) + assert mock_update.call_count == total + assert mock_log.call_count == total + + +# =========================================================================== +# MultimodalLmEvalWrapper.generate_until — partial scorer wiring +# =========================================================================== + + +def test_multimodal_generate_until_invokes_partial_scorer(): + """The multimodal override scores the post-processed text lm-eval sees.""" + from tensorrt_llm.evaluate.lm_eval import _RunningScoreTracker + + fake_output = MagicMock() + fake_output.result.return_value.outputs = [MagicMock(text="reasoning42")] + fake_llm = MagicMock() + fake_llm.tokenizer = MagicMock() + fake_llm.input_processor = MagicMock() + fake_llm.generate_async.return_value = fake_output + + with patch.object(MultimodalLmEvalWrapper, "_get_model_type", return_value="gemma3"): + wrapper = MultimodalLmEvalWrapper( + fake_llm, + sampling_params=None, + model_type="gemma3", + post_process_fn=lambda s: s.split("")[-1], + partial_scores_every=1, + partial_scoring_task_dict={"fake_task": _FakeTask()}, + ) + + fake_request = MagicMock() + fake_request.args = ("prompt", {}, {"visual": [MagicMock()]}) + + with ( + patch( + "tensorrt_llm.evaluate.lm_eval.prompt_inputs", + side_effect=lambda p: {"prompt": p}, + ), + patch( + "tensorrt_llm.evaluate.lm_eval.convert_image_mode", + side_effect=lambda img, mode: img, + ), + patch.object(_RunningScoreTracker, "update") as mock_update, + patch.object(_RunningScoreTracker, "maybe_log") as mock_log, + ): + results = wrapper.generate_until([fake_request], disable_tqdm=True) + + assert results == ["42"] + # The scorer must see the post-processed text, not the raw output. + mock_update.assert_called_once_with(fake_request, "42") + mock_log.assert_called_once_with(1, 1) + + +# =========================================================================== +# End-to-end: real lm-eval evaluator over a mocked LLM +# =========================================================================== +# +# Runs lm_eval.evaluator.evaluate() — real ConfigurableTask, real filter +# pipeline, real metric aggregation — against LmEvalWrapper wrapping a +# mocked LLM that returns canned responses. This exercises the exact +# calling conventions between the harness and the wrapper (instance +# shapes, filtered_resps, the process_results list convention) that pure +# unit mocks can get subtly wrong; the GSM8K first-character bug above +# survived precisely because nothing ran the real harness loop. + + +_E2E_DOCS = [ + {"question": "2+2?", "answer": "4"}, + {"question": "3+4?", "answer": "7"}, + {"question": "5+6?", "answer": "11"}, + {"question": "10-3?", "answer": "7"}, +] +# Model answers: 3 correct, 1 wrong ("12" != "11") -> exact_match 0.75. +_E2E_RESPONSES = [ + "The answer is 4.", + "The answer is 7.", + "The answer is 12.", + "The answer is 7.", +] + + +def _toy_task(): + """A real generate_until ConfigurableTask over an in-memory dataset.""" + import datasets + from lm_eval.api.task import ConfigurableTask + + return ConfigurableTask( + config={ + "task": "toy_arith", + "custom_dataset": lambda **kwargs: datasets.DatasetDict( + {"test": datasets.Dataset.from_list(_E2E_DOCS)} + ), + "test_split": "test", + "output_type": "generate_until", + "doc_to_text": "Q: {{question}}\nA:", + "doc_to_target": "{{answer}}", + "generation_kwargs": {"until": ["\n"], "do_sample": False}, + "filter_list": [ + { + "name": "strict-match", + "filter": [ + {"function": "regex", "regex_pattern": r"(-?[0-9]+)"}, + {"function": "take_first"}, + ], + } + ], + "metric_list": [ + { + "metric": "exact_match", + "aggregation": "mean", + "higher_is_better": True, + } + ], + } + ) + + +def _canned_llm(responses): + """LLM whose generate_async yields the canned texts in submission order.""" + llm = MagicMock() + llm.tokenizer = MagicMock() + response_iter = iter(responses) + + def generate_async(prompt, sampling_params=None, streaming=False): + text = next(response_iter) + output = MagicMock() + output.result.return_value = _text_result(text) + return output + + llm.generate_async = generate_async + return llm + + +def test_e2e_harness_final_score_over_mocked_llm(): + """The real lm-eval evaluator scores canned responses correctly.""" + from lm_eval.evaluator import evaluate + + task = _toy_task() + wrapper = LmEvalWrapper(llm=_canned_llm(_E2E_RESPONSES)) + results = evaluate( + lm=wrapper, + task_dict={"toy_arith": task}, + bootstrap_iters=0, + log_samples=False, + ) + score = results["results"]["toy_arith"]["exact_match,strict-match"] + assert score == pytest.approx(0.75) + + +def test_e2e_partial_scores_match_final_score(): + """Partial-score estimates over the full corpus agree with the harness. + + Uses the REAL task's filters and process_results inside + _RunningScoreTracker (no fakes), so a calling-convention mismatch + between the tracker and lm-eval internals disables the tracker and + fails this test. + """ + from lm_eval.evaluator import evaluate + + task = _toy_task() + task_dict = {"toy_arith": task} + wrapper = LmEvalWrapper( + llm=_canned_llm(_E2E_RESPONSES), + partial_scores_every=2, + partial_scoring_task_dict=task_dict, + ) + with patch("tensorrt_llm.evaluate.lm_eval.logger") as mock_logger: + results = evaluate( + lm=wrapper, + task_dict=task_dict, + bootstrap_iters=0, + log_samples=False, + ) + messages = [call.args[0] for call in mock_logger.info.call_args_list] + assert not any("Partial scoring disabled" in m for m in messages), ( + "tracker was disabled by a scoring failure against the real task" + ) + partial = [m for m in messages if "Partial scores" in m] + # interval=2 over 4 responses -> logs at 2/4 and 4/4. + assert len(partial) == 2 + assert "2/4" in partial[0] + assert "4/4" in partial[1] + # The final running estimate agrees with the true score (0~100 scale). + assert "75.00" in partial[1] + score = results["results"]["toy_arith"]["exact_match,strict-match"] + assert score == pytest.approx(0.75) + + +def test_e2e_windowed_matches_final_score(monkeypatch): + """The windowed path produces the same harness score as submit-all. + + Windowing must be a pure scheduling change — outputs are collected in + submission order, so the score is identical to the default path. + """ + from lm_eval.evaluator import evaluate + + monkeypatch.setenv(MAX_IN_FLIGHT_ENV_VAR, "2") + task = _toy_task() + wrapper = LmEvalWrapper(llm=_canned_llm(_E2E_RESPONSES)) + assert wrapper.max_in_flight == 2 + results = evaluate( + lm=wrapper, + task_dict={"toy_arith": task}, + bootstrap_iters=0, + log_samples=False, + ) + score = results["results"]["toy_arith"]["exact_match,strict-match"] + assert score == pytest.approx(0.75) diff --git a/tests/unittest/others/test_quantize_calib_dataset.py b/tests/unittest/others/test_quantize_calib_dataset.py deleted file mode 100644 index 7a2e0549baf4..000000000000 --- a/tests/unittest/others/test_quantize_calib_dataset.py +++ /dev/null @@ -1,83 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import tempfile - -import torch -from utils.util import skip_no_modelopt - - -@skip_no_modelopt -def test_is_cnn_dailymail_local_repo(): - from tensorrt_llm.quantization.quantize_by_modelopt import _is_cnn_dailymail_local_repo - - # Non-existent path - assert not _is_cnn_dailymail_local_repo("/does/not/exist") - - # Empty directory - with tempfile.TemporaryDirectory() as d: - assert not _is_cnn_dailymail_local_repo(d) - - # 3.0.0 subdir - with tempfile.TemporaryDirectory() as d: - os.makedirs(os.path.join(d, "3.0.0")) - assert _is_cnn_dailymail_local_repo(d) - - # Other versions not detected - for version in ("1.0.0", "2.0.0"): - with tempfile.TemporaryDirectory() as d: - os.makedirs(os.path.join(d, version)) - assert not _is_cnn_dailymail_local_repo(d), ( - f"version subdir {version} should not be detected" - ) - - # Directory with the cnn_dailymail.py builder script - with tempfile.TemporaryDirectory() as d: - open(os.path.join(d, "cnn_dailymail.py"), "w").close() - assert _is_cnn_dailymail_local_repo(d) - - # Directory with unrelated content - with tempfile.TemporaryDirectory() as d: - os.makedirs(os.path.join(d, "train")) - open(os.path.join(d, "data.parquet"), "w").close() - assert not _is_cnn_dailymail_local_repo(d) - - -@skip_no_modelopt -def test_get_calib_dataloader_local_cnn_dailymail(monkeypatch): - from tensorrt_llm.quantization import quantize_by_modelopt - - captured = {} - - def fake_load_dataset(path, **kwargs): - captured["kwargs"] = kwargs - return {"article": ["calibration article"] * 2} - - def fake_tokenizer(dataset, **kwargs): - return {"input_ids": torch.ones(len(dataset), 4, dtype=torch.long)} - - monkeypatch.setattr(quantize_by_modelopt, "load_dataset", fake_load_dataset) - - with tempfile.TemporaryDirectory() as d: - # Name lacks "cnn_dailymail". - os.makedirs(os.path.join(d, "3.0.0")) - dataloader = quantize_by_modelopt.get_calib_dataloader( - dataset_name_or_dir=d, - tokenizer=fake_tokenizer, - calib_size=2, - ) - - assert captured["kwargs"].get("name") == "3.0.0" - assert len(list(dataloader)) == 2 diff --git a/tests/unittest/others/test_time_breakdown.py b/tests/unittest/others/test_time_breakdown.py index ae45744cade5..412b95734160 100644 --- a/tests/unittest/others/test_time_breakdown.py +++ b/tests/unittest/others/test_time_breakdown.py @@ -16,6 +16,8 @@ import unittest from unittest.mock import patch +from tensorrt_llm.serve.scripts.benchmark_serving import ( + _read_new_perf_metrics, _snapshot_perf_metrics) from tensorrt_llm.serve.scripts.time_breakdown import (RequestDataParser, RequestTimeBreakdown, TimingMetric, @@ -393,9 +395,50 @@ def test_parse_json_file(self): finally: os.unlink(temp_file) + def test_parse_jsonl_file(self): + """Test parsing server-produced JSONL records.""" + with tempfile.NamedTemporaryFile(mode='w', + suffix='.jsonl', + delete=False) as f: + for record in self.test_data: + f.write(json.dumps(record) + '\n') + temp_file = f.name + + try: + timing_data = self.analyzer.parse_json_file(temp_file) + self.assertEqual(len(timing_data), 2) + self.assertEqual(timing_data[0]['ctx_server_arrival_time'], 1.0) + self.assertEqual(timing_data[1]['ctx_server_arrival_time'], 2.0) + finally: + os.unlink(temp_file) + + def test_read_new_disagg_metrics_for_benchmark(self): + """Test reading only new combined records from a metrics directory.""" + with tempfile.TemporaryDirectory() as output_dir: + disagg_path = os.path.join( + output_dir, "perf_metrics-disagg-host-1-start.jsonl") + with open(disagg_path, "w", encoding="utf-8") as output: + output.write('{"disagg_request_id":0}\n') + offsets = _snapshot_perf_metrics(output_dir) + + with open(disagg_path, "a", encoding="utf-8") as output: + output.write('{"disagg_request_id":1}\n') + for kind in ("context", "generation"): + worker_path = os.path.join( + output_dir, f"perf_metrics-{kind}-host-2-start.jsonl") + with open(worker_path, "w", encoding="utf-8") as output: + output.write('{"request_id":1}\n') + + records = _read_new_perf_metrics(output_dir, + offsets, + expected_count=1, + timeout=0.1) + + self.assertEqual(records, [{"disagg_request_id": 1}]) + def test_parse_json_file_not_found(self): """Test parsing a non-existent file.""" - with self.assertRaises(SystemExit): + with self.assertRaises(FileNotFoundError): self.analyzer.parse_json_file('non_existent_file.json') def test_parse_json_file_invalid_json(self): @@ -406,7 +449,7 @@ def test_parse_json_file_invalid_json(self): temp_file = f.name try: - with self.assertRaises(SystemExit): + with self.assertRaises(ValueError): self.analyzer.parse_json_file(temp_file) finally: os.unlink(temp_file) diff --git a/tests/unittest/pytest.ini b/tests/unittest/pytest.ini index 0f76aadd5886..48c994b91cf9 100644 --- a/tests/unittest/pytest.ini +++ b/tests/unittest/pytest.ini @@ -4,10 +4,12 @@ asyncio_default_fixture_loop_scope = module threadleak = True # ThreadPoolExecutor-\d+_\d+ excludes worker threads leaked by torch._dynamo/torch.compile. # Thread-\d+ \(_manager_spawn\) is the MPIPoolExecutor manager thread of a pool cached -# for reuse by the NEXT test, and session-reuse-* are the reuse layer's own threads -# (tests/test_common/session_reuse.py); they legitimately outlive the test they start under. -threadleak_exclude = asyncio_\d+|rpc_client_loop|rpc_client_worker_\d+|rpc_server_worker_\d+|InductorSubproc|subproc_worker_timer|ThreadPoolExecutor-\d+_\d+|Thread-\d+ \(_manager_spawn\)|session-reuse-\w+ -addopts = --durations=0 -W ignore::DeprecationWarning -p test_common.session_reuse_hooks +# for reuse by the NEXT test (or prefetched in the background for it); +# session-reuse-* are the reuse layer's own threads (tests/test_common/session_reuse.py) +# and session-prefetch-* are the prefetcher's (tests/test_common/session_prefetcher.py); +# they legitimately outlive the test they start under. +threadleak_exclude = asyncio_\d+|rpc_client_loop|rpc_client_worker_\d+|rpc_server_worker_\d+|InductorSubproc|subproc_worker_timer|ThreadPoolExecutor-\d+_\d+|Thread-\d+ \(_manager_spawn\)|session-reuse-\w+|session-prefetch-\w+ +addopts = --durations=0 -W ignore::DeprecationWarning -p test_common.session_reuse_hooks -p test_common.s3_output_hooks pythonpath = auto_deploy/_utils_test ../../examples/auto_deploy diff --git a/tests/unittest/scaffolding/test_scaffolding.py b/tests/unittest/scaffolding/test_scaffolding.py index d6d322936583..fad6c4dbf794 100644 --- a/tests/unittest/scaffolding/test_scaffolding.py +++ b/tests/unittest/scaffolding/test_scaffolding.py @@ -29,7 +29,8 @@ def create_scaffolding_llm_with_majority_vote_controller( trtllm_worker = create_trtllm_worker(deepseek_distill_7b_path) workers = {} - prototype_generation_controller = NativeGenerationController() + prototype_generation_controller = NativeGenerationController( + sampling_params={"max_tokens": 100}) workers[NativeGenerationController.WorkerTag.GENERATION] = trtllm_worker prototype_majority_vote_controller = MajorityVoteController( @@ -48,23 +49,28 @@ def create_scaffolding_llm_with_majority_vote_controller( def test_unbatched_scaffolding_sync(default_prompt, deepseek_distill_7b_path): scaffolding_llm = create_scaffolding_llm_with_native_generation_controller( deepseek_distill_7b_path) - result = scaffolding_llm.generate(default_prompt) - assert isinstance(result.outputs[0].text, str) and len( - result.outputs[0].text) > 0, "Output should be a non-empty string" - scaffolding_llm.shutdown(shutdown_workers=True) + try: + result = scaffolding_llm.generate(default_prompt) + assert isinstance(result.outputs[0].text, str) and len( + result.outputs[0].text) > 0, "Output should be a non-empty string" + finally: + scaffolding_llm.shutdown(shutdown_workers=True) def test_batched_scaffolding_sync(default_prompt, deepseek_distill_7b_path): scaffolding_llm = create_scaffolding_llm_with_native_generation_controller( deepseek_distill_7b_path) - batch_size = 3 - prompts = [default_prompt] * batch_size - results = scaffolding_llm.generate(prompts) - assert len(results) == batch_size - for result in results: - assert isinstance(result.outputs[0].text, str) and len( - result.outputs[0].text) > 0, "Output should be a non-empty string" - scaffolding_llm.shutdown(shutdown_workers=True) + try: + batch_size = 3 + prompts = [default_prompt] * batch_size + results = scaffolding_llm.generate(prompts) + assert len(results) == batch_size + for result in results: + assert isinstance(result.outputs[0].text, str) and len( + result.outputs[0].text + ) > 0, "Output should be a non-empty string" + finally: + scaffolding_llm.shutdown(shutdown_workers=True) def test_async_scaffolding_generation(default_prompt, deepseek_distill_7b_path): @@ -72,11 +78,14 @@ def test_async_scaffolding_generation(default_prompt, deepseek_distill_7b_path): async def run_async_test(): scaffolding_llm = create_scaffolding_llm_with_native_generation_controller( deepseek_distill_7b_path) - future = scaffolding_llm.generate_async(default_prompt) - result = await future.aresult() - assert isinstance(result.outputs[0].text, str) and len( - result.outputs[0].text) > 0, "Output should be a non-empty string" - scaffolding_llm.shutdown(shutdown_workers=True) + try: + future = scaffolding_llm.generate_async(default_prompt) + result = await future.aresult() + assert isinstance(result.outputs[0].text, str) and len( + result.outputs[0].text + ) > 0, "Output should be a non-empty string" + finally: + scaffolding_llm.shutdown(shutdown_workers=True) import asyncio asyncio.run(run_async_test()) @@ -85,7 +94,9 @@ async def run_async_test(): def test_majority_vote(default_prompt, deepseek_distill_7b_path): scaffolding_llm = create_scaffolding_llm_with_majority_vote_controller( deepseek_distill_7b_path, samples_num=3) - result = scaffolding_llm.generate(default_prompt) - assert isinstance(result.outputs[0].text, str) and len( - result.outputs[0].text) > 0, "Output should be a non-empty string" - scaffolding_llm.shutdown(shutdown_workers=True) + try: + result = scaffolding_llm.generate(default_prompt) + assert isinstance(result.outputs[0].text, str) and len( + result.outputs[0].text) > 0, "Output should be a non-empty string" + finally: + scaffolding_llm.shutdown(shutdown_workers=True) diff --git a/tests/unittest/scaffolding/test_worker.py b/tests/unittest/scaffolding/test_worker.py index 6faa353fc5a1..b5aed772232e 100644 --- a/tests/unittest/scaffolding/test_worker.py +++ b/tests/unittest/scaffolding/test_worker.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from pathlib import Path # isort: off @@ -55,7 +70,8 @@ def server(model_name: str, backend: str, num_postprocess_workers: int): args.extend(["--num_postprocess_workers", f"{num_postprocess_workers}"]) args.extend(["--kv_cache_free_gpu_memory_fraction", "0.5"]) remote_server = RemoteOpenAIServer(model_path, args) - return remote_server + yield remote_server + remote_server.terminate() def create_trtoai_worker(model_name, async_client): @@ -68,24 +84,25 @@ def create_trtoai_worker(model_name, async_client): @pytest.mark.asyncio(loop_scope="module") def test_trtoai_worker_generation(default_prompt, model_name, server): worker = create_trtoai_worker(model_name, server.get_async_client()) - task = GenerationTask.create_from_prompt(default_prompt) - status = asyncio.run(worker.run_task(task)) try: + task = GenerationTask.create_from_prompt(default_prompt) + task.max_tokens = 100 + status = asyncio.run(worker.run_task(task)) assert status == TaskStatus.SUCCESS, "Generation Task is not successful with TRTOpenaiWorker" - except AssertionError as e: + finally: worker.shutdown() - server.__exit__(None, None, None) - raise e @pytest.mark.asyncio(loop_scope="module") def test_trtoai_worker_chat(default_prompt, model_name, server): worker = create_trtoai_worker(model_name, server.get_async_client()) - task = ChatTask.create_from_messages([UserMessage(default_prompt)]) - task.max_tokens = 100 - status = asyncio.run(worker.run_task(task)) - assert status == TaskStatus.SUCCESS, "Chat Task is not successful with TRTOpenaiWorker" - worker.shutdown() + try: + task = ChatTask.create_from_messages([UserMessage(default_prompt)]) + task.max_tokens = 100 + status = asyncio.run(worker.run_task(task)) + assert status == TaskStatus.SUCCESS, "Chat Task is not successful with TRTOpenaiWorker" + finally: + worker.shutdown() def create_trtllm_worker(model_path): @@ -94,10 +111,10 @@ def create_trtllm_worker(model_path): def test_trtllm_worker_generation(default_prompt, deepseek_distill_7b_path): worker = create_trtllm_worker(deepseek_distill_7b_path) - task = GenerationTask.create_from_prompt(default_prompt) - status = asyncio.run(worker.run_task(task)) try: + task = GenerationTask.create_from_prompt(default_prompt) + task.max_tokens = 100 + status = asyncio.run(worker.run_task(task)) assert status == TaskStatus.SUCCESS, "Generation Task is not successful with TRTLLMWorker" - except AssertionError as e: + finally: worker.shutdown() - raise e diff --git a/tests/unittest/scripts/test_cluster_env.py b/tests/unittest/scripts/test_cluster_env.py new file mode 100644 index 000000000000..26905068b10f --- /dev/null +++ b/tests/unittest/scripts/test_cluster_env.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +from pathlib import Path +from types import ModuleType + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +CLUSTER_ENV_PATH = REPO_ROOT / "jenkins" / "scripts" / "perf" / "cluster_env.py" + + +@pytest.fixture(scope="module") +def cluster_env_module() -> ModuleType: + """Load cluster_env.py without requiring jenkins to be a Python package.""" + spec = importlib.util.spec_from_file_location("cluster_env", CLUSTER_ENV_PATH) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("stage_name", "expected_gpu"), + ( + ("DGX_B200-8_GPUs-PyTorch-PerfSanity-1", "B200"), + ("DGX_GB200-4_GPUs-PyTorch-PerfSanity-1", "GB200"), + ("dgx_gb300-4_gpus-pytorch-perfsanity-1", "GB300"), + ("unknown-stage", ""), + ("", ""), + ), +) +def test_gpu_type_from_stage_name( + cluster_env_module: ModuleType, stage_name: str, expected_gpu: str +) -> None: + assert cluster_env_module.gpu_type_from_stage_name(stage_name) == expected_gpu + + +@pytest.mark.parametrize( + ("supported_gpus", "expected_gpu"), + ( + (["B200"], "B200"), + (["b200", "gb200"], "GB200"), + (["GB300", "B300"], "GB300"), + (["L40S"], ""), + ([], ""), + ), +) +def test_gpu_type_from_supported_gpus( + cluster_env_module: ModuleType, supported_gpus: list[str], expected_gpu: str +) -> None: + assert cluster_env_module.gpu_type_from_supported_gpus(supported_gpus) == expected_gpu + + +@pytest.mark.parametrize( + ("cluster_name", "expected_export"), + ( + ( + "GCP-NRT-CS-001", + "export UCX_NET_DEVICES=rocep145s0:1,rocep146s0:1,rocep152s0:1," + "rocep153s0:1,rocep198s0:1,rocep199s0:1,rocep205s0:1,rocep206s0:1 " + "UCX_IB_GID_INDEX=auto UCX_IB_TRAFFIC_CLASS=52 UCX_IB_SL=0", + ), + ( + "nsc-svg-slurm-1", + "export UCX_NET_DEVICES=mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1," + "mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1", + ), + ( + "oci-aga-cs-001", + "export UCX_TLS=^tcp,rc_gda,gga UCX_IB_MLX5_DEVX=n " + "UCX_NET_DEVICES=" + "rdma_vf_rail0:1,rdma_vf_rail1:1,rdma_vf_rail2:1,rdma_vf_rail3:1 " + "UCX_IB_TRAFFIC_CLASS=96 TRTLLM_NIXL_NUM_THREADS=1", + ), + ("aws-cmh", "export UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp"), + ("aws-dfw-prod", "export UCX_TLS=^gdr_copy"), + ), +) +def test_get_ucx_tls_cmd_selects_cluster_rule( + cluster_env_module: ModuleType, cluster_name: str, expected_export: str +) -> None: + command = cluster_env_module.get_ucx_tls_cmd(cluster_name, "B200") + + assert command == f"{cluster_env_module.BASE_UCX_UNSET} && {expected_export} &&" + + +@pytest.mark.parametrize("cluster_name", ("", "unknown-cluster")) +def test_get_ucx_tls_cmd_uses_base_unset_for_unknown_cluster( + cluster_env_module: ModuleType, cluster_name: str +) -> None: + command = cluster_env_module.get_ucx_tls_cmd(cluster_name, "GB300") + + assert command == f"{cluster_env_module.BASE_UCX_UNSET} &&" diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py new file mode 100644 index 000000000000..55c8ead43706 --- /dev/null +++ b/tests/unittest/scripts/test_perf_submit.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +from pathlib import Path +from types import ModuleType + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +SUBMIT_PATHS = ( + REPO_ROOT / "jenkins" / "scripts" / "perf" / "submit.py", + REPO_ROOT / "jenkins" / "scripts" / "perf" / "local" / "submit.py", +) +DISAGG_CONFIG_DIR = REPO_ROOT / "tests" / "scripts" / "perf-sanity" / "disaggregated" + + +@pytest.fixture(params=SUBMIT_PATHS, ids=("ci", "local")) +def submit_module(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> ModuleType: + monkeypatch.syspath_prepend(str(request.param.parent)) + spec = importlib.util.spec_from_file_location( + f"perf_submit_{request.param.parent.name}", request.param + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("config_name", "expected_queue_size"), + ( + ("gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL", 1), + ( + "gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL", + 128, + ), + ( + "gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL", + 512, + ), + ( + "gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL", + 4096, + ), + ), +) +def test_gen_only_queue_size_does_not_exceed_executor_capacity( + submit_module, config_name, expected_queue_size +): + with open(DISAGG_CONFIG_DIR / f"{config_name}.yaml") as config_file: + config = yaml.safe_load(config_file) + + concurrency = config["benchmark"]["concurrency_list"] + + assert ( + submit_module.get_benchmark_request_queue_size(config, concurrency) == expected_queue_size + ) + + +def test_gen_only_queue_size_preserves_reachable_concurrency(submit_module): + config = { + "worker_config": { + "gen": { + "max_batch_size": 8, + "tensor_parallel_size": 4, + "enable_attention_dp": True, + } + } + } + + assert submit_module.get_benchmark_request_queue_size(config, 16) == 16 diff --git a/tests/unittest/test_s3_output.py b/tests/unittest/test_s3_output.py index f7593af7cdf9..899ad5a8fe27 100644 --- a/tests/unittest/test_s3_output.py +++ b/tests/unittest/test_s3_output.py @@ -13,26 +13,57 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io +import logging +import os import subprocess import sys +from types import SimpleNamespace -from test_common.s3_output import FDRedirector, UploadLogPlugin +import pytest +from test_common import s3_output_hooks +from test_common.s3_output import ( + FDRedirector, + FileSlice, + FileSliceReader, + SessionCapture, + UploadLogPlugin, +) class Report: - def __init__(self): + def __init__(self, when=None, outcome=None): self.sections = [] + self.when = when + self.outcome = outcome + + +class CaptureContext: + def __init__(self): + self.closed = False + + def __exit__(self, exc_type, exc_val, exc_tb): + self.closed = True class RecordingS3Client: def __init__(self): self.uploads = [] + self.fileobj_uploads = [] def upload_file(self, filepath, bucket, object_key, ExtraArgs=None): self.uploads.append((filepath, bucket, object_key, ExtraArgs)) + def upload_fileobj(self, fileobj, bucket, object_key, ExtraArgs=None): + self.fileobj_uploads.append((fileobj.read(), bucket, object_key, ExtraArgs)) + -def make_plugin(tmp_path, inline_output_max_bytes): +def make_plugin( + tmp_path, + inline_output_max_bytes, + capture_mode="timestamped", + session_capture=None, +): return UploadLogPlugin( endpoint_url="https://example.com", aws_access_key_id="user", @@ -41,7 +72,9 @@ def make_plugin(tmp_path, inline_output_max_bytes): upload_path="logs", output_path=str(tmp_path), skip_upload=True, + capture_mode=capture_mode, inline_output_max_bytes=inline_output_max_bytes, + session_capture=session_capture, ) @@ -51,6 +84,85 @@ def write_log(tmp_path, test_name, filename, content): (test_dir / filename).write_text(content, encoding="utf-8") +def complete_makereport_hook(plugin, nodeid, report): + item = type("Item", (), {"nodeid": nodeid})() + hook = plugin.pytest_runtest_makereport(item, call=None) + next(hook) + with pytest.raises(StopIteration) as stop: + hook.send(report) + assert stop.value.value is report + + +def test_capture_stays_open_after_successful_setup_report(tmp_path): + nodeid = "test_module.py::test_case" + plugin = make_plugin(tmp_path, inline_output_max_bytes=256) + capture = CaptureContext() + plugin._active_capture[nodeid] = {"stdout_redir": capture} + + complete_makereport_hook(plugin, nodeid, Report(when="setup", outcome="passed")) + + assert not capture.closed + assert nodeid in plugin._active_capture + plugin._close_capture(plugin._active_capture.pop(nodeid)) + + +@pytest.mark.parametrize( + ("when", "outcome"), + [ + ("call", "passed"), + ("call", "failed"), + ("setup", "failed"), + ("setup", "skipped"), + ], +) +def test_capture_closes_before_terminal_report(tmp_path, when, outcome): + nodeid = "test_module.py::test_case" + plugin = make_plugin(tmp_path, inline_output_max_bytes=256) + capture = CaptureContext() + plugin._active_capture[nodeid] = {"stdout_redir": capture} + + complete_makereport_hook(plugin, nodeid, Report(when=when, outcome=outcome)) + + assert capture.closed + assert nodeid not in plugin._active_capture + + +def test_teardown_fallback_warns_when_capture_is_still_active(tmp_path, caplog): + nodeid = "test_module.py::test_case" + plugin = make_plugin(tmp_path, inline_output_max_bytes=256) + capture = CaptureContext() + plugin._active_capture[nodeid] = {"stdout_redir": capture} + item = type("Item", (), {"nodeid": nodeid})() + + caplog.set_level(logging.WARNING, logger="test_common.s3_output") + hook = plugin.pytest_runtest_teardown(item, nextitem=None) + next(hook) + + assert capture.closed + assert nodeid not in plugin._active_capture + assert "remained active until teardown" in caplog.text + + with pytest.raises(StopIteration): + next(hook) + + +def test_session_capture_closes_after_teardown(tmp_path): + nodeid = "test_module.py::test_case" + plugin = make_plugin(tmp_path, inline_output_max_bytes=256, capture_mode="session") + capture = CaptureContext() + plugin._active_capture[nodeid] = {"stdout_redir": capture} + item = type("Item", (), {"nodeid": nodeid})() + + hook = plugin.pytest_runtest_teardown(item, nextitem=None) + next(hook) + assert not capture.closed + + with pytest.raises(StopIteration): + next(hook) + assert capture.closed + assert nodeid not in plugin._active_capture + + def test_small_stdout_is_inlined_without_upload(tmp_path): test_name = "test-small" write_log(tmp_path, test_name, "stdout.log", "ok\n") @@ -135,6 +247,199 @@ def test_deferred_stdout_is_removed_after_upload_finishes(tmp_path): assert not (tmp_path / test_name).exists() +def test_file_slice_upload_reads_only_test_range(tmp_path): + test_name = "test-slice" + spool = tmp_path / "stdout-spool.log" + prefix = b"previous test\n" + content = b"parent\nchild\nparent again\n" + spool.write_bytes(prefix + content + b"next test\n") + plugin = make_plugin(tmp_path, inline_output_max_bytes=3, capture_mode="session") + plugin.skip_upload = False + plugin.s3 = RecordingS3Client() + plugin._captured_slices[(test_name, "stdout.log")] = FileSlice( + str(spool), len(prefix), len(content) + ) + report = Report() + + plugin.upload_and_report(report, test_name, "stdout.log", "Captured stdout") + + assert plugin.s3.fileobj_uploads == [ + ( + content, + "bucket", + "logs/test-slice/stdout.log", + {"ContentType": "text/plain"}, + ) + ] + assert "uploaded to" in report.sections[0][1] + + +def test_file_slice_reader_supports_upload_size_probe(tmp_path): + spool = tmp_path / "stdout-spool.log" + spool.write_bytes(b"prefix\ntest output\nsuffix\n") + file_slice = FileSlice(str(spool), len(b"prefix\n"), len(b"test output\n")) + + with io.BufferedReader(FileSliceReader(file_slice)) as source_file: + assert source_file.tell() == 0 + assert source_file.seek(0, os.SEEK_END) == file_slice.size + assert source_file.tell() == file_slice.size + assert source_file.seek(0) == 0 + assert source_file.read() == b"test output\n" + + +def test_session_capture_preserves_parent_and_child_stdout_order(tmp_path): + capture = SessionCapture(str(tmp_path)) + capture.start() + child = None + try: + child = subprocess.Popen( + [ + sys.executable, + "-c", + "import os, sys; sys.stdin.buffer.read(1); " + "os.write(1, b'child stdout\\n'); os.write(2, b'child stderr\\n')", + ], + stdin=subprocess.PIPE, + ) + offsets = capture.snapshot() + os.write(1, b"parent before\n") + child.stdin.write(b"x") + child.stdin.close() + assert child.wait(timeout=10) == 0 + os.write(1, b"parent after\n") + slices = capture.slices_since(offsets) + finally: + if child is not None and child.poll() is None: + child.kill() + child.wait() + capture.stop() + + with io.BufferedReader(FileSliceReader(slices["stdout.log"])) as stdout_file: + assert stdout_file.read() == b"parent before\nchild stdout\nparent after\n" + with io.BufferedReader(FileSliceReader(slices["stderr.log"])) as stderr_file: + assert stderr_file.read() == b"child stderr\n" + capture.remove_files() + + +class _EarlyConfig: + def __init__(self, capture="no", numprocesses=None): + self.option = SimpleNamespace(numprocesses=numprocesses) + self.known_args_namespace = SimpleNamespace( + capture=capture, + numprocesses=numprocesses, + ) + self.cleanups = [] + + def add_cleanup(self, cleanup): + self.cleanups.append(cleanup) + + +def test_early_capture_preserves_conftest_child_output_for_case(tmp_path, monkeypatch): + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + early_config = _EarlyConfig() + initial_hook = s3_output_hooks.pytest_load_initial_conftests( + early_config, + [ + "-s", + "--s3-upload-path=logs", + f"--output-dir={tmp_path}", + ], + ) + child = None + capture = None + try: + next(initial_hook) + state = s3_output_hooks._capture_state(early_config) + assert state is not None + capture = state.capture + assert capture is not None + child = subprocess.Popen( + [ + sys.executable, + "-c", + "import os, sys; sys.stdin.buffer.read(1); " + "os.write(1, b'child stdout\\n'); os.write(2, b'child stderr\\n')", + ], + stdin=subprocess.PIPE, + ) + + with pytest.raises(StopIteration): + next(initial_hook) + + plugin = make_plugin( + tmp_path, + inline_output_max_bytes=0, + capture_mode="session", + session_capture=capture, + ) + session = SimpleNamespace(config=SimpleNamespace()) + session_hook = plugin.pytest_sessionstart(session) + next(session_hook) + with pytest.raises(StopIteration): + next(session_hook) + + offsets = capture.snapshot() + os.write(1, b"parent before\n") + child.stdin.write(b"x") + child.stdin.close() + assert child.wait(timeout=10) == 0 + os.write(1, b"parent after\n") + slices = capture.slices_since(offsets) + capture.stop() + + with io.BufferedReader(FileSliceReader(slices["stdout.log"])) as stdout_file: + assert stdout_file.read() == b"parent before\nchild stdout\nparent after\n" + with io.BufferedReader(FileSliceReader(slices["stderr.log"])) as stderr_file: + assert stderr_file.read() == b"child stderr\n" + finally: + if child is not None and child.poll() is None: + child.kill() + child.wait() + if capture is not None: + capture.stop() + for cleanup in early_config.cleanups: + cleanup() + + +def test_early_capture_skips_xdist_controller(tmp_path, monkeypatch): + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + early_config = _EarlyConfig(numprocesses=2) + initial_hook = s3_output_hooks.pytest_load_initial_conftests( + early_config, + [ + "-s", + "-n", + "2", + "--s3-upload-path=logs", + f"--output-dir={tmp_path}", + ], + ) + + next(initial_hook) + assert s3_output_hooks._capture_state(early_config) is None + with pytest.raises(StopIteration): + next(initial_hook) + + +def test_s3_plugin_registration_runs_on_xdist_worker_only(monkeypatch): + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + registered_configs = [] + + def register_plugin(config, session_capture=None): + registered_configs.append(config) + return object() + + monkeypatch.setattr(s3_output_hooks.s3_output, "register_plugin", register_plugin) + controller_config = _EarlyConfig(numprocesses=2) + s3_output_hooks.pytest_configure(controller_config) + assert registered_configs == [] + + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw0") + worker_config = _EarlyConfig(numprocesses=2) + s3_output_hooks.pytest_configure(worker_config) + assert registered_configs == [worker_config] + + def test_small_log_file_is_not_inlined(tmp_path): test_name = "test-log" write_log(tmp_path, test_name, "logging.log", "ok\n") diff --git a/tests/unittest/tools/test_config_database_sync.py b/tests/unittest/tools/test_config_database_sync.py index 83e5670f6783..2a20d2707052 100644 --- a/tests/unittest/tools/test_config_database_sync.py +++ b/tests/unittest/tools/test_config_database_sync.py @@ -22,6 +22,7 @@ from pathlib import Path import pytest +import yaml REPO_ROOT = Path(__file__).parent.parent.parent.parent.resolve() EXPECTED_MODEL_METADATA = { @@ -58,6 +59,7 @@ sys.modules[_db_spec.name] = _db_module _db_spec.loader.exec_module(_db_module) generate_tests = _db_module.generate_tests +generate_server_name = _db_module.generate_server_name TEST_LIST_PATH = _db_module.TEST_LIST_PATH PERF_SANITY_DIR = _db_module.PERF_SANITY_DIR @@ -85,6 +87,9 @@ def test_config_db_json_generation(self): int(r.osl), int(r.concurrency), r.config_path, + r.profile, + r.validated_trtllm_commit, + r.validated_trtllm_version, ) for r in recipes } @@ -126,6 +131,9 @@ def test_config_db_json_generation(self): int(e.get("osl")), int(e.get("concurrency")), e.get("config_path"), + e.get("profile"), + e.get("validated_trtllm_commit"), + e.get("validated_trtllm_version"), ) self.assertIn( key, @@ -152,6 +160,104 @@ def test_config_db_json_generation(self): "Generated config_db.json is missing entries from lookup.yaml.", ) + def test_profile_and_validated_commit_metadata(self) -> None: + commit = "93CB6518B6D6DBD6095748189E626DB731F44545" + version = "1.3.0rc14" + recipes = [] + for profile in ("latency", "balanced", "throughput"): + recipes.append( + { + "model": "example/model", + "arch": "ExampleForCausalLM", + "gpu": "B200_NVL", + "num_gpus": 8, + "isl": 1024, + "osl": 1024, + "concurrency": 256, + "config_path": f"examples/configs/database/example_{profile}.yaml", + "profile": profile, + "validated_trtllm_commit": commit, + "validated_trtllm_version": version, + } + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + yaml_path = Path(tmp_dir) / "lookup.yaml" + output_path = Path(tmp_dir) / "config_db.json" + yaml_path.write_text(yaml.safe_dump(recipes), encoding="utf-8") + generate_json(yaml_path, output_path) + payload = json.loads(output_path.read_text(encoding="utf-8")) + + self.assertEqual( + [entry["profile"] for entry in payload["entries"]], + ["latency", "balanced", "throughput"], + ) + self.assertEqual( + {entry["performance_profile"] for entry in payload["entries"]}, + {"Min Latency", "Balanced", "Max Throughput"}, + ) + self.assertTrue( + all(entry["validated_trtllm_commit"] == commit.lower() for entry in payload["entries"]) + ) + self.assertTrue( + all(entry["validated_trtllm_version"] == version for entry in payload["entries"]) + ) + + parsed = RecipeList.model_validate(recipes) + server_names = [generate_server_name(recipe) for recipe in parsed] + self.assertEqual(len(server_names), len(set(server_names))) + self.assertTrue(server_names[0].endswith("_latency")) + + def test_recipe_metadata_validation(self) -> None: + base = { + "model": "example/model", + "arch": "ExampleForCausalLM", + "gpu": "B200_NVL", + "num_gpus": 8, + "isl": 1024, + "osl": 1024, + "concurrency": 256, + "config_path": "examples/configs/database/example.yaml", + } + + with self.assertRaisesRegex(ValueError, "full 40-character Git SHA"): + RecipeList.model_validate( + [ + { + **base, + "validated_trtllm_commit": "deadbeef", + "validated_trtllm_version": "1.3.0rc14", + } + ] + ) + + with self.assertRaisesRegex(ValueError, "must be provided together"): + RecipeList.model_validate([{**base, "validated_trtllm_commit": "a" * 40}]) + + with self.assertRaisesRegex(ValueError, "profile is only allowed"): + RecipeList.model_validate([{**base, "profile": "balanced"}]) + + endpoint_conflict = [ + {**base, "config_path": "latency.yaml", "profile": "latency"}, + {**base, "config_path": "throughput.yaml", "profile": "throughput"}, + ] + parsed = RecipeList.model_validate(endpoint_conflict) + self.assertEqual([recipe.profile for recipe in parsed], ["latency", "throughput"]) + + missing_endpoint = [ + {**base, "config_path": "latency.yaml", "profile": "latency"}, + {**base, "config_path": "balanced.yaml", "profile": "balanced"}, + ] + with self.assertRaisesRegex(ValueError, "exactly one latency and throughput"): + RecipeList.model_validate(missing_endpoint) + + duplicate_profile = [ + {**base, "config_path": "latency-a.yaml", "profile": "latency"}, + {**base, "config_path": "latency-b.yaml", "profile": "latency"}, + ] + with self.assertRaisesRegex(ValueError, "exactly one latency and throughput"): + RecipeList.model_validate(duplicate_profile) + @pytest.mark.skip(reason="https://nvbugs/6337224") def test_config_database_tests_sync(self): """Test that config database test files are synchronized with lookup.yaml. diff --git a/tests/unittest/tools/test_config_selector.js b/tests/unittest/tools/test_config_selector.js index d5f57bb7cb6b..ce489982cc69 100644 --- a/tests/unittest/tools/test_config_selector.js +++ b/tests/unittest/tools/test_config_selector.js @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + const assert = require("node:assert/strict"); const fs = require("node:fs"); const path = require("node:path"); @@ -77,6 +80,50 @@ test("selector exports a pure view-model helper for compatibility logic", () => assert.equal(typeof selector.createSelectorViewModel, "function"); }); +test("selector builds a validated TensorRT-LLM commit link", () => { + const selector = loadSelectorExports(); + const commit = "93cb6518b6d6dbd6095748189e626db731f44545"; + assert.equal( + selector.validatedCommitUrl(commit), + `https://github.com/NVIDIA/TensorRT-LLM/commit/${commit}` + ); + assert.equal(selector.validatedCommitUrl("not-a-commit"), ""); +}); + +test("selector resolves same-concurrency conflicts by explicit profile", () => { + const selector = loadSelectorExports(); + const entries = ["latency", "balanced", "throughput"].map((profile) => ({ + model: "example/model", + gpu: "B200_NVL", + gpu_display: "8xB200_NVL", + num_gpus: 8, + isl: 1024, + osl: 1024, + concurrency: 256, + profile, + performance_profile: profile, + config_path: `${profile}.yaml`, + command: profile, + })); + const baseState = { + model: "example/model", + topology: "8|B200_NVL", + islOsl: "1024|1024", + concurrency: "256", + }; + + const unresolved = selector.createSelectorViewModel(entries, {}, baseState); + assert.equal(unresolved.finalEntries.length, 3); + assert.match(unresolved.message, /Available Profile options/); + + const resolved = selector.createSelectorViewModel(entries, {}, { + ...baseState, + profile: "balanced", + }); + assert.equal(resolved.finalEntries.length, 1); + assert.equal(resolved.resolvedEntry.profile, "balanced"); +}); + test("selector keeps both 4x and 8x B200 topologies visible for DeepSeek NVFP4", () => { const { selector, payload, entries } = setupDeepSeekNvfp4(); @@ -91,7 +138,7 @@ test("selector keeps both 4x and 8x B200 topologies visible for DeepSeek NVFP4", assert.ok(findOption(view.groups.topology, "8|B200_NVL")); }); -test("selector marks 1024/1024 unavailable for 4x B200 on DeepSeek NVFP4", () => { +test("selector includes automated 1024/1024 configs for 4x B200", () => { const { selector, payload, entries } = setupDeepSeekNvfp4(); const view = selector.createSelectorViewModel(entries, payload.models, { @@ -101,16 +148,31 @@ test("selector marks 1024/1024 unavailable for 4x B200 on DeepSeek NVFP4", () => concurrency: "", }); - assert.equal(findOption(view.groups.islOsl, "1024|1024").status, "incompatible"); + assert.equal(findOption(view.groups.islOsl, "1024|1024").status, "available"); assert.equal(findOption(view.groups.islOsl, "1024|8192").status, "available"); assert.equal(findOption(view.groups.islOsl, "8192|1024").status, "available"); }); test("selector preserves invalid active selections and explains the clash", () => { - const { selector, payload, entries } = setupDeepSeekNvfp4(); - - const view = selector.createSelectorViewModel(entries, payload.models, { - model: DEEPSEEK_NVFP4_MODEL, + const selector = loadSelectorExports(); + const entries = [ + [4, 1024, 8192], + [4, 8192, 1024], + [8, 1024, 1024], + ].map(([numGpus, isl, osl]) => ({ + model: "example/model", + gpu: "B200_NVL", + gpu_display: `${numGpus}xB200_NVL`, + num_gpus: numGpus, + isl, + osl, + concurrency: 1, + config_path: `${numGpus}-${isl}-${osl}.yaml`, + command: `${numGpus}-${isl}-${osl}`, + })); + + const view = selector.createSelectorViewModel(entries, {}, { + model: "example/model", topology: "4|B200_NVL", islOsl: "1024|1024", concurrency: "", diff --git a/tests/unittest/tools/test_prepare_dataset.py b/tests/unittest/tools/test_prepare_dataset.py index 948cde1e0928..e4acfe83032c 100644 --- a/tests/unittest/tools/test_prepare_dataset.py +++ b/tests/unittest/tools/test_prepare_dataset.py @@ -16,15 +16,14 @@ _DEFAULT_OUTPUT_STDEV = 10 _TEST_TASK_IDS = [0, 1, 2] _TOKENIZER_SUBPATH = "llama-models-v2/tinyllama-tarot-v1/" -_PREPARE_DATASET_SCRIPT_PATH = "benchmarks/cpp/prepare_dataset.py" class TestPrepareDatasetLora: """ - Test suite for prepare_dataset.py CLI tool LoRA metadata generation + Test suite for the trtllm-bench prepare-dataset LoRA metadata generation functionality. - This test class validates that the prepare_dataset.py script correctly + This test class validates that trtllm-bench prepare-dataset correctly generates LoRA request metadata when LoRA-specific parameters are provided. It covers both fixed task ID and random task ID scenarios. """ @@ -50,7 +49,7 @@ def temp_lora_dir(self) -> str: def _build_base_command(self, output_path: Path) -> List[str]: """ - Build the base command for running prepare_dataset.py. + Build the base command for running trtllm-bench prepare-dataset. Args: output_path: Path to the output dataset file @@ -71,7 +70,8 @@ def _build_base_command(self, output_path: Path) -> List[str]: tokenizer_dir = model_cache / _TOKENIZER_SUBPATH cmd.extend(["--model", str(tokenizer_dir)]) - # Always add --stdout flag since we parse stdout output + # Write to a file via --output rather than --stdout: trtllm-bench prints + # an import-time banner on stdout that would corrupt the parsed output. cmd.extend(["prepare-dataset", "--output", f"{output_path}"]) return cmd @@ -110,7 +110,7 @@ def _add_synthetic_data_arguments(self, cmd: List[str]) -> None: def _run_prepare_dataset(self, **kwargs) -> str: """ - Execute prepare_dataset.py with specified parameters and capture + Execute trtllm-bench prepare-dataset with specified parameters and capture output. Args: @@ -140,7 +140,7 @@ def _run_prepare_dataset(self, **kwargs) -> str: def _parse_json_output(self, output: str) -> List[Dict[str, Any]]: """ - Parse JSON lines from prepare_dataset.py output. + Parse JSON lines from the prepare-dataset output. Args: output: Raw stdout output containing JSON lines diff --git a/tests/unittest/tools/test_test_to_stage_mapping.py b/tests/unittest/tools/test_test_to_stage_mapping.py index 65b27e755b44..f8689ff12398 100644 --- a/tests/unittest/tools/test_test_to_stage_mapping.py +++ b/tests/unittest/tools/test_test_to_stage_mapping.py @@ -97,6 +97,17 @@ def test_s3_stdout_echo_requires_explicit_opt_in(): context = lines[max(0, idx - 3):idx] assert any('if (ENABLE_S3_ECHO_STDOUT)' in line for line in context) + progress_lines = [ + idx for idx, line in enumerate(lines) + if 'console_output_style=progress-even-when-capture-no' in line + ] + assert progress_lines, 'Expected upload-only pytest progress configuration' + + for idx in progress_lines: + context = lines[max(0, idx - 3):idx] + assert any('if (ENABLE_UPLOAD_TEST_RESULTS)' in line + for line in context) + @pytest.mark.skip(reason="https://nvbugs/5547275") @pytest.mark.parametrize("direction", @@ -182,17 +193,22 @@ def test_search_functionality(stage_query, sample_test_cases): @pytest.mark.parametrize('file_format', ['txt', 'yml']) -def test_cli_functionality(tmp_path, sample_test_cases, file_format): +def test_cli_functionality(tmp_path, stage_query, sample_test_cases, + file_format): """Test CLI functionality with sample data.""" - if not sample_test_cases: - pytest.skip("No test cases available") + # Use the first sample that maps to at least one stage (some test-db + # files, e.g. multi-node perf-sanity lists, have no L0 stage). + test_case = next( + (t for t in sample_test_cases if stage_query.tests_to_stages([t])), + None) + if test_case is None: + pytest.skip("No sampled test maps to any stage") - # Use only first sample for CLI test test_file = tmp_path / f'sample_tests.{file_format}' if file_format == 'txt': - test_file.write_text(f'{sample_test_cases[0]}\n') + test_file.write_text(f'{test_case}\n') else: # yml - test_file.write_text(f'- {sample_test_cases[0]}\n') + test_file.write_text(f'- {test_case}\n') script = os.path.join(SCRIPTS_DIR, 'test_to_stage_mapping.py') cmd = [sys.executable, script, '--test-list', str(test_file)] @@ -200,7 +216,7 @@ def test_cli_functionality(tmp_path, sample_test_cases, file_format): lines = output.decode().strip().splitlines() # Should return at least one stage - assert lines, f"No stages returned for test '{sample_test_cases[0]}'" + assert lines, f"No stages returned for test '{test_case}'" def test_backend_filtering_consistency(stage_query): @@ -246,8 +262,15 @@ def test_backend_filtering_consistency(stage_query): f"at least one stage containing '{backend.upper()}', " \ f"but got stages: {stages}" - # Check that test does NOT map to stages of other backends - other_backends = all_backends - {backend} + # Check that test does NOT map to stages of backends it is not + # declared under (tests may legitimately be listed under several + # backends across test-db files). + declared_backends = { + b.strip() + for _, _, b in stage_query.test_map[test_name] + if b and b.strip() + } + other_backends = all_backends - declared_backends for stage in stages: stage_upper = stage.upper() for other_backend in other_backends: diff --git a/tests/unittest/utils/cpp_paths.py b/tests/unittest/utils/cpp_paths.py index b147b5271138..feac0f698cab 100644 --- a/tests/unittest/utils/cpp_paths.py +++ b/tests/unittest/utils/cpp_paths.py @@ -4,9 +4,6 @@ import pytest -import tensorrt_llm.bindings as _tb -from tensorrt_llm.bindings.internal.testing import ModelSpec - _sys.path.append(_os.path.join(_os.path.dirname(__file__), '..', '..', '..')) @@ -15,70 +12,3 @@ def llm_root() -> _pl.Path: environ_root = _os.environ.get("LLM_ROOT", None) return _pl.Path(environ_root) if environ_root is not None else _pl.Path( __file__).resolve().parent.parent.parent.parent - - -@pytest.fixture(scope="module") -def resource_path(llm_root: _pl.Path) -> _pl.Path: - return llm_root / "cpp" / "tests" / "resources" - - -@pytest.fixture(scope="module") -def data_path(resource_path: _pl.Path) -> _pl.Path: - return resource_path / "data" - - -@pytest.fixture(scope="module") -def input_data_path(data_path): - return data_path / "input_tokens.npy" - - -@pytest.fixture(scope="module") -def engine_path(resource_path: _pl.Path) -> _pl.Path: - return resource_path / "models" / "rt_engine" - - -def get_base_model_spec() -> ModelSpec: - model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) - model_spec_obj.use_gpt_plugin().set_kv_cache_type( - _tb.KVCacheType.PAGED).use_packed_input() - return model_spec_obj - - -@pytest.fixture(scope="module") -def model_path(engine_path): - return engine_path / f"gpt2/{get_base_model_spec().get_model_path()}/tp1-pp1-cp1-gpu" - - -@pytest.fixture(scope="module") -def model_path_return_logits(engine_path): - return engine_path / f"gpt2/{get_base_model_spec().gather_logits().get_model_path()}/tp1-pp1-cp1-gpu" - - -@pytest.fixture -def model_path_lora(engine_path: _pl.Path) -> _pl.Path: - return engine_path / f"gpt2/{get_base_model_spec().use_lora_plugin().get_model_path()}/tp1-pp1-cp1-gpu" - - -@pytest.fixture -def model_path_draft_tokens_external(engine_path: _pl.Path) -> _pl.Path: - return engine_path / f"gpt2/{get_base_model_spec().use_draft_tokens_external_decoding().get_model_path()}/tp1-pp1-cp1-gpu" - - -@pytest.fixture -def lora_config_path(data_path: _pl.Path) -> _pl.Path: - return data_path / "lora-test-weights-gpt2-tp1" - - -@pytest.fixture(scope="module") -def results_data_path(data_path: _pl.Path) -> _pl.Path: - return data_path / f"gpt2/sampling/{get_base_model_spec().get_results_file()}" - - -@pytest.fixture(scope="module") -def results_data_path_beam_width_2(data_path: _pl.Path) -> _pl.Path: - return data_path / f"gpt2/beam_search_2/{get_base_model_spec().get_results_file()}" - - -@pytest.fixture(scope="module") -def results_data_path_fmhafp32acc(data_path: _pl.Path) -> _pl.Path: - return data_path / f"gpt2/sampling/{get_base_model_spec().enable_context_fmha_fp32_acc().get_results_file()}"